From 4c9ac71667b2369e22d345ce360ca75c637dd41d Mon Sep 17 00:00:00 2001 From: SigureMo Date: Sat, 22 Oct 2022 20:39:42 +0800 Subject: [PATCH 1/4] update config --- .pre-commit-config.yaml | 14 ++++++++++---- .style.yapf | 3 --- pyproject.toml | 4 ++++ 3 files changed, 14 insertions(+), 7 deletions(-) delete mode 100644 .style.yapf create mode 100644 pyproject.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fda8a37c2ec00..b58e1b0a4380d 100755 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,11 +28,17 @@ repos: python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs1.py| python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py )$ -- repo: https://github.com/google/yapf - rev: v0.32.0 +- repo: https://github.com/psf/black.git + rev: 22.8.0 hooks: - - id: yapf - files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$ + - id: black + files: (.*\.(py|pyi|bzl)|BUILD|.*\.BUILD|WORKSPACE)$ + # Temporary exclude, will be formatted in a separate PR + exclude: | + (?x)^( + python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py| + python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py + )$ - repo: https://github.com/PyCQA/flake8 rev: 4.0.1 hooks: diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index 4741fb4f3bbc6..0000000000000 --- a/.style.yapf +++ /dev/null @@ -1,3 +0,0 @@ -[style] -based_on_style = pep8 -column_limit = 80 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000..e9513d0648ba1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,4 @@ +[tool.black] +exclude = "build" +line-length = 80 +skip-string-normalization = true From da4687063db6130ac933f3d0df52d35fc0c4d684 Mon Sep 17 00:00:00 2001 From: SigureMo Date: Sat, 22 Oct 2022 20:57:46 +0800 Subject: [PATCH 2/4] re-blacken python code --- .cmake-format.py | 2 +- cmake/copyfile.py | 4 +- cmake/make_resource.py | 16 +- .../generate_file_structures.py | 57 +- .../generator/codegen_utils.py | 195 +- .../generator/eager_gen.py | 1267 ++-- .../generator/python_c_gen.py | 293 +- .../api/full_ILSVRC2012_val_preprocess.py | 95 +- .../api/full_pascalvoc_test_preprocess.py | 152 +- .../api/test_detection_dataset_preprocess.py | 4 +- .../fluid/pybind/generate_file_structures.py | 3 +- paddle/infrt/tests/models/abs_model.py | 6 +- .../tests/models/efficientnet-b4/model.py | 5 +- .../efficientnet-b4/net/efficientnet.py | 206 +- .../tests/models/efficientnet-b4/net/utils.py | 292 +- paddle/infrt/tests/models/linear.py | 12 +- paddle/infrt/tests/models/resnet50_model.py | 5 +- paddle/phi/api/yaml/generator/api_base.py | 651 +- paddle/phi/api/yaml/generator/api_gen.py | 210 +- .../api/yaml/generator/backward_api_gen.py | 201 +- .../phi/api/yaml/generator/cross_validate.py | 27 +- paddle/phi/api/yaml/generator/filters.py | 27 +- paddle/phi/api/yaml/generator/generate_op.py | 189 +- .../api/yaml/generator/generate_sparse_op.py | 80 +- .../yaml/generator/intermediate_api_gen.py | 79 +- .../api/yaml/generator/ops_extra_info_gen.py | 49 +- paddle/phi/api/yaml/generator/parse_api.py | 9 +- paddle/phi/api/yaml/generator/parse_utils.py | 192 +- .../phi/api/yaml/generator/sparse_api_gen.py | 167 +- .../api/yaml/generator/sparse_bw_api_gen.py | 86 +- .../phi/api/yaml/generator/strings_api_gen.py | 172 +- paddle/phi/api/yaml/generator/type_mapping.py | 24 +- .../yaml/generator/wrapped_infermeta_gen.py | 74 +- python/paddle/__init__.py | 12 +- python/paddle/amp/auto_cast.py | 31 +- python/paddle/amp/grad_scaler.py | 39 +- python/paddle/audio/__init__.py | 8 +- python/paddle/audio/backends/backend.py | 26 +- python/paddle/audio/backends/init_backend.py | 9 +- python/paddle/audio/backends/wave_backend.py | 19 +- python/paddle/audio/datasets/dataset.py | 25 +- python/paddle/audio/datasets/esc50.py | 39 +- python/paddle/audio/datasets/tess.py | 49 +- python/paddle/audio/features/layers.py | 277 +- python/paddle/audio/functional/functional.py | 97 +- python/paddle/audio/functional/window.py | 165 +- python/paddle/audio/utils/error.py | 1 + python/paddle/autograd/__init__.py | 1 + python/paddle/autograd/backward_mode.py | 18 +- python/paddle/autograd/py_layer.py | 44 +- python/paddle/autograd/saved_tensors_hooks.py | 7 +- python/paddle/batch.py | 6 +- python/paddle/callbacks.py | 11 +- python/paddle/check_import_scipy.py | 6 +- python/paddle/common_ops_import.py | 22 +- python/paddle/cost_model/cost_model.py | 33 +- python/paddle/dataset/cifar.py | 47 +- python/paddle/dataset/common.py | 39 +- python/paddle/dataset/conll05.py | 47 +- python/paddle/dataset/flowers.py | 84 +- python/paddle/dataset/image.py | 49 +- python/paddle/dataset/imdb.py | 37 +- python/paddle/dataset/imikolov.py | 43 +- python/paddle/dataset/mnist.py | 40 +- python/paddle/dataset/movielens.py | 61 +- python/paddle/dataset/tests/cifar_test.py | 13 +- python/paddle/dataset/tests/flowers_test.py | 10 +- python/paddle/dataset/tests/imikolov_test.py | 21 +- python/paddle/dataset/tests/mnist_test.py | 7 +- python/paddle/dataset/tests/test_image.py | 2 +- python/paddle/dataset/tests/voc2012_test.py | 1 - python/paddle/dataset/tests/wmt16_test.py | 26 +- python/paddle/dataset/uci_housing.py | 54 +- python/paddle/dataset/voc2012.py | 9 +- python/paddle/dataset/wmt14.py | 46 +- python/paddle/dataset/wmt16.py | 161 +- python/paddle/device/__init__.py | 65 +- python/paddle/device/cuda/__init__.py | 32 +- python/paddle/device/cuda/graphs.py | 125 +- python/paddle/distributed/__init__.py | 46 +- .../distributed/auto_parallel/callbacks.py | 79 +- .../distributed/auto_parallel/cluster.py | 114 +- .../distributed/auto_parallel/cluster_v2.py | 29 +- .../distributed/auto_parallel/completion.py | 1166 ++- .../distributed/auto_parallel/converter.py | 208 +- .../auto_parallel/cost/base_cost.py | 240 +- .../auto_parallel/cost/comm_op_cost.py | 102 +- .../auto_parallel/cost/comp_op_cost.py | 400 +- .../auto_parallel/cost/estimate_cost.py | 196 +- .../auto_parallel/cost/tensor_cost.py | 25 +- .../distributed/auto_parallel/cost_model.py | 285 +- .../auto_parallel/dist_attribute.py | 154 +- .../distributed/auto_parallel/dist_context.py | 465 +- .../distributed/auto_parallel/dist_loader.py | 152 +- .../distributed/auto_parallel/dist_op.py | 153 +- .../distributed/auto_parallel/dist_saver.py | 30 +- .../distributed/auto_parallel/dist_tensor.py | 222 +- .../distributed/auto_parallel/engine.py | 675 +- .../paddle/distributed/auto_parallel/graph.py | 6 +- .../distributed/auto_parallel/helper.py | 61 +- .../distributed/auto_parallel/interface.py | 83 +- .../distributed/auto_parallel/mapper.py | 88 +- .../auto_parallel/operators/common.py | 162 +- .../auto_parallel/operators/dist_assign.py | 10 +- .../dist_check_finite_and_unscale.py | 127 +- .../auto_parallel/operators/dist_default.py | 217 +- .../auto_parallel/operators/dist_eltwise.py | 106 +- .../auto_parallel/operators/dist_embedding.py | 480 +- .../dist_fill_constant_batch_size_like.py | 32 +- .../operators/dist_fused_attention.py | 72 +- .../operators/dist_fused_feedforward.py | 74 +- .../auto_parallel/operators/dist_matmul.py | 1648 ++-- .../auto_parallel/operators/dist_pnorm.py | 168 +- .../operators/dist_reduce_sum_p.py | 56 +- .../auto_parallel/operators/dist_reshape.py | 239 +- .../auto_parallel/operators/dist_shape.py | 7 +- .../auto_parallel/operators/dist_slice.py | 26 +- .../auto_parallel/operators/dist_softmax.py | 49 +- .../auto_parallel/operators/dist_split.py | 23 +- .../auto_parallel/operators/dist_transpose.py | 58 +- .../operators/dist_update_loss_scaling.py | 115 +- .../distributed/auto_parallel/parallelizer.py | 306 +- .../auto_parallel/parallelizer_v2.py | 196 +- .../distributed/auto_parallel/partitioner.py | 339 +- .../distributed/auto_parallel/planner.py | 691 +- .../distributed/auto_parallel/planner_v2.py | 9 +- .../auto_parallel/process_group.py | 52 +- .../distributed/auto_parallel/process_mesh.py | 44 +- .../auto_parallel/process_mesh_v2.py | 40 +- .../distributed/auto_parallel/reshard.py | 1563 ++-- .../distributed/auto_parallel/strategy.py | 15 +- .../auto_parallel/tuner/algorithms.py | 5 +- .../distributed/auto_parallel/tuner/config.py | 6 +- .../auto_parallel/tuner/optimization_tuner.py | 239 +- .../auto_parallel/tuner/parallel_tuner.py | 644 +- .../auto_parallel/tuner/profiler.py | 75 +- .../auto_parallel/tuner/recorder.py | 4 +- .../auto_parallel/tuner/storable.py | 1 - .../distributed/auto_parallel/tuner/trial.py | 29 +- .../auto_parallel/tuner/tunable_space.py | 40 +- .../auto_parallel/tuner/tunable_variable.py | 53 +- .../paddle/distributed/auto_parallel/utils.py | 728 +- python/paddle/distributed/cloud_utils.py | 63 +- python/paddle/distributed/collective.py | 659 +- .../distributed/communication/all_reduce.py | 32 +- .../paddle/distributed/communication/group.py | 12 +- .../distributed/communication/reduce.py | 1 + .../communication/stream/__init__.py | 12 +- .../communication/stream/all_gather.py | 50 +- .../communication/stream/all_reduce.py | 53 +- .../communication/stream/alltoall.py | 70 +- .../communication/stream/alltoall_single.py | 52 +- .../communication/stream/broadcast.py | 8 +- .../distributed/communication/stream/recv.py | 6 +- .../communication/stream/reduce.py | 22 +- .../communication/stream/reduce_scatter.py | 119 +- .../communication/stream/scatter.py | 68 +- .../distributed/communication/stream/send.py | 6 +- python/paddle/distributed/elastic.py | 10 +- python/paddle/distributed/entry_attr.py | 6 +- python/paddle/distributed/fleet/__init__.py | 20 +- .../paddle/distributed/fleet/ascend_utils.py | 30 +- .../fleet/base/distributed_strategy.py | 421 +- .../fleet/base/meta_optimizer_factory.py | 4 +- .../fleet/base/orthogonal_strategy.py | 60 +- .../fleet/base/private_helper_function.py | 10 +- .../distributed/fleet/base/role_maker.py | 287 +- .../distributed/fleet/base/runtime_factory.py | 1 - .../fleet/base/strategy_compiler.py | 36 +- .../distributed/fleet/base/strategy_group.py | 52 +- .../paddle/distributed/fleet/base/topology.py | 119 +- .../distributed/fleet/base/util_factory.py | 243 +- .../paddle/distributed/fleet/cloud_utils.py | 48 +- .../fleet/data_generator/data_generator.py | 43 +- .../distributed/fleet/dataset/dataset.py | 124 +- .../fleet/dataset/index_dataset.py | 21 +- .../distributed/fleet/elastic/__init__.py | 7 +- .../distributed/fleet/elastic/collective.py | 13 +- .../distributed/fleet/elastic/manager.py | 162 +- python/paddle/distributed/fleet/fleet.py | 405 +- .../distributed/fleet/fleet_executor_utils.py | 290 +- python/paddle/distributed/fleet/launch.py | 312 +- .../paddle/distributed/fleet/launch_utils.py | 1002 ++- .../distributed/fleet/layers/mpu/mp_layers.py | 265 +- .../distributed/fleet/layers/mpu/mp_ops.py | 668 +- .../distributed/fleet/layers/mpu/random.py | 104 +- .../fleet/meta_optimizers/amp_optimizer.py | 87 +- .../ascend/ascend_optimizer.py | 154 +- .../meta_optimizers/ascend/ascend_parser.py | 2202 +++--- .../fleet/meta_optimizers/asp_optimizer.py | 32 +- .../fleet/meta_optimizers/common.py | 283 +- .../fleet/meta_optimizers/dgc_optimizer.py | 56 +- .../dygraph_sharding_optimizer.py | 56 +- .../heter_parallel_optimizer.py | 21 +- .../hybrid_parallel_gradscaler.py | 18 +- .../hybrid_parallel_optimizer.py | 140 +- .../sharding_optimizer_stage2.py | 186 +- .../fp16_allreduce_optimizer.py | 88 +- .../gradient_merge_optimizer.py | 37 +- .../graph_execution_optimizer.py | 150 +- .../fleet/meta_optimizers/lamb_optimizer.py | 64 +- .../fleet/meta_optimizers/lars_optimizer.py | 62 +- .../meta_optimizers/localsgd_optimizer.py | 478 +- .../meta_optimizers/meta_optimizer_base.py | 81 +- .../parameter_server_graph_optimizer.py | 19 +- .../parameter_server_optimizer.py | 216 +- .../meta_optimizers/pipeline_optimizer.py | 208 +- .../fleet/meta_optimizers/ps_optimizer.py | 122 +- .../meta_optimizers/raw_program_optimizer.py | 339 +- .../meta_optimizers/recompute_optimizer.py | 57 +- .../meta_optimizers/sharding/fp16_helper.py | 217 +- .../sharding/gradient_clip_helper.py | 80 +- .../sharding/offload_helper.py | 222 +- .../fleet/meta_optimizers/sharding/prune.py | 43 +- .../fleet/meta_optimizers/sharding/shard.py | 45 +- .../fleet/meta_optimizers/sharding/utils.py | 820 +- .../sharding/weight_decay_helper.py | 9 +- .../meta_optimizers/sharding_optimizer.py | 1267 ++-- .../tensor_parallel_optimizer.py | 180 +- .../fleet/meta_parallel/meta_parallel_base.py | 6 +- .../parallel_layers/pp_layers.py | 327 +- .../fleet/meta_parallel/pipeline_parallel.py | 293 +- .../pp_utils/p2p_communication.py | 490 +- .../fleet/meta_parallel/pp_utils/utils.py | 22 +- .../group_sharded_optimizer_stage2.py | 213 +- .../sharding/group_sharded_stage2.py | 256 +- .../sharding/group_sharded_stage3.py | 496 +- .../sharding/group_sharded_storage.py | 86 +- .../sharding/group_sharded_utils.py | 112 +- .../meta_parallel/sharding/sharding_stage2.py | 265 +- .../meta_parallel/sharding/sharding_stage3.py | 557 +- .../meta_parallel/sharding/sharding_utils.py | 112 +- .../fleet/meta_parallel/sharding_parallel.py | 1 - .../fleet/meta_parallel/tensor_parallel.py | 6 +- .../distributed/fleet/metrics/metric.py | 9 +- python/paddle/distributed/fleet/model.py | 49 +- python/paddle/distributed/fleet/optimizer.py | 55 +- .../distributed/fleet/recompute/recompute.py | 194 +- .../fleet/recompute/recompute_hybrid.py | 104 +- .../fleet/runtime/collective_runtime.py | 16 +- .../fleet/runtime/parameter_server_runtime.py | 534 +- .../distributed/fleet/runtime/runtime_base.py | 1 - .../distributed/fleet/runtime/the_one_ps.py | 574 +- python/paddle/distributed/fleet/scaler.py | 53 +- .../distributed/fleet/utils/__init__.py | 14 +- python/paddle/distributed/fleet/utils/fs.py | 77 +- .../distributed/fleet/utils/http_server.py | 9 +- .../fleet/utils/hybrid_parallel_inference.py | 296 +- .../fleet/utils/hybrid_parallel_util.py | 114 +- .../fleet/utils/internal_storage.py | 105 +- .../distributed/fleet/utils/log_util.py | 5 +- .../paddle/distributed/fleet/utils/ps_util.py | 132 +- .../distributed/launch/context/__init__.py | 9 +- .../distributed/launch/context/args_envs.py | 193 +- .../distributed/launch/context/device.py | 8 +- .../distributed/launch/context/event.py | 1 - .../paddle/distributed/launch/context/node.py | 10 +- .../distributed/launch/context/resource.py | 1 - .../launch/controllers/collective.py | 59 +- .../launch/controllers/controller.py | 55 +- .../launch/controllers/ipu_controller.py | 109 +- .../distributed/launch/controllers/master.py | 37 +- .../distributed/launch/controllers/ps.py | 106 +- .../distributed/launch/controllers/rpc.py | 24 +- .../distributed/launch/controllers/watcher.py | 6 +- .../distributed/launch/job/container.py | 31 +- python/paddle/distributed/launch/job/job.py | 15 +- python/paddle/distributed/launch/job/pod.py | 27 +- python/paddle/distributed/launch/main.py | 1 + .../distributed/launch/plugins/__init__.py | 16 +- .../paddle/distributed/launch/plugins/test.py | 25 +- .../distributed/launch/utils/kv_client.py | 8 +- .../distributed/launch/utils/kv_server.py | 9 +- .../paddle/distributed/launch/utils/nvsmi.py | 28 +- .../launch/utils/process_context.py | 33 +- python/paddle/distributed/metric/metrics.py | 170 +- python/paddle/distributed/models/moe/utils.py | 123 +- python/paddle/distributed/parallel.py | 111 +- .../paddle/distributed/parallel_with_gloo.py | 17 +- .../distributed/passes/auto_parallel_amp.py | 519 +- ...uto_parallel_data_parallel_optimization.py | 250 +- .../distributed/passes/auto_parallel_fp16.py | 425 +- .../passes/auto_parallel_grad_clip.py | 104 +- .../passes/auto_parallel_gradient_merge.py | 273 +- .../passes/auto_parallel_quantization.py | 153 +- .../passes/auto_parallel_recompute.py | 178 +- .../passes/auto_parallel_sharding.py | 485 +- python/paddle/distributed/passes/cpp_pass.py | 39 +- .../distributed/passes/fuse_all_reduce.py | 95 +- python/paddle/distributed/passes/pass_base.py | 58 +- .../paddle/distributed/passes/pass_utils.py | 5 +- .../distributed/passes/ps_server_pass.py | 129 +- .../distributed/passes/ps_trainer_pass.py | 759 +- python/paddle/distributed/ps/coordinator.py | 120 +- python/paddle/distributed/ps/the_one_ps.py | 708 +- .../paddle/distributed/ps/utils/ps_factory.py | 12 +- python/paddle/distributed/ps/utils/public.py | 756 +- python/paddle/distributed/rpc/rpc.py | 50 +- .../paddle/distributed/sharding/__init__.py | 5 +- .../distributed/sharding/group_sharded.py | 144 +- python/paddle/distributed/spawn.py | 189 +- .../paddle/distributed/utils/launch_utils.py | 181 +- python/paddle/distributed/utils/moe_utils.py | 121 +- python/paddle/distribution/__init__.py | 21 +- python/paddle/distribution/beta.py | 9 +- python/paddle/distribution/categorical.py | 60 +- python/paddle/distribution/constraint.py | 14 +- python/paddle/distribution/dirichlet.py | 52 +- python/paddle/distribution/distribution.py | 63 +- .../paddle/distribution/exponential_family.py | 6 +- python/paddle/distribution/gumbel.py | 30 +- python/paddle/distribution/independent.py | 23 +- python/paddle/distribution/kl.py | 77 +- python/paddle/distribution/laplace.py | 57 +- python/paddle/distribution/lognormal.py | 8 +- python/paddle/distribution/multinomial.py | 57 +- python/paddle/distribution/normal.py | 105 +- python/paddle/distribution/transform.py | 212 +- .../distribution/transformed_distribution.py | 38 +- python/paddle/distribution/uniform.py | 101 +- python/paddle/distribution/variable.py | 37 +- python/paddle/fft.py | 292 +- python/paddle/fluid/__init__.py | 68 +- python/paddle/fluid/average.py | 20 +- python/paddle/fluid/backward.py | 1009 ++- python/paddle/fluid/clip.py | 227 +- python/paddle/fluid/communicator.py | 52 +- python/paddle/fluid/compiler.py | 326 +- .../contrib/decoder/beam_search_decoder.py | 319 +- .../extend_optimizer_with_weight_decay.py | 74 +- .../paddle/fluid/contrib/layers/metric_op.py | 336 +- python/paddle/fluid/contrib/layers/nn.py | 1389 ++-- .../paddle/fluid/contrib/layers/rnn_impl.py | 358 +- .../paddle/fluid/contrib/memory_usage_calc.py | 8 +- .../fluid/contrib/mixed_precision/amp_nn.py | 96 +- .../contrib/mixed_precision/bf16/amp_lists.py | 45 +- .../contrib/mixed_precision/bf16/amp_utils.py | 279 +- .../contrib/mixed_precision/bf16/decorator.py | 124 +- .../contrib/mixed_precision/decorator.py | 310 +- .../contrib/mixed_precision/fp16_lists.py | 36 +- .../contrib/mixed_precision/fp16_utils.py | 283 +- python/paddle/fluid/contrib/model_stat.py | 9 +- python/paddle/fluid/contrib/op_frequence.py | 24 +- python/paddle/fluid/contrib/optimizer.py | 149 +- .../contrib/quantize/quantize_transpiler.py | 312 +- .../contrib/slim/quantization/adaround.py | 295 +- .../slim/quantization/cal_kl_threshold.py | 39 +- .../quantization/imperative/fuse_utils.py | 100 +- .../slim/quantization/imperative/ptq.py | 148 +- .../slim/quantization/imperative/ptq_hooks.py | 7 +- .../quantization/imperative/ptq_quantizer.py | 87 +- .../quantization/imperative/ptq_registry.py | 34 +- .../slim/quantization/imperative/qat.py | 306 +- .../slim/quantization/imperative/utils.py | 38 +- .../post_training_quantization.py | 1172 +-- .../quantization/quant2_int8_mkldnn_pass.py | 332 +- .../quantization/quant_int8_mkldnn_pass.py | 104 +- .../slim/quantization/quantization_pass.py | 1632 ++-- .../quantization/quantize_transpiler_v2.py | 366 +- .../fluid/contrib/slim/quantization/utils.py | 85 +- .../contrib/slim/tests/convert_model2dot.py | 51 +- .../slim/tests/imperative_test_utils.py | 190 +- ...t2_int8_image_classification_comparison.py | 271 +- .../slim/tests/quant2_int8_lstm_model.py | 188 +- .../slim/tests/quant2_int8_nlp_comparison.py | 236 +- ...nt_int8_image_classification_comparison.py | 193 +- .../contrib/slim/tests/save_quant_model.py | 135 +- .../fluid/contrib/slim/tests/test_graph.py | 59 +- .../slim/tests/test_imperative_out_scale.py | 136 +- .../contrib/slim/tests/test_imperative_ptq.py | 227 +- .../contrib/slim/tests/test_imperative_qat.py | 157 +- .../slim/tests/test_imperative_qat_amp.py | 102 +- .../tests/test_imperative_qat_channelwise.py | 8 +- .../slim/tests/test_imperative_qat_fuse.py | 7 +- .../slim/tests/test_imperative_qat_lsq.py | 166 +- .../tests/test_imperative_qat_user_defined.py | 151 +- .../slim/tests/test_imperative_skip_op.py | 68 +- .../test_moving_average_abs_max_scale_op.py | 33 +- ...t_post_training_quantization_lstm_model.py | 262 +- .../test_post_training_quantization_mnist.py | 516 +- ..._post_training_quantization_mobilenetv1.py | 340 +- ..._training_quantization_program_resnet50.py | 222 +- ...est_post_training_quantization_resnet50.py | 45 +- .../test_post_training_quantization_while.py | 358 +- .../tests/test_quant2_int8_mkldnn_pass.py | 236 +- .../tests/test_quantization_mkldnn_pass.py | 118 +- .../slim/tests/test_quantization_pass.py | 810 +- .../tests/test_quantization_scale_pass.py | 103 +- .../slim/tests/test_quantize_transpiler_v2.py | 143 +- .../tests/test_user_defined_quantization.py | 126 +- .../test_weight_quantization_mobilenetv1.py | 226 +- .../paddle/fluid/contrib/sparsity/__init__.py | 19 +- python/paddle/fluid/contrib/sparsity/asp.py | 252 +- .../contrib/sparsity/supported_layer_list.py | 43 +- python/paddle/fluid/contrib/sparsity/utils.py | 114 +- .../fluid/contrib/tests/test_amp_list.py | 12 +- .../fluid/contrib/tests/test_bf16_utils.py | 94 +- .../fluid/contrib/tests/test_correlation.py | 152 +- .../fluid/contrib/tests/test_fp16_utils.py | 17 +- .../tests/test_image_classification_fp16.py | 299 +- .../contrib/tests/test_model_cast_to_bf16.py | 106 +- .../tests/test_multi_precision_fp16_train.py | 209 +- .../contrib/tests/test_quantize_transpiler.py | 167 +- .../contrib/tests/test_weight_decay_extend.py | 115 +- python/paddle/fluid/core.py | 122 +- python/paddle/fluid/data.py | 18 +- python/paddle/fluid/data_feed_desc.py | 6 +- python/paddle/fluid/data_feeder.py | 206 +- python/paddle/fluid/dataloader/__init__.py | 10 +- .../paddle/fluid/dataloader/batch_sampler.py | 118 +- python/paddle/fluid/dataloader/collate.py | 12 +- .../fluid/dataloader/dataloader_iter.py | 207 +- python/paddle/fluid/dataloader/dataset.py | 69 +- python/paddle/fluid/dataloader/fetcher.py | 85 +- python/paddle/fluid/dataloader/flat.py | 50 +- python/paddle/fluid/dataloader/sampler.py | 75 +- python/paddle/fluid/dataloader/worker.py | 103 +- python/paddle/fluid/dataset.py | 168 +- python/paddle/fluid/debugger.py | 90 +- python/paddle/fluid/device_worker.py | 273 +- .../paddle/fluid/distribute_lookup_table.py | 6 +- python/paddle/fluid/distributed/downpour.py | 91 +- python/paddle/fluid/distributed/fleet.py | 25 +- python/paddle/fluid/distributed/helper.py | 22 +- python/paddle/fluid/distributed/node.py | 70 +- .../paddle/fluid/distributed/ps_instance.py | 16 +- python/paddle/fluid/distributed/ps_pb2.py | 1229 +-- python/paddle/fluid/dygraph/amp/auto_cast.py | 154 +- .../paddle/fluid/dygraph/amp/loss_scaler.py | 176 +- python/paddle/fluid/dygraph/base.py | 172 +- python/paddle/fluid/dygraph/checkpoint.py | 56 +- python/paddle/fluid/dygraph/container.py | 17 +- .../dygraph_to_static/assert_transformer.py | 21 +- .../dygraph_to_static/ast_transformer.py | 93 +- .../dygraph_to_static/base_transformer.py | 271 +- .../basic_api_transformer.py | 33 +- .../break_continue_transformer.py | 175 +- .../dygraph_to_static/call_transformer.py | 13 +- .../dygraph_to_static/cast_transformer.py | 8 +- .../dygraph_to_static/convert_call_func.py | 97 +- .../dygraph_to_static/convert_operators.py | 272 +- .../create_variable_transformer.py | 29 +- .../decorator_transformer.py | 87 +- .../early_return_transformer.py | 27 +- .../fluid/dygraph/dygraph_to_static/error.py | 188 +- .../dygraph_to_static/function_spec.py | 101 +- .../dygraph_to_static/ifelse_transformer.py | 220 +- .../dygraph_to_static/logging_utils.py | 24 +- .../dygraph_to_static/logical_transformer.py | 21 +- .../dygraph_to_static/loop_transformer.py | 237 +- .../dygraph/dygraph_to_static/origin_info.py | 68 +- .../dygraph_to_static/partial_program.py | 431 +- .../dygraph_to_static/print_transformer.py | 12 +- .../dygraph_to_static/program_translator.py | 432 +- .../dygraph_to_static/return_transformer.py | 191 +- .../dygraph_to_static/static_analysis.py | 83 +- .../tensor_shape_transformer.py | 8 +- .../dygraph_to_static/typehint_transformer.py | 8 +- .../fluid/dygraph/dygraph_to_static/utils.py | 526 +- .../dygraph_to_static/variable_trans_func.py | 14 +- python/paddle/fluid/dygraph/inplace_utils.py | 7 +- python/paddle/fluid/dygraph/io.py | 572 +- python/paddle/fluid/dygraph/jit.py | 427 +- python/paddle/fluid/dygraph/layer_hooks.py | 11 +- .../fluid/dygraph/layer_object_helper.py | 63 +- python/paddle/fluid/dygraph/layers.py | 434 +- .../fluid/dygraph/learning_rate_scheduler.py | 345 +- python/paddle/fluid/dygraph/math_op_patch.py | 221 +- python/paddle/fluid/dygraph/nn.py | 2081 +++--- python/paddle/fluid/dygraph/parallel.py | 265 +- .../paddle/fluid/dygraph/parallel_helper.py | 19 +- python/paddle/fluid/dygraph/rnn.py | 175 +- python/paddle/fluid/dygraph/tracer.py | 85 +- .../fluid/dygraph/varbase_patch_methods.py | 214 +- python/paddle/fluid/dygraph_utils.py | 12 +- python/paddle/fluid/entry_attr.py | 8 +- python/paddle/fluid/evaluator.py | 234 +- python/paddle/fluid/executor.py | 1376 ++-- python/paddle/fluid/framework.py | 1625 ++-- python/paddle/fluid/generator.py | 3 +- python/paddle/fluid/graphviz.py | 153 +- .../incubate/checkpoint/auto_checkpoint.py | 233 +- .../incubate/checkpoint/checkpoint_saver.py | 81 +- .../fluid/incubate/data_generator/__init__.py | 48 +- .../fluid/incubate/fleet/base/fleet_base.py | 68 +- .../paddle/fluid/incubate/fleet/base/mode.py | 1 + .../fluid/incubate/fleet/base/role_maker.py | 211 +- .../incubate/fleet/collective/__init__.py | 336 +- .../distribute_transpiler/__init__.py | 514 +- .../distributed_strategy.py | 151 +- .../parameter_server/ir/heter_trainer_pass.py | 51 +- .../fleet/parameter_server/ir/pserver_pass.py | 547 +- .../fleet/parameter_server/ir/public.py | 643 +- .../fleet/parameter_server/ir/trainer_pass.py | 991 +-- .../fleet/parameter_server/ir/ufind.py | 2 +- .../parameter_server/ir/vars_metatools.py | 115 +- .../incubate/fleet/parameter_server/mode.py | 1 + .../fleet/parameter_server/pslib/__init__.py | 461 +- .../fleet/parameter_server/pslib/node.py | 550 +- .../pslib/optimizer_factory.py | 575 +- .../fleet/parameter_server/pslib/ps_pb2.py | 1510 ++-- .../fleet/tests/ctr_dataset_reader.py | 19 +- .../incubate/fleet/tests/fleet_deep_ctr.py | 136 +- .../fluid/incubate/fleet/utils/fleet_util.py | 747 +- .../paddle/fluid/incubate/fleet/utils/hdfs.py | 54 +- .../fluid/incubate/fleet/utils/http_server.py | 10 +- .../fluid/incubate/fleet/utils/utils.py | 326 +- python/paddle/fluid/inference/__init__.py | 19 +- python/paddle/fluid/inference/wrapper.py | 39 +- python/paddle/fluid/initializer.py | 846 ++- python/paddle/fluid/input.py | 79 +- python/paddle/fluid/install_check.py | 67 +- python/paddle/fluid/io.py | 829 +- python/paddle/fluid/ir.py | 148 +- python/paddle/fluid/layer_helper.py | 76 +- python/paddle/fluid/layer_helper_base.py | 365 +- python/paddle/fluid/layers/collective.py | 174 +- python/paddle/fluid/layers/control_flow.py | 1907 +++-- python/paddle/fluid/layers/detection.py | 1935 ++--- python/paddle/fluid/layers/device.py | 9 +- python/paddle/fluid/layers/distributions.py | 161 +- python/paddle/fluid/layers/io.py | 317 +- .../fluid/layers/layer_function_generator.py | 125 +- .../fluid/layers/learning_rate_scheduler.py | 310 +- python/paddle/fluid/layers/loss.py | 724 +- python/paddle/fluid/layers/math_op_patch.py | 311 +- python/paddle/fluid/layers/metric_op.py | 188 +- python/paddle/fluid/layers/nn.py | 6637 +++++++++-------- python/paddle/fluid/layers/ops.py | 236 +- python/paddle/fluid/layers/rnn.py | 1542 ++-- python/paddle/fluid/layers/sequence_lod.py | 437 +- python/paddle/fluid/layers/tensor.py | 887 ++- python/paddle/fluid/layers/utils.py | 159 +- python/paddle/fluid/lazy_init.py | 9 +- python/paddle/fluid/lod_tensor.py | 25 +- python/paddle/fluid/memory_analysis.py | 14 +- python/paddle/fluid/metrics.py | 181 +- python/paddle/fluid/multiprocess_utils.py | 11 +- python/paddle/fluid/net_drawer.py | 29 +- python/paddle/fluid/nets.py | 286 +- python/paddle/fluid/op.py | 48 +- python/paddle/fluid/optimizer.py | 4099 ++++++---- python/paddle/fluid/parallel_executor.py | 78 +- python/paddle/fluid/param_attr.py | 74 +- python/paddle/fluid/profiler.py | 43 +- python/paddle/fluid/reader.py | 536 +- python/paddle/fluid/regularizer.py | 54 +- .../tests/book/notest_understand_sentiment.py | 282 +- .../fluid/tests/book/test_fit_a_line.py | 92 +- .../tests/book/test_image_classification.py | 133 +- .../tests/book/test_label_semantic_roles.py | 351 +- .../tests/book/test_machine_translation.py | 197 +- .../fluid/tests/book/test_recognize_digits.py | 157 +- .../tests/book/test_recommender_system.py | 210 +- .../tests/book/test_rnn_encoder_decoder.py | 195 +- .../fluid/tests/book/test_word2vec_book.py | 230 +- .../custom_kernel_dot_c_setup.py | 21 +- .../custom_kernel/custom_kernel_dot_setup.py | 30 +- .../custom_kernel/test_custom_kernel_dot.py | 22 +- .../custom_kernel/test_custom_kernel_load.py | 30 +- .../custom_raw_op_kernel_op_setup.py | 16 +- .../tests/custom_op/custom_relu_setup.py | 4 +- .../fluid/tests/custom_op/test_check_abi.py | 18 +- .../tests/custom_op/test_context_pool.py | 7 +- .../tests/custom_op/test_custom_attrs_jit.py | 42 +- .../tests/custom_op/test_custom_concat.py | 61 +- .../fluid/tests/custom_op/test_custom_conj.py | 35 +- .../tests/custom_op/test_custom_linear.py | 92 +- .../custom_op/test_custom_raw_op_kernel_op.py | 19 +- .../tests/custom_op/test_custom_relu_model.py | 200 +- .../custom_op/test_custom_relu_op_jit.py | 47 +- .../custom_op/test_custom_relu_op_setup.py | 208 +- .../custom_op/test_custom_simple_slice.py | 13 +- .../custom_op/test_custom_tanh_double_grad.py | 33 +- .../tests/custom_op/test_dispatch_jit.py | 45 +- .../tests/custom_op/test_multi_out_jit.py | 23 +- .../fluid/tests/custom_op/test_sysconfig.py | 1 - python/paddle/fluid/tests/custom_op/utils.py | 2 +- .../custom_device_multi_process_collective.py | 17 +- .../custom_runtime/process_group_xccl.py | 39 +- .../test_collective_process_group_xccl.py | 86 +- .../custom_runtime/test_custom_cpu_plugin.py | 78 +- .../test_custom_cpu_profiler_plugin.py | 22 +- .../test_custom_cpu_to_static.py | 154 +- .../fluid/tests/test_beam_search_decoder.py | 146 +- python/paddle/fluid/tests/test_data_feeder.py | 36 +- python/paddle/fluid/tests/test_detection.py | 1002 +-- python/paddle/fluid/tests/test_error_clip.py | 30 +- python/paddle/fluid/tests/test_if_else_op.py | 67 +- python/paddle/fluid/tests/test_lod_tensor.py | 118 +- .../tests/test_python_operator_overriding.py | 33 +- python/paddle/fluid/tests/test_sequential.py | 1 - .../paddle/fluid/tests/unittests/__init__.py | 2 + .../fluid/tests/unittests/ascend_group.py | 108 +- .../ascend_multi_process_collective.py | 17 +- .../tests/unittests/asp/asp_pruning_base.py | 57 +- .../asp/test_asp_customized_pruning.py | 221 +- .../asp/test_asp_optimize_dynamic.py | 167 +- .../unittests/asp/test_asp_optimize_static.py | 199 +- .../unittests/asp/test_asp_pruning_dynamic.py | 71 +- .../unittests/asp/test_asp_pruning_static.py | 69 +- .../tests/unittests/asp/test_asp_save_load.py | 115 +- .../tests/unittests/asp/test_asp_utils.py | 133 +- .../asp/test_fleet_with_asp_dynamic.py | 95 +- .../asp/test_fleet_with_asp_sharding.py | 47 +- .../asp/test_fleet_with_asp_static.py | 144 +- .../tests/unittests/auto_checkpoint_utils.py | 28 +- .../auto_parallel/amp_pass_unittest.py | 9 +- .../auto_parallel_relaunch_model.py | 92 +- ...auto_parallel_relaunch_with_gpt_planner.py | 148 +- .../auto_parallel_relaunch_with_planner.py | 45 +- .../auto_parallel/clip_grad_by_global_norm.py | 11 +- .../unittests/auto_parallel/converter.py | 12 +- .../unittests/auto_parallel/engine_api.py | 278 +- .../unittests/auto_parallel/engine_api_dp.py | 61 +- .../unittests/auto_parallel/get_gpt_model.py | 85 +- .../gradient_merge_pass_unittest.py | 19 +- .../auto_parallel/high_order_grad.py | 32 +- .../auto_parallel/iterable_dataset.py | 72 +- .../auto_parallel/optimization_tuner_api.py | 62 +- .../auto_parallel/recompute_pass_unittest.py | 5 +- .../auto_parallel/sharding_pass_unittest.py | 23 +- .../test_auto_parallel_relaunch.py | 39 +- .../unittests/auto_parallel/test_base_cost.py | 154 +- .../unittests/auto_parallel/test_cluster.py | 11 +- .../auto_parallel/test_cluster_v2.py | 9 +- .../unittests/auto_parallel/test_comm_cost.py | 127 +- .../unittests/auto_parallel/test_comp_cost.py | 105 +- .../test_conditional_block_reshard.py | 45 +- .../unittests/auto_parallel/test_converter.py | 20 +- .../auto_parallel/test_dist_assign.py | 17 +- .../auto_parallel/test_dist_attr_v2.py | 55 +- .../auto_parallel/test_dist_context.py | 189 +- .../auto_parallel/test_dist_embedding.py | 38 +- .../auto_parallel/test_dist_matmul.py | 94 +- .../auto_parallel/test_dist_op_cost.py | 333 +- .../auto_parallel/test_dist_pnorm.py | 25 +- .../auto_parallel/test_dist_reshape.py | 11 +- .../auto_parallel/test_dist_shape.py | 11 +- .../auto_parallel/test_dist_slice.py | 16 +- .../auto_parallel/test_dist_split.py | 11 +- .../auto_parallel/test_engine_api.py | 18 +- .../auto_parallel/test_engine_api_dp.py | 18 +- .../auto_parallel/test_engine_callbacks.py | 62 +- .../auto_parallel/test_high_order_grad.py | 18 +- .../unittests/auto_parallel/test_interface.py | 115 +- .../auto_parallel/test_iterable_dataset.py | 18 +- .../auto_parallel/test_lr_grad_clip.py | 47 +- .../auto_parallel/test_new_cost_model.py | 34 +- .../test_optimization_tuner_api.py | 18 +- .../auto_parallel/test_parallel_tuner.py | 131 +- .../auto_parallel/test_parallel_tuner_full.py | 140 +- .../test_parallel_tuner_predict.py | 139 +- .../unittests/auto_parallel/test_pass_amp.py | 18 +- .../auto_parallel/test_pass_grad_clip.py | 23 +- .../auto_parallel/test_pass_gradient_merge.py | 23 +- .../auto_parallel/test_pass_quantization.py | 1 - .../auto_parallel/test_pass_recompute.py | 18 +- .../auto_parallel/test_pass_sharding.py | 18 +- .../auto_parallel/test_prim_dist_op.py | 86 +- .../auto_parallel/test_process_mesh.py | 68 +- .../auto_parallel/test_process_mesh_v2.py | 36 +- .../unittests/auto_parallel/test_recorder.py | 36 +- .../test_relaunch_with_gpt_planner.py | 37 +- .../test_relaunch_with_planner.py | 37 +- .../unittests/auto_parallel/test_strategy.py | 1 - .../unittests/auto_parallel/test_to_static.py | 91 +- .../unittests/auto_parallel/test_trial.py | 1 - .../auto_parallel/test_tunable_space.py | 8 +- .../auto_parallel/test_tunable_variable.py | 31 +- .../auto_parallel/test_while_op_completion.py | 118 +- .../auto_parallel/test_while_op_partition.py | 112 +- .../unittests/auto_parallel_autoconvert.py | 283 +- .../unittests/auto_parallel_data_unshard.py | 91 +- .../unittests/auto_parallel_gpt_model.py | 796 +- .../unittests/auto_parallel_save_load.py | 201 +- .../fluid/tests/unittests/autograd/config.py | 26 +- .../test_autograd_functional_dynamic.py | 419 +- .../autograd/test_autograd_functional_prim.py | 215 +- .../test_autograd_functional_static.py | 228 +- .../autograd/test_jvp_and_transpose.py | 323 +- .../tests/unittests/autograd/test_minimize.py | 30 +- .../unittests/autograd/test_orig2prim.py | 530 +- .../unittests/autograd/test_prim2orig.py | 284 +- .../tests/unittests/autograd/test_primapi.py | 809 +- .../tests/unittests/autograd/test_primops.py | 208 +- .../unittests/autograd/test_transform.py | 239 +- .../fluid/tests/unittests/autograd/utils.py | 84 +- .../paddle/fluid/tests/unittests/benchmark.py | 48 +- .../fluid/tests/unittests/benchmark_sum_op.py | 1 - .../tests/unittests/c_embedding_op_base.py | 13 +- .../tests/unittests/check_nan_inf_base.py | 23 +- .../unittests/check_nan_inf_base_dygraph.py | 13 +- .../collective/collective_allgather_api.py | 28 +- .../collective_allgather_api_dygraph.py | 1 - ...collective_allgather_object_api_dygraph.py | 1 - .../collective/collective_allreduce_api.py | 7 +- .../collective_allreduce_api_dygraph.py | 1 - .../collective_allreduce_new_group_api.py | 7 +- .../collective/collective_allreduce_op.py | 30 +- .../collective_allreduce_op_wait.py | 42 +- .../collective/collective_alltoall_api.py | 7 +- .../collective_alltoall_api_dygraph.py | 1 - .../collective/collective_alltoall_single.py | 43 +- .../collective_alltoall_single_api_dygraph.py | 1 - .../collective/collective_barrier_api.py | 1 - .../collective_batch_isend_irecv.py | 6 +- .../collective/collective_broadcast_api.py | 7 +- .../collective_broadcast_api_dygraph.py | 1 - .../collective/collective_broadcast_op.py | 33 +- .../collective/collective_concat_op.py | 24 +- .../collective/collective_global_gather.py | 59 +- .../collective_global_gather_dygraph.py | 19 +- .../collective/collective_global_scatter.py | 50 +- .../collective_global_scatter_dygraph.py | 19 +- .../collective/collective_identity_op.py | 23 +- .../collective_isend_irecv_api_dygraph.py | 1 - .../collective/collective_reduce_api.py | 7 +- .../collective_reduce_api_dygraph.py | 1 - .../collective/collective_reduce_op.py | 33 +- .../collective_reduce_op_calc_stream.py | 38 +- .../collective/collective_reduce_scatter.py | 7 +- .../collective_reduce_scatter_api_dygraph.py | 1 - .../collective/collective_scatter_api.py | 17 +- .../collective_scatter_api_dygraph.py | 13 +- .../collective/collective_scatter_op.py | 34 +- .../collective/collective_sendrecv_api.py | 11 +- .../collective_sendrecv_api_dygraph.py | 1 - .../collective/collective_sendrecv_op.py | 48 +- .../collective_sendrecv_op_array.py | 42 +- .../collective_sendrecv_op_dynamic_shape.py | 57 +- .../collective/collective_split_op.py | 24 +- .../collective/column_parallel_linear_api.py | 15 +- ...munication_stream_allgather_api_dygraph.py | 60 +- ...munication_stream_allreduce_api_dygraph.py | 19 +- ...mmunication_stream_alltoall_api_dygraph.py | 78 +- ...tion_stream_alltoall_single_api_dygraph.py | 25 +- ...munication_stream_broadcast_api_dygraph.py | 19 +- ...communication_stream_reduce_api_dygraph.py | 30 +- ...ation_stream_reduce_scatter_api_dygraph.py | 50 +- ...ommunication_stream_scatter_api_dygraph.py | 40 +- ...mmunication_stream_sendrecv_api_dygraph.py | 33 +- .../fleet/auto_parallel_parallelizer.py | 83 +- .../collective/fleet/c_comm_init_op.py | 46 +- .../fleet/dist_mnist_gradient_merge.py | 31 +- ...dist_mnist_gradient_merge_raw_optimizer.py | 20 +- .../fleet/dygraph_group_sharded_api.py | 137 +- .../fleet/dygraph_group_sharded_api_eager.py | 165 +- .../fleet/dygraph_group_sharded_stage2.py | 149 +- ...graph_group_sharded_stage2_comm_overlap.py | 149 +- .../dygraph_group_sharded_stage2_offload.py | 61 +- .../fleet/dygraph_group_sharded_stage3.py | 257 +- .../dygraph_group_sharded_stage3_offload.py | 147 +- .../dygraph_sharding_optimizer_stage2.py | 47 +- .../fleet/dygraph_sharding_stage2.py | 147 +- .../fleet/dygraph_sharding_stage2_offload.py | 55 +- .../fleet/dygraph_sharding_stage3.py | 255 +- .../fleet/dygraph_sharding_stage3_offload.py | 142 +- .../hybrid_parallel_communicate_group.py | 39 +- .../fleet/hybrid_parallel_inference_helper.py | 81 +- .../fleet/hybrid_parallel_mp_amp.py | 28 +- .../fleet/hybrid_parallel_mp_clip_grad.py | 17 +- .../fleet/hybrid_parallel_mp_fp16.py | 22 +- .../fleet/hybrid_parallel_mp_layers.py | 105 +- .../fleet/hybrid_parallel_mp_model.py | 129 +- .../fleet/hybrid_parallel_mp_random.py | 3 +- .../fleet/hybrid_parallel_pp_amp.py | 66 +- .../fleet/hybrid_parallel_pp_clip_grad.py | 33 +- .../fleet/hybrid_parallel_pp_embedding.py | 63 +- .../fleet/hybrid_parallel_pp_fp16.py | 86 +- ...id_parallel_pp_layer_with_virtual_stage.py | 27 +- .../fleet/hybrid_parallel_pp_recompute.py | 46 +- .../fleet/hybrid_parallel_pp_save_load.py | 34 +- ...arallel_pp_save_load_with_virtual_stage.py | 39 +- .../fleet/hybrid_parallel_pp_transformer.py | 39 +- ...allel_pp_transformer_with_virtual_stage.py | 32 +- .../collective/fleet/hybrid_parallel_qat.py | 169 +- .../fleet/hybrid_parallel_sharding_model.py | 220 +- .../fleet/hybrid_parallel_shared_weight.py | 79 +- .../unittests/collective/fleet/new_group.py | 23 +- .../fleet/parallel_class_center_sample.py | 54 +- ...parallel_dygraph_control_flow_different.py | 70 +- .../parallel_dygraph_control_flow_same.py | 32 +- .../fleet/parallel_dygraph_no_sync.py | 34 +- .../parallel_dygraph_no_sync_control_flow.py | 16 +- ...parallel_dygraph_no_sync_gradient_check.py | 34 +- .../parallel_dygraph_no_sync_unused_params.py | 16 +- .../fleet/parallel_dygraph_se_resnext.py | 271 +- .../fleet/parallel_dygraph_sync_batch_norm.py | 74 +- .../fleet/parallel_dygraph_transformer.py | 969 ++- .../fleet/parallel_margin_cross_entropy.py | 143 +- .../collective/fleet/pipeline_mnist.py | 87 +- .../fleet/pipeline_mnist_multi_device.py | 87 +- .../fleet/pipeline_mnist_one_device.py | 70 +- .../fleet/static_model_parallel_by_col.py | 54 +- .../fleet/static_model_parallel_by_row.py | 51 +- .../fleet/static_model_parallel_embedding.py | 31 +- .../collective/fleet/test_auto_checkpoint.py | 27 +- .../collective/fleet/test_auto_checkpoint1.py | 3 +- .../collective/fleet/test_auto_checkpoint2.py | 3 +- .../collective/fleet/test_auto_checkpoint3.py | 3 +- .../fleet/test_auto_checkpoint_dist_basic.py | 14 +- .../fleet/test_auto_checkpoint_multiple.py | 25 +- .../fleet/test_communicator_half_async.py | 24 +- .../fleet/test_communicator_sync.py | 6 +- .../collective/fleet/test_dgc_momentum_op.py | 58 +- .../unittests/collective/fleet/test_dgc_op.py | 28 +- .../collective/fleet/test_dgc_optimizer.py | 107 +- .../fleet/test_dist_mnist_dgc_nccl.py | 33 +- .../fleet/test_dist_mnist_gradient_merge.py | 45 +- .../fleet/test_dist_se_resnext_dgc.py | 12 +- .../fleet/test_distributed_strategy.py | 135 +- .../fleet/test_dygraph_recompute.py | 186 +- .../fleet/test_dygraph_recompute_for_eager.py | 179 +- .../test_dygraph_sharding_optimizer_stage2.py | 5 +- .../fleet/test_dygraph_sharding_stage2.py | 5 +- .../fleet/test_dygraph_sharding_stage3.py | 5 +- .../collective/fleet/test_fleet_amp_init.py | 51 +- .../fleet/test_fleet_amp_meta_optimizer.py | 32 +- .../collective/fleet/test_fleet_checkpoint.py | 78 +- .../fleet/test_fleet_dgc_meta_optimizer.py | 26 +- .../fleet/test_fleet_distributed_strategy.py | 90 +- ...est_fleet_fp16_allreduce_meta_optimizer.py | 27 +- ...est_fleet_gradient_merge_meta_optimizer.py | 22 +- ...st_fleet_graph_execution_meta_optimizer.py | 281 +- .../fleet/test_fleet_graph_executor.py | 35 +- .../fleet/test_fleet_hybrid_meta_optimizer.py | 1064 ++- .../fleet/test_fleet_lamb_meta_optimizer.py | 54 +- .../fleet/test_fleet_lars_meta_optimizer.py | 71 +- .../test_fleet_localsgd_meta_optimizer.py | 14 +- .../collective/fleet/test_fleet_log.py | 1 - .../fleet/test_fleet_meta_optimizer_base.py | 44 +- .../test_fleet_pipeline_meta_optimizer.py | 49 +- ..._pipeline_meta_optimizer_with_recompute.py | 32 +- .../fleet/test_fleet_private_function.py | 6 +- .../test_fleet_raw_program_meta_optimizer.py | 16 +- .../test_fleet_recompute_meta_optimizer.py | 25 +- .../fleet/test_fleet_rolemaker_new.py | 24 +- .../test_fleet_sharding_meta_optimizer.py | 3107 ++++++-- .../fleet/test_fleet_static_mp_layers.py | 66 +- .../collective/fleet/test_fleet_utils.py | 90 +- .../unittests/collective/fleet/test_hdfs1.py | 37 +- .../unittests/collective/fleet/test_hdfs2.py | 8 +- .../unittests/collective/fleet/test_hdfs3.py | 8 +- .../test_hybrid_parallel_inference_helper.py | 1 - .../test_imperative_auto_mixed_precision.py | 862 ++- ...perative_auto_mixed_precision_for_eager.py | 860 ++- .../collective/fleet/test_ir_pass_pipeline.py | 1 - .../collective/fleet/test_mixed_precision.py | 55 +- .../test_parallel_class_center_sample.py | 1 - .../test_parallel_dygraph_control_flow.py | 26 +- .../fleet/test_parallel_dygraph_mnist.py | 23 +- .../fleet/test_parallel_dygraph_mp_layers.py | 1 - .../fleet/test_parallel_dygraph_no_sync.py | 46 +- ...parallel_dygraph_no_sync_gradient_check.py | 6 +- ...test_parallel_dygraph_pipeline_parallel.py | 17 +- ...ph_pipeline_parallel_with_virtual_stage.py | 7 +- .../fleet/test_parallel_dygraph_qat.py | 51 +- .../fleet/test_parallel_dygraph_se_resnext.py | 17 +- ...test_parallel_dygraph_sharding_parallel.py | 5 +- .../test_parallel_dygraph_sparse_embedding.py | 24 +- ...el_dygraph_sparse_embedding_over_height.py | 22 +- .../test_parallel_dygraph_sync_batch_norm.py | 11 +- .../test_parallel_dygraph_tensor_parallel.py | 1 - .../test_parallel_dygraph_transformer.py | 22 +- .../test_parallel_dygraph_unused_variables.py | 17 +- .../test_parallel_margin_cross_entropy.py | 6 +- .../collective/fleet/test_pipeline.py | 28 +- .../collective/fleet/test_recv_save_op.py | 120 +- .../unittests/collective/fleet/test_rnn_dp.py | 130 +- .../fleet/test_static_model_parallel.py | 34 +- .../collective/fleet/test_tcp_store.py | 1 - .../collective/init_process_group.py | 1 - .../unittests/collective/multinode/common.py | 8 +- .../collective/multinode/dygraph_hybrid_dp.py | 18 +- .../multinode/dygraph_hybrid_dpppmp.py | 77 +- .../multinode/dygraph_hybrid_fp16.py | 92 +- .../multinode/dygraph_hybrid_recompute.py | 50 +- .../mn_dygraph_group_sharded_stage3.py | 260 +- .../multinode/mn_dygraph_sharding_stage2.py | 152 +- .../multinode/test_collective_multi_nodes.py | 40 +- .../test_multinode_dygraph_hybrid_dp.py | 7 +- .../test_multinode_dygraph_hybrid_dpppmp.py | 19 +- .../test_multinode_dygraph_sharding.py | 17 +- .../collective/orthogonal_strategy.py | 24 +- .../collective/parallel_embedding_api.py | 27 +- .../collective/process_group_gloo.py | 16 +- .../unittests/collective/process_group_mpi.py | 95 +- .../collective/process_group_nccl.py | 134 +- .../collective/row_parallel_linear_api.py | 15 +- .../unittests/collective/strategy_group.py | 16 +- .../unittests/collective/test_allreduce.py | 1 - .../unittests/collective/test_broadcast.py | 1 - .../unittests/collective/test_c_concat.py | 1 - .../unittests/collective/test_c_identity.py | 1 - .../unittests/collective/test_c_split.py | 1 - .../test_collective_allgather_api.py | 95 +- .../test_collective_allgather_object_api.py | 53 +- .../test_collective_allreduce_api.py | 63 +- .../test_collective_alltoall_api.py | 23 +- .../test_collective_alltoall_single.py | 1 - .../test_collective_alltoall_single_api.py | 23 +- .../collective/test_collective_barrier_api.py | 6 +- .../test_collective_batch_isend_irecv.py | 1 - .../test_collective_broadcast_api.py | 58 +- .../test_collective_cpu_barrier_with_gloo.py | 35 +- .../test_collective_global_gather.py | 30 +- .../test_collective_global_scatter.py | 30 +- .../test_collective_isend_irecv_api.py | 23 +- .../collective/test_collective_optimizer.py | 14 +- .../test_collective_process_group.py | 1 - .../collective/test_collective_reduce.py | 1 - .../collective/test_collective_reduce_api.py | 48 +- .../test_collective_reduce_scatter.py | 1 - .../test_collective_reduce_scatter_api.py | 23 +- .../collective/test_collective_scatter.py | 1 - .../collective/test_collective_scatter_api.py | 53 +- .../collective/test_collective_sendrecv.py | 11 +- .../test_collective_sendrecv_api.py | 25 +- .../test_collective_split_col_linear.py | 6 +- .../test_collective_split_embedding.py | 6 +- ...llective_split_embedding_none_divisible.py | 13 +- .../test_collective_split_row_linear.py | 6 +- .../collective/test_collective_wait.py | 7 +- .../collective/test_communication_api_base.py | 26 +- ...test_communication_stream_allgather_api.py | 21 +- ...test_communication_stream_allreduce_api.py | 21 +- .../test_communication_stream_alltoall_api.py | 21 +- ...ommunication_stream_alltoall_single_api.py | 21 +- ...test_communication_stream_broadcast_api.py | 21 +- .../test_communication_stream_reduce_api.py | 21 +- ...communication_stream_reduce_scatter_api.py | 21 +- .../test_communication_stream_scatter_api.py | 21 +- .../test_communication_stream_sendrecv_api.py | 21 +- .../collective/test_eager_dist_api.py | 1 - .../collective/test_gen_nccl_id_op.py | 18 +- .../collective/test_new_group_api.py | 6 +- .../collective/world_size_and_rank.py | 9 +- .../unittests/collective_allgather_op.py | 33 +- .../unittests/collective_reducescatter.py | 7 +- .../unittests/collective_reducescatter_op.py | 33 +- .../tests/unittests/ctr_dataset_reader.py | 31 +- .../fluid/tests/unittests/decorator_helper.py | 4 - .../fluid/tests/unittests/detected_gpu.py | 6 +- .../tests/unittests/dist_allreduce_op.py | 44 +- .../paddle/fluid/tests/unittests/dist_ctr.py | 94 +- .../fluid/tests/unittests/dist_ctr_reader.py | 5 +- .../fluid/tests/unittests/dist_fleet_ctr.py | 180 +- .../tests/unittests/dist_fleet_ctr_ps_gpu.py | 50 +- .../tests/unittests/dist_fleet_debug_gloo.py | 27 +- .../dist_fleet_heter_pipeline_ctr.py | 102 +- .../dist_fleet_raw_program_optimizer.py | 49 +- ...et_raw_program_optimizer_fuse_allreduce.py | 49 +- .../tests/unittests/dist_fleet_simnet_bow.py | 127 +- .../dist_fleet_sparse_embedding_ctr.py | 87 +- .../fluid/tests/unittests/dist_mnist.py | 55 +- .../tests/unittests/dist_mnist_batch_merge.py | 21 +- .../unittests/dist_mnist_fp16_allreduce.py | 35 +- .../fluid/tests/unittests/dist_mnist_lars.py | 31 +- .../fluid/tests/unittests/dist_save_load.py | 111 +- .../fluid/tests/unittests/dist_se_resnext.py | 217 +- .../tests/unittests/dist_sharding_save.py | 42 +- .../unittests/dist_text_classification.py | 92 +- .../fluid/tests/unittests/dist_transformer.py | 1318 ++-- .../fluid/tests/unittests/dist_word2vec.py | 62 +- .../distributed_fused_lamb_test_base.py | 102 +- .../auto_parallel_pass_test_base.py | 185 +- .../check_pass_conflict_example.py | 2 - .../distributed_passes/dist_pass_test_base.py | 121 +- .../unittests/distributed_passes/model_zoo.py | 43 +- .../distributed_passes/pass_run_main.py | 30 +- .../distributed_passes/ps_pass_test_base.py | 80 +- .../test_auto_parallel_amp_pass.py | 13 +- ...arallel_data_parallel_optimization_pass.py | 57 +- .../test_auto_parallel_fp16_pass.py | 32 +- .../test_auto_parallel_gradient_merge_pass.py | 152 +- .../test_auto_parallel_recompute_pass.py | 19 +- .../test_auto_parallel_sharding_pass.py | 13 +- .../test_dist_fuse_adam_pass.py | 20 +- .../test_dist_fuse_all_reduce_pass.py | 11 +- .../test_dist_fuse_bn_act_pass.py | 8 +- .../test_dist_fuse_bn_add_act_pass.py | 8 +- .../test_dist_fuse_momentum_pass.py | 19 +- ...test_dist_fuse_relu_depthwise_conv_pass.py | 8 +- .../test_dist_fuse_sgd_pass.py | 20 +- .../test_dist_inplace_addto_pass.py | 11 +- .../distributed_passes/test_ps_server_pass.py | 1 - .../test_ps_trainer_pass.py | 21 +- .../distributed_passes/test_white_lists.py | 17 +- .../tests/unittests/distribution/config.py | 2 +- .../tests/unittests/distribution/mock_data.py | 10 +- .../unittests/distribution/parameterize.py | 64 +- .../distribution/test_dirichlet_op.py | 8 +- .../distribution/test_distribution.py | 25 +- .../distribution/test_distribution_beta.py | 37 +- .../test_distribution_beta_static.py | 141 +- .../test_distribution_categorical.py | 156 +- .../test_distribution_constraint.py | 37 +- .../test_distribution_dirichlet.py | 46 +- .../test_distribution_dirichlet_static.py | 73 +- .../test_distribution_expfamily.py | 40 +- .../test_distribution_expfamily_static.py | 13 +- .../distribution/test_distribution_gumbel.py | 149 +- .../test_distribution_gumbel_static.py | 162 +- .../test_distribution_independent.py | 60 +- .../test_distribution_independent_static.py | 102 +- .../distribution/test_distribution_laplace.py | 199 +- .../test_distribution_laplace_static.py | 228 +- .../test_distribution_lognormal.py | 216 +- .../test_distribution_lognormal_static.py | 202 +- .../test_distribution_multinomial.py | 106 +- .../test_distribution_multinomial_static.py | 124 +- .../distribution/test_distribution_normal.py | 419 +- .../test_distribution_transform.py | 981 ++- .../test_distribution_transform_static.py | 960 ++- ...t_distribution_transformed_distribution.py | 20 +- ...ibution_transformed_distribution_static.py | 52 +- .../distribution/test_distribution_uniform.py | 193 +- .../test_distribution_variable.py | 39 +- .../tests/unittests/distribution/test_kl.py | 116 +- .../unittests/distribution/test_kl_static.py | 145 +- .../tests/unittests/dygraph_fleet_api.py | 7 +- .../unittests/dygraph_recompute_hybrid.py | 138 +- .../dygraph_to_static/bert_dygraph_model.py | 407 +- .../unittests/dygraph_to_static/bert_utils.py | 197 +- .../unittests/dygraph_to_static/darknet.py | 158 +- .../unittests/dygraph_to_static/decos.py | 3 - .../dygraph_to_static/ifelse_simple_func.py | 101 +- .../dygraph_to_static/predictor_utils.py | 5 +- .../seq2seq_dygraph_model.py | 528 +- .../dygraph_to_static/seq2seq_utils.py | 6 +- .../dygraph_to_static/simnet_dygraph_model.py | 140 +- .../simnet_dygraph_model_v2.py | 135 +- .../dygraph_to_static/test_assert.py | 37 +- .../dygraph_to_static/test_ast_util.py | 13 +- .../test_basic_api_transformation.py | 150 +- .../unittests/dygraph_to_static/test_bert.py | 167 +- .../unittests/dygraph_to_static/test_bmn.py | 533 +- .../dygraph_to_static/test_break_continue.py | 29 +- .../dygraph_to_static/test_cache_program.py | 42 +- .../unittests/dygraph_to_static/test_cast.py | 74 +- .../test_closure_analysis.py | 124 +- .../dygraph_to_static/test_container.py | 22 +- .../dygraph_to_static/test_convert_call.py | 69 +- .../test_convert_call_generator.py | 4 +- .../test_convert_operators.py | 126 +- .../test_cpu_cuda_to_tensor.py | 32 +- .../dygraph_to_static/test_cycle_gan.py | 490 +- .../dygraph_to_static/test_declarative.py | 158 +- .../test_decorator_transform.py | 17 +- .../dygraph_to_static/test_deepcopy.py | 5 +- .../unittests/dygraph_to_static/test_dict.py | 95 +- .../dygraph_to_static/test_drop_path.py | 2 - .../test_duplicate_output.py | 4 +- .../dygraph_to_static/test_fetch_feed.py | 26 +- .../dygraph_to_static/test_for_enumerate.py | 51 +- .../dygraph_to_static/test_full_name_usage.py | 27 +- .../dygraph_to_static/test_function_spec.py | 24 +- .../unittests/dygraph_to_static/test_grad.py | 41 +- .../test_gradient_aggregation.py | 24 +- .../dygraph_to_static/test_grid_generator.py | 59 +- .../dygraph_to_static/test_ifelse.py | 89 +- .../dygraph_to_static/test_isinstance.py | 18 +- .../test_jit_property_save.py | 6 +- .../unittests/dygraph_to_static/test_lac.py | 477 +- .../dygraph_to_static/test_lambda.py | 18 +- .../dygraph_to_static/test_layer_hook.py | 15 +- .../unittests/dygraph_to_static/test_len.py | 32 +- .../unittests/dygraph_to_static/test_list.py | 51 +- .../dygraph_to_static/test_logging_utils.py | 31 +- .../dygraph_to_static/test_logical.py | 33 +- .../unittests/dygraph_to_static/test_loop.py | 93 +- .../unittests/dygraph_to_static/test_lstm.py | 46 +- .../unittests/dygraph_to_static/test_mnist.py | 262 +- .../dygraph_to_static/test_mnist_amp.py | 33 +- .../dygraph_to_static/test_mnist_pure_fp16.py | 50 +- .../dygraph_to_static/test_mobile_net.py | 564 +- .../dygraph_to_static/test_multi_forward.py | 10 +- .../dygraph_to_static/test_op_attr.py | 22 +- .../dygraph_to_static/test_param_guard.py | 15 +- .../dygraph_to_static/test_params_no_grad.py | 8 +- .../dygraph_to_static/test_partial_program.py | 55 +- .../unittests/dygraph_to_static/test_place.py | 1 - .../unittests/dygraph_to_static/test_print.py | 15 +- .../test_program_translator.py | 152 +- .../dygraph_to_static/test_ptb_lm.py | 163 +- .../dygraph_to_static/test_ptb_lm_v2.py | 207 +- .../test_reinforcement_learning.py | 29 +- .../dygraph_to_static/test_resnet.py | 252 +- .../dygraph_to_static/test_resnet_amp.py | 33 +- .../test_resnet_pure_fp16.py | 42 +- .../dygraph_to_static/test_resnet_v2.py | 254 +- .../dygraph_to_static/test_return.py | 38 +- .../dygraph_to_static/test_rollback.py | 17 +- .../test_save_inference_model.py | 111 +- .../dygraph_to_static/test_save_load.py | 28 +- .../dygraph_to_static/test_se_resnet.py | 402 +- .../dygraph_to_static/test_sentiment.py | 229 +- .../dygraph_to_static/test_seq2seq.py | 140 +- .../dygraph_to_static/test_setter_helper.py | 6 +- .../dygraph_to_static/test_simnet.py | 57 +- .../dygraph_to_static/test_simnet_v2.py | 62 +- .../unittests/dygraph_to_static/test_slice.py | 95 +- .../dygraph_to_static/test_spec_names.py | 2 - .../dygraph_to_static/test_static_analysis.py | 33 +- .../dygraph_to_static/test_tensor_methods.py | 4 - .../dygraph_to_static/test_tensor_shape.py | 64 +- .../dygraph_to_static/test_to_tensor.py | 19 +- .../dygraph_to_static/test_transformer.py | 393 +- .../unittests/dygraph_to_static/test_tsm.py | 287 +- .../dygraph_to_static/test_typehint.py | 9 +- .../dygraph_to_static/test_typing.py | 9 - .../unittests/dygraph_to_static/test_utils.py | 5 +- .../test_variable_trans_func.py | 14 +- .../dygraph_to_static/test_warning.py | 11 +- .../dygraph_to_static/test_word2vec.py | 113 +- .../dygraph_to_static/test_yolov3.py | 40 +- .../transformer_dygraph_model.py | 831 ++- .../dygraph_to_static/transformer_util.py | 186 +- .../dygraph_to_static/tsm_config_utils.py | 22 +- .../unittests/dygraph_to_static/yolov3.py | 239 +- .../fluid/tests/unittests/elastic_demo.py | 16 +- .../fluid/tests/unittests/fake_reader.py | 31 +- .../fluid/tests/unittests/feed_data_reader.py | 9 +- .../tests/unittests/fft/spectral_op_np.py | 16 +- .../fluid/tests/unittests/fft/test_fft.py | 2006 +++-- .../fft/test_fft_with_static_graph.py | 2117 ++++-- .../tests/unittests/fft/test_spectral_op.py | 282 +- .../fluid/tests/unittests/find_ports.py | 3 +- .../unittests/fleet_heter_ps_training.py | 99 +- .../unittests/fleet_meta_optimizer_base.py | 110 +- .../tests/unittests/fleet_ps_training.py | 16 +- .../fluid/tests/unittests/gradient_checker.py | 362 +- .../fluid/tests/unittests/hccl_tools.py | 67 +- .../fluid/tests/unittests/hdfs_test_utils.py | 18 +- .../unittests/hybrid_parallel_pp_alexnet.py | 42 +- .../unittests/hybrid_parallel_pp_layer.py | 21 +- .../test_standalone_controlflow.py | 43 +- .../interpreter/test_standalone_executor.py | 146 +- .../test_standalone_multiply_write.py | 10 +- .../deprecated/test_custom_nllloss_ipu.py | 28 +- .../custom_ops/test_checkpointoutput_ipu.py | 19 +- .../custom_ops/test_custom_leaky_relu_ipu.py | 49 +- .../ipu/custom_ops/test_custom_ops_ipu.py | 93 +- .../test_dist_data_parallel_ipu.py | 82 +- .../distributed/test_dist_pod128_sample.py | 53 +- .../ipu/distributed/test_dist_sample.py | 98 +- .../fluid/tests/unittests/ipu/op_test_ipu.py | 88 +- .../unittests/ipu/test_activation_ops_ipu.py | 33 +- .../ipu/test_affine_channel_op_ipu.py | 21 +- .../unittests/ipu/test_arg_max_op_ipu.py | 8 +- .../unittests/ipu/test_arg_min_op_ipu.py | 8 +- .../unittests/ipu/test_argsort_op_ipu.py | 9 +- .../tests/unittests/ipu/test_assign_op_ipu.py | 21 +- .../tests/unittests/ipu/test_avg_shard_ipu.py | 35 +- .../unittests/ipu/test_batch_norm_op_ipu.py | 16 +- .../ipu/test_binary_cross_entropy_op_ipu.py | 19 +- .../tests/unittests/ipu/test_bmm_op_ipu.py | 13 +- .../tests/unittests/ipu/test_cast_op_ipu.py | 24 +- .../tests/unittests/ipu/test_clip_op_ipu.py | 125 +- .../tests/unittests/ipu/test_concat_op_ipu.py | 18 +- .../ipu/test_conv2d_transpose_op_ipu.py | 33 +- .../tests/unittests/ipu/test_conv_op_ipu.py | 32 +- .../ipu/test_cross_entropy2_op_ipu.py | 46 +- .../tests/unittests/ipu/test_cumsum_op_ipu.py | 24 +- .../unittests/ipu/test_data_norm_op_ipu.py | 30 +- .../tests/unittests/ipu/test_dist_op_ipu.py | 20 +- .../tests/unittests/ipu/test_dot_op_ipu.py | 14 +- .../unittests/ipu/test_dropout_op_ipu.py | 15 +- .../unittests/ipu/test_dy2static_fp16_ipu.py | 36 +- .../tests/unittests/ipu/test_dy2static_ipu.py | 133 +- .../unittests/ipu/test_elemetwise_x_op_ipu.py | 20 +- .../tests/unittests/ipu/test_equal_op_ipu.py | 15 +- .../unittests/ipu/test_eval_model_ipu.py | 55 +- .../unittests/ipu/test_expand_as_v2_op_ipu.py | 27 +- .../tests/unittests/ipu/test_expand_op_ipu.py | 23 +- .../unittests/ipu/test_expand_v2_op_ipu.py | 17 +- .../ipu/test_fill_any_like_op_ipu.py | 14 +- .../ipu/test_fill_constant_op_ipu.py | 2 - .../test_flatten_contiguous_range_op_ipu.py | 10 +- .../unittests/ipu/test_flatten_op_ipu.py | 9 +- .../tests/unittests/ipu/test_flip_op_ipu.py | 11 +- .../unittests/ipu/test_fp16_support_ipu.py | 41 +- .../tests/unittests/ipu/test_gather_op_ipu.py | 14 +- .../tests/unittests/ipu/test_gelu_op_ipu.py | 8 +- .../unittests/ipu/test_gradient_clip_ipu.py | 39 +- .../unittests/ipu/test_greater_op_ipu.py | 17 +- .../unittests/ipu/test_groupnorm_op_ipu.py | 31 +- .../unittests/ipu/test_huber_loss_op_ipu.py | 19 +- .../unittests/ipu/test_identity_loss_ipu.py | 30 +- .../ipu/test_inference_model_io_ipu.py | 59 +- .../unittests/ipu/test_instancenorm_op_ipu.py | 29 +- .../unittests/ipu/test_interpolate_ops_ipu.py | 25 +- .../unittests/ipu/test_ipu_shard_api_ipu.py | 16 +- .../unittests/ipu/test_ipu_strategy_ipu.py | 23 +- .../unittests/ipu/test_kldiv_loss_op_ipu.py | 19 +- .../unittests/ipu/test_layernorm_op_ipu.py | 42 +- .../unittests/ipu/test_logical_not_op_ipu.py | 7 +- .../unittests/ipu/test_logical_x_op_ipu.py | 18 +- .../unittests/ipu/test_lookuptable_op_ipu.py | 10 +- .../ipu/test_lookuptable_v2_op_ipu.py | 10 +- .../unittests/ipu/test_lr_sheduler_ipu.py | 26 +- .../ipu/test_margin_rank_loss_op_ipu.py | 20 +- .../tests/unittests/ipu/test_matmul_op_ipu.py | 26 +- .../unittests/ipu/test_matmul_serilize_ipu.py | 28 +- .../unittests/ipu/test_matmul_v2_op_ipu.py | 23 +- .../tests/unittests/ipu/test_mean_op_ipu.py | 7 +- .../unittests/ipu/test_meshgrid_op_ipu.py | 55 +- .../ipu/test_mixed_precision_inference_ipu.py | 43 +- .../ipu/test_mixed_precision_training_ipu.py | 43 +- .../unittests/ipu/test_model_parallel_ipu.py | 123 +- .../unittests/ipu/test_model_pipeline_ipu.py | 32 +- .../tests/unittests/ipu/test_mul_op_ipu.py | 15 +- .../unittests/ipu/test_not_equal_op_ipu.py | 24 +- .../unittests/ipu/test_one_hot_op_ipu.py | 8 +- .../unittests/ipu/test_one_hot_v2_op_ipu.py | 8 +- .../tests/unittests/ipu/test_optimizer_ipu.py | 70 +- .../tests/unittests/ipu/test_p_norm_op_ipu.py | 9 +- .../tests/unittests/ipu/test_pad_op_ipu.py | 22 +- .../unittests/ipu/test_pool_avg_op_ipu.py | 27 +- .../unittests/ipu/test_pool_max_op_ipu.py | 27 +- .../tests/unittests/ipu/test_pow_op_ipu.py | 24 +- .../tests/unittests/ipu/test_prelu_op_ipu.py | 14 +- .../tests/unittests/ipu/test_print_op_ipu.py | 38 +- .../unittests/ipu/test_rank_loss_op_ipu.py | 19 +- .../unittests/ipu/test_reduce_x_op_ipu.py | 26 +- .../ipu/test_reshape_inplace_op_ipu.py | 8 +- .../unittests/ipu/test_reshape_op_ipu.py | 9 +- .../tests/unittests/ipu/test_save_load_ipu.py | 78 +- .../tests/unittests/ipu/test_scale_op_ipu.py | 24 +- .../ipu/test_scaled_optimizer_state_ipu.py | 57 +- .../unittests/ipu/test_set_batch_size_ipu.py | 40 +- .../unittests/ipu/test_set_ipu_shard_api.py | 129 +- .../unittests/ipu/test_share_data_op_ipu.py | 38 +- ...igmoid_cross_entropy_with_logits_op_ipu.py | 22 +- .../tests/unittests/ipu/test_slice_op_ipu.py | 38 +- .../unittests/ipu/test_softmax_op_ipu.py | 8 +- .../test_softmax_with_cross_entropy_op_ipu.py | 63 +- .../tests/unittests/ipu/test_split_op_ipu.py | 8 +- .../unittests/ipu/test_squeeze_op_ipu.py | 9 +- .../tests/unittests/ipu/test_stack_op_ipu.py | 24 +- .../ipu/test_strided_slice_op_ipu.py | 45 +- .../tests/unittests/ipu/test_sum_op_ipu.py | 36 +- .../tests/unittests/ipu/test_tile_op_ipu.py | 21 +- .../tests/unittests/ipu/test_topk_op_ipu.py | 17 +- .../unittests/ipu/test_transpose_op_ipu.py | 10 +- .../tests/unittests/ipu/test_unary_ops_ipu.py | 29 +- .../unittests/ipu/test_unsqueeze_op_ipu.py | 9 +- .../unittests/ipu/test_unstack_op_ipu.py | 9 +- .../unittests/ipu/test_varname_inplace_ipu.py | 23 +- .../unittests/ipu/test_warpctc_op_ipu.py | 67 +- .../unittests/ipu/test_weight_decay_ipu.py | 35 +- .../unittests/ipu/test_weight_sharing_ipu.py | 41 +- .../tests/unittests/ipu/test_where_op_ipu.py | 17 +- .../unittests/ipu/test_yolo_box_op_ipu.py | 9 +- .../unittests/ir/inference/auto_scan_test.py | 485 +- .../ir/inference/inference_pass_test.py | 141 +- .../unittests/ir/inference/program_config.py | 248 +- .../ir/inference/quant_dequant_test.py | 202 +- ...ive_pool2d_convert_global_pass_autoscan.py | 80 +- .../test_conv_act_mkldnn_fuse_pass.py | 147 +- .../test_conv_bias_mkldnn_fuse_pass.py | 117 +- .../ir/inference/test_conv_bn_fuse_pass.py | 160 +- ...est_conv_elementwise_add2_act_fuse_pass.py | 266 +- ...test_conv_elementwise_add_act_fuse_pass.py | 135 +- .../test_conv_elementwise_add_fuse_pass.py | 92 +- .../test_conv_eltwiseadd_bn_fuse_pass.py | 208 +- .../test_conv_transpose_bn_fuse_pass.py | 161 +- ..._conv_transpose_eltwiseadd_bn_fuse_pass.py | 235 +- .../test_delete_c_identity_op_pass.py | 29 +- .../test_emb_eltwise_layernorm_fuse_pass.py | 335 +- ...test_fc_elementwise_layernorm_fuse_pass.py | 92 +- .../ir/inference/test_fc_fuse_pass.py | 55 +- .../ir/inference/test_fc_gru_fuse_pass.py | 46 +- .../ir/inference/test_fc_lstm_fuse_pass.py | 16 +- .../test_flatten2_matmul_fuse_pass.py | 39 +- .../test_identity_scale_clean_pass.py | 32 +- .../ir/inference/test_layer_norm_fuse_pass.py | 75 +- .../test_layernorm_shift_partition_pass.py | 574 +- .../inference/test_map_matmul_to_mul_pass.py | 41 +- .../test_map_matmul_v2_to_matmul_pass.py | 41 +- .../test_map_matmul_v2_to_mul_pass.py | 39 +- .../inference/test_matmul_scale_fuse_pass.py | 38 +- .../test_matmul_v2_scale_fuse_pass.py | 40 +- .../test_merge_layernorm_fuse_pass.py | 219 +- .../test_mkldnn_batch_norm_act_fuse_pass.py | 74 +- .../test_mkldnn_conv3d_bias_fuse_pass.py | 107 +- .../ir/inference/test_mkldnn_conv3d_op.py | 61 +- .../test_mkldnn_conv_activation_fuse_pass.py | 26 +- ...st_mkldnn_conv_affine_channel_fuse_pass.py | 119 +- .../test_mkldnn_conv_bias_fuse_pass.py | 164 +- ...kldnn_conv_concat_relu_mkldnn_fuse_pass.py | 172 +- ...t_mkldnn_conv_elementwise_add_fuse_pass.py | 111 +- .../test_mkldnn_conv_gelu_fuse_pass.py | 77 +- ...test_mkldnn_conv_hard_sigmoid_fuse_pass.py | 76 +- .../test_mkldnn_conv_hard_swish_fuse_pass.py | 81 +- .../test_mkldnn_conv_mish_fuse_pass.py | 73 +- ...st_mkldnn_conv_transpose_bias_fuse_pass.py | 80 +- .../test_mkldnn_cpu_bfloat16_pass.py | 12 +- .../test_mkldnn_depthwise_conv_pass.py | 76 +- .../test_mkldnn_elt_act_fuse_pass.py | 154 +- .../test_mkldnn_elt_act_fuse_pass_new.py | 47 +- .../inference/test_mkldnn_fc_act_fuse_pass.py | 42 +- ...est_mkldnn_fc_elementwise_add_fuse_pass.py | 59 +- .../test_mkldnn_inplace_fuse_pass.py | 20 +- ...test_mkldnn_int8_scale_calculation_pass.py | 103 +- .../inference/test_mkldnn_log_softmax_op.py | 30 +- ...test_mkldnn_matmul_activation_fuse_pass.py | 105 +- ...ul_elementwise_add_activation_fuse_pass.py | 135 +- ...mkldnn_matmul_elementwise_add_fuse_pass.py | 54 +- .../test_mkldnn_matmul_op_output_fuse_pass.py | 53 +- ...ldnn_matmul_transpose_reshape_fuse_pass.py | 83 +- ...t_mkldnn_matmul_v2_activation_fuse_pass.py | 100 +- ...dnn_matmul_v2_elementwise_add_fuse_pass.py | 49 +- ...n_matmul_v2_transpose_reshape_fuse_pass.py | 105 +- .../ir/inference/test_mkldnn_matmulv2_op.py | 91 +- .../ir/inference/test_mkldnn_mish_op.py | 44 +- .../ir/inference/test_mkldnn_pad2d_op.py | 42 +- .../ir/inference/test_mkldnn_pad3d_op.py | 55 +- .../ir/inference/test_mkldnn_prelu_op.py | 58 +- ...ldnn_reshape_transpose_matmul_fuse_pass.py | 133 +- ...n_reshape_transpose_matmul_v2_fuse_pass.py | 26 +- .../test_mkldnn_scale_matmul_fuse_pass.py | 146 +- .../ir/inference/test_mkldnn_shape_op.py | 31 +- ...test_mkldnn_shuffle_channel_detect_pass.py | 99 +- .../test_mkldnn_shuffle_channel_op.py | 26 +- ...st_mkldnn_softplus_activation_fuse_pass.py | 61 +- .../ir/inference/test_mul_gru_fuse_pass.py | 141 +- .../ir/inference/test_mul_lstm_fuse_pass.py | 110 +- .../test_multihead_matmul_fuse_pass_v3.py | 330 +- .../test_onednn_fc_activation_fuse_pass.py | 127 +- .../test_repeated_fc_relu_fuse_pass.py | 128 +- .../test_reshape2_matmul_fuse_pass.py | 39 +- .../inference/test_seq_concat_fc_fuse_pass.py | 131 +- .../test_seqconv_eltadd_relu_fuse_pass.py | 104 +- .../test_seqpool_cvm_concat_fuse_pass_py.py | 167 +- .../test_shuffle_channel_detect_pass.py | 74 +- ...t_simplify_with_basic_ops_pass_autoscan.py | 136 +- .../test_squared_mat_sub_fuse_pass.py | 200 +- .../test_squeeze2_matmul_fuse_pass.py | 39 +- ...test_transpose_flatten_concat_fuse_pass.py | 26 +- .../ir/inference/test_trt_activation_pass.py | 220 +- .../inference/test_trt_affine_channel_op.py | 39 +- .../inference/test_trt_anchor_generator_op.py | 53 +- .../test_trt_c_allreduce_infer_script.py | 53 +- .../ir/inference/test_trt_conv3d_op.py | 107 +- .../inference/test_trt_conv3d_transpose_op.py | 58 +- .../ir/inference/test_trt_conv_pass.py | 150 +- .../test_trt_conv_quant_dequant_pass.py | 172 +- .../inference/test_trt_convert_activation.py | 61 +- .../test_trt_convert_affine_channel.py | 65 +- .../test_trt_convert_anchor_generator.py | 78 +- .../ir/inference/test_trt_convert_arg_max.py | 55 +- .../inference/test_trt_convert_batch_norm.py | 195 +- .../test_trt_convert_bilinear_interp_v2.py | 86 +- .../ir/inference/test_trt_convert_bmm.py | 63 +- .../inference/test_trt_convert_c_allreduce.py | 10 +- .../ir/inference/test_trt_convert_cast.py | 50 +- .../ir/inference/test_trt_convert_clip.py | 76 +- .../ir/inference/test_trt_convert_concat.py | 177 +- .../ir/inference/test_trt_convert_conv2d.py | 127 +- .../test_trt_convert_conv2d_fusion.py | 133 +- .../test_trt_convert_conv2d_transpose.py | 218 +- .../test_trt_convert_conv3d_transpose.py | 77 +- .../test_trt_convert_deformable_conv.py | 184 +- .../test_trt_convert_depthwise_conv2d.py | 126 +- ..._trt_convert_depthwise_conv2d_transpose.py | 114 +- .../ir/inference/test_trt_convert_dropout.py | 76 +- .../inference/test_trt_convert_elementwise.py | 438 +- .../test_trt_convert_emb_eltwise_layernorm.py | 303 +- .../ir/inference/test_trt_convert_equal.py | 84 +- .../inference/test_trt_convert_expand_v2.py | 115 +- .../ir/inference/test_trt_convert_fc.py | 135 +- .../test_trt_convert_fill_constant.py | 84 +- .../ir/inference/test_trt_convert_flatten.py | 180 +- ...st_trt_convert_flatten_contiguous_range.py | 49 +- .../test_trt_convert_fused_token_prune.py | 80 +- .../ir/inference/test_trt_convert_gather.py | 127 +- .../inference/test_trt_convert_gather_nd.py | 256 +- .../ir/inference/test_trt_convert_gelu.py | 45 +- .../test_trt_convert_grid_sampler.py | 47 +- .../inference/test_trt_convert_group_norm.py | 81 +- .../test_trt_convert_hard_sigmoid.py | 35 +- .../inference/test_trt_convert_hard_swish.py | 59 +- .../test_trt_convert_instance_norm.py | 89 +- .../ir/inference/test_trt_convert_inverse.py | 34 +- .../inference/test_trt_convert_layer_norm.py | 168 +- .../inference/test_trt_convert_leaky_relu.py | 55 +- .../ir/inference/test_trt_convert_matmul.py | 144 +- .../inference/test_trt_convert_matmul_v2.py | 217 +- .../ir/inference/test_trt_convert_mish.py | 57 +- .../test_trt_convert_multiclass_nms.py | 138 +- .../test_trt_convert_multiclass_nms3.py | 147 +- .../test_trt_convert_multihead_matmul.py | 1070 ++- .../test_trt_convert_nearest_interp.py | 88 +- .../test_trt_convert_nearest_interp_v2.py | 55 +- .../ir/inference/test_trt_convert_pad.py | 59 +- .../ir/inference/test_trt_convert_pad3d.py | 40 +- .../ir/inference/test_trt_convert_pool2d.py | 120 +- .../test_trt_convert_preln_residual_bias.py | 134 +- ...test_trt_convert_preln_residual_no_bias.py | 109 +- .../ir/inference/test_trt_convert_prelu.py | 112 +- .../inference/test_trt_convert_reduce_mean.py | 77 +- .../inference/test_trt_convert_reduce_sum.py | 74 +- .../ir/inference/test_trt_convert_reshape.py | 140 +- .../ir/inference/test_trt_convert_rnn.py | 213 +- .../inference/test_trt_convert_roi_align.py | 145 +- .../ir/inference/test_trt_convert_roll.py | 56 +- .../ir/inference/test_trt_convert_scale.py | 106 +- .../test_trt_convert_scatter_nd_add.py | 48 +- .../ir/inference/test_trt_convert_shape.py | 40 +- .../test_trt_convert_shuffle_channel.py | 51 +- .../ir/inference/test_trt_convert_silu.py | 47 +- .../test_trt_convert_skip_layernorm.py | 110 +- .../ir/inference/test_trt_convert_slice.py | 79 +- .../ir/inference/test_trt_convert_softmax.py | 47 +- .../ir/inference/test_trt_convert_split.py | 162 +- .../ir/inference/test_trt_convert_squeeze2.py | 57 +- .../ir/inference/test_trt_convert_stack.py | 90 +- .../test_trt_convert_strided_slice.py | 129 +- .../ir/inference/test_trt_convert_sum.py | 114 +- .../ir/inference/test_trt_convert_swish.py | 47 +- .../ir/inference/test_trt_convert_tile.py | 44 +- .../ir/inference/test_trt_convert_top_k.py | 50 +- .../ir/inference/test_trt_convert_top_k_v2.py | 67 +- .../inference/test_trt_convert_transpose.py | 60 +- .../ir/inference/test_trt_convert_unary.py | 65 +- .../ir/inference/test_trt_convert_unfold.py | 42 +- .../inference/test_trt_convert_unsqueeze2.py | 51 +- .../ir/inference/test_trt_convert_yolo_box.py | 126 +- .../test_trt_convert_yolo_box_head.py | 48 +- .../ir/inference/test_trt_deformable_conv.py | 44 +- .../ir/inference/test_trt_dynamic_shape.py | 35 +- .../ir/inference/test_trt_elementwise_op.py | 35 +- .../ir/inference/test_trt_fc_fuse_pass.py | 197 +- .../test_trt_fc_fuse_quant_dequant_pass.py | 213 +- .../test_trt_flatten2_matmul_fuse_pass.py | 40 +- .../ir/inference/test_trt_flatten_op.py | 41 +- .../ir/inference/test_trt_gather_nd_op.py | 55 +- .../ir/inference/test_trt_gather_op.py | 48 +- .../ir/inference/test_trt_inspector.py | 29 +- .../ir/inference/test_trt_instance_norm_op.py | 15 +- .../unittests/ir/inference/test_trt_matmul.py | 74 +- .../test_trt_matmul_quant_dequant.py | 210 +- .../inference/test_trt_multiclass_nms3_op.py | 154 +- .../inference/test_trt_multiclass_nms_op.py | 74 +- .../inference/test_trt_nearest_interp_op.py | 66 +- .../test_trt_nearest_interp_v2_op.py | 66 +- .../unittests/ir/inference/test_trt_pad_op.py | 16 +- .../ir/inference/test_trt_pool3d_op.py | 233 +- .../ir/inference/test_trt_pool_op.py | 78 +- .../ir/inference/test_trt_reduce_mean_op.py | 168 +- .../ir/inference/test_trt_reduce_sum_op.py | 46 +- .../ir/inference/test_trt_reshape_op.py | 67 +- .../ir/inference/test_trt_roi_align_op.py | 42 +- .../ir/inference/test_trt_scale_op.py | 34 +- .../test_trt_shuffle_channel_detect_pass.py | 13 +- .../test_trt_slice_dynamic_plugin.py | 54 +- .../ir/inference/test_trt_slice_plugin.py | 52 +- .../test_trt_squeeze2_matmul_fuse_pass.py | 40 +- .../ir/inference/test_trt_subgraph_pass.py | 287 +- .../ir/inference/test_trt_tile_op.py | 34 +- ..._trt_transpose_flatten_concat_fuse_pass.py | 22 +- .../inference/test_trt_tuned_dynamic_shape.py | 37 +- .../ir/inference/test_trt_yolo_box_op.py | 107 +- .../test_unsqueeze2_eltwise_fuse_pass.py | 30 +- .../ir/inference/test_yolo_box_post.py | 80 +- .../fluid/tests/unittests/ir/pass_test.py | 60 +- .../ir/test_convert_to_mixed_precision.py | 78 +- .../unittests/ir/test_fuse_resnet_unit.py | 43 +- ...r_embedding_eltwise_layernorm_fuse_pass.py | 159 +- .../unittests/ir/test_ir_fc_fuse_pass.py | 15 +- .../unittests/ir/test_ir_fusion_group_pass.py | 33 +- .../unittests/ir/test_ir_generate_pass.py | 77 +- .../ir/test_ir_graph_to_program_pass.py | 17 +- .../test_ir_preln_residual_bias_fuse_pass.py | 40 +- .../ir/test_ir_skip_layernorm_pass.py | 17 +- .../ir/test_ir_subgraph_python_interface.py | 19 +- .../unittests/ir/test_ir_yolo_box_pass.py | 61 +- .../unittests/ir_memory_optimize_net_base.py | 73 +- .../tests/unittests/jit_load_rename_var.py | 5 +- .../tests/unittests/launch_function_helper.py | 1 - .../mkldnn/check_flags_mkldnn_ops_on_off.py | 24 +- .../mkldnn/check_flags_use_mkldnn.py | 14 +- .../tests/unittests/mkldnn/mkldnn_op_test.py | 92 +- .../mkldnn/test_activation_bf16_mkldnn_op.py | 63 +- .../mkldnn/test_activation_mkldnn_op.py | 67 +- .../mkldnn/test_batch_norm_mkldnn_op.py | 83 +- .../mkldnn/test_bilinear_interp_mkldnn_op.py | 56 +- .../test_bilinear_interp_v2_mkldnn_op.py | 41 +- .../unittests/mkldnn/test_cast_mkldnn_op.py | 24 +- .../unittests/mkldnn/test_clip_mkldnn_op.py | 38 +- .../mkldnn/test_concat_bf16_mkldnn_op.py | 33 +- .../mkldnn/test_concat_int8_mkldnn_op.py | 23 +- .../unittests/mkldnn/test_concat_mkldnn_op.py | 11 +- .../mkldnn/test_conv2d_bf16_mkldnn_op.py | 103 +- .../mkldnn/test_conv2d_int8_mkldnn_op.py | 136 +- .../unittests/mkldnn/test_conv2d_mkldnn_op.py | 48 +- .../test_conv2d_transpose_bf16_mkldnn_op.py | 35 +- .../mkldnn/test_conv2d_transpose_mkldnn_op.py | 18 +- .../unittests/mkldnn/test_conv3d_mkldnn_op.py | 19 +- .../mkldnn/test_dequantize_mkldnn_op.py | 23 +- .../test_elementwise_add_bf16_mkldnn_op.py | 74 +- .../mkldnn/test_elementwise_add_mkldnn_op.py | 35 +- .../mkldnn/test_elementwise_div_mkldnn_op.py | 65 +- .../test_elementwise_mul_bf16_mkldnn_op.py | 55 +- .../mkldnn/test_elementwise_mul_mkldnn_op.py | 32 +- .../mkldnn/test_elementwise_sub_mkldnn_op.py | 85 +- .../mkldnn/test_expand_v2_mkldnn_op.py | 35 +- .../mkldnn/test_fc_bf16_mkldnn_op.py | 17 +- .../mkldnn/test_fc_int8_mkldnn_op.py | 8 +- .../unittests/mkldnn/test_fc_mkldnn_op.py | 12 +- .../mkldnn/test_fill_constant_mkldnn_op.py | 32 +- .../mkldnn/test_flags_mkldnn_ops_on_off.py | 21 +- .../unittests/mkldnn/test_flags_use_mkldnn.py | 11 +- .../mkldnn/test_flatten_mkldnn_op.py | 37 +- .../mkldnn/test_fusion_gru_bf16_mkldnn_op.py | 51 +- .../mkldnn/test_fusion_gru_int8_mkldnn_op.py | 88 +- .../mkldnn/test_fusion_gru_mkldnn_op.py | 10 +- .../mkldnn/test_fusion_lstm_bf16_mkldnn_op.py | 53 +- .../mkldnn/test_fusion_lstm_int8_mkldnn_op.py | 58 +- .../mkldnn/test_fusion_lstm_mkldnn_op.py | 9 +- .../mkldnn/test_gaussian_random_mkldnn_op.py | 8 +- .../mkldnn/test_layer_norm_bf16_mkldnn_op.py | 75 +- .../mkldnn/test_layer_norm_mkldnn_op.py | 72 +- .../mkldnn/test_log_softmax_mkldnn_op.py | 19 +- .../unittests/mkldnn/test_lrn_mkldnn_op.py | 28 +- .../mkldnn/test_matmul_bf16_mkldnn_op.py | 55 +- .../unittests/mkldnn/test_matmul_mkldnn_op.py | 175 +- .../mkldnn/test_matmul_v2_mkldnn_op.py | 215 +- .../mkldnn/test_mul_int8_mkldnn_op.py | 29 +- .../unittests/mkldnn/test_mul_mkldnn_op.py | 35 +- .../mkldnn/test_multi_gru_mkldnn_op.py | 161 +- .../mkldnn/test_nearest_interp_mkldnn_op.py | 49 +- .../test_nearest_interp_v2_mkldnn_op.py | 48 +- ...st_onnx_format_quantization_mobilenetv1.py | 238 +- .../mkldnn/test_pool2d_bf16_mkldnn_op.py | 92 +- .../mkldnn/test_pool2d_int8_mkldnn_op.py | 44 +- .../unittests/mkldnn/test_pool2d_mkldnn_op.py | 32 +- .../unittests/mkldnn/test_prelu_mkldnn_op.py | 50 +- .../mkldnn/test_quantize_mkldnn_op.py | 32 +- .../mkldnn/test_reduce_bf16_mkldnn_op.py | 52 +- .../unittests/mkldnn/test_reduce_mkldnn_op.py | 49 +- .../mkldnn/test_requantize_mkldnn_op.py | 146 +- .../unittests/mkldnn/test_reshape_bf16_op.py | 31 +- .../mkldnn/test_reshape_mkldnn_op.py | 73 +- .../mkldnn/test_scale_bf16_mkldnn_op.py | 31 +- .../unittests/mkldnn/test_scale_mkldnn_op.py | 14 +- .../unittests/mkldnn/test_shape_mkldnn_op.py | 4 - .../mkldnn/test_shuffle_channel_mkldnn_op.py | 8 +- .../unittests/mkldnn/test_slice_mkldnn_op.py | 45 +- .../mkldnn/test_softmax_bf16_mkldnn_op.py | 25 +- .../mkldnn/test_softmax_mkldnn_op.py | 45 +- .../mkldnn/test_softplus_mkldnn_op.py | 24 +- .../mkldnn/test_split_bf16_mkldnn_op.py | 36 +- .../unittests/mkldnn/test_split_mkldnn_op.py | 23 +- .../mkldnn/test_squeeze2_mkldnn_op.py | 36 +- .../unittests/mkldnn/test_stack_mkldnn_op.py | 15 +- .../mkldnn/test_sum_bf16_mkldnn_op.py | 6 +- .../unittests/mkldnn/test_sum_mkldnn_op.py | 12 +- .../mkldnn/test_transpose_bf16_mkldnn_op.py | 14 +- .../mkldnn/test_transpose_int8_mkldnn_op.py | 25 +- .../mkldnn/test_transpose_mkldnn_op.py | 13 +- .../tests/unittests/mlu/c_comm_init_op_mlu.py | 46 +- .../unittests/mlu/collective_allgather_api.py | 12 +- .../unittests/mlu/collective_allgather_op.py | 33 +- .../unittests/mlu/collective_allreduce_api.py | 12 +- .../unittests/mlu/collective_allreduce_op.py | 30 +- .../unittests/mlu/collective_broadcast_api.py | 12 +- .../unittests/mlu/collective_broadcast_op.py | 33 +- .../unittests/mlu/collective_reduce_api.py | 12 +- .../unittests/mlu/collective_reduce_op.py | 33 +- .../tests/unittests/mlu/multi_process_mlu.py | 41 +- .../tests/unittests/mlu/nproc_process_mlu.py | 9 +- .../mlu/parallel_dygraph_sync_batch_norm.py | 74 +- .../unittests/mlu/sync_batch_norm_op_mlu.py | 40 +- .../tests/unittests/mlu/test_abs_op_mlu.py | 12 +- .../unittests/mlu/test_accuracy_op_mlu.py | 58 +- .../tests/unittests/mlu/test_adam_op_mlu.py | 47 +- .../tests/unittests/mlu/test_adamw_op_mlu.py | 40 +- .../test_amp_check_finite_and_scale_op_mlu.py | 6 - .../unittests/mlu/test_arg_max_op_mlu.py | 76 +- .../unittests/mlu/test_argsort_op_mlu.py | 8 +- .../tests/unittests/mlu/test_assign_op_mlu.py | 1 - .../unittests/mlu/test_assign_value_op_mlu.py | 12 +- .../unittests/mlu/test_batch_norm_op_mlu.py | 346 +- .../mlu/test_batch_norm_op_mlu_v2.py | 56 +- .../tests/unittests/mlu/test_bce_loss_mlu.py | 246 +- .../mlu/test_bce_with_logits_loss_mlu.py | 254 +- .../mlu/test_bilinear_interp_v2_op_mlu.py | 252 +- .../tests/unittests/mlu/test_cast_op_mlu.py | 24 +- .../tests/unittests/mlu/test_clip_op_mlu.py | 7 - .../mlu/test_coalesce_tensor_op_mlu.py | 32 +- .../mlu/test_collective_allgather.py | 26 +- .../mlu/test_collective_allgather_api_mlu.py | 16 +- .../mlu/test_collective_allreduce_api_mlu.py | 16 +- .../mlu/test_collective_allreduce_max.py | 31 +- .../mlu/test_collective_allreduce_min.py | 31 +- .../mlu/test_collective_allreduce_prod.py | 31 +- .../mlu/test_collective_allreduce_sum.py | 31 +- .../mlu/test_collective_api_base_mlu.py | 102 +- .../unittests/mlu/test_collective_base_mlu.py | 206 +- .../mlu/test_collective_broadcast.py | 26 +- .../mlu/test_collective_broadcast_api_mlu.py | 16 +- .../mlu/test_collective_reduce_api_mlu.py | 1 - .../mlu/test_collective_reduce_max.py | 11 +- .../mlu/test_collective_reduce_min.py | 11 +- .../mlu/test_collective_reduce_prod.py | 11 +- .../mlu/test_collective_reduce_sum.py | 11 +- .../unittests/mlu/test_compare_op_mlu.py | 50 +- .../tests/unittests/mlu/test_concat_op_mlu.py | 44 +- .../mlu/test_conv2d_op_depthwise_conv_mlu.py | 23 +- .../tests/unittests/mlu/test_conv2d_op_mlu.py | 132 +- .../mlu/test_conv2d_transposed_op_mlu.py | 319 +- .../tests/unittests/mlu/test_cumsum_op_mlu.py | 86 +- .../mlu/test_deformable_conv_op_mlu.py | 117 +- .../unittests/mlu/test_dropout_op_mlu.py | 114 +- .../mlu/test_elementwise_add_op_mlu.py | 116 +- .../mlu/test_elementwise_div_op_mlu.py | 84 +- .../mlu/test_elementwise_max_op_mlu.py | 174 +- .../mlu/test_elementwise_min_op_mlu.py | 89 +- .../mlu/test_elementwise_mul_op_mlu.py | 63 +- .../mlu/test_elementwise_pow_op_mlu.py | 125 +- .../mlu/test_elementwise_sub_op_mlu.py | 60 +- .../tests/unittests/mlu/test_exp_op_mlu.py | 3 - .../unittests/mlu/test_expand_as_v2_op_mlu.py | 45 +- .../unittests/mlu/test_expand_v2_op_mlu.py | 65 +- .../mlu/test_fill_any_like_op_mlu.py | 8 - ...st_fill_constant_batch_size_like_op_mlu.py | 46 +- .../mlu/test_fill_constant_op_mlu.py | 284 +- .../unittests/mlu/test_flatten2_op_mlu.py | 20 +- .../test_flatten_contigous_range_op_mlu.py | 109 +- .../unittests/mlu/test_flatten_op_mlu.py | 4 - .../tests/unittests/mlu/test_floor_op_mlu.py | 2 - .../unittests/mlu/test_gather_nd_op_mlu.py | 43 +- .../tests/unittests/mlu/test_gather_op_mlu.py | 28 +- .../mlu/test_gaussian_random_op_mlu.py | 4 +- .../tests/unittests/mlu/test_gelu_op_mlu.py | 34 +- .../unittests/mlu/test_grid_sampler_op_mlu.py | 80 +- .../unittests/mlu/test_hard_sigmoid_op_mlu.py | 32 +- .../unittests/mlu/test_hard_swish_op_mlu.py | 74 +- .../unittests/mlu/test_huber_loss_op_mlu.py | 48 +- .../mlu/test_iou_similarity_op_mlu.py | 10 +- .../unittests/mlu/test_label_smooth_op_mlu.py | 23 +- .../unittests/mlu/test_layer_norm_op_mlu.py | 279 +- .../unittests/mlu/test_leaky_relu_op_mlu.py | 35 +- .../tests/unittests/mlu/test_log_op_mlu.py | 59 +- .../unittests/mlu/test_log_softmax_op_mlu.py | 34 +- .../unittests/mlu/test_logical_op_mlu.py | 140 +- .../mlu/test_lookup_table_v2_op_mlu.py | 20 +- .../mlu/test_masked_select_op_mlu.py | 37 +- .../tests/unittests/mlu/test_matmul_op_mlu.py | 38 +- .../unittests/mlu/test_matmul_v2_op_mlu.py | 45 +- .../tests/unittests/mlu/test_mean_op_mlu.py | 2 - .../unittests/mlu/test_merged_adam_op_mlu.py | 167 +- .../mlu/test_merged_momentum_op_mlu.py | 295 +- .../unittests/mlu/test_meshgrid_op_mlu.py | 132 +- .../unittests/mlu/test_momentum_op_mlu.py | 252 +- .../mlu/test_nearest_interp_v2_op_mlu.py | 253 +- .../unittests/mlu/test_one_hot_v2_op_mlu.py | 88 +- ...st_parallel_dygraph_sync_batch_norm_mlu.py | 88 +- .../tests/unittests/mlu/test_pool2d_op_mlu.py | 694 +- .../unittests/mlu/test_prior_box_op_mlu.py | 83 +- .../unittests/mlu/test_randperm_op_mlu.py | 170 +- .../tests/unittests/mlu/test_range_op_mlu.py | 13 +- .../unittests/mlu/test_reciprocal_op_mlu.py | 8 +- .../unittests/mlu/test_reduce_max_op_mlu.py | 41 +- .../unittests/mlu/test_reduce_mean_op_mlu.py | 26 +- .../unittests/mlu/test_reduce_min_op_mlu.py | 41 +- .../unittests/mlu/test_reduce_prod_op_mlu.py | 8 +- .../unittests/mlu/test_reduce_sum_op_mlu.py | 50 +- .../tests/unittests/mlu/test_relu6_op_mlu.py | 29 +- .../tests/unittests/mlu/test_relu_op_mlu.py | 29 +- .../unittests/mlu/test_reshape2_op_mlu.py | 5 +- .../tests/unittests/mlu/test_rnn_op_mlu.py | 69 +- .../unittests/mlu/test_roi_align_op_mlu.py | 124 +- .../tests/unittests/mlu/test_scale_op_mlu.py | 28 +- .../unittests/mlu/test_scatter_op_mlu.py | 77 +- .../unittests/mlu/test_set_value_op_mlu.py | 85 +- .../tests/unittests/mlu/test_shape_op_mlu.py | 9 +- ...igmoid_cross_entropy_with_logits_op_mlu.py | 85 +- .../unittests/mlu/test_sigmoid_op_mlu.py | 8 +- .../tests/unittests/mlu/test_size_op_mlu.py | 34 +- .../tests/unittests/mlu/test_slice_op_mlu.py | 160 +- .../unittests/mlu/test_softmax_op_mlu.py | 30 +- .../test_softmax_with_cross_entropy_op_mlu.py | 51 +- .../tests/unittests/mlu/test_spawn_mlu.py | 8 +- .../tests/unittests/mlu/test_split_op_mlu.py | 32 +- .../tests/unittests/mlu/test_sqrt_op_mlu.py | 9 +- .../mlu/test_squared_l2_norm_op_mlu.py | 10 +- .../unittests/mlu/test_squeeze2_op_mlu.py | 6 +- .../unittests/mlu/test_squeeze_op_mlu.py | 6 - .../tests/unittests/mlu/test_stack_op_mlu.py | 25 +- .../mlu/test_strided_slice_op_mlu.py | 233 +- .../tests/unittests/mlu/test_sum_op_mlu.py | 11 +- .../mlu/test_sync_batch_norm_base_mlu.py | 256 +- .../test_sync_batch_norm_op_mlu_baseline.py | 7 +- .../mlu/test_sync_batch_norm_op_mlu_extra.py | 55 +- .../tests/unittests/mlu/test_tanh_op_mlu.py | 28 +- .../tests/unittests/mlu/test_tile_op_mlu.py | 30 +- .../tests/unittests/mlu/test_top_k_op_mlu.py | 2 - .../unittests/mlu/test_top_k_v2_op_mlu.py | 115 +- .../unittests/mlu/test_transpose_op_mlu.py | 43 +- .../unittests/mlu/test_tril_triu_op_mlu.py | 64 +- .../test_truncated_gaussian_random_op_mlu.py | 24 +- .../mlu/test_uniform_random_op_mlu.py | 23 +- .../unittests/mlu/test_unsqueeze2_op_mlu.py | 36 +- .../unittests/mlu/test_unsqueeze_op_mlu.py | 7 +- .../unittests/mlu/test_unstack_op_mlu.py | 5 - .../unittests/mlu/test_where_index_op_mlu.py | 25 +- .../tests/unittests/mlu/test_where_op_mlu.py | 147 +- .../unittests/mlu/test_yolo_box_op_mlu.py | 195 +- .../fluid/tests/unittests/multi_process.py | 41 +- .../tests/unittests/my_data_generator.py | 10 +- .../fluid/tests/unittests/nproc_process.py | 9 +- .../npu/collective_identity_op_npu.py | 23 +- .../tests/unittests/npu/process_group_hccl.py | 32 +- .../unittests/npu/sync_batch_norm_op_npu.py | 49 +- .../tests/unittests/npu/test_abs_op_npu.py | 1 - .../unittests/npu/test_accuracy_op_npu.py | 12 +- .../tests/unittests/npu/test_adam_op_npu.py | 150 +- .../tests/unittests/npu/test_adamw_op_npu.py | 40 +- .../test_amp_check_finite_and_scale_op_npu.py | 71 +- .../unittests/npu/test_arg_max_op_npu.py | 74 +- .../unittests/npu/test_arg_min_op_npu.py | 60 +- .../unittests/npu/test_argsort_op_npu.py | 53 +- .../tests/unittests/npu/test_assign_op_npu.py | 1 - .../unittests/npu/test_assign_value_op_npu.py | 38 +- .../tests/unittests/npu/test_atan_op_npu.py | 5 +- .../unittests/npu/test_batch_norm_op_npu.py | 284 +- .../tests/unittests/npu/test_bce_loss_npu.py | 259 +- .../npu/test_beam_search_decode_op_npu.py | 36 +- .../unittests/npu/test_beam_search_op_npu.py | 101 +- .../npu/test_bilinear_interp_v2_op_npu.py | 82 +- .../unittests/npu/test_box_coder_op_npu.py | 86 +- .../unittests/npu/test_c_embedding_op_npu.py | 6 +- .../unittests/npu/test_c_identity_npu.py | 7 +- .../tests/unittests/npu/test_cast_op_npu.py | 9 +- .../unittests/npu/test_clip_by_norm_op_npu.py | 14 +- .../tests/unittests/npu/test_clip_op_npu.py | 30 +- .../npu/test_coalesce_tensor_op_npu.py | 9 +- .../unittests/npu/test_collective_base_npu.py | 131 +- .../npu/test_collective_process_group_hccl.py | 1 - .../unittests/npu/test_compare_op_npu.py | 50 +- .../tests/unittests/npu/test_concat_op_npu.py | 33 +- .../npu/test_conv2d_op_depthwise_conv_npu.py | 184 +- .../tests/unittests/npu/test_conv2d_op_npu.py | 163 +- .../npu/test_conv2d_transpose_op_npu.py | 206 +- .../tests/unittests/npu/test_conv3d_op_npu.py | 437 +- .../npu/test_conv3d_transpose_op_npu.py | 85 +- .../tests/unittests/npu/test_cos_op_npu.py | 28 +- .../tests/unittests/npu/test_crop_op_npu.py | 13 +- .../tests/unittests/npu/test_cumsum_op_npu.py | 113 +- .../npu/test_density_prior_box_op_npu.py | 77 +- .../unittests/npu/test_dropout_op_npu.py | 120 +- .../npu/test_elementwise_add_op_npu.py | 131 +- .../npu/test_elementwise_div_op_npu.py | 51 +- .../npu/test_elementwise_floordiv_op_npu.py | 4 +- .../npu/test_elementwise_max_op_npu.py | 154 +- .../npu/test_elementwise_min_op_npu.py | 89 +- .../npu/test_elementwise_mod_op_npu.py | 14 +- .../npu/test_elementwise_mul_op_npu.py | 57 +- .../npu/test_elementwise_pow_op_npu.py | 152 +- .../npu/test_elementwise_sub_op_npu.py | 74 +- .../tests/unittests/npu/test_exp_op_npu.py | 2 - .../unittests/npu/test_expand_as_v2_op_npu.py | 33 +- .../tests/unittests/npu/test_expand_op_npu.py | 23 +- .../unittests/npu/test_expand_v2_op_npu.py | 67 +- .../tests/unittests/npu/test_eye_op_npu.py | 28 +- .../npu/test_fill_any_like_op_npu.py | 8 - ...st_fill_constant_batch_size_like_op_npu.py | 26 +- .../npu/test_fill_constant_op_npu.py | 14 +- .../npu/test_fill_zeros_like_op_npu.py | 6 - .../npu/test_flags_check_nan_inf_npu.py | 1 - .../unittests/npu/test_flatten2_op_npu.py | 6 +- .../test_flatten_contiguous_range_op_npu.py | 107 +- .../unittests/npu/test_float_status_op_npu.py | 2 - .../unittests/npu/test_gather_nd_op_npu.py | 43 +- .../tests/unittests/npu/test_gather_op_npu.py | 53 +- .../npu/test_gaussian_random_op_npu.py | 18 +- .../tests/unittests/npu/test_gelu_op_npu.py | 34 +- .../unittests/npu/test_group_norm_op_npu.py | 62 +- .../unittests/npu/test_hard_sigmoid_op_npu.py | 20 +- .../unittests/npu/test_hard_swish_op_npu.py | 57 +- .../unittests/npu/test_huber_loss_op_npu.py | 61 +- .../unittests/npu/test_increment_op_npu.py | 29 +- .../unittests/npu/test_index_sample_op_npu.py | 43 +- .../unittests/npu/test_index_select_op_npu.py | 70 +- .../npu/test_instance_norm_op_npu.py | 1 - .../npu/test_iou_similarity_op_npu.py | 10 +- .../unittests/npu/test_is_empty_op_npu.py | 41 +- .../unittests/npu/test_kldiv_loss_op_npu.py | 43 +- .../unittests/npu/test_label_smooth_op_npu.py | 24 +- .../unittests/npu/test_layer_norm_op_npu.py | 152 +- .../unittests/npu/test_leaky_relu_op_npu.py | 35 +- .../unittests/npu/test_log_loss_op_npu.py | 17 +- .../tests/unittests/npu/test_log_op_npu.py | 28 +- .../unittests/npu/test_log_softmax_op_npu.py | 38 +- .../unittests/npu/test_logical_op_npu.py | 137 +- .../npu/test_lookup_table_v2_op_npu.py | 20 +- .../npu/test_masked_select_op_npu.py | 38 +- .../tests/unittests/npu/test_matmul_op_npu.py | 40 +- .../unittests/npu/test_matmulv2_op_npu.py | 45 +- .../tests/unittests/npu/test_mean_op_npu.py | 2 - .../tests/unittests/npu/test_memcpy_op_npu.py | 93 +- .../npu/test_merged_momentum_op_npu.py | 295 +- .../unittests/npu/test_meshgrid_op_npu.py | 157 +- .../unittests/npu/test_mixed_precision_npu.py | 55 +- .../unittests/npu/test_momentum_op_npu.py | 111 +- .../tests/unittests/npu/test_mul_op_npu.py | 175 +- .../unittests/npu/test_multinomial_op_npu.py | 31 +- .../npu/test_nearest_interp_op_npu.py | 198 +- .../npu/test_nearest_interp_v2_op_npu.py | 116 +- .../tests/unittests/npu/test_norm_op_npu.py | 25 +- .../tests/unittests/npu/test_npu_place.py | 7 +- .../unittests/npu/test_one_hot_op_npu.py | 36 +- .../unittests/npu/test_one_hot_v2_op_npu.py | 43 +- .../tests/unittests/npu/test_p_norm_op_npu.py | 28 +- .../tests/unittests/npu/test_pad3d_op_npu.py | 325 +- .../tests/unittests/npu/test_pad_op_npu.py | 28 +- .../npu/test_parallel_dygraph_mnist_npu.py | 27 +- .../tests/unittests/npu/test_pool2d_op_npu.py | 207 +- .../tests/unittests/npu/test_pow_op_npu.py | 28 +- .../unittests/npu/test_prior_box_op_npu.py | 86 +- .../unittests/npu/test_randperm_op_npu.py | 19 +- .../tests/unittests/npu/test_range_npu.py | 16 +- .../unittests/npu/test_reciprocal_op_npu.py | 12 +- .../unittests/npu/test_reduce_any_op_npu.py | 30 +- .../unittests/npu/test_reduce_max_op_npu.py | 104 +- .../unittests/npu/test_reduce_mean_op_npu.py | 26 +- .../unittests/npu/test_reduce_min_op_npu.py | 104 +- .../unittests/npu/test_reduce_prod_op_npu.py | 62 +- .../unittests/npu/test_reduce_sum_op_npu.py | 49 +- .../tests/unittests/npu/test_relu6_op_npu.py | 29 +- .../tests/unittests/npu/test_relu_op_npu.py | 35 +- .../unittests/npu/test_reshape2_op_npu.py | 5 +- .../unittests/npu/test_rmsprop_op_npu.py | 52 +- .../unittests/npu/test_roi_align_op_npu.py | 124 +- .../unittests/npu/test_run_program_op_npu.py | 132 +- .../unittests/npu/test_sampling_id_op_npu.py | 1 - .../tests/unittests/npu/test_save_load_npu.py | 64 +- .../tests/unittests/npu/test_scale_op_npu.py | 20 +- .../unittests/npu/test_scatter_op_npu.py | 5 - .../tests/unittests/npu/test_seed_op_npu.py | 2 - .../unittests/npu/test_sequence_mask_npu.py | 54 +- .../unittests/npu/test_set_value_op_npu.py | 88 +- .../tests/unittests/npu/test_sgd_op_npu.py | 27 +- .../tests/unittests/npu/test_shape_op_npu.py | 5 - .../unittests/npu/test_shard_index_op.py | 6 +- ...igmoid_cross_entropy_with_logits_op_npu.py | 105 +- .../unittests/npu/test_sigmoid_op_npu.py | 18 +- .../tests/unittests/npu/test_sin_op_npu.py | 10 +- .../tests/unittests/npu/test_size_op_npu.py | 36 +- .../tests/unittests/npu/test_slice_op_npu.py | 137 +- .../npu/test_smooth_l1_loss_op_npu.py | 79 +- .../unittests/npu/test_softmax_op_npu.py | 27 +- .../test_softmax_with_cross_entropy_op_npu.py | 51 +- .../tests/unittests/npu/test_split_op_npu.py | 20 +- .../tests/unittests/npu/test_sqrt_op_npu.py | 28 +- .../tests/unittests/npu/test_square_op_npu.py | 28 +- .../npu/test_squared_l2_norm_op_npu.py | 9 +- .../unittests/npu/test_squeeze_op_npu.py | 42 +- .../tests/unittests/npu/test_stack_op_npu.py | 44 +- .../npu/test_strided_slice_op_npu.py | 236 +- .../tests/unittests/npu/test_sum_op_npu.py | 11 +- .../tests/unittests/npu/test_swish_op_npu.py | 13 +- .../npu/test_sync_batch_norm_base_npu.py | 279 +- .../test_sync_batch_norm_op_npu_baseline.py | 12 +- .../npu/test_sync_batch_norm_op_npu_extra.py | 34 +- .../npu/test_take_along_axis_op_npu.py | 28 +- .../tests/unittests/npu/test_tanh_op_npu.py | 28 +- .../tests/unittests/npu/test_tile_op_npu.py | 25 +- .../tests/unittests/npu/test_top_k_op_npu.py | 49 +- .../unittests/npu/test_top_k_v2_op_npu.py | 103 +- .../unittests/npu/test_transpose_op_npu.py | 19 +- .../unittests/npu/test_tril_triu_op_npu.py | 61 +- .../test_truncated_gaussian_random_op_npu.py | 14 +- .../npu/test_uniform_random_op_npu.py | 23 +- .../unittests/npu/test_unsqueeze_op_npu.py | 10 +- .../unittests/npu/test_unstack_op_npu.py | 5 - .../test_update_loss_scaling_min_op_npu.py | 5 +- .../npu/test_update_loss_scaling_op_npu.py | 151 +- .../unittests/npu/test_where_index_npu.py | 26 +- .../tests/unittests/npu/test_where_op_npu.py | 52 +- .../tests/unittests/npu/test_while_op_npu.py | 32 +- .../paddle/fluid/tests/unittests/op_test.py | 1485 ++-- .../fluid/tests/unittests/op_test_xpu.py | 321 +- ...allel_dygraph_dataparallel_with_pylayer.py | 9 +- .../parallel_dygraph_gradient_check.py | 34 +- ...el_dygraph_gradient_check_in_eager_mode.py | 52 +- .../tests/unittests/parallel_dygraph_mnist.py | 136 +- .../unittests/parallel_dygraph_none_var.py | 32 +- .../parallel_dygraph_shared_unused_var.py | 16 +- .../parallel_dygraph_sparse_embedding.py | 65 +- .../parallel_dygraph_sparse_embedding_fp64.py | 68 +- ...el_dygraph_sparse_embedding_over_height.py | 34 +- .../parallel_dygraph_unused_variables.py | 68 +- .../unittests/parallel_executor_test_base.py | 219 +- .../tests/unittests/ps/dataset_generator_A.py | 4 +- .../tests/unittests/ps/dataset_generator_B.py | 6 +- .../fluid/tests/unittests/ps/fl_ps_trainer.py | 84 +- .../tests/unittests/ps/ps_dnn_trainer.py | 271 +- .../fluid/tests/unittests/ps/test_fl_ps.py | 5 +- .../tests/unittests/ps/test_the_one_ps.py | 16 +- .../fluid/tests/unittests/ps_dnn_model.py | 207 +- .../tests/unittests/py_precise_roi_pool.py | 72 +- .../fluid/tests/unittests/rnn/convert.py | 28 +- .../fluid/tests/unittests/rnn/rnn_numpy.py | 311 +- .../tests/unittests/rnn/test_rnn_cells.py | 54 +- .../unittests/rnn/test_rnn_cells_static.py | 125 +- .../rnn/test_rnn_cudnn_params_packing.py | 8 +- .../tests/unittests/rnn/test_rnn_nets.py | 98 +- .../unittests/rnn/test_rnn_nets_static.py | 146 +- .../tests/unittests/rnn/test_wrappers.py | 36 +- .../unittests/rpc/rpc_launch_sync_add.py | 6 +- .../fluid/tests/unittests/rpc/test_rpc.py | 3 - .../tests/unittests/rpc/test_rpc_base.py | 49 +- .../sequence/test_sequence_concat.py | 17 +- .../unittests/sequence/test_sequence_conv.py | 177 +- .../sequence/test_sequence_enumerate_op.py | 41 +- .../sequence/test_sequence_erase_op.py | 7 +- .../sequence/test_sequence_expand.py | 11 +- .../sequence/test_sequence_expand_as.py | 5 - .../sequence/test_sequence_first_step.py | 13 +- .../sequence/test_sequence_last_step.py | 13 +- .../unittests/sequence/test_sequence_mask.py | 54 +- .../sequence/test_sequence_pad_op.py | 39 +- .../unittests/sequence/test_sequence_pool.py | 125 +- .../sequence/test_sequence_reshape.py | 19 +- .../sequence/test_sequence_reverse.py | 25 +- .../sequence/test_sequence_scatter_op.py | 9 +- .../sequence/test_sequence_slice_op.py | 15 +- .../sequence/test_sequence_softmax_op.py | 19 +- .../test_sequence_topk_avg_pooling.py | 80 +- .../sequence/test_sequence_unpad_op.py | 15 +- .../fluid/tests/unittests/seresnext_net.py | 165 +- .../tests/unittests/seresnext_test_base.py | 39 +- .../tests/unittests/simnet_dataset_reader.py | 1 - .../fluid/tests/unittests/simple_nets.py | 49 +- .../tests/unittests/spawn_runner_base.py | 30 +- .../static_model_parallel_fused_attention.py | 99 +- ...static_model_parallel_fused_feedforward.py | 85 +- ..._model_parallel_fused_multi_transformer.py | 53 +- .../fluid/tests/unittests/test_Tensor_type.py | 1 - .../fluid/tests/unittests/test_accuracy_op.py | 58 +- .../unittests/test_activation_nn_grad.py | 205 +- .../tests/unittests/test_activation_op.py | 864 ++- .../unittests/test_activation_sparse_op.py | 2 - .../fluid/tests/unittests/test_adadelta_op.py | 102 +- .../fluid/tests/unittests/test_adagrad_op.py | 77 +- .../tests/unittests/test_adagrad_op_v2.py | 29 +- .../fluid/tests/unittests/test_adam_op.py | 542 +- .../test_adam_optimizer_fp32_fp64.py | 7 +- .../fluid/tests/unittests/test_adamax_api.py | 50 +- .../fluid/tests/unittests/test_adamax_op.py | 56 +- .../fluid/tests/unittests/test_adamw_op.py | 421 +- .../unittests/test_adaptive_avg_pool1d.py | 77 +- .../unittests/test_adaptive_avg_pool2d.py | 214 +- .../unittests/test_adaptive_avg_pool3d.py | 230 +- .../unittests/test_adaptive_max_pool1d.py | 67 +- .../unittests/test_adaptive_max_pool2d.py | 179 +- .../unittests/test_adaptive_max_pool3d.py | 221 +- .../test_add_position_encoding_op.py | 48 +- .../unittests/test_add_reader_dependency.py | 48 +- .../fluid/tests/unittests/test_addmm_op.py | 286 +- .../tests/unittests/test_affine_channel_op.py | 26 +- .../unittests/test_affine_grid_function.py | 85 +- .../tests/unittests/test_affine_grid_op.py | 100 +- .../tests/unittests/test_allclose_layer.py | 163 +- .../fluid/tests/unittests/test_allclose_op.py | 47 +- .../fluid/tests/unittests/test_allgather.py | 1 - .../test_amp_check_finite_and_scale_op.py | 3 - .../unittests/test_anchor_generator_op.py | 36 +- .../fluid/tests/unittests/test_angle_op.py | 35 +- .../unittests/test_apply_pass_to_program.py | 60 +- .../fluid/tests/unittests/test_arange.py | 30 +- .../tests/unittests/test_arg_min_max_op.py | 66 +- .../tests/unittests/test_arg_min_max_v2_op.py | 128 +- .../fluid/tests/unittests/test_argsort_op.py | 148 +- .../unittests/test_array_read_write_op.py | 79 +- .../tests/unittests/test_ascend_trigger.py | 12 +- .../fluid/tests/unittests/test_assert_op.py | 27 +- .../fluid/tests/unittests/test_assign_op.py | 100 +- .../tests/unittests/test_assign_pos_op.py | 34 +- .../tests/unittests/test_assign_value_op.py | 38 +- .../test_async_ssa_graph_executor_mnist.py | 130 +- .../fluid/tests/unittests/test_atan2_op.py | 23 +- .../tests/unittests/test_attention_lstm_op.py | 63 +- .../tests/unittests/test_attribute_var.py | 20 +- .../fluid/tests/unittests/test_auc_op.py | 74 +- .../unittests/test_auc_single_pred_op.py | 32 +- .../test_auto_growth_gpu_memory_limit.py | 7 +- .../test_auto_parallel_autoconvert.py | 1 - .../unittests/test_auto_parallel_cluster.py | 16 +- .../test_auto_parallel_completion.py | 541 +- .../test_auto_parallel_completion_gpt.py | 635 +- .../test_auto_parallel_cost_model.py | 163 +- .../test_auto_parallel_data_unshard.py | 1 - .../test_auto_parallel_dist_tensor.py | 161 +- .../unittests/test_auto_parallel_graph.py | 1 - .../unittests/test_auto_parallel_mapper.py | 174 +- .../test_auto_parallel_partitioner.py | 1401 ++-- .../test_auto_parallel_partitioner_gpt.py | 733 +- .../unittests/test_auto_parallel_reshard.py | 228 +- .../test_auto_parallel_reshard_dpmppp.py | 138 +- .../test_auto_parallel_reshard_mppp.py | 174 +- .../test_auto_parallel_reshard_serial.py | 120 +- .../unittests/test_auto_parallel_save_load.py | 1 - .../unittests/test_auto_parallel_searcher.py | 186 +- .../test_auto_search_dist_matmul_op.py | 565 +- .../unittests/test_auto_search_dist_op.py | 470 +- .../test_avoid_twice_initialization.py | 31 +- .../fluid/tests/unittests/test_backward.py | 169 +- ...test_backward_infer_var_data_type_shape.py | 13 +- .../fluid/tests/unittests/test_base_layer.py | 254 +- .../tests/unittests/test_basic_gru_api.py | 269 +- .../tests/unittests/test_basic_gru_unit_op.py | 78 +- .../tests/unittests/test_basic_lstm_api.py | 217 +- .../unittests/test_basic_lstm_unit_op.py | 84 +- .../tests/unittests/test_basic_rnn_name.py | 127 +- .../fluid/tests/unittests/test_batch_fc_op.py | 34 +- .../tests/unittests/test_batch_norm_op.py | 350 +- .../tests/unittests/test_batch_norm_op_v2.py | 95 +- .../tests/unittests/test_batch_sampler.py | 75 +- .../fluid/tests/unittests/test_bce_loss.py | 265 +- .../unittests/test_bce_with_logits_loss.py | 349 +- .../unittests/test_beam_search_decode_op.py | 71 +- .../tests/unittests/test_beam_search_op.py | 278 +- .../tests/unittests/test_bernoulli_op.py | 12 +- .../paddle/fluid/tests/unittests/test_bfgs.py | 82 +- .../tests/unittests/test_bicubic_interp_op.py | 326 +- .../unittests/test_bicubic_interp_v2_op.py | 443 +- .../unittests/test_bilateral_slice_op.py | 55 +- .../tests/unittests/test_bilinear_api.py | 27 +- .../unittests/test_bilinear_interp_op.py | 214 +- .../unittests/test_bilinear_interp_v2_op.py | 327 +- .../test_bilinear_tensor_product_op.py | 13 +- .../fluid/tests/unittests/test_bincount_op.py | 72 +- .../unittests/test_bipartite_match_op.py | 44 +- .../fluid/tests/unittests/test_bitwise_op.py | 73 +- .../tests/unittests/test_block_rename_var.py | 22 +- .../fluid/tests/unittests/test_bmm_op.py | 40 +- .../fluid/tests/unittests/test_box_clip_op.py | 31 +- .../tests/unittests/test_box_coder_op.py | 111 +- .../test_box_decoder_and_assign_op.py | 6 +- .../fluid/tests/unittests/test_boxps.py | 68 +- .../fluid/tests/unittests/test_bpr_loss_op.py | 5 +- .../tests/unittests/test_broadcast_error.py | 1 - .../tests/unittests/test_broadcast_shape.py | 9 +- .../unittests/test_broadcast_tensors_op.py | 114 +- .../tests/unittests/test_broadcast_to_op.py | 40 +- .../tests/unittests/test_bucketize_api.py | 41 +- .../test_buffer_shared_memory_reuse_pass.py | 78 +- ...euse_pass_and_fuse_optimization_op_pass.py | 2 - .../unittests/test_c_comm_init_all_op.py | 9 +- .../tests/unittests/test_c_embedding_op.py | 6 +- .../tests/unittests/test_calc_gradient.py | 58 +- .../paddle/fluid/tests/unittests/test_case.py | 203 +- .../fluid/tests/unittests/test_cast_op.py | 61 +- .../fluid/tests/unittests/test_center_loss.py | 62 +- .../tests/unittests/test_channel_shuffle.py | 110 +- .../unittests/test_check_import_scipy.py | 1 - .../tests/unittests/test_checkpoint_saver.py | 1 - .../fluid/tests/unittests/test_cholesky_op.py | 41 +- .../tests/unittests/test_cholesky_solve_op.py | 72 +- .../tests/unittests/test_chunk_eval_op.py | 211 +- .../fluid/tests/unittests/test_chunk_op.py | 19 +- .../unittests/test_class_center_sample_op.py | 165 +- .../tests/unittests/test_clip_by_norm_op.py | 40 +- .../fluid/tests/unittests/test_clip_op.py | 144 +- .../unittests/test_coalesce_tensor_op.py | 127 +- .../test_collect_fpn_proposals_op.py | 45 +- .../unittests/test_collective_api_base.py | 361 +- .../tests/unittests/test_collective_base.py | 232 +- .../unittests/test_communicator_async.py | 4 +- .../tests/unittests/test_communicator_geo.py | 24 +- .../unittests/test_communicator_ps_gpu.py | 14 +- .../fluid/tests/unittests/test_compare_op.py | 154 +- .../tests/unittests/test_compare_reduce_op.py | 9 - .../fluid/tests/unittests/test_compat.py | 1 - .../tests/unittests/test_compiled_program.py | 84 +- .../fluid/tests/unittests/test_complex_abs.py | 48 +- .../tests/unittests/test_complex_cast.py | 59 +- .../test_complex_elementwise_layers.py | 46 +- .../tests/unittests/test_complex_getitem.py | 1 - .../test_complex_grad_accumulated.py | 60 +- .../tests/unittests/test_complex_kron.py | 45 +- .../tests/unittests/test_complex_matmul.py | 68 +- .../fluid/tests/unittests/test_complex_op.py | 74 +- .../tests/unittests/test_complex_reshape.py | 25 +- .../tests/unittests/test_complex_simplenet.py | 27 +- .../tests/unittests/test_complex_sum_layer.py | 7 +- .../unittests/test_complex_trace_layer.py | 12 +- .../tests/unittests/test_complex_transpose.py | 7 +- .../tests/unittests/test_complex_variable.py | 28 +- .../tests/unittests/test_complex_view_op.py | 33 +- .../fluid/tests/unittests/test_concat_op.py | 164 +- .../paddle/fluid/tests/unittests/test_cond.py | 434 +- .../tests/unittests/test_conditional_block.py | 36 +- .../fluid/tests/unittests/test_conj_op.py | 53 +- .../fluid/tests/unittests/test_const_value.py | 1 - .../tests/unittests/test_context_manager.py | 1 - .../tests/unittests/test_conv1d_layer.py | 198 +- .../unittests/test_conv1d_transpose_layer.py | 230 +- .../fluid/tests/unittests/test_conv2d_api.py | 458 +- .../tests/unittests/test_conv2d_fusion_op.py | 47 +- .../tests/unittests/test_conv2d_layer.py | 273 +- .../fluid/tests/unittests/test_conv2d_op.py | 460 +- .../test_conv2d_op_depthwise_conv.py | 38 +- .../unittests/test_conv2d_transpose_layer.py | 226 +- .../unittests/test_conv2d_transpose_op.py | 512 +- ...test_conv2d_transpose_op_depthwise_conv.py | 5 - .../tests/unittests/test_conv3d_layer.py | 206 +- .../fluid/tests/unittests/test_conv3d_op.py | 677 +- .../unittests/test_conv3d_transpose_layer.py | 237 +- .../unittests/test_conv3d_transpose_op.py | 216 +- .../test_conv3d_transpose_part2_op.py | 188 +- .../tests/unittests/test_conv_nn_grad.py | 345 +- .../tests/unittests/test_conv_shift_op.py | 1 - .../unittests/test_conv_transpose_nn_grad.py | 254 +- .../paddle/fluid/tests/unittests/test_corr.py | 33 +- .../fluid/tests/unittests/test_cos_sim_op.py | 73 +- .../unittests/test_cosine_embedding_loss.py | 260 +- .../unittests/test_cosine_similarity_api.py | 12 +- .../fluid/tests/unittests/test_cost_model.py | 26 +- .../tests/unittests/test_count_nonzero_api.py | 10 +- .../paddle/fluid/tests/unittests/test_cov.py | 215 +- .../tests/unittests/test_cpuonly_spawn.py | 2 - .../tests/unittests/test_create_global_var.py | 1 - .../unittests/test_create_op_doc_string.py | 1 - .../tests/unittests/test_create_parameter.py | 17 +- .../tests/unittests/test_crf_decoding_op.py | 114 +- .../fluid/tests/unittests/test_crop_op.py | 20 +- .../tests/unittests/test_crop_tensor_op.py | 52 +- .../tests/unittests/test_cross_entropy2_op.py | 39 +- .../unittests/test_cross_entropy_loss.py | 1540 ++-- .../tests/unittests/test_cross_entropy_op.py | 286 +- .../fluid/tests/unittests/test_cross_op.py | 52 +- .../fluid/tests/unittests/test_crypto.py | 2 - .../fluid/tests/unittests/test_ctc_align.py | 148 +- .../unittests/test_cuda_cudnn_version.py | 1 - .../tests/unittests/test_cuda_device_count.py | 1 - .../test_cuda_device_name_capability.py | 5 +- .../tests/unittests/test_cuda_empty_cache.py | 1 - .../fluid/tests/unittests/test_cuda_graph.py | 68 +- .../test_cuda_graph_partial_graph.py | 2 - .../test_cuda_graph_partial_graph_static.py | 8 +- ...est_cuda_graph_partial_graph_static_run.py | 8 +- .../test_cuda_max_memory_allocated.py | 23 +- .../test_cuda_max_memory_reserved.py | 23 +- .../unittests/test_cuda_memory_allocated.py | 7 +- .../unittests/test_cuda_memory_reserved.py | 7 +- .../tests/unittests/test_cuda_random_seed.py | 71 +- .../tests/unittests/test_cuda_stream_event.py | 15 +- .../tests/unittests/test_cudnn_grucell.py | 114 +- .../tests/unittests/test_cudnn_lstmcell.py | 185 +- .../fluid/tests/unittests/test_cumprod_op.py | 28 +- .../fluid/tests/unittests/test_cumsum_op.py | 169 +- .../tests/unittests/test_custom_grad_input.py | 66 +- .../fluid/tests/unittests/test_cvm_op.py | 78 +- .../unittests/test_cyclic_cifar_dataset.py | 1 - .../paddle/fluid/tests/unittests/test_data.py | 17 +- .../tests/unittests/test_data_generator.py | 27 - .../tests/unittests/test_data_norm_op.py | 123 +- .../unittests/test_dataloader_autotune.py | 51 +- .../unittests/test_dataloader_dataset.py | 24 +- .../unittests/test_dataloader_early_reset.py | 10 +- .../unittests/test_dataloader_keep_order.py | 61 +- .../unittests/test_dataloader_unkeep_order.py | 76 +- .../fluid/tests/unittests/test_dataset.py | 607 +- .../test_dataset_consistency_inspection.py | 347 +- .../unittests/test_dataset_dataloader.py | 72 +- .../tests/unittests/test_dataset_download.py | 1 - .../fluid/tests/unittests/test_debugger.py | 59 +- .../unittests/test_decayed_adagrad_op.py | 11 +- .../unittests/test_decoupled_py_reader.py | 76 +- .../test_decoupled_py_reader_data_check.py | 38 +- .../tests/unittests/test_default_dtype.py | 2 - .../unittests/test_default_scope_funcs.py | 11 +- .../tests/unittests/test_deform_conv2d.py | 324 +- .../unittests/test_deformable_conv_op.py | 364 +- .../unittests/test_deformable_conv_v1_op.py | 241 +- .../test_deformable_psroi_pooling.py | 355 +- .../fluid/tests/unittests/test_deg2rad.py | 21 +- .../unittests/test_density_prior_box_op.py | 71 +- .../unittests/test_deprecated_decorator.py | 21 +- ...t_deprecated_memory_optimize_interfaces.py | 2 - .../unittests/test_dequantize_abs_max_op.py | 5 +- .../tests/unittests/test_dequantize_log_op.py | 3 +- .../fluid/tests/unittests/test_desc_clone.py | 131 +- .../fluid/tests/unittests/test_detach.py | 143 +- .../tests/unittests/test_detection_map_op.py | 133 +- .../tests/unittests/test_determinant_op.py | 13 +- .../fluid/tests/unittests/test_device.py | 26 +- .../tests/unittests/test_device_guard.py | 99 +- .../paddle/fluid/tests/unittests/test_diag.py | 3 - .../fluid/tests/unittests/test_diag_embed.py | 24 +- .../fluid/tests/unittests/test_diag_v2.py | 88 +- .../fluid/tests/unittests/test_diagflat.py | 11 +- .../fluid/tests/unittests/test_diagonal_op.py | 57 +- .../fluid/tests/unittests/test_diff_op.py | 111 +- .../fluid/tests/unittests/test_digamma_op.py | 4 - .../unittests/test_directory_migration.py | 192 +- .../unittests/test_disable_signal_handler.py | 22 +- .../tests/unittests/test_dist_allreduce_op.py | 2 +- .../fluid/tests/unittests/test_dist_base.py | 1023 ++- .../tests/unittests/test_dist_dygraph_apis.py | 1 - .../test_dist_fleet_a_sync_optimizer_async.py | 6 +- .../test_dist_fleet_a_sync_optimizer_auto.py | 17 +- ..._dist_fleet_a_sync_optimizer_auto_async.py | 29 +- ...st_dist_fleet_a_sync_optimizer_auto_geo.py | 17 +- .../test_dist_fleet_a_sync_optimizer_geo.py | 28 +- .../test_dist_fleet_a_sync_optimizer_sync.py | 6 +- .../tests/unittests/test_dist_fleet_base.py | 251 +- .../tests/unittests/test_dist_fleet_ctr.py | 59 +- .../tests/unittests/test_dist_fleet_ctr2.py | 30 +- .../tests/unittests/test_dist_fleet_decay.py | 41 +- .../tests/unittests/test_dist_fleet_geo.py | 19 +- .../tests/unittests/test_dist_fleet_gloo.py | 74 +- .../unittests/test_dist_fleet_heter_base.py | 336 +- .../test_dist_fleet_heter_program.py | 67 +- .../tests/unittests/test_dist_fleet_infer.py | 44 +- .../tests/unittests/test_dist_fleet_ps.py | 106 +- .../tests/unittests/test_dist_fleet_ps10.py | 41 +- .../tests/unittests/test_dist_fleet_ps11.py | 93 +- .../tests/unittests/test_dist_fleet_ps12.py | 66 +- .../tests/unittests/test_dist_fleet_ps13.py | 93 +- .../tests/unittests/test_dist_fleet_ps2.py | 99 +- .../tests/unittests/test_dist_fleet_ps3.py | 106 +- .../tests/unittests/test_dist_fleet_ps4.py | 100 +- .../tests/unittests/test_dist_fleet_ps5.py | 117 +- .../tests/unittests/test_dist_fleet_ps6.py | 100 +- .../tests/unittests/test_dist_fleet_ps7.py | 41 +- .../tests/unittests/test_dist_fleet_ps8.py | 41 +- .../tests/unittests/test_dist_fleet_ps9.py | 41 +- .../test_dist_fleet_raw_program_optimizer.py | 12 +- ...et_raw_program_optimizer_fuse_allreduce.py | 5 +- .../tests/unittests/test_dist_fleet_simnet.py | 17 +- .../test_dist_fleet_sparse_embedding_ctr.py | 158 +- .../test_dist_fleet_trainer_desc_config.py | 20 +- .../test_dist_lookup_sparse_table_fuse_ops.py | 89 +- .../test_dist_mnist_backward_deps.py | 2 +- .../unittests/test_dist_mnist_batch_merge.py | 40 +- .../unittests/test_dist_mnist_fleet_save.py | 19 +- .../unittests/test_dist_mnist_fleetapi.py | 16 +- .../test_dist_mnist_fp16_allreduce.py | 2 +- .../unittests/test_dist_mnist_hallreduce.py | 12 +- .../tests/unittests/test_dist_mnist_lars.py | 1 - .../unittests/test_dist_mnist_multi_comm.py | 12 +- .../tests/unittests/test_dist_mnist_pg.py | 16 +- .../test_dist_mnist_ring_allreduce.py | 2 +- .../tests/unittests/test_dist_mnist_train.py | 38 +- .../unittests/test_dist_mnist_with_program.py | 4 +- .../fluid/tests/unittests/test_dist_oneps.py | 1 - .../fluid/tests/unittests/test_dist_op.py | 66 +- .../tests/unittests/test_dist_save_load.py | 89 +- .../unittests/test_dist_se_resnext_nccl.py | 26 +- .../unittests/test_dist_se_resnext_sync.py | 11 +- .../unittests/test_dist_sharding_save.py | 37 +- .../unittests/test_dist_sparse_load_ps0.py | 16 +- .../unittests/test_dist_sparse_load_ps1.py | 4 +- .../test_dist_sparse_tensor_load_adagrad.py | 5 +- .../test_dist_sparse_tensor_load_adam.py | 5 +- .../test_dist_sparse_tensor_load_ftrl.py | 5 +- .../test_dist_sparse_tensor_load_momentum.py | 5 +- .../test_dist_sparse_tensor_load_rmsprop.py | 5 +- .../test_dist_sparse_tensor_load_sgd.py | 15 +- .../test_dist_text_classification.py | 22 +- .../fluid/tests/unittests/test_dist_train.py | 71 +- .../tests/unittests/test_dist_transformer.py | 50 +- .../tests/unittests/test_dist_transpiler.py | 1133 ++- .../tests/unittests/test_dist_tree_index.py | 89 +- .../tests/unittests/test_dist_word2vec.py | 33 +- .../test_distribute_fpn_proposals_op.py | 113 +- ...est_distributed_fused_lamb_op_with_clip.py | 17 +- ...buted_fused_lamb_op_with_gradient_merge.py | 19 +- ..._distributed_fused_lamb_op_without_clip.py | 1 - .../tests/unittests/test_distributions.py | 680 +- .../fluid/tests/unittests/test_dot_op.py | 179 +- .../fluid/tests/unittests/test_downpoursgd.py | 53 +- .../fluid/tests/unittests/test_dpsgd_op.py | 7 +- .../tests/unittests/test_dropout_nd_op.py | 65 +- .../fluid/tests/unittests/test_dropout_op.py | 774 +- .../unittests/test_dygraph_mnist_fp16.py | 132 +- .../test_dygraph_mode_of_unittest.py | 3 +- .../unittests/test_dygraph_multi_forward.py | 200 +- .../unittests/test_dygraph_spectral_norm.py | 54 +- .../unittests/test_dygraph_weight_norm.py | 66 +- .../fluid/tests/unittests/test_dyn_rnn.py | 215 +- .../test_dynamic_rnn_stop_gradient.py | 40 +- .../unittests/test_dynrnn_gradient_check.py | 124 +- .../unittests/test_dynrnn_static_input.py | 83 +- .../test_eager_deletion_delete_vars.py | 76 +- .../test_eager_deletion_dynamic_rnn_base.py | 33 +- .../unittests/test_eager_deletion_gru_net.py | 22 +- .../unittests/test_eager_deletion_lstm_net.py | 28 +- .../test_eager_deletion_padding_rnn.py | 433 +- .../test_eager_deletion_recurrent_op.py | 266 +- .../unittests/test_eager_deletion_while_op.py | 59 +- .../tests/unittests/test_eager_run_program.py | 90 +- .../tests/unittests/test_eager_trace_op.py | 19 +- .../tests/unittests/test_edit_distance_op.py | 44 +- .../unittests/test_egr_code_generate_api.py | 10 +- .../tests/unittests/test_egr_python_api.py | 378 +- .../unittests/test_egr_string_tensor_api.py | 39 +- .../fluid/tests/unittests/test_eig_op.py | 164 +- .../fluid/tests/unittests/test_eigh_op.py | 112 +- .../fluid/tests/unittests/test_eigvals_op.py | 181 +- .../fluid/tests/unittests/test_eigvalsh_op.py | 97 +- .../fluid/tests/unittests/test_einsum.py | 203 +- .../fluid/tests/unittests/test_einsum_op.py | 29 +- .../fluid/tests/unittests/test_einsum_v2.py | 247 +- .../unittests/test_elementwise_add_op.py | 230 +- .../unittests/test_elementwise_div_op.py | 151 +- .../unittests/test_elementwise_floordiv_op.py | 6 +- .../unittests/test_elementwise_gradient_op.py | 62 +- .../test_elementwise_heaviside_op.py | 72 +- .../unittests/test_elementwise_max_op.py | 100 +- .../unittests/test_elementwise_min_op.py | 103 +- .../unittests/test_elementwise_mod_op.py | 24 +- .../unittests/test_elementwise_mul_op.py | 164 +- .../unittests/test_elementwise_nn_grad.py | 148 +- .../unittests/test_elementwise_pow_op.py | 72 +- .../unittests/test_elementwise_sub_op.py | 148 +- .../paddle/fluid/tests/unittests/test_ema.py | 36 +- .../fluid/tests/unittests/test_ema_fleet.py | 35 +- .../test_embedding_id_stop_gradient.py | 11 +- .../tests/unittests/test_empty_like_op.py | 52 +- .../fluid/tests/unittests/test_empty_op.py | 80 +- .../fluid/tests/unittests/test_entry_attr.py | 24 +- .../fluid/tests/unittests/test_entry_attr2.py | 17 +- .../fluid/tests/unittests/test_erf_op.py | 2 - .../fluid/tests/unittests/test_erfinv_op.py | 20 +- .../fluid/tests/unittests/test_exception.py | 13 +- .../tests/unittests/test_executor_and_mul.py | 16 +- .../test_executor_and_use_program_cache.py | 138 +- .../unittests/test_executor_check_feed.py | 32 +- .../test_executor_check_fetch_list.py | 23 +- .../test_executor_feed_non_tensor.py | 123 +- ..._executor_return_tensor_not_overwriting.py | 12 +- .../tests/unittests/test_expand_as_op.py | 35 +- .../tests/unittests/test_expand_as_v2_op.py | 34 +- .../fluid/tests/unittests/test_expand_op.py | 74 +- .../tests/unittests/test_expand_v2_op.py | 107 +- .../tests/unittests/test_exponential_op.py | 242 +- .../fluid/tests/unittests/test_eye_op.py | 29 +- .../unittests/test_fake_dequantize_op.py | 81 +- .../tests/unittests/test_fake_init_op.py | 6 +- .../tests/unittests/test_fake_quantize_op.py | 298 +- .../unittests/test_faster_tokenizer_op.py | 220 +- .../fluid/tests/unittests/test_fc_op.py | 41 +- .../test_feed_data_check_shape_type.py | 211 +- .../tests/unittests/test_feed_fetch_method.py | 1 - .../tests/unittests/test_fetch_handler.py | 15 +- .../unittests/test_fetch_lod_tensor_array.py | 34 +- .../tests/unittests/test_fetch_unmerged.py | 85 +- .../fluid/tests/unittests/test_fetch_var.py | 2 - .../tests/unittests/test_fill_any_like_op.py | 24 +- .../fluid/tests/unittests/test_fill_any_op.py | 17 +- .../test_fill_constant_batch_size_like.py | 22 +- .../tests/unittests/test_fill_constant_op.py | 305 +- .../unittests/test_fill_diagonal_tensor_op.py | 11 +- .../fluid/tests/unittests/test_fill_op.py | 21 +- .../unittests/test_fill_zeros_like2_op.py | 5 - .../unittests/test_fill_zeros_like_op.py | 2 - .../unittests/test_filter_by_instag_op.py | 48 +- .../fluid/tests/unittests/test_flatten2_op.py | 20 +- .../test_flatten_contiguous_range_op.py | 111 +- .../fluid/tests/unittests/test_flatten_op.py | 4 - .../fluid/tests/unittests/test_fleet.py | 38 +- .../tests/unittests/test_fleet_api_input.py | 239 +- .../unittests/test_fleet_ascend_utils.py | 35 +- .../fluid/tests/unittests/test_fleet_auto.py | 17 +- .../fluid/tests/unittests/test_fleet_base.py | 50 +- .../tests/unittests/test_fleet_base_2.py | 41 +- .../tests/unittests/test_fleet_base_3.py | 34 +- .../tests/unittests/test_fleet_base_4.py | 6 +- .../tests/unittests/test_fleet_base_single.py | 34 +- .../test_fleet_elastic_collective.py | 8 +- .../unittests/test_fleet_elastic_init.py | 2 - .../unittests/test_fleet_elastic_manager.py | 114 +- .../test_fleet_exe_dist_model_run.py | 41 +- .../test_fleet_exe_dist_model_tensor.py | 52 +- .../tests/unittests/test_fleet_executor.py | 39 +- .../test_fleet_executor_multi_devices.py | 10 +- .../test_fleet_executor_origin_scheduler.py | 39 +- .../test_fleet_executor_task_node.py | 16 +- .../unittests/test_fleet_executor_utils.py | 12 +- .../test_fleet_executor_with_task_nodes.py | 41 +- .../unittests/test_fleet_gradient_scale.py | 31 +- .../tests/unittests/test_fleet_metric.py | 25 +- .../tests/unittests/test_fleet_nocvm_1.py | 50 +- .../fluid/tests/unittests/test_fleet_ps.py | 21 +- .../unittests/test_fleet_pyramid_hash.py | 15 +- .../tests/unittests/test_fleet_rolemaker.py | 33 +- .../tests/unittests/test_fleet_rolemaker_2.py | 36 +- .../tests/unittests/test_fleet_rolemaker_3.py | 34 +- .../tests/unittests/test_fleet_rolemaker_4.py | 9 +- .../unittests/test_fleet_rolemaker_init.py | 27 +- .../tests/unittests/test_fleet_runtime.py | 26 +- .../unittests/test_fleet_unitaccessor.py | 35 +- .../fluid/tests/unittests/test_fleet_util.py | 76 +- .../paddle/fluid/tests/unittests/test_flip.py | 56 +- .../fluid/tests/unittests/test_fmax_op.py | 104 +- .../fluid/tests/unittests/test_fmin_op.py | 104 +- .../fluid/tests/unittests/test_fold_op.py | 138 +- .../fluid/tests/unittests/test_frac_api.py | 12 +- .../fluid/tests/unittests/test_frame_op.py | 10 +- .../unittests/test_framework_debug_str.py | 1 - .../fluid/tests/unittests/test_frexp_api.py | 17 +- .../tests/unittests/test_fs_interface.py | 1 - .../fluid/tests/unittests/test_fsp_op.py | 27 +- .../fluid/tests/unittests/test_ftrl_op.py | 97 +- .../tests/unittests/test_full_like_op.py | 62 +- .../fluid/tests/unittests/test_full_op.py | 225 +- .../tests/unittests/test_function_hook.py | 2 - .../tests/unittests/test_functional_conv1d.py | 27 +- .../test_functional_conv1d_transpose.py | 27 +- .../tests/unittests/test_functional_conv2d.py | 236 +- .../test_functional_conv2d_transpose.py | 250 +- .../tests/unittests/test_functional_conv3d.py | 237 +- .../test_functional_conv3d_transpose.py | 269 +- .../unittests/test_fuse_all_reduce_pass.py | 78 +- .../tests/unittests/test_fuse_bn_act_pass.py | 74 +- .../unittests/test_fuse_bn_add_act_pass.py | 229 +- .../test_fuse_elewise_add_act_pass.py | 43 +- .../unittests/test_fuse_gemm_epilogue_pass.py | 261 +- .../unittests/test_fuse_optimizer_pass.py | 147 +- .../test_fuse_relu_depthwise_conv_pass.py | 68 +- .../unittests/test_fused_attention_op.py | 224 +- .../unittests/test_fused_attention_op_api.py | 350 +- ...sed_bias_dropout_residual_layer_norm_op.py | 111 +- ...bias_dropout_residual_layer_norm_op_api.py | 100 +- .../test_fused_elemwise_activation_op.py | 293 +- .../unittests/test_fused_emb_seq_pool_op.py | 69 +- .../test_fused_embedding_fc_lstm_op.py | 100 +- .../test_fused_fc_elementwise_layernorm_op.py | 16 +- .../unittests/test_fused_feedforward_op.py | 317 +- .../unittests/test_fused_gate_attention_op.py | 184 +- .../test_fused_gemm_epilogue_grad_op.py | 128 +- .../unittests/test_fused_gemm_epilogue_op.py | 303 +- .../tests/unittests/test_fused_matmul_bias.py | 20 +- .../test_fused_multi_transformer_int8_op.py | 414 +- .../test_fused_multi_transformer_op.py | 547 +- .../test_fused_multihead_matmul_op.py | 58 +- .../unittests/test_fused_token_prune_op.py | 79 +- .../test_fused_transformer_encoder_layer.py | 131 +- ...st_fused_transformer_with_amp_decorator.py | 32 +- .../tests/unittests/test_fusion_gru_op.py | 82 +- .../tests/unittests/test_fusion_lstm_op.py | 84 +- .../test_fusion_repeated_fc_relu_op.py | 29 +- .../test_fusion_seqconv_eltadd_relu_op.py | 12 +- .../test_fusion_seqexpand_concat_fc_op.py | 33 +- .../test_fusion_seqpool_concat_op.py | 19 +- .../test_fusion_seqpool_cvm_concat_op.py | 19 +- .../test_fusion_squared_mat_sub_op.py | 6 +- ...test_fusion_transpose_flatten_concat_op.py | 44 +- .../unittests/test_gast_with_compatibility.py | 2 - .../tests/unittests/test_gather_nd_op.py | 66 +- .../fluid/tests/unittests/test_gather_op.py | 80 +- .../tests/unittests/test_gather_tree_op.py | 107 +- .../unittests/test_gaussian_random_op.py | 212 +- .../paddle/fluid/tests/unittests/test_gcd.py | 34 +- .../fluid/tests/unittests/test_gelu_op.py | 18 +- .../unittests/test_generate_mask_labels_op.py | 154 +- .../test_generate_proposal_labels_op.py | 265 +- .../unittests/test_generate_proposals_op.py | 225 +- .../test_generate_proposals_v2_op.py | 124 +- .../fluid/tests/unittests/test_generator.py | 6 +- .../unittests/test_generator_dataloader.py | 73 +- .../test_get_all_op_or_kernel_names.py | 6 +- .../unittests/test_get_device_properties.py | 2 - .../test_get_inputs_outputs_in_block.py | 7 +- .../tests/unittests/test_get_places_op.py | 1 - .../tests/unittests/test_get_set_flags.py | 4 +- .../test_get_tensor_from_selected_rows_op.py | 1 - .../test_global_var_getter_setter.py | 2 - .../paddle/fluid/tests/unittests/test_glu.py | 2 - .../test_gpu_package_without_gpu_device.py | 16 +- .../unittests/test_grad_clip_minimize.py | 53 +- .../tests/unittests/test_gradient_clip.py | 264 +- .../unittests/test_graph_khop_sampler.py | 176 +- .../tests/unittests/test_graph_reindex.py | 469 +- .../unittests/test_graph_sample_neighbors.py | 348 +- .../unittests/test_graph_send_recv_op.py | 308 +- .../unittests/test_graph_send_ue_recv_op.py | 487 +- .../tests/unittests/test_graph_send_uv_op.py | 159 +- .../unittests/test_grid_sample_function.py | 73 +- .../tests/unittests/test_grid_sampler_op.py | 272 +- .../tests/unittests/test_group_norm_op.py | 141 +- .../tests/unittests/test_group_norm_op_v2.py | 58 +- .../fluid/tests/unittests/test_gru_op.py | 111 +- .../fluid/tests/unittests/test_gru_rnn_op.py | 49 +- .../fluid/tests/unittests/test_gru_unit_op.py | 130 +- .../tests/unittests/test_gumbel_softmax_op.py | 35 +- .../fluid/tests/unittests/test_hash_op.py | 55 +- .../unittests/test_hinge_embedding_loss.py | 121 +- .../tests/unittests/test_hinge_loss_op.py | 1 - .../tests/unittests/test_histogram_op.py | 63 +- .../fluid/tests/unittests/test_hsigmoid_op.py | 497 +- .../tests/unittests/test_huber_loss_op.py | 40 +- .../test_hybrid_parallel_topology.py | 104 +- .../tests/unittests/test_identity_loss_op.py | 24 +- .../fluid/tests/unittests/test_identity_op.py | 1 - .../tests/unittests/test_iinfo_and_finfo.py | 22 +- .../tests/unittests/test_im2sequence_op.py | 193 +- .../test_image_classification_layer.py | 51 +- .../unittests/test_imperative_auto_prune.py | 62 +- .../tests/unittests/test_imperative_basic.py | 339 +- .../test_imperative_container_layerdict.py | 21 +- .../test_imperative_container_layerlist.py | 32 +- ...test_imperative_container_parameterlist.py | 27 +- .../test_imperative_container_sequential.py | 11 +- .../test_imperative_data_loader_base.py | 53 +- .../test_imperative_data_loader_exception.py | 48 +- .../test_imperative_data_loader_exit_func.py | 17 +- .../test_imperative_data_loader_fds_clear.py | 33 +- .../test_imperative_data_loader_process.py | 30 +- .../test_imperative_data_parallel.py | 14 +- .../unittests/test_imperative_decorator.py | 14 +- .../tests/unittests/test_imperative_deepcf.py | 199 +- .../unittests/test_imperative_double_grad.py | 292 +- .../unittests/test_imperative_framework.py | 34 +- .../tests/unittests/test_imperative_gan.py | 132 +- .../tests/unittests/test_imperative_gnn.py | 97 +- .../tests/unittests/test_imperative_group.py | 55 +- .../test_imperative_hook_for_layer.py | 102 +- .../unittests/test_imperative_layer_apply.py | 49 +- .../test_imperative_layer_children.py | 16 +- .../test_imperative_layer_trainable.py | 1 - .../tests/unittests/test_imperative_layers.py | 192 +- .../test_imperative_load_static_param.py | 233 +- ..._imperative_lod_tensor_to_selected_rows.py | 112 +- .../tests/unittests/test_imperative_mnist.py | 215 +- .../test_imperative_mnist_sorted_gradient.py | 98 +- .../test_imperative_named_members.py | 70 +- .../unittests/test_imperative_numpy_bridge.py | 7 +- .../test_imperative_ocr_attention_model.py | 568 +- .../unittests/test_imperative_optimizer.py | 506 +- .../unittests/test_imperative_optimizer_v2.py | 487 +- ...test_imperative_parallel_coalesce_split.py | 13 +- .../test_imperative_partitial_backward.py | 10 +- .../unittests/test_imperative_ptb_rnn.py | 288 +- ...test_imperative_ptb_rnn_sorted_gradient.py | 124 +- .../test_imperative_recurrent_usage.py | 41 +- .../test_imperative_reinforcement.py | 60 +- .../tests/unittests/test_imperative_resnet.py | 275 +- .../test_imperative_resnet_sorted_gradient.py | 113 +- .../unittests/test_imperative_save_load.py | 498 +- .../unittests/test_imperative_save_load_v2.py | 537 +- .../unittests/test_imperative_se_resnext.py | 402 +- .../test_imperative_selected_rows.py | 33 +- ..._imperative_selected_rows_to_lod_tensor.py | 110 +- .../test_imperative_signal_handler.py | 5 - ...perative_star_gan_with_gradient_penalty.py | 436 +- .../test_imperative_static_runner_mnist.py | 278 +- .../test_imperative_static_runner_while.py | 122 +- .../test_imperative_tensor_clear_gradient.py | 2 - .../test_imperative_thread_local_has_grad.py | 2 - ...imperative_trace_non_persistable_inputs.py | 46 +- ..._imperative_transformer_sorted_gradient.py | 1151 +-- .../unittests/test_imperative_triple_grad.py | 199 +- .../test_imperative_using_non_zero_gpu.py | 1 - .../fluid/tests/unittests/test_increment.py | 10 +- .../tests/unittests/test_index_add_op.py | 153 +- .../tests/unittests/test_index_sample_op.py | 40 +- .../tests/unittests/test_index_select_op.py | 79 +- .../test_infer_no_need_buffer_slots.py | 36 +- .../fluid/tests/unittests/test_infer_shape.py | 1 - .../tests/unittests/test_inference_api.py | 61 +- .../unittests/test_inference_model_io.py | 334 +- .../fluid/tests/unittests/test_initializer.py | 575 +- .../tests/unittests/test_initializer_nn.py | 427 +- .../fluid/tests/unittests/test_inner.py | 48 +- .../fluid/tests/unittests/test_inplace.py | 74 +- .../tests/unittests/test_inplace_abn_op.py | 130 +- .../unittests/test_inplace_addto_strategy.py | 75 +- .../test_inplace_and_clear_gradient.py | 2 - .../test_inplace_auto_generated_apis.py | 42 +- ...test_inplace_softmax_with_cross_entropy.py | 92 +- .../fluid/tests/unittests/test_input_spec.py | 34 +- .../tests/unittests/test_install_check.py | 1 - .../tests/unittests/test_instance_norm_op.py | 171 +- .../unittests/test_instance_norm_op_v2.py | 16 +- .../fluid/tests/unittests/test_inverse_op.py | 51 +- .../tests/unittests/test_io_save_load.py | 35 +- .../tests/unittests/test_iou_similarity_op.py | 7 +- .../fluid/tests/unittests/test_ir_graph.py | 15 +- .../tests/unittests/test_ir_inplace_pass.py | 15 +- .../test_ir_memory_optimize_ifelse_op.py | 31 +- .../unittests/test_ir_memory_optimize_nlp.py | 28 +- .../unittests/test_ir_memory_optimize_pass.py | 17 +- .../test_ir_memory_optimize_transformer.py | 7 +- .../fluid/tests/unittests/test_is_complex.py | 1 - .../fluid/tests/unittests/test_is_empty_op.py | 21 +- .../fluid/tests/unittests/test_is_integer.py | 1 - .../fluid/tests/unittests/test_is_tensor.py | 10 +- .../fluid/tests/unittests/test_isclose_op.py | 63 +- .../fluid/tests/unittests/test_isfinite_op.py | 24 +- .../tests/unittests/test_isfinite_v2_op.py | 26 +- .../fluid/tests/unittests/test_jit_layer.py | 11 +- .../unittests/test_jit_pre_save_hooks.py | 14 +- .../tests/unittests/test_jit_save_load.py | 846 ++- .../tests/unittests/test_kldiv_loss_op.py | 18 +- .../fluid/tests/unittests/test_kron_op.py | 75 +- .../fluid/tests/unittests/test_kthvalue_op.py | 48 +- .../fluid/tests/unittests/test_l1_loss.py | 64 +- .../fluid/tests/unittests/test_l1_norm_op.py | 3 +- .../unittests/test_label_smooth_functional.py | 57 +- .../tests/unittests/test_label_smooth_op.py | 19 +- .../fluid/tests/unittests/test_lamb_op.py | 101 +- .../fluid/tests/unittests/test_lambv2_op.py | 136 +- .../tests/unittests/test_launch_coverage.py | 77 +- .../tests/unittests/test_layer_norm_op.py | 359 +- .../tests/unittests/test_layer_norm_op_v2.py | 12 +- .../fluid/tests/unittests/test_layers.py | 4576 +++++++----- .../tests/unittests/test_layout_autotune.py | 20 +- .../fluid/tests/unittests/test_lazy_init.py | 79 +- .../fluid/tests/unittests/test_lbfgs.py | 66 +- .../paddle/fluid/tests/unittests/test_lcm.py | 34 +- .../unittests/test_learning_rate_scheduler.py | 388 +- .../fluid/tests/unittests/test_lerp_op.py | 49 +- .../fluid/tests/unittests/test_lgamma_op.py | 10 +- .../unittests/test_limit_by_capacity_op.py | 55 +- .../fluid/tests/unittests/test_linalg_cond.py | 24 +- .../tests/unittests/test_linalg_lstsq_op.py | 171 +- .../tests/unittests/test_linalg_pinv_op.py | 106 +- .../fluid/tests/unittests/test_linear.py | 40 +- .../unittests/test_linear_chain_crf_op.py | 118 +- .../tests/unittests/test_linear_interp_op.py | 366 +- .../unittests/test_linear_interp_v2_op.py | 411 +- .../fluid/tests/unittests/test_linspace.py | 24 +- .../unittests/test_listen_and_serv_op.py | 44 +- .../fluid/tests/unittests/test_load_op.py | 22 +- .../fluid/tests/unittests/test_load_op_xpu.py | 19 +- .../test_load_state_dict_from_old_format.py | 117 +- .../unittests/test_load_vars_shape_check.py | 1 - .../unittests/test_locality_aware_nms_op.py | 417 +- .../tests/unittests/test_lod_append_op.py | 33 +- .../unittests/test_lod_array_length_op.py | 5 +- .../tests/unittests/test_lod_rank_table.py | 7 +- .../tests/unittests/test_lod_reset_op.py | 23 +- .../tests/unittests/test_lod_tensor_array.py | 10 +- .../unittests/test_lod_tensor_array_ops.py | 142 +- .../fluid/tests/unittests/test_log_loss_op.py | 7 +- .../fluid/tests/unittests/test_log_softmax.py | 52 +- .../tests/unittests/test_logcumsumexp_op.py | 90 +- .../fluid/tests/unittests/test_logical_op.py | 149 +- .../fluid/tests/unittests/test_logit_op.py | 32 +- .../fluid/tests/unittests/test_logspace.py | 38 +- .../fluid/tests/unittests/test_logsumexp.py | 37 +- .../fluid/tests/unittests/test_lookahead.py | 75 +- .../unittests/test_lookup_table_bf16_op.py | 132 +- .../unittests/test_lookup_table_dequant_op.py | 1 - .../tests/unittests/test_lookup_table_op.py | 109 +- .../unittests/test_lookup_table_v2_bf16_op.py | 50 +- .../unittests/test_lookup_table_v2_op.py | 71 +- .../tests/unittests/test_lr_scheduler.py | 818 +- .../fluid/tests/unittests/test_lrn_op.py | 190 +- .../tests/unittests/test_lstm_cudnn_op.py | 348 +- .../fluid/tests/unittests/test_lstm_op.py | 245 +- .../tests/unittests/test_lstm_unit_op.py | 66 +- .../fluid/tests/unittests/test_lstmp_op.py | 316 +- .../fluid/tests/unittests/test_lu_op.py | 67 +- .../tests/unittests/test_lu_unpack_op.py | 77 +- .../fluid/tests/unittests/test_manual_seed.py | 1 - .../unittests/test_margin_cross_entropy_op.py | 332 +- .../unittests/test_margin_rank_loss_op.py | 31 +- .../fluid/tests/unittests/test_marker_op.py | 3 +- .../tests/unittests/test_masked_select_op.py | 29 +- .../unittests/test_match_matrix_tensor_op.py | 24 +- .../tests/unittests/test_math_op_patch.py | 193 +- .../unittests/test_math_op_patch_var_base.py | 245 +- .../fluid/tests/unittests/test_matmul_op.py | 140 +- .../unittests/test_matmul_op_with_head.py | 124 +- .../tests/unittests/test_matmul_v2_op.py | 256 +- .../tests/unittests/test_matrix_nms_op.py | 262 +- .../tests/unittests/test_matrix_power_op.py | 86 +- .../tests/unittests/test_matrix_rank_op.py | 101 +- .../unittests/test_max_min_amax_amin_op.py | 50 +- .../fluid/tests/unittests/test_max_op.py | 40 +- .../fluid/tests/unittests/test_maximum_op.py | 57 +- .../fluid/tests/unittests/test_maxout_op.py | 26 +- .../fluid/tests/unittests/test_mean_iou.py | 73 +- .../fluid/tests/unittests/test_mean_op.py | 112 +- .../fluid/tests/unittests/test_median.py | 15 +- .../fluid/tests/unittests/test_memcpy_op.py | 151 +- .../tests/unittests/test_memory_analysis.py | 17 +- .../test_memory_reuse_exclude_feed_var.py | 16 +- .../tests/unittests/test_memory_usage.py | 16 +- .../unittests/test_merge_selectedrows_op.py | 1 - .../tests/unittests/test_merged_adam_op.py | 167 +- .../unittests/test_merged_momentum_op.py | 309 +- .../fluid/tests/unittests/test_meshgrid_op.py | 191 +- .../fluid/tests/unittests/test_min_op.py | 40 +- .../unittests/test_mine_hard_examples_op.py | 48 +- .../fluid/tests/unittests/test_minimum_op.py | 57 +- .../fluid/tests/unittests/test_minus_op.py | 3 +- .../test_mix_precision_all_reduce_fuse.py | 38 +- .../fluid/tests/unittests/test_mode_op.py | 41 +- .../tests/unittests/test_modelaverage.py | 136 +- .../unittests/test_modified_huber_loss_op.py | 24 +- .../fluid/tests/unittests/test_momentum_op.py | 396 +- .../fluid/tests/unittests/test_monitor.py | 24 +- .../fluid/tests/unittests/test_mse_loss.py | 239 +- .../fluid/tests/unittests/test_mul_nn_grad.py | 43 +- .../fluid/tests/unittests/test_mul_op.py | 123 +- .../tests/unittests/test_multi_dot_op.py | 100 +- .../test_multi_label_soft_margin_loss.py | 231 +- .../tests/unittests/test_multiclass_nms_op.py | 464 +- .../unittests/test_multihead_attention.py | 57 +- .../tests/unittests/test_multimarginloss.py | 342 +- .../tests/unittests/test_multinomial_op.py | 82 +- .../tests/unittests/test_multiplex_op.py | 28 +- .../fluid/tests/unittests/test_multiply.py | 32 +- .../test_multiprocess_dataloader_dataset.py | 171 +- .../test_multiprocess_dataloader_dynamic.py | 102 +- .../test_multiprocess_dataloader_exception.py | 104 +- ...ess_dataloader_iterable_dataset_dynamic.py | 103 +- ...ocess_dataloader_iterable_dataset_split.py | 37 +- ...cess_dataloader_iterable_dataset_static.py | 157 +- .../test_multiprocess_dataloader_static.py | 196 +- .../test_multiprocess_reader_exception.py | 44 +- .../fluid/tests/unittests/test_mv_op.py | 33 +- .../test_naive_best_fit_gpu_memory_limit.py | 7 +- .../fluid/tests/unittests/test_name_scope.py | 1 - .../fluid/tests/unittests/test_nan_inf.py | 18 +- .../fluid/tests/unittests/test_nanmean_api.py | 21 +- .../fluid/tests/unittests/test_nanmedian.py | 66 +- .../fluid/tests/unittests/test_nansum_api.py | 47 +- .../paddle/fluid/tests/unittests/test_nce.py | 274 +- .../tests/unittests/test_nearest_interp_op.py | 216 +- .../unittests/test_nearest_interp_v2_op.py | 365 +- .../fluid/tests/unittests/test_neg_op.py | 12 +- .../unittests/test_network_with_dtype.py | 6 +- .../fluid/tests/unittests/test_newprofiler.py | 153 +- .../unittests/test_newprofiler_helper.py | 1 - .../fluid/tests/unittests/test_nll_loss.py | 813 +- .../fluid/tests/unittests/test_nms_op.py | 4 +- .../tests/unittests/test_nn_dice_loss.py | 15 +- .../test_nn_functional_embedding_dygraph.py | 6 +- .../test_nn_functional_embedding_static.py | 94 +- .../unittests/test_nn_functional_hot_op.py | 86 +- .../fluid/tests/unittests/test_nn_grad.py | 307 +- .../unittests/test_nn_margin_rank_loss.py | 159 +- .../tests/unittests/test_nn_matmul_v2_grad.py | 428 +- .../test_nn_quant_functional_layers.py | 3 +- .../tests/unittests/test_nn_sigmoid_op.py | 2 - .../fluid/tests/unittests/test_nonzero_api.py | 25 +- .../fluid/tests/unittests/test_norm_all.py | 449 +- .../tests/unittests/test_norm_nn_grad.py | 126 +- .../fluid/tests/unittests/test_norm_op.py | 35 +- .../fluid/tests/unittests/test_normal.py | 68 +- .../unittests/test_normalization_wrapper.py | 55 +- .../fluid/tests/unittests/test_normalize.py | 10 +- .../tests/unittests/test_npair_loss_op.py | 191 +- .../tests/unittests/test_number_count_op.py | 19 +- .../fluid/tests/unittests/test_numel_op.py | 34 +- .../fluid/tests/unittests/test_one_hot_op.py | 85 +- .../tests/unittests/test_one_hot_v2_op.py | 91 +- .../fluid/tests/unittests/test_ones_like.py | 36 +- .../fluid/tests/unittests/test_ones_op.py | 11 +- .../fluid/tests/unittests/test_onnx_export.py | 17 +- .../unittests/test_op_function_generator.py | 5 +- .../tests/unittests/test_op_name_conflict.py | 95 +- .../tests/unittests/test_op_support_gpu.py | 6 +- .../fluid/tests/unittests/test_op_version.py | 22 +- .../fluid/tests/unittests/test_operator.py | 33 +- .../tests/unittests/test_operator_desc.py | 95 +- .../fluid/tests/unittests/test_ops_nms.py | 132 +- .../fluid/tests/unittests/test_optimizer.py | 1359 ++-- .../unittests/test_optimizer_for_varbase.py | 37 +- .../tests/unittests/test_optimizer_grad.py | 107 +- .../test_optimizer_in_control_flow.py | 128 +- .../fluid/tests/unittests/test_outer.py | 48 +- .../tests/unittests/test_overlap_add_op.py | 13 +- .../fluid/tests/unittests/test_pad2d_op.py | 51 +- .../fluid/tests/unittests/test_pad3d_op.py | 568 +- .../tests/unittests/test_pad_constant_like.py | 42 +- .../fluid/tests/unittests/test_pad_op.py | 55 +- .../test_paddle_imperative_double_grad.py | 192 +- .../unittests/test_paddle_multiprocessing.py | 35 +- .../tests/unittests/test_paddle_save_load.py | 504 +- .../unittests/test_paddle_save_load_binary.py | 130 +- .../tests/unittests/test_pairwise_distance.py | 451 +- .../test_parallel_dygraph_dataparallel.py | 89 +- ...t_parallel_dygraph_dataparallel_cpuonly.py | 43 +- ...graph_sparse_embedding_diff_length_gloo.py | 11 +- ..._parallel_dygraph_sparse_embedding_gloo.py | 22 +- ...graph_sparse_embedding_over_height_gloo.py | 4 +- .../test_parallel_dygraph_transformer_gloo.py | 22 +- ..._parallel_dygraph_unused_variables_gloo.py | 33 +- .../unittests/test_parallel_executor_crf.py | 257 +- .../test_parallel_executor_drop_scope.py | 21 +- .../test_parallel_executor_dry_run.py | 40 +- ..._parallel_executor_feed_persistable_var.py | 34 +- .../test_parallel_executor_fetch_feed.py | 122 +- ...st_parallel_executor_fetch_isolated_var.py | 45 +- ...test_parallel_executor_fix_op_run_order.py | 23 +- ...el_executor_inference_feed_partial_data.py | 82 +- .../unittests/test_parallel_executor_mnist.py | 146 +- .../unittests/test_parallel_executor_pg.py | 42 +- .../test_parallel_executor_profiler.py | 11 +- .../test_parallel_executor_run_cinn.py | 36 +- ...arallel_executor_run_load_infer_program.py | 63 +- ...st_parallel_executor_seresnext_base_cpu.py | 19 +- ...st_parallel_executor_seresnext_base_gpu.py | 19 +- ...utor_seresnext_with_fuse_all_reduce_cpu.py | 14 +- ...utor_seresnext_with_fuse_all_reduce_gpu.py | 15 +- ...llel_executor_seresnext_with_reduce_cpu.py | 39 +- ...llel_executor_seresnext_with_reduce_gpu.py | 11 +- ...test_parallel_executor_test_while_train.py | 58 +- .../test_parallel_executor_transformer.py | 143 +- .../fluid/tests/unittests/test_parameter.py | 64 +- .../tests/unittests/test_partial_concat_op.py | 20 +- .../tests/unittests/test_partial_sum_op.py | 14 +- .../tests/unittests/test_pipeline_parallel.py | 2 - .../tests/unittests/test_pixel_shuffle.py | 125 +- .../tests/unittests/test_pixel_unshuffle.py | 136 +- .../fluid/tests/unittests/test_poisson_op.py | 243 +- .../unittests/test_polygon_box_transform.py | 40 +- .../fluid/tests/unittests/test_pool1d_api.py | 343 +- .../fluid/tests/unittests/test_pool2d_api.py | 628 +- .../fluid/tests/unittests/test_pool2d_op.py | 904 ++- .../fluid/tests/unittests/test_pool3d_api.py | 547 +- .../fluid/tests/unittests/test_pool3d_op.py | 797 +- .../fluid/tests/unittests/test_pool_max_op.py | 50 +- .../test_positive_negative_pair_op.py | 44 +- .../paddle/fluid/tests/unittests/test_pow.py | 35 +- .../test_pow2_decay_with_linear_warmup_op.py | 27 +- .../unittests/test_precision_recall_op.py | 55 +- .../fluid/tests/unittests/test_prelu_op.py | 182 +- .../fluid/tests/unittests/test_print_op.py | 41 +- .../tests/unittests/test_prior_box_op.py | 110 +- .../fluid/tests/unittests/test_prod_op.py | 140 +- .../fluid/tests/unittests/test_profiler.py | 139 +- .../unittests/test_profiler_statistic.py | 1035 ++- .../fluid/tests/unittests/test_program.py | 103 +- .../tests/unittests/test_program_code.py | 21 +- .../unittests/test_program_prune_backward.py | 161 +- .../tests/unittests/test_program_to_string.py | 13 +- .../fluid/tests/unittests/test_protobuf.py | 1 - .../tests/unittests/test_protobuf_descs.py | 9 +- .../unittests/test_proximal_adagrad_op.py | 1 - .../tests/unittests/test_proximal_gd_op.py | 1 - .../tests/unittests/test_prroi_pool_op.py | 212 +- .../fluid/tests/unittests/test_prune.py | 694 +- .../test_prune_gate_by_capacity_op.py | 68 +- .../tests/unittests/test_ps_dispatcher.py | 17 +- .../tests/unittests/test_psroi_pool_op.py | 242 +- .../unittests/test_pull_gpups_sparse_op.py | 20 +- .../tests/unittests/test_put_along_axis_op.py | 61 +- .../fluid/tests/unittests/test_py_func_op.py | 134 +- .../unittests/test_py_reader_combination.py | 43 +- .../unittests/test_py_reader_error_msg.py | 44 +- .../test_py_reader_lod_level_share.py | 5 +- .../unittests/test_py_reader_pin_memory.py | 52 +- .../unittests/test_py_reader_push_pop.py | 44 +- .../unittests/test_py_reader_return_list.py | 36 +- .../test_py_reader_sample_generator.py | 40 +- .../test_py_reader_using_executor.py | 130 +- .../fluid/tests/unittests/test_pylayer_op.py | 137 +- .../tests/unittests/test_pyramid_hash_op.py | 11 +- .../test_python_bf16_numpy_datatype.py | 1 - .../fluid/tests/unittests/test_qr_op.py | 72 +- .../test_quantile_and_nanquantile.py | 81 +- .../fluid/tests/unittests/test_query_op.py | 1 - .../fluid/tests/unittests/test_queue.py | 67 +- .../fluid/tests/unittests/test_rad2deg.py | 22 +- .../fluid/tests/unittests/test_rand_op.py | 20 +- .../tests/unittests/test_randint_like.py | 210 +- .../fluid/tests/unittests/test_randint_op.py | 75 +- .../fluid/tests/unittests/test_randn_op.py | 33 +- .../tests/unittests/test_random_crop_op.py | 21 +- .../tests/unittests/test_random_routing_op.py | 24 +- .../fluid/tests/unittests/test_random_seed.py | 171 +- .../fluid/tests/unittests/test_randperm_op.py | 183 +- .../fluid/tests/unittests/test_range.py | 13 +- .../tests/unittests/test_rank_attention_op.py | 61 +- .../tests/unittests/test_rank_loss_op.py | 9 +- .../unittests/test_raw_program_optimizer.py | 31 +- .../tests/unittests/test_reader_reset.py | 29 +- .../tests/unittests/test_real_imag_op.py | 46 +- .../tests/unittests/test_recurrent_op.py | 261 +- .../fluid/tests/unittests/test_reduce_op.py | 298 +- .../tests/unittests/test_reducescatter.py | 6 +- .../tests/unittests/test_reducescatter_api.py | 1 - .../fluid/tests/unittests/test_registry.py | 1 - .../fluid/tests/unittests/test_regularizer.py | 204 +- .../tests/unittests/test_regularizer_api.py | 109 +- .../fluid/tests/unittests/test_renorm_op.py | 78 +- .../unittests/test_reorder_lod_tensor.py | 124 +- .../unittests/test_repeat_interleave_op.py | 76 +- .../tests/unittests/test_require_version.py | 56 +- .../test_reset_grad_inplace_version.py | 16 +- .../fluid/tests/unittests/test_reshape_op.py | 105 +- .../unittests/test_resnet50_with_cinn.py | 45 +- .../tests/unittests/test_retain_graph.py | 79 +- .../test_retinanet_detection_output.py | 393 +- .../fluid/tests/unittests/test_reverse_op.py | 93 +- .../fluid/tests/unittests/test_rmsprop_op.py | 251 +- .../tests/unittests/test_rnn_cell_api.py | 633 +- .../tests/unittests/test_rnn_decode_api.py | 707 +- .../unittests/test_rnn_memory_helper_op.py | 101 +- .../fluid/tests/unittests/test_rnn_op.py | 81 +- .../tests/unittests/test_roi_align_op.py | 140 +- .../test_roi_perspective_transform_op.py | 235 +- .../fluid/tests/unittests/test_roi_pool_op.py | 83 +- .../fluid/tests/unittests/test_roll_op.py | 53 +- .../fluid/tests/unittests/test_rot90_op.py | 99 +- .../fluid/tests/unittests/test_row_conv_op.py | 83 +- .../unittests/test_rpn_target_assign_op.py | 695 +- .../fluid/tests/unittests/test_rrelu_op.py | 225 +- .../paddle/fluid/tests/unittests/test_run.py | 18 +- ...est_run_fluid_by_module_or_command_line.py | 2 - .../tests/unittests/test_run_program_op.py | 229 +- .../test_runtime_and_compiletime_exception.py | 9 +- .../tests/unittests/test_sample_logits_op.py | 68 +- .../tests/unittests/test_sampling_id_op.py | 1 - ...est_save_inference_model_conditional_op.py | 74 +- .../unittests/test_save_model_without_var.py | 22 +- .../unittests/test_saved_tensors_hooks.py | 4 - .../fluid/tests/unittests/test_scale_op.py | 70 +- .../test_scaled_dot_product_attention.py | 96 +- .../tests/unittests/test_scatter_nd_op.py | 226 +- .../fluid/tests/unittests/test_scatter_op.py | 145 +- .../fluid/tests/unittests/test_scope.py | 4 +- .../tests/unittests/test_searchsorted_op.py | 130 +- .../fluid/tests/unittests/test_seed_op.py | 9 +- .../fluid/tests/unittests/test_segment_ops.py | 91 +- .../unittests/test_select_input_output_op.py | 48 +- .../tests/unittests/test_selected_rows.py | 14 +- .../fluid/tests/unittests/test_selu_op.py | 32 +- .../tests/unittests/test_set_bool_attr.py | 13 +- .../tests/unittests/test_set_value_op.py | 457 +- .../fluid/tests/unittests/test_sgd_op.py | 229 +- .../fluid/tests/unittests/test_sgd_op_bf16.py | 203 +- .../paddle/fluid/tests/unittests/test_sgn.py | 27 +- .../fluid/tests/unittests/test_shape_op.py | 4 - .../tests/unittests/test_shard_index_op.py | 6 +- .../tests/unittests/test_share_data_op.py | 2 - .../tests/unittests/test_shrink_rnn_memory.py | 37 +- .../tests/unittests/test_shuffle_batch_op.py | 7 +- .../unittests/test_shuffle_channel_op.py | 10 +- ...st_sigmoid_cross_entropy_with_logits_op.py | 155 +- .../unittests/test_sigmoid_focal_loss.py | 194 +- .../unittests/test_sigmoid_focal_loss_op.py | 132 +- .../fluid/tests/unittests/test_sign_op.py | 75 +- .../fluid/tests/unittests/test_signal.py | 370 +- .../unittests/test_similarity_focus_op.py | 80 +- .../tests/unittests/test_simple_rnn_op.py | 53 +- .../fluid/tests/unittests/test_size_op.py | 34 +- .../fluid/tests/unittests/test_slice_op.py | 346 +- .../fluid/tests/unittests/test_slice_var.py | 43 +- .../tests/unittests/test_smooth_l1_loss.py | 109 +- .../tests/unittests/test_smooth_l1_loss_op.py | 83 +- .../tests/unittests/test_soft_margin_loss.py | 133 +- .../fluid/tests/unittests/test_softmax2d.py | 33 +- .../unittests/test_softmax_mask_fuse_op.py | 35 +- ...est_softmax_mask_fuse_upper_triangle_op.py | 31 +- .../fluid/tests/unittests/test_softmax_op.py | 178 +- .../test_softmax_with_cross_entropy_op.py | 294 +- .../fluid/tests/unittests/test_solve_op.py | 242 +- .../fluid/tests/unittests/test_sort_op.py | 32 +- .../tests/unittests/test_space_to_depth_op.py | 37 +- .../tests/unittests/test_sparse_addmm_op.py | 40 +- .../unittests/test_sparse_attention_op.py | 220 +- .../tests/unittests/test_sparse_conv_op.py | 192 +- .../tests/unittests/test_sparse_copy_op.py | 1 - .../unittests/test_sparse_elementwise_op.py | 122 +- .../test_sparse_fused_attention_op.py | 76 +- .../unittests/test_sparse_is_same_shape.py | 149 +- .../tests/unittests/test_sparse_matmul_op.py | 86 +- .../tests/unittests/test_sparse_model.py | 11 +- .../unittests/test_sparse_momentum_op.py | 126 +- .../tests/unittests/test_sparse_mv_op.py | 46 +- .../tests/unittests/test_sparse_norm_op.py | 106 +- .../tests/unittests/test_sparse_pooling_op.py | 34 +- .../tests/unittests/test_sparse_reshape_op.py | 57 +- .../tests/unittests/test_sparse_softmax_op.py | 43 +- .../unittests/test_sparse_transpose_op.py | 19 +- .../tests/unittests/test_sparse_unary_op.py | 45 +- .../tests/unittests/test_sparse_utils_op.py | 224 +- .../test_spawn_and_init_parallel_env.py | 18 +- .../tests/unittests/test_spectral_norm_op.py | 43 +- .../test_split_and_merge_lod_tensor_op.py | 241 +- .../fluid/tests/unittests/test_split_op.py | 169 +- .../tests/unittests/test_split_program.py | 59 +- .../fluid/tests/unittests/test_splits_api.py | 50 +- .../fluid/tests/unittests/test_spp_op.py | 32 +- .../tests/unittests/test_square_error_cost.py | 19 +- .../unittests/test_squared_l2_distance_op.py | 15 +- .../unittests/test_squared_l2_norm_op.py | 14 +- .../fluid/tests/unittests/test_squeeze2_op.py | 10 +- .../fluid/tests/unittests/test_squeeze_op.py | 66 +- .../fluid/tests/unittests/test_stack_op.py | 81 +- ...t_static_model_parallel_fused_attention.py | 12 +- ...static_model_parallel_fused_feedforward.py | 12 +- ..._model_parallel_fused_multi_transformer.py | 5 +- .../tests/unittests/test_static_save_load.py | 1240 +-- .../unittests/test_static_save_load_bf16.py | 97 +- .../unittests/test_static_save_load_large.py | 39 +- ...tatic_shape_inferrence_for_shape_tensor.py | 7 +- .../fluid/tests/unittests/test_std_layer.py | 17 +- .../fluid/tests/unittests/test_stft_op.py | 13 +- .../tests/unittests/test_strided_slice_op.py | 341 +- .../fluid/tests/unittests/test_subtract_op.py | 57 +- .../fluid/tests/unittests/test_sum_op.py | 257 +- .../fluid/tests/unittests/test_svd_op.py | 201 +- .../fluid/tests/unittests/test_switch.py | 24 +- .../tests/unittests/test_switch_autotune.py | 47 +- .../fluid/tests/unittests/test_switch_case.py | 319 +- .../unittests/test_sync_batch_norm_op.py | 164 +- .../paddle/fluid/tests/unittests/test_take.py | 144 +- .../unittests/test_take_along_axis_op.py | 28 +- .../tests/unittests/test_target_assign_op.py | 65 +- .../tests/unittests/test_tdm_child_op.py | 33 +- .../tests/unittests/test_tdm_sampler_op.py | 109 +- .../test_teacher_student_sigmoid_loss_op.py | 37 +- .../tests/unittests/test_temporal_shift_op.py | 53 +- .../fluid/tests/unittests/test_tensor.py | 60 +- .../unittests/test_tensor_array_to_tensor.py | 96 +- .../tests/unittests/test_tensor_copy_from.py | 1 - .../tests/unittests/test_tensor_fill_.py | 16 +- .../unittests/test_tensor_fill_diagonal_.py | 134 +- .../test_tensor_fill_diagonal_tensor.py | 125 +- .../test_tensor_fill_diagonal_tensor_.py | 127 +- .../unittests/test_tensor_register_hook.py | 184 +- ...st_tensor_scalar_type_promotion_dynamic.py | 1 - ...est_tensor_scalar_type_promotion_static.py | 1 - .../tests/unittests/test_tensor_to_list.py | 6 +- .../tests/unittests/test_tensor_to_numpy.py | 12 +- .../unittests/test_tensor_type_promotion.py | 21 +- .../fluid/tests/unittests/test_tensor_uva.py | 18 +- .../tests/unittests/test_tensor_zero_.py | 6 +- .../fluid/tests/unittests/test_tensordot.py | 228 +- .../fluid/tests/unittests/test_tf32_cublas.py | 2 - .../fluid/tests/unittests/test_tf32_cudnn.py | 1 - .../fluid/tests/unittests/test_tile_op.py | 68 +- .../fluid/tests/unittests/test_top_k_op.py | 1 - .../fluid/tests/unittests/test_top_k_v2_op.py | 299 +- .../fluid/tests/unittests/test_trace_op.py | 34 +- .../unittests/test_traced_layer_err_msg.py | 145 +- .../fluid/tests/unittests/test_trainable.py | 36 +- .../tests/unittests/test_trainer_desc.py | 2 +- .../tests/unittests/test_transfer_dtype_op.py | 15 +- .../unittests/test_transfer_layout_op.py | 42 +- .../tests/unittests/test_transformer_api.py | 934 ++- .../tests/unittests/test_translated_layer.py | 71 +- .../tests/unittests/test_transpose_op.py | 121 +- .../tests/unittests/test_tree_conv_op.py | 147 +- .../unittests/test_triangular_solve_op.py | 62 +- .../tests/unittests/test_tril_indices_op.py | 48 +- .../tests/unittests/test_tril_triu_op.py | 78 +- .../unittests/test_trilinear_interp_op.py | 297 +- .../unittests/test_trilinear_interp_v2_op.py | 364 +- .../unittests/test_triplet_margin_loss.py | 423 +- .../test_triplet_margin_with_distance_loss.py | 292 +- .../tests/unittests/test_triu_indices_op.py | 28 +- .../fluid/tests/unittests/test_trunc_op.py | 4 - .../test_truncated_gaussian_random_op.py | 24 +- .../fluid/tests/unittests/test_unbind_op.py | 44 +- .../fluid/tests/unittests/test_unfold_op.py | 77 +- .../unittests/test_uniform_random_bf16_op.py | 129 +- .../test_uniform_random_inplace_op.py | 24 +- .../tests/unittests/test_uniform_random_op.py | 274 +- .../fluid/tests/unittests/test_unique.py | 161 +- .../unittests/test_unique_consecutive_op.py | 70 +- .../fluid/tests/unittests/test_unique_name.py | 2 - .../unittests/test_unique_with_counts.py | 49 +- .../fluid/tests/unittests/test_unpool1d_op.py | 143 +- .../fluid/tests/unittests/test_unpool3d_op.py | 312 +- .../fluid/tests/unittests/test_unpool_op.py | 297 +- .../tests/unittests/test_unsqueeze2_op.py | 64 +- .../tests/unittests/test_unsqueeze_op.py | 112 +- .../fluid/tests/unittests/test_unstack_op.py | 5 - .../unittests/test_update_loss_scaling_op.py | 199 +- .../fluid/tests/unittests/test_var_base.py | 403 +- .../fluid/tests/unittests/test_var_conv_2d.py | 138 +- .../fluid/tests/unittests/test_var_info.py | 4 +- .../fluid/tests/unittests/test_variable.py | 558 +- .../tests/unittests/test_variance_layer.py | 17 +- .../fluid/tests/unittests/test_version.py | 4 +- .../test_view_op_reuse_allocation.py | 20 +- .../tests/unittests/test_viterbi_decode_op.py | 35 +- .../fluid/tests/unittests/test_warpctc_op.py | 405 +- .../tests/unittests/test_weight_decay.py | 129 +- .../unittests/test_weight_normalization.py | 72 +- .../fluid/tests/unittests/test_where_index.py | 25 +- .../fluid/tests/unittests/test_where_op.py | 147 +- .../tests/unittests/test_while_loop_op.py | 286 +- .../fluid/tests/unittests/test_while_op.py | 61 +- .../fluid/tests/unittests/test_yolo_box_op.py | 195 +- .../tests/unittests/test_yolov3_loss_op.py | 308 +- .../tests/unittests/test_zero_dim_shape.py | 38 +- .../fluid/tests/unittests/test_zeropad2d.py | 30 +- .../tests/unittests/test_zeros_like_op.py | 48 +- .../fluid/tests/unittests/test_zeros_op.py | 14 +- .../paddle/fluid/tests/unittests/testsuite.py | 51 +- .../unittests/tokenizer/bert_tokenizer.py | 182 +- .../unittests/tokenizer/tokenizer_utils.py | 505 +- .../tests/unittests/transformer_model.py | 370 +- python/paddle/fluid/tests/unittests/utils.py | 2 - .../white_list/check_shape_white_list.py | 21 +- .../compile_vs_runtime_white_list.py | 32 +- .../white_list/no_grad_set_white_list.py | 2 - .../white_list/op_accuracy_white_list.py | 138 +- .../white_list/op_threshold_white_list.py | 85 +- .../xpu/collective_allgather_op_xpu.py | 33 +- .../xpu/collective_allreduce_op_xpu.py | 34 +- .../xpu/collective_identity_op_xpu.py | 23 +- .../unittests/xpu/get_test_cover_info.py | 80 +- .../unittests/xpu/test_accuracy_op_xpu.py | 15 +- .../unittests/xpu/test_activation_op_xpu.py | 177 +- .../tests/unittests/xpu/test_adam_op_xpu.py | 162 +- .../tests/unittests/xpu/test_adamw_op_xpu.py | 115 +- .../xpu/test_affine_channel_op_xpu.py | 34 +- .../test_amp_check_finite_and_scale_op_xpu.py | 11 +- .../unittests/xpu/test_arg_max_op_xpu.py | 51 +- .../unittests/xpu/test_argsort_op_xpu.py | 39 +- .../tests/unittests/xpu/test_assign_op_xpu.py | 1 + .../unittests/xpu/test_assign_value_op_xpu.py | 31 +- .../unittests/xpu/test_batch_norm_op_xpu.py | 230 +- .../unittests/xpu/test_bce_loss_op_xpu.py | 17 +- .../xpu/test_bilinear_interp_v2_op_xpu.py | 157 +- .../unittests/xpu/test_bitwise_op_xpu.py | 79 +- .../tests/unittests/xpu/test_bmm_op_xpu.py | 12 +- .../unittests/xpu/test_c_embedding_op_xpu.py | 6 +- .../tests/unittests/xpu/test_cast_op_xpu.py | 21 +- .../unittests/xpu/test_clip_by_norm_op_xpu.py | 8 +- .../tests/unittests/xpu/test_clip_op_xpu.py | 37 +- .../xpu/test_coalesce_tensor_op_xpu.py | 50 +- .../xpu/test_collective_allgather_xpu.py | 23 +- .../xpu/test_collective_allreduce_xpu.py | 23 +- .../unittests/xpu/test_collective_base_xpu.py | 240 +- .../xpu/test_collective_identity_xpu.py | 23 +- .../unittests/xpu/test_compare_op_xpu.py | 47 +- .../tests/unittests/xpu/test_concat_op_xpu.py | 33 +- .../tests/unittests/xpu/test_conv2d_op_xpu.py | 289 +- .../xpu/test_conv2d_transpose_op_xpu.py | 96 +- .../xpu/test_deformable_conv_op_xpu.py | 211 +- .../xpu/test_depthwise_conv2d_op_xpu.py | 24 +- .../unittests/xpu/test_device_guard_xpu.py | 99 +- .../unittests/xpu/test_dropout_op_xpu.py | 76 +- .../xpu/test_elementwise_add_op_xpu.py | 98 +- .../xpu/test_elementwise_add_op_xpu_kp.py | 169 +- .../xpu/test_elementwise_div_op_xpu.py | 132 +- .../xpu/test_elementwise_floordiv_op_xpu.py | 12 +- .../xpu/test_elementwise_max_op_xpu.py | 101 +- .../xpu/test_elementwise_min_op_xpu.py | 101 +- .../xpu/test_elementwise_mod_op_xpu.py | 19 +- .../xpu/test_elementwise_mul_op_xpu.py | 102 +- .../xpu/test_elementwise_pow_op_xpu.py | 71 +- .../xpu/test_elementwise_sub_op_xpu.py | 70 +- .../tests/unittests/xpu/test_empty_op_xpu.py | 44 +- .../unittests/xpu/test_expand_as_v2_op_xpu.py | 42 +- .../unittests/xpu/test_expand_v2_op_xpu.py | 63 +- .../xpu/test_fill_any_like_op_xpu.py | 12 +- .../xpu/test_fill_constant_op_xpu.py | 79 +- .../unittests/xpu/test_flatten2_op_xpu.py | 13 +- .../test_flatten_contiguous_range_op_xpu.py | 109 +- .../unittests/xpu/test_flatten_op_xpu.py | 11 +- .../xpu/test_fleet_exe_dist_model_run_xpu.py | 35 +- .../test_fused_gemm_epilogue_grad_op_xpu.py | 39 +- .../xpu/test_fused_gemm_epilogue_op_xpu.py | 76 +- .../test_fused_resnet_basic_block_op_xpu.py | 225 +- .../unittests/xpu/test_gather_nd_op_xpu.py | 55 +- .../tests/unittests/xpu/test_gather_op_xpu.py | 21 +- .../xpu/test_gaussian_random_op_xpu.py | 163 +- .../unittests/xpu/test_gen_bkcl_id_op.py | 18 +- .../xpu/test_generate_proposals_v2_op_xpu.py | 287 +- .../unittests/xpu/test_grid_sampler_op_xpu.py | 89 +- .../unittests/xpu/test_huber_loss_op_xpu.py | 36 +- .../unittests/xpu/test_index_select_op_xpu.py | 86 +- .../xpu/test_instance_norm_op_xpu.py | 24 +- .../xpu/test_iou_similarity_op_xpu.py | 20 +- .../unittests/xpu/test_label_smooth_op_xpu.py | 32 +- .../tests/unittests/xpu/test_lamb_op_xpu.py | 61 +- .../unittests/xpu/test_layer_norm_op_xpu.py | 37 +- .../unittests/xpu/test_log_loss_op_xpu.py | 6 +- .../unittests/xpu/test_log_softmax_op_xpu.py | 29 +- .../unittests/xpu/test_logical_op_xpu.py | 72 +- .../unittests/xpu/test_logsumexp_op_xpu.py | 10 +- .../xpu/test_lookup_table_v2_op_xpu.py | 60 +- .../xpu/test_masked_select_op_xpu.py | 36 +- .../tests/unittests/xpu/test_matmul_op_xpu.py | 228 +- .../unittests/xpu/test_matmul_v2_op_xpu.py | 33 +- .../tests/unittests/xpu/test_mean_op_xpu.py | 27 +- .../xpu/test_merged_momentum_op_xpu.py | 12 +- .../xpu/test_merged_momentum_op_xpu_base.py | 160 +- .../unittests/xpu/test_momentum_op_xpu.py | 48 +- .../tests/unittests/xpu/test_mul_op_xpu.py | 86 +- .../xpu/test_nearest_interp_v2_op_xpu.py | 143 +- .../unittests/xpu/test_one_hot_op_xpu.py | 46 +- .../unittests/xpu/test_one_hot_v2_op_xpu.py | 76 +- .../tests/unittests/xpu/test_p_norm_op_xpu.py | 25 +- .../tests/unittests/xpu/test_pool2d_op_xpu.py | 221 +- ...st_pow2_decay_with_linear_warmup_op_xpu.py | 27 +- .../unittests/xpu/test_prior_box_op_xpu.py | 104 +- .../tests/unittests/xpu/test_range_xpu.py | 21 +- .../unittests/xpu/test_reduce_all_op_xpu.py | 27 +- .../unittests/xpu/test_reduce_amax_op_xpu.py | 19 +- .../unittests/xpu/test_reduce_amin_op_xpu.py | 19 +- .../unittests/xpu/test_reduce_any_op_xpu.py | 24 +- .../unittests/xpu/test_reduce_max_op_xpu.py | 21 +- .../unittests/xpu/test_reduce_mean_op_xpu.py | 46 +- .../unittests/xpu/test_reduce_min_op_xpu.py | 21 +- .../unittests/xpu/test_reduce_prod_op_xpu.py | 51 +- .../unittests/xpu/test_reduce_sum_op_xpu.py | 21 +- .../unittests/xpu/test_refactor_op_xpu.py | 111 +- .../unittests/xpu/test_reshape2_op_xpu.py | 42 +- .../unittests/xpu/test_rmsprop_op_xpu.py | 57 +- .../tests/unittests/xpu/test_rnn_op_xpu.py | 94 +- .../unittests/xpu/test_roi_align_op_xpu.py | 139 +- .../unittests/xpu/test_sampling_id_op_xpu.py | 1 - .../tests/unittests/xpu/test_scale_op_xpu.py | 17 +- .../unittests/xpu/test_scatter_op_xpu.py | 40 +- .../xpu/test_sequence_conv_op_xpu.py | 302 +- .../xpu/test_sequence_unpad_op_xpu.py | 23 +- .../tests/unittests/xpu/test_sgd_op_xpu.py | 16 +- .../tests/unittests/xpu/test_shape_op_xpu.py | 14 +- ...igmoid_cross_entropy_with_logits_op_xpu.py | 152 +- .../tests/unittests/xpu/test_sign_op_xpu.py | 17 +- .../tests/unittests/xpu/test_slice_op_xpu.py | 39 +- .../unittests/xpu/test_softmax_op_xpu.py | 13 +- .../test_softmax_with_cross_entropy_op_xpu.py | 81 +- .../tests/unittests/xpu/test_split_op_xpu.py | 17 +- .../unittests/xpu/test_squeeze2_op_xpu.py | 13 +- .../unittests/xpu/test_squeeze_op_xpu.py | 18 +- .../tests/unittests/xpu/test_stack_op_xpu.py | 24 +- .../tests/unittests/xpu/test_sum_op_xpu.py | 35 +- .../xpu/test_temporal_shift_op_xpu.py | 30 +- .../tests/unittests/xpu/test_tile_op_xpu.py | 31 +- .../tests/unittests/xpu/test_top_k_op_xpu.py | 23 +- .../unittests/xpu/test_top_k_v2_op_xpu.py | 29 +- .../unittests/xpu/test_transpose_op_xpu.py | 20 +- .../unittests/xpu/test_tril_triu_op_xpu.py | 59 +- .../test_truncated_gaussian_random_op_xpu.py | 18 +- .../xpu/test_uniform_random_op_xpu.py | 7 +- .../unittests/xpu/test_unsqueeze2_op_xpu.py | 43 +- .../unittests/xpu/test_unsqueeze_op_xpu.py | 14 +- .../xpu/test_update_loss_scaling_op_xpu.py | 160 +- .../unittests/xpu/test_where_index_xpu.py | 36 +- .../tests/unittests/xpu/test_where_op_xpu.py | 64 +- .../tests/unittests/xpu/test_while_op_xpu.py | 32 +- .../tests/unittests/xpu/test_xpu_place.py | 1 - python/paddle/fluid/trainer_desc.py | 104 +- python/paddle/fluid/trainer_factory.py | 86 +- python/paddle/fluid/transpiler/__init__.py | 5 +- .../fluid/transpiler/ascend_transpiler.py | 43 +- python/paddle/fluid/transpiler/collective.py | 688 +- .../fluid/transpiler/details/checkport.py | 10 +- .../paddle/fluid/transpiler/details/ufind.py | 2 +- .../transpiler/details/vars_distributed.py | 128 +- .../fluid/transpiler/distribute_transpiler.py | 1361 ++-- .../fluid/transpiler/geo_sgd_transpiler.py | 179 +- .../memory_optimization_transpiler.py | 21 +- .../paddle/fluid/transpiler/ps_dispatcher.py | 4 +- python/paddle/fluid/unique_name.py | 1 + python/paddle/fluid/variable_index.py | 312 +- python/paddle/fluid/wrapped_decorator.py | 1 - python/paddle/framework/__init__.py | 15 +- python/paddle/framework/framework.py | 14 +- python/paddle/framework/io.py | 258 +- python/paddle/framework/random.py | 2 +- python/paddle/geometric/math.py | 132 +- .../geometric/message_passing/send_recv.py | 279 +- .../paddle/geometric/message_passing/utils.py | 41 +- python/paddle/geometric/reindex.py | 152 +- python/paddle/geometric/sampling/neighbors.py | 108 +- python/paddle/hapi/callbacks.py | 216 +- python/paddle/hapi/dynamic_flops.py | 69 +- python/paddle/hapi/hub.py | 67 +- python/paddle/hapi/model.py | 721 +- python/paddle/hapi/model_summary.py | 97 +- python/paddle/hapi/progressbar.py | 74 +- python/paddle/hapi/static_flops.py | 12 +- python/paddle/hub.py | 4 +- python/paddle/incubate/__init__.py | 8 +- python/paddle/incubate/asp/__init__.py | 19 +- python/paddle/incubate/autograd/__init__.py | 10 +- python/paddle/incubate/autograd/functional.py | 116 +- python/paddle/incubate/autograd/primapi.py | 54 +- python/paddle/incubate/autograd/primops.py | 263 +- python/paddle/incubate/autograd/primreg.py | 43 +- python/paddle/incubate/autograd/primrules.py | 437 +- python/paddle/incubate/autograd/primx.py | 103 +- python/paddle/incubate/autograd/utils.py | 3 +- python/paddle/incubate/autotune.py | 3 +- .../incubate/distributed/fleet/__init__.py | 5 +- .../distributed/models/moe/gate/base_gate.py | 1 - .../models/moe/gate/gshard_gate.py | 57 +- .../distributed/models/moe/gate/naive_gate.py | 9 +- .../models/moe/gate/switch_gate.py | 47 +- .../distributed/models/moe/grad_clip.py | 76 +- .../distributed/models/moe/moe_layer.py | 236 +- .../incubate/distributed/models/moe/utils.py | 42 +- .../incubate/multiprocessing/reductions.py | 55 +- python/paddle/incubate/nn/__init__.py | 6 +- .../nn/functional/fused_matmul_bias.py | 32 +- .../nn/functional/fused_transformer.py | 738 +- .../paddle/incubate/nn/layer/fused_linear.py | 35 +- .../incubate/nn/layer/fused_transformer.py | 566 +- python/paddle/incubate/operators/__init__.py | 14 +- .../incubate/operators/graph_khop_sampler.py | 125 +- .../incubate/operators/graph_reindex.py | 94 +- .../operators/graph_sample_neighbors.py | 117 +- .../incubate/operators/graph_send_recv.py | 109 +- .../paddle/incubate/operators/resnet_unit.py | 297 +- .../incubate/operators/softmax_mask_fuse.py | 11 +- .../softmax_mask_fuse_upper_triangle.py | 8 +- .../optimizer/distributed_fused_lamb.py | 272 +- .../incubate/optimizer/functional/bfgs.py | 72 +- .../incubate/optimizer/functional/lbfgs.py | 147 +- .../optimizer/functional/line_search.py | 165 +- .../incubate/optimizer/functional/utils.py | 27 +- python/paddle/incubate/optimizer/lookahead.py | 66 +- .../paddle/incubate/optimizer/modelaverage.py | 153 +- .../incubate/passes/fuse_resnet_unit_pass.py | 119 +- python/paddle/incubate/tensor/math.py | 168 +- python/paddle/incubate/xpu/resnet_block.py | 545 +- python/paddle/inference/__init__.py | 18 +- python/paddle/io/__init__.py | 21 +- python/paddle/jit/__init__.py | 11 +- .../paddle/jit/dy2static/convert_call_func.py | 4 +- .../paddle/jit/dy2static/convert_operators.py | 32 +- .../jit/dy2static/variable_trans_func.py | 6 +- python/paddle/jit/layer.py | 7 +- python/paddle/linalg.py | 4 +- python/paddle/metric/__init__.py | 9 +- python/paddle/metric/metrics.py | 86 +- python/paddle/nn/__init__.py | 36 +- python/paddle/nn/functional/activation.py | 466 +- python/paddle/nn/functional/common.py | 836 ++- python/paddle/nn/functional/conv.py | 912 ++- python/paddle/nn/functional/distance.py | 34 +- python/paddle/nn/functional/extension.py | 123 +- python/paddle/nn/functional/input.py | 89 +- python/paddle/nn/functional/loss.py | 1617 ++-- python/paddle/nn/functional/norm.py | 428 +- python/paddle/nn/functional/pooling.py | 1507 ++-- .../paddle/nn/functional/sparse_attention.py | 36 +- python/paddle/nn/functional/vision.py | 201 +- python/paddle/nn/initializer/__init__.py | 19 +- python/paddle/nn/initializer/assign.py | 14 +- python/paddle/nn/initializer/dirac.py | 221 +- python/paddle/nn/initializer/kaiming.py | 24 +- python/paddle/nn/initializer/orthogonal.py | 218 +- python/paddle/nn/initializer/uniform.py | 9 +- python/paddle/nn/initializer/xavier.py | 14 +- python/paddle/nn/layer/activation.py | 74 +- python/paddle/nn/layer/common.py | 499 +- python/paddle/nn/layer/container.py | 18 +- python/paddle/nn/layer/conv.py | 628 +- python/paddle/nn/layer/distance.py | 7 +- python/paddle/nn/layer/loss.py | 482 +- python/paddle/nn/layer/norm.py | 647 +- python/paddle/nn/layer/pooling.py | 386 +- python/paddle/nn/layer/rnn.py | 576 +- python/paddle/nn/layer/transformer.py | 448 +- python/paddle/nn/layer/vision.py | 33 +- python/paddle/nn/quant/functional_layers.py | 9 - python/paddle/nn/quant/lsq.py | 247 +- python/paddle/nn/quant/quant_layers.py | 768 +- python/paddle/nn/utils/__init__.py | 15 +- python/paddle/nn/utils/spectral_norm_hook.py | 56 +- .../paddle/nn/utils/transform_parameters.py | 51 +- python/paddle/nn/utils/weight_norm_hook.py | 32 +- python/paddle/onnx/export.py | 15 +- python/paddle/optimizer/__init__.py | 14 +- python/paddle/optimizer/adadelta.py | 85 +- python/paddle/optimizer/adagrad.py | 75 +- python/paddle/optimizer/adam.py | 432 +- python/paddle/optimizer/adamax.py | 161 +- python/paddle/optimizer/adamw.py | 355 +- python/paddle/optimizer/lamb.py | 227 +- python/paddle/optimizer/lr.py | 480 +- python/paddle/optimizer/momentum.py | 361 +- python/paddle/optimizer/optimizer.py | 552 +- python/paddle/optimizer/rmsprop.py | 111 +- python/paddle/optimizer/sgd.py | 107 +- python/paddle/profiler/__init__.py | 13 +- python/paddle/profiler/profiler.py | 242 +- python/paddle/profiler/profiler_statistic.py | 979 ++- python/paddle/profiler/statistic_helper.py | 27 +- python/paddle/profiler/timer.py | 95 +- python/paddle/profiler/utils.py | 36 +- python/paddle/quantization/__init__.py | 41 +- python/paddle/reader/decorator.py | 48 +- python/paddle/reader/tests/decorator_test.py | 37 +- python/paddle/signal.py | 290 +- python/paddle/sparse/binary.py | 7 +- python/paddle/sparse/creation.py | 83 +- .../paddle/sparse/nn/functional/activation.py | 7 +- python/paddle/sparse/nn/functional/conv.py | 155 +- python/paddle/sparse/nn/functional/pooling.py | 32 +- .../sparse/nn/functional/transformer.py | 25 +- python/paddle/sparse/nn/layer/conv.py | 199 +- python/paddle/sparse/nn/layer/norm.py | 180 +- python/paddle/sparse/nn/layer/pooling.py | 37 +- python/paddle/sparse/unary.py | 6 +- python/paddle/static/__init__.py | 65 +- python/paddle/static/amp/__init__.py | 8 +- python/paddle/static/input.py | 50 +- python/paddle/static/io.py | 169 +- python/paddle/static/nn/__init__.py | 2 +- python/paddle/static/nn/common.py | 68 +- python/paddle/static/quantization/__init__.py | 60 +- python/paddle/static/sparsity/__init__.py | 27 +- python/paddle/sysconfig.py | 2 + python/paddle/tensor/array.py | 85 +- python/paddle/tensor/attribute.py | 78 +- python/paddle/tensor/creation.py | 716 +- python/paddle/tensor/einsum.py | 181 +- .../paddle/tensor/layer_function_generator.py | 107 +- python/paddle/tensor/linalg.py | 1408 ++-- python/paddle/tensor/logic.py | 318 +- python/paddle/tensor/manipulation.py | 1578 ++-- python/paddle/tensor/math.py | 1267 ++-- python/paddle/tensor/ops.py | 177 +- python/paddle/tensor/random.py | 259 +- python/paddle/tensor/search.py | 369 +- python/paddle/tensor/stat.py | 134 +- python/paddle/tensor/to_string.py | 189 +- .../paddle/tests/dist_hapi_mnist_dynamic.py | 33 +- python/paddle/tests/dist_hapi_mnist_static.py | 33 +- .../tests/dist_hapi_pure_fp16_static.py | 22 +- python/paddle/tests/hapi_mnist_bf16_static.py | 47 +- python/paddle/tests/hubconf.py | 3 +- python/paddle/tests/test_async_read_write.py | 106 +- python/paddle/tests/test_audio_backend.py | 45 +- python/paddle/tests/test_audio_datasets.py | 60 +- python/paddle/tests/test_audio_functions.py | 190 +- .../paddle/tests/test_audio_logmel_feature.py | 145 +- python/paddle/tests/test_audio_mel_feature.py | 59 +- .../paddle/tests/test_callback_early_stop.py | 100 +- .../test_callback_reduce_lr_on_plateau.py | 88 +- python/paddle/tests/test_callback_visualdl.py | 17 +- python/paddle/tests/test_callbacks.py | 22 +- python/paddle/tests/test_dataset_cifar.py | 4 - python/paddle/tests/test_dataset_conll05.py | 1 - python/paddle/tests/test_dataset_imdb.py | 2 - python/paddle/tests/test_dataset_imikolov.py | 2 - python/paddle/tests/test_dataset_movielens.py | 2 - .../paddle/tests/test_dataset_uci_housing.py | 5 - python/paddle/tests/test_dataset_voc.py | 3 - python/paddle/tests/test_dataset_wmt.py | 20 +- python/paddle/tests/test_datasets.py | 17 +- python/paddle/tests/test_dist_hapi_model.py | 44 +- python/paddle/tests/test_dlpack.py | 36 +- python/paddle/tests/test_download.py | 41 +- python/paddle/tests/test_hapi_amp.py | 116 +- python/paddle/tests/test_hapi_hub.py | 148 +- python/paddle/tests/test_hapi_hub_model.py | 1 - python/paddle/tests/test_logger.py | 1 - python/paddle/tests/test_metrics.py | 127 +- python/paddle/tests/test_model.py | 607 +- python/paddle/tests/test_ops_roi_align.py | 71 +- python/paddle/tests/test_ops_roi_pool.py | 71 +- python/paddle/tests/test_pretrained_model.py | 1 - python/paddle/tests/test_progressbar.py | 3 +- python/paddle/tests/test_read_file.py | 11 +- python/paddle/tests/test_transforms.py | 652 +- python/paddle/tests/test_vision_models.py | 1 - python/paddle/text/__init__.py | 13 +- python/paddle/text/datasets/conll05.py | 104 +- python/paddle/text/datasets/imdb.py | 25 +- python/paddle/text/datasets/imikolov.py | 39 +- python/paddle/text/datasets/movielens.py | 76 +- python/paddle/text/datasets/uci_housing.py | 42 +- python/paddle/text/datasets/wmt14.py | 50 +- python/paddle/text/datasets/wmt16.py | 92 +- python/paddle/text/viterbi_decode.py | 66 +- python/paddle/utils/__init__.py | 4 +- python/paddle/utils/cpp_extension/__init__.py | 8 +- .../utils/cpp_extension/cpp_extension.py | 230 +- .../utils/cpp_extension/extension_utils.py | 377 +- python/paddle/utils/deprecated.py | 65 +- python/paddle/utils/dlpack.py | 8 +- python/paddle/utils/download.py | 73 +- python/paddle/utils/gast/ast3.py | 33 +- python/paddle/utils/gast/astn.py | 2 - python/paddle/utils/gast/gast.py | 1501 ++-- python/paddle/utils/image_util.py | 51 +- python/paddle/utils/install_check.py | 81 +- python/paddle/utils/lazy_import.py | 4 +- python/paddle/utils/op_version.py | 7 +- python/paddle/utils/profiler.py | 43 +- python/paddle/utils/unique_name.py | 4 +- python/paddle/vision/__init__.py | 4 +- python/paddle/vision/datasets/__init__.py | 12 +- python/paddle/vision/datasets/cifar.py | 61 +- python/paddle/vision/datasets/flowers.py | 59 +- python/paddle/vision/datasets/folder.py | 79 +- python/paddle/vision/datasets/mnist.py | 79 +- python/paddle/vision/datasets/voc2012.py | 35 +- python/paddle/vision/image.py | 12 +- python/paddle/vision/models/__init__.py | 82 +- python/paddle/vision/models/alexnet.py | 51 +- python/paddle/vision/models/densenet.py | 249 +- python/paddle/vision/models/googlenet.py | 63 +- python/paddle/vision/models/inceptionv3.py | 606 +- python/paddle/vision/models/lenet.py | 35 +- python/paddle/vision/models/mobilenetv1.py | 250 +- python/paddle/vision/models/mobilenetv2.py | 119 +- python/paddle/vision/models/mobilenetv3.py | 257 +- python/paddle/vision/models/resnet.py | 274 +- python/paddle/vision/models/shufflenetv2.py | 232 +- python/paddle/vision/models/squeezenet.py | 104 +- python/paddle/vision/models/vgg.py | 94 +- python/paddle/vision/ops.py | 1051 ++- python/paddle/vision/transforms/__init__.py | 2 +- python/paddle/vision/transforms/functional.py | 352 +- .../vision/transforms/functional_cv2.py | 255 +- .../vision/transforms/functional_pil.py | 135 +- .../vision/transforms/functional_tensor.py | 283 +- python/paddle/vision/transforms/transforms.py | 356 +- r/example/mobilenet.py | 2 +- .../CspChromeTraceFormatter.py | 1 - tools/CrossStackProfiler/CspFileReader.py | 128 +- tools/CrossStackProfiler/CspReporter.py | 152 +- tools/CrossStackProfiler/DCGMFileReader.py | 86 +- tools/CrossStackProfiler/NetFileReader.py | 37 +- tools/CrossStackProfiler/ProfileFileReader.py | 197 +- tools/analysisPyXml.py | 51 +- tools/check_api_compatible.py | 77 +- tools/check_api_source_without_core_ops.py | 6 +- tools/check_ctest_hung.py | 3 +- tools/check_op_benchmark_result.py | 88 +- tools/check_op_desc.py | 142 +- tools/check_op_kernel_same_dtypes.py | 39 +- tools/check_op_register_type.py | 35 +- tools/check_pr_approval.py | 7 +- tools/check_ut.py | 2 +- tools/codestyle/docstring_checker.py | 91 +- tools/codestyle/test_docstring_checker.py | 42 +- tools/continuous_integration/bisect.py | 111 +- tools/count_api_without_core_ops.py | 46 +- tools/coverage/coverage_diff.py | 2 +- tools/coverage/coverage_diff_list.py | 2 +- tools/coverage/coverage_lines.py | 18 +- tools/coverage/cuda_clean.py | 4 +- tools/coverage/gcda_clean.py | 7 +- tools/coverage/python_coverage.py | 19 +- tools/externalError/spider.py | 63 +- tools/final_ut_parallel_rule.py | 50 +- tools/gen_ut_cmakelists.py | 168 +- tools/get_op_list.py | 12 +- tools/get_pr_ut.py | 178 +- tools/get_single_test_cov.py | 42 +- tools/get_ut_file_map.py | 13 +- tools/get_ut_mem_map.py | 20 +- tools/group_case_for_parallel.py | 27 +- tools/handle_h_cu_file.py | 18 +- tools/infrt/fake_models/multi_fc.py | 35 +- ...rate_pd_op_dialect_from_paddle_op_maker.py | 229 +- tools/infrt/generate_phi_kernel_dialect.py | 126 +- tools/infrt/get_compat_kernel_signature.py | 40 +- tools/infrt/get_phi_kernel_info.py | 84 +- tools/infrt/print_kernel_pass_info.py | 24 +- tools/jetson_infer_op.py | 54 +- tools/parallel_UT_rule.py | 1208 ++- tools/print_op_desc.py | 7 +- tools/print_signatures.py | 123 +- tools/prune_for_jetson.py | 74 +- tools/pyCov_multithreading.py | 14 +- tools/remove_grad_op_and_kernel.py | 71 +- tools/sampcd_processor.py | 212 +- tools/summary_env.py | 25 +- tools/test_check_api_compatible.py | 68 +- tools/test_check_pr_approval.py | 60 +- tools/test_print_signatures.py | 9 +- tools/test_runner.py | 12 +- tools/test_sampcd_processor.py | 275 +- tools/timeline.py | 87 +- tools/windows/get_prec_ut_list.py | 2 +- 3576 files changed, 247166 insertions(+), 177981 deletions(-) diff --git a/.cmake-format.py b/.cmake-format.py index a6f4af8f8a39a..0181300410770 100644 --- a/.cmake-format.py +++ b/.cmake-format.py @@ -109,5 +109,5 @@ "SRCS": '*', "DEPS": '*', } - } + }, } diff --git a/cmake/copyfile.py b/cmake/copyfile.py index 7ba4d95049dc7..552e8ea1a2b14 100644 --- a/cmake/copyfile.py +++ b/cmake/copyfile.py @@ -21,7 +21,7 @@ def main(): src = sys.argv[1] dst = sys.argv[2] - if os.path.isdir(src): #copy directory + if os.path.isdir(src): # copy directory pathList = os.path.split(src) dst = os.path.join(dst, pathList[-1]) if not os.path.exists(dst): @@ -31,7 +31,7 @@ def main(): shutil.rmtree(dst) shutil.copytree(src, dst) print("overwritten copy directory: {0} --->>> {1}".format(src, dst)) - else: #copy file, wildcard + else: # copy file, wildcard if not os.path.exists(dst): os.makedirs(dst) srcFiles = glob.glob(src) diff --git a/cmake/make_resource.py b/cmake/make_resource.py index 936aefd4e6e69..ad8ee179d60c2 100644 --- a/cmake/make_resource.py +++ b/cmake/make_resource.py @@ -20,7 +20,15 @@ out = sys.argv[2] var = re.sub(r'[ .-]', '_', os.path.basename(res)) -open(out, "w").write("const unsigned char " + var + "[] = {" + - ",".join(["0x%02x" % ord(c) - for c in open(res).read()]) + ",0};\n" + - "const unsigned " + var + "_size = sizeof(" + var + ");\n") +open(out, "w").write( + "const unsigned char " + + var + + "[] = {" + + ",".join(["0x%02x" % ord(c) for c in open(res).read()]) + + ",0};\n" + + "const unsigned " + + var + + "_size = sizeof(" + + var + + ");\n" +) diff --git a/paddle/fluid/eager/auto_code_generator/generate_file_structures.py b/paddle/fluid/eager/auto_code_generator/generate_file_structures.py index 19ea5982f5883..5555c17f1b6c6 100644 --- a/paddle/fluid/eager/auto_code_generator/generate_file_structures.py +++ b/paddle/fluid/eager/auto_code_generator/generate_file_structures.py @@ -41,8 +41,9 @@ def GenerateFileStructureForFinalDygraph(eager_dir): os.mkdir(directory) # Empty files - dygraph_forward_api_h_path = os.path.join(generated_dir, - "dygraph_functions.h") + dygraph_forward_api_h_path = os.path.join( + generated_dir, "dygraph_functions.h" + ) empty_files = [dygraph_forward_api_h_path] empty_files.append(os.path.join(forwards_dir, "dygraph_functions.cc")) empty_files.append(os.path.join(nodes_dir, "nodes.cc")) @@ -83,32 +84,41 @@ def GenerateFileStructureForIntermediateDygraph(eager_dir, split_count): os.mkdir(directory) # Empty files - dygraph_forward_api_h_path = os.path.join(generated_dir, - "dygraph_forward_api.h") + dygraph_forward_api_h_path = os.path.join( + generated_dir, "dygraph_forward_api.h" + ) empty_files = [dygraph_forward_api_h_path] empty_files.append(os.path.join(nodes_dir, "nodes.h")) for i in range(split_count): empty_files.append( - os.path.join(forwards_dir, - "dygraph_forward_functions" + str(i + 1) + ".cc")) - empty_files.append(os.path.join(nodes_dir, - "nodes" + str(i + 1) + ".cc")) + os.path.join( + forwards_dir, "dygraph_forward_functions" + str(i + 1) + ".cc" + ) + ) + empty_files.append( + os.path.join(nodes_dir, "nodes" + str(i + 1) + ".cc") + ) empty_files.append( - os.path.join(forwards_dir, "dygraph_forward_functions_args_info.cc")) + os.path.join(forwards_dir, "dygraph_forward_functions_args_info.cc") + ) empty_files.append( - os.path.join(forwards_dir, - "dygraph_forward_functions_args_type_info.cc")) + os.path.join( + forwards_dir, "dygraph_forward_functions_args_type_info.cc" + ) + ) empty_files.append( - os.path.join(forwards_dir, "dygraph_forward_functions_returns_info.cc")) + os.path.join(forwards_dir, "dygraph_forward_functions_returns_info.cc") + ) for path in empty_files: if not os.path.exists(path): open(path, 'a').close() # CMakeLists nodes_level_cmakelist_path = os.path.join(nodes_dir, "CMakeLists.txt") - generated_level_cmakelist_path = os.path.join(generated_dir, - "CMakeLists.txt") + generated_level_cmakelist_path = os.path.join( + generated_dir, "CMakeLists.txt" + ) forwards_level_cmakelist_path = os.path.join(forwards_dir, "CMakeLists.txt") with open(nodes_level_cmakelist_path, "w") as f: @@ -120,9 +130,11 @@ def GenerateFileStructureForIntermediateDygraph(eager_dir, split_count): for i in range(split_count): f.write( " COMMAND ${CMAKE_COMMAND} -E copy_if_different \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/nodes/nodes" - + str(i + 1) + - ".tmp.cc\" \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/nodes/nodes" - + str(i + 1) + ".cc\"\n") + + str(i + 1) + + ".tmp.cc\" \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/nodes/nodes" + + str(i + 1) + + ".cc\"\n" + ) f.write(" DEPENDS legacy_eager_codegen\n") f.write(" VERBATIM)\n") @@ -142,9 +154,11 @@ def GenerateFileStructureForIntermediateDygraph(eager_dir, split_count): for i in range(split_count): f.write( " COMMAND ${CMAKE_COMMAND} -E copy_if_different \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/forwards/dygraph_forward_functions" - + str(i + 1) + - ".tmp.cc\" \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/forwards/dygraph_forward_functions" - + str(i + 1) + ".cc\"\n") + + str(i + 1) + + ".tmp.cc\" \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/forwards/dygraph_forward_functions" + + str(i + 1) + + ".cc\"\n" + ) f.write( " COMMAND ${CMAKE_COMMAND} -E copy_if_different \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/forwards/dygraph_forward_functions_args_info.tmp.cc\" \"${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/fluid_generated/forwards/dygraph_forward_functions_args_info.cc\"\n" ) @@ -167,7 +181,8 @@ def GenerateFileStructureForIntermediateDygraph(eager_dir, split_count): "${fluid_manual_functions} DEPS ${eager_deps} ${fluid_deps} ${GLOB_OP_LIB} ${GLOB_OPERATOR_DEPS})\n" ) f.write( - "add_dependencies(dygraph_function copy_dygraph_forward_functions)") + "add_dependencies(dygraph_function copy_dygraph_forward_functions)" + ) with open(generated_level_cmakelist_path, "w") as f: f.write("add_subdirectory(forwards)\nadd_subdirectory(nodes)") diff --git a/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py b/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py index 91a16a0fdf05f..587dfb571b00d 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py +++ b/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py @@ -18,20 +18,45 @@ ######################## ### Global Variables ### ######################## -ops_to_fill_zero_for_empty_grads = set([ - "split_grad", "split_with_num_grad", "rnn_grad", "matmul_double_grad", - "matmul_triple_grad", "sigmoid_double_grad", "sigmoid_triple_grad", - "add_double_grad", "add_triple_grad", "multiply_grad", - "multiply_double_grad", "multiply_triple_grad", "conv2d_grad_grad", - "batch_norm_double_grad", "tanh_double_grad", "tanh_triple_grad", - "subtract_double_grad", "divide_double_grad", "log_double_grad", - "elu_double_grad", "leaky_relu_double_grad", "sqrt_double_grad", - "rsqrt_double_grad", "square_double_grad", "celu_double_grad", - "pad_double_grad", "pad3d_double_grad", "squeeze_double_grad", - "unsqueeze_double_grad", "instance_norm_double_grad", "conv3d_double_grad", - "depthwise_conv2d_grad_grad", "concat_double_grad", "expand_grad", - "argsort_grad" -]) +ops_to_fill_zero_for_empty_grads = set( + [ + "split_grad", + "split_with_num_grad", + "rnn_grad", + "matmul_double_grad", + "matmul_triple_grad", + "sigmoid_double_grad", + "sigmoid_triple_grad", + "add_double_grad", + "add_triple_grad", + "multiply_grad", + "multiply_double_grad", + "multiply_triple_grad", + "conv2d_grad_grad", + "batch_norm_double_grad", + "tanh_double_grad", + "tanh_triple_grad", + "subtract_double_grad", + "divide_double_grad", + "log_double_grad", + "elu_double_grad", + "leaky_relu_double_grad", + "sqrt_double_grad", + "rsqrt_double_grad", + "square_double_grad", + "celu_double_grad", + "pad_double_grad", + "pad3d_double_grad", + "squeeze_double_grad", + "unsqueeze_double_grad", + "instance_norm_double_grad", + "conv3d_double_grad", + "depthwise_conv2d_grad_grad", + "concat_double_grad", + "expand_grad", + "argsort_grad", + ] +) # For API dispatch used at python-level # { op_name : [arg_name, ...] } @@ -40,22 +65,31 @@ core_ops_args_type_info = {} yaml_types_mapping = { - 'int' : 'int', 'int32_t' : 'int32_t', 'int64_t' : 'int64_t', 'size_t' : 'size_t', \ - 'float' : 'float', 'double' : 'double', 'bool' : 'bool', \ - 'str' : 'std::string', \ - 'str[]' : 'std::vector', 'float[]' : 'std::vector', \ - 'Place' : 'paddle::Place', 'DataLayout' : 'phi::DataLayout', 'DataType' : 'paddle::experimental::DataType', \ - 'int64_t[]' : 'std::vector', 'int[]' : 'std::vector', - 'Tensor' : 'Tensor', - 'Tensor[]' : 'std::vector', - 'Tensor[Tensor[]]' : 'std::vector>', - 'Scalar' : 'paddle::experimental::Scalar', - 'Scalar(int)' : 'paddle::experimental::Scalar', - 'Scalar(int64_t)' : 'paddle::experimental::Scalar', - 'Scalar(float)' : 'paddle::experimental::Scalar', - 'Scalar(double)' : 'paddle::experimental::Scalar', - 'Scalar[]' : 'std::vector', - 'IntArray' : 'paddle::experimental::IntArray' + 'int': 'int', + 'int32_t': 'int32_t', + 'int64_t': 'int64_t', + 'size_t': 'size_t', + 'float': 'float', + 'double': 'double', + 'bool': 'bool', + 'str': 'std::string', + 'str[]': 'std::vector', + 'float[]': 'std::vector', + 'Place': 'paddle::Place', + 'DataLayout': 'phi::DataLayout', + 'DataType': 'paddle::experimental::DataType', + 'int64_t[]': 'std::vector', + 'int[]': 'std::vector', + 'Tensor': 'Tensor', + 'Tensor[]': 'std::vector', + 'Tensor[Tensor[]]': 'std::vector>', + 'Scalar': 'paddle::experimental::Scalar', + 'Scalar(int)': 'paddle::experimental::Scalar', + 'Scalar(int64_t)': 'paddle::experimental::Scalar', + 'Scalar(float)': 'paddle::experimental::Scalar', + 'Scalar(double)': 'paddle::experimental::Scalar', + 'Scalar[]': 'std::vector', + 'IntArray': 'paddle::experimental::IntArray', } @@ -81,7 +115,8 @@ def ReadBwdFile(filepath): if contents is not None: for content in contents: assert 'backward_op' in content.keys(), AssertMessage( - 'backward_op', content.keys()) + 'backward_op', content.keys() + ) if 'backward_op' in content.keys(): api_name = content['backward_op'] @@ -116,7 +151,8 @@ def IsPlainTensorType(string): def IsVectorTensorType(string): vector_tensor_types = [ - 'std::vector>', 'std::vector' + 'std::vector>', + 'std::vector', ] if string in vector_tensor_types: return True @@ -147,7 +183,6 @@ def RemoveConstAndReference(string): def GetGradNodeName(string): - def str2Hump(text): arr = filter(None, text.split('_')) res = '' @@ -166,7 +201,6 @@ def GetDygraphForwardFunctionName(string): def GetDygraphLogName(string): - def str2Hump(text): arr = filter(None, text.split('_')) res = '' @@ -236,10 +270,14 @@ def ParseYamlArgs(string): m = re.search(pattern, arg) arg_type = m.group(1).strip() arg_name = m.group(3).split("=")[0].strip() - default_value = m.group(3).split("=")[1].strip() if len( - m.group(3).split("=")) > 1 else None - - assert arg_type in yaml_types_mapping.keys( + default_value = ( + m.group(3).split("=")[1].strip() + if len(m.group(3).split("=")) > 1 + else None + ) + + assert ( + arg_type in yaml_types_mapping.keys() ), f"The argument type {arg_type} in yaml config is not supported in yaml_types_mapping." if arg_type in ["DataType", "DataLayout"] and default_value is not None: default_value = f"paddle::experimental::{default_value}" @@ -277,7 +315,8 @@ def ParseYamlReturns(string): else: ret_type = ret.strip() - assert ret_type in yaml_types_mapping.keys( + assert ( + ret_type in yaml_types_mapping.keys() ), f"The return type {ret_type} in yaml config is not supported in yaml_types_mapping." ret_type = yaml_types_mapping[ret_type] @@ -295,7 +334,9 @@ def ParseYamlForwardFromBackward(string): wspace = r'\s*' fargs = r'(.*?)' frets = r'(.*)' - pattern = fr'{fname}{wspace}\({wspace}{fargs}{wspace}\){wspace}->{wspace}{frets}' + pattern = ( + fr'{fname}{wspace}\({wspace}{fargs}{wspace}\){wspace}->{wspace}{frets}' + ) m = re.search(pattern, string) function_name = m.group(1) @@ -359,38 +400,44 @@ def ParseYamlInplaceInfo(string): ### Generator Base ### ######################## class FunctionGeneratorBase: - def __init__(self, forward_api_contents, namespace): self.forward_api_contents = forward_api_contents self.namespace = namespace - self.is_forward_only = False if 'backward' in forward_api_contents.keys( - ) else True + self.is_forward_only = ( + False if 'backward' in forward_api_contents.keys() else True + ) self.forward_api_name = "" - self.orig_forward_inputs_list = [ - ] #[ [arg_name, arg_type, orig_position], ...] - self.orig_forward_attrs_list = [ - ] #[ [attr_name, attr_type, default_value, orig_position], ...] - self.orig_forward_returns_list = [ - ] #[ [ret_name, ret_type, orig_position], ...] + self.orig_forward_inputs_list = ( + [] + ) # [ [arg_name, arg_type, orig_position], ...] + self.orig_forward_attrs_list = ( + [] + ) # [ [attr_name, attr_type, default_value, orig_position], ...] + self.orig_forward_returns_list = ( + [] + ) # [ [ret_name, ret_type, orig_position], ...] # Processed Forward Data - self.forward_inputs_position_map = { - } #{ "name" : [type, fwd_position] } - self.forward_outputs_position_map = { - } #{ "name" : [type, fwd_position] } + self.forward_inputs_position_map = ( + {} + ) # { "name" : [type, fwd_position] } + self.forward_outputs_position_map = ( + {} + ) # { "name" : [type, fwd_position] } # Special Op Attributes - self.optional_inputs = [] #[name, ...] - self.no_need_buffers = [] #[name, ...] - self.intermediate_outputs = [] #[name, ...] - self.forward_inplace_map = {} #{name : name, ...} + self.optional_inputs = [] # [name, ...] + self.no_need_buffers = [] # [name, ...] + self.intermediate_outputs = [] # [name, ...] + self.forward_inplace_map = {} # {name : name, ...} def ParseForwardInplaceInfo(self): forward_api_contents = self.forward_api_contents - if 'inplace' not in forward_api_contents.keys(): return + if 'inplace' not in forward_api_contents.keys(): + return inplace_map_str = forward_api_contents['inplace'] self.forward_inplace_map = ParseYamlInplaceInfo(inplace_map_str) @@ -432,19 +479,26 @@ def CollectOriginalForwardInfo(self): forward_args_str = forward_api_contents['args'] forward_returns_str = forward_api_contents['output'] - assert 'op' in forward_api_contents.keys( + assert ( + 'op' in forward_api_contents.keys() ), "Unable to find \"op\" in forward_api_contents keys" - assert 'args' in forward_api_contents.keys( + assert ( + 'args' in forward_api_contents.keys() ), "Unable to find \"args\" in forward_api_contents keys" - assert 'output' in forward_api_contents.keys( + assert ( + 'output' in forward_api_contents.keys() ), "Unable to find \"output\" in forward_api_contents keys" # Collect Original Forward Inputs/Outputs and then perform validation checks - self.orig_forward_inputs_list, self.orig_forward_attrs_list, self.orig_forward_returns_list = ParseYamlForward( - forward_args_str, forward_returns_str) - - def DetermineForwardPositionMap(self, forward_inputs_list, - forward_returns_list): + ( + self.orig_forward_inputs_list, + self.orig_forward_attrs_list, + self.orig_forward_returns_list, + ) = ParseYamlForward(forward_args_str, forward_returns_str) + + def DetermineForwardPositionMap( + self, forward_inputs_list, forward_returns_list + ): for i in range(len(forward_inputs_list)): forward_input = forward_inputs_list[i] input_name = forward_input[0] @@ -452,13 +506,14 @@ def DetermineForwardPositionMap(self, forward_inputs_list, input_pos = forward_input[2] self.forward_inputs_position_map[input_name] = [ - input_type, input_pos + input_type, + input_pos, ] for i in range(len(forward_returns_list)): forward_return = forward_returns_list[i] if len(forward_return[0]) == 0: - if (len(forward_returns_list) == 1): + if len(forward_returns_list) == 1: return_name = "out" else: return_name = "out_{}".format(i + 1) @@ -468,12 +523,12 @@ def DetermineForwardPositionMap(self, forward_inputs_list, return_pos = forward_return[2] self.forward_outputs_position_map[return_name] = [ - return_type, return_pos + return_type, + return_pos, ] class GeneratorBase: - def __init__(self, api_yaml_path): self.namespace = "" self.api_yaml_path = api_yaml_path diff --git a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py index cf51fbb9d07dd..fe0b021a2e5b5 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py +++ b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py @@ -15,12 +15,19 @@ import re import argparse import os -from codegen_utils import core_ops_returns_info, core_ops_args_info, core_ops_args_type_info +from codegen_utils import ( + core_ops_returns_info, + core_ops_args_info, + core_ops_args_type_info, +) from codegen_utils import ReadBwdFile from codegen_utils import FindForwardName, GetGradNodeName, GetSavedName from codegen_utils import IsPlainTensorType, IsVectorTensorType from codegen_utils import GetConstReference, RemoveConstAndReference -from codegen_utils import GetDygraphForwardFunctionName, GetIntermediateAPIFunctionName +from codegen_utils import ( + GetDygraphForwardFunctionName, + GetIntermediateAPIFunctionName, +) from codegen_utils import GetAutoGradMetaName, GetAutoGradMetaVectorName from codegen_utils import GetInplacedFunctionName from codegen_utils import ParseYamlForwardFromBackward @@ -39,7 +46,11 @@ # Black Ops list that's NO NEED to apply code generation black_ops_list = [ - "conv2d", "conv2d_grad", "conv2d_grad_grad", "add_n", "add_n_grad" + "conv2d", + "conv2d_grad", + "conv2d_grad_grad", + "add_n", + "add_n_grad", ] @@ -48,7 +59,8 @@ ########### def ParseArguments(): parser = argparse.ArgumentParser( - description='Eager Code Generator Args Parser') + description='Eager Code Generator Args Parser' + ) parser.add_argument('--nodes_h_path', type=str) parser.add_argument('--nodes_cc_path', type=str) parser.add_argument('--forwards_h_path', type=str) @@ -63,54 +75,44 @@ def ParseArguments(): ######################## ## Code Gen Templates ## ######################## -SET_PLAIN_TENSOR_WRAPPER_TEMPLATE = \ -""" void SetTensorWrapper{}(const paddle::experimental::Tensor& {}) {{ +SET_PLAIN_TENSOR_WRAPPER_TEMPLATE = """ void SetTensorWrapper{}(const paddle::experimental::Tensor& {}) {{ {} = egr::TensorWrapper({}, {}); }} """ -SET_VECTOR_TENSOR_WRAPPER_TEMPLATE = \ -""" void SetTensorWrapper{}(const std::vector& {}) {{ +SET_VECTOR_TENSOR_WRAPPER_TEMPLATE = """ void SetTensorWrapper{}(const std::vector& {}) {{ for(const auto& eager_tensor : {}) {{ {}.emplace_back(egr::TensorWrapper(eager_tensor, {})); }}; }} """ -PLAIN_TENSOR_MEMBER_TEMPLATE = \ -""" egr::TensorWrapper {}; +PLAIN_TENSOR_MEMBER_TEMPLATE = """ egr::TensorWrapper {}; """ -VECTOR_TENSOR_MEMBER_TEMPLATE = \ -""" std::vector {}; +VECTOR_TENSOR_MEMBER_TEMPLATE = """ std::vector {}; """ -CLEAR_TENSOR_WRAPPER_TEMPLATE = \ -""" {}.clear(); +CLEAR_TENSOR_WRAPPER_TEMPLATE = """ {}.clear(); """ -CLEAR_VECTOR_TENSOR_WRAPPERS_TEMPLATE = \ -""" for (auto& tw : {}) {{ +CLEAR_VECTOR_TENSOR_WRAPPERS_TEMPLATE = """ for (auto& tw : {}) {{ tw.clear(); }} """ -SET_ATTR_METHOD_TEMPLATE = \ -""" void SetAttribute{}({} {}) {{ +SET_ATTR_METHOD_TEMPLATE = """ void SetAttribute{}({} {}) {{ {} = {}; }} """ -ATTRIBUTE_MEMBER_WITH_DEFAULT_TEMPLATE = \ -""" {} {} = {}; +ATTRIBUTE_MEMBER_WITH_DEFAULT_TEMPLATE = """ {} {} = {}; """ -ATTRIBUTE_MEMBER_TEMPLATE = \ -""" {} {}; +ATTRIBUTE_MEMBER_TEMPLATE = """ {} {}; """ -NODE_DECLARATION_TEMPLATE = \ -""" +NODE_DECLARATION_TEMPLATE = """ class {} : public egr::GradNodeBase {{ public: {}() : egr::GradNodeBase() {{}} @@ -143,8 +145,7 @@ class {} : public egr::GradNodeBase {{ {}}}; """ -GRAD_FUNCTION_TEMPLATE = \ -""" +GRAD_FUNCTION_TEMPLATE = """ paddle::small_vector, egr::kSlotSmallVectorSize> {}::operator()(paddle::small_vector, egr::kSlotSmallVectorSize>& grads, bool create_graph, bool is_new_grad) {{ VLOG(3) << \"Running AD API GRAD: \" << \"{}\"; // Fill Zero For GradIn Tensors @@ -182,8 +183,7 @@ class {} : public egr::GradNodeBase {{ }} """ -FORWARD_FUNCTION_TEMPLATE = \ -""" +FORWARD_FUNCTION_TEMPLATE = """ {} {}({}) {{ VLOG(3) << \"Running AD API: \" << \"{}\"; // Dygraph Record Event @@ -222,8 +222,7 @@ class {} : public egr::GradNodeBase {{ }} """ -AFTER_LOG_PRINT_TEMPLATE = \ -""" +AFTER_LOG_PRINT_TEMPLATE = """ if(VLOG_IS_ON(4)){{ const char* INPUT_PRINT_TEMPLATE = \"{{ Input: [%s], Output: [%s] }} \"; {} @@ -231,8 +230,7 @@ class {} : public egr::GradNodeBase {{ }} """ -BEFORE_LOG_PRINT_TEMPLATE = \ -""" +BEFORE_LOG_PRINT_TEMPLATE = """ if(VLOG_IS_ON(3)){{ const char* INPUT_PRINT_TEMPLATE = \"{{ Input: [%s]}} \"; {} @@ -240,8 +238,7 @@ class {} : public egr::GradNodeBase {{ }} """ -FORWARD_ONLY_FUNCTION_TEMPLATE = \ -""" +FORWARD_ONLY_FUNCTION_TEMPLATE = """ {} {}({}) {{ VLOG(3) << \"Running AD API: \" << \"{}\"; // Dygraph Record Event @@ -265,8 +262,7 @@ class {} : public egr::GradNodeBase {{ }} """ -FORWARD_BODY_TEMPLATE = \ -""" if(require_any_grad) {{ +FORWARD_BODY_TEMPLATE = """ if(require_any_grad) {{ {} egr::EagerUtils::PassStopGradient({}); @@ -288,8 +284,7 @@ class {} : public egr::GradNodeBase {{ }} """ -HIHGER_ORDER_DERIVATIVE_VALUE_TEMPLATE = \ -""" if(trace_backward) {{ +HIHGER_ORDER_DERIVATIVE_VALUE_TEMPLATE = """ if(trace_backward) {{ {} // Node Construction {} @@ -309,15 +304,13 @@ class {} : public egr::GradNodeBase {{ }} """ -NAMESPACE_WRAPPER_TEMPLATE = \ -""" +NAMESPACE_WRAPPER_TEMPLATE = """ namespace {} {{ {} }} """ -NODE_CC_FILE_TEMPLATE = \ -""" +NODE_CC_FILE_TEMPLATE = """ #include "glog/logging.h" #include "paddle/phi/api/all.h" #include "paddle/phi/api/backward/backward_api.h" @@ -337,8 +330,7 @@ class {} : public egr::GradNodeBase {{ {} """ -NODE_H_FILE_TEMPLATE = \ -""" +NODE_H_FILE_TEMPLATE = """ #pragma once #include "paddle/fluid/eager/tensor_wrapper.h" #include "paddle/fluid/eager/grad_node_info.h" @@ -347,8 +339,7 @@ class {} : public egr::GradNodeBase {{ {} """ -FORWARD_CC_FILE_TEMPLATE = \ -""" +FORWARD_CC_FILE_TEMPLATE = """ #include "paddle/phi/api/lib/dygraph_api.h" #include "paddle/fluid/eager/api/generated/eager_generated/forwards/dygraph_functions.h" #include "paddle/fluid/eager/api/generated/eager_generated/backwards/nodes.h" @@ -367,8 +358,7 @@ class {} : public egr::GradNodeBase {{ {} """ -FORWARD_H_FILE_TEMPLATE = \ -""" +FORWARD_H_FILE_TEMPLATE = """ #pragma once #include "glog/logging.h" #include "paddle/fluid/eager/autograd_meta.h" @@ -383,8 +373,7 @@ class {} : public egr::GradNodeBase {{ {} """ -CORE_OPS_INFO_TEMPLATE = \ -""" +CORE_OPS_INFO_TEMPLATE = """ std::unordered_map> core_ops_args_info = {{ {} }}; @@ -397,28 +386,24 @@ class {} : public egr::GradNodeBase {{ """ -CORE_OPS_DECLARATION_TEMPLATE = \ -""" +CORE_OPS_DECLARATION_TEMPLATE = """ extern std::unordered_map> core_ops_args_info; extern std::unordered_map> core_ops_args_type_info; extern std::unordered_map> core_ops_returns_info; """ -CHECK_INPLACE_TEMPLATE = \ -""" +CHECK_INPLACE_TEMPLATE = """ egr::EagerUtils::CheckInplace({}, {}, require_any_grad); """ -BUMP_INPLACE_VERSION_TEMPLATE = \ -""" +BUMP_INPLACE_VERSION_TEMPLATE = """ // Bump Inplace Version {}.bump_inplace_version(); VLOG(3) << \"Tensor(\" << {}.name() << \") uses Inplace Strategy.\"; """ -AMP_LOGIC_TEMPLATE = \ -""" if (egr::Controller::Instance().GetAMPLevel() != paddle::imperative::AmpLevel::O0) {{ +AMP_LOGIC_TEMPLATE = """ if (egr::Controller::Instance().GetAMPLevel() != paddle::imperative::AmpLevel::O0) {{ VLOG(5) << "Check and Prepare For AMP"; {} paddle::small_vector, egr::kSlotSmallVectorSize> amp_tensors_vector = {}; @@ -431,8 +416,7 @@ class {} : public egr::GradNodeBase {{ }} }} """ -LAYOUT_LOGIC_TEMPLATE=\ -""" +LAYOUT_LOGIC_TEMPLATE = """ if (egr::Controller::Instance().UseLayoutAutoTune()) {{ paddle::small_vector, egr::kSlotSmallVectorSize> tensors_vector = {}; {} @@ -445,26 +429,22 @@ class {} : public egr::GradNodeBase {{ return {}; }} """ -CREATE_PLAIN_OPTIONAL_TENSOR_TEMPLATE = \ -""" +CREATE_PLAIN_OPTIONAL_TENSOR_TEMPLATE = """ paddle::optional {}_optional; if({}.initialized()) {}_optional = paddle::make_optional({}); """ -CREATE_RECOVER_OPTIONAL_TENSOR_TEMPLATE = \ -""" +CREATE_RECOVER_OPTIONAL_TENSOR_TEMPLATE = """ paddle::optional {}_optional; if( {}.impl() ) {}_optional = paddle::make_optional({}); """ -CREATE_RECOVER_OPTIONAL_VECTOR_TENSOR_TEMPLATE = \ -""" +CREATE_RECOVER_OPTIONAL_VECTOR_TENSOR_TEMPLATE = """ paddle::optional> {}_optional; if( !{}.empty() ) {}_optional = paddle::make_optional>({}); """ -CHECK_BACKWARD_INPLACE_TEMPLATE = \ -""" +CHECK_BACKWARD_INPLACE_TEMPLATE = """ bool can_be_inplaced = false; if ({}.initialized()) {{ VLOG(10) << {}.name() << "({}) use_count: " << {}.impl().use_count(); @@ -473,15 +453,12 @@ class {} : public egr::GradNodeBase {{ }} }}""" -CHECK_NAN_AND_INF_TEMPLATE = \ -""" if (FLAGS_check_nan_inf) {{ egr::CheckTensorHasNanOrInf("{}", {}); }} +CHECK_NAN_AND_INF_TEMPLATE = """ if (FLAGS_check_nan_inf) {{ egr::CheckTensorHasNanOrInf("{}", {}); }} """ inplace_optional_out_type_map = { - "Tensor": - "paddle::optional&", - "std::vector": - "paddle::optional>&" + "Tensor": "paddle::optional&", + "std::vector": "paddle::optional>&", } @@ -490,13 +467,16 @@ def ExtractForwardApiNameFormInvoke(invoke_config): if api_name[-1] == '_': api_name = api_name[:-1] return re.search( - r"(?P[a-zA-Z0-9_]+)(?P_intermediate)?", - api_name).group('api_name') + r"(?P[a-zA-Z0-9_]+)(?P_intermediate)?", api_name + ).group('api_name') def IsInvokeForwardApi(api_contents, forward_api_name_list): - return 'invoke' in api_contents and ExtractForwardApiNameFormInvoke( - api_contents['invoke']) in forward_api_name_list + return ( + 'invoke' in api_contents + and ExtractForwardApiNameFormInvoke(api_contents['invoke']) + in forward_api_name_list + ) ####################### @@ -531,7 +511,8 @@ def GenerateCoreOpInfoDefinition(): op_returns_info_str = "\n".join(op_returns_info_list) core_ops_info_definition_str = CORE_OPS_INFO_TEMPLATE.format( - op_args_info_str, op_types_info_str, op_returns_info_str) + op_args_info_str, op_types_info_str, op_returns_info_str + ) return core_ops_info_definition_str @@ -540,23 +521,27 @@ def GenerateCoreOpInfoDefinition(): ## Generator Class ## ##################### class DygraphFunctionGeneratorBase(FunctionGeneratorBase): - - def __init__(self, forward_api_contents, grad_api_contents, - forward_apis_dict, namespace): + def __init__( + self, + forward_api_contents, + grad_api_contents, + forward_apis_dict, + namespace, + ): self.forward_api_contents = forward_api_contents # Members from Parent: - #self.namespace - #self.forward_api_contents - #self.forward_api_name - #self.orig_forward_inputs_list - #self.orig_forward_attrs_list - #self.orig_forward_returns_list - #self.forward_inputs_position_map - #self.forward_outputs_position_map - #self.optional_inputs - #self.no_need_buffers - #self.intermediate_outputs - #self.forward_inplace_map + # self.namespace + # self.forward_api_contents + # self.forward_api_name + # self.orig_forward_inputs_list + # self.orig_forward_attrs_list + # self.orig_forward_returns_list + # self.forward_inputs_position_map + # self.forward_outputs_position_map + # self.optional_inputs + # self.no_need_buffers + # self.intermediate_outputs + # self.forward_inplace_map FunctionGeneratorBase.__init__(self, forward_api_contents, namespace) self.forward_apis_dict = forward_apis_dict @@ -566,33 +551,43 @@ def __init__(self, forward_api_contents, grad_api_contents, self.backward_forward_str = "" self.backward_api_name = "" - self.forward_attrs_list = [ - ] #[ [attr_name, attr_type, default_value, orig_position], ...] - self.forward_inputs_list = [ - ] #[ [arg_name, arg_type, orig_position], ...] - self.forward_returns_list = [ - ] #[ [ret_name, ret_type, orig_position], ...] - - self.backward_attrs_list = [ - ] #[ [attr_name, attr_type, default_value, orig_position], ...] - self.backward_inputs_list = [ - ] #[ [arg_name, arg_type, orig_position], ...] - self.backward_returns_list = [ - ] #[ [ret_name, ret_type, orig_position], ...] + self.forward_attrs_list = ( + [] + ) # [ [attr_name, attr_type, default_value, orig_position], ...] + self.forward_inputs_list = ( + [] + ) # [ [arg_name, arg_type, orig_position], ...] + self.forward_returns_list = ( + [] + ) # [ [ret_name, ret_type, orig_position], ...] + + self.backward_attrs_list = ( + [] + ) # [ [attr_name, attr_type, default_value, orig_position], ...] + self.backward_inputs_list = ( + [] + ) # [ [arg_name, arg_type, orig_position], ...] + self.backward_returns_list = ( + [] + ) # [ [ret_name, ret_type, orig_position], ...] # SlotNameMatched Backward Data - self.backward_forward_inputs_map = { - } #{ "name" : [type, is_fwd_input, orig_position] ...} - self.backward_grad_inputs_map = { - } #{ "name" : [type, fwd_position, orig_position] ...} - self.backward_grad_outputs_map = { - } #{ "name" : [type, fwd_position, orig_position] ...} - - self.backward_inplace_map = {} #{name : name, ...} + self.backward_forward_inputs_map = ( + {} + ) # { "name" : [type, is_fwd_input, orig_position] ...} + self.backward_grad_inputs_map = ( + {} + ) # { "name" : [type, fwd_position, orig_position] ...} + self.backward_grad_outputs_map = ( + {} + ) # { "name" : [type, fwd_position, orig_position] ...} + + self.backward_inplace_map = {} # {name : name, ...} def ParseBackwardInplaceInfo(self): grad_api_contents = self.grad_api_contents - if 'inplace' not in grad_api_contents.keys(): return + if 'inplace' not in grad_api_contents.keys(): + return inplace_map_str = grad_api_contents['inplace'] self.backward_inplace_map = ParseYamlInplaceInfo(inplace_map_str) @@ -601,21 +596,28 @@ def DygraphYamlValidationCheck(self): forward_api_contents = self.forward_api_contents grad_api_contents = self.grad_api_contents - assert 'op' in forward_api_contents.keys( + assert ( + 'op' in forward_api_contents.keys() ), "Unable to find \"op\" in ops.yaml" - assert 'args' in forward_api_contents.keys( + assert ( + 'args' in forward_api_contents.keys() ), "Unable to find \"args\" in ops.yaml" - assert 'output' in forward_api_contents.keys( + assert ( + 'output' in forward_api_contents.keys() ), "Unable to find \"output\" in ops.yaml" if grad_api_contents is not None: - assert 'backward' in forward_api_contents.keys( + assert ( + 'backward' in forward_api_contents.keys() ), "Unable to find \"backward\" in ops.yaml" - assert 'args' in grad_api_contents.keys( + assert ( + 'args' in grad_api_contents.keys() ), "Unable to find \"args\" in backward.yaml" - assert 'output' in grad_api_contents.keys( + assert ( + 'output' in grad_api_contents.keys() ), "Unable to find \"output\" in backward.yaml" - assert 'forward' in grad_api_contents.keys( + assert ( + 'forward' in grad_api_contents.keys() ), "Unable to find \"forward\" in backward.yaml" def ForwardsValidationCheck(self): @@ -634,9 +636,11 @@ def ForwardsValidationCheck(self): orig_input_pos = orig_forward_inputs_list[i][2] assert forward_input_type == orig_input_type, AssertMessage( - forward_input_type, orig_input_type) + forward_input_type, orig_input_type + ) assert forward_input_pos == orig_input_pos, AssertMessage( - forward_input_pos, orig_input_pos) + forward_input_pos, orig_input_pos + ) for i in range(len(forward_attrs_list)): orig_attr_type = orig_forward_attrs_list[i][1] @@ -644,9 +648,11 @@ def ForwardsValidationCheck(self): forward_attr_type = forward_attrs_list[i][1] forward_attr_pos = forward_attrs_list[i][3] assert orig_attr_type == forward_attr_type, AssertMessage( - orig_attr_type, forward_attr_type) + orig_attr_type, forward_attr_type + ) assert orig_attr_pos == forward_attr_pos, AssertMessage( - orig_attr_pos, forward_attr_pos) + orig_attr_pos, forward_attr_pos + ) for i in range(len(forward_returns_list)): orig_return_type = orig_forward_returns_list[i][1] @@ -655,9 +661,11 @@ def ForwardsValidationCheck(self): forward_return_pos = forward_returns_list[i][2] assert orig_return_type == forward_return_type, AssertMessage( - orig_return_type, forward_return_type) + orig_return_type, forward_return_type + ) assert orig_return_pos == forward_return_pos, AssertMessage( - orig_return_pos, forward_return_pos) + orig_return_pos, forward_return_pos + ) # Check Order: Inputs, Attributes max_input_position = -1 @@ -666,7 +674,8 @@ def ForwardsValidationCheck(self): for _, _, _, pos in forward_attrs_list: assert pos > max_input_position, AssertMessage( - pos, max_input_position) + pos, max_input_position + ) def BackwardValidationCheck(self): backward_forward_inputs_map = self.backward_forward_inputs_map @@ -681,13 +690,15 @@ def BackwardValidationCheck(self): max_grad_tensor_position = -1 for _, (_, _, pos) in backward_grad_inputs_map.items(): assert pos > max_fwd_input_position, AssertMessage( - pos, max_grad_tensor_position) + pos, max_grad_tensor_position + ) max_grad_tensor_position = max(max_grad_tensor_position, pos) max_attr_position = -1 for _, _, _, pos in backward_attrs_list: assert pos > max_grad_tensor_position, AssertMessage( - pos, max_grad_tensor_position) + pos, max_grad_tensor_position + ) max_attr_position = max(max_attr_position, pos) def IntermediateValidationCheck(self): @@ -699,11 +710,13 @@ def IntermediateValidationCheck(self): """ intermediate_positions = range( len(forward_returns_list) - len(intermediate_outputs), - len(forward_returns_list)) + len(forward_returns_list), + ) for ret_name, _, pos in forward_returns_list: if ret_name in intermediate_outputs: assert pos in intermediate_positions, AssertMessage( - pos, intermediate_positions) + pos, intermediate_positions + ) def CollectBackwardInfo(self): forward_api_contents = self.forward_api_contents @@ -714,20 +727,32 @@ def CollectBackwardInfo(self): backward_args_str = grad_api_contents['args'] backward_returns_str = grad_api_contents['output'] - self.backward_inputs_list, self.backward_attrs_list, self.backward_returns_list = ParseYamlBackward( - backward_args_str, backward_returns_str) + ( + self.backward_inputs_list, + self.backward_attrs_list, + self.backward_returns_list, + ) = ParseYamlBackward(backward_args_str, backward_returns_str) def CollectForwardInfoFromBackwardContents(self): backward_forward_str = self.backward_forward_str - self.forward_inputs_list, self.forward_attrs_list, self.forward_returns_list = ParseYamlForwardFromBackward( - backward_forward_str) + ( + self.forward_inputs_list, + self.forward_attrs_list, + self.forward_returns_list, + ) = ParseYamlForwardFromBackward(backward_forward_str) def CollectForwardInfoFromYamlForward(self): - self.forward_inputs_list, self.forward_attrs_list, self.forward_returns_list = ParseYamlForwardFromBackward( - self.forward_api_contents['args'] + " -> " + - self.forward_api_contents['output']) + ( + self.forward_inputs_list, + self.forward_attrs_list, + self.forward_returns_list, + ) = ParseYamlForwardFromBackward( + self.forward_api_contents['args'] + + " -> " + + self.forward_api_contents['output'] + ) def SlotNameMatching(self): backward_inputs_list = self.backward_inputs_list @@ -743,35 +768,48 @@ def SlotNameMatching(self): backward_fwd_name = FindForwardName(backward_input_name) if backward_fwd_name: # Grad Input - assert backward_fwd_name in forward_outputs_position_map.keys( - ), AssertMessage(backward_fwd_name, - forward_outputs_position_map.keys()) + assert ( + backward_fwd_name in forward_outputs_position_map.keys() + ), AssertMessage( + backward_fwd_name, forward_outputs_position_map.keys() + ) matched_forward_output_type = forward_outputs_position_map[ - backward_fwd_name][0] + backward_fwd_name + ][0] matched_forward_output_pos = forward_outputs_position_map[ - backward_fwd_name][1] + backward_fwd_name + ][1] self.backward_grad_inputs_map[backward_input_name] = [ - backward_input_type, matched_forward_output_pos, - backward_input_pos + backward_input_type, + matched_forward_output_pos, + backward_input_pos, ] else: # TensorWrapper Input if backward_input_name in forward_inputs_position_map.keys(): tensor_wrapper_type = forward_inputs_position_map[ - backward_input_name][0] + backward_input_name + ][0] self.backward_forward_inputs_map[backward_input_name] = [ - backward_input_type, True, backward_input_pos + backward_input_type, + True, + backward_input_pos, ] elif backward_input_name in forward_outputs_position_map.keys(): tensor_wrapper_type = forward_outputs_position_map[ - backward_input_name][0] + backward_input_name + ][0] self.backward_forward_inputs_map[backward_input_name] = [ - backward_input_type, False, backward_input_pos + backward_input_type, + False, + backward_input_pos, ] else: - assert False, f"Cannot find {backward_input_name} in forward position map" + assert ( + False + ), f"Cannot find {backward_input_name} in forward position map" for backward_output in backward_returns_list: backward_output_name = backward_output[0] @@ -779,19 +817,26 @@ def SlotNameMatching(self): backward_output_pos = backward_output[2] backward_fwd_name = FindForwardName(backward_output_name) - assert backward_fwd_name is not None, f"Detected {backward_fwd_name} = None" - assert backward_fwd_name in forward_inputs_position_map.keys( - ), AssertMessage(backward_fwd_name, - forward_inputs_position_map.keys()) + assert ( + backward_fwd_name is not None + ), f"Detected {backward_fwd_name} = None" + assert ( + backward_fwd_name in forward_inputs_position_map.keys() + ), AssertMessage( + backward_fwd_name, forward_inputs_position_map.keys() + ) matched_forward_input_type = forward_inputs_position_map[ - backward_fwd_name][0] + backward_fwd_name + ][0] matched_forward_input_pos = forward_inputs_position_map[ - backward_fwd_name][1] + backward_fwd_name + ][1] self.backward_grad_outputs_map[backward_output_name] = [ - backward_output_type, matched_forward_input_pos, - backward_output_pos + backward_output_type, + matched_forward_input_pos, + backward_output_pos, ] def GetPassStopGradientArgsList(self, forward_outputs_position_map): @@ -815,7 +860,8 @@ def GenerateNodeCreationCodes(self, for_backward=False): # Pass Stop Gradient Args pass_stop_gradient_args_str = self.GetPassStopGradientArgsList( - forward_outputs_position_map) + forward_outputs_position_map + ) # Node Construction num_backward_inputs = len(forward_outputs_position_map.keys()) @@ -839,7 +885,9 @@ def GenerateNodeCreationCodes(self, for_backward=False): for name, _, default_val_attr, _ in backward_attrs_list: if name in forward_attrs_name_set: - set_attributes = f"{indent}grad_node->SetAttribute{name}({name});" + set_attributes = ( + f"{indent}grad_node->SetAttribute{name}({name});" + ) else: set_attributes = f"{indent}grad_node->SetAttribute{name}({default_val_attr});" set_attributes_list.append(set_attributes) @@ -849,31 +897,41 @@ def GenerateNodeCreationCodes(self, for_backward=False): set_input_tensor_wrappers_list = [] set_output_tensor_wrappers_list = [] num_fwd_outputs = len(forward_outputs_position_map.keys()) - for name, (atype, is_fwd_input, - pos) in backward_forward_inputs_map.items(): - is_optional = (name in optional_inputs) + for name, ( + atype, + is_fwd_input, + pos, + ) in backward_forward_inputs_map.items(): + is_optional = name in optional_inputs if is_fwd_input: if is_optional: set_tensor_wrappers = f"{indent}if({name}) grad_node->SetTensorWrapper{name}(*{name});" else: - set_tensor_wrappers = f"{indent}grad_node->SetTensorWrapper{name}({name});" + set_tensor_wrappers = ( + f"{indent}grad_node->SetTensorWrapper{name}({name});" + ) set_input_tensor_wrappers_list.append(set_tensor_wrappers) else: # Forwad's output as backward's input if num_fwd_outputs > 1: # Aligned with forward output position - assert name in forward_outputs_position_map.keys( + assert ( + name in forward_outputs_position_map.keys() ), AssertMessage(name, forward_outputs_position_map.keys()) if is_optional: set_tensor_wrappers = f"{indent}if({name}) grad_node->SetTensorWrapper{name}(*{name});" else: - set_tensor_wrappers = f"{indent}grad_node->SetTensorWrapper{name}({name});" + set_tensor_wrappers = ( + f"{indent}grad_node->SetTensorWrapper{name}({name});" + ) set_output_tensor_wrappers_list.append(set_tensor_wrappers) set_input_tensor_wrappers_str = "\n".join( - set_input_tensor_wrappers_list) + set_input_tensor_wrappers_list + ) set_output_tensor_wrappers_str = "\n".join( - set_output_tensor_wrappers_list) + set_output_tensor_wrappers_list + ) # SetGradOutMeta & SetEdges grad_node_out_list = [] @@ -882,19 +940,24 @@ def GenerateNodeCreationCodes(self, for_backward=False): for name, (_, pos) in forward_inputs_position_map.items(): # Has corresponding grad output has_corresponding_grad_output = False - for _, (_, corresponding_pos, - _) in backward_grad_outputs_map.items(): + for _, ( + _, + corresponding_pos, + _, + ) in backward_grad_outputs_map.items(): if pos == corresponding_pos: has_corresponding_grad_output = True if not has_corresponding_grad_output: continue grad_node_out_list.append(name) - is_optional = (name in self.optional_inputs) + is_optional = name in self.optional_inputs if is_optional: set_grad_out_meta = f"{indent}if({name}.get_ptr() != nullptr) grad_node->SetGradOutMeta(*({name}.get_ptr()), {pos});" else: - set_grad_out_meta = f"{indent}grad_node->SetGradOutMeta({name}, {pos});" + set_grad_out_meta = ( + f"{indent}grad_node->SetGradOutMeta({name}, {pos});" + ) set_grad_out_meta_list.append(set_grad_out_meta) set_grad_out_meta_str = "\n".join(set_grad_out_meta_list) @@ -915,8 +978,12 @@ def GenerateNodeCreationCodes(self, for_backward=False): {indent} egr::EagerUtils::SetHistory({output_autograd_meta_name}, grad_node); {indent}}}""" - set_grad_in_meta = f"{indent}grad_node->SetGradInMeta({name}, {pos});" - set_retain_grad = f"{indent}egr::EagerUtils::CheckAndRetainGrad({name});" + set_grad_in_meta = ( + f"{indent}grad_node->SetGradInMeta({name}, {pos});" + ) + set_retain_grad = ( + f"{indent}egr::EagerUtils::CheckAndRetainGrad({name});" + ) set_out_rank_list.append(set_out_rank) set_history_list.append(set_history) @@ -932,18 +999,33 @@ def GenerateNodeCreationCodes(self, for_backward=False): node_creation_event_str = f"{indent}paddle::platform::RecordEvent node_creation_record_event(\"{node_event_name}\", paddle::platform::TracerEventType::OperatorInner, 1);\n" if not for_backward: self.node_creation_str = FORWARD_BODY_TEMPLATE.format( - node_creation_event_str, pass_stop_gradient_args_str, - node_construction_str, set_attributes_str, - set_input_tensor_wrappers_str, set_grad_out_meta_str, - set_out_rank_str, set_history_str, set_grad_in_meta_str, - set_retain_grad_str, set_output_tensor_wrappers_str) + node_creation_event_str, + pass_stop_gradient_args_str, + node_construction_str, + set_attributes_str, + set_input_tensor_wrappers_str, + set_grad_out_meta_str, + set_out_rank_str, + set_history_str, + set_grad_in_meta_str, + set_retain_grad_str, + set_output_tensor_wrappers_str, + ) else: - self.node_creation_str = HIHGER_ORDER_DERIVATIVE_VALUE_TEMPLATE.format( - node_creation_event_str, node_construction_str, - set_attributes_str, set_input_tensor_wrappers_str, - set_grad_out_meta_str, set_out_rank_str, set_history_str, - set_grad_in_meta_str, set_retain_grad_str, - set_output_tensor_wrappers_str) + self.node_creation_str = ( + HIHGER_ORDER_DERIVATIVE_VALUE_TEMPLATE.format( + node_creation_event_str, + node_construction_str, + set_attributes_str, + set_input_tensor_wrappers_str, + set_grad_out_meta_str, + set_out_rank_str, + set_history_str, + set_grad_in_meta_str, + set_retain_grad_str, + set_output_tensor_wrappers_str, + ) + ) self.grad_node_out_list = grad_node_out_list @@ -988,8 +1070,9 @@ def run(self): ## Process Parsed Contents ## ############################# # Initialize forward_inputs_position_map, forward_outputs_position_map - self.DetermineForwardPositionMap(self.forward_inputs_list, - self.forward_returns_list) + self.DetermineForwardPositionMap( + self.forward_inputs_list, self.forward_returns_list + ) if self.grad_api_contents is not None: # Initialize backward_forward_inputs_map, backward_grad_inputs_map, backward_grad_outputs_map @@ -999,31 +1082,51 @@ def run(self): class DygraphForwardFunctionGenerator(DygraphFunctionGeneratorBase): - - def __init__(self, forward_api_contents, grad_api_contents, - forward_apis_dict, namespace): - DygraphFunctionGeneratorBase.__init__(self, forward_api_contents, - grad_api_contents, - forward_apis_dict, namespace) + def __init__( + self, + forward_api_contents, + grad_api_contents, + forward_apis_dict, + namespace, + ): + DygraphFunctionGeneratorBase.__init__( + self, + forward_api_contents, + grad_api_contents, + forward_apis_dict, + namespace, + ) # Generated Results self.forward_definition_str = "" self.forward_declaration_str = "" - def GenerateForwardLayoutAutotune(self, forward_api_name, - amp_tensors_vector_list, - layout_tensors_vector_optional_list, - layout_autotune_list_str, - returns_type_str, returns_str, - amp_inputs_call_args_str): + def GenerateForwardLayoutAutotune( + self, + forward_api_name, + amp_tensors_vector_list, + layout_tensors_vector_optional_list, + layout_autotune_list_str, + returns_type_str, + returns_str, + amp_inputs_call_args_str, + ): intermediate_outputs = self.intermediate_outputs forward_attrs_list = self.forward_attrs_list forward_outputs_position_map = self.forward_outputs_position_map - num_outputs = len( - forward_outputs_position_map.keys()) - len(intermediate_outputs) + num_outputs = len(forward_outputs_position_map.keys()) - len( + intermediate_outputs + ) # for layout autotune attr lightly_sensitive_attr = [ - 'axis', 'axes', 'dim', 'dims', 'start', 'end', 'stop', 'perm' + 'axis', + 'axes', + 'dim', + 'dims', + 'start', + 'end', + 'stop', + 'perm', ] heavily_sensitive_attr = ['data_format', 'data_layout'] layout_autotune_attr = [] @@ -1037,15 +1140,17 @@ def GenerateForwardLayoutAutotune(self, forward_api_name, heavily_flag = False for name, atype, default_val, pos in forward_attrs_list: for attr_name in lightly_sensitive_attr: - if name.find(attr_name) != -1 and (name - not in layout_autotune_attr): + if name.find(attr_name) != -1 and ( + name not in layout_autotune_attr + ): lightly_flag = True layout_autotune_attr.append(name) layout_autotune_attr_type_list.append(atype) if lightly_flag is False: for attr_name in heavily_sensitive_attr: if name.find(attr_name) != -1 and ( - name not in layout_autotune_attr): + name not in layout_autotune_attr + ): layout_autotune_attr.append(name) layout_autotune_attr_type_list.append(atype) heavily_flag = True @@ -1074,7 +1179,8 @@ def GenerateForwardLayoutAutotune(self, forward_api_name, if num_outputs == 1: result_name = returns_str layout_autotune_outs_list.append( - f"transformer -> SetOutTensorLayout(&{returns_str});\n") + f"transformer -> SetOutTensorLayout(&{returns_str});\n" + ) else: for name, (rtype, pos) in forward_outputs_position_map.items(): if name in intermediate_outputs: @@ -1083,11 +1189,13 @@ def GenerateForwardLayoutAutotune(self, forward_api_name, f" auto& {name} = std::get<{len(layout_tmp_result_list)}>(api_result);\n" ) layout_autotune_outs_list.append( - f" transformer -> SetOutTensorLayout(&{name});\n") + f" transformer -> SetOutTensorLayout(&{name});\n" + ) layout_tmp_result_list.append(f"{name}") - tensors_vector_list_str = "{ " + ",".join( - amp_tensors_vector_list) + " }" + tensors_vector_list_str = ( + "{ " + ",".join(amp_tensors_vector_list) + " }" + ) if len(amp_tensors_vector_list) == 0: layout_logic_str = "" @@ -1096,9 +1204,13 @@ def GenerateForwardLayoutAutotune(self, forward_api_name, layout_logic_str = LAYOUT_LOGIC_TEMPLATE.format( tensors_vector_list_str, " ".join(layout_tensors_vector_optional_list), - " ".join(layout_autotune_attr_code_list) + " " + - layout_autotune_list_str, after_call_str, - " ".join(layout_autotune_outs_list), returns_str) + " ".join(layout_autotune_attr_code_list) + + " " + + layout_autotune_list_str, + after_call_str, + " ".join(layout_autotune_outs_list), + returns_str, + ) return layout_logic_str @@ -1106,8 +1218,11 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): namespace = self.namespace if self.forward_api_name[-1] == '_' and not is_inplaced: return - forward_api_name = GetInplacedFunctionName( - self.forward_api_name) if is_inplaced else self.forward_api_name + forward_api_name = ( + GetInplacedFunctionName(self.forward_api_name) + if is_inplaced + else self.forward_api_name + ) forward_inputs_position_map = self.forward_inputs_position_map forward_outputs_position_map = self.forward_outputs_position_map @@ -1122,7 +1237,8 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): # Get Function Args num_inputs = len(forward_attrs_list) + len( - forward_inputs_position_map.keys()) + forward_inputs_position_map.keys() + ) inputs_args_definition_list = ["" for i in range(num_inputs)] inputs_args_declaration_list = ["" for i in range(num_inputs)] inputs_call_list = ["" for i in range(num_inputs)] @@ -1138,10 +1254,14 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): for name, (ttype, pos) in forward_inputs_position_map.items(): inputs_call_list[pos] = f"{name}" amp_inputs_call_list[pos] = f"new_{name}" - is_optional = (name in optional_inputs) + is_optional = name in optional_inputs if IsPlainTensorType(ttype): if is_optional: - if self.is_forward_only and is_inplaced and forward_inplace_map and name in forward_inplace_map.keys( + if ( + self.is_forward_only + and is_inplaced + and forward_inplace_map + and name in forward_inplace_map.keys() ): arg_str = f"paddle::optional& {name}" else: @@ -1159,7 +1279,10 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): f"auto new_{name} = transformer->TransInTensor(\"{name}\", {name});\n" ) else: - if is_inplaced and forward_inplace_map and name in forward_inplace_map.keys( + if ( + is_inplaced + and forward_inplace_map + and name in forward_inplace_map.keys() ): arg_str = f"paddle::experimental::Tensor& {name}" amp_tensors_vector_list.append(f"{{{name}}}") @@ -1178,7 +1301,11 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): else: assert IsVectorTensorType(ttype) if is_optional: - if self.is_forward_only and is_inplaced and forward_inplace_map and name in forward_inplace_map.keys( + if ( + self.is_forward_only + and is_inplaced + and forward_inplace_map + and name in forward_inplace_map.keys() ): arg_str = f"paddle::optional>& {name}" else: @@ -1193,9 +1320,14 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): f"auto new_{name} = transformer->TransInTensors(\"{name}\", {name});\n" ) else: - if is_inplaced and forward_inplace_map and name in forward_inplace_map.keys( + if ( + is_inplaced + and forward_inplace_map + and name in forward_inplace_map.keys() ): - arg_str = f"std::vector& {name}" + arg_str = ( + f"std::vector& {name}" + ) else: arg_str = f"const std::vector& {name}" amp_tensors_vector_list.append(f"{name}") @@ -1215,7 +1347,8 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): amp_inputs_call_list[pos] = name if default_val is not None: inputs_args_declaration_list[ - pos] = f"{atype} {name} = {default_val}" + pos + ] = f"{atype} {name} = {default_val}" else: inputs_args_declaration_list[pos] = f"{atype} {name}" inputs_args_definition_list[pos] = f"{atype} {name}" @@ -1228,8 +1361,9 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): function_name = forward_api_name if len(intermediate_outputs) > 0: if is_inplaced: - function_name = GetIntermediateAPIFunctionName( - forward_api_name[:-1]) + '_' + function_name = ( + GetIntermediateAPIFunctionName(forward_api_name[:-1]) + '_' + ) else: function_name = GetIntermediateAPIFunctionName(function_name) @@ -1237,12 +1371,14 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): if is_inplaced and len(forward_outputs_position_map) == 1: api_out_type = "auto&" forward_call_str = f"{indent}{api_out_type} api_result = paddle::experimental::{namespace}{function_name}({inputs_call_args_str});" - num_outputs = len( - forward_outputs_position_map.keys()) - len(intermediate_outputs) + num_outputs = len(forward_outputs_position_map.keys()) - len( + intermediate_outputs + ) # Check Nan and Inf check_nan_inf_str = CHECK_NAN_AND_INF_TEMPLATE.format( - function_name, "api_result") + function_name, "api_result" + ) # Get Outputs get_outputs_str = "" @@ -1250,7 +1386,9 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): if num_outputs == 1 and len(intermediate_outputs) == 0: get_outputs_str += f"{indent}auto& {name} = api_result;\n" else: - get_outputs_str += f"{indent}auto& {name} = std::get<{pos}>(api_result);\n" + get_outputs_str += ( + f"{indent}auto& {name} = std::get<{pos}>(api_result);\n" + ) # Get return type list & outputs returns_type_list = ["" for i in range(num_outputs)] @@ -1261,32 +1399,46 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): returns_list[pos] = f"{name}" if IsPlainTensorType(rtype): - if is_inplaced and forward_inplace_map and name in forward_inplace_map.values( + if ( + is_inplaced + and forward_inplace_map + and name in forward_inplace_map.values() ): ind = list(forward_inplace_map.values()).index(name) - if list(forward_inplace_map.keys() - )[ind] in self.optional_inputs: + if ( + list(forward_inplace_map.keys())[ind] + in self.optional_inputs + ): returns_type_list[pos] = inplace_optional_out_type_map[ - rtype] + rtype + ] else: returns_type_list[pos] = "paddle::experimental::Tensor&" else: returns_type_list[pos] = "paddle::experimental::Tensor" else: assert IsVectorTensorType(rtype) - if is_inplaced and forward_inplace_map and name in forward_inplace_map.values( + if ( + is_inplaced + and forward_inplace_map + and name in forward_inplace_map.values() ): ind = list(forward_inplace_map.values()).index(name) - if list(forward_inplace_map.keys() - )[ind] in self.optional_inputs: + if ( + list(forward_inplace_map.keys())[ind] + in self.optional_inputs + ): returns_type_list[pos] = inplace_optional_out_type_map[ - rtype] + rtype + ] else: returns_type_list[ - pos] = "std::vector&" + pos + ] = "std::vector&" else: returns_type_list[ - pos] = "std::vector" + pos + ] = "std::vector" if num_outputs == 1: returns_str = returns_list[0] @@ -1306,29 +1458,40 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): for name, (ttype, pos) in forward_inputs_position_map.items(): # Has corresponding grad output has_corresponding_grad_output = False - for _, (_, corresponding_pos, - _) in backward_grad_outputs_map.items(): + for _, ( + _, + corresponding_pos, + _, + ) in backward_grad_outputs_map.items(): if pos == corresponding_pos: has_corresponding_grad_output = True - if has_corresponding_grad_output or ( - name in forward_inplace_map and forward_api_name - not in inplace_check_blacklist) or self.is_forward_only: + if ( + has_corresponding_grad_output + or ( + name in forward_inplace_map + and forward_api_name not in inplace_check_blacklist + ) + or self.is_forward_only + ): input_autograd_meta_name = GetAutoGradMetaName(name) if IsPlainTensorType(ttype): input_autograd_meta = f"{indent}egr::AutogradMeta* {input_autograd_meta_name} = egr::EagerUtils::nullable_autograd_meta({name});" else: assert IsVectorTensorType(ttype) - input_autograd_meta_vec_name = GetAutoGradMetaVectorName( - name) + input_autograd_meta_vec_name = ( + GetAutoGradMetaVectorName(name) + ) input_autograd_meta = f"{indent}std::vector {input_autograd_meta_vec_name} = egr::EagerUtils::nullable_autograd_meta({name});\n" input_autograd_meta += f"{indent}std::vector* {input_autograd_meta_name} = &{input_autograd_meta_vec_name};" inputs_autograd_meta_list.append(input_autograd_meta) compute_require_grad_args_list.append( - input_autograd_meta_name) + input_autograd_meta_name + ) inputs_autograd_meta_str = "\n".join(inputs_autograd_meta_list) compute_require_grad_args_str = ",".join( - compute_require_grad_args_list) + compute_require_grad_args_list + ) # 2. Get Output AutoGradMeta outputs_autograd_meta_list = [] @@ -1363,11 +1526,16 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): for inplace_name in forward_inplace_map.keys(): if forward_api_name not in inplace_check_blacklist: inplace_autograd_meta_name = GetAutoGradMetaName( - inplace_name) + inplace_name + ) check_inplace_str += CHECK_INPLACE_TEMPLATE.format( - inplace_name, inplace_autograd_meta_name) - bump_inplace_version_str += BUMP_INPLACE_VERSION_TEMPLATE.format( - inplace_name, inplace_name) + inplace_name, inplace_autograd_meta_name + ) + bump_inplace_version_str += ( + BUMP_INPLACE_VERSION_TEMPLATE.format( + inplace_name, inplace_name + ) + ) # Node Creation self.GenerateNodeCreationCodes() @@ -1375,36 +1543,56 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): dygraph_event_str = f"{indent}paddle::platform::RecordEvent dygraph_entrance_record_event(\"{forward_api_name} dygraph\", paddle::platform::TracerEventType::Operator, 1);\n" forward_ad_function_name = GetDygraphForwardFunctionName( - forward_api_name) + forward_api_name + ) # Forward amp logic - kernel_trans2_op_name_str = f"auto op_name = phi::TransToFluidOpName(\"{forward_api_name}\");" - amp_tensors_vector_list_str = "{ " + ",".join( - amp_tensors_vector_list) + " }" + kernel_trans2_op_name_str = ( + f"auto op_name = phi::TransToFluidOpName(\"{forward_api_name}\");" + ) + amp_tensors_vector_list_str = ( + "{ " + ",".join(amp_tensors_vector_list) + " }" + ) amp_tensors_vector_optional_list_str = " ".join( - amp_tensors_vector_optional_list) + amp_tensors_vector_optional_list + ) amp_get_dst_dtype_str = "auto amp_dst_dtype = egr::GetAmpDestDtype(op_name, amp_tensors_vector);\n" - amp_autocast_list_str = " ".join( - amp_autocast_list) + " " + " ".join( - amp_autocast_optional_list) + amp_autocast_list_str = ( + " ".join(amp_autocast_list) + + " " + + " ".join(amp_autocast_optional_list) + ) amp_inputs_call_args_str = ", ".join(amp_inputs_call_list) - amp_call_str = f"return {forward_ad_function_name}({amp_inputs_call_args_str});" + amp_call_str = ( + f"return {forward_ad_function_name}({amp_inputs_call_args_str});" + ) if is_inplaced or (forward_api_name == "cast"): amp_logic_str = "\n VLOG(5) << \" No AMP for {} because it is a inplace or cast api. \"; ".format( - forward_ad_function_name) + forward_ad_function_name + ) else: amp_logic_str = AMP_LOGIC_TEMPLATE.format( - kernel_trans2_op_name_str, amp_tensors_vector_list_str, - amp_tensors_vector_optional_list_str, amp_get_dst_dtype_str, - amp_autocast_list_str, amp_call_str) + kernel_trans2_op_name_str, + amp_tensors_vector_list_str, + amp_tensors_vector_optional_list_str, + amp_get_dst_dtype_str, + amp_autocast_list_str, + amp_call_str, + ) # Forward layout autotune layout_autotune_list_str = " ".join( - layout_autotune_list) + " ".join(layout_autotune_optional_list) + layout_autotune_list + ) + " ".join(layout_autotune_optional_list) layout_logic_str = self.GenerateForwardLayoutAutotune( - forward_api_name, amp_tensors_vector_list, - layout_tensors_vector_optional_list, layout_autotune_list_str, - returns_type_str, returns_str, amp_inputs_call_args_str) + forward_api_name, + amp_tensors_vector_list, + layout_tensors_vector_optional_list, + layout_autotune_list_str, + returns_type_str, + returns_str, + amp_inputs_call_args_str, + ) # For inputs outputs prepare for logging var_str = f"\n{indent} std::string input_str = \"\";" @@ -1426,23 +1614,50 @@ def GenerateForwardDefinitionAndDeclaration(self, is_inplaced): if self.is_forward_only: if len(amp_tensors_vector_list) == 0: amp_logic_str = "\n VLOG(7) << \" No AMP for {} because it has no input. \"; ".format( - forward_ad_function_name) - self.forward_definition_str += FORWARD_ONLY_FUNCTION_TEMPLATE.format( - returns_type_str, forward_ad_function_name, - inputs_args_definition_str, forward_api_name, dygraph_event_str, - amp_logic_str, layout_logic_str, forward_api_name, - before_log_str, forward_call_str, get_outputs_str, - forward_api_name, log_str, returns_str) + forward_ad_function_name + ) + self.forward_definition_str += ( + FORWARD_ONLY_FUNCTION_TEMPLATE.format( + returns_type_str, + forward_ad_function_name, + inputs_args_definition_str, + forward_api_name, + dygraph_event_str, + amp_logic_str, + layout_logic_str, + forward_api_name, + before_log_str, + forward_call_str, + get_outputs_str, + forward_api_name, + log_str, + returns_str, + ) + ) else: self.forward_definition_str += FORWARD_FUNCTION_TEMPLATE.format( - returns_type_str, forward_ad_function_name, - inputs_args_definition_str, forward_api_name, dygraph_event_str, - amp_logic_str, layout_logic_str, inputs_autograd_meta_str, - forward_api_name, before_log_str, forward_call_str, - check_nan_inf_str, get_outputs_str, outputs_autograd_meta_str, - compute_require_grad_args_str, check_inplace_str, - bump_inplace_version_str, node_creation_str, forward_api_name, - log_str, returns_str) + returns_type_str, + forward_ad_function_name, + inputs_args_definition_str, + forward_api_name, + dygraph_event_str, + amp_logic_str, + layout_logic_str, + inputs_autograd_meta_str, + forward_api_name, + before_log_str, + forward_call_str, + check_nan_inf_str, + get_outputs_str, + outputs_autograd_meta_str, + compute_require_grad_args_str, + check_inplace_str, + bump_inplace_version_str, + node_creation_str, + forward_api_name, + log_str, + returns_str, + ) self.forward_declaration_str += f"{returns_type_str} {forward_ad_function_name}({inputs_args_declaration_str});\n" @@ -1451,21 +1666,27 @@ def GenerateInplacedForwardDygraphFunctions(self): forward_api_name = self.forward_api_name forward_api_contents = self.forward_api_contents - if forward_api_name != "sum" and "inplace" in forward_api_contents.keys( + if ( + forward_api_name != "sum" + and "inplace" in forward_api_contents.keys() ): # Function Definition and Declaration Generation self.GenerateForwardDefinitionAndDeclaration(is_inplaced=True) self.UpdateCoreOpsInformation(is_inplaced=True) def UpdateCoreOpsInformation(self, is_inplaced): - forward_api_name = GetInplacedFunctionName( - self.forward_api_name) if is_inplaced else self.forward_api_name + forward_api_name = ( + GetInplacedFunctionName(self.forward_api_name) + if is_inplaced + else self.forward_api_name + ) forward_inputs_position_map = self.forward_inputs_position_map forward_outputs_position_map = self.forward_outputs_position_map forward_attrs_list = self.forward_attrs_list - num_args = len( - forward_inputs_position_map.keys()) + len(forward_attrs_list) + num_args = len(forward_inputs_position_map.keys()) + len( + forward_attrs_list + ) num_returns = len(forward_outputs_position_map.keys()) fwd_api_name = "" + forward_api_name @@ -1503,16 +1724,21 @@ def run(self): class DygraphNodeGenerator(DygraphFunctionGeneratorBase): - - def __init__(self, - forward_api_contents, - grad_api_contents, - forward_apis_dict, - namespace, - next_grad_api_contents=None): - DygraphFunctionGeneratorBase.__init__(self, forward_api_contents, - grad_api_contents, - forward_apis_dict, namespace) + def __init__( + self, + forward_api_contents, + grad_api_contents, + forward_apis_dict, + namespace, + next_grad_api_contents=None, + ): + DygraphFunctionGeneratorBase.__init__( + self, + forward_api_contents, + grad_api_contents, + forward_apis_dict, + namespace, + ) # Record name mapping from forward_var_name to grad_var_names self.to_next_grad_name_mapping = {} # {name : name} @@ -1569,8 +1795,11 @@ def GenerateHigherOrderNodeCreationCode(self): backward_api_contents = next_grad_api_contents next_node_generator = DygraphFunctionGeneratorBase( - forward_api_contents, backward_api_contents, forward_apis_dict, - namespace) + forward_api_contents, + backward_api_contents, + forward_apis_dict, + namespace, + ) next_node_generator.run() next_node_generator.GenerateNodeCreationCodes(True) @@ -1579,7 +1808,11 @@ def GenerateHigherOrderNodeCreationCode(self): self.RecordGrad2NextGradNameMapping(next_node_generator) if next_node_generator is not None: - return next_grad_node_creation_str, next_grad_node_out_list, next_node_generator.backward_forward_inputs_map + return ( + next_grad_node_creation_str, + next_grad_node_out_list, + next_node_generator.backward_forward_inputs_map, + ) else: return next_grad_node_creation_str, next_grad_node_out_list, None @@ -1593,30 +1826,45 @@ def GenerateNodeDeclaration(self): set_tensor_wrapper_methods_str = "" tensor_wrapper_members_str = "" clear_tensor_wrapper_str = "" - for tname, (ttype, is_fwd_input, - _) in backward_forward_inputs_map.items(): + for tname, ( + ttype, + is_fwd_input, + _, + ) in backward_forward_inputs_map.items(): no_need_buffer = "true" if tname in no_need_buffers else "false" tensor_wrapper_name = GetSavedName(tname) if IsPlainTensorType(ttype): - set_tensor_wrapper_methods_str += SET_PLAIN_TENSOR_WRAPPER_TEMPLATE.format( - tname, tname, tensor_wrapper_name, tname, no_need_buffer) + set_tensor_wrapper_methods_str += ( + SET_PLAIN_TENSOR_WRAPPER_TEMPLATE.format( + tname, tname, tensor_wrapper_name, tname, no_need_buffer + ) + ) - tensor_wrapper_members_str += PLAIN_TENSOR_MEMBER_TEMPLATE.format( - tensor_wrapper_name) + tensor_wrapper_members_str += ( + PLAIN_TENSOR_MEMBER_TEMPLATE.format(tensor_wrapper_name) + ) - clear_tensor_wrapper_str += CLEAR_TENSOR_WRAPPER_TEMPLATE.format( - tensor_wrapper_name) + clear_tensor_wrapper_str += ( + CLEAR_TENSOR_WRAPPER_TEMPLATE.format(tensor_wrapper_name) + ) else: assert IsVectorTensorType(ttype) - set_tensor_wrapper_methods_str += SET_VECTOR_TENSOR_WRAPPER_TEMPLATE.format( - tname, tname, tname, tensor_wrapper_name, no_need_buffer) + set_tensor_wrapper_methods_str += ( + SET_VECTOR_TENSOR_WRAPPER_TEMPLATE.format( + tname, tname, tname, tensor_wrapper_name, no_need_buffer + ) + ) - tensor_wrapper_members_str += VECTOR_TENSOR_MEMBER_TEMPLATE.format( - tensor_wrapper_name) + tensor_wrapper_members_str += ( + VECTOR_TENSOR_MEMBER_TEMPLATE.format(tensor_wrapper_name) + ) - clear_tensor_wrapper_str += CLEAR_VECTOR_TENSOR_WRAPPERS_TEMPLATE.format( - tensor_wrapper_name) + clear_tensor_wrapper_str += ( + CLEAR_VECTOR_TENSOR_WRAPPERS_TEMPLATE.format( + tensor_wrapper_name + ) + ) # SetAttributes & Attribute Members set_attribute_methods_str = "" @@ -1624,27 +1872,44 @@ def GenerateNodeDeclaration(self): for aname, atype, default_val, _ in backward_attrs_list: saved_attr_name = GetSavedName(aname) set_attribute_methods_str += SET_ATTR_METHOD_TEMPLATE.format( - aname, GetConstReference(atype), aname, saved_attr_name, aname) + aname, GetConstReference(atype), aname, saved_attr_name, aname + ) if default_val: - attribute_members_str += ATTRIBUTE_MEMBER_WITH_DEFAULT_TEMPLATE.format( - RemoveConstAndReference(atype), saved_attr_name, - default_val) + attribute_members_str += ( + ATTRIBUTE_MEMBER_WITH_DEFAULT_TEMPLATE.format( + RemoveConstAndReference(atype), + saved_attr_name, + default_val, + ) + ) else: attribute_members_str += ATTRIBUTE_MEMBER_TEMPLATE.format( - RemoveConstAndReference(atype), saved_attr_name) + RemoveConstAndReference(atype), saved_attr_name + ) grad_node_name = GetGradNodeName(self.backward_api_name) self.node_declaration_str = NODE_DECLARATION_TEMPLATE.format( - grad_node_name, grad_node_name, grad_node_name, grad_node_name, - grad_node_name, clear_tensor_wrapper_str, grad_node_name, - grad_node_name, set_tensor_wrapper_methods_str, - set_attribute_methods_str, tensor_wrapper_members_str, - attribute_members_str) - - def GenerateNodeDefinition(self, next_grad_node_creation_str, - next_grad_node_out_list, - backward_forward_inputs_map_next): + grad_node_name, + grad_node_name, + grad_node_name, + grad_node_name, + grad_node_name, + clear_tensor_wrapper_str, + grad_node_name, + grad_node_name, + set_tensor_wrapper_methods_str, + set_attribute_methods_str, + tensor_wrapper_members_str, + attribute_members_str, + ) + + def GenerateNodeDefinition( + self, + next_grad_node_creation_str, + next_grad_node_out_list, + backward_forward_inputs_map_next, + ): namespace = self.namespace forward_api_name = self.forward_api_name backward_api_name = self.backward_api_name @@ -1655,21 +1920,30 @@ def GenerateNodeDefinition(self, next_grad_node_creation_str, backward_inplace_map = self.backward_inplace_map indent = GetIndent(1) - is_invoke_forward_api = IsInvokeForwardApi(self.grad_api_contents, - self.forward_apis_dict) + is_invoke_forward_api = IsInvokeForwardApi( + self.grad_api_contents, self.forward_apis_dict + ) # Construct grad_api function args # Order: TensorWrappers, GradTensors, Attributes - grad_api_args_len = len(backward_forward_inputs_map.keys()) + len( - backward_grad_inputs_map.keys()) + len(backward_attrs_list) + grad_api_args_len = ( + len(backward_forward_inputs_map.keys()) + + len(backward_grad_inputs_map.keys()) + + len(backward_attrs_list) + ) grad_api_args = ["" for i in range(grad_api_args_len)] get_grad_in_args_list = [] # Fill Grad Ins with Zero fill_zero_str = "" if backward_api_name in ops_to_fill_zero_for_empty_grads: - fill_zero_str = f"{indent}const auto& input_metas = this->InputMeta();\n" - for name, (ttype, fwd_position, - grad_api_position) in backward_grad_inputs_map.items(): + fill_zero_str = ( + f"{indent}const auto& input_metas = this->InputMeta();\n" + ) + for name, ( + ttype, + fwd_position, + grad_api_position, + ) in backward_grad_inputs_map.items(): if name in self.optional_inputs: if IsPlainTensorType(ttype): fill_zero_str += f"{indent}egr::EagerUtils::FillZeroForEmptyOptionalGradInput(&grads[{fwd_position}][0], input_metas[{fwd_position}][0]);\n" @@ -1684,40 +1958,67 @@ def GenerateNodeDefinition(self, next_grad_node_creation_str, inplace_check_str = "" optional_inplace_var_name = [] # Grad Ins from TensorWrappers - for name, (backward_input_type, is_fwd_input, - grad_api_position), in backward_forward_inputs_map.items(): + for ( + name, + (backward_input_type, is_fwd_input, grad_api_position), + ) in backward_forward_inputs_map.items(): tensor_wrapper_name = GetSavedName(name) transformed_tensor_name = self.TransformToNextGradName(name) - is_optional = (name in self.optional_inputs) + is_optional = name in self.optional_inputs tensor_wrapper_recover_str = f"{indent}auto {transformed_tensor_name} = egr::EagerUtils::RecoverTensorWrapper(&this->{tensor_wrapper_name});" if backward_inplace_map and name in backward_inplace_map.keys(): if len(next_grad_node_creation_str) > 0: - if (transformed_tensor_name - in backward_forward_inputs_map_next) and ( - backward_forward_inputs_map_next[ - transformed_tensor_name][1]): + if ( + transformed_tensor_name + in backward_forward_inputs_map_next + ) and ( + backward_forward_inputs_map_next[ + transformed_tensor_name + ][1] + ): optional_inplace_var_name.append( - transformed_tensor_name) - tensor_wrapper_intermidiate_tensor_str = f"(&this->{tensor_wrapper_name})->get_intermidiate_tensor()" + transformed_tensor_name + ) + tensor_wrapper_intermidiate_tensor_str = ( + f"(&this->{tensor_wrapper_name})->get_intermidiate_tensor()" + ) inplace_check_str += CHECK_BACKWARD_INPLACE_TEMPLATE.format( - transformed_tensor_name, transformed_tensor_name, name, - transformed_tensor_name, transformed_tensor_name, - transformed_tensor_name, transformed_tensor_name, - tensor_wrapper_intermidiate_tensor_str) + transformed_tensor_name, + transformed_tensor_name, + name, + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + tensor_wrapper_intermidiate_tensor_str, + ) inplace_grad_input_str = transformed_tensor_name if is_optional: if backward_input_type == "std::vector": - tensor_wrapper_recover_str += "\n" + CREATE_RECOVER_OPTIONAL_VECTOR_TENSOR_TEMPLATE.format( - transformed_tensor_name, transformed_tensor_name, - transformed_tensor_name, transformed_tensor_name) + tensor_wrapper_recover_str += ( + "\n" + + CREATE_RECOVER_OPTIONAL_VECTOR_TENSOR_TEMPLATE.format( + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + ) + ) else: - tensor_wrapper_recover_str += "\n" + CREATE_RECOVER_OPTIONAL_TENSOR_TEMPLATE.format( - transformed_tensor_name, transformed_tensor_name, - transformed_tensor_name, transformed_tensor_name) + tensor_wrapper_recover_str += ( + "\n" + + CREATE_RECOVER_OPTIONAL_TENSOR_TEMPLATE.format( + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + ) + ) - grad_api_args[ - grad_api_position] = transformed_tensor_name + "_optional" + grad_api_args[grad_api_position] = ( + transformed_tensor_name + "_optional" + ) else: grad_api_args[grad_api_position] = transformed_tensor_name @@ -1725,37 +2026,57 @@ def GenerateNodeDefinition(self, next_grad_node_creation_str, get_grad_in_args_list.append(tensor_wrapper_recover_str) # Grad Ins from grads - for name, (ttype, fwd_position, - grad_api_position) in backward_grad_inputs_map.items(): + for name, ( + ttype, + fwd_position, + grad_api_position, + ) in backward_grad_inputs_map.items(): transformed_tensor_name = self.TransformToNextGradName(name) - is_optional = (name in self.optional_inputs) + is_optional = name in self.optional_inputs if IsPlainTensorType(ttype): get_tensor_str = f"{indent}auto& {transformed_tensor_name} = hooked_grads[{fwd_position}][0];" # Inplace in backward op if backward_inplace_map and name in backward_inplace_map.keys(): if len(next_grad_node_creation_str) > 0: - if (transformed_tensor_name - in backward_forward_inputs_map_next) and ( - backward_forward_inputs_map_next[ - transformed_tensor_name][1]): + if ( + transformed_tensor_name + in backward_forward_inputs_map_next + ) and ( + backward_forward_inputs_map_next[ + transformed_tensor_name + ][1] + ): optional_inplace_var_name.append( - transformed_tensor_name) + transformed_tensor_name + ) grads_tensor_str = f"grads[{fwd_position}][0]" inplace_check_str += CHECK_BACKWARD_INPLACE_TEMPLATE.format( - transformed_tensor_name, transformed_tensor_name, name, - transformed_tensor_name, transformed_tensor_name, - transformed_tensor_name, transformed_tensor_name, - grads_tensor_str) + transformed_tensor_name, + transformed_tensor_name, + name, + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + grads_tensor_str, + ) inplace_grad_input_str = transformed_tensor_name if is_optional: - get_tensor_str += "\n" + CREATE_PLAIN_OPTIONAL_TENSOR_TEMPLATE.format( - transformed_tensor_name, transformed_tensor_name, - transformed_tensor_name, transformed_tensor_name) + get_tensor_str += ( + "\n" + + CREATE_PLAIN_OPTIONAL_TENSOR_TEMPLATE.format( + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + transformed_tensor_name, + ) + ) grad_api_args[ - grad_api_position] = f"{transformed_tensor_name}_optional" + grad_api_position + ] = f"{transformed_tensor_name}_optional" else: grad_api_args[grad_api_position] = transformed_tensor_name else: @@ -1768,7 +2089,9 @@ def GenerateNodeDefinition(self, next_grad_node_creation_str, # Grad Attrs for name, _, _, grad_api_position in backward_attrs_list: saved_attribute_name = GetSavedName(name) - get_attr_str = f"{indent}auto& {name} = this->{saved_attribute_name};" + get_attr_str = ( + f"{indent}auto& {name} = this->{saved_attribute_name};" + ) grad_api_args[grad_api_position] = name get_grad_in_args_list.append(get_attr_str) @@ -1790,13 +2113,18 @@ def GenerateNodeDefinition(self, next_grad_node_creation_str, # Grad Outputs out_index = -1 out_assign_str = "" - for name, (ttype, fwd_position, - grad_api_position) in backward_grad_outputs_map.items(): + for name, ( + ttype, + fwd_position, + grad_api_position, + ) in backward_grad_outputs_map.items(): transformed_tensor_name = self.TransformToNextGradName(name) out_index = out_index + 1 if is_invoke_forward_api: if len(backward_grad_outputs_map) == 1: - out_assign_str += f"{indent}*api_output_{out_index} = api_output;\n" + out_assign_str += ( + f"{indent}*api_output_{out_index} = api_output;\n" + ) else: out_assign_str += f"{indent}*api_output_{out_index} = std::get<{out_index}>(api_output);\n" else: @@ -1809,7 +2137,9 @@ def GenerateNodeDefinition(self, next_grad_node_creation_str, }}""" if IsPlainTensorType(ttype): - if backward_inplace_map and name in backward_inplace_map.values( + if ( + backward_inplace_map + and name in backward_inplace_map.values() ): inplace_str = f""" if (api_output_{out_index} != nullptr && can_be_inplaced) {{ egr::EagerUtils::HandleViewBetweenInputAndOutput({inplace_grad_input_str}, api_output_{out_index}); @@ -1844,14 +2174,19 @@ def GenerateNodeDefinition(self, next_grad_node_creation_str, if is_invoke_forward_api: autograd_api_out = "auto" - if len(self.backward_inplace_map) > 0 and len( - backward_grad_outputs_map) == 1: + if ( + len(self.backward_inplace_map) > 0 + and len(backward_grad_outputs_map) == 1 + ): autograd_api_out = "auto&" - forward_api_name = self.grad_api_contents['invoke'].split( - '(')[0].strip() + forward_api_name = ( + self.grad_api_contents['invoke'].split('(')[0].strip() + ) autograd_api = self.grad_api_contents['invoke'].replace( forward_api_name, - GetDygraphForwardFunctionName(forward_api_name), 1) + GetDygraphForwardFunctionName(forward_api_name), + 1, + ) grad_function_call_str = f""" if (trace_backward) {{ {indent}{autograd_api_out} api_output = {autograd_api}; @@ -1865,7 +2200,8 @@ def GenerateNodeDefinition(self, next_grad_node_creation_str, # Check Nan and Inf check_nan_inf_str = CHECK_NAN_AND_INF_TEMPLATE.format( - backward_api_name, "returns") + backward_api_name, "returns" + ) # Prepare for Node Creation if Necessary outputs_autograd_meta_str = "" @@ -1878,14 +2214,19 @@ def GenerateNodeDefinition(self, next_grad_node_creation_str, # TODO(jiabin): Optimize this with SetStopGradient instead of Pass Stop gradient num_fwd_outputs = len(backward_grad_outputs_map.keys()) - for name, (rtype, pos, - grad_api_position) in backward_grad_outputs_map.items(): + for name, ( + rtype, + pos, + grad_api_position, + ) in backward_grad_outputs_map.items(): transformed_tensor_name = self.TransformToNextGradName(name) output_autograd_meta_name = GetAutoGradMetaName( - transformed_tensor_name) + transformed_tensor_name + ) output_autograd_meta_vec_name = GetAutoGradMetaVectorName( - transformed_tensor_name) + transformed_tensor_name + ) if IsPlainTensorType(rtype): output_autograd_meta = f""" auto& {transformed_tensor_name} = returns[{pos}][0]; @@ -1923,15 +2264,20 @@ def GenerateNodeDefinition(self, next_grad_node_creation_str, # For inputs outputs prepare for logging var_str = f"\n{indent} std::string input_str = \"\";" var_str += f"\n{indent} std::string output_str = \"\";" - for name, (ttype, fwd_position, - grad_api_position) in backward_grad_inputs_map.items(): + for name, ( + ttype, + fwd_position, + grad_api_position, + ) in backward_grad_inputs_map.items(): new_name = self.TransformToNextGradName(name) var_str += f"\n{indent} const char* TENSOR_{new_name.upper()}_TEMPLATE = \" \\n( {new_name} , [%s]), \";" var_str += f"\n{indent} std::string input_{new_name}_str = paddle::string::Sprintf(TENSOR_{new_name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({new_name}));" var_str += f"\n{indent} input_str += input_{new_name}_str; " - for name, (backward_input_type, is_fwd_input, - grad_api_position), in backward_forward_inputs_map.items(): + for ( + name, + (backward_input_type, is_fwd_input, grad_api_position), + ) in backward_forward_inputs_map.items(): new_name = self.TransformToNextGradName(name) var_str += f"\n{indent} const char* TENSOR_{new_name.upper()}_TEMPLATE = \" \\n( {new_name} , [%s]), \";" var_str += f"\n{indent} std::string input_{new_name}_str = paddle::string::Sprintf(TENSOR_{new_name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({new_name}));" @@ -1939,8 +2285,11 @@ def GenerateNodeDefinition(self, next_grad_node_creation_str, before_log_str = BEFORE_LOG_PRINT_TEMPLATE.format(var_str) - for name, (ttype, fwd_position, - grad_api_position) in backward_grad_outputs_map.items(): + for name, ( + ttype, + fwd_position, + grad_api_position, + ) in backward_grad_outputs_map.items(): new_name = self.TransformToNextGradName(name) var_str += f"\n{indent} const char* TENSOR_{new_name.upper()}_TEMPLATE = \" \\n ( {new_name} , [%s]), \";" var_str += f"\n{indent} std::string output_{new_name}_str = paddle::string::Sprintf(TENSOR_{new_name.upper()}_TEMPLATE, egr::EagerUtils::TensorStr({new_name}));" @@ -1949,13 +2298,24 @@ def GenerateNodeDefinition(self, next_grad_node_creation_str, log_str = AFTER_LOG_PRINT_TEMPLATE.format(var_str) self.node_definition_str = GRAD_FUNCTION_TEMPLATE.format( - grad_node_name, self.backward_api_name, fill_zero_str, - get_grad_in_args_str, grad_function_prepare_str, - compute_require_next_grad_str, inplace_check_str, - inplace_for_grad_outs_str, self.backward_api_name, before_log_str, - grad_function_call_str, check_nan_inf_str, - outputs_autograd_meta_str, next_grad_node_creation_str, - self.backward_api_name, log_str, returns_str) + grad_node_name, + self.backward_api_name, + fill_zero_str, + get_grad_in_args_str, + grad_function_prepare_str, + compute_require_next_grad_str, + inplace_check_str, + inplace_for_grad_outs_str, + self.backward_api_name, + before_log_str, + grad_function_call_str, + check_nan_inf_str, + outputs_autograd_meta_str, + next_grad_node_creation_str, + self.backward_api_name, + log_str, + returns_str, + ) def run(self): super().run() @@ -1966,18 +2326,22 @@ def run(self): ## Code Generation ## ##################### # Higher-order GradNode generation - next_grad_node_creation_str, next_grad_node_out_list, backward_forward_inputs_map = self.GenerateHigherOrderNodeCreationCode( - ) + ( + next_grad_node_creation_str, + next_grad_node_out_list, + backward_forward_inputs_map, + ) = self.GenerateHigherOrderNodeCreationCode() self.GenerateNodeDeclaration() - self.GenerateNodeDefinition(next_grad_node_creation_str, - next_grad_node_out_list, - backward_forward_inputs_map) + self.GenerateNodeDefinition( + next_grad_node_creation_str, + next_grad_node_out_list, + backward_forward_inputs_map, + ) class DygraphForwardAndNodesGenerator(GeneratorBase): - def __init__(self, api_yaml_path, backward_yaml_path): # Parent members: # self.namespace @@ -1995,8 +2359,9 @@ def __init__(self, api_yaml_path, backward_yaml_path): self.node_definition_str = "" def CollectIsForwardOnly(self, forward_api_contents): - self.is_forward_only = False if 'backward' in forward_api_contents.keys( - ) else True + self.is_forward_only = ( + False if 'backward' in forward_api_contents.keys() else True + ) def ParseYamlContents(self): self.ParseForwardYamlContents() @@ -2010,11 +2375,13 @@ def ParseYamlContents(self): def GetBackwardAPIContents(self, forward_api_contents): grad_api_dict = self.grad_api_dict - if 'backward' not in forward_api_contents.keys(): return None + if 'backward' not in forward_api_contents.keys(): + return None backward_api_name = forward_api_contents['backward'] assert backward_api_name in grad_api_dict.keys(), AssertMessage( - backward_api_name, grad_api_dict.keys()) + backward_api_name, grad_api_dict.keys() + ) backward_api_contents = grad_api_dict[backward_api_name] return backward_api_contents @@ -2028,7 +2395,8 @@ def GenerateCode(self): namespace = self.namespace for forward_api_contents in forward_api_list: - if forward_api_contents['op'] in black_ops_list: continue + if forward_api_contents['op'] in black_ops_list: + continue self.CollectIsForwardOnly(forward_api_contents) @@ -2036,34 +2404,50 @@ def GenerateCode(self): backward_api_contents = None else: backward_api_contents = self.GetBackwardAPIContents( - forward_api_contents) + forward_api_contents + ) # Generate Dygraph Forward Function function_generator = DygraphForwardFunctionGenerator( - forward_api_contents, backward_api_contents, forward_apis_dict, - namespace) + forward_api_contents, + backward_api_contents, + forward_apis_dict, + namespace, + ) function_generator.run() - self.forward_definition_str += function_generator.forward_definition_str + "\n" - self.forward_declaration_str += function_generator.forward_declaration_str + "\n" + self.forward_definition_str += ( + function_generator.forward_definition_str + "\n" + ) + self.forward_declaration_str += ( + function_generator.forward_declaration_str + "\n" + ) # Generate Dygraph GradNode Function while True: if backward_api_contents is None: break next_grad_api_contents = self.GetBackwardAPIContents( - backward_api_contents) + backward_api_contents + ) - node_generator = DygraphNodeGenerator(forward_api_contents, - backward_api_contents, - forward_apis_dict, - namespace, - next_grad_api_contents) + node_generator = DygraphNodeGenerator( + forward_api_contents, + backward_api_contents, + forward_apis_dict, + namespace, + next_grad_api_contents, + ) node_generator.run() - self.node_declaration_str += node_generator.node_declaration_str + "\n" - self.node_definition_str += node_generator.node_definition_str + "\n" + self.node_declaration_str += ( + node_generator.node_declaration_str + "\n" + ) + self.node_definition_str += ( + node_generator.node_definition_str + "\n" + ) - if next_grad_api_contents is None: break + if next_grad_api_contents is None: + break # Detect if there exists higher-order GradNode forward_api_contents = backward_api_contents @@ -2076,13 +2460,17 @@ def GenerateCode(self): if namespace.endswith("::"): namespace = namespace[:-2] self.forward_definition_str = NAMESPACE_WRAPPER_TEMPLATE.format( - namespace, self.forward_definition_str) + namespace, self.forward_definition_str + ) self.forward_declaration_str = NAMESPACE_WRAPPER_TEMPLATE.format( - namespace, self.forward_declaration_str) + namespace, self.forward_declaration_str + ) self.node_declaration_str = NAMESPACE_WRAPPER_TEMPLATE.format( - namespace, self.node_declaration_str) + namespace, self.node_declaration_str + ) self.node_definition_str = NAMESPACE_WRAPPER_TEMPLATE.format( - namespace, self.node_definition_str) + namespace, self.node_definition_str + ) def run(self): self.ParseYamlContents() @@ -2118,8 +2506,9 @@ def GenerateForwardCCFile(filepath, forward_definition_str): os.remove(filepath) core_ops_info_str = GenerateCoreOpInfoDefinition() - file_contents = FORWARD_CC_FILE_TEMPLATE.format(core_ops_info_str, - forward_definition_str) + file_contents = FORWARD_CC_FILE_TEMPLATE.format( + core_ops_info_str, forward_definition_str + ) with open(filepath, 'a') as f: f.write(file_contents) @@ -2130,7 +2519,8 @@ def GenerateForwardHFile(filepath, forward_function_declaration_str): core_ops_info_str = GenerateCoreOpInfoDeclaration() file_contents = FORWARD_H_FILE_TEMPLATE.format( - core_ops_info_str, forward_function_declaration_str) + core_ops_info_str, forward_function_declaration_str + ) with open(filepath, 'a') as f: f.write(file_contents) @@ -2157,8 +2547,9 @@ def GenerateForwardHFile(filepath, forward_function_declaration_str): else: backward_yaml_path = None - generator = DygraphForwardAndNodesGenerator(api_yaml_path, - backward_yaml_path) + generator = DygraphForwardAndNodesGenerator( + api_yaml_path, backward_yaml_path + ) generator.run() node_declaration_str += generator.node_declaration_str + "\n" diff --git a/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py b/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py index 5aac3574ba772..dc38d3d46b293 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py +++ b/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py @@ -25,7 +25,7 @@ def SkipAPIGeneration(forward_api_name): - return (forward_api_name in skipped_forward_api_names) + return forward_api_name in skipped_forward_api_names atype_to_parsing_function = { @@ -61,28 +61,25 @@ def FindParsingFunctionFromAttributeType(atype): ########################## ## Refactored Functions ## ########################## -PARSE_PYTHON_C_TENSORS_TEMPLATE = \ -" auto {} = {}(\"{}\", \"{}\", args, {}, {});\n" +PARSE_PYTHON_C_TENSORS_TEMPLATE = ( + " auto {} = {}(\"{}\", \"{}\", args, {}, {});\n" +) -PARSE_PYTHON_C_ARGS_TEMPLATE = \ -""" PyObject* {}_obj = PyTuple_GET_ITEM(args, {}); +PARSE_PYTHON_C_ARGS_TEMPLATE = """ PyObject* {}_obj = PyTuple_GET_ITEM(args, {}); {} {} = {}({}_obj, \"{}\", {}); """ -RECORD_EVENT_TEMPLATE = \ -"paddle::platform::RecordEvent {}(\"{} {}\", paddle::platform::TracerEventType::UserDefined, 1);" +RECORD_EVENT_TEMPLATE = "paddle::platform::RecordEvent {}(\"{} {}\", paddle::platform::TracerEventType::UserDefined, 1);" -RETURN_INPLACE_PYOBJECT_TEMPLATE = \ -""" +RETURN_INPLACE_PYOBJECT_TEMPLATE = """ inplace_var_idx_map[{}] = {}; """ -PYTHON_C_FUNCTION_TEMPLATE = \ -""" +PYTHON_C_FUNCTION_TEMPLATE = """ static PyObject * eager_api_{}(PyObject *self, PyObject *args, PyObject *kwargs) {{ {} PyThreadState *tstate = nullptr; @@ -115,8 +112,7 @@ def FindParsingFunctionFromAttributeType(atype): NOAMP_DYGRAPH_FUNCTION_TEMPLATE = "decltype({}({})) out = {}({});" -FUNCTION_SET_DEVICE_TEMPLATE = \ -"""{} if (paddle::platform::is_gpu_place(place)) {{ +FUNCTION_SET_DEVICE_TEMPLATE = """{} if (paddle::platform::is_gpu_place(place)) {{ #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) phi::backends::gpu::SetDeviceId(place.device); VLOG(1) <<"CurrentDeviceId: " << phi::backends::gpu::GetCurrentDeviceId() << " from " << (int)place.device; @@ -136,16 +132,13 @@ def FindParsingFunctionFromAttributeType(atype): }} """ -FUNCTION_NAME_TEMPLATE = \ -"{}{}{}" +FUNCTION_NAME_TEMPLATE = "{}{}{}" -PYTHON_C_FUNCTION_REG_TEMPLATE = \ -" {{\"{}{}\", (PyCFunction)(void(*)(void)) {}eager_api_{}, METH_VARARGS | METH_KEYWORDS, \"C++ interface function for {} in dygraph.\"}},\n" +PYTHON_C_FUNCTION_REG_TEMPLATE = " {{\"{}{}\", (PyCFunction)(void(*)(void)) {}eager_api_{}, METH_VARARGS | METH_KEYWORDS, \"C++ interface function for {} in dygraph.\"}},\n" -PYTHON_C_WRAPPER_TEMPLATE = \ -""" +PYTHON_C_WRAPPER_TEMPLATE = """ #include #include "paddle/fluid/platform/enforce.h" #include "paddle/phi/api/include/strings_api.h" @@ -184,8 +177,7 @@ def FindParsingFunctionFromAttributeType(atype): """ -CORE_OPS_INFO = \ -""" +CORE_OPS_INFO = """ static PyObject * eager_get_core_ops_args_info(PyObject *self) { PyThreadState *tstate = nullptr; try { @@ -230,15 +222,13 @@ def FindParsingFunctionFromAttributeType(atype): """ -CORE_OPS_INFO_REGISTRY = \ -""" +CORE_OPS_INFO_REGISTRY = """ {\"get_core_ops_args_info\", (PyCFunction)(void(*)(void))eager_get_core_ops_args_info, METH_NOARGS, \"C++ interface function for eager_get_core_ops_args_info.\"}, {\"get_core_ops_args_type_info\", (PyCFunction)(void(*)(void))eager_get_core_ops_args_type_info, METH_NOARGS, \"C++ interface function for eager_get_core_ops_args_type_info.\"}, {\"get_core_ops_returns_info\", (PyCFunction)(void(*)(void))eager_get_core_ops_returns_info, METH_NOARGS, \"C++ interface function for eager_get_core_ops_returns_info.\"}, """ -NAMESPACE_WRAPPER_TEMPLATE = \ -"""namespace {} {{ +NAMESPACE_WRAPPER_TEMPLATE = """namespace {} {{ {} }} """ @@ -248,21 +238,20 @@ def FindParsingFunctionFromAttributeType(atype): ## Generator Classes ## ####################### class PythonCSingleFunctionGenerator(FunctionGeneratorBase): - def __init__(self, forward_api_contents, namespace): # Members from Parent: - #self.namespace - #self.forward_api_contents - #self.forward_api_name - #self.orig_forward_inputs_list - #self.orig_forward_attrs_list - #self.orig_forward_returns_list - #self.forward_inputs_position_map - #self.forward_outputs_position_map - #self.optional_inputs - #self.no_need_buffers - #self.intermediate_outputs - #self.forward_inplace_map + # self.namespace + # self.forward_api_contents + # self.forward_api_name + # self.orig_forward_inputs_list + # self.orig_forward_attrs_list + # self.orig_forward_returns_list + # self.forward_inputs_position_map + # self.forward_outputs_position_map + # self.optional_inputs + # self.no_need_buffers + # self.intermediate_outputs + # self.forward_inplace_map FunctionGeneratorBase.__init__(self, forward_api_contents, namespace) self.is_forward_only = True @@ -273,8 +262,9 @@ def __init__(self, forward_api_contents, namespace): def CollectIsForwardOnly(self): forward_api_contents = self.forward_api_contents - self.is_forward_only = False if 'backward' in forward_api_contents.keys( - ) else True + self.is_forward_only = ( + False if 'backward' in forward_api_contents.keys() else True + ) def GeneratePythonCFunction(self): namespace = self.namespace @@ -293,25 +283,53 @@ def GeneratePythonCFunction(self): for name, (ttype, pos) in forward_inputs_position_map.items(): if forward_inplace_map and name in forward_inplace_map.keys(): inplace_args_pos_map[name] = pos - is_optional = (name in optional_inputs) + is_optional = name in optional_inputs if IsVectorTensorType(ttype): if is_optional: - get_eager_tensor_str += PARSE_PYTHON_C_TENSORS_TEMPLATE.format( - name, "GetOptionalTensorListFromArgs", forward_api_name, - name, pos, "true") + get_eager_tensor_str += ( + PARSE_PYTHON_C_TENSORS_TEMPLATE.format( + name, + "GetOptionalTensorListFromArgs", + forward_api_name, + name, + pos, + "true", + ) + ) else: - get_eager_tensor_str += PARSE_PYTHON_C_TENSORS_TEMPLATE.format( - name, "GetTensorListFromArgs", forward_api_name, name, - pos, "false") + get_eager_tensor_str += ( + PARSE_PYTHON_C_TENSORS_TEMPLATE.format( + name, + "GetTensorListFromArgs", + forward_api_name, + name, + pos, + "false", + ) + ) else: if is_optional: - get_eager_tensor_str += PARSE_PYTHON_C_TENSORS_TEMPLATE.format( - name, "GetOptionalTensorFromArgs", forward_api_name, - name, pos, "true") + get_eager_tensor_str += ( + PARSE_PYTHON_C_TENSORS_TEMPLATE.format( + name, + "GetOptionalTensorFromArgs", + forward_api_name, + name, + pos, + "true", + ) + ) else: - get_eager_tensor_str += PARSE_PYTHON_C_TENSORS_TEMPLATE.format( - name, "GetTensorFromArgs", forward_api_name, name, pos, - "false") + get_eager_tensor_str += ( + PARSE_PYTHON_C_TENSORS_TEMPLATE.format( + name, + "GetTensorFromArgs", + forward_api_name, + name, + pos, + "false", + ) + ) if forward_inplace_map: for name, (ttype, pos) in forward_outputs_position_map.items(): @@ -319,26 +337,40 @@ def GeneratePythonCFunction(self): inplace_returns_pos_map[name] = pos parse_attributes_str = "" - expected_place_str = " auto place = egr::Controller::Instance().GetExpectedPlace();\n" + expected_place_str = ( + " auto place = egr::Controller::Instance().GetExpectedPlace();\n" + ) # Generate Python-C Attributes Parsing Logic for name, atype, _, pos in orig_forward_attrs_list: parsing_function_name = FindParsingFunctionFromAttributeType(atype) # Used input argument place if specified from Python frontend. - if len(expected_place_str - ) != 0 and parsing_function_name == "CastPyArg2Place": + if ( + len(expected_place_str) != 0 + and parsing_function_name == "CastPyArg2Place" + ): expected_place_str = "" - assert name == "place", "Only support 'place' as template argument name in FUNCTION_SET_DEVICE_TEMPLATE." + assert ( + name == "place" + ), "Only support 'place' as template argument name in FUNCTION_SET_DEVICE_TEMPLATE." parse_attributes_str += PARSE_PYTHON_C_ARGS_TEMPLATE.format( - name, pos, atype, name, parsing_function_name, name, - forward_api_name, pos) + name, + pos, + atype, + name, + parsing_function_name, + name, + forward_api_name, + pos, + ) set_device_str = FUNCTION_SET_DEVICE_TEMPLATE.format(expected_place_str) # Generate Dygraph Function Call Logic - num_args = len( - forward_inputs_position_map.keys()) + len(orig_forward_attrs_list) + num_args = len(forward_inputs_position_map.keys()) + len( + orig_forward_attrs_list + ) dygraph_function_call_list = ["" for i in range(num_args)] for name, (_, pos) in forward_inputs_position_map.items(): dygraph_function_call_list[pos] = f"{name}" @@ -348,23 +380,34 @@ def GeneratePythonCFunction(self): # Generate Python-C Function Definitions fwd_function_name = FUNCTION_NAME_TEMPLATE.format( - "::", namespace, GetForwardFunctionName(forward_api_name)) + "::", namespace, GetForwardFunctionName(forward_api_name) + ) return_str = " return ToPyObject(out);" # Generate Record Event for performance profiling pythonc_record_event_str = RECORD_EVENT_TEMPLATE.format( - "pythonc_record_event", forward_api_name, "pybind_imperative_func") + "pythonc_record_event", forward_api_name, "pybind_imperative_func" + ) noamp_dygraph_function_str = NOAMP_DYGRAPH_FUNCTION_TEMPLATE.format( - fwd_function_name, dygraph_function_call_str, fwd_function_name, - dygraph_function_call_str) + fwd_function_name, + dygraph_function_call_str, + fwd_function_name, + dygraph_function_call_str, + ) # Generate Python-C Function Definetion self.python_c_function_str = PYTHON_C_FUNCTION_TEMPLATE.format( - forward_api_name, pythonc_record_event_str, forward_api_name, - get_eager_tensor_str, parse_attributes_str, set_device_str, - noamp_dygraph_function_str, return_str) + forward_api_name, + pythonc_record_event_str, + forward_api_name, + get_eager_tensor_str, + parse_attributes_str, + set_device_str, + noamp_dygraph_function_str, + return_str, + ) # Set prefix of forward_api_name to avoid conflicts prefix = self.namespace.strip("::") @@ -372,37 +415,63 @@ def GeneratePythonCFunction(self): # Generate Python-C Function Registration self.python_c_function_reg_str = PYTHON_C_FUNCTION_REG_TEMPLATE.format( - forward_api_name_prefix, forward_api_name, namespace, - forward_api_name, forward_api_name) + forward_api_name_prefix, + forward_api_name, + namespace, + forward_api_name, + forward_api_name, + ) if forward_inplace_map: inplaced_forward_api_name = GetInplacedFunctionName( - self.forward_api_name) + self.forward_api_name + ) inplaced_fwd_function_name = FUNCTION_NAME_TEMPLATE.format( - "::", namespace, - GetForwardFunctionName(inplaced_forward_api_name)) - - inplace_noamp_dygraph_function_str = NOAMP_DYGRAPH_FUNCTION_TEMPLATE.format( - inplaced_fwd_function_name, dygraph_function_call_str, - inplaced_fwd_function_name, dygraph_function_call_str) + "::", + namespace, + GetForwardFunctionName(inplaced_forward_api_name), + ) + + inplace_noamp_dygraph_function_str = ( + NOAMP_DYGRAPH_FUNCTION_TEMPLATE.format( + inplaced_fwd_function_name, + dygraph_function_call_str, + inplaced_fwd_function_name, + dygraph_function_call_str, + ) + ) return_str = " std::map inplace_var_idx_map;" for inplace_input, inplace_output in forward_inplace_map.items(): return_str += RETURN_INPLACE_PYOBJECT_TEMPLATE.format( inplace_returns_pos_map[inplace_output], - inplace_args_pos_map[inplace_input]) - return_str += " return ToPyObject(out, args, inplace_var_idx_map);" + inplace_args_pos_map[inplace_input], + ) + return_str += ( + " return ToPyObject(out, args, inplace_var_idx_map);" + ) # Generate Python-C Function Definetion python_c_inplace_func_str = PYTHON_C_FUNCTION_TEMPLATE.format( - inplaced_forward_api_name, pythonc_record_event_str, - inplaced_forward_api_name, get_eager_tensor_str, - parse_attributes_str, set_device_str, - inplace_noamp_dygraph_function_str, return_str) - - python_c_inplace_func_reg_str = PYTHON_C_FUNCTION_REG_TEMPLATE.format( - forward_api_name_prefix, inplaced_forward_api_name, namespace, - inplaced_forward_api_name, inplaced_forward_api_name) + inplaced_forward_api_name, + pythonc_record_event_str, + inplaced_forward_api_name, + get_eager_tensor_str, + parse_attributes_str, + set_device_str, + inplace_noamp_dygraph_function_str, + return_str, + ) + + python_c_inplace_func_reg_str = ( + PYTHON_C_FUNCTION_REG_TEMPLATE.format( + forward_api_name_prefix, + inplaced_forward_api_name, + namespace, + inplaced_forward_api_name, + inplaced_forward_api_name, + ) + ) # self.forward_api_name ending with '_' means it only has inplace api if self.forward_api_name[-1] == '_': @@ -427,11 +496,13 @@ def run(self): # Initialized orig_forward_inputs_list, orig_forward_returns_list, orig_forward_attrs_list self.CollectOriginalForwardInfo() - if SkipAPIGeneration(self.forward_api_name): return False + if SkipAPIGeneration(self.forward_api_name): + return False # Initialized forward_inputs_position_map, forward_outputs_position_map - self.DetermineForwardPositionMap(self.orig_forward_inputs_list, - self.orig_forward_returns_list) + self.DetermineForwardPositionMap( + self.orig_forward_inputs_list, self.orig_forward_returns_list + ) # Code Generation self.GeneratePythonCFunction() @@ -440,7 +511,6 @@ def run(self): class PythonCGenerator(GeneratorBase): - def __init__(self, path): # Parent members: # self.namespace @@ -458,12 +528,17 @@ def GeneratePythonCFunctions(self): forward_api_list = self.forward_api_list for forward_api_content in forward_api_list: f_generator = PythonCSingleFunctionGenerator( - forward_api_content, namespace) + forward_api_content, namespace + ) status = f_generator.run() if status == True: - self.python_c_functions_str += f_generator.python_c_function_str + "\n" - self.python_c_functions_reg_str += f_generator.python_c_function_reg_str + self.python_c_functions_str += ( + f_generator.python_c_function_str + "\n" + ) + self.python_c_functions_reg_str += ( + f_generator.python_c_function_reg_str + ) def AttachNamespace(self): namespace = self.namespace @@ -473,7 +548,8 @@ def AttachNamespace(self): if namespace.endswith("::"): namespace = namespace[:-2] self.python_c_functions_str = NAMESPACE_WRAPPER_TEMPLATE.format( - namespace, python_c_functions_str) + namespace, python_c_functions_str + ) def run(self): # Infer namespace from yaml_path @@ -494,7 +570,8 @@ def run(self): ############################ def ParseArguments(): parser = argparse.ArgumentParser( - description='Eager Code Generator Args Parser') + description='Eager Code Generator Args Parser' + ) parser.add_argument('--api_yaml_path', type=str) parser.add_argument('--output_path', type=str) @@ -508,15 +585,18 @@ def GenerateCoreOpsInfoMap(): def GeneratePythonCWrappers(python_c_function_str, python_c_function_reg_str): - core_ops_infos_definition, core_ops_infos_registry = GenerateCoreOpsInfoMap( - ) + ( + core_ops_infos_definition, + core_ops_infos_registry, + ) = GenerateCoreOpsInfoMap() python_c_function_str += core_ops_infos_definition python_c_function_reg_str += core_ops_infos_registry python_c_function_reg_str += " {nullptr,nullptr,0,nullptr}" - python_c_str = PYTHON_C_WRAPPER_TEMPLATE.format(python_c_function_str, - python_c_function_reg_str) + python_c_str = PYTHON_C_WRAPPER_TEMPLATE.format( + python_c_function_str, python_c_function_reg_str + ) return python_c_str @@ -538,11 +618,16 @@ def GeneratePythonCFile(filepath, python_c_str): py_c_generator = PythonCGenerator(api_yaml_path) py_c_generator.run() - generated_python_c_functions += py_c_generator.python_c_functions_str + "\n" - generated_python_c_registration += py_c_generator.python_c_functions_reg_str + generated_python_c_functions += ( + py_c_generator.python_c_functions_str + "\n" + ) + generated_python_c_registration += ( + py_c_generator.python_c_functions_reg_str + ) - python_c_str = GeneratePythonCWrappers(generated_python_c_functions, - generated_python_c_registration) + python_c_str = GeneratePythonCWrappers( + generated_python_c_functions, generated_python_c_registration + ) output_path = args.output_path for path in [output_path]: diff --git a/paddle/fluid/inference/tests/api/full_ILSVRC2012_val_preprocess.py b/paddle/fluid/inference/tests/api/full_ILSVRC2012_val_preprocess.py index 5cda10f873cd3..219ce72077cd5 100644 --- a/paddle/fluid/inference/tests/api/full_ILSVRC2012_val_preprocess.py +++ b/paddle/fluid/inference/tests/api/full_ILSVRC2012_val_preprocess.py @@ -94,8 +94,9 @@ def download_concat(cache_folder, zip_path): def print_processbar(done_percentage): done_filled = done_percentage * '=' empty_filled = (100 - done_percentage) * ' ' - sys.stdout.write("\r[%s%s]%d%%" % - (done_filled, empty_filled, done_percentage)) + sys.stdout.write( + "\r[%s%s]%d%%" % (done_filled, empty_filled, done_percentage) + ) sys.stdout.flush() @@ -137,7 +138,7 @@ def convert_Imagenet_tar2bin(tar_file, output_file): val_dict[name] = label for img_name in dataset.keys(): - remove_len = (len(FOLDER_NAME)) + remove_len = len(FOLDER_NAME) img_name_prim = img_name[remove_len:] label = val_dict[img_name_prim] label_int = (int)(label) @@ -156,19 +157,25 @@ def run_convert(): retry = 0 try_limit = 3 - while not (os.path.exists(output_file) - and os.path.getsize(output_file) == FULL_SIZE_BYTES): + while not ( + os.path.exists(output_file) + and os.path.getsize(output_file) == FULL_SIZE_BYTES + ): if os.path.exists(output_file): sys.stderr.write( - "\n\nThe existing binary file[{}] is broken. Start to generate new one...\n\n" - .format(output_file)) + "\n\nThe existing binary file[{}] is broken. Start to generate new one...\n\n".format( + output_file + ) + ) os.remove(output_file) if retry < try_limit: retry = retry + 1 else: raise RuntimeError( - "Can not convert the dataset to binary file with try limit {0}". - format(try_limit)) + "Can not convert the dataset to binary file with try limit {0}".format( + try_limit + ) + ) download_concat(cache_folder, zip_path) convert_Imagenet_tar2bin(zip_path, output_file) print("\nSuccess! The binary file can be found at {0}".format(output_file)) @@ -193,63 +200,75 @@ def convert_Imagenet_local2bin(args): if not os.path.exists(img_path): continue - #save image(float32) to file + # save image(float32) to file img = Image.open(img_path) img = process_image(img) np_img = np.array(img) - of.seek(SIZE_INT64 + - SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * idx) + of.seek( + SIZE_INT64 + SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * idx + ) of.write(np_img.astype('float32').tobytes()) - #save label(int64_t) to file + # save label(int64_t) to file label_int = (int)(label) np_label = np.array(label_int) - of.seek(SIZE_INT64 + - SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * num_images + - idx * SIZE_INT64) + of.seek( + SIZE_INT64 + + SIZE_FLOAT32 * DATA_DIM * DATA_DIM * 3 * num_images + + idx * SIZE_INT64 + ) of.write(np_label.astype('int64').tobytes()) # The bin file should contain # number of images + all images data + all corresponding labels # so the file target_size should be as follows - target_size = SIZE_INT64 + num_images * 3 * args.data_dim * args.data_dim * SIZE_FLOAT32 + num_images * SIZE_INT64 - if (os.path.getsize(bin_file_path) == target_size): + target_size = ( + SIZE_INT64 + + num_images * 3 * args.data_dim * args.data_dim * SIZE_FLOAT32 + + num_images * SIZE_INT64 + ) + if os.path.getsize(bin_file_path) == target_size: print( - "Success! The user data output binary file can be found at: {0}" - .format(bin_file_path)) + "Success! The user data output binary file can be found at: {0}".format( + bin_file_path + ) + ) else: print("Conversion failed!") def main_preprocess_Imagenet(args): parser = argparse.ArgumentParser( - description= - "Convert the full Imagenet val set or local data to binary file.", + description="Convert the full Imagenet val set or local data to binary file.", usage=None, - add_help=True) + add_help=True, + ) parser.add_argument( '--local', action="store_true", - help="If used, user need to set --data_dir and then convert file") - parser.add_argument("--data_dir", - default="", - type=str, - help="Dataset root directory") + help="If used, user need to set --data_dir and then convert file", + ) + parser.add_argument( + "--data_dir", default="", type=str, help="Dataset root directory" + ) parser.add_argument( "--label_list", type=str, default="val_list.txt", - help= - "List of object labels with same sequence as denoted in the annotation file" + help="List of object labels with same sequence as denoted in the annotation file", + ) + parser.add_argument( + "--output_file", + type=str, + default="imagenet_small.bin", + help="File path of the output binary file", + ) + parser.add_argument( + "--data_dim", + type=int, + default=DATA_DIM, + help="Image preprocess with data_dim width and height", ) - parser.add_argument("--output_file", - type=str, - default="imagenet_small.bin", - help="File path of the output binary file") - parser.add_argument("--data_dim", - type=int, - default=DATA_DIM, - help="Image preprocess with data_dim width and height") args = parser.parse_args() if args.local: diff --git a/paddle/fluid/inference/tests/api/full_pascalvoc_test_preprocess.py b/paddle/fluid/inference/tests/api/full_pascalvoc_test_preprocess.py index 74739530b4724..6fc2e072e00cc 100644 --- a/paddle/fluid/inference/tests/api/full_pascalvoc_test_preprocess.py +++ b/paddle/fluid/inference/tests/api/full_pascalvoc_test_preprocess.py @@ -24,7 +24,9 @@ import tarfile import argparse -DATA_URL = "http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar" +DATA_URL = ( + "http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar" +) DATA_DIR = os.path.expanduser("~/.cache/paddle/dataset/pascalvoc/") TAR_FILE = "VOCtest_06-Nov-2007.tar" TAR_PATH = os.path.join(DATA_DIR, TAR_FILE) @@ -106,8 +108,9 @@ def convert_pascalvoc_local2bin(args): for object in objects: bbox_sample = [] # start from 1 - bbox_sample.append(float(label_list.index( - object.find('name').text))) + bbox_sample.append( + float(label_list.index(object.find('name').text)) + ) bbox = object.find('bndbox') difficult = float(object.find('difficult').text) bbox_sample.append(float(bbox.find('xmin').text) / im_width) @@ -118,7 +121,8 @@ def convert_pascalvoc_local2bin(args): bbox_labels.append(bbox_sample) bbox_labels = np.array(bbox_labels) - if len(bbox_labels) == 0: continue + if len(bbox_labels) == 0: + continue lbls.extend(bbox_labels[:, 0]) boxes.extend(bbox_labels[:, 1:5]) @@ -135,11 +139,17 @@ def convert_pascalvoc_local2bin(args): # number of images + all images data + an array that represent object numbers of each image # + labels of all objects in images + bboxes of all objects + difficulties of all objects # so the target size should be as follows: - target_size = SIZE_INT64 + image_nums * 3 * args.resize_h * args.resize_h * SIZE_FLOAT32 + image_nums * SIZE_INT64 + object_nums_sum * ( - SIZE_INT64 + 4 * SIZE_FLOAT32 + SIZE_INT64) - if (os.path.getsize(output_file_path) == target_size): - print("Success! \nThe local data output binary file can be found at: ", - output_file_path) + target_size = ( + SIZE_INT64 + + image_nums * 3 * args.resize_h * args.resize_h * SIZE_FLOAT32 + + image_nums * SIZE_INT64 + + object_nums_sum * (SIZE_INT64 + 4 * SIZE_FLOAT32 + SIZE_INT64) + ) + if os.path.getsize(output_file_path) == target_size: + print( + "Success! \nThe local data output binary file can be found at: ", + output_file_path, + ) else: print("Conversion failed!") @@ -147,8 +157,9 @@ def convert_pascalvoc_local2bin(args): def print_processbar(done_percentage): done_filled = done_percentage * '=' empty_filled = (100 - done_percentage) * ' ' - sys.stdout.write("\r[%s%s]%d%%" % - (done_filled, empty_filled, done_percentage)) + sys.stdout.write( + "\r[%s%s]%d%%" % (done_filled, empty_filled, done_percentage) + ) sys.stdout.flush() @@ -163,13 +174,30 @@ def convert_pascalvoc_tar2bin(tar_path, data_out_path): # map label to number (index) label_list = [ - "background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", - "car", "cat", "chair", "cow", "diningtable", "dog", "horse", - "motorbike", "person", "pottedplant", "sheep", "sofa", "train", - "tvmonitor" + "background", + "aeroplane", + "bicycle", + "bird", + "boat", + "bottle", + "bus", + "car", + "cat", + "chair", + "cow", + "diningtable", + "dog", + "horse", + "motorbike", + "person", + "pottedplant", + "sheep", + "sofa", + "train", + "tvmonitor", ] print_processbar(0) - #read from tar file and write to bin + # read from tar file and write to bin tar = tarfile.open(tar_path, "r") f_test = tar.extractfile(TEST_LIST_KEY).read() lines = f_test.split('\n') @@ -210,8 +238,9 @@ def convert_pascalvoc_tar2bin(tar_path, data_out_path): for object in objects: bbox_sample = [] - bbox_sample.append(float(label_list.index( - object.find('name').text))) + bbox_sample.append( + float(label_list.index(object.find('name').text)) + ) bbox = object.find('bndbox') difficult = float(object.find('difficult').text) bbox_sample.append(float(bbox.find('xmin').text) / im_width) @@ -222,7 +251,8 @@ def convert_pascalvoc_tar2bin(tar_path, data_out_path): bbox_labels.append(bbox_sample) bbox_labels = np.array(bbox_labels) - if len(bbox_labels) == 0: continue + if len(bbox_labels) == 0: + continue lbls.extend(bbox_labels[:, 0]) boxes.extend(bbox_labels[:, 1:5]) difficults.extend(bbox_labels[:, -1]) @@ -258,12 +288,16 @@ def download_pascalvoc(data_url, data_dir, tar_targethash, tar_path): def run_convert(): try_limit = 2 retry = 0 - while not (os.path.exists(DATA_OUT_PATH) and os.path.getsize(DATA_OUT_PATH) - == BIN_FULLSIZE and BIN_TARGETHASH == hashlib.md5( - open(DATA_OUT_PATH, 'rb').read()).hexdigest()): + while not ( + os.path.exists(DATA_OUT_PATH) + and os.path.getsize(DATA_OUT_PATH) == BIN_FULLSIZE + and BIN_TARGETHASH + == hashlib.md5(open(DATA_OUT_PATH, 'rb').read()).hexdigest() + ): if os.path.exists(DATA_OUT_PATH): sys.stderr.write( - "The existing binary file is broken. It is being removed...\n") + "The existing binary file is broken. It is being removed...\n" + ) os.remove(DATA_OUT_PATH) if retry < try_limit: retry = retry + 1 @@ -275,52 +309,60 @@ def run_convert(): def main_pascalvoc_preprocess(args): parser = argparse.ArgumentParser( - description= - "Convert the full pascalvoc val set or local data to binary file.", + description="Convert the full pascalvoc val set or local data to binary file.", usage=None, - add_help=True) + add_help=True, + ) parser.add_argument( '--local', action="store_true", - help="If used, user need to set --data_dir and then convert file") - parser.add_argument("--data_dir", - default="", - type=str, - help="Dataset root directory") + help="If used, user need to set --data_dir and then convert file", + ) + parser.add_argument( + "--data_dir", default="", type=str, help="Dataset root directory" + ) parser.add_argument( "--img_annotation_list", type=str, default="test_100.txt", - help= - "A file containing the image file path and corresponding annotation file path" + help="A file containing the image file path and corresponding annotation file path", ) parser.add_argument( "--label_file", type=str, default="label_list", - help= - "List of object labels with same sequence as denoted in the annotation file" + help="List of object labels with same sequence as denoted in the annotation file", + ) + parser.add_argument( + "--output_file", + type=str, + default="pascalvoc_small.bin", + help="File path of the output binary file", + ) + parser.add_argument( + "--resize_h", + type=int, + default=RESIZE_H, + help="Image preprocess with resize_h", + ) + parser.add_argument( + "--resize_w", + type=int, + default=RESIZE_W, + help="Image prerocess with resize_w", + ) + parser.add_argument( + "--mean_value", + type=str, + default=MEAN_VALUE, + help="Image preprocess with mean_value", + ) + parser.add_argument( + "--ap_version", + type=str, + default=AP_VERSION, + help="Image preprocess with ap_version", ) - parser.add_argument("--output_file", - type=str, - default="pascalvoc_small.bin", - help="File path of the output binary file") - parser.add_argument("--resize_h", - type=int, - default=RESIZE_H, - help="Image preprocess with resize_h") - parser.add_argument("--resize_w", - type=int, - default=RESIZE_W, - help="Image prerocess with resize_w") - parser.add_argument("--mean_value", - type=str, - default=MEAN_VALUE, - help="Image preprocess with mean_value") - parser.add_argument("--ap_version", - type=str, - default=AP_VERSION, - help="Image preprocess with ap_version") args = parser.parse_args() if args.local: convert_pascalvoc_local2bin(args) diff --git a/paddle/fluid/inference/tests/api/test_detection_dataset_preprocess.py b/paddle/fluid/inference/tests/api/test_detection_dataset_preprocess.py index 57920a8b27065..4ae32b0f77a84 100644 --- a/paddle/fluid/inference/tests/api/test_detection_dataset_preprocess.py +++ b/paddle/fluid/inference/tests/api/test_detection_dataset_preprocess.py @@ -17,13 +17,13 @@ class Test_Preprocess(unittest.TestCase): - def test_local_convert(self): os.system("python full_pascalvoc_test_preprocess.py --choice=local") def test_online_convert(self): os.system( - "python full_pascalvoc_test_preprocess.py --choice=VOC_test_2007") + "python full_pascalvoc_test_preprocess.py --choice=VOC_test_2007" + ) if __name__ == '__main__': diff --git a/paddle/fluid/pybind/generate_file_structures.py b/paddle/fluid/pybind/generate_file_structures.py index 869b27050a08c..4904f97dc2fdd 100644 --- a/paddle/fluid/pybind/generate_file_structures.py +++ b/paddle/fluid/pybind/generate_file_structures.py @@ -25,7 +25,8 @@ for i in range(split_count): empty_files.append( - os.path.join(pybind_dir, "op_function" + str(i + 1) + ".cc")) + os.path.join(pybind_dir, "op_function" + str(i + 1) + ".cc") + ) for path in empty_files: if not os.path.exists(path): diff --git a/paddle/infrt/tests/models/abs_model.py b/paddle/infrt/tests/models/abs_model.py index c94338be7788b..56e1bc8fa710f 100644 --- a/paddle/infrt/tests/models/abs_model.py +++ b/paddle/infrt/tests/models/abs_model.py @@ -19,7 +19,6 @@ class AbsNet(paddle.nn.Layer): - def __init__(self): super(AbsNet, self).__init__() @@ -32,6 +31,7 @@ def forward(self, x): # build network model = AbsNet() # save inferencing format model - net = to_static(model, - input_spec=[InputSpec(shape=[None, 1, 28, 28], name='x')]) + net = to_static( + model, input_spec=[InputSpec(shape=[None, 1, 28, 28], name='x')] + ) paddle.jit.save(net, sys.argv[1]) diff --git a/paddle/infrt/tests/models/efficientnet-b4/model.py b/paddle/infrt/tests/models/efficientnet-b4/model.py index 0c6163f3df272..1b493897e654a 100644 --- a/paddle/infrt/tests/models/efficientnet-b4/model.py +++ b/paddle/infrt/tests/models/efficientnet-b4/model.py @@ -20,6 +20,7 @@ import sys model = EfficientNet.from_name('efficientnet-b4') -net = to_static(model, - input_spec=[InputSpec(shape=[None, 3, 256, 256], name='x')]) +net = to_static( + model, input_spec=[InputSpec(shape=[None, 3, 256, 256], name='x')] +) paddle.jit.save(net, sys.argv[1]) diff --git a/paddle/infrt/tests/models/efficientnet-b4/net/efficientnet.py b/paddle/infrt/tests/models/efficientnet-b4/net/efficientnet.py index 75f6780484dd9..c14be25a81590 100644 --- a/paddle/infrt/tests/models/efficientnet-b4/net/efficientnet.py +++ b/paddle/infrt/tests/models/efficientnet-b4/net/efficientnet.py @@ -16,9 +16,15 @@ import paddle.nn as nn import paddle.nn.functional as F -from .utils import (round_filters, round_repeats, drop_connect, - get_same_padding_conv2d, get_model_params, - efficientnet_params, load_pretrained_weights) +from .utils import ( + round_filters, + round_repeats, + drop_connect, + get_same_padding_conv2d, + get_model_params, + efficientnet_params, + load_pretrained_weights, +) class MBConvBlock(nn.Layer): @@ -38,8 +44,9 @@ def __init__(self, block_args, global_params): self._block_args = block_args self._bn_mom = global_params.batch_norm_momentum self._bn_eps = global_params.batch_norm_epsilon - self.has_se = (self._block_args.se_ratio - is not None) and (0 < self._block_args.se_ratio <= 1) + self.has_se = (self._block_args.se_ratio is not None) and ( + 0 < self._block_args.se_ratio <= 1 + ) self.id_skip = block_args.id_skip # skip connection and drop connect # Get static or dynamic convolution depending on image size @@ -47,15 +54,19 @@ def __init__(self, block_args, global_params): # Expansion phase inp = self._block_args.input_filters # number of input channels - oup = self._block_args.input_filters * self._block_args.expand_ratio # number of output channels + oup = ( + self._block_args.input_filters * self._block_args.expand_ratio + ) # number of output channels if self._block_args.expand_ratio != 1: - self._expand_conv = Conv2d(in_channels=inp, - out_channels=oup, - kernel_size=1, - bias_attr=False) - self._bn0 = nn.BatchNorm2D(num_features=oup, - momentum=self._bn_mom, - epsilon=self._bn_eps) + self._expand_conv = Conv2d( + in_channels=inp, + out_channels=oup, + kernel_size=1, + bias_attr=False, + ) + self._bn0 = nn.BatchNorm2D( + num_features=oup, momentum=self._bn_mom, epsilon=self._bn_eps + ) # Depthwise convolution phase k = self._block_args.kernel_size @@ -66,32 +77,40 @@ def __init__(self, block_args, global_params): groups=oup, # groups makes it depthwise kernel_size=k, stride=s, - bias_attr=False) - self._bn1 = nn.BatchNorm2D(num_features=oup, - momentum=self._bn_mom, - epsilon=self._bn_eps) + bias_attr=False, + ) + self._bn1 = nn.BatchNorm2D( + num_features=oup, momentum=self._bn_mom, epsilon=self._bn_eps + ) # Squeeze and Excitation layer, if desired if self.has_se: num_squeezed_channels = max( 1, - int(self._block_args.input_filters * self._block_args.se_ratio)) - self._se_reduce = Conv2d(in_channels=oup, - out_channels=num_squeezed_channels, - kernel_size=1) - self._se_expand = Conv2d(in_channels=num_squeezed_channels, - out_channels=oup, - kernel_size=1) + int(self._block_args.input_filters * self._block_args.se_ratio), + ) + self._se_reduce = Conv2d( + in_channels=oup, + out_channels=num_squeezed_channels, + kernel_size=1, + ) + self._se_expand = Conv2d( + in_channels=num_squeezed_channels, + out_channels=oup, + kernel_size=1, + ) # Output phase final_oup = self._block_args.output_filters - self._project_conv = Conv2d(in_channels=oup, - out_channels=final_oup, - kernel_size=1, - bias_attr=False) - self._bn2 = nn.BatchNorm2D(num_features=final_oup, - momentum=self._bn_mom, - epsilon=self._bn_eps) + self._project_conv = Conv2d( + in_channels=oup, + out_channels=final_oup, + kernel_size=1, + bias_attr=False, + ) + self._bn2 = nn.BatchNorm2D( + num_features=final_oup, momentum=self._bn_mom, epsilon=self._bn_eps + ) self._swish = nn.Hardswish() def forward(self, inputs, drop_connect_rate=None): @@ -111,18 +130,26 @@ def forward(self, inputs, drop_connect_rate=None): if self.has_se: x_squeezed = F.adaptive_avg_pool2d(x, 1) x_squeezed = self._se_expand( - self._swish(self._se_reduce(x_squeezed))) + self._swish(self._se_reduce(x_squeezed)) + ) x = F.sigmoid(x_squeezed) * x x = self._bn2(self._project_conv(x)) # Skip connection and drop connect - input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters - if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters: + input_filters, output_filters = ( + self._block_args.input_filters, + self._block_args.output_filters, + ) + if ( + self.id_skip + and self._block_args.stride == 1 + and input_filters == output_filters + ): if drop_connect_rate: - x = drop_connect(x, - prob=drop_connect_rate, - training=self.training) + x = drop_connect( + x, prob=drop_connect_rate, training=self.training + ) x = x + inputs # skip connection return x @@ -161,15 +188,14 @@ def __init__(self, blocks_args=None, global_params=None): # Stem in_channels = 3 # rgb out_channels = round_filters( - 32, self._global_params) # number of output channels - self._conv_stem = Conv2d(in_channels, - out_channels, - kernel_size=3, - stride=2, - bias_attr=False) - self._bn0 = nn.BatchNorm2D(num_features=out_channels, - momentum=bn_mom, - epsilon=bn_eps) + 32, self._global_params + ) # number of output channels + self._conv_stem = Conv2d( + in_channels, out_channels, kernel_size=3, stride=2, bias_attr=False + ) + self._bn0 = nn.BatchNorm2D( + num_features=out_channels, momentum=bn_mom, epsilon=bn_eps + ) # Build blocks self._blocks = nn.LayerList([]) @@ -177,32 +203,37 @@ def __init__(self, blocks_args=None, global_params=None): # Update block input and output filters based on depth multiplier. block_args = block_args._replace( - input_filters=round_filters(block_args.input_filters, - self._global_params), - output_filters=round_filters(block_args.output_filters, - self._global_params), - num_repeat=round_repeats(block_args.num_repeat, - self._global_params)) + input_filters=round_filters( + block_args.input_filters, self._global_params + ), + output_filters=round_filters( + block_args.output_filters, self._global_params + ), + num_repeat=round_repeats( + block_args.num_repeat, self._global_params + ), + ) # The first block needs to take care of stride and filter size increase. self._blocks.append(MBConvBlock(block_args, self._global_params)) if block_args.num_repeat > 1: block_args = block_args._replace( - input_filters=block_args.output_filters, stride=1) + input_filters=block_args.output_filters, stride=1 + ) for _ in range(block_args.num_repeat - 1): - self._blocks.append(MBConvBlock(block_args, - self._global_params)) + self._blocks.append( + MBConvBlock(block_args, self._global_params) + ) # Head in_channels = block_args.output_filters # output of final block out_channels = round_filters(1280, self._global_params) - self._conv_head = Conv2d(in_channels, - out_channels, - kernel_size=1, - bias_attr=False) - self._bn1 = nn.BatchNorm2D(num_features=out_channels, - momentum=bn_mom, - epsilon=bn_eps) + self._conv_head = Conv2d( + in_channels, out_channels, kernel_size=1, bias_attr=False + ) + self._bn1 = nn.BatchNorm2D( + num_features=out_channels, momentum=bn_mom, epsilon=bn_eps + ) # Final linear layer self._avg_pooling = nn.AdaptiveAvgPool2D(1) @@ -217,7 +248,7 @@ def set_swish(self, memory_efficient=True): block.set_swish(memory_efficient) def extract_features(self, inputs): - """ Returns output of the final convolution layer """ + """Returns output of the final convolution layer""" # Stem x = self._swish(self._bn0(self._conv_stem(inputs))) @@ -235,7 +266,7 @@ def extract_features(self, inputs): return x def forward(self, inputs): - """ Calls extract_features to extract features, applies final linear layer, and returns logits. """ + """Calls extract_features to extract features, applies final linear layer, and returns logits.""" bs = inputs.shape[0] # Convolution layers x = self.extract_features(inputs) @@ -250,31 +281,33 @@ def forward(self, inputs): @classmethod def from_name(cls, model_name, override_params=None): cls._check_model_name_is_valid(model_name) - blocks_args, global_params = get_model_params(model_name, - override_params) + blocks_args, global_params = get_model_params( + model_name, override_params + ) return cls(blocks_args, global_params) @classmethod - def from_pretrained(cls, - model_name, - advprop=False, - num_classes=1000, - in_channels=3): - model = cls.from_name(model_name, - override_params={'num_classes': num_classes}) - load_pretrained_weights(model, - model_name, - load_fc=(num_classes == 1000), - advprop=advprop) + def from_pretrained( + cls, model_name, advprop=False, num_classes=1000, in_channels=3 + ): + model = cls.from_name( + model_name, override_params={'num_classes': num_classes} + ) + load_pretrained_weights( + model, model_name, load_fc=(num_classes == 1000), advprop=advprop + ) if in_channels != 3: Conv2d = get_same_padding_conv2d( - image_size=model._global_params.image_size) + image_size=model._global_params.image_size + ) out_channels = round_filters(32, model._global_params) - model._conv_stem = Conv2d(in_channels, - out_channels, - kernel_size=3, - stride=2, - bias_attr=False) + model._conv_stem = Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=2, + bias_attr=False, + ) return model @classmethod @@ -285,8 +318,9 @@ def get_image_size(cls, model_name): @classmethod def _check_model_name_is_valid(cls, model_name): - """ Validates model name. """ + """Validates model name.""" valid_models = ['efficientnet-b' + str(i) for i in range(9)] if model_name not in valid_models: - raise ValueError('model_name should be one of: ' + - ', '.join(valid_models)) + raise ValueError( + 'model_name should be one of: ' + ', '.join(valid_models) + ) diff --git a/paddle/infrt/tests/models/efficientnet-b4/net/utils.py b/paddle/infrt/tests/models/efficientnet-b4/net/utils.py index 62caf759ad3ca..9a060306995bb 100644 --- a/paddle/infrt/tests/models/efficientnet-b4/net/utils.py +++ b/paddle/infrt/tests/models/efficientnet-b4/net/utils.py @@ -22,25 +22,44 @@ import paddle.nn.functional as F # Parameters for the entire model (stem, all blocks, and head) -GlobalParams = collections.namedtuple('GlobalParams', [ - 'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate', 'num_classes', - 'width_coefficient', 'depth_coefficient', 'depth_divisor', 'min_depth', - 'drop_connect_rate', 'image_size' -]) +GlobalParams = collections.namedtuple( + 'GlobalParams', + [ + 'batch_norm_momentum', + 'batch_norm_epsilon', + 'dropout_rate', + 'num_classes', + 'width_coefficient', + 'depth_coefficient', + 'depth_divisor', + 'min_depth', + 'drop_connect_rate', + 'image_size', + ], +) # Parameters for an individual model block -BlockArgs = collections.namedtuple('BlockArgs', [ - 'kernel_size', 'num_repeat', 'input_filters', 'output_filters', - 'expand_ratio', 'id_skip', 'stride', 'se_ratio' -]) +BlockArgs = collections.namedtuple( + 'BlockArgs', + [ + 'kernel_size', + 'num_repeat', + 'input_filters', + 'output_filters', + 'expand_ratio', + 'id_skip', + 'stride', + 'se_ratio', + ], +) # Change namedtuple defaults -GlobalParams.__new__.__defaults__ = (None, ) * len(GlobalParams._fields) -BlockArgs.__new__.__defaults__ = (None, ) * len(BlockArgs._fields) +GlobalParams.__new__.__defaults__ = (None,) * len(GlobalParams._fields) +BlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields) def round_filters(filters, global_params): - """ Calculate and round number of filters based on depth multiplier. """ + """Calculate and round number of filters based on depth multiplier.""" multiplier = global_params.width_coefficient if not multiplier: return filters @@ -48,15 +67,16 @@ def round_filters(filters, global_params): min_depth = global_params.min_depth filters *= multiplier min_depth = min_depth or divisor - new_filters = max(min_depth, - int(filters + divisor / 2) // divisor * divisor) + new_filters = max( + min_depth, int(filters + divisor / 2) // divisor * divisor + ) if new_filters < 0.9 * filters: # prevent rounding by more than 10% new_filters += divisor return int(new_filters) def round_repeats(repeats, global_params): - """ Round number of filters based on depth multiplier. """ + """Round number of filters based on depth multiplier.""" multiplier = global_params.depth_coefficient if not multiplier: return repeats @@ -76,8 +96,8 @@ def drop_connect(inputs, prob, training): def get_same_padding_conv2d(image_size=None): - """ Chooses static padding if you have specified an image size, and dynamic padding otherwise. - Static padding is necessary for ONNX exporting of models. """ + """Chooses static padding if you have specified an image size, and dynamic padding otherwise. + Static padding is necessary for ONNX exporting of models.""" if image_size is None: return Conv2dDynamicSamePadding else: @@ -85,26 +105,31 @@ def get_same_padding_conv2d(image_size=None): class Conv2dDynamicSamePadding(nn.Conv2D): - """ 2D Convolutions like TensorFlow, for a dynamic image size """ - - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - dilation=1, - groups=1, - bias_attr=None): - super().__init__(in_channels, - out_channels, - kernel_size, - stride, - 0, - dilation, - groups, - bias_attr=bias_attr) - self.stride = self._stride if len( - self._stride) == 2 else [self._stride[0]] * 2 + """2D Convolutions like TensorFlow, for a dynamic image size""" + + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + dilation=1, + groups=1, + bias_attr=None, + ): + super().__init__( + in_channels, + out_channels, + kernel_size, + stride, + 0, + dilation, + groups, + bias_attr=bias_attr, + ) + self.stride = ( + self._stride if len(self._stride) == 2 else [self._stride[0]] * 2 + ) def forward(self, x): ih, iw = x.shape[-2:] @@ -112,65 +137,84 @@ def forward(self, x): sh, sw = self.stride oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) pad_h = max( - (oh - 1) * self.stride[0] + (kh - 1) * self._dilation[0] + 1 - ih, - 0) + (oh - 1) * self.stride[0] + (kh - 1) * self._dilation[0] + 1 - ih, 0 + ) pad_w = max( - (ow - 1) * self.stride[1] + (kw - 1) * self._dilation[1] + 1 - iw, - 0) + (ow - 1) * self.stride[1] + (kw - 1) * self._dilation[1] + 1 - iw, 0 + ) if pad_h > 0 or pad_w > 0: - x = F.pad(x, [ - pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2 - ]) - return F.conv2d(x, self.weight, self.bias, self.stride, self._padding, - self._dilation, self._groups) + x = F.pad( + x, + [ + pad_w // 2, + pad_w - pad_w // 2, + pad_h // 2, + pad_h - pad_h // 2, + ], + ) + return F.conv2d( + x, + self.weight, + self.bias, + self.stride, + self._padding, + self._dilation, + self._groups, + ) class Conv2dStaticSamePadding(nn.Conv2D): - """ 2D Convolutions like TensorFlow, for a fixed image size""" - - def __init__(self, - in_channels, - out_channels, - kernel_size, - image_size=None, - **kwargs): + """2D Convolutions like TensorFlow, for a fixed image size""" + + def __init__( + self, in_channels, out_channels, kernel_size, image_size=None, **kwargs + ): if 'stride' in kwargs and isinstance(kwargs['stride'], list): kwargs['stride'] = kwargs['stride'][0] super().__init__(in_channels, out_channels, kernel_size, **kwargs) - self.stride = self._stride if len( - self._stride) == 2 else [self._stride[0]] * 2 + self.stride = ( + self._stride if len(self._stride) == 2 else [self._stride[0]] * 2 + ) # Calculate padding based on image size and save it assert image_size is not None - ih, iw = image_size if type(image_size) == list else [ - image_size, image_size - ] + ih, iw = ( + image_size if type(image_size) == list else [image_size, image_size] + ) kh, kw = self.weight.shape[-2:] sh, sw = self.stride oh, ow = math.ceil(ih / sh), math.ceil(iw / sw) pad_h = max( - (oh - 1) * self.stride[0] + (kh - 1) * self._dilation[0] + 1 - ih, - 0) + (oh - 1) * self.stride[0] + (kh - 1) * self._dilation[0] + 1 - ih, 0 + ) pad_w = max( - (ow - 1) * self.stride[1] + (kw - 1) * self._dilation[1] + 1 - iw, - 0) + (ow - 1) * self.stride[1] + (kw - 1) * self._dilation[1] + 1 - iw, 0 + ) if pad_h > 0 or pad_w > 0: - self.static_padding = nn.Pad2D([ - pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2 - ]) + self.static_padding = nn.Pad2D( + [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2] + ) else: self.static_padding = Identity() def forward(self, x): x = self.static_padding(x) - x = F.conv2d(x, self.weight, self.bias, self.stride, self._padding, - self._dilation, self._groups) + x = F.conv2d( + x, + self.weight, + self.bias, + self.stride, + self._padding, + self._dilation, + self._groups, + ) return x class Identity(nn.Layer): - - def __init__(self, ): + def __init__( + self, + ): super().__init__() def forward(self, x): @@ -178,7 +222,7 @@ def forward(self, x): def efficientnet_params(model_name): - """ Map EfficientNet model name to parameter coefficients. """ + """Map EfficientNet model name to parameter coefficients.""" params_dict = { # Coefficients: width,depth,resolution,dropout 'efficientnet-b0': (1.0, 1.0, 224, 0.2), @@ -196,11 +240,11 @@ def efficientnet_params(model_name): class BlockDecoder(object): - """ Block Decoder for readability, straight from the official TensorFlow repository """ + """Block Decoder for readability, straight from the official TensorFlow repository""" @staticmethod def _decode_block_string(block_string): - """ Gets a block through a string notation of arguments. """ + """Gets a block through a string notation of arguments.""" assert isinstance(block_string, str) ops = block_string.split('_') @@ -212,8 +256,9 @@ def _decode_block_string(block_string): options[key] = value # Check stride - assert (('s' in options and len(options['s']) == 1) or - (len(options['s']) == 2 and options['s'][0] == options['s'][1])) + assert ('s' in options and len(options['s']) == 1) or ( + len(options['s']) == 2 and options['s'][0] == options['s'][1] + ) return BlockArgs( kernel_size=int(options['k']), @@ -223,7 +268,8 @@ def _decode_block_string(block_string): expand_ratio=int(options['e']), id_skip=('noskip' not in block_string), se_ratio=float(options['se']) if 'se' in options else None, - stride=[int(options['s'][0])]) + stride=[int(options['s'][0])], + ) @staticmethod def _encode_block_string(block): @@ -234,7 +280,7 @@ def _encode_block_string(block): 's%d%d' % (block.strides[0], block.strides[1]), 'e%s' % block.expand_ratio, 'i%d' % block.input_filters, - 'o%d' % block.output_filters + 'o%d' % block.output_filters, ] if 0 < block.se_ratio <= 1: args.append('se%s' % block.se_ratio) @@ -270,13 +316,15 @@ def encode(blocks_args): return block_strings -def efficientnet(width_coefficient=None, - depth_coefficient=None, - dropout_rate=0.2, - drop_connect_rate=0.2, - image_size=None, - num_classes=1000): - """ Get block arguments according to parameter and coefficients. """ +def efficientnet( + width_coefficient=None, + depth_coefficient=None, + dropout_rate=0.2, + drop_connect_rate=0.2, + image_size=None, + num_classes=1000, +): + """Get block arguments according to parameter and coefficients.""" blocks_args = [ 'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25', @@ -305,67 +353,51 @@ def efficientnet(width_coefficient=None, def get_model_params(model_name, override_params): - """ Get the block args and global params for a given model """ + """Get the block args and global params for a given model""" if model_name.startswith('efficientnet'): w, d, s, p = efficientnet_params(model_name) - blocks_args, global_params = efficientnet(width_coefficient=w, - depth_coefficient=d, - dropout_rate=p, - image_size=s) + blocks_args, global_params = efficientnet( + width_coefficient=w, + depth_coefficient=d, + dropout_rate=p, + image_size=s, + ) else: - raise NotImplementedError('model name is not pre-defined: %s' % - model_name) + raise NotImplementedError( + 'model name is not pre-defined: %s' % model_name + ) if override_params: global_params = global_params._replace(**override_params) return blocks_args, global_params url_map = { - 'efficientnet-b0': - '/home/aistudio/data/weights/efficientnet-b0-355c32eb.pdparams', - 'efficientnet-b1': - '/home/aistudio/data/weights/efficientnet-b1-f1951068.pdparams', - 'efficientnet-b2': - '/home/aistudio/data/weights/efficientnet-b2-8bb594d6.pdparams', - 'efficientnet-b3': - '/home/aistudio/data/weights/efficientnet-b3-5fb5a3c3.pdparams', - 'efficientnet-b4': - '/home/aistudio/data/weights/efficientnet-b4-6ed6700e.pdparams', - 'efficientnet-b5': - '/home/aistudio/data/weights/efficientnet-b5-b6417697.pdparams', - 'efficientnet-b6': - '/home/aistudio/data/weights/efficientnet-b6-c76e70fd.pdparams', - 'efficientnet-b7': - '/home/aistudio/data/weights/efficientnet-b7-dcc49843.pdparams', + 'efficientnet-b0': '/home/aistudio/data/weights/efficientnet-b0-355c32eb.pdparams', + 'efficientnet-b1': '/home/aistudio/data/weights/efficientnet-b1-f1951068.pdparams', + 'efficientnet-b2': '/home/aistudio/data/weights/efficientnet-b2-8bb594d6.pdparams', + 'efficientnet-b3': '/home/aistudio/data/weights/efficientnet-b3-5fb5a3c3.pdparams', + 'efficientnet-b4': '/home/aistudio/data/weights/efficientnet-b4-6ed6700e.pdparams', + 'efficientnet-b5': '/home/aistudio/data/weights/efficientnet-b5-b6417697.pdparams', + 'efficientnet-b6': '/home/aistudio/data/weights/efficientnet-b6-c76e70fd.pdparams', + 'efficientnet-b7': '/home/aistudio/data/weights/efficientnet-b7-dcc49843.pdparams', } url_map_advprop = { - 'efficientnet-b0': - '/home/aistudio/data/weights/adv-efficientnet-b0-b64d5a18.pdparams', - 'efficientnet-b1': - '/home/aistudio/data/weights/adv-efficientnet-b1-0f3ce85a.pdparams', - 'efficientnet-b2': - '/home/aistudio/data/weights/adv-efficientnet-b2-6e9d97e5.pdparams', - 'efficientnet-b3': - '/home/aistudio/data/weights/adv-efficientnet-b3-cdd7c0f4.pdparams', - 'efficientnet-b4': - '/home/aistudio/data/weights/adv-efficientnet-b4-44fb3a87.pdparams', - 'efficientnet-b5': - '/home/aistudio/data/weights/adv-efficientnet-b5-86493f6b.pdparams', - 'efficientnet-b6': - '/home/aistudio/data/weights/adv-efficientnet-b6-ac80338e.pdparams', - 'efficientnet-b7': - '/home/aistudio/data/weights/adv-efficientnet-b7-4652b6dd.pdparams', - 'efficientnet-b8': - '/home/aistudio/data/weights/adv-efficientnet-b8-22a8fe65.pdparams', + 'efficientnet-b0': '/home/aistudio/data/weights/adv-efficientnet-b0-b64d5a18.pdparams', + 'efficientnet-b1': '/home/aistudio/data/weights/adv-efficientnet-b1-0f3ce85a.pdparams', + 'efficientnet-b2': '/home/aistudio/data/weights/adv-efficientnet-b2-6e9d97e5.pdparams', + 'efficientnet-b3': '/home/aistudio/data/weights/adv-efficientnet-b3-cdd7c0f4.pdparams', + 'efficientnet-b4': '/home/aistudio/data/weights/adv-efficientnet-b4-44fb3a87.pdparams', + 'efficientnet-b5': '/home/aistudio/data/weights/adv-efficientnet-b5-86493f6b.pdparams', + 'efficientnet-b6': '/home/aistudio/data/weights/adv-efficientnet-b6-ac80338e.pdparams', + 'efficientnet-b7': '/home/aistudio/data/weights/adv-efficientnet-b7-4652b6dd.pdparams', + 'efficientnet-b8': '/home/aistudio/data/weights/adv-efficientnet-b8-22a8fe65.pdparams', } -def load_pretrained_weights(model, - model_name, - weights_path=None, - load_fc=True, - advprop=False): +def load_pretrained_weights( + model, model_name, weights_path=None, load_fc=True, advprop=False +): """Loads pretrained weights from weights path or download using url. Args: model (Module): The whole model of efficientnet. diff --git a/paddle/infrt/tests/models/linear.py b/paddle/infrt/tests/models/linear.py index 1a6c6f095c704..71dc4e0d1c901 100644 --- a/paddle/infrt/tests/models/linear.py +++ b/paddle/infrt/tests/models/linear.py @@ -28,13 +28,12 @@ # define a random dataset class RandomDataset(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64') + label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype('int64') return image, label def __len__(self): @@ -42,7 +41,6 @@ def __len__(self): class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) @@ -71,11 +69,9 @@ def train(layer, loader, loss_fn, opt): # create data loader dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) -loader = paddle.io.DataLoader(dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) +loader = paddle.io.DataLoader( + dataset, batch_size=BATCH_SIZE, shuffle=True, drop_last=True, num_workers=2 +) # train train(layer, loader, loss_fn, adam) diff --git a/paddle/infrt/tests/models/resnet50_model.py b/paddle/infrt/tests/models/resnet50_model.py index af45de0c8a609..ff84c3988abcb 100644 --- a/paddle/infrt/tests/models/resnet50_model.py +++ b/paddle/infrt/tests/models/resnet50_model.py @@ -19,6 +19,7 @@ import sys model = resnet50(True) -net = to_static(model, - input_spec=[InputSpec(shape=[None, 3, 256, 256], name='x')]) +net = to_static( + model, input_spec=[InputSpec(shape=[None, 3, 256, 256], name='x')] +) paddle.jit.save(net, sys.argv[1]) diff --git a/paddle/phi/api/yaml/generator/api_base.py b/paddle/phi/api/yaml/generator/api_base.py index 534158eb31cbe..e67023d2faf71 100644 --- a/paddle/phi/api/yaml/generator/api_base.py +++ b/paddle/phi/api/yaml/generator/api_base.py @@ -20,7 +20,6 @@ class BaseAPI(object): - def __init__(self, api_item_yaml): self.api = self.get_api_name(api_item_yaml) @@ -34,8 +33,12 @@ def __init__(self, api_item_yaml): # names : [], list of output names # types : [], list of output types # out_size_expr : [], expression for getting size of vector - self.inputs, self.attrs, self.outputs, self.optional_vars = self.parse_args( - self.api, api_item_yaml) + ( + self.inputs, + self.attrs, + self.outputs, + self.optional_vars, + ) = self.parse_args(self.api, api_item_yaml) self.is_base_api = True if 'invoke' in api_item_yaml: @@ -44,7 +47,8 @@ def __init__(self, api_item_yaml): else: if 'infer_meta' in api_item_yaml: self.infer_meta = self.parse_infer_meta( - api_item_yaml['infer_meta']) + api_item_yaml['infer_meta'] + ) self.kernel = self.parse_kernel(api_item_yaml['kernel']) self.data_transform = self.parse_data_transform(api_item_yaml) self.inplace_map, self.view_map = {}, {} @@ -52,18 +56,16 @@ def __init__(self, api_item_yaml): self.gene_input_func = { "const Tensor&": { "dense": self.gene_dense_input, - "selected_rows": self.gene_selected_rows_input + "selected_rows": self.gene_selected_rows_input, }, "const paddle::optional&": { "dense": self.gene_dense_input, - "selected_rows": self.gene_selected_rows_input - }, - "const std::vector&": { - "dense": self.gene_vec_dense_input + "selected_rows": self.gene_selected_rows_input, }, + "const std::vector&": {"dense": self.gene_vec_dense_input}, "const paddle::optional>&": { "dense": self.gene_optional_vec_dense_input - } + }, } def get_api_name(self, api_item_yaml): @@ -75,21 +77,19 @@ def get_api_func_name(self): def get_input_tensor_args(self, inplace_flag=False): input_args = [] inplace_type_map = { - "const Tensor&": - "Tensor&", - "const paddle::optional&": - "paddle::optional&", - "const std::vector&": - "std::vector&", - "const paddle::optional>&": - "paddle::optional>&" + "const Tensor&": "Tensor&", + "const paddle::optional&": "paddle::optional&", + "const std::vector&": "std::vector&", + "const paddle::optional>&": "paddle::optional>&", } for name in self.inputs['names']: name = name.split('@')[0] if inplace_flag and name in self.inplace_map.values(): input_args.append( - inplace_type_map[self.inputs['input_info'][name]] + ' ' + - name) + inplace_type_map[self.inputs['input_info'][name]] + + ' ' + + name + ) else: input_args.append(self.inputs['input_info'][name] + ' ' + name) return input_args @@ -100,8 +100,9 @@ def get_declare_args(self, inplace_flag=False): default_value = '' if self.attrs['attr_info'][name][1] is not None: default_value = ' = ' + self.attrs['attr_info'][name][1] - declare_args.append(self.attrs['attr_info'][name][0] + ' ' + name + - default_value) + declare_args.append( + self.attrs['attr_info'][name][0] + ' ' + name + default_value + ) return ", ".join(declare_args) @@ -118,28 +119,35 @@ def parse_args(self, api_name, api_item_yaml): optional_vars = [ item.strip() for item in api_item_yaml['optional'].split(',') ] - inputs, attrs = self.parse_input_and_attr(api_name, - api_item_yaml['args'], - optional_vars) + inputs, attrs = self.parse_input_and_attr( + api_name, api_item_yaml['args'], optional_vars + ) output_type_list, output_names, out_size_expr = self.parse_output( - api_name, api_item_yaml['output']) - return inputs, attrs, { - 'names': output_names, - 'types': output_type_list, - 'out_size_expr': out_size_expr - }, optional_vars + api_name, api_item_yaml['output'] + ) + return ( + inputs, + attrs, + { + 'names': output_names, + 'types': output_type_list, + 'out_size_expr': out_size_expr, + }, + optional_vars, + ) def parse_input_and_attr(self, api_name, args_config, optional_vars=[]): inputs = {'names': [], 'input_info': {}} attrs = {'names': [], 'attr_info': {}} args_str = args_config.strip() - assert args_str.startswith('(') and args_str.endswith(')'), \ - f"Args declaration should start with '(' and end with ')', please check the args of {api_name} in yaml." + assert args_str.startswith('(') and args_str.endswith( + ')' + ), f"Args declaration should start with '(' and end with ')', please check the args of {api_name} in yaml." args_str = args_str[1:-1] args_list = args_str.split(',') input_types_map = { 'Tensor': 'const Tensor&', - 'Tensor[]': 'const std::vector&' + 'Tensor[]': 'const std::vector&', } attr_types_map = { 'IntArray': 'const IntArray&', @@ -177,7 +185,7 @@ def parse_input_and_attr(self, api_name, args_config, optional_vars=[]): 'bool': 'paddle::optional', 'Place': 'paddle::optional', 'DataLayout': 'paddle::optional', - 'DataType': 'paddle::optional' + 'DataType': 'paddle::optional', } for item in args_list: @@ -188,10 +196,12 @@ def parse_input_and_attr(self, api_name, args_config, optional_vars=[]): for in_type_symbol, in_type in input_types_map.items(): if type_and_name[0] == in_type_symbol: input_name = type_and_name[1].strip() - assert len(input_name) > 0, \ - f"The input tensor name should not be empty. Please check the args of {api_name} in yaml." - assert len(attrs['names']) == 0, \ - f"The input Tensor should appear before attributes. please check the position of {api_name}:input({input_name}) in yaml" + assert ( + len(input_name) > 0 + ), f"The input tensor name should not be empty. Please check the args of {api_name} in yaml." + assert ( + len(attrs['names']) == 0 + ), f"The input Tensor should appear before attributes. please check the position of {api_name}:input({input_name}) in yaml" if input_name in optional_vars: in_type = optional_types_trans[in_type_symbol] @@ -206,9 +216,10 @@ def parse_input_and_attr(self, api_name, args_config, optional_vars=[]): # match the attribute for attr_type_symbol, attr_type in attr_types_map.items(): if type_and_name[0] == attr_type_symbol: - attr_name = item[len(attr_type_symbol):].strip() - assert len(attr_name) > 0, \ - f"The attribute name should not be empty. Please check the args of {api_name} in yaml." + attr_name = item[len(attr_type_symbol) :].strip() + assert ( + len(attr_name) > 0 + ), f"The attribute name should not be empty. Please check the args of {api_name} in yaml." default_value = None if '=' in attr_name: attr_infos = attr_name.split('=') @@ -218,7 +229,9 @@ def parse_input_and_attr(self, api_name, args_config, optional_vars=[]): if attr_name in optional_vars: attr_type = optional_types_trans[attr_type_symbol] - default_value_str = "" if default_value is None else '=' + default_value + default_value_str = ( + "" if default_value is None else '=' + default_value + ) attrs['names'].append(attr_name) attrs['attr_info'][attr_name] = (attr_type, default_value) break @@ -226,25 +239,34 @@ def parse_input_and_attr(self, api_name, args_config, optional_vars=[]): return inputs, attrs def parse_output(self, api_name, output_config): - def parse_output_item(output_item): output_type_map = { 'Tensor': 'Tensor', - 'Tensor[]': 'std::vector' + 'Tensor[]': 'std::vector', } result = re.search( r"(?P[a-zA-Z0-9_[\]]+)\s*(?P\([a-zA-Z0-9_@]+\))?\s*(?P\{[^\}]+\})?", - output_item) - assert result is not None, f"{api_name} : the output config parse error." + output_item, + ) + assert ( + result is not None + ), f"{api_name} : the output config parse error." out_type = result.group('out_type') - assert out_type in output_type_map, \ - f"{api_name} : Output type error: the output type only support Tensor and Tensor[], \ + assert ( + out_type in output_type_map + ), f"{api_name} : Output type error: the output type only support Tensor and Tensor[], \ but now is {out_type}." - out_name = 'out' if result.group('name') is None else result.group( - 'name')[1:-1] - out_size_expr = None if result.group( - 'expr') is None else result.group('expr')[1:-1] + out_name = ( + 'out' + if result.group('name') is None + else result.group('name')[1:-1] + ) + out_size_expr = ( + None + if result.group('expr') is None + else result.group('expr')[1:-1] + ) return output_type_map[out_type], out_name, out_size_expr temp_list = output_config.split(',') @@ -286,7 +308,7 @@ def parse_kernel(self, kernel_config): 'layout': None, 'data_type': None, 'use_gpudnn': 'false', - 'dispatch': {} + 'dispatch': {}, } if 'backend' in kernel_config and len(kernel_config['backend']) > 0: kernel['backend'] = kernel_config['backend'] @@ -301,7 +323,8 @@ def parse_kernel(self, kernel_config): if isinstance(kernel['use_gpudnn'], bool): kernel['use_gpudnn'] = str(kernel['use_gpudnn']).lower() kernel_funcs = re.compile(r'([a-zA-Z0-9_]+)\s*({[^}]+})?').findall( - kernel_config['func']) + kernel_config['func'] + ) def parse_kernel_in_out_type(in_out_str): if len(in_out_str) == 0: @@ -313,11 +336,17 @@ def parse_kernel_in_out_type(in_out_str): # check the tensor type for item in inputs: assert item in [ - 'dense', 'selected_rows', 'sparse_coo', 'sparse_csr' + 'dense', + 'selected_rows', + 'sparse_coo', + 'sparse_csr', ], f"{self.api} : Invalid input tensor type ('{item}'), here we only support 'dense', 'selected_rows', 'sparse_coo' and 'sparse_csr'." for item in outputs: assert item in [ - 'dense', 'selected_rows', 'sparse_coo', 'sparse_csr' + 'dense', + 'selected_rows', + 'sparse_coo', + 'sparse_csr', ], f"{self.api} : Invalid output tensor type ('{item}'), here we only support 'dense', 'selected_rows', 'sparse_coo' and 'sparse_csr'." return (inputs, outputs) @@ -325,7 +354,8 @@ def parse_kernel_in_out_type(in_out_str): for func_item in kernel_funcs: kernel['func'].append(func_item[0]) kernel['dispatch'][func_item[0]] = parse_kernel_in_out_type( - func_item[1]) + func_item[1] + ) return kernel @@ -334,10 +364,12 @@ def parse_data_transform(self, api_item_yaml): if 'data_transform' in api_item_yaml: if 'skip_transform' in api_item_yaml['data_transform']: data_transform['skip_transform'] = api_item_yaml[ - 'data_transform']['skip_transform'] + 'data_transform' + ]['skip_transform'] if 'support_trans_dtype' in api_item_yaml['data_transform']: data_transform['support_trans_dtype'] = api_item_yaml[ - 'data_transform']['support_trans_dtype'] + 'data_transform' + ]['support_trans_dtype'] return data_transform @@ -356,9 +388,12 @@ def gene_api_declaration(self): if self.is_base_api and len(self.inplace_map) > 0: if api_func_name[-1] != '_': api_func_name += '_' - api_declaration = api_declaration + f""" + api_declaration = ( + api_declaration + + f""" PADDLE_API {self.get_return_type(inplace_flag=True)} {api_func_name}({self.get_declare_args(inplace_flag=True)}); """ + ) return api_declaration @@ -368,11 +403,13 @@ def gene_kernel_backend_select(self): if self.kernel['backend'] is not None: if '>' in self.kernel['backend']: vars_list = self.kernel['backend'].split('>') - assert len( - vars_list - ) == 2, f"{self.api} api: The number of params to set backend with '>' only allows 2, but received {len(vars_list)}." - assert (vars_list[0].strip() in self.attrs['names']) and (self.attrs['attr_info'][vars_list[0].strip()][0] == 'const Place&'), \ - f"{self.api} api: When use '>' to set kernel backend, the first param should be a attribute with Place type." + assert ( + len(vars_list) == 2 + ), f"{self.api} api: The number of params to set backend with '>' only allows 2, but received {len(vars_list)}." + assert (vars_list[0].strip() in self.attrs['names']) and ( + self.attrs['attr_info'][vars_list[0].strip()][0] + == 'const Place&' + ), f"{self.api} api: When use '>' to set kernel backend, the first param should be a attribute with Place type." backend_select_code = f""" kernel_backend = ParseBackendWithInputOrder({vars_list[0].strip()}, {vars_list[1].strip()}); """ @@ -404,16 +441,19 @@ def gene_kernel_select(self) -> str: attr_data_type_count = 0 for attr_name in attrs['names']: if attrs['attr_info'][attr_name][0] == 'const Place&': - assert kernel['backend'] is not None, \ - f"{api} api: When there is a parameter with 'Place' type in attributes, you must set backend of kernel manually." + assert ( + kernel['backend'] is not None + ), f"{api} api: When there is a parameter with 'Place' type in attributes, you must set backend of kernel manually." attr_backend_count = attr_backend_count + 1 if attrs['attr_info'][attr_name][0] == 'DataLayout': - assert kernel['layout'] is not None, \ - f"{api} api: When there is a parameter with 'DataLayout' type in attributes, you must set layout of kernel manually." + assert ( + kernel['layout'] is not None + ), f"{api} api: When there is a parameter with 'DataLayout' type in attributes, you must set layout of kernel manually." attr_layout_count = attr_layout_count + 1 if attrs['attr_info'][attr_name][0] == 'DataType': - assert kernel['data_type'] is not None, \ - f"{api} api: When there is a parameter with 'DataType' type in attributes, you must set data_type of kernel manually." + assert ( + kernel['data_type'] is not None + ), f"{api} api: When there is a parameter with 'DataType' type in attributes, you must set data_type of kernel manually." attr_data_type_count = attr_data_type_count + 1 # preprocess kernel configures @@ -422,48 +462,67 @@ def gene_kernel_select(self) -> str: if kernel['layout'] is not None: if '>' in kernel['layout']: vars_list = kernel['layout'].split('>') - assert len( - vars_list - ) == 2, f"{api} api: The number of params to set layout with '>' only allows 2, but received {len(vars_list)}." - assert vars_list[0].strip() in attrs['names'] and attrs['attr_info'][vars_list[0].strip()][0] == 'DataLayout', \ - f"{api} api: When use '>' to set kernel layout, the first param should be a attribute with DataLayout type." - kernel_select_code = kernel_select_code + f""" + assert ( + len(vars_list) == 2 + ), f"{api} api: The number of params to set layout with '>' only allows 2, but received {len(vars_list)}." + assert ( + vars_list[0].strip() in attrs['names'] + and attrs['attr_info'][vars_list[0].strip()][0] + == 'DataLayout' + ), f"{api} api: When use '>' to set kernel layout, the first param should be a attribute with DataLayout type." + kernel_select_code = ( + kernel_select_code + + f""" kernel_layout = ParseLayoutWithInputOrder({vars_list[0].strip()}, {vars_list[1].strip()}); """ + ) else: vars_list = kernel['layout'].split(',') - assert len( - vars_list - ) == 1, f"{api} api: The number of params to set layout must be 1, but received {len(vars_list)}." - kernel_select_code = kernel_select_code + f""" + assert ( + len(vars_list) == 1 + ), f"{api} api: The number of params to set layout must be 1, but received {len(vars_list)}." + kernel_select_code = ( + kernel_select_code + + f""" kernel_layout = ParseLayout({vars_list[0].strip()}); """ + ) if kernel['data_type'] is not None: if '>' in kernel['data_type']: vars_list = kernel['data_type'].split('>') - assert len( - vars_list - ) == 2, f"{api} api: The number of params to set data_type with '>' only allows 2, but received {len(vars_list)}." - assert vars_list[0].strip() in attrs['names'] and attrs['attr_info'][vars_list[0].strip()][0] == 'DataType', \ - f"{api} api: When use '>' to set kernel data_type, the first param should be a attribute with DataType type." - kernel_select_code = kernel_select_code + f""" + assert ( + len(vars_list) == 2 + ), f"{api} api: The number of params to set data_type with '>' only allows 2, but received {len(vars_list)}." + assert ( + vars_list[0].strip() in attrs['names'] + and attrs['attr_info'][vars_list[0].strip()][0] + == 'DataType' + ), f"{api} api: When use '>' to set kernel data_type, the first param should be a attribute with DataType type." + kernel_select_code = ( + kernel_select_code + + f""" kernel_data_type = ParseDataTypeWithInputOrder({vars_list[0].strip()}, {vars_list[1].strip()}); """ + ) else: vars_list = kernel['data_type'].split(',') - assert len( - vars_list - ) == 1, f"{api} api: The number of params to set data_type only allows 1, but received {len(vars_list)}." - kernel_select_code = kernel_select_code + f""" + assert ( + len(vars_list) == 1 + ), f"{api} api: The number of params to set data_type only allows 1, but received {len(vars_list)}." + kernel_select_code = ( + kernel_select_code + + f""" kernel_data_type = ParseDataType({vars_list[0].strip()}); """ + ) if len(input_names) == 0: - assert attr_backend_count > 0 and attr_data_type_count > 0, \ - f"{api} api: When there is no input tensor, the args must have 'Place' and 'DataType'." + assert ( + attr_backend_count > 0 and attr_data_type_count > 0 + ), f"{api} api: When there is no input tensor, the args must have 'Place' and 'DataType'." kernel_select_args = "" for input_name in input_names: @@ -475,7 +534,9 @@ def gene_kernel_select(self) -> str: kernel_select_code = kernel_key_item_init + kernel_select_code if len(input_names) > 0: - kernel_select_code = kernel_select_code + f""" + kernel_select_code = ( + kernel_select_code + + f""" if (kernel_backend == Backend::UNDEFINED || kernel_layout == DataLayout::UNDEFINED || kernel_data_type == DataType::UNDEFINED ) {{ @@ -491,6 +552,7 @@ def gene_kernel_select(self) -> str: kernel_data_type = kernel_key.dtype(); }} }}""" + ) return kernel_select_code @@ -499,37 +561,62 @@ def gene_infer_meta(self, kernel_output_names, code_indent) -> str: attr_names = self.attrs['names'] infer_meta = self.infer_meta - infer_meta_params = infer_meta['param'] if infer_meta[ - 'param'] is not None else input_names + attr_names + infer_meta_params = ( + infer_meta['param'] + if infer_meta['param'] is not None + else input_names + attr_names + ) # generate meta tensors meta_tensor_code = "" param_code = "" for param in infer_meta_params: if param in input_names: if self.inputs['input_info'][param] == "const Tensor&": - param_code = param_code + "MakeMetaTensor(*" + PREFIX_TENSOR_NAME + param + "), " - elif self.inputs['input_info'][ - param] == "const std::vector&": - meta_tensor_code = meta_tensor_code + f""" + param_code = ( + param_code + + "MakeMetaTensor(*" + + PREFIX_TENSOR_NAME + + param + + "), " + ) + elif ( + self.inputs['input_info'][param] + == "const std::vector&" + ): + meta_tensor_code = ( + meta_tensor_code + + f""" {code_indent} auto {param}_meta_vec = MakeMetaTensor({PREFIX_TENSOR_NAME}{param}); {code_indent} std::vector {param}_metas({param}_meta_vec.size()); {code_indent} for (size_t i = 0; i < {param}_meta_vec.size(); ++i) {{ {code_indent} {param}_metas[i] = &{param}_meta_vec[i]; {code_indent} }} """ + ) param_code = param_code + param + "_metas, " - elif self.inputs['input_info'][ - param] == "const paddle::optional>&": - meta_tensor_code = meta_tensor_code + f""" + elif ( + self.inputs['input_info'][param] + == "const paddle::optional>&" + ): + meta_tensor_code = ( + meta_tensor_code + + f""" {code_indent} auto {param}_meta_vec = MakeMetaTensor({PREFIX_TENSOR_NAME}{param}); {code_indent} paddle::optional> {param}_metas({param}_meta_vec.size()); {code_indent} for (size_t i = 0; i < {param}_meta_vec.size(); ++i) {{ {code_indent} {param}_metas->at(i) = &{param}_meta_vec[i]; {code_indent} }} """ + ) param_code = param_code + param + "_metas, " elif param in self.optional_vars: - param_code = param_code + "MakeMetaTensor(" + PREFIX_TENSOR_NAME + param + "), " + param_code = ( + param_code + + "MakeMetaTensor(" + + PREFIX_TENSOR_NAME + + param + + "), " + ) else: raise ValueError( f"{self.api} : Param of infer_meta error : {self.inputs['input_info'][param]} type is not supported." @@ -545,22 +632,37 @@ def gene_infer_meta(self, kernel_output_names, code_indent) -> str: for i, out_name in enumerate(kernel_output_names): if self.outputs['types'][i] == 'std::vector': - meta_tensor_code = meta_tensor_code + f""" + meta_tensor_code = ( + meta_tensor_code + + f""" {code_indent} auto {out_name}_{PREFIX_META_TENSOR_NAME}vec = MakeMetaTensor({out_name}); {code_indent} std::vector {out_name}_metas({out_name}_{PREFIX_META_TENSOR_NAME}vec.size()); {code_indent} for (size_t i = 0; i < {out_name}_{PREFIX_META_TENSOR_NAME}vec.size(); ++i) {{ {code_indent} {out_name}_metas[i] = {out_name}[i] ? &{out_name}_{PREFIX_META_TENSOR_NAME}vec[i] : nullptr; {code_indent} }}""" + ) param_code = param_code + out_name + '_metas, ' else: - meta_tensor_code = meta_tensor_code + code_indent + " phi::MetaTensor " + out_name.replace( - 'kernel_', - PREFIX_META_TENSOR_NAME) + "(" + out_name + ");\n" + meta_tensor_code = ( + meta_tensor_code + + code_indent + + " phi::MetaTensor " + + out_name.replace('kernel_', PREFIX_META_TENSOR_NAME) + + "(" + + out_name + + ");\n" + ) if len(kernel_output_names) == 1: - param_code = param_code + f"&{out_name.replace('kernel_', PREFIX_META_TENSOR_NAME)}, " + param_code = ( + param_code + + f"&{out_name.replace('kernel_', PREFIX_META_TENSOR_NAME)}, " + ) else: - param_code = param_code + f"{out_name} ? &{out_name.replace('kernel_', PREFIX_META_TENSOR_NAME)} : nullptr, " + param_code = ( + param_code + + f"{out_name} ? &{out_name.replace('kernel_', PREFIX_META_TENSOR_NAME)} : nullptr, " + ) param_code = param_code[:-2] return f"""{meta_tensor_code} @@ -575,10 +677,9 @@ def gene_trans_flag(self, input_name): trans_flag = "{false, true}" return trans_flag - def gene_dense_input(self, - input_name, - input_name_tensor_map, - code_indent=''): + def gene_dense_input( + self, input_name, input_name_tensor_map, code_indent='' + ): input_tensor_code = "" trans_flag = self.gene_trans_flag(input_name) input_names = self.inputs['names'] @@ -588,15 +689,18 @@ def gene_dense_input(self, kernel_param = input_names + attr_names input_name_tensor_map[input_name].append( - (f"{PREFIX_TENSOR_NAME}{input_name}", False)) - input_tensor_code = input_tensor_code + f""" + (f"{PREFIX_TENSOR_NAME}{input_name}", False) + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name} = PrepareData({input_name}, kernel.InputAt({kernel_param.index(input_name)}), {trans_flag});""" + ) return input_tensor_code - def gene_selected_rows_input(self, - input_name, - input_name_tensor_map, - code_indent=''): + def gene_selected_rows_input( + self, input_name, input_name_tensor_map, code_indent='' + ): input_tensor_code = "" trans_flag = self.gene_trans_flag(input_name) input_names = self.inputs['names'] @@ -606,16 +710,19 @@ def gene_selected_rows_input(self, kernel_param = input_names + attr_names input_name_tensor_map[input_name].append( - (f"{PREFIX_TENSOR_NAME}{input_name}", False)) - input_tensor_code = input_tensor_code + f""" + (f"{PREFIX_TENSOR_NAME}{input_name}", False) + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name} = TensorToSelectedRows({input_name}); """ + ) return input_tensor_code - def gene_optional_vec_dense_input(self, - input_name, - input_name_tensor_map, - code_indent=''): + def gene_optional_vec_dense_input( + self, input_name, input_name_tensor_map, code_indent='' + ): input_tensor_code = "" trans_flag = self.gene_trans_flag(input_name) input_names = self.inputs['names'] @@ -625,13 +732,20 @@ def gene_optional_vec_dense_input(self, kernel_param = input_names + attr_names if input_name in self.inplace_map.values(): input_name_tensor_map[input_name].append( - (f"{PREFIX_TENSOR_NAME}{input_name}", True)) - input_tensor_code = input_tensor_code + f""" + (f"{PREFIX_TENSOR_NAME}{input_name}", True) + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} paddle::optional> {PREFIX_TENSOR_NAME}{input_name} = TensorToConstDenseTensorPtr({input_name});""" + ) else: input_name_tensor_map[input_name].append( - (f"{PREFIX_TENSOR_NAME}{input_name}_vec", True)) - input_tensor_code = input_tensor_code + f""" + (f"{PREFIX_TENSOR_NAME}{input_name}_vec", True) + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name}_vec = PrepareData({input_name}, kernel.InputAt({kernel_param.index(input_name)}), {trans_flag}); {code_indent} paddle::optional> {PREFIX_TENSOR_NAME}{input_name}; {code_indent} if ({PREFIX_TENSOR_NAME}{input_name}_vec){{ @@ -640,12 +754,12 @@ def gene_optional_vec_dense_input(self, {code_indent} {PREFIX_TENSOR_NAME}{input_name}->at(i) = &{PREFIX_TENSOR_NAME}{input_name}_vec->at(i); {code_indent} }} {code_indent} }}""" + ) return input_tensor_code - def gene_vec_dense_input(self, - input_name, - input_name_tensor_map, - code_indent=''): + def gene_vec_dense_input( + self, input_name, input_name_tensor_map, code_indent='' + ): input_tensor_code = "" trans_flag = self.gene_trans_flag(input_name) input_names = self.inputs['names'] @@ -656,18 +770,26 @@ def gene_vec_dense_input(self, if input_name in self.inplace_map.values(): input_name_tensor_map[input_name].append( - (f"{PREFIX_TENSOR_NAME}{input_name}", True)) - input_tensor_code = input_tensor_code + f""" + (f"{PREFIX_TENSOR_NAME}{input_name}", True) + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} std::vector {PREFIX_TENSOR_NAME}{input_name} = TensorToConstDenseTensorPtr({input_name});""" + ) else: input_name_tensor_map[input_name].append( - (f"{PREFIX_TENSOR_NAME}{input_name}_vec", True)) - input_tensor_code = input_tensor_code + f""" + (f"{PREFIX_TENSOR_NAME}{input_name}_vec", True) + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name}_vec = PrepareData({input_name}, kernel.InputAt({kernel_param.index(input_name)}), {trans_flag}); {code_indent} std::vector {PREFIX_TENSOR_NAME}{input_name}({PREFIX_TENSOR_NAME}{input_name}_vec->size()); {code_indent} for (size_t i = 0; i < {PREFIX_TENSOR_NAME}{input_name}.size(); ++i) {{ {code_indent} {PREFIX_TENSOR_NAME}{input_name}[i] = &{PREFIX_TENSOR_NAME}{input_name}_vec->at(i); {code_indent} }}""" + ) return input_tensor_code def gene_input(self, kernel_tensor_type=None, code_indent=''): @@ -683,55 +805,62 @@ def gene_input(self, kernel_tensor_type=None, code_indent=''): if input_name in kernel_param: # input is dense tensor api_tensor_type = self.inputs['input_info'][input_name] - phi_tensor_type = 'dense' if kernel_tensor_type is None else kernel_tensor_type[ - 0][kernel_param.index(input_name)] + phi_tensor_type = ( + 'dense' + if kernel_tensor_type is None + else kernel_tensor_type[0][kernel_param.index(input_name)] + ) if api_tensor_type in self.gene_input_func.keys(): input_tensor_code += self.gene_input_func[api_tensor_type][ - phi_tensor_type](input_name, input_name_tensor_map, - code_indent) + phi_tensor_type + ](input_name, input_name_tensor_map, code_indent) else: # do nothing pass else: if input_name in self.infer_meta['param']: if input_name in self.optional_vars: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} paddle::optional {PREFIX_TENSOR_NAME}{input_name} = {input_name} ? paddle::optional(*{input_name}->impl()) : paddle::none;""" + ) else: - if self.inputs['input_info'][ - input_name] == "const std::vector&": - input_tensor_code = input_tensor_code + f""" + if ( + self.inputs['input_info'][input_name] + == "const std::vector&" + ): + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name}_uq_ptr = TensorToDenseTensor({input_name}); {code_indent} const auto& {PREFIX_TENSOR_NAME}{input_name} = *{PREFIX_TENSOR_NAME}{input_name}_uq_ptr;""" + ) else: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name} = {input_name}.impl();""" + ) return input_name_tensor_map, input_tensor_code def get_kernel_args(self, kernel_tensor_type=None, code_indent=''): dense_input_trans_map = { - 'const Tensor&': - 'const phi::DenseTensor&', - 'const std::vector&': - 'const std::vector&', - 'const paddle::optional': - 'paddle::optional', - 'const paddle::optional&': - 'const paddle::optional&', - 'const paddle::optional>&': - 'const paddle::optional>&' + 'const Tensor&': 'const phi::DenseTensor&', + 'const std::vector&': 'const std::vector&', + 'const paddle::optional': 'paddle::optional', + 'const paddle::optional&': 'const paddle::optional&', + 'const paddle::optional>&': 'const paddle::optional>&', } dense_out_trans_map = { 'Tensor': 'phi::DenseTensor*', - 'std::vector': 'std::vector&' + 'std::vector': 'std::vector&', } sr_input_trans_map = { - 'const Tensor&': - 'const phi::SelectedRows&', - 'const paddle::optional&': - 'const paddle::optional&' + 'const Tensor&': 'const phi::SelectedRows&', + 'const paddle::optional&': 'const paddle::optional&', } sr_out_trans_map = {'Tensor': 'phi::SelectedRows*'} input_names = self.inputs['names'] @@ -744,10 +873,14 @@ def get_kernel_args(self, kernel_tensor_type=None, code_indent=''): kernel_param = input_names + attr_names input_name_tensor_map, input_tensor_code = self.gene_input( - kernel_tensor_type, code_indent) + kernel_tensor_type, code_indent + ) - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} if(platform::RecordOpInfoSupplement::IsEnabled()){{""" + ) single_tensor_names = [] list_tensor_names = [] for input_name, input_tensors in input_name_tensor_map.items(): @@ -760,54 +893,96 @@ def get_kernel_args(self, kernel_tensor_type=None, code_indent=''): else: list_tensor_names.append(input_name) if not single_tensor_names: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} std::vector>> input_shapes;""" + ) else: for input_name in single_tensor_names: if input_name in self.optional_vars: input_tensors = input_name_tensor_map[input_name] - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} std::vector {input_name}_record_shapes;""" + ) for input_tensor, _ in input_tensors: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} if({input_tensor}){{ {code_indent} {input_name}_record_shapes.push_back((*{input_tensor}).dims()); {code_indent} }}""" + ) - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} std::vector>> input_shapes{{""" + ) for input_name in single_tensor_names[:-1]: if input_name in self.optional_vars: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} {{"{input_name}", {input_name}_record_shapes}},""" + ) else: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} {{"{input_name}", {{""" + ) input_tensors = input_name_tensor_map[input_name] for input_tensor, _ in input_tensors[:-1]: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} (*{input_tensor}).dims(),""" - input_tensor_code = input_tensor_code + f""" + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} (*{input_tensors[-1][0]}).dims()}}}},""" + ) if single_tensor_names[-1] in self.optional_vars: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} {{"{single_tensor_names[-1]}", {code_indent} {single_tensor_names[-1]}_record_shapes}}}};""" + ) else: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} {{"{single_tensor_names[-1]}", {{""" + ) input_tensors = input_name_tensor_map[single_tensor_names[-1]] for input_tensor, _ in input_tensors[:-1]: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} (*{input_tensor}).dims(),""" - input_tensor_code = input_tensor_code + f""" + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} (*{input_tensors[-1][0]}).dims()}}}}}};""" + ) if list_tensor_names: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} std::vector ddims_vec;""" + ) for input_name in list_tensor_names: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} ddims_vec.clear();""" + ) for input_tensor, is_vector in input_name_tensor_map[input_name]: if is_vector: input_tensor_truncate = input_tensor[:-4] @@ -815,29 +990,44 @@ def get_kernel_args(self, kernel_tensor_type=None, code_indent=''): input_tensor_truncate = input_tensor if input_name in self.optional_vars: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} if ({input_tensor_truncate}){{ {code_indent} ddims_vec.reserve({input_tensor_truncate}->size()); {code_indent} for (size_t i = 0; i < {input_tensor_truncate}->size(); ++i) {{ {code_indent} ddims_vec.emplace_back((*{input_tensor_truncate}->at(i)).dims()); {code_indent} }} {code_indent} }}""" + ) else: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} ddims_vec.reserve({input_tensor_truncate}.size()); {code_indent} for (size_t i = 0; i < {input_tensor_truncate}.size(); ++i) {{ {code_indent} ddims_vec.emplace_back((*{input_tensor_truncate}[i]).dims()); {code_indent} }}""" + ) else: - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" ddims_vec.emplace_back((*{input_tensor}).dims()); {code_indent} """ - input_tensor_code = input_tensor_code + f""" + ) + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} input_shapes.emplace_back("{input_name}", ddims_vec);""" + ) - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} platform::RecordOpInfoSupplement("{self.api}", input_shapes); {code_indent} }}""" + ) kernel_args = ["*dev_ctx"] for param in kernel_param: if param in input_names: @@ -846,20 +1036,27 @@ def get_kernel_args(self, kernel_tensor_type=None, code_indent=''): else: if self.inputs['input_info'][param] == "const Tensor&": kernel_args.append("*" + PREFIX_TENSOR_NAME + param) - elif self.inputs['input_info'][ - param] == "const std::vector&": + elif ( + self.inputs['input_info'][param] + == "const std::vector&" + ): kernel_args.append(PREFIX_TENSOR_NAME + param) else: # do nothing pass # input is dense tensor - if kernel_tensor_type is None or kernel_tensor_type[0][ - kernel_param.index(param)] == 'dense': + if ( + kernel_tensor_type is None + or kernel_tensor_type[0][kernel_param.index(param)] + == 'dense' + ): kernel_args_type_list.append( - dense_input_trans_map[input_infos[param]]) + dense_input_trans_map[input_infos[param]] + ) else: # input is selected_rows kernel_args_type_list.append( - sr_input_trans_map[input_infos[param]]) + sr_input_trans_map[input_infos[param]] + ) elif param in attr_names: # set attr for kernel_context if 'IntArray' in self.attrs['attr_info'][param][0]: @@ -867,14 +1064,16 @@ def get_kernel_args(self, kernel_tensor_type=None, code_indent=''): param = 'phi::IntArray(' + param + ')' elif 'vector' in self.attrs['attr_info'][param][0]: kernel_args_type_list.append( - 'const std::vector&') + 'const std::vector&' + ) param = param elif 'Scalar' in self.attrs['attr_info'][param][0]: kernel_args_type_list.append('const phi::Scalar&') param = 'phi::Scalar(' + param + ')' else: kernel_args_type_list.append( - self.attrs['attr_info'][param][0]) + self.attrs['attr_info'][param][0] + ) kernel_args.append(param) elif isinstance(param, bool): kernel_args.append(str(param).lower()) @@ -883,7 +1082,10 @@ def get_kernel_args(self, kernel_tensor_type=None, code_indent=''): for i, out_type in enumerate(self.outputs['types']): # output is dense tensor - if kernel_tensor_type is None or kernel_tensor_type[1][i] == 'dense': + if ( + kernel_tensor_type is None + or kernel_tensor_type[1][i] == 'dense' + ): kernel_args_type_list.append(dense_out_trans_map[out_type]) else: # output is selected_rows kernel_args_type_list.append(sr_out_trans_map[out_type]) @@ -897,28 +1099,36 @@ def gene_return_code(self): return "return api_output;" # Override by child class - def gene_output(self, - out_dtype_list, - out_tensor_type_list=None, - code_indent='', - inplace_flag=False): + def gene_output( + self, + out_dtype_list, + out_tensor_type_list=None, + code_indent='', + inplace_flag=False, + ): return None, None, None def gen_kernel_code(self, kernel_name, code_indent, inplace_flag=False): kernel_dispatch = self.kernel['dispatch'][kernel_name] input_tensors, kernel_args, kernel_signature = self.get_kernel_args( - kernel_dispatch, code_indent) + kernel_dispatch, code_indent + ) out_tensor_type_list = kernel_dispatch[1] if kernel_dispatch else None outputs_args, kernel_output_names, output_create = self.gene_output( - self.outputs['types'], out_tensor_type_list, code_indent, - inplace_flag) + self.outputs['types'], + out_tensor_type_list, + code_indent, + inplace_flag, + ) fallback_kernel_output_trans = "" for kernel_out in outputs_args: - fallback_kernel_output_trans += (f""" + fallback_kernel_output_trans += f""" {code_indent} TransDataBackend({kernel_out}, kernel_backend, {kernel_out});""" - ) - cudnn_args = '' if self.kernel[ - 'use_gpudnn'] == 'false' else ', ' + self.kernel['use_gpudnn'] + cudnn_args = ( + '' + if self.kernel['use_gpudnn'] == 'false' + else ', ' + self.kernel['use_gpudnn'] + ) return f""" {code_indent} VLOG(6) << "{self.api} API kernel key: [" << kernel_backend << ", " << kernel_layout << ", "<< kernel_data_type << "]"; {code_indent} auto kernel_result = phi::KernelFactory::Instance().SelectKernelOrThrowError( @@ -952,8 +1162,9 @@ def gen_kernel_code(self, kernel_name, code_indent, inplace_flag=False): {code_indent} {self.gene_return_code()}""" def get_condition_code(self, kernel_name): - assert self.kernel['dispatch'][kernel_name], \ - f"{self.api} api: the tensor type of inputs and outputs for kernel isn't set, see also 'kernel:func' of 'scale' in ops.yaml." + assert self.kernel['dispatch'][ + kernel_name + ], f"{self.api} api: the tensor type of inputs and outputs for kernel isn't set, see also 'kernel:func' of 'scale' in ops.yaml." input_types = self.kernel['dispatch'][kernel_name][0] condition_list = [] for i, in_type in enumerate(input_types): @@ -964,7 +1175,8 @@ def get_condition_code(self, kernel_name): ) else: condition_list.append( - f"{self.inputs['names'][i]}.is_dense_tensor()") + f"{self.inputs['names'][i]}.is_dense_tensor()" + ) else: if self.inputs['names'][i] in self.optional_vars: condition_list.append( @@ -972,7 +1184,8 @@ def get_condition_code(self, kernel_name): ) else: condition_list.append( - f"{self.inputs['names'][i]}.is_selected_rows()") + f"{self.inputs['names'][i]}.is_selected_rows()" + ) return " && ".join(condition_list) def gene_dispatch_code(self, kernel_name, inplace_flag=False): @@ -995,18 +1208,25 @@ def gene_base_api_code(self, inplace_flag=False): kernel_dispatch_code = '' for kernel_name in self.kernel['func']: kernel_dispatch_code += self.gene_dispatch_code( - kernel_name, inplace_flag) - return api_code + f""" + kernel_name, inplace_flag + ) + return ( + api_code + + f""" {kernel_dispatch_code} PADDLE_THROW(phi::errors::Unimplemented( "The kernel of ({self.api}) for input tensors is unimplemented, please check the type of input tensors.")); }} """ + ) else: - return api_code + self.gen_kernel_code(self.kernel['func'][0], '', - inplace_flag) + """ + return ( + api_code + + self.gen_kernel_code(self.kernel['func'][0], '', inplace_flag) + + """ } """ + ) def gene_invoke_code(self, invoke_code, params_code): return f""" @@ -1034,8 +1254,9 @@ def adjust_name(matched): return matched_str[0:-1] + '_val' + matched_str[-1] invoke_code = re.sub(pattern, adjust_name, self.invoke) - params_code = re.sub(pattern, adjust_name, - self.get_define_args()) + params_code = re.sub( + pattern, adjust_name, self.get_define_args() + ) else: invoke_code = self.invoke params_code = self.get_define_args() diff --git a/paddle/phi/api/yaml/generator/api_gen.py b/paddle/phi/api/yaml/generator/api_gen.py index 9cfc54a4c58d5..4b8a06dc21992 100644 --- a/paddle/phi/api/yaml/generator/api_gen.py +++ b/paddle/phi/api/yaml/generator/api_gen.py @@ -20,23 +20,24 @@ inplace_out_type_map = { "Tensor": "Tensor&", - "std::vector": "std::vector&" + "std::vector": "std::vector&", } inplace_optional_out_type_map = { "Tensor": "paddle::optional&", - "std::vector": "paddle::optional>&" + "std::vector": "paddle::optional>&", } class ForwardAPI(BaseAPI): - def __init__(self, api_item_yaml): super(ForwardAPI, self).__init__(api_item_yaml) self.is_dygraph_api, self.intermediate_outs = self.parse_intermediate( - api_item_yaml) + api_item_yaml + ) self.inplace_map, self.view_map = self.parse_inplace_and_view( - api_item_yaml) + api_item_yaml + ) def get_api_func_name(self): if self.is_dygraph_api: @@ -47,17 +48,26 @@ def get_api_func_name(self): def gene_input(self, kernel_tensor_type=None, code_indent=''): kernel_param = self.kernel['param'] input_name_tensor_map, input_tensor_code = super().gene_input( - kernel_tensor_type, code_indent) + kernel_tensor_type, code_indent + ) # generate the input that is in view list for i, input_name in enumerate(self.inputs['names']): - if input_name in self.view_map.values( - ) and input_name not in input_name_tensor_map.keys(): - if kernel_tensor_type is None or kernel_tensor_type[0][ - kernel_param.index(input_name)] == 'dense': + if ( + input_name in self.view_map.values() + and input_name not in input_name_tensor_map.keys() + ): + if ( + kernel_tensor_type is None + or kernel_tensor_type[0][kernel_param.index(input_name)] + == 'dense' + ): trans_flag = self.gene_trans_flag(input_name) - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name} = PrepareData({input_name}, kernel.InputAt(0), {trans_flag});""" + ) else: # do nothing pass @@ -87,10 +97,12 @@ def parse_inplace_and_view(self, api_item_yaml): result = re.search(r"(?P\w+)\s*->\s*(?P\w+)", item) in_val = result.group('in') out_val = result.group('out') - assert in_val in self.inputs['names'], \ - f"{self.api} : {mode} input error: the input var name('{in_val}') is not found in the input args of {self.api}." - assert out_val in self.outputs['names'], \ - f"{self.api} : {mode} output error: the output var name('{out_val}') is not found in the output args of {self.api}." + assert ( + in_val in self.inputs['names'] + ), f"{self.api} : {mode} input error: the input var name('{in_val}') is not found in the input args of {self.api}." + assert ( + out_val in self.outputs['names'] + ), f"{self.api} : {mode} output error: the output var name('{out_val}') is not found in the output args of {self.api}." if mode == 'inplace': inplace_map[out_val] = in_val @@ -106,7 +118,8 @@ def get_return_type_with_intermediate(self, inplace_flag=False): if inplace_flag and out_name in self.inplace_map: if self.inplace_map[out_name] in self.optional_vars: out_type_list.append( - inplace_optional_out_type_map[out_type]) + inplace_optional_out_type_map[out_type] + ) else: out_type_list.append(inplace_out_type_map[out_type]) else: @@ -124,7 +137,8 @@ def get_return_type(self, inplace_flag=False): if inplace_flag and out_name in self.inplace_map: if self.inplace_map[out_name] in self.optional_vars: out_type_list.append( - inplace_optional_out_type_map[out_type]) + inplace_optional_out_type_map[out_type] + ) else: out_type_list.append(inplace_out_type_map[out_type]) elif self.is_dygraph_api or out_name not in self.intermediate_outs: @@ -151,11 +165,13 @@ def gene_return_code(self): ] return 'return std::make_tuple(' + ", ".join(selected_code) + ');' - def gene_output(self, - out_dtype_list, - out_tensor_type_list=None, - code_indent='', - inplace_flag=False): + def gene_output( + self, + out_dtype_list, + out_tensor_type_list=None, + code_indent='', + inplace_flag=False, + ): kernel_output = [] output_names = [] output_create = "" @@ -164,29 +180,48 @@ def gene_output(self, if len(out_dtype_list) == 1: kernel_output.append('kernel_out') output_names.append('kernel_out') - inplace_assign = " = " + self.inplace_map[ - self.outputs['names'][0]] if inplace_flag and self.outputs[ - 'names'][0] in self.inplace_map else "" + inplace_assign = ( + " = " + self.inplace_map[self.outputs['names'][0]] + if inplace_flag and self.outputs['names'][0] in self.inplace_map + else "" + ) output_create = f""" {code_indent} {return_type} api_output{inplace_assign};""" - set_out_func = 'SetKernelOutput' if out_tensor_type_list is None or out_tensor_type_list[ - 0] == 'dense' else 'SetSelectedRowsKernelOutput' + set_out_func = ( + 'SetKernelOutput' + if out_tensor_type_list is None + or out_tensor_type_list[0] == 'dense' + else 'SetSelectedRowsKernelOutput' + ) if return_type == 'std::vector': - assert self.outputs['out_size_expr'][0] is not None, \ - f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." - output_create = output_create + f""" + assert ( + self.outputs['out_size_expr'][0] is not None + ), f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." + output_create = ( + output_create + + f""" {code_indent} auto kernel_out = {set_out_func}({self.outputs['out_size_expr'][0]}, &api_output);""" + ) else: - output_create = output_create + f""" + output_create = ( + output_create + + f""" {code_indent} auto kernel_out = {set_out_func}(&api_output);""" - - if not inplace_flag and self.view_map is not None and self.outputs[ - 'names'][0] in self.view_map: - output_create = output_create + f""" + ) + + if ( + not inplace_flag + and self.view_map is not None + and self.outputs['names'][0] in self.view_map + ): + output_create = ( + output_create + + f""" {code_indent} kernel_out->ShareBufferWith(*{PREFIX_TENSOR_NAME}{self.view_map[self.outputs['names'][0]]}); {code_indent} kernel_out->ShareInplaceVersionCounterWith(*{PREFIX_TENSOR_NAME}{self.view_map[self.outputs['names'][0]]}); {code_indent} VLOG(3) << "Perform View between Output and Input Tensor, share allocation and inplace version.";""" + ) elif len(out_dtype_list) > 1: output_create = f""" @@ -206,47 +241,74 @@ def gene_output(self, for i in range(len(out_dtype_list)): kernel_output.append(f'kernel_out_{i}') output_names.append(f'kernel_out_{i}') - set_out_func = 'SetKernelOutput' if out_tensor_type_list is None or out_tensor_type_list[ - i] == 'dense' else 'SetSelectedRowsKernelOutput' + set_out_func = ( + 'SetKernelOutput' + if out_tensor_type_list is None + or out_tensor_type_list[i] == 'dense' + else 'SetSelectedRowsKernelOutput' + ) get_out_code = f"&std::get<{i}>(api_output)" - if self.outputs['names'][ - i] in self.inplace_map and self.inplace_map[ - self.outputs['names'][i]] in self.optional_vars: + if ( + self.outputs['names'][i] in self.inplace_map + and self.inplace_map[self.outputs['names'][i]] + in self.optional_vars + ): get_out_code = f"std::get<{i}>(api_output).get_ptr()" if out_dtype_list[i] == 'std::vector': - assert self.outputs['out_size_expr'][i] is not None, \ - f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." + assert ( + self.outputs['out_size_expr'][i] is not None + ), f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." # Special case for inplace vector and inplace optional if self.outputs['names'][i] in self.inplace_map: set_out_func = "SetInplaceVectorKernelOutput" - if self.inplace_map[self.outputs['names'] - [i]] in self.optional_vars: - set_out_func = "SetInplaceOptionalVectorKernelOutput" + if ( + self.inplace_map[self.outputs['names'][i]] + in self.optional_vars + ): + set_out_func = ( + "SetInplaceOptionalVectorKernelOutput" + ) get_out_code = f"std::get<{i}>(api_output)" - output_create = output_create + f""" + output_create = ( + output_create + + f""" {code_indent} auto kernel_out_{i} = {set_out_func}({self.outputs['out_size_expr'][i]}, {get_out_code});""" + ) else: - output_create = output_create + f""" + output_create = ( + output_create + + f""" {code_indent} auto kernel_out_{i} = {set_out_func}({get_out_code});""" + ) - if not inplace_flag and self.view_map is not None and self.outputs[ - 'names'][i] in self.view_map: + if ( + not inplace_flag + and self.view_map is not None + and self.outputs['names'][i] in self.view_map + ): if out_dtype_list[i] == 'Tensor': - output_create = output_create + f""" + output_create = ( + output_create + + f""" {code_indent} kernel_out_{i}->ShareBufferWith(*{PREFIX_TENSOR_NAME}{self.view_map[self.outputs['names'][i]]}); {code_indent} kernel_out_{i}->ShareInplaceVersionCounterWith(*{PREFIX_TENSOR_NAME}{self.view_map[self.outputs['names'][i]]}); {code_indent} VLOG(3) << "Perform View between Output and Input Tensor, share allocation and inplace version.";""" + ) else: raise ValueError( - "{} : Output error: only support Tensor type when use view in yaml. But get {}" - .format(self.api, out_dtype_list[i])) + "{} : Output error: only support Tensor type when use view in yaml. But get {}".format( + self.api, out_dtype_list[i] + ) + ) else: raise ValueError( "{} : Output error: the output should not be empty.".format( - self.api)) + self.api + ) + ) return kernel_output, output_names, output_create @@ -288,15 +350,18 @@ def source_include(header_file_path): def api_namespace(): - return (""" + return ( + """ namespace paddle { namespace experimental { -""", """ +""", + """ } // namespace experimental } // namespace paddle -""") +""", + ) def generate_api(api_yaml_path, header_file_path, source_file_path): @@ -338,19 +403,26 @@ def generate_api(api_yaml_path, header_file_path, source_file_path): def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle C++ API files') - parser.add_argument('--api_yaml_path', - help='path to api yaml file', - nargs='+', - default='paddle/phi/api/yaml/ops.yaml') - - parser.add_argument('--api_header_path', - help='output of generated api header code file', - default='paddle/phi/api/include/api.h') - - parser.add_argument('--api_source_path', - help='output of generated api source code file', - default='paddle/phi/api/lib/api.cc') + description='Generate PaddlePaddle C++ API files' + ) + parser.add_argument( + '--api_yaml_path', + help='path to api yaml file', + nargs='+', + default='paddle/phi/api/yaml/ops.yaml', + ) + + parser.add_argument( + '--api_header_path', + help='output of generated api header code file', + default='paddle/phi/api/include/api.h', + ) + + parser.add_argument( + '--api_source_path', + help='output of generated api source code file', + default='paddle/phi/api/lib/api.cc', + ) options = parser.parse_args() diff --git a/paddle/phi/api/yaml/generator/backward_api_gen.py b/paddle/phi/api/yaml/generator/backward_api_gen.py index 16052e8193774..f06af0ead4b63 100644 --- a/paddle/phi/api/yaml/generator/backward_api_gen.py +++ b/paddle/phi/api/yaml/generator/backward_api_gen.py @@ -20,7 +20,6 @@ class BackwardAPI(BaseAPI): - def __init__(self, backward_item_yaml): super(BackwardAPI, self).__init__(backward_item_yaml) self.check_args(backward_item_yaml['forward']) @@ -33,12 +32,18 @@ def parse_forward_config(self, forward_config): # api_name (const Tensor& input, ... , int attr, ...) -> Tensor(out) result = re.search( r"(?P[a-z][a-z0-9_]+)\s*(?P\([^\)]+\))\s*->\s*(?P.+)", - forward_config) + forward_config, + ) api = result.group('op') - _, outputs, _, = self.parse_output(self.api, result.group('outputs')) + ( + _, + outputs, + _, + ) = self.parse_output(self.api, result.group('outputs')) outputs = [item.split('@')[0] for item in outputs] - fw_inputs, fw_attrs = self.parse_input_and_attr(api, - result.group('args')) + fw_inputs, fw_attrs = self.parse_input_and_attr( + api, result.group('args') + ) return api, fw_inputs, fw_attrs, outputs @@ -54,27 +59,34 @@ def parse_no_need_buffer(self, api_item_yaml): def check_args(self, forward_config): # parse the forward and backward config _, fw_inputs, fw_attrs, fw_outputs = self.parse_forward_config( - forward_config) + forward_config + ) # check the inputs of backward for input in self.inputs['names']: if input not in fw_inputs['names'] and input not in fw_outputs: if input.endswith('_grad'): original_name = input[:-5] - assert original_name in fw_outputs, \ - f"{self.api} : Input Tensor error: the input tensor({input}) of backward should be an input or output or grad of output in forward api. \ + assert ( + original_name in fw_outputs + ), f"{self.api} : Input Tensor error: the input tensor({input}) of backward should be an input or output or grad of output in forward api. \ Please check the forward of {self.api} in yaml." # check the attributes of backward for attr in self.attrs['names']: - assert (attr in fw_attrs['names'] and self.attrs['attr_info'][attr][0] == fw_attrs['attr_info'][attr][0]) or \ - self.attrs['attr_info'][attr][1] is not None, \ - f"{self.api} : Attribute error: The attribute({attr}) of backward isn't consistent with forward api or doesn't have default value. \ + assert ( + attr in fw_attrs['names'] + and self.attrs['attr_info'][attr][0] + == fw_attrs['attr_info'][attr][0] + ) or self.attrs['attr_info'][attr][ + 1 + ] is not None, f"{self.api} : Attribute error: The attribute({attr}) of backward isn't consistent with forward api or doesn't have default value. \ Please check the args of {self.api} in yaml." # check the output of backward - assert len(self.outputs['types']) <= len(fw_inputs['names']), \ - f"{self.api} : Output error: The number of outputs should be less then the number of inputs of forward api. \ + assert len(self.outputs['types']) <= len( + fw_inputs['names'] + ), f"{self.api} : Output error: The number of outputs should be less then the number of inputs of forward api. \ Please check the output of {self.api} in yaml." def get_declare_args(self, inplace_flag=False): @@ -83,13 +95,16 @@ def get_declare_args(self, inplace_flag=False): def get_define_args(self, inplace_flag=False): out_type_map = { 'Tensor': 'Tensor*', - 'std::vector': 'std::vector' + 'std::vector': 'std::vector', } intputs_and_attrs = super(BackwardAPI, self).get_define_args() outs = [] for i, name in enumerate(self.outputs['names']): - outs.append(out_type_map[self.outputs['types'][i]] + ' ' + - name.split('@')[0]) + outs.append( + out_type_map[self.outputs['types'][i]] + + ' ' + + name.split('@')[0] + ) result = intputs_and_attrs + ', ' + ", ".join(outs) return result @@ -100,7 +115,8 @@ def gene_api_declaration(self): if not self.is_base_api: invoke_func_name = self.invoke.split('(')[0] if (not invoke_func_name.endswith("_grad")) and ( - not invoke_func_name.endswith('_impl')): + not invoke_func_name.endswith('_impl') + ): return "" api_func_name = self.get_api_func_name() api_declaration = f""" @@ -124,11 +140,13 @@ def gene_kernel_backend_select(self): def get_return_type(self, inplace_flag=False): return 'void' - def gene_output(self, - out_dtype_list, - out_tensor_type_list=None, - code_indent='', - inplace_flag=False): + def gene_output( + self, + out_dtype_list, + out_tensor_type_list=None, + code_indent='', + inplace_flag=False, + ): kernel_output = [] output_names = [] output_create = "" @@ -136,60 +154,101 @@ def gene_output(self, if len(out_dtype_list) == 1: kernel_output.append('kernel_out') output_names.append('kernel_out') - inplace_assign = " = " + self.inplace_map[self.outputs['names'][ - 0]] if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][0] in self.inplace_map else "" + inplace_assign = ( + " = " + self.inplace_map[self.outputs['names'][0]] + if inplace_flag + and self.inplace_map is not None + and self.outputs['names'][0] in self.inplace_map + else "" + ) output_create = "" - set_out_func = 'SetKernelOutput' if out_tensor_type_list is None or out_tensor_type_list[ - 0] == 'dense' else 'SetSelectedRowsKernelOutput' + set_out_func = ( + 'SetKernelOutput' + if out_tensor_type_list is None + or out_tensor_type_list[0] == 'dense' + else 'SetSelectedRowsKernelOutput' + ) if out_dtype_list[0] == 'std::vector': - assert self.outputs['out_size_expr'] is not None, \ - f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." - output_create = output_create + f""" + assert ( + self.outputs['out_size_expr'] is not None + ), f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." + output_create = ( + output_create + + f""" {code_indent} auto kernel_out = {set_out_func}(&{self.outputs['names'][0]});""" + ) else: - output_create = output_create + f""" + output_create = ( + output_create + + f""" {code_indent} auto kernel_out = {set_out_func}({self.outputs['names'][0]});""" + ) elif len(out_dtype_list) > 1: output_create = "" for i, out_type_item in enumerate(out_dtype_list): kernel_output.append(f'kernel_out_{i}') output_names.append(f'kernel_out_{i}') - set_out_func = 'SetKernelOutput' if out_tensor_type_list is None or out_tensor_type_list[ - i] == 'dense' else 'SetSelectedRowsKernelOutput' + set_out_func = ( + 'SetKernelOutput' + if out_tensor_type_list is None + or out_tensor_type_list[i] == 'dense' + else 'SetSelectedRowsKernelOutput' + ) if out_type_item == 'Tensor': - if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][i] in self.inplace_map: - output_create = output_create + f""" + if ( + inplace_flag + and self.inplace_map is not None + and self.outputs['names'][i] in self.inplace_map + ): + output_create = ( + output_create + + f""" {code_indent} *{self.outputs['names'][i]} = {self.inplace_map[self.outputs['names'][i]]};""" + ) - output_create = output_create + f""" + output_create = ( + output_create + + f""" {code_indent} auto kernel_out_{i} = {set_out_func}({self.outputs['names'][i]});""" + ) else: - if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][i] in self.inplace_map: - output_create = output_create + f""" + if ( + inplace_flag + and self.inplace_map is not None + and self.outputs['names'][i] in self.inplace_map + ): + output_create = ( + output_create + + f""" {code_indent} *{self.outputs['names'][i]} = {self.inplace_map[self.outputs['names'][i]]};""" - - assert self.outputs['out_size_expr'][i] is not None, \ - f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." - output_create = output_create + f""" + ) + + assert ( + self.outputs['out_size_expr'][i] is not None + ), f"{self.api}: The out size expr : '{{expr}}' should be set when output has Tensor[]. You can refer 'split' api." + output_create = ( + output_create + + f""" {code_indent} auto kernel_out_{i} = {set_out_func}(&{self.outputs['names'][i]});""" + ) else: raise ValueError( "{} : Output error: the output should not be empty.".format( - self.api)) + self.api + ) + ) return kernel_output, output_names, output_create def gene_invoke_code(self, invoke_code, params_code): invoke_func_name = invoke_code.split('(')[0].strip() if invoke_func_name.endswith('_grad') or invoke_func_name.endswith( - '_impl'): + '_impl' + ): return f""" PADDLE_API {self.get_return_type()} {self.api}({params_code}) {{ {invoke_code}; @@ -235,19 +294,23 @@ def source_include(header_file_path): def backward_api_namespace(): - return (""" + return ( + """ namespace paddle { namespace experimental { -""", """ +""", + """ } // namespace experimental } // namespace paddle -""") +""", + ) -def generate_backward_api(backward_yaml_path, header_file_path, - source_file_path): +def generate_backward_api( + backward_yaml_path, header_file_path, source_file_path +): bw_apis = [] for each_api_yaml in backward_yaml_path: @@ -283,18 +346,25 @@ def generate_backward_api(backward_yaml_path, header_file_path, def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle C++ backward API files') - parser.add_argument('--backward_yaml_path', - help='path to backward yaml file', - nargs='+', - default='paddle/phi/api/yaml/backward.yaml') - parser.add_argument('--backward_header_path', - help='output of generated backward header code file', - default='paddle/phi/api/backward/backward_api.h') - - parser.add_argument('--backward_source_path', - help='output of generated backward source code file', - default='paddle/phi/api/lib/backward_api.cc') + description='Generate PaddlePaddle C++ backward API files' + ) + parser.add_argument( + '--backward_yaml_path', + help='path to backward yaml file', + nargs='+', + default='paddle/phi/api/yaml/backward.yaml', + ) + parser.add_argument( + '--backward_header_path', + help='output of generated backward header code file', + default='paddle/phi/api/backward/backward_api.h', + ) + + parser.add_argument( + '--backward_source_path', + help='output of generated backward source code file', + default='paddle/phi/api/lib/backward_api.cc', + ) options = parser.parse_args() @@ -302,8 +372,9 @@ def main(): header_file_path = options.backward_header_path source_file_path = options.backward_source_path - generate_backward_api(backward_yaml_path, header_file_path, - source_file_path) + generate_backward_api( + backward_yaml_path, header_file_path, source_file_path + ) if __name__ == '__main__': diff --git a/paddle/phi/api/yaml/generator/cross_validate.py b/paddle/phi/api/yaml/generator/cross_validate.py index 63e94eab12a7e..f42ff8680a4a4 100644 --- a/paddle/phi/api/yaml/generator/cross_validate.py +++ b/paddle/phi/api/yaml/generator/cross_validate.py @@ -34,17 +34,22 @@ def main(forward_api_yaml_paths, backward_api_yaml_paths): if __name__ == "__main__": current_dir = Path(__file__).parent / "temp" parser = argparse.ArgumentParser( - description="Parse api yaml into canonical format.") - parser.add_argument('--forward_yaml_paths', - type=str, - nargs='+', - default=str(current_dir / "api.parsed.yaml"), - help="forward api yaml file.") - parser.add_argument('--backward_yaml_paths', - type=str, - nargs='+', - default=str(current_dir / "backward_api.parsed.yaml"), - help="backward api yaml file.") + description="Parse api yaml into canonical format." + ) + parser.add_argument( + '--forward_yaml_paths', + type=str, + nargs='+', + default=str(current_dir / "api.parsed.yaml"), + help="forward api yaml file.", + ) + parser.add_argument( + '--backward_yaml_paths', + type=str, + nargs='+', + default=str(current_dir / "backward_api.parsed.yaml"), + help="backward api yaml file.", + ) args = parser.parse_args() main(args.forward_yaml_paths, args.backward_yaml_paths) diff --git a/paddle/phi/api/yaml/generator/filters.py b/paddle/phi/api/yaml/generator/filters.py index 30af0dee34dab..9654f8a1991e7 100644 --- a/paddle/phi/api/yaml/generator/filters.py +++ b/paddle/phi/api/yaml/generator/filters.py @@ -15,12 +15,20 @@ import itertools import re -from type_mapping import (input_types_map, optional_input_types_map, - attr_types_map, opmaker_attr_types_map, - output_type_map) -from type_mapping import (dense_input_types_map, dense_optional_input_types_map, - dense_output_types_map, sr_output_types_map, - phi_attr_types_map) +from type_mapping import ( + input_types_map, + optional_input_types_map, + attr_types_map, + opmaker_attr_types_map, + output_type_map, +) +from type_mapping import ( + dense_input_types_map, + dense_optional_input_types_map, + dense_output_types_map, + sr_output_types_map, + phi_attr_types_map, +) def quote(s): @@ -104,7 +112,7 @@ def to_input_name(s): is more common. """ match = re.match(r"(d\d*)(\w+)", s) - assert (match.group(1) != ""), "it should be a grad style name." + assert match.group(1) != "", "it should be a grad style name." return match.group(2) @@ -117,9 +125,10 @@ def cartesian_prod_attrs(attrs): items.append((name, "{}Tensor".format(name))) elif type_name == "IntArray": items.append( - (name, "{}Tensor".format(name), "{}TensorList".format(name))) + (name, "{}Tensor".format(name), "{}TensorList".format(name)) + ) else: - items.append((name, )) + items.append((name,)) _combinations = itertools.product(*items) combinations = [] diff --git a/paddle/phi/api/yaml/generator/generate_op.py b/paddle/phi/api/yaml/generator/generate_op.py index b78fbecd3fbb7..a7efe85398045 100644 --- a/paddle/phi/api/yaml/generator/generate_op.py +++ b/paddle/phi/api/yaml/generator/generate_op.py @@ -19,18 +19,32 @@ import yaml from jinja2 import Environment, FileSystemLoader, StrictUndefined -from filters import to_op_attr_type, to_opmaker_name, to_opmaker_name_cstr, to_pascal_case -from tests import is_base_api, is_vec, is_scalar, is_initializer_list, supports_inplace, supports_no_need_buffer +from filters import ( + to_op_attr_type, + to_opmaker_name, + to_opmaker_name_cstr, + to_pascal_case, +) +from tests import ( + is_base_api, + is_vec, + is_scalar, + is_initializer_list, + supports_inplace, + supports_no_need_buffer, +) from filters import to_input_name, cartesian_prod_mapping from parse_utils import to_named_dict file_loader = FileSystemLoader(Path(__file__).parent / "templates") -env = Environment(loader=file_loader, - keep_trailing_newline=True, - trim_blocks=True, - lstrip_blocks=True, - undefined=StrictUndefined, - extensions=['jinja2.ext.do']) +env = Environment( + loader=file_loader, + keep_trailing_newline=True, + trim_blocks=True, + lstrip_blocks=True, + undefined=StrictUndefined, + extensions=['jinja2.ext.do'], +) env.filters["to_op_attr_type"] = to_op_attr_type env.filters["to_opmaker_name"] = to_opmaker_name env.filters["to_pascal_case"] = to_pascal_case @@ -54,7 +68,6 @@ def restruct_io(api): # replace name of op and params for OpMaker def replace_compat_name(api_op_map, forward_api_dict, backward_api_dict): - def get_api_and_op_name(api_item): names = api_item.split('(') if len(names) == 1: @@ -74,7 +87,8 @@ def get_api_and_op_name(api_item): forward_api_item['op_name'] = op_name if 'backward' in api_args and has_backward: bw_api_name, bw_op_name = get_api_and_op_name( - api_args['backward'].split(',')[0]) + api_args['backward'].split(',')[0] + ) forward_api_item['backward'] = bw_op_name backward_api_item['op_name'] = bw_op_name @@ -100,8 +114,10 @@ def get_api_and_op_name(api_item): ] if forward_api_item['kernel']['data_type']: forward_api_item['kernel']['data_type']['candidates'] = [ - args_map[param] if param in args_map else param for param in - forward_api_item['kernel']['data_type']['candidates'] + args_map[param] if param in args_map else param + for param in forward_api_item['kernel']['data_type'][ + 'candidates' + ] ] if forward_api_item['kernel']['backend']: forward_api_item['kernel']['backend']['candidates'] = [ @@ -128,25 +144,32 @@ def get_api_and_op_name(api_item): for args_item in backward_api_item['inputs']: if args_item['name'] in args_map: args_item['name'] = args_map[args_item['name']] - elif args_item['name'].endswith( - '_grad') and args_item['name'][:-5] in args_map: - args_map[args_item['name']] = args_map[args_item['name'] - [:-5]] + '_grad' + elif ( + args_item['name'].endswith('_grad') + and args_item['name'][:-5] in args_map + ): + args_map[args_item['name']] = ( + args_map[args_item['name'][:-5]] + '_grad' + ) args_item['name'] = args_map[args_item['name']] for args_item in backward_api_item['attrs']: if args_item['name'] in args_map: args_item['name'] = args_map[args_item['name']] for args_item in backward_api_item['outputs']: - if args_item['name'].endswith( - '_grad') and args_item['name'][:-5] in args_map: - args_map[args_item['name']] = args_map[args_item['name'] - [:-5]] + '_grad' + if ( + args_item['name'].endswith('_grad') + and args_item['name'][:-5] in args_map + ): + args_map[args_item['name']] = ( + args_map[args_item['name'][:-5]] + '_grad' + ) args_item['name'] = args_map[args_item['name']] if 'invoke' in backward_api_item: backward_api_item['invoke']['args'] = [ args_map[param.strip()] - if param.strip() in args_map else param.strip() + if param.strip() in args_map + else param.strip() for param in backward_api_item['invoke']['args'].split(',') ] continue @@ -161,18 +184,24 @@ def get_api_and_op_name(api_item): ] if backward_api_item['kernel']['data_type']: backward_api_item['kernel']['data_type']['candidates'] = [ - args_map[param] if param in args_map else param for param in - backward_api_item['kernel']['data_type']['candidates'] + args_map[param] if param in args_map else param + for param in backward_api_item['kernel']['data_type'][ + 'candidates' + ] ] if backward_api_item['kernel']['backend']: backward_api_item['kernel']['backend']['candidates'] = [ - args_map[param] if param in args_map else param for param in - backward_api_item['kernel']['backend']['candidates'] + args_map[param] if param in args_map else param + for param in backward_api_item['kernel']['backend'][ + 'candidates' + ] ] if backward_api_item['kernel']['layout']: backward_api_item['kernel']['layout']['candidates'] = [ - args_map[param] if param in args_map else param for param in - backward_api_item['kernel']['layout']['candidates'] + args_map[param] if param in args_map else param + for param in backward_api_item['kernel']['layout'][ + 'candidates' + ] ] if backward_api_item['no_need_buffer']: backward_api_item['no_need_buffer'] = [ @@ -193,36 +222,43 @@ def process_invoke_op(forward_api_dict, backward_api_dict): bw_api['invoke']['attrs'] = [] bw_api['invoke']['outputs'] = [] for input_item in reuse_op['inputs']: - bw_api['invoke']['inputs'].append({ - 'name': - input_item['name'], - 'value': - args_list[args_index] - }) + bw_api['invoke']['inputs'].append( + { + 'name': input_item['name'], + 'value': args_list[args_index], + } + ) args_index = args_index + 1 for attr in reuse_op['attrs']: if args_index < len(args_list): - attr_value = f"this->GetAttr(\"{args_list[args_index]}\")" if args_list[ - args_index] in bw_api['attr_dict'] else args_list[ - args_index] - bw_api['invoke']['attrs'].append({ - 'name': attr['name'], - 'value': attr_value - }) + attr_value = ( + f"this->GetAttr(\"{args_list[args_index]}\")" + if args_list[args_index] in bw_api['attr_dict'] + else args_list[args_index] + ) + bw_api['invoke']['attrs'].append( + {'name': attr['name'], 'value': attr_value} + ) args_index = args_index + 1 else: break for idx, output_item in enumerate(reuse_op['outputs']): - bw_api['invoke']['outputs'].append({ - 'name': - output_item['name'], - 'value': - bw_api['outputs'][idx]['name'] - }) - - -def main(ops_yaml_path, backward_yaml_path, op_compat_yaml_path, - op_version_yaml_path, output_op_path, output_arg_map_path): + bw_api['invoke']['outputs'].append( + { + 'name': output_item['name'], + 'value': bw_api['outputs'][idx]['name'], + } + ) + + +def main( + ops_yaml_path, + backward_yaml_path, + op_compat_yaml_path, + op_version_yaml_path, + output_op_path, + output_arg_map_path, +): with open(ops_yaml_path, "rt") as f: apis = yaml.safe_load(f) apis = [restruct_io(api) for api in apis] @@ -273,9 +309,9 @@ def main(ops_yaml_path, backward_yaml_path, op_compat_yaml_path, op_template = env.get_template('op.c.j2') with open(output_op_path, "wt") as f: - msg = op_template.render(apis=apis, - backward_apis=backward_apis, - api_dict=api_dict) + msg = op_template.render( + apis=apis, backward_apis=backward_apis, api_dict=api_dict + ) f.write(msg) ks_template = env.get_template('ks.c.j2') @@ -286,28 +322,35 @@ def main(ops_yaml_path, backward_yaml_path, op_compat_yaml_path, if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Generate operator file from api yaml.") - parser.add_argument('--ops_yaml_path', - type=str, - help="parsed ops yaml file.") - parser.add_argument('--backward_yaml_path', - type=str, - help="parsed backward ops yaml file.") - parser.add_argument('--op_compat_yaml_path', - type=str, - help="ops args compat yaml file.") - parser.add_argument('--op_version_yaml_path', - type=str, - help="ops version yaml file.") - parser.add_argument("--output_op_path", - type=str, - help="path to save generated operators.") + description="Generate operator file from api yaml." + ) + parser.add_argument( + '--ops_yaml_path', type=str, help="parsed ops yaml file." + ) + parser.add_argument( + '--backward_yaml_path', type=str, help="parsed backward ops yaml file." + ) + parser.add_argument( + '--op_compat_yaml_path', type=str, help="ops args compat yaml file." + ) + parser.add_argument( + '--op_version_yaml_path', type=str, help="ops version yaml file." + ) + parser.add_argument( + "--output_op_path", type=str, help="path to save generated operators." + ) parser.add_argument( "--output_arg_map_path", type=str, - help="path to save generated argument mapping functions.") + help="path to save generated argument mapping functions.", + ) args = parser.parse_args() - main(args.ops_yaml_path, args.backward_yaml_path, args.op_compat_yaml_path, - args.op_version_yaml_path, args.output_op_path, - args.output_arg_map_path) + main( + args.ops_yaml_path, + args.backward_yaml_path, + args.op_compat_yaml_path, + args.op_version_yaml_path, + args.output_op_path, + args.output_arg_map_path, + ) diff --git a/paddle/phi/api/yaml/generator/generate_sparse_op.py b/paddle/phi/api/yaml/generator/generate_sparse_op.py index e4d7f44856bfd..48ba0d81eca3d 100644 --- a/paddle/phi/api/yaml/generator/generate_sparse_op.py +++ b/paddle/phi/api/yaml/generator/generate_sparse_op.py @@ -19,19 +19,33 @@ import yaml from jinja2 import Environment, FileSystemLoader, StrictUndefined -from filters import to_op_attr_type, to_opmaker_name, to_opmaker_name_cstr, to_pascal_case -from tests import is_base_api, is_vec, is_scalar, is_initializer_list, supports_inplace, supports_no_need_buffer +from filters import ( + to_op_attr_type, + to_opmaker_name, + to_opmaker_name_cstr, + to_pascal_case, +) +from tests import ( + is_base_api, + is_vec, + is_scalar, + is_initializer_list, + supports_inplace, + supports_no_need_buffer, +) from filters import to_input_name, cartesian_prod_mapping from parse_utils import to_named_dict from generate_op import process_invoke_op file_loader = FileSystemLoader(Path(__file__).parent / "templates") -env = Environment(loader=file_loader, - keep_trailing_newline=True, - trim_blocks=True, - lstrip_blocks=True, - undefined=StrictUndefined, - extensions=['jinja2.ext.do']) +env = Environment( + loader=file_loader, + keep_trailing_newline=True, + trim_blocks=True, + lstrip_blocks=True, + undefined=StrictUndefined, + extensions=['jinja2.ext.do'], +) env.filters["to_op_attr_type"] = to_op_attr_type env.filters["to_opmaker_name"] = to_opmaker_name env.filters["to_pascal_case"] = to_pascal_case @@ -56,8 +70,9 @@ def restruct_io(api): SPARSE_OP_PREFIX = 'sparse_' -def main(api_yaml_path, backward_yaml_path, output_op_path, - output_arg_map_path): +def main( + api_yaml_path, backward_yaml_path, output_op_path, output_arg_map_path +): with open(api_yaml_path, "rt") as f: apis = yaml.safe_load(f) apis = [restruct_io(api) for api in apis] @@ -86,8 +101,9 @@ def main(api_yaml_path, backward_yaml_path, output_op_path, for bw_api in backward_apis: if 'invoke' in bw_api: if bw_api['invoke']['func'] in forward_api_dict: - bw_api['invoke'][ - 'func'] = SPARSE_OP_PREFIX + bw_api['invoke']['func'] + bw_api['invoke']['func'] = ( + SPARSE_OP_PREFIX + bw_api['invoke']['func'] + ) # fill backward field for an api if another api claims it as forward for name, backward_api in backward_api_dict.items(): @@ -111,9 +127,9 @@ def main(api_yaml_path, backward_yaml_path, output_op_path, op_template = env.get_template('sparse_op.c.j2') with open(output_op_path, "wt") as f: - msg = op_template.render(apis=apis, - backward_apis=backward_apis, - api_dict=api_dict) + msg = op_template.render( + apis=apis, backward_apis=backward_apis, api_dict=api_dict + ) f.write(msg) ks_template = env.get_template('sparse_ks.c.j2') @@ -124,21 +140,29 @@ def main(api_yaml_path, backward_yaml_path, output_op_path, if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Generate operator file from api yaml.") - parser.add_argument('--ops_yaml_path', - type=str, - help="parsed sparse ops yaml file.") - parser.add_argument('--backward_ops_yaml_path', - type=str, - help="parsed backward sparse ops yaml file.") - parser.add_argument("--output_op_path", - type=str, - help="path to save generated operators.") + description="Generate operator file from api yaml." + ) + parser.add_argument( + '--ops_yaml_path', type=str, help="parsed sparse ops yaml file." + ) + parser.add_argument( + '--backward_ops_yaml_path', + type=str, + help="parsed backward sparse ops yaml file.", + ) + parser.add_argument( + "--output_op_path", type=str, help="path to save generated operators." + ) parser.add_argument( "--output_arg_map_path", type=str, - help="path to save generated argument mapping functions.") + help="path to save generated argument mapping functions.", + ) args = parser.parse_args() - main(args.ops_yaml_path, args.backward_ops_yaml_path, args.output_op_path, - args.output_arg_map_path) + main( + args.ops_yaml_path, + args.backward_ops_yaml_path, + args.output_op_path, + args.output_arg_map_path, + ) diff --git a/paddle/phi/api/yaml/generator/intermediate_api_gen.py b/paddle/phi/api/yaml/generator/intermediate_api_gen.py index d9562e189e867..1f1a55b9d1a37 100644 --- a/paddle/phi/api/yaml/generator/intermediate_api_gen.py +++ b/paddle/phi/api/yaml/generator/intermediate_api_gen.py @@ -58,28 +58,37 @@ def source_include(header_file_path): def api_namespace(): - return (""" + return ( + """ namespace paddle { namespace experimental { -""", """ +""", + """ } // namespace experimental } // namespace paddle -""") +""", + ) def sparse_namespace(): - return (""" + return ( + """ namespace sparse { -""", """ +""", + """ } // namespace sparse -""") +""", + ) -def generate_intermediate_api(api_yaml_path, sparse_api_yaml_path, - dygraph_header_file_path, - dygraph_source_file_path): +def generate_intermediate_api( + api_yaml_path, + sparse_api_yaml_path, + dygraph_header_file_path, + dygraph_source_file_path, +): dygraph_header_file = open(dygraph_header_file_path, 'w') dygraph_source_file = open(dygraph_source_file_path, 'w') @@ -132,23 +141,32 @@ def generate_intermediate_api(api_yaml_path, sparse_api_yaml_path, def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle C++ Sparse API files') - parser.add_argument('--api_yaml_path', - nargs='+', - help='path to api yaml file', - default='paddle/phi/api/yaml/ops.yaml') - - parser.add_argument('--sparse_api_yaml_path', - help='path to sparse api yaml file', - default='paddle/phi/api/yaml/sparse_ops.yaml') - - parser.add_argument('--dygraph_api_header_path', - help='output of generated dygraph api header code file', - default='paddle/phi/api/lib/dygraph_api.h') - - parser.add_argument('--dygraph_api_source_path', - help='output of generated dygraph api source code file', - default='paddle/phi/api/lib/dygraph_api.cc') + description='Generate PaddlePaddle C++ Sparse API files' + ) + parser.add_argument( + '--api_yaml_path', + nargs='+', + help='path to api yaml file', + default='paddle/phi/api/yaml/ops.yaml', + ) + + parser.add_argument( + '--sparse_api_yaml_path', + help='path to sparse api yaml file', + default='paddle/phi/api/yaml/sparse_ops.yaml', + ) + + parser.add_argument( + '--dygraph_api_header_path', + help='output of generated dygraph api header code file', + default='paddle/phi/api/lib/dygraph_api.h', + ) + + parser.add_argument( + '--dygraph_api_source_path', + help='output of generated dygraph api source code file', + default='paddle/phi/api/lib/dygraph_api.cc', + ) options = parser.parse_args() @@ -157,9 +175,12 @@ def main(): dygraph_header_file_path = options.dygraph_api_header_path dygraph_source_file_path = options.dygraph_api_source_path - generate_intermediate_api(api_yaml_path, sparse_api_yaml_path, - dygraph_header_file_path, - dygraph_source_file_path) + generate_intermediate_api( + api_yaml_path, + sparse_api_yaml_path, + dygraph_header_file_path, + dygraph_source_file_path, + ) if __name__ == '__main__': diff --git a/paddle/phi/api/yaml/generator/ops_extra_info_gen.py b/paddle/phi/api/yaml/generator/ops_extra_info_gen.py index 85f1d1657557e..77596b9f2d46b 100644 --- a/paddle/phi/api/yaml/generator/ops_extra_info_gen.py +++ b/paddle/phi/api/yaml/generator/ops_extra_info_gen.py @@ -52,16 +52,20 @@ def map_code_template(attrs_str, attrs_checker_str): 'int64_t[]': 'std::vector', 'float[]': 'std::vector', 'double[]': 'std::vector', - 'str[]': 'std::vector' + 'str[]': 'std::vector', } def parse_attr(attr_str): result = re.search( r"(?P[a-zA-Z0-9_[\]]+)\s+(?P[a-zA-Z0-9_]+)\s*=\s*(?P\S+)", - attr_str) - return ATTR_TYPE_STRING_MAP[result.group('attr_type')], result.group( - 'name'), result.group('default_val') + attr_str, + ) + return ( + ATTR_TYPE_STRING_MAP[result.group('attr_type')], + result.group('name'), + result.group('default_val'), + ) def generate_extra_info(op_compat_yaml_path, ops_extra_info_path): @@ -93,14 +97,16 @@ def get_op_name(api_item): ) if attr_type.startswith("std::vector"): attr_map_list.append( - f"{{\"{attr_name}\", {attr_type}{default_val}}}") + f"{{\"{attr_name}\", {attr_type}{default_val}}}" + ) else: attr_map_list.append( f"{{\"{attr_name}\", {attr_type}{{{default_val}}}}}" ) api_extra_attr_map = ", ".join(attr_map_list) api_extra_attr_checkers = ",\n ".join( - attr_checker_func_list) + attr_checker_func_list + ) extra_map_str_list.append( f"{{\"{get_op_name(op_compat_args['op'])}\", {{ {api_extra_attr_map} }}}}" ) @@ -111,28 +117,37 @@ def get_op_name(api_item): for bw_item in op_compat_args['backward'].split(','): bw_op_name = get_op_name(bw_item) extra_map_str_list.append( - f"{{\"{bw_op_name}\", {{ {api_extra_attr_map} }}}}") + f"{{\"{bw_op_name}\", {{ {api_extra_attr_map} }}}}" + ) extra_checker_str_list.append( f"{{\"{bw_op_name}\", {{ {api_extra_attr_checkers} }}}}" ) ops_extra_info_file = open(ops_extra_info_path, 'w') ops_extra_info_file.write( - map_code_template(",\n ".join(extra_map_str_list), - ",\n ".join(extra_checker_str_list))) + map_code_template( + ",\n ".join(extra_map_str_list), + ",\n ".join(extra_checker_str_list), + ) + ) ops_extra_info_file.close() def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle Extra Param Info for Op') - parser.add_argument('--op_compat_yaml_path', - help='path to api compat yaml file', - default='paddle/phi/api/yaml/op_compat.yaml') - - parser.add_argument('--ops_extra_info_path', - help='output of generated extra_prama_info code file', - default='paddle/fluid/operators/ops_extra_info.cc') + description='Generate PaddlePaddle Extra Param Info for Op' + ) + parser.add_argument( + '--op_compat_yaml_path', + help='path to api compat yaml file', + default='paddle/phi/api/yaml/op_compat.yaml', + ) + + parser.add_argument( + '--ops_extra_info_path', + help='output of generated extra_prama_info code file', + default='paddle/fluid/operators/ops_extra_info.cc', + ) options = parser.parse_args() diff --git a/paddle/phi/api/yaml/generator/parse_api.py b/paddle/phi/api/yaml/generator/parse_api.py index 4536c86653821..64037216b24e2 100644 --- a/paddle/phi/api/yaml/generator/parse_api.py +++ b/paddle/phi/api/yaml/generator/parse_api.py @@ -36,11 +36,12 @@ def main(api_yaml_path, output_path, backward): if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Parse api yaml into canonical format.") + description="Parse api yaml into canonical format." + ) parser.add_argument('--api_yaml_path', type=str, help="api yaml file.") - parser.add_argument("--output_path", - type=str, - help="path to save parsed yaml file.") + parser.add_argument( + "--output_path", type=str, help="path to save parsed yaml file." + ) parser.add_argument("--backward", action="store_true", default=False) args = parser.parse_args() diff --git a/paddle/phi/api/yaml/generator/parse_utils.py b/paddle/phi/api/yaml/generator/parse_utils.py index ed8068b40e827..8220432cf849e 100644 --- a/paddle/phi/api/yaml/generator/parse_utils.py +++ b/paddle/phi/api/yaml/generator/parse_utils.py @@ -34,39 +34,42 @@ def parse_arg(api_name: str, s: str) -> Dict[str, str]: 2. typename name = default_value """ typename, rest = [item.strip() for item in s.split(" ", 1)] - assert len( - typename - ) > 0, f"The arg typename should not be empty. Please check the args of {api_name} in yaml." + assert ( + len(typename) > 0 + ), f"The arg typename should not be empty. Please check the args of {api_name} in yaml." - assert rest.count( - "=") <= 1, f"There is more than 1 = in an arg in {api_name}" + assert ( + rest.count("=") <= 1 + ), f"There is more than 1 = in an arg in {api_name}" if rest.count("=") == 1: name, default_value = [item.strip() for item in rest.split("=", 1)] - assert len( - name - ) > 0, f"The arg name should not be empty. Please check the args of {api_name} in yaml." - assert len( - default_value - ) > 0, f"The default value should not be empty. Please check the args of {api_name} in yaml." + assert ( + len(name) > 0 + ), f"The arg name should not be empty. Please check the args of {api_name} in yaml." + assert ( + len(default_value) > 0 + ), f"The default value should not be empty. Please check the args of {api_name} in yaml." return { "typename": typename, "name": name, - "default_value": default_value + "default_value": default_value, } else: name = rest.strip() - assert len( - name - ) > 0, f"The arg name should not be empty. Please check the args of {api_name} in yaml." + assert ( + len(name) > 0 + ), f"The arg name should not be empty. Please check the args of {api_name} in yaml." return {"typename": typename, "name": name} -def parse_input_and_attr(api_name: str, - arguments: str) -> Tuple[List, List, Dict, Dict]: +def parse_input_and_attr( + api_name: str, arguments: str +) -> Tuple[List, List, Dict, Dict]: args_str = arguments.strip() - assert args_str.startswith('(') and args_str.endswith(')'), \ - (f"Args declaration should start with '(' and end with ')', " - f"please check the args of {api_name} in yaml.") + assert args_str.startswith('(') and args_str.endswith(')'), ( + f"Args declaration should start with '(' and end with ')', " + f"please check the args of {api_name} in yaml." + ) args_str = args_str[1:-1] args = parse_plain_list(args_str) @@ -80,14 +83,17 @@ def parse_input_and_attr(api_name: str, typename = item["typename"] name = item["name"] if is_input(typename): - assert len(attrs) == 0, \ - (f"The input Tensor should appear before attributes. " + assert len(attrs) == 0, ( + f"The input Tensor should appear before attributes. " f"please check the position of {api_name}:input({name}) " - f"in yaml.") + f"in yaml." + ) inputs.append(item) elif is_attr(typename): if met_attr_with_default_value: - assert "default_value" in item, f"{api_name}: Arguments with default value should not precede those without default value" + assert ( + "default_value" in item + ), f"{api_name}: Arguments with default value should not precede those without default value" elif "default_value" in item: met_attr_with_default_value = True attrs.append(item) @@ -100,7 +106,8 @@ def parse_output(api_name: str, s: str) -> Dict[str, str]: """parse an output, typename or typename(name).""" match = re.search( r"(?P[a-zA-Z0-9_[\]]+)\s*(?P\([a-zA-Z0-9_@]+\))?\s*(?P\{[^\}]+\})?", - s) + s, + ) typename = match.group("out_type") name = match.group("name") size_expr = match.group("expr") @@ -108,13 +115,15 @@ def parse_output(api_name: str, s: str) -> Dict[str, str]: name = name[1:-1] if name is not None else 'out' size_expr = size_expr[1:-1] if size_expr is not None else None - assert is_output(typename), \ - (f"Invalid output type: {typename} in api: {api_name}." - f"Supported types are Tensor and Tensor[]") + assert is_output(typename), ( + f"Invalid output type: {typename} in api: {api_name}." + f"Supported types are Tensor and Tensor[]" + ) if size_expr is not None: - assert is_vec(typename), \ - (f"Invalid output size: output {name} in api: {api_name} is " - f"not a vector but has size expr") + assert is_vec(typename), ( + f"Invalid output size: output {name} in api: {api_name} is " + f"not a vector but has size expr" + ) return {"typename": typename, "name": name, "size": size_expr} else: return {"typename": typename, "name": name} @@ -148,8 +157,9 @@ def parse_plain_list(s: str, sep=",") -> List[str]: return items -def parse_kernel(api_name: str, kernel_config: Dict[str, - Any]) -> Dict[str, Any]: +def parse_kernel( + api_name: str, kernel_config: Dict[str, Any] +) -> Dict[str, Any]: # kernel : # func : [], Kernel functions (example: scale, scale_sr) # param : [], Input params of kernel @@ -163,7 +173,7 @@ def parse_kernel(api_name: str, kernel_config: Dict[str, 'backend': None, 'layout': None, 'data_type': None, - 'dispatch': {} + 'dispatch': {}, } if 'param' in kernel_config: kernel['param'] = kernel_config['param'] @@ -178,7 +188,8 @@ def parse_kernel(api_name: str, kernel_config: Dict[str, kernel['data_type'] = parse_candidates(kernel_config["data_type"]) kernel_funcs = re.compile(r'([a-zA-Z0-9_]+)\s*({[^}]+})?').findall( - kernel_config['func']) + kernel_config['func'] + ) def parse_kernel_in_out_type(in_out_str): if len(in_out_str) == 0: @@ -190,11 +201,17 @@ def parse_kernel_in_out_type(in_out_str): # check the tensor type for item in inputs: assert item in [ - 'dense', 'selected_rows', 'sparse_coo', 'sparse_csr' + 'dense', + 'selected_rows', + 'sparse_coo', + 'sparse_csr', ], f"{api_name} : Invalid input tensor type ('{item}'), here we only support 'dense', 'selected_rows', 'sparse_coo' and 'sparse_csr'." for item in outputs: assert item in [ - 'dense', 'selected_rows', 'sparse_coo', 'sparse_csr' + 'dense', + 'selected_rows', + 'sparse_coo', + 'sparse_csr', ], f"{api_name} : Invalid output tensor type ('{item}'), here we only support 'dense', 'selected_rows', 'sparse_coo' and 'sparse_csr'." return (inputs, outputs) @@ -202,7 +219,8 @@ def parse_kernel_in_out_type(in_out_str): for func_item in kernel_funcs: kernel['func'].append(func_item[0]) kernel['dispatch'][func_item[0]] = parse_kernel_in_out_type( - func_item[1]) + func_item[1] + ) return kernel @@ -228,10 +246,9 @@ def parse_invoke(api_name: str, invoke_config: str) -> Dict[str, Any]: def extract_type_and_name(records: List[Dict]) -> List[Dict]: """extract type and name from forward call, it is simpler than forward api.""" - extracted = [{ - "name": item["name"], - "typename": item["typename"] - } for item in records] + extracted = [ + {"name": item["name"], "typename": item["typename"]} for item in records + ] return extracted @@ -239,7 +256,8 @@ def parse_forward(api_name: str, forward_config: str) -> Dict[str, Any]: # api_name (const Tensor& input, ... , int attr, ...) -> Tensor(out) result = re.search( r"(?P[a-z][a-z0-9_]+)\s*(?P\([^\)]+\))\s*->\s*(?P.+)", - forward_config) + forward_config, + ) api = result.group("op") outputs = parse_outputs(api_name, result.group("outputs")) outputs = extract_type_and_name(outputs) @@ -251,7 +269,7 @@ def parse_forward(api_name: str, forward_config: str) -> Dict[str, Any]: "name": api, "inputs": inputs, "attrs": attrs, - "outputs": outputs + "outputs": outputs, } return forward_cfg @@ -267,13 +285,19 @@ def parse_api_entry(api_entry: Dict[str, Any], name_field="op"): typename = attr["typename"] default_value = attr["default_value"] if typename == "DataType": - assert "DataType" in default_value, f"invalid DataType default value in {api_name}" + assert ( + "DataType" in default_value + ), f"invalid DataType default value in {api_name}" # remove namespace - default_value = default_value[default_value.find("DataType"):] + default_value = default_value[default_value.find("DataType") :] attr["default_value"] = default_value elif typename == "DataLayout": - assert "DataLayout" in default_value, f"invalid DataLayout default value in {api_name}" - default_value = default_value[default_value.find("DataLayout"):] + assert ( + "DataLayout" in default_value + ), f"invalid DataLayout default value in {api_name}" + default_value = default_value[ + default_value.find("DataLayout") : + ] attr["default_value"] = default_value input_names = [item["name"] for item in inputs] @@ -286,7 +310,9 @@ def parse_api_entry(api_entry: Dict[str, Any], name_field="op"): if "optional" in api_entry: optional_args = parse_plain_list(api_entry["optional"]) for name in optional_args: - assert name in input_names, f"{api_name} has an optional input: '{name}' which is not an input." + assert ( + name in input_names + ), f"{api_name} has an optional input: '{name}' which is not an input." for input in inputs: if input["name"] in optional_args: input["optional"] = True @@ -297,7 +323,9 @@ def parse_api_entry(api_entry: Dict[str, Any], name_field="op"): if "intermediate" in api_entry: intermediate_outs = parse_plain_list(api_entry["intermediate"]) for name in intermediate_outs: - assert name in output_names, f"{api_name} has an intermediate output: '{name}' which is not an output." + assert ( + name in output_names + ), f"{api_name} has an intermediate output: '{name}' which is not an output." for output in outputs: if output["name"] in intermediate_outs: output["intermediate"] = True @@ -308,7 +336,9 @@ def parse_api_entry(api_entry: Dict[str, Any], name_field="op"): if "no_need_buffer" in api_entry: no_buffer_args = parse_plain_list(api_entry["no_need_buffer"]) for name in no_buffer_args: - assert name in input_names, f"{api_name} has an no buffer input: '{name}' which is not an input." + assert ( + name in input_names + ), f"{api_name} has an no buffer input: '{name}' which is not an input." for input in inputs: if input["name"] in no_buffer_args: input["no_need_buffer"] = True @@ -322,7 +352,7 @@ def parse_api_entry(api_entry: Dict[str, Any], name_field="op"): "inputs": inputs, "attrs": attrs, "outputs": outputs, - "no_need_buffer": no_buffer_args + "no_need_buffer": no_buffer_args, } # invokes another api? @@ -344,11 +374,13 @@ def parse_api_entry(api_entry: Dict[str, Any], name_field="op"): inplace_pairs = parse_inplace(api_name, api_entry["inplace"]) else: inplace_pairs = None - api.update({ - "infer_meta": infer_meta, - "kernel": kernel, - "inplace": inplace_pairs - }) + api.update( + { + "infer_meta": infer_meta, + "kernel": kernel, + "inplace": inplace_pairs, + } + ) else: # invoke invoke = parse_invoke(api_name, api_entry["invoke"]) @@ -367,8 +399,9 @@ def parse_api_entry(api_entry: Dict[str, Any], name_field="op"): if "forward" in api_entry: forward = parse_forward(api_name, api_entry["forward"]) # validate_fb - validate_backward_inputs(api_name, forward["inputs"], - forward["outputs"], inputs) + validate_backward_inputs( + api_name, forward["inputs"], forward["outputs"], inputs + ) validate_backward_attrs(api_name, forward["attrs"], attrs) validate_backward_outputs(api_name, forward["inputs"], outputs) else: @@ -384,23 +417,27 @@ def validate_backward_attrs(api, forward_attrs, backward_attrs): # this is a not-that-clean trick to allow backward api to has more attrs # than the forward api, as long as they all have default value for i in range(-num_exceptional_attrs, 0): - assert "default_value" in backward_attrs[ - i], f"{api} has exceptional attr without default value" + assert ( + "default_value" in backward_attrs[i] + ), f"{api} has exceptional attr without default value" -def validate_backward_inputs(api, forward_inputs, forward_outputs, - backward_inputs): +def validate_backward_inputs( + api, forward_inputs, forward_outputs, backward_inputs +): foward_input_names = [item["name"] for item in forward_inputs] forward_output_names = [item["name"] for item in forward_outputs] backward_input_names = [item["name"] for item in backward_inputs] assert len(backward_input_names) <= len(foward_input_names) + 2 * len( - forward_output_names), f"{api} has too many inputs." + forward_output_names + ), f"{api} has too many inputs." def validate_backward_outputs(api, forward_inputs, backward_outputs): assert len(backward_outputs) <= len( - forward_inputs), f"{api} has too many outputs" + forward_inputs + ), f"{api} has too many outputs" def cross_validate(apis): @@ -419,15 +456,17 @@ def cross_validate(apis): f"Something Wrong here, {name}'s forward api({fw_name}) does not claim {name} as its backward." ) else: - assert fw_api[ - "backward"] == name, f"{name}: backward and forward name mismatch" + assert ( + fw_api["backward"] == name + ), f"{name}: backward and forward name mismatch" assert len(fw_call["inputs"]) <= len( fw_api["inputs"] ), f"{name}: forward call has more inputs than the api" for (input, input_) in zip(fw_call["inputs"], fw_api["inputs"]): - assert input["typename"] == input_[ - "typename"], f"type mismatch in {name} and {fw_name}" + assert ( + input["typename"] == input_["typename"] + ), f"type mismatch in {name} and {fw_name}" assert len(fw_call["attrs"]) <= len( fw_api["attrs"] @@ -439,13 +478,16 @@ def cross_validate(apis): r"Scalar(\(\w+\))*", attr_["typename"] ), f"type mismatch in {name} and {fw_name}" else: - assert attr["typename"] == attr_[ - "typename"], f"type mismatch in {name} and {fw_name}" + assert ( + attr["typename"] == attr_["typename"] + ), f"type mismatch in {name} and {fw_name}" assert len(fw_call["outputs"]) == len( fw_api["outputs"] ), f"{name}: forward call has more outputs than the api" - for (output, output_) in zip(fw_call["outputs"], - fw_api["outputs"]): - assert output["typename"] == output_[ - "typename"], f"type mismatch in {name} and {fw_name}" + for (output, output_) in zip( + fw_call["outputs"], fw_api["outputs"] + ): + assert ( + output["typename"] == output_["typename"] + ), f"type mismatch in {name} and {fw_name}" diff --git a/paddle/phi/api/yaml/generator/sparse_api_gen.py b/paddle/phi/api/yaml/generator/sparse_api_gen.py index f013fed1b3ce1..15f4a4c30b32d 100644 --- a/paddle/phi/api/yaml/generator/sparse_api_gen.py +++ b/paddle/phi/api/yaml/generator/sparse_api_gen.py @@ -20,7 +20,6 @@ class SparseAPI(ForwardAPI): - def __init__(self, api_item_yaml): super(SparseAPI, self).__init__(api_item_yaml) @@ -30,11 +29,13 @@ def gene_api_declaration(self): {super(SparseAPI, self).gene_api_declaration()} """ - def gene_output(self, - out_dtype_list, - out_tensor_type_list=None, - code_indent='', - inplace_flag=False): + def gene_output( + self, + out_dtype_list, + out_tensor_type_list=None, + code_indent='', + inplace_flag=False, + ): kernel_output = [] output_names = [] output_create = "" @@ -42,15 +43,19 @@ def gene_output(self, output_type_map = { 'dense': 'TensorType::DENSE_TENSOR', 'sparse_coo': 'TensorType::SPARSE_COO', - 'sparse_csr': 'TensorType::SPARSE_CSR' + 'sparse_csr': 'TensorType::SPARSE_CSR', } if len(out_dtype_list) == 1: kernel_output.append('kernel_out') output_names.append('kernel_out') - inplace_assign = " = " + self.inplace_map[self.outputs['names'][ - 0]] if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][0] in self.inplace_map else "" + inplace_assign = ( + " = " + self.inplace_map[self.outputs['names'][0]] + if inplace_flag + and self.inplace_map is not None + and self.outputs['names'][0] in self.inplace_map + else "" + ) output_create = f""" {return_type} api_output{inplace_assign}; auto* kernel_out = SetSparseKernelOutput(&api_output, {output_type_map[out_dtype_list[0]]});""" @@ -65,8 +70,9 @@ def gene_output(self, for out_name in self.outputs['names']: if out_name in self.inplace_map: - output_create = output_create + self.inplace_map[ - out_name] + ', ' + output_create = ( + output_create + self.inplace_map[out_name] + ', ' + ) else: output_create += 'Tensor(), ' output_create = output_create[:-2] + '};' @@ -74,28 +80,30 @@ def gene_output(self, for i in range(len(out_dtype_list)): kernel_output.append(f'kernel_out_{i}') output_names.append(f'kernel_out_{i}') - output_create = output_create + f""" + output_create = ( + output_create + + f""" auto* kernel_out_{i} = SetSparseKernelOutput(&std::get<{i}>(api_output), {output_type_map[out_dtype_list[i]]});""" + ) else: raise ValueError( "{} : Output error: the output should not be empty.".format( - self.api)) + self.api + ) + ) return kernel_output, output_names, output_create def gen_sparse_kernel_context(self, kernel_output_names): input_trans_map = { - 'const Tensor&': - 'const phi::TenseBase&', - 'const std::vector&': - 'const std::vector&', - 'const paddle::optional&': - 'paddle::optional' + 'const Tensor&': 'const phi::TenseBase&', + 'const std::vector&': 'const std::vector&', + 'const paddle::optional&': 'paddle::optional', } out_trans_map = { 'Tensor': 'phi::TenseBase*', - 'std::vector': 'std::vector' + 'std::vector': 'std::vector', } input_names = self.inputs['names'] input_infos = self.inputs['input_info'] @@ -109,11 +117,17 @@ def gen_sparse_kernel_context(self, kernel_output_names): for param in kernel_param: if param in input_names: if param in self.optional_vars: - kernel_context_code = kernel_context_code + f""" + kernel_context_code = ( + kernel_context_code + + f""" kernel_context.EmplaceBackInput({param} ? {param}->impl().get() : nullptr);""" + ) else: - kernel_context_code = kernel_context_code + f""" + kernel_context_code = ( + kernel_context_code + + f""" kernel_context.EmplaceBackInput({param}.impl().get());""" + ) continue if param in attr_names: @@ -126,12 +140,18 @@ def gen_sparse_kernel_context(self, kernel_output_names): param = str(param).lower() else: param + str(param) + ", " - kernel_context_code = kernel_context_code + f""" + kernel_context_code = ( + kernel_context_code + + f""" kernel_context.EmplaceBackAttr({param});""" + ) for out_name in kernel_output_names: - kernel_context_code = kernel_context_code + f""" + kernel_context_code = ( + kernel_context_code + + f""" kernel_context.EmplaceBackOutput({out_name});""" + ) return kernel_context_code @@ -141,20 +161,25 @@ def prepare_input(self): attr_names = self.attrs['names'] infer_meta = self.infer_meta - infer_meta_params = infer_meta['param'] if infer_meta[ - 'param'] is not None else input_names + attr_names + infer_meta_params = ( + infer_meta['param'] + if infer_meta['param'] is not None + else input_names + attr_names + ) create_input_var_code = "" tensor_type_map = { 'dense': 'phi::DenseTensor', 'sparse_coo': 'phi::SparseCooTensor', - 'sparse_csr': 'phi::SparseCsrTensor' + 'sparse_csr': 'phi::SparseCsrTensor', } for param in infer_meta_params: if param in input_names: var_name = "auto " + PREFIX_TENSOR_NAME + param + " = " if self.inputs['input_info'][param] == "const Tensor&": - create_input_var_code = create_input_var_code + var_name + param + ".impl();\n" + create_input_var_code = ( + create_input_var_code + var_name + param + ".impl();\n" + ) elif param in self.optional_vars: tensor_type = 'phi::DenseTensor' for name, input_type in zip(input_names, input_types): @@ -162,17 +187,35 @@ def prepare_input(self): tensor_type = tensor_type_map[input_type] break optional_var = "paddle::optional<" + tensor_type + ">(" - create_input_var_code = create_input_var_code + var_name + param + " ? " + optional_var + "*static_cast<" + tensor_type + "*>((*" + param + ").impl().get())) : " + optional_var + "paddle::none);\n" + create_input_var_code = ( + create_input_var_code + + var_name + + param + + " ? " + + optional_var + + "*static_cast<" + + tensor_type + + "*>((*" + + param + + ").impl().get())) : " + + optional_var + + "paddle::none);\n" + ) return f"""{create_input_var_code}""" def gen_sparse_kernel_code(self, kernel_name, inplace_flag=False): _, kernel_output_names, output_create = self.gene_output( - self.kernel['dispatch'][kernel_name][1], None, '', inplace_flag) + self.kernel['dispatch'][kernel_name][1], None, '', inplace_flag + ) kernel_context_code = self.gen_sparse_kernel_context( - kernel_output_names) - return_code = "" if len( - self.gene_return_code()) == 0 else " " + self.gene_return_code() + kernel_output_names + ) + return_code = ( + "" + if len(self.gene_return_code()) == 0 + else " " + self.gene_return_code() + ) return f""" VLOG(6) << "{self.api} api sparse kernel key: [" << kernel_backend << ", " << kernel_layout << ", "<< kernel_data_type << "]"; auto kernel_result = phi::KernelFactory::Instance().SelectKernelOrThrowError( @@ -190,12 +233,13 @@ def gen_sparse_kernel_code(self, kernel_name, inplace_flag=False): {return_code}""" def get_condition_code(self, kernel_name): - assert self.kernel['dispatch'][kernel_name], \ - f"{self.api} api: the tensor type of inputs and outputs for kernel isn't set, see also 'kernel:func' of 'conv3d' in sparse_ops.yaml." + assert self.kernel['dispatch'][ + kernel_name + ], f"{self.api} api: the tensor type of inputs and outputs for kernel isn't set, see also 'kernel:func' of 'conv3d' in sparse_ops.yaml." input_types = self.kernel['dispatch'][kernel_name][0] sparse_type_map = { 'sparse_coo': 'DataLayout::SPARSE_COO', - 'sparse_csr': 'DataLayout::SPARSE_CSR' + 'sparse_csr': 'DataLayout::SPARSE_CSR', } condition_list = [] tensor_type_list = [] @@ -212,10 +256,12 @@ def get_condition_code(self, kernel_name): else: if in_type == 'sparse_coo': condition_list.append( - f"{self.inputs['names'][i]}.is_sparse_coo_tensor()") + f"{self.inputs['names'][i]}.is_sparse_coo_tensor()" + ) else: condition_list.append( - f"{self.inputs['names'][i]}.is_sparse_csr_tensor()") + f"{self.inputs['names'][i]}.is_sparse_csr_tensor()" + ) tensor_type_list.append(in_type) self.inputs['tensor_type'] = tensor_type_list @@ -235,7 +281,8 @@ def gene_base_api_code(self, inplace_flag=False): kernel_dispatch_code = f"{self.gene_kernel_select()}\n" for kernel_name in self.kernel['func']: kernel_dispatch_code += self.gene_dispatch_code( - kernel_name, inplace_flag) + kernel_name, inplace_flag + ) return f""" PADDLE_API {self.get_return_type(inplace_flag)} {api_func_name}({self.get_define_args(inplace_flag)}) {{ @@ -281,17 +328,20 @@ def source_include(header_file_path): def api_namespace(): - return (""" + return ( + """ namespace paddle { namespace experimental { namespace sparse { -""", """ +""", + """ } // namespace sparse } // namespace experimental } // namespace paddle -""") +""", + ) def generate_api(api_yaml_path, header_file_path, source_file_path): @@ -327,18 +377,25 @@ def generate_api(api_yaml_path, header_file_path, source_file_path): def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle C++ Sparse API files') - parser.add_argument('--api_yaml_path', - help='path to sparse api yaml file', - default='paddle/phi/api/yaml/sparse_ops.yaml') - - parser.add_argument('--api_header_path', - help='output of generated api header code file', - default='paddle/phi/api/include/sparse_api.h') - - parser.add_argument('--api_source_path', - help='output of generated api source code file', - default='paddle/phi/api/lib/sparse_api.cc') + description='Generate PaddlePaddle C++ Sparse API files' + ) + parser.add_argument( + '--api_yaml_path', + help='path to sparse api yaml file', + default='paddle/phi/api/yaml/sparse_ops.yaml', + ) + + parser.add_argument( + '--api_header_path', + help='output of generated api header code file', + default='paddle/phi/api/include/sparse_api.h', + ) + + parser.add_argument( + '--api_source_path', + help='output of generated api source code file', + default='paddle/phi/api/lib/sparse_api.cc', + ) options = parser.parse_args() diff --git a/paddle/phi/api/yaml/generator/sparse_bw_api_gen.py b/paddle/phi/api/yaml/generator/sparse_bw_api_gen.py index 819e217c65daf..e17a4f34de095 100644 --- a/paddle/phi/api/yaml/generator/sparse_bw_api_gen.py +++ b/paddle/phi/api/yaml/generator/sparse_bw_api_gen.py @@ -20,7 +20,6 @@ class SparseBackwardAPI(SparseAPI, BackwardAPI): - def __init__(self, bw_api_item_yaml): BackwardAPI.__init__(self, bw_api_item_yaml) @@ -45,26 +44,32 @@ def get_declare_args(self, inplace_flag=False): def get_define_args(self, inplace_flag=False): return BackwardAPI.get_define_args(self) - def gene_output(self, - out_dtype_list, - out_tensor_type_list=None, - code_indent='', - inplace_flag=False): + def gene_output( + self, + out_dtype_list, + out_tensor_type_list=None, + code_indent='', + inplace_flag=False, + ): kernel_output = [] output_names = [] output_create = "" output_type_map = { 'dense': 'TensorType::DENSE_TENSOR', 'sparse_coo': 'TensorType::SPARSE_COO', - 'sparse_csr': 'TensorType::SPARSE_CSR' + 'sparse_csr': 'TensorType::SPARSE_CSR', } if len(out_dtype_list) == 1: kernel_output.append('kernel_out') output_names.append('kernel_out') - inplace_assign = " = " + self.inplace_map[self.outputs['names'][ - 0]] if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][0] in self.inplace_map else "" + inplace_assign = ( + " = " + self.inplace_map[self.outputs['names'][0]] + if inplace_flag + and self.inplace_map is not None + and self.outputs['names'][0] in self.inplace_map + else "" + ) output_create = f""" auto kernel_out = SetSparseKernelOutput({self.outputs['names'][0]}, {output_type_map[out_dtype_list[0]]});""" @@ -74,18 +79,29 @@ def gene_output(self, for i, out_type_item in enumerate(out_dtype_list): kernel_output.append(f'kernel_out_{i}') output_names.append(f'kernel_out_{i}') - if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][i] in self.inplace_map: - output_create = output_create + f""" + if ( + inplace_flag + and self.inplace_map is not None + and self.outputs['names'][i] in self.inplace_map + ): + output_create = ( + output_create + + f""" *{self.outputs['names'][i]} = {self.inplace_map[self.outputs['names'][i]]};""" + ) - output_create = output_create + f""" + output_create = ( + output_create + + f""" auto kernel_out_{i} = SetSparseKernelOutput({self.outputs['names'][i]}, {output_type_map[out_dtype_list[i]]});""" + ) else: raise ValueError( "{} : Output error: the output should not be empty.".format( - self.api)) + self.api + ) + ) return kernel_output, output_names, output_create @@ -122,17 +138,20 @@ def source_include(header_file_path): def api_namespace(): - return (""" + return ( + """ namespace paddle { namespace experimental { namespace sparse { -""", """ +""", + """ } // namespace sparse } // namespace experimental } // namespace paddle -""") +""", + ) def generate_api(api_yaml_path, header_file_path, source_file_path): @@ -166,18 +185,25 @@ def generate_api(api_yaml_path, header_file_path, source_file_path): def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle C++ Sparse API files') - parser.add_argument('--api_yaml_path', - help='path to sparse api yaml file', - default='paddle/phi/api/yaml/sparse_backward.yaml') - - parser.add_argument('--api_header_path', - help='output of generated api header code file', - default='paddle/phi/api/backward/sparse_bw_api.h') - - parser.add_argument('--api_source_path', - help='output of generated api source code file', - default='paddle/phi/api/lib/sparse_bw_api.cc') + description='Generate PaddlePaddle C++ Sparse API files' + ) + parser.add_argument( + '--api_yaml_path', + help='path to sparse api yaml file', + default='paddle/phi/api/yaml/sparse_backward.yaml', + ) + + parser.add_argument( + '--api_header_path', + help='output of generated api header code file', + default='paddle/phi/api/backward/sparse_bw_api.h', + ) + + parser.add_argument( + '--api_source_path', + help='output of generated api source code file', + default='paddle/phi/api/lib/sparse_bw_api.cc', + ) options = parser.parse_args() diff --git a/paddle/phi/api/yaml/generator/strings_api_gen.py b/paddle/phi/api/yaml/generator/strings_api_gen.py index b34075b681b27..907f126d949af 100644 --- a/paddle/phi/api/yaml/generator/strings_api_gen.py +++ b/paddle/phi/api/yaml/generator/strings_api_gen.py @@ -22,7 +22,6 @@ class StringsAPI(ForwardAPI): - def __init__(self, api_item_yaml): super(StringsAPI, self).__init__(api_item_yaml) @@ -48,11 +47,13 @@ def get_tensor_type(self, kernel_tensor_out_type): } return tensor_type_dict[kernel_tensor_out_type] - def gene_output(self, - out_dtype_list, - out_tensor_type_list=None, - code_indent='', - inplace_flag=False): + def gene_output( + self, + out_dtype_list, + out_tensor_type_list=None, + code_indent='', + inplace_flag=False, + ): kernel_output = [] output_names = [] output_create = "" @@ -62,11 +63,16 @@ def gene_output(self, kernel_output.append('kernel_out') output_names.append('kernel_out') kernel_tensor_out_type = self.get_kernel_tensor_out_type( - self.outputs['names'][0]) + self.outputs['names'][0] + ) tensor_type = self.get_tensor_type(kernel_tensor_out_type) - inplace_assign = " = " + self.inplace_map[self.outputs['names'][ - 0]] if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][0] in self.inplace_map else "" + inplace_assign = ( + " = " + self.inplace_map[self.outputs['names'][0]] + if inplace_flag + and self.inplace_map is not None + and self.outputs['names'][0] in self.inplace_map + else "" + ) output_create = f""" {return_type} api_output{inplace_assign}; {tensor_type}* kernel_out = dynamic_cast<{tensor_type}*>(SetStringsKernelOutput(&api_output, {kernel_tensor_out_type}));""" @@ -79,37 +85,45 @@ def gene_output(self, kernel_output.append(f'kernel_out_{i}') output_names.append(f'kernel_out_{i}') kernel_tensor_out_type = self.get_kernel_tensor_out_type( - self.outputs['names'][i]) + self.outputs['names'][i] + ) tensor_type = self.get_tensor_type(kernel_tensor_out_type) - if inplace_flag and self.inplace_map is not None and self.outputs[ - 'names'][i] in self.inplace_map: - output_create = output_create + f""" + if ( + inplace_flag + and self.inplace_map is not None + and self.outputs['names'][i] in self.inplace_map + ): + output_create = ( + output_create + + f""" std::get<{i}>(api_output) = {self.inplace_map[self.outputs['names'][i]]};""" + ) - output_create = output_create + f""" + output_create = ( + output_create + + f""" {tensor_type}* kernel_out_{i} = dynamic_cast<{tensor_type}*>(SetStringsKernelOutput(&std::get<{i}>(api_output), {kernel_tensor_out_type}));""" + ) else: raise ValueError( "{} : Output error: the output should not be empty.".format( - self.api)) + self.api + ) + ) return kernel_output, output_names, output_create def get_kernel_args(self, code_indent): input_trans_map = { - 'const Tensor&': - 'const phi::StringTensor&', - 'const std::vector&': - 'const std::vector&', - 'const paddle::optional&': - 'paddle::optional', - 'const paddle::optional>&': - 'paddle::optional&>' + 'const Tensor&': 'const phi::StringTensor&', + 'const std::vector&': 'const std::vector&', + 'const paddle::optional&': 'paddle::optional', + 'const paddle::optional>&': 'paddle::optional&>', } out_trans_map = { 'Tensor': 'phi::StringTensor*', - 'std::vector': 'std::vector&' + 'std::vector': 'std::vector&', } input_names = self.inputs['names'] input_infos = self.inputs['input_info'] @@ -122,21 +136,36 @@ def get_kernel_args(self, code_indent): input_tensor_code = "" # set input_tensor_code for i, input_name in enumerate(input_names): - input_tensor_code = input_tensor_code + f""" + input_tensor_code = ( + input_tensor_code + + f""" {code_indent} auto {PREFIX_TENSOR_NAME}{input_name} = TensorToStringTensor({input_name});""" + ) # set kernel_args kernel_args = "*dev_ctx, " for param in kernel_param: if param in input_names: if param in self.optional_vars: - kernel_args = kernel_args + PREFIX_TENSOR_NAME + param + ", " + kernel_args = ( + kernel_args + PREFIX_TENSOR_NAME + param + ", " + ) else: if self.inputs['input_info'][param] == "const Tensor&": - kernel_args = kernel_args + "*" + PREFIX_TENSOR_NAME + param + ", " - elif self.inputs['input_info'][ - input_name] == "const std::vector&": - kernel_args = kernel_args + PREFIX_TENSOR_NAME + param + ", " + kernel_args = ( + kernel_args + + "*" + + PREFIX_TENSOR_NAME + + param + + ", " + ) + elif ( + self.inputs['input_info'][input_name] + == "const std::vector&" + ): + kernel_args = ( + kernel_args + PREFIX_TENSOR_NAME + param + ", " + ) else: # do nothing pass @@ -152,7 +181,8 @@ def get_kernel_args(self, code_indent): param = 'phi::Scalar(' + param + ')' else: kernel_args_type_list.append( - self.attrs['attr_info'][param][0]) + self.attrs['attr_info'][param][0] + ) kernel_args = kernel_args + param + ", " elif isinstance(param, bool): kernel_args = kernel_args + str(param).lower() + ", " @@ -169,9 +199,11 @@ def get_kernel_args(self, code_indent): def gen_string_tensor_kernel_code(self, inplace_flag=False, code_indent=""): input_tensors, kernel_args, kernel_signature = self.get_kernel_args( - code_indent) + code_indent + ) outputs_args, kernel_output_names, output_create = self.gene_output( - self.outputs['types'], None, '', inplace_flag) + self.outputs['types'], None, '', inplace_flag + ) return f""" // 1. Get kernel signature and kernel @@ -214,8 +246,9 @@ def gene_kernel_select(self) -> str: attr_data_type_count = 0 for attr_name in attrs['names']: if attrs['attr_info'][attr_name][0] == 'Backend': - assert kernel['backend'] is not None, \ - f"{api} api: When there is a parameter with 'Backend' type in attributes, you must set backend of kernel manually." + assert ( + kernel['backend'] is not None + ), f"{api} api: When there is a parameter with 'Backend' type in attributes, you must set backend of kernel manually." attr_backend_count = attr_backend_count + 1 # preprocess kernel configures @@ -223,22 +256,30 @@ def gene_kernel_select(self) -> str: if kernel['backend'] is not None: if '>' in kernel['backend']: vars_list = kernel['backend'].split('>') - assert len( - vars_list - ) == 2, f"{api} api: The number of params to set backend with '>' only allows 2, but received {len(vars_list)}." - assert (vars_list[0].strip() in attrs['names']) and (attrs['attr_info'][vars_list[0].strip()][0] == 'const Place&'), \ - f"{api} api: When use '>' to set kernel backend, the first param should be a attribute with Place type." - kernel_select_code = kernel_select_code + f""" + assert ( + len(vars_list) == 2 + ), f"{api} api: The number of params to set backend with '>' only allows 2, but received {len(vars_list)}." + assert (vars_list[0].strip() in attrs['names']) and ( + attrs['attr_info'][vars_list[0].strip()][0] + == 'const Place&' + ), f"{api} api: When use '>' to set kernel backend, the first param should be a attribute with Place type." + kernel_select_code = ( + kernel_select_code + + f""" kernel_backend = ParseBackendWithInputOrder({vars_list[0].strip()}, {vars_list[1].strip()}); """ + ) else: args_str = "" for ele in kernel['backend'].split(','): args_str = args_str + ele.strip() + ', ' - kernel_select_code = kernel_select_code + f""" + kernel_select_code = ( + kernel_select_code + + f""" kernel_backend = ParseBackend({args_str[:-2]}); """ + ) kernel_select_args = "" for input_name in input_names: @@ -250,10 +291,13 @@ def gene_kernel_select(self) -> str: kernel_select_code = kernel_key_item_init + kernel_select_code if len(input_names) > 0: - kernel_select_code = kernel_select_code + f""" + kernel_select_code = ( + kernel_select_code + + f""" auto kernel_key_set = ParseKernelKeyByInputArgs({kernel_select_args}); auto kernel_key = kernel_key_set.GetHighestPriorityKernelKey(); kernel_backend = kernel_key.backend();""" + ) return kernel_select_code @@ -300,17 +344,20 @@ def api_register(): def api_namespace(): - return (""" + return ( + """ namespace paddle { namespace experimental { namespace strings { -""", """ +""", + """ } // namespace strings } // namespace experimental } // namespace paddle -""") +""", + ) def generate_api(api_yaml_path, header_file_path, source_file_path): @@ -346,18 +393,25 @@ def generate_api(api_yaml_path, header_file_path, source_file_path): def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle C++ Strings API files') - parser.add_argument('--api_yaml_path', - help='path to sparse api yaml file', - default='paddle/phi/api/yaml/strings_ops.yaml') - - parser.add_argument('--api_header_path', - help='output of generated api header code file', - default='paddle/phi/api/include/strings_api.h') - - parser.add_argument('--api_source_path', - help='output of generated api source code file', - default='paddle/phi/api/lib/strings_api.cc') + description='Generate PaddlePaddle C++ Strings API files' + ) + parser.add_argument( + '--api_yaml_path', + help='path to sparse api yaml file', + default='paddle/phi/api/yaml/strings_ops.yaml', + ) + + parser.add_argument( + '--api_header_path', + help='output of generated api header code file', + default='paddle/phi/api/include/strings_api.h', + ) + + parser.add_argument( + '--api_source_path', + help='output of generated api source code file', + default='paddle/phi/api/lib/strings_api.cc', + ) options = parser.parse_args() diff --git a/paddle/phi/api/yaml/generator/type_mapping.py b/paddle/phi/api/yaml/generator/type_mapping.py index 9e65797660229..8aec1bcc49a5e 100644 --- a/paddle/phi/api/yaml/generator/type_mapping.py +++ b/paddle/phi/api/yaml/generator/type_mapping.py @@ -15,7 +15,7 @@ # type mapping: types in yaml -> types in c++ API input_types_map = { 'Tensor': 'const Tensor&', - 'Tensor[]': 'const std::vector&' + 'Tensor[]': 'const std::vector&', } optional_input_types_map = { @@ -81,15 +81,17 @@ output_type_map = {'Tensor': 'Tensor', 'Tensor[]': 'std::vector'} -#------------------------------ phi attr ------------------------------ +# ------------------------------ phi attr ------------------------------ phi_attr_types_map = attr_types_map.copy() -phi_attr_types_map.update({ - 'IntArray': 'const phi::IntArray&', - 'Scalar': 'const phi::Scalar&', - 'Scalar[]': 'std::vector&' -}) +phi_attr_types_map.update( + { + 'IntArray': 'const phi::IntArray&', + 'Scalar': 'const phi::Scalar&', + 'Scalar[]': 'std::vector&', + } +) -#--------------------------- phi dense tensor --------------------------- +# --------------------------- phi dense tensor --------------------------- # type mapping to phi, used in implementation dense_input_types_map = { 'Tensor': 'const phi::DenseTensor&', @@ -98,15 +100,15 @@ dense_optional_input_types_map = { 'Tensor': 'paddle::optional', - 'Tensor[]': 'paddle::optional&>' + 'Tensor[]': 'paddle::optional&>', } dense_output_types_map = { 'Tensor': 'phi::DenseTensor*', - 'Tensor[]': 'std::vector' + 'Tensor[]': 'std::vector', } -#---------------------- phi selected rows------------------------------ +# ---------------------- phi selected rows------------------------------ # type mapping to phi, used in implementation sr_input_types_map = { 'Tensor': 'const phi::SelectedRows&', diff --git a/paddle/phi/api/yaml/generator/wrapped_infermeta_gen.py b/paddle/phi/api/yaml/generator/wrapped_infermeta_gen.py index 5a120ad08b3ec..c14259349ce60 100644 --- a/paddle/phi/api/yaml/generator/wrapped_infermeta_gen.py +++ b/paddle/phi/api/yaml/generator/wrapped_infermeta_gen.py @@ -39,33 +39,38 @@ def gene_wrapped_infermeta_and_register(api): if kernel_params == api.infer_meta['param']: return '', '', register_code - assert len(api.infer_meta['param']) <= len(kernel_params), \ - f"{api.api} api: Parameters error. The params of infer_meta should be a subset of kernel params." + assert len(api.infer_meta['param']) <= len( + kernel_params + ), f"{api.api} api: Parameters error. The params of infer_meta should be a subset of kernel params." tensor_type_map = { 'const Tensor&': 'const MetaTensor&', - 'const std::vector&': - 'const std::vector&', + 'const std::vector&': 'const std::vector&', 'Tensor': 'MetaTensor*', 'std::vector': 'std::vector', - 'const paddle::optional&': 'const MetaTensor&' + 'const paddle::optional&': 'const MetaTensor&', } wrapped_infermeta_name = get_wrapped_infermeta_name( - api.kernel['func'][0]) + api.kernel['func'][0] + ) args = [] for input_name in api.inputs['names']: if input_name in kernel_params: args.append( - tensor_type_map[api.inputs['input_info'][input_name]] + - ' ' + input_name) + tensor_type_map[api.inputs['input_info'][input_name]] + + ' ' + + input_name + ) for attr_name in api.attrs['names']: if attr_name in kernel_params: - args.append(api.attrs['attr_info'][attr_name][0] + ' ' + - attr_name) + args.append( + api.attrs['attr_info'][attr_name][0] + ' ' + attr_name + ) for i, out_type in enumerate(api.outputs['types']): - args.append(tensor_type_map[out_type] + ' ' + - api.outputs['names'][i]) + args.append( + tensor_type_map[out_type] + ' ' + api.outputs['names'][i] + ) invoke_param = api.infer_meta['param'] invoke_param.extend(api.outputs['names']) @@ -112,15 +117,19 @@ def source_include(header_file_path): def api_namespace(): - return (""" + return ( + """ namespace phi { -""", """ +""", + """ } // namespace phi -""") +""", + ) -def generate_wrapped_infermeta_and_register(api_yaml_path, header_file_path, - source_file_path): +def generate_wrapped_infermeta_and_register( + api_yaml_path, header_file_path, source_file_path +): apis = [] for each_api_yaml in api_yaml_path: with open(each_api_yaml, 'r') as f: @@ -145,8 +154,11 @@ def generate_wrapped_infermeta_and_register(api_yaml_path, header_file_path, for api in apis: api_item = ForwardAPI(api) - declare_code, defind_code, register_code = gene_wrapped_infermeta_and_register( - api_item) + ( + declare_code, + defind_code, + register_code, + ) = gene_wrapped_infermeta_and_register(api_item) header_file.write(declare_code) source_file.write(defind_code) if infermeta_register_code.find(register_code) == -1: @@ -163,20 +175,25 @@ def generate_wrapped_infermeta_and_register(api_yaml_path, header_file_path, def main(): parser = argparse.ArgumentParser( - description='Generate PaddlePaddle C++ API files') - parser.add_argument('--api_yaml_path', - help='path to api yaml file', - nargs='+', - default='paddle/phi/api/yaml/ops.yaml') + description='Generate PaddlePaddle C++ API files' + ) + parser.add_argument( + '--api_yaml_path', + help='path to api yaml file', + nargs='+', + default='paddle/phi/api/yaml/ops.yaml', + ) parser.add_argument( '--wrapped_infermeta_header_path', help='output of generated wrapped_infermeta header code file', - default='paddle/phi/infermeta/generated.h') + default='paddle/phi/infermeta/generated.h', + ) parser.add_argument( '--wrapped_infermeta_source_path', help='output of generated wrapped_infermeta source code file', - default='paddle/phi/infermeta/generated.cc') + default='paddle/phi/infermeta/generated.cc', + ) options = parser.parse_args() @@ -184,8 +201,9 @@ def main(): header_file_path = options.wrapped_infermeta_header_path source_file_path = options.wrapped_infermeta_source_path - generate_wrapped_infermeta_and_register(api_yaml_path, header_file_path, - source_file_path) + generate_wrapped_infermeta_and_register( + api_yaml_path, header_file_path, source_file_path + ) if __name__ == '__main__': diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index 9671520edd27e..abe3184ee9185 100755 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -17,9 +17,11 @@ from paddle.cuda_env import * # noqa: F403 except ImportError: import sys - sys.stderr.write('''Warning with import paddle: you should not + + sys.stderr.write( + '''Warning with import paddle: you should not import paddle from the source directory; please install paddlepaddle*.whl firstly.''' - ) + ) from .batch import batch # noqa: F401 from .framework import monkey_patch_variable @@ -52,6 +54,7 @@ from .framework.dtype import bool # noqa: F401 from .framework.dtype import complex64 # noqa: F401 from .framework.dtype import complex128 # noqa: F401 + if fluid.framework._in_eager_mode_: Tensor = framework.core.eager.Tensor else: @@ -112,8 +115,8 @@ from .tensor.creation import assign # noqa: F401 from .tensor.creation import complex # noqa: F401 from .tensor.creation import clone # noqa: F401 -from .tensor.creation import tril_indices #noqa: F401 -from .tensor.creation import triu_indices #noqa: F401 +from .tensor.creation import tril_indices # noqa: F401 +from .tensor.creation import triu_indices # noqa: F401 from .tensor.linalg import matmul # noqa: F401 from .tensor.linalg import dot # noqa: F401 from .tensor.linalg import norm # noqa: F401 @@ -382,6 +385,7 @@ # CINN has to set a flag to include a lib if is_compiled_with_cinn(): import os + package_dir = os.path.dirname(os.path.abspath(__file__)) runtime_include_dir = os.path.join(package_dir, "libs") cuh_file = os.path.join(runtime_include_dir, "cinn_cuda_runtime_source.cuh") diff --git a/python/paddle/amp/auto_cast.py b/python/paddle/amp/auto_cast.py index ffb1648634253..930e97f62f4d0 100644 --- a/python/paddle/amp/auto_cast.py +++ b/python/paddle/amp/auto_cast.py @@ -18,11 +18,13 @@ __all__ = [] -def auto_cast(enable=True, - custom_white_list=None, - custom_black_list=None, - level='O1', - dtype='float16'): +def auto_cast( + enable=True, + custom_white_list=None, + custom_black_list=None, + level='O1', + dtype='float16', +): """ Create a context which enables auto-mixed-precision(AMP) of operators executed in dynamic graph mode. If enabled, the input data type (float32 or float16) of each operator is decided @@ -78,12 +80,14 @@ def auto_cast(enable=True, return amp_guard(enable, custom_white_list, custom_black_list, level, dtype) -def decorate(models, - optimizers=None, - level='O1', - dtype='float16', - master_weight=None, - save_dtype=None): +def decorate( + models, + optimizers=None, + level='O1', + dtype='float16', + master_weight=None, + save_dtype=None, +): """ Decorate models and optimizers for auto-mixed-precision. When level is O1(amp), the decorate will do nothing. When level is O2(pure float16/bfloat16), the decorate will cast all parameters of models to float16/bfloat16, except BatchNorm and LayerNorm. @@ -147,5 +151,6 @@ def decorate(models, output = model(data) print(output.dtype) # FP16 """ - return amp_decorate(models, optimizers, level, dtype, master_weight, - save_dtype) + return amp_decorate( + models, optimizers, level, dtype, master_weight, save_dtype + ) diff --git a/python/paddle/amp/grad_scaler.py b/python/paddle/amp/grad_scaler.py index 9fe7f71d2be01..7fe0db4d04aad 100644 --- a/python/paddle/amp/grad_scaler.py +++ b/python/paddle/amp/grad_scaler.py @@ -75,18 +75,25 @@ class GradScaler(AmpScaler): optimizer.clear_grad() """ - def __init__(self, - enable=True, - init_loss_scaling=2.**15, - incr_ratio=2.0, - decr_ratio=0.5, - incr_every_n_steps=1000, - decr_every_n_nan_or_inf=2, - use_dynamic_loss_scaling=True): - super(GradScaler, - self).__init__(enable, init_loss_scaling, incr_ratio, decr_ratio, - incr_every_n_steps, decr_every_n_nan_or_inf, - use_dynamic_loss_scaling) + def __init__( + self, + enable=True, + init_loss_scaling=2.0**15, + incr_ratio=2.0, + decr_ratio=0.5, + incr_every_n_steps=1000, + decr_every_n_nan_or_inf=2, + use_dynamic_loss_scaling=True, + ): + super(GradScaler, self).__init__( + enable, + init_loss_scaling, + incr_ratio, + decr_ratio, + incr_every_n_steps, + decr_every_n_nan_or_inf, + use_dynamic_loss_scaling, + ) def scale(self, var): """ @@ -192,7 +199,8 @@ def step(self, optimizer): optimizer_state = self._optimizer_states[id(optimizer)] if optimizer_state["state"] is OptimizerState.STEPPED: raise RuntimeError( - "step() has already been called since the last update().") + "step() has already been called since the last update()." + ) # unscale the grad if optimizer_state["state"] is OptimizerState.INIT: @@ -570,8 +578,9 @@ def set_decr_every_n_nan_or_inf(self, new_decr_every_n_nan_or_inf): scaler.set_decr_every_n_nan_or_inf(new_decr_every_n_nan_or_inf) print(scaler.get_decr_every_n_nan_or_inf()) # 3 """ - super(GradScaler, - self).set_decr_every_n_nan_or_inf(new_decr_every_n_nan_or_inf) + super(GradScaler, self).set_decr_every_n_nan_or_inf( + new_decr_every_n_nan_or_inf + ) def state_dict(self): """ diff --git a/python/paddle/audio/__init__.py b/python/paddle/audio/__init__.py index ee768ab6d029c..3b6e25e2b6c75 100644 --- a/python/paddle/audio/__init__.py +++ b/python/paddle/audio/__init__.py @@ -20,5 +20,11 @@ from .backends.backend import info, load, save __all__ = [ - "functional", "features", "datasets", "backends", "load", "info", "save" + "functional", + "features", + "datasets", + "backends", + "load", + "info", + "save", ] diff --git a/python/paddle/audio/backends/backend.py b/python/paddle/audio/backends/backend.py index fbfd11d20e0b5..d092968a68ecf 100644 --- a/python/paddle/audio/backends/backend.py +++ b/python/paddle/audio/backends/backend.py @@ -19,10 +19,16 @@ class AudioInfo: - """ Audio info, return type of backend info function """ - - def __init__(self, sample_rate: int, num_samples: int, num_channels: int, - bits_per_sample: int, encoding: str): + """Audio info, return type of backend info function""" + + def __init__( + self, + sample_rate: int, + num_samples: int, + num_channels: int, + bits_per_sample: int, + encoding: str, + ): self.sample_rate = sample_rate self.num_samples = num_samples self.num_channels = num_channels @@ -61,11 +67,13 @@ def info(filepath: str) -> AudioInfo: raise NotImplementedError("please set audio backend") -def load(filepath: Union[str, Path], - frame_offset: int = 0, - num_frames: int = -1, - normalize: bool = True, - channels_first: bool = True) -> Tuple[paddle.Tensor, int]: +def load( + filepath: Union[str, Path], + frame_offset: int = 0, + num_frames: int = -1, + normalize: bool = True, + channels_first: bool = True, +) -> Tuple[paddle.Tensor, int]: """Load audio data from file.Load the audio content start form frame_offset, and get num_frames. Args: diff --git a/python/paddle/audio/backends/init_backend.py b/python/paddle/audio/backends/init_backend.py index a066e4e23a64e..3ca77ba316f98 100644 --- a/python/paddle/audio/backends/init_backend.py +++ b/python/paddle/audio/backends/init_backend.py @@ -35,7 +35,7 @@ def _check_version(version: str) -> bool: def list_available_backends() -> List[str]: - """ List available backends, the backends in paddleaudio and the default backend. + """List available backends, the backends in paddleaudio and the default backend. Returns: List[str]: The list of available backends. @@ -82,7 +82,8 @@ def list_available_backends() -> List[str]: if _check_version(version) == False: err_msg = ( "the version of paddleaudio installed is {},\n" - "please ensure the paddleaudio >= 1.0.2.").format(version) + "please ensure the paddleaudio >= 1.0.2." + ).format(version) raise ImportError(err_msg) backends = paddleaudio.backends.list_audio_backends() backends.append("wave_backend") @@ -90,7 +91,7 @@ def list_available_backends() -> List[str]: def get_current_backend() -> str: - """ Get the name of the current audio backend + """Get the name of the current audio backend Returns: str: The name of the current backend, @@ -124,6 +125,7 @@ def get_current_backend() -> str: current_backend = None if "paddleaudio" in sys.modules: import paddleaudio + current_backend = paddleaudio.backends.get_audio_backend() if paddle.audio.load == paddleaudio.load: return current_backend @@ -171,6 +173,7 @@ def set_backend(backend_name: str): module = wave_backend else: import paddleaudio + paddleaudio.backends.set_audio_backend(backend_name) module = paddleaudio diff --git a/python/paddle/audio/backends/wave_backend.py b/python/paddle/audio/backends/wave_backend.py index 66f2d48fe19a5..4be5592099d97 100644 --- a/python/paddle/audio/backends/wave_backend.py +++ b/python/paddle/audio/backends/wave_backend.py @@ -80,15 +80,18 @@ def info(filepath: str) -> AudioInfo: bits_per_sample = file_.getsampwidth() * 8 encoding = "PCM_S" # default WAV encoding, only support file_obj.close() - return AudioInfo(sample_rate, sample_frames, channels, bits_per_sample, - encoding) + return AudioInfo( + sample_rate, sample_frames, channels, bits_per_sample, encoding + ) -def load(filepath: Union[str, Path], - frame_offset: int = 0, - num_frames: int = -1, - normalize: bool = True, - channels_first: bool = True) -> Tuple[paddle.Tensor, int]: +def load( + filepath: Union[str, Path], + frame_offset: int = 0, + num_frames: int = -1, + normalize: bool = True, + channels_first: bool = True, +) -> Tuple[paddle.Tensor, int]: """Load audio data from file. load the audio content start form frame_offset, and get num_frames. Args: @@ -154,7 +157,7 @@ def load(filepath: Union[str, Path], waveform = np.reshape(audio_norm, (frames, channels)) if num_frames != -1: - waveform = waveform[frame_offset:frame_offset + num_frames, :] + waveform = waveform[frame_offset : frame_offset + num_frames, :] waveform = paddle.to_tensor(waveform) if channels_first: waveform = paddle.transpose(waveform, perm=[1, 0]) diff --git a/python/paddle/audio/datasets/dataset.py b/python/paddle/audio/datasets/dataset.py index 67fda01f3fde9..27e84cbe9c7b2 100644 --- a/python/paddle/audio/datasets/dataset.py +++ b/python/paddle/audio/datasets/dataset.py @@ -25,7 +25,7 @@ 'melspectrogram': MelSpectrogram, 'mfcc': MFCC, 'logmelspectrogram': LogMelSpectrogram, - 'spectrogram': Spectrogram + 'spectrogram': Spectrogram, } @@ -34,12 +34,14 @@ class AudioClassificationDataset(paddle.io.Dataset): Base class of audio classification dataset. """ - def __init__(self, - files: List[str], - labels: List[int], - feat_type: str = 'raw', - sample_rate: int = None, - **kwargs): + def __init__( + self, + files: List[str], + labels: List[int], + feat_type: str = 'raw', + sample_rate: int = None, + **kwargs, + ): """ Ags: files (:obj:`List[str]`): A list of absolute path of audio files. @@ -59,7 +61,9 @@ def __init__(self, self.feat_type = feat_type self.sample_rate = sample_rate - self.feat_config = kwargs # Pass keyword arguments to customize feature config + self.feat_config = ( + kwargs # Pass keyword arguments to customize feature config + ) def _get_data(self, input_file: str): raise NotImplementedError @@ -78,8 +82,9 @@ def _convert_to_record(self, idx): if feat_func is not None: waveform = waveform.unsqueeze(0) # (batch_size, T) if self.feat_type != 'spectrogram': - feature_extractor = feat_func(sr=self.sample_rate, - **self.feat_config) + feature_extractor = feat_func( + sr=self.sample_rate, **self.feat_config + ) else: feature_extractor = feat_func(**self.feat_config) record['feat'] = feature_extractor(waveform).squeeze(0) diff --git a/python/paddle/audio/datasets/esc50.py b/python/paddle/audio/datasets/esc50.py index f702fe518facb..4d106c2712974 100644 --- a/python/paddle/audio/datasets/esc50.py +++ b/python/paddle/audio/datasets/esc50.py @@ -133,22 +133,24 @@ class ESC50(AudioClassificationDataset): meta = os.path.join('ESC-50-master', 'meta', 'esc50.csv') meta_info = collections.namedtuple( 'META_INFO', - ('filename', 'fold', 'target', 'category', 'esc10', 'src_file', 'take')) + ('filename', 'fold', 'target', 'category', 'esc10', 'src_file', 'take'), + ) audio_path = os.path.join('ESC-50-master', 'audio') - def __init__(self, - mode: str = 'train', - split: int = 1, - feat_type: str = 'raw', - archive=None, - **kwargs): + def __init__( + self, + mode: str = 'train', + split: int = 1, + feat_type: str = 'raw', + archive=None, + **kwargs + ): if archive is not None: self.archive = archive files, labels = self._get_data(mode, split) - super(ESC50, self).__init__(files=files, - labels=labels, - feat_type=feat_type, - **kwargs) + super(ESC50, self).__init__( + files=files, labels=labels, feat_type=feat_type, **kwargs + ) def _get_meta_info(self) -> List[collections.namedtuple]: ret = [] @@ -158,12 +160,15 @@ def _get_meta_info(self) -> List[collections.namedtuple]: return ret def _get_data(self, mode: str, split: int) -> Tuple[List[str], List[int]]: - if not os.path.isdir(os.path.join(DATA_HOME, self.audio_path)) or \ - not os.path.isfile(os.path.join(DATA_HOME, self.meta)): - download.get_path_from_url(self.archive['url'], - DATA_HOME, - self.archive['md5'], - decompress=True) + if not os.path.isdir( + os.path.join(DATA_HOME, self.audio_path) + ) or not os.path.isfile(os.path.join(DATA_HOME, self.meta)): + download.get_path_from_url( + self.archive['url'], + DATA_HOME, + self.archive['md5'], + decompress=True, + ) meta_info = self._get_meta_info() diff --git a/python/paddle/audio/datasets/tess.py b/python/paddle/audio/datasets/tess.py index 0f375aa2b0172..5cdf8cc65f38b 100644 --- a/python/paddle/audio/datasets/tess.py +++ b/python/paddle/audio/datasets/tess.py @@ -71,8 +71,7 @@ class TESS(AudioClassificationDataset): """ archive = { - 'url': - 'https://bj.bcebos.com/paddleaudio/datasets/TESS_Toronto_emotional_speech_set.zip', + 'url': 'https://bj.bcebos.com/paddleaudio/datasets/TESS_Toronto_emotional_speech_set.zip', 'md5': '1465311b24d1de704c4c63e4ccc470c7', } @@ -85,28 +84,30 @@ class TESS(AudioClassificationDataset): 'ps', # pleasant surprise 'sad', ] - meta_info = collections.namedtuple('META_INFO', - ('speaker', 'word', 'emotion')) + meta_info = collections.namedtuple( + 'META_INFO', ('speaker', 'word', 'emotion') + ) audio_path = 'TESS_Toronto_emotional_speech_set' - def __init__(self, - mode='train', - n_folds=5, - split=1, - feat_type='raw', - archive=None, - **kwargs): - """ - - """ - assert split <= n_folds, f'The selected split should not be larger than n_fold, but got {split} > {n_folds}' + def __init__( + self, + mode='train', + n_folds=5, + split=1, + feat_type='raw', + archive=None, + **kwargs, + ): + """ """ + assert ( + split <= n_folds + ), f'The selected split should not be larger than n_fold, but got {split} > {n_folds}' if archive is not None: self.archive = archive files, labels = self._get_data(mode, n_folds, split) - super(TESS, self).__init__(files=files, - labels=labels, - feat_type=feat_type, - **kwargs) + super(TESS, self).__init__( + files=files, labels=labels, feat_type=feat_type, **kwargs + ) def _get_meta_info(self, files) -> List[collections.namedtuple]: ret = [] @@ -117,10 +118,12 @@ def _get_meta_info(self, files) -> List[collections.namedtuple]: def _get_data(self, mode, n_folds, split) -> Tuple[List[str], List[int]]: if not os.path.isdir(os.path.join(DATA_HOME, self.audio_path)): - download.get_path_from_url(self.archive['url'], - DATA_HOME, - self.archive['md5'], - decompress=True) + download.get_path_from_url( + self.archive['url'], + DATA_HOME, + self.archive['md5'], + decompress=True, + ) wav_files = [] for root, _, files in os.walk(os.path.join(DATA_HOME, self.audio_path)): diff --git a/python/paddle/audio/features/layers.py b/python/paddle/audio/features/layers.py index d21a24d34241f..a732a88dd953e 100644 --- a/python/paddle/audio/features/layers.py +++ b/python/paddle/audio/features/layers.py @@ -61,15 +61,17 @@ class Spectrogram(nn.Layer): feats = feature_extractor(waveform) """ - def __init__(self, - n_fft: int = 512, - hop_length: Optional[int] = 512, - win_length: Optional[int] = None, - window: str = 'hann', - power: float = 1.0, - center: bool = True, - pad_mode: str = 'reflect', - dtype: str = 'float32') -> None: + def __init__( + self, + n_fft: int = 512, + hop_length: Optional[int] = 512, + win_length: Optional[int] = None, + window: str = 'hann', + power: float = 1.0, + center: bool = True, + pad_mode: str = 'reflect', + dtype: str = 'float32', + ) -> None: super(Spectrogram, self).__init__() assert power > 0, 'Power of spectrogram must be > 0.' @@ -78,17 +80,18 @@ def __init__(self, if win_length is None: win_length = n_fft - self.fft_window = get_window(window, - win_length, - fftbins=True, - dtype=dtype) - self._stft = partial(paddle.signal.stft, - n_fft=n_fft, - hop_length=hop_length, - win_length=win_length, - window=self.fft_window, - center=center, - pad_mode=pad_mode) + self.fft_window = get_window( + window, win_length, fftbins=True, dtype=dtype + ) + self._stft = partial( + paddle.signal.stft, + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + window=self.fft_window, + center=center, + pad_mode=pad_mode, + ) self.register_buffer('fft_window', self.fft_window) def forward(self, x: Tensor) -> Tensor: @@ -143,31 +146,35 @@ class MelSpectrogram(nn.Layer): feats = feature_extractor(waveform) """ - def __init__(self, - sr: int = 22050, - n_fft: int = 2048, - hop_length: Optional[int] = 512, - win_length: Optional[int] = None, - window: str = 'hann', - power: float = 2.0, - center: bool = True, - pad_mode: str = 'reflect', - n_mels: int = 64, - f_min: float = 50.0, - f_max: Optional[float] = None, - htk: bool = False, - norm: Union[str, float] = 'slaney', - dtype: str = 'float32') -> None: + def __init__( + self, + sr: int = 22050, + n_fft: int = 2048, + hop_length: Optional[int] = 512, + win_length: Optional[int] = None, + window: str = 'hann', + power: float = 2.0, + center: bool = True, + pad_mode: str = 'reflect', + n_mels: int = 64, + f_min: float = 50.0, + f_max: Optional[float] = None, + htk: bool = False, + norm: Union[str, float] = 'slaney', + dtype: str = 'float32', + ) -> None: super(MelSpectrogram, self).__init__() - self._spectrogram = Spectrogram(n_fft=n_fft, - hop_length=hop_length, - win_length=win_length, - window=window, - power=power, - center=center, - pad_mode=pad_mode, - dtype=dtype) + self._spectrogram = Spectrogram( + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + window=window, + power=power, + center=center, + pad_mode=pad_mode, + dtype=dtype, + ) self.n_mels = n_mels self.f_min = f_min self.f_max = f_max @@ -175,14 +182,16 @@ def __init__(self, self.norm = norm if f_max is None: f_max = sr // 2 - self.fbank_matrix = compute_fbank_matrix(sr=sr, - n_fft=n_fft, - n_mels=n_mels, - f_min=f_min, - f_max=f_max, - htk=htk, - norm=norm, - dtype=dtype) + self.fbank_matrix = compute_fbank_matrix( + sr=sr, + n_fft=n_fft, + n_mels=n_mels, + f_min=f_min, + f_max=f_max, + htk=htk, + norm=norm, + dtype=dtype, + ) self.register_buffer('fbank_matrix', self.fbank_matrix) def forward(self, x: Tensor) -> Tensor: @@ -240,40 +249,44 @@ class LogMelSpectrogram(nn.Layer): feats = feature_extractor(waveform) """ - def __init__(self, - sr: int = 22050, - n_fft: int = 512, - hop_length: Optional[int] = None, - win_length: Optional[int] = None, - window: str = 'hann', - power: float = 2.0, - center: bool = True, - pad_mode: str = 'reflect', - n_mels: int = 64, - f_min: float = 50.0, - f_max: Optional[float] = None, - htk: bool = False, - norm: Union[str, float] = 'slaney', - ref_value: float = 1.0, - amin: float = 1e-10, - top_db: Optional[float] = None, - dtype: str = 'float32') -> None: + def __init__( + self, + sr: int = 22050, + n_fft: int = 512, + hop_length: Optional[int] = None, + win_length: Optional[int] = None, + window: str = 'hann', + power: float = 2.0, + center: bool = True, + pad_mode: str = 'reflect', + n_mels: int = 64, + f_min: float = 50.0, + f_max: Optional[float] = None, + htk: bool = False, + norm: Union[str, float] = 'slaney', + ref_value: float = 1.0, + amin: float = 1e-10, + top_db: Optional[float] = None, + dtype: str = 'float32', + ) -> None: super(LogMelSpectrogram, self).__init__() - self._melspectrogram = MelSpectrogram(sr=sr, - n_fft=n_fft, - hop_length=hop_length, - win_length=win_length, - window=window, - power=power, - center=center, - pad_mode=pad_mode, - n_mels=n_mels, - f_min=f_min, - f_max=f_max, - htk=htk, - norm=norm, - dtype=dtype) + self._melspectrogram = MelSpectrogram( + sr=sr, + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + window=window, + power=power, + center=center, + pad_mode=pad_mode, + n_mels=n_mels, + f_min=f_min, + f_max=f_max, + htk=htk, + norm=norm, + dtype=dtype, + ) self.ref_value = ref_value self.amin = amin @@ -288,10 +301,12 @@ def forward(self, x: Tensor) -> Tensor: Tensor: Log mel spectrograms with shape `(N, n_mels, num_frames)`. """ mel_feature = self._melspectrogram(x) - log_mel_feature = power_to_db(mel_feature, - ref_value=self.ref_value, - amin=self.amin, - top_db=self.top_db) + log_mel_feature = power_to_db( + mel_feature, + ref_value=self.ref_value, + amin=self.amin, + top_db=self.top_db, + ) return log_mel_feature @@ -338,45 +353,50 @@ class MFCC(nn.Layer): feats = feature_extractor(waveform) """ - def __init__(self, - sr: int = 22050, - n_mfcc: int = 40, - n_fft: int = 512, - hop_length: Optional[int] = None, - win_length: Optional[int] = None, - window: str = 'hann', - power: float = 2.0, - center: bool = True, - pad_mode: str = 'reflect', - n_mels: int = 64, - f_min: float = 50.0, - f_max: Optional[float] = None, - htk: bool = False, - norm: Union[str, float] = 'slaney', - ref_value: float = 1.0, - amin: float = 1e-10, - top_db: Optional[float] = None, - dtype: str = 'float32') -> None: + def __init__( + self, + sr: int = 22050, + n_mfcc: int = 40, + n_fft: int = 512, + hop_length: Optional[int] = None, + win_length: Optional[int] = None, + window: str = 'hann', + power: float = 2.0, + center: bool = True, + pad_mode: str = 'reflect', + n_mels: int = 64, + f_min: float = 50.0, + f_max: Optional[float] = None, + htk: bool = False, + norm: Union[str, float] = 'slaney', + ref_value: float = 1.0, + amin: float = 1e-10, + top_db: Optional[float] = None, + dtype: str = 'float32', + ) -> None: super(MFCC, self).__init__() - assert n_mfcc <= n_mels, 'n_mfcc cannot be larger than n_mels: %d vs %d' % ( - n_mfcc, n_mels) - self._log_melspectrogram = LogMelSpectrogram(sr=sr, - n_fft=n_fft, - hop_length=hop_length, - win_length=win_length, - window=window, - power=power, - center=center, - pad_mode=pad_mode, - n_mels=n_mels, - f_min=f_min, - f_max=f_max, - htk=htk, - norm=norm, - ref_value=ref_value, - amin=amin, - top_db=top_db, - dtype=dtype) + assert ( + n_mfcc <= n_mels + ), 'n_mfcc cannot be larger than n_mels: %d vs %d' % (n_mfcc, n_mels) + self._log_melspectrogram = LogMelSpectrogram( + sr=sr, + n_fft=n_fft, + hop_length=hop_length, + win_length=win_length, + window=window, + power=power, + center=center, + pad_mode=pad_mode, + n_mels=n_mels, + f_min=f_min, + f_max=f_max, + htk=htk, + norm=norm, + ref_value=ref_value, + amin=amin, + top_db=top_db, + dtype=dtype, + ) self.dct_matrix = create_dct(n_mfcc=n_mfcc, n_mels=n_mels, dtype=dtype) self.register_buffer('dct_matrix', self.dct_matrix) @@ -389,6 +409,9 @@ def forward(self, x: Tensor) -> Tensor: Tensor: Mel frequency cepstral coefficients with shape `(N, n_mfcc, num_frames)`. """ log_mel_feature = self._log_melspectrogram(x) - mfcc = paddle.matmul(log_mel_feature.transpose( - (0, 2, 1)), self.dct_matrix).transpose((0, 2, 1)) # (B, n_mels, L) + mfcc = paddle.matmul( + log_mel_feature.transpose((0, 2, 1)), self.dct_matrix + ).transpose( + (0, 2, 1) + ) # (B, n_mels, L) return mfcc diff --git a/python/paddle/audio/functional/functional.py b/python/paddle/audio/functional/functional.py index 69feab4366757..d8fb243615307 100644 --- a/python/paddle/audio/functional/functional.py +++ b/python/paddle/audio/functional/functional.py @@ -20,8 +20,9 @@ from paddle import Tensor -def hz_to_mel(freq: Union[Tensor, float], - htk: bool = False) -> Union[Tensor, float]: +def hz_to_mel( + freq: Union[Tensor, float], htk: bool = False +) -> Union[Tensor, float]: """Convert Hz to Mels. Args: @@ -61,11 +62,13 @@ def hz_to_mel(freq: Union[Tensor, float], logstep = math.log(6.4) / 27.0 # step size for log region if isinstance(freq, Tensor): - target = min_log_mel + paddle.log( - freq / min_log_hz + 1e-10) / logstep # prevent nan with 1e-10 + target = ( + min_log_mel + paddle.log(freq / min_log_hz + 1e-10) / logstep + ) # prevent nan with 1e-10 mask = (freq > min_log_hz).astype(freq.dtype) mels = target * mask + mels * ( - 1 - mask) # will replace by masked_fill OP in future + 1 - mask + ) # will replace by masked_fill OP in future else: if freq >= min_log_hz: mels = min_log_mel + math.log(freq / min_log_hz + 1e-10) / logstep @@ -73,8 +76,9 @@ def hz_to_mel(freq: Union[Tensor, float], return mels -def mel_to_hz(mel: Union[float, Tensor], - htk: bool = False) -> Union[float, Tensor]: +def mel_to_hz( + mel: Union[float, Tensor], htk: bool = False +) -> Union[float, Tensor]: """Convert mel bin numbers to frequencies. Args: @@ -96,7 +100,7 @@ def mel_to_hz(mel: Union[float, Tensor], """ if htk: - return 700.0 * (10.0**(mel / 2595.0) - 1.0) + return 700.0 * (10.0 ** (mel / 2595.0) - 1.0) f_min = 0.0 f_sp = 200.0 / 3 @@ -109,18 +113,21 @@ def mel_to_hz(mel: Union[float, Tensor], target = min_log_hz * paddle.exp(logstep * (mel - min_log_mel)) mask = (mel > min_log_mel).astype(mel.dtype) freqs = target * mask + freqs * ( - 1 - mask) # will replace by masked_fill OP in future + 1 - mask + ) # will replace by masked_fill OP in future else: if mel >= min_log_mel: freqs = min_log_hz * math.exp(logstep * (mel - min_log_mel)) return freqs -def mel_frequencies(n_mels: int = 64, - f_min: float = 0.0, - f_max: float = 11025.0, - htk: bool = False, - dtype: str = 'float32') -> Tensor: +def mel_frequencies( + n_mels: int = 64, + f_min: float = 0.0, + f_max: float = 11025.0, + htk: bool = False, + dtype: str = 'float32', +) -> Tensor: """Compute mel frequencies. Args: @@ -177,14 +184,16 @@ def fft_frequencies(sr: int, n_fft: int, dtype: str = 'float32') -> Tensor: return paddle.linspace(0, float(sr) / 2, int(1 + n_fft // 2), dtype=dtype) -def compute_fbank_matrix(sr: int, - n_fft: int, - n_mels: int = 64, - f_min: float = 0.0, - f_max: Optional[float] = None, - htk: bool = False, - norm: Union[str, float] = 'slaney', - dtype: str = 'float32') -> Tensor: +def compute_fbank_matrix( + sr: int, + n_fft: int, + n_mels: int = 64, + f_min: float = 0.0, + f_max: Optional[float] = None, + htk: bool = False, + norm: Union[str, float] = 'slaney', + dtype: str = 'float32', +) -> Tensor: """Compute fbank matrix. Args: @@ -220,15 +229,13 @@ def compute_fbank_matrix(sr: int, fftfreqs = fft_frequencies(sr=sr, n_fft=n_fft, dtype=dtype) # 'Center freqs' of mel bands - uniformly spaced between limits - mel_f = mel_frequencies(n_mels + 2, - f_min=f_min, - f_max=f_max, - htk=htk, - dtype=dtype) + mel_f = mel_frequencies( + n_mels + 2, f_min=f_min, f_max=f_max, htk=htk, dtype=dtype + ) - fdiff = mel_f[1:] - mel_f[:-1] #np.diff(mel_f) + fdiff = mel_f[1:] - mel_f[:-1] # np.diff(mel_f) ramps = mel_f.unsqueeze(1) - fftfreqs.unsqueeze(0) - #ramps = np.subtract.outer(mel_f, fftfreqs) + # ramps = np.subtract.outer(mel_f, fftfreqs) for i in range(n_mels): # lower and upper slopes for all bins @@ -236,12 +243,13 @@ def compute_fbank_matrix(sr: int, upper = ramps[i + 2] / fdiff[i + 1] # .. then intersect them with each other and zero - weights[i] = paddle.maximum(paddle.zeros_like(lower), - paddle.minimum(lower, upper)) + weights[i] = paddle.maximum( + paddle.zeros_like(lower), paddle.minimum(lower, upper) + ) # Slaney-style mel is scaled to be approx constant energy per channel if norm == 'slaney': - enorm = 2.0 / (mel_f[2:n_mels + 2] - mel_f[:n_mels]) + enorm = 2.0 / (mel_f[2 : n_mels + 2] - mel_f[:n_mels]) weights *= enorm.unsqueeze(1) elif isinstance(norm, int) or isinstance(norm, float): weights = paddle.nn.functional.normalize(weights, p=norm, axis=-1) @@ -249,10 +257,12 @@ def compute_fbank_matrix(sr: int, return weights -def power_to_db(spect: Tensor, - ref_value: float = 1.0, - amin: float = 1e-10, - top_db: Optional[float] = 80.0) -> Tensor: +def power_to_db( + spect: Tensor, + ref_value: float = 1.0, + amin: float = 1e-10, + top_db: Optional[float] = 80.0, +) -> Tensor: """Convert a power spectrogram (amplitude squared) to decibel (dB) units. The function computes the scaling `10 * log10(x / ref)` in a numerically stable way. Args: @@ -291,10 +301,12 @@ def power_to_db(spect: Tensor, return log_spec -def create_dct(n_mfcc: int, - n_mels: int, - norm: Optional[str] = 'ortho', - dtype: str = 'float32') -> Tensor: +def create_dct( + n_mfcc: int, + n_mels: int, + norm: Optional[str] = 'ortho', + dtype: str = 'float32', +) -> Tensor: """Create a discrete cosine transform(DCT) matrix. Args: @@ -316,8 +328,9 @@ def create_dct(n_mfcc: int, """ n = paddle.arange(n_mels, dtype=dtype) k = paddle.arange(n_mfcc, dtype=dtype).unsqueeze(1) - dct = paddle.cos(math.pi / float(n_mels) * (n + 0.5) * - k) # size (n_mfcc, n_mels) + dct = paddle.cos( + math.pi / float(n_mels) * (n + 0.5) * k + ) # size (n_mfcc, n_mels) if norm is None: dct *= 2.0 else: diff --git a/python/paddle/audio/functional/window.py b/python/paddle/audio/functional/window.py index 17ccdce9ef663..844e2fc26335f 100644 --- a/python/paddle/audio/functional/window.py +++ b/python/paddle/audio/functional/window.py @@ -31,7 +31,7 @@ def _acosh(x: Union[Tensor, float]) -> Tensor: def _extend(M: int, sym: bool) -> bool: - """Extend window by 1 sample if needed for DFT-even symmetry. """ + """Extend window by 1 sample if needed for DFT-even symmetry.""" if not sym: return M + 1, True else: @@ -39,7 +39,7 @@ def _extend(M: int, sym: bool) -> bool: def _len_guards(M: int) -> bool: - """Handle small or incorrect window lengths. """ + """Handle small or incorrect window lengths.""" if int(M) != M or M < 0: raise ValueError('Window length M must be a non-negative integer') @@ -47,101 +47,99 @@ def _len_guards(M: int) -> bool: def _truncate(w: Tensor, needed: bool) -> Tensor: - """Truncate window by 1 sample if needed for DFT-even symmetry. """ + """Truncate window by 1 sample if needed for DFT-even symmetry.""" if needed: return w[:-1] else: return w -def _general_gaussian(M: int, - p, - sig, - sym: bool = True, - dtype: str = 'float64') -> Tensor: +def _general_gaussian( + M: int, p, sig, sym: bool = True, dtype: str = 'float64' +) -> Tensor: """Compute a window with a generalized Gaussian shape. This function is consistent with scipy.signal.windows.general_gaussian(). """ if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) n = paddle.arange(0, M, dtype=dtype) - (M - 1.0) / 2.0 - w = paddle.exp(-0.5 * paddle.abs(n / sig)**(2 * p)) + w = paddle.exp(-0.5 * paddle.abs(n / sig) ** (2 * p)) return _truncate(w, needs_trunc) -def _general_cosine(M: int, - a: float, - sym: bool = True, - dtype: str = 'float64') -> Tensor: +def _general_cosine( + M: int, a: float, sym: bool = True, dtype: str = 'float64' +) -> Tensor: """Compute a generic weighted sum of cosine terms window. This function is consistent with scipy.signal.windows.general_cosine(). """ if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) fac = paddle.linspace(-math.pi, math.pi, M, dtype=dtype) - w = paddle.zeros((M, ), dtype=dtype) + w = paddle.zeros((M,), dtype=dtype) for k in range(len(a)): w += a[k] * paddle.cos(k * fac) return _truncate(w, needs_trunc) -def _general_hamming(M: int, - alpha: float, - sym: bool = True, - dtype: str = 'float64') -> Tensor: +def _general_hamming( + M: int, alpha: float, sym: bool = True, dtype: str = 'float64' +) -> Tensor: """Compute a generalized Hamming window. This function is consistent with scipy.signal.windows.general_hamming() """ - return _general_cosine(M, [alpha, 1. - alpha], sym, dtype=dtype) + return _general_cosine(M, [alpha, 1.0 - alpha], sym, dtype=dtype) -def _taylor(M: int, - nbar=4, - sll=30, - norm=True, - sym: bool = True, - dtype: str = 'float64') -> Tensor: +def _taylor( + M: int, nbar=4, sll=30, norm=True, sym: bool = True, dtype: str = 'float64' +) -> Tensor: """Compute a Taylor window. The Taylor window taper function approximates the Dolph-Chebyshev window's constant sidelobe level for a parameterized number of near-in sidelobes. """ if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) # Original text uses a negative sidelobe level parameter and then negates # it in the calculation of B. To keep consistent with other methods we # assume the sidelobe level parameter to be positive. - B = 10**(sll / 20) + B = 10 ** (sll / 20) A = _acosh(B) / math.pi - s2 = nbar**2 / (A**2 + (nbar - 0.5)**2) + s2 = nbar**2 / (A**2 + (nbar - 0.5) ** 2) ma = paddle.arange(1, nbar, dtype=dtype) - Fm = paddle.empty((nbar - 1, ), dtype=dtype) + Fm = paddle.empty((nbar - 1,), dtype=dtype) signs = paddle.empty_like(ma) signs[::2] = 1 signs[1::2] = -1 m2 = ma * ma for mi in range(len(ma)): - numer = signs[mi] * paddle.prod(1 - m2[mi] / s2 / (A**2 + - (ma - 0.5)**2)) + numer = signs[mi] * paddle.prod( + 1 - m2[mi] / s2 / (A**2 + (ma - 0.5) ** 2) + ) if mi == 0: - denom = 2 * paddle.prod(1 - m2[mi] / m2[mi + 1:]) + denom = 2 * paddle.prod(1 - m2[mi] / m2[mi + 1 :]) elif mi == len(ma) - 1: denom = 2 * paddle.prod(1 - m2[mi] / m2[:mi]) else: - denom = 2 * paddle.prod(1 - m2[mi] / m2[:mi]) * paddle.prod( - 1 - m2[mi] / m2[mi + 1:]) + denom = ( + 2 + * paddle.prod(1 - m2[mi] / m2[:mi]) + * paddle.prod(1 - m2[mi] / m2[mi + 1 :]) + ) Fm[mi] = numer / denom def W(n): return 1 + 2 * paddle.matmul( Fm.unsqueeze(0), - paddle.cos(2 * math.pi * ma.unsqueeze(1) * (n - M / 2. + 0.5) / M)) + paddle.cos(2 * math.pi * ma.unsqueeze(1) * (n - M / 2.0 + 0.5) / M), + ) w = W(paddle.arange(0, M, dtype=dtype)) @@ -169,18 +167,17 @@ def _hann(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor: return _general_hamming(M, 0.5, sym, dtype=dtype) -def _tukey(M: int, - alpha=0.5, - sym: bool = True, - dtype: str = 'float64') -> Tensor: +def _tukey( + M: int, alpha=0.5, sym: bool = True, dtype: str = 'float64' +) -> Tensor: """Compute a Tukey window. The Tukey window is also known as a tapered cosine window. """ if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) if alpha <= 0: - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) elif alpha >= 1.0: return hann(M, sym=sym) @@ -188,57 +185,55 @@ def _tukey(M: int, n = paddle.arange(0, M, dtype=dtype) width = int(alpha * (M - 1) / 2.0) - n1 = n[0:width + 1] - n2 = n[width + 1:M - width - 1] - n3 = n[M - width - 1:] + n1 = n[0 : width + 1] + n2 = n[width + 1 : M - width - 1] + n3 = n[M - width - 1 :] w1 = 0.5 * (1 + paddle.cos(math.pi * (-1 + 2.0 * n1 / alpha / (M - 1)))) w2 = paddle.ones(n2.shape, dtype=dtype) - w3 = 0.5 * (1 + paddle.cos(math.pi * (-2.0 / alpha + 1 + 2.0 * n3 / alpha / - (M - 1)))) + w3 = 0.5 * ( + 1 + + paddle.cos(math.pi * (-2.0 / alpha + 1 + 2.0 * n3 / alpha / (M - 1))) + ) w = paddle.concat([w1, w2, w3]) return _truncate(w, needs_trunc) -def _kaiser(M: int, - beta: float, - sym: bool = True, - dtype: str = 'float64') -> Tensor: +def _kaiser( + M: int, beta: float, sym: bool = True, dtype: str = 'float64' +) -> Tensor: """Compute a Kaiser window. The Kaiser window is a taper formed by using a Bessel function. """ raise NotImplementedError() -def _gaussian(M: int, - std: float, - sym: bool = True, - dtype: str = 'float64') -> Tensor: +def _gaussian( + M: int, std: float, sym: bool = True, dtype: str = 'float64' +) -> Tensor: """Compute a Gaussian window. The Gaussian widows has a Gaussian shape defined by the standard deviation(std). """ if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) n = paddle.arange(0, M, dtype=dtype) - (M - 1.0) / 2.0 sig2 = 2 * std * std - w = paddle.exp(-n**2 / sig2) + w = paddle.exp(-(n**2) / sig2) return _truncate(w, needs_trunc) -def _exponential(M: int, - center=None, - tau=1., - sym: bool = True, - dtype: str = 'float64') -> Tensor: - """Compute an exponential (or Poisson) window. """ +def _exponential( + M: int, center=None, tau=1.0, sym: bool = True, dtype: str = 'float64' +) -> Tensor: + """Compute an exponential (or Poisson) window.""" if sym and center is not None: raise ValueError("If sym==True, center must be None.") if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) if center is None: @@ -251,10 +246,9 @@ def _exponential(M: int, def _triang(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor: - """Compute a triangular window. - """ + """Compute a triangular window.""" if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) n = paddle.arange(1, (M + 1) // 2 + 1, dtype=dtype) @@ -273,12 +267,13 @@ def _bohman(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor: The Bohman window is the autocorrelation of a cosine window. """ if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) fac = paddle.abs(paddle.linspace(-1, 1, M, dtype=dtype)[1:-1]) w = (1 - fac) * paddle.cos(math.pi * fac) + 1.0 / math.pi * paddle.sin( - math.pi * fac) + math.pi * fac + ) w = _cat([0, w, 0], dtype) return _truncate(w, needs_trunc) @@ -295,20 +290,21 @@ def _blackman(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor: def _cosine(M: int, sym: bool = True, dtype: str = 'float64') -> Tensor: - """Compute a window with a simple cosine shape. - """ + """Compute a window with a simple cosine shape.""" if _len_guards(M): - return paddle.ones((M, ), dtype=dtype) + return paddle.ones((M,), dtype=dtype) M, needs_trunc = _extend(M, sym) - w = paddle.sin(math.pi / M * (paddle.arange(0, M, dtype=dtype) + .5)) + w = paddle.sin(math.pi / M * (paddle.arange(0, M, dtype=dtype) + 0.5)) return _truncate(w, needs_trunc) -def get_window(window: Union[str, Tuple[str, float]], - win_length: int, - fftbins: bool = True, - dtype: str = 'float64') -> Tensor: +def get_window( + window: Union[str, Tuple[str, float]], + win_length: int, + fftbins: bool = True, + dtype: str = 'float64', +) -> Tensor: """Return a window of a given length and type. Args: @@ -340,19 +336,22 @@ def get_window(window: Union[str, Tuple[str, float]], args = window[1:] elif isinstance(window, str): if window in ['gaussian', 'exponential']: - raise ValueError("The '" + window + "' window needs one or " - "more parameters -- pass a tuple.") + raise ValueError( + "The '" + window + "' window needs one or " + "more parameters -- pass a tuple." + ) else: winstr = window else: - raise ValueError("%s as window type is not supported." % - str(type(window))) + raise ValueError( + "%s as window type is not supported." % str(type(window)) + ) try: winfunc = eval('_' + winstr) except NameError as e: raise ValueError("Unknown window type.") from e - params = (win_length, ) + args + params = (win_length,) + args kwargs = {'sym': sym} return winfunc(*params, dtype=dtype, **kwargs) diff --git a/python/paddle/audio/utils/error.py b/python/paddle/audio/utils/error.py index ab239a24970ad..244340b99b5ec 100644 --- a/python/paddle/audio/utils/error.py +++ b/python/paddle/audio/utils/error.py @@ -17,4 +17,5 @@ class ParameterError(Exception): """Exception class for Parameter checking""" + pass diff --git a/python/paddle/autograd/__init__.py b/python/paddle/autograd/__init__.py index 70fc9647cd489..19742cf6afe6c 100644 --- a/python/paddle/autograd/__init__.py +++ b/python/paddle/autograd/__init__.py @@ -18,6 +18,7 @@ from . import backward_mode # noqa: F401 from .backward_mode import backward # noqa: F401 from ..fluid.framework import _in_eager_mode_ + if _in_eager_mode_: from .py_layer import EagerPyLayer as PyLayer # noqa: F401 from .py_layer import EagerPyLayerContext as PyLayerContext # noqa: F401 diff --git a/python/paddle/autograd/backward_mode.py b/python/paddle/autograd/backward_mode.py index 83c84a01afb27..9f673e62e21c5 100644 --- a/python/paddle/autograd/backward_mode.py +++ b/python/paddle/autograd/backward_mode.py @@ -83,15 +83,13 @@ def check_tensors(in_out_list, name): assert len(in_out_list) > 0, "{} connot be empyt".format(name) for each_var in in_out_list: assert isinstance( - each_var, - (paddle.Tensor, core.eager.Tensor - )), "Elements of {} must be paddle.Tensor".format(name) + each_var, (paddle.Tensor, core.eager.Tensor) + ), "Elements of {} must be paddle.Tensor".format(name) return in_out_list else: assert isinstance( - in_out_list, - (paddle.Tensor, core.eager.Tensor - )), "{} must be Tensor or list of Tensor".format(name) + in_out_list, (paddle.Tensor, core.eager.Tensor) + ), "{} must be Tensor or list of Tensor".format(name) return [in_out_list] tensors = check_tensors(tensors, "tensors") @@ -117,12 +115,14 @@ def check_tensors(in_out_list, name): if len(grad_tensors) > 0: assert len(tensors) == len( - grad_tensors), "The length of grad_tensors must be equal to tensors" + grad_tensors + ), "The length of grad_tensors must be equal to tensors" assert isinstance(retain_graph, bool), "retain_graph must be True or False" if framework._in_eager_mode_: core.eager.run_backward(tensors, grad_tensors, retain_graph) else: - core.dygraph_run_backward(tensors, grad_tensors, retain_graph, - framework._dygraph_tracer()) + core.dygraph_run_backward( + tensors, grad_tensors, retain_graph, framework._dygraph_tracer() + ) diff --git a/python/paddle/autograd/py_layer.py b/python/paddle/autograd/py_layer.py index ee7c2e36f73a1..254aabb04b3cb 100644 --- a/python/paddle/autograd/py_layer.py +++ b/python/paddle/autograd/py_layer.py @@ -124,9 +124,7 @@ def backward(ctx, dy): def with_mateclass(meta, *bases): - class impl(meta): - def __new__(cls, name, temp_bases, attrs): return meta(name, bases, attrs) @@ -134,7 +132,6 @@ def __new__(cls, name, temp_bases, attrs): class CPyLayer(object): - @classmethod @dygraph_only def apply(cls, *args, **kwargs): @@ -182,12 +179,14 @@ def backward(ctx, dy): class PyLayerBackward(LegacyPyLayerContext): - def backward(self, *args, **kwargs): with paddle.fluid.dygraph.guard(): with paddle.fluid.dygraph.no_grad(): - if self._amp_state and 'enable' in self._amp_state and self._amp_state[ - 'enable']: + if ( + self._amp_state + and 'enable' in self._amp_state + and self._amp_state['enable'] + ): with auto_cast(**args[0]._amp_state): return self._forward_cls.backward(*args, **kwargs) else: @@ -197,10 +196,10 @@ def backward(self, *args, **kwargs): class LayerMeta(type): - def __init__(cls, name, bases, attrs): - cls._backward_function = type(name + '_backward', (PyLayerBackward, ), - {"_forward_cls": cls}) + cls._backward_function = type( + name + '_backward', (PyLayerBackward,), {"_forward_cls": cls} + ) return super(LayerMeta, cls).__init__(name, bases, attrs) @@ -292,7 +291,8 @@ def backward(ctx, dy): return grad """ raise NotImplementedError( - "You must implement the forward function for PyLayer.") + "You must implement the forward function for PyLayer." + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -332,11 +332,11 @@ def backward(ctx, dy): """ raise NotImplementedError( - "You must implement the backward function for PyLayer.") + "You must implement the backward function for PyLayer." + ) class EagerPyLayerContext(object): - def save_for_backward(self, *tensors): """ Saves given tensors that backward need. Use ``saved_tensor`` in the `backward` to get the saved tensors. @@ -542,25 +542,22 @@ def backward(ctx, grad, grad2): class EagerPyLayerBackward(core.eager.PyLayer, EagerPyLayerContext): - def backward(self, *args): return self._forward_cls.backward(self, *args) class EagerPyLayerMeta(type): - def __init__(cls, name, bases, attrs): - cls._backward_function = type(name + '_backward', - (EagerPyLayerBackward, ), - {"_forward_cls": cls}) + cls._backward_function = type( + name + '_backward', (EagerPyLayerBackward,), {"_forward_cls": cls} + ) return super(EagerPyLayerMeta, cls).__init__(name, bases, attrs) class EagerPyLayer( - with_mateclass(EagerPyLayerMeta, core.eager.PyLayer, - EagerPyLayerContext)): - + with_mateclass(EagerPyLayerMeta, core.eager.PyLayer, EagerPyLayerContext) +): @staticmethod def forward(ctx, *args, **kwargs): """ @@ -597,7 +594,8 @@ def backward(ctx, dy): return grad """ raise NotImplementedError( - "You must implement the forward function for PyLayer.") + "You must implement the forward function for PyLayer." + ) @staticmethod def backward(ctx, *args): @@ -637,11 +635,11 @@ def backward(ctx, dy): """ raise NotImplementedError( - "You must implement the backward function for PyLayer.") + "You must implement the backward function for PyLayer." + ) def once_differentiable(backward): - def wrapper(ctx, *args): with paddle.fluid.dygraph.no_grad(): outputs = backward(ctx, *args) diff --git a/python/paddle/autograd/saved_tensors_hooks.py b/python/paddle/autograd/saved_tensors_hooks.py index ff5fecf5ccd5a..8906dd98b4991 100644 --- a/python/paddle/autograd/saved_tensors_hooks.py +++ b/python/paddle/autograd/saved_tensors_hooks.py @@ -17,7 +17,7 @@ __all__ = [] -class saved_tensors_hooks(): +class saved_tensors_hooks: """ Dynamic graph, registers a pair of pack / unpack hooks for saved tensors. @@ -104,8 +104,9 @@ def __init__(self, pack_hook, unpack_hook): self.unpack_hook = unpack_hook def __enter__(self): - core.eager.register_saved_tensors_hooks(self.pack_hook, - self.unpack_hook) + core.eager.register_saved_tensors_hooks( + self.pack_hook, self.unpack_hook + ) def __exit__(self, *args): core.eager.reset_saved_tensors_hooks() diff --git a/python/paddle/batch.py b/python/paddle/batch.py index b87ddc7485882..13ba5a00c0c0c 100644 --- a/python/paddle/batch.py +++ b/python/paddle/batch.py @@ -66,7 +66,9 @@ def batch_reader(): # Batch size check batch_size = int(batch_size) if batch_size <= 0: - raise ValueError("batch_size should be a positive integeral value, " - "but got batch_size={}".format(batch_size)) + raise ValueError( + "batch_size should be a positive integeral value, " + "but got batch_size={}".format(batch_size) + ) return batch_reader diff --git a/python/paddle/callbacks.py b/python/paddle/callbacks.py index 46f69aae1bbfa..b2da04c83df86 100644 --- a/python/paddle/callbacks.py +++ b/python/paddle/callbacks.py @@ -20,7 +20,12 @@ from .hapi.callbacks import EarlyStopping # noqa: F401 from .hapi.callbacks import ReduceLROnPlateau # noqa: F401 -__all__ = [ #noqa - 'Callback', 'ProgBarLogger', 'ModelCheckpoint', 'VisualDL', 'LRScheduler', - 'EarlyStopping', 'ReduceLROnPlateau' +__all__ = [ # noqa + 'Callback', + 'ProgBarLogger', + 'ModelCheckpoint', + 'VisualDL', + 'LRScheduler', + 'EarlyStopping', + 'ReduceLROnPlateau', ] diff --git a/python/paddle/check_import_scipy.py b/python/paddle/check_import_scipy.py index 9bbd061cb3771..473a4301813cd 100644 --- a/python/paddle/check_import_scipy.py +++ b/python/paddle/check_import_scipy.py @@ -20,10 +20,10 @@ def check_import_scipy(OsName): import scipy.io as scio # noqa: F401 except ImportError as e: print_info = str(e) - if (len(print_info) > 0): + if len(print_info) > 0: if 'DLL load failed' in print_info: raise ImportError( - print_info + - "\nplease download Visual C++ Redistributable from https://support.microsoft.com/en-us/topic/the-latest-supported-visual-c-downloads-2647da03-1eea-4433-9aff-95f26a218cc0" + print_info + + "\nplease download Visual C++ Redistributable from https://support.microsoft.com/en-us/topic/the-latest-supported-visual-c-downloads-2647da03-1eea-4433-9aff-95f26a218cc0" ) return diff --git a/python/paddle/common_ops_import.py b/python/paddle/common_ops_import.py index 00ef853b995d9..47bce838936d4 100644 --- a/python/paddle/common_ops_import.py +++ b/python/paddle/common_ops_import.py @@ -13,13 +13,29 @@ # limitations under the License. from paddle.fluid.layer_helper import LayerHelper # noqa: F401 from paddle.fluid.param_attr import ParamAttr # noqa: F401 -from paddle.fluid.framework import convert_np_dtype_to_dtype_, _non_static_mode, _varbase_creator, in_dygraph_mode, _in_legacy_dygraph # noqa: F401 -from paddle.fluid.framework import device_guard, default_main_program, dygraph_only, _dygraph_tracer # noqa: F401 +from paddle.fluid.framework import ( # noqa: F401 + convert_np_dtype_to_dtype_, + _non_static_mode, + _varbase_creator, + in_dygraph_mode, + _in_legacy_dygraph, +) +from paddle.fluid.framework import ( # noqa: F401 + device_guard, + default_main_program, + dygraph_only, + _dygraph_tracer, +) from paddle.fluid.framework import OpProtoHolder, Variable # noqa: F401 from paddle.fluid.initializer import Constant # noqa: F401 from paddle.fluid.core import VarDesc # noqa: F401 from paddle.fluid import core, dygraph_utils # noqa: F401 -from paddle.fluid.data_feeder import check_type, check_dtype, check_variable_and_dtype, convert_dtype # noqa: F401 +from paddle.fluid.data_feeder import ( # noqa: F401 + check_type, + check_dtype, + check_variable_and_dtype, + convert_dtype, +) from paddle.fluid.layers import fill_constant, utils, scale # noqa: F401 from paddle.tensor.layer_function_generator import templatedoc # noqa: F401 import paddle.fluid as fluid # noqa: F401 diff --git a/python/paddle/cost_model/cost_model.py b/python/paddle/cost_model/cost_model.py index a59ff31a683a4..8797868287ba0 100644 --- a/python/paddle/cost_model/cost_model.py +++ b/python/paddle/cost_model/cost_model.py @@ -20,8 +20,7 @@ from paddle.fluid import core -class CostModel(): - +class CostModel: def __init__(self): pass @@ -30,11 +29,12 @@ def build_program(self): main_program = static.Program() startup_program = static.Program() - with static.program_guard(main_program=main_program, - startup_program=startup_program): - data = paddle.static.data(name='X', - shape=[None, 1], - dtype='float32') + with static.program_guard( + main_program=main_program, startup_program=startup_program + ): + data = paddle.static.data( + name='X', shape=[None, 1], dtype='float32' + ) hidden = paddle.static.nn.fc(data, 10) loss = paddle.mean(hidden) paddle.optimizer.SGD(learning_rate=0.01).minimize(loss) @@ -43,11 +43,13 @@ def build_program(self): return startup_program, main_program - def profile_measure(self, - startup_program, - main_program, - device='gpu', - fetch_cost_list=['time']): + def profile_measure( + self, + startup_program, + main_program, + device='gpu', + fetch_cost_list=['time'], + ): place = paddle.set_device('gpu') x = np.random.random(size=(10, 1)).astype('float32') @@ -61,8 +63,9 @@ def profile_measure(self, cost_data = cost_model.ProfileMeasure(device) def static_cost_data(self): - static_cost_data_path = os.path.join(os.path.dirname(__file__), - "static_op_benchmark.json") + static_cost_data_path = os.path.join( + os.path.dirname(__file__), "static_op_benchmark.json" + ) with open(static_cost_data_path, 'r') as load_f: load_dict = json.load(load_f) self._static_cost_data = load_dict @@ -79,7 +82,7 @@ def get_static_op_time(self, op_name, forward=True, dtype="float32"): op_cost = {} for op_data in self._static_cost_data: if (op_data["op"] == op_name) and (dtype in op_data["config"]): - if (forward): + if forward: op_cost["op_time"] = op_data["paddle_gpu_time"] else: op_cost["op_time"] = op_data["paddle_gpu_time_backward"] diff --git a/python/paddle/dataset/cifar.py b/python/paddle/dataset/cifar.py index 0a05ecde4738a..80fd43eb52f71 100644 --- a/python/paddle/dataset/cifar.py +++ b/python/paddle/dataset/cifar.py @@ -43,7 +43,6 @@ def reader_creator(filename, sub_name, cycle=False): - def read_batch(batch): data = batch[b'data'] labels = batch.get(b'labels', batch.get(b'fine_labels', None)) @@ -54,8 +53,11 @@ def read_batch(batch): def reader(): while True: with tarfile.open(filename, mode='r') as f: - names = (each_item.name for each_item in f - if sub_name in each_item.name) + names = ( + each_item.name + for each_item in f + if sub_name in each_item.name + ) for name in names: batch = pickle.load(f.extractfile(name), encoding='bytes') @@ -72,7 +74,8 @@ def reader(): since="2.0.0", update_to="paddle.vision.datasets.Cifar100", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train100(): """ CIFAR-100 training set creator. @@ -85,14 +88,16 @@ def train100(): """ return reader_creator( paddle.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5), - 'train') + 'train', + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Cifar100", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test100(): """ CIFAR-100 test set creator. @@ -105,14 +110,16 @@ def test100(): """ return reader_creator( paddle.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5), - 'test') + 'test', + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Cifar10", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train10(cycle=False): """ CIFAR-10 training set creator. @@ -125,17 +132,19 @@ def train10(cycle=False): :return: Training reader creator :rtype: callable """ - return reader_creator(paddle.dataset.common.download( - CIFAR10_URL, 'cifar', CIFAR10_MD5), - 'data_batch', - cycle=cycle) + return reader_creator( + paddle.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5), + 'data_batch', + cycle=cycle, + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Cifar10", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test10(cycle=False): """ CIFAR-10 test set creator. @@ -148,17 +157,19 @@ def test10(cycle=False): :return: Test reader creator. :rtype: callable """ - return reader_creator(paddle.dataset.common.download( - CIFAR10_URL, 'cifar', CIFAR10_MD5), - 'test_batch', - cycle=cycle) + return reader_creator( + paddle.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5), + 'test_batch', + cycle=cycle, + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Cifar10", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5) paddle.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5) diff --git a/python/paddle/dataset/common.py b/python/paddle/dataset/common.py index 97e986c70068a..157cdeb91fec7 100644 --- a/python/paddle/dataset/common.py +++ b/python/paddle/dataset/common.py @@ -74,8 +74,8 @@ def download(url, module_name, md5sum, save_name=None): os.makedirs(dirname) filename = os.path.join( - dirname, - url.split('/')[-1] if save_name is None else save_name) + dirname, url.split('/')[-1] if save_name is None else save_name + ) if os.path.exists(filename) and md5file(filename) == md5sum: return filename @@ -90,9 +90,12 @@ def download(url, module_name, md5sum, save_name=None): else: raise RuntimeError( "Cannot download {0} within retry limit {1}".format( - url, retry_limit)) - sys.stderr.write("Cache file %s not found, downloading %s \n" % - (filename, url)) + url, retry_limit + ) + ) + sys.stderr.write( + "Cache file %s not found, downloading %s \n" % (filename, url) + ) sys.stderr.write("Begin to download\n") try: r = requests.get(url, stream=True) @@ -108,8 +111,9 @@ def download(url, module_name, md5sum, save_name=None): total_iter = total_length / chunk_size + 1 log_interval = total_iter // 20 if total_iter > 20 else 1 log_index = 0 - bar = paddle.hapi.progressbar.ProgressBar(total_iter, - name='item') + bar = paddle.hapi.progressbar.ProgressBar( + total_iter, name='item' + ) for data in r.iter_content(chunk_size=chunk_size): f.write(data) log_index += 1 @@ -127,12 +131,15 @@ def download(url, module_name, md5sum, save_name=None): def fetch_all(): for module_name in [ - x for x in dir(paddle.dataset) if not x.startswith("__") + x for x in dir(paddle.dataset) if not x.startswith("__") ]: if "fetch" in dir( - importlib.import_module("paddle.dataset.%s" % module_name)): - getattr(importlib.import_module("paddle.dataset.%s" % module_name), - "fetch")() + importlib.import_module("paddle.dataset.%s" % module_name) + ): + getattr( + importlib.import_module("paddle.dataset.%s" % module_name), + "fetch", + )() def split(reader, line_count, suffix="%05d.pickle", dumper=pickle.dump): @@ -173,10 +180,9 @@ def split(reader, line_count, suffix="%05d.pickle", dumper=pickle.dump): dumper(lines, f) -def cluster_files_reader(files_pattern, - trainer_count, - trainer_id, - loader=pickle.load): +def cluster_files_reader( + files_pattern, trainer_count, trainer_id, loader=pickle.load +): """ Create a reader that yield element from the given files, select a file set according trainer count and trainer_id @@ -216,4 +222,5 @@ def _check_exists_and_download(path, url, md5, module_name, download=True): return paddle.dataset.common.download(url, module_name, md5) else: raise ValueError( - '{} not exists and auto download disabled'.format(path)) + '{} not exists and auto download disabled'.format(path) + ) diff --git a/python/paddle/dataset/conll05.py b/python/paddle/dataset/conll05.py index 9333d7350e8c6..22038594f60c9 100644 --- a/python/paddle/dataset/conll05.py +++ b/python/paddle/dataset/conll05.py @@ -83,7 +83,8 @@ def reader(): wf = tf.extractfile(words_name) pf = tf.extractfile(props_name) with gzip.GzipFile(fileobj=wf) as words_file, gzip.GzipFile( - fileobj=pf) as props_file: + fileobj=pf + ) as props_file: sentences = [] labels = [] one_seg = [] @@ -116,16 +117,17 @@ def reader(): lbl_seq.append('I-' + cur_tag) is_in_bracket = False elif l.find('(') != -1 and l.find(')') != -1: - cur_tag = l[1:l.find('*')] + cur_tag = l[1 : l.find('*')] lbl_seq.append('B-' + cur_tag) is_in_bracket = False elif l.find('(') != -1 and l.find(')') == -1: - cur_tag = l[1:l.find('*')] + cur_tag = l[1 : l.find('*')] lbl_seq.append('B-' + cur_tag) is_in_bracket = True else: - raise RuntimeError('Unexpected label: %s' % - l) + raise RuntimeError( + 'Unexpected label: %s' % l + ) yield sentences, verb_list[i], lbl_seq @@ -143,11 +145,9 @@ def reader(): return reader -def reader_creator(corpus_reader, - word_dict=None, - predicate_dict=None, - label_dict=None): - +def reader_creator( + corpus_reader, word_dict=None, predicate_dict=None, label_dict=None +): def reader(): for sentence, predicate, labels in corpus_reader(): @@ -193,8 +193,7 @@ def reader(): pred_idx = [predicate_dict.get(predicate)] * sen_len label_idx = [label_dict.get(w) for w in labels] - yield word_idx, ctx_n2_idx, ctx_n1_idx, \ - ctx_0_idx, ctx_p1_idx, ctx_p2_idx, pred_idx, mark, label_idx + yield word_idx, ctx_n2_idx, ctx_n1_idx, ctx_0_idx, ctx_p1_idx, ctx_p2_idx, pred_idx, mark, label_idx return reader @@ -203,17 +202,21 @@ def reader(): since="2.0.0", update_to="paddle.text.datasets.Conll05st", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def get_dict(): """ Get the word, verb and label dictionary of Wikipedia corpus. """ word_dict = load_dict( - paddle.dataset.common.download(WORDDICT_URL, 'conll05st', WORDDICT_MD5)) + paddle.dataset.common.download(WORDDICT_URL, 'conll05st', WORDDICT_MD5) + ) verb_dict = load_dict( - paddle.dataset.common.download(VERBDICT_URL, 'conll05st', VERBDICT_MD5)) + paddle.dataset.common.download(VERBDICT_URL, 'conll05st', VERBDICT_MD5) + ) label_dict = load_label_dict( - paddle.dataset.common.download(TRGDICT_URL, 'conll05st', TRGDICT_MD5)) + paddle.dataset.common.download(TRGDICT_URL, 'conll05st', TRGDICT_MD5) + ) return word_dict, verb_dict, label_dict @@ -221,7 +224,8 @@ def get_dict(): since="2.0.0", update_to="paddle.text.datasets.Conll05st", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def get_embedding(): """ Get the trained word vector based on Wikipedia corpus. @@ -233,7 +237,8 @@ def get_embedding(): since="2.0.0", update_to="paddle.text.datasets.Conll05st", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(): """ Conll05 test set creator. @@ -250,7 +255,8 @@ def test(): reader = corpus_reader( paddle.dataset.common.download(DATA_URL, 'conll05st', DATA_MD5), words_name='conll05st-release/test.wsj/words/test.wsj.words.gz', - props_name='conll05st-release/test.wsj/props/test.wsj.props.gz') + props_name='conll05st-release/test.wsj/props/test.wsj.props.gz', + ) return reader_creator(reader, word_dict, verb_dict, label_dict) @@ -258,7 +264,8 @@ def test(): since="2.0.0", update_to="paddle.text.datasets.Conll05st", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(WORDDICT_URL, 'conll05st', WORDDICT_MD5) paddle.dataset.common.download(VERBDICT_URL, 'conll05st', VERBDICT_MD5) diff --git a/python/paddle/dataset/flowers.py b/python/paddle/dataset/flowers.py index 42c0e7081d58e..41ffc488aa1ac 100644 --- a/python/paddle/dataset/flowers.py +++ b/python/paddle/dataset/flowers.py @@ -63,11 +63,9 @@ def default_mapper(is_train, sample): ''' img, label = sample img = load_image_bytes(img) - img = simple_transform(img, - 256, - 224, - is_train, - mean=[103.94, 116.78, 123.68]) + img = simple_transform( + img, 256, 224, is_train, mean=[103.94, 116.78, 123.68] + ) return img.flatten().astype('float32'), label @@ -75,14 +73,16 @@ def default_mapper(is_train, sample): test_mapper = functools.partial(default_mapper, False) -def reader_creator(data_file, - label_file, - setid_file, - dataset_name, - mapper, - buffered_size=1024, - use_xmap=True, - cycle=False): +def reader_creator( + data_file, + label_file, + setid_file, + dataset_name, + mapper, + buffered_size=1024, + use_xmap=True, + cycle=False, +): ''' 1. read images from tar file and merge images into batch files in 102flowers.tgz_batch/ @@ -138,7 +138,8 @@ def reader(): since="2.0.0", update_to="paddle.vision.datasets.Flowers", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(mapper=train_mapper, buffered_size=1024, use_xmap=True, cycle=False): ''' Create flowers training set reader. @@ -157,21 +158,24 @@ def train(mapper=train_mapper, buffered_size=1024, use_xmap=True, cycle=False): :return: train data reader :rtype: callable ''' - return reader_creator(download(DATA_URL, 'flowers', DATA_MD5), - download(LABEL_URL, 'flowers', LABEL_MD5), - download(SETID_URL, 'flowers', SETID_MD5), - TRAIN_FLAG, - mapper, - buffered_size, - use_xmap, - cycle=cycle) + return reader_creator( + download(DATA_URL, 'flowers', DATA_MD5), + download(LABEL_URL, 'flowers', LABEL_MD5), + download(SETID_URL, 'flowers', SETID_MD5), + TRAIN_FLAG, + mapper, + buffered_size, + use_xmap, + cycle=cycle, + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Flowers", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(mapper=test_mapper, buffered_size=1024, use_xmap=True, cycle=False): ''' Create flowers test set reader. @@ -190,21 +194,24 @@ def test(mapper=test_mapper, buffered_size=1024, use_xmap=True, cycle=False): :return: test data reader :rtype: callable ''' - return reader_creator(download(DATA_URL, 'flowers', DATA_MD5), - download(LABEL_URL, 'flowers', LABEL_MD5), - download(SETID_URL, 'flowers', SETID_MD5), - TEST_FLAG, - mapper, - buffered_size, - use_xmap, - cycle=cycle) + return reader_creator( + download(DATA_URL, 'flowers', DATA_MD5), + download(LABEL_URL, 'flowers', LABEL_MD5), + download(SETID_URL, 'flowers', SETID_MD5), + TEST_FLAG, + mapper, + buffered_size, + use_xmap, + cycle=cycle, + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.Flowers", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def valid(mapper=test_mapper, buffered_size=1024, use_xmap=True): ''' Create flowers validation set reader. @@ -221,10 +228,15 @@ def valid(mapper=test_mapper, buffered_size=1024, use_xmap=True): :return: test data reader :rtype: callable ''' - return reader_creator(download(DATA_URL, 'flowers', DATA_MD5), - download(LABEL_URL, 'flowers', LABEL_MD5), - download(SETID_URL, 'flowers', SETID_MD5), VALID_FLAG, - mapper, buffered_size, use_xmap) + return reader_creator( + download(DATA_URL, 'flowers', DATA_MD5), + download(LABEL_URL, 'flowers', LABEL_MD5), + download(SETID_URL, 'flowers', SETID_MD5), + VALID_FLAG, + mapper, + buffered_size, + use_xmap, + ) def fetch(): diff --git a/python/paddle/dataset/image.py b/python/paddle/dataset/image.py index 98ad9299c1b08..c8d5124f0feac 100644 --- a/python/paddle/dataset/image.py +++ b/python/paddle/dataset/image.py @@ -31,6 +31,7 @@ """ import numpy as np + # FIXME(minqiyang): this is an ugly fix for the numpy bug reported here # https://github.com/numpy/numpy/issues/12497 import subprocess @@ -42,9 +43,11 @@ # will be the C++ execubable on Windows if sys.platform == 'win32' and 'python.exe' not in interpreter: interpreter = sys.exec_prefix + os.sep + 'python.exe' -import_cv2_proc = subprocess.Popen([interpreter, "-c", "import cv2"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) +import_cv2_proc = subprocess.Popen( + [interpreter, "-c", "import cv2"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, +) out, err = import_cv2_proc.communicate() retcode = import_cv2_proc.poll() if retcode != 0: @@ -65,6 +68,7 @@ def _check_cv2(): if cv2 is None: import sys + sys.stderr.write( '''Warning with paddle image module: opencv-python should be imported, or paddle image module could NOT work; please install opencv-python first.''' @@ -74,10 +78,9 @@ def _check_cv2(): return True -def batch_images_from_tar(data_file, - dataset_name, - img2label, - num_per_batch=1024): +def batch_images_from_tar( + data_file, dataset_name, img2label, num_per_batch=1024 +): """ Read images from tar file and batch them into batch file. @@ -115,9 +118,11 @@ def batch_images_from_tar(data_file, output = {} output['label'] = labels output['data'] = data - pickle.dump(output, - open('%s/batch_%d' % (out_path, file_id), 'wb'), - protocol=2) + pickle.dump( + output, + open('%s/batch_%d' % (out_path, file_id), 'wb'), + protocol=2, + ) file_id += 1 data = [] labels = [] @@ -125,9 +130,9 @@ def batch_images_from_tar(data_file, output = {} output['label'] = labels output['data'] = data - pickle.dump(output, - open('%s/batch_%d' % (out_path, file_id), 'wb'), - protocol=2) + pickle.dump( + output, open('%s/batch_%d' % (out_path, file_id), 'wb'), protocol=2 + ) with open(meta_file, 'a') as meta: for file in os.listdir(out_path): @@ -321,12 +326,9 @@ def left_right_flip(im, is_color=True): return im[:, ::-1] -def simple_transform(im, - resize_size, - crop_size, - is_train, - is_color=True, - mean=None): +def simple_transform( + im, resize_size, crop_size, is_train, is_color=True, mean=None +): """ Simply data argumentation for training. These operations include resizing, croping and flipping. @@ -377,12 +379,9 @@ def simple_transform(im, return im -def load_and_transform(filename, - resize_size, - crop_size, - is_train, - is_color=True, - mean=None): +def load_and_transform( + filename, resize_size, crop_size, is_train, is_color=True, mean=None +): """ Load image from the input file `filename` and transform image for data argumentation. Please refer to the `simple_transform` interface diff --git a/python/paddle/dataset/imdb.py b/python/paddle/dataset/imdb.py index bed7f03090bf9..e95a9e6df0066 100644 --- a/python/paddle/dataset/imdb.py +++ b/python/paddle/dataset/imdb.py @@ -29,7 +29,7 @@ __all__ = [] -#URL = 'http://ai.stanford.edu/%7Eamaas/data/sentiment/aclImdb_v1.tar.gz' +# URL = 'http://ai.stanford.edu/%7Eamaas/data/sentiment/aclImdb_v1.tar.gz' URL = 'https://dataset.bj.bcebos.com/imdb%2FaclImdb_v1.tar.gz' MD5 = '7c2ac02c03563afcf9b574c7e56c153a' @@ -49,7 +49,8 @@ def tokenize(pattern): if bool(pattern.match(tf.name)): # newline and punctuations removal and ad-hoc tokenization. yield tarf.extractfile(tf).read().rstrip(b'\n\r').translate( - None, string.punctuation.encode('latin-1')).lower().split() + None, string.punctuation.encode('latin-1') + ).lower().split() tf = tarf.next() @@ -77,7 +78,8 @@ def build_dict(pattern, cutoff): since="2.0.0", update_to="paddle.text.datasets.Imdb", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def reader_creator(pos_pattern, neg_pattern, word_idx): UNK = word_idx[''] INS = [] @@ -100,7 +102,8 @@ def reader(): since="2.0.0", update_to="paddle.text.datasets.Imdb", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(word_idx): """ IMDB training set creator. @@ -113,15 +116,19 @@ def train(word_idx): :return: Training reader creator :rtype: callable """ - return reader_creator(re.compile(r"aclImdb/train/pos/.*\.txt$"), - re.compile(r"aclImdb/train/neg/.*\.txt$"), word_idx) + return reader_creator( + re.compile(r"aclImdb/train/pos/.*\.txt$"), + re.compile(r"aclImdb/train/neg/.*\.txt$"), + word_idx, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imdb", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(word_idx): """ IMDB test set creator. @@ -134,15 +141,19 @@ def test(word_idx): :return: Test reader creator :rtype: callable """ - return reader_creator(re.compile(r"aclImdb/test/pos/.*\.txt$"), - re.compile(r"aclImdb/test/neg/.*\.txt$"), word_idx) + return reader_creator( + re.compile(r"aclImdb/test/pos/.*\.txt$"), + re.compile(r"aclImdb/test/neg/.*\.txt$"), + word_idx, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imdb", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def word_dict(): """ Build a word dictionary from the corpus. @@ -151,13 +162,15 @@ def word_dict(): :rtype: dict """ return build_dict( - re.compile(r"aclImdb/((train)|(test))/((pos)|(neg))/.*\.txt$"), 150) + re.compile(r"aclImdb/((train)|(test))/((pos)|(neg))/.*\.txt$"), 150 + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imdb", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(URL, 'imdb', MD5) diff --git a/python/paddle/dataset/imikolov.py b/python/paddle/dataset/imikolov.py index b94513ec13afe..4630d88e21a7c 100644 --- a/python/paddle/dataset/imikolov.py +++ b/python/paddle/dataset/imikolov.py @@ -26,7 +26,7 @@ __all__ = [] -#URL = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz' +# URL = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz' URL = 'https://dataset.bj.bcebos.com/imikolov%2Fsimple-examples.tgz' MD5 = '30177ea32e27c525793142b6bf2c8e2d' @@ -57,9 +57,10 @@ def build_dict(min_word_freq=50): train_filename = './simple-examples/data/ptb.train.txt' test_filename = './simple-examples/data/ptb.valid.txt' with tarfile.open( - paddle.dataset.common.download(paddle.dataset.imikolov.URL, - 'imikolov', - paddle.dataset.imikolov.MD5)) as tf: + paddle.dataset.common.download( + paddle.dataset.imikolov.URL, 'imikolov', paddle.dataset.imikolov.MD5 + ) + ) as tf: trainf = tf.extractfile(train_filename) testf = tf.extractfile(test_filename) word_freq = word_count(testf, word_count(trainf)) @@ -78,12 +79,14 @@ def build_dict(min_word_freq=50): def reader_creator(filename, word_idx, n, data_type): - def reader(): with tarfile.open( - paddle.dataset.common.download( - paddle.dataset.imikolov.URL, 'imikolov', - paddle.dataset.imikolov.MD5)) as tf: + paddle.dataset.common.download( + paddle.dataset.imikolov.URL, + 'imikolov', + paddle.dataset.imikolov.MD5, + ) + ) as tf: f = tf.extractfile(filename) UNK = word_idx[''] @@ -94,13 +97,14 @@ def reader(): if len(l) >= n: l = [word_idx.get(w, UNK) for w in l] for i in range(n, len(l) + 1): - yield tuple(l[i - n:i]) + yield tuple(l[i - n : i]) elif DataType.SEQ == data_type: l = l.strip().split() l = [word_idx.get(w, UNK) for w in l] src_seq = [word_idx['']] + l trg_seq = l + [word_idx['']] - if n > 0 and len(src_seq) > n: continue + if n > 0 and len(src_seq) > n: + continue yield src_seq, trg_seq else: assert False, 'Unknow data type' @@ -112,7 +116,8 @@ def reader(): since="2.0.0", update_to="paddle.text.datasets.Imikolov", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(word_idx, n, data_type=DataType.NGRAM): """ imikolov training set creator. @@ -129,15 +134,17 @@ def train(word_idx, n, data_type=DataType.NGRAM): :return: Training reader creator :rtype: callable """ - return reader_creator('./simple-examples/data/ptb.train.txt', word_idx, n, - data_type) + return reader_creator( + './simple-examples/data/ptb.train.txt', word_idx, n, data_type + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imikolov", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(word_idx, n, data_type=DataType.NGRAM): """ imikolov test set creator. @@ -154,14 +161,16 @@ def test(word_idx, n, data_type=DataType.NGRAM): :return: Test reader creator :rtype: callable """ - return reader_creator('./simple-examples/data/ptb.valid.txt', word_idx, n, - data_type) + return reader_creator( + './simple-examples/data/ptb.valid.txt', word_idx, n, data_type + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.Imikolov", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(URL, "imikolov", MD5) diff --git a/python/paddle/dataset/mnist.py b/python/paddle/dataset/mnist.py index 12f03dd837d45..ec1cb9855cfe4 100644 --- a/python/paddle/dataset/mnist.py +++ b/python/paddle/dataset/mnist.py @@ -38,7 +38,6 @@ def reader_creator(image_filename, label_filename, buffer_size): - def reader(): with gzip.GzipFile(image_filename, 'rb') as image_file: img_buf = image_file.read() @@ -53,14 +52,16 @@ def reader(): # image file : 16B magic_byte_img = '>IIII' magic_img, image_num, rows, cols = struct.unpack_from( - magic_byte_img, img_buf, offset_img) + magic_byte_img, img_buf, offset_img + ) offset_img += struct.calcsize(magic_byte_img) offset_lab = 0 # label file : 8B magic_byte_lab = '>II' magic_lab, label_num = struct.unpack_from( - magic_byte_lab, lab_buf, offset_lab) + magic_byte_lab, lab_buf, offset_lab + ) offset_lab += struct.calcsize(magic_byte_lab) while True: @@ -72,11 +73,12 @@ def reader(): step_label += buffer_size fmt_images = '>' + str(buffer_size * rows * cols) + 'B' - images_temp = struct.unpack_from(fmt_images, img_buf, - offset_img) + images_temp = struct.unpack_from( + fmt_images, img_buf, offset_img + ) images = numpy.reshape( - images_temp, - (buffer_size, rows * cols)).astype('float32') + images_temp, (buffer_size, rows * cols) + ).astype('float32') offset_img += struct.calcsize(fmt_images) images = images / 255.0 @@ -93,7 +95,8 @@ def reader(): since="2.0.0", update_to="paddle.vision.datasets.MNIST", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(): """ MNIST training set creator. @@ -105,17 +108,22 @@ def train(): :rtype: callable """ return reader_creator( - paddle.dataset.common.download(TRAIN_IMAGE_URL, 'mnist', - TRAIN_IMAGE_MD5), - paddle.dataset.common.download(TRAIN_LABEL_URL, 'mnist', - TRAIN_LABEL_MD5), 100) + paddle.dataset.common.download( + TRAIN_IMAGE_URL, 'mnist', TRAIN_IMAGE_MD5 + ), + paddle.dataset.common.download( + TRAIN_LABEL_URL, 'mnist', TRAIN_LABEL_MD5 + ), + 100, + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.MNIST", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(): """ MNIST test set creator. @@ -129,14 +137,16 @@ def test(): return reader_creator( paddle.dataset.common.download(TEST_IMAGE_URL, 'mnist', TEST_IMAGE_MD5), paddle.dataset.common.download(TEST_LABEL_URL, 'mnist', TEST_LABEL_MD5), - 100) + 100, + ) @deprecated( since="2.0.0", update_to="paddle.vision.datasets.MNIST", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(TRAIN_IMAGE_URL, 'mnist', TRAIN_IMAGE_MD5) paddle.dataset.common.download(TRAIN_LABEL_URL, 'mnist', TRAIN_LABEL_MD5) diff --git a/python/paddle/dataset/movielens.py b/python/paddle/dataset/movielens.py index 513e5d2a5d65e..fd57ad8edf758 100644 --- a/python/paddle/dataset/movielens.py +++ b/python/paddle/dataset/movielens.py @@ -33,7 +33,7 @@ age_table = [1, 18, 25, 35, 45, 50, 56] -#URL = 'http://files.grouplens.org/datasets/movielens/ml-1m.zip' +# URL = 'http://files.grouplens.org/datasets/movielens/ml-1m.zip' URL = 'https://dataset.bj.bcebos.com/movielens%2Fml-1m.zip' MD5 = 'c4d9eecfca2ab87c1945afe126590906' @@ -53,13 +53,17 @@ def value(self): Get information from a movie. """ return [ - self.index, [CATEGORIES_DICT[c] for c in self.categories], - [MOVIE_TITLE_DICT[w.lower()] for w in self.title.split()] + self.index, + [CATEGORIES_DICT[c] for c in self.categories], + [MOVIE_TITLE_DICT[w.lower()] for w in self.title.split()], ] def __str__(self): return "" % ( - self.index, self.title, self.categories) + self.index, + self.title, + self.categories, + ) def __repr__(self): return self.__str__() @@ -84,8 +88,11 @@ def value(self): def __str__(self): return "" % ( - self.index, "M" if self.is_male else "F", age_table[self.age], - self.job_id) + self.index, + "M" if self.is_male else "F", + age_table[self.age], + self.job_id, + ) def __repr__(self): return str(self) @@ -117,7 +124,8 @@ def __initialize_meta_info__(): categories_set.add(c) title = pattern.match(title).group(1) MOVIE_INFO[int(movie_id)] = MovieInfo( - index=movie_id, categories=categories, title=title) + index=movie_id, categories=categories, title=title + ) for w in title.split(): title_word_set.add(w.lower()) @@ -137,10 +145,9 @@ def __initialize_meta_info__(): for line in user_file: line = line.decode(encoding='latin') uid, gender, age, job, _ = line.strip().split("::") - USER_INFO[int(uid)] = UserInfo(index=uid, - gender=gender, - age=age, - job_id=job) + USER_INFO[int(uid)] = UserInfo( + index=uid, gender=gender, age=age, job_id=job + ) return fn @@ -166,7 +173,8 @@ def __reader__(rand_seed=0, test_ratio=0.1, is_test=False): since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def __reader_creator__(**kwargs): return lambda: __reader__(**kwargs) @@ -179,7 +187,8 @@ def __reader_creator__(**kwargs): since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def get_movie_title_dict(): """ Get movie title dictionary. @@ -199,7 +208,8 @@ def __max_index_info__(a, b): since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def max_movie_id(): """ Get the maximum value of movie id. @@ -212,7 +222,8 @@ def max_movie_id(): since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def max_user_id(): """ Get the maximum value of user id. @@ -232,21 +243,24 @@ def __max_job_id_impl__(a, b): since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def max_job_id(): """ Get the maximum value of job id. """ __initialize_meta_info__() - return functools.reduce(__max_job_id_impl__, - list(USER_INFO.values())).job_id + return functools.reduce( + __max_job_id_impl__, list(USER_INFO.values()) + ).job_id @deprecated( since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def movie_categories(): """ Get movie categories dictionary. @@ -259,7 +273,8 @@ def movie_categories(): since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def user_info(): """ Get user info dictionary. @@ -272,7 +287,8 @@ def user_info(): since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def movie_info(): """ Get movie info dictionary. @@ -294,7 +310,8 @@ def unittest(): since="2.0.0", update_to="paddle.text.datasets.Movielens", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(URL, "movielens", MD5) diff --git a/python/paddle/dataset/tests/cifar_test.py b/python/paddle/dataset/tests/cifar_test.py index f7bdc47d9aac6..6f25975dd8e0f 100644 --- a/python/paddle/dataset/tests/cifar_test.py +++ b/python/paddle/dataset/tests/cifar_test.py @@ -19,7 +19,6 @@ class TestCIFAR(unittest.TestCase): - def check_reader(self, reader): sum = 0 label = 0 @@ -32,25 +31,29 @@ def check_reader(self, reader): def test_test10(self): instances, max_label_value = self.check_reader( - paddle.dataset.cifar.test10()) + paddle.dataset.cifar.test10() + ) self.assertEqual(instances, 10000) self.assertEqual(max_label_value, 9) def test_train10(self): instances, max_label_value = self.check_reader( - paddle.dataset.cifar.train10()) + paddle.dataset.cifar.train10() + ) self.assertEqual(instances, 50000) self.assertEqual(max_label_value, 9) def test_test100(self): instances, max_label_value = self.check_reader( - paddle.dataset.cifar.test100()) + paddle.dataset.cifar.test100() + ) self.assertEqual(instances, 10000) self.assertEqual(max_label_value, 99) def test_train100(self): instances, max_label_value = self.check_reader( - paddle.dataset.cifar.train100()) + paddle.dataset.cifar.train100() + ) self.assertEqual(instances, 50000) self.assertEqual(max_label_value, 99) diff --git a/python/paddle/dataset/tests/flowers_test.py b/python/paddle/dataset/tests/flowers_test.py index 31a49edf9751f..55a27b0ce8fca 100644 --- a/python/paddle/dataset/tests/flowers_test.py +++ b/python/paddle/dataset/tests/flowers_test.py @@ -19,7 +19,6 @@ class TestFlowers(unittest.TestCase): - def check_reader(self, reader): sum = 0 label = 0 @@ -33,19 +32,22 @@ def check_reader(self, reader): def test_train(self): instances, max_label_value = self.check_reader( - paddle.dataset.flowers.train()) + paddle.dataset.flowers.train() + ) self.assertEqual(instances, 6149) self.assertEqual(max_label_value, 102) def test_test(self): instances, max_label_value = self.check_reader( - paddle.dataset.flowers.test()) + paddle.dataset.flowers.test() + ) self.assertEqual(instances, 1020) self.assertEqual(max_label_value, 102) def test_valid(self): instances, max_label_value = self.check_reader( - paddle.dataset.flowers.valid()) + paddle.dataset.flowers.valid() + ) self.assertEqual(instances, 1020) self.assertEqual(max_label_value, 102) diff --git a/python/paddle/dataset/tests/imikolov_test.py b/python/paddle/dataset/tests/imikolov_test.py index f7f025eb54076..f8d8b182e2d77 100644 --- a/python/paddle/dataset/tests/imikolov_test.py +++ b/python/paddle/dataset/tests/imikolov_test.py @@ -21,7 +21,6 @@ class TestMikolov(unittest.TestCase): - def check_reader(self, reader, n): for l in reader(): self.assertEqual(len(l), n) @@ -30,16 +29,18 @@ def test_train(self): n = 5 self.check_reader(paddle.dataset.imikolov.train(WORD_DICT, n), n) - first_line = 'aer banknote berlitz calloway centrust cluett fromstein '\ - 'gitano guterman hydro-quebec ipo kia memotec mlx nahb punts '\ + first_line = ( + 'aer banknote berlitz calloway centrust cluett fromstein ' + 'gitano guterman hydro-quebec ipo kia memotec mlx nahb punts ' 'rake regatta rubens sim snack-food ssangyong swapo wachter' + ) first_line = [ WORD_DICT.get(ch, WORD_DICT['']) for ch in first_line.split(' ') ] for l in paddle.dataset.imikolov.train( - WORD_DICT, n=-1, - data_type=paddle.dataset.imikolov.DataType.SEQ)(): + WORD_DICT, n=-1, data_type=paddle.dataset.imikolov.DataType.SEQ + )(): read_line = l[0][1:] break self.assertEqual(first_line, read_line) @@ -48,15 +49,17 @@ def test_test(self): n = 5 self.check_reader(paddle.dataset.imikolov.test(WORD_DICT, n), n) - first_line = 'consumers may want to move their telephones a little '\ - 'closer to the tv set' + first_line = ( + 'consumers may want to move their telephones a little ' + 'closer to the tv set' + ) first_line = [ WORD_DICT.get(ch, WORD_DICT['']) for ch in first_line.split(' ') ] for l in paddle.dataset.imikolov.test( - WORD_DICT, n=-1, - data_type=paddle.dataset.imikolov.DataType.SEQ)(): + WORD_DICT, n=-1, data_type=paddle.dataset.imikolov.DataType.SEQ + )(): read_line = l[0][1:] break self.assertEqual(first_line, read_line) diff --git a/python/paddle/dataset/tests/mnist_test.py b/python/paddle/dataset/tests/mnist_test.py index d2ebacc8c73f0..978fa714f7267 100644 --- a/python/paddle/dataset/tests/mnist_test.py +++ b/python/paddle/dataset/tests/mnist_test.py @@ -19,7 +19,6 @@ class TestMNIST(unittest.TestCase): - def check_reader(self, reader): sum = 0 label = 0 @@ -32,13 +31,15 @@ def check_reader(self, reader): def test_train(self): instances, max_label_value = self.check_reader( - paddle.dataset.mnist.train()) + paddle.dataset.mnist.train() + ) self.assertEqual(instances, 60000) self.assertEqual(max_label_value, 9) def test_test(self): instances, max_label_value = self.check_reader( - paddle.dataset.mnist.test()) + paddle.dataset.mnist.test() + ) self.assertEqual(instances, 10000) self.assertEqual(max_label_value, 9) diff --git a/python/paddle/dataset/tests/test_image.py b/python/paddle/dataset/tests/test_image.py index 61979b865fda3..953e70c41865d 100644 --- a/python/paddle/dataset/tests/test_image.py +++ b/python/paddle/dataset/tests/test_image.py @@ -32,7 +32,7 @@ class Image(unittest.TestCase): """ def test_resize_flip_chw(self): - """ resize """ + """resize""" imgdir = sys.argv[0].replace('test_image.py', 'cat.jpg') images = image.load_image(imgdir) images = image.resize_short(images, 256) diff --git a/python/paddle/dataset/tests/voc2012_test.py b/python/paddle/dataset/tests/voc2012_test.py index 43f96cd9377ae..7db2f5abd385f 100644 --- a/python/paddle/dataset/tests/voc2012_test.py +++ b/python/paddle/dataset/tests/voc2012_test.py @@ -19,7 +19,6 @@ class TestVOC(unittest.TestCase): - def check_reader(self, reader): sum = 0 label = 0 diff --git a/python/paddle/dataset/tests/wmt16_test.py b/python/paddle/dataset/tests/wmt16_test.py index 1b0b44c974f6a..58f405b26e306 100644 --- a/python/paddle/dataset/tests/wmt16_test.py +++ b/python/paddle/dataset/tests/wmt16_test.py @@ -19,7 +19,6 @@ class TestWMT16(unittest.TestCase): - def checkout_one_sample(self, sample): # train data has 3 field: source language word indices, # target language word indices, and target next word indices. @@ -37,23 +36,30 @@ def checkout_one_sample(self, sample): def test_train(self): for idx, sample in enumerate( - paddle.dataset.wmt16.train(src_dict_size=100000, - trg_dict_size=100000)()): - if idx >= 10: break + paddle.dataset.wmt16.train( + src_dict_size=100000, trg_dict_size=100000 + )() + ): + if idx >= 10: + break self.checkout_one_sample(sample) def test_test(self): for idx, sample in enumerate( - paddle.dataset.wmt16.test(src_dict_size=1000, - trg_dict_size=1000)()): - if idx >= 10: break + paddle.dataset.wmt16.test(src_dict_size=1000, trg_dict_size=1000)() + ): + if idx >= 10: + break self.checkout_one_sample(sample) def test_val(self): for idx, sample in enumerate( - paddle.dataset.wmt16.validation(src_dict_size=1000, - trg_dict_size=1000)()): - if idx >= 10: break + paddle.dataset.wmt16.validation( + src_dict_size=1000, trg_dict_size=1000 + )() + ): + if idx >= 10: + break self.checkout_one_sample(sample) def test_get_dict(self): diff --git a/python/paddle/dataset/uci_housing.py b/python/paddle/dataset/uci_housing.py index 66c6ab3c47989..6c085920e2e05 100644 --- a/python/paddle/dataset/uci_housing.py +++ b/python/paddle/dataset/uci_housing.py @@ -31,8 +31,19 @@ URL = 'http://paddlemodels.bj.bcebos.com/uci_housing/housing.data' MD5 = 'd4accdce7a25600298819f8e28e8d593' feature_names = [ - 'CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', - 'PTRATIO', 'B', 'LSTAT' + 'CRIM', + 'ZN', + 'INDUS', + 'CHAS', + 'NOX', + 'RM', + 'AGE', + 'DIS', + 'RAD', + 'TAX', + 'PTRATIO', + 'B', + 'LSTAT', ] UCI_TRAIN_DATA = None @@ -44,14 +55,15 @@ def feature_range(maximums, minimums): import matplotlib + matplotlib.use('Agg') import matplotlib.pyplot as plt + fig, ax = plt.subplots() feature_num = len(maximums) - ax.bar(list(range(feature_num)), - maximums - minimums, - color='r', - align='center') + ax.bar( + list(range(feature_num)), maximums - minimums, color='r', align='center' + ) ax.set_title('feature scale') plt.xticks(list(range(feature_num)), feature_names) plt.xlim([-1, feature_num]) @@ -70,10 +82,13 @@ def load_data(filename, feature_num=14, ratio=0.8): data = np.fromfile(filename, sep=' ') data = data.reshape(data.shape[0] // feature_num, feature_num) - maximums, minimums, avgs = data.max(axis=0), data.min( - axis=0), data.sum(axis=0) / data.shape[0] + maximums, minimums, avgs = ( + data.max(axis=0), + data.min(axis=0), + data.sum(axis=0) / data.shape[0], + ) # if you want to print the distribution of input data, you could use function of feature_range - #feature_range(maximums[:-1], minimums[:-1]) + # feature_range(maximums[:-1], minimums[:-1]) for i in range(feature_num - 1): data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i]) offset = int(data.shape[0] * ratio) @@ -85,7 +100,8 @@ def load_data(filename, feature_num=14, ratio=0.8): since="2.0.0", update_to="paddle.text.datasets.UCIHousing", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(): """ UCI_HOUSING training set creator. @@ -110,7 +126,8 @@ def reader(): since="2.0.0", update_to="paddle.text.datasets.UCIHousing", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(): """ UCI_HOUSING test set creator. @@ -132,10 +149,9 @@ def reader(): def fluid_model(): - parameter_tar = paddle.dataset.common.download(FLUID_URL_MODEL, - 'uci_housing', - FLUID_MD5_MODEL, - 'fit_a_line.fluid.tar') + parameter_tar = paddle.dataset.common.download( + FLUID_URL_MODEL, 'uci_housing', FLUID_MD5_MODEL, 'fit_a_line.fluid.tar' + ) tar = tarfile.TarFile(parameter_tar, mode='r') dirpath = tempfile.mkdtemp() @@ -148,7 +164,8 @@ def fluid_model(): since="2.0.0", update_to="paddle.text.datasets.UCIHousing", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def predict_reader(): """ It returns just one tuple data to do inference. @@ -158,13 +175,14 @@ def predict_reader(): """ global UCI_TEST_DATA load_data(paddle.dataset.common.download(URL, 'uci_housing', MD5)) - return (UCI_TEST_DATA[0][:-1], ) + return (UCI_TEST_DATA[0][:-1],) @deprecated( since="2.0.0", update_to="paddle.text.datasets.UCIHousing", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(URL, 'uci_housing', MD5) diff --git a/python/paddle/dataset/voc2012.py b/python/paddle/dataset/voc2012.py index a949872fb0bab..2a80eab1e84b9 100644 --- a/python/paddle/dataset/voc2012.py +++ b/python/paddle/dataset/voc2012.py @@ -68,7 +68,8 @@ def reader(): since="2.0.0", update_to="paddle.vision.datasets.VOC2012", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(): """ Create a train dataset reader containing 2913 images in HWC order. @@ -80,7 +81,8 @@ def train(): since="2.0.0", update_to="paddle.vision.datasets.VOC2012", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(): """ Create a test dataset reader containing 1464 images in HWC order. @@ -92,7 +94,8 @@ def test(): since="2.0.0", update_to="paddle.vision.datasets.VOC2012", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def val(): """ Create a val dataset reader containing 1449 images in HWC order. diff --git a/python/paddle/dataset/wmt14.py b/python/paddle/dataset/wmt14.py index 33ad05c36d019..6d7c2639d2fa4 100644 --- a/python/paddle/dataset/wmt14.py +++ b/python/paddle/dataset/wmt14.py @@ -27,12 +27,14 @@ __all__ = [] -URL_DEV_TEST = ('http://www-lium.univ-lemans.fr/~schwenk/' - 'cslm_joint_paper/data/dev+test.tgz') +URL_DEV_TEST = ( + 'http://www-lium.univ-lemans.fr/~schwenk/' + 'cslm_joint_paper/data/dev+test.tgz' +) MD5_DEV_TEST = '7d7897317ddd8ba0ae5c5fa7248d3ff5' # this is a small set of data for test. The original data is too large and # will be add later. -URL_TRAIN = ('http://paddlemodels.bj.bcebos.com/wmt/wmt14.tgz') +URL_TRAIN = 'http://paddlemodels.bj.bcebos.com/wmt/wmt14.tgz' MD5_TRAIN = '0791583d57d5beb693b9414c5b36798c' # BLEU of this trained model is 26.92 URL_MODEL = 'http://paddlemodels.bj.bcebos.com/wmt%2Fwmt14.tgz' @@ -45,7 +47,6 @@ def __read_to_dict(tar_file, dict_size): - def __to_dict(fd, size): out_dict = dict() for line_count, line in enumerate(fd): @@ -57,13 +58,15 @@ def __to_dict(fd, size): with tarfile.open(tar_file, mode='r') as f: names = [ - each_item.name for each_item in f + each_item.name + for each_item in f if each_item.name.endswith("src.dict") ] assert len(names) == 1 src_dict = __to_dict(f.extractfile(names[0]), dict_size) names = [ - each_item.name for each_item in f + each_item.name + for each_item in f if each_item.name.endswith("trg.dict") ] assert len(names) == 1 @@ -72,12 +75,12 @@ def __to_dict(fd, size): def reader_creator(tar_file, file_name, dict_size): - def reader(): src_dict, trg_dict = __read_to_dict(tar_file, dict_size) with tarfile.open(tar_file, mode='r') as f: names = [ - each_item.name for each_item in f + each_item.name + for each_item in f if each_item.name.endswith(file_name) ] for name in names: @@ -112,7 +115,8 @@ def reader(): since="2.0.0", update_to="paddle.text.datasets.WMT14", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(dict_size): """ WMT14 training set creator. @@ -126,14 +130,17 @@ def train(dict_size): """ return reader_creator( paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN), - 'train/train', dict_size) + 'train/train', + dict_size, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT14", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(dict_size): """ WMT14 test set creator. @@ -147,25 +154,31 @@ def test(dict_size): """ return reader_creator( paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN), - 'test/test', dict_size) + 'test/test', + dict_size, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT14", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def gen(dict_size): return reader_creator( paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN), - 'gen/gen', dict_size) + 'gen/gen', + dict_size, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT14", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def get_dict(dict_size, reverse=True): # if reverse = False, return dict = {'a':'001', 'b':'002', ...} # else reverse = true, return dict = {'001':'a', '002':'b', ...} @@ -181,7 +194,8 @@ def get_dict(dict_size, reverse=True): since="2.0.0", update_to="paddle.text.datasets.WMT14", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN) paddle.dataset.common.download(URL_MODEL, 'wmt14', MD5_MODEL) diff --git a/python/paddle/dataset/wmt16.py b/python/paddle/dataset/wmt16.py index f5a97172a7d82..01b13cd136847 100644 --- a/python/paddle/dataset/wmt16.py +++ b/python/paddle/dataset/wmt16.py @@ -37,7 +37,7 @@ __all__ = [] -DATA_URL = ("http://paddlemodels.bj.bcebos.com/wmt/wmt16.tar.gz") +DATA_URL = "http://paddlemodels.bj.bcebos.com/wmt/wmt16.tar.gz" DATA_MD5 = "0c38be43600334966403524a40dcd81e" TOTAL_EN_WORDS = 11250 @@ -54,7 +54,8 @@ def __build_dict(tar_file, dict_size, save_path, lang): for line in f.extractfile("wmt16/train"): line = line.decode() line_split = line.strip().split("\t") - if len(line_split) != 2: continue + if len(line_split) != 2: + continue sen = line_split[0] if lang == "en" else line_split[1] for w in sen.split(): word_dict[w] += 1 @@ -62,17 +63,21 @@ def __build_dict(tar_file, dict_size, save_path, lang): with open(save_path, "wb") as fout: fout.write(("%s\n%s\n%s\n" % (START_MARK, END_MARK, UNK_MARK)).encode()) for idx, word in enumerate( - sorted(word_dict.items(), key=lambda x: x[1], reverse=True)): - if idx + 3 == dict_size: break + sorted(word_dict.items(), key=lambda x: x[1], reverse=True) + ): + if idx + 3 == dict_size: + break fout.write(word[0].encode()) fout.write(b'\n') def __load_dict(tar_file, dict_size, lang, reverse=False): - dict_path = os.path.join(paddle.dataset.common.DATA_HOME, - "wmt16/%s_%d.dict" % (lang, dict_size)) - if not os.path.exists(dict_path) or (len(open(dict_path, "rb").readlines()) - != dict_size): + dict_path = os.path.join( + paddle.dataset.common.DATA_HOME, "wmt16/%s_%d.dict" % (lang, dict_size) + ) + if not os.path.exists(dict_path) or ( + len(open(dict_path, "rb").readlines()) != dict_size + ): __build_dict(tar_file, dict_size, dict_path, lang) word_dict = {} @@ -87,18 +92,20 @@ def __load_dict(tar_file, dict_size, lang, reverse=False): def __get_dict_size(src_dict_size, trg_dict_size, src_lang): src_dict_size = min( - src_dict_size, (TOTAL_EN_WORDS if src_lang == "en" else TOTAL_DE_WORDS)) + src_dict_size, (TOTAL_EN_WORDS if src_lang == "en" else TOTAL_DE_WORDS) + ) trg_dict_size = min( - trg_dict_size, (TOTAL_DE_WORDS if src_lang == "en" else TOTAL_EN_WORDS)) + trg_dict_size, (TOTAL_DE_WORDS if src_lang == "en" else TOTAL_EN_WORDS) + ) return src_dict_size, trg_dict_size def reader_creator(tar_file, file_name, src_dict_size, trg_dict_size, src_lang): - def reader(): src_dict = __load_dict(tar_file, src_dict_size, src_lang) - trg_dict = __load_dict(tar_file, trg_dict_size, - ("de" if src_lang == "en" else "en")) + trg_dict = __load_dict( + tar_file, trg_dict_size, ("de" if src_lang == "en" else "en") + ) # the index for start mark, end mark, and unk are the same in source # language and target language. Here uses the source language @@ -117,9 +124,11 @@ def reader(): if len(line_split) != 2: continue src_words = line_split[src_col].split() - src_ids = [start_id - ] + [src_dict.get(w, unk_id) - for w in src_words] + [end_id] + src_ids = ( + [start_id] + + [src_dict.get(w, unk_id) for w in src_words] + + [end_id] + ) trg_words = line_split[trg_col].split() trg_ids = [trg_dict.get(w, unk_id) for w in trg_words] @@ -136,7 +145,8 @@ def reader(): since="2.0.0", update_to="paddle.text.datasets.WMT16", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def train(src_dict_size, trg_dict_size, src_lang="en"): """ WMT16 train set reader. @@ -172,24 +182,31 @@ def train(src_dict_size, trg_dict_size, src_lang="en"): """ if src_lang not in ["en", "de"]: - raise ValueError("An error language type. Only support: " - "en (for English); de(for Germany).") - src_dict_size, trg_dict_size = __get_dict_size(src_dict_size, trg_dict_size, - src_lang) - - return reader_creator(tar_file=paddle.dataset.common.download( - DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz"), - file_name="wmt16/train", - src_dict_size=src_dict_size, - trg_dict_size=trg_dict_size, - src_lang=src_lang) + raise ValueError( + "An error language type. Only support: " + "en (for English); de(for Germany)." + ) + src_dict_size, trg_dict_size = __get_dict_size( + src_dict_size, trg_dict_size, src_lang + ) + + return reader_creator( + tar_file=paddle.dataset.common.download( + DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz" + ), + file_name="wmt16/train", + src_dict_size=src_dict_size, + trg_dict_size=trg_dict_size, + src_lang=src_lang, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT16", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def test(src_dict_size, trg_dict_size, src_lang="en"): """ WMT16 test set reader. @@ -224,25 +241,32 @@ def test(src_dict_size, trg_dict_size, src_lang="en"): """ if src_lang not in ["en", "de"]: - raise ValueError("An error language type. " - "Only support: en (for English); de(for Germany).") - - src_dict_size, trg_dict_size = __get_dict_size(src_dict_size, trg_dict_size, - src_lang) - - return reader_creator(tar_file=paddle.dataset.common.download( - DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz"), - file_name="wmt16/test", - src_dict_size=src_dict_size, - trg_dict_size=trg_dict_size, - src_lang=src_lang) + raise ValueError( + "An error language type. " + "Only support: en (for English); de(for Germany)." + ) + + src_dict_size, trg_dict_size = __get_dict_size( + src_dict_size, trg_dict_size, src_lang + ) + + return reader_creator( + tar_file=paddle.dataset.common.download( + DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz" + ), + file_name="wmt16/test", + src_dict_size=src_dict_size, + trg_dict_size=trg_dict_size, + src_lang=src_lang, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT16", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def validation(src_dict_size, trg_dict_size, src_lang="en"): """ WMT16 validation set reader. @@ -276,24 +300,31 @@ def validation(src_dict_size, trg_dict_size, src_lang="en"): callable: The validation reader. """ if src_lang not in ["en", "de"]: - raise ValueError("An error language type. " - "Only support: en (for English); de(for Germany).") - src_dict_size, trg_dict_size = __get_dict_size(src_dict_size, trg_dict_size, - src_lang) - - return reader_creator(tar_file=paddle.dataset.common.download( - DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz"), - file_name="wmt16/val", - src_dict_size=src_dict_size, - trg_dict_size=trg_dict_size, - src_lang=src_lang) + raise ValueError( + "An error language type. " + "Only support: en (for English); de(for Germany)." + ) + src_dict_size, trg_dict_size = __get_dict_size( + src_dict_size, trg_dict_size, src_lang + ) + + return reader_creator( + tar_file=paddle.dataset.common.download( + DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz" + ), + file_name="wmt16/val", + src_dict_size=src_dict_size, + trg_dict_size=trg_dict_size, + src_lang=src_lang, + ) @deprecated( since="2.0.0", update_to="paddle.text.datasets.WMT16", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def get_dict(lang, dict_size, reverse=False): """ return the word dictionary for the specified language. @@ -312,11 +343,14 @@ def get_dict(lang, dict_size, reverse=False): dict: The word dictionary for the specific language. """ - if lang == "en": dict_size = min(dict_size, TOTAL_EN_WORDS) - else: dict_size = min(dict_size, TOTAL_DE_WORDS) + if lang == "en": + dict_size = min(dict_size, TOTAL_EN_WORDS) + else: + dict_size = min(dict_size, TOTAL_DE_WORDS) - dict_path = os.path.join(paddle.dataset.common.DATA_HOME, - "wmt16/%s_%d.dict" % (lang, dict_size)) + dict_path = os.path.join( + paddle.dataset.common.DATA_HOME, "wmt16/%s_%d.dict" % (lang, dict_size) + ) assert os.path.exists(dict_path), "Word dictionary does not exist. " "Please invoke paddle.dataset.wmt16.train/test/validation first " "to build the dictionary." @@ -328,9 +362,10 @@ def get_dict(lang, dict_size, reverse=False): since="2.0.0", update_to="paddle.text.datasets.WMT16", level=1, - reason="Please use new dataset API which supports paddle.io.DataLoader") + reason="Please use new dataset API which supports paddle.io.DataLoader", +) def fetch(): - """download the entire dataset. - """ - paddle.v4.dataset.common.download(DATA_URL, "wmt16", DATA_MD5, - "wmt16.tar.gz") + """download the entire dataset.""" + paddle.v4.dataset.common.download( + DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz" + ) diff --git a/python/paddle/device/__init__.py b/python/paddle/device/__init__.py index e9894309999a4..6e14fd504784e 100644 --- a/python/paddle/device/__init__.py +++ b/python/paddle/device/__init__.py @@ -199,39 +199,49 @@ def _convert_to_place(device): place = core.CPUPlace() elif lower_device == 'gpu': if not core.is_compiled_with_cuda(): - raise ValueError("The device should not be 'gpu', " - "since PaddlePaddle is not compiled with CUDA") + raise ValueError( + "The device should not be 'gpu', " + "since PaddlePaddle is not compiled with CUDA" + ) place = core.CUDAPlace(ParallelEnv().dev_id) elif lower_device == 'xpu': if not core.is_compiled_with_xpu(): - raise ValueError("The device should not be 'xpu', " - "since PaddlePaddle is not compiled with XPU") + raise ValueError( + "The device should not be 'xpu', " + "since PaddlePaddle is not compiled with XPU" + ) selected_xpus = os.getenv("FLAGS_selected_xpus", "0").split(",") device_id = int(selected_xpus[0]) place = core.XPUPlace(device_id) elif lower_device == 'npu': if not core.is_compiled_with_npu(): - raise ValueError("The device should not be 'npu', " - "since PaddlePaddle is not compiled with NPU") + raise ValueError( + "The device should not be 'npu', " + "since PaddlePaddle is not compiled with NPU" + ) selected_npus = os.getenv("FLAGS_selected_npus", "0").split(",") device_id = int(selected_npus[0]) place = core.NPUPlace(device_id) elif lower_device == 'ipu': if not core.is_compiled_with_ipu(): raise ValueError( - "The device should not be 'ipu', " \ - "since PaddlePaddle is not compiled with IPU") + "The device should not be 'ipu', " + "since PaddlePaddle is not compiled with IPU" + ) place = core.IPUPlace() elif lower_device == 'mlu': if not core.is_compiled_with_mlu(): - raise ValueError("The device should not be 'mlu', " - "since PaddlePaddle is not compiled with MLU") + raise ValueError( + "The device should not be 'mlu', " + "since PaddlePaddle is not compiled with MLU" + ) selected_mlus = os.getenv("FLAGS_selected_mlus", "0").split(",") device_id = int(selected_mlus[0]) place = core.MLUPlace(device_id) elif device in core.get_all_custom_device_type(): - selected_devices = os.getenv("FLAGS_selected_{}s".format(device), - "0").split(",") + selected_devices = os.getenv( + "FLAGS_selected_{}s".format(device), "0" + ).split(",") device_id = int(selected_devices[0]) place = core.CustomPlace(device, device_id) else: @@ -239,7 +249,12 @@ def _convert_to_place(device): avaliable_xpu_device = re.match(r'xpu:\d+', lower_device) avaliable_npu_device = re.match(r'npu:\d+', lower_device) avaliable_mlu_device = re.match(r'mlu:\d+', lower_device) - if not avaliable_gpu_device and not avaliable_xpu_device and not avaliable_npu_device and not avaliable_mlu_device: + if ( + not avaliable_gpu_device + and not avaliable_xpu_device + and not avaliable_npu_device + and not avaliable_mlu_device + ): device_info_list = device.split(':', 1) device_type = device_info_list[0] if device_type in core.get_all_custom_device_type(): @@ -248,15 +263,20 @@ def _convert_to_place(device): place = core.CustomPlace(device_type, device_id) else: raise ValueError( - "The device must be a string which is like 'cpu', {}". - format(', '.join("'{}', '{}:x'".format(x, x) - for x in ['gpu', 'xpu', 'npu', 'mlu'] + - core.get_all_custom_device_type()))) + "The device must be a string which is like 'cpu', {}".format( + ', '.join( + "'{}', '{}:x'".format(x, x) + for x in ['gpu', 'xpu', 'npu', 'mlu'] + + core.get_all_custom_device_type() + ) + ) + ) if avaliable_gpu_device: if not core.is_compiled_with_cuda(): raise ValueError( "The device should not be {}, since PaddlePaddle is " - "not compiled with CUDA".format(avaliable_gpu_device)) + "not compiled with CUDA".format(avaliable_gpu_device) + ) device_info_list = device.split(':', 1) device_id = device_info_list[1] device_id = int(device_id) @@ -265,7 +285,8 @@ def _convert_to_place(device): if not core.is_compiled_with_xpu(): raise ValueError( "The device should not be {}, since PaddlePaddle is " - "not compiled with XPU".format(avaliable_xpu_device)) + "not compiled with XPU".format(avaliable_xpu_device) + ) device_info_list = device.split(':', 1) device_id = device_info_list[1] device_id = int(device_id) @@ -274,7 +295,8 @@ def _convert_to_place(device): if not core.is_compiled_with_npu(): raise ValueError( "The device should not be {}, since PaddlePaddle is " - "not compiled with NPU".format(avaliable_npu_device)) + "not compiled with NPU".format(avaliable_npu_device) + ) device_info_list = device.split(':', 1) device_id = device_info_list[1] device_id = int(device_id) @@ -283,7 +305,8 @@ def _convert_to_place(device): if not core.is_compiled_with_mlu(): raise ValueError( "The device should not be {}, since PaddlePaddle is " - "not compiled with mlu".format(avaliable_mlu_device)) + "not compiled with mlu".format(avaliable_mlu_device) + ) device_info_list = device.split(':', 1) device_id = device_info_list[1] device_id = int(device_id) diff --git a/python/paddle/device/cuda/__init__.py b/python/paddle/device/cuda/__init__.py index fa0f194969bc8..316f9de612265 100644 --- a/python/paddle/device/cuda/__init__.py +++ b/python/paddle/device/cuda/__init__.py @@ -124,8 +124,11 @@ def device_count(): ''' - num_gpus = core.get_cuda_device_count() if hasattr( - core, 'get_cuda_device_count') else 0 + num_gpus = ( + core.get_cuda_device_count() + if hasattr(core, 'get_cuda_device_count') + else 0 + ) return num_gpus @@ -165,7 +168,7 @@ def extract_cuda_device_id(device, op_name): Return: int: The id of the given device. If device is None, return the id of current device. ''' - if (device is None): + if device is None: return core.get_cuda_current_device_id() if isinstance(device, int): @@ -178,15 +181,19 @@ def extract_cuda_device_id(device, op_name): else: raise ValueError( "The current string {} is not expected. Because {} only support string which is like 'gpu:x'. " - "Please input appropriate string again!".format( - device, op_name)) + "Please input appropriate string again!".format(device, op_name) + ) else: raise ValueError( "The device type {} is not expected. Because {} only support int, str or paddle.CUDAPlace. " - "Please input appropriate device again!".format(device, op_name)) + "Please input appropriate device again!".format(device, op_name) + ) - assert device_id >= 0, f"The device id must be not less than 0, but got id = {device_id}." - assert device_id < device_count( + assert ( + device_id >= 0 + ), f"The device id must be not less than 0, but got id = {device_id}." + assert ( + device_id < device_count() ), f"The device id {device_id} exceeds gpu card number {device_count()}" return device_id @@ -424,7 +431,8 @@ def get_device_properties(device=None): raise ValueError( "The API paddle.device.cuda.get_device_properties is not supported in " "CPU-only PaddlePaddle. Please reinstall PaddlePaddle with GPU support " - "to call this API.") + "to call this API." + ) if device is not None: if isinstance(device, int): @@ -438,12 +446,14 @@ def get_device_properties(device=None): raise ValueError( "The current string {} is not expected. Because paddle.device." "cuda.get_device_properties only support string which is like 'gpu:x'. " - "Please input appropriate string again!".format(device)) + "Please input appropriate string again!".format(device) + ) else: raise ValueError( "The device type {} is not expected. Because paddle.device.cuda." "get_device_properties only support int, str or paddle.CUDAPlace. " - "Please input appropriate device again!".format(device)) + "Please input appropriate device again!".format(device) + ) else: device_id = -1 diff --git a/python/paddle/device/cuda/graphs.py b/python/paddle/device/cuda/graphs.py index 44ce2c5eea7d0..ea9962f22997c 100644 --- a/python/paddle/device/cuda/graphs.py +++ b/python/paddle/device/cuda/graphs.py @@ -16,7 +16,11 @@ import paddle from paddle.fluid import core from paddle.fluid.layers.utils import _hash_with_id -from paddle.fluid.core import is_compiled_with_cuda, is_compiled_with_rocm, CUDAPlace +from paddle.fluid.core import ( + is_compiled_with_cuda, + is_compiled_with_rocm, + CUDAPlace, +) import warnings if is_compiled_with_cuda() and not is_compiled_with_rocm(): @@ -24,6 +28,7 @@ def is_cuda_graph_supported(): return True + else: CoreCUDAGraph = None @@ -36,9 +41,10 @@ def is_cuda_graph_supported(): class CUDAGraph: - def __init__(self, place=None, mode="thread_local"): - assert CoreCUDAGraph is not None, "CUDA Graph is only supported on PaddlePaddle compiled with NVIDIA GPU." + assert ( + CoreCUDAGraph is not None + ), "CUDA Graph is only supported on PaddlePaddle compiled with NVIDIA GPU." self._graph = None if place is None: @@ -65,7 +71,8 @@ def print_to_dot_files(self, dirname, flags=None): dirname = dirname.name os.makedirs(name=dirname, exist_ok=True) assert os.path.isdir( - dirname), "The dirname {} should be a directory".format(dirname) + dirname + ), "The dirname {} should be a directory".format(dirname) if flags is None: flags = 2047 # only all information. It can be any integer inside [1, 2048) self._graph.print_to_dot_files(dirname, flags) @@ -76,6 +83,7 @@ def wrap_cuda_graph(function, mode="thread_local", memory_pool="default"): if not paddle.in_dynamic_mode(): # static mode from paddle.fluid.framework import _cuda_graph_guard + global cuda_graph_id graph_id = str(cuda_graph_id) cuda_graph_id += 1 @@ -86,13 +94,15 @@ def wrap_cuda_graph(function, mode="thread_local", memory_pool="default"): else: raise ValueError( "memory_pool should be one of default or new under static mode, but got", - memory_pool) + memory_pool, + ) return _cuda_graph_guard( - mode + ';' + str(memory_pool_id) + ';' + - graph_id)(lambda *args, **kwargs: function(*args, **kwargs)) + mode + ';' + str(memory_pool_id) + ';' + graph_id + )(lambda *args, **kwargs: function(*args, **kwargs)) from paddle.jit import to_static from paddle.nn import Layer + new_function = to_static(function) if isinstance(function, Layer): mock_func = new_function.forward @@ -105,7 +115,9 @@ def wrap_cuda_graph(function, mode="thread_local", memory_pool="default"): mock_func._cuda_graph_pool_id = CoreCUDAGraph.gen_new_memory_pool_id() else: if isinstance(memory_pool, Layer): - mock_func._cuda_graph_pool_id = memory_pool.forward._cuda_graph_pool_id + mock_func._cuda_graph_pool_id = ( + memory_pool.forward._cuda_graph_pool_id + ) else: mock_func._cuda_graph_pool_id = memory_pool._cuda_graph_pool_id return new_function @@ -224,16 +236,21 @@ def get_cuda_graph_sections(program): for idx, op in enumerate(block.ops): if op.type == 'conditional_block' or op.type == 'while': - assert op._cuda_graph_attr is None, "Cuda graph not support conditional block op and while op." + assert ( + op._cuda_graph_attr is None + ), "Cuda graph not support conditional block op and while op." if op.has_attr('is_test') and op.attr('is_test'): is_test = True # find cuda graph sections if op._cuda_graph_attr is not None: - assert isinstance(op._cuda_graph_attr, - str), "cuda_graph_attr should be a str" + assert isinstance( + op._cuda_graph_attr, str + ), "cuda_graph_attr should be a str" cuda_graph_attrs = op._cuda_graph_attr.split(';') - assert len(cuda_graph_attrs) == 3, "cuda graph attr should have three fields: " \ - "cuda graph mode, cuda graph memory pool id, cuda graph id" + assert len(cuda_graph_attrs) == 3, ( + "cuda graph attr should have three fields: " + "cuda graph mode, cuda graph memory pool id, cuda graph id" + ) local_cuda_graph_id = int(cuda_graph_attrs[2]) if local_cuda_graph_id == current_cuda_graph_id: if len(internal_section) > 0: @@ -241,12 +258,16 @@ def get_cuda_graph_sections(program): internal_idx ), "len of internal section should be equal with len of internal idx" for internal_op in internal_section: - loss_related = (int(internal_op.attr(op_role_attr_name)) - == loss_op_role) or int( - (internal_op.attr(op_role_attr_name) - ) == loss_grad_op_role) - sub_block_related = (op.type == 'conditional_block' - or op.type == 'while') + loss_related = ( + int(internal_op.attr(op_role_attr_name)) + == loss_op_role + ) or int( + (internal_op.attr(op_role_attr_name)) + == loss_grad_op_role + ) + sub_block_related = ( + op.type == 'conditional_block' or op.type == 'while' + ) if loss_related or sub_block_related: # If loss_related is True # The internal section contains loss related ops, @@ -260,8 +281,9 @@ def get_cuda_graph_sections(program): internal_section = [] internal_idx = [] # Beside clear the internal section, a new cuda graph section should be recorded - assert len(current_section) == len(current_idx), \ - "num of section's op is not equal with the idx" + assert len(current_section) == len( + current_idx + ), "num of section's op is not equal with the idx" if len(current_section) > 0: # store previous section cuda_graph_sections.append(current_section) @@ -282,7 +304,9 @@ def get_cuda_graph_sections(program): # internal ops and idx belong to no section, just clear it internal_section = [] internal_idx = [] - current_cuda_graph_id = local_cuda_graph_id # start record a new section + current_cuda_graph_id = ( + local_cuda_graph_id # start record a new section + ) assert len(current_section) == len( current_idx ), "num of section's op is not equal with num of idx" @@ -299,7 +323,8 @@ def get_cuda_graph_sections(program): # handle the last section assert len(current_section) == len( - current_idx), "num of section's op is not equal with num of idx" + current_idx + ), "num of section's op is not equal with num of idx" if len(current_section) > 0: # store previous section cuda_graph_sections.append(current_section) @@ -308,9 +333,15 @@ def get_cuda_graph_sections(program): return cuda_graph_sections, sections_idx, is_test -def replace_cuda_graph_section(ins_and_outs, section_program, section_idx, - origin_program, cuda_graph_section, order, - is_test): +def replace_cuda_graph_section( + ins_and_outs, + section_program, + section_idx, + origin_program, + cuda_graph_section, + order, + is_test, +): """ Use section_program and ins_and_outs to initialize a run_program_op, and replace the section_idx marks ops in the origin program. @@ -344,8 +375,9 @@ def replace_cuda_graph_section(ins_and_outs, section_program, section_idx, memory_pool_id = int(attrs[1]) break - assert mode is not None and memory_pool_id is not None, \ - "mode and memory pool id should be specified in cuda graph attr" + assert ( + mode is not None and memory_pool_id is not None + ), "mode and memory pool id should be specified in cuda graph attr" cuda_graph_var = origin_block.create_var( name="cuda_graph_" + str(order), @@ -372,7 +404,7 @@ def replace_cuda_graph_section(ins_and_outs, section_program, section_idx, outputs={ 'Out': outs, 'OutScope': out_scope_var, - 'CUDAGraph': cuda_graph_var + 'CUDAGraph': cuda_graph_var, }, attrs={ 'global_block': section_program.global_block(), @@ -386,7 +418,8 @@ def replace_cuda_graph_section(ins_and_outs, section_program, section_idx, 'use_interpretorcore': False, 'forward_global_block': section_program.global_block(), 'backward_global_block': section_program.global_block(), - }) + }, + ) def cuda_graph_transform(program): @@ -408,9 +441,11 @@ def cuda_graph_transform(program): # A cuda graph section contains all ops marked with same cuda graph id and # some ops inserted by some optimizers (amp, sharding for example) between ops with same id. cuda_graph_sections, sections_idx, is_test = get_cuda_graph_sections( - program) - assert len(cuda_graph_sections) == len(sections_idx), \ - "num of cuda graph sections is not equal with num of idx sections" + program + ) + assert len(cuda_graph_sections) == len( + sections_idx + ), "num of cuda graph sections is not equal with num of idx sections" # step 2: construct new program for each section and find inputs and outputs of each section. # The inputs are variables generated outside the section but will be used by this section. @@ -420,23 +455,27 @@ def cuda_graph_transform(program): for i in range(len(cuda_graph_sections)): # creating new program for current section section_program, ins_outs = construct_program_and_find_ins_outs( - cuda_graph_sections[i], program, sections_idx[i]) + cuda_graph_sections[i], program, sections_idx[i] + ) ins_and_outs.append(ins_outs) section_programs.append(section_program) - assert len(section_programs) == len(cuda_graph_sections), \ - "the num of cuda graph sections should be equal with the num of new program" + assert len(section_programs) == len( + cuda_graph_sections + ), "the num of cuda graph sections should be equal with the num of new program" # step 3: replace the ops in original program with run_program_op. # Will remove all ops in the section from origin program, and use run_program_op to replace them. for i in reversed(range(len(cuda_graph_sections))): # carry out the replacement in reversed order, to keep the previous idx intact - replace_cuda_graph_section(ins_and_outs[i], - section_programs[i], - sections_idx[i], - program, - cuda_graph_sections[i], - order=i, - is_test=is_test) + replace_cuda_graph_section( + ins_and_outs[i], + section_programs[i], + sections_idx[i], + program, + cuda_graph_sections[i], + order=i, + is_test=is_test, + ) # NOTE: user should hold these program, for now just return these program back to caller return section_programs diff --git a/python/paddle/distributed/__init__.py b/python/paddle/distributed/__init__.py index 13c305b58da73..2bce21be7cc61 100644 --- a/python/paddle/distributed/__init__.py +++ b/python/paddle/distributed/__init__.py @@ -72,12 +72,42 @@ from . import rpc __all__ = [ # noqa - "spawn", "launch", "scatter", "broadcast", "ParallelEnv", "new_group", - "init_parallel_env", "gloo_init_parallel_env", "gloo_barrier", - "gloo_release", "QueueDataset", "split", "CountFilterEntry", - "ShowClickEntry", "get_world_size", "get_group", "all_gather", - "all_gather_object", "InMemoryDataset", "barrier", "all_reduce", "alltoall", - "alltoall_single", "send", "reduce", "recv", "ReduceOp", "wait", "get_rank", - "ProbabilityEntry", "ParallelMode", "is_initialized", - "destroy_process_group", "isend", "irecv", "reduce_scatter", "rpc", "stream" + "spawn", + "launch", + "scatter", + "broadcast", + "ParallelEnv", + "new_group", + "init_parallel_env", + "gloo_init_parallel_env", + "gloo_barrier", + "gloo_release", + "QueueDataset", + "split", + "CountFilterEntry", + "ShowClickEntry", + "get_world_size", + "get_group", + "all_gather", + "all_gather_object", + "InMemoryDataset", + "barrier", + "all_reduce", + "alltoall", + "alltoall_single", + "send", + "reduce", + "recv", + "ReduceOp", + "wait", + "get_rank", + "ProbabilityEntry", + "ParallelMode", + "is_initialized", + "destroy_process_group", + "isend", + "irecv", + "reduce_scatter", + "rpc", + "stream", ] diff --git a/python/paddle/distributed/auto_parallel/callbacks.py b/python/paddle/distributed/auto_parallel/callbacks.py index 17ce5bd71b816..26371cc4ccb91 100644 --- a/python/paddle/distributed/auto_parallel/callbacks.py +++ b/python/paddle/distributed/auto_parallel/callbacks.py @@ -16,22 +16,30 @@ import time import paddle -from paddle.hapi.callbacks import ProgBarLogger, ModelCheckpoint, LRScheduler, CallbackList, Callback +from paddle.hapi.callbacks import ( + ProgBarLogger, + ModelCheckpoint, + LRScheduler, + CallbackList, + Callback, +) from .interface import CollectionNames, get_collection -def config_callbacks(callbacks=None, - engine=None, - batch_size=None, - epochs=None, - steps=None, - log_freq=2, - verbose=2, - save_freq=1, - save_dir=None, - metrics=None, - acc_step=1, - mode='train'): +def config_callbacks( + callbacks=None, + engine=None, + batch_size=None, + epochs=None, + steps=None, + log_freq=2, + verbose=2, + save_freq=1, + save_dir=None, + metrics=None, + acc_step=1, + mode='train', +): cbks = callbacks or [] cbks = cbks if isinstance(cbks, (list, tuple)) else [cbks] @@ -74,7 +82,6 @@ def config_callbacks(callbacks=None, class ProgBarLoggerAuto(ProgBarLogger): - def __init__(self, log_freq=1, verbose=2): super(ProgBarLoggerAuto, self).__init__(log_freq, verbose) @@ -110,16 +117,22 @@ def _updates(self, logs, mode): cnt = timer['count'] if timer['count'] > 0 else 1.0 samples = timer['samples'] if timer['samples'] > 0 else 1.0 values.append( - ('avg_reader_cost', "%.5f sec" % (timer['data_time'] / cnt))) + ('avg_reader_cost', "%.5f sec" % (timer['data_time'] / cnt)) + ) values.append( - ('avg_batch_cost', "%.5f sec" % (timer['batch_time'] / cnt))) + ('avg_batch_cost', "%.5f sec" % (timer['batch_time'] / cnt)) + ) values.append( - ('ips', "%.5f samples/sec" % - (samples / (timer['data_time'] + timer['batch_time'])))) + ( + 'ips', + "%.5f samples/sec" + % (samples / (timer['data_time'] + timer['batch_time'])), + ) + ) timer['count'] = 0 timer['samples'] = 0 - timer['data_time'] = 0. - timer['batch_time'] = 0. + timer['data_time'] = 0.0 + timer['batch_time'] = 0.0 progbar.update(steps, values) @@ -130,7 +143,8 @@ def on_eval_batch_end(self, step, logs=None): self.evaled_samples += samples self._eval_timer['batch_time'] += ( - time.time() - self._eval_timer['batch_data_end_time']) + time.time() - self._eval_timer['batch_data_end_time'] + ) self._eval_timer['count'] += 1 samples = self.params['batch_size'] self._eval_timer['samples'] += samples @@ -143,7 +157,6 @@ def on_eval_batch_end(self, step, logs=None): class LRSchedulerAuto(LRScheduler): - def __init__(self, by_step=True, by_epoch=False): super(LRSchedulerAuto, self).__init__(by_step, by_epoch) @@ -156,15 +169,18 @@ def on_train_batch_end(self, step, logs=None): self.train_step += 1 if self.by_step and self.train_step % self.acc_step == 0: - if self.model._optimizer and \ - hasattr(self.model._optimizer, '_learning_rate') and \ - isinstance(self.model._optimizer._learning_rate, - paddle.optimizer.lr.LRScheduler): + if ( + self.model._optimizer + and hasattr(self.model._optimizer, '_learning_rate') + and isinstance( + self.model._optimizer._learning_rate, + paddle.optimizer.lr.LRScheduler, + ) + ): self.model._optimizer._learning_rate.step() class History(Callback): - def __init__(self): self.history = {} @@ -181,7 +197,6 @@ def on_epoch_end(self, epoch, logs=None): class Profiler(Callback): - def __init__(self, *args, **kwargs): self.prof = paddle.profiler.Profiler(*args, **kwargs) @@ -197,8 +212,11 @@ def on_train_begin(self, logs=None): def on_train_batch_end(self, step, logs=None): self.train_step += 1 self.prof.step(num_samples=self.batch_size) - print("step {}:{}".format(self.train_step, - self.prof.step_info(unit='samples'))) + print( + "step {}:{}".format( + self.train_step, self.prof.step_info(unit='samples') + ) + ) def on_train_end(self, logs=None): self.prof.stop() @@ -206,7 +224,6 @@ def on_train_end(self, logs=None): class ModelCheckpointAuto(ModelCheckpoint): - def __init__(self, *args, **kwargs): super(ModelCheckpointAuto, self).__init__(*args, **kwargs) diff --git a/python/paddle/distributed/auto_parallel/cluster.py b/python/paddle/distributed/auto_parallel/cluster.py index e17f83eb41907..ca66a770e3c4c 100644 --- a/python/paddle/distributed/auto_parallel/cluster.py +++ b/python/paddle/distributed/auto_parallel/cluster.py @@ -128,8 +128,15 @@ def memory(self, value): def __str__(self): str = "" str += "global_id: {}, local_id: {}, machine_id: {}, type: {}, model: {}, dp_flops: {}, sp_flops: {}, memory: {}".format( - self.global_id, self.local_id, self.machine.id, self.type.name, - self.model, self.dp_gflops, self.sp_gflops, self.memory) + self.global_id, + self.local_id, + self.machine.id, + self.type.name, + self.model, + self.dp_gflops, + self.sp_gflops, + self.memory, + ) return str def __repr__(self): @@ -202,8 +209,12 @@ def hop(self, value): def __str__(self): str = "" str += "source_global_id: {}, target_global_id: {}, type: {}, bandwidth: {}, latency: {}".format( - self.source.global_id, self.target.global_id, self.type, - self.bandwidth, self.latency) + self.source.global_id, + self.target.global_id, + self.type, + self.bandwidth, + self.latency, + ) return str def __repr__(self): @@ -211,7 +222,6 @@ def __repr__(self): class Machine: - def __init__(self, id): self._id = id self._hostname = None @@ -292,7 +302,6 @@ def __repr__(self): class AlphaLatency: - def __init__(self, alpha_latency): assert isinstance(alpha_latency, dict) self._base = alpha_latency.get("base", None) @@ -304,12 +313,15 @@ def __init__(self, alpha_latency): self._switch = float(self._switch) except: raise TypeError("The switch latency must be float") - self._base_ring = self._base.get( - "ring", None) if self._base is not None else None - self._base_tree = self._base.get( - "tree", None) if self._base is not None else None - self._base_inter = self._base.get( - "inter", None) if self._base is not None else None + self._base_ring = ( + self._base.get("ring", None) if self._base is not None else None + ) + self._base_tree = ( + self._base.get("tree", None) if self._base is not None else None + ) + self._base_inter = ( + self._base.get("inter", None) if self._base is not None else None + ) if self._base_ring is not None: try: self._base_ring = float(self._base_ring) @@ -416,19 +428,21 @@ def __init__(self): # which have the same number accelerators. self._num_devices_per_machine = None - def gen_default_config_cluster(self, - gpu_model="V100", - cpu_model="6271C", - node_count=1, - device_count=1, - gpu_memory=32, - cpu_memory=503, - inter_bandwidth=24, - intra_bandwidth=235, - gpu_dp_gflops=7800, - gpu_sp_gflops=15700, - cpu_dp_gflops=75, - cpu_sp_gflops=150): + def gen_default_config_cluster( + self, + gpu_model="V100", + cpu_model="6271C", + node_count=1, + device_count=1, + gpu_memory=32, + cpu_memory=503, + inter_bandwidth=24, + intra_bandwidth=235, + gpu_dp_gflops=7800, + gpu_sp_gflops=15700, + cpu_dp_gflops=75, + cpu_sp_gflops=150, + ): """Generate cluster by default config.""" gpu_models = ["V100", "A100", "H100", "A2", "A10", "A16", "A30", "A40"] xpu_models = ["XPU"] @@ -602,25 +616,31 @@ def add_machine(self, machine): prev_machine = self._machines[machine.id - 1] offset = prev_machine._non_accelerator_cumulative_count for global_id in machine.devices: - if machine.devices[ - global_id].type not in Device.NON_ACCELERATOR_TYPE: + if ( + machine.devices[global_id].type + not in Device.NON_ACCELERATOR_TYPE + ): rank_id = global_id - offset self._rank_to_device_id[rank_id] = global_id self._device_id_to_rank[global_id] = rank_id - machine._non_accelerator_cumulative_count = len( - machine.devices) - len( - machine.accelerators - ) + prev_machine._non_accelerator_cumulative_count + machine._non_accelerator_cumulative_count = ( + len(machine.devices) + - len(machine.accelerators) + + prev_machine._non_accelerator_cumulative_count + ) else: for global_id in machine.devices: - if machine.devices[ - global_id].type not in Device.NON_ACCELERATOR_TYPE: + if ( + machine.devices[global_id].type + not in Device.NON_ACCELERATOR_TYPE + ): rank_id = global_id self._rank_to_device_id[rank_id] = global_id self._device_id_to_rank[global_id] = rank_id machine.accelerators[global_id] = machine.devices[global_id] machine._non_accelerator_cumulative_count = len( - machine.devices) - len(machine.accelerators) + machine.devices + ) - len(machine.accelerators) @property def alpha_latency(self): @@ -696,7 +716,8 @@ def _build_from_dict(self, cluster_info): if "alpha_latency" in cluster_info: self._alpha_latency = AlphaLatency( - cluster_info.get("alpha_latency")) + cluster_info.get("alpha_latency") + ) else: self._alpha_latecy = None @@ -732,7 +753,7 @@ def get_beta(self, source_device_id, target_device_id): else: bandwidth = link.bandwidth - if bandwidth == 0.: + if bandwidth == 0.0: beta = 0 else: beta = 1 / (bandwidth * (convert_base**3 / 10**6)) @@ -813,13 +834,16 @@ def get_default_cluster(): global_device_count = int(global_device_count) assert global_device_count % local_device_count == 0 node_count = int(global_device_count) // local_device_count - print("Node Count: ", - node_count, - "Local Device Size: ", - local_device_count, - "World size: ", - paddle.distributed.get_world_size(), - flush=True) - cluster.gen_default_config_cluster(node_count=node_count, - device_count=local_device_count) + print( + "Node Count: ", + node_count, + "Local Device Size: ", + local_device_count, + "World size: ", + paddle.distributed.get_world_size(), + flush=True, + ) + cluster.gen_default_config_cluster( + node_count=node_count, device_count=local_device_count + ) return cluster diff --git a/python/paddle/distributed/auto_parallel/cluster_v2.py b/python/paddle/distributed/auto_parallel/cluster_v2.py index 866ab338ea264..debcb078f6eb4 100644 --- a/python/paddle/distributed/auto_parallel/cluster_v2.py +++ b/python/paddle/distributed/auto_parallel/cluster_v2.py @@ -75,10 +75,10 @@ class DeviceMesh(core.DeviceMesh): def __init__(self, name, mesh, dim_names=None): self._name = name - if not isinstance(mesh, list) and \ - not isinstance(mesh, np.ndarray): + if not isinstance(mesh, list) and not isinstance(mesh, np.ndarray): raise ValueError( - 'The mesh must be an instance of list or np.ndarray.') + 'The mesh must be an instance of list or np.ndarray.' + ) if isinstance(mesh, list): mesh = np.array(mesh) @@ -87,24 +87,29 @@ def __init__(self, name, mesh, dim_names=None): self._shape = list(self._mesh.shape) self._device_ids = self._mesh.flatten().tolist() - assert all(isinstance(p, int) for p in self._device_ids), \ - ("All elements of the mesh be integer") - assert min( - self._device_ids) >= 0, ('All elements of the mesh must be >= 0.') + assert all( + isinstance(p, int) for p in self._device_ids + ), "All elements of the mesh be integer" + assert ( + min(self._device_ids) >= 0 + ), 'All elements of the mesh must be >= 0.' unique_device_ids = set(self._device_ids) assert len(unique_device_ids) == len( - self._device_ids), ('All elements of the mesh must be unique.') + self._device_ids + ), 'All elements of the mesh must be unique.' if dim_names is not None: - assert len(dim_names) == len(self._shape), \ - ("The length of dims_names must be same as the shape of the mesh.") + assert len(dim_names) == len( + self._shape + ), "The length of dims_names must be same as the shape of the mesh." self._dim_names = dim_names else: self._dim_names = ["d" + str(i) for i in range(len(self._shape))] # Follow the requirement for using pybind11 - core.DeviceMesh.__init__(self, self._name, self._shape, - self._device_ids, self._dim_names) + core.DeviceMesh.__init__( + self, self._name, self._shape, self._device_ids, self._dim_names + ) @property def mesh(self): diff --git a/python/paddle/distributed/auto_parallel/completion.py b/python/paddle/distributed/auto_parallel/completion.py index db387ef2bb216..ba879e89f8dec 100644 --- a/python/paddle/distributed/auto_parallel/completion.py +++ b/python/paddle/distributed/auto_parallel/completion.py @@ -55,7 +55,8 @@ def _compute_compatible_process_mesh_two(pm1, pm2): compatible_result = None for process_mesh in process_mesh_list: compatible, compatible_result = _compute_compatible_process_mesh_two( - compatible_result, process_mesh) + compatible_result, process_mesh + ) if not compatible: return None return copy.deepcopy(compatible_result) @@ -78,7 +79,8 @@ def _compute_compatible_dim_mapping_two(dm1, dm2): compatible_result = -1 for mapping in dim_mapping_list: compatible, compatible_result = _compute_compatible_dim_mapping_two( - compatible_result, mapping) + compatible_result, mapping + ) if not compatible: return None return compatible_result @@ -86,7 +88,7 @@ def _compute_compatible_dim_mapping_two(dm1, dm2): def compute_compatible_dims_mapping(dims_mapping_list): """Compute the compatible dims mapping given a list of dims mapping. - Each of dims mapping is also a list. + Each of dims mapping is also a list. """ if not dims_mapping_list: return None @@ -99,7 +101,8 @@ def compute_compatible_dims_mapping(dims_mapping_list): compatible_result = [] for dim_mappings in zip(*dims_mapping_list): compatible_dim_mapping = compute_compatible_dim_mapping( - list(dim_mappings)) + list(dim_mappings) + ) if compatible_dim_mapping is None: return None compatible_result.append(compatible_dim_mapping) @@ -125,7 +128,8 @@ def _validate_dims_mapping(dims_mapping, process_mesh): return False for i in range(len(dims_mapping)): if dims_mapping[i] < -1 or dims_mapping[i] >= len( - process_mesh.topology): + process_mesh.topology + ): return False for i in range(len(process_mesh.topology)): if dims_mapping.count(i) > 1: @@ -134,7 +138,6 @@ def _validate_dims_mapping(dims_mapping, process_mesh): class Completer: - def __init__(self, dist_context): assert dist_context is not None self._dist_context = dist_context @@ -146,12 +149,15 @@ def _update_tensor_node_dims_mapping(self, tensor_node, fwd=True): return False tensor_desc = tensor_node.var() # Skip reader tensor - if tensor_desc.type() == core.VarDesc.VarType.READER \ - or tensor_desc.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or tensor_desc.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + tensor_desc.type() == core.VarDesc.VarType.READER + or tensor_desc.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or tensor_desc.type == core.VarDesc.VarType.STEP_SCOPES + ): return False tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) + tensor_node + ) assert tensor_dist_attr is not None if tensor_dist_attr.is_annotated("dims_mapping"): return False @@ -160,48 +166,74 @@ def _update_tensor_node_dims_mapping(self, tensor_node, fwd=True): dims_mapping_list = [] for pred_op_node in tensor_node.inputs: if pred_op_node.op() is not None: - if pred_op_node.op().type() == "create_py_reader" \ - or pred_op_node.op().type() == "create_double_buffer_reader" \ - or pred_op_node.op().type() == "read": + if ( + pred_op_node.op().type() == "create_py_reader" + or pred_op_node.op().type() + == "create_double_buffer_reader" + or pred_op_node.op().type() == "read" + ): continue - op_dist_attr = self._dist_context.get_op_dist_attr_for_graph( - pred_op_node) - if op_dist_attr.process_mesh == tensor_dist_attr.process_mesh: + op_dist_attr = ( + self._dist_context.get_op_dist_attr_for_graph( + pred_op_node + ) + ) + if ( + op_dist_attr.process_mesh + == tensor_dist_attr.process_mesh + ): op_dims_mapping = op_dist_attr.get_output_dims_mapping( - tensor_desc.name()) + tensor_desc.name() + ) dims_mapping_list.append(op_dims_mapping) dims_mapping_list.append(tensor_dims_mapping) compatible_dims_mapping = compute_compatible_dims_mapping( - dims_mapping_list) - if not _validate_dims_mapping(compatible_dims_mapping, - tensor_dist_attr.process_mesh): + dims_mapping_list + ) + if not _validate_dims_mapping( + compatible_dims_mapping, tensor_dist_attr.process_mesh + ): return False - if (compatible_dims_mapping is not None) and \ - (compatible_dims_mapping != tensor_dims_mapping): + if (compatible_dims_mapping is not None) and ( + compatible_dims_mapping != tensor_dims_mapping + ): tensor_dist_attr.dims_mapping = compatible_dims_mapping changed = True else: dims_mapping_list = [] for succ_op_node in tensor_node.outputs: if succ_op_node.op() is not None: - if succ_op_node.op().type() == "create_py_reader" \ - or succ_op_node.op().type() == "create_double_buffer_reader" \ - or succ_op_node.op().type() == "read": + if ( + succ_op_node.op().type() == "create_py_reader" + or succ_op_node.op().type() + == "create_double_buffer_reader" + or succ_op_node.op().type() == "read" + ): continue - op_dist_attr = self._dist_context.get_op_dist_attr_for_graph( - succ_op_node) - if op_dist_attr.process_mesh == tensor_dist_attr.process_mesh: + op_dist_attr = ( + self._dist_context.get_op_dist_attr_for_graph( + succ_op_node + ) + ) + if ( + op_dist_attr.process_mesh + == tensor_dist_attr.process_mesh + ): op_dims_mapping = op_dist_attr.get_input_dims_mapping( - tensor_desc.name()) + tensor_desc.name() + ) dims_mapping_list.append(op_dims_mapping) dims_mapping_list.append(tensor_dims_mapping) compatible_dims_mapping = compute_compatible_dims_mapping( - dims_mapping_list) - if not _validate_dims_mapping(compatible_dims_mapping, - tensor_dist_attr.process_mesh): + dims_mapping_list + ) + if not _validate_dims_mapping( + compatible_dims_mapping, tensor_dist_attr.process_mesh + ): return False - if (compatible_dims_mapping is not None) and \ - (compatible_dims_mapping != tensor_dims_mapping): + if (compatible_dims_mapping is not None) and ( + compatible_dims_mapping != tensor_dims_mapping + ): tensor_dist_attr.dims_mapping = compatible_dims_mapping changed = True return changed @@ -212,10 +244,12 @@ def _update_op_node_dims_mapping(self, op_node, fwd=True): return False # Skip reader op op_desc = op_node.op() - if op_desc.type() == "create_py_reader" \ - or op_desc.type() == "create_double_buffer_reader" \ - or op_desc.type() == "while" \ - or op_desc.type() == "read": + if ( + op_desc.type() == "create_py_reader" + or op_desc.type() == "create_double_buffer_reader" + or op_desc.type() == "while" + or op_desc.type() == "read" + ): return False dist_op = self._dist_context.get_dist_op_for_graph(op_node) op_dist_attr = dist_op.dist_attr @@ -227,28 +261,42 @@ def _update_op_node_dims_mapping(self, op_node, fwd=True): continue tensor_desc = tensor_node.var() if op_dist_attr.is_annotated_input_dims_mapping( - tensor_desc.name()): + tensor_desc.name() + ): continue - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) - if op_dist_attr.process_mesh == tensor_dist_attr.process_mesh: + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph( + tensor_node + ) + ) + if ( + op_dist_attr.process_mesh + == tensor_dist_attr.process_mesh + ): tensor_dims_mapping = tensor_dist_attr.dims_mapping op_dims_mapping = op_dist_attr.get_input_dims_mapping( - tensor_desc.name()) - compatible_dims_mapping = compute_compatible_dims_mapping( - [op_dims_mapping, tensor_dims_mapping]) + tensor_desc.name() + ) + compatible_dims_mapping = ( + compute_compatible_dims_mapping( + [op_dims_mapping, tensor_dims_mapping] + ) + ) if not _validate_dims_mapping( - compatible_dims_mapping, - op_dist_attr.process_mesh): + compatible_dims_mapping, op_dist_attr.process_mesh + ): continue - if (compatible_dims_mapping is not None) and \ - (compatible_dims_mapping != op_dims_mapping): + if (compatible_dims_mapping is not None) and ( + compatible_dims_mapping != op_dims_mapping + ): op_dist_attr.set_input_dims_mapping( - tensor_desc.name(), compatible_dims_mapping) + tensor_desc.name(), compatible_dims_mapping + ) changed = True # Find the most compatible implemenetations from the distributed operator - op_dist_impls = find_compatible_distributed_operator_impls(dist_op, - fwd=True) + op_dist_impls = find_compatible_distributed_operator_impls( + dist_op, fwd=True + ) if op_dist_impls is not None: not_compatible = True backup_op_dist_attr = copy.deepcopy(op_dist_attr) @@ -257,8 +305,10 @@ def _update_op_node_dims_mapping(self, op_node, fwd=True): dim_changed = op_dist_impl.update_dims_mapping(dist_op) if dim_changed: changed = True - if op_dist_impl.is_auto_compatible(dist_op) \ - and dist_op.validate_dist_attr(): + if ( + op_dist_impl.is_auto_compatible(dist_op) + and dist_op.validate_dist_attr() + ): if op_dist_impl.type == "elementwise": op_dist_attr.impl_type = "default" else: @@ -283,28 +333,42 @@ def _update_op_node_dims_mapping(self, op_node, fwd=True): continue tensor_desc = tensor_node.var() if op_dist_attr.is_annotated_output_dims_mapping( - tensor_desc.name()): + tensor_desc.name() + ): continue - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) - if op_dist_attr.process_mesh == tensor_dist_attr.process_mesh: + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph( + tensor_node + ) + ) + if ( + op_dist_attr.process_mesh + == tensor_dist_attr.process_mesh + ): tensor_dims_mapping = tensor_dist_attr.dims_mapping op_dims_mapping = op_dist_attr.get_output_dims_mapping( - tensor_desc.name()) - compatible_dims_mapping = compute_compatible_dims_mapping( - [op_dims_mapping, tensor_dims_mapping]) + tensor_desc.name() + ) + compatible_dims_mapping = ( + compute_compatible_dims_mapping( + [op_dims_mapping, tensor_dims_mapping] + ) + ) if not _validate_dims_mapping( - compatible_dims_mapping, - op_dist_attr.process_mesh): + compatible_dims_mapping, op_dist_attr.process_mesh + ): continue - if (compatible_dims_mapping is not None) and \ - (compatible_dims_mapping != op_dims_mapping): + if (compatible_dims_mapping is not None) and ( + compatible_dims_mapping != op_dims_mapping + ): op_dist_attr.set_output_dims_mapping( - tensor_desc.name(), compatible_dims_mapping) + tensor_desc.name(), compatible_dims_mapping + ) changed = True # Find the most compatible implemenetations from the distributed operator op_dist_impls = find_compatible_distributed_operator_impls( - dist_op, fwd=False) + dist_op, fwd=False + ) if op_dist_impls is not None: not_compatible = True backup_op_dist_attr = copy.deepcopy(op_dist_attr) @@ -313,8 +377,10 @@ def _update_op_node_dims_mapping(self, op_node, fwd=True): dim_changed = op_dist_impl.update_dims_mapping(dist_op) if dim_changed: changed = True - if op_dist_impl.is_auto_compatible(dist_op) \ - and dist_op.validate_dist_attr(): + if ( + op_dist_impl.is_auto_compatible(dist_op) + and dist_op.validate_dist_attr() + ): if op_dist_impl.type == "elementwise": op_dist_attr.impl_type = "default" else: @@ -338,24 +404,33 @@ def _update_dims_mapping_between_graphs(self): changed = False for parent_node, child_node in self._node_pairs_between_graphs: parent_node_dist_attr = self._dist_context.get_dist_attr_for_graph( - parent_node) + parent_node + ) child_node_dist_attr = self._dist_context.get_dist_attr_for_graph( - child_node) - if parent_node_dist_attr.process_mesh != child_node_dist_attr.process_mesh: + child_node + ) + if ( + parent_node_dist_attr.process_mesh + != child_node_dist_attr.process_mesh + ): continue parent_node_dims_mapping = parent_node_dist_attr.dims_mapping child_node_dims_mapping = child_node_dist_attr.dims_mapping compatible_dims_mapping = compute_compatible_dims_mapping( - [parent_node_dims_mapping, child_node_dims_mapping]) - if not _validate_dims_mapping(compatible_dims_mapping, - parent_node_dist_attr.process_mesh): + [parent_node_dims_mapping, child_node_dims_mapping] + ) + if not _validate_dims_mapping( + compatible_dims_mapping, parent_node_dist_attr.process_mesh + ): return False - if (compatible_dims_mapping is not None) \ - and (compatible_dims_mapping != parent_node_dims_mapping): + if (compatible_dims_mapping is not None) and ( + compatible_dims_mapping != parent_node_dims_mapping + ): parent_node_dist_attr.dims_mapping = compatible_dims_mapping changed = True - if (compatible_dims_mapping is not None) \ - and (compatible_dims_mapping != child_node_dims_mapping): + if (compatible_dims_mapping is not None) and ( + compatible_dims_mapping != child_node_dims_mapping + ): child_node_dist_attr.dims_mapping = compatible_dims_mapping changed = True return changed @@ -365,11 +440,15 @@ def _update_dims_mapping_for_special(self): op_nodes = self._dist_context._serial_ordered_op_nodes # NOTE: this list may be changed if Paddle changes the existing rules. related_reader_ops = [ - "create_py_reader", "create_double_buffer_reader", "read" + "create_py_reader", + "create_double_buffer_reader", + "read", ] for op_node in op_nodes: - if op_node.op() is not None \ - and op_node.op().type() in related_reader_ops: + if ( + op_node.op() is not None + and op_node.op().type() in related_reader_ops + ): continue op_dist_attr = self._dist_context.get_dist_attr_for_graph(op_node) for tensor_node in op_node.outputs: @@ -377,11 +456,18 @@ def _update_dims_mapping_for_special(self): if tensor_node.var().type() == core.VarDesc.VarType.READER: continue tensor_desc = tensor_node.var() - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) - if op_dist_attr.process_mesh == tensor_dist_attr.process_mesh: + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph( + tensor_node + ) + ) + if ( + op_dist_attr.process_mesh + == tensor_dist_attr.process_mesh + ): op_dims_mapping = op_dist_attr.get_output_dims_mapping( - tensor_desc.name()) + tensor_desc.name() + ) tensor_dist_attr.dims_mapping = op_dims_mapping def _update_dims_mapping(self): @@ -390,17 +476,22 @@ def _update_dims_mapping(self): while not reach_fix_point: changed = False for is_fwd in [True, False]: - all_nodes = self._dist_context.serial_ordered_nodes \ - if is_fwd else reversed(self._dist_context.serial_ordered_nodes) + all_nodes = ( + self._dist_context.serial_ordered_nodes + if is_fwd + else reversed(self._dist_context.serial_ordered_nodes) + ) for node in all_nodes: if node.is_var() and node.var() is not None: tensor_changed = self._update_tensor_node_dims_mapping( - node, fwd=is_fwd) + node, fwd=is_fwd + ) if tensor_changed: changed = True if node.is_op() and node.op() is not None: op_changed = self._update_op_node_dims_mapping( - node, fwd=is_fwd) + node, fwd=is_fwd + ) if op_changed: changed = True graph_changed = self._update_dims_mapping_between_graphs() @@ -419,12 +510,16 @@ def _update_process_mesh_by_nearest(self, op_node, nearest_op_node): if not op_dist_attr.is_annotated("process_mesh"): process_mesh = op_dist_attr.process_mesh nearest_op_dis_attr = self._dist_context.get_dist_attr_for_graph( - nearest_op_node) + nearest_op_node + ) nearest_process_mesh = nearest_op_dis_attr.process_mesh compatible_process_mesh = compute_compatible_process_mesh( - [process_mesh, nearest_process_mesh]) - if compatible_process_mesh is not None \ - and process_mesh != compatible_process_mesh: + [process_mesh, nearest_process_mesh] + ) + if ( + compatible_process_mesh is not None + and process_mesh != compatible_process_mesh + ): op_dist_attr.process_mesh = compatible_process_mesh # Skip the process_mesh setting of inputs and outputs of while_op if op_dist_attr.op_type == "while": @@ -432,43 +527,60 @@ def _update_process_mesh_by_nearest(self, op_node, nearest_op_node): # Set the process mesh of the op node's leaf-inputs for tensor_node in op_node.inputs: if tensor_node.is_var() and tensor_node.var() is not None: - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph( + tensor_node + ) + ) if tensor_dist_attr.is_annotated("process_mesh"): continue # Skip the non-leaf var node if len(tensor_node.inputs) != 0: continue compatible_process_mesh = compute_compatible_process_mesh( - [tensor_dist_attr.process_mesh, op_dist_attr.process_mesh]) - if compatible_process_mesh is not None \ - and tensor_dist_attr.process_mesh != compatible_process_mesh: + [tensor_dist_attr.process_mesh, op_dist_attr.process_mesh] + ) + if ( + compatible_process_mesh is not None + and tensor_dist_attr.process_mesh != compatible_process_mesh + ): tensor_dist_attr.process_mesh = compatible_process_mesh # Set the process mesh of the op node's outputs for tensor_node in op_node.outputs: if tensor_node.is_var() and tensor_node.var() is not None: - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph( + tensor_node + ) + ) if tensor_dist_attr.is_annotated("process_mesh"): continue compatible_process_mesh = compute_compatible_process_mesh( - [tensor_dist_attr.process_mesh, op_dist_attr.process_mesh]) - if compatible_process_mesh is not None \ - and tensor_dist_attr.process_mesh != compatible_process_mesh: + [tensor_dist_attr.process_mesh, op_dist_attr.process_mesh] + ) + if ( + compatible_process_mesh is not None + and tensor_dist_attr.process_mesh != compatible_process_mesh + ): tensor_dist_attr.process_mesh = compatible_process_mesh def _update_process_mesh_for_specials(self): - def _find_nearest_tensor_node_before(nodes, idx, var_name): for node in reversed(nodes[:idx]): - if node.is_var() and node.var() is not None \ - and node.var().name() == var_name: + if ( + node.is_var() + and node.var() is not None + and node.var().name() == var_name + ): return node def _find_nearest_tensor_node_after(nodes, idx, var_name): - for node in nodes[idx + 1:]: - if node.is_var() and node.var() is not None \ - and node.var().name() == var_name: + for node in nodes[idx + 1 :]: + if ( + node.is_var() + and node.var() is not None + and node.var().name() == var_name + ): return node def _find_nodes_related_to_cond(source_node): @@ -486,28 +598,42 @@ def _find_nodes_related_to_cond(source_node): neighbors = cur.inputs + cur.outputs for node in neighbors: if node.is_var() and node.var() is not None: - if node.var().type() != core.VarDesc.VarType.READER \ - and len(node.var().shape()) == 1: + if ( + node.var().type() != core.VarDesc.VarType.READER + and len(node.var().shape()) == 1 + ): frontier.append(node) related_nodes.append(node) if node.is_op() and node.op() is not None: flag = True - if node.op().type() == "create_py_reader" \ - or node.op().type() == "create_double_buffer_reader" \ - or node.op().type() == "read": + if ( + node.op().type() == "create_py_reader" + or node.op().type() == "create_double_buffer_reader" + or node.op().type() == "read" + ): flag = False for tensor_node in node.inputs: - if tensor_node.is_var() and tensor_node.var( - ) is not None: - if tensor_node.var().type() in __not_shape_var_type__ \ - or len(tensor_node.var().shape()) != 1: + if ( + tensor_node.is_var() + and tensor_node.var() is not None + ): + if ( + tensor_node.var().type() + in __not_shape_var_type__ + or len(tensor_node.var().shape()) != 1 + ): flag = False break for tensor_node in node.outputs: - if tensor_node.is_var() and tensor_node.var( - ) is not None: - if tensor_node.var().type() in __not_shape_var_type__ \ - or len(tensor_node.var().shape()) != 1: + if ( + tensor_node.is_var() + and tensor_node.var() is not None + ): + if ( + tensor_node.var().type() + in __not_shape_var_type__ + or len(tensor_node.var().shape()) != 1 + ): flag = False break if flag: @@ -532,27 +658,32 @@ def _make_dims_mapping_replicate(dist_attr): dims_mapping = dist_attr.get_output_dims_mapping(arg_name) for _ in dims_mapping: new_dims_mapping.append(-1) - dist_attr.set_output_dims_mapping(arg_name, - new_dims_mapping) + dist_attr.set_output_dims_mapping( + arg_name, new_dims_mapping + ) # Amend the process meshes related to while_op for while_op_node, while_op_node_idx in self._while_op_nodes.values(): sub_graph_id = while_op_node.op()._block_attr_id("sub_block") sub_graph = self._dist_context.serial_graph.get_sub_graph( - sub_graph_id) + sub_graph_id + ) sub_graph_nodes = list(sub_graph.all_nodes()) while_dist_op = self._dist_context.get_dist_op_for_graph( - while_op_node) + while_op_node + ) while_op_dist_attr = while_dist_op.dist_attr # Step 1: set the process mesh of while_op to the merged process mesh of its subblock merged_process_mesh = while_op_dist_attr.process_mesh for node in sub_graph_nodes: - if (node.is_var() and node.var() is not None) \ - or (node.is_op() and node.op() is not None): + if (node.is_var() and node.var() is not None) or ( + node.is_op() and node.op() is not None + ): dist_attr = self._dist_context.get_dist_attr_for_graph(node) merged_process_mesh = merge_process_mesh_two( - merged_process_mesh, dist_attr.process_mesh) + merged_process_mesh, dist_attr.process_mesh + ) while_op_dist_attr.process_mesh = merged_process_mesh _make_dims_mapping_replicate(while_op_dist_attr) @@ -562,97 +693,143 @@ def _make_dims_mapping_replicate(dist_attr): cond_tensor_name = while_op_node.op().input("Condition")[0] cond_tensor_node = None for node in while_op_node.inputs: - if node.is_var() and node.var() is not None \ - and node.var().name() == cond_tensor_name: + if ( + node.is_var() + and node.var() is not None + and node.var().name() == cond_tensor_name + ): cond_tensor_node = node cond_tensor_related_nodes.append(cond_tensor_node) break cond_tensor_related_nodes.extend( - _find_nodes_related_to_cond(cond_tensor_node)) + _find_nodes_related_to_cond(cond_tensor_node) + ) # Step 2.2: Find related nodes of cond var in the subgraph of while_op cond_tensor_node = None for node in reversed(sub_graph_nodes): - if node.is_var() and node.var() is not None \ - and node.var().name() == cond_tensor_name \ - and len(node.outputs) == 0: + if ( + node.is_var() + and node.var() is not None + and node.var().name() == cond_tensor_name + and len(node.outputs) == 0 + ): cond_tensor_node = node break cond_tensor_related_nodes.extend( - _find_nodes_related_to_cond(cond_tensor_node)) + _find_nodes_related_to_cond(cond_tensor_node) + ) # Step 2.3: Add the StepScops output of while_op stepscopes_tensor_name = while_op_node.op().output("StepScopes")[0] stepscopes_tensor_node = None for output_node in while_op_node.outputs: - if output_node.is_var() and output_node.var() is not None \ - and output_node.var().name() == stepscopes_tensor_name: + if ( + output_node.is_var() + and output_node.var() is not None + and output_node.var().name() == stepscopes_tensor_name + ): stepscopes_tensor_node = output_node cond_tensor_related_nodes.append(stepscopes_tensor_node) # Step 2.4: Set the process meshes of all nodes related to cond var to the process mesh of while op for node in cond_tensor_related_nodes: tensor_dist_attr = self._dist_context.get_dist_attr_for_graph( - node) + node + ) tensor_dist_attr.process_mesh = merged_process_mesh _make_dims_mapping_replicate(tensor_dist_attr) # Step 3: set the process meshes of the inputs in while_op to the process meshes of the outside input nodes while_op_inputs_dist_attrs = while_op_dist_attr.inputs_dist_attrs - for tensor_name, tensor_dist_attr in while_op_inputs_dist_attrs.items( - ): + for ( + tensor_name, + tensor_dist_attr, + ) in while_op_inputs_dist_attrs.items(): nearest_tensor_node = _find_nearest_tensor_node_before( - self._dist_context.serial_ordered_nodes, while_op_node_idx, - tensor_name) - nearest_tensor_dist_attr = self._dist_context.get_dist_attr_for_graph( - nearest_tensor_node) - tensor_dist_attr.process_mesh = nearest_tensor_dist_attr.process_mesh + self._dist_context.serial_ordered_nodes, + while_op_node_idx, + tensor_name, + ) + nearest_tensor_dist_attr = ( + self._dist_context.get_dist_attr_for_graph( + nearest_tensor_node + ) + ) + tensor_dist_attr.process_mesh = ( + nearest_tensor_dist_attr.process_mesh + ) # Step 4: set the process meshes of the outputs in while_op to the process meshes of the outside output nodes while_op_outputs_dist_attrs = while_op_dist_attr.outputs_dist_attrs - for tensor_name, tensor_dist_attr in while_op_outputs_dist_attrs.items( - ): + for ( + tensor_name, + tensor_dist_attr, + ) in while_op_outputs_dist_attrs.items(): nearest_tensor_node = _find_nearest_tensor_node_before( - self._dist_context.serial_ordered_nodes, while_op_node_idx, - tensor_name) + self._dist_context.serial_ordered_nodes, + while_op_node_idx, + tensor_name, + ) if nearest_tensor_node is None: nearest_tensor_node = _find_nearest_tensor_node_after( self._dist_context.serial_ordered_nodes, - while_op_node_idx, tensor_name) - nearest_tensor_dist_attr = self._dist_context.get_dist_attr_for_graph( - nearest_tensor_node) - tensor_dist_attr.process_mesh = nearest_tensor_dist_attr.process_mesh + while_op_node_idx, + tensor_name, + ) + nearest_tensor_dist_attr = ( + self._dist_context.get_dist_attr_for_graph( + nearest_tensor_node + ) + ) + tensor_dist_attr.process_mesh = ( + nearest_tensor_dist_attr.process_mesh + ) # Amend the process meshes related to array for array_node_list in self._array_nodes.values(): merged_process_mesh = None for array_node in array_node_list: dist_attr = self._dist_context.get_dist_attr_for_graph( - array_node) + array_node + ) merged_process_mesh = merge_process_mesh_two( - merged_process_mesh, dist_attr.process_mesh) + merged_process_mesh, dist_attr.process_mesh + ) for array_node in array_node_list: dist_attr = self._dist_context.get_dist_attr_for_graph( - array_node) + array_node + ) dist_attr.process_mesh = merged_process_mesh _make_dims_mapping_replicate(dist_attr) def _update_process_mesh_between_graphs(self): for parent_node, child_node in self._node_pairs_between_graphs: parent_node_dist_attr = self._dist_context.get_dist_attr_for_graph( - parent_node) + parent_node + ) child_node_dist_attr = self._dist_context.get_dist_attr_for_graph( - child_node) - parent_node_dist_attr.process_mesh = child_node_dist_attr.process_mesh - compatible_process_mesh = compute_compatible_process_mesh([ - parent_node_dist_attr.process_mesh, + child_node + ) + parent_node_dist_attr.process_mesh = ( child_node_dist_attr.process_mesh - ]) - if compatible_process_mesh is not None \ - and parent_node_dist_attr.process_mesh != compatible_process_mesh: + ) + compatible_process_mesh = compute_compatible_process_mesh( + [ + parent_node_dist_attr.process_mesh, + child_node_dist_attr.process_mesh, + ] + ) + if ( + compatible_process_mesh is not None + and parent_node_dist_attr.process_mesh + != compatible_process_mesh + ): parent_node_dist_attr.process_mesh = compatible_process_mesh - if compatible_process_mesh is not None \ - and child_node_dist_attr.process_mesh != compatible_process_mesh: + if ( + compatible_process_mesh is not None + and child_node_dist_attr.process_mesh != compatible_process_mesh + ): child_node_dist_attr.process_mesh = compatible_process_mesh def _update_process_mesh(self): @@ -661,8 +838,9 @@ def _update_process_mesh(self): # Step 1: Set the annotated process meshes from tensors to the first ops using them ordered_tensor_nodes = self._dist_context._serial_ordered_tensor_nodes for tensor_node in ordered_tensor_nodes: - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph(tensor_node) + ) if not tensor_dist_attr.is_annotated("process_mesh"): continue first_op_node = None @@ -680,13 +858,18 @@ def _update_process_mesh(self): if first_op_node is None: continue op_dist_attr = self._dist_context.get_dist_attr_for_graph( - first_op_node) + first_op_node + ) if op_dist_attr is not None and not op_dist_attr.is_annotated( - "process_mesh"): + "process_mesh" + ): compatible_process_mesh = compute_compatible_process_mesh( - [tensor_dist_attr.process_mesh, op_dist_attr.process_mesh]) - if compatible_process_mesh is not None \ - and op_dist_attr.process_mesh != compatible_process_mesh: + [tensor_dist_attr.process_mesh, op_dist_attr.process_mesh] + ) + if ( + compatible_process_mesh is not None + and op_dist_attr.process_mesh != compatible_process_mesh + ): op_dist_attr.process_mesh = compatible_process_mesh # Step 2: set the process meshes of ops with the nearest op before them @@ -694,8 +877,10 @@ def _update_process_mesh(self): idx_of_first_op_node_has_process_mesh = -1 for idx, op_node in enumerate(ordered_op_nodes): op_dist_attr = self._dist_context.get_dist_attr_for_graph(op_node) - if op_dist_attr.process_mesh is not None \ - and idx_of_first_op_node_has_process_mesh == -1: + if ( + op_dist_attr.process_mesh is not None + and idx_of_first_op_node_has_process_mesh == -1 + ): idx_of_first_op_node_has_process_mesh = idx # Reuse the following method to set the related tensors for same op node self._update_process_mesh_by_nearest(op_node, op_node) @@ -703,17 +888,20 @@ def _update_process_mesh(self): if idx_of_first_op_node_has_process_mesh + 1 > len(ordered_op_nodes): return None for idx, op_node in enumerate( - ordered_op_nodes[idx_of_first_op_node_has_process_mesh + 1:]): + ordered_op_nodes[idx_of_first_op_node_has_process_mesh + 1 :] + ): original_idx = idx_of_first_op_node_has_process_mesh + idx + 1 nearest_op_node = ordered_op_nodes[original_idx - 1] nearest_op_dist_attr = self._dist_context.get_dist_attr_for_graph( - nearest_op_node) + nearest_op_node + ) op_dist_attr = self._dist_context.get_dist_attr_for_graph(op_node) assert nearest_op_dist_attr.process_mesh is not None self._update_process_mesh_by_nearest(op_node, nearest_op_node) # Step 2.3: set the process meshes of ops by the nearest op node before the first op node nearest_op_node = ordered_op_nodes[ - idx_of_first_op_node_has_process_mesh] + idx_of_first_op_node_has_process_mesh + ] for op_node in ordered_op_nodes[:idx_of_first_op_node_has_process_mesh]: self._update_process_mesh_by_nearest(op_node, nearest_op_node) @@ -750,21 +938,31 @@ def _prepare(self): if node.is_var() and node.var() is not None: if node.node.graph_id() != 0: for before_node in reversed(all_nodes[:idx]): - if before_node.is_var() and before_node.var() is not None \ - and before_node.node.graph_id() == node.node.graph_id() - 1 \ - and before_node.var().name() == node.var().name(): + if ( + before_node.is_var() + and before_node.var() is not None + and before_node.node.graph_id() + == node.node.graph_id() - 1 + and before_node.var().name() == node.var().name() + ): self._node_pairs_between_graphs.append( - (before_node, node)) - for after_node in all_nodes[idx + 1:]: - if after_node.is_var() and after_node.var() is not None \ - and after_node.node.graph_id() == node.node.graph_id() - 1 \ - and after_node.var().name() == node.var().name(): + (before_node, node) + ) + for after_node in all_nodes[idx + 1 :]: + if ( + after_node.is_var() + and after_node.var() is not None + and after_node.node.graph_id() + == node.node.graph_id() - 1 + and after_node.var().name() == node.var().name() + ): self._node_pairs_between_graphs.append( - (after_node, node)) + (after_node, node) + ) self._has_prepared = True def complete_forward_annotation(self, serial_main_program=None): - """ Complete annotation for the partial annotated serial_main_program. + """Complete annotation for the partial annotated serial_main_program. Arguments: serial_main_program: partial annotated serial_main_program. Returns:e @@ -818,8 +1016,9 @@ def _update_dist_attr_for_dp(self): # TODO: we must ensure the world process group contains all ranks ranks = get_world_process_group().ranks process_mesh = ProcessMesh(ranks) - for dist_tensor in self._dist_context._dist_tensors_for_program.values( - ): + for ( + dist_tensor + ) in self._dist_context._dist_tensors_for_program.values(): serial_tensor = dist_tensor.serial_tensor tensor_dist_attr = dist_tensor.dist_attr tensor_dist_attr.process_mesh = process_mesh @@ -838,27 +1037,35 @@ def _update_dist_attr_for_dp(self): if not serial_tensor.is_parameter: if arg_name not in input_xshape_arg_names: old_dims_mapping = op_dist_attr.get_input_dims_mapping( - arg_name) + arg_name + ) if len(old_dims_mapping) > 0: new_dims_mapping = [0] + [ -1 for _ in range(len(old_dims_mapping) - 1) ] op_dist_attr.set_input_dims_mapping( - arg_name, new_dims_mapping) + arg_name, new_dims_mapping + ) else: old_dims_mapping = op_dist_attr.get_input_dims_mapping( - arg_name) + arg_name + ) if len(old_dims_mapping) > 1: new_dims_mapping = [-1, 0] + [ -1 for _ in range(len(old_dims_mapping) - 2) ] op_dist_attr.set_input_dims_mapping( - arg_name, new_dims_mapping) + arg_name, new_dims_mapping + ) # Set tensor's dims_mapping by the op's - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - serial_tensor) - tensor_dist_attr.dims_mapping = op_dist_attr.get_input_dims_mapping( - arg_name) + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + serial_tensor + ) + ) + tensor_dist_attr.dims_mapping = ( + op_dist_attr.get_input_dims_mapping(arg_name) + ) output_xshape_arg_names = [] if "XShape" in op_desc.output_names(): output_xshape_arg_names = op_desc.output("XShape") @@ -867,37 +1074,48 @@ def _update_dist_attr_for_dp(self): if not serial_tensor.is_parameter: if arg_name not in output_xshape_arg_names: old_dims_mapping = op_dist_attr.get_output_dims_mapping( - arg_name) + arg_name + ) if len(old_dims_mapping) > 0: new_dims_mapping = [0] + [ -1 for _ in range(len(old_dims_mapping) - 1) ] op_dist_attr.set_output_dims_mapping( - arg_name, new_dims_mapping) + arg_name, new_dims_mapping + ) else: old_dims_mapping = op_dist_attr.get_output_dims_mapping( - arg_name) + arg_name + ) if len(old_dims_mapping) > 1: new_dims_mapping = [-1, 0] + [ -1 for _ in range(len(old_dims_mapping) - 2) ] op_dist_attr.set_output_dims_mapping( - arg_name, new_dims_mapping) + arg_name, new_dims_mapping + ) # Set tensor's dims_mapping by the op's - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - serial_tensor) - tensor_dist_attr.dims_mapping = op_dist_attr.get_output_dims_mapping( - arg_name) + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + serial_tensor + ) + ) + tensor_dist_attr.dims_mapping = ( + op_dist_attr.get_output_dims_mapping(arg_name) + ) op_dist_impls = find_compatible_distributed_operator_impls( - dist_op, partial=False) + dist_op, partial=False + ) if op_dist_impls is not None: not_compatible = True backup_op_dist_attr = copy.deepcopy(op_dist_attr) for op_dist_impl in op_dist_impls: op_dist_impl.update_dims_mapping(dist_op) - if op_dist_impl.is_auto_compatible(dist_op) \ - and dist_op.validate_dist_attr(): + if ( + op_dist_impl.is_auto_compatible(dist_op) + and dist_op.validate_dist_attr() + ): op_dist_attr.impl_type = op_dist_impl.type op_dist_attr.impl_idx = op_dist_impl.idx not_compatible = False @@ -939,24 +1157,36 @@ def _complete_tensor_dist_attr_by_op(self, serial_main_program=None): # Use the first op to set the tensor dist attr if tensor_name in has_set_dist_attr: continue - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) - tensor_dist_attr.process_mesh = op_dist_attr.process_mesh - tensor_dist_attr.dims_mapping = op_dist_attr.get_input_dims_mapping( - tensor_name) if tensor.is_parameter else [ - -1 for i in tensor_desc.shape() - ] + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph( + tensor_node + ) + ) + tensor_dist_attr.process_mesh = ( + op_dist_attr.process_mesh + ) + tensor_dist_attr.dims_mapping = ( + op_dist_attr.get_input_dims_mapping(tensor_name) + if tensor.is_parameter + else [-1 for i in tensor_desc.shape()] + ) has_set_dist_attr.add(tensor_name) for tensor_node in node.outputs: if tensor_node.is_var() and tensor_node.var() is not None: tensor_name = tensor_node.var().name() if tensor_name in has_set_dist_attr: continue - tensor_dist_attr = self._dist_context.get_tensor_dist_attr_for_graph( - tensor_node) - tensor_dist_attr.process_mesh = op_dist_attr.process_mesh - tensor_dist_attr.dims_mapping = op_dist_attr.get_output_dims_mapping( - tensor_name) + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_graph( + tensor_node + ) + ) + tensor_dist_attr.process_mesh = ( + op_dist_attr.process_mesh + ) + tensor_dist_attr.dims_mapping = ( + op_dist_attr.get_output_dims_mapping(tensor_name) + ) has_set_dist_attr.add(tensor_name) self._update_process_mesh_for_specials() @@ -1007,79 +1237,108 @@ def _get_op_by_id(ops, id): for idx in range(0, len(ops)): op = ops[idx] if int(op.attr('op_role')) == int( - core.op_proto_and_checker_maker.OpRole.Forward): + core.op_proto_and_checker_maker.OpRole.Forward + ): continue if int(op.attr('op_role')) == int( - core.op_proto_and_checker_maker.OpRole.Backward) and int( - ops[idx - 1].attr('op_role')) == int( - core.op_proto_and_checker_maker.OpRole.Forward): + core.op_proto_and_checker_maker.OpRole.Backward + ) and int(ops[idx - 1].attr('op_role')) == int( + core.op_proto_and_checker_maker.OpRole.Forward + ): appended_grad_times += 1 if int(op.attr('op_role')) == int( - int(core.op_proto_and_checker_maker.OpRole.Backward) - | int(core.op_proto_and_checker_maker.OpRole.Loss)): + int(core.op_proto_and_checker_maker.OpRole.Backward) + | int(core.op_proto_and_checker_maker.OpRole.Loss) + ): assert op.type == "fill_constant" break # complete the annotation of grad op (xxx_grad op or sum op) # xxx_grad op will have a corresponding forward op in grad_op_id_to_op_id grad_op = ops[idx] - if grad_op.desc.original_id( - ) in dist_op_context.grad_op_id_to_op_id: + if ( + grad_op.desc.original_id() + in dist_op_context.grad_op_id_to_op_id + ): # TODO support the case where one forward op corresponding to multiple xxx_grad op forward_op = _get_op_by_id( - ops, dist_op_context.grad_op_id_to_op_id[ - grad_op.desc.original_id()]) + ops, + dist_op_context.grad_op_id_to_op_id[ + grad_op.desc.original_id() + ], + ) assert forward_op is not None - fwd_op_dist_attr = self._dist_context.get_op_dist_attr_for_program( - forward_op) + fwd_op_dist_attr = ( + self._dist_context.get_op_dist_attr_for_program(forward_op) + ) fwd_op_process_mesh = fwd_op_dist_attr.process_mesh grad_op_dist_attr = OperatorDistributedAttribute() grad_op_dist_attr.process_mesh = fwd_op_process_mesh for input_name in grad_op.input_arg_names: - if input_name not in forward_op.input_arg_names and input_name not in forward_op.output_arg_names: + if ( + input_name not in forward_op.input_arg_names + and input_name not in forward_op.output_arg_names + ): if input_name in grad_var_to_var[appended_grad_times]: fwd_name = grad_var_to_var[appended_grad_times][ - input_name] - ref_dims_mapping = fwd_op_dist_attr.get_output_dims_mapping( - fwd_name) + input_name + ] + ref_dims_mapping = ( + fwd_op_dist_attr.get_output_dims_mapping( + fwd_name + ) + ) else: input_var = vars[input_name] ref_dims_mapping = self._dist_context.get_tensor_dist_attr_for_program( - input_var).dims_mapping + input_var + ).dims_mapping else: if fwd_op_dist_attr.get_input_dims_mapping(input_name): - ref_dims_mapping = fwd_op_dist_attr.get_input_dims_mapping( - input_name) + ref_dims_mapping = ( + fwd_op_dist_attr.get_input_dims_mapping( + input_name + ) + ) else: - ref_dims_mapping = fwd_op_dist_attr.get_output_dims_mapping( - input_name) - assert ref_dims_mapping is not None, "[{}] 's dims mapping is NONE".format( - input_name) + ref_dims_mapping = ( + fwd_op_dist_attr.get_output_dims_mapping( + input_name + ) + ) + assert ( + ref_dims_mapping is not None + ), "[{}] 's dims mapping is NONE".format(input_name) grad_op_dist_attr.set_input_dims_mapping( - input_name, ref_dims_mapping) + input_name, ref_dims_mapping + ) for output_name in grad_op.output_arg_names: assert output_name in grad_var_to_var[appended_grad_times] fwd_name = grad_var_to_var[appended_grad_times][output_name] ref_dims_mapping = fwd_op_dist_attr.get_input_dims_mapping( - fwd_name) + fwd_name + ) # var output_var = vars[output_name] tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.dims_mapping = ref_dims_mapping tensor_dist_attr.process_mesh = fwd_op_process_mesh self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr) + output_var, tensor_dist_attr + ) # op grad_op_dist_attr.set_output_dims_mapping( - output_name, ref_dims_mapping) + output_name, ref_dims_mapping + ) self._dist_context.set_op_dist_attr_for_program( - grad_op, grad_op_dist_attr) + grad_op, grad_op_dist_attr + ) # grad ops that have not a corresponding mapping in grad_op_id_to_op_id else: @@ -1087,14 +1346,20 @@ def _get_op_by_id(ops, id): if grad_op.type == 'sum': assert all(map(_is_grad_var_name, grad_op.input_arg_names)) output_name = grad_op.output_arg_names[0] - assert output_name in grad_var_to_var[appended_grad_times], \ - "sum op's output '{}' has no corresponding var".format( - output_name) + assert ( + output_name in grad_var_to_var[appended_grad_times] + ), "sum op's output '{}' has no corresponding var".format( + output_name + ) ref_fwd_var_name = grad_var_to_var[appended_grad_times][ - output_name] + output_name + ] ref_fwd_var = vars[ref_fwd_var_name] - ref_fwd_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - ref_fwd_var) + ref_fwd_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + ref_fwd_var + ) + ) ref_fwd_dims_mapping = ref_fwd_dist_attr.dims_mapping ref_fwd_process_mesh = ref_fwd_dist_attr.process_mesh # output @@ -1103,21 +1368,27 @@ def _get_op_by_id(ops, id): tensor_dist_attr.process_mesh = ref_fwd_process_mesh output_var = vars[output_name] self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr) + output_var, tensor_dist_attr + ) # op grad_op_dist_attr = OperatorDistributedAttribute() grad_op_dist_attr.process_mesh = ref_fwd_process_mesh for var_name in grad_op.input_arg_names: grad_op_dist_attr.set_input_dims_mapping( - var_name, ref_fwd_dims_mapping) + var_name, ref_fwd_dims_mapping + ) grad_op_dist_attr.set_output_dims_mapping( - output_name, ref_fwd_dims_mapping) + output_name, ref_fwd_dims_mapping + ) elif grad_op.type == 'fill_any_like': ref_var_name = grad_op.input_arg_names[0] ref_var = vars[ref_var_name] - ref_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - ref_var) + ref_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + ref_var + ) + ) ref_dims_mapping = ref_dist_attr.dims_mapping ref_process_mesh = ref_dist_attr.process_mesh # output @@ -1127,24 +1398,29 @@ def _get_op_by_id(ops, id): output_var_name = grad_op.output_arg_names[0] output_var = vars[output_var_name] self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr) + output_var, tensor_dist_attr + ) # op grad_op_dist_attr = OperatorDistributedAttribute() grad_op_dist_attr.process_mesh = ref_process_mesh grad_op_dist_attr.set_input_dims_mapping( - ref_var_name, ref_dims_mapping) + ref_var_name, ref_dims_mapping + ) grad_op_dist_attr.set_output_dims_mapping( - output_var_name, ref_dims_mapping) + output_var_name, ref_dims_mapping + ) elif grad_op.type in ['shape', 'fill_constant']: continue else: - raise ValueError("got unexpect op [{}]".format( - str(grad_op.type))) + raise ValueError( + "got unexpect op [{}]".format(str(grad_op.type)) + ) self._dist_context.set_op_dist_attr_for_program( - grad_op, grad_op_dist_attr) + grad_op, grad_op_dist_attr + ) def complete_backward_annotation(self, serial_main_program=None): """Complete the annotation of vars and ops in the backward phase for parallel program.""" @@ -1161,9 +1437,9 @@ def _is_grad_var_name(name): def _get_forward_varname_from_grad_varname(grad_var_name): assert _is_grad_var_name( - grad_var_name), "[{}] is not a grad varnme.".format( - grad_var_name) - return grad_var_name[:grad_var_name.find("@GRAD")] + grad_var_name + ), "[{}] is not a grad varnme.".format(grad_var_name) + return grad_var_name[: grad_var_name.find("@GRAD")] def _get_op_by_id(ops, id): for op in ops: @@ -1174,160 +1450,216 @@ def _get_op_by_id(ops, id): first_backward_op_idx = -1 for idx, op in enumerate(serial_main_program.global_block().ops): if int(op.attr('op_role')) == int( - int(core.op_proto_and_checker_maker.OpRole.Backward) - | int(core.op_proto_and_checker_maker.OpRole.Loss)): + int(core.op_proto_and_checker_maker.OpRole.Backward) + | int(core.op_proto_and_checker_maker.OpRole.Loss) + ): assert op.type == "fill_constant" first_backward_op_idx = idx break - assert first_backward_op_idx >= 0, "No backward procedure found in this program." + assert ( + first_backward_op_idx >= 0 + ), "No backward procedure found in this program." ops = list(serial_main_program.global_block().ops) vars = serial_main_program.global_block().vars dist_op_context = self._dist_context.dist_op_context - grad_var_to_var = dist_op_context.grad_var_to_var[len( - dist_op_context.grad_var_to_var)] + grad_var_to_var = dist_op_context.grad_var_to_var[ + len(dist_op_context.grad_var_to_var) + ] for idx in range(first_backward_op_idx, len(ops)): # complete the initial grad loss op if idx == first_backward_op_idx: assert ops[idx].type == "fill_constant" - assert len( - ops[idx].input_arg_names - ) == 0, "first backward op should has only ONE output, but got [{}]".format( - len(ops[idx].input_arg_names)) - assert len( - ops[idx].output_arg_names - ) == 1, "first backward op should has only ONE output, but got [{}]".format( - len(ops[idx].output_arg_names)) + assert ( + len(ops[idx].input_arg_names) == 0 + ), "first backward op should has only ONE output, but got [{}]".format( + len(ops[idx].input_arg_names) + ) + assert ( + len(ops[idx].output_arg_names) == 1 + ), "first backward op should has only ONE output, but got [{}]".format( + len(ops[idx].output_arg_names) + ) grad_var = vars[ops[idx].output_arg_names[0]] forward_var_name = _get_forward_varname_from_grad_varname( - grad_var.name) + grad_var.name + ) forward_var = vars[forward_var_name] # TODO complete other attribte for grad var tensor_dist_attr = TensorDistributedAttribute() - process_mesh = self._dist_context.get_tensor_dist_attr_for_program( - forward_var).process_mesh - dims_mapping = self._dist_context.get_tensor_dist_attr_for_program( - forward_var).dims_mapping + process_mesh = ( + self._dist_context.get_tensor_dist_attr_for_program( + forward_var + ).process_mesh + ) + dims_mapping = ( + self._dist_context.get_tensor_dist_attr_for_program( + forward_var + ).dims_mapping + ) tensor_dist_attr.dims_mapping = dims_mapping tensor_dist_attr.process_mesh = process_mesh self._dist_context.set_tensor_dist_attr_for_program( - grad_var, tensor_dist_attr) + grad_var, tensor_dist_attr + ) op_dist_attr = OperatorDistributedAttribute() op_dist_attr.process_mesh = process_mesh - op_dist_attr.set_output_dims_mapping(grad_var.name, - dims_mapping) + op_dist_attr.set_output_dims_mapping( + grad_var.name, dims_mapping + ) self._dist_context.set_op_dist_attr_for_program( - ops[idx], op_dist_attr) + ops[idx], op_dist_attr + ) continue # complete the annotation of grad op (xxx_grad op or sum op) # xxx_grad op will have a corresponding forward op in grad_op_id_to_op_id grad_op = ops[idx] - if grad_op.desc.original_id( - ) in dist_op_context.grad_op_id_to_op_id: + if ( + grad_op.desc.original_id() + in dist_op_context.grad_op_id_to_op_id + ): # TODO support the case where one forward op corresponding to multiple xxx_grad op forward_op = _get_op_by_id( ops[:first_backward_op_idx], dist_op_context.grad_op_id_to_op_id[ - grad_op.desc.original_id()]) + grad_op.desc.original_id() + ], + ) assert forward_op is not None if grad_op.type == "concat" and forward_op.type == "split": - forward_op_dist_attr = self._dist_context.get_op_dist_attr_for_program( - forward_op) + forward_op_dist_attr = ( + self._dist_context.get_op_dist_attr_for_program( + forward_op + ) + ) output_var = vars[grad_op.desc.output('Out')[0]] split_input_var_name = forward_op.input("X")[0] - ref_dims_mapping = forward_op_dist_attr.get_input_dims_mapping( - split_input_var_name) + ref_dims_mapping = ( + forward_op_dist_attr.get_input_dims_mapping( + split_input_var_name + ) + ) ref_mesh = forward_op_dist_attr.process_mesh grad_op_dist_attr = OperatorDistributedAttribute() for input_name in grad_op.input_arg_names: grad_op_dist_attr.set_input_dims_mapping( - input_name, ref_dims_mapping) + input_name, ref_dims_mapping + ) output_var_dist_attr = TensorDistributedAttribute() output_var_dist_attr.dims_mapping = ref_dims_mapping output_var_dist_attr.process_mesh = ref_mesh self._dist_context.set_tensor_dist_attr_for_program( - output_var, output_var_dist_attr) + output_var, output_var_dist_attr + ) grad_op_dist_attr.set_output_dims_mapping( - output_var.name, ref_dims_mapping) + output_var.name, ref_dims_mapping + ) grad_op_dist_attr.process_mesh = ref_mesh self._dist_context.set_op_dist_attr_for_program( - grad_op, grad_op_dist_attr) + grad_op, grad_op_dist_attr + ) grad_op_dist_attr.impl_type = fwd_op_dist_attr.impl_type grad_op_dist_attr.impl_idx = fwd_op_dist_attr.impl_idx continue - fwd_op_dist_attr = self._dist_context.get_op_dist_attr_for_program( - forward_op) + fwd_op_dist_attr = ( + self._dist_context.get_op_dist_attr_for_program(forward_op) + ) fwd_op_process_mesh = fwd_op_dist_attr.process_mesh grad_op_dist_attr = OperatorDistributedAttribute() grad_op_dist_attr.process_mesh = fwd_op_process_mesh for input_name in grad_op.input_arg_names: - if input_name not in forward_op.input_arg_names and input_name not in forward_op.output_arg_names: + if ( + input_name not in forward_op.input_arg_names + and input_name not in forward_op.output_arg_names + ): if input_name in grad_var_to_var: fwd_name = grad_var_to_var[input_name] - ref_dims_mapping = fwd_op_dist_attr.get_output_dims_mapping( - fwd_name) + ref_dims_mapping = ( + fwd_op_dist_attr.get_output_dims_mapping( + fwd_name + ) + ) else: input_var = vars[input_name] ref_dims_mapping = self._dist_context.get_tensor_dist_attr_for_program( - input_var).dims_mapping + input_var + ).dims_mapping else: if fwd_op_dist_attr.get_input_dims_mapping(input_name): - ref_dims_mapping = fwd_op_dist_attr.get_input_dims_mapping( - input_name) + ref_dims_mapping = ( + fwd_op_dist_attr.get_input_dims_mapping( + input_name + ) + ) else: - ref_dims_mapping = fwd_op_dist_attr.get_output_dims_mapping( - input_name) - assert ref_dims_mapping is not None, "[{}] 's dims mapping is NONE".format( - input_name) + ref_dims_mapping = ( + fwd_op_dist_attr.get_output_dims_mapping( + input_name + ) + ) + assert ( + ref_dims_mapping is not None + ), "[{}] 's dims mapping is NONE".format(input_name) grad_op_dist_attr.set_input_dims_mapping( - input_name, ref_dims_mapping) + input_name, ref_dims_mapping + ) for output_name in grad_op.output_arg_names: assert output_name in grad_var_to_var fwd_name = grad_var_to_var[output_name] ref_dims_mapping = fwd_op_dist_attr.get_input_dims_mapping( - fwd_name) + fwd_name + ) # var output_var = vars[output_name] tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.dims_mapping = ref_dims_mapping tensor_dist_attr.process_mesh = fwd_op_process_mesh self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr) + output_var, tensor_dist_attr + ) # op grad_op_dist_attr.set_output_dims_mapping( - output_name, ref_dims_mapping) + output_name, ref_dims_mapping + ) grad_op_dist_attr.impl_type = fwd_op_dist_attr.impl_type grad_op_dist_attr.impl_idx = fwd_op_dist_attr.impl_idx self._dist_context.set_op_dist_attr_for_program( - grad_op, grad_op_dist_attr) + grad_op, grad_op_dist_attr + ) # grad ops that have not a corresponding mapping in grad_op_id_to_op_id else: if grad_op.type == 'sum': assert all(map(_is_grad_var_name, grad_op.input_arg_names)) output_name = grad_op.output_arg_names[0] - assert output_name in grad_var_to_var, "sum op's output '{}' has no corresponding var".format( - output_name) + assert ( + output_name in grad_var_to_var + ), "sum op's output '{}' has no corresponding var".format( + output_name + ) ref_fwd_var_name = grad_var_to_var[output_name] ref_fwd_var = vars[ref_fwd_var_name] - ref_fwd_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - ref_fwd_var) + ref_fwd_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + ref_fwd_var + ) + ) ref_fwd_dims_mapping = ref_fwd_dist_attr.dims_mapping ref_fwd_process_mesh = ref_fwd_dist_attr.process_mesh @@ -1337,24 +1669,30 @@ def _get_op_by_id(ops, id): tensor_dist_attr.process_mesh = ref_fwd_process_mesh output_var = vars[output_name] self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr) + output_var, tensor_dist_attr + ) # op grad_op_dist_attr = OperatorDistributedAttribute() grad_op_dist_attr.process_mesh = ref_fwd_process_mesh for var_name in grad_op.input_arg_names: grad_op_dist_attr.set_input_dims_mapping( - var_name, ref_fwd_dims_mapping) + var_name, ref_fwd_dims_mapping + ) grad_op_dist_attr.set_output_dims_mapping( - output_name, ref_fwd_dims_mapping) + output_name, ref_fwd_dims_mapping + ) grad_op_dist_attr.impl_type = "default" grad_op_dist_attr.impl_idx = 0 elif grad_op.type == 'fill_any_like': ref_var_name = grad_op.input_arg_names[0] ref_var = vars[ref_var_name] - ref_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - ref_var) + ref_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + ref_var + ) + ) ref_dims_mapping = ref_dist_attr.dims_mapping ref_process_mesh = ref_dist_attr.process_mesh # output @@ -1364,27 +1702,35 @@ def _get_op_by_id(ops, id): output_var_name = grad_op.output_arg_names[0] output_var = vars[output_var_name] self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr) + output_var, tensor_dist_attr + ) # op grad_op_dist_attr = OperatorDistributedAttribute() grad_op_dist_attr.process_mesh = ref_process_mesh grad_op_dist_attr.set_input_dims_mapping( - ref_var_name, ref_dims_mapping) + ref_var_name, ref_dims_mapping + ) grad_op_dist_attr.set_output_dims_mapping( - output_var_name, ref_dims_mapping) + output_var_name, ref_dims_mapping + ) else: - raise ValueError("got unexpect op [{}]".format( - str(grad_op.type))) + raise ValueError( + "got unexpect op [{}]".format(str(grad_op.type)) + ) self._dist_context.set_op_dist_attr_for_program( - grad_op, grad_op_dist_attr) + grad_op, grad_op_dist_attr + ) def complete_update_annotation(self, serial_main_program): """Complete the annotation of vars and ops in the update phase for parallel program.""" # Copy the dist tensors and dist ops annotated by users from the default context # global mesh - from paddle.distributed.auto_parallel.process_group import get_world_process_group + from paddle.distributed.auto_parallel.process_group import ( + get_world_process_group, + ) + world_ranks = get_world_process_group().ranks # Notice: serial_main_program is actually a dist_main_program of current rank, @@ -1403,17 +1749,22 @@ def complete_update_annotation(self, serial_main_program): if int(op.attr('op_role')) == int(OpRole.Optimize): if is_gradient_clip_op(op): if op.type in [ - "sum", "sqrt", "fill_constant", "elementwise_max", - "elementwise_div" + "sum", + "sqrt", + "fill_constant", + "elementwise_max", + "elementwise_div", ]: op_dist_attr = OperatorDistributedAttribute() op_dist_attr.process_mesh = world_ranks for in_name in op.input_arg_names: in_var = vars[in_name] in_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - in_var) + in_var + ) op_dist_attr.set_input_dist_attr( - in_name, in_dist_attr) + in_name, in_dist_attr + ) for out_name in op.output_arg_names: out_var = vars[out_name] out_dist_attr = TensorDistributedAttribute() @@ -1422,22 +1773,30 @@ def complete_update_annotation(self, serial_main_program): -1 for _ in range(len(out_var.shape)) ] self._dist_context.set_tensor_dist_attr_for_program( - out_var, out_dist_attr) + out_var, out_dist_attr + ) op_dist_attr.set_output_dist_attr( - out_name, out_dist_attr) + out_name, out_dist_attr + ) else: in_var = vars[op.input("X")[0]] - in_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - in_var) + in_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + in_var + ) + ) assert in_dist_attr is not None ref_process_mesh = in_dist_attr.process_mesh ref_dims_mapping = in_dist_attr.dims_mapping - if op.type == "cast" and \ - ops[idx + 1].type == "elementwise_mul": + if ( + op.type == "cast" + and ops[idx + 1].type == "elementwise_mul" + ): ref_var = vars[ops[idx + 1].input("X")[0]] ref_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - ref_var) + ref_var + ) assert ref_dist_attr is not None ref_process_mesh = ref_dist_attr.process_mesh @@ -1447,51 +1806,72 @@ def complete_update_annotation(self, serial_main_program): if out_var.shape == in_var.shape: out_dist_attr.dims_mapping = ref_dims_mapping else: - assert len( - out_var.shape) == 1 and out_var.shape[0] == 1 + assert ( + len(out_var.shape) == 1 + and out_var.shape[0] == 1 + ) out_dist_attr.dims_mapping = [-1] self._dist_context.set_tensor_dist_attr_for_program( - out_var, out_dist_attr) + out_var, out_dist_attr + ) op_dist_attr = OperatorDistributedAttribute() op_dist_attr.process_mesh = ref_process_mesh op_dist_attr.set_input_dist_attr( - in_var.name, in_dist_attr) + in_var.name, in_dist_attr + ) op_dist_attr.set_output_dist_attr( - out_var.name, out_dist_attr) + out_var.name, out_dist_attr + ) self._dist_context.set_op_dist_attr_for_program( - op, op_dist_attr) + op, op_dist_attr + ) if "Grad" in op.input_names and "Param" in ops[idx].input_names: - assert len( - op.input("Param")) == 1, "Only support one-to-one now." - assert len( - op.input("Grad")) == 1, "Only support one-to-one now." + assert ( + len(op.input("Param")) == 1 + ), "Only support one-to-one now." + assert ( + len(op.input("Grad")) == 1 + ), "Only support one-to-one now." param = vars[op.input("Param")[0]] grad_var = vars[op.input("Grad")[0]] - param_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - param) + param_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + param + ) + ) assert param_dist_attr is not None - ref_process_mesh = self._dist_context.get_tensor_dist_attr_for_program( - param).process_mesh + ref_process_mesh = ( + self._dist_context.get_tensor_dist_attr_for_program( + param + ).process_mesh + ) assert ref_process_mesh is not None - ref_dims_mapping = self._dist_context.get_tensor_dist_attr_for_program( - param).dims_mapping + ref_dims_mapping = ( + self._dist_context.get_tensor_dist_attr_for_program( + param + ).dims_mapping + ) assert ref_dims_mapping is not None op_dist_attr = OperatorDistributedAttribute() op_dist_attr.process_mesh = ref_process_mesh - op_dist_attr.set_input_dims_mapping(grad_var.name, - ref_dims_mapping) - op_dist_attr.set_input_dims_mapping(param.name, - ref_dims_mapping) + op_dist_attr.set_input_dims_mapping( + grad_var.name, ref_dims_mapping + ) + op_dist_attr.set_input_dims_mapping( + param.name, ref_dims_mapping + ) op_dist_attr.set_output_dims_mapping( - param.name, ref_dims_mapping) + param.name, ref_dims_mapping + ) learning_var = vars[op.input("LearningRate")[0]] op_dist_attr.set_input_dims_mapping(learning_var.name, [-1]) op_dist_attr.set_output_dims_mapping( - learning_var.name, [-1]) + learning_var.name, [-1] + ) if not learning_rate_completed: learning_rate_completed = True @@ -1499,18 +1879,19 @@ def complete_update_annotation(self, serial_main_program): var_dist_attr.process_mesh = world_ranks var_dist_attr.dims_mapping = [-1] self._dist_context.set_tensor_dist_attr_for_program( - learning_var, var_dist_attr) + learning_var, var_dist_attr + ) for input_name in op.desc.input_names(): if input_name in [ - 'Param', - 'Grad', - 'LearningRate', - "SkipUpdate", - "Beta1Tensor", - "Beta2Tensor", - "EpsilonTensor", + 'Param', + 'Grad', + 'LearningRate', + "SkipUpdate", + "Beta1Tensor", + "Beta2Tensor", + "EpsilonTensor", ]: continue if len(op.desc.input(input_name)) == 0: @@ -1523,22 +1904,28 @@ def complete_update_annotation(self, serial_main_program): if "Beta1Pow" in input_name or "Beta2Pow" in input_name: input_var_attr.dims_mapping = [-1] op_dist_attr.set_input_dims_mapping( - input_var.name, [-1]) + input_var.name, [-1] + ) op_dist_attr.set_output_dims_mapping( - input_var.name, [-1]) + input_var.name, [-1] + ) else: input_var_attr.dims_mapping = ref_dims_mapping op_dist_attr.set_input_dims_mapping( - input_var.name, ref_dims_mapping) + input_var.name, ref_dims_mapping + ) op_dist_attr.set_output_dims_mapping( - input_var.name, ref_dims_mapping) + input_var.name, ref_dims_mapping + ) input_var_attr.process_mesh = ref_process_mesh self._dist_context.set_tensor_dist_attr_for_program( - input_var, input_var_attr) + input_var, input_var_attr + ) self._dist_context.set_op_dist_attr_for_program( - op, op_dist_attr) + op, op_dist_attr + ) continue def complete_prim_annotation(self, serial_main_program=None): @@ -1574,14 +1961,18 @@ def complete_prim_annotation(self, serial_main_program=None): def _init_global_mesh_for_program(self): # Copy the dist tensors and dist ops annotated by users from the default context # global mesh - from paddle.distributed.auto_parallel.process_group import get_world_process_group + from paddle.distributed.auto_parallel.process_group import ( + get_world_process_group, + ) + world_ranks = get_world_process_group().ranks for block in self._dist_context._serial_main_program.blocks: for tensor in block.vars.values(): # Copy the distributed tensors in the default context dist_tensor = self._dist_context.get_dist_tensor_for_program( - tensor) + tensor + ) assert dist_tensor is not None dist_tensor.dist_attr.process_mesh = world_ranks for op in block.ops: @@ -1592,7 +1983,8 @@ def _init_global_mesh_for_program(self): # Find the most compatible implemenetations from the distributed operator op_dist_impls = find_compatible_distributed_operator_impls( - dist_op, fwd=True) + dist_op, fwd=True + ) if op_dist_impls is not None: backup_op_dist_attr = copy.deepcopy(dist_op.dist_attr) for op_dist_impl in op_dist_impls: diff --git a/python/paddle/distributed/auto_parallel/converter.py b/python/paddle/distributed/auto_parallel/converter.py index 21cc2a22368f5..d0fae414b1981 100644 --- a/python/paddle/distributed/auto_parallel/converter.py +++ b/python/paddle/distributed/auto_parallel/converter.py @@ -45,30 +45,42 @@ def __init__(self, tensors_dict, pre_strategy, cur_strategy): def _check_tensor_dict(self, tensors_dict): if not tensors_dict: - raise ValueError("'tensors_dict' is None, " - "the tensors to be converted cannot be None.") + raise ValueError( + "'tensors_dict' is None, " + "the tensors to be converted cannot be None." + ) if not isinstance(tensors_dict, dict): raise TypeError( - "The type of 'tensors_dict' should be 'dict', but got '{}'.". - format(str(type(tensors_dict)))) + "The type of 'tensors_dict' should be 'dict', but got '{}'.".format( + str(type(tensors_dict)) + ) + ) return tensors_dict def _check_pre_strategy(self, pre_strategy): if not pre_strategy: - raise ValueError("'pre_strategy' is None, " - "there are not tensors in pre process.") + raise ValueError( + "'pre_strategy' is None, " + "there are not tensors in pre process." + ) if not isinstance(pre_strategy, dict): - raise TypeError("The type of 'pre_strategy' should be 'dict', " - "but got '{}'.".format(str(type(pre_strategy)))) + raise TypeError( + "The type of 'pre_strategy' should be 'dict', " + "but got '{}'.".format(str(type(pre_strategy))) + ) return pre_strategy def _check_cur_strategy(self, cur_strategy): if not cur_strategy: - warnings.warn("'cur_strategy' is None, " - "there are not tensors in cur process") + warnings.warn( + "'cur_strategy' is None, " + "there are not tensors in cur process" + ) if not isinstance(cur_strategy, dict): - raise TypeError("The type of 'cur_strategy' should be 'dict', " - "but got '{}'.".format(str(type(cur_strategy)))) + raise TypeError( + "The type of 'cur_strategy' should be 'dict', " + "but got '{}'.".format(str(type(cur_strategy))) + ) return cur_strategy def convert(self, strict=True): @@ -131,42 +143,60 @@ def convert(self, strict=True): cur_dist_attr = self._cur_strategy[tensor_name] try: tensors_dict[tensor_name] = Converter.merge_and_slice( - tensor_list, pre_dist_attr, cur_dist_attr) + tensor_list, pre_dist_attr, cur_dist_attr + ) except ValueError as err: raise ValueError( - "Fail to convert tensor '{}'. ".format(str(tensor_name)) + - str(err)) + "Fail to convert tensor '{}'. ".format(str(tensor_name)) + + str(err) + ) for tensor_name in self._pre_strategy: if tensor_name not in self._cur_strategy: tensor_not_in_cur.append(tensor_name) if not strict: - tensors_dict, tensor_match_with_pre, tensor_match_with_cur = self.convert_with_prefix_match( - tensors_dict, tensor_not_in_pre, tensor_not_in_cur) + ( + tensors_dict, + tensor_match_with_pre, + tensor_match_with_cur, + ) = self.convert_with_prefix_match( + tensors_dict, tensor_not_in_pre, tensor_not_in_cur + ) else: - tensors_dict, tensor_match_with_pre, tensor_match_with_cur = tensors_dict, [], [] + tensors_dict, tensor_match_with_pre, tensor_match_with_cur = ( + tensors_dict, + [], + [], + ) tensor_not_in_pre = set(tensor_not_in_pre) - set(tensor_match_with_pre) tensor_not_in_cur = set(tensor_not_in_cur) - set(tensor_match_with_cur) if tensor_not_in_pre: warnings.warn( "tensors [{}] are not found in last training strategy.".format( - str(tensor_not_in_pre))) + str(tensor_not_in_pre) + ) + ) if tensor_not_in_cur: warnings.warn( - "tensors [{}] are not found in current training strategy.". - format(str(tensor_not_in_cur))) + "tensors [{}] are not found in current training strategy.".format( + str(tensor_not_in_cur) + ) + ) if tensor_not_in_ckpt: warnings.warn( "tensors [{}] are found in pre_strategy, but are not found" - "in checkpoint files, please check your checkpoint files.". - format(str(tensor_not_in_ckpt))) + "in checkpoint files, please check your checkpoint files.".format( + str(tensor_not_in_ckpt) + ) + ) return tensors_dict - def convert_with_prefix_match(self, tensors_dict, tensor_not_in_pre, - tensor_not_in_cur): + def convert_with_prefix_match( + self, tensors_dict, tensor_not_in_pre, tensor_not_in_cur + ): # the name which in cur_process and can match with pre_process tensor_match_with_pre = [] # the name which in pre_process and can match with cur_process @@ -174,7 +204,7 @@ def convert_with_prefix_match(self, tensors_dict, tensor_not_in_pre, for cur_name in tensor_not_in_pre: prefix_name = cur_name while prefix_name.find("_") != -1: - prefix_name = prefix_name[:prefix_name.rfind("_")] + prefix_name = prefix_name[: prefix_name.rfind("_")] for pre_name in tensor_not_in_cur: if prefix_name in pre_name: # 'cur_name' of cur_process can match with 'pre_name' of pre_process @@ -185,14 +215,20 @@ def convert_with_prefix_match(self, tensors_dict, tensor_not_in_pre, cur_dist_attr = self._cur_strategy[cur_name] try: tensors_dict[cur_name] = Converter.merge_and_slice( - pre_tensor_list, pre_dist_attr, cur_dist_attr) + pre_tensor_list, pre_dist_attr, cur_dist_attr + ) except ValueError as err: raise ValueError( "Fail to convert tensor '{}' by '{}'. ".format( - str(cur_name), str(pre_name)) + str(err)) + str(cur_name), str(pre_name) + ) + + str(err) + ) self._logger.info( "tensor [{}] is matched with tensor [{}]".format( - cur_name, pre_name)) + cur_name, pre_name + ) + ) tensor_match_with_pre.append(cur_name) tensor_match_with_cur.append(pre_name) break @@ -221,8 +257,9 @@ def merge_and_slice(tensor_list, pre_dist_attr, cur_dist_attr): cur_dims_mapping = cur_dist_attr["dims_mapping"] if len(set(pre_dims_mapping)) > 1 or -1 not in pre_dims_mapping: # merge tensor - tensor = Converter.merge_with_dist_attr(tensor_list, - pre_dist_attr) + tensor = Converter.merge_with_dist_attr( + tensor_list, pre_dist_attr + ) else: # skip merge tensor tensor = tensor_list[0] @@ -235,7 +272,7 @@ def merge_and_slice(tensor_list, pre_dist_attr, cur_dist_attr): @staticmethod def merge_with_dist_attr(tensor_list, dist_attr): - """ Merge tensor with distributed attribute """ + """Merge tensor with distributed attribute""" from .reshard import Resharder dims_mapping = dist_attr["dims_mapping"] @@ -243,44 +280,62 @@ def merge_with_dist_attr(tensor_list, dist_attr): process_group = dist_attr["process_group"] # get the complete shape of the tensor complete_shape = Resharder.compute_complete_shape( - tensor_list[0].shape, process_shape, dims_mapping) + tensor_list[0].shape, process_shape, dims_mapping + ) # merge the tensor with dist_attr partition_tensor_list = [] merged_partiton = [] for process in process_group: partition_index = Resharder.compute_partition_index( - process, complete_shape, dims_mapping, process_shape, - process_group) + process, + complete_shape, + dims_mapping, + process_shape, + process_group, + ) index = process_group.index(process) if partition_index not in merged_partiton: merged_partiton.append(partition_index) - Converter.merge(partition_tensor_list, tensor_list[index], - partition_index, complete_shape) + Converter.merge( + partition_tensor_list, + tensor_list[index], + partition_index, + complete_shape, + ) if len(partition_tensor_list) != 1: - raise ValueError("Fail to merge tensor with dist_attr '{}'.".format( - str(dist_attr))) + raise ValueError( + "Fail to merge tensor with dist_attr '{}'.".format( + str(dist_attr) + ) + ) complete_tensor = partition_tensor_list[0][0] return complete_tensor @staticmethod def slice_with_dist_attr(tensor, dist_attr): - """ Slice tensor with distributed attribute """ + """Slice tensor with distributed attribute""" dims_mapping = dist_attr["dims_mapping"] process_shape = dist_attr["process_shape"] process_group = dist_attr["process_group"] # slice the tensor with dist_attr partition_index_list = Converter._get_split_indices( - tensor.shape, dims_mapping, process_shape, process_group) - sliced_tensor_list = Converter.split(tensor, partition_index_list, - len(partition_index_list)) + tensor.shape, dims_mapping, process_shape, process_group + ) + sliced_tensor_list = Converter.split( + tensor, partition_index_list, len(partition_index_list) + ) # get the current tensor's index in sliced_tensor_list rank_id = paddle.distributed.get_rank() sliced_tensor_index = Converter._get_sliced_index( - rank_id, tensor.shape, dims_mapping, process_shape, process_group) + rank_id, tensor.shape, dims_mapping, process_shape, process_group + ) if sliced_tensor_index not in range(len(sliced_tensor_list)): - raise ValueError("Fail to slice tensor with dist_attr '{}'.".format( - str(dist_attr))) + raise ValueError( + "Fail to slice tensor with dist_attr '{}'.".format( + str(dist_attr) + ) + ) sliced_tensor = sliced_tensor_list[sliced_tensor_index] return sliced_tensor @@ -319,21 +374,32 @@ def merge(partition_tensor_list, tensor, partition_index, complete_shape): else: i = 0 while i < len(partition_tensor_list): - concat_axis, first_order, new_partition = Resharder.compute_concat_info( - partition_tensor_list[i][1], partition_index) + ( + concat_axis, + first_order, + new_partition, + ) = Resharder.compute_concat_info( + partition_tensor_list[i][1], partition_index + ) if concat_axis != -1: if first_order == 0: new_tensor = np.concatenate( (partition_tensor_list[i][0], tensor), - axis=concat_axis) + axis=concat_axis, + ) else: new_tensor = np.concatenate( (tensor, partition_tensor_list[i][0]), - axis=concat_axis) + axis=concat_axis, + ) partition_tensor_list.pop(i) - Converter.merge(partition_tensor_list, new_tensor, - new_partition, complete_shape) + Converter.merge( + partition_tensor_list, + new_tensor, + new_partition, + complete_shape, + ) break i += 1 @@ -361,19 +427,21 @@ def split(complete_tensor, partition_index_list, length): """ sliced_tensor_list = [] axis = len(complete_tensor.shape) - length - sliced_tensor = np.split(complete_tensor, - partition_index_list[axis], - axis=axis) + sliced_tensor = np.split( + complete_tensor, partition_index_list[axis], axis=axis + ) if length == 1: return sliced_tensor for tensor in sliced_tensor: sliced_tensor_list.extend( - Converter.split(tensor, partition_index_list, length - 1)) + Converter.split(tensor, partition_index_list, length - 1) + ) return sliced_tensor_list @staticmethod - def _get_split_indices(complete_shape, dims_mapping, process_shape, - process_group): + def _get_split_indices( + complete_shape, dims_mapping, process_shape, process_group + ): """ Get split indices of every dimension. @@ -398,22 +466,31 @@ def _get_split_indices(complete_shape, dims_mapping, process_shape, split_indices_list = [] for process in process_group: partition_index = Resharder.compute_partition_index( - process, complete_shape, dims_mapping, process_shape, - process_group) + process, + complete_shape, + dims_mapping, + process_shape, + process_group, + ) if split_indices_list: for dim in range(len(partition_index)): split_indices_list[dim].extend(partition_index[dim]) else: split_indices_list = partition_index split_indices_list = list( - map(lambda x, y: list(set(x) - set([y]) - set([0])), - split_indices_list, complete_shape)) + map( + lambda x, y: list(set(x) - set([y]) - set([0])), + split_indices_list, + complete_shape, + ) + ) split_indices_list = [sorted(x) for x in split_indices_list] return split_indices_list @staticmethod - def _get_sliced_index(rank_id, complete_shape, dims_mapping, process_shape, - process_group): + def _get_sliced_index( + rank_id, complete_shape, dims_mapping, process_shape, process_group + ): """ Get sliced_tensor's index of current rank in all sliced tensors list. @@ -442,7 +519,8 @@ def _get_sliced_index(rank_id, complete_shape, dims_mapping, process_shape, from .reshard import Resharder partition_index = Resharder.compute_partition_index( - rank_id, complete_shape, dims_mapping, process_shape, process_group) + rank_id, complete_shape, dims_mapping, process_shape, process_group + ) sliced_index = 0 for i, shape in enumerate(complete_shape): if dims_mapping[i] == -1: diff --git a/python/paddle/distributed/auto_parallel/cost/base_cost.py b/python/paddle/distributed/auto_parallel/cost/base_cost.py index 5ac81052c7625..2ce98a0a0511e 100644 --- a/python/paddle/distributed/auto_parallel/cost/base_cost.py +++ b/python/paddle/distributed/auto_parallel/cost/base_cost.py @@ -25,8 +25,12 @@ from ..dist_tensor import DistributedTensor COMM_OP_TYPE = [ - "send_v2", "recv_v2", "c_broadcast", "c_allgather", "c_allreduce_sum", - "c_identity" + "send_v2", + "recv_v2", + "c_broadcast", + "c_allgather", + "c_allreduce_sum", + "c_identity", ] NON_COMP_TYPE = ["while"] + COMM_OP_TYPE _g_op_cost_factory = {} @@ -93,8 +97,9 @@ def build_comp_desc_from_dist_op(dist_op, dist_context): var_name_list = op.input(input_name) var_desc = [] for var_name in var_name_list: - var = get_var_with_recursion(var_name, op.block, - op.block.program) + var = get_var_with_recursion( + var_name, op.block, op.block.program + ) # Use op input_dims_mapping dims_mapping = dist_attr.get_input_dims_mapping(var_name) global_sizes = var.shape @@ -102,18 +107,34 @@ def build_comp_desc_from_dist_op(dist_op, dist_context): shard_sizes = None topology = process_mesh.topology shape = DistributedTensor.get_local_sizes( - global_sizes, dims_mapping, topology, processes, process, - shard_sizes) + global_sizes, + dims_mapping, + topology, + processes, + process, + shard_sizes, + ) var_desc.append((var.dtype, shape)) # For special op such as embedding and its grad op - if op.type == "c_embedding" or op.type == "lookup_table_v2" or op.type == "c_embedding_grad" or op.type == "lookup_table_v2_grad": + if ( + op.type == "c_embedding" + or op.type == "lookup_table_v2" + or op.type == "c_embedding_grad" + or op.type == "lookup_table_v2_grad" + ): if input_name == "W": - embedding_row_dim_mapping = dist_attr.get_input_dims_mapping( - op.input(input_name)[0])[0] + embedding_row_dim_mapping = ( + dist_attr.get_input_dims_mapping( + op.input(input_name)[0] + )[0] + ) relative_idx = _get_idx_in_axis( - processes, dist_attr.process_mesh.topology, - embedding_row_dim_mapping, process) + processes, + dist_attr.process_mesh.topology, + embedding_row_dim_mapping, + process, + ) per_part_size = shape[0] relative_idx = relative_idx * per_part_size desc["attrs"]["start_index"] = relative_idx @@ -126,8 +147,9 @@ def build_comp_desc_from_dist_op(dist_op, dist_context): var_desc = [] for var_name in var_name_list: # Use op output_dims_mapping - var = get_var_with_recursion(var_name, op.block, - op.block.program) + var = get_var_with_recursion( + var_name, op.block, op.block.program + ) dist_attr = dist_op.dist_attr dims_mapping = dist_attr.get_output_dims_mapping(var_name) process_mesh = dist_attr.process_mesh @@ -136,8 +158,13 @@ def build_comp_desc_from_dist_op(dist_op, dist_context): processes = process_mesh.processes topology = process_mesh.topology shape = DistributedTensor.get_local_sizes( - global_sizes, dims_mapping, topology, processes, process, - shard_sizes) + global_sizes, + dims_mapping, + topology, + processes, + process, + shard_sizes, + ) var_desc.append((var.dtype, shape)) # For special op such as fill_constant_batch_size_like @@ -150,8 +177,9 @@ def build_comp_desc_from_dist_op(dist_op, dist_context): # Modify target shape for idx, axis in enumerate(dims_mapping): if axis >= 0: - shape_list[idx] = shape_list[ - idx] // process_mesh_shape[axis] + shape_list[idx] = ( + shape_list[idx] // process_mesh_shape[axis] + ) desc["attrs"]["shape"] = shape_list output_desc[out_name] = var_desc @@ -209,13 +237,15 @@ def _parse_dtype(dtype): return parse_result -def build_comm_desc_from_dist_op(op_type, - dist_op, - ctx, - var_names, - attrs=None, - parallel_axis=None, - group_ranks=None): +def build_comm_desc_from_dist_op( + op_type, + dist_op, + ctx, + var_names, + attrs=None, + parallel_axis=None, + group_ranks=None, +): """Build descriptions of communication op distributed on the processes.""" from ..reshard import get_var_with_recursion @@ -256,19 +286,26 @@ def build_comm_desc_from_dist_op(op_type, has_found = True break assert has_found - var = get_var_with_recursion(var_name, serial_op.block, - serial_op.block.program) + var = get_var_with_recursion( + var_name, serial_op.block, serial_op.block.program + ) - dims_mapping = dist_attr.get_input_dims_mapping( - var_name - ) if var_name in dist_op.serial_op.input_arg_names else dist_attr.get_output_dims_mapping( - var_name) + dims_mapping = ( + dist_attr.get_input_dims_mapping(var_name) + if var_name in dist_op.serial_op.input_arg_names + else dist_attr.get_output_dims_mapping(var_name) + ) global_sizes = var.shape shard_sizes = None topology = process_mesh.topology shape = DistributedTensor.get_local_sizes( - global_sizes, dims_mapping, topology, processes, process, - shard_sizes) + global_sizes, + dims_mapping, + topology, + processes, + process, + shard_sizes, + ) input_list.append((var.dtype, shape)) # NOTE: The input_name of comm ops used usually is X. @@ -278,9 +315,12 @@ def build_comm_desc_from_dist_op(op_type, if parallel_axis is not None: process_mesh_shape = process_mesh.topology process_mesh_group = process_mesh.processes - comm_group_ranks = _get_comm_group(process_mesh_group, - process_mesh_shape, - parallel_axis, rank_id) + comm_group_ranks = _get_comm_group( + process_mesh_group, + process_mesh_shape, + parallel_axis, + rank_id, + ) elif group_ranks is not None: comm_group_ranks = group_ranks else: @@ -322,8 +362,9 @@ def build_comm_costs_from_descs(op_cost_class, ctx, processes, descs, cluster): group_ranks = desc["group_ranks"] if group_ranks not in group_ranks_list: group_ranks_list.append(group_ranks) - comm_op_cost = op_cost_class(op_desc=desc, - comm_context=comm_context) + comm_op_cost = op_cost_class( + op_desc=desc, comm_context=comm_context + ) comm_op_cost_list.append(comm_op_cost) return comm_op_cost_list @@ -336,8 +377,9 @@ def build_comp_costs_from_descs(op_cost_class, ctx, processes, descs, cluster): return costs -def build_dp_costs(result, dist_op, ctx, var_names, attrs, parallel_axis, - cluster): +def build_dp_costs( + result, dist_op, ctx, var_names, attrs, parallel_axis, cluster +): """DP cost contains a allreduce_sum op cost and a scale op cost""" # The costs will be appended in the given result. from ..reshard import get_var_with_recursion @@ -370,10 +412,15 @@ def build_dp_costs(result, dist_op, ctx, var_names, attrs, parallel_axis, ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_cost_list = build_comm_costs_from_descs( - _g_op_cost_factory["c_allreduce_sum"], ctx, processes, - c_allreduce_sum_descs, cluster) + _g_op_cost_factory["c_allreduce_sum"], + ctx, + processes, + c_allreduce_sum_descs, + cluster, + ) result.append(comm_cost_list) # The scale op just on the group_ranks @@ -386,24 +433,33 @@ def build_dp_costs(result, dist_op, ctx, var_names, attrs, parallel_axis, desc = {} desc["op"] = op_type desc["inputs"] = {} - dims_mapping = dist_attr.get_input_dims_mapping( - var_name) if dist_attr.get_input_dims_mapping( - var_name - ) is not None else dist_attr.get_output_dims_mapping(var_name) - var = get_var_with_recursion(var_name, dist_op.serial_op.block, - dist_op.serial_op.block.program) + dims_mapping = ( + dist_attr.get_input_dims_mapping(var_name) + if dist_attr.get_input_dims_mapping(var_name) is not None + else dist_attr.get_output_dims_mapping(var_name) + ) + var = get_var_with_recursion( + var_name, + dist_op.serial_op.block, + dist_op.serial_op.block.program, + ) global_sizes = var.shape shard_sizes = None topology = process_mesh.topology - shape = DistributedTensor.get_local_sizes(global_sizes, - dims_mapping, topology, - processes, rank, - shard_sizes) + shape = DistributedTensor.get_local_sizes( + global_sizes, + dims_mapping, + topology, + processes, + rank, + shard_sizes, + ) desc["inputs"]["X"] = [(var.dtype, shape)] attrs = {"scale": 1.0 / dp_degree} desc["attrs"] = attrs - scale_op_cost = _g_op_cost_factory["scale"](op_desc=desc, - cluster=cluster) + scale_op_cost = _g_op_cost_factory["scale"]( + op_desc=desc, cluster=cluster + ) scale_costs[rank] = scale_op_cost result.append(scale_costs) @@ -442,7 +498,7 @@ def _post_init(self): if alpha_latency is None: # set default self.base_ring = 8.4 - self.base_tree = 0. + self.base_tree = 0.0 # self.base_inter_ring = 9.6 # self.base_inter_tree = 28 # NVL in default @@ -457,7 +513,7 @@ def _post_init(self): self.base_ring = base_ring if base_ring is not None else 8.4 base_tree = alpha_latency.base_tree - self.base_tree = base_tree if base_tree is not None else 0. + self.base_tree = base_tree if base_tree is not None else 0.0 intra_ring = alpha_latency.intra_ring if intra_ring == LinkType.NVL: @@ -521,10 +577,16 @@ def get_max_beta(self, ranks): for i in range(len(ranks)): for j in range(i + 1, len(ranks)): forward_order_beta = self.cluster.get_beta( - ranks[i], ranks[j]) + ranks[i], ranks[j] + ) backward_order_beta = self.cluster.get_beta( - ranks[j], ranks[i]) - beta = forward_order_beta if forward_order_beta > backward_order_beta else backward_order_beta + ranks[j], ranks[i] + ) + beta = ( + forward_order_beta + if forward_order_beta > backward_order_beta + else backward_order_beta + ) if max_beta == None: max_beta = beta else: @@ -547,7 +609,6 @@ def get_hops(self, ranks): class Cost: - def __init__(self, time=0, memory=0, flops=0): self.time = time self.memory = memory @@ -557,14 +618,14 @@ def _check_time(self, val): assert val >= 0, "Time must be greater than or equal to 0." def _check_memory(self, val): - assert isinstance( - val, - int) and val >= 0, "Memory must be int and greater than equal to 0." + assert ( + isinstance(val, int) and val >= 0 + ), "Memory must be int and greater than equal to 0." def _check_flops(self, val): - assert isinstance( - val, - int) and val >= 0, "FLOPs must be int and greater than equal to 0." + assert ( + isinstance(val, int) and val >= 0 + ), "FLOPs must be int and greater than equal to 0." @property def time(self): @@ -598,7 +659,7 @@ def __add__(self, rhs): time = self.time + rhs.time memory = self.memory + rhs.memory flops = self.flops + rhs.flops - assert (time >= 0 and memory >= 0 and flops >= 0) + assert time >= 0 and memory >= 0 and flops >= 0 return Cost(time, memory, flops) def __sub__(self, rhs): @@ -606,12 +667,11 @@ def __sub__(self, rhs): time = self.time - rhs.time memory = self.memory - rhs.memory flops = self.flops - rhs.flops - assert (time >= 0 and memory >= 0 and flops >= 0) + assert time >= 0 and memory >= 0 and flops >= 0 return Cost(time, memory, flops) class OpCost: - def __init__(self, op=None, op_desc=None): self._op = op self._op_desc = op_desc @@ -666,12 +726,12 @@ def __add__(self, rhs): time = self.cost.time + rhs.cost.time memory = self.cost.memory + rhs.cost.memory flops = self.cost.flops + rhs.cost.flops - assert (time >= 0 and memory >= 0 and flops >= 0) + assert time >= 0 and memory >= 0 and flops >= 0 elif isinstance(rhs, Cost): time = self.time + rhs.time memory = self.memory + rhs.memory flops = self.flops + rhs.flops - assert (time >= 0 and memory >= 0 and flops >= 0) + assert time >= 0 and memory >= 0 and flops >= 0 return Cost(time, memory, flops) def __sub__(self, rhs): @@ -683,12 +743,12 @@ def __sub__(self, rhs): time = self.cost.time - rhs.cost.time memory = self.cost.memory - rhs.cost.memory flops = self.cost.flops - rhs.cost.flops - assert (time >= 0 and memory >= 0 and flops >= 0) + assert time >= 0 and memory >= 0 and flops >= 0 elif isinstance(rhs, Cost): time = self.time - rhs.time memory = self.memory - rhs.memory flops = self.flops - rhs.flops - assert (time >= 0 and memory >= 0 and flops >= 0) + assert time >= 0 and memory >= 0 and flops >= 0 return Cost(time, memory, flops) @@ -721,8 +781,9 @@ def comm_count(self): vars = self.op.block.vars # NOTE: The tensor communicated input_name is "X" in default. Otherwise, this function should be overrided var_name = self.op.input("X")[0] - var = get_var_with_recursion(var_name, self.op.block, - self.program) + var = get_var_with_recursion( + var_name, self.op.block, self.program + ) dtype = var.dtype shape = var.shape elif self.op_desc is not None: @@ -756,7 +817,8 @@ def machine_count(self): if self._machine_count is None: cluster = self._comm_context.cluster self._machine_count = cluster.get_involved_machine_count( - self.group_ranks) + self.group_ranks + ) return self._machine_count @property @@ -775,8 +837,10 @@ def group_ranks(self): process_group = get_process_group(ring_id) if process_group is None: raise ValueError( - "There not exists process group whose ring_id is {}.". - format(ring_id)) + "There not exists process group whose ring_id is {}.".format( + ring_id + ) + ) self._group_ranks = process_group.ranks return self._group_ranks @@ -786,7 +850,9 @@ def _check_comm_op_type(cls): if cls.OP_TYPE not in COMM_OP_TYPE: raise TypeError( "Please Check op type in {}, but got {}.".format( - COMM_OP_TYPE, cls.OP_TYPE)) + COMM_OP_TYPE, cls.OP_TYPE + ) + ) class CompOpCost(OpCost): @@ -804,7 +870,9 @@ def _check_comp_op_type(cls): if cls.OP_TYPE in NON_COMP_TYPE: raise TypeError( "Please Check op type not in {}, but got {}.".format( - NON_COMP_TYPE, cls.OP_TYPE)) + NON_COMP_TYPE, cls.OP_TYPE + ) + ) def register_op_cost(cls): @@ -821,12 +889,12 @@ def register(op_type): def calc_time_by_modeling(op=None, desc=None, cluster=None): op_type = op.type if op is not None else desc["op"] if op_type in COMM_OP_TYPE: - op_cost = _g_op_cost_factory[op_type](op=op, - op_desc=desc, - comm_context=CommContext(cluster)) + op_cost = _g_op_cost_factory[op_type]( + op=op, op_desc=desc, comm_context=CommContext(cluster) + ) elif op_type not in NON_COMP_TYPE: - op_cost = _g_op_cost_factory[op_type](op=op, - op_desc=desc, - cluster=cluster) + op_cost = _g_op_cost_factory[op_type]( + op=op, op_desc=desc, cluster=cluster + ) time = op_cost.calc_time() return time diff --git a/python/paddle/distributed/auto_parallel/cost/comm_op_cost.py b/python/paddle/distributed/auto_parallel/cost/comm_op_cost.py index b81df4dbe8656..2e8e347bd8ec6 100644 --- a/python/paddle/distributed/auto_parallel/cost/comm_op_cost.py +++ b/python/paddle/distributed/auto_parallel/cost/comm_op_cost.py @@ -22,9 +22,9 @@ class AllreduceSumOpCost(CommOpCost): OP_TYPE = "c_allreduce_sum" def __init__(self, op=None, op_desc=None, comm_context=None): - super(AllreduceSumOpCost, self).__init__(op=op, - op_desc=op_desc, - comm_context=comm_context) + super(AllreduceSumOpCost, self).__init__( + op=op, op_desc=op_desc, comm_context=comm_context + ) def calc_time(self): # use tree if cross machine and use ring if in a single machine @@ -39,22 +39,41 @@ def calc_time(self): def calc_time_ring(self): alpha = self.comm_context.base_ring - alpha += 2 * (self.rank_count - - self.machine_count) * self.comm_context.intra_ring - alpha += 2 * (self.machine_count - 1) * ( - self.comm_context.inter_ring + self.hops * self.comm_context.switch) + alpha += ( + 2 + * (self.rank_count - self.machine_count) + * self.comm_context.intra_ring + ) + alpha += ( + 2 + * (self.machine_count - 1) + * ( + self.comm_context.inter_ring + + self.hops * self.comm_context.switch + ) + ) beta = self.comm_context.get_max_beta(self.group_ranks) - time = alpha + 2 * (self.rank_count - - 1) / self.rank_count * self.comm_count * beta + time = ( + alpha + + 2 + * (self.rank_count - 1) + / self.rank_count + * self.comm_count + * beta + ) return time def calc_time_tree(self): alpha = self.comm_context.base_tree - alpha += 2 * (self.rank_count / self.machine_count - - 1) * self.comm_context.intra_tree + alpha += ( + 2 + * (self.rank_count / self.machine_count - 1) + * self.comm_context.intra_tree + ) alpha += math.log2(self.machine_count) * ( - self.comm_context.inter_tree + self.hops * self.comm_context.switch) + self.comm_context.inter_tree + self.hops * self.comm_context.switch + ) beta = self.comm_context.get_max_beta(self.group_ranks) time = alpha + 2 * self.comm_count * beta @@ -67,9 +86,9 @@ class AllgatherOpCost(CommOpCost): OP_TYPE = "c_allgather" def __init__(self, op=None, op_desc=None, comm_context=None): - super(AllgatherOpCost, self).__init__(op=op, - op_desc=op_desc, - comm_context=comm_context) + super(AllgatherOpCost, self).__init__( + op=op, op_desc=op_desc, comm_context=comm_context + ) def calc_time(self): time = self.calc_time_ring() @@ -77,13 +96,17 @@ def calc_time(self): def calc_time_ring(self): alpha = self.comm_context.base_ring - alpha += (self.rank_count - - self.machine_count) * self.comm_context.intra_ring + alpha += ( + self.rank_count - self.machine_count + ) * self.comm_context.intra_ring alpha += (self.machine_count - 1) * ( - self.comm_context.inter_ring + self.hops * self.comm_context.switch) + self.comm_context.inter_ring + self.hops * self.comm_context.switch + ) beta = self.comm_context.get_max_beta(self.group_ranks) - time = alpha + (self.rank_count - - 1) / self.rank_count * self.comm_count * beta + time = ( + alpha + + (self.rank_count - 1) / self.rank_count * self.comm_count * beta + ) return time @@ -92,9 +115,9 @@ class BroadcastOpCost(CommOpCost): OP_TYPE = "c_broadcast" def __init__(self, op=None, op_desc=None, comm_context=None): - super(BroadcastOpCost, self).__init__(op=op, - op_desc=op_desc, - comm_context=comm_context) + super(BroadcastOpCost, self).__init__( + op=op, op_desc=op_desc, comm_context=comm_context + ) def calc_time(self): time = self.calc_time_ring() @@ -103,7 +126,10 @@ def calc_time(self): def calc_time_ring(self): alpha = self.comm_context.base_ring if self.machine_count > 1: - alpha += self.comm_context.inter_ring + self.hops * self.comm_context.switch + alpha += ( + self.comm_context.inter_ring + + self.hops * self.comm_context.switch + ) else: alpha += self.comm_context.intra_ring beta = self.comm_context.get_max_beta(self.group_ranks) @@ -117,9 +143,9 @@ class IdentityOpCost(CommOpCost): OP_TYPE = "c_identity" def __init__(self, op=None, op_desc=None, comm_context=None): - super(IdentityOpCost, self).__init__(op=op, - op_desc=op_desc, - comm_context=comm_context) + super(IdentityOpCost, self).__init__( + op=op, op_desc=op_desc, comm_context=comm_context + ) def calc_time(self): return 0 @@ -130,14 +156,17 @@ class RecvOpCost(CommOpCost): OP_TYPE = "recv_v2" def __init__(self, op=None, op_desc=None, comm_context=None): - super(RecvOpCost, self).__init__(op=op, - op_desc=op_desc, - comm_context=comm_context) + super(RecvOpCost, self).__init__( + op=op, op_desc=op_desc, comm_context=comm_context + ) def calc_time(self): alpha = self.comm_context.base_ring if self.machine_count > 1: - alpha += self.comm_context.inter_ring + self.hops * self.comm_context.switch + alpha += ( + self.comm_context.inter_ring + + self.hops * self.comm_context.switch + ) else: alpha += self.comm_context.intra_ring beta = self.comm_context.get_max_beta(self.group_ranks) @@ -150,14 +179,17 @@ class SendOpCost(CommOpCost): OP_TYPE = "send_v2" def __init__(self, op=None, op_desc=None, comm_context=None): - super(SendOpCost, self).__init__(op=op, - op_desc=op_desc, - comm_context=comm_context) + super(SendOpCost, self).__init__( + op=op, op_desc=op_desc, comm_context=comm_context + ) def calc_time(self): alpha = self.comm_context.base_ring if self.machine_count > 1: - alpha += self.comm_context.inter_ring + self.hops * self.comm_context.switch + alpha += ( + self.comm_context.inter_ring + + self.hops * self.comm_context.switch + ) else: alpha += self.comm_context.intra_ring beta = self.comm_context.get_max_beta(self.group_ranks) diff --git a/python/paddle/distributed/auto_parallel/cost/comp_op_cost.py b/python/paddle/distributed/auto_parallel/cost/comp_op_cost.py index 938a9465701fa..51c3d6069a691 100644 --- a/python/paddle/distributed/auto_parallel/cost/comp_op_cost.py +++ b/python/paddle/distributed/auto_parallel/cost/comp_op_cost.py @@ -20,9 +20,9 @@ class AdamOpCost(CompOpCost): OP_TYPE = "adam" def __init__(self, op=None, op_desc=None, cluster=None): - super(AdamOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(AdamOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -39,9 +39,9 @@ class AssignOpCost(CompOpCost): OP_TYPE = "assign" def __init__(self, op=None, op_desc=None, cluster=None): - super(AssignOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(AssignOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -58,9 +58,9 @@ class AssignValueOpCost(CompOpCost): OP_TYPE = "assign_value" def __init__(self, op=None, op_desc=None, cluster=None): - super(AssignValueOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(AssignValueOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -77,9 +77,9 @@ class BeamSearchOpCost(CompOpCost): OP_TYPE = "beam_search" def __init__(self, op=None, op_desc=None, cluster=None): - super(BeamSearchOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(BeamSearchOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -96,9 +96,9 @@ class BeamSearchDecodeOpCost(CompOpCost): OP_TYPE = "beam_search_decode" def __init__(self, op=None, op_desc=None, cluster=None): - super(BeamSearchDecodeOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(BeamSearchDecodeOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -115,9 +115,9 @@ class CastOpCost(CompOpCost): OP_TYPE = "cast" def __init__(self, op=None, op_desc=None, cluster=None): - super(CastOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(CastOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -134,9 +134,9 @@ class ConcatOpCost(CompOpCost): OP_TYPE = "concat" def __init__(self, op=None, op_desc=None, cluster=None): - super(ConcatOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ConcatOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -153,9 +153,9 @@ class DropoutOpCost(CompOpCost): OP_TYPE = "dropout" def __init__(self, op=None, op_desc=None, cluster=None): - super(DropoutOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(DropoutOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -172,9 +172,9 @@ class DropoutGradOpCost(CompOpCost): OP_TYPE = "dropout_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(DropoutGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(DropoutGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -191,9 +191,9 @@ class ElementwiseAddOpCost(CompOpCost): OP_TYPE = "elementwise_add" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseAddOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseAddOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -210,9 +210,9 @@ class ElementwiseAddGradOpCost(CompOpCost): OP_TYPE = "elementwise_add_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseAddGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseAddGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -229,9 +229,9 @@ class ElementwiseDivOpCost(CompOpCost): OP_TYPE = "elementwise_div" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseDivOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseDivOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -248,9 +248,9 @@ class ElementwiseDivGradOpCost(CompOpCost): OP_TYPE = "elementwise_div_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseDivGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseDivGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -267,9 +267,9 @@ class ElementwiseMulOpCost(CompOpCost): OP_TYPE = "elementwise_mul" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseMulOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseMulOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -286,9 +286,9 @@ class ElementwiseMulGradOpCost(CompOpCost): OP_TYPE = "elementwise_mul_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseMulGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseMulGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -305,9 +305,9 @@ class ElementwiseSubOpCost(CompOpCost): OP_TYPE = "elementwise_sub" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseSubOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseSubOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -324,9 +324,9 @@ class ElementwiseSubGradOpCost(CompOpCost): OP_TYPE = "elementwise_sub_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(ElementwiseSubGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ElementwiseSubGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -343,9 +343,9 @@ class EmbeddingOpCost(CompOpCost): OP_TYPE = "c_embedding" def __init__(self, op=None, op_desc=None, cluster=None): - super(EmbeddingOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(EmbeddingOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -362,9 +362,9 @@ class EmbeddingGradOpCost(CompOpCost): OP_TYPE = "c_embedding_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(EmbeddingGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(EmbeddingGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -381,9 +381,9 @@ class FillConstantOpCost(CompOpCost): OP_TYPE = "fill_constant" def __init__(self, op=None, op_desc=None, cluster=None): - super(FillConstantOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(FillConstantOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -400,9 +400,9 @@ class FillConstantBatchSizeLikeOpCost(CompOpCost): OP_TYPE = "fill_constant_batch_size_like" def __init__(self, op=None, op_desc=None, cluster=None): - super(FillConstantBatchSizeLikeOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(FillConstantBatchSizeLikeOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -419,8 +419,9 @@ class FusedSoftmaxMaskUpperTriangleOpCost(CompOpCost): OP_TYPE = "fused_softmax_mask_upper_triangle" def __init__(self, op=None, op_desc=None, cluster=None): - super(FusedSoftmaxMaskUpperTriangleOpCost, - self).__init__(op=op, op_desc=op_desc, cluster=cluster) + super(FusedSoftmaxMaskUpperTriangleOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -437,8 +438,9 @@ class FusedSoftmaxMaskUpperTriangleGradOpCost(CompOpCost): OP_TYPE = "fused_softmax_mask_upper_triangle_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(FusedSoftmaxMaskUpperTriangleGradOpCost, - self).__init__(op=op, op_desc=op_desc, cluster=cluster) + super(FusedSoftmaxMaskUpperTriangleGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -455,9 +457,9 @@ class GatherOpCost(CompOpCost): OP_TYPE = "gather" def __init__(self, op=None, op_desc=None, cluster=None): - super(GatherOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(GatherOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -474,9 +476,9 @@ class GeluOpCost(CompOpCost): OP_TYPE = "gelu" def __init__(self, op=None, op_desc=None, cluster=None): - super(GeluOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(GeluOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -493,9 +495,9 @@ class GeluGradOpCost(CompOpCost): OP_TYPE = "gelu_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(GeluGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(GeluGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -512,9 +514,9 @@ class GreaterEqualOpCost(CompOpCost): OP_TYPE = "greater_equal" def __init__(self, op=None, op_desc=None, cluster=None): - super(GreaterEqualOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(GreaterEqualOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -531,9 +533,9 @@ class IncrementOpCost(CompOpCost): OP_TYPE = "increment" def __init__(self, op=None, op_desc=None, cluster=None): - super(IncrementOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(IncrementOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -546,9 +548,9 @@ class IsEmptyOpCost(CompOpCost): OP_TYPE = "is_empty" def __init__(self, op=None, op_desc=None, cluster=None): - super(IsEmptyOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(IsEmptyOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -561,9 +563,9 @@ class LayerNormOpCost(CompOpCost): OP_TYPE = "layer_norm" def __init__(self, op=None, op_desc=None, cluster=None): - super(LayerNormOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LayerNormOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -580,9 +582,9 @@ class LayerNormGradOpCost(CompOpCost): OP_TYPE = "layer_norm_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(LayerNormGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LayerNormGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -599,9 +601,9 @@ class LessThanOpCost(CompOpCost): OP_TYPE = "less_than" def __init__(self, op=None, op_desc=None, cluster=None): - super(LessThanOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LessThanOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -618,9 +620,9 @@ class LogicalNotOpCost(CompOpCost): OP_TYPE = "logical_not" def __init__(self, op=None, op_desc=None, cluster=None): - super(LogicalNotOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LogicalNotOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -637,9 +639,9 @@ class LogicalAndOpCost(CompOpCost): OP_TYPE = "logical_and" def __init__(self, op=None, op_desc=None, cluster=None): - super(LogicalAndOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LogicalAndOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -656,9 +658,9 @@ class LodResetOpCost(CompOpCost): OP_TYPE = "lod_reset" def __init__(self, op=None, op_desc=None, cluster=None): - super(LodResetOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LodResetOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -692,9 +694,9 @@ class LookupTableV2OpCost(CompOpCost): OP_TYPE = "lookup_table_v2" def __init__(self, op=None, op_desc=None, cluster=None): - super(LookupTableV2OpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LookupTableV2OpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -711,9 +713,9 @@ class LookupTableV2GradOpCost(CompOpCost): OP_TYPE = "lookup_table_v2_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(LookupTableV2GradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(LookupTableV2GradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -730,9 +732,9 @@ class MatmulOpCost(CompOpCost): OP_TYPE = "matmul" def __init__(self, op=None, op_desc=None, cluster=None): - super(MatmulOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(MatmulOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -749,9 +751,9 @@ class MatmulGradOpCost(CompOpCost): OP_TYPE = "matmul_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(MatmulGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(MatmulGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -768,9 +770,9 @@ class MatmulV2OpCost(CompOpCost): OP_TYPE = "matmul_v2" def __init__(self, op=None, op_desc=None, cluster=None): - super(MatmulV2OpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(MatmulV2OpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -787,9 +789,9 @@ class MatmulV2GradOpCost(CompOpCost): OP_TYPE = "matmul_v2_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(MatmulV2GradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(MatmulV2GradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -806,9 +808,9 @@ class MemcpyOpCost(CompOpCost): OP_TYPE = "memcpy" def __init__(self, op=None, op_desc=None, cluster=None): - super(MemcpyOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(MemcpyOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -842,9 +844,9 @@ class MulGradOpCost(CompOpCost): OP_TYPE = "mul_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(MulGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(MulGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -861,9 +863,9 @@ class OneHotOpCost(CompOpCost): OP_TYPE = "one_hot" def __init__(self, op=None, op_desc=None, cluster=None): - super(OneHotOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(OneHotOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -880,9 +882,9 @@ class ReadFromArrayOpCost(CompOpCost): OP_TYPE = "read_from_array" def __init__(self, op=None, op_desc=None, cluster=None): - super(ReadFromArrayOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ReadFromArrayOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -899,9 +901,9 @@ class ReduceSumOpCost(CompOpCost): OP_TYPE = "reduce_sum" def __init__(self, op=None, op_desc=None, cluster=None): - super(ReduceSumOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ReduceSumOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -918,9 +920,9 @@ class ReduceSumGradOpCost(CompOpCost): OP_TYPE = "reduce_sum_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(ReduceSumGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ReduceSumGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -937,9 +939,9 @@ class Reshape2OpCost(CompOpCost): OP_TYPE = "reshape2" def __init__(self, op=None, op_desc=None, cluster=None): - super(Reshape2OpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(Reshape2OpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -956,9 +958,9 @@ class Reshape2GradOpCost(CompOpCost): OP_TYPE = "reshape2_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(Reshape2GradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(Reshape2GradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -975,9 +977,9 @@ class ReduceMeanOpCost(CompOpCost): OP_TYPE = "reduce_mean" def __init__(self, op=None, op_desc=None, cluster=None): - super(ReduceMeanOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ReduceMeanOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -994,9 +996,9 @@ class ReduceMeanGradOpCost(CompOpCost): OP_TYPE = "reduce_mean_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(ReduceMeanGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ReduceMeanGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1013,9 +1015,9 @@ class SamplingIdOpCost(CompOpCost): OP_TYPE = "sampling_id" def __init__(self, op=None, op_desc=None, cluster=None): - super(SamplingIdOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SamplingIdOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1032,9 +1034,9 @@ class ScaleOpCost(CompOpCost): OP_TYPE = "scale" def __init__(self, op=None, op_desc=None, cluster=None): - super(ScaleOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(ScaleOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1051,9 +1053,9 @@ class SliceOpCost(CompOpCost): OP_TYPE = "slice" def __init__(self, op=None, op_desc=None, cluster=None): - super(SliceOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SliceOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1070,9 +1072,9 @@ class SoftmaxOpCost(CompOpCost): OP_TYPE = "softmax" def __init__(self, op=None, op_desc=None, cluster=None): - super(SoftmaxOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SoftmaxOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1089,9 +1091,9 @@ class SoftmaxGradOpCost(CompOpCost): OP_TYPE = "softmax_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(SoftmaxGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SoftmaxGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1108,9 +1110,9 @@ class SoftmaxWithCrossEntropyOpCost(CompOpCost): OP_TYPE = "softmax_with_cross_entropy" def __init__(self, op=None, op_desc=None, cluster=None): - super(SoftmaxWithCrossEntropyOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SoftmaxWithCrossEntropyOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1127,9 +1129,9 @@ class SoftmaxWithCrossEntropyGradOpCost(CompOpCost): OP_TYPE = "softmax_with_cross_entropy_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(SoftmaxWithCrossEntropyGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SoftmaxWithCrossEntropyGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1146,9 +1148,9 @@ class SplitOpCost(CompOpCost): OP_TYPE = "split" def __init__(self, op=None, op_desc=None, cluster=None): - super(SplitOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SplitOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1165,9 +1167,9 @@ class Squeeze2OpCost(CompOpCost): OP_TYPE = "squeeze2" def __init__(self, op=None, op_desc=None, cluster=None): - super(Squeeze2OpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(Squeeze2OpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1184,9 +1186,9 @@ class SquareOpCost(CompOpCost): OP_TYPE = "square" def __init__(self, op=None, op_desc=None, cluster=None): - super(SquareOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SquareOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1203,9 +1205,9 @@ class SquareGradOpCost(CompOpCost): OP_TYPE = "square_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(SquareGradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(SquareGradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1239,9 +1241,9 @@ class TopKOpCost(CompOpCost): OP_TYPE = "top_k" def __init__(self, op=None, op_desc=None, cluster=None): - super(TopKOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(TopKOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1258,9 +1260,9 @@ class Transpose2OpCost(CompOpCost): OP_TYPE = "transpose2" def __init__(self, op=None, op_desc=None, cluster=None): - super(Transpose2OpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(Transpose2OpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1277,9 +1279,9 @@ class Transpose2GradOpCost(CompOpCost): OP_TYPE = "transpose2_grad" def __init__(self, op=None, op_desc=None, cluster=None): - super(Transpose2GradOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(Transpose2GradOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1296,9 +1298,9 @@ class Unsqueeze2OpCost(CompOpCost): OP_TYPE = "unsqueeze2" def __init__(self, op=None, op_desc=None, cluster=None): - super(Unsqueeze2OpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(Unsqueeze2OpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): @@ -1315,9 +1317,9 @@ class WriteToArrayOpCost(CompOpCost): OP_TYPE = "write_to_array" def __init__(self, op=None, op_desc=None, cluster=None): - super(WriteToArrayOpCost, self).__init__(op=op, - op_desc=op_desc, - cluster=cluster) + super(WriteToArrayOpCost, self).__init__( + op=op, op_desc=op_desc, cluster=cluster + ) # For a concrete COMP OP, the calc_time and calc_flops function need to be overrided def calc_flops(self): diff --git a/python/paddle/distributed/auto_parallel/cost/estimate_cost.py b/python/paddle/distributed/auto_parallel/cost/estimate_cost.py index fba9d7919c406..cac8bf9f277e4 100644 --- a/python/paddle/distributed/auto_parallel/cost/estimate_cost.py +++ b/python/paddle/distributed/auto_parallel/cost/estimate_cost.py @@ -26,12 +26,9 @@ class CostEstimator: _sepical_op_type = ["fused_attention", "fused_feedforward"] - def __init__(self, - program, - cluster, - mode="modeling", - rank=None, - loop_count=10): + def __init__( + self, program, cluster, mode="modeling", rank=None, loop_count=10 + ): self._program = program self._cluster = cluster self._check_mode(mode) @@ -40,7 +37,8 @@ def __init__(self, self._loop_count = loop_count self._global_cost = Cost() self._local_cost_mapping = {} - self._detailed_cost = OrderedDict( + self._detailed_cost = ( + OrderedDict() ) # {`op_id`: {"reshard": [], "dist_op": [], "local_cost": local_cost}}} self._bubble_time_mapping = {} self._ordered_ops = [] @@ -105,7 +103,8 @@ def local_bubble_time(self, rank=None): def _check_mode(self, mode): if mode not in ["modeling", "profiling"]: raise ValueError( - "Just support modeling and profiling, but got {}".format(mode)) + "Just support modeling and profiling, but got {}".format(mode) + ) def _is_special_var_name(self, var_name): special_var_name = ["lod_tensor_blocking_queue_0"] @@ -115,6 +114,7 @@ def _is_special_var_name(self, var_name): def _estimate_core(self, dist_context, resharder, block): from ..reshard import get_var_with_recursion + ops = block.ops loop_count = None if block.desc.id != self.program.global_block().desc.id: @@ -131,8 +131,9 @@ def _estimate_core(self, dist_context, resharder, block): if int(op.attr('op_role')) == int(OpRole.Optimize): continue if op.type in [ - "create_py_reader", "create_double_buffer_reader", - "read" + "create_py_reader", + "create_double_buffer_reader", + "read", ]: continue @@ -171,14 +172,16 @@ def _estimate_core(self, dist_context, resharder, block): max_time = rank_cost.time for rank in group_ranks: - self.local_cost( - rank).time = max_time + cost.time + self.local_cost(rank).time = ( + max_time + cost.time + ) if rank not in self._bubble_time_mapping: self._bubble_time_mapping[rank] = 0 self._bubble_time_mapping[rank] += ( - max_time - cost_time[rank]) + max_time - cost_time[rank] + ) for rank in local_comp_cost: for comp_cost in local_comp_cost[rank]: @@ -190,15 +193,19 @@ def _estimate_core(self, dist_context, resharder, block): processes = op_dist_attr.process_mesh.processes container = get_distributed_operator_impl_container( - op_dist_attr.impl_type) + op_dist_attr.impl_type + ) dist_impl = container.impls[op_dist_attr.impl_idx] - dist_op_cost = dist_impl.calc_cost(op.attr('op_role'), dist_op, - dist_context, self.cluster) + dist_op_cost = dist_impl.calc_cost( + op.attr('op_role'), dist_op, dist_context, self.cluster + ) detail["dist_op_cost"] = dist_op_cost if dist_op_cost is None: - assert dist_op.serial_op.type in CostEstimator._sepical_op_type + assert ( + dist_op.serial_op.type in CostEstimator._sepical_op_type + ) continue for item in dist_op_cost: if isinstance(item, list): @@ -216,12 +223,14 @@ def _estimate_core(self, dist_context, resharder, block): if max_time < rank_cost.time: max_time = rank_cost.time for rank in group_ranks: - self.local_cost( - rank).time = max_time + comm_op_cost.time + self.local_cost(rank).time = ( + max_time + comm_op_cost.time + ) if rank not in self._bubble_time_mapping: self._bubble_time_mapping[rank] = 0 self._bubble_time_mapping[rank] += ( - max_time - cost_time[rank]) + max_time - cost_time[rank] + ) elif isinstance(item, dict): # Op just one for rank in processes: @@ -246,8 +255,11 @@ def _calculate_bytes(self, sizes, dtype): dtype_factor = 8 elif dtype == paddle.float32 or dtype == paddle.int32: dtype_factor = 4 - elif dtype == paddle.float16 or dtype == paddle.bfloat16 \ - or dtype == paddle.int16: + elif ( + dtype == paddle.float16 + or dtype == paddle.bfloat16 + or dtype == paddle.int16 + ): dtype_factor = 2 elif dtype == paddle.int8 or dtype == paddle.uint8: dtype_factor = 1 @@ -269,8 +281,9 @@ def _convert_pm_and_dm_to_str(process_mesh, dims_mapping): memories = {} self.max_memories = {} - var_info = { - } # var_name: [[process_mesh, dims_mapping], [id]], [[process_mesh, dims_mapping], [id]]} + var_info = ( + {} + ) # var_name: [[process_mesh, dims_mapping], [id]], [[process_mesh, dims_mapping], [id]]} for block in self.program.blocks: for op in block.ops: @@ -279,18 +292,22 @@ def _convert_pm_and_dm_to_str(process_mesh, dims_mapping): for op_id, op in self._ordered_ops: if op.type in [ - "create_py_reader", "create_double_buffer_reader", "read" + "create_py_reader", + "create_double_buffer_reader", + "read", ]: continue dist_op = dist_context.get_dist_op_for_program(op) process_mesh = dist_op.dist_attr.process_mesh for var_name in op.input_arg_names: input_dims_mapping = dist_op.dist_attr.get_input_dims_mapping( - var_name) + var_name + ) if var_name not in var_info: var_info[var_name] = {} - key = _convert_pm_and_dm_to_str(process_mesh, - input_dims_mapping) + key = _convert_pm_and_dm_to_str( + process_mesh, input_dims_mapping + ) if key not in var_info[var_name]: var_info[var_name][key] = {} # It is even partition now @@ -299,21 +316,27 @@ def _convert_pm_and_dm_to_str(process_mesh, dims_mapping): global_sizes = var.shape dtype = var.dtype sizes = DistributedTensor.get_local_sizes( - global_sizes, input_dims_mapping, process_mesh.topology, - process_mesh.processes) + global_sizes, + input_dims_mapping, + process_mesh.topology, + process_mesh.processes, + ) var_info[var_name][key]["memory"] = self._calculate_bytes( - sizes, dtype) + sizes, dtype + ) if "position" not in var_info[var_name][key]: var_info[var_name][key]["position"] = [] var_info[var_name][key]["position"].append(op_id) for var_name in op.output_arg_names: output_dims_mapping = dist_op.dist_attr.get_output_dims_mapping( - var_name) + var_name + ) if var_name not in var_info: var_info[var_name] = {} - key = _convert_pm_and_dm_to_str(process_mesh, - output_dims_mapping) + key = _convert_pm_and_dm_to_str( + process_mesh, output_dims_mapping + ) if key not in var_info[var_name]: var_info[var_name][key] = {} if "memory" not in var_info[var_name][key]: @@ -321,10 +344,14 @@ def _convert_pm_and_dm_to_str(process_mesh, dims_mapping): global_sizes = var.shape dtype = var.dtype sizes = DistributedTensor.get_local_sizes( - global_sizes, output_dims_mapping, - process_mesh.topology, process_mesh.processes) + global_sizes, + output_dims_mapping, + process_mesh.topology, + process_mesh.processes, + ) var_info[var_name][key]["memory"] = self._calculate_bytes( - sizes, dtype) + sizes, dtype + ) if "position" not in var_info[var_name][key]: var_info[var_name][key]["position"] = [] var_info[var_name][key]["position"].append(op_id) @@ -332,7 +359,9 @@ def _convert_pm_and_dm_to_str(process_mesh, dims_mapping): has_used_vars = set() for op_id, op in self._ordered_ops: if op.type in [ - "create_py_reader", "create_double_buffer_reader", "read" + "create_py_reader", + "create_double_buffer_reader", + "read", ]: continue can_free_memories = {} @@ -341,9 +370,11 @@ def _convert_pm_and_dm_to_str(process_mesh, dims_mapping): process_mesh = dist_op.dist_attr.process_mesh for var_name in op.input_arg_names: input_dims_mapping = dist_op.dist_attr.get_input_dims_mapping( - var_name) - key = _convert_pm_and_dm_to_str(process_mesh, - input_dims_mapping) + var_name + ) + key = _convert_pm_and_dm_to_str( + process_mesh, input_dims_mapping + ) has_used_var = var_name + key var = dist_op.get_serial_input(var_name) # Not used @@ -363,13 +394,16 @@ def _convert_pm_and_dm_to_str(process_mesh, dims_mapping): if process not in can_free_memories: can_free_memories[process] = 0 can_free_memories[process] += var_info[ - var_name][key]["memory"] + var_name + ][key]["memory"] for var_name in op.output_arg_names: output_dims_mapping = dist_op.dist_attr.get_output_dims_mapping( - var_name) - key = _convert_pm_and_dm_to_str(process_mesh, - output_dims_mapping) + var_name + ) + key = _convert_pm_and_dm_to_str( + process_mesh, output_dims_mapping + ) has_used_var = var_name + key var = dist_op.get_serial_output(var_name) # Not used @@ -389,7 +423,8 @@ def _convert_pm_and_dm_to_str(process_mesh, dims_mapping): if process not in can_free_memories: can_free_memories[process] = 0 can_free_memories[process] += var_info[ - var_name][key]["memory"] + var_name + ][key]["memory"] # Calc peak memory for process in memories: @@ -413,8 +448,12 @@ def _convert_pm_and_dm_to_str(process_mesh, dims_mapping): def estimate(self, dist_context, resharder=None): self.prepare() from ..reshard import Resharder - resharder = Resharder(self.program, None, self.rank, dist_context, - []) if resharder is None else resharder + + resharder = ( + Resharder(self.program, None, self.rank, dist_context, []) + if resharder is None + else resharder + ) block = self.program.global_block() self._estimate_core(dist_context, resharder, block) @@ -446,7 +485,7 @@ def _pretty_print_memory_cost(self): memories = [ int(item // 1e6) for item in list(self.max_memories.values()) ] - for memory in (memories + header): + for memory in memories + header: if len(str(memory)) > max_len: max_len = len(str(memory)) max_len += 4 # for pretty print of center @@ -476,7 +515,7 @@ def _pretty_print_global(self): max_len = 0 header = ["Execution Time(ms)", "Max Memory(MiB)"] vals = [round(self.global_cost.time, 3), int(self.max_memory // 1e6)] - for memory in (vals + header): + for memory in vals + header: if len(str(memory)) > max_len: max_len = len(str(memory)) max_len += 4 # for pretty print of center @@ -506,16 +545,27 @@ def pretty_print_cost(self): def get_cost_from_engine(engine, mode): from ..utils import to_list + # Construct cost estimator by original main program - serial_main_prog = engine._serial_main_progs[mode].clone( - ) if mode in engine._serial_main_progs else engine._orig_main_prog.clone() + serial_main_prog = ( + engine._serial_main_progs[mode].clone() + if mode in engine._serial_main_progs + else engine._orig_main_prog.clone() + ) - serial_startup_prog = engine._serial_startup_progs[mode].clone( - ) if mode in engine._serial_startup_progs else engine._orig_startup_prog.clone( + serial_startup_prog = ( + engine._serial_startup_progs[mode].clone() + if mode in engine._serial_startup_progs + else engine._orig_startup_prog.clone() + ) + losses = ( + to_list(engine._loss) + if ( + not isinstance(engine._loss, paddle.nn.Layer) + and not callable(engine._loss) + ) + else engine._losses ) - losses = to_list( - engine._loss) if (not isinstance(engine._loss, paddle.nn.Layer) - and not callable(engine._loss)) else engine._losses if mode in engine._dist_contexts: dist_context = engine._dist_contexts[mode] @@ -523,33 +573,45 @@ def get_cost_from_engine(engine, mode): else: from ..completion import Completer from ..dist_context import DistributedContext - dist_context = DistributedContext(serial_main_prog, serial_startup_prog, - engine._optimizer, losses, {}, - {"loss": losses}, engine._cluster, - engine._strategy) + + dist_context = DistributedContext( + serial_main_prog, + serial_startup_prog, + engine._optimizer, + losses, + {}, + {"loss": losses}, + engine._cluster, + engine._strategy, + ) completer = Completer(dist_context) completer.complete_forward_annotation() dist_context.block_state.parse_forward_blocks( - dist_context.serial_main_program) + dist_context.serial_main_program + ) if mode == "eval" or mode == "predict": cost_estimator = CostEstimator(serial_main_prog, engine._cluster) elif mode == "train": from ..parallelizer_v2 import Parallelizer + # Get serial main program with backward serial_optimizer = engine._optimizer parallelizer = Parallelizer(mode, completer, dist_context) # Generate backward loss_name = dist_context.serial_loss.name serial_loss = serial_main_prog.global_block()._var_recursive(loss_name) - params_grads = parallelizer._generate_backward(serial_main_prog, - serial_startup_prog, - serial_loss) + params_grads = parallelizer._generate_backward( + serial_main_prog, serial_startup_prog, serial_loss + ) # Generate optimizer optimizer_ops = parallelizer._generate_optimizer( - serial_main_prog, serial_startup_prog, serial_optimizer, - params_grads) + serial_main_prog, + serial_startup_prog, + serial_optimizer, + params_grads, + ) cost_estimator = CostEstimator(serial_main_prog, engine._cluster) # Estimate global_cost and max memory diff --git a/python/paddle/distributed/auto_parallel/cost/tensor_cost.py b/python/paddle/distributed/auto_parallel/cost/tensor_cost.py index 9741020da6512..0303e29749f9a 100644 --- a/python/paddle/distributed/auto_parallel/cost/tensor_cost.py +++ b/python/paddle/distributed/auto_parallel/cost/tensor_cost.py @@ -22,7 +22,6 @@ class TensorCost: - def __init__(self, tensor=None, dist_tensor=None, shape=None, dtype=None): self._check_args(tensor, dist_tensor, shape, dtype) self._tensor = tensor @@ -49,31 +48,35 @@ def dtype(self): def _check_args(self, tensor, dist_tensor, shape, dtype): if tensor is not None: - assert (shape is None and dist_tensor is None and dtype is None) + assert shape is None and dist_tensor is None and dtype is None if not isinstance(tensor, Variable): raise TypeError( "Please check tensor type is Variable, but got {}".format( - type(tensor))) + type(tensor) + ) + ) elif dist_tensor is not None: - assert (tensor is None and shape is None) + assert tensor is None and shape is None if not isinstance(dist_tensor, DistributedTensor): raise TypeError( - "Please check dist_tensor type is DistributedTensor, but got {}" - .format(type(dist_tensor))) + "Please check dist_tensor type is DistributedTensor, but got {}".format( + type(dist_tensor) + ) + ) elif shape is not None: - assert (tensor is None and dist_tensor is None - and dtype is not None) + assert tensor is None and dist_tensor is None and dtype is not None if not isinstance(shape, (list, set)): raise TypeError( "Please check shape type is list or set, but got {}".format( - type(shape))) + type(shape) + ) + ) elif dtype is not None: - assert (tensor is None and dist_tensor is None - and shape is not None) + assert tensor is None and dist_tensor is None and shape is not None @property def cost(self): diff --git a/python/paddle/distributed/auto_parallel/cost_model.py b/python/paddle/distributed/auto_parallel/cost_model.py index cdcc19e27d792..dfff44d0bde75 100644 --- a/python/paddle/distributed/auto_parallel/cost_model.py +++ b/python/paddle/distributed/auto_parallel/cost_model.py @@ -36,7 +36,6 @@ class CostNodeType(Enum): class Cost(object): - def __init__(self): self.runtime = None self.static_mem = None @@ -51,7 +50,6 @@ class CostModelMode(Enum): class CostNode(object): - def __init__(self, node, node_type, id=None): self.id = id self.node = node @@ -72,7 +70,6 @@ def cost(self, cost): class MergedOpsCostNode(CostNode): - def __init__(self, node_type, id=None, base_node_list=None, is_bwd=False): super(MergedOpsCostNode, self).__init__(None, node_type, id) self.node_list = base_node_list @@ -80,13 +77,9 @@ def __init__(self, node_type, id=None, base_node_list=None, is_bwd=False): class CommOpCostNode(CostNode): - - def __init__(self, - node, - node_type, - id=None, - comm_node_list=None, - is_bwd=False): + def __init__( + self, node, node_type, id=None, comm_node_list=None, is_bwd=False + ): super(CommOpCostNode, self).__init__(node, node_type, id) self.node_list = comm_node_list self.ranks = [] @@ -108,8 +101,9 @@ def init_comm_cost(self, cluster=None): comm_volumn = np.prod(self.input_shape) * 4 if 'allreduce' in self.comm_type: - self._cost = comm_volumn / (BANDWIDTH * num_ranks / - (2 * (num_ranks - 1))) + self._cost = comm_volumn / ( + BANDWIDTH * num_ranks / (2 * (num_ranks - 1)) + ) elif 'gather' in self.comm_type: self._cost = comm_volumn / (BANDWIDTH * num_ranks / (num_ranks - 1)) elif 'broadcast' in self.comm_type: @@ -121,14 +115,15 @@ def init_comm_cost(self, cluster=None): class TensorCostNode(CostNode): - - def __init__(self, - node, - node_type, - id=None, - base_node_list=None, - batch_size=None, - shared_node_id=None): + def __init__( + self, + node, + node_type, + id=None, + base_node_list=None, + batch_size=None, + shared_node_id=None, + ): super(TensorCostNode, self).__init__(node, node_type, id) if node.name == "create_py_reader_0" or node.name == "double_buffer_0": self.shape = [2, 2] @@ -163,7 +158,6 @@ def get_size(self): class CompOpCostNode(CostNode): - def __init__(self, node, node_type, id=None, is_bwd=False, is_optim=False): super(CompOpCostNode, self).__init__(node, node_type, id) self.is_bwd = is_bwd @@ -179,7 +173,6 @@ def init_comp_cost(self, cost_data): class PipeEvent(object): - def __init__(self, stage_id, event_name, duration, start_time=-1): self.stage_id = stage_id self.name = event_name @@ -189,15 +182,16 @@ def __init__(self, stage_id, event_name, duration, start_time=-1): class CostModel(object): - - def __init__(self, - mode=CostModelMode.BENCHMARKING, - cluster=None, - batch_size=1, - microbatch_num=1, - opcall_overhead=0, - standalone_cost_data=None, - pipeline_config=None): + def __init__( + self, + mode=CostModelMode.BENCHMARKING, + cluster=None, + batch_size=1, + microbatch_num=1, + opcall_overhead=0, + standalone_cost_data=None, + pipeline_config=None, + ): self.mode = mode # parameters @@ -229,17 +223,20 @@ def __init__(self, self.optim_time = [] def _parse_sub_program(self, program, nodes, graph, cost_data, sub_idx): - assert len( - program.blocks) == 1, "Program more than 1 block not supported." + assert ( + len(program.blocks) == 1 + ), "Program more than 1 block not supported." block = program.blocks[0] var_id = "lod_tensor_blocking_queue_0" new_var = program.global_block().create_var( name=var_id, dtype=paddle.float32, - type=core.VarDesc.VarType.LOD_TENSOR) - nodes[var_id] = TensorCostNode(new_var, CostNodeType.VARIABLE, - "lod_tensor_blocking_queue_0") + type=core.VarDesc.VarType.LOD_TENSOR, + ) + nodes[var_id] = TensorCostNode( + new_var, CostNodeType.VARIABLE, "lod_tensor_blocking_queue_0" + ) for var in block.vars.values(): var_id = var.name # if var.name == "create_py_reader_0" or var.name == "double_buffer_0": @@ -249,13 +246,17 @@ def _parse_sub_program(self, program, nodes, graph, cost_data, sub_idx): for op in block.ops: op_id = op.type + "_" + str(op.idx) - if op.type.startswith('c_') or op.type.startswith( - 'send') or op.type.startswith('recv'): + if ( + op.type.startswith('c_') + or op.type.startswith('send') + or op.type.startswith('recv') + ): is_bwd = False - if op.type.startswith( - 'c_' - ) and op.type != "c_sync_calc_stream" and not op.type.startswith( - 'c_embedding'): + if ( + op.type.startswith('c_') + and op.type != "c_sync_calc_stream" + and not op.type.startswith('c_embedding') + ): ring_id = op.attr('ring_id') if ring_id not in self.ring2rank: self.ring2rank[ring_id] = set() @@ -265,14 +266,17 @@ def _parse_sub_program(self, program, nodes, graph, cost_data, sub_idx): is_bwd = '@GRAD' in op.output('Out')[0] elif op.type.startswith('send'): is_bwd = '@GRAD' in op.input('X')[0] - op_node = CommOpCostNode(op, CostNodeType.COMMUNICATION, op_id, - is_bwd) + op_node = CommOpCostNode( + op, CostNodeType.COMMUNICATION, op_id, is_bwd + ) else: - is_bwd = (int(op.attr('op_role')) == int( - OpRole.Backward)) or "@GRAD" in op.input_arg_names + is_bwd = ( + int(op.attr('op_role')) == int(OpRole.Backward) + ) or "@GRAD" in op.input_arg_names is_optim = 'LearningRate' in op.input_names - op_node = CompOpCostNode(op, CostNodeType.COMPUTATION, op_id, - is_bwd, is_optim) + op_node = CompOpCostNode( + op, CostNodeType.COMPUTATION, op_id, is_bwd, is_optim + ) op_node.init_comp_cost(cost_data) nodes[op_id] = op_node @@ -310,17 +314,20 @@ def _parse_sub_program(self, program, nodes, graph, cost_data, sub_idx): for pred_id in graph[node_id][PRED]: pred = nodes[pred_id] if pred.type == CostNodeType.COMPUTATION and ( - pred_id in graph[node_id][SUCC]): + pred_id in graph[node_id][SUCC] + ): graph[pred_id][SUCC].remove(node_id) graph[node_id][PRED].remove(pred_id) write_op_cnt += 1 new_var_id = node_id + '_write_{}'.format(write_op_cnt) - new_var = TensorCostNode(node.node, - CostNodeType.VARIABLE, - new_var_id, - shared_node_id=node_id) + new_var = TensorCostNode( + node.node, + CostNodeType.VARIABLE, + new_var_id, + shared_node_id=node_id, + ) graph[new_var_id] = [[], []] graph[pred_id][SUCC].append(new_var_id) @@ -346,23 +353,31 @@ def parse_program(self, distributed_program): self.op_graph.append({}) self.runtime_graph.append({}) self._parse_sub_program( - sub_prog, self.nodes[sub_idx], self.origin_graph[sub_idx], - self.cost_data[0 if self.rank2pp is None else self. - rank2pp[sub_idx]], sub_idx) + sub_prog, + self.nodes[sub_idx], + self.origin_graph[sub_idx], + self.cost_data[ + 0 if self.rank2pp is None else self.rank2pp[sub_idx] + ], + sub_idx, + ) return self.nodes def _find_succ_op(self, node_id, sub_idx=0): succ_ops_id = [] for succ_id in self.origin_graph[sub_idx][node_id][SUCC]: succ = self.nodes[sub_idx][succ_id] - if succ.type == CostNodeType.COMMUNICATION or \ - succ.type == CostNodeType.COMPUTATION: + if ( + succ.type == CostNodeType.COMMUNICATION + or succ.type == CostNodeType.COMPUTATION + ): succ_ops_id.append(succ_id) elif succ.type == CostNodeType.VARIABLE: succ_ops_id = succ_ops_id + self._find_succ_op(succ_id, sub_idx) else: raise NotImplementedError( - 'This type of node not supported yet:{}'.format(succ.type)) + 'This type of node not supported yet:{}'.format(succ.type) + ) return succ_ops_id def build_op_graph(self): @@ -392,9 +407,11 @@ def merge_comm(self): for sub_idx in range(self.total_rank): for node_id, edges in self.op_graph[sub_idx].items(): node = self.nodes[sub_idx][node_id] - if node_id.startswith('c_') and not node.id.startswith( - "c_sync_calc_stream") and not node.id.startswith( - 'c_embedding'): + if ( + node_id.startswith('c_') + and not node.id.startswith("c_sync_calc_stream") + and not node.id.startswith('c_embedding') + ): ring_id = node.node.attr('ring_id') node.set_ranks(list(self.ring2rank[ring_id])) node.init_comm_cost(self.cluster) @@ -420,13 +437,17 @@ def _merge_node(self, to_merge_node_list, merge_type='linear', nodes=None): else: raise NotImplementedError( 'This type of merging is not supported:{}'.format( - merge_type)) + merge_type + ) + ) merged_node_id = 'merged_' + str(len(nodes)) is_bwd = to_merge_node_list[0].is_bwd - merged_node = MergedOpsCostNode(CostNodeType.MERGED, - id=merged_node_id, - base_node_list=nodes_list, - is_bwd=is_bwd) + merged_node = MergedOpsCostNode( + CostNodeType.MERGED, + id=merged_node_id, + base_node_list=nodes_list, + is_bwd=is_bwd, + ) merged_node.cost = node_cost return merged_node_id, merged_node @@ -440,12 +461,12 @@ def merge_linear(self): ''' cnt = 0 for sub_idx in range(self.total_rank): - cnt += self._merge_linear(self.nodes[sub_idx], - self.runtime_graph[sub_idx], - is_bwd=False) - cnt += self._merge_linear(self.nodes[sub_idx], - self.runtime_graph[sub_idx], - is_bwd=True) + cnt += self._merge_linear( + self.nodes[sub_idx], self.runtime_graph[sub_idx], is_bwd=False + ) + cnt += self._merge_linear( + self.nodes[sub_idx], self.runtime_graph[sub_idx], is_bwd=True + ) return cnt def merge_branch(self): @@ -461,12 +482,12 @@ def merge_branch(self): ''' cnt = 0 for sub_idx in range(self.total_rank): - cnt += self._merge_branch(self.nodes[sub_idx], - self.runtime_graph[sub_idx], - is_bwd=False) - cnt += self._merge_branch(self.nodes[sub_idx], - self.runtime_graph[sub_idx], - is_bwd=True) + cnt += self._merge_branch( + self.nodes[sub_idx], self.runtime_graph[sub_idx], is_bwd=False + ) + cnt += self._merge_branch( + self.nodes[sub_idx], self.runtime_graph[sub_idx], is_bwd=True + ) return cnt def _merge_linear(self, nodes, runtime_graph, is_bwd=False): @@ -484,7 +505,8 @@ def _merge_linear(self, nodes, runtime_graph, is_bwd=False): pred_id = edges[PRED][0] pred = nodes[pred_id] merged_node_id, merged_node = self._merge_node( - [node, pred], merge_type='linear', nodes=nodes) + [node, pred], merge_type='linear', nodes=nodes + ) nodes[merged_node_id] = merged_node runtime_graph[merged_node_id] = [[], []] @@ -492,7 +514,8 @@ def _merge_linear(self, nodes, runtime_graph, is_bwd=False): succ = None try: runtime_graph[merged_node_id][SUCC] = copy.deepcopy( - edges[SUCC]) + edges[SUCC] + ) if len(runtime_graph[pred_id][SUCC]) > 1: # predecessor has more than 1 successor @@ -501,7 +524,8 @@ def _merge_linear(self, nodes, runtime_graph, is_bwd=False): succ.remove(node_id) runtime_graph[merged_node_id][SUCC] += succ runtime_graph[merged_node_id][PRED] = runtime_graph[ - pred_id][PRED] + pred_id + ][PRED] except: pass try: @@ -567,16 +591,20 @@ def _merge_branch(self, nodes, runtime_graph, is_bwd=False): to_merge = True try: - if len(edges[SUCC]) < 1 or len( - runtime_graph[edges[SUCC][0]][SUCC]) < 1: + if ( + len(edges[SUCC]) < 1 + or len(runtime_graph[edges[SUCC][0]][SUCC]) < 1 + ): continue except: continue end_node_id = runtime_graph[edges[SUCC][0]][SUCC][0] for i in succ_nodes_id: try: - if len(runtime_graph[i][SUCC]) != 1 or \ - runtime_graph[i][SUCC][0] != end_node_id: + if ( + len(runtime_graph[i][SUCC]) != 1 + or runtime_graph[i][SUCC][0] != end_node_id + ): to_merge = False # if branches has different end node, we don't merge them break except: @@ -584,7 +612,8 @@ def _merge_branch(self, nodes, runtime_graph, is_bwd=False): if to_merge and len(succ_nodes_id) > 1: to_merge_node_list = [nodes[i] for i in succ_nodes_id] merged_node_id, merged_node = self._merge_node( - to_merge_node_list, merge_type='branch', nodes=nodes) + to_merge_node_list, merge_type='branch', nodes=nodes + ) nodes[merged_node_id] = merged_node runtime_graph[merged_node_id] = [[], []] @@ -605,7 +634,6 @@ def _merge_branch(self, nodes, runtime_graph, is_bwd=False): return reduct_cnt def get_runtime_cost(self): - def get_node_cost(node): node_cost = node.cost + self.opcall_overhead if isinstance(node, MergedOpsCostNode): @@ -635,7 +663,8 @@ def get_mem(self): top_list = [] for sub_idx in range(self.total_rank): static_mem, cur_mem, top_mem = self._simulate_mem( - self.nodes[sub_idx], self.origin_graph[sub_idx]) + self.nodes[sub_idx], self.origin_graph[sub_idx] + ) static_list.append(static_mem) top_list.append(top_mem) return static_list, top_list @@ -670,8 +699,9 @@ def _simulate_mem(self, nodes, origin_graph): static_mem += size cur_mem += size edges = sim_graph[node_id] - if not (node.type == CostNodeType.VARIABLE - and node.node.persistable): + if not ( + node.type == CostNodeType.VARIABLE and node.node.persistable + ): for succ_id in edges[SUCC]: sim_graph[succ_id][PRED].remove(node_id) if len(sim_graph[succ_id][PRED]) == 0: @@ -680,8 +710,10 @@ def _simulate_mem(self, nodes, origin_graph): pred = nodes if pred.type == CostNodeType.VARIABLE: sim_graph[pred_id][SUCC].remove(node_id) - if len(sim_graph[pred_id] - [SUCC]) == 0 and not pred.node.persistable: + if ( + len(sim_graph[pred_id][SUCC]) == 0 + and not pred.node.persistable + ): cur_mem -= pred.get_size() return static_mem, cur_mem, top_mem @@ -713,16 +745,22 @@ def _simulate_pipeline(self): event_list.append(e) if stid != stage_num - 1: q.put( - PipeEvent(stid + 1, - 'fwd', - self.fwd_time[stid + 1], - start_time=e.e_time)) + PipeEvent( + stid + 1, + 'fwd', + self.fwd_time[stid + 1], + start_time=e.e_time, + ) + ) else: q.put( - PipeEvent(stid, - 'bwd', - self.bwd_time[stid], - start_time=e.e_time)) + PipeEvent( + stid, + 'bwd', + self.bwd_time[stid], + start_time=e.e_time, + ) + ) fwd_cnt[stid] -= 1 global_time[stid] = e.e_time else: @@ -733,18 +771,24 @@ def _simulate_pipeline(self): event_list.append(e) if stid != 0: q.put( - PipeEvent(stid - 1, - 'bwd', - self.bwd_time[stid - 1], - start_time=e.e_time)) + PipeEvent( + stid - 1, + 'bwd', + self.bwd_time[stid - 1], + start_time=e.e_time, + ) + ) fwd_cnt[stid] += 1 bwd_cnt[stid] -= 1 if bwd_cnt[stid] == 0: q.put( - PipeEvent(stid, - 'optim', - self.optim_time[stid], - start_time=e.e_time)) + PipeEvent( + stid, + 'optim', + self.optim_time[stid], + start_time=e.e_time, + ) + ) global_time[stid] = e.e_time elif e.name == 'optim': e.s_time = max(global_time[stid], e.s_time) @@ -754,7 +798,9 @@ def _simulate_pipeline(self): else: raise NotImplementedError( 'This type of pipe event is not supported yet.{}'.format( - e.name)) + e.name + ) + ) for t in global_time: total_time = max(total_time, t) @@ -784,8 +830,13 @@ def init(self, distributed_program): self.build_runtime_graph() -def estimate_cost(distributed_program, cluster, pipeline_config, - standalone_cost_data, batch_size): +def estimate_cost( + distributed_program, + cluster, + pipeline_config, + standalone_cost_data, + batch_size, +): """ Estimated cost from distributed program, cluster model and distributed settings. @@ -798,10 +849,12 @@ def estimate_cost(distributed_program, cluster, pipeline_config, """ # the following line is left for now, cluster model will be involved in the future assert cluster is None, "For now, cluster remains None" - cm_ctx = CostModel(cluster=cluster, - batch_size=batch_size, - standalone_cost_data=standalone_cost_data, - pipeline_config=pipeline_config) + cm_ctx = CostModel( + cluster=cluster, + batch_size=batch_size, + standalone_cost_data=standalone_cost_data, + pipeline_config=pipeline_config, + ) cm_ctx.init(distributed_program) cost = cm_ctx.get_cost() return cost diff --git a/python/paddle/distributed/auto_parallel/dist_attribute.py b/python/paddle/distributed/auto_parallel/dist_attribute.py index 04af4ad77e5a1..19aaa07f24df6 100644 --- a/python/paddle/distributed/auto_parallel/dist_attribute.py +++ b/python/paddle/distributed/auto_parallel/dist_attribute.py @@ -17,11 +17,17 @@ from .process_mesh import ProcessMesh _g_tensor_dist_attr_field_keys = [ - "process_mesh", "dims_mapping", "shard_sizes", "device_placement" + "process_mesh", + "dims_mapping", + "shard_sizes", + "device_placement", ] _g_op_dist_attr_field_keys = [ - "process_mesh", "impl_type", "impl_idx", "is_recompute" + "process_mesh", + "impl_type", + "impl_idx", + "is_recompute", ] _g_op_input_suffix = "@input" @@ -50,7 +56,6 @@ def append_op_output_suffix(name): class TensorDistributedAttribute: - def __init__(self): # The process mesh of distributed operator attribute must is the same as # the process meshes of all input and output distributed attributed @@ -67,8 +72,9 @@ def process_mesh(self): @process_mesh.setter def process_mesh(self, process_mesh): if process_mesh is not None: - assert isinstance(process_mesh, (list, ProcessMesh)), \ - "The type of process_mesh must be list or ProcessMesh." + assert isinstance( + process_mesh, (list, ProcessMesh) + ), "The type of process_mesh must be list or ProcessMesh." if isinstance(process_mesh, list): process_mesh = ProcessMesh(process_mesh) self._process_mesh = copy.deepcopy(process_mesh) @@ -80,12 +86,15 @@ def dims_mapping(self): @dims_mapping.setter def dims_mapping(self, dims_mapping): if dims_mapping is not None: - assert isinstance(dims_mapping, list), \ - "The type of dims_mapping must be list." - assert all(isinstance(x, int) for x in dims_mapping), \ - ("All elements of dims_mapping must be integer") - assert all(x >= -1 for x in dims_mapping), \ - ("All elements of dims_mapping must be greater than or equal to -1.") + assert isinstance( + dims_mapping, list + ), "The type of dims_mapping must be list." + assert all( + isinstance(x, int) for x in dims_mapping + ), "All elements of dims_mapping must be integer" + assert all( + x >= -1 for x in dims_mapping + ), "All elements of dims_mapping must be greater than or equal to -1." self._dims_mapping = copy.deepcopy(dims_mapping) @property @@ -109,37 +118,44 @@ def device_placement(self, device_placement): def init(self, dist_attr): if dist_attr is None: return - assert isinstance(dist_attr, (dict, TensorDistributedAttribute)), \ - "The type of dist_attr must be dict or TensorDistributedAttribute." + assert isinstance( + dist_attr, (dict, TensorDistributedAttribute) + ), "The type of dist_attr must be dict or TensorDistributedAttribute." if isinstance(dist_attr, dict): for key, value in dist_attr.items(): if key in get_tensor_dist_attr_field_keys(): field_property = TensorDistributedAttribute.__dict__.get( - key, None) + key, None + ) if field_property: field_property.fset(self, value) else: assert False, "No setter for {} in args {}.".format( - key, dist_attr) + key, dist_attr + ) elif isinstance(dist_attr, TensorDistributedAttribute): for key in get_tensor_dist_attr_field_keys(): field_property = TensorDistributedAttribute.__dict__.get( - key, None) + key, None + ) if field_property: field_property.fset(self, field_property.fget(dist_attr)) else: assert False, "No setter for {} in args {}.".format( - key, dist_attr) + key, dist_attr + ) self._is_annotated = copy.deepcopy(dist_attr._is_annotated) def reset(self, skip_dist_attr_field_names=None): - if skip_dist_attr_field_names is None or \ - (skip_dist_attr_field_names is not None \ - and "process_mesh" not in skip_dist_attr_field_names): + if skip_dist_attr_field_names is None or ( + skip_dist_attr_field_names is not None + and "process_mesh" not in skip_dist_attr_field_names + ): self._process_mesh = None - if skip_dist_attr_field_names is None or \ - (skip_dist_attr_field_names is not None \ - and "dims_mapping" not in skip_dist_attr_field_names): + if skip_dist_attr_field_names is None or ( + skip_dist_attr_field_names is not None + and "dims_mapping" not in skip_dist_attr_field_names + ): for i, _ in enumerate(self._dims_mapping): self._dims_mapping[i] = -1 self._is_annotated = {} @@ -160,8 +176,9 @@ def mark_annotated(self, dist_attr_field_name): def mark_annotated_as(self, dist_attr): if dist_attr is None: return - assert isinstance(dist_attr, (dict, TensorDistributedAttribute)), \ - "The type of dist_attr must be dict or TensorDistributedAttribute." + assert isinstance( + dist_attr, (dict, TensorDistributedAttribute) + ), "The type of dist_attr must be dict or TensorDistributedAttribute." if isinstance(dist_attr, dict): for key in dist_attr.keys(): if key in get_tensor_dist_attr_field_keys(): @@ -189,21 +206,22 @@ def __str__(self): annotated_str = "annotated" else: annotated_str = "non-annotated" - str += "\n\t\tprocess_mesh ({}): {},".format(annotated_str, - self.process_mesh) + str += "\n\t\tprocess_mesh ({}): {},".format( + annotated_str, self.process_mesh + ) if self.is_annotated("dims_mapping"): annotated_str = "annotated" else: annotated_str = "non-annotated" - str += "\n\t\tdims_mapping ({}): {}".format(annotated_str, - self.dims_mapping) + str += "\n\t\tdims_mapping ({}): {}".format( + annotated_str, self.dims_mapping + ) str += "\n\t}" return str class OperatorDistributedAttribute: - def __init__(self): self._process_mesh = None self._op_type = None @@ -221,8 +239,9 @@ def process_mesh(self): @process_mesh.setter def process_mesh(self, process_mesh): if process_mesh is not None: - assert isinstance(process_mesh, (list, ProcessMesh)), \ - "The type of process_mesh must be list or ProcessMesh." + assert isinstance( + process_mesh, (list, ProcessMesh) + ), "The type of process_mesh must be list or ProcessMesh." if isinstance(process_mesh, list): process_mesh = ProcessMesh(process_mesh) self._process_mesh = copy.deepcopy(process_mesh) @@ -337,8 +356,9 @@ def set_output_dims_mapping(self, name, dims_mapping): def init(self, dist_attr): if dist_attr is None: return - assert isinstance(dist_attr, (dict, OperatorDistributedAttribute)), \ - "The type of dist_attr must be dict or OperatorDistributedAttribute." + assert isinstance( + dist_attr, (dict, OperatorDistributedAttribute) + ), "The type of dist_attr must be dict or OperatorDistributedAttribute." if isinstance(dist_attr, dict): for key, value in dist_attr.items(): if isinstance(key, Variable): @@ -350,31 +370,41 @@ def init(self, dist_attr): self.set_output_dist_attr(key.name, tensor_dist_attr) else: if key in get_op_dist_attr_field_keys(): - field_property = OperatorDistributedAttribute.__dict__.get( - key, None) + field_property = ( + OperatorDistributedAttribute.__dict__.get(key, None) + ) if field_property: field_property.fset(self, value) else: assert False, "No setter for {} in args {}.".format( - key, dist_attr) + key, dist_attr + ) elif isinstance(dist_attr, OperatorDistributedAttribute): - for tensor_name, tensor_dist_attr in dist_attr.inputs_dist_attrs.items( - ): + for ( + tensor_name, + tensor_dist_attr, + ) in dist_attr.inputs_dist_attrs.items(): self.set_input_dist_attr( - tensor_name, dist_attr.get_input_dist_attr(tensor_name)) - for tensor_name, tensor_dist_attr in dist_attr.outputs_dist_attrs.items( - ): + tensor_name, dist_attr.get_input_dist_attr(tensor_name) + ) + for ( + tensor_name, + tensor_dist_attr, + ) in dist_attr.outputs_dist_attrs.items(): self.set_output_dist_attr( - tensor_name, dist_attr.get_output_dist_attr(tensor_name)) + tensor_name, dist_attr.get_output_dist_attr(tensor_name) + ) self._is_annotated = copy.deepcopy(dist_attr._is_annotated) for key in get_op_dist_attr_field_keys(): field_property = OperatorDistributedAttribute.__dict__.get( - key, None) + key, None + ) if field_property: field_property.fset(self, field_property.fget(dist_attr)) else: assert False, "No setter for {} in args {}.".format( - key, dist_attr) + key, dist_attr + ) # Make sure proscess_meshes in dist op be same if self.op_type == "while": return None @@ -390,8 +420,9 @@ def init(self, dist_attr): if shared_process_mesh is None: shared_process_mesh = process_mesh else: - assert process_mesh == shared_process_mesh, \ - "ProcessMeshes in DistributedOperator must be the same." + assert ( + process_mesh == shared_process_mesh + ), "ProcessMeshes in DistributedOperator must be the same." self.process_mesh = shared_process_mesh def reset(self, skip_dist_attr_field_names=None): @@ -399,9 +430,10 @@ def reset(self, skip_dist_attr_field_names=None): tensor_dist_attr.reset(skip_dist_attr_field_names) for tensor_dist_attr in self.outputs_dist_attrs.values(): tensor_dist_attr.reset(skip_dist_attr_field_names) - if skip_dist_attr_field_names is None or \ - (skip_dist_attr_field_names is not None \ - and "process_mesh" not in skip_dist_attr_field_names): + if skip_dist_attr_field_names is None or ( + skip_dist_attr_field_names is not None + and "process_mesh" not in skip_dist_attr_field_names + ): self._process_mesh = None self.impl_type = "default" self.impl_idx = 0 @@ -428,8 +460,9 @@ def mark_annotated(self, attr_name): def mark_annotated_as(self, dist_attr): if dist_attr is None: return - assert isinstance(dist_attr, (dict, OperatorDistributedAttribute)), \ - "The type of dist_attr must be dict or OperatorDistributedAttribute." + assert isinstance( + dist_attr, (dict, OperatorDistributedAttribute) + ), "The type of dist_attr must be dict or OperatorDistributedAttribute." if isinstance(dist_attr, dict): for key, value in dist_attr.items(): if isinstance(key, Variable): @@ -458,15 +491,19 @@ def mark_annotated_as(self, dist_attr): self._is_annotated = copy.deepcopy(dist_attr._is_annotated) if self.is_annotated("process_mesh"): process_mesh_annotated = True - for tensor_name, tensor_dist_attr in dist_attr.inputs_dist_attrs.items( - ): + for ( + tensor_name, + tensor_dist_attr, + ) in dist_attr.inputs_dist_attrs.items(): input_dist_attr = self.get_input_dist_attr(tensor_name) if input_dist_attr is not None: input_dist_attr.mark_annotated_as(tensor_dist_attr) if input_dist_attr.is_annotated("process_mesh"): process_mesh_annotated = True - for tensor_name, tensor_dist_attr in dist_attr.outputs_dist_attrs.items( - ): + for ( + tensor_name, + tensor_dist_attr, + ) in dist_attr.outputs_dist_attrs.items(): output_dist_attr = self.get_output_dist_attr(tensor_name) if output_dist_attr is not None: output_dist_attr.mark_annotated_as(tensor_dist_attr) @@ -523,8 +560,9 @@ def __str__(self): annotated_str = "annotated" else: annotated_str = "non-annotated" - str += "\n\t\tprocess_mesh ({}): {},".format(annotated_str, - self.process_mesh) + str += "\n\t\tprocess_mesh ({}): {},".format( + annotated_str, self.process_mesh + ) for arg_name, tensor_dist_attr in self.inputs_dist_attrs.items(): str += "\n\t\t{}'s (input): {},".format(arg_name, tensor_dist_attr) diff --git a/python/paddle/distributed/auto_parallel/dist_context.py b/python/paddle/distributed/auto_parallel/dist_context.py index 008c4b987074e..12c224ac27fa5 100644 --- a/python/paddle/distributed/auto_parallel/dist_context.py +++ b/python/paddle/distributed/auto_parallel/dist_context.py @@ -50,15 +50,17 @@ class DistributedContext: One auto-parallel run should use its own DistributedContext to avoid interfering other run. """ - def __init__(self, - serial_main_prog=None, - serial_startup_prog=None, - serial_optimizer=None, - serial_loss=None, - feed_vars={}, - fetch_vars={}, - cluster=None, - strategy=None): + def __init__( + self, + serial_main_prog=None, + serial_startup_prog=None, + serial_optimizer=None, + serial_loss=None, + feed_vars={}, + fetch_vars={}, + cluster=None, + strategy=None, + ): # Data members related to original programs (unchanged) self._original_serial_main_program = serial_main_prog self._original_serial_startup_program = serial_startup_prog @@ -107,7 +109,7 @@ def __init__(self, # self._tensor_id_to_tensor_node_ids = {} self._is_initialized = False - #TODO: need a better way to remove the following flag + # TODO: need a better way to remove the following flag self._need_copy_dist_attr_to_graph = False self._backup_pass_context_stack = [] self._backup_block_state_stack = [] @@ -189,7 +191,8 @@ def block_state(self): @property def has_annotation(self): return len(self._dist_tensors_for_program) or len( - self._dist_ops_for_program) + self._dist_ops_for_program + ) @property def gradient_scale(self): @@ -209,18 +212,23 @@ def data_parallel(self, dp): def _backup_serial_info(self, mode): self._backup_serial_main_program_stack.append( - self._serial_main_program.clone()) + self._serial_main_program.clone() + ) self._backup_serial_startup_program_stack.append( - self._serial_startup_program.clone()) - self._backup_pass_context_stack.append(copy.deepcopy( - self._pass_context)) + self._serial_startup_program.clone() + ) + self._backup_pass_context_stack.append( + copy.deepcopy(self._pass_context) + ) self._backup_block_state_stack.append(copy.deepcopy(self._block_state)) def _backup_dist_info(self, mode): self._backup_dist_tensors_for_program_stack.append( - copy.deepcopy(self._dist_tensors_for_program)) + copy.deepcopy(self._dist_tensors_for_program) + ) self._backup_dist_ops_for_program_stack.append( - copy.deepcopy(self._dist_ops_for_program)) + copy.deepcopy(self._dist_ops_for_program) + ) def _backup(self, serial=True, serial_mode=None, dist=True, dist_mode=None): # Use this function carefully @@ -237,7 +245,8 @@ def _restore_serial_loss(self): block_idx = loss.block.idx var_name = loss.name var = self._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) self._serial_loss = var elif len(self._original_serial_loss) == 0: self._serial_loss = [] @@ -247,7 +256,8 @@ def _restore_serial_loss(self): block_idx = self._original_serial_loss.block.idx var_name = self._original_serial_loss.name var = self._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) self._serial_loss = var def _restore_serial_feed_vars(self): @@ -257,7 +267,8 @@ def _restore_serial_feed_vars(self): block_idx = var.block.idx var_name = var.name var = self._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) new_var_list.append(var) self._serial_feed_vars[key] = new_var_list @@ -272,7 +283,8 @@ def _restore_serial_fetch_vars(self): block_idx = var.block.idx var_name = var.name var = self._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) new_inner_var_list.append(var) new_var_list.append(new_inner_var_list) else: @@ -280,22 +292,27 @@ def _restore_serial_fetch_vars(self): block_idx = var.block.idx var_name = var.name var = self._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) new_var_list.append(var) self._serial_fetch_vars[key] = new_var_list def _restore_serial_info(self, mode="to_backup"): if mode == "to_backup": - self._serial_main_program = self._backup_serial_main_program_stack.pop( + self._serial_main_program = ( + self._backup_serial_main_program_stack.pop() ) - self._serial_startup_program = self._backup_serial_startup_program_stack.pop( + self._serial_startup_program = ( + self._backup_serial_startup_program_stack.pop() ) elif mode == "to_original": assert self._original_serial_main_program is not None assert self._original_serial_startup_program is not None - self._serial_main_program = self._original_serial_main_program.clone( + self._serial_main_program = ( + self._original_serial_main_program.clone() ) - self._serial_startup_program = self._original_serial_startup_program.clone( + self._serial_startup_program = ( + self._original_serial_startup_program.clone() ) self._restore_serial_loss() @@ -307,21 +324,27 @@ def _restore_serial_info(self, mode="to_backup"): def _restore_dist_info(self, mode="to_backup"): if mode == "to_backup": - self._dist_tensors_for_program = self._backup_dist_tensors_for_program_stack.pop( + self._dist_tensors_for_program = ( + self._backup_dist_tensors_for_program_stack.pop() ) - self._dist_ops_for_program = self._backup_dist_ops_for_program_stack.pop( + self._dist_ops_for_program = ( + self._backup_dist_ops_for_program_stack.pop() ) elif mode == "to_original": assert self._original_dist_tensors_for_program assert self._original_dist_ops_for_program self._dist_tensors_for_program = copy.deepcopy( - self._original_dist_tensors_for_program) + self._original_dist_tensors_for_program + ) self._dist_ops_for_program = copy.deepcopy( - self._original_dist_ops_for_program) + self._original_dist_ops_for_program + ) elif mode == "to_default": new_tensors_ids = [] - for tensor_id, dist_tensor in self._dist_tensors_for_program.items( - ): + for ( + tensor_id, + dist_tensor, + ) in self._dist_tensors_for_program.items(): if tensor_id in self._tensors_ids: dist_tensor.dist_attr.reset() else: @@ -338,8 +361,10 @@ def _restore_dist_info(self, mode="to_backup"): self._dist_ops_for_program.pop(op_id) else: new_tensors_ids = [] - for tensor_id, dist_tensor in self._dist_tensors_for_program.items( - ): + for ( + tensor_id, + dist_tensor, + ) in self._dist_tensors_for_program.items(): new_tensors_ids.append(tensor_id) for tensor_id in new_tensors_ids: self._dist_tensors_for_program.pop(tensor_id) @@ -354,11 +379,13 @@ def _restore_dist_info(self, mode="to_backup"): self._need_copy_dist_attr_to_graph = True self._process_meshes = [] - def _restore(self, - serial=True, - serial_mode="to_backup", - dist=True, - dist_mode="to_backup"): + def _restore( + self, + serial=True, + serial_mode="to_backup", + dist=True, + dist_mode="to_backup", + ): # Use this function carefully if serial: self._restore_serial_info(serial_mode) @@ -369,11 +396,13 @@ def initialize(self, with_graph=True): if not self._is_initialized: if not self._serial_main_program: if self._original_serial_main_program: - self._serial_main_program = self._original_serial_main_program.clone( + self._serial_main_program = ( + self._original_serial_main_program.clone() ) if not self._serial_startup_program: if self._original_serial_startup_program: - self._serial_startup_program = self._original_serial_startup_program.clone( + self._serial_startup_program = ( + self._original_serial_startup_program.clone() ) if not self._serial_loss: self._restore_serial_loss() @@ -387,9 +416,11 @@ def initialize(self, with_graph=True): self._init_dist_attr_for_program() # Backup the original distributed information for later restore self._original_dist_tensors_for_program = copy.deepcopy( - self._dist_tensors_for_program) + self._dist_tensors_for_program + ) self._original_dist_ops_for_program = copy.deepcopy( - self._dist_ops_for_program) + self._dist_ops_for_program + ) self._tensors_ids = list(self._dist_tensors_for_program.keys()) self._ops_ids = list(self._dist_ops_for_program.keys()) self._is_initialized = True @@ -397,7 +428,8 @@ def initialize(self, with_graph=True): if with_graph: set_flags({"FLAGS_convert_all_blocks": True}) self._serial_graph = framework.IrGraph( - core.Graph(self._serial_main_program.desc)) + core.Graph(self._serial_main_program.desc) + ) self._init_dist_attr_for_graph() self._need_copy_dist_attr_to_graph = False @@ -405,8 +437,9 @@ def initialize(self, with_graph=True): self.copy_dist_attr_from_program_to_graph() def add_process_mesh(self, process_mesh): - assert isinstance(process_mesh, ProcessMesh), \ - 'The type of dim_mapping must be ProcessMesh.' + assert isinstance( + process_mesh, ProcessMesh + ), 'The type of dim_mapping must be ProcessMesh.' if process_mesh not in self.process_meshes: self._process_meshes.append(process_mesh) @@ -428,7 +461,8 @@ def get_dist_tensor_for_program(self, serial_tensor): else: serial_tensor_id = serial_tensor.desc.original_id() dist_tensor = self._dist_tensors_for_program.get( - serial_tensor_id, None) + serial_tensor_id, None + ) if dist_tensor: return dist_tensor else: @@ -468,7 +502,8 @@ def get_tensor_dist_attr_for_program(self, serial_tensor): else: serial_tensor_id = serial_tensor.desc.original_id() dist_tensor = self._dist_tensors_for_program.get( - serial_tensor_id, None) + serial_tensor_id, None + ) if dist_tensor: return dist_tensor.dist_attr else: @@ -487,8 +522,9 @@ def set_tensor_dist_attr_for_program(self, serial_tensor, dist_attr): def get_tensor_dist_attr_for_graph(self, serial_tensor_node): serial_tensor_node_id = _node_id(serial_tensor_node) - dist_tensor = self._dist_tensors_for_graph.get(serial_tensor_node_id, - None) + dist_tensor = self._dist_tensors_for_graph.get( + serial_tensor_node_id, None + ) if dist_tensor: return dist_tensor.dist_attr else: @@ -530,7 +566,8 @@ def get_dist_attr_for_graph(self, serial_node): if serial_node.is_var() and serial_node.var() is not None: serial_tensor_node_id = _node_id(serial_node) dist_tensor = self._dist_tensors_for_graph.get( - serial_tensor_node_id, None) + serial_tensor_node_id, None + ) if dist_tensor: return dist_tensor.dist_attr else: @@ -557,7 +594,8 @@ def _init_dist_attr_for_program(self, no_default=False): for tensor in block.vars.values(): # Copy the distributed tensors in the default context default_dist_tensor = default_ctx.get_dist_tensor_for_program( - tensor) + tensor + ) if default_dist_tensor and default_ctx is not self: self.add_dist_tensor_for_program(default_dist_tensor) current_dist_tensor = self.get_dist_tensor_for_program(tensor) @@ -574,12 +612,13 @@ def _init_dist_attr_for_program(self, no_default=False): dist_op = DistributedOperator(op) self.add_dist_op_for_program(dist_op) self._original_dist_tensors_for_program = copy.deepcopy( - self._dist_tensors_for_program) + self._dist_tensors_for_program + ) self._original_dist_ops_for_program = copy.deepcopy( - self._dist_ops_for_program) + self._dist_ops_for_program + ) def _order_nodes_by_program_order(self): - def _contains(nodes, target_node): for node in nodes: if _node_id(node) == _node_id(target_node): @@ -598,11 +637,14 @@ def _contains(nodes, target_node): if node.is_op() and node.op() is not None: serial_ordered_op_nodes.append(node) serial_ordered_tensor_nodes.sort( - key=lambda node: node.node.original_desc_id()) + key=lambda node: node.node.original_desc_id() + ) serial_ordered_op_nodes.sort( - key=lambda node: node.node.original_desc_id()) + key=lambda node: node.node.original_desc_id() + ) num_nodes_before = len(serial_ordered_tensor_nodes) + len( - serial_ordered_op_nodes) + serial_ordered_op_nodes + ) new_serial_ordered_tensor_nodes = [] new_serial_ordered_op_nodes = [] @@ -610,9 +652,11 @@ def _contains(nodes, target_node): for op_node in serial_ordered_op_nodes: tensor_nodes = [] for tensor_node in op_node.inputs: - if tensor_node.is_var() \ - and tensor_node.var() is not None \ - and not _contains(new_serial_ordered_nodes, tensor_node): + if ( + tensor_node.is_var() + and tensor_node.var() is not None + and not _contains(new_serial_ordered_nodes, tensor_node) + ): tensor_nodes.append(tensor_node) new_serial_ordered_tensor_nodes.append(tensor_node) tensor_nodes.sort(key=lambda node: node.node.original_desc_id()) @@ -621,23 +665,27 @@ def _contains(nodes, target_node): new_serial_ordered_op_nodes.append(op_node) tensor_nodes = [] for tensor_node in op_node.outputs: - if tensor_node.is_var() \ - and tensor_node.var() is not None \ - and not _contains(new_serial_ordered_nodes, tensor_node): + if ( + tensor_node.is_var() + and tensor_node.var() is not None + and not _contains(new_serial_ordered_nodes, tensor_node) + ): tensor_nodes.append(tensor_node) new_serial_ordered_tensor_nodes.append(tensor_node) tensor_nodes.sort(key=lambda node: node.node.original_desc_id()) new_serial_ordered_nodes.extend(tensor_nodes) new_serial_ordered_tensor_nodes.sort( - key=lambda node: node.node.original_desc_id()) + key=lambda node: node.node.original_desc_id() + ) new_serial_ordered_op_nodes.sort( - key=lambda node: node.node.original_desc_id()) + key=lambda node: node.node.original_desc_id() + ) self._serial_ordered_tensor_nodes = new_serial_ordered_tensor_nodes self._serial_ordered_op_nodes = new_serial_ordered_op_nodes self._serial_ordered_nodes = new_serial_ordered_nodes assert len(self._serial_ordered_nodes) == len( - self._serial_ordered_tensor_nodes) + len( - self._serial_ordered_op_nodes) + self._serial_ordered_tensor_nodes + ) + len(self._serial_ordered_op_nodes) self._serial_orphan_tensor_nodes = [] for tensor_node in serial_ordered_tensor_nodes: if not _contains(self._serial_ordered_tensor_nodes, tensor_node): @@ -654,34 +702,49 @@ def _init_dist_attr_for_graph(self): if node.is_var() and node.var() is not None: dist_tensor = None tensor_id = node.node.original_desc_id() - for cur_tensor_id, cur_dist_tensor in self._dist_tensors_for_program.items( - ): - if tensor_id == cur_tensor_id \ - or tensor_id == cur_dist_tensor.serial_tensor.desc.original_id(): + for ( + cur_tensor_id, + cur_dist_tensor, + ) in self._dist_tensors_for_program.items(): + if ( + tensor_id == cur_tensor_id + or tensor_id + == cur_dist_tensor.serial_tensor.desc.original_id() + ): dist_tensor = cur_dist_tensor - self._node_id_to_tensor_id[_node_id( - node)] = cur_tensor_id - assert dist_tensor is not None, \ - "Tensor must have a distributed tensor after the initialization for program." + self._node_id_to_tensor_id[ + _node_id(node) + ] = cur_tensor_id + assert ( + dist_tensor is not None + ), "Tensor must have a distributed tensor after the initialization for program." serial_tensor_node_id = _node_id(node) - new_dist_tensor = DistributedTensor(dist_tensor.serial_tensor, - dist_tensor.dist_attr) + new_dist_tensor = DistributedTensor( + dist_tensor.serial_tensor, dist_tensor.dist_attr + ) self._dist_tensors_for_graph[ - serial_tensor_node_id] = new_dist_tensor + serial_tensor_node_id + ] = new_dist_tensor if node.is_op() and node.op() is not None: dist_op = None op_id = node.node.original_desc_id() - for cur_op_id, cur_dist_op in self._dist_ops_for_program.items( - ): - if op_id == cur_op_id \ - or op_id == cur_dist_op.serial_op.desc.original_id(): + for ( + cur_op_id, + cur_dist_op, + ) in self._dist_ops_for_program.items(): + if ( + op_id == cur_op_id + or op_id == cur_dist_op.serial_op.desc.original_id() + ): dist_op = cur_dist_op self._node_id_to_op_id[_node_id(node)] = cur_op_id - assert dist_op is not None, \ - "Operator must have a distributed operator after the initialization for program." + assert ( + dist_op is not None + ), "Operator must have a distributed operator after the initialization for program." serial_op_node_id = _node_id(node) - new_dist_op = DistributedOperator(dist_op.serial_op, - dist_op.dist_attr) + new_dist_op = DistributedOperator( + dist_op.serial_op, dist_op.dist_attr + ) self._dist_ops_for_graph[serial_op_node_id] = new_dist_op def clear_dist_info_for_program(self): @@ -697,36 +760,51 @@ def copy_dist_attr_from_program_to_graph(self): if node.is_var() and node.var() is not None: dist_tensor = None tensor_id = node.node.original_desc_id() - for cur_tensor_id, cur_dist_tensor in self._dist_tensors_for_program.items( - ): - if tensor_id == cur_tensor_id \ - or tensor_id == cur_dist_tensor.serial_tensor.desc.original_id(): + for ( + cur_tensor_id, + cur_dist_tensor, + ) in self._dist_tensors_for_program.items(): + if ( + tensor_id == cur_tensor_id + or tensor_id + == cur_dist_tensor.serial_tensor.desc.original_id() + ): dist_tensor = cur_dist_tensor - assert dist_tensor is not None, \ - "Tensor must have a distributed tensor after the initialization for program." + assert ( + dist_tensor is not None + ), "Tensor must have a distributed tensor after the initialization for program." serial_tensor_node_id = _node_id(node) - new_dist_tensor = DistributedTensor(dist_tensor.serial_tensor, - dist_tensor.dist_attr) + new_dist_tensor = DistributedTensor( + dist_tensor.serial_tensor, dist_tensor.dist_attr + ) self._dist_tensors_for_graph[ - serial_tensor_node_id] = new_dist_tensor + serial_tensor_node_id + ] = new_dist_tensor if node.is_op() and node.op() is not None: dist_op = None op_id = node.node.original_desc_id() - for cur_op_id, cur_dist_op in self._dist_ops_for_program.items( - ): - if op_id == cur_op_id \ - or op_id == cur_dist_op.serial_op.desc.original_id(): + for ( + cur_op_id, + cur_dist_op, + ) in self._dist_ops_for_program.items(): + if ( + op_id == cur_op_id + or op_id == cur_dist_op.serial_op.desc.original_id() + ): dist_op = cur_dist_op - assert dist_op is not None, \ - "Operator must have a distributed operator after the initialization for program." + assert ( + dist_op is not None + ), "Operator must have a distributed operator after the initialization for program." serial_op_node_id = _node_id(node) - new_dist_op = DistributedOperator(dist_op.serial_op, - dist_op.dist_attr) + new_dist_op = DistributedOperator( + dist_op.serial_op, dist_op.dist_attr + ) self._dist_ops_for_graph[serial_op_node_id] = new_dist_op def copy_dist_attr_from_graph_to_program(self): - assert self._is_initialized, \ - "Both program and graph must be initialized." + assert ( + self._is_initialized + ), "Both program and graph must be initialized." updated_tensors = {} # all_nodes = self._serial_graph.all_nodes() all_nodes = self._serial_ordered_nodes @@ -736,11 +814,15 @@ def copy_dist_attr_from_graph_to_program(self): updated = updated_tensors.get(tensor_id, False) # If a var has multiples var nodes in graph, only use the first one for now if not updated: - tensor_dist_attr_for_graph = self.get_tensor_dist_attr_for_graph( - node) + tensor_dist_attr_for_graph = ( + self.get_tensor_dist_attr_for_graph(node) + ) dist_tensor_for_program = self._dist_tensors_for_program[ - tensor_id] - dist_tensor_for_program.dist_attr = tensor_dist_attr_for_graph + tensor_id + ] + dist_tensor_for_program.dist_attr = ( + tensor_dist_attr_for_graph + ) updated_tensors[tensor_id] = True if node.is_op() and node.op() is not None: op_id = self._node_id_to_op_id[_node_id(node)] @@ -752,22 +834,26 @@ def copy_dist_attr_from_graph_to_program(self): for orphan_node in self._serial_orphan_tensor_nodes: serial_tensor_id = orphan_node.var().id() dist_tensor = self._dist_tensors_for_program.get( - serial_tensor_id, None) + serial_tensor_id, None + ) if dist_tensor: dist_tensor.dist_attr.process_mesh = self._process_meshes[0] else: serial_tensor_id = orphan_node.var().original_id() dist_tensor = self._dist_tensors_for_program.get( - serial_tensor_id, None) + serial_tensor_id, None + ) dist_tensor.dist_attr.process_mesh = self._process_meshes[0] def amend_dist_attr_for_program(self): for dist_tensor in self._dist_tensors_for_program.values(): serial_tensor = dist_tensor.serial_tensor dist_attr = dist_tensor.dist_attr - if serial_tensor.type == core.VarDesc.VarType.READER \ - or serial_tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + serial_tensor.type == core.VarDesc.VarType.READER + or serial_tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES + ): tensor_shape = [] else: tensor_shape = serial_tensor.shape @@ -777,8 +863,11 @@ def amend_dist_attr_for_program(self): # If the dimension of tensor is less than the sharding dimension of process mesh, # we just amend the dimension mapping to -1. (Is this really OK?) for i in range(len(tensor_shape)): - if dims_mapping[i] != -1 and tensor_shape[i] > 0 \ - and process_mesh_shape[dims_mapping[i]] > tensor_shape[i]: + if ( + dims_mapping[i] != -1 + and tensor_shape[i] > 0 + and process_mesh_shape[dims_mapping[i]] > tensor_shape[i] + ): dims_mapping[i] = -1 if dims_mapping[i] != -1 and len(process_mesh_processes) == 1: dims_mapping[i] = -1 @@ -792,9 +881,13 @@ def amend_dist_attr_for_program(self): if dist_op.get_serial_input(arg_name) is None: tensor_shape = [] else: - if dist_op.get_serial_input(arg_name).type == core.VarDesc.VarType.READER \ - or dist_op.get_serial_input(arg_name).type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or dist_op.serial_op.type == "create_py_reader": + if ( + dist_op.get_serial_input(arg_name).type + == core.VarDesc.VarType.READER + or dist_op.get_serial_input(arg_name).type + == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or dist_op.serial_op.type == "create_py_reader" + ): tensor_shape = [] else: tensor_shape = dist_op.get_serial_input(arg_name).shape @@ -802,16 +895,27 @@ def amend_dist_attr_for_program(self): # If the dimension of tensor is less than the sharding dimension of process mesh, # we just amend the dimension mapping to -1. (Is this really OK?) for i in range(len(tensor_shape)): - if dims_mapping[i] != -1 and tensor_shape[i] > 0 \ - and process_mesh_shape[dims_mapping[i]] > tensor_shape[i]: + if ( + dims_mapping[i] != -1 + and tensor_shape[i] > 0 + and process_mesh_shape[dims_mapping[i]] + > tensor_shape[i] + ): dims_mapping[i] = -1 - if dims_mapping[i] != -1 and len( - process_mesh_processes) == 1: + if ( + dims_mapping[i] != -1 + and len(process_mesh_processes) == 1 + ): dims_mapping[i] = -1 for arg_name in serial_op.output_arg_names: - if dist_op.get_serial_output(arg_name).type == core.VarDesc.VarType.READER \ - or dist_op.get_serial_output(arg_name).type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or dist_op.get_serial_output(arg_name).type == core.VarDesc.VarType.STEP_SCOPES: + if ( + dist_op.get_serial_output(arg_name).type + == core.VarDesc.VarType.READER + or dist_op.get_serial_output(arg_name).type + == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or dist_op.get_serial_output(arg_name).type + == core.VarDesc.VarType.STEP_SCOPES + ): tensor_shape = [] else: tensor_shape = dist_op.get_serial_output(arg_name).shape @@ -819,11 +923,17 @@ def amend_dist_attr_for_program(self): # If the dimension of tensor is less than the sharding dimension of process mesh, # we just amend the dimension mapping to -1. (Is this really OK?) for i in range(len(tensor_shape)): - if dims_mapping[i] != -1 and tensor_shape[i] > 0 \ - and process_mesh_shape[dims_mapping[i]] > tensor_shape[i]: + if ( + dims_mapping[i] != -1 + and tensor_shape[i] > 0 + and process_mesh_shape[dims_mapping[i]] + > tensor_shape[i] + ): dims_mapping[i] = -1 - if dims_mapping[i] != -1 and len( - process_mesh_processes) == 1: + if ( + dims_mapping[i] != -1 + and len(process_mesh_processes) == 1 + ): dims_mapping[i] = -1 if len(process_mesh_processes) == 1: dist_op.dist_attr.impl_type = "default" @@ -831,30 +941,44 @@ def amend_dist_attr_for_program(self): def validate_dist_attr_for_program(self): if not self._is_initialized: - assert False, \ - "Program must be initialized before validating its distributed attributes" + assert ( + False + ), "Program must be initialized before validating its distributed attributes" for block in self.serial_main_program.blocks: for tensor in block.vars.values(): dist_tensor = self.get_dist_tensor_for_program(tensor) - assert dist_tensor is not None, \ - "Tensor {} does not have a distributed attribute.".format( - dist_tensor.serial_tensor.name) - if (dist_tensor - is not None) and (not dist_tensor.validate_dist_attr()): - assert False, "Tensor {} (id: {}, original_id: {}) has a wrong distributed attributes {}.".format( + assert ( + dist_tensor is not None + ), "Tensor {} does not have a distributed attribute.".format( + dist_tensor.serial_tensor.name + ) + if (dist_tensor is not None) and ( + not dist_tensor.validate_dist_attr() + ): + assert ( + False + ), "Tensor {} (id: {}, original_id: {}) has a wrong distributed attributes {}.".format( dist_tensor.serial_tensor.name, dist_tensor.serial_tensor.desc.id(), dist_tensor.serial_tensor.desc.original_id(), - dist_tensor.dist_attr) + dist_tensor.dist_attr, + ) for op in block.ops: dist_op = self.get_dist_op_for_program(op) - assert dist_op is not None, \ - "Operator {} does not have a distributed attribute.".format( - dist_op.serial_op.type) + assert ( + dist_op is not None + ), "Operator {} does not have a distributed attribute.".format( + dist_op.serial_op.type + ) if (dist_op is not None) and (not dist_op.validate_dist_attr()): - assert False, "Operator {} (id: {}, original_id: {}) has a wrong distributed attributes {} .".format( - dist_op.serial_op.type, dist_op.serial_op.desc.id(), - dist_op.serial_op.desc.original_id(), dist_op.dist_attr) + assert ( + False + ), "Operator {} (id: {}, original_id: {}) has a wrong distributed attributes {} .".format( + dist_op.serial_op.type, + dist_op.serial_op.desc.id(), + dist_op.serial_op.desc.original_id(), + dist_op.dist_attr, + ) return True def __deepcopy__(self, memo): @@ -863,15 +987,27 @@ def __deepcopy__(self, memo): memo[id(self)] = result for k, v in self.__dict__.items(): if k in [ - "_original_serial_main_program", "_original_serial_startup_program", \ - "_serial_main_program", "_serial_startup_program", "_serial_graph", \ - "_dist_main_programs", "_dist_startup_programs", \ - "_serial_ordered_nodes", "_serial_ordered_tensor_nodes", \ - "_serial_ordered_op_nodes", "_original_serial_loss", \ - "_original_serial_feed_vars", "_original_serial_fetch_vars", \ - "_serial_loss", "_serial_feed_vars", "_serial_fetch_vars", "_serial_optimizer", \ - "_backup_serial_main_program_stack", "_backup_serial_startup_program_stack", \ - "_pass_context"]: + "_original_serial_main_program", + "_original_serial_startup_program", + "_serial_main_program", + "_serial_startup_program", + "_serial_graph", + "_dist_main_programs", + "_dist_startup_programs", + "_serial_ordered_nodes", + "_serial_ordered_tensor_nodes", + "_serial_ordered_op_nodes", + "_original_serial_loss", + "_original_serial_feed_vars", + "_original_serial_fetch_vars", + "_serial_loss", + "_serial_feed_vars", + "_serial_fetch_vars", + "_serial_optimizer", + "_backup_serial_main_program_stack", + "_backup_serial_startup_program_stack", + "_pass_context", + ]: setattr(result, k, v) else: setattr(result, k, copy.deepcopy(v, memo)) @@ -913,8 +1049,12 @@ def __deepcopy__(self, memo): memo[id(self)] = result for k, v in self.__dict__.items(): if k in [ - "_dst_main_program", "_dst_startup_program", "_cur_src_op", - "_work_block", "_main_block", "_startup_block" + "_dst_main_program", + "_dst_startup_program", + "_cur_src_op", + "_work_block", + "_main_block", + "_startup_block", ]: setattr(result, k, v) else: @@ -994,7 +1134,6 @@ def prepare_context(self, src_op): class BlockState(object): - def __init__(self): self.nblock = 0 self.forward_indices = [] @@ -1011,8 +1150,11 @@ def parse_forward_blocks(self, program): for idx, block in enumerate(program.blocks): assert idx == block.idx, "index doesn't match" - assert block.forward_block_idx == -1, "forward_block_idx of forward block [{}] is not [{}]".format( - idx, block.forward_block_idx) + assert ( + block.forward_block_idx == -1 + ), "forward_block_idx of forward block [{}] is not [{}]".format( + idx, block.forward_block_idx + ) self.forward_indices.append(idx) self.nblock += 1 @@ -1021,7 +1163,8 @@ def parse_forward_blocks(self, program): def parse_backward_blocks(self, program): assert 0 in self.forward_indices, "forward block idx are{}".format( - self.forward_indices) + self.forward_indices + ) self.backward_to_forward_index_map[0] = 0 for idx, block in enumerate(program.blocks): diff --git a/python/paddle/distributed/auto_parallel/dist_loader.py b/python/paddle/distributed/auto_parallel/dist_loader.py index 9eb62b1b74e8a..f16c55a011c7c 100644 --- a/python/paddle/distributed/auto_parallel/dist_loader.py +++ b/python/paddle/distributed/auto_parallel/dist_loader.py @@ -17,12 +17,18 @@ import paddle from paddle.io import BatchSampler, IterableDataset -from paddle.fluid.dataloader.batch_sampler import _InfiniteIterableSampler, DistributedBatchSampler -from paddle.fluid.dataloader.dataloader_iter import _DatasetKind, default_collate_fn, default_convert_fn +from paddle.fluid.dataloader.batch_sampler import ( + _InfiniteIterableSampler, + DistributedBatchSampler, +) +from paddle.fluid.dataloader.dataloader_iter import ( + _DatasetKind, + default_collate_fn, + default_convert_fn, +) class DistributedDataLoaderBase(metaclass=abc.ABCMeta): - @abc.abstractmethod def __iter__(self): raise NotImplementedError @@ -33,24 +39,25 @@ def __next__(self): class DistributedDataLoaderFromGenerator(DistributedDataLoaderBase): - - def __init__(self, - dataset, - feed_list=None, - capacity=None, - use_double_buffer=True, - iterable=True, - return_list=False, - use_multiprocess=False, - drop_last=True, - places=None, - batch_size=1, - epochs=1, - steps_per_epoch=None, - collate_fn=None, - split_data=True, - data_parallel_world_size=[], - data_parallel_rank=[]): + def __init__( + self, + dataset, + feed_list=None, + capacity=None, + use_double_buffer=True, + iterable=True, + return_list=False, + use_multiprocess=False, + drop_last=True, + places=None, + batch_size=1, + epochs=1, + steps_per_epoch=None, + collate_fn=None, + split_data=True, + data_parallel_world_size=[], + data_parallel_rank=[], + ): self.dataset = dataset self.feed_list = feed_list self.capacity = capacity @@ -80,12 +87,15 @@ def __init__(self, else: if isinstance(dataset, IterableDataset): self.batch_sampler = _InfiniteIterableSampler( - dataset, batch_size) + dataset, batch_size + ) else: - self.batch_sampler = BatchSampler(dataset, - batch_size=batch_size, - shuffle=False, - drop_last=drop_last) + self.batch_sampler = BatchSampler( + dataset, + batch_size=batch_size, + shuffle=False, + drop_last=drop_last, + ) self.auto_collate_batch = self.batch_sampler is not None self.sampler_iter = iter(self.index_sampler) @@ -96,8 +106,12 @@ def __init__(self, self.collate_fn = collate_fn or default_convert_fn self.dataset_fetcher = _DatasetKind.create_fetcher( - self.dataset_kind, self.dataset, self.auto_collate_batch, - self.collate_fn, self.drop_last) + self.dataset_kind, + self.dataset, + self.auto_collate_batch, + self.collate_fn, + self.drop_last, + ) self._steps = self._infer_steps() self._inner_dataloader = self._create_inner_dataloader() @@ -146,18 +160,21 @@ def index_sampler(self): return _InfiniteIterableSampler(self.dataset, 1) def _create_inner_dataloader(self): - def data_generator(): while True: try: indices = next(self.sampler_iter) batch = self.dataset_fetcher.fetch(indices) - if batch is None: break + if batch is None: + break except StopIteration: self.dataset_fetcher = _DatasetKind.create_fetcher( - self.dataset_kind, self.dataset, - self.auto_collate_batch, self.collate_fn, - self.drop_last) + self.dataset_kind, + self.dataset, + self.auto_collate_batch, + self.collate_fn, + self.drop_last, + ) break partial_data = [] @@ -168,11 +185,16 @@ def data_generator(): continue batch_size = array.shape[0] - assert batch_size % self.dp_world_sizes[i] == 0, \ - "batch_size [{}] is not divisible by dp_world_size [{}]".format(str(batch_size), str(self.dp_world_sizes[i])) + assert ( + batch_size % self.dp_world_sizes[i] == 0 + ), "batch_size [{}] is not divisible by dp_world_size [{}]".format( + str(batch_size), str(self.dp_world_sizes[i]) + ) partial_data.append( - np.split(array, - self.dp_world_sizes[i])[self.dp_ranks[i]]) + np.split(array, self.dp_world_sizes[i])[ + self.dp_ranks[i] + ] + ) yield partial_data @@ -184,33 +206,35 @@ def data_generator(): iterable=False, return_list=self.return_list, use_multiprocess=self.use_multiprocess, - drop_last=self.drop_last) + drop_last=self.drop_last, + ) dataloader.set_batch_generator(data_generator, self.places) return dataloader class DistributedDataLoader(DistributedDataLoaderBase): - - def __init__(self, - dataset, - feed_list=None, - places=None, - return_list=True, - batch_size=1, - shuffle=False, - drop_last=False, - collate_fn=None, - num_workers=0, - use_buffer_reader=True, - use_shared_memory=True, - timeout=0, - worker_init_fn=None, - epochs=1, - steps_per_epoch=None, - split_data=True, - data_parallel_world_size=[], - data_parallel_rank=[]): + def __init__( + self, + dataset, + feed_list=None, + places=None, + return_list=True, + batch_size=1, + shuffle=False, + drop_last=False, + collate_fn=None, + num_workers=0, + use_buffer_reader=True, + use_shared_memory=True, + timeout=0, + worker_init_fn=None, + epochs=1, + steps_per_epoch=None, + split_data=True, + data_parallel_world_size=[], + data_parallel_rank=[], + ): self.dataset = dataset self.feed_list = feed_list self.return_list = return_list @@ -231,8 +255,13 @@ def __init__(self, self.split_data = split_data # TODO: rank info self.batch_sampler = DistributedBatchSampler( - self.dataset, self.batch_size, self.dp_world_sizes[0], - self.dp_ranks[0], self.shuffle, self.drop_last) + self.dataset, + self.batch_size, + self.dp_world_sizes[0], + self.dp_ranks[0], + self.shuffle, + self.drop_last, + ) self._inner_dataloader = self._create_inner_dataloader() def __iter__(self): @@ -253,7 +282,8 @@ def _create_inner_dataloader(self): use_buffer_reader=self.use_buffer_reader, use_shared_memory=self.use_shared_memory, timeout=self.timeout, - worker_init_fn=self.worker_init_fn) + worker_init_fn=self.worker_init_fn, + ) self.data = (x for x in dataloader) return dataloader diff --git a/python/paddle/distributed/auto_parallel/dist_op.py b/python/paddle/distributed/auto_parallel/dist_op.py index 004436458b13a..41b4696174a5a 100644 --- a/python/paddle/distributed/auto_parallel/dist_op.py +++ b/python/paddle/distributed/auto_parallel/dist_op.py @@ -23,7 +23,6 @@ class DistributedOperator: - def __init__(self, serial_op, dist_attr=None): self._serial_op = serial_op self._serial_inputs = {} @@ -74,28 +73,34 @@ def _init_default_dist_attr(self): if tensor is None: tensor_shape = [] else: - if tensor.type == core.VarDesc.VarType.READER \ - or tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + tensor.type == core.VarDesc.VarType.READER + or tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): tensor_shape = [] else: tensor_shape = tensor.shape if self._dist_attr.get_input_dims_mapping(tensor_name) is None: tensor_dims_mapping = [-1 for _ in range(len(tensor_shape))] - self._dist_attr.set_input_dims_mapping(tensor_name, - tensor_dims_mapping) + self._dist_attr.set_input_dims_mapping( + tensor_name, tensor_dims_mapping + ) for tensor_name in self._serial_op.output_arg_names: tensor = self._serial_op.block._var_recursive(tensor_name) - if tensor.type == core.VarDesc.VarType.READER \ - or tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or tensor.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + tensor.type == core.VarDesc.VarType.READER + or tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or tensor.type == core.VarDesc.VarType.STEP_SCOPES + ): tensor_shape = [] else: tensor_shape = tensor.shape self._serial_outputs[tensor_name] = tensor if self._dist_attr.get_output_dims_mapping(tensor_name) is None: tensor_dims_mapping = [-1 for _ in range(len(tensor_shape))] - self._dist_attr.set_output_dims_mapping(tensor_name, - tensor_dims_mapping) + self._dist_attr.set_output_dims_mapping( + tensor_name, tensor_dims_mapping + ) if self._dist_attr.op_type is None: self._dist_attr.op_type = self.serial_op.type if self._dist_attr.impl_type is None: @@ -113,8 +118,10 @@ def _filter_dist_attr(self, dist_attr): new_dist_attr = {} for key, value in dist_attr.items(): if isinstance(key, Variable): - if key.name in self._serial_op.input_arg_names \ - or key.name in self._serial_op.output_arg_names: + if ( + key.name in self._serial_op.input_arg_names + or key.name in self._serial_op.output_arg_names + ): new_dist_attr[key] = value else: new_dist_attr[key] = value @@ -125,13 +132,15 @@ def _filter_dist_attr(self, dist_attr): for tensor_name in self._serial_op.input_arg_names: tensor_dist_attr = dist_attr.get_input_dist_attr(tensor_name) if tensor_dist_attr: - new_dist_attr.set_input_dist_attr(tensor_name, - tensor_dist_attr) + new_dist_attr.set_input_dist_attr( + tensor_name, tensor_dist_attr + ) for tensor_name in self._serial_op.output_arg_names: tensor_dist_attr = dist_attr.get_output_dist_attr(tensor_name) if tensor_dist_attr: - new_dist_attr.set_output_dist_attr(tensor_name, - tensor_dist_attr) + new_dist_attr.set_output_dist_attr( + tensor_name, tensor_dist_attr + ) else: assert False, "Cannot recognize the {} parameter.".format(dist_attr) return new_dist_attr @@ -142,8 +151,10 @@ def validate_dist_attr(self): for name in self.serial_op.input_arg_names: input_dist_attr = self.dist_attr.get_input_dist_attr(name) dims_mapping = input_dist_attr.dims_mapping - if self.get_serial_input( - name).type == core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + self.get_serial_input(name).type + == core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): shape = [] else: shape = self.get_serial_input(name).shape @@ -151,7 +162,8 @@ def validate_dist_attr(self): return False for i in range(len(dims_mapping)): if dims_mapping[i] < -1 or dims_mapping[i] >= len( - self.dist_attr.process_mesh.topology): + self.dist_attr.process_mesh.topology + ): return False for i in range(len(self.dist_attr.process_mesh.topology)): if dims_mapping.count(i) > 1: @@ -162,8 +174,12 @@ def validate_dist_attr(self): for name in self.serial_op.output_arg_names: output_dist_attr = self.dist_attr.get_output_dist_attr(name) dims_mapping = output_dist_attr.dims_mapping - if self.get_serial_output(name).type == core.VarDesc.VarType.LOD_TENSOR_ARRAY\ - or self.get_serial_output(name).type == core.VarDesc.VarType.STEP_SCOPES: + if ( + self.get_serial_output(name).type + == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or self.get_serial_output(name).type + == core.VarDesc.VarType.STEP_SCOPES + ): shape = [] else: shape = self.get_serial_output(name).shape @@ -171,7 +187,8 @@ def validate_dist_attr(self): return False for i in range(len(dims_mapping)): if dims_mapping[i] < -1 or dims_mapping[i] >= len( - self.dist_attr.process_mesh.topology): + self.dist_attr.process_mesh.topology + ): return False for i in range(len(self.dist_attr.process_mesh.topology)): if dims_mapping.count(i) > 1: @@ -181,8 +198,9 @@ def validate_dist_attr(self): return True def __str__(self): - str = "{{op type: {}, op id: {}".format(self.serial_op.desc.type(), - self.serial_op.desc.id()) + str = "{{op type: {}, op id: {}".format( + self.serial_op.desc.type(), self.serial_op.desc.id() + ) # str += ", {}".format(self.dist_attr) # return str @@ -191,8 +209,9 @@ def __str__(self): annotated_str = "annotated" else: annotated_str = "non-annotated" - str += ", process_mesh ({}): {}".format(annotated_str, - self.dist_attr.process_mesh) + str += ", process_mesh ({}): {}".format( + annotated_str, self.dist_attr.process_mesh + ) for arg_name in self.serial_op.desc.input_arg_names(): dims_mapping = self.dist_attr.get_input_dims_mapping(arg_name) @@ -208,7 +227,8 @@ def __str__(self): else: is_parameter_str = "non-parameter" str += ", {}'s dims_mapping (input, {}, {}): {}".format( - arg_name, annotated_str, is_parameter_str, dims_mapping) + arg_name, annotated_str, is_parameter_str, dims_mapping + ) for arg_name in self.serial_op.desc.output_arg_names(): dims_mapping = self.dist_attr.get_output_dims_mapping(arg_name) @@ -224,12 +244,14 @@ def __str__(self): else: is_parameter_str = "non-parameter" str += ", {}'s dims_mapping (output, {}, {}): {}".format( - arg_name, annotated_str, is_parameter_str, dims_mapping) + arg_name, annotated_str, is_parameter_str, dims_mapping + ) str += ", pipeline stage: {}".format(None) str += ", dist_impl idx: {} , dist_impl type {} }}".format( - self.dist_attr._impl_idx, self.dist_attr._impl_type) + self.dist_attr._impl_idx, self.dist_attr._impl_type + ) return str @@ -238,7 +260,11 @@ def __deepcopy__(self, memo): result = cls.__new__(cls) memo[id(self)] = result for k, v in self.__dict__.items(): - if k == "_serial_op" or k == "_serial_inputs" or k == "_serial_outputs": + if ( + k == "_serial_op" + or k == "_serial_inputs" + or k == "_serial_outputs" + ): setattr(result, k, v) else: setattr(result, k, copy.deepcopy(v, memo)) @@ -246,9 +272,9 @@ def __deepcopy__(self, memo): class DistributedOperatorHelper: - - def __init__(self, serial_op, process_mesh, in_dims_mappings, - out_dims_mappings): + def __init__( + self, serial_op, process_mesh, in_dims_mappings, out_dims_mappings + ): self._serial_op = serial_op self._process_mesh = process_mesh self._in_dims_mappings = in_dims_mappings @@ -258,8 +284,11 @@ def __call__(self, *args, **kwargs): tensor_to_dims_mapping = {} index = 0 if self._in_dims_mappings: - assert len(args) + len(kwargs) == len(self._in_dims_mappings), \ - "The length of dims_mapping {} does not matching the length output {}.".format(len(self._in_dims_mappings), len(args) + len(kwargs)) + assert len(args) + len(kwargs) == len( + self._in_dims_mappings + ), "The length of dims_mapping {} does not matching the length output {}.".format( + len(self._in_dims_mappings), len(args) + len(kwargs) + ) for arg in args: if isinstance(arg, Variable) and self._in_dims_mappings: tensor_to_dims_mapping[arg.name] = self._in_dims_mappings[index] @@ -283,13 +312,17 @@ def __call__(self, *args, **kwargs): raise ValueError("Unrecognized outpout.") if self._out_dims_mappings: - assert len(new_output) == len(self._out_dims_mappings), \ - "The length of dims_mapping {} does not matching the length output {}.".format(len(self._out_dims_mappings), len(new_output)) + assert len(new_output) == len( + self._out_dims_mappings + ), "The length of dims_mapping {} does not matching the length output {}.".format( + len(self._out_dims_mappings), len(new_output) + ) for i, item in enumerate(new_output): if isinstance(item, Variable) and self._out_dims_mappings: tensor_to_dims_mapping[item.name] = self._out_dims_mappings[i] from .dist_context import get_default_distributed_context + default_dist_ctx = get_default_distributed_context() for idx in range(op_size, new_op_size): op = cur_block.ops[idx] @@ -298,48 +331,62 @@ def __call__(self, *args, **kwargs): if name in tensor_to_dims_mapping.keys(): tensor = dist_op.get_serial_input(name) tensor_dist_attr = dist_op.dist_attr.get_input_dist_attr( - name) + name + ) dims_mapping = tensor_to_dims_mapping[name] if tensor is None: tensor_shape = [] else: - if tensor.type == core.VarDesc.VarType.READER \ - or tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or tensor.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + tensor.type == core.VarDesc.VarType.READER + or tensor.type + == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or tensor.type == core.VarDesc.VarType.STEP_SCOPES + ): tensor_shape = [] else: tensor_shape = tensor.shape if dims_mapping is not None: dims_mapping = tensor_to_dims_mapping[name] shard_spec = convert_to_shard_spec( - dims_mapping, self._process_mesh) - assert verify_shard_spec(shard_spec, tensor_shape, self._process_mesh), \ - "For tensor {}, shard_spec {} is invalid with tensor_shape {} and process_mesh {}.".format( - name, shard_spec, tensor_shape, self._process_mesh) + dims_mapping, self._process_mesh + ) + assert verify_shard_spec( + shard_spec, tensor_shape, self._process_mesh + ), "For tensor {}, shard_spec {} is invalid with tensor_shape {} and process_mesh {}.".format( + name, shard_spec, tensor_shape, self._process_mesh + ) tensor_dist_attr.dims_mapping = dims_mapping tensor_dist_attr.mark_annotated("dims_mapping") for name in dist_op.serial_op.output_arg_names: if name in tensor_to_dims_mapping.keys(): tensor = dist_op.get_serial_output(name) tensor_dist_attr = dist_op.dist_attr.get_output_dist_attr( - name) + name + ) dims_mapping = tensor_to_dims_mapping[name] if tensor is None: tensor_shape = [] else: - if tensor.type == core.VarDesc.VarType.READER \ - or tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or tensor.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + tensor.type == core.VarDesc.VarType.READER + or tensor.type + == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or tensor.type == core.VarDesc.VarType.STEP_SCOPES + ): tensor_shape = [] else: tensor_shape = tensor.shape if dims_mapping is not None: dims_mapping = tensor_to_dims_mapping[name] shard_spec = convert_to_shard_spec( - dims_mapping, self._process_mesh) - assert verify_shard_spec(shard_spec, tensor_shape, self._process_mesh), \ - "For tensor {}, shard_spec {} is invalid with tensor_shape {} and process_mesh {}.".format( - name, shard_spec, tensor_shape, self._process_mesh) + dims_mapping, self._process_mesh + ) + assert verify_shard_spec( + shard_spec, tensor_shape, self._process_mesh + ), "For tensor {}, shard_spec {} is invalid with tensor_shape {} and process_mesh {}.".format( + name, shard_spec, tensor_shape, self._process_mesh + ) tensor_dist_attr.dims_mapping = dims_mapping tensor_dist_attr.mark_annotated("dims_mapping") dist_op.dist_attr.process_mesh = self._process_mesh diff --git a/python/paddle/distributed/auto_parallel/dist_saver.py b/python/paddle/distributed/auto_parallel/dist_saver.py index a885bf7592ba9..f2766bff602e3 100644 --- a/python/paddle/distributed/auto_parallel/dist_saver.py +++ b/python/paddle/distributed/auto_parallel/dist_saver.py @@ -50,16 +50,13 @@ def _process_path(path): class DistributedSaver: - def __init__(self): self._logger = get_logger(logging.INFO) def save(self, path, serial_program, dist_main_program, dist_context): - def _save_state(program, path, mode="param"): state = { - k: np.array(v) - for k, v in program.state_dict(mode).items() + k: np.array(v) for k, v in program.state_dict(mode).items() } with open(path, "wb") as f: pickle.dump(state, f) @@ -105,8 +102,9 @@ def load(self, path, load_optimizer=True): def _load_file(filename, dirname, suffix="pdparams"): file_list = [] for file in os.listdir(dirname): - if check_filename('{}(.*)_dist(.*).{}'.format(filename, suffix), - file): + if check_filename( + '{}(.*)_dist(.*).{}'.format(filename, suffix), file + ): file_list.append(os.path.join(dirname, file)) file_list.sort() return file_list @@ -134,14 +132,16 @@ def _load_state(filename, dirname, suffix="pdparams"): # load path.pdparam and path.pdopt param_state_dict = _load_state(filename, dirname) - opt_state_dict = _load_state(filename, dirname, - "pdopt") if load_optimizer else {} + opt_state_dict = ( + _load_state(filename, dirname, "pdopt") if load_optimizer else {} + ) state_dict = dict(param_state_dict, **opt_state_dict) # load path.pdattr dist_attr_file_list = _load_file(filename, dirname, "pdattr") self._logger.info( - "Load distributed attribute file: {}".format(dist_attr_file_list)) + "Load distributed attribute file: {}".format(dist_attr_file_list) + ) dist_attr = {} for dist_attr_file in dist_attr_file_list: with open(dist_attr_file, 'rb') as f: @@ -206,11 +206,13 @@ def save_inference_model(self, path, feed_vars, fetch_vars, exe, **kwargs): # NOTE: `paddle.static.save_inference_model` does not support subblock. dist_filename = filename + "_dist" + str(rank_id) dist_path = os.path.join(dirname, dist_filename) - paddle.static.save_inference_model(dist_path, - dist_feed_vars, - dist_fetch_vars, - exe, - program=dist_main_prog) + paddle.static.save_inference_model( + dist_path, + dist_feed_vars, + dist_fetch_vars, + exe, + program=dist_main_prog, + ) def _save_rank_mapping(self, dirname): path = os.path.join(dirname, 'rank_mapping.csv') diff --git a/python/paddle/distributed/auto_parallel/dist_tensor.py b/python/paddle/distributed/auto_parallel/dist_tensor.py index e07269fab25f9..88c754f06f3d8 100644 --- a/python/paddle/distributed/auto_parallel/dist_tensor.py +++ b/python/paddle/distributed/auto_parallel/dist_tensor.py @@ -33,32 +33,45 @@ class DistributedTensor: """ @staticmethod - def _validate_sizes_and_dist_attr(sizes, - dims_mapping, - topology, - processes, - rank=None, - shard_sizes=None): - if not (isinstance(sizes, (list, tuple)) - and all(map(lambda x: isinstance(x, int) and x >= 0, sizes))): + def _validate_sizes_and_dist_attr( + sizes, dims_mapping, topology, processes, rank=None, shard_sizes=None + ): + if not ( + isinstance(sizes, (list, tuple)) + and all(map(lambda x: isinstance(x, int) and x >= 0, sizes)) + ): raise ValueError( - "The sizes must be list or tuple and item in sizes must be non-negative integer, but got {}" - .format(sizes)) - if not (isinstance(dims_mapping, (list, tuple)) and all( - map(lambda x: isinstance(x, int) and x >= -1, dims_mapping))): + "The sizes must be list or tuple and item in sizes must be non-negative integer, but got {}".format( + sizes + ) + ) + if not ( + isinstance(dims_mapping, (list, tuple)) + and all(map(lambda x: isinstance(x, int) and x >= -1, dims_mapping)) + ): raise ValueError( - "The dims_mapping must be list or tuple and item in dims_mapping must >= -1, but got {}" - .format(dims_mapping)) - if not (isinstance(processes, (list, tuple)) and all( - map(lambda x: isinstance(x, int) and x >= 0, processes))): + "The dims_mapping must be list or tuple and item in dims_mapping must >= -1, but got {}".format( + dims_mapping + ) + ) + if not ( + isinstance(processes, (list, tuple)) + and all(map(lambda x: isinstance(x, int) and x >= 0, processes)) + ): raise ValueError( - "The processes must be list or tuple and item in processes must be integer, but got {}" - .format(processes)) - if not (isinstance(topology, (list, tuple)) - and all(map(lambda x: isinstance(x, int) and x > 0, topology))): + "The processes must be list or tuple and item in processes must be integer, but got {}".format( + processes + ) + ) + if not ( + isinstance(topology, (list, tuple)) + and all(map(lambda x: isinstance(x, int) and x > 0, topology)) + ): raise ValueError( - "The topology must be list or tuple and item in topology must be non-negative integer, but got {}" - .format(topology)) + "The topology must be list or tuple and item in topology must be non-negative integer, but got {}".format( + topology + ) + ) if rank is not None and not (isinstance(rank, int) and rank >= 0): raise ValueError("The rank must >= 0, but got {}".format(rank)) @@ -67,16 +80,17 @@ def _validate_sizes_and_dist_attr(sizes, raise ValueError("Only support even sharding now.") @staticmethod - def get_local_sizes(global_sizes, - dims_mapping, - topology, - processes, - rank=None, - shard_sizes=None): - DistributedTensor._validate_sizes_and_dist_attr(global_sizes, - dims_mapping, topology, - processes, rank, - shard_sizes) + def get_local_sizes( + global_sizes, + dims_mapping, + topology, + processes, + rank=None, + shard_sizes=None, + ): + DistributedTensor._validate_sizes_and_dist_attr( + global_sizes, dims_mapping, topology, processes, rank, shard_sizes + ) local_sizes = [] # for even sharding, the local sizes of every rank are equal @@ -92,16 +106,12 @@ def get_local_sizes(global_sizes, return local_sizes @staticmethod - def get_local_offsets(global_sizes, - dims_mapping, - topology, - processes, - rank, - shard_sizes=None): - local_sizes = DistributedTensor.get_local_sizes(global_sizes, - dims_mapping, topology, - processes, rank, - shard_sizes) + def get_local_offsets( + global_sizes, dims_mapping, topology, processes, rank, shard_sizes=None + ): + local_sizes = DistributedTensor.get_local_sizes( + global_sizes, dims_mapping, topology, processes, rank, shard_sizes + ) local_offsets = [] rank_relatvie = processes.index(rank) coordinate = _linear_idx2coordinate(topology, rank_relatvie) @@ -110,21 +120,23 @@ def get_local_offsets(global_sizes, if dims_mapping[i] == -1: local_offsets.append(0) else: - local_offsets.append(coordinate[dims_mapping[i]] * - local_sizes[i]) + local_offsets.append( + coordinate[dims_mapping[i]] * local_sizes[i] + ) return local_offsets @staticmethod - def get_global_sizes(local_sizes, - dims_mapping, - topology, - processes, - rank=None, - shard_sizes=None): - DistributedTensor._validate_sizes_and_dist_attr(local_sizes, - dims_mapping, topology, - processes, rank, - shard_sizes) + def get_global_sizes( + local_sizes, + dims_mapping, + topology, + processes, + rank=None, + shard_sizes=None, + ): + DistributedTensor._validate_sizes_and_dist_attr( + local_sizes, dims_mapping, topology, processes, rank, shard_sizes + ) global_sizes = [] for idx, item in enumerate(local_sizes): if dims_mapping[idx] == -1: @@ -134,25 +146,24 @@ def get_global_sizes(local_sizes, return global_sizes @staticmethod - def get_local_shard(global_sizes, - dims_mapping, - topology, - processes, - rank, - shard_sizes=None): + def get_local_shard( + global_sizes, dims_mapping, topology, processes, rank, shard_sizes=None + ): local_offsets = DistributedTensor.get_local_offsets( - global_sizes, dims_mapping, topology, processes, rank, shard_sizes) - local_sizes = DistributedTensor.get_local_sizes(global_sizes, - dims_mapping, topology, - processes, rank, - shard_sizes) + global_sizes, dims_mapping, topology, processes, rank, shard_sizes + ) + local_sizes = DistributedTensor.get_local_sizes( + global_sizes, dims_mapping, topology, processes, rank, shard_sizes + ) assert len(local_sizes) == len( local_offsets ), "The length of local_sizes must be equal to local_offsets, but got {} and {}.".format( - len(local_sizes), len(local_offsets)) + len(local_sizes), len(local_offsets) + ) local_end_offsets = list( - map(lambda x: x[0] + x[1], zip(local_offsets, local_sizes))) + map(lambda x: x[0] + x[1], zip(local_offsets, local_sizes)) + ) local_shard = list(zip(local_offsets, local_end_offsets)) return local_shard @@ -167,7 +178,11 @@ def __init__(self, serial_tensor, dist_attr=None, dist_context=None): self._local_tensor_map = {} from .dist_context import get_default_distributed_context - self._dist_context = dist_context if dist_context is not None else get_default_distributed_context( + + self._dist_context = ( + dist_context + if dist_context is not None + else get_default_distributed_context() ) # TODO: Add Automatically to dist_context after initialized and it will be adapted in the future. # self._dist_context.add_dist_tensor_for_program(self) @@ -193,9 +208,12 @@ def dist_attr(self, dist_attr): def _init_default_dist_attr(self): if self._dist_attr.dims_mapping is None: - if self.serial_tensor.type == core.VarDesc.VarType.READER \ - or self.serial_tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or self.serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + self.serial_tensor.type == core.VarDesc.VarType.READER + or self.serial_tensor.type + == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or self.serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES + ): tensor_shape = [] else: tensor_shape = self._serial_tensor.shape @@ -203,17 +221,21 @@ def _init_default_dist_attr(self): self._dist_attr.dims_mapping = tensor_dims_mapping def validate_dist_attr(self): - if self.serial_tensor.type == core.VarDesc.VarType.READER \ - or self.serial_tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or self.serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + self.serial_tensor.type == core.VarDesc.VarType.READER + or self.serial_tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or self.serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES + ): return True tensor_shape = self.serial_tensor.shape if len(tensor_shape) != len(self.dist_attr.dims_mapping): return False for i in range(len(self.dist_attr.dims_mapping)): if self.dist_attr.dims_mapping[ - i] < -1 or self.dist_attr.dims_mapping[i] >= len( - self.dist_attr.process_mesh.topology): + i + ] < -1 or self.dist_attr.dims_mapping[i] >= len( + self.dist_attr.process_mesh.topology + ): return False for i in range(len(self.dist_attr.process_mesh.topology)): if self.dist_attr.dims_mapping.count(i) > 1: @@ -228,10 +250,9 @@ def local_sizes(self, rank=None): shard_sizes = self.dist_attr.shard_sizes processes = self.dist_attr.process_mesh.processes topology = self.dist_attr.process_mesh.topology - local_sizes = DistributedTensor.get_local_sizes(global_sizes, - dims_mapping, topology, - processes, rank, - shard_sizes) + local_sizes = DistributedTensor.get_local_sizes( + global_sizes, dims_mapping, topology, processes, rank, shard_sizes + ) return local_sizes @@ -247,8 +268,13 @@ def local_offsets(self, rank=None): processes = self.dist_attr.process_mesh.processes topology = self.dist_attr.process_mesh.topology local_offsets = DistributedTensor.get_local_offsets( - global_sizes, dims_mapping, topology, processes, rank, - shard_sizes) + global_sizes, + dims_mapping, + topology, + processes, + rank, + shard_sizes, + ) self._local_offsets_map[rank] = local_offsets return local_offsets @@ -268,8 +294,13 @@ def local_shard(self, rank=None): processes = self.dist_attr.process_mesh.processes topology = self.dist_attr.process_mesh.topology local_shard = DistributedTensor.get_local_shard( - global_sizes, dims_mapping, topology, processes, rank, - shard_sizes) + global_sizes, + dims_mapping, + topology, + processes, + rank, + shard_sizes, + ) self._local_shard_map[rank] = local_shard return local_shard @@ -324,8 +355,9 @@ def _copy_kwargs(serial_tensor): if rank is not None and not (isinstance(rank, int) and rank >= 0): raise ValueError("The rank must >= 0, but got {}".format(rank)) if block is not None and not isinstance(block, Block): - raise TypeError("The block must be Block, but got {}.".format( - type(block))) + raise TypeError( + "The block must be Block, but got {}.".format(type(block)) + ) rank = paddle.distributed.get_rank() if rank is None else rank if block is None: @@ -350,8 +382,9 @@ def _copy_kwargs(serial_tensor): def local_tensor(self, rank=None): rank = paddle.distributed.get_rank() if rank is None else rank - assert rank in self._local_tensor_map, "The rank {} local tensor has not been created.".format( - rank) + assert ( + rank in self._local_tensor_map + ), "The rank {} local tensor has not been created.".format(rank) return self._local_tensor_map[rank] def __deepcopy__(self, memo): @@ -367,7 +400,8 @@ def __deepcopy__(self, memo): def __str__(self): str = "{{tensor name: {}, tensor id: {}".format( - self.serial_tensor.desc.name(), self.serial_tensor.desc.id()) + self.serial_tensor.desc.name(), self.serial_tensor.desc.id() + ) # str += ", {}".format(self.dist_attr) # return str @@ -376,8 +410,9 @@ def __str__(self): annotated_str = "annotated" else: annotated_str = "non-annotated" - str += ", process_mesh ({}): {}".format(annotated_str, - self.dist_attr.process_mesh) + str += ", process_mesh ({}): {}".format( + annotated_str, self.dist_attr.process_mesh + ) str += ", is_parameter: {}".format(self.serial_tensor.is_parameter) @@ -385,8 +420,9 @@ def __str__(self): annotated_str = "annotated" else: annotated_str = "non-annotated" - str += ", dims_mapping ({}): {}".format(annotated_str, - self.dist_attr.dims_mapping) + str += ", dims_mapping ({}): {}".format( + annotated_str, self.dist_attr.dims_mapping + ) if self.dist_attr.is_annotated("shard_mask"): annotated_str = "annotated" diff --git a/python/paddle/distributed/auto_parallel/engine.py b/python/paddle/distributed/auto_parallel/engine.py index 1d6bce931d28e..1581beb2a511c 100644 --- a/python/paddle/distributed/auto_parallel/engine.py +++ b/python/paddle/distributed/auto_parallel/engine.py @@ -41,7 +41,10 @@ from .parallelizer_v2 import Parallelizer from .dist_op import DistributedOperator from .dist_saver import DistributedSaver -from .dist_loader import DistributedDataLoaderFromGenerator, DistributedDataLoader +from .dist_loader import ( + DistributedDataLoaderFromGenerator, + DistributedDataLoader, +) from .utils import to_list, get_dist_attr, get_lr from .process_group import new_process_group, get_all_process_groups from .dist_context import DistributedContext, get_default_distributed_context @@ -115,16 +118,21 @@ class Engine: """ - def __init__(self, - model=None, - loss=None, - optimizer=None, - metrics=None, - cluster=None, - strategy=None): - - if model and not isinstance(model, - paddle.nn.Layer) and not callable(model): + def __init__( + self, + model=None, + loss=None, + optimizer=None, + metrics=None, + cluster=None, + strategy=None, + ): + + if ( + model + and not isinstance(model, paddle.nn.Layer) + and not callable(model) + ): raise TypeError( "'model must be sub classes of `paddle.nn.Layer` or any callable function." ) @@ -132,18 +140,20 @@ def __init__(self, self._loss = loss if optimizer and not isinstance( - optimizer, - (paddle.optimizer.Optimizer, paddle.fluid.optimizer.Optimizer)): + optimizer, + (paddle.optimizer.Optimizer, paddle.fluid.optimizer.Optimizer), + ): raise TypeError( "'optimizer' must be object of class `paddle.optimizer.Optimizer`" - " or `paddle.fluid.optimizer.Optimizer`.") + " or `paddle.fluid.optimizer.Optimizer`." + ) self._optimizer = self._validate_opt(optimizer) metrics = metrics or [] for metric in to_list(metrics): - assert isinstance(metric, Metric), \ - "{} is not sub class of Metric".format( - metric.__class__.__name__) + assert isinstance( + metric, Metric + ), "{} is not sub class of Metric".format(metric.__class__.__name__) self._metrics = to_list(metrics) if cluster and not isinstance(cluster, Cluster): @@ -159,8 +169,9 @@ def __init__(self, self._strategy = strategy or Strategy() if os.getenv("POD_NAME"): - print("Distribute training by paddle.distributed.launch", - flush=True) + print( + "Distribute training by paddle.distributed.launch", flush=True + ) fleet.init(is_collective=True) self._executor = None @@ -185,7 +196,7 @@ def __init__(self, self._has_prepared_reader = { "train": False, "eval": False, - "predict": False + "predict": False, } self._inputs_spec = [] self._labels_spec = [] @@ -220,8 +231,10 @@ def _prepare_data_spec(self, data, split, batch_size): labels = sample[split:] else: raise ValueError( - "Data should be a Dataset or IterableDatset, but received {}.". - format(type(data).__name__)) + "Data should be a Dataset or IterableDatset, but received {}.".format( + type(data).__name__ + ) + ) inputs = to_list(inputs) labels = to_list(labels) @@ -264,34 +277,40 @@ def _infer_item_spec(item, name, batch_size, specs): labels_spec = self._validate_spec(labels_spec) return inputs_spec, labels_spec - def _prepare_data_tensor(self, - inputs_spec, - labels_spec, - inputs=None, - labels=None): + def _prepare_data_tensor( + self, inputs_spec, labels_spec, inputs=None, labels=None + ): if _non_static_mode() or self._dygraph_mode: return None, None inputs_spec = inputs_spec if inputs_spec else [] labels_spec = labels_spec if labels_spec else [] if inputs_spec: - assert isinstance(inputs_spec, list), \ - "inputs should be list, but received {}".format(type(inputs_spec)) + assert isinstance( + inputs_spec, list + ), "inputs should be list, but received {}".format( + type(inputs_spec) + ) if inputs is None: inputs = [s._create_feed_layer() for s in inputs_spec] else: - assert isinstance(inputs, list), \ - "inputs should be list, but received {}".format(type(inputs)) + assert isinstance( + inputs, list + ), "inputs should be list, but received {}".format(type(inputs)) for input_spec, input in zip(inputs_spec, inputs): if input_spec.shape != input.shape: input.desc.set_shape(input_spec.shape) if labels_spec: - assert isinstance(labels_spec, list), \ - "labels should be list, but received {}".format(type(labels_spec)) + assert isinstance( + labels_spec, list + ), "labels should be list, but received {}".format( + type(labels_spec) + ) if labels is None: labels = [s._create_feed_layer() for s in labels_spec] else: - assert isinstance(labels, list), \ - "labels should be list, but received {}".format(type(labels)) + assert isinstance( + labels, list + ), "labels should be list, but received {}".format(type(labels)) for label_spec, label in zip(labels_spec, labels): if label_spec.shape != label.shape: label.desc.set_shape(label_spec.shape) @@ -304,7 +323,9 @@ def _prepare_reader(self): # NOTE: this list may be changed if Paddle changes the existing rules. related_reader_ops = [ - "create_py_reader", "create_double_buffer_reader", "read" + "create_py_reader", + "create_double_buffer_reader", + "read", ] # remove the first three ops if multiple run fit/evaluate/predict if dist_main_block.ops[0].type == 'create_py_reader': @@ -322,9 +343,9 @@ def _prepare_reader(self): for idx in reversed(reader_op_indices): new_op_desc = dist_main_block.desc._prepend_op() new_op_desc.copy_from(dist_main_block.ops[idx].desc) - new_op = Operator(dist_main_block, - new_op_desc, - type=new_op_desc.type()) + new_op = Operator( + dist_main_block, new_op_desc, type=new_op_desc.type() + ) new_reader_ops.append(new_op) dist_op = DistributedOperator(new_op) dist_context.add_dist_op_for_program(dist_op) @@ -355,16 +376,22 @@ def _prepare_feed(self, data, user_feeds, mode): else: raise ValueError("Unsupported data {}".format(data)) if user_feeds is not None: - assert isinstance(user_feeds, dict), \ - "user_feeds must be a dict, but receive {}".format(type(user_feeds).__name__) + assert isinstance( + user_feeds, dict + ), "user_feeds must be a dict, but receive {}".format( + type(user_feeds).__name__ + ) for name, data in user_feeds.items(): feeds[name] = data return feeds def _prepare_fetch(self, user_fetches, mode): if user_fetches is not None: - assert isinstance(user_fetches, list), \ - "user_fetches must be a list, but receive {}".format(type(user_fetches).__name__) + assert isinstance( + user_fetches, list + ), "user_fetches must be a list, but receive {}".format( + type(user_fetches).__name__ + ) fetch_names = [] fetch_indices = [] @@ -396,14 +423,16 @@ def _process_fetch_group(group_name, var_list): _process_fetch_group("fetches", var_list) return fetch_names, fetch_indices - def _prepare_logger(self, - outs, - epoch=None, - step=None, - lr=None, - fetch_names=None, - fetch_indices=None, - mode=None): + def _prepare_logger( + self, + outs, + epoch=None, + step=None, + lr=None, + fetch_names=None, + fetch_indices=None, + mode=None, + ): logs = {} if epoch is not None: logs["epoch"] = epoch @@ -470,9 +499,9 @@ def _build(self, mode): inputs_spec = self._inputs_spec labels_spec = self._labels_spec if self._labels_spec else [] - self.program_helper = ProgramHelper(self._model, self._loss, - self._metrics, inputs_spec, - labels_spec) + self.program_helper = ProgramHelper( + self._model, self._loss, self._metrics, inputs_spec, labels_spec + ) # build forward main program self.program_helper.build_program(mode) @@ -505,8 +534,9 @@ def _build(self, mode): serial_main_prog = self._orig_main_prog.clone() serial_startup_prog = self._orig_startup_prog.clone() if not self._skip_build: - with static.program_guard(serial_main_prog, serial_startup_prog), \ - utils.unique_name.guard(): + with static.program_guard( + serial_main_prog, serial_startup_prog + ), utils.unique_name.guard(): outputs = to_list(self._model(*inputs)) if mode != "predict" and self._loss: losses = to_list(self._loss(*(outputs + labels))) @@ -515,7 +545,8 @@ def _build(self, mode): if mode != "predict" and (outputs or labels): for metric in self._metrics: metrics.append( - to_list(metric.compute(*(outputs + labels)))) + to_list(metric.compute(*(outputs + labels))) + ) else: losses = to_list(self._loss) self.losses = losses @@ -532,7 +563,7 @@ def _build(self, mode): fetch_vars = { "outputs": flatten(outputs), "loss": losses, - "metrics": metrics + "metrics": metrics, } if mode != "train": @@ -540,8 +571,15 @@ def _build(self, mode): self._set_recompute_ckpts() self._dist_contexts[mode] = DistributedContext( - serial_main_prog, serial_startup_prog, self._optimizer, losses, - feed_vars, fetch_vars, self._cluster, self._strategy) + serial_main_prog, + serial_startup_prog, + self._optimizer, + losses, + feed_vars, + fetch_vars, + self._cluster, + self._strategy, + ) self._dist_contexts[mode].gradient_scale = self._strategy.gradient_scale def _optimization_tuning(self, mode, dataset, batch_size): @@ -558,20 +596,24 @@ def _optimization_tuning(self, mode, dataset, batch_size): dataset.dp_rank = self._dp_ranks from .tuner.optimization_tuner import OptimizationTuner - self._optimization_tuner = OptimizationTuner(self._tuning.to_dict(), - self._dist_contexts[mode], - dataset, - self._inputs_spec, - self._labels_spec, - batch_size=batch_size, - rank=self._cur_rank) + + self._optimization_tuner = OptimizationTuner( + self._tuning.to_dict(), + self._dist_contexts[mode], + dataset, + self._inputs_spec, + self._labels_spec, + batch_size=batch_size, + rank=self._cur_rank, + ) self._optimization_tuner.tune() if self._tuning.run_after_tuning: # update the strategy self._dist_contexts[ - mode]._strategy = self._optimization_tuner.get_best_config() + mode + ]._strategy = self._optimization_tuner.get_best_config() def _plan(self, mode): if self._planned_mode is None: @@ -596,7 +638,8 @@ def _plan(self, mode): self._dp_ranks = [] for feed_var in feed_list: dp_world_size, dp_rank = self._get_input_split_info( - feed_var, self._dist_contexts[mode]) + feed_var, self._dist_contexts[mode] + ) self._dp_world_sizes.append(dp_world_size) self._dp_ranks.append(dp_rank) @@ -604,8 +647,9 @@ def _parallel(self, mode, all_ranks=False): # Parallelize program based on the planner's results # For now, the completer has to be passed to the planner, # because we may use it to complete the annotation of the backwarkward and update. - parallelizer = Parallelizer(mode, self._planners[mode].completer, - self._dist_contexts[mode]) + parallelizer = Parallelizer( + mode, self._planners[mode].completer, self._dist_contexts[mode] + ) if not all_ranks: parallelizer.parallel(self._cur_rank) else: @@ -623,22 +667,30 @@ def _init_dist_context(self, mode): for ib, block in enumerate(origin_main_prog.blocks): for iop, op in enumerate(block.ops): ref_op = ref_blocks[ib].ops[iop] - assert op.type == ref_op.type, \ - "'{}' mode op '{}' is different with '{}' op '{}'. ".format(mode, op.type, ref_mode, ref_op.type) - ref_op_dist_attr = ref_dist_context.get_op_dist_attr_for_program( - ref_op) + assert ( + op.type == ref_op.type + ), "'{}' mode op '{}' is different with '{}' op '{}'. ".format( + mode, op.type, ref_mode, ref_op.type + ) + ref_op_dist_attr = ( + ref_dist_context.get_op_dist_attr_for_program(ref_op) + ) dist_context.set_op_dist_attr_for_program(op, ref_op_dist_attr) def _initialize(self, mode): # Get the current content from the distributed context self._serial_main_progs[mode] = self._dist_contexts[ - mode].serial_main_program + mode + ].serial_main_program self._serial_startup_progs[mode] = self._dist_contexts[ - mode].serial_startup_program + mode + ].serial_startup_program self._dist_main_progs[mode] = self._dist_contexts[ - mode].dist_main_programs + mode + ].dist_main_programs self._dist_startup_progs[mode] = self._dist_contexts[ - mode].dist_startup_programs + mode + ].dist_startup_programs self._feed_vars[mode] = self._dist_contexts[mode].serial_feed_vars self._fetch_vars[mode] = self._dist_contexts[mode].serial_fetch_vars self._optimizer = self._dist_contexts[mode]._serial_optimizer @@ -685,30 +737,33 @@ def _initialize(self, mode): self._executor.run(prune_startup_prog) if hasattr(self, "_state_dict") and hasattr(self, "_dist_attr"): - self._set_state_dict(mode, self._strict, self._state_dict, - self._dist_attr) + self._set_state_dict( + mode, self._strict, self._state_dict, self._dist_attr + ) if self._strategy.reinit: self._logger.info("NOTE: parameters will be re-initialized.") dist_startup_prog = self._dist_startup_progs[mode][self._cur_rank] self._executor.run(dist_startup_prog) - def fit(self, - train_data, - train_sample_split=None, - batch_size=1, - epochs=1, - steps_per_epoch=None, - log_freq=10, - save_dir=None, - save_freq=1, - valid_data=None, - valid_sample_split=None, - valid_freq=1, - valid_steps=None, - collate_fn=None, - callbacks=None, - verbose=2): + def fit( + self, + train_data, + train_sample_split=None, + batch_size=1, + epochs=1, + steps_per_epoch=None, + log_freq=10, + save_dir=None, + save_freq=1, + valid_data=None, + valid_sample_split=None, + valid_freq=1, + valid_steps=None, + collate_fn=None, + callbacks=None, + verbose=2, + ): """ Trains the model for a fixed number of epochs. If `valid_data` is set, evaluation will be done at the end of each epoch. @@ -777,16 +832,19 @@ def fit(self, """ self._mode = 'train' self._inputs_spec, self._labels_spec = self._prepare_data_spec( - train_data, train_sample_split, batch_size) + train_data, train_sample_split, batch_size + ) self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) if not self._has_prepared[self._mode]: self._prepare_program(self._mode) else: self._switch_mode(self._mode) - assert self._mode in self._dist_main_progs, \ - "train model is not ready, please call `engine._prepare_program('train')` first." + assert ( + self._mode in self._dist_main_progs + ), "train model is not ready, please call `engine._prepare_program('train')` first." train_dataloader = self._prepare_dataloader_from_generator( dataset=train_data, @@ -795,7 +853,8 @@ def fit(self, batch_size=batch_size, epochs=epochs, steps_per_epoch=steps_per_epoch, - collate_fn=collate_fn) + collate_fn=collate_fn, + ) fetch_names, fetch_indices = self._prepare_fetch(None, mode=self._mode) @@ -824,21 +883,35 @@ def fit(self, self.main_program, fetch_list=fetch_names, use_program_cache=self._strategy.use_cache, - return_numpy=self._strategy.return_numpy) + return_numpy=self._strategy.return_numpy, + ) except core.EOFException: break lr = get_lr(self._optimizer) - logs = self._prepare_logger(outs, epoch, step, lr, fetch_names, - fetch_indices, self._mode) + logs = self._prepare_logger( + outs, + epoch, + step, + lr, + fetch_names, + fetch_indices, + self._mode, + ) cbks.on_batch_end('train', step, logs) if valid_data and (epoch + 1) % valid_freq == 0: - val_logs = self.evaluate(valid_data, valid_sample_split, - batch_size, valid_steps, log_freq, - collate_fn, callbacks, verbose) + val_logs = self.evaluate( + valid_data, + valid_sample_split, + batch_size, + valid_steps, + log_freq, + collate_fn, + callbacks, + verbose, + ) val_logs = { - "val_" + name: val - for name, val in val_logs.items() + "val_" + name: val for name, val in val_logs.items() } logs.update(val_logs) self._switch_mode("train") @@ -850,15 +923,17 @@ def fit(self, cbks.on_end('train', logs) return self.history - def evaluate(self, - valid_data, - valid_sample_split=None, - batch_size=1, - steps=None, - log_freq=10, - collate_fn=None, - callbacks=None, - verbose=2): + def evaluate( + self, + valid_data, + valid_sample_split=None, + batch_size=1, + steps=None, + log_freq=10, + collate_fn=None, + callbacks=None, + verbose=2, + ): """ Evaluate the loss and metrics of the model on evaluation data. @@ -907,23 +982,27 @@ def evaluate(self, """ self._mode = 'eval' self._inputs_spec, self._labels_spec = self._prepare_data_spec( - valid_data, valid_sample_split, batch_size) + valid_data, valid_sample_split, batch_size + ) self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) if not self._has_prepared[self._mode]: self._prepare_program(self._mode) else: self._switch_mode(self._mode) - assert self._mode in self._dist_main_progs, \ - "eval model is not ready, please call `engine._prepare_program('eval')` first." + assert ( + self._mode in self._dist_main_progs + ), "eval model is not ready, please call `engine._prepare_program('eval')` first." valid_dataloader = self._prepare_dataloader_from_generator( dataset=valid_data, capacity=70, iterable=False, batch_size=batch_size, steps_per_epoch=steps, - collate_fn=collate_fn) + collate_fn=collate_fn, + ) fetch_names, fetch_indices = self._prepare_fetch(None, mode=self._mode) @@ -937,10 +1016,9 @@ def evaluate(self, ) eval_steps = valid_dataloader._steps - cbks.on_begin('eval', { - 'steps': eval_steps, - 'metrics': self._metrics_name() - }) + cbks.on_begin( + 'eval', {'steps': eval_steps, 'metrics': self._metrics_name()} + ) logs = {} for step, _ in enumerate(valid_dataloader): cbks.on_batch_begin('eval', step, logs) @@ -949,24 +1027,28 @@ def evaluate(self, self.main_program, fetch_list=fetch_names, use_program_cache=self._strategy.use_cache, - return_numpy=self._strategy.return_numpy) + return_numpy=self._strategy.return_numpy, + ) except core.EOFException: break - logs = self._prepare_logger(outs, None, step, None, fetch_names, - fetch_indices, self._mode) + logs = self._prepare_logger( + outs, None, step, None, fetch_names, fetch_indices, self._mode + ) cbks.on_batch_end('eval', step, logs) cbks.on_end('eval', logs) self._reset_metrics() return logs - def predict(self, - test_data, - test_sample_split=None, - batch_size=1, - steps=None, - collate_fn=None, - callbacks=None, - verbose=2): + def predict( + self, + test_data, + test_sample_split=None, + batch_size=1, + steps=None, + collate_fn=None, + callbacks=None, + verbose=2, + ): """ Compute the output predictions on testing data. @@ -1012,16 +1094,19 @@ def predict(self, """ self._mode = 'predict' self._inputs_spec, self._labels_spec = self._prepare_data_spec( - test_data, test_sample_split, batch_size) + test_data, test_sample_split, batch_size + ) self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) if not self._has_prepared[self._mode]: self._prepare_program(self._mode) else: self._switch_mode(self._mode) - assert self._mode in self._dist_main_progs, \ - "predict model is not ready, please call `engine._prepare_program('predict')` first." + assert ( + self._mode in self._dist_main_progs + ), "predict model is not ready, please call `engine._prepare_program('predict')` first." test_dataloader = self._prepare_dataloader_from_generator( dataset=test_data, @@ -1029,7 +1114,8 @@ def predict(self, iterable=False, batch_size=batch_size, steps_per_epoch=steps, - collate_fn=collate_fn) + collate_fn=collate_fn, + ) fetch_names, fetch_indices = self._prepare_fetch(None, mode=self._mode) @@ -1045,37 +1131,43 @@ def predict(self, self.main_program, fetch_list=fetch_names, use_program_cache=self._strategy.use_cache, - return_numpy=self._strategy.return_numpy) + return_numpy=self._strategy.return_numpy, + ) except core.EOFException: break - logs = self._prepare_logger(outs, None, step, None, fetch_names, - fetch_indices, self._mode) + logs = self._prepare_logger( + outs, None, step, None, fetch_names, fetch_indices, self._mode + ) cbks.on_batch_end('predict', step, logs) outputs.append(list(logs["outputs"].values())) cbks.on_end('predict', logs) return outputs - def dataloader(self, - dataset, - batch_size=1, - shuffle=False, - drop_last=False, - collate_fn=None, - num_workers=0, - use_buffer_reader=True, - use_shared_memory=True, - timeout=0, - worker_init_fn=None, - epochs=1, - steps_per_epoch=None, - sample_split=1, - mode=None): + def dataloader( + self, + dataset, + batch_size=1, + shuffle=False, + drop_last=False, + collate_fn=None, + num_workers=0, + use_buffer_reader=True, + use_shared_memory=True, + timeout=0, + worker_init_fn=None, + epochs=1, + steps_per_epoch=None, + sample_split=1, + mode=None, + ): if mode is not None: self.to_mode(mode) self._inputs_spec, self._labels_spec = self._prepare_data_spec( - dataset, sample_split, batch_size) + dataset, sample_split, batch_size + ) self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) if not self._has_prepared[self._mode]: self._prepare_program(self._mode) else: @@ -1093,28 +1185,33 @@ def dataloader(self, timeout=timeout, worker_init_fn=worker_init_fn, epochs=epochs, - steps_per_epoch=steps_per_epoch) + steps_per_epoch=steps_per_epoch, + ) return dataloader - def dataloader_from_generator(self, - dataset, - capacity=70, - use_double_buffer=True, - iterable=True, - use_multiprocess=False, - drop_last=True, - batch_size=1, - epochs=1, - steps_per_epoch=None, - collate_fn=None, - sample_split=1, - mode=None): + def dataloader_from_generator( + self, + dataset, + capacity=70, + use_double_buffer=True, + iterable=True, + use_multiprocess=False, + drop_last=True, + batch_size=1, + epochs=1, + steps_per_epoch=None, + collate_fn=None, + sample_split=1, + mode=None, + ): if mode is not None: self.to_mode(mode) self._inputs_spec, self._labels_spec = self._prepare_data_spec( - dataset, sample_split, batch_size) + dataset, sample_split, batch_size + ) self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) if not self._has_prepared[self._mode]: self._prepare_program(self._mode) else: @@ -1130,17 +1227,20 @@ def dataloader_from_generator(self, batch_size=batch_size, epochs=epochs, steps_per_epoch=steps_per_epoch, - collate_fn=collate_fn) + collate_fn=collate_fn, + ) return dataloader - def prepare(self, - inputs_spec=None, - labels_spec=None, - inputs=None, - labels=None, - main_program=None, - startup_program=None, - mode=None): + def prepare( + self, + inputs_spec=None, + labels_spec=None, + inputs=None, + labels=None, + main_program=None, + startup_program=None, + mode=None, + ): if mode is not None: self.to_mode(mode) if inputs or labels: @@ -1148,7 +1248,8 @@ def prepare(self, self._inputs_spec = inputs_spec self._labels_spec = labels_spec self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec, inputs, labels) + self._inputs_spec, self._labels_spec, inputs, labels + ) self._orig_main_prog = main_program if self._orig_main_prog is None: self._orig_main_prog = static.default_main_program() @@ -1164,7 +1265,8 @@ def prepare(self, self._labels_spec = labels_spec self._outside_dataloader = True self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) self._orig_main_prog = main_program if self._orig_main_prog is None: self._orig_main_prog = static.default_main_program() @@ -1176,44 +1278,55 @@ def prepare(self, else: self._switch_mode(self._mode) else: - assert self._inputs_spec and self._labels_spec, \ - "Please call the dataloader(...) before calling prepare(...)" + assert ( + self._inputs_spec and self._labels_spec + ), "Please call the dataloader(...) before calling prepare(...)" def run(self, data=None, feed=None, fetch_list=None, mode=None): if mode is not None: self.to_mode(mode) feed_dict = self._prepare_feed(data, feed, self._mode) fetch_names, fetch_indices = self._prepare_fetch(fetch_list, self._mode) - if self._outside_dataloader and not self._has_prepared_reader[ - self._mode]: + if ( + self._outside_dataloader + and not self._has_prepared_reader[self._mode] + ): self._prepare_reader() - outs = self._executor.run(self.main_program, - feed=feed_dict, - fetch_list=fetch_names, - use_program_cache=self._strategy.use_cache, - return_numpy=self._strategy.return_numpy) - logs = self._prepare_logger(outs, None, None, None, fetch_names, - fetch_indices, self._mode) + outs = self._executor.run( + self.main_program, + feed=feed_dict, + fetch_list=fetch_names, + use_program_cache=self._strategy.use_cache, + return_numpy=self._strategy.return_numpy, + ) + logs = self._prepare_logger( + outs, None, None, None, fetch_names, fetch_indices, self._mode + ) return logs - def _prepare_dataloader(self, - dataset, - return_list=True, - batch_size=1, - shuffle=False, - drop_last=False, - collate_fn=None, - num_workers=0, - use_buffer_reader=True, - use_shared_memory=True, - timeout=0, - worker_init_fn=None, - epochs=1, - steps_per_epoch=None): + def _prepare_dataloader( + self, + dataset, + return_list=True, + batch_size=1, + shuffle=False, + drop_last=False, + collate_fn=None, + num_workers=0, + use_buffer_reader=True, + use_shared_memory=True, + timeout=0, + worker_init_fn=None, + epochs=1, + steps_per_epoch=None, + ): if self._strategy.gradient_merge and batch_size is not None: - assert batch_size % self._k_steps == 0, \ - "Requires batch_size:[{}] to be divisible by k_steps:[{}].".format(batch_size, self._k_steps) + assert ( + batch_size % self._k_steps == 0 + ), "Requires batch_size:[{}] to be divisible by k_steps:[{}].".format( + batch_size, self._k_steps + ) batch_size //= self._k_steps dist_main_prog = self._dist_main_progs[self._mode][self._cur_rank] @@ -1257,26 +1370,32 @@ def _prepare_dataloader(self, steps_per_epoch=steps_per_epoch, split_data=self._strategy.split_data, data_parallel_world_size=self._dp_world_sizes, - data_parallel_rank=self._dp_ranks) + data_parallel_rank=self._dp_ranks, + ) return dataloader - def _prepare_dataloader_from_generator(self, - dataset, - capacity=None, - use_double_buffer=True, - iterable=True, - return_list=False, - use_multiprocess=False, - drop_last=True, - batch_size=1, - epochs=1, - steps_per_epoch=None, - collate_fn=None): + def _prepare_dataloader_from_generator( + self, + dataset, + capacity=None, + use_double_buffer=True, + iterable=True, + return_list=False, + use_multiprocess=False, + drop_last=True, + batch_size=1, + epochs=1, + steps_per_epoch=None, + collate_fn=None, + ): if self._strategy.gradient_merge and batch_size is not None: - assert batch_size % self._k_steps == 0, \ - "Requires batch_size:[{}] to be divisible by k_steps:[{}].".format(batch_size, self._k_steps) + assert ( + batch_size % self._k_steps == 0 + ), "Requires batch_size:[{}] to be divisible by k_steps:[{}].".format( + batch_size, self._k_steps + ) batch_size //= self._k_steps dist_main_prog = self._dist_main_progs[self._mode][self._cur_rank] @@ -1317,16 +1436,19 @@ def _prepare_dataloader_from_generator(self, collate_fn=collate_fn, split_data=self._strategy.split_data, data_parallel_world_size=self._dp_world_sizes, - data_parallel_rank=self._dp_ranks) + data_parallel_rank=self._dp_ranks, + ) self._prepare_reader() return dataloader def _tune(self, tune_data, tune_sample_split=None, batch_size=1): self._mode = 'train' self._inputs_spec, self._labels_spec = self._prepare_data_spec( - tune_data, tune_sample_split, batch_size) + tune_data, tune_sample_split, batch_size + ) self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) self._optimization_tuning(self._mode, tune_data, batch_size) def _validate_spec(self, specs): @@ -1337,12 +1459,17 @@ def _validate_spec(self, specs): assert isinstance(spec, InputSpec) if spec.name is None: raise ValueError( - "Requires Input[{}].name != None, but receive `None` with {}." - .format(i, spec)) + "Requires Input[{}].name != None, but receive `None` with {}.".format( + i, spec + ) + ) if self._k_steps > 1: shape = list(spec.shape) - assert shape[0] % self._k_steps == 0, \ - "Requires batch_size[{}] to be divisible by k_steps[{}].".format(spec.shape[0], self._k_steps) + assert ( + shape[0] % self._k_steps == 0 + ), "Requires batch_size[{}] to be divisible by k_steps[{}].".format( + spec.shape[0], self._k_steps + ) shape[0] //= self._k_steps spec.shape = shape return specs @@ -1360,16 +1487,20 @@ def _get_input_split_info(self, var, dist_context): dims_mapping = tensor_dist_attr.dims_mapping if self._cur_rank not in process_mesh.processes: - rank_id = _get_corresponding_rank(dist_context, process_mesh, - self._cur_rank) + rank_id = _get_corresponding_rank( + dist_context, process_mesh, self._cur_rank + ) else: rank_id = self._cur_rank batch_size_axis = dims_mapping[0] if batch_size_axis > -1 and process_mesh.topology[batch_size_axis] > 1: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, - batch_size_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh.processes, + process_mesh.topology, + batch_size_axis, + rank_id, + ) return len(group_ranks), group_ranks.index(rank_id) return 1, 0 @@ -1382,10 +1513,12 @@ def _set_recompute_ckpts(self): # extract ckpts by specific model if isinstance(self._model, paddle.nn.Layer): - if hasattr(self._model, - "gpt") and self._model.__class__.__name__ in [ - 'GPTForPretraining', 'GPTForPretrainingAuto' - ]: + if hasattr( + self._model, "gpt" + ) and self._model.__class__.__name__ in [ + 'GPTForPretraining', + 'GPTForPretrainingAuto', + ]: exact_ckpts = self._model.gpt.checkpoints else: exact_ckpts = recompute.checkpoints @@ -1397,7 +1530,7 @@ def _set_recompute_ckpts(self): recompute.checkpoints = exact_ckpts[:] logs = { 'Model Class': self._model.__class__.__name__, - 'Applied Recompute ckpts': exact_ckpts + 'Applied Recompute ckpts': exact_ckpts, } self._logger.info(logs) @@ -1422,8 +1555,11 @@ def _switch_mode(self, mode): self._optimizer = self._dist_contexts[mode]._serial_optimizer def to_mode(self, mode): - assert mode in ["train", "eval", "predict"], \ - "mode {} should be one of ['train', 'eval', 'predict']".format(mode) + assert mode in [ + "train", + "eval", + "predict", + ], "mode {} should be one of ['train', 'eval', 'predict']".format(mode) self._mode = mode def _set_state_dict(self, mode, strict, state_dict, dist_attr): @@ -1484,20 +1620,24 @@ def save(self, path, training=True): serial_program = self._serial_main_progs[self._mode] dist_main_prog = self._dist_main_progs[self._mode][self._cur_rank] dist_context = self._dist_contexts[self._mode] - self._saver.save(path, - serial_program=serial_program, - dist_main_program=dist_main_prog, - dist_context=dist_context) + self._saver.save( + path, + serial_program=serial_program, + dist_main_program=dist_main_prog, + dist_context=dist_context, + ) else: assert "predict" in self._dist_main_progs feed_vars = self._feed_vars["predict"]['inputs'] fetch_vars = self._fetch_vars["predict"]['outputs'] dist_main_prog = self._dist_main_progs["predict"][self._cur_rank] - self._saver.save_inference_model(path, - feed_vars, - fetch_vars, - self._executor, - program=dist_main_prog) + self._saver.save_inference_model( + path, + feed_vars, + fetch_vars, + self._executor, + program=dist_main_prog, + ) def load(self, path, strict=True, load_optimizer=True): """ @@ -1547,7 +1687,8 @@ def load(self, path, strict=True, load_optimizer=True): """ self._strict = strict self._state_dict, self._dist_attr = self._saver.load( - path, load_optimizer) + path, load_optimizer + ) return self._state_dict, self._dist_attr def cost(self, inputs_spec=None, labels_spec=None, mode="train"): @@ -1577,14 +1718,18 @@ def cost(self, inputs_spec=None, labels_spec=None, mode="train"): # Check mode accepted_modes = ["train", "predict", "eval"] if mode not in accepted_modes: - raise ValueError("The mode {} is not in accepted modes {}".format( - mode, accepted_modes)) + raise ValueError( + "The mode {} is not in accepted modes {}".format( + mode, accepted_modes + ) + ) self.to_mode(mode) if inputs_spec is not None: self._inputs_spec, self._labels_spec = inputs_spec, labels_spec self._inputs, self._labels = self._prepare_data_tensor( - self._inputs_spec, self._labels_spec) + self._inputs_spec, self._labels_spec + ) self._build(mode) self._plan(mode) else: diff --git a/python/paddle/distributed/auto_parallel/graph.py b/python/paddle/distributed/auto_parallel/graph.py index de6505071abfe..d671d95aa9f4a 100644 --- a/python/paddle/distributed/auto_parallel/graph.py +++ b/python/paddle/distributed/auto_parallel/graph.py @@ -14,7 +14,6 @@ class Node: - def __init__(self, id, **attrs): # Each node must has a unique id self._id = id @@ -48,7 +47,6 @@ def __str__(self): class Edge: - def __init__(self, src_id, tgt_id, **attrs): # The id of source node in an Edge self._src_id = src_id @@ -85,12 +83,12 @@ def __contains__(self, attr_name): def __str__(self): str = "" str += "(src_id: {}, tgt_id: {}, attrs: {})".format( - self.src_id, self.tgt_id, self._attrs) + self.src_id, self.tgt_id, self._attrs + ) return str class Graph: - def __init__(self, **attrs): # _nodes is dict for storing the nodes of the graph. # The key of this dict is the node id. diff --git a/python/paddle/distributed/auto_parallel/helper.py b/python/paddle/distributed/auto_parallel/helper.py index 3173f51e249ab..f73668a2ba95c 100644 --- a/python/paddle/distributed/auto_parallel/helper.py +++ b/python/paddle/distributed/auto_parallel/helper.py @@ -20,7 +20,9 @@ from paddle.fluid.framework import Parameter from paddle.fluid.framework import program_guard from paddle.fluid.executor import global_scope -from paddle.fluid.dygraph.dygraph_to_static.program_translator import StaticFunction +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + StaticFunction, +) from .utils import to_list from .utils import get_logger @@ -174,7 +176,6 @@ def startup_program(self): class BuildInfo: - def __init__(self): self.clear() @@ -224,8 +225,9 @@ def build_program(self, mode): # skip if we has already built program. if self.build_info.has_cache(mode, True): self._logger.info( - "Already build program with mode = %s, use cached program." % - mode) + "Already build program with mode = %s, use cached program." + % mode + ) return self._logger.info("start to build program for mode = %s." % mode) @@ -249,21 +251,24 @@ def _build_startup_program(self): self.lazy_init = True return for param in self.concrete_program.parameters: - Parameter(name=param.name, - desc=param, - type=param.type, - shape=param.shape, - dtype=param.dtype, - stop_gradient=param.stop_gradient, - block=self.startup_program.global_block()) + Parameter( + name=param.name, + desc=param, + type=param.type, + shape=param.shape, + dtype=param.dtype, + stop_gradient=param.stop_gradient, + block=self.startup_program.global_block(), + ) def apply_optimizer(self, optimizer): """ Append backward and generate optimizer operations. """ self._verify_optimizer(optimizer) - self._logger.info("start to apply optimizer: %s ", - type(optimizer).__name__) + self._logger.info( + "start to apply optimizer: %s ", type(optimizer).__name__ + ) # clear optimizer parameters original_params = optimizer._parameter_list optimizer._parameter_list = None @@ -276,13 +281,17 @@ def apply_optimizer(self, optimizer): def _verify_optimizer(self, optimizer): assert optimizer is not None - assert hasattr(optimizer, - "minimize"), "Optimizer must have minimize() method." - assert self.proxy_layer.mode == 'train', "Required mode == 'train', but received '%s'" % self.proxy_layer.mode - assert len( - self.loss_vars - ) == 1, "Required len(loss_vars) == 1, but received len(loss_vars) = %s" % len( - self.loss_vars) + assert hasattr( + optimizer, "minimize" + ), "Optimizer must have minimize() method." + assert self.proxy_layer.mode == 'train', ( + "Required mode == 'train', but received '%s'" + % self.proxy_layer.mode + ) + assert len(self.loss_vars) == 1, ( + "Required len(loss_vars) == 1, but received len(loss_vars) = %s" + % len(self.loss_vars) + ) def to(self, mode): """ @@ -291,7 +300,8 @@ def to(self, mode): assert mode in ['train', 'eval', 'predict'] func = getattr(self.proxy_layer, '_' + mode) assert isinstance( - func, StaticFunction), "Please call build_program(mode) firstly." + func, StaticFunction + ), "Please call build_program(mode) firstly." self.proxy_layer.set_mode(mode) def static_func(self): @@ -299,7 +309,9 @@ def static_func(self): Return StaticFunction instance with underly target mode. """ assert self.proxy_layer.mode in [ - 'train', 'eval', 'predict' + 'train', + 'eval', + 'predict', ], "Please call build_program(mode) firstly." func_name = '_' + self.proxy_layer.mode return getattr(self.proxy_layer, func_name) @@ -317,13 +329,14 @@ def init(self, main_program, place, dist_context): dist_attr = { "dims_mapping": var_dist_attr.dims_mapping, "process_shape": var_dist_attr.process_mesh.topology, - "process_group": var_dist_attr.process_mesh.processes + "process_group": var_dist_attr.process_mesh.processes, } # slice param_value with dist_attr # share sliced_param_value with param_tensor in global_scope param_tensor = global_scope().var(param.name).get_tensor() sliced_param = Converter.slice_with_dist_attr( - param.numpy(), dist_attr) + param.numpy(), dist_attr + ) param_tensor.set(sliced_param, place) @property diff --git a/python/paddle/distributed/auto_parallel/interface.py b/python/paddle/distributed/auto_parallel/interface.py index 0abdc0ea76893..124f622d40f04 100644 --- a/python/paddle/distributed/auto_parallel/interface.py +++ b/python/paddle/distributed/auto_parallel/interface.py @@ -63,29 +63,39 @@ def shard_tensor(x, process_mesh=None, shard_spec=None): """ if process_mesh is not None: - assert isinstance(process_mesh, ProcessMesh), \ - "Argument process_mesh {} is not an instance of ProcessMesh".format(process_mesh) + assert isinstance( + process_mesh, ProcessMesh + ), "Argument process_mesh {} is not an instance of ProcessMesh".format( + process_mesh + ) else: process_mesh = get_current_process_mesh() - assert process_mesh is not None, \ - "Specify the process mesh argument or use ProcessMesh context manager first." - assert isinstance(shard_spec, list), \ - "Argument shard_spec {} is not an instance of list".format(shard_spec) + assert ( + process_mesh is not None + ), "Specify the process mesh argument or use ProcessMesh context manager first." + assert isinstance( + shard_spec, list + ), "Argument shard_spec {} is not an instance of list".format(shard_spec) dist_tensor = DistributedTensor(x) serial_tensor = dist_tensor.serial_tensor dist_tensor.dist_attr.process_mesh = process_mesh - if serial_tensor.type == core.VarDesc.VarType.READER \ - or serial_tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - or serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES: + if ( + serial_tensor.type == core.VarDesc.VarType.READER + or serial_tensor.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + or serial_tensor.type == core.VarDesc.VarType.STEP_SCOPES + ): tensor_shape = [] else: tensor_shape = serial_tensor.shape if shard_spec is not None: - assert verify_shard_spec(shard_spec, tensor_shape, process_mesh), \ - "For tensor {}, shard_spec {} is invalid with tensor_shape {} and process_mesh {}.".format( - serial_tensor.name, shard_spec, tensor_shape, process_mesh) + assert verify_shard_spec( + shard_spec, tensor_shape, process_mesh + ), "For tensor {}, shard_spec {} is invalid with tensor_shape {} and process_mesh {}.".format( + serial_tensor.name, shard_spec, tensor_shape, process_mesh + ) dist_tensor.dist_attr.dims_mapping = convert_to_dims_mapping( - shard_spec, process_mesh) + shard_spec, process_mesh + ) if process_mesh is not None: dist_tensor.dist_attr.mark_annotated("process_mesh") if shard_spec is not None: @@ -140,41 +150,54 @@ def shard_op(op, process_mesh=None, in_shard_specs=None, out_shard_specs=None): """ if process_mesh is not None: - assert isinstance(process_mesh, ProcessMesh), \ - "Argument process_mesh {} is not an instance of ProcessMesh".format(process_mesh) + assert isinstance( + process_mesh, ProcessMesh + ), "Argument process_mesh {} is not an instance of ProcessMesh".format( + process_mesh + ) else: process_mesh = get_current_process_mesh() - assert process_mesh is not None, \ - "Specify the process mesh argument or use ProcessMesh context manager first." + assert ( + process_mesh is not None + ), "Specify the process mesh argument or use ProcessMesh context manager first." in_dims_mappings = [] if in_shard_specs is not None: - assert all((isinstance(shard_spec, list) or shard_spec is None) for shard_spec in in_shard_specs), \ - "in_shard_spec {} is not a list of list or None".format(in_shard_specs) + assert all( + (isinstance(shard_spec, list) or shard_spec is None) + for shard_spec in in_shard_specs + ), "in_shard_spec {} is not a list of list or None".format( + in_shard_specs + ) for shard_spec in in_shard_specs: if shard_spec is not None: in_dims_mappings.append( - convert_to_dims_mapping(shard_spec, process_mesh)) + convert_to_dims_mapping(shard_spec, process_mesh) + ) else: in_dims_mappings.append(None) out_dims_mappings = [] if out_shard_specs is not None: - assert all((isinstance(shard_spec, list) or shard_spec is None) for shard_spec in out_shard_specs), \ - "out_shard_spec {} is not a list of list or None".format(out_shard_specs) + assert all( + (isinstance(shard_spec, list) or shard_spec is None) + for shard_spec in out_shard_specs + ), "out_shard_spec {} is not a list of list or None".format( + out_shard_specs + ) for shard_spec in out_shard_specs: if shard_spec is not None: out_dims_mappings.append( - convert_to_dims_mapping(shard_spec, process_mesh)) + convert_to_dims_mapping(shard_spec, process_mesh) + ) else: out_dims_mappings.append(None) - op = DistributedOperatorHelper(op, process_mesh, in_dims_mappings, - out_dims_mappings) + op = DistributedOperatorHelper( + op, process_mesh, in_dims_mappings, out_dims_mappings + ) return op def recompute(op): - class RecomputeOperator: - def __init__(self, op): self._op = op @@ -215,11 +238,13 @@ def add_to_collection(collection_name, value, name=None): _g_collections[collection_name] = [] if name is not None: for _, v in _g_collections[collection_name]: - if v == value: return + if v == value: + return _g_collections[collection_name].append((name, value)) else: for _, v in _g_collections[collection_name]: - if v == value: return + if v == value: + return _g_collections[collection_name].append((None, value)) diff --git a/python/paddle/distributed/auto_parallel/mapper.py b/python/paddle/distributed/auto_parallel/mapper.py index f8c0792c580f0..d7b599aaa1c44 100644 --- a/python/paddle/distributed/auto_parallel/mapper.py +++ b/python/paddle/distributed/auto_parallel/mapper.py @@ -24,9 +24,16 @@ def is_collective_comm_op(op): comm_list = [ - "c_allreduce_sum", "c_allreduce_min", "c_allreduce_max", - "c_allreduce_prod", "c_reduce_sum", "c_reduce_min", "c_reduce_max", - "c_reduce_prod", "c_broadcast", "c_allgather" + "c_allreduce_sum", + "c_allreduce_min", + "c_allreduce_max", + "c_allreduce_prod", + "c_reduce_sum", + "c_reduce_min", + "c_reduce_max", + "c_reduce_prod", + "c_broadcast", + "c_allgather", ] if op.type in comm_list: return True @@ -127,7 +134,8 @@ def analyze_comm_requirements_from_op(op, rank, g_process_group_map): if comm_volume is not None: comm_requirements_to_ranks[tgt_rank] = {} comm_requirements_to_ranks[tgt_rank][ - "comm_volume"] = comm_volume + "comm_volume" + ] = comm_volume elif is_p2p_comm_op(op): tgt_rank = op.attr("peer") comm_volume = get_comm_volume(op, rank, tgt_rank) @@ -149,28 +157,33 @@ def analyze_requirements_for_program(src_info, rank): for block in program.blocks: for op in block.ops: cur_comm_requirements_to_ranks = analyze_comm_requirements_from_op( - op, rank, g_process_group_map) + op, rank, g_process_group_map + ) for tgt_rank, link_info in cur_comm_requirements_to_ranks.items(): if tgt_rank in comm_requirements_to_ranks: comm_requirements_to_ranks[tgt_rank][ - "comm_volume"] += link_info["comm_volume"] + "comm_volume" + ] += link_info["comm_volume"] else: comm_requirements_to_ranks[tgt_rank] = {} comm_requirements_to_ranks[tgt_rank][ - "comm_volume"] = link_info["comm_volume"] + "comm_volume" + ] = link_info["comm_volume"] return resource_requirements, comm_requirements_to_ranks def build_process_graph(distributed_program): graph = Graph() for src_rank, src_info in distributed_program.items(): - resource_requirements, comm_requirements_to_ranks = analyze_requirements_for_program( - src_info, src_rank) + ( + resource_requirements, + comm_requirements_to_ranks, + ) = analyze_requirements_for_program(src_info, src_rank) graph.add_node(src_rank, resource_requirements=resource_requirements) for tgt_rank, comm_requirements in comm_requirements_to_ranks.items(): - graph.add_edge(src_rank, - tgt_rank, - comm_requirements=comm_requirements) + graph.add_edge( + src_rank, tgt_rank, comm_requirements=comm_requirements + ) return graph @@ -185,14 +198,17 @@ def build_cluster_graph(cluster): for machine in cluster.machines.values(): for device in machine.devices.values(): graph.add_node(device.global_id, device=device) - if cuda_visible_devices and device.local_id not in cuda_visible_devices: + if ( + cuda_visible_devices + and device.local_id not in cuda_visible_devices + ): graph.nodes[device.global_id]["occupied"] = True else: graph.nodes[device.global_id]["occupied"] = False for link in machine.links.values(): - graph.add_edge(link.source.global_id, - link.target.global_id, - link=link) + graph.add_edge( + link.source.global_id, link.target.global_id, link=link + ) return graph @@ -222,7 +238,8 @@ def select_unvisited_rank_node(rank_node_list): queue = deque() root_rank_node = select_unvisited_rank_node( - list(process_graph.nodes.values())) + list(process_graph.nodes.values()) + ) while root_rank_node is not None: queue.append(root_rank_node) while queue: @@ -232,48 +249,61 @@ def select_unvisited_rank_node(rank_node_list): device_type = cur_rank_node["resource_requirements"]["device_type"] cur_device_node = None for device_node in cluster_graph.nodes.values(): - if (device_node["device"].type - == device_type) and (not device_node["occupied"]): + if (device_node["device"].type == device_type) and ( + not device_node["occupied"] + ): device_node["occupied"] = True cur_rank_node["visited"] = True cur_rank_node["device"] = device_node["device"] cur_device_node = device_node break - assert cur_device_node, "Cannot find a device to satisfy the requirement." + assert ( + cur_device_node + ), "Cannot find a device to satisfy the requirement." nbr_rank_edges = [] for nbr_rank_node_id, nbr_rank_edge in process_graph.adjs[ - cur_rank_node.id].items(): - assert nbr_rank_edge.src_id == cur_rank_node.id and nbr_rank_edge.tgt_id == nbr_rank_node_id + cur_rank_node.id + ].items(): + assert ( + nbr_rank_edge.src_id == cur_rank_node.id + and nbr_rank_edge.tgt_id == nbr_rank_node_id + ) queue.append(process_graph.nodes[nbr_rank_node_id]) nbr_rank_edges.append(nbr_rank_edge) nbr_rank_edges.sort(key=sort_by_comm_volume) nbr_device_edges = [] for nbr_device_edge in cluster_graph.adjs[ - cur_device_node.id].values(): + cur_device_node.id + ].values(): nbr_device_edges.append(nbr_device_edge) nbr_device_edges.sort(key=sort_by_comm_bandwidth) for nbr_rank_edge in nbr_rank_edges: - src_rank_node = process_graph.nodes[ - nbr_rank_edge.src_id]["visited"] + src_rank_node = process_graph.nodes[nbr_rank_edge.src_id][ + "visited" + ] if src_rank_node: continue device_type = src_rank_node["resource_requirements"][ - "device_type"] + "device_type" + ] nbr_rank_node = process_graph.nodes[nbr_rank_edge.tgt_id] for nbr_device_edge in nbr_device_edges: nbr_device_node = cluster_graph.nodes[ - nbr_device_edge.tgt_id] + nbr_device_edge.tgt_id + ] if (nbr_device_node["device"].type == device_type) and ( - not nbr_device_node["occupied"]): + not nbr_device_node["occupied"] + ): nbr_device_node["occupied"] = True nbr_rank_node["visited"] = True nbr_rank_node["device"] = nbr_device_node["device"] break root_rank_node = select_unvisited_rank_node( - list(process_graph.nodes.values())) + list(process_graph.nodes.values()) + ) rank_mapping = {} for rank, rank_node in process_graph.nodes.items(): diff --git a/python/paddle/distributed/auto_parallel/operators/common.py b/python/paddle/distributed/auto_parallel/operators/common.py index 7bd51d3f98671..af9c53a88ea86 100644 --- a/python/paddle/distributed/auto_parallel/operators/common.py +++ b/python/paddle/distributed/auto_parallel/operators/common.py @@ -21,16 +21,22 @@ _g_distributed_operator_impl_containers = {} _g_elementwise_ops = [ - "elementwise", "gelu", "dropout", "cast", "gather", "concat", - "fused_softmax_mask_upper_triangle" + "elementwise", + "gelu", + "dropout", + "cast", + "gather", + "concat", + "fused_softmax_mask_upper_triangle", ] BACKWARD_ONLY_DIST_OPS = {'check_finite_and_unscale', 'update_loss_scaling'} -class ParallelMode(): +class ParallelMode: """ the parallel mode for communication or auxiliary operator """ + DataParallel = "auto_parallel/data_parallel" ModelParallel = "auto_parallel/model_parallel" PipelineParalel = "auto_parallel/pipeline_paralel" @@ -46,7 +52,6 @@ def is_elementwise_op(op_type): class DistributedOperatorImplContainer: - def __init__(self, op_type): self._type = op_type self._impls = [] @@ -64,8 +69,9 @@ def impls(self): return self._impls def register_impl(self, dist_impl): - assert self.type == dist_impl.type, \ - "Op type of container must be same as that of the implementation." + assert ( + self.type == dist_impl.type + ), "Op type of container must be same as that of the implementation." impl_idx = len(self.impls) dist_impl.idx = impl_idx self._impls.append(dist_impl) @@ -96,7 +102,6 @@ def get_compatible_impls(self, dist_op): class DistributedOperatorImpl(abc.ABC): - def __init__(self, name): self._name = name self._type = None @@ -181,54 +186,69 @@ def find_compatible_distributed_operator_impls(dist_op, fwd=True, partial=True): op_type = dist_op.serial_op.type dist_op_impl_container = get_distributed_operator_impl_container(op_type) dist_op_eltwise_impl_container = get_distributed_operator_impl_container( - "elementwise") + "elementwise" + ) dist_op_default_impl_container = get_distributed_operator_impl_container( - "default") + "default" + ) compatible_impls = [] if partial: if fwd: # First, find impls in the corresponding container if dist_op_impl_container: compatible_impls.extend( - dist_op_impl_container.get_input_compatible_impls(dist_op)) + dist_op_impl_container.get_input_compatible_impls(dist_op) + ) # Second, find impls in the elementwise container if dist_op_eltwise_impl_container and is_elementwise_op(op_type): compatible_impls.extend( dist_op_eltwise_impl_container.get_input_compatible_impls( - dist_op)) + dist_op + ) + ) # Third, find impls in the default container if dist_op_default_impl_container: compatible_impls.extend( dist_op_default_impl_container.get_input_compatible_impls( - dist_op)) + dist_op + ) + ) else: # First, find impls in the corresponding container if dist_op_impl_container: compatible_impls.extend( - dist_op_impl_container.get_output_compatible_impls(dist_op)) + dist_op_impl_container.get_output_compatible_impls(dist_op) + ) # Second, find impls in the elementwise container if dist_op_eltwise_impl_container and is_elementwise_op(op_type): compatible_impls.extend( dist_op_eltwise_impl_container.get_output_compatible_impls( - dist_op)) + dist_op + ) + ) # Third, find impls in the default container if dist_op_default_impl_container: compatible_impls.extend( dist_op_default_impl_container.get_output_compatible_impls( - dist_op)) + dist_op + ) + ) else: # First, find impls in the corresponding container if dist_op_impl_container: compatible_impls.extend( - dist_op_impl_container.get_compatible_impls(dist_op)) + dist_op_impl_container.get_compatible_impls(dist_op) + ) # Second, find impls in the elementwise container if dist_op_eltwise_impl_container and is_elementwise_op(op_type): compatible_impls.extend( - dist_op_eltwise_impl_container.get_compatible_impls(dist_op)) + dist_op_eltwise_impl_container.get_compatible_impls(dist_op) + ) # Third, find impls in the default container if dist_op_default_impl_container: compatible_impls.extend( - dist_op_default_impl_container.get_compatible_impls(dist_op)) + dist_op_default_impl_container.get_compatible_impls(dist_op) + ) if compatible_impls: # For now, just return the first compatible impl @@ -241,11 +261,11 @@ def find_compatible_distributed_operator_impls(dist_op, fwd=True, partial=True): def is_parameter_related(varname, block): if ".subprog_" in varname: - varname = varname[:varname.index(".subprog_")] + varname = varname[: varname.index(".subprog_")] if ".cast_fp" in varname: - varname = varname[:varname.index(".cast_fp")] + varname = varname[: varname.index(".cast_fp")] if ".quantized" in varname: - varname = varname[:varname.index(".quantized")] + varname = varname[: varname.index(".quantized")] assert block.has_var(varname) var = block.var(varname) return var.is_parameter @@ -277,8 +297,9 @@ def infer_shape(block, src_var, src_var_dist_attr, op_input_dist_attr): return exact_shape -def set_comm_op_dist_attr_for_program(new_op, process_mesh, tensor_dist_attr, - ctx): +def set_comm_op_dist_attr_for_program( + new_op, process_mesh, tensor_dist_attr, ctx +): assert process_mesh is not None assert tensor_dist_attr is not None @@ -303,9 +324,11 @@ def naive_copy_op_dist_attr_for_program(new_op, ref_op, ctx): assert len(new_op.input(input_name)) == 1 ref_tensor_dist_attr = ref_dist_attr.get_input_dist_attr( - ref_op.input(input_name)[0]) + ref_op.input(input_name)[0] + ) new_op_dist_attr.set_input_dist_attr( - new_op.input(input_name)[0], ref_tensor_dist_attr) + new_op.input(input_name)[0], ref_tensor_dist_attr + ) for output_name in ref_op.output_names: assert output_name in new_op.output_names @@ -313,9 +336,11 @@ def naive_copy_op_dist_attr_for_program(new_op, ref_op, ctx): assert len(new_op.output(output_name)) == 1 ref_tensor_dist_attr = ref_dist_attr.get_output_dist_attr( - ref_op.output(output_name)[0]) + ref_op.output(output_name)[0] + ) new_op_dist_attr.set_output_dist_attr( - new_op.output(output_name)[0], ref_tensor_dist_attr) + new_op.output(output_name)[0], ref_tensor_dist_attr + ) ctx.set_op_dist_attr_for_program(new_op, new_op_dist_attr) @@ -348,9 +373,12 @@ def get_data_parallel_group(dist_ctx, op, act_grad_names, rank): batch_size_axis = var_dim_mapping[0] if len(var_dim_mapping) > 0 else -1 if batch_size_axis > -1 and mesh_shape[batch_size_axis] > 1: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, - batch_size_axis, rank) + group_ranks = _get_comm_group( + process_mesh.processes, + process_mesh.topology, + batch_size_axis, + rank, + ) dp_group = new_process_group(group_ranks) break @@ -377,33 +405,39 @@ def sync_and_scale_gradients(dist_ctx, op, dp_group, allreduce_var_names): for var_name in allreduce_var_names: added_ops = [] grad_var = main_block.var(var_name) - allreduce_op = main_block.append_op(type='c_allreduce_sum', - inputs={'X': [grad_var]}, - outputs={'Out': [grad_var]}, - attrs={ - 'ring_id': dp_group.id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Backward - }) - allreduce_op._set_attr('op_namescope', - str('/') + ParallelMode.DataParallel) + allreduce_op = main_block.append_op( + type='c_allreduce_sum', + inputs={'X': [grad_var]}, + outputs={'Out': [grad_var]}, + attrs={ + 'ring_id': dp_group.id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Backward, + }, + ) + allreduce_op._set_attr( + 'op_namescope', str('/') + ParallelMode.DataParallel + ) added_ops.append(allreduce_op) if dist_ctx.gradient_scale: - scale_op = main_block.append_op(type='scale', - inputs={'X': grad_var}, - outputs={'Out': grad_var}, - attrs={ - 'scale': 1.0 / dp_degree, - OP_ROLE_KEY: OpRole.Backward - }) - scale_op._set_attr('op_namescope', - str('/') + ParallelMode.DataParallel) + scale_op = main_block.append_op( + type='scale', + inputs={'X': grad_var}, + outputs={'Out': grad_var}, + attrs={'scale': 1.0 / dp_degree, OP_ROLE_KEY: OpRole.Backward}, + ) + scale_op._set_attr( + 'op_namescope', str('/') + ParallelMode.DataParallel + ) added_ops.append(scale_op) dims_mapping = op_dist_attr.get_output_dims_mapping(grad_var.name) - assert dims_mapping is not None, "Unexception: dims_mapping of output [{}] of op [{}] is None".format( - grad_var.name, op_dist_attr.op_type) + assert ( + dims_mapping is not None + ), "Unexception: dims_mapping of output [{}] of op [{}] is None".format( + grad_var.name, op_dist_attr.op_type + ) # NOTE auxiliary op's dist attr should follow dist_op not dist_tensor for new_op in added_ops: new_op_attr = OperatorDistributedAttribute() @@ -413,8 +447,9 @@ def sync_and_scale_gradients(dist_ctx, op, dp_group, allreduce_var_names): dist_ctx.set_op_dist_attr_for_program(new_op, new_op_attr) -def gradient_synchronization(dist_ctx, op, act_grad_names, out_grad_names, - rank): +def gradient_synchronization( + dist_ctx, op, act_grad_names, out_grad_names, rank +): """ conduct the allreudce and scaling（dp size）for gradients of model parameters for operator in data parallelism. @@ -430,8 +465,11 @@ def gradient_synchronization(dist_ctx, op, act_grad_names, out_grad_names, if not is_in_backward_phase(dist_ctx): return - if is_optimize_op(op) or len(act_grad_names) == 0 or len( - out_grad_names) == 0: + if ( + is_optimize_op(op) + or len(act_grad_names) == 0 + or len(out_grad_names) == 0 + ): return dp_group = get_data_parallel_group(dist_ctx, op, act_grad_names, rank) @@ -443,13 +481,19 @@ def gradient_synchronization(dist_ctx, op, act_grad_names, out_grad_names, def is_data_parallel_scale_op(op): - return op.type == "scale" and op.desc.has_attr("op_namescope") \ - and ParallelMode.DataParallel in op.desc.attr("op_namescope") + return ( + op.type == "scale" + and op.desc.has_attr("op_namescope") + and ParallelMode.DataParallel in op.desc.attr("op_namescope") + ) def is_data_parallel_reduce_op(op): - return op.type in ["c_reduce_sum", "c_allreduce_sum"] and op.desc.has_attr("op_namescope") \ - and ParallelMode.DataParallel in op.desc.attr("op_namescope") + return ( + op.type in ["c_reduce_sum", "c_allreduce_sum"] + and op.desc.has_attr("op_namescope") + and ParallelMode.DataParallel in op.desc.attr("op_namescope") + ) def is_in_backward_phase(dist_ctx): diff --git a/python/paddle/distributed/auto_parallel/operators/dist_assign.py b/python/paddle/distributed/auto_parallel/operators/dist_assign.py index 96923f461a73d..15382977aabb7 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_assign.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_assign.py @@ -21,7 +21,6 @@ class DistributedAssign(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedAssign, self).__init__(op_type) @@ -30,7 +29,6 @@ def __init__(self, op_type): class DistributedAssignImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedAssignImpl, self).__init__(name) self._forward_implemented = True @@ -43,8 +41,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -70,7 +69,8 @@ def update_dims_mapping(self, dist_op): for i in range(len(x_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True diff --git a/python/paddle/distributed/auto_parallel/operators/dist_check_finite_and_unscale.py b/python/paddle/distributed/auto_parallel/operators/dist_check_finite_and_unscale.py index 72a4eda103dbd..c99cfe21f7279 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_check_finite_and_unscale.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_check_finite_and_unscale.py @@ -22,23 +22,24 @@ from ..utils import set_dist_op_desc_original_id from ..process_group import new_process_group from ..dist_attribute import OperatorDistributedAttribute -from paddle.distributed.auto_parallel.process_group import get_world_process_group +from paddle.distributed.auto_parallel.process_group import ( + get_world_process_group, +) world_process_group = get_world_process_group() class DistributedCheckFiniteAndUnscale(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedCheckFiniteAndUnscale, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedCheckFiniteAndUnscale("check_finite_and_unscale")) + DistributedCheckFiniteAndUnscale("check_finite_and_unscale") +) class DistributedCheckFiniteAndUnscaleImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedCheckFiniteAndUnscaleImpl, self).__init__(name) self._name = name @@ -80,8 +81,11 @@ def backward(ctx, *args, **kwargs): backward_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id dist_attr = ctx.get_op_dist_attr_for_program(backward_op) - assert dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(backward_op)) + assert ( + dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format( + str(backward_op) + ) assert rank_id in dist_attr.process_mesh.processes @@ -89,25 +93,33 @@ def backward(ctx, *args, **kwargs): assert 'Scale' in kwargs, "input [{}] is not given".format('Scale') assert 'Out' in kwargs, "input [{}] is not given".format('Out') assert 'FoundInfinite' in kwargs, "output [{}] is not given".format( - 'FoundInfinite') + 'FoundInfinite' + ) - assert len( + assert ( + len(kwargs['Scale']) == 1 + ), "check_finite_and_unscale input Scale take 1 variable but got {}".format( kwargs['Scale'] - ) == 1, "check_finite_and_unscale input Scale take 1 variable but got {}".format( - kwargs['Scale']) - assert len( + ) + assert ( + len(kwargs['FoundInfinite']) == 1 + ), "check_finite_and_unscale input FoundInfinite take 1 variable but got {}".format( kwargs['FoundInfinite'] - ) == 1, "check_finite_and_unscale input FoundInfinite take 1 variable but got {}".format( - kwargs['FoundInfinite']) + ) assert len(kwargs['X']) == len( kwargs['Out'] ), "check_finite_and_unscale got [{}] X and [{}] Out, which are supposed to be equal".format( - len(kwargs['X']), len(kwargs['Out'])) + len(kwargs['X']), len(kwargs['Out']) + ) filter_vars = [] for varname in kwargs['X']: - if rank_id in ctx.get_tensor_dist_attr_for_program( - main_block.var(varname)).process_mesh.processes: + if ( + rank_id + in ctx.get_tensor_dist_attr_for_program( + main_block.var(varname) + ).process_mesh.processes + ): filter_vars.append(varname) # replicate op in dist program @@ -121,55 +133,70 @@ def backward(ctx, *args, **kwargs): group = new_process_group(world_process_group.ranks) inf_var = main_block.var(kwargs['FoundInfinite'][0]) - inf_var_int32 = main_block.create_var(name=inf_var.name + "@cast_int32", - shape=inf_var.shape, - dtype=core.VarDesc.VarType.INT32) + inf_var_int32 = main_block.create_var( + name=inf_var.name + "@cast_int32", + shape=inf_var.shape, + dtype=core.VarDesc.VarType.INT32, + ) set_var_dist_attr( - ctx, inf_var_int32, + ctx, + inf_var_int32, ctx.get_tensor_dist_attr_for_program(inf_var).dims_mapping, - ctx.get_tensor_dist_attr_for_program(inf_var).process_mesh) - cast_op1 = main_block.append_op(type='cast', - inputs={'X': inf_var}, - outputs={'Out': inf_var_int32}, - attrs={ - "in_dtype": inf_var.dtype, - "out_dtype": inf_var_int32.dtype, - OP_ROLE_KEY: OpRole.Optimize - }) - allreduce_op = main_block.append_op(type='c_allreduce_max', - inputs={'X': inf_var_int32}, - outputs={'Out': inf_var_int32}, - attrs={ - 'ring_id': group.id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Optimize - }) - cast_op2 = main_block.append_op(type='cast', - inputs={'X': inf_var_int32}, - outputs={'Out': inf_var}, - attrs={ - "in_dtype": inf_var_int32.dtype, - "out_dtype": inf_var.dtype, - OP_ROLE_KEY: OpRole.Optimize - }) + ctx.get_tensor_dist_attr_for_program(inf_var).process_mesh, + ) + cast_op1 = main_block.append_op( + type='cast', + inputs={'X': inf_var}, + outputs={'Out': inf_var_int32}, + attrs={ + "in_dtype": inf_var.dtype, + "out_dtype": inf_var_int32.dtype, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) + allreduce_op = main_block.append_op( + type='c_allreduce_max', + inputs={'X': inf_var_int32}, + outputs={'Out': inf_var_int32}, + attrs={ + 'ring_id': group.id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) + cast_op2 = main_block.append_op( + type='cast', + inputs={'X': inf_var_int32}, + outputs={'Out': inf_var}, + attrs={ + "in_dtype": inf_var_int32.dtype, + "out_dtype": inf_var.dtype, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) for op in [cast_op1, allreduce_op, cast_op2]: new_op_dist_attr = OperatorDistributedAttribute() for varname in op.input_arg_names: var_dist_attr = ctx.get_tensor_dist_attr_for_program( - main_block.var(varname)) + main_block.var(varname) + ) assert var_dist_attr is not None new_op_dist_attr.set_input_dims_mapping( - varname, var_dist_attr.dims_mapping) + varname, var_dist_attr.dims_mapping + ) for varname in op.output_arg_names: var_dist_attr = ctx.get_tensor_dist_attr_for_program( - main_block.var(varname)) + main_block.var(varname) + ) new_op_dist_attr.set_output_dims_mapping( - varname, var_dist_attr.dims_mapping) + varname, var_dist_attr.dims_mapping + ) new_op_dist_attr.process_mesh = var_dist_attr.process_mesh ctx.set_op_dist_attr_for_program(op, new_op_dist_attr) register_distributed_operator_impl( "check_finite_and_unscale", - DistributedCheckFiniteAndUnscaleImpl("check_finite_and_unscale")) + DistributedCheckFiniteAndUnscaleImpl("check_finite_and_unscale"), +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_default.py b/python/paddle/distributed/auto_parallel/operators/dist_default.py index 9c1e8b0487180..36b9b8b2ee45a 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_default.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_default.py @@ -38,35 +38,41 @@ def prim_operator_data_parallel_functor(ctx, src_op): var_name = src_op.output_arg_names[0] if var_name in ctx.grads_params: - assert var_name not in ctx.synced_gradient, "in primtive mode, grad is already {} synced".format( - var_name) + assert ( + var_name not in ctx.synced_gradient + ), "in primtive mode, grad is already {} synced".format(var_name) ctx.synced_gradient.add(var_name) sync_group = new_process_group(ctx.data_parallel_group) - allreduce_op = main_block.append_op(type='c_allreduce_sum', - inputs={'X': [var_name]}, - outputs={'Out': [var_name]}, - attrs={ - 'ring_id': sync_group.id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Backward - }) + allreduce_op = main_block.append_op( + type='c_allreduce_sum', + inputs={'X': [var_name]}, + outputs={'Out': [var_name]}, + attrs={ + 'ring_id': sync_group.id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Backward, + }, + ) param = ctx.grads_params[var_name] startup_block = dist_op_context.startup_block - new_op = startup_block.append_op(type='c_broadcast', - inputs={'X': [param]}, - outputs={'Out': [param]}, - attrs={ - 'ring_id': sync_group.id, - 'root': 0, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward - }) + new_op = startup_block.append_op( + type='c_broadcast', + inputs={'X': [param]}, + outputs={'Out': [param]}, + attrs={ + 'ring_id': sync_group.id, + 'root': 0, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) grad_var = main_block.var(var_name) dims_mapping = ctx.get_tensor_dist_attr_for_program( - grad_var).dims_mapping + grad_var + ).dims_mapping dist_attr = ctx.get_op_dist_attr_for_program(src_op) process_mesh = dist_attr.process_mesh op_attr = OperatorDistributedAttribute() @@ -79,7 +85,6 @@ def prim_operator_data_parallel_functor(ctx, src_op): class DistributedDefault(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedDefault, self).__init__(op_type) @@ -89,7 +94,6 @@ def __init__(self, op_type): # Replicated Default class DistributedDefaultImpl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedDefaultImpl0, self).__init__(name) self._forward_implemented = True @@ -107,13 +111,14 @@ def calc_cost(self, op_role, dist_op, ctx, cluster): def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes op_type = dist_op.serial_op.type - cost_mapping = build_comp_costs_from_descs(_g_op_cost_factory[op_type], - ctx, processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + _g_op_cost_factory[op_type], ctx, processes, desc_mapping, cluster + ) res_cost = [cost_mapping] return res_cost @@ -121,16 +126,17 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): def calc_bwd_cost(self, dist_op, ctx, cluster): # calc comp op cost res = [] - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) dist_attr = dist_op.dist_attr process_mesh = dist_attr.process_mesh processes = process_mesh.processes backward_op = dist_op.serial_op op_type = backward_op.type - cost_mapping = build_comp_costs_from_descs(_g_op_cost_factory[op_type], - ctx, processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + _g_op_cost_factory[op_type], ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) main_block = backward_op.block @@ -139,7 +145,8 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and not is_parameter_related( - varname, main_block): + varname, main_block + ): var_dim_mapping = dist_attr.get_input_dims_mapping(varname) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] @@ -151,16 +158,25 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and is_parameter_related( - varname, main_block): + varname, main_block + ): var_dim_mapping = dist_attr.get_input_dims_mapping( - varname) + varname + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [varname + "@GRAD"] - build_dp_costs(res, dist_op, ctx, var_names, attrs, - parallel_axis, cluster) + build_dp_costs( + res, + dist_op, + ctx, + var_names, + attrs, + parallel_axis, + cluster, + ) return res def is_input_compatible(self, dist_op): @@ -304,8 +320,10 @@ def is_auto_compatible(self, dist_op): batch_dim_mappings.append(dims_mapping[1]) # Check batch dim mapping compatibility - if not all(batch_dim_mappings[0] == dim_mapping - for dim_mapping in batch_dim_mappings): + if not all( + batch_dim_mappings[0] == dim_mapping + for dim_mapping in batch_dim_mappings + ): return False return True @@ -342,7 +360,8 @@ def update_dims_mapping(self, dist_op): for arg_name in op_desc.output_arg_names(): if op_desc.type() == 'fill_any_like': input_tensor = dist_op.get_serial_input( - op_desc.input_arg_names()[0]) + op_desc.input_arg_names()[0] + ) if input_tensor.is_parameter: continue serial_tensor = dist_op.get_serial_output(arg_name) @@ -359,7 +378,8 @@ def update_dims_mapping(self, dist_op): return changed compatible_dim_mapping = compute_compatible_dim_mapping( - batch_dim_mappings) + batch_dim_mappings + ) if compatible_dim_mapping is None: return False @@ -369,19 +389,24 @@ def update_dims_mapping(self, dist_op): continue dims_mapping = op_dist_attr.get_input_dims_mapping(arg_name) if arg_name not in input_xshape_arg_names: - if len(dims_mapping) >= 1 and \ - compatible_dim_mapping != dims_mapping[0]: + if ( + len(dims_mapping) >= 1 + and compatible_dim_mapping != dims_mapping[0] + ): dims_mapping[0] = compatible_dim_mapping changed = True else: - if len(dims_mapping) >= 2 and \ - compatible_dim_mapping != dims_mapping[1]: + if ( + len(dims_mapping) >= 2 + and compatible_dim_mapping != dims_mapping[1] + ): dims_mapping[1] = compatible_dim_mapping changed = True for arg_name in op_desc.output_arg_names(): if op_desc.type() == 'fill_any_like': input_tensor = dist_op.get_serial_input( - op_desc.input_arg_names()[0]) + op_desc.input_arg_names()[0] + ) if input_tensor.is_parameter: continue if op_desc.type() in ["shape", "slice"]: @@ -391,13 +416,17 @@ def update_dims_mapping(self, dist_op): continue dims_mapping = op_dist_attr.get_output_dims_mapping(arg_name) if arg_name not in output_xshape_arg_names: - if len(dims_mapping - ) >= 1 and compatible_dim_mapping != dims_mapping[0]: + if ( + len(dims_mapping) >= 1 + and compatible_dim_mapping != dims_mapping[0] + ): dims_mapping[0] = compatible_dim_mapping changed = True else: - if len(dims_mapping - ) >= 2 and compatible_dim_mapping != dims_mapping[1]: + if ( + len(dims_mapping) >= 2 + and compatible_dim_mapping != dims_mapping[1] + ): dims_mapping[1] = compatible_dim_mapping changed = True @@ -414,17 +443,20 @@ def forward(ctx, *args, **kwargs): # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) # replicate op in dist program dist_op_desc = main_block.append_op(type='nop').desc @@ -437,6 +469,7 @@ def forward(ctx, *args, **kwargs): # data parallel synchronization for primtive operators from paddle.incubate.autograd import prim_enabled + if prim_enabled(): assert is_prim_op(src_op) prim_operator_data_parallel_functor(ctx, src_op) @@ -447,9 +480,11 @@ def forward(ctx, *args, **kwargs): return for varname in dist_op_desc.input_arg_names(): - if startup_block.has_var(varname) and startup_block.var( - varname - ).is_parameter and varname not in dist_op_context.already_init_sync_vars: + if ( + startup_block.has_var(varname) + and startup_block.var(varname).is_parameter + and varname not in dist_op_context.already_init_sync_vars + ): dist_op_context.already_init_sync_vars.add(varname) param = startup_block.var(varname) param_dist_attr = ctx.get_tensor_dist_attr_for_program(param) @@ -458,38 +493,41 @@ def forward(ctx, *args, **kwargs): # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, process_mesh, rank_id + ) # NOTE all not splited axis should be presented in mesh for axis, size in enumerate(process_mesh.topology): if size <= 1 or axis in dims_mapping: pass else: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, - axis, rank_id) + group_ranks = _get_comm_group( + process_mesh.processes, + process_mesh.topology, + axis, + rank_id, + ) sync_group = new_process_group(group_ranks) - new_op = startup_block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': - sync_group.id, - 'root': - 0, - 'use_calc_stream': - True, - OP_ROLE_KEY: - OpRole.Forward - }) + new_op = startup_block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': sync_group.id, + 'root': 0, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) # set distributed attribute op_attr = OperatorDistributedAttribute() op_attr.process_mesh = process_mesh - op_attr.set_output_dims_mapping(param.name, - dims_mapping) + op_attr.set_output_dims_mapping( + param.name, dims_mapping + ) op_attr.set_input_dims_mapping(param.name, dims_mapping) ctx.set_op_dist_attr_for_program(new_op, op_attr) @@ -501,24 +539,30 @@ def backward(ctx, *args, **kwargs): main_block = dist_op_context.work_block backward_op = dist_op_context.cur_src_op dist_attr = ctx.get_op_dist_attr_for_program(backward_op) - assert dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(backward_op)) + assert ( + dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format( + str(backward_op) + ) rank_id = dist_op_context.rank_id # check validation of inputs / outputs for input_name in backward_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( backward_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in backward_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( backward_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) # replicate op in dist program dist_op_desc = main_block.append_op(type='nop').desc @@ -535,7 +579,8 @@ def backward(ctx, *args, **kwargs): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and not is_parameter_related( - varname, main_block): + varname, main_block + ): act_grad_names.append(varname) out_grad_names = [] @@ -548,9 +593,11 @@ def backward(ctx, *args, **kwargs): if is_parameter_related(fwd_name, main_block): out_grad_names.append(varname) - gradient_synchronization(ctx, backward_op, act_grad_names, - out_grad_names, rank_id) + gradient_synchronization( + ctx, backward_op, act_grad_names, out_grad_names, rank_id + ) register_distributed_operator_impl( - "default", DistributedDefaultImpl0("replicate_parallel")) + "default", DistributedDefaultImpl0("replicate_parallel") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_eltwise.py b/python/paddle/distributed/auto_parallel/operators/dist_eltwise.py index 1dc163c0af44b..c9f5eb8051555 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_eltwise.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_eltwise.py @@ -27,18 +27,17 @@ class DistributedElementwise(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedElementwise, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedElementwise("elementwise")) + DistributedElementwise("elementwise") +) # Replicated Elementwise class DistributedElementwiseImpl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedElementwiseImpl0, self).__init__(name) self._forward_implemented = False @@ -56,13 +55,14 @@ def calc_cost(self, op_role, dist_op, ctx, cluster): def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes op_type = dist_op.serial_op.type - cost_mapping = build_comp_costs_from_descs(_g_op_cost_factory[op_type], - ctx, processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + _g_op_cost_factory[op_type], ctx, processes, desc_mapping, cluster + ) res_cost = [cost_mapping] return res_cost @@ -70,16 +70,17 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): def calc_bwd_cost(self, dist_op, ctx, cluster): # calc comp op cost res = [] - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) dist_attr = dist_op.dist_attr process_mesh = dist_attr.process_mesh processes = process_mesh.processes backward_op = dist_op.serial_op op_type = backward_op.type - cost_mapping = build_comp_costs_from_descs(_g_op_cost_factory[op_type], - ctx, processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + _g_op_cost_factory[op_type], ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) main_block = backward_op.block @@ -88,7 +89,8 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and not is_parameter_related( - varname, main_block): + varname, main_block + ): var_dim_mapping = dist_attr.get_input_dims_mapping(varname) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] @@ -100,16 +102,25 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and is_parameter_related( - varname, main_block): + varname, main_block + ): var_dim_mapping = dist_attr.get_input_dims_mapping( - varname) + varname + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [varname + "@GRAD"] - build_dp_costs(res, dist_op, ctx, var_names, attrs, - parallel_axis, cluster) + build_dp_costs( + res, + dist_op, + ctx, + var_names, + attrs, + parallel_axis, + cluster, + ) return res def is_input_compatible(self, dist_op): @@ -189,8 +200,9 @@ def is_auto_compatible(self, dist_op): for dims_mapping in dims_mapping_list: if idx < len(dims_mapping): dim_mappings.append(dims_mapping[-(idx + 1)]) - if not all(dim_mappings[0] == dim_mapping - for dim_mapping in dim_mappings): + if not all( + dim_mappings[0] == dim_mapping for dim_mapping in dim_mappings + ): return False return True @@ -216,10 +228,13 @@ def update_dims_mapping(self, dist_op): -1 for _ in range(input_max_dims_mapping_len) ] for i in range(input_dims_mapping_lens[arg_name]): - new_idx = (input_max_dims_mapping_len - - input_dims_mapping_lens[arg_name]) + i + new_idx = ( + input_max_dims_mapping_len + - input_dims_mapping_lens[arg_name] + ) + i new_dims_mapping[new_idx] = input_dims_mapping_dict[ - arg_name][i] + arg_name + ][i] dims_mapping_list.append(new_dims_mapping) else: dims_mapping_list.append(input_dims_mapping_dict[arg_name]) @@ -240,10 +255,13 @@ def update_dims_mapping(self, dist_op): -1 for _ in range(output_max_dims_mapping_len) ] for i in range(output_dims_mapping_lens[arg_name]): - new_idx = (output_max_dims_mapping_len - - output_dims_mapping_lens[arg_name]) + i + new_idx = ( + output_max_dims_mapping_len + - output_dims_mapping_lens[arg_name] + ) + i new_dims_mapping[new_idx] = output_dims_mapping_dict[ - arg_name][i] + arg_name + ][i] dims_mapping_list.append(new_dims_mapping) else: dims_mapping_list.append(output_dims_mapping_dict[arg_name]) @@ -251,7 +269,8 @@ def update_dims_mapping(self, dist_op): assert input_max_dims_mapping_len == output_max_dims_mapping_len max_dims_mapping_len = input_max_dims_mapping_len compatible_dims_mapping = compute_compatible_dims_mapping( - dims_mapping_list) + dims_mapping_list + ) if compatible_dims_mapping is None: return False @@ -261,17 +280,20 @@ def update_dims_mapping(self, dist_op): -1 for _ in range(input_dims_mapping_lens[arg_name]) ] for i in range(input_dims_mapping_lens[arg_name]): - new_idx = (max_dims_mapping_len - - input_dims_mapping_lens[arg_name]) + i + new_idx = ( + max_dims_mapping_len - input_dims_mapping_lens[arg_name] + ) + i new_dims_mapping[i] = compatible_dims_mapping[new_idx] if new_dims_mapping != input_dims_mapping_dict[arg_name]: - op_dist_attr.set_input_dims_mapping(arg_name, - new_dims_mapping) + op_dist_attr.set_input_dims_mapping( + arg_name, new_dims_mapping + ) changed = True else: if compatible_dims_mapping != input_dims_mapping_dict[arg_name]: op_dist_attr.set_input_dims_mapping( - arg_name, compatible_dims_mapping) + arg_name, compatible_dims_mapping + ) changed = True for arg_name in output_arg_names: @@ -280,17 +302,24 @@ def update_dims_mapping(self, dist_op): -1 for _ in range(output_dims_mapping_lens[arg_name]) ] for i in range(output_dims_mapping_lens[arg_name]): - new_idx = (max_dims_mapping_len - - output_dims_mapping_lens[arg_name]) + i + new_idx = ( + max_dims_mapping_len + - output_dims_mapping_lens[arg_name] + ) + i new_dims_mapping[i] = compatible_dims_mapping[new_idx] if new_dims_mapping != output_dims_mapping_dict[arg_name]: op_dist_attr.set_output_dims_mapping( - arg_name, new_dims_mapping) + arg_name, new_dims_mapping + ) changed = True else: - if compatible_dims_mapping != output_dims_mapping_dict[arg_name]: + if ( + compatible_dims_mapping + != output_dims_mapping_dict[arg_name] + ): op_dist_attr.set_output_dims_mapping( - arg_name, compatible_dims_mapping) + arg_name, compatible_dims_mapping + ) changed = True return changed @@ -305,4 +334,5 @@ def backward(ctx, *args, **kwargs): register_distributed_operator_impl( - "elementwise", DistributedElementwiseImpl0("replicate_parallel")) + "elementwise", DistributedElementwiseImpl0("replicate_parallel") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_embedding.py b/python/paddle/distributed/auto_parallel/operators/dist_embedding.py index 513dffb5fca10..fa76d9aff3253 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_embedding.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_embedding.py @@ -17,7 +17,11 @@ from .common import DistributedOperatorImpl from .common import register_distributed_operator_impl_container from .common import gradient_synchronization -from .common import naive_copy_op_dist_attr_for_program, register_distributed_operator_impl, set_comm_op_dist_attr_for_program +from .common import ( + naive_copy_op_dist_attr_for_program, + register_distributed_operator_impl, + set_comm_op_dist_attr_for_program, +) from ..utils import is_dim_shard from ..utils import is_dim_replicate from ..utils import compute_compatible_and_update_dim_mapping @@ -26,33 +30,48 @@ from paddle.fluid.data_feeder import check_variable_and_dtype, check_dtype from paddle.distributed.fleet.meta_optimizers.common import OP_ROLE_KEY, OpRole from ..process_group import new_process_group -from ..utils import _get_comm_group, _get_idx_in_axis, _get_corresponding_rank, set_var_dist_attr +from ..utils import ( + _get_comm_group, + _get_idx_in_axis, + _get_corresponding_rank, + set_var_dist_attr, +) from ..cost import build_comp_desc_from_dist_op, build_comm_desc_from_dist_op -from ..cost import build_comm_costs_from_descs, build_comp_costs_from_descs, build_dp_costs +from ..cost import ( + build_comm_costs_from_descs, + build_comp_costs_from_descs, + build_dp_costs, +) from ..cost import EmbeddingOpCost, EmbeddingGradOpCost -from paddle.distributed.auto_parallel.cost.comm_op_cost import AllreduceSumOpCost, IdentityOpCost +from paddle.distributed.auto_parallel.cost.comm_op_cost import ( + AllreduceSumOpCost, + IdentityOpCost, +) class DistributedEmbedding(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedEmbedding, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedEmbedding("lookup_table_v2")) + DistributedEmbedding("lookup_table_v2") +) register_distributed_operator_impl_container( - DistributedEmbedding("c_embedding")) + DistributedEmbedding("c_embedding") +) register_distributed_operator_impl_container( - DistributedEmbedding("lookup_table")) + DistributedEmbedding("lookup_table") +) def adopt_lookup_table_v1(ctx, main_block, src_op, Ids_var): - assert len( - Ids_var.shape - ) == 3, "input Ids to lookup_table should have 3 dimensions but got [{}] with shape [{}]".format( - Ids_var.name, Ids_var.shape) + assert ( + len(Ids_var.shape) == 3 + ), "input Ids to lookup_table should have 3 dimensions but got [{}] with shape [{}]".format( + Ids_var.name, Ids_var.shape + ) if not Ids_var.stop_gradient: raise NotImplementedError( 'Requiring the gradient of Ids of lookup_table(v1）dist op is not currently supported. Please open an issue with details on your use case so that we can prioritize adding this (for instance, adversarial training for language model).' @@ -60,59 +79,72 @@ def adopt_lookup_table_v1(ctx, main_block, src_op, Ids_var): target_shape = list(Ids_var.shape[:-1]) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["dist_reshape", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["dist_reshape", 'tmp']) + ), dtype=Ids_var.dtype, shape=target_shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) target_shape = [0] + list(Ids_var.shape[:-1]) xshape_var = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["dist_Xshape", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["dist_Xshape", 'tmp']) + ), dtype=Ids_var.dtype, shape=target_shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) # TODO use inplace reshape for memory saving - reshape_op = main_block.append_op(type='reshape2', - inputs={'X': [Ids_var]}, - outputs={ - 'Out': [intermediate_var_0], - 'XShape': [xshape_var] - }, - attrs={ - "shape": [0, -1], - }) + reshape_op = main_block.append_op( + type='reshape2', + inputs={'X': [Ids_var]}, + outputs={'Out': [intermediate_var_0], 'XShape': [xshape_var]}, + attrs={ + "shape": [0, -1], + }, + ) # set dist attr op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) Ids_var_dist_attr = op_dist_attr.get_input_dist_attr(Ids_var.name) assert Ids_var_dist_attr is not None intermediate_var_0_dist_attr = set_var_dist_attr( - ctx, intermediate_var_0, Ids_var_dist_attr.dims_mapping, - Ids_var_dist_attr.process_mesh) - set_var_dist_attr(ctx, xshape_var, - [-1] + list(Ids_var_dist_attr.dims_mapping), - Ids_var_dist_attr.process_mesh) + ctx, + intermediate_var_0, + Ids_var_dist_attr.dims_mapping, + Ids_var_dist_attr.process_mesh, + ) + set_var_dist_attr( + ctx, + xshape_var, + [-1] + list(Ids_var_dist_attr.dims_mapping), + Ids_var_dist_attr.process_mesh, + ) op_dist_attr.del_input_dist_attr(Ids_var.name) - op_dist_attr.set_input_dist_attr(intermediate_var_0.name, - intermediate_var_0_dist_attr) + op_dist_attr.set_input_dist_attr( + intermediate_var_0.name, intermediate_var_0_dist_attr + ) new_op_dist_attr = OperatorDistributedAttribute() new_op_dist_attr.process_mesh = Ids_var_dist_attr.process_mesh new_op_dist_attr.impl_type = "default" new_op_dist_attr.impl_idx = 0 - new_op_dist_attr.set_input_dims_mapping(Ids_var.name, - Ids_var_dist_attr.dims_mapping) - new_op_dist_attr.set_output_dims_mapping(intermediate_var_0.name, - Ids_var_dist_attr.dims_mapping) + new_op_dist_attr.set_input_dims_mapping( + Ids_var.name, Ids_var_dist_attr.dims_mapping + ) + new_op_dist_attr.set_output_dims_mapping( + intermediate_var_0.name, Ids_var_dist_attr.dims_mapping + ) new_op_dist_attr.set_output_dims_mapping( - xshape_var.name, [-1] + list(Ids_var_dist_attr.dims_mapping)) + xshape_var.name, [-1] + list(Ids_var_dist_attr.dims_mapping) + ) ctx.set_op_dist_attr_for_program(reshape_op, new_op_dist_attr) return intermediate_var_0 @@ -120,7 +152,6 @@ def adopt_lookup_table_v1(ctx, main_block, src_op, Ids_var): # RowParallel class DistributedEmbeddingImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedEmbeddingImpl, self).__init__(name) self._forward_implemented = True @@ -138,17 +169,19 @@ def calc_cost(self, op_role, dist_op, ctx, cluster): def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes # embedding need start_index - cost_mapping = build_comp_costs_from_descs(EmbeddingOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + EmbeddingOpCost, ctx, processes, desc_mapping, cluster + ) serial_op = dist_op.serial_op parallel_axis = dist_op.dist_attr.get_input_dims_mapping( - serial_op.input("W")[0])[0] + serial_op.input("W")[0] + )[0] attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = serial_op.output("Out") c_allreduce_sum_desc_mapping = build_comm_desc_from_dist_op( @@ -157,11 +190,16 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - AllreduceSumOpCost, ctx, processes, c_allreduce_sum_desc_mapping, - cluster) + AllreduceSumOpCost, + ctx, + processes, + c_allreduce_sum_desc_mapping, + cluster, + ) res_cost = [cost_mapping, comm_op_cost_list] @@ -175,7 +213,8 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): dist_attr = dist_op.dist_attr embedding_row_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("W")[0])[0] + backward_op.input("W")[0] + )[0] parallel_axis = embedding_row_dim_mapping attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = [backward_op.input("Out@GRAD")[0]] @@ -185,33 +224,38 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) process_mesh = dist_attr.process_mesh processes = process_mesh.processes comm_op_cost_list = build_comm_costs_from_descs( - IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster) + IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster + ) res.append(comm_op_cost_list) # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) - cost_mapping = build_comp_costs_from_descs(EmbeddingGradOpCost, ctx, - processes, desc_mapping, - cluster) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) + cost_mapping = build_comp_costs_from_descs( + EmbeddingGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("Ids")[0]) + backward_op.input("Ids")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] if batch_size_axis > -1 and mesh_shape[batch_size_axis] > 1: parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('W@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res @@ -223,7 +267,8 @@ def is_input_compatible(self, dist_op): ids_dims_mapping = op_dist_attr.get_input_dims_mapping(ids_name) w_dims_mapping = op_dist_attr.get_input_dims_mapping(w_name) if is_dim_replicate(w_dims_mapping[-2]) or is_dim_shard( - w_dims_mapping[-1]): + w_dims_mapping[-1] + ): return False # Other dimensions must be replicate except the batch dimension for mapping in ids_dims_mapping[1:]: @@ -243,8 +288,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -256,7 +302,7 @@ def is_auto_compatible(self, dist_op): ids_dims_mapping = op_dist_attr.get_input_dims_mapping(ids_name) w_dims_mapping = op_dist_attr.get_input_dims_mapping(w_name) - if ids_dims_mapping != out_dims_mapping[:len(ids_dims_mapping)]: + if ids_dims_mapping != out_dims_mapping[: len(ids_dims_mapping)]: return False return True @@ -274,12 +320,14 @@ def update_dims_mapping(self, dist_op): for i in range(len(ids_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [ids_dims_mapping, out_dims_mapping], [i, i]) + [ids_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True dim_changed = compute_compatible_and_update_dim_mapping( - [w_dims_mapping, out_dims_mapping], [-1, -1]) + [w_dims_mapping, out_dims_mapping], [-1, -1] + ) if dim_changed: changed = True @@ -297,26 +345,30 @@ def forward(ctx, *args, **kwargs): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # check validation of inputs / outputs assert 'Ids' in kwargs, "input [{}] is not given".format('Ids') assert 'W' in kwargs, "input [{}] is not given".format('W') assert 'Out' in kwargs, "output [{}] is not given".format('Out') - assert len( + assert ( + len(kwargs['Ids']) == 1 + ), "row_parallel_embedding input Ids take 1 variable but got {}".format( kwargs['Ids'] - ) == 1, "row_parallel_embedding input Ids take 1 variable but got {}".format( - kwargs['Ids']) - assert len( + ) + assert ( + len(kwargs['W']) == 1 + ), "row_parallel_embedding input W take 1 variable but got {}".format( kwargs['W'] - ) == 1, "row_parallel_embedding input W take 1 variable but got {}".format( - kwargs['W']) - assert len( + ) + assert ( + len(kwargs['Out']) == 1 + ), "row_parallel_embedding output Out take 1 variable but got {}".format( kwargs['Out'] - ) == 1, "row_parallel_embedding output Out take 1 variable but got {}".format( - kwargs['Out']) + ) Ids_var = main_block.var(kwargs['Ids'][0]) Weight_var = main_block._var_recursive(kwargs['W'][0]) @@ -328,70 +380,85 @@ def forward(ctx, *args, **kwargs): # got dist attribute info embedding_row_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[0] - assert embedding_row_dim_mapping >= 0, "row_parallel_embedding's row should be divided by a specific mesh axis, but got [{}]".format( - embedding_row_dim_mapping) + Weight_var.name + )[0] + assert ( + embedding_row_dim_mapping >= 0 + ), "row_parallel_embedding's row should be divided by a specific mesh axis, but got [{}]".format( + embedding_row_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in process_mesh_group: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # A generalized method to caculate embedding offset using cartisian product - relative_idx = _get_idx_in_axis(process_mesh_group, process_mesh_shape, - embedding_row_dim_mapping, rank_id) + relative_idx = _get_idx_in_axis( + process_mesh_group, + process_mesh_shape, + embedding_row_dim_mapping, + rank_id, + ) per_part_size = Weight_var.shape[0] relative_idx = relative_idx * per_part_size # TODO caculate ring id parallel_axis = embedding_row_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # append op - check_variable_and_dtype(Ids_var, 'input', ['int32', 'int64'], - 'c_embedding') + check_variable_and_dtype( + Ids_var, 'input', ['int32', 'int64'], 'c_embedding' + ) # infer new var shape with op dist attr out_tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(Out_var) assert out_tensor_dist_attr is not None out_var_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert out_var_dist_attr is not None - ref_shape = infer_shape(main_block, Out_var, out_tensor_dist_attr, - out_var_dist_attr) + ref_shape = infer_shape( + main_block, Out_var, out_tensor_dist_attr, out_var_dist_attr + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_embedding", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_embedding", 'tmp']) + ), dtype=Weight_var.dtype, shape=Out_var.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=Out_var.stop_gradient) + stop_gradient=Out_var.stop_gradient, + ) # set intermediate_var_0's dist_attr with Out_var's dist_attr - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - out_var_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, out_var_dist_attr + ) check_variable_and_dtype( - Out_var, 'tensor', + Out_var, + 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'c_allreduce_sum') + 'c_allreduce_sum', + ) c_embedding_op = main_block.append_op( type='c_embedding', - inputs={ - 'Ids': [Ids_var], - 'W': [Weight_var] - }, + inputs={'Ids': [Ids_var], 'W': [Weight_var]}, outputs={'Out': [intermediate_var_0]}, attrs={ "start_index": relative_idx, - OP_ROLE_KEY: src_op.attr('op_role') - }) + OP_ROLE_KEY: src_op.attr('op_role'), + }, + ) if intermediate_var_0.shape != ref_shape: intermediate_var_0.desc.set_shape(ref_shape) @@ -404,8 +471,9 @@ def forward(ctx, *args, **kwargs): 'ring_id': group.id, 'use_calc_stream': True, 'use_model_parallel': True, - OP_ROLE_KEY: src_op.attr('op_role') - }) + OP_ROLE_KEY: src_op.attr('op_role'), + }, + ) if Out_var.shape != ref_shape: Out_var.desc.set_shape(ref_shape) @@ -418,15 +486,19 @@ def forward(ctx, *args, **kwargs): for input_varname in c_embedding_op.desc.input_arg_names(): input_dist_attr = op_dist_attr.get_input_dist_attr(input_varname) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - embedding_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + embedding_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) output_varname = c_embedding_op.desc.output_arg_names()[0] output_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - embedding_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) + op_dist_attr + ) + embedding_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) ctx.set_op_dist_attr_for_program(c_embedding_op, embedding_op_dist_attr) # allreduce @@ -438,16 +510,20 @@ def forward(ctx, *args, **kwargs): input_var = main_block.var(input_varname) tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(input_var) assert tensor_dist_attr is not None - allreduce_op_dist_attr.set_input_dist_attr(input_varname, - tensor_dist_attr) + allreduce_op_dist_attr.set_input_dist_attr( + input_varname, tensor_dist_attr + ) for output_varname in c_allreduce_sum_op.desc.output_arg_names(): output_dist_attr = op_dist_attr.get_output_dist_attr(output_varname) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - allreduce_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) - ctx.set_op_dist_attr_for_program(c_allreduce_sum_op, - allreduce_op_dist_attr) + op_dist_attr + ) + allreduce_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) + ctx.set_op_dist_attr_for_program( + c_allreduce_sum_op, allreduce_op_dist_attr + ) # param initialization sync if Weight_var.is_parameter and not op_dist_attr.is_recompute: @@ -464,20 +540,25 @@ def forward(ctx, *args, **kwargs): if size <= 1 or axis in dim_mapping: pass else: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, axis, - rank_id) + group_ranks = _get_comm_group( + process_mesh.processes, + process_mesh.topology, + axis, + rank_id, + ) sync_group = new_process_group(group_ranks) - startup_block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': sync_group.id, - 'root': 0, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward - }) + startup_block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': sync_group.id, + 'root': 0, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -488,35 +569,43 @@ def backward(ctx, *args, **kwargs): backward_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id dist_attr = ctx.get_op_dist_attr_for_program(backward_op) - assert dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(backward_op)) + assert ( + dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format( + str(backward_op) + ) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, dist_attr.process_mesh, rank_id + ) assert 'Ids' in kwargs, "input [{}] is not given".format('Ids') assert 'W' in kwargs, "input [{}] is not given".format('W') assert 'Out@GRAD' in kwargs, "input [{}] is not given".format('Out') assert 'W@GRAD' in kwargs, "output [{}] is not given".format('W@GRAD') - assert len( + assert ( + len(kwargs['Ids']) == 1 + ), "row_parallel_embedding input Ids take 1 variable but got {}".format( kwargs['Ids'] - ) == 1, "row_parallel_embedding input Ids take 1 variable but got {}".format( - kwargs['Ids']) - assert len( + ) + assert ( + len(kwargs['W']) == 1 + ), "row_parallel_embedding input Ids take 1 variable but got {}".format( kwargs['W'] - ) == 1, "row_parallel_embedding input Ids take 1 variable but got {}".format( - kwargs['W']) - assert len( - kwargs['Out@GRAD'] - ) == 1, "row_parallel_embedding input Ids take 1 variable but got {}".format( - kwargs['Out']) - assert len( + ) + assert ( + len(kwargs['Out@GRAD']) == 1 + ), "row_parallel_embedding input Ids take 1 variable but got {}".format( + kwargs['Out'] + ) + assert ( + len(kwargs['W@GRAD']) == 1 + ), "row_parallel_embedding output Ids take 1 variable but got {}".format( kwargs['W@GRAD'] - ) == 1, "row_parallel_embedding output Ids take 1 variable but got {}".format( - kwargs['W@GRAD']) + ) Ids_var = main_block.var(kwargs['Ids'][0]) Weight_var = main_block.var(kwargs['W'][0]) @@ -524,39 +613,57 @@ def backward(ctx, *args, **kwargs): Weight_grad = main_block.var(kwargs['W@GRAD'][0]) embedding_row_dim_mapping = dist_attr.get_input_dims_mapping( - Weight_var.name)[0] - assert embedding_row_dim_mapping >= 0, "row_parallel_embedding's row should be divided by a specific mesh axis, but got [{}]".format( - embedding_row_dim_mapping) + Weight_var.name + )[0] + assert ( + embedding_row_dim_mapping >= 0 + ), "row_parallel_embedding's row should be divided by a specific mesh axis, but got [{}]".format( + embedding_row_dim_mapping + ) process_mesh_shape = dist_attr.process_mesh.topology process_mesh_group = dist_attr.process_mesh.processes # A generalized method to caculate embedding offset using cartisian product - relative_idx = _get_idx_in_axis(process_mesh_group, process_mesh_shape, - embedding_row_dim_mapping, rank_id) + relative_idx = _get_idx_in_axis( + process_mesh_group, + process_mesh_shape, + embedding_row_dim_mapping, + rank_id, + ) per_part_size = Weight_var.shape[0] relative_idx = relative_idx * per_part_size check_variable_and_dtype( - Out_grad, 'tensor', - ['float16', 'float32', 'float64', 'int32', 'int64'], '_c_identity') + Out_grad, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + '_c_identity', + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_embedding", '@tmp_0@GRAD'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_embedding", '@tmp_0@GRAD']) + ), dtype=Out_grad.dtype, shape=Out_grad.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=Out_grad.stop_gradient) + stop_gradient=Out_grad.stop_gradient, + ) # copy X_var's dist_attr to intermediate_var_0's dist_attr out_grad_dist_attr = dist_attr.get_input_dist_attr(Out_grad.name) assert out_grad_dist_attr is not None - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - out_grad_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, out_grad_dist_attr + ) - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - embedding_row_dim_mapping, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, + process_mesh_shape, + embedding_row_dim_mapping, + rank_id, + ) group = new_process_group(group_ranks) c_identity_op = main_block.append_op( @@ -568,41 +675,54 @@ def backward(ctx, *args, **kwargs): 'use_calc_stream': True, 'use_model_parallel': True, OP_ROLE_KEY: OpRole.Backward, - }) - check_variable_and_dtype(intermediate_var_0, 'x', - ['float16', 'float32', 'float64'], 'linear') - check_dtype(intermediate_var_0.dtype, 'dtype', - ['float16', 'float32', 'float64'], 'linear') + }, + ) + check_variable_and_dtype( + intermediate_var_0, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + intermediate_var_0.dtype, + 'dtype', + ['float16', 'float32', 'float64'], + 'linear', + ) - set_comm_op_dist_attr_for_program(c_identity_op, dist_attr.process_mesh, - out_grad_dist_attr, ctx) + set_comm_op_dist_attr_for_program( + c_identity_op, dist_attr.process_mesh, out_grad_dist_attr, ctx + ) c_embedding_grad_op_desc = main_block.append_op(type='nop').desc c_embedding_grad_op_desc.set_type("c_embedding_grad") c_embedding_grad_op_desc.set_input('Ids', [Ids_var.name]) c_embedding_grad_op_desc.set_input('W', [Weight_var.name]) - c_embedding_grad_op_desc.set_input('Out@GRAD', - [intermediate_var_0.name]) + c_embedding_grad_op_desc.set_input( + 'Out@GRAD', [intermediate_var_0.name] + ) c_embedding_grad_op_desc.set_output('W@GRAD', [Weight_grad.name]) c_embedding_grad_op_desc._set_attr('start_index', relative_idx) c_embedding_grad_op_desc._set_attr(OP_ROLE_KEY, OpRole.Backward) c_embedding_grad_op = main_block.ops[-1] assert c_embedding_grad_op.type == "c_embedding_grad" - naive_copy_op_dist_attr_for_program(c_embedding_grad_op, backward_op, - ctx) + naive_copy_op_dist_attr_for_program( + c_embedding_grad_op, backward_op, ctx + ) # data parallel gradient synchronization act_grad_names = [Ids_var.name] out_grad_names = [kwargs['W@GRAD'][0]] - gradient_synchronization(ctx, backward_op, act_grad_names, - out_grad_names, rank_id) + gradient_synchronization( + ctx, backward_op, act_grad_names, out_grad_names, rank_id + ) -register_distributed_operator_impl("lookup_table_v2", - DistributedEmbeddingImpl("row_parallel")) -register_distributed_operator_impl("c_embedding", - DistributedEmbeddingImpl("row_parallel")) -register_distributed_operator_impl("lookup_table", - DistributedEmbeddingImpl("row_parallel")) +register_distributed_operator_impl( + "lookup_table_v2", DistributedEmbeddingImpl("row_parallel") +) +register_distributed_operator_impl( + "c_embedding", DistributedEmbeddingImpl("row_parallel") +) +register_distributed_operator_impl( + "lookup_table", DistributedEmbeddingImpl("row_parallel") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_fill_constant_batch_size_like.py b/python/paddle/distributed/auto_parallel/operators/dist_fill_constant_batch_size_like.py index a6b48165fa9d6..8e3fc06756c5d 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_fill_constant_batch_size_like.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_fill_constant_batch_size_like.py @@ -25,17 +25,16 @@ class DistributedFillConstantBatchSizeLike(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedFillConstantBatchSizeLike, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedFillConstantBatchSizeLike("fill_constant_batch_size_like")) + DistributedFillConstantBatchSizeLike("fill_constant_batch_size_like") +) class DistributedFillConstantBatchSizeLikeImpl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedFillConstantBatchSizeLikeImpl0, self).__init__(name) self._forward_implemented = True @@ -45,7 +44,8 @@ def calc_cost(self, op_role, dist_op, ctx, cluster): cost = None if int(op_role) == int(OpRole.Backward): raise ValueError( - "The fill_constant_batch_size_like has no grad op.") + "The fill_constant_batch_size_like has no grad op." + ) else: cost = self.calc_fwd_cost(dist_op, ctx, cluster) assert cost is not None @@ -53,13 +53,18 @@ def calc_cost(self, op_role, dist_op, ctx, cluster): def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes op_type = dist_op.serial_op.type cost_mapping = build_comp_costs_from_descs( - FillConstantBatchSizeLikeOpCost, ctx, processes, desc_mapping, - cluster) + FillConstantBatchSizeLikeOpCost, + ctx, + processes, + desc_mapping, + cluster, + ) res_cost = [cost_mapping] return res_cost @@ -81,8 +86,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc op_dist_attr = dist_op.dist_attr @@ -105,7 +111,8 @@ def update_dims_mapping(self, dist_op): # only the batch size dimemsion of input and output are relative. dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [0, 0]) + [x_dims_mapping, out_dims_mapping], [0, 0] + ) if dim_changed: changed = True @@ -143,4 +150,5 @@ def backward(ctx, *args, **kwargs): register_distributed_operator_impl( "fill_constant_batch_size_like", - DistributedFillConstantBatchSizeLikeImpl0("fill_by_shape")) + DistributedFillConstantBatchSizeLikeImpl0("fill_by_shape"), +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_fused_attention.py b/python/paddle/distributed/auto_parallel/operators/dist_fused_attention.py index 0c8d8d7b76844..783a9ce5a1838 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_fused_attention.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_fused_attention.py @@ -24,17 +24,16 @@ class DistributedFusedAttention(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedFusedAttention, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedFusedAttention("fused_attention")) + DistributedFusedAttention("fused_attention") +) class DistributedFusedAttentionImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedFusedAttentionImpl, self).__init__(name) self._forward_implemented = True @@ -60,10 +59,12 @@ def is_input_compatible(self, dist_op): if is_dim_shard(mapping): return False if len(qkv_w_dims_mapping) != 4 or is_dim_replicate( - qkv_w_dims_mapping[head_axis]): + qkv_w_dims_mapping[head_axis] + ): return False if len(qkv_bias_dims_mapping) != 3 or is_dim_replicate( - qkv_bias_dims_mapping[head_axis]): + qkv_bias_dims_mapping[head_axis] + ): return False if is_dim_replicate(out_w_dims_mapping[0]): return False @@ -71,10 +72,13 @@ def is_input_compatible(self, dist_op): return False replicated_dims = [ - qkv_w_dims_mapping[0], qkv_w_dims_mapping[-2], - qkv_w_dims_mapping[-1], qkv_bias_dims_mapping[0], - qkv_bias_dims_mapping[-1], out_w_dims_mapping[-1], - out_bias_dims_mapping[-1] + qkv_w_dims_mapping[0], + qkv_w_dims_mapping[-2], + qkv_w_dims_mapping[-1], + qkv_bias_dims_mapping[0], + qkv_bias_dims_mapping[-1], + out_w_dims_mapping[-1], + out_bias_dims_mapping[-1], ] for mapping in replicated_dims: if is_dim_shard(mapping): @@ -100,8 +104,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -128,7 +133,8 @@ def update_dims_mapping(self, dist_op): out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) for i in range(len(x_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True @@ -145,22 +151,28 @@ def forward(ctx, *args, **kwargs): op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # infer logic comm presentation head_axis = 1 qkv_w = src_op.input('QKVW')[0] - qkv_w_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - qkv_w)[head_axis] - assert qkv_w_col_dim_mapping >= 0, "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - qkv_w_col_dim_mapping) + qkv_w_col_dim_mapping = op_dist_attr.get_input_dims_mapping(qkv_w)[ + head_axis + ] + assert ( + qkv_w_col_dim_mapping >= 0 + ), "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + qkv_w_col_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = qkv_w_col_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # insert op @@ -181,20 +193,25 @@ def backward(ctx, *args, **kwargs): op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # infer logic comm presentation out_w = src_op.input('OutLinearW')[0] out_w_col_dim_mapping = op_dist_attr.get_input_dims_mapping(out_w)[-1] - assert out_w_col_dim_mapping >= 0, "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - out_w_col_dim_mapping) + assert ( + out_w_col_dim_mapping >= 0 + ), "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + out_w_col_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = out_w_col_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # insert op @@ -207,4 +224,5 @@ def backward(ctx, *args, **kwargs): register_distributed_operator_impl( - "fused_attention", DistributedFusedAttentionImpl("tensor_parallel")) + "fused_attention", DistributedFusedAttentionImpl("tensor_parallel") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_fused_feedforward.py b/python/paddle/distributed/auto_parallel/operators/dist_fused_feedforward.py index a9b29c2054f9b..9cb435d4aa85e 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_fused_feedforward.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_fused_feedforward.py @@ -24,17 +24,16 @@ class DistributedFusedFeedForward(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedFusedFeedForward, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedFusedFeedForward("fused_feedforward")) + DistributedFusedFeedForward("fused_feedforward") +) class DistributedFusedFeedForwardImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedFusedFeedForwardImpl, self).__init__(name) self._forward_implemented = True @@ -51,24 +50,30 @@ def is_input_compatible(self, dist_op): x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) linear1_weight_dims_mapping = op_dist_attr.get_input_dims_mapping( - linear1_weight) + linear1_weight + ) linear1_bias_dims_mapping = op_dist_attr.get_input_dims_mapping( - linear1_bias) + linear1_bias + ) linear2_weight_dims_mapping = op_dist_attr.get_input_dims_mapping( - linear2_weight) + linear2_weight + ) linear2_bias_dims_mapping = op_dist_attr.get_input_dims_mapping( - linear2_bias) + linear2_bias + ) for mapping in x_dims_mapping[1:-1]: if is_dim_shard(mapping): return False if is_dim_shard(linear1_weight_dims_mapping[-2]) or is_dim_replicate( - linear1_weight_dims_mapping[-1]): + linear1_weight_dims_mapping[-1] + ): return False if is_dim_replicate(linear1_bias_dims_mapping[-1]): return False if is_dim_replicate(linear2_weight_dims_mapping[-2]) or is_dim_shard( - linear2_weight_dims_mapping[-1]): + linear2_weight_dims_mapping[-1] + ): return False if is_dim_shard(linear2_bias_dims_mapping[-1]): return False @@ -91,8 +96,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -119,7 +125,8 @@ def update_dims_mapping(self, dist_op): out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) for i in range(len(x_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True @@ -136,21 +143,27 @@ def forward(ctx, *args, **kwargs): op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # infer logic comm presentation linear1_weight = src_op.input('Linear1Weight')[0] linear1_weight_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - linear1_weight)[-1] - assert linear1_weight_col_dim_mapping >= 0, "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - linear1_weight_col_dim_mapping) + linear1_weight + )[-1] + assert ( + linear1_weight_col_dim_mapping >= 0 + ), "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + linear1_weight_col_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = linear1_weight_col_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # insert op @@ -172,21 +185,27 @@ def backward(ctx, *args, **kwargs): op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # infer logic comm presentation linear2_weight = src_op.input('Linear2Weight')[0] linear2_weight_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - linear2_weight)[-1] - assert linear2_weight_col_dim_mapping >= 0, "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - linear2_weight_col_dim_mapping) + linear2_weight + )[-1] + assert ( + linear2_weight_col_dim_mapping >= 0 + ), "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + linear2_weight_col_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = linear2_weight_col_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # insert op @@ -199,4 +218,5 @@ def backward(ctx, *args, **kwargs): register_distributed_operator_impl( - "fused_feedforward", DistributedFusedFeedForwardImpl("tensor_parallel")) + "fused_feedforward", DistributedFusedFeedForwardImpl("tensor_parallel") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_matmul.py b/python/paddle/distributed/auto_parallel/operators/dist_matmul.py index fd7852e469977..b13bbf740c791 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_matmul.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_matmul.py @@ -34,20 +34,31 @@ from ..process_group import new_process_group from ..utils import _get_comm_group, _get_corresponding_rank from .dist_default import DistributedDefaultImpl0 -from ..cost import build_comp_desc_from_dist_op, build_comm_desc_from_dist_op, build_dp_costs +from ..cost import ( + build_comp_desc_from_dist_op, + build_comm_desc_from_dist_op, + build_dp_costs, +) from ..cost import build_comm_costs_from_descs, build_comp_costs_from_descs from ..cost import MatmulV2OpCost, MatmulOpCost, MulOpCost from ..cost import MatmulV2GradOpCost, MatmulGradOpCost, MulGradOpCost -from paddle.distributed.auto_parallel.cost.comm_op_cost import AllreduceSumOpCost, IdentityOpCost +from paddle.distributed.auto_parallel.cost.comm_op_cost import ( + AllreduceSumOpCost, + IdentityOpCost, +) def trans_x_y_dims_mapping(trans_x, trans_y, x_dims_mapping, y_dims_mapping): if trans_x: - x_dims_mapping[-1], x_dims_mapping[-2] = x_dims_mapping[ - -2], x_dims_mapping[-1] + x_dims_mapping[-1], x_dims_mapping[-2] = ( + x_dims_mapping[-2], + x_dims_mapping[-1], + ) if trans_y: - y_dims_mapping[-1], y_dims_mapping[-2] = y_dims_mapping[ - -2], y_dims_mapping[-1] + y_dims_mapping[-1], y_dims_mapping[-2] = ( + y_dims_mapping[-2], + y_dims_mapping[-1], + ) def copy_op_with_new_input_output(ctx, block, src_op, **kwargs): @@ -120,13 +131,17 @@ def _update_dims_mapping_for_matmul(dist_op): for i in range(new_out_dims_mapping_len - 2): broadcast_out_dims_mapping.append(out_dims_mapping[i]) - compatible_dims_mapping = compute_compatible_dims_mapping([ - broadcast_x_dims_mapping, broadcast_y_dims_mapping, - broadcast_out_dims_mapping - ]) + compatible_dims_mapping = compute_compatible_dims_mapping( + [ + broadcast_x_dims_mapping, + broadcast_y_dims_mapping, + broadcast_out_dims_mapping, + ] + ) if compatible_dims_mapping is None: - trans_x_y_dims_mapping(trans_x, trans_y, x_dims_mapping, - y_dims_mapping) + trans_x_y_dims_mapping( + trans_x, trans_y, x_dims_mapping, y_dims_mapping + ) return False for i in range(new_x_dims_mapping_len - 2): @@ -149,17 +164,20 @@ def _update_dims_mapping_for_matmul(dist_op): # The following which uses negative index can be work # when len(out_dims_mapping) > 2 and len(out_dims_mapping) <=2 dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, y_dims_mapping], [-1, -2]) + [x_dims_mapping, y_dims_mapping], [-1, -2] + ) if dim_changed: changed = True dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [-2, -2]) + [x_dims_mapping, out_dims_mapping], [-2, -2] + ) if dim_changed: changed = True dim_changed = compute_compatible_and_update_dim_mapping( - [y_dims_mapping, out_dims_mapping], [-1, -1]) + [y_dims_mapping, out_dims_mapping], [-1, -1] + ) if dim_changed: changed = True @@ -199,7 +217,8 @@ def _is_auto_compatible_for_matmul(dist_op): x_dims_mapping = copy.deepcopy(op_dist_attr.get_input_dims_mapping(x_name)) y_dims_mapping = copy.deepcopy(op_dist_attr.get_input_dims_mapping(y_name)) out_dims_mapping = copy.deepcopy( - op_dist_attr.get_output_dims_mapping(out_name)) + op_dist_attr.get_output_dims_mapping(out_name) + ) x_dims_mapping_len = len(x_dims_mapping) y_dims_mapping_len = len(y_dims_mapping) out_dims_mapping_len = len(out_dims_mapping) @@ -231,22 +250,23 @@ def _is_auto_compatible_for_matmul(dist_op): for i in range(out_dims_mapping_len - 2): broadcast_out_dims_mapping.append(out_dims_mapping[i]) - is_same = ((broadcast_x_dims_mapping == broadcast_y_dims_mapping) - and (broadcast_x_dims_mapping == broadcast_out_dims_mapping)) + is_same = (broadcast_x_dims_mapping == broadcast_y_dims_mapping) and ( + broadcast_x_dims_mapping == broadcast_out_dims_mapping + ) if not is_same: return False # The following which uses negative index can be work # when len(out_dims_mapping) > 2 and len(out_dims_mapping) <=2 - is_same = (x_dims_mapping[-1] == y_dims_mapping[-2]) + is_same = x_dims_mapping[-1] == y_dims_mapping[-2] if not is_same: return False - is_same = (x_dims_mapping[-2] == out_dims_mapping[-2]) + is_same = x_dims_mapping[-2] == out_dims_mapping[-2] if not is_same: return False - is_same = (y_dims_mapping[-1] == out_dims_mapping[-1]) + is_same = y_dims_mapping[-1] == out_dims_mapping[-1] if not is_same: return False @@ -262,8 +282,9 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): backward_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id dist_attr = ctx.get_op_dist_attr_for_program(backward_op) - assert dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(backward_op)) + assert ( + dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(backward_op)) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in dist_attr.process_mesh.processes: @@ -274,22 +295,26 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): assert 'Out@GRAD' in kwargs, "input [{}] is not given".format('Out@GRAD') assert 'Y@GRAD' in kwargs, "output [{}] is not given".format('Y@GRAD') assert 'X@GRAD' in kwargs, "output [{}] is not given".format('X@GRAD') - assert len( + assert ( + len(kwargs['Y']) == 1 + ), "row_parallel_embedding input Ids take 1 variable but got {}".format( kwargs['Y'] - ) == 1, "row_parallel_embedding input Ids take 1 variable but got {}".format( - kwargs['Y']) - assert len( + ) + assert ( + len(kwargs['X']) == 1 + ), "row_parallel_embedding input Ids take 1 variable but got {}".format( kwargs['X'] - ) == 1, "row_parallel_embedding input Ids take 1 variable but got {}".format( - kwargs['X']) - assert len( - kwargs['Out@GRAD'] - ) == 1, "row_parallel_embedding input Ids take 1 variable but got {}".format( - kwargs['Out']) - assert len( + ) + assert ( + len(kwargs['Out@GRAD']) == 1 + ), "row_parallel_embedding input Ids take 1 variable but got {}".format( + kwargs['Out'] + ) + assert ( + len(kwargs['Y@GRAD']) == 1 + ), "row_parallel_embedding output Ids take 1 variable but got {}".format( kwargs['Y@GRAD'] - ) == 1, "row_parallel_embedding output Ids take 1 variable but got {}".format( - kwargs['Y@GRAD']) + ) X_var = main_block.var(kwargs['X'][0]) Y_var = main_block._var_recursive(kwargs['Y'][0]) @@ -299,7 +324,8 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): assert not is_parameter_related( X_var.name, main_block ), "left operand(X) [{}] of dist matmul should not be parameter".format( - X_var.name) + X_var.name + ) X_var_dims_mapping = dist_attr.get_input_dims_mapping(X_var.name) Y_var_dim_mapping = dist_attr.get_input_dims_mapping(Y_var.name) @@ -336,28 +362,34 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): parallel_axis = Y_var_dim_mapping[0] check_variable_and_dtype( - Out_grad, 'tensor', + Out_grad, + 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - '_c_identity') + '_c_identity', + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_identity", 'tmp'])) + "@GRAD", + name=unique_name.generate_with_ignorable_key( + ".".join(["c_identity", 'tmp']) + ) + + "@GRAD", dtype=Out_grad.dtype, shape=Out_grad.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=Out_grad.stop_gradient) + stop_gradient=Out_grad.stop_gradient, + ) # copy X_var's dist_attr to intermediate_var_0's dist_attr out_grad_dist_attr = dist_attr.get_input_dist_attr(Out_grad.name) assert out_grad_dist_attr is not None - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - out_grad_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, out_grad_dist_attr + ) - group_ranks = _get_comm_group(process_mesh_group, - process_mesh_shape, parallel_axis, - rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) c_identity_op = main_block.append_op( type='c_identity', @@ -368,20 +400,29 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): 'use_calc_stream': True, 'use_model_parallel': True, OP_ROLE_KEY: OpRole.Backward, - }) - check_variable_and_dtype(intermediate_var_0, 'x', - ['float16', 'float32', 'float64'], - 'linear') - check_dtype(intermediate_var_0.dtype, 'dtype', - ['float16', 'float32', 'float64'], 'linear') - set_comm_op_dist_attr_for_program(c_identity_op, - dist_attr.process_mesh, - out_grad_dist_attr, ctx) + }, + ) + check_variable_and_dtype( + intermediate_var_0, + 'x', + ['float16', 'float32', 'float64'], + 'linear', + ) + check_dtype( + intermediate_var_0.dtype, + 'dtype', + ['float16', 'float32', 'float64'], + 'linear', + ) + set_comm_op_dist_attr_for_program( + c_identity_op, dist_attr.process_mesh, out_grad_dist_attr, ctx + ) new_kwargs = copy.deepcopy(kwargs) new_kwargs['Out@GRAD'] = [intermediate_var_0.name] matmul_op_desc = copy_op_with_new_input_output( - ctx, main_block, backward_op, **new_kwargs) + ctx, main_block, backward_op, **new_kwargs + ) else: # col parallel: matmul + allreduce assert Y_var_dim_mapping[0] < 0 @@ -394,28 +435,36 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): assert len(kwargs['X@GRAD']) == 1 X_grad = main_block.var(kwargs['X@GRAD'][0]) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_identity", 'tmp'])) + "@GRAD", + name=unique_name.generate_with_ignorable_key( + ".".join(["c_identity", 'tmp']) + ) + + "@GRAD", dtype=X_grad.dtype, shape=X_grad.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=X_grad.stop_gradient) + stop_gradient=X_grad.stop_gradient, + ) X_grad_dist_attr = dist_attr.get_output_dist_attr(X_grad.name) assert X_grad_dist_attr is not None - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - X_grad_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, X_grad_dist_attr + ) new_kwargs['X@GRAD'] = [intermediate_var_0.name] matmul_op_desc = copy_op_with_new_input_output( - ctx, main_block, backward_op, **new_kwargs) + ctx, main_block, backward_op, **new_kwargs + ) # NOTE (JZ-LIANG) trick to skip one allreduce if left operand has not grad if has_x_grad: - group_ranks = _get_comm_group(process_mesh_group, - process_mesh_shape, parallel_axis, - rank_id) + group_ranks = _get_comm_group( + process_mesh_group, + process_mesh_shape, + parallel_axis, + rank_id, + ) group = new_process_group(group_ranks) c_allreduce_sum_op = main_block.append_op( type='c_allreduce_sum', @@ -425,15 +474,20 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): 'ring_id': group.id, 'use_calc_stream': True, 'use_model_parallel': True, - OP_ROLE_KEY: OpRole.Backward - }) - set_comm_op_dist_attr_for_program(c_allreduce_sum_op, - dist_attr.process_mesh, - X_grad_dist_attr, ctx) + OP_ROLE_KEY: OpRole.Backward, + }, + ) + set_comm_op_dist_attr_for_program( + c_allreduce_sum_op, + dist_attr.process_mesh, + X_grad_dist_attr, + ctx, + ) else: # replicate - matmul_op_desc = copy_op_with_new_input_output(ctx, main_block, - backward_op, **kwargs) + matmul_op_desc = copy_op_with_new_input_output( + ctx, main_block, backward_op, **kwargs + ) # data parallel gradient synchronization act_grad_names = [X_var.name] @@ -445,8 +499,9 @@ def _right_operand_parameter_matmul_backward(ctx, *args, **kwargs): if trans_x: trans_x_y_dims_mapping(True, False, X_var_dims_mapping, None) - gradient_synchronization(ctx, backward_op, act_grad_names, out_grad_names, - rank_id) + gradient_synchronization( + ctx, backward_op, act_grad_names, out_grad_names, rank_id + ) if trans_x: trans_x_y_dims_mapping(True, False, X_var_dims_mapping, None) @@ -469,23 +524,25 @@ def _init_param_sync(Weight_var, dist_op_context, startup_block, ctx, rank_id): if size <= 1 or axis in dim_mapping: pass else: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, axis, rank_id) + group_ranks = _get_comm_group( + process_mesh.processes, process_mesh.topology, axis, rank_id + ) sync_group = new_process_group(group_ranks) - startup_block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': sync_group.id, - 'root': 0, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward - }) + startup_block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': sync_group.id, + 'root': 0, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) class DistributedMatmul(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedMatmul, self).__init__(op_type) @@ -495,7 +552,6 @@ def __init__(self, op_type): # ColumnParallel class DistributedMatmulImpl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMatmulImpl0, self).__init__(name) self._forward_implemented = True @@ -518,7 +574,8 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): main_block = backward_op.block vars = main_block.vars Y_var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("Y")[0]) + backward_op.input("Y")[0] + ) # col parallel: matmul + allreduce assert Y_var_dim_mapping[0] < 0 parallel_axis = Y_var_dim_mapping[1] @@ -528,13 +585,14 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): assert len(backward_op.output("X@GRAD")) == 1 # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) process_mesh = dist_attr.process_mesh processes = process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulGradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # calc comm op cost @@ -547,40 +605,52 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - AllreduceSumOpCost, ctx, processes, - c_allreduce_sum_desc_mapping, cluster) + AllreduceSumOpCost, + ctx, + processes, + c_allreduce_sum_desc_mapping, + cluster, + ) res.append(comm_op_cost_list) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulOpCost, ctx, processes, - desc_mapping, cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulOpCost, ctx, processes, desc_mapping, cluster + ) # calc comm op cost serial_op = dist_op.serial_op vars = serial_op.block.vars parallel_axis = dist_op.dist_attr.get_input_dims_mapping( - serial_op.input("Y")[0])[-1] + serial_op.input("Y")[0] + )[-1] attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = serial_op.input("X") c_identity_desc_mapping = build_comm_desc_from_dist_op( @@ -589,10 +659,12 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster) + IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster + ) res_cost = [comm_op_cost_list, cost_mapping] return res_cost @@ -603,16 +675,19 @@ def is_input_compatible(self, dist_op): x_name = op_desc.input('X')[0] y_name = op_desc.input('Y')[0] x_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(x_name)) + op_dist_attr.get_input_dims_mapping(x_name) + ) y_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(y_name)) + op_dist_attr.get_input_dims_mapping(y_name) + ) trans_x = op_desc.attr('transpose_X') trans_y = op_desc.attr('transpose_Y') trans_x_y_dims_mapping(trans_x, trans_y, x_dims_mapping, y_dims_mapping) if is_dim_shard(x_dims_mapping[-1]): return False if is_dim_shard(y_dims_mapping[-2]) or is_dim_replicate( - y_dims_mapping[-1]): + y_dims_mapping[-1] + ): return False for mapping in x_dims_mapping[1:-1]: if is_dim_shard(mapping): @@ -632,8 +707,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): return False @@ -658,28 +734,33 @@ def forward(ctx, *args, **kwargs): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Weight_var = main_block.var(kwargs['Y'][0]) @@ -689,18 +770,24 @@ def forward(ctx, *args, **kwargs): # TODO infer logic comm presentation matmul_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-1] + Weight_var.name + )[-1] if trans_y: matmul_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-2] - assert matmul_col_dim_mapping >= 0, "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - matmul_col_dim_mapping) + Weight_var.name + )[-2] + assert ( + matmul_col_dim_mapping >= 0 + ), "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + matmul_col_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = matmul_col_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # infer new var shape with op dist attr @@ -708,31 +795,39 @@ def forward(ctx, *args, **kwargs): assert x_tensor_dist_attr is not None identity_var_dist_attr = op_dist_attr.get_input_dist_attr(X_var.name) assert identity_var_dist_attr is not None - ref_shape_x = infer_shape(main_block, X_var, x_tensor_dist_attr, - identity_var_dist_attr) + ref_shape_x = infer_shape( + main_block, X_var, x_tensor_dist_attr, identity_var_dist_attr + ) # infer out var shape with op dist attr out_tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(Out_var) assert out_tensor_dist_attr is not None out_var_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert out_var_dist_attr is not None - ref_shape_out = infer_shape(main_block, Out_var, out_tensor_dist_attr, - out_var_dist_attr) + ref_shape_out = infer_shape( + main_block, Out_var, out_tensor_dist_attr, out_var_dist_attr + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_identity", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_identity", 'tmp']) + ), dtype=X_var.dtype, shape=X_var.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=X_var.stop_gradient) + stop_gradient=X_var.stop_gradient, + ) # set intermediate_var_0's dist_attr with X_var's dist_attr - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - identity_var_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, identity_var_dist_attr + ) check_variable_and_dtype( - X_var, 'tensor', - ['float16', 'float32', 'float64', 'int32', 'int64'], '_c_identity') + X_var, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + '_c_identity', + ) c_identity_op = main_block.append_op( type='c_identity', @@ -742,26 +837,31 @@ def forward(ctx, *args, **kwargs): 'ring_id': group.id, 'use_calc_stream': True, 'use_model_parallel': True, - OP_ROLE_KEY: src_op.attr('op_role') - }) + OP_ROLE_KEY: src_op.attr('op_role'), + }, + ) if intermediate_var_0.shape != ref_shape_x: intermediate_var_0.desc.set_shape(ref_shape_x) - check_variable_and_dtype(intermediate_var_0, 'x', - ['float16', 'float32', 'float64'], 'linear') - check_dtype(intermediate_var_0.dtype, 'dtype', - ['float16', 'float32', 'float64'], 'linear') + check_variable_and_dtype( + intermediate_var_0, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + intermediate_var_0.dtype, + 'dtype', + ['float16', 'float32', 'float64'], + 'linear', + ) attrs = { 'transpose_X': trans_x, 'transpose_Y': trans_y, 'alpha': 1, - OP_ROLE_KEY: src_op.attr('op_role') + OP_ROLE_KEY: src_op.attr('op_role'), } inputs = {'X': [intermediate_var_0], 'Y': [Weight_var]} - matmul_op = main_block.append_op(type='matmul', - inputs=inputs, - outputs={'Out': Out_var}, - attrs=attrs) + matmul_op = main_block.append_op( + type='matmul', inputs=inputs, outputs={'Out': Out_var}, attrs=attrs + ) if Out_var.shape != ref_shape_out: Out_var.desc.set_shape(ref_shape_out) @@ -775,13 +875,16 @@ def forward(ctx, *args, **kwargs): input_varname = c_identity_op.desc.input_arg_names()[0] input_dist_attr = op_dist_attr.get_input_dist_attr(input_varname) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - identity_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + identity_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) # output output_varname = c_identity_op.desc.output_arg_names()[0] - identity_op_dist_attr.set_output_dist_attr(output_varname, - input_dist_attr) + identity_op_dist_attr.set_output_dist_attr( + output_varname, input_dist_attr + ) # set op dist attr ctx.set_op_dist_attr_for_program(c_identity_op, identity_op_dist_attr) @@ -794,31 +897,39 @@ def forward(ctx, *args, **kwargs): for input_varname in matmul_op.desc.input_arg_names(): if input_varname in src_op.desc.input_arg_names(): input_dist_attr = op_dist_attr.get_input_dist_attr( - input_varname) + input_varname + ) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmul_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + matmul_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) else: input_var = main_block.var(input_varname) tensor_dist_attr = ctx.get_tensor_dist_attr_for_program( - input_var) - matmul_op_dist_attr.set_input_dist_attr(input_varname, - tensor_dist_attr) + input_var + ) + matmul_op_dist_attr.set_input_dist_attr( + input_varname, tensor_dist_attr + ) # output output_varname = matmul_op.desc.output_arg_names()[0] output_dist_attr = op_dist_attr.get_output_dist_attr(output_varname) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmul_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) + op_dist_attr + ) + matmul_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) # set op dist attr ctx.set_op_dist_attr_for_program(matmul_op, matmul_op_dist_attr) # init param sync if Weight_var.is_parameter and not op_dist_attr.is_recompute: - _init_param_sync(Weight_var, dist_op_context, startup_block, ctx, - rank_id) + _init_param_sync( + Weight_var, dist_op_context, startup_block, ctx, rank_id + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -827,7 +938,6 @@ def backward(ctx, *args, **kwargs): # RowParallel class DistributedMatmulImpl1(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMatmulImpl1, self).__init__(name) self._forward_implemented = True @@ -850,7 +960,8 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): main_block = backward_op.block vars = main_block.vars Y_var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("Y")[0]) + backward_op.input("Y")[0] + ) assert Y_var_dim_mapping[1] < 0 parallel_axis = Y_var_dim_mapping[0] @@ -863,50 +974,60 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) process_mesh = dist_attr.process_mesh processes = process_mesh.processes comm_op_cost_list = build_comm_costs_from_descs( - IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster) + IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster + ) res.append(comm_op_cost_list) # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) - cost_mapping = build_comp_costs_from_descs(MatmulGradOpCost, ctx, - processes, desc_mapping, - cluster) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) + cost_mapping = build_comp_costs_from_descs( + MatmulGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulOpCost, ctx, processes, - desc_mapping, cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulOpCost, ctx, processes, desc_mapping, cluster + ) # calc comm op cost serial_op = dist_op.serial_op vars = serial_op.block.vars parallel_axis = dist_op.dist_attr.get_input_dims_mapping( - serial_op.input("Y")[0])[-2] + serial_op.input("Y")[0] + )[-2] attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = serial_op.output("Out") @@ -916,11 +1037,16 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - AllreduceSumOpCost, ctx, processes, c_allreduce_sum_desc_mapping, - cluster) + AllreduceSumOpCost, + ctx, + processes, + c_allreduce_sum_desc_mapping, + cluster, + ) res_cost = [cost_mapping, comm_op_cost_list] @@ -932,16 +1058,19 @@ def is_input_compatible(self, dist_op): x_name = op_desc.input('X')[0] y_name = op_desc.input('Y')[0] x_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(x_name)) + op_dist_attr.get_input_dims_mapping(x_name) + ) y_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(y_name)) + op_dist_attr.get_input_dims_mapping(y_name) + ) trans_x = op_desc.attr('transpose_X') trans_y = op_desc.attr('transpose_Y') trans_x_y_dims_mapping(trans_x, trans_y, x_dims_mapping, y_dims_mapping) if is_dim_replicate(x_dims_mapping[-1]): return False if is_dim_replicate(y_dims_mapping[-2]) or is_dim_shard( - y_dims_mapping[-1]): + y_dims_mapping[-1] + ): return False # Other dimensions must be replicate except the batch dimension for mapping in x_dims_mapping[1:-1]: @@ -963,8 +1092,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): return False @@ -989,28 +1119,33 @@ def forward(ctx, *args, **kwargs): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Weight_var = main_block.var(kwargs['Y'][0]) @@ -1020,29 +1155,37 @@ def forward(ctx, *args, **kwargs): # TODO infer logic comm presentation matmul_row_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-2] + Weight_var.name + )[-2] if trans_y: matmul_row_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-1] - assert matmul_row_dim_mapping >= 0, "row_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - matmul_row_dim_mapping) + Weight_var.name + )[-1] + assert ( + matmul_row_dim_mapping >= 0 + ), "row_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + matmul_row_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = matmul_row_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) - check_variable_and_dtype(X_var, 'x', ['float16', 'float32', 'float64'], - 'linear') - check_dtype(X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], - 'linear') + check_variable_and_dtype( + X_var, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], 'linear' + ) attrs = { 'transpose_X': trans_x, 'transpose_Y': trans_y, 'alpha': 1, - OP_ROLE_KEY: src_op.attr('op_role') + OP_ROLE_KEY: src_op.attr('op_role'), } inputs = {'X': X_var, 'Y': Weight_var} @@ -1051,27 +1194,33 @@ def forward(ctx, *args, **kwargs): assert out_tensor_dist_attr is not None out_var_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert out_var_dist_attr is not None - ref_shape = infer_shape(main_block, Out_var, out_tensor_dist_attr, - out_var_dist_attr) + ref_shape = infer_shape( + main_block, Out_var, out_tensor_dist_attr, out_var_dist_attr + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_allreduce_sum", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_allreduce_sum", 'tmp']) + ), shape=Out_var.shape, dtype=Out_var.dtype, type=Out_var.type, lod_level=Out_var.lod_level, persistable=False, is_data=False, - need_check_feed=Out_var.desc.need_check_feed()) + need_check_feed=Out_var.desc.need_check_feed(), + ) # set intermediate_var_0's dist_attr with Out_var's dist_attr - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - out_var_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, out_var_dist_attr + ) - matmul_op = main_block.append_op(type='matmul', - inputs=inputs, - outputs={'Out': intermediate_var_0}, - attrs=attrs) + matmul_op = main_block.append_op( + type='matmul', + inputs=inputs, + outputs={'Out': intermediate_var_0}, + attrs=attrs, + ) if intermediate_var_0.shape != ref_shape: intermediate_var_0.desc.set_shape(ref_shape) @@ -1083,8 +1232,9 @@ def forward(ctx, *args, **kwargs): 'ring_id': group.id, 'use_calc_stream': True, 'use_model_parallel': True, - OP_ROLE_KEY: src_op.attr('op_role') - }) + OP_ROLE_KEY: src_op.attr('op_role'), + }, + ) if Out_var.shape != ref_shape: Out_var.desc.set_shape(ref_shape) @@ -1097,15 +1247,19 @@ def forward(ctx, *args, **kwargs): for input_varname in matmul_op.desc.input_arg_names(): input_dist_attr = op_dist_attr.get_input_dist_attr(input_varname) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmul_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + matmul_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) output_varname = matmul_op.desc.output_arg_names()[0] output_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmul_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) + op_dist_attr + ) + matmul_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) ctx.set_op_dist_attr_for_program(matmul_op, matmul_op_dist_attr) # allreduce @@ -1117,21 +1271,26 @@ def forward(ctx, *args, **kwargs): input_var = main_block.var(input_varname) tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(input_var) assert tensor_dist_attr is not None - allreduce_op_dist_attr.set_input_dist_attr(input_varname, - tensor_dist_attr) + allreduce_op_dist_attr.set_input_dist_attr( + input_varname, tensor_dist_attr + ) for output_varname in c_allreduce_sum_op.desc.output_arg_names(): output_dist_attr = op_dist_attr.get_output_dist_attr(output_varname) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - allreduce_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) - ctx.set_op_dist_attr_for_program(c_allreduce_sum_op, - allreduce_op_dist_attr) + op_dist_attr + ) + allreduce_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) + ctx.set_op_dist_attr_for_program( + c_allreduce_sum_op, allreduce_op_dist_attr + ) # init param sync if Weight_var.is_parameter and not op_dist_attr.is_recompute: - _init_param_sync(Weight_var, dist_op_context, startup_block, ctx, - rank_id) + _init_param_sync( + Weight_var, dist_op_context, startup_block, ctx, rank_id + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -1140,7 +1299,6 @@ def backward(ctx, *args, **kwargs): # ReplicateParallel class DistributedMatmulImpl2(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMatmulImpl2, self).__init__(name) @@ -1161,38 +1319,45 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): vars = main_block.vars # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) process_mesh = dist_attr.process_mesh processes = process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulGradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulOpCost, ctx, processes, - desc_mapping, cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulOpCost, ctx, processes, desc_mapping, cluster + ) res_cost = [cost_mapping] return res_cost @@ -1208,13 +1373,15 @@ def is_input_compatible(self, dist_op): if is_dim_shard(x_dims_mapping[-1]): return False if is_valid_list_index(x_dims_mapping, -2) and is_dim_shard( - x_dims_mapping[-2]): + x_dims_mapping[-2] + ): return False if is_dim_shard(y_dims_mapping[-1]): return False if is_valid_list_index(y_dims_mapping, -2) and is_dim_shard( - y_dims_mapping[-2]): + y_dims_mapping[-2] + ): return False return True @@ -1228,14 +1395,16 @@ def is_output_compatible(self, dist_op): if is_dim_shard(out_dims_mapping[-1]): return False if is_valid_list_index(out_dims_mapping, -2) and is_dim_shard( - out_dims_mapping[-2]): + out_dims_mapping[-2] + ): return False return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): @@ -1259,16 +1428,18 @@ def backward(ctx, *args, **kwargs): _right_operand_parameter_matmul_backward(ctx, *args, **kwargs) -register_distributed_operator_impl("matmul", - DistributedMatmulImpl0("column_parallel")) -register_distributed_operator_impl("matmul", - DistributedMatmulImpl1("row_parallel")) -register_distributed_operator_impl("matmul", - DistributedMatmulImpl2("replicate_parallel")) +register_distributed_operator_impl( + "matmul", DistributedMatmulImpl0("column_parallel") +) +register_distributed_operator_impl( + "matmul", DistributedMatmulImpl1("row_parallel") +) +register_distributed_operator_impl( + "matmul", DistributedMatmulImpl2("replicate_parallel") +) class DistributedMatmulV2(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedMatmulV2, self).__init__(op_type) @@ -1278,7 +1449,6 @@ def __init__(self, op_type): # ColumnParallel class DistributedMatmulV2Impl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMatmulV2Impl0, self).__init__(name) self._forward_implemented = True @@ -1301,7 +1471,8 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): main_block = backward_op.block vars = main_block.vars Y_var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("Y")[0]) + backward_op.input("Y")[0] + ) process_mesh = dist_attr.process_mesh processes = process_mesh.processes # col parallel: matmul + allreduce @@ -1315,12 +1486,13 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): assert len(backward_op.output("X@GRAD")) == 1 # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) - cost_mapping = build_comp_costs_from_descs(MatmulV2GradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulV2GradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # calc comm op cost @@ -1333,45 +1505,55 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - AllreduceSumOpCost, ctx, processes, - c_allreduce_sum_desc_mapping, cluster) + AllreduceSumOpCost, + ctx, + processes, + c_allreduce_sum_desc_mapping, + cluster, + ) res.append(comm_op_cost_list) # need gradient allreduce process_mesh = dist_attr.process_mesh var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost # TODO: trans shape if trans_x or trans_y is True - comp_desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + comp_desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - comp_cost_mapping = build_comp_costs_from_descs(MatmulV2OpCost, ctx, - processes, - comp_desc_mapping, - cluster) + comp_cost_mapping = build_comp_costs_from_descs( + MatmulV2OpCost, ctx, processes, comp_desc_mapping, cluster + ) # calc comm op cost serial_op = dist_op.serial_op vars = serial_op.block.vars parallel_axis = dist_op.dist_attr.get_input_dims_mapping( - serial_op.input("Y")[0])[-1] + serial_op.input("Y")[0] + )[-1] attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = serial_op.input("X") @@ -1381,9 +1563,11 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster) + IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster + ) res_cost = [comm_op_cost_list, comp_cost_mapping] return res_cost @@ -1394,16 +1578,19 @@ def is_input_compatible(self, dist_op): x_name = op_desc.input('X')[0] y_name = op_desc.input('Y')[0] x_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(x_name)) + op_dist_attr.get_input_dims_mapping(x_name) + ) y_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(y_name)) + op_dist_attr.get_input_dims_mapping(y_name) + ) trans_x = op_desc.attr('trans_x') trans_y = op_desc.attr('trans_y') trans_x_y_dims_mapping(trans_x, trans_y, x_dims_mapping, y_dims_mapping) if is_dim_shard(x_dims_mapping[-1]): return False if is_dim_shard(y_dims_mapping[-2]) or is_dim_replicate( - y_dims_mapping[-1]): + y_dims_mapping[-1] + ): return False for mapping in x_dims_mapping[1:-1]: if is_dim_shard(mapping): @@ -1423,8 +1610,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): return False @@ -1449,28 +1637,33 @@ def forward(ctx, *args, **kwargs): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Weight_var = main_block._var_recursive(kwargs['Y'][0]) @@ -1480,18 +1673,24 @@ def forward(ctx, *args, **kwargs): # TODO infer logic comm presentation matmul_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-1] + Weight_var.name + )[-1] if trans_y: matmul_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-2] - assert matmul_col_dim_mapping >= 0, "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - matmul_col_dim_mapping) + Weight_var.name + )[-2] + assert ( + matmul_col_dim_mapping >= 0 + ), "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + matmul_col_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = matmul_col_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # infer new var shape with op dist attr @@ -1499,31 +1698,39 @@ def forward(ctx, *args, **kwargs): assert x_tensor_dist_attr is not None identity_var_dist_attr = op_dist_attr.get_input_dist_attr(X_var.name) assert identity_var_dist_attr is not None - ref_shape_x = infer_shape(main_block, X_var, x_tensor_dist_attr, - identity_var_dist_attr) + ref_shape_x = infer_shape( + main_block, X_var, x_tensor_dist_attr, identity_var_dist_attr + ) # infer out var shape with op dist attr out_tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(Out_var) assert out_tensor_dist_attr is not None out_var_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert out_var_dist_attr is not None - ref_shape_out = infer_shape(main_block, Out_var, out_tensor_dist_attr, - out_var_dist_attr) + ref_shape_out = infer_shape( + main_block, Out_var, out_tensor_dist_attr, out_var_dist_attr + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_identity", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_identity", 'tmp']) + ), dtype=X_var.dtype, shape=X_var.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=X_var.stop_gradient) + stop_gradient=X_var.stop_gradient, + ) # set intermediate_var_0's dist_attr with X_var's dist_attr - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - identity_var_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, identity_var_dist_attr + ) check_variable_and_dtype( - X_var, 'tensor', - ['float16', 'float32', 'float64', 'int32', 'int64'], '_c_identity') + X_var, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + '_c_identity', + ) c_identity_op = main_block.append_op( type='c_identity', inputs={'X': [X_var]}, @@ -1533,24 +1740,32 @@ def forward(ctx, *args, **kwargs): 'use_calc_stream': True, 'use_model_parallel': True, OP_ROLE_KEY: src_op.attr('op_role'), - }) + }, + ) if intermediate_var_0.shape != ref_shape_x: intermediate_var_0.desc.set_shape(ref_shape_x) - check_variable_and_dtype(intermediate_var_0, 'x', - ['float16', 'float32', 'float64'], 'linear') - check_dtype(intermediate_var_0.dtype, 'dtype', - ['float16', 'float32', 'float64'], 'linear') + check_variable_and_dtype( + intermediate_var_0, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + intermediate_var_0.dtype, + 'dtype', + ['float16', 'float32', 'float64'], + 'linear', + ) attrs = { 'trans_x': trans_x, 'trans_y': trans_y, - OP_ROLE_KEY: src_op.attr('op_role') + OP_ROLE_KEY: src_op.attr('op_role'), } inputs = {'X': [intermediate_var_0], 'Y': [Weight_var]} - matmul_v2_op = main_block.append_op(type='matmul_v2', - inputs=inputs, - outputs={'Out': Out_var}, - attrs=attrs) + matmul_v2_op = main_block.append_op( + type='matmul_v2', + inputs=inputs, + outputs={'Out': Out_var}, + attrs=attrs, + ) if Out_var.shape != ref_shape_out: Out_var.desc.set_shape(ref_shape_out) @@ -1564,13 +1779,16 @@ def forward(ctx, *args, **kwargs): input_varname = c_identity_op.desc.input_arg_names()[0] input_dist_attr = op_dist_attr.get_input_dist_attr(input_varname) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - identity_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + identity_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) # output output_varname = c_identity_op.desc.output_arg_names()[0] - identity_op_dist_attr.set_output_dist_attr(output_varname, - input_dist_attr) + identity_op_dist_attr.set_output_dist_attr( + output_varname, input_dist_attr + ) ctx.set_op_dist_attr_for_program(c_identity_op, identity_op_dist_attr) # matmulv2 @@ -1581,29 +1799,37 @@ def forward(ctx, *args, **kwargs): for input_varname in matmul_v2_op.desc.input_arg_names(): if input_varname in src_op.desc.input_arg_names(): input_dist_attr = op_dist_attr.get_input_dist_attr( - input_varname) + input_varname + ) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) + op_dist_attr + ) matmulv2_op_dist_attr.set_input_dist_attr( - input_varname, input_dist_attr) + input_varname, input_dist_attr + ) else: input_var = main_block.var(input_varname) tensor_dist_attr = ctx.get_tensor_dist_attr_for_program( - input_var) + input_var + ) matmulv2_op_dist_attr.set_input_dist_attr( - input_varname, tensor_dist_attr) + input_varname, tensor_dist_attr + ) for output_varname in matmul_v2_op.desc.output_arg_names(): output_dist_attr = op_dist_attr.get_output_dist_attr(output_varname) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmulv2_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) + op_dist_attr + ) + matmulv2_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) ctx.set_op_dist_attr_for_program(matmul_v2_op, matmulv2_op_dist_attr) # init param sync if Weight_var.is_parameter and not op_dist_attr.is_recompute: - _init_param_sync(Weight_var, dist_op_context, startup_block, ctx, - rank_id) + _init_param_sync( + Weight_var, dist_op_context, startup_block, ctx, rank_id + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -1612,7 +1838,6 @@ def backward(ctx, *args, **kwargs): # RowParallel class DistributedMatmulV2Impl1(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMatmulV2Impl1, self).__init__(name) self._forward_implemented = True @@ -1635,7 +1860,8 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): main_block = backward_op.block vars = main_block.vars Y_var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("Y")[0]) + backward_op.input("Y")[0] + ) assert Y_var_dim_mapping[1] < 0 parallel_axis = Y_var_dim_mapping[0] @@ -1650,50 +1876,59 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster) + IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster + ) res.append(comm_op_cost_list) # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) - cost_mapping = build_comp_costs_from_descs(MatmulV2GradOpCost, ctx, - processes, desc_mapping, - cluster) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) + cost_mapping = build_comp_costs_from_descs( + MatmulV2GradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # need gradient allreduce process_mesh = dist_attr.process_mesh var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulV2OpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulV2OpCost, ctx, processes, desc_mapping, cluster + ) # calc comm op cost serial_op = dist_op.serial_op vars = serial_op.block.vars parallel_axis = dist_op.dist_attr.get_input_dims_mapping( - serial_op.input("Y")[0])[-2] + serial_op.input("Y")[0] + )[-2] attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = serial_op.output("Out") @@ -1703,11 +1938,16 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - AllreduceSumOpCost, ctx, processes, c_allreduce_sum_desc_mapping, - cluster) + AllreduceSumOpCost, + ctx, + processes, + c_allreduce_sum_desc_mapping, + cluster, + ) res_cost = [cost_mapping, comm_op_cost_list] return res_cost @@ -1718,16 +1958,19 @@ def is_input_compatible(self, dist_op): x_name = op_desc.input('X')[0] y_name = op_desc.input('Y')[0] x_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(x_name)) + op_dist_attr.get_input_dims_mapping(x_name) + ) y_dims_mapping = copy.deepcopy( - op_dist_attr.get_input_dims_mapping(y_name)) + op_dist_attr.get_input_dims_mapping(y_name) + ) trans_x = op_desc.attr('trans_x') trans_y = op_desc.attr('trans_y') trans_x_y_dims_mapping(trans_x, trans_y, x_dims_mapping, y_dims_mapping) if is_dim_replicate(x_dims_mapping[-1]): return False if is_dim_replicate(y_dims_mapping[-2]) or is_dim_shard( - y_dims_mapping[-1]): + y_dims_mapping[-1] + ): return False # Other dimensions must be replicate except the batch dimension for mapping in x_dims_mapping[1:-1]: @@ -1749,8 +1992,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): return False @@ -1775,28 +2019,33 @@ def forward(ctx, *args, **kwargs): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Weight_var = main_block._var_recursive(kwargs['Y'][0]) @@ -1806,28 +2055,36 @@ def forward(ctx, *args, **kwargs): # TODO infer logic comm presentation matmul_row_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-2] + Weight_var.name + )[-2] if trans_y: matmul_row_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-1] - assert matmul_row_dim_mapping >= 0, "row_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - matmul_row_dim_mapping) + Weight_var.name + )[-1] + assert ( + matmul_row_dim_mapping >= 0 + ), "row_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + matmul_row_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = matmul_row_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) - check_variable_and_dtype(X_var, 'x', ['float16', 'float32', 'float64'], - 'linear') - check_dtype(X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], - 'linear') + check_variable_and_dtype( + X_var, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], 'linear' + ) attrs = { 'trans_x': trans_x, 'trans_y': trans_y, - OP_ROLE_KEY: src_op.attr('op_role') + OP_ROLE_KEY: src_op.attr('op_role'), } inputs = {'X': X_var, 'Y': Weight_var} @@ -1836,27 +2093,33 @@ def forward(ctx, *args, **kwargs): assert out_tensor_dist_attr is not None out_var_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert out_var_dist_attr is not None - ref_shape = infer_shape(main_block, Out_var, out_tensor_dist_attr, - out_var_dist_attr) + ref_shape = infer_shape( + main_block, Out_var, out_tensor_dist_attr, out_var_dist_attr + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_allreduce_sum", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_allreduce_sum", 'tmp']) + ), shape=Out_var.shape, dtype=Out_var.dtype, type=Out_var.type, lod_level=Out_var.lod_level, persistable=False, is_data=False, - need_check_feed=Out_var.desc.need_check_feed()) + need_check_feed=Out_var.desc.need_check_feed(), + ) # set intermediate_var_0's dist_attr with Out_var's dist_attr - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - out_var_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, out_var_dist_attr + ) - matmul_v2_op = main_block.append_op(type='matmul_v2', - inputs=inputs, - outputs={'Out': intermediate_var_0}, - attrs=attrs) + matmul_v2_op = main_block.append_op( + type='matmul_v2', + inputs=inputs, + outputs={'Out': intermediate_var_0}, + attrs=attrs, + ) if intermediate_var_0.shape != ref_shape: intermediate_var_0.desc.set_shape(ref_shape) @@ -1868,8 +2131,9 @@ def forward(ctx, *args, **kwargs): 'ring_id': group.id, 'use_calc_stream': True, 'use_model_parallel': True, - OP_ROLE_KEY: src_op.attr('op_role') - }) + OP_ROLE_KEY: src_op.attr('op_role'), + }, + ) if Out_var.shape != ref_shape: Out_var.desc.set_shape(ref_shape) @@ -1882,15 +2146,19 @@ def forward(ctx, *args, **kwargs): for input_varname in matmul_v2_op.desc.input_arg_names(): input_dist_attr = op_dist_attr.get_input_dist_attr(input_varname) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmulv2_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + matmulv2_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) output_varname = matmul_v2_op.desc.output_arg_names()[0] output_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmulv2_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) + op_dist_attr + ) + matmulv2_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) ctx.set_op_dist_attr_for_program(matmul_v2_op, matmulv2_op_dist_attr) # allreduce @@ -1902,21 +2170,26 @@ def forward(ctx, *args, **kwargs): input_var = main_block.var(input_varname) tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(input_var) assert tensor_dist_attr is not None - allreduce_op_dist_attr.set_input_dist_attr(input_varname, - tensor_dist_attr) + allreduce_op_dist_attr.set_input_dist_attr( + input_varname, tensor_dist_attr + ) for output_varname in c_allreduce_sum_op.desc.output_arg_names(): output_dist_attr = op_dist_attr.get_output_dist_attr(output_varname) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - allreduce_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) - ctx.set_op_dist_attr_for_program(c_allreduce_sum_op, - allreduce_op_dist_attr) + op_dist_attr + ) + allreduce_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) + ctx.set_op_dist_attr_for_program( + c_allreduce_sum_op, allreduce_op_dist_attr + ) # init param sync if Weight_var.is_parameter and not op_dist_attr.is_recompute: - _init_param_sync(Weight_var, dist_op_context, startup_block, ctx, - rank_id) + _init_param_sync( + Weight_var, dist_op_context, startup_block, ctx, rank_id + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -1925,7 +2198,6 @@ def backward(ctx, *args, **kwargs): # ReplicateParallel class DistributedMatmulV2Impl2(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMatmulV2Impl2, self).__init__(name) @@ -1947,38 +2219,44 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): process_mesh = dist_attr.process_mesh # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulV2GradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulV2GradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MatmulV2OpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MatmulV2OpCost, ctx, processes, desc_mapping, cluster + ) res_cost = [cost_mapping] @@ -1995,13 +2273,15 @@ def is_input_compatible(self, dist_op): if is_dim_shard(x_dims_mapping[-1]): return False if is_valid_list_index(x_dims_mapping, -2) and is_dim_shard( - x_dims_mapping[-2]): + x_dims_mapping[-2] + ): return False if is_dim_shard(y_dims_mapping[-1]): return False if is_valid_list_index(y_dims_mapping, -2) and is_dim_shard( - y_dims_mapping[-2]): + y_dims_mapping[-2] + ): return False return True @@ -2016,14 +2296,16 @@ def is_output_compatible(self, dist_op): if is_dim_shard(out_dims_mapping[-1]): return False if is_valid_list_index(out_dims_mapping, -2) and is_dim_shard( - out_dims_mapping[-2]): + out_dims_mapping[-2] + ): return False return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): @@ -2047,16 +2329,18 @@ def backward(ctx, *args, **kwargs): _right_operand_parameter_matmul_backward(ctx, *args, **kwargs) -register_distributed_operator_impl("matmul_v2", - DistributedMatmulV2Impl0("column_parallel")) -register_distributed_operator_impl("matmul_v2", - DistributedMatmulV2Impl1("row_parallel")) register_distributed_operator_impl( - "matmul_v2", DistributedMatmulV2Impl2("replicate_parallel")) + "matmul_v2", DistributedMatmulV2Impl0("column_parallel") +) +register_distributed_operator_impl( + "matmul_v2", DistributedMatmulV2Impl1("row_parallel") +) +register_distributed_operator_impl( + "matmul_v2", DistributedMatmulV2Impl2("replicate_parallel") +) class DistributedMul(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedMul, self).__init__(op_type) @@ -2066,7 +2350,6 @@ def __init__(self, op_type): # ColumnParallel class DistributedMulImpl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMulImpl0, self).__init__(name) self._forward_implemented = True @@ -2089,7 +2372,8 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): main_block = backward_op.block vars = main_block.vars Y_var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("Y")[0]) + backward_op.input("Y")[0] + ) # col parallel: matmul + allreduce assert Y_var_dim_mapping[0] < 0 parallel_axis = Y_var_dim_mapping[1] @@ -2099,13 +2383,14 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): assert len(backward_op.output("X@GRAD")) == 1 # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) process_mesh = dist_attr.process_mesh processes = process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MulGradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MulGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # calc comm op cost @@ -2118,40 +2403,52 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - AllreduceSumOpCost, ctx, processes, - c_allreduce_sum_desc_mapping, cluster) + AllreduceSumOpCost, + ctx, + processes, + c_allreduce_sum_desc_mapping, + cluster, + ) res.append(comm_op_cost_list) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MulOpCost, ctx, processes, - desc_mapping, cluster) + cost_mapping = build_comp_costs_from_descs( + MulOpCost, ctx, processes, desc_mapping, cluster + ) # calc comm op cost serial_op = dist_op.serial_op vars = serial_op.block.vars parallel_axis = dist_op.dist_attr.get_input_dims_mapping( - serial_op.input("Y")[0])[-1] + serial_op.input("Y")[0] + )[-1] attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = serial_op.input("X") c_identity_desc_mapping = build_comm_desc_from_dist_op( @@ -2160,10 +2457,12 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) comm_op_cost_list = build_comm_costs_from_descs( - IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster) + IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster + ) res_cost = [comm_op_cost_list, cost_mapping] return res_cost @@ -2178,7 +2477,8 @@ def is_input_compatible(self, dist_op): if is_dim_shard(x_dims_mapping[-1]): return False if is_dim_shard(y_dims_mapping[-2]) or is_dim_replicate( - y_dims_mapping[-1]): + y_dims_mapping[-1] + ): return False for mapping in x_dims_mapping[1:-1]: if is_dim_shard(mapping): @@ -2198,8 +2498,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): @@ -2226,28 +2527,33 @@ def forward(ctx, *args, **kwargs): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Weight_var = main_block._var_recursive(kwargs['Y'][0]) @@ -2255,15 +2561,20 @@ def forward(ctx, *args, **kwargs): # TODO infer logic comm presentation matmul_col_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-1] - assert matmul_col_dim_mapping >= 0, "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - matmul_col_dim_mapping) + Weight_var.name + )[-1] + assert ( + matmul_col_dim_mapping >= 0 + ), "col_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + matmul_col_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = matmul_col_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) # infer new var shape with op dist attr @@ -2271,31 +2582,39 @@ def forward(ctx, *args, **kwargs): assert x_tensor_dist_attr is not None identity_var_dist_attr = op_dist_attr.get_input_dist_attr(X_var.name) assert identity_var_dist_attr is not None - ref_shape_x = infer_shape(main_block, X_var, x_tensor_dist_attr, - identity_var_dist_attr) + ref_shape_x = infer_shape( + main_block, X_var, x_tensor_dist_attr, identity_var_dist_attr + ) # infer out var shape with op dist attr out_tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(Out_var) assert out_tensor_dist_attr is not None out_var_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert out_var_dist_attr is not None - ref_shape_out = infer_shape(main_block, Out_var, out_tensor_dist_attr, - out_var_dist_attr) + ref_shape_out = infer_shape( + main_block, Out_var, out_tensor_dist_attr, out_var_dist_attr + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_identity", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_identity", 'tmp']) + ), dtype=X_var.dtype, shape=X_var.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=X_var.stop_gradient) + stop_gradient=X_var.stop_gradient, + ) # set intermediate_var_0's dist_attr with X_var's dist_attr - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - identity_var_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, identity_var_dist_attr + ) check_variable_and_dtype( - X_var, 'tensor', - ['float16', 'float32', 'float64', 'int32', 'int64'], '_c_identity') + X_var, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + '_c_identity', + ) c_identity_op = main_block.append_op( type='c_identity', inputs={'X': [X_var]}, @@ -2304,20 +2623,26 @@ def forward(ctx, *args, **kwargs): 'ring_id': group.id, 'use_calc_stream': True, 'use_model_parallel': True, - OP_ROLE_KEY: src_op.attr('op_role') - }) + OP_ROLE_KEY: src_op.attr('op_role'), + }, + ) if intermediate_var_0.shape != ref_shape_x: intermediate_var_0.desc.set_shape(ref_shape_x) - check_variable_and_dtype(intermediate_var_0, 'x', - ['float16', 'float32', 'float64'], 'linear') - check_dtype(intermediate_var_0.dtype, 'dtype', - ['float16', 'float32', 'float64'], 'linear') + check_variable_and_dtype( + intermediate_var_0, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + intermediate_var_0.dtype, + 'dtype', + ['float16', 'float32', 'float64'], + 'linear', + ) # attrs = {'trans_x': False, 'trans_y': False} attrs = { "x_num_col_dims": src_op.desc.attr("x_num_col_dims"), "y_num_col_dims": src_op.desc.attr("y_num_col_dims"), - OP_ROLE_KEY: src_op.attr('op_role') + OP_ROLE_KEY: src_op.attr('op_role'), } inputs = {'X': intermediate_var_0, 'Y': Weight_var} @@ -2331,16 +2656,15 @@ def forward(ctx, *args, **kwargs): inputs_original_shape[var_name] = var.shape input_tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(var) input_var_dist_attr = op_dist_attr.get_input_dist_attr(var.name) - input_ref_shape = infer_shape(main_block, var, - input_tensor_dist_attr, - input_var_dist_attr) + input_ref_shape = infer_shape( + main_block, var, input_tensor_dist_attr, input_var_dist_attr + ) inputs_ref_shape[var_name] = input_ref_shape var.desc.set_shape(input_ref_shape) - mul_op = main_block.append_op(type='mul', - inputs=inputs, - outputs={'Out': Out_var}, - attrs=attrs) + mul_op = main_block.append_op( + type='mul', inputs=inputs, outputs={'Out': Out_var}, attrs=attrs + ) if Out_var.shape != ref_shape_out: Out_var.desc.set_shape(ref_shape_out) @@ -2359,13 +2683,16 @@ def forward(ctx, *args, **kwargs): input_varname = c_identity_op.desc.input_arg_names()[0] input_dist_attr = op_dist_attr.get_input_dist_attr(input_varname) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - identity_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + identity_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) # output output_varname = c_identity_op.desc.output_arg_names()[0] - identity_op_dist_attr.set_output_dist_attr(output_varname, - input_dist_attr) + identity_op_dist_attr.set_output_dist_attr( + output_varname, input_dist_attr + ) ctx.set_op_dist_attr_for_program(c_identity_op, identity_op_dist_attr) # matmulv2 @@ -2376,29 +2703,37 @@ def forward(ctx, *args, **kwargs): for input_varname in mul_op.desc.input_arg_names(): if input_varname in src_op.desc.input_arg_names(): input_dist_attr = op_dist_attr.get_input_dist_attr( - input_varname) + input_varname + ) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) + op_dist_attr + ) matmulv2_op_dist_attr.set_input_dist_attr( - input_varname, input_dist_attr) + input_varname, input_dist_attr + ) else: input_var = main_block.var(input_varname) tensor_dist_attr = ctx.get_tensor_dist_attr_for_program( - input_var) + input_var + ) matmulv2_op_dist_attr.set_input_dist_attr( - input_varname, tensor_dist_attr) + input_varname, tensor_dist_attr + ) for output_varname in mul_op.desc.output_arg_names(): output_dist_attr = op_dist_attr.get_output_dist_attr(output_varname) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmulv2_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) + op_dist_attr + ) + matmulv2_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) ctx.set_op_dist_attr_for_program(mul_op, matmulv2_op_dist_attr) # init param sync if Weight_var.is_parameter and not op_dist_attr.is_recompute: - _init_param_sync(Weight_var, dist_op_context, startup_block, ctx, - rank_id) + _init_param_sync( + Weight_var, dist_op_context, startup_block, ctx, rank_id + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -2407,7 +2742,6 @@ def backward(ctx, *args, **kwargs): # RowParallel class DistributedMulImpl1(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMulImpl1, self).__init__(name) self._forward_implemented = True @@ -2431,7 +2765,8 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): main_block = backward_op.block vars = main_block.vars Y_var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("Y")[0]) + backward_op.input("Y")[0] + ) assert Y_var_dim_mapping[1] < 0 parallel_axis = Y_var_dim_mapping[0] @@ -2444,49 +2779,59 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) processes = process_mesh.processes comm_op_cost_list = build_comm_costs_from_descs( - IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster) + IdentityOpCost, ctx, processes, c_identity_desc_mapping, cluster + ) res.append(comm_op_cost_list) # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) - cost_mapping = build_comp_costs_from_descs(MulGradOpCost, ctx, - processes, desc_mapping, - cluster) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) + cost_mapping = build_comp_costs_from_descs( + MulGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MulOpCost, ctx, processes, - desc_mapping, cluster) + cost_mapping = build_comp_costs_from_descs( + MulOpCost, ctx, processes, desc_mapping, cluster + ) # calc comm op cost serial_op = dist_op.serial_op vars = serial_op.block.vars parallel_axis = dist_op.dist_attr.get_input_dims_mapping( - serial_op.input("Y")[0])[-2] + serial_op.input("Y")[0] + )[-2] attrs = {"use_calc_stream": True, "use_model_parallel": True} var_names = serial_op.output("Out") @@ -2496,12 +2841,17 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): ctx, var_names, attrs=attrs, - parallel_axis=parallel_axis) + parallel_axis=parallel_axis, + ) # print("dist_matmul.py dist_op: ", dist_op) comm_op_cost_list = build_comm_costs_from_descs( - AllreduceSumOpCost, ctx, processes, c_allreduce_sum_desc_mapping, - cluster) + AllreduceSumOpCost, + ctx, + processes, + c_allreduce_sum_desc_mapping, + cluster, + ) res_cost = [cost_mapping, comm_op_cost_list] @@ -2517,7 +2867,8 @@ def is_input_compatible(self, dist_op): if is_dim_replicate(x_dims_mapping[-1]): return False if is_dim_replicate(y_dims_mapping[-2]) or is_dim_shard( - y_dims_mapping[-1]): + y_dims_mapping[-1] + ): return False # Other dimensions must be replicate except the batch dimension for mapping in x_dims_mapping[1:-1]: @@ -2539,8 +2890,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): @@ -2567,28 +2919,33 @@ def forward(ctx, *args, **kwargs): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # FIXME (JZ-LIANG) Remove this hack to support any op mesh group for Pipeline Parallelism if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Weight_var = main_block._var_recursive(kwargs['Y'][0]) @@ -2596,26 +2953,33 @@ def forward(ctx, *args, **kwargs): # TODO infer logic comm presentation matmul_row_dim_mapping = op_dist_attr.get_input_dims_mapping( - Weight_var.name)[-2] - assert matmul_row_dim_mapping >= 0, "row_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( - matmul_row_dim_mapping) + Weight_var.name + )[-2] + assert ( + matmul_row_dim_mapping >= 0 + ), "row_parallel_matmul's row should be divided by a specific mesh axis, but got [{}]".format( + matmul_row_dim_mapping + ) process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes parallel_axis = matmul_row_dim_mapping - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - parallel_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, parallel_axis, rank_id + ) group = new_process_group(group_ranks) - check_variable_and_dtype(X_var, 'x', ['float16', 'float32', 'float64'], - 'linear') - check_dtype(X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], - 'linear') + check_variable_and_dtype( + X_var, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], 'linear' + ) # attrs = {'trans_x': False, 'trans_y': False} attrs = { "x_num_col_dims": src_op.desc.attr("x_num_col_dims"), "y_num_col_dims": src_op.desc.attr("y_num_col_dims"), - OP_ROLE_KEY: src_op.attr('op_role') + OP_ROLE_KEY: src_op.attr('op_role'), } inputs = {'X': X_var, 'Y': Weight_var} @@ -2624,22 +2988,26 @@ def forward(ctx, *args, **kwargs): assert out_tensor_dist_attr is not None out_var_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert out_var_dist_attr is not None - ref_shape = infer_shape(main_block, Out_var, out_tensor_dist_attr, - out_var_dist_attr) + ref_shape = infer_shape( + main_block, Out_var, out_tensor_dist_attr, out_var_dist_attr + ) intermediate_var_0 = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ["c_allreduce_sum", 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(["c_allreduce_sum", 'tmp']) + ), shape=Out_var.shape, dtype=Out_var.dtype, type=Out_var.type, lod_level=Out_var.lod_level, persistable=False, is_data=False, - need_check_feed=Out_var.desc.need_check_feed()) + need_check_feed=Out_var.desc.need_check_feed(), + ) # set intermediate_var_0's dist_attr with Out_var's dist_attr - ctx.set_tensor_dist_attr_for_program(intermediate_var_0, - out_var_dist_attr) + ctx.set_tensor_dist_attr_for_program( + intermediate_var_0, out_var_dist_attr + ) inputs_ref_shape = {} inputs_original_shape = {} @@ -2648,16 +3016,18 @@ def forward(ctx, *args, **kwargs): inputs_original_shape[var_name] = var.shape input_tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(var) input_var_dist_attr = op_dist_attr.get_input_dist_attr(var.name) - input_ref_shape = infer_shape(main_block, var, - input_tensor_dist_attr, - input_var_dist_attr) + input_ref_shape = infer_shape( + main_block, var, input_tensor_dist_attr, input_var_dist_attr + ) inputs_ref_shape[var_name] = input_ref_shape var.desc.set_shape(input_ref_shape) - mul_op = main_block.append_op(type='mul', - inputs=inputs, - outputs={'Out': intermediate_var_0}, - attrs=attrs) + mul_op = main_block.append_op( + type='mul', + inputs=inputs, + outputs={'Out': intermediate_var_0}, + attrs=attrs, + ) if intermediate_var_0.shape != ref_shape: intermediate_var_0.desc.set_shape(ref_shape) @@ -2675,8 +3045,9 @@ def forward(ctx, *args, **kwargs): 'ring_id': group.id, 'use_calc_stream': True, 'use_model_parallel': True, - OP_ROLE_KEY: src_op.attr('op_role') - }) + OP_ROLE_KEY: src_op.attr('op_role'), + }, + ) if Out_var.shape != ref_shape: Out_var.desc.set_shape(ref_shape) @@ -2690,15 +3061,19 @@ def forward(ctx, *args, **kwargs): for input_varname in mul_op.desc.input_arg_names(): input_dist_attr = op_dist_attr.get_input_dist_attr(input_varname) assert input_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmulv2_op_dist_attr.set_input_dist_attr(input_varname, - input_dist_attr) + op_dist_attr + ) + matmulv2_op_dist_attr.set_input_dist_attr( + input_varname, input_dist_attr + ) output_varname = mul_op.desc.output_arg_names()[0] output_dist_attr = op_dist_attr.get_output_dist_attr(Out_var.name) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - matmulv2_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) + op_dist_attr + ) + matmulv2_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) ctx.set_op_dist_attr_for_program(mul_op, matmulv2_op_dist_attr) # allreduce @@ -2710,21 +3085,26 @@ def forward(ctx, *args, **kwargs): input_var = main_block.var(input_varname) tensor_dist_attr = ctx.get_tensor_dist_attr_for_program(input_var) assert tensor_dist_attr is not None - allreduce_op_dist_attr.set_input_dist_attr(input_varname, - tensor_dist_attr) + allreduce_op_dist_attr.set_input_dist_attr( + input_varname, tensor_dist_attr + ) for output_varname in c_allreduce_sum_op.desc.output_arg_names(): output_dist_attr = op_dist_attr.get_output_dist_attr(output_varname) assert output_dist_attr is not None, "dist_attr is {}".format( - op_dist_attr) - allreduce_op_dist_attr.set_output_dist_attr(output_varname, - output_dist_attr) - ctx.set_op_dist_attr_for_program(c_allreduce_sum_op, - allreduce_op_dist_attr) + op_dist_attr + ) + allreduce_op_dist_attr.set_output_dist_attr( + output_varname, output_dist_attr + ) + ctx.set_op_dist_attr_for_program( + c_allreduce_sum_op, allreduce_op_dist_attr + ) # init param sync if Weight_var.is_parameter and not op_dist_attr.is_recompute: - _init_param_sync(Weight_var, dist_op_context, startup_block, ctx, - rank_id) + _init_param_sync( + Weight_var, dist_op_context, startup_block, ctx, rank_id + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -2733,7 +3113,6 @@ def backward(ctx, *args, **kwargs): # ReplicateParallel class DistributedMulImpl2(DistributedOperatorImpl): - def __init__(self, name): super(DistributedMulImpl2, self).__init__(name) @@ -2754,38 +3133,45 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): vars = main_block.vars # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) process_mesh = dist_attr.process_mesh processes = process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MulGradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + MulGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) # need gradient allreduce var_dim_mapping = dist_attr.get_input_dims_mapping( - backward_op.input("X")[0]) + backward_op.input("X")[0] + ) mesh_shape = process_mesh.topology batch_size_axis = var_dim_mapping[0] - if batch_size_axis > -1 and mesh_shape[ - batch_size_axis] > 1 and is_parameter_related( - backward_op.input("Y")[0], main_block): + if ( + batch_size_axis > -1 + and mesh_shape[batch_size_axis] > 1 + and is_parameter_related(backward_op.input("Y")[0], main_block) + ): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [backward_op.output('Y@GRAD')[0]] - build_dp_costs(res, dist_op, ctx, var_names, attrs, parallel_axis, - cluster) + build_dp_costs( + res, dist_op, ctx, var_names, attrs, parallel_axis, cluster + ) return res def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(MulOpCost, ctx, processes, - desc_mapping, cluster) + cost_mapping = build_comp_costs_from_descs( + MulOpCost, ctx, processes, desc_mapping, cluster + ) res_cost = [cost_mapping] return res_cost @@ -2801,12 +3187,14 @@ def is_input_compatible(self, dist_op): if is_dim_shard(x_dims_mapping[-1]): return False if is_valid_list_index(x_dims_mapping, -2) and is_dim_shard( - x_dims_mapping[-2]): + x_dims_mapping[-2] + ): return False if is_dim_shard(y_dims_mapping[-1]): return False if is_valid_list_index(y_dims_mapping, -2) and is_dim_shard( - y_dims_mapping[-2]): + y_dims_mapping[-2] + ): return False return True @@ -2821,14 +3209,16 @@ def is_output_compatible(self, dist_op): if is_dim_shard(out_dims_mapping[-1]): return False if is_valid_list_index(out_dims_mapping, -2) and is_dim_shard( - out_dims_mapping[-2]): + out_dims_mapping[-2] + ): return False return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False if not _is_auto_compatible_for_matmul(dist_op): @@ -2852,8 +3242,10 @@ def backward(ctx, *args, **kwargs): _right_operand_parameter_matmul_backward(ctx, *args, **kwargs) -register_distributed_operator_impl("mul", - DistributedMulImpl0("column_parallel")) +register_distributed_operator_impl( + "mul", DistributedMulImpl0("column_parallel") +) register_distributed_operator_impl("mul", DistributedMulImpl1("row_parallel")) -register_distributed_operator_impl("mul", - DistributedMulImpl2("replicate_parallel")) +register_distributed_operator_impl( + "mul", DistributedMulImpl2("replicate_parallel") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_pnorm.py b/python/paddle/distributed/auto_parallel/operators/dist_pnorm.py index c56f121430a73..68d2351e1abe9 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_pnorm.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_pnorm.py @@ -20,8 +20,15 @@ from .common import register_distributed_operator_impl from ..process_group import new_process_group from ..utils import is_dim_shard, is_dim_replicate, _get_corresponding_rank -from ..utils import compute_compatible_dim_mapping, set_dist_op_desc_original_id, _get_comm_group -from ..dist_attribute import TensorDistributedAttribute, OperatorDistributedAttribute +from ..utils import ( + compute_compatible_dim_mapping, + set_dist_op_desc_original_id, + _get_comm_group, +) +from ..dist_attribute import ( + TensorDistributedAttribute, + OperatorDistributedAttribute, +) from paddle.fluid import core from paddle.fluid.framework import Operator @@ -29,7 +36,6 @@ class DistributedPNorm(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedPNorm, self).__init__(op_type) @@ -39,7 +45,6 @@ def __init__(self, op_type): # Row Parallel class DistributedPNormImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedPNormImpl, self).__init__(name) self._forward_implemented = True @@ -62,15 +67,18 @@ def is_output_compatible(self, dist_op): return True def is_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)) or \ - (not self.is_compatible(dist_op)): + if ( + (not self.is_input_compatible(dist_op)) + or (not self.is_output_compatible(dist_op)) + or (not self.is_compatible(dist_op)) + ): return False return True @@ -90,20 +98,25 @@ def update_dims_mapping(self, dist_op): batch_dim_mappings.append(dims_mapping[0]) compatible_dim_mapping = compute_compatible_dim_mapping( - batch_dim_mappings) + batch_dim_mappings + ) if compatible_dim_mapping is None: return False for arg_name in op_desc.input_arg_names(): dims_mapping = op_dist_attr.get_input_dims_mapping(arg_name) - if len(dims_mapping - ) >= 1 and compatible_dim_mapping != dims_mapping[0]: + if ( + len(dims_mapping) >= 1 + and compatible_dim_mapping != dims_mapping[0] + ): dims_mapping[0] = compatible_dim_mapping changed = True for arg_name in op_desc.output_arg_names(): dims_mapping = op_dist_attr.get_output_dims_mapping(arg_name) - if len(dims_mapping - ) >= 1 and compatible_dim_mapping != dims_mapping[0]: + if ( + len(dims_mapping) >= 1 + and compatible_dim_mapping != dims_mapping[0] + ): dims_mapping[0] = compatible_dim_mapping changed = True @@ -122,21 +135,25 @@ def forward(ctx, *args, **kwargs): # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) if rank_id not in op_dist_attr.process_mesh.processes: - rank_id = _get_corresponding_rank(ctx, op_dist_attr.process_mesh, - rank_id) + rank_id = _get_corresponding_rank( + ctx, op_dist_attr.process_mesh, rank_id + ) X_var = main_block.var(kwargs['X'][0]) in_dims_mapping = op_dist_attr.get_input_dims_mapping(X_var.name) @@ -145,14 +162,17 @@ def forward(ctx, *args, **kwargs): break process_mesh_shape = op_dist_attr.process_mesh.topology process_mesh_group = op_dist_attr.process_mesh.processes - group_ranks = _get_comm_group(process_mesh_group, process_mesh_shape, - axis, rank_id) + group_ranks = _get_comm_group( + process_mesh_group, process_mesh_shape, axis, rank_id + ) group = new_process_group(group_ranks) - check_variable_and_dtype(X_var, 'x', ['float16', 'float32', 'float64'], - 'norm') - check_dtype(X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], - 'norm') + check_variable_and_dtype( + X_var, 'x', ['float16', 'float32', 'float64'], 'norm' + ) + check_dtype( + X_var.dtype, 'dtype', ['float16', 'float32', 'float64'], 'norm' + ) # 2. insert c_allgather op # create c_allgather output var @@ -162,32 +182,37 @@ def forward(ctx, *args, **kwargs): shape=X_var.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=X_var.stop_gradient) + stop_gradient=X_var.stop_gradient, + ) # set allgather_out tensor dist_attr allgather_out_dist_attr = TensorDistributedAttribute() allgather_out_dist_attr.process_mesh = op_dist_attr.process_mesh allgather_out_dist_attr.dims_mapping = [ -1 for i in range(len(allgather_out.shape)) ] - ctx.set_tensor_dist_attr_for_program(allgather_out, - allgather_out_dist_attr) - c_allgather_op = main_block.append_op(type='c_allgather', - inputs={'X': [X_var]}, - outputs={'Out': [allgather_out]}, - attrs={ - 'ring_id': group.id, - 'use_calc_stream': True, - 'nranks': group.nranks, - 'op_role': - src_op.attr('op_role') - }) + ctx.set_tensor_dist_attr_for_program( + allgather_out, allgather_out_dist_attr + ) + c_allgather_op = main_block.append_op( + type='c_allgather', + inputs={'X': [X_var]}, + outputs={'Out': [allgather_out]}, + attrs={ + 'ring_id': group.id, + 'use_calc_stream': True, + 'nranks': group.nranks, + 'op_role': src_op.attr('op_role'), + }, + ) # set c_allgather op dist_attr allgather_op_dist_attr = OperatorDistributedAttribute() allgather_op_dist_attr.process_mesh = op_dist_attr.process_mesh - allgather_op_dist_attr.set_input_dims_mapping(X_var.name, - in_dims_mapping) + allgather_op_dist_attr.set_input_dims_mapping( + X_var.name, in_dims_mapping + ) allgather_op_dist_attr.set_output_dims_mapping( - allgather_out.name, allgather_out_dist_attr.dims_mapping) + allgather_out.name, allgather_out_dist_attr.dims_mapping + ) ctx.set_op_dist_attr_for_program(c_allgather_op, allgather_op_dist_attr) # 3. copy p_norm op desc and reset input name @@ -203,7 +228,8 @@ def forward(ctx, *args, **kwargs): dist_op_desc.set_output(output_name, kwargs[output_name]) pnorm_op = Operator(main_block, dist_op_desc) op_dist_attr.set_input_dims_mapping( - allgather_out.name, allgather_out_dist_attr.dims_mapping) + allgather_out.name, allgather_out_dist_attr.dims_mapping + ) ctx.set_op_dist_attr_for_program(pnorm_op, op_dist_attr) @staticmethod @@ -219,17 +245,20 @@ def backward(ctx, *args, **kwargs): # check validation of inputs / outputs for input_name in backward_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( backward_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in backward_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( backward_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) X_grad_var = main_block.var(kwargs['X@GRAD'][0]) @@ -244,7 +273,8 @@ def backward(ctx, *args, **kwargs): shape=new_X_var.shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=X_grad_var.stop_gradient) + stop_gradient=X_grad_var.stop_gradient, + ) new_kwargs['X@GRAD'] = [new_X_grad.name] new_X_var_dist_attr = ctx.get_tensor_dist_attr_for_program(new_X_var) ctx.set_tensor_dist_attr_for_program(new_X_grad, new_X_var_dist_attr) @@ -258,10 +288,12 @@ def backward(ctx, *args, **kwargs): for output_name in backward_op.desc.output_names(): dist_op_desc.set_output(output_name, new_kwargs[output_name]) p_norm_grad_op = Operator(main_block, dist_op_desc) - op_dist_attr.set_input_dims_mapping(new_X_var.name, - new_X_var_dist_attr.dims_mapping) - op_dist_attr.set_output_dims_mapping(new_X_grad.name, - new_X_var_dist_attr.dims_mapping) + op_dist_attr.set_input_dims_mapping( + new_X_var.name, new_X_var_dist_attr.dims_mapping + ) + op_dist_attr.set_output_dims_mapping( + new_X_grad.name, new_X_var_dist_attr.dims_mapping + ) ctx.set_op_dist_attr_for_program(p_norm_grad_op, op_dist_attr) # 2. insert slice op @@ -271,8 +303,12 @@ def backward(ctx, *args, **kwargs): from ..reshard import Resharder partition_idx = Resharder.compute_partition_index( - rank_id, new_X_grad.shape, dims_mapping, process_mesh_shape, - process_mesh_group) + rank_id, + new_X_grad.shape, + dims_mapping, + process_mesh_shape, + process_mesh_group, + ) slice_starts = [] slice_ends = [] slices_axes = [] @@ -287,22 +323,28 @@ def backward(ctx, *args, **kwargs): "starts": slice_starts, "ends": slice_ends, "infer_flags": infer_flags, - "op_role": backward_op.attr('op_role') + "op_role": backward_op.attr('op_role'), } - slice_op = main_block.append_op(type='slice', - inputs={'Input': [new_X_grad]}, - outputs={'Out': [X_grad_var]}, - attrs=attrs) + slice_op = main_block.append_op( + type='slice', + inputs={'Input': [new_X_grad]}, + outputs={'Out': [X_grad_var]}, + attrs=attrs, + ) X_grad_var_dims_mapping = op_dist_attr.get_output_dims_mapping( - X_grad_var.name) + X_grad_var.name + ) slice_op_dist_attr = OperatorDistributedAttribute() slice_op_dist_attr.process_mesh = op_dist_attr.process_mesh slice_op_dist_attr.set_input_dims_mapping( - new_X_grad.name, new_X_var_dist_attr.dims_mapping) - slice_op_dist_attr.set_output_dims_mapping(X_grad_var.name, - X_grad_var_dims_mapping) + new_X_grad.name, new_X_var_dist_attr.dims_mapping + ) + slice_op_dist_attr.set_output_dims_mapping( + X_grad_var.name, X_grad_var_dims_mapping + ) ctx.set_op_dist_attr_for_program(slice_op, slice_op_dist_attr) -register_distributed_operator_impl("p_norm", - DistributedPNormImpl("row_parallel")) +register_distributed_operator_impl( + "p_norm", DistributedPNormImpl("row_parallel") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_reduce_sum_p.py b/python/paddle/distributed/auto_parallel/operators/dist_reduce_sum_p.py index 77372257f4f75..d007aeda423f5 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_reduce_sum_p.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_reduce_sum_p.py @@ -23,18 +23,17 @@ class DistributedReduceSumPrimtive(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedReduceSumPrimtive, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedReduceSumPrimtive("reduce_sum_p")) + DistributedReduceSumPrimtive("reduce_sum_p") +) # Batch Dimension ReduceSum Primitive class DistributedReduceSumPrimtiveImpl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedReduceSumPrimtiveImpl0, self).__init__(name) self._forward_implemented = True @@ -56,7 +55,7 @@ def is_output_compatible(self, dist_op): output_name = outputs[0] output_var = dist_op.serial_op.block.var(output_name) - if output_var.shape != (1, ): + if output_var.shape != (1,): return False return True @@ -66,7 +65,8 @@ def is_auto_compatible(self, dist_op): op_dist_attr = dist_op.dist_attr return self.is_input_compatible(dist_op) and self.is_output_compatible( - dist_op) + dist_op + ) def update_dims_mapping(self, dist_op): changed = False @@ -85,17 +85,20 @@ def forward(ctx, *args, **kwargs): # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) # replicate op in dist program dist_op_desc = main_block.append_op(type='nop').desc @@ -109,14 +112,16 @@ def forward(ctx, *args, **kwargs): # batch dimension synchronization var_name = src_op.output_arg_names[0] sync_group = new_process_group(ctx.data_parallel_group) - allreduce_op = main_block.append_op(type='c_allreduce_sum', - inputs={'X': [var_name]}, - outputs={'Out': [var_name]}, - attrs={ - 'ring_id': sync_group.id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward - }) + allreduce_op = main_block.append_op( + type='c_allreduce_sum', + inputs={'X': [var_name]}, + outputs={'Out': [var_name]}, + attrs={ + 'ring_id': sync_group.id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) # dist attr var = main_block.var(var_name) @@ -124,19 +129,24 @@ def forward(ctx, *args, **kwargs): op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) new_op_attr = OperatorDistributedAttribute() new_op_attr.process_mesh = op_dist_attr.process_mesh - new_op_attr.set_output_dims_mapping(var.name, - tensor_dist_attr.dims_mapping) - new_op_attr.set_input_dims_mapping(var.name, - tensor_dist_attr.dims_mapping) + new_op_attr.set_output_dims_mapping( + var.name, tensor_dist_attr.dims_mapping + ) + new_op_attr.set_input_dims_mapping( + var.name, tensor_dist_attr.dims_mapping + ) ctx.set_op_dist_attr_for_program(allreduce_op, new_op_attr) @staticmethod def backward(ctx, *args, **kwargs): raise RuntimeError( - "primitive operator does NOT have backward function, op type: {}". - format(str(op.type))) + "primitive operator does NOT have backward function, op type: {}".format( + str(op.type) + ) + ) register_distributed_operator_impl( "reduce_sum_p", - DistributedReduceSumPrimtiveImpl0("batch_dimension_reduce_sum_p")) + DistributedReduceSumPrimtiveImpl0("batch_dimension_reduce_sum_p"), +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_reshape.py b/python/paddle/distributed/auto_parallel/operators/dist_reshape.py index e9f32b80ca13c..67b32dbecae85 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_reshape.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_reshape.py @@ -27,7 +27,6 @@ class DistributedReshape2(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedReshape2, self).__init__(op_type) @@ -36,7 +35,6 @@ def __init__(self, op_type): class DistributedReshapeImpl0(DistributedOperatorImpl): - def __init__(self, name): super(DistributedReshapeImpl0, self).__init__(name) self._forward_implemented = True @@ -66,19 +64,21 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): for idx, axis in enumerate(dim_mapping): if axis >= 0: if len(shape_list) > idx: - shape_list[ - idx] = shape_list[idx] // process_mesh_shape[axis] + shape_list[idx] = ( + shape_list[idx] // process_mesh_shape[axis] + ) # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_attr.process_mesh.processes for key in desc_mapping: desc_mapping[key]["shape"] = shape_list - cost_mapping = build_comp_costs_from_descs(Reshape2OpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Reshape2OpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) return res @@ -86,16 +86,17 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): def calc_bwd_cost(self, dist_op, ctx, cluster): # calc comp op cost res = [] - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) dist_attr = dist_op.dist_attr process_mesh = dist_attr.process_mesh processes = process_mesh.processes op_type = dist_op.serial_op.type - cost_mapping = build_comp_costs_from_descs(Reshape2GradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Reshape2GradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) backward_op = dist_op.serial_op @@ -105,7 +106,8 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and is_parameter_related( - varname, main_block): + varname, main_block + ): # NOTE input var's dim_mapping of backward op should be the same with input var instead of corresponding varname of forward op var_dim_mapping = dist_attr.get_input_dims_mapping(varname) @@ -115,8 +117,15 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [varname + "@GRAD"] - build_dp_costs(res, dist_op, ctx, var_names, attrs, - parallel_axis, cluster) + build_dp_costs( + res, + dist_op, + ctx, + var_names, + attrs, + parallel_axis, + cluster, + ) return res @@ -150,8 +159,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -160,7 +170,8 @@ def is_auto_compatible(self, dist_op): out_name = op_desc.output('Out')[0] x_shape_name = op_desc.output('XShape')[0] x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) @@ -186,11 +197,13 @@ def update_dims_mapping(self, dist_op): x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) for i in range(len(x_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True @@ -210,23 +223,27 @@ def forward(ctx, *args, **kwargs): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Out_var = main_block.var(kwargs['Out'][0]) @@ -247,8 +264,9 @@ def forward(ctx, *args, **kwargs): for idx, axis in enumerate(dim_mapping): if axis >= 0: if len(shape_list) > idx: - shape_list[ - idx] = shape_list[idx] // process_mesh_shape[axis] + shape_list[idx] = ( + shape_list[idx] // process_mesh_shape[axis] + ) # create op new_op_desc = main_block.append_op(type='nop').desc @@ -267,7 +285,6 @@ def backward(ctx, *args, **kwargs): class DistributedReshapeImpl1(DistributedOperatorImpl): - def __init__(self, name): super(DistributedReshapeImpl1, self).__init__(name) self._forward_implemented = True @@ -297,19 +314,21 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): for idx, axis in enumerate(dim_mapping): if axis >= 0: if len(shape_list) > idx: - shape_list[ - idx] = shape_list[idx] // process_mesh_shape[axis] + shape_list[idx] = ( + shape_list[idx] // process_mesh_shape[axis] + ) # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_attr.process_mesh.processes for key in desc_mapping: desc_mapping[key]["shape"] = shape_list - cost_mapping = build_comp_costs_from_descs(Reshape2OpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Reshape2OpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) return res @@ -317,16 +336,17 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): def calc_bwd_cost(self, dist_op, ctx, cluster): # calc comp op cost res = [] - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) dist_attr = dist_op.dist_attr process_mesh = dist_attr.process_mesh processes = process_mesh.processes op_type = dist_op.serial_op.type - cost_mapping = build_comp_costs_from_descs(Reshape2GradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Reshape2GradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) backward_op = dist_op.serial_op @@ -336,7 +356,8 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and not is_parameter_related( - varname, main_block): + varname, main_block + ): # NOTE input var's dim_mapping of backward op should be the same with input var instead of corresponding varname of forward op var_dim_mapping = dist_attr.get_input_dims_mapping(varname) @@ -346,8 +367,15 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [varname + "@GRAD"] - build_dp_costs(res, dist_op, ctx, var_names, attrs, - parallel_axis, cluster) + build_dp_costs( + res, + dist_op, + ctx, + var_names, + attrs, + parallel_axis, + cluster, + ) return res @@ -381,8 +409,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -393,7 +422,8 @@ def is_auto_compatible(self, dist_op): x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) if is_dim_shard(x_dims_mapping[-1]): return False @@ -420,11 +450,13 @@ def update_dims_mapping(self, dist_op): x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) for i in range(len(out_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True @@ -444,23 +476,27 @@ def forward(ctx, *args, **kwargs): src_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Out_var = main_block.var(kwargs['Out'][0]) @@ -481,8 +517,9 @@ def forward(ctx, *args, **kwargs): for idx, axis in enumerate(dim_mapping): if axis >= 0: if len(shape_list) > idx: - shape_list[ - idx] = shape_list[idx] // process_mesh_shape[axis] + shape_list[idx] = ( + shape_list[idx] // process_mesh_shape[axis] + ) # create op new_op_desc = main_block.append_op(type='nop').desc @@ -501,7 +538,6 @@ def backward(ctx, *args, **kwargs): class DistributedReshapeImpl2(DistributedOperatorImpl): - def __init__(self, name): super(DistributedReshapeImpl2, self).__init__(name) self._forward_implemented = True @@ -531,19 +567,21 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): for idx, axis in enumerate(dim_mapping): if axis >= 0: if len(shape_list) > idx: - shape_list[ - idx] = shape_list[idx] // process_mesh_shape[axis] + shape_list[idx] = ( + shape_list[idx] // process_mesh_shape[axis] + ) # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_attr.process_mesh.processes for key in desc_mapping: desc_mapping[key]["shape"] = shape_list - cost_mapping = build_comp_costs_from_descs(Reshape2OpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Reshape2OpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) return res @@ -551,16 +589,17 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): def calc_bwd_cost(self, dist_op, ctx, cluster): # calc comp op cost res = [] - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) dist_attr = dist_op.dist_attr process_mesh = dist_attr.process_mesh processes = process_mesh.processes op_type = dist_op.serial_op.type - cost_mapping = build_comp_costs_from_descs(Reshape2GradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Reshape2GradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) backward_op = dist_op.serial_op @@ -570,7 +609,8 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and not is_parameter_related( - varname, main_block): + varname, main_block + ): # NOTE input var's dim_mapping of backward op should be the same with input var instead of corresponding varname of forward op var_dim_mapping = dist_attr.get_input_dims_mapping(varname) @@ -580,8 +620,15 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [varname + "@GRAD"] - build_dp_costs(res, dist_op, ctx, var_names, attrs, - parallel_axis, cluster) + build_dp_costs( + res, + dist_op, + ctx, + var_names, + attrs, + parallel_axis, + cluster, + ) return res @@ -612,8 +659,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -624,7 +672,8 @@ def is_auto_compatible(self, dist_op): x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) for idx, item in enumerate(x_dims_mapping[:-1]): if out_dims_mapping[idx] != item: @@ -648,11 +697,13 @@ def update_dims_mapping(self, dist_op): x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) for i in range(len(out_dims_mapping) - 1): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True @@ -671,23 +722,27 @@ def forward(ctx, *args, **kwargs): main_block = dist_op_context.work_block src_op = dist_op_context.cur_src_op op_dist_attr = ctx.get_op_dist_attr_for_program(src_op) - assert op_dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(src_op)) + assert ( + op_dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format(str(src_op)) # check validation of inputs / outputs for input_name in src_op.desc.input_names(): assert input_name in kwargs, "input [{}] is not given".format( - input_name) + input_name + ) assert len(kwargs[input_name]) == len( src_op.desc.input(input_name) ), "number of tensor for input [{}] is not match".format(input_name) for output_name in src_op.desc.output_names(): assert output_name in kwargs, "input [{}] is not given".format( - output_name) + output_name + ) assert len(kwargs[output_name]) == len( src_op.desc.output(output_name) ), "number of tensor for input [{}] is not match".format( - output_name) + output_name + ) X_var = main_block.var(kwargs['X'][0]) Out_var = main_block.var(kwargs['Out'][0]) @@ -708,8 +763,9 @@ def forward(ctx, *args, **kwargs): for idx, axis in enumerate(out_dim_mapping): if axis >= 0: if len(shape_list) > idx: - shape_list[ - idx] = shape_list[idx] // process_mesh_shape[axis] + shape_list[idx] = ( + shape_list[idx] // process_mesh_shape[axis] + ) # create op new_op_desc = main_block.append_op(type='nop').desc @@ -727,9 +783,12 @@ def backward(ctx, *args, **kwargs): DistributedDefaultImpl0.backward(ctx, *args, **kwargs) -register_distributed_operator_impl("reshape2", - DistributedReshapeImpl0("add_one_dim_back")) register_distributed_operator_impl( - "reshape2", DistributedReshapeImpl1("remove_one_dim_back")) -register_distributed_operator_impl("reshape2", - DistributedReshapeImpl2("same_dim_shape")) + "reshape2", DistributedReshapeImpl0("add_one_dim_back") +) +register_distributed_operator_impl( + "reshape2", DistributedReshapeImpl1("remove_one_dim_back") +) +register_distributed_operator_impl( + "reshape2", DistributedReshapeImpl2("same_dim_shape") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_shape.py b/python/paddle/distributed/auto_parallel/operators/dist_shape.py index 313f296ab9624..47254cd99854a 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_shape.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_shape.py @@ -21,7 +21,6 @@ class DistributedShape(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedShape, self).__init__(op_type) @@ -30,7 +29,6 @@ def __init__(self, op_type): class DistributedShapeImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedShapeImpl, self).__init__(name) self._forward_implemented = True @@ -52,8 +50,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False return True diff --git a/python/paddle/distributed/auto_parallel/operators/dist_slice.py b/python/paddle/distributed/auto_parallel/operators/dist_slice.py index 1c9b0e482a13c..82562c8b07564 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_slice.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_slice.py @@ -22,7 +22,6 @@ class DistributedSlice(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedSlice, self).__init__(op_type) @@ -31,7 +30,6 @@ def __init__(self, op_type): class DistributedSliceImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedSliceImpl, self).__init__(name) self._forward_implemented = True @@ -75,8 +73,9 @@ def is_output_compatible(self, dist_op): return True def is_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -87,7 +86,8 @@ def is_compatible(self, dist_op): in_dims_mapping = op_dist_attr.get_input_dims_mapping(in_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) if len(in_dims_mapping) - len(decrease_axis) != 0 and len( - out_dims_mapping) != len(in_dims_mapping) - len(decrease_axis): + out_dims_mapping + ) != len(in_dims_mapping) - len(decrease_axis): return False new_out_dims_mapping = [] @@ -102,9 +102,11 @@ def is_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)) or \ - (not self.is_compatible(dist_op)): + if ( + (not self.is_input_compatible(dist_op)) + or (not self.is_output_compatible(dist_op)) + or (not self.is_compatible(dist_op)) + ): return False return True @@ -135,7 +137,8 @@ def update_dims_mapping(self, dist_op): assert len(ref_dims_mapping) == len(out_dims_mapping) for i in range(len(out_dims_mapping)): compatible_dim_mapping = compute_compatible_dim_mapping( - [out_dims_mapping[i], ref_dims_mapping[i]]) + [out_dims_mapping[i], ref_dims_mapping[i]] + ) if compatible_dim_mapping is None: continue if ref_dims_mapping[i] != compatible_dim_mapping: @@ -156,5 +159,6 @@ def backward(ctx, *args, **kwargs): DistributedDefaultImpl0.backward(ctx, *args, **kwargs) -register_distributed_operator_impl("slice", - DistributedSliceImpl("decrease_in_axis")) +register_distributed_operator_impl( + "slice", DistributedSliceImpl("decrease_in_axis") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_softmax.py b/python/paddle/distributed/auto_parallel/operators/dist_softmax.py index 8d85c2c19fcca..6a980f724639b 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_softmax.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_softmax.py @@ -27,7 +27,6 @@ class DistributedSoftmax(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedSoftmax, self).__init__(op_type) @@ -36,7 +35,6 @@ def __init__(self, op_type): class DistributedSoftmaxImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedSoftmaxImpl, self).__init__(name) self._forward_implemented = False @@ -53,12 +51,13 @@ def calc_cost(self, op_role, dist_op, ctx, cluster): def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes - cost_mapping = build_comp_costs_from_descs(SoftmaxOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + SoftmaxOpCost, ctx, processes, desc_mapping, cluster + ) res_cost = [cost_mapping] return res_cost @@ -66,14 +65,15 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): def calc_bwd_cost(self, dist_op, ctx, cluster): # calc comp op cost res = [] - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) dist_attr = dist_op.dist_attr process_mesh = dist_attr.process_mesh processes = process_mesh.processes - cost_mapping = build_comp_costs_from_descs(SoftmaxGradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + SoftmaxGradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) backward_op = dist_op.serial_op @@ -83,7 +83,8 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and is_parameter_related( - varname, main_block): + varname, main_block + ): # NOTE input var's dim_mapping of backward op should be the same with input var instead of corresponding varname of forward op var_dim_mapping = dist_attr.get_input_dims_mapping(varname) @@ -93,8 +94,15 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [varname + "@GRAD"] - build_dp_costs(res, dist_op, ctx, var_names, attrs, - parallel_axis, cluster) + build_dp_costs( + res, + dist_op, + ctx, + var_names, + attrs, + parallel_axis, + cluster, + ) return res @@ -129,8 +137,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -159,7 +168,8 @@ def update_dims_mapping(self, dist_op): for i in range(len(x_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True @@ -175,4 +185,5 @@ def backward(ctx, *args, **kwargs): register_distributed_operator_impl( - "softmax", DistributedSoftmaxImpl("replicate_last_axis")) + "softmax", DistributedSoftmaxImpl("replicate_last_axis") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_split.py b/python/paddle/distributed/auto_parallel/operators/dist_split.py index cf02e00a33764..8411d6d83dd6e 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_split.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_split.py @@ -22,7 +22,6 @@ class DistributedSplit(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedSplit, self).__init__(op_type) @@ -31,7 +30,6 @@ def __init__(self, op_type): class DistributedSplitImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedSplitImpl, self).__init__(name) self._forward_implemented = True @@ -62,8 +60,9 @@ def is_output_compatible(self, dist_op): return True def is_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -91,16 +90,19 @@ def update_dims_mapping(self, dist_op): out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) for i in range(len(x_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [x_dims_mapping, out_dims_mapping], [i, i]) + [x_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True return changed def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)) or \ - (not self.is_compatible(dist_op)): + if ( + (not self.is_input_compatible(dist_op)) + or (not self.is_output_compatible(dist_op)) + or (not self.is_compatible(dist_op)) + ): return False return True @@ -114,5 +116,6 @@ def backward(ctx, *args, **kwargs): DistributedDefaultImpl0.backward(ctx, *args, **kwargs) -register_distributed_operator_impl("split", - DistributedSplitImpl("replicate_in_axis")) +register_distributed_operator_impl( + "split", DistributedSplitImpl("replicate_in_axis") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_transpose.py b/python/paddle/distributed/auto_parallel/operators/dist_transpose.py index 96220bf0d50fb..6bacb8bace409 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_transpose.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_transpose.py @@ -26,17 +26,16 @@ class DistributedTranspose2(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedTranspose2, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedTranspose2("transpose2")) + DistributedTranspose2("transpose2") +) class DistributedTranspose2Impl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedTranspose2Impl, self).__init__(name) self._forward_implemented = False @@ -49,8 +48,9 @@ def is_output_compatible(self, dist_op): return True def is_auto_compatible(self, dist_op): - if (not self.is_input_compatible(dist_op)) or \ - (not self.is_output_compatible(dist_op)): + if (not self.is_input_compatible(dist_op)) or ( + not self.is_output_compatible(dist_op) + ): return False op_desc = dist_op.serial_op.desc @@ -60,7 +60,8 @@ def is_auto_compatible(self, dist_op): out_name = op_desc.output('Out')[0] x_shape_name = op_desc.output('XShape')[0] x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) new_dims_mapping = [-1 for i in range(len(x_dims_mapping))] @@ -91,7 +92,8 @@ def update_dims_mapping(self, dist_op): x_dims_mapping = op_dist_attr.get_input_dims_mapping(x_name) out_dims_mapping = op_dist_attr.get_output_dims_mapping(out_name) x_shape_dims_mapping = op_dist_attr.get_output_dims_mapping( - x_shape_name) + x_shape_name + ) perm = op_desc.attr('axis') assert len(x_dims_mapping) == len(perm) @@ -102,7 +104,8 @@ def update_dims_mapping(self, dist_op): for i in range(len(out_dims_mapping)): dim_changed = compute_compatible_and_update_dim_mapping( - [new_dims_mapping, out_dims_mapping], [i, i]) + [new_dims_mapping, out_dims_mapping], [i, i] + ) if dim_changed: changed = True @@ -127,13 +130,14 @@ def calc_cost(self, op_role, dist_op, ctx, cluster): def calc_fwd_cost(self, dist_op, ctx, cluster): # calc comp op cost - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) processes = dist_op.dist_attr.process_mesh.processes op_type = dist_op.serial_op.type - cost_mapping = build_comp_costs_from_descs(Transpose2OpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Transpose2OpCost, ctx, processes, desc_mapping, cluster + ) res_cost = [cost_mapping] return res_cost @@ -141,15 +145,16 @@ def calc_fwd_cost(self, dist_op, ctx, cluster): def calc_bwd_cost(self, dist_op, ctx, cluster): # calc comp op cost res = [] - desc_mapping = build_comp_desc_from_dist_op(dist_op=dist_op, - dist_context=ctx) + desc_mapping = build_comp_desc_from_dist_op( + dist_op=dist_op, dist_context=ctx + ) dist_attr = dist_op.dist_attr process_mesh = dist_attr.process_mesh processes = process_mesh.processes op_type = dist_op.serial_op.type - cost_mapping = build_comp_costs_from_descs(Transpose2GradOpCost, ctx, - processes, desc_mapping, - cluster) + cost_mapping = build_comp_costs_from_descs( + Transpose2GradOpCost, ctx, processes, desc_mapping, cluster + ) res.append(cost_mapping) backward_op = dist_op.serial_op @@ -159,7 +164,8 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): for input_name in backward_op.desc.input_names(): for varname in backward_op.desc.input(input_name): if "@GRAD" not in varname and is_parameter_related( - varname, main_block): + varname, main_block + ): # NOTE input var's dim_mapping of backward op should be the same with input var instead of corresponding varname of forward op var_dim_mapping = dist_attr.get_input_dims_mapping(varname) @@ -169,8 +175,15 @@ def calc_bwd_cost(self, dist_op, ctx, cluster): parallel_axis = batch_size_axis attrs = {"use_calc_stream": True} var_names = [varname + "@GRAD"] - build_dp_costs(res, dist_op, ctx, var_names, attrs, - parallel_axis, cluster) + build_dp_costs( + res, + dist_op, + ctx, + var_names, + attrs, + parallel_axis, + cluster, + ) return res @staticmethod @@ -183,4 +196,5 @@ def backward(ctx, *args, **kwargs): register_distributed_operator_impl( - "transpose2", DistributedTranspose2Impl("same_mapping_transpose")) + "transpose2", DistributedTranspose2Impl("same_mapping_transpose") +) diff --git a/python/paddle/distributed/auto_parallel/operators/dist_update_loss_scaling.py b/python/paddle/distributed/auto_parallel/operators/dist_update_loss_scaling.py index cbbcaef5ee47f..389473660799c 100644 --- a/python/paddle/distributed/auto_parallel/operators/dist_update_loss_scaling.py +++ b/python/paddle/distributed/auto_parallel/operators/dist_update_loss_scaling.py @@ -20,17 +20,16 @@ class DistributedUpdateLossScaling(DistributedOperatorImplContainer): - def __init__(self, op_type): super(DistributedUpdateLossScaling, self).__init__(op_type) register_distributed_operator_impl_container( - DistributedUpdateLossScaling("update_loss_scaling")) + DistributedUpdateLossScaling("update_loss_scaling") +) class DistributedUpdateLossScalingImpl(DistributedOperatorImpl): - def __init__(self, name): super(DistributedUpdateLossScalingImpl, self).__init__(name) self._name = name @@ -60,7 +59,8 @@ def update_dims_mapping(self, dist_op): @staticmethod def forward(ctx, *args, **kwargs): raise RuntimeError( - "DistributedUpdateLossScalingImpl's forward should not be called !") + "DistributedUpdateLossScalingImpl's forward should not be called !" + ) @staticmethod def backward(ctx, *args, **kwargs): @@ -71,59 +71,89 @@ def backward(ctx, *args, **kwargs): backward_op = dist_op_context.cur_src_op rank_id = dist_op_context.rank_id dist_attr = ctx.get_op_dist_attr_for_program(backward_op) - assert dist_attr is not None, "backward op [{}] don't have dist attribute !".format( - str(backward_op)) + assert ( + dist_attr is not None + ), "backward op [{}] don't have dist attribute !".format( + str(backward_op) + ) assert rank_id in dist_attr.process_mesh.processes assert 'X' in kwargs, "input [{}] is not given".format('X') assert 'FoundInfinite' in kwargs, "input [{}] is not given".format( - 'FoundInfinite') + 'FoundInfinite' + ) assert 'PrevLossScaling' in kwargs, "input [{}] is not given".format( - 'PrevLossScaling') + 'PrevLossScaling' + ) assert 'InGoodSteps' in kwargs, "input [{}] is not given".format( - 'InGoodSteps') + 'InGoodSteps' + ) assert 'InBadSteps' in kwargs, "input [{}] is not given".format( - 'InBadSteps') + 'InBadSteps' + ) assert 'Out' in kwargs, "output [{}] is not given".format('Out') assert 'LossScaling' in kwargs, "output [{}] is not given".format( - 'LossScaling') + 'LossScaling' + ) assert 'OutGoodSteps' in kwargs, "output [{}] is not given".format( - 'OutGoodSteps') + 'OutGoodSteps' + ) assert 'OutBadSteps' in kwargs, "output [{}] is not given".format( - 'OutBadSteps') - - assert len(kwargs['FoundInfinite']) == 1, \ - "update_loss_scaling input FoundInfinite take 1 variable but got {}".format( - kwargs['FoundInfinite']) - assert len(kwargs['PrevLossScaling']) == 1, \ - "update_loss_scaling input PrevLossScaling take 1 variable but got {}".format( - kwargs['PrevLossScaling']) - assert len(kwargs['InGoodSteps']) == 1, \ - "update_loss_scaling input InGoodSteps take 1 variable but got {}".format( - kwargs['InGoodSteps']) - assert len(kwargs['InBadSteps']) == 1, \ - "update_loss_scaling input InBadSteps take 1 variable but got {}".format( - kwargs['InBadSteps']) - assert len(kwargs['LossScaling']) == 1, \ - "update_loss_scaling output LossScaling take 1 variable but got {}".format( - kwargs['LossScaling']) - assert len(kwargs['OutGoodSteps']) == 1, \ - "update_loss_scaling output OutGoodSteps take 1 variable but got {}".format( - kwargs['OutGoodSteps']) - assert len(kwargs['OutBadSteps']) == 1, \ - "update_loss_scaling output OutBadSteps take 1 variable but got {}".format( - kwargs['OutBadSteps']) - - assert len(kwargs['X']) == len(kwargs['Out']), \ - "update_loss_scaling got [{}] X and [{}] Out, which are supposed to be equal".format( - len(kwargs['X']), len(kwargs['Out'])) + 'OutBadSteps' + ) + + assert ( + len(kwargs['FoundInfinite']) == 1 + ), "update_loss_scaling input FoundInfinite take 1 variable but got {}".format( + kwargs['FoundInfinite'] + ) + assert ( + len(kwargs['PrevLossScaling']) == 1 + ), "update_loss_scaling input PrevLossScaling take 1 variable but got {}".format( + kwargs['PrevLossScaling'] + ) + assert ( + len(kwargs['InGoodSteps']) == 1 + ), "update_loss_scaling input InGoodSteps take 1 variable but got {}".format( + kwargs['InGoodSteps'] + ) + assert ( + len(kwargs['InBadSteps']) == 1 + ), "update_loss_scaling input InBadSteps take 1 variable but got {}".format( + kwargs['InBadSteps'] + ) + assert ( + len(kwargs['LossScaling']) == 1 + ), "update_loss_scaling output LossScaling take 1 variable but got {}".format( + kwargs['LossScaling'] + ) + assert ( + len(kwargs['OutGoodSteps']) == 1 + ), "update_loss_scaling output OutGoodSteps take 1 variable but got {}".format( + kwargs['OutGoodSteps'] + ) + assert ( + len(kwargs['OutBadSteps']) == 1 + ), "update_loss_scaling output OutBadSteps take 1 variable but got {}".format( + kwargs['OutBadSteps'] + ) + + assert len(kwargs['X']) == len( + kwargs['Out'] + ), "update_loss_scaling got [{}] X and [{}] Out, which are supposed to be equal".format( + len(kwargs['X']), len(kwargs['Out']) + ) filter_vars = [] for varname in kwargs['X']: - if rank_id in ctx.get_tensor_dist_attr_for_program( - main_block.var(varname)).process_mesh.processes: + if ( + rank_id + in ctx.get_tensor_dist_attr_for_program( + main_block.var(varname) + ).process_mesh.processes + ): filter_vars.append(varname) # replicate op in dist program @@ -136,4 +166,5 @@ def backward(ctx, *args, **kwargs): register_distributed_operator_impl( "update_loss_scaling", - DistributedUpdateLossScalingImpl("update_loss_scaling")) + DistributedUpdateLossScalingImpl("update_loss_scaling"), +) diff --git a/python/paddle/distributed/auto_parallel/parallelizer.py b/python/paddle/distributed/auto_parallel/parallelizer.py index b25d1d8b33bf7..aad15642f1b3e 100644 --- a/python/paddle/distributed/auto_parallel/parallelizer.py +++ b/python/paddle/distributed/auto_parallel/parallelizer.py @@ -79,8 +79,12 @@ def __init__(self, fleet): self._pass_context = PassContext() self._need_rank_mapping = os.getenv("PADDLE_NEED_RANK_MAPPING") - self._need_rank_mapping = True if self._need_rank_mapping and \ - self._need_rank_mapping.lower() == 'true' else False + self._need_rank_mapping = ( + True + if self._need_rank_mapping + and self._need_rank_mapping.lower() == 'true' + else False + ) # self._pass_context = None def _remove_distributed_attrs(self, main_program): @@ -93,8 +97,9 @@ def _remove_distributed_attrs(self, main_program): if suffix in attr_name: op._remove_attr(attr_name) - def _apply_pre_optimization_passes(self, main_program, startup_program, - loss, params_grads, no_grad_set): + def _apply_pre_optimization_passes( + self, main_program, startup_program, loss, params_grads, no_grad_set + ): # apply amp pass if self._dist_strategy.amp: config = copy.deepcopy(self._dist_strategy.amp_configs) @@ -104,13 +109,15 @@ def _apply_pre_optimization_passes(self, main_program, startup_program, if config["use_pure_fp16"]: config["base_opt"] = self._optimizer auto_parallel_fp16_pass = new_pass("auto_parallel_fp16", config) - auto_parallel_fp16_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_fp16_pass.apply( + [main_program], [startup_program], self._pass_context + ) loss = auto_parallel_fp16_pass.get_loss() else: auto_parallel_amp_pass = new_pass("auto_parallel_amp", config) - auto_parallel_amp_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_amp_pass.apply( + [main_program], [startup_program], self._pass_context + ) loss = auto_parallel_amp_pass.get_loss() # apply recompute pass @@ -119,14 +126,22 @@ def _apply_pre_optimization_passes(self, main_program, startup_program, config["dist_context"] = self._dist_context config["no_grad_set"] = copy.deepcopy(no_grad_set) config["loss"] = loss - auto_parallel_recompute_pass = new_pass("auto_parallel_recompute", - config) - auto_parallel_recompute_pass.apply([main_program], - [startup_program], - self._pass_context) - - def _generate_backward(self, main_program, startup_program, loss, - parameter_list, no_grad_set, callbacks): + auto_parallel_recompute_pass = new_pass( + "auto_parallel_recompute", config + ) + auto_parallel_recompute_pass.apply( + [main_program], [startup_program], self._pass_context + ) + + def _generate_backward( + self, + main_program, + startup_program, + loss, + parameter_list, + no_grad_set, + callbacks, + ): with program_guard(main_program, startup_program): params_grads = append_backward( @@ -134,7 +149,8 @@ def _generate_backward(self, main_program, startup_program, loss, parameter_list, no_grad_set, callbacks, - distop_context=self._dist_context.dist_op_context) + distop_context=self._dist_context.dist_op_context, + ) self._completer = Completer(self._dist_context) self._completer.complete_backward_annotation(main_program) self._dist_context.block_state.parse_backward_blocks(main_program) @@ -153,18 +169,21 @@ def _apply_optimize(self, main_program, startup_program, params_grads): return optimize_ops - def _apply_post_optimization_passes(self, main_program, startup_program, - rank, params_grads): + def _apply_post_optimization_passes( + self, main_program, startup_program, rank, params_grads + ): if self._dist_strategy.sharding: config = copy.deepcopy(self._dist_strategy.sharding_configs) config["dist_context"] = self._dist_context config["params_grads"] = params_grads config["global_rank"] = rank - auto_parallel_sharding_pass = new_pass("auto_parallel_sharding", - config) - auto_parallel_sharding_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_sharding_pass = new_pass( + "auto_parallel_sharding", config + ) + auto_parallel_sharding_pass.apply( + [main_program], [startup_program], self._pass_context + ) params_grads = self._pass_context.get_attr("params_grads") config = copy.deepcopy(self._dist_strategy.sharding_configs) @@ -172,18 +191,20 @@ def _apply_post_optimization_passes(self, main_program, startup_program, config["params_grads"] = params_grads config["rank_id"] = rank auto_parallel_clip_pass = new_pass("auto_parallel_grad_clip", config) - auto_parallel_clip_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_clip_pass.apply( + [main_program], [startup_program], self._pass_context + ) if self._dist_strategy.gradient_merge: config = copy.deepcopy(self._dist_strategy.gradient_merge_configs) config["dist_context"] = self._dist_context config["params_grads"] = params_grads auto_parallel_gradient_merge_pass = new_pass( - "auto_parallel_gradient_merge_pass", config) - auto_parallel_gradient_merge_pass.apply([main_program], - [startup_program], - self._pass_context) + "auto_parallel_gradient_merge_pass", config + ) + auto_parallel_gradient_merge_pass.apply( + [main_program], [startup_program], self._pass_context + ) def _get_dist_program(self, rank, dist_context=None, relaunch_phase=False): completed_main_program = None @@ -197,8 +218,9 @@ def _get_dist_program(self, rank, dist_context=None, relaunch_phase=False): self._dist_context = DistributedContext() _logger.info("Start annotation dist attr.") self._completer = Completer(self._dist_context) - completed_main_program = self._completer.complete_forward_annotation( - serial_main_program) + completed_main_program = ( + self._completer.complete_forward_annotation(serial_main_program) + ) else: completed_main_program = serial_main_program self._dist_context = copy.deepcopy(dist_context) @@ -208,34 +230,54 @@ def _get_dist_program(self, rank, dist_context=None, relaunch_phase=False): # serial backward pass params_grads = self._generate_backward( - completed_main_program, serial_startup_program, serial_loss, - self._parameter_list, self._no_grad_set, self._callbacks) + completed_main_program, + serial_startup_program, + serial_loss, + self._parameter_list, + self._no_grad_set, + self._callbacks, + ) # serial forward pass - self._apply_pre_optimization_passes(completed_main_program, - serial_startup_program, serial_loss, - params_grads, self._no_grad_set) + self._apply_pre_optimization_passes( + completed_main_program, + serial_startup_program, + serial_loss, + params_grads, + self._no_grad_set, + ) # Logical partition partitioner = Partitioner(self._dist_context, rank) - dist_main_prog, dist_startup_prog, dist_params_grads = partitioner.partition( - completed_main_program, serial_startup_program, params_grads) + ( + dist_main_prog, + dist_startup_prog, + dist_params_grads, + ) = partitioner.partition( + completed_main_program, serial_startup_program, params_grads + ) # TODO refactor the placement of optimizer # generate optimize program - dist_optimize_ops = self._apply_optimize(dist_main_prog, - dist_startup_prog, - dist_params_grads) + dist_optimize_ops = self._apply_optimize( + dist_main_prog, dist_startup_prog, dist_params_grads + ) set_grad_var_shape(dist_main_prog, self._dist_context) make_data_unshard(dist_main_prog, dist_startup_prog, self._dist_context) - resharder = Resharder(dist_main_prog, dist_startup_prog, rank, - self._dist_context, dist_params_grads) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank, + self._dist_context, + dist_params_grads, + ) resharder.reshard() - self._apply_post_optimization_passes(dist_main_prog, dist_startup_prog, - rank, dist_params_grads) + self._apply_post_optimization_passes( + dist_main_prog, dist_startup_prog, rank, dist_params_grads + ) g_process_group_map = None if not relaunch_phase: g_process_group_map = copy.deepcopy(_g_process_group_map) @@ -243,14 +285,22 @@ def _get_dist_program(self, rank, dist_context=None, relaunch_phase=False): _g_process_group_map[0] = ProcessGroup(0, []) for process_mesh in self._dist_context._process_meshes: _g_process_group_map[0].add_ranks(process_mesh.processes) - return dist_optimize_ops, dist_params_grads, dist_startup_prog, dist_main_prog, g_process_group_map - - def parallelize(self, - loss, - startup_program, - parameter_list=None, - no_grad_set=None, - callbacks=None): + return ( + dist_optimize_ops, + dist_params_grads, + dist_startup_prog, + dist_main_prog, + g_process_group_map, + ) + + def parallelize( + self, + loss, + startup_program, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): assert startup_program is not None self._loss = loss self._startup_program = startup_program @@ -261,25 +311,27 @@ def parallelize(self, if self._enable_auto_mapping and self._need_rank_mapping: # Do the mapping pass before parallelization - assert self._cluster is not None, \ - "The cluster must not be none when using auto mapping." + assert ( + self._cluster is not None + ), "The cluster must not be none when using auto mapping." dist_programs = {} world_process_group = get_world_process_group() dist_context = None # auto search if self._dist_strategy.auto_search: logging.info("Start searching dist attr.") - serial_program_info = SerialProgramInfo(self._main_program, - self._startup_program, - self._loss, - self._optimizer, - self._cluster) - planner = Planner(serial_program_info, - self, - algorithm_config={ - "name": "mcmc", - "max_search_times": 5 - }) + serial_program_info = SerialProgramInfo( + self._main_program, + self._startup_program, + self._loss, + self._optimizer, + self._cluster, + ) + planner = Planner( + serial_program_info, + self, + algorithm_config={"name": "mcmc", "max_search_times": 5}, + ) dist_context, _ = planner.search() logging.info("End searching dist attr.") @@ -288,31 +340,42 @@ def parallelize(self, logging.info("Start serialize searched dist attr") cwd = pathlib.Path().resolve() searched_dist_context_path = os.path.join( - cwd, f"searched_dist_context_{time.time()}.pkl") + cwd, f"searched_dist_context_{time.time()}.pkl" + ) saved_dist_context = {} ops_dist_attr = {} tensors_dist_attr = {} for key, dist_op in dist_context._dist_ops_for_program.items(): ops_dist_attr[key] = dist_op.dist_attr - for key, dist_tensor in dist_context._dist_tensors_for_program.items( - ): + for ( + key, + dist_tensor, + ) in dist_context._dist_tensors_for_program.items(): tensors_dist_attr[key] = dist_tensor.dist_attr saved_dist_context["ops_dist_attr"] = ops_dist_attr saved_dist_context["tensors_dist_attr"] = tensors_dist_attr saved_dist_context[ - "process_meshes"] = dist_context._process_meshes - with open(searched_dist_context_path, - "wb") as dist_context_file: + "process_meshes" + ] = dist_context._process_meshes + with open( + searched_dist_context_path, "wb" + ) as dist_context_file: pickle.dump(saved_dist_context, dist_context_file) os.environ[ - 'PADDLE_SEARCHED_DIST_CONTEXT_PATH'] = searched_dist_context_path + 'PADDLE_SEARCHED_DIST_CONTEXT_PATH' + ] = searched_dist_context_path logging.info( f"End serialize searched dist attr to {searched_dist_context_path}" ) for rank in world_process_group.ranks: - dist_optimize_ops, dist_params_grads, dist_startup_prog, dist_main_prog, g_process_group_map = self._get_dist_program( - rank, dist_context) + ( + dist_optimize_ops, + dist_params_grads, + dist_startup_prog, + dist_main_prog, + g_process_group_map, + ) = self._get_dist_program(rank, dist_context) dist_programs[rank] = [dist_main_prog, g_process_group_map] # Do the mapping between the distributed program graph and the cluster graph @@ -324,27 +387,42 @@ def parallelize(self, json.dump(rank_mapping, rank_mapping_file) enable_elastic = os.getenv("PADDLE_ENABLE_ELASTIC") - enable_elastic = True if enable_elastic and enable_elastic.lower( - ) == 'true' else False + enable_elastic = ( + True + if enable_elastic and enable_elastic.lower() == 'true' + else False + ) if enable_elastic: print("Auto mapping finished, now do elastic re-launch") - sys.exit(paddle.distributed.fleet.elastic.manager. - ELASTIC_AUTO_PARALLEL_EXIT_CODE) + sys.exit( + paddle.distributed.fleet.elastic.manager.ELASTIC_AUTO_PARALLEL_EXIT_CODE + ) original_cmd_args = os.getenv("PADDLE_ORIGINAL_CMD_ARGS") rank_mapping_args = " ".join( - ["--rank_mapping_path", self._rank_mapping_path]) + ["--rank_mapping_path", self._rank_mapping_path] + ) if os.environ.get("WITH_COVERAGE", "OFF") == "ON": coverage_args = ["-m", "coverage", "run", "--branch", "-p"] else: coverage_args = [] - new_cmd_args = "-m paddle.distributed.fleet.launch" + " " + rank_mapping_args + " " + original_cmd_args - new_cmd = [sys.executable, "-u" - ] + coverage_args + shlex.split(new_cmd_args) + new_cmd_args = ( + "-m paddle.distributed.fleet.launch" + + " " + + rank_mapping_args + + " " + + original_cmd_args + ) + new_cmd = ( + [sys.executable, "-u"] + + coverage_args + + shlex.split(new_cmd_args) + ) new_process = subprocess.Popen(new_cmd) new_process.wait() - assert new_process.returncode == 0, \ - "Launch failed with rank mapping" + assert ( + new_process.returncode == 0 + ), "Launch failed with rank mapping" print("Successfully do the second launch for auto mapping!") sys.exit(0) else: @@ -352,27 +430,32 @@ def parallelize(self, rank = paddle.distributed.get_rank() dist_context = None searched_dist_context_path = os.getenv( - "PADDLE_SEARCHED_DIST_CONTEXT_PATH", None) + "PADDLE_SEARCHED_DIST_CONTEXT_PATH", None + ) if searched_dist_context_path is not None: - with open(searched_dist_context_path, - "rb") as dist_context_file: + with open( + searched_dist_context_path, "rb" + ) as dist_context_file: saved_dist_context = pickle.load(dist_context_file) dist_context = DistributedContext() for op in self._main_program.global_block().ops: dist_attr = saved_dist_context["ops_dist_attr"][ - op.desc.id()] + op.desc.id() + ] dist_op = DistributedOperator(op, dist_attr) dist_context.add_dist_op_for_program(dist_op) vars = self._main_program.global_block().vars for var in vars.values(): dist_attr = saved_dist_context["tensors_dist_attr"][ - var.desc.id()] + var.desc.id() + ] dist_tensor = DistributedTensor(var, dist_attr) dist_context.add_dist_tensor_for_program(dist_tensor) dist_context._process_meshes = saved_dist_context[ - "process_meshes"] + "process_meshes" + ] else: if self._dist_strategy.auto_search: @@ -381,13 +464,16 @@ def parallelize(self, self._startup_program, self._loss, self._optimizer, - cluster=self._cluster) - planner = Planner(serial_program_info, - self, - algorithm_config={ - "name": "mcmc", - "max_search_times": 5 - }) + cluster=self._cluster, + ) + planner = Planner( + serial_program_info, + self, + algorithm_config={ + "name": "mcmc", + "max_search_times": 5, + }, + ) dist_context, _ = planner.search() # rebuild g_process_group @@ -395,8 +481,13 @@ def parallelize(self, pg0 = get_process_group(0) for process_mesh in dist_context._process_meshes: pg0.add_ranks(process_mesh.processes) - dist_optimize_ops, dist_params_grads, dist_startup_prog, dist_main_prog, _ = self._get_dist_program( - rank, dist_context, relaunch_phase=True) + ( + dist_optimize_ops, + dist_params_grads, + dist_startup_prog, + dist_main_prog, + _, + ) = self._get_dist_program(rank, dist_context, relaunch_phase=True) # NOTE: This is a trick to fix hang in pipeline mode when dist context is searched by planner if self._dist_strategy.auto_search: @@ -424,14 +515,25 @@ def parallelize(self, # with inference. self._remove_distributed_attrs(dist_main_prog) - return dist_optimize_ops, dist_params_grads, dist_startup_prog, dist_main_prog + return ( + dist_optimize_ops, + dist_params_grads, + dist_startup_prog, + dist_main_prog, + ) def __deepcopy__(self, memo): cls = self.__class__ result = cls.__new__(cls) memo[id(self)] = result for k, v in self.__dict__.items(): - if k == "_main_program" or k == "_startup_program" or k == "_dist_context" or k == "_fleet" or k == "_loss": + if ( + k == "_main_program" + or k == "_startup_program" + or k == "_dist_context" + or k == "_fleet" + or k == "_loss" + ): setattr(result, k, v) else: setattr(result, k, copy.deepcopy(v, memo)) diff --git a/python/paddle/distributed/auto_parallel/parallelizer_v2.py b/python/paddle/distributed/auto_parallel/parallelizer_v2.py index d6bb5efc8eb91..7a7ee603e9c61 100644 --- a/python/paddle/distributed/auto_parallel/parallelizer_v2.py +++ b/python/paddle/distributed/auto_parallel/parallelizer_v2.py @@ -29,7 +29,6 @@ class Parallelizer: - def __init__(self, mode, completer, dist_context): self._mode = mode self._completer = completer @@ -54,73 +53,123 @@ def parallel(self, rank): if self._mode == "train" and serial_optimizer: # Generate backward serial_loss = self._dist_context.serial_loss - params_grads = self._generate_backward(serial_main_program, - serial_startup_program, - serial_loss) + params_grads = self._generate_backward( + serial_main_program, serial_startup_program, serial_loss + ) # Apply pre optimization passes time0 = time.time() - serial_main_program, serial_startup_program, params_grads = self._apply_pre_optimization( - serial_main_program, serial_startup_program, serial_loss, - serial_optimizer, params_grads) + ( + serial_main_program, + serial_startup_program, + params_grads, + ) = self._apply_pre_optimization( + serial_main_program, + serial_startup_program, + serial_loss, + serial_optimizer, + params_grads, + ) self._logger.debug( - "within parallel apply_pre_optimization time: {}, mode {}". - format(time.time() - time0, self._mode)) + "within parallel apply_pre_optimization time: {}, mode {}".format( + time.time() - time0, self._mode + ) + ) # Do logical partition time0 = time.time() partitioner = Partitioner(self._dist_context, rank) - dist_main_prog, dist_startup_prog, dist_params_grads = partitioner.partition( - serial_main_program, serial_startup_program, params_grads) + ( + dist_main_prog, + dist_startup_prog, + dist_params_grads, + ) = partitioner.partition( + serial_main_program, serial_startup_program, params_grads + ) self._logger.debug( "within parallel partitioner time: {}, mode {}".format( - time.time() - time0, self._mode)) + time.time() - time0, self._mode + ) + ) # Generate optimizer time0 = time.time() - self._generate_optimizer(dist_main_prog, dist_startup_prog, - serial_optimizer, dist_params_grads) + self._generate_optimizer( + dist_main_prog, + dist_startup_prog, + serial_optimizer, + dist_params_grads, + ) self._logger.debug( "within parallel optimizer time: {}, mode {}".format( - time.time() - time0, self._mode)) + time.time() - time0, self._mode + ) + ) # Do reshard process time0 = time.time() set_grad_var_shape(dist_main_prog, self._dist_context) - resharder = Resharder(dist_main_prog, dist_startup_prog, rank, - self._dist_context, dist_params_grads) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank, + self._dist_context, + dist_params_grads, + ) resharder.reshard() self._logger.debug( "within parallel reshard time: {}, mode {}".format( - time.time() - time0, self._mode)) + time.time() - time0, self._mode + ) + ) # Apply post optimization passes time0 = time.time() - self._apply_post_optimization(dist_main_prog, dist_startup_prog, - rank, dist_params_grads) + self._apply_post_optimization( + dist_main_prog, dist_startup_prog, rank, dist_params_grads + ) self._logger.debug( - "within parallel apply_post_optimization time: {}, mode {}". - format(time.time() - time0, self._mode)) + "within parallel apply_post_optimization time: {}, mode {}".format( + time.time() - time0, self._mode + ) + ) else: # Apply pre optimization passes time0 = time.time() - self._apply_pre_optimization(serial_main_program, - serial_startup_program, None, None, - None) + self._apply_pre_optimization( + serial_main_program, serial_startup_program, None, None, None + ) self._logger.debug( - "within parallel apply_pre_optimization time: {}, mode {}". - format(time.time() - time0, self._mode)) + "within parallel apply_pre_optimization time: {}, mode {}".format( + time.time() - time0, self._mode + ) + ) # Do logical partition time0 = time.time() partitioner = Partitioner(self._dist_context, rank) - dist_main_prog, dist_startup_prog, dist_params_grads = partitioner.partition( - serial_main_program, serial_startup_program, []) + ( + dist_main_prog, + dist_startup_prog, + dist_params_grads, + ) = partitioner.partition( + serial_main_program, serial_startup_program, [] + ) # Do reshard process self._logger.debug( "within parallel partitioner time: {}, mode {}".format( - time.time() - time0, self._mode)) + time.time() - time0, self._mode + ) + ) time0 = time.time() - resharder = Resharder(dist_main_prog, dist_startup_prog, rank, - self._dist_context, [], 1) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank, + self._dist_context, + [], + 1, + ) resharder.reshard() self._logger.debug( "within parallel reshard time: {}, mode {}".format( - time.time() - time0, self._mode)) + time.time() - time0, self._mode + ) + ) # Clone program for test if self._mode != 'train': dist_main_prog = dist_main_prog.clone(for_test=True) @@ -133,13 +182,15 @@ def parallel(self, rank): def _generate_backward(self, main_program, startup_program, loss): with program_guard(main_program, startup_program): params_grads = append_backward( - loss, distop_context=self._dist_context.dist_op_context) + loss, distop_context=self._dist_context.dist_op_context + ) self._completer.complete_backward_annotation(main_program) self._dist_context.block_state.parse_backward_blocks(main_program) return params_grads - def _generate_optimizer(self, main_program, startup_program, optimizer, - params_grads): + def _generate_optimizer( + self, main_program, startup_program, optimizer, params_grads + ): # NOTE: `apply_gradients` will add an Accumulator for a parameter only once, # but optimizer will be called repeatedly in re-launch, so optimizer need to be copied. optimizer = copy.deepcopy(optimizer) @@ -150,8 +201,9 @@ def _generate_optimizer(self, main_program, startup_program, optimizer, self._completer.complete_update_annotation(main_program) return optimizer_ops - def _apply_pre_optimization(self, main_program, startup_program, loss, - optimizer, params_grads): + def _apply_pre_optimization( + self, main_program, startup_program, loss, optimizer, params_grads + ): if self._strategy is None: return @@ -162,10 +214,11 @@ def _apply_pre_optimization(self, main_program, startup_program, loss, config["dist_context"] = self._dist_context config["params_grads"] = params_grads auto_parallel_quantization_pass = new_pass( - "auto_parallel_quantization", config) - auto_parallel_quantization_pass.apply([main_program], - [startup_program], - self._pass_context) + "auto_parallel_quantization", config + ) + auto_parallel_quantization_pass.apply( + [main_program], [startup_program], self._pass_context + ) main_program = self._pass_context.get_attr("main_program") startup_program = self._pass_context.get_attr("startup_program") params_grads = self._pass_context.get_attr("params_grads") @@ -176,18 +229,22 @@ def _apply_pre_optimization(self, main_program, startup_program, loss, config["dist_context"] = self._dist_context config["params_grads"] = params_grads config["loss"] = loss - config["input_data"] = self._dist_context.serial_feed_vars["inputs"] \ + config["input_data"] = ( + self._dist_context.serial_feed_vars["inputs"] + self._dist_context.serial_feed_vars["labels"] + ) if config["use_pure_fp16"]: config["base_opt"] = optimizer auto_parallel_fp16_pass = new_pass("auto_parallel_fp16", config) - auto_parallel_fp16_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_fp16_pass.apply( + [main_program], [startup_program], self._pass_context + ) loss = auto_parallel_fp16_pass.get_loss() else: auto_parallel_amp_pass = new_pass("auto_parallel_amp", config) - auto_parallel_amp_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_amp_pass.apply( + [main_program], [startup_program], self._pass_context + ) loss = auto_parallel_amp_pass.get_loss() # apply recompute pass @@ -197,16 +254,18 @@ def _apply_pre_optimization(self, main_program, startup_program, loss, config["dist_context"] = self._dist_context config["no_grad_set"] = None config["loss"] = loss - auto_parallel_recompute_pass = new_pass("auto_parallel_recompute", - config) - auto_parallel_recompute_pass.apply([main_program], - [startup_program], - self._pass_context) + auto_parallel_recompute_pass = new_pass( + "auto_parallel_recompute", config + ) + auto_parallel_recompute_pass.apply( + [main_program], [startup_program], self._pass_context + ) return main_program, startup_program, params_grads - def _apply_post_optimization(self, main_program, startup_program, rank, - params_grads): + def _apply_post_optimization( + self, main_program, startup_program, rank, params_grads + ): if self._strategy is None: return @@ -223,10 +282,12 @@ def _apply_post_optimization(self, main_program, startup_program, rank, config["dist_context"] = self._dist_context config["params_grads"] = params_grads config["global_rank"] = rank - auto_parallel_sharding_pass = new_pass("auto_parallel_sharding", - config) - auto_parallel_sharding_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_sharding_pass = new_pass( + "auto_parallel_sharding", config + ) + auto_parallel_sharding_pass.apply( + [main_program], [startup_program], self._pass_context + ) params_grads = self._pass_context.get_attr("params_grads") # GradClip is train-only optimization @@ -235,10 +296,12 @@ def _apply_post_optimization(self, main_program, startup_program, rank, config["dist_context"] = self._dist_context config["params_grads"] = params_grads config["rank_id"] = rank - auto_parallel_clip_pass = new_pass("auto_parallel_grad_clip", - config) - auto_parallel_clip_pass.apply([main_program], [startup_program], - self._pass_context) + auto_parallel_clip_pass = new_pass( + "auto_parallel_grad_clip", config + ) + auto_parallel_clip_pass.apply( + [main_program], [startup_program], self._pass_context + ) # gradient_merge is then train-only optimization if self._mode == "train" and self._strategy.gradient_merge.enable: @@ -246,7 +309,8 @@ def _apply_post_optimization(self, main_program, startup_program, rank, config["dist_context"] = self._dist_context config["params_grads"] = params_grads auto_parallel_gradient_merge_pass = new_pass( - "auto_parallel_gradient_merge_pass", config) - auto_parallel_gradient_merge_pass.apply([main_program], - [startup_program], - self._pass_context) + "auto_parallel_gradient_merge_pass", config + ) + auto_parallel_gradient_merge_pass.apply( + [main_program], [startup_program], self._pass_context + ) diff --git a/python/paddle/distributed/auto_parallel/partitioner.py b/python/paddle/distributed/auto_parallel/partitioner.py index 32917627672d5..2ba3398132437 100644 --- a/python/paddle/distributed/auto_parallel/partitioner.py +++ b/python/paddle/distributed/auto_parallel/partitioner.py @@ -17,7 +17,9 @@ from paddle.fluid import core from paddle.fluid import core from paddle.fluid.framework import Parameter, Program -from paddle.distributed.auto_parallel.operators.common import get_distributed_operator_impl_container +from paddle.distributed.auto_parallel.operators.common import ( + get_distributed_operator_impl_container, +) from paddle.distributed.auto_parallel.dist_context import DistributedContext from .dist_attribute import OperatorDistributedAttribute from .utils import is_backward_op, is_forward_op, is_loss_op, is_optimize_op @@ -25,7 +27,8 @@ __varname_not_in_block__ = ["lod_tensor_blocking_queue"] __not_shape_var_type__ = [ - core.VarDesc.VarType.READER, core.VarDesc.VarType.STEP_SCOPES + core.VarDesc.VarType.READER, + core.VarDesc.VarType.STEP_SCOPES, ] @@ -51,25 +54,29 @@ def __init__(self, dist_context, rank_id=0): """ if not isinstance(dist_context, DistributedContext): raise TypeError( - "dist_context be paddle.fluid.DistributedContext, got %s here" % - type(dist_context)) + "dist_context be paddle.fluid.DistributedContext, got %s here" + % type(dist_context) + ) self._dist_context = dist_context self._rank_id = rank_id self._serial2dist_varname_mapping = {} self._dist_varname_suffix = "" - def partition(self, serial_main_program, serial_startup_program, - params_grads): + def partition( + self, serial_main_program, serial_startup_program, params_grads + ): if not isinstance(serial_main_program, (Program)): raise TypeError( - "main_program be paddle.fluid.framework.program, got %s here" % - type(serial_main_program)) + "main_program be paddle.fluid.framework.program, got %s here" + % type(serial_main_program) + ) # check if shard annotated serial program valid if not self._is_valid_annotated_program(serial_main_program): raise RuntimeError( - "Not all vars or ops are annotated in main program !") + "Not all vars or ops are annotated in main program !" + ) # init distop helper dist_op_context = self._dist_context.dist_op_context @@ -81,22 +88,31 @@ def partition(self, serial_main_program, serial_startup_program, partitioned_startup_prog = None else: partitioned_startup_prog = self.partition_startup_program( - serial_main_program, serial_startup_program) + serial_main_program, serial_startup_program + ) dist_op_context.dst_startup_program = partitioned_startup_prog # partition main program - partitioned_main_prog, partitioned_params_grads = self.partition_main_program( - serial_main_program, params_grads) + ( + partitioned_main_prog, + partitioned_params_grads, + ) = self.partition_main_program(serial_main_program, params_grads) - return partitioned_main_prog, partitioned_startup_prog, partitioned_params_grads + return ( + partitioned_main_prog, + partitioned_startup_prog, + partitioned_params_grads, + ) - def partition_startup_program(self, serial_main_program, - serial_startup_program): + def partition_startup_program( + self, serial_main_program, serial_startup_program + ): if not isinstance(serial_startup_program, (Program)): raise TypeError( - "dist_context be paddle.fluid.framework.program, got %s here" % - type(serial_startup_program)) + "dist_context be paddle.fluid.framework.program, got %s here" + % type(serial_startup_program) + ) partitioned_startup_prog = fluid.Program() ref_block = serial_main_program.global_block() @@ -109,27 +125,33 @@ def partition_startup_program(self, serial_main_program, assert var.persistable new_name = var.name + self._dist_varname_suffix temp_varname_map[var.name] = new_name - target_shape = _partition_var(self._dist_context, ref_block, - target_block, var.name, new_name) + target_shape = _partition_var( + self._dist_context, ref_block, target_block, var.name, new_name + ) var2shape[new_name] = target_shape # ops for op in serial_startup_program.global_block().ops: # TODO if var not belong to this rank, should be filtered output_vars = op.desc.output_arg_names() - assert len( - output_vars - ) == 1, "initializer should output only ONE variable, but got [{}]".format( - str(op.desc)) - assert temp_varname_map[output_vars[ - 0]] in var2shape, "try to initialize [{}] which is not a persistable var".format( - output_vars[0]) + assert ( + len(output_vars) == 1 + ), "initializer should output only ONE variable, but got [{}]".format( + str(op.desc) + ) + assert ( + temp_varname_map[output_vars[0]] in var2shape + ), "try to initialize [{}] which is not a persistable var".format( + output_vars[0] + ) new_op_desc = target_block.desc.append_op() new_op_desc.copy_from(op.desc) - new_op_desc._rename_output(output_vars[0], - temp_varname_map[output_vars[0]]) - new_op_desc._set_attr("shape", - var2shape[temp_varname_map[output_vars[0]]]) + new_op_desc._rename_output( + output_vars[0], temp_varname_map[output_vars[0]] + ) + new_op_desc._set_attr( + "shape", var2shape[temp_varname_map[output_vars[0]]] + ) target_block._sync_with_cpp() # set distribute atrribute @@ -137,14 +159,17 @@ def partition_startup_program(self, serial_main_program, assert new_op.type == new_op_desc.type() assert new_op.desc == new_op_desc output_var = target_block.var(output_vars[0]) - output_var_attr = self._dist_context.get_tensor_dist_attr_for_program( - output_var) + output_var_attr = ( + self._dist_context.get_tensor_dist_attr_for_program(output_var) + ) op_attr = OperatorDistributedAttribute() op_attr.process_mesh = output_var_attr.process_mesh - op_attr.set_output_dims_mapping(output_var.name, - output_var_attr.dims_mapping) - op_attr.set_input_dims_mapping(output_var.name, - output_var_attr.dims_mapping) + op_attr.set_output_dims_mapping( + output_var.name, output_var_attr.dims_mapping + ) + op_attr.set_input_dims_mapping( + output_var.name, output_var_attr.dims_mapping + ) self._dist_context.set_op_dist_attr_for_program(new_op, op_attr) return partitioned_startup_prog @@ -166,7 +191,8 @@ def partition_main_program(self, serial_main_program, params_and_grads): target_block = partitioned_main_prog.blocks[0] else: target_block = partitioned_main_prog._create_block( - parent_idx=ref_block.parent_idx) + parent_idx=ref_block.parent_idx + ) assert ref_block.idx == target_block.idx target_block._set_forward_block_idx(ref_block.forward_block_idx) dist_op_context.work_block = target_block @@ -181,8 +207,9 @@ def partition_main_program(self, serial_main_program, params_and_grads): for attr_name in op.all_attrs(): if op.attr_type(attr_name) == core.AttrType.BLOCK: relative_id = op._block_attr_id(attr_name) - op._set_attr(attr_name, - partitioned_main_prog.block(relative_id)) + op._set_attr( + attr_name, partitioned_main_prog.block(relative_id) + ) partitioned_params_and_grads = [] for p, g in params_and_grads: @@ -193,7 +220,8 @@ def partition_main_program(self, serial_main_program, params_and_grads): else: assert g.name in self._serial2dist_varname_mapping dist_g = self._get_dist_var_by_serial_var( - g, partitioned_main_prog) + g, partitioned_main_prog + ) partitioned_params_and_grads.append((dist_p, dist_g)) return partitioned_main_prog, partitioned_params_and_grads @@ -217,71 +245,112 @@ def partition_block(self, ref_block, target_block): for idx in range(len(serial_ops)): if idx <= last_fwd_op_idx: forward_op_id2forward_op[ - serial_ops[idx].desc.original_id()] = serial_ops[idx] + serial_ops[idx].desc.original_id() + ] = serial_ops[idx] # partiiton appended_grad_times = 0 for idx, op in enumerate(serial_ops): op_dist_attr = self._dist_context.get_op_dist_attr_for_program(op) - if is_backward_op(op) and (is_forward_op(serial_ops[idx - 1]) - or is_loss_op(serial_ops[idx - 1])): + if is_backward_op(op) and ( + is_forward_op(serial_ops[idx - 1]) + or is_loss_op(serial_ops[idx - 1]) + ): if not op_dist_attr.is_recompute: appended_grad_times += 1 # partititon input variables for serial_input_varname in op.desc.input_arg_names(): - if serial_input_varname not in self._serial2dist_varname_mapping: - new_varname = serial_input_varname + self._dist_varname_suffix + if ( + serial_input_varname + not in self._serial2dist_varname_mapping + ): + new_varname = ( + serial_input_varname + self._dist_varname_suffix + ) if ref_block.has_var(serial_input_varname): - _partition_var(self._dist_context, ref_block, - target_block, serial_input_varname, - new_varname) + _partition_var( + self._dist_context, + ref_block, + target_block, + serial_input_varname, + new_varname, + ) else: for varname_not_in_block in __varname_not_in_block__: - assert varname_not_in_block in serial_input_varname, \ - "{} is not found".format(serial_input_varname) + assert ( + varname_not_in_block in serial_input_varname + ), "{} is not found".format(serial_input_varname) self._serial2dist_varname_mapping[ - serial_input_varname] = new_varname + serial_input_varname + ] = new_varname # partition output vars for serial_output_varname in op.desc.output_arg_names(): - if serial_output_varname not in self._serial2dist_varname_mapping: - new_varname = serial_output_varname + self._dist_varname_suffix - _partition_var(self._dist_context, ref_block, target_block, - serial_output_varname, new_varname) + if ( + serial_output_varname + not in self._serial2dist_varname_mapping + ): + new_varname = ( + serial_output_varname + self._dist_varname_suffix + ) + _partition_var( + self._dist_context, + ref_block, + target_block, + serial_output_varname, + new_varname, + ) self._serial2dist_varname_mapping[ - serial_output_varname] = new_varname + serial_output_varname + ] = new_varname # partition op if is_forward_op(op) or op_dist_attr.is_recompute: kinputs, koutputs = dist_op_context.prepare_context(op) dist_op_forward_impl = _get_dist_op_forward_implement( - op, self._dist_context) - dist_op_forward_impl.forward(self._dist_context, **kinputs, - **koutputs) + op, self._dist_context + ) + dist_op_forward_impl.forward( + self._dist_context, **kinputs, **koutputs + ) elif is_backward_op(op): kinputs, koutputs = dist_op_context.prepare_context(op) dist_op_backward_impl = _get_dist_op_backward_implement( - op, self._dist_context, forward_op_id2forward_op) - grad_var_to_var = self._dist_context.dist_op_context.grad_var_to_var[ - appended_grad_times] + op, self._dist_context, forward_op_id2forward_op + ) + grad_var_to_var = ( + self._dist_context.dist_op_context.grad_var_to_var[ + appended_grad_times + ] + ) dist_op_backward_impl.backward( - self._dist_context, **kinputs, **koutputs, - **{"grad_var_to_var": grad_var_to_var}) + self._dist_context, + **kinputs, + **koutputs, + **{"grad_var_to_var": grad_var_to_var} + ) elif is_optimize_op(op): # NOTE: BACKWARD_ONLY_DIST_OPS's op_role must 2 because of 1F1B PASS kinputs, koutputs = dist_op_context.prepare_context(op) dist_op_opt_impl = _get_dist_op_backward_implement( - op, self._dist_context, forward_op_id2forward_op) - dist_op_opt_impl.backward(self._dist_context, **kinputs, - **koutputs, **{"grad_var_to_var": {}}) + op, self._dist_context, forward_op_id2forward_op + ) + dist_op_opt_impl.backward( + self._dist_context, + **kinputs, + **koutputs, + **{"grad_var_to_var": {}} + ) else: raise NotImplementedError( - "partitioner only support forward and backward, optimize ops, but got {}" - .format(str(op))) + "partitioner only support forward and backward, optimize ops, but got {}".format( + str(op) + ) + ) def _is_valid_annotated_program(self, program): @@ -293,13 +362,16 @@ def _is_valid_annotated_program(self, program): ] var_dist_attrs = [ self._dist_context.get_tensor_dist_attr_for_program(var) - for var in vars_ if (var.type not in __not_shape_var_type__) + for var in vars_ + if (var.type not in __not_shape_var_type__) ] - all_ops_annotated = all(dist_attr is not None - for dist_attr in op_dist_attrs) - all_vars_annotated = all(dist_attr is not None - for dist_attr in var_dist_attrs) + all_ops_annotated = all( + dist_attr is not None for dist_attr in op_dist_attrs + ) + all_vars_annotated = all( + dist_attr is not None for dist_attr in var_dist_attrs + ) return all_ops_annotated and all_vars_annotated @@ -323,22 +395,26 @@ def _get_dist_shape(var, dist_attr): assert len(var_shape) == len( mapping ), "variable shape [{}] and dim_mapping [{}] is NOT match !".format( - var_shape, mapping) + var_shape, mapping + ) new_shape = [] for idx in range(len(var_shape)): if var_shape[idx] == -1 or mapping[idx] == -1: new_shape.append(var_shape[idx]) else: - assert var_shape[idx] % mesh[mapping[ - idx]] == 0, "un-event partition: var_shape[idx]=[{}], mesh[{}]".format( - var_shape[idx], mesh[mapping[idx]]) + assert ( + var_shape[idx] % mesh[mapping[idx]] == 0 + ), "un-event partition: var_shape[idx]=[{}], mesh[{}]".format( + var_shape[idx], mesh[mapping[idx]] + ) new_shape.append(var_shape[idx] // mesh[mapping[idx]]) return new_shape -def _partition_parameter(dist_context, src_var, dst_block, dst_varname, - dst_shape): +def _partition_parameter( + dist_context, src_var, dst_block, dst_varname, dst_shape +): # NOTE hack to copied Parameter # not initialized parameter, need to initialize it copied_kwargs = {} @@ -348,39 +424,45 @@ def _partition_parameter(dist_context, src_var, dst_block, dst_varname, copied_kwargs['do_model_average'] = src_var.do_model_average copied_kwargs['need_clip'] = src_var.need_clip - param = Parameter(block=dst_block, - type=src_var.type, - name=dst_varname, - shape=dst_shape, - dtype=src_var.dtype, - lod_level=src_var.lod_level, - error_clip=src_var.error_clip, - stop_gradient=src_var.stop_gradient, - is_data=src_var.is_data, - belong_to_optimizer=src_var.belong_to_optimizer, - **copied_kwargs) + param = Parameter( + block=dst_block, + type=src_var.type, + name=dst_varname, + shape=dst_shape, + dtype=src_var.dtype, + lod_level=src_var.lod_level, + error_clip=src_var.error_clip, + stop_gradient=src_var.stop_gradient, + is_data=src_var.is_data, + belong_to_optimizer=src_var.belong_to_optimizer, + **copied_kwargs + ) return param -def _partition_intermediate_var(dist_context, src_var, dst_block, dst_varname, - dst_shape): - var = dst_block.create_var(type=src_var.type, - name=dst_varname, - shape=dst_shape, - dtype=src_var.dtype, - lod_level=src_var.lod_level, - persistable=src_var.persistable, - error_clip=src_var.error_clip, - stop_gradient=src_var.stop_gradient, - is_data=src_var.is_data, - belong_to_optimizer=src_var.belong_to_optimizer) +def _partition_intermediate_var( + dist_context, src_var, dst_block, dst_varname, dst_shape +): + var = dst_block.create_var( + type=src_var.type, + name=dst_varname, + shape=dst_shape, + dtype=src_var.dtype, + lod_level=src_var.lod_level, + persistable=src_var.persistable, + error_clip=src_var.error_clip, + stop_gradient=src_var.stop_gradient, + is_data=src_var.is_data, + belong_to_optimizer=src_var.belong_to_optimizer, + ) return var -def _partition_var(dist_context, src_block, dst_block, src_varname, - dst_varname): +def _partition_var( + dist_context, src_block, dst_block, src_varname, dst_varname +): """ partition include: split + replicate """ @@ -388,44 +470,53 @@ def _partition_var(dist_context, src_block, dst_block, src_varname, if src_var.type in __not_shape_var_type__: persist = getattr(src_var, 'persistable', False) - new_var = dst_block.create_var(type=src_var.type, - name=dst_varname, - persistable=persist, - stop_gradient=True) + new_var = dst_block.create_var( + type=src_var.type, + name=dst_varname, + persistable=persist, + stop_gradient=True, + ) target_shape = None else: dist_attr = dist_context.get_tensor_dist_attr_for_program(src_var) target_shape = _get_dist_shape(src_var, dist_attr) if isinstance(src_var, Parameter): - new_var = _partition_parameter(dist_context, src_var, dst_block, - dst_varname, target_shape) + new_var = _partition_parameter( + dist_context, src_var, dst_block, dst_varname, target_shape + ) else: - new_var = _partition_intermediate_var(dist_context, src_var, - dst_block, dst_varname, - target_shape) + new_var = _partition_intermediate_var( + dist_context, src_var, dst_block, dst_varname, target_shape + ) dist_attr = copy.deepcopy( - dist_context.get_tensor_dist_attr_for_program(src_var)) + dist_context.get_tensor_dist_attr_for_program(src_var) + ) assert dist_attr is not None dist_context.set_tensor_dist_attr_for_program(new_var, dist_attr) return target_shape -def _get_dist_op_backward_implement(backward_op, dist_context, - forward_op_id2forward_op): +def _get_dist_op_backward_implement( + backward_op, dist_context, forward_op_id2forward_op +): dist_op_context = dist_context.dist_op_context if backward_op.desc.original_id() in dist_op_context.grad_op_id_to_op_id: forward_op_id = dist_op_context.grad_op_id_to_op_id[ - backward_op.desc.original_id()] + backward_op.desc.original_id() + ] forward_op = forward_op_id2forward_op[forward_op_id] forward_op_dist_attr = dist_context.get_op_dist_attr_for_program( - forward_op) + forward_op + ) dist_op_impl_container = get_distributed_operator_impl_container( - forward_op_dist_attr.impl_type) + forward_op_dist_attr.impl_type + ) dist_op_impl = dist_op_impl_container.get_impl( - forward_op_dist_attr.impl_idx) + forward_op_dist_attr.impl_idx + ) return dist_op_impl # # NOTE trick for dist ops that only have backward implement @@ -433,7 +524,8 @@ def _get_dist_op_backward_implement(backward_op, dist_context, op_dist_attr = dist_context.get_op_dist_attr_for_program(backward_op) assert op_dist_attr.impl_idx >= 0 dist_op_impl = get_distributed_operator_impl_container( - op_dist_attr.impl_type).get_impl(op_dist_attr.impl_idx) + op_dist_attr.impl_type + ).get_impl(op_dist_attr.impl_idx) return dist_op_impl dist_op = get_distributed_operator_impl_container("default") @@ -443,6 +535,7 @@ def _get_dist_op_backward_implement(backward_op, dist_context, def _get_dist_op_forward_implement(forward_op, dist_context): dist_attr = dist_context.get_op_dist_attr_for_program(forward_op) dist_op_impl_container = get_distributed_operator_impl_container( - dist_attr.impl_type) + dist_attr.impl_type + ) dist_op_impl = dist_op_impl_container.get_impl(dist_attr.impl_idx) return dist_op_impl diff --git a/python/paddle/distributed/auto_parallel/planner.py b/python/paddle/distributed/auto_parallel/planner.py index d01fe50c0d41c..df7fd63a6ed3d 100755 --- a/python/paddle/distributed/auto_parallel/planner.py +++ b/python/paddle/distributed/auto_parallel/planner.py @@ -32,7 +32,10 @@ from .utils import update_op_dims_mapping_by_elementwise_like_dist_impl from .utils import get_all_distributed_main_program from .dist_context import DistributedContext, DistributedOperatorContext -from .dist_attribute import OperatorDistributedAttribute, TensorDistributedAttribute +from .dist_attribute import ( + OperatorDistributedAttribute, + TensorDistributedAttribute, +) paddle.seed(123) random.seed(123) @@ -40,18 +43,19 @@ class PlanFilter: - @staticmethod - def check_dims_mapping_for_tensor(process_mesh_topology, tensor_shape, - dims_mapping): + def check_dims_mapping_for_tensor( + process_mesh_topology, tensor_shape, dims_mapping + ): valid = True assert len(tensor_shape) == len(dims_mapping) for idx, dim_mapping in enumerate(dims_mapping): if dim_mapping != -1: - if tensor_shape[idx] % process_mesh_topology[ - dim_mapping] != 0 or dims_mapping.count( - dim_mapping) > 1: + if ( + tensor_shape[idx] % process_mesh_topology[dim_mapping] != 0 + or dims_mapping.count(dim_mapping) > 1 + ): valid = False if dim_mapping != -1 and process_mesh_topology[0] == 1: valid = False @@ -65,7 +69,8 @@ def check_dims_mapping_for_op(op, op_dist_attr, vars): for var_name in op.input_arg_names: dims_mapping = op_dist_attr.get_input_dims_mapping(var_name) if not PlanFilter.check_dims_mapping_for_tensor( - process_mesh.topology, vars[var_name].shape, dims_mapping): + process_mesh.topology, vars[var_name].shape, dims_mapping + ): return False if vars[var_name].is_data and len(dims_mapping) > 1: for dim in dims_mapping[1:]: @@ -75,7 +80,8 @@ def check_dims_mapping_for_op(op, op_dist_attr, vars): for var_name in op.output_arg_names: dims_mapping = op_dist_attr.get_output_dims_mapping(var_name) if not PlanFilter.check_dims_mapping_for_tensor( - process_mesh.topology, vars[var_name].shape, dims_mapping): + process_mesh.topology, vars[var_name].shape, dims_mapping + ): return False return True @@ -83,7 +89,11 @@ def check_dims_mapping_for_op(op, op_dist_attr, vars): @staticmethod def check_dims_mapping_for_special_op(op, op_dist_attr, vars): # NOTE: Those ops has some partition limits, and will be solved when corresponding dist op implemented in the future. - if op.type == "elementwise_add" or op.type == 'layer_norm' or op.type == "softmax_with_cross_entropy": + if ( + op.type == "elementwise_add" + or op.type == 'layer_norm' + or op.type == "softmax_with_cross_entropy" + ): for name in op.input_arg_names: for item in op_dist_attr.get_input_dims_mapping(name): if item != -1: @@ -104,20 +114,25 @@ def check_dims_mapping_for_special_op(op, op_dist_attr, vars): class PlanSpace: not_enum_ops = ["create_py_reader", "create_double_buffer_reader", "read"] special_vars = [ - "lod_tensor_blocking_queue_0", "create_py_reader_0", "double_buffer_0" + "lod_tensor_blocking_queue_0", + "create_py_reader_0", + "double_buffer_0", ] @staticmethod - def _enum_dims_mapping(process_mesh_topology, visited, path, depth, res, - tensor_shape): + def _enum_dims_mapping( + process_mesh_topology, visited, path, depth, res, tensor_shape + ): """Enumerate dims mapping of tensor by the given process_mesh_topology""" nums = list(range(-1, len(process_mesh_topology))) if depth == len(tensor_shape): valid = True for idx, item in enumerate(path): if item != -1: - if tensor_shape[idx] % process_mesh_topology[ - item] != 0 or path.count(item) > 1: + if ( + tensor_shape[idx] % process_mesh_topology[item] != 0 + or path.count(item) > 1 + ): valid = False if valid: res.append(copy.deepcopy(path)) @@ -128,15 +143,23 @@ def _enum_dims_mapping(process_mesh_topology, visited, path, depth, res, if i != 0: visited[i] = True path.append(nums[i]) - PlanSpace._enum_dims_mapping(process_mesh_topology, visited, - path, depth + 1, res, tensor_shape) + PlanSpace._enum_dims_mapping( + process_mesh_topology, + visited, + path, + depth + 1, + res, + tensor_shape, + ) visited[i] = False path.pop() @staticmethod def enum_process_mesh_topology(processes): """Enumerate all process meshes with the given processes.""" - assert processes >= 1, "The processes must be number and greater than 0." + assert ( + processes >= 1 + ), "The processes must be number and greater than 0." # compute divisors divisors = [] for i in range(1, processes + 1): @@ -182,7 +205,8 @@ def _enum_valid_dist_attr_for_op(program, op, process_mesh): dims_mapping_dict = OrderedDict() op_valid_dist_attrs = [] dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + op.type + ) # enumerate all valid dims mapping of tensor when process mesh given for var_name in chain(op.input_arg_names, op.output_arg_names): @@ -193,15 +217,22 @@ def _enum_valid_dist_attr_for_op(program, op, process_mesh): depth = 0 path = [] dims_mapping_list = [] - PlanSpace._enum_dims_mapping(process_mesh.topology, visited, path, - depth, dims_mapping_list, - vars[var_name].shape) + PlanSpace._enum_dims_mapping( + process_mesh.topology, + visited, + path, + depth, + dims_mapping_list, + vars[var_name].shape, + ) dims_mapping_dict[var_name] = copy.deepcopy(dims_mapping_list) # compose dims mapping composed_dims_mapping_list = list( product( - *[dims_mapping_dict[key] for key in dims_mapping_dict.keys()])) + *[dims_mapping_dict[key] for key in dims_mapping_dict.keys()] + ) + ) for composed_dims_mapping in composed_dims_mapping_list: op_dist_attr = OperatorDistributedAttribute() op_dist_attr.process_mesh = process_mesh @@ -209,15 +240,19 @@ def _enum_valid_dist_attr_for_op(program, op, process_mesh): for idx, dims_mapping in enumerate(composed_dims_mapping): if var_names[idx] in op.input_arg_names: - op_dist_attr.set_input_dims_mapping(var_names[idx], - dims_mapping) + op_dist_attr.set_input_dims_mapping( + var_names[idx], dims_mapping + ) elif var_names[idx] in op.output_arg_names: op_dist_attr.set_output_dims_mapping( - var_names[idx], dims_mapping) + var_names[idx], dims_mapping + ) else: raise ValueError( - "The {varname} is not input or output of op {op}.". - format(varname='var_names[idx]', op='op')) + "The {varname} is not input or output of op {op}.".format( + varname='var_names[idx]', op='op' + ) + ) dist_op = DistributedOperator(op, op_dist_attr) if dist_op_impl_container is None: @@ -226,14 +261,16 @@ def _enum_valid_dist_attr_for_op(program, op, process_mesh): valid = True try: changed = update_op_dims_mapping_by_elementwise_like_dist_impl( - dist_op) + dist_op + ) except Exception as e: valid = False if valid and not changed: if PlanFilter.check_dims_mapping_for_op( - op, dist_op.dist_attr, vars + op, dist_op.dist_attr, vars ) and PlanFilter.check_dims_mapping_for_special_op( - op, dist_op.dist_attr, vars): + op, dist_op.dist_attr, vars + ): dist_op.dist_attr.impl_type = "elementwise" dist_op.dist_attr.impl_idx = 0 op_valid_dist_attrs.append(dist_op.dist_attr) @@ -243,14 +280,16 @@ def _enum_valid_dist_attr_for_op(program, op, process_mesh): valid = True try: changed = update_op_dims_mapping_by_default_dist_impl( - dist_op) + dist_op + ) except Exception as e: valid = False if valid and not changed: if PlanFilter.check_dims_mapping_for_op( - op, dist_op.dist_attr, vars + op, dist_op.dist_attr, vars ) and PlanFilter.check_dims_mapping_for_special_op( - op, dist_op.dist_attr, vars): + op, dist_op.dist_attr, vars + ): dist_op.dist_attr.impl_type = "default" dist_op.dist_attr.impl_idx = 0 op_valid_dist_attrs.append(dist_op.dist_attr) @@ -261,7 +300,8 @@ def _enum_valid_dist_attr_for_op(program, op, process_mesh): for idx, impl in enumerate(impls): if impl.is_auto_compatible(dist_op): if PlanFilter.check_dims_mapping_for_op( - op, dist_op.dist_attr, vars): + op, dist_op.dist_attr, vars + ): dist_op.dist_attr.impl_type = dist_op.serial_op.type dist_op.dist_attr.impl_idx = idx op_valid_dist_attrs.append(dist_op.dist_attr) @@ -273,10 +313,12 @@ def _enum_valid_dist_attr_for_op(program, op, process_mesh): dist_op = DistributedOperator(op, op_dist_attr) for var_name in op.input_arg_names: op_dist_attr.set_input_dims_mapping( - vars[var_name], [-1 for i in vars[var_name].shape]) + vars[var_name], [-1 for i in vars[var_name].shape] + ) for var_name in op.output_arg_names: op_dist_attr.set_output_dims_mapping( - vars[var_name], [-1 for i in vars[var_name].shape]) + vars[var_name], [-1 for i in vars[var_name].shape] + ) dist_op.dist_attr.impl_type = "default" dist_op.dist_attr.impl_idx = 0 op_valid_dist_attrs.append(dist_op.dist_attr) @@ -284,9 +326,9 @@ def _enum_valid_dist_attr_for_op(program, op, process_mesh): return op_valid_dist_attrs @staticmethod - def enum_valid_dist_attr_for_program(program, - process_mesh_topology, - is_pipeline=False): + def enum_valid_dist_attr_for_program( + program, process_mesh_topology, is_pipeline=False + ): """Enumerate valid distributed attributes for all ops in program.""" valid_dist_attr_dict = OrderedDict() ops = program.global_block().ops @@ -304,16 +346,31 @@ def enum_valid_dist_attr_for_program(program, if len(process_mesh_topology) > 1: process_mesh_shape = process_mesh_topology[:-1] per_process_mesh_group = processes // pipeline_stages - pipeline_process_meshes = [auto.ProcessMesh(mesh=np.array(global_group[i*per_process_mesh_group: \ - (i+1)*per_process_mesh_group]).reshape(process_mesh_shape).tolist()) for i in range(pipeline_stages)] + pipeline_process_meshes = [ + auto.ProcessMesh( + mesh=np.array( + global_group[ + i + * per_process_mesh_group : (i + 1) + * per_process_mesh_group + ] + ) + .reshape(process_mesh_shape) + .tolist() + ) + for i in range(pipeline_stages) + ] elif len(process_mesh_topology) == 1: pipeline_process_meshes = [ auto.ProcessMesh(mesh=[i]) for i in range(pipeline_stages) ] else: if len(process_mesh_topology) > 1: - global_process_mesh = auto.ProcessMesh(mesh=np.array( - global_group).reshape(process_mesh_topology).tolist()) + global_process_mesh = auto.ProcessMesh( + mesh=np.array(global_group) + .reshape(process_mesh_topology) + .tolist() + ) else: global_process_mesh = auto.ProcessMesh(mesh=global_group) @@ -323,8 +380,11 @@ def enum_valid_dist_attr_for_program(program, op_process_mesh = global_process_mesh pipeline_stage = -1 if pipeline_process_meshes is not None: - pipeline_stage = idx // op_count_per_stage if idx // op_count_per_stage < len( - pipeline_process_meshes) else idx // op_count_per_stage - 1 + pipeline_stage = ( + idx // op_count_per_stage + if idx // op_count_per_stage < len(pipeline_process_meshes) + else idx // op_count_per_stage - 1 + ) if pipeline_stage >= len(pipeline_process_meshes): pipeline_stage = len(pipeline_process_meshes) - 1 op_process_mesh = pipeline_process_meshes[pipeline_stage] @@ -338,7 +398,8 @@ def enum_valid_dist_attr_for_program(program, else: dims_mapping = [-1 for i in vars[var_name].shape] op_dist_attr.set_input_dims_mapping( - var_name, dims_mapping) + var_name, dims_mapping + ) for var_name in op.output_arg_names: if var_name in PlanSpace.special_vars: @@ -346,24 +407,31 @@ def enum_valid_dist_attr_for_program(program, else: dims_mapping = [-1 for i in vars[var_name].shape] op_dist_attr.set_output_dims_mapping( - var_name, dims_mapping) + var_name, dims_mapping + ) op_valid_dist_attrs = [op_dist_attr] pipeline_stage = 0 if pipeline_stage != -1 else pipeline_stage else: op_valid_dist_attrs = PlanSpace._enum_valid_dist_attr_for_op( - program, op, op_process_mesh) + program, op, op_process_mesh + ) - assert op_valid_dist_attrs is not None, "Enumerate {} valid distributed attribute failed.".format( - op) + assert ( + op_valid_dist_attrs is not None + ), "Enumerate {} valid distributed attribute failed.".format(op) valid_dist_attr_dict[op.desc.id()] = [ - op_valid_dist_attrs, pipeline_stage + op_valid_dist_attrs, + pipeline_stage, ] - return valid_dist_attr_dict, pipeline_process_meshes, global_process_mesh + return ( + valid_dist_attr_dict, + pipeline_process_meshes, + global_process_mesh, + ) class SearchAlgorithm: - def __init__(self, name): self._name = name @@ -376,7 +444,6 @@ def search(self): class MCMC(SearchAlgorithm): - def __init__(self, serial_program_info, parallelizer, max_search_times=5): super(MCMC, self).__init__("mcmc") self._serial_program_info = serial_program_info @@ -395,32 +462,49 @@ def parallelizer(self): def max_search_times(self): return self._max_search_times - def make_special_op_unshard(self, op, ops, vars, dist_context, - valid_dist_attr_dict): + def make_special_op_unshard( + self, op, ops, vars, dist_context, valid_dist_attr_dict + ): if op.type == "softmax_with_cross_entropy": for var_name in op.input_arg_names: dims_mapping = dist_context.get_op_dist_attr_for_program( - op).get_input_dims_mapping(var_name) - if dims_mapping != dist_context.get_tensor_dist_attr_for_program( - vars[var_name]).dims_mapping: + op + ).get_input_dims_mapping(var_name) + if ( + dims_mapping + != dist_context.get_tensor_dist_attr_for_program( + vars[var_name] + ).dims_mapping + ): has_changed = False for search_op in ops: if var_name in search_op.output_arg_names: op_dist_attr_list = valid_dist_attr_dict[ - search_op.desc.id()][0] + search_op.desc.id() + ][0] for op_dist_attr in op_dist_attr_list: - if op_dist_attr.get_output_dims_mapping( - var_name) == dims_mapping: + if ( + op_dist_attr.get_output_dims_mapping( + var_name + ) + == dims_mapping + ): dist_context.set_op_dist_attr_for_program( - search_op, op_dist_attr) + search_op, op_dist_attr + ) for name in search_op.output_arg_names: - tensor_dist_attr = TensorDistributedAttribute( + tensor_dist_attr = ( + TensorDistributedAttribute() + ) + tensor_dist_attr.process_mesh = ( + op_dist_attr.process_mesh ) - tensor_dist_attr.process_mesh = op_dist_attr.process_mesh tensor_dist_attr.dims_mapping = op_dist_attr.get_output_dims_mapping( - name) + name + ) dist_context.set_tensor_dist_attr_for_program( - vars[name], tensor_dist_attr) + vars[name], tensor_dist_attr + ) has_changed = True break if has_changed: @@ -430,8 +514,13 @@ def make_special_op_unshard(self, op, ops, vars, dist_context, "Change softmax_with_cross_entropy dist attr failed" ) - def init_program(self, valid_dist_attr_dict, program, - pipeline_process_meshes, global_process_mesh): + def init_program( + self, + valid_dist_attr_dict, + program, + pipeline_process_meshes, + global_process_mesh, + ): ops = program.global_block().ops vars = program.global_block().vars new_dist_context = DistributedContext() @@ -439,32 +528,44 @@ def init_program(self, valid_dist_attr_dict, program, for op in ops: op_valid_dist_attr_list = valid_dist_attr_dict[op.desc.id()][0] random_op_dist_attr = np.random.randint( - len(op_valid_dist_attr_list)) + len(op_valid_dist_attr_list) + ) init_op_dist_attr = op_valid_dist_attr_list[random_op_dist_attr] new_dist_context.set_op_dist_attr_for_program(op, init_op_dist_attr) for var_name in op.input_arg_names: if var_name == "lod_tensor_blocking_queue_0": continue - if new_dist_context.get_tensor_dist_attr_for_program( - vars[var_name]) is None: + if ( + new_dist_context.get_tensor_dist_attr_for_program( + vars[var_name] + ) + is None + ): tensor_dist_attr = TensorDistributedAttribute() - tensor_dist_attr.process_mesh = init_op_dist_attr.process_mesh - tensor_dist_attr.dims_mapping = init_op_dist_attr.get_input_dims_mapping( - var_name) + tensor_dist_attr.process_mesh = ( + init_op_dist_attr.process_mesh + ) + tensor_dist_attr.dims_mapping = ( + init_op_dist_attr.get_input_dims_mapping(var_name) + ) new_dist_context.set_tensor_dist_attr_for_program( - vars[var_name], tensor_dist_attr) + vars[var_name], tensor_dist_attr + ) for var_name in op.output_arg_names: tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.process_mesh = init_op_dist_attr.process_mesh - tensor_dist_attr.dims_mapping = init_op_dist_attr.get_output_dims_mapping( - var_name) + tensor_dist_attr.dims_mapping = ( + init_op_dist_attr.get_output_dims_mapping(var_name) + ) new_dist_context.set_tensor_dist_attr_for_program( - vars[var_name], tensor_dist_attr) + vars[var_name], tensor_dist_attr + ) # NOTE: this is a temporary solution to make softmax_with_cross_entropy unshard - self.make_special_op_unshard(op, ops, vars, new_dist_context, - valid_dist_attr_dict) + self.make_special_op_unshard( + op, ops, vars, new_dist_context, valid_dist_attr_dict + ) # add process meshes to distributed context if global_process_mesh is not None: @@ -475,16 +576,19 @@ def init_program(self, valid_dist_attr_dict, program, return new_dist_context - def estimate_searched_strategy_cost(self, - dist_context, - pipeline_process_meshes=None): + def estimate_searched_strategy_cost( + self, dist_context, pipeline_process_meshes=None + ): cost = None # get all distributed programs all_dist_main_program = get_all_distributed_main_program( - self.serial_program_info, dist_context, self.parallelizer) - pipeline_config = [ - process_mesh.processes for process_mesh in pipeline_process_meshes - ] if pipeline_process_meshes is not None else None + self.serial_program_info, dist_context, self.parallelizer + ) + pipeline_config = ( + [process_mesh.processes for process_mesh in pipeline_process_meshes] + if pipeline_process_meshes is not None + else None + ) microbatch_size = 1 for program in all_dist_main_program: searched_batch_size = False @@ -497,14 +601,17 @@ def estimate_searched_strategy_cost(self, break from .utils import get_standalone_cost_data + standalone_cost_data = get_standalone_cost_data(all_dist_main_program) # cost model does not support cluster argument - cost = estimate_cost(all_dist_main_program, - cluster=None, - pipeline_config=pipeline_config, - standalone_cost_data=standalone_cost_data, - batch_size=microbatch_size) + cost = estimate_cost( + all_dist_main_program, + cluster=None, + pipeline_config=pipeline_config, + standalone_cost_data=standalone_cost_data, + batch_size=microbatch_size, + ) return cost @@ -514,10 +621,12 @@ def set_tensor_dist_attr(self, op, op_dist_attr, vars, dist_context): process_mesh = op_dist_attr.process_mesh tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.process_mesh = process_mesh - tensor_dist_attr.dims_mapping = op_dist_attr.get_output_dims_mapping( - var_name) + tensor_dist_attr.dims_mapping = ( + op_dist_attr.get_output_dims_mapping(var_name) + ) dist_context.set_tensor_dist_attr_for_program( - vars[var_name], tensor_dist_attr) + vars[var_name], tensor_dist_attr + ) # set input tensor distributed attribute if input is data or parameter for var_name in op.input_arg_names: @@ -525,27 +634,34 @@ def set_tensor_dist_attr(self, op, op_dist_attr, vars, dist_context): process_mesh = op_dist_attr.process_mesh tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.process_mesh = process_mesh - tensor_dist_attr.dims_mapping = op_dist_attr.get_input_dims_mapping( - var_name) + tensor_dist_attr.dims_mapping = ( + op_dist_attr.get_input_dims_mapping(var_name) + ) dist_context.set_tensor_dist_attr_for_program( - vars[var_name], tensor_dist_attr) + vars[var_name], tensor_dist_attr + ) def change_process_mesh(self, op, changed_process_mesh, vars, dist_context): dist_context.get_op_dist_attr_for_program( - op).process_mesh = changed_process_mesh + op + ).process_mesh = changed_process_mesh for var_name in op.output_arg_names: dist_context.get_tensor_dist_attr_for_program( - vars[var_name]).process_mesh = changed_process_mesh + vars[var_name] + ).process_mesh = changed_process_mesh for var_name in op.input_arg_names: if vars[var_name].is_parameter or vars[var_name].is_data: dist_context.get_tensor_dist_attr_for_program( - vars[var_name]).process_mesh = changed_process_mesh - - def search_once(self, - program, - valid_dist_attr_dict, - dist_context, - pipeline_process_meshes=None): + vars[var_name] + ).process_mesh = changed_process_mesh + + def search_once( + self, + program, + valid_dist_attr_dict, + dist_context, + pipeline_process_meshes=None, + ): raw_ops = program.global_block().ops ops = [] for op in raw_ops: @@ -561,9 +677,11 @@ def search_once(self, op_valid_dist_attr_list = valid_dist_attr_dict[selected_op.desc.id()][0] pipeline_stage = valid_dist_attr_dict[selected_op.desc.id()][1] random_selected_dist_attr_idx = np.random.randint( - len(op_valid_dist_attr_list)) + len(op_valid_dist_attr_list) + ) selected_op_dist_attr = copy.deepcopy( - op_valid_dist_attr_list[random_selected_dist_attr_idx]) + op_valid_dist_attr_list[random_selected_dist_attr_idx] + ) start_idx = ops[0].desc.id() if pipeline_stage > -1: @@ -574,142 +692,215 @@ def search_once(self, if changed_mode == 0: # not change the process mesh, just change dims mapping new_dist_context.set_op_dist_attr_for_program( - selected_op, selected_op_dist_attr) - self.set_tensor_dist_attr(selected_op, selected_op_dist_attr, - vars, new_dist_context) + selected_op, selected_op_dist_attr + ) + self.set_tensor_dist_attr( + selected_op, selected_op_dist_attr, vars, new_dist_context + ) elif changed_mode == 1: changed_stage = pipeline_stage - 1 - if changed_stage == -1 or random_selected_op_idx == len(ops) - 1 or \ - (random_selected_op_idx + 1 == len(ops) - 1 and new_valid_dist_attr_dict[ops[random_selected_op_idx + 1].desc.id()][1] == pipeline_stage + 1 ): + if ( + changed_stage == -1 + or random_selected_op_idx == len(ops) - 1 + or ( + random_selected_op_idx + 1 == len(ops) - 1 + and new_valid_dist_attr_dict[ + ops[random_selected_op_idx + 1].desc.id() + ][1] + == pipeline_stage + 1 + ) + ): new_dist_context.set_op_dist_attr_for_program( - selected_op, selected_op_dist_attr) - self.set_tensor_dist_attr(selected_op, - selected_op_dist_attr, vars, - new_dist_context) + selected_op, selected_op_dist_attr + ) + self.set_tensor_dist_attr( + selected_op, + selected_op_dist_attr, + vars, + new_dist_context, + ) else: selected_op_process_mesh = pipeline_process_meshes[ - pipeline_stage] + pipeline_stage + ] next_op_id = ops[random_selected_op_idx + 1].desc.id() - if new_valid_dist_attr_dict[next_op_id][ - 1] == pipeline_stage + 1 and random_selected_op_idx + 1 != len( - ops) - 1: + if ( + new_valid_dist_attr_dict[next_op_id][1] + == pipeline_stage + 1 + and random_selected_op_idx + 1 != len(ops) - 1 + ): new_valid_dist_attr_dict[next_op_id][1] = pipeline_stage for op_dist_attr in new_valid_dist_attr_dict[ - next_op_id][0]: + next_op_id + ][0]: op_dist_attr.process_mesh = selected_op_process_mesh # set next op dist attr in the discontext and output/input tensor process mesh self.change_process_mesh( ops[random_selected_op_idx + 1], - selected_op_process_mesh, vars, new_dist_context) + selected_op_process_mesh, + vars, + new_dist_context, + ) # change the selected op stage and output dist attr - new_valid_dist_attr_dict[ - selected_op.desc.id()][1] = changed_stage + new_valid_dist_attr_dict[selected_op.desc.id()][ + 1 + ] = changed_stage new_process_mesh = pipeline_process_meshes[changed_stage] selected_op_dist_attr.process_mesh = new_process_mesh for op_dist_attr in new_valid_dist_attr_dict[ - selected_op.desc.id()][0]: + selected_op.desc.id() + ][0]: op_dist_attr.process_mesh = new_process_mesh new_dist_context.set_op_dist_attr_for_program( - selected_op, selected_op_dist_attr) + selected_op, selected_op_dist_attr + ) - self.set_tensor_dist_attr(selected_op, - selected_op_dist_attr, vars, - new_dist_context) + self.set_tensor_dist_attr( + selected_op, + selected_op_dist_attr, + vars, + new_dist_context, + ) # change the pre op stage for idx in range(random_selected_op_idx - 1, -1, -1): stage = new_valid_dist_attr_dict[ops[idx].desc.id()][1] valid_dist_attr_list = new_valid_dist_attr_dict[ - ops[idx].desc.id()][0] + ops[idx].desc.id() + ][0] new_process_mesh = pipeline_process_meshes[ - changed_stage] + changed_stage + ] if stage == changed_stage + 1: - new_valid_dist_attr_dict[ - ops[idx].desc.id()][1] = changed_stage + new_valid_dist_attr_dict[ops[idx].desc.id()][ + 1 + ] = changed_stage for op_dist_attr in valid_dist_attr_list: op_dist_attr.process_mesh = new_process_mesh new_dist_context.get_op_dist_attr_for_program( - ops[idx]).process_mesh = new_process_mesh + ops[idx] + ).process_mesh = new_process_mesh # change process mesh of the output and input tensor - self.change_process_mesh(ops[idx], new_process_mesh, - vars, new_dist_context) + self.change_process_mesh( + ops[idx], + new_process_mesh, + vars, + new_dist_context, + ) else: break else: changed_stage = pipeline_stage + 1 - if changed_stage == len( - pipeline_process_meshes) or random_selected_op_idx == 0 or \ - (new_valid_dist_attr_dict[ops[random_selected_op_idx - 1].desc.id()][1] == pipeline_stage - 1 and (random_selected_op_idx == 1)): + if ( + changed_stage == len(pipeline_process_meshes) + or random_selected_op_idx == 0 + or ( + new_valid_dist_attr_dict[ + ops[random_selected_op_idx - 1].desc.id() + ][1] + == pipeline_stage - 1 + and (random_selected_op_idx == 1) + ) + ): new_dist_context.set_op_dist_attr_for_program( - selected_op, selected_op_dist_attr) - self.set_tensor_dist_attr(selected_op, - selected_op_dist_attr, vars, - new_dist_context) + selected_op, selected_op_dist_attr + ) + self.set_tensor_dist_attr( + selected_op, + selected_op_dist_attr, + vars, + new_dist_context, + ) else: selected_op_process_mesh = pipeline_process_meshes[ - pipeline_stage] + pipeline_stage + ] pre_op_id = ops[random_selected_op_idx - 1].desc.id() - if new_valid_dist_attr_dict[pre_op_id][ - 1] == pipeline_stage - 1 and random_selected_op_idx != 1: + if ( + new_valid_dist_attr_dict[pre_op_id][1] + == pipeline_stage - 1 + and random_selected_op_idx != 1 + ): new_valid_dist_attr_dict[pre_op_id][1] = pipeline_stage for op_dist_attr in new_valid_dist_attr_dict[pre_op_id][ - 0]: + 0 + ]: op_dist_attr.process_mesh = selected_op_process_mesh # set pre op dist attr in the discontext and output tensor process mesh self.change_process_mesh( ops[random_selected_op_idx - 1], - selected_op_process_mesh, vars, new_dist_context) + selected_op_process_mesh, + vars, + new_dist_context, + ) # change the selected op stage and output tensor dist attr - new_valid_dist_attr_dict[ - selected_op.desc.id()][1] = changed_stage + new_valid_dist_attr_dict[selected_op.desc.id()][ + 1 + ] = changed_stage new_process_mesh = pipeline_process_meshes[changed_stage] selected_op_dist_attr.process_mesh = new_process_mesh for op_dist_attr in new_valid_dist_attr_dict[ - selected_op.desc.id()][0]: + selected_op.desc.id() + ][0]: op_dist_attr.process_mesh = new_process_mesh new_dist_context.set_op_dist_attr_for_program( - selected_op, selected_op_dist_attr) - self.set_tensor_dist_attr(selected_op, - selected_op_dist_attr, vars, - new_dist_context) + selected_op, selected_op_dist_attr + ) + self.set_tensor_dist_attr( + selected_op, + selected_op_dist_attr, + vars, + new_dist_context, + ) # change the next op stage for idx in range(random_selected_op_idx + 1, len(ops)): stage = new_valid_dist_attr_dict[ops[idx].desc.id()][1] valid_dist_attr_list = new_valid_dist_attr_dict[ - ops[idx].desc.id()][0] + ops[idx].desc.id() + ][0] new_process_mesh = pipeline_process_meshes[ - changed_stage] + changed_stage + ] if stage == changed_stage - 1: - new_valid_dist_attr_dict[ - ops[idx].desc.id()][1] = changed_stage + new_valid_dist_attr_dict[ops[idx].desc.id()][ + 1 + ] = changed_stage for op_dist_attr in valid_dist_attr_list: op_dist_attr.process_mesh = new_process_mesh new_dist_context.get_op_dist_attr_for_program( - ops[idx]).process_mesh = new_process_mesh + ops[idx] + ).process_mesh = new_process_mesh # change the output tensor dist attr - self.change_process_mesh(ops[idx], new_process_mesh, - vars, new_dist_context) + self.change_process_mesh( + ops[idx], + new_process_mesh, + vars, + new_dist_context, + ) else: break else: new_dist_context.set_op_dist_attr_for_program( - selected_op, selected_op_dist_attr) - self.set_tensor_dist_attr(selected_op, selected_op_dist_attr, vars, - new_dist_context) + selected_op, selected_op_dist_attr + ) + self.set_tensor_dist_attr( + selected_op, selected_op_dist_attr, vars, new_dist_context + ) for op in ops: # make softmax_with_cross_entropy unshard if op.type == "softmax_with_cross_entropy": - self.make_special_op_unshard(op, ops, vars, new_dist_context, - valid_dist_attr_dict) + self.make_special_op_unshard( + op, ops, vars, new_dist_context, valid_dist_attr_dict + ) break if new_valid_dist_attr_dict is None: @@ -717,22 +908,29 @@ def search_once(self, else: return new_valid_dist_attr_dict, new_dist_context - def _search_core(self, - valid_dist_attr_dict, - init_dist_context, - pipeline_process_meshes=None): + def _search_core( + self, + valid_dist_attr_dict, + init_dist_context, + pipeline_process_meshes=None, + ): times = 0 best_dist_context = init_dist_context cost = self.estimate_searched_strategy_cost( - init_dist_context, pipeline_process_meshes).runtime + init_dist_context, pipeline_process_meshes + ).runtime min_cost = cost while times < self.max_search_times: times += 1 new_dist_context = self.search_once( - self.serial_program_info.train_program, valid_dist_attr_dict, - best_dist_context, pipeline_process_meshes)[1] + self.serial_program_info.train_program, + valid_dist_attr_dict, + best_dist_context, + pipeline_process_meshes, + )[1] cur_cost = self.estimate_searched_strategy_cost( - new_dist_context, pipeline_process_meshes).runtime + new_dist_context, pipeline_process_meshes + ).runtime if (min_cost - cur_cost) > 0: best_dist_context = copy.deepcopy(new_dist_context) min_cost = cur_cost @@ -744,35 +942,57 @@ def search(self): start_time = time.time() train_program = self.serial_program_info.train_program cluster = self.serial_program_info.cluster - processes = paddle.distributed.get_world_size( - ) if cluster is None else len(cluster.get_all_devices("GPU")) + processes = ( + paddle.distributed.get_world_size() + if cluster is None + else len(cluster.get_all_devices("GPU")) + ) assert processes > 0, "Get process failed." process_mesh_topology_list = PlanSpace.enum_process_mesh_topology( - processes) + processes + ) searched_dist_context = None min_cost = None searched_pipeline_dist_context = None pipeline_min_cost = None for process_mesh_topology in process_mesh_topology_list: - print("MCMC search: search process mesh {} with pipeline mode.". - format(process_mesh_topology)) - valid_dist_attr_dict, pipeline_process_meshes, global_process_mesh = PlanSpace.enum_valid_dist_attr_for_program( - train_program, process_mesh_topology, True) - init_dist_context = self.init_program(valid_dist_attr_dict, - train_program, - pipeline_process_meshes, - global_process_mesh) + print( + "MCMC search: search process mesh {} with pipeline mode.".format( + process_mesh_topology + ) + ) + ( + valid_dist_attr_dict, + pipeline_process_meshes, + global_process_mesh, + ) = PlanSpace.enum_valid_dist_attr_for_program( + train_program, process_mesh_topology, True + ) + init_dist_context = self.init_program( + valid_dist_attr_dict, + train_program, + pipeline_process_meshes, + global_process_mesh, + ) best_dist_context, cost = self._search_core( - valid_dist_attr_dict, init_dist_context, - pipeline_process_meshes) + valid_dist_attr_dict, init_dist_context, pipeline_process_meshes + ) print( - "MCMC search: the min cost is {} in the process mesh {} with pipeline mode." - .format(cost, process_mesh_topology)) + "MCMC search: the min cost is {} in the process mesh {} with pipeline mode.".format( + cost, process_mesh_topology + ) + ) best_dist_context._dist_op_context = DistributedOperatorContext() - pipeline_min_cost = cost if pipeline_min_cost is None else pipeline_min_cost - searched_pipeline_dist_context = best_dist_context if searched_pipeline_dist_context is None else searched_pipeline_dist_context + pipeline_min_cost = ( + cost if pipeline_min_cost is None else pipeline_min_cost + ) + searched_pipeline_dist_context = ( + best_dist_context + if searched_pipeline_dist_context is None + else searched_pipeline_dist_context + ) if pipeline_min_cost > cost: searched_pipeline_dist_context = best_dist_context pipeline_min_cost = cost @@ -783,23 +1003,41 @@ def search(self): # if process_mesh_topology shape is 3, include pipeline mode by default if len(process_mesh_topology) == 3: continue - print("MCMC search: search process mesh {} without pipeline mode.". - format(process_mesh_topology)) - valid_dist_attr_dict, pipeline_process_meshes, global_process_mesh = PlanSpace.enum_valid_dist_attr_for_program( - train_program, process_mesh_topology, False) - init_dist_context = self.init_program(valid_dist_attr_dict, - train_program, - pipeline_process_meshes, - global_process_mesh) + print( + "MCMC search: search process mesh {} without pipeline mode.".format( + process_mesh_topology + ) + ) + ( + valid_dist_attr_dict, + pipeline_process_meshes, + global_process_mesh, + ) = PlanSpace.enum_valid_dist_attr_for_program( + train_program, process_mesh_topology, False + ) + init_dist_context = self.init_program( + valid_dist_attr_dict, + train_program, + pipeline_process_meshes, + global_process_mesh, + ) best_dist_context, cost = self._search_core( - valid_dist_attr_dict, init_dist_context, - pipeline_process_meshes) + valid_dist_attr_dict, init_dist_context, pipeline_process_meshes + ) print( - "MCMC search: the min cost is {} in the process mesh {} without pipeline mode." - .format(cost, process_mesh_topology)) + "MCMC search: the min cost is {} in the process mesh {} without pipeline mode.".format( + cost, process_mesh_topology + ) + ) best_dist_context._dist_op_context = DistributedOperatorContext() - non_pipeline_min_cost = cost if non_pipeline_min_cost is None else non_pipeline_min_cost - searched_non_pipeline_dist_context = best_dist_context if searched_non_pipeline_dist_context is None else searched_non_pipeline_dist_context + non_pipeline_min_cost = ( + cost if non_pipeline_min_cost is None else non_pipeline_min_cost + ) + searched_non_pipeline_dist_context = ( + best_dist_context + if searched_non_pipeline_dist_context is None + else searched_non_pipeline_dist_context + ) if non_pipeline_min_cost > cost: searched_non_pipeline_dist_context = best_dist_context non_pipeline_min_cost = cost @@ -820,22 +1058,23 @@ def search(self): pg0.add_ranks(process_mesh.processes) end_time = time.time() print( - "End MCMC searching: the min cost is {} and the search time is {}s." - .format(min_cost, end_time - start_time)) + "End MCMC searching: the min cost is {} and the search time is {}s.".format( + min_cost, end_time - start_time + ) + ) return searched_dist_context, min_cost class Planner: - - def __init__(self, - serial_program_info, - parallelizer, - algorithm_config=None): + def __init__( + self, serial_program_info, parallelizer, algorithm_config=None + ): self._serial_program_info = serial_program_info self._parallelizer = parallelizer self._algorithm_config = algorithm_config self._algorithm_searcher = self.create_algorithm_searcher( - algorithm_config) + algorithm_config + ) @property def serial_program_info(self): @@ -861,13 +1100,19 @@ def create_algorithm_searcher(self, algorithm_config): if name == "mcmc": # NOTE: Only GPU clusters are supported now. max_search_times = algorithm_config.get("max_search_times", None) - algorithm_searcher = MCMC( - self.serial_program_info, self.parallelizer, - max_search_times) if max_search_times is not None else MCMC( - self.serial_program_info, self.parallelizer) + algorithm_searcher = ( + MCMC( + self.serial_program_info, + self.parallelizer, + max_search_times, + ) + if max_search_times is not None + else MCMC(self.serial_program_info, self.parallelizer) + ) else: raise NotImplementedError( - "Other search algorithms have not been supported now.") + "Other search algorithms have not been supported now." + ) return algorithm_searcher diff --git a/python/paddle/distributed/auto_parallel/planner_v2.py b/python/paddle/distributed/auto_parallel/planner_v2.py index 8e2c0c4617b0f..0f9792911dddc 100755 --- a/python/paddle/distributed/auto_parallel/planner_v2.py +++ b/python/paddle/distributed/auto_parallel/planner_v2.py @@ -18,7 +18,6 @@ class Planner: - def __init__(self, mode, dist_context): self._mode = mode self._dist_context = dist_context @@ -39,8 +38,9 @@ def __init__(self, mode, dist_context): self._strategy = dist_context.strategy # set parallel tuner for auto search if self._strategy.auto_mode == "full": - self._parallel_tuner = ParallelTuner(self._dist_context, - mode=self._mode) + self._parallel_tuner = ParallelTuner( + self._dist_context, mode=self._mode + ) @property def completer(self): @@ -53,4 +53,5 @@ def plan(self): self._completer.complete_forward_annotation() # parse forward sub block self._dist_context.block_state.parse_forward_blocks( - self._dist_context.serial_main_program) + self._dist_context.serial_main_program + ) diff --git a/python/paddle/distributed/auto_parallel/process_group.py b/python/paddle/distributed/auto_parallel/process_group.py index ba63a71643466..9883f116f4eea 100644 --- a/python/paddle/distributed/auto_parallel/process_group.py +++ b/python/paddle/distributed/auto_parallel/process_group.py @@ -31,10 +31,11 @@ def get_all_process_groups(): def get_process_group(group_id, g_process_group_map=None): global _g_process_group_map - return _g_process_group_map.get( - group_id, - None) if g_process_group_map is None else g_process_group_map.get( - group_id, None) + return ( + _g_process_group_map.get(group_id, None) + if g_process_group_map is None + else g_process_group_map.get(group_id, None) + ) def get_world_process_group(): @@ -75,10 +76,11 @@ def new_process_group(ranks, group_id=None): # the instantiation process in a more general way. In the future, the process group may # handle the communication implementation choice. class ProcessGroup: - def __init__(self, group_id, ranks): if group_id == 0 and get_process_group(0) is not None: - assert group_id != 0, "Process group id 0 is reserved for all ranks." + assert ( + group_id != 0 + ), "Process group id 0 is reserved for all ranks." self._group_id = group_id self._ranks = sorted(ranks) # Add the current ranks into group 0 @@ -103,8 +105,9 @@ def add_ranks(self, new_ranks): if set(new_ranks) <= set(self.ranks): return else: - assert self.is_instantiate() == False, \ - "Cannot add new ranks after instantiating the process group" + assert ( + self.is_instantiate() == False + ), "Cannot add new ranks after instantiating the process group" self._ranks.extend(new_ranks) self._ranks = sorted(list(set(self.ranks))) @@ -112,8 +115,9 @@ def local_rank(self, global_rank): if global_rank in self.ranks: return self.ranks.index(global_rank) else: - assert False, \ - "Rank {} doesn't belong to this group".format(global_rank) + assert False, "Rank {} doesn't belong to this group".format( + global_rank + ) def is_instantiate(self): return self._is_instantiate @@ -137,22 +141,27 @@ def instantiate(self): if core.is_compiled_with_cuda(): place = core.CUDAPlace(genv.device_id) - core.NCCLParallelContext(strategy, - place).init_with_ring_id(ring_id) + core.NCCLParallelContext(strategy, place).init_with_ring_id( + ring_id + ) else: - assert False, ("No CUDA device found") + assert False, "No CUDA device found" # TODO(shenliang03): This is a temporary solution to solve the problem of # hang caused by cross-creation of new_group paddle.disable_static() - paddle.set_device('gpu:%d' % - paddle.distributed.ParallelEnv().dev_id) - tmp = paddle.to_tensor( - [1], dtype="int32") if _non_static_mode() else fill_constant( - [0], dtype="int32", value="1") + paddle.set_device( + 'gpu:%d' % paddle.distributed.ParallelEnv().dev_id + ) + tmp = ( + paddle.to_tensor([1], dtype="int32") + if _non_static_mode() + else fill_constant([0], dtype="int32", value="1") + ) # use legacy ops - _legacy_C_ops.c_allreduce_sum_(tmp, 'use_calc_stream', True, - 'ring_id', self.id) + _legacy_C_ops.c_allreduce_sum_( + tmp, 'use_calc_stream', True, 'ring_id', self.id + ) _legacy_C_ops.c_sync_calc_stream(tmp, tmp) paddle.enable_static() @@ -173,7 +182,8 @@ def __ne__(self, other): def __str__(self): string = "id: {}, nranks: {}, ranks: {}.".format( - self.id, self.nranks, ", ".join(map(str, self.ranks))) + self.id, self.nranks, ", ".join(map(str, self.ranks)) + ) return string def __hash__(self): diff --git a/python/paddle/distributed/auto_parallel/process_mesh.py b/python/paddle/distributed/auto_parallel/process_mesh.py index afe9f13e3ab43..9d8a44b92d2e2 100644 --- a/python/paddle/distributed/auto_parallel/process_mesh.py +++ b/python/paddle/distributed/auto_parallel/process_mesh.py @@ -68,10 +68,10 @@ def __init__(self, mesh=None, dim_names=None, shape=None, process_ids=None): assert process_ids is not None mesh = np.array(process_ids).reshape(shape) - if not isinstance(mesh, list) and \ - not isinstance(mesh, np.ndarray): + if not isinstance(mesh, list) and not isinstance(mesh, np.ndarray): raise ValueError( - 'The mesh must be an instance of list or np.ndarray.') + 'The mesh must be an instance of list or np.ndarray.' + ) if isinstance(mesh, list): mesh = np.array(mesh) @@ -79,30 +79,37 @@ def __init__(self, mesh=None, dim_names=None, shape=None, process_ids=None): self._shape = list(self._mesh.shape) self._process_ids = self._mesh.flatten().tolist() - assert all(isinstance(p, int) for p in self._process_ids), \ - ("All elements of the mesh must be integer") - assert min( - self._process_ids) >= 0, ('All elements of the mesh must be >= 0.') + assert all( + isinstance(p, int) for p in self._process_ids + ), "All elements of the mesh must be integer" + assert ( + min(self._process_ids) >= 0 + ), 'All elements of the mesh must be >= 0.' unique_process_ids = set(self._process_ids) assert len(unique_process_ids) == len( - self._process_ids), ('All elements of the mesh must be unique.') + self._process_ids + ), 'All elements of the mesh must be unique.' if dim_names is not None: - assert len(dim_names) == len(self._shape), \ - ("The length of dims_names must be same as the shape of the mesh.") + assert len(dim_names) == len( + self._shape + ), "The length of dims_names must be same as the shape of the mesh." self._dim_names = copy.deepcopy(dim_names) else: self._dim_names = ["d" + str(i) for i in range(len(self._shape))] unique_dim_names = set(self._dim_names) - assert len(unique_dim_names) == len(self._dim_names), ( - 'All dim_names {} must be unique.'.format(dim_names)) + assert len(unique_dim_names) == len( + self._dim_names + ), 'All dim_names {} must be unique.'.format(dim_names) # Store all process meshes from .dist_context import get_default_distributed_context + default_dist_cxt = get_default_distributed_context() default_dist_cxt.add_process_mesh(self) # Add new processes to process group 0 from .process_group import get_process_group + pg0 = get_process_group(0) pg0.add_ranks(self.processes) @@ -183,20 +190,24 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, exc_traceback): from .dist_tensor import DistributedTensor from .dist_op import DistributedOperator + default_prog = paddle.fluid.default_main_program() cur_block = default_prog.current_block() new_var_names = list(cur_block.vars.keys()) new_op_size = len(cur_block.ops) from .dist_context import get_default_distributed_context + default_dist_ctx = get_default_distributed_context() for name in new_var_names: if name not in self._old_var_names: tensor = cur_block.vars[name] dist_tensor = default_dist_ctx.get_dist_tensor_for_program( - tensor) + tensor + ) if dist_tensor is None: - dist_tensor = DistributedTensor(cur_block.vars[name], - {"process_mesh": self}) + dist_tensor = DistributedTensor( + cur_block.vars[name], {"process_mesh": self} + ) dist_tensor.dist_attr.mark_annotated("process_mesh") default_dist_ctx.add_dist_tensor_for_program(dist_tensor) else: @@ -229,5 +240,6 @@ def __ne__(self, other): def __str__(self): str = "shape {}, process_ids {}, dim_nams {}".format( - self.shape, self.process_ids, self.dim_names) + self.shape, self.process_ids, self.dim_names + ) return str diff --git a/python/paddle/distributed/auto_parallel/process_mesh_v2.py b/python/paddle/distributed/auto_parallel/process_mesh_v2.py index bc0de1748f124..0b427c9d16187 100644 --- a/python/paddle/distributed/auto_parallel/process_mesh_v2.py +++ b/python/paddle/distributed/auto_parallel/process_mesh_v2.py @@ -44,10 +44,10 @@ class ProcessMesh(core.ProcessMesh): """ def __init__(self, mesh, dim_names=None): - if not isinstance(mesh, list) and \ - not isinstance(mesh, np.ndarray): + if not isinstance(mesh, list) and not isinstance(mesh, np.ndarray): raise ValueError( - 'The mesh must be an instance of list or np.ndarray.') + 'The mesh must be an instance of list or np.ndarray.' + ) if isinstance(mesh, list): mesh = np.array(mesh) @@ -56,24 +56,29 @@ def __init__(self, mesh, dim_names=None): self._shape = list(self._mesh.shape) self._process_ids = self._mesh.flatten().tolist() - assert all(isinstance(p, int) for p in self._process_ids), \ - ("All elements of the mesh must be integer") - assert min( - self._process_ids) >= 0, ('All elements of the mesh must be >= 0.') + assert all( + isinstance(p, int) for p in self._process_ids + ), "All elements of the mesh must be integer" + assert ( + min(self._process_ids) >= 0 + ), 'All elements of the mesh must be >= 0.' unique_process_ids = set(self._process_ids) assert len(unique_process_ids) == len( - self._process_ids), ('All elements of the mesh must be unique.') + self._process_ids + ), 'All elements of the mesh must be unique.' if dim_names is not None: - assert len(dim_names) == len(self._shape), \ - ("The length of dims_names must be same as the shape of the mesh.") + assert len(dim_names) == len( + self._shape + ), "The length of dims_names must be same as the shape of the mesh." self._dim_names = dim_names else: self._dim_names = ["d" + str(i) for i in range(len(self._shape))] # Follow the requirement for using pybind11 - core.ProcessMesh.__init__(self, self._shape, self._process_ids, - self._dim_names) + core.ProcessMesh.__init__( + self, self._shape, self._process_ids, self._dim_names + ) @property def mesh(self): @@ -107,15 +112,20 @@ def _compute_compatible_of_two_process_meshes(pm1, pm2): compatible_result = None for process_mesh in process_meshes: - compatible, compatible_result = _compute_compatible_of_two_process_meshes( - compatible_result, process_mesh) + ( + compatible, + compatible_result, + ) = _compute_compatible_of_two_process_meshes( + compatible_result, process_mesh + ) if not compatible: return None if compatible_result.empty(): return None if isinstance(compatible_result, core.ProcessMesh): mesh = np.array(compatible_result.process_ids).reshape( - compatible_result.shape) + compatible_result.shape + ) return ProcessMesh(mesh, compatible_result.dim_names) elif isinstance(compatible_result, ProcessMesh): return ProcessMesh(compatible_result.mesh, compatible_result.dim_names) diff --git a/python/paddle/distributed/auto_parallel/reshard.py b/python/paddle/distributed/auto_parallel/reshard.py index 46057ad97c4e8..44521c05994dd 100644 --- a/python/paddle/distributed/auto_parallel/reshard.py +++ b/python/paddle/distributed/auto_parallel/reshard.py @@ -32,7 +32,11 @@ # NOTE: If op in _g_special_ops or _g_gradient_clip_ops, it will not be resharded. _g_special_ops = ['check_finite_and_unscale', 'update_loss_scaling'] _g_gradient_clip_ops = [ - "sum", "sqrt", "fill_constant", "elementwise_max", "elementwise_div" + "sum", + "sqrt", + "fill_constant", + "elementwise_max", + "elementwise_div", ] _g_subblock_ops = ["while", "conditional_block"] @@ -264,20 +268,25 @@ class Inserter: def insert_cast_op(block, idx, tensor, op_role, tensor_type): # to avoid name conflict with framework new_var_name = paddle.fluid.unique_name.generate_with_ignorable_key( - ".".join(["cast@RESHARD", 'tmp'])) - out = block.create_var(name=new_var_name, - dtype=tensor_type, - type=tensor.type, - lod_level=tensor.lod_level) - cast_op = block._insert_op(idx, - type='cast', - inputs={'X': [tensor]}, - outputs={'Out': [out]}, - attrs={ - 'in_dtype': tensor.dtype, - 'out_dtype': out.dtype, - 'op_role': op_role - }) + ".".join(["cast@RESHARD", 'tmp']) + ) + out = block.create_var( + name=new_var_name, + dtype=tensor_type, + type=tensor.type, + lod_level=tensor.lod_level, + ) + cast_op = block._insert_op( + idx, + type='cast', + inputs={'X': [tensor]}, + outputs={'Out': [out]}, + attrs={ + 'in_dtype': tensor.dtype, + 'out_dtype': out.dtype, + 'op_role': op_role, + }, + ) cast_op._set_attr('op_namescope', "/auto_parallel/reshard") return out @@ -287,16 +296,18 @@ def insert_send_op(block, idx, tensor, src, dst, op_role): op_type = 'send_v2' # use pair comm group process_group = new_process_group([src, dst]) - send_op = block._insert_op(idx, - type=op_type, - inputs={'X': [tensor]}, - attrs={ - 'ring_id': process_group.id, - 'peer': process_group.ranks.index(dst), - 'use_calc_stream': True, - 'op_role': op_role, - 'dynamic_shape': True - }) + send_op = block._insert_op( + idx, + type=op_type, + inputs={'X': [tensor]}, + attrs={ + 'ring_id': process_group.id, + 'peer': process_group.ranks.index(dst), + 'use_calc_stream': True, + 'op_role': op_role, + 'dynamic_shape': True, + }, + ) send_op._set_attr('op_namescope', "/auto_parallel/reshard") @staticmethod @@ -305,19 +316,21 @@ def insert_recv_op(block, idx, tensor, src, dst, op_role): op_type = 'recv_v2' # use pair group process_group = new_process_group([src, dst]) - recv_op = block._insert_op(idx, - type=op_type, - inputs={'X': [tensor]}, - outputs={'Out': [tensor]}, - attrs={ - 'ring_id': process_group.id, - 'peer': process_group.ranks.index(src), - 'out_shape': tensor.shape, - 'dtype': tensor.dtype, - 'use_calc_stream': True, - 'op_role': op_role, - 'dynamic_shape': True - }) + recv_op = block._insert_op( + idx, + type=op_type, + inputs={'X': [tensor]}, + outputs={'Out': [tensor]}, + attrs={ + 'ring_id': process_group.id, + 'peer': process_group.ranks.index(src), + 'out_shape': tensor.shape, + 'dtype': tensor.dtype, + 'use_calc_stream': True, + 'op_role': op_role, + 'dynamic_shape': True, + }, + ) recv_op._set_attr('op_namescope', "/auto_parallel/reshard") @staticmethod @@ -325,21 +338,23 @@ def insert_reset_lod_op(block, idx, X, Y, op_role): """Insert reset_lod op into block at the given index.""" new_var_name = paddle.fluid.unique_name.generate_with_ignorable_key( - ".".join(["reset_lod@RESHARD", 'tmp'])) - reset_lod_out = block.create_var(name=new_var_name, - shape=X.shape, - type=X.type, - dtype=X.dtype, - lod_level=X.lod_level) - - reset_op = block._insert_op(idx, - type="lod_reset", - inputs={ - 'X': X, - 'Y': Y - }, - outputs={'Out': reset_lod_out}, - attrs={'op_role': op_role}) + ".".join(["reset_lod@RESHARD", 'tmp']) + ) + reset_lod_out = block.create_var( + name=new_var_name, + shape=X.shape, + type=X.type, + dtype=X.dtype, + lod_level=X.lod_level, + ) + + reset_op = block._insert_op( + idx, + type="lod_reset", + inputs={'X': X, 'Y': Y}, + outputs={'Out': reset_lod_out}, + attrs={'op_role': op_role}, + ) reset_op._set_attr('op_namescope', "/auto_parallel/reshard") return reset_lod_out @@ -355,24 +370,29 @@ def insert_concat_op(block, idx, tensors, axis, op_role): with paddle.static.program_guard(block.program): out = block.create_var( name=paddle.fluid.unique_name.generate_with_ignorable_key( - ".".join([helper.name, 'tmp'])), + ".".join([helper.name, 'tmp']) + ), dtype=tensors[0].dtype, shape=None, lod_level=tensors[0].lod_level, type=tensors[0].type, persistable=False, - stop_gradient=False) - concat_op = block._insert_op(idx, - type='concat', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs) + stop_gradient=False, + ) + concat_op = block._insert_op( + idx, + type='concat', + inputs=inputs, + outputs={'Out': [out]}, + attrs=attrs, + ) concat_op._set_attr('op_namescope', "/auto_parallel/reshard") return out @staticmethod - def insert_slice_op(block, idx, tensor, starts, ends, axes, new_var_name, - op_role): + def insert_slice_op( + block, idx, tensor, starts, ends, axes, new_var_name, op_role + ): """Insert slice op into block at the given block.""" # This is a hack to insert split op to get slice tensor # 1. [128, 128] => [64, 128]: split @@ -387,19 +407,19 @@ def insert_slice_op(block, idx, tensor, starts, ends, axes, new_var_name, # use assign if len(diff_dims) == 0: - out = block.create_var(name=new_var_name, - dtype=tensor.dtype, - type=tensor.type, - shape=slice_shape, - lod_level=tensor.lod_level) + out = block.create_var( + name=new_var_name, + dtype=tensor.dtype, + type=tensor.type, + shape=slice_shape, + lod_level=tensor.lod_level, + ) inputs = {'X': [tensor]} outputs = {"Out": [out]} attrs = {"in_place": False} - slice_op = block._insert_op(idx, - type="assign", - inputs=inputs, - outputs=outputs, - attrs=attrs) + slice_op = block._insert_op( + idx, type="assign", inputs=inputs, outputs=outputs, attrs=attrs + ) slice_op._set_attr('op_namescope', "/auto_parallel/reshard") return out @@ -420,23 +440,27 @@ def insert_slice_op(block, idx, tensor, starts, ends, axes, new_var_name, new_shape.append(item // num_or_sections) with paddle.static.program_guard(block.program): outs = [ - block.create_var(name=paddle.fluid.unique_name. - generate_with_ignorable_key(".".join( - ['split@RESHARD', 'tmp'])), - dtype=tensor.dtype, - shape=None, - type=tensor.type, - persistable=False, - lod_level=tensor.lod_level, - stop_gradient=False) + block.create_var( + name=paddle.fluid.unique_name.generate_with_ignorable_key( + ".".join(['split@RESHARD', 'tmp']) + ), + dtype=tensor.dtype, + shape=None, + type=tensor.type, + persistable=False, + lod_level=tensor.lod_level, + stop_gradient=False, + ) for i in range(num_or_sections) ] out = outs[cur_idx] - split_op = block._insert_op(idx, - type="split", - inputs=inputs, - outputs={'Out': outs}, - attrs=attrs) + split_op = block._insert_op( + idx, + type="split", + inputs=inputs, + outputs={'Out': outs}, + attrs=attrs, + ) split_op._set_attr('op_namescope', "/auto_parallel/reshard") return out @@ -449,17 +473,21 @@ def insert_slice_op(block, idx, tensor, starts, ends, axes, new_var_name, "starts": starts, "ends": ends, "infer_flags": infer_flags, - 'op_role': op_role + 'op_role': op_role, } - out = block.create_var(name=new_var_name, - dtype=tensor.dtype, - type=tensor.type, - lod_level=tensor.lod_level) - slice_op = block._insert_op(idx, - type="slice", - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs) + out = block.create_var( + name=new_var_name, + dtype=tensor.dtype, + type=tensor.type, + lod_level=tensor.lod_level, + ) + slice_op = block._insert_op( + idx, + type="slice", + inputs=inputs, + outputs={'Out': [out]}, + attrs=attrs, + ) slice_op._set_attr('op_namescope', "/auto_parallel/reshard") return out @@ -480,19 +508,20 @@ def insert_split_op(block, idx, tensor, num_or_sections, op_role, axis=0): outs = [ block.create_var( name=paddle.fluid.unique_name.generate_with_ignorable_key( - ".".join([helper.name, 'tmp'])), + ".".join([helper.name, 'tmp']) + ), dtype=tensor.dtype, shape=None, lod_level=tensor.lod_level, type=tensor.type, persistable=False, - stop_gradient=False) for i in range(num_or_sections) + stop_gradient=False, + ) + for i in range(num_or_sections) ] - split_op = block._insert_op(idx, - type="split", - inputs=inputs, - outputs={'Out': outs}, - attrs=attrs) + split_op = block._insert_op( + idx, type="split", inputs=inputs, outputs={'Out': outs}, attrs=attrs + ) split_op._set_attr('op_namescope', "/auto_parallel/reshard") return outs @@ -505,27 +534,30 @@ def insert_fill_constant_op(block, idx, op_role): with paddle.static.program_guard(block.program): out = block.create_var( name=paddle.fluid.unique_name.generate_with_ignorable_key( - ".".join([helper.name, 'tmp'])), + ".".join([helper.name, 'tmp']) + ), dtype=paddle.int64, shape=None, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) inputs = {} attrs = {'force_cpu': False} attrs['str_value'] = str(int("1")) attrs['value'] = int("1") attrs['dtype'] = out.dtype attrs['op_role'] = op_role - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=[0], - op_type='fill_constant') - fillconstant_op = block._insert_op(idx, - type='fill_constant', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs) + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=[0], op_type='fill_constant' + ) + fillconstant_op = block._insert_op( + idx, + type='fill_constant', + inputs=inputs, + outputs={'Out': [out]}, + attrs=attrs, + ) out.stop_gradient = True fillconstant_op._set_attr('op_namescope', "/auto_parallel/reshard") return out @@ -541,7 +573,8 @@ def insert_allgather_op(block, idx, tensor, ranks, op_role): if not group.is_instantiate(): # insert fill_constant op fill_constant_out = Inserter.insert_fill_constant_op( - block, idx, op_role) + block, idx, op_role + ) fill_constant_out.stop_gradient = True # insert c_allreduce_sum op @@ -553,8 +586,9 @@ def insert_allgather_op(block, idx, tensor, ranks, op_role): attrs={ 'ring_id': 0, 'use_calc_stream': True, - 'op_role': op_role - }) + 'op_role': op_role, + }, + ) allreduce_op._set_attr('op_namescope', "/auto_parallel/reshard") # insert c_sync_calc_stream op sync_calc_op = block._insert_op( @@ -562,7 +596,8 @@ def insert_allgather_op(block, idx, tensor, ranks, op_role): type="c_sync_calc_stream", inputs={'X': [fill_constant_out]}, outputs={'Out': [fill_constant_out]}, - attrs={'op_role': op_role}) + attrs={'op_role': op_role}, + ) sync_calc_op._set_attr('op_namescope', "/auto_parallel/reshard") idx_offset = 3 @@ -573,37 +608,42 @@ def insert_allgather_op(block, idx, tensor, ranks, op_role): with paddle.static.program_guard(block.program): allgather_out = block.create_var( name=paddle.fluid.unique_name.generate_with_ignorable_key( - ".".join([helper.name, 'tmp'])), + ".".join([helper.name, 'tmp']) + ), dtype=tensor.dtype, shape=None, lod_level=tensor.lod_level, type=tensor.type, persistable=False, - stop_gradient=False) - allgather_op = block._insert_op(idx + idx_offset, - type=op_type, - inputs={'X': [tensor]}, - outputs={'Out': [allgather_out]}, - attrs={ - 'ring_id': group.id, - 'use_calc_stream': True, - 'nranks': group.nranks, - 'op_role': op_role - }) + stop_gradient=False, + ) + allgather_op = block._insert_op( + idx + idx_offset, + type=op_type, + inputs={'X': [tensor]}, + outputs={'Out': [allgather_out]}, + attrs={ + 'ring_id': group.id, + 'use_calc_stream': True, + 'nranks': group.nranks, + 'op_role': op_role, + }, + ) allgather_op._set_attr('op_namescope', "/auto_parallel/reshard") idx_offset += 1 # insert split op - split_out = Inserter.insert_split_op(block, idx + idx_offset, - allgather_out, group.nranks, - op_role) + split_out = Inserter.insert_split_op( + block, idx + idx_offset, allgather_out, group.nranks, op_role + ) idx_offset += 1 tensor_list.extend(split_out) return tensor_list, idx_offset @staticmethod - def concat_partitions_with_op(partition_tensor_list, tensor, - partition_index, block, idx, op_role): + def concat_partitions_with_op( + partition_tensor_list, tensor, partition_index, block, idx, op_role + ): """Concat the tensors and insert concat op.""" if not partition_tensor_list: partition_tensor_list.append((tensor, partition_index)) @@ -611,18 +651,42 @@ def concat_partitions_with_op(partition_tensor_list, tensor, i = 0 has_concat = False while i < len(partition_tensor_list): - concat_axis, first_order, new_partition = Resharder.compute_concat_info( - partition_tensor_list[i][1], partition_index) + ( + concat_axis, + first_order, + new_partition, + ) = Resharder.compute_concat_info( + partition_tensor_list[i][1], partition_index + ) if concat_axis != -1: has_concat = True - _ = Inserter.insert_concat_op(block, idx[0], [partition_tensor_list[i][0], tensor], concat_axis, op_role) \ - if first_order == 0 else \ - Inserter.insert_concat_op(block, idx[0], [tensor, partition_tensor_list[i][0]], concat_axis, op_role) + _ = ( + Inserter.insert_concat_op( + block, + idx[0], + [partition_tensor_list[i][0], tensor], + concat_axis, + op_role, + ) + if first_order == 0 + else Inserter.insert_concat_op( + block, + idx[0], + [tensor, partition_tensor_list[i][0]], + concat_axis, + op_role, + ) + ) partition_tensor_list.pop(i) idx[0] += 1 - Inserter.concat_partitions_with_op(partition_tensor_list, _, - new_partition, block, - idx, op_role) + Inserter.concat_partitions_with_op( + partition_tensor_list, + _, + new_partition, + block, + idx, + op_role, + ) break i += 1 if not has_concat: @@ -636,7 +700,9 @@ class Remover: def remove_no_need_ops(auto_parallel_main_prog, dist_context, rank_id): """Remove no need ops in the main program""" not_remove_op_ref = [ - "create_py_reader", "create_double_buffer_reader", "read" + "create_py_reader", + "create_double_buffer_reader", + "read", ] # NOTE: The nested sub block is not be supported now. @@ -660,7 +726,9 @@ def remove_no_need_ops(auto_parallel_main_prog, dist_context, rank_id): for var_name in op.output_arg_names: dim_list.extend( get_var_with_recursion( - var_name, block, auto_parallel_main_prog).shape) + var_name, block, auto_parallel_main_prog + ).shape + ) for i in range(idx, -1, -1): if ops[i].type == "create_py_reader": ops[i]._set_attr("shape_concat", dim_list) @@ -671,10 +739,13 @@ def remove_no_need_ops(auto_parallel_main_prog, dist_context, rank_id): if op.type == "c_sync_comm_stream": need_save = [] for var_name in op.input_arg_names: - process_mesh = dist_context.get_tensor_dist_attr_for_program( - get_var_with_recursion( - var_name, block, - auto_parallel_main_prog)).process_mesh + process_mesh = ( + dist_context.get_tensor_dist_attr_for_program( + get_var_with_recursion( + var_name, block, auto_parallel_main_prog + ) + ).process_mesh + ) if rank_id in process_mesh.processes: need_save.append(var_name) if not need_save: @@ -690,15 +761,19 @@ def remove_no_need_ops(auto_parallel_main_prog, dist_context, rank_id): op_dist_attr = dist_context.get_op_dist_attr_for_program(op) if op_dist_attr is not None: op_process_mesh = op_dist_attr.process_mesh - if rank_id not in op_process_mesh.processes and op.type not in not_remove_op_ref: + if ( + rank_id not in op_process_mesh.processes + and op.type not in not_remove_op_ref + ): remove_op_idx.append(idx) for idx in remove_op_idx[::-1]: block._remove_op(idx) @staticmethod - def remove_no_need_vars(auto_parallel_main_prog, dist_params_grads, - feed_var_names): + def remove_no_need_vars( + auto_parallel_main_prog, dist_params_grads, feed_var_names + ): """Remove no need vars in the main program""" for block_idx, block in enumerate(auto_parallel_main_prog.blocks): remove_vars = set() @@ -721,7 +796,10 @@ def remove_no_need_vars(auto_parallel_main_prog, dist_params_grads, param_grad_map = {} for op in ops: if int(op.attr('op_role')) == int(OpRole.Optimize): - if "Param" in op.input_names and "Grad" in op.input_names: + if ( + "Param" in op.input_names + and "Grad" in op.input_names + ): param_name = op.input("Param")[0] grad_name = op.input("Grad")[0] param_grad_map[param_name] = grad_name @@ -740,7 +818,9 @@ def remove_no_need_vars(auto_parallel_main_prog, dist_params_grads, grad_name = dist_params_grads[idx][1].name if grad_name != param_grad_map[param_name]: dist_params_grads[idx] = ( - vars[param_name], vars[param_grad_map[param_name]]) + vars[param_name], + vars[param_grad_map[param_name]], + ) idx += 1 for var in remove_vars: @@ -749,23 +829,28 @@ def remove_no_need_vars(auto_parallel_main_prog, dist_params_grads, block._remove_var(var) @staticmethod - def remove_no_need_in_main(auto_parallel_main_prog, dist_context, rank_id, - dist_params_grads): + def remove_no_need_in_main( + auto_parallel_main_prog, dist_context, rank_id, dist_params_grads + ): """Remove no need vars and ops in the main program.""" - Remover.remove_no_need_ops(auto_parallel_main_prog, dist_context, - rank_id) - Resharder.change_while_op_input_and_output(auto_parallel_main_prog, - dist_context) + Remover.remove_no_need_ops( + auto_parallel_main_prog, dist_context, rank_id + ) + Resharder.change_while_op_input_and_output( + auto_parallel_main_prog, dist_context + ) # 'feed_var_names' cannot be removed from auto_parallel_main_prog feed_var_names = [] for var in sum(list(dist_context.serial_feed_vars.values()), []): feed_var_names.append(var.name) - Remover.remove_no_need_vars(auto_parallel_main_prog, dist_params_grads, - feed_var_names) + Remover.remove_no_need_vars( + auto_parallel_main_prog, dist_params_grads, feed_var_names + ) @staticmethod - def remove_no_need_in_startup(auto_parallel_main_prog, - auto_parallel_startup_prog): + def remove_no_need_in_startup( + auto_parallel_main_prog, auto_parallel_startup_prog + ): """Remove no need vars and ops in the startup program.""" main_input_vars = set() main_ops = auto_parallel_main_prog.global_block().ops @@ -850,28 +935,43 @@ class Resharder: dist_params_grads (list): The list contains the tuple of param and grad. batch_size (int): The batch size. Default: None. """ + while_block_info = {} - def __init__(self, - auto_parallel_main_prog, - auto_parallel_startup_prog, - rank_id, - dist_context, - dist_params_grads, - batch_size=None): - assert isinstance(auto_parallel_main_prog, Program), "The type of auto_parallel_main_prog should be Program, " \ - "but got {}.".format(type(auto_parallel_main_prog)) + def __init__( + self, + auto_parallel_main_prog, + auto_parallel_startup_prog, + rank_id, + dist_context, + dist_params_grads, + batch_size=None, + ): + assert isinstance(auto_parallel_main_prog, Program), ( + "The type of auto_parallel_main_prog should be Program, " + "but got {}.".format(type(auto_parallel_main_prog)) + ) if auto_parallel_startup_prog is not None: - assert isinstance(auto_parallel_main_prog, Program), "The type of auto_parallel_startup_prog should be Program or None, " \ - "but got {}.".format(type(auto_parallel_startup_prog)) - assert isinstance(rank_id, int), "The type of rank_id should be int, " \ - "but got {}.".format(type(rank_id)) - assert isinstance(dist_context, DistributedContext), "The type of dist_context should be DistributedContext, " \ - "but got {}.".format(type(dist_context)) + assert isinstance(auto_parallel_main_prog, Program), ( + "The type of auto_parallel_startup_prog should be Program or None, " + "but got {}.".format(type(auto_parallel_startup_prog)) + ) + assert isinstance( + rank_id, int + ), "The type of rank_id should be int, " "but got {}.".format( + type(rank_id) + ) + assert isinstance(dist_context, DistributedContext), ( + "The type of dist_context should be DistributedContext, " + "but got {}.".format(type(dist_context)) + ) if batch_size is not None: - assert isinstance(batch_size, int), "The type of batch_size should be int, " \ - "but got {}.".format(type(batch_size)) + assert isinstance( + batch_size, int + ), "The type of batch_size should be int, " "but got {}.".format( + type(batch_size) + ) self._auto_parallel_main_prog = auto_parallel_main_prog self._auto_parallel_startup_prog = auto_parallel_startup_prog @@ -943,29 +1043,37 @@ def compute_process_index(process, process_group, process_shape): for i in range(len(process_shape)): idx = relative_process // (product // process_shape[i]) product = product // process_shape[i] - relative_process = relative_process - relative_process // product * product + relative_process = ( + relative_process - relative_process // product * product + ) process_index.append(idx) return process_index @staticmethod - def compute_partition_index(process, complete_shape, dims_mapping, - process_shape, process_group): + def compute_partition_index( + process, complete_shape, dims_mapping, process_shape, process_group + ): """Compute the partition index in complete tensor.""" partition_shape = Resharder.compute_partition_shape( - complete_shape, dims_mapping, process_shape) - process_index = Resharder.compute_process_index(process, process_group, - process_shape) + complete_shape, dims_mapping, process_shape + ) + process_index = Resharder.compute_process_index( + process, process_group, process_shape + ) partition_index = [] for i in range(len(complete_shape)): if dims_mapping[i] == -1: partition_index.append([0, partition_shape[i]]) else: - partition_index.append([ - process_index[dims_mapping[i]] * partition_shape[i], - (process_index[dims_mapping[i]] + 1) * partition_shape[i] - ]) + partition_index.append( + [ + process_index[dims_mapping[i]] * partition_shape[i], + (process_index[dims_mapping[i]] + 1) + * partition_shape[i], + ] + ) return partition_index @@ -980,12 +1088,16 @@ def compute_concat_info(partition_index_x, partition_index_y): for idx, item in enumerate(partition_index_x): if item != partition_index_y[idx]: differ_count += 1 - if item[1] == partition_index_y[idx][ - 0] and item[0] < partition_index_y[idx][1]: + if ( + item[1] == partition_index_y[idx][0] + and item[0] < partition_index_y[idx][1] + ): concat_axis = idx new_partition.append([item[0], partition_index_y[idx][1]]) - elif item[0] == partition_index_y[idx][ - 1] and item[1] > partition_index_y[idx][0]: + elif ( + item[0] == partition_index_y[idx][1] + and item[1] > partition_index_y[idx][0] + ): first_order = 1 concat_axis = idx new_partition.append([partition_index_y[idx][0], item[1]]) @@ -1018,12 +1130,14 @@ def concat_partitions(partition_index_list, partition_index): has_concat = False while i < len(partition_index_list): concat_axis, _, new_partition = Resharder.compute_concat_info( - partition_index_list[i], partition_index) + partition_index_list[i], partition_index + ) if concat_axis != -1: has_concat = True partition_index_list.pop(i) - Resharder.concat_partitions(partition_index_list, - new_partition) + Resharder.concat_partitions( + partition_index_list, new_partition + ) break i += 1 if not has_concat: @@ -1035,7 +1149,8 @@ def change_while_op_input_and_output(auto_parallel_main_prog, dist_context): for sub_block_idx in Resharder.while_block_info: sub_block = auto_parallel_main_prog.blocks[sub_block_idx] parent_while_op_id = Resharder.while_block_info[sub_block_idx][ - "op_id"] + "op_id" + ] parent_block = auto_parallel_main_prog.blocks[sub_block.parent_idx] sub_block_op_inputs = set() @@ -1043,10 +1158,12 @@ def change_while_op_input_and_output(auto_parallel_main_prog, dist_context): for op in sub_block.ops: # skip the input and output of operators inserted in the reshard phase dist_op = dist_context.get_dist_op_for_program(op) - if dist_op or (op.type == "slice" and not dist_op) or ( - op.type == "split" - and not dist_op) or (op.type == "assign" - and not dist_op): + if ( + dist_op + or (op.type == "slice" and not dist_op) + or (op.type == "split" and not dist_op) + or (op.type == "assign" and not dist_op) + ): for var_name in op.output_arg_names: if var_name not in sub_block_op_outputs: sub_block_op_outputs.append(var_name) @@ -1077,8 +1194,9 @@ def change_while_op_input_and_output(auto_parallel_main_prog, dist_context): for var_name in while_op.output("Out"): for output_name in sub_block_op_outputs[::-1]: if output_name.find(var_name) != -1 and ( - len(var_name) == len(output_name) - or "@RESHARD" in output_name): + len(var_name) == len(output_name) + or "@RESHARD" in output_name + ): if output_name not in new_Out: new_Out.append(output_name) assert new_Out @@ -1087,8 +1205,9 @@ def change_while_op_input_and_output(auto_parallel_main_prog, dist_context): def is_overlapped(self, shape_x, shape_y): """Judge whether two partitions intersect on the specified dimension.""" overlapped = False - if (shape_y[0] <= shape_x[0] < shape_y[1]) or (shape_x[0] <= shape_y[0] - < shape_x[1]): + if (shape_y[0] <= shape_x[0] < shape_y[1]) or ( + shape_x[0] <= shape_y[0] < shape_x[1] + ): overlapped = True return overlapped @@ -1116,8 +1235,9 @@ def is_condition_replicative(self, op): # the dims mapping of condition tensor should be replicative for var_name in input_cond: - var = get_var_with_recursion(var_name, sub_block, - self.auto_parallel_main_prog) + var = get_var_with_recursion( + var_name, sub_block, self.auto_parallel_main_prog + ) dist_tensor = self.dist_context.get_dist_tensor_for_program(var) tensor_dist_attr = dist_tensor.dist_attr var_dims_mapping = tensor_dist_attr.dims_mapping @@ -1140,13 +1260,22 @@ def need_reshard(self, dist_tensor, dist_attr, op_input=True, dist_op=None): if op_input: op_input_dims_mapping = dist_attr[1] if all( - map(lambda x: x, [ - tensor_dims_mapping, tensor_process_mesh, - op_input_dims_mapping, op_process_mesh - ])): + map( + lambda x: x, + [ + tensor_dims_mapping, + tensor_process_mesh, + op_input_dims_mapping, + op_process_mesh, + ], + ) + ): # judge whether need reshard by dims_mapping if tensor_dims_mapping != op_input_dims_mapping: - if tensor_process_mesh not in self.dist_context.process_meshes: + if ( + tensor_process_mesh + not in self.dist_context.process_meshes + ): # assert whether -1 when union. for item in tensor_dims_mapping: if item != -1: @@ -1170,10 +1299,16 @@ def need_reshard(self, dist_tensor, dist_attr, op_input=True, dist_op=None): else: op_output_dims_mapping = dist_attr[1] if all( - map(lambda x: x, [ - tensor_dims_mapping, tensor_process_mesh, - op_output_dims_mapping, op_process_mesh - ])): + map( + lambda x: x, + [ + tensor_dims_mapping, + tensor_process_mesh, + op_output_dims_mapping, + op_process_mesh, + ], + ) + ): if tensor_dims_mapping != op_output_dims_mapping: raise ValueError( "It is not supported that tensor dims mapping is different from op output dims mapping." @@ -1190,10 +1325,9 @@ def get_op_process_meshes(self, op): op_process_mesh = dist_op.dist_attr.process_mesh for process_mesh in self.dist_context.process_meshes: - if set(process_mesh.processes) & (set( - op_process_mesh.processes)) and len( - process_mesh.processes) < len( - op_process_mesh.processes): + if set(process_mesh.processes) & ( + set(op_process_mesh.processes) + ) and len(process_mesh.processes) < len(op_process_mesh.processes): process_meshes.append(process_mesh) # it means the process mesh is not a union when process meshes is null @@ -1235,34 +1369,47 @@ def find_op_desc_seq(self, dist_tensor, dist_attr, serial=False): new_shape[0] = self.batch_size source_tensor.desc.set_shape(new_shape) - complete_shape = Resharder.compute_complete_shape( - source_tensor.shape, source_process_shape, - source_dims_mapping) if not serial else source_tensor.shape + complete_shape = ( + Resharder.compute_complete_shape( + source_tensor.shape, source_process_shape, source_dims_mapping + ) + if not serial + else source_tensor.shape + ) op_desc_seq = {} # TODO: if the target process group has the same process with source process group - if set(target_process_group).intersection(set( - source_process_group)) and set(target_process_group).difference( - set(source_process_group)): + if set(target_process_group).intersection( + set(source_process_group) + ) and set(target_process_group).difference(set(source_process_group)): pass elif target_process_group != source_process_group: partition_process_mapping_list = [] for source_process in source_process_group: # get partition index of source process - source_partition_index = Resharder.compute_partition_index(source_process, complete_shape, source_dims_mapping, \ - source_process_shape, source_process_group) + source_partition_index = Resharder.compute_partition_index( + source_process, + complete_shape, + source_dims_mapping, + source_process_shape, + source_process_group, + ) if not partition_process_mapping_list: # the item in partition_process_mapping_list is source_partition_index, which processes and whether has been used partition_process_mapping_list.append( - [source_partition_index, [source_process], [False]]) + [source_partition_index, [source_process], [False]] + ) else: partition_list = list( - [item[0] for item in partition_process_mapping_list]) + [item[0] for item in partition_process_mapping_list] + ) process_list = list( - [item[1] for item in partition_process_mapping_list]) + [item[1] for item in partition_process_mapping_list] + ) has_used = list( - [item[2] for item in partition_process_mapping_list]) + [item[2] for item in partition_process_mapping_list] + ) if partition_list.count(source_partition_index) == 1: index = partition_list.index(source_partition_index) @@ -1270,32 +1417,52 @@ def find_op_desc_seq(self, dist_tensor, dist_attr, serial=False): has_used[index].append(False) else: partition_process_mapping_list.append( - [source_partition_index, [source_process], [False]]) + [source_partition_index, [source_process], [False]] + ) for target_process in target_process_group: # has_sent means the source_partition_index has been sent to target_process has_sent = [] target_partition_index = Resharder.compute_partition_index( - target_process, complete_shape, target_dims_mapping, - target_process_shape, target_process_group) + target_process, + complete_shape, + target_dims_mapping, + target_process_shape, + target_process_group, + ) partition_index_list = [] all_partition_index_list = [] for source_process in source_process_group: source_partition_index = Resharder.compute_partition_index( - source_process, complete_shape, source_dims_mapping, - source_process_shape, source_process_group) + source_process, + complete_shape, + source_dims_mapping, + source_process_shape, + source_process_group, + ) to_send_process = None - if all(_ for _ in list(map(self.is_overlapped, source_partition_index, target_partition_index))) \ - and source_partition_index not in has_sent: - idx = list([ - item[0] for item in partition_process_mapping_list - ]).index(source_partition_index) - has_used = list([ - item[2] for item in partition_process_mapping_list - ])[idx] - process_list = list([ - item[1] for item in partition_process_mapping_list - ])[idx] + if ( + all( + _ + for _ in list( + map( + self.is_overlapped, + source_partition_index, + target_partition_index, + ) + ) + ) + and source_partition_index not in has_sent + ): + idx = list( + [item[0] for item in partition_process_mapping_list] + ).index(source_partition_index) + has_used = list( + [item[2] for item in partition_process_mapping_list] + )[idx] + process_list = list( + [item[1] for item in partition_process_mapping_list] + )[idx] i = 0 while i < len(has_used): if not has_used[i]: @@ -1308,7 +1475,9 @@ def find_op_desc_seq(self, dist_tensor, dist_attr, serial=False): has_used = list(map(lambda x: False, has_used)) to_send_process = process_list[0] has_used[0] = True - assert to_send_process is not None, "Failed to find the send process." + assert ( + to_send_process is not None + ), "Failed to find the send process." if to_send_process not in op_desc_seq.keys(): op_desc_seq[to_send_process] = [] @@ -1317,25 +1486,30 @@ def find_op_desc_seq(self, dist_tensor, dist_attr, serial=False): all_partition_index_list.append(source_partition_index) # append send and recv op desc - is_bool = ( - dist_tensor.serial_tensor.dtype == paddle.bool) - send_op_desc = SendOpDesc(source_partition_index, - to_send_process, - target_process, - is_bool=is_bool) - recv_op_desc = RecvOpDesc(source_partition_index, - to_send_process, - target_process, - is_bool=is_bool) + is_bool = dist_tensor.serial_tensor.dtype == paddle.bool + send_op_desc = SendOpDesc( + source_partition_index, + to_send_process, + target_process, + is_bool=is_bool, + ) + recv_op_desc = RecvOpDesc( + source_partition_index, + to_send_process, + target_process, + is_bool=is_bool, + ) op_desc_seq[to_send_process].append(send_op_desc) op_desc_seq[target_process].append(recv_op_desc) has_sent.append(source_partition_index) - Resharder.concat_partitions(partition_index_list, - source_partition_index) + Resharder.concat_partitions( + partition_index_list, source_partition_index + ) # append concat op desc op_desc_seq[target_process].append( - ConcatOpDesc(all_partition_index_list)) + ConcatOpDesc(all_partition_index_list) + ) # append slice op desc slice_starts = [] @@ -1345,17 +1519,21 @@ def find_op_desc_seq(self, dist_tensor, dist_attr, serial=False): to_slice_tensor_shape = [] for idx, item in enumerate(concatenated_partition_index): - slice_starts.append(target_partition_index[idx][0] - - item[0]) + slice_starts.append( + target_partition_index[idx][0] - item[0] + ) slice_ends.append(target_partition_index[idx][1] - item[0]) slices_axes.append(idx) to_slice_tensor_shape.append(item[1] - item[0]) op_desc_seq[target_process].append( - SliceOpDesc(slice_starts, - slice_ends, - slices_axes, - shape=to_slice_tensor_shape)) + SliceOpDesc( + slice_starts, + slice_ends, + slices_axes, + shape=to_slice_tensor_shape, + ) + ) # in the same process group, it will use allgahther and slice op. else: @@ -1365,16 +1543,26 @@ def find_op_desc_seq(self, dist_tensor, dist_attr, serial=False): process_index = [] for source_process in source_process_group: source_partition_index = Resharder.compute_partition_index( - source_process, complete_shape, source_dims_mapping, - source_process_shape, source_process_group) + source_process, + complete_shape, + source_dims_mapping, + source_process_shape, + source_process_group, + ) if source_partition_index not in partition_index_list: partition_index_list.append(source_partition_index) - process_index.append([[ - source_process, - ], source_partition_index]) + process_index.append( + [ + [ + source_process, + ], + source_partition_index, + ] + ) else: - process_index[partition_index_list.index( - source_partition_index)][0].append(source_process) + process_index[ + partition_index_list.index(source_partition_index) + ][0].append(source_process) for i in range(len(process_index[0][0])): group = [] @@ -1388,28 +1576,50 @@ def find_op_desc_seq(self, dist_tensor, dist_attr, serial=False): slice_ends = [] slices_axes = [] target_partition_index = Resharder.compute_partition_index( - process, complete_shape, target_dims_mapping, - target_process_shape, target_process_group) + process, + complete_shape, + target_dims_mapping, + target_process_shape, + target_process_group, + ) for idx, item in enumerate(target_partition_index): slice_starts.append(item[0]) slice_ends.append(item[1]) slices_axes.append(idx) to_slice_tensor_shape = dist_tensor.global_sizes() - slice_op_desc = SliceOpDesc(starts=slice_starts, - ends=slice_ends, - axes=slices_axes, - shape=to_slice_tensor_shape) - allgather_shape = None if not serial else dist_tensor.local_sizes( - rank=process) - op_desc_seq[process] = [AllGatherOpDesc(group=group, shape=allgather_shape, is_bool=(source_tensor.dtype == paddle.bool)), - ConcatOpDesc(partition_index_list=all_partition_index_list), slice_op_desc] \ - if len(group) > 1 else [slice_op_desc] + slice_op_desc = SliceOpDesc( + starts=slice_starts, + ends=slice_ends, + axes=slices_axes, + shape=to_slice_tensor_shape, + ) + allgather_shape = ( + None + if not serial + else dist_tensor.local_sizes(rank=process) + ) + op_desc_seq[process] = ( + [ + AllGatherOpDesc( + group=group, + shape=allgather_shape, + is_bool=(source_tensor.dtype == paddle.bool), + ), + ConcatOpDesc( + partition_index_list=all_partition_index_list + ), + slice_op_desc, + ] + if len(group) > 1 + else [slice_op_desc] + ) return op_desc_seq - def parse_op_desc(self, block, op_desc_seq, var_name, reshard_op, - dist_attr): + def parse_op_desc( + self, block, op_desc_seq, var_name, reshard_op, dist_attr + ): """Parse op desc sequence and insert op in the block""" tensor_list = [] partition_tensor_list = [] @@ -1422,55 +1632,84 @@ def parse_op_desc(self, block, op_desc_seq, var_name, reshard_op, if op.desc.id == reshard_op.desc.id: idx = index break - assert idx is not None, "The op for reshard cannot be found in the rank {} program.".format( - self.rank_id) + assert ( + idx is not None + ), "The op for reshard cannot be found in the rank {} program.".format( + self.rank_id + ) matched_op = block.ops[idx] - source_tensor = get_var_with_recursion(var_name, block, - self.auto_parallel_main_prog) + source_tensor = get_var_with_recursion( + var_name, block, self.auto_parallel_main_prog + ) for op_desc in op_desc_list: if isinstance(op_desc, AllGatherOpDesc): # noqa: F401 if var_name not in self.has_allgather.keys(): self.has_allgather[var_name] = [] - if not self.has_allgather[var_name] or op_desc.group not in list( - map(lambda x: x[0], self.has_allgather[var_name])): + if not self.has_allgather[ + var_name + ] or op_desc.group not in list( + map(lambda x: x[0], self.has_allgather[var_name]) + ): if op_desc.is_bool: # for bool data allgather, cast to int64 -> allgather -> cast bool out_cast = Inserter.insert_cast_op( - block, idx, source_tensor, - reshard_op.attr('op_role'), paddle.int64) + block, + idx, + source_tensor, + reshard_op.attr('op_role'), + paddle.int64, + ) tensor_list, idx_offset = Inserter.insert_allgather_op( - block, idx + 1, out_cast, op_desc.group, - reshard_op.attr('op_role')) + block, + idx + 1, + out_cast, + op_desc.group, + reshard_op.attr('op_role'), + ) idx += idx_offset tensor_name_list = [] for var in tensor_list: out_cast = Inserter.insert_cast_op( - block, idx, var, reshard_op.attr('op_role'), - paddle.bool) + block, + idx, + var, + reshard_op.attr('op_role'), + paddle.bool, + ) tensor_name_list.append(out_cast.name) idx += 1 self.has_allgather[var_name].append( - [op_desc.group, tensor_name_list]) + [op_desc.group, tensor_name_list] + ) else: tensor_list, idx_offset = Inserter.insert_allgather_op( - block, idx, source_tensor, op_desc.group, - reshard_op.attr('op_role')) + block, + idx, + source_tensor, + op_desc.group, + reshard_op.attr('op_role'), + ) idx += idx_offset tensor_name_list = [var.name for var in tensor_list] self.has_allgather[var_name].append( - [op_desc.group, tensor_name_list]) + [op_desc.group, tensor_name_list] + ) else: for item in self.has_allgather[var_name]: if op_desc.group == item[0]: tensor_list = [ get_var_with_recursion( - var_name, block, - self.auto_parallel_main_prog) + var_name, + block, + self.auto_parallel_main_prog, + ) for var_name in item[1] ] break - assert tensor_list, "The result of parsing allgather op should not be None." + assert ( + tensor_list + ), "The result of parsing allgather op should not be None." elif isinstance(op_desc, SendOpDesc): if var_name not in self.has_sent.keys(): @@ -1478,16 +1717,30 @@ def parse_op_desc(self, block, op_desc_seq, var_name, reshard_op, if op_desc.dst not in self.has_sent[var_name]: if op_desc.is_bool: out_cast = Inserter.insert_cast_op( - block, idx, source_tensor, - reshard_op.attr('op_role'), paddle.int64) - Inserter.insert_send_op(block, idx + 1, out_cast, - op_desc.src, op_desc.dst, - reshard_op.attr('op_role')) + block, + idx, + source_tensor, + reshard_op.attr('op_role'), + paddle.int64, + ) + Inserter.insert_send_op( + block, + idx + 1, + out_cast, + op_desc.src, + op_desc.dst, + reshard_op.attr('op_role'), + ) idx += 2 else: - Inserter.insert_send_op(block, idx, source_tensor, - op_desc.src, op_desc.dst, - reshard_op.attr('op_role')) + Inserter.insert_send_op( + block, + idx, + source_tensor, + op_desc.src, + op_desc.dst, + reshard_op.attr('op_role'), + ) idx += 1 self.has_sent[var_name].append(op_desc.dst) @@ -1506,13 +1759,23 @@ def parse_op_desc(self, block, op_desc_seq, var_name, reshard_op, shape=shape, lod_level=source_tensor.lod_level, dtype=paddle.int64, - type=source_tensor.type) - Inserter.insert_recv_op(block, idx, recv_tensor, - op_desc.src, op_desc.dst, - reshard_op.attr('op_role')) + type=source_tensor.type, + ) + Inserter.insert_recv_op( + block, + idx, + recv_tensor, + op_desc.src, + op_desc.dst, + reshard_op.attr('op_role'), + ) out_cast = Inserter.insert_cast_op( - block, idx + 1, recv_tensor, - reshard_op.attr('op_role'), paddle.bool) + block, + idx + 1, + recv_tensor, + reshard_op.attr('op_role'), + paddle.bool, + ) tensor_list.append(out_cast) idx += 2 self.has_recv[var_name][op_desc.src] = out_cast @@ -1522,26 +1785,45 @@ def parse_op_desc(self, block, op_desc_seq, var_name, reshard_op, shape=shape, lod_level=source_tensor.lod_level, dtype=source_tensor.dtype, - type=source_tensor.type) - Inserter.insert_recv_op(block, idx, recv_tensor, - op_desc.src, op_desc.dst, - reshard_op.attr('op_role')) + type=source_tensor.type, + ) + Inserter.insert_recv_op( + block, + idx, + recv_tensor, + op_desc.src, + op_desc.dst, + reshard_op.attr('op_role'), + ) # for lod tensor, need reset lod after received if recv_tensor.lod_level != 0: set_lod = False # use data lod to reset tensor lod - for tmp_block in self.auto_parallel_main_prog.blocks: + for ( + tmp_block + ) in self.auto_parallel_main_prog.blocks: for tmp_var_name in tmp_block.vars: tmp_var = tmp_block.vars[tmp_var_name] - if tmp_var.is_data and tmp_var.lod_level == recv_tensor.lod_level: - reset_lod_out = Inserter.insert_reset_lod_op( - block, idx + 1, recv_tensor, - tmp_var, reshard_op.attr('op_role')) + if ( + tmp_var.is_data + and tmp_var.lod_level + == recv_tensor.lod_level + ): + reset_lod_out = ( + Inserter.insert_reset_lod_op( + block, + idx + 1, + recv_tensor, + tmp_var, + reshard_op.attr('op_role'), + ) + ) tensor_list.append(reset_lod_out) idx += 2 self.has_recv[var_name][ - op_desc.src] = reset_lod_out + op_desc.src + ] = reset_lod_out set_lod = True break if set_lod: @@ -1559,16 +1841,24 @@ def parse_op_desc(self, block, op_desc_seq, var_name, reshard_op, idx_list = [idx] for index, tensor in enumerate(tensor_list): Inserter.concat_partitions_with_op( - partition_tensor_list, tensor, - partition_index_list[index], block, idx_list, - reshard_op.attr('op_role')) + partition_tensor_list, + tensor, + partition_index_list[index], + block, + idx_list, + reshard_op.attr('op_role'), + ) idx = idx_list[0] elif isinstance(op_desc, SliceOpDesc): - assert len( - partition_tensor_list) == 1 or not partition_tensor_list - to_slice_tensor = partition_tensor_list[0][0] if len( - partition_tensor_list) == 1 else source_tensor + assert ( + len(partition_tensor_list) == 1 or not partition_tensor_list + ) + to_slice_tensor = ( + partition_tensor_list[0][0] + if len(partition_tensor_list) == 1 + else source_tensor + ) new_name = unique_name.generate(var_name + "@RESHARD") target_tensor = Inserter.insert_slice_op( block, @@ -1578,7 +1868,8 @@ def parse_op_desc(self, block, op_desc_seq, var_name, reshard_op, ends=op_desc.ends, axes=op_desc.axes, new_var_name=new_name, - op_role=reshard_op.attr('op_role')) + op_role=reshard_op.attr('op_role'), + ) process_mesh = dist_attr[0] dims_mapping = dist_attr[1] @@ -1587,83 +1878,119 @@ def parse_op_desc(self, block, op_desc_seq, var_name, reshard_op, tensor_attr.dims_mapping = dims_mapping tensor_attr.process_mesh = process_mesh self.dist_context.set_tensor_dist_attr_for_program( - target_tensor, tensor_attr) + target_tensor, tensor_attr + ) if matched_op.type == "while": # var_reshard_mapping means the while op input need be changed to - if "var_reshard_mapping" not in Resharder.while_block_info[ - op.attr("sub_block").id].keys(): - Resharder.while_block_info[op.attr( - "sub_block").id]["var_reshard_mapping"] = {} - if var_name not in Resharder.while_block_info[op.attr( - "sub_block").id]["var_reshard_mapping"].keys(): + if ( + "var_reshard_mapping" + not in Resharder.while_block_info[ + op.attr("sub_block").id + ].keys() + ): Resharder.while_block_info[op.attr("sub_block").id][ - "var_reshard_mapping"][var_name] = [] + "var_reshard_mapping" + ] = {} + if ( + var_name + not in Resharder.while_block_info[ + op.attr("sub_block").id + ]["var_reshard_mapping"].keys() + ): + Resharder.while_block_info[op.attr("sub_block").id][ + "var_reshard_mapping" + ][var_name] = [] Resharder.while_block_info[op.attr("sub_block").id][ - "var_reshard_mapping"][var_name].append( - [dist_attr, target_tensor.name]) + "var_reshard_mapping" + ][var_name].append([dist_attr, target_tensor.name]) # rename op input name according to new name for op in block.ops: # just for while op while_op_X_append = [] for name in op.input_arg_names: - op_dist_attr = self.dist_context.get_op_dist_attr_for_program( - op) + op_dist_attr = ( + self.dist_context.get_op_dist_attr_for_program(op) + ) if name == var_name and op_dist_attr is not None: if op.desc.id() == matched_op.desc.id(): if matched_op.type == "while": old_name = name new_name = target_tensor.name assert old_name != new_name - op_input_dist_attr = op_dist_attr.get_input_dist_attr( - old_name) + op_input_dist_attr = ( + op_dist_attr.get_input_dist_attr( + old_name + ) + ) op_dist_attr.set_input_dist_attr( - new_name, op_input_dist_attr) + new_name, op_input_dist_attr + ) op_dist_attr.set_input_dims_mapping( - new_name, dims_mapping) - if old_name in op_dist_attr._inputs_dist_attrs: + new_name, dims_mapping + ) + if ( + old_name + in op_dist_attr._inputs_dist_attrs + ): op_dist_attr.del_input_dist_attr( - old_name) + old_name + ) while_op_X_append.append(new_name) continue else: op.desc._rename_input( - name, target_tensor.name) + name, target_tensor.name + ) old_name = name new_name = target_tensor.name assert old_name != new_name - op_input_dist_attr = op_dist_attr.get_input_dist_attr( - old_name) + op_input_dist_attr = ( + op_dist_attr.get_input_dist_attr( + old_name + ) + ) op_dist_attr.set_input_dist_attr( - new_name, op_input_dist_attr) + new_name, op_input_dist_attr + ) op_dist_attr.set_input_dims_mapping( - new_name, dims_mapping) + new_name, dims_mapping + ) op_dist_attr.del_input_dist_attr(old_name) continue op_process_mesh = op_dist_attr.process_mesh - op_input_dims_mapping = op_dist_attr.get_input_dims_mapping( - var_name) + op_input_dims_mapping = ( + op_dist_attr.get_input_dims_mapping(var_name) + ) # NOTE: For op whose process mesh is a union, its input will not be renamed by other op reshard result now which means that it will have more reshard operation. - if op_process_mesh == process_mesh and op_input_dims_mapping == dims_mapping: + if ( + op_process_mesh == process_mesh + and op_input_dims_mapping == dims_mapping + ): op.desc._rename_input(name, target_tensor.name) old_name = name new_name = target_tensor.name assert old_name != new_name - op_input_dist_attr = op_dist_attr.get_input_dist_attr( - old_name) + op_input_dist_attr = ( + op_dist_attr.get_input_dist_attr(old_name) + ) op_dist_attr.set_input_dist_attr( - new_name, op_input_dist_attr) + new_name, op_input_dist_attr + ) op_dist_attr.set_input_dims_mapping( - new_name, dims_mapping) + new_name, dims_mapping + ) op_dist_attr.del_input_dist_attr(old_name) # for while op, the input X should reset if while_op_X_append: proto = OpProtoHolder.instance().get_op_proto(op.type) - op.desc.set_input(proto.inputs[0].name, - op.input("X") + while_op_X_append) + op.desc.set_input( + proto.inputs[0].name, + op.input("X") + while_op_X_append, + ) def _get_subblock_input_attrs(self, op, var_name): # NOTE: Multi while loop is not supported @@ -1681,11 +2008,14 @@ def _get_subblock_input_attrs(self, op, var_name): if name == var_name: process_mesh = dist_attr.process_mesh input_dims_mapping = dist_attr.get_input_dims_mapping( - var_name) + var_name + ) has_exist = False for input_attr in input_attrs: - if process_mesh == input_attr[ - 0] and input_dims_mapping == input_attr[1]: + if ( + process_mesh == input_attr[0] + and input_dims_mapping == input_attr[1] + ): has_exist = True break if not has_exist: @@ -1698,10 +2028,9 @@ def _get_common_op_input_attrs(self, op, var_name): dist_attr = dist_op.dist_attr op_process_mesh = dist_attr.process_mesh for process_mesh in self.dist_context.process_meshes: - if set(process_mesh.processes) & (set( - op_process_mesh.processes)) and len( - process_mesh.processes) < len( - op_process_mesh.processes): + if set(process_mesh.processes) & ( + set(op_process_mesh.processes) + ) and len(process_mesh.processes) < len(op_process_mesh.processes): process_meshes.append(process_mesh) # it means that the process mesh is not a union when process meshes is none @@ -1737,7 +2066,8 @@ def _remove_global_process_mesh(self): for process in process_mesh.processes: processes.add(process) for idx, process_mesh in enumerate( - self.dist_context.process_meshes): + self.dist_context.process_meshes + ): if len(set(process_mesh.processes)) == len(processes): global_process_mesh_idx = idx break @@ -1757,7 +2087,8 @@ def _remove_global_process_mesh(self): def _change_subblock_op_input_and_output(self, block_idx, block): if "var_reshard_mapping" in Resharder.while_block_info[block_idx]: var_reshard_mapping = Resharder.while_block_info[block_idx][ - "var_reshard_mapping"] + "var_reshard_mapping" + ] for op in block.ops: for var_name in op.input_arg_names: if var_name in var_reshard_mapping: @@ -1766,9 +2097,11 @@ def _change_subblock_op_input_and_output(self, block_idx, block): dist_attr = dist_op.dist_attr target_name = None for item in var_reshard_mapping[var_name]: - if dist_attr.process_mesh == item[0][ - 0] and dist_attr.get_input_dims_mapping( - var_name) == item[0][1]: + if ( + dist_attr.process_mesh == item[0][0] + and dist_attr.get_input_dims_mapping(var_name) + == item[0][1] + ): target_name = item[1] break if target_name is None: @@ -1776,15 +2109,18 @@ def _change_subblock_op_input_and_output(self, block_idx, block): else: op.desc._rename_input(var_name, target_name) dist_op = self.dist_context.get_dist_op_for_program( - op) + op + ) op_dist_attr = dist_op.dist_attr old_name = var_name new_name = target_name assert old_name != new_name - op_input_dist_attr = op_dist_attr.get_input_dist_attr( - old_name) + op_input_dist_attr = ( + op_dist_attr.get_input_dist_attr(old_name) + ) op_dist_attr.set_input_dist_attr( - new_name, op_input_dist_attr) + new_name, op_input_dist_attr + ) op_dist_attr.del_input_dist_attr(old_name) # the outputs also need to be renamed when the output name is the same with input name in inplace op @@ -1804,9 +2140,11 @@ def _change_subblock_op_input_and_output(self, block_idx, block): new_name = target_name assert old_name != new_name op_output_dist_attr = op_dist_attr.get_output_dist_attr( - old_name) + old_name + ) op_dist_attr.set_output_dist_attr( - new_name, op_output_dist_attr) + new_name, op_output_dist_attr + ) op_dist_attr.del_output_dist_attr(old_name) def _reshard_input(self, block): @@ -1821,18 +2159,22 @@ def _reshard_input(self, block): dist_op = self.dist_context.get_dist_op_for_program(op) if dist_op is not None: - op_input_dist_attrs = [ - ] # [(op_process_mesh, op_input_dims_mapping), (op_process_mesh, op_input_dims_mapping)] + op_input_dist_attrs = ( + [] + ) # [(op_process_mesh, op_input_dims_mapping), (op_process_mesh, op_input_dims_mapping)] if op.type in _g_subblock_ops: if not self.is_condition_replicative(op): raise ValueError( "Please check the condition due to the dims mapping is not replicative." ) - if op.attr( - "sub_block").id not in Resharder.while_block_info: + if ( + op.attr("sub_block").id + not in Resharder.while_block_info + ): Resharder.while_block_info[op.attr("sub_block").id] = {} - Resharder.while_block_info[op.attr( - "sub_block").id]["op_id"] = op.desc.id() + Resharder.while_block_info[op.attr("sub_block").id][ + "op_id" + ] = op.desc.id() if op.type == "while": # condition var process mesh is the same with op and dims_mapping is replicative, so it do not need reshard @@ -1849,17 +2191,24 @@ def _reshard_input(self, block): # skip lod_tensor_blocking_queue_? name if "lod_tensor_blocking_queue" in var_name: continue - var = get_var_with_recursion(var_name, block, - self.auto_parallel_main_prog) + var = get_var_with_recursion( + var_name, block, self.auto_parallel_main_prog + ) dist_tensor = self.dist_context.get_dist_tensor_for_program( - var) + var + ) # judge whether union tensor dims_mapping all -1 is_union_process_mesh_tensor = False - if dist_tensor.dist_attr.process_mesh not in self.dist_context.process_meshes and self.dist_context.process_meshes: + if ( + dist_tensor.dist_attr.process_mesh + not in self.dist_context.process_meshes + and self.dist_context.process_meshes + ): is_union_process_mesh_tensor = True assert dist_tensor.dist_attr.dims_mapping.count( - -1) == len(dist_tensor.dist_attr.dims_mapping) + -1 + ) == len(dist_tensor.dist_attr.dims_mapping) op_input_attrs = self.get_op_input_attrs(op, var_name) for input_attr in op_input_attrs: @@ -1869,18 +2218,23 @@ def _reshard_input(self, block): if is_union_process_mesh_tensor: # if op process mesh is subset of union tensor process mesh, need no reshard if set(input_attr[0].processes) <= set( - dist_tensor.dist_attr.process_mesh.processes + dist_tensor.dist_attr.process_mesh.processes ): continue if dist_tensor is not None and self.need_reshard( - dist_tensor, input_attr): + dist_tensor, input_attr + ): reshard_op_desc = self.find_op_desc_seq( - dist_tensor, input_attr) - self.parse_op_desc(block, reshard_op_desc, var_name, - op, input_attr) + dist_tensor, input_attr + ) + self.parse_op_desc( + block, reshard_op_desc, var_name, op, input_attr + ) cur_op_count = len(block.ops) - idx_offset = idx_offset + cur_op_count - pre_op_count + idx_offset = ( + idx_offset + cur_op_count - pre_op_count + ) pre_op_count = cur_op_count idx = idx + idx_offset + 1 else: @@ -1895,34 +2249,43 @@ def _hadnle_recv(self, block, idx, var, op, send_rank, recv_rank): shape=var.shape, lod_level=var.lod_level, dtype=paddle.int64, - type=var.type) - Inserter.insert_recv_op(block, idx + 1, - recv_cast_out, send_rank, recv_rank, - op.attr('op_role')) + type=var.type, + ) + Inserter.insert_recv_op( + block, + idx + 1, + recv_cast_out, + send_rank, + recv_rank, + op.attr('op_role'), + ) reset_lod_out = None if var.lod_level != 0: set_lod = False for tmp_block in self.auto_parallel_main_prog.blocks: for tmp_var_name in tmp_block.vars: tmp_var = tmp_block.vars[tmp_var_name] - if tmp_var.is_data and tmp_var.lod_level == var.lod_level: + if ( + tmp_var.is_data + and tmp_var.lod_level == var.lod_level + ): reset_lod_out = block.create_var( - name=unique_name.generate(var.name + - "@RESETLOD"), + name=unique_name.generate( + var.name + "@RESETLOD" + ), shape=recv_cast_out.shape, type=recv_cast_out.type, dtype=recv_cast_out.dtype, - lod_level=recv_cast_out.lod_level) + lod_level=recv_cast_out.lod_level, + ) idx += 1 block._insert_op( idx, type="lod_reset", - inputs={ - 'X': recv_cast_out, - 'Y': tmp_var - }, + inputs={'X': recv_cast_out, 'Y': tmp_var}, outputs={'Out': reset_lod_out}, - attrs={'op_role': op.attr("op_role")}) + attrs={'op_role': op.attr("op_role")}, + ) set_lod = True break if set_lod: @@ -1930,18 +2293,21 @@ def _hadnle_recv(self, block, idx, var, op, send_rank, recv_rank): assert set_lod is True # cast int64 to bool - block._insert_op(idx + 2, - type='cast', - inputs={ - 'X': [recv_cast_out] if - reset_lod_out is None else [reset_lod_out] - }, - outputs={'Out': [var]}, - attrs={ - 'in_dtype': recv_cast_out.dtype, - 'out_dtype': var.dtype, - 'op_role': op.attr('op_role') - }) + block._insert_op( + idx + 2, + type='cast', + inputs={ + 'X': [recv_cast_out] + if reset_lod_out is None + else [reset_lod_out] + }, + outputs={'Out': [var]}, + attrs={ + 'in_dtype': recv_cast_out.dtype, + 'out_dtype': var.dtype, + 'op_role': op.attr('op_role'), + }, + ) else: if var.lod_level != 0: recv_out = block.create_var( @@ -1949,50 +2315,75 @@ def _hadnle_recv(self, block, idx, var, op, send_rank, recv_rank): shape=var.shape, lod_level=var.lod_level, dtype=var.int64, - type=var.type) - Inserter.insert_recv_op(block, idx + 1, recv_out, send_rank, - recv_rank, op.attr('op_role')) + type=var.type, + ) + Inserter.insert_recv_op( + block, + idx + 1, + recv_out, + send_rank, + recv_rank, + op.attr('op_role'), + ) set_lod = False for tmp_block in self.auto_parallel_main_prog.blocks: for tmp_var_name in tmp_block.vars: tmp_var = tmp_block.vars[tmp_var_name] - if tmp_var.is_data and tmp_var.lod_level == var.lod_level: + if ( + tmp_var.is_data + and tmp_var.lod_level == var.lod_level + ): idx += 1 block._insert_op( idx, type="lod_reset", - inputs={ - 'X': recv_out, - 'Y': tmp_var - }, + inputs={'X': recv_out, 'Y': tmp_var}, outputs={'Out': var}, - attrs={'op_role': op.attr("op_role")}) + attrs={'op_role': op.attr("op_role")}, + ) set_lod = True break if set_lod: break assert set_lod is True else: - Inserter.insert_recv_op(block, idx + 1, var, send_rank, - recv_rank, op.attr('op_role')) + Inserter.insert_recv_op( + block, + idx + 1, + var, + send_rank, + recv_rank, + op.attr('op_role'), + ) def _handle_send(self, block, idx, var, op, send_rank, recv_rank): if var.dtype == paddle.bool: - cast_out = Inserter.insert_cast_op(block, idx + 1, var, - op.attr('op_role'), paddle.int64) - Inserter.insert_send_op(block, idx + 2, cast_out, send_rank, - recv_rank, op.attr('op_role')) + cast_out = Inserter.insert_cast_op( + block, idx + 1, var, op.attr('op_role'), paddle.int64 + ) + Inserter.insert_send_op( + block, + idx + 2, + cast_out, + send_rank, + recv_rank, + op.attr('op_role'), + ) else: - Inserter.insert_send_op(block, idx + 1, var, send_rank, recv_rank, - op.attr('op_role')) + Inserter.insert_send_op( + block, idx + 1, var, send_rank, recv_rank, op.attr('op_role') + ) def _reshard_output(self, block): # insert send and recv op if output process mesh is different from tensor process mesh idx = 0 # skip reader and ops whose process mesh is union skip_ops = [ - "create_py_reader", "create_double_buffer_reader", "read", - "write_to_array", "read_from_array" + "create_py_reader", + "create_double_buffer_reader", + "read", + "write_to_array", + "read_from_array", ] global _g_special_ops skip_ops += _g_special_ops @@ -2004,59 +2395,86 @@ def _reshard_output(self, block): if dist_op is not None and op.type not in skip_ops: idx_offset = 0 for var_name in op.output_arg_names: - var = get_var_with_recursion(var_name, block, - self.auto_parallel_main_prog) + var = get_var_with_recursion( + var_name, block, self.auto_parallel_main_prog + ) dist_tensor = self.dist_context.get_dist_tensor_for_program( - var) + var + ) tensor_process_mesh = dist_tensor.dist_attr.process_mesh output_attr = [ dist_op.dist_attr.process_mesh, - dist_op.dist_attr.get_output_dims_mapping(var_name) + dist_op.dist_attr.get_output_dims_mapping(var_name), ] if dist_tensor is not None and self.need_reshard( - dist_tensor, output_attr, False): + dist_tensor, output_attr, False + ): tensor_processes = set( - tensor_process_mesh.processes) - ( - set(tensor_process_mesh.processes) - & set(output_attr[0].processes)) + tensor_process_mesh.processes + ) - ( + set(tensor_process_mesh.processes) + & set(output_attr[0].processes) + ) if tensor_processes: if len(tensor_processes) != len( - output_attr[0].processes): + output_attr[0].processes + ): if dist_tensor.dist_attr.dims_mapping.count( - -1) != len( - dist_tensor.dist_attr.dims_mapping - ) or output_attr[1].count(-1) != len( - output_attr[1]): + -1 + ) != len( + dist_tensor.dist_attr.dims_mapping + ) or output_attr[ + 1 + ].count( + -1 + ) != len( + output_attr[1] + ): raise ValueError( - "The dims_mapping must be -1") + "The dims_mapping must be -1" + ) else: for index, tensor_process in enumerate( - tensor_processes): + tensor_processes + ): recv_rank = tensor_process actual_index = index if index >= len( - output_attr[0].processes): + output_attr[0].processes + ): actual_index = ( - index - - len(output_attr[0].processes) + index + - len(output_attr[0].processes) ) % len(output_attr[0].processes) item = output_attr[0].processes[ - actual_index] + actual_index + ] if recv_rank == item: continue if self.rank_id == item: # if send bool data, cast then send self._handle_send( - block, idx, var, op, item, - recv_rank) + block, + idx, + var, + op, + item, + recv_rank, + ) if self.rank_id == recv_rank: # if recv bool data, recv then cast self._hadnle_recv( - block, idx, var, op, item, - recv_rank) + block, + idx, + var, + op, + item, + recv_rank, + ) else: for index, tensor_process in enumerate( - tensor_processes): + tensor_processes + ): recv_rank = tensor_process item = output_attr[0].processes[index] if recv_rank == item: @@ -2064,16 +2482,18 @@ def _reshard_output(self, block): if self.rank_id == item: # if send bool data, cast then send self._handle_send( - block, idx, var, op, item, - recv_rank) + block, idx, var, op, item, recv_rank + ) if self.rank_id == recv_rank: # if recv bool data, recv then cast self._hadnle_recv( - block, idx, var, op, item, - recv_rank) + block, idx, var, op, item, recv_rank + ) cur_op_count = len(block.ops) - idx_offset = idx_offset + cur_op_count - pre_op_count + idx_offset = ( + idx_offset + cur_op_count - pre_op_count + ) pre_op_count = cur_op_count idx = idx + idx_offset + 1 @@ -2095,13 +2515,17 @@ def reshard(self): self._reshard_output(block) # remove no need vars and ops in the main program - Remover.remove_no_need_in_main(self.auto_parallel_main_prog, - self.dist_context, self.rank_id, - self.dist_params_grads) + Remover.remove_no_need_in_main( + self.auto_parallel_main_prog, + self.dist_context, + self.rank_id, + self.dist_params_grads, + ) # remove no need vars and ops in the startip program - Remover.remove_no_need_in_startup(self.auto_parallel_main_prog, - self.auto_parallel_startup_prog) + Remover.remove_no_need_in_startup( + self.auto_parallel_main_prog, self.auto_parallel_startup_prog + ) # reset some variable when remove operation ended Resharder.while_block_info = {} @@ -2119,47 +2543,68 @@ def get_cost(self, op, tensor, cluster): return reshard_op_cost else: dist_tensor = self.dist_context.get_dist_tensor_for_program( - tensor) + tensor + ) # simplified processing: ignore union process mesh and output reshard dist_op = self.dist_context.get_dist_op_for_program(op) dims_mapping = dist_op.dist_attr.get_input_dims_mapping( - tensor.name) + tensor.name + ) process_mesh = dist_op.dist_attr.process_mesh dist_attr = [process_mesh, dims_mapping] if dist_tensor is not None and self.need_reshard( - dist_tensor, dist_attr): + dist_tensor, dist_attr + ): if tensor_name not in self._has_resharded: self._has_resharded[tensor_name] = [dist_op] else: for item in self._has_resharded[tensor_name]: item_dist_attr = item.dist_attr - item_dims_mapping = item_dist_attr.get_input_dims_mapping( - tensor_name) + item_dims_mapping = ( + item_dist_attr.get_input_dims_mapping( + tensor_name + ) + ) item_process_mesh = item_dist_attr.process_mesh - if dims_mapping == item_dims_mapping and item_process_mesh == process_mesh: + if ( + dims_mapping == item_dims_mapping + and item_process_mesh == process_mesh + ): return reshard_op_cost self._has_resharded[tensor_name].append(dist_op) - reshard_op_desc = self.find_op_desc_seq(dist_tensor, - dist_attr, - serial=True) + reshard_op_desc = self.find_op_desc_seq( + dist_tensor, dist_attr, serial=True + ) dtype = dist_tensor.serial_tensor.dtype reshard_op_cost = self.parse_op_desc_for_cost( - reshard_op_desc, dtype, cluster) + reshard_op_desc, dtype, cluster + ) return reshard_op_cost - def _concat_partitions_for_cost(self, partition_tensor_list, - partition_index, dtype, rank_id, - local_rank_comp_cost, cluster): + def _concat_partitions_for_cost( + self, + partition_tensor_list, + partition_index, + dtype, + rank_id, + local_rank_comp_cost, + cluster, + ): if not partition_tensor_list: partition_tensor_list.append(partition_index) else: i = 0 has_concat = False while i < len(partition_tensor_list): - concat_axis, first_order, new_partition = Resharder.compute_concat_info( - partition_tensor_list[i], partition_index) + ( + concat_axis, + first_order, + new_partition, + ) = Resharder.compute_concat_info( + partition_tensor_list[i], partition_index + ) if concat_axis != -1: has_concat = True concat_desc = {} @@ -2167,31 +2612,38 @@ def _concat_partitions_for_cost(self, partition_tensor_list, concat_desc["attrs"] = {"axis": concat_axis} if first_order == 0: concat_desc["inputs"] = { - "X": [(dtype, partition_tensor_list[i]), - (dtype, partition_index)] + "X": [ + (dtype, partition_tensor_list[i]), + (dtype, partition_index), + ] } else: concat_desc["inputs"] = { - "X": [(dtype, partition_index), - (dtype, partition_tensor_list[i])] + "X": [ + (dtype, partition_index), + (dtype, partition_tensor_list[i]), + ] } partition_tensor_list.pop(i) if rank_id not in local_rank_comp_cost: local_rank_comp_cost[rank_id] = [] local_rank_comp_cost[rank_id].append( - ConcatOpCost(op_desc=concat_desc, cluster=cluster)) - self._concat_partitions_for_cost(partition_tensor_list, - new_partition, dtype, - rank_id, - local_rank_comp_cost, - cluster) + ConcatOpCost(op_desc=concat_desc, cluster=cluster) + ) + self._concat_partitions_for_cost( + partition_tensor_list, + new_partition, + dtype, + rank_id, + local_rank_comp_cost, + cluster, + ) break i += 1 if not has_concat: partition_tensor_list.append(partition_index) def parse_op_desc_for_cost(self, reshard_op_desc, dtype, cluster): - def _get_idx(comm_ranks, group_ranks): res, is_the_same = None, False idx = 0 @@ -2222,28 +2674,41 @@ def _get_idx(comm_ranks, group_ranks): if isinstance(op_desc, SendOpDesc): group_ranks = [key, op_desc.dst] shape = op_desc.shape - send_desc = build_comm_desc("send_v2", group_ranks, dtype, - shape) + send_desc = build_comm_desc( + "send_v2", group_ranks, dtype, shape + ) idx, is_the_same = _get_idx(comm_ranks, group_ranks) if idx is None: - comm_costs.append([ - (group_ranks, - SendOpCost(op_desc=send_desc, - comm_context=comm_context)) - ]) + comm_costs.append( + [ + ( + group_ranks, + SendOpCost( + op_desc=send_desc, + comm_context=comm_context, + ), + ) + ] + ) comm_ranks.append(set(group_ranks)) else: if not is_the_same: comm_costs[idx].append( - (group_ranks, - SendOpCost(op_desc=send_desc, - comm_context=comm_context))) + ( + group_ranks, + SendOpCost( + op_desc=send_desc, + comm_context=comm_context, + ), + ) + ) elif isinstance(op_desc, AllGatherOpDesc): # NOTE: fill_const and other unnecessary op is not calculated because those cost is very small group_ranks = op_desc.group shape = op_desc.shape - allgather_desc = build_comm_desc("c_allgather", group_ranks, - dtype, shape) + allgather_desc = build_comm_desc( + "c_allgather", group_ranks, dtype, shape + ) split_inputs_shape = [] for idx, dim in enumerate(shape): if idx == 0: @@ -2252,18 +2717,29 @@ def _get_idx(comm_ranks, group_ranks): split_inputs_shape.append(dim) idx, is_the_same = _get_idx(comm_ranks, group_ranks) if idx is None: - comm_costs.append([ - (group_ranks, - AllgatherOpCost(op_desc=allgather_desc, - comm_context=comm_context)) - ]) + comm_costs.append( + [ + ( + group_ranks, + AllgatherOpCost( + op_desc=allgather_desc, + comm_context=comm_context, + ), + ) + ] + ) comm_ranks.append(set(group_ranks)) else: if not is_the_same: comm_costs[idx].append( - (group_ranks, - AllgatherOpCost(op_desc=allgather_desc, - comm_context=comm_context))) + ( + group_ranks, + AllgatherOpCost( + op_desc=allgather_desc, + comm_context=comm_context, + ), + ) + ) # calc the split op cost if key not in local_rank_comp_cost: local_rank_comp_cost[key] = [] @@ -2274,19 +2750,27 @@ def _get_idx(comm_ranks, group_ranks): } split_desc["attrs"] = {"num": len(group_ranks), "axis": 0} local_rank_comp_cost[key].append( - SplitOpCost(op_desc=split_desc, cluster=cluster)) + SplitOpCost(op_desc=split_desc, cluster=cluster) + ) elif isinstance(op_desc, ConcatOpDesc): partition_index_list = op_desc._partition_index_list for idx, partion_idex in enumerate(partition_index_list): self._concat_partitions_for_cost( - partition_tensor_list, partion_idex, dtype, key, - local_rank_comp_cost, cluster) + partition_tensor_list, + partion_idex, + dtype, + key, + local_rank_comp_cost, + cluster, + ) elif isinstance(op_desc, SliceOpDesc): if key not in local_rank_comp_cost: local_rank_comp_cost[key] = [] - assert len( - partition_tensor_list) == 1 or not partition_tensor_list + assert ( + len(partition_tensor_list) == 1 + or not partition_tensor_list + ) to_slice_tensor_shape = [] if len(partition_tensor_list) == 1: for item in partition_tensor_list[0]: @@ -2300,13 +2784,14 @@ def _get_idx(comm_ranks, group_ranks): "axes": op_desc.axes, "starts": op_desc.starts, "ends": op_desc.ends, - "infer_flags": infer_flags + "infer_flags": infer_flags, } slice_desc["inputs"] = { "Input": [(dtype, to_slice_tensor_shape)] } local_rank_comp_cost[key].append( - SliceOpCost(op_desc=slice_desc, cluster=cluster)) + SliceOpCost(op_desc=slice_desc, cluster=cluster) + ) res = (comm_costs, local_rank_comp_cost) diff --git a/python/paddle/distributed/auto_parallel/strategy.py b/python/paddle/distributed/auto_parallel/strategy.py index a504bbfed29b4..2440472badad3 100644 --- a/python/paddle/distributed/auto_parallel/strategy.py +++ b/python/paddle/distributed/auto_parallel/strategy.py @@ -17,7 +17,6 @@ class BaseConfig(object): - def __init__(self, category, config_dict=None): self._category = category self._config_dict = None @@ -27,7 +26,9 @@ def __init__(self, category, config_dict=None): else: raise ValueError( "Expected a dictionary. But received: {}".format( - config_dict)) + config_dict + ) + ) # Initialize attributes by the default config config = constants.get_category_default_config(self._category) for field, default_value in config.items(): @@ -73,49 +74,42 @@ def __deepcopy__(self, memo): class RecomputeConfig(BaseConfig): - def __init__(self, config_dict=None): category = constants.RECOMPUTE super(RecomputeConfig, self).__init__(category, config_dict) class AMPConfig(BaseConfig): - def __init__(self, config_dict=None): category = constants.AMP super(AMPConfig, self).__init__(category, config_dict) class ShardingConfig(BaseConfig): - def __init__(self, config_dict=None): category = constants.SHARDING super(ShardingConfig, self).__init__(category, config_dict) class GradientMergeConfig(BaseConfig): - def __init__(self, config_dict=None): category = constants.GRADIENT_MERGE super(GradientMergeConfig, self).__init__(category, config_dict) class QATConfig(BaseConfig): - def __init__(self, config_dict=None): category = constants.QAT super(QATConfig, self).__init__(category, config_dict) class TuningConfig(BaseConfig): - def __init__(self, config_dict=None): category = constants.TUNING super(TuningConfig, self).__init__(category, config_dict) class DatasetConfig(BaseConfig): - def __init__(self, config_dict=None): category = constants.DATASET super(DatasetConfig, self).__init__(category, config_dict) @@ -161,7 +155,8 @@ def __init__(self, config=None): # self._config_dict = yaml.load(yaml_file, Loader=yaml.Loader) else: raise ValueError( - "Expected a dictionary. But received: {}".format(config)) + "Expected a dictionary. But received: {}".format(config) + ) else: self._config_dict = {} diff --git a/python/paddle/distributed/auto_parallel/tuner/algorithms.py b/python/paddle/distributed/auto_parallel/tuner/algorithms.py index f892a7838fe7a..efc3358ebe41a 100644 --- a/python/paddle/distributed/auto_parallel/tuner/algorithms.py +++ b/python/paddle/distributed/auto_parallel/tuner/algorithms.py @@ -31,6 +31,7 @@ class AlgorithmBase(ABC): In another word, the key "algorithm" for this class is the search space pruning rules specific for the given optimization scenario. """ + _REGISTERED_ALGORITHMS = {} name = None @@ -88,7 +89,6 @@ def get_config_from_trial(self, trial): def register_algor(name): - def impl(cls): AlgorithmBase._register(name, cls) cls.name = name @@ -121,7 +121,8 @@ def _init_spaces(self): assert set(stage_range).issubset( set([0, 1, 2, 3]) ), "Sharding Stage should belong into range within 0 - 3 but got {}.".format( - stage_range) + stage_range + ) stage_range.sort(reverse=True) else: stage_range = list(range(self._max_stage + 1)).sort(reverse=True) diff --git a/python/paddle/distributed/auto_parallel/tuner/config.py b/python/paddle/distributed/auto_parallel/tuner/config.py index 9073013127ab1..4ed6340eccb6f 100644 --- a/python/paddle/distributed/auto_parallel/tuner/config.py +++ b/python/paddle/distributed/auto_parallel/tuner/config.py @@ -109,8 +109,10 @@ def _initialize(self, user_config): self._project_dir = project_dir for p in _tuning_supported_passes: - if getattr(self._dist_strategy, p) and _get_pass_config( - self._dist_strategy, p).enable_tuning: + if ( + getattr(self._dist_strategy, p) + and _get_pass_config(self._dist_strategy, p).enable_tuning + ): # TODO distinguish different args of each passes self._tuning_passes_name.add(p) diff --git a/python/paddle/distributed/auto_parallel/tuner/optimization_tuner.py b/python/paddle/distributed/auto_parallel/tuner/optimization_tuner.py index 518e4fda1187d..a5cdbc7f95a91 100644 --- a/python/paddle/distributed/auto_parallel/tuner/optimization_tuner.py +++ b/python/paddle/distributed/auto_parallel/tuner/optimization_tuner.py @@ -34,7 +34,10 @@ from paddle.distributed.auto_parallel.completion import Completer from paddle.distributed.auto_parallel.reshard import Resharder from paddle.distributed.auto_parallel.partitioner import Partitioner -from paddle.distributed.auto_parallel.process_group import clear_all_process_groups, get_all_process_groups +from paddle.distributed.auto_parallel.process_group import ( + clear_all_process_groups, + get_all_process_groups, +) from paddle.distributed.auto_parallel.utils import debug_program from paddle.distributed.auto_parallel.utils import set_grad_var_shape @@ -82,9 +85,8 @@ def parse_process_groups(): def get_metric(results): assert isinstance( - results, - dict), "results should be type of dictionary, but got {}.".format( - type(results)) + results, dict + ), "results should be type of dictionary, but got {}.".format(type(results)) if 'Throughtput' in results and isinstance(results['Throughtput'], float): return float(results['Throughtput']) else: @@ -108,20 +110,25 @@ def _copy_context(ref_dist_context): clear_all_process_groups() new_dist_context = DistributedContext() - new_dist_context._serial_main_program = ref_dist_context.serial_main_program.clone( - for_test=False) - new_dist_context._serial_startup_program = ref_dist_context.serial_startup_program.clone( - for_test=False) + new_dist_context._serial_main_program = ( + ref_dist_context.serial_main_program.clone(for_test=False) + ) + new_dist_context._serial_startup_program = ( + ref_dist_context.serial_startup_program.clone(for_test=False) + ) # mapping variable into new dist context if getattr(ref_dist_context, '_params_grads', None): new_dist_context._params_grads = _get_new_params_grads( new_dist_context.serial_main_program, ref_dist_context.serial_main_program, - ref_dist_context._params_grads) + ref_dist_context._params_grads, + ) new_dist_context._serial_loss = _get_new_loss( new_dist_context.serial_main_program, - ref_dist_context.serial_main_program, ref_dist_context.serial_loss) + ref_dist_context.serial_main_program, + ref_dist_context.serial_loss, + ) for key, var_list in ref_dist_context._serial_feed_vars.items(): new_var_list = [] @@ -129,7 +136,8 @@ def _copy_context(ref_dist_context): block_idx = var.block.idx var_name = var.name var = new_dist_context._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) new_var_list.append(var) new_dist_context._serial_feed_vars[key] = new_var_list @@ -143,7 +151,8 @@ def _copy_context(ref_dist_context): block_idx = var.block.idx var_name = var.name var = new_dist_context._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) new_inner_var_list.append(var) new_var_list.append(new_inner_var_list) else: @@ -151,21 +160,26 @@ def _copy_context(ref_dist_context): block_idx = var.block.idx var_name = var.name var = new_dist_context._serial_main_program.blocks[ - block_idx]._var_recursive(var_name) + block_idx + ]._var_recursive(var_name) new_var_list.append(var) new_dist_context._serial_fetch_vars[key] = new_var_list # copy information in forward and backward new_dist_context._serial_optimizer = copy.deepcopy( - ref_dist_context.serial_optimizer) + ref_dist_context.serial_optimizer + ) new_dist_context._dist_tensors_for_program = copy.deepcopy( - ref_dist_context._dist_tensors_for_program) + ref_dist_context._dist_tensors_for_program + ) new_dist_context._dist_ops_for_program = copy.deepcopy( - ref_dist_context._dist_ops_for_program) + ref_dist_context._dist_ops_for_program + ) for pm in ref_dist_context.process_meshes: new_dist_context.add_process_mesh(pm) new_dist_context._dist_op_context = copy.deepcopy( - ref_dist_context._dist_op_context) + ref_dist_context._dist_op_context + ) new_dist_context._block_state = copy.deepcopy(ref_dist_context.block_state) return new_dist_context @@ -229,28 +243,39 @@ def device_id(self): def _build_programs_without_optimization(self): serial_main_program = self._baseline_dist_context.serial_main_program - serial_startup_program = self._baseline_dist_context.serial_startup_program + serial_startup_program = ( + self._baseline_dist_context.serial_startup_program + ) serial_loss = self._baseline_dist_context.serial_loss with program_guard(serial_main_program, serial_startup_program): params_grads = append_backward( serial_loss, - distop_context=self._baseline_dist_context.dist_op_context) + distop_context=self._baseline_dist_context.dist_op_context, + ) self._baseline_completer.complete_backward_annotation( - serial_main_program) + serial_main_program + ) self._baseline_dist_context.block_state.parse_backward_blocks( - serial_main_program) + serial_main_program + ) self._baseline_dist_context._params_grads = params_grads if self._config.verbose: baseline_dir = os.path.join(self.project_dir, "baseline") if not os.path.exists(baseline_dir): pathlib.Path(baseline_dir).mkdir(parents=True, exist_ok=True) - debug_program(self._baseline_dist_context._serial_main_program, - baseline_dir, "main") - debug_program(self._baseline_dist_context._serial_startup_program, - baseline_dir, "startup") + debug_program( + self._baseline_dist_context._serial_main_program, + baseline_dir, + "main", + ) + debug_program( + self._baseline_dist_context._serial_startup_program, + baseline_dir, + "startup", + ) def _select_tuning_algorithm(self): @@ -275,18 +300,22 @@ def _apply_optimization(self, trial): # TODO AMP Pass should not use loss var config["loss"] = dist_context.serial_loss - config["input_data"] = self._baseline_dist_context.serial_feed_vars["inputs"] \ + config["input_data"] = ( + self._baseline_dist_context.serial_feed_vars["inputs"] + self._baseline_dist_context.serial_feed_vars["labels"] + ) if config["use_pure_fp16"]: config["base_opt"] = dist_context.serial_optimizer auto_parallel_fp16_pass = new_pass("auto_parallel_fp16", config) - auto_parallel_fp16_pass.apply([main_program], [startup_program], - pass_context) + auto_parallel_fp16_pass.apply( + [main_program], [startup_program], pass_context + ) dist_context.serial_loss = auto_parallel_fp16_pass.get_loss() else: auto_parallel_amp_pass = new_pass("auto_parallel_amp", config) - auto_parallel_amp_pass.apply([main_program], [startup_program], - pass_context) + auto_parallel_amp_pass.apply( + [main_program], [startup_program], pass_context + ) dist_context.serial_loss = auto_parallel_amp_pass.get_loss() if new_strategy.recompute.enable: @@ -294,27 +323,40 @@ def _apply_optimization(self, trial): config["dist_context"] = dist_context config["no_grad_set"] = None config["loss"] = dist_context.serial_loss - auto_parallel_recompute_pass = new_pass("auto_parallel_recompute", - config) - auto_parallel_recompute_pass.apply([main_program], - [startup_program], pass_context) + auto_parallel_recompute_pass = new_pass( + "auto_parallel_recompute", config + ) + auto_parallel_recompute_pass.apply( + [main_program], [startup_program], pass_context + ) # Do logical partition partitioner = Partitioner(dist_context, self.rank) - dist_main_prog, dist_startup_prog, dist_params_grads = partitioner.partition( - main_program, startup_program, dist_context._params_grads) + ( + dist_main_prog, + dist_startup_prog, + dist_params_grads, + ) = partitioner.partition( + main_program, startup_program, dist_context._params_grads + ) # Generate optimizer # FIXME should be remove from apply pass after pass support optimizers with program_guard(dist_main_prog, dist_startup_prog): optimizer_ops = dist_context.serial_optimizer.apply_gradients( - dist_params_grads) + dist_params_grads + ) completer.complete_update_annotation(dist_main_prog) # Do reshard process set_grad_var_shape(dist_main_prog, dist_context) - resharder = Resharder(dist_main_prog, dist_startup_prog, self.rank, - dist_context, dist_params_grads) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + self.rank, + dist_context, + dist_params_grads, + ) resharder.reshard() if new_strategy.sharding.enable: @@ -322,21 +364,27 @@ def _apply_optimization(self, trial): config["dist_context"] = dist_context config["params_grads"] = dist_params_grads config["global_rank"] = self.rank - auto_parallel_sharding_pass = new_pass("auto_parallel_sharding", - config) - auto_parallel_sharding_pass.apply([dist_main_prog], - [dist_startup_prog], pass_context) + auto_parallel_sharding_pass = new_pass( + "auto_parallel_sharding", config + ) + auto_parallel_sharding_pass.apply( + [dist_main_prog], [dist_startup_prog], pass_context + ) if new_strategy.gradient_merge.enable: config = copy.deepcopy(new_strategy.gradient_merge.to_dict()) config["dist_context"] = dist_context config["params_grads"] = dist_params_grads auto_parallel_gradient_merge_pass = new_pass( - "auto_parallel_gradient_merge_pass", config) - auto_parallel_gradient_merge_pass.apply([dist_main_prog], - [dist_startup_prog], - pass_context) - trial.main_program, trial.startup_program = dist_main_prog, dist_startup_prog + "auto_parallel_gradient_merge_pass", config + ) + auto_parallel_gradient_merge_pass.apply( + [dist_main_prog], [dist_startup_prog], pass_context + ) + trial.main_program, trial.startup_program = ( + dist_main_prog, + dist_startup_prog, + ) return trial def _get_profile_context(self, trial, result_path): @@ -344,16 +392,18 @@ def _get_profile_context(self, trial, result_path): profile_ctx = {} profile_ctx['distributed_env'] = copy.deepcopy( - paddle.distributed.ParallelEnv()) + paddle.distributed.ParallelEnv() + ) profile_ctx['group_map'] = parse_process_groups() profile_ctx[ - "loss_var_name"] = self._baseline_dist_context.serial_loss.name + "loss_var_name" + ] = self._baseline_dist_context.serial_loss.name profile_ctx[ - "main_program_decs"] = trial.main_program.desc.serialize_to_string( - ) + "main_program_decs" + ] = trial.main_program.desc.serialize_to_string() profile_ctx[ - "startup_program_decs"] = trial.startup_program.desc.serialize_to_string( - ) + "startup_program_decs" + ] = trial.startup_program.desc.serialize_to_string() self._dataset.batch_size = self._batch_size self._dataset.input_names = self._get_input_names() @@ -375,19 +425,25 @@ def _launch_profile(self, ctx_path, trial_dir): else: coverage_args = [] - profile_args = " ".join([ - "--rank", - str(self.rank), - "--device_id", - str(self.device_id), - "--ctx_filename", - ctx_path, - "--profile_start_step", - str(self._config.profile_start_step), - "--profile_end_step", - str(self._config.profile_end_step), - ]) - cmd_args = "-m paddle.distributed.auto_parallel.tuner.profiler" + " " + profile_args + profile_args = " ".join( + [ + "--rank", + str(self.rank), + "--device_id", + str(self.device_id), + "--ctx_filename", + ctx_path, + "--profile_start_step", + str(self._config.profile_start_step), + "--profile_end_step", + str(self._config.profile_end_step), + ] + ) + cmd_args = ( + "-m paddle.distributed.auto_parallel.tuner.profiler" + + " " + + profile_args + ) cmd = [sys.executable, "-u"] + coverage_args + shlex.split(cmd_args) parent_env = copy.copy(os.environ.copy()) @@ -400,10 +456,11 @@ def _launch_profile(self, ctx_path, trial_dir): # TODO if any rank hang or fail, kill all processes self._logger.debug("Executing cmd:\n{} .".format(" ".join(cmd))) # new_process = subprocess.Popen(cmd, env=new_env) - with open(os.path.join(trial_dir, "stdout.log" + str(self.rank)), - "wb") as out, open( - os.path.join(trial_dir, "stderr.log" + str(self.rank)), - "wb") as err: + with open( + os.path.join(trial_dir, "stdout.log" + str(self.rank)), "wb" + ) as out, open( + os.path.join(trial_dir, "stderr.log" + str(self.rank)), "wb" + ) as err: result = subprocess.Popen(cmd, stdout=out, stderr=err, env=new_env) result.wait() out.flush() @@ -455,13 +512,18 @@ def _evaluate_trial(self, trial): elif self._config.mode == "COSTMODEL": raise NotImplementedError( - "COSTMODEL mode for optimization tuning is not supported yet!") + "COSTMODEL mode for optimization tuning is not supported yet!" + ) else: - raise NotImplementedError("invalid evaluation mode: {}".format( - self._config.mode)) + raise NotImplementedError( + "invalid evaluation mode: {}".format(self._config.mode) + ) - self._logger.info("Trial {} evaluation finish with {}.".format( - trial.name, parse_results(results))) + self._logger.info( + "Trial {} evaluation finish with {}.".format( + trial.name, parse_results(results) + ) + ) return results def _update(self, i, trial, results): @@ -496,11 +558,12 @@ def summary(self): Tuning Result Summary Run total {} trials with {} min. The best trial is: [{}], whose configuration is following: - """.format(len(self._finished_trials), - (time.time() - self._tuning_start_time) / 60, - best_trial.name) - summary_ += "\n" + best_trial.summary() + "\n"\ - + """.format( + len(self._finished_trials), + (time.time() - self._tuning_start_time) / 60, + best_trial.name, + ) + summary_ += "\n" + best_trial.summary() + "\n" self._logger.info(summary_) with open(os.path.join(self.project_dir, "summary.txt"), "w+") as fw: for line in summary_.split("\n"): @@ -532,7 +595,8 @@ def tune(self): self._tuning_start_time = time.time() self._algorithm.collect_model_info( self._baseline_dist_context.serial_main_program, - self._baseline_dist_context.serial_startup_program) + self._baseline_dist_context.serial_startup_program, + ) # main search loop i = 0 @@ -554,10 +618,15 @@ def tune(self): # early stop i += 1 - if self._config.early_stop and self._config.early_stop <= i - self._best_iter: + if ( + self._config.early_stop + and self._config.early_stop <= i - self._best_iter + ): self._logger.info( - "Early stop the Tuning since there is no better trial found within [{}] trials" - .format(self._config.early_stop)) + "Early stop the Tuning since there is no better trial found within [{}] trials".format( + self._config.early_stop + ) + ) break # step5: summary the best config and return diff --git a/python/paddle/distributed/auto_parallel/tuner/parallel_tuner.py b/python/paddle/distributed/auto_parallel/tuner/parallel_tuner.py index 24ee382f7f75a..0ed03defbed86 100644 --- a/python/paddle/distributed/auto_parallel/tuner/parallel_tuner.py +++ b/python/paddle/distributed/auto_parallel/tuner/parallel_tuner.py @@ -33,15 +33,16 @@ class ParallelTuner: - - def __init__(self, - dist_context, - mode="train", - max_trials=25, - tuner_id=None, - seed=None, - logger=None, - loop_count=10): + def __init__( + self, + dist_context, + mode="train", + max_trials=25, + tuner_id=None, + seed=None, + logger=None, + loop_count=10, + ): self._loop_count = loop_count self._estimator = None self._dist_context = dist_context @@ -49,7 +50,8 @@ def __init__(self, self._mode = mode self._cluster = self._dist_context.cluster self._num_machines = self._cluster.get_num_machines() - self._num_devices_per_machine = self._cluster.get_num_devices_per_machine( + self._num_devices_per_machine = ( + self._cluster.get_num_devices_per_machine() ) self._space = TunableSpace() self._objective = "time" @@ -58,15 +60,17 @@ def __init__(self, self._tuner_id = tuner_id self._seed = seed if seed is not None else 9999 - print("seed", - self._seed, - "mode", - self._mode, - "num_machies", - self._num_machines, - "num_devices_per_machine", - self._num_devices_per_machine, - flush=True) + print( + "seed", + self._seed, + "mode", + self._mode, + "num_machies", + self._num_machines, + "num_devices_per_machine", + self._num_devices_per_machine, + flush=True, + ) self._seed_state = self._seed self._logger = logger self._max_collisions = 3 @@ -89,8 +93,12 @@ def __init__(self, self._cached_candidates_info = defaultdict(list) self._special_ops = [ - "create_py_reader", "create_double_buffer_reader", "read", "while", - "read_from_array", "write_to_array" + "create_py_reader", + "create_double_buffer_reader", + "read", + "while", + "read_from_array", + "write_to_array", ] # Each parallel strategy has two elements. The First one is for distributed tensors, @@ -100,44 +108,62 @@ def __init__(self, self._completer = Completer(self._dist_context) - self._parallelizer = Parallelizer(self._mode, self._completer, - self._dist_context) + self._parallelizer = Parallelizer( + self._mode, self._completer, self._dist_context + ) - def _generate_combination(self, - elements, - target, - idx, - partial_candidate, - candidates, - num_candidates=None): + def _generate_combination( + self, + elements, + target, + idx, + partial_candidate, + candidates, + num_candidates=None, + ): if target == 0: candidates.append(copy.deepcopy(partial_candidate)) return - if target < 0 or idx == len(elements) \ - or len(candidates) > num_candidates: + if ( + target < 0 + or idx == len(elements) + or len(candidates) > num_candidates + ): return # Use partial_candidate.append(elements[idx]) - self._generate_combination(elements, target - elements[idx], idx, - partial_candidate, candidates, - num_candidates) + self._generate_combination( + elements, + target - elements[idx], + idx, + partial_candidate, + candidates, + num_candidates, + ) # Not use partial_candidate.pop() - self._generate_combination(elements, target, idx + 1, partial_candidate, - candidates, num_candidates) - - def _permute_combination(self, - combination, - target, - check, - partial_candidate, - candidates, - num_candidates=None, - skip_prob=None): - if num_candidates is not None \ - and len(candidates) == num_candidates: + self._generate_combination( + elements, + target, + idx + 1, + partial_candidate, + candidates, + num_candidates, + ) + + def _permute_combination( + self, + combination, + target, + check, + partial_candidate, + candidates, + num_candidates=None, + skip_prob=None, + ): + if num_candidates is not None and len(candidates) == num_candidates: return if len(partial_candidate) == len(combination): @@ -149,13 +175,22 @@ def _permute_combination(self, continue if self._rng.choice([True, False], p=[skip_prob, 1 - skip_prob]): continue - if i > 0 and combination[i] == combination[i - 1] \ - and check[i -1] == 0: + if ( + i > 0 + and combination[i] == combination[i - 1] + and check[i - 1] == 0 + ): continue check[i] = 1 - self._permute_combination(combination, target, check, - partial_candidate + [combination[i]], - candidates, num_candidates, skip_prob) + self._permute_combination( + combination, + target, + check, + partial_candidate + [combination[i]], + candidates, + num_candidates, + skip_prob, + ) check[i] = 0 def _partition_number(self, target): @@ -166,8 +201,14 @@ def _partition_number(self, target): seed_candidates = [] num_seed_candidates = 1000 partial_results = [] - self._generate_combination(elements, target, 0, partial_results, - seed_candidates, num_seed_candidates) + self._generate_combination( + elements, + target, + 0, + partial_results, + seed_candidates, + num_seed_candidates, + ) candidates = [] for seed_candidate in seed_candidates: @@ -178,10 +219,16 @@ def _partition_number(self, target): if target <= 8: skip_prob = 0.0 else: - skip_prob = (len(seed_candidate) / target) - self._permute_combination(seed_candidate, target, check, [], - cur_candidates, num_cur_candidates, - skip_prob) + skip_prob = len(seed_candidate) / target + self._permute_combination( + seed_candidate, + target, + check, + [], + cur_candidates, + num_cur_candidates, + skip_prob, + ) candidates.extend(cur_candidates) return candidates @@ -190,8 +237,9 @@ def _partition_devices(self, num_machines, num_devices_per_machine): intra_node_partitions = self._partition_number(num_devices_per_machine) return inter_node_partitions, intra_node_partitions - def _generate_process_mesh_list(self, inter_node_partition, - intra_node_partition): + def _generate_process_mesh_list( + self, inter_node_partition, intra_node_partition + ): process_mesh_list = [] start_row = 0 start_col = 0 @@ -200,8 +248,9 @@ def _generate_process_mesh_list(self, inter_node_partition, for n in intra_node_partition: process_mesh = [] for p in range(m): - start = (start_row + - p) * self._num_devices_per_machine + start_col + start = ( + start_row + p + ) * self._num_devices_per_machine + start_col tmp = [] for q in range(n): tmp.append(start + q) @@ -211,8 +260,9 @@ def _generate_process_mesh_list(self, inter_node_partition, start_row += m return process_mesh_list - def _generate_dims_mapping_candidates_helper(self, dims_mapping, dims_list, - start, visited, candidates): + def _generate_dims_mapping_candidates_helper( + self, dims_mapping, dims_list, start, visited, candidates + ): if start == len(dims_mapping) or all(visited): candidates.append(copy.deepcopy(dims_mapping)) return @@ -222,15 +272,17 @@ def _generate_dims_mapping_candidates_helper(self, dims_mapping, dims_list, dims_mapping[start] = dim visited[idx] = True self._generate_dims_mapping_candidates_helper( - dims_mapping, dims_list, start + 1, visited, candidates) + dims_mapping, dims_list, start + 1, visited, candidates + ) visited[idx] = False dims_mapping[start] = -1 - self._generate_dims_mapping_candidates_helper(dims_mapping, dims_list, - start + 1, visited, - candidates) + self._generate_dims_mapping_candidates_helper( + dims_mapping, dims_list, start + 1, visited, candidates + ) - def _generate_dims_mapping_candidates(self, dims_mapping_len, - process_mesh_len): + def _generate_dims_mapping_candidates( + self, dims_mapping_len, process_mesh_len + ): assert dims_mapping_len >= 1 and process_mesh_len >= 1 key = (dims_mapping_len, process_mesh_len) if key in self._cached_dims_mapping_candidates: @@ -239,8 +291,9 @@ def _generate_dims_mapping_candidates(self, dims_mapping_len, dims_mapping = [-1 for i in range(dims_mapping_len)] dims_list = [i for i in range(process_mesh_len)] visited = [False for i in range(process_mesh_len)] - self._generate_dims_mapping_candidates_helper(dims_mapping, dims_list, - 0, visited, candidates) + self._generate_dims_mapping_candidates_helper( + dims_mapping, dims_list, 0, visited, candidates + ) self._cached_dims_mapping_candidates[key] = candidates return candidates @@ -257,12 +310,14 @@ def _generate_dist_attr_candidates(self, op_id, dist_op): key.append(input_name) for input_arg_name in serial_op.input(input_name): key.append( - len(op_dist_attr.get_input_dims_mapping(input_arg_name))) + len(op_dist_attr.get_input_dims_mapping(input_arg_name)) + ) for output_name in serial_op.output_names: key.append(output_name) for output_arg_name in serial_op.output(output_name): key.append( - len(op_dist_attr.get_output_dims_mapping(output_arg_name))) + len(op_dist_attr.get_output_dims_mapping(output_arg_name)) + ) key = tuple(key) if key in self._cached_candidates_info: @@ -274,18 +329,26 @@ def _generate_dist_attr_candidates(self, op_id, dist_op): i = 0 for input_name in serial_op.input_names: for input_arg_name in serial_op.input(input_name): - cached_dims_mapping = cached_dist_attr.get_input_dims_mapping( - cached_input_arg_names[i]) + cached_dims_mapping = ( + cached_dist_attr.get_input_dims_mapping( + cached_input_arg_names[i] + ) + ) new_op_dist_attr.set_input_dims_mapping( - input_arg_name, cached_dims_mapping) + input_arg_name, cached_dims_mapping + ) i += 1 i = 0 for output_name in serial_op.output_names: for output_arg_name in serial_op.output(output_name): - cached_dims_mapping = cached_dist_attr.get_output_dims_mapping( - cached_output_arg_names[i]) + cached_dims_mapping = ( + cached_dist_attr.get_output_dims_mapping( + cached_output_arg_names[i] + ) + ) new_op_dist_attr.set_output_dims_mapping( - output_arg_name, cached_dims_mapping) + output_arg_name, cached_dims_mapping + ) i += 1 cached_dist_attr_candidates.append(new_op_dist_attr) return cached_dist_attr_candidates @@ -315,22 +378,28 @@ def _generate_dist_attr_candidates(self, op_id, dist_op): input_names.append(tensor_name) if dims_mapping_len < 1: dims_mapping_generated.append( - [copy.deepcopy(original_dims_mapping)]) + [copy.deepcopy(original_dims_mapping)] + ) else: dims_mapping_generated.append( self._generate_dims_mapping_candidates( - dims_mapping_len, process_mesh_len)) + dims_mapping_len, process_mesh_len + ) + ) input_dims_mapping_candidates = [] for dims_mapping_list in itertools.product(*dims_mapping_generated): dims_mapping_list = list(dims_mapping_list) assert len(dims_mapping_list) == len(input_names) for i, dims_mapping in enumerate(dims_mapping_list): - new_op_dist_attr.set_input_dims_mapping(input_names[i], - dims_mapping) - new_dist_op = DistributedOperator(dist_op.serial_op, - new_op_dist_attr) + new_op_dist_attr.set_input_dims_mapping( + input_names[i], dims_mapping + ) + new_dist_op = DistributedOperator( + dist_op.serial_op, new_op_dist_attr + ) dist_op_impls = find_compatible_distributed_operator_impls( - new_dist_op, fwd=True) + new_dist_op, fwd=True + ) if dist_op_impls is not None: input_dims_mapping_candidates.append(dims_mapping_list) @@ -344,42 +413,58 @@ def _generate_dist_attr_candidates(self, op_id, dist_op): output_names.append(tensor_name) if dims_mapping_len < 1: dims_mapping_generated.append( - [copy.deepcopy(original_dims_mapping)]) + [copy.deepcopy(original_dims_mapping)] + ) else: dims_mapping_generated.append( self._generate_dims_mapping_candidates( - dims_mapping_len, process_mesh_len)) + dims_mapping_len, process_mesh_len + ) + ) output_dims_mapping_candidates = [] for dims_mapping_list in itertools.product(*dims_mapping_generated): dims_mapping_list = list(dims_mapping_list) assert len(dims_mapping_list) == len(output_names) for i, dims_mapping in enumerate(dims_mapping_list): new_op_dist_attr.set_output_dims_mapping( - output_names[i], dims_mapping) - new_dist_op = DistributedOperator(dist_op.serial_op, - new_op_dist_attr) + output_names[i], dims_mapping + ) + new_dist_op = DistributedOperator( + dist_op.serial_op, new_op_dist_attr + ) dist_op_impls = find_compatible_distributed_operator_impls( - new_dist_op, fwd=False) + new_dist_op, fwd=False + ) if dist_op_impls is not None: output_dims_mapping_candidates.append(dims_mapping_list) if not input_dims_mapping_candidates and output_dims_mapping_candidates: - inout_dims_mapping_generated = [[[[-2]]], - output_dims_mapping_candidates] - elif input_dims_mapping_candidates and not output_dims_mapping_candidates: inout_dims_mapping_generated = [ - input_dims_mapping_candidates, [[[-2]]] + [[[-2]]], + output_dims_mapping_candidates, ] - elif not input_dims_mapping_candidates and not output_dims_mapping_candidates: + elif ( + input_dims_mapping_candidates and not output_dims_mapping_candidates + ): + inout_dims_mapping_generated = [ + input_dims_mapping_candidates, + [[[-2]]], + ] + elif ( + not input_dims_mapping_candidates + and not output_dims_mapping_candidates + ): inout_dims_mapping_generated = [[[[-2]]], [[[-2]]]] else: inout_dims_mapping_generated = [ - input_dims_mapping_candidates, output_dims_mapping_candidates + input_dims_mapping_candidates, + output_dims_mapping_candidates, ] # Find valid dims_mapping generated for both inputs and outputs cached_dist_attr_candidates = [] for inout_dims_mapping_list in itertools.product( - *inout_dims_mapping_generated): + *inout_dims_mapping_generated + ): assert len(inout_dims_mapping_list) == 2 if input_dims_mapping_candidates: assert len(inout_dims_mapping_list[0]) == len(input_names) @@ -389,35 +474,45 @@ def _generate_dist_attr_candidates(self, op_id, dist_op): for i, dims_mapping in enumerate(inout_dims_mapping_list[0]): if dims_mapping != [-2]: new_op_dist_attr.set_input_dims_mapping( - input_names[i], dims_mapping) + input_names[i], dims_mapping + ) # set the dims_mappings for outputs for i, dims_mapping in enumerate(inout_dims_mapping_list[1]): if dims_mapping != [-2]: new_op_dist_attr.set_output_dims_mapping( - output_names[i], dims_mapping) - new_dist_op = DistributedOperator(dist_op.serial_op, - new_op_dist_attr) + output_names[i], dims_mapping + ) + new_dist_op = DistributedOperator( + dist_op.serial_op, new_op_dist_attr + ) dist_op_impls = find_compatible_distributed_operator_impls( - new_dist_op, partial=False) + new_dist_op, partial=False + ) if dist_op_impls is None: continue for dist_op_impl in dist_op_impls: new_op_dist_attr.impl_type = dist_op_impl.type new_op_dist_attr.impl_idx = dist_op_impl.idx cached_dist_attr_candidates.append( - copy.deepcopy(new_op_dist_attr)) + copy.deepcopy(new_op_dist_attr) + ) self._cached_candidates_info[key].append(cached_dist_attr_candidates) return self._cached_candidates_info[key][2] def construct_space(self): inter_node_partitions, intra_node_partitions = self._partition_devices( - self._num_machines, self._num_devices_per_machine) - self._space.choice("inter_node_partitions", - inter_node_partitions, - default=inter_node_partitions[0]) - self._space.choice("intra_node_partitions", - intra_node_partitions, - default=intra_node_partitions[0]) + self._num_machines, self._num_devices_per_machine + ) + self._space.choice( + "inter_node_partitions", + inter_node_partitions, + default=inter_node_partitions[0], + ) + self._space.choice( + "intra_node_partitions", + intra_node_partitions, + default=intra_node_partitions[0], + ) dist_ops = self._dist_context._dist_ops_for_program for op_id, dist_op in dist_ops.items(): @@ -433,17 +528,22 @@ def construct_space(self): if op_type in self._exclude_op_types: del self._concerned_dist_ops[op_id] - print("Number of the concered dist ops", - len(self._concerned_dist_ops), - flush=True) + print( + "Number of the concered dist ops", + len(self._concerned_dist_ops), + flush=True, + ) search_space = 1 for op_id, dist_op in self._concerned_dist_ops.items(): op_dist_attr_candidates = self._generate_dist_attr_candidates( - op_id, dist_op) + op_id, dist_op + ) search_space *= len(op_dist_attr_candidates) - self._space.choice(str(op_id), - op_dist_attr_candidates, - default=op_dist_attr_candidates[0]) + self._space.choice( + str(op_id), + op_dist_attr_candidates, + default=op_dist_attr_candidates[0], + ) def _compute_values_hash(self, values): keys = sorted(values.keys()) @@ -522,10 +622,10 @@ def _generate_pipeline_starts(self, process_mesh_list): for _ in pipeline_starts[1:-1]: directions.append(Boolean("direction")) sizes.append( - IntRange("size", - start=0, - stop=half_ops_per_stage, - endpoint=True)) + IntRange( + "size", start=0, stop=half_ops_per_stage, endpoint=True + ) + ) for i, start in enumerate(pipeline_starts[1:-1]): direction = directions[i].random(self._seed) size = sizes[i].random(self._seed) @@ -538,19 +638,23 @@ def _generate_pipeline_starts(self, process_mesh_list): # Don't change the last start new_pipeline_starts.append(pipeline_starts[-1]) # Validate the new starts - print("Adjusted pipeline starts", - new_pipeline_starts, - half_ops_per_stage, - pipeline_starts, - flush=True) + print( + "Adjusted pipeline starts", + new_pipeline_starts, + half_ops_per_stage, + pipeline_starts, + flush=True, + ) for i, new_start in enumerate(new_pipeline_starts[1:]): assert new_start > new_pipeline_starts[i] return new_pipeline_starts else: - print("Non-adjusted pipeline starts", - pipeline_starts, - half_ops_per_stage, - flush=True) + print( + "Non-adjusted pipeline starts", + pipeline_starts, + half_ops_per_stage, + flush=True, + ) return pipeline_starts def _apply_pipeline_partition(self, process_mesh_list): @@ -620,12 +724,12 @@ def _amend_dist_attr(self): # if dim_mapping != -1 \ # and (tensor_shape[i] % process_shape[dim_mapping] != 0 \ # or dynamic_dims[i] == 1): - if dim_mapping != -1 \ - and (tensor_shape[i] % process_shape[dim_mapping] != 0): + if dim_mapping != -1 and ( + tensor_shape[i] % process_shape[dim_mapping] != 0 + ): dims_mapping[i] = -1 # it is a fix-bug - if dim_mapping != -1 \ - and process_shape[dim_mapping] == 1: + if dim_mapping != -1 and process_shape[dim_mapping] == 1: dims_mapping[i] = -1 for arg_name in dist_attr.outputs_dist_attrs.keys(): @@ -651,20 +755,22 @@ def _amend_dist_attr(self): else: continue for i, dim_mapping in enumerate(dims_mapping): - if dim_mapping != -1 \ - and (tensor_shape[i] % process_shape[dim_mapping] != 0): + if dim_mapping != -1 and ( + tensor_shape[i] % process_shape[dim_mapping] != 0 + ): dims_mapping[i] = -1 # it is a fix-bug - if dim_mapping != -1 \ - and process_shape[dim_mapping] == 1: + if dim_mapping != -1 and process_shape[dim_mapping] == 1: dims_mapping[i] = -1 dist_op_impls = find_compatible_distributed_operator_impls( - dist_op, partial=False) + dist_op, partial=False + ) serial_op_type = dist_op.serial_op.type if dist_op_impls is not None and ( - serial_op_type != "fused_softmax_mask_upper_triangle" - or self._check_fused_softmax_mask_upper_triangle(dist_op)): + serial_op_type != "fused_softmax_mask_upper_triangle" + or self._check_fused_softmax_mask_upper_triangle(dist_op) + ): dist_op.dist_attr.impl_type = dist_op_impls[0].type dist_op.dist_attr.impl_idx = dist_op_impls[0].idx else: @@ -684,14 +790,20 @@ def _check_fused_softmax_mask_upper_triangle(self, dist_op): """The last_but_one dim shoule be equal to last dim.""" input_name = dist_op.serial_op.input_arg_names[0] input_dims_mapping = dist_op.dist_attr.get_input_dims_mapping( - input_name) + input_name + ) topology = dist_op.dist_attr.process_mesh.topology input_tensor = dist_op.get_serial_input(input_name) - last_but_one_dim = input_tensor.shape[-2] // topology[ - input_dims_mapping[-2]] if input_dims_mapping[ - -2] != -1 else input_tensor.shape[-2] - last_dim = input_tensor.shape[-1] // topology[input_dims_mapping[ - -1]] if input_dims_mapping[-1] != -1 else input_tensor.shape[-1] + last_but_one_dim = ( + input_tensor.shape[-2] // topology[input_dims_mapping[-2]] + if input_dims_mapping[-2] != -1 + else input_tensor.shape[-2] + ) + last_dim = ( + input_tensor.shape[-1] // topology[input_dims_mapping[-1]] + if input_dims_mapping[-1] != -1 + else input_tensor.shape[-1] + ) if last_but_one_dim == last_dim: return True return False @@ -709,30 +821,37 @@ def _eval_trial(self, trial): inter_node_partition = trial.space.values["inter_node_partitions"] intra_node_partition = trial.space.values["intra_node_partitions"] process_mesh_list = self._generate_process_mesh_list( - inter_node_partition, intra_node_partition) + inter_node_partition, intra_node_partition + ) print("\tprocess_mesh list", process_mesh_list, flush=True) op_id_to_process_mesh = self._apply_pipeline_partition( - process_mesh_list) + process_mesh_list + ) if op_id_to_process_mesh is None: print("Operators are less than pipeline stages", flush=True) return results op_id_to_dist_attr = {} for name, value in trial.space.values.items(): - if name != "inter_node_partitions" \ - and name !="intra_node_partitions": + if ( + name != "inter_node_partitions" + and name != "intra_node_partitions" + ): op_id_to_dist_attr[int(name)] = value end_time = time.time() cur_sample_time = end_time - start_time - self._sample_time = (num_prev_trials * self._sample_time + - cur_sample_time) / self._num_trials - print("\tsample_time", - num_prev_trials, - self._num_trials, - self._sample_time, - cur_sample_time, - flush=True) + self._sample_time = ( + num_prev_trials * self._sample_time + cur_sample_time + ) / self._num_trials + print( + "\tsample_time", + num_prev_trials, + self._num_trials, + self._sample_time, + cur_sample_time, + flush=True, + ) assert len(op_id_to_process_mesh) == len(op_id_to_dist_attr) @@ -740,42 +859,52 @@ def _eval_trial(self, trial): for op_id, process_mesh in op_id_to_process_mesh.items(): dist_op = self._dist_context._dist_ops_for_program[op_id] dist_op.dist_attr = copy.deepcopy(op_id_to_dist_attr[op_id]) - assert dist_op.dist_attr.impl_type == op_id_to_dist_attr[ - op_id].impl_type - assert dist_op.dist_attr.impl_idx == op_id_to_dist_attr[ - op_id].impl_idx + assert ( + dist_op.dist_attr.impl_type + == op_id_to_dist_attr[op_id].impl_type + ) + assert ( + dist_op.dist_attr.impl_idx == op_id_to_dist_attr[op_id].impl_idx + ) dist_op.dist_attr.process_mesh = process_mesh self._amend_dist_attr() self._completer._complete_tensor_dist_attr_by_op() self._dist_context.block_state.parse_forward_blocks( - self._dist_context.serial_main_program) + self._dist_context.serial_main_program + ) end_time = time.time() cur_complete_time = end_time - start_time - self._complete_time = (num_prev_trials * self._complete_time + - cur_complete_time) / self._num_trials - print("\tcomplete_time", - num_prev_trials, - self._num_trials, - self._complete_time, - cur_complete_time, - flush=True) + self._complete_time = ( + num_prev_trials * self._complete_time + cur_complete_time + ) / self._num_trials + print( + "\tcomplete_time", + num_prev_trials, + self._num_trials, + self._complete_time, + cur_complete_time, + flush=True, + ) start_time = time.time() estimate_time = self._estimate_trial() end_time = time.time() cur_estimate_time = end_time - start_time - self._estimate_time = (num_prev_trials * self._estimate_time + - cur_estimate_time) / self._num_trials - print("\testimate_time", - num_prev_trials, - self._num_trials, - self._estimate_time, - cur_estimate_time, - estimate_time, - flush=True) + self._estimate_time = ( + num_prev_trials * self._estimate_time + cur_estimate_time + ) / self._num_trials + print( + "\testimate_time", + num_prev_trials, + self._num_trials, + self._estimate_time, + cur_estimate_time, + estimate_time, + flush=True, + ) results = {"estimate_time": estimate_time} return results @@ -791,12 +920,14 @@ def _estimate_trial(self): self._estimator = CostEstimator( self._dist_context.serial_main_program, self._cluster, - loop_count=self._loop_count) + loop_count=self._loop_count, + ) elif self._mode == "predict": self._estimator = CostEstimator( self._dist_context.serial_main_program, self._cluster, - loop_count=self._loop_count) + loop_count=self._loop_count, + ) elif self._mode == "train": # get serial main program with backward serial_main_program = self._dist_context.serial_main_program @@ -806,18 +937,23 @@ def _estimate_trial(self): # Generate backward serial_loss = self._dist_context.serial_fetch_vars["loss"][0] params_grads = self._parallelizer._generate_backward( - serial_main_program, serial_startup_program, serial_loss) + serial_main_program, serial_startup_program, serial_loss + ) # Generate optimizer optimizer_ops = self._parallelizer._generate_optimizer( - serial_main_program, serial_startup_program, serial_optimizer, - params_grads) - self._estimator = CostEstimator(serial_main_program, - self._cluster, - loop_count=self._loop_count) + serial_main_program, + serial_startup_program, + serial_optimizer, + params_grads, + ) + self._estimator = CostEstimator( + serial_main_program, self._cluster, loop_count=self._loop_count + ) max_memory = self._estimator._estimate_max_memory_by_dist_op( - self._dist_context) + self._dist_context + ) print("\tmax_memory", "{:,}".format(max_memory), flush=True) # The max memory must be less than 80% 32GB (hard code) if max_memory > 32 * 0.8 * 1024 * 1024 * 1024: @@ -829,18 +965,23 @@ def _estimate_trial(self): def _store_init_parallel_strategy(self): # If there is no annotation information, use the dp as the initial parallel strategy. # TODO: we should need a better way to set up the initial parallel strategy. - if not self._dist_context.has_annotation \ - or not self._dist_context.process_meshes: + if ( + not self._dist_context.has_annotation + or not self._dist_context.process_meshes + ): ranks = self._num_machines * self._num_devices_per_machine tensor_node = self._dist_context._serial_ordered_tensor_nodes[0] tensor_node_id = _node_id(tensor_node) tensor = self._dist_context._dist_tensors_for_graph[ - tensor_node_id].serial_tensor + tensor_node_id + ].serial_tensor tensor_dist_attr = self._dist_context._dist_tensors_for_graph[ - tensor_node_id].dist_attr + tensor_node_id + ].dist_attr tensor_dist_attr.process_mesh = ProcessMesh(list(range(ranks))) self._dist_context._process_meshes.append( - tensor_dist_attr.process_mesh) + tensor_dist_attr.process_mesh + ) tensor_dist_attr.dims_mapping = [0] + [ -1 for _ in range(len(tensor.shape) - 1) ] @@ -851,23 +992,30 @@ def _store_init_parallel_strategy(self): # Do the sharding propagation self._completer.complete_forward_annotation() self._dist_context.block_state.parse_forward_blocks( - self._dist_context.serial_main_program) + self._dist_context.serial_main_program + ) # Backup the intital parallel strategy self._init_parallel_strategy[0] = copy.deepcopy( - self._dist_context._dist_tensors_for_program) + self._dist_context._dist_tensors_for_program + ) self._init_parallel_strategy[1] = copy.deepcopy( - self._dist_context._dist_ops_for_program) + self._dist_context._dist_ops_for_program + ) self._init_parallel_strategy[2] = copy.deepcopy( - self._dist_context.process_meshes) + self._dist_context.process_meshes + ) # Initialize the best parallel strategy to the initial one self._best_parallel_strategy[0] = copy.deepcopy( - self._dist_context._dist_tensors_for_program) + self._dist_context._dist_tensors_for_program + ) self._best_parallel_strategy[1] = copy.deepcopy( - self._dist_context._dist_ops_for_program) + self._dist_context._dist_ops_for_program + ) self._best_parallel_strategy[2] = copy.deepcopy( - self._dist_context._process_meshes) + self._dist_context._process_meshes + ) def _store_best_parallel_strategy(self): # Swap the best and the current parallel strategy @@ -876,9 +1024,11 @@ def _store_best_parallel_strategy(self): tmp[1] = self._best_parallel_strategy[1] tmp[2] = self._best_parallel_strategy[2] self._best_parallel_strategy[ - 0] = self._dist_context._dist_tensors_for_program + 0 + ] = self._dist_context._dist_tensors_for_program self._best_parallel_strategy[ - 1] = self._dist_context._dist_ops_for_program + 1 + ] = self._dist_context._dist_ops_for_program self._best_parallel_strategy[2] = self._dist_context._process_meshes self._dist_context._dist_tensors_for_program = tmp[0] self._dist_context._dist_ops_for_program = tmp[1] @@ -894,19 +1044,23 @@ def tune(self): # We have to restore the distributed context, because the estimation of one trail need to # generate the backward and update parts. Since we will do the tuning process, # here we only need to reset all distributed information to the default one. - self._dist_context._restore(serial=True, - serial_mode="to_backup", - dist=True, - dist_mode="to_default") + self._dist_context._restore( + serial=True, + serial_mode="to_backup", + dist=True, + dist_mode="to_default", + ) best_time = init_time start_time = time.time() self.construct_space() end_time = time.time() - print("construct_space time", - self._num_trials, - end_time - start_time, - flush=True) + print( + "construct_space time", + self._num_trials, + end_time - start_time, + flush=True, + ) create_trial_time = 0.0 eval_trial_time = 0.0 self._sample_time = 0.0 @@ -921,14 +1075,17 @@ def tune(self): num_prev_trials = self._num_trials - 1 end_time = time.time() cur_create_trial_time = end_time - start_time - create_trial_time = (num_prev_trials * create_trial_time + - cur_create_trial_time) / self._num_trials - print("create_trial time", - num_prev_trials, - self._num_trials, - create_trial_time, - cur_create_trial_time, - flush=True) + create_trial_time = ( + num_prev_trials * create_trial_time + cur_create_trial_time + ) / self._num_trials + print( + "create_trial time", + num_prev_trials, + self._num_trials, + create_trial_time, + cur_create_trial_time, + flush=True, + ) if trial.status == TrialStatus.STOPPED: break # We need to backup the distributed context, because the evaluation of one trail will @@ -940,15 +1097,18 @@ def tune(self): results = self._eval_trial(trial) end_time = time.time() cur_eval_trial_time = end_time - start_time - eval_trial_time = (num_prev_trials * eval_trial_time + - cur_eval_trial_time) / self._num_trials - print("eval_trial time", - num_prev_trials, - self._num_trials, - eval_trial_time, - cur_eval_trial_time, - "\n", - flush=True) + eval_trial_time = ( + num_prev_trials * eval_trial_time + cur_eval_trial_time + ) / self._num_trials + print( + "eval_trial time", + num_prev_trials, + self._num_trials, + eval_trial_time, + cur_eval_trial_time, + "\n", + flush=True, + ) cur_time = results["estimate_time"] if cur_time < best_time: @@ -956,13 +1116,17 @@ def tune(self): self._store_best_parallel_strategy() best_time = cur_time # We need to restore the distributed context and reset the distributed information to the default. - self._dist_context._restore(serial=True, - serial_mode="to_backup", - dist=True, - dist_mode="to_default") + self._dist_context._restore( + serial=True, + serial_mode="to_backup", + dist=True, + dist_mode="to_default", + ) # Select the best parallel strategy - self._dist_context._dist_tensors_for_program = self._best_parallel_strategy[ - 0] + self._dist_context._dist_tensors_for_program = ( + self._best_parallel_strategy[0] + ) self._dist_context._dist_ops_for_program = self._best_parallel_strategy[ - 1] + 1 + ] self._dist_context._process_meshes = self._best_parallel_strategy[2] diff --git a/python/paddle/distributed/auto_parallel/tuner/profiler.py b/python/paddle/distributed/auto_parallel/tuner/profiler.py index 4b2655028bf7f..1aeafbea76410 100644 --- a/python/paddle/distributed/auto_parallel/tuner/profiler.py +++ b/python/paddle/distributed/auto_parallel/tuner/profiler.py @@ -22,8 +22,13 @@ import paddle from paddle.fluid.framework import Program, _current_expected_place from paddle.fluid.framework import Operator -from paddle.distributed.auto_parallel.process_group import get_all_process_groups, new_process_group -from paddle.distributed.auto_parallel.dist_loader import DistributedDataLoaderFromGenerator +from paddle.distributed.auto_parallel.process_group import ( + get_all_process_groups, + new_process_group, +) +from paddle.distributed.auto_parallel.dist_loader import ( + DistributedDataLoaderFromGenerator, +) from paddle.distributed.collective import _get_global_env paddle.enable_static() @@ -44,25 +49,32 @@ def parse_args(): "--profile_start_step", default=10, type=int, - help="integer indicates the warmup step before starting profile.") - parser.add_argument("--profile_end_step", - default=30, - type=int, - help="integer indicates at the end step of profile.") - parser.add_argument("--rank", - type=int, - required=True, - help="the rank id of the this process.") - parser.add_argument("--device_id", - type=int, - required=True, - help="the device id of the this process.") + help="integer indicates the warmup step before starting profile.", + ) + parser.add_argument( + "--profile_end_step", + default=30, + type=int, + help="integer indicates at the end step of profile.", + ) + parser.add_argument( + "--rank", + type=int, + required=True, + help="the rank id of the this process.", + ) + parser.add_argument( + "--device_id", + type=int, + required=True, + help="the device id of the this process.", + ) parser.add_argument( "--ctx_filename", type=str, required=True, - help= - "the filename to the profile context file saved by optimizaiton tuner") + help="the filename to the profile context file saved by optimizaiton tuner", + ) args = parser.parse_args() @@ -109,11 +121,9 @@ def get_cpp_error_type(error): return error_type -def create_dataloader(main_program, - startup_program, - profile_ctx, - epochs=1, - steps_per_epoch=None): +def create_dataloader( + main_program, startup_program, profile_ctx, epochs=1, steps_per_epoch=None +): dataset = profile_ctx["dataset"] main_block = main_program.global_block() @@ -141,7 +151,8 @@ def create_dataloader(main_program, epochs=epochs, steps_per_epoch=steps_per_epoch, data_parallel_world_size=dataset.dp_world_size, - data_parallel_rank=dataset.dp_rank) + data_parallel_rank=dataset.dp_rank, + ) # move read op from the end of program to the start of program new_op_size = len(main_block.ops) @@ -162,8 +173,12 @@ def init_comm(profile_ctx): dist_env = profile_ctx['distributed_env'] genv = _get_global_env() genv = dist_env - print("current process rank: {}, device_id: {}, ip: {}.", genv.rank, - genv.device_id, genv.current_endpoint) + print( + "current process rank: {}, device_id: {}, ip: {}.", + genv.rank, + genv.device_id, + genv.current_endpoint, + ) # init nccl comm group_map = profile_ctx['group_map'] @@ -201,8 +216,9 @@ def profiler(args): """ # load ctx if not os.path.isfile(args.ctx_filename): - raise ValueError("There is no profile context named {}.".format( - args.ctx_filename)) + raise ValueError( + "There is no profile context named {}.".format(args.ctx_filename) + ) with open(args.ctx_filename, 'rb') as f: profile_ctx = pickle.load(f, encoding='latin1') @@ -240,8 +256,9 @@ def profiler(args): print("step: %d, loss_print: %f" % (eval_step, loss[0])) eval_step += 1 - avg_tput = 1.0 * (args.profile_end_step - - args.profile_start_step) / duration + avg_tput = ( + 1.0 * (args.profile_end_step - args.profile_start_step) / duration + ) result_dict = { "Throughtput": avg_tput, diff --git a/python/paddle/distributed/auto_parallel/tuner/recorder.py b/python/paddle/distributed/auto_parallel/tuner/recorder.py index 4d088379d8b3b..2c838cfb14921 100644 --- a/python/paddle/distributed/auto_parallel/tuner/recorder.py +++ b/python/paddle/distributed/auto_parallel/tuner/recorder.py @@ -71,7 +71,9 @@ def __init__(self, direction="min"): if direction not in {"min", "max"}: raise ValueError( "direction should be one of {{min, max}}, but got: {}.".format( - direction)) + direction + ) + ) self._direction = direction self._records = {} diff --git a/python/paddle/distributed/auto_parallel/tuner/storable.py b/python/paddle/distributed/auto_parallel/tuner/storable.py index 18a0669d62286..fb03070ad099a 100644 --- a/python/paddle/distributed/auto_parallel/tuner/storable.py +++ b/python/paddle/distributed/auto_parallel/tuner/storable.py @@ -19,7 +19,6 @@ class Storable(object): - def get_state(self): raise NotImplementedError diff --git a/python/paddle/distributed/auto_parallel/tuner/trial.py b/python/paddle/distributed/auto_parallel/tuner/trial.py index 2c8963322a3d6..b152e407f5f18 100644 --- a/python/paddle/distributed/auto_parallel/tuner/trial.py +++ b/python/paddle/distributed/auto_parallel/tuner/trial.py @@ -32,11 +32,9 @@ class TrialStatus: class Trial(Storable): - - def __init__(self, - tunable_space, - trial_id=None, - status=TrialStatus.RUNNING): + def __init__( + self, tunable_space, trial_id=None, status=TrialStatus.RUNNING + ): self._id = _generate_trial_id() if trial_id is None else trial_id self._space = tunable_space self._recorder = MetricsRecorder() @@ -115,13 +113,14 @@ def from_state(cls, state): class OptimizationTunerTrial(Trial): - - def __init__(self, - config, - name, - changed_configs, - trial_id=None, - status=TrialStatus.RUNNING): + def __init__( + self, + config, + name, + changed_configs, + trial_id=None, + status=TrialStatus.RUNNING, + ): super(OptimizationTunerTrial, self).__init__(config, trial_id, status) self._name = name self._changed_configs = changed_configs @@ -140,7 +139,8 @@ def summary(self): h1_format = " " + "|{{:^{}s}}|\n".format(length) h2_format = " " + "|{{:>{}s}}{}{{:^{}s}}|\n".format( - max_k, " " * spacing, max_v) + max_k, " " * spacing, max_v + ) border = " +" + "".join(["="] * length) + "+" line = " +" + "".join(["-"] * length) + "+" @@ -158,7 +158,8 @@ def summary(self): keys = my_configs.to_dict().keys() for key in keys: draws += h2_format.format( - key, str(my_configs.to_dict().get(key, None))) + key, str(my_configs.to_dict().get(key, None)) + ) result_res = draws + border return result_res diff --git a/python/paddle/distributed/auto_parallel/tuner/tunable_space.py b/python/paddle/distributed/auto_parallel/tuner/tunable_space.py index 2009f1d911cbf..e3e503401b453 100644 --- a/python/paddle/distributed/auto_parallel/tuner/tunable_space.py +++ b/python/paddle/distributed/auto_parallel/tuner/tunable_space.py @@ -104,29 +104,24 @@ def choice(self, name, values, default=None): return self._retrieve(tv) def int_range(self, name, start, stop, step=1, default=None): - tv = IntRange(name=name, - start=start, - stop=stop, - step=step, - default=default) + tv = IntRange( + name=name, start=start, stop=stop, step=step, default=default + ) return self._retrieve(tv) def float_range(self, name, start, stop, step=None, default=None): - tv = FloatRange(name=name, - start=start, - stop=stop, - step=step, - default=default) + tv = FloatRange( + name=name, start=start, stop=stop, step=step, default=default + ) return self._retrieve(tv) def get_state(self): return { - "variables": [{ - "class_name": v.__class__.__name__, - "state": v.get_state() - } for v in self._variables.values()], - "values": - dict((k, v) for (k, v) in self.values.items()) + "variables": [ + {"class_name": v.__class__.__name__, "state": v.get_state()} + for v in self._variables.values() + ], + "values": dict((k, v) for (k, v) in self.values.items()), } @classmethod @@ -146,11 +141,16 @@ def _deserialize_tunable_variable(state): if isinstance(state, classes): return state - if (not isinstance(state, dict) or "class_name" not in state - or "state" not in state): + if ( + not isinstance(state, dict) + or "class_name" not in state + or "state" not in state + ): raise ValueError( - "Expect state to be a python dict containing class_name and state as keys, but found {}" - .format(state)) + "Expect state to be a python dict containing class_name and state as keys, but found {}".format( + state + ) + ) cls_name = state["class_name"] cls = cls_name_to_cls[cls_name] diff --git a/python/paddle/distributed/auto_parallel/tuner/tunable_variable.py b/python/paddle/distributed/auto_parallel/tuner/tunable_variable.py index 31dd07aad374c..7b09f3a3b9a27 100644 --- a/python/paddle/distributed/auto_parallel/tuner/tunable_variable.py +++ b/python/paddle/distributed/auto_parallel/tuner/tunable_variable.py @@ -50,7 +50,9 @@ def __init__(self, name, default): if not isinstance(default, (str, int, float, bool)): raise ValueError( "Fixed must be an str, int, float or bool, but found {}".format( - default)) + default + ) + ) self._default = default def random(self, seed=None): @@ -69,7 +71,8 @@ def __init__(self, name, default=False): super(Boolean, self).__init__(name=name, default=default) if default not in {True, False}: raise ValueError( - "default must be a Python boolean, but got {}".format(default)) + "default must be a Python boolean, but got {}".format(default) + ) def random(self, seed=None): rng = np.random.default_rng(seed) @@ -77,19 +80,21 @@ def random(self, seed=None): def __repr__(self): return 'Boolean(name: "{}", default: {})'.format( - self.name, self.default) + self.name, self.default + ) class Choice(TunableVariable): - def __init__(self, name, values, default=None): super(Choice, self).__init__(name=name, default=default) types = set(type(v) for v in values) if len(types) > 1: raise TypeError( - "Choice can contain only one type of value, but found values: {} with types: {}." - .format(str(values), str(types))) + "Choice can contain only one type of value, but found values: {} with types: {}.".format( + str(values), str(types) + ) + ) self._is_unknown_type = False if isinstance(values[0], str): @@ -115,8 +120,10 @@ def __init__(self, name, values, default=None): if default is not None and default not in values: raise ValueError( - "The default value should be one of the choices {}, but found {}" - .format(values, default)) + "The default value should be one of the choices {}, but found {}".format( + values, default + ) + ) self._default = default @property @@ -142,7 +149,8 @@ def get_state(self): def __repr__(self): return 'Choice(name: "{}", values: {}, default: {})'.format( - self.name, self.values, self.default) + self.name, self.values, self.default + ) class IntRange(TunableVariable): @@ -187,13 +195,15 @@ def get_state(self): def _check_int(self, val): int_val = int(val) if int_val != val: - raise ValueError("Expects val is an int, but found: {}.".format( - str(val))) + raise ValueError( + "Expects val is an int, but found: {}.".format(str(val)) + ) return int_val def __repr__(self): return "IntRange(name: {}, start: {}, stop: {}, step: {}, default: {})".format( - self.name, self.start, self.stop, self.step, self.default) + self.name, self.start, self.stop, self.step, self.default + ) class FloatRange(TunableVariable): @@ -201,13 +211,9 @@ class FloatRange(TunableVariable): Float range. """ - def __init__(self, - name, - start, - stop, - step=None, - default=None, - endpoint=False): + def __init__( + self, name, start, stop, step=None, default=None, endpoint=False + ): super(FloatRange, self).__init__(name=name, default=default) self.stop = float(stop) self.start = float(start) @@ -246,5 +252,10 @@ def get_state(self): def __repr__(self): return "FloatRange(name: {}, start: {}, stop: {}, step: {}, default: {}, endpoint: {})".format( - self.name, self.start, self.stop, self.step, self.default, - self.endpoint) + self.name, + self.start, + self.stop, + self.step, + self.default, + self.endpoint, + ) diff --git a/python/paddle/distributed/auto_parallel/utils.py b/python/paddle/distributed/auto_parallel/utils.py index cf6f506f8c563..8b7ec647f6e76 100644 --- a/python/paddle/distributed/auto_parallel/utils.py +++ b/python/paddle/distributed/auto_parallel/utils.py @@ -23,12 +23,18 @@ import paddle.fluid.core as core from paddle.distributed.fleet.meta_optimizers.common import OpRole -from paddle.distributed.auto_parallel.process_group import get_all_process_groups +from paddle.distributed.auto_parallel.process_group import ( + get_all_process_groups, +) from paddle.fluid.io import is_parameter, is_belong_to_optimizer -from paddle.distributed.auto_parallel.dist_attribute import TensorDistributedAttribute, OperatorDistributedAttribute +from paddle.distributed.auto_parallel.dist_attribute import ( + TensorDistributedAttribute, + OperatorDistributedAttribute, +) __not_shape_var_type__ = [ - core.VarDesc.VarType.READER, core.VarDesc.VarType.STEP_SCOPES + core.VarDesc.VarType.READER, + core.VarDesc.VarType.STEP_SCOPES, ] @@ -39,7 +45,8 @@ def get_logger(log_level, name="auto_parallel"): logger.setLevel(log_level) log_handler = logging.StreamHandler() log_format = logging.Formatter( - '%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s') + '%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s' + ) log_handler.setFormatter(log_format) logger.addHandler(log_handler) return logger @@ -114,8 +121,11 @@ def verify_shard_spec(shard_spec, tensor_shape, process_mesh): if not verify_dims_mapping(dims_mapping, process_mesh): return False for i in range(len(tensor_shape)): - if dims_mapping[i] != -1 and tensor_shape[i] > 0 \ - and tensor_shape[i] % process_mesh.shape[dims_mapping[i]] != 0: + if ( + dims_mapping[i] != -1 + and tensor_shape[i] > 0 + and tensor_shape[i] % process_mesh.shape[dims_mapping[i]] != 0 + ): return False return True @@ -141,14 +151,17 @@ def compute_compatible_dims_mapping(dims_mapping_list): return None length = len(dims_mapping_list[0]) for dims_mapping in dims_mapping_list: - assert dims_mapping is not None, \ - "Dims mapping must not be None for compatible computation" - assert len(dims_mapping) == length, \ - "The length of dims_mapping in list must be same for compatible computation." + assert ( + dims_mapping is not None + ), "Dims mapping must not be None for compatible computation" + assert ( + len(dims_mapping) == length + ), "The length of dims_mapping in list must be same for compatible computation." compatible_result = [] for dim_mappings in zip(*dims_mapping_list): compatible_dim_mapping = compute_compatible_dim_mapping( - list(dim_mappings)) + list(dim_mappings) + ) if compatible_dim_mapping is None: return None compatible_result.append(compatible_dim_mapping) @@ -161,7 +174,10 @@ def compute_compatible_process_mesh(process_mesh_list): return compatible_process_mesh for process_mesh in process_mesh_list: if process_mesh is not None: - if compatible_process_mesh is None or compatible_process_mesh == process_mesh: + if ( + compatible_process_mesh is None + or compatible_process_mesh == process_mesh + ): compatible_process_mesh = process_mesh else: return None @@ -201,15 +217,18 @@ def remove_distributed_attr_suffix(name): def check_distributed_attr_for_program(program, dist_context=None): from .dist_context import get_default_distributed_context + if dist_context is None: dist_context = get_default_distributed_context() - assert dist_context.is_initialized_for_program(), \ - "Distributed attributes must be initialized before check." + assert ( + dist_context.is_initialized_for_program() + ), "Distributed attributes must be initialized before check." for block in program.blocks: for tensor in block.vars.values(): dist_tensor = dist_context.get_dist_tensor_for_graph(tensor) tensor_dist_attr = dist_context.get_tensor_dist_attr_for_program( - tensor) + tensor + ) if (tensor_dist_attr is not None) and (not dist_tensor.is_valid()): return False for op in block.ops: @@ -229,6 +248,7 @@ def print_program_with_dist_attr(program, dist_context=None): lock.acquire() from .dist_context import get_default_distributed_context from .dist_context import set_default_distributed_context + if dist_context is None: dist_context = get_default_distributed_context() print(program, flush=True) @@ -256,7 +276,8 @@ def _get_comm_group(processes, shape, axis, rank): # NOTE _linear_idx2coordinate assume processes mesh start with 0 and continuous # tricks to support processes mesh when it is not start with 0 or continuous assert rank in processes, "rank [{}] is NOT in processes group {}".format( - rank, processes) + rank, processes + ) rank_relatvie = processes.index(rank) coordinate = _linear_idx2coordinate(shape, rank_relatvie) coordinates_in_group = [coordinate[:] for i in range(shape[axis])] @@ -325,14 +346,19 @@ def _coordinate2linear_idx(mesh_shape, coordinate): assert len(mesh_shape) == len( coordinate ), "coordinate should have the same size as mesh shape, but got shape: {}, coordinate: {}".format( - mesh_shape, coordinate) + mesh_shape, coordinate + ) for i in range(len(mesh_shape)): - assert coordinate[ - i] >= 0, "index in dimension [{}] is least than zero. coordinate: {}".format( - i, coordinate) - assert coordinate[i] < mesh_shape[ - i], "index beyond extent in dimension [{}]. shape: {}, coordinate: {}".format( - i, mesh_shape, coordinate) + assert ( + coordinate[i] >= 0 + ), "index in dimension [{}] is least than zero. coordinate: {}".format( + i, coordinate + ) + assert ( + coordinate[i] < mesh_shape[i] + ), "index beyond extent in dimension [{}]. shape: {}, coordinate: {}".format( + i, mesh_shape, coordinate + ) base = mesh_shape[-1] linear_idx = coordinate[-1] @@ -365,11 +391,13 @@ def _linear_idx2coordinate(mesh_shape, linear_idx): """ assert linear_idx >= 0, "linear index [{}] is least than zero".format( - linear_idx) + linear_idx + ) assert linear_idx < np.prod( mesh_shape ), "linear index beyond the extent of mesh shape. shape: {}, linear index: {}".format( - mesh_shape, linear_idx) + mesh_shape, linear_idx + ) base = 1 coordinate = [-1] * len(mesh_shape) @@ -392,15 +420,17 @@ def _get_corresponding_rank(dist_context, target_mesh, rank): coordinate = None for mesh in dist_context.process_meshes: if rank in mesh.processes and mesh.topology == target_mesh.topology: - coordinate = _linear_idx2coordinate(mesh.topology, - mesh.processes.index(rank)) + coordinate = _linear_idx2coordinate( + mesh.topology, mesh.processes.index(rank) + ) break # assert coordinate is not None, "could NOT found rank [{}] in any registered mesh".format( # rank) if coordinate is not None: - return target_mesh.processes[_coordinate2linear_idx( - mesh.topology, coordinate)] + return target_mesh.processes[ + _coordinate2linear_idx(mesh.topology, coordinate) + ] else: return target_mesh.processes[0] @@ -412,7 +442,8 @@ def _get_unshard_dist_shape(var, dist_attr): assert len(var_shape) == len( mapping ), "variable shape [{}] and dim_mapping [{}] is NOT match !".format( - var_shape, mapping) + var_shape, mapping + ) new_shape = [] for idx in range(len(var_shape)): if var_shape[idx] == -1 or mapping[idx] == -1: @@ -425,13 +456,15 @@ def _get_unshard_dist_shape(var, dist_attr): def make_data_unshard(dist_main_prog, dist_startup_prog, dist_context=None): from .dist_context import get_default_distributed_context + if dist_context is None: dist_context = get_default_distributed_context() for var in dist_main_prog.list_vars(): if var.is_data: tensor_dist_attr = dist_context.get_tensor_dist_attr_for_program( - var) + var + ) inverse_shape = _get_unshard_dist_shape(var, tensor_dist_attr) var.desc.set_shape(inverse_shape) dim_mapping = tensor_dist_attr.dims_mapping @@ -441,62 +474,76 @@ def make_data_unshard(dist_main_prog, dist_startup_prog, dist_context=None): def _update_addition_info(addition_info): - """ Update default addition_info with inputs """ + """Update default addition_info with inputs""" add_info = {"epoch": 0, "batch": 0, "batch_size": 0} if not addition_info: return add_info elif not isinstance(addition_info, dict): - raise TypeError("The type of 'addition_info' should be 'dict', " - "but got '{}'.".format(str(type(addition_info)))) + raise TypeError( + "The type of 'addition_info' should be 'dict', " + "but got '{}'.".format(str(type(addition_info))) + ) else: for item, value in addition_info.items(): if item not in ["epoch", "batch", "batch_size"]: raise ValueError( "The key of 'addition_info' should be one of the " "['epoch', 'batch', 'batch_size'], but got '{}'.".format( - str(item))) + str(item) + ) + ) if not isinstance(value, int): raise ValueError( "The value of 'addition_info' should be 'int', " - "but got '{}'.".format(str(type(value)))) + "but got '{}'.".format(str(type(value))) + ) add_info[item] = value return add_info def _check_valid_path(file_path): - """ Validity check of input file path """ + """Validity check of input file path""" if not file_path: return file_path elif isinstance(file_path, list): for file in file_path: if not isinstance(file, str): - raise TypeError("The type of file path should be 'str', " - "but got '{}'.".format(str(type(file)))) + raise TypeError( + "The type of file path should be 'str', " + "but got '{}'.".format(str(type(file))) + ) if not os.path.exists(file): raise ValueError( - "The file path '{}' does not exist.".format(file)) + "The file path '{}' does not exist.".format(file) + ) return file_path else: - raise TypeError("The type of file path should be 'list', " - "but got '{}'.".format(str(type(file_path)))) + raise TypeError( + "The type of file path should be 'list', " + "but got '{}'.".format(str(type(file_path))) + ) def _check_param_dict(param_dict): if not param_dict: raise ValueError("'param_dict' cannot be None.") elif not isinstance(param_dict, dict): - raise TypeError("The type of 'param_dict' should be 'dict', " - "but got '{}'.".format(str(type(param_dict)))) + raise TypeError( + "The type of 'param_dict' should be 'dict', " + "but got '{}'.".format(str(type(param_dict))) + ) else: for name, value in param_dict.items(): if not isinstance(name, str): raise TypeError( "The type of key of 'param_dict' should be 'str', " - "but got '{}'.".format(str(type(name)))) + "but got '{}'.".format(str(type(name))) + ) if not isinstance(value, paddle.fluid.LoDTensor): raise TypeError( "The type of value of 'param_dict' should be 'LoDTensor', " - "but got '{}'.".format(str(type(value)))) + "but got '{}'.".format(str(type(value))) + ) return param_dict @@ -504,33 +551,40 @@ def _check_dist_attr(dist_attr): if not dist_attr: return dist_attr elif not isinstance(dist_attr, dict): - raise TypeError("The type of 'dist_attr' should be 'dict', " - "but got '{}'.".format(str(type(dist_attr)))) + raise TypeError( + "The type of 'dist_attr' should be 'dict', " + "but got '{}'.".format(str(type(dist_attr))) + ) else: for name, value in dist_attr.items(): if not isinstance(name, str): raise TypeError( "The type of param name of 'dist_attr' should be 'str', " - "but got '{}'.".format(str(type(name)))) + "but got '{}'.".format(str(type(name))) + ) if not isinstance(value, dict): raise TypeError( "The type of distributed attribute should be 'dict', " - "but got '{}'".format(str(type(value)))) + "but got '{}'".format(str(type(value))) + ) attr = ['process_shape', 'process_group', 'dims_mapping'] if list(value.keys()) != attr: raise ValueError( "The key of distributed attribute should be " "'['process_shape', 'process_group', 'dims_mapping']', " - "but got {}.".format(str(value.keys()))) + "but got {}.".format(str(value.keys())) + ) return dist_attr -def save_distributed_checkpoint(program, - checkpoint_path, - dist_attr_path, - addition_info=None, - is_integrated=False, - dist_context=None): +def save_distributed_checkpoint( + program, + checkpoint_path, + dist_attr_path, + addition_info=None, + is_integrated=False, + dist_context=None, +): """ Save model parameter state, optimzer state, distributed attribute and additional information of each rank. @@ -569,7 +623,8 @@ def save_distributed_checkpoint(program, else: # TODO: integrate param before save raise NotImplementedError( - "Integrating parameter has not been implemented.") + "Integrating parameter has not been implemented." + ) def load_distributed_checkpoint(checkpoint_path, dist_attr_path): @@ -597,10 +652,10 @@ def load_distributed_checkpoint(checkpoint_path, dist_attr_path): './dist_attr_rank1.pdattr'] param_dict, dist_attr, add_info = load_distributed_checkpoint(ckpt_path, dist_attr_path) """ - assert _check_valid_path(checkpoint_path), \ - "'checkpoint_path' cannot be None." - assert _check_valid_path(dist_attr_path), \ - "'dist_attr_path' cannot be None." + assert _check_valid_path( + checkpoint_path + ), "'checkpoint_path' cannot be None." + assert _check_valid_path(dist_attr_path), "'dist_attr_path' cannot be None." state_dict_info = _load_distributed_state_dict(checkpoint_path) dist_attr = _load_distributed_attribute(dist_attr_path) @@ -609,10 +664,9 @@ def load_distributed_checkpoint(checkpoint_path, dist_attr_path): return param_dict, dist_attr, addition_info -def load_checkpoint_into_program(checkpoint_path, - dist_attr_path, - program, - dist_context=None): +def load_checkpoint_into_program( + checkpoint_path, dist_attr_path, program, dist_context=None +): """ Load parameter, optimizer, distributed attribute and addition_info into model. @@ -641,10 +695,10 @@ def load_checkpoint_into_program(checkpoint_path, from .dist_context import get_default_distributed_context assert isinstance(program, paddle.fluid.framework.Program) - assert _check_valid_path(checkpoint_path), \ - "'checkpoint_path' cannot be None." - assert _check_valid_path(dist_attr_path), \ - "'dist_attr_path' cannot be None." + assert _check_valid_path( + checkpoint_path + ), "'checkpoint_path' cannot be None." + assert _check_valid_path(dist_attr_path), "'dist_attr_path' cannot be None." if dist_context is None: dist_context = get_default_distributed_context() all_state_dict_info = _load_distributed_state_dict(checkpoint_path) @@ -652,9 +706,9 @@ def load_checkpoint_into_program(checkpoint_path, all_cur_dist_attr = get_dist_attr(program, dist_context) all_param_dict = all_state_dict_info["model"] addition_info = all_state_dict_info["addition_info"] - sliced_param_dict = merge_and_slice_parameter(all_param_dict, - all_pre_dist_attr, - all_cur_dist_attr) + sliced_param_dict = merge_and_slice_parameter( + all_param_dict, all_pre_dist_attr, all_cur_dist_attr + ) load_parameter_into_program(sliced_param_dict, program) return addition_info @@ -676,28 +730,31 @@ def load_parameter_into_program(param_dict, program): def _save_distributed_attribute(program, dist_attr_path, dist_context): - """ Save distributed attribute of all parameters """ + """Save distributed attribute of all parameters""" # TODO: just save a complete distributed attribute file rank_id = paddle.distributed.get_rank() - dist_attr_name = os.path.join(dist_attr_path, - "dist_attr_rank{}.pdattr".format(rank_id)) + dist_attr_name = os.path.join( + dist_attr_path, "dist_attr_rank{}.pdattr".format(rank_id) + ) dist_attr_dict = { "model": get_dist_attr(program, dist_context), - "world_size": paddle.distributed.get_world_size() + "world_size": paddle.distributed.get_world_size(), } paddle.save(dist_attr_dict, dist_attr_name) logging.info( - "Already saved distributed attribute to '{}'.".format(dist_attr_path)) + "Already saved distributed attribute to '{}'.".format(dist_attr_path) + ) def _load_distributed_attribute(dist_attr_path): - """ Load parameters' distributed attribute from dist_attr_path """ + """Load parameters' distributed attribute from dist_attr_path""" total_dist_attr = {} for dist_attr_file in dist_attr_path: dist_attr = paddle.load(dist_attr_file) pre_world_size = dist_attr["world_size"] - assert pre_world_size == len(dist_attr_path), \ - "The number of 'dist_attr_path' must be equal to the last training world size." + assert pre_world_size == len( + dist_attr_path + ), "The number of 'dist_attr_path' must be equal to the last training world size." for name, attr in dist_attr["model"].items(): if name not in total_dist_attr: total_dist_attr[name] = attr @@ -706,27 +763,29 @@ def _load_distributed_attribute(dist_attr_path): def _save_distributed_state_dict(program, addition_info, checkpoint_path): - """ Save parameters' state_dict """ + """Save parameters' state_dict""" rank = paddle.distributed.get_rank() - ckpt_file_name = os.path.join(checkpoint_path, - "model_state_rank{}.pdmodel".format(rank)) + ckpt_file_name = os.path.join( + checkpoint_path, "model_state_rank{}.pdmodel".format(rank) + ) state_dict = { "model": program.state_dict(), "world_size": paddle.distributed.get_world_size(), - "addition_info": addition_info + "addition_info": addition_info, } paddle.save(state_dict, ckpt_file_name) logging.info("Already saved model to '{}'.".format(checkpoint_path)) def _load_distributed_state_dict(checkpoint_path): - """ Load parameters' state_dict from checkpoint_path """ + """Load parameters' state_dict from checkpoint_path""" all_state_dict = {} for idx, ckpt_file in enumerate(checkpoint_path): state_dict_info = paddle.load(ckpt_file, return_numpy=True) pre_world_size = state_dict_info["world_size"] - assert pre_world_size == len(checkpoint_path), \ - "The number of 'checkpoint_path' must be equal to the last training world size." + assert pre_world_size == len( + checkpoint_path + ), "The number of 'checkpoint_path' must be equal to the last training world size." if idx == 0: addition_info = state_dict_info["addition_info"] for name, value in state_dict_info["model"].items(): @@ -737,7 +796,7 @@ def _load_distributed_state_dict(checkpoint_path): all_state_dict_info = { "model": all_state_dict, - "addition_info": addition_info + "addition_info": addition_info, } return all_state_dict_info @@ -758,13 +817,14 @@ def get_dist_attr(program, dist_context=None): for var in program.list_vars(): if is_parameter(var) or is_belong_to_optimizer(var): tensor_dist_attr = dist_context.get_tensor_dist_attr_for_program( - var) + var + ) process_mesh = tensor_dist_attr.process_mesh dims_mapping = tensor_dist_attr.dims_mapping dist_attr[var.name] = { "process_shape": process_mesh.topology, "process_group": process_mesh.processes, - "dims_mapping": dims_mapping + "dims_mapping": dims_mapping, } return dist_attr @@ -782,19 +842,26 @@ def merge_and_slice_parameter(dist_param_dict, pre_dist_attr, cur_dist_attr): dist_param_dict(dict): parameters' value of current rank. """ assert _check_dist_attr(pre_dist_attr), "'pre_dist_attr' cannot be None." - assert isinstance(dist_param_dict, dict), \ - "The type of 'dist_param_dict' should be 'dict', but got {}.".format( - str(type(dist_param_dict))) + assert isinstance( + dist_param_dict, dict + ), "The type of 'dist_param_dict' should be 'dict', but got {}.".format( + str(type(dist_param_dict)) + ) for name, value in dist_param_dict.items(): if not isinstance(name, str): - raise TypeError("The key of 'dist_param_dict' is parameter's name, " - "and its type should be 'str', but got {}.".format( - str(type(name)))) + raise TypeError( + "The key of 'dist_param_dict' is parameter's name, " + "and its type should be 'str', but got {}.".format( + str(type(name)) + ) + ) if not isinstance(value, list) or not all( - isinstance(v, np.ndarray) for v in value): + isinstance(v, np.ndarray) for v in value + ): raise TypeError( "The value of 'dist_param_dict' is parameter's value of all ranks, " - "and its type should be 'list(numpy.ndarray)'.") + "and its type should be 'list(numpy.ndarray)'." + ) if cur_dist_attr is None: return {} @@ -822,7 +889,8 @@ def merge_and_slice_parameter(dist_param_dict, pre_dist_attr, cur_dist_attr): cur_dims_mapping = cur_attr["dims_mapping"] if len(set(pre_dims_mapping)) > 1 or -1 not in pre_dims_mapping: complete_param = _merge_parameter_with_dist_attr( - pre_param, pre_attr) + pre_param, pre_attr + ) dist_param_dict[var_name] = complete_param else: complete_param = pre_param[0] @@ -830,7 +898,8 @@ def merge_and_slice_parameter(dist_param_dict, pre_dist_attr, cur_dist_attr): if len(set(cur_dims_mapping)) > 1 or -1 not in cur_dims_mapping: sliced_param = _slice_parameter_with_dist_attr( - complete_param, cur_attr) + complete_param, cur_attr + ) dist_param_dict[var_name] = sliced_param for var_name in pre_dist_attr: @@ -841,67 +910,81 @@ def merge_and_slice_parameter(dist_param_dict, pre_dist_attr, cur_dist_attr): if param_not_in_pre: warnings.warn( "Parameters '{}' are not found in last training process.".format( - str(param_not_in_pre))) + str(param_not_in_pre) + ) + ) if param_not_in_cur: warnings.warn( "Parameters '{}' are not found in current training process.".format( - str(param_not_in_cur))) + str(param_not_in_cur) + ) + ) return dist_param_dict def _merge_parameter_with_dist_attr(param_list, dist_attr): - """ Merge parameter with distributed attribute """ + """Merge parameter with distributed attribute""" from .reshard import Resharder dims_mapping = dist_attr["dims_mapping"] process_shape = dist_attr["process_shape"] process_group = dist_attr["process_group"] # get the complete shape of the parameter - complete_shape = Resharder.compute_complete_shape(param_list[0].shape, - process_shape, - dims_mapping) + complete_shape = Resharder.compute_complete_shape( + param_list[0].shape, process_shape, dims_mapping + ) # merge the parameter with dist_attr partition_param_list = [] merged_partiton = [] for process in process_group: partition_index = Resharder.compute_partition_index( - process, complete_shape, dims_mapping, process_shape, process_group) + process, complete_shape, dims_mapping, process_shape, process_group + ) index = process_group.index(process) if partition_index not in merged_partiton: merged_partiton.append(partition_index) - _merge_parameter(partition_param_list, param_list[index], - partition_index, complete_shape) + _merge_parameter( + partition_param_list, + param_list[index], + partition_index, + complete_shape, + ) - assert len(partition_param_list) == 1 or not partition_param_list, \ - "Fail to merge parameter" + assert ( + len(partition_param_list) == 1 or not partition_param_list + ), "Fail to merge parameter" complete_param = partition_param_list[0][0] return complete_param def _slice_parameter_with_dist_attr(param, dist_attr): - """ Slice parameter with distributed attribute """ - param = np.array(param) if isinstance(param, - paddle.fluid.LoDTensor) else param + """Slice parameter with distributed attribute""" + param = ( + np.array(param) if isinstance(param, paddle.fluid.LoDTensor) else param + ) dims_mapping = dist_attr["dims_mapping"] process_shape = dist_attr["process_shape"] process_group = dist_attr["process_group"] # slice the parameter with dist_attr - partition_index_list = _get_split_indices(param.shape, dims_mapping, - process_shape, process_group) - sliced_param_list = _slice_parameter(param, partition_index_list, - len(partition_index_list)) + partition_index_list = _get_split_indices( + param.shape, dims_mapping, process_shape, process_group + ) + sliced_param_list = _slice_parameter( + param, partition_index_list, len(partition_index_list) + ) # get the current parameter's index in sliced_param_list rank_id = paddle.distributed.get_rank() - sliced_param_index = _get_sliced_param_index(rank_id, param.shape, - dims_mapping, process_shape, - process_group) + sliced_param_index = _get_sliced_param_index( + rank_id, param.shape, dims_mapping, process_shape, process_group + ) sliced_param = sliced_param_list[sliced_param_index] return sliced_param -def _merge_parameter(partition_param_list, param, partition_index, - complete_shape): +def _merge_parameter( + partition_param_list, param, partition_index, complete_shape +): """ Merge partitial parameters to a complete one. @@ -935,19 +1018,30 @@ def _merge_parameter(partition_param_list, param, partition_index, else: i = 0 while i < len(partition_param_list): - concat_axis, first_order, new_partition = Resharder.compute_concat_info( - partition_param_list[i][1], partition_index) + ( + concat_axis, + first_order, + new_partition, + ) = Resharder.compute_concat_info( + partition_param_list[i][1], partition_index + ) if concat_axis != -1: if first_order == 0: new_param = np.concatenate( - (partition_param_list[i][0], param), axis=concat_axis) + (partition_param_list[i][0], param), axis=concat_axis + ) else: new_param = np.concatenate( - (param, partition_param_list[i][0]), axis=concat_axis) + (param, partition_param_list[i][0]), axis=concat_axis + ) partition_param_list.pop(i) - _merge_parameter(partition_param_list, new_param, new_partition, - complete_shape) + _merge_parameter( + partition_param_list, + new_param, + new_partition, + complete_shape, + ) break i += 1 @@ -975,19 +1069,21 @@ def _slice_parameter(complete_param, partition_index_list, length): """ sliced_param_list = [] axis = len(complete_param.shape) - length - sliced_param = np.split(complete_param, - partition_index_list[axis], - axis=axis) + sliced_param = np.split( + complete_param, partition_index_list[axis], axis=axis + ) if length == 1: return sliced_param for param in sliced_param: sliced_param_list.extend( - _slice_parameter(param, partition_index_list, length - 1)) + _slice_parameter(param, partition_index_list, length - 1) + ) return sliced_param_list -def _get_sliced_param_index(rank, complete_shape, dims_mapping, process_shape, - process_group): +def _get_sliced_param_index( + rank, complete_shape, dims_mapping, process_shape, process_group +): """ Get sliced_param's index of current rank in all sliced parameters list. @@ -1015,10 +1111,9 @@ def _get_sliced_param_index(rank, complete_shape, dims_mapping, process_shape, """ from .reshard import Resharder - partition_index = Resharder.compute_partition_index(rank, complete_shape, - dims_mapping, - process_shape, - process_group) + partition_index = Resharder.compute_partition_index( + rank, complete_shape, dims_mapping, process_shape, process_group + ) sliced_param_index = 0 for i, shape in enumerate(complete_shape): if dims_mapping[i] == -1: @@ -1033,8 +1128,9 @@ def _get_sliced_param_index(rank, complete_shape, dims_mapping, process_shape, return sliced_param_index -def _get_split_indices(complete_shape, dims_mapping, process_shape, - process_group): +def _get_split_indices( + complete_shape, dims_mapping, process_shape, process_group +): """ Get split indices of every dimension. @@ -1059,15 +1155,20 @@ def _get_split_indices(complete_shape, dims_mapping, process_shape, split_indices_list = [] for process in process_group: partition_index = Resharder.compute_partition_index( - process, complete_shape, dims_mapping, process_shape, process_group) + process, complete_shape, dims_mapping, process_shape, process_group + ) if split_indices_list: for dim in range(len(partition_index)): split_indices_list[dim].extend(partition_index[dim]) else: split_indices_list = partition_index split_indices_list = list( - map(lambda x, y: list(set(x) - set([y]) - set([0])), split_indices_list, - complete_shape)) + map( + lambda x, y: list(set(x) - set([y]) - set([0])), + split_indices_list, + complete_shape, + ) + ) split_indices_list = [sorted(x) for x in split_indices_list] return split_indices_list @@ -1086,8 +1187,10 @@ def set_grad_var_shape(program, dist_context): if int(op.attr('op_role')) != int(OpRole.Backward): continue - if int(block.ops[idx-1].attr('op_role')) == int(OpRole.Forward) or \ - int(block.ops[idx-1].attr('op_role')) == 257: + if ( + int(block.ops[idx - 1].attr('op_role')) == int(OpRole.Forward) + or int(block.ops[idx - 1].attr('op_role')) == 257 + ): appended_grad_times += 1 if op.type in ["check_finite_and_unscale", "update_loss_scaling"]: @@ -1105,61 +1208,102 @@ def set_grad_var_shape(program, dist_context): continue if var_name in grad_var_to_var[appended_grad_times]: forward_var_name = grad_var_to_var[appended_grad_times][ - var_name] + var_name + ] else: - forward_var_name = var_name[:var_name.find("@GRAD")] + forward_var_name = var_name[: var_name.find("@GRAD")] if op.type in [ - "c_allreduce_sum", "c_identity", "scale", "cast", - "fill_any_like" + "c_allreduce_sum", + "c_identity", + "scale", + "cast", + "fill_any_like", ]: forward_var_name = op.input_arg_names[0] - elif op.type == "matmul_v2_grad" or op.type == "matmul_grad" or op.type == "mul_grad": + elif ( + op.type == "matmul_v2_grad" + or op.type == "matmul_grad" + or op.type == "mul_grad" + ): forward_var_name = None for output_name in op.output_names: if var_name in op.output(output_name): assert "@GRAD" in output_name - input_name = output_name[:output_name.find("@GRAD")] + input_name = output_name[: output_name.find("@GRAD")] assert len(op.input(input_name)) == 1 forward_var_name = op.input(input_name)[0] assert forward_var_name is not None need_set_shape_list = [ - "reshape2_grad", "softmax_with_cross_entropy_grad", - "transpose2_grad", "softmax_grad", "cross_entropy_grad2", - "dropout_grad", "tanh_grad", "slice", "assign", - "matmul_v2_triple_grad", "elementwise_add_triple_grad", - "fill_constant", "sqrt_grad", + "reshape2_grad", + "softmax_with_cross_entropy_grad", + "transpose2_grad", + "softmax_grad", + "cross_entropy_grad2", + "dropout_grad", + "tanh_grad", + "slice", + "assign", + "matmul_v2_triple_grad", + "elementwise_add_triple_grad", + "fill_constant", + "sqrt_grad", "fused_softmax_mask_upper_triangle_grad", - "flatten_contiguous_range_grad", "relu_grad" + "flatten_contiguous_range_grad", + "relu_grad", ] forward_list = [ - "reshape2", "softmax_with_cross_entropy", "transpose2", - "softmax", "cross_entropy2", "dropout", "tanh", - ["slice_grad", "c_allgather"], "assign", "matmul_v2_grad_grad", - "elementwise_add_grad_grad", "shape", "sqrt", - "fused_softmax_mask_upper_triangle", "flatten_contiguous_range", - "relu" + "reshape2", + "softmax_with_cross_entropy", + "transpose2", + "softmax", + "cross_entropy2", + "dropout", + "tanh", + ["slice_grad", "c_allgather"], + "assign", + "matmul_v2_grad_grad", + "elementwise_add_grad_grad", + "shape", + "sqrt", + "fused_softmax_mask_upper_triangle", + "flatten_contiguous_range", + "relu", ] if op.type in need_set_shape_list: for forward_op in block.ops: idx = need_set_shape_list.index(op.type) forward_op_name = forward_list[idx] - if forward_op.type in forward_op_name and forward_var_name in forward_op.input_arg_names: - op_dist_attr = dist_context.get_op_dist_attr_for_program( - forward_op) + if ( + forward_op.type in forward_op_name + and forward_var_name in forward_op.input_arg_names + ): + op_dist_attr = ( + dist_context.get_op_dist_attr_for_program( + forward_op + ) + ) break forward_input_dist_attr = op_dist_attr.get_input_dist_attr( - forward_var_name) - assert forward_input_dist_attr is not None, f"{forward_var_name, str(op)}" + forward_var_name + ) + assert ( + forward_input_dist_attr is not None + ), f"{forward_var_name, str(op)}" forward_var = vars[forward_var_name] - forward_var_dist_attr = dist_context.get_tensor_dist_attr_for_program( - forward_var) + forward_var_dist_attr = ( + dist_context.get_tensor_dist_attr_for_program(forward_var) + ) assert forward_var_dist_attr is not None grad_var = vars[var_name] - ref_shape = infer_shape(block, forward_var, forward_var_dist_attr, - forward_input_dist_attr) + ref_shape = infer_shape( + block, + forward_var, + forward_var_dist_attr, + forward_input_dist_attr, + ) if list(grad_var.shape) != ref_shape: grad_var.desc.set_shape(ref_shape) @@ -1171,28 +1315,33 @@ def set_grad_var_shape(program, dist_context): def is_forward_op(op): op_role = int(op.attr('op_role')) - return OP_ROLE_KEY in op.attr_names and (op_role == int(OpRole.Forward) - or op_role == int(OpRole.Loss)) + return OP_ROLE_KEY in op.attr_names and ( + op_role == int(OpRole.Forward) or op_role == int(OpRole.Loss) + ) def is_backward_op(op): - return OP_ROLE_KEY in op.attr_names and \ - int(op.all_attrs()[OP_ROLE_KEY]) & int(OpRole.Backward) + return OP_ROLE_KEY in op.attr_names and int( + op.all_attrs()[OP_ROLE_KEY] + ) & int(OpRole.Backward) def is_optimize_op(op): - return OP_ROLE_KEY in op.attr_names and \ - int(op.all_attrs()[OP_ROLE_KEY]) & int(OpRole.Optimize) + return OP_ROLE_KEY in op.attr_names and int( + op.all_attrs()[OP_ROLE_KEY] + ) & int(OpRole.Optimize) def is_lr_sched_op(op): - return OP_ROLE_KEY in op.attr_names and \ - int(op.all_attrs()[OP_ROLE_KEY]) & int(OpRole.Optimize.LRSched) + return OP_ROLE_KEY in op.attr_names and int( + op.all_attrs()[OP_ROLE_KEY] + ) & int(OpRole.Optimize.LRSched) def is_loss_op(op): - return OP_ROLE_KEY in op.attr_names and \ - int(op.all_attrs()[OP_ROLE_KEY]) == (int(OpRole.Forward) | int(OpRole.Loss)) + return OP_ROLE_KEY in op.attr_names and int( + op.all_attrs()[OP_ROLE_KEY] + ) == (int(OpRole.Forward) | int(OpRole.Loss)) def is_loss_grad_op(op): @@ -1203,8 +1352,9 @@ def is_loss_grad_op(op): def is_gradient_clip_op(op): - return op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/gradient_clip") + return op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/gradient_clip") def is_prim_op(op): @@ -1215,8 +1365,9 @@ def get_loss_op(block): loss_ops = [] for op in block.ops: if is_loss_op(op): - assert len(op.desc.output_arg_names() - ) == 1, "loss op should only output loss var" + assert ( + len(op.desc.output_arg_names()) == 1 + ), "loss op should only output loss var" loss_ops.append(op) assert len(loss_ops) == 1, "num of loss op is not equal to one" @@ -1236,7 +1387,8 @@ def set_var_dist_attr(dist_context, var, dims_mapping, process_mesh, **kwargs): def naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - new_op, process_mesh, ref_mapping, ctx): + new_op, process_mesh, ref_mapping, ctx +): assert process_mesh is not None assert ref_mapping is not None @@ -1270,9 +1422,11 @@ def update_op_dims_mapping_by_default_dist_impl(dist_op): dims_mapping = op_dist_attr.get_input_dims_mapping(arg_name) if len(dims_mapping) > 1: for idx, mapping in enumerate(dims_mapping[1:]): - assert mapping == -1, \ - "{} only the batch dimension (0-dim) can be sharded, but the dimension {} is sharded by {} part."\ - .format(op_desc.type(), idx, mapping) + assert ( + mapping == -1 + ), "{} only the batch dimension (0-dim) can be sharded, but the dimension {} is sharded by {} part.".format( + op_desc.type(), idx, mapping + ) batch_dim_mappings.append(dims_mapping[0]) for arg_name in op_desc.output_arg_names(): serial_tensor = dist_op.get_serial_output(arg_name) @@ -1282,23 +1436,31 @@ def update_op_dims_mapping_by_default_dist_impl(dist_op): if arg_name not in xshape_arg_names: if len(dims_mapping) > 1: for idx, mapping in enumerate(dims_mapping[1:]): - assert mapping == -1, \ - "{} only the batch dimension (0-dim) can be sharded, but the dimension {} is sharded by {} part."\ - .format(op_desc.type(), idx, mapping) + assert ( + mapping == -1 + ), "{} only the batch dimension (0-dim) can be sharded, but the dimension {} is sharded by {} part.".format( + op_desc.type(), idx, mapping + ) batch_dim_mappings.append(dims_mapping[0]) else: - assert dims_mapping[0] == -1, \ - "{} only the batch dimension (1-dim) of XShape can be sharded, but the dimension 0 is sharded by {} part."\ - .format(op_desc.type(), mapping) + assert ( + dims_mapping[0] == -1 + ), "{} only the batch dimension (1-dim) of XShape can be sharded, but the dimension 0 is sharded by {} part.".format( + op_desc.type(), mapping + ) if len(dims_mapping) > 2: for idx, mapping in enumerate(dims_mapping[2:]): - assert mapping == -1, \ - "{} only the batch dimension (1-dim) of XShape can be sharded, but the dimension {} is sharded by {} part."\ - .format(op_desc.type(), idx, mapping) + assert ( + mapping == -1 + ), "{} only the batch dimension (1-dim) of XShape can be sharded, but the dimension {} is sharded by {} part.".format( + op_desc.type(), idx, mapping + ) batch_dim_mappings.append(dims_mapping[1]) compatible_dim_mapping = compute_compatible_dim_mapping(batch_dim_mappings) - assert compatible_dim_mapping is not None, "There is no compatible dim mapping." + assert ( + compatible_dim_mapping is not None + ), "There is no compatible dim mapping." for arg_name in op_desc.input_arg_names(): serial_tensor = dist_op.get_serial_input(arg_name) if serial_tensor.is_parameter: @@ -1344,8 +1506,9 @@ def update_op_dims_mapping_by_elementwise_like_dist_impl(dist_op): if input_dims_mapping_lens[arg_name] < max_dims_mapping_len: new_dims_mapping = [-1 for _ in range(max_dims_mapping_len)] for i in range(input_dims_mapping_lens[arg_name]): - new_idx = (max_dims_mapping_len - - input_dims_mapping_lens[arg_name]) + i + new_idx = ( + max_dims_mapping_len - input_dims_mapping_lens[arg_name] + ) + i new_dims_mapping[new_idx] = input_dims_mapping_dict[arg_name][i] dims_mapping_list.append(new_dims_mapping) else: @@ -1357,7 +1520,9 @@ def update_op_dims_mapping_by_elementwise_like_dist_impl(dist_op): dims_mapping_list.append(dims_mapping) compatible_dims_mapping = compute_compatible_dims_mapping(dims_mapping_list) - assert compatible_dims_mapping is not None, "There is no compatible dim mapping." + assert ( + compatible_dims_mapping is not None + ), "There is no compatible dim mapping." for arg_name in input_arg_names: if input_dims_mapping_lens[arg_name] < max_dims_mapping_len: @@ -1365,55 +1530,64 @@ def update_op_dims_mapping_by_elementwise_like_dist_impl(dist_op): -1 for _ in range(input_dims_mapping_lens[arg_name]) ] for i in range(input_dims_mapping_lens[arg_name]): - new_idx = (max_dims_mapping_len - - input_dims_mapping_lens[arg_name]) + i + new_idx = ( + max_dims_mapping_len - input_dims_mapping_lens[arg_name] + ) + i new_dims_mapping[i] = compatible_dims_mapping[new_idx] if new_dims_mapping != input_dims_mapping_dict[arg_name]: op_dist_attr.set_input_dims_mapping(arg_name, new_dims_mapping) changed = True else: if compatible_dims_mapping != input_dims_mapping_dict[arg_name]: - op_dist_attr.set_input_dims_mapping(arg_name, - compatible_dims_mapping) + op_dist_attr.set_input_dims_mapping( + arg_name, compatible_dims_mapping + ) changed = True for arg_name in output_arg_names: dims_mapping = op_dist_attr.get_output_dims_mapping(arg_name) if compatible_dims_mapping != dims_mapping: - op_dist_attr.set_output_dims_mapping(arg_name, - compatible_dims_mapping) + op_dist_attr.set_output_dims_mapping( + arg_name, compatible_dims_mapping + ) changed = True return changed -def get_all_distributed_main_program(serial_program_info, dist_context, - parallelizer): +def get_all_distributed_main_program( + serial_program_info, dist_context, parallelizer +): "Get all distributed main programs by dist_context." from .dist_context import DistributedOperatorContext + cluster = serial_program_info.cluster copied_parallelizer = copy.deepcopy(parallelizer) all_dist_main_program = [] - ranks = paddle.distributed.get_world_size() if cluster is None else len( - cluster.get_all_devices("GPU")) + ranks = ( + paddle.distributed.get_world_size() + if cluster is None + else len(cluster.get_all_devices("GPU")) + ) for rank_id in range(ranks): used_dist_context = copy.deepcopy(dist_context) used_dist_context._dist_op_context = DistributedOperatorContext() - _, _, dist_startup_program, dist_main_program, _ = copied_parallelizer._get_dist_program( - rank_id, used_dist_context) + ( + _, + _, + dist_startup_program, + dist_main_program, + _, + ) = copied_parallelizer._get_dist_program(rank_id, used_dist_context) all_dist_main_program.append(dist_main_program) return all_dist_main_program class SerialProgramInfo: - - def __init__(self, - train_program, - satrtup_program, - loss, - optimizer, - cluster=None): + def __init__( + self, train_program, satrtup_program, loss, optimizer, cluster=None + ): self._train_program = train_program self._startup_program = satrtup_program self._loss = loss @@ -1442,7 +1616,6 @@ def cluster(self): def get_standalone_cost_data(distributed_programs): - def _compute_runtime(op_cost, op, vars): runtime = 0 try: @@ -1455,32 +1628,47 @@ def _compute_runtime(op_cost, op, vars): parsed_info = op_config.split("\n") variable = "(Variable)" for info in parsed_info: - variable = "(Variable)" if "(Variable)" in info else "(list" + variable = ( + "(Variable)" if "(Variable)" in info else "(list" + ) if variable in info: - arg_name_lower = info[:info.find(variable) - 1] + arg_name_lower = info[: info.find(variable) - 1] shape_left_boundary = info.find("[") shape_right_boundary = info.find("]") - assert shape_left_boundary > 0 and shape_right_boundary > 0 and shape_right_boundary > shape_left_boundary, "Get shape failed." - shape = info[shape_left_boundary + - 1:shape_right_boundary].split(",") + assert ( + shape_left_boundary > 0 + and shape_right_boundary > 0 + and shape_right_boundary > shape_left_boundary + ), "Get shape failed." + shape = info[ + shape_left_boundary + 1 : shape_right_boundary + ].split(",") shape = list(map(lambda x: int(x.strip()), shape)) dtype_factor = 1 total_static_input_size += reduce(lambda x, y: x * y, shape) if op.type == "c_embedding": - arg_name_lower = "w" if arg_name_lower == "weight" else "ids" + arg_name_lower = ( + "w" if arg_name_lower == "weight" else "ids" + ) for arg_name in op.input_names: if arg_name.lower() == arg_name_lower: for var_name in op.input(arg_name): var = vars[var_name] total_actual_input_size += reduce( - lambda x, y: x * y, var.shape) + lambda x, y: x * y, var.shape + ) break - assert total_static_input_size > 0 and total_actual_input_size > 0, "Get input size failed." + assert ( + total_static_input_size > 0 and total_actual_input_size > 0 + ), "Get input size failed." - actual_runtime = total_actual_input_size / total_static_input_size * runtime + actual_runtime = ( + total_actual_input_size / total_static_input_size * runtime + ) return actual_runtime import paddle.cost_model as cm + cost_model = cm.CostModel() cost_model.static_cost_data() DEFAULT_MULTIPLE = 2 @@ -1491,13 +1679,16 @@ def _compute_runtime(op_cost, op, vars): "reshape2": "reshape", "unsqueeze2": "unsqueeze", "reduce_sum": "sum", - "elementwise_div": "divide" + "elementwise_div": "divide", } standalone_cost_data = [] # skip ops not_enum_ops = [ - "create_py_reader", "create_double_buffer_reader", "read", "assign" + "create_py_reader", + "create_double_buffer_reader", + "read", + "assign", ] for distributed_program in distributed_programs: cost_data = {} @@ -1507,26 +1698,33 @@ def _compute_runtime(op_cost, op, vars): if op.type in not_enum_ops: cost_data[op.desc.id()] = runtime continue - dtype = str(vars[op.input_arg_names[0]].dtype - ) if op.input_arg_names else "float32" + dtype = ( + str(vars[op.input_arg_names[0]].dtype) + if op.input_arg_names + else "float32" + ) if int(op.attr('op_role')) == int(OpRole.Backward): if "_grad" in op.type: forward_op_name = op.type[:-5] if forward_op_name in OP_NAME_MAPPING.keys(): forward_op_name = OP_NAME_MAPPING[forward_op_name] - op_cost = cost_model.get_static_op_time(forward_op_name, - forward=False, - dtype=dtype) + op_cost = cost_model.get_static_op_time( + forward_op_name, forward=False, dtype=dtype + ) if op_cost: runtime = _compute_runtime(op_cost, op, vars) else: - op_cost = cost_model.get_static_op_time(forward_op_name, - dtype=dtype) + op_cost = cost_model.get_static_op_time( + forward_op_name, dtype=dtype + ) if op_cost: runtime = 2 * _compute_runtime(op_cost, op, vars) elif int(op.attr('op_role')) == int(OpRole.Forward): - op_name = OP_NAME_MAPPING[ - op.type] if op.type in OP_NAME_MAPPING.keys() else op.type + op_name = ( + OP_NAME_MAPPING[op.type] + if op.type in OP_NAME_MAPPING.keys() + else op.type + ) op_cost = cost_model.get_static_op_time(op_name) if op_cost: runtime = _compute_runtime(op_cost, op, vars) @@ -1565,7 +1763,8 @@ def to_list(value): def debug_program(program, path, name): filename = os.path.join( - path, name + '_program' + ".%d" % (paddle.distributed.get_rank())) + path, name + '_program' + ".%d" % (paddle.distributed.get_rank()) + ) with open(filename, 'w') as f: f.write(str(program)) @@ -1599,9 +1798,11 @@ def get_lr(optimizer): return optimizer._learning_rate() else: raise TypeError( - "'optimizer' must be object of class `paddle.optimizer.Optimizer`" \ - " or `paddle.fluid.optimizer.Optimizer`, but got {}.".format(type(optimizer)) + "'optimizer' must be object of class `paddle.optimizer.Optimizer`" + " or `paddle.fluid.optimizer.Optimizer`, but got {}.".format( + type(optimizer) ) + ) def initialize_pg_in_full_mode(all_process_groups, cur_rank): @@ -1631,21 +1832,28 @@ def initialize_pg_in_full_mode(all_process_groups, cur_rank): if is_send: recv_rank = process_group.ranks[1] recv_rank_ip, recv_rank_port = genv.trainer_endpoints[ - recv_rank].split(":") + recv_rank + ].split(":") connect_port = int(recv_rank_port) + magic_num - client_socket = socket.socket(socket.AF_INET, - socket.SOCK_STREAM) + client_socket = socket.socket( + socket.AF_INET, socket.SOCK_STREAM + ) client_socket.connect((recv_rank_ip, connect_port)) client_socket.send(str(cur_rank).encode('utf-8')) rank = client_socket.recv(buff_size).decode('utf-8') rank = int(rank) if rank != recv_rank: raise ValueError( - "Please check comm pair, the recv rank should be {} but got {}." - .format(recv_rank, rank)) + "Please check comm pair, the recv rank should be {} but got {}.".format( + recv_rank, rank + ) + ) else: - print("It is able to instantiate {} as sender now.".format( - process_group.ranks)) + print( + "It is able to instantiate {} as sender now.".format( + process_group.ranks + ) + ) client_socket.close() else: send_rank = process_group.ranks[0] @@ -1657,10 +1865,14 @@ def initialize_pg_in_full_mode(all_process_groups, cur_rank): has_recv_by_socket.append(rank) else: client_sockets[send_rank].send( - str(cur_rank).encode("utf-8")) + str(cur_rank).encode("utf-8") + ) client_sockets[send_rank].close() - print("It is able to instantiate {} as recver now.". - format(process_group.ranks)) + print( + "It is able to instantiate {} as recver now.".format( + process_group.ranks + ) + ) break process_group.instantiate() server_socket.close() diff --git a/python/paddle/distributed/cloud_utils.py b/python/paddle/distributed/cloud_utils.py index b186ff64baf55..2e1d85205a538 100644 --- a/python/paddle/distributed/cloud_utils.py +++ b/python/paddle/distributed/cloud_utils.py @@ -13,7 +13,11 @@ # limitations under the License. import os -from paddle.distributed.utils.launch_utils import get_cluster, get_gpus, get_cluster_from_args +from paddle.distributed.utils.launch_utils import ( + get_cluster, + get_gpus, + get_cluster_from_args, +) from paddle.distributed.utils.launch_utils import logger __all__ = [] @@ -23,7 +27,7 @@ def get_cloud_cluster(args_node_ips, args_node_ip, args_port, selected_devices): """ args_node_ips:string, args_node_ip:string, args_port: int, selected_devices:list """ - #you can automatically get ip info while using paddlecloud multi nodes mode. + # you can automatically get ip info while using paddlecloud multi nodes mode. node_ips = os.getenv("PADDLE_TRAINERS") assert node_ips is not None, "PADDLE_TRAINERS should not be None" @@ -41,16 +45,23 @@ def get_cloud_cluster(args_node_ips, args_node_ip, args_port, selected_devices): node_rank = int(node_rank) if node_ip != "127.0.0.1" and node_ip != args_node_ip: - logger.warning("Please NOTE: When using paddlecloud, node_ip is \ + logger.warning( + "Please NOTE: When using paddlecloud, node_ip is \ automatically got from POD_IP. Your input node_ip: {} doesn't equals to \ -node_ip: {} from paddlecloud environment.".format(args_node_ip, node_ip)) +node_ip: {} from paddlecloud environment.".format( + args_node_ip, node_ip + ) + ) if args_node_ips != "127.0.0.1" and args_node_ips != ",".join(node_ips): logger.warning( "Please NOTE: When using paddlecloud, cluster_node_ips is \ automatically got from PADDLE_TRAINERS(multi nodes) or POD_IP(single node).\ Your input cluster_node_ips: {} doesn't equals to IPs: {} from \ -paddlecloud environment.".format(args_node_ips, node_ips)) +paddlecloud environment.".format( + args_node_ips, node_ips + ) + ) # DISTRIBUTED_TRAINER_ENDPOINTS: new environment since paddlecloud 1.8.4 # e.g: DISTRIBUTED_TRAINER_ENDPOINTS="ip1:port1,ip1:port2,ip1:port3,ip1:port4,ip2:port5,ip2:port6,ip2:port7,ip2:port8" @@ -61,10 +72,13 @@ def get_cloud_cluster(args_node_ips, args_node_ip, args_port, selected_devices): try: paddle_port = int(os.getenv("PADDLE_PORT", "")) - if paddle_ports_num >= len( - selected_devices) and paddle_port != args_port: + if ( + paddle_ports_num >= len(selected_devices) + and paddle_port != args_port + ): logger.warning( - "Use Cloud specified port:{}.".format(paddle_port)) + "Use Cloud specified port:{}.".format(paddle_port) + ) started_port = paddle_port except Exception as e: @@ -84,15 +98,21 @@ def get_cloud_cluster(args_node_ips, args_node_ip, args_port, selected_devices): assert num_nodes * paddle_ports_num == len(trainer_endpoints_ori) for i in range(num_nodes): trainer_endpoints.append( - trainer_endpoints_ori[i * paddle_ports_num:(i + 1) * - paddle_ports_num]) + trainer_endpoints_ori[ + i * paddle_ports_num : (i + 1) * paddle_ports_num + ] + ) - logger.debug("parsed from args: node_ips:{} \ + logger.debug( + "parsed from args: node_ips:{} \ node_ip:{} node_rank:{} trainer_endpoints:{}".format( - node_ips, node_ip, node_rank, trainer_endpoints)) + node_ips, node_ip, node_rank, trainer_endpoints + ) + ) - cluster, pod = get_cluster(node_ips, node_ip, trainer_endpoints, - selected_devices) + cluster, pod = get_cluster( + node_ips, node_ip, trainer_endpoints, selected_devices + ) return cluster, cluster.pods[node_rank] @@ -104,15 +124,22 @@ def get_cluster_and_pod(args): # parse arguments, used for cloud-single-machine and local selected_devices = get_gpus(args.selected_devices) trainers_num = _get_trainers_num() - logger.debug("parsed from args trainerss_num:{} selected_devices:{}".format( - trainers_num, selected_devices)) + logger.debug( + "parsed from args trainerss_num:{} selected_devices:{}".format( + trainers_num, selected_devices + ) + ) cluster = None pod = None if args.use_paddlecloud and trainers_num != 1: - cluster, pod = get_cloud_cluster(args.cluster_node_ips, args.node_ip, - args.started_port, selected_devices) + cluster, pod = get_cloud_cluster( + args.cluster_node_ips, + args.node_ip, + args.started_port, + selected_devices, + ) logger.info("get cluster from cloud:{}".format(cluster)) else: cluster, pod = get_cluster_from_args(args, selected_devices) diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py index fa1b3e00d4760..77258f7036cbd 100644 --- a/python/paddle/distributed/collective.py +++ b/python/paddle/distributed/collective.py @@ -92,8 +92,9 @@ def _get_group_map(): global _group_map if _global_env_gid not in _group_map: genv = _get_global_env() - _group_map[_global_env_gid] = Group(genv.rank, 0, - list(range(genv.world_size))) + _group_map[_global_env_gid] = Group( + genv.rank, 0, list(range(genv.world_size)) + ) return _group_map @@ -108,8 +109,10 @@ def _get_group_map_by_name(): def _get_default_group(): global _group_map_by_name - assert is_initialized(), ("Call paddle.distributed.init_parallel_env first " - "to initialize the distributed environment.") + assert is_initialized(), ( + "Call paddle.distributed.init_parallel_env first " + "to initialize the distributed environment." + ) return _get_group_map_by_name()[_default_group_name] @@ -165,21 +168,23 @@ def get_group(id=0): return gm[id] if id in gm else None -def _new_process_group_impl(backend, - store, - rank, - world_size, - group_name, - pg_options, - group_id=0, - src_rank=None, - dst_rank=None): +def _new_process_group_impl( + backend, + store, + rank, + world_size, + group_name, + pg_options, + group_id=0, + src_rank=None, + dst_rank=None, +): pg = None genv = _get_global_env() if backend != 'heter': assert src_rank is None and dst_rank is None, ( - "src_rank and dst_rank " - "can only be set for heter backend.") + "src_rank and dst_rank " "can only be set for heter backend." + ) assert backend in _valid_backend_list, "Unsupported backend: %s." % backend if backend == "gloo": place = core.CPUPlace() @@ -208,24 +213,27 @@ def _new_process_group_impl(backend, switch_ep = os.getenv("CLUSTER_SWITCH", None) assert switch_ep, "please set the CLUSTER_SWITCH variable." cluster_size_cumsum = np.cumsum(cluster_size) - cluster_offset = 0 if cluster_id == 0 else cluster_size_cumsum[ - cluster_id - 1] + cluster_offset = ( + 0 if cluster_id == 0 else cluster_size_cumsum[cluster_id - 1] + ) global_rank = cluster_offset + rank global_world_size = cluster_size_cumsum[-1] global_rank, global_world_size = _get_global_config(backend, rank) - pg = core.ProcessGroupHeter(store, - rank=global_rank, - world_size=global_world_size, - place=place, - gid=group_id, - local_rank=rank, - local_size=world_size, - gloo_rank=cluster_id, - gloo_size=len(cluster_size), - with_switch=True, - switch_endpoint=switch_ep, - src_rank=src_rank, - dst_rank=dst_rank) + pg = core.ProcessGroupHeter( + store, + rank=global_rank, + world_size=global_world_size, + place=place, + gid=group_id, + local_rank=rank, + local_size=world_size, + gloo_rank=cluster_id, + gloo_size=len(cluster_size), + with_switch=True, + switch_endpoint=switch_ep, + src_rank=src_rank, + dst_rank=dst_rank, + ) return pg @@ -271,10 +279,12 @@ def barrier(group=None): if not isinstance(ring_id, int): raise ValueError("The type of 'group' for barrier must be int.") helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [temp]}, - outputs={'Out': [temp]}, - attrs={'ring_id': ring_id}) + helper.append_op( + type=op_type, + inputs={'X': [temp]}, + outputs={'Out': [temp]}, + attrs={'ring_id': ring_id}, + ) # _custom_gid provides a way for users to @@ -296,7 +306,7 @@ def _barrier_by_tcp_store(group_name, store, timeout): return barrier_prefix = "Barrier/" + group_name + "/" - is_master = (global_rank == 0) + is_master = global_rank == 0 def _check_keys_ready(wait_keys): start_time = time.time() @@ -309,9 +319,12 @@ def _check_keys_ready(wait_keys): "Keys {} are not ready sinck rank {} is waiting them." "Two reason may cause this error:\n 1. The create process group api should be called by all ranks.\n" " 2. Try to increase the waiting time.\n".format( - group_name, wait_keys, global_rank)) + group_name, wait_keys, global_rank + ) + ) wait_keys = list( - filter(lambda key: int(store.get(key)) != 1, wait_keys)) + filter(lambda key: int(store.get(key)) != 1, wait_keys) + ) # all the workers set their exiting key and exit # the master will wait for all workers' exiting key, ensure to exit in the end @@ -363,22 +376,25 @@ def new_group(ranks=None, backend=None, timeout=_default_timeout): ranks = global_ranks assert len(ranks) <= len(global_ranks), ( "Size of new group must be less than or " - "equal to that of the default global group.") + "equal to that of the default global group." + ) size = len(ranks) ranks = sorted(ranks) if backend == 'heter' or (size > 1 and global_rank in ranks): rank = 0 if backend == 'heter' else ranks.index(global_rank) src_rank = ranks[0] if backend == 'heter' else None dst_rank = ranks[1] if backend == 'heter' else None - pg = _new_process_group_impl(backend, - _default_store, - rank, - size, - group_name, - pg_options=None, - group_id=gid, - src_rank=src_rank, - dst_rank=dst_rank) + pg = _new_process_group_impl( + backend, + _default_store, + rank, + size, + group_name, + pg_options=None, + group_id=gid, + src_rank=src_rank, + dst_rank=dst_rank, + ) else: rank = -1 pg = None @@ -386,7 +402,7 @@ def new_group(ranks=None, backend=None, timeout=_default_timeout): _group_map_by_name[group_name] = group _group_map[gid] = group _group_map_backend[group] = backend - #TODO: The method below is a new method for group management, will replace the previous + # TODO: The method below is a new method for group management, will replace the previous # three in the future. _add_new_group(group) @@ -402,7 +418,7 @@ def new_group(ranks=None, backend=None, timeout=_default_timeout): if not backend: backend = 'nccl' - assert backend == 'nccl', ("backend other than nccl is not supported yet") + assert backend == 'nccl', "backend other than nccl is not supported yet" genv = _get_global_env() global_rank = genv.rank @@ -431,30 +447,36 @@ def new_group(ranks=None, backend=None, timeout=_default_timeout): if core.is_compiled_with_cuda(): place = core.CUDAPlace(genv.device_id) - core.NCCLParallelContext(strategy, - place).init_with_ring_id(ring_id) + core.NCCLParallelContext(strategy, place).init_with_ring_id( + ring_id + ) elif core.is_compiled_with_npu(): place = core.NPUPlace(genv.device_id) - core.HCCLParallelContext(strategy, - place).init_with_ring_id(ring_id) + core.HCCLParallelContext(strategy, place).init_with_ring_id( + ring_id + ) elif core.is_compiled_with_mlu(): place = core.MLUPlace(genv.device_id) - core.CNCLParallelContext(strategy, - place).init_with_ring_id(ring_id) + core.CNCLParallelContext(strategy, place).init_with_ring_id( + ring_id + ) elif core.is_compiled_with_xpu(): place = core.XPUPlace(genv.device_id) - core.BKCLParallelContext(strategy, - place).init_with_ring_id(ring_id) + core.BKCLParallelContext(strategy, place).init_with_ring_id( + ring_id + ) else: - assert False, ("no cuda device found") + assert False, "no cuda device found" else: return gp # TODO(shenliang03): This is a temporary solution to solve the problem of # hang caused by cross-creation of new_group - tmp = paddle.to_tensor( - [1], dtype="int32") if _non_static_mode() else fill_constant( - [0], dtype="int32", value="1") + tmp = ( + paddle.to_tensor([1], dtype="int32") + if _non_static_mode() + else fill_constant([0], dtype="int32", value="1") + ) paddle.distributed.all_reduce(tmp, sync_op=True) paddle.distributed.wait(tmp) return gp @@ -586,8 +608,9 @@ def _sync_calc_stream(tensor): def _sync_comm_stream(tensor, ring_id=0): if _non_static_mode(): - return _legacy_C_ops.c_sync_comm_stream([tensor], [tensor], 'ring_id', - ring_id) + return _legacy_C_ops.c_sync_comm_stream( + [tensor], [tensor], 'ring_id', ring_id + ) op_type = 'c_sync_comm_stream' @@ -648,7 +671,7 @@ def broadcast(tensor, src, group=None, sync_op=True): if in_dygraph_mode(): group = _get_default_group() if group is None else group gsrc = group.get_group_rank(src) - assert gsrc >= 0, ("src rank out of group, need global rank") + assert gsrc >= 0, "src rank out of group, need global rank" task = group.process_group.broadcast(tensor, gsrc) if sync_op: task.wait() @@ -659,28 +682,48 @@ def broadcast(tensor, src, group=None, sync_op=True): use_calc_stream = sync_op ring_id = ring_id = 0 if group is None else group.id gsrc = src if group is None else group.get_group_rank(src) - assert gsrc >= 0, ("src rank out of group, need global rank") + assert gsrc >= 0, "src rank out of group, need global rank" if _non_static_mode(): - return _legacy_C_ops.c_broadcast(tensor, tensor, 'root', gsrc, - 'use_calc_stream', use_calc_stream, - 'ring_id', ring_id) + return _legacy_C_ops.c_broadcast( + tensor, + tensor, + 'root', + gsrc, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + ) op_type = 'c_broadcast' - check_variable_and_dtype(tensor, 'tensor', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'int8', 'uint8', - 'bool' - ], 'broadcast') + check_variable_and_dtype( + tensor, + 'tensor', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'int8', + 'uint8', + 'bool', + ], + 'broadcast', + ) helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [tensor]}, - outputs={'Out': [tensor]}, - attrs={ - 'root': gsrc, - 'use_calc_stream': use_calc_stream, - 'ring_id': ring_id, - }) + helper.append_op( + type=op_type, + inputs={'X': [tensor]}, + outputs={'Out': [tensor]}, + attrs={ + 'root': gsrc, + 'use_calc_stream': use_calc_stream, + 'ring_id': ring_id, + }, + ) def reduce(tensor, dst, op=ReduceOp.SUM, group=None, sync_op=True): @@ -730,7 +773,7 @@ def reduce(tensor, dst, op=ReduceOp.SUM, group=None, sync_op=True): op_type = _get_reduce_op(op, "reduce") group = _get_default_group() if group is None else group gdst = group.get_group_rank(dst) - assert gdst >= 0, ("dst rank out of group, need global rank") + assert gdst >= 0, "dst rank out of group, need global rank" task = group.process_group.reduce(tensor, gdst, op_type) if sync_op: task.wait() @@ -741,34 +784,72 @@ def reduce(tensor, dst, op=ReduceOp.SUM, group=None, sync_op=True): use_calc_stream = sync_op ring_id = 0 if group is None else group.id gdst = dst if group is None else group.get_group_rank(dst) - assert gdst >= 0, ("dst rank out of group, need global rank") + assert gdst >= 0, "dst rank out of group, need global rank" if _non_static_mode(): if op == ReduceOp.SUM: - return _legacy_C_ops.c_reduce_sum(tensor, tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id, 'root_id', gdst) + return _legacy_C_ops.c_reduce_sum( + tensor, + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'root_id', + gdst, + ) elif op == ReduceOp.MAX: - return _legacy_C_ops.c_reduce_max(tensor, tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id, 'root_id', gdst) + return _legacy_C_ops.c_reduce_max( + tensor, + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'root_id', + gdst, + ) elif op == ReduceOp.MIN: - return _legacy_C_ops.c_reduce_min(tensor, tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id, 'root_id', gdst) + return _legacy_C_ops.c_reduce_min( + tensor, + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'root_id', + gdst, + ) elif op == ReduceOp.PROD: - return _legacy_C_ops.c_reduce_prod(tensor, tensor, - 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id, 'root_id', gdst) + return _legacy_C_ops.c_reduce_prod( + tensor, + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'root_id', + gdst, + ) else: raise ValueError("Unknown parameter: {}.".format(op)) op_type = 'c_reduce' - check_variable_and_dtype(tensor, 'tensor', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'int8', 'uint8', - 'bool' - ], 'reduce') + check_variable_and_dtype( + tensor, + 'tensor', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'int8', + 'uint8', + 'bool', + ], + 'reduce', + ) if op == ReduceOp.SUM: op_type = 'c_reduce_sum' @@ -780,14 +861,16 @@ def reduce(tensor, dst, op=ReduceOp.SUM, group=None, sync_op=True): op_type = 'c_reduce_prod' helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [tensor]}, - outputs={'Out': [tensor]}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream, - 'root_id': gdst, - }) + helper.append_op( + type=op_type, + inputs={'X': [tensor]}, + outputs={'Out': [tensor]}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + 'root_id': gdst, + }, + ) def all_gather(tensor_list, tensor, group=None, sync_op=True): @@ -840,8 +923,9 @@ def convert_to_complex(list_of_tensor): list_of_complex.append(paddle.as_complex(tensor)) return list_of_complex - is_input_complex = (tensor.dtype == paddle.complex64 - or tensor.dtype == paddle.complex128) + is_input_complex = ( + tensor.dtype == paddle.complex64 or tensor.dtype == paddle.complex128 + ) if is_input_complex: tensor = paddle.as_real(tensor) @@ -868,33 +952,68 @@ def convert_to_complex(list_of_tensor): nranks = _get_global_group().nranks if group is None else group.nranks if _non_static_mode(): - out = _legacy_C_ops.c_allgather(tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', ring_id, - 'nranks', nranks) + out = _legacy_C_ops.c_allgather( + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'nranks', + nranks, + ) else: op_type = 'c_allgather' helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=tensor.dtype) if not isinstance(tensor_list, list): - raise ValueError("The type of 'tensor_list' for all_gather " - "should be list.") + raise ValueError( + "The type of 'tensor_list' for all_gather " "should be list." + ) for elem in tensor_list: - check_variable_and_dtype(elem, 'tensor_list', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'bool', - 'int8', 'uint8', 'complex64', 'complex128' - ], 'all_gather') - check_variable_and_dtype(tensor, 'tensor', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'bool', 'int8', - 'uint8', 'complex64', 'complex128' - ], 'all_gather') - helper.append_op(type=op_type, - inputs={'X': [tensor]}, - outputs={'Out': [out]}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream, - 'nranks': nranks - }) + check_variable_and_dtype( + elem, + 'tensor_list', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'bool', + 'int8', + 'uint8', + 'complex64', + 'complex128', + ], + 'all_gather', + ) + check_variable_and_dtype( + tensor, + 'tensor', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'bool', + 'int8', + 'uint8', + 'complex64', + 'complex128', + ], + 'all_gather', + ) + helper.append_op( + type=op_type, + inputs={'X': [tensor]}, + outputs={'Out': [out]}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + 'nranks': nranks, + }, + ) list_of_tensor = paddle.split(out, nranks, 0) if is_input_complex: @@ -950,7 +1069,8 @@ def all_gather_object(object_list, obj, group=None): print(object_list) # [{'foo': [1, 2, 3]}, {'bar': [4, 5, 6]}] (2 GPUs) """ - assert in_dygraph_mode( + assert ( + in_dygraph_mode() ), "all_gather_object doesn't support static graph mode." tensor, len_of_tensor = _convert_object_to_tensor(obj) @@ -971,7 +1091,8 @@ def all_gather_object(object_list, obj, group=None): all_gather(tensor_list, input_tensor, group) for i, tensor in enumerate(tensor_list): object_list.append( - _convert_tensor_to_object(tensor, list_len_of_tensor[i])) + _convert_tensor_to_object(tensor, list_len_of_tensor[i]) + ) def scatter(tensor, tensor_list=None, src=0, group=None, sync_op=True): @@ -1033,7 +1154,7 @@ def scatter(tensor, tensor_list=None, src=0, group=None, sync_op=True): gsrc = src if group is None else group.get_group_rank(src) rank = _get_global_group().rank if group is None else group.rank nranks = _get_global_group().nranks if group is None else group.nranks - assert gsrc >= 0, ("src rank out of group, need global rank") + assert gsrc >= 0, "src rank out of group, need global rank" if rank != gsrc: tensor_list = [] @@ -1050,24 +1171,46 @@ def scatter(tensor, tensor_list=None, src=0, group=None, sync_op=True): use_calc_stream = sync_op if _non_static_mode(): - return _legacy_C_ops.c_scatter(temp, tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', ring_id, - 'nranks', nranks, 'root', gsrc) + return _legacy_C_ops.c_scatter( + temp, + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'nranks', + nranks, + 'root', + gsrc, + ) op_type = 'c_scatter' - check_variable_and_dtype(tensor, 'tensor', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'int8', 'uint8', - 'bool' - ], 'scatter') + check_variable_and_dtype( + tensor, + 'tensor', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'int8', + 'uint8', + 'bool', + ], + 'scatter', + ) helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [temp]}, - outputs={'Out': [tensor]}, - attrs={ - 'ring_id': ring_id, - 'root': gsrc, - 'use_calc_stream': use_calc_stream, - 'nranks': nranks, - }) + helper.append_op( + type=op_type, + inputs={'X': [temp]}, + outputs={'Out': [tensor]}, + attrs={ + 'ring_id': ring_id, + 'root': gsrc, + 'use_calc_stream': use_calc_stream, + 'nranks': nranks, + }, + ) def alltoall(in_tensor_list, out_tensor_list, group=None, sync_op=True): @@ -1119,7 +1262,7 @@ def alltoall(in_tensor_list, out_tensor_list, group=None, sync_op=True): if in_dygraph_mode(): group = _get_default_group() if group is None else group backend = _group_map_backend[group] - assert backend != 'gloo', ("backend gloo is not supported yet") + assert backend != 'gloo', "backend gloo is not supported yet" else: ring_id = 0 if group is None else group.id @@ -1140,44 +1283,56 @@ def alltoall(in_tensor_list, out_tensor_list, group=None, sync_op=True): use_calc_stream = sync_op if _non_static_mode(): - out = _legacy_C_ops.alltoall(temp, 'use_calc_stream', use_calc_stream, - 'ring_id', ring_id) + out = _legacy_C_ops.alltoall( + temp, 'use_calc_stream', use_calc_stream, 'ring_id', ring_id + ) else: op_type = 'alltoall' helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference( - dtype=in_tensor_list[0].dtype) + dtype=in_tensor_list[0].dtype + ) if not isinstance(in_tensor_list, list): - raise ValueError("The type of 'in_tensor_list' for all_to_all " - "should be list.") + raise ValueError( + "The type of 'in_tensor_list' for all_to_all " "should be list." + ) for elem in in_tensor_list: check_variable_and_dtype( - elem, 'in_tensor_list', + elem, + 'in_tensor_list', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'all_to_all') + 'all_to_all', + ) if not isinstance(out_tensor_list, list): - raise ValueError("The type of 'out_tensor_list' for all_to_all " - "should be list.") + raise ValueError( + "The type of 'out_tensor_list' for all_to_all " + "should be list." + ) if len(out_tensor_list) != 0: - raise ValueError("The 'out_tensor_list' for all_to_all " - "must be an empty list.") - helper.append_op(type=op_type, - inputs={'X': [temp]}, - outputs={'Out': [out]}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream, - }) + raise ValueError( + "The 'out_tensor_list' for all_to_all " "must be an empty list." + ) + helper.append_op( + type=op_type, + inputs={'X': [temp]}, + outputs={'Out': [out]}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + }, + ) out_tensor_list.extend(paddle.split(out, nranks, 0)) -def alltoall_single(in_tensor, - out_tensor, - in_split_sizes=None, - out_split_sizes=None, - group=None, - sync_op=True): +def alltoall_single( + in_tensor, + out_tensor, + in_split_sizes=None, + out_split_sizes=None, + group=None, + sync_op=True, +): """ Scatter a single input tensor to all participators and gather the received tensors in out_tensor. @@ -1250,13 +1405,14 @@ def alltoall_single(in_tensor, group = _get_default_group() if group is None else group backend = _group_map_backend[group] - assert backend != 'gloo', ("backend gloo is not supported yet") + assert backend != 'gloo', "backend gloo is not supported yet" in_split_sizes = [] if in_split_sizes is None else in_split_sizes out_split_sizes = [] if out_split_sizes is None else out_split_sizes - task = group.process_group.alltoall_single(in_tensor, out_tensor, - in_split_sizes, out_split_sizes) + task = group.process_group.alltoall_single( + in_tensor, out_tensor, in_split_sizes, out_split_sizes + ) if sync_op: task.wait() return @@ -1305,7 +1461,7 @@ def send(tensor, dst=0, group=None, sync_op=True): if in_dygraph_mode(): group = _get_default_group() if group is None else group backend = _group_map_backend[group] - assert backend != 'gloo', ("backend gloo is not supported yet") + assert backend != 'gloo', "backend gloo is not supported yet" task = group.process_group.send(tensor, dst) if sync_op: task.wait() @@ -1317,21 +1473,33 @@ def send(tensor, dst=0, group=None, sync_op=True): ring_id = 0 if group is None else group.id if _non_static_mode(): - return _legacy_C_ops.send_v2(tensor, 'use_calc_stream', use_calc_stream, - 'ring_id', ring_id, 'peer', dst) + return _legacy_C_ops.send_v2( + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'peer', + dst, + ) op_type = 'send_v2' check_variable_and_dtype( - tensor, 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'send') + tensor, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'send', + ) helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [tensor]}, - attrs={ - 'ring_id': ring_id, - 'peer': dst, - 'use_calc_stream': use_calc_stream, - }) + helper.append_op( + type=op_type, + inputs={'X': [tensor]}, + attrs={ + 'ring_id': ring_id, + 'peer': dst, + 'use_calc_stream': use_calc_stream, + }, + ) def recv(tensor, src=0, group=None, sync_op=True): @@ -1372,7 +1540,7 @@ def recv(tensor, src=0, group=None, sync_op=True): if in_dygraph_mode(): group = _get_default_group() if group is None else group backend = _group_map_backend[group] - assert backend != 'gloo', ("backend gloo is not supported yet") + assert backend != 'gloo', "backend gloo is not supported yet" task = group.process_group.recv(tensor, src) if sync_op: task.wait() @@ -1384,37 +1552,58 @@ def recv(tensor, src=0, group=None, sync_op=True): ring_id = 0 if group is None else group.id if _non_static_mode(): - return _legacy_C_ops.recv_v2(tensor, 'use_calc_stream', use_calc_stream, - 'ring_id', ring_id, 'peer', src, 'dtype', - tensor.dtype, 'out_shape', tensor.shape) + return _legacy_C_ops.recv_v2( + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'peer', + src, + 'dtype', + tensor.dtype, + 'out_shape', + tensor.shape, + ) op_type = 'recv_v2' check_variable_and_dtype( - tensor, 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'recv') + tensor, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'recv', + ) helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - outputs={'Out': [tensor]}, - attrs={ - 'ring_id': ring_id, - 'peer': src, - 'out_shape': tensor.shape, - 'dtype': tensor.dtype, - 'use_calc_stream': use_calc_stream, - }) + helper.append_op( + type=op_type, + outputs={'Out': [tensor]}, + attrs={ + 'ring_id': ring_id, + 'peer': src, + 'out_shape': tensor.shape, + 'dtype': tensor.dtype, + 'use_calc_stream': use_calc_stream, + }, + ) def _check_single_tensor(tensor, tensor_name): if not isinstance(tensor, (core.eager.Tensor, paddle.Tensor)): - raise RuntimeError("Invalid function argument. Expected parameter {}" - "to be of type paddle.Tensor, but it's {}".format( - tensor_name, type(tensor))) + raise RuntimeError( + "Invalid function argument. Expected parameter {}" + "to be of type paddle.Tensor, but it's {}".format( + tensor_name, type(tensor) + ) + ) def _check_tensor_list(tensor_list, tensor_name): - if not isinstance(tensor_list, list) or \ - not all(isinstance(t, (core.eager.Tensor, paddle.Tensor)) for t in tensor_list): - raise RuntimeError("Invalid function argument. Expected parameter {}" - "to be of type paddle.Tensor".format(tensor_name)) + if not isinstance(tensor_list, list) or not all( + isinstance(t, (core.eager.Tensor, paddle.Tensor)) for t in tensor_list + ): + raise RuntimeError( + "Invalid function argument. Expected parameter {}" + "to be of type paddle.Tensor".format(tensor_name) + ) def isend(tensor, dst, group=None): @@ -1459,9 +1648,9 @@ def isend(tensor, dst, group=None): if in_dygraph_mode(): group = _get_default_group() if group is None else group backend = _group_map_backend[group] - assert backend != 'gloo', ("backend gloo is not supported yet") + assert backend != 'gloo', "backend gloo is not supported yet" group_dst_rank = group.get_group_rank(dst) - assert group_dst_rank >= 0, ("dst rank out of group, need global rank") + assert group_dst_rank >= 0, "dst rank out of group, need global rank" return group.process_group.send(tensor, group_dst_rank) else: raise RuntimeError("Only support eager dygraph mode.") @@ -1508,9 +1697,9 @@ def irecv(tensor, src=None, group=None): if in_dygraph_mode(): group = _get_default_group() if group is None else group backend = _group_map_backend[group] - assert backend != 'gloo', ("backend gloo is not supported yet") + assert backend != 'gloo', "backend gloo is not supported yet" group_src_rank = group.get_group_rank(src) - assert group_src_rank >= 0, ("src rank out of group, need global rank") + assert group_src_rank >= 0, "src rank out of group, need global rank" return group.process_group.recv(tensor, group_src_rank) else: raise RuntimeError("Only support eager dygraph mode.") @@ -1536,9 +1725,11 @@ class P2POp(object): def __init__(self, op, tensor, peer, group=None): if op not in [isend, irecv]: - raise RuntimeError("Invalid ``op`` function. Expected ``op`` " - "to be of type ``paddle.distributed.isend`` or " - "``paddle.distributed.irecv``.") + raise RuntimeError( + "Invalid ``op`` function. Expected ``op`` " + "to be of type ``paddle.distributed.isend`` or " + "``paddle.distributed.irecv``." + ) _check_single_tensor(tensor, "tensor") self.op = op @@ -1564,13 +1755,17 @@ def _check_p2p_op_list(p2p_op_list): all ops use the same backend. """ if not isinstance(p2p_op_list, list) or not all( - isinstance(p2p_op, P2POp) for p2p_op in p2p_op_list): - raise RuntimeError("Invalid ``p2p_op_list``. Each op is expected to " - "to be of type ``paddle.distributed.P2POp``.") + isinstance(p2p_op, P2POp) for p2p_op in p2p_op_list + ): + raise RuntimeError( + "Invalid ``p2p_op_list``. Each op is expected to " + "to be of type ``paddle.distributed.P2POp``." + ) backend = _group_map_backend[p2p_op_list[0].group] - if not all(backend == _group_map_backend[p2p_op.group] - for p2p_op in p2p_op_list): + if not all( + backend == _group_map_backend[p2p_op.group] for p2p_op in p2p_op_list + ): raise RuntimeError("All groups need to use the same backend.") @@ -1647,11 +1842,9 @@ def batch_isend_irecv(p2p_op_list): raise RuntimeError("Don't support static graph mode currently.") -def reduce_scatter(tensor, - tensor_list, - op=ReduceOp.SUM, - group=None, - sync_op=True): +def reduce_scatter( + tensor, tensor_list, op=ReduceOp.SUM, group=None, sync_op=True +): """ Reduces, then scatters a list of tensors to all processes in a group @@ -1702,7 +1895,7 @@ def reduce_scatter(tensor, op_type = _get_reduce_op(op, "reduce_scatter") group = _get_default_group() if group is None else group backend = _group_map_backend[group] - assert backend != 'gloo', ("backend gloo is not supported yet") + assert backend != 'gloo', "backend gloo is not supported yet" temp = paddle.concat(tensor_list, axis=0) task = group.process_group._reduce_scatter_base(tensor, temp, op_type) @@ -1715,11 +1908,9 @@ def reduce_scatter(tensor, raise RuntimeError("Don't support static graph mode currently.") -def _reduce_scatter_base(output, - input, - op=ReduceOp.SUM, - group=None, - sync_op=True): +def _reduce_scatter_base( + output, input, op=ReduceOp.SUM, group=None, sync_op=True +): """ Reduces, then scatters a flattened tensor to all processes in a group. diff --git a/python/paddle/distributed/communication/all_reduce.py b/python/paddle/distributed/communication/all_reduce.py index 737e0cbbfb56c..7a09d779e8fe7 100644 --- a/python/paddle/distributed/communication/all_reduce.py +++ b/python/paddle/distributed/communication/all_reduce.py @@ -58,30 +58,28 @@ def all_reduce(tensor, op=ReduceOp.SUM, group=None, sync_op=True): # [[5, 7, 9], [5, 7, 9]] (2 GPUs) """ if not framework._in_legacy_dygraph(): - return stream.all_reduce(tensor, - op=op, - group=group, - sync_op=sync_op, - use_calc_stream=False) + return stream.all_reduce( + tensor, op=op, group=group, sync_op=sync_op, use_calc_stream=False + ) # code below will be removed after we remove the old dygraph use_calc_stream = sync_op ring_id = 0 if group is None else group.id if op == ReduceOp.SUM: - return paddle._legacy_C_ops.c_allreduce_sum_(tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id) + return paddle._legacy_C_ops.c_allreduce_sum_( + tensor, 'use_calc_stream', use_calc_stream, 'ring_id', ring_id + ) elif op == ReduceOp.MAX: - return paddle._legacy_C_ops.c_allreduce_max_(tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id) + return paddle._legacy_C_ops.c_allreduce_max_( + tensor, 'use_calc_stream', use_calc_stream, 'ring_id', ring_id + ) elif op == ReduceOp.MIN: - return paddle._legacy_C_ops.c_allreduce_min_(tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id) + return paddle._legacy_C_ops.c_allreduce_min_( + tensor, 'use_calc_stream', use_calc_stream, 'ring_id', ring_id + ) elif op == ReduceOp.PROD: - return paddle._legacy_C_ops.c_allreduce_prod_(tensor, 'use_calc_stream', - use_calc_stream, - 'ring_id', ring_id) + return paddle._legacy_C_ops.c_allreduce_prod_( + tensor, 'use_calc_stream', use_calc_stream, 'ring_id', ring_id + ) else: raise ValueError("Unknown parameter: {}.".format(op)) diff --git a/python/paddle/distributed/communication/group.py b/python/paddle/distributed/communication/group.py index 6b4e545b245d1..60f1264fe8088 100644 --- a/python/paddle/distributed/communication/group.py +++ b/python/paddle/distributed/communication/group.py @@ -13,7 +13,7 @@ # limitations under the License. -class Group(): +class Group: """ The abstract representation of group. """ @@ -69,14 +69,15 @@ def get_group_rank(self, rank): def __repr__(self): debug_str = "rank: {}, nranks: {}, id: {}, ranks: ".format( - self.rank, self.nranks, self.id) + self.rank, self.nranks, self.id + ) debug_str += ", ".join(map(str, self.ranks)) debug_str += "; name: " debug_str += self.name if self.name else "None" return debug_str -class _GroupManager(): +class _GroupManager: global_group_id = 0 group_map_by_id = {} @@ -89,6 +90,7 @@ def _get_global_group(): def _add_new_group(group): if group.id in _GroupManager.group_map_by_id: - raise RuntimeError("The group with id {} already exist.".format( - group.id)) + raise RuntimeError( + "The group with id {} already exist.".format(group.id) + ) _GroupManager.group_map_by_id[group.id] = group diff --git a/python/paddle/distributed/communication/reduce.py b/python/paddle/distributed/communication/reduce.py index 5caa5bebedfd8..8628e83b626ef 100644 --- a/python/paddle/distributed/communication/reduce.py +++ b/python/paddle/distributed/communication/reduce.py @@ -46,6 +46,7 @@ class ReduceOp: print(data) # [[5, 7, 9], [5, 7, 9]] (2 GPUs) """ + SUM = 0 MAX = 1 MIN = 2 diff --git a/python/paddle/distributed/communication/stream/__init__.py b/python/paddle/distributed/communication/stream/__init__.py index 43952ce5541a3..63ec858557b3b 100644 --- a/python/paddle/distributed/communication/stream/__init__.py +++ b/python/paddle/distributed/communication/stream/__init__.py @@ -24,6 +24,14 @@ from .send import send __all__ = [ - "all_gather", "all_reduce", "alltoall", "alltoall_single", "broadcast", - "reduce", "reduce_scatter", "recv", "scatter", "send" + "all_gather", + "all_reduce", + "alltoall", + "alltoall_single", + "broadcast", + "reduce", + "reduce_scatter", + "recv", + "scatter", + "send", ] diff --git a/python/paddle/distributed/communication/stream/all_gather.py b/python/paddle/distributed/communication/stream/all_gather.py index 9eb961cda171d..cdd7f98554b3a 100644 --- a/python/paddle/distributed/communication/stream/all_gather.py +++ b/python/paddle/distributed/communication/stream/all_gather.py @@ -27,33 +27,39 @@ def _check_tensor_shape(tensor, shape, nranks=1): def _check_tensor_list_shape(tensor_list, shape, nranks=1): if len(tensor_list) != nranks: raise RuntimeError( - 'The tensor_list for all_gather is not correctly-sized.') + 'The tensor_list for all_gather is not correctly-sized.' + ) for tensor in tensor_list: if tensor.shape != shape: raise RuntimeError( - 'The tensor_list for all_gather is not correctly-sized.') + 'The tensor_list for all_gather is not correctly-sized.' + ) -def _all_gather_into_tensor_in_dygraph(out_tensor, in_tensor, group, sync_op, - use_calc_stream): +def _all_gather_into_tensor_in_dygraph( + out_tensor, in_tensor, group, sync_op, use_calc_stream +): group = collective._get_default_group() if group is None else group _check_tensor_shape(out_tensor, in_tensor.shape, group.nranks) if use_calc_stream: return group.process_group.allgather_into_tensor_on_calc_stream( - in_tensor, out_tensor) + in_tensor, out_tensor + ) - task = group.process_group.allgather_into_tensor(in_tensor, out_tensor, - sync_op) + task = group.process_group.allgather_into_tensor( + in_tensor, out_tensor, sync_op + ) if sync_op: task.wait() return task -def _all_gather_in_dygraph(tensor_list, tensor, group, sync_op, - use_calc_stream): +def _all_gather_in_dygraph( + tensor_list, tensor, group, sync_op, use_calc_stream +): group = collective._get_default_group() if group is None else group if len(tensor_list) == 0: @@ -71,11 +77,13 @@ def _all_gather_in_dygraph(tensor_list, tensor, group, sync_op, return task -def all_gather(tensor_or_tensor_list, - tensor, - group=None, - sync_op=True, - use_calc_stream=False): +def all_gather( + tensor_or_tensor_list, + tensor, + group=None, + sync_op=True, + use_calc_stream=False, +): """ Gather tensors across devices to a correctly-sized tensor or a tensor list. @@ -122,16 +130,18 @@ def all_gather(tensor_or_tensor_list, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if framework.in_dygraph_mode(): if paddle.is_tensor(tensor_or_tensor_list): - return _all_gather_into_tensor_in_dygraph(tensor_or_tensor_list, - tensor, group, sync_op, - use_calc_stream) + return _all_gather_into_tensor_in_dygraph( + tensor_or_tensor_list, tensor, group, sync_op, use_calc_stream + ) else: - return _all_gather_in_dygraph(tensor_or_tensor_list, tensor, group, - sync_op, use_calc_stream) + return _all_gather_in_dygraph( + tensor_or_tensor_list, tensor, group, sync_op, use_calc_stream + ) raise RuntimeError( "paddle.distributed.stream.all_gather is only supported in dygraph mode now." diff --git a/python/paddle/distributed/communication/stream/all_reduce.py b/python/paddle/distributed/communication/stream/all_reduce.py index 0ba161a078ab8..97eda9d0f9b44 100644 --- a/python/paddle/distributed/communication/stream/all_reduce.py +++ b/python/paddle/distributed/communication/stream/all_reduce.py @@ -34,10 +34,21 @@ def _all_reduce_in_dygraph(tensor, op, group, sync_op, use_calc_stream): def _all_reduce_in_static_mode(tensor, op, group, sync_op, use_calc_stream): - data_feeder.check_variable_and_dtype(tensor, 'tensor', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'int8', 'uint8', - 'bool' - ], 'all_reduce') + data_feeder.check_variable_and_dtype( + tensor, + 'tensor', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'int8', + 'uint8', + 'bool', + ], + 'all_reduce', + ) op_type = _get_reduce_op(op, "all_reduce") ring_id = 0 if group is None else group.id @@ -48,22 +59,19 @@ def _all_reduce_in_static_mode(tensor, op, group, sync_op, use_calc_stream): # TODO: Support task and use task.wait in static mode # Use use_calc_stream rather than sync_op helper = layer_helper.LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [tensor]}, - outputs={'Out': [tensor]}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': sync_op - }) + helper.append_op( + type=op_type, + inputs={'X': [tensor]}, + outputs={'Out': [tensor]}, + attrs={'ring_id': ring_id, 'use_calc_stream': sync_op}, + ) return None -def all_reduce(tensor, - op=ReduceOp.SUM, - group=None, - sync_op=True, - use_calc_stream=False): +def all_reduce( + tensor, op=ReduceOp.SUM, group=None, sync_op=True, use_calc_stream=False +): """ Perform specific reduction (for example, sum, max) on inputs across devices. @@ -106,11 +114,14 @@ def all_reduce(tensor, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if framework.in_dygraph_mode(): - return _all_reduce_in_dygraph(tensor, op, group, sync_op, - use_calc_stream) + return _all_reduce_in_dygraph( + tensor, op, group, sync_op, use_calc_stream + ) else: - return _all_reduce_in_static_mode(tensor, op, group, sync_op, - use_calc_stream) + return _all_reduce_in_static_mode( + tensor, op, group, sync_op, use_calc_stream + ) diff --git a/python/paddle/distributed/communication/stream/alltoall.py b/python/paddle/distributed/communication/stream/alltoall.py index b216906d04568..5faf106263125 100644 --- a/python/paddle/distributed/communication/stream/alltoall.py +++ b/python/paddle/distributed/communication/stream/alltoall.py @@ -25,22 +25,26 @@ def _check_tensor_shape(tensor, shape, nranks=1): def _check_tensor_list_shape(tensor_list, shape, nranks=1): if len(tensor_list) != nranks: raise RuntimeError( - 'The tensor_list for alltoall is not correctly-sized.') + 'The tensor_list for alltoall is not correctly-sized.' + ) for tensor in tensor_list: if tensor.shape != shape: raise RuntimeError( - 'The tensor_list for alltoall is not correctly-sized.') + 'The tensor_list for alltoall is not correctly-sized.' + ) -def _alltoall_tensor_in_dygraph(out_tensor, in_tensor, group, sync_op, - use_calc_stream): +def _alltoall_tensor_in_dygraph( + out_tensor, in_tensor, group, sync_op, use_calc_stream +): group = collective._get_default_group() if group is None else group _check_tensor_shape(out_tensor, in_tensor.shape, group.nranks) if use_calc_stream: return group.process_group.alltoall_tensor_on_calc_stream( - in_tensor, out_tensor) + in_tensor, out_tensor + ) task = group.process_group.alltoall_tensor(in_tensor, out_tensor, sync_op) if sync_op: @@ -49,8 +53,9 @@ def _alltoall_tensor_in_dygraph(out_tensor, in_tensor, group, sync_op, return task -def _alltoall_in_dygraph(out_tensor_list, in_tensor_list, group, sync_op, - use_calc_stream): +def _alltoall_in_dygraph( + out_tensor_list, in_tensor_list, group, sync_op, use_calc_stream +): group = collective._get_default_group() if group is None else group if len(in_tensor_list) == 0: @@ -61,26 +66,31 @@ def _alltoall_in_dygraph(out_tensor_list, in_tensor_list, group, sync_op, paddle.empty_like(tensor) for tensor in in_tensor_list ] else: - _check_tensor_list_shape(out_tensor_list, in_tensor_list[0].shape, - group.nranks) + _check_tensor_list_shape( + out_tensor_list, in_tensor_list[0].shape, group.nranks + ) if use_calc_stream: return group.process_group.alltoall_on_calc_stream( - in_tensor_list, out_tensor_list) + in_tensor_list, out_tensor_list + ) - task = group.process_group.alltoall(in_tensor_list, out_tensor_list, - sync_op) + task = group.process_group.alltoall( + in_tensor_list, out_tensor_list, sync_op + ) if sync_op: task.wait() return task -def alltoall(out_tensor_or_tensor_list, - in_tensor_or_tensor_list, - group=None, - sync_op=True, - use_calc_stream=False): +def alltoall( + out_tensor_or_tensor_list, + in_tensor_or_tensor_list, + group=None, + sync_op=True, + use_calc_stream=False, +): """ Scatter a tensor (or a tensor list) across devices and gather outputs to another tensor (or a tensor list, respectively). @@ -130,7 +140,8 @@ def alltoall(out_tensor_or_tensor_list, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if out_tensor_or_tensor_list is None: raise RuntimeError("The output should be specified.") @@ -141,16 +152,25 @@ def alltoall(out_tensor_or_tensor_list, out_is_tensor = paddle.is_tensor(out_tensor_or_tensor_list) in_is_tensor = paddle.is_tensor(in_tensor_or_tensor_list) if out_is_tensor and in_is_tensor: - return _alltoall_tensor_in_dygraph(out_tensor_or_tensor_list, - in_tensor_or_tensor_list, group, - sync_op, use_calc_stream) + return _alltoall_tensor_in_dygraph( + out_tensor_or_tensor_list, + in_tensor_or_tensor_list, + group, + sync_op, + use_calc_stream, + ) elif not out_is_tensor and not in_is_tensor: - return _alltoall_in_dygraph(out_tensor_or_tensor_list, - in_tensor_or_tensor_list, group, - sync_op, use_calc_stream) + return _alltoall_in_dygraph( + out_tensor_or_tensor_list, + in_tensor_or_tensor_list, + group, + sync_op, + use_calc_stream, + ) else: raise RuntimeError( - "The output and input should be both tensor or tensor list.") + "The output and input should be both tensor or tensor list." + ) raise RuntimeError( "paddle.distributed.stream.alltoall is only supported in dygraph mode now." diff --git a/python/paddle/distributed/communication/stream/alltoall_single.py b/python/paddle/distributed/communication/stream/alltoall_single.py index b2187cc06e343..75eb289032636 100644 --- a/python/paddle/distributed/communication/stream/alltoall_single.py +++ b/python/paddle/distributed/communication/stream/alltoall_single.py @@ -16,9 +16,15 @@ from paddle.distributed import collective -def _alltoall_single_in_dygraph(out_tensor, in_tensor, out_split_sizes, - in_split_sizes, group, sync_op, - use_calc_stream): +def _alltoall_single_in_dygraph( + out_tensor, + in_tensor, + out_split_sizes, + in_split_sizes, + group, + sync_op, + use_calc_stream, +): group = collective._get_default_group() if group is None else group if out_split_sizes is None: @@ -28,24 +34,27 @@ def _alltoall_single_in_dygraph(out_tensor, in_tensor, out_split_sizes, if use_calc_stream: return group.process_group.alltoall_single_on_calc_stream( - in_tensor, out_tensor, in_split_sizes, out_split_sizes) + in_tensor, out_tensor, in_split_sizes, out_split_sizes + ) - task = group.process_group.alltoall_single(in_tensor, out_tensor, - in_split_sizes, out_split_sizes, - sync_op) + task = group.process_group.alltoall_single( + in_tensor, out_tensor, in_split_sizes, out_split_sizes, sync_op + ) if sync_op: task.wait() return task -def alltoall_single(out_tensor, - in_tensor, - out_split_sizes=None, - in_split_sizes=None, - group=None, - sync_op=True, - use_calc_stream=False): +def alltoall_single( + out_tensor, + in_tensor, + out_split_sizes=None, + in_split_sizes=None, + group=None, + sync_op=True, + use_calc_stream=False, +): """ Split and Scatter the splitted input tensor to the out tensor across devices. @@ -116,12 +125,19 @@ def alltoall_single(out_tensor, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if framework.in_dygraph_mode(): - return _alltoall_single_in_dygraph(out_tensor, in_tensor, - out_split_sizes, in_split_sizes, - group, sync_op, use_calc_stream) + return _alltoall_single_in_dygraph( + out_tensor, + in_tensor, + out_split_sizes, + in_split_sizes, + group, + sync_op, + use_calc_stream, + ) raise RuntimeError( "paddle.distributed.stream.alltoall_single is only supported in dygraph mode now." diff --git a/python/paddle/distributed/communication/stream/broadcast.py b/python/paddle/distributed/communication/stream/broadcast.py index 06bde316937a9..3672b02811e6e 100644 --- a/python/paddle/distributed/communication/stream/broadcast.py +++ b/python/paddle/distributed/communication/stream/broadcast.py @@ -72,11 +72,13 @@ def broadcast(tensor, src=0, group=None, sync_op=True, use_calc_stream=False): if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be True in sync op behavior.") + "use_calc_stream can only be True in sync op behavior." + ) if framework.in_dygraph_mode(): - return _broadcast_in_dygraph(tensor, src, group, sync_op, - use_calc_stream) + return _broadcast_in_dygraph( + tensor, src, group, sync_op, use_calc_stream + ) raise RuntimeError( "paddle.distributed.stream.broadcast is only supported in dygraph mode now." diff --git a/python/paddle/distributed/communication/stream/recv.py b/python/paddle/distributed/communication/stream/recv.py index 25a8173788473..2658379a412a7 100644 --- a/python/paddle/distributed/communication/stream/recv.py +++ b/python/paddle/distributed/communication/stream/recv.py @@ -73,10 +73,12 @@ def recv(tensor, src=0, group=None, sync_op=True, use_calc_stream=False): if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be True in sync op behavior.") + "use_calc_stream can only be True in sync op behavior." + ) if framework.in_dygraph_mode(): return _recv_in_dygraph(tensor, src, group, sync_op, use_calc_stream) raise RuntimeError( - "paddle.distributed.stream.recv is only supported in dygraph mode now.") + "paddle.distributed.stream.recv is only supported in dygraph mode now." + ) diff --git a/python/paddle/distributed/communication/stream/reduce.py b/python/paddle/distributed/communication/stream/reduce.py index b0f7f5c884743..d7f0fc6b2b778 100644 --- a/python/paddle/distributed/communication/stream/reduce.py +++ b/python/paddle/distributed/communication/stream/reduce.py @@ -30,12 +30,14 @@ def _reduce_in_dygraph(tensor, dst, op, group, sync_op, use_calc_stream): return task -def reduce(tensor, - dst=0, - op=ReduceOp.SUM, - group=None, - sync_op=True, - use_calc_stream=False): +def reduce( + tensor, + dst=0, + op=ReduceOp.SUM, + group=None, + sync_op=True, + use_calc_stream=False, +): """ Perform specific reduction (for example, sum, max) on a tensor across devices and send to the destintion device. @@ -82,11 +84,13 @@ def reduce(tensor, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if framework.in_dygraph_mode(): - return _reduce_in_dygraph(tensor, dst, op, group, sync_op, - use_calc_stream) + return _reduce_in_dygraph( + tensor, dst, op, group, sync_op, use_calc_stream + ) raise RuntimeError( "paddle.distributed.stream.reduce is only supported in dygraph mode now." diff --git a/python/paddle/distributed/communication/stream/reduce_scatter.py b/python/paddle/distributed/communication/stream/reduce_scatter.py index 68ba8e18e404c..ef375fa5be412 100644 --- a/python/paddle/distributed/communication/stream/reduce_scatter.py +++ b/python/paddle/distributed/communication/stream/reduce_scatter.py @@ -23,26 +23,31 @@ def _check_tensor_shape(tensor, shape, nranks=1): expect_shape[0] //= nranks if list(tensor.shape) != expect_shape: raise RuntimeError( - "The in_tensor for reduce_scatter is not correctly-sized.") + "The in_tensor for reduce_scatter is not correctly-sized." + ) def _check_tensor_list_shape(tensor_list, shape, nranks=1): if len(tensor_list) != nranks: raise RuntimeError( - "The tensor_list for reduce_scatter is not correctly-sized.") + "The tensor_list for reduce_scatter is not correctly-sized." + ) for tensor in tensor_list: if tensor.shape != shape: raise RuntimeError( - "The tensor_list for reduce_scatter is not correctly-sized.") - - -def _reduce_scatter_tensor_in_dygraph(out_tensor, - in_tensor, - op, - group, - sync_op, - use_calc_stream, - caller="reduce_scatter"): + "The tensor_list for reduce_scatter is not correctly-sized." + ) + + +def _reduce_scatter_tensor_in_dygraph( + out_tensor, + in_tensor, + op, + group, + sync_op, + use_calc_stream, + caller="reduce_scatter", +): op_type = _get_reduce_op(op, caller) group = _get_global_group() if group is None else group @@ -50,18 +55,21 @@ def _reduce_scatter_tensor_in_dygraph(out_tensor, if use_calc_stream: return group.process_group.reduce_scatter_tensor_on_calc_stream( - in_tensor, out_tensor, op_type) + in_tensor, out_tensor, op_type + ) - task = group.process_group.reduce_scatter_tensor(in_tensor, out_tensor, - op_type, sync_op) + task = group.process_group.reduce_scatter_tensor( + in_tensor, out_tensor, op_type, sync_op + ) if sync_op: task.wait() return task -def _reduce_scatter_in_dygraph(tensor, tensor_list, op, group, sync_op, - use_calc_stream): +def _reduce_scatter_in_dygraph( + tensor, tensor_list, op, group, sync_op, use_calc_stream +): op_type = _get_reduce_op(op, "reduce_scatter") group = _get_global_group() if group is None else group @@ -69,22 +77,26 @@ def _reduce_scatter_in_dygraph(tensor, tensor_list, op, group, sync_op, if use_calc_stream: return group.process_group.reduce_scatter_on_calc_stream( - tensor_list, tensor, op_type) + tensor_list, tensor, op_type + ) - task = group.process_group.reduce_scatter(tensor_list, tensor, op_type, - sync_op) + task = group.process_group.reduce_scatter( + tensor_list, tensor, op_type, sync_op + ) if sync_op: task.wait() return task -def reduce_scatter(tensor, - tensor_or_tensor_list, - op=ReduceOp.SUM, - group=None, - sync_op=True, - use_calc_stream=False): +def reduce_scatter( + tensor, + tensor_or_tensor_list, + op=ReduceOp.SUM, + group=None, + sync_op=True, + use_calc_stream=False, +): """ Reduce, then scatter a tensor (or a tensor list) across devices. @@ -132,29 +144,42 @@ def reduce_scatter(tensor, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if framework.in_dygraph_mode(): if paddle.is_tensor(tensor_or_tensor_list): - return _reduce_scatter_tensor_in_dygraph(tensor, - tensor_or_tensor_list, op, - group, sync_op, - use_calc_stream) + return _reduce_scatter_tensor_in_dygraph( + tensor, + tensor_or_tensor_list, + op, + group, + sync_op, + use_calc_stream, + ) else: - return _reduce_scatter_in_dygraph(tensor, tensor_or_tensor_list, op, - group, sync_op, use_calc_stream) + return _reduce_scatter_in_dygraph( + tensor, + tensor_or_tensor_list, + op, + group, + sync_op, + use_calc_stream, + ) raise RuntimeError( "paddle.distributed.stream.reduce_scatter is only supported in dygraph mode now." ) -def _reduce_scatter_base(out_tensor, - in_tensor, - op=ReduceOp.SUM, - group=None, - sync_op=True, - use_calc_stream=False): +def _reduce_scatter_base( + out_tensor, + in_tensor, + op=ReduceOp.SUM, + group=None, + sync_op=True, + use_calc_stream=False, +): """ Reduce, then scatter a flattened tensor across devices. @@ -202,13 +227,19 @@ def _reduce_scatter_base(out_tensor, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if framework.in_dygraph_mode(): - return _reduce_scatter_tensor_in_dygraph(out_tensor, in_tensor, op, - group, sync_op, - use_calc_stream, - "_reduce_scatter_base") + return _reduce_scatter_tensor_in_dygraph( + out_tensor, + in_tensor, + op, + group, + sync_op, + use_calc_stream, + "_reduce_scatter_base", + ) raise RuntimeError( "paddle.distributed.stream._reduce_scatter_base is only supported in dygraph mode now." diff --git a/python/paddle/distributed/communication/stream/scatter.py b/python/paddle/distributed/communication/stream/scatter.py index 9b0fcaf4b086a..199885c1fde91 100644 --- a/python/paddle/distributed/communication/stream/scatter.py +++ b/python/paddle/distributed/communication/stream/scatter.py @@ -28,15 +28,18 @@ def _check_tensor_shape(tensor, shape, nranks=1): def _check_tensor_list_shape(tensor_list, shape, nranks=1): if len(tensor_list) != nranks: raise RuntimeError( - "The tensor_list for scatter is not correctly-sized.") + "The tensor_list for scatter is not correctly-sized." + ) for tensor in tensor_list: if tensor.shape != shape: raise RuntimeError( - "The tensor_list for scatter is not correctly-sized.") + "The tensor_list for scatter is not correctly-sized." + ) -def _scatter_tensor_in_dygraph(out_tensor, in_tensor, src, group, sync_op, - use_calc_stream): +def _scatter_tensor_in_dygraph( + out_tensor, in_tensor, src, group, sync_op, use_calc_stream +): group = collective._get_default_group() if group is None else group src_rank = group.get_group_rank(src) @@ -50,18 +53,21 @@ def _scatter_tensor_in_dygraph(out_tensor, in_tensor, src, group, sync_op, if use_calc_stream: return group.process_group.scatter_tensor_on_calc_stream( - in_tensor, out_tensor, src) + in_tensor, out_tensor, src + ) - task = group.process_group.scatter_tensor(in_tensor, out_tensor, src, - sync_op) + task = group.process_group.scatter_tensor( + in_tensor, out_tensor, src, sync_op + ) if sync_op: task.wait() return task -def _scatter_in_dygraph(tensor, tensor_list, src, group, sync_op, - use_calc_stream): +def _scatter_in_dygraph( + tensor, tensor_list, src, group, sync_op, use_calc_stream +): group = collective._get_default_group() if group is None else group src_rank = group.get_group_rank(src) @@ -73,14 +79,16 @@ def _scatter_in_dygraph(tensor, tensor_list, src, group, sync_op, if rank == src_rank: if len(tensor_list) == 0: raise RuntimeError( - "The tensor_list should not be empty on src rank.") + "The tensor_list should not be empty on src rank." + ) _check_tensor_list_shape(tensor_list, tensor.shape, nranks) else: tensor_list = [tensor for _ in range(nranks)] if use_calc_stream: return group.process_group.scatter_on_calc_stream( - tensor_list, tensor, src) + tensor_list, tensor, src + ) task = group.process_group.scatter(tensor_list, tensor, src, sync_op) if sync_op: @@ -89,12 +97,14 @@ def _scatter_in_dygraph(tensor, tensor_list, src, group, sync_op, return task -def scatter(tensor, - tensor_or_tensor_list=None, - src=0, - group=None, - sync_op=True, - use_calc_stream=False): +def scatter( + tensor, + tensor_or_tensor_list=None, + src=0, + group=None, + sync_op=True, + use_calc_stream=False, +): """ Scatter a tensor (or a tensor list) across devices. @@ -143,19 +153,31 @@ def scatter(tensor, if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be true in sync op behavior.") + "use_calc_stream can only be true in sync op behavior." + ) if tensor_or_tensor_list is None: raise RuntimeError("The input should be specified.") if framework.in_dygraph_mode(): if paddle.is_tensor(tensor_or_tensor_list): - return _scatter_tensor_in_dygraph(tensor, tensor_or_tensor_list, - src, group, sync_op, - use_calc_stream) + return _scatter_tensor_in_dygraph( + tensor, + tensor_or_tensor_list, + src, + group, + sync_op, + use_calc_stream, + ) else: - return _scatter_in_dygraph(tensor, tensor_or_tensor_list, src, - group, sync_op, use_calc_stream) + return _scatter_in_dygraph( + tensor, + tensor_or_tensor_list, + src, + group, + sync_op, + use_calc_stream, + ) raise RuntimeError( "paddle.distributed.stream.scatter is only supported in dygraph mode now." diff --git a/python/paddle/distributed/communication/stream/send.py b/python/paddle/distributed/communication/stream/send.py index 41ec2c0141b12..206a8cfaa6f01 100644 --- a/python/paddle/distributed/communication/stream/send.py +++ b/python/paddle/distributed/communication/stream/send.py @@ -73,10 +73,12 @@ def send(tensor, dst=0, group=None, sync_op=True, use_calc_stream=False): if not sync_op and use_calc_stream: raise RuntimeError( - "use_calc_stream can only be True in sync op behavior.") + "use_calc_stream can only be True in sync op behavior." + ) if framework.in_dygraph_mode(): return _send_in_dygraph(tensor, dst, group, sync_op, use_calc_stream) raise RuntimeError( - "paddle.distributed.stream.send is only supported in dygraph mode now.") + "paddle.distributed.stream.send is only supported in dygraph mode now." + ) diff --git a/python/paddle/distributed/elastic.py b/python/paddle/distributed/elastic.py index b13eb07ee28c0..d1fd9a790f2d4 100644 --- a/python/paddle/distributed/elastic.py +++ b/python/paddle/distributed/elastic.py @@ -17,7 +17,6 @@ class Command(object): - def __init__(self, server, name): import etcd3 @@ -47,14 +46,15 @@ def close(self): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Elastic Command') - parser.add_argument("--elastic_server", - type=str, - help="etcd server host:port") + parser.add_argument( + "--elastic_server", type=str, help="etcd server host:port" + ) parser.add_argument("--job_id", type=str, help="job unique id") parser.add_argument( "--np", type=str, - help="job pod/node number, need to be 'MIN' or 'MIN:MAX' format") + help="job pod/node number, need to be 'MIN' or 'MIN:MAX' format", + ) parser.add_argument("action", type=str, help="action to take") args = parser.parse_args() diff --git a/python/paddle/distributed/entry_attr.py b/python/paddle/distributed/entry_attr.py index a118314778186..f5043ac084cd6 100644 --- a/python/paddle/distributed/entry_attr.py +++ b/python/paddle/distributed/entry_attr.py @@ -124,11 +124,13 @@ def __init__(self, count_filter): if not isinstance(count_filter, int): raise ValueError( - "count_filter must be a valid integer greater than 0") + "count_filter must be a valid integer greater than 0" + ) if count_filter < 0: raise ValueError( - "count_filter must be a valid integer greater or equal than 0") + "count_filter must be a valid integer greater or equal than 0" + ) self._name = "count_filter_entry" self._count_filter = count_filter diff --git a/python/paddle/distributed/fleet/__init__.py b/python/paddle/distributed/fleet/__init__.py index b75d84edf29b2..aebedefeaafcb 100755 --- a/python/paddle/distributed/fleet/__init__.py +++ b/python/paddle/distributed/fleet/__init__.py @@ -24,7 +24,9 @@ from .dataset import FileInstantDataset # noqa: F401 from .dataset import BoxPSDataset # noqa: F401 from .data_generator.data_generator import MultiSlotDataGenerator # noqa: F401 -from .data_generator.data_generator import MultiSlotStringDataGenerator # noqa: F401 +from .data_generator.data_generator import ( + MultiSlotStringDataGenerator, +) # noqa: F401 from . import metrics # noqa: F401 from .base.topology import CommunicateTopology from .base.topology import HybridCommunicateGroup # noqa: F401 @@ -34,11 +36,17 @@ from .scaler import distributed_scaler from .utils import log_util -__all__ = [ #noqa - "CommunicateTopology", "UtilBase", "HybridCommunicateGroup", - "MultiSlotStringDataGenerator", "UserDefinedRoleMaker", - "DistributedStrategy", "Role", "MultiSlotDataGenerator", - "PaddleCloudRoleMaker", "Fleet" +__all__ = [ # noqa + "CommunicateTopology", + "UtilBase", + "HybridCommunicateGroup", + "MultiSlotStringDataGenerator", + "UserDefinedRoleMaker", + "DistributedStrategy", + "Role", + "MultiSlotDataGenerator", + "PaddleCloudRoleMaker", + "Fleet", ] fleet = Fleet() diff --git a/python/paddle/distributed/fleet/ascend_utils.py b/python/paddle/distributed/fleet/ascend_utils.py index 6ff31555a1222..ee10cd78a5ed0 100644 --- a/python/paddle/distributed/fleet/ascend_utils.py +++ b/python/paddle/distributed/fleet/ascend_utils.py @@ -14,7 +14,11 @@ import os import json -from paddle.distributed.fleet.launch_utils import DeviceMode, get_cluster, get_host_name_ip +from paddle.distributed.fleet.launch_utils import ( + DeviceMode, + get_cluster, + get_host_name_ip, +) __all__ = [] @@ -80,17 +84,18 @@ def _get_ascend_rankfile(rank_table_file_path): assert nodes is not None, "DLS_TASK_NUMBER didn't set!" for node in range(int(nodes)): node_ip = os.getenv("VC_CUSTOM{}_HOSTS".format(node), None) - assert node_ip is not None, "VC_CUSTOM{}_HOSTS didn't set!".format( - node) + assert ( + node_ip is not None + ), "VC_CUSTOM{}_HOSTS didn't set!".format(node) node_ips.append(node_ip) return node_ips, device_count node_ips.append(server['server_id']) return node_ips, device_count -def get_cloud_cluster(rank_table_file=None, - device_mode=DeviceMode.ASCEND_NPU, - start_port=6070): +def get_cloud_cluster( + rank_table_file=None, device_mode=DeviceMode.ASCEND_NPU, start_port=6070 +): """ Args: rank_table_file: string, ascend npu rank file path @@ -110,8 +115,12 @@ def get_cloud_cluster(rank_table_file=None, else: _, node_ip = get_host_name_ip() - assert node_ip in node_ips, "Can't find your local ip {%s} in node_ips: {%s}" \ - % (node_ip, node_ips) + assert ( + node_ip in node_ips + ), "Can't find your local ip {%s} in node_ips: {%s}" % ( + node_ip, + node_ips, + ) else: # single trainer (single ascend card) node_ips = ["127.0.0.1"] @@ -127,5 +136,6 @@ def get_cloud_cluster(rank_table_file=None, for ip in node_ips: trainer_endpoints.append(["%s:%d" % (ip, port) for port in free_ports]) - return get_cluster(node_ips, node_ip, trainer_endpoints, device_mode, - devices_per_proc) + return get_cluster( + node_ips, node_ip, trainer_endpoints, device_mode, devices_per_proc + ) diff --git a/python/paddle/distributed/fleet/base/distributed_strategy.py b/python/paddle/distributed/fleet/base/distributed_strategy.py index c54df36cec637..c32b1f2d68c59 100755 --- a/python/paddle/distributed/fleet/base/distributed_strategy.py +++ b/python/paddle/distributed/fleet/base/distributed_strategy.py @@ -26,7 +26,6 @@ def __non_auto_func_called__(func): - def __impl__(*args, **kwargs): global non_auto_func_called non_auto_func_called = False @@ -129,7 +128,8 @@ def __init__(self): key = 'FLAGS_cudnn_batchnorm_spatial_persistent' if _global_flags().is_public(key): self.strategy.cudnn_batchnorm_spatial_persistent = bool( - _global_flags()[key]) + _global_flags()[key] + ) key = 'FLAGS_conv_workspace_size_limit' if _global_flags().is_public(key): self.strategy.conv_workspace_size_limit = int(_global_flags()[key]) @@ -144,8 +144,9 @@ def __init__(self): def __setattr__(self, key, value): if self.__lock_attr and not hasattr(self, key): - raise TypeError("%s is not a attribute of %s" % - (key, self.__class__.__name__)) + raise TypeError( + "%s is not a attribute of %s" % (key, self.__class__.__name__) + ) object.__setattr__(self, key, value) def save_to_prototxt(self, output): @@ -180,7 +181,8 @@ def load_from_prototxt(self, pb_file): """ with open(pb_file, 'r') as f: self.strategy = google.protobuf.text_format.Merge( - str(f.read()), self.strategy) + str(f.read()), self.strategy + ) @property def execution_strategy(self): @@ -203,8 +205,11 @@ def execution_strategy(self): execution_strategy = paddle.fluid.ExecutionStrategy() fields = self.strategy.execution_strategy.DESCRIPTOR.fields for f in fields: - setattr(execution_strategy, f.name, - getattr(self.strategy.execution_strategy, f.name)) + setattr( + execution_strategy, + f.name, + getattr(self.strategy.execution_strategy, f.name), + ) return execution_strategy @execution_strategy.setter @@ -212,8 +217,11 @@ def execution_strategy(self): def execution_strategy(self, strategy): fields = self.strategy.execution_strategy.DESCRIPTOR.fields for f in fields: - setattr(self.strategy.execution_strategy, f.name, - getattr(strategy, f.name)) + setattr( + self.strategy.execution_strategy, + f.name, + getattr(strategy, f.name), + ) @property def build_strategy(self): @@ -261,8 +269,9 @@ def build_strategy(self, strategy): value = ReduceStrategyFleet(value) setattr(self.strategy.build_strategy, f.name, value) elif f.label == 3: # repeated field - getattr(self.strategy.build_strategy, - f.name).extend(getattr(strategy, f.name)) + getattr(self.strategy.build_strategy, f.name).extend( + getattr(strategy, f.name) + ) @property def gradient_scale_configs(self): @@ -282,8 +291,11 @@ def gradient_scale_configs(self): @gradient_scale_configs.setter @is_strict_auto def gradient_scale_configs(self, config): - check_configs_key(self.strategy.gradient_scale_configs, config, - 'gradient_scale_configs') + check_configs_key( + self.strategy.gradient_scale_configs, + config, + 'gradient_scale_configs', + ) assign_configs_value(self.strategy.gradient_scale_configs, config) @property @@ -318,8 +330,10 @@ def a_sync(self, flag): self.a_sync_configs = {"k_steps": 0} else: raise ValueError( - "The type of `flag` is invalid, expected type is bool, but received {}" - .format(type(flag))) + "The type of `flag` is invalid, expected type is bool, but received {}".format( + type(flag) + ) + ) @property def a_sync_configs(self): @@ -365,8 +379,9 @@ def a_sync_configs(self): @a_sync_configs.setter @is_strict_auto def a_sync_configs(self, configs): - check_configs_key(self.strategy.a_sync_configs, configs, - "a_sync_configs") + check_configs_key( + self.strategy.a_sync_configs, configs, "a_sync_configs" + ) assign_configs_value(self.strategy.a_sync_configs, configs) @property @@ -430,14 +445,17 @@ def adam_d2sum(self, flag): self.strategy.adam_d2sum = flag else: raise ValueError( - "The type of `flag` is invalid, expected type is bool, but received {}" - .format(type(flag))) + "The type of `flag` is invalid, expected type is bool, but received {}".format( + type(flag) + ) + ) @trainer_desc_configs.setter @is_strict_auto def trainer_desc_configs(self, configs): - check_configs_key(self.strategy.trainer_desc_configs, configs, - "trainer_desc_configs") + check_configs_key( + self.strategy.trainer_desc_configs, configs, "trainer_desc_configs" + ) assign_configs_value(self.strategy.trainer_desc_configs, configs) @property @@ -465,8 +483,9 @@ def fs_client_param(self): @fs_client_param.setter @is_strict_auto def fs_client_param(self, configs): - check_configs_key(self.strategy.fs_client_param, configs, - "fs_client_param") + check_configs_key( + self.strategy.fs_client_param, configs, "fs_client_param" + ) assign_configs_value(self.strategy.fs_client_param, configs) @property @@ -477,6 +496,7 @@ def sparse_table_configs(self): @is_strict_auto def sparse_table_configs(self, configs): from google.protobuf.descriptor import FieldDescriptor + table_param = self.strategy.downpour_table_param def set_table_config(msg, config_name, configs, index=0): @@ -493,8 +513,9 @@ def set_table_config(msg, config_name, configs, index=0): data = getattr(msg, field.name).add() set_table_config(data, name, configs, i) else: - set_table_config(getattr(msg, field.name), name, - configs) + set_table_config( + getattr(msg, field.name), name, configs + ) else: # print("not message:", name) if name not in configs: @@ -513,132 +534,204 @@ def set_table_config(msg, config_name, configs, index=0): for table_name in configs: table_data = table_param.add() table_data.table_name = table_name - set_table_config(table_data, "table_parameters." + table_name, - configs[table_name]) + set_table_config( + table_data, + "table_parameters." + table_name, + configs[table_name], + ) @sparse_table_configs.setter def fleet_desc_configs(self, configs): - support_sparse_key_list = ['sparse_table_class', 'sparse_compress_in_save', 'sparse_shard_num', \ - 'sparse_accessor_class', 'sparse_learning_rate', 'sparse_initial_g2sum', 'sparse_initial_range', \ - 'sparse_weight_bounds', 'sparse_fea_dim', 'sparse_embedx_dim', 'sparse_embedx_threshold', 'sparse_nonclk_coeff', \ - 'sparse_click_coeff', 'sparse_base_threshold', 'sparse_delta_threshold', 'sparse_delta_keep_days', \ - 'sparse_delete_after_unseen_days', 'sparse_show_click_decay_rate', 'sparse_delete_threshold', \ - 'sparse_converter', 'sparse_deconverter', 'sparse_enable_cache', 'sparse_cache_rate', \ - 'sparse_cache_file_num', 'sparse_beta1_decay_rate', 'sparse_beta2_decay_rate', \ - 'sparse_ada_epsilon', 'sparse_optimizer', 'sparse_ssd_unseenday_threshold', - 'embed_sparse_optimizer', 'embed_sparse_learning_rate', 'embed_sparse_weight_bounds', \ - 'embed_sparse_initial_range', 'embed_sparse_initial_g2sum', 'embed_sparse_beta1_decay_rate', \ - 'embed_sparse_beta2_decay_rate', 'embedx_sparse_optimizer', 'embedx_sparse_learning_rate', \ - 'embedx_sparse_weight_bounds', 'embedx_sparse_initial_range', 'embedx_sparse_initial_g2sum', \ - 'embedx_sparse_beta1_decay_rate', 'embedx_sparse_beta2_decay_rate', 'feature_learning_rate', 'nodeid_slot'] + support_sparse_key_list = [ + 'sparse_table_class', + 'sparse_compress_in_save', + 'sparse_shard_num', + 'sparse_accessor_class', + 'sparse_learning_rate', + 'sparse_initial_g2sum', + 'sparse_initial_range', + 'sparse_weight_bounds', + 'sparse_fea_dim', + 'sparse_embedx_dim', + 'sparse_embedx_threshold', + 'sparse_nonclk_coeff', + 'sparse_click_coeff', + 'sparse_base_threshold', + 'sparse_delta_threshold', + 'sparse_delta_keep_days', + 'sparse_delete_after_unseen_days', + 'sparse_show_click_decay_rate', + 'sparse_delete_threshold', + 'sparse_converter', + 'sparse_deconverter', + 'sparse_enable_cache', + 'sparse_cache_rate', + 'sparse_cache_file_num', + 'sparse_beta1_decay_rate', + 'sparse_beta2_decay_rate', + 'sparse_ada_epsilon', + 'sparse_optimizer', + 'sparse_ssd_unseenday_threshold', + 'embed_sparse_optimizer', + 'embed_sparse_learning_rate', + 'embed_sparse_weight_bounds', + 'embed_sparse_initial_range', + 'embed_sparse_initial_g2sum', + 'embed_sparse_beta1_decay_rate', + 'embed_sparse_beta2_decay_rate', + 'embedx_sparse_optimizer', + 'embedx_sparse_learning_rate', + 'embedx_sparse_weight_bounds', + 'embedx_sparse_initial_range', + 'embedx_sparse_initial_g2sum', + 'embedx_sparse_beta1_decay_rate', + 'embedx_sparse_beta2_decay_rate', + 'feature_learning_rate', + 'nodeid_slot', + ] support_sparse_table_class = ['DownpourSparseTable'] support_sparse_accessor_class = [ - 'DownpourSparseValueAccessor', 'DownpourCtrAccessor', - 'DownpourCtrDoubleAccessor', 'DownpourUnitAccessor', - 'DownpourDoubleUnitAccessor', 'DownpourCtrDymfAccessor' + 'DownpourSparseValueAccessor', + 'DownpourCtrAccessor', + 'DownpourCtrDoubleAccessor', + 'DownpourUnitAccessor', + 'DownpourDoubleUnitAccessor', + 'DownpourCtrDymfAccessor', ] table_param = self.strategy.downpour_table_param def add_graph_config(graph, strategy): - graph.feature_learning_rate = strategy.get('feature_learning_rate', - 0.05) + graph.feature_learning_rate = strategy.get( + 'feature_learning_rate', 0.05 + ) graph.nodeid_slot = strategy.get('nodeid_slot', 9008) def sparse_optimizer_config(sgd, strategy, prefix): - optimizer_name = strategy.get(prefix + "sparse_optimizer", - "adagrad") + optimizer_name = strategy.get( + prefix + "sparse_optimizer", "adagrad" + ) sgd.name = optimizer_name if optimizer_name == "naive": sgd.name = "SparseNaiveSGDRule" sgd.naive.learning_rate = strategy.get( - prefix + 'sparse_learning_rate', 0.05) + prefix + 'sparse_learning_rate', 0.05 + ) sgd.naive.initial_range = strategy.get( - prefix + 'sparse_initial_range', 1e-4) - bounds = strategy.get(prefix + 'sparse_weight_bounds', - [-10, 10]) + prefix + 'sparse_initial_range', 1e-4 + ) + bounds = strategy.get( + prefix + 'sparse_weight_bounds', [-10, 10] + ) sgd.naive.weight_bounds.extend(bounds) elif optimizer_name == "adagrad": sgd.name = 'SparseAdaGradSGDRule' sgd.adagrad.learning_rate = strategy.get( - prefix + 'sparse_learning_rate', 0.05) + prefix + 'sparse_learning_rate', 0.05 + ) sgd.adagrad.initial_range = strategy.get( - prefix + 'sparse_initial_range', 1e-4) + prefix + 'sparse_initial_range', 1e-4 + ) if prefix == "embed_": sgd.adagrad.initial_range = 0 sgd.adagrad.initial_g2sum = strategy.get( - prefix + 'sparse_initial_g2sum', 3) - bounds = strategy.get(prefix + 'sparse_weight_bounds', - [-10, 10]) + prefix + 'sparse_initial_g2sum', 3 + ) + bounds = strategy.get( + prefix + 'sparse_weight_bounds', [-10, 10] + ) sgd.adagrad.weight_bounds.extend(bounds) elif optimizer_name == "std_adagrad": sgd.name = 'StdAdaGradSGDRule' sgd.adagrad.learning_rate = strategy.get( - prefix + 'sparse_learning_rate', 0.05) + prefix + 'sparse_learning_rate', 0.05 + ) sgd.adagrad.initial_range = strategy.get( - prefix + 'sparse_initial_range', 1e-4) + prefix + 'sparse_initial_range', 1e-4 + ) if prefix == "embed_": sgd.adagrad.initial_range = 0 sgd.adagrad.initial_g2sum = strategy.get( - prefix + 'sparse_initial_g2sum', 3) - bounds = strategy.get(prefix + 'sparse_weight_bounds', - [-10, 10]) + prefix + 'sparse_initial_g2sum', 3 + ) + bounds = strategy.get( + prefix + 'sparse_weight_bounds', [-10, 10] + ) sgd.adagrad.weight_bounds.extend(bounds) elif optimizer_name == "adam": sgd.name = 'SparseAdamSGDRule' sgd.adam.learning_rate = strategy.get( - prefix + 'sparse_learning_rate', 0.001) + prefix + 'sparse_learning_rate', 0.001 + ) sgd.adam.initial_range = strategy.get( - prefix + 'sparse_initial_range', 1e-4) + prefix + 'sparse_initial_range', 1e-4 + ) sgd.adam.beta1_decay_rate = strategy.get( - prefix + 'sparse_beta1_decay_rate', 0.9) + prefix + 'sparse_beta1_decay_rate', 0.9 + ) sgd.adam.beta2_decay_rate = strategy.get( - prefix + 'sparse_beta2_decay_rate', 0.999) + prefix + 'sparse_beta2_decay_rate', 0.999 + ) sgd.adam.ada_epsilon = strategy.get( - prefix + 'sparse_ada_epsilon', 1e-8) - bounds = strategy.get(prefix + 'sparse_weight_bounds', - [-10, 10]) + prefix + 'sparse_ada_epsilon', 1e-8 + ) + bounds = strategy.get( + prefix + 'sparse_weight_bounds', [-10, 10] + ) sgd.adam.weight_bounds.extend(bounds) elif optimizer_name == "shared_adam": sgd.name = 'SparseSharedAdamSGDRule' sgd.adam.learning_rate = strategy.get( - prefix + 'sparse_learning_rate', 0.001) + prefix + 'sparse_learning_rate', 0.001 + ) sgd.adam.initial_range = strategy.get( - prefix + 'sparse_initial_range', 1e-4) + prefix + 'sparse_initial_range', 1e-4 + ) sgd.adam.beta1_decay_rate = strategy.get( - prefix + 'sparse_beta1_decay_rate', 0.9) + prefix + 'sparse_beta1_decay_rate', 0.9 + ) sgd.adam.beta2_decay_rate = strategy.get( - prefix + 'sparse_beta2_decay_rate', 0.999) + prefix + 'sparse_beta2_decay_rate', 0.999 + ) sgd.adam.ada_epsilon = strategy.get( - prefix + 'sparse_ada_epsilon', 1e-8) - bounds = strategy.get(prefix + 'sparse_weight_bounds', - [-10, 10]) + prefix + 'sparse_ada_epsilon', 1e-8 + ) + bounds = strategy.get( + prefix + 'sparse_weight_bounds', [-10, 10] + ) sgd.adam.weight_bounds.extend(bounds) def set_sparse_table_config(table_data, config): for key in config: if key not in support_sparse_key_list: raise ValueError("strategy key '%s' not support" % (key)) - table_class = config.get("sparse_table_class", - "DownpourSparseTable") + table_class = config.get( + "sparse_table_class", "DownpourSparseTable" + ) if table_class not in support_sparse_table_class: raise ValueError( "support sparse_table_class: ['DownpourSparseTable'], but actual %s" - % (table_class)) + % (table_class) + ) table_data.table_class = 'MemorySparseTable' table_data.shard_num = config.get('sparse_shard_num', 1000) table_data.enable_sparse_table_cache = config.get( - 'sparse_enable_cache', True) + 'sparse_enable_cache', True + ) table_data.sparse_table_cache_rate = config.get( - 'sparse_cache_rate', 0.00055) + 'sparse_cache_rate', 0.00055 + ) table_data.sparse_table_cache_file_num = config.get( - 'sparse_cache_file_num', 16) + 'sparse_cache_file_num', 16 + ) - accessor_class = config.get("sparse_accessor_class", - "DownpourCtrAccessor") + accessor_class = config.get( + "sparse_accessor_class", "DownpourCtrAccessor" + ) if accessor_class not in support_sparse_accessor_class: raise ValueError( "support sparse_accessor_class: ['DownpourSparseValueAccessor', 'DownpourCtrAccessor', 'DownpourCtrDoubleAccessor', 'DownpourUnitAccessor', 'DownpourDoubleUnitAccessor'], but actual %s" - % (accessor_class)) + % (accessor_class) + ) if accessor_class.find("Double") >= 0: table_data.accessor.accessor_class = 'CtrDoubleAccessor' @@ -653,7 +746,8 @@ def set_sparse_table_config(table_data, config): table_data.accessor.embedx_dim = config.get('sparse_embedx_dim', 8) table_data.accessor.fea_dim = table_data.accessor.embedx_dim + 3 table_data.accessor.embedx_threshold = config.get( - 'sparse_embedx_threshold', 10) + 'sparse_embedx_threshold', 10 + ) if accessor_class == 'DownpourUnitAccessor': table_data.accessor.ctr_accessor_param.show_scale = False @@ -661,23 +755,32 @@ def set_sparse_table_config(table_data, config): table_data.accessor.ctr_accessor_param.show_scale = True table_data.accessor.ctr_accessor_param.nonclk_coeff = config.get( - 'sparse_nonclk_coeff', 0.1) + 'sparse_nonclk_coeff', 0.1 + ) table_data.accessor.ctr_accessor_param.click_coeff = config.get( - 'sparse_click_coeff', 1) + 'sparse_click_coeff', 1 + ) table_data.accessor.ctr_accessor_param.base_threshold = config.get( - 'sparse_base_threshold', 1.5) + 'sparse_base_threshold', 1.5 + ) table_data.accessor.ctr_accessor_param.delta_threshold = config.get( - 'sparse_delta_threshold', 0.25) + 'sparse_delta_threshold', 0.25 + ) table_data.accessor.ctr_accessor_param.delta_keep_days = config.get( - 'sparse_delta_keep_days', 16) - table_data.accessor.ctr_accessor_param.show_click_decay_rate = config.get( - 'sparse_show_click_decay_rate', 0.98) - table_data.accessor.ctr_accessor_param.delete_threshold = config.get( - 'sparse_delete_threshold', 0.8) - table_data.accessor.ctr_accessor_param.delete_after_unseen_days = config.get( - 'sparse_delete_after_unseen_days', 30) - table_data.accessor.ctr_accessor_param.ssd_unseenday_threshold = config.get( - 'sparse_ssd_unseenday_threshold', 1) + 'sparse_delta_keep_days', 16 + ) + table_data.accessor.ctr_accessor_param.show_click_decay_rate = ( + config.get('sparse_show_click_decay_rate', 0.98) + ) + table_data.accessor.ctr_accessor_param.delete_threshold = ( + config.get('sparse_delete_threshold', 0.8) + ) + table_data.accessor.ctr_accessor_param.delete_after_unseen_days = ( + config.get('sparse_delete_after_unseen_days', 30) + ) + table_data.accessor.ctr_accessor_param.ssd_unseenday_threshold = ( + config.get('sparse_ssd_unseenday_threshold', 1) + ) converter = config.get('sparse_converter', "") deconverter = config.get('sparse_deconverter', "") @@ -691,23 +794,33 @@ def set_sparse_table_config(table_data, config): save_data2.converter = converter save_data2.deconverter = deconverter - if accessor_class == 'DownpourCtrAccessor' or accessor_class == 'DownpourCtrDoubleAccessor': - sparse_optimizer_config(table_data.accessor.embed_sgd_param, - config, '') - sparse_optimizer_config(table_data.accessor.embedx_sgd_param, - config, '') + if ( + accessor_class == 'DownpourCtrAccessor' + or accessor_class == 'DownpourCtrDoubleAccessor' + ): + sparse_optimizer_config( + table_data.accessor.embed_sgd_param, config, '' + ) + sparse_optimizer_config( + table_data.accessor.embedx_sgd_param, config, '' + ) else: - sparse_optimizer_config(table_data.accessor.embed_sgd_param, - config, 'embed_') - sparse_optimizer_config(table_data.accessor.embedx_sgd_param, - config, 'embedx_') + sparse_optimizer_config( + table_data.accessor.embed_sgd_param, config, 'embed_' + ) + sparse_optimizer_config( + table_data.accessor.embedx_sgd_param, config, 'embedx_' + ) add_graph_config(table_data.accessor.graph_sgd_param, config) if not configs: print("fleet desc config is empty") else: for table_name in configs: - if table_name == 'dense_table' or table_name == 'datanorm_table': + if ( + table_name == 'dense_table' + or table_name == 'datanorm_table' + ): continue if type(configs[table_name]) != dict: continue @@ -1136,8 +1249,9 @@ def recompute_configs(self): @recompute_configs.setter @is_strict_auto def recompute_configs(self, configs): - check_configs_key(self.strategy.recompute_configs, configs, - "checkpoint_configs") + check_configs_key( + self.strategy.recompute_configs, configs, "checkpoint_configs" + ) assign_configs_value(self.strategy.recompute_configs, configs) @property @@ -1230,8 +1344,9 @@ def sharding_configs(self): @sharding_configs.setter @is_strict_auto def sharding_configs(self, configs): - check_configs_key(self.strategy.sharding_configs, configs, - "sharding_configs") + check_configs_key( + self.strategy.sharding_configs, configs, "sharding_configs" + ) assign_configs_value(self.strategy.sharding_configs, configs) @property @@ -1412,8 +1527,9 @@ def pipeline_configs(self): @pipeline_configs.setter @is_strict_auto def pipeline_configs(self, configs): - check_configs_key(self.strategy.pipeline_configs, configs, - "pipeline_configs") + check_configs_key( + self.strategy.pipeline_configs, configs, "pipeline_configs" + ) assign_configs_value(self.strategy.pipeline_configs, configs) @property @@ -1467,8 +1583,11 @@ def tensor_parallel_configs(self): @tensor_parallel_configs.setter @is_strict_auto def tensor_parallel_configs(self, configs): - check_configs_key(self.strategy.tensor_parallel_configs, configs, - "tensor_parallel_configs") + check_configs_key( + self.strategy.tensor_parallel_configs, + configs, + "tensor_parallel_configs", + ) assign_configs_value(self.strategy.tensor_parallel_configs, configs) @property @@ -1501,8 +1620,9 @@ def hybrid_configs(self): @hybrid_configs.setter def hybrid_configs(self, configs): - check_configs_key(self.strategy.hybrid_configs, configs, - "hybrid_configs") + check_configs_key( + self.strategy.hybrid_configs, configs, "hybrid_configs" + ) assign_configs_value(self.strategy.hybrid_configs, configs) @property @@ -1558,8 +1678,9 @@ def localsgd_configs(self): @localsgd_configs.setter @is_strict_auto def localsgd_configs(self, configs): - check_configs_key(self.strategy.localsgd_configs, configs, - "localsgd_configs") + check_configs_key( + self.strategy.localsgd_configs, configs, "localsgd_configs" + ) assign_configs_value(self.strategy.localsgd_configs, configs) @property @@ -1617,8 +1738,11 @@ def adaptive_localsgd_configs(self): @adaptive_localsgd_configs.setter @is_strict_auto def adaptive_localsgd_configs(self, configs): - check_configs_key(self.strategy.adaptive_localsgd_configs, configs, - "adaptive_localsgd_configs") + check_configs_key( + self.strategy.adaptive_localsgd_configs, + configs, + "adaptive_localsgd_configs", + ) assign_configs_value(self.strategy.adaptive_localsgd_configs, configs) @property @@ -1762,8 +1886,9 @@ def gradient_merge_configs(self): @gradient_merge_configs.setter @is_strict_auto def gradient_merge_configs(self, configs): - check_configs_key(self.strategy.gradient_merge_configs, configs, - "gradient_configs") + check_configs_key( + self.strategy.gradient_merge_configs, configs, "gradient_configs" + ) assign_configs_value(self.strategy.gradient_merge_configs, configs) @property @@ -2243,7 +2368,8 @@ def __repr__(self): h1_format = " " + "|{{:^{}s}}|\n".format(length) h2_format = " " + "|{{:>{}s}}{}{{:^{}s}}|\n".format( - max_k, " " * spacing, max_v) + max_k, " " * spacing, max_v + ) border = " +" + "".join(["="] * length) + "+" line = " +" + "".join(["-"] * length) + "+" @@ -2268,37 +2394,48 @@ def __repr__(self): if getattr(self.strategy, f.name): draws += border + "\n" draws += h1_format.format( - "{}=True <-> {}_configs".format(f.name, f.name)) + "{}=True <-> {}_configs".format(f.name, f.name) + ) draws += line + "\n" - my_configs = getattr(self.strategy, - f.name + "_configs") + my_configs = getattr( + self.strategy, f.name + "_configs" + ) config_fields = my_configs.DESCRIPTOR.fields for ff in config_fields: if isinstance( - getattr(my_configs, - ff.name), google.protobuf.pyext. - _message.RepeatedScalarContainer): + getattr(my_configs, ff.name), + google.protobuf.pyext._message.RepeatedScalarContainer, + ): values = getattr(my_configs, ff.name) for i, v in enumerate(values): if i == 0: draws += h2_format.format( - ff.name, str(v)) + ff.name, str(v) + ) else: draws += h2_format.format( - "", str(v)) + "", str(v) + ) else: draws += h2_format.format( ff.name, - str(getattr(my_configs, ff.name))) + str(getattr(my_configs, ff.name)), + ) else: env_draws += h2_format.format( - f.name, str(getattr(self.strategy, f.name))) + f.name, str(getattr(self.strategy, f.name)) + ) else: env_draws += h2_format.format( - f.name, str(getattr(self.strategy, f.name))) - - result_res = draws + border + "\n" + h1_format.format( - "Environment Flags, Communication Flags") + f.name, str(getattr(self.strategy, f.name)) + ) + + result_res = ( + draws + + border + + "\n" + + h1_format.format("Environment Flags, Communication Flags") + ) result_res += env_draws build_strategy_str = border + "\n" @@ -2308,7 +2445,8 @@ def __repr__(self): fields = self.strategy.build_strategy.DESCRIPTOR.fields for f in fields: build_strategy_str += h2_format.format( - f.name, str(getattr(self.strategy.build_strategy, f.name))) + f.name, str(getattr(self.strategy.build_strategy, f.name)) + ) build_strategy_str += border + "\n" execution_strategy_str = h1_format.format("Execution Strategy") @@ -2317,7 +2455,8 @@ def __repr__(self): fields = self.strategy.execution_strategy.DESCRIPTOR.fields for f in fields: execution_strategy_str += h2_format.format( - f.name, str(getattr(self.strategy.execution_strategy, f.name))) + f.name, str(getattr(self.strategy.execution_strategy, f.name)) + ) execution_strategy_str += border + "\n" result_res += build_strategy_str + execution_strategy_str diff --git a/python/paddle/distributed/fleet/base/meta_optimizer_factory.py b/python/paddle/distributed/fleet/base/meta_optimizer_factory.py index c2e55cfc38d63..380a9b8c177af 100755 --- a/python/paddle/distributed/fleet/base/meta_optimizer_factory.py +++ b/python/paddle/distributed/fleet/base/meta_optimizer_factory.py @@ -17,7 +17,8 @@ __all__ = [] meta_optimizer_names = list( - filter(lambda name: name.endswith("Optimizer"), dir())) + filter(lambda name: name.endswith("Optimizer"), dir()) +) # Because HybridParallelOptimizer is dygraph optimizer, it # should be removed @@ -26,7 +27,6 @@ class MetaOptimizerFactory(object): - def __init__(self): pass diff --git a/python/paddle/distributed/fleet/base/orthogonal_strategy.py b/python/paddle/distributed/fleet/base/orthogonal_strategy.py index d2ba6e4461b24..d0fec2cfdb2d5 100644 --- a/python/paddle/distributed/fleet/base/orthogonal_strategy.py +++ b/python/paddle/distributed/fleet/base/orthogonal_strategy.py @@ -19,7 +19,7 @@ from paddle.distributed.fleet.base.strategy_group import StrategyGroupBase -class OrthogonalStrategy(): +class OrthogonalStrategy: """ A hybrid of multiple distributed strategies. Strategies need to be orthogonal, means the ranks are organized like a square if there are two strategies, a cube if there aree three strategies, etc. @@ -57,8 +57,9 @@ def __init__(self, list_of_strategy, fused_strategy_dict={}): strategy[0] for strategy in list_of_strategy ] self._list_of_degree = [strategy[1] for strategy in list_of_strategy] - self._coordinate = collections.namedtuple('Coordinate', - self._list_of_strategy_name) + self._coordinate = collections.namedtuple( + 'Coordinate', self._list_of_strategy_name + ) self._check_valid_strategy() ranges = [range(degree) for degree in self._list_of_degree] @@ -66,14 +67,16 @@ def __init__(self, list_of_strategy, fused_strategy_dict={}): self._coordinate(*coord) for coord in itertools.product(*ranges) ] self._coord_to_rank_dict = dict( - zip(list_of_coord, range(len(list_of_coord)))) + zip(list_of_coord, range(len(list_of_coord))) + ) for idx, strategy in enumerate(list_of_strategy): strategy_name = strategy[0] self._name_to_degree_dict[strategy_name] = strategy[1] self._rank_list_dict[strategy_name] = self._calc_rank_list(idx) self._name_to_group_dict[strategy_name] = strategy[2]( - self._rank_list_dict[strategy_name]) + self._rank_list_dict[strategy_name] + ) self._name_to_fused_group_dict = {} self._create_fused_group() @@ -88,8 +91,9 @@ def strategy_group(self, name): Returns: An instance of specific strategy group. """ - assert name in self._list_of_strategy_name, "Strategy group {} is not created.".format( - name) + assert ( + name in self._list_of_strategy_name + ), "Strategy group {} is not created.".format(name) return self._name_to_group_dict[name] def fused_strategy_group(self, name): @@ -102,8 +106,9 @@ def fused_strategy_group(self, name): Returns: (StrategyGroupBase): An instance of strategy group. """ - assert name in self._name_to_fused_group_dict, "Fused strategy group {} is not created.".format( - name) + assert ( + name in self._name_to_fused_group_dict + ), "Fused strategy group {} is not created.".format(name) return self._name_to_fused_group_dict[name] def rank_in_strategy(self, name): @@ -116,33 +121,42 @@ def rank_in_strategy(self, name): Returns: (Integer): Local rank in specific strategy. """ - assert name in self._list_of_strategy_name, "Strategy group {} is not created.".format( - name) + assert ( + name in self._list_of_strategy_name + ), "Strategy group {} is not created.".format(name) return self._name_to_group_dict[name].group.rank def _check_valid_strategy(self): assert len(self._list_of_strategy_name) == len( set(self._list_of_strategy_name) ), "Defined duplicated strategies: {}".format(list_of_strategy) - num_of_ranks = functools.reduce(lambda x, y: x * y, - self._list_of_degree) - assert num_of_ranks == dist.get_world_size( + num_of_ranks = functools.reduce( + lambda x, y: x * y, self._list_of_degree + ) + assert ( + num_of_ranks == dist.get_world_size() ), "There are total {} ranks, but need {} ranks in this strategy.".format( - dist.get_world_size(), num_of_ranks) + dist.get_world_size(), num_of_ranks + ) for fused_strategy in self._fused_strategy_dict.values(): for strategy in fused_strategy: - assert strategy in self._list_of_strategy_name, "Can not fuse strategy {} without defined previous.".format( - strategy) + assert ( + strategy in self._list_of_strategy_name + ), "Can not fuse strategy {} without defined previous.".format( + strategy + ) def _create_fused_group(self): for name in self._fused_strategy_dict: fused_strategy = self._fused_strategy_dict[name] non_fused_strategy = list( - set(self._list_of_strategy_name).difference(fused_strategy)) + set(self._list_of_strategy_name).difference(fused_strategy) + ) non_fused_ranges = [] for strategy in non_fused_strategy: non_fused_ranges.append( - range(self._name_to_degree_dict[strategy])) + range(self._name_to_degree_dict[strategy]) + ) fused_ranges = [] for strategy in fused_strategy: fused_ranges.append(range(self._name_to_degree_dict[strategy])) @@ -156,8 +170,9 @@ def _create_fused_group(self): for fused_ranks in itertools.product(*fused_ranges): for i, fused_rank in enumerate(fused_ranks): coord_dict[fused_strategy[i]] = fused_rank - ranks.append(self._coord_to_rank_dict[self._coordinate( - **coord_dict)]) + ranks.append( + self._coord_to_rank_dict[self._coordinate(**coord_dict)] + ) rank_list.append(ranks) self._name_to_fused_group_dict[name] = StrategyGroupBase(rank_list) @@ -175,7 +190,8 @@ def _calc_rank_list(self, strategy_axis): coord_list = list(coord) coord_list.insert(strategy_axis, val) ranks.append( - self._coord_to_rank_dict[self._coordinate(*coord_list)]) + self._coord_to_rank_dict[self._coordinate(*coord_list)] + ) rank_list.append(ranks) return rank_list diff --git a/python/paddle/distributed/fleet/base/private_helper_function.py b/python/paddle/distributed/fleet/base/private_helper_function.py index 2745f398152a4..b67146065899a 100644 --- a/python/paddle/distributed/fleet/base/private_helper_function.py +++ b/python/paddle/distributed/fleet/base/private_helper_function.py @@ -39,8 +39,9 @@ def wait_server_ready(endpoints): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): @@ -52,8 +53,9 @@ def wait_server_ready(endpoints): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + str(not_ready_endpoints) + - "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: diff --git a/python/paddle/distributed/fleet/base/role_maker.py b/python/paddle/distributed/fleet/base/role_maker.py index a0b39c3b7a372..28dace611a092 100755 --- a/python/paddle/distributed/fleet/base/role_maker.py +++ b/python/paddle/distributed/fleet/base/role_maker.py @@ -20,7 +20,9 @@ import paddle import paddle.fluid as fluid -from paddle.distributed.fleet.base.private_helper_function import wait_server_ready +from paddle.distributed.fleet.base.private_helper_function import ( + wait_server_ready, +) __all__ = [] @@ -49,10 +51,13 @@ def __init__(self): self._nodes_comm = None self._comm_world = ["worker", "server", "all"] - self._err_init = "gloo is not initialized, will not communicator with other nodes" + self._err_init = ( + "gloo is not initialized, will not communicator with other nodes" + ) self._err_type = "gloo initialized error, please check arguments" self._err_world = "argument error, comm_world must in {}".format( - self._comm_world) + self._comm_world + ) self._is_initialized = False self._init_timeout_seconds = 3600 @@ -67,14 +72,16 @@ def __init__(self): self._server_num = -1 self._need_init_all = False - def init(self, - rendezvous, - role, - role_id, - worker_num, - server_num, - need_init_all=False, - kwargs=None): + def init( + self, + rendezvous, + role, + role_id, + worker_num, + server_num, + need_init_all=False, + kwargs=None, + ): self._rendezvous = rendezvous self._role = role @@ -110,8 +117,9 @@ def init(self, if not ip or not port: raise ValueError(self._err_type) - http_server = self._init_http(ip, port, self._prefix, - start_http_server, http_server_d) + http_server = self._init_http( + ip, port, self._prefix, start_http_server, http_server_d + ) else: raise ValueError(self._err_type) @@ -119,15 +127,15 @@ def init(self, self._http_server = http_server def _init_fs(self, fs_path, prefix): - def init(rank, nodes, role): gloo = fluid.core.Gloo() gloo.set_rank(rank) gloo.set_size(nodes) gloo.set_prefix(prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_hdfs_store(os.path.join(fs_path, role), "", "") gloo.init() return gloo @@ -147,15 +155,15 @@ def init(rank, nodes, role): self._nodes_comm = gloo def _init_dfs(self, dfs_name, dfs_ugi, dfs_path, prefix): - def init(rank, nodes, role): gloo = fluid.core.Gloo() gloo.set_rank(rank) gloo.set_size(nodes) gloo.set_prefix(prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_hdfs_store(os.path.join(dfs_path, role), dfs_name, dfs_ugi) gloo.init() return gloo @@ -175,15 +183,17 @@ def init(rank, nodes, role): self._nodes_comm = gloo def _init_http(self, ip, port, prefix, start_http_server, http_server_d): - def __start_kv_server(http_server_d, size_d): print("start http_server: {}, {}".format(port, size_d)) from paddle.distributed.fleet.utils.http_server import KVServer + http_server = KVServer(port, size_d) http_server.start() wait_seconds = 5 - while http_server_d.get("running", - False) or not http_server.should_stop(): + while ( + http_server_d.get("running", False) + or not http_server.should_stop() + ): time.sleep(wait_seconds) http_server.stop() @@ -196,8 +206,9 @@ def init_kv_server(http_server_d): http_server_d["running"] = True # child process for http server - _http_server = Process(target=__start_kv_server, - args=(http_server_d, size_d)) + _http_server = Process( + target=__start_kv_server, args=(http_server_d, size_d) + ) _http_server.daemon = True # set running status to True # start child process @@ -210,8 +221,9 @@ def init(rank, nodes, role): gloo.set_size(nodes) gloo.set_prefix(prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_http_store(ip, port, 'worker') ep = ":".join([ip, str(port)]) wait_server_ready([ep]) @@ -285,8 +297,12 @@ def __get_default_iface_from_gateway(self): gateway = None if len(item) > gateway_idx: gateway = item[gateway_idx] - if gateway and gateway != '*' and gateway != "0.0.0.0" and len( - item) > iface_idx: + if ( + gateway + and gateway != '*' + and gateway != "0.0.0.0" + and len(item) > iface_idx + ): return item[iface_idx] return "lo" @@ -294,8 +310,9 @@ def __get_default_iface_from_interfaces(self): """ get default physical interface """ - res = os.popen("ip -f inet addr | awk NR%3==1").read().strip().split( - "\n") + res = ( + os.popen("ip -f inet addr | awk NR%3==1").read().strip().split("\n") + ) for item in res: if "BROADCAST" in item: return item.split(":")[1].strip() @@ -470,8 +487,11 @@ def _get_pserver_endpoints(self): def to_string(self): return "role: {}, current_id: {}, worker_endpoints: {}, server_endpoints: {}".format( - self._role, self._current_id, self._worker_endpoints, - self._server_endpoints) + self._role, + self._current_id, + self._worker_endpoints, + self._server_endpoints, + ) def _all_gather(self, input, comm_world="worker"): print("warning: RoleMakerBase does not have all gather worker.") @@ -493,13 +513,13 @@ def _barrier(self, comm_world): """ print("warning: RoleMakerBase does not have barrier worker.") - #def _is_heter_worker(self): + # def _is_heter_worker(self): # """ # Return is_heter_worker() of current process # """ # raise NotImplementedError("Please implement this method in child class") - #def _heter_worker_num(self): + # def _heter_worker_num(self): # """ # Get current total heter-worker number. # @@ -508,14 +528,14 @@ def _barrier(self, comm_world): # """ # raise NotImplementedError("Please implement this method in child class") - #def _get_heter_worker_endpoints(self): + # def _get_heter_worker_endpoints(self): # """ # Returns: # string: all heter_trainers'endpoints # """ # raise NotImplementedError("Please implement this method in child class") - #def _get_heter_worker_endpoint(self): + # def _get_heter_worker_endpoint(self): # """ # Returns: # int: corresponding heter_trainer's endpoint @@ -524,7 +544,6 @@ def _barrier(self, comm_world): class PaddleCloudRoleMaker(RoleMakerBase): - def __init__(self, is_collective=False, **kwargs): super(PaddleCloudRoleMaker, self).__init__() self._is_collective = is_collective @@ -578,24 +597,24 @@ def _heter_device_type(self): def _get_stage_id(self): """ - return stage id of current heter worker - """ + return stage id of current heter worker + """ if not self._role_is_generated: self._generate_role() return self._stage_id def _get_stage_trainers(self): """ - return trainer num of all stages - """ + return trainer num of all stages + """ if not self._role_is_generated: self._generate_role() return self._stage_trainers def _get_num_stage(self): """ - return stage num - """ + return stage num + """ if not self._role_is_generated: self._generate_role() return self._stage_num @@ -667,8 +686,11 @@ def _server_num(self): """ if not self._role_is_generated: self._generate_role() - return len(self._get_pserver_endpoints() - ) if self._get_pserver_endpoints() is not None else 0 + return ( + len(self._get_pserver_endpoints()) + if self._get_pserver_endpoints() is not None + else 0 + ) def _node_num(self): """ @@ -712,7 +734,9 @@ def _get_trainer_endpoints(self): def _get_trainer_endpoint(self): if not self._role_is_generated: self._generate_role() - assert self._role == Role.WORKER, "get_trainer_endpoint should be called by trainer" + assert ( + self._role == Role.WORKER + ), "get_trainer_endpoint should be called by trainer" return self._cur_endpoint def _get_heter_worker_endpoints(self): @@ -722,7 +746,9 @@ def _get_heter_worker_endpoints(self): """ if not self._role_is_generated: self._generate_role() - assert self._heter_trainer_endpoints != [], "Heter Worker Endpoints Not initialized" + assert ( + self._heter_trainer_endpoints != [] + ), "Heter Worker Endpoints Not initialized" return self._heter_trainer_endpoints def _get_heter_worker_endpoint(self): @@ -732,7 +758,9 @@ def _get_heter_worker_endpoint(self): """ if not self._role_is_generated: self._generate_role() - assert self._role == Role.HETER_WORKER, "_get_heter_worker_endpoint should be invoked by heter worker" + assert ( + self._role == Role.HETER_WORKER + ), "_get_heter_worker_endpoint should be invoked by heter worker" return self._cur_endpoint def _get_pserver_endpoints(self): @@ -755,7 +783,8 @@ def _get_previous_trainers(self): if not self._role_is_generated: self._generate_role() assert self._role in ( - Role.WORKER, Role.HETER_WORKER + Role.WORKER, + Role.HETER_WORKER, ), "_get_previous_trainers should be invoked by trainer or heter worker" return self._previous_heter_trainer_endpoints @@ -766,7 +795,8 @@ def _get_next_trainers(self): if not self._role_is_generated: self._generate_role() assert self._role in ( - Role.WORKER, Role.HETER_WORKER + Role.WORKER, + Role.HETER_WORKER, ), "_get_next_trainers should be invoked by trainer or heter worker" return self._next_heter_trainer_endpoints @@ -820,8 +850,9 @@ def _ps_env(self): # each role will execute it else: self._worker_endpoints = [] - self._coordinator_endpoints = os.getenv("PADDLE_COORDINATOR_ENDPOINTS", - "") + self._coordinator_endpoints = os.getenv( + "PADDLE_COORDINATOR_ENDPOINTS", "" + ) if self._coordinator_endpoints == "": print("fl-ps > coordinator address is null!") else: @@ -838,22 +869,31 @@ def _ps_env(self): # each role will execute it training_role = os.getenv("TRAINING_ROLE", None) if training_role == None: raise ValueError( - "Can not find TRAINING_ROLE, please check your environment.") + "Can not find TRAINING_ROLE, please check your environment." + ) if training_role not in [ - "TRAINER", "PSERVER", "HETER_TRAINER", "COORDINATOR" + "TRAINER", + "PSERVER", + "HETER_TRAINER", + "COORDINATOR", ]: raise ValueError( - "TRAINING_ROLE must be PSERVER or TRAINER or HETER_TRAINER or COORDINATOR, but get {}, please check your environment." - .format(training_role)) + "TRAINING_ROLE must be PSERVER or TRAINER or HETER_TRAINER or COORDINATOR, but get {}, please check your environment.".format( + training_role + ) + ) # For Heter Parameter Server env setting next_heter_trainer_eplist = os.getenv( - "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST", "") + "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST", "" + ) previous_heter_trainer_eplist = os.getenv( - "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST", "") + "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST", "" + ) all_heter_trainer_eplist = os.getenv( - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST", "") + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST", "" + ) if all_heter_trainer_eplist != "": self._heter_trainer_endpoints = all_heter_trainer_eplist.split(",") @@ -863,11 +903,13 @@ def _ps_env(self): # each role will execute it if previous_heter_trainer_eplist == "": assert training_role in ( "TRAINER", - "PSERVER"), "training_role should be trainer or pserver" + "PSERVER", + ), "training_role should be trainer or pserver" else: try: - self._previous_heter_trainer_endpoints = previous_heter_trainer_eplist.split( - ",") + self._previous_heter_trainer_endpoints = ( + previous_heter_trainer_eplist.split(",") + ) except: raise ValueError( "Can not Find PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST in env or its format doesn't match the requirement: 'IP:PORT,IP:PORT' ." @@ -875,12 +917,14 @@ def _ps_env(self): # each role will execute it if next_heter_trainer_eplist == "": assert training_role in ( - "HETER_TRAINER", "PSERVER" + "HETER_TRAINER", + "PSERVER", ), "training_role should be heter trainer or pserver" else: try: - self._next_heter_trainer_endpoints = next_heter_trainer_eplist.split( - ",") + self._next_heter_trainer_endpoints = ( + next_heter_trainer_eplist.split(",") + ) except: raise ValueError( "Can not Find PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST in env or its format doesn't match the requirement: 'IP:PORT,IP:PORT' ." @@ -902,7 +946,8 @@ def _ps_env(self): # each role will execute it self._stage_id = os.getenv("STAGE_ID", None) if self._stage_id == None: raise ValueError( - "Can not find STAGE_ID, please check your environment.") + "Can not find STAGE_ID, please check your environment." + ) self._stage_id = int(self._stage_id) self._stage_num = os.getenv("STAGE_NUM", None) if self._stage_num == None: @@ -910,8 +955,9 @@ def _ps_env(self): # each role will execute it "Can not find STAGE_NUM, please check your environment." ) self._stage_num = int(self._stage_num) - self._stage_trainers = os.getenv("PADDLE_STAGE_TRAINERS_NUM", - None) + self._stage_trainers = os.getenv( + "PADDLE_STAGE_TRAINERS_NUM", None + ) if self._stage_trainers == None: raise ValueError( "Can not find PADDLE_STAGE_TRAINERS_NUM, please check your environment." @@ -920,11 +966,13 @@ def _ps_env(self): # each role will execute it cur_port = os.getenv("PADDLE_PORT", None) if cur_port == None: raise ValueError( - "Can not find PADDLE_PORT, please check your environment.") + "Can not find PADDLE_PORT, please check your environment." + ) cur_ip = os.getenv("POD_IP", None) if cur_ip == None: raise ValueError( - "Can not find POD_IP, please check your environment.") + "Can not find POD_IP, please check your environment." + ) curr_endpoint = ":".join([cur_ip, cur_port]) self._cur_endpoint = curr_endpoint elif training_role == "COORDINATOR": @@ -936,11 +984,13 @@ def _ps_env(self): # each role will execute it cur_port = os.getenv("PADDLE_PORT", None) if cur_port == None: raise ValueError( - "Can not find PADDLE_PORT, please check your environment.") + "Can not find PADDLE_PORT, please check your environment." + ) cur_ip = os.getenv("POD_IP", None) if cur_ip == None: raise ValueError( - "Can not find POD_IP, please check your environment.") + "Can not find POD_IP, please check your environment." + ) curr_endpoint = ":".join([cur_ip, cur_port]) self._cur_endpoint = curr_endpoint current_id = self._server_endpoints.index(self._cur_endpoint) @@ -949,12 +999,14 @@ def _ps_env(self): # each role will execute it self._stage_id = os.getenv("STAGE_ID", None) if self._stage_id == None: raise ValueError( - "Can not find STAGE_ID, please check your environment.") + "Can not find STAGE_ID, please check your environment." + ) self._stage_id = int(self._stage_id) self._stage_num = os.getenv("STAGE_NUM", None) if self._stage_num == None: raise ValueError( - "Can not find STAGE_NUM, please check your environment.") + "Can not find STAGE_NUM, please check your environment." + ) self._stage_num = int(self._stage_num) self._stage_trainers = os.getenv("PADDLE_STAGE_TRAINERS_NUM", None) @@ -964,47 +1016,57 @@ def _ps_env(self): # each role will execute it ) self._stage_trainers = eval(self._stage_trainers) - self._heter_trainer_device_type = os.getenv("HETER_DEVICE_TYPE", - None) + self._heter_trainer_device_type = os.getenv( + "HETER_DEVICE_TYPE", None + ) if self._heter_trainer_device_type == None: raise ValueError( "Can not find HETER_DEVICE_TYPE, please check your environment." ) assert self._heter_trainer_device_type in ( - "cpu", "gpu", - "xpu"), "HETER_DEVICE_TYPE should be cpu,gpu or xpu" + "cpu", + "gpu", + "xpu", + ), "HETER_DEVICE_TYPE should be cpu,gpu or xpu" if self._heter_trainer_device_type == "gpu": heter_device_id = os.getenv("FLAGS_selected_gpus", "0") self._heter_trainer_device = ":".join( - (self._heter_trainer_device_type, heter_device_id)) + (self._heter_trainer_device_type, heter_device_id) + ) if self._heter_trainer_device == "xpu": heter_device_id = os.getenv("FLAGS_selected_xpus", "0") self._heter_trainer_device = ":".join( - (self._heter_trainer_device_type, heter_device_id)) + (self._heter_trainer_device_type, heter_device_id) + ) cur_port = os.getenv("PADDLE_PORT", None) if cur_port == None: raise ValueError( - "Can not find PADDLE_PORT, please check your environment.") + "Can not find PADDLE_PORT, please check your environment." + ) cur_ip = os.getenv("POD_IP", None) if cur_ip == None: raise ValueError( - "Can not find POD_IP, please check your environment.") + "Can not find POD_IP, please check your environment." + ) curr_endpoint = ":".join([cur_ip, cur_port]) self._cur_endpoint = curr_endpoint - current_id = all_heter_trainer_eplist.split(",").index( - curr_endpoint) + trainers_num + current_id = ( + all_heter_trainer_eplist.split(",").index(curr_endpoint) + + trainers_num + ) self._trainers_num = trainers_num self._role = role self._current_id = current_id self._nodes_num = len( - set([x.split(':')[0] for x in self._worker_endpoints])) + set([x.split(':')[0] for x in self._worker_endpoints]) + ) def _collective_env(self): self._current_id = int(os.getenv("PADDLE_TRAINER_ID", "0")) self._training_role = os.getenv("PADDLE_TRAINING_ROLE", "TRAINER") - assert (self._training_role == "TRAINER") + assert self._training_role == "TRAINER" self._role = Role.WORKER self._worker_endpoints = os.getenv("PADDLE_TRAINER_ENDPOINTS") self._cur_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT") @@ -1016,7 +1078,8 @@ def _collective_env(self): self._worker_endpoints = self._worker_endpoints.split(",") self._trainers_num = len(self._worker_endpoints) self._nodes_num = len( - set([x.split(':')[0] for x in self._worker_endpoints])) + set([x.split(':')[0] for x in self._worker_endpoints]) + ) self._local_rank = os.getenv("PADDLE_RANK_IN_NODE") self._local_device_ids = os.getenv("PADDLE_LOCAL_DEVICE_IDS") self._world_device_ids = os.getenv("PADDLE_WORLD_DEVICE_IDS") @@ -1031,7 +1094,9 @@ def _gloo_init(self): rendezvous_type = int(os.getenv("PADDLE_GLOO_RENDEZVOUS", "0")) prefix = os.getenv("SYS_JOB_ID", "") if rendezvous_type not in [ - Gloo.RENDEZVOUS.HDFS, Gloo.RENDEZVOUS.HTTP, Gloo.RENDEZVOUS.FILE + Gloo.RENDEZVOUS.HDFS, + Gloo.RENDEZVOUS.HTTP, + Gloo.RENDEZVOUS.FILE, ]: raise ValueError(self._gloo._err_type) @@ -1081,16 +1146,21 @@ def _gloo_init(self): type = "HTTP" else: type = "FILE" - print("Gloo init with {}: need_init_all: {}, args: {}".format( - type, need_init_all, kwargs)) - - self._gloo.init(rendezvous=rendezvous_type, - role=self._role, - role_id=self._role_id(), - worker_num=self._worker_num(), - server_num=self._server_num(), - need_init_all=need_init_all, - kwargs=kwargs) + print( + "Gloo init with {}: need_init_all: {}, args: {}".format( + type, need_init_all, kwargs + ) + ) + + self._gloo.init( + rendezvous=rendezvous_type, + role=self._role, + role_id=self._role_id(), + worker_num=self._worker_num(), + server_num=self._server_num(), + need_init_all=need_init_all, + kwargs=kwargs, + ) if rendezvous_type == Gloo.RENDEZVOUS.HTTP: http_server_d['running'] = False @@ -1110,11 +1180,10 @@ def _generate_role(self): class UserDefinedRoleMaker(PaddleCloudRoleMaker): - def __init__(self, is_collective=False, init_gloo=False, **kwargs): - super(UserDefinedRoleMaker, self).__init__(is_collective=is_collective, - init_gloo=init_gloo, - **kwargs) + super(UserDefinedRoleMaker, self).__init__( + is_collective=is_collective, init_gloo=init_gloo, **kwargs + ) self._init_gloo = init_gloo def _user_defined_ps_env(self): @@ -1123,19 +1192,22 @@ def _user_defined_ps_env(self): self._trainers_num = self._kwargs.get("worker_num", 0) if self._trainers_num == 0: - assert (len(self._worker_endpoints) > 0) + assert len(self._worker_endpoints) > 0 self._trainers_num = len(self._worker_endpoints) self._role = self._kwargs.get("role") self._current_id = self._kwargs.get("current_id") - if self._role == Role.WORKER and len( - self._worker_endpoints) > self._current_id: + if ( + self._role == Role.WORKER + and len(self._worker_endpoints) > self._current_id + ): self._cur_endpoint = self._worker_endpoints[self._current_id] elif self._role == Role.SERVER: self._cur_endpoint = self._server_endpoints[self._current_id] self._nodes_num = len( - set([x.split(':')[0] for x in self._worker_endpoints])) + set([x.split(':')[0] for x in self._worker_endpoints]) + ) def _user_defined_collective_env(self): self._worker_endpoints = self._kwargs.get("worker_endpoints") @@ -1143,7 +1215,8 @@ def _user_defined_collective_env(self): self._trainers_num = len(self._worker_endpoints) self._training_role = Role.WORKER self._nodes_num = len( - set([x.split(':')[0] for x in self._worker_endpoints])) + set([x.split(':')[0] for x in self._worker_endpoints]) + ) def _generate_role(self): """ diff --git a/python/paddle/distributed/fleet/base/runtime_factory.py b/python/paddle/distributed/fleet/base/runtime_factory.py index 3fa7dbb285c4e..e9006409674a7 100644 --- a/python/paddle/distributed/fleet/base/runtime_factory.py +++ b/python/paddle/distributed/fleet/base/runtime_factory.py @@ -18,7 +18,6 @@ class RuntimeFactory(object): - def __init__(self): pass diff --git a/python/paddle/distributed/fleet/base/strategy_compiler.py b/python/paddle/distributed/fleet/base/strategy_compiler.py index fa9dc1e0cdc14..14db852f7708d 100644 --- a/python/paddle/distributed/fleet/base/strategy_compiler.py +++ b/python/paddle/distributed/fleet/base/strategy_compiler.py @@ -107,7 +107,6 @@ def maximum_path_len_algo(optimizer_list): class StrategyCompilerBase(object): - def __init__(self): pass @@ -143,6 +142,7 @@ def _get_applied_graph_list(self): def _get_valid_strategy(self, dist_strategy, can_not_apply_optimizer_list): import copy + valid_strategy = copy.deepcopy(dist_strategy) invalid_optimizers = [] for candidate in self._meta_optimizer_candidates: @@ -171,9 +171,15 @@ def _get_valid_strategy(self, dist_strategy, can_not_apply_optimizer_list): we will remove grad fusion and sync batch-norm """ - def generate_optimizer(self, loss, role_maker, optimizer, - user_defined_strategy, meta_optimizer_list, - graph_optimizer_list): + def generate_optimizer( + self, + loss, + role_maker, + optimizer, + user_defined_strategy, + meta_optimizer_list, + graph_optimizer_list, + ): self._user_defined_strategy = user_defined_strategy self._meta_optimizer_candidates = meta_optimizer_list self._graph_optimizer_candidates = graph_optimizer_list @@ -190,12 +196,19 @@ def generate_optimizer(self, loss, role_maker, optimizer, # and graph_optimizer, the corresponding distributed strategy # should be updated. - self._meta_optimizers = [] if meta_optimizers is None else meta_optimizers - self._graph_optimizers = [] if graph_optimizers is None else graph_optimizers + self._meta_optimizers = ( + [] if meta_optimizers is None else meta_optimizers + ) + self._graph_optimizers = ( + [] if graph_optimizers is None else graph_optimizers + ) - return_meta = None if meta_optimizers == None else meta_optimizers[0] - return_graph = None if graph_optimizers == None else graph_optimizers[ - 0] + return_meta = ( + None if meta_optimizers == None else meta_optimizers[0] + ) + return_graph = ( + None if graph_optimizers == None else graph_optimizers[0] + ) if meta_optimizers == None or graph_optimizers == None: return return_meta, return_graph @@ -205,7 +218,10 @@ def generate_optimizer(self, loss, role_maker, optimizer, need_graph_opt = True for graph_opt in graph_optimizers: for program_opt in meta_optimizers: - if graph_opt.__class__.__name__ in program_opt.meta_optimizers_black_list: + if ( + graph_opt.__class__.__name__ + in program_opt.meta_optimizers_black_list + ): need_graph_opt = False if not need_graph_opt: return_graph = None diff --git a/python/paddle/distributed/fleet/base/strategy_group.py b/python/paddle/distributed/fleet/base/strategy_group.py index 94ab04b6c4e93..5b0ac3cfe0e40 100644 --- a/python/paddle/distributed/fleet/base/strategy_group.py +++ b/python/paddle/distributed/fleet/base/strategy_group.py @@ -15,7 +15,7 @@ import paddle.distributed as dist -class StrategyGroupBase(): +class StrategyGroupBase: """ The base class of communication group with distributed strategy. @@ -39,7 +39,8 @@ class StrategyGroupBase(): """ def __init__(self, list_of_ranks): - assert dist.is_initialized( + assert ( + dist.is_initialized() ), "The global communication group need to be initialized." assert len(list_of_ranks), "The list_of_ranks can not be empty." self._rank = dist.get_rank() @@ -57,8 +58,9 @@ def world_size(self): world_size_list = [] for ranks in self._list_of_ranks: world_size_list.append(len(ranks)) - is_value = all(world_size == world_size_list[0] - for world_size in world_size_list) + is_value = all( + world_size == world_size_list[0] for world_size in world_size_list + ) return world_size_list[0] if is_value else world_size_list @property @@ -77,10 +79,11 @@ def _create_group(self): group = dist.new_group(ranks=ranks) if self._rank in ranks: list_of_group.append(group) - assert len( - list_of_group - ) > 0, "Rank {} does not belong to the list_of_ranks {}.".format( - self._rank, self._list_of_ranks) + assert ( + len(list_of_group) > 0 + ), "Rank {} does not belong to the list_of_ranks {}.".format( + self._rank, self._list_of_ranks + ) return list_of_group if len(list_of_group) > 1 else list_of_group[0] @@ -99,8 +102,8 @@ class DPGroup(StrategyGroupBase): def __init__(self, list_of_ranks): super(DPGroup, self).__init__(list_of_ranks) assert not isinstance( - self.group, list), "Rank {} belongs to multi dp groups".format( - self._rank) + self.group, list + ), "Rank {} belongs to multi dp groups".format(self._rank) class MPGroup(StrategyGroupBase): @@ -118,8 +121,8 @@ class MPGroup(StrategyGroupBase): def __init__(self, list_of_ranks): super(MPGroup, self).__init__(list_of_ranks) assert not isinstance( - self.group, list), "Rank {} belongs to multi mp groups".format( - self._rank) + self.group, list + ), "Rank {} belongs to multi mp groups".format(self._rank) class ShardingGroup(StrategyGroupBase): @@ -137,8 +140,8 @@ class ShardingGroup(StrategyGroupBase): def __init__(self, list_of_ranks): super(ShardingGroup, self).__init__(list_of_ranks) assert not isinstance( - self.group, - list), "Rank {} belongs to multi sharding groups".format(self._rank) + self.group, list + ), "Rank {} belongs to multi sharding groups".format(self._rank) class PPGroup(StrategyGroupBase): @@ -156,8 +159,8 @@ class PPGroup(StrategyGroupBase): def __init__(self, list_of_ranks): super(PPGroup, self).__init__(list_of_ranks) assert not isinstance( - self.group, list), "Rank {} belongs to multi pp groups".format( - self._rank) + self.group, list + ), "Rank {} belongs to multi pp groups".format(self._rank) self._send_next_group = None self._send_prev_group = None @@ -197,7 +200,12 @@ def p2p_groups(self): Returns: Four subgroups including send/recv to/from prev/next. """ - return self._send_next_group, self._send_prev_group, self._recv_next_group, self._recv_prev_group + return ( + self._send_next_group, + self._send_prev_group, + self._recv_next_group, + self._recv_prev_group, + ) def _create_p2p_group(self): degree = self.world_size @@ -223,5 +231,11 @@ def _create_p2p_group(self): elif self._rank == prev_rank: self._recv_next_group = prev_group - assert self._send_next_group and self._send_prev_group and self._recv_next_group and self._recv_prev_group,\ - "Error occurs while creating p2p group for rank {}.".format(self._rank) + assert ( + self._send_next_group + and self._send_prev_group + and self._recv_next_group + and self._recv_prev_group + ), "Error occurs while creating p2p group for rank {}.".format( + self._rank + ) diff --git a/python/paddle/distributed/fleet/base/topology.py b/python/paddle/distributed/fleet/base/topology.py index d679894d3e208..6fa1521d58c46 100644 --- a/python/paddle/distributed/fleet/base/topology.py +++ b/python/paddle/distributed/fleet/base/topology.py @@ -40,6 +40,7 @@ class ParallelMode(object): print(parallel_mode.DATA_PARALLEL) # 0 """ + DATA_PARALLEL = 0 TENSOR_PARALLEL = 1 PIPELINE_PARALLEL = 2 @@ -47,14 +48,16 @@ class ParallelMode(object): class CommunicateTopology(object): - - def __init__(self, - hybrid_group_names=["data", "pipe", "sharding", "model"], - dims=[1, 1, 1, 1]): + def __init__( + self, + hybrid_group_names=["data", "pipe", "sharding", "model"], + dims=[1, 1, 1, 1], + ): self._parallel_names = hybrid_group_names self._dims = dims - self.coordinate = collections.namedtuple('Coordinate', - self._parallel_names) + self.coordinate = collections.namedtuple( + 'Coordinate', self._parallel_names + ) self._world_size = reduce(lambda x, y: x * y, self._dims) ranges = [range(d) for d in self._dims] @@ -62,7 +65,8 @@ def __init__(self, self._coord2rank = dict(zip(all_coordinate, range(len(all_coordinate)))) self._rank2coord = dict( - zip(self._coord2rank.values(), self._coord2rank.keys())) + zip(self._coord2rank.values(), self._coord2rank.keys()) + ) def get_hybrid_group_names(self): return self._parallel_names @@ -87,7 +91,8 @@ def get_coord(self, rank): def get_axis_list(self, axis_name, index): axis = self._parallel_names.index(axis_name) ranks = [ - self._coord2rank[coord] for coord in self._coord2rank.keys() + self._coord2rank[coord] + for coord in self._coord2rank.keys() if coord[axis] == index ] ranks.sort() @@ -129,7 +134,6 @@ def get_rank_from_stage(self, global_rank, **kwargs): class HybridCommunicateGroup(object): - def __init__(self, topology): self.nranks = paddle.distributed.get_world_size() self.global_rank = paddle.distributed.get_rank() @@ -145,10 +149,16 @@ def __init__(self, topology): self._sharding_parallel_id = self._get_sharding_parallel_id() self.stage_id = self._get_pipe_parallel_id() - assert self._check_vaild_topo( - ), "Here is an unreasonable topogy setting. world_size: {}, but" \ - "mp_num: {}, sharding_num: {}, pp_num: {}, dp_num: {}".format(self.nranks, - self._mp_degree, self._sharding_degree, self._pp_degree, self._dp_degree) + assert self._check_vaild_topo(), ( + "Here is an unreasonable topogy setting. world_size: {}, but" + "mp_num: {}, sharding_num: {}, pp_num: {}, dp_num: {}".format( + self.nranks, + self._mp_degree, + self._sharding_degree, + self._pp_degree, + self._dp_degree, + ) + ) # create comm group for data parallel self._dp_group, self._dp_comm_group = self._set_comm_group("data") @@ -161,26 +171,43 @@ def __init__(self, topology): # create comm group for sharding parallel self._sharding_group, self._sharding_comm_group = self._set_comm_group( - "sharding") + "sharding" + ) # create global group for check inf_nan / clip global norm self._check_group, self._check_comm_group = self._set_check_group( - "data") + "data" + ) # create p2p group - self.is_first_stage = (self.stage_id == 0) - self.is_last_stage = (self.stage_id == (self._pp_degree - 1)) + self.is_first_stage = self.stage_id == 0 + self.is_last_stage = self.stage_id == (self._pp_degree - 1) # create p2p_groups if self._pp_degree > 1: self._set_p2p_group() - debug_str = "HybridParallelInfo: rank_id: %d, mp_degree: %d, " \ - "sharding_degree: %d, pp_degree: %d, dp_degree: %d" % (self.global_rank, self._mp_degree, - self._sharding_degree, self._pp_degree, self._dp_degree) - debug_str += ", mp_group: %s, sharding_group: %s, pp_group: %s, dp_group: %s, check/clip group: %s" % ( - self._mp_group, self._sharding_group, self._pp_group, - self._dp_group, self._check_group) + debug_str = ( + "HybridParallelInfo: rank_id: %d, mp_degree: %d, " + "sharding_degree: %d, pp_degree: %d, dp_degree: %d" + % ( + self.global_rank, + self._mp_degree, + self._sharding_degree, + self._pp_degree, + self._dp_degree, + ) + ) + debug_str += ( + ", mp_group: %s, sharding_group: %s, pp_group: %s, dp_group: %s, check/clip group: %s" + % ( + self._mp_group, + self._sharding_group, + self._pp_group, + self._dp_group, + self._check_group, + ) + ) logger.info(debug_str) global _HYBRID_PARALLEL_GROUP @@ -192,7 +219,12 @@ def get_parallel_mode(self): # adding its parallel logic within that parallelism # when use sharding alone, it should have its own parallelism for its parallel logic # TODO modify 3 others parallel to support sharding - if self._mp_degree == 1 and self._pp_degree == 1 and self._dp_degree == 1 and self._sharding_degree > 1: + if ( + self._mp_degree == 1 + and self._pp_degree == 1 + and self._dp_degree == 1 + and self._sharding_degree > 1 + ): return ParallelMode.SHARDING_PARALLEL elif self._mp_degree == 1 and self._pp_degree == 1: return ParallelMode.DATA_PARALLEL @@ -203,7 +235,13 @@ def get_parallel_mode(self): return ParallelMode.PIPELINE_PARALLEL def _check_vaild_topo(self): - return self._dp_degree * self._mp_degree * self._pp_degree * self._sharding_degree == self.nranks + return ( + self._dp_degree + * self._mp_degree + * self._pp_degree + * self._sharding_degree + == self.nranks + ) def _set_comm_group(self, parallel_method="data"): parallel_group = [] @@ -265,14 +303,16 @@ def _set_p2p_group(self): self.prev_rank = prev_rank next_group = paddle.distributed.new_group( - ranks=[curr_rank, next_rank]) + ranks=[curr_rank, next_rank] + ) if self.global_rank == curr_rank: self.send_next_group = next_group elif self.global_rank == next_rank: self.recv_prev_group = next_group prev_group = paddle.distributed.new_group( - ranks=[prev_rank, curr_rank]) + ranks=[prev_rank, curr_rank] + ) if self.global_rank == curr_rank: self.send_prev_group = prev_group @@ -336,7 +376,12 @@ def get_pipe_parallel_group(self): return self._pp_comm_group def get_p2p_groups(self): - return self.send_next_group, self.send_prev_group, self.recv_next_group, self.recv_prev_group + return ( + self.send_next_group, + self.send_prev_group, + self.recv_next_group, + self.recv_prev_group, + ) # sharding parallel message: def _get_sharding_parallel_id(self): @@ -360,23 +405,25 @@ def get_check_parallel_group(self): return self._check_comm_group def get_rank_from_stage(self, stage_id, **kwargs): - return self._topo.get_rank_from_stage(self.global_rank, - pipe=stage_id, - **kwargs) + return self._topo.get_rank_from_stage( + self.global_rank, pipe=stage_id, **kwargs + ) class _CommunicateGroup(object): - """ tmp for static """ + """tmp for static""" def __init__(self): global _HYBRID_PARALLEL_GROUP _HYBRID_PARALLEL_GROUP = self self.groups = dict() - def set_comm_group(self, group_name, group_rank, group_size, ring_id, - group_ranks): - group = paddle.distributed.collective.Group(group_rank, ring_id, - group_ranks) + def set_comm_group( + self, group_name, group_rank, group_size, ring_id, group_ranks + ): + group = paddle.distributed.collective.Group( + group_rank, ring_id, group_ranks + ) self.groups[group_name] = group def get_group(self, group_name): diff --git a/python/paddle/distributed/fleet/base/util_factory.py b/python/paddle/distributed/fleet/base/util_factory.py index 95d287811e6cb..5af1acb941358 100755 --- a/python/paddle/distributed/fleet/base/util_factory.py +++ b/python/paddle/distributed/fleet/base/util_factory.py @@ -32,7 +32,6 @@ class UtilFactory(object): - def _create_util(self, context=None): util = UtilBase() if context is not None and "valid_strategy" in context: @@ -43,7 +42,6 @@ def _create_util(self, context=None): class UtilBase(object): - def __init__(self): self.role_maker = None self.dist_strategy = None @@ -221,7 +219,7 @@ def get_heter_file_shard(self, files): trainer_files = [[]] * trainers begin = 0 for i in range(trainers): - trainer_files[i] = files[begin:begin + blocks[i]] + trainer_files[i] = files[begin : begin + blocks[i]] begin += blocks[i] return trainer_files[trainer_id] @@ -279,7 +277,7 @@ def get_file_shard(self, files): trainer_files = [[]] * trainers begin = 0 for i in range(trainers): - trainer_files[i] = files[begin:begin + blocks[i]] + trainer_files[i] = files[begin : begin + blocks[i]] begin += blocks[i] return trainer_files[trainer_id] @@ -323,7 +321,6 @@ def _save_program(self, program, model_filename='__model__', is_text=False): f.write(program.desc.serialize_to_string()) def _load_program(self, path, is_text): - def load_program_binary(path): """load program from binary string file""" with open(path, "rb") as f: @@ -347,8 +344,9 @@ def load_program_text(path): def _program_type_trans(self, prog_dir, prog_fn, is_text): prog = self._load_program(os.path.join(prog_dir, prog_fn), is_text) prog_out_fn = prog_fn + ".bin" if is_text else prog_fn + ".pbtxt" - self._save_program(prog, os.path.join(prog_dir, prog_out_fn), - 1 - is_text) + self._save_program( + prog, os.path.join(prog_dir, prog_out_fn), 1 - is_text + ) return prog_out_fn def _visualize_graphviz(self, program, output_dir, output_filename): @@ -357,29 +355,37 @@ def _visualize_graphviz(self, program, output_dir, output_filename): pdf_path = os.path.join(output_dir, output_filename + '.pdf') debugger.draw_block_graphviz(block, path=dot_path) cmd = ["dot", "-Tpdf", dot_path, "-o", pdf_path] - p = subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) p.wait() def _proto_check(self, config): - train_prog = self._load_program(config.train_prog_path, - config.is_text_train_program) - pruned_prog = self._load_program(config.pruned_prog_path, - config.is_text_pruned_program) + train_prog = self._load_program( + config.train_prog_path, config.is_text_train_program + ) + pruned_prog = self._load_program( + config.pruned_prog_path, config.is_text_pruned_program + ) is_match = True - pruned_vars = [(v.name, v) for v in pruned_prog.list_vars() - if fluid.io.is_persistable(v)] + pruned_vars = [ + (v.name, v) + for v in pruned_prog.list_vars() + if fluid.io.is_persistable(v) + ] pruned_vars = OrderedDict(pruned_vars) pruned_vars_name = [name for name in pruned_vars] print("persistable vars in pruned program: {}".format(pruned_vars_name)) # feed and fetch op is added in pruned program when pruning, not need to be found in train program feed_fetch_type_list = [ - core.VarDesc.VarType.FEED_MINIBATCH, core.VarDesc.VarType.FETCH_LIST + core.VarDesc.VarType.FEED_MINIBATCH, + core.VarDesc.VarType.FETCH_LIST, ] for var_name in pruned_vars: @@ -392,21 +398,28 @@ def _proto_check(self, config): except ValueError as e: print( "Not find variable '%s' in train program. please check pruning." - % var_name) + % var_name + ) is_match = False continue - if var.shape != train_prog_var.shape or var.dtype != train_prog_var.dtype: + if ( + var.shape != train_prog_var.shape + or var.dtype != train_prog_var.dtype + ): print( - "variable: {} not match. in pruned program shape: {} dtype:{}, in train program shape: {} dtype: {}" - .format(var_name, var.shape, var.dtype, - train_prog_var.shape, train_prog_var.dtype)) + "variable: {} not match. in pruned program shape: {} dtype:{}, in train program shape: {} dtype: {}".format( + var_name, + var.shape, + var.dtype, + train_prog_var.shape, + train_prog_var.dtype, + ) + ) is_match = False return is_match def _params_check(self, config): - def feed_gen(batch_size, feeded_vars_dims, feeded_vars_filelist): - def reader(batch_size, fn, dim): data = [] if isinstance(dim, list) or isinstance(dim, tuple): @@ -437,39 +450,57 @@ def reader(batch_size, fn, dim): prog = self._load_program( os.path.join(config.dump_model_dir, config.dump_program_filename), - config.is_text_dump_program) + config.is_text_dump_program, + ) if config.is_text_dump_program: model_filename = self._program_type_trans( - config.dump_model_dir, config.dump_program_filename, - config.is_text_dump_program) + config.dump_model_dir, + config.dump_program_filename, + config.is_text_dump_program, + ) saved_params = [ v for v in prog.list_vars() if fluid.io.is_persistable(v) ] - print("persistable vars in dump program: {}".format( - [v.name for v in saved_params])) + print( + "persistable vars in dump program: {}".format( + [v.name for v in saved_params] + ) + ) def check_not_expected_ops(prog, not_expected_op_types): op_types_set = set() for op in prog.global_block().ops: - if op.type in not_expected_op_types and op.type not in op_types_set: + if ( + op.type in not_expected_op_types + and op.type not in op_types_set + ): op_types_set.add(op.type) return op_types_set not_expected_op_types = check_not_expected_ops(prog, ["lookup_table"]) if len(not_expected_op_types) > 0: print( - "find op type '{}' in program, please check if your program is pruned correctly !" - .format(list(not_expected_op_types))) + "find op type '{}' in program, please check if your program is pruned correctly !".format( + list(not_expected_op_types) + ) + ) return False place = fluid.CPUPlace() exe = fluid.Executor(place) scope = fluid.core.Scope() with fluid.scope_guard(scope): - inference_program, feed_target_names, fetch_targets = \ - fluid.io.load_inference_model(config.dump_model_dir, exe, model_filename=model_filename, - params_filename=config.save_params_filename) + ( + inference_program, + feed_target_names, + fetch_targets, + ) = fluid.io.load_inference_model( + config.dump_model_dir, + exe, + model_filename=model_filename, + params_filename=config.save_params_filename, + ) # check program vars and saved vars shape orig_para_shape = { @@ -480,13 +511,17 @@ def check_not_expected_ops(prog, not_expected_op_types): var_temp = fluid.global_scope().find_var(each_var.name) assert var_temp != None, "can't not find var: " + each_var.name new_shape = (np.array(var_temp.get_tensor())).shape - assert each_var.name in orig_para_shape, each_var.name + "MUST in var list" + assert each_var.name in orig_para_shape, ( + each_var.name + "MUST in var list" + ) orig_shape = orig_para_shape.get(each_var.name) if new_shape != orig_shape: raise RuntimeError( "Shape not matching: the Program requires a parameter with a shape of ({}), " - "while the loaded parameter (namely [ {} ]) has a shape of ({})." - .format(orig_shape, each_var.name, new_shape)) + "while the loaded parameter (namely [ {} ]) has a shape of ({}).".format( + orig_shape, each_var.name, new_shape + ) + ) # check feed/fetch vars in program and config feed_config = config.feed_config @@ -498,10 +533,15 @@ def check_not_expected_ops(prog, not_expected_op_types): print("warning! no fetch targets in program.") fetch_list = fetch_targets feed_name_list = feed_target_names - if feed_config.feeded_vars_names is not None and feed_target_names != feed_config.feeded_vars_names: + if ( + feed_config.feeded_vars_names is not None + and feed_target_names != feed_config.feeded_vars_names + ): print( - "warning! feed vars in program and config are diff: feed in program: {}. feed in config {}." - .format(feed_target_names, feed_config.feeded_vars_names)) + "warning! feed vars in program and config are diff: feed in program: {}. feed in config {}.".format( + feed_target_names, feed_config.feeded_vars_names + ) + ) feed_name_list = feed_config.feeded_vars_names # remove feed op in inference_program. new feed op will be added in exe.run global_block = inference_program.global_block() @@ -512,10 +552,15 @@ def check_not_expected_ops(prog, not_expected_op_types): need_to_remove_op_index.append(i) for index in need_to_remove_op_index[::-1]: global_block._remove_op(index) - if fetch_config.fetch_vars_names is not None and fetch_targets_names != fetch_config.fetch_vars_names: + if ( + fetch_config.fetch_vars_names is not None + and fetch_targets_names != fetch_config.fetch_vars_names + ): print( - "warning! fetch vars in program and config are diff: fetch in program: {}. fetch in config {}." - .format(fetch_targets_names, fetch_config.fetch_vars_names)) + "warning! fetch vars in program and config are diff: fetch in program: {}. fetch in config {}.".format( + fetch_targets_names, fetch_config.fetch_vars_names + ) + ) fetch_list = [ inference_program.global_block().var(i) for i in fetch_config.fetch_vars_names @@ -535,75 +580,105 @@ def check_not_expected_ops(prog, not_expected_op_types): # try dump fetch_targets feed_tensors = [] - assert len(feed_config.feeded_vars_names) == len( - feed_config.feeded_vars_dims) == len( - feed_config.feeded_vars_types) + assert ( + len(feed_config.feeded_vars_names) + == len(feed_config.feeded_vars_dims) + == len(feed_config.feeded_vars_types) + ) # check program vars and feed tensor shape in config for i in range(len(feed_config.feeded_vars_names)): var = inference_program.global_block().var( - feed_config.feeded_vars_names[i]) - if not isinstance(feed_config.feeded_vars_dims[i], - (list, tuple)): - tensor_shape = (feed_config.feeded_vars_dims[i], ) + feed_config.feeded_vars_names[i] + ) + if not isinstance( + feed_config.feeded_vars_dims[i], (list, tuple) + ): + tensor_shape = (feed_config.feeded_vars_dims[i],) else: tensor_shape = tuple(feed_config.feeded_vars_dims[i]) feed_config.feeded_vars_dims[i] = tensor_shape var_shape = var.shape[1:] if tensor_shape != var_shape: raise RuntimeError( - "feed variable '{}' shape not match. infer program shape: {}. feed tensor shape: {}" - .format(feed_config.feeded_vars_names[i], var_shape, - tensor_shape)) + "feed variable '{}' shape not match. infer program shape: {}. feed tensor shape: {}".format( + feed_config.feeded_vars_names[i], + var_shape, + tensor_shape, + ) + ) if not feed_config.feeded_vars_filelist: print("generate random feed vars.") for i in range(len(feed_config.feeded_vars_names)): var = inference_program.global_block().var( - feed_config.feeded_vars_names[i]) + feed_config.feeded_vars_names[i] + ) # create fake feed tensor. if lod_level > 1, should create_lod_tensor() if var.lod_level == 0: feed_tensors.append( - np.array(np.random.random( - tuple([config.batch_size] + - list(feed_config.feeded_vars_dims[i]))), - dtype=feed_config.feeded_vars_types[i])) + np.array( + np.random.random( + tuple( + [config.batch_size] + + list(feed_config.feeded_vars_dims[i]) + ) + ), + dtype=feed_config.feeded_vars_types[i], + ) + ) elif var.lod_level == 1: - t = np.array(np.random.random( - tuple([config.batch_size] + - list(feed_config.feeded_vars_dims[i]))), - dtype=feed_config.feeded_vars_types[i]) + t = np.array( + np.random.random( + tuple( + [config.batch_size] + + list(feed_config.feeded_vars_dims[i]) + ) + ), + dtype=feed_config.feeded_vars_types[i], + ) feed_tensors.append( - fluid.create_lod_tensor(t, - [[1] * config.batch_size], - place)) + fluid.create_lod_tensor( + t, [[1] * config.batch_size], place + ) + ) else: raise RuntimeError( "vars with lod_level >= 2 is not supported now in this infer program check tool." ) - results = exe.run(inference_program, - feed={ - name: feed_tensors[i] - for i, name in enumerate(feed_name_list) - }, - fetch_list=fetch_list, - return_numpy=return_numpy) + results = exe.run( + inference_program, + feed={ + name: feed_tensors[i] + for i, name in enumerate(feed_name_list) + }, + fetch_list=fetch_list, + return_numpy=return_numpy, + ) else: - print("load feed vars from files: {}.".format( - feed_config.feeded_vars_filelist)) + print( + "load feed vars from files: {}.".format( + feed_config.feeded_vars_filelist + ) + ) feed_vars = [ inference_program.global_block().var( - feed_config.feeded_vars_names[i]) + feed_config.feeded_vars_names[i] + ) for i in range(len(feed_config.feeded_vars_names)) ] feeder = fluid.DataFeeder(feed_list=feed_vars, place=place) - batch_feed = feed_gen(config.batch_size, - feed_config.feeded_vars_dims, - feed_config.feeded_vars_filelist) + batch_feed = feed_gen( + config.batch_size, + feed_config.feeded_vars_dims, + feed_config.feeded_vars_filelist, + ) slots = [batch_feed] - results = exe.run(inference_program, - feed=feeder.feed(slots), - fetch_list=fetch_list, - return_numpy=return_numpy) + results = exe.run( + inference_program, + feed=feeder.feed(slots), + fetch_list=fetch_list, + return_numpy=return_numpy, + ) for i, v in enumerate(fetch_list): print("fetch_targets name: %s" % v.name) print("fetch_targets: {}".format(results[i])) diff --git a/python/paddle/distributed/fleet/cloud_utils.py b/python/paddle/distributed/fleet/cloud_utils.py index cfd8a9ff4e2e7..cf74f8446705f 100644 --- a/python/paddle/distributed/fleet/cloud_utils.py +++ b/python/paddle/distributed/fleet/cloud_utils.py @@ -18,14 +18,13 @@ __all__ = [] -def get_cloud_cluster(args_node_ips, - device_mode, - devices_per_proc, - args_port=6170): +def get_cloud_cluster( + args_node_ips, device_mode, devices_per_proc, args_port=6170 +): """ args_node_ips:string, device_mode:DeviceMode(Int), device_per_proc:list, args_port: int """ - #you can automatically get ip info while using paddlecloud multi nodes mode. + # you can automatically get ip info while using paddlecloud multi nodes mode. node_ips = os.getenv("PADDLE_TRAINERS") assert node_ips is not None, "PADDLE_TRAINERS should not be None" @@ -47,7 +46,10 @@ def get_cloud_cluster(args_node_ips, "Please NOTE: When using paddlecloud, cluster_node_ips is \ automatically got from PADDLE_TRAINERS(multi nodes) or POD_IP(single node).\ Your input cluster_node_ips: {} doesn't equals to IPs: {} from \ -paddlecloud environment.".format(args_node_ips, node_ips)) +paddlecloud environment.".format( + args_node_ips, node_ips + ) + ) # DISTRIBUTED_TRAINER_ENDPOINTS: new environment since paddlecloud 1.8.4 # e.g: DISTRIBUTED_TRAINER_ENDPOINTS="ip1:port1,ip1:port2,ip1:port3,ip1:port4,ip2:port5,ip2:port6,ip2:port7,ip2:port8" @@ -58,10 +60,13 @@ def get_cloud_cluster(args_node_ips, try: paddle_port = int(os.getenv("PADDLE_PORT", "")) - if paddle_ports_num >= len( - devices_per_proc) and paddle_port != args_port: + if ( + paddle_ports_num >= len(devices_per_proc) + and paddle_port != args_port + ): logger.warning( - "Use Cloud specified port:{}.".format(paddle_port)) + "Use Cloud specified port:{}.".format(paddle_port) + ) started_port = paddle_port except Exception as e: @@ -81,15 +86,21 @@ def get_cloud_cluster(args_node_ips, assert num_nodes * paddle_ports_num == len(trainer_endpoints_ori) for i in range(num_nodes): trainer_endpoints.append( - trainer_endpoints_ori[i * paddle_ports_num:(i + 1) * - paddle_ports_num]) + trainer_endpoints_ori[ + i * paddle_ports_num : (i + 1) * paddle_ports_num + ] + ) - logger.debug("parsed from args: node_ips:{} \ + logger.debug( + "parsed from args: node_ips:{} \ node_ip:{} node_rank:{} trainer_endpoints:{}".format( - node_ips, node_ip, node_rank, trainer_endpoints)) + node_ips, node_ip, node_rank, trainer_endpoints + ) + ) - cluster, pod = get_cluster(node_ips, node_ip, trainer_endpoints, - device_mode, devices_per_proc) + cluster, pod = get_cluster( + node_ips, node_ip, trainer_endpoints, device_mode, devices_per_proc + ) return cluster, cluster.pods[node_rank] @@ -98,7 +109,12 @@ def use_paddlecloud(): node_ip = os.getenv("POD_IP") node_rank = os.getenv("PADDLE_TRAINER_ID") paddle_ports_num = os.getenv("TRAINER_PORTS_NUM") - if node_ips is None or node_ip is None or node_rank is None or paddle_ports_num is None: + if ( + node_ips is None + or node_ip is None + or node_rank is None + or paddle_ports_num is None + ): return False else: return True diff --git a/python/paddle/distributed/fleet/data_generator/data_generator.py b/python/paddle/distributed/fleet/data_generator/data_generator.py index d43c376bb0c01..9e9ec7b61bc46 100644 --- a/python/paddle/distributed/fleet/data_generator/data_generator.py +++ b/python/paddle/distributed/fleet/data_generator/data_generator.py @@ -147,7 +147,8 @@ def _gen_str(self, line): Return a string data that can be read directly by the datafeed. ''' raise NotImplementedError( - "pls use MultiSlotDataGenerator or PairWiseDataGenerator") + "pls use MultiSlotDataGenerator or PairWiseDataGenerator" + ) def generate_sample(self, line): ''' @@ -187,8 +188,9 @@ def local_iter(): ''' raise NotImplementedError( - "Please rewrite this function to return a list or tuple: " + - "[(name, [feasign, ...]), ...] or ((name, [feasign, ...]), ...)") + "Please rewrite this function to return a list or tuple: " + + "[(name, [feasign, ...]), ...] or ((name, [feasign, ...]), ...)" + ) def generate_batch(self, samples): ''' @@ -236,7 +238,6 @@ def local_iter(): # add more generalized DataGenerator that can adapt user-defined slot # for example, [(name, float_list), (name, str_list), (name, int_list)] class MultiSlotStringDataGenerator(DataGenerator): - def _gen_str(self, line): ''' Further processing the output of the process() function rewritten by @@ -267,7 +268,8 @@ def _gen_str(self, line): if not isinstance(line, list) and not isinstance(line, tuple): raise ValueError( "the output of process() must be in list or tuple type" - "Examples: [('words', ['1926', '08', '17']), ('label', ['1'])]") + "Examples: [('words', ['1926', '08', '17']), ('label', ['1'])]" + ) output = "" for index, item in enumerate(line): name, elements = item @@ -281,7 +283,6 @@ def _gen_str(self, line): class MultiSlotDataGenerator(DataGenerator): - def _gen_str(self, line): ''' Further processing the output of the process() function rewritten by @@ -316,7 +317,8 @@ def _gen_str(self, line): if not isinstance(line, list) and not isinstance(line, tuple): raise ValueError( "the output of process() must be in list or tuple type" - "Example: [('words', [1926, 08, 17]), ('label', [1])]") + "Example: [('words', [1926, 08, 17]), ('label', [1])]" + ) output = "" if self._proto_info is None: @@ -326,8 +328,9 @@ def _gen_str(self, line): if not isinstance(name, str): raise ValueError("name%s must be in str type" % type(name)) if not isinstance(elements, list): - raise ValueError("elements%s must be in list type" % - type(elements)) + raise ValueError( + "elements%s must be in list type" % type(elements) + ) if not elements: raise ValueError( "the elements of each field can not be empty, you need padding it in process()." @@ -340,10 +343,12 @@ def _gen_str(self, line): if isinstance(elem, float): self._proto_info[-1] = (name, "float") elif not isinstance(elem, int) and not isinstance( - elem, long): + elem, long + ): raise ValueError( - "the type of element%s must be in int or float" % - type(elem)) + "the type of element%s must be in int or float" + % type(elem) + ) output += " " + str(elem) else: if len(line) != len(self._proto_info): @@ -355,8 +360,9 @@ def _gen_str(self, line): if not isinstance(name, str): raise ValueError("name%s must be in str type" % type(name)) if not isinstance(elements, list): - raise ValueError("elements%s must be in list type" % - type(elements)) + raise ValueError( + "elements%s must be in list type" % type(elements) + ) if not elements: raise ValueError( "the elements of each field can not be empty, you need padding it in process()." @@ -364,7 +370,8 @@ def _gen_str(self, line): if name != self._proto_info[index][0]: raise ValueError( "the field name of two given line are not match: require<%s>, get<%s>." - % (self._proto_info[index][0], name)) + % (self._proto_info[index][0], name) + ) if output: output += " " output += str(len(elements)) @@ -373,9 +380,11 @@ def _gen_str(self, line): if isinstance(elem, float): self._proto_info[index] = (name, "float") elif not isinstance(elem, int) and not isinstance( - elem, long): + elem, long + ): raise ValueError( "the type of element%s must be in int or float" - % type(elem)) + % type(elem) + ) output += " " + str(elem) return output + "\n" diff --git a/python/paddle/distributed/fleet/dataset/dataset.py b/python/paddle/distributed/fleet/dataset/dataset.py index 58e265c834723..b74f700391cf5 100755 --- a/python/paddle/distributed/fleet/dataset/dataset.py +++ b/python/paddle/distributed/fleet/dataset/dataset.py @@ -21,10 +21,10 @@ class DatasetBase(object): - """ Base dataset class. """ + """Base dataset class.""" def __init__(self): - """ Init. """ + """Init.""" # define class name here # to decide whether we need create in memory instance self.proto_desc = data_feed_pb2.DataFeedDesc() @@ -35,15 +35,17 @@ def __init__(self): self.use_ps_gpu = False self.psgpu = None - def init(self, - batch_size=1, - thread_num=1, - use_var=[], - pipe_command="cat", - input_type=0, - fs_name="", - fs_ugi="", - download_cmd="cat"): + def init( + self, + batch_size=1, + thread_num=1, + use_var=[], + pipe_command="cat", + input_type=0, + fs_name="", + fs_ugi="", + download_cmd="cat", + ): """ should be called only once in user's python scripts to initialize setings of dataset instance. Normally, it is called by InMemoryDataset or QueueDataset. @@ -271,8 +273,9 @@ def _dynamic_adjust_before_train(self, thread_num): def _dynamic_adjust_after_train(self): pass - def _check_use_var_with_data_generator(self, var_list, data_generator_class, - test_file): + def _check_use_var_with_data_generator( + self, var_list, data_generator_class, test_file + ): """ Var consistency insepection of use_var_list and data_generator data. @@ -304,32 +307,38 @@ def _check_use_var_with_data_generator(self, var_list, data_generator_class, if var_len != data_gen_len: raise ValueError( "var length mismatch error: var_list = %s vs data_generator = %s" - % (var_len, data_gen_len)) + % (var_len, data_gen_len) + ) for i, ele in enumerate(user_parsed_line): if len(ele[1]) == 0: raise ValueError( "var length error: var %s's length in data_generator is 0" - % ele[0]) + % ele[0] + ) if var_list[ - i].dtype == core.VarDesc.VarType.FP32 and not all( - isinstance(ele, float) for ele in ele[1]): + i + ].dtype == core.VarDesc.VarType.FP32 and not all( + isinstance(ele, float) for ele in ele[1] + ): raise TypeError( "var dtype mismatch error: var name = %s, var type in var_list = %s, while var in data_generator contains non-float value, which is %s \n" "Please check if order of var_list and data_generator are aligned. \n" "Please check if var's type in data_generator is correct." - % (ele[0], "float", ele[1])) + % (ele[0], "float", ele[1]) + ) - if (var_list[i].dtype == core.VarDesc.VarType.INT64 - or var_list[i].dtype - == core.VarDesc.VarType.INT32) and not all( - isinstance(ele, int) for ele in ele[1]): + if ( + var_list[i].dtype == core.VarDesc.VarType.INT64 + or var_list[i].dtype == core.VarDesc.VarType.INT32 + ) and not all(isinstance(ele, int) for ele in ele[1]): raise TypeError( "var dtype mismatch error: var name = %s, var type in var_list = %s, while var in data_generator contains non-int value, which is %s \n" "Please check if order of var_list and data_generator are aligned. \n" "Please check if var's type in data_generator is correct." - % (ele[0], "int", ele[1])) + % (ele[0], "int", ele[1]) + ) else: break @@ -353,7 +362,7 @@ class InMemoryDataset(DatasetBase): """ def __init__(self): - """ Init. """ + """Init.""" super(InMemoryDataset, self).__init__() self.proto_desc.name = "MultiSlotInMemoryDataFeed" self.fleet_send_batch_size = None @@ -588,14 +597,16 @@ def init(self, **kwargs): data_feed_type = "MultiSlotInMemoryDataFeed" self._set_feed_type(data_feed_type) - super(InMemoryDataset, self).init(batch_size=batch_size, - thread_num=thread_num, - use_var=use_var, - pipe_command=pipe_command, - input_type=input_type, - fs_name=fs_name, - fs_ugi=fs_ugi, - download_cmd=download_cmd) + super(InMemoryDataset, self).init( + batch_size=batch_size, + thread_num=thread_num, + use_var=use_var, + pipe_command=pipe_command, + input_type=input_type, + fs_name=fs_name, + fs_ugi=fs_ugi, + download_cmd=download_cmd, + ) if kwargs.get("queue_num", -1) > 0: queue_num = kwargs.get("queue_num", -1) @@ -606,7 +617,7 @@ def _set_feed_type(self, data_feed_type): Set data_feed_desc """ self.proto_desc.name = data_feed_type - if (self.proto_desc.name == "SlotRecordInMemoryDataFeed"): + if self.proto_desc.name == "SlotRecordInMemoryDataFeed": self.dataset = core.Dataset("SlotRecordDataset") def _prepare_to_run(self): @@ -779,11 +790,12 @@ def _set_generate_unique_feasigns(self, generate_uni_feasigns, shard_num): self.gen_uni_feasigns = generate_uni_feasigns self.local_shard_num = shard_num - def _generate_local_tables_unlock(self, table_id, fea_dim, read_thread_num, - consume_thread_num, shard_num): - self.dataset.generate_local_tables_unlock(table_id, fea_dim, - read_thread_num, - consume_thread_num, shard_num) + def _generate_local_tables_unlock( + self, table_id, fea_dim, read_thread_num, consume_thread_num, shard_num + ): + self.dataset.generate_local_tables_unlock( + table_id, fea_dim, read_thread_num, consume_thread_num, shard_num + ) def set_date(self, date): """ @@ -821,11 +833,25 @@ def set_date(self, date): if self.use_ps_gpu and core._is_compiled_with_heterps(): self.psgpu.set_date(year, month, day) - def tdm_sample(self, tree_name, tree_path, tdm_layer_counts, - start_sample_layer, with_hierachy, seed, id_slot): - self.dataset.tdm_sample(tree_name, tree_path, tdm_layer_counts, - start_sample_layer, with_hierachy, seed, - id_slot) + def tdm_sample( + self, + tree_name, + tree_path, + tdm_layer_counts, + start_sample_layer, + with_hierachy, + seed, + id_slot, + ): + self.dataset.tdm_sample( + tree_name, + tree_path, + tdm_layer_counts, + start_sample_layer, + with_hierachy, + seed, + id_slot, + ) def load_into_memory(self, is_shuffle=False): """ @@ -1112,12 +1138,14 @@ def get_memory_data_size(self, fleet=None): """ import numpy as np + local_data_size = self.dataset.get_memory_data_size() local_data_size = np.array([local_data_size]) if fleet is not None: global_data_size = local_data_size * 0 - fleet._role_maker.all_reduce_worker(local_data_size, - global_data_size) + fleet._role_maker.all_reduce_worker( + local_data_size, global_data_size + ) return global_data_size[0] return local_data_size[0] @@ -1166,12 +1194,14 @@ def get_shuffle_data_size(self, fleet=None): """ import numpy as np + local_data_size = self.dataset.get_shuffle_data_size() local_data_size = np.array([local_data_size]) if fleet is not None: global_data_size = local_data_size * 0 - fleet._role_maker.all_reduce_worker(local_data_size, - global_data_size) + fleet._role_maker.all_reduce_worker( + local_data_size, global_data_size + ) return global_data_size[0] return local_data_size[0] diff --git a/python/paddle/distributed/fleet/dataset/index_dataset.py b/python/paddle/distributed/fleet/dataset/index_dataset.py index 8b5a9c5a45bf6..1dda709fffd58 100644 --- a/python/paddle/distributed/fleet/dataset/index_dataset.py +++ b/python/paddle/distributed/fleet/dataset/index_dataset.py @@ -17,13 +17,11 @@ class Index(object): - def __init__(self, name): self._name = name class TreeIndex(Index): - def __init__(self, name, path): super(TreeIndex, self).__init__(name) self._wrapper = core.IndexWrapper() @@ -67,7 +65,7 @@ def get_children_codes(self, ancestor, level): def get_travel_path(self, child, ancestor): res = [] - while (child > ancestor): + while child > ancestor: res.append(child) child = int((child - 1) / self._branch) return res @@ -76,17 +74,18 @@ def get_pi_relation(self, ids, level): codes = self.get_ancestor_codes(ids, level) return dict(zip(ids, codes)) - def init_layerwise_sampler(self, - layer_sample_counts, - start_sample_layer=1, - seed=0): + def init_layerwise_sampler( + self, layer_sample_counts, start_sample_layer=1, seed=0 + ): assert self._layerwise_sampler is None self._layerwise_sampler = core.IndexSampler("by_layerwise", self._name) - self._layerwise_sampler.init_layerwise_conf(layer_sample_counts, - start_sample_layer, seed) + self._layerwise_sampler.init_layerwise_conf( + layer_sample_counts, start_sample_layer, seed + ) def layerwise_sample(self, user_input, index_input, with_hierarchy=False): if self._layerwise_sampler is None: raise ValueError("please init layerwise_sampler first.") - return self._layerwise_sampler.sample(user_input, index_input, - with_hierarchy) + return self._layerwise_sampler.sample( + user_input, index_input, with_hierarchy + ) diff --git a/python/paddle/distributed/fleet/elastic/__init__.py b/python/paddle/distributed/fleet/elastic/__init__.py index b80a66c6f01d0..447bedb21d592 100644 --- a/python/paddle/distributed/fleet/elastic/__init__.py +++ b/python/paddle/distributed/fleet/elastic/__init__.py @@ -25,12 +25,12 @@ def enable_elastic(args, distribute_mode): - #elastic_level = os.getenv('PADDLE_ELASTIC_FAULT_TOLERANC_LEVEL') - #if not elastic_level and (elastic_level != ElasticLevel.FAULT_TOLERANCE and + # elastic_level = os.getenv('PADDLE_ELASTIC_FAULT_TOLERANC_LEVEL') + # if not elastic_level and (elastic_level != ElasticLevel.FAULT_TOLERANCE and # elastic_level != ElasticLevel.ELASTIC): # return False - #if distribute_mode != DistributeMode.COLLECTIVE: + # if distribute_mode != DistributeMode.COLLECTIVE: # return False if not args.elastic_server and not os.getenv('PADDLE_ELASTIC_SERVER'): @@ -50,6 +50,7 @@ def launch_elastic(args, distribute_mode): server = args.elastic_server or os.getenv('PADDLE_ELASTIC_SERVER') srv, port = server.split(':') import etcd3 + etcd_client = etcd3.client(host=srv, port=port) elastic = ElasticManager(args, etcd_client) diff --git a/python/paddle/distributed/fleet/elastic/collective.py b/python/paddle/distributed/fleet/elastic/collective.py index aa5c962b99467..bdb8a6c577094 100644 --- a/python/paddle/distributed/fleet/elastic/collective.py +++ b/python/paddle/distributed/fleet/elastic/collective.py @@ -17,12 +17,15 @@ import shutil import paddle -from paddle.distributed.fleet.launch_utils import logger, pull_worker_log, start_local_trainers +from paddle.distributed.fleet.launch_utils import ( + logger, + pull_worker_log, + start_local_trainers, +) from paddle.distributed.fleet.elastic.manager import LauncherInterface class CollectiveLauncher(LauncherInterface): - def __init__(self, args): self.args = args self.procs = [] @@ -33,7 +36,8 @@ def launch(self): self.tmp_dir = tempfile.mkdtemp() cluster, pod = paddle.distributed.fleet.launch.get_cluster_info(args) global_envs = paddle.distributed.fleet.launch.get_global_envs( - args, self.tmp_dir) + args, self.tmp_dir + ) self.procs = start_local_trainers( cluster, @@ -41,7 +45,8 @@ def launch(self): training_script=args.training_script, training_script_args=args.training_script_args, log_dir=args.log_dir, - envs=global_envs) + envs=global_envs, + ) for idx, proc in enumerate(self.procs): logger.info("launch proc_id:{} idx:{}".format(proc.proc.pid, idx)) diff --git a/python/paddle/distributed/fleet/elastic/manager.py b/python/paddle/distributed/fleet/elastic/manager.py index 84cf95742e309..9397f6d10d4a6 100644 --- a/python/paddle/distributed/fleet/elastic/manager.py +++ b/python/paddle/distributed/fleet/elastic/manager.py @@ -54,7 +54,6 @@ class ElasticStatus: class LauncherInterface(object): - def __init__(self, args): self.args = args self.procs = [] @@ -67,8 +66,9 @@ def _terminate_procs(self): os.killpg(os.getpgid(p.proc.pid), signal.SIGTERM) if p.log_fn: p.log_fn.close() - logger.info("terminate process group gid:{}".format( - p.proc.pid)) + logger.info( + "terminate process group gid:{}".format(p.proc.pid) + ) time.sleep(1) for p in self.procs: @@ -105,8 +105,10 @@ def _check_procs(self): return ret logger.error("ABORT!!! ABORT!!! ABORT!!!") logger.error( - "ERROR rank {} error with exit code {}, check log for detail." - .format(p.rank, ret)) + "ERROR rank {} error with exit code {}, check log for detail.".format( + p.rank, ret + ) + ) result = ret if not alive and result is None: return 0 @@ -124,7 +126,6 @@ def watch(self): class ElasticManager(object): - def __init__(self, args, etcd_client): self.args = args @@ -137,11 +138,14 @@ def __init__(self, args, etcd_client): self.host = host if host else self._get_host() - (self.device_mode, - self.devices_per_proc) = launch_utils.get_device_proc_info(args) + ( + self.device_mode, + self.devices_per_proc, + ) = launch_utils.get_device_proc_info(args) self.elastic_timeout = int( - os.getenv('PADDLE_ELASTIC_TIMEOUT', ELASTIC_TIMEOUT)) + os.getenv('PADDLE_ELASTIC_TIMEOUT', ELASTIC_TIMEOUT) + ) elastic_ttl = int(os.getenv('PADDLE_ELASTIC_TTL', ELASTIC_TTL)) self.start_port = None @@ -158,7 +162,8 @@ def __init__(self, args, etcd_client): self.np = len(node_ips) self.start_port = int(os.getenv("FLAGS_START_PORT", "6170")) self.dist_endpoints = self._host_to_endpoints( - node_ips, self.devices_per_proc, self.start_port) + node_ips, self.devices_per_proc, self.start_port + ) self.trainer_endpoints_list = [ "%s:%d" % (ip, self.start_port) for ip in node_ips ] @@ -172,10 +177,12 @@ def __init__(self, args, etcd_client): # auto correct the value of elastic_level # 1: Fault tolerant, 2: Elastic self.elastic_level = int( - os.getenv('PADDLE_ELASTIC_FAULT_TOLERANC_LEVEL', - ElasticLevel.FAULT_TOLERANCE)) - if self.min_np == self.max_np or \ - (self.min_np > 0 and self.max_np == 0): + os.getenv( + 'PADDLE_ELASTIC_FAULT_TOLERANC_LEVEL', + ElasticLevel.FAULT_TOLERANCE, + ) + ) + if self.min_np == self.max_np or (self.min_np > 0 and self.max_np == 0): self.elastic_level = ElasticLevel.FAULT_TOLERANCE logger.info('start job with ElasticLevel.FAULT_TOLERANCE') if self.min_np > 0 and self.max_np > self.min_np: @@ -183,12 +190,15 @@ def __init__(self, args, etcd_client): logger.info('start job with ElasticLevel.ELASTIC') # compatible with kuberntes service discovery - if not server and os.getenv( - 'PADDLE_ELASTIC_ETCD_SERVICE_HOST') and os.getenv( - 'PADDLE_ELASTIC_ETCD_SERVICE_PORT'): + if ( + not server + and os.getenv('PADDLE_ELASTIC_ETCD_SERVICE_HOST') + and os.getenv('PADDLE_ELASTIC_ETCD_SERVICE_PORT') + ): server = '{}:{}'.format( os.getenv('PADDLE_ELASTIC_ETCD_SERVICE_HOST'), - os.getenv('PADDLE_ELASTIC_ETCD_SERVICE_PORT')) + os.getenv('PADDLE_ELASTIC_ETCD_SERVICE_PORT'), + ) logger.debug('init with server {} host {}'.format(server, host)) @@ -202,8 +212,10 @@ def __init__(self, args, etcd_client): if not server or ':' not in server or not name or not self.np: logger.info( - 'Elastic is not enabled with server {} name {} and np {}'. - format(server, name, self.np)) + 'Elastic is not enabled with server {} name {} and np {}'.format( + server, name, self.np + ) + ) self.enable = False return else: @@ -218,9 +230,11 @@ def __init__(self, args, etcd_client): self.endpoints_path = self.prefix + '/endpoints' node_tag = ''.join( - random.choice('abcdefghijklmnopqrstuvwxyz') for _ in range(6)) - self.host_path = '{}/{}{}'.format(self.node_prefix, node_tag, - time.time()) + random.choice('abcdefghijklmnopqrstuvwxyz') for _ in range(6) + ) + self.host_path = '{}/{}{}'.format( + self.node_prefix, node_tag, time.time() + ) ''' 0 group mode, be aware of healthy status of other workers 1 decouple mode, check own status only @@ -241,7 +255,8 @@ def host_call_back(event): self.elastic_startup_time = None host_watch = self.etcd.add_watch_prefix_callback( - self.node_prefix, host_call_back) + self.node_prefix, host_call_back + ) host_lease = self.etcd.lease(elastic_ttl) # register etcd lease heartbeat @@ -260,51 +275,61 @@ def lease_heartbeat(): ) if self.curr_host not in hosts: logger.info( - f"[lease_heartbeat] register host={self.curr_host}") - self.etcd.put(self.host_path, - self.curr_host.encode('latin-1'), - lease=host_lease) + f"[lease_heartbeat] register host={self.curr_host}" + ) + self.etcd.put( + self.host_path, + self.curr_host.encode('latin-1'), + lease=host_lease, + ) except Exception as e: logger.error( "[lease_heartbeat] internal error:{} {}".format( - e, traceback.format_exc())) + e, traceback.format_exc() + ) + ) break time.sleep(elastic_ttl / 3) - keepalived_thread = threading.Thread(name='lease_heartbeat', - target=lease_heartbeat, - daemon=True) + keepalived_thread = threading.Thread( + name='lease_heartbeat', target=lease_heartbeat, daemon=True + ) keepalived_thread.start() - self.etcd.put(self.host_path, - self.curr_host.encode('latin-1'), - lease=host_lease) + self.etcd.put( + self.host_path, self.curr_host.encode('latin-1'), lease=host_lease + ) # endpoints handle DISTRIBUTED_TRAINER_ENDPOINTS and PADDLE_TRAINERS self.etcd.put( self.endpoints_path, - '{}|{}'.format(self.dist_endpoints, - self.trainers).encode('latin-1')) + '{}|{}'.format(self.dist_endpoints, self.trainers).encode( + 'latin-1' + ), + ) def endpoints_call_back(event): if not self.dist_endpoints: return edps = six.ensure_str(self.etcd.get(self.endpoints_path)[0] or '') self.dist_endpoints, self.trainers = edps.split('|') - logger.info("set DISTRIBUTED_TRAINER_ENDPOINTS {} ".format( - self.dist_endpoints)) + logger.info( + "set DISTRIBUTED_TRAINER_ENDPOINTS {} ".format( + self.dist_endpoints + ) + ) logger.info("set PADDLE_TRAINERS {} ".format(self.trainers)) - endpoints_watch = self.etcd.add_watch_callback(self.endpoints_path, - endpoints_call_back) + endpoints_watch = self.etcd.add_watch_callback( + self.endpoints_path, endpoints_call_back + ) self.watches = [host_watch, endpoints_watch] self.launcher = None - def _host_to_endpoints(self, - ip_port_list: list, - devices_per_proc: list, - start_port: int = 6170) -> str: + def _host_to_endpoints( + self, ip_port_list: list, devices_per_proc: list, start_port: int = 6170 + ) -> str: endpoint_list = [] for ip_port in ip_port_list: endpoints = ip_port.split(":") @@ -347,11 +372,13 @@ def pre_hook(self): return logger.info("execute pre_hook...") current_env = copy.copy(os.environ.copy()) - out, err = subprocess.Popen(self.args.elastic_pre_hook, - env=current_env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True).communicate() + out, err = subprocess.Popen( + self.args.elastic_pre_hook, + env=current_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + ).communicate() if err: logger.warning("pre_hook exec failed") else: @@ -377,7 +404,8 @@ def _parse_np(self, np: str): max_np = min_np if min_np > max_np else max_np else: raise ValueError( - f'the np={np} needs to be in "MIN" or "MIN:MAX" format') + f'the np={np} needs to be in "MIN" or "MIN:MAX" format' + ) return min_np, max_np @@ -436,8 +464,10 @@ def _match(self, host_list: list = None): return False def _update_endpoint(self, endpoints, hosts): - self.etcd.put(self.endpoints_path, - '{}|{}'.format(endpoints, hosts).encode('latin-1')) + self.etcd.put( + self.endpoints_path, + '{}|{}'.format(endpoints, hosts).encode('latin-1'), + ) def _update_fault_tolrance(self): rank = int(os.getenv('PADDLE_TRAINER_ID', -1)) @@ -447,8 +477,11 @@ def _update_fault_tolrance(self): if self.curr_host in self.dist_endpoints: os.environ['DISTRIBUTED_TRAINER_ENDPOINTS'] = self.dist_endpoints os.environ['PADDLE_TRAINERS'] = self.trainers - logger.info("update env DISTRIBUTED_TRAINER_ENDPOINTS {} ".format( - self.dist_endpoints)) + logger.info( + "update env DISTRIBUTED_TRAINER_ENDPOINTS {} ".format( + self.dist_endpoints + ) + ) logger.info("update env PADDLE_TRAINERS {} ".format(self.trainers)) return @@ -476,9 +509,11 @@ def _update_elastic_scale_out(self): host_endpoints.append(curr_host_port) os.environ['PADDLE_TRAINER_ID'] = '{}'.format( - host_endpoints.index(self.curr_host)) + host_endpoints.index(self.curr_host) + ) hosts = ','.join( - [host_port.split(":")[0] for host_port in host_endpoints]) + [host_port.split(":")[0] for host_port in host_endpoints] + ) self.args.ips = hosts os.environ['PADDLE_TRAINERS'] = hosts self.np = len(host_endpoints) @@ -521,12 +556,14 @@ def _update_elastic_scale_in(self): ip_list = [ip_port.split(":")[0] for ip_port in sorted_endpoints] hosts = ','.join(ip_list) - new_endpoints = self._host_to_endpoints(sorted_endpoints, - self.devices_per_proc) + new_endpoints = self._host_to_endpoints( + sorted_endpoints, self.devices_per_proc + ) self.args.ips = hosts os.environ['PADDLE_TRAINER_ID'] = '{}'.format( - sorted_endpoints.index(self.curr_host)) + sorted_endpoints.index(self.curr_host) + ) os.environ['PADDLE_TRAINERS'] = hosts self.np = len(sorted_endpoints) os.environ['PADDLE_TRAINER_ENDPOINTS'] = ','.join(sorted_endpoints) @@ -560,8 +597,9 @@ def wait(self): logger.info('ready with hosts {}'.format(self.hosts)) self._update_hosts() return - logger.info('not ready for np {} with hosts {}'.format( - self.np, self.hosts)) + logger.info( + 'not ready for np {} with hosts {}'.format(self.np, self.hosts) + ) idx += 1 time.sleep(2) return diff --git a/python/paddle/distributed/fleet/fleet.py b/python/paddle/distributed/fleet/fleet.py index 7a6af30f91283..db3aae28a2018 100644 --- a/python/paddle/distributed/fleet/fleet.py +++ b/python/paddle/distributed/fleet/fleet.py @@ -55,12 +55,12 @@ def apply_ir_passes(main_program, startup_program, config): ) build_strategy.fuse_all_optimizer_ops = False - return apply_build_strategy(main_program, startup_program, build_strategy, - pass_attrs) + return apply_build_strategy( + main_program, startup_program, build_strategy, pass_attrs + ) def _inited_runtime_handler_(func): - def __impl__(*args, **kwargs): cls = args[0] @@ -73,15 +73,17 @@ def __impl__(*args, **kwargs): def _is_non_distributed_check_(func): - def __impl__(*args, **kwargs): cls = args[0] - if cls._role_maker is not None and cls._role_maker._is_non_distributed( - ) is True: + if ( + cls._role_maker is not None + and cls._role_maker._is_non_distributed() is True + ): logger.warning( - "%s() function doesn't work when use non_distributed fleet." % - (func.__name__)) + "%s() function doesn't work when use non_distributed fleet." + % (func.__name__) + ) return return func(*args, **kwargs) @@ -161,11 +163,13 @@ def __init__(self): self._context = {} self.user_defined_optimizer = paddle.optimizer.Optimizer(0.0) - def init(self, - role_maker=None, - is_collective=False, - strategy=None, - log_level="INFO"): + def init( + self, + role_maker=None, + is_collective=False, + strategy=None, + log_level="INFO", + ): """ Initialize role_maker in Fleet. @@ -240,22 +244,28 @@ def init(self, if isinstance(is_collective, bool): self._is_collective = is_collective self._role_maker = PaddleCloudRoleMaker( - is_collective=self._is_collective) + is_collective=self._is_collective + ) else: raise ValueError( - "`is_collective` should be instance of `bool`, but got {}". - format(type(is_collective))) + "`is_collective` should be instance of `bool`, but got {}".format( + type(is_collective) + ) + ) else: if isinstance(role_maker, RoleMakerBase): self._role_maker = role_maker self._is_collective = role_maker._is_collective else: raise ValueError( - "`role_maker` should be subclass of `RoleMakerBase`, but got {}" - .format(type(role_maker))) + "`role_maker` should be subclass of `RoleMakerBase`, but got {}".format( + type(role_maker) + ) + ) self._role_maker._generate_role() import paddle.distributed.fleet as fleet + fleet.util._set_role_maker(self._role_maker) self.strategy_compiler = StrategyCompiler() @@ -276,17 +286,20 @@ def init(self, return if parallel_helper._is_parallel_ctx_initialized(): logger.warning( - "The dygraph parallel environment has been initialized.") + "The dygraph parallel environment has been initialized." + ) else: # FLAGS_nccl_nrings is used for dynamic graph multi-stream communication if "FLAGS_nccl_nrings" in os.environ: logger.warning( "You have set the environment variable FLAGS_nccl_nrings " "outside the program, so the nccl_comm_num in " - "DistributedStrategy will not take effect here.") + "DistributedStrategy will not take effect here." + ) else: os.environ["FLAGS_nccl_nrings"] = str( - self._user_defined_strategy.nccl_comm_num) + self._user_defined_strategy.nccl_comm_num + ) paddle.distributed.init_parallel_env() # hybrid parallel not support for npu/xpu @@ -308,17 +321,24 @@ def init(self, global_ring_id = 3 if use_sharding else 0 global_ranks = list(range(global_world_size)) - if tp._HYBRID_PARALLEL_GROUP is None: tp._CommunicateGroup() + if tp._HYBRID_PARALLEL_GROUP is None: + tp._CommunicateGroup() cg = tp._HYBRID_PARALLEL_GROUP self._hcg = cg - cg.set_comm_group('global', global_rank, global_world_size, - global_ring_id, global_ranks) + cg.set_comm_group( + 'global', + global_rank, + global_world_size, + global_ring_id, + global_ranks, + ) use_tensor_parallel = self._user_defined_strategy.tensor_parallel use_mp = use_sharding or use_tensor_parallel # hybrid group - if use_mp is False: return + if use_mp is False: + return mp_degree_sharding = 1 mp_degree_tensor_parallel = 1 @@ -327,14 +347,21 @@ def init(self, mp_degree_sharding = int(sharding_configs['mp_degree']) if use_tensor_parallel: - tensor_parallel_configs = self._user_defined_strategy.tensor_parallel_configs + tensor_parallel_configs = ( + self._user_defined_strategy.tensor_parallel_configs + ) mp_degree_tensor_parallel = int( - tensor_parallel_configs['tensor_parallel_degree']) + tensor_parallel_configs['tensor_parallel_degree'] + ) if use_sharding and use_tensor_parallel: assert mp_degree_sharding == mp_degree_tensor_parallel - mp_degree = mp_degree_sharding if use_sharding else mp_degree_tensor_parallel + mp_degree = ( + mp_degree_sharding + if use_sharding + else mp_degree_tensor_parallel + ) if mp_degree > 1: assert global_world_size % mp_degree == 0 @@ -343,16 +370,17 @@ def init(self, mp_rank = global_rank % mp_degree mp_group_id = global_rank // mp_degree mp_group_ranks = [ - idx for idx in global_ranks + idx + for idx in global_ranks if idx // mp_degree == mp_group_id ] - cg.set_comm_group('model', mp_rank, mp_degree, mp_ring_id, - mp_group_ranks) + cg.set_comm_group( + 'model', mp_rank, mp_degree, mp_ring_id, mp_group_ranks + ) return self def _init_hybrid_parallel_env(self): - """initialize the hybrid environment - """ + """initialize the hybrid environment""" self.hybrid_configs = self._user_defined_strategy.hybrid_configs self.dp_degree = self.hybrid_configs["dp_degree"] self.mp_degree = self.hybrid_configs["mp_degree"] @@ -361,7 +389,9 @@ def _init_hybrid_parallel_env(self): assert self.mp_degree >= 0, "mp_degree should be greater or equal to 0" assert self.pp_degree >= 0, "pp_degree should be greater or equal to 0" - assert self.sharding_degree >= 0, "sharding_degree should be greater or equal to 0" + assert ( + self.sharding_degree >= 0 + ), "sharding_degree should be greater or equal to 0" self.mp_degree = max(self.mp_degree, 1) self.pp_degree = max(self.pp_degree, 1) @@ -375,14 +405,19 @@ def _init_hybrid_parallel_env(self): self._topology = tp.CommunicateTopology( hybrid_group_names=["data", "pipe", "sharding", "model"], dims=[ - self.dp_degree, self.pp_degree, self.sharding_degree, - self.mp_degree - ]) + self.dp_degree, + self.pp_degree, + self.sharding_degree, + self.mp_degree, + ], + ) self._hcg = tp.HybridCommunicateGroup(self._topology) if self.mp_degree > 1: - tensor_parallel_configs = self._user_defined_strategy.tensor_parallel_configs + tensor_parallel_configs = ( + self._user_defined_strategy.tensor_parallel_configs + ) tensor_init_seed = tensor_parallel_configs["tensor_init_seed"] if tensor_init_seed == -1: model_parallel_random_seed() @@ -822,29 +857,29 @@ def save(self, dirname, feed=[], fetch=[], **configs): for name in fetch_var_names ] - self._runtime_handle._save_inference_model(executor, dirname, - feeded_var_names, - fetch_vars, None, True, - 0) + self._runtime_handle._save_inference_model( + executor, dirname, feeded_var_names, fetch_vars, None, True, 0 + ) else: increment_mode = 0 if "mode" in configs: increment_mode = int(configs["mode"]) - self._runtime_handle._save_persistables(executor, - dirname, - main_program=None, - mode=increment_mode) + self._runtime_handle._save_persistables( + executor, dirname, main_program=None, mode=increment_mode + ) @is_non_distributed_check @inited_runtime_handler - def save_inference_model(self, - executor, - dirname, - feeded_var_names, - target_vars, - main_program=None, - export_for_deployment=True, - mode=0): + def save_inference_model( + self, + executor, + dirname, + feeded_var_names, + target_vars, + main_program=None, + export_for_deployment=True, + mode=0, + ): """ save inference model for inference. @@ -865,10 +900,15 @@ def save_inference_model(self, """ - self._runtime_handle._save_inference_model(executor, dirname, - feeded_var_names, - target_vars, main_program, - export_for_deployment, mode) + self._runtime_handle._save_inference_model( + executor, + dirname, + feeded_var_names, + target_vars, + main_program, + export_for_deployment, + mode, + ) @is_non_distributed_check @inited_runtime_handler @@ -913,8 +953,9 @@ def save_persistables(self, executor, dirname, main_program=None, mode=0): fleet.save_persistables(exe, "dirname", paddle.static.default_main_program()) """ - self._runtime_handle._save_persistables(executor, dirname, main_program, - mode) + self._runtime_handle._save_persistables( + executor, dirname, main_program, mode + ) @is_non_distributed_check @inited_runtime_handler @@ -953,12 +994,9 @@ def save_one_table(self, table_id, path, mode): @is_non_distributed_check @inited_runtime_handler - def save_dense_params(self, - executor, - dirname, - scope, - program, - var_names=None): + def save_dense_params( + self, executor, dirname, scope, program, var_names=None + ): """ save fleet one table from path @@ -982,8 +1020,9 @@ def save_dense_params(self, fleet.save_dense_params(exe, "path", scope=paddle.static.global_scope(), program=paddle.static.default_main_program()) """ - self._runtime_handle._save_dense_params(executor, dirname, scope, - program, var_names) + self._runtime_handle._save_dense_params( + executor, dirname, scope, program, var_names + ) def shrink(self, threshold=None): self._runtime_handle._shrink(threshold) @@ -1027,7 +1066,8 @@ def distributed_optimizer(self, optimizer, strategy=None): "in fleet.init(). The strategy here is only for compatibility. " "If the strategy in fleet.distributed_optimizer() is " "not None, then it will overwrite the DistributedStrategy in fleet.init(), " - "which will take effect in distributed training.") + "which will take effect in distributed training." + ) self._user_defined_strategy = copy.deepcopy(strategy) self._context = {} @@ -1046,21 +1086,19 @@ def _get_amp_optimizer(self): if hasattr(self.user_defined_optimizer, 'amp_init'): amp_optimizer = self.user_defined_optimizer - assert amp_optimizer is not None, \ - "amp_init can only be used when the amp(auto mixed precision) strategy is turned on." + assert ( + amp_optimizer is not None + ), "amp_init can only be used when the amp(auto mixed precision) strategy is turned on." return amp_optimizer def get_loss_scaling(self): - """Return the real-time loss scaling factor. - """ + """Return the real-time loss scaling factor.""" amp_optimizer = self._get_amp_optimizer() return amp_optimizer.get_loss_scaling() - def amp_init(self, - place, - scope=None, - test_program=None, - use_fp16_test=False): + def amp_init( + self, place, scope=None, test_program=None, use_fp16_test=False + ): """ Init the amp training, such as cast fp32 parameters to fp16 type. @@ -1145,11 +1183,9 @@ def _get_applied_graph_list(self): else: return self._context["applied_graph_list"] - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): """ Add distributed operations to minimize ``loss`` by updating ``parameter_list``. @@ -1201,23 +1237,27 @@ def minimize(self, """ if not isinstance(loss, list): - return self._minimize_impl(loss, startup_program, parameter_list, - no_grad_set) + return self._minimize_impl( + loss, startup_program, parameter_list, no_grad_set + ) else: - if paddle.fluid.framework._non_static_mode( - ) or self._role_maker._is_non_distributed() or self._is_collective: + if ( + paddle.fluid.framework._non_static_mode() + or self._role_maker._is_non_distributed() + or self._is_collective + ): raise ValueError("loss can be list only in PS mode") - return self._minimize_losses_impl(loss, startup_program, - parameter_list, no_grad_set) - - def _minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + return self._minimize_losses_impl( + loss, startup_program, parameter_list, no_grad_set + ) + + def _minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): context = {} context["user_defined_strategy"] = copy.deepcopy( - self._user_defined_strategy) + self._user_defined_strategy + ) if paddle.fluid.framework._non_static_mode(): # imitate target optimizer retrieval target_opt = self.user_defined_optimizer @@ -1230,49 +1270,62 @@ def _minimize_impl(self, if not hasattr(self.origin_main_program, "distributed_info_"): setattr(self.origin_main_program, "distributed_info_", dict()) self.origin_main_program.distributed_info_[ - "dp_degree"] = self._user_defined_strategy.sharding_configs[ - "dp_degree"] + "dp_degree" + ] = self._user_defined_strategy.sharding_configs["dp_degree"] self.origin_main_program.distributed_info_[ - "mp_degree"] = self._user_defined_strategy.sharding_configs[ - "mp_degree"] + "mp_degree" + ] = self._user_defined_strategy.sharding_configs["mp_degree"] self.origin_main_program.distributed_info_[ - "pp_degree"] = self._user_defined_strategy.sharding_configs[ - "pp_degree"] + "pp_degree" + ] = self._user_defined_strategy.sharding_configs["pp_degree"] self.origin_main_program.distributed_info_[ - "sharding_degree"] = self._user_defined_strategy.sharding_configs[ - "sharding_degree"] + "sharding_degree" + ] = self._user_defined_strategy.sharding_configs["sharding_degree"] context["origin_main_program"] = self.origin_main_program context["origin_main_programs"] = [self.origin_main_program] context["loss"] = loss if startup_program == None: - self.origin_startup_program = \ + self.origin_startup_program = ( paddle.static.default_startup_program().clone(for_test=False) + ) startup_program = paddle.static.default_startup_program() else: - self.origin_startup_program = \ - startup_program.clone(for_test=False) + self.origin_startup_program = startup_program.clone(for_test=False) context["origin_startup_program"] = startup_program context["origin_startup_programs"] = [startup_program] context["role_maker"] = self._role_maker # Use the auto-parallel's routines instead - if self._user_defined_strategy.semi_auto or self._user_defined_strategy.auto_search: + if ( + self._user_defined_strategy.semi_auto + or self._user_defined_strategy.auto_search + ): from ..auto_parallel.parallelizer import AutoParallelizer + auto_parallelizer = AutoParallelizer(self) - optimize_ops, params_grads, dist_startup_prog, dist_main_prog = auto_parallelizer.parallelize( - loss, startup_program, parameter_list, no_grad_set) + ( + optimize_ops, + params_grads, + dist_startup_prog, + dist_main_prog, + ) = auto_parallelizer.parallelize( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads, dist_startup_prog, dist_main_prog # compile time - distributed_optimizer_list = \ + distributed_optimizer_list = ( MetaOptimizerFactory()._get_valid_meta_optimizers( - self.user_defined_optimizer) + self.user_defined_optimizer + ) + ) context["user_defined_strategy"] = copy.deepcopy( - self._user_defined_strategy) + self._user_defined_strategy + ) copy_user_defined_strategy = copy.deepcopy(self._user_defined_strategy) # trigger the auto-parallel in very strict condition @@ -1290,9 +1343,12 @@ def _minimize_impl(self, can_not_apply_optimizer_list = [] # recall meta optimizers for ranking for opt in distributed_optimizer_list: - opt._set_basic_info(loss, self._role_maker, - self.user_defined_optimizer, - copy_user_defined_strategy) + opt._set_basic_info( + loss, + self._role_maker, + self.user_defined_optimizer, + copy_user_defined_strategy, + ) if opt._can_apply() and not opt._is_graph_out(): valid_optimizer_list.append(opt) elif opt._can_apply() and opt._is_graph_out(): @@ -1300,19 +1356,27 @@ def _minimize_impl(self, else: can_not_apply_optimizer_list.append(opt) # combine recalled meta optimizers to be a valid meta optimizer - meta_optimizer, graph_optimizer = \ - self.strategy_compiler.generate_optimizer( - loss, self._role_maker, self.user_defined_optimizer, - copy_user_defined_strategy, valid_optimizer_list, - valid_graph_optimizer_list) + ( + meta_optimizer, + graph_optimizer, + ) = self.strategy_compiler.generate_optimizer( + loss, + self._role_maker, + self.user_defined_optimizer, + copy_user_defined_strategy, + valid_optimizer_list, + valid_graph_optimizer_list, + ) valid_strategy = self.strategy_compiler._get_valid_strategy( - copy_user_defined_strategy, can_not_apply_optimizer_list) + copy_user_defined_strategy, can_not_apply_optimizer_list + ) context["valid_strategy"] = copy.deepcopy(valid_strategy) logger.debug("valid_strategy: " + str(context["valid_strategy"])) - logger.debug("user_defined_strategy: " + - str(context["user_defined_strategy"])) + logger.debug( + "user_defined_strategy: " + str(context["user_defined_strategy"]) + ) applied_meta_list = self.strategy_compiler._get_applied_meta_list() applied_graph_list = self.strategy_compiler._get_applied_graph_list() @@ -1333,41 +1397,48 @@ def _minimize_impl(self, self._runtime_handle = RuntimeFactory()._create_runtime(context) compiled_program = compiler.CompiledProgram( - self.origin_main_program).with_data_parallel( - loss_name=loss.name, share_vars_from=None) + self.origin_main_program + ).with_data_parallel(loss_name=loss.name, share_vars_from=None) loss.block.program._graph = compiled_program - return self.user_defined_optimizer.minimize(loss, - startup_program, - parameter_list, - no_grad_set=no_grad_set) + return self.user_defined_optimizer.minimize( + loss, startup_program, parameter_list, no_grad_set=no_grad_set + ) if meta_optimizer: - logger.debug("before minimize program id: " + - str(id(loss.block.program))) + logger.debug( + "before minimize program id: " + str(id(loss.block.program)) + ) optimize_ops, params_grads = meta_optimizer.minimize( - loss, startup_program, parameter_list, no_grad_set=no_grad_set) - logger.debug("after minimize program id: " + - str(id(loss.block.program))) + loss, startup_program, parameter_list, no_grad_set=no_grad_set + ) + logger.debug( + "after minimize program id: " + str(id(loss.block.program)) + ) default_program = paddle.static.default_main_program() logger.debug("default program id: " + str(id(default_program))) if id(default_program) != id(loss.block.program): paddle.fluid.framework.switch_main_program(loss.block.program) - logger.debug("default program id after switch: " + - str(id(default_program))) + logger.debug( + "default program id after switch: " + str(id(default_program)) + ) else: optimize_ops, params_grads = self.user_defined_optimizer.minimize( - loss, startup_program, parameter_list, no_grad_set=no_grad_set) + loss, startup_program, parameter_list, no_grad_set=no_grad_set + ) context["program_optimize_ops"] = optimize_ops context["program_params_grads"] = params_grads if graph_optimizer: - logger.debug("before graph minimize program id: " + - str(id(loss.block.program))) + logger.debug( + "before graph minimize program id: " + + str(id(loss.block.program)) + ) optimize_ops, params_grads = graph_optimizer.minimize( - loss, startup_program, parameter_list, no_grad_set=no_grad_set) + loss, startup_program, parameter_list, no_grad_set=no_grad_set + ) # since we do not encourage users to use graph operations # if a graph optimizer takes effect, mostly # optimizers_ops and params_grads are None @@ -1382,8 +1453,10 @@ def _minimize_impl(self, opt_info = {} if program._fleet_opt is None else program._fleet_opt opt_info["mpi_size"] = self.worker_num() opt_info["mpi_rank"] = self.worker_index() - for k, v in self._user_defined_strategy.trainer_desc_configs.items( - ): + for ( + k, + v, + ) in self._user_defined_strategy.trainer_desc_configs.items(): if v or k not in opt_info: opt_info[k] = v program._fleet_opt = opt_info @@ -1392,15 +1465,18 @@ def _minimize_impl(self, self._runtime_handle = RuntimeFactory()._create_runtime(context) import paddle.distributed.fleet as fleet + fleet.util._set_strategy(context["valid_strategy"]) return optimize_ops, params_grads - def _minimize_losses_impl(self, - losses, - startup_programs=None, - parameter_list=None, - no_grad_set=None): + def _minimize_losses_impl( + self, + losses, + startup_programs=None, + parameter_list=None, + no_grad_set=None, + ): context = {} # cache original feed forward program @@ -1416,7 +1492,8 @@ def _minimize_losses_impl(self, startup_programs = [paddle.static.default_startup_program()] else: raise ValueError( - "startup_program can't be None when loss is list.") + "startup_program can't be None when loss is list." + ) self.origin_startup_program = startup_programs[0].clone(for_test=False) context["origin_startup_program"] = startup_programs[0] context["origin_startup_programs"] = [] @@ -1426,7 +1503,8 @@ def _minimize_losses_impl(self, context["role_maker"] = self._role_maker context["user_defined_strategy"] = copy.deepcopy( - self._user_defined_strategy) + self._user_defined_strategy + ) context["valid_strategy"] = copy.deepcopy(self._user_defined_strategy) @@ -1439,12 +1517,17 @@ def _minimize_losses_impl(self, params_grads = [] from .meta_optimizers import ParameterServerOptimizer + ps_optimizer = ParameterServerOptimizer(self.user_defined_optimizer) - ps_optimizer._set_basic_info(losses, self._role_maker, - self.user_defined_optimizer, - self._user_defined_strategy) + ps_optimizer._set_basic_info( + losses, + self._role_maker, + self.user_defined_optimizer, + self._user_defined_strategy, + ) optimize_ops, params_grads = ps_optimizer.minimize_losses_impl( - losses, startup_programs, parameter_list, no_grad_set=no_grad_set) + losses, startup_programs, parameter_list, no_grad_set=no_grad_set + ) # default_program = paddle.static.default_main_program() @@ -1459,18 +1542,24 @@ def _minimize_losses_impl(self, opt_info = {} if program._fleet_opt is None else program._fleet_opt opt_info["mpi_size"] = self.worker_num() opt_info["mpi_rank"] = self.worker_index() - for k, v in self._user_defined_strategy.trainer_desc_configs.items( - ): + for ( + k, + v, + ) in self._user_defined_strategy.trainer_desc_configs.items(): if v or k not in opt_info: opt_info[k] = v program._fleet_opt = opt_info - logger.debug("fleet base opt info: " + str(id(program)) + - str(program._fleet_opt)) + logger.debug( + "fleet base opt info: " + + str(id(program)) + + str(program._fleet_opt) + ) if self._runtime_handle is None: self._runtime_handle = RuntimeFactory()._create_runtime(context) import paddle.distributed.fleet as fleet + fleet.util._set_strategy(context["valid_strategy"]) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/fleet_executor_utils.py b/python/paddle/distributed/fleet/fleet_executor_utils.py index 48ef34f4603ca..4eb4108d5d595 100644 --- a/python/paddle/distributed/fleet/fleet_executor_utils.py +++ b/python/paddle/distributed/fleet/fleet_executor_utils.py @@ -22,16 +22,18 @@ class TaskNode: Python side TaskNode, connection to the c++ side TaskNode """ - def __init__(self, - rank, - max_run_times, - max_slot_times, - role=None, - node_type=None, - task_id=0, - ops=None, - program=None, - lazy_initialize=False): + def __init__( + self, + rank, + max_run_times, + max_slot_times, + role=None, + node_type=None, + task_id=0, + ops=None, + program=None, + lazy_initialize=False, + ): """ :param rank (int): Current rank of the task node. :param max_run_times (int): The max run times of the task node. @@ -43,10 +45,12 @@ def __init__(self, :param program (Program): An instance of Program to init the task node. :param lazy_initialize (bool): In user-defined task, the program may change adding feed/fetch op. As efficient consideration, the task node will have the C++ object later. """ - assert ((ops is not None) ^ (program is not None)), \ - "Should provide only one of ops or program to task node." - assert (not ((ops is not None) and lazy_initialize)), \ - "Lazy initialization doesn't support with ops list" + assert (ops is not None) ^ ( + program is not None + ), "Should provide only one of ops or program to task node." + assert not ( + (ops is not None) and lazy_initialize + ), "Lazy initialization doesn't support with ops list" self.id = int(task_id) self.rank = rank self.max_run_times = max_run_times @@ -61,20 +65,28 @@ def __init__(self, self.downstreams = [] if not lazy_initialize: if ops is not None: - assert role is not None and task_id is not None, \ - "If init task node with ops, should provide `role` and `task_id`." - self.node = core.TaskNode(role, ops, rank, task_id, - max_run_times, max_slot_times) + assert ( + role is not None and task_id is not None + ), "If init task node with ops, should provide `role` and `task_id`." + self.node = core.TaskNode( + role, ops, rank, task_id, max_run_times, max_slot_times + ) else: - self.node = core.TaskNode(program.desc, rank, self.id, - max_run_times, max_slot_times) + self.node = core.TaskNode( + program.desc, rank, self.id, max_run_times, max_slot_times + ) if self.node_type: self.node.set_type(self.node_type) def task_node(self): if self.lazy_initialize: - self.node = core.TaskNode(self.program.desc, self.rank, self.id, - self.max_run_times, self.max_slot_times) + self.node = core.TaskNode( + self.program.desc, + self.rank, + self.id, + self.max_run_times, + self.max_slot_times, + ) if self.node_type: self.node.set_type(self.node_type) if self.run_pre_steps: @@ -89,12 +101,15 @@ def task_node(self): return self.node def set_program(self, program): - assert self.lazy_initialize, \ - "Inside program is unchangable for immediate initialized task node. Set the lazy_initialize to be true if the inside program need to be update. Remember to do all your change before eval node.task_node()." + assert ( + self.lazy_initialize + ), "Inside program is unchangable for immediate initialized task node. Set the lazy_initialize to be true if the inside program need to be update. Remember to do all your change before eval node.task_node()." self.program = program def get_program(self): - assert self.program is not None, "The task node is not initialized using program" + assert ( + self.program is not None + ), "The task node is not initialized using program" return self.program def set_run_pre_steps(self, steps): @@ -142,10 +157,16 @@ def _invalide_coord(self, coord): :param coord: The coord to be tested :return: False if valid, True if invalid. """ - return coord['mp_idx'] < 0 or coord['mp_idx'] >= self.mp_degree or \ - coord['sharding_idx'] < 0 or coord['sharding_idx'] >= self.sharding_degree or \ - coord['pp_idx'] < 0 or coord['pp_idx'] >= self.pp_degree or \ - coord['dp_idx'] < 0 or coord['dp_idx'] >= self.dp_degree + return ( + coord['mp_idx'] < 0 + or coord['mp_idx'] >= self.mp_degree + or coord['sharding_idx'] < 0 + or coord['sharding_idx'] >= self.sharding_degree + or coord['pp_idx'] < 0 + or coord['pp_idx'] >= self.pp_degree + or coord['dp_idx'] < 0 + or coord['dp_idx'] >= self.dp_degree + ) def coord_to_rank(self, coord): """ @@ -155,9 +176,15 @@ def coord_to_rank(self, coord): """ if self._invalide_coord(coord): return -1 - return int(coord['dp_idx'] * self.pp_degree * self.sharding_degree * self.mp_degree + \ - coord['pp_idx'] * self.sharding_degree * self.mp_degree + \ - coord['sharding_idx'] * self.mp_degree + coord['mp_idx']) + return int( + coord['dp_idx'] + * self.pp_degree + * self.sharding_degree + * self.mp_degree + + coord['pp_idx'] * self.sharding_degree * self.mp_degree + + coord['sharding_idx'] * self.mp_degree + + coord['mp_idx'] + ) def rank_to_coord(self, rank): """ @@ -176,17 +203,14 @@ def rank_to_coord(self, rank): 'mp_idx': int(mp_idx), 'sharding_idx': int(sharding_idx), 'pp_idx': int(pp_idx), - 'dp_idx': int(dp_idx) + 'dp_idx': int(dp_idx), } class FleetExecutorUtils: - - def __init__(self, - dist_strategy=None, - rank=None, - nrank=None, - max_run_times=None): + def __init__( + self, dist_strategy=None, rank=None, nrank=None, max_run_times=None + ): self.dist_strategy = dist_strategy self.rank = rank self.nrank = nrank @@ -206,12 +230,14 @@ def is_lr_sched_op(self, op_role): return op_role == int(OpRole.Optimize.LRSched) def is_forward_op(self, op_role): - return (op_role == int(OpRole.Forward)) or \ - (op_role == (int(OpRole.Forward) | int(OpRole.Loss))) + return (op_role == int(OpRole.Forward)) or ( + op_role == (int(OpRole.Forward) | int(OpRole.Loss)) + ) def is_backward_op(self, op_role): - return (op_role == int(OpRole.Backward)) or \ - (op_role == (int(OpRole.Backward) | int(OpRole.Loss))) + return (op_role == int(OpRole.Backward)) or ( + op_role == (int(OpRole.Backward) | int(OpRole.Loss)) + ) def split_program_to_op_list(self, program): op_list_map = {"lr": [], "fwd": [], "bwd": [], "opt": []} @@ -233,17 +259,19 @@ def split_program_to_op_list(self, program): return op_list_map def convert_op_list_to_program(self, op_list, complete_program): - #TODO(liyurui): Complete this convert logic + # TODO(liyurui): Complete this convert logic program_map = { "lr": Program(), "fwd": Program(), "bwd": Program(), - "opt": Program() + "opt": Program(), } return program_map def build_1f1b_dependency(self, task_node_map): - assert not self.is_auto_parallel, "Handly add dependency should not be invoked in auto parallel mode" + assert ( + not self.is_auto_parallel + ), "Handly add dependency should not be invoked in auto parallel mode" # Generated the dependency based on this graph: # lr(1:m) -> forward -> backward -> (m:1)optimize # ↑ ↓ @@ -253,8 +281,9 @@ def build_1f1b_dependency(self, task_node_map): # add dependency intra stage cur_start_id = self.rank * self.num_of_functionality - pp_buff_size = int(self.dist_strategy['pp_degree'] - - self.coord['pp_idx']) + pp_buff_size = int( + self.dist_strategy['pp_degree'] - self.coord['pp_idx'] + ) task_node_map["lr"].add_downstream_task(cur_start_id + 1) task_node_map["fwd"].add_upstream_task(cur_start_id) task_node_map["fwd"].add_downstream_task(cur_start_id + 2, pp_buff_size) @@ -267,8 +296,8 @@ def build_1f1b_dependency(self, task_node_map): downstream_coord['pp_idx'] = downstream_coord['pp_idx'] + 1 pp_upstream = self.coord_sys.coord_to_rank(upstream_coord) pp_downstream = self.coord_sys.coord_to_rank(downstream_coord) - first_stage = (pp_upstream == -1) - last_stage = (pp_downstream == -1) + first_stage = pp_upstream == -1 + last_stage = pp_downstream == -1 prev_pp_start_id = pp_upstream * self.num_of_functionality next_pp_start_id = pp_downstream * self.num_of_functionality if not first_stage: @@ -282,31 +311,39 @@ def build_1f1b_dependency(self, task_node_map): def construct_task_nodes_1f1b(self, program_map): max_slot_times = int(self.max_run_times - self.coord['pp_idx']) cur_start_id = int(self.rank * self.num_of_functionality) - lr_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - program=program_map["lr"], - task_id=cur_start_id) - fwd_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - program=program_map["fwd"], - task_id=cur_start_id + 1) - bwd_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - program=program_map["bwd"], - task_id=cur_start_id + 2) - opt_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - program=program_map["opt"], - task_id=cur_start_id + 3) + lr_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + program=program_map["lr"], + task_id=cur_start_id, + ) + fwd_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + program=program_map["fwd"], + task_id=cur_start_id + 1, + ) + bwd_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + program=program_map["bwd"], + task_id=cur_start_id + 2, + ) + opt_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + program=program_map["opt"], + task_id=cur_start_id + 3, + ) return { "lr": lr_task_node, "fwd": fwd_task_node, "bwd": bwd_task_node, - "opt": opt_task_node + "opt": opt_task_node, } def task_id_to_rank(self): @@ -319,51 +356,61 @@ def task_id_to_rank(self): def construct_task_nodes_1f1b_op_list(self, op_list_map): max_slot_times = int(self.max_run_times - self.coord['pp_idx']) cur_start_id = int(self.rank * self.num_of_functionality) - lr_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - role=int(OpRole.Optimize.LRSched), - ops=op_list_map["lr"], - task_id=cur_start_id, - node_type="Amplifier") + lr_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + role=int(OpRole.Optimize.LRSched), + ops=op_list_map["lr"], + task_id=cur_start_id, + node_type="Amplifier", + ) lr_task_node.set_run_pre_steps(self.max_run_times) - fwd_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - role=int(OpRole.Forward), - ops=op_list_map["fwd"], - task_id=cur_start_id + 1, - node_type="Compute") - bwd_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - role=int(OpRole.Backward), - ops=op_list_map["bwd"], - task_id=cur_start_id + 2, - node_type="Compute") - opt_task_node = TaskNode(rank=self.rank, - max_run_times=self.max_run_times, - max_slot_times=max_slot_times, - role=int(OpRole.Optimize), - ops=op_list_map["opt"], - task_id=cur_start_id + 3, - node_type="Amplifier") + fwd_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + role=int(OpRole.Forward), + ops=op_list_map["fwd"], + task_id=cur_start_id + 1, + node_type="Compute", + ) + bwd_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + role=int(OpRole.Backward), + ops=op_list_map["bwd"], + task_id=cur_start_id + 2, + node_type="Compute", + ) + opt_task_node = TaskNode( + rank=self.rank, + max_run_times=self.max_run_times, + max_slot_times=max_slot_times, + role=int(OpRole.Optimize), + ops=op_list_map["opt"], + task_id=cur_start_id + 3, + node_type="Amplifier", + ) opt_task_node.set_run_pre_steps(self.max_run_times) opt_task_node.set_run_at_offset(self.max_run_times - 1) return { "lr": lr_task_node, "fwd": fwd_task_node, "bwd": bwd_task_node, - "opt": opt_task_node + "opt": opt_task_node, } -def run1f1b(program, - rank, - max_run_times, - dist_opt, - nrank, - with_standalone_executor=False): +def run1f1b( + program, + rank, + max_run_times, + dist_opt, + nrank, + with_standalone_executor=False, +): """ Split the program to support 1f1b pipeline scheduler. This funct will split the program based on the op_role. @@ -380,24 +427,29 @@ def run1f1b(program, task_id_to_rank (dict): task nodes' ids to it's corresponding rank """ print("fleet executor will use python side 1f1b scheduler.") - fleet_executor_utils = FleetExecutorUtils(dist_strategy=dist_opt, - rank=rank, - nrank=nrank, - max_run_times=max_run_times) + fleet_executor_utils = FleetExecutorUtils( + dist_strategy=dist_opt, + rank=rank, + nrank=nrank, + max_run_times=max_run_times, + ) op_list_map = fleet_executor_utils.split_program_to_op_list(program) task_node_map = None if with_standalone_executor: program_map = fleet_executor_utils.convert_op_list_to_program( - op_list_map, program) + op_list_map, program + ) task_node_map = fleet_executor_utils.construct_task_nodes_1f1b( - program_map) + program_map + ) else: op_desc_list_map = {"lr": [], "fwd": [], "bwd": [], "opt": []} for key in op_list_map: for op in op_list_map[key]: op_desc_list_map[key].append(op.desc) task_node_map = fleet_executor_utils.construct_task_nodes_1f1b_op_list( - op_desc_list_map) + op_desc_list_map + ) task_node_map = fleet_executor_utils.build_1f1b_dependency(task_node_map) task_id_to_rank = fleet_executor_utils.task_id_to_rank() task_node_list = [task_node_map[key].task_node() for key in task_node_map] @@ -414,10 +466,12 @@ def origin(program, rank): task_id_to_rank (dict): a fake dict, since there is no upstream or downstream, this dict won't be used """ print("fleet executor will use python side origin scheduler.") - task_node = TaskNode(program=program, - rank=rank, - node_type="Compute", - max_run_times=1, - max_slot_times=1) + task_node = TaskNode( + program=program, + rank=rank, + node_type="Compute", + max_run_times=1, + max_slot_times=1, + ) task_id_to_rank = {task_node.task_id(): rank} return [task_node.task_node()], task_id_to_rank diff --git a/python/paddle/distributed/fleet/launch.py b/python/paddle/distributed/fleet/launch.py index a50c08cbaf96a..ca301c52a7f25 100755 --- a/python/paddle/distributed/fleet/launch.py +++ b/python/paddle/distributed/fleet/launch.py @@ -65,10 +65,21 @@ import paddle.fluid as fluid from paddle.distributed.fleet import launch_utils from paddle.distributed.fleet.launch_utils import ( - get_host_name_ip, find_free_ports, logger, get_cluster, DeviceMode, - start_local_trainers, direct_start, watch_local_trainers, - terminate_local_procs, DistributeMode, ParameterServerLauncher, get_logger, - check_backend, block_windows_and_macos) + get_host_name_ip, + find_free_ports, + logger, + get_cluster, + DeviceMode, + start_local_trainers, + direct_start, + watch_local_trainers, + terminate_local_procs, + DistributeMode, + ParameterServerLauncher, + get_logger, + check_backend, + block_windows_and_macos, +) from paddle.distributed.fleet import cloud_utils from paddle.distributed.fleet import ascend_utils @@ -92,33 +103,38 @@ def _parse_args(): parser = ArgumentParser( description='''start paddle training using multi-process mode. see: http://www.paddlepaddle.org/documentation/docs/zh/1.6/user_guides/howto/training/cluster_howto.html#permalink-8--nccl2- -''') +''' + ) base_group = parser.add_argument_group("Base Parameters") base_group.add_argument( "--log_dir", type=str, default="log", - help="The path for each process's log. Default --log_dir=log/") + help="The path for each process's log. Default --log_dir=log/", + ) base_group.add_argument( "--backend", type=str, default=os.environ.get('PADDLE_DISTRI_BACKEND', 'auto'), help="Specifize the backend, can be gloo|nccl|bkcl|auto|hccl|heter. " - "Default value is auto which perfers nccl or bkcl.") + "Default value is auto which perfers nccl or bkcl.", + ) base_group.add_argument( "--nproc_per_node", type=int, default=None, help="The number of processes to launch on a node." "In gpu training, it should be less or equal to the gpus number of you system(or you set by --gpus). And so each process can" - " bound to one or average number of gpus.") + " bound to one or average number of gpus.", + ) base_group.add_argument( "--run_mode", type=str, default=None, - help="run mode of job, can be:collective/ps/ps-heter") + help="run mode of job, can be:collective/ps/ps-heter", + ) if fluid.core.is_compiled_with_cuda(): base_group.add_argument( @@ -127,7 +143,7 @@ def _parse_args(): default=None, help="It's for gpu training." "For example:" - "--gpus=\"0,1,2,3\" will launch four training processes each bound to one gpu." + "--gpus=\"0,1,2,3\" will launch four training processes each bound to one gpu.", ) base_group.add_argument("--selected_gpus", dest="gpus") @@ -137,7 +153,7 @@ def _parse_args(): type=str, default=None, help="It's for xpu training. For example: " - "--xpus=\"0,1,2,3\" will launch four training processes each bound to one xpu." + "--xpus=\"0,1,2,3\" will launch four training processes each bound to one xpu.", ) base_group.add_argument("--selected_xpus", dest="xpus") @@ -147,7 +163,7 @@ def _parse_args(): type=str, default=None, help="It's for xpu training. For example: " - "--npus=\"0,1,2,3\" will launch four training processes each bound to one npu." + "--npus=\"0,1,2,3\" will launch four training processes each bound to one npu.", ) base_group.add_argument("--selected_npus", dest="npus") @@ -157,16 +173,18 @@ def _parse_args(): type=str, default=None, help="It's for mlu training. For example: " - "--mlus=\"0,1,2,3\" will launch four training processes each bound to one mlu." + "--mlus=\"0,1,2,3\" will launch four training processes each bound to one mlu.", ) base_group.add_argument("--selected_mlus", dest="mlus") - base_group.add_argument("training_script", - type=str, - help="The full path to the single GPU training " - "program/script to be launched in parallel, " - "followed by all the arguments for the " - "training script") + base_group.add_argument( + "training_script", + type=str, + help="The full path to the single GPU training " + "program/script to be launched in parallel, " + "followed by all the arguments for the " + "training script", + ) base_group.add_argument('training_script_args', nargs=REMAINDER) @@ -177,79 +195,86 @@ def _parse_args(): "--ips", type=str, default="127.0.0.1", - help="Paddle cluster nodes ips, such as 192.168.0.16,192.168.0.17..") + help="Paddle cluster nodes ips, such as 192.168.0.16,192.168.0.17..", + ) collective_group.add_argument( "--cluster_topo_path", type=str, default=None, help="A json format file will be stored in this path which is used" - "to represent the cluster topology information for auto parallel.") + "to represent the cluster topology information for auto parallel.", + ) collective_group.add_argument( "--rank_mapping_path", type=str, default=None, help="A json format file will be stored in this path which is used" - "to map processes to machines for auto parallel.") + "to map processes to machines for auto parallel.", + ) collective_group.add_argument( "--enable_auto_mapping", type=bool, default=False, - help="Set true to enable the lazy launch for auto-parallel scenario.") + help="Set true to enable the lazy launch for auto-parallel scenario.", + ) ps_group = parser.add_argument_group("Parameter-Server Parameters") # for parameter server - ps_group.add_argument("--servers", - type=str, - default="", - help="User defined servers ip:port") - ps_group.add_argument("--workers", - type=str, - default="", - help="User defined workers ip:port") - ps_group.add_argument("--coordinators", - type=str, - default="", - help="User defined coordinators ip:port") + ps_group.add_argument( + "--servers", type=str, default="", help="User defined servers ip:port" + ) + ps_group.add_argument( + "--workers", type=str, default="", help="User defined workers ip:port" + ) + ps_group.add_argument( + "--coordinators", + type=str, + default="", + help="User defined coordinators ip:port", + ) ps_group.add_argument( "--heter_workers", type=str, default="", - help="User defined heter workers in each stage ip1:port1;ip2:port2") + help="User defined heter workers in each stage ip1:port1;ip2:port2", + ) ps_group.add_argument( "--heter_devices", type=str, default="", - help="User defined heter devices in each stage cpu;gpu;cpu") + help="User defined heter devices in each stage cpu;gpu;cpu", + ) ps_group.add_argument("--worker_num", type=int, help="number of workers") - ps_group.add_argument("--coordinator_num", - type=int, - help="number of coordinators") + ps_group.add_argument( + "--coordinator_num", type=int, help="number of coordinators" + ) ps_group.add_argument("--server_num", type=int, help="number of servers") - ps_group.add_argument("--heter_worker_num", - type=str, - help="number of heter_workers in each stage 1;2;3") + ps_group.add_argument( + "--heter_worker_num", + type=str, + help="number of heter_workers in each stage 1;2;3", + ) ps_group.add_argument("--http_port", type=int, help="Gloo http Port") # parameter elastic mode elastic_group = parser.add_argument_group("Elastic Parameters") - elastic_group.add_argument("--elastic_server", - type=str, - help="etcd server host:port") - elastic_group.add_argument("--elastic_pre_hook", - type=str, - help="elastic pre_hook shell cmd") + elastic_group.add_argument( + "--elastic_server", type=str, help="etcd server host:port" + ) + elastic_group.add_argument( + "--elastic_pre_hook", type=str, help="elastic pre_hook shell cmd" + ) elastic_group.add_argument("--job_id", type=str, help="job unique id") elastic_group.add_argument("--np", type=int, help="job pod/node number") elastic_group.add_argument("--scale", type=int, default=0, help="scale np") - elastic_group.add_argument("--host", - type=str, - help="bind host, default to POD_IP env") - elastic_group.add_argument("--force", - type=bool, - default=False, - help="update np force") + elastic_group.add_argument( + "--host", type=str, help="bind host, default to POD_IP env" + ) + elastic_group.add_argument( + "--force", type=bool, default=False, help="update np force" + ) known_args, _ = parser.parse_known_args() return known_args @@ -265,16 +290,23 @@ def get_cluster_from_args(args, device_mode, devices_per_proc): else: _, node_ip = get_host_name_ip() - assert node_ip in node_ips, "Can't find your local ip {%s} in node_ips: {%s}" \ - % (node_ip, node_ips) + assert ( + node_ip in node_ips + ), "Can't find your local ip {%s} in node_ips: {%s}" % (node_ip, node_ips) node_rank = node_ips.index(node_ip) - logger.debug("parsed from args: node_ips:{} node_ip:{} node_rank:{}".format( - node_ips, node_ip, node_rank)) + logger.debug( + "parsed from args: node_ips:{} node_ip:{} node_rank:{}".format( + node_ips, node_ip, node_rank + ) + ) free_ports = None - if not cloud_utils.use_paddlecloud() and len( - node_ips) <= 1 and os.environ.get('FLAGS_START_PORT') is None: + if ( + not cloud_utils.use_paddlecloud() + and len(node_ips) <= 1 + and os.environ.get('FLAGS_START_PORT') is None + ): free_ports = find_free_ports(len(devices_per_proc)) if free_ports is not None: free_ports = list(free_ports) @@ -291,17 +323,21 @@ def get_cluster_from_args(args, device_mode, devices_per_proc): trainer_endpoints = [] for ip in node_ips: trainer_endpoints.append(["%s:%d" % (ip, port) for port in free_ports]) - return get_cluster(node_ips, node_ip, trainer_endpoints, device_mode, - devices_per_proc) + return get_cluster( + node_ips, node_ip, trainer_endpoints, device_mode, devices_per_proc + ) def cpuonly_check(args): if args.ips and len(args.ips.split(',')) > 1: raise RuntimeError( "CPUONLY launch only support single trainer, that is len(ips)=1, but got %s." - % args.ips) + % args.ips + ) if args.run_mode: - assert args.run_mode == 'cpuonly', "CPUONLY launch only support run mode is CPUONLY" + assert ( + args.run_mode == 'cpuonly' + ), "CPUONLY launch only support run mode is CPUONLY" if args.servers: raise RuntimeError("CPUONLY launch can't have --servers as arguments.") return True @@ -309,15 +345,20 @@ def cpuonly_check(args): def get_cluster_info(args): # parse arguments, used for cloud-single-machine and local - if args.backend == 'gloo': cpuonly_check(args) + if args.backend == 'gloo': + cpuonly_check(args) if args.enable_auto_mapping: (device_mode, devices_per_proc) = (DeviceMode.GPU, []) else: - (device_mode, - devices_per_proc) = launch_utils.get_device_proc_info(args) + (device_mode, devices_per_proc) = launch_utils.get_device_proc_info( + args + ) trainers_num = cloud_utils.get_trainers_num() - logger.debug("parsed from args trainerss_num:{} mode:{} devices:{}".format( - trainers_num, device_mode, devices_per_proc)) + logger.debug( + "parsed from args trainerss_num:{} mode:{} devices:{}".format( + trainers_num, device_mode, devices_per_proc + ) + ) cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES") @@ -329,52 +370,69 @@ def get_cluster_info(args): start_port = os.environ.get('FLAGS_START_PORT') # auto mapping between processes and devices for auto-parallel if args.enable_auto_mapping == True: - assert args.cluster_topo_path is not None, \ - "The cluster topology must be provied when enabling auto mapping." + assert ( + args.cluster_topo_path is not None + ), "The cluster topology must be provied when enabling auto mapping." rank_mapping_path = args.rank_mapping_path or os.getenv( - "PADDLE_RANK_MAPPING_PATH") + "PADDLE_RANK_MAPPING_PATH" + ) if not rank_mapping_path: os.environ["PADDLE_NEED_RANK_MAPPING"] = str(True) os.environ["PADDLE_ENABLE_ELASTIC"] = str( - enable_elastic(args, device_mode)) + enable_elastic(args, device_mode) + ) cwd = pathlib.Path().resolve() - rank_mapping_path = os.path.join(cwd, - "auto_parallel_rank_mapping.json") + rank_mapping_path = os.path.join( + cwd, "auto_parallel_rank_mapping.json" + ) os.environ["PADDLE_RANK_MAPPING_PATH"] = str(rank_mapping_path) original_args = sys.argv[1:] os.environ["PADDLE_ORIGINAL_CMD_ARGS"] = " ".join(original_args) os.environ["PADDLE_CLUSTER_TOPO_PATH"] = str(args.cluster_topo_path) os.environ["PADDLE_ENABLE_AUTO_MAPPING"] = str( - args.enable_auto_mapping) - cluster, pod = launch_utils.get_mapped_cluster_from_args_without_rank_mapping( - args, device_mode) + args.enable_auto_mapping + ) + ( + cluster, + pod, + ) = launch_utils.get_mapped_cluster_from_args_without_rank_mapping( + args, device_mode + ) else: os.environ["PADDLE_NEED_RANK_MAPPING"] = str(False) os.environ["PADDLE_ENABLE_ELASTIC"] = str( - enable_elastic(args, device_mode)) + enable_elastic(args, device_mode) + ) os.environ["PADDLE_CLUSTER_TOPO_PATH"] = str(args.cluster_topo_path) os.environ["PADDLE_RANK_MAPPING_PATH"] = str(rank_mapping_path) os.environ["PADDLE_ENABLE_AUTO_MAPPING"] = str( - args.enable_auto_mapping) - cluster, pod = launch_utils.get_mapped_cluster_from_args_with_rank_mapping( - args, device_mode) + args.enable_auto_mapping + ) + ( + cluster, + pod, + ) = launch_utils.get_mapped_cluster_from_args_with_rank_mapping( + args, device_mode + ) elif cloud_utils.use_paddlecloud() and trainers_num != 1: - cluster, pod = cloud_utils.get_cloud_cluster(args.ips, device_mode, - devices_per_proc, - start_port) + cluster, pod = cloud_utils.get_cloud_cluster( + args.ips, device_mode, devices_per_proc, start_port + ) logger.debug("get cluster from cloud:{}".format(cluster)) elif device_mode == DeviceMode.ASCEND_NPU: # for ascend - cluster, pod = ascend_utils.get_cloud_cluster(rank_table_file=os.getenv( - "RANK_TABLE_FILE", None), - device_mode=device_mode, - start_port=start_port) + cluster, pod = ascend_utils.get_cloud_cluster( + rank_table_file=os.getenv("RANK_TABLE_FILE", None), + device_mode=device_mode, + start_port=start_port, + ) else: # trainers_num = 1 or not use paddlecloud ips="a,b" - cluster, pod = get_cluster_from_args(args, device_mode, - devices_per_proc) + cluster, pod = get_cluster_from_args( + args, device_mode, devices_per_proc + ) logger.debug("get cluster from args:{}".format(cluster)) return cluster, pod @@ -394,12 +452,14 @@ def launch_collective(args): cluster, pod = get_cluster_info(args) global_envs = get_global_envs(args, tmp_dir) - procs = start_local_trainers(cluster, - pod, - training_script=args.training_script, - training_script_args=args.training_script_args, - log_dir=args.log_dir, - envs=global_envs) + procs = start_local_trainers( + cluster, + pod, + training_script=args.training_script, + training_script_args=args.training_script_args, + log_dir=args.log_dir, + envs=global_envs, + ) for idx, proc in enumerate(procs): print("launch proc_id:{} idx:{}".format(proc.proc.pid, idx)) @@ -431,7 +491,7 @@ def launch_ps(args, distribute_mode): if cloud_flag and distribute_mode == DistributeMode.PS: direct_start(args) return - #elif cloud_flag and distribute_mode == DistributeMode.PS_HETER: + # elif cloud_flag and distribute_mode == DistributeMode.PS_HETER: # cloud_ps_heter_env_set(args) # args.workers = os.getenv("PADDLE_TRAINER_ENDPOINTS") # args.servers = os.getenv("PADDLE_PSERVERS_IP_PORT_LIST") @@ -443,7 +503,8 @@ def launch_ps(args, distribute_mode): def infer_backend(args): - if args.backend != "auto": return + if args.backend != "auto": + return if fluid.core.is_compiled_with_cuda(): args.backend = 'nccl' elif fluid.core.is_compiled_with_npu(): @@ -469,8 +530,14 @@ def which_distributed_mode(args): return DistributeMode.PS_HETER ps_args = [ - '--worker_num', '--server_num', '--heter_worker_num', '--servers', - '--workers', '--heter_workers', '--heter_devices', '--http_port' + '--worker_num', + '--server_num', + '--heter_worker_num', + '--servers', + '--workers', + '--heter_workers', + '--heter_devices', + '--http_port', ] collective_args = ['--ips'] @@ -482,7 +549,8 @@ def which_distributed_mode(args): ps_arg for ps_arg in ps_args if ps_arg in " ".join(sys.argv[1:-1]) ] has_collective_args = [ - co_arg for co_arg in collective_args + co_arg + for co_arg in collective_args if co_arg in " ".join(sys.argv[1:-1]) ] @@ -504,8 +572,10 @@ def which_distributed_mode(args): if len(has_ps_args) > 0: logger.info( - "Run parameter-sever mode. pserver arguments:{}, accelerators count:{}" - .format(has_ps_args, accelerators)) + "Run parameter-sever mode. pserver arguments:{}, accelerators count:{}".format( + has_ps_args, accelerators + ) + ) has_ps_heter_args = list(set(has_ps_args) & set(ps_heter_args)) has_coordinator_args = list(set(has_ps_args) & set(coordinator_args)) if len(has_ps_heter_args) > 0: @@ -515,23 +585,29 @@ def which_distributed_mode(args): elif len(has_collective_args) > 0: logger.info( "Run collective mode. gpu arguments:{}, cuda count:{}".format( - has_collective_args, accelerators)) + has_collective_args, accelerators + ) + ) return DistributeMode.COLLECTIVE else: - if not fluid.core.is_compiled_with_cuda( - ) and not fluid.core.is_compiled_with_xpu( - ) and not fluid.core.is_compiled_with_mlu(): + if ( + not fluid.core.is_compiled_with_cuda() + and not fluid.core.is_compiled_with_xpu() + and not fluid.core.is_compiled_with_mlu() + ): if args.servers: logger.warning( "Not found distinct arguments and not compiled with cuda or xpu or npu or mlu. " - "But found args.servers not empty, default use ps mode") + "But found args.servers not empty, default use ps mode" + ) return DistributeMode.PS else: return DistributeMode.COLLECTIVE else: logger.warning( "Not found distinct arguments and compiled with cuda or xpu or npu or mlu. " - "Default use collective mode") + "Default use collective mode" + ) return DistributeMode.COLLECTIVE @@ -716,19 +792,23 @@ def launch(): if args.backend == 'auto': distribute_mode = which_distributed_mode( - args) # which_distributed_mode must modify args.backend + args + ) # which_distributed_mode must modify args.backend else: - assert args.run_mode == 'collective' or args.run_mode == None, "When backend is not 'auto', run mode must be collective" + assert ( + args.run_mode == 'collective' or args.run_mode == None + ), "When backend is not 'auto', run mode must be collective" check_backend(args.backend) distribute_mode = DistributeMode.COLLECTIVE - #assert args.backend in ['gloo', 'nccl', 'bkcl', 'cncl', 'heter', 'unknown'] + # assert args.backend in ['gloo', 'nccl', 'bkcl', 'cncl', 'heter', 'unknown'] if args.backend == 'gloo': logger.warning("launch start with CPUONLY mode") block_windows_and_macos( - args.backend) # raise error when using gloo on windows or macos + args.backend + ) # raise error when using gloo on windows or macos if enable_elastic(args, distribute_mode): launch_elastic(args, distribute_mode) diff --git a/python/paddle/distributed/fleet/launch_utils.py b/python/paddle/distributed/fleet/launch_utils.py index d29741cbef605..4ec2aa07787cb 100755 --- a/python/paddle/distributed/fleet/launch_utils.py +++ b/python/paddle/distributed/fleet/launch_utils.py @@ -35,19 +35,21 @@ logger.propagate = False -class DistributeMode(): +class DistributeMode: """ There are various mode for fleetrun, each of them is designed for different model. """ + COLLECTIVE = 0 PS = 1 PS_HETER = 2 -class DeviceMode(): +class DeviceMode: """ Training devices type """ + UNKNOWN = -1 CPU = 0 GPU = 1 @@ -59,7 +61,6 @@ class DeviceMode(): class Cluster(object): - def __init__(self, hdfs): self.job_server = None self.pods = [] @@ -68,8 +69,11 @@ def __init__(self, hdfs): def __str__(self): return "job_server:{} pods:{} job_stage_flag:{} hdfs:{}".format( - self.job_server, [str(pod) for pod in self.pods], - self.job_stage_flag, self.hdfs) + self.job_server, + [str(pod) for pod in self.pods], + self.job_stage_flag, + self.hdfs, + ) def __eq__(self, cluster): if len(self.pods) != len(cluster.pods): @@ -115,8 +119,9 @@ def pods_endpoints(self): r = [] for pod in self.pods: ep = "{}:{}".format(pod.addr, pod.port) - assert pod.port != None and pod.addr != None, "{} not a valid endpoint".format( - ep) + assert ( + pod.port != None and pod.addr != None + ), "{} not a valid endpoint".format(ep) r.append(ep) return r @@ -129,7 +134,6 @@ def get_pod_by_id(self, pod_id): class JobServer(object): - def __init__(self): self.endpoint = None @@ -144,7 +148,6 @@ def __ne__(self, j): class Trainer(object): - def __init__(self): self.accelerators = [] self.endpoint = None @@ -153,14 +156,14 @@ def __init__(self): def __str__(self): return "accelerator:{} endpoint:{} rank:{}".format( - self.accelerators, self.endpoint, self.rank) + self.accelerators, self.endpoint, self.rank + ) def __eq__(self, t): if len(self.accelerators) != len(t.accelerators): return False - if self.endpoint != t.endpoint or \ - self.rank != t.rank: + if self.endpoint != t.endpoint or self.rank != t.rank: return False for a, b in zip(self.accelerators, t.accelerators): @@ -177,7 +180,6 @@ def rank(self): class Pod(object): - def __init__(self): self.rank = None self.id = None @@ -194,29 +196,39 @@ def __init__(self): def __str__(self): return "rank:{} id:{} addr:{} port:{} visible_accelerator:{} trainers:{} servers:{} \ workers:{} heter_workers:{} coordinators:{}".format( - self.rank, self.id, self.addr, self.port, self.accelerators, - [str(t) for t in self.trainers], [str(s) for s in self.servers], - [str(w) - for w in self.workers], [str(h) for h in self.heter_workers], - [str(c) for c in self.coordinators]) + self.rank, + self.id, + self.addr, + self.port, + self.accelerators, + [str(t) for t in self.trainers], + [str(s) for s in self.servers], + [str(w) for w in self.workers], + [str(h) for h in self.heter_workers], + [str(c) for c in self.coordinators], + ) def __eq__(self, pod): - if self.rank != pod.rank or \ - self.id != pod.id or \ - self.addr != pod.addr or \ - self.port != pod.port: + if ( + self.rank != pod.rank + or self.id != pod.id + or self.addr != pod.addr + or self.port != pod.port + ): logger.debug("pod {} != {}".format(self, pod)) return False if len(self.trainers) != len(pod.trainers): - logger.debug("trainers {} != {}".format(self.trainers, - pod.trainers)) + logger.debug( + "trainers {} != {}".format(self.trainers, pod.trainers) + ) return False for i in range(len(self.trainers)): if self.trainers[i] != pod.trainers[i]: - logger.debug("trainer {} != {}".format(self.trainers[i], - pod.trainers[i])) + logger.debug( + "trainer {} != {}".format(self.trainers[i], pod.trainers[i]) + ) return False if len(self.servers) != len(pod.servers): @@ -225,8 +237,9 @@ def __eq__(self, pod): for i in range(len(self.servers)): if self.servers[i] != pod.servers[i]: - logger.debug("servers {} != {}".format(self.servers[i], - pod.servers[i])) + logger.debug( + "servers {} != {}".format(self.servers[i], pod.servers[i]) + ) return False if len(self.workers) != len(pod.workers): @@ -235,8 +248,9 @@ def __eq__(self, pod): for i in range(len(self.workers)): if self.workers[i] != pod.workers[i]: - logger.debug("workers {} != {}".format(self.workers[i], - pod.workers[i])) + logger.debug( + "workers {} != {}".format(self.workers[i], pod.workers[i]) + ) return False return True @@ -267,15 +281,17 @@ def get_logger(log_level=20, name="root"): log_handler = logging.StreamHandler() log_format = logging.Formatter( - '%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s') + '%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s' + ) log_handler.setFormatter(log_format) logger.addHandler(log_handler) return logger -def get_cluster(node_ips, node_ip, trainer_endpoints, device_mode, - devices_per_proc): +def get_cluster( + node_ips, node_ip, trainer_endpoints, device_mode, devices_per_proc +): assert type(trainer_endpoints) is list, "trainer_endpoints must be list" cluster = Cluster(hdfs=None) trainer_rank = 0 @@ -292,7 +308,11 @@ def get_cluster(node_ips, node_ip, trainer_endpoints, device_mode, ), "current trainer_endpoints size should be greater equal than acclerators size." for i in range(len(devices_per_proc)): trainer = Trainer() - if device_mode == DeviceMode.GPU or device_mode == DeviceMode.ASCEND_NPU or device_mode == DeviceMode.MLU: + if ( + device_mode == DeviceMode.GPU + or device_mode == DeviceMode.ASCEND_NPU + or device_mode == DeviceMode.MLU + ): if isinstance(devices_per_proc[i], (list, tuple)): trainer.accelerators.extend(devices_per_proc[i]) pod.accelerators.extend(devices_per_proc[i]) @@ -371,21 +391,23 @@ def add_arguments(argname, type, default, help, argparser, **kwargs): args = parser.parse_args() """ type = strtobool if type == bool else type - argparser.add_argument("--" + argname, - default=default, - type=type, - help=help + ' Default: %(default)s.', - **kwargs) + argparser.add_argument( + "--" + argname, + default=default, + type=type, + help=help + ' Default: %(default)s.', + **kwargs + ) def find_free_ports(num): - def __free_port(): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: # Note(wangxi): Close the connection with a TCP RST instead # of a TCP FIN, to avoid time_wait state. - s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, - struct.pack('ii', 1, 0)) + s.setsockopt( + socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0) + ) s.bind(('', 0)) return s.getsockname()[1] @@ -429,7 +451,8 @@ def pretty_print_envs(envs, header=None): max_k = max(max_k, len(k)) h_format = " " + "|{{:>{}s}}{}{{:^{}s}}|\n".format( - max_k, " " * spacing, max_v) + max_k, " " * spacing, max_v + ) l_format = " " + "|{{:>{}s}}{{}}{{:^{}s}}|\n".format(max_k, max_v) length = max_k + max_v + spacing @@ -461,7 +484,6 @@ def pretty_print_envs(envs, header=None): class TrainerProc(object): - def __init__(self): self.proc = None self.log_fn = None @@ -483,12 +505,9 @@ def run_with_coverage(*args): return _run_with_coverage -def start_local_trainers(cluster, - pod, - training_script, - training_script_args, - log_dir=None, - envs=None): +def start_local_trainers( + cluster, pod, training_script, training_script_args, log_dir=None, envs=None +): if envs is None: current_env = copy.copy(os.environ.copy()) @@ -507,74 +526,87 @@ def start_local_trainers(cluster, procs = [] for idx, t in enumerate(pod.trainers): proc_env = { - "PADDLE_TRAINER_ID": - "%d" % t.rank, - "PADDLE_CURRENT_ENDPOINT": - "%s" % t.endpoint, - "PADDLE_TRAINERS_NUM": - "%d" % cluster.trainers_nranks(), - "PADDLE_TRAINER_ENDPOINTS": - ",".join(cluster.trainers_endpoints()), - "PADDLE_RANK_IN_NODE": - str(idx), - "PADDLE_LOCAL_DEVICE_IDS": - ",".join([str(acc) for acc in t.accelerators]), - "PADDLE_WORLD_DEVICE_IDS": - ",".join(res), + "PADDLE_TRAINER_ID": "%d" % t.rank, + "PADDLE_CURRENT_ENDPOINT": "%s" % t.endpoint, + "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), + "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), + "PADDLE_RANK_IN_NODE": str(idx), + "PADDLE_LOCAL_DEVICE_IDS": ",".join( + [str(acc) for acc in t.accelerators] + ), + "PADDLE_WORLD_DEVICE_IDS": ",".join(res), } # The following three environnement variables are used for auto mapping if current_env.get("PADDLE_CLUSTER_TOPO_PATH", None) is not None: proc_env["PADDLE_CLUSTER_TOPO_PATH"] = current_env[ - "PADDLE_CLUSTER_TOPO_PATH"] + "PADDLE_CLUSTER_TOPO_PATH" + ] if current_env.get("PADDLE_RANK_MAPPING_PATH", None) is not None: proc_env["PADDLE_RANK_MAPPING_PATH"] = current_env[ - "PADDLE_RANK_MAPPING_PATH"] + "PADDLE_RANK_MAPPING_PATH" + ] if current_env.get("PADDLE_ENABLE_AUTO_MAPPING", None) is not None: proc_env["PADDLE_ENABLE_AUTO_MAPPING"] = current_env[ - "PADDLE_ENABLE_AUTO_MAPPING"] + "PADDLE_ENABLE_AUTO_MAPPING" + ] if len(t.accelerators) > 0 and pod.device_mode == DeviceMode.GPU: proc_env["FLAGS_selected_gpus"] = "%s" % ",".join( - [str(g) for g in t.accelerators]) + [str(g) for g in t.accelerators] + ) - elif len(t.accelerators - ) > 0 and pod.device_mode == DeviceMode.ASCEND_NPU: + elif ( + len(t.accelerators) > 0 and pod.device_mode == DeviceMode.ASCEND_NPU + ): proc_env["FLAGS_selected_npus"] = "%s" % ",".join( - [str(g) for g in t.accelerators]) + [str(g) for g in t.accelerators] + ) elif len(t.accelerators) > 0 and pod.device_mode == DeviceMode.MLU: proc_env["FLAGS_selected_mlus"] = "%s" % ",".join( - [str(g) for g in t.accelerators]) + [str(g) for g in t.accelerators] + ) if len(t.accelerators) > 0: proc_env["FLAGS_selected_accelerators"] = "%s" % ",".join( - [str(g) for g in t.accelerators]) + [str(g) for g in t.accelerators] + ) # to do: same code style in future if fluid.core.is_compiled_with_xpu() and len(t.accelerators) > 0: proc_env["FLAGS_selected_xpus"] = "%s" % ",".join( - [str(g) for g in t.accelerators]) + [str(g) for g in t.accelerators] + ) current_env.update(proc_env) coverage_args = [] - if run_with_coverage() or os.environ.get("WITH_COVERAGE", - "OFF") == "ON": + if ( + run_with_coverage() + or os.environ.get("WITH_COVERAGE", "OFF") == "ON" + ): coverage_args = ["-m", "coverage", "run", "--branch", "-p"] - cmd = [sys.executable, "-u"] + coverage_args + [training_script - ] + training_script_args + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [training_script] + + training_script_args + ) logger.debug("start trainer proc{} env:{}".format(cmd, current_env)) if idx == 0: - logger.info("Local start {} processes. First process distributed " - "environment info (Only For Debug): {}".format( - len(pod.trainers), - pretty_print_envs(proc_env, - ("Distributed Envs", "Value")))) + logger.info( + "Local start {} processes. First process distributed " + "environment info (Only For Debug): {}".format( + len(pod.trainers), + pretty_print_envs(proc_env, ("Distributed Envs", "Value")), + ) + ) logger.info( "details about PADDLE_TRAINER_ENDPOINTS can be found in " "{}/endpoints.log, and detail running logs maybe found in " - "{}/workerlog.0".format(log_dir, log_dir)) + "{}/workerlog.0".format(log_dir, log_dir) + ) fn = None pre_fn = None if os.name == 'nt' else os.setsid if log_dir is not None: @@ -584,16 +616,17 @@ def start_local_trainers(cluster, with open("%s/endpoints.log" % log_dir, "w") as f: f.write("PADDLE_TRAINER_ENDPOINTS: \n") f.write("\n".join(cluster.trainers_endpoints())) - if current_env.get("PADDLE_ENABLE_AUTO_MAPPING") is not None \ - and current_env.get("PADDLE_NEED_RANK_MAPPING").lower() == "true": + if ( + current_env.get("PADDLE_ENABLE_AUTO_MAPPING") is not None + and current_env.get("PADDLE_NEED_RANK_MAPPING").lower() + == "true" + ): fn = open("%s/prelaunchlog.%d" % (log_dir, idx), "a") else: fn = open("%s/workerlog.%d" % (log_dir, idx), "a") - proc = subprocess.Popen(cmd, - env=current_env, - stdout=fn, - stderr=fn, - preexec_fn=pre_fn) + proc = subprocess.Popen( + cmd, env=current_env, stdout=fn, stderr=fn, preexec_fn=pre_fn + ) else: proc = subprocess.Popen(cmd, env=current_env, preexec_fn=pre_fn) @@ -620,8 +653,9 @@ def pull_worker_log(tp): except UnicodeEncodeError: sys.stdout.write( 'UnicodeEncodeError occurs at this line. ' - 'Please refer to the original log file "%s"\n' % - tp.log_fn.name) + 'Please refer to the original log file "%s"\n' + % tp.log_fn.name + ) tp.log_offset = fin.tell() @@ -652,14 +686,18 @@ def watch_local_trainers(procs, nranks): return except SystemExit: logger.error( - "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log." - .format(nranks, error_rank)) + "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log.".format( + nranks, error_rank + ) + ) terminate_local_procs(procs) raise except: logger.error( - "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log." - .format(nranks, error_rank)) + "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log.".format( + nranks, error_rank + ) + ) terminate_local_procs(procs) return @@ -680,17 +718,22 @@ def get_gpus(gpus): # therefore gpus=0,1,2,3 cuda_visible_devices_list = cuda_visible_devices.split(',') for x in gpus.split(','): - assert x in cuda_visible_devices_list, "Can't find "\ - "your gpus %s in CUDA_VISIBLE_DEVICES[%s]."\ + assert x in cuda_visible_devices_list, ( + "Can't find " + "your gpus %s in CUDA_VISIBLE_DEVICES[%s]." % (x, cuda_visible_devices) + ) res_gpus = [ cuda_visible_devices_list.index(x.strip()) for x in gpus.split(',') ] - logger.info("Change selected_gpus into reletive values. --ips:{} " - "will change into relative_ips:{} according to your " - "CUDA_VISIBLE_DEVICES:{}".format( - gpus, res_gpus, cuda_visible_devices_list)) + logger.info( + "Change selected_gpus into reletive values. --ips:{} " + "will change into relative_ips:{} according to your " + "CUDA_VISIBLE_DEVICES:{}".format( + gpus, res_gpus, cuda_visible_devices_list + ) + ) return res_gpus @@ -709,17 +752,23 @@ def get_xpus(xpus): # therefore xpus=0,1,2,3 xpu_visible_devices_list = xpu_visible_devices.split(',') for x in xpus.split(','): - assert x in xpu_visible_devices_list, "Can't find "\ - "your xpus %s in XPU_VISIBLE_DEVICES[%s]."\ - % (x, xpu_visible_devices) + assert ( + x in xpu_visible_devices_list + ), "Can't find " "your xpus %s in XPU_VISIBLE_DEVICES[%s]." % ( + x, + xpu_visible_devices, + ) res_xpus = [ xpu_visible_devices_list.index(x.strip()) for x in xpus.split(',') ] - logger.info("Change selected_xpus into reletive values. --ips:{} " - "will change into relative_ips:{} according to your " - "XPU_VISIBLE_DEVICES:{}".format( - xpus, res_xpus, xpu_visible_devices_list)) + logger.info( + "Change selected_xpus into reletive values. --ips:{} " + "will change into relative_ips:{} according to your " + "XPU_VISIBLE_DEVICES:{}".format( + xpus, res_xpus, xpu_visible_devices_list + ) + ) return res_xpus @@ -738,17 +787,22 @@ def get_npus(npus): # therefore npus=0,1,2,3 npu_visible_devices_list = npu_visible_devices.split(',') for x in npus.split(','): - assert x in npu_visible_devices_list, "Can't find "\ - "your npus %s in ASCEND_VISIBLE_DEVICES[%s]."\ + assert x in npu_visible_devices_list, ( + "Can't find " + "your npus %s in ASCEND_VISIBLE_DEVICES[%s]." % (x, npu_visible_devices) + ) res_npus = [ npu_visible_devices_list.index(x.strip()) for x in npus.split(',') ] - logger.info("Change selected_npus into reletive values. --ips:{} " - "will change into relative_ips:{} according to your " - "ASCEND_VISIBLE_DEVICES:{}".format( - npus, res_npus, npu_visible_devices_list)) + logger.info( + "Change selected_npus into reletive values. --ips:{} " + "will change into relative_ips:{} according to your " + "ASCEND_VISIBLE_DEVICES:{}".format( + npus, res_npus, npu_visible_devices_list + ) + ) return res_npus @@ -767,33 +821,45 @@ def get_mlus(mlus): # therefore mlus=0,1,2,3 mlu_visible_devices_list = mlu_visible_devices.split(',') for x in mlus.split(','): - assert x in mlu_visible_devices_list, "Can't find "\ - "your mlus %s in MLU_VISIBLE_DEVICES[%s]."\ - % (x, mlu_visible_devices) + assert ( + x in mlu_visible_devices_list + ), "Can't find " "your mlus %s in MLU_VISIBLE_DEVICES[%s]." % ( + x, + mlu_visible_devices, + ) res_mlus = [ mlu_visible_devices_list.index(x.strip()) for x in mlus.split(',') ] - logger.info("Change selected_mlus into reletive values. --ips:{} " - "will change into relative_ips:{} according to your " - "MLU_VISIBLE_DEVICES:{}".format( - mlus, res_mlus, mlu_visible_devices_list)) + logger.info( + "Change selected_mlus into reletive values. --ips:{} " + "will change into relative_ips:{} according to your " + "MLU_VISIBLE_DEVICES:{}".format( + mlus, res_mlus, mlu_visible_devices_list + ) + ) return res_mlus def get_device_mode(backend): if backend == 'heter': - if fluid.core.is_compiled_with_cuda() and \ - fluid.core.get_cuda_device_count() > 0: + if ( + fluid.core.is_compiled_with_cuda() + and fluid.core.get_cuda_device_count() > 0 + ): print("launch train in heter mode with GPU device.") return DeviceMode.GPU - if fluid.core.is_compiled_with_xpu() and \ - fluid.core.get_xpu_device_count() > 0: + if ( + fluid.core.is_compiled_with_xpu() + and fluid.core.get_xpu_device_count() > 0 + ): print("launch train in heter mode with XPU device.") return DeviceMode.XPU - if fluid.core.is_compiled_with_npu() and \ - fluid.core.get_npu_device_count() > 0: + if ( + fluid.core.is_compiled_with_npu() + and fluid.core.get_npu_device_count() > 0 + ): print("launch train in heter mode with NPU device.") return DeviceMode.ASCEND_NPU @@ -801,8 +867,7 @@ def get_device_mode(backend): print("launch train in ascend npu mode!") return DeviceMode.ASCEND_NPU - if backend == 'nccl' and \ - fluid.core.get_cuda_device_count() > 0: + if backend == 'nccl' and fluid.core.get_cuda_device_count() > 0: print("launch train in GPU mode!") return DeviceMode.GPU @@ -830,62 +895,80 @@ def get_device_proc_info(args): if device_mode == DeviceMode.GPU: gpus = get_gpus(args.gpus) if args.nproc_per_node is not None: - assert (len(gpus) % int(args.nproc_per_node)) ==0, \ - "gpus' number:{} mod args.nproc_per_node:{} must == 0".format(len(gpus), args.nproc_per_node) + assert ( + len(gpus) % int(args.nproc_per_node) + ) == 0, "gpus' number:{} mod args.nproc_per_node:{} must == 0".format( + len(gpus), args.nproc_per_node + ) n = int(len(gpus) / int(args.nproc_per_node)) - devices_per_proc = [gpus[i:i + n] for i in range(0, len(gpus), n)] + devices_per_proc = [gpus[i : i + n] for i in range(0, len(gpus), n)] else: devices_per_proc = gpus elif device_mode == DeviceMode.ASCEND_NPU: npus = get_npus(args.npus) if args.nproc_per_node is not None: - assert (len(npus) % int(args.nproc_per_node)) ==0, \ - "npus' number:{} mod args.nproc_per_node:{} must == 0".format(len(npus), args.nproc_per_node) + assert ( + len(npus) % int(args.nproc_per_node) + ) == 0, "npus' number:{} mod args.nproc_per_node:{} must == 0".format( + len(npus), args.nproc_per_node + ) n = int(len(npus) / int(args.nproc_per_node)) - devices_per_proc = [npus[i:i + n] for i in range(0, len(npus), n)] + devices_per_proc = [npus[i : i + n] for i in range(0, len(npus), n)] else: devices_per_proc = npus elif device_mode == DeviceMode.XPU: xpus = get_xpus(args.xpus) if args.nproc_per_node is not None: - assert (len(xpus) % int(args.nproc_per_node)) == 0, \ - "xpus' number:{} mod args.nproc_per_node:{} must == 0".format(len(xpus), args.nproc_per_node) + assert ( + len(xpus) % int(args.nproc_per_node) + ) == 0, "xpus' number:{} mod args.nproc_per_node:{} must == 0".format( + len(xpus), args.nproc_per_node + ) n = int(len(xpus) / int(args.nproc_per_node)) - devices_per_proc = [xpus[i:i + n] for i in range(0, len(xpus), n)] + devices_per_proc = [xpus[i : i + n] for i in range(0, len(xpus), n)] else: devices_per_proc = xpus elif device_mode == DeviceMode.MLU: mlus = get_mlus(args.mlus) if args.nproc_per_node is not None: - assert (len(mlus) % int(args.nproc_per_node)) ==0, \ - "mlus' number:{} mod args.nproc_per_node:{} must == 0".format(len(mlus), args.nproc_per_node) + assert ( + len(mlus) % int(args.nproc_per_node) + ) == 0, "mlus' number:{} mod args.nproc_per_node:{} must == 0".format( + len(mlus), args.nproc_per_node + ) n = int(len(mlus) / int(args.nproc_per_node)) - devices_per_proc = [mlus[i:i + n] for i in range(0, len(mlus), n)] + devices_per_proc = [mlus[i : i + n] for i in range(0, len(mlus), n)] else: devices_per_proc = mlus elif device_mode == DeviceMode.CPU: if hasattr(args, "paddle_cpuonly") and args.nproc_per_node is None: - #NOTE (xiongkun03) set it to cpu core number + # NOTE (xiongkun03) set it to cpu core number args.nproc_per_node = multiprocessing.cpu_count() if args.nproc_per_node is None: devices_per_proc = [0] else: devices_per_proc = [x for x in range(0, args.nproc_per_node)] else: - assert False, "Can't support device_mode:{}, support only cpu|gpu|xpu now.".format( - device_mode) + assert ( + False + ), "Can't support device_mode:{}, support only cpu|gpu|xpu now.".format( + device_mode + ) return (device_mode, devices_per_proc) def direct_start(args): # run ps-cpu mode on paddlecloud, using given envs - cmd = [sys.executable, "-u", args.training_script] + \ - args.training_script_args + cmd = [ + sys.executable, + "-u", + args.training_script, + ] + args.training_script_args proc = subprocess.Popen(cmd) proc.wait() return @@ -907,7 +990,7 @@ def get_custom_endpoints(origin_endpoints, offset=0): return paddle_user_define_endpoints -#def cloud_ps_heter_env_set(args): +# def cloud_ps_heter_env_set(args): # environs = {} # # paddle_trainer_endpoints = os.getenv("TRAINER_IP_PORT_LIST", "") @@ -947,12 +1030,13 @@ def get_custom_endpoints(origin_endpoints, offset=0): # pretty_print_envs(environs))) -def get_mapped_cluster_without_rank_mapping(node_ips, node_ip, - trainer_endpoints, device_mode, - node_ranks): +def get_mapped_cluster_without_rank_mapping( + node_ips, node_ip, trainer_endpoints, device_mode, node_ranks +): assert type(trainer_endpoints) is list, "trainer_endpoints must be list" - assert device_mode == DeviceMode.GPU, \ - "Only support get mapped cluster for gpu now." + assert ( + device_mode == DeviceMode.GPU + ), "Only support get mapped cluster for gpu now." cluster = Cluster(hdfs=None) for node_rank, ip in enumerate(node_ips): pod = Pod() @@ -976,8 +1060,9 @@ def get_mapped_cluster_without_rank_mapping(node_ips, node_ip, def get_mapped_cluster_from_args_without_rank_mapping(args, device_mode): - assert device_mode == DeviceMode.GPU, \ - "Only support get mapped cluster for gpu now." + assert ( + device_mode == DeviceMode.GPU + ), "Only support get mapped cluster for gpu now." gpus_num = fluid.core.get_cuda_device_count() # parse ip-ranks json file @@ -999,17 +1084,21 @@ def get_mapped_cluster_from_args_without_rank_mapping(args, device_mode): else: _, node_ip = get_host_name_ip() - assert node_ip in node_ips, \ - "Can't find your local ip {%s} in node_ips: {%s}" % (node_ip, node_ips) + assert ( + node_ip in node_ips + ), "Can't find your local ip {%s} in node_ips: {%s}" % (node_ip, node_ips) node_rank = node_ips.index(node_ip) - assert len(node_ranks) == len(node_ips), \ - "ranks length should be equal to ips length." + assert len(node_ranks) == len( + node_ips + ), "ranks length should be equal to ips length." - logger.debug("parsed from args: node_ips:{} node_ip:{} " - "node_rank:{} node_ranks:{}".format(node_ips, node_ip, - node_rank, - node_ranks[node_rank])) + logger.debug( + "parsed from args: node_ips:{} node_ip:{} " + "node_rank:{} node_ranks:{}".format( + node_ips, node_ip, node_rank, node_ranks[node_rank] + ) + ) # NOTE: there are different number of global mapped ranks on each node. free_ports = [] @@ -1019,30 +1108,40 @@ def get_mapped_cluster_from_args_without_rank_mapping(args, device_mode): if os.environ.get('PADDLE_PORT') is not None: start_port = int(os.getenv("PADDLE_PORT", "")) free_ports = [ - x for x in range(start_port, start_port + - len(node_ranks[node_rank])) + x + for x in range( + start_port, start_port + len(node_ranks[node_rank]) + ) ] elif os.environ.get('FLAGS_START_PORT') is not None: start_port = int(os.environ.get('FLAGS_START_PORT')) free_ports = [ - x for x in range(start_port, start_port + - len(node_ranks[node_rank])) + x + for x in range( + start_port, start_port + len(node_ranks[node_rank]) + ) ] else: free_ports = find_free_ports(len(node_ranks[node_rank])) trainer_endpoints.append(["%s:%d" % (ip, port) for port in free_ports]) - return get_mapped_cluster_without_rank_mapping(node_ips, node_ip, - trainer_endpoints, - device_mode, node_ranks) + return get_mapped_cluster_without_rank_mapping( + node_ips, node_ip, trainer_endpoints, device_mode, node_ranks + ) -def get_mapped_cluster_with_rank_mapping(node_ips, node_ip, trainer_endpoints, - device_mode, node_ranks, - node_rank_mappings): +def get_mapped_cluster_with_rank_mapping( + node_ips, + node_ip, + trainer_endpoints, + device_mode, + node_ranks, + node_rank_mappings, +): assert type(trainer_endpoints) is list, "trainer_endpoints must be list" - assert device_mode == DeviceMode.GPU, \ - "Only support get mapped cluster for gpu now." + assert ( + device_mode == DeviceMode.GPU + ), "Only support get mapped cluster for gpu now." def get_relative_gpu_id(gpu_id): cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES") @@ -1052,8 +1151,10 @@ def get_relative_gpu_id(gpu_id): cuda_visible_devices_list = cuda_visible_devices.split(',') relative_id = cuda_visible_devices_list.index(str(gpu_id)) logger.info( - "Change gpu id from {} to {} based on CUDA_VISIBLE_DEVICES {}". - format(gpu_id, relative_id, cuda_visible_devices_list)) + "Change gpu id from {} to {} based on CUDA_VISIBLE_DEVICES {}".format( + gpu_id, relative_id, cuda_visible_devices_list + ) + ) return relative_id cluster = Cluster(hdfs=None) @@ -1069,12 +1170,15 @@ def get_relative_gpu_id(gpu_id): cur_node_rank_mapping = node_rank_mappings[node_rank] for i in range(len(ranks_per_node)): trainer = Trainer() - local_device_ids = cur_node_rank_mapping["ranks"][str( - ranks_per_node[i])] - assert len(local_device_ids) == 1, \ - "Only support one process to one device mapping" - trainer.accelerators.append(get_relative_gpu_id( - local_device_ids[0])) + local_device_ids = cur_node_rank_mapping["ranks"][ + str(ranks_per_node[i]) + ] + assert ( + len(local_device_ids) == 1 + ), "Only support one process to one device mapping" + trainer.accelerators.append( + get_relative_gpu_id(local_device_ids[0]) + ) trainer.endpoint = "%s" % (cur_node_endpoints[i]) trainer.rank = ranks_per_node[i] pod.trainers.append(trainer) @@ -1085,13 +1189,15 @@ def get_relative_gpu_id(gpu_id): def get_mapped_cluster_from_args_with_rank_mapping(args, device_mode): - assert device_mode == DeviceMode.GPU, \ - "Only support get mapped cluster for gpu now." + assert ( + device_mode == DeviceMode.GPU + ), "Only support get mapped cluster for gpu now." gpus_num = fluid.core.get_cuda_device_count() # parse ip-ranks json file rank_mapping_path = args.rank_mapping_path or os.getenv( - "PADDLE_RANK_MAPPING_PATH") + "PADDLE_RANK_MAPPING_PATH" + ) rank_mapping = None with open(rank_mapping_path, "r") as json_file: rank_mapping = json.load(json_file) @@ -1118,19 +1224,24 @@ def get_mapped_cluster_from_args_with_rank_mapping(args, device_mode): else: _, node_ip = get_host_name_ip() - assert node_ip in node_ips, \ - "Can't find your local ip {%s} in node_ips: {%s}" % (node_ip, node_ips) + assert ( + node_ip in node_ips + ), "Can't find your local ip {%s} in node_ips: {%s}" % (node_ip, node_ips) node_rank = node_ips.index(node_ip) - assert len(node_ranks[node_rank]) <= gpus_num, \ - "number of ranks mapped to one node should not exceed the avaiable ones." - assert len(node_ranks) == len(node_ips), \ - "ranks length should be equal to ips length." - - logger.debug("parsed from args: node_ips:{} node_ip:{} " - "node_rank:{} node_ranks:{}".format(node_ips, node_ip, - node_rank, - node_ranks[node_rank])) + assert ( + len(node_ranks[node_rank]) <= gpus_num + ), "number of ranks mapped to one node should not exceed the avaiable ones." + assert len(node_ranks) == len( + node_ips + ), "ranks length should be equal to ips length." + + logger.debug( + "parsed from args: node_ips:{} node_ip:{} " + "node_rank:{} node_ranks:{}".format( + node_ips, node_ip, node_rank, node_ranks[node_rank] + ) + ) # NOTE: there are different number of global mapped ranks on each node. free_ports = [] @@ -1140,26 +1251,34 @@ def get_mapped_cluster_from_args_with_rank_mapping(args, device_mode): if os.environ.get('PADDLE_PORT') is not None: start_port = int(os.getenv("PADDLE_PORT", "")) free_ports = [ - x for x in range(start_port, start_port + - len(node_ranks[node_rank])) + x + for x in range( + start_port, start_port + len(node_ranks[node_rank]) + ) ] elif os.environ.get('FLAGS_START_PORT') is not None: start_port = int(os.environ.get('FLAGS_START_PORT')) free_ports = [ - x for x in range(start_port, start_port + - len(node_ranks[node_rank])) + x + for x in range( + start_port, start_port + len(node_ranks[node_rank]) + ) ] else: free_ports = find_free_ports(len(node_ranks[node_rank])) trainer_endpoints.append(["%s:%d" % (ip, port) for port in free_ports]) - return get_mapped_cluster_with_rank_mapping(node_ips, node_ip, - trainer_endpoints, device_mode, - node_ranks, node_rank_mappings) + return get_mapped_cluster_with_rank_mapping( + node_ips, + node_ip, + trainer_endpoints, + device_mode, + node_ranks, + node_rank_mappings, + ) class ParameterServerLauncher(object): - def __init__(self, args, distribute_mode): self.args = args self.distribute_mode = distribute_mode @@ -1200,17 +1319,21 @@ def get_role_endpoints(self, args): if args.server_num: self.server_num = args.server_num if args.servers: - assert len( - args.servers.split(",") - ) == self.server_num, "The server_num and servers doesn't match. Expect servers endpoints num epual to server_num, but received servers enpoint num: {} and server_num {}".format( - len(args.servers.split(",")), self.server_num) + assert ( + len(args.servers.split(",")) == self.server_num + ), "The server_num and servers doesn't match. Expect servers endpoints num epual to server_num, but received servers enpoint num: {} and server_num {}".format( + len(args.servers.split(",")), self.server_num + ) self.server_endpoints = args.servers else: ports = get_ports(self.server_num, 0) self.server_endpoints = ",".join( - ["127.0.0.1:" + str(x) for x in ports]) + ["127.0.0.1:" + str(x) for x in ports] + ) else: - assert args.servers != "", "The setting of Parameter-Server must has server_num or servers." + assert ( + args.servers != "" + ), "The setting of Parameter-Server must has server_num or servers." self.server_endpoints = args.servers self.server_num = len(self.server_endpoints.split(",")) @@ -1218,18 +1341,22 @@ def get_role_endpoints(self, args): if args.worker_num: self.worker_num = args.worker_num if args.workers: - assert len( - args.workers.split(",") - ) == self.worker_num, "The worker_num and workers doesn't match. Expect workers endpoints num epual to worker_num, but received workers enpoint num: {} and worker_num {}".format( - len(args.workers.split(",")), self.worker_num) + assert ( + len(args.workers.split(",")) == self.worker_num + ), "The worker_num and workers doesn't match. Expect workers endpoints num epual to worker_num, but received workers enpoint num: {} and worker_num {}".format( + len(args.workers.split(",")), self.worker_num + ) self.worker_endpoints = args.workers else: ports = get_ports(self.worker_num, self.server_num) self.worker_endpoints = ",".join( - ["127.0.0.1:" + str(x) for x in ports]) + ["127.0.0.1:" + str(x) for x in ports] + ) else: - assert args.workers != "", "The setting of Parameter-Server must has worker_num or workers." + assert ( + args.workers != "" + ), "The setting of Parameter-Server must has worker_num or workers." worker_endpoints_ips = [ x.strip().split(":")[0] for x in args.workers.split(",") ] @@ -1243,13 +1370,20 @@ def get_role_endpoints(self, args): start_port = 6170 worker_endpoints_port = range( start_port + self.server_num, - start_port + self.server_num + self.worker_num, 1) + start_port + self.server_num + self.worker_num, + 1, + ) # create endpoints str worker_endpoints = [] for i in range(self.worker_num): - worker_endpoints.append(":".join( - (worker_endpoints_ips[i], - str(worker_endpoints_port[i])))) + worker_endpoints.append( + ":".join( + ( + worker_endpoints_ips[i], + str(worker_endpoints_port[i]), + ) + ) + ) self.worker_endpoints = ",".join(worker_endpoints) else: self.worker_endpoints = args.workers @@ -1259,21 +1393,25 @@ def get_role_endpoints(self, args): self.with_coordinator = True self.coordinator_num = args.coordinator_num if args.coordinators: - assert len( - args.coordinators.split(",") - ) == self.coordinator_num, "The coordinator_num and coordinators doesn't match. Expect coordinators endpoints num epual to coordinator_num, but received coordinator enpoint num: {} and coordinator_num {}".format( - len(args.coordinators.split(",")), self.coordinator_num) + assert ( + len(args.coordinators.split(",")) == self.coordinator_num + ), "The coordinator_num and coordinators doesn't match. Expect coordinators endpoints num epual to coordinator_num, but received coordinator enpoint num: {} and coordinator_num {}".format( + len(args.coordinators.split(",")), self.coordinator_num + ) self.coordinator_endpoints = args.coordinators else: ports = get_ports(self.coordinator_num, 1) self.coordinator_endpoints = ",".join( - ["127.0.0.1:" + str(x) for x in ports]) + ["127.0.0.1:" + str(x) for x in ports] + ) print(">>> use default coordinator addr(only one process)") # get heter worker envs if self.distribute_mode == DistributeMode.PS_HETER: - assert args.heter_devices != "", "The setting of Parameter-Server heter mode must has heter_devices." + assert ( + args.heter_devices != "" + ), "The setting of Parameter-Server heter mode must has heter_devices." self.stage_device_map[1] = "cpu" # for cpu trainer heter_devices_list = args.heter_devices.split(";") for i in range(len(heter_devices_list)): @@ -1292,19 +1430,22 @@ def get_role_endpoints(self, args): self.stage_heter_trainer_num ), "The stage_num and heter_workers doesn't match. Expect heter_workers endpoints stage num epual to heter_worker_num stage, but received heter_workers enpoint stage num: {} and heter_worker_num stage {}".format( len(args.heter_workers.split(";")), - len(self.stage_heter_trainer_num)) + len(self.stage_heter_trainer_num), + ) heter_worker_endpoints_list = args.heter_workers.split(";") self.heter_worker_endpoints = "" for i in range(len(self.stage_heter_trainer_num)): if self.heter_worker_endpoints != "": self.heter_worker_endpoints += "," heter_worker_endpoints = heter_worker_endpoints_list[ - i].split(",") - assert len( - heter_worker_endpoints - ) == self.stage_heter_trainer_num[ - i], "The heter trainer num in stage {} is not equal in args.heter_worker_num and args.heter_workers".format( - i) + i + ].split(",") + assert ( + len(heter_worker_endpoints) + == self.stage_heter_trainer_num[i] + ), "The heter trainer num in stage {} is not equal in args.heter_worker_num and args.heter_workers".format( + i + ) heter_worker_endpoints_ips = [ x.strip().split(":")[0] @@ -1319,20 +1460,28 @@ def get_role_endpoints(self, args): # if no port value in heter_worker_endpoint, will set default port values. heter_worker_endpoints_port = get_ports( len(heter_worker_endpoints_ips), - self.worker_num + self.server_num + - self.heter_worker_num) + self.worker_num + + self.server_num + + self.heter_worker_num, + ) new_heter_worker_endpoints = [] for j in range(len(heter_worker_endpoints_ips)): - new_heter_worker_endpoints.append(":".join( - (heter_worker_endpoints_ips[j], - str(heter_worker_endpoints_port[j])))) + new_heter_worker_endpoints.append( + ":".join( + ( + heter_worker_endpoints_ips[j], + str(heter_worker_endpoints_port[j]), + ) + ) + ) ip_port_list = ",".join(new_heter_worker_endpoints) else: ip_port_list = ",".join(heter_worker_endpoints) self.stage_heter_map[i + 2] = ip_port_list - self.stage_list.extend([i + 2] * - len(ip_port_list.split(','))) + self.stage_list.extend( + [i + 2] * len(ip_port_list.split(',')) + ) self.heter_worker_num += self.stage_heter_trainer_num[i] self.heter_worker_endpoints += ip_port_list @@ -1340,27 +1489,36 @@ def get_role_endpoints(self, args): for i in range(len(self.stage_heter_trainer_num)): heter_trainer_num = self.stage_heter_trainer_num[i] ports = get_ports( - heter_trainer_num, self.server_num + - self.worker_num + self.heter_worker_num) + heter_trainer_num, + self.server_num + + self.worker_num + + self.heter_worker_num, + ) ip_port_list = ",".join( - ["127.0.0.1:" + str(x) for x in ports]) + ["127.0.0.1:" + str(x) for x in ports] + ) self.stage_heter_map[i + 2] = ip_port_list - self.stage_list.extend([i + 2] * - len(ip_port_list.split(','))) + self.stage_list.extend( + [i + 2] * len(ip_port_list.split(',')) + ) self.heter_worker_num += heter_trainer_num if self.heter_worker_endpoints != "": self.heter_worker_endpoints += "," self.heter_worker_endpoints += ip_port_list else: - assert args.heter_workers != "", "The setting of Parameter-Server heter mode must has heter_worker_num or heter_workers." + assert ( + args.heter_workers != "" + ), "The setting of Parameter-Server heter mode must has heter_worker_num or heter_workers." self.stage_heter_trainer_num = [] heter_worker_endpoints_list = args.heter_workers.split(";") self.heter_worker_endpoints = "" for i in range(len(heter_worker_endpoints_list)): heter_worker_endpoints = heter_worker_endpoints_list[ - i].split(",") + i + ].split(",") self.stage_heter_trainer_num.append( - len(heter_worker_endpoints)) + len(heter_worker_endpoints) + ) heter_worker_endpoints_ips = [ x.strip().split(":")[0] for x in heter_worker_endpoints ] @@ -1371,29 +1529,39 @@ def get_role_endpoints(self, args): if 1 in heter_worker_endpoints_len: # if no port value in heter_worker_endpoint, will set default port values. heter_worker_endpoints_port = get_ports( - len(heter_worker_endpoints_ips), self.worker_num + - self.server_num + self.heter_worker_num) + len(heter_worker_endpoints_ips), + self.worker_num + + self.server_num + + self.heter_worker_num, + ) new_heter_worker_endpoints = [] for j in range(len(heter_worker_endpoints_ips)): - new_heter_worker_endpoints.append(":".join( - (heter_worker_endpoints_ips[j], - str(heter_worker_endpoints_port[j])))) + new_heter_worker_endpoints.append( + ":".join( + ( + heter_worker_endpoints_ips[j], + str(heter_worker_endpoints_port[j]), + ) + ) + ) ip_port_list = ",".join(new_heter_worker_endpoints) else: ip_port_list = ",".join(heter_worker_endpoints) self.stage_heter_map[i + 2] = ip_port_list - self.stage_list.extend([i + 2] * - len(ip_port_list.split(','))) + self.stage_list.extend( + [i + 2] * len(ip_port_list.split(',')) + ) self.heter_worker_num += self.stage_heter_trainer_num[-1] if self.heter_worker_endpoints != "": self.heter_worker_endpoints += "," self.heter_worker_endpoints += ip_port_list - self.stage_trainer_num = [self.worker_num - ] + self.stage_heter_trainer_num + self.stage_trainer_num = [ + self.worker_num + ] + self.stage_heter_trainer_num self.stage_num = len(self.stage_trainer_num) # get http_port @@ -1401,7 +1569,8 @@ def get_role_endpoints(self, args): http_port = [args.http_port] else: http_port = get_ports( - 1, self.server_num + self.worker_num + self.heter_worker_num) + 1, self.server_num + self.worker_num + self.heter_worker_num + ) http_ip = self.server_endpoints.split(",")[0].split(":")[0] self.http_port = http_ip + ":" + str(http_port[0]) @@ -1461,13 +1630,17 @@ def get_role_endpoints(self, args): else: self.current_node_ip = pod_ip if not self.distribute_mode == DistributeMode.PS_HETER: - assert self.current_node_ip in self.node_ips, "Can't find your local ip {%s} in args.servers and args.workers ips: {%s}" \ - % (self.current_node_ip, self.node_ips) + assert self.current_node_ip in self.node_ips, ( + "Can't find your local ip {%s} in args.servers and args.workers ips: {%s}" + % (self.current_node_ip, self.node_ips) + ) if self.current_node_ip in self.node_ips: self.node_rank = self.node_ips.index(self.current_node_ip) logger.debug( - "parsed from args: node_ips:{} current_node_ip:{} node_rank:{}". - format(self.node_ips, self.current_node_ip, self.node_rank)) + "parsed from args: node_ips:{} current_node_ip:{} node_rank:{}".format( + self.node_ips, self.current_node_ip, self.node_rank + ) + ) def start_ps(self): if self.current_node_ip not in self.node_ips: @@ -1484,16 +1657,20 @@ def start_ps(self): for i in range(len(self.server_endpoints_ips)): if ip == self.server_endpoints_ips[i]: server = Trainer() - server.endpoint = "%s:%s" % (ip, - self.server_endpoints_port[i]) + server.endpoint = "%s:%s" % ( + ip, + self.server_endpoints_port[i], + ) server.rank = server_rank server_rank += 1 pod.servers.append(server) for j in range(len(self.worker_endpoints_ips)): if ip == self.worker_endpoints_ips[j]: worker = Trainer() - worker.endpoint = "%s:%s" % (ip, - self.worker_endpoints_port[j]) + worker.endpoint = "%s:%s" % ( + ip, + self.worker_endpoints_port[j], + ) worker.rank = worker_rank worker.stage = 1 worker_rank += 1 @@ -1502,7 +1679,9 @@ def start_ps(self): if ip == self.coordinator_endpoints_ips[m]: coordinator = Trainer() coordinator.endpoint = "%s:%s" % ( - ip, self.coordinator_endpoints_port[m]) + ip, + self.coordinator_endpoints_port[m], + ) coordinator.rank = coordinator_rank coordinator.stage = 1 coordinator_rank += 1 @@ -1512,7 +1691,9 @@ def start_ps(self): if ip == self.heter_worker_endpoints_ips[k]: heter_worker = Trainer() heter_worker.endpoint = "%s:%s" % ( - ip, self.heter_worker_endpoints_port[k]) + ip, + self.heter_worker_endpoints_port[k], + ) heter_worker.rank = heter_worker_rank heter_worker.stage = self.stage_list[k] heter_worker_rank += 1 @@ -1528,19 +1709,19 @@ def start_ps(self): "worker": [], "coordinator": [], "server": [], - "heter_worker": [] + "heter_worker": [], } self.cmds = { "worker": [], "coordinator": [], "server": [], - "heter_worker": [] + "heter_worker": [], } self.log_fns = { "worker": [], "coordinator": [], "server": [], - "heter_worker": [] + "heter_worker": [], } self.start_pod_server(self.args, pod) @@ -1551,9 +1732,13 @@ def start_ps(self): self.start_pod_heter_worker(self.args, pod) logger.info( - "Please check servers, workers, coordinator and heter_worker logs in {}/workerlog.*, {}/serverlog.* , {}/coordinatorlog.*, and {}/heterlog.*" - .format(self.args.log_dir, self.args.log_dir, self.args.log_dir, - self.args.log_dir)) + "Please check servers, workers, coordinator and heter_worker logs in {}/workerlog.*, {}/serverlog.* , {}/coordinatorlog.*, and {}/heterlog.*".format( + self.args.log_dir, + self.args.log_dir, + self.args.log_dir, + self.args.log_dir, + ) + ) # 4. wait for finish training if len(self.procs["worker"]) > 0: @@ -1609,8 +1794,7 @@ def start_pod_server(self, args, pod): "PADDLE_PSERVERS_IP_PORT_LIST": self.server_endpoints, "PADDLE_TRAINER_ENDPOINTS": self.worker_endpoints, "PADDLE_COORDINATOR_ENDPOINTS": self.coordinator_endpoints, - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST": - self.heter_worker_endpoints, + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST": self.heter_worker_endpoints, "PADDLE_PORT": cur_server.endpoint.split(":")[1], "TRAINING_ROLE": "PSERVER", "PADDLE_TRAINERS_NUM": str(self.worker_num), @@ -1618,7 +1802,7 @@ def start_pod_server(self, args, pod): "PADDLE_WITH_GLOO": str(os.getenv("PADDLE_WITH_GLOO", "0")), "PADDLE_GLOO_RENDEZVOUS": "3", "PADDLE_GLOO_FS_PATH": self.gloo_rendezvous_dir, - "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port + "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port, } else: proc_env = { @@ -1632,12 +1816,15 @@ def start_pod_server(self, args, pod): "PADDLE_WITH_GLOO": str(os.getenv("PADDLE_WITH_GLOO", "0")), "PADDLE_GLOO_RENDEZVOUS": "3", "PADDLE_GLOO_FS_PATH": self.gloo_rendezvous_dir, - "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port + "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port, } current_env.update(proc_env) - cmd = [sys.executable, "-u", args.training_script - ] + args.training_script_args + cmd = [ + sys.executable, + "-u", + args.training_script, + ] + args.training_script_args self.cmds["server"].append(cmd) if idx == 0: @@ -1645,17 +1832,19 @@ def start_pod_server(self, args, pod): "Local server start {} processes. First process distributed " "environment info (Only For Debug): {}".format( len(pod.servers), - pretty_print_envs(proc_env, - ("Distributed Envs", "Value")))) + pretty_print_envs( + proc_env, ("Distributed Envs", "Value") + ), + ) + ) if args.log_dir is not None: os.system("mkdir -p {}".format(args.log_dir)) fn = open("%s/serverlog.%d" % (args.log_dir, idx), "w") self.log_fns["server"].append(fn) - proc = subprocess.Popen(cmd, - env=current_env, - stdout=fn, - stderr=fn) + proc = subprocess.Popen( + cmd, env=current_env, stdout=fn, stderr=fn + ) else: proc = subprocess.Popen(cmd, env=current_env) @@ -1685,56 +1874,38 @@ def start_pod_worker(self, args, pod): device_list = [str(x) for x in range(0, heter_device_num)] for idx, cur_worker in enumerate(pod.workers): - device_id = "0" if heter_device_num == 0 else str( - device_list[(idx) % heter_device_num]) + device_id = ( + "0" + if heter_device_num == 0 + else str(device_list[(idx) % heter_device_num]) + ) if self.distribute_mode == DistributeMode.PS_HETER: proc_env = { - "PADDLE_PSERVERS_IP_PORT_LIST": - self.server_endpoints, - "PADDLE_TRAINER_ENDPOINTS": - self.worker_endpoints, - "PADDLE_TRAINERS_NUM": - str(self.worker_num), - "PADDLE_COORDINATOR_ENDPOINTS": - self.coordinator_endpoints, - "PADDLE_STAGE_TRAINERS_NUM": - str(self.stage_trainer_num), - "STAGE_ID": - "1", - "STAGE_NUM": - str(self.stage_num), - "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST": - "", - "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST": - self.stage_heter_map[2], - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST": - self.heter_worker_endpoints, - "HETER_DEVICE_TYPE": - self.stage_device_map[1], - "TRAINING_ROLE": - "TRAINER", - "POD_IP": - cur_worker.endpoint.split(":")[0], - "PADDLE_PORT": - cur_worker.endpoint.split(":")[1], - "PADDLE_TRAINER_ID": - str(cur_worker.rank), - "PADDLE_WITH_GLOO": - str(os.getenv("PADDLE_WITH_GLOO", "0")), - "PADDLE_GLOO_RENDEZVOUS": - "3", - "PADDLE_GLOO_FS_PATH": - self.gloo_rendezvous_dir, - "FLAGS_selected_gpus": - "0", - "FLAGS_selected_xpus": - "0", - "CUDA_VISIBLE_DEVICES": - device_id, - "XPU_VISIBLE_DEVICES": - device_id, - "PADDLE_GLOO_HTTP_ENDPOINT": - self.http_port + "PADDLE_PSERVERS_IP_PORT_LIST": self.server_endpoints, + "PADDLE_TRAINER_ENDPOINTS": self.worker_endpoints, + "PADDLE_TRAINERS_NUM": str(self.worker_num), + "PADDLE_COORDINATOR_ENDPOINTS": self.coordinator_endpoints, + "PADDLE_STAGE_TRAINERS_NUM": str(self.stage_trainer_num), + "STAGE_ID": "1", + "STAGE_NUM": str(self.stage_num), + "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST": "", + "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST": self.stage_heter_map[ + 2 + ], + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST": self.heter_worker_endpoints, + "HETER_DEVICE_TYPE": self.stage_device_map[1], + "TRAINING_ROLE": "TRAINER", + "POD_IP": cur_worker.endpoint.split(":")[0], + "PADDLE_PORT": cur_worker.endpoint.split(":")[1], + "PADDLE_TRAINER_ID": str(cur_worker.rank), + "PADDLE_WITH_GLOO": str(os.getenv("PADDLE_WITH_GLOO", "0")), + "PADDLE_GLOO_RENDEZVOUS": "3", + "PADDLE_GLOO_FS_PATH": self.gloo_rendezvous_dir, + "FLAGS_selected_gpus": "0", + "FLAGS_selected_xpus": "0", + "CUDA_VISIBLE_DEVICES": device_id, + "XPU_VISIBLE_DEVICES": device_id, + "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port, } else: proc_env = { @@ -1753,12 +1924,15 @@ def start_pod_worker(self, args, pod): "FLAGS_selected_xpus": "0", "CUDA_VISIBLE_DEVICES": device_id, "XPU_VISIBLE_DEVICES": device_id, - "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port + "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port, } current_env.update(proc_env) - cmd = [sys.executable, "-u", args.training_script - ] + args.training_script_args + cmd = [ + sys.executable, + "-u", + args.training_script, + ] + args.training_script_args self.cmds["worker"].append(cmd) if idx == 0: @@ -1766,17 +1940,19 @@ def start_pod_worker(self, args, pod): "Local worker start {} processes. First process distributed " "environment info (Only For Debug): {}".format( len(pod.workers), - pretty_print_envs(proc_env, - ("Distributed Envs", "Value")))) + pretty_print_envs( + proc_env, ("Distributed Envs", "Value") + ), + ) + ) if args.log_dir is not None: os.system("mkdir -p {}".format(args.log_dir)) fn = open("%s/workerlog.%d" % (args.log_dir, idx), "w") self.log_fns["worker"].append(fn) - proc = subprocess.Popen(cmd, - env=current_env, - stdout=fn, - stderr=fn) + proc = subprocess.Popen( + cmd, env=current_env, stdout=fn, stderr=fn + ) else: proc = subprocess.Popen(cmd, env=current_env) @@ -1816,12 +1992,15 @@ def start_pod_coordinator(self, args, pod): "FLAGS_selected_xpus": "0", "CUDA_VISIBLE_DEVICES": device_id, "XPU_VISIBLE_DEVICES": device_id, - "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port + "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port, } current_env.update(proc_env) - cmd = [sys.executable, "-u", args.training_script - ] + args.training_script_args + cmd = [ + sys.executable, + "-u", + args.training_script, + ] + args.training_script_args self.cmds["coordinator"].append(cmd) if idx == 0: @@ -1829,17 +2008,19 @@ def start_pod_coordinator(self, args, pod): "Local coordinator start {} processes. First process distributed " "environment info (Only For Debug): {}".format( len(pod.coordinators), - pretty_print_envs(proc_env, - ("Distributed Envs", "Value")))) + pretty_print_envs( + proc_env, ("Distributed Envs", "Value") + ), + ) + ) if args.log_dir is not None: os.system("mkdir -p {}".format(args.log_dir)) fn = open("%s/coordinator.%d" % (args.log_dir, idx), "w") self.log_fns["coordinator"].append(fn) - proc = subprocess.Popen(cmd, - env=current_env, - stdout=fn, - stderr=fn) + proc = subprocess.Popen( + cmd, env=current_env, stdout=fn, stderr=fn + ) else: proc = subprocess.Popen(cmd, env=current_env) @@ -1869,58 +2050,48 @@ def start_pod_heter_worker(self, args, pod): device_list = [str(x) for x in range(0, heter_device_num)] for idx, cur_heter_worker in enumerate(pod.heter_workers): - device_id = "0" if heter_device_num == 0 else str( - device_list[(idx) % heter_device_num]) + device_id = ( + "0" + if heter_device_num == 0 + else str(device_list[(idx) % heter_device_num]) + ) stage_id = cur_heter_worker.stage proc_env = { - "PADDLE_PSERVERS_IP_PORT_LIST": - self.server_endpoints, - "PADDLE_TRAINER_ENDPOINTS": - self.worker_endpoints, - "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST": - self.stage_heter_map[stage_id + 1] - if stage_id <= self.stage_num - 1 else "", - "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST": - self.stage_heter_map[stage_id - 1], - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST": - self.heter_worker_endpoints, - "HETER_DEVICE_TYPE": - self.stage_device_map[stage_id], - "STAGE_ID": - str(stage_id), - "STAGE_NUM": - str(self.stage_num), - "PADDLE_PORT": - cur_heter_worker.endpoint.split(":")[1], - "TRAINING_ROLE": - "HETER_TRAINER", - "PADDLE_TRAINERS_NUM": - str(self.worker_num), - "PADDLE_STAGE_TRAINERS_NUM": - str(self.stage_trainer_num), - "POD_IP": - cur_heter_worker.endpoint.split(":")[0], - "PADDLE_WITH_GLOO": - str(os.getenv("PADDLE_WITH_GLOO", "0")), - "PADDLE_GLOO_RENDEZVOUS": - "3", - "PADDLE_GLOO_FS_PATH": - self.gloo_rendezvous_dir, - "FLAGS_selected_gpus": - "0", - "FLAGS_selected_xpus": - "0", - "CUDA_VISIBLE_DEVICES": - device_id, - "XPU_VISIBLE_DEVICES": - device_id, - "PADDLE_GLOO_HTTP_ENDPOINT": - self.http_port + "PADDLE_PSERVERS_IP_PORT_LIST": self.server_endpoints, + "PADDLE_TRAINER_ENDPOINTS": self.worker_endpoints, + "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST": self.stage_heter_map[ + stage_id + 1 + ] + if stage_id <= self.stage_num - 1 + else "", + "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST": self.stage_heter_map[ + stage_id - 1 + ], + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST": self.heter_worker_endpoints, + "HETER_DEVICE_TYPE": self.stage_device_map[stage_id], + "STAGE_ID": str(stage_id), + "STAGE_NUM": str(self.stage_num), + "PADDLE_PORT": cur_heter_worker.endpoint.split(":")[1], + "TRAINING_ROLE": "HETER_TRAINER", + "PADDLE_TRAINERS_NUM": str(self.worker_num), + "PADDLE_STAGE_TRAINERS_NUM": str(self.stage_trainer_num), + "POD_IP": cur_heter_worker.endpoint.split(":")[0], + "PADDLE_WITH_GLOO": str(os.getenv("PADDLE_WITH_GLOO", "0")), + "PADDLE_GLOO_RENDEZVOUS": "3", + "PADDLE_GLOO_FS_PATH": self.gloo_rendezvous_dir, + "FLAGS_selected_gpus": "0", + "FLAGS_selected_xpus": "0", + "CUDA_VISIBLE_DEVICES": device_id, + "XPU_VISIBLE_DEVICES": device_id, + "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port, } current_env.update(proc_env) - cmd = [sys.executable, "-u", args.training_script - ] + args.training_script_args + cmd = [ + sys.executable, + "-u", + args.training_script, + ] + args.training_script_args self.cmds["heter_worker"].append(cmd) if idx == 0: @@ -1928,17 +2099,19 @@ def start_pod_heter_worker(self, args, pod): "Local heter_worker start {} processes. First process distributed " "environment info (Only For Debug): {}".format( len(pod.heter_workers), - pretty_print_envs(proc_env, - ("Distributed Envs", "Value")))) + pretty_print_envs( + proc_env, ("Distributed Envs", "Value") + ), + ) + ) if args.log_dir is not None: os.system("mkdir -p {}".format(args.log_dir)) fn = open("%s/heterlog.%d" % (args.log_dir, idx), "w") self.log_fns["heter_worker"].append(fn) - proc = subprocess.Popen(cmd, - env=current_env, - stdout=fn, - stderr=fn) + proc = subprocess.Popen( + cmd, env=current_env, stdout=fn, stderr=fn + ) else: proc = subprocess.Popen(cmd, env=current_env) @@ -1955,13 +2128,21 @@ def start_pod_heter_worker(self, args, pod): def check_backend(backend): if backend not in [ - 'nccl', 'gloo', 'bkcl', 'cncl', 'auto', 'hccl', 'heter', 'xccl' + 'nccl', + 'gloo', + 'bkcl', + 'cncl', + 'auto', + 'hccl', + 'heter', + 'xccl', ]: raise ValueError( "paddle.distributed initialize error, " "backend argument can only be one of " "'nccl', 'gloo', 'bkcl', 'auto', 'hccl', 'heter', 'xccl' " - "but got %s" % backend) + "but got %s" % backend + ) if backend == 'nccl' and not fluid.core.is_compiled_with_cuda(): raise ValueError( @@ -1989,7 +2170,8 @@ def check_backend(backend): def block_windows_and_macos(backend): - if backend != 'gloo': return + if backend != 'gloo': + return if utils.OS_NAME.startswith('darwin'): # MACOS , block raise ValueError( "You are going to using gloo on macos, but currently is not supported" diff --git a/python/paddle/distributed/fleet/layers/mpu/mp_layers.py b/python/paddle/distributed/fleet/layers/mpu/mp_layers.py index 673f9b0f8a7ab..7ce317f8a38b0 100644 --- a/python/paddle/distributed/fleet/layers/mpu/mp_layers.py +++ b/python/paddle/distributed/fleet/layers/mpu/mp_layers.py @@ -86,27 +86,38 @@ def forward(self, x): return x """ - def __init__(self, - num_embeddings, - embedding_dim, - weight_attr=None, - mp_group=None, - name=None): + def __init__( + self, + num_embeddings, + embedding_dim, + weight_attr=None, + mp_group=None, + name=None, + ): super(VocabParallelEmbedding, self).__init__() - self.model_parallel_group = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group( - ) if mp_group is None else mp_group - self.world_size = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size( - ) if mp_group is None else mp_group.nranks - self.rank = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_rank( - ) if mp_group is None else mp_group.rank + self.model_parallel_group = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group() + if mp_group is None + else mp_group + ) + self.world_size = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size() + if mp_group is None + else mp_group.nranks + ) + self.rank = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_rank() + if mp_group is None + else mp_group.rank + ) self.origin_num_embeddings = num_embeddings - self.is_mp = (self.world_size > 1) + self.is_mp = self.world_size > 1 - assert num_embeddings % self.world_size == 0, ( - "The length of the vocabulary must be divisible by the parallelism degree of MP" - ) + assert ( + num_embeddings % self.world_size == 0 + ), "The length of the vocabulary must be divisible by the parallelism degree of MP" per_part_size = num_embeddings // self.world_size @@ -118,15 +129,19 @@ def __init__(self, if self.is_mp and paddle.in_dynamic_mode(): with get_rng_state_tracker().rng_state(): - self.weight = self.create_parameter(attr=self._weight_attr, - shape=self._size, - dtype=self._dtype, - is_bias=False) + self.weight = self.create_parameter( + attr=self._weight_attr, + shape=self._size, + dtype=self._dtype, + is_bias=False, + ) else: - self.weight = self.create_parameter(attr=self._weight_attr, - shape=self._size, - dtype=self._dtype, - is_bias=False) + self.weight = self.create_parameter( + attr=self._weight_attr, + shape=self._size, + dtype=self._dtype, + is_bias=False, + ) self.weight.is_distributed = True if self.is_mp else False @@ -136,17 +151,22 @@ def forward(self, x): self.weight, x, start_index=self.vocab_start_index, - name=self._name) - output = mp_ops._mp_allreduce(output_parallel, - group=self.model_parallel_group, - use_calc_stream=True, - use_model_parallel=True) + name=self._name, + ) + output = mp_ops._mp_allreduce( + output_parallel, + group=self.model_parallel_group, + use_calc_stream=True, + use_model_parallel=True, + ) else: - output = F.embedding(x, - weight=self.weight, - padding_idx=None, - sparse=False, - name=self._name) + output = F.embedding( + x, + weight=self.weight, + padding_idx=None, + sparse=False, + name=self._name, + ) return output @@ -200,29 +220,39 @@ def forward(self, x): return x """ - def __init__(self, - in_features, - out_features, - weight_attr=None, - has_bias=None, - gather_output=True, - fuse_matmul_bias=False, - mp_group=None, - name=None): + def __init__( + self, + in_features, + out_features, + weight_attr=None, + has_bias=None, + gather_output=True, + fuse_matmul_bias=False, + mp_group=None, + name=None, + ): super(ColumnParallelLinear, self).__init__() - self.model_parallel_group = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group( - ) if mp_group is None else mp_group - self.world_size = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size( - ) if mp_group is None else mp_group.nranks + self.model_parallel_group = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group() + if mp_group is None + else mp_group + ) + self.world_size = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size() + if mp_group is None + else mp_group.nranks + ) self._name = name - self.is_mp = (self.world_size > 1) + self.is_mp = self.world_size > 1 self.gather_output = gather_output assert out_features % self.world_size == 0, ( "Number of column of the weight for linear ({}) must be" " divisible by model parallel size ({})".format( - out_features, self.world_size)) + out_features, self.world_size + ) + ) self.output_size_per_partition = out_features // self.world_size self._weight_attr = weight_attr @@ -234,13 +264,15 @@ def __init__(self, shape=[in_features, self.output_size_per_partition], attr=self._weight_attr, dtype=self._dtype, - is_bias=False) + is_bias=False, + ) else: self.weight = self.create_parameter( shape=[in_features, self.output_size_per_partition], attr=self._weight_attr, dtype=self._dtype, - is_bias=False) + is_bias=False, + ) self.weight.is_distributed = True if self.is_mp else False @@ -250,7 +282,8 @@ def __init__(self, shape=[self.output_size_per_partition], attr=paddle.nn.initializer.Constant(value=0.0), dtype=self._dtype, - is_bias=True) + is_bias=True, + ) self.bias.is_distributed = True if self.is_mp else False else: self.bias = None @@ -263,26 +296,29 @@ def __init__(self, "You set fuse_matmul_bias=True in ColumnParallelLinear, " "however, the paddle you are using not support this operation. " "Please set fuse_matmul_bias=False or use paddle compiled " - "with cuda 11.6 or higher.") + "with cuda 11.6 or higher." + ) from paddle.incubate.nn.functional import fused_linear + self.linear = fused_linear def forward(self, x): # use inner api to process identity if self.is_mp: - input_parallel = mp_ops._c_identity(x, - group=self.model_parallel_group) + input_parallel = mp_ops._c_identity( + x, group=self.model_parallel_group + ) else: input_parallel = x - output_parallel = self.linear(input_parallel, - self.weight, - self.bias, - name=self._name) + output_parallel = self.linear( + input_parallel, self.weight, self.bias, name=self._name + ) if self.gather_output and self.is_mp: - output = mp_ops._c_concat(output_parallel, - group=self.model_parallel_group) + output = mp_ops._c_concat( + output_parallel, group=self.model_parallel_group + ) else: output = output_parallel return output @@ -338,15 +374,17 @@ def forward(self, x): return x """ - def __init__(self, - in_features, - out_features, - weight_attr=None, - has_bias=True, - input_is_parallel=False, - fuse_matmul_bias=False, - mp_group=None, - name=None): + def __init__( + self, + in_features, + out_features, + weight_attr=None, + has_bias=True, + input_is_parallel=False, + fuse_matmul_bias=False, + mp_group=None, + name=None, + ): super(RowParallelLinear, self).__init__() self.in_features = in_features @@ -356,18 +394,29 @@ def __init__(self, self._dtype = self._helper.get_default_dtype() self._name = name - self.model_parallel_group = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group( - ) if mp_group is None else mp_group - self.world_size = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size( - ) if mp_group is None else mp_group.nranks - self.rank = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_rank( - ) if mp_group is None else mp_group.rank + self.model_parallel_group = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group() + if mp_group is None + else mp_group + ) + self.world_size = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size() + if mp_group is None + else mp_group.nranks + ) + self.rank = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_rank() + if mp_group is None + else mp_group.rank + ) - self.is_mp = (self.world_size > 1) + self.is_mp = self.world_size > 1 assert in_features % self.world_size == 0, ( "Number of row of the weight for linear ({}) must be" " divisible by model parallel size ({})".format( - in_features, self.world_size)) + in_features, self.world_size + ) + ) self.input_size_per_partition = in_features // self.world_size @@ -377,13 +426,15 @@ def __init__(self, shape=[self.input_size_per_partition, self.out_features], attr=self._weight_attr, dtype=self._dtype, - is_bias=False) + is_bias=False, + ) else: self.weight = self.create_parameter( shape=[self.input_size_per_partition, self.out_features], attr=self._weight_attr, dtype=self._dtype, - is_bias=False) + is_bias=False, + ) self.weight.is_distributed = True if self.is_mp else False @@ -392,7 +443,8 @@ def __init__(self, shape=[self.out_features], attr=paddle.nn.initializer.Constant(value=0.0), dtype=self._dtype, - is_bias=True) + is_bias=True, + ) else: self.bias = None @@ -404,8 +456,10 @@ def __init__(self, "You set fuse_matmul_bias=True in RowParallelLinear, " "however, the paddle you are using not support this operation. " "Please set fuse_matmul_bias=False or use paddle compiled " - "with cuda 11.6 or higher.") + "with cuda 11.6 or higher." + ) from paddle.incubate.nn.functional import fused_linear + self.linear = fused_linear def forward(self, x): @@ -416,19 +470,20 @@ def forward(self, x): input_parallel = mp_ops._c_split(x, group=self.model_parallel_group) if self.is_mp: - output_parallel = self.linear(input_parallel, - self.weight, - name=self._name) - output_ = mp_ops._mp_allreduce(output_parallel, - group=self.model_parallel_group, - use_calc_stream=True, - use_model_parallel=True) + output_parallel = self.linear( + input_parallel, self.weight, name=self._name + ) + output_ = mp_ops._mp_allreduce( + output_parallel, + group=self.model_parallel_group, + use_calc_stream=True, + use_model_parallel=True, + ) output = output_ + self.bias if self.bias is not None else output_ else: - output = self.linear(input_parallel, - self.weight, - self.bias, - name=self._name) + output = self.linear( + input_parallel, self.weight, self.bias, name=self._name + ) return output @@ -451,14 +506,24 @@ class ParallelCrossEntropy(Layer): def __init__(self, mp_group=None, name=None): super(ParallelCrossEntropy, self).__init__() self.name = name - self.model_parallel_group = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group( - ) if mp_group is None else mp_group - self.world_size = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size( - ) if mp_group is None else mp_group.nranks - self.rank = tp._HYBRID_PARALLEL_GROUP.get_model_parallel_rank( - ) if mp_group is None else mp_group.rank + self.model_parallel_group = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_group() + if mp_group is None + else mp_group + ) + self.world_size = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_world_size() + if mp_group is None + else mp_group.nranks + ) + self.rank = ( + tp._HYBRID_PARALLEL_GROUP.get_model_parallel_rank() + if mp_group is None + else mp_group.rank + ) def forward(self, input, label): loss = mp_ops._c_softmax_with_cross_entropy( - input, label, group=self.model_parallel_group) + input, label, group=self.model_parallel_group + ) return loss diff --git a/python/paddle/distributed/fleet/layers/mpu/mp_ops.py b/python/paddle/distributed/fleet/layers/mpu/mp_ops.py index 30c2a7ea3c474..b1627d5a3b79c 100644 --- a/python/paddle/distributed/fleet/layers/mpu/mp_ops.py +++ b/python/paddle/distributed/fleet/layers/mpu/mp_ops.py @@ -47,12 +47,17 @@ def _c_identity(tensor, group=None): from paddle.autograd import PyLayer class c_identity_eager(PyLayer): - @staticmethod def forward(ctx, tensor): - return _legacy_C_ops.c_identity(tensor, 'use_calc_stream', True, - 'ring_id', group.id, - 'use_model_parallel', True) + return _legacy_C_ops.c_identity( + tensor, + 'use_calc_stream', + True, + 'ring_id', + group.id, + 'use_model_parallel', + True, + ) @staticmethod def backward(ctx, dy): @@ -63,25 +68,36 @@ def backward(ctx, dy): return c_identity_eager.apply(tensor) elif _in_legacy_dygraph(): - return _legacy_C_ops.c_identity(tensor, 'use_calc_stream', True, - 'ring_id', ring_id, - 'use_model_parallel', True) + return _legacy_C_ops.c_identity( + tensor, + 'use_calc_stream', + True, + 'ring_id', + ring_id, + 'use_model_parallel', + True, + ) op_type = 'c_identity' helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=tensor.dtype) check_variable_and_dtype( - tensor, 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - '_c_identity') - - helper.append_op(type=op_type, - inputs={'X': tensor}, - outputs={'Out': out}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - 'use_model_parallel': True, - }) + tensor, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + '_c_identity', + ) + + helper.append_op( + type=op_type, + inputs={'X': tensor}, + outputs={'Out': out}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + 'use_model_parallel': True, + }, + ) return out @@ -107,29 +123,43 @@ def _c_concat(tensor, group=None): nranks = group.nranks if _non_static_mode(): - return _legacy_C_ops.c_concat(tensor, 'ring_id', ring_id, - 'use_calc_stream', True, 'rank', rank, - 'nranks', nranks, 'use_model_parallel', - True) + return _legacy_C_ops.c_concat( + tensor, + 'ring_id', + ring_id, + 'use_calc_stream', + True, + 'rank', + rank, + 'nranks', + nranks, + 'use_model_parallel', + True, + ) op_type = 'c_concat' helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=tensor.dtype) check_variable_and_dtype( - tensor, 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - '_c_concat') - - helper.append_op(type=op_type, - inputs={'X': tensor}, - outputs={'Out': out}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - 'use_model_parallel': True, - 'nranks': nranks, - 'rank': rank - }) + tensor, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + '_c_concat', + ) + + helper.append_op( + type=op_type, + inputs={'X': tensor}, + outputs={'Out': out}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + 'use_model_parallel': True, + 'nranks': nranks, + 'rank': rank, + }, + ) return out @@ -152,42 +182,61 @@ def _c_split(tensor, group=None): global_rank = collective._get_global_env().rank rank = global_rank if group is None else group.get_group_rank(global_rank) - nranks = collective._get_global_env( - ).world_size if group is None else group.nranks + nranks = ( + collective._get_global_env().world_size + if group is None + else group.nranks + ) if _non_static_mode(): - return _legacy_C_ops.c_split(tensor, 'use_calc_stream', True, 'ring_id', - ring_id, 'rank', rank, 'nranks', nranks, - 'use_model_parallel', True) + return _legacy_C_ops.c_split( + tensor, + 'use_calc_stream', + True, + 'ring_id', + ring_id, + 'rank', + rank, + 'nranks', + nranks, + 'use_model_parallel', + True, + ) op_type = 'c_split' helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=tensor.dtype) check_variable_and_dtype( - tensor, 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - '_c_split') - - helper.append_op(type=op_type, - inputs={'X': tensor}, - outputs={'Out': out}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - 'rank': rank, - 'nranks': nranks, - 'use_model_parallel': True, - }) + tensor, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + '_c_split', + ) + + helper.append_op( + type=op_type, + inputs={'X': tensor}, + outputs={'Out': out}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + 'rank': rank, + 'nranks': nranks, + 'use_model_parallel': True, + }, + ) return out -def _mp_allreduce(tensor, - op=ReduceOp.SUM, - group=None, - use_calc_stream=True, - use_model_parallel=True): - """[it is same as allreduce above, but it supports model parallel. And it support inplace startegy] - """ +def _mp_allreduce( + tensor, + op=ReduceOp.SUM, + group=None, + use_calc_stream=True, + use_model_parallel=True, +): + """[it is same as allreduce above, but it supports model parallel. And it support inplace startegy]""" if group is not None and not group.is_member(): return @@ -198,38 +247,57 @@ def _mp_allreduce(tensor, from paddle.autograd import PyLayer class mp_allreduce_eager(PyLayer): - @staticmethod - def forward(ctx, tensor, group, use_calc_stream, - use_model_parallel): + def forward( + ctx, tensor, group, use_calc_stream, use_model_parallel + ): ctx.ring_id = group.id if use_calc_stream: op_type = collective._get_reduce_op(op, "_mp_allreduce") group.process_group.allreduce_on_calc_stream( - tensor, op_type) + tensor, op_type + ) return tensor else: return _legacy_C_ops.c_allreduce_sum_( - tensor, 'use_calc_stream', use_calc_stream, 'ring_id', - ring_id, "use_model_parallel", use_model_parallel) + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + "use_model_parallel", + use_model_parallel, + ) @staticmethod def backward(ctx, dy): - return _legacy_C_ops.c_identity(dy, 'use_calc_stream', True, - 'ring_id', ctx.ring_id, - 'use_model_parallel', True) - - return mp_allreduce_eager.apply(tensor, group, use_calc_stream, - use_model_parallel) + return _legacy_C_ops.c_identity( + dy, + 'use_calc_stream', + True, + 'ring_id', + ctx.ring_id, + 'use_model_parallel', + True, + ) + + return mp_allreduce_eager.apply( + tensor, group, use_calc_stream, use_model_parallel + ) ring_id = 0 if group is None else group.id if _in_legacy_dygraph(): if op == ReduceOp.SUM: - return _legacy_C_ops.c_allreduce_sum_(tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id, "use_model_parallel", - use_model_parallel) + return _legacy_C_ops.c_allreduce_sum_( + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + "use_model_parallel", + use_model_parallel, + ) else: raise ValueError("Unknown parameter: {}.".format(op)) @@ -238,17 +306,22 @@ def backward(ctx, dy): out = helper.create_variable_for_type_inference(dtype=tensor.dtype) check_variable_and_dtype( - tensor, 'tensor', ['float16', 'float32', 'float64', 'int32', 'int64'], - op_type) - - helper.append_op(type=op_type, - inputs={'X': tensor}, - outputs={'Out': out}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream, - 'use_model_parallel': use_model_parallel, - }) + tensor, + 'tensor', + ['float16', 'float32', 'float64', 'int32', 'int64'], + op_type, + ) + + helper.append_op( + type=op_type, + inputs={'X': tensor}, + outputs={'Out': out}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + 'use_model_parallel': use_model_parallel, + }, + ) return out @@ -267,21 +340,21 @@ def _c_lookup_table(table, index, start_index=0, name=None): Tensor. """ if _non_static_mode(): - return _legacy_C_ops.c_embedding(table, index, "start_index", - start_index) + return _legacy_C_ops.c_embedding( + table, index, "start_index", start_index + ) op_type = 'c_embedding' helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='table') check_variable_and_dtype(index, 'input', ['int32', 'int64'], op_type) tmp = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='c_embedding', - inputs={ - 'Ids': index, - 'W': table - }, - outputs={'Out': tmp}, - attrs={"start_index": start_index}) + helper.append_op( + type='c_embedding', + inputs={'Ids': index, 'W': table}, + outputs={'Out': tmp}, + attrs={"start_index": start_index}, + ) return tmp @@ -290,63 +363,75 @@ class _Linear(layers.Layer): Linear """ - def __init__(self, - in_features, - out_features, - weight_attr=None, - bias_attr=None, - name=None): + def __init__( + self, + in_features, + out_features, + weight_attr=None, + bias_attr=None, + name=None, + ): super(_Linear, self).__init__() self._dtype = self._helper.get_default_dtype() self._weight_attr = weight_attr self._bias_attr = bias_attr - self.weight = self.create_parameter(shape=[in_features, out_features], - attr=self._weight_attr, - dtype=self._dtype, - is_bias=False) - self.bias = self.create_parameter(shape=[out_features], - attr=self._bias_attr, - dtype=self._dtype, - is_bias=True) + self.weight = self.create_parameter( + shape=[in_features, out_features], + attr=self._weight_attr, + dtype=self._dtype, + is_bias=False, + ) + self.bias = self.create_parameter( + shape=[out_features], + attr=self._bias_attr, + dtype=self._dtype, + is_bias=True, + ) self.name = name def forward(self, input): - out = _linear(x=input, - weight=self.weight, - bias=self.bias, - name=self.name) + out = _linear( + x=input, weight=self.weight, bias=self.bias, name=self.name + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' return 'in_features={}, out_features={}, dtype={}{}'.format( - self.weight.shape[0], self.weight.shape[1], self._dtype, name_str) + self.weight.shape[0], self.weight.shape[1], self._dtype, name_str + ) -def _c_softmax_with_cross_entropy(logits, - label, - group=None, - return_softmax=False): +def _c_softmax_with_cross_entropy( + logits, label, group=None, return_softmax=False +): if group is not None and not group.is_member(): return ring_id = 0 if group is None else group.id global_rank = collective._get_global_env().rank rank = global_rank if group is None else group.get_group_rank(global_rank) - nranks = collective._get_global_env( - ).world_size if group is None else group.nranks + nranks = ( + collective._get_global_env().world_size + if group is None + else group.nranks + ) input_dims = len(list(logits.shape)) label_dims = len(list(label.shape)) if input_dims - 1 != label_dims and input_dims != label_dims: raise ValueError( 'Expected nput_dims - 1 = label_dims or input_dims == label_dims\ - (got nput_dims{}, label_dims{})'.format(input_dims, label_dims)) + (got nput_dims{}, label_dims{})'.format( + input_dims, label_dims + ) + ) if input_dims - 1 == label_dims: label = paddle.unsqueeze(label, axis=-1) if _non_static_mode(): softmax, loss = _legacy_C_ops.c_softmax_with_cross_entropy( - logits, label, 'ring_id', ring_id, 'rank', rank, 'nranks', nranks) + logits, label, 'ring_id', ring_id, 'rank', rank, 'nranks', nranks + ) if not return_softmax: return loss else: @@ -360,16 +445,12 @@ def _c_softmax_with_cross_entropy(logits, helper = LayerHelper('c_softmax_with_cross_entropy', **locals()) softmax = helper.create_variable_for_type_inference(dtype=logits.dtype) loss = helper.create_variable_for_type_inference(dtype=logits.dtype) - helper.append_op(type='c_softmax_with_cross_entropy', - inputs={ - 'Logits': logits, - 'Label': label - }, - outputs={ - 'Softmax': softmax, - 'Loss': loss - }, - attrs=attrs) + helper.append_op( + type='c_softmax_with_cross_entropy', + inputs={'Logits': logits, 'Label': label}, + outputs={'Softmax': softmax, 'Loss': loss}, + attrs=attrs, + ) if return_softmax: return loss, softmax @@ -383,19 +464,30 @@ def _linear(x, weight, bias=None, name=None): """ if _non_static_mode(): pre_bias = _varbase_creator(dtype=x.dtype) - _legacy_C_ops.matmul(x, weight, pre_bias, 'transpose_X', False, - 'transpose_Y', False, "alpha", 1) - return dygraph_utils._append_bias_in_dygraph(pre_bias, - bias, - axis=len(x.shape) - 1) + _legacy_C_ops.matmul( + x, + weight, + pre_bias, + 'transpose_X', + False, + 'transpose_Y', + False, + "alpha", + 1, + ) + return dygraph_utils._append_bias_in_dygraph( + pre_bias, bias, axis=len(x.shape) - 1 + ) else: helper = LayerHelper('linear', **locals()) dtype = x.dtype - assert len( - x.shape) < 4, "X latitude is not supported greater than 3 now." + assert ( + len(x.shape) < 4 + ), "X latitude is not supported greater than 3 now." - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'linear') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'linear' + ) check_dtype(dtype, 'dtype', ['float16', 'float32', 'float64'], 'linear') inputs = {'X': [x], 'Y': [weight]} @@ -405,19 +497,17 @@ def _linear(x, weight, bias=None, name=None): 'alpha': 1, } tmp = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='matmul_v2', - inputs=inputs, - outputs={'Out': tmp}, - attrs=attrs) + helper.append_op( + type='matmul_v2', inputs=inputs, outputs={'Out': tmp}, attrs=attrs + ) if bias is not None: res = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='elementwise_add', - inputs={ - 'X': [tmp], - 'Y': [bias] - }, - outputs={'Out': [res]}, - attrs={'axis': len(x.shape) - 1}) + helper.append_op( + type='elementwise_add', + inputs={'X': [tmp], 'Y': [bias]}, + outputs={'Out': [res]}, + attrs={'axis': len(x.shape) - 1}, + ) else: res = tmp return res @@ -436,18 +526,20 @@ def _set_var_distributed(var): main_block._find_var_recursive(var.name).is_distributed = True -def _parallel_linear(x, - num_rows, - num_cols, - axis, - param_attr, - bias_attr, - gather_out, - inner_rank, - nranks, - split_tensor, - name, - group=None): +def _parallel_linear( + x, + num_rows, + num_cols, + axis, + param_attr, + bias_attr, + gather_out, + inner_rank, + nranks, + split_tensor, + name, + group=None, +): """ Parallel Linear @@ -466,21 +558,25 @@ def _parallel_linear(x, else: x = _c_identity(x, group=group) - linear = paddle.nn.Linear(num_rows, - num_cols, - weight_attr=param_attr, - bias_attr=bias_attr, - name=name) + linear = paddle.nn.Linear( + num_rows, + num_cols, + weight_attr=param_attr, + bias_attr=bias_attr, + name=name, + ) # NOTE: npu linear function use matmul_v2 but linear use matmul - linear_function = _linear if core.is_compiled_with_npu()\ - else paddle.nn.functional.linear + linear_function = ( + _linear if core.is_compiled_with_npu() else paddle.nn.functional.linear + ) linear_out = linear_function( x, linear.weight, # NOTE(wangxi): row split, bias need add after allreduce None if axis == 0 else linear.bias, - linear.name) + linear.name, + ) _set_var_distributed(linear.weight) # set is_distributed for splited bias @@ -489,7 +585,8 @@ def _parallel_linear(x, if axis == 1 and linear._bias_attr != False: _set_var_distributed(linear.bias) - if not gather_out: return linear_out + if not gather_out: + return linear_out out_shape = list(linear_out.shape) out_shape[0] *= 1 if axis == 0 else nranks @@ -501,40 +598,47 @@ def _parallel_linear(x, lod_level=linear_out.lod_level, persistable=False, is_data=False, - need_check_feed=linear_out.desc.need_check_feed()) + need_check_feed=linear_out.desc.need_check_feed(), + ) if axis == 0: - main_block.append_op(type='c_allreduce_sum', - inputs={'X': linear_out}, - outputs={'Out': out}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - 'use_model_parallel': True - }) + main_block.append_op( + type='c_allreduce_sum', + inputs={'X': linear_out}, + outputs={'Out': out}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + 'use_model_parallel': True, + }, + ) if linear.bias is not None: out = out + linear.bias else: - main_block.append_op(type='c_concat', - inputs={'X': linear_out}, - outputs={'Out': out}, - attrs={ - 'rank': inner_rank, - 'ring_id': ring_id, - 'nranks': nranks, - 'use_calc_stream': True, - 'use_model_parallel': True - }) + main_block.append_op( + type='c_concat', + inputs={'X': linear_out}, + outputs={'Out': out}, + attrs={ + 'rank': inner_rank, + 'ring_id': ring_id, + 'nranks': nranks, + 'use_calc_stream': True, + 'use_model_parallel': True, + }, + ) return out -def _parallel_embedding(x, - per_part_embeddings, - origin_size, - param_attr, - inner_rank, - num_partitions, - name, - group=None): +def _parallel_embedding( + x, + per_part_embeddings, + origin_size, + param_attr, + inner_rank, + num_partitions, + name, + group=None, +): """ Parallel Embedding """ @@ -551,43 +655,43 @@ def _parallel_embedding(x, dtype = helper.get_default_dtype() size = [per_part_size, origin_size[1]] - weight = helper.create_parameter(attr=param_attr, - shape=size, - dtype=dtype, - is_bias=False) + weight = helper.create_parameter( + attr=param_attr, shape=size, dtype=dtype, is_bias=False + ) if num_partitions == 1: - return paddle.nn.functional.embedding(x, - weight=weight, - padding_idx=None, - sparse=False, - name=name) + return paddle.nn.functional.embedding( + x, weight=weight, padding_idx=None, sparse=False, name=name + ) startup_block = paddle.static.default_startup_program().global_block() main_block = paddle.static.default_main_program().global_block() startup_block.vars[weight.name].is_distributed = True main_block.vars[weight.name].is_distributed = True - output_parallel = _c_lookup_table(weight, - x, - start_index=vocab_start_index, - name=name) - out = _mp_allreduce(output_parallel, - group=group, - use_calc_stream=True, - use_model_parallel=True) + output_parallel = _c_lookup_table( + weight, x, start_index=vocab_start_index, name=name + ) + out = _mp_allreduce( + output_parallel, + group=group, + use_calc_stream=True, + use_model_parallel=True, + ) return out -def split(x, - size, - operation, - axis=0, - num_partitions=1, - gather_out=True, - weight_attr=None, - bias_attr=None, - name=None): +def split( + x, + size, + operation, + axis=0, + num_partitions=1, + gather_out=True, + weight_attr=None, + bias_attr=None, + name=None, +): """ Split the weight of the specified operation into multiple devices @@ -706,14 +810,16 @@ def split(x, num_partitions=2) """ - assert isinstance( - size, - (list, tuple)), ("The type of size for " - "paddle.distributed.split must be list or tuple.") - assert len(size) == 2, ("Number of elements in size of " - "paddle.distributed.split must be two.") - assert isinstance(operation, str), ("The type of operation for " - "paddle.distributed.split must be str.") + assert isinstance(size, (list, tuple)), ( + "The type of size for " + "paddle.distributed.split must be list or tuple." + ) + assert len(size) == 2, ( + "Number of elements in size of " "paddle.distributed.split must be two." + ) + assert isinstance(operation, str), ( + "The type of operation for " "paddle.distributed.split must be str." + ) supported_operations = [ 'linear', 'embedding', @@ -721,16 +827,22 @@ def split(x, assert operation in supported_operations, ( "The operation for " "paddle.distributed.split must be one of {}.".format( - supported_operations)) + supported_operations + ) + ) if _non_static_mode(): raise ValueError( "paddle.distributed.split cannot be used in dynamic " "graph mode, plese use ParallelEmbedding, ParallelRowLinear, " - "ParallelColumnLinear instead.") + "ParallelColumnLinear instead." + ) else: from paddle.distributed.fleet import fleet - assert fleet._role_maker, ("To use paddle.distributed.split, " - "you must call fleet.init() firstly.") + + assert fleet._role_maker, ( + "To use paddle.distributed.split, " + "you must call fleet.init() firstly." + ) rank = fleet.worker_index() nranks = fleet.worker_num() @@ -738,21 +850,28 @@ def split(x, inner_rank = rank % num_partitions if operation == "embedding": - assert axis == 0, ("We only support to split the weight of embedding " - "along the first axis now.") - assert size[0] % num_partitions == 0, \ - "The length of the vocabulary must be divisible by num_partitions " \ - "but received vocabulary={} num_partitions={}".format(size[0], num_partitions) + assert axis == 0, ( + "We only support to split the weight of embedding " + "along the first axis now." + ) + assert size[0] % num_partitions == 0, ( + "The length of the vocabulary must be divisible by num_partitions " + "but received vocabulary={} num_partitions={}".format( + size[0], num_partitions + ) + ) per_part_size = size[0] // num_partitions - emb_out = _parallel_embedding(x, - per_part_size, - size, - weight_attr, - inner_rank, - num_partitions, - name, - group=None) + emb_out = _parallel_embedding( + x, + per_part_size, + size, + weight_attr, + inner_rank, + num_partitions, + name, + group=None, + ) return emb_out else: should_split = False @@ -760,32 +879,41 @@ def split(x, assert size[0] % num_partitions == 0, ( "Number of rows of the weight for linear ({}) must be" " divisible by num_partitions ({})".format( - size[0], num_partitions)) + size[0], num_partitions + ) + ) per_part_size = size[0] // num_partitions linear_size = (per_part_size, size[1]) - if x.shape[-1] == size[0]: should_split = True + if x.shape[-1] == size[0]: + should_split = True elif axis == 1: assert size[1] % num_partitions == 0, ( "Number of column of the weight for linear ({}) must be" " divisible by num_partitions ({})".format( - size[1], num_partitions)) + size[1], num_partitions + ) + ) per_part_size = size[1] // num_partitions linear_size = (size[0], per_part_size) else: - raise ValueError("The value of axis must be 0 or 1, but the value " - "given is {}.".format(axis)) - - linear_out = _parallel_linear(x, - linear_size[0], - linear_size[1], - axis, - weight_attr, - bias_attr, - gather_out, - inner_rank, - num_partitions, - should_split, - name=name, - group=None) + raise ValueError( + "The value of axis must be 0 or 1, but the value " + "given is {}.".format(axis) + ) + + linear_out = _parallel_linear( + x, + linear_size[0], + linear_size[1], + axis, + weight_attr, + bias_attr, + gather_out, + inner_rank, + num_partitions, + should_split, + name=name, + group=None, + ) return linear_out diff --git a/python/paddle/distributed/fleet/layers/mpu/random.py b/python/paddle/distributed/fleet/layers/mpu/random.py index ff082695cb7ee..17442c1938a1d 100644 --- a/python/paddle/distributed/fleet/layers/mpu/random.py +++ b/python/paddle/distributed/fleet/layers/mpu/random.py @@ -85,6 +85,7 @@ def get_rng_state_tracker(): def model_parallel_random_seed(seed=None): import paddle.distributed.fleet as fleet + hcg = fleet.get_hybrid_communicate_group() rank = hcg.get_model_parallel_rank() @@ -105,23 +106,23 @@ def determinate_seed(rng_name): helper = LayerHelper('seed', **locals()) out = helper.create_variable_for_type_inference(dtype=paddle.int32) # set force_cpu to reduce sync copy from CPU->GPU->CPU, and reduce pipeline hang - helper.append_op(type='seed', - outputs={'Out': out}, - attrs={ - 'deterministic': True, - 'rng_name': rng_name, - 'force_cpu': True - }) + helper.append_op( + type='seed', + outputs={'Out': out}, + attrs={'deterministic': True, 'rng_name': rng_name, 'force_cpu': True}, + ) return out -def dropout(x, - p=0.5, - axis=None, - rng_name=None, - training=True, - mode="upscale_in_train", - name=None): +def dropout( + x, + p=0.5, + axis=None, + rng_name=None, + training=True, + mode="upscale_in_train", + name=None, +): """ Dropout is a regularization technique for reducing overfitting by preventing neuron co-adaption during training. The dropout operator randomly sets the @@ -191,53 +192,66 @@ def dropout(x, raise TypeError("p argument should be a number(int|float) or Variable") # fast return for p == 0 - if isinstance(p, (int, float)) and p == 0: return x + if isinstance(p, (int, float)) and p == 0: + return x assert 0 <= p <= 1, ValueError("p argument should between 0 and 1") - assert mode in ('downscale_in_infer', 'upscale_in_train'), \ - ValueError( - "mode argument should be 'downscale_in_infer' or 'upscale_in_train'") + assert mode in ('downscale_in_infer', 'upscale_in_train'), ValueError( + "mode argument should be 'downscale_in_infer' or 'upscale_in_train'" + ) - assert axis is None, \ - TypeError("unsupport axis when using random seed generator") + assert axis is None, TypeError( + "unsupport axis when using random seed generator" + ) - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer # dygraph using tracker, doesn't need determinate seed if _non_static_mode(): - out, mask = _legacy_C_ops.dropout(x, 'dropout_prob', p, 'is_test', - not training, 'fix_seed', False, - 'seed', 0, 'dropout_implementation', - mode) + out, mask = _legacy_C_ops.dropout( + x, + 'dropout_prob', + p, + 'is_test', + not training, + 'fix_seed', + False, + 'seed', + 0, + 'dropout_implementation', + mode, + ) return out seed = determinate_seed(rng_name) if isinstance(p, Variable) and not p.shape != [1]: raise TypeError( - "Required p.shape == [1] if type(p) is Variable, but received p.shape = {}" - .format(p.shape)) + "Required p.shape == [1] if type(p) is Variable, but received p.shape = {}".format( + p.shape + ) + ) helper = LayerHelper('dropout', **locals()) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'dropout') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'dropout' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) mask = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) - - helper.append_op(type='dropout', - inputs={ - 'X': [x], - 'Seed': seed - }, - outputs={ - 'Out': [out], - 'Mask': [mask] - }, - attrs={ - 'dropout_prob': p, - 'is_test': not training, - 'dropout_implementation': mode, - }) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) + + helper.append_op( + type='dropout', + inputs={'X': [x], 'Seed': seed}, + outputs={'Out': [out], 'Mask': [mask]}, + attrs={ + 'dropout_prob': p, + 'is_test': not training, + 'dropout_implementation': mode, + }, + ) return out diff --git a/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py index 78a53ccdba55e..f9e8b6b8e21d5 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py @@ -18,7 +18,6 @@ class AMPOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(AMPOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -32,11 +31,12 @@ def __init__(self, optimizer): ] self.meta_optimizers_black_list = ["DGCOptimizer"] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(AMPOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(AMPOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) def _init_wrapped_opt(self): if self.wrapped_opt is not None: @@ -48,14 +48,21 @@ def _init_wrapped_opt(self): custom_black_list = set(config['custom_black_list']) custom_black_varnames = set(config['custom_black_varnames']) amp_lists = mixed_precision.AutoMixedPrecisionLists( - custom_white_list, custom_black_list, custom_black_varnames) + custom_white_list, custom_black_list, custom_black_varnames + ) self.wrapped_opt = mixed_precision.decorate( - self.inner_opt, amp_lists, config['init_loss_scaling'], - config['incr_every_n_steps'], config['decr_every_n_nan_or_inf'], - config['incr_ratio'], config['decr_ratio'], - config['use_dynamic_loss_scaling'], config['use_pure_fp16'], - config['use_fp16_guard']) + self.inner_opt, + amp_lists, + config['init_loss_scaling'], + config['incr_every_n_steps'], + config['decr_every_n_nan_or_inf'], + config['incr_ratio'], + config['decr_ratio'], + config['use_dynamic_loss_scaling'], + config['use_pure_fp16'], + config['use_fp16_guard'], + ) # if worker_num > 1, all cards will communication with each other, # add is_distributed to optimize amp, overlap communication and @@ -87,46 +94,46 @@ def _enable_strategy(self, dist_strategy, context): "decr_every_n_nan_or_inf": 2, "incr_ratio": 2.0, "decr_ratio": 0.8, - "use_dynamic_loss_scaling": True + "use_dynamic_loss_scaling": True, } - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): # maybe inner_opt of other meta optimizer self._init_wrapped_opt() - return self.wrapped_opt.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + return self.wrapped_opt.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) def apply_gradients(self, params_grads): return self.wrapped_opt.apply_gradients(params_grads=params_grads) def apply_optimize(self, loss, startup_program, params_grads): - return self.wrapped_opt.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) - - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + return self.wrapped_opt.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) + + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): self._init_wrapped_opt() - optimize_ops, params_grads = \ - self.wrapped_opt.minimize(loss, startup_program, - parameter_list, no_grad_set) + optimize_ops, params_grads = self.wrapped_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads - def amp_init(self, - place, - scope=None, - test_program=None, - use_fp16_test=False): - return self.wrapped_opt.amp_init(place, scope, test_program, - use_fp16_test) + def amp_init( + self, place, scope=None, test_program=None, use_fp16_test=False + ): + return self.wrapped_opt.amp_init( + place, scope, test_program, use_fp16_test + ) def get_loss_scaling(self): return self.wrapped_opt.get_loss_scaling() diff --git a/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_optimizer.py index c78c855a28552..ee3f96e243fe2 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_optimizer.py @@ -25,7 +25,6 @@ class AscendIRParser(object): - def __init__(self, auto_dp=False, world_rank_size=1): self.graph_idx = 0 self.hcom_endpoints = {} @@ -39,24 +38,33 @@ def _construct_input_map(self, input_varlist): for id, var in enumerate(input_varlist): if var.is_data: # input data ge_input = core.GEOperatorFactory.create_operator( - var.name, "Data").set_attr_int32("index", id) + var.name, "Data" + ).set_attr_int32("index", id) ret_map[var.name] = ge_input ge_in_operator.append(ge_input) else: # param, learning ... ge_input = core.GEOperatorFactory.create_operator( - var.name, "Variable") + var.name, "Variable" + ) ge_input.update_output_desc( "y", - core.GETensorDesc(core.GEShape(var.shape), - core.GEFormat.FORMAT_ND, - core.GEDataType.DT_FLOAT)) + core.GETensorDesc( + core.GEShape(var.shape), + core.GEFormat.FORMAT_ND, + core.GEDataType.DT_FLOAT, + ), + ) ret_map[var.name] = ge_input return ge_in_operator, ret_map def _endpoint_to_world_rank_id(self, endpoint): world_endpoints = fleet.worker_endpoints() - assert endpoint in world_endpoints, "endpoint (%s) not in worker_endpoints (%s) " % ( - endpoint, fleet.world_device_ids()) + assert ( + endpoint in world_endpoints + ), "endpoint (%s) not in worker_endpoints (%s) " % ( + endpoint, + fleet.world_device_ids(), + ) return world_endpoints.index(endpoint) def parse_op(self, op): @@ -72,13 +80,16 @@ def parse_op(self, op): self.hcom_endpoints[nccl_id] = other_endpoints[:] self.hcom_endpoints[nccl_id].insert(rank, endpoint) - print("nccl_id (%s) registered endpoints %s" % - (nccl_id, self.hcom_endpoints[nccl_id])) + print( + "nccl_id (%s) registered endpoints %s" + % (nccl_id, self.hcom_endpoints[nccl_id]) + ) elif op.type == 'c_comm_init': nccl_id = op.input_arg_names[0] nranks = op.attr("nranks") - assert nranks == len(self.hcom_endpoints[nccl_id] - ), "nranks doesn't match endpoint count" + assert nranks == len( + self.hcom_endpoints[nccl_id] + ), "nranks doesn't match endpoint count" rank = op.attr("rank") ring_id = op.attr("ring_id") @@ -88,24 +99,29 @@ def parse_op(self, op): for endpoint in self.hcom_endpoints[nccl_id] ] self.groups_to_create.append( - HcomGroupConfig(name=group_name, - nranks=nranks, - rank_ids=global_rank_ids)) - print("append to create group: %s, with rank_ids: %s" % - (group_name, global_rank_ids)) + HcomGroupConfig( + name=group_name, nranks=nranks, rank_ids=global_rank_ids + ) + ) + print( + "append to create group: %s, with rank_ids: %s" + % (group_name, global_rank_ids) + ) elif op.type in ascend_parser.registerd_op: op_parser = self.parser_factory.create_parse( - ascend_parser.registerd_op[op.type]) + ascend_parser.registerd_op[op.type] + ) op_parser.apply(op) else: - assert False, "Op[%s] has not been registered, so we have to skip it" % ( - op.type) - - def _parse_program(self, - graph_name, - program, - input_varlist=[], - fetch_list=[]): + assert ( + False + ), "Op[%s] has not been registered, so we have to skip it" % ( + op.type + ) + + def _parse_program( + self, graph_name, program, input_varlist=[], fetch_list=[] + ): begin_graph_idx = self.graph_idx ge_in_operator = [] ge_out_operator = [] @@ -121,7 +137,8 @@ def _parse_program(self, ge_in_operator, self.var2geop = self._construct_input_map(input_varlist) self.parser_factory = ascend_parser.AscendParserFactory( - graph, self.var2geop) + graph, self.var2geop + ) for i, curop in list(enumerate(block.ops)): self.parse_op(curop) @@ -150,28 +167,36 @@ def _parse_program(self, input_varlist = [var for var in input_varlist if var.is_data] - block.append_op(type="ascend_trigger", - inputs={"FeedList": input_varlist}, - outputs={"FetchList": fetch_list}, - attrs={'graph_idx': self.graph_idx}) + block.append_op( + type="ascend_trigger", + inputs={"FeedList": input_varlist}, + outputs={"FetchList": fetch_list}, + attrs={'graph_idx': self.graph_idx}, + ) self.graph_idx += 1 return graph - def parse_program(self, startup_program, main_program, input_varlist, - fetch_list): + def parse_program( + self, startup_program, main_program, input_varlist, fetch_list + ): startup_graph = self._parse_program("startup", startup_program) - main_graph = self._parse_program("main", main_program, input_varlist, - fetch_list) + main_graph = self._parse_program( + "main", main_program, input_varlist, fetch_list + ) if self._auto_dp and self._world_rank_size > 1: - assert len(self.groups_to_create - ) == 0, "can't parse program under auto_dp mode" + assert ( + len(self.groups_to_create) == 0 + ), "can't parse program under auto_dp mode" from paddle.distributed import fleet + self.groups_to_create.append( - HcomGroupConfig(name="hcom_group_0", - nranks=fleet.world_size(), - rank_ids=[x - for x in range(fleet.world_size())])) + HcomGroupConfig( + name="hcom_group_0", + nranks=fleet.world_size(), + rank_ids=[x for x in range(fleet.world_size())], + ) + ) return startup_graph, main_graph @@ -179,7 +204,6 @@ def parse_program(self, startup_program, main_program, input_varlist, # AscendOptimizer is a wrapper for basic optimizer now # We will make it part of fleet meta_optimizer in the future class AscendOptimizer(Optimizer): - def __init__(self, optimizer, fetch_list=[]): self.inner_opt = optimizer self.fetch_list = fetch_list @@ -209,28 +233,34 @@ def _get_input_varlist(self, program): ret_list.append(var) return ret_list - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - auto_dp=False, - rank_table_file=None, - precision_mode="must_keep_origin_dtype"): + def minimize( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + auto_dp=False, + rank_table_file=None, + precision_mode="must_keep_origin_dtype", + ): minimized = None if self.inner_opt: - minimized = self.inner_opt.minimize(loss, - startup_program=startup_program) + minimized = self.inner_opt.minimize( + loss, startup_program=startup_program + ) self.ascend_instance = core.AscendInstance() from paddle.distributed import fleet + if auto_dp and fleet.world_size() > 1: from paddle.fluid.transpiler import ascend_transpiler - t = ascend_transpiler.AscendTranspiler(startup_program, - loss.block.program) + + t = ascend_transpiler.AscendTranspiler( + startup_program, loss.block.program + ) t.transpile() - #print(loss.block.program) + # print(loss.block.program) # Config about Graph Engine can be found in https://support.huaweicloud.com/ config = { @@ -251,17 +281,21 @@ def minimize(self, self.ascend_instance.init_global_resources() main_block = loss.block - self.parser = AscendIRParser(auto_dp=auto_dp, - world_rank_size=fleet.world_size()) + self.parser = AscendIRParser( + auto_dp=auto_dp, world_rank_size=fleet.world_size() + ) input_varlist = self._get_input_varlist(main_block.program) startup_graph, main_graph = self.parser.parse_program( - startup_program, main_block.program, input_varlist, self.fetch_list) + startup_program, main_block.program, input_varlist, self.fetch_list + ) for cfg in self.parser.groups_to_create: - print("create group (%s), nranks: %d, rank_ids: %s" % - (cfg.name, cfg.nranks, cfg.rank_ids)) + print( + "create group (%s), nranks: %d, rank_ids: %s" + % (cfg.name, cfg.nranks, cfg.rank_ids) + ) hccl.create_group(cfg.name, cfg.nranks, cfg.rank_ids) self.ascend_instance.add_ascend_subgraph(0, startup_graph) diff --git a/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_parser.py b/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_parser.py index e09d8ec8bf77c..52447952e55a3 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_parser.py +++ b/python/paddle/distributed/fleet/meta_optimizers/ascend/ascend_parser.py @@ -56,7 +56,7 @@ "slice": "SliceParser", "top_k": "TopkParser", "accuracy": "AccuracyParser", - #"increment": "IncrementParser", + # "increment": "IncrementParser", "lookup_table": "LookupTableParser", "truncated_gaussian_random": "TruncatedNormalParser", "c_allgather": "AllGatherParser", @@ -71,7 +71,6 @@ "equal": "EqualParser", "expand": "ExpandParser", "squeeze2": "SqueezeParser", - ## backwords "matmul_grad": "MatMulGradParser", "mul_grad": "MulGradParser", @@ -94,17 +93,15 @@ "gather_grad": "GatherGradParser", "transpose2_grad": "TransposeGradParser", "layer_norm_grad": "LayerNormGradParser", - ## opt "sgd": "SGDParser", - #"adam": "AdamParser", + # "adam": "AdamParser", } global_cnt = -1 global_input_cnt = -1 class AscendHelper(object): - def __init__(self): self.dtype2ge_map = { 0: core.GEDataType.DT_BOOL, @@ -113,7 +110,7 @@ def __init__(self): 3: core.GEDataType.DT_INT64, 4: core.GEDataType.DT_FLOAT16, 5: core.GEDataType.DT_FLOAT, - 6: core.GEDataType.DT_DOUBLE + 6: core.GEDataType.DT_DOUBLE, } self.dtype2np_map = { 0: "bool", @@ -122,23 +119,24 @@ def __init__(self): 3: "int64", 4: "float16", 5: "float32", - 6: "float64" + 6: "float64", } self.dtype2paddle_inv_map = {"VarType.FP32": 0, "VarType.FP16": 1} def dtype2ge(self, dtype): assert dtype in self.dtype2ge_map, "dtype[%d] is not supported %d" % ( - dtype) + dtype + ) return self.dtype2ge_map[dtype] def dtype2np(self, index): assert index in self.dtype2np_map, "index[%d] is not supported %d" % ( - index) + index + ) return self.dtype2np_map[index] class AscendParserFactory(object): - def __init__(self, graph, var2geop): self.graph = graph self.var2geop = var2geop @@ -152,7 +150,6 @@ def create_parse(self, parser_class): class AscendParserBase(object): - def __init__(self, graph, var2geop): self.graph = graph self.var2geop = var2geop @@ -161,35 +158,42 @@ def __init__(self, graph, var2geop): def _get_ge_input(self, input_var_name): assert input_var_name in self.var2geop, "var %s not created before" % ( - input_var_name) + input_var_name + ) return self.var2geop[input_var_name] def update_output(self, geop_list, index_list): output_num = len(self.op.output_names) - assert output_num == len( - index_list - ), "Parser[%s]'s output number[%d] is not equal to parameters number[%d]" % ( - self.parser_name, len(index_list), output_num) + assert output_num == len(index_list), ( + "Parser[%s]'s output number[%d] is not equal to parameters number[%d]" + % (self.parser_name, len(index_list), output_num) + ) for output_id in range(output_num): arguments = self.op.output(self.op.output_names[output_id]) if len(arguments) > 0: - assert len(arguments) == len( - index_list[output_id] - ), "Parser[%s]'s %dth argument number[%d] is not equal to paddle's number[%d]" % ( - self.parser_name, output_id, len( - index_list[output_id]), len(arguments)) + assert len(arguments) == len(index_list[output_id]), ( + "Parser[%s]'s %dth argument number[%d] is not equal to paddle's number[%d]" + % ( + self.parser_name, + output_id, + len(index_list[output_id]), + len(arguments), + ) + ) for i in range(len(arguments)): self.var2geop[arguments[i]] = geop_list[ - index_list[output_id][i]] + index_list[output_id][i] + ] for geop in geop_list: self.graph.add_op(geop) def apply(self, op): self.op = op - assert self.op.type == self.parser_name, "op [%s] != parser_name[%s]" % ( - self.op.type, self.parser_name) - #print("begin to parse op %s" % (self.parser_name)) + assert ( + self.op.type == self.parser_name + ), "op [%s] != parser_name[%s]" % (self.op.type, self.parser_name) + # print("begin to parse op %s" % (self.parser_name)) geop_list, index_list = self._apply() self.update_output(geop_list, index_list) @@ -205,33 +209,43 @@ def _accumulated_op_id(self): return name def _create_ge_tensor(self, shape, dtype, value): - tensor_desc = core.GETensorDesc(core.GEShape(shape), - core.GEFormat.FORMAT_ND, - self.ascend_helper.dtype2ge(dtype)) + tensor_desc = core.GETensorDesc( + core.GEShape(shape), + core.GEFormat.FORMAT_ND, + self.ascend_helper.dtype2ge(dtype), + ) tensor = core.GETensor(tensor_desc) - data = (value * np.ones( - (shape))).reshape(shape).astype(self.ascend_helper.dtype2np(dtype)) + data = ( + (value * np.ones((shape))) + .reshape(shape) + .astype(self.ascend_helper.dtype2np(dtype)) + ) buf = data.tobytes() data_8 = np.frombuffer(buf, dtype=np.uint8) tensor.set_data(data_8) return tensor def _get_ge_tensor(self, shape, dtype, value_list): - tensor_desc = core.GETensorDesc(core.GEShape(shape), - core.GEFormat.FORMAT_ND, - self.ascend_helper.dtype2ge(dtype)) + tensor_desc = core.GETensorDesc( + core.GEShape(shape), + core.GEFormat.FORMAT_ND, + self.ascend_helper.dtype2ge(dtype), + ) tensor = core.GETensor(tensor_desc) - data = np.array(value_list).reshape(shape).astype( - self.ascend_helper.dtype2np(dtype)) + data = ( + np.array(value_list) + .reshape(shape) + .astype(self.ascend_helper.dtype2np(dtype)) + ) buf = data.tobytes() data_8 = np.frombuffer(buf, dtype=np.uint8) tensor.set_data(data_8) tensor_const = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) return tensor_const @@ -242,21 +256,28 @@ def _get_variable(self, shape, dtype, tensor): type = core.GEDataType.DT_FLOAT var = core.GEOperatorFactory.create_operator( - "variable" + self._accumulated_op_id(), "Variable") + "variable" + self._accumulated_op_id(), "Variable" + ) var.update_output_desc( "y", - core.GETensorDesc(core.GEShape(shape), core.GEFormat.FORMAT_ND, - type)) - assign = core.GEOperatorFactory.create_operator( - "assign" + self._accumulated_op_id(), - "Assign").set_input("value", tensor).set_input("ref", var) + core.GETensorDesc( + core.GEShape(shape), core.GEFormat.FORMAT_ND, type + ), + ) + assign = ( + core.GEOperatorFactory.create_operator( + "assign" + self._accumulated_op_id(), "Assign" + ) + .set_input("value", tensor) + .set_input("ref", var) + ) return assign def _create_shape_tensor(self): - tensor_desc = core.GETensorDesc(core.GEShape([2]), - core.GEFormat.FORMAT_ND, - core.GEDataType.DT_INT32) + tensor_desc = core.GETensorDesc( + core.GEShape([2]), core.GEFormat.FORMAT_ND, core.GEDataType.DT_INT32 + ) tensor = core.GETensor(tensor_desc) data = np.ones((2)).astype("int32").reshape([2]) @@ -268,16 +289,19 @@ def _create_shape_tensor(self): def _get_GEtensor_shape(self, tensor): tensor_shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), - "Shape").set_input("x", tensor) - tensor_shape = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", tensor_shape).set_attr_int32("dst_type", 0) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", tensor) + tensor_shape = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", tensor_shape) + .set_attr_int32("dst_type", 0) + ) return tensor_shape class AddParser(AscendParserBase): - def __init__(self, graph, var2geop): super(AddParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_add" @@ -285,14 +309,17 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - add = core.GEOperatorFactory.create_operator( - "add" + self._accumulated_op_id(), - "Add").set_input("x1", x).set_input("x2", y) + add = ( + core.GEOperatorFactory.create_operator( + "add" + self._accumulated_op_id(), "Add" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [add], [[0]] class DotSubParser(AscendParserBase): - def __init__(self, graph, var2geop): super(DotSubParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_sub" @@ -300,14 +327,17 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - sub = core.GEOperatorFactory.create_operator( - "sub" + self._accumulated_op_id(), - "Sub").set_input("x1", x).set_input("x2", y) + sub = ( + core.GEOperatorFactory.create_operator( + "sub" + self._accumulated_op_id(), "Sub" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [sub], [[0]] class DotMulParser(AscendParserBase): - def __init__(self, graph, var2geop): super(DotMulParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_mul" @@ -315,14 +345,17 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - mul = core.GEOperatorFactory.create_operator( - "dotmul" + self._accumulated_op_id(), - "Mul").set_input("x1", x).set_input("x2", y) + mul = ( + core.GEOperatorFactory.create_operator( + "dotmul" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [mul], [[0]] class DotDivParser(AscendParserBase): - def __init__(self, graph, var2geop): super(DotDivParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_div" @@ -330,14 +363,17 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - div = core.GEOperatorFactory.create_operator( - "dotdiv" + self._accumulated_op_id(), - "Div").set_input("x1", x).set_input("x2", y) + div = ( + core.GEOperatorFactory.create_operator( + "dotdiv" + self._accumulated_op_id(), "Div" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [div], [[0]] class DotPowParser(AscendParserBase): - def __init__(self, graph, var2geop): super(DotPowParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_pow" @@ -345,14 +381,17 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - pow = core.GEOperatorFactory.create_operator( - "dotpow" + self._accumulated_op_id(), - "Pow").set_input("x1", x).set_input("x2", y) + pow = ( + core.GEOperatorFactory.create_operator( + "dotpow" + self._accumulated_op_id(), "Pow" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [pow], [[0]] class LessParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LessParser, self).__init__(graph, var2geop) self.parser_name = "less_than" @@ -360,14 +399,17 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - less_than = core.GEOperatorFactory.create_operator( - "less_than" + self._accumulated_op_id(), - "Less").set_input("x1", x).set_input("x2", y) + less_than = ( + core.GEOperatorFactory.create_operator( + "less_than" + self._accumulated_op_id(), "Less" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [less_than], [[0]] class MaxParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MaxParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_max" @@ -375,14 +417,17 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - max_out = core.GEOperatorFactory.create_operator( - "max" + self._accumulated_op_id(), - "Maximum").set_input("x1", x).set_input("x2", y) + max_out = ( + core.GEOperatorFactory.create_operator( + "max" + self._accumulated_op_id(), "Maximum" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [max_out], [[0]] class MinParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MinParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_min" @@ -390,15 +435,18 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - min_out = core.GEOperatorFactory.create_operator( - "min" + self._accumulated_op_id(), - "Minimum").set_input("x1", x).set_input("x2", y) + min_out = ( + core.GEOperatorFactory.create_operator( + "min" + self._accumulated_op_id(), "Minimum" + ) + .set_input("x1", x) + .set_input("x2", y) + ) return [min_out], [[0]] ## cal class LogParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LogParser, self).__init__(graph, var2geop) self.parser_name = "log" @@ -406,12 +454,12 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) log = core.GEOperatorFactory.create_operator( - "log" + self._accumulated_op_id(), "Log").set_input("x", x) + "log" + self._accumulated_op_id(), "Log" + ).set_input("x", x) return [log], [[0]] class SqrtParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SqrtParser, self).__init__(graph, var2geop) self.parser_name = "sqrt" @@ -419,12 +467,12 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) sqrt = core.GEOperatorFactory.create_operator( - "sqrt" + self._accumulated_op_id(), "Sqrt").set_input("x", x) + "sqrt" + self._accumulated_op_id(), "Sqrt" + ).set_input("x", x) return [sqrt], [[0]] class PowParser(AscendParserBase): - def __init__(self, graph, var2geop): super(PowParser, self).__init__(graph, var2geop) self.parser_name = "pow" @@ -432,17 +480,19 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) factor = self.op.attr("factor") - pow_value = core.GEOperatorFactory.create_operator( - "pow" + self._accumulated_op_id(), - "Power").set_input("x", x).set_attr_float( - "power", - factor).set_attr_float("scale", - 1.0).set_attr_float("shift", 0.0) + pow_value = ( + core.GEOperatorFactory.create_operator( + "pow" + self._accumulated_op_id(), "Power" + ) + .set_input("x", x) + .set_attr_float("power", factor) + .set_attr_float("scale", 1.0) + .set_attr_float("shift", 0.0) + ) return [pow_value], [[0]] class SquareParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SquareParser, self).__init__(graph, var2geop) self.parser_name = "square" @@ -450,12 +500,12 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) square = core.GEOperatorFactory.create_operator( - "square" + self._accumulated_op_id(), "Square").set_input("x", x) + "square" + self._accumulated_op_id(), "Square" + ).set_input("x", x) return [square], [[0]] class SumParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SumParser, self).__init__(graph, var2geop) self.parser_name = "sum" @@ -466,19 +516,26 @@ def _apply(self): assert False, "the size of input list must large or equal 2" x = self._get_ge_input(self.op.input_arg_names[0]) y = self._get_ge_input(self.op.input_arg_names[1]) - sum = core.GEOperatorFactory.create_operator( - "sum" + self._accumulated_op_id(), - "Add").set_input("x1", x).set_input("x2", y) + sum = ( + core.GEOperatorFactory.create_operator( + "sum" + self._accumulated_op_id(), "Add" + ) + .set_input("x1", x) + .set_input("x2", y) + ) for i in range(2, len_list): y = self._get_ge_input(self.op.input_arg_names[i]) - sum = core.GEOperatorFactory.create_operator( - "sum" + self._accumulated_op_id(), - "Add").set_input("x1", sum).set_input("x2", y) + sum = ( + core.GEOperatorFactory.create_operator( + "sum" + self._accumulated_op_id(), "Add" + ) + .set_input("x1", sum) + .set_input("x2", y) + ) return [sum], [[0]] class LogicalNotParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LogicalNotParser, self).__init__(graph, var2geop) self.parser_name = "logical_not" @@ -486,28 +543,30 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) logical_not = core.GEOperatorFactory.create_operator( - "logical_not" + self._accumulated_op_id(), - "LogicalNot").set_input("x", x) + "logical_not" + self._accumulated_op_id(), "LogicalNot" + ).set_input("x", x) return [logical_not], [[0]] class MeanParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MeanParser, self).__init__(graph, var2geop) self.parser_name = "mean" def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) - mean = core.GEOperatorFactory.create_operator( - "mean" + self._accumulated_op_id(), "ReduceMeanD").set_input( - "x", x).set_attr_bool("keep_dims", - False).set_attr_vec_int32("axes", []) + mean = ( + core.GEOperatorFactory.create_operator( + "mean" + self._accumulated_op_id(), "ReduceMeanD" + ) + .set_input("x", x) + .set_attr_bool("keep_dims", False) + .set_attr_vec_int32("axes", []) + ) return [mean], [[0]] class ReduceSumParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReduceSumParser, self).__init__(graph, var2geop) self.parser_name = "reduce_sum" @@ -520,14 +579,18 @@ def _apply(self): x_shape = self.op.block.var(self.op.input_arg_names[0]).shape if reduce_all: axes = list(range(len(x_shape))) - reduce_sum = core.GEOperatorFactory.create_operator( - "reduce_sum" + self._accumulated_op_id(), - "ReduceSumD").set_input("x", x, 0).set_attr_vec_int32( - "axes", axes).set_attr_bool("keep_dims", keep_dims) + reduce_sum = ( + core.GEOperatorFactory.create_operator( + "reduce_sum" + self._accumulated_op_id(), "ReduceSumD" + ) + .set_input("x", x, 0) + .set_attr_vec_int32("axes", axes) + .set_attr_bool("keep_dims", keep_dims) + ) return [reduce_sum], [[0]] -#class IncrementParser(AscendParserBase): +# class IncrementParser(AscendParserBase): # def __init__(self, graph, var2geop): # super(IncrementParser, self).__init__(graph, var2geop) # self.parser_name = "increment" @@ -544,7 +607,6 @@ def _apply(self): ## matrix cal class MatMulParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MatMulParser, self).__init__(graph, var2geop) self.parser_name = "matmul" @@ -559,24 +621,31 @@ def _apply(self): x2_shape = self.op.block.var(self.op.input_arg_names[1]).shape if len(x1_shape) > 2: - matmul = core.GEOperatorFactory.create_operator( - "matmul" + self._accumulated_op_id(), "BatchMatMul").set_input( - "x1", x).set_input("x2", y).set_attr_bool( - "adj_x1", - transpose_x).set_attr_bool("adj_x2", transpose_y) + matmul = ( + core.GEOperatorFactory.create_operator( + "matmul" + self._accumulated_op_id(), "BatchMatMul" + ) + .set_input("x1", x) + .set_input("x2", y) + .set_attr_bool("adj_x1", transpose_x) + .set_attr_bool("adj_x2", transpose_y) + ) elif len(x1_shape) == 2: - matmul = core.GEOperatorFactory.create_operator( - "matmul" + self._accumulated_op_id(), - "MatMul").set_input("x1", x).set_input("x2", y).set_attr_bool( - "transpose_x1", - transpose_x).set_attr_bool("transpose_x2", transpose_y) + matmul = ( + core.GEOperatorFactory.create_operator( + "matmul" + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", x) + .set_input("x2", y) + .set_attr_bool("transpose_x1", transpose_x) + .set_attr_bool("transpose_x2", transpose_y) + ) else: assert False, "not support" return [matmul], [[0]] class MulParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MulParser, self).__init__(graph, var2geop) self.parser_name = "mul" @@ -591,48 +660,72 @@ def _apply(self): if x_num_col_dims == 1 and y_num_col_dims == 1: if len(shape_x1) == 2 and len(shape_x2) == 2: - matmul = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "MatMul").set_input("x1", x).set_input("x2", y) + matmul = ( + core.GEOperatorFactory.create_operator( + "mul" + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", x) + .set_input("x2", y) + ) elif len(shape_x1) == 3 and len(shape_x2) == 2: flatten_x1 = core.GEOperatorFactory.create_operator( - "flatten" + self._accumulated_op_id(), - "Flatten").set_input("x", x) - matmul = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "MatMul").set_input("x1", flatten_x1, - 0).set_input("x2", y, 0) + "flatten" + self._accumulated_op_id(), "Flatten" + ).set_input("x", x) + matmul = ( + core.GEOperatorFactory.create_operator( + "mul" + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", flatten_x1, 0) + .set_input("x2", y, 0) + ) else: assert False, "not support" else: if len(shape_x1) == 3 and len(shape_x2) == 2: assert x_num_col_dims == 2, "only support 2" - flatten_x1 = core.GEOperatorFactory.create_operator( - "flatten" + self._accumulated_op_id(), - "FlattenV2").set_input("x", x).set_attr_int32( - "axis", 0).set_attr_int32("end_axis", 1) - matmul_m = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "MatMul").set_input("x1", flatten_x1, - 0).set_input("x2", y, 0) - matmul_transpose = core.GEOperatorFactory.create_operator( - "transpose" + self._accumulated_op_id(), - "TransposeD").set_input("x", matmul_m).set_attr_vec_int32( - "perm", [1, 0]) + flatten_x1 = ( + core.GEOperatorFactory.create_operator( + "flatten" + self._accumulated_op_id(), "FlattenV2" + ) + .set_input("x", x) + .set_attr_int32("axis", 0) + .set_attr_int32("end_axis", 1) + ) + matmul_m = ( + core.GEOperatorFactory.create_operator( + "mul" + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", flatten_x1, 0) + .set_input("x2", y, 0) + ) + matmul_transpose = ( + core.GEOperatorFactory.create_operator( + "transpose" + self._accumulated_op_id(), "TransposeD" + ) + .set_input("x", matmul_m) + .set_attr_vec_int32("perm", [1, 0]) + ) tensor = self._create_ge_tensor( - [3], 2, [shape_x2[1], shape_x1[0], shape_x1[1]]) + [3], 2, [shape_x2[1], shape_x1[0], shape_x1[1]] + ) const_shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor) - reshape_matmul = core.GEOperatorFactory.create_operator( - "reshape" + self._accumulated_op_id(), - "Reshape").set_input("x", matmul_transpose).set_input( - "shape", const_shape).set_attr_int32("axis", 0) - matmul = core.GEOperatorFactory.create_operator( - "transpose" + self._accumulated_op_id(), - "TransposeD").set_input("x", - reshape_matmul).set_attr_vec_int32( - "perm", [1, 2, 0]) + "shape" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) + reshape_matmul = ( + core.GEOperatorFactory.create_operator( + "reshape" + self._accumulated_op_id(), "Reshape" + ) + .set_input("x", matmul_transpose) + .set_input("shape", const_shape) + .set_attr_int32("axis", 0) + ) + matmul = ( + core.GEOperatorFactory.create_operator( + "transpose" + self._accumulated_op_id(), "TransposeD" + ) + .set_input("x", reshape_matmul) + .set_attr_vec_int32("perm", [1, 2, 0]) + ) else: assert False, "not support" @@ -640,7 +733,6 @@ def _apply(self): class LayerNormParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LayerNormParser, self).__init__(graph, var2geop) self.parser_name = "layer_norm" @@ -654,44 +746,65 @@ def _apply(self): x_dtype = self.op.block.var(self.op.input_arg_names[2]).dtype shape_tensor = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), "Shape").set_input("x", x) - scale_expand = core.GEOperatorFactory.create_operator( - "broadcast_to_d" + self._accumulated_op_id(), - "BroadcastTo").set_input("x", - scale).set_input("shape", shape_tensor) - bias_expand = core.GEOperatorFactory.create_operator( - "broadcast_to_d" + self._accumulated_op_id(), - "BroadcastTo").set_input("x", - bias).set_input("shape", shape_tensor) - layer_norm = core.GEOperatorFactory.create_operator( - "layer_norm" + self._accumulated_op_id(), - "LayerNorm").set_input("x", x).set_input( - "gamma", - scale_expand).set_input("beta", bias_expand).set_attr_int32( - "begin_norm_axis", begin_norm_axis).set_attr_int32( - "begin_params_axis", - begin_norm_axis).set_attr_float("epsilon", epsilon) - - cast_dtype = 0 if self.ascend_helper.dtype2paddle_inv_map[str( - x_dtype)] == 0 else 1 - y = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", layer_norm, - 0).set_attr_int32("dst_type", cast_dtype) - mean = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", layer_norm, - 1).set_attr_int32("dst_type", cast_dtype) - variance = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", layer_norm, - 2).set_attr_int32("dst_type", cast_dtype) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", x) + scale_expand = ( + core.GEOperatorFactory.create_operator( + "broadcast_to_d" + self._accumulated_op_id(), "BroadcastTo" + ) + .set_input("x", scale) + .set_input("shape", shape_tensor) + ) + bias_expand = ( + core.GEOperatorFactory.create_operator( + "broadcast_to_d" + self._accumulated_op_id(), "BroadcastTo" + ) + .set_input("x", bias) + .set_input("shape", shape_tensor) + ) + layer_norm = ( + core.GEOperatorFactory.create_operator( + "layer_norm" + self._accumulated_op_id(), "LayerNorm" + ) + .set_input("x", x) + .set_input("gamma", scale_expand) + .set_input("beta", bias_expand) + .set_attr_int32("begin_norm_axis", begin_norm_axis) + .set_attr_int32("begin_params_axis", begin_norm_axis) + .set_attr_float("epsilon", epsilon) + ) + + cast_dtype = ( + 0 + if self.ascend_helper.dtype2paddle_inv_map[str(x_dtype)] == 0 + else 1 + ) + y = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", layer_norm, 0) + .set_attr_int32("dst_type", cast_dtype) + ) + mean = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", layer_norm, 1) + .set_attr_int32("dst_type", cast_dtype) + ) + variance = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", layer_norm, 2) + .set_attr_int32("dst_type", cast_dtype) + ) return [y, mean, variance], [[1], [2], [0]] ## activate function class ReluParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReluParser, self).__init__(graph, var2geop) self.parser_name = "relu" @@ -699,12 +812,12 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) relu = core.GEOperatorFactory.create_operator( - "relu" + self._accumulated_op_id(), "Relu").set_input("x", x) + "relu" + self._accumulated_op_id(), "Relu" + ).set_input("x", x) return [relu], [[0]] class GeluParser(AscendParserBase): - def __init__(self, graph, var2geop): super(GeluParser, self).__init__(graph, var2geop) self.parser_name = "gelu" @@ -712,12 +825,12 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) gelu = core.GEOperatorFactory.create_operator( - "gelu" + self._accumulated_op_id(), "Gelu").set_input("x", x) + "gelu" + self._accumulated_op_id(), "Gelu" + ).set_input("x", x) return [gelu], [[0]] class TanhParser(AscendParserBase): - def __init__(self, graph, var2geop): super(TanhParser, self).__init__(graph, var2geop) self.parser_name = "tanh" @@ -725,13 +838,13 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) tanh = core.GEOperatorFactory.create_operator( - "tanh" + self._accumulated_op_id(), "Tanh").set_input("x", x) + "tanh" + self._accumulated_op_id(), "Tanh" + ).set_input("x", x) return [tanh], [[0]] ## loss function class SoftmaxWithCrossEntropyParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SoftmaxWithCrossEntropyParser, self).__init__(graph, var2geop) self.parser_name = "softmax_with_cross_entropy" @@ -742,48 +855,65 @@ def _apply(self): cls_num = self.op.block.var(self.op.input_arg_names[1]).shape[1] softmax = core.GEOperatorFactory.create_operator( - "softmax" + self._accumulated_op_id(), - "SoftmaxV2").set_input("x", logits) - label = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", label).set_attr_int32("dst_type", 3) + "softmax" + self._accumulated_op_id(), "SoftmaxV2" + ).set_input("x", logits) + label = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", label) + .set_attr_int32("dst_type", 3) + ) tensoron = self._create_ge_tensor([1], 5, 1) on = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensoron) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensoron) tensoroff = self._create_ge_tensor([1], 5, 0) off = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensoroff) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensoroff) self._mark_as_input(on) self._mark_as_input(off) - onehot = core.GEOperatorFactory.create_operator( - "onehot" + self._accumulated_op_id(), - "OneHotD").set_input("x", - label).set_input("on_value", on).set_input( - "off_value", - off).set_attr_int32("depth", cls_num) + onehot = ( + core.GEOperatorFactory.create_operator( + "onehot" + self._accumulated_op_id(), "OneHotD" + ) + .set_input("x", label) + .set_input("on_value", on) + .set_input("off_value", off) + .set_attr_int32("depth", cls_num) + ) squeeze = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "Squeeze").set_input("x", onehot) - - loss_all = core.GEOperatorFactory.create_operator( - "loss" + self._accumulated_op_id(), - "SoftmaxCrossEntropyWithLogits").set_input("features", - logits).set_input( - "labels", squeeze) - loss = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", loss_all, 0).set_attr_int32("dst_type", 0) - loss_expand = core.GEOperatorFactory.create_operator( - "unsqueeze" + self._accumulated_op_id(), - "Unsqueeze").set_input("x", loss).set_attr_vec_int32("axes", [1]) + "mul" + self._accumulated_op_id(), "Squeeze" + ).set_input("x", onehot) + + loss_all = ( + core.GEOperatorFactory.create_operator( + "loss" + self._accumulated_op_id(), + "SoftmaxCrossEntropyWithLogits", + ) + .set_input("features", logits) + .set_input("labels", squeeze) + ) + loss = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", loss_all, 0) + .set_attr_int32("dst_type", 0) + ) + loss_expand = ( + core.GEOperatorFactory.create_operator( + "unsqueeze" + self._accumulated_op_id(), "Unsqueeze" + ) + .set_input("x", loss) + .set_attr_vec_int32("axes", [1]) + ) return [label, softmax, loss_expand], [[2], [1]] class SoftMaxParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SoftMaxParser, self).__init__(graph, var2geop) self.parser_name = "softmax" @@ -792,16 +922,18 @@ def _apply(self): logits = self._get_ge_input(self.op.input_arg_names[0]) axes = self.op.attr("axis") - softmax = core.GEOperatorFactory.create_operator( - "softmax" + self._accumulated_op_id(), - "SoftmaxV2").set_input("x", - logits).set_attr_vec_int32("axes", [axes]) + softmax = ( + core.GEOperatorFactory.create_operator( + "softmax" + self._accumulated_op_id(), "SoftmaxV2" + ) + .set_input("x", logits) + .set_attr_vec_int32("axes", [axes]) + ) return [softmax], [[0]] ## general class ShapeParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ShapeParser, self).__init__(graph, var2geop) self.parser_name = "shape" @@ -809,12 +941,12 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), "Shape").set_input("x", x) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", x) return [shape], [[0]] class FillConstantParser(AscendParserBase): - def __init__(self, graph, var2geop): super(FillConstantParser, self).__init__(graph, var2geop) self.parser_name = "fill_constant" @@ -826,27 +958,35 @@ def _apply(self): tensor = self._create_ge_tensor(shape, dtype, value) const = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) self._mark_as_input(const) if self.op.block.var(self.op.output('Out')[0]).persistable: - #print("%s is Persistable in fill_constant" % + # print("%s is Persistable in fill_constant" % # (self.op.output('Out')[0])) var = core.GEOperatorFactory.create_operator( - self.op.output('Out')[0], "Variable") + self.op.output('Out')[0], "Variable" + ) var.update_output_desc( "y", - core.GETensorDesc(core.GEShape(shape), core.GEFormat.FORMAT_ND, - core.GEDataType.DT_FLOAT)) - assign = core.GEOperatorFactory.create_operator( - "assign" + self._accumulated_op_id(), - "Assign").set_input("value", const).set_input("ref", var) + core.GETensorDesc( + core.GEShape(shape), + core.GEFormat.FORMAT_ND, + core.GEDataType.DT_FLOAT, + ), + ) + assign = ( + core.GEOperatorFactory.create_operator( + "assign" + self._accumulated_op_id(), "Assign" + ) + .set_input("value", const) + .set_input("ref", var) + ) return [const], [[0]] return [const], [[0]] class TruncatedNormalParser(AscendParserBase): - def __init__(self, graph, var2geop): super(TruncatedNormalParser, self).__init__(graph, var2geop) self.parser_name = "truncated_gaussian_random" @@ -860,24 +1000,24 @@ def _apply(self): tensor1 = self._create_ge_tensor([len(shape)], 2, shape) shape_tensor = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor1) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor1) tensor2 = self._create_ge_tensor([1], dtype, mean) mean_tensor = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor2) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor2) tensor3 = self._create_ge_tensor([1], dtype, std) std_tensor = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor3) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor3) tensor4 = self._create_ge_tensor([1], dtype, mean - 2 * std) min_tensor = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor4) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor4) tensor5 = self._create_ge_tensor([1], dtype, mean + 2 * std) max_tensor = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor5) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor5) self._mark_as_input(shape_tensor) self._mark_as_input(mean_tensor) @@ -885,34 +1025,50 @@ def _apply(self): self._mark_as_input(min_tensor) self._mark_as_input(max_tensor) - truncated_normal = core.GEOperatorFactory.create_operator( - "truncated_normal" + self._accumulated_op_id(), - "ParameterizedTruncatedNormal").set_input( - "shape", - shape_tensor).set_input("means", mean_tensor).set_input( - "stdevs", - std_tensor).set_input("min", min_tensor).set_input( - "max", max_tensor).set_attr_int32("seed", 0) + truncated_normal = ( + core.GEOperatorFactory.create_operator( + "truncated_normal" + self._accumulated_op_id(), + "ParameterizedTruncatedNormal", + ) + .set_input("shape", shape_tensor) + .set_input("means", mean_tensor) + .set_input("stdevs", std_tensor) + .set_input("min", min_tensor) + .set_input("max", max_tensor) + .set_attr_int32("seed", 0) + ) ## wirte the output of truncatedNormal from startup_program to main_program if self.op.block.var(self.op.output('Out')[0]).persistable: - #print("%s is Persistable in truncated_normal" % + # print("%s is Persistable in truncated_normal" % # (self.op.output('Out')[0])) var = core.GEOperatorFactory.create_operator( - self.op.output('Out')[0], "Variable") + self.op.output('Out')[0], "Variable" + ) var.update_output_desc( "y", - core.GETensorDesc(core.GEShape(shape), core.GEFormat.FORMAT_ND, - core.GEDataType.DT_FLOAT)) - assign = core.GEOperatorFactory.create_operator( - "assign" + self._accumulated_op_id(), - "Assign").set_input("value", - truncated_normal).set_input("ref", var) + core.GETensorDesc( + core.GEShape(shape), + core.GEFormat.FORMAT_ND, + core.GEDataType.DT_FLOAT, + ), + ) + assign = ( + core.GEOperatorFactory.create_operator( + "assign" + self._accumulated_op_id(), "Assign" + ) + .set_input("value", truncated_normal) + .set_input("ref", var) + ) return [ - shape_tensor, mean_tensor, std_tensor, min_tensor, max_tensor, - truncated_normal + shape_tensor, + mean_tensor, + std_tensor, + min_tensor, + max_tensor, + truncated_normal, ], [[-1]] - #else: + # else: # print( # "self.op.output('Out')[0] is not persistable in truncated_noraml" # ) @@ -920,7 +1076,6 @@ def _apply(self): class GatherParser(AscendParserBase): - def __init__(self, graph, var2geop): super(GatherParser, self).__init__(graph, var2geop) self.parser_name = "gather" @@ -930,16 +1085,18 @@ def _apply(self): x = self._get_ge_input(self.op.input_arg_names[1]) clo = self.op.block.var(self.op.input_arg_names[1]).shape[-1] - gather = core.GEOperatorFactory.create_operator( - "gather" + self._accumulated_op_id(), - "Gather").set_input("x", x).set_input("indices", - index).set_attr_bool( - "validate_indices", True) + gather = ( + core.GEOperatorFactory.create_operator( + "gather" + self._accumulated_op_id(), "Gather" + ) + .set_input("x", x) + .set_input("indices", index) + .set_attr_bool("validate_indices", True) + ) return [gather], [[0]] class ScatterParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ScatterParser, self).__init__(graph, var2geop) self.parser_name = "scatter" @@ -952,25 +1109,35 @@ def _apply(self): index_shape = self.op.block.var(self.op.input_arg_names[0]).shape if len(index_shape) == 1: - index = core.GEOperatorFactory.create_operator( - "unsqueeze" + self.getid(), - "Unsqueeze").set_input("x", - index).set_attr_vec_int32("axes", [1]) + index = ( + core.GEOperatorFactory.create_operator( + "unsqueeze" + self.getid(), "Unsqueeze" + ) + .set_input("x", index) + .set_attr_vec_int32("axes", [1]) + ) if not overwrite: - scatter_value = core.GEOperatorFactory.create_operator( - "scatter" + self._accumulated_op_id(), - "TensorScatterAdd").set_input("x", x).set_input( - "indices", index).set_input("updates", updates) + scatter_value = ( + core.GEOperatorFactory.create_operator( + "scatter" + self._accumulated_op_id(), "TensorScatterAdd" + ) + .set_input("x", x) + .set_input("indices", index) + .set_input("updates", updates) + ) else: - scatter_value = core.GEOperatorFactory.create_operator( - "scatter" + self._accumulated_op_id(), - "TensorScatterUpdate").set_input("x", x).set_input( - "indices", index).set_input("updates", updates) + scatter_value = ( + core.GEOperatorFactory.create_operator( + "scatter" + self._accumulated_op_id(), "TensorScatterUpdate" + ) + .set_input("x", x) + .set_input("indices", index) + .set_input("updates", updates) + ) return [x, index, updates, scatter_value], [[-1]] class CastParser(AscendParserBase): - def __init__(self, graph, var2geop): super(CastParser, self).__init__(graph, var2geop) self.parser_name = "cast" @@ -978,14 +1145,17 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) dtype = self.op.attr("out_dtype") - cast = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", x).set_attr_int32("dst_type", dtype) + cast = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", x) + .set_attr_int32("dst_type", dtype) + ) return [cast], [[0]] class AssignParser(AscendParserBase): - def __init__(self, graph, var2geop): super(AssignParser, self).__init__(graph, var2geop) self.parser_name = "assign" @@ -993,14 +1163,17 @@ def __init__(self, graph, var2geop): def _apply(self): const = self._get_ge_input(self.op.input_arg_names[0]) var = self._get_ge_input(self.op.input_arg_names[1]) - assign = core.GEOperatorFactory.create_operator( - "assign" + self._accumulated_op_id(), - "Assign").set_input("value", const).set_input("ref", var) + assign = ( + core.GEOperatorFactory.create_operator( + "assign" + self._accumulated_op_id(), "Assign" + ) + .set_input("value", const) + .set_input("ref", var) + ) return [assign], [[0]] class ScaleParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ScaleParser, self).__init__(graph, var2geop) self.parser_name = "scale" @@ -1012,27 +1185,36 @@ def _apply(self): bias_after_scale = self.op.attr("bias_after_scale") if bias_after_scale: - scale_value = core.GEOperatorFactory.create_operator( - "scale" + self._accumulated_op_id(), - "Power").set_input("x", x).set_attr_float( - "power", - 1.0).set_attr_float("scale", - scale).set_attr_float("shift", bias) + scale_value = ( + core.GEOperatorFactory.create_operator( + "scale" + self._accumulated_op_id(), "Power" + ) + .set_input("x", x) + .set_attr_float("power", 1.0) + .set_attr_float("scale", scale) + .set_attr_float("shift", bias) + ) else: - x_add_bias = core.GEOperatorFactory.create_operator( - "adds" + self._accumulated_op_id(), - "Adds").set_input("x", x).set_attr_float("value", bias) - scale_value = core.GEOperatorFactory.create_operator( - "scale" + self._accumulated_op_id(), - "Power").set_input("x", x_add_bias).set_attr_float( - "power", - 1.0).set_attr_float("scale", - scale).set_attr_float("shift", 0.0) + x_add_bias = ( + core.GEOperatorFactory.create_operator( + "adds" + self._accumulated_op_id(), "Adds" + ) + .set_input("x", x) + .set_attr_float("value", bias) + ) + scale_value = ( + core.GEOperatorFactory.create_operator( + "scale" + self._accumulated_op_id(), "Power" + ) + .set_input("x", x_add_bias) + .set_attr_float("power", 1.0) + .set_attr_float("scale", scale) + .set_attr_float("shift", 0.0) + ) return [scale_value], [[0]] class SliceParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SliceParser, self).__init__(graph, var2geop) self.parser_name = "slice" @@ -1058,18 +1240,22 @@ def _apply(self): cnt += 1 size = [ends_cor[i] - starts_cor[i] for i in range(len(axes_cor))] - assert len(axes_cor) == len(starts_cor) == len( - ends_cor), "the three fields must have same size" - slice_value = core.GEOperatorFactory.create_operator( - "slice" + self._accumulated_op_id(), - "SliceD").set_input("x", x).set_attr_vec_int32( - "offsets", starts_cor).set_attr_vec_int32("size", size) + assert ( + len(axes_cor) == len(starts_cor) == len(ends_cor) + ), "the three fields must have same size" + slice_value = ( + core.GEOperatorFactory.create_operator( + "slice" + self._accumulated_op_id(), "SliceD" + ) + .set_input("x", x) + .set_attr_vec_int32("offsets", starts_cor) + .set_attr_vec_int32("size", size) + ) return [slice_value], [[0]] class ReshapeParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReshapeParser, self).__init__(graph, var2geop) self.parser_name = "reshape2" @@ -1092,21 +1278,24 @@ def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) tensor = self._create_ge_tensor([len(shape)], 2, shape) const_shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor) - reshape = core.GEOperatorFactory.create_operator( - "reshape" + self._accumulated_op_id(), - "Reshape").set_input("x", x).set_input("shape", - const_shape).set_attr_int32( - "axis", 0) + "shape" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) + reshape = ( + core.GEOperatorFactory.create_operator( + "reshape" + self._accumulated_op_id(), "Reshape" + ) + .set_input("x", x) + .set_input("shape", const_shape) + .set_attr_int32("axis", 0) + ) x_shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), "Shape").set_input("x", x) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", x) return [x_shape, reshape], [[1], [0]] class TransposeParser(AscendParserBase): - def __init__(self, graph, var2geop): super(TransposeParser, self).__init__(graph, var2geop) self.parser_name = "transpose2" @@ -1114,17 +1303,21 @@ def __init__(self, graph, var2geop): def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) perm = self.op.attr("axis") - transpose = core.GEOperatorFactory.create_operator( - "transpose" + self._accumulated_op_id(), - "TransposeD").set_input("x", x).set_attr_vec_int32("perm", perm) + transpose = ( + core.GEOperatorFactory.create_operator( + "transpose" + self._accumulated_op_id(), "TransposeD" + ) + .set_input("x", x) + .set_attr_vec_int32("perm", perm) + ) x_shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), "Shape").set_input("x", x) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", x) return [x_shape, transpose], [[1], [0]] class AccuracyParser(AscendParserBase): - def __init__(self, graph, var2geop): super(AccuracyParser, self).__init__(graph, var2geop) self.parser_name = "accuracy" @@ -1134,42 +1327,73 @@ def _apply(self): label = self._get_ge_input(self.op.input_arg_names[1]) logits = self._get_ge_input(self.op.input_arg_names[2]) - pred = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", pred).set_attr_int32("dst_type", 3) - label = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", label).set_attr_int32("dst_type", 3) - equal = core.GEOperatorFactory.create_operator( - "equal" + self._accumulated_op_id(), - "Equal").set_input("x1", pred).set_input("x2", label) - cast = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", equal).set_attr_int32("dst_type", 0) - acc = core.GEOperatorFactory.create_operator( - "mean" + self._accumulated_op_id(), "ReduceMeanD").set_input( - "x", cast).set_attr_bool("keep_dims", - False).set_attr_vec_int32("axes", []) - correct = core.GEOperatorFactory.create_operator( - "sum" + self._accumulated_op_id(), "ReduceSumD").set_input( - "x", cast).set_attr_bool("keep_dims", - False).set_attr_vec_int32("axes", []) - ones_tensor = core.GEOperatorFactory.create_operator( - "oneslike" + self._accumulated_op_id(), - "OnesLike").set_input("x", label) + pred = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", pred) + .set_attr_int32("dst_type", 3) + ) + label = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", label) + .set_attr_int32("dst_type", 3) + ) + equal = ( + core.GEOperatorFactory.create_operator( + "equal" + self._accumulated_op_id(), "Equal" + ) + .set_input("x1", pred) + .set_input("x2", label) + ) + cast = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", equal) + .set_attr_int32("dst_type", 0) + ) + acc = ( + core.GEOperatorFactory.create_operator( + "mean" + self._accumulated_op_id(), "ReduceMeanD" + ) + .set_input("x", cast) + .set_attr_bool("keep_dims", False) + .set_attr_vec_int32("axes", []) + ) + correct = ( + core.GEOperatorFactory.create_operator( + "sum" + self._accumulated_op_id(), "ReduceSumD" + ) + .set_input("x", cast) + .set_attr_bool("keep_dims", False) + .set_attr_vec_int32("axes", []) + ) ones_tensor = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", ones_tensor).set_attr_int32("dst_type", 0) - total = core.GEOperatorFactory.create_operator( - "sum" + self._accumulated_op_id(), - "ReduceSumD").set_input("x", ones_tensor).set_attr_bool( - "keep_dims", False).set_attr_vec_int32("axes", []) + "oneslike" + self._accumulated_op_id(), "OnesLike" + ).set_input("x", label) + ones_tensor = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", ones_tensor) + .set_attr_int32("dst_type", 0) + ) + total = ( + core.GEOperatorFactory.create_operator( + "sum" + self._accumulated_op_id(), "ReduceSumD" + ) + .set_input("x", ones_tensor) + .set_attr_bool("keep_dims", False) + .set_attr_vec_int32("axes", []) + ) return [acc, correct, total], [[0], [1], [2]] class TopkParser(AscendParserBase): - def __init__(self, graph, var2geop): super(TopkParser, self).__init__(graph, var2geop) self.parser_name = "top_k" @@ -1180,25 +1404,40 @@ def _apply(self): tensor = self._create_ge_tensor([1], 2, k) const_k = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor) - cast_x = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", x).set_attr_int32("dst_type", 1) - topk = core.GEOperatorFactory.create_operator( - "topk" + self._accumulated_op_id(), - "TopK").set_input("x", cast_x).set_input("k", const_k) - value = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", topk, 0).set_attr_int32("dst_type", 0) - index = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", topk, 1).set_attr_int32("dst_type", 0) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) + cast_x = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", x) + .set_attr_int32("dst_type", 1) + ) + topk = ( + core.GEOperatorFactory.create_operator( + "topk" + self._accumulated_op_id(), "TopK" + ) + .set_input("x", cast_x) + .set_input("k", const_k) + ) + value = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", topk, 0) + .set_attr_int32("dst_type", 0) + ) + index = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", topk, 1) + .set_attr_int32("dst_type", 0) + ) return [value, index], [[1], [0]] class LookupTableParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LookupTableParser, self).__init__(graph, var2geop) self.parser_name = "lookup_table" @@ -1207,17 +1446,24 @@ def _apply(self): ids = self._get_ge_input(self.op.input_arg_names[0]) w = self._get_ge_input(self.op.input_arg_names[1]) - ids_squeeze = core.GEOperatorFactory.create_operator( - "squeeze" + self._accumulated_op_id(), - "Squeeze").set_input("x", ids).set_attr_vec_int32("axes", [-1]) - out = core.GEOperatorFactory.create_operator( - "lookup" + self._accumulated_op_id(), - "Gather").set_input("x", w).set_input("indices", ids_squeeze) + ids_squeeze = ( + core.GEOperatorFactory.create_operator( + "squeeze" + self._accumulated_op_id(), "Squeeze" + ) + .set_input("x", ids) + .set_attr_vec_int32("axes", [-1]) + ) + out = ( + core.GEOperatorFactory.create_operator( + "lookup" + self._accumulated_op_id(), "Gather" + ) + .set_input("x", w) + .set_input("indices", ids_squeeze) + ) return [out], [[0]] class StackParser(AscendParserBase): - def __init__(self, graph, var2geop): super(StackParser, self).__init__(graph, var2geop) self.parser_name = "stack" @@ -1226,30 +1472,37 @@ def _apply(self): tiles = len(self.op.input_arg_names) data_x_lst = [] for index in range(tiles): - data_x_lst.append(self._get_ge_input( - self.op.input_arg_names[index])) + data_x_lst.append( + self._get_ge_input(self.op.input_arg_names[index]) + ) axis = self.op.attr("axis") data_x = data_x_lst[0] tensor = self._create_ge_tensor([1], 2, axis) tensor_axis = core.GEOperatorFactory.create_operator( - "axis" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor) - expand = core.GEOperatorFactory.create_operator( - "expand" + self._accumulated_op_id(), - "ExpandDims").set_input("x", data_x).set_input("axis", tensor_axis) - - stack = core.GEOperatorFactory.create_operator( - "stack" + self._accumulated_op_id(), "TileWithAxis").set_input( - "x", - expand).set_attr_int32("axis", - axis).set_attr_int32("tiles", tiles) + "axis" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) + expand = ( + core.GEOperatorFactory.create_operator( + "expand" + self._accumulated_op_id(), "ExpandDims" + ) + .set_input("x", data_x) + .set_input("axis", tensor_axis) + ) + + stack = ( + core.GEOperatorFactory.create_operator( + "stack" + self._accumulated_op_id(), "TileWithAxis" + ) + .set_input("x", expand) + .set_attr_int32("axis", axis) + .set_attr_int32("tiles", tiles) + ) return [stack], [[0]] class UnSqueezeParser(AscendParserBase): - def __init__(self, graph, var2geop): super(UnSqueezeParser, self).__init__(graph, var2geop) self.parser_name = "unsqueeze2" @@ -1258,18 +1511,21 @@ def _apply(self): x = self._get_ge_input(self.op.input_arg_names[0]) axes = self.op.attr('axes') - output = core.GEOperatorFactory.create_operator( - "unsqueeze" + self._accumulated_op_id(), - "Unsqueeze").set_input("x", x).set_attr_vec_int32("axes", axes) + output = ( + core.GEOperatorFactory.create_operator( + "unsqueeze" + self._accumulated_op_id(), "Unsqueeze" + ) + .set_input("x", x) + .set_attr_vec_int32("axes", axes) + ) shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), - "Shape").set_input("x", output) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", output) return [shape, output], [[1], [0]] ## parallel class AllGatherParser(AscendParserBase): - def __init__(self, graph, var2geop): super(AllGatherParser, self).__init__(graph, var2geop) self.parser_name = "c_allgather" @@ -1279,15 +1535,18 @@ def _apply(self): rank_size = self.op.attr("rank_size") group = self.op.attr("group") - allgather = core.GEOperatorFactory.create_operator( - "allgather" + self._accumulated_op_id(), - "HcomAllGather").set_input("x", x).set_attr_int32( - "rank_size", rank_size).set_attr_string("group", group) + allgather = ( + core.GEOperatorFactory.create_operator( + "allgather" + self._accumulated_op_id(), "HcomAllGather" + ) + .set_input("x", x) + .set_attr_int32("rank_size", rank_size) + .set_attr_string("group", group) + ) return [allgather], [[0]] class AllReduceParser(AscendParserBase): - def __init__(self, graph, var2geop, reduction): super(AllReduceParser, self).__init__(graph, var2geop) self.parser_name = "c_allreduce_" + reduction @@ -1298,13 +1557,17 @@ def _apply(self): reduction = self.reduction ring_id = self.op.attr("ring_id") group = "hcom_group_" + str(ring_id) - fusion = None #self.op.attr("fusion") - fusion_id = None #self.op.attr("fusion_id") - - allreduce = core.GEOperatorFactory.create_operator( - "allreduce" + self._accumulated_op_id(), - "HcomAllReduce").set_input("x", x).set_attr_string( - "reduction", reduction).set_attr_string("group", group) + fusion = None # self.op.attr("fusion") + fusion_id = None # self.op.attr("fusion_id") + + allreduce = ( + core.GEOperatorFactory.create_operator( + "allreduce" + self._accumulated_op_id(), "HcomAllReduce" + ) + .set_input("x", x) + .set_attr_string("reduction", reduction) + .set_attr_string("group", group) + ) if fusion is not None: allreduce.set_attr_int32("fusion", fusion) @@ -1314,19 +1577,16 @@ def _apply(self): class AllReduceSumParser(AllReduceParser): - def __init__(self, graph, var2geop): super(AllReduceSumParser, self).__init__(graph, var2geop, 'sum') class AllReduceMaxParser(AllReduceParser): - def __init__(self, graph, var2geop): super(AllReduceMaxParser, self).__init__(graph, var2geop, 'max') class BroadcastParser(AscendParserBase): - def __init__(self, graph, var2geop): super(BroadcastParser, self).__init__(graph, var2geop) self.parser_name = "c_broadcast" @@ -1336,15 +1596,18 @@ def _apply(self): root_rank = self.op.attr("root_rank") group = self.op.attr("group") - broadcast = core.GEOperatorFactory.create_operator( - "broadcast" + self._accumulated_op_id(), - "HcomBroadcast").set_input("x", x).set_attr_int32( - "root_rank", root_rank).set_attr_string("group", group) + broadcast = ( + core.GEOperatorFactory.create_operator( + "broadcast" + self._accumulated_op_id(), "HcomBroadcast" + ) + .set_input("x", x) + .set_attr_int32("root_rank", root_rank) + .set_attr_string("group", group) + ) return [broadcast], [[0]] class ReduceScatterParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReduceScatterParser, self).__init__(graph, var2geop) self.parser_name = "c_reduce_scatter" @@ -1355,17 +1618,19 @@ def _apply(self): group = self.op.attr("group") rank_size = self.op.attr("rank_size") - reduce_scatter = core.GEOperatorFactory.create_operator( - "reducescatter" + self._accumulated_op_id(), - "HcomReduceScatter").set_input("x", x).set_attr_string( - "reduction", - reduction).set_attr_string("group", group).set_attr_int32( - "rank_size", rank_size) + reduce_scatter = ( + core.GEOperatorFactory.create_operator( + "reducescatter" + self._accumulated_op_id(), "HcomReduceScatter" + ) + .set_input("x", x) + .set_attr_string("reduction", reduction) + .set_attr_string("group", group) + .set_attr_int32("rank_size", rank_size) + ) return [reduce_scatter], [[0]] class SendParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SendParser, self).__init__(graph, var2geop) self.parser_name = "c_send" @@ -1376,15 +1641,19 @@ def _apply(self): dest_rank = self.op.attr("dest_rank") group = self.op.attr("group") - send = core.GEOperatorFactory.create_operator( - "send" + self._accumulated_op_id(), "HcomSend").set_input( - "x", x).set_attr_int32("sr_tag", sr_tag).set_attr_int32( - "dest_rank", dest_rank).set_attr_string("group", group) + send = ( + core.GEOperatorFactory.create_operator( + "send" + self._accumulated_op_id(), "HcomSend" + ) + .set_input("x", x) + .set_attr_int32("sr_tag", sr_tag) + .set_attr_int32("dest_rank", dest_rank) + .set_attr_string("group", group) + ) return [send], [[0]] class ReceiveParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReceiveParser, self).__init__(graph, var2geop) self.parser_name = "c_receive" @@ -1397,19 +1666,21 @@ def _apply(self): shape = self.op.attr("shape") dtype = self.op.attr("dtype") - receive = core.GEOperatorFactory.create_operator( - "receive" + self._accumulated_op_id(), - "HcomReceive").set_input("x", x).set_attr_int32( - "sr_tag", - sr_tag).set_attr_int32("src_rank", src_rank).set_attr_string( - "group", group).set_attr_vec_int32("shape", - shape).set_attr_int32( - "dtype", dtype) + receive = ( + core.GEOperatorFactory.create_operator( + "receive" + self._accumulated_op_id(), "HcomReceive" + ) + .set_input("x", x) + .set_attr_int32("sr_tag", sr_tag) + .set_attr_int32("src_rank", src_rank) + .set_attr_string("group", group) + .set_attr_vec_int32("shape", shape) + .set_attr_int32("dtype", dtype) + ) return [receive], [[0]] class RangeParser(AscendParserBase): - def __init__(self, graph, var2geop): super(RangeParser, self).__init__(graph, var2geop) self.parser_name = "range" @@ -1420,17 +1691,19 @@ def _apply(self): end = self._get_ge_input(self.op.input_arg_names[1]) delta = self._get_ge_input(self.op.input_arg_names[2]) - ge_range = core.GEOperatorFactory.create_operator( - "range" + self._accumulated_op_id(), "Range")\ - .set_input("start", end)\ - .set_input("limit", start) \ - .set_input("delta", delta) + ge_range = ( + core.GEOperatorFactory.create_operator( + "range" + self._accumulated_op_id(), "Range" + ) + .set_input("start", end) + .set_input("limit", start) + .set_input("delta", delta) + ) return [ge_range], [[0]] class UniformRandomParser(AscendParserBase): - def __init__(self, graph, var2geop): super(UniformRandomParser, self).__init__(graph, var2geop) self.parser_name = "uniform_random" @@ -1442,35 +1715,42 @@ def _apply(self): max_v = self.op.attr("max") seed = self.op.attr("seed") dtype = self.op.attr("dtype") - assert max_v > min_v, "assert max_v > min_v, but received " + \ - "as max_v={}, min_v={} ".format(max_v, min_v) + assert max_v > min_v, ( + "assert max_v > min_v, but received " + + "as max_v={}, min_v={} ".format(max_v, min_v) + ) tensor1 = self._create_ge_tensor([len(shape)], 2, shape) shape_tensor = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor1) - - ge_ur = core.GEOperatorFactory.create_operator( - "uniform_random" + self._accumulated_op_id(), "RandomUniform")\ - .set_input("shape", shape_tensor)\ - .set_attr_dtype("dtype", self.ascend_helper.dtype2ge(dtype)) \ - .set_attr_int32("seed", seed)\ + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor1) + + ge_ur = ( + core.GEOperatorFactory.create_operator( + "uniform_random" + self._accumulated_op_id(), "RandomUniform" + ) + .set_input("shape", shape_tensor) + .set_attr_dtype("dtype", self.ascend_helper.dtype2ge(dtype)) + .set_attr_int32("seed", seed) .set_attr_int32("seed2", seed) + ) scale = max_v - min_v - scale_value = core.GEOperatorFactory.create_operator( - "scale" + self._accumulated_op_id(), - "Power").set_input("x", ge_ur).set_attr_float( - "power", - 1.0).set_attr_float("scale", - scale).set_attr_float("shift", min_v) + scale_value = ( + core.GEOperatorFactory.create_operator( + "scale" + self._accumulated_op_id(), "Power" + ) + .set_input("x", ge_ur) + .set_attr_float("power", 1.0) + .set_attr_float("scale", scale) + .set_attr_float("shift", min_v) + ) return [scale_value], [[0]] class EqualParser(AscendParserBase): - def __init__(self, graph, var2geop): super(EqualParser, self).__init__(graph, var2geop) self.parser_name = "equal" @@ -1478,15 +1758,17 @@ def __init__(self, graph, var2geop): def _apply(self): data_x1 = self._get_ge_input(self.op.input_arg_names[0]) data_x2 = self._get_ge_input(self.op.input_arg_names[1]) - equal = core.GEOperatorFactory.create_operator("equal" \ - + self._accumulated_op_id(), "Equal")\ - .set_input("x1", data_x1)\ - .set_input("x2", data_x2) + equal = ( + core.GEOperatorFactory.create_operator( + "equal" + self._accumulated_op_id(), "Equal" + ) + .set_input("x1", data_x1) + .set_input("x2", data_x2) + ) return [equal], [[0]] class ExpandParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ExpandParser, self).__init__(graph, var2geop) self.parser_name = "expand" @@ -1496,19 +1778,21 @@ def _apply(self): expand_times = self.op.attr('expand_times') tensor = self._create_ge_tensor([len(expand_times)], 2, expand_times) - expand_tensor = core.GEOperatorFactory.\ - create_operator("const" + self._accumulated_op_id(), "Const")\ - .set_attr_tensor("value", tensor) - - assign = core.GEOperatorFactory\ - .create_operator("tile" + self._accumulated_op_id(), "Tile")\ - .set_input("x", data_x1_shape)\ - .set_input("multiples", expand_tensor) + expand_tensor = core.GEOperatorFactory.create_operator( + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) + + assign = ( + core.GEOperatorFactory.create_operator( + "tile" + self._accumulated_op_id(), "Tile" + ) + .set_input("x", data_x1_shape) + .set_input("multiples", expand_tensor) + ) return [assign], [[0]] class SqueezeParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SqueezeParser, self).__init__(graph, var2geop) self.parser_name = "squeeze2" @@ -1517,26 +1801,28 @@ def _apply(self): tensor = self._get_ge_input(self.op.input_arg_names[0]) axes = self.op.attr("axes") - data_squeezed = core.GEOperatorFactory\ - .create_operator("squeeze" + self._accumulated_op_id(), "Squeeze")\ - .set_input("x", tensor)\ - .set_attr_vec_int32("axes", axes) + data_squeezed = ( + core.GEOperatorFactory.create_operator( + "squeeze" + self._accumulated_op_id(), "Squeeze" + ) + .set_input("x", tensor) + .set_attr_vec_int32("axes", axes) + ) shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), - "Shape").set_input("x", data_squeezed) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", data_squeezed) return [shape, data_squeezed], [[1], [0]] -#****************************************************************# -#*************************** *************************# -#*************************** *************************# -#*************************** GradParser *************************# -#*************************** *************************# -#*************************** *************************# -#****************************************************************# +# ****************************************************************# +# *************************** *************************# +# *************************** *************************# +# *************************** GradParser *************************# +# *************************** *************************# +# *************************** *************************# +# ****************************************************************# ## grad class ReduceSumGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReduceSumGradParser, self).__init__(graph, var2geop) self.parser_name = "reduce_sum_grad" @@ -1546,24 +1832,27 @@ def _apply(self): input = self._get_ge_input(self.op.input_arg_names[1]) shape_tensor = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), - "Shape").set_input("x", input, 0) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", input, 0) tensoron = self._create_ge_tensor([1], 2, -1) const = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensoron) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensoron) self._mark_as_input(const) - reduce_sum = core.GEOperatorFactory.create_operator( - "broadcast_to_d" + self._accumulated_op_id(), - "BroadcastTo").set_input("x", x).set_input("shape", shape_tensor) - #reduce_sum = core.GEOperatorFactory.create_operator("expand" + self._accumulated_op_id(), "ExpandDims").set_input("x", reduce_sum).set_input("axis", const) + reduce_sum = ( + core.GEOperatorFactory.create_operator( + "broadcast_to_d" + self._accumulated_op_id(), "BroadcastTo" + ) + .set_input("x", x) + .set_input("shape", shape_tensor) + ) + # reduce_sum = core.GEOperatorFactory.create_operator("expand" + self._accumulated_op_id(), "ExpandDims").set_input("x", reduce_sum).set_input("axis", const) return [reduce_sum], [[0]] class MatMulGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MatMulGradParser, self).__init__(graph, var2geop) self.parser_name = "matmul_grad" @@ -1581,63 +1870,91 @@ def _apply(self): if len(x_shape) > 2: if transpose_y: - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "BatchMatMul").set_input("x1", out_grad).set_input( - "x2", - y).set_attr_bool("adj_x1", - False).set_attr_bool("adj_x2", False) - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "BatchMatMul").set_input("x1", out_grad).set_input( - "x2", - x).set_attr_bool("adj_x1", - True).set_attr_bool("adj_x2", False) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), + "BatchMatMul", + ) + .set_input("x1", out_grad) + .set_input("x2", y) + .set_attr_bool("adj_x1", False) + .set_attr_bool("adj_x2", False) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), + "BatchMatMul", + ) + .set_input("x1", out_grad) + .set_input("x2", x) + .set_attr_bool("adj_x1", True) + .set_attr_bool("adj_x2", False) + ) else: - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "BatchMatMul").set_input("x1", out_grad).set_input( - "x2", - y).set_attr_bool("adj_x1", - False).set_attr_bool("adj_x2", True) - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "BatchMatMul").set_input( - "x1", x).set_input("x2", out_grad).set_attr_bool( - "adj_x1", True).set_attr_bool("adj_x2", False) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), + "BatchMatMul", + ) + .set_input("x1", out_grad) + .set_input("x2", y) + .set_attr_bool("adj_x1", False) + .set_attr_bool("adj_x2", True) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), + "BatchMatMul", + ) + .set_input("x1", x) + .set_input("x2", out_grad) + .set_attr_bool("adj_x1", True) + .set_attr_bool("adj_x2", False) + ) else: if transpose_y: - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", out_grad).set_input( - "x2", y).set_attr_bool("transpose_x1", - False).set_attr_bool( - "transpose_x2", False) - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", out_grad).set_input( - "x2", x).set_attr_bool("transpose_x1", - True).set_attr_bool( - "transpose_x2", False) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", out_grad) + .set_input("x2", y) + .set_attr_bool("transpose_x1", False) + .set_attr_bool("transpose_x2", False) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", out_grad) + .set_input("x2", x) + .set_attr_bool("transpose_x1", True) + .set_attr_bool("transpose_x2", False) + ) else: - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", out_grad).set_input( - "x2", y).set_attr_bool("transpose_x1", - False).set_attr_bool( - "transpose_x2", True) - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", x).set_input( - "x2", out_grad).set_attr_bool("transpose_x1", - True).set_attr_bool( - "transpose_x2", False) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", out_grad) + .set_input("x2", y) + .set_attr_bool("transpose_x1", False) + .set_attr_bool("transpose_x2", True) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", x) + .set_input("x2", out_grad) + .set_attr_bool("transpose_x1", True) + .set_attr_bool("transpose_x2", False) + ) return [x_grad, y_grad], [[0], [1]] class MulGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MulGradParser, self).__init__(graph, var2geop) self.parser_name = "mul_grad" @@ -1655,78 +1972,114 @@ def _apply(self): if x_num_col_dims == 1 and y_num_col_dims == 1: if len(shape_x) == 2 and len(shape_y) == 2: - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", out_grad).set_input( - "x2", y).set_attr_bool("transpose_x1", - False).set_attr_bool( - "transpose_x2", True) - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", x).set_input( - "x2", out_grad).set_attr_bool("transpose_x1", - True).set_attr_bool( - "transpose_x2", False) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", out_grad) + .set_input("x2", y) + .set_attr_bool("transpose_x1", False) + .set_attr_bool("transpose_x2", True) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", x) + .set_input("x2", out_grad) + .set_attr_bool("transpose_x1", True) + .set_attr_bool("transpose_x2", False) + ) elif len(shape_x) == 3 and len(shape_y) == 2: flatten_x = core.GEOperatorFactory.create_operator( - "flatten" + self._accumulated_op_id(), - "Flatten").set_input("x", x) - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", out_grad).set_input( - "x2", y).set_attr_bool("transpose_x1", - False).set_attr_bool( - "transpose_x2", True) + "flatten" + self._accumulated_op_id(), "Flatten" + ).set_input("x", x) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", out_grad) + .set_input("x2", y) + .set_attr_bool("transpose_x1", False) + .set_attr_bool("transpose_x2", True) + ) if len(shape_out_grad) == 2: - x_grad = core.GEOperatorFactory.create_operator( - "unsqueeze" + self._accumulated_op_id(), - "Unsqueeze").set_input("x", x_grad).set_attr_vec_int32( - "axes", [1]) - - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", flatten_x).set_input( - "x2", out_grad).set_attr_bool("transpose_x1", - True).set_attr_bool( - "transpose_x2", False) + x_grad = ( + core.GEOperatorFactory.create_operator( + "unsqueeze" + self._accumulated_op_id(), "Unsqueeze" + ) + .set_input("x", x_grad) + .set_attr_vec_int32("axes", [1]) + ) + + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", flatten_x) + .set_input("x2", out_grad) + .set_attr_bool("transpose_x1", True) + .set_attr_bool("transpose_x2", False) + ) else: if len(shape_x) == 3 and len(shape_y) == 2: assert x_num_col_dims == 2, "only support 2" - flatten_x = core.GEOperatorFactory.create_operator( - "flatten" + self._accumulated_op_id(), - "FlattenV2").set_input("x", x).set_attr_int32( - "axis", 0).set_attr_int32("end_axis", 1) - flatten_out_grad = core.GEOperatorFactory.create_operator( - "flatten" + self._accumulated_op_id(), - "FlattenV2").set_input("x", out_grad).set_attr_int32( - "axis", 0).set_attr_int32("end_axis", 1) - - y_unsqueeze = core.GEOperatorFactory.create_operator( - "unsqueeze" + self._accumulated_op_id(), - "Unsqueeze").set_input("x", - y).set_attr_vec_int32("axes", [0]) - y_stack = core.GEOperatorFactory.create_operator( - "stack" + self._accumulated_op_id(), - "TileWithAxis").set_input("x", y_unsqueeze).set_attr_int32( - "axis", 0).set_attr_int32("tiles", shape_out_grad[0]) - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "BatchMatMul").set_input("x1", out_grad).set_input( - "x2", y_stack).set_attr_bool("adj_x1", - False).set_attr_bool( - "adj_x2", True) - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "MatMul").set_input("x1", flatten_x).set_input( - "x2", flatten_out_grad).set_attr_bool( - "transpose_x1", - True).set_attr_bool("transpose_x2", False) + flatten_x = ( + core.GEOperatorFactory.create_operator( + "flatten" + self._accumulated_op_id(), "FlattenV2" + ) + .set_input("x", x) + .set_attr_int32("axis", 0) + .set_attr_int32("end_axis", 1) + ) + flatten_out_grad = ( + core.GEOperatorFactory.create_operator( + "flatten" + self._accumulated_op_id(), "FlattenV2" + ) + .set_input("x", out_grad) + .set_attr_int32("axis", 0) + .set_attr_int32("end_axis", 1) + ) + + y_unsqueeze = ( + core.GEOperatorFactory.create_operator( + "unsqueeze" + self._accumulated_op_id(), "Unsqueeze" + ) + .set_input("x", y) + .set_attr_vec_int32("axes", [0]) + ) + y_stack = ( + core.GEOperatorFactory.create_operator( + "stack" + self._accumulated_op_id(), "TileWithAxis" + ) + .set_input("x", y_unsqueeze) + .set_attr_int32("axis", 0) + .set_attr_int32("tiles", shape_out_grad[0]) + ) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), + "BatchMatMul", + ) + .set_input("x1", out_grad) + .set_input("x2", y_stack) + .set_attr_bool("adj_x1", False) + .set_attr_bool("adj_x2", True) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "MatMul" + ) + .set_input("x1", flatten_x) + .set_input("x2", flatten_out_grad) + .set_attr_bool("transpose_x1", True) + .set_attr_bool("transpose_x2", False) + ) return [x_grad, y_grad], [[0], [1]] class ReluGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReluGradParser, self).__init__(graph, var2geop) self.parser_name = "relu_grad" @@ -1734,15 +2087,17 @@ def __init__(self, graph, var2geop): def _apply(self): out = self._get_ge_input(self.op.input_arg_names[0]) out_grad = self._get_ge_input(self.op.input_arg_names[1]) - relu_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "ReluGrad").set_input("gradients", - out_grad).set_input("features", out) + relu_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "ReluGrad" + ) + .set_input("gradients", out_grad) + .set_input("features", out) + ) return [relu_grad], [[0]] class SoftmaxWithCrossEntropyGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SoftmaxWithCrossEntropyGradParser, self).__init__(graph, var2geop) self.parser_name = "softmax_with_cross_entropy_grad" @@ -1759,39 +2114,53 @@ def _apply(self): tensoron = self._create_ge_tensor([1], 5, 1) on = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensoron) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensoron) tensoroff = self._create_ge_tensor([1], 5, 0) off = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensoroff) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensoroff) self._mark_as_input(on) self._mark_as_input(off) - label = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", label).set_attr_int32("dst_type", 3) - onehot = core.GEOperatorFactory.create_operator( - "onehot" + self._accumulated_op_id(), - "OneHotD").set_input("x", - label).set_input("on_value", on).set_input( - "off_value", - off).set_attr_int32("depth", cls_num) + label = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", label) + .set_attr_int32("dst_type", 3) + ) + onehot = ( + core.GEOperatorFactory.create_operator( + "onehot" + self._accumulated_op_id(), "OneHotD" + ) + .set_input("x", label) + .set_input("on_value", on) + .set_input("off_value", off) + .set_attr_int32("depth", cls_num) + ) squeeze = core.GEOperatorFactory.create_operator( - "suqeeze" + self._accumulated_op_id(), - "Squeeze").set_input("x", onehot) - sub = core.GEOperatorFactory.create_operator( - "sub" + self._accumulated_op_id(), - "Sub").set_input("x1", softmax).set_input("x2", squeeze) - grad = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "Mul").set_input("x1", loss_grad).set_input("x2", sub) + "suqeeze" + self._accumulated_op_id(), "Squeeze" + ).set_input("x", onehot) + sub = ( + core.GEOperatorFactory.create_operator( + "sub" + self._accumulated_op_id(), "Sub" + ) + .set_input("x1", softmax) + .set_input("x2", squeeze) + ) + grad = ( + core.GEOperatorFactory.create_operator( + "mul" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", loss_grad) + .set_input("x2", sub) + ) return [on, off, label, onehot, grad], [[-1]] class DotMulGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(DotMulGradParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_mul_grad" @@ -1801,18 +2170,25 @@ def _apply(self): out_1 = self._get_ge_input(self.op.input_arg_names[1]) out_2 = self._get_ge_input(self.op.input_arg_names[2]) - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "Mul").set_input("x1", out_grad).set_input("x2", out_2) - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "Mul").set_input("x1", out_1).set_input("x2", out_grad) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", out_grad) + .set_input("x2", out_2) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", out_1) + .set_input("x2", out_grad) + ) return [x_grad, y_grad], [[0], [1]] class DotAddGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(DotAddGradParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_add_grad" @@ -1828,36 +2204,53 @@ def _apply(self): x_grad = out_grad cur_time_x = len(out_grad_shape) - len(out_1_shape) for i in range(cur_time_x): - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "ReduceSumD").set_input("x", x_grad).set_attr_vec_int32( - "axes", [0]).set_attr_bool("keep_dims", False) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "ReduceSumD" + ) + .set_input("x", x_grad) + .set_attr_vec_int32("axes", [0]) + .set_attr_bool("keep_dims", False) + ) for axis, size in enumerate(out_1_shape): if size == 1: - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "ReduceSumD").set_input("x", x_grad).set_attr_vec_int32( - "axes", [axis]).set_attr_bool("keep_dims", True) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), + "ReduceSumD", + ) + .set_input("x", x_grad) + .set_attr_vec_int32("axes", [axis]) + .set_attr_bool("keep_dims", True) + ) y_grad = out_grad cur_time_y = len(out_grad_shape) - len(out_2_shape) for i in range(cur_time_y): - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "ReduceSumD").set_input("x", y_grad).set_attr_vec_int32( - "axes", [0]).set_attr_bool("keep_dims", False) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "ReduceSumD" + ) + .set_input("x", y_grad) + .set_attr_vec_int32("axes", [0]) + .set_attr_bool("keep_dims", False) + ) for axis, size in enumerate(out_2_shape): if size == 1: - y_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "ReduceSumD").set_input("x", y_grad).set_attr_vec_int32( - "axes", [axis]).set_attr_bool("keep_dims", True) + y_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), + "ReduceSumD", + ) + .set_input("x", y_grad) + .set_attr_vec_int32("axes", [axis]) + .set_attr_bool("keep_dims", True) + ) return [x_grad, y_grad], [[0], [1]] class DotDivGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(DotDivGradParser, self).__init__(graph, var2geop) self.parser_name = "elementwise_div_grad" @@ -1868,41 +2261,68 @@ def _apply(self): x = self._get_ge_input(self.op.input_arg_names[2]) y = self._get_ge_input(self.op.input_arg_names[3]) - y_power = core.GEOperatorFactory.create_operator( - "power" + self._accumulated_op_id(), - "Power").set_input("x", y).set_attr_float("power", -1) + y_power = ( + core.GEOperatorFactory.create_operator( + "power" + self._accumulated_op_id(), "Power" + ) + .set_input("x", y) + .set_attr_float("power", -1) + ) tensor_zeros = core.GEOperatorFactory.create_operator( - "zeroslike" + self._accumulated_op_id(), - "ZerosLike").set_input("x", x) - x_zero = core.GEOperatorFactory.create_operator( - "equal" + self._accumulated_op_id(), - "Equal").set_input("x1", x).set_input("x2", tensor_zeros) + "zeroslike" + self._accumulated_op_id(), "ZerosLike" + ).set_input("x", x) + x_zero = ( + core.GEOperatorFactory.create_operator( + "equal" + self._accumulated_op_id(), "Equal" + ) + .set_input("x1", x) + .set_input("x2", tensor_zeros) + ) x_nozero = core.GEOperatorFactory.create_operator( - "logical_not" + self._accumulated_op_id(), - "LogicalNot").set_input("x", x_zero) - x_nozero_f = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", x_nozero).set_attr_int32("dst_type", 0) - x_grad_w = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "Mul").set_input("x1", x_nozero_f).set_input("x2", y_power) - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "Mul").set_input("x1", x_grad_w).set_input("x2", out_grad) - - y_grad_w = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "Mul").set_input("x1", out).set_input("x2", y_power) - y_grad = core.GEOperatorFactory.create_operator( - "mul" + self._accumulated_op_id(), - "Mul").set_input("x1", y_grad_w).set_input("x2", out_grad) + "logical_not" + self._accumulated_op_id(), "LogicalNot" + ).set_input("x", x_zero) + x_nozero_f = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", x_nozero) + .set_attr_int32("dst_type", 0) + ) + x_grad_w = ( + core.GEOperatorFactory.create_operator( + "mul" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", x_nozero_f) + .set_input("x2", y_power) + ) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", x_grad_w) + .set_input("x2", out_grad) + ) + + y_grad_w = ( + core.GEOperatorFactory.create_operator( + "mul" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", out) + .set_input("x2", y_power) + ) + y_grad = ( + core.GEOperatorFactory.create_operator( + "mul" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", y_grad_w) + .set_input("x2", out_grad) + ) return [x_grad, y_grad], [[0], [1]] class SoftmaxGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SoftmaxGradParser, self).__init__(graph, var2geop) self.parser_name = "softmax_grad" @@ -1911,15 +2331,17 @@ def _apply(self): out = self._get_ge_input(self.op.input_arg_names[0]) out_grad = self._get_ge_input(self.op.input_arg_names[1]) - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "SoftmaxGrad").set_input("softmax", - out).set_input("grad_softmax", out_grad) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "SoftmaxGrad" + ) + .set_input("softmax", out) + .set_input("grad_softmax", out_grad) + ) return [x_grad], [[0]] class ReshapeGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(ReshapeGradParser, self).__init__(graph, var2geop) self.parser_name = "reshape2_grad" @@ -1931,20 +2353,24 @@ def _apply(self): if x_shape_list[0] == 0: x_shape_delzero = x_shape_list[1:] - tensor = self._create_ge_tensor([len(x_shape_delzero)], 2, - x_shape_delzero) + tensor = self._create_ge_tensor( + [len(x_shape_delzero)], 2, x_shape_delzero + ) const_shape = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", tensor) - x_grad = core.GEOperatorFactory.create_operator( - "reshape" + self._accumulated_op_id(), - "Reshape").set_input("x", out_grad).set_input("shape", const_shape) + "shape" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", tensor) + x_grad = ( + core.GEOperatorFactory.create_operator( + "reshape" + self._accumulated_op_id(), "Reshape" + ) + .set_input("x", out_grad) + .set_input("shape", const_shape) + ) return [x_grad], [[0]] class GatherGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(GatherGradParser, self).__init__(graph, var2geop) self.parser_name = "gather_grad" @@ -1959,24 +2385,30 @@ def _apply(self): x_shape = self.op.block.var(self.op.input_arg_names[2]).shape if len(index_shape) == 1: - index = core.GEOperatorFactory.create_operator( - "unsqueeze" + self._accumulated_op_id(), - "Unsqueeze").set_input("x", - index).set_attr_vec_int32("axes", [1]) + index = ( + core.GEOperatorFactory.create_operator( + "unsqueeze" + self._accumulated_op_id(), "Unsqueeze" + ) + .set_input("x", index) + .set_attr_vec_int32("axes", [1]) + ) tensor_zeros = core.GEOperatorFactory.create_operator( - "zeroslike" + self._accumulated_op_id(), - "ZerosLike").set_input("x", x) - x_grad = core.GEOperatorFactory.create_operator( - "scatter" + self._accumulated_op_id(), - "TensorScatterUpdate").set_input("x", tensor_zeros).set_input( - "indices", index).set_input("updates", out_grad) + "zeroslike" + self._accumulated_op_id(), "ZerosLike" + ).set_input("x", x) + x_grad = ( + core.GEOperatorFactory.create_operator( + "scatter" + self._accumulated_op_id(), "TensorScatterUpdate" + ) + .set_input("x", tensor_zeros) + .set_input("indices", index) + .set_input("updates", out_grad) + ) return [tensor_zeros, x_grad], [[-1]] class TransposeGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(TransposeGradParser, self).__init__(graph, var2geop) self.parser_name = "transpose2_grad" @@ -1990,16 +2422,18 @@ def _apply(self): out_grad_shape = self.op.block.var(self.op.input_arg_names[0]).shape assert list(map(lambda x: out_grad_shape[x], perm)) == list(x_shape) - x_grad = core.GEOperatorFactory.create_operator( - "transpose" + self._accumulated_op_id(), - "TransposeD").set_input("x", - out_grad).set_attr_vec_int32("perm", perm) + x_grad = ( + core.GEOperatorFactory.create_operator( + "transpose" + self._accumulated_op_id(), "TransposeD" + ) + .set_input("x", out_grad) + .set_attr_vec_int32("perm", perm) + ) return [x_grad], [[0]] class LayerNormGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LayerNormGradParser, self).__init__(graph, var2geop) self.parser_name = "layer_norm_grad" @@ -2013,33 +2447,48 @@ def _apply(self): out_grad = self._get_ge_input(self.op.input_arg_names[5]) x_dtype = self.op.block.var(self.op.input_arg_names[4]).dtype - x_grad = core.GEOperatorFactory.create_operator( - self.parser_name + self._accumulated_op_id(), - "LayerNormGrad").set_input("dy", out_grad).set_input( - "x", x).set_input("variance", - variance).set_input("mean", mean).set_input( - "gamma", scale) - - cast_dtype = 0 if self.ascend_helper.dtype2paddle_inv_map[str( - x_dtype)] == 0 else 1 - out_x_grad = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", x_grad, - 0).set_attr_int32("dst_type", cast_dtype) - out_scale_grad = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", x_grad, - 1).set_attr_int32("dst_type", cast_dtype) - out_bias_grad = core.GEOperatorFactory.create_operator( - "cast" + self._accumulated_op_id(), - "Cast").set_input("x", x_grad, - 2).set_attr_int32("dst_type", cast_dtype) + x_grad = ( + core.GEOperatorFactory.create_operator( + self.parser_name + self._accumulated_op_id(), "LayerNormGrad" + ) + .set_input("dy", out_grad) + .set_input("x", x) + .set_input("variance", variance) + .set_input("mean", mean) + .set_input("gamma", scale) + ) + + cast_dtype = ( + 0 + if self.ascend_helper.dtype2paddle_inv_map[str(x_dtype)] == 0 + else 1 + ) + out_x_grad = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", x_grad, 0) + .set_attr_int32("dst_type", cast_dtype) + ) + out_scale_grad = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", x_grad, 1) + .set_attr_int32("dst_type", cast_dtype) + ) + out_bias_grad = ( + core.GEOperatorFactory.create_operator( + "cast" + self._accumulated_op_id(), "Cast" + ) + .set_input("x", x_grad, 2) + .set_attr_int32("dst_type", cast_dtype) + ) return [out_x_grad, out_scale_grad, out_bias_grad], [[2], [1], [0]] class TanhGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(TanhGradParser, self).__init__(graph, var2geop) self.parser_name = 'tanh_grad' @@ -2047,15 +2496,18 @@ def __init__(self, graph, var2geop): def _apply(self): y = self._get_ge_input(self.op.input_arg_names[0]) out_grad = self._get_ge_input(self.op.input_arg_names[1]) - tanh_grad = core.GEOperatorFactory.create_operator( - "tanh_grad" + self._accumulated_op_id(), - "TanhGrad").set_input("y", y).set_input("dy", out_grad) + tanh_grad = ( + core.GEOperatorFactory.create_operator( + "tanh_grad" + self._accumulated_op_id(), "TanhGrad" + ) + .set_input("y", y) + .set_input("dy", out_grad) + ) return [tanh_grad], [[0]] class LogGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LogGradParser, self).__init__(graph, var2geop) self.parser_name = 'log_grad' @@ -2063,14 +2515,17 @@ def __init__(self, graph, var2geop): def _apply(self): grad = self._get_ge_input(self.op.input_arg_names[0]) input = self._get_ge_input(self.op.input_arg_names[1]) - log_grad = core.GEOperatorFactory.create_operator( - "log_grad" + self._accumulated_op_id(), - "DivNoNan").set_input("x1", grad).set_input("x2", input) + log_grad = ( + core.GEOperatorFactory.create_operator( + "log_grad" + self._accumulated_op_id(), "DivNoNan" + ) + .set_input("x1", grad) + .set_input("x2", input) + ) return [log_grad], [[0]] class SqrtGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SqrtGradParser, self).__init__(graph, var2geop) self.parser_name = "sqrt_grad" @@ -2078,14 +2533,17 @@ def __init__(self, graph, var2geop): def _apply(self): y = self._get_ge_input(self.op.input_arg_names[0]) out_grad = self._get_ge_input(self.op.input_arg_names[1]) - sqrt_grad = core.GEOperatorFactory.create_operator( - "sqrt_grad" + self._accumulated_op_id(), - "SqrtGrad").set_input("y", y).set_input("dy", out_grad) + sqrt_grad = ( + core.GEOperatorFactory.create_operator( + "sqrt_grad" + self._accumulated_op_id(), "SqrtGrad" + ) + .set_input("y", y) + .set_input("dy", out_grad) + ) return [sqrt_grad] class PowGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(PowGradParser, self).__init__(graph, var2geop) self.parser_name = "pow_grad" @@ -2097,31 +2555,46 @@ def _apply(self): shape_tensor = self._create_shape_tensor() shape_tensor = core.GEOperatorFactory.create_operator( - "shape" + self._accumulated_op_id(), "Shape").set_input("x", x) + "shape" + self._accumulated_op_id(), "Shape" + ).set_input("x", x) factor_scale = self._create_ge_tensor([1], 5, factor) factor_scale = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", factor_scale) - factor_tensor = core.GEOperatorFactory.create_operator( - "broadcast_to_d" + self._accumulated_op_id(), - "BroadcastTo").set_input("x", factor_scale).set_input( - "shape", shape_tensor) - - x_power = core.GEOperatorFactory.create_operator( - "x_power" + self._accumulated_op_id(), - "Power").set_input("x", x).set_attr_float("power", factor - 1) - x_power_mul_factor = core.GEOperatorFactory.create_operator( - "x_power_mul_factor" + self._accumulated_op_id(), - "Mul").set_input("x1", x).set_input("x2", factor_tensor) - x_power_mul_factor_grad = core.GEOperatorFactory.create_operator( - "x_power_mul_factor_grad" + self._accumulated_op_id(), - "Mul").set_input("x1", x_power_mul_factor).set_input("x2", grad) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", factor_scale) + factor_tensor = ( + core.GEOperatorFactory.create_operator( + "broadcast_to_d" + self._accumulated_op_id(), "BroadcastTo" + ) + .set_input("x", factor_scale) + .set_input("shape", shape_tensor) + ) + + x_power = ( + core.GEOperatorFactory.create_operator( + "x_power" + self._accumulated_op_id(), "Power" + ) + .set_input("x", x) + .set_attr_float("power", factor - 1) + ) + x_power_mul_factor = ( + core.GEOperatorFactory.create_operator( + "x_power_mul_factor" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", x) + .set_input("x2", factor_tensor) + ) + x_power_mul_factor_grad = ( + core.GEOperatorFactory.create_operator( + "x_power_mul_factor_grad" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", x_power_mul_factor) + .set_input("x2", grad) + ) return [x_power_mul_factor_grad], [[0]] class GeluGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(GeluGradParser, self).__init__(graph, var2geop) self.parser_name = "gelu_grad" @@ -2131,17 +2604,21 @@ def _apply(self): x = self._get_ge_input(self.op.input_arg_names[1]) y = core.GEOperatorFactory.create_operator( - "gelu" + self._accumulated_op_id(), "Gelu").set_input("x", x) - gelu_grad = core.GEOperatorFactory.create_operator( - "gelu_grad" + self._accumulated_op_id(), - "GeluGrad").set_input("x", x).set_input("dy", - grad).set_input("y", y) + "gelu" + self._accumulated_op_id(), "Gelu" + ).set_input("x", x) + gelu_grad = ( + core.GEOperatorFactory.create_operator( + "gelu_grad" + self._accumulated_op_id(), "GeluGrad" + ) + .set_input("x", x) + .set_input("dy", grad) + .set_input("y", y) + ) return [gelu_grad], [[0]] class MeanGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(MeanGradParser, self).__init__(graph, var2geop) self.parser_name = "mean_grad" @@ -2151,25 +2628,36 @@ def _apply(self): x = self._get_ge_input(self.op.input_arg_names[1]) ones_tensor = core.GEOperatorFactory.create_operator( - "one_tensor" + self._accumulated_op_id(), - "OnesLike").set_input("x", x) - sum = core.GEOperatorFactory.create_operator( - "mean" + self._accumulated_op_id(), - "ReduceSumD").set_input("x", ones_tensor).set_attr_bool( - "keep_dims", False).set_attr_vec_int32("axes", []) - mean = core.GEOperatorFactory.create_operator( - "x_power" + self._accumulated_op_id(), - "Power").set_input("x", sum).set_attr_float("power", -1) - - mean_grad = core.GEOperatorFactory.create_operator( - "mean_grad" + self._accumulated_op_id(), - "Mul").set_input("x1", mean).set_input("x2", grad) + "one_tensor" + self._accumulated_op_id(), "OnesLike" + ).set_input("x", x) + sum = ( + core.GEOperatorFactory.create_operator( + "mean" + self._accumulated_op_id(), "ReduceSumD" + ) + .set_input("x", ones_tensor) + .set_attr_bool("keep_dims", False) + .set_attr_vec_int32("axes", []) + ) + mean = ( + core.GEOperatorFactory.create_operator( + "x_power" + self._accumulated_op_id(), "Power" + ) + .set_input("x", sum) + .set_attr_float("power", -1) + ) + + mean_grad = ( + core.GEOperatorFactory.create_operator( + "mean_grad" + self._accumulated_op_id(), "Mul" + ) + .set_input("x1", mean) + .set_input("x2", grad) + ) return [mean_grad], [[0]] class SliceGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SliceGradParser, self).__init__(graph, var2geop) self.parser_name = "slice_grad" @@ -2200,15 +2688,18 @@ def _apply(self): starts_cor[0] = 0 ends_cor[0] = 0 paddings = [[s, e] for (s, e) in zip(starts_cor, ends_cor)] - slice_value = core.GEOperatorFactory.create_operator( - "slice_grad" + self._accumulated_op_id(), "PadD").set_input( - "x", grad).set_attr_vec_vec_int64("paddings", paddings) + slice_value = ( + core.GEOperatorFactory.create_operator( + "slice_grad" + self._accumulated_op_id(), "PadD" + ) + .set_input("x", grad) + .set_attr_vec_vec_int64("paddings", paddings) + ) return [slice_value], [[0]] class LookUpTableGradParser(AscendParserBase): - def __init__(self, graph, var2geop): super(LookUpTableGradParser, self).__init__(graph, var2geop) self.parser_name = "lookup_table_grad" @@ -2222,28 +2713,39 @@ def _apply(self): shape_grad = self.op.block.var(self.op.input_arg_names[1]).shape shape_embedding = self.op.block.var(self.op.input_arg_names[2]).shape - ids_flatten = core.GEOperatorFactory.create_operator( - "flatten" + self._accumulated_op_id(), "FlattenV2").set_input( - "x", ids).set_attr_int32("axis", - 0).set_attr_int32("end_axis", 1) - grad_flatten = core.GEOperatorFactory.create_operator( - "flatten" + self._accumulated_op_id(), "FlattenV2").set_input( - "x", grad).set_attr_int32("axis", - 0).set_attr_int32("end_axis", 1) + ids_flatten = ( + core.GEOperatorFactory.create_operator( + "flatten" + self._accumulated_op_id(), "FlattenV2" + ) + .set_input("x", ids) + .set_attr_int32("axis", 0) + .set_attr_int32("end_axis", 1) + ) + grad_flatten = ( + core.GEOperatorFactory.create_operator( + "flatten" + self._accumulated_op_id(), "FlattenV2" + ) + .set_input("x", grad) + .set_attr_int32("axis", 0) + .set_attr_int32("end_axis", 1) + ) tensor_zeros = core.GEOperatorFactory.create_operator( - "zeroslike" + self._accumulated_op_id(), - "ZerosLike").set_input("x", embedding) - embedding_grad = core.GEOperatorFactory.create_operator( - "scatteradd" + self._accumulated_op_id(), - "TensorScatterAdd").set_input("x", tensor_zeros).set_input( - "indices", ids_flatten).set_input("updates", grad_flatten) + "zeroslike" + self._accumulated_op_id(), "ZerosLike" + ).set_input("x", embedding) + embedding_grad = ( + core.GEOperatorFactory.create_operator( + "scatteradd" + self._accumulated_op_id(), "TensorScatterAdd" + ) + .set_input("x", tensor_zeros) + .set_input("indices", ids_flatten) + .set_input("updates", grad_flatten) + ) return [embedding_grad], [[0]] class SGDParser(AscendParserBase): - def __init__(self, graph, var2geop): super(SGDParser, self).__init__(graph, var2geop) self.parser_name = "sgd" @@ -2252,15 +2754,18 @@ def _apply(self): grad = self._get_ge_input(self.op.input_arg_names[0]) lr = self._get_ge_input(self.op.input_arg_names[1]) param = self._get_ge_input(self.op.input_arg_names[2]) - sgd = core.GEOperatorFactory.create_operator( - "momentum" + self._accumulated_op_id(), - "ApplyGradientDescent").set_input("var", param).set_input( - "alpha", lr).set_input("delta", grad) + sgd = ( + core.GEOperatorFactory.create_operator( + "momentum" + self._accumulated_op_id(), "ApplyGradientDescent" + ) + .set_input("var", param) + .set_input("alpha", lr) + .set_input("delta", grad) + ) return [sgd], [[0]] class AdamParser(AscendParserBase): - def __init__(self, graph, var2geop): super(AdamParser, self).__init__(graph, var2geop) self.parser_name = "adam" @@ -2278,26 +2783,29 @@ def _apply(self): epsilon = self.op.attr('epsilon') beta1 = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", - self._create_ge_tensor([1], 5, beta1)) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", self._create_ge_tensor([1], 5, beta1)) beta2 = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", - self._create_ge_tensor([1], 5, beta2)) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", self._create_ge_tensor([1], 5, beta2)) epsilon = core.GEOperatorFactory.create_operator( - "const" + self._accumulated_op_id(), - "Const").set_attr_tensor("value", - self._create_ge_tensor([1], 5, epsilon)) - - adam = core.GEOperatorFactory.create_operator( - "adam" + self._accumulated_op_id(), - "ApplyAdam").set_input("var", param).set_input( - "m", moment1).set_input("v", moment2).set_input( - "beta1_power", beta1_power).set_input( - "beta2_power", - beta2_power).set_input("lr", lr).set_input( - "beta1", beta1).set_input("beta2", beta2).set_input( - "epsilon", epsilon).set_input("grad", grad) + "const" + self._accumulated_op_id(), "Const" + ).set_attr_tensor("value", self._create_ge_tensor([1], 5, epsilon)) + + adam = ( + core.GEOperatorFactory.create_operator( + "adam" + self._accumulated_op_id(), "ApplyAdam" + ) + .set_input("var", param) + .set_input("m", moment1) + .set_input("v", moment2) + .set_input("beta1_power", beta1_power) + .set_input("beta2_power", beta2_power) + .set_input("lr", lr) + .set_input("beta1", beta1) + .set_input("beta2", beta2) + .set_input("epsilon", epsilon) + .set_input("grad", grad) + ) return [adam], [[0]] diff --git a/python/paddle/distributed/fleet/meta_optimizers/asp_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/asp_optimizer.py index 2047c3172c260..59a75d1b054cd 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/asp_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/asp_optimizer.py @@ -19,23 +19,26 @@ class ASPOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(ASPOptimizer, self).__init__(optimizer) self.inner_opt = optimizer # we do not allow meta optimizer to be inner optimizer currently self.meta_optimizers_white_list = [ - "AMPOptimizer", "LarsOptimizer", "LambOptimizer", - "GraphExecutionOptimizer", "RecomputeOptimizer", - "GradientMergeOptimizer" + "AMPOptimizer", + "LarsOptimizer", + "LambOptimizer", + "GraphExecutionOptimizer", + "RecomputeOptimizer", + "GradientMergeOptimizer", ] self.meta_optimizers_black_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(ASPOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(ASPOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) def _can_apply(self): if not self.role_maker._is_collective: @@ -52,17 +55,16 @@ def _disable_strategy(self, dist_strategy): def _enable_strategy(self, dist_strategy, context): dist_strategy.asp = True - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): optimize_ops, params_grads = ASPHelper._minimize( self.inner_opt, loss, startup_program=startup_program, parameter_list=parameter_list, - no_grad_set=no_grad_set) + no_grad_set=no_grad_set, + ) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/common.py b/python/paddle/distributed/fleet/meta_optimizers/common.py index ed04188de20f0..7a3c89f1e9dea 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/common.py +++ b/python/paddle/distributed/fleet/meta_optimizers/common.py @@ -27,8 +27,11 @@ def is_update_op(op): - return 'Param' in op.input_names and 'Grad' in op.input_names and \ - "LearningRate" in op.input_names + return ( + 'Param' in op.input_names + and 'Grad' in op.input_names + and "LearningRate" in op.input_names + ) def is_loss_grad_op(op): @@ -39,17 +42,18 @@ def is_loss_grad_op(op): def is_backward_op(op): - return OP_ROLE_KEY in op.attr_names and \ - int(op.all_attrs()[OP_ROLE_KEY]) & int(OpRole.Backward) + return OP_ROLE_KEY in op.attr_names and int( + op.all_attrs()[OP_ROLE_KEY] + ) & int(OpRole.Backward) def is_optimizer_op(op): - return OP_ROLE_KEY in op.attr_names and \ - int(op.all_attrs()[OP_ROLE_KEY]) & int(OpRole.Optimize) + return OP_ROLE_KEY in op.attr_names and int( + op.all_attrs()[OP_ROLE_KEY] + ) & int(OpRole.Optimize) class CollectiveHelper(object): - def __init__(self, role_maker, nrings=1, wait_port=True): self.nrings = nrings self.wait_port = wait_port @@ -63,21 +67,27 @@ def update_startup_program(self, startup_program=None): endpoints = self.role_maker._get_trainer_endpoints() current_endpoint = endpoints[self.role_maker._worker_index()] for ring_id in range(self.nrings): - self._init_communicator(self.startup_program, - current_endpoint, endpoints, - self.role_maker._worker_index(), ring_id, - self.wait_port) + self._init_communicator( + self.startup_program, + current_endpoint, + endpoints, + self.role_maker._worker_index(), + ring_id, + self.wait_port, + ) self._broadcast_params() - def _init_communicator(self, - program, - current_endpoint, - endpoints, - rank, - ring_id, - wait_port, - global_ring_id=None, - sync=True): + def _init_communicator( + self, + program, + current_endpoint, + endpoints, + rank, + ring_id, + wait_port, + global_ring_id=None, + sync=True, + ): # if current_endpoint is None, it means just for sync, # no group is created. if current_endpoint: @@ -89,32 +99,40 @@ def _init_communicator(self, wait_server_ready(other_endpoints) def _add_sync_by_allreduce(block): - sync_var = block.create_var(name=unique_name.generate('sync_var'), - dtype=core.VarDesc.VarType.INT32, - persistable=False, - stop_gradient=True) - block.append_op(type='fill_constant', - inputs={}, - outputs={'Out': [sync_var]}, - attrs={ - 'shape': [1], - 'dtype': sync_var.dtype, - 'value': 1, - 'force_cpu': False, - OP_ROLE_KEY: OpRole.Forward - }) - block.append_op(type='c_allreduce_sum', - inputs={'X': [sync_var]}, - outputs={'Out': [sync_var]}, - attrs={ - 'ring_id': global_ring_id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward - }) - block.append_op(type='c_sync_calc_stream', - inputs={'X': sync_var}, - outputs={'Out': sync_var}, - attrs={OP_ROLE_KEY: OpRole.Forward}) + sync_var = block.create_var( + name=unique_name.generate('sync_var'), + dtype=core.VarDesc.VarType.INT32, + persistable=False, + stop_gradient=True, + ) + block.append_op( + type='fill_constant', + inputs={}, + outputs={'Out': [sync_var]}, + attrs={ + 'shape': [1], + 'dtype': sync_var.dtype, + 'value': 1, + 'force_cpu': False, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + block.append_op( + type='c_allreduce_sum', + inputs={'X': [sync_var]}, + outputs={'Out': [sync_var]}, + attrs={ + 'ring_id': global_ring_id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + block.append_op( + type='c_sync_calc_stream', + inputs={'X': sync_var}, + outputs={'Out': sync_var}, + attrs={OP_ROLE_KEY: OpRole.Forward}, + ) block = program.global_block() if current_endpoint is None: @@ -123,79 +141,93 @@ def _add_sync_by_allreduce(block): _add_sync_by_allreduce(block) return - comm_id_var = block.create_var(name=unique_name.generate('comm_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) + comm_id_var = block.create_var( + name=unique_name.generate('comm_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) if core.is_compiled_with_cuda(): - block.append_op(type='c_gen_nccl_id', - inputs={}, - outputs={'Out': comm_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints, - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) - block.append_op(type='c_comm_init', - inputs={'X': comm_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_gen_nccl_id', + inputs={}, + outputs={'Out': comm_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + block.append_op( + type='c_comm_init', + inputs={'X': comm_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Forward, + }, + ) elif core.is_compiled_with_xpu(): - block.append_op(type='c_gen_bkcl_id', - inputs={}, - outputs={'Out': comm_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints, - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) - block.append_op(type='c_comm_init', - inputs={'X': comm_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_gen_bkcl_id', + inputs={}, + outputs={'Out': comm_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + block.append_op( + type='c_comm_init', + inputs={'X': comm_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Forward, + }, + ) elif core.is_compiled_with_npu(): - block.append_op(type='c_gen_hccl_id', - inputs={}, - outputs={'Out': comm_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints, - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) - block.append_op(type='c_comm_init_hccl', - inputs={'X': comm_id_var}, - outputs={}, - attrs={ - 'rank': rank, - 'ring_id': ring_id, - 'device_id': - int(os.getenv("FLAGS_selected_npus")), - 'rank_ids': nranks, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_gen_hccl_id', + inputs={}, + outputs={'Out': comm_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + block.append_op( + type='c_comm_init_hccl', + inputs={'X': comm_id_var}, + outputs={}, + attrs={ + 'rank': rank, + 'ring_id': ring_id, + 'device_id': int(os.getenv("FLAGS_selected_npus")), + 'rank_ids': nranks, + OP_ROLE_KEY: OpRole.Forward, + }, + ) else: raise ValueError( "comm_id must be generated in paddlepaddle-xpu or paddlepaddle-xpu." ) - if sync: _add_sync_by_allreduce(block) + if sync: + _add_sync_by_allreduce(block) def _wait(self, current_endpoint, endpoints): - assert (self.wait_port) + assert self.wait_port other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) wait_server_ready(other_endpoints) @@ -208,20 +240,21 @@ def _broadcast_params(self): continue ring_id = (ring_id + 1) % self.nrings - block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - 'root': 0, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + 'root': 0, + OP_ROLE_KEY: OpRole.Forward, + }, + ) for ring_id in range(self.nrings): - block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: OpRole.Forward}, + ) diff --git a/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py index d25cf9680236f..15090f7826544 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py @@ -19,7 +19,6 @@ class DGCOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(DGCOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -28,11 +27,12 @@ def __init__(self, optimizer): self.meta_optimizers_white_list = [] self.meta_optimizers_black_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(DGCOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(DGCOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) def _init_dgc_opt(self): if self.dgc_opt is not None: @@ -62,7 +62,8 @@ def _init_dgc_opt(self): num_trainers=self.role_maker._worker_num(), regularization=opt.regularization, grad_clip=opt._grad_clip, - name=opt._name) + name=opt._name, + ) def _can_apply(self): if not self.role_maker._is_collective: @@ -88,15 +89,18 @@ def _enable_strategy(self, dist_strategy, context): dist_strategy.dgc = True dist_strategy.dgc_configs = {"rampup_begin_step": 0, "rampup_step": 1} - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): self._init_dgc_opt() - return self.dgc_opt.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + return self.dgc_opt.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) def apply_gradients(self, params_grads): self._init_dgc_opt() @@ -104,17 +108,15 @@ def apply_gradients(self, params_grads): def apply_optimize(self, loss, startup_program, params_grads): self._init_dgc_opt() - return self.dgc_opt.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) - - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + return self.dgc_opt.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) + + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): self._init_dgc_opt() - optimize_ops, params_grads = \ - self.dgc_opt.minimize(loss, startup_program, - parameter_list, no_grad_set) + optimize_ops, params_grads = self.dgc_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py index 958ebeb517d61..7e5f6983867e1 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py @@ -42,17 +42,26 @@ class DygraphShardingOptimizer(object): # 3. dynamic trainable params, which is the case bewteen pretraining and finetuning # 4. option to choose fuse comm (more GPU MEM need) or un-fuse comm - def __init__(self, hcg, user_defined_strategy, params, - inner_optimizer_class, **inner_optimizer_kargs): + def __init__( + self, + hcg, + user_defined_strategy, + params, + inner_optimizer_class, + **inner_optimizer_kargs + ): if not isinstance(params, list): raise TypeError( "`parameters` argument given to the DygraphShardingOptimizer should be " - "an iterable of paddle Tensors, but got argument type is `{}`.". - format(type(params))) + "an iterable of paddle Tensors, but got argument type is `{}`.".format( + type(params) + ) + ) self._parameter_list = params self._reference_is_trainable_params = list( - map(_is_trainable, self._parameter_list)) + map(_is_trainable, self._parameter_list) + ) self._inner_optimizer_class = inner_optimizer_class self._inner_optimizer_kargs = inner_optimizer_kargs @@ -102,8 +111,11 @@ def _partition_parameters(self): rank = sizes.index(min(sizes)) mapping[rank].append(param) numel = reduce(lambda x, y: x * y, param.shape) - assert numel > 0, "param [{}] should larger than 0, but it is [{}]".format( - param.name, numel) + assert ( + numel > 0 + ), "param [{}] should larger than 0, but it is [{}]".format( + param.name, numel + ) sizes[rank] += numel return mapping @@ -127,7 +139,8 @@ def _buid_inner_optimizer(self): # update related ops: clip, regular, opt self._inner_optimizer = self._inner_optimizer_class( parameters=self._rank2params[self._sharding_rank], - **self._inner_optimizer_kargs) + **self._inner_optimizer_kargs + ) def _sharding_sync_parameters(self): """ @@ -146,7 +159,8 @@ def _sharding_sync_parameters(self): # instead of the relative logic rank id within group src=self._hcg.get_sharding_parallel_group().ranks[rank], group=self._hcg.get_sharding_parallel_group(), - sync_op=True) + sync_op=True, + ) def _update_trainable(self): """ @@ -154,21 +168,23 @@ def _update_trainable(self): """ raise NotImplementedError - def minimize(self, - loss, - startup_program=None, - parameters=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameters=None, no_grad_set=None + ): # NOTE in dygraph mode, the only different between step and minimize is that minimize # allow user to customize the parameters for updating on each step input_param_names = set([param.name for param in parameters]) parameters = list( - filter(lambda x: x.name in input_param_names, - self._rank2params[self._sharding_rank])) - result = self._inner_optimizer.minimize(loss, startup_program, - parameters, no_grad_set) + filter( + lambda x: x.name in input_param_names, + self._rank2params[self._sharding_rank], + ) + ) + result = self._inner_optimizer.minimize( + loss, startup_program, parameters, no_grad_set + ) # sync parameters across sharding ranks self._sharding_sync_parameters() @@ -187,7 +203,9 @@ def step(self): # TODO is it a good way to make _grad_clip a property @property def _grad_clip(self): - assert self._inner_optimizer is not None, "inner opt of sharding is not initiliazed." + assert ( + self._inner_optimizer is not None + ), "inner opt of sharding is not initiliazed." return self._inner_optimizer._grad_clip def __getattr__(self, item): diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/heter_parallel_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/heter_parallel_optimizer.py index cda8c9e30cefa..a2a65d995ad7c 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/heter_parallel_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/heter_parallel_optimizer.py @@ -20,7 +20,8 @@ def _obtain_optimizer_parameters_list(optimizer): if getattr(optimizer, '_param_groups', None) and isinstance( - optimizer._param_groups[0], dict): + optimizer._param_groups[0], dict + ): parameters_list = [] for group in optimizer._param_groups: for param in group['params']: @@ -47,19 +48,19 @@ def step(self): self._inner_opt.step() @imperative_base.no_grad - def minimize(self, - loss, - startup_program=None, - parameters=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameters=None, no_grad_set=None + ): # minimize does not support parameters in the form of param_group, # so no need use _obtain_optimizer_parameters_list - parameter_list = parameters if parameters \ - else self._inner_opt._parameter_list + parameter_list = ( + parameters if parameters else self._inner_opt._parameter_list + ) - return self._inner_opt.minimize(loss, startup_program, parameter_list, - no_grad_set) + return self._inner_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) def __getattr__(self, item): return getattr(self._inner_opt, item) diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py index bd762a202f371..d768411dea5fe 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_gradscaler.py @@ -21,12 +21,12 @@ class HybridParallelGradScaler: - def __init__(self, scaler, hcg): self._scaler = scaler self._hcg = hcg self._use_dp_mode = ( - self._hcg.get_parallel_mode() == ParallelMode.DATA_PARALLEL) + self._hcg.get_parallel_mode() == ParallelMode.DATA_PARALLEL + ) def scale(self, var): return self._scaler.scale(var) @@ -56,20 +56,22 @@ def _unscale(self, optimizer): if not self._enable: return param_grads = [ - param._grad_ivar() for param in optimizer._parameter_list + param._grad_ivar() + for param in optimizer._parameter_list if param._grad_ivar() is not None ] - _legacy_C_ops.check_finite_and_unscale(param_grads, self._scale, - param_grads, self._found_inf) + _legacy_C_ops.check_finite_and_unscale( + param_grads, self._scale, param_grads, self._found_inf + ) # allreduce_max found_inf in check_group if not self._use_dp_mode: self._found_inf = paddle.cast(self._found_inf, dtype="int32") # TODO(shenliang03) Since the minimize call in the optimizer is # after the gradscaler, check_finite needs to synchronize global # information. In the future, we should use check_group - paddle.distributed.all_reduce(self._found_inf, - op=paddle.distributed.ReduceOp.MAX, - group=None) + paddle.distributed.all_reduce( + self._found_inf, op=paddle.distributed.ReduceOp.MAX, group=None + ) self._found_inf = paddle.cast(self._found_inf, dtype="bool") def __getattr__(self, item): diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py index 2cc43abee92aa..33922b7f35d9c 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py @@ -14,7 +14,10 @@ import paddle from paddle.fluid.clip import ClipGradByGlobalNorm -from ...utils.hybrid_parallel_util import fused_allreduce_gradients, sharding_reduce_gradients +from ...utils.hybrid_parallel_util import ( + fused_allreduce_gradients, + sharding_reduce_gradients, +) from ...base.topology import ParallelMode from paddle.fluid.dygraph import base as imperative_base from paddle.fluid import framework @@ -27,7 +30,8 @@ def _obtain_optimizer_parameters_list(optimizer): if getattr(optimizer, '_param_groups', None) and isinstance( - optimizer._param_groups[0], dict): + optimizer._param_groups[0], dict + ): parameters_list = [] for group in optimizer._param_groups: for param in group['params']: @@ -39,7 +43,6 @@ def _obtain_optimizer_parameters_list(optimizer): class HybridParallelClipGrad: - def __init__(self, clip, hcg): self._clip = clip self._hcg = hcg @@ -65,7 +68,8 @@ def _dygraph_clip(self, params_grads): not_shared_enable = (not hasattr(p, 'is_firstly_shared')) or ( hasattr(p, 'is_firstly_shared') - and getattr(p, 'is_firstly_shared', True)) + and getattr(p, 'is_firstly_shared', True) + ) if not_shared_enable: if p.is_distributed: @@ -81,69 +85,83 @@ def _dygraph_clip(self, params_grads): # global norm of distributed FP16 params_and_grads if len(sum_square_dist_fp16) == 0: - global_norm_dist_fp16 = paddle.to_tensor([0.], dtype=paddle.float32) + global_norm_dist_fp16 = paddle.to_tensor( + [0.0], dtype=paddle.float32 + ) else: global_norm_dist_fp16 = layers.concat(sum_square_dist_fp16) global_norm_dist_fp16 = layers.reduce_sum(global_norm_dist_fp16) - global_norm_dist_fp16 = paddle.cast(global_norm_dist_fp16, - dtype=paddle.float32) + global_norm_dist_fp16 = paddle.cast( + global_norm_dist_fp16, dtype=paddle.float32 + ) # global norm of non-distributed FP16 params_and_grads if len(sum_square_not_dist_fp16) == 0: - global_norm_not_dist_fp16 = paddle.to_tensor([0.], - dtype=paddle.float32) + global_norm_not_dist_fp16 = paddle.to_tensor( + [0.0], dtype=paddle.float32 + ) else: global_norm_not_dist_fp16 = layers.concat(sum_square_not_dist_fp16) global_norm_not_dist_fp16 = layers.reduce_sum( - global_norm_not_dist_fp16) - global_norm_not_dist_fp16 = paddle.cast(global_norm_not_dist_fp16, - dtype=paddle.float32) + global_norm_not_dist_fp16 + ) + global_norm_not_dist_fp16 = paddle.cast( + global_norm_not_dist_fp16, dtype=paddle.float32 + ) # global norm of distributed FP32 params_and_grads - global_norm_dist_fp32 = layers.concat(sum_square_dist_fp32) if len( - sum_square_dist_fp32) != 0 else paddle.to_tensor( - [0.], dtype=paddle.float32) + global_norm_dist_fp32 = ( + layers.concat(sum_square_dist_fp32) + if len(sum_square_dist_fp32) != 0 + else paddle.to_tensor([0.0], dtype=paddle.float32) + ) global_norm_dist_fp32 = layers.reduce_sum(global_norm_dist_fp32) # global norm of non-distributed FP32 params_and_grads - global_norm_not_dist_fp32 = layers.concat( - sum_square_not_dist_fp32 - ) if len(sum_square_not_dist_fp32) != 0 else paddle.to_tensor( - [0.], dtype=paddle.float32) + global_norm_not_dist_fp32 = ( + layers.concat(sum_square_not_dist_fp32) + if len(sum_square_not_dist_fp32) != 0 + else paddle.to_tensor([0.0], dtype=paddle.float32) + ) global_norm_not_dist_fp32 = layers.reduce_sum(global_norm_not_dist_fp32) global_norm_var_dist = global_norm_dist_fp16 + global_norm_dist_fp32 - global_norm_var_not_dist = global_norm_not_dist_fp16 + global_norm_not_dist_fp32 + global_norm_var_not_dist = ( + global_norm_not_dist_fp16 + global_norm_not_dist_fp32 + ) # add all reduce to get global norm of distributed params_and_grads if self._hcg.get_model_parallel_world_size() > 1: paddle.distributed.all_reduce( - global_norm_var_dist, - group=self._hcg.get_check_parallel_group()) + global_norm_var_dist, group=self._hcg.get_check_parallel_group() + ) # add all reduce to get global norm of non-distributed params_and_grads in groups of pp if self._hcg.get_pipe_parallel_world_size() > 1: paddle.distributed.all_reduce( global_norm_var_not_dist, - group=self._hcg.get_pipe_parallel_group()) + group=self._hcg.get_pipe_parallel_group(), + ) # In Sharding mode, param and grad is mapping different rank in optimizer. # ClipGradByGlobalNorm need allreduce to get globol norm if self._hcg.get_sharding_parallel_world_size() > 1: paddle.distributed.all_reduce( global_norm_var_not_dist, - group=self._hcg.get_sharding_parallel_group()) - - global_norm_var_fp32 = layers.sqrt(global_norm_var_dist + - global_norm_var_not_dist) - - max_global_norm = layers.fill_constant(shape=[1], - dtype=global_norm_var_fp32.dtype, - value=self.clip_norm) - clip_var = layers.elementwise_div(x=max_global_norm, - y=layers.elementwise_max( - x=global_norm_var_fp32, - y=max_global_norm)) + group=self._hcg.get_sharding_parallel_group(), + ) + + global_norm_var_fp32 = layers.sqrt( + global_norm_var_dist + global_norm_var_not_dist + ) + + max_global_norm = layers.fill_constant( + shape=[1], dtype=global_norm_var_fp32.dtype, value=self.clip_norm + ) + clip_var = layers.elementwise_div( + x=max_global_norm, + y=layers.elementwise_max(x=global_norm_var_fp32, y=max_global_norm), + ) clip_var_fp16 = paddle.cast(clip_var, paddle.float16) for p, g in params_grads: if g is None: @@ -173,35 +191,43 @@ def __init__(self, optimizer, hcg, strategy): self._hcg = hcg self._use_dp_mode = ( - self._hcg.get_parallel_mode() == ParallelMode.DATA_PARALLEL) + self._hcg.get_parallel_mode() == ParallelMode.DATA_PARALLEL + ) - self._need_dp = (self._hcg.get_data_parallel_world_size() > 1) + self._need_dp = self._hcg.get_data_parallel_world_size() > 1 # NOTE(shenliang03): Because of the pure DataParallel mode, the gradient synchronization # is achieved through reducer, so there is no need to call fuse_allreduce in optimizer. self._dp_enable = not self._use_dp_mode and self._need_dp - self._sharding_enable = (self._hcg.get_sharding_parallel_world_size() > - 1) + self._sharding_enable = self._hcg.get_sharding_parallel_world_size() > 1 - if isinstance(self._inner_opt._grad_clip, - ClipGradByGlobalNorm) and not self._use_dp_mode: - logger.warning("While using ClipGradByGlobalNorm in TensorParallel, PipelineParallel " \ - "or Sharding, the grad clip of original optimizer will be changed.") + if ( + isinstance(self._inner_opt._grad_clip, ClipGradByGlobalNorm) + and not self._use_dp_mode + ): + logger.warning( + "While using ClipGradByGlobalNorm in TensorParallel, PipelineParallel " + "or Sharding, the grad clip of original optimizer will be changed." + ) if self._sharding_enable: # change sharding inner_optimizer's _grad_clip - self._inner_opt._inner_optimizer._grad_clip = HybridParallelClipGrad( - self._inner_opt._grad_clip, hcg) + self._inner_opt._inner_optimizer._grad_clip = ( + HybridParallelClipGrad(self._inner_opt._grad_clip, hcg) + ) else: self._inner_opt._grad_clip = HybridParallelClipGrad( - self._inner_opt._grad_clip, hcg) + self._inner_opt._grad_clip, hcg + ) if self._inner_opt._parameter_list and isinstance( - self._inner_opt._parameter_list[0], dict): + self._inner_opt._parameter_list[0], dict + ): for item in self._inner_opt._param_groups: if "grad_clip" in item.keys(): item["grad_clip"] = HybridParallelClipGrad( - self._inner_opt._grad_clip, hcg) + self._inner_opt._grad_clip, hcg + ) @imperative_base.no_grad @framework.dygraph_only @@ -216,16 +242,15 @@ def step(self): self._inner_opt.step() @imperative_base.no_grad - def minimize(self, - loss, - startup_program=None, - parameters=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameters=None, no_grad_set=None + ): # minimize does not support parameters in the form of param_group, # so no need use _obtain_optimizer_parameters_list - parameter_list = parameters if parameters \ - else self._inner_opt._parameter_list + parameter_list = ( + parameters if parameters else self._inner_opt._parameter_list + ) # Here sharding should use global parameter list if self._sharding_enable: @@ -234,8 +259,9 @@ def minimize(self, if self._dp_enable: fused_allreduce_gradients(list(parameter_list), self._hcg) - return self._inner_opt.minimize(loss, startup_program, parameter_list, - no_grad_set) + return self._inner_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) def __getattr__(self, item): return getattr(self._inner_opt, item) diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/sharding_optimizer_stage2.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/sharding_optimizer_stage2.py index 3468ec7a3a7b8..ed1435209540c 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/sharding_optimizer_stage2.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/sharding_optimizer_stage2.py @@ -30,10 +30,19 @@ from paddle.fluid import core from paddle.optimizer import Optimizer from paddle.fluid.clip import ClipGradByGlobalNorm -from paddle.distributed.collective import _get_global_group, new_group, broadcast, wait +from paddle.distributed.collective import ( + _get_global_group, + new_group, + broadcast, + wait, +) from ...utils.internal_storage import ParamStorage, GradStorage -from ...meta_parallel.sharding.sharding_utils import Type, device_guard, ShardingClipGrad +from ...meta_parallel.sharding.sharding_utils import ( + Type, + device_guard, + ShardingClipGrad, +) # CUDA alignment 256 bytes, cpu alignment 4096 bytes alignment = {"gpu": 256, "cpu": 4096} @@ -61,19 +70,22 @@ class ShardingOptimizerStage2(Optimizer): # 4. Support offload function. # 5. Support the establishment of independent communication groups. # 6. Broadcast_fp16 is not supported now. - def __init__(self, - params, - optim, - group=None, - offload=False, - device="gpu", - pertrain_sync_models=True, - **kw): + def __init__( + self, + params, + optim, + group=None, + offload=False, + device="gpu", + pertrain_sync_models=True, + **kw + ): super().__init__(optim._learning_rate, params, kw) # Segmentation information - self._dtype_rank_params = OrderedDict( + self._dtype_rank_params = ( + OrderedDict() ) # {dtype:[param1,param2]} device, rank, params self._param2rank = {} self.__segment_params = [] @@ -84,17 +96,26 @@ def __init__(self, self._optim_defaults = kw self._optim = optim - assert hasattr(self._optim, "_master_weights" - ), "Must use optimizer with _master_weights attribute" + assert hasattr( + self._optim, "_master_weights" + ), "Must use optimizer with _master_weights attribute" self._local_params = params self._default_device = device - self._pfp16 = len( - list( - filter(lambda x: x.trainable and x.dtype == Type.fp16.value, - self._local_params))) > 0 + self._pfp16 = ( + len( + list( + filter( + lambda x: x.trainable and x.dtype == Type.fp16.value, + self._local_params, + ) + ) + ) + > 0 + ) - self.group = new_group( - _get_global_group().ranks) if group is None else group + self.group = ( + new_group(_get_global_group().ranks) if group is None else group + ) self.world_size = self.group.nranks self.rank = self.group.rank @@ -110,19 +131,24 @@ def __init__(self, logging.warning( "While using ClipGradByGlobalNorm in ShardingOptimizer, the grad clip of original optimizer will be changed." ) - self._optim._grad_clip = ShardingClipGrad(self._optim._grad_clip, - paddle.get_device(), - self.group) + self._optim._grad_clip = ShardingClipGrad( + self._optim._grad_clip, paddle.get_device(), self.group + ) if self._optim._parameter_list and isinstance( - self._optim._parameter_list[0], dict): + self._optim._parameter_list[0], dict + ): for item in self._optim._param_groups: if "grad_clip" in item.keys(): item["grad_clip"] = ShardingClipGrad( - self._optim._grad_clip, paddle.get_device(), - self.group) + self._optim._grad_clip, + paddle.get_device(), + self.group, + ) if offload: - assert self._pfp16, "Only support offload strategy while using \'Adam\', \'AdamW\' and \'Momentum\' optimizer with AMP/Pure FP16" + assert ( + self._pfp16 + ), "Only support offload strategy while using \'Adam\', \'AdamW\' and \'Momentum\' optimizer with AMP/Pure FP16" self.offload = offload # Using for offload self.offload_device = "cpu" @@ -143,10 +169,9 @@ def _sync_params_and_buffers(self): """ for p in self._local_params: - broadcast(p, - src=self._global_root_rank, - group=self.group, - sync_op=True) + broadcast( + p, src=self._global_root_rank, group=self.group, sync_op=True + ) # Multi stream operation will be supported later wait(tensor=p, group=self.group, use_calc_stream=True) @@ -159,16 +184,17 @@ def _generate_master_params(self, trainable_params): name=param.name, value=param.cast(dtype=Type.fp32.value).numpy(), place=core.CPUPlace(), - stop_gradient=param.stop_gradient) + stop_gradient=param.stop_gradient, + ) else: for param in trainable_params: if param.dtype == Type.fp16.value: self._optim._master_weights[param.name] = paddle.cast( - param, Type.fp32.value) + param, Type.fp32.value + ) def _update_opt_status(self): - """Update optimizer status and parameter storage information, and special functions to be developed. - """ + """Update optimizer status and parameter storage information, and special functions to be developed.""" # func 1 self._integration_params() @@ -222,8 +248,9 @@ def dtype_rank_params(self): self._dtype_rank_params[param.dtype] = [ [] for _ in range(self.world_size) ] - self._dtype_rank_params[param.dtype][self.param2rank[ - param.name]].append(param) + self._dtype_rank_params[param.dtype][ + self.param2rank[param.name] + ].append(param) # Sort per rank params by size for dtype in self._dtype_rank_params.keys(): @@ -243,7 +270,8 @@ def rank_buffer_size(self): if dtype not in self._rank_buffer_size.keys(): self._rank_buffer_size[dtype] = {} for dst_rank, per_rank_params in enumerate( - self.dtype_rank_params[dtype]): + self.dtype_rank_params[dtype] + ): if dst_rank not in self._rank_buffer_size[dtype].keys(): self._rank_buffer_size[dtype][dst_rank] = 0 for param in per_rank_params: @@ -251,11 +279,15 @@ def rank_buffer_size(self): continue size = np.prod(param.shape) * align[dtype] remaining = size % alignment[self._default_device] - ali = 0 if remaining == 0 else alignment[ - self._default_device] - remaining + ali = ( + 0 + if remaining == 0 + else alignment[self._default_device] - remaining + ) align_ = ali // align[dtype] - self._rank_buffer_size[dtype][dst_rank] += np.prod( - param.shape) + align_ + self._rank_buffer_size[dtype][dst_rank] += ( + np.prod(param.shape) + align_ + ) self._param2align[param.name] = align_ return self._rank_buffer_size @@ -274,23 +306,27 @@ def _integration_params(self): # Merge all the trainable params in a single InternalStorage trainable_params = list( - filter(lambda x: x.trainable, params)) + filter(lambda x: x.trainable, params) + ) if self._pfp16 and dst_rank == self.rank: self._generate_master_params(trainable_params) if trainable_params: param_storage = ParamStorage( size=self.rank_buffer_size[dtype][dst_rank], dtype=dtype, - device=self._default_device) + device=self._default_device, + ) - param_storage.add_rank_params(trainable_params, - self._param2align) + param_storage.add_rank_params( + trainable_params, self._param2align + ) self.param_storages[dtype][dst_rank] = param_storage # Clear the InternalStorage keys which are not in use anymore dtype_in_use = list(self.dtype_rank_params.keys()) dtype_to_pop = list( - filter(lambda x: x not in dtype_in_use, self.param_storages.keys())) + filter(lambda x: x not in dtype_in_use, self.param_storages.keys()) + ) for d in dtype_to_pop: self.param_storages.pop(d) @@ -300,8 +336,11 @@ def _integration_params(self): for param in cpu_master_params: size = np.prod(param.shape) * align[Type.fp32.value] remaining = size % alignment[self.offload_device] - ali = 0 if remaining == 0 else alignment[ - self.offload_device] - remaining + ali = ( + 0 + if remaining == 0 + else alignment[self.offload_device] - remaining + ) align_ = ali // align[Type.fp32.value] self.offload_buffer_size += np.prod(param.shape) + align_ self.offload_param2align[param.name] = align_ @@ -311,9 +350,11 @@ def _integration_params(self): self.offload_params = ParamStorage( size=self.offload_buffer_size, dtype=Type.fp32.value, - device=self.offload_device) + device=self.offload_device, + ) self.offload_params.add_rank_params( - cpu_master_params, self.offload_param2align, False) + cpu_master_params, self.offload_param2align, False + ) self.offload_params.buffer.stop_gradient = False self.offload_grads = GradStorage( @@ -322,14 +363,16 @@ def _integration_params(self): device=self.offload_device, destination=self.rank, parm2align=self.offload_param2align, - convert_cpu=True) + convert_cpu=True, + ) for p in cpu_master_params: self.offload_grads.add_grad( - p, self.offload_param2align[p.name]) + p, self.offload_param2align[p.name] + ) self._optim._master_weights[ - self.offload_params.buffer. - name] = self.offload_params.buffer + self.offload_params.buffer.name + ] = self.offload_params.buffer def _offload_acc_grad(self, param_name, grad_fp32_cpu): """accumulate grads with offload strategy""" @@ -337,12 +380,14 @@ def _offload_acc_grad(self, param_name, grad_fp32_cpu): if param_name in self._master_params.keys(): if self._master_params[param_name].grad is None: self._master_params[param_name]._copy_gradient_from( - grad_fp32_cpu) + grad_fp32_cpu + ) else: self._master_params[param_name].grad.add_(grad_fp32_cpu) self.offload_params.buffer._copy_gradient_from( - self.offload_grads.buffer) + self.offload_grads.buffer + ) def _offload_scale_grad(self, scale_size): """scale grads with offload strategy""" @@ -362,7 +407,7 @@ def step(self): if self.offload: params_list = [self.offload_params.buffer] - #TODO(Baibaifan): Offload will support param_groups later + # TODO(Baibaifan): Offload will support param_groups later if not isinstance(self._optim._param_groups[0], dict): self._optim._parameter_list = params_list self._optim._param_groups = params_list @@ -376,8 +421,10 @@ def step(self): for param in self._local_params: if param.name in self._master_params.keys(): param.set_value( - self._master_params[param.name].cuda(dev_id).cast( - dtype=param.dtype)) + self._master_params[param.name] + .cuda(dev_id) + .cast(dtype=param.dtype) + ) else: self._optim.step() @@ -386,7 +433,8 @@ def step(self): def minimize(self): raise RuntimeError( - "optimizer.minimize() not support now, please use optimizer.step()") + "optimizer.minimize() not support now, please use optimizer.step()" + ) def set_state_dict(self, state_dict): self._optim.set_state_dict(state_dict) @@ -408,12 +456,16 @@ def _broadcast_params(self): # Exchange all the shards with the other ranks for dtype_per_rank in self.param_storages.values(): for dst_rank, internal_storage in dtype_per_rank.items(): - broadcast(tensor=internal_storage.buffer, - src=self.group.ranks[dst_rank], - group=self.group, - sync_op=True) + broadcast( + tensor=internal_storage.buffer, + src=self.group.ranks[dst_rank], + group=self.group, + sync_op=True, + ) # Multi stream operation will be supported later - wait(tensor=internal_storage.buffer, - group=self.group, - use_calc_stream=True) + wait( + tensor=internal_storage.buffer, + group=self.group, + use_calc_stream=True, + ) diff --git a/python/paddle/distributed/fleet/meta_optimizers/fp16_allreduce_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/fp16_allreduce_optimizer.py index 93857461b26d2..3834f85bffaeb 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/fp16_allreduce_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/fp16_allreduce_optimizer.py @@ -18,7 +18,6 @@ class FP16AllReduceOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(FP16AllReduceOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -34,11 +33,12 @@ def __init__(self, optimizer): ] self.meta_optimizers_black_list = ["DGCOptimizer"] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(FP16AllReduceOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(FP16AllReduceOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) def _can_apply(self): if not self.role_maker._is_collective: @@ -84,28 +84,30 @@ def fp16_compression(param_and_grads): else: op._remove_attr(op_maker.kOpRoleVarAttrName()) - new_grad = block.create_var(name=unique_name.generate(grad.name + - ".cast_fp16"), - dtype=core.VarDesc.VarType.FP16, - persistable=False, - stop_gradient=True) + new_grad = block.create_var( + name=unique_name.generate(grad.name + ".cast_fp16"), + dtype=core.VarDesc.VarType.FP16, + persistable=False, + stop_gradient=True, + ) with block.program._backward_role_guard(): - cast_op = block.append_op(type="cast", - inputs={"X": grad}, - outputs={"Out": new_grad}, - attrs={ - "in_dtype": - core.VarDesc.VarType.FP32, - "out_dtype": - core.VarDesc.VarType.FP16 - }, - stop_gradient=True) + cast_op = block.append_op( + type="cast", + inputs={"X": grad}, + outputs={"Out": new_grad}, + attrs={ + "in_dtype": core.VarDesc.VarType.FP32, + "out_dtype": core.VarDesc.VarType.FP16, + }, + stop_gradient=True, + ) backward = op_maker.OpRole.Backward cast_op._set_attr(op_maker.kOpRoleAttrName(), backward) - cast_op._set_attr(op_maker.kOpRoleVarAttrName(), - [param.name, new_grad.name]) + cast_op._set_attr( + op_maker.kOpRoleVarAttrName(), [param.name, new_grad.name] + ) new_grad.op = cast_op new_param_and_grads.append((param, new_grad, True)) @@ -122,30 +124,32 @@ def fp16_compression(param_and_grads): continue block = grad.block - new_grad = block.create_var(name=unique_name.generate(grad.name + - ".cast_fp32"), - dtype=core.VarDesc.VarType.FP32, - persistable=False, - stop_gradient=True) + new_grad = block.create_var( + name=unique_name.generate(grad.name + ".cast_fp32"), + dtype=core.VarDesc.VarType.FP32, + persistable=False, + stop_gradient=True, + ) with block.program._optimized_guard( - [param, grad]), framework.name_scope('fp16_allreduce'): - cast_op = block.append_op(type="cast", - inputs={"X": grad}, - outputs={"Out": new_grad}, - attrs={ - "in_dtype": - core.VarDesc.VarType.FP16, - "out_dtype": - core.VarDesc.VarType.FP32 - }, - stop_gradient=True) + [param, grad] + ), framework.name_scope('fp16_allreduce'): + cast_op = block.append_op( + type="cast", + inputs={"X": grad}, + outputs={"Out": new_grad}, + attrs={ + "in_dtype": core.VarDesc.VarType.FP16, + "out_dtype": core.VarDesc.VarType.FP32, + }, + stop_gradient=True, + ) ret_param_and_grads.append((param, new_grad)) return ret_param_and_grads def apply_optimize(self, loss, startup_program, params_grads): new_params_grads = self.fp16_compression(params_grads) - return self.inner_opt.apply_optimize(loss, - startup_program=startup_program, - params_grads=new_params_grads) + return self.inner_opt.apply_optimize( + loss, startup_program=startup_program, params_grads=new_params_grads + ) diff --git a/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py index 10175f8936a70..eb97122587f36 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py @@ -18,7 +18,6 @@ class GradientMergeOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(GradientMergeOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -32,26 +31,30 @@ def __init__(self, optimizer): ] self.meta_optimizers_black_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(GradientMergeOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(GradientMergeOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) def _init_wrapped_opt(self): config = self.user_defined_strategy.gradient_merge_configs self.wrapped_opt = GM(self.inner_opt) self.wrapped_opt._set_k_steps( - self.user_defined_strategy.gradient_merge_configs["k_steps"]) + self.user_defined_strategy.gradient_merge_configs["k_steps"] + ) self.wrapped_opt._set_avg( - self.user_defined_strategy.gradient_merge_configs["avg"]) + self.user_defined_strategy.gradient_merge_configs["avg"] + ) def _can_apply(self): if not self.role_maker._is_collective: return False - can_apply = (self.user_defined_strategy.gradient_merge == True) and \ - self.user_defined_strategy.gradient_merge_configs["k_steps"] > 1 + can_apply = ( + self.user_defined_strategy.gradient_merge == True + ) and self.user_defined_strategy.gradient_merge_configs["k_steps"] > 1 return can_apply def _disable_strategy(self, dist_strategy): @@ -62,13 +65,11 @@ def _enable_strategy(self, dist_strategy, context): # we currently do not support auto-enable GradientMerge return - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): self._init_wrapped_opt() - optimize_ops, params_grads = \ - self.wrapped_opt.minimize(loss, startup_program, - parameter_list, no_grad_set) + optimize_ops, params_grads = self.wrapped_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py index a5b0856a66ff4..dd2ccfc7ff7d2 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py @@ -24,7 +24,6 @@ class GraphExecutionOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(GraphExecutionOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -45,12 +44,14 @@ def _can_apply(self): return False return not self.user_defined_strategy.without_graph_optimization - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): pass # should fix the variable @@ -69,75 +70,77 @@ def _setup_nccl_op(self, startup_program, main_program, build_strategy): if trainer_id == 0 and not paddle.is_compiled_with_npu(): wait_server_ready(other_trainers) - if build_strategy.reduce_strategy == BuildStrategy.ReduceStrategy._NoReduce: + if ( + build_strategy.reduce_strategy + == BuildStrategy.ReduceStrategy._NoReduce + ): return if core.is_compiled_with_cuda(): comm_id_var = startup_program.global_block().create_var( - name="NCCLID", persistable=True, type=core.VarDesc.VarType.RAW) + name="NCCLID", persistable=True, type=core.VarDesc.VarType.RAW + ) for i in range(1, build_strategy.nccl_comm_num): startup_program.global_block().create_var( name="NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) if build_strategy.use_hierarchical_allreduce: for i in range(0, build_strategy.nccl_comm_num): startup_program.global_block().create_var( name="Hierarchical_inter_NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().create_var( name="Hierarchical_exter_NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().append_op( type="gen_nccl_id", inputs={}, outputs={"NCCLID": comm_id_var}, attrs={ - "trainers": - trainer_endpoints, - "trainer_id": - trainer_id, - "nccl_comm_num": - build_strategy.nccl_comm_num, - "use_hierarchical_allreduce": - build_strategy.use_hierarchical_allreduce, - "hierarchical_allreduce_inter_ranks": - build_strategy.hierarchical_allreduce_inter_nranks - }) + "trainers": trainer_endpoints, + "trainer_id": trainer_id, + "nccl_comm_num": build_strategy.nccl_comm_num, + "use_hierarchical_allreduce": build_strategy.use_hierarchical_allreduce, + "hierarchical_allreduce_inter_ranks": build_strategy.hierarchical_allreduce_inter_nranks, + }, + ) elif core.is_compiled_with_xpu(): comm_id_var = startup_program.global_block().create_var( - name="BKCLID", persistable=True, type=core.VarDesc.VarType.RAW) + name="BKCLID", persistable=True, type=core.VarDesc.VarType.RAW + ) - #NOTE(liuyuhui) Baidu Kunlun Communication Library(BKCL) currently do not support multi machines. - assert build_strategy.bkcl_comm_num == 1, \ - "Baidu Kunlun Communication Library(BKCL) currently do not support multi machines." + # NOTE(liuyuhui) Baidu Kunlun Communication Library(BKCL) currently do not support multi machines. + assert ( + build_strategy.bkcl_comm_num == 1 + ), "Baidu Kunlun Communication Library(BKCL) currently do not support multi machines." for i in range(1, build_strategy.bkcl_comm_num): startup_program.global_block().create_var( name="BKCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().append_op( type="gen_bkcl_id", inputs={}, outputs={"BKCLID": comm_id_var}, attrs={ - "trainers": - trainer_endpoints, - "trainer_id": - trainer_id, - "nccl_comm_num": - build_strategy.nccl_comm_num, - "use_hierarchical_allreduce": - build_strategy.use_hierarchical_allreduce, - "hierarchical_allreduce_inter_ranks": - build_strategy.hierarchical_allreduce_inter_nranks - }) + "trainers": trainer_endpoints, + "trainer_id": trainer_id, + "nccl_comm_num": build_strategy.nccl_comm_num, + "use_hierarchical_allreduce": build_strategy.use_hierarchical_allreduce, + "hierarchical_allreduce_inter_ranks": build_strategy.hierarchical_allreduce_inter_nranks, + }, + ) else: raise ValueError( "comm_id must be generated in paddlepaddle-xpu or paddlepaddle-gpu." @@ -147,27 +150,32 @@ def _try_to_compile(self, startup_program, main_program, loss): dist_strategy = self.user_defined_strategy local_build_strategy = dist_strategy.build_strategy - local_build_strategy.use_hierarchical_allreduce = \ + local_build_strategy.use_hierarchical_allreduce = ( dist_strategy.use_hierarchical_allreduce - local_build_strategy.hierarchical_allreduce_inter_nranks = \ + ) + local_build_strategy.hierarchical_allreduce_inter_nranks = ( dist_strategy.hierarchical_allreduce_inter_nranks - local_build_strategy.sync_batch_norm = \ - dist_strategy.sync_batch_norm - local_build_strategy.fuse_all_reduce_ops = \ + ) + local_build_strategy.sync_batch_norm = dist_strategy.sync_batch_norm + local_build_strategy.fuse_all_reduce_ops = ( dist_strategy.fuse_all_reduce_ops - local_build_strategy.nccl_comm_num = \ - dist_strategy.nccl_comm_num + ) + local_build_strategy.nccl_comm_num = dist_strategy.nccl_comm_num - gradient_scale_configs = self.user_defined_strategy.gradient_scale_configs + gradient_scale_configs = ( + self.user_defined_strategy.gradient_scale_configs + ) scale_strategys = { 'avg': BuildStrategy.GradientScaleStrategy.CoeffNumDevice, 'sum': BuildStrategy.GradientScaleStrategy.One, 'customized': BuildStrategy.GradientScaleStrategy.Customized, } - assert gradient_scale_configs['scale_strategy'] in scale_strategys, \ - "gradient_scale_configs.scale_strategy must be 'avg', 'sum' or 'customized'" - local_build_strategy.gradient_scale_strategy = \ - scale_strategys[gradient_scale_configs['scale_strategy']] + assert ( + gradient_scale_configs['scale_strategy'] in scale_strategys + ), "gradient_scale_configs.scale_strategy must be 'avg', 'sum' or 'customized'" + local_build_strategy.gradient_scale_strategy = scale_strategys[ + gradient_scale_configs['scale_strategy'] + ] if self.user_defined_strategy.recompute == True: logging.warn( @@ -180,7 +188,9 @@ def _try_to_compile(self, startup_program, main_program, loss): node_num = self.role_maker._node_num() if self.role_maker._is_collective: - assert worker_num >= 1, "nccl2 worker_num must >= 1, now:{}" % worker_num + assert worker_num >= 1, ( + "nccl2 worker_num must >= 1, now:{}" % worker_num + ) if worker_num <= 1: # local mode @@ -198,8 +208,8 @@ def _try_to_compile(self, startup_program, main_program, loss): sync_allreduce = dist_strategy.sync_nccl_allreduce if sync_allreduce: exe_strategy.num_threads = max( - local_build_strategy.nccl_comm_num + 1, - exe_strategy.num_threads) + local_build_strategy.nccl_comm_num + 1, exe_strategy.num_threads + ) if local_build_strategy.nccl_comm_num > 1: logging.warn( "nccl_comm_num > 1, you may need to set sync_nccl_allreduce=False to ensure that different nccl comms can overlap" @@ -217,17 +227,23 @@ def _try_to_compile(self, startup_program, main_program, loss): # NOTE. compatible with compiler, otherwise these values will be overwritten by compiler main_program._nccl_comm_num = local_build_strategy.nccl_comm_num - main_program._use_hierarchical_allreduce = local_build_strategy.use_hierarchical_allreduce - main_program._hierarchical_allreduce_inter_nranks = local_build_strategy.hierarchical_allreduce_inter_nranks + main_program._use_hierarchical_allreduce = ( + local_build_strategy.use_hierarchical_allreduce + ) + main_program._hierarchical_allreduce_inter_nranks = ( + local_build_strategy.hierarchical_allreduce_inter_nranks + ) # TODO(guru4elephant): should be an independent optimizer if worker_num > 1: - self._setup_nccl_op(startup_program, main_program, - local_build_strategy) + self._setup_nccl_op( + startup_program, main_program, local_build_strategy + ) local_build_strategy.num_trainers = self.role_maker._worker_num() local_build_strategy.trainer_id = self.role_maker._worker_index() - local_build_strategy.trainers_endpoints = self.role_maker._get_trainer_endpoints( + local_build_strategy.trainers_endpoints = ( + self.role_maker._get_trainer_endpoints() ) local_build_strategy.enable_backward_optimizer_op_deps = True @@ -237,7 +253,8 @@ def _try_to_compile(self, startup_program, main_program, loss): loss_name=loss.name, build_strategy=local_build_strategy, exec_strategy=exe_strategy, - share_vars_from=None) + share_vars_from=None, + ) return self._compiled_program @@ -249,15 +266,14 @@ def _enable_strategy(self, dist_strategy, context): # by default, graph execution strategy is enabled return - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): if startup_program == None: startup_program = paddle.static.default_startup_program() - compiled_program = self._try_to_compile(startup_program, - loss.block.program, loss) + compiled_program = self._try_to_compile( + startup_program, loss.block.program, loss + ) loss.block.program._graph = compiled_program # just return self.optimizer_ops and self.param_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py index 3dc5bed03aeac..22f225469c2fc 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py @@ -20,7 +20,6 @@ class LambOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(LambOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -29,11 +28,12 @@ def __init__(self, optimizer): self.meta_optimizers_white_list = ["GraphExecutionOptimizer"] self.meta_optimizers_black_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(LambOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(LambOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) opt = self.inner_opt if not isinstance(opt, AdamOptimizer): @@ -63,7 +63,8 @@ def exclude_fn(param): regularization=opt.regularization, grad_clip=opt._grad_clip, exclude_from_weight_decay_fn=_exclude_from_weight_decay_fn, - name=opt._name) + name=opt._name, + ) def _can_apply(self): if not self.role_maker._is_collective: @@ -72,8 +73,10 @@ def _can_apply(self): if self.user_defined_strategy.lamb: if not isinstance(self.inner_opt, AdamOptimizer): logging.warn( - "lamb need the inner optimizer to be AdamOptimizer optimizer but got {}." - .format(self.inner_opt.type)) + "lamb need the inner optimizer to be AdamOptimizer optimizer but got {}.".format( + self.inner_opt.type + ) + ) return False return True return False @@ -86,33 +89,34 @@ def _enable_strategy(self, dist_strategy, context): dist_strategy.lamb = True dist_strategy.lamb_configs = { "lamb_weight_decay": 0.01, - "exclude_from_weight_decay": [] + "exclude_from_weight_decay": [], } - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): - return self.lamb_opt.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): + return self.lamb_opt.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) # the following function will be used by AMP if both LARS and AMP are turn on together. def apply_gradients(self, params_grads): return self.lamb_opt.apply_gradients(params_grads=params_grads) def apply_optimize(self, loss, startup_program, params_grads): - return self.lamb_opt.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) - - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - optimize_ops, params_grads = \ - self.lamb_opt.minimize(loss, startup_program, - parameter_list, no_grad_set) + return self.lamb_opt.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) + + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + optimize_ops, params_grads = self.lamb_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py index 44f8fe473e2f9..e584783581e4a 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py @@ -19,7 +19,6 @@ class LarsOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(LarsOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -28,11 +27,12 @@ def __init__(self, optimizer): self.meta_optimizers_white_list = ["GraphExecutionOptimizer"] self.meta_optimizers_black_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(LarsOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(LarsOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) opt = self.inner_opt if not isinstance(opt, Momentum): @@ -50,7 +50,8 @@ def _set_basic_info(self, loss, role_maker, user_defined_optimizer, grad_clip=opt._grad_clip, name=opt._name, exclude_from_weight_decay=configs['exclude_from_weight_decay'], - epsilon=configs['epsilon']) + epsilon=configs['epsilon'], + ) def _can_apply(self): if not self.role_maker._is_collective: @@ -59,8 +60,10 @@ def _can_apply(self): if self.user_defined_strategy.lars: if not isinstance(self.inner_opt, Momentum): logging.warn( - "lars need the inner optimizer to be Momentum optimizer but got {}." - .format(self.inner_opt.type)) + "lars need the inner optimizer to be Momentum optimizer but got {}.".format( + self.inner_opt.type + ) + ) return False return True return False @@ -76,30 +79,31 @@ def _enable_strategy(self, dist_strategy, context): "lars_weight_decay": 0.0005, } - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): - return self.lars_opt.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): + return self.lars_opt.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) # the following function will be used by AMP if both LARS and AMP are turn on together. def apply_gradients(self, params_grads): return self.lars_opt.apply_gradients(params_grads=params_grads) def apply_optimize(self, loss, startup_program, params_grads): - return self.lars_opt.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) - - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - optimize_ops, params_grads = \ - self.lars_opt.minimize(loss, startup_program, - parameter_list, no_grad_set) + return self.lars_opt.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) + + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + optimize_ops, params_grads = self.lars_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py index 3431ad7e9145c..aba540fe7e783 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py @@ -22,7 +22,6 @@ class LocalSGDOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(LocalSGDOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -43,10 +42,12 @@ def _can_apply(self): if self.role_maker._worker_num() <= 1: return False - return isinstance(self.inner_opt, paddle.optimizer.momentum.Momentum) \ - or isinstance(self.inner_opt, paddle.fluid.optimizer.Momentum) \ - or isinstance(self.inner_opt, paddle.optimizer.sgd.SGD) \ + return ( + isinstance(self.inner_opt, paddle.optimizer.momentum.Momentum) + or isinstance(self.inner_opt, paddle.fluid.optimizer.Momentum) + or isinstance(self.inner_opt, paddle.optimizer.sgd.SGD) or isinstance(self.inner_opt, paddle.fluid.optimizer.SGD) + ) def _disable_strategy(self, dist_strategy): dist_strategy.localsgd = False @@ -69,11 +70,13 @@ def create_snapshot_vars(self, program): p2s = [] for param in non_dist_params: - snapshot = block.create_var(name=self.snapshot_name(param.name), - shape=param.shape, - persistable=True, - stop_gradient=True, - dtype=param.dtype) + snapshot = block.create_var( + name=self.snapshot_name(param.name), + shape=param.shape, + persistable=True, + stop_gradient=True, + dtype=param.dtype, + ) p2s.append([param, snapshot]) return p2s @@ -82,17 +85,17 @@ def init_snapshot_vars(self, startup_program, param2snapshot): for param, snapshot in param2snapshot: layers.assign(param, snapshot) - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - minimized = self.inner_opt.minimize(loss, - startup_program=startup_program) + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + minimized = self.inner_opt.minimize( + loss, startup_program=startup_program + ) k_steps_value = self.user_defined_strategy.localsgd_configs['k_steps'] begin_step_value = self.user_defined_strategy.localsgd_configs[ - 'begin_step'] + 'begin_step' + ] if startup_program is None: startup_program = default_startup_program() @@ -107,78 +110,90 @@ def minimize_impl(self, p2s = self.create_snapshot_vars(main_block.program) with program_guard(main_block.program, startup_program): step = layers.autoincreased_step_counter(begin=1) - k_steps = layers.create_global_var(name="k_steps", - shape=[1], - value=k_steps_value, - dtype='int64', - persistable=True) - - begin_step = layers.create_global_var(name="begin_step", - shape=[1], - value=begin_step_value, - dtype='int64', - persistable=True) - - last_step = layers.create_global_var(name="last_step", - shape=[1], - value=begin_step_value, - dtype='int64', - persistable=True) + k_steps = layers.create_global_var( + name="k_steps", + shape=[1], + value=k_steps_value, + dtype='int64', + persistable=True, + ) + + begin_step = layers.create_global_var( + name="begin_step", + shape=[1], + value=begin_step_value, + dtype='int64', + persistable=True, + ) + + last_step = layers.create_global_var( + name="last_step", + shape=[1], + value=begin_step_value, + dtype='int64', + persistable=True, + ) def communicate(): sub_block = default_main_program().current_block() ring_id = -1 for param, snapshot in p2s: - sub_block.append_op(type='elementwise_sub', - inputs={ - 'X': [snapshot], - 'Y': [param] - }, - outputs={'Out': [param]}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) - sub_block.append_op(type='c_sync_calc_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) + sub_block.append_op( + type='elementwise_sub', + inputs={'X': [snapshot], 'Y': [param]}, + outputs={'Out': [param]}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) + sub_block.append_op( + type='c_sync_calc_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) ring_id = (ring_id + 1) % self.nrings - sub_block.append_op(type='c_allreduce_sum', - inputs={'X': [param]}, - outputs={'Out': [param]}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Optimize - }) + sub_block.append_op( + type='c_allreduce_sum', + inputs={'X': [param]}, + outputs={'Out': [param]}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) for ring_id in range(self.nrings): - sub_block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Optimize - }) + sub_block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) for param, snapshot in p2s: - sub_block.append_op(type='scale', - inputs={'X': [param]}, - outputs={'Out': [param]}, - attrs={ - 'scale': - 1.0 / self.role_maker._worker_num(), - OP_ROLE_KEY: - OpRole.Optimize - }) - sub_block.append_op(type='elementwise_sub', - inputs={ - 'X': [snapshot], - 'Y': [param] - }, - outputs={'Out': [param]}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) - sub_block.append_op(type='assign', - inputs={'X': [param]}, - outputs={'Out': [snapshot]}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) + sub_block.append_op( + type='scale', + inputs={'X': [param]}, + outputs={'Out': [param]}, + attrs={ + 'scale': 1.0 / self.role_maker._worker_num(), + OP_ROLE_KEY: OpRole.Optimize, + }, + ) + sub_block.append_op( + type='elementwise_sub', + inputs={'X': [snapshot], 'Y': [param]}, + outputs={'Out': [param]}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) + sub_block.append_op( + type='assign', + inputs={'X': [param]}, + outputs={'Out': [snapshot]}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) layers.assign(step, last_step) def begin_localsgd(): @@ -189,13 +204,13 @@ def begin_localsgd(): class AdaptiveLocalSGDOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(AdaptiveLocalSGDOptimizer, self).__init__(optimizer) self.inner_opt = optimizer self.meta_optimizers_white_list = ['AMPOptimizer'] self.meta_optimizers_black_list = [ - "GraphExecutionOptimizer", "LocalSGDOptimizer" + "GraphExecutionOptimizer", + "LocalSGDOptimizer", ] self.snapshot_key = '@SNAPSHOT' @@ -209,10 +224,12 @@ def _can_apply(self): if self.role_maker._worker_num() <= 1: return False - return isinstance(self.inner_opt, paddle.optimizer.momentum.Momentum) \ - or isinstance(self.inner_opt, paddle.fluid.optimizer.Momentum) \ - or isinstance(self.inner_opt, paddle.optimizer.sgd.SGD) \ + return ( + isinstance(self.inner_opt, paddle.optimizer.momentum.Momentum) + or isinstance(self.inner_opt, paddle.fluid.optimizer.Momentum) + or isinstance(self.inner_opt, paddle.optimizer.sgd.SGD) or isinstance(self.inner_opt, paddle.fluid.optimizer.SGD) + ) def _disable_strategy(self, dist_strategy): dist_strategy.adaptive_localsgd = False @@ -222,7 +239,7 @@ def _enable_strategy(self, dist_strategy, context): dist_strategy.adaptive_localsgd = True dist_strategy.adaptive_localsgd_configs = { "init_k_steps": 1, - "begin_step": 1 + "begin_step": 1, } def snapshot_name(self, param_name): @@ -238,11 +255,13 @@ def create_snapshot_vars(self, program): p2s = [] for param in non_dist_params: - snapshot = block.create_var(name=self.snapshot_name(param.name), - shape=param.shape, - persistable=True, - stop_gradient=True, - dtype=param.dtype) + snapshot = block.create_var( + name=self.snapshot_name(param.name), + shape=param.shape, + persistable=True, + stop_gradient=True, + dtype=param.dtype, + ) p2s.append([param, snapshot]) return p2s @@ -252,40 +271,46 @@ def init_snapshot_vars(self, startup_program, param2snapshot): layers.assign(param, snapshot) def _generate_avg_loss(self, program_block, loss, avg_loss): - program_block.append_op(type='c_allreduce_sum', - inputs={'X': [loss]}, - outputs={'Out': [avg_loss]}, - attrs={ - 'ring_id': 0, - OP_ROLE_KEY: OpRole.Optimize, - 'use_calc_stream': True - }) - program_block.append_op(type='c_sync_calc_stream', - inputs={'X': [avg_loss]}, - outputs={'Out': [avg_loss]}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) - - program_block.append_op(type='scale', - inputs={'X': [avg_loss]}, - outputs={'Out': [avg_loss]}, - attrs={ - 'scale': - 1.0 / self.role_maker._worker_num(), - OP_ROLE_KEY: OpRole.Optimize - }) - - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - minimized = self.inner_opt.minimize(loss, - startup_program=startup_program) + program_block.append_op( + type='c_allreduce_sum', + inputs={'X': [loss]}, + outputs={'Out': [avg_loss]}, + attrs={ + 'ring_id': 0, + OP_ROLE_KEY: OpRole.Optimize, + 'use_calc_stream': True, + }, + ) + program_block.append_op( + type='c_sync_calc_stream', + inputs={'X': [avg_loss]}, + outputs={'Out': [avg_loss]}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) + + program_block.append_op( + type='scale', + inputs={'X': [avg_loss]}, + outputs={'Out': [avg_loss]}, + attrs={ + 'scale': 1.0 / self.role_maker._worker_num(), + OP_ROLE_KEY: OpRole.Optimize, + }, + ) + + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + minimized = self.inner_opt.minimize( + loss, startup_program=startup_program + ) init_k_steps = self.user_defined_strategy.adaptive_localsgd_configs[ - 'init_k_steps'] + 'init_k_steps' + ] begin_step_value = self.user_defined_strategy.adaptive_localsgd_configs[ - 'begin_step'] + 'begin_step' + ] if startup_program is None: startup_program = default_startup_program() @@ -301,41 +326,53 @@ def minimize_impl(self, with program_guard(main_block.program, startup_program): step = layers.autoincreased_step_counter(begin=1) - k_steps = layers.create_global_var(name="k_steps", - shape=[1], - value=int(init_k_steps), - dtype='int64', - persistable=True) - - begin_step = layers.create_global_var(name="begin_step", - shape=[1], - value=int(begin_step_value), - dtype='int64', - persistable=True) - - last_step = layers.create_global_var(name="last_step", - shape=[1], - value=int(0), - dtype='int64', - persistable=True) - - avg_loss = layers.create_global_var(name="avg_loss", - shape=[1], - value=float(0), - dtype=loss.dtype, - persistable=True) - - lr_0 = layers.create_global_var(name="lr_0", - shape=[1], - value=float(0), - dtype='float32', - persistable=True) - - loss_0 = layers.create_global_var(name="loss_0", - shape=[1], - value=float(0), - dtype='float32', - persistable=True) + k_steps = layers.create_global_var( + name="k_steps", + shape=[1], + value=int(init_k_steps), + dtype='int64', + persistable=True, + ) + + begin_step = layers.create_global_var( + name="begin_step", + shape=[1], + value=int(begin_step_value), + dtype='int64', + persistable=True, + ) + + last_step = layers.create_global_var( + name="last_step", + shape=[1], + value=int(0), + dtype='int64', + persistable=True, + ) + + avg_loss = layers.create_global_var( + name="avg_loss", + shape=[1], + value=float(0), + dtype=loss.dtype, + persistable=True, + ) + + lr_0 = layers.create_global_var( + name="lr_0", + shape=[1], + value=float(0), + dtype='float32', + persistable=True, + ) + + loss_0 = layers.create_global_var( + name="loss_0", + shape=[1], + value=float(0), + dtype='float32', + persistable=True, + ) global_lr = self.inner_opt._global_learning_rate() @@ -350,75 +387,90 @@ def communicate(): sub_block = default_main_program().current_block() ring_id = -1 for param, snapshot in p2s: - sub_block.append_op(type='elementwise_sub', - inputs={ - 'X': [snapshot], - 'Y': [param] - }, - outputs={'Out': [param]}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) - sub_block.append_op(type='c_sync_calc_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) + sub_block.append_op( + type='elementwise_sub', + inputs={'X': [snapshot], 'Y': [param]}, + outputs={'Out': [param]}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) + sub_block.append_op( + type='c_sync_calc_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) ring_id = (ring_id + 1) % self.nrings - sub_block.append_op(type='c_allreduce_sum', - inputs={'X': [param]}, - outputs={'Out': [param]}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Optimize - }) + sub_block.append_op( + type='c_allreduce_sum', + inputs={'X': [param]}, + outputs={'Out': [param]}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) for ring_id in range(self.nrings): - sub_block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Optimize - }) + sub_block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) for param, snapshot in p2s: - sub_block.append_op(type='scale', - inputs={'X': [param]}, - outputs={'Out': [param]}, - attrs={ - 'scale': - 1.0 / self.role_maker._worker_num(), - OP_ROLE_KEY: - OpRole.Optimize - }) - sub_block.append_op(type='elementwise_sub', - inputs={ - 'X': [snapshot], - 'Y': [param] - }, - outputs={'Out': [param]}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) - sub_block.append_op(type='assign', - inputs={'X': [param]}, - outputs={'Out': [snapshot]}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) + sub_block.append_op( + type='scale', + inputs={'X': [param]}, + outputs={'Out': [param]}, + attrs={ + 'scale': 1.0 / self.role_maker._worker_num(), + OP_ROLE_KEY: OpRole.Optimize, + }, + ) + sub_block.append_op( + type='elementwise_sub', + inputs={'X': [snapshot], 'Y': [param]}, + outputs={'Out': [param]}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) + sub_block.append_op( + type='assign', + inputs={'X': [param]}, + outputs={'Out': [snapshot]}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) layers.assign(step, last_step) def communicate_avg_loss(): communicate() self._generate_avg_loss(main_block, loss, avg_loss) - next_local_steps = layers.cast(layers.ceil( - layers.sqrt(lr_0 * avg_loss / (global_lr * loss_0) * - float(init_k_steps))), - dtype='int64') - max_local_steps = layers.fill_constant(shape=[1], - dtype='int64', - value=16) - min_local_steps = layers.fill_constant(shape=[1], - dtype='int64', - value=1) + next_local_steps = layers.cast( + layers.ceil( + layers.sqrt( + lr_0 + * avg_loss + / (global_lr * loss_0) + * float(init_k_steps) + ) + ), + dtype='int64', + ) + max_local_steps = layers.fill_constant( + shape=[1], dtype='int64', value=16 + ) + min_local_steps = layers.fill_constant( + shape=[1], dtype='int64', value=1 + ) next_local_steps = layers.elementwise_min( - next_local_steps, max_local_steps) + next_local_steps, max_local_steps + ) next_local_steps = layers.elementwise_max( - next_local_steps, min_local_steps) + next_local_steps, min_local_steps + ) layers.assign(next_local_steps, k_steps) def begin_localsgd(): diff --git a/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py b/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py index 35e11221b6f63..87085a322c303 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py +++ b/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py @@ -18,7 +18,6 @@ class MetaOptimizerBase(Optimizer): - def __init__(self, optimizer): self.inner_opt = optimizer self._learning_rate = self.inner_opt._learning_rate @@ -26,8 +25,9 @@ def __init__(self, optimizer): self.meta_optimizers_white_list = [] self.meta_optimizers_black_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): self.loss = loss self.role_maker = role_maker self.user_defined_optimizer = user_defined_optimizer @@ -50,52 +50,57 @@ def _can_update(self, optimizer): def _disable_strategy(self, dist_strategy): raise NotImplementedError( "you should implement disable strategy in {}".format( - type(self).__name__)) + type(self).__name__ + ) + ) def _enable_strategy(self, dist_strategy, context=None): raise NotImplementedError( "you should implement enable strategy in {}".format( - type(self).__name__)) + type(self).__name__ + ) + ) def apply_gradients(self, params_grads): return self.inner_opt.apply_gradients(params_grads=params_grads) - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): - return self.inner_opt.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): + return self.inner_opt.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) def apply_optimize(self, loss, startup_program, params_grads): - return self.inner_opt.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) - - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - params_grads = self.backward(loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) - - optimize_ops = self.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) + return self.inner_opt.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) + + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + params_grads = self.backward( + loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) + + optimize_ops = self.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) return optimize_ops, params_grads - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - optimize_ops, params_grads = self.minimize_impl(loss, startup_program, - parameter_list, - no_grad_set) + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + optimize_ops, params_grads = self.minimize_impl( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py index 42784113822c3..f94215aad0938 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py @@ -18,7 +18,6 @@ class ParameterServerGraphOptimizer(ParameterServerOptimizer): - def __init__(self, optimizer): super(ParameterServerGraphOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -59,18 +58,18 @@ def _try_to_compile(self, main_program, loss): self._compiled_program = compiler.CompiledProgram(main_program) - self._compiled_program.with_data_parallel(loss_name=loss.name, - build_strategy=build_strategy, - exec_strategy=exec_strategy, - share_vars_from=None) + self._compiled_program.with_data_parallel( + loss_name=loss.name, + build_strategy=build_strategy, + exec_strategy=exec_strategy, + share_vars_from=None, + ) return self._compiled_program - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): program = loss.block.program compiled_program = self._try_to_compile(program, loss) program._graph = compiled_program diff --git a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py index c04215d45656c..d18c5d1fdcb7a 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py @@ -24,23 +24,24 @@ class ParameterServerOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(ParameterServerOptimizer, self).__init__(optimizer) self.inner_opt = optimizer # we do not allow meta optimizer to be inner optimizer currently self.meta_optimizers_white_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(ParameterServerOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(ParameterServerOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) - #self.micro_batch_size = user_defined_strategy.pipeline_configs[ + # self.micro_batch_size = user_defined_strategy.pipeline_configs[ # 'micro_batch_size'] self.num_microbatches = user_defined_strategy.pipeline_configs[ - 'accumulate_steps'] + 'accumulate_steps' + ] def _is_graph_out(self): return False @@ -66,11 +67,13 @@ def get_dist_env(self): 'trainer_id': trainer_id, 'num_trainers': num_trainers, 'current_endpoint': current_endpoint, - 'trainer_endpoints': trainer_endpoints + 'trainer_endpoints': trainer_endpoints, } def _get_distributed_strategy(self): - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + StrategyFactory, + ) k_steps = self.user_defined_strategy.a_sync_configs["k_steps"] strategy = None @@ -90,7 +93,9 @@ def _get_distributed_strategy(self): return strategy def _build_trainer_programs(self, compiled_config): - from paddle.fluid.incubate.fleet.parameter_server.ir import trainer_pass as worker + from paddle.fluid.incubate.fleet.parameter_server.ir import ( + trainer_pass as worker, + ) _main = compiled_config.origin_main_program.clone() _startup = compiled_config.origin_startup_program.clone() @@ -98,49 +103,66 @@ def _build_trainer_programs(self, compiled_config): use_ps_gpu = self.user_defined_strategy.a_sync_configs["use_ps_gpu"] if not compiled_config.is_geo_mode(): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _add_lr_decay_table_pass + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _add_lr_decay_table_pass, + ) + _add_lr_decay_table_pass( - _main, compiled_config, - self.user_defined_strategy.a_sync_configs["lr_decay_steps"]) + _main, + compiled_config, + self.user_defined_strategy.a_sync_configs["lr_decay_steps"], + ) # for main program - _main = worker.distributed_ops_pass(_main, compiled_config, - use_ps_gpu) + _main = worker.distributed_ops_pass( + _main, compiled_config, use_ps_gpu + ) if not use_ps_gpu: _main = worker.delete_optimizer_pass(_main, compiled_config) _main = worker.append_send_ops_pass(_main, compiled_config) _startup = worker.delete_extra_optimizes_pass( - _startup, compiled_config) + _startup, compiled_config + ) # for startup program _startup = worker.fake_init_ops_pass(_startup, compiled_config) if use_ps_gpu: _main = worker.ps_gpu_pass(_main) - from paddle.fluid.transpiler.collective import SingleProcessMultiThread + from paddle.fluid.transpiler.collective import ( + SingleProcessMultiThread, + ) + t = SingleProcessMultiThread() env = self.get_dist_env() - t.transpile(startup_program=_startup, - main_program=_main, - rank=env["trainer_id"], - endpoints=env["trainer_endpoints"], - current_endpoint=env['current_endpoint'], - wait_port=False) + t.transpile( + startup_program=_startup, + main_program=_main, + rank=env["trainer_id"], + endpoints=env["trainer_endpoints"], + current_endpoint=env['current_endpoint'], + wait_port=False, + ) compiled_config.set_origin_ps_main_program(_main) compiled_config.set_origin_ps_startup_program(_startup) # for heter program if self.role_maker._is_heter_parameter_server_mode: - from paddle.fluid.incubate.fleet.parameter_server.ir import heter_trainer_pass as heter_worker + from paddle.fluid.incubate.fleet.parameter_server.ir import ( + heter_trainer_pass as heter_worker, + ) + if self.role_maker._is_heter_worker(): # for heter worker stage_id = self.role_maker._get_stage_id() device = self.role_maker._heter_device_type().lower() _main = heter_worker.split_heter_worker_ops_pass( - _main, compiled_config, stage_id, device) + _main, compiled_config, stage_id, device + ) else: # for default worker _main = heter_worker.split_trainer_ops_pass( - _main, compiled_config) + _main, compiled_config + ) else: _main = worker.append_send_ops_pass(_main, compiled_config) _startup = _startup @@ -148,7 +170,8 @@ def _build_trainer_programs(self, compiled_config): compiled_config.set_origin_ps_startup_program(_startup) launch_barrier = self.user_defined_strategy.a_sync_configs[ - "launch_barrier"] + "launch_barrier" + ] launch_barrier_flag = int(os.getenv("FLAGS_LAUNCH_BARRIER", "1")) if launch_barrier and launch_barrier_flag: # for trainer wait server ready @@ -165,11 +188,16 @@ def _build_pserver_programs(self, compiled_config): _main = fluid.Program() _startup = fluid.Program() - from paddle.fluid.incubate.fleet.parameter_server.ir import pserver_pass as server + from paddle.fluid.incubate.fleet.parameter_server.ir import ( + pserver_pass as server, + ) if not compiled_config.is_geo_mode(): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_optimize_ops + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_optimize_ops, + ) + is_sgd_adam = False main_program = compiled_config.get_origin_main_program() @@ -178,11 +206,16 @@ def _build_pserver_programs(self, compiled_config): if len(ops) == 0: return _main, _startup - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _add_lr_decay_table_pass + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _add_lr_decay_table_pass, + ) + lr_decay_steps = self.user_defined_strategy.a_sync_configs[ - "lr_decay_steps"] - _add_lr_decay_table_pass(main_program, compiled_config, - lr_decay_steps) + "lr_decay_steps" + ] + _add_lr_decay_table_pass( + main_program, compiled_config, lr_decay_steps + ) for op in ops: if op.type in ["sgd", "adam"]: @@ -195,37 +228,44 @@ def _build_pserver_programs(self, compiled_config): _main = server.add_listen_and_serv_pass(_main, compiled_config) _main = server.add_rpc_global_flags_pass(_main, compiled_config) _main = server.add_optimizer_pass(_main, compiled_config) - _main = server.large_scale_sparse_pass(_main, _main, - compiled_config, False) + _main = server.large_scale_sparse_pass( + _main, _main, compiled_config, False + ) _startup = server.build_pserver_startup_program_pass( - _startup, _main, compiled_config) - _startup = server.large_scale_sparse_pass(_startup, _main, - compiled_config, True) + _startup, _main, compiled_config + ) + _startup = server.large_scale_sparse_pass( + _startup, _main, compiled_config, True + ) if not compiled_config.is_sync_mode(): _main = server.delete_unused_in_main_pass( - _main, compiled_config) + _main, compiled_config + ) _startup = server.delete_unused_in_startup_pass( - _startup, _main, compiled_config) + _startup, _main, compiled_config + ) else: _main = server.add_listen_and_serv_pass(_main, compiled_config) _main = server.add_rpc_global_flags_pass(_main, compiled_config) _main = server.add_geo_optimizer_pass(_main, compiled_config) _startup = server.build_pserver_startup_program_pass( - _startup, _main, compiled_config) + _startup, _main, compiled_config + ) _startup = server.delete_unused_in_startup_pass( - _startup, _main, compiled_config) + _startup, _main, compiled_config + ) return _main, _startup def _can_apply_geo(self, dist_strategy, program): - def get_sys_free_mem(): plat = platform.system() if platform.system() == "Darwin": - vm = subprocess.Popen(['vm_stat'], - stdout=subprocess.PIPE).communicate()[0] + vm = subprocess.Popen( + ['vm_stat'], stdout=subprocess.PIPE + ).communicate()[0] # Process vm_stat vmLines = vm.split('\n') sep = re.compile(r':[\s]+') @@ -233,8 +273,9 @@ def get_sys_free_mem(): for row in range(1, len(vmLines) - 2): rowText = vmLines[row].strip() rowElements = sep.split(rowText) - vmStats[(rowElements[0])] = int( - rowElements[1].strip(r'\.')) * 4096 + vmStats[(rowElements[0])] = ( + int(rowElements[1].strip(r'\.')) * 4096 + ) return vmStats["Pages free"] elif platform.system() == "Linux": mems = {} @@ -246,22 +287,27 @@ def get_sys_free_mem(): return free else: raise ValueError( - "%s platform is unsupported is parameter server optimizer" % - (platform.system())) + "%s platform is unsupported is parameter server optimizer" + % (platform.system()) + ) if not isinstance(self.inner_opt, fluid.optimizer.SGDOptimizer): return False free = get_sys_free_mem() - from paddle.fluid.incubate.fleet.parameter_server.ir import vars_metatools + from paddle.fluid.incubate.fleet.parameter_server.ir import ( + vars_metatools, + ) processed_var_names = set(["@EMPTY@"]) param_memory_size = 0 for varname in program.global_block().vars: var = program.global_block().vars[varname] - if not var.persistable or var.desc.type( - ) != core.VarDesc.VarType.LOD_TENSOR: + if ( + not var.persistable + or var.desc.type() != core.VarDesc.VarType.LOD_TENSOR + ): continue param = vars_metatools.create_var_struct(var) param_memory_size += param.m_size @@ -287,15 +333,18 @@ def get_sys_free_mem(): if x < 0: if neg_dim_count >= 1: raise ValueError( - "Var %s has more than one negative dim." % - (var_name)) + "Var %s has more than one negative dim." + % (var_name) + ) neg_dim_count += 1 - data_count *= (-x) + data_count *= -x else: data_count *= x program_tmp_vars[var_name] = ( - data_count, neg_dim_count, - vars_metatools.dtype_to_size[var.dtype]) + data_count, + neg_dim_count, + vars_metatools.dtype_to_size[var.dtype], + ) for varname in program_tmp_vars: data_count, neg_dim_count, type_size = program_tmp_vars[varname] @@ -309,27 +358,32 @@ def get_sys_free_mem(): else: return False - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - self.inner_opt.minimize(loss, startup_program, parameter_list, - no_grad_set) + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + self.inner_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) strategy = self._get_distributed_strategy() _origin_main_program = loss.block.program _origin_startup_program = startup_program - from paddle.fluid.incubate.fleet.parameter_server.ir import public as public - - compiled_config = public.CompileTimeStrategy(_origin_main_program, - _origin_startup_program, - strategy, self.role_maker) + from paddle.fluid.incubate.fleet.parameter_server.ir import ( + public as public, + ) + + compiled_config = public.CompileTimeStrategy( + _origin_main_program, + _origin_startup_program, + strategy, + self.role_maker, + ) compiled_config.strategy = strategy if self.role_maker._is_worker() or self.role_maker._is_heter_worker(): main_program, startup_program = self._build_trainer_programs( - compiled_config) + compiled_config + ) if self.role_maker._is_heter_parameter_server_mode: _origin_startup_program._heter_pipeline_opt = { "startup_program": startup_program, @@ -340,12 +394,12 @@ def minimize_impl(self, loss.block.program._heter_pipeline_opt = { "trainer": "HeterPipelineTrainer", "device_worker": "HeterSection", - "trainers": self.role_maker._get_stage_trainers( - ), ## trainer num in each stage + "trainers": self.role_maker._get_stage_trainers(), ## trainer num in each stage "trainer_id": int(self.role_maker._role_id()), "pipeline_stage": int(self.role_maker._get_stage_id()) - 1, - "num_pipeline_stages": - int(self.role_maker._get_num_stage()), + "num_pipeline_stages": int( + self.role_maker._get_num_stage() + ), "section_program": main_program, "num_microbatches": self.num_microbatches, "heter_place": self.role_maker._heter_device(), @@ -356,13 +410,14 @@ def minimize_impl(self, elif self.role_maker._is_server(): main_program, startup_program = self._build_pserver_programs( - compiled_config) + compiled_config + ) loss.block.program = main_program fluid.framework.switch_startup_program(startup_program) return None, None def _disable_strategy(self, dist_strategy): - #if self.role_maker._is_heter_parameter_server_mode: + # if self.role_maker._is_heter_parameter_server_mode: # dist_strategy.pipeline = False # dist_strategy.pipeline_configs = { # "micro_batch_size": 1, @@ -374,7 +429,7 @@ def _disable_strategy(self, dist_strategy): dist_strategy.a_sync_configs = a_sync_configs def _enable_strategy(self, dist_strategy, context): - #if self.role_maker._is_heter_parameter_server_mode: + # if self.role_maker._is_heter_parameter_server_mode: # dist_strategy.pipeline = True # dist_strategy.pipeline_configs = { # "micro_batch_size": 1, @@ -387,8 +442,9 @@ def _enable_strategy(self, dist_strategy, context): dist_strategy.a_sync = True a_sync_configs = dist_strategy.a_sync_configs - is_geo = self._can_apply_geo(dist_strategy, - context["origin_main_program"]) + is_geo = self._can_apply_geo( + dist_strategy, context["origin_main_program"] + ) if is_geo: a_sync_configs["k_steps"] = 800 diff --git a/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py index 690ccdfea5f0c..dfb8fe5b22438 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py @@ -14,13 +14,19 @@ import paddle.fluid as fluid from paddle.fluid.optimizer import PipelineOptimizer as PO from .meta_optimizer_base import MetaOptimizerBase -from .common import CollectiveHelper, OP_ROLE_KEY, OP_ROLE_VAR_KEY, OpRole, is_backward_op, is_loss_grad_op +from .common import ( + CollectiveHelper, + OP_ROLE_KEY, + OP_ROLE_VAR_KEY, + OpRole, + is_backward_op, + is_loss_grad_op, +) __all__ = [] class PipelineOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(PipelineOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -35,17 +41,21 @@ def __init__(self, optimizer): self.dp_ring_id = 2 self.start_pipeline_ring_id = 20 # Just a magic number - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(PipelineOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(PipelineOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) self.micro_batch_size = user_defined_strategy.pipeline_configs[ - 'micro_batch_size'] + 'micro_batch_size' + ] self.num_microbatches = user_defined_strategy.pipeline_configs[ - 'accumulate_steps'] + 'accumulate_steps' + ] self.schedule_mode = user_defined_strategy.pipeline_configs[ - 'schedule_mode'] + 'schedule_mode' + ] self.use_sharding = user_defined_strategy.sharding def _can_apply(self): @@ -83,23 +93,25 @@ def _broadcast_params(self, ring_id): if param.is_distributed: continue - block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - 'root': 0, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + 'root': 0, + OP_ROLE_KEY: OpRole.Forward, + }, + ) - if not param: return # no parameter on this device - block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) + if not param: + return # no parameter on this device + block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: OpRole.Forward}, + ) def _get_process_group_info(self): # global ring info @@ -121,12 +133,16 @@ def _init_process_group(self, pipeline_pair, pipeline_ring_map): self._get_process_group_info() collective_helper = CollectiveHelper(self.role_maker, wait_port=False) # Create global ring for all gpus (ring_id = 0) - collective_helper._init_communicator(self.startup_program, - self.current_endpoint, - self.global_endpoints, - self.global_rank, - self.global_ring_id, True, - self.global_ring_id, True) + collective_helper._init_communicator( + self.startup_program, + self.current_endpoint, + self.global_endpoints, + self.global_rank, + self.global_ring_id, + True, + self.global_ring_id, + True, + ) # Create pipeline rings if self.inner_parallelism > 1: pipeline_id = self.rank // self.inner_parallelism @@ -139,41 +155,62 @@ def _init_process_group(self, pipeline_pair, pipeline_ring_map): second_node = pair[1] + start_index if self.rank != first_node and self.rank != second_node: collective_helper._init_communicator( - self.startup_program, None, None, None, None, False, - self.global_ring_id, True) + self.startup_program, + None, + None, + None, + None, + False, + self.global_ring_id, + True, + ) continue pipeline_endpoints = [ - self.endpoints[first_node], self.endpoints[second_node] + self.endpoints[first_node], + self.endpoints[second_node], ] pipeline_rank = 0 if self.rank == first_node else 1 pipeline_nranks = 2 - collective_helper._init_communicator(self.startup_program, - self.current_endpoint, - pipeline_endpoints, - pipeline_rank, ring_id, - False, self.global_ring_id, - True) + collective_helper._init_communicator( + self.startup_program, + self.current_endpoint, + pipeline_endpoints, + pipeline_rank, + ring_id, + False, + self.global_ring_id, + True, + ) # Create dp rings if self.pipeline_num > 1: collective_helper._init_communicator( - self.startup_program, self.current_endpoint, self.dp_endpoints, - self.dp_rank, self.dp_ring_id, True, self.global_ring_id, True) + self.startup_program, + self.current_endpoint, + self.dp_endpoints, + self.dp_rank, + self.dp_ring_id, + True, + self.global_ring_id, + True, + ) self._broadcast_params(self.dp_ring_id) - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): self.endpoints = self.role_maker._get_trainer_endpoints() self.current_endpoint = self.endpoints[self.role_maker._worker_index()] self.rank = self.role_maker._worker_index() self.nranks = self.role_maker._worker_num() - self.wrapped_opt = PO(self.inner_opt, - num_microbatches=self.num_microbatches) - orig_startup_program = startup_program if startup_program else fluid.default_startup_program( + self.wrapped_opt = PO( + self.inner_opt, num_microbatches=self.num_microbatches + ) + orig_startup_program = ( + startup_program + if startup_program + else fluid.default_startup_program() ) block = loss.block program = block.program @@ -187,10 +224,18 @@ def minimize_impl(self, program._pipeline_opt['use_sharding'] = False program._pipeline_opt['mp_degree'] = 1 program._pipeline_opt['mp_rank'] = 0 - optimize_ops, params_grads, prog_list, pp_pair, ring_map = self.wrapped_opt.minimize( - loss, startup_program, parameter_list, no_grad_set) + ( + optimize_ops, + params_grads, + prog_list, + pp_pair, + ring_map, + ) = self.wrapped_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) self.startup_program = orig_startup_program._pipeline_opt[ - 'startup_program'] + 'startup_program' + ] self.inner_parallelism = program._pipeline_opt['inner_parallelism'] assert self.nranks % self.inner_parallelism == 0 assert prog_list @@ -217,18 +262,21 @@ def _insert_loss_grad_ops(self, loss, pipeline_num): for idx, op in reversed(list(enumerate(block.ops))): if is_loss_grad_op(op): loss_grad_var = block.vars[op.output_arg_names[0]] - block._insert_op(idx + 1, - type='scale', - inputs={'X': loss_grad_var}, - outputs={'Out': loss_grad_var}, - attrs={ - 'scale': 1.0 / pipeline_num, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op( + idx + 1, + type='scale', + inputs={'X': loss_grad_var}, + outputs={'Out': loss_grad_var}, + attrs={ + 'scale': 1.0 / pipeline_num, + OP_ROLE_KEY: OpRole.Backward, + }, + ) def _insert_allreduce_ops(self, ring_id): - block = self.main_program._pipeline_opt['section_program'].global_block( - ) + block = self.main_program._pipeline_opt[ + 'section_program' + ].global_block() origin_block = self.main_program.global_block() grad = None processed_param_name = set() @@ -237,9 +285,9 @@ def _insert_allreduce_ops(self, ring_id): if is_backward_op(op) and not first_optimize_op_idx: first_optimize_op_idx = idx + 1 # no optimize phase - if first_optimize_op_idx == len(block.ops): return - if is_backward_op(op) and \ - OP_ROLE_VAR_KEY in op.attr_names: + if first_optimize_op_idx == len(block.ops): + return + if is_backward_op(op) and OP_ROLE_VAR_KEY in op.attr_names: op_role_var = op.all_attrs()[OP_ROLE_VAR_KEY] if len(op_role_var) == 0: continue @@ -248,21 +296,25 @@ def _insert_allreduce_ops(self, ring_id): for i in range(0, len(op_role_var), 2): param_name = op_role_var[i] param = block.vars[op_role_var[i]] - if param_name in processed_param_name: continue + if param_name in processed_param_name: + continue processed_param_name.add(param_name) grad_name = op_role_var[i + 1] - if 'MERGED' not in grad_name: grad_name += '@MERGED' + if 'MERGED' not in grad_name: + grad_name += '@MERGED' grad = block.vars[grad_name] origin_param = origin_block.vars[op_role_var[i]] if origin_param.is_distributed: continue - block._insert_op(first_optimize_op_idx + offset, - type='c_allreduce_sum', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Optimize - }) + block._insert_op( + first_optimize_op_idx + offset, + type='c_allreduce_sum', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) diff --git a/python/paddle/distributed/fleet/meta_optimizers/ps_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/ps_optimizer.py index 523a674110293..000ef98bcaec4 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/ps_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/ps_optimizer.py @@ -19,24 +19,31 @@ import re import os import platform -from paddle.distributed.ps.utils.public import TrainerRuntimeConfig, build_var_distributed, dtype_to_size, get_dist_env, get_var_mem_size, logger +from paddle.distributed.ps.utils.public import ( + TrainerRuntimeConfig, + build_var_distributed, + dtype_to_size, + get_dist_env, + get_var_mem_size, + logger, +) from paddle.distributed.passes import PassContext from paddle.distributed.ps.utils.ps_factory import PsProgramBuilderFactory class ParameterServerOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(ParameterServerOptimizer, self).__init__(optimizer) self.inner_opt = optimizer # we do not allow meta optimizer to be inner optimizer currently self.meta_optimizers_white_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(ParameterServerOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(ParameterServerOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) def _set_origin_programs(self, losses): self.origin_main_programs = [] @@ -66,31 +73,40 @@ def _init_ps_pass_context(self, loss, startup_program): logger.info("ps_mode: {}".format(attrs['ps_mode'])) attrs['role_maker'] = self.role_maker attrs[ - 'is_heter_ps_mode'] = self.role_maker._is_heter_parameter_server_mode + 'is_heter_ps_mode' + ] = self.role_maker._is_heter_parameter_server_mode attrs['is_worker'] = self.role_maker._is_worker() attrs['is_server'] = self.role_maker._is_server() attrs['is_heter_worker'] = self.role_maker._is_heter_worker() - logger.info("this process is heter? {}".format( - attrs['is_heter_worker'])) + logger.info( + "this process is heter? {}".format(attrs['is_heter_worker']) + ) attrs['use_ps_gpu'] = self.user_defined_strategy.a_sync_configs[ - "use_ps_gpu"] + "use_ps_gpu" + ] attrs['lr_decay_steps'] = self.user_defined_strategy.a_sync_configs[ - "lr_decay_steps"] + "lr_decay_steps" + ] # FL attrs['local_sparse'] = attrs[ - "user_defined_strategy"].trainer_desc_configs["local_sparse"] + "user_defined_strategy" + ].trainer_desc_configs["local_sparse"] attrs['remote_sparse'] = attrs[ - "user_defined_strategy"].trainer_desc_configs["remote_sparse"] + "user_defined_strategy" + ].trainer_desc_configs["remote_sparse"] attrs['is_fl_ps_mode'] = self.user_defined_strategy.is_fl_ps_mode attrs[ - 'with_coordinator'] = self.user_defined_strategy.is_with_coordinator + 'with_coordinator' + ] = self.user_defined_strategy.is_with_coordinator attrs['k_steps'] = self.user_defined_strategy.a_sync_configs["k_steps"] attrs['launch_barrier'] = self.user_defined_strategy.a_sync_configs[ - "launch_barrier"] + "launch_barrier" + ] attrs['launch_barrier_flag'] = int( - os.getenv("FLAGS_LAUNCH_BARRIER", "1")) + os.getenv("FLAGS_LAUNCH_BARRIER", "1") + ) build_var_distributed(attrs) @@ -111,31 +127,32 @@ def _can_apply(self): k_steps = self.user_defined_strategy.a_sync_configs["k_steps"] return True if k_steps >= 0 else False - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - self.inner_opt.minimize(loss, startup_program, parameter_list, - no_grad_set) + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + self.inner_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) if startup_program == None: startup_program = paddle.static.default_startup_program() - -# print("program after inner optimizer minimize:", -# str(loss.block.program)) + # print("program after inner optimizer minimize:", + # str(loss.block.program)) self._set_origin_programs([loss]) self._init_ps_pass_context(loss, startup_program) ps_builder = PsProgramBuilderFactory()._create_ps_program_builder( - self.pass_ctx) + self.pass_ctx + ) ps_builder._build_programs() return None, None - def minimize_losses_impl(self, - losses, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_losses_impl( + self, + losses, + startup_program=None, + parameter_list=None, + no_grad_set=None, + ): if parameter_list is None: parameter_list = [None] * len(losses) for idx, loss in enumerate(losses): @@ -148,18 +165,19 @@ def minimize_losses_impl(self, startup_prog = startup_program[idx] self._init_ps_pass_context(loss, startup_prog) ps_builder = PsProgramBuilderFactory()._create_ps_program_builder( - self.pass_ctx) + self.pass_ctx + ) ps_builder._build_programs() startup_program[idx] = self.pass_ctx._attrs['cloned_startup'] return None, None def _can_apply_geo(self, program): - def get_sys_free_mem(): plat = platform.system() if platform.system() == "Darwin": - vm = subprocess.Popen(['vm_stat'], - stdout=subprocess.PIPE).communicate()[0] + vm = subprocess.Popen( + ['vm_stat'], stdout=subprocess.PIPE + ).communicate()[0] # Process vm_stat vmLines = vm.split('\n') sep = re.compile(r':[\s]+') @@ -167,8 +185,9 @@ def get_sys_free_mem(): for row in range(1, len(vmLines) - 2): rowText = vmLines[row].strip() rowElements = sep.split(rowText) - vmStats[(rowElements[0])] = int( - rowElements[1].strip(r'\.')) * 4096 + vmStats[(rowElements[0])] = ( + int(rowElements[1].strip(r'\.')) * 4096 + ) return vmStats["Pages free"] elif platform.system() == "Linux": mems = {} @@ -180,8 +199,9 @@ def get_sys_free_mem(): return free else: raise ValueError( - "%s platform is unsupported is parameter server optimizer" % - (platform.system())) + "%s platform is unsupported is parameter server optimizer" + % (platform.system()) + ) if not isinstance(self.inner_opt, fluid.optimizer.SGDOptimizer): return False @@ -191,8 +211,10 @@ def get_sys_free_mem(): param_memory_size = 0 for varname in program.global_block().vars: var = program.global_block().vars[varname] - if not var.persistable or var.desc.type( - ) != core.VarDesc.VarType.LOD_TENSOR: + if ( + not var.persistable + or var.desc.type() != core.VarDesc.VarType.LOD_TENSOR + ): continue param_memory_size += get_var_mem_size(var) processed_var_names.add(varname) @@ -217,14 +239,18 @@ def get_sys_free_mem(): if x < 0: if neg_dim_count >= 1: raise ValueError( - "Var %s has more than one negative dim." % - (var_name)) + "Var %s has more than one negative dim." + % (var_name) + ) neg_dim_count += 1 - data_count *= (-x) + data_count *= -x else: data_count *= x - program_tmp_vars[var_name] = (data_count, neg_dim_count, - dtype_to_size[var.dtype]) + program_tmp_vars[var_name] = ( + data_count, + neg_dim_count, + dtype_to_size[var.dtype], + ) for varname in program_tmp_vars: data_count, neg_dim_count, type_size = program_tmp_vars[varname] diff --git a/python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py index 9c7f213105eda..6ae89a9754ebd 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/raw_program_optimizer.py @@ -14,11 +14,18 @@ import paddle.fluid as fluid from paddle.fluid import core, unique_name from .meta_optimizer_base import MetaOptimizerBase -from .common import OpRole, OP_ROLE_KEY, OP_ROLE_VAR_KEY, CollectiveHelper, is_loss_grad_op, is_backward_op, is_optimizer_op +from .common import ( + OpRole, + OP_ROLE_KEY, + OP_ROLE_VAR_KEY, + CollectiveHelper, + is_loss_grad_op, + is_backward_op, + is_optimizer_op, +) class RawProgramOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(RawProgramOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -36,16 +43,23 @@ def __init__(self, optimizer): ] self.global_ring_id = 0 - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(RawProgramOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) - self.without_graph_optimization = user_defined_strategy.without_graph_optimization + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(RawProgramOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) + self.without_graph_optimization = ( + user_defined_strategy.without_graph_optimization + ) self.fuse_all_reduce_ops = user_defined_strategy.fuse_all_reduce_ops if self.fuse_all_reduce_ops: - self.fuse_grad_size_in_num = user_defined_strategy.fuse_grad_size_in_num - self.calc_comm_same_stream = user_defined_strategy._calc_comm_same_stream + self.fuse_grad_size_in_num = ( + user_defined_strategy.fuse_grad_size_in_num + ) + self.calc_comm_same_stream = ( + user_defined_strategy._calc_comm_same_stream + ) def _can_apply(self): if not self.role_maker._is_collective: @@ -68,23 +82,25 @@ def _broadcast_params(self, ring_id): if param.is_distributed: continue - block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - 'root': 0, - OP_ROLE_KEY: OpRole.Forward - }) - - if not param: return # no parameter on this device - block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + 'root': 0, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + + if not param: + return # no parameter on this device + block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: OpRole.Forward}, + ) def _get_process_group_info(self): # global ring info @@ -96,19 +112,21 @@ def _init_process_group(self): self._get_process_group_info() collective_helper = CollectiveHelper(self.role_maker, wait_port=False) # Create global ring for all gpus (ring_id = 0) - collective_helper._init_communicator(self.startup_program, - self.current_endpoint, - self.global_endpoints, - self.global_rank, - self.global_ring_id, True, - self.global_ring_id, True) + collective_helper._init_communicator( + self.startup_program, + self.current_endpoint, + self.global_endpoints, + self.global_rank, + self.global_ring_id, + True, + self.global_ring_id, + True, + ) self._broadcast_params(self.global_ring_id) - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): self.endpoints = self.role_maker._get_trainer_endpoints() self.current_endpoint = self.endpoints[self.role_maker._worker_index()] self.rank = self.role_maker._worker_index() @@ -122,7 +140,8 @@ def minimize_impl(self, self.main_program = program optimize_ops, params_grads = self.inner_opt.minimize( - loss, startup_program, parameter_list, no_grad_set) + loss, startup_program, parameter_list, no_grad_set + ) if self.nranks == 1: return optimize_ops, params_grads self._init_process_group() @@ -147,7 +166,9 @@ def _find_gradient_merge_block(self): if gm_cond_var_name is None: return None - cond_op = None # false_fn of gm is None, so we should only find one block + cond_op = ( + None # false_fn of gm is None, so we should only find one block + ) for op in self.main_program.global_block().ops: if op.type != 'conditional_block' or 'Cond' not in op.input_names: continue @@ -173,8 +194,7 @@ def _insert_allreduce_ops_for_gm(self, gm_block): param_vars = [] grad_vars = [] for op in block.ops: - if is_backward_op(op) and \ - OP_ROLE_VAR_KEY in op.attr_names: + if is_backward_op(op) and OP_ROLE_VAR_KEY in op.attr_names: op_role_var = op.attr(OP_ROLE_VAR_KEY) assert len(op_role_var) % 2 == 0 for i in range(0, len(op_role_var), 2): @@ -188,35 +208,41 @@ def _insert_allreduce_ops_for_gm(self, gm_block): if not grad_vars: return - gm_block._insert_op(first_optimize_op_idx, - type="c_sync_calc_stream", - inputs={'X': grad_vars[0]}, - outputs={'Out': grad_vars[0]}, - attrs={OP_ROLE_KEY: OpRole.Backward}) + gm_block._insert_op( + first_optimize_op_idx, + type="c_sync_calc_stream", + inputs={'X': grad_vars[0]}, + outputs={'Out': grad_vars[0]}, + attrs={OP_ROLE_KEY: OpRole.Backward}, + ) insert_op_num = 1 ring_id = self.global_ring_id # NOTE: can perform fuse allreduce inside the loop in the future for i, (p, g) in enumerate(zip(param_vars, grad_vars)): - gm_block._insert_op(first_optimize_op_idx + insert_op_num, - type="c_allreduce_sum", - inputs={'X': g}, - outputs={'Out': g}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Backward, - }) + gm_block._insert_op( + first_optimize_op_idx + insert_op_num, + type="c_allreduce_sum", + inputs={'X': g}, + outputs={'Out': g}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Backward, + }, + ) insert_op_num += 1 - gm_block._insert_op(first_optimize_op_idx + insert_op_num, - type="c_sync_comm_stream", - inputs={'X': grad_vars}, - outputs={'Out': grad_vars}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Backward, - }) + gm_block._insert_op( + first_optimize_op_idx + insert_op_num, + type="c_sync_comm_stream", + inputs={'X': grad_vars}, + outputs={'Out': grad_vars}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Backward, + }, + ) def _transpile_main_program(self, loss): self._insert_loss_grad_ops(loss) @@ -240,14 +266,16 @@ def _insert_loss_grad_ops(self, loss): for idx, op in reversed(list(enumerate(block.ops))): if is_loss_grad_op(op): loss_grad_var = block.vars[op.output_arg_names[0]] - block._insert_op(idx + 1, - type='scale', - inputs={'X': loss_grad_var}, - outputs={'Out': loss_grad_var}, - attrs={ - 'scale': 1.0 / self.nranks, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op( + idx + 1, + type='scale', + inputs={'X': loss_grad_var}, + outputs={'Out': loss_grad_var}, + attrs={ + 'scale': 1.0 / self.nranks, + OP_ROLE_KEY: OpRole.Backward, + }, + ) def _insert_allreduce_ops(self): block = self.main_program.global_block() @@ -255,8 +283,7 @@ def _insert_allreduce_ops(self): grad = None grad_vars = [] for idx, op in reversed(list(enumerate(block.ops))): - if is_backward_op(op) and \ - OP_ROLE_VAR_KEY in op.attr_names: + if is_backward_op(op) and OP_ROLE_VAR_KEY in op.attr_names: op_role_var = op.attr(OP_ROLE_VAR_KEY) if len(op_role_var) == 0: continue @@ -271,36 +298,39 @@ def _insert_allreduce_ops(self): continue grad_vars.append(grad) - block._insert_op(idx + offset, - type='c_sync_calc_stream', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - OP_ROLE_KEY: OpRole.Backward, - }) + block._insert_op( + idx + offset, + type='c_sync_calc_stream', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + OP_ROLE_KEY: OpRole.Backward, + }, + ) offset += 1 - block._insert_op(idx + offset, - type='c_allreduce_sum', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op( + idx + offset, + type='c_allreduce_sum', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Backward, + }, + ) if grad is None: return for idx, op in enumerate(block.ops): if is_optimizer_op(op): - block._insert_op(idx, - type='c_sync_comm_stream', - inputs={'X': grad_vars}, - outputs={'Out': grad_vars}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op( + idx, + type='c_sync_comm_stream', + inputs={'X': grad_vars}, + outputs={'Out': grad_vars}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: OpRole.Backward}, + ) break # This function helps reduce the number of allreduce by integrating op, which can save communication time. @@ -318,16 +348,16 @@ def _allreduce_fusion_program(self): # find all grad params for idx, op in enumerate(block.ops): - if first_backward_idx == -1 and \ - is_backward_op(op): + if first_backward_idx == -1 and is_backward_op(op): first_backward_idx = idx - if is_backward_op(op) and \ - OP_ROLE_VAR_KEY in op.attr_names: + if is_backward_op(op) and OP_ROLE_VAR_KEY in op.attr_names: op_role_var = op.attr(OP_ROLE_VAR_KEY) if len(op_role_var) == 0: continue - assert len(op_role_var) % 2 == 0, "vars need to be one param var followed by one grad var, " \ - "but got odd number of vars" + assert len(op_role_var) % 2 == 0, ( + "vars need to be one param var followed by one grad var, " + "but got odd number of vars" + ) for i in range(0, len(op_role_var), 2): param_name = op_role_var[i] param = block.var(param_name) @@ -338,7 +368,8 @@ def _allreduce_fusion_program(self): param_grads.append((param, grad)) outputs_name_to_idx = self.__get_ouputs_name_to_idx( - first_backward_idx, block) + first_backward_idx, block + ) # structure of grad_param_segments is # [([grad0, grad1], [param0, param1]), ([grad2, grad3], [param2, param3])] @@ -348,9 +379,11 @@ def _allreduce_fusion_program(self): last_dtype = None # split the grad based on dtype and fused size for param, grad in param_grads: - if len(grad_param_segments) == 0 \ - or len(grad_param_segments[-1][0]) == self.fuse_grad_size_in_num \ - or grad.dtype != last_dtype: + if ( + len(grad_param_segments) == 0 + or len(grad_param_segments[-1][0]) == self.fuse_grad_size_in_num + or grad.dtype != last_dtype + ): grad_param_segments.append(([grad], [param])) last_dtype = grad.dtype else: @@ -366,34 +399,40 @@ def _allreduce_fusion_program(self): # not to use reversed since needs the absolute index value grad_segment, param_segment = grad_param_segments[i] # insert coalesce tensor - fused_var = block.create_var(name=unique_name.generate( - 'FusedOutput_{}'.format(grad_segment[0].name)), - dtype=grad_segment[0].dtype, - persistable=False, - stop_gradient=True) + fused_var = block.create_var( + name=unique_name.generate( + 'FusedOutput_{}'.format(grad_segment[0].name) + ), + dtype=grad_segment[0].dtype, + persistable=False, + stop_gradient=True, + ) fused_vars[i] = fused_var after_idx = outputs_name_to_idx[grad_segment[-1]][1] - block._insert_op_without_sync(after_idx + 1, - type='c_allreduce_sum', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': - self.calc_comm_same_stream, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op_without_sync( + after_idx + 1, + type='c_allreduce_sum', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': self.calc_comm_same_stream, + OP_ROLE_KEY: OpRole.Backward, + }, + ) if not self.calc_comm_same_stream: block._insert_op_without_sync( after_idx + 1, type='c_sync_calc_stream', inputs={'X': fused_var}, outputs={'Out': fused_var}, - attrs={OP_ROLE_KEY: OpRole.Backward}) + attrs={OP_ROLE_KEY: OpRole.Backward}, + ) # update the outputs_name_to_idx after insertion of sync/allreduce ops outputs_name_to_idx = self.__get_ouputs_name_to_idx( - first_backward_idx, block) + first_backward_idx, block + ) # the before_idx is not guaranteed sorted, therefore we have to find the # topology to insert the coalesce ops pos_for_coalesce = {} @@ -407,25 +446,26 @@ def _allreduce_fusion_program(self): pos_for_coalesce[i] = before_idx # insert the coalesce op based on the sorted before_idx - pos_for_coalesce = sorted(pos_for_coalesce.items(), - key=lambda kv: (kv[1], kv[0]), - reverse=True) + pos_for_coalesce = sorted( + pos_for_coalesce.items(), + key=lambda kv: (kv[1], kv[0]), + reverse=True, + ) for i, before_idx in pos_for_coalesce: grad_segment, param_segment = grad_param_segments[i] fused_var = fused_vars[i] - block._insert_op_without_sync(before_idx, - type="coalesce_tensor", - inputs={"Input": param_segment}, - outputs={ - "Output": grad_segment, - "FusedOutput": fused_var - }, - attrs={ - "copy_data": False, - "use_align": True, - "dtype": grad_segment[0].dtype, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op_without_sync( + before_idx, + type="coalesce_tensor", + inputs={"Input": param_segment}, + outputs={"Output": grad_segment, "FusedOutput": fused_var}, + attrs={ + "copy_data": False, + "use_align": True, + "dtype": grad_segment[0].dtype, + OP_ROLE_KEY: OpRole.Backward, + }, + ) if self.calc_comm_same_stream: block._sync_with_cpp() @@ -434,14 +474,13 @@ def _allreduce_fusion_program(self): # insert the sync comm op for idx, op in enumerate(block.ops): if is_optimizer_op(op): - block._insert_op_without_sync(idx, - type='c_sync_comm_stream', - inputs={'X': fused_vars}, - outputs={'Out': fused_vars}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op_without_sync( + idx, + type='c_sync_comm_stream', + inputs={'X': fused_vars}, + outputs={'Out': fused_vars}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: OpRole.Backward}, + ) break block._sync_with_cpp() @@ -465,6 +504,8 @@ def __get_ouputs_name_to_idx(self, first_backward_idx, block): # the first idx and the last ids are identical outputs_name_to_idx[var] = (idx, idx) else: - outputs_name_to_idx[var] = (outputs_name_to_idx[var][0], - idx) + outputs_name_to_idx[var] = ( + outputs_name_to_idx[var][0], + idx, + ) return outputs_name_to_idx diff --git a/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py index c9054c793f491..524c3a123abc0 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py @@ -18,7 +18,6 @@ class RecomputeOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(RecomputeOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -32,11 +31,12 @@ def __init__(self, optimizer): ] self.meta_optimizers_black_list = [] - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(RecomputeOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(RecomputeOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) def _init_wrapped_opt(self): if self.wrapped_opt is not None: @@ -56,8 +56,10 @@ def _can_apply(self): return False if self.user_defined_strategy.recompute == True: - if len(self.user_defined_strategy.recompute_configs["checkpoints"] - ) == 0: + if ( + len(self.user_defined_strategy.recompute_configs["checkpoints"]) + == 0 + ): return False else: return True @@ -70,32 +72,33 @@ def _enable_strategy(self, dist_strategy, context): # we do not support automatically recompute checkpoints currently return - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): # maybe inner_opt of other meta optimizer self._init_wrapped_opt() - return self.wrapped_opt.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + return self.wrapped_opt.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) def apply_gradients(self, params_grads): return self.wrapped_opt.apply_gradients(params_grads=params_grads) def apply_optimize(self, loss, startup_program, params_grads): - return self.wrapped_opt.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) + return self.wrapped_opt.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): self._init_wrapped_opt() - optimize_ops, params_grads = \ - self.wrapped_opt.minimize(loss, startup_program, - parameter_list, no_grad_set) + optimize_ops, params_grads = self.wrapped_opt.minimize( + loss, startup_program, parameter_list, no_grad_set + ) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/fp16_helper.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/fp16_helper.py index 126c7d1ca04ba..e4db252cf7c95 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/fp16_helper.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/fp16_helper.py @@ -12,7 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle.distributed.fleet.meta_optimizers.common import is_optimizer_op, OP_ROLE_KEY, OpRole +from paddle.distributed.fleet.meta_optimizers.common import ( + is_optimizer_op, + OP_ROLE_KEY, + OpRole, +) from paddle.fluid import core @@ -20,7 +24,6 @@ class FP16Utils(object): - def __init__(self): pass @@ -30,16 +33,20 @@ def is_fp16_cast_op(block, op, params): return False if is_optimizer_op(op): return False - assert (len(op.desc.input_arg_names()) == 1) - assert (len(op.desc.output_arg_names()) == 1) - input_name, output_name = op.desc.input_arg_names( - )[0], op.desc.output_arg_names()[0] + assert len(op.desc.input_arg_names()) == 1 + assert len(op.desc.output_arg_names()) == 1 + input_name, output_name = ( + op.desc.input_arg_names()[0], + op.desc.output_arg_names()[0], + ) if input_name not in params: return False input_var = block.var(input_name) output_var = block.var(output_name) - if input_var.dtype != core.VarDesc.VarType.FP32 or \ - output_var.dtype != core.VarDesc.VarType.FP16: + if ( + input_var.dtype != core.VarDesc.VarType.FP32 + or output_var.dtype != core.VarDesc.VarType.FP16 + ): return False return True @@ -49,14 +56,18 @@ def is_fp32_cast_op(block, op): return False if not is_optimizer_op(op): return False - assert (len(op.desc.input_arg_names()) == 1) - assert (len(op.desc.output_arg_names()) == 1) - input_name, output_name = op.desc.input_arg_names( - )[0], op.desc.output_arg_names()[0] + assert len(op.desc.input_arg_names()) == 1 + assert len(op.desc.output_arg_names()) == 1 + input_name, output_name = ( + op.desc.input_arg_names()[0], + op.desc.output_arg_names()[0], + ) input_var = block.var(input_name) output_var = block.var(output_name) - if input_var.dtype != core.VarDesc.VarType.FP16 or \ - output_var.dtype != core.VarDesc.VarType.FP32: + if ( + input_var.dtype != core.VarDesc.VarType.FP16 + or output_var.dtype != core.VarDesc.VarType.FP32 + ): return False return True @@ -64,7 +75,8 @@ def is_fp32_cast_op(block, op): def remove_cast_op(block, params, segment, offset): inserted_op_num = 0 for op_idx in reversed( - range(offset + segment._start_idx, offset + segment._end_idx)): + range(offset + segment._start_idx, offset + segment._end_idx) + ): op = block.ops[op_idx] if FP16Utils.is_fp16_cast_op(block, op, params): block._remove_op(op_idx, sync=False) @@ -84,13 +96,16 @@ def prune_fp16(block, shard, reduced_grads_to_param, ring_ids): continue output_name = op.desc.output_arg_names()[0] # TODO (JZ-LIANG) revise this for uniform mixed parallelism - param_name = output_name.strip( - "@GRAD@MERGED" - ) if "@MERGED" in output_name else output_name.strip("@GRAD") + param_name = ( + output_name.strip("@GRAD@MERGED") + if "@MERGED" in output_name + else output_name.strip("@GRAD") + ) if param_name not in shard.global_params: raise ValueError( "Output 'X' of cast_op must be a grad of" - "model param, but {} is not a grad".format(output_name)) + "model param, but {} is not a grad".format(output_name) + ) if output_name in reduced_grads_to_param: continue if shard.has_param(param_name): @@ -117,7 +132,8 @@ def prune_fp16(block, shard, reduced_grads_to_param, ring_ids): if param_name not in shard.global_params: raise ValueError( "Input 'X' of check_finite_and_unscale must" - "be grads, but {} is not a grad".format(input_name)) + "be grads, but {} is not a grad".format(input_name) + ) if shard.has_param(param_name): reversed_x.append(input_name) reversed_x_paramname.append(param_name) @@ -127,55 +143,73 @@ def prune_fp16(block, shard, reduced_grads_to_param, ring_ids): # the grad checking should take the all and only param in the current shard to_check_param = set(reversed_x_paramname) should_check_param = set(shard.global_params).intersection( - set([param for param, worker_idx in shard.global_param2device.items() \ - if worker_idx == shard.worker_idx])) - assert to_check_param == should_check_param, "amp \ + set( + [ + param + for param, worker_idx in shard.global_param2device.items() + if worker_idx == shard.worker_idx + ] + ) + ) + assert ( + to_check_param == should_check_param + ), "amp \ check_finite_and_unscale checking miss [{}] and got unexpected [{}]".format( should_check_param - to_check_param, - to_check_param - should_check_param) + to_check_param - should_check_param, + ) if update_loss_scaling_op_idx == -1: return inf_var = block.var(inf_var_name) - inf_var_int32 = block.create_var(name=inf_var_name + "@cast_int32", - shape=inf_var.shape, - dtype=core.VarDesc.VarType.INT32) - - block._insert_op_without_sync(update_loss_scaling_op_idx, - type='cast', - inputs={'X': inf_var}, - outputs={'Out': inf_var_int32}, - attrs={ - "in_dtype": inf_var.dtype, - "out_dtype": inf_var_int32.dtype, - OP_ROLE_KEY: OpRole.Optimize - }) + inf_var_int32 = block.create_var( + name=inf_var_name + "@cast_int32", + shape=inf_var.shape, + dtype=core.VarDesc.VarType.INT32, + ) + + block._insert_op_without_sync( + update_loss_scaling_op_idx, + type='cast', + inputs={'X': inf_var}, + outputs={'Out': inf_var_int32}, + attrs={ + "in_dtype": inf_var.dtype, + "out_dtype": inf_var_int32.dtype, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) update_loss_scaling_op_idx += 1 # allreduce(mp)->allreduce(sharding)->allreduce(pp) for ring_id in ring_ids: - if ring_id == -1: continue + if ring_id == -1: + continue # this allreduce communication should not overlap with calc - block._insert_op_without_sync(update_loss_scaling_op_idx, - type='c_allreduce_max', - inputs={'X': inf_var_int32}, - outputs={'Out': inf_var_int32}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Optimize - }) + block._insert_op_without_sync( + update_loss_scaling_op_idx, + type='c_allreduce_max', + inputs={'X': inf_var_int32}, + outputs={'Out': inf_var_int32}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) update_loss_scaling_op_idx += 1 - block._insert_op_without_sync(update_loss_scaling_op_idx, - type='cast', - inputs={'X': inf_var_int32}, - outputs={'Out': inf_var}, - attrs={ - "in_dtype": inf_var_int32.dtype, - "out_dtype": inf_var.dtype, - OP_ROLE_KEY: OpRole.Optimize - }) + block._insert_op_without_sync( + update_loss_scaling_op_idx, + type='cast', + inputs={'X': inf_var_int32}, + outputs={'Out': inf_var}, + attrs={ + "in_dtype": inf_var_int32.dtype, + "out_dtype": inf_var.dtype, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) update_loss_scaling_op_idx += 1 block._sync_with_cpp() @@ -197,42 +231,51 @@ def sync_amp_check_nan_inf(block, ring_ids): # 1. inf_var_int32 = allreduce_max(inf_var_int32) # 3. inf_var = cast(inf_var_int32) inf_var = block.var(inf_var_name) - inf_var_int32 = block.create_var(name=inf_var_name + "@cast_int32", - shape=inf_var.shape, - dtype=core.VarDesc.VarType.INT32) - block._insert_op_without_sync(update_loss_scaling_op_idx, - type='cast', - inputs={'X': inf_var}, - outputs={'Out': inf_var_int32}, - attrs={ - "in_dtype": inf_var.dtype, - "out_dtype": inf_var_int32.dtype, - OP_ROLE_KEY: OpRole.Optimize - }) + inf_var_int32 = block.create_var( + name=inf_var_name + "@cast_int32", + shape=inf_var.shape, + dtype=core.VarDesc.VarType.INT32, + ) + block._insert_op_without_sync( + update_loss_scaling_op_idx, + type='cast', + inputs={'X': inf_var}, + outputs={'Out': inf_var_int32}, + attrs={ + "in_dtype": inf_var.dtype, + "out_dtype": inf_var_int32.dtype, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) update_loss_scaling_op_idx += 1 # allreduce(mp)->allreduce(pp) for ring_id in ring_ids: - if ring_id == -1: continue - block._insert_op_without_sync(update_loss_scaling_op_idx, - type='c_allreduce_max', - inputs={'X': inf_var_int32}, - outputs={'Out': inf_var_int32}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Optimize - }) + if ring_id == -1: + continue + block._insert_op_without_sync( + update_loss_scaling_op_idx, + type='c_allreduce_max', + inputs={'X': inf_var_int32}, + outputs={'Out': inf_var_int32}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) update_loss_scaling_op_idx += 1 - block._insert_op_without_sync(update_loss_scaling_op_idx, - type='cast', - inputs={'X': inf_var_int32}, - outputs={'Out': inf_var}, - attrs={ - "in_dtype": inf_var_int32.dtype, - "out_dtype": inf_var.dtype, - OP_ROLE_KEY: OpRole.Optimize - }) + block._insert_op_without_sync( + update_loss_scaling_op_idx, + type='cast', + inputs={'X': inf_var_int32}, + outputs={'Out': inf_var}, + attrs={ + "in_dtype": inf_var_int32.dtype, + "out_dtype": inf_var.dtype, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) update_loss_scaling_op_idx += 1 block._sync_with_cpp() diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/gradient_clip_helper.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/gradient_clip_helper.py index 03d955842f5fc..563757d35f405 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/gradient_clip_helper.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/gradient_clip_helper.py @@ -18,13 +18,13 @@ class GradientClipHelper(object): - def __init__(self, mp_ring_id): self.mp_ring_id = mp_ring_id def _is_gradient_clip_op(self, op): - return op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/gradient_clip") + return op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/gradient_clip") def prune_gradient_clip(self, block, shard, ring_ids): """ @@ -51,8 +51,9 @@ def prune_gradient_clip(self, block, shard, ring_ids): param_name = input_name.strip("@GRAD@MERGED") else: param_name = input_name.strip("@GRAD") - if shard.is_param(param_name) and \ - not shard.has_param(param_name): + if shard.is_param(param_name) and not shard.has_param( + param_name + ): deperate_op = True elif shard.is_param(param_name): reversed_x_paramname.append(param_name) @@ -84,7 +85,7 @@ def prune_gradient_clip(self, block, shard, ring_ids): reversed_inputs.append(input_name) op.desc.set_input("X", reversed_inputs) - assert (len(op.desc.output_arg_names()) == 1) + assert len(op.desc.output_arg_names()) == 1 sum_res = op.desc.output_arg_names()[0] # NOTE(wangxi): If we have 2 param, but sharding is 4, @@ -96,26 +97,25 @@ def prune_gradient_clip(self, block, shard, ring_ids): namescope = op.attr("op_namescope") block._remove_op(idx, sync=False) - op = block._insert_op_without_sync(idx, - type='fill_constant', - inputs={}, - outputs={'Out': sum_res}, - attrs={ - 'shape': - sum_var.shape, - 'dtype': - sum_var.dtype, - 'value': - 0.0, - OP_ROLE_KEY: - OpRole.Optimize - }) + op = block._insert_op_without_sync( + idx, + type='fill_constant', + inputs={}, + outputs={'Out': sum_res}, + attrs={ + 'shape': sum_var.shape, + 'dtype': sum_var.dtype, + 'value': 0.0, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) op._set_attr('op_namescope', namescope) # allreduce(mp)->allreduce(sharding)->allreduce(pp) idx_offset = 1 for ring_id in ring_ids: - if ring_id == -1: continue + if ring_id == -1: + continue # this allreduce should not overlap with calc and should be scheduled in calc stream block._insert_op_without_sync( idx + idx_offset, @@ -127,18 +127,28 @@ def prune_gradient_clip(self, block, shard, ring_ids): 'op_namescope': "/gradient_clip_model_parallelism", 'use_calc_stream': True, OP_ROLE_KEY: OpRole.Optimize, - }) + }, + ) idx_offset += 1 # the grad sum here should take the all and only param in the current shard to_check_param = set(reversed_x_paramname) - should_check_param = set(shard.global_params).intersection(set( - [param for param, worker_idx in shard.global_param2device.items() \ - if worker_idx == shard.worker_idx])) - assert to_check_param == should_check_param, "amp check_finite_and_unscale \ + should_check_param = set(shard.global_params).intersection( + set( + [ + param + for param, worker_idx in shard.global_param2device.items() + if worker_idx == shard.worker_idx + ] + ) + ) + assert ( + to_check_param == should_check_param + ), "amp check_finite_and_unscale \ checking miss [{}] and got unexpected [{}]".format( should_check_param - to_check_param, - to_check_param - should_check_param) + to_check_param - should_check_param, + ) for var_name in deperated_vars: block._remove_var(var_name, sync=False) @@ -178,8 +188,12 @@ def sync_global_norm(self, block, ring_ids, mp_rank): # by global norm. Those vars either doesn't have is_distributed attr # or the is_distributed attr has been set as False. # Therefore, we prune those duplicated vars for grad clip. - if mp_rank >= 1 and (not (hasattr(input_var, 'is_distributed') - and input_var.is_distributed)): + if mp_rank >= 1 and ( + not ( + hasattr(input_var, 'is_distributed') + and input_var.is_distributed + ) + ): removed_op_idx.add(idx) for output_name in op.output_arg_names: removed_tmp_var.add(output_name) @@ -222,8 +236,9 @@ def sync_global_norm(self, block, ring_ids, mp_rank): 'shape': sum_rst_var.shape, 'dtype': sum_rst_var.dtype, 'value': 0.0, - OP_ROLE_KEY: OpRole.Optimize - }) + OP_ROLE_KEY: OpRole.Optimize, + }, + ) fill_constant_op._set_attr('op_namescope', namescope) self._insert_allreduce(block, ring_ids, idx, sum_rst_var) break @@ -245,4 +260,5 @@ def _insert_allreduce(block, ring_ids, idx, var): 'op_namescope': "/gradient_clip_model_parallelism", 'use_calc_stream': True, OP_ROLE_KEY: OpRole.Optimize, - }) + }, + ) diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/offload_helper.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/offload_helper.py index 968709717d1b0..ac10bb42383d1 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/offload_helper.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/offload_helper.py @@ -56,21 +56,25 @@ def __init__(self, mp_ring_id=None, dp_ring_id=None): def _insert_cast_op(self, block, idx, src_name, dst_name): src_var = block.var(src_name) if not block.has_var(dst_name): - block.create_var(name=dst_name, - shape=src_var.shape, - dtype=core.VarDesc.VarType.FP16, - persistable=True) + block.create_var( + name=dst_name, + shape=src_var.shape, + dtype=core.VarDesc.VarType.FP16, + persistable=True, + ) dst_var = block.var(dst_name) assert dst_var.dtype == core.VarDesc.VarType.FP16 - block._insert_op_without_sync(idx, - type='cast', - inputs={'X': src_var}, - outputs={'Out': dst_var}, - attrs={ - 'in_dtype': src_var.dtype, - 'out_dtype': dst_var.dtype, - OP_ROLE_KEY: OpRole.Optimize - }) + block._insert_op_without_sync( + idx, + type='cast', + inputs={'X': src_var}, + outputs={'Out': dst_var}, + attrs={ + 'in_dtype': src_var.dtype, + 'out_dtype': dst_var.dtype, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) def _insert_broadcast_op(self, block, idx, param_name): rings = [] @@ -86,36 +90,42 @@ def _insert_broadcast_op(self, block, idx, param_name): # the insert op order is: mp, dp for ring in rings: - block._insert_op_without_sync(idx, - type="c_broadcast", - inputs={'X': param_name}, - outputs={'Out': param_name}, - attrs={ - 'ring_id': ring, - 'root': 0, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward, - }) + block._insert_op_without_sync( + idx, + type="c_broadcast", + inputs={'X': param_name}, + outputs={'Out': param_name}, + attrs={ + 'ring_id': ring, + 'root': 0, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) def _insert_memcpy_op(self, block, idx, src_name, dst_name, dst_place_type): src_var = block.var(src_name) dst_var = block.var(dst_name) - block._insert_op_without_sync(idx, - type='memcpy', - inputs={'X': src_var}, - outputs={'Out': dst_var}, - attrs={ - 'dst_place_type': dst_place_type, - OP_ROLE_KEY: OpRole.Optimize, - }) + block._insert_op_without_sync( + idx, + type='memcpy', + inputs={'X': src_var}, + outputs={'Out': dst_var}, + attrs={ + 'dst_place_type': dst_place_type, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) def _insert_fetch_op(self, block, idx, src_name, dst_name): - self._insert_memcpy_op(block, idx, src_name, dst_name, - OffloadHelper.cuda_place_type) + self._insert_memcpy_op( + block, idx, src_name, dst_name, OffloadHelper.cuda_place_type + ) def _insert_offload_op(self, block, idx, src_name, dst_name): - self._insert_memcpy_op(block, idx, src_name, dst_name, - OffloadHelper.cuda_pinned_place_type) + self._insert_memcpy_op( + block, idx, src_name, dst_name, OffloadHelper.cuda_pinned_place_type + ) def _get_offload_var_name(self, name): return unique_name.generate(name + '@offload') @@ -124,10 +134,12 @@ def _create_offload_var(self, var_name, offload_var_name, blocks): for block in blocks: var = block.var(var_name) var.persistable = False - offload_var = block.create_var(name=offload_var_name, - shape=var.shape, - dtype=var.dtype, - persistable=True) + offload_var = block.create_var( + name=offload_var_name, + shape=var.shape, + dtype=var.dtype, + persistable=True, + ) def offload_fp32param(self, block, startup_block, offload=True): """ @@ -188,13 +200,15 @@ def remove_param(input_name): if 'subprog' not in output_name: assert output_name == input_name + '.cast_fp16' - assert input_name not in param_to_fp16, \ - "There must be only one cast op from fp32 param to fp16 param." + assert ( + input_name not in param_to_fp16 + ), "There must be only one cast op from fp32 param to fp16 param." param_to_fp16[input_name] = output_name else: # fp16-->recompute_var - assert input_name in param_to_fp16, \ - "param must first be cast to fp16" + assert ( + input_name in param_to_fp16 + ), "param must first be cast to fp16" fp16_param = param_to_fp16[input_name] fp16_param_to_recompute[fp16_param] = output_name recompute_to_fp16[output_name] = fp16_param @@ -205,24 +219,28 @@ def remove_param(input_name): for idx, op in reversed(list(enumerate(block.ops))): if is_update_op(op): param = op.desc.input("Param")[0] - if param not in param_to_idx: continue + if param not in param_to_idx: + continue # step3.1: create offload_var offload_var_name = self._get_offload_var_name(param) param_name_to_offload_name[param] = offload_var_name if offload: - self._create_offload_var(param, offload_var_name, - [block, startup_block]) + self._create_offload_var( + param, offload_var_name, [block, startup_block] + ) # step3.2: insert cast op and offload op - self._insert_offload_op(block, idx + 1, param, - offload_var_name) + self._insert_offload_op( + block, idx + 1, param, offload_var_name + ) assert param in param_to_fp16 fp16_param_name = param_to_fp16[param] fp16_param_var = block.var(fp16_param_name) fp16_param_var.persistable = True - self._insert_cast_op(block, idx + 1, param, - param_to_fp16[param]) + self._insert_cast_op( + block, idx + 1, param, param_to_fp16[param] + ) if offload: # step3.3: insert fetch op @@ -242,8 +260,9 @@ def remove_param(input_name): op._rename_input(input_name, recompute_to_fp16[input_name]) for output_name in op.desc.output_arg_names(): if output_name in recompute_to_fp16: - op._rename_output(output_name, - recompute_to_fp16[output_name]) + op._rename_output( + output_name, recompute_to_fp16[output_name] + ) # step4: remove recompute_param for name in recompute_to_fp16.keys(): @@ -262,14 +281,23 @@ def remove_param(input_name): var_name = out_name if offload: offload_var_name = param_name_to_offload_name[var_name] - self._insert_offload_op(startup_block, insert_idx, - var_name, offload_var_name) - self._insert_cast_op(startup_block, insert_idx, var_name, - param_to_fp16[var_name]) + self._insert_offload_op( + startup_block, + insert_idx, + var_name, + offload_var_name, + ) + self._insert_cast_op( + startup_block, + insert_idx, + var_name, + param_to_fp16[var_name], + ) # NOTE(wangxi): cast and offload should insert after broadcast param. # the insert op order is: {mp, dp}broadcast, cast, offload - self._insert_broadcast_op(startup_block, insert_idx, - var_name) + self._insert_broadcast_op( + startup_block, insert_idx, var_name + ) visited_vars.add(out_name) @@ -322,14 +350,16 @@ def offload(self, block, startup_block): offload_var_name = self._get_offload_var_name(var_name) vars_name_to_offload_name[var_name] = offload_var_name - self._create_offload_var(var_name, offload_var_name, - [block, startup_block]) + self._create_offload_var( + var_name, offload_var_name, [block, startup_block] + ) # step2: insert offload op for var_name in vars_name: offload_var_name = vars_name_to_offload_name[var_name] - self._insert_offload_op(block, idx + 1, var_name, - offload_var_name) + self._insert_offload_op( + block, idx + 1, var_name, offload_var_name + ) # step3: insert fetch op for var_name in vars_name: @@ -347,18 +377,17 @@ def offload(self, block, startup_block): var_name = out_name offload_var_name = vars_name_to_offload_name[var_name] # insert offload op after var is generated - self._insert_offload_op(startup_block, idx + 1, var_name, - offload_var_name) + self._insert_offload_op( + startup_block, idx + 1, var_name, offload_var_name + ) visited_vars.add(out_name) block._sync_with_cpp() startup_block._sync_with_cpp() - def opt_sharding_cast_fp32param(self, - block, - startup_block, - params, - offload=False): + def opt_sharding_cast_fp32param( + self, block, startup_block, params, offload=False + ): """ (p_fp16) = cast(p) (p_fp16_recompute) = cast(p) @@ -420,13 +449,15 @@ def remove_param(input_name): if 'subprog' not in output_name: assert output_name == input_name + '.cast_fp16' - assert input_name not in param_to_fp16, \ - "There must be only one cast op from fp32 param to fp16 param." + assert ( + input_name not in param_to_fp16 + ), "There must be only one cast op from fp32 param to fp16 param." param_to_fp16[input_name] = output_name else: # fp16-->recompute_var - assert input_name in param_to_fp16, \ - "param must first be cast to fp16" + assert ( + input_name in param_to_fp16 + ), "param must first be cast to fp16" fp16_param = param_to_fp16[input_name] fp16_param_to_recompute[fp16_param] = output_name recompute_to_fp16[output_name] = fp16_param @@ -443,19 +474,22 @@ def remove_param(input_name): offload_var_name = self._get_offload_var_name(param) param_name_to_offload_name[param] = offload_var_name if offload: - self._create_offload_var(param, offload_var_name, - [block, startup_block]) + self._create_offload_var( + param, offload_var_name, [block, startup_block] + ) # step3.2: insert cast op and offload op - self._insert_offload_op(block, idx + 1, param, - offload_var_name) + self._insert_offload_op( + block, idx + 1, param, offload_var_name + ) assert param in param_to_fp16 fp16_param_name = param_to_fp16[param] fp16_param_var = block.var(fp16_param_name) fp16_param_var.persistable = True - self._insert_cast_op(block, idx + 1, param, - param_to_fp16[param]) + self._insert_cast_op( + block, idx + 1, param, param_to_fp16[param] + ) if offload: # step3.3: insert fetch op @@ -476,8 +510,9 @@ def remove_param(input_name): op._rename_input(input_name, recompute_to_fp16[input_name]) for output_name in op.desc.output_arg_names(): if output_name in recompute_to_fp16: - op._rename_output(output_name, - recompute_to_fp16[output_name]) + op._rename_output( + output_name, recompute_to_fp16[output_name] + ) # step4: remove recompute_param for name in recompute_to_fp16.keys(): @@ -508,22 +543,31 @@ def remove_param(input_name): insert_idx = len(startup_block.ops) for idx, op in reversed(list(enumerate(startup_block.ops))): for out_name in op.output_arg_names: - if out_name in visited_vars: continue + if out_name in visited_vars: + continue if out_name in param_to_fp16: var_name = out_name if offload: self._insert_offload_op( - startup_block, idx + 1, var_name, - param_name_to_offload_name[var_name]) - - self._insert_cast_op(startup_block, insert_idx, var_name, - param_to_fp16[var_name]) + startup_block, + idx + 1, + var_name, + param_name_to_offload_name[var_name], + ) + + self._insert_cast_op( + startup_block, + insert_idx, + var_name, + param_to_fp16[var_name], + ) # NOTE(wangxi): cast and offload should insert after broadcast param. # the insert op order is: {mp, dp}broadcast, cast, offload - self._insert_broadcast_op(startup_block, insert_idx, - var_name) + self._insert_broadcast_op( + startup_block, insert_idx, var_name + ) if var_name not in local_params: param = startup_block.var(out_name) diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/prune.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/prune.py index adbc00f25deb6..895fd2f7acb27 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/prune.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/prune.py @@ -16,7 +16,6 @@ class ProgramDeps(object): - def __init__(self, block, start_vars, end_vars): self._block = block # vars where to start to build the deps @@ -45,7 +44,9 @@ def get_var_deps(self, var_name): else: return None - def _build_deps(self, ): + def _build_deps( + self, + ): for var_name in self._start_vars: self._var_to_use_op[var_name] = [] @@ -53,8 +54,9 @@ def _build_deps(self, ): for idx, op in enumerate(self._block.ops): if op.type in [ - "c_allreduce_sum", "c_sync_comm_stream", - "c_calc_comm_stream" + "c_allreduce_sum", + "c_sync_comm_stream", + "c_calc_comm_stream", ]: continue input_vars = op.desc.input_arg_names() @@ -77,11 +79,13 @@ def _build_deps(self, ): self._var_to_generate_op[output_name].append(idx) if op.type == "conditional_block": # subblock - assert (op.desc.has_attr("sub_block")) + assert op.desc.has_attr("sub_block") subblock_idx = op.desc.attr("sub_block").id subblock_deps = ProgramDeps( self._block.program.block(subblock_idx), - op.desc.input_arg_names(), op.desc.output_arg_names()) + op.desc.input_arg_names(), + op.desc.output_arg_names(), + ) self._sub_block_deps[subblock_idx] = subblock_deps subblock_deps._father_block_deps = self @@ -93,17 +97,24 @@ def crop_input_var_from_op(self, op_idx, var_name): raise ValueError( "op_idx: {} is not in self._var_to_use_op[{}], " "self._var_to_use_op[{}] is {}".format( - op_idx, var_name, var_name, - self._var_to_use_op[var_name])) + op_idx, + var_name, + var_name, + self._var_to_use_op[var_name], + ) + ) self._var_to_use_op[var_name].remove(op_idx) # update _should_removed_var if var_name in self._start_vars: self._should_removed_var.discard(var_name) - elif self._var_to_use_op[ - var_name] == []: # no more deps of this var + elif ( + self._var_to_use_op[var_name] == [] + ): # no more deps of this var self._should_removed_var.add(var_name) - elif self._var_to_generate_op[var_name][-1] >= self._var_to_use_op[ - var_name][-1]: + elif ( + self._var_to_generate_op[var_name][-1] + >= self._var_to_use_op[var_name][-1] + ): # there are circle in the graph self._should_removed_var.add(var_name) else: # input_name should not be deleted @@ -111,11 +122,13 @@ def crop_input_var_from_op(self, op_idx, var_name): def crop_output_var_from_op(self, op_idx, var_name): if var_name in self._var_to_generate_op: - assert (op_idx in self._var_to_generate_op[var_name]) + assert op_idx in self._var_to_generate_op[var_name] self._var_to_generate_op[var_name].remove(op_idx) if self._block.has_var(var_name): - if var_name not in self._var_to_generate_op or self._var_to_generate_op[ - var_name] == []: + if ( + var_name not in self._var_to_generate_op + or self._var_to_generate_op[var_name] == [] + ): self._block._remove_var(var_name, sync=False) def remove_op(self, op_idx, reserved_vars=None): diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/shard.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/shard.py index e6490d62a5d59..d33d04098d051 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/shard.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/shard.py @@ -15,14 +15,17 @@ import re from paddle.distributed.fleet.meta_optimizers.common import is_optimizer_op from paddle.distributed.fleet.meta_optimizers.sharding.utils import get_var_size -from paddle.distributed.fleet.meta_optimizers.sharding.fp16_helper import FP16Utils +from paddle.distributed.fleet.meta_optimizers.sharding.fp16_helper import ( + FP16Utils, +) __all__ = [] class Shard(object): - - def __init__(self, ): + def __init__( + self, + ): self.global_params = set([]) self.worker_idx = -1 self.worker_num = -1 @@ -37,19 +40,25 @@ def setup(self, params_grads, worker_idx, worker_num): self.worker_num = worker_num # global_param2device contains fp32 params and fp16 params # device2global_params only contains fp32 params - self.global_param2device, self.device2global_params \ - = self._split_params(params_grads, worker_idx, worker_num) + ( + self.global_param2device, + self.device2global_params, + ) = self._split_params(params_grads, worker_idx, worker_num) def has_param(self, var_name): - return var_name in self.global_param2device and \ - self._var_device_id(var_name) == self.worker_idx + return ( + var_name in self.global_param2device + and self._var_device_id(var_name) == self.worker_idx + ) def has_opt_var(self, var_name): return self._var_device_id(var_name) == self.worker_idx def has_var(self, var_name): - return self._var_device_id(var_name) == -1 or \ - self._var_device_id(var_name) == self.worker_idx + return ( + self._var_device_id(var_name) == -1 + or self._var_device_id(var_name) == self.worker_idx + ) def _split_params(self, params_grads, worker_idx, worker_num): param2device = {} @@ -74,8 +83,11 @@ def _var_device_id(self, var_name): if var_name in self.global_param2device: return self.global_param2device[var_name] for suffix in [ - "_moment1_0", "_moment2_0", "_beta1_pow_acc_0", - "_beta2_pow_acc_0", "_velocity_0" + "_moment1_0", + "_moment2_0", + "_beta1_pow_acc_0", + "_beta2_pow_acc_0", + "_velocity_0", ]: base_name = re.sub(suffix, '', var_name) if base_name in self.global_param2device: @@ -105,7 +117,8 @@ def find_broadcast_params(self, block): fp16_to_fp32[output_name] = input_name param_usage[input_name] -= 1 self.global_param2device[output_name] = self.global_param2device[ - input_name] + input_name + ] for param, usage in param_usage.items(): if usage > 0: @@ -122,8 +135,11 @@ def is_opti_var(self, var_name): if var_name in self.global_params: return True for suffix in [ - "_moment1_0", "_moment2_0", "_beta1_pow_acc_0", - "_beta2_pow_acc_0", "_velocity_0" + "_moment1_0", + "_moment2_0", + "_beta1_pow_acc_0", + "_beta2_pow_acc_0", + "_velocity_0", ]: base_name = re.sub(suffix, '', var_name) if base_name in self.global_params: @@ -140,7 +156,6 @@ def filter_grads(self, grads): class ProgramSegment(object): - def __init__(self, block): self._block = block self._allreduce_vars = [] diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py index b5f72aac893e0..1ec9457854dce 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/utils.py @@ -14,7 +14,11 @@ import paddle from paddle.fluid import core, unique_name from functools import reduce -from paddle.distributed.fleet.meta_optimizers.common import is_loss_grad_op, is_backward_op, is_optimizer_op +from paddle.distributed.fleet.meta_optimizers.common import ( + is_loss_grad_op, + is_backward_op, + is_optimizer_op, +) from paddle.distributed.fleet.meta_optimizers.common import OP_ROLE_KEY, OpRole import re @@ -42,7 +46,10 @@ def check_broadcast(block): "var_name areadly exist: {}" "the old pos is {}, the new pos is {}".format( var_name, - broadcast_vars[var_name]["broadcast_pos"], idx)) + broadcast_vars[var_name]["broadcast_pos"], + idx, + ) + ) broadcast_vars[var_name] = { "fill_constant_pos": -1, "broadcast_pos": idx, @@ -69,17 +76,21 @@ def check_broadcast(block): var_name = op.desc.input_arg_names()[0] if "@BroadCast" in var_name: if broadcast_vars[var_name]["fill_constant_pos"] != -1: - assert (last_sync_calc_op_idx != -1) - assert (broadcast_vars[var_name]["fill_constant_pos"] < - last_sync_calc_op_idx) - assert (last_sync_calc_op_idx < idx) + assert last_sync_calc_op_idx != -1 + assert ( + broadcast_vars[var_name]["fill_constant_pos"] + < last_sync_calc_op_idx + ) + assert last_sync_calc_op_idx < idx continue for input_name in op.desc.input_arg_names(): if input_name in broadcast_vars: - assert (broadcast_vars[input_name]["broadcast_pos"] != -1) - assert (broadcast_vars[input_name]["broadcast_pos"] < - last_sync_comm_op_idx) - assert (last_sync_comm_op_idx < idx) + assert broadcast_vars[input_name]["broadcast_pos"] != -1 + assert ( + broadcast_vars[input_name]["broadcast_pos"] + < last_sync_comm_op_idx + ) + assert last_sync_comm_op_idx < idx return @@ -135,8 +146,10 @@ def check_allreduce_sum(block, shard, sharding_ring_id, dp_ring_id=-1): if var_name in vars_status and vars_status[var_name] == 0: vars_status[var_name] = 1 for var_name in dp_grads_status: - if var_name in dp_grads_status and dp_grads_status[ - var_name] == 0: + if ( + var_name in dp_grads_status + and dp_grads_status[var_name] == 0 + ): dp_grads_status[var_name] = 1 # check sharding allreduce and reduce but skip megatron allreduce elif op.type == "c_allreduce_sum" or op.type == "c_reduce_sum": @@ -144,7 +157,9 @@ def check_allreduce_sum(block, shard, sharding_ring_id, dp_ring_id=-1): var_name = op.desc.input_arg_names()[0] ring_id = op.desc.attr("ring_id") if ring_id == sharding_ring_id: - assert op.type == "c_reduce_sum", "Grad in Sharding group should be reduce rather than allreduce" + assert ( + op.type == "c_reduce_sum" + ), "Grad in Sharding group should be reduce rather than allreduce" if var_name in vars_status: _status = vars_status[var_name] else: @@ -152,12 +167,15 @@ def check_allreduce_sum(block, shard, sharding_ring_id, dp_ring_id=-1): if _status == -1: raise ValueError( "{} is not generated, but you are" - "trying to all-reduce it".format(var_name)) + "trying to all-reduce it".format(var_name) + ) if _status == 0: - raise ValueError("There should be a sync_calc op " - "after generate Var: {} and before the" - "c_allreduce_sum op".format(var_name)) - assert (_status == 1) + raise ValueError( + "There should be a sync_calc op " + "after generate Var: {} and before the" + "c_allreduce_sum op".format(var_name) + ) + assert _status == 1 if var_name in vars_status: vars_status[var_name] = 2 else: @@ -193,29 +211,37 @@ def check_allreduce_sum(block, shard, sharding_ring_id, dp_ring_id=-1): if vars_status[input_name] != 3: raise ValueError( "There should be a sync_comm op " - "after allreduce the Var: {}".format(input_name)) + "after allreduce the Var: {}".format(input_name) + ) raise ValueError( - "The reduce output grad [{}] should NOT be be used in Non-root rank." - .format(input_name)) + "The reduce output grad [{}] should NOT be be used in Non-root rank.".format( + input_name + ) + ) if input_name in dp_grads_status: if dp_ring_id == -1: if dp_grads_status[input_name] != 3: raise ValueError( "There should be a sync_comm op " - "after allreduce the Var: {}".format( - input_name)) + "after allreduce the Var: {}".format(input_name) + ) else: if dp_grads_status[input_name] != 5: raise ValueError( "The grad in shard should be allreduce and sync" - "twice before usage {}".format(input_name)) + "twice before usage {}".format(input_name) + ) for output_name in op.desc.output_arg_names(): - if output_name in vars_status and \ - vars_status[output_name] == -1: + if ( + output_name in vars_status + and vars_status[output_name] == -1 + ): vars_status[output_name] = 0 - if output_name in dp_grads_status and \ - dp_grads_status[output_name] == -1: + if ( + output_name in dp_grads_status + and dp_grads_status[output_name] == -1 + ): dp_grads_status[output_name] = 0 # check sharding with amp @@ -234,9 +260,9 @@ def get_valid_op_role(block, insert_idx): return OpRole.Forward or OpRole.Backward """ op_role = block.ops[insert_idx].attr('op_role') - if (insert_idx >= len(block.ops)) or (op_role in [ - int(OpRole.Backward), int(OpRole.Optimize) - ]): + if (insert_idx >= len(block.ops)) or ( + op_role in [int(OpRole.Backward), int(OpRole.Optimize)] + ): return OpRole.Backward if op_role in [int(OpRole.Forward), int(OpRole.Loss)]: return OpRole.Forward @@ -249,11 +275,13 @@ def insert_sync_calc_op(block, insert_idx, calc_dep_vars): _insert_sync_calc_op """ op_role = get_valid_op_role(block, insert_idx) - block._insert_op_without_sync(insert_idx, - type='c_sync_calc_stream', - inputs={'X': calc_dep_vars}, - outputs={'Out': calc_dep_vars}, - attrs={OP_ROLE_KEY: op_role}) + block._insert_op_without_sync( + insert_idx, + type='c_sync_calc_stream', + inputs={'X': calc_dep_vars}, + outputs={'Out': calc_dep_vars}, + attrs={OP_ROLE_KEY: op_role}, + ) return @@ -262,14 +290,13 @@ def insert_sync_comm_op(block, insert_idx, ring_id, comm_dep_vars): insert sync_comm_op for single var """ op_role = get_valid_op_role(block, insert_idx) - block._insert_op_without_sync(insert_idx, - type='c_sync_comm_stream', - inputs={'X': comm_dep_vars}, - outputs={'Out': comm_dep_vars}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type='c_sync_comm_stream', + inputs={'X': comm_dep_vars}, + outputs={'Out': comm_dep_vars}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: op_role}, + ) return 1 @@ -282,14 +309,13 @@ def insert_sync_comm_ops(block, insert_idx, ring_id, comm_dep_vars): return 0 op_role = get_valid_op_role(block, insert_idx) - block._insert_op_without_sync(insert_idx, - type='c_sync_comm_stream', - inputs={'X': comm_dep_vars}, - outputs={'Out': comm_dep_vars}, - attrs={ - 'ring_id': int(ring_id), - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type='c_sync_comm_stream', + inputs={'X': comm_dep_vars}, + outputs={'Out': comm_dep_vars}, + attrs={'ring_id': int(ring_id), OP_ROLE_KEY: op_role}, + ) return 1 @@ -300,15 +326,17 @@ def insert_fill_constant_ops(block, insert_idx, fill_constant_vars): op_role = get_valid_op_role(block, insert_idx) for broadcast_name in fill_constant_vars: broadcast_var = block.var(broadcast_name) - block._insert_op_without_sync(insert_idx, - type="fill_constant", - outputs={"Out": broadcast_var.name}, - attrs={ - "shape": broadcast_var.shape, - "dtype": broadcast_var.dtype, - "value": 0.0, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type="fill_constant", + outputs={"Out": broadcast_var.name}, + attrs={ + "shape": broadcast_var.shape, + "dtype": broadcast_var.dtype, + "value": 0.0, + OP_ROLE_KEY: op_role, + }, + ) return @@ -318,58 +346,69 @@ def insert_cast_ops(block, insert_idx, cast_ops): """ op_role = get_valid_op_role(block, insert_idx) for fp16_name, fp32_name in cast_ops.items(): - block._insert_op_without_sync(insert_idx, - type="cast", - inputs={"X": fp32_name}, - outputs={"Out": fp16_name}, - attrs={ - "in_dtype": core.VarDesc.VarType.FP32, - "out_dtype": - core.VarDesc.VarType.FP16, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type="cast", + inputs={"X": fp32_name}, + outputs={"Out": fp16_name}, + attrs={ + "in_dtype": core.VarDesc.VarType.FP32, + "out_dtype": core.VarDesc.VarType.FP16, + OP_ROLE_KEY: op_role, + }, + ) return -def insert_allreduce_ops(block, - insert_idx, - ring_id, - allreduce_vars, - op_role=OpRole.Backward, - use_calc_stream=False, - user_defined_strategy=None): +def insert_allreduce_ops( + block, + insert_idx, + ring_id, + allreduce_vars, + op_role=OpRole.Backward, + use_calc_stream=False, + user_defined_strategy=None, +): """ _add_allreduce_ops """ if len(allreduce_vars) == 0: return - if user_defined_strategy and \ - user_defined_strategy.fuse_all_reduce_ops and \ - not user_defined_strategy.fuse_grad_merge: + if ( + user_defined_strategy + and user_defined_strategy.fuse_all_reduce_ops + and not user_defined_strategy.fuse_grad_merge + ): # If fuse_grad_merge is enable, the grad vars have already been fused during # gradient merge pass, therefore, those vars are not need to be fused here - insert_fused_allreduce_ops(block, insert_idx, ring_id, allreduce_vars, - op_role, use_calc_stream, - user_defined_strategy.fuse_grad_size_in_MB) + insert_fused_allreduce_ops( + block, + insert_idx, + ring_id, + allreduce_vars, + op_role, + use_calc_stream, + user_defined_strategy.fuse_grad_size_in_MB, + ) else: for var in allreduce_vars: - block._insert_op_without_sync(insert_idx, - type='c_allreduce_sum', - inputs={'X': var}, - outputs={'Out': var}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': - use_calc_stream, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type='c_allreduce_sum', + inputs={'X': var}, + outputs={'Out': var}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + OP_ROLE_KEY: op_role, + }, + ) return class FuseHelper(object): - @staticmethod def sort_vars_by_dtype(block, vars_name): fp32_vars = [] @@ -389,17 +428,19 @@ def sort_vars_by_dtype(block, vars_name): return fp32_vars @staticmethod - def get_fused_groups(block, vars_name, fuse_size=32.): - """ coalesce tensor, get fused group """ + def get_fused_groups(block, vars_name, fuse_size=32.0): + """coalesce tensor, get fused group""" groups = [] - cur_size = 0. + cur_size = 0.0 last_dtype = None for var_name in vars_name: real_var = block.var(var_name) var_size = get_var_size(real_var) - if cur_size + var_size > fuse_size \ - or len(groups) == 0 \ - or real_var.dtype != last_dtype: + if ( + cur_size + var_size > fuse_size + or len(groups) == 0 + or real_var.dtype != last_dtype + ): groups.append([real_var]) cur_size = var_size last_dtype = real_var.dtype @@ -409,11 +450,9 @@ def get_fused_groups(block, vars_name, fuse_size=32.): return groups @staticmethod - def insert_coalesce_tensor(block, - index, - groups, - op_role=OpRole.Backward, - prefix="Output"): + def insert_coalesce_tensor( + block, index, groups, op_role=OpRole.Backward, prefix="Output" + ): fused_vars = [] insert_num = 0 for group in groups: @@ -423,247 +462,305 @@ def insert_coalesce_tensor(block, fused_vars.append(group[0]) continue - fused_var = block.create_var(name=unique_name.generate( - 'Fused{}_{}'.format(prefix, group[0].name)), - dtype=group[0].dtype, - persistable=False, - stop_gradient=True) + fused_var = block.create_var( + name=unique_name.generate( + 'Fused{}_{}'.format(prefix, group[0].name) + ), + dtype=group[0].dtype, + persistable=False, + stop_gradient=True, + ) fused_vars.append(fused_var) - block._insert_op_without_sync(index, - type="coalesce_tensor", - inputs={"Input": group}, - outputs={ - "Output": group, - "FusedOutput": fused_var - }, - attrs={ - "copy_data": True, - "use_align": True, - "dtype": group[0].dtype, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + index, + type="coalesce_tensor", + inputs={"Input": group}, + outputs={"Output": group, "FusedOutput": fused_var}, + attrs={ + "copy_data": True, + "use_align": True, + "dtype": group[0].dtype, + OP_ROLE_KEY: op_role, + }, + ) insert_num += 1 return fused_vars, insert_num -def insert_fused_allreduce_ops(block, - insert_idx, - ring_id, - allreduce_vars, - op_role=OpRole.Backward, - use_calc_stream=False, - fuse_grad_size_in_MB=32): - groups = FuseHelper.get_fused_groups(block, allreduce_vars, - fuse_grad_size_in_MB) +def insert_fused_allreduce_ops( + block, + insert_idx, + ring_id, + allreduce_vars, + op_role=OpRole.Backward, + use_calc_stream=False, + fuse_grad_size_in_MB=32, +): + groups = FuseHelper.get_fused_groups( + block, allreduce_vars, fuse_grad_size_in_MB + ) - fused_vars, insert_num = FuseHelper.insert_coalesce_tensor(block, - insert_idx, - groups, - op_role, - prefix="Grad") + fused_vars, insert_num = FuseHelper.insert_coalesce_tensor( + block, insert_idx, groups, op_role, prefix="Grad" + ) for fused_var in fused_vars: - block._insert_op_without_sync(insert_idx + insert_num, - type='c_allreduce_sum', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx + insert_num, + type='c_allreduce_sum', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + OP_ROLE_KEY: op_role, + }, + ) if not use_calc_stream: - block._insert_op_without_sync(insert_idx + insert_num, - type='c_sync_calc_stream', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={OP_ROLE_KEY: op_role}) - - -def insert_fused_reduce_ops(block, - insert_idx, - ring_id, - reduce_vars, - shard, - op_role=OpRole.Backward, - use_calc_stream=False, - rank=None, - fuse_grad_size=32): + block._insert_op_without_sync( + insert_idx + insert_num, + type='c_sync_calc_stream', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={OP_ROLE_KEY: op_role}, + ) + + +def insert_fused_reduce_ops( + block, + insert_idx, + ring_id, + reduce_vars, + shard, + op_role=OpRole.Backward, + use_calc_stream=False, + rank=None, + fuse_grad_size=32, +): nranks = shard.worker_num device_to_vars = [[] for _ in range(nranks)] for var in reduce_vars: root_id = get_grad_device(var, shard) - assert 0 <= root_id < nranks, "root_id should >=0 and < nranks, " \ - "but now nranks={}, the root_id of var={} is {}"\ - .format(nranks, var, root_id) + assert 0 <= root_id < nranks, ( + "root_id should >=0 and < nranks, " + "but now nranks={}, the root_id of var={} is {}".format( + nranks, var, root_id + ) + ) device_to_vars[root_id].append(var) for root_id, vars_name in enumerate(device_to_vars): groups = FuseHelper.get_fused_groups(block, vars_name, fuse_grad_size) fused_vars, insert_num = FuseHelper.insert_coalesce_tensor( - block, insert_idx, groups, op_role, prefix="Grad") + block, insert_idx, groups, op_role, prefix="Grad" + ) for fused_var in fused_vars: - block._insert_op_without_sync(insert_idx + insert_num, - type='c_reduce_sum', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={ - 'ring_id': ring_id, - 'root_id': root_id, - 'use_calc_stream': - use_calc_stream, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx + insert_num, + type='c_reduce_sum', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={ + 'ring_id': ring_id, + 'root_id': root_id, + 'use_calc_stream': use_calc_stream, + OP_ROLE_KEY: op_role, + }, + ) if not use_calc_stream: - block._insert_op_without_sync(insert_idx + insert_num, - type='c_sync_calc_stream', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={OP_ROLE_KEY: op_role}) + block._insert_op_without_sync( + insert_idx + insert_num, + type='c_sync_calc_stream', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={OP_ROLE_KEY: op_role}, + ) return [] if rank is None else device_to_vars[rank] -def insert_reduce_ops(block, - insert_idx, - ring_id, - reduce_vars, - shard, - op_role=OpRole.Backward, - use_calc_stream=False, - rank=None, - strategy=None): +def insert_reduce_ops( + block, + insert_idx, + ring_id, + reduce_vars, + shard, + op_role=OpRole.Backward, + use_calc_stream=False, + rank=None, + strategy=None, +): """ _add_reduce_ops """ - if strategy and strategy.fuse_all_reduce_ops and \ - not strategy.fuse_grad_merge: - return insert_fused_reduce_ops(block, insert_idx, ring_id, reduce_vars, - shard, op_role, use_calc_stream, rank, - strategy.fuse_grad_size_in_MB) + if ( + strategy + and strategy.fuse_all_reduce_ops + and not strategy.fuse_grad_merge + ): + return insert_fused_reduce_ops( + block, + insert_idx, + ring_id, + reduce_vars, + shard, + op_role, + use_calc_stream, + rank, + strategy.fuse_grad_size_in_MB, + ) grad_in_this_device = [] for var in reduce_vars: grad_var = var - if strategy and strategy.fuse_all_reduce_ops and \ - strategy.fuse_grad_merge: + if ( + strategy + and strategy.fuse_all_reduce_ops + and strategy.fuse_grad_merge + ): # TODO(wangxi): if support fp16_allreduce, need be # 'FusedMergedGrad.cast_fp16._' grad_var = var.replace('FusedMergedGrad_', '') root_id = get_grad_device(grad_var, shard) - assert root_id >= 0, "root id should be a positive int, but now root id is {}".format( - root_id) + assert ( + root_id >= 0 + ), "root id should be a positive int, but now root id is {}".format( + root_id + ) if rank is not None and rank == root_id: grad_in_this_device.append(var) - block._insert_op_without_sync(insert_idx, - type='c_reduce_sum', - inputs={'X': var}, - outputs={'Out': var}, - attrs={ - 'ring_id': ring_id, - 'root_id': root_id, - 'use_calc_stream': use_calc_stream, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type='c_reduce_sum', + inputs={'X': var}, + outputs={'Out': var}, + attrs={ + 'ring_id': ring_id, + 'root_id': root_id, + 'use_calc_stream': use_calc_stream, + OP_ROLE_KEY: op_role, + }, + ) return grad_in_this_device -def insert_fused_broadcast_param_ops(block, - insert_idx, - ring_id, - params, - shard, - op_role=OpRole.Optimize, - use_calc_stream=False, - rank=None, - fuse_size=32): +def insert_fused_broadcast_param_ops( + block, + insert_idx, + ring_id, + params, + shard, + op_role=OpRole.Optimize, + use_calc_stream=False, + rank=None, + fuse_size=32, +): nranks = shard.worker_num device_to_vars = [[] for _ in range(nranks)] for var in params: root_id = shard.device(var) - assert 0 <= root_id < nranks, "root_id should >=0 and < nranks, " \ - "but now nranks={}, the root_id of var={} is {}"\ - .format(nranks, var, root_id) + assert 0 <= root_id < nranks, ( + "root_id should >=0 and < nranks, " + "but now nranks={}, the root_id of var={} is {}".format( + nranks, var, root_id + ) + ) device_to_vars[root_id].append(var) for root_id, vars_name in enumerate(device_to_vars): groups = FuseHelper.get_fused_groups(block, vars_name, fuse_size) fused_vars, insert_num = FuseHelper.insert_coalesce_tensor( - block, insert_idx, groups, op_role, prefix="Param") + block, insert_idx, groups, op_role, prefix="Param" + ) for fused_var in fused_vars: - block._insert_op_without_sync(insert_idx + insert_num, - type='c_broadcast', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={ - 'ring_id': ring_id, - 'root': root_id, - 'use_calc_stream': - use_calc_stream, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx + insert_num, + type='c_broadcast', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={ + 'ring_id': ring_id, + 'root': root_id, + 'use_calc_stream': use_calc_stream, + OP_ROLE_KEY: op_role, + }, + ) if not use_calc_stream: - block._insert_op_without_sync(insert_idx + insert_num, - type='c_sync_calc_stream', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={OP_ROLE_KEY: op_role}) + block._insert_op_without_sync( + insert_idx + insert_num, + type='c_sync_calc_stream', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={OP_ROLE_KEY: op_role}, + ) return [] if rank is None else device_to_vars[rank] -def insert_broadcast_param_ops(block, - insert_idx, - ring_id, - params, - shard, - op_role=OpRole.Optimize, - use_calc_stream=False, - rank=None, - strategy=None): +def insert_broadcast_param_ops( + block, + insert_idx, + ring_id, + params, + shard, + op_role=OpRole.Optimize, + use_calc_stream=False, + rank=None, + strategy=None, +): """ add broadcast param ops """ if strategy and strategy.fuse_all_reduce_ops: # TODO(wangxi): put fused var in startup_program, only need exec once - return insert_fused_broadcast_param_ops(block, insert_idx, ring_id, - params, shard, op_role, - use_calc_stream, rank, - strategy.fuse_grad_size_in_MB) + return insert_fused_broadcast_param_ops( + block, + insert_idx, + ring_id, + params, + shard, + op_role, + use_calc_stream, + rank, + strategy.fuse_grad_size_in_MB, + ) param_in_this_device = [] for param in params: root_id = shard.device(param) - assert root_id >= 0, "root id should be a positive int, but now root id is {}".format( - root_id) + assert ( + root_id >= 0 + ), "root id should be a positive int, but now root id is {}".format( + root_id + ) if rank is not None and rank == root_id: param_in_this_device.append(param) - block._insert_op_without_sync(insert_idx, - type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - 'root': root_id, - 'use_calc_stream': use_calc_stream, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + 'root': root_id, + 'use_calc_stream': use_calc_stream, + OP_ROLE_KEY: op_role, + }, + ) return param_in_this_device -def fuse_opt_broadcast_param_ops(block, - ring_id, - shard, - op_role=OpRole.Optimize, - strategy=None): +def fuse_opt_broadcast_param_ops( + block, ring_id, shard, op_role=OpRole.Optimize, strategy=None +): """ fuse optimizer sharding broadcast param ops """ @@ -689,26 +786,30 @@ def fuse_opt_broadcast_param_ops(block, groups = FuseHelper.get_fused_groups(block, vars_name, fuse_size) fused_vars, insert_num = FuseHelper.insert_coalesce_tensor( - block, insert_idx, groups, op_role, prefix="Param") + block, insert_idx, groups, op_role, prefix="Param" + ) for fused_var in fused_vars: - block._insert_op_without_sync(insert_idx + insert_num, - type='c_broadcast', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={ - 'ring_id': ring_id, - 'root': root_id, - 'use_calc_stream': True, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx + insert_num, + type='c_broadcast', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={ + 'ring_id': ring_id, + 'root': root_id, + 'use_calc_stream': True, + OP_ROLE_KEY: op_role, + }, + ) block._sync_with_cpp() def get_grad_device(grad_name, shard): assert "@GRAD" in grad_name, "[{}] should be a grad variable.".format( - grad_name) + grad_name + ) base_name = None # NOTE: mind the traversal order possible_suffixes = [ @@ -725,8 +826,9 @@ def get_grad_device(grad_name, shard): base_name = re.sub(suffix, '', grad_name) break - assert base_name in shard.global_param2device, "[{}] should be a param variable.".format( - base_name) + assert ( + base_name in shard.global_param2device + ), "[{}] should be a param variable.".format(base_name) return shard.global_param2device[base_name] @@ -760,15 +862,17 @@ def insert_broadcast_ops(block, insert_idx, ring_id, broadcast2root): """ op_role = get_valid_op_role(block, insert_idx) for broadcast_name, root_device in broadcast2root: - block._insert_op_without_sync(insert_idx, - type='c_broadcast', - inputs={'X': broadcast_name}, - outputs={'Out': broadcast_name}, - attrs={ - 'ring_id': ring_id, - 'root': root_device, - OP_ROLE_KEY: op_role - }) + block._insert_op_without_sync( + insert_idx, + type='c_broadcast', + inputs={'X': broadcast_name}, + outputs={'Out': broadcast_name}, + attrs={ + 'ring_id': ring_id, + 'root': root_device, + OP_ROLE_KEY: op_role, + }, + ) return @@ -793,8 +897,12 @@ def get_var_size(param): var size in MB """ assert -1 not in param.shape - return reduce(lambda x, y: x * y, - param.shape) * DtypeToSize[param.dtype] / 1024.0 / 1024.0 + return ( + reduce(lambda x, y: x * y, param.shape) + * DtypeToSize[param.dtype] + / 1024.0 + / 1024.0 + ) def insert_scale_loss_grad_ops(block, scale=1.0): @@ -804,9 +912,10 @@ def insert_scale_loss_grad_ops(block, scale=1.0): ''' for idx, op in reversed(list(enumerate(block.ops))): if is_loss_grad_op(op): - assert op.type == 'fill_constant', \ - "loss_grad_op must be fill_constant op, " \ + assert op.type == 'fill_constant', ( + "loss_grad_op must be fill_constant op, " "but this op is {}".format(op.type) + ) assert op.has_attr('value') loss_scale = float(op.attr('value')) loss_scale = loss_scale / scale @@ -825,8 +934,9 @@ def comm_analyse(main_program): if op.type == "c_broadcast": var_name = op.desc.input_arg_names()[0] # convert MB to KB - broadcast_vars[var_name] = get_var_size( - block.var(var_name)) * 1024.0 + broadcast_vars[var_name] = ( + get_var_size(block.var(var_name)) * 1024.0 + ) elif op.type == "c_allreduce_sum": var_name = op.desc.input_arg_names()[0] reduce_vars[var_name] = get_var_size(block.var(var_name)) * 1024.0 @@ -836,14 +946,14 @@ def comm_analyse(main_program): for k, v in broadcast_vars.items(): print("broadcast: {}: {} KB".format(k, v)) - if (int(v / gap) in varsize_count): + if int(v / gap) in varsize_count: varsize_count[int(v / gap)] += 1 else: varsize_count[int(v / gap)] = 1 for k, v in reduce_vars.items(): print("allreduce: {}: {} KB".format(k, v)) - if (int(v / gap) in varsize_count): + if int(v / gap) in varsize_count: varsize_count[int(v / gap)] += 1 else: varsize_count[int(v / gap)] = 1 @@ -852,8 +962,9 @@ def comm_analyse(main_program): sorted_varsize = sorted(varsize_count.items(), key=lambda x: x[0]) for varsize, count in sorted_varsize: print("NCCL size {}~{} KB: {}".format(varsize, varsize + 1, count)) - f.write("NCCL size {}~{} KB: {}\n".format(varsize, varsize + 1, - count)) + f.write( + "NCCL size {}~{} KB: {}\n".format(varsize, varsize + 1, count) + ) def add_sync_comm(program, sharding_ring_id): @@ -863,7 +974,7 @@ def add_sync_comm(program, sharding_ring_id): add the sync_comm op for the test prog. """ - #NOTE (liangjianzhong): only support one comm stream by now, use more than one + # NOTE (liangjianzhong): only support one comm stream by now, use more than one # comm streams will cause error. should be revise in future. assert sharding_ring_id >= 0, "sharding_ring_id should larger than zero" @@ -877,15 +988,15 @@ def add_sync_comm(program, sharding_ring_id): for input_name in op.desc.input_arg_names(): not_sync_vars.remove(input_name) if not_sync_vars: - block.append_op(type='c_sync_comm_stream', - inputs={'X': list(not_sync_vars)}, - outputs={'Out': list(not_sync_vars)}, - attrs={ - 'ring_id': - sharding_ring_id, - 'op_role': - core.op_proto_and_checker_maker.OpRole.Forward - }) + block.append_op( + type='c_sync_comm_stream', + inputs={'X': list(not_sync_vars)}, + outputs={'Out': list(not_sync_vars)}, + attrs={ + 'ring_id': sharding_ring_id, + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, + }, + ) return @@ -905,8 +1016,14 @@ def is_opt_vars(var): # support EMA optimizer with '_ema_0', # support offload with '@offload_0' and '.cast_fp16' checks = [ - "_moment1_0", "_moment2_0", "_beta1_pow_acc_0", "_beta2_pow_acc_0", - "_velocity_0", "_ema_0", "@offload_0", ".cast_fp16" + "_moment1_0", + "_moment2_0", + "_beta1_pow_acc_0", + "_beta2_pow_acc_0", + "_velocity_0", + "_ema_0", + "@offload_0", + ".cast_fp16", ] for check in checks: if var.name.endswith(check) and var.persistable: @@ -919,24 +1036,27 @@ def is_gradient_merge_vars(var): return var.name.endswith("@GradiantMerge") def is_trainable(var): - return isinstance(var, - paddle.fluid.framework.Parameter) and var.trainable + return ( + isinstance(var, paddle.fluid.framework.Parameter) and var.trainable + ) def sharding_predicate(var): - return is_trainable(var) or is_opt_vars(var) or is_gradient_merge_vars( - var) + return ( + is_trainable(var) or is_opt_vars(var) or is_gradient_merge_vars(var) + ) if int(os.environ.get('PADDLE_TRAINER_ID', 0)) == 0: - paddle.fluid.io.save_persistables(exe, - dirname, - main_program=main_program, - filename=None) + paddle.fluid.io.save_persistables( + exe, dirname, main_program=main_program, filename=None + ) else: - paddle.fluid.io.save_vars(exe, - dirname, - main_program=main_program, - predicate=sharding_predicate, - filename=None) + paddle.fluid.io.save_vars( + exe, + dirname, + main_program=main_program, + predicate=sharding_predicate, + filename=None, + ) return @@ -944,22 +1064,28 @@ def sharding_predicate(var): def append_naive_sync(block, sync_var, ring_id): # NOTE (JZ-LIANG) update this to use barrier sync for more elegent logic # sync within global - block.append_op(type="fill_constant", - outputs={"Out": sync_var}, - attrs={ - "shape": sync_var.shape, - "dtype": sync_var.dtype, - "value": int(1), - }) - block.append_op(type='c_allreduce_sum', - inputs={'X': sync_var}, - outputs={'Out': sync_var}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward - }) - block.append_op(type='c_sync_calc_stream', - inputs={'X': [sync_var]}, - outputs={'Out': [sync_var]}, - attrs={OP_ROLE_KEY: OpRole.Forward}) + block.append_op( + type="fill_constant", + outputs={"Out": sync_var}, + attrs={ + "shape": sync_var.shape, + "dtype": sync_var.dtype, + "value": int(1), + }, + ) + block.append_op( + type='c_allreduce_sum', + inputs={'X': sync_var}, + outputs={'Out': sync_var}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + block.append_op( + type='c_sync_calc_stream', + inputs={'X': [sync_var]}, + outputs={'Out': [sync_var]}, + attrs={OP_ROLE_KEY: OpRole.Forward}, + ) diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py b/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py index 42c52af44311c..3d5d8aa2a3851 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding/weight_decay_helper.py @@ -18,13 +18,13 @@ class WeightDecayHelper(object): - def __init__(self): pass def _is_weight_decay_op(self, op): - return op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/regularization") + return op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/regularization") def prune_weight_decay(self, block, shard): for idx, op in reversed(list(enumerate(block.ops))): @@ -33,7 +33,8 @@ def prune_weight_decay(self, block, shard): if OP_ROLE_VAR_KEY not in op.attr_names: raise ValueError( "The Weight Dacay op should hold op_role_var attribute" - "but the {} op does not hold op_role_var".format(op.type)) + "but the {} op does not hold op_role_var".format(op.type) + ) op_role_var = op.all_attrs()[OP_ROLE_VAR_KEY] if not shard.has_param(op_role_var[0]): block._remove_op(idx, sync=False) diff --git a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py index 073de83d19acb..da0a08aefa109 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/sharding_optimizer.py @@ -94,7 +94,7 @@ def _enable_strategy(self, dist_strategy, context): dist_strategy.sharding_configs = {"segment_broadcast_MB": 32} def _get_sharding_segment_strategy(self): - """ get + """get self._sharding_segment_strategy 1. if by_size: self._broadcast_MB 2. if by_anchors: self._sharding_segment_anchors @@ -107,21 +107,26 @@ def _get_sharding_segment_strategy(self): if segment_strategy == "segment_broadcast_MB": self._broadcast_MB = sharding_configs["segment_broadcast_MB"] - assert self._broadcast_MB > 0, "segment size should larger than zero !" + assert ( + self._broadcast_MB > 0 + ), "segment size should larger than zero !" elif segment_strategy == "segment_anchors": self._sharding_segment_anchors = sharding_configs["segment_anchors"] - assert len(self._sharding_segment_anchors - ) > 0, "you should set the sharding segment anchors !" + assert ( + len(self._sharding_segment_anchors) > 0 + ), "you should set the sharding segment anchors !" self._backward_remain_anchors = self._sharding_segment_anchors[:] self._forward_remain_anchors = [] else: raise NotImplementedError( "the sharding segment strategy [{}] is not implemented".format( - str(segment_strategy))) + str(segment_strategy) + ) + ) self._sharding_segment_strategy = segment_strategy def _get_hybrid_degree(self): - """ get + """get self.hybrid_dp self.sharding_degree self.mp_degree @@ -145,21 +150,32 @@ def _get_hybrid_degree(self): assert strategy.pipeline is True if os.getenv("PADDLE_MANUAL_PIPELINE_STAGE", None): - assert pp_degree == 2, ("For manually set pipeline, only " - "pp_degree = 2 is supported.") - assert global_world_size == mp_degree * sharding_degree * dp_degree, \ - "global work size [{}], mp_degree [{}], sharding_degree [{}], dp_degree [{}].".format( - global_world_size, mp_degree, sharding_degree, dp_degree) + assert pp_degree == 2, ( + "For manually set pipeline, only " "pp_degree = 2 is supported." + ) + assert ( + global_world_size == mp_degree * sharding_degree * dp_degree + ), "global work size [{}], mp_degree [{}], sharding_degree [{}], dp_degree [{}].".format( + global_world_size, mp_degree, sharding_degree, dp_degree + ) else: - assert global_world_size == mp_degree * sharding_degree * pp_degree * dp_degree, \ - "global work size [{}], mp_degree [{}], sharding_degree [{}], pp_degree [{}], dp_degree [{}].".format( - global_world_size, mp_degree, sharding_degree, pp_degree, dp_degree) + assert ( + global_world_size + == mp_degree * sharding_degree * pp_degree * dp_degree + ), "global work size [{}], mp_degree [{}], sharding_degree [{}], pp_degree [{}], dp_degree [{}].".format( + global_world_size, + mp_degree, + sharding_degree, + pp_degree, + dp_degree, + ) # FIXME (JZ-LIANG) deprecated hybrid_dp if sharding_configs["hybrid_dp"]: logger.warning( "[hybrid_dp] API setting is deprecated. Now when " - "dp_degree >= 2, its will be in hybrid dp mode automatically") + "dp_degree >= 2, its will be in hybrid dp mode automatically" + ) assert dp_degree >= 1 self.hybrid_dp = True if dp_degree > 1 else False @@ -169,7 +185,7 @@ def _get_hybrid_degree(self): self.dp_degree = dp_degree def _get_hybrid_dp_mode(self): - """ get + """get self.hybrid_dp_mode = 'pp_hybrid_dp' or 'sharding_hybrid_dp' self.gradient_merge_mode = 'pp_gm' or 'sharding_gm' self._gradient_merge_acc_step @@ -193,9 +209,10 @@ def _get_hybrid_dp_mode(self): if self.pp_degree > 1: dp_mode = "pp_hybrid_dp" else: - assert self.sharding_degree > 1, \ - "by now we only support five kind of hybrid dp: sharding_hybrid_dp, " \ + assert self.sharding_degree > 1, ( + "by now we only support five kind of hybrid dp: sharding_hybrid_dp, " "mp_sharding_hybrid_dp, pp_hybrid_dp, mp_sharding_pp_hybrid_dp, sharding_pp_hybrid_dp." + ) dp_mode = "sharding_hybrid_dp" # gradient merge @@ -208,23 +225,33 @@ def _get_hybrid_dp_mode(self): gm_mode = "pp_gm" gm_acc_step = strategy.pipeline_configs['accumulate_steps'] gradient_scale_configs = strategy.gradient_scale_configs - assert gradient_scale_configs['scale_strategy'] == 'avg', \ - 'For pipeline mode, the ' 'gradient scale mode should ' \ - 'be "avg", but got {}'.format(gradient_scale_configs['scale_strategy']) + assert gradient_scale_configs['scale_strategy'] == 'avg', ( + 'For pipeline mode, the ' + 'gradient scale mode should ' + 'be "avg", but got {}'.format( + gradient_scale_configs['scale_strategy'] + ) + ) # Note (Yuang Liu): this avg_loss flag determines where to do the average op for grad merge. # If True, will do sum firstly for gradient merge, then do scale by gm_acc_step. # If False, will scale loss by gm_acc_step first, then do sum for gradient merge. self.scale_gradient = gradient_scale_configs['scale_gradient'] if gm_acc_step > 1: - logger.info("Gradient merge in [{}], acc step = [{}]".format( - gm_mode, gm_acc_step)) + logger.info( + "Gradient merge in [{}], acc step = [{}]".format( + gm_mode, gm_acc_step + ) + ) optimizer_sharding = False # TODO(wangxi): need support dp_as_opt_sharding with sharding # need support without pp in future - if self.sharding_degree == 1 and self.dp_degree > 1 \ - and sharding_configs['_dp_as_optimizer_sharding'] \ - and self.pp_degree > 1: + if ( + self.sharding_degree == 1 + and self.dp_degree > 1 + and sharding_configs['_dp_as_optimizer_sharding'] + and self.pp_degree > 1 + ): optimizer_sharding = True self.hybrid_dp_mode = dp_mode @@ -234,19 +261,23 @@ def _get_hybrid_dp_mode(self): # this feature is design for ascend, and should NOT be used in GPU training self.pp_allreduce_in_optimize = sharding_configs[ - "pp_allreduce_in_optimize"] + "pp_allreduce_in_optimize" + ] - def _inner_opt_minimize(self, loss, startup_program, parameter_list, - no_grad_set): + def _inner_opt_minimize( + self, loss, startup_program, parameter_list, no_grad_set + ): pipeline_configs = self.user_defined_strategy.pipeline_configs if self.inner_opt is None: raise ValueError( - "self.inner_opt of ShardingOptimizer should not be None.") + "self.inner_opt of ShardingOptimizer should not be None." + ) if self.pp_degree > 1: pp_optimizer = fluid.optimizer.PipelineOptimizer( - self.inner_opt, self._gradient_merge_acc_step) + self.inner_opt, self._gradient_merge_acc_step + ) self._pp_optimizer = pp_optimizer global_rank = self.role_maker._worker_index() @@ -263,17 +294,25 @@ def _inner_opt_minimize(self, loss, startup_program, parameter_list, 'global_ring_id': 3, 'mp_degree': self.mp_degree, 'mp_rank': global_rank % self.mp_degree, - 'scale_gradient': self.scale_gradient + 'scale_gradient': self.scale_gradient, } main_program = loss.block.program main_program._pipeline_opt = pipeline_opt - optimize_ops, params_grads, program_list, self.pipeline_pair, self.pp_ring_map = pp_optimizer.minimize( - loss, startup_program, parameter_list, no_grad_set) + ( + optimize_ops, + params_grads, + program_list, + self.pipeline_pair, + self.pp_ring_map, + ) = pp_optimizer.minimize( + loss, startup_program, parameter_list, no_grad_set + ) assert self.pp_degree == len(program_list) else: optimize_ops, params_grads = self.inner_opt.minimize( - loss, startup_program, parameter_list, no_grad_set) + loss, startup_program, parameter_list, no_grad_set + ) if startup_program is None: startup_program = default_startup_program() @@ -282,8 +321,9 @@ def _inner_opt_minimize(self, loss, startup_program, parameter_list, startup_program = startup_program._pipeline_opt['startup_program'] print("pp_rank:", self.pp_rank) if os.getenv("PADDLE_MANUAL_PIPELINE_STAGE", None): - main_program = program_list[int( - os.getenv("PADDLE_MANUAL_PIPELINE_STAGE"))] + main_program = program_list[ + int(os.getenv("PADDLE_MANUAL_PIPELINE_STAGE")) + ] else: main_program = program_list[self.pp_rank] with open("main_%d" % self.role_maker._worker_index(), 'w') as f: @@ -309,14 +349,16 @@ def _inner_opt_minimize(self, loss, startup_program, parameter_list, return optimize_ops, params_grads def _apply_sharding_pass(self, params_grads): - if self.sharding_degree == 1: return + if self.sharding_degree == 1: + return main_block = self._main_program.global_block() startup_block = self._startup_program.global_block() # step1: build shard - self._build_shard(params_grads, self.sharding_rank, - self.sharding_degree) + self._build_shard( + params_grads, self.sharding_rank, self.sharding_degree + ) # step2: split_program self._split_program(main_block) @@ -328,13 +370,16 @@ def _apply_sharding_pass(self, params_grads): # step4: remove unneeded ops and vars from block self._prune_main_program( - main_block, self._shard, - [self.mp_ring_id, self.sharding_ring_id, self.pp_ring_id]) + main_block, + self._shard, + [self.mp_ring_id, self.sharding_ring_id, self.pp_ring_id], + ) self._prune_startup_program(startup_block, self._shard) def _apply_opt_sharding_pass(self, params_grads): - """ outer dp as optimizer sharding """ - if self._optimizer_sharding is False: return + """outer dp as optimizer sharding""" + if self._optimizer_sharding is False: + return main_block = self._main_program.global_block() startup_block = self._startup_program.global_block() @@ -348,12 +393,15 @@ def _apply_opt_sharding_pass(self, params_grads): # step4: remove unneeded ops and vars from block self._prune_main_program( - main_block, self._shard, - [self.mp_ring_id, self.pp_ring_id, self.dp_ring_id]) + main_block, + self._shard, + [self.mp_ring_id, self.pp_ring_id, self.dp_ring_id], + ) self._prune_startup_program(startup_block, self._shard) def _insert_allreduce_for_pp(self, params_grads): - if self.pp_degree == 1: return + if self.pp_degree == 1: + return strategy = self.user_defined_strategy sharding_configs = strategy.sharding_configs @@ -373,10 +421,12 @@ def _insert_allreduce_for_pp(self, params_grads): main_block._remove_op(idx) for idx, op in reversed(list(enumerate(main_block.ops))): - if op.type != 'cast': continue + if op.type != 'cast': + continue in_name = op.input_arg_names[0] - if in_name not in self._params: continue - #if self._shard.has_param(param_name): continue + if in_name not in self._params: + continue + # if self._shard.has_param(param_name): continue if in_name not in main_block.vars: main_block._remove_op(idx) @@ -386,7 +436,8 @@ def _insert_allreduce_for_pp(self, params_grads): shard = self._shard if self._optimizer_sharding else None accumulated_grad_names = self._pp_optimizer._accumulate_gradients( - main_block, strategy=strategy, shard=shard) + main_block, strategy=strategy, shard=shard + ) len_of_ops = len(main_block.ops) if self.scale_gradient: @@ -394,8 +445,9 @@ def _insert_allreduce_for_pp(self, params_grads): first_optimize_op_index = get_first_optimize_op_idx(main_block) if self.pp_allreduce_in_optimize: - logger.info("Pipeline Persistable grad is {}".format( - accumulated_grad_names)) + logger.info( + "Pipeline Persistable grad is {}".format(accumulated_grad_names) + ) # FIXME(wangxi): accumulated_grad get from pipeline is not # include sharding's param@BroadCast grad when # pp_allreduce_in_optimize @@ -407,10 +459,11 @@ def _insert_allreduce_for_pp(self, params_grads): self._shard, core.op_proto_and_checker_maker.OpRole.Optimize, use_calc_stream=True, - rank=self.sharding_rank) + rank=self.sharding_rank, + ) logger.info("PP-Sharding grad is {}".format(accumulated_grad_names)) - first_optimize_op_index += (len(main_block.ops) - len_of_ops) + first_optimize_op_index += len(main_block.ops) - len_of_ops len_of_ops = len(main_block.ops) if self._optimizer_sharding: @@ -423,10 +476,12 @@ def _insert_allreduce_for_pp(self, params_grads): OpRole.Optimize, use_calc_stream=True, rank=self.dp_rank, - strategy=strategy) + strategy=strategy, + ) logger.info( - "Optimizer grad in this rank {}".format(accumulated_grad_names)) - first_optimize_op_index += (len(main_block.ops) - len_of_ops) + "Optimizer grad in this rank {}".format(accumulated_grad_names) + ) + first_optimize_op_index += len(main_block.ops) - len_of_ops len_of_ops = len(main_block.ops) # NOTE(wangxi): we fused after optimize_cast @@ -434,14 +489,17 @@ def _insert_allreduce_for_pp(self, params_grads): optimizer_param = utils.insert_broadcast_param_ops( main_block, len_of_ops, - self.dp_ring_id, [x[0].name for x in params_grads], + self.dp_ring_id, + [x[0].name for x in params_grads], self._shard, OpRole.Optimize, use_calc_stream=True, rank=self.dp_rank, - strategy=None if optimize_cast else strategy) + strategy=None if optimize_cast else strategy, + ) logger.info( - "Optimizer param in this rank {}".format(optimizer_param)) + "Optimizer param in this rank {}".format(optimizer_param) + ) if not strategy.fuse_grad_merge and not optimize_cast: assert len(accumulated_grad_names) == len(optimizer_param) elif self.hybrid_dp and self.hybrid_dp_mode == "pp_hybrid_dp": @@ -452,15 +510,20 @@ def _insert_allreduce_for_pp(self, params_grads): accumulated_grad_names, core.op_proto_and_checker_maker.OpRole.Optimize, use_calc_stream=True, - user_defined_strategy=strategy) - first_optimize_op_index += (len(main_block.ops) - len_of_ops) + user_defined_strategy=strategy, + ) + first_optimize_op_index += len(main_block.ops) - len_of_ops len_of_ops = len(main_block.ops) # FIXME(wangxi): if fp16_allreduce, put cast fp16->fp32 to there? def _avg_grad_merge_after_sum(self, main_block, accumulated_grad_names): - if self.user_defined_strategy.amp and \ - self.user_defined_strategy.amp_configs['use_dynamic_loss_scaling']: + if ( + self.user_defined_strategy.amp + and self.user_defined_strategy.amp_configs[ + 'use_dynamic_loss_scaling' + ] + ): # For AMP, if using dynamic loss scaling the avg # operation can be simple done by modify the LossScaling op. for idx, op in enumerate(main_block.ops): @@ -471,7 +534,8 @@ def _avg_grad_merge_after_sum(self, main_block, accumulated_grad_names): loss_scale_tmp_var = main_block.create_var( name=loss_scale_tmp_var_name, shape=loss_scaling_var.shape, - dtype=loss_scaling_var.dtype) + dtype=loss_scaling_var.dtype, + ) main_block._insert_op_without_sync( idx, type='scale', @@ -481,8 +545,9 @@ def _avg_grad_merge_after_sum(self, main_block, accumulated_grad_names): 'scale': self._gradient_merge_acc_step, 'bias': 0.0, 'bias_after_scale': False, - OP_ROLE_KEY: OpRole.Optimize - }) + OP_ROLE_KEY: OpRole.Optimize, + }, + ) op._rename_input(loss_scale_name, loss_scale_tmp_var_name) break else: @@ -493,7 +558,9 @@ def _avg_grad_merge_after_sum(self, main_block, accumulated_grad_names): if is_optimizer_op(op) and op.type != 'c_sync_comm_stream': tmp_first_opt_idx = idx break - assert tmp_first_opt_idx is not None, 'Occurs some errors, no optimize ops' + assert ( + tmp_first_opt_idx is not None + ), 'Occurs some errors, no optimize ops' for grad in accumulated_grad_names: main_block._insert_op_without_sync( tmp_first_opt_idx, @@ -504,14 +571,17 @@ def _avg_grad_merge_after_sum(self, main_block, accumulated_grad_names): 'scale': 1.0 / self._gradient_merge_acc_step, 'bias': 0.0, 'bias_after_scale': False, - OP_ROLE_KEY: OpRole.Optimize - }) + OP_ROLE_KEY: OpRole.Optimize, + }, + ) def _adapt_amp_clip_without_sharding(self): # if not use sharding, adapt amp/clip, for remain parallelism. # cast --> amp --> clip --> opt - if self.sharding_degree > 1: return - if self._optimizer_sharding: return + if self.sharding_degree > 1: + return + if self._optimizer_sharding: + return main_block = self._main_program.global_block() startup_block = self._startup_program.global_block() @@ -525,9 +595,9 @@ def _adapt_amp_clip_without_sharding(self): FP16Utils.sync_amp_check_nan_inf(main_block, rings) gradientclip_helper = GradientClipHelper(None) - gradientclip_helper.sync_global_norm(main_block, - [self.mp_ring_id, self.pp_ring_id], - self.mp_rank) + gradientclip_helper.sync_global_norm( + main_block, [self.mp_ring_id, self.pp_ring_id], self.mp_rank + ) def _insert_loss_grad_scale_op(self): main_block = self._main_program.global_block() @@ -548,8 +618,9 @@ def _apply_optimize_offload_pass(self, params_grads): mp_ring_id = self.mp_ring_id if self.mp_degree > 1 else None dp_ring_id = self.dp_ring_id if self.dp_degree > 1 else None - offload_helper = OffloadHelper(mp_ring_id=mp_ring_id, - dp_ring_id=dp_ring_id) + offload_helper = OffloadHelper( + mp_ring_id=mp_ring_id, dp_ring_id=dp_ring_id + ) # optimize offload should be enable while gradient merge is enable and # acc_step is quite large (e.g. >> 100). Since its memcpy could not be @@ -565,32 +636,32 @@ def _apply_optimize_offload_pass(self, params_grads): # will take more memory, but will be faster. Trade space for time. if self._optimizer_sharding: offload_helper.opt_sharding_cast_fp32param( - main_block, startup_block, - [x[0].name for x in params_grads]) + main_block, startup_block, [x[0].name for x in params_grads] + ) # NOTE(wangxi): fused after optimize_cast - utils.fuse_opt_broadcast_param_ops(main_block, - dp_ring_id, - self._shard, - strategy=strategy) + utils.fuse_opt_broadcast_param_ops( + main_block, dp_ring_id, self._shard, strategy=strategy + ) else: offload_helper.cast_fp32param_in_optimize( - main_block, startup_block) + main_block, startup_block + ) def _dump_program_for_debug(self): main_block = self._main_program.global_block() startup_block = self._startup_program.global_block() - with open("start_sharding_%d" % self.role_maker._worker_index(), - 'w') as f: + with open( + "start_sharding_%d" % self.role_maker._worker_index(), 'w' + ) as f: f.writelines(str(startup_block.program)) - with open("main_sharding_%d" % self.role_maker._worker_index(), - 'w') as f: + with open( + "main_sharding_%d" % self.role_maker._worker_index(), 'w' + ) as f: f.writelines(str(main_block.program)) - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): # TODO: (JZ-LIANG) support multiple comm in future # self._nrings = self.user_defined_strategy.nccl_comm_num self._nrings_sharding = 1 @@ -605,7 +676,8 @@ def minimize_impl(self, # inner optimize minimize optimize_ops, params_grads = self._inner_opt_minimize( - loss, startup_program, parameter_list, no_grad_set) + loss, startup_program, parameter_list, no_grad_set + ) self._init_comm() @@ -654,13 +726,15 @@ def _init_pair_comm(self, pair, ring_id): ] pp_rank = 0 if self.pp_rank == pair[0] else 1 if os.getenv("PADDLE_MANUAL_PIPELINE_STAGE", None) is None: - self._collective_helper._init_communicator(self._startup_program, - self.current_endpoint, - pp_group_endpoints, - pp_rank, - ring_id, - False, - sync=False) + self._collective_helper._init_communicator( + self._startup_program, + self.current_endpoint, + pp_group_endpoints, + pp_rank, + ring_id, + False, + sync=False, + ) def _init_npu_pipeline_comm(self, startup_block): # NOTE(wangxi): some bug with hccl, must set pp_degree be even number @@ -678,15 +752,22 @@ def _init_npu_pipeline_comm(self, startup_block): my_pair.append(pair) # for example: self.pp_rank=2, self.pp_degree=4 - send_to_next_pair = (self.pp_rank, (self.pp_rank + 1) % self.pp_degree - ) # 2->3 + send_to_next_pair = ( + self.pp_rank, + (self.pp_rank + 1) % self.pp_degree, + ) # 2->3 recv_from_next_pair = ( - (self.pp_rank + 1) % self.pp_degree, self.pp_rank) # 3->2 + (self.pp_rank + 1) % self.pp_degree, + self.pp_rank, + ) # 3->2 recv_from_prev_pair = ( - (self.pp_rank - 1 + self.pp_degree) % self.pp_degree, self.pp_rank + (self.pp_rank - 1 + self.pp_degree) % self.pp_degree, + self.pp_rank, ) # 1->2 - send_to_prev_pair = (self.pp_rank, (self.pp_rank - 1 + self.pp_degree) % - self.pp_degree) # 2->1 + send_to_prev_pair = ( + self.pp_rank, + (self.pp_rank - 1 + self.pp_degree) % self.pp_degree, + ) # 2->1 even = (self.pp_rank % 2) == 0 @@ -695,54 +776,66 @@ def _init_npu_pipeline_comm(self, startup_block): ring_id = self.pp_ring_map[pair[0] * 1000 + pair[1]] self._init_pair_comm(pair, ring_id) my_pair.remove(pair) - logger.info("pair0(even->odd): pp pair:{}, ring_id: {}".format( - pair, ring_id)) + logger.info( + "pair0(even->odd): pp pair:{}, ring_id: {}".format(pair, ring_id) + ) # 2. even recv from next, odd send to prev, 1->0, 3->2 pair = recv_from_next_pair if even else send_to_prev_pair ring_id = self.pp_ring_map[pair[0] * 1000 + pair[1]] self._init_pair_comm(pair, ring_id) my_pair.remove(pair) - logger.info("pair1(even<-odd): pp pair:{}, ring_id: {}".format( - pair, ring_id)) + logger.info( + "pair1(even<-odd): pp pair:{}, ring_id: {}".format(pair, ring_id) + ) # if pp_degree is 2, only need pair(0->1, 1->0) if self.pp_degree > 2: # 3. odd send to next, even recv from prev, 1->2, 3->0 pair = send_to_next_pair if not even else recv_from_prev_pair - ring_id = self.pp_ring_map.get(pair[0] * 1000 + pair[1], - max_ring_id + - 1) # 3->0 not in pp_ring_map + ring_id = self.pp_ring_map.get( + pair[0] * 1000 + pair[1], max_ring_id + 1 + ) # 3->0 not in pp_ring_map self._init_pair_comm(pair, ring_id) if self.pp_rank != 0 and self.pp_rank != self.pp_degree - 1: my_pair.remove(pair) - logger.info("pair2(odd->even): pp pair:{}, ring_id: {}".format( - pair, ring_id)) + logger.info( + "pair2(odd->even): pp pair:{}, ring_id: {}".format( + pair, ring_id + ) + ) # 4. odd recv from next, even send to prev, 2->1, 0->3 pair = recv_from_next_pair if not even else send_to_prev_pair - ring_id = self.pp_ring_map.get(pair[0] * 1000 + pair[1], - max_ring_id + - 2) # 0->3 not in pp_ring_map + ring_id = self.pp_ring_map.get( + pair[0] * 1000 + pair[1], max_ring_id + 2 + ) # 0->3 not in pp_ring_map self._init_pair_comm(pair, ring_id) if self.pp_rank != 0 and self.pp_rank != self.pp_degree - 1: my_pair.remove(pair) - logger.info("pair3(odd<-even): pp pair:{}, ring_id: {}".format( - pair, ring_id)) + logger.info( + "pair3(odd<-even): pp pair:{}, ring_id: {}".format( + pair, ring_id + ) + ) - assert len(my_pair) == 0, "Current pipeline does not support cross stage communication, " \ - "please check unexpected pair {}".format(my_pair) + assert len(my_pair) == 0, ( + "Current pipeline does not support cross stage communication, " + "please check unexpected pair {}".format(my_pair) + ) def _init_pipeline_comm(self, startup_block): # TODO (JZ-LIANG) to unify pp_rank_ and pp_rank if os.getenv("PADDLE_MANUAL_PIPELINE_STAGE", None) is None: - self._collective_helper._init_communicator(self._startup_program, - self.current_endpoint, - self.pp_group_endpoints, - self.pp_rank, - self.pp_ring_id, - False, - sync=False) + self._collective_helper._init_communicator( + self._startup_program, + self.current_endpoint, + self.pp_group_endpoints, + self.pp_rank, + self.pp_ring_id, + False, + sync=False, + ) if core.is_compiled_with_npu(): self._init_npu_pipeline_comm(startup_block) @@ -762,13 +855,15 @@ def _init_comm(self): # mp ring if self.mp_degree > 1: - self._collective_helper._init_communicator(self._startup_program, - self.current_endpoint, - self.mp_group_endpoints, - self.mp_rank, - self.mp_ring_id, - False, - sync=False) + self._collective_helper._init_communicator( + self._startup_program, + self.current_endpoint, + self.mp_group_endpoints, + self.mp_rank, + self.mp_ring_id, + False, + sync=False, + ) # sharding ring if self.sharding_degree > 1: @@ -779,7 +874,8 @@ def _init_comm(self): self.sharding_rank, self.sharding_ring_id, False, - sync=False) + sync=False, + ) # pp ring if self.pp_degree > 1: @@ -787,13 +883,15 @@ def _init_comm(self): # pure dp ring if self.dp_degree > 1: - self._collective_helper._init_communicator(self._startup_program, - self.current_endpoint, - self.dp_group_endpoints, - self.dp_rank, - self.dp_ring_id, - False, - sync=False) + self._collective_helper._init_communicator( + self._startup_program, + self.current_endpoint, + self.dp_group_endpoints, + self.dp_rank, + self.dp_ring_id, + False, + sync=False, + ) startup_block._sync_with_cpp() @@ -804,9 +902,12 @@ def _build_shard(self, params_grads, shard_rank, shard_size): # step 3: get broadcast vars self._broadcast_vars = self._shard.find_broadcast_params( - self._main_program.global_block()) + self._main_program.global_block() + ) - def _wait(self, ): + def _wait( + self, + ): endpoints = self.global_endpoints[:] current_endpoint = endpoints[self.global_rank] if self.global_rank == 0: @@ -831,7 +932,7 @@ def _split_program(self, block): segment._end_idx = last_backward_op_idx for op_idx in reversed(range(last_backward_op_idx)): op = block.ops[op_idx] - assert (int(op.attr('op_role')) != int(OpRole.Optimize)) + assert int(op.attr('op_role')) != int(OpRole.Optimize) if self._sharding_segment_strategy == "segment_broadcast_MB": if segment._param_mem >= self._broadcast_MB: segment = self.collect_segment(segment, op_idx, block) @@ -845,21 +946,27 @@ def _split_program(self, block): if ".cast_fp16@GRAD" not in input_name: continue else: - input_name = input_name[:input_name. - find(".cast_fp16@GRAD")] + input_name = input_name[ + : input_name.find(".cast_fp16@GRAD") + ] if input_name in self._backward_remain_anchors: segment = self.collect_segment( - segment, op_idx, block) - assert input_name not in self._forward_remain_anchors, "segment anchor [{}] met twice !".format( - input_name) + segment, op_idx, block + ) + assert ( + input_name not in self._forward_remain_anchors + ), "segment anchor [{}] met twice !".format( + input_name + ) self._backward_remain_anchors.remove(input_name) self._forward_remain_anchors.append(input_name) elif int(op.attr('op_role')) == int(OpRole.Forward): for output_name in op.desc.output_arg_names(): if output_name in self._forward_remain_anchors: segment = self.collect_segment( - segment, op_idx, block) + segment, op_idx, block + ) self._forward_remain_anchors.remove(output_name) # find broadcast vars @@ -875,47 +982,49 @@ def _split_program(self, block): if self._shard.has_param(input_name): broadcast_var_name = input_name else: - broadcast_var_name = unique_name.generate(input_name + - "@BroadCast") + broadcast_var_name = unique_name.generate( + input_name + "@BroadCast" + ) segment._fill_constant_vars.append(broadcast_var_name) # (JZ-LIANG) should use Param base name ? broadcast_var_base_name = input_name if "subprog" in broadcast_var_base_name: # remove suffix - broadcast_var_base_name = broadcast_var_base_name[: - broadcast_var_base_name - .find( - ".subprog" - )] + broadcast_var_base_name = broadcast_var_base_name[ + : broadcast_var_base_name.find(".subprog") + ] - var2broadcast_time[ - broadcast_var_base_name] = var2broadcast_time.get( - broadcast_var_base_name, 0) + 1 + var2broadcast_time[broadcast_var_base_name] = ( + var2broadcast_time.get(broadcast_var_base_name, 0) + 1 + ) segment._param2broadcast[input_name] = broadcast_var_name segment._broadcast_vars.append( - (broadcast_var_name, self._shard.device(input_name))) + (broadcast_var_name, self._shard.device(input_name)) + ) segment._param_mem += get_var_size( - self._main_program.global_block().var(input_name)) + self._main_program.global_block().var(input_name) + ) # find reduce vars if self.pp_degree > 1 and self.pp_allreduce_in_optimize: # place pipeline gradient allreduce in optimize pass else: - if is_backward_op(op) and \ - OP_ROLE_VAR_KEY in op.attr_names: + if is_backward_op(op) and OP_ROLE_VAR_KEY in op.attr_names: op_role_var = op.all_attrs()[OP_ROLE_VAR_KEY] if len(op_role_var) != 0: assert len(op_role_var) % 2 == 0 for i in range(0, len(op_role_var), 2): - param, reduced_grad = op_role_var[i], op_role_var[i - + - 1] + param, reduced_grad = ( + op_role_var[i], + op_role_var[i + 1], + ) segment._allreduce_vars.append(reduced_grad) - assert (reduced_grad - not in self._reduced_grads_to_param) + assert ( + reduced_grad not in self._reduced_grads_to_param + ) self._reduced_grads_to_param[reduced_grad] = param # find cast op @@ -930,29 +1039,40 @@ def _split_program(self, block): self._segments.insert(0, segment) if self._sharding_segment_strategy == "segment_anchors": - assert len( - self._forward_remain_anchors) == 0, "remain anchors {}".format( - self._forward_remain_anchors) - assert len( - self._backward_remain_anchors) == 0, "remain anchors {}".format( - self._backward_remain_anchors) + assert ( + len(self._forward_remain_anchors) == 0 + ), "remain anchors {}".format(self._forward_remain_anchors) + assert ( + len(self._backward_remain_anchors) == 0 + ), "remain anchors {}".format(self._backward_remain_anchors) if self._verbose: - for varname in sorted(var2broadcast_time, - key=var2broadcast_time.get, - reverse=True): - logger.info("Sharding broadcast: [{}] times [{}]".format( - var2broadcast_time[varname], varname)) + for varname in sorted( + var2broadcast_time, key=var2broadcast_time.get, reverse=True + ): + logger.info( + "Sharding broadcast: [{}] times [{}]".format( + var2broadcast_time[varname], varname + ) + ) for idx_ in range(len(self._segments)): logger.info("segment [{}] :".format(idx_)) - logger.info("start op: [{}] [{}]".format( - block.ops[self._segments[idx_]._start_idx].desc.type(), - block.ops[self._segments[idx_]. - _start_idx].desc.input_arg_names())) - logger.info("end op: [{}] [{}]".format( - block.ops[self._segments[idx_]._end_idx].desc.type(), - block.ops[ - self._segments[idx_]._end_idx].desc.input_arg_names())) + logger.info( + "start op: [{}] [{}]".format( + block.ops[self._segments[idx_]._start_idx].desc.type(), + block.ops[ + self._segments[idx_]._start_idx + ].desc.input_arg_names(), + ) + ) + logger.info( + "end op: [{}] [{}]".format( + block.ops[self._segments[idx_]._end_idx].desc.type(), + block.ops[ + self._segments[idx_]._end_idx + ].desc.input_arg_names(), + ) + ) return def _prune_main_program(self, block, shard, rings): @@ -985,17 +1105,18 @@ def _prune_main_program(self, block, shard, rings): input_names = op.desc.input_arg_names() output_names = op.desc.output_arg_names() # FIXME(wangxi): need use grads, pipeline grad is @GRAD@MERGE - if op.type == "c_allreduce_sum" and \ - op.attr('use_model_parallel') is False: - assert (len(output_names) == 1) + if ( + op.type == "c_allreduce_sum" + and op.attr('use_model_parallel') is False + ): + assert len(output_names) == 1 output_name = output_names[0] reduced_grads.append(output_name) # prune optimizer state and param pruned_opti_vars = [] for var_name in list(block.vars.keys()): - if shard.is_opti_var(var_name) and \ - not shard.has_opt_var(var_name): + if shard.is_opti_var(var_name) and not shard.has_opt_var(var_name): pruned_opti_vars.append(var_name) program_deps = ProgramDeps(block, reduced_grads, pruned_opti_vars) @@ -1006,17 +1127,17 @@ def _prune_main_program(self, block, shard, rings): # Prune for idx, op in reversed(list(enumerate(block.ops))): if op.type in [ - "c_allreduce_sum", - "c_sync_comm_stream", - "c_calc_comm_stream", - "c_gen_nccl_id", - "c_comm_init", - 'send_v2', - 'recv_v2', + "c_allreduce_sum", + "c_sync_comm_stream", + "c_calc_comm_stream", + "c_gen_nccl_id", + "c_comm_init", + 'send_v2', + 'recv_v2', ]: pass elif op.type == "conditional_block": - assert (op.desc.has_attr("sub_block")) + assert op.desc.has_attr("sub_block") subblock_idx = op.desc.attr("sub_block").id subblock_deps = program_deps.get_sub_block_deps(subblock_idx) # only prune amp subblock @@ -1032,7 +1153,8 @@ def _prune_main_program(self, block, shard, rings): reversed_output_vars.append(output_name) # prune for sub_op_idx, _ in reversed( - list(enumerate(subblock_deps._block.ops))): + list(enumerate(subblock_deps._block.ops)) + ): if subblock_deps.should_remove_op(sub_op_idx): subblock_deps.remove_op(sub_op_idx) reversed_input_vars = [] @@ -1048,7 +1170,9 @@ def _prune_main_program(self, block, shard, rings): # _should_removed_var: opt state not cur shard if program_deps.should_remove_op(idx): # NOTE(wangxi): need reserve all param in optimizer_sharding - reserved_vars = self._params if self._optimizer_sharding else None + reserved_vars = ( + self._params if self._optimizer_sharding else None + ) program_deps.remove_op(idx, reserved_vars) # NOTE (JZ-LIANG) revise and unify logic here @@ -1059,7 +1183,8 @@ def _prune_main_program(self, block, shard, rings): # remove inputs that not on this card reserved_x = [] for var_name in op.desc.input("X"): - if block.has_var(var_name): reserved_x.append(var_name) + if block.has_var(var_name): + reserved_x.append(var_name) op.desc.set_input('X', reserved_x) block._sync_with_cpp() return @@ -1082,175 +1207,280 @@ def _add_broadcast_allreduce(self, block): # NOTE (JZ-LIANG) revise and unify logic here # fix the _end_idx for segments[-1] if pp is used. new_end_idx = self._segments[-1]._end_idx - for idx in range(self._segments[-1]._end_idx - 1, - self._segments[-1]._start_idx - 1, -1): + for idx in range( + self._segments[-1]._end_idx - 1, + self._segments[-1]._start_idx - 1, + -1, + ): op = block.ops[idx] if op.type == "fill_constant" or op.type == "sum": - if "MERGED" in op.output_arg_names[0]: new_end_idx = idx + 1 + if "MERGED" in op.output_arg_names[0]: + new_end_idx = idx + 1 elif op.type == "cast": - if "@TMP" in op.output_arg_names[0]: new_end_idx = idx + 1 + if "@TMP" in op.output_arg_names[0]: + new_end_idx = idx + 1 self._segments[-1]._end_idx = new_end_idx if self._segments[-1]._allreduce_vars: shard_allredue_vars = self._shard.filter_grads( - self._segments[-1]._allreduce_vars) - if self.gradient_merge_mode != "sharding_gm" or self._gradient_merge_acc_step <= 1: - if self.hybrid_dp and self.hybrid_dp_mode == "sharding_hybrid_dp" and len( - shard_allredue_vars) >= 1: - insert_sync_comm_ops(block, self._segments[-1]._end_idx, - self.dp_ring_id, shard_allredue_vars) + self._segments[-1]._allreduce_vars + ) + if ( + self.gradient_merge_mode != "sharding_gm" + or self._gradient_merge_acc_step <= 1 + ): + if ( + self.hybrid_dp + and self.hybrid_dp_mode == "sharding_hybrid_dp" + and len(shard_allredue_vars) >= 1 + ): + insert_sync_comm_ops( + block, + self._segments[-1]._end_idx, + self.dp_ring_id, + shard_allredue_vars, + ) insert_allreduce_ops( block, self._segments[-1]._end_idx, self.dp_ring_id, shard_allredue_vars, - user_defined_strategy=self.user_defined_strategy) + user_defined_strategy=self.user_defined_strategy, + ) # gradient merge - elif self.gradient_merge_mode == "sharding_gm" and self._gradient_merge_acc_step > 1: + elif ( + self.gradient_merge_mode == "sharding_gm" + and self._gradient_merge_acc_step > 1 + ): self.create_persistable_gradients_and_insert_merge_ops( - block, self._startup_program.global_block(), - self._segments[-1]._end_idx, shard_allredue_vars, - self._shard) - - insert_sync_comm_ops(block, self._segments[-1]._end_idx, - self.sharding_ring_id, - self._segments[-1]._allreduce_vars) + block, + self._startup_program.global_block(), + self._segments[-1]._end_idx, + shard_allredue_vars, + self._shard, + ) + + insert_sync_comm_ops( + block, + self._segments[-1]._end_idx, + self.sharding_ring_id, + self._segments[-1]._allreduce_vars, + ) # allreduce --> reduce - insert_reduce_ops(block, - self._segments[-1]._end_idx, - self.sharding_ring_id, - self._segments[-1]._allreduce_vars, - self._shard, - op_role=OpRole.Backward, - use_calc_stream=False) + insert_reduce_ops( + block, + self._segments[-1]._end_idx, + self.sharding_ring_id, + self._segments[-1]._allreduce_vars, + self._shard, + op_role=OpRole.Backward, + use_calc_stream=False, + ) for idx, segment in reversed(list(enumerate(self._segments))): - allreduce_vars = self._segments[ - idx - 1]._allreduce_vars if idx > 0 else [] - broadcast_vars = self._segments[ - idx + - 1]._broadcast_vars if idx < len(self._segments) - 1 else [] - fill_constant_vars = self._segments[ - idx + - 2]._fill_constant_vars if idx < len(self._segments) - 2 else [] - cast_ops = self._segments[ - idx + 2]._cast_ops if idx < len(self._segments) - 2 else {} + allreduce_vars = ( + self._segments[idx - 1]._allreduce_vars if idx > 0 else [] + ) + broadcast_vars = ( + self._segments[idx + 1]._broadcast_vars + if idx < len(self._segments) - 1 + else [] + ) + fill_constant_vars = ( + self._segments[idx + 2]._fill_constant_vars + if idx < len(self._segments) - 2 + else [] + ) + cast_ops = ( + self._segments[idx + 2]._cast_ops + if idx < len(self._segments) - 2 + else {} + ) for op_idx in reversed(range(segment._start_idx, segment._end_idx)): op = block.ops[op_idx] for input_name in op.desc.input_arg_names(): - if input_name in segment._param2broadcast and \ - input_name != segment._param2broadcast[input_name]: - op._rename_input(input_name, - segment._param2broadcast[input_name]) + if ( + input_name in segment._param2broadcast + and input_name != segment._param2broadcast[input_name] + ): + op._rename_input( + input_name, segment._param2broadcast[input_name] + ) for param_name, broadcast_name in segment._param2broadcast.items(): if param_name != broadcast_name: block.create_var( name=broadcast_name, - shape=self._main_program.global_block().var( - param_name).shape, - dtype=self._main_program.global_block().var( - param_name).dtype, - persistable=False) + shape=self._main_program.global_block() + .var(param_name) + .shape, + dtype=self._main_program.global_block() + .var(param_name) + .dtype, + persistable=False, + ) # step1: remove cast ops block._sync_with_cpp() segment._end_idx += FP16Utils.remove_cast_op( - block, self._params, segment, 0) + block, self._params, segment, 0 + ) # step2: add Sync ops shard_allredue_vars = self._shard.filter_grads(allreduce_vars) - if self.gradient_merge_mode != "sharding_gm" or self._gradient_merge_acc_step <= 1: - if self.hybrid_dp and self.hybrid_dp_mode == "sharding_hybrid_dp" and len( - shard_allredue_vars) >= 1: - insert_sync_comm_ops(block, segment._end_idx, - self.dp_ring_id, shard_allredue_vars) + if ( + self.gradient_merge_mode != "sharding_gm" + or self._gradient_merge_acc_step <= 1 + ): + if ( + self.hybrid_dp + and self.hybrid_dp_mode == "sharding_hybrid_dp" + and len(shard_allredue_vars) >= 1 + ): + insert_sync_comm_ops( + block, + segment._end_idx, + self.dp_ring_id, + shard_allredue_vars, + ) broad_cast_vars = [x[0] for x in broadcast_vars] if len(broad_cast_vars) > 0: - insert_sync_comm_ops(block, segment._end_idx, - self.sharding_ring_id, - broad_cast_vars) + insert_sync_comm_ops( + block, + segment._end_idx, + self.sharding_ring_id, + broad_cast_vars, + ) else: comm_dep_vars = allreduce_vars + [ x[0] for x in broadcast_vars ] if len(comm_dep_vars) > 0: - insert_sync_comm_ops(block, segment._end_idx, - self.sharding_ring_id, - comm_dep_vars) + insert_sync_comm_ops( + block, + segment._end_idx, + self.sharding_ring_id, + comm_dep_vars, + ) # gradient merge - elif self.gradient_merge_mode == "sharding_gm" and self._gradient_merge_acc_step > 1: + elif ( + self.gradient_merge_mode == "sharding_gm" + and self._gradient_merge_acc_step > 1 + ): broad_cast_vars = [x[0] for x in broadcast_vars] if len(broad_cast_vars) > 0: - insert_sync_comm_ops(block, segment._end_idx, - self.sharding_ring_id, broad_cast_vars) + insert_sync_comm_ops( + block, + segment._end_idx, + self.sharding_ring_id, + broad_cast_vars, + ) - calc_dep_vars = fill_constant_vars + [ - k for k, v in cast_ops.items() - ] + self._segments[idx]._allreduce_vars + calc_dep_vars = ( + fill_constant_vars + + [k for k, v in cast_ops.items()] + + self._segments[idx]._allreduce_vars + ) if len(calc_dep_vars) > 0: - insert_sync_calc_op(block, segment._end_idx, - [calc_dep_vars[-1]]) + insert_sync_calc_op( + block, segment._end_idx, [calc_dep_vars[-1]] + ) # step3: insert `fill_constant` ops - insert_fill_constant_ops(block, segment._end_idx, - fill_constant_vars) + insert_fill_constant_ops( + block, segment._end_idx, fill_constant_vars + ) # step4: add `cast` ops insert_cast_ops(block, segment._end_idx, cast_ops) # step5: add broadcast ops # gradient merge - if self.gradient_merge_mode == "sharding_gm" and self._gradient_merge_acc_step > 1: + if ( + self.gradient_merge_mode == "sharding_gm" + and self._gradient_merge_acc_step > 1 + ): self.create_persistable_gradients_and_insert_merge_ops( - block, self._startup_program.global_block(), - segment._start_idx, shard_allredue_vars, self._shard) + block, + self._startup_program.global_block(), + segment._start_idx, + shard_allredue_vars, + self._shard, + ) - insert_broadcast_ops(block, segment._start_idx, - self.sharding_ring_id, broadcast_vars) + insert_broadcast_ops( + block, segment._start_idx, self.sharding_ring_id, broadcast_vars + ) # step6: add all_reduce ops # dp - if self.gradient_merge_mode != "sharding_gm" or self._gradient_merge_acc_step <= 1: - if self.hybrid_dp and self.hybrid_dp_mode == "sharding_hybrid_dp" and len( - shard_allredue_vars) >= 1: + if ( + self.gradient_merge_mode != "sharding_gm" + or self._gradient_merge_acc_step <= 1 + ): + if ( + self.hybrid_dp + and self.hybrid_dp_mode == "sharding_hybrid_dp" + and len(shard_allredue_vars) >= 1 + ): insert_allreduce_ops( block, segment._start_idx, self.dp_ring_id, shard_allredue_vars, - user_defined_strategy=self.user_defined_strategy) - insert_sync_comm_ops(block, segment._start_idx, - self.sharding_ring_id, allreduce_vars) + user_defined_strategy=self.user_defined_strategy, + ) + insert_sync_comm_ops( + block, + segment._start_idx, + self.sharding_ring_id, + allreduce_vars, + ) # gradient merge - elif self.gradient_merge_mode == "sharding_gm" and self._gradient_merge_acc_step > 1: - insert_sync_comm_ops(block, segment._start_idx, - self.sharding_ring_id, allreduce_vars) + elif ( + self.gradient_merge_mode == "sharding_gm" + and self._gradient_merge_acc_step > 1 + ): + insert_sync_comm_ops( + block, + segment._start_idx, + self.sharding_ring_id, + allreduce_vars, + ) # sharding # allreduce --> reduce # TODO temp change if len(allreduce_vars) > 0: - insert_reduce_ops(block, - segment._start_idx, - self.sharding_ring_id, - allreduce_vars, - self._shard, - op_role=OpRole.Backward, - use_calc_stream=False) + insert_reduce_ops( + block, + segment._start_idx, + self.sharding_ring_id, + allreduce_vars, + self._shard, + op_role=OpRole.Backward, + use_calc_stream=False, + ) block._sync_with_cpp() if self._segments[0]._broadcast_vars: broadcast_vars = [x[0] for x in self._segments[0]._broadcast_vars] - insert_sync_comm_ops(block, self._segments[0]._start_idx, - self.sharding_ring_id, broadcast_vars) - insert_broadcast_ops(block, self._segments[0]._start_idx, - self.sharding_ring_id, - self._segments[0]._broadcast_vars) + insert_sync_comm_ops( + block, + self._segments[0]._start_idx, + self.sharding_ring_id, + broadcast_vars, + ) + insert_broadcast_ops( + block, + self._segments[0]._start_idx, + self.sharding_ring_id, + self._segments[0]._broadcast_vars, + ) fill_constant_vars = [] for x in self._segments[:2]: @@ -1264,12 +1494,14 @@ def _add_broadcast_allreduce(self, block): calc_deps_vars = fill_constant_vars + [k for k, v in cast_ops.items()] if fill_constant_vars or cast_ops: - insert_sync_calc_op(block, self._segments[0]._start_idx, - [calc_deps_vars[-1]]) + insert_sync_calc_op( + block, self._segments[0]._start_idx, [calc_deps_vars[-1]] + ) if fill_constant_vars: - insert_fill_constant_ops(block, self._segments[0]._start_idx, - fill_constant_vars) + insert_fill_constant_ops( + block, self._segments[0]._start_idx, fill_constant_vars + ) if cast_ops: insert_cast_ops(block, self._segments[0]._start_idx, cast_ops) @@ -1283,7 +1515,7 @@ def _prune_startup_program(self, block, shard): continue if self._optimizer_sharding and shard.is_param(output_name): continue - #TODO why do we remove op, when only one var is removed + # TODO why do we remove op, when only one var is removed block._remove_op(idx, sync=False) break @@ -1312,16 +1544,29 @@ def _build_groups(self): self.global_rank = self.role_maker._worker_index() self.global_endpoints = self.role_maker._get_trainer_endpoints() self.current_endpoint = self.global_endpoints[self.global_rank] - self._collective_helper = CollectiveHelper(self.role_maker, - nrings=self._nrings_sharding) - assert self.global_word_size % self.mp_degree == 0, \ - "global_word_size: {} should be divisible to the mp_degree: {}".format(self.global_word_size, self.mp_degree) - assert self.global_word_size % self.sharding_degree == 0, \ - "global_word_size: {} should be divisible to the sharding_degree: {}".format(self.global_word_size, self.sharding_degree) - assert self.global_word_size % self.pp_degree == 0, \ - "global_word_size: {} should be divisible to the pp_degree: {}".format(self.global_word_size, self.pp_degree) - assert self.global_word_size % self.dp_degree == 0, \ - "global_word_size: {} should be divisible to the dp_degree: {}".format(self.global_word_size, self.dp_degree) + self._collective_helper = CollectiveHelper( + self.role_maker, nrings=self._nrings_sharding + ) + assert ( + self.global_word_size % self.mp_degree == 0 + ), "global_word_size: {} should be divisible to the mp_degree: {}".format( + self.global_word_size, self.mp_degree + ) + assert ( + self.global_word_size % self.sharding_degree == 0 + ), "global_word_size: {} should be divisible to the sharding_degree: {}".format( + self.global_word_size, self.sharding_degree + ) + assert ( + self.global_word_size % self.pp_degree == 0 + ), "global_word_size: {} should be divisible to the pp_degree: {}".format( + self.global_word_size, self.pp_degree + ) + assert ( + self.global_word_size % self.dp_degree == 0 + ), "global_word_size: {} should be divisible to the dp_degree: {}".format( + self.global_word_size, self.dp_degree + ) # mp group if self.mp_degree > 1: @@ -1329,14 +1574,16 @@ def _build_groups(self): self.mp_rank = self.global_rank % self.mp_degree self.mp_group_id = self.global_rank // self.mp_degree self.mp_group_endpoints = [ - ep for idx, ep in enumerate(self.global_endpoints) + ep + for idx, ep in enumerate(self.global_endpoints) if idx // self.mp_degree == self.mp_group_id ] assert self.current_endpoint in self.mp_group_endpoints - assert len( - self.mp_group_endpoints - ) == self.mp_degree, "num of mp worker in group is [{}], but mp group size is [{}]".format( - len(self.mp_group_endpoints), self.mp_degree) + assert ( + len(self.mp_group_endpoints) == self.mp_degree + ), "num of mp worker in group is [{}], but mp group size is [{}]".format( + len(self.mp_group_endpoints), self.mp_degree + ) else: self.mp_degree = 1 self.mp_ring_id = -1 @@ -1347,23 +1594,28 @@ def _build_groups(self): # sharding if self.sharding_degree > 1: self.sharding_ring_id = 1 - self.sharding_rank = (self.global_rank // - self.mp_degree) % self.sharding_degree - self.sharding_group_id = self.global_rank // (self.mp_degree * - self.sharding_degree) + self.sharding_rank = ( + self.global_rank // self.mp_degree + ) % self.sharding_degree + self.sharding_group_id = self.global_rank // ( + self.mp_degree * self.sharding_degree + ) # mp + sharding + ... if self.mp_degree > 1: self.sharding_group_endpoints = [ - ep for idx, ep in enumerate(self.global_endpoints) - if (idx // (self.mp_degree * self.sharding_degree)) == self. - sharding_group_id and idx % self.mp_degree == self.mp_rank + ep + for idx, ep in enumerate(self.global_endpoints) + if (idx // (self.mp_degree * self.sharding_degree)) + == self.sharding_group_id + and idx % self.mp_degree == self.mp_rank ] # sharding + ... else: self.sharding_group_endpoints = [ - ep for idx, ep in enumerate(self.global_endpoints) - if (idx // (self.mp_degree * self.sharding_degree) - ) == self.sharding_group_id + ep + for idx, ep in enumerate(self.global_endpoints) + if (idx // (self.mp_degree * self.sharding_degree)) + == self.sharding_group_id ] assert self.current_endpoint in self.sharding_group_endpoints else: @@ -1378,20 +1630,28 @@ def _build_groups(self): self.pp_pair_ring_id = 20 # pipeline global ring_id set to 4 for sharding0, mp1, dp2, global3 self.pp_ring_id = 4 - self.pp_rank = self.global_rank // (self.sharding_degree * - self.mp_degree) % self.pp_degree + self.pp_rank = ( + self.global_rank + // (self.sharding_degree * self.mp_degree) + % self.pp_degree + ) # (NOTE): Already adjust for (outter-pure) dp self.pp_group_id = self.global_rank // ( - self.mp_degree * self.sharding_degree * self.pp_degree) + self.mp_degree * self.sharding_degree * self.pp_degree + ) pp_first_stage_idx = self.global_rank % ( - self.sharding_degree * self.mp_degree) + self.pp_group_id * ( - self.mp_degree * self.sharding_degree * self.pp_degree) + self.sharding_degree * self.mp_degree + ) + self.pp_group_id * ( + self.mp_degree * self.sharding_degree * self.pp_degree + ) pp_stage_offset = self.sharding_degree * self.mp_degree self.pp_group_endpoints = [] for i in range(self.pp_degree): self.pp_group_endpoints.append( - self.global_endpoints[pp_first_stage_idx + - pp_stage_offset * i]) + self.global_endpoints[ + pp_first_stage_idx + pp_stage_offset * i + ] + ) assert self.current_endpoint in self.pp_group_endpoints else: self.pp_ring_id = -1 @@ -1407,29 +1667,48 @@ def _build_groups(self): # sharding-hybrid-dp as one senario of outter-pure-dp local_pp_degree = self.pp_degree if os.getenv("PADDLE_MANUAL_PIPELINE_STAGE", None): - assert self.pp_degree == 2, ("For manually set pipeline, only " - "pp_degree = 2 is supported.") - assert self.global_word_size == self.mp_degree * self.sharding_degree * self.dp_degree, \ - "global work size [{}], mp_degree [{}], sharding_degree [{}], dp_degree [{}].".format( - self.global_word_size, self.mp_degree, self.sharding_degree, self.dp_degree) + assert self.pp_degree == 2, ( + "For manually set pipeline, only " "pp_degree = 2 is supported." + ) + assert ( + self.global_word_size + == self.mp_degree * self.sharding_degree * self.dp_degree + ), "global work size [{}], mp_degree [{}], sharding_degree [{}], dp_degree [{}].".format( + self.global_word_size, + self.mp_degree, + self.sharding_degree, + self.dp_degree, + ) local_pp_degree = 1 else: - assert self.global_word_size == self.mp_degree * self.sharding_degree * self.pp_degree * self.dp_degree, "mp_degree: [{}], sharding_degree: [{}], pp_degree: [{}], dp_degree: [{}]; BUT global nrank: [{}]".format( - self.mp_degree, self.sharding_degree, self.pp_degree, - self.dp_degree, self.global_word_size) + assert ( + self.global_word_size + == self.mp_degree + * self.sharding_degree + * self.pp_degree + * self.dp_degree + ), "mp_degree: [{}], sharding_degree: [{}], pp_degree: [{}], dp_degree: [{}]; BUT global nrank: [{}]".format( + self.mp_degree, + self.sharding_degree, + self.pp_degree, + self.dp_degree, + self.global_word_size, + ) if self.dp_degree > 1: self.dp_ring_id = 2 self.dp_rank = self.global_rank // ( - self.sharding_degree * self.mp_degree * local_pp_degree) + self.sharding_degree * self.mp_degree * local_pp_degree + ) dp_first_rank_idx = self.global_rank % ( - self.sharding_degree * self.mp_degree * local_pp_degree) - dp_offset = (self.sharding_degree * self.mp_degree * - local_pp_degree) + self.sharding_degree * self.mp_degree * local_pp_degree + ) + dp_offset = self.sharding_degree * self.mp_degree * local_pp_degree self.dp_group_endpoints = [] for i in range(self.dp_degree): self.dp_group_endpoints.append( - self.global_endpoints[dp_first_rank_idx + dp_offset * i]) + self.global_endpoints[dp_first_rank_idx + dp_offset * i] + ) assert self.current_endpoint in self.dp_group_endpoints logger.info("Hybrid DP mode turn on !") else: @@ -1458,8 +1737,9 @@ def _build_groups(self): logger.info("sharding group size: {}".format(self.sharding_degree)) logger.info("sharding rank: {}".format(self.sharding_rank)) logger.info("sharding group id: {}".format(self.sharding_group_id)) - logger.info("sharding group endpoints: {}".format( - self.sharding_group_endpoints)) + logger.info( + "sharding group endpoints: {}".format(self.sharding_group_endpoints) + ) logger.info("sharding ring id: {}".format(self.sharding_ring_id)) logger.info("#####" * 6) @@ -1472,15 +1752,15 @@ def _build_groups(self): logger.info("pure dp group size: {}".format(self.dp_degree)) logger.info("pure dp rank: {}".format(self.dp_rank)) - logger.info("pure dp group endpoints: {}".format( - self.dp_group_endpoints)) + logger.info( + "pure dp group endpoints: {}".format(self.dp_group_endpoints) + ) logger.info("pure dp ring id: {}".format(self.dp_ring_id)) logger.info("#####" * 6) return def _recreate_not_persist_param_as_var(self): - def recreate_not_persist_param_as_var(program): block = program.global_block() params = block.all_parameters() @@ -1504,14 +1784,16 @@ def recreate_not_persist_param_as_var(program): is_distributed = param.is_distributed block._remove_var(name, sync=False) - var = block.create_var(name=name, - shape=shape, - dtype=dtype, - type=type, - lod_level=lod_level, - stop_gradient=stop_gradient, - trainable=trainable, - persistable=False) + var = block.create_var( + name=name, + shape=shape, + dtype=dtype, + type=type, + lod_level=lod_level, + stop_gradient=stop_gradient, + trainable=trainable, + persistable=False, + ) if have_dist_attr: var.is_distributed = is_distributed @@ -1547,7 +1829,8 @@ def _initialization_broadcast(self): broadcast_params.add(op.desc.output_arg_names()[0]) for param in params_name: - if param in broadcast_params: continue + if param in broadcast_params: + continue rings = [] # need sync not distributed param in mp group @@ -1557,30 +1840,37 @@ def _initialization_broadcast(self): rings.append(self.dp_ring_id) for ring in rings: - startup_block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring, - 'root': 0, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Forward - }) + startup_block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring, + 'root': 0, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Forward, + }, + ) startup_block._sync_with_cpp() # sharding gradient merge def create_persistable_gradients_and_insert_merge_ops( - self, main_block, startup_block, insert_idx, grad_names, shard): + self, main_block, startup_block, insert_idx, grad_names, shard + ): for grad_name in grad_names: - assert get_grad_device( - grad_name, shard - ) == shard.worker_idx, "try to merge gradient not belong to current shard: [{}]".format( - grad_name) + assert ( + get_grad_device(grad_name, shard) == shard.worker_idx + ), "try to merge gradient not belong to current shard: [{}]".format( + grad_name + ) persistable_grad_name = grad_name + '@GradiantMerge' - assert grad_name not in self._grad2merged_grad, "grad [{}] already in grad2merged_grad, maybe you meet sharing weight case !".format( - grad_name) + assert ( + grad_name not in self._grad2merged_grad + ), "grad [{}] already in grad2merged_grad, maybe you meet sharing weight case !".format( + grad_name + ) self._grad2merged_grad[grad_name] = persistable_grad_name grad_var = main_block.var(grad_name) # create var @@ -1588,36 +1878,38 @@ def create_persistable_gradients_and_insert_merge_ops( name=persistable_grad_name, shape=grad_var.shape, dtype=grad_var.dtype, - persistable=True) + persistable=True, + ) startup_gradient_merge_var = startup_block.create_var( name=persistable_grad_name, shape=grad_var.shape, dtype=grad_var.dtype, - persistable=True) + persistable=True, + ) # merge gradient main_block._insert_op_without_sync( insert_idx, type="elementwise_add", - inputs={ - 'X': grad_name, - 'Y': gradient_merge_var - }, + inputs={'X': grad_name, 'Y': gradient_merge_var}, outputs={'Out': gradient_merge_var}, attrs={ 'axis': -1, 'use_mkldnn': False, - OP_ROLE_KEY: OpRole.Backward - }) + OP_ROLE_KEY: OpRole.Backward, + }, + ) # startup initialization - startup_block.append_op(type="fill_constant", - outputs={"Out": startup_gradient_merge_var}, - attrs={ - "shape": grad_var.shape, - "dtype": grad_var.dtype, - "value": float(0), - }) + startup_block.append_op( + type="fill_constant", + outputs={"Out": startup_gradient_merge_var}, + attrs={ + "shape": grad_var.shape, + "dtype": grad_var.dtype, + "value": float(0), + }, + ) main_block._sync_with_cpp() startup_block._sync_with_cpp() @@ -1630,14 +1922,17 @@ def _create_gm_cond(self, main_block): value=int(self._gradient_merge_acc_step), dtype='int32', persistable=True, - force_cpu=True) + force_cpu=True, + ) - zero_var = layers.create_global_var(name="gradient_merge_zero", - shape=[1], - value=int(0), - dtype='int32', - persistable=True, - force_cpu=True) + zero_var = layers.create_global_var( + name="gradient_merge_zero", + shape=[1], + value=int(0), + dtype='int32', + persistable=True, + force_cpu=True, + ) # Add step var & cond var current_step_var = layers.create_global_var( @@ -1646,42 +1941,40 @@ def _create_gm_cond(self, main_block): value=int(0), dtype='int32', persistable=True, - force_cpu=True) + force_cpu=True, + ) - cond_var = main_block.create_var(name="gradient_merge_cond", - shape=[1], - dtype='bool') + cond_var = main_block.create_var( + name="gradient_merge_cond", shape=[1], dtype='bool' + ) with device_guard("cpu"): # step_var = (step_var + 1) % k_step - main_block.append_op(type='increment', - inputs={'X': [current_step_var]}, - outputs={'Out': [current_step_var]}, - attrs={ - 'step': float(1), - OP_ROLE_KEY: OpRole.Optimize - }) - - main_block.append_op(type='elementwise_mod', - inputs={ - 'X': current_step_var, - 'Y': acc_step_var - }, - outputs={'Out': current_step_var}, - attrs={ - 'axis': -1, - OP_ROLE_KEY: OpRole.Optimize, - 'use_mkldnn': False - }) + main_block.append_op( + type='increment', + inputs={'X': [current_step_var]}, + outputs={'Out': [current_step_var]}, + attrs={'step': float(1), OP_ROLE_KEY: OpRole.Optimize}, + ) + + main_block.append_op( + type='elementwise_mod', + inputs={'X': current_step_var, 'Y': acc_step_var}, + outputs={'Out': current_step_var}, + attrs={ + 'axis': -1, + OP_ROLE_KEY: OpRole.Optimize, + 'use_mkldnn': False, + }, + ) # cond_var = (step_var == 0) - main_block.append_op(type='equal', - inputs={ - 'X': current_step_var, - 'Y': zero_var - }, - outputs={'Out': cond_var}, - attrs={OP_ROLE_KEY: OpRole.Optimize}) + main_block.append_op( + type='equal', + inputs={'X': current_step_var, 'Y': zero_var}, + outputs={'Out': cond_var}, + attrs={OP_ROLE_KEY: OpRole.Optimize}, + ) # paddle.static.Print(current_step_var, message="in FWBW last conditional") return cond_var @@ -1708,35 +2001,37 @@ def _true_apply_gradient(self): # allreduce grad@gradientmerge if self.hybrid_dp: - assert self.dp_ring_id >= 0, "dp_ring_id should larger than 0 when in sharding&DP mode" + assert ( + self.dp_ring_id >= 0 + ), "dp_ring_id should larger than 0 when in sharding&DP mode" for grad, merged_grad in self._grad2merged_grad.items(): merged_grad_var = main_block.var(merged_grad) - cur_block.append_op(type='c_allreduce_sum', - inputs={'X': merged_grad_var}, - outputs={'Out': merged_grad_var}, - attrs={ - 'ring_id': self.dp_ring_id, - 'use_calc_stream': True, - OP_ROLE_KEY: OpRole.Optimize - }) + cur_block.append_op( + type='c_allreduce_sum', + inputs={'X': merged_grad_var}, + outputs={'Out': merged_grad_var}, + attrs={ + 'ring_id': self.dp_ring_id, + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) # grad@gradientmerge / acc_step for grad, merged_grad in self._grad2merged_grad.items(): # grad /= k_steps merged_grad_var = main_block.var(merged_grad) - cur_block.append_op(type='scale', - inputs={'X': merged_grad_var}, - outputs={'Out': merged_grad_var}, - attrs={ - 'scale': - 1.0 / float(self._gradient_merge_acc_step), - 'bias': - 0.0, - 'bias_after_scale': - False, - OP_ROLE_KEY: - OpRole.Optimize - }) + cur_block.append_op( + type='scale', + inputs={'X': merged_grad_var}, + outputs={'Out': merged_grad_var}, + attrs={ + 'scale': 1.0 / float(self._gradient_merge_acc_step), + 'bias': 0.0, + 'bias_after_scale': False, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) # re-create optimize ops already_moved_var_names = [] @@ -1747,15 +2042,19 @@ def _true_apply_gradient(self): for input_name in new_op_desc.input_arg_names(): if input_name in self._grad2merged_grad: new_op_desc._rename_input( - input_name, self._grad2merged_grad[input_name]) + input_name, self._grad2merged_grad[input_name] + ) for output_name in new_op_desc.output_arg_names(): if output_name in self._grad2merged_grad: new_op_desc._rename_output( - output_name, self._grad2merged_grad[output_name]) + output_name, self._grad2merged_grad[output_name] + ) # move non temp optimize vars from block0 to cond block - if output_name not in already_moved_var_names and output_name not in self._grad2merged_grad.keys( + if ( + output_name not in already_moved_var_names + and output_name not in self._grad2merged_grad.keys() ): var_ = self._main_program.global_block().var(output_name) if not var_.persistable: @@ -1764,11 +2063,14 @@ def _true_apply_gradient(self): shape_ = var_.shape type_ = var_.dtype self._main_program.global_block()._remove_var( - var_.name, sync=False) - self.cond_block.create_var(name=name_, - shape=shape_, - dtype=type_, - persistable=False) + var_.name, sync=False + ) + self.cond_block.create_var( + name=name_, + shape=shape_, + dtype=type_, + persistable=False, + ) already_moved_var_names.append(name_) self._main_program.global_block()._sync_with_cpp() @@ -1777,14 +2079,16 @@ def _true_apply_gradient(self): # fill zero to grad@gradientmerge for grad, merged_grad in self._grad2merged_grad.items(): merged_grad_var = main_block.var(merged_grad) - cur_block.append_op(type='fill_constant', - outputs={'Out': merged_grad_var}, - attrs={ - "shape": merged_grad_var.shape, - "dtype": merged_grad_var.dtype, - "value": float(0), - OP_ROLE_KEY: OpRole.Optimize - }) + cur_block.append_op( + type='fill_constant', + outputs={'Out': merged_grad_var}, + attrs={ + "shape": merged_grad_var.shape, + "dtype": merged_grad_var.dtype, + "value": float(0), + OP_ROLE_KEY: OpRole.Optimize, + }, + ) # lr_var = main_block.var("gradient_merge_current_step") # paddle.static.Print(lr_var, message="in OPTIMIZE last conditional") @@ -1796,7 +2100,10 @@ def _sharding_gradient_merge(self): create cond block """ - if self.gradient_merge_mode != "sharding_gm" or self._gradient_merge_acc_step <= 1: + if ( + self.gradient_merge_mode != "sharding_gm" + or self._gradient_merge_acc_step <= 1 + ): return main_block = self._main_program.global_block() @@ -1815,7 +2122,8 @@ def _sharding_gradient_merge(self): main_block._remove_op(op_idx, sync=False) tmp_copy_block._sync_with_cpp() self.original_optimize_ops_desc = list( - reversed(self.original_optimize_ops_desc)) + reversed(self.original_optimize_ops_desc) + ) # back to block 0 self._main_program._rollback() @@ -1832,18 +2140,17 @@ def _sharding_gradient_merge(self): # cond op step_scope = self._main_program.global_block().create_var( - type=core.VarDesc.VarType.STEP_SCOPES) + type=core.VarDesc.VarType.STEP_SCOPES + ) conditional_block_op = self._main_program.global_block().append_op( type='conditional_block', inputs={ 'Cond': cond, 'Input': [], }, - outputs={ - 'Out': [], - 'Scope': [step_scope] - }, + outputs={'Out': [], 'Scope': [step_scope]}, attrs={ 'sub_block': cond_block, 'is_scalar_condition': True, - }) + }, + ) diff --git a/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py index 5f01552c71bde..0cd86ad08bde2 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/tensor_parallel_optimizer.py @@ -13,13 +13,20 @@ import paddle.fluid as fluid from .meta_optimizer_base import MetaOptimizerBase -from .common import CollectiveHelper, OP_ROLE_KEY, OP_ROLE_VAR_KEY, OpRole, is_backward_op, is_loss_grad_op, is_optimizer_op +from .common import ( + CollectiveHelper, + OP_ROLE_KEY, + OP_ROLE_VAR_KEY, + OpRole, + is_backward_op, + is_loss_grad_op, + is_optimizer_op, +) __all__ = [] class TensorParallelOptimizer(MetaOptimizerBase): - def __init__(self, optimizer): super(TensorParallelOptimizer, self).__init__(optimizer) self.inner_opt = optimizer @@ -36,13 +43,15 @@ def __init__(self, optimizer): self.global_ring_id = 1 self.dp_ring_id = 2 - def _set_basic_info(self, loss, role_maker, user_defined_optimizer, - user_defined_strategy): - super(TensorParallelOptimizer, - self)._set_basic_info(loss, role_maker, user_defined_optimizer, - user_defined_strategy) + def _set_basic_info( + self, loss, role_maker, user_defined_optimizer, user_defined_strategy + ): + super(TensorParallelOptimizer, self)._set_basic_info( + loss, role_maker, user_defined_optimizer, user_defined_strategy + ) self.mp_degree = user_defined_strategy.tensor_parallel_configs[ - 'tensor_parallel_degree'] + 'tensor_parallel_degree' + ] def _can_apply(self): if not self.role_maker._is_collective: @@ -69,23 +78,25 @@ def _broadcast_params(self, ring_id, mp_mode): if param.is_distributed and mp_mode: continue - block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - 'root': 0, - OP_ROLE_KEY: OpRole.Forward - }) - - if not param: return # no parameter on this device - block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward - }) + block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + 'root': 0, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + + if not param: + return # no parameter on this device + block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: OpRole.Forward}, + ) def _get_process_group_info(self): # global ring info @@ -98,7 +109,8 @@ def _get_process_group_info(self): self.mp_nranks = self.mp_degree mp_group = self.rank // self.mp_degree self.mp_endpoints = [ - self.endpoints[i] for i in range(self.global_nranks) + self.endpoints[i] + for i in range(self.global_nranks) if i // self.mp_degree == mp_group ] @@ -117,33 +129,47 @@ def _init_process_group(self): collective_helper = CollectiveHelper(self.role_maker, wait_port=False) # Create global ring for all gpus - collective_helper._init_communicator(self.startup_program, - self.current_endpoint, - self.global_endpoints, - self.global_rank, - self.global_ring_id, True, - self.global_ring_id, True) + collective_helper._init_communicator( + self.startup_program, + self.current_endpoint, + self.global_endpoints, + self.global_rank, + self.global_ring_id, + True, + self.global_ring_id, + True, + ) # Create model parallel ring for all gpus - collective_helper._init_communicator(self.startup_program, - self.current_endpoint, - self.mp_endpoints, self.mp_rank, - self.mp_ring_id, True, - self.global_ring_id, True) + collective_helper._init_communicator( + self.startup_program, + self.current_endpoint, + self.mp_endpoints, + self.mp_rank, + self.mp_ring_id, + True, + self.global_ring_id, + True, + ) self._broadcast_params(self.mp_ring_id, mp_mode=True) # Create dp rings if self.nranks > self.mp_degree: collective_helper._init_communicator( - self.startup_program, self.current_endpoint, self.dp_endpoints, - self.dp_rank, self.dp_ring_id, True, self.global_ring_id, True) + self.startup_program, + self.current_endpoint, + self.dp_endpoints, + self.dp_rank, + self.dp_ring_id, + True, + self.global_ring_id, + True, + ) self._broadcast_params(self.dp_ring_id, mp_mode=False) - def minimize_impl(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize_impl( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): self.endpoints = self.role_maker._get_trainer_endpoints() self.current_endpoint = self.endpoints[self.role_maker._worker_index()] self.startup_program = startup_program @@ -151,7 +177,8 @@ def minimize_impl(self, self.startup_program = fluid.default_startup_program() optimize_ops, params_grads = self.inner_opt.minimize( - loss, self.startup_program, parameter_list, no_grad_set) + loss, self.startup_program, parameter_list, no_grad_set + ) self.main_program = loss.block.program self.nranks = len(self.endpoints) @@ -180,14 +207,16 @@ def _insert_loss_grad_ops(self, loss, dp_degree): for idx, op in reversed(list(enumerate(block.ops))): if is_loss_grad_op(op): loss_grad_var = block.vars[op.output_arg_names[0]] - block._insert_op(idx + 1, - type='scale', - inputs={'X': loss_grad_var}, - outputs={'Out': loss_grad_var}, - attrs={ - 'scale': 1.0 / dp_degree, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op( + idx + 1, + type='scale', + inputs={'X': loss_grad_var}, + outputs={'Out': loss_grad_var}, + attrs={ + 'scale': 1.0 / dp_degree, + OP_ROLE_KEY: OpRole.Backward, + }, + ) break def _insert_allreduce_ops(self, loss, ring_id): @@ -205,33 +234,36 @@ def _insert_allreduce_ops(self, loss, ring_id): grad = block.vars[op_role_var[i + 1]] if offset == idx: offset += 1 - block._insert_op(offset, - type='c_sync_calc_stream', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={OP_ROLE_KEY: OpRole.Backward}) + block._insert_op( + offset, + type='c_sync_calc_stream', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={OP_ROLE_KEY: OpRole.Backward}, + ) offset += 1 - block._insert_op(offset, - type='c_allreduce_sum', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op( + offset, + type='c_allreduce_sum', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Backward, + }, + ) if grad is None: return for idx, op in list(enumerate(block.ops)): if is_optimizer_op(op): - block._insert_op(idx, - type='c_sync_comm_stream', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op( + idx, + type='c_sync_comm_stream', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={'ring_id': ring_id, OP_ROLE_KEY: OpRole.Backward}, + ) break diff --git a/python/paddle/distributed/fleet/meta_parallel/meta_parallel_base.py b/python/paddle/distributed/fleet/meta_parallel/meta_parallel_base.py index f5b8660bd88d4..9386893728ab6 100644 --- a/python/paddle/distributed/fleet/meta_parallel/meta_parallel_base.py +++ b/python/paddle/distributed/fleet/meta_parallel/meta_parallel_base.py @@ -18,10 +18,10 @@ class MetaParallelBase(Layer): - def __init__(self, layers, hcg, strategy): - super(MetaParallelBase, - self).__init__(layers.full_name() + "_meta_parallel_base") + super(MetaParallelBase, self).__init__( + layers.full_name() + "_meta_parallel_base" + ) self._layers = layers self._hcg = hcg self._strategy = strategy diff --git a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py index f1a812391801b..fec5005627b88 100755 --- a/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py +++ b/python/paddle/distributed/fleet/meta_parallel/parallel_layers/pp_layers.py @@ -54,7 +54,6 @@ class LayerDesc(object): - def __init__(self, layer_func, *inputs, **kwargs): self.layer_func = layer_func self.inputs = inputs @@ -62,25 +61,28 @@ def __init__(self, layer_func, *inputs, **kwargs): if not issubclass(layer_func, Layer): raise TypeError( - "The input(layer_func) should be a derived class of Layer.") + "The input(layer_func) should be a derived class of Layer." + ) def build_layer(self): return self.layer_func(*self.inputs, **self.kwargs) def __repr__(self): - return layer_to_str(self.layer_func.__name__, *self.inputs, - **self.kwargs) + return layer_to_str( + self.layer_func.__name__, *self.inputs, **self.kwargs + ) class SharedLayerDesc(LayerDesc): - - def __init__(self, - key, - layer_func, - forward_func=None, - shared_weight_attr='weight', - *inputs, - **kwargs): + def __init__( + self, + key, + layer_func, + forward_func=None, + shared_weight_attr='weight', + *inputs, + **kwargs + ): super(SharedLayerDesc, self).__init__(layer_func, *inputs, **kwargs) self.layer_name = key self.forward_func = forward_func @@ -88,12 +90,13 @@ def __init__(self, class SegmentLayers(object): - - def __init__(self, - layers_desc, - num_parts, - method="uniform", - num_virtual_pipeline_stage=None): + def __init__( + self, + layers_desc, + num_parts, + method="uniform", + num_virtual_pipeline_stage=None, + ): self._layers_desc = layers_desc self.method = method self.num_parts = num_parts @@ -101,7 +104,9 @@ def __init__(self, self.num_virtual_pipeline_stage = num_virtual_pipeline_stage if self.num_virtual_pipeline_stage is not None: self.total_parts = num_parts * self.num_virtual_pipeline_stage - assert self.num_items >= self.num_parts, "layer number should be greater than number of segments" + assert ( + self.num_items >= self.num_parts + ), "layer number should be greater than number of segments" def do_segment(self): if self.method == "uniform": @@ -115,12 +120,17 @@ def do_segment(self): for idx in weight_idxs: weights[idx] = 1 - actual_num_parts = self.num_parts if self.num_virtual_pipeline_stage is None else self.total_parts - - assert sum( - weights - ) % actual_num_parts == 0, "number of layers ({}) should be divided by part number({})".format( - sum(weights), actual_num_parts) + actual_num_parts = ( + self.num_parts + if self.num_virtual_pipeline_stage is None + else self.total_parts + ) + + assert ( + sum(weights) % actual_num_parts == 0 + ), "number of layers ({}) should be divided by part number({})".format( + sum(weights), actual_num_parts + ) part_size = sum(weights) // actual_num_parts result = [0 for _ in range(actual_num_parts + 1)] @@ -153,8 +163,9 @@ def _gen_layer_weight(self, layername): if regex.search(name): weight_idxs.append(idx) - assert len( - weight_idxs) > 0, "weight_idxs' length should be greater than 0" + assert ( + len(weight_idxs) > 0 + ), "weight_idxs' length should be greater than 0" return weight_idxs def uniform(self, num_items, num_parts): @@ -167,7 +178,6 @@ def uniform(self, num_items, num_parts): class PipelineLayerChunk(Layer): - def __init__(self): super(PipelineLayerChunk, self).__init__() self.run_function = [] @@ -189,7 +199,8 @@ def forward(self, *args, **kwargs): # behavior under recompute circumstance. raise PermissionError( "The forward function of PipelineLayerChunk cannot be called directly. " - "Please call forward function of PipelineLayer.") + "Please call forward function of PipelineLayer." + ) class PipelineLayer(Layer): @@ -271,32 +282,41 @@ def __init__(self, num_classes=10, **kwargs): """ - def __init__(self, - layers, - num_stages=None, - topology=None, - loss_fn=None, - seg_method="uniform", - recompute_interval=0, - recompute_ctx=None, - num_virtual_pipeline_stages=None): + def __init__( + self, + layers, + num_stages=None, + topology=None, + loss_fn=None, + seg_method="uniform", + recompute_interval=0, + recompute_ctx=None, + num_virtual_pipeline_stages=None, + ): super(PipelineLayer, self).__init__() if num_stages is None and topology is None: raise ValueError("should provide num_stages or topology") if num_virtual_pipeline_stages: - assert isinstance(num_virtual_pipeline_stages, int), \ - "virtual_pipeline_stage should be None or an int" + assert isinstance( + num_virtual_pipeline_stages, int + ), "virtual_pipeline_stage should be None or an int" if num_virtual_pipeline_stages > 1: logger.info( "set num_virtual_pipeline_stages > 1 means using interleave scheduler instead of 1f1b scheduler" ) - assert isinstance(seg_method, str), \ - "seg_method should be a str for interleave scheduler" - assert seg_method.startswith('layer:'), \ - "seg_method shoud be start with layer: for interleave scheduler" - - self._num_virtual_pipeline_stages = 1 if num_virtual_pipeline_stages is None else num_virtual_pipeline_stages + assert isinstance( + seg_method, str + ), "seg_method should be a str for interleave scheduler" + assert seg_method.startswith( + 'layer:' + ), "seg_method shoud be start with layer: for interleave scheduler" + + self._num_virtual_pipeline_stages = ( + 1 + if num_virtual_pipeline_stages is None + else num_virtual_pipeline_stages + ) # lazy import import paddle.distributed as dist @@ -310,13 +330,17 @@ def __init__(self, self.recompute_ctx = recompute_ctx if recompute_interval > 0: - assert recompute_ctx is not None, "recompute_ctx must be not None for recompute." + assert ( + recompute_ctx is not None + ), "recompute_ctx must be not None for recompute." offload = recompute_ctx.get('offload', False) partition = recompute_ctx.get('partition', False) logger.info( - "Start Recompute for PipeLineParallel. recompute_offload: {}, recompute_partition: {}" - .format(offload, partition)) + "Start Recompute for PipeLineParallel. recompute_offload: {}, recompute_partition: {}".format( + offload, partition + ) + ) world_size = dist.get_world_size() self.global_rank = dist.get_rank() @@ -325,22 +349,28 @@ def __init__(self, self._stage_id = self._topo.get_coord(self.global_rank).pipe self._num_stages = self._topo.get_dim_size("pipe") if num_stages: - assert self._num_stages == num_stages, "num_stages should be equal to be %d" % ( - self._num_stages) + assert ( + self._num_stages == num_stages + ), "num_stages should be equal to be %d" % (self._num_stages) else: # construct default topology if world_size % num_stages != 0: raise ValueError( "should provide correct num_stages({}) " "which can be divided by world_size({})".format( - num_stages, world_size)) + num_stages, world_size + ) + ) dp_num = world_size // num_stages - self._topo = fleet.CommunicateTopology(["data", "pipe", "model"], - [dp_num, num_stages, 1]) + self._topo = fleet.CommunicateTopology( + ["data", "pipe", "model"], [dp_num, num_stages, 1] + ) self._stage_id = self._topo.get_coord(self.global_rank).pipe self._num_stages = self._topo.get_dim_size("pipe") - self._total_stages_with_virtual_stages = self._num_stages * self._num_virtual_pipeline_stages + self._total_stages_with_virtual_stages = ( + self._num_stages * self._num_virtual_pipeline_stages + ) # initialize segment self._layers_desc = list(self.layers) @@ -378,16 +408,22 @@ def get_stage_from_index(self, layer_idx): start_idx = virtual_pp_rank * self._num_stages for stage in range(self._num_stages): # stage mark the real pp stage - if self.segment_parts[start_idx + - stage] <= layer_idx < self.segment_parts[ - start_idx + stage + 1]: + if ( + self.segment_parts[start_idx + stage] + <= layer_idx + < self.segment_parts[start_idx + stage + 1] + ): return stage def get_num_virtual_stages(self): return self._num_virtual_pipeline_stages def get_model_chunks(self): - return None if self._num_virtual_pipeline_stages == 1 else self._model_chunks + return ( + None + if self._num_virtual_pipeline_stages == 1 + else self._model_chunks + ) def _construct_shared_comm(self): shared_comm = {} @@ -395,17 +431,21 @@ def _construct_shared_comm(self): return layers_desc = self._layers_desc - shared_layer_names = set(s.layer_name for s in layers_desc - if isinstance(s, SharedLayerDesc)) + shared_layer_names = set( + s.layer_name for s in layers_desc if isinstance(s, SharedLayerDesc) + ) for key in shared_layer_names: shared_layers = [] for idx, layer in enumerate(layers_desc): - if isinstance(layer, - SharedLayerDesc) and layer.layer_name == key: + if ( + isinstance(layer, SharedLayerDesc) + and layer.layer_name == key + ): shared_layers.append(idx) shared_stages = set( - self.get_stage_from_index(idx) for idx in shared_layers) + self.get_stage_from_index(idx) for idx in shared_layers + ) self._dp_degree = self._topo.get_dim('data') self._mp_degree = self._topo.get_dim('model') self._sharding_degree = self._topo.get_dim('sharding') @@ -422,7 +462,9 @@ def _construct_shared_comm(self): pipe=s, data=dp, sharding=sharding, - model=mp)) + model=mp, + ) + ) group = paddle.distributed.new_group(ranks=shared_ranks) if self.global_rank in shared_ranks: @@ -431,8 +473,9 @@ def _construct_shared_comm(self): shared_comm[key] = { 'ranks': shared_ranks, 'group': group, - 'weight_attr': - self.shared_weight_attrs[key], + 'weight_attr': self.shared_weight_attrs[ + key + ], 'layer': self.shared_layers[key], } return shared_comm @@ -440,10 +483,11 @@ def _construct_shared_comm(self): def _synchronize_shared_weights(self): for key, comm in self.shared_comm.items(): with paddle.framework.no_grad(): - paddle.distributed.broadcast(getattr(comm['layer'], - comm['weight_attr']), - src=min(comm['ranks']), - group=comm['group']) + paddle.distributed.broadcast( + getattr(comm['layer'], comm['weight_attr']), + src=min(comm['ranks']), + group=comm['group'], + ) for param in comm['layer'].parameters(): if self.global_rank != min(comm['ranks']): @@ -455,8 +499,9 @@ def allreduce_shared_weight_gradients(self): # need use trace_op to allreduce weight if in_dygraph_mode(): with paddle.framework.no_grad(): - paddle.distributed.all_reduce(param.grad, - group=comm['group']) + paddle.distributed.all_reduce( + param.grad, group=comm['group'] + ) else: with paddle.framework.no_grad(): paddle.fluid.framework._dygraph_tracer().trace_op( @@ -465,8 +510,9 @@ def allreduce_shared_weight_gradients(self): outputs={'Out': param._grad_ivar()}, attrs={ 'ring_id': comm['group'].id, - 'use_calc_stream': True - }) + 'use_calc_stream': True, + }, + ) def _segment_network_for_interleave(self, seg_method): logger.info("start segment network for interleave scheduler") @@ -474,14 +520,20 @@ def _segment_network_for_interleave(self, seg_method): self._layers_desc, num_parts=self._num_stages, method=seg_method, - num_virtual_pipeline_stage=self._num_virtual_pipeline_stages) + num_virtual_pipeline_stage=self._num_virtual_pipeline_stages, + ) self.segment_parts = seg.do_segment() - logger.info("segment result:" + - ", ".join(str(arg) for arg in self.segment_parts)) + logger.info( + "segment result:" + + ", ".join(str(arg) for arg in self.segment_parts) + ) - for i in range(self._stage_id, self._total_stages_with_virtual_stages, - self._num_stages): + for i in range( + self._stage_id, + self._total_stages_with_virtual_stages, + self._num_stages, + ): # If there are 2 real pp stages and 2 virtual pp stages, and the model has 8 layers. # Layers [0, 1], [4, 5] will be assigned to the first real pp stage. # Layers [2, 3], [6, 7] will be assigned to the second real pp stage. @@ -497,13 +549,15 @@ def _segment_network_for_interleave(self, seg_method): def _segment_network(self, seg_method): logger.info("start segment network..") - seg = SegmentLayers(self._layers_desc, - num_parts=self._num_stages, - method=seg_method) + seg = SegmentLayers( + self._layers_desc, num_parts=self._num_stages, method=seg_method + ) self.segment_parts = seg.do_segment() - logger.info("segment result:" + - ", ".join(str(arg) for arg in self.segment_parts)) + logger.info( + "segment result:" + + ", ".join(str(arg) for arg in self.segment_parts) + ) self._start_pos = self.segment_parts[self._stage_id] self._end_pos = self.segment_parts[self._stage_id + 1] @@ -511,22 +565,30 @@ def _segment_network(self, seg_method): def _print_segmentation_for_debug(self): # print information for debug - for stage in range(self._num_stages * - self._num_virtual_pipeline_stages): + for stage in range( + self._num_stages * self._num_virtual_pipeline_stages + ): start = self.segment_parts[stage] end = self.segment_parts[stage + 1] - logger.info("stage={}, global_rank={} ,layer_number={}".format( - stage, self.global_rank, end - start)) + logger.info( + "stage={}, global_rank={} ,layer_number={}".format( + stage, self.global_rank, end - start + ) + ) for index, layer in enumerate(self._layers_desc[start:end]): logger.info("{}: {}".format(index + start, str(layer))) if self._num_virtual_pipeline_stages > 1: for stage in range(self._num_stages): - stage_to_virtual_stage_info = "stage {} contains virtual stages: ".format( - stage) - for i in range(stage, self._total_stages_with_virtual_stages, - self._num_stages): + stage_to_virtual_stage_info = ( + "stage {} contains virtual stages: ".format(stage) + ) + for i in range( + stage, + self._total_stages_with_virtual_stages, + self._num_stages, + ): stage_to_virtual_stage_info += " {},".format(i) logger.info(stage_to_virtual_stage_info) @@ -572,9 +634,11 @@ def _build_layer_impl(self, start, end): if layer.layer_name not in self.shared_layers: self.shared_layers[layer.layer_name] = layer.build_layer() self.shared_weight_attrs[ - layer.layer_name] = layer.shared_weight_attr + layer.layer_name + ] = layer.shared_weight_attr for param in self.shared_layers[ - layer.layer_name].parameters(): + layer.layer_name + ].parameters(): setattr(param, "is_firstly_shared", True) if layer.forward_func is None: @@ -582,8 +646,11 @@ def _build_layer_impl(self, start, end): else: run_function.append( - partial(layer.forward_func, - self.shared_layers[layer.layer_name])) + partial( + layer.forward_func, + self.shared_layers[layer.layer_name], + ) + ) elif isinstance(layer, LayerDesc): model = layer.build_layer() @@ -612,11 +679,15 @@ def execute_func(*x): def forward(self, input, chunk_id=None): if chunk_id is not None: assert isinstance(chunk_id, int), "chunk_id should be an int" - assert self._num_virtual_pipeline_stages > 1, \ - "chunk_id is only valid when using virtual pipeline stage" - assert chunk_id < len(self._model_chunks), \ - "The virtual pipeline only has {} chunks, " \ - "but received chunk_id {}.".format(len(self._model_chunks), chunk_id) + assert ( + self._num_virtual_pipeline_stages > 1 + ), "chunk_id is only valid when using virtual pipeline stage" + assert chunk_id < len(self._model_chunks), ( + "The virtual pipeline only has {} chunks, " + "but received chunk_id {}.".format( + len(self._model_chunks), chunk_id + ) + ) # Get the target model chunk. model_chunk = self._model_chunks[chunk_id] # Update the self.run_function to the target run functions. @@ -634,20 +705,25 @@ def forward(self, input, chunk_id=None): funcs = self.run_function[start_idx:end_idx] if not isinstance(input, tuple): - input = (input, ) + input = (input,) if self._need_recompute(funcs, input): input = recompute_hybrid( self.recompute_ctx, - self.forward_function(start_idx, end_idx), *input) + self.forward_function(start_idx, end_idx), + *input + ) else: input = self.forward_function(start_idx, end_idx)(*input) return input def _need_recompute(self, funcs, inputs): - if not any(input_.stop_gradient == False - for input_ in inputs if isinstance(input_, paddle.Tensor)): + if not any( + input_.stop_gradient == False + for input_ in inputs + if isinstance(input_, paddle.Tensor) + ): return False params = [f.parameters() for f in funcs if isinstance(f, Layer)] @@ -671,11 +747,18 @@ def _offset_dirname(ckpt_dir, local_layer_idx, local_chunk_id=None): if self._num_virtual_pipeline_stages > 1: # add virtual pipeline info to the save path assert local_chunk_id is not None - virtual_pipeline_stage_message = "-virtual_pp_stage_{:0>2d}".format( - local_chunk_id) - layer_save_path = os.path.join(ckpt_dir, - 'layer_{:0>2d}'.format(idx)) - layer_save_path = layer_save_path + virtual_pipeline_stage_message + rank_message + '-model_states.pdparams' + virtual_pipeline_stage_message = ( + "-virtual_pp_stage_{:0>2d}".format(local_chunk_id) + ) + layer_save_path = os.path.join( + ckpt_dir, 'layer_{:0>2d}'.format(idx) + ) + layer_save_path = ( + layer_save_path + + virtual_pipeline_stage_message + + rank_message + + '-model_states.pdparams' + ) return layer_save_path def _save_model(run_functions, local_chunk_id=None): @@ -698,7 +781,8 @@ def _save_model(run_functions, local_chunk_id=None): def set_state_dir(self, path): assert os.path.exists( - path), "{} not found, please check the path".format(path) + path + ), "{} not found, please check the path".format(path) def _load_model(run_functions, local_chunk_id=None): for idx, layer in enumerate(run_functions): @@ -712,21 +796,26 @@ def _load_model(run_functions, local_chunk_id=None): pos_offset = self._start_poss[local_chunk_id] layer_idx = idx + pos_offset layer_save_path = os.path.join( - path, 'layer_{0:0>2d}'.format(layer_idx)) + path, 'layer_{0:0>2d}'.format(layer_idx) + ) if self._num_virtual_pipeline_stages > 1: # add virtual pipeline info to the path assert local_chunk_id is not None - layer_save_path = layer_save_path + "-virtual_pp_stage_{:0>2d}".format( - local_chunk_id) - model_files = glob.glob(layer_save_path + - "*model_states.pdparams") + layer_save_path = ( + layer_save_path + + "-virtual_pp_stage_{:0>2d}".format(local_chunk_id) + ) + model_files = glob.glob( + layer_save_path + "*model_states.pdparams" + ) model_files.sort() mp_rank = self._topo.get_coord(self.global_rank).model mp_world_size = self._topo.get_dim('model') num_files = len(model_files) - load_param_path = model_files[mp_rank * num_files // - mp_world_size] + load_param_path = model_files[ + mp_rank * num_files // mp_world_size + ] model_state_dict = paddle.load(load_param_path) layer.set_state_dict(model_state_dict) diff --git a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py index 937122e8a7b33..89a3d619218b8 100755 --- a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py @@ -29,27 +29,31 @@ class PipelineParallel(MetaParallelBase): - def __init__(self, layers, hcg, strategy): if not isinstance(layers, PipelineLayer): raise TypeError( - "The Layer should be a derived class of PipelineLayer.") + "The Layer should be a derived class of PipelineLayer." + ) super(PipelineParallel, self).__init__(layers, hcg, strategy) self.use_data_parallel = self._hcg.get_data_parallel_world_size() > 1 self.use_model_parallel = self._hcg.get_model_parallel_world_size() > 1 - self.use_sharding_parallel = self._hcg.get_sharding_parallel_world_size( - ) > 1 + self.use_sharding_parallel = ( + self._hcg.get_sharding_parallel_world_size() > 1 + ) self.total_loss = None self.micro_batch_size = self._strategy.pipeline_configs[ - 'micro_batch_size'] + 'micro_batch_size' + ] self.accumulate_steps = self._strategy.pipeline_configs[ - 'accumulate_steps'] + 'accumulate_steps' + ] # If sent tensor are not the same from different hosts, # they shouldn't been sent partially and then concated as a whole tensor. self._enable_partial_send_recv = self._strategy.pipeline_configs[ - 'enable_partial_send_recv'] + 'enable_partial_send_recv' + ] self._using_cache = self._strategy.pipeline_configs['p2p_cache_shape'] self.num_stages = self._hcg.get_pipe_parallel_world_size() @@ -61,16 +65,20 @@ def __init__(self, layers, hcg, strategy): self._real_pp_world_size = self.num_stages self._real_pp_rank = self.stage_id - p2p.initialize_p2p_groups(hcg, self._using_cache, - self._enable_partial_send_recv) + p2p.initialize_p2p_groups( + hcg, self._using_cache, self._enable_partial_send_recv + ) self.global_rank = self._hcg.get_global_rank() self.micro_batch_id = 0 self._compute_loss = True - logger.info("Pipeline Info -- num_stages: {}, stage_id: {}".format( - self.num_stages, self.stage_id)) + logger.info( + "Pipeline Info -- num_stages: {}, stage_id: {}".format( + self.num_stages, self.stage_id + ) + ) if self.use_model_parallel: logger.info("start broadcast mp parameters") @@ -122,7 +130,7 @@ def forward_backward_pipeline(self, data, scaler=None): # store data id for micro_batch self.micro_batch_id = 0 - startup_steps = (self.num_stages - self.stage_id - 1) + startup_steps = self.num_stages - self.stage_id - 1 startup_steps = min(startup_steps, self.accumulate_steps) steady_steps = self.accumulate_steps - startup_steps @@ -142,39 +150,46 @@ def forward_backward_pipeline(self, data, scaler=None): input_tensor = p2p.recv_forward(self.is_pipeline_first_stage()) for i in range(steady_steps): - last_iter = (i == (steady_steps - 1)) + last_iter = i == (steady_steps - 1) output_tensor = self._forward_step(input_tensor) output_tensor_grad = p2p.send_forward_recv_backward( - output_tensor, self.is_pipeline_last_stage()) + output_tensor, self.is_pipeline_last_stage() + ) input_buffers.append(input_tensor) output_buffers.append(output_tensor) input_tensor, output_tensor = input_buffers.pop( - 0), output_buffers.pop(0) + 0 + ), output_buffers.pop(0) - input_tensor_grad = self._backward_step(input_tensor, output_tensor, - output_tensor_grad) + input_tensor_grad = self._backward_step( + input_tensor, output_tensor, output_tensor_grad + ) if last_iter: input_tensor = None - p2p.send_backward(input_tensor_grad, - self.is_pipeline_first_stage()) + p2p.send_backward( + input_tensor_grad, self.is_pipeline_first_stage() + ) else: input_tensor = p2p.send_backward_recv_forward( - input_tensor_grad, self.is_pipeline_first_stage()) + input_tensor_grad, self.is_pipeline_first_stage() + ) for i in range(startup_steps): input_tensor = input_buffers.pop(0) output_tensor = output_buffers.pop(0) output_tensor_grad = p2p.recv_backward( - self.is_pipeline_last_stage()) + self.is_pipeline_last_stage() + ) - input_tensor_grad = self._backward_step(input_tensor, output_tensor, - output_tensor_grad) + input_tensor_grad = self._backward_step( + input_tensor, output_tensor, output_tensor_grad + ) p2p.send_backward(input_tensor_grad, self.is_pipeline_first_stage()) self._layers.allreduce_shared_weight_gradients() @@ -186,17 +201,20 @@ def _prepare_training(self, data, optimizer, lr_scheduler): # reset the virtual pp rank for each run self.set_virtual_pipeline_rank(0) - assert isinstance(optimizer, HybridParallelOptimizer), ( - 'optimizer should be HybridParallelOptimizer subclass.') + assert isinstance( + optimizer, HybridParallelOptimizer + ), 'optimizer should be HybridParallelOptimizer subclass.' - assert fluid.framework._dygraph_tracer()._has_grad, ( - 'Please enable the generation of gradients.') + assert ( + fluid.framework._dygraph_tracer()._has_grad + ), 'Please enable the generation of gradients.' if self.is_pipeline_first_stage( - ignore_virtual=True) or self.is_pipeline_last_stage( - ignore_virtual=True): - assert data is not None, ( - "For the first and the last stage, the data must be set.") + ignore_virtual=True + ) or self.is_pipeline_last_stage(ignore_virtual=True): + assert ( + data is not None + ), "For the first and the last stage, the data must be set." else: data = None @@ -233,7 +251,7 @@ def eval_batch(self, data, compute_loss=False): # store total loss of entire batch self.total_loss = None - startup_steps = (self.num_stages - self.stage_id - 1) + startup_steps = self.num_stages - self.stage_id - 1 startup_steps = min(startup_steps, self.accumulate_steps) steady_steps = self.accumulate_steps - startup_steps @@ -253,7 +271,7 @@ def eval_batch(self, data, compute_loss=False): input_tensor = p2p.recv_forward(self.is_pipeline_first_stage()) for i in range(steady_steps): - last_iter = (i == (steady_steps - 1)) + last_iter = i == (steady_steps - 1) output_tensor = self._forward_step(input_tensor) p2p.send_forward(output_tensor, self.is_pipeline_last_stage()) @@ -282,13 +300,14 @@ def _forward_step(self, input_tensor, chunk_id=None): if self.is_pipeline_last_stage(): # train calculate loss for train if self._compute_loss: - assert self._layers._loss_fn is not None, "loss function should exist to compute loss" + assert ( + self._layers._loss_fn is not None + ), "loss function should exist to compute loss" labels = self._load_micro_batch(self.micro_batch_id) output_tensor = self._layers._loss_fn(output_tensor, labels) assert isinstance( - output_tensor, - (paddle.Tensor, core.eager.Tensor - )), "Currently, loss_fn should obtain Paddle.Tensor dtype" + output_tensor, (paddle.Tensor, core.eager.Tensor) + ), "Currently, loss_fn should obtain Paddle.Tensor dtype" with paddle.amp.auto_cast(enable=False): if self.accumulate_steps > 1: @@ -318,16 +337,20 @@ def _backward_step(self, input_tensor, output_tensor, output_tensor_grad): assert len(outputs) == len(output_tensor_grad) paddle.autograd.backward( tensors=outputs, - grad_tensors=[t for t in output_tensor_grad]) + grad_tensors=[t for t in output_tensor_grad], + ) else: - paddle.autograd.backward(tensors=[output_tensor], - grad_tensors=[output_tensor_grad]) + paddle.autograd.backward( + tensors=[output_tensor], + grad_tensors=[output_tensor_grad], + ) input_tensor_grad = None if input_tensor is not None: if isinstance(input_tensor, tuple): input_tensor_grad = tuple( - [t.grad for t in input_tensor if not t.stop_gradient]) + [t.grad for t in input_tensor if not t.stop_gradient] + ) else: input_tensor_grad = input_tensor.grad return input_tensor_grad @@ -341,31 +364,39 @@ def _load_micro_batch(self, cache_id): if self.is_pipeline_first_stage(): assert len(inputs) == 2, "length of input should be 2" if isinstance(inputs[0], tuple): - assert len( - inputs[0] - ) > 1, "If you use tuple for input data, it should have at least two inputs." + assert ( + len(inputs[0]) > 1 + ), "If you use tuple for input data, it should have at least two inputs." batch_size = inputs[0][0].shape[0] - assert self.micro_batch_size * self.accumulate_steps == batch_size, ( + assert ( + self.micro_batch_size * self.accumulate_steps == batch_size + ), ( "batch_size needs to be divisible by micro_batch_size. Currently, " "batch_size = %d, micro_batch_size = %d, accumulate_steps = %d." - % - (batch_size, self.micro_batch_size, self.accumulate_steps)) + % (batch_size, self.micro_batch_size, self.accumulate_steps) + ) data = [input[begin:end, :].detach() for input in inputs[0]] return tuple(data) else: batch_size = inputs[0].shape[0] - assert self.micro_batch_size * self.accumulate_steps == batch_size + assert ( + self.micro_batch_size * self.accumulate_steps == batch_size + ) return inputs[0][begin:end, :].detach() elif self.is_pipeline_last_stage(): assert len(inputs) == 2, "length of input should be 2" if isinstance(inputs[1], tuple): batch_size = inputs[1][0].shape[0] - assert self.micro_batch_size * self.accumulate_steps == batch_size + assert ( + self.micro_batch_size * self.accumulate_steps == batch_size + ) data = [input[begin:end, :].detach() for input in inputs[1]] return tuple(data) else: batch_size = inputs[1].shape[0] - assert self.micro_batch_size * self.accumulate_steps == batch_size + assert ( + self.micro_batch_size * self.accumulate_steps == batch_size + ) return inputs[1][begin:end, :].detach() else: # No data input is required for other stages @@ -375,34 +406,40 @@ def _broadcast_final_loss(self): # Since the last backward run in interleave will set the virtual rank to 0, # here we need to check last stage ignoring virtual stage. if self.is_pipeline_last_stage(ignore_virtual=True): - assert self.total_loss is not None, "train_batch() in last stage should obtain vaild loss" + assert ( + self.total_loss is not None + ), "train_batch() in last stage should obtain vaild loss" loss = self.total_loss.detach() - is_fp32 = paddle.to_tensor( - 1) if loss.dtype == paddle.float32 else paddle.to_tensor(0) - paddle.distributed.broadcast(is_fp32, - src=self.global_rank, - sync_op=True, - group=self.pp_group) - paddle.distributed.broadcast(loss, - src=self.global_rank, - sync_op=True, - group=self.pp_group) + is_fp32 = ( + paddle.to_tensor(1) + if loss.dtype == paddle.float32 + else paddle.to_tensor(0) + ) + paddle.distributed.broadcast( + is_fp32, src=self.global_rank, sync_op=True, group=self.pp_group + ) + paddle.distributed.broadcast( + loss, src=self.global_rank, sync_op=True, group=self.pp_group + ) else: is_fp32 = paddle.to_tensor(1) paddle.distributed.broadcast( is_fp32, src=self._hcg.get_rank_from_stage(self.num_stages - 1), sync_op=True, - group=self.pp_group) - loss = paddle.zeros(shape=[ - 1 - ], dtype="float32") if is_fp32.numpy()[0] else paddle.zeros( - shape=[1], dtype="float16") + group=self.pp_group, + ) + loss = ( + paddle.zeros(shape=[1], dtype="float32") + if is_fp32.numpy()[0] + else paddle.zeros(shape=[1], dtype="float16") + ) paddle.distributed.broadcast( loss, src=self._hcg.get_rank_from_stage(self.num_stages - 1), sync_op=True, - group=self.pp_group) + group=self.pp_group, + ) return loss def _optimizer_step(self): @@ -421,11 +458,12 @@ class PipelineParallelWithInterleave(PipelineParallel): # pipeline parallel with interleave scheduler def __init__(self, layers, hcg, strategy): - super(PipelineParallelWithInterleave, self).__init__(layers=layers, - hcg=hcg, - strategy=strategy) + super(PipelineParallelWithInterleave, self).__init__( + layers=layers, hcg=hcg, strategy=strategy + ) assert layers.get_num_virtual_stages() > 1 - assert framework.in_dygraph_mode( + assert ( + framework.in_dygraph_mode() ), "virtual pipeline stage with interleave only support eager dygraph mode" # setup for interleave scheduler self.num_model_chunks = layers.get_num_virtual_stages() @@ -436,11 +474,12 @@ def __init__(self, layers, hcg, strategy): self._virtual_pp_rank = 0 def _get_virtual_pp_rank(self, micro_step, forward): - virtual_pp_stage = micro_step % (self.num_stages * - self.num_model_chunks) + virtual_pp_stage = micro_step % ( + self.num_stages * self.num_model_chunks + ) virtual_pp_stage = virtual_pp_stage // self.num_stages if not forward: - virtual_pp_stage = (self.num_model_chunks - virtual_pp_stage - 1) + virtual_pp_stage = self.num_model_chunks - virtual_pp_stage - 1 return virtual_pp_stage def _forward_step_helper(self, micro_step): @@ -455,7 +494,8 @@ def _forward_step_helper(self, micro_step): if self.is_pipeline_first_stage(): if len(self.input_tensors[virtual_pp_rank]) == len( - self.output_tensors[virtual_pp_rank]): + self.output_tensors[virtual_pp_rank] + ): self.input_tensors[virtual_pp_rank].append(None) input_tensor = self.input_tensors[virtual_pp_rank][-1] output_tensor = self._forward_step(input_tensor, virtual_pp_rank) @@ -484,21 +524,22 @@ def _backward_step_helper(self, micro_step): input_tensor = self.input_tensors[virtual_pp_rank].pop(0) output_tensor = self.output_tensors[virtual_pp_rank].pop(0) output_tensor_grad = self.output_tensor_grads[virtual_pp_rank].pop(0) - input_tensor_grad = self._backward_step(input_tensor, output_tensor, - output_tensor_grad) + input_tensor_grad = self._backward_step( + input_tensor, output_tensor, output_tensor_grad + ) return input_tensor_grad - def interleave_pipeline(self, - data, - scaler, - forward_only=False, - compute_loss=True): + def interleave_pipeline( + self, data, scaler, forward_only=False, compute_loss=True + ): # use interleave scheduling strategy. # this strategy is inspired by: # https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/schedules.py if not compute_loss: - assert not forward_only, "compute_loss can only be set to False when forward_only is set to True" + assert ( + not forward_only + ), "compute_loss can only be set to False when forward_only is set to True" # init some attributes for this batch run self.scaler = scaler @@ -530,15 +571,17 @@ def interleave_pipeline(self, self.set_virtual_pipeline_rank(0) self.input_tensors[0].append( - p2p.recv_forward(self.is_pipeline_first_stage(), sync_recv=False)) + p2p.recv_forward(self.is_pipeline_first_stage(), sync_recv=False) + ) # run startup steps for micro_step in range(startup_steps): output_tensor = self._forward_step_helper(micro_step) # determine whether recv forward tensor or not - next_virtual_pp_rank = self._get_virtual_pp_rank(micro_step + 1, - forward=True) + next_virtual_pp_rank = self._get_virtual_pp_rank( + micro_step + 1, forward=True + ) recv_prev = True if self.is_pipeline_first_stage(ignore_virtual=True): if next_virtual_pp_rank == 0: @@ -552,24 +595,33 @@ def interleave_pipeline(self, if self.is_pipeline_last_stage(): output_tensor = None - if micro_step == (startup_steps - - 1) and not forward_only and not all_startup_steps: + if ( + micro_step == (startup_steps - 1) + and not forward_only + and not all_startup_steps + ): input_tensor_grad = None recv_next = True if self.is_pipeline_last_stage(ignore_virtual=True): recv_next = False # the last startup step needs on four direction comm to set up for steady 1f1b - input_tensor, output_tensor_grad = p2p.send_forward_backward_recv_forward_backward( + ( + input_tensor, + output_tensor_grad, + ) = p2p.send_forward_backward_recv_forward_backward( output_tensor, input_tensor_grad, recv_prev=recv_prev, - recv_next=recv_next) - self.output_tensor_grads[self.num_model_chunks - - 1].append(output_tensor_grad) + recv_next=recv_next, + ) + self.output_tensor_grads[self.num_model_chunks - 1].append( + output_tensor_grad + ) else: input_tensor = p2p.send_forward_recv_forward( - output_tensor, recv_prev=recv_prev) + output_tensor, recv_prev=recv_prev + ) self.input_tensors[next_virtual_pp_rank].append(input_tensor) # run 1f1b steady steps @@ -581,7 +633,8 @@ def interleave_pipeline(self, # backward backward_micro_step_id = micro_step input_tensor_grad = self._backward_step_helper( - backward_micro_step_id) + backward_micro_step_id + ) # four directions comm # send output tensor to downstream @@ -591,14 +644,16 @@ def interleave_pipeline(self, # last stage doesn't send rst to downstream forward_virtual_pp_rank = self._get_virtual_pp_rank( - forward_micro_step_id, forward=True) + forward_micro_step_id, forward=True + ) self.set_virtual_pipeline_rank(forward_virtual_pp_rank) if self.is_pipeline_last_stage(): output_tensor = None # first stage doesn't send grad to upstream backward_virtual_pp_rank = self._get_virtual_pp_rank( - backward_micro_step_id, forward=False) + backward_micro_step_id, forward=False + ) self.set_virtual_pipeline_rank(backward_virtual_pp_rank) if self.is_pipeline_first_stage(): input_tensor_grad = None @@ -607,14 +662,16 @@ def interleave_pipeline(self, recv_prev = True if self.is_pipeline_first_stage(ignore_virtual=True): next_forward_virtual_pp_rank = self._get_virtual_pp_rank( - forward_micro_step_id - (self.num_stages - 1), forward=True) + forward_micro_step_id - (self.num_stages - 1), forward=True + ) if next_forward_virtual_pp_rank == (self.num_model_chunks - 1): # first pp stage and first virtual stage recv_prev = False next_forward_virtual_pp_rank += 1 else: next_forward_virtual_pp_rank = self._get_virtual_pp_rank( - forward_micro_step_id + 1, forward=True) + forward_micro_step_id + 1, forward=True + ) # last iteration doesn't need recv from upstream if micro_step == (steady_steps - 1): @@ -625,53 +682,67 @@ def interleave_pipeline(self, if self.is_pipeline_last_stage(ignore_virtual=True): next_backward_virtual_pp_rank = self._get_virtual_pp_rank( backward_micro_step_id - (self.num_stages - 1), - forward=False) + forward=False, + ) if next_backward_virtual_pp_rank == 0: # last pp stage and last virtual stage recv_next = False next_backward_virtual_pp_rank -= 1 else: next_backward_virtual_pp_rank = self._get_virtual_pp_rank( - backward_micro_step_id + 1, forward=False) + backward_micro_step_id + 1, forward=False + ) - input_tensor, output_tensor_grad = p2p.send_forward_backward_recv_forward_backward( + ( + input_tensor, + output_tensor_grad, + ) = p2p.send_forward_backward_recv_forward_backward( output_tensor, input_tensor_grad, recv_prev=recv_prev, - recv_next=recv_next) + recv_next=recv_next, + ) if recv_prev: self.input_tensors[next_forward_virtual_pp_rank].append( - input_tensor) + input_tensor + ) if recv_next: self.output_tensor_grads[next_backward_virtual_pp_rank].append( - output_tensor_grad) + output_tensor_grad + ) # remaining backward steps if not forward_only: if all_startup_steps: self.output_tensor_grads[self.num_model_chunks - 1].append( - p2p.recv_backward(self.is_pipeline_last_stage(), - sync_recv=False)) + p2p.recv_backward( + self.is_pipeline_last_stage(), sync_recv=False + ) + ) for micro_step in range(steady_steps, num_steps): # cooldown loop input_tensor_grad = self._backward_step_helper(micro_step) next_backward_virtual_pp_rank = self._get_virtual_pp_rank( - micro_step + 1, forward=False) + micro_step + 1, forward=False + ) recv_next = True if self.is_pipeline_last_stage(ignore_virtual=True): - if next_backward_virtual_pp_rank == (self.num_model_chunks - - 1): + if next_backward_virtual_pp_rank == ( + self.num_model_chunks - 1 + ): recv_next = False if micro_step == (num_steps - 1): recv_next = False self.output_tensor_grads[next_backward_virtual_pp_rank].append( - p2p.send_backward_recv_backward(input_tensor_grad, - recv_next=recv_next)) + p2p.send_backward_recv_backward( + input_tensor_grad, recv_next=recv_next + ) + ) self._layers.allreduce_shared_weight_gradients() diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py index ecd3dc7ab9136..e97cf3c02302f 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py +++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/p2p_communication.py @@ -30,12 +30,23 @@ def initialize_p2p_groups(hcg, use_cache=True, enable_partial_send_recv=True): _hcg = hcg _use_cache = use_cache _enable_partial_send_recv = enable_partial_send_recv - send_next_group, send_prev_group, recv_next_group, recv_prev_group = _hcg.get_p2p_groups( + ( + send_next_group, + send_prev_group, + recv_next_group, + recv_prev_group, + ) = _hcg.get_p2p_groups() + + debug_str = ( + "P2pInfo: send_next_group: %s, send_prev_group: %s, " + "recv_next_group: %s, recv_prev_group: %s" + % ( + repr(send_next_group), + repr(send_prev_group), + repr(recv_next_group), + repr(recv_prev_group), + ) ) - - debug_str = "P2pInfo: send_next_group: %s, send_prev_group: %s, " \ - "recv_next_group: %s, recv_prev_group: %s" % (repr(send_next_group), - repr(send_prev_group),repr(recv_next_group), repr(recv_prev_group)) logger.info(debug_str) @@ -150,9 +161,11 @@ def set_send_message(self, tensor): self.send_dtype_message = paddle_2_number(tensor.dtype) elif isinstance(tensor, tuple): self.send_shape_message = tuple( - [d.shape for d in tensor if not d.stop_gradient]) + [d.shape for d in tensor if not d.stop_gradient] + ) self.send_dtype_message = tuple( - [paddle_2_number(d.dtype) for d in tensor]) + [paddle_2_number(d.dtype) for d in tensor] + ) _send_recv_meta = SendRecvMeta() @@ -166,84 +179,117 @@ def _is_valid_send_recv_partial(tensor, mp_degree): return mp_degree > 1 and tensor_numel % mp_degree == 0 -def _partial_send_op(tensor, group, use_calc_stream, ring_id, dst, nranks, - rank_id): +def _partial_send_op( + tensor, group, use_calc_stream, ring_id, dst, nranks, rank_id +): dst_rank_in_group = dst if group is None else group.get_group_rank(dst) if _in_legacy_dygraph(): - return _legacy_C_ops.partial_send(tensor.detach(), 'use_calc_stream', - use_calc_stream, 'ring_id', ring_id, - 'peer', dst_rank_in_group, 'num', - nranks, 'id', rank_id) + return _legacy_C_ops.partial_send( + tensor.detach(), + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'peer', + dst_rank_in_group, + 'num', + nranks, + 'id', + rank_id, + ) elif in_dygraph_mode(): - group = paddle.distributed.collective._get_default_group( - ) if group is None else group - comm_op = group.process_group.send_partial_on_calc_stream \ - if use_calc_stream else group.process_group.send_partial + group = ( + paddle.distributed.collective._get_default_group() + if group is None + else group + ) + comm_op = ( + group.process_group.send_partial_on_calc_stream + if use_calc_stream + else group.process_group.send_partial + ) return comm_op(tensor, dst_rank_in_group, nranks, rank_id) -def send_partial(tensor, - dst=0, - nranks=1, - rank_id=0, - group=None, - use_calc_stream=True): +def send_partial( + tensor, dst=0, nranks=1, rank_id=0, group=None, use_calc_stream=True +): # dst: local rank in group if group is not None and not group.is_member(): return ring_id = 0 if group is None else group.id - dst_rank = _hcg._get_p2p_next_rank( - ) if dst == 1 else _hcg._get_p2p_prev_rank() + dst_rank = ( + _hcg._get_p2p_next_rank() if dst == 1 else _hcg._get_p2p_prev_rank() + ) if _is_valid_send_recv_partial(tensor, nranks): - return _partial_send_op(tensor, group, use_calc_stream, ring_id, - dst_rank, nranks, rank_id) + return _partial_send_op( + tensor, group, use_calc_stream, ring_id, dst_rank, nranks, rank_id + ) else: if _in_legacy_dygraph(): - send_op = lambda x, dst, group: \ - paddle.distributed.send(x, dst, group, use_calc_stream) + send_op = lambda x, dst, group: paddle.distributed.send( + x, dst, group, use_calc_stream + ) elif in_dygraph_mode(): send_op = paddle.distributed.isend return send_op(tensor.detach(), dst=dst_rank, group=group) -def _partial_recv_op(tensor, group, use_calc_stream, ring_id, src, nranks, - rank_id): +def _partial_recv_op( + tensor, group, use_calc_stream, ring_id, src, nranks, rank_id +): src_rank_in_group = src if group is None else group.get_group_rank(src) if _in_legacy_dygraph(): assert use_calc_stream - return _legacy_C_ops.partial_recv(tensor.detach(), 'use_calc_stream', - use_calc_stream, 'ring_id', ring_id, - 'peer', src_rank_in_group, 'num', - nranks, 'id', rank_id, 'dtype', - tensor.dtype, 'out_shape', - tensor.shape) + return _legacy_C_ops.partial_recv( + tensor.detach(), + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'peer', + src_rank_in_group, + 'num', + nranks, + 'id', + rank_id, + 'dtype', + tensor.dtype, + 'out_shape', + tensor.shape, + ) elif in_dygraph_mode(): - group = paddle.distributed.collective._get_default_group( - ) if group is None else group - comm_op = group.process_group.recv_partial_on_calc_stream \ - if use_calc_stream else group.process_group.recv_partial + group = ( + paddle.distributed.collective._get_default_group() + if group is None + else group + ) + comm_op = ( + group.process_group.recv_partial_on_calc_stream + if use_calc_stream + else group.process_group.recv_partial + ) return comm_op(tensor, src_rank_in_group, nranks, rank_id) -def recv_partial(tensor, - src=0, - nranks=1, - rank_id=0, - group=None, - use_calc_stream=True): +def recv_partial( + tensor, src=0, nranks=1, rank_id=0, group=None, use_calc_stream=True +): # src: local rank in group if group is not None and not group.is_member(): return ring_id = 0 if group is None else group.id - src_rank = _hcg._get_p2p_prev_rank( - ) if src == 0 else _hcg._get_p2p_next_rank() + src_rank = ( + _hcg._get_p2p_prev_rank() if src == 0 else _hcg._get_p2p_next_rank() + ) if _is_valid_send_recv_partial(tensor, nranks): - return _partial_recv_op(tensor, group, use_calc_stream, ring_id, - src_rank, nranks, rank_id) + return _partial_recv_op( + tensor, group, use_calc_stream, ring_id, src_rank, nranks, rank_id + ) else: if _in_legacy_dygraph() or use_calc_stream: recv_op = paddle.distributed.recv @@ -252,42 +298,52 @@ def recv_partial(tensor, return recv_op(tensor.detach(), src=src_rank, group=group) -def _partial_allgather_op(tensor, group, use_calc_stream, ring_id, nranks, - rank_id): +def _partial_allgather_op( + tensor, group, use_calc_stream, ring_id, nranks, rank_id +): if _in_legacy_dygraph(): - return _legacy_C_ops.partial_allgather_(tensor.detach(), - 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id, 'nranks', nranks, - 'rank', rank_id) + return _legacy_C_ops.partial_allgather_( + tensor.detach(), + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'nranks', + nranks, + 'rank', + rank_id, + ) elif in_dygraph_mode(): - group = paddle.distributed.collective._get_default_group( - ) if group is None else group - comm_op = group.process_group.all_gather_partial_on_calc_stream \ - if use_calc_stream else group.process_group.all_gather_partial + group = ( + paddle.distributed.collective._get_default_group() + if group is None + else group + ) + comm_op = ( + group.process_group.all_gather_partial_on_calc_stream + if use_calc_stream + else group.process_group.all_gather_partial + ) return comm_op(tensor, tensor, nranks, rank_id) -def allgather_partial(tensor, - nranks=1, - rank_id=0, - group=None, - use_calc_stream=True): +def allgather_partial( + tensor, nranks=1, rank_id=0, group=None, use_calc_stream=True +): if not _is_valid_send_recv_partial(tensor, nranks): return tensor if group is not None and not group.is_member(): return ring_id = 0 if group is None else group.id - return _partial_allgather_op(tensor, group, use_calc_stream, ring_id, - nranks, rank_id) + return _partial_allgather_op( + tensor, group, use_calc_stream, ring_id, nranks, rank_id + ) -def _p2p_helper(tensor_send_next, - tensor_send_prev, - recv_prev, - recv_next, - sync_recv=True): +def _p2p_helper( + tensor_send_next, tensor_send_prev, recv_prev, recv_next, sync_recv=True +): global _hcg tensor_recv_prev = None @@ -310,15 +366,17 @@ def _p2p_helper(tensor_send_next, if isinstance(recv_shape_msg, tuple): tensor_recv_prev = [] for idx, shape in enumerate(recv_shape_msg): - tmp = paddle.empty(shape=shape, - dtype=number_2_dtype(recv_dtype_msg[idx])) + tmp = paddle.empty( + shape=shape, dtype=number_2_dtype(recv_dtype_msg[idx]) + ) tmp.stop_gradient = recv_stop_gradient[idx] tensor_recv_prev.append(tmp) tensor_recv_prev = tuple(tensor_recv_prev) else: tensor_recv_prev = paddle.empty( - shape=recv_shape_msg, dtype=number_2_dtype(recv_dtype_msg)) + shape=recv_shape_msg, dtype=number_2_dtype(recv_dtype_msg) + ) tensor_recv_prev.stop_gradient = recv_stop_gradient if recv_next: @@ -326,12 +384,15 @@ def _p2p_helper(tensor_send_next, tensor_recv_next = [] for idx, shape in enumerate(send_shape_msg): tensor_recv_next.append( - paddle.empty(shape=shape, - dtype=number_2_dtype(send_dtype_msg[idx]))) + paddle.empty( + shape=shape, dtype=number_2_dtype(send_dtype_msg[idx]) + ) + ) tensor_recv_next = tuple(tensor_recv_next) else: tensor_recv_next = paddle.empty( - shape=send_shape_msg, dtype=number_2_dtype(send_dtype_msg)) + shape=send_shape_msg, dtype=number_2_dtype(send_dtype_msg) + ) # TODO(Yuang Liu): use batch_isend_irecv replace all these comm ops tasks = [] @@ -340,51 +401,63 @@ def _p2p_helper(tensor_send_next, if isinstance(tensor_send_prev, tuple): for d in tensor_send_prev: paddle.distributed.wait(d, use_calc_stream=True) - send_partial(d, - dst=0, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.send_prev_group, - use_calc_stream=False) + send_partial( + d, + dst=0, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.send_prev_group, + use_calc_stream=False, + ) else: paddle.distributed.wait(tensor_send_prev, use_calc_stream=True) - send_partial(tensor_send_prev, - dst=0, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.send_prev_group, - use_calc_stream=False) + send_partial( + tensor_send_prev, + dst=0, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.send_prev_group, + use_calc_stream=False, + ) if tensor_recv_prev is not None: if isinstance(tensor_recv_prev, tuple): for d in tensor_recv_prev: - task = recv_partial(d, - src=0, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.recv_prev_group, - use_calc_stream=sync_recv) + task = recv_partial( + d, + src=0, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.recv_prev_group, + use_calc_stream=sync_recv, + ) if sync_recv: - allgather_partial(d, - nranks=mp_degree, - rank_id=mp_rank, - group=mp_group, - use_calc_stream=True) + allgather_partial( + d, + nranks=mp_degree, + rank_id=mp_rank, + group=mp_group, + use_calc_stream=True, + ) else: tasks.append(task) else: - task = recv_partial(tensor_recv_prev, - src=0, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.recv_prev_group, - use_calc_stream=sync_recv) + task = recv_partial( + tensor_recv_prev, + src=0, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.recv_prev_group, + use_calc_stream=sync_recv, + ) if sync_recv: - allgather_partial(tensor_recv_prev, - nranks=mp_degree, - rank_id=mp_rank, - group=mp_group, - use_calc_stream=True) + allgather_partial( + tensor_recv_prev, + nranks=mp_degree, + rank_id=mp_rank, + group=mp_group, + use_calc_stream=True, + ) else: tasks.append(task) @@ -392,52 +465,64 @@ def _p2p_helper(tensor_send_next, if isinstance(tensor_send_next, tuple): for d in tensor_send_next: paddle.distributed.wait(d, use_calc_stream=True) - send_partial(d, - dst=1, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.send_next_group, - use_calc_stream=False) + send_partial( + d, + dst=1, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.send_next_group, + use_calc_stream=False, + ) else: paddle.distributed.wait(tensor_send_next, use_calc_stream=True) - send_partial(tensor_send_next, - dst=1, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.send_next_group, - use_calc_stream=False) + send_partial( + tensor_send_next, + dst=1, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.send_next_group, + use_calc_stream=False, + ) if tensor_recv_next is not None: if isinstance(tensor_recv_next, tuple): for d in tensor_recv_next: - task = recv_partial(d, - src=1, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.recv_next_group, - use_calc_stream=sync_recv) + task = recv_partial( + d, + src=1, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.recv_next_group, + use_calc_stream=sync_recv, + ) if sync_recv: - allgather_partial(d, - nranks=mp_degree, - rank_id=mp_rank, - group=mp_group, - use_calc_stream=True) + allgather_partial( + d, + nranks=mp_degree, + rank_id=mp_rank, + group=mp_group, + use_calc_stream=True, + ) else: tasks.append(task) else: - task = recv_partial(tensor_recv_next, - src=1, - nranks=mp_degree, - rank_id=mp_rank, - group=_hcg.recv_next_group, - use_calc_stream=sync_recv) + task = recv_partial( + tensor_recv_next, + src=1, + nranks=mp_degree, + rank_id=mp_rank, + group=_hcg.recv_next_group, + use_calc_stream=sync_recv, + ) if sync_recv: - allgather_partial(tensor_recv_next, - nranks=mp_degree, - rank_id=mp_rank, - group=mp_group, - use_calc_stream=True) + allgather_partial( + tensor_recv_next, + nranks=mp_degree, + rank_id=mp_rank, + group=mp_group, + use_calc_stream=True, + ) else: tasks.append(task) @@ -463,11 +548,13 @@ def _p2p_helper(tensor_send_next, tensors_for_all_gather.append(tensor_recv_next) for tensor in tensors_for_all_gather: - allgather_partial(tensor, - nranks=mp_degree, - rank_id=mp_rank, - group=mp_group, - use_calc_stream=True) + allgather_partial( + tensor, + nranks=mp_degree, + rank_id=mp_rank, + group=mp_group, + use_calc_stream=True, + ) return tensor_recv_prev, tensor_recv_next @@ -480,11 +567,13 @@ def recv_forward(pp_first_stage, sync_recv=True): _send_recv_meta.recv_meta(_hcg.recv_prev_group) _send_recv_meta.has_recv_meta = _use_cache - input_tensor, _ = _p2p_helper(tensor_send_next=None, - tensor_send_prev=None, - recv_prev=True, - recv_next=False, - sync_recv=sync_recv) + input_tensor, _ = _p2p_helper( + tensor_send_next=None, + tensor_send_prev=None, + recv_prev=True, + recv_next=False, + sync_recv=sync_recv, + ) return input_tensor @@ -492,11 +581,13 @@ def recv_backward(pp_last_stage, sync_recv=True): if pp_last_stage: output_tensor_grad = None else: - _, output_tensor_grad = _p2p_helper(tensor_send_next=None, - tensor_send_prev=None, - recv_prev=False, - recv_next=True, - sync_recv=sync_recv) + _, output_tensor_grad = _p2p_helper( + tensor_send_next=None, + tensor_send_prev=None, + recv_prev=False, + recv_next=True, + sync_recv=sync_recv, + ) return output_tensor_grad @@ -507,28 +598,34 @@ def send_forward(output_tensor, pp_last_stage): _send_recv_meta.send_meta(output_tensor, _hcg.send_next_group) _send_recv_meta.has_send_meta = _use_cache - _p2p_helper(tensor_send_next=output_tensor, - tensor_send_prev=None, - recv_prev=False, - recv_next=False) + _p2p_helper( + tensor_send_next=output_tensor, + tensor_send_prev=None, + recv_prev=False, + recv_next=False, + ) def send_backward(input_tensor_grad, pp_first_stage): if not pp_first_stage: - _p2p_helper(tensor_send_next=None, - tensor_send_prev=input_tensor_grad, - recv_prev=False, - recv_next=False) + _p2p_helper( + tensor_send_next=None, + tensor_send_prev=input_tensor_grad, + recv_prev=False, + recv_next=False, + ) def send_forward_recv_backward(output_tensor, pp_last_stage): if pp_last_stage: output_tensor_grad = None else: - _, output_tensor_grad = _p2p_helper(tensor_send_next=output_tensor, - tensor_send_prev=None, - recv_prev=False, - recv_next=True) + _, output_tensor_grad = _p2p_helper( + tensor_send_next=output_tensor, + tensor_send_prev=None, + recv_prev=False, + recv_next=True, + ) return output_tensor_grad @@ -536,16 +633,18 @@ def send_backward_recv_forward(input_tensor_grad, pp_first_stage): if pp_first_stage: input_tensor = None else: - input_tensor, _ = _p2p_helper(tensor_send_next=None, - tensor_send_prev=input_tensor_grad, - recv_prev=True, - recv_next=False) + input_tensor, _ = _p2p_helper( + tensor_send_next=None, + tensor_send_prev=input_tensor_grad, + recv_prev=True, + recv_next=False, + ) return input_tensor -def send_forward_backward_recv_forward_backward(output_tensor, - input_tensor_grad, recv_prev, - recv_next): +def send_forward_backward_recv_forward_backward( + output_tensor, input_tensor_grad, recv_prev, recv_next +): # always have to send dytpe info to downstream if not _send_recv_meta.has_send_meta: _send_recv_meta.set_send_message(output_tensor) @@ -559,7 +658,8 @@ def send_forward_backward_recv_forward_backward(output_tensor, tensor_send_prev=input_tensor_grad, recv_prev=recv_prev, recv_next=recv_next, - sync_recv=False) + sync_recv=False, + ) return input_tensor, output_tensor_grad @@ -573,19 +673,23 @@ def send_forward_recv_forward(output_tensor, recv_prev): _send_recv_meta.recv_meta(_hcg.recv_prev_group) _send_recv_meta.has_recv_meta = _use_cache - input_tensor, _ = _p2p_helper(tensor_send_next=output_tensor, - tensor_send_prev=None, - recv_prev=recv_prev, - recv_next=False, - sync_recv=False) + input_tensor, _ = _p2p_helper( + tensor_send_next=output_tensor, + tensor_send_prev=None, + recv_prev=recv_prev, + recv_next=False, + sync_recv=False, + ) return input_tensor def send_backward_recv_backward(input_tensor_grad, recv_next): - _, output_tensor_grad = _p2p_helper(tensor_send_next=None, - tensor_send_prev=input_tensor_grad, - recv_prev=False, - recv_next=recv_next, - sync_recv=False) + _, output_tensor_grad = _p2p_helper( + tensor_send_next=None, + tensor_send_prev=input_tensor_grad, + recv_prev=False, + recv_next=recv_next, + sync_recv=False, + ) return output_tensor_grad diff --git a/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py b/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py index 9f5d868a1570d..e33f9d4689cec 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py +++ b/python/paddle/distributed/fleet/meta_parallel/pp_utils/utils.py @@ -28,7 +28,7 @@ paddle.float32: 1, paddle.float64: 2, paddle.int32: 3, - paddle.int64: 4 + paddle.int64: 4, } NUMBER_TO_DTYPE = { @@ -36,7 +36,7 @@ 1: "float32", 2: "float64", 3: "int32", - 4: "int64" + 4: "int64", } @@ -88,7 +88,17 @@ def _all_gather(tensor, group=None, use_calc_stream=True): if group is not None and not group.is_member(): return ring_id = 0 if group is None else group.id - nranks = paddle.distributed.collective._get_global_group( - ).nranks if group is None else group.nranks - return _legacy_C_ops.c_allgather(tensor, 'use_calc_stream', use_calc_stream, - 'ring_id', ring_id, 'nranks', nranks) + nranks = ( + paddle.distributed.collective._get_global_group().nranks + if group is None + else group.nranks + ) + return _legacy_C_ops.c_allgather( + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'nranks', + nranks, + ) diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_optimizer_stage2.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_optimizer_stage2.py index 7981375649040..2668b5808996d 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_optimizer_stage2.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_optimizer_stage2.py @@ -31,7 +31,11 @@ from paddle.fluid import core from paddle.optimizer import Optimizer from paddle.fluid.clip import ClipGradByGlobalNorm -from paddle.distributed.collective import _get_global_group, broadcast, new_group +from paddle.distributed.collective import ( + _get_global_group, + broadcast, + new_group, +) from .group_sharded_storage import ParamStorage, GradStorage from .group_sharded_utils import Type, device_guard, GroupShardedClipGrad @@ -63,21 +67,24 @@ class GroupShardedOptimizerStage2(Optimizer): # 4. Support offload function. # 5. Support the establishment of independent communication groups. # 6. Broadcast_fp16 is not supported now. - def __init__(self, - params, - optim, - group=None, - offload=False, - device="gpu", - pertrain_sync_models=True, - dp_group=None, - **kw): + def __init__( + self, + params, + optim, + group=None, + offload=False, + device="gpu", + pertrain_sync_models=True, + dp_group=None, + **kw + ): super().__init__(learning_rate=optim._learning_rate, parameters=params) assert core.is_compiled_with_cuda(), "Only GPU is supported now" # Segmentation information - self._dtype_rank_params = OrderedDict( + self._dtype_rank_params = ( + OrderedDict() ) # {dtype:[param1,param2]} device, rank, params self._param2rank = {} self.__segment_params = [] @@ -92,8 +99,9 @@ def __init__(self, # record the last task used for comm overlap for sharding stage 2 self._comm_task = None - assert hasattr(self._optim, "_master_weights" - ), "Must use optimizer with _master_weights attribute" + assert hasattr( + self._optim, "_master_weights" + ), "Must use optimizer with _master_weights attribute" # Support parameter group and parameter list self._local_params = [] @@ -104,10 +112,17 @@ def __init__(self, self._local_params.extend(list(params)) self._default_device = device - self._pfp16 = len( - list( - filter(lambda x: x.trainable and x.dtype == Type.fp16.value, - self._local_params))) > 0 + self._pfp16 = ( + len( + list( + filter( + lambda x: x.trainable and x.dtype == Type.fp16.value, + self._local_params, + ) + ) + ) + > 0 + ) self._broadcast_overlap = False self._forward_pre_hook_remove_helper = [] @@ -116,12 +131,14 @@ def __init__(self, # Have to sort the params to make sure all params are in the forward using order. self._broadcast_order_params = sorted( self.local_params, - key=lambda x: int(x.name.split('.')[0].split('_')[-1])) + key=lambda x: int(x.name.split('.')[0].split('_')[-1]), + ) except ValueError: self._broadcast_order_params = None - self._group = new_group( - _get_global_group().ranks) if group is None else group + self._group = ( + new_group(_get_global_group().ranks) if group is None else group + ) # only support to combine stage2 and dp hybrid parallel now. self._dp_group = dp_group @@ -141,15 +158,19 @@ def __init__(self, ) self._optim._grad_clip = GroupShardedClipGrad( - self._optim._grad_clip, paddle.get_device(), self._group) + self._optim._grad_clip, paddle.get_device(), self._group + ) if self._optim._parameter_list and isinstance( - self._optim._parameter_list[0], dict): + self._optim._parameter_list[0], dict + ): for item in self._optim._param_groups: if "grad_clip" in item.keys(): item["grad_clip"] = self._optim._grad_clip if offload: - assert self._pfp16, "Only support offload strategy while using \'Adam\', \'AdamW\' and \'Momentum\' optimizer with AMP/Pure FP16" + assert ( + self._pfp16 + ), "Only support offload strategy while using \'Adam\', \'AdamW\' and \'Momentum\' optimizer with AMP/Pure FP16" self.offload = offload # Using for offload self.offload_device = "cpu" @@ -171,16 +192,17 @@ def _sync_params_and_buffers(self): """ for p in self._local_params: - broadcast(p, - src=self._global_root_rank, - group=self._group, - sync_op=True) + broadcast( + p, src=self._global_root_rank, group=self._group, sync_op=True + ) if self._dp_group: - broadcast(p, - src=self._dp_group.ranks[0], - group=self._dp_group, - sync_op=True) + broadcast( + p, + src=self._dp_group.ranks[0], + group=self._dp_group, + sync_op=True, + ) def _update_task(self, task): if self._reduce_overlap: @@ -194,15 +216,15 @@ def _set_reduce_overlap(self, reduce_overlap): # Enable gradients' reduces overlap with backward calculation. self._reduce_overlap = reduce_overlap - def _set_broadcast_overlap(self, - broadcast_overlap, - layers=None, - num_groups=None): + def _set_broadcast_overlap( + self, broadcast_overlap, layers=None, num_groups=None + ): # Enable post optimizer broadcasts overlap with the forward calculation of next batch. self._broadcast_overlap = broadcast_overlap if self._broadcast_overlap: - assert layers is not None, \ - "To enable broadcast overlap forward, please pass the module to the function." + assert ( + layers is not None + ), "To enable broadcast overlap forward, please pass the module to the function." self._layers = layers warnings.warn( "Setting overlap broadcast means the `paddle.device.cuda.synchronize()` " @@ -212,7 +234,8 @@ def _set_broadcast_overlap(self, # Params' names should be like column_linear_32.w_0 patter to get the best performance. warnings.warn( r"The param name passed to the optimizer doesn't follow .+_[0-9]+\..+ patter, " - "overlap broadcast may harm the performance.") + "overlap broadcast may harm the performance." + ) self._broadcast_order_params = self._local_params if num_groups is None or num_groups > len(self._broadcast_order_params): @@ -222,9 +245,9 @@ def _set_broadcast_overlap(self, ) num_groups = 1 - assert isinstance( - num_groups, - int) and num_groups > 0, "num_groups should be a positive integer" + assert ( + isinstance(num_groups, int) and num_groups > 0 + ), "num_groups should be a positive integer" self._number_of_broadcast_groups = num_groups self._broadcast_groups = [ @@ -244,7 +267,8 @@ def _generate_master_params(self, trainable_params): name=param.name, value=param.cast(dtype=Type.fp32.value).numpy(), place=core.CPUPlace(), - stop_gradient=param.stop_gradient) + stop_gradient=param.stop_gradient, + ) else: for param in trainable_params: if param.dtype == Type.fp16.value: @@ -253,8 +277,7 @@ def _generate_master_params(self, trainable_params): self._optim._master_weights[param.name] = master_tensor def _update_opt_status(self): - """Update optimizer status and parameter storage information, and special functions to be developed. - """ + """Update optimizer status and parameter storage information, and special functions to be developed.""" # func 1 self._integration_params() @@ -302,14 +325,16 @@ def dtype_rank_params(self): if len(self._dtype_rank_params) == 0: # Assign the parameters of each rank according to the type trainable_params = list( - filter(lambda x: x.trainable, self._local_params)) + filter(lambda x: x.trainable, self._local_params) + ) for param in trainable_params: if param.dtype not in self._dtype_rank_params.keys(): self._dtype_rank_params[param.dtype] = [ [] for _ in range(self.world_size) ] - self._dtype_rank_params[param.dtype][self.param2rank[ - param.name]].append(param) + self._dtype_rank_params[param.dtype][ + self.param2rank[param.name] + ].append(param) # Sort per rank params by size for dtype in self._dtype_rank_params.keys(): @@ -329,7 +354,8 @@ def rank_buffer_size(self): if dtype not in self._rank_buffer_size.keys(): self._rank_buffer_size[dtype] = {} for dst_rank, per_rank_params in enumerate( - self.dtype_rank_params[dtype]): + self.dtype_rank_params[dtype] + ): if dst_rank not in self._rank_buffer_size[dtype].keys(): self._rank_buffer_size[dtype][dst_rank] = 0 for param in per_rank_params: @@ -337,11 +363,15 @@ def rank_buffer_size(self): continue size = param._numel() * align[dtype] remaining = size % alignment[self._default_device] - ali = 0 if remaining == 0 else alignment[ - self._default_device] - remaining + ali = ( + 0 + if remaining == 0 + else alignment[self._default_device] - remaining + ) align_ = ali // align[dtype] - self._rank_buffer_size[dtype][dst_rank] += param._numel( - ) + align_ + self._rank_buffer_size[dtype][dst_rank] += ( + param._numel() + align_ + ) self._param2align[param.name] = align_ return self._rank_buffer_size @@ -360,23 +390,27 @@ def _integration_params(self): # Merge all the trainable params in a single InternalStorage trainable_params = list( - filter(lambda x: x.trainable, params)) + filter(lambda x: x.trainable, params) + ) if self._pfp16 and dst_rank == self._rank: self._generate_master_params(trainable_params) if trainable_params: param_storage = ParamStorage( size=self.rank_buffer_size[dtype][dst_rank], dtype=dtype, - device=self._default_device) + device=self._default_device, + ) - param_storage.add_rank_params(trainable_params, - self._param2align) + param_storage.add_rank_params( + trainable_params, self._param2align + ) self.param_storages[dtype][dst_rank] = param_storage # Clear the InternalStorage keys which are not in use anymore dtype_in_use = list(self.dtype_rank_params.keys()) dtype_to_pop = list( - filter(lambda x: x not in dtype_in_use, self.param_storages.keys())) + filter(lambda x: x not in dtype_in_use, self.param_storages.keys()) + ) for d in dtype_to_pop: self.param_storages.pop(d) @@ -386,8 +420,11 @@ def _integration_params(self): for param in cpu_master_params: size = param._numel() * align[Type.fp32.value] remaining = size % alignment[self.offload_device] - ali = 0 if remaining == 0 else alignment[ - self.offload_device] - remaining + ali = ( + 0 + if remaining == 0 + else alignment[self.offload_device] - remaining + ) align_ = ali // align[Type.fp32.value] self.offload_buffer_size += param._numel() + align_ self.offload_param2align[param.name] = align_ @@ -397,10 +434,12 @@ def _integration_params(self): self.offload_params = ParamStorage( size=self.offload_buffer_size, dtype=Type.fp32.value, - device=self.offload_device) + device=self.offload_device, + ) self.offload_params.buffer.name = "offload_buffer" self.offload_params.add_rank_params( - cpu_master_params, self.offload_param2align, False) + cpu_master_params, self.offload_param2align, False + ) self.offload_params.buffer.stop_gradient = False self.offload_grads = GradStorage( @@ -409,14 +448,16 @@ def _integration_params(self): device=self.offload_device, destination=self._rank, parm2align=self.offload_param2align, - convert_cpu=True) + convert_cpu=True, + ) for p in cpu_master_params: self.offload_grads.add_grad( - p, self.offload_param2align[p.name]) + p, self.offload_param2align[p.name] + ) self._optim._master_weights[ - self.offload_params.buffer. - name] = self.offload_params.buffer + self.offload_params.buffer.name + ] = self.offload_params.buffer def _offload_acc_grad(self, param_name, grad_fp32_cpu): """accumulate grads with offload strategy""" @@ -424,12 +465,14 @@ def _offload_acc_grad(self, param_name, grad_fp32_cpu): if param_name in self._master_params.keys(): if self._master_params[param_name].grad is None: self._master_params[param_name]._copy_gradient_from( - grad_fp32_cpu) + grad_fp32_cpu + ) else: self._master_params[param_name].grad.add_(grad_fp32_cpu) self.offload_params.buffer._copy_gradient_from( - self.offload_grads.buffer) + self.offload_grads.buffer + ) def _offload_scale_grad(self, scale_size): """scale grads with offload strategy""" @@ -456,7 +499,7 @@ def step(self): if self.offload: params_list = [self.offload_params.buffer] - #TODO(Baibaifan): Offload will support param_groups later + # TODO(Baibaifan): Offload will support param_groups later if not isinstance(self._optim._param_groups[0], dict): self._optim._parameter_list = params_list self._optim._param_groups = params_list @@ -468,8 +511,11 @@ def step(self): for param in self._local_params: if param.name in self._master_params.keys(): - param.set_value(self._master_params[param.name].cuda( - self.dev_id).cast(dtype=param.dtype)) + param.set_value( + self._master_params[param.name] + .cuda(self.dev_id) + .cast(dtype=param.dtype) + ) else: self._optim.step() @@ -478,7 +524,8 @@ def step(self): def minimize(self): raise RuntimeError( - "optimizer.minimize() not support now, please use optimizer.step()") + "optimizer.minimize() not support now, please use optimizer.step()" + ) def set_state_dict(self, state_dict): self._optim.set_state_dict(state_dict) @@ -501,10 +548,12 @@ def _broadcast_params(self): else: for dtype_per_rank in self.param_storages.values(): for dst_rank, internal_storage in dtype_per_rank.items(): - broadcast(tensor=internal_storage.buffer, - src=self._group.ranks[dst_rank], - group=self._group, - sync_op=True) + broadcast( + tensor=internal_storage.buffer, + src=self._group.ranks[dst_rank], + group=self._group, + sync_op=True, + ) def _forward_pre_hook_function(self, tasks): # Since the layers will call pre hook by `forward_pre_hook(self, inputs)`, @@ -527,10 +576,12 @@ def _broadcast_params_overlap_forward(self): if x.trainable: group = self._broadcast_groups[group_idx] group_idx = (group_idx + 1) % self._number_of_broadcast_groups - task = broadcast(tensor=x, - src=group.ranks[self._param2rank[x.name]], - group=group, - sync_op=False) + task = broadcast( + tensor=x, + src=group.ranks[self._param2rank[x.name]], + group=group, + sync_op=False, + ) assert x.name not in param2task param2task[x.name] = task @@ -544,4 +595,6 @@ def _broadcast_params_overlap_forward(self): tasks.append(param2task[param.name]) self._forward_pre_hook_remove_helper.append( layer.register_forward_pre_hook( - self._forward_pre_hook_function(tasks))) + self._forward_pre_hook_function(tasks) + ) + ) diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage2.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage2.py index 6c0716f7bbb31..044111cc3490a 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage2.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage2.py @@ -58,38 +58,50 @@ class GroupShardedStage2(nn.Layer): # 5. Support the establishment of independent communication groups. def __init__( - self, - layer, - sharding_optimizer, - group=None, - sync_buffers=False, - buffer_max_size=2**23, #8MB - auto_refresh_trainable=True, - device="gpu", - dp_group=None): + self, + layer, + sharding_optimizer, + group=None, + sync_buffers=False, + buffer_max_size=2**23, # 8MB + auto_refresh_trainable=True, + device="gpu", + dp_group=None, + ): super().__init__() # training options self._layer = layer - self._sharding_optimizers = [ - sharding_optimizer - ] if not isinstance(sharding_optimizer, list) else sharding_optimizer + self._sharding_optimizers = ( + [sharding_optimizer] + if not isinstance(sharding_optimizer, list) + else sharding_optimizer + ) assert all( list( - map(lambda opt: isinstance(opt, GroupShardedOptimizerStage2), - self._sharding_optimizers)) + map( + lambda opt: isinstance(opt, GroupShardedOptimizerStage2), + self._sharding_optimizers, + ) + ) ), "Please use GroupShardedOptimizerStage2 optimizer" self._sync_buffers = sync_buffers self._auto_refresh_trainable = auto_refresh_trainable # Communication related attributes - self._group = collective.new_group( - collective._get_global_group().ranks) if group is None else group + self._group = ( + collective.new_group(collective._get_global_group().ranks) + if group is None + else group + ) self._world_size_scaling = 1.0 / self._group.nranks - assert self._group.nranks > 1, "Training must be distributed, ranks must be greater than 1" + assert ( + self._group.nranks > 1 + ), "Training must be distributed, ranks must be greater than 1" self._rank = self._group.rank self._global_root_rank = self._group.ranks[ - 0] # picking ranks index 0 as the reference + 0 + ] # picking ranks index 0 as the reference self._default_device = device self._dp_group = dp_group @@ -106,15 +118,17 @@ def __init__( self._trainable_param2rank = {} self._trainable_param2align = {} self._trainable_params = list( - filter(lambda x: x.trainable, self._all_params)) + filter(lambda x: x.trainable, self._all_params) + ) self._trainable_mask = list(map(_trainable, self._trainable_params)) self._param_grads = [] # Set grad storage size & Display param sizes and model sizes model_size = sum([p._numel() for p in self._layer.parameters()]) assert buffer_max_size >= 0, "buffer_max_size must be GE than 0." - self._buffer_max_size = self._rank_buffer_size(buffer_max_size, - model_size) + self._buffer_max_size = self._rank_buffer_size( + buffer_max_size, model_size + ) self._use_grad_storage = buffer_max_size > 0 self._grad_storages = {} # {dtype: {rank: GradStorage}} self._has_grad_storage = [] @@ -123,11 +137,12 @@ def __init__( # Offload # TODO(haohongxiang): Now it's not be supported for multi-optimizers using Offload strategy self._offload_optims = list( - filter(lambda optim: optim.offload, self._sharding_optimizers)) + filter(lambda optim: optim.offload, self._sharding_optimizers) + ) if len(self._offload_optims) > 0: - assert len( - self._sharding_optimizers - ) == 1, "Only support offload strategy for single optimizer" + assert ( + len(self._sharding_optimizers) == 1 + ), "Only support offload strategy for single optimizer" self._offload = len(self._offload_optims) > 0 self._offload_device = "cpu" @@ -170,17 +185,21 @@ def forward(self, *inputs, **kwargs): return fw def set_state_dict(self, state_dict, use_structured_name=True): - self._layer.set_state_dict(state_dict, - use_structured_name=use_structured_name) - - def state_dict(self, - destination=None, - include_sublayers=True, - structured_name_prefix=""): + self._layer.set_state_dict( + state_dict, use_structured_name=use_structured_name + ) + + def state_dict( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + ): return self._layer.state_dict( destination=destination, include_sublayers=include_sublayers, - structured_name_prefix=structured_name_prefix) + structured_name_prefix=structured_name_prefix, + ) def _clear_gradients(self): """ @@ -188,8 +207,10 @@ def _clear_gradients(self): """ # Release grad storages for dtype in self._grad_storages.keys(): - if not self._offload and self._rank in self._grad_storages[ - dtype].keys(): + if ( + not self._offload + and self._rank in self._grad_storages[dtype].keys() + ): self._grad_storages[dtype][self._rank].buffer.zero_() # Release grads of params @@ -213,10 +234,13 @@ def _grad_scale(self): # Scale grad storages for dtype in self._grad_storages.keys(): - if not self._offload and self._rank in self._grad_storages[ - dtype].keys(): + if ( + not self._offload + and self._rank in self._grad_storages[dtype].keys() + ): self._grad_storages[dtype][self._rank].buffer.scale_( - scale=scale_factor) + scale=scale_factor + ) # Scale grads of params with paddle.no_grad(): @@ -246,7 +270,9 @@ def to(self, device=None, dtype=None, blocking=True): Synchronously or asynchronously convert the data type of the layer, the device is not supported now. """ assert isinstance(device, str), "Device must be type str" - assert device == self._default_device, "New devices are not supported, because of the optimizer state is not sync" + assert ( + device == self._default_device + ), "New devices are not supported, because of the optimizer state is not sync" self._layer.to(device=device, dtype=dtype, blocking=blocking) @@ -254,14 +280,15 @@ def to(self, device=None, dtype=None, blocking=True): self._fresh_trainable() def _fresh_trainable(self): - """ Whether to update training parameters. """ + """Whether to update training parameters.""" # Make sure that this is not done while gradients are waiting to be reduced (if no_sync context for instance) if reduce(lambda x, y: x or y, self._grad_reduced, False): logging.warning("Grads waiting to be reduced.") self._trainable_params = list( - filter(lambda x: x.trainable, self._all_params)) + filter(lambda x: x.trainable, self._all_params) + ) self._trainable_params.sort(key=lambda x: x._numel()) self._trainable_param2rank = {} @@ -271,14 +298,19 @@ def _fresh_trainable(self): optim._update_opt_status() # Get the parameters split by the optimizer according to rank - for per_rank_params in optim.dtype_rank_params.values( + for ( + per_rank_params + ) in ( + optim.dtype_rank_params.values() ): # all the params from all ranks for params in per_rank_params: for param in filter(lambda x: x.trainable, params): self._trainable_param2rank[ - param.name] = optim.param2rank[param.name] + param.name + ] = optim.param2rank[param.name] self._trainable_param2align[ - param.name] = optim._param2align[param.name] + param.name + ] = optim._param2align[param.name] # Create grad_storage self._setup_use_grad_storage() @@ -292,16 +324,17 @@ def __sync_buffers(self): """ for buffer in self._layer.buffers(include_sublayers=True): - collective.broadcast(buffer, - self._global_root_rank, - self._group, - sync_op=True) + collective.broadcast( + buffer, self._global_root_rank, self._group, sync_op=True + ) if self._dp_group and self._dp_group.nranks > 1: - collective.broadcast(buffer, - self._dp_group.ranks[0], - self._dp_group, - sync_op=True) + collective.broadcast( + buffer, + self._dp_group.ranks[0], + self._dp_group, + sync_op=True, + ) def __getattr__(self, name): """Forward missing attributes to wrapped layer.""" @@ -327,9 +360,9 @@ def _set_reduce_overlap(self, reduce_overlap): # model._set_reduce_overlap(True) self._reduce_overlap = reduce_overlap if self._reduce_overlap: - assert len( - self._sharding_optimizers - ) == 1, "Only support comm overlap strategy for single optimizer" + assert ( + len(self._sharding_optimizers) == 1 + ), "Only support comm overlap strategy for single optimizer" self._sharding_optimizers[0]._set_reduce_overlap(reduce_overlap) def _get_reduce_fn(self, index, param, dst_rank): @@ -345,7 +378,9 @@ def _get_reduce_fn(self, index, param, dst_rank): def reduce(*_): # Skip gradient reduction, do not change status information if self._grad_reduced[index]: - assert param.grad is not None, "Parameter gradient cannot be None" + assert ( + param.grad is not None + ), "Parameter gradient cannot be None" # Change reduce information self._grad_reduced[index] = False @@ -356,26 +391,35 @@ def cleanup(): param.clear_gradient(False) elif self._offload: tmp_grad = param.grad.cast( - dtype=Type.fp32.value).cpu() + dtype=Type.fp32.value + ).cpu() self._sharding_optimizers[0]._offload_acc_grad( - param.name, tmp_grad) + param.name, tmp_grad + ) del tmp_grad param.clear_gradient(False) # Synchronize the reduce parameter gradient asynchronize self._sharding_optimizers[0]._update_task( - collective.reduce(tensor=param.grad, - dst=self._group.ranks[dst_rank], - group=self._group, - sync_op=not self._reduce_overlap)) + collective.reduce( + tensor=param.grad, + dst=self._group.ranks[dst_rank], + group=self._group, + sync_op=not self._reduce_overlap, + ) + ) if self._dp_group and self._dp_group.nranks > 1: - assert not self._reduce_overlap, 'dp + stage2 hybrid parallel only Synchronize due to the new communication lib.' - #TODO(wuhuachao):after the new communication lib upgrading, overlapping the comm of dp + stage2. - collective.all_reduce(tensor=param.grad, - group=self._dp_group, - sync_op=True) + assert ( + not self._reduce_overlap + ), 'dp + stage2 hybrid parallel only Synchronize due to the new communication lib.' + # TODO(wuhuachao):after the new communication lib upgrading, overlapping the comm of dp + stage2. + collective.all_reduce( + tensor=param.grad, + group=self._dp_group, + sync_op=True, + ) # Clear the task flow and trigger callback to clear the redundant gradient # self._clear_task_flow() @@ -388,7 +432,9 @@ def cleanup(): def reduce(*_): # Skip gradient reduction, do not change status information if self._grad_reduced[index]: - assert param.grad is not None, "Parameter gradient cannot be None" + assert ( + param.grad is not None + ), "Parameter gradient cannot be None" # Change reduce information self._grad_reduced[index] = False @@ -410,9 +456,11 @@ def cleanup(): for p in grad_storage._params: with device_guard(): tmp_grad = p.grad.cast( - dtype=Type.fp32.value) + dtype=Type.fp32.value + ) self._sharding_optimizers[ - 0]._offload_acc_grad(p.name, tmp_grad) + 0 + ]._offload_acc_grad(p.name, tmp_grad) p.clear_gradient(False) grad_storage._device = self._default_device grad_storage.buffer._clear_data() @@ -425,14 +473,20 @@ def cleanup(): tensor=grad_storage.buffer, dst=self._group.ranks[grad_storage.destination], group=self._group, - sync_op=not self._reduce_overlap)) + sync_op=not self._reduce_overlap, + ) + ) if self._dp_group and self._dp_group.nranks > 1: - assert not self._reduce_overlap, 'dp + stage2 hybrid parallel only Synchronize due to the new communication lib.' - #TODO(wuhuachao):after the new communication lib upgrading, overlapping the comm of dp + stage2. - collective.all_reduce(tensor=grad_storage.buffer, - group=self._dp_group, - sync_op=True) + assert ( + not self._reduce_overlap + ), 'dp + stage2 hybrid parallel only Synchronize due to the new communication lib.' + # TODO(wuhuachao):after the new communication lib upgrading, overlapping the comm of dp + stage2. + collective.all_reduce( + tensor=grad_storage.buffer, + group=self._dp_group, + sync_op=True, + ) cleanup() @@ -460,7 +514,8 @@ def _setup_backward_hooks(self): reduce_function = self._get_reduce_fn(index, param, dst_rank) self._bw_hooks.append( - param._register_backward_hook(reduce_function)) + param._register_backward_hook(reduce_function) + ) def _setup_use_grad_storage(self): """ @@ -483,25 +538,32 @@ def _setup_use_grad_storage(self): dtype=param.dtype, device=self._default_device, destination=dst_rank, - parm2align=self._trainable_param2align) + parm2align=self._trainable_param2align, + ) # Criteria to decide whether this parameter is to be put in GradStorage if self._grad_storages[param.dtype][dst_rank].can_add_grad_view( - param, self._trainable_param2align[param.name]): + param, self._trainable_param2align[param.name] + ): self._grad_storages[param.dtype][dst_rank].add_grad( - param, self._trainable_param2align[param.name]) + param, self._trainable_param2align[param.name] + ) self._has_grad_storage[index] = True else: self._param_grads.append(param.name) print( - "Can not add param: {}, param's shape: {}, param align: {}, grad_storages fill: {}, " - .format(param.name, param.shape, - self._trainable_param2align[param.name], - self._grad_storages[param.dtype][dst_rank]._fill)) + "Can not add param: {}, param's shape: {}, param align: {}, grad_storages fill: {}, ".format( + param.name, + param.shape, + self._trainable_param2align[param.name], + self._grad_storages[param.dtype][dst_rank]._fill, + ) + ) for dtype in self._grad_storages.keys(): self._grad_storage_list.extend( - list(self._grad_storages[dtype].values())) + list(self._grad_storages[dtype].values()) + ) # def _clear_task_flow(self): # """Try to consume the previous tasks.""" @@ -553,21 +615,27 @@ def _rank_buffer_size(self, buffer_max_size, model_size): if Type.fp16.value in rank_buffer_size.keys(): # FP16 GradStorage and model size logger_.info( - "====== FP16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======" - .format(rank_buffer_size[Type.fp16.value] / 2**19, - model_size / 2**19)) + "====== FP16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======".format( + rank_buffer_size[Type.fp16.value] / 2**19, + model_size / 2**19, + ) + ) if Type.bf16.value in rank_buffer_size.keys(): # FP16 GradStorage and model size logger_.info( - "====== BF16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======" - .format(rank_buffer_size[Type.bf16.value] / 2**19, - model_size / 2**19)) + "====== BF16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======".format( + rank_buffer_size[Type.bf16.value] / 2**19, + model_size / 2**19, + ) + ) if Type.fp32.value in rank_buffer_size.keys(): # FP32 GradStorage and model size logger_.info( - "====== FP32 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======" - .format(rank_buffer_size[Type.fp32.value] / 2**18, - model_size / 2**18)) + "====== FP32 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======".format( + rank_buffer_size[Type.fp32.value] / 2**18, + model_size / 2**18, + ) + ) return rank_buffer_size def _redefine_opt_step(self): diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py index 00ce653e1df7d..e80ea0531f359 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_stage3.py @@ -72,16 +72,18 @@ class GroupShardedStage3(nn.Layer): # 3. Support offload function. # 4. Support the establishment of independent communication groups. - def __init__(self, - layer, - optimizer, - group=None, - sync_buffers=False, - device="gpu", - segment_size=2**20, - pertrain_sync_models=True, - offload=False, - sync_comm=False): + def __init__( + self, + layer, + optimizer, + group=None, + sync_buffers=False, + device="gpu", + segment_size=2**20, + pertrain_sync_models=True, + offload=False, + sync_comm=False, + ): super().__init__() # Default configs @@ -96,27 +98,40 @@ def __init__(self, self._segment_size = segment_size global DEV - DEV = "cpu" if paddle.get_device() == "cpu" else paddle.get_device( - ).split(":")[0] + DEV = ( + "cpu" + if paddle.get_device() == "cpu" + else paddle.get_device().split(":")[0] + ) global DEV_ID - DEV_ID = 0 if paddle.get_device() == "cpu" else int( - paddle.get_device().split(":")[1]) + DEV_ID = ( + 0 + if paddle.get_device() == "cpu" + else int(paddle.get_device().split(":")[1]) + ) global param2dtype param2dtype = dict() # Communication group establishment - self._group = collective.new_group( - collective._get_global_group().ranks) if group is None else group + self._group = ( + collective.new_group(collective._get_global_group().ranks) + if group is None + else group + ) self._world_size_scaling = 1.0 / self._group.nranks - assert self._group.nranks > 1, "Training must be distributed, ranks must be greater than 1." + assert ( + self._group.nranks > 1 + ), "Training must be distributed, ranks must be greater than 1." self._rank = self._group.rank self._global_root_rank = self._group.ranks[ - 0] # picking ranks index 0 as the reference + 0 + ] # picking ranks index 0 as the reference # Parameter segmentation for global ranks # After flatten -> self._param2buffer_size, self._param2buffer, self._trainable_params self._param2buffer_size = dict() # {param.name: size} - self._param2buffer = dict( + self._param2buffer = ( + dict() ) # {param.name: [(start0, end0),(start1, end1), ...]} self._trainable_params = dict() # {id(layer): [trainable_params]} self._unslice_params = set() # param's numel <= segment_size @@ -124,9 +139,11 @@ def __init__(self, self._grad_storages = dict() # {param.dtype: GradStorage} assert not isinstance( - optimizer, list), "Multiple optimizers are not supported now." - self._optim = _OptimizerWrapper(optimizer, self._offload, self._group, - self._update_params_slice) + optimizer, list + ), "Multiple optimizers are not supported now." + self._optim = _OptimizerWrapper( + optimizer, self._offload, self._group, self._update_params_slice + ) self._ori_parameter_list = self._optim._parameter_list self._ori_param_groups = self._optim._param_groups @@ -136,9 +153,11 @@ def __init__(self, "While using ClipGradByGlobalNorm in GroupShardedStage3, the grad clip of original optimizer will be changed." ) self._optim._grad_clip = GroupShardedClipGrad( - self._optim._grad_clip, paddle.get_device(), self._group) + self._optim._grad_clip, paddle.get_device(), self._group + ) if self._optim._parameter_list and isinstance( - self._optim._parameter_list[0], dict): + self._optim._parameter_list[0], dict + ): for item in self._optim._param_groups: if "grad_clip" in item.keys(): item["grad_clip"] = self._optim._grad_clip @@ -177,22 +196,24 @@ def _sync_params_and_buffers(self): """ for p in self._layer.parameters(): - collective.broadcast(p, - src=self._global_root_rank, - group=self._group, - sync_op=True) + collective.broadcast( + p, src=self._global_root_rank, group=self._group, sync_op=True + ) def _clear_gradients(self): assert len(self._trainable_params.keys()) > 0 current_layer_params = self._layer.parameters(include_sublayers=True) # 1.Handle param's slice trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) for param in trainable_params: - assert hasattr(param, "fw_storage" - ), "Find {} don't have fw_storage attribute.".format( - param.name) + assert hasattr( + param, "fw_storage" + ), "Find {} don't have fw_storage attribute.".format(param.name) param.fw_storage.clear_gradient(False) param.bw_storage._clear() @@ -206,8 +227,10 @@ def _clear_gradients(self): param.clear_gradient(False) tmp_var = param.cuda(DEV_ID) - if tmp_var.dtype == Type.fp32.value and param2dtype[ - param.name] == Type.fp16.value: + if ( + tmp_var.dtype == Type.fp32.value + and param2dtype[param.name] == Type.fp16.value + ): tmp_var = paddle.cast(tmp_var, Type.fp16.value) tmp_var._share_buffer_to(param) del tmp_var @@ -222,9 +245,11 @@ def _update_params_slice(self): if not isinstance(self._optim._param_groups[0], dict): slice_params = [param.fw_storage for param in update_list] self._optim._parameter_list = slice_params + list( - self._unslice_params) + self._unslice_params + ) self._optim._param_groups = slice_params + list( - self._unslice_params) + self._unslice_params + ) else: for param_group in self._optim._param_groups: p_group = [] @@ -250,17 +275,21 @@ def forward(self, *inputs, **kwargs): return fw def set_state_dict(self, state_dict, use_structured_name=True): - self._layer.set_state_dict(state_dict, - use_structured_name=use_structured_name) - - def state_dict(self, - destination=None, - include_sublayers=True, - structured_name_prefix=""): + self._layer.set_state_dict( + state_dict, use_structured_name=use_structured_name + ) + + def state_dict( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + ): return self._layer.state_dict( destination=destination, include_sublayers=include_sublayers, - structured_name_prefix=structured_name_prefix) + structured_name_prefix=structured_name_prefix, + ) def _handle_unslice_params(self): buffer_size = dict() @@ -287,9 +316,11 @@ def _handle_unslice_params(self): dtype=param.dtype, device=self._default_device, destination=self._rank, - parm2align=self._unslice_params2align) + parm2align=self._unslice_params2align, + ) self._grad_storages[param.dtype].add_grad( - param, self._unslice_params2align[param.name]) + param, self._unslice_params2align[param.name] + ) def _segment_rank_params(self, layer, name="last_layer"): """ @@ -329,8 +360,11 @@ def _add_manage_info(trainable_param): align_ = self._param2align(param) offset = align_ + param._numel() - buffer_size = offset if offset % self._group.nranks == 0 else offset + self._group.nranks - ( - offset % self._group.nranks) + buffer_size = ( + offset + if offset % self._group.nranks == 0 + else offset + self._group.nranks - (offset % self._group.nranks) + ) self._param2buffer_size[param.name] = buffer_size # 2.Combination param buffer @@ -339,7 +373,8 @@ def _add_manage_info(trainable_param): for rank_ in range(self._group.nranks): self._param2buffer[param.name].append( - (rank_ * pre_buffer, (rank_ + 1) * pre_buffer)) + (rank_ * pre_buffer, (rank_ + 1) * pre_buffer) + ) # Record param's dtype param2dtype[param.name] = param.dtype @@ -351,10 +386,11 @@ def _param_storage(self, param, buffer_size): This is a function to simplify the handling of parameter InternalStorages. """ assert isinstance(buffer_size, int) - value = np.zeros( - buffer_size, - dtype=np.float16) if Type.fp16.value == param.dtype else np.zeros( - buffer_size, dtype=np.float32) + value = ( + np.zeros(buffer_size, dtype=np.float16) + if Type.fp16.value == param.dtype + else np.zeros(buffer_size, dtype=np.float32) + ) buffer = core.eager.Tensor(value=value, place=core.CPUPlace()) param_shape = param.shape @@ -376,16 +412,19 @@ def _param_storage(self, param, buffer_size): if self._offload: with device_guard(): tmp_tensor = buffer._slice(start, end) - param.fw_storage = core.eager.Tensor(value=tmp_tensor, - place=core.CPUPlace(), - name="slice@" + param.name) + param.fw_storage = core.eager.Tensor( + value=tmp_tensor, + place=core.CPUPlace(), + name="slice@" + param.name, + ) with device_guard(): - param.master_weight = paddle.cast(param.fw_storage, - Type.fp32.value) + param.master_weight = paddle.cast( + param.fw_storage, Type.fp32.value + ) else: - param.fw_storage = core.eager.Tensor(value=buffer._slice( - start, end), - name="slice@" + param.name) + param.fw_storage = core.eager.Tensor( + value=buffer._slice(start, end), name="slice@" + param.name + ) param.status = "part" # Updata optimizer master weights @@ -414,20 +453,32 @@ def _register_forward_hooks(self, layer): self._register_forward_hooks(sub_layer) def _register_forward_all_hooks(self, sub_layer, task_flow): - def _forward_pre_hook(layer, inputs): - return ForwardPreHooks(layer, self._order_tracer, - self._trainable_params, - self._param2buffer_size, self._group, - self._sync_comm, self._offload, task_flow) + return ForwardPreHooks( + layer, + self._order_tracer, + self._trainable_params, + self._param2buffer_size, + self._group, + self._sync_comm, + self._offload, + task_flow, + ) def _forward_post_hook(layer, inputs, outputs): - return ForwardPostHooks.apply(outputs, layer, self._order_tracer, - self._trainable_params, - self._param2buffer, - self._param2buffer_size, self._rank, - self._group, self._sync_comm, - self._offload, task_flow) + return ForwardPostHooks.apply( + outputs, + layer, + self._order_tracer, + self._trainable_params, + self._param2buffer, + self._param2buffer_size, + self._rank, + self._group, + self._sync_comm, + self._offload, + task_flow, + ) # register previous forward hooks sub_layer.register_forward_pre_hook(_forward_pre_hook) @@ -442,10 +493,9 @@ def _sync_buffers(self): """ for buffer in self._layer.buffers(include_sublayers=True): - collective.broadcast(buffer, - self._global_root_rank, - self._group, - sync_op=True) + collective.broadcast( + buffer, self._global_root_rank, self._group, sync_op=True + ) def __getattr__(self, name): """Forward missing attributes to wrapped layer.""" @@ -462,14 +512,16 @@ def _update_params(self): assert len(self._trainable_params.keys()) > 0 current_layer_params = self._layer.parameters(include_sublayers=True) trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) # 1.Handle param's slice for param in trainable_params: assert hasattr( - param, - "fw_storage"), "Find {} don't have fw_storage attribute".format( - param.name) + param, "fw_storage" + ), "Find {} don't have fw_storage attribute".format(param.name) # Gradient average if self._offload: with device_guard(): @@ -507,16 +559,21 @@ def get_all_parameters(self, convert2cpu=False): assert len(self._trainable_params.keys()) > 0 current_layer_params = self._layer.parameters(include_sublayers=True) trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) - t_flow = _allgather_buffer(trainable_params, - self._group, - param2buffer_size=self._param2buffer_size, - use_calc_stream=True, - task_flow=TaskFlow(), - sync_wait=True, - offload=self._offload, - convert2cpu=convert2cpu) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) + t_flow = _allgather_buffer( + trainable_params, + self._group, + param2buffer_size=self._param2buffer_size, + use_calc_stream=True, + task_flow=TaskFlow(), + sync_wait=True, + offload=self._offload, + convert2cpu=convert2cpu, + ) if convert2cpu: for param in trainable_params: t_flow.full_param[param.name][0]._share_buffer_to(param) @@ -527,15 +584,17 @@ def get_all_parameters(self, convert2cpu=False): def _register_backward_hooks(self): current_layer_params = self._layer.parameters(include_sublayers=True) trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) for param in trainable_params: allreduce_function = self._get_allreduce_fn(param) param._register_backward_hook(allreduce_function) def _get_allreduce_fn(self, param): - @paddle.autograd.no_grad() def allreduce_(*_): if param.name in self._task_flow.full_grad.keys(): @@ -545,21 +604,25 @@ def allreduce_(*_): start, end = self._param2buffer[param.name][self._rank] if param.bw_storage is None: - param.bw_storage = full_grad._slice(start, - end).detach().clone() + param.bw_storage = ( + full_grad._slice(start, end).detach().clone() + ) if self._offload: param.bw_storage = _device2cpu(param.bw_storage, True) else: if self._offload: cpu_grad = _device2cpu( - full_grad._slice(start, end).detach().clone(), True) + full_grad._slice(start, end).detach().clone(), True + ) with device_guard(): param.bw_storage = paddle.add( - param.bw_storage, cpu_grad) + param.bw_storage, cpu_grad + ) else: param.bw_storage = paddle.add( param.bw_storage, - full_grad._slice(start, end).detach().clone()) + full_grad._slice(start, end).detach().clone(), + ) param.clear_gradient(False) del self._task_flow.full_grad[param.name] @@ -568,8 +631,12 @@ def allreduce_(*_): param.use_count = 0 param._clear_data() start, end = self._param2buffer[param.name][self._rank] - param.fw_storage = self._task_flow.full_param[ - param.name][0]._slice(start, end).detach().clone() + param.fw_storage = ( + self._task_flow.full_param[param.name][0] + ._slice(start, end) + .detach() + .clone() + ) param.status = "part" del self._task_flow.full_param[param.name] @@ -583,8 +650,9 @@ def _param2align(self, param): # CUDA alignment 256 bytes size = param._numel() * align[param.dtype] remaining = size % alignment[self._default_device] - ali = 0 if remaining == 0 else alignment[ - self._default_device] - remaining + ali = ( + 0 if remaining == 0 else alignment[self._default_device] - remaining + ) align_ = ali // align[param.dtype] return align_ @@ -618,8 +686,16 @@ def _opt_clear(self): self._optim.clear_grad = MethodType(_opt_clear, self._optim) -def ForwardPreHooks(layer, order_tracer, trainable_params, param2buffer_size, - group, sync_comm, offload, task_flow): +def ForwardPreHooks( + layer, + order_tracer, + trainable_params, + param2buffer_size, + group, + sync_comm, + offload, + task_flow, +): # Record layer's id layer_id = id(layer) @@ -634,35 +710,55 @@ def ForwardPreHooks(layer, order_tracer, trainable_params, param2buffer_size, # Whether to use calc stream task_flow.use_calc[layer_id] = use_calc # wait current layer params - _wait_layer(trainable_params[layer_id], task_flow, group, - param2buffer_size, use_calc, offload) - - if layer_id == order_tracer["layer"][-1]: return + _wait_layer( + trainable_params[layer_id], + task_flow, + group, + param2buffer_size, + use_calc, + offload, + ) + + if layer_id == order_tracer["layer"][-1]: + return order_ = order_tracer[layer_id] layer_id = order_tracer["layer"][order_ + 1] - _allgather_buffer(trainable_params[layer_id], - group, - param2buffer_size=param2buffer_size, - use_calc_stream=use_calc, - task_flow=task_flow, - sync_wait=sync_wait, - offload=offload) + _allgather_buffer( + trainable_params[layer_id], + group, + param2buffer_size=param2buffer_size, + use_calc_stream=use_calc, + task_flow=task_flow, + sync_wait=sync_wait, + offload=offload, + ) return class ForwardPostHooks(PyLayer): - @staticmethod - def forward(ctx, inputs, layer, order_tracer, trainable_params, - param2buffer, param2buffer_size, rank, group, sync_comm, - offload, task_flow): + def forward( + ctx, + inputs, + layer, + order_tracer, + trainable_params, + param2buffer, + param2buffer_size, + rank, + group, + sync_comm, + offload, + task_flow, + ): layer_id = id(layer) # release current layer full params - _release_param(trainable_params[layer_id], param2buffer, rank, - task_flow, offload) + _release_param( + trainable_params[layer_id], param2buffer, rank, task_flow, offload + ) if layer_id not in order_tracer.keys(): order_ = order_tracer["order"] @@ -670,7 +766,7 @@ def forward(ctx, inputs, layer, order_tracer, trainable_params, order_tracer["order"] += 1 order_tracer["layer"].append(layer_id) - #Record fw info + # Record fw info ctx.order_tracer = order_tracer ctx.task_flow = task_flow ctx.group = group @@ -698,32 +794,43 @@ def backward(ctx, *args): # Allgather params synchronization if sync_comm: use_calc, sync_wait = True, True - _allgather_buffer(trainable_params[layer_id], - group, - param2buffer_size=param2buffer_size, - use_calc_stream=use_calc, - task_flow=task_flow, - sync_wait=sync_wait, - offload=offload) + _allgather_buffer( + trainable_params[layer_id], + group, + param2buffer_size=param2buffer_size, + use_calc_stream=use_calc, + task_flow=task_flow, + sync_wait=sync_wait, + offload=offload, + ) else: - _wait_layer(trainable_params[layer_id], task_flow, group, - param2buffer_size, use_calc, offload) + _wait_layer( + trainable_params[layer_id], + task_flow, + group, + param2buffer_size, + use_calc, + offload, + ) # Create params's grad - _create_params_grad(trainable_params[layer_id], param2buffer_size, - task_flow) + _create_params_grad( + trainable_params[layer_id], param2buffer_size, task_flow + ) # Whether to use calc stream task_flow.use_calc[layer_id] = use_calc if layer_id != order_tracer["layer"][0] and not sync_comm: layer_next_id = order_tracer["layer"][order_tracer[layer_id] - 1] - _allgather_buffer(trainable_params[layer_next_id], - group, - param2buffer_size=param2buffer_size, - use_calc_stream=use_calc, - task_flow=task_flow, - sync_wait=sync_wait, - offload=offload) + _allgather_buffer( + trainable_params[layer_next_id], + group, + param2buffer_size=param2buffer_size, + use_calc_stream=use_calc, + task_flow=task_flow, + sync_wait=sync_wait, + offload=offload, + ) return args @@ -733,22 +840,22 @@ class TaskFlow: Task flows, one way linked list for task acquisition. """ - def __init__(self, - full_param=dict(), - full_grad=dict(), - use_calc=dict(), - callback=None): + def __init__( + self, + full_param=dict(), + full_grad=dict(), + use_calc=dict(), + callback=None, + ): self.full_param = full_param self.full_grad = full_grad self.use_calc = use_calc self.callback = callback -def _release_param(trainable_params, - param2buffer, - rank, - task_flow, - offload=False): +def _release_param( + trainable_params, param2buffer, rank, task_flow, offload=False +): for param in trainable_params: # async communicate share weight not clear param.use_count -= 1 @@ -757,8 +864,12 @@ def _release_param(trainable_params, if param.name in task_flow.full_param.keys(): start, end = param2buffer[param.name][rank] with paddle.amp.auto_cast(enable=False): - param.fw_storage = task_flow.full_param[ - param.name][0]._slice(start, end).detach().clone() + param.fw_storage = ( + task_flow.full_param[param.name][0] + ._slice(start, end) + .detach() + .clone() + ) param.status = "part" del task_flow.full_param[param.name] @@ -767,12 +878,14 @@ def _release_param(trainable_params, return -def _wait_layer(trainable_params, - task_flow, - group, - param2buffer_size, - use_calc_stream, - offload=False): +def _wait_layer( + trainable_params, + task_flow, + group, + param2buffer_size, + use_calc_stream, + offload=False, +): for param in trainable_params: if param.status == "all": @@ -787,25 +900,29 @@ def _wait_layer(trainable_params, param.status = "all" param.use_count += 1 else: - _allgather_buffer(trainable_params, - group, - param2buffer_size=param2buffer_size, - use_calc_stream=True, - task_flow=task_flow, - sync_wait=True, - offload=offload) + _allgather_buffer( + trainable_params, + group, + param2buffer_size=param2buffer_size, + use_calc_stream=True, + task_flow=task_flow, + sync_wait=True, + offload=offload, + ) break return task_flow -def _allgather_buffer(trainable_params, - group, - param2buffer_size, - use_calc_stream, - task_flow, - sync_wait=False, - offload=False, - convert2cpu=False): +def _allgather_buffer( + trainable_params, + group, + param2buffer_size, + use_calc_stream, + task_flow, + sync_wait=False, + offload=False, + convert2cpu=False, +): for param in trainable_params: if param.status == "all": @@ -846,8 +963,9 @@ def _create_params_grad(trainable_params, param2buffer_size, task_flow): if param.name in task_flow.full_grad.keys(): continue assert isinstance(param2buffer_size[param.name], int) - temp_grad = paddle.zeros([param2buffer_size[param.name]], - dtype=param.dtype) + temp_grad = paddle.zeros( + [param2buffer_size[param.name]], dtype=param.dtype + ) temp_tensor = temp_grad._slice(0, param._numel()) temp_tensor.get_tensor()._set_dims(param.shape) param._copy_gradient_from(temp_tensor) @@ -875,13 +993,14 @@ def _UnsliceParam(param): def _VarBaseWrapper(param): varbase = param.fw_storage - tmp_param = EagerParamBase(shape=varbase.shape, - dtype=varbase.dtype, - name="slice@" + param.name) + tmp_param = EagerParamBase( + shape=varbase.shape, dtype=varbase.dtype, name="slice@" + param.name + ) varbase._share_buffer_to(tmp_param) tmp_param.regularizer = param.regularizer tmp_param.optimize_attr['learning_rate'] = param.optimize_attr[ - 'learning_rate'] + 'learning_rate' + ] varbase._clear() return tmp_param @@ -906,14 +1025,17 @@ def _device2cpu(trans_param, convert_dtype=False): def _cpu2device(param): tmp_p = param.fw_storage.cuda(DEV_ID) - if tmp_p.dtype == Type.fp32.value and param2dtype[ - param.name] == Type.fp16.value: + if ( + tmp_p.dtype == Type.fp32.value + and param2dtype[param.name] == Type.fp16.value + ): tmp_p = paddle.cast(tmp_p, Type.fp16.value) return tmp_p def _current_layer_params(layer): - return layer.parameters( - include_sublayers=False) + list(layer.extra_parameters) if hasattr( - layer, "extra_parameters") else layer.parameters( - include_sublayers=False) + return ( + layer.parameters(include_sublayers=False) + list(layer.extra_parameters) + if hasattr(layer, "extra_parameters") + else layer.parameters(include_sublayers=False) + ) diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_storage.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_storage.py index 66e9617f00efd..72e9ebfcb7d88 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_storage.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_storage.py @@ -46,30 +46,40 @@ def __init__(self, size, dtype, device, convert_cpu=False): # The flatten tensor size = [size] if isinstance(size, int) else size if convert_cpu: - value = np.zeros( - size, - dtype=np.float16) if Type.fp16.value == dtype else np.zeros( - size, dtype=np.float32) + value = ( + np.zeros(size, dtype=np.float16) + if Type.fp16.value == dtype + else np.zeros(size, dtype=np.float32) + ) self.buffer = core.eager.Tensor(value=value, place=core.CPUPlace()) if dtype == Type.bf16.value: self.buffer = paddle.cast(self.buffer, dtype=paddle.bfloat16) else: self.buffer = paddle.zeros(size, dtype=dtype) - self.dev_id = 0 if paddle.get_device() == "cpu" else int( - paddle.get_device().split(":")[1]) + self.dev_id = ( + 0 + if paddle.get_device() == "cpu" + else int(paddle.get_device().split(":")[1]) + ) def to(self, device, dtype=None, keep_alignment=True): """ Move the underlying buffer """ - assert self.buffer is not None, "Cannot move a collapsed bucket, please rebuild it" - assert (dtype == Type.fp32.value - or Type.fp16.value), "Conversion type is not supported now" + assert ( + self.buffer is not None + ), "Cannot move a collapsed bucket, please rebuild it" + assert ( + dtype == Type.fp32.value or Type.fp16.value + ), "Conversion type is not supported now" if self._device != device: - tmp_buffer = self.buffer.cuda( - self.dev_id) if device == "gpu" else self.buffer.cpu() + tmp_buffer = ( + self.buffer.cuda(self.dev_id) + if device == "gpu" + else self.buffer.cpu() + ) for param in self._params: param.clear_gradient(False) @@ -107,17 +117,18 @@ def add_rank_params(self, trainable_params, param2align, convert_gpu=True): Add new parameters to the InternalStorage. Params becomes a view of this InternalStorage buffer. """ - assert all([ - id(param) not in self._param_ids for param in trainable_params - ]), "The same param cannot be checked in twice" + assert all( + [id(param) not in self._param_ids for param in trainable_params] + ), "The same param cannot be checked in twice" assert self.buffer is not None self.param2align = param2align cpu_param_shape = list() for param in trainable_params: - p_shape = self._add_param_as_view(param, param2align[param.name], - convert_gpu) + p_shape = self._add_param_as_view( + param, param2align[param.name], convert_gpu + ) cpu_param_shape.append(p_shape) if convert_gpu: @@ -127,8 +138,9 @@ def add_rank_params(self, trainable_params, param2align, convert_gpu=True): self._fill = 0 for idx, param in enumerate(trainable_params): - self._convert_buffer(param, cpu_param_shape[idx], - param2align[param.name]) + self._convert_buffer( + param, cpu_param_shape[idx], param2align[param.name] + ) self._params.append(param) self._param_ids.append(id(param)) @@ -138,7 +150,8 @@ def _add_param_as_view(self, param, align, convert_gpu=True): assert ( param.dtype == self.buffer.dtype ), "Different types for the InternalStorage and the param, cannot proceed: {} - {}".format( - param.dtype, self.buffer.dtype) + param.dtype, self.buffer.dtype + ) var_end = self._fill + param._numel() offset = var_end + align @@ -199,13 +212,9 @@ class GradStorage(InternalStorage): This is a basic class to simplify the handling of gradient InternalStorages """ - def __init__(self, - size, - dtype, - device, - destination, - parm2align, - convert_cpu=False): + def __init__( + self, size, dtype, device, destination, parm2align, convert_cpu=False + ): if isinstance(size, np.int64): size = size.tolist() super().__init__(size, dtype, device, convert_cpu) @@ -219,21 +228,21 @@ def __init__(self, self.sent = False def reset_checked_in(self): - """ Reset the counter of the parameter grads which have been checked in - """ + """Reset the counter of the parameter grads which have been checked in""" self.params_checked_in = 0 self.sent = False @property def all_checked_in(self): - """ Judge all the expected gradient check-in happened """ + """Judge all the expected gradient check-in happened""" return len(self._params) == self.params_checked_in def can_add_grad_view(self, param, align): - """ Is there enough InternalStorage to add this parameter gradient, and whether this param have already checked in. - """ - return self._fill + param._numel() + align <= self._max_size and id( - param) not in self._param_ids + """Is there enough InternalStorage to add this parameter gradient, and whether this param have already checked in.""" + return ( + self._fill + param._numel() + align <= self._max_size + and id(param) not in self._param_ids + ) def to(self, device, dtype=None, keep_alignment=True): """ @@ -253,9 +262,9 @@ def add_grad(self, param, align): Add a new parameter gradient to the InternalStorage. Param.grad becomes a view of this InternalStorage buffer. """ - assert id( - param - ) not in self._param_ids, "The same gradients cannot be checked in twice" + assert ( + id(param) not in self._param_ids + ), "The same gradients cannot be checked in twice" self._add_grad_as_view(param, align) self._params.append(param) @@ -302,8 +311,9 @@ def _array_grads(self): @paddle.autograd.no_grad() def _add_grad_as_view(self, param, align): - assert param._numel( - ) > 0, "Cannot add a gradient to a released InternalStorage, please rebuild" + assert ( + param._numel() > 0 + ), "Cannot add a gradient to a released InternalStorage, please rebuild" assert param.dtype == self.buffer.dtype grad_end = self._fill + param._numel() diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py index 0981b2cab76fd..4107bd83db936 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/group_sharded_utils.py @@ -39,13 +39,13 @@ class Type(Enum): """ Type of trainable parameters """ + fp16 = paddle.float16 bf16 = paddle.bfloat16 fp32 = paddle.float32 class GroupShardedClipGrad: - def __init__(self, clip, device, group): self._clip = clip self._device = device @@ -66,49 +66,64 @@ def _dygraph_clip(self, params_grads): merge_grad = g if g.type == core.VarDesc.VarType.SELECTED_ROWS: merge_grad = layers.get_tensor_from_selected_rows( - layers.merge_selected_rows(g)) + layers.merge_selected_rows(g) + ) square = layers.square(merge_grad) sum_square = layers.reduce_sum(square) if p.dtype == paddle.float16: - if p_slice: sum_square_fp16.append(sum_square) - else: unslice_params_fp16.append(sum_square) + if p_slice: + sum_square_fp16.append(sum_square) + else: + unslice_params_fp16.append(sum_square) elif p.dtype == paddle.float32: - if p_slice: sum_square_fp32.append(sum_square) - else: unslice_params_fp32.append(sum_square) + if p_slice: + sum_square_fp32.append(sum_square) + else: + unslice_params_fp32.append(sum_square) # global norm of non-distributed FP16 params_and_grads if len(sum_square_fp16) == 0: - global_norm_fp16 = paddle.to_tensor([0.], dtype=paddle.float32) + global_norm_fp16 = paddle.to_tensor([0.0], dtype=paddle.float32) else: global_norm_fp16 = layers.concat(sum_square_fp16) global_norm_fp16 = layers.reduce_sum(global_norm_fp16) - global_norm_fp16 = paddle.cast(global_norm_fp16, - dtype=paddle.float32) + global_norm_fp16 = paddle.cast( + global_norm_fp16, dtype=paddle.float32 + ) # global norm of non-distributed FP16 params_and_grads for unslice parameters if len(unslice_params_fp16) == 0: - global_unslice_fp16 = paddle.to_tensor([0.], dtype=paddle.float32) + global_unslice_fp16 = paddle.to_tensor([0.0], dtype=paddle.float32) else: global_unslice_fp16 = layers.concat(unslice_params_fp16) global_unslice_fp16 = layers.reduce_sum(global_unslice_fp16) - global_unslice_fp16 = paddle.cast(global_unslice_fp16, - dtype=paddle.float32) + global_unslice_fp16 = paddle.cast( + global_unslice_fp16, dtype=paddle.float32 + ) # global norm of non-distributed FP32 params_and_grads - global_norm_fp32 = layers.concat( - sum_square_fp32) if len(sum_square_fp32) != 0 else paddle.to_tensor( - [0.], dtype=paddle.float32) + global_norm_fp32 = ( + layers.concat(sum_square_fp32) + if len(sum_square_fp32) != 0 + else paddle.to_tensor([0.0], dtype=paddle.float32) + ) global_norm_fp32 = layers.reduce_sum(global_norm_fp32) # global norm of non-distributed FP32 params_and_grads for unslice parameters - global_unslice_fp32 = layers.concat(unslice_params_fp32) if len( - unslice_params_fp32) != 0 else paddle.to_tensor( - [0.], dtype=paddle.float32) + global_unslice_fp32 = ( + layers.concat(unslice_params_fp32) + if len(unslice_params_fp32) != 0 + else paddle.to_tensor([0.0], dtype=paddle.float32) + ) global_unslice_fp32 = layers.reduce_sum(global_unslice_fp32) global_unslice_var = global_unslice_fp16 + global_unslice_fp32 - global_norm_var = global_norm_fp16 + global_norm_fp32 + 1.0 / self._group.nranks * global_unslice_var + global_norm_var = ( + global_norm_fp16 + + global_norm_fp32 + + 1.0 / self._group.nranks * global_unslice_var + ) # add all reduce to get global norm of distributed params_and_grads dev_id = int(self._device.split(":")[1]) @@ -119,14 +134,14 @@ def _dygraph_clip(self, params_grads): paddle.distributed.all_reduce(global_norm_var, group=self._group) global_norm_var = layers.sqrt(global_norm_var) - max_global_norm = layers.fill_constant(shape=[1], - dtype=global_norm_var.dtype, - value=self.clip_norm) - - clip_var = layers.elementwise_div(x=max_global_norm, - y=layers.elementwise_max( - x=global_norm_var, - y=max_global_norm)) + max_global_norm = layers.fill_constant( + shape=[1], dtype=global_norm_var.dtype, value=self.clip_norm + ) + + clip_var = layers.elementwise_div( + x=max_global_norm, + y=layers.elementwise_max(x=global_norm_var, y=max_global_norm), + ) clip_var_fp16 = paddle.cast(clip_var, paddle.float16) for p, g in params_grads: @@ -165,7 +180,6 @@ def device_guard(dev_id=0, device="cpu"): @dygraph_only def GroupShardedScaler(scaler): - def unscale_method(self, optimizer): if not self._enable: return @@ -177,14 +191,16 @@ def unscale_method(self, optimizer): optimizer.update_scaler = True if getattr(optimizer._optim, '_param_groups', None) and isinstance( - optimizer._optim._param_groups[0], dict): + optimizer._optim._param_groups[0], dict + ): for group in optimizer._optim._param_groups: for param in group['params']: if param.grad is not None: param_grads.append(param.grad) if param.grad.dtype in [ - core.VarDesc.VarType.FP16, paddle.float16 + core.VarDesc.VarType.FP16, + paddle.float16, ]: param_grads_fp16.append(param.grad) else: @@ -194,7 +210,8 @@ def unscale_method(self, optimizer): if param.grad is not None: param_grads.append(param.grad) if param.grad.dtype in [ - core.VarDesc.VarType.FP16, paddle.float16 + core.VarDesc.VarType.FP16, + paddle.float16, ]: param_grads_fp16.append(param.grad) else: @@ -204,27 +221,34 @@ def unscale_method(self, optimizer): temp_found_inf_fp32 = to_variable(np.array([0]).astype(np.bool_)) device = "cpu" if optimizer.offload else "gpu" - dev_id = 0 if device == "cpu" else int( - paddle.get_device().split(":")[1]) + dev_id = ( + 0 if device == "cpu" else int(paddle.get_device().split(":")[1]) + ) with device_guard(dev_id, device): if len(param_grads_fp16): - _legacy_C_ops.check_finite_and_unscale(param_grads_fp16, - self._scale, - param_grads_fp16, - temp_found_inf_fp16) + _legacy_C_ops.check_finite_and_unscale( + param_grads_fp16, + self._scale, + param_grads_fp16, + temp_found_inf_fp16, + ) if len(param_grads_fp32): - _legacy_C_ops.check_finite_and_unscale(param_grads_fp32, - self._scale, - param_grads_fp32, - temp_found_inf_fp32) + _legacy_C_ops.check_finite_and_unscale( + param_grads_fp32, + self._scale, + param_grads_fp32, + temp_found_inf_fp32, + ) self._found_inf = 1 if temp_found_inf_fp16 or temp_found_inf_fp32 else 0 is_found_inf = paddle.to_tensor([self._found_inf], dtype="int32") - paddle.distributed.all_reduce(is_found_inf, - op=paddle.distributed.ReduceOp.MAX, - group=optimizer._group) + paddle.distributed.all_reduce( + is_found_inf, + op=paddle.distributed.ReduceOp.MAX, + group=optimizer._group, + ) self._found_inf = is_found_inf.numpy()[0] scaler._unscale = MethodType(unscale_method, scaler) diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage2.py b/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage2.py index 5933d11037eaf..63ce53f01c439 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage2.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage2.py @@ -35,7 +35,9 @@ from paddle.distributed.collective import _get_global_group from ...utils.internal_storage import GradStorage -from ...meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ShardingOptimizerStage2 +from ...meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ( + ShardingOptimizerStage2, +) from .sharding_utils import Taskflow, Type @@ -59,42 +61,55 @@ class ShardingStage2(nn.Layer): # 5. Support the establishment of independent communication groups. def __init__( - self, - layer, - sharding_optimizer, - group=None, - sync_buffers=False, - buffer_max_size=2**23, #8MB - auto_refresh_trainable=True, - device="gpu"): + self, + layer, + sharding_optimizer, + group=None, + sync_buffers=False, + buffer_max_size=2**23, # 8MB + auto_refresh_trainable=True, + device="gpu", + ): super().__init__() # training options self._layer = layer - self._sharding_optimizers = [ - sharding_optimizer - ] if not isinstance(sharding_optimizer, list) else sharding_optimizer + self._sharding_optimizers = ( + [sharding_optimizer] + if not isinstance(sharding_optimizer, list) + else sharding_optimizer + ) assert all( list( - map(lambda opt: isinstance(opt, ShardingOptimizerStage2), - self._sharding_optimizers)) + map( + lambda opt: isinstance(opt, ShardingOptimizerStage2), + self._sharding_optimizers, + ) + ) ), "Please use ShardingOptimizerStage2 optimizer" self._sync_buffers = sync_buffers self._auto_refresh_trainable = auto_refresh_trainable # Communication related attributes - self._group = dist.new_group( - _get_global_group().ranks) if group is None else group + self._group = ( + dist.new_group(_get_global_group().ranks) + if group is None + else group + ) self._world_size_scaling = 1.0 / self._group.nranks - assert self._group.nranks > 1, "Training must be distributed, ranks must be greater than 1" + assert ( + self._group.nranks > 1 + ), "Training must be distributed, ranks must be greater than 1" self._rank = self._group.rank self._global_root_rank = self._group.ranks[ - 0] # picking rank 0 as the reference + 0 + ] # picking rank 0 as the reference self._default_device = device # Global statistical parameters self._all_params = list( - chain(*[optim.local_params for optim in self._sharding_optimizers])) + chain(*[optim.local_params for optim in self._sharding_optimizers]) + ) self._trainable_params = [] self._grad_reduced = [] self._trainable_param2rank = {} @@ -103,11 +118,13 @@ def __init__( self._param_grads = [] # Set grad storage size & Display param sizes and model sizes - model_size = sum([np.prod(p.shape) - for p in self._layer.parameters()]).item() + model_size = sum( + [np.prod(p.shape) for p in self._layer.parameters()] + ).item() assert buffer_max_size >= 0, "buffer_max_size must be GE than 0." - self._buffer_max_size = self._rank_buffer_size(buffer_max_size, - model_size) + self._buffer_max_size = self._rank_buffer_size( + buffer_max_size, model_size + ) self._use_grad_storage = buffer_max_size > 0 self._grad_storages = {} # {dtype: {rank: GradStorage}} self._has_grad_storage = [] @@ -116,11 +133,12 @@ def __init__( # Offload # TODO(haohongxiang): Now it's not be supported for multi-optimizers using Offload strategy self._offload_optims = list( - filter(lambda optim: optim.offload, self._sharding_optimizers)) + filter(lambda optim: optim.offload, self._sharding_optimizers) + ) if len(self._offload_optims) > 0: - assert len( - self._sharding_optimizers - ) == 1, "Only support offload strategy for single optimizer" + assert ( + len(self._sharding_optimizers) == 1 + ), "Only support offload strategy for single optimizer" self._offload = self._sharding_optimizers[0].offload self._offload_device = "cpu" @@ -163,16 +181,19 @@ def forward(self, *inputs, **kwargs): return fw def set_state_dict(self, state_dict, use_structured_name=True): - self._layer.set_state_dict(state_dict, - use_structured_name=use_structured_name) - - def state_dict(self, - destination=None, - include_sublayers=True, - structured_name_prefix=""): - return self._layer.state_dict(destination=None, - include_sublayers=True, - structured_name_prefix="") + self._layer.set_state_dict( + state_dict, use_structured_name=use_structured_name + ) + + def state_dict( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + ): + return self._layer.state_dict( + destination=None, include_sublayers=True, structured_name_prefix="" + ) def _clear_gradients(self): """ @@ -180,8 +201,10 @@ def _clear_gradients(self): """ # Release grad storages for dtype in self._grad_storages.keys(): - if not self._offload and self._rank in self._grad_storages[ - dtype].keys(): + if ( + not self._offload + and self._rank in self._grad_storages[dtype].keys() + ): self._grad_storages[dtype][self._rank].buffer.zero_() # Release grads of params @@ -199,10 +222,13 @@ def _grad_scale(self): """ # Scale grad storages for dtype in self._grad_storages.keys(): - if not self._offload and self._rank in self._grad_storages[ - dtype].keys(): + if ( + not self._offload + and self._rank in self._grad_storages[dtype].keys() + ): self._grad_storages[dtype][self._rank].buffer.scale_( - scale=self._world_size_scaling) + scale=self._world_size_scaling + ) # Scale grads of params for param in self._trainable_params: @@ -213,7 +239,8 @@ def _grad_scale(self): # Scale grads of master params with offload strategy if self._offload: self._sharding_optimizers[0]._offload_scale_grad( - self._world_size_scaling) + self._world_size_scaling + ) def _init_internal_storage(self, needs_fresh): """ @@ -232,7 +259,9 @@ def to(self, device=None, dtype=None, blocking=True): Synchronously or asynchronously convert the data type of the layer, the device is not supported now. """ assert isinstance(device, str), "Device must be type str" - assert device == self._default_device, "New devices are not supported, because of the optimizer state is not sync" + assert ( + device == self._default_device + ), "New devices are not supported, because of the optimizer state is not sync" self._layer.to(device=device, dtype=dtype, blocking=blocking) @@ -240,14 +269,15 @@ def to(self, device=None, dtype=None, blocking=True): self._fresh_trainable() def _fresh_trainable(self): - """ Whether to update training parameters. """ + """Whether to update training parameters.""" # Make sure that this is not done while gradients are waiting to be reduced (if no_sync context for instance) if reduce(lambda x, y: x or y, self._grad_reduced, False): logging.warning("Grads waiting to be reduced.") self._trainable_params = list( - filter(lambda x: x.trainable, self._all_params)) + filter(lambda x: x.trainable, self._all_params) + ) self._trainable_params.sort(key=lambda x: np.prod(x.shape)) self._trainable_param2rank = {} @@ -257,14 +287,19 @@ def _fresh_trainable(self): optim.update_opt_status() # Get the parameters split by the optimizer according to rank - for per_rank_params in optim.dtype_rank_params.values( + for ( + per_rank_params + ) in ( + optim.dtype_rank_params.values() ): # all the params from all ranks for params in per_rank_params: for param in filter(lambda x: x.trainable, params): self._trainable_param2rank[ - param.name] = optim.param2rank[param.name] + param.name + ] = optim.param2rank[param.name] self._trainable_param2align[ - param.name] = optim._param2align[param.name] + param.name + ] = optim._param2align[param.name] self._setup_use_grad_storage() @@ -278,10 +313,9 @@ def __sync_buffers(self): """ for buffer in self._layer.buffers(include_sublayers=True): - dist.broadcast(buffer, - self._global_root_rank, - self._group, - sync_op=True) + dist.broadcast( + buffer, self._global_root_rank, self._group, sync_op=True + ) # Multi stream operation will be supported later dist.wait(tensor=buffer, group=self._group, use_calc_stream=True) @@ -315,7 +349,9 @@ def _get_reduce_fn(self, index, param, dst_rank): def reduce(*_): # Skip gradient reduction, do not change status information if self._grad_reduced[index]: - assert param.grad is not None, "Parameter gradient cannot be None" + assert ( + param.grad is not None + ), "Parameter gradient cannot be None" # Change reduce information self._grad_reduced[index] = False @@ -327,22 +363,29 @@ def cleanup(): elif self._offload: self._sharding_optimizers[0]._offload_acc_grad( param.name, - param.grad.cast(dtype=Type.fp32.value).cpu()) + param.grad.cast(dtype=Type.fp32.value).cpu(), + ) param.clear_gradient(False) # Synchronize the reduce parameter gradient self._tasks_flow.append( - Taskflow(task=dist.reduce( - tensor=param.grad, - dst=self._group.ranks[dst_rank], - group=self._group, - sync_op=True), - callback=cleanup)) + Taskflow( + task=dist.reduce( + tensor=param.grad, + dst=self._group.ranks[dst_rank], + group=self._group, + sync_op=True, + ), + callback=cleanup, + ) + ) # Multi stream operation will be supported later - dist.wait(tensor=param.grad, - group=self._group, - use_calc_stream=True) + dist.wait( + tensor=param.grad, + group=self._group, + use_calc_stream=True, + ) # Clear the task flow and trigger callback to clear the redundant gradient self._clear_task_flow() @@ -353,7 +396,9 @@ def cleanup(): def reduce(*_): # Skip gradient reduction, do not change status information if self._grad_reduced[index]: - assert param.grad is not None, "Parameter gradient cannot be None" + assert ( + param.grad is not None + ), "Parameter gradient cannot be None" # Change reduce information self._grad_reduced[index] = False @@ -370,35 +415,43 @@ def cleanup(): p.clear_gradient(False) p._gradient_set_empty(False) - grad_storage.buffer.value().get_tensor()._clear( - ) + grad_storage.buffer.value().get_tensor()._clear() elif self._offload: grad_storage.to(device=self._offload_device) for p in grad_storage._params: self._sharding_optimizers[ - 0]._offload_acc_grad( - p.name, - p.grad.cast(dtype=Type.fp32.value)) + 0 + ]._offload_acc_grad( + p.name, + p.grad.cast(dtype=Type.fp32.value), + ) p.clear_gradient(False) p._gradient_set_empty(False) grad_storage._device = self._default_device - grad_storage.buffer.value().get_tensor()._clear( - ) + grad_storage.buffer.value().get_tensor()._clear() # Reduce the bucket grad_storage.sent = True self._tasks_flow.append( - Taskflow(task=dist.reduce( - tensor=grad_storage.buffer, - dst=self._group.ranks[grad_storage.destination], - group=self._group, - sync_op=True), - callback=cleanup)) + Taskflow( + task=dist.reduce( + tensor=grad_storage.buffer, + dst=self._group.ranks[ + grad_storage.destination + ], + group=self._group, + sync_op=True, + ), + callback=cleanup, + ) + ) # Multi stream operation will be supported later - dist.wait(tensor=grad_storage.buffer, - group=self._group, - use_calc_stream=True) + dist.wait( + tensor=grad_storage.buffer, + group=self._group, + use_calc_stream=True, + ) # Clear the task flow and trigger callback to clear the redundant gradient self._clear_task_flow() @@ -424,7 +477,8 @@ def _setup_backward_hooks(self): reduce_function = self._get_reduce_fn(index, param, dst_rank) self._bw_hooks.append( - param._register_backward_hook(reduce_function)) + param._register_backward_hook(reduce_function) + ) def _setup_use_grad_storage(self): """ @@ -447,27 +501,36 @@ def _setup_use_grad_storage(self): dtype=param.dtype, device=self._default_device, destination=dst_rank, - parm2align=self._trainable_param2align) + parm2align=self._trainable_param2align, + ) # Criteria to decide whether this parameter is to be put in GradStorage if self._grad_storages[param.dtype][dst_rank].can_add_grad_view( - param, self._trainable_param2align[param.name]): + param, self._trainable_param2align[param.name] + ): self._grad_storages[param.dtype][dst_rank].add_grad( - param, self._trainable_param2align[param.name]) + param, self._trainable_param2align[param.name] + ) self._has_grad_storage[index] = True else: self._param_grads.append(param.name) print( - "Can not add param: {}, param's shape: {}, param align: {}, grad_storages fill: {}, " - .format(param.name, param.shape, - self._trainable_param2align[param.name], - self._grad_storages[param.dtype][dst_rank]._fill)) + "Can not add param: {}, param's shape: {}, param align: {}, grad_storages fill: {}, ".format( + param.name, + param.shape, + self._trainable_param2align[param.name], + self._grad_storages[param.dtype][dst_rank]._fill, + ) + ) self._grad_storage_list = list( - chain(*[ - self._grad_storages[dtype].values() - for dtype in self._grad_storages.keys() - ])) + chain( + *[ + self._grad_storages[dtype].values() + for dtype in self._grad_storages.keys() + ] + ) + ) def _clear_task_flow(self): """Try to consume the previous tasks.""" @@ -518,15 +581,19 @@ def _rank_buffer_size(self, buffer_max_size, model_size): if Type.fp16.value in rank_buffer_size.keys(): # FP16 GradStorage and model size print( - "====== FP16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======" - .format(rank_buffer_size[Type.fp16.value] / 2**19, - model_size / 2**19)) + "====== FP16 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======".format( + rank_buffer_size[Type.fp16.value] / 2**19, + model_size / 2**19, + ) + ) if Type.fp32.value in rank_buffer_size.keys(): # FP32 GradStorage and model size print( - "====== FP32 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======" - .format(rank_buffer_size[Type.fp32.value] / 2**18, - model_size / 2**18)) + "====== FP32 GradStorage size: {:.2f}M parameters, Model size {:.2f}M parameters ======".format( + rank_buffer_size[Type.fp32.value] / 2**18, + model_size / 2**18, + ) + ) return rank_buffer_size def _redefine_opt_step(self): diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage3.py b/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage3.py index 02e701e8990db..deae0cddd291c 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage3.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_stage3.py @@ -59,16 +59,18 @@ class ShardingStage3(nn.Layer): # 3. Support offload function. # 4. Support the establishment of independent communication groups. - def __init__(self, - layer, - optimizer, - group=None, - sync_buffers=False, - device="gpu", - segment_size=2**15, - pertrain_sync_models=True, - offload=False, - sync_comm=False): + def __init__( + self, + layer, + optimizer, + group=None, + sync_buffers=False, + device="gpu", + segment_size=2**15, + pertrain_sync_models=True, + offload=False, + sync_comm=False, + ): super().__init__() # Default configs @@ -83,28 +85,41 @@ def __init__(self, self._segment_size = segment_size global DEV - DEV = "cpu" if paddle.get_device() == "cpu" else paddle.get_device( - ).split(":")[0] + DEV = ( + "cpu" + if paddle.get_device() == "cpu" + else paddle.get_device().split(":")[0] + ) global DEV_ID - DEV_ID = 0 if paddle.get_device() == "cpu" else int( - paddle.get_device().split(":")[1]) + DEV_ID = ( + 0 + if paddle.get_device() == "cpu" + else int(paddle.get_device().split(":")[1]) + ) global param2dtype param2dtype = dict() # Communication group establishment - self._group = dist.new_group( - _get_global_group().ranks) if group is None else group + self._group = ( + dist.new_group(_get_global_group().ranks) + if group is None + else group + ) self._world_size_scaling = 1.0 / self._group.nranks - assert self._group.nranks > 1, "Training must be distributed, ranks must be greater than 1." + assert ( + self._group.nranks > 1 + ), "Training must be distributed, ranks must be greater than 1." self._rank = self._group.rank self._global_root_rank = self._group.ranks[ - 0] # picking rank 0 as the reference + 0 + ] # picking rank 0 as the reference self._global_ranks = self._group.ranks # Parameter segmentation for global ranks # After flatten -> self._param2buffer_size, self._param2buffer, self._trainable_params self._param2buffer_size = dict() # {param.name: size} - self._param2buffer = dict( + self._param2buffer = ( + dict() ) # {param.name: [(start0, end0),(start1, end1), ...]} self._trainable_params = dict() # {id(layer): [trainable_params]} self._unslice_params = set() # param's numel <= segment_size @@ -112,9 +127,11 @@ def __init__(self, self._grad_storages = dict() # {param.dtype: GradStorage} assert not isinstance( - optimizer, list), "Multiple optimizers are not supported now." - self._optim = _OptimizerWrapper(optimizer, self._offload, self._group, - self._update_params_slice) + optimizer, list + ), "Multiple optimizers are not supported now." + self._optim = _OptimizerWrapper( + optimizer, self._offload, self._group, self._update_params_slice + ) self._ori_parameter_list = self._optim._parameter_list self._ori_param_groups = self._optim._param_groups @@ -123,9 +140,9 @@ def __init__(self, logging.warning( "While using ClipGradByGlobalNorm in ShardingStage3, the grad clip of original optimizer will be changed." ) - self._optim._grad_clip = ShardingClipGrad(self._optim._grad_clip, - paddle.get_device(), - self._group) + self._optim._grad_clip = ShardingClipGrad( + self._optim._grad_clip, paddle.get_device(), self._group + ) # Synchronous all ranks models if pertrain_sync_models: @@ -161,10 +178,9 @@ def _sync_params_and_buffers(self): """ for p in self._layer.parameters(): - dist.broadcast(p, - src=self._global_root_rank, - group=self._group, - sync_op=True) + dist.broadcast( + p, src=self._global_root_rank, group=self._group, sync_op=True + ) # Multi stream operation will be supported later dist.wait(tensor=p, group=self._group, use_calc_stream=True) @@ -174,12 +190,15 @@ def _clear_gradients(self): current_layer_params = self._layer.parameters(include_sublayers=True) # 1.Handle param's slice trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) for param in trainable_params: - assert hasattr(param, "fw_storage" - ), "Find {} don't have fw_storage attribute.".format( - param.name) + assert hasattr( + param, "fw_storage" + ), "Find {} don't have fw_storage attribute.".format(param.name) param.fw_storage.clear_gradient(False) param.fw_storage._gradient_set_empty(False) @@ -195,8 +214,10 @@ def _clear_gradients(self): param._gradient_set_empty(False) tmp_var = param.cuda(DEV_ID) - if tmp_var.dtype == Type.fp32.value and param2dtype[ - param.name] == Type.fp16.value: + if ( + tmp_var.dtype == Type.fp32.value + and param2dtype[param.name] == Type.fp16.value + ): tmp_var = paddle.cast(tmp_var, Type.fp16.value) tmp_var._share_buffer_to(param) tmp_var._clear() @@ -211,9 +232,11 @@ def _update_params_slice(self): if not isinstance(self._optim._param_groups[0], dict): slice_params = [param.fw_storage for param in update_list] self._optim._parameter_list = slice_params + list( - self._unslice_params) + self._unslice_params + ) self._optim._param_groups = slice_params + list( - self._unslice_params) + self._unslice_params + ) else: for param_group in self._optim._param_groups: p_group = [] @@ -239,16 +262,19 @@ def forward(self, *inputs, **kwargs): return fw def set_state_dict(self, state_dict, use_structured_name=True): - self._layer.set_state_dict(state_dict, - use_structured_name=use_structured_name) - - def state_dict(self, - destination=None, - include_sublayers=True, - structured_name_prefix=""): - return self._layer.state_dict(destination=None, - include_sublayers=True, - structured_name_prefix="") + self._layer.set_state_dict( + state_dict, use_structured_name=use_structured_name + ) + + def state_dict( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + ): + return self._layer.state_dict( + destination=None, include_sublayers=True, structured_name_prefix="" + ) def _handle_unslice_params(self): buffer_size = dict() @@ -258,7 +284,8 @@ def _handle_unslice_params(self): # Updata optimizer master weights if param.dtype == Type.fp16.value and not self._offload: self._optim._master_weights[param.name] = paddle.cast( - param, Type.fp32.value) + param, Type.fp32.value + ) if self._offload: param.master_weight = paddle.cast(param, Type.fp32.value).cpu() param2dtype[param.name] = param.dtype @@ -274,9 +301,11 @@ def _handle_unslice_params(self): dtype=param.dtype, device=self._default_device, destination=self._rank, - parm2align=self._unslice_params2align) + parm2align=self._unslice_params2align, + ) self._grad_storages[param.dtype].add_grad( - param, self._unslice_params2align[param.name]) + param, self._unslice_params2align[param.name] + ) def _segment_rank_params(self, layer, name="last_layer"): """ @@ -316,8 +345,11 @@ def _add_manage_info(trainable_param): align_ = self._param2align(param) offset = align_ + param._numel() - buffer_size = offset if offset % self._group.nranks == 0 else offset + self._group.nranks - ( - offset % self._group.nranks) + buffer_size = ( + offset + if offset % self._group.nranks == 0 + else offset + self._group.nranks - (offset % self._group.nranks) + ) self._param2buffer_size[param.name] = buffer_size # 2.Combination param buffer @@ -326,7 +358,8 @@ def _add_manage_info(trainable_param): for rank_ in range(self._group.nranks): self._param2buffer[param.name].append( - (rank_ * pre_buffer, (rank_ + 1) * pre_buffer)) + (rank_ * pre_buffer, (rank_ + 1) * pre_buffer) + ) # Record param's dtype param2dtype[param.name] = param.dtype @@ -339,10 +372,11 @@ def _param_storage(self, param, buffer_size): This is a function to simplify the handling of parameter InternalStorages. """ assert isinstance(buffer_size, int) - value = np.zeros( - buffer_size, - dtype=np.float16) if Type.fp16.value == param.dtype else np.zeros( - buffer_size, dtype=np.float32) + value = ( + np.zeros(buffer_size, dtype=np.float16) + if Type.fp16.value == param.dtype + else np.zeros(buffer_size, dtype=np.float32) + ) buffer = core.VarBase(value=value, place=core.CPUPlace()) param_shape = param.shape @@ -353,30 +387,37 @@ def _param_storage(self, param, buffer_size): start, end = self._param2buffer[param.name][self._rank] # Copy the current param value - tmp_var = core.VarBase(tensor=buffer._slice(0, param._numel()), - place=core.CPUPlace()) + tmp_var = core.VarBase( + tensor=buffer._slice(0, param._numel()), place=core.CPUPlace() + ) param_cpu = param.cpu() - tmp_var.value().get_tensor().set(param_cpu.value().get_tensor(), - core.CPUPlace()) + tmp_var.value().get_tensor().set( + param_cpu.value().get_tensor(), core.CPUPlace() + ) param.value().get_tensor()._set_dims(param_shape) # Current rank param_storage if self._offload: - param.fw_storage = core.VarBase(buffer._slice(start, end), - core.CPUPlace(), - "slice@" + param.name) + param.fw_storage = core.VarBase( + buffer._slice(start, end), + core.CPUPlace(), + "slice@" + param.name, + ) with device_guard(device="cpu"): - param.master_weight = paddle.cast(param.fw_storage, - Type.fp32.value) + param.master_weight = paddle.cast( + param.fw_storage, Type.fp32.value + ) else: - param.fw_storage = core.VarBase(buffer._slice(start, end), - "slice@" + param.name) + param.fw_storage = core.VarBase( + buffer._slice(start, end), "slice@" + param.name + ) param.status = "part" # Updata optimizer master weights if param.dtype == Type.fp16.value and not self._offload: self._optim._master_weights[param.fw_storage.name] = paddle.cast( - param.fw_storage, Type.fp32.value) + param.fw_storage, Type.fp32.value + ) param._clear() def _register_forward_hooks(self, layer): @@ -398,20 +439,33 @@ def _register_forward_hooks(self, layer): self._register_forward_hooks(sub_layer) def _register_forward_all_hooks(self, sub_layer, task_flow): - def _forward_pre_hook(layer, inputs): - return ForwardPreHooks(layer, self._order_tracer, - self._trainable_params, self._param2buffer, - self._rank, self._group, self._sync_comm, - self._offload, task_flow) + return ForwardPreHooks( + layer, + self._order_tracer, + self._trainable_params, + self._param2buffer, + self._rank, + self._group, + self._sync_comm, + self._offload, + task_flow, + ) def _forward_post_hook(layer, inputs, outputs): - return ForwardPostHooks.apply(outputs, layer, self._order_tracer, - self._trainable_params, - self._param2buffer, - self._param2buffer_size, self._rank, - self._group, self._sync_comm, - self._offload, task_flow) + return ForwardPostHooks.apply( + outputs, + layer, + self._order_tracer, + self._trainable_params, + self._param2buffer, + self._param2buffer_size, + self._rank, + self._group, + self._sync_comm, + self._offload, + task_flow, + ) # register previous forward hooks sub_layer.register_forward_pre_hook(_forward_pre_hook) @@ -426,10 +480,9 @@ def _sync_buffers(self): """ for buffer in self._layer.buffers(include_sublayers=True): - dist.broadcast(buffer, - self._global_root_rank, - self._group, - sync_op=True) + dist.broadcast( + buffer, self._global_root_rank, self._group, sync_op=True + ) # Multi stream operation will be supported later dist.wait(tensor=buffer, group=self._group, use_calc_stream=True) @@ -448,14 +501,16 @@ def _update_params(self): assert len(self._trainable_params.keys()) > 0 current_layer_params = self._layer.parameters(include_sublayers=True) trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) # 1.Handle param's slice for param in trainable_params: assert hasattr( - param, - "fw_storage"), "Find {} don't have fw_storage attribute".format( - param.name) + param, "fw_storage" + ), "Find {} don't have fw_storage attribute".format(param.name) # Gradient average if self._offload: with device_guard(device="cpu"): @@ -470,12 +525,14 @@ def _update_params(self): # 2.Handle unslice param for grad_storage in self._grad_storages.values(): grad_storage.buffer.scale_(scale=self._world_size_scaling) - dist.all_reduce(tensor=grad_storage.buffer, - group=self._group, - sync_op=True) - dist.wait(tensor=grad_storage.buffer, - group=self._group, - use_calc_stream=True) + dist.all_reduce( + tensor=grad_storage.buffer, group=self._group, sync_op=True + ) + dist.wait( + tensor=grad_storage.buffer, + group=self._group, + use_calc_stream=True, + ) if self._offload: for param in list(self._unslice_params): @@ -500,15 +557,20 @@ def get_all_parameters(self, convert2cpu=False): assert len(self._trainable_params.keys()) > 0 current_layer_params = self._layer.parameters(include_sublayers=True) trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) - t_flow = _allgather_buffer(trainable_params, - self._group, - use_calc_stream=True, - task_flow=TaskFlow(), - sync_wait=True, - offload=self._offload, - convert2cpu=convert2cpu) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) + t_flow = _allgather_buffer( + trainable_params, + self._group, + use_calc_stream=True, + task_flow=TaskFlow(), + sync_wait=True, + offload=self._offload, + convert2cpu=convert2cpu, + ) if convert2cpu: for param in trainable_params: t_flow.full_param[param.name]._share_buffer_to(param) @@ -519,49 +581,60 @@ def get_all_parameters(self, convert2cpu=False): def _register_backward_hooks(self): current_layer_params = self._layer.parameters(include_sublayers=True) trainable_params = list( - filter(lambda p: p.trainable and p not in self._unslice_params, - current_layer_params)) + filter( + lambda p: p.trainable and p not in self._unslice_params, + current_layer_params, + ) + ) for param in trainable_params: allreduce_function = self._get_allreduce_fn(param) param._register_backward_hook(allreduce_function) def _get_allreduce_fn(self, param): - @paddle.autograd.no_grad() def allreduce_(*_): if param.name in self._task_flow.full_grad.keys(): full_grad = self._task_flow.full_grad[param.name] # Only support sync allreduce current rank's layer now - dist.all_reduce(tensor=full_grad, - group=self._group, - sync_op=True) - dist.wait(tensor=full_grad, - group=self._group, - use_calc_stream=True) + dist.all_reduce( + tensor=full_grad, group=self._group, sync_op=True + ) + dist.wait( + tensor=full_grad, group=self._group, use_calc_stream=True + ) start, end = self._param2buffer[param.name][self._rank] if param.bw_storage is None: - param.bw_storage = core.VarBase(full_grad._slice( - start, end)).detach().clone() + param.bw_storage = ( + core.VarBase(full_grad._slice(start, end)) + .detach() + .clone() + ) if self._offload: param.bw_storage = _device2cpu(param.bw_storage, True) else: if self._offload: cpu_grad = _device2cpu( - core.VarBase(full_grad._slice( - start, end)).detach().clone(), True) + core.VarBase(full_grad._slice(start, end)) + .detach() + .clone(), + True, + ) with device_guard(device="cpu"): param.bw_storage = paddle.add( - param.bw_storage, cpu_grad) + param.bw_storage, cpu_grad + ) else: # param.bw_storage.add_( # core.VarBase(full_grad._slice(start, end)) # .detach().clone()) param.bw_storage = paddle.add( param.bw_storage, - core.VarBase(full_grad._slice( - start, end)).detach().clone()) + core.VarBase(full_grad._slice(start, end)) + .detach() + .clone(), + ) param.clear_gradient(False) param._gradient_set_empty(False) tmp_var = self._task_flow.full_grad.pop(param.name) @@ -572,10 +645,16 @@ def allreduce_(*_): param.use_count = 0 param._clear() start, end = self._param2buffer[param.name][self._rank] - param.fw_storage = core.VarBase( - self._task_flow.full_param[param.name]._slice( - start, end), - param.name + "@slice").detach().clone() + param.fw_storage = ( + core.VarBase( + self._task_flow.full_param[param.name]._slice( + start, end + ), + param.name + "@slice", + ) + .detach() + .clone() + ) param.status = "part" tmp_var = self._task_flow.full_param.pop(param.name) tmp_var._clear() @@ -590,8 +669,9 @@ def _param2align(self, param): # CUDA alignment 256 bytes size = param._numel() * align[param.dtype] remaining = size % alignment[self._default_device] - ali = 0 if remaining == 0 else alignment[ - self._default_device] - remaining + ali = ( + 0 if remaining == 0 else alignment[self._default_device] - remaining + ) align_ = ali // align[param.dtype] return align_ @@ -625,8 +705,17 @@ def _opt_clear(self): self._optim.clear_grad = MethodType(_opt_clear, self._optim) -def ForwardPreHooks(layer, order_tracer, trainable_params, param2buffer, rank, - group, sync_comm, offload, task_flow): +def ForwardPreHooks( + layer, + order_tracer, + trainable_params, + param2buffer, + rank, + group, + sync_comm, + offload, + task_flow, +): # Record layer's id layer_id = id(layer) @@ -641,34 +730,49 @@ def ForwardPreHooks(layer, order_tracer, trainable_params, param2buffer, rank, # Whether to use calc stream task_flow.use_calc[layer_id] = use_calc # wait current layer params - _wait_layer(trainable_params[layer_id], task_flow, group, use_calc, - offload) + _wait_layer( + trainable_params[layer_id], task_flow, group, use_calc, offload + ) - if layer_id == order_tracer["layer"][-1]: return + if layer_id == order_tracer["layer"][-1]: + return order_ = order_tracer[layer_id] layer_id = order_tracer["layer"][order_ + 1] - _allgather_buffer(trainable_params[layer_id], - group, - use_calc_stream=use_calc, - task_flow=task_flow, - sync_wait=sync_wait, - offload=offload) + _allgather_buffer( + trainable_params[layer_id], + group, + use_calc_stream=use_calc, + task_flow=task_flow, + sync_wait=sync_wait, + offload=offload, + ) return class ForwardPostHooks(PyLayer): - @staticmethod - def forward(ctx, inputs, layer, order_tracer, trainable_params, - param2buffer, param2buffer_size, rank, group, sync_comm, - offload, task_flow): + def forward( + ctx, + inputs, + layer, + order_tracer, + trainable_params, + param2buffer, + param2buffer_size, + rank, + group, + sync_comm, + offload, + task_flow, + ): layer_id = id(layer) # release current layer full params - _release_param(trainable_params[layer_id], param2buffer, rank, - task_flow, offload) + _release_param( + trainable_params[layer_id], param2buffer, rank, task_flow, offload + ) if layer_id not in order_tracer.keys(): order_ = order_tracer["order"] @@ -676,7 +780,7 @@ def forward(ctx, inputs, layer, order_tracer, trainable_params, order_tracer["order"] += 1 order_tracer["layer"].append(layer_id) - #Record bw info + # Record bw info ctx.order_tracer = order_tracer ctx.task_flow = task_flow ctx.group = group @@ -705,30 +809,36 @@ def backward(ctx, *args): # Allgather params synchronization if sync_comm: use_calc, sync_wait = True, True - _allgather_buffer(trainable_params[layer_id], - group, - use_calc_stream=use_calc, - task_flow=task_flow, - sync_wait=sync_wait, - offload=offload) + _allgather_buffer( + trainable_params[layer_id], + group, + use_calc_stream=use_calc, + task_flow=task_flow, + sync_wait=sync_wait, + offload=offload, + ) else: - _wait_layer(trainable_params[layer_id], task_flow, group, use_calc, - offload) + _wait_layer( + trainable_params[layer_id], task_flow, group, use_calc, offload + ) # Create params's grad - _create_params_grad(trainable_params[layer_id], param2buffer_size, - task_flow) + _create_params_grad( + trainable_params[layer_id], param2buffer_size, task_flow + ) # Whether to use calc stream task_flow.use_calc[layer_id] = use_calc if layer_id != order_tracer["layer"][0] and not sync_comm: layer_next_id = order_tracer["layer"][order_tracer[layer_id] - 1] - _allgather_buffer(trainable_params[layer_next_id], - group, - use_calc_stream=use_calc, - task_flow=task_flow, - sync_wait=sync_wait, - offload=offload) + _allgather_buffer( + trainable_params[layer_next_id], + group, + use_calc_stream=use_calc, + task_flow=task_flow, + sync_wait=sync_wait, + offload=offload, + ) return args @@ -738,22 +848,22 @@ class TaskFlow: Task flows, one way linked list for task acquisition. """ - def __init__(self, - full_param=dict(), - full_grad=dict(), - use_calc=dict(), - callback=None): + def __init__( + self, + full_param=dict(), + full_grad=dict(), + use_calc=dict(), + callback=None, + ): self.full_param = full_param self.full_grad = full_grad self.use_calc = use_calc self.callback = callback -def _release_param(trainable_params, - param2buffer, - rank, - task_flow, - offload=False): +def _release_param( + trainable_params, param2buffer, rank, task_flow, offload=False +): for param in trainable_params: # async communicate share weight not clear param.use_count -= 1 @@ -762,9 +872,14 @@ def _release_param(trainable_params, if param.name in task_flow.full_param.keys(): start, end = param2buffer[param.name][rank] with paddle.amp.auto_cast(enable=False): - param.fw_storage = core.VarBase( - task_flow.full_param[param.name]._slice(start, end), - param.name + "@slice").detach().clone() + param.fw_storage = ( + core.VarBase( + task_flow.full_param[param.name]._slice(start, end), + param.name + "@slice", + ) + .detach() + .clone() + ) param.status = "part" tmp_var = task_flow.full_param.pop(param.name) tmp_var._clear() @@ -774,11 +889,9 @@ def _release_param(trainable_params, return -def _wait_layer(trainable_params, - task_flow, - group, - use_calc_stream, - offload=False): +def _wait_layer( + trainable_params, task_flow, group, use_calc_stream, offload=False +): paddle.device.cuda.synchronize() for param in trainable_params: if param.status == "all": @@ -786,30 +899,35 @@ def _wait_layer(trainable_params, continue if param.name in task_flow.full_param.keys(): full_param = task_flow.full_param[param.name] - core.VarBase(full_param._slice( - 0, param._numel()))._share_buffer_to(param) + core.VarBase(full_param._slice(0, param._numel()))._share_buffer_to( + param + ) param.fw_storage._clear() param.fw_storage = None param.status = "all" param.use_count += 1 else: - _allgather_buffer(trainable_params, - group, - use_calc_stream=True, - task_flow=task_flow, - sync_wait=True, - offload=offload) + _allgather_buffer( + trainable_params, + group, + use_calc_stream=True, + task_flow=task_flow, + sync_wait=True, + offload=offload, + ) break return task_flow -def _allgather_buffer(trainable_params, - group, - use_calc_stream, - task_flow, - sync_wait=False, - offload=False, - convert2cpu=False): +def _allgather_buffer( + trainable_params, + group, + use_calc_stream, + task_flow, + sync_wait=False, + offload=False, + convert2cpu=False, +): for param in trainable_params: if param.status == "all": @@ -820,18 +938,21 @@ def _allgather_buffer(trainable_params, param.fw_storage = _cpu2device(param) with paddle.amp.auto_cast(enable=False): - full_param = _all_gather(param.fw_storage, - group, - use_calc_stream=use_calc_stream) + full_param = _all_gather( + param.fw_storage, group, use_calc_stream=use_calc_stream + ) # Allgather current layer in the 1st step synchronously if sync_wait: with paddle.amp.auto_cast(enable=False): - dist.wait(tensor=full_param, - group=group, - use_calc_stream=use_calc_stream) - core.VarBase(full_param._slice( - 0, param._numel()))._share_buffer_to(param) + dist.wait( + tensor=full_param, + group=group, + use_calc_stream=use_calc_stream, + ) + core.VarBase(full_param._slice(0, param._numel()))._share_buffer_to( + param + ) param.fw_storage._clear() param.fw_storage = None param.status = "all" @@ -855,10 +976,12 @@ def _create_params_grad(trainable_params, param2buffer_size, task_flow): if param.name in task_flow.full_grad.keys(): continue assert isinstance(param2buffer_size[param.name], int) - temp_grad = paddle.zeros([param2buffer_size[param.name]], - dtype=param.dtype) + temp_grad = paddle.zeros( + [param2buffer_size[param.name]], dtype=param.dtype + ) param._copy_gradient_from( - core.VarBase(temp_grad._slice(0, param._numel()))) + core.VarBase(temp_grad._slice(0, param._numel())) + ) task_flow.full_grad[param.name] = temp_grad return task_flow @@ -882,13 +1005,14 @@ def _UnsliceParam(param): def _VarBaseWrapper(param): varbase = param.fw_storage - tmp_param = ParamBase(shape=varbase.shape, - dtype=varbase.dtype, - name="slice@" + param.name) + tmp_param = ParamBase( + shape=varbase.shape, dtype=varbase.dtype, name="slice@" + param.name + ) varbase._share_buffer_to(tmp_param) tmp_param.regularizer = param.regularizer tmp_param.optimize_attr['learning_rate'] = param.optimize_attr[ - 'learning_rate'] + 'learning_rate' + ] varbase._clear() return tmp_param @@ -913,14 +1037,17 @@ def _device2cpu(trans_param, convert_dtype=False): def _cpu2device(param): tmp_p = param.fw_storage.cuda(DEV_ID) - if tmp_p.dtype == Type.fp32.value and param2dtype[ - param.name] == Type.fp16.value: + if ( + tmp_p.dtype == Type.fp32.value + and param2dtype[param.name] == Type.fp16.value + ): tmp_p = paddle.cast(tmp_p, Type.fp16.value) return tmp_p def _current_layer_params(layer): - return layer.parameters( - include_sublayers=False) + list(layer.extra_parameters) if hasattr( - layer, "extra_parameters") else layer.parameters( - include_sublayers=False) + return ( + layer.parameters(include_sublayers=False) + list(layer.extra_parameters) + if hasattr(layer, "extra_parameters") + else layer.parameters(include_sublayers=False) + ) diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_utils.py b/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_utils.py index 0e7725e3e21f8..07cf159c3e66f 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_utils.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding/sharding_utils.py @@ -40,13 +40,13 @@ class Type(Enum): """ Type of trainable parameters """ + fp16 = paddle.float16 bf16 = paddle.bfloat16 fp32 = paddle.float32 class ShardingClipGrad: - def __init__(self, clip, device, group): self._clip = clip self._device = device @@ -67,49 +67,64 @@ def _dygraph_clip(self, params_grads): merge_grad = g if g.type == core.VarDesc.VarType.SELECTED_ROWS: merge_grad = layers.get_tensor_from_selected_rows( - layers.merge_selected_rows(g)) + layers.merge_selected_rows(g) + ) square = layers.square(merge_grad) sum_square = layers.reduce_sum(square) if p.dtype == paddle.float16: - if p_slice: sum_square_fp16.append(sum_square) - else: unslice_params_fp16.append(sum_square) + if p_slice: + sum_square_fp16.append(sum_square) + else: + unslice_params_fp16.append(sum_square) elif p.dtype == paddle.float32: - if p_slice: sum_square_fp32.append(sum_square) - else: unslice_params_fp32.append(sum_square) + if p_slice: + sum_square_fp32.append(sum_square) + else: + unslice_params_fp32.append(sum_square) # global norm of non-distributed FP16 params_and_grads if len(sum_square_fp16) == 0: - global_norm_fp16 = paddle.to_tensor([0.], dtype=paddle.float32) + global_norm_fp16 = paddle.to_tensor([0.0], dtype=paddle.float32) else: global_norm_fp16 = layers.concat(sum_square_fp16) global_norm_fp16 = layers.reduce_sum(global_norm_fp16) - global_norm_fp16 = paddle.cast(global_norm_fp16, - dtype=paddle.float32) + global_norm_fp16 = paddle.cast( + global_norm_fp16, dtype=paddle.float32 + ) # global norm of non-distributed FP16 params_and_grads for unslice parameter if len(unslice_params_fp16) == 0: - global_unslice_fp16 = paddle.to_tensor([0.], dtype=paddle.float32) + global_unslice_fp16 = paddle.to_tensor([0.0], dtype=paddle.float32) else: global_unslice_fp16 = layers.concat(unslice_params_fp16) global_unslice_fp16 = layers.reduce_sum(global_unslice_fp16) - global_unslice_fp16 = paddle.cast(global_unslice_fp16, - dtype=paddle.float32) + global_unslice_fp16 = paddle.cast( + global_unslice_fp16, dtype=paddle.float32 + ) # global norm of non-distributed FP32 params_and_grads - global_norm_fp32 = layers.concat( - sum_square_fp32) if len(sum_square_fp32) != 0 else paddle.to_tensor( - [0.], dtype=paddle.float32) + global_norm_fp32 = ( + layers.concat(sum_square_fp32) + if len(sum_square_fp32) != 0 + else paddle.to_tensor([0.0], dtype=paddle.float32) + ) global_norm_fp32 = layers.reduce_sum(global_norm_fp32) # global norm of non-distributed FP32 params_and_grads for unslice parameter - global_unslice_fp32 = layers.concat(unslice_params_fp32) if len( - unslice_params_fp32) != 0 else paddle.to_tensor( - [0.], dtype=paddle.float32) + global_unslice_fp32 = ( + layers.concat(unslice_params_fp32) + if len(unslice_params_fp32) != 0 + else paddle.to_tensor([0.0], dtype=paddle.float32) + ) global_unslice_fp32 = layers.reduce_sum(global_unslice_fp32) global_unslice_var = global_unslice_fp16 + global_unslice_fp32 - global_norm_var = global_norm_fp16 + global_norm_fp32 + 1.0 / self._group.nranks * global_unslice_var + global_norm_var = ( + global_norm_fp16 + + global_norm_fp32 + + 1.0 / self._group.nranks * global_unslice_var + ) # add all reduce to get global norm of distributed params_and_grads dev_id = int(self._device.split(":")[1]) @@ -117,14 +132,14 @@ def _dygraph_clip(self, params_grads): paddle.distributed.all_reduce(global_norm_var, group=self._group) global_norm_var = layers.sqrt(global_norm_var) - max_global_norm = layers.fill_constant(shape=[1], - dtype=global_norm_var.dtype, - value=self.clip_norm) - - clip_var = layers.elementwise_div(x=max_global_norm, - y=layers.elementwise_max( - x=global_norm_var, - y=max_global_norm)) + max_global_norm = layers.fill_constant( + shape=[1], dtype=global_norm_var.dtype, value=self.clip_norm + ) + + clip_var = layers.elementwise_div( + x=max_global_norm, + y=layers.elementwise_max(x=global_norm_var, y=max_global_norm), + ) clip_var_fp16 = paddle.cast(clip_var, paddle.float16) for p, g in params_grads: @@ -163,7 +178,6 @@ def device_guard(dev_id=0, device="cpu"): @dygraph_only def ShardingScaler(scaler): - def unscale_method(self, optimizer): if not self._enable: return @@ -175,14 +189,16 @@ def unscale_method(self, optimizer): optimizer.update_scaler = True if getattr(optimizer._optim, '_param_groups', None) and isinstance( - optimizer._optim._param_groups[0], dict): + optimizer._optim._param_groups[0], dict + ): for group in optimizer._optim._param_groups: for param in group['params']: if param._grad_ivar() is not None: param_grads.append(param._grad_ivar()) if param._grad_ivar().dtype in [ - core.VarDesc.VarType.FP16, paddle.float16 + core.VarDesc.VarType.FP16, + paddle.float16, ]: param_grads_fp16.append(param._grad_ivar()) else: @@ -192,7 +208,8 @@ def unscale_method(self, optimizer): if param.grad is not None: param_grads.append(param.grad) if param.grad.dtype in [ - core.VarDesc.VarType.FP16, paddle.float16 + core.VarDesc.VarType.FP16, + paddle.float16, ]: param_grads_fp16.append(param.grad) else: @@ -202,27 +219,34 @@ def unscale_method(self, optimizer): temp_found_inf_fp32 = to_variable(np.array([0]).astype(np.bool_)) device = "cpu" if optimizer.offload else "gpu" - dev_id = 0 if device == "cpu" else int( - paddle.get_device().split(":")[1]) + dev_id = ( + 0 if device == "cpu" else int(paddle.get_device().split(":")[1]) + ) with device_guard(dev_id, device): if len(param_grads_fp16): - _legacy_C_ops.check_finite_and_unscale(param_grads_fp16, - self._scale, - param_grads_fp16, - temp_found_inf_fp16) + _legacy_C_ops.check_finite_and_unscale( + param_grads_fp16, + self._scale, + param_grads_fp16, + temp_found_inf_fp16, + ) if len(param_grads_fp32): - _legacy_C_ops.check_finite_and_unscale(param_grads_fp32, - self._scale, - param_grads_fp32, - temp_found_inf_fp32) + _legacy_C_ops.check_finite_and_unscale( + param_grads_fp32, + self._scale, + param_grads_fp32, + temp_found_inf_fp32, + ) self._found_inf = 1 if temp_found_inf_fp16 or temp_found_inf_fp32 else 0 is_found_inf = paddle.to_tensor([self._found_inf], dtype="int32") - paddle.distributed.all_reduce(is_found_inf, - op=paddle.distributed.ReduceOp.MAX, - group=optimizer.group) + paddle.distributed.all_reduce( + is_found_inf, + op=paddle.distributed.ReduceOp.MAX, + group=optimizer.group, + ) self._found_inf = is_found_inf.numpy()[0] scaler._unscale = MethodType(unscale_method, scaler) diff --git a/python/paddle/distributed/fleet/meta_parallel/sharding_parallel.py b/python/paddle/distributed/fleet/meta_parallel/sharding_parallel.py index fc3f195f7dea5..65fd317bd7623 100644 --- a/python/paddle/distributed/fleet/meta_parallel/sharding_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/sharding_parallel.py @@ -20,7 +20,6 @@ class ShardingParallel(MetaParallelBase): - def __init__(self, layers, hcg, **kwargs): super(ShardingParallel, self).__init__(layers, hcg, **kwargs) diff --git a/python/paddle/distributed/fleet/meta_parallel/tensor_parallel.py b/python/paddle/distributed/fleet/meta_parallel/tensor_parallel.py index 2e2072e9a3e2e..5a7d0ae6c770e 100755 --- a/python/paddle/distributed/fleet/meta_parallel/tensor_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/tensor_parallel.py @@ -15,14 +15,16 @@ from .meta_parallel_base import MetaParallelBase from ..utils.hybrid_parallel_util import broadcast_dp_parameters from ..utils.hybrid_parallel_util import broadcast_input_data -from ..utils.hybrid_parallel_util import broadcast_mp_parameters, broadcast_sharding_parameters +from ..utils.hybrid_parallel_util import ( + broadcast_mp_parameters, + broadcast_sharding_parameters, +) from ..utils.log_util import logger __all__ = [] class TensorParallel(MetaParallelBase): - def __init__(self, layers, hcg, **kwargs): super(TensorParallel, self).__init__(layers, hcg, **kwargs) diff --git a/python/paddle/distributed/fleet/metrics/metric.py b/python/paddle/distributed/fleet/metrics/metric.py index 6aa3793cd65d5..34c1c2968c820 100644 --- a/python/paddle/distributed/fleet/metrics/metric.py +++ b/python/paddle/distributed/fleet/metrics/metric.py @@ -257,7 +257,8 @@ def mae(abserr, total_ins_num, scope=None, util=None): abserr = np.array(scope.find_var(abserr).get_tensor()) if isinstance(total_ins_num, Variable): total_ins_num = np.array( - scope.find_var(total_ins_num.name).get_tensor()) + scope.find_var(total_ins_num.name).get_tensor() + ) elif isinstance(total_ins_num, str): total_ins_num = np.array(scope.find_var(total_ins_num).get_tensor()) @@ -306,7 +307,8 @@ def rmse(sqrerr, total_ins_num, scope=None, util=None): sqrerr = np.array(scope.find_var(sqrerr).get_tensor()) if isinstance(total_ins_num, Variable): total_ins_num = np.array( - scope.find_var(total_ins_num.name).get_tensor()) + scope.find_var(total_ins_num.name).get_tensor() + ) elif isinstance(total_ins_num, str): total_ins_num = np.array(scope.find_var(total_ins_num).get_tensor()) old_metric_shape = np.array(sqrerr.shape) @@ -355,7 +357,8 @@ def mse(sqrerr, total_ins_num, scope=None, util=None): sqrerr = np.array(scope.find_var(sqrerr).get_tensor()) if isinstance(total_ins_num, Variable): total_ins_num = np.array( - scope.find_var(total_ins_num.name).get_tensor()) + scope.find_var(total_ins_num.name).get_tensor() + ) elif isinstance(total_ins_num, str): total_ins_num = np.array(scope.find_var(total_ins_num).get_tensor()) old_metric_shape = np.array(sqrerr.shape) diff --git a/python/paddle/distributed/fleet/model.py b/python/paddle/distributed/fleet/model.py index 632e016f3d350..21e6d07ad55e3 100644 --- a/python/paddle/distributed/fleet/model.py +++ b/python/paddle/distributed/fleet/model.py @@ -15,7 +15,12 @@ import paddle from .base.topology import ParallelMode from .meta_parallel import TensorParallel -from .meta_parallel import PipelineParallel, ShardingParallel, PipelineParallelWithInterleave, PipelineLayer +from .meta_parallel import ( + PipelineParallel, + ShardingParallel, + PipelineParallelWithInterleave, + PipelineLayer, +) from paddle.fluid.dygraph.varbase_patch_methods import _grad_scalar from paddle.distributed import fleet @@ -89,19 +94,23 @@ def forward(self, x): amp_enable = True amp_level = "O2" if strategy.amp_configs['use_pure_fp16'] else "O1" if amp_level.upper() == "O2": - model = paddle.amp.decorate(models=model, - optimizers=None, - level="O2", - master_weight=None, - save_dtype=None) + model = paddle.amp.decorate( + models=model, + optimizers=None, + level="O2", + master_weight=None, + save_dtype=None, + ) init_loss_scaling = strategy.amp_configs['init_loss_scaling'] incr_ratio = strategy.amp_configs['incr_ratio'] decr_ratio = strategy.amp_configs['decr_ratio'] incr_every_n_steps = strategy.amp_configs['incr_every_n_steps'] decr_every_n_nan_or_inf = strategy.amp_configs[ - 'decr_every_n_nan_or_inf'] + 'decr_every_n_nan_or_inf' + ] use_dynamic_loss_scaling = strategy.amp_configs[ - 'use_dynamic_loss_scaling'] + 'use_dynamic_loss_scaling' + ] global _grad_scalar _grad_scalar = paddle.amp.GradScaler( @@ -110,14 +119,16 @@ def forward(self, x): decr_ratio=decr_ratio, incr_every_n_steps=incr_every_n_steps, decr_every_n_nan_or_inf=decr_every_n_nan_or_inf, - use_dynamic_loss_scaling=use_dynamic_loss_scaling) + use_dynamic_loss_scaling=use_dynamic_loss_scaling, + ) if strategy.heter_ccl_mode == True: distributed_model = paddle.DataParallel( model, comm_buffer_size=strategy.fuse_grad_size_in_MB, last_comm_buffer_size=strategy.last_comm_group_size_MB, - find_unused_parameters=strategy.find_unused_parameters) + find_unused_parameters=strategy.find_unused_parameters, + ) return distributed_model if fleet_env._hcg.get_parallel_mode() == ParallelMode.SHARDING_PARALLEL: @@ -127,8 +138,13 @@ def forward(self, x): # NOTE (JZ-LIANG) init parameters broadcast within sharding group # normally it should be done inside DataParallel if fleet_env.sharding_degree > 1: - from paddle.distributed.fleet.utils.hybrid_parallel_util import broadcast_sharding_parameters - assert fleet_env.sharding_degree == fleet_env._hcg.get_sharding_parallel_world_size( + from paddle.distributed.fleet.utils.hybrid_parallel_util import ( + broadcast_sharding_parameters, + ) + + assert ( + fleet_env.sharding_degree + == fleet_env._hcg.get_sharding_parallel_world_size() ) broadcast_sharding_parameters(model, fleet_env._hcg) model = paddle.DataParallel( @@ -136,7 +152,8 @@ def forward(self, x): comm_buffer_size=strategy.fuse_grad_size_in_MB, last_comm_buffer_size=strategy.last_comm_group_size_MB, find_unused_parameters=strategy.find_unused_parameters, - group=fleet_env._hcg.get_data_parallel_group()) + group=fleet_env._hcg.get_data_parallel_group(), + ) elif fleet_env._hcg.get_parallel_mode() == ParallelMode.TENSOR_PARALLEL: model = TensorParallel(model, fleet_env._hcg, strategy=strategy) elif fleet_env._hcg.get_parallel_mode() == ParallelMode.PIPELINE_PARALLEL: @@ -148,8 +165,8 @@ def forward(self, x): model = PipelineParallel(model, fleet_env._hcg, strategy=strategy) else: # interleave pipeline - model = PipelineParallelWithInterleave(model, - fleet_env._hcg, - strategy=strategy) + model = PipelineParallelWithInterleave( + model, fleet_env._hcg, strategy=strategy + ) return model diff --git a/python/paddle/distributed/fleet/optimizer.py b/python/paddle/distributed/fleet/optimizer.py index 42567465c551b..37a3a896f6b05 100644 --- a/python/paddle/distributed/fleet/optimizer.py +++ b/python/paddle/distributed/fleet/optimizer.py @@ -21,27 +21,27 @@ def _dygraph_distributed_optimizer(optimizer, strategy=None): """ - Optimizer for distributed training. - For the distributed training, this method would rebuild a new instance of DistributedOptimizer. - Which has basic Optimizer function and special features for distributed training. - Args: - optimizer(Optimizer): The executor to run for init server. - strategy(DistributedStrategy): Extra properties for distributed optimizer. - It is recommended to use DistributedStrategy in fleet.init(). The strategy - here is for compatibility. If the strategy in fleet.distributed_optimizer() - is not None, then it will overwrite the DistributedStrategy in fleet.init(), - which will take effect in distributed training. - Returns: - Fleet: instance of fleet. - Examples: - .. code-block:: python - import paddle - import paddle.distributed.fleet as fleet - fleet.init(is_collective=True) - strategy = fleet.DistributedStrategy() - optimizer = paddle.optimizer.SGD(learning_rate=0.001) - optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) - """ + Optimizer for distributed training. + For the distributed training, this method would rebuild a new instance of DistributedOptimizer. + Which has basic Optimizer function and special features for distributed training. + Args: + optimizer(Optimizer): The executor to run for init server. + strategy(DistributedStrategy): Extra properties for distributed optimizer. + It is recommended to use DistributedStrategy in fleet.init(). The strategy + here is for compatibility. If the strategy in fleet.distributed_optimizer() + is not None, then it will overwrite the DistributedStrategy in fleet.init(), + which will take effect in distributed training. + Returns: + Fleet: instance of fleet. + Examples: + .. code-block:: python + import paddle + import paddle.distributed.fleet as fleet + fleet.init(is_collective=True) + strategy = fleet.DistributedStrategy() + optimizer = paddle.optimizer.SGD(learning_rate=0.001) + optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) + """ fleet_env = fleet.fleet fleet_env.user_defined_optimizer = optimizer @@ -52,18 +52,21 @@ def _dygraph_distributed_optimizer(optimizer, strategy=None): "in fleet_env.init(). The strategy here is only for compatibility. " "If the strategy in fleet_env.distributed_optimizer() is " "not None, then it will overwrite the DistributedStrategy in fleet_env.init(), " - "which will take effect in distributed training.") + "which will take effect in distributed training." + ) fleet_env._user_defined_strategy = copy.deepcopy(strategy) fleet_env._context = {} if fleet_env.worker_num() > 1: if fleet_env._user_defined_strategy.heter_ccl_mode == False: - return HybridParallelOptimizer(optimizer, fleet_env._hcg, - fleet_env._user_defined_strategy) + return HybridParallelOptimizer( + optimizer, fleet_env._hcg, fleet_env._user_defined_strategy + ) else: - return HeterParallelOptimizer(optimizer, - fleet_env._user_defined_strategy) + return HeterParallelOptimizer( + optimizer, fleet_env._user_defined_strategy + ) else: return optimizer diff --git a/python/paddle/distributed/fleet/recompute/recompute.py b/python/paddle/distributed/fleet/recompute/recompute.py index 03b0c9d73f06b..2657c60f02c9a 100755 --- a/python/paddle/distributed/fleet/recompute/recompute.py +++ b/python/paddle/distributed/fleet/recompute/recompute.py @@ -40,16 +40,23 @@ def detach_variable(inputs): def check_recompute_necessary(inputs): - if not any(input_.stop_gradient == False for input_ in inputs - if isinstance(input_, (core.eager.Tensor, paddle.Tensor))): + if not any( + input_.stop_gradient == False + for input_ in inputs + if isinstance(input_, (core.eager.Tensor, paddle.Tensor)) + ): logger.warning( "[Recompute]: None of the inputs to current recompute block need grad, " - "therefore there is NO need to recompute this block in backward !") + "therefore there is NO need to recompute this block in backward !" + ) @contextlib.contextmanager def swith_rng_state_tracker(rng_state, tracker): - from paddle.distributed.fleet.meta_parallel.parallel_layers.random import get_rng_state_tracker + from paddle.distributed.fleet.meta_parallel.parallel_layers.random import ( + get_rng_state_tracker, + ) + orig_cuda_rng_state = paddle.get_cuda_rng_state() orig_cuda_rng_tracker = get_rng_state_tracker().get_states_tracker() @@ -63,10 +70,11 @@ def swith_rng_state_tracker(rng_state, tracker): class LegacyRecomputeFunction(LegacyPyLayer): - @staticmethod def forward(ctx, run_function, preserve_rng_state, *args): - from paddle.distributed.fleet.meta_parallel.parallel_layers.random import get_rng_state_tracker + from paddle.distributed.fleet.meta_parallel.parallel_layers.random import ( + get_rng_state_tracker, + ) # store for recomputing ctx.run_function = run_function @@ -95,30 +103,37 @@ def forward(ctx, run_function, preserve_rng_state, *args): cur_device = paddle.get_device() if 'gpu:' not in cur_device: raise RuntimeError( - "Recompute with RNG perserve is not support current device: {}." - .format(cur_device)) + "Recompute with RNG perserve is not support current device: {}.".format( + cur_device + ) + ) ctx.fw_cuda_rng_state = paddle.get_cuda_rng_state() - ctx.fwd_cuda_rng_state_tracker = get_rng_state_tracker( - ).get_states_tracker() + ctx.fwd_cuda_rng_state_tracker = ( + get_rng_state_tracker().get_states_tracker() + ) # TODO support AMP tracer = framework._dygraph_tracer() - ctx.is_fw_autocast = False if tracer._amp_level == core.AmpLevel.O0 else True + ctx.is_fw_autocast = ( + False if tracer._amp_level == core.AmpLevel.O0 else True + ) if tracer._amp_level == core.AmpLevel.O2: ctx.amp_level = 'O2' elif tracer._amp_level in (core.AmpLevel.O1, core.AmpLevel.O0): ctx.amp_level = 'O1' else: - raise ValueError("unsupported amp level: {}".format( - tracer._amp_level)) + raise ValueError( + "unsupported amp level: {}".format(tracer._amp_level) + ) if tracer._amp_dtype == 'float16': ctx.amp_dtype = 'float16' elif tracer._amp_dtype in ('bfloat16', 'float32'): ctx.amp_dtype = 'bfloat16' else: - raise ValueError("unsupported amp dtype: {}".format( - tracer._amp_dtype)) + raise ValueError( + "unsupported amp dtype: {}".format(tracer._amp_dtype) + ) ctx.amp_white_list, ctx.amp_black_list = tracer._get_amp_op_list() @@ -145,27 +160,31 @@ def backward(ctx, *args): # NOTE support AMP # need restore auto_cast state as well as w/b list if ctx.preserve_rng_state: - with swith_rng_state_tracker(ctx.fw_cuda_rng_state, - ctx.fwd_cuda_rng_state_tracker): + with swith_rng_state_tracker( + ctx.fw_cuda_rng_state, ctx.fwd_cuda_rng_state_tracker + ): with paddle.amp.auto_cast( - enable=ctx.is_fw_autocast, - custom_white_list=ctx.amp_white_list, - custom_black_list=ctx.amp_black_list, - level=ctx.amp_level, - dtype=ctx.amp_dtype): + enable=ctx.is_fw_autocast, + custom_white_list=ctx.amp_white_list, + custom_black_list=ctx.amp_black_list, + level=ctx.amp_level, + dtype=ctx.amp_dtype, + ): detached_inputs = detach_variable(tuple(inputs)) outputs = ctx.run_function(*detached_inputs) else: - with paddle.amp.auto_cast(enable=ctx.is_fw_autocast, - custom_white_list=ctx.amp_white_list, - custom_black_list=ctx.amp_black_list, - level=ctx.amp_level, - dtype=ctx.amp_dtype): + with paddle.amp.auto_cast( + enable=ctx.is_fw_autocast, + custom_white_list=ctx.amp_white_list, + custom_black_list=ctx.amp_black_list, + level=ctx.amp_level, + dtype=ctx.amp_dtype, + ): detached_inputs = detach_variable(tuple(inputs)) outputs = ctx.run_function(*detached_inputs) if isinstance(outputs, core.VarBase): - outputs = (outputs, ) + outputs = (outputs,) assert len(outputs) == len(args) # run backward() with only tensor that requires grad @@ -176,8 +195,10 @@ def backward(ctx, *args): # the following backward_inputs_with_grad is used to avoid this case. backward_inputs_with_grad = [] for i in range(len(outputs)): - if isinstance(outputs[i], - core.VarBase) and not outputs[i].stop_gradient: + if ( + isinstance(outputs[i], core.VarBase) + and not outputs[i].stop_gradient + ): forward_outputs_with_grad.append(outputs[i]) backward_inputs_with_grad.append(args[i]) @@ -188,19 +209,24 @@ def backward(ctx, *args): # actually backward with paddle.amp.auto_cast(enable=False): - paddle.autograd.backward(forward_outputs_with_grad, - backward_inputs_with_grad) + paddle.autograd.backward( + forward_outputs_with_grad, backward_inputs_with_grad + ) - grads = list(inp._grad_ivar() for inp in detached_inputs - if isinstance(inp, core.VarBase)) + grads = list( + inp._grad_ivar() + for inp in detached_inputs + if isinstance(inp, core.VarBase) + ) return grads class RecomputeFunction(PyLayer): - @staticmethod def forward(ctx, run_function, preserve_rng_state, *args, **kwargs): - from paddle.distributed.fleet.meta_parallel.parallel_layers.random import get_rng_state_tracker + from paddle.distributed.fleet.meta_parallel.parallel_layers.random import ( + get_rng_state_tracker, + ) # store for recomputing ctx.run_function = run_function @@ -230,30 +256,37 @@ def forward(ctx, run_function, preserve_rng_state, *args, **kwargs): cur_device = paddle.get_device() if 'gpu:' not in cur_device: raise RuntimeError( - "Recompute with RNG perserve is not support current device: {}." - .format(cur_device)) + "Recompute with RNG perserve is not support current device: {}.".format( + cur_device + ) + ) ctx.fw_cuda_rng_state = paddle.get_cuda_rng_state() - ctx.fwd_cuda_rng_state_tracker = get_rng_state_tracker( - ).get_states_tracker() + ctx.fwd_cuda_rng_state_tracker = ( + get_rng_state_tracker().get_states_tracker() + ) # TODO support AMP tracer = framework._dygraph_tracer() - ctx.is_fw_autocast = False if tracer._amp_level == core.AmpLevel.O0 else True + ctx.is_fw_autocast = ( + False if tracer._amp_level == core.AmpLevel.O0 else True + ) if tracer._amp_level == core.AmpLevel.O2: ctx.amp_level = 'O2' elif tracer._amp_level in (core.AmpLevel.O1, core.AmpLevel.O0): ctx.amp_level = 'O1' else: - raise ValueError("unsupported amp level: {}".format( - tracer._amp_level)) + raise ValueError( + "unsupported amp level: {}".format(tracer._amp_level) + ) if tracer._amp_dtype == 'float16': ctx.amp_dtype = 'float16' elif tracer._amp_dtype in ('bfloat16', 'float32'): ctx.amp_dtype = 'bfloat16' else: - raise ValueError("unsupported amp dtype: {}".format( - tracer._amp_dtype)) + raise ValueError( + "unsupported amp dtype: {}".format(tracer._amp_dtype) + ) ctx.amp_white_list, ctx.amp_black_list = tracer._get_amp_op_list() @@ -280,28 +313,33 @@ def backward(ctx, *args): # NOTE support AMP # need restore auto_cast state as well as w/b list if ctx.preserve_rng_state: - with swith_rng_state_tracker(ctx.fw_cuda_rng_state, - ctx.fwd_cuda_rng_state_tracker): + with swith_rng_state_tracker( + ctx.fw_cuda_rng_state, ctx.fwd_cuda_rng_state_tracker + ): with paddle.amp.auto_cast( - enable=ctx.is_fw_autocast, - custom_white_list=ctx.amp_white_list, - custom_black_list=ctx.amp_black_list, - level=ctx.amp_level, - dtype=ctx.amp_dtype): + enable=ctx.is_fw_autocast, + custom_white_list=ctx.amp_white_list, + custom_black_list=ctx.amp_black_list, + level=ctx.amp_level, + dtype=ctx.amp_dtype, + ): detached_inputs = detach_variable(tuple(inputs)) - outputs = ctx.run_function(*detached_inputs, - **ctx.kwargs) + outputs = ctx.run_function( + *detached_inputs, **ctx.kwargs + ) else: - with paddle.amp.auto_cast(enable=ctx.is_fw_autocast, - custom_white_list=ctx.amp_white_list, - custom_black_list=ctx.amp_black_list, - level=ctx.amp_level, - dtype=ctx.amp_dtype): + with paddle.amp.auto_cast( + enable=ctx.is_fw_autocast, + custom_white_list=ctx.amp_white_list, + custom_black_list=ctx.amp_black_list, + level=ctx.amp_level, + dtype=ctx.amp_dtype, + ): detached_inputs = detach_variable(tuple(inputs)) outputs = ctx.run_function(*detached_inputs, **ctx.kwargs) if isinstance(outputs, (core.VarBase, core.eager.Tensor)): - outputs = (outputs, ) + outputs = (outputs,) assert len(outputs) == len(args) # run backward() with only tensor that requires grad @@ -312,10 +350,10 @@ def backward(ctx, *args): # the following backward_inputs_with_grad is used to avoid this case. backward_inputs_with_grad = [] for i in range(len(outputs)): - if isinstance( - outputs[i], - (core.VarBase, - core.eager.Tensor)) and not outputs[i].stop_gradient: + if ( + isinstance(outputs[i], (core.VarBase, core.eager.Tensor)) + and not outputs[i].stop_gradient + ): forward_outputs_with_grad.append(outputs[i]) backward_inputs_with_grad.append(args[i]) @@ -326,17 +364,22 @@ def backward(ctx, *args): # actually backward with paddle.amp.auto_cast(enable=False): - paddle.autograd.backward(forward_outputs_with_grad, - backward_inputs_with_grad) + paddle.autograd.backward( + forward_outputs_with_grad, backward_inputs_with_grad + ) if in_dygraph_mode(): grads = tuple( - inp._grad_ivar() for inp in detached_inputs - if isinstance(inp, (core.VarBase, core.eager.Tensor))) + inp._grad_ivar() + for inp in detached_inputs + if isinstance(inp, (core.VarBase, core.eager.Tensor)) + ) else: grads = list( - inp._grad_ivar() for inp in detached_inputs - if isinstance(inp, (core.VarBase, core.eager.Tensor))) + inp._grad_ivar() + for inp in detached_inputs + if isinstance(inp, (core.VarBase, core.eager.Tensor)) + ) return grads @@ -494,7 +537,6 @@ def recompute_sequential(ctx, functions, *args, **kwargs): preserve_rng_state = ctx.get('preserve_rng_state', True) def _run_func(begin, end, funcs): - def do_run(input): for i in range(begin, end + 1): input = funcs[i](input) @@ -510,8 +552,10 @@ def do_run(input): end = -1 for begin in range(0, segment_size * (segments - 1), segment_size): end = begin + segment_size - 1 - args = recompute(_run_func(begin, end, functions), - *args, - preserve_rng_state=preserve_rng_state, - **kwargs) + args = recompute( + _run_func(begin, end, functions), + *args, + preserve_rng_state=preserve_rng_state, + **kwargs + ) return _run_func(end + 1, len(functions) - 1, functions)(args) diff --git a/python/paddle/distributed/fleet/recompute/recompute_hybrid.py b/python/paddle/distributed/fleet/recompute/recompute_hybrid.py index 9c006c1e044f0..7f0d22726e596 100644 --- a/python/paddle/distributed/fleet/recompute/recompute_hybrid.py +++ b/python/paddle/distributed/fleet/recompute/recompute_hybrid.py @@ -17,7 +17,11 @@ from paddle.autograd import PyLayer from paddle.fluid import framework from ..meta_parallel.parallel_layers.random import get_rng_state_tracker -from .recompute import check_recompute_necessary, detach_variable, swith_rng_state_tracker +from .recompute import ( + check_recompute_necessary, + detach_variable, + swith_rng_state_tracker, +) from ..meta_parallel.pp_utils import utils __all__ = [] @@ -32,8 +36,11 @@ def _split_activation(tensor, mp_group): tensor_numel = paddle.numel(tensor) assert tensor_numel != 0, "can't recompute zero element" - assert tensor_numel % mp_degree == 0, "The capacity of the activation ({}) cannot be divisible by mp_degree({})".format( - tensor_numel, mp_degree) + assert ( + tensor_numel % mp_degree == 0 + ), "The capacity of the activation ({}) cannot be divisible by mp_degree({})".format( + tensor_numel, mp_degree + ) # use inplace operation to save memory data = tensor.flatten_() @@ -69,8 +76,16 @@ class _HPRecomputeFunction(PyLayer): """ @staticmethod - def forward(ctx, run_function, all_outputs, mp_group, offload, partition, - *args, **kwargs): + def forward( + ctx, + run_function, + all_outputs, + mp_group, + offload, + partition, + *args, + **kwargs + ): check_recompute_necessary(args) # store for recomputing @@ -80,8 +95,9 @@ def forward(ctx, run_function, all_outputs, mp_group, offload, partition, # store the rng states ctx.fwd_cuda_rng_state = paddle.get_cuda_rng_state() - ctx.fwd_cuda_rng_state_tracker = get_rng_state_tracker( - ).get_states_tracker() + ctx.fwd_cuda_rng_state_tracker = ( + get_rng_state_tracker().get_states_tracker() + ) # save config info ctx.mp_group = mp_group @@ -95,20 +111,25 @@ def forward(ctx, run_function, all_outputs, mp_group, offload, partition, tensor_inputs = [] cur_device = paddle.get_device() - assert 'gpu:' in paddle.get_device( + assert ( + 'gpu:' in paddle.get_device() ), "Recompute with RNG is not support current device: {}.".format( - cur_device) + cur_device + ) # TODO support AMP tracer = framework._dygraph_tracer() - ctx.is_fw_autocast = False if tracer._amp_level == core.AmpLevel.O0 else True + ctx.is_fw_autocast = ( + False if tracer._amp_level == core.AmpLevel.O0 else True + ) if tracer._amp_level == core.AmpLevel.O2: ctx.amp_level = 'O2' elif tracer._amp_level in (core.AmpLevel.O1, core.AmpLevel.O0): ctx.amp_level = 'O1' else: - raise ValueError("unsupported amp level: {}".format( - tracer._amp_level)) + raise ValueError( + "unsupported amp level: {}".format(tracer._amp_level) + ) ctx.amp_white_list, ctx.amp_black_list = tracer._get_amp_op_list() with paddle.no_grad(): @@ -119,8 +140,9 @@ def forward(ctx, run_function, all_outputs, mp_group, offload, partition, state = arg.stop_gradient if partition: ctx.tensor_shapes.append(arg.shape) - partition = _split_activation(arg.detach(), - mp_group).clone() + partition = _split_activation( + arg.detach(), mp_group + ).clone() # TODO(shenliang03) not use calculate stream to D2H to speed arg = partition.cpu() if offload else partition else: @@ -154,38 +176,44 @@ def backward(ctx, *args): for i, idx in enumerate(tensor_indices): if ctx.partition: state = tensors[i].stop_gradient - tensors[i] = _merge_activation( - tensors[i], - ctx.mp_group).detach().reshape_(tensor_shapes[i]) + tensors[i] = ( + _merge_activation(tensors[i], ctx.mp_group) + .detach() + .reshape_(tensor_shapes[i]) + ) tensors[i].stop_gradient = state - inputs[idx] = tensors[i].cuda( - device_id) if ctx.offload else tensors[i] + inputs[idx] = ( + tensors[i].cuda(device_id) if ctx.offload else tensors[i] + ) tracer = framework._dygraph_tracer() tracer._has_grad = True # need restore auto_cast state as well as w/b list - with swith_rng_state_tracker(ctx.fwd_cuda_rng_state, - ctx.fwd_cuda_rng_state_tracker): - with paddle.amp.auto_cast(enable=ctx.is_fw_autocast, - custom_white_list=ctx.amp_white_list, - custom_black_list=ctx.amp_black_list, - level=ctx.amp_level): + with swith_rng_state_tracker( + ctx.fwd_cuda_rng_state, ctx.fwd_cuda_rng_state_tracker + ): + with paddle.amp.auto_cast( + enable=ctx.is_fw_autocast, + custom_white_list=ctx.amp_white_list, + custom_black_list=ctx.amp_black_list, + level=ctx.amp_level, + ): detached_inputs = detach_variable(tuple(inputs)) outputs = ctx.run_function(*detached_inputs, **ctx.kwargs) if isinstance(outputs, (core.VarBase, core.eager.Tensor)): - outputs = (outputs, ) + outputs = (outputs,) assert len(outputs) == len(args) forward_outputs_with_grad = [] backward_inputs = [] for i in range(len(outputs)): - if isinstance( - outputs[i], - (core.VarBase, - core.eager.Tensor)) and not outputs[i].stop_gradient: + if ( + isinstance(outputs[i], (core.VarBase, core.eager.Tensor)) + and not outputs[i].stop_gradient + ): forward_outputs_with_grad.append(outputs[i]) backward_inputs.append(args[i]) @@ -196,8 +224,11 @@ def backward(ctx, *args): # actually backward paddle.autograd.backward(forward_outputs_with_grad, backward_inputs) - grads = tuple(inp._grad_ivar() for inp in detached_inputs - if isinstance(inp, (core.VarBase, core.eager.Tensor))) + grads = tuple( + inp._grad_ivar() + for inp in detached_inputs + if isinstance(inp, (core.VarBase, core.eager.Tensor)) + ) return grads @@ -226,14 +257,17 @@ def recompute_hybrid(ctx, function, *args, **kwargs): """ mp_group = ctx.get('mp_group', None) - assert mp_group is not None, "ctx must contains mp_group and mp_group can not be None." + assert ( + mp_group is not None + ), "ctx must contains mp_group and mp_group can not be None." offload = ctx.get('offload', False) partition = ctx.get('partition', False) all_outputs = [] - _HPRecomputeFunction.apply(function, all_outputs, mp_group, offload, - partition, *args, **kwargs) + _HPRecomputeFunction.apply( + function, all_outputs, mp_group, offload, partition, *args, **kwargs + ) if len(all_outputs) == 1: return all_outputs[0] diff --git a/python/paddle/distributed/fleet/runtime/collective_runtime.py b/python/paddle/distributed/fleet/runtime/collective_runtime.py index 5b66bf79398ac..7ccd0ecf7d000 100644 --- a/python/paddle/distributed/fleet/runtime/collective_runtime.py +++ b/python/paddle/distributed/fleet/runtime/collective_runtime.py @@ -19,28 +19,32 @@ class CollectiveRuntime(RuntimeBase): - def __init__(self): super(CollectiveRuntime, self).__init__() def _init_worker(self): logging.warn( - "You should not call 'init_worker' method for collective mode.") + "You should not call 'init_worker' method for collective mode." + ) def _run_worker(self): logging.warn( - "You should not call 'run_worker' method for collective mode.") + "You should not call 'run_worker' method for collective mode." + ) def _init_server(self, *args, **kwargs): logging.warn( - "You should not call 'init_server' method for collective mode.") + "You should not call 'init_server' method for collective mode." + ) def _run_server(self): logging.warn( - "You should not call 'run_server' method for collective mode.") + "You should not call 'run_server' method for collective mode." + ) def _stop_worker(self): logging.warn( - "You should not call 'stop_worker' method for collective mode.") + "You should not call 'stop_worker' method for collective mode." + ) # save inference model should be added here diff --git a/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py b/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py index 062a6d5abf584..e2c5e5da29ea4 100644 --- a/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py +++ b/python/paddle/distributed/fleet/runtime/parameter_server_runtime.py @@ -30,7 +30,6 @@ class ParameterServerRuntime(RuntimeBase): - def __init__(self): super(ParameterServerRuntime, self).__init__() self._communicator = None @@ -46,7 +45,9 @@ def _set_basic_info(self, context): def _get_distributed_strategy(self): strategy = None - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + StrategyFactory, + ) dist_strategy = self.context["valid_strategy"] k_steps = dist_strategy.a_sync_configs["k_steps"] @@ -66,19 +67,21 @@ def _get_distributed_strategy(self): return strategy def build_compiled_startegy(self): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import CompileTimeStrategy - - compiled_config = CompileTimeStrategy(self.origin_main_program, - self.origin_main_program, - self.async_strategy, - self.role_maker) + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + CompileTimeStrategy, + ) + + compiled_config = CompileTimeStrategy( + self.origin_main_program, + self.origin_main_program, + self.async_strategy, + self.role_maker, + ) return compiled_config - def _load_sparse_params(self, - executor, - dirname, - varnames, - main_program=None): + def _load_sparse_params( + self, executor, dirname, varnames, main_program=None + ): assert vars != None check_vars = [] load_prog = Program() @@ -88,12 +91,15 @@ def _in_varnames(var): return var.name in varnames load_vars = list( - filter(_in_varnames, - fluid.default_main_program().list_vars())) + filter(_in_varnames, fluid.default_main_program().list_vars()) + ) if main_program is None: main_program = self.origin_main_program - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_varname_parts + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_varname_parts, + ) + for each_var in load_vars: assert isinstance(each_var, Variable) @@ -104,29 +110,31 @@ def _in_varnames(var): if not os.path.exists(var_path): raise ValueError( "SelectedRows var {} can not find at {}".format( - new_var.name, var_path)) + new_var.name, var_path + ) + ) if os.path.isfile(var_path): - load_block.append_op(type='sparse_tensor_load', - inputs={}, - outputs={'Out': [new_var]}, - attrs={ - 'file_path': - os.path.join(dirname, origin_varname), - 'node_index': - self.role_maker._server_index(), - 'node_num': - self.role_maker._server_num(), - 'shape': - each_var.shape - }) + load_block.append_op( + type='sparse_tensor_load', + inputs={}, + outputs={'Out': [new_var]}, + attrs={ + 'file_path': os.path.join(dirname, origin_varname), + 'node_index': self.role_maker._server_index(), + 'node_num': self.role_maker._server_num(), + 'shape': each_var.shape, + }, + ) check_vars.append(each_var) executor.run(load_prog) def _load_distributed_params(self, dirname, varnames): from paddle.fluid.communicator import LargeScaleKV - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_varname_parts + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_varname_parts, + ) scale_kv = LargeScaleKV() for varname in varnames: @@ -136,12 +144,13 @@ def _load_distributed_params(self, dirname, varnames): @staticmethod def __exclude_vars(exclude_var_names=[]): - def is_valid(var): if var.name in exclude_var_names: return False - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_varname_parts + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_varname_parts, + ) origin_varname, _, _ = _get_varname_parts(var.name) if origin_varname.endswith("@GRAD"): @@ -150,25 +159,29 @@ def is_valid(var): if origin_varname == "learning_rate_0": return False - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable return is_valid def _init_worker(self): - def sync_strategy_envs(): kwargs = {} kwargs[ - "pserver_endpoints"] = self.role_maker._get_pserver_endpoints() + "pserver_endpoints" + ] = self.role_maker._get_pserver_endpoints() kwargs["trainer_id"] = self.role_maker._worker_index() return kwargs def geo_strategy_envs(): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablenames + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablenames, + ) def get_sparse_attrs(): opt_init_map = {} @@ -176,13 +189,17 @@ def get_sparse_attrs(): opt_init_map["fill_constant"] = ["value"] opt_init_map["uniform_random"] = ["seed", "min", "max"] opt_init_map["truncated_gaussian_random"] = [ - "seed", "mean", "std" + "seed", + "mean", + "std", ] - dist_varnames = get_sparse_tablenames(self.origin_main_program, - True) + dist_varnames = get_sparse_tablenames( + self.origin_main_program, True + ) sparse_varnames = get_sparse_tablenames( - self.origin_main_program, False) + self.origin_main_program, False + ) if len(dist_varnames) != 0: raise ValueError( @@ -191,15 +208,18 @@ def get_sparse_attrs(): init_attrs = [] for value_name in sparse_varnames: - value_var = self.origin_main_program.global_block( - ).vars[value_name] + value_var = self.origin_main_program.global_block().vars[ + value_name + ] value_attr = [ value_name, - ",".join([str(dim) for dim in value_var.shape]) + ",".join([str(dim) for dim in value_var.shape]), ] for op in self.origin_startup_program.global_block().ops: - if op.type in opt_init_map.keys( - ) and value_name == op.output("Out")[0]: + if ( + op.type in opt_init_map.keys() + and value_name == op.output("Out")[0] + ): init_attr = [op.type] for attr in opt_init_map[op.type]: init_attr.append(str(op.attr(attr))) @@ -213,10 +233,15 @@ def get_sparse_attrs(): kwargs["sparse_attrs"] = get_sparse_attrs() return kwargs - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_lr_ops, _has_global_step + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_lr_ops, + _has_global_step, + ) - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import \ - SyncStrategy, GeoStrategy + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + SyncStrategy, + GeoStrategy, + ) trainer_config = self.async_strategy.get_trainer_runtime_config() print(trainer_config) @@ -228,7 +253,9 @@ def get_sparse_attrs(): wait_server_ready(self.role_maker._get_pserver_endpoints()) # for ps-heter mode, wait heter worker ready - if self.role_maker._is_heter_parameter_server_mode and self.role_maker._is_worker( + if ( + self.role_maker._is_heter_parameter_server_mode + and self.role_maker._is_worker() ): wait_server_ready(self.role_maker._get_heter_worker_endpoints()) @@ -252,15 +279,18 @@ def get_sparse_attrs(): if self.compiled_strategy.is_geo_mode(): recv_ctx = self.compiled_strategy.get_communicator_recv_context( - recv_type=4) + recv_type=4 + ) else: recv_ctx = self.compiled_strategy.get_communicator_recv_context( - recv_type=1) + recv_type=1 + ) from paddle.fluid.communicator import Communicator + self._communicator = Communicator( - trainer_config.mode, kwargs, - trainer_config.get_communicator_flags()) + trainer_config.mode, kwargs, trainer_config.get_communicator_flags() + ) self._communicator.init_with_ctx(send_ctx, recv_ctx) if not self._communicator.is_running(): @@ -271,21 +301,30 @@ def get_sparse_attrs(): def _get_executor(self): executor = fluid.Executor(fluid.CPUPlace()) if self.role_maker._is_heter_parameter_server_mode: - heter_worker_device_guard = self.context[ - "valid_strategy"].a_sync_configs[ - "heter_worker_device_guard"].upper() + heter_worker_device_guard = ( + self.context["valid_strategy"] + .a_sync_configs["heter_worker_device_guard"] + .upper() + ) if heter_worker_device_guard not in ["GPU", "XPU", "CPU"]: - raise ValueError("Heter Worker Not Support Device {}".format( - heter_worker_device_guard)) + raise ValueError( + "Heter Worker Not Support Device {}".format( + heter_worker_device_guard + ) + ) if self.role_maker._is_heter_worker(): if heter_worker_device_guard == "GPU": executor = Executor( fluid.CUDAPlace( - int(os.getenv("FLAGS_selected_gpus", "0")))) + int(os.getenv("FLAGS_selected_gpus", "0")) + ) + ) elif heter_worker_device_guard == "XPU": executor = Executor( fluid.XPUPlace( - int(os.getenv("FLAGS_selected_xpus", "0")))) + int(os.getenv("FLAGS_selected_xpus", "0")) + ) + ) return executor def _init_server(self, *args, **kwargs): @@ -297,8 +336,10 @@ def _init_server(self, *args, **kwargs): model_dirname = None executor = self._get_executor() - if self.role_maker._is_heter_worker( - ) and self.context["valid_strategy"].a_sync_configs["launch_barrier"]: + if ( + self.role_maker._is_heter_worker() + and self.context["valid_strategy"].a_sync_configs["launch_barrier"] + ): # for heter trainer wait server ready wait_server_ready(self.role_maker._get_pserver_endpoints()) executor.run(fluid.default_startup_program()) @@ -310,26 +351,35 @@ def _init_server(self, *args, **kwargs): sparse_varnames = self.compiled_strategy.get_sparse_varname_on_ps(False) sparse_related_optimize_varnames = [] for var_name in sparse_varnames: - sparse_related_optimize_varnames += self.compiled_strategy.get_optimize_varname_on_ps( - var_name) + sparse_related_optimize_varnames += ( + self.compiled_strategy.get_optimize_varname_on_ps(var_name) + ) sparse_related_optimize_varnames = list( - set(sparse_related_optimize_varnames)) + set(sparse_related_optimize_varnames) + ) distribtued_varnames = self.compiled_strategy.get_sparse_varname_on_ps( - True) + True + ) distributed_related_optimize_varnames = [] for var_name in distribtued_varnames: - distributed_related_optimize_varnames += self.compiled_strategy.get_optimize_varname_on_ps( - var_name) + distributed_related_optimize_varnames += ( + self.compiled_strategy.get_optimize_varname_on_ps(var_name) + ) distributed_related_optimize_varnames = list( - set(distributed_related_optimize_varnames)) + set(distributed_related_optimize_varnames) + ) remaining_vars = list( filter( ParameterServerRuntime.__exclude_vars( - sparse_varnames + distribtued_varnames + - sparse_related_optimize_varnames + - distributed_related_optimize_varnames), - fluid.default_main_program().list_vars())) + sparse_varnames + + distribtued_varnames + + sparse_related_optimize_varnames + + distributed_related_optimize_varnames + ), + fluid.default_main_program().list_vars(), + ) + ) if not model_dirname: return @@ -338,21 +388,26 @@ def _init_server(self, *args, **kwargs): raise ValueError("There is no directory named '%s'", model_dirname) # load dense - fluid.io.load_vars(executor, - main_program=fluid.default_main_program(), - dirname=model_dirname, - vars=remaining_vars) + fluid.io.load_vars( + executor, + main_program=fluid.default_main_program(), + dirname=model_dirname, + vars=remaining_vars, + ) # load sparse - self._load_sparse_params(executor=executor, - dirname=model_dirname, - varnames=sparse_varnames + - sparse_related_optimize_varnames) + self._load_sparse_params( + executor=executor, + dirname=model_dirname, + varnames=sparse_varnames + sparse_related_optimize_varnames, + ) # load large scale - self._load_distributed_params(dirname=model_dirname, - varnames=distribtued_varnames + - distributed_related_optimize_varnames) + self._load_distributed_params( + dirname=model_dirname, + varnames=distribtued_varnames + + distributed_related_optimize_varnames, + ) def _run_server(self): executor = self._get_executor() @@ -365,8 +420,15 @@ def _stop_worker(self): def _get_optimizer_status(self, op, param_name): supported_opts = [ - "sgd", "adam", "adagrad", "adamax", "momentum", "lars_momentum", - "rmsprop", "decayed_adagrad", "ftrl" + "sgd", + "adam", + "adagrad", + "adamax", + "momentum", + "lars_momentum", + "rmsprop", + "decayed_adagrad", + "ftrl", ] reshaped_val_map = {} @@ -377,7 +439,9 @@ def _get_optimizer_status(self, op, param_name): reshaped_val_map["momentum"] = ["velocity_0"] reshaped_val_map["lars_momentum"] = ["velocity_0"] reshaped_val_map["rmsprop"] = [ - "momentum_0", "mean_square_0", "mean_grad_0" + "momentum_0", + "mean_square_0", + "mean_grad_0", ] reshaped_val_map["decayed_adagrad"] = ["moment_0"] reshaped_val_map["ftrl"] = ["squared_0", "linear_0"] @@ -388,8 +452,10 @@ def _get_optimizer_status(self, op, param_name): if op not in supported_opts: raise ValueError( - "fleet can not support optimizer: {}, only this can be supported: {}" - .format(op, supported_opts)) + "fleet can not support optimizer: {}, only this can be supported: {}".format( + op, supported_opts + ) + ) reshaped_names = [ param_name + "_" + val for val in reshaped_val_map[op] @@ -404,12 +470,17 @@ def _get_optimizer_status(self, op, param_name): return reshaped_names, origin_names def _get_optimizer_op(self, param_name): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_optimize_ops + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_optimize_ops, + ) opts = _get_optimize_ops(self.origin_main_program) for op in opts: - if "Param" in op.input_names and \ - "LearningRate" in op.input_names and op.input("Param")[0] == param_name: + if ( + "Param" in op.input_names + and "LearningRate" in op.input_names + and op.input("Param")[0] == param_name + ): return op def _save_dense_params(self, executor, dirname, context, main_program): @@ -428,27 +499,24 @@ def _save_dense_params(self, executor, dirname, context, main_program): optimizer = self._get_optimizer_op(varname) reshaped_varnames, origin_varnames = self._get_optimizer_status( - optimizer.type, varname) + optimizer.type, varname + ) for var_name in [varname] + reshaped_varnames + origin_varnames: var = self.origin_main_program.global_block().vars[var_name] - block.append_op(type='recv_save', - attrs={ - "trainer_id": - self.role_maker._worker_index(), - "shape": - var.shape, - "slice_shapes": - [",".join([str(i) for i in var.shape])], - "slice_varnames": [var.name], - "remote_varnames": [var.name], - "is_sparse": - False, - "endpoints": - var_ctx.split_endpoints(), - "file_path": - os.path.join(dirname, var.name) - }) + block.append_op( + type='recv_save', + attrs={ + "trainer_id": self.role_maker._worker_index(), + "shape": var.shape, + "slice_shapes": [",".join([str(i) for i in var.shape])], + "slice_varnames": [var.name], + "remote_varnames": [var.name], + "is_sparse": False, + "endpoints": var_ctx.split_endpoints(), + "file_path": os.path.join(dirname, var.name), + }, + ) executor.run(prog) return local_vars @@ -467,7 +535,8 @@ def _save_sparse_params(self, executor, dirname, context, main_program): optimizer = self._get_optimizer_op(varname) reshaped_varnames, origin_varnames = self._get_optimizer_status( - optimizer.type, varname) + optimizer.type, varname + ) var = self.origin_main_program.global_block().vars[varname] slice_shapes = [] @@ -476,37 +545,34 @@ def _save_sparse_params(self, executor, dirname, context, main_program): for section in var_ctx.sections(): slice_shapes.append(str(section) + dims1) - block.append_op(type='recv_save', - attrs={ - "trainer_id": - self.role_maker._worker_index(), - "shape": - var.shape, - "slice_shapes": - slice_shapes, - "slice_varnames": - var_ctx.split_varnames(), - "remote_varnames": - var_ctx.split_varnames(), - "is_sparse": - True, - "endpoints": - var_ctx.split_endpoints(), - "pserver_num": - len(self.role_maker._get_pserver_endpoints()), - "file_path": - os.path.join(dirname, var.name) - }) + block.append_op( + type='recv_save', + attrs={ + "trainer_id": self.role_maker._worker_index(), + "shape": var.shape, + "slice_shapes": slice_shapes, + "slice_varnames": var_ctx.split_varnames(), + "remote_varnames": var_ctx.split_varnames(), + "is_sparse": True, + "endpoints": var_ctx.split_endpoints(), + "pserver_num": len( + self.role_maker._get_pserver_endpoints() + ), + "file_path": os.path.join(dirname, var.name), + }, + ) for reshaped_varname in reshaped_varnames: - var = self.origin_main_program.global_block( - ).vars[reshaped_varname] + var = self.origin_main_program.global_block().vars[ + reshaped_varname + ] slice_varnames = [] remote_varnames = [] for i in range(len(var_ctx.split_varnames())): - slice_varnames.append("{}.block{}".format( - reshaped_varname, i)) + slice_varnames.append( + "{}.block{}".format(reshaped_varname, i) + ) remote_varnames.append(reshaped_varname) block.append_op( @@ -519,32 +585,31 @@ def _save_sparse_params(self, executor, dirname, context, main_program): "remote_varnames": remote_varnames, "is_sparse": True, "endpoints": var_ctx.split_endpoints(), - "pserver_num": - len(self.role_maker._get_pserver_endpoints()), - "file_path": os.path.join(dirname, var.name) - }) + "pserver_num": len( + self.role_maker._get_pserver_endpoints() + ), + "file_path": os.path.join(dirname, var.name), + }, + ) for origin_varname in origin_varnames: - var = self.origin_main_program.global_block( - ).vars[origin_varname] - - block.append_op(type='recv_save', - attrs={ - "trainer_id": - self.role_maker._worker_index(), - "shape": - var.shape, - "slice_shapes": - [",".join([str(i) for i in var.shape])], - "slice_varnames": [origin_varname], - "remote_varnames": [origin_varname], - "is_sparse": - False, - "endpoints": - var_ctx.split_endpoints()[:1], - "file_path": - os.path.join(dirname, var.name) - }) + var = self.origin_main_program.global_block().vars[ + origin_varname + ] + + block.append_op( + type='recv_save', + attrs={ + "trainer_id": self.role_maker._worker_index(), + "shape": var.shape, + "slice_shapes": [",".join([str(i) for i in var.shape])], + "slice_varnames": [origin_varname], + "remote_varnames": [origin_varname], + "is_sparse": False, + "endpoints": var_ctx.split_endpoints()[:1], + "file_path": os.path.join(dirname, var.name), + }, + ) executor.run(prog) return context.keys() @@ -553,58 +618,71 @@ def _save_distributed_params(self, executor, dirname, context, mode): block = prog.global_block() for name, var_ctx in context.items(): - block.append_op(type='checkpoint_notify', - attrs={ - "varname": name, - "mode": mode, - "slice_varnames": var_ctx.split_varnames(), - "remote_varnames": var_ctx.split_varnames(), - "endpoints": var_ctx.split_endpoints(), - "dirname": dirname - }) + block.append_op( + type='checkpoint_notify', + attrs={ + "varname": name, + "mode": mode, + "slice_varnames": var_ctx.split_varnames(), + "remote_varnames": var_ctx.split_varnames(), + "endpoints": var_ctx.split_endpoints(), + "dirname": dirname, + }, + ) executor.run(prog) return context.keys() - def _save_distributed_persistables(self, executor, dirname, main_program, - mode): + def _save_distributed_persistables( + self, executor, dirname, main_program, mode + ): dense_ctx = self.compiled_strategy.get_communicator_recv_context( - recv_type=1, use_origin_program=True) + recv_type=1, use_origin_program=True + ) sparse_ctx = self.compiled_strategy.get_communicator_recv_context( - recv_type=2, use_origin_program=True) + recv_type=2, use_origin_program=True + ) distributed_ctx = self.compiled_strategy.get_communicator_recv_context( - recv_type=3, use_origin_program=True) + recv_type=3, use_origin_program=True + ) - recv_dense_varnames = self._save_dense_params(executor, dirname, - dense_ctx, main_program) + recv_dense_varnames = self._save_dense_params( + executor, dirname, dense_ctx, main_program + ) - recv_sparse_varnames = self._save_sparse_params(executor, dirname, - sparse_ctx, - main_program) + recv_sparse_varnames = self._save_sparse_params( + executor, dirname, sparse_ctx, main_program + ) recv_distributed_varnames = self._save_distributed_params( - executor, dirname, distributed_ctx, mode) + executor, dirname, distributed_ctx, mode + ) - saved_varnames = recv_dense_varnames + list( - recv_sparse_varnames) + list(recv_distributed_varnames) + saved_varnames = ( + recv_dense_varnames + + list(recv_sparse_varnames) + + list(recv_distributed_varnames) + ) remaining_vars = list( - filter(ParameterServerRuntime.__exclude_vars(saved_varnames), - main_program.list_vars())) - - fluid.io.save_vars(executor, - main_program=main_program, - dirname=dirname, - vars=remaining_vars) - - def _ps_inference_save_persistables(self, - executor, - dirname, - main_program=None, - mode=0, - **kwargs): + filter( + ParameterServerRuntime.__exclude_vars(saved_varnames), + main_program.list_vars(), + ) + ) + + fluid.io.save_vars( + executor, + main_program=main_program, + dirname=dirname, + vars=remaining_vars, + ) + + def _ps_inference_save_persistables( + self, executor, dirname, main_program=None, mode=0, **kwargs + ): """ This function filters out all variables with `persistable==True` from the give `main_program` and then saves these variables to the folder `dirname` @@ -634,16 +712,19 @@ def _ps_inference_save_persistables(self, "in fleet.save_persistables() function, main_program must be as Program type, CompiledProgram is not allowed" ) - self._save_distributed_persistables(executor, dirname, main_program, - mode) - - def _ps_inference_save_inference_model(self, - executor, - dirname, - feeded_var_names, - target_vars, - main_program=None, - export_for_deployment=True): + self._save_distributed_persistables( + executor, dirname, main_program, mode + ) + + def _ps_inference_save_inference_model( + self, + executor, + dirname, + feeded_var_names, + target_vars, + main_program=None, + export_for_deployment=True, + ): """ Prune the given `main_program` to build a new program especially for inference, and then save it and all related parameters to given `dirname` by the `executor`. @@ -664,14 +745,28 @@ def _ps_inference_save_inference_model(self, raise TypeError( "in fleet.save_inference_model() function, main_program must be as Program type, CompiledProgram is not allowed" ) - fluid.io.save_inference_model(dirname, feeded_var_names, - target_vars, executor, main_program, - None, None, export_for_deployment) + fluid.io.save_inference_model( + dirname, + feeded_var_names, + target_vars, + executor, + main_program, + None, + None, + export_for_deployment, + ) else: - fluid.io.save_inference_model(dirname, feeded_var_names, - target_vars, executor, - self.origin_main_program, None, None, - export_for_deployment, True) + fluid.io.save_inference_model( + dirname, + feeded_var_names, + target_vars, + executor, + self.origin_main_program, + None, + None, + export_for_deployment, + True, + ) model_basename = "__model__" model_filename = os.path.join(dirname, model_basename) @@ -681,10 +776,9 @@ def _ps_inference_save_inference_model(self, program = Program.parse_from_string(program_desc_str) program._copy_dist_param_info_from(fluid.default_main_program()) - self._ps_inference_save_persistables(executor, - dirname, - program, - mode=0) + self._ps_inference_save_persistables( + executor, dirname, program, mode=0 + ) def _save_inference_model(self, *args, **kwargs): self._ps_inference_save_inference_model(*args, **kwargs) diff --git a/python/paddle/distributed/fleet/runtime/runtime_base.py b/python/paddle/distributed/fleet/runtime/runtime_base.py index 38bb31ce3fc1d..2e8bacfbc3b1d 100644 --- a/python/paddle/distributed/fleet/runtime/runtime_base.py +++ b/python/paddle/distributed/fleet/runtime/runtime_base.py @@ -16,7 +16,6 @@ class RuntimeBase(object): - def __init__(self): pass diff --git a/python/paddle/distributed/fleet/runtime/the_one_ps.py b/python/paddle/distributed/fleet/runtime/the_one_ps.py index e9765c9e2e606..5a0be9a1e018f 100644 --- a/python/paddle/distributed/fleet/runtime/the_one_ps.py +++ b/python/paddle/distributed/fleet/runtime/the_one_ps.py @@ -35,8 +35,12 @@ def conv_indent(indent): def parse_table_class(varname, o_main_program): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import is_distributed_sparse_op - from paddle.fluid.incubate.fleet.parameter_server.ir.public import is_sparse_op + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + is_distributed_sparse_op, + ) + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + is_sparse_op, + ) for op in o_main_program.global_block().ops: if not is_distributed_sparse_op(op) and not is_sparse_op(op): @@ -44,7 +48,11 @@ def parse_table_class(varname, o_main_program): param_name = op.input("W")[0] - if param_name == varname and op.type == "lookup_table" or op.type == "lookup_table_v2": + if ( + param_name == varname + and op.type == "lookup_table" + or op.type == "lookup_table_v2" + ): if op.has_attr('table_class') and op.attr("table_class") != "none": return op.attr('table_class') else: @@ -90,7 +98,10 @@ def get_default_accessor_proto(accessor, varname, o_main_program): for sgd_param in [accessor.embed_sgd_param, accessor.embedx_sgd_param]: if not sgd_param.HasField("name"): sgd_param.name = "SparseAdaGradSGDRule" - if sgd_param.name == "SparseAdaGradSGDRule" or sgd_param.name == "StdAdaGradSGDRule": + if ( + sgd_param.name == "SparseAdaGradSGDRule" + or sgd_param.name == "StdAdaGradSGDRule" + ): if not sgd_param.adagrad.HasField("learning_rate"): sgd_param.adagrad.learning_rate = 0.05 if not sgd_param.adagrad.HasField("initial_g2sum"): @@ -130,17 +141,20 @@ def check_embedding_dim(accessor, varname, o_main_program): fea_dim = accessor.fea_dim if fea_dim != embedding_dim: raise ValueError( - "The fea_dim is wrong, it will be sparse_embedding_dim: {}, but got {}" - .format(embedding_dim, fea_dim)) + "The fea_dim is wrong, it will be sparse_embedding_dim: {}, but got {}".format( + embedding_dim, fea_dim + ) + ) embedx_dim = accessor.embedx_dim if embedx_dim != embedding_dim - 3: raise ValueError( - "The embedx_dim is wrong, it will be sparse_embedding_dim - 3: {}, but got {}" - .format(embedding_dim - 3, embedx_dim)) + "The embedx_dim is wrong, it will be sparse_embedding_dim - 3: {}, but got {}".format( + embedding_dim - 3, embedx_dim + ) + ) class Accessor: - def __init__(self): self.accessor_class = "" self.optimizer = None @@ -157,12 +171,12 @@ def to_string(self, indent): attrs += "\n" if self.optimizer is not None: attrs += self.optimizer.to_string(indent) - return accessor_str.format(conv_indent(indent), attrs, - conv_indent(indent)) + return accessor_str.format( + conv_indent(indent), attrs, conv_indent(indent) + ) class CommonAccessor: - def __init__(self): self.accessor_class = "" self.table_name = None @@ -183,26 +197,45 @@ def __init__(self): def define_optimize_map(self): opt_input_map = {} opt_input_map["sgd"] = [("Param", None), ("LearningRate", 1)] - opt_input_map["adam"] = [("Param", None), ("Moment1", None), - ("Moment2", None), ("Beta1Pow", 1), - ("Beta2Pow", 1), ("LearningRate", 1)] - opt_input_map["adam_d2sum"] = [("Param", None), ("D2Sum", None), - ("G2Sum", None), ("Moment", None), - ("MomentDecayRate", 1), - ("AdaDecayRate", 1), ("AdaEpsilon", 1), - ("LearningRate", 1)] + opt_input_map["adam"] = [ + ("Param", None), + ("Moment1", None), + ("Moment2", None), + ("Beta1Pow", 1), + ("Beta2Pow", 1), + ("LearningRate", 1), + ] + opt_input_map["adam_d2sum"] = [ + ("Param", None), + ("D2Sum", None), + ("G2Sum", None), + ("Moment", None), + ("MomentDecayRate", 1), + ("AdaDecayRate", 1), + ("AdaEpsilon", 1), + ("LearningRate", 1), + ] opt_input_map["sum"] = [("Param", None)] - opt_input_map["naive_adagrad"] = [("Param", None), ("G2Sum", 1), - ("LearningRate", 1)] + opt_input_map["naive_adagrad"] = [ + ("Param", None), + ("G2Sum", 1), + ("LearningRate", 1), + ] opt_attr_map = {} opt_attr_map["sgd"] = [] opt_attr_map["sum"] = [] opt_attr_map["naive_adagrad"] = [] - opt_attr_map["adam"] = [("beta1", "f"), ("beta2", "f"), - ("epsilon", "f")] - opt_attr_map["adam_d2sum"] = [("beta1", "f"), ("beta2", "f"), - ("epsilon", "f")] + opt_attr_map["adam"] = [ + ("beta1", "f"), + ("beta2", "f"), + ("epsilon", "f"), + ] + opt_attr_map["adam_d2sum"] = [ + ("beta1", "f"), + ("beta2", "f"), + ("epsilon", "f"), + ] opt_init_map = {} opt_init_map["gaussian_random"] = ["seed", "mean", "std"] @@ -215,8 +248,12 @@ def define_optimize_map(self): self.opt_init_map = opt_init_map def parse_entry(self, varname, o_main_program): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import is_distributed_sparse_op - from paddle.fluid.incubate.fleet.parameter_server.ir.public import is_sparse_op + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + is_distributed_sparse_op, + ) + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + is_sparse_op, + ) for op in o_main_program.global_block().ops: if not is_distributed_sparse_op(op) and not is_sparse_op(op): @@ -250,8 +287,10 @@ def get_initializer_attr(self, value_name, o_startup_program): origin_var_name = value_name for op in o_startup_program.global_block().ops: - if op.type in self.opt_init_map.keys( - ) and origin_var_name == op.output("Out")[0]: + if ( + op.type in self.opt_init_map.keys() + and origin_var_name == op.output("Out")[0] + ): init_attr = [op.type] for attr in self.opt_init_map[op.type]: init_attr.append(str(op.attr(attr))) @@ -259,9 +298,19 @@ def get_initializer_attr(self, value_name, o_startup_program): break return attr_str - def parse_by_optimizer(self, grad_name, is_sparse, size, single_dim, - compiled_strategy, adam_d2sum): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_optimize_ops + def parse_by_optimizer( + self, + grad_name, + is_sparse, + size, + single_dim, + compiled_strategy, + adam_d2sum, + ): + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_optimize_ops, + ) + param_name = compiled_strategy.grad_name_to_param_name[grad_name] main_program, startup_program = compiled_strategy.get_origin_programs() pserver_id = compiled_strategy.get_role_id() @@ -270,8 +319,9 @@ def parse_by_optimizer(self, grad_name, is_sparse, size, single_dim, oop = None for op in optimizer_ops: - if ("Param" in op.input_names) and (op.input("Param")[0] - == param_name): + if ("Param" in op.input_names) and ( + op.input("Param")[0] == param_name + ): oop = op break @@ -311,7 +361,7 @@ def parse_by_optimizer(self, grad_name, is_sparse, size, single_dim, for (formal_name, shape) in param_varnames: params.append(formal_name) if self.accessor_class == "adam_d2sum": - #for dims + # for dims if shape is None: if is_sparse: shape = single_dim @@ -319,18 +369,24 @@ def parse_by_optimizer(self, grad_name, is_sparse, size, single_dim, shape = self.get_shard(size, pserver_num, pserver_id) dims.append(shape) - #for initializers + # for initializers if formal_name == "Param" or formal_name == "LearningRate": - param = main_program.global_block().vars[oop.input( - formal_name)[0]] - #TODO: for dense learning_rate, can be different from sparse lr - if formal_name == "LearningRate" and param.name != "learning_rate_0": + param = main_program.global_block().vars[ + oop.input(formal_name)[0] + ] + # TODO: for dense learning_rate, can be different from sparse lr + if ( + formal_name == "LearningRate" + and param.name != "learning_rate_0" + ): warnings.warn("will support decay soon") - param = main_program.global_block( - ).vars["learning_rate_0"] + param = main_program.global_block().vars[ + "learning_rate_0" + ] initializer = self.get_initializer_attr( - param.name, startup_program) + param.name, startup_program + ) elif formal_name == "MomentDecayRate": initializer = "fill_constant&0.99" elif formal_name == "AdaDecayRate": @@ -346,23 +402,30 @@ def parse_by_optimizer(self, grad_name, is_sparse, size, single_dim, initializer = "fill_constant&0" initializers.append(initializer) else: - param = main_program.global_block().vars[oop.input( - formal_name)[0]] - if formal_name == "LearningRate" and param.name != "learning_rate_0": + param = main_program.global_block().vars[ + oop.input(formal_name)[0] + ] + if ( + formal_name == "LearningRate" + and param.name != "learning_rate_0" + ): warnings.warn("will support decay soon") - param = main_program.global_block( - ).vars["learning_rate_0"] + param = main_program.global_block().vars[ + "learning_rate_0" + ] if shape is None: if is_sparse: shape = single_dim else: - shape = self.get_shard(size, pserver_num, - pserver_id) + shape = self.get_shard( + size, pserver_num, pserver_id + ) dims.append(shape) initializer = self.get_initializer_attr( - param.name, startup_program) + param.name, startup_program + ) initializers.append(initializer) for (attr_varname, type_) in attr_varnames: @@ -401,12 +464,12 @@ def to_string(self, indent): attrs += "initializers: \"{}\" ".format(initializer) attrs += "\n" - return accessor_str.format(conv_indent(indent), attrs, - conv_indent(indent)) + return accessor_str.format( + conv_indent(indent), attrs, conv_indent(indent) + ) class Tensor: - def __init__(self): self.main_program_id = None self.startup_program_id = None @@ -422,14 +485,15 @@ def to_string(self, indent): attrs += "startup_program_id: {} ".format(str(self.startup_program_id)) attrs += "main_program_id: {} ".format(str(self.main_program_id)) attrs += "tensor_table_class: \"{}\" ".format( - str(self.tensor_table_class)) + str(self.tensor_table_class) + ) attrs += "\n" - return program_str.format(conv_indent(indent), attrs, - conv_indent(indent)) + return program_str.format( + conv_indent(indent), attrs, conv_indent(indent) + ) class Table: - def __init__(self): self.id = -1 self.table_class = None @@ -458,9 +522,9 @@ def to_string(self, indent): if self.accessor_proto is not None: accessor_str = "{}accessor {{{}\n{}}}" - accessor_str = accessor_str.format(conv_indent(indent), - self.accessor_proto, - conv_indent(indent)) + accessor_str = accessor_str.format( + conv_indent(indent), self.accessor_proto, conv_indent(indent) + ) attrs += accessor_str + "\n" elif self.accessor is not None: attrs += self.accessor.to_string(indent) @@ -478,7 +542,6 @@ def to_string(self, indent): class Service: - def __init__(self): self.server_class = "BrpcPsServer" self.client_class = "BrpcPsClient" @@ -496,12 +559,12 @@ def to_string(self, indent): attrs += "start_server_port: {} ".format(self.start_server_port) attrs += "server_thread_num: {} ".format(self.server_thread_num) - return service_str.format(conv_indent(indent), attrs, - conv_indent(indent)) + return service_str.format( + conv_indent(indent), attrs, conv_indent(indent) + ) class DownpourServer: - def __init__(self): self.service = None self.tables = [] @@ -526,12 +589,12 @@ def to_string(self, indent): for table in self.tables: table_strs += "\n" table_strs += table.to_string(indent) - return server_str.format(conv_indent(indent), table_strs, - conv_indent(indent)) + return server_str.format( + conv_indent(indent), table_strs, conv_indent(indent) + ) class Server: - def __init__(self): self.servers = [] @@ -552,7 +615,6 @@ def __str__(self): class DownpourWorker: - def __init__(self): self.tables = [] @@ -569,12 +631,12 @@ def to_string(self, indent): table_strs += "\n" table_strs += table.to_string(indent) - return worker_str.format(conv_indent(indent), table_strs, - conv_indent(indent)) + return worker_str.format( + conv_indent(indent), table_strs, conv_indent(indent) + ) class Worker: - def __init__(self): self.workers = [] @@ -595,7 +657,6 @@ def __str__(self): class fsClient: - def __init__(self, proto): self.proto = proto self.uri = proto.uri @@ -605,6 +666,7 @@ def __init__(self, proto): def to_string(self): from google.protobuf import text_format + proto_txt = text_format.MessageToString(self.proto) if proto_txt: fs_str = "fs_client_param {{\n{}}}" @@ -614,7 +676,6 @@ def to_string(self): class TheOnePSRuntime(RuntimeBase): - def __init__(self): super(TheOnePSRuntime, self).__init__() self._communicator = None @@ -634,8 +695,9 @@ def _set_basic_info(self, context): def _get_distributed_strategy(self): strategy = None - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import \ - StrategyFactory + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + StrategyFactory, + ) dist_strategy = self.context["valid_strategy"] k_steps = dist_strategy.a_sync_configs["k_steps"] @@ -657,19 +719,24 @@ def _get_distributed_strategy(self): return strategy def build_compiled_startegy(self): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import CompileTimeStrategy - - compiled_config = CompileTimeStrategy(self.origin_main_program, - self.origin_main_program, - self.async_strategy, - self.role_maker) + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + CompileTimeStrategy, + ) + + compiled_config = CompileTimeStrategy( + self.origin_main_program, + self.origin_main_program, + self.async_strategy, + self.role_maker, + ) if self.async_strategy.use_ps_gpu: compiled_config.use_ps_gpu = True return compiled_config def _init_worker(self): - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import \ - SyncStrategy + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + SyncStrategy, + ) is_sync = self.compiled_strategy.is_sync_mode() worker = self._get_fleet_proto(is_server=False, is_sync=is_sync) @@ -690,7 +757,8 @@ def _init_worker(self): def sync_strategy_envs(): kwargs = {} kwargs[ - "pserver_endpoints"] = self.role_maker._get_pserver_endpoints() + "pserver_endpoints" + ] = self.role_maker._get_pserver_endpoints() kwargs["trainer_id"] = self.role_maker._worker_index() return kwargs @@ -712,11 +780,13 @@ def sync_strategy_envs(): string_hosts.append(pshost.serialize_to_string()) dense_map = self.compiled_strategy.get_the_one_recv_context( - split_dense_table=self.role_maker._is_heter_parameter_server_mode) + split_dense_table=self.role_maker._is_heter_parameter_server_mode + ) send_ctx = self.compiled_strategy.get_the_one_send_context( split_dense_table=self.role_maker._is_heter_parameter_server_mode, use_origin_program=self.role_maker._is_heter_parameter_server_mode, - ep_list=endpoints) + ep_list=endpoints, + ) trainer_config = self.async_strategy.get_trainer_runtime_config() debug = bool(int(os.getenv("PSERVER_DEBUG", "0"))) @@ -732,7 +802,7 @@ def sync_strategy_envs(): kwargs['need_global_step'] = "0" kwargs["trainer_id"] = self.role_maker._role_id() kwargs["trainers"] = self.role_maker._worker_num() - #if self.role_maker._is_heter_worker(): + # if self.role_maker._is_heter_worker(): # kwargs["trainer_id"] += kwargs["trainers"] for table in server.servers[0].tables: @@ -745,13 +815,16 @@ def sync_strategy_envs(): kwargs.update(sync_kwargs) from paddle.fluid.communicator import Communicator, HeterClient + self._communicator = Communicator( - trainer_config.mode, kwargs, - trainer_config.get_communicator_flags()) - self._communicator.init_with_ctx(send_ctx, dense_map, proto_txt, - string_hosts, fluid.global_scope()) + trainer_config.mode, kwargs, trainer_config.get_communicator_flags() + ) + self._communicator.init_with_ctx( + send_ctx, dense_map, proto_txt, string_hosts, fluid.global_scope() + ) import paddle.distributed.fleet as fleet + fleet.util.barrier() info = self._communicator.get_client_info() if isinstance(info, list) and len(info) > 0: @@ -774,11 +847,14 @@ def sync_strategy_envs(): is_test = bool(int(os.getenv("TEST_MODE", "0"))) - if self.role_maker._is_first_worker( - ) and self.role_maker._is_heter_parameter_server_mode: + if ( + self.role_maker._is_first_worker() + and self.role_maker._is_heter_parameter_server_mode + ): # for ps-heter mode load all parameters on first_worker init_params = self.compiled_strategy.get_the_one_recv_context( - split_dense_table=True, use_origin_program=True) + split_dense_table=True, use_origin_program=True + ) else: init_params = dense_map @@ -798,8 +874,10 @@ def sync_strategy_envs(): if launch_barrier and launch_barrier_flag: # for trainer wait server ready wait_server_ready(self.role_maker._get_pserver_endpoints()) - if self.role_maker._is_heter_parameter_server_mode and self.role_maker._get_next_trainers( - ) != []: + if ( + self.role_maker._is_heter_parameter_server_mode + and self.role_maker._get_next_trainers() != [] + ): wait_server_ready(self.role_maker._get_next_trainers()) if self.role_maker._is_heter_parameter_server_mode: previous_trainers = [] @@ -808,14 +886,13 @@ def sync_strategy_envs(): next_trainers = [] if self.role_maker._get_next_trainers() != []: next_trainers = self.role_maker._get_next_trainers() - self._heter_client = HeterClient(next_trainers, - previous_trainers, - self.role_maker._role_id()) - - def _push_sparse_param(self, - var_name, - table_id=-1, - scope=fluid.global_scope()): + self._heter_client = HeterClient( + next_trainers, previous_trainers, self.role_maker._role_id() + ) + + def _push_sparse_param( + self, var_name, table_id=-1, scope=fluid.global_scope() + ): self._communicator.push_sparse_param(var_name, table_id, scope) def _get_executor(self): @@ -825,20 +902,23 @@ def _get_executor(self): heter_device_type = self.role_maker._heter_device_type().upper() if heter_device_type not in ["GPU", "XPU", "CPU"]: raise ValueError( - "Heter Worker Not Support Device {}".format( - device_type)) + "Heter Worker Not Support Device {}".format(device_type) + ) if heter_device_type == "GPU": executor = Executor( fluid.CUDAPlace( - int(os.getenv("FLAGS_selected_gpus", "0")))) + int(os.getenv("FLAGS_selected_gpus", "0")) + ) + ) elif heter_device_type == "XPU": executor = Executor( fluid.XPUPlace( - int(os.getenv("FLAGS_selected_xpus", "0")))) + int(os.getenv("FLAGS_selected_xpus", "0")) + ) + ) return executor def _get_fleet_proto(self, is_server, is_sync, **kwargs): - def _build_merge_accessor(ctx): accessor = Accessor() accessor.accessor_class = "CommMergeAccessor" @@ -872,7 +952,8 @@ def _build_barrier_table(idx): trainer_num = self.compiled_strategy.get_trainers() if self.role_maker._is_heter_parameter_server_mode: trainer_num += len( - self.role_maker._get_heter_worker_endpoints()) + self.role_maker._get_heter_worker_endpoints() + ) common.trainer_num = trainer_num common.attrs = "" common.dims = [] @@ -918,27 +999,32 @@ def _add_tensor_table(tables): for table_name in tensor_table_dict: if tensor_table_dict[table_name]["startup_program"] != None: tensor_table_dict[table_name][ - "startup_program_id"] = program_idx + "startup_program_id" + ] = program_idx self._server_sub_program.append( - tensor_table_dict[table_name]["startup_program"].desc) + tensor_table_dict[table_name]["startup_program"].desc + ) program_idx += 1 if tensor_table_dict[table_name]["main_program"] != None: tensor_table_dict[table_name][ - "main_program_id"] = program_idx + "main_program_id" + ] = program_idx self._server_sub_program.append( - tensor_table_dict[table_name]["main_program"].desc) + tensor_table_dict[table_name]["main_program"].desc + ) program_idx += 1 # Todo: Hard code for lr_decay table apply table id - new_table = _build_tensor_table(len(tables), - tensor_table_dict[table_name]) + new_table = _build_tensor_table( + len(tables), tensor_table_dict[table_name] + ) tables.append(new_table) return tables def _get_tables(): send_ctx = self.compiled_strategy.get_the_one_send_context( use_origin_program=True, - split_dense_table=self.role_maker. - _is_heter_parameter_server_mode) + split_dense_table=self.role_maker._is_heter_parameter_server_mode, + ) tables = [] for idx, (name, ctx) in enumerate(send_ctx.items()): @@ -953,14 +1039,18 @@ def _get_tables(): table.type = "PS_SPARSE_TABLE" table.shard_num = 256 - common.table_name = self.compiled_strategy.grad_name_to_param_name[ - ctx.origin_varnames()[0]] + common.table_name = ( + self.compiled_strategy.grad_name_to_param_name[ + ctx.origin_varnames()[0] + ] + ) if self.compiled_strategy.is_geo_mode(): table.table_class = "MemorySparseGeoTable" else: all_table_proto = self.context[ - "user_defined_strategy"].sparse_table_configs + "user_defined_strategy" + ].sparse_table_configs table_proto = all_table_proto.add() for proto in all_table_proto: if proto.table_name == common.table_name: @@ -970,11 +1060,13 @@ def _get_tables(): table.table_class = table_proto.table_class else: table.table_class = parse_table_class( - common.table_name, self.origin_main_program) + common.table_name, self.origin_main_program + ) if table.table_class != 'MemorySparseTable': table.table_class = 'MemorySparseTable' warnings.warn( - "The PS mode must use MemorySparseTable.") + "The PS mode must use MemorySparseTable." + ) if table_proto.HasField("shard_num"): table.shard_num = table_proto.shard_num @@ -988,15 +1080,21 @@ def _get_tables(): warnings.warn( "The accessor of sparse table is not set, use default value." ) - get_default_accessor_proto(table_proto.accessor, - common.table_name, - self.origin_main_program) - check_embedding_dim(table_proto.accessor, - common.table_name, - self.origin_main_program) + get_default_accessor_proto( + table_proto.accessor, + common.table_name, + self.origin_main_program, + ) + check_embedding_dim( + table_proto.accessor, + common.table_name, + self.origin_main_program, + ) from google.protobuf import text_format + table.accessor_proto = text_format.MessageToString( - table_proto.accessor) + table_proto.accessor + ) else: table.type = "PS_DENSE_TABLE" table.table_class = "MemoryDenseTable" @@ -1005,14 +1103,18 @@ def _get_tables(): adam_d2sum = self.context["user_defined_strategy"].adam_d2sum common.parse_by_optimizer( - ctx.origin_varnames()[0], ctx.is_sparse(), + ctx.origin_varnames()[0], + ctx.is_sparse(), ctx.sections()[0], ctx.sections()[1] if ctx.is_sparse() else 1, - self.compiled_strategy, adam_d2sum) + self.compiled_strategy, + adam_d2sum, + ) if ctx.is_sparse(): - common.parse_entry(common.table_name, - self.origin_main_program) + common.parse_entry( + common.table_name, self.origin_main_program + ) if is_sync: common.sync = "true" @@ -1071,7 +1173,8 @@ def _init_server(self, dirname=None, var_names=None, **kwargs): server = self._get_fleet_proto(is_server=True, is_sync=is_sync) proto_txt = str(server) fs_client = fsClient( - self.context["user_defined_strategy"].fs_client_param) + self.context["user_defined_strategy"].fs_client_param + ) proto_txt = proto_txt + "\n" + fs_client.to_string() debug = bool(int(os.getenv("PSERVER_DEBUG", "0"))) @@ -1085,10 +1188,13 @@ def _init_server(self, dirname=None, var_names=None, **kwargs): string_hosts.append(pshost.serialize_to_string()) self._server = fluid.core.DistFleetWrapper() - self._server.init_server(proto_txt, string_hosts, role_id, trainers, - self._server_sub_program) + self._server.init_server( + proto_txt, string_hosts, role_id, trainers, self._server_sub_program + ) - from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablenames + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablenames, + ) dist_varnames = get_sparse_tablenames(self.origin_main_program, True) sparse_varnames = get_sparse_tablenames(self.origin_main_program, False) @@ -1101,8 +1207,10 @@ def _init_server(self, dirname=None, var_names=None, **kwargs): for var_name in var_names: if var_name not in distributed_varnames: raise ValueError( - "fleet.init server can only load sparse variables in {}" - .format(distributed_varnames)) + "fleet.init server can only load sparse variables in {}".format( + distributed_varnames + ) + ) load_varnames = var_names if dirname is None or not load_varnames: @@ -1132,19 +1240,22 @@ def _run_server(self): def _stop_worker(self): self._communicator.stop() if self.role_maker._is_heter_parameter_server_mode: - assert self._heter_client != None, "heter client should not be None in heterps mode" + assert ( + self._heter_client != None + ), "heter client should not be None in heterps mode" self._heter_client.stop() - #executor = self._get_executor() - #executor.close() + # executor = self._get_executor() + # executor.close() @staticmethod def __exclude_vars(exclude_var_names=[]): - def is_valid(var): if var.name in exclude_var_names: return False - from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_varname_parts + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_varname_parts, + ) origin_varname, _, _ = _get_varname_parts(var.name) if origin_varname.endswith("@GRAD"): @@ -1153,9 +1264,11 @@ def is_valid(var): if origin_varname == "learning_rate_0": return False - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable @@ -1168,11 +1281,16 @@ def _get_inference_model_path(self, dirname): model_path = os.path.join(dirname, "dnn_plugin") return model_path - def _save_sparse_params(self, executor, dirname, context, main_program, - mode): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablenames + def _save_sparse_params( + self, executor, dirname, context, main_program, mode + ): + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablenames, + ) + distributed_varnames = get_sparse_tablenames( - self.compiled_strategy.origin_main_program, True) + self.compiled_strategy.origin_main_program, True + ) values = [] model_path = self._get_inference_model_path(dirname) for id, names in context.items(): @@ -1188,23 +1306,24 @@ def _save_sparse_params(self, executor, dirname, context, main_program, # self._worker.save_all_model(dirname, mode) return values - def _save_distributed_persistables(self, - executor, - dirname, - main_program, - mode=0): + def _save_distributed_persistables( + self, executor, dirname, main_program, mode=0 + ): denses = self.compiled_strategy.get_the_one_recv_context( is_dense=True, split_dense_table=self.role_maker._is_heter_parameter_server_mode, - use_origin_program=True) + use_origin_program=True, + ) sparses = self.compiled_strategy.get_the_one_recv_context( is_dense=False, split_dense_table=self.role_maker._is_heter_parameter_server_mode, - use_origin_program=True) + use_origin_program=True, + ) - sparse_varnames = self._save_sparse_params(executor, dirname, sparses, - main_program, mode) + sparse_varnames = self._save_sparse_params( + executor, dirname, sparses, main_program, mode + ) recv_dense_varnames = [] for id, names in denses.items(): @@ -1214,24 +1333,25 @@ def _save_distributed_persistables(self, saved_varnames = sparse_varnames remaining_vars = list( - filter(TheOnePSRuntime.__exclude_vars(saved_varnames), - main_program.list_vars())) + filter( + TheOnePSRuntime.__exclude_vars(saved_varnames), + main_program.list_vars(), + ) + ) import paddle + for var in remaining_vars: # if var.name not in recv_dense_varnames: # continue tensor = var.get_value() - paddle.save(tensor, - os.path.join(dirname, var.name), - use_binary_format=True) - - def _ps_inference_save_persistables(self, - executor, - dirname, - main_program=None, - mode=0, - **kwargs): + paddle.save( + tensor, os.path.join(dirname, var.name), use_binary_format=True + ) + + def _ps_inference_save_persistables( + self, executor, dirname, main_program=None, mode=0, **kwargs + ): """ This function filters out all variables with `persistable==True` from the give `main_program` and then saves these variables to the folder `dirname` @@ -1250,7 +1370,8 @@ def _ps_inference_save_persistables(self, if not isinstance(executor, Executor): raise TypeError( - "in fleet.save() function, executor must be as Executor type") + "in fleet.save() function, executor must be as Executor type" + ) if main_program is None: main_program = self.compiled_strategy.get_origin_ps_main_program() @@ -1265,14 +1386,16 @@ def _ps_inference_save_persistables(self, # mode) self._worker.save_all_model(dirname, mode) - def _ps_inference_save_inference_model(self, - executor, - dirname, - feeded_var_names, - target_vars, - main_program=None, - export_for_deployment=True, - mode=0): + def _ps_inference_save_inference_model( + self, + executor, + dirname, + feeded_var_names, + target_vars, + main_program=None, + export_for_deployment=True, + mode=0, + ): """ Prune the given `main_program` to build a new program especially for inference, and then save it and all related parameters to given `dirname` by the `executor`. @@ -1285,10 +1408,14 @@ def _ps_inference_save_inference_model(self, if not isinstance(executor, Executor): raise TypeError( - "in fleet.save() function, executor must be as Executor type") + "in fleet.save() function, executor must be as Executor type" + ) import paddle - program = self.origin_main_program if main_program is None else main_program + + program = ( + self.origin_main_program if main_program is None else main_program + ) if isinstance(program, CompiledProgram): raise TypeError( @@ -1299,8 +1426,9 @@ def _ps_inference_save_inference_model(self, program.global_block().var(name) for name in feeded_var_names ] - infer_program = paddle.static.normalize_program(program, feed_vars, - target_vars) + infer_program = paddle.static.normalize_program( + program, feed_vars, target_vars + ) infer_program._copy_dist_param_info_from(program) @@ -1312,29 +1440,38 @@ def _ps_inference_save_inference_model(self, sparses = self.compiled_strategy.get_the_one_recv_context( is_dense=False, split_dense_table=self.role_maker._is_heter_parameter_server_mode, - use_origin_program=True) - sparse_names = self._save_sparse_params(executor, dirname, sparses, - main_program, mode) + use_origin_program=True, + ) + sparse_names = self._save_sparse_params( + executor, dirname, sparses, main_program, mode + ) denses = self.compiled_strategy.get_the_one_recv_context( is_dense=True, split_dense_table=self.role_maker._is_heter_parameter_server_mode, - use_origin_program=True) + use_origin_program=True, + ) # TODO(zhaocaibei123): for GEO: should call GeoCommunicator::RecvDense self._communicator.pull_dense(denses) generate_vars = self.context[ - "user_defined_strategy"].trainer_desc_configs["stat_var_names"] + "user_defined_strategy" + ].trainer_desc_configs["stat_var_names"] generate_vars = [var for var in generate_vars] remaining_vars = list( - filter(TheOnePSRuntime.__exclude_vars(sparse_names), - infer_program.list_vars())) + filter( + TheOnePSRuntime.__exclude_vars(sparse_names), + infer_program.list_vars(), + ) + ) for var in remaining_vars: tensor = var.get_value() - paddle.save(tensor, - os.path.join(model_path, var.name), - use_binary_format=True) + paddle.save( + tensor, + os.path.join(model_path, var.name), + use_binary_format=True, + ) def _save_inference_model(self, *args, **kwargs): self._ps_inference_save_inference_model(*args, **kwargs) @@ -1343,9 +1480,13 @@ def _save_persistables(self, *args, **kwargs): self._ps_inference_save_persistables(*args, **kwargs) def _load_sparse_params(self, dirname, context, main_program, mode): - from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablenames + from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablenames, + ) + distributed_varnames = get_sparse_tablenames( - self.compiled_strategy.origin_main_program, True) + self.compiled_strategy.origin_main_program, True + ) values = [] for id, names in context.items(): if names[0] not in distributed_varnames: @@ -1356,10 +1497,9 @@ def _load_sparse_params(self, dirname, context, main_program, mode): values.extend(names) return values - def _ps_inference_load_inference_model(self, - dirname, - mode=0, - main_program=None): + def _ps_inference_load_inference_model( + self, dirname, mode=0, main_program=None + ): if main_program is None: main_program = self.compiled_strategy.get_origin_ps_main_program() @@ -1371,14 +1511,17 @@ def _ps_inference_load_inference_model(self, denses = self.compiled_strategy.get_the_one_recv_context( is_dense=True, split_dense_table=self.role_maker._is_heter_parameter_server_mode, - use_origin_program=True) + use_origin_program=True, + ) sparses = self.compiled_strategy.get_the_one_recv_context( is_dense=False, split_dense_table=self.role_maker._is_heter_parameter_server_mode, - use_origin_program=True) + use_origin_program=True, + ) - sparse_varnames = self._load_sparse_params(dirname, sparses, - main_program, mode) + sparse_varnames = self._load_sparse_params( + dirname, sparses, main_program, mode + ) recv_dense_varnames = [] for id, names in denses.items(): @@ -1387,14 +1530,18 @@ def _ps_inference_load_inference_model(self, loaded_varnames = sparse_varnames remaining_vars = list( - filter(TheOnePSRuntime.__exclude_vars(loaded_varnames), - main_program.list_vars())) + filter( + TheOnePSRuntime.__exclude_vars(loaded_varnames), + main_program.list_vars(), + ) + ) if dirname.startswith("afs:") or dirname.startswith("hdfs:"): model_path = "./dnn_plugin" else: model_path = os.path.join(dirname, "dnn_plugin") import paddle + for var in remaining_vars: if var.name not in recv_dense_varnames: continue @@ -1421,13 +1568,14 @@ def _shrink(self, threshold=None): else: threshold = 0 import paddle.distributed.fleet as fleet + fleet.util.barrier() if self.role_maker._is_first_worker(): sparses = self.compiled_strategy.get_the_one_recv_context( is_dense=False, - split_dense_table=self.role_maker. - _is_heter_parameter_server_mode, - use_origin_program=True) + split_dense_table=self.role_maker._is_heter_parameter_server_mode, + use_origin_program=True, + ) for id, names in sparses.items(): self._worker.shrink_sparse_table(id, threshold) diff --git a/python/paddle/distributed/fleet/scaler.py b/python/paddle/distributed/fleet/scaler.py index 583c2819d8df0..60bc82016802c 100644 --- a/python/paddle/distributed/fleet/scaler.py +++ b/python/paddle/distributed/fleet/scaler.py @@ -23,12 +23,12 @@ def distributed_scaler(scaler): - def unscale_method(self, optimizer): if not self._enable: return if getattr(optimizer, '_param_groups', None) and isinstance( - optimizer._param_groups[0], dict): + optimizer._param_groups[0], dict + ): param_grads = [] param_grads_fp16 = [] param_grads_fp32 = [] @@ -36,38 +36,47 @@ def unscale_method(self, optimizer): for param in group['params']: if param._grad_ivar() is not None: param_grads.append(param._grad_ivar()) - if param._grad_ivar( - ).dtype == core.VarDesc.VarType.FP16: + if ( + param._grad_ivar().dtype + == core.VarDesc.VarType.FP16 + ): param_grads_fp16.append(param._grad_ivar()) else: param_grads_fp32.append(param._grad_ivar()) else: param_grads = [ - param._grad_ivar() for param in optimizer._parameter_list + param._grad_ivar() + for param in optimizer._parameter_list if param._grad_ivar() is not None ] param_grads_fp16 = [ - param._grad_ivar() for param in optimizer._parameter_list - if (param._grad_ivar() is not None) and ( - param._grad_ivar().dtype == core.VarDesc.VarType.FP16) + param._grad_ivar() + for param in optimizer._parameter_list + if (param._grad_ivar() is not None) + and (param._grad_ivar().dtype == core.VarDesc.VarType.FP16) ] param_grads_fp32 = [ - param._grad_ivar() for param in optimizer._parameter_list - if (param._grad_ivar() is not None) and ( - param._grad_ivar().dtype == core.VarDesc.VarType.FP32) + param._grad_ivar() + for param in optimizer._parameter_list + if (param._grad_ivar() is not None) + and (param._grad_ivar().dtype == core.VarDesc.VarType.FP32) ] temp_found_inf_fp16 = to_variable(np.array([0]).astype(np.bool_)) temp_found_inf_fp32 = to_variable(np.array([0]).astype(np.bool_)) if len(param_grads_fp16): - _legacy_C_ops.check_finite_and_unscale(param_grads_fp16, - self._scale, - param_grads_fp16, - temp_found_inf_fp16) + _legacy_C_ops.check_finite_and_unscale( + param_grads_fp16, + self._scale, + param_grads_fp16, + temp_found_inf_fp16, + ) if len(param_grads_fp32): - _legacy_C_ops.check_finite_and_unscale(param_grads_fp32, - self._scale, - param_grads_fp32, - temp_found_inf_fp32) + _legacy_C_ops.check_finite_and_unscale( + param_grads_fp32, + self._scale, + param_grads_fp32, + temp_found_inf_fp32, + ) self._found_inf = 1 if temp_found_inf_fp16 or temp_found_inf_fp32 else 0 is_found_inf = paddle.to_tensor([self._found_inf], dtype="int32") @@ -75,9 +84,9 @@ def unscale_method(self, optimizer): # TODO(shenliang03) Since dp allreduce in the optimizer is # after the gradscaler, check_finite needs to synchronize global # information. In the future, we should use check_group to speed. - paddle.distributed.all_reduce(is_found_inf, - op=paddle.distributed.ReduceOp.MAX, - group=None) + paddle.distributed.all_reduce( + is_found_inf, op=paddle.distributed.ReduceOp.MAX, group=None + ) self._found_inf = is_found_inf.numpy()[0] # Only data_parallel doesn't need to modify scaler diff --git a/python/paddle/distributed/fleet/utils/__init__.py b/python/paddle/distributed/fleet/utils/__init__.py index 93fc890d05af5..340370e5e55ea 100644 --- a/python/paddle/distributed/fleet/utils/__init__.py +++ b/python/paddle/distributed/fleet/utils/__init__.py @@ -22,14 +22,14 @@ from . import log_util # noqa: F401 from . import hybrid_parallel_util # noqa: F401 -__all__ = [ #noqa - "LocalFS", "recompute", "DistributedInfer", "HDFSClient" -] +__all__ = ["LocalFS", "recompute", "DistributedInfer", "HDFSClient"] # noqa -@deprecated(since="2.4.0", - update_to="paddle.distributed.fleet.recompute", - level=1, - reason="Please use new recompute API(fleet.recompute) ") +@deprecated( + since="2.4.0", + update_to="paddle.distributed.fleet.recompute", + level=1, + reason="Please use new recompute API(fleet.recompute) ", +) def recompute(function, *args, **kwargs): return fleet.recompute.recompute(function, *args, **kwargs) diff --git a/python/paddle/distributed/fleet/utils/fs.py b/python/paddle/distributed/fleet/utils/fs.py index cf0a6475d0e57..9daddcdb8568a 100644 --- a/python/paddle/distributed/fleet/utils/fs.py +++ b/python/paddle/distributed/fleet/utils/fs.py @@ -47,7 +47,6 @@ class FSShellCmdAborted(ExecuteError): class FS(object): - @abc.abstractmethod def ls_dir(self, fs_path): raise NotImplementedError @@ -171,7 +170,8 @@ def mkdirs(self, fs_path): client.delete("test_mkdirs") """ assert not os.path.isfile(fs_path), "{} is already a file".format( - fs_path) + fs_path + ) os.system("mkdir -p {}".format(fs_path)) def rename(self, fs_src_path, fs_dst_path): @@ -379,9 +379,7 @@ def list_dirs(self, fs_path): def _handle_errors(max_time_out=None): - def decorator(f): - @functools.wraps(f) def handler(*args, **kwargs): o = args[0] @@ -400,16 +398,20 @@ def handler(*args, **kwargs): # important: only ExecuteError need to retry except ExecuteError as e: if time.time() - start >= time_out: - raise FSTimeOut("args:{} timeout:{}".format( - args, - time.time() - start)) + raise FSTimeOut( + "args:{} timeout:{}".format( + args, time.time() - start + ) + ) time.sleep(inter) if time.time() - last_print_time > 30: - print("hadoop operator timeout:args:{} timeout:{}".format( - args, - time.time() - start)) + print( + "hadoop operator timeout:args:{} timeout:{}".format( + args, time.time() - start + ) + ) last_print_time = time.time() return handler @@ -443,11 +445,12 @@ class HDFSClient(FS): """ def __init__( - self, - hadoop_home, - configs, - time_out=5 * 60 * 1000, # ms - sleep_inter=1000): # ms + self, + hadoop_home, + configs, + time_out=5 * 60 * 1000, # ms + sleep_inter=1000, + ): # ms self.pre_commands = [] hadoop_bin = '%s/bin/hadoop' % hadoop_home self.pre_commands.append(hadoop_bin) @@ -463,7 +466,8 @@ def __init__( self._sleep_inter = sleep_inter self._base_cmd = " ".join(self.pre_commands) self._bd_err_re = re.compile( - r'\s?responseErrorMsg\s?\:.*, errorCode\:\s?[0-9]+, path\:') + r'\s?responseErrorMsg\s?\:.*, errorCode\:\s?[0-9]+, path\:' + ) def _run_cmd(self, cmd, redirect_stderr=False, retry_times=5): exe_cmd = "{} -{}".format(self._base_cmd, cmd) @@ -771,8 +775,9 @@ def get_local_files(path): procs = [] for i in range(multi_processes): process_datas = self._split_files(all_files, i, multi_processes) - p = multiprocessing.Process(target=__subprocess_upload, - args=(fs_path, process_datas)) + p = multiprocessing.Process( + target=__subprocess_upload, args=(fs_path, process_datas) + ) procs.append(p) p.start() @@ -841,8 +846,9 @@ def __subprocess_download(local_path, datas): procs = [] for i in range(multi_processes): process_datas = self._split_files(all_files, i, multi_processes) - p = multiprocessing.Process(target=__subprocess_download, - args=(local_path, process_datas)) + p = multiprocessing.Process( + target=__subprocess_download, args=(local_path, process_datas) + ) procs.append(p) p.start() @@ -938,7 +944,8 @@ def mv(self, fs_src_path, fs_dst_path, overwrite=False, test_exists=True): if test_exists: if not self.is_exist(fs_src_path): raise FSFileNotExistsError( - "{} is not exists".format(fs_src_path)) + "{} is not exists".format(fs_src_path) + ) if self.is_exist(fs_dst_path): raise FSFileExistsError("{} exists already".format(fs_dst_path)) @@ -954,8 +961,7 @@ def _try_mv(self, fs_src_path, fs_dst_path): if ret != 0: raise ExecuteError(cmd) except Exception as e: - if not self.is_exist(fs_src_path) and \ - self.is_exist(fs_dst_path): + if not self.is_exist(fs_src_path) and self.is_exist(fs_dst_path): return raise e @@ -1103,7 +1109,7 @@ def _split_files(self, files, trainer_id, trainers): trainer_files = [[]] * trainers begin = 0 for i in range(trainers): - trainer_files[i] = files[begin:begin + blocks[i]] + trainer_files[i] = files[begin : begin + blocks[i]] begin += blocks[i] return trainer_files[trainer_id] @@ -1121,13 +1127,15 @@ def list_files_info(self, path_list): file_list = [] - #concat filelist can speed up 'hadoop ls' + # concat filelist can speed up 'hadoop ls' str_concat = "" for path in path_list: str_concat += path + " " - cmd = "ls " + str_concat + " | awk '{if ($8 != \"\") {print $5\" \"$8 }}'" + cmd = ( + "ls " + str_concat + " | awk '{if ($8 != \"\") {print $5\" \"$8 }}'" + ) ret, lines = self._run_cmd(cmd) - if (len(lines) == 0): + if len(lines) == 0: logger.warning("list_files empty, path[%s]" % path_list) return [] for line in lines: @@ -1155,10 +1163,7 @@ class AFSClient(FS): client.ls_dir("hdfs:/test_hdfs_client") """ - def __init__( - self, - time_out=5 * 60 * 1000, # ms - sleep_inter=1000): # ms + def __init__(self, time_out=5 * 60 * 1000, sleep_inter=1000): # ms # ms self._fs = core.AfsWrapper() self._time_out = time_out @@ -1392,8 +1397,9 @@ def __subprocess_download(local_path, datas): procs = [] for i in range(multi_processes): process_datas = self._split_files(all_files, i, multi_processes) - p = multiprocessing.Process(target=__subprocess_download, - args=(local_path, process_datas)) + p = multiprocessing.Process( + target=__subprocess_download, args=(local_path, process_datas) + ) procs.append(p) p.start() @@ -1448,7 +1454,8 @@ def mv(self, fs_src_path, fs_dst_path, overwrite=False, test_exists=True): if test_exists: if not self.is_exist(fs_src_path): raise FSFileNotExistsError( - "{} is not exists".format(fs_src_path)) + "{} is not exists".format(fs_src_path) + ) if self.is_exist(fs_dst_path): raise FSFileExistsError("{} exists already".format(fs_dst_path)) @@ -1552,7 +1559,7 @@ def _split_files(self, files, trainer_id, trainers): trainer_files = [[]] * trainers begin = 0 for i in range(trainers): - trainer_files[i] = files[begin:begin + blocks[i]] + trainer_files[i] = files[begin : begin + blocks[i]] begin += blocks[i] return trainer_files[trainer_id] diff --git a/python/paddle/distributed/fleet/utils/http_server.py b/python/paddle/distributed/fleet/utils/http_server.py index a1251c46f3c89..1dd78136d9809 100644 --- a/python/paddle/distributed/fleet/utils/http_server.py +++ b/python/paddle/distributed/fleet/utils/http_server.py @@ -35,9 +35,9 @@ def get_logger(name, level, fmt): return logger -_http_server_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_http_server_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class KVHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): @@ -167,7 +167,8 @@ def start(self): start server until user calls stop to let it quit. """ self.listen_thread = threading.Thread( - target=lambda: self.http_server.serve_forever()) + target=lambda: self.http_server.serve_forever() + ) self.listen_thread.start() def stop(self): diff --git a/python/paddle/distributed/fleet/utils/hybrid_parallel_inference.py b/python/paddle/distributed/fleet/utils/hybrid_parallel_inference.py index fd81c5b11f4a0..0c5bff02ed820 100644 --- a/python/paddle/distributed/fleet/utils/hybrid_parallel_inference.py +++ b/python/paddle/distributed/fleet/utils/hybrid_parallel_inference.py @@ -181,15 +181,17 @@ class HybridParallelInferenceHelper(object): print(res) """ - def __init__(self, - startup_program, - main_program, - num_mp=1, - num_pp=1, - micro_batch_size=1, - beam_size=1, - init_comm=True, - role_maker=None): + def __init__( + self, + startup_program, + main_program, + num_mp=1, + num_pp=1, + micro_batch_size=1, + beam_size=1, + init_comm=True, + role_maker=None, + ): assert isinstance(startup_program, Program) assert isinstance(main_program, Program) @@ -225,7 +227,8 @@ def __init__(self, if role_maker is None: self.role_maker = fleet.base.role_maker.PaddleCloudRoleMaker( - is_collective=True) + is_collective=True + ) else: if isinstance(role_maker, fleet.base.role_maker.RoleMakerBase): assert role_maker._is_collective == True @@ -249,7 +252,8 @@ def __init__(self, self.global_nranks = self.nranks arr = np.arange(0, self.num_pp * self.num_mp).reshape( - [self.num_pp, self.num_mp]) + [self.num_pp, self.num_mp] + ) ipp, imp = np.where(arr == self.rank) ipp = ipp[0] imp = imp[0] @@ -269,31 +273,43 @@ def _init_communication_group(self): num_pp = len(dev_ids) num_pp = max(1, num_pp) assert num_pp == self.num_pp, 'num_pp: {}, self.num_pp: {}'.format( - num_pp, self.num_pp) + num_pp, self.num_pp + ) collective_helper = fleet.meta_optimizers.common.CollectiveHelper( - self.role_maker, wait_port=False) + self.role_maker, wait_port=False + ) # Create global rings - collective_helper._init_communicator(self._startup_program, - self.current_endpoint, - self.global_endpoints, - self.global_rank, - self.global_ring_id, True, - self.global_ring_id, True) + collective_helper._init_communicator( + self._startup_program, + self.current_endpoint, + self.global_endpoints, + self.global_rank, + self.global_ring_id, + True, + self.global_ring_id, + True, + ) # Create mp rings if self.num_mp > 1: mp_endpoints = [self.endpoints[mp_idx] for mp_idx in self.mp_group] mp_rank = [ - idx for idx, mp_idx in enumerate(self.mp_group) + idx + for idx, mp_idx in enumerate(self.mp_group) if mp_idx == self.rank ][0] - collective_helper._init_communicator(self._startup_program, - self.current_endpoint, - mp_endpoints, mp_rank, - self.mp_ring_id, True, - self.global_ring_id, True) + collective_helper._init_communicator( + self._startup_program, + self.current_endpoint, + mp_endpoints, + mp_rank, + self.mp_ring_id, + True, + self.global_ring_id, + True, + ) # Create pipeline rings if self.num_pp > 1: @@ -305,20 +321,32 @@ def _init_communication_group(self): second_node = self.pp_group[pair[1]] if self.rank != first_node and self.rank != second_node: collective_helper._init_communicator( - self._startup_program, None, None, None, None, False, - self.global_ring_id, True) + self._startup_program, + None, + None, + None, + None, + False, + self.global_ring_id, + True, + ) continue pipeline_endpoints = [ - self.endpoints[first_node], self.endpoints[second_node] + self.endpoints[first_node], + self.endpoints[second_node], ] pipeline_rank = 0 if self.rank == first_node else 1 - collective_helper._init_communicator(self._startup_program, - self.current_endpoint, - pipeline_endpoints, - pipeline_rank, ring_id, - False, self.global_ring_id, - True) + collective_helper._init_communicator( + self._startup_program, + self.current_endpoint, + pipeline_endpoints, + pipeline_rank, + ring_id, + False, + self.global_ring_id, + True, + ) def _get_input_output_info(self, block): ''' @@ -345,7 +373,10 @@ def _update_param_device_map(self): for each_block in self._main_program.blocks: for op in each_block.ops: for var_name in op.input_arg_names: - if var_name not in params or var_name in self._param_device_map: + if ( + var_name not in params + or var_name in self._param_device_map + ): continue device = op.attr(self._op_device_key) @@ -374,7 +405,8 @@ def _split_program(self, program, stage, block_idx): if op.type == "while": sub_block_id = int(op.attr('sub_block').id) sub_used_var_names = self._split_program( - program, stage, sub_block_id) + program, stage, sub_block_id + ) used_var_names.update(sub_used_var_names) @@ -409,25 +441,24 @@ def _split_program(self, program, stage, block_idx): return used_var_names - -# def _find_post_op(self, index, var_name): -# """ -# Find the post op that has variable named var_name as input. -# """ -# # bugfix for uniform hybrid parallelism -# if '.cast_fp32' in var_name: -# var_name = var_name.replace('.cast_fp32', '') -# if '.cast_fp16' in var_name: -# var_name = var_name.replace('.cast_fp16', '') - -# post_ops = self._input_var_to_op[var_name] -# if post_ops == None: return None -# result_op = None -# for post_op, post_idx in reversed(post_ops): -# if post_idx > index: -# result_op = post_op -# break -# return result_op + # def _find_post_op(self, index, var_name): + # """ + # Find the post op that has variable named var_name as input. + # """ + # # bugfix for uniform hybrid parallelism + # if '.cast_fp32' in var_name: + # var_name = var_name.replace('.cast_fp32', '') + # if '.cast_fp16' in var_name: + # var_name = var_name.replace('.cast_fp16', '') + + # post_ops = self._input_var_to_op[var_name] + # if post_ops == None: return None + # result_op = None + # for post_op, post_idx in reversed(post_ops): + # if post_idx > index: + # result_op = post_op + # break + # return result_op def _find_prev_op(self, index, var_name): """ @@ -435,7 +466,8 @@ def _find_prev_op(self, index, var_name): variable named var_name. """ prev_ops = self._output_var_to_op[var_name] - if prev_ops == None: return None + if prev_ops == None: + return None result_op = None for prev_op, prev_idx in reversed(prev_ops): if prev_idx < index: @@ -481,26 +513,31 @@ def _check_validation(self, block): pre_stage_id = None for op in block.ops: - assert op.has_attr(self._op_role_key), ("{} has no {} set .".format( - op.type, self._op_role_key)) + assert op.has_attr(self._op_role_key), "{} has no {} set .".format( + op.type, self._op_role_key + ) op_role = op.attr(self._op_role_key) - assert op_role == int(self._op_role.Forward), ( - "Only forward is supported for inference.") + assert op_role == int( + self._op_role.Forward + ), "Only forward is supported for inference." if not op._has_kernel(op.type): assert op.type in [ - "while", "conditional_block" - ], ("The only supported op without kernel is while.") + "while", + "conditional_block", + ], "The only supported op without kernel is while." sub_block_id = op.attr('sub_block').id sub_block = block.program.block(sub_block_id) self._check_validation(sub_block) - assert op.has_attr( - self._op_device_key), ("{} has no {} set.".format( - op.type, self._op_device_key)) + assert op.has_attr(self._op_device_key), "{} has no {} set.".format( + op.type, self._op_device_key + ) device = op.attr(self._op_device_key) - assert device, ("{} has no {} set.".format(op.type, - self._op_device_key)) - if device.split(':')[1] == "all": continue + assert device, "{} has no {} set.".format( + op.type, self._op_device_key + ) + if device.split(':')[1] == "all": + continue dev_type = device.split(':')[0] assert dev_type == self._device @@ -522,14 +559,17 @@ def _insert_sendrecv_ops_for_boundaries(self, block, is_while_block): for index, op in enumerate(list(block.ops)): cur_device = op.attr(self._op_device_key) - if cur_device.split(':')[-1] == "all": continue + if cur_device.split(':')[-1] == "all": + continue for var_name in op.input_arg_names: if not block.has_var(var_name) and block._find_var_recursive( - var_name): + var_name + ): continue var = block.var(var_name) # skip data var - if var.is_data: continue + if var.is_data: + continue prev_device = None generate_ops = self._output_var_to_op.get(var_name) if generate_ops is None: @@ -540,21 +580,24 @@ def _insert_sendrecv_ops_for_boundaries(self, block, is_while_block): prev_op = self._find_prev_op(index, var_name) if not prev_device: - prev_device = prev_op.attr(self._op_device_key) \ - if prev_op else None + prev_device = ( + prev_op.attr(self._op_device_key) if prev_op else None + ) if prev_device is None or prev_device.split(":")[-1] == "all": continue - if prev_device == cur_device: continue + if prev_device == cur_device: + continue if var_name not in input_var_to_device: input_var_to_device[var_name] = [] if (cur_device, prev_device) in input_var_to_device[var_name]: continue - assert self._device == cur_device.split( - ':')[0], "More than one device type found." + assert ( + self._device == cur_device.split(':')[0] + ), "More than one device type found." device_type = cur_device.split(':')[0] + ':' def _insert_send_recv(cur_id, prev_id): @@ -568,7 +611,8 @@ def _insert_send_recv(cur_id, prev_id): _insert_send_recv(cur_id - 1, prev_id) _insert_send_recv(cur_id, cur_id - 1) input_var_to_device[var_name].append( - (cur_dev, prev_dev)) + (cur_dev, prev_dev) + ) return assert cur_id - prev_id == 1 @@ -577,7 +621,10 @@ def _insert_send_recv(cur_id, prev_id): op_role = op.attr(self._op_role_key) var = block.vars[var_name] pair = (prev_id, cur_id) - if is_while_block and pair not in self._pipeline_pair_in_while: + if ( + is_while_block + and pair not in self._pipeline_pair_in_while + ): self._pipeline_pair_in_while.append(pair) # 1000 is just a magic number @@ -599,14 +646,16 @@ def _insert_send_recv(cur_id, prev_id): self._op_role_key: op_role, 'use_calc_stream': True, 'peer': 1, - 'ring_id': ring_id - }) + 'ring_id': ring_id, + }, + ) extra_index_info['index'] += 1 var_shape = list(var.shape) if var_shape[0] < 0: if is_while_block: - var_shape[ - 0] = self.micro_batch_size * self.beam_size + var_shape[0] = ( + self.micro_batch_size * self.beam_size + ) else: var_shape[0] = self.micro_batch_size @@ -621,17 +670,24 @@ def _insert_send_recv(cur_id, prev_id): self._op_role_key: op_role, 'use_calc_stream': True, 'peer': 0, - 'ring_id': ring_id - }) + 'ring_id': ring_id, + }, + ) extra_index_info['index'] += 1 - _insert_send_recv(int(cur_device.split(':')[1]), - int(prev_device.split(':')[1])) + _insert_send_recv( + int(cur_device.split(':')[1]), + int(prev_device.split(':')[1]), + ) block._sync_with_cpp() def _insert_sendrecv_ops_in_while_block( - self, block, sync_in_while_lastpp2firstpp_var_names, - sync_in_while_var_names, stage): + self, + block, + sync_in_while_lastpp2firstpp_var_names, + sync_in_while_var_names, + stage, + ): dev_ids = [] for pair in self._pipeline_pair_in_while: prev_id, cur_id = pair @@ -646,13 +702,16 @@ def _insert_sendrecv_ops_in_while_block( first_id = min(dev_ids) last_id = max(dev_ids) - assert len(block.ops) > 2, "It must have more than 2 ops in while sub block, " \ - "layers.assign(layers.cast(cond_int, dtype='bool'), cond) must at end of while block, " \ + assert len(block.ops) > 2, ( + "It must have more than 2 ops in while sub block, " + "layers.assign(layers.cast(cond_int, dtype='bool'), cond) must at end of while block, " "because nccl cannot send bool dtype var" + ) index = len(block.ops) - 2 for prev_id in dev_ids: - if prev_id == cur_id: continue + if prev_id == cur_id: + continue assert cur_id > prev_id pair = (prev_id, cur_id) @@ -667,7 +726,10 @@ def _insert_sendrecv_ops_in_while_block( ring_id = self._pp_ring_map[pair_key] if cur_id == last_id and prev_id == first_id: - var_names = sync_in_while_lastpp2firstpp_var_names + sync_in_while_var_names + var_names = ( + sync_in_while_lastpp2firstpp_var_names + + sync_in_while_var_names + ) else: var_names = sync_in_while_var_names @@ -679,19 +741,24 @@ def _insert_sendrecv_ops_in_while_block( type='send_v2', inputs={'X': var}, attrs={ - self._op_device_key: - self._device + ':' + str(cur_id), + self._op_device_key: self._device + + ':' + + str(cur_id), self._op_role_key: int(self._op_role.Forward), 'use_calc_stream': True, 'peer': 0, - 'ring_id': ring_id - }) + 'ring_id': ring_id, + }, + ) else: var_shape = list(var.shape) print(var_name) if len(var.shape) > 0: - var_shape[0] = self.micro_batch_size if var_shape[ - 0] < 0 else var_shape[0] + var_shape[0] = ( + self.micro_batch_size + if var_shape[0] < 0 + else var_shape[0] + ) block._insert_op_without_sync( index=index, type='recv_v2', @@ -699,13 +766,15 @@ def _insert_sendrecv_ops_in_while_block( attrs={ 'out_shape': var_shape, 'dtype': var.dtype, - self._op_device_key: - self._device + ':' + str(prev_id), + self._op_device_key: self._device + + ':' + + str(prev_id), self._op_role_key: int(self._op_role.Forward), 'use_calc_stream': True, 'peer': 1, - 'ring_id': ring_id - }) + 'ring_id': ring_id, + }, + ) index += 1 block._sync_with_cpp() @@ -721,13 +790,16 @@ def _get_while_block(self): if op.type == 'while': sub_block_id = op.attr('sub_block').id num_while += 1 - if sub_block_id: return op, self._main_program.block(sub_block_id) + if sub_block_id: + return op, self._main_program.block(sub_block_id) return None, None - def gen_infer_program(self, - sync_in_while_lastpp2firstpp_var_names=None, - sync_in_while_var_names=None, - debug=False): + def gen_infer_program( + self, + sync_in_while_lastpp2firstpp_var_names=None, + sync_in_while_var_names=None, + debug=False, + ): """ Generate inference program. Params: @@ -764,15 +836,19 @@ def gen_infer_program(self, while_op, while_block = self._get_while_block() if while_block: out_var_to_op, in_var_to_op = self._get_input_output_info( - while_block) + while_block + ) self._output_var_to_op = out_var_to_op self._input_var_to_op = in_var_to_op self._insert_sendrecv_ops_for_boundaries(while_block, True) self._insert_sendrecv_ops_in_while_block( - while_block, sync_in_while_lastpp2firstpp_var_names, - sync_in_while_var_names, self._stage) + while_block, + sync_in_while_lastpp2firstpp_var_names, + sync_in_while_var_names, + self._stage, + ) # step3: split programs self._split_program(self._startup_program, self._stage, 0) diff --git a/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py b/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py index 93b9ce4ef799c..fec3e455f8ab2 100644 --- a/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py +++ b/python/paddle/distributed/fleet/utils/hybrid_parallel_util.py @@ -15,7 +15,11 @@ from paddle import framework import paddle from paddle.fluid import core -from paddle.fluid.dygraph.parallel import _split_tensors, sync_params_buffers, build_groups +from paddle.fluid.dygraph.parallel import ( + _split_tensors, + sync_params_buffers, + build_groups, +) from paddle.fluid.framework import in_dygraph_mode, _in_legacy_dygraph from .log_util import logger @@ -30,7 +34,8 @@ def _apply_collective_grads(parameters, comm_group): for param in parameters: if param.trainable and (param._grad_ivar() is not None): g_var = param._grad_ivar() - assert not g_var._is_sparse( + assert ( + not g_var._is_sparse() ), "Now, it doesn't support sparse parameters" grad_vars.append(g_var) assert g_var not in grad_var_set @@ -38,19 +43,20 @@ def _apply_collective_grads(parameters, comm_group): coalesced_grads_and_vars = build_groups(grad_vars, 128 * 1024 * 1024) - nranks = paddle.distributed.get_world_size( - ) if comm_group is None else comm_group.nranks + nranks = ( + paddle.distributed.get_world_size() + if comm_group is None + else comm_group.nranks + ) for coalesced_grad, _, _ in coalesced_grads_and_vars: # need to div nranks div_factor = paddle.to_tensor(nranks, dtype=coalesced_grad.dtype) paddle.fluid.framework._dygraph_tracer().trace_op( type="elementwise_div", - inputs={ - 'X': coalesced_grad, - 'Y': div_factor - }, + inputs={'X': coalesced_grad, 'Y': div_factor}, outputs={'Out': coalesced_grad}, - attrs={'axis': -1}) + attrs={'axis': -1}, + ) paddle.distributed.all_reduce(coalesced_grad, group=comm_group) _split_tensors(coalesced_grads_and_vars) @@ -63,7 +69,8 @@ def _apply_collective_grads_eager(parameters, comm_group): for param in parameters: if param.trainable and (param._grad_ivar() is not None): g_var = param._grad_ivar() - assert not g_var.is_sparse( + assert ( + not g_var.is_sparse() ), "Now, it doesn't support sparse parameters" grad_vars.append(g_var) assert g_var not in grad_var_set @@ -71,8 +78,11 @@ def _apply_collective_grads_eager(parameters, comm_group): coalesced_grads_and_vars = build_groups(grad_vars, 128 * 1024 * 1024) - nranks = paddle.distributed.get_world_size( - ) if comm_group is None else comm_group.nranks + nranks = ( + paddle.distributed.get_world_size() + if comm_group is None + else comm_group.nranks + ) for coalesced_grad, _, _ in coalesced_grads_and_vars: # need to div nranks coalesced_grad.scale_(1.0 / nranks) @@ -87,20 +97,18 @@ def _broadcast_data_help(data, shape, dtype, hcg): mp_rank = hcg.get_model_parallel_rank() shape_gpu = paddle.to_tensor(shape, dtype="int32") - paddle.distributed.broadcast(shape_gpu, - src=src_rank, - group=model_parallel_group, - sync_op=True) + paddle.distributed.broadcast( + shape_gpu, src=src_rank, group=model_parallel_group, sync_op=True + ) if mp_rank != 0: input_data = paddle.zeros(shape_gpu, dtype=dtype) else: input_data = data - paddle.distributed.broadcast(input_data, - src=src_rank, - group=model_parallel_group, - sync_op=True) + paddle.distributed.broadcast( + input_data, src=src_rank, group=model_parallel_group, sync_op=True + ) if mp_rank != 0: if in_dygraph_mode(): @@ -109,7 +117,8 @@ def _broadcast_data_help(data, shape, dtype, hcg): else: data.value().get_tensor()._clear() data.value().get_tensor()._share_data_with( - input_data.value().get_tensor()) + input_data.value().get_tensor() + ) def broadcast_input_data(hcg, *inputs, **kwargs): @@ -117,8 +126,11 @@ def broadcast_input_data(hcg, *inputs, **kwargs): for v in inputs: if isinstance(v, (core.VarBase, core.eager.Tensor)): with framework.no_grad(): - if "gpu" in cur_device and in_dygraph_mode() \ - and not v.place.is_gpu_place(): + if ( + "gpu" in cur_device + and in_dygraph_mode() + and not v.place.is_gpu_place() + ): v_gpu = v.cuda(int(cur_device.split(":")[1])) v._clear_data() v_gpu._share_buffer_to(v) @@ -129,8 +141,11 @@ def broadcast_input_data(hcg, *inputs, **kwargs): for k, v in kwargs.items(): if isinstance(v, (core.VarBase, core.eager.Tensor)): with framework.no_grad(): - if "gpu" in cur_device and in_dygraph_mode() \ - and not v.place.is_gpu_place(): + if ( + "gpu" in cur_device + and in_dygraph_mode() + and not v.place.is_gpu_place() + ): v_gpu = v.cuda(int(cur_device.split(":")[1])) v._clear_data() v_gpu._share_buffer_to(v) @@ -144,26 +159,27 @@ def broadcast_input_data(hcg, *inputs, **kwargs): def broadcast_mp_parameters(model, hcg): model_parallel_group = hcg.get_model_parallel_group() src_rank = hcg.get_model_parallel_group_src_rank() - sync_params_buffers(model, - model_parallel_group, - src_rank, - is_model_parallel=True) + sync_params_buffers( + model, model_parallel_group, src_rank, is_model_parallel=True + ) def broadcast_dp_parameters(model, hcg): data_parallel_group = hcg.get_data_parallel_group() src_rank = hcg.get_data_parallel_group_src_rank() - sync_params_buffers(model, - data_parallel_group, - src_rank, - is_model_parallel=False) + sync_params_buffers( + model, data_parallel_group, src_rank, is_model_parallel=False + ) def fused_allreduce_gradients(parameter_list, hcg): data_parallel_group = None if hcg is None else hcg.get_data_parallel_group() logger.debug("dp start fuse allreduce gradients") - apply_func = _apply_collective_grads_eager if in_dygraph_mode( - ) else _apply_collective_grads + apply_func = ( + _apply_collective_grads_eager + if in_dygraph_mode() + else _apply_collective_grads + ) with framework.no_grad(): apply_func(parameter_list, data_parallel_group) @@ -182,7 +198,8 @@ def sharding_reduce_gradients(parameter_list, hcg): paddle.distributed.all_reduce( param.grad, group=hcg.get_sharding_parallel_group(), - sync_op=True) + sync_op=True, + ) elif _in_legacy_dygraph(): g_var = param._grad_ivar() @@ -195,20 +212,20 @@ def sharding_reduce_gradients(parameter_list, hcg): outputs={'Out': g_var}, attrs={ 'ring_id': hcg.get_sharding_parallel_group().id, - 'use_calc_stream': True - }) + 'use_calc_stream': True, + }, + ) # grad / sharding_rank - div_factor = paddle.to_tensor(sharding_nrank, - dtype=g_var.dtype) + div_factor = paddle.to_tensor( + sharding_nrank, dtype=g_var.dtype + ) paddle.fluid.framework._dygraph_tracer().trace_op( type="elementwise_div", - inputs={ - 'X': g_var, - 'Y': div_factor - }, + inputs={'X': g_var, 'Y': div_factor}, outputs={'Out': g_var}, - attrs={'axis': -1}) + attrs={'axis': -1}, + ) def broadcast_sharding_parameters(model, hcg): @@ -216,7 +233,6 @@ def broadcast_sharding_parameters(model, hcg): logger.debug("sharding start init parameters sync") sharding_parallel_group = hcg.get_sharding_parallel_group() src_rank = hcg.get_sharding_parallel_group_src_rank() - sync_params_buffers(model, - sharding_parallel_group, - src_rank, - is_model_parallel=False) + sync_params_buffers( + model, sharding_parallel_group, src_rank, is_model_parallel=False + ) diff --git a/python/paddle/distributed/fleet/utils/internal_storage.py b/python/paddle/distributed/fleet/utils/internal_storage.py index 89011a6fe1c17..e0f2ef0a04640 100644 --- a/python/paddle/distributed/fleet/utils/internal_storage.py +++ b/python/paddle/distributed/fleet/utils/internal_storage.py @@ -47,10 +47,11 @@ def __init__(self, size, dtype, device, convert_cpu=False): # The actual flat tensor size = [size] if isinstance(size, int) else size if convert_cpu: - value = np.zeros( - size, - dtype=np.float16) if Type.fp16.value == dtype else np.zeros( - size, dtype=np.float32) + value = ( + np.zeros(size, dtype=np.float16) + if Type.fp16.value == dtype + else np.zeros(size, dtype=np.float32) + ) self.buffer = core.VarBase(value=value, place=core.CPUPlace()) else: self.buffer = paddle.zeros(size, dtype=dtype) @@ -59,16 +60,25 @@ def to(self, device, dtype=None, keep_alignment=True): """ Move the underlying buffer """ - assert self.buffer is not None, "Cannot move a collapsed bucket, please rebuild it" - assert (dtype == Type.fp32.value - or Type.fp16.value), "Conversion type is not supported now" + assert ( + self.buffer is not None + ), "Cannot move a collapsed bucket, please rebuild it" + assert ( + dtype == Type.fp32.value or Type.fp16.value + ), "Conversion type is not supported now" - dev_id = 0 if paddle.get_device() == "cpu" else int( - paddle.get_device().split(":")[1]) + dev_id = ( + 0 + if paddle.get_device() == "cpu" + else int(paddle.get_device().split(":")[1]) + ) if self._device != device: - tmp_buffer = self.buffer.cuda( - dev_id) if device == "gpu" else self.buffer.cpu() + tmp_buffer = ( + self.buffer.cuda(dev_id) + if device == "gpu" + else self.buffer.cpu() + ) for param in self._params: param.clear_gradient(False) param._gradient_set_empty(False) @@ -106,17 +116,18 @@ def add_rank_params(self, trainable_params, param2align, convert_gpu=True): Add new parameters to the InternalStorage. Params becomes a view of this InternalStorage buffer. """ - assert all([ - id(param) not in self._param_ids for param in trainable_params - ]), "The same param cannot be checked in twice" + assert all( + [id(param) not in self._param_ids for param in trainable_params] + ), "The same param cannot be checked in twice" assert self.buffer is not None self.param2align = param2align cpu_param_shape = list() for param in trainable_params: - p_shape = self._add_param_as_view(param, param2align[param.name], - convert_gpu) + p_shape = self._add_param_as_view( + param, param2align[param.name], convert_gpu + ) cpu_param_shape.append(p_shape) if convert_gpu: @@ -127,8 +138,9 @@ def add_rank_params(self, trainable_params, param2align, convert_gpu=True): self._fill = 0 for idx, param in enumerate(trainable_params): - self._convert_buffer(param, cpu_param_shape[idx], - param2align[param.name]) + self._convert_buffer( + param, cpu_param_shape[idx], param2align[param.name] + ) self._params.append(param) self._param_ids.append(id(param)) @@ -138,7 +150,8 @@ def _add_param_as_view(self, param, align, convert_gpu=True): assert ( param.dtype == self.buffer.dtype ), "Different types for the InternalStorage and the param, cannot proceed: {} - {}".format( - param.dtype, self.buffer.dtype) + param.dtype, self.buffer.dtype + ) var_end = self._fill + np.prod(param.shape) offset = var_end + align @@ -152,11 +165,15 @@ def _add_param_as_view(self, param, align, convert_gpu=True): param.stop_gradient = origin_state # Copy the current param value - dev_id = 0 if paddle.get_device() == "cpu" else int( - paddle.get_device().split(":")[1]) + dev_id = ( + 0 + if paddle.get_device() == "cpu" + else int(paddle.get_device().split(":")[1]) + ) with device_guard(dev_id, "cpu"): tmp_var = core.VarBase( - tensor=self.buffer._slice(self._fill, var_end)) + tensor=self.buffer._slice(self._fill, var_end) + ) if convert_gpu: param_cpu = param.cpu() param.value().get_tensor()._clear() @@ -199,13 +216,9 @@ class GradStorage(InternalStorage): This is a basic class to simplify the handling of gradient InternalStorages """ - def __init__(self, - size, - dtype, - device, - destination, - parm2align, - convert_cpu=False): + def __init__( + self, size, dtype, device, destination, parm2align, convert_cpu=False + ): if isinstance(size, np.int64): size = size.tolist() super().__init__(size, dtype, device, convert_cpu) @@ -219,22 +232,21 @@ def __init__(self, self.sent = False def reset_checked_in(self): - """ Reset the counter of the parameter grads which have been checked in - """ + """Reset the counter of the parameter grads which have been checked in""" self.params_checked_in = 0 self.sent = False @property def all_checked_in(self): - """ Judge all the expected gradient check-in happened """ + """Judge all the expected gradient check-in happened""" return len(self._params) == self.params_checked_in def can_add_grad_view(self, param, align): - """ Is there enough InternalStorage to add this parameter gradient, and whether this param have already checked in. - """ - return self._fill + np.prod( - param.shape) + align <= self._max_size and id( - param) not in self._param_ids + """Is there enough InternalStorage to add this parameter gradient, and whether this param have already checked in.""" + return ( + self._fill + np.prod(param.shape) + align <= self._max_size + and id(param) not in self._param_ids + ) def to(self, device, dtype=None, keep_alignment=True): """ @@ -254,9 +266,9 @@ def add_grad(self, param, align): Add a new parameter gradient to the InternalStorage. Param.grad becomes a view of this InternalStorage buffer. """ - assert id( - param - ) not in self._param_ids, "The same gradients cannot be checked in twice" + assert ( + id(param) not in self._param_ids + ), "The same gradients cannot be checked in twice" self._add_grad_as_view(param, align) self._params.append(param) @@ -304,9 +316,9 @@ def _array_grads(self): @fluid.dygraph.no_grad def _add_grad_as_view(self, param, align): - assert np.prod( - self.buffer.shape - ) > 0, "Cannot add a gradient to a released InternalStorage, please rebuild" + assert ( + np.prod(self.buffer.shape) > 0 + ), "Cannot add a gradient to a released InternalStorage, please rebuild" assert param.dtype == self.buffer.dtype grad_end = self._fill + np.prod(param.shape) @@ -314,8 +326,11 @@ def _add_grad_as_view(self, param, align): assert offset <= np.prod(self.buffer.shape) # Copy the current grad value to InternalStorage - dev_id = 0 if paddle.get_device() == "cpu" else int( - paddle.get_device().split(":")[1]) + dev_id = ( + 0 + if paddle.get_device() == "cpu" + else int(paddle.get_device().split(":")[1]) + ) if self._device == "cpu": with device_guard(dev_id, self._device): tmp_var = core.VarBase(self.buffer._slice(self._fill, grad_end)) diff --git a/python/paddle/distributed/fleet/utils/log_util.py b/python/paddle/distributed/fleet/utils/log_util.py index 34b1caa863cb2..07bcbea4b4109 100644 --- a/python/paddle/distributed/fleet/utils/log_util.py +++ b/python/paddle/distributed/fleet/utils/log_util.py @@ -67,7 +67,8 @@ def layer_to_str(base, *args, **kwargs): if kwargs: name += ", " if kwargs: - name += ", ".join("{}={}".format(key, str(value)) - for key, value in kwargs.items()) + name += ", ".join( + "{}={}".format(key, str(value)) for key, value in kwargs.items() + ) name += ")" return name diff --git a/python/paddle/distributed/fleet/utils/ps_util.py b/python/paddle/distributed/fleet/utils/ps_util.py index fbf713bed9363..9b079d64bb530 100644 --- a/python/paddle/distributed/fleet/utils/ps_util.py +++ b/python/paddle/distributed/fleet/utils/ps_util.py @@ -29,21 +29,21 @@ def __init__(self, main_program=None, startup_program=None): if main_program: self.origin_main_program = main_program.clone() else: - self.origin_main_program = paddle.static.default_main_program( - ).clone() + self.origin_main_program = ( + paddle.static.default_main_program().clone() + ) if startup_program: self.origin_startup_program = startup_program else: - self.origin_startup_program = paddle.static.default_startup_program( + self.origin_startup_program = ( + paddle.static.default_startup_program() ) self.sparse_table_maps = None - def init_distributed_infer_env(self, - exe, - loss, - role_maker=None, - dirname=None): + def init_distributed_infer_env( + self, exe, loss, role_maker=None, dirname=None + ): import paddle.distributed.fleet as fleet if fleet.fleet._runtime_handle is None: @@ -52,10 +52,12 @@ def init_distributed_infer_env(self, fake_optimizer = paddle.optimizer.SGD() strategy = fleet.DistributedStrategy() strategy.a_sync = True - optimizer = fleet.distributed_optimizer(fake_optimizer, - strategy=strategy) - optimizer.minimize(loss, - startup_program=self.origin_startup_program) + optimizer = fleet.distributed_optimizer( + fake_optimizer, strategy=strategy + ) + optimizer.minimize( + loss, startup_program=self.origin_startup_program + ) if fleet.is_server(): fleet.init_server(dirname=dirname) @@ -88,36 +90,45 @@ def _init_dense_params(self, exe=None, dirname=None): if dirname is not None and exe is not None: all_persist_vars = [ - v for v in self.origin_main_program.list_vars() + v + for v in self.origin_main_program.list_vars() if paddle.static.io.is_persistable(v) ] - dense_persist_vars = [(v.name, v) for v in all_persist_vars - if v.name not in sparse_table_maps] + dense_persist_vars = [ + (v.name, v) + for v in all_persist_vars + if v.name not in sparse_table_maps + ] need_load_vars = [ - v[1] for v in dense_persist_vars + v[1] + for v in dense_persist_vars if os.path.isfile(os.path.join(dirname, v[0])) ] - paddle.static.load_vars(exe, - dirname, - main_program=self.origin_main_program, - vars=need_load_vars) + paddle.static.load_vars( + exe, + dirname, + main_program=self.origin_main_program, + vars=need_load_vars, + ) def get_dist_infer_program(self): varname2tables = self._get_sparse_table_map() - convert_program = self._convert_program(self.origin_main_program, - varname2tables) + convert_program = self._convert_program( + self.origin_main_program, varname2tables + ) return convert_program def _convert_program(self, main_program, varname2tables): - def distributed_ops_pass(program): SPARSE_OP_TYPE_DICT = {"lookup_table": "W", "lookup_table_v2": "W"} def _get_pull_sparse_ops(_program): pull_sparse_ops = {} for op in _program.global_block().ops: - if op.type in SPARSE_OP_TYPE_DICT.keys() \ - and op.attr('remote_prefetch') is True: + if ( + op.type in SPARSE_OP_TYPE_DICT.keys() + and op.attr('remote_prefetch') is True + ): param_name = op.input(SPARSE_OP_TYPE_DICT[op.type])[0] ops = pull_sparse_ops.get(param_name, []) ops.append(op) @@ -125,7 +136,6 @@ def _get_pull_sparse_ops(_program): return pull_sparse_ops def _pull_sparse_fuse(_program, pull_sparse_ops): - def dag_check_up_and_reorder(program, inputs, outputs): global_block = program.global_block() min_output_index = len(global_block.ops) @@ -151,7 +161,8 @@ def dag_check_up_and_reorder(program, inputs, outputs): if out_var.name in ins: output_indexes[idx] = 1 min_output_index = min( - min_output_index, idx) + min_output_index, idx + ) for i in range(len(global_block.ops)): if input_indexes[i] == 1 and output_indexes[i] == 1: @@ -162,8 +173,9 @@ def dag_check_up_and_reorder(program, inputs, outputs): if min_output_index < max_input_index: move_ops = [] - for i in range(min_output_index + 1, - len(input_indexes)): + for i in range( + min_output_index + 1, len(input_indexes) + ): if input_indexes[i] == 1: move_ops.append((global_block.ops[i], i)) for i, op in enumerate(move_ops): @@ -179,8 +191,9 @@ def dag_check_up_and_reorder(program, inputs, outputs): for k in range(0, len(op.input_names)): ins = op.input(op.input_names[k]) op_inputs.append(ins) - for j in range(pos - 1, min_output_index - 1, - -1): + for j in range( + pos - 1, min_output_index - 1, -1 + ): op1 = global_block.ops[j] if op1 in visited: continue @@ -209,27 +222,35 @@ def dag_check_up_and_reorder(program, inputs, outputs): queue.sort() for index in queue: desc = global_block.desc._insert_op( - min_output_index) + min_output_index + ) desc.copy_from(global_block.ops[index].desc) global_block.desc._remove_op( - index + 1, index + 2) + index + 1, index + 2 + ) global_block.ops[index].desc = desc insert_op = global_block.ops.pop(index) input_state = input_indexes.pop(index) output_state = output_indexes.pop(index) - global_block.ops.insert(min_output_index, - insert_op) - input_indexes.insert(min_output_index, - input_state) - output_indexes.insert(min_output_index, - output_state) + global_block.ops.insert( + min_output_index, insert_op + ) + input_indexes.insert( + min_output_index, input_state + ) + output_indexes.insert( + min_output_index, output_state + ) min_output_index = min_output_index + 1 assert global_block.desc.op_size() == len( - global_block.ops) + global_block.ops + ) for i in range(len(global_block.ops)): - assert global_block.desc.op( - i) == global_block.ops[i].desc + assert ( + global_block.desc.op(i) + == global_block.ops[i].desc + ) for param, ops in pull_sparse_ops.items(): all_ops = program.global_block().ops @@ -243,8 +264,10 @@ def dag_check_up_and_reorder(program, inputs, outputs): if w.name not in varname2tables.keys(): raise ValueError( - "can not find variable {}, please check your configuration" - .format(w.name)) + "can not find variable {}, please check your configuration".format( + w.name + ) + ) table_id = varname2tables[w.name] @@ -264,8 +287,9 @@ def dag_check_up_and_reorder(program, inputs, outputs): program.global_block()._remove_op(idx) inputs_idxs = [-1] * len(inputs) - outputs_idxs = [len(program.global_block().ops) + 1 - ] * len(outputs) + outputs_idxs = [len(program.global_block().ops) + 1] * len( + outputs + ) for idx, op in enumerate(program.global_block().ops): for i in range(0, len(op.output_names)): @@ -273,13 +297,15 @@ def dag_check_up_and_reorder(program, inputs, outputs): for in_id, in_var in enumerate(inputs): if in_var.name in outs: inputs_idxs[in_id] = max( - idx, inputs_idxs[in_id]) + idx, inputs_idxs[in_id] + ) for i in range(0, len(op.input_names)): ins = op.input(op.input_names[i]) for out_id, out_var in enumerate(outputs): if out_var.name in ins: outputs_idxs[out_id] = min( - idx, outputs_idxs[out_id]) + idx, outputs_idxs[out_id] + ) if min(outputs_idxs) - max(inputs_idxs) >= 1: distributed_idx = max(inputs_idxs) + 1 @@ -287,18 +313,16 @@ def dag_check_up_and_reorder(program, inputs, outputs): program.global_block()._insert_op( index=distributed_idx, type="distributed_lookup_table", - inputs={ - "Ids": inputs, - 'W': w - }, + inputs={"Ids": inputs, 'W': w}, outputs={"Outputs": outputs}, attrs={ "is_distributed": is_distributed, "padding_idx": padding_idx, "table_id": table_id, "is_test": True, - "lookup_table_version": op_type - }) + "lookup_table_version": op_type, + }, + ) else: raise ValueError( "something wrong with Fleet, submit a issue is recommended" diff --git a/python/paddle/distributed/launch/context/__init__.py b/python/paddle/distributed/launch/context/__init__.py index 9936b534edfcd..037fc0efbc51b 100644 --- a/python/paddle/distributed/launch/context/__init__.py +++ b/python/paddle/distributed/launch/context/__init__.py @@ -22,7 +22,6 @@ class Context(object): - def __init__(self, enable_plugin=True): self.args, self.unknown_args = parse_args() self.envs = fetch_envs() @@ -54,8 +53,9 @@ def is_legacy_mode(self): return False if len(self.unknown_args) > 0: - self.logger.warning("Compatible mode enable with args {}".format( - self.unknown_args)) + self.logger.warning( + "Compatible mode enable with args {}".format(self.unknown_args) + ) return True return False @@ -75,7 +75,8 @@ def get_logger(self, level=logging.INFO): logger = logging.getLogger("LAUNCH") logger.setLevel(self.args.log_level.upper() or level) formatter = logging.Formatter( - fmt='%(name)s %(levelname)s %(asctime)s %(message)s') + fmt='%(name)s %(levelname)s %(asctime)s %(message)s' + ) ch = logging.StreamHandler() ch.setFormatter(formatter) logger.addHandler(ch) diff --git a/python/paddle/distributed/launch/context/args_envs.py b/python/paddle/distributed/launch/context/args_envs.py index 104ad1b789f54..2013ba6a3d7ba 100644 --- a/python/paddle/distributed/launch/context/args_envs.py +++ b/python/paddle/distributed/launch/context/args_envs.py @@ -52,126 +52,135 @@ def parse_args(): base_group = parser.add_argument_group("Base Parameters") - base_group.add_argument("--master", - type=str, - default=None, - help="the master/rendezvous server, ip:port") - - base_group.add_argument("--legacy", - type=bool, - default=False, - help="use legacy launch") - - base_group.add_argument("--rank", - type=int, - default=-1, - help="the node rank") - - base_group.add_argument("--log_level", - type=str, - default="INFO", - help="log level. Default INFO") - - base_group.add_argument("--nnodes", - type=str, - default="1", - help="the number of nodes, i.e. pod/node number") - - base_group.add_argument("--nproc_per_node", - type=int, - default=None, - help="the number of processes in a pod") + base_group.add_argument( + "--master", + type=str, + default=None, + help="the master/rendezvous server, ip:port", + ) + + base_group.add_argument( + "--legacy", type=bool, default=False, help="use legacy launch" + ) + + base_group.add_argument( + "--rank", type=int, default=-1, help="the node rank" + ) + + base_group.add_argument( + "--log_level", type=str, default="INFO", help="log level. Default INFO" + ) + + base_group.add_argument( + "--nnodes", + type=str, + default="1", + help="the number of nodes, i.e. pod/node number", + ) + + base_group.add_argument( + "--nproc_per_node", + type=int, + default=None, + help="the number of processes in a pod", + ) base_group.add_argument( "--log_dir", type=str, default="log", - help="the path for each process's log. Default ./log") - base_group.add_argument("--run_mode", - type=str, - default=None, - help="run mode of the job, collective/ps/ps-heter") - - base_group.add_argument("--job_id", - type=str, - default="default", - help="unique id of the job. Default default") - - base_group.add_argument("--devices", - "--gpus", - "--npus", - "--xpus", - type=str, - default=None, - help="accelerate devices. as --gpus,npus,xpus") + help="the path for each process's log. Default ./log", + ) + base_group.add_argument( + "--run_mode", + type=str, + default=None, + help="run mode of the job, collective/ps/ps-heter", + ) + + base_group.add_argument( + "--job_id", + type=str, + default="default", + help="unique id of the job. Default default", + ) + + base_group.add_argument( + "--devices", + "--gpus", + "--npus", + "--xpus", + type=str, + default=None, + help="accelerate devices. as --gpus,npus,xpus", + ) base_group.add_argument("--host", type=str, default=None, help="host ip") - base_group.add_argument("--ips", - type=str, - default=None, - help="nodes ips, e.g. 10.10.1.1,10.10.1.2") + base_group.add_argument( + "--ips", + type=str, + default=None, + help="nodes ips, e.g. 10.10.1.1,10.10.1.2", + ) - base_group.add_argument("--start_port", - type=int, - default=6070, - help="fix port start with") + base_group.add_argument( + "--start_port", type=int, default=6070, help="fix port start with" + ) - base_group.add_argument("training_script", - type=str, - help="the full path of py script," - "followed by arguments for the " - "training script") + base_group.add_argument( + "training_script", + type=str, + help="the full path of py script," + "followed by arguments for the " + "training script", + ) base_group.add_argument('training_script_args', nargs=REMAINDER) ps_group = parser.add_argument_group("Parameter-Server Parameters") # for parameter server - ps_group.add_argument("--servers", - type=str, - default='', - help="servers endpoints full list") - ps_group.add_argument("--trainers", - type=str, - default='', - help="trainers endpoints full list") - - ps_group.add_argument("--trainer_num", - type=int, - default=None, - help="number of trainers") - ps_group.add_argument("--server_num", - type=int, - default=None, - help="number of servers") - ps_group.add_argument("--gloo_port", - type=int, - default=6767, - help="gloo http port") - ps_group.add_argument("--with_gloo", - type=str, - default="1", - help="use gloo or not") + ps_group.add_argument( + "--servers", type=str, default='', help="servers endpoints full list" + ) + ps_group.add_argument( + "--trainers", type=str, default='', help="trainers endpoints full list" + ) + + ps_group.add_argument( + "--trainer_num", type=int, default=None, help="number of trainers" + ) + ps_group.add_argument( + "--server_num", type=int, default=None, help="number of servers" + ) + ps_group.add_argument( + "--gloo_port", type=int, default=6767, help="gloo http port" + ) + ps_group.add_argument( + "--with_gloo", type=str, default="1", help="use gloo or not" + ) # parameter elastic mode elastic_group = parser.add_argument_group("Elastic Parameters") - elastic_group.add_argument("--max_restart", - type=int, - default=3, - help="the times can restart. Default 3") + elastic_group.add_argument( + "--max_restart", + type=int, + default=3, + help="the times can restart. Default 3", + ) elastic_group.add_argument( "--elastic_level", type=int, default=-1, - help= - "elastic level: -1 disable, 0 failed exit, peers hold, 1 internal restart" + help="elastic level: -1 disable, 0 failed exit, peers hold, 1 internal restart", ) elastic_group.add_argument( "--elastic_timeout", type=int, default=30, - help="seconds to wait before elastic job begin to train") + help="seconds to wait before elastic job begin to train", + ) return parser.parse_known_args() diff --git a/python/paddle/distributed/launch/context/device.py b/python/paddle/distributed/launch/context/device.py index 14997df24590f..f05bc1b776869 100644 --- a/python/paddle/distributed/launch/context/device.py +++ b/python/paddle/distributed/launch/context/device.py @@ -28,7 +28,6 @@ class DeviceType: class Device(object): - def __init__(self, dtype=None, memory="", labels=""): self._dtype = dtype self._memory = memory @@ -102,7 +101,8 @@ def parse_device(self): if 'PADDLE_XCCL_BACKEND' in os.environ: dev._dtype = DeviceType.CUSTOM_DEVICE visible_devices_str = '{}_VISIBLE_DEVICES'.format( - os.getenv('PADDLE_XCCL_BACKEND').upper()) + os.getenv('PADDLE_XCCL_BACKEND').upper() + ) if visible_devices_str in os.environ: visible_devices = os.getenv(visible_devices_str) elif 'CUDA_VISIBLE_DEVICES' in os.environ: @@ -127,7 +127,6 @@ def parse_device(self): @classmethod def detect_device(self): - def get_custom_devices_count(device_type): all_custom_devices = get_available_custom_device() all_custom_devices = [ @@ -144,7 +143,8 @@ def get_custom_devices_count(device_type): dev._dtype = DeviceType.CUSTOM_DEVICE num = get_custom_devices_count(custom_device_type) visible_devices_str = '{}_VISIBLE_DEVICES'.format( - custom_device_type.upper()) + custom_device_type.upper() + ) if visible_devices_str in os.environ: visible_devices = os.getenv(visible_devices_str) elif fluid.core.is_compiled_with_cuda(): diff --git a/python/paddle/distributed/launch/context/event.py b/python/paddle/distributed/launch/context/event.py index cb39e1529fc82..3859edb83f29c 100644 --- a/python/paddle/distributed/launch/context/event.py +++ b/python/paddle/distributed/launch/context/event.py @@ -14,7 +14,6 @@ class Event(object): - def __init__(self, kind="status", message="", fatal=False): self.kind = kind self.message = message diff --git a/python/paddle/distributed/launch/context/node.py b/python/paddle/distributed/launch/context/node.py index e81c91e1bca9f..b8c1a2a14f5ce 100644 --- a/python/paddle/distributed/launch/context/node.py +++ b/python/paddle/distributed/launch/context/node.py @@ -22,7 +22,6 @@ class Node(object): - def __init__(self): # self.device = Device.detect_device() self.device = Device.parse_device() @@ -54,8 +53,9 @@ def get_ports_occupied(self): def _get_free_port(self, port=0): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: - s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, - struct.pack('ii', 1, 0)) + s.setsockopt( + socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0) + ) try: s.bind(('', port)) return s.getsockname()[1] @@ -81,8 +81,8 @@ def get_free_port(self): @classmethod def is_server_ready(self, ip, port): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: - #sock.settimeout(0.01) - #sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # sock.settimeout(0.01) + # sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) result = sock.connect_ex((ip, int(port))) diff --git a/python/paddle/distributed/launch/context/resource.py b/python/paddle/distributed/launch/context/resource.py index d523c3c5cdfe8..b7baf6fd7b621 100644 --- a/python/paddle/distributed/launch/context/resource.py +++ b/python/paddle/distributed/launch/context/resource.py @@ -14,6 +14,5 @@ class Resource(object): - def __init__(self): self.devices = [] diff --git a/python/paddle/distributed/launch/controllers/collective.py b/python/paddle/distributed/launch/controllers/collective.py index dd0a3cc34aa3a..be38b4057f0d7 100644 --- a/python/paddle/distributed/launch/controllers/collective.py +++ b/python/paddle/distributed/launch/controllers/collective.py @@ -19,7 +19,6 @@ class CollectiveController(Controller): - @classmethod def enable(cls, ctx): # collective is the default mode @@ -31,7 +30,11 @@ def enable(cls, ctx): return False def build_pod(self): - if self.ctx.args.master is None and self.ctx.args.start_port and self.ctx.args.ips: + if ( + self.ctx.args.master is None + and self.ctx.args.start_port + and self.ctx.args.ips + ): self._build_pod_with_args() else: self._build_pod_with_master() @@ -48,15 +51,18 @@ def _build_pod_with_args(self): self.ctx.logger.debug("job endpoints: {}".format(job_endpoints)) - rank_offset = ips.index( - self.ctx.node.ip - ) * self.pod.replicas if self.ctx.node.ip in ips else 0 + rank_offset = ( + ips.index(self.ctx.node.ip) * self.pod.replicas + if self.ctx.node.ip in ips + else 0 + ) self.save_pod_log(job_endpoints) selected_dev_key = self.ctx.node.device.get_selected_device_key() selected_dev_list = self.ctx.node.device.get_selected_devices( - self.ctx.args.devices) + self.ctx.args.devices + ) for i in range(self.pod.replicas): e = { @@ -101,19 +107,24 @@ def _build_pod_with_master(self): for p in self.ctx.node.get_free_ports(self.pod.replicas) ] - data = json.dumps({ - 'name': self.pod.name, - 'rank': self.pod.rank, - 'replicas': self.pod.replicas, - 'dtype': self.ctx.node.device.dtype, - 'candidate': '{}:{}'.format(self.ctx.node.ip, port), - 'endpoints': ",".join(endpoints), - }) - - peer_list, rank = self.master.sync_peers('/{}/info'.format(self.job.id), - self.pod.name, data, - self.job.replicas, - self.pod.rank) + data = json.dumps( + { + 'name': self.pod.name, + 'rank': self.pod.rank, + 'replicas': self.pod.replicas, + 'dtype': self.ctx.node.device.dtype, + 'candidate': '{}:{}'.format(self.ctx.node.ip, port), + 'endpoints': ",".join(endpoints), + } + ) + + peer_list, rank = self.master.sync_peers( + '/{}/info'.format(self.job.id), + self.pod.name, + data, + self.job.replicas, + self.pod.rank, + ) self.pod.rank = rank if len(peer_list) < 1: @@ -136,7 +147,8 @@ def _build_pod_with_master(self): self.pod.reset() selected_dev_key = self.ctx.node.device.get_selected_device_key() selected_dev_list = self.ctx.node.device.get_selected_devices( - self.ctx.args.devices) + self.ctx.args.devices + ) for i in range(self.pod.replicas): e = { "PADDLE_MASTER": collective_master, @@ -170,7 +182,6 @@ def _build_pod_with_master(self): class CollectiveElasticController(CollectiveController): - @classmethod def enable(cls, ctx): if ctx.args.master and ctx.args.master.startswith("etcd://"): @@ -200,9 +211,9 @@ def run(self): self.ctx.logger.info("Waiting peer ready...") - ok, replicas = self.master.wait_peer_ready(self.job.replicas_min, - self.job.replicas_max, - timeout) + ok, replicas = self.master.wait_peer_ready( + self.job.replicas_min, self.job.replicas_max, timeout + ) if ok: self.job.replicas = replicas else: diff --git a/python/paddle/distributed/launch/controllers/controller.py b/python/paddle/distributed/launch/controllers/controller.py index 21b9ce2d5c80c..9ff18d5b5d511 100644 --- a/python/paddle/distributed/launch/controllers/controller.py +++ b/python/paddle/distributed/launch/controllers/controller.py @@ -32,7 +32,6 @@ class ControleMode: class ControllerBase(object): - def __init__(self, ctx): signal.signal(signal.SIGTERM, self.signal_handler) signal.signal(signal.SIGABRT, self.signal_handler) @@ -43,9 +42,11 @@ def __init__(self, ctx): self.watcher = Watcher(self.ctx) - self.job = Job(nnodes=self.ctx.args.nnodes, - mode=self.ctx.args.run_mode, - jid=self.ctx.args.job_id) + self.job = Job( + nnodes=self.ctx.args.nnodes, + mode=self.ctx.args.run_mode, + jid=self.ctx.args.job_id, + ) self.pod = Pod() self.ctx.set_envs({"POD_NAME": self.pod.name}) @@ -74,14 +75,14 @@ def watch(self) -> bool: ''' watch self and peer status, return true to exit ''' - #TODO(kuizhiqing) unify ctx.status and master status + # TODO(kuizhiqing) unify ctx.status and master status self.ctx.logger.info("Watching {}".format(self.pod)) while not self.ctx.status.is_done(): status = self.pod.watch(timeout=2) - #if self.ctx.continous_log(): + # if self.ctx.continous_log(): # default to print log self.pod.logs() @@ -120,8 +121,10 @@ def watch(self) -> bool: return False # peer failure - if self.ctx.status.is_restarting( - ) and self.master.get_status() != self.ctx.status.COMPLETED: + if ( + self.ctx.status.is_restarting() + and self.master.get_status() != self.ctx.status.COMPLETED + ): self.pod.stop(timeout=30) return False @@ -190,12 +193,9 @@ def _get_out_err_file(self, out=None, err=None): err = os.path.join(self.ctx.args.log_dir, err) return out, (err or out) - def new_container(self, - entrypoint=None, - envs={}, - use_ctx_env=True, - out=None, - err=None): + def new_container( + self, entrypoint=None, envs={}, use_ctx_env=True, out=None, err=None + ): c = Container( entrypoint=(entrypoint or self._get_entrypoint()), env=(self.ctx.get_envs() if use_ctx_env else {}), @@ -204,18 +204,19 @@ def new_container(self, c.update_env(envs) return c - def add_container(self, - container=None, - entrypoint=None, - envs={}, - log_file=None, - is_init=False): + def add_container( + self, + container=None, + entrypoint=None, + envs={}, + log_file=None, + is_init=False, + ): if not container: - container = self.new_container(entrypoint=entrypoint, - envs=envs, - out=log_file, - err=log_file) + container = self.new_container( + entrypoint=entrypoint, envs=envs, out=log_file, err=log_file + ) if is_init: self.pod.add_init_container(container) @@ -241,8 +242,10 @@ def save_pod_log(self, info): if not self.ctx.args.log_dir: return - f = os.path.join(self.ctx.args.log_dir, - '{}.{}.log'.format(self.job.id, self.pod.name)) + f = os.path.join( + self.ctx.args.log_dir, + '{}.{}.log'.format(self.job.id, self.pod.name), + ) try: os.makedirs(os.path.dirname(f), exist_ok=True) with open(f, 'a+') as fd: diff --git a/python/paddle/distributed/launch/controllers/ipu_controller.py b/python/paddle/distributed/launch/controllers/ipu_controller.py index 92dc2960ab624..ea342e5ebe03a 100644 --- a/python/paddle/distributed/launch/controllers/ipu_controller.py +++ b/python/paddle/distributed/launch/controllers/ipu_controller.py @@ -21,7 +21,6 @@ class IPUController(CollectiveController): - @classmethod def enable(cls, ctx): if ctx.args.training_script == "ipu": @@ -33,26 +32,31 @@ def enable(cls, ctx): def parse_ipu_args(self, args_list): parser = argparse.ArgumentParser() - parser.add_argument("--hosts", - type=str, - help="The hosts for IPU distributd training.") - parser.add_argument("--nproc_per_host", - type=int, - help="The number of processes launched per host.") - parser.add_argument("--ipus_per_replica", - type=int, - help="The number of IPUs requested per replica.") - parser.add_argument("--ipu_partition", - type=str, - help="The partition name of IPU devices.") - parser.add_argument("--vipu_server", - type=str, - help="The ip of the IPU device manager.") + parser.add_argument( + "--hosts", type=str, help="The hosts for IPU distributd training." + ) + parser.add_argument( + "--nproc_per_host", + type=int, + help="The number of processes launched per host.", + ) + parser.add_argument( + "--ipus_per_replica", + type=int, + help="The number of IPUs requested per replica.", + ) + parser.add_argument( + "--ipu_partition", + type=str, + help="The partition name of IPU devices.", + ) + parser.add_argument( + "--vipu_server", type=str, help="The ip of the IPU device manager." + ) parser.add_argument( "training_script", type=str, - help= - "The full path to the IPU distributed training program/script to be launched in parallel. e.g., ``training.py``." + help="The full path to the IPU distributed training program/script to be launched in parallel. e.g., ``training.py``.", ) parser.add_argument('training_script_args', nargs=argparse.REMAINDER) return parser.parse_args(args_list) @@ -64,19 +68,27 @@ def replace_training_script(self): num_ipus = int(self.ctx.args.devices) # The number of replicas for data parallel - assert (num_ipus % poprun_args.ipus_per_replica) == 0, \ - "The number of IPUs:{} mod the number of IPUs per replica:{} must == 0".format(num_ipus, poprun_args.ipus_per_replica) + assert ( + num_ipus % poprun_args.ipus_per_replica + ) == 0, "The number of IPUs:{} mod the number of IPUs per replica:{} must == 0".format( + num_ipus, poprun_args.ipus_per_replica + ) num_replicas = num_ipus // poprun_args.ipus_per_replica self.ctx.logger.info( - "The number of total replicas is {}.".format(num_replicas)) + "The number of total replicas is {}.".format(num_replicas) + ) # The number of processes num_nodes = len(poprun_args.hosts.split(',')) num_procs = num_nodes * poprun_args.nproc_per_host self.ctx.logger.info( - "The number of total processes is {}.".format(num_procs)) - assert (num_replicas % num_procs) == 0, \ - "The number of replicas:{} mod the number of processes:{} must == 0".format(num_replicas, num_procs) + "The number of total processes is {}.".format(num_procs) + ) + assert ( + num_replicas % num_procs + ) == 0, "The number of replicas:{} mod the number of processes:{} must == 0".format( + num_replicas, num_procs + ) # hosts and endpoints hosts = poprun_args.hosts.replace(' ', '').split(',') @@ -87,26 +99,36 @@ def replace_training_script(self): poprun_command.append('--num-instances={}'.format(num_procs)) poprun_command.append('--num-replicas={}'.format(num_replicas)) - poprun_command.append('--ipus-per-replica={}'.format( - poprun_args.ipus_per_replica)) + poprun_command.append( + '--ipus-per-replica={}'.format(poprun_args.ipus_per_replica) + ) poprun_command.append('--host={}'.format(','.join(hosts))) - poprun_command.append('--vipu-partition={}'.format( - poprun_args.ipu_partition)) - poprun_command.append('--vipu-server-host={}'.format( - poprun_args.vipu_server)) + poprun_command.append( + '--vipu-partition={}'.format(poprun_args.ipu_partition) + ) + poprun_command.append( + '--vipu-server-host={}'.format(poprun_args.vipu_server) + ) - poprun_command.extend([ - '--update-partition=no', '--vipu-server-timeout=120', - '--print-topology=yes', '--numa-aware=yes' - ]) + poprun_command.extend( + [ + '--update-partition=no', + '--vipu-server-timeout=120', + '--print-topology=yes', + '--numa-aware=yes', + ] + ) # global envs global_envs = '--mpi-local-args=\'' log_level = os.getenv('POPART_LOG_LEVEL', None) if log_level: global_envs += '-x POPART_LOG_LEVEL={} '.format(log_level) - global_envs += '-x PADDLE_TRAINERS_NUM={} -x PADDLE_TRAINER_ENDPOINTS={}'.format( - num_procs, ','.join(endpoints)) + global_envs += ( + '-x PADDLE_TRAINERS_NUM={} -x PADDLE_TRAINER_ENDPOINTS={}'.format( + num_procs, ','.join(endpoints) + ) + ) global_envs += '\'' poprun_command.append(global_envs) @@ -115,8 +137,10 @@ def replace_training_script(self): cur_endpoint = endpoints[idx // poprun_args.nproc_per_host] rank_in_node = idx % poprun_args.nproc_per_host poprun_command.append( - '--instance-mpi-local-args={}:\"-x PADDLE_TRAINER_ID={} -x PADDLE_CURRENT_ENDPOINT={} -x PADDLE_RANK_IN_NODE={}\"' - .format(idx, idx, cur_endpoint, rank_in_node)) + '--instance-mpi-local-args={}:\"-x PADDLE_TRAINER_ID={} -x PADDLE_CURRENT_ENDPOINT={} -x PADDLE_RANK_IN_NODE={}\"'.format( + idx, idx, cur_endpoint, rank_in_node + ) + ) # executor poprun_command.append(sys.executable) @@ -142,12 +166,9 @@ def _get_entrypoint(self): entrypoint = [" ".join(entrypoint)] return entrypoint - def new_container(self, - entrypoint=None, - envs={}, - use_ctx_env=True, - out=None, - err=None): + def new_container( + self, entrypoint=None, envs={}, use_ctx_env=True, out=None, err=None + ): c = Container( entrypoint=(entrypoint or self._get_entrypoint()), env=(self.ctx.get_envs() if use_ctx_env else {}), diff --git a/python/paddle/distributed/launch/controllers/master.py b/python/paddle/distributed/launch/controllers/master.py index da240ceffa9f3..128852e092cfb 100644 --- a/python/paddle/distributed/launch/controllers/master.py +++ b/python/paddle/distributed/launch/controllers/master.py @@ -64,7 +64,6 @@ def factory(cls, ctx): class HTTPMaster(Master): - def lazy_init(self): if self.initialized: return @@ -83,7 +82,8 @@ def lazy_init(self): break except Exception as e: self.ctx.logger.warning( - "start master failed {}".format(e)) + "start master failed {}".format(e) + ) time.sleep(0.1) continue else: @@ -94,7 +94,9 @@ def lazy_init(self): print("Copy the following command to other nodes to run.") cmd = [ - sys.executable.split('/')[-1], "-m", "paddle.distributed.launch" + sys.executable.split('/')[-1], + "-m", + "paddle.distributed.launch", ] cmd.extend(["--master", self.endpoint]) cmd.extend(sys.argv[1:]) @@ -104,7 +106,8 @@ def lazy_init(self): if int(self.ctx.args.rank) >= 0: self.ctx.logger.warning( - "--rank set in the command may not compatible in auto mode") + "--rank set in the command may not compatible in auto mode" + ) if '127.0.0.1' in self.endpoint: self.endpoint = self.endpoint.replace('127.0.0.1', self.ctx.node.ip) @@ -173,7 +176,6 @@ def sync_peers(self, prefix, key, value, size, rank=-1) -> (list, int): class ETCDMaster(Master): - def __init__(self, ctx): super().__init__(ctx) @@ -225,7 +227,8 @@ def sync_peers(self, prefix, key, value, size, rank=-1) -> (list, int): ii = int(six.ensure_str(k.key).split('/')[-1]) if ii < 0: self.ctx.logger.error( - "rank {} error in sync".format(ii)) + "rank {} error in sync".format(ii) + ) ret[ii] = six.ensure_str(v) return ret, rank else: @@ -241,7 +244,7 @@ def register_heartbeat(self, job_id, pod_id, ttl=10): lease = self.client.lease(ttl) - #self.client.delete_prefix(self.job_prefix) + # self.client.delete_prefix(self.job_prefix) beat_path = "{}/{}".format(self.heartbeat_prefix, pod_id) self.client.put(beat_path, pod_id.encode('latin-1'), lease=lease) @@ -250,16 +253,17 @@ def _beat_watch(event): self.ctx.status.restart() beat_watch = self.client.add_watch_prefix_callback( - self.heartbeat_prefix, _beat_watch) + self.heartbeat_prefix, _beat_watch + ) def _heartbeat(): while not self.ctx.status.is_done(): try: lease.refresh() if pod_id not in self.fetch_peer_alive(): - self.client.put(beat_path, - pod_id.encode('latin-1'), - lease=lease) + self.client.put( + beat_path, pod_id.encode('latin-1'), lease=lease + ) self.ctx.logger.debug("Heartbeat register again") except Exception as e: self.ctx.logger.error("Heartbeat error {}".format(e)) @@ -267,9 +271,9 @@ def _heartbeat(): self.ctx.logger.debug("Heartbeat done") self.client.cancel_watch(beat_watch) - self.beat_thread = threading.Thread(name='heartbeat', - target=_heartbeat, - daemon=True) + self.beat_thread = threading.Thread( + name='heartbeat', target=_heartbeat, daemon=True + ) self.beat_thread.start() def fetch_peer_alive(self): @@ -311,7 +315,8 @@ def set_status(self, status): assert self.client.put( self.job_prefix, status.encode('latin-1'), - lease=self.client.lease(600)), "set status failed {}".format(status) + lease=self.client.lease(600), + ), "set status failed {}".format(status) def get_status(self): return six.ensure_str(self.client.get(self.job_prefix)[0] or '') @@ -320,4 +325,4 @@ def stop(self): if hasattr(self, 'beat_thread'): self.ctx.status.done() # daemon thread - #self.beat_thread.join() + # self.beat_thread.join() diff --git a/python/paddle/distributed/launch/controllers/ps.py b/python/paddle/distributed/launch/controllers/ps.py index f785311a52540..b78ab34d49096 100644 --- a/python/paddle/distributed/launch/controllers/ps.py +++ b/python/paddle/distributed/launch/controllers/ps.py @@ -19,12 +19,15 @@ class PSController(Controller): - @classmethod def enable(cls, ctx): - if ctx.args.run_mode == ControleMode.PS or ctx.args.server_num or len( - ctx.args.servers) > 0 or ctx.args.trainer_num or len( - ctx.args.trainers) > 0: + if ( + ctx.args.run_mode == ControleMode.PS + or ctx.args.server_num + or len(ctx.args.servers) > 0 + or ctx.args.trainer_num + or len(ctx.args.trainers) > 0 + ): ctx.logger.debug("{} enabled".format(cls.__name__)) ctx.args.run_mode = ControleMode.PS return True @@ -59,6 +62,7 @@ def _build_pod_with_args(self): self.save_pod_log([server_endpoints, trainer_endpoints]) import tempfile + gloo_rendezvous_dir = tempfile.mkdtemp() if os.path.exists(gloo_rendezvous_dir): shutil.rmtree(gloo_rendezvous_dir) @@ -70,7 +74,7 @@ def _build_pod_with_args(self): "PADDLE_GLOO_RENDEZVOUS": "3", "PADDLE_GLOO_FS_PATH": gloo_rendezvous_dir, "PADDLE_GLOO_HTTP_ENDPOINT": gloo_http, - "PADDLE_WITH_GLOO": self.ctx.args.with_gloo + "PADDLE_WITH_GLOO": self.ctx.args.with_gloo, } for i in range(server_num): @@ -124,19 +128,24 @@ def _build_pod_with_master(self): for p in self.ctx.node.get_free_ports(trainer_num) ] - data = json.dumps({ - 'name': self.pod.name, - 'rank': self.pod.rank, - 'servers': servers, - 'trainers': trainers, - 'dtype': self.ctx.node.device.dtype, - 'gloo_port': self.ctx.node.get_free_port(), - }) + data = json.dumps( + { + 'name': self.pod.name, + 'rank': self.pod.rank, + 'servers': servers, + 'trainers': trainers, + 'dtype': self.ctx.node.device.dtype, + 'gloo_port': self.ctx.node.get_free_port(), + } + ) - peer_list, rank = self.master.sync_peers('/{}/info'.format(self.job.id), - self.pod.name, data, - self.job.replicas, - self.pod.rank) + peer_list, rank = self.master.sync_peers( + '/{}/info'.format(self.job.id), + self.pod.name, + data, + self.job.replicas, + self.pod.rank, + ) self.ctx.logger.debug("sync peers done {}".format(peer_list)) @@ -146,17 +155,19 @@ def _build_pod_with_master(self): server_endpoints = [j for i in peer_list for j in i['servers']] trainer_endpoints = [j for i in peer_list for j in i['trainers']] - #rank_offset = sum([i['replicas'] for i in peer_list[:rank]]) + # rank_offset = sum([i['replicas'] for i in peer_list[:rank]]) server_rank_offset = sum([len(i['servers']) for i in peer_list[:rank]]) trainer_rank_offset = sum( - [len(i['trainers']) for i in peer_list[:rank]]) + [len(i['trainers']) for i in peer_list[:rank]] + ) self.pod.rank = rank self.pod.replicas = server_num + trainer_num import tempfile + gloo_rendezvous_dir = tempfile.mkdtemp() if os.path.exists(gloo_rendezvous_dir): shutil.rmtree(gloo_rendezvous_dir) @@ -168,27 +179,21 @@ def _build_pod_with_master(self): "PADDLE_GLOO_RENDEZVOUS": "3", "PADDLE_GLOO_FS_PATH": gloo_rendezvous_dir, "PADDLE_GLOO_HTTP_ENDPOINT": gloo_http, - "PADDLE_WITH_GLOO": self.ctx.args.with_gloo + "PADDLE_WITH_GLOO": self.ctx.args.with_gloo, } for i in range(server_num): e = { - "PADDLE_NNODES": - "{}".format(self.job.replicas), - "PADDLE_PSERVERS_IP_PORT_LIST": - ",".join(server_endpoints), - "PADDLE_TRAINER_ENDPOINTS": - ",".join(trainer_endpoints), - "PADDLE_PORT": - server_endpoints[i + server_rank_offset].split(":")[1], - "PADDLE_ROLE": - "PSERVER", - "TRAINING_ROLE": - "PSERVER", - "PADDLE_TRAINERS_NUM": - "{}".format(len(trainer_endpoints)), - "POD_IP": - self.ctx.node.ip, + "PADDLE_NNODES": "{}".format(self.job.replicas), + "PADDLE_PSERVERS_IP_PORT_LIST": ",".join(server_endpoints), + "PADDLE_TRAINER_ENDPOINTS": ",".join(trainer_endpoints), + "PADDLE_PORT": server_endpoints[i + server_rank_offset].split( + ":" + )[1], + "PADDLE_ROLE": "PSERVER", + "TRAINING_ROLE": "PSERVER", + "PADDLE_TRAINERS_NUM": "{}".format(len(trainer_endpoints)), + "POD_IP": self.ctx.node.ip, } e.update(_gloo_envs) log_file = "serverlog.{}".format(i) @@ -196,24 +201,17 @@ def _build_pod_with_master(self): for i in range(trainer_num): e = { - "PADDLE_NNODES": - "{}".format(self.job.replicas), - "PADDLE_PSERVERS_IP_PORT_LIST": - ",".join(server_endpoints), - "PADDLE_TRAINER_ENDPOINTS": - ",".join(trainer_endpoints), - "PADDLE_PORT": - trainer_endpoints[i + trainer_rank_offset].split(":")[1], - "PADDLE_ROLE": - "TRAINER", - "TRAINING_ROLE": - "TRAINER", - "PADDLE_TRAINER_ID": - "{}".format(i + trainer_rank_offset), - "PADDLE_TRAINERS_NUM": - "{}".format(len(trainer_endpoints)), - "POD_IP": - self.ctx.node.ip, + "PADDLE_NNODES": "{}".format(self.job.replicas), + "PADDLE_PSERVERS_IP_PORT_LIST": ",".join(server_endpoints), + "PADDLE_TRAINER_ENDPOINTS": ",".join(trainer_endpoints), + "PADDLE_PORT": trainer_endpoints[i + trainer_rank_offset].split( + ":" + )[1], + "PADDLE_ROLE": "TRAINER", + "TRAINING_ROLE": "TRAINER", + "PADDLE_TRAINER_ID": "{}".format(i + trainer_rank_offset), + "PADDLE_TRAINERS_NUM": "{}".format(len(trainer_endpoints)), + "POD_IP": self.ctx.node.ip, } e.update(_gloo_envs) log_file = "workerlog.{}".format(i) diff --git a/python/paddle/distributed/launch/controllers/rpc.py b/python/paddle/distributed/launch/controllers/rpc.py index d68c389da9e03..0d3c314ec7788 100644 --- a/python/paddle/distributed/launch/controllers/rpc.py +++ b/python/paddle/distributed/launch/controllers/rpc.py @@ -18,7 +18,6 @@ class RpcController(Controller): - @classmethod def enable(cls, ctx): if ctx.args.run_mode == ControleMode.RPC: @@ -28,8 +27,9 @@ def enable(cls, ctx): return False def build_pod(self): - assert (self.ctx.args.master - is not None), "Master is None, Please set master address!" + assert ( + self.ctx.args.master is not None + ), "Master is None, Please set master address!" self._build_pod_with_master() def _build_pod_with_master(self): @@ -47,14 +47,16 @@ def _build_pod_with_master(self): for p in self.ctx.node.get_free_ports(self.pod.replicas) ] - data = json.dumps({ - "name": self.pod.name, - "rank": self.pod.rank, - "replicas": self.pod.replicas, - "dtype": self.ctx.node.device.dtype, - "candidate": "{}:{}".format(self.ctx.node.ip, port), - "endpoints": ",".join(endpoints), - }) + data = json.dumps( + { + "name": self.pod.name, + "rank": self.pod.rank, + "replicas": self.pod.replicas, + "dtype": self.ctx.node.device.dtype, + "candidate": "{}:{}".format(self.ctx.node.ip, port), + "endpoints": ",".join(endpoints), + } + ) peer_list, rank = self.master.sync_peers( "/{}/info".format(self.job.id), self.pod.name, diff --git a/python/paddle/distributed/launch/controllers/watcher.py b/python/paddle/distributed/launch/controllers/watcher.py index 4b8e346e7908f..a9c1f50966687 100644 --- a/python/paddle/distributed/launch/controllers/watcher.py +++ b/python/paddle/distributed/launch/controllers/watcher.py @@ -20,7 +20,6 @@ class Watcher(object): - def __init__(self, ctx): self.ctx = ctx @@ -31,8 +30,9 @@ def __init__(self, ctx): # gpu log file self.gpus = self.ctx.args.devices or self.ctx.node.device.labels if len(self.gpus) > 0: - fn = os.path.join(self.ctx.args.log_dir, - "{}.gpu.log".format(self.ctx.args.job_id)) + fn = os.path.join( + self.ctx.args.log_dir, "{}.gpu.log".format(self.ctx.args.job_id) + ) os.makedirs(os.path.dirname(fn), exist_ok=True) self.gpu_fd = open(fn, 'w') else: diff --git a/python/paddle/distributed/launch/job/container.py b/python/paddle/distributed/launch/job/container.py index 55223bacdd848..6eb313ea57965 100644 --- a/python/paddle/distributed/launch/job/container.py +++ b/python/paddle/distributed/launch/job/container.py @@ -89,7 +89,8 @@ def update_env(self, env={}, **kwargs): def _valide_env(self): for k, v in self._env.items(): assert isinstance(k, str) and isinstance( - v, str), 'env {}:{} must be str'.format(k, v) + v, str + ), 'env {}:{} must be str'.format(k, v) def _get_fd(self, pth): if not pth: @@ -120,11 +121,13 @@ def start(self): self._log_handler.seek(0, 2) self._log_start_offset = self._log_handler.tell() - self._proc = ProcessContext(self._entrypoint, - env=self._env, - out=self._stdout, - err=self._stderr, - shell=self._shell) + self._proc = ProcessContext( + self._entrypoint, + env=self._env, + out=self._stdout, + err=self._stderr, + shell=self._shell, + ) self._proc.start() @@ -159,13 +162,15 @@ def status(self): return Status.FAILED def __str__(self): - return 'Container rank {} status {} cmd {} code {} log {} \nenv {}'.format( - self._rank, - self.status, - self._entrypoint, - self.exit_code, - self.errfile, - self._env, + return ( + 'Container rank {} status {} cmd {} code {} log {} \nenv {}'.format( + self._rank, + self.status, + self._entrypoint, + self.exit_code, + self.errfile, + self._env, + ) ) def logs(self, fn=None, offset=0, whence=1, limit=1000): diff --git a/python/paddle/distributed/launch/job/job.py b/python/paddle/distributed/launch/job/job.py index 4bad1209c1859..f5c805e31bf63 100644 --- a/python/paddle/distributed/launch/job/job.py +++ b/python/paddle/distributed/launch/job/job.py @@ -20,7 +20,6 @@ class JobMode: class Job(object): - def __init__(self, jid='default', mode=JobMode.COLLECTIVE, nnodes="1"): self._mode = mode self._id = jid @@ -34,8 +33,13 @@ def __init__(self, jid='default', mode=JobMode.COLLECTIVE, nnodes="1"): def __str__(self): return "Job: {}, mode {}, replicas {}[{}:{}], elastic {}".format( - self.id, self.mode, self._replicas, self._replicas_min, - self._replicas_max, self.elastic) + self.id, + self.mode, + self._replicas, + self._replicas_min, + self._replicas_max, + self.elastic, + ) @property def mode(self): @@ -76,6 +80,9 @@ def set_replicas(self, nnodes: str): self._elastic = True else: self._replicas = int(np) - self._replicas_min, self._replicas_max = self._replicas, self._replicas + self._replicas_min, self._replicas_max = ( + self._replicas, + self._replicas, + ) self._elastic = False diff --git a/python/paddle/distributed/launch/job/pod.py b/python/paddle/distributed/launch/job/pod.py index 960a52aa3f368..b65aad6e0fcb4 100644 --- a/python/paddle/distributed/launch/job/pod.py +++ b/python/paddle/distributed/launch/job/pod.py @@ -21,17 +21,17 @@ class PodSepc(object): - def __init__(self): self._name = ''.join( - random.choice('abcdefghijklmnopqrstuvwxyz') for _ in range(6)) + random.choice('abcdefghijklmnopqrstuvwxyz') for _ in range(6) + ) # by controller self._init_containers: List[Container] = [] self._containers: List[Container] = [] - #self.resource: Resource = None - #self.status: Status = None + # self.resource: Resource = None + # self.status: Status = None self._rank = -1 self._init_timeout = None @@ -41,14 +41,13 @@ def __init__(self): class Pod(PodSepc): - def __init__(self): super().__init__() def __str__(self): - return "Pod: {}, replicas {}, status {}".format(self.name, - self.replicas, - self.status) + return "Pod: {}, replicas {}, status {}".format( + self.name, self.replicas, self.status + ) def failed_container(self): cs = [] @@ -183,11 +182,13 @@ def tail(self, idx=None): else: self._containers[idx].tail() - def watch(self, - all_list=[Status.COMPLETED], - any_list=[Status.FAILED], - interval=1, - timeout=-1): + def watch( + self, + all_list=[Status.COMPLETED], + any_list=[Status.FAILED], + interval=1, + timeout=-1, + ): ''' watch return if any container status in any_list or all container status in all_list diff --git a/python/paddle/distributed/launch/main.py b/python/paddle/distributed/launch/main.py index a6a0c973208fd..c960239f73319 100644 --- a/python/paddle/distributed/launch/main.py +++ b/python/paddle/distributed/launch/main.py @@ -290,6 +290,7 @@ def launch(): # legacy mode from paddle.distributed.fleet import launch + launch.launch() else: diff --git a/python/paddle/distributed/launch/plugins/__init__.py b/python/paddle/distributed/launch/plugins/__init__.py index 91a2dee09a3b8..2d7632d642cf2 100644 --- a/python/paddle/distributed/launch/plugins/__init__.py +++ b/python/paddle/distributed/launch/plugins/__init__.py @@ -27,7 +27,7 @@ def log(ctx): def process_args(ctx): # reset device by args - #argdev = ctx.args.gpus or ctx.args.xpus or ctx.args.npus + # argdev = ctx.args.gpus or ctx.args.xpus or ctx.args.npus argdev = ctx.args.devices if argdev: for d in argdev.split(','): @@ -49,7 +49,8 @@ def collective_compatible(ctx): ctx.args.master = eps[0] if ':' in eps[0] else '{}:6768'.format(eps[0]) ctx.args.nnodes = len(hosts) ctx.logger.info( - 'args reset by env PADDLE_TRAINER_ENDPOINTS\n{}'.format(eps)) + 'args reset by env PADDLE_TRAINER_ENDPOINTS\n{}'.format(eps) + ) if 'DISTRIBUTED_TRAINER_ENDPOINTS' in ctx.envs: eps = ctx.envs['DISTRIBUTED_TRAINER_ENDPOINTS'].split(',') @@ -57,7 +58,8 @@ def collective_compatible(ctx): ctx.args.master = eps[0] ctx.args.nnodes = len(hosts) ctx.logger.info( - 'args reset by env DISTRIBUTED_TRAINER_ENDPOINTS\n{}'.format(eps)) + 'args reset by env DISTRIBUTED_TRAINER_ENDPOINTS\n{}'.format(eps) + ) def rewrite_host_ip(ctx): @@ -72,9 +74,13 @@ def test_mode(ctx): if int(ctx.args.nnodes) < 2: ctx.args.nnodes = 2 ctx.args.training_script = '{}/test.py'.format( - os.path.dirname(__file__)) + os.path.dirname(__file__) + ) enabled_plugins = [ - test_mode, collective_compatible, rewrite_host_ip, process_args + test_mode, + collective_compatible, + rewrite_host_ip, + process_args, ] diff --git a/python/paddle/distributed/launch/plugins/test.py b/python/paddle/distributed/launch/plugins/test.py index ae4c111351ea3..3dd7ab886e61b 100644 --- a/python/paddle/distributed/launch/plugins/test.py +++ b/python/paddle/distributed/launch/plugins/test.py @@ -31,13 +31,12 @@ # define a random dataset class RandomDataset(Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([3, 224, 224]).astype('float32') - label = np.random.randint(0, class_dim - 1, (1, )).astype('int64') + label = np.random.randint(0, class_dim - 1, (1,)).astype('int64') return image, label def __len__(self): @@ -49,7 +48,8 @@ def optimizer_setting(parameter_list=None): learning_rate=base_lr, momentum=momentum_rate, weight_decay=paddle.regularizer.L2Decay(l2_decay), - parameters=parameter_list) + parameters=parameter_list, + ) return optimizer @@ -62,11 +62,13 @@ def train_resnet(): resnet = fleet.distributed_model(resnet) dataset = RandomDataset(batch_num * batch_size) - train_loader = DataLoader(dataset, - batch_size=batch_size, - shuffle=True, - drop_last=True, - num_workers=2) + train_loader = DataLoader( + dataset, + batch_size=batch_size, + shuffle=True, + drop_last=True, + num_workers=2, + ) print("Distributed training start...") for eop in range(epoch): @@ -86,14 +88,17 @@ def train_resnet(): optimizer.step() resnet.clear_gradients() - print("[Epoch %d, batch %d] loss: %.5f, acc1: %.5f, acc5: %.5f" % - (eop, batch_id, avg_loss, acc_top1, acc_top5)) + print( + "[Epoch %d, batch %d] loss: %.5f, acc1: %.5f, acc5: %.5f" + % (eop, batch_id, avg_loss, acc_top1, acc_top5) + ) print("Distributed training completed") if __name__ == '__main__': import os + nnodes = os.getenv('PADDLE_NNODES') cn = os.getenv('PADDLE_LOCAL_SIZE') print(f"Prepare distributed training with {nnodes} nodes {cn} cards") diff --git a/python/paddle/distributed/launch/utils/kv_client.py b/python/paddle/distributed/launch/utils/kv_client.py index a66ca800c58c2..8ed46053de0d3 100644 --- a/python/paddle/distributed/launch/utils/kv_client.py +++ b/python/paddle/distributed/launch/utils/kv_client.py @@ -17,10 +17,12 @@ class KVClient(object): - def __init__(self, endpoint='localhost:2379'): - self.endpoint = endpoint if endpoint.startswith( - "http://") else "http://{}".format(endpoint) + self.endpoint = ( + endpoint + if endpoint.startswith("http://") + else "http://{}".format(endpoint) + ) def put(self, key, value): key = key if key.startswith('/') else "/{}".format(key) diff --git a/python/paddle/distributed/launch/utils/kv_server.py b/python/paddle/distributed/launch/utils/kv_server.py index ddf5685c988b7..7c8807d3a16b4 100644 --- a/python/paddle/distributed/launch/utils/kv_server.py +++ b/python/paddle/distributed/launch/utils/kv_server.py @@ -22,7 +22,6 @@ class KVHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): - def do_GET(self): with self.server.kv_lock: ret = {} @@ -69,7 +68,6 @@ def log_message(self, format, *args): class KVServer(HTTPServer, object): - def __init__(self, port): super(KVServer, self).__init__(('', port), KVHandler) self.kv_lock = threading.Lock() @@ -90,8 +88,7 @@ def stop(self): self.stopped = True -class PKVServer(): - +class PKVServer: def __init__(self, port): self._server = KVServer(port) @@ -114,11 +111,11 @@ def stopped(self): if __name__ == '__main__': - #kv = PKVServer(8090) + # kv = PKVServer(8090) kv = KVServer(8090) kv.start() import time - #print("serve at 8090 for 600 s") + # print("serve at 8090 for 600 s") time.sleep(600) diff --git a/python/paddle/distributed/launch/utils/nvsmi.py b/python/paddle/distributed/launch/utils/nvsmi.py index 785704be3ff74..d1a14f11666d2 100644 --- a/python/paddle/distributed/launch/utils/nvsmi.py +++ b/python/paddle/distributed/launch/utils/nvsmi.py @@ -19,7 +19,6 @@ class Info(object): - def __repr__(self): return str(self.__dict__) @@ -76,20 +75,28 @@ def query_smi(query=None, query_type="gpu", index=None, dtype=None): def get_gpu_info(index=None): q = "index,uuid,driver_version,name,gpu_serial,display_active,display_mode".split( - ",") + "," + ) d = [int, str, str, str, str, str, str] - index = index if index is None or isinstance( - index, list) else str(index).split(",") + index = ( + index + if index is None or isinstance(index, list) + else str(index).split(",") + ) return query_smi(q, index=index, dtype=d) def get_gpu_util(index=None): q = "index,utilization.gpu,memory.total,memory.used,memory.free,timestamp".split( - ",") + "," + ) d = [int, int, int, int, int, str] - index = index if index is None or isinstance( - index, list) else str(index).split(",") + index = ( + index + if index is None or isinstance(index, list) + else str(index).split(",") + ) return query_smi(q, index=index, dtype=d) @@ -97,8 +104,11 @@ def get_gpu_util(index=None): def get_gpu_process(index=None): q = "pid,process_name,gpu_uuid,gpu_name,used_memory".split(",") d = [int, str, str, str, int] - index = index if index is None or isinstance( - index, list) else str(index).split(",") + index = ( + index + if index is None or isinstance(index, list) + else str(index).split(",") + ) return query_smi(q, index=index, query_type="compute", dtype=d) diff --git a/python/paddle/distributed/launch/utils/process_context.py b/python/paddle/distributed/launch/utils/process_context.py index 5d8505aa66eb3..682a857f2ee6c 100644 --- a/python/paddle/distributed/launch/utils/process_context.py +++ b/python/paddle/distributed/launch/utils/process_context.py @@ -17,15 +17,16 @@ class ProcessContext(object): - - def __init__(self, - cmd, - env=os.environ, - out=sys.stdout, - err=sys.stderr, - group=True, - preexec_fn=None, - shell=False): + def __init__( + self, + cmd, + env=os.environ, + out=sys.stdout, + err=sys.stderr, + group=True, + preexec_fn=None, + shell=False, + ): self._cmd = cmd self._env = env self._preexec_fn = preexec_fn @@ -38,12 +39,14 @@ def __init__(self, def _start(self): pre_fn = os.setsid if self._group else None - self._proc = subprocess.Popen(self._cmd, - env=self._env, - stdout=self._stdout, - stderr=self._stderr, - preexec_fn=self._preexec_fn or pre_fn, - shell=self._shell) + self._proc = subprocess.Popen( + self._cmd, + env=self._env, + stdout=self._stdout, + stderr=self._stderr, + preexec_fn=self._preexec_fn or pre_fn, + shell=self._shell, + ) def _close_std(self): try: diff --git a/python/paddle/distributed/metric/metrics.py b/python/paddle/distributed/metric/metrics.py index 9f8573183b37d..64f62d85251ab 100644 --- a/python/paddle/distributed/metric/metrics.py +++ b/python/paddle/distributed/metric/metrics.py @@ -22,15 +22,17 @@ # read metric config from yaml and init MetricMsg in fleet_wrapper -def init_metric(metric_ptr, - metric_yaml_path, - cmatch_rank_var="", - mask_var="", - uid_var="", - phase=-1, - cmatch_rank_group="", - ignore_rank=False, - bucket_size=1000000): +def init_metric( + metric_ptr, + metric_yaml_path, + cmatch_rank_var="", + mask_var="", + uid_var="", + phase=-1, + cmatch_rank_group="", + ignore_rank=False, + bucket_size=1000000, +): yaml_fobj = open(metric_yaml_path) if sys.version.startswith('2.7.13'): content = yaml.load(yaml_fobj) @@ -49,53 +51,102 @@ def init_metric(metric_ptr, phase = 1 if is_join else 0 if metric_runner['method'] == 'AucCalculator': - metric_ptr.init_metric(metric_runner['method'], - metric_runner['name'], - metric_runner['label'], - metric_runner['target'], cmatch_rank_var, - mask_var, uid_var, phase, cmatch_rank_group, - ignore_rank, bucket_size) + metric_ptr.init_metric( + metric_runner['method'], + metric_runner['name'], + metric_runner['label'], + metric_runner['target'], + cmatch_rank_var, + mask_var, + uid_var, + phase, + cmatch_rank_group, + ignore_rank, + bucket_size, + ) elif metric_runner['method'] == 'MultiTaskAucCalculator': metric_ptr.init_metric( - metric_runner['method'], metric_runner['name'], - metric_runner['label'], metric_runner['target'], - metric_runner['cmatch_var'], mask_var, uid_var, phase, - metric_runner['cmatch_group'], ignore_rank, bucket_size) + metric_runner['method'], + metric_runner['name'], + metric_runner['label'], + metric_runner['target'], + metric_runner['cmatch_var'], + mask_var, + uid_var, + phase, + metric_runner['cmatch_group'], + ignore_rank, + bucket_size, + ) elif metric_runner['method'] == 'CmatchRankAucCalculator': metric_ptr.init_metric( - metric_runner['method'], metric_runner['name'], - metric_runner['label'], metric_runner['target'], - metric_runner['cmatch_var'], mask_var, uid_var, phase, - metric_runner['cmatch_group'], metric_runner['ignore_rank'], - bucket_size) + metric_runner['method'], + metric_runner['name'], + metric_runner['label'], + metric_runner['target'], + metric_runner['cmatch_var'], + mask_var, + uid_var, + phase, + metric_runner['cmatch_group'], + metric_runner['ignore_rank'], + bucket_size, + ) elif metric_runner['method'] == 'MaskAucCalculator': - metric_ptr.init_metric(metric_runner['method'], - metric_runner['name'], - metric_runner['label'], - metric_runner['target'], cmatch_rank_var, - metric_runner['mask'], uid_var, phase, - cmatch_rank_group, ignore_rank, bucket_size) + metric_ptr.init_metric( + metric_runner['method'], + metric_runner['name'], + metric_runner['label'], + metric_runner['target'], + cmatch_rank_var, + metric_runner['mask'], + uid_var, + phase, + cmatch_rank_group, + ignore_rank, + bucket_size, + ) elif metric_runner['method'] == 'CmatchRankMaskAucCalculator': metric_ptr.init_metric( - metric_runner['method'], metric_runner['name'], - metric_runner['label'], metric_runner['target'], - metric_runner['cmatch_var'], metric_runner['mask'], uid_var, - phase, metric_runner['cmatch_group'], - metric_runner['ignore_rank'], bucket_size) + metric_runner['method'], + metric_runner['name'], + metric_runner['label'], + metric_runner['target'], + metric_runner['cmatch_var'], + metric_runner['mask'], + uid_var, + phase, + metric_runner['cmatch_group'], + metric_runner['ignore_rank'], + bucket_size, + ) elif metric_runner['method'] == 'WuAucCalculator': - metric_ptr.init_metric(metric_runner['method'], - metric_runner['name'], - metric_runner['label'], - metric_runner['target'], cmatch_rank_var, - mask_var, metric_runner['uid'], phase, - cmatch_rank_group, ignore_rank, bucket_size) + metric_ptr.init_metric( + metric_runner['method'], + metric_runner['name'], + metric_runner['label'], + metric_runner['target'], + cmatch_rank_var, + mask_var, + metric_runner['uid'], + phase, + cmatch_rank_group, + ignore_rank, + bucket_size, + ) else: - metric_ptr.init_metric(metric_runner['method'], - metric_runner['name'], - metric_runner['label'], - metric_runner['target'], cmatch_rank_var, - mask_var, phase, cmatch_rank_group, - ignore_rank, bucket_size) + metric_ptr.init_metric( + metric_runner['method'], + metric_runner['name'], + metric_runner['label'], + metric_runner['target'], + cmatch_rank_var, + mask_var, + phase, + cmatch_rank_group, + ignore_rank, + bucket_size, + ) def print_metric(metric_ptr, name): @@ -104,14 +155,27 @@ def print_metric(metric_ptr, name): """ if name.find("wuauc") != -1: metric = metric_ptr.get_wuauc_metric_msg(name) - monitor_msg = "%s: User Count=%.0f INS Count=%.0f UAUC=%.6f WUAUC=%.6f "\ - % (name, metric[0], metric[1], metric[4], metric[5]) + monitor_msg = ( + "%s: User Count=%.0f INS Count=%.0f UAUC=%.6f WUAUC=%.6f " + % (name, metric[0], metric[1], metric[4], metric[5]) + ) else: metric = metric_ptr.get_metric_msg(name) - monitor_msg = "%s: AUC=%.6f BUCKET_ERROR=%.6f MAE=%.6f RMSE=%.6f "\ - "Actual CTR=%.6f Predicted CTR=%.6f COPC=%.6f INS Count=%.0f"\ - % (name, metric[0], metric[1], metric[2], metric[3], metric[4], - metric[5], metric[6], metric[7]) + monitor_msg = ( + "%s: AUC=%.6f BUCKET_ERROR=%.6f MAE=%.6f RMSE=%.6f " + "Actual CTR=%.6f Predicted CTR=%.6f COPC=%.6f INS Count=%.0f" + % ( + name, + metric[0], + metric[1], + metric[2], + metric[3], + metric[4], + metric[5], + metric[6], + metric[7], + ) + ) # logger.info(monitor_msg) return monitor_msg diff --git a/python/paddle/distributed/models/moe/utils.py b/python/paddle/distributed/models/moe/utils.py index 4c6ac50344982..2011e558701e7 100644 --- a/python/paddle/distributed/models/moe/utils.py +++ b/python/paddle/distributed/models/moe/utils.py @@ -51,10 +51,12 @@ def _number_count(numbers, upper_range): helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=numbers.dtype) - helper.append_op(type=op_type, - inputs={'numbers': numbers}, - outputs={'Out': out}, - attrs={'upper_range': upper_range}) + helper.append_op( + type=op_type, + inputs={'numbers': numbers}, + outputs={'Out': out}, + attrs={'upper_range': upper_range}, + ) return out @@ -98,29 +100,31 @@ def _assign_pos(x, cum_count): helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=cum_count.dtype) - helper.append_op(type=op_type, - inputs={ - 'X': [x], - 'cum_count': [cum_count], - "eff_num_len": [cum_count[-1]] - }, - outputs={'Out': [out]}) + helper.append_op( + type=op_type, + inputs={ + 'X': [x], + 'cum_count': [cum_count], + "eff_num_len": [cum_count[-1]], + }, + outputs={'Out': [out]}, + ) return out def _random_routing(topk_idx, topk_value, prob, topk=2): r""" - random routing topk gate idx - ``` - out = topk_idx - for i in len(topk_idx): - if topk * value[i][topk-1] < prob[i]: - out[i][topk-1] = -1 - ``` - Args: - topk_idx: gate idx, shape=(N, topk) - topk_value: values, shape = topk_idx.shape - prob: random prob, shape=(topk_idx.shape[0],) + random routing topk gate idx + ``` + out = topk_idx + for i in len(topk_idx): + if topk * value[i][topk-1] < prob[i]: + out[i][topk-1] = -1 + ``` + Args: + topk_idx: gate idx, shape=(N, topk) + topk_value: values, shape = topk_idx.shape + prob: random prob, shape=(topk_idx.shape[0],) """ if topk == 2: if in_dygraph_mode(): @@ -155,25 +159,27 @@ def _limit_by_capacity(expert_count, capacity, n_worker): print(out) # the result: [1, 2, 2, 4, 3, 3] """ if in_dygraph_mode(): - return _legacy_C_ops.limit_by_capacity(expert_count, capacity, - 'n_worker', n_worker) + return _legacy_C_ops.limit_by_capacity( + expert_count, capacity, 'n_worker', n_worker + ) elif _in_legacy_dygraph(): - return core.ops.limit_by_capacity(expert_count, capacity, 'n_worker', - n_worker) + return core.ops.limit_by_capacity( + expert_count, capacity, 'n_worker', n_worker + ) else: op_type = 'limit_by_capacity' helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference( - dtype=expert_count.dtype) - - helper.append_op(type=op_type, - inputs={ - 'expert_count': expert_count, - 'capacity': capacity - }, - outputs={'Out': out}, - attrs={'n_worker': n_worker}) + dtype=expert_count.dtype + ) + + helper.append_op( + type=op_type, + inputs={'expert_count': expert_count, 'capacity': capacity}, + outputs={'Out': out}, + attrs={'n_worker': n_worker}, + ) return out @@ -202,30 +208,35 @@ def _prune_gate_by_capacity(gate_idx, expert_count, n_expert, n_worker): [1, 3, 3, 3, -1, 2, 1, 1]) """ if in_dygraph_mode(): - return _legacy_C_ops.prune_gate_by_capacity(gate_idx, expert_count, - "n_expert", n_expert, - "n_worker", n_worker) + return _legacy_C_ops.prune_gate_by_capacity( + gate_idx, expert_count, "n_expert", n_expert, "n_worker", n_worker + ) elif _in_legacy_dygraph(): - return core.ops.prune_gate_by_capacity(gate_idx, expert_count, - "n_expert", n_expert, "n_worker", - n_worker) - check_variable_and_dtype(gate_idx, 'GateIdx', ['int32', 'int64'], - 'paddle.distributed.utils.prune_gate_by_capacity') - check_variable_and_dtype(expert_count, 'ExpertCount', ['int32', 'int64'], - 'paddle.distributed.utils.prune_gate_by_capacity') + return core.ops.prune_gate_by_capacity( + gate_idx, expert_count, "n_expert", n_expert, "n_worker", n_worker + ) + check_variable_and_dtype( + gate_idx, + 'GateIdx', + ['int32', 'int64'], + 'paddle.distributed.utils.prune_gate_by_capacity', + ) + check_variable_and_dtype( + expert_count, + 'ExpertCount', + ['int32', 'int64'], + 'paddle.distributed.utils.prune_gate_by_capacity', + ) helper = LayerHelper('prune_gate_by_capacity', **locals()) new_gate_idx = helper.create_variable_for_type_inference( - dtype=gate_idx.dtype) - helper.append_op(type='prune_gate_by_capacity', - inputs={ - 'GateIdx': gate_idx, - "ExpertCount": expert_count - }, - outputs={'NewGateIdx': new_gate_idx}, - attrs={ - "n_expert": n_expert, - "n_worker": n_worker - }) + dtype=gate_idx.dtype + ) + helper.append_op( + type='prune_gate_by_capacity', + inputs={'GateIdx': gate_idx, "ExpertCount": expert_count}, + outputs={'NewGateIdx': new_gate_idx}, + attrs={"n_expert": n_expert, "n_worker": n_worker}, + ) return new_gate_idx diff --git a/python/paddle/distributed/parallel.py b/python/paddle/distributed/parallel.py index 49cb60349f7af..2a74e6bb2cff7 100644 --- a/python/paddle/distributed/parallel.py +++ b/python/paddle/distributed/parallel.py @@ -26,7 +26,9 @@ from paddle.fluid.dygraph import parallel_helper from paddle.distributed.fleet.launch_utils import check_backend from paddle.fluid.dygraph.parallel import ParallelEnv -from paddle.distributed.fleet.base.private_helper_function import wait_server_ready # noqa: F401 +from paddle.distributed.fleet.base.private_helper_function import ( + wait_server_ready, +) # noqa: F401 from paddle.distributed.collective import _set_group_map from paddle.distributed.collective import _set_group_map_by_name from paddle.distributed.collective import _get_group_map_by_name @@ -57,6 +59,7 @@ def _get_global_parallel_env(): def _start_kv_server(port, http_server_d, size): from paddle.distributed.fleet.utils.http_server import KVServer + http_server = KVServer(int(port), size=size) http_server.start() wait_seconds = 3 @@ -67,10 +70,15 @@ def _start_kv_server(port, http_server_d, size): def _is_cpuonly(backend): check_backend(backend) - if (backend in ['auto', 'nccl', 'bkcl', 'hccl', 'heter', 'cncl'] and - (core.is_compiled_with_cuda() or core.is_compiled_with_xpu() - or core.is_compiled_with_npu() - or core.is_compiled_with_mlu())) or backend == 'xccl': + if ( + backend in ['auto', 'nccl', 'bkcl', 'hccl', 'heter', 'cncl'] + and ( + core.is_compiled_with_cuda() + or core.is_compiled_with_xpu() + or core.is_compiled_with_npu() + or core.is_compiled_with_mlu() + ) + ) or backend == 'xccl': # passes 'auto' and can use cuda or xpu, use the default logics. so return False return False @@ -81,9 +89,10 @@ def _is_cpuonly(backend): def _check_var_exists(var_name): var = os.environ.get(var_name, None) if var is None: - raise ValueError("paddle.distributed initialize error, " - "environment variable %s is needed, but not set." % - var_name) + raise ValueError( + "paddle.distributed initialize error, " + "environment variable %s is needed, but not set." % var_name + ) def init_parallel_env(): @@ -161,15 +170,21 @@ def train(): backend = os.environ.get('PADDLE_DISTRI_BACKEND', 'auto') is_cpu_only = _is_cpuonly(backend) # 1. gpu xpu check, must be gpu or xpu, - if not (is_cpu_only or core.is_compiled_with_cuda() - or core.is_compiled_with_xpu() or core.is_compiled_with_npu() - or core.is_compiled_with_mlu()): + if not ( + is_cpu_only + or core.is_compiled_with_cuda() + or core.is_compiled_with_xpu() + or core.is_compiled_with_npu() + or core.is_compiled_with_mlu() + ): raise NotImplementedError( - "If you want to use CPU-only version, please use 'gloo' as backend") + "If you want to use CPU-only version, please use 'gloo' as backend" + ) if backend == "xccl": FLAGS_selected_custom_devices = 'FLAGS_selected_{}s'.format( - parallel_env.device_type) + parallel_env.device_type + ) _check_var_exists(FLAGS_selected_custom_devices) else: if not is_cpu_only and core.is_compiled_with_cuda(): @@ -197,8 +212,9 @@ def train(): # they need to call a function to change default place, # here just set correctly place to users if backend == "xccl": - place = core.CustomPlace(parallel_env.device_type, - parallel_env.device_id) + place = core.CustomPlace( + parallel_env.device_type, parallel_env.device_id + ) elif is_cpu_only: place = core.CPUPlace() elif core.is_compiled_with_cuda(): @@ -222,11 +238,15 @@ def train(): assert rank >= 0 and world_size > rank and world_size > 1, ( "rank must be non-negative and world_size must be the " "maximum rank plus one. Moreover, at least two processes are " - "required to create a process group.") + "required to create a process group." + ) master_addr = os.getenv("MASTER_ADDR", None) master_port = os.getenv("MASTER_PORT", None) - endpoints = ":".join([master_addr, master_port - ]) if master_addr and master_port else None + endpoints = ( + ":".join([master_addr, master_port]) + if master_addr and master_port + else None + ) if endpoints is None: endpoints = os.getenv("PADDLE_MASTER", None) if endpoints is None: @@ -235,23 +255,28 @@ def train(): "The environment variable 'MASTER_ADDR' and 'MASTER_PORT' " "must be specified, for example 'export MASTER_ADDR=127.0.0.1' " "and 'export MASTER_ADDR=54612'. Or you can start your training" - "with paddle.distributed.run module.") + "with paddle.distributed.run module." + ) master_addr, master_port = endpoints.split(":") master_port = int(master_port) is_master = rank == 0 stop_check_timeout = int(os.getenv("FLAGS_stop_check_timeout", "900")) - default_store = core.TCPStore(master_addr, - master_port, - is_master, - world_size, - timeout=stop_check_timeout) + default_store = core.TCPStore( + master_addr, + master_port, + is_master, + world_size, + timeout=stop_check_timeout, + ) _set_default_store(default_store) - pg = _new_process_group_impl(backend, - default_store, - rank, - world_size, - _default_group_name, - pg_options=None) + pg = _new_process_group_impl( + backend, + default_store, + rank, + world_size, + _default_group_name, + pg_options=None, + ) ranks = list(range(world_size)) group = Group(rank, 0, ranks, pg=pg, name=_default_group_name) _set_group_map_by_name(_default_group_name, group) @@ -277,8 +302,10 @@ def train(): size = {'_worker': parallel_env.world_size} if backend == "heter": size = {'_worker': len(node_num)} - http_server = Process(target=_start_kv_server, - args=(int(ep_rank_0[1]), http_server_d, size)) + http_server = Process( + target=_start_kv_server, + args=(int(ep_rank_0[1]), http_server_d, size), + ) http_server.daemon = True http_server_d["running"] = True http_server.start() @@ -296,22 +323,28 @@ def train(): # init nccl or hccl or bkcl or heter context if is_cpu_only: parallel_helper._set_parallel_ctx( - core.GLOOParallelContext(strategy, place)) - elif (backend == "heter"): + core.GLOOParallelContext(strategy, place) + ) + elif backend == "heter": parallel_helper._set_parallel_ctx( - core.HeterParallelContext(strategy, parallel_env.device_id)) + core.HeterParallelContext(strategy, parallel_env.device_id) + ) elif core.is_compiled_with_cuda(): parallel_helper._set_parallel_ctx( - core.NCCLParallelContext(strategy, place)) + core.NCCLParallelContext(strategy, place) + ) elif core.is_compiled_with_xpu(): parallel_helper._set_parallel_ctx( - core.BKCLParallelContext(strategy, place)) + core.BKCLParallelContext(strategy, place) + ) elif core.is_compiled_with_npu(): parallel_helper._set_parallel_ctx( - core.HCCLParallelContext(strategy, place)) + core.HCCLParallelContext(strategy, place) + ) elif core.is_compiled_with_mlu(): parallel_helper._set_parallel_ctx( - core.CNCLParallelContext(strategy, place)) + core.CNCLParallelContext(strategy, place) + ) if backend != "heter": other_endpoints = strategy.trainer_endpoints[:] diff --git a/python/paddle/distributed/parallel_with_gloo.py b/python/paddle/distributed/parallel_with_gloo.py index a5630239948c0..eeb91332070b4 100755 --- a/python/paddle/distributed/parallel_with_gloo.py +++ b/python/paddle/distributed/parallel_with_gloo.py @@ -17,7 +17,9 @@ # deprecated module import from paddle.fluid import core -from paddle.distributed.fleet.base.private_helper_function import wait_server_ready +from paddle.distributed.fleet.base.private_helper_function import ( + wait_server_ready, +) __all__ = [] @@ -26,6 +28,7 @@ def _start_kv_server(port, http_server_d, size): from paddle.distributed.fleet.utils.http_server import KVServer + http_server = KVServer(int(port), size=size) http_server.start() wait_seconds = 3 @@ -89,8 +92,9 @@ def test_gloo_init_with_multiprocess(num_of_ranks): test_gloo_init_with_multiprocess(2) """ - assert (rank_num < 2) is False, \ - "rank_num should greater than or equal to 2 for parallel environment initialzation." + assert ( + rank_num < 2 + ) is False, "rank_num should greater than or equal to 2 for parallel environment initialzation." # init gloo context manager = Manager() @@ -100,9 +104,10 @@ def test_gloo_init_with_multiprocess(num_of_ranks): if rank_id == 0: # The scope for worker used by http server is '_worker' size = {'_worker': rank_num} - http_server_proc = Process(target=_start_kv_server, - args=(int(server_endpoint.split(":")[1]), - http_server_status, size)) + http_server_proc = Process( + target=_start_kv_server, + args=(int(server_endpoint.split(":")[1]), http_server_status, size), + ) http_server_proc.daemon = True http_server_status["running"] = True http_server_proc.start() diff --git a/python/paddle/distributed/passes/auto_parallel_amp.py b/python/paddle/distributed/passes/auto_parallel_amp.py index dc51cced37d92..3305982f5055c 100644 --- a/python/paddle/distributed/passes/auto_parallel_amp.py +++ b/python/paddle/distributed/passes/auto_parallel_amp.py @@ -18,25 +18,48 @@ from .pass_base import PassBase, register_pass from paddle.distributed.fleet.meta_optimizers.common import OpRole from paddle.fluid.data_feeder import check_variable_and_dtype, check_type -from paddle.distributed.auto_parallel.utils import get_loss_op, set_var_dist_attr -from paddle.distributed.auto_parallel.utils import naive_set_dist_op_attr_for_program_by_mesh_and_mapping -from paddle.distributed.auto_parallel.process_group import get_world_process_group -from paddle.fluid.contrib.mixed_precision.fp16_utils import AutoMixedPrecisionLists -from paddle.fluid.contrib.mixed_precision.fp16_utils import _keep_fp32_input, _keep_fp32_output, find_op_index -from paddle.fluid.contrib.mixed_precision.fp16_utils import _valid_types, find_true_post_op, find_true_prev_op -from paddle.fluid.contrib.mixed_precision.fp16_utils import _is_in_black_varnames, _dtype_to_str, _rename_arg -from paddle.distributed.auto_parallel.dist_attribute import OperatorDistributedAttribute +from paddle.distributed.auto_parallel.utils import ( + get_loss_op, + set_var_dist_attr, +) +from paddle.distributed.auto_parallel.utils import ( + naive_set_dist_op_attr_for_program_by_mesh_and_mapping, +) +from paddle.distributed.auto_parallel.process_group import ( + get_world_process_group, +) +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + AutoMixedPrecisionLists, +) +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + _keep_fp32_input, + _keep_fp32_output, + find_op_index, +) +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + _valid_types, + find_true_post_op, + find_true_prev_op, +) +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + _is_in_black_varnames, + _dtype_to_str, + _rename_arg, +) +from paddle.distributed.auto_parallel.dist_attribute import ( + OperatorDistributedAttribute, +) from ..auto_parallel.utils import is_forward_op, is_backward_op, is_loss_op world_process_group = get_world_process_group() class AMPState(object): - def __init__(self, block): self._block = block - self._op_fp16_dict = { - } # op_id --> True/False. 'True' means that the current op is in fp16 mode. + self._op_fp16_dict = ( + {} + ) # op_id --> True/False. 'True' means that the current op is in fp16 mode. self._var_name_dict = {} # fwd_op_id --> {old_name: cast_name} self.is_train = False @@ -55,7 +78,8 @@ def _build_state(self, amp_lists, dist_context): elif int(op.attr('op_role')) == int(OpRole.Backward): if op.desc.original_id() in dist_op_context.grad_op_id_to_op_id: fwd_op_id = dist_op_context.grad_op_id_to_op_id[ - op.desc.original_id()] + op.desc.original_id() + ] if self._is_fp16_op(fwd_op_id) == True: self._op_fp16_dict[op.desc.original_id()] = True elif self._is_fp16_op(fwd_op_id) == False: @@ -78,7 +102,8 @@ def _mark_black_white_ops(self, amp_lists): if op.type == 'create_py_reader' or op.type == 'read': continue if amp_lists.black_varnames is not None and _is_in_black_varnames( - op, amp_lists): + op, amp_lists + ): self._op_fp16_dict[op.desc.original_id()] = False continue if op.type in amp_lists.black_list: @@ -98,17 +123,24 @@ def _mark_black_white_ops(self, amp_lists): continue elif in_var.op is op: prev_op = find_true_prev_op( - ops, op, in_var_name) + ops, op, in_var_name + ) if prev_op is None: continue else: prev_op = in_var.op # if it's one of inputs - if self._is_fp16_op(prev_op.desc.original_id()) == False or \ - prev_op.type in amp_lists.black_list: + if ( + self._is_fp16_op(prev_op.desc.original_id()) + == False + or prev_op.type in amp_lists.black_list + ): is_black_op = True - elif self._is_fp16_op(prev_op.desc.original_id()) == True or \ - prev_op.type in amp_lists.white_list: + elif ( + self._is_fp16_op(prev_op.desc.original_id()) + == True + or prev_op.type in amp_lists.white_list + ): is_white_op = True if is_black_op: self._op_fp16_dict[op.desc.original_id()] = False @@ -131,19 +163,28 @@ def cast_forward_program(self, dist_context): break if self._is_fp16_op(op.desc.original_id()) == False: num_cast_ops = self._insert_cast_op_forward( - op, idx, core.VarDesc.VarType.FP16, - core.VarDesc.VarType.FP32, dist_context) + op, + idx, + core.VarDesc.VarType.FP16, + core.VarDesc.VarType.FP32, + dist_context, + ) elif self._is_fp16_op(op.desc.original_id()) == True: num_cast_ops = self._insert_cast_op_forward( - op, idx, core.VarDesc.VarType.FP32, - core.VarDesc.VarType.FP16, dist_context) + op, + idx, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP16, + dist_context, + ) else: pass idx += num_cast_ops + 1 self._block._sync_with_cpp() - def _insert_cast_op_forward(self, op, idx, src_dtype, dst_dtype, - dist_context): + def _insert_cast_op_forward( + self, op, idx, src_dtype, dst_dtype, dist_context + ): """ only for forward cast modified from paddle.fluid.contrib.mixed_precision @@ -152,38 +193,45 @@ def _insert_cast_op_forward(self, op, idx, src_dtype, dst_dtype, var_name_dict = {} for in_name in op.input_names: if src_dtype == core.VarDesc.VarType.FP32 and _keep_fp32_input( - op, in_name): + op, in_name + ): continue for in_var_name in op.input(in_name): in_var = self._block._find_var_recursive(in_var_name) if in_var.type not in _valid_types or in_var.dtype == dst_dtype: continue if in_var.dtype == src_dtype: - cast_name = in_var.name + '.cast_' + _dtype_to_str( - dst_dtype) + cast_name = ( + in_var.name + '.cast_' + _dtype_to_str(dst_dtype) + ) out_var = self._block.vars.get(cast_name) var_name_dict[in_var.name] = cast_name consume_op_attr = dist_context.get_op_dist_attr_for_program( - op) + op + ) assert consume_op_attr is not None if out_var is None or out_var.dtype != dst_dtype: # NOTE we make the cast op and var's dist attr as the op that consume the # cast var instead of the op which generates the var in_var_dist_attr = consume_op_attr.get_input_dist_attr( - in_var.name) + in_var.name + ) assert in_var_dist_attr is not None ref_mesh = in_var_dist_attr.process_mesh ref_mapping = in_var_dist_attr.dims_mapping consume_op_attr.set_input_dist_attr( - cast_name, in_var_dist_attr) + cast_name, in_var_dist_attr + ) out_var = self._block.create_var( name=cast_name, dtype=dst_dtype, persistable=False, - stop_gradient=in_var.stop_gradient) - set_var_dist_attr(dist_context, out_var, ref_mapping, - ref_mesh) + stop_gradient=in_var.stop_gradient, + ) + set_var_dist_attr( + dist_context, out_var, ref_mapping, ref_mesh + ) cast_op = self._block._insert_op_without_sync( idx, @@ -193,22 +241,29 @@ def _insert_cast_op_forward(self, op, idx, src_dtype, dst_dtype, attrs={ "in_dtype": in_var.dtype, "out_dtype": out_var.dtype, - }) + }, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - cast_op, ref_mesh, ref_mapping, dist_context) + cast_op, ref_mesh, ref_mapping, dist_context + ) num_cast_ops += 1 else: in_var_dist_attr = consume_op_attr.get_input_dist_attr( - in_var.name) + in_var.name + ) consume_op_attr.set_input_dist_attr( - cast_name, in_var_dist_attr) + cast_name, in_var_dist_attr + ) _rename_arg(op, in_var.name, cast_name) else: if op.has_attr('in_dtype'): op._set_attr('in_dtype', dst_dtype) self._var_name_dict[op.desc.original_id()] = var_name_dict - if src_dtype == core.VarDesc.VarType.FP32 and dst_dtype == core.VarDesc.VarType.FP16: + if ( + src_dtype == core.VarDesc.VarType.FP32 + and dst_dtype == core.VarDesc.VarType.FP16 + ): for out_name in op.output_names: if _keep_fp32_output(op, out_name): continue @@ -238,8 +293,9 @@ def cast_backward_program(self, params_grads, dist_context): # NOTE: the map in `grad_var_to_var` may be changed when the var is casted, # which will affect the dist_op to insert allreduce_sum op. op_dist_attr = dist_context.get_op_dist_attr_for_program(grad_op) - if is_backward_op(grad_op) and (is_forward_op(ops[idx - 1]) - or is_loss_op(ops[idx - 1])): + if is_backward_op(grad_op) and ( + is_forward_op(ops[idx - 1]) or is_loss_op(ops[idx - 1]) + ): if not op_dist_attr.is_recompute: appended_grad_times += 1 @@ -248,14 +304,22 @@ def cast_backward_program(self, params_grads, dist_context): if grad_op_orig_id in dist_op_context.grad_op_id_to_op_id: if self._is_fp16_op(grad_op_orig_id) == False: # fp32 num_cast_ops = self._insert_cast_op_backward( - grad_op, idx, core.VarDesc.VarType.FP16, - core.VarDesc.VarType.FP32, dist_context, - appended_grad_times) + grad_op, + idx, + core.VarDesc.VarType.FP16, + core.VarDesc.VarType.FP32, + dist_context, + appended_grad_times, + ) elif self._is_fp16_op(grad_op_orig_id) == True: # fp16 num_cast_ops = self._insert_cast_op_backward( - grad_op, idx, core.VarDesc.VarType.FP32, - core.VarDesc.VarType.FP16, dist_context, - appended_grad_times) + grad_op, + idx, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP16, + dist_context, + appended_grad_times, + ) elif grad_op.type == "sum": in_var_name = grad_op.desc.input_arg_names()[0] src_dtype = self._block.var(in_var_name).dtype @@ -270,15 +334,24 @@ def cast_backward_program(self, params_grads, dist_context): else: raise ValueError( "'{}' op is not supported in the complete amp pass.".format( - grad_op.type)) + grad_op.type + ) + ) idx += num_cast_ops + 1 self._block._sync_with_cpp() _update_backward_cast_ops(params_grads, dist_context) - def _insert_cast_op_backward(self, grad_op, idx, src_dtype, dst_dtype, - dist_context, appended_grad_times): - """ only for backward cast """ + def _insert_cast_op_backward( + self, + grad_op, + idx, + src_dtype, + dst_dtype, + dist_context, + appended_grad_times, + ): + """only for backward cast""" def _keep_fp32_input(op, in_name): op_type = op.type @@ -299,7 +372,8 @@ def _keep_fp32_output(op, out_name): for in_name in grad_op.input_names: if src_dtype == core.VarDesc.VarType.FP32 and _keep_fp32_input( - grad_op, in_name): + grad_op, in_name + ): for in_var_name in grad_op.input(in_name): in_var = self._block._find_var_recursive(in_var_name) assert in_var.dtype == core.VarDesc.VarType.FP32 @@ -309,24 +383,34 @@ def _keep_fp32_output(op, out_name): in_var = self._block._find_var_recursive(in_var_name) if in_var.dtype == src_dtype: consume_op_attr = dist_context.get_op_dist_attr_for_program( - grad_op) + grad_op + ) if in_var_name in self._var_name_dict[fwd_op_id]: # NOTE: if in_var of consume grad_op has been casted before, # it should be renamed and reset dist_attr. cast_name = self._var_name_dict[fwd_op_id][in_var_name] grad_op.desc._rename_input(in_var_name, cast_name) in_var_dist_attr = consume_op_attr.get_input_dist_attr( - in_var_name) + in_var_name + ) consume_op_attr.set_input_dist_attr( - cast_name, in_var_dist_attr) + cast_name, in_var_dist_attr + ) else: - assert in_var.dtype == dst_dtype, "op [{}] expect input [{}] to be dtype [{}] BUT got [{}]. {}".format( - grad_op.type, in_name, dst_dtype, in_var.dtype, - str(grad_op)) + assert ( + in_var.dtype == dst_dtype + ), "op [{}] expect input [{}] to be dtype [{}] BUT got [{}]. {}".format( + grad_op.type, + in_name, + dst_dtype, + in_var.dtype, + str(grad_op), + ) for out_name in grad_op.output_names: if src_dtype == core.VarDesc.VarType.FP32 and _keep_fp32_output( - grad_op, out_name): + grad_op, out_name + ): for out_var_name in grad_op.output(out_name): out_var = self._block._find_var_recursive(out_var_name) assert out_var.dtype == core.VarDesc.VarType.FP32 @@ -334,7 +418,7 @@ def _keep_fp32_output(op, out_name): for out_var_name in grad_op.output(out_name): out_var = self._block._find_var_recursive(out_var_name) - out_var_name_prefix = out_var_name[:out_var_name.find("@")] + out_var_name_prefix = out_var_name[: out_var_name.find("@")] fwd_var = self._block._find_var_recursive(out_var_name_prefix) # NOTE: the out_var's dtype of consume grad_op should equal to the fwd_var's dtype if out_var.dtype != fwd_var.dtype: @@ -345,34 +429,45 @@ def _keep_fp32_output(op, out_name): # NOTE: if out_var of consume grad_op has been casted before, # it should be renamed and reset dist_attr, then we insert cast op to # convert the cast_var to original dtype - consume_op_attr = dist_context.get_op_dist_attr_for_program( - grad_op) + consume_op_attr = ( + dist_context.get_op_dist_attr_for_program(grad_op) + ) fwd_cast_name = self._var_name_dict[fwd_op_id][ - out_var_name_prefix] + out_var_name_prefix + ] suffix = "" if "@RENAME" in out_var_name: - suffix = out_var_name[out_var_name.find("@RENAME"):] + suffix = out_var_name[ + out_var_name.find("@RENAME") : + ] cast_name = fwd_cast_name + "@GRAD" + suffix cast_var = self._block.vars.get(cast_name) if cast_var is None or cast_var.dtype != dst_dtype: grad_op.desc._rename_output(out_var_name, cast_name) - out_var_dist_attr = consume_op_attr.get_output_dist_attr( - out_var_name) + out_var_dist_attr = ( + consume_op_attr.get_output_dist_attr( + out_var_name + ) + ) ref_mesh = out_var_dist_attr.process_mesh ref_mapping = out_var_dist_attr.dims_mapping consume_op_attr.set_output_dist_attr( - cast_name, out_var_dist_attr) + cast_name, out_var_dist_attr + ) assert ref_mapping is not None cast_var = self._block.create_var( name=cast_name, shape=out_var.shape, dtype=dst_dtype, persistable=False, - stop_gradient=out_var.stop_gradient) - set_var_dist_attr(dist_context, cast_var, - ref_mapping, ref_mesh) + stop_gradient=out_var.stop_gradient, + ) + set_var_dist_attr( + dist_context, cast_var, ref_mapping, ref_mesh + ) dist_op_context.grad_var_to_var[ - appended_grad_times][cast_name] = fwd_cast_name + appended_grad_times + ][cast_name] = fwd_cast_name cast_op = self._block._insert_op( idx + 1, @@ -382,13 +477,15 @@ def _keep_fp32_output(op, out_name): attrs={ "in_dtype": cast_var.dtype, "out_dtype": out_var.dtype, - "op_role": OpRole.Backward - }) + "op_role": OpRole.Backward, + }, + ) cast_op._remove_attr("op_role_var") cast_op._remove_attr("op_namescope") cast_op._remove_attr("with_quant_attr") naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - cast_op, ref_mesh, ref_mapping, dist_context) + cast_op, ref_mesh, ref_mapping, dist_context + ) num_cast_ops += 1 else: assert out_var.dtype == dst_dtype @@ -409,15 +506,18 @@ def _update_backward_cast_ops(params_grads, dist_context): for p, g in params_grads: op = g.op if g.dtype == core.VarDesc.VarType.FP32 and op.type == 'cast': - if int(op.attr('op_role')) == int( - OpRole.Backward) and op.has_attr('op_role_var'): + if int(op.attr('op_role')) == int(OpRole.Backward) and op.has_attr( + 'op_role_var' + ): op._remove_attr("op_role_var") post_ops = find_true_post_op(main_block.ops, op, g.name) if post_ops: - raise ValueError("The cast op {0}'s output should not be" - "used by a non-optimize op, however, it" - "is used by {1}".format(op, post_ops[0])) + raise ValueError( + "The cast op {0}'s output should not be" + "used by a non-optimize op, however, it" + "is used by {1}".format(op, post_ops[0]) + ) if op == main_block.ops[-1]: continue @@ -425,23 +525,29 @@ def _update_backward_cast_ops(params_grads, dist_context): # add new op in the python and cpp at the same time new_op_desc = main_block.desc.append_op() new_op_desc.copy_from(op.desc) - new_op = paddle.fluid.framework.Operator(block=main_block, - desc=new_op_desc, - type=None, - inputs=None, - outputs=None, - attrs=None) + new_op = paddle.fluid.framework.Operator( + block=main_block, + desc=new_op_desc, + type=None, + inputs=None, + outputs=None, + attrs=None, + ) main_block.ops.append(new_op) # dist attr param_dist_attr = dist_context.get_tensor_dist_attr_for_program(p) output_dist_attr = dist_context.get_tensor_dist_attr_for_program( - main_block.var(op.output_arg_names[0])) + main_block.var(op.output_arg_names[0]) + ) assert param_dist_attr is not None assert output_dist_attr is not None naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - new_op, param_dist_attr.process_mesh, - param_dist_attr.dims_mapping, dist_context) + new_op, + param_dist_attr.process_mesh, + param_dist_attr.dims_mapping, + dist_context, + ) output_dist_attr.process_mesh = param_dist_attr.process_mesh output_dist_attr.dims_mapping = param_dist_attr.dims_mapping @@ -462,26 +568,34 @@ def _check_and_update_gradient(params_grads, loss_scaling, dist_context): grads = [g for _, g in params_grads] check_type(grads, 'x', (tuple, list), 'check_finite_and_unscale') for e in grads: - check_variable_and_dtype(e, "x", ['float16', 'float32', 'float64'], - 'check_finite_and_unscale') + check_variable_and_dtype( + e, + "x", + ['float16', 'float32', 'float64'], + 'check_finite_and_unscale', + ) found_inf = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ['find_infinite_scale', 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join(['find_infinite_scale', 'tmp']) + ), shape=[1], dtype='bool', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) set_var_dist_attr(dist_context, found_inf, [-1], world_process_group.ranks) inputs = {'X': grads, 'Scale': loss_scaling} outputs = {'Out': grads, 'FoundInfinite': found_inf} attrs = {'op_role': OpRole.Optimize} - new_op = main_block.append_op(type='check_finite_and_unscale', - inputs=inputs, - outputs=outputs, - attrs=attrs) + new_op = main_block.append_op( + type='check_finite_and_unscale', + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) new_op_dist_attr = OperatorDistributedAttribute() new_op_dist_attr.process_mesh = world_process_group.ranks @@ -491,17 +605,18 @@ def _check_and_update_gradient(params_grads, loss_scaling, dist_context): for g in grads: g_dist_attr = dist_context.get_tensor_dist_attr_for_program(g) assert g_dist_attr is not None - new_op_dist_attr.set_input_dims_mapping(g.name, - g_dist_attr.dims_mapping) - new_op_dist_attr.set_output_dims_mapping(g.name, - g_dist_attr.dims_mapping) + new_op_dist_attr.set_input_dims_mapping( + g.name, g_dist_attr.dims_mapping + ) + new_op_dist_attr.set_output_dims_mapping( + g.name, g_dist_attr.dims_mapping + ) dist_context.set_op_dist_attr_for_program(new_op, new_op_dist_attr) return grads, found_inf @register_pass("auto_parallel_amp") class AMPPass(PassBase): - def __init__(self): super(AMPPass, self).__init__() self.set_attr("loss", None) @@ -552,7 +667,8 @@ def _apply_single_impl(self, main_program, startup_program, context): amp_lists = AutoMixedPrecisionLists( set(self.get_attr("custom_white_list")), set(self.get_attr("custom_black_list")), - set(self.get_attr("custom_black_varnames"))) + set(self.get_attr("custom_black_varnames")), + ) with paddle.static.program_guard(main_program, startup_program): amp_state = AMPState(main_program.global_block()) @@ -566,10 +682,13 @@ def _apply_single_impl(self, main_program, startup_program, context): self._init_amp_var() self._scale_loss() - if self.get_attr("use_dynamic_loss_scaling" - ) or self.get_attr("init_loss_scaling") != 1.0: + if ( + self.get_attr("use_dynamic_loss_scaling") + or self.get_attr("init_loss_scaling") != 1.0 + ): grads, found_inf = _check_and_update_gradient( - params_grads, self._loss_scaling, self.dist_context) + params_grads, self._loss_scaling, self.dist_context + ) if self.get_attr("use_dynamic_loss_scaling"): self._update_loss_scaling(grads, found_inf) @@ -580,9 +699,14 @@ def _init_amp_var(self): shape=[1], value=self.get_attr("init_loss_scaling"), dtype='float32', - persistable=True) - set_var_dist_attr(self.dist_context, self._loss_scaling, [-1], - world_process_group.ranks) + persistable=True, + ) + set_var_dist_attr( + self.dist_context, + self._loss_scaling, + [-1], + world_process_group.ranks, + ) if self.get_attr("use_dynamic_loss_scaling"): self._num_good_steps = paddle.static.create_global_var( @@ -590,18 +714,28 @@ def _init_amp_var(self): shape=[1], value=0, dtype='int32', - persistable=True) - set_var_dist_attr(self.dist_context, self._num_good_steps, [-1], - world_process_group.ranks) + persistable=True, + ) + set_var_dist_attr( + self.dist_context, + self._num_good_steps, + [-1], + world_process_group.ranks, + ) self._num_bad_steps = paddle.static.create_global_var( name=unique_name.generate("num_bad_steps"), shape=[1], value=0, dtype='int32', - persistable=True) - set_var_dist_attr(self.dist_context, self._num_bad_steps, [-1], - world_process_group.ranks) + persistable=True, + ) + set_var_dist_attr( + self.dist_context, + self._num_bad_steps, + [-1], + world_process_group.ranks, + ) def _scale_loss(self): @@ -613,18 +747,22 @@ def _scale_loss(self): assert loss is not None loss_op = loss.op loss_op_dist_attr = self.dist_context.get_op_dist_attr_for_program( - loss_op) + loss_op + ) if loss.dtype != core.VarDesc.VarType.FP32: tmp_name = unique_name.generate(loss.name + ".cast_fp32") - cast_loss = main_block.create_var(name=tmp_name, - dtype=core.VarDesc.VarType.FP32) + cast_loss = main_block.create_var( + name=tmp_name, dtype=core.VarDesc.VarType.FP32 + ) loss_dist_attr = self.dist_context.get_tensor_dist_attr_for_program( - loss) + loss + ) ref_mesh = loss_op_dist_attr.process_mesh self.dist_context.set_tensor_dist_attr_for_program( - cast_loss, loss_dist_attr) + cast_loss, loss_dist_attr + ) # forward loss_op_idx = find_op_index(main_block.desc, loss_op.desc) @@ -637,22 +775,28 @@ def _scale_loss(self): "in_dtype": loss.dtype, "out_dtype": core.VarDesc.VarType.FP32, 'op_role': loss_op.all_attrs()[OP_ROLE_KEY], - }) + }, + ) - loss_op._set_attr(OP_ROLE_KEY, - core.op_proto_and_checker_maker.OpRole.Forward) + loss_op._set_attr( + OP_ROLE_KEY, core.op_proto_and_checker_maker.OpRole.Forward + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - cast_op, ref_mesh, [-1], self.dist_context) + cast_op, ref_mesh, [-1], self.dist_context + ) # backward first_backward_op = main_block.ops[loss_op_idx + 2] - assert first_backward_op.type == "fill_constant" and int( - first_backward_op.all_attrs()[OP_ROLE_KEY]) == 257 + assert ( + first_backward_op.type == "fill_constant" + and int(first_backward_op.all_attrs()[OP_ROLE_KEY]) == 257 + ) cast_loss_grad = main_block.create_var( name=unique_name.generate(tmp_name + "@GRAD"), shape=loss.shape, dtype=core.VarDesc.VarType.FP32, - persistable=loss.persistable) + persistable=loss.persistable, + ) set_var_dist_attr(self.dist_context, cast_loss_grad, [-1], ref_mesh) pre_grad_name = first_backward_op.output_arg_names[0] @@ -666,14 +810,18 @@ def _scale_loss(self): "in_dtype": core.VarDesc.VarType.FP32, "out_dtype": core.VarDesc.VarType.FP16, 'op_role': core.op_proto_and_checker_maker.OpRole.Backward, - }) + }, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - cast_grad_op, ref_mesh, [-1], self.dist_context) + cast_grad_op, ref_mesh, [-1], self.dist_context + ) loss_op = cast_op loss = cast_loss - if self.get_attr("use_dynamic_loss_scaling" - ) or self.get_attr("init_loss_scaling") != 1.0: + if ( + self.get_attr("use_dynamic_loss_scaling") + or self.get_attr("init_loss_scaling") != 1.0 + ): loss_op_idx = find_op_index(main_block.desc, loss_op.desc) @@ -683,63 +831,76 @@ def _scale_loss(self): name=unique_name.generate("scaled_loss"), shape=loss.shape, dtype=loss.dtype, - persistable=loss.persistable) - set_var_dist_attr(self.dist_context, self._scaled_loss, [-1], - ref_mesh) + persistable=loss.persistable, + ) + set_var_dist_attr( + self.dist_context, self._scaled_loss, [-1], ref_mesh + ) elementwise_mul_op = main_block._insert_op( loss_op_idx + 1, type='elementwise_mul', - inputs={ - 'X': [loss], - 'Y': [self._loss_scaling] - }, + inputs={'X': [loss], 'Y': [self._loss_scaling]}, outputs={'Out': [self._scaled_loss]}, attrs={ 'op_role': loss_op.all_attrs()[OP_ROLE_KEY], - }) - loss_op._set_attr(OP_ROLE_KEY, - core.op_proto_and_checker_maker.OpRole.Forward) + }, + ) + loss_op._set_attr( + OP_ROLE_KEY, core.op_proto_and_checker_maker.OpRole.Forward + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - elementwise_mul_op, ref_mesh, [-1], self.dist_context) + elementwise_mul_op, ref_mesh, [-1], self.dist_context + ) # backward first_backward_op = main_block.ops[loss_op_idx + 2] - assert first_backward_op.type == "fill_constant" and int( - first_backward_op.all_attrs()[OP_ROLE_KEY]) == 257 + assert ( + first_backward_op.type == "fill_constant" + and int(first_backward_op.all_attrs()[OP_ROLE_KEY]) == 257 + ) self._scaled_loss_grad = main_block.create_var( name=unique_name.generate("scaled_loss") + "@GRAD", shape=loss.shape, dtype=loss.dtype, - persistable=loss.persistable) - set_var_dist_attr(self.dist_context, self._scaled_loss_grad, [-1], - ref_mesh) + persistable=loss.persistable, + ) + set_var_dist_attr( + self.dist_context, self._scaled_loss_grad, [-1], ref_mesh + ) pre_grad_name = first_backward_op.output_arg_names[0] - first_backward_op._rename_output(pre_grad_name, - self._scaled_loss_grad.name) + first_backward_op._rename_output( + pre_grad_name, self._scaled_loss_grad.name + ) # FIXME(JZ-LIANG) a trick to insert backward op main_block._sync_with_cpp() elementwise_mul_grad_op_desc = main_block.desc._insert_op( - loss_op_idx + 3) + loss_op_idx + 3 + ) elementwise_mul_grad_op_desc.set_type("elementwise_mul_grad") elementwise_mul_grad_op_desc.set_input( - 'Out@GRAD', [self._scaled_loss_grad.name]) + 'Out@GRAD', [self._scaled_loss_grad.name] + ) elementwise_mul_grad_op_desc.set_input('X', [loss.name]) - elementwise_mul_grad_op_desc.set_input('Y', - [self._loss_scaling.name]) + elementwise_mul_grad_op_desc.set_input( + 'Y', [self._loss_scaling.name] + ) elementwise_mul_grad_op_desc.set_output('X@GRAD', [pre_grad_name]) elementwise_mul_grad_op_desc.set_output('Y@GRAD', []) elementwise_mul_grad_op_desc._set_attr( - OP_ROLE_KEY, core.op_proto_and_checker_maker.OpRole.Backward) + OP_ROLE_KEY, core.op_proto_and_checker_maker.OpRole.Backward + ) elementwise_mul_grad_op_desc._set_attr('axis', -1) elementwise_mul_grad_op = paddle.fluid.framework.Operator( - main_block, elementwise_mul_grad_op_desc) + main_block, elementwise_mul_grad_op_desc + ) main_block.ops.insert(loss_op_idx + 3, elementwise_mul_grad_op) main_block._sync_with_cpp() elementwise_mul_grad_op = main_block.ops[loss_op_idx + 3] assert elementwise_mul_grad_op.type == "elementwise_mul_grad" naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - elementwise_mul_grad_op, ref_mesh, [-1], self.dist_context) + elementwise_mul_grad_op, ref_mesh, [-1], self.dist_context + ) else: self._scaled_loss = loss @@ -751,31 +912,39 @@ def _update_loss_scaling(self, grads, found_inf): main_block = paddle.static.default_main_program().global_block() main_block._sync_with_cpp() - check_variable_and_dtype(self._loss_scaling, "prev_loss_scaling", - ['float32', 'float64'], "update_loss_scaling") + check_variable_and_dtype( + self._loss_scaling, + "prev_loss_scaling", + ['float32', 'float64'], + "update_loss_scaling", + ) check_type(grads, 'x', (tuple, list), 'update_loss_scaling') for e in grads: - check_variable_and_dtype(e, "x", ['float16', 'float32', 'float64'], - 'update_loss_scaling') + check_variable_and_dtype( + e, "x", ['float16', 'float32', 'float64'], 'update_loss_scaling' + ) if e.dtype == core.VarDesc.VarType.FP16: - assert self._loss_scaling.dtype == core.VarDesc.VarType.FP32, \ - "The dtype of prev_loss_scaling should be float32 when the dtype of x is float16." + assert ( + self._loss_scaling.dtype == core.VarDesc.VarType.FP32 + ), "The dtype of prev_loss_scaling should be float32 when the dtype of x is float16." else: - assert self._loss_scaling.dtype == e.dtype, "The dtype of prev_loss_scaling should be equal to the dtype of x." + assert ( + self._loss_scaling.dtype == e.dtype + ), "The dtype of prev_loss_scaling should be equal to the dtype of x." inputs = { 'X': grads, 'FoundInfinite': found_inf, 'PrevLossScaling': self._loss_scaling, 'InGoodSteps': self._num_good_steps, - 'InBadSteps': self._num_bad_steps + 'InBadSteps': self._num_bad_steps, } outputs = { 'Out': grads, 'LossScaling': self._loss_scaling, 'OutGoodSteps': self._num_good_steps, - 'OutBadSteps': self._num_bad_steps + 'OutBadSteps': self._num_bad_steps, } attrs = { @@ -784,13 +953,15 @@ def _update_loss_scaling(self, grads, found_inf): 'incr_ratio': self.get_attr("incr_ratio"), 'decr_ratio': self.get_attr("decr_ratio"), 'stop_update': self.get_attr("stop_update"), - 'op_role': OpRole.Optimize + 'op_role': OpRole.Optimize, } - new_op = main_block.append_op(type='update_loss_scaling', - inputs=inputs, - outputs=outputs, - attrs=attrs) + new_op = main_block.append_op( + type='update_loss_scaling', + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) new_op_dist_attr = OperatorDistributedAttribute() new_op_dist_attr.process_mesh = world_process_group.ranks @@ -800,10 +971,12 @@ def _update_loss_scaling(self, grads, found_inf): for g in grads: g_dist_attr = self.dist_context.get_tensor_dist_attr_for_program(g) assert g_dist_attr is not None - new_op_dist_attr.set_input_dims_mapping(g.name, - g_dist_attr.dims_mapping) - new_op_dist_attr.set_output_dims_mapping(g.name, - g_dist_attr.dims_mapping) + new_op_dist_attr.set_input_dims_mapping( + g.name, g_dist_attr.dims_mapping + ) + new_op_dist_attr.set_output_dims_mapping( + g.name, g_dist_attr.dims_mapping + ) self.dist_context.set_op_dist_attr_for_program(new_op, new_op_dist_attr) main_block._sync_with_cpp() diff --git a/python/paddle/distributed/passes/auto_parallel_data_parallel_optimization.py b/python/paddle/distributed/passes/auto_parallel_data_parallel_optimization.py index 8470aa5109961..fc938d6523030 100644 --- a/python/paddle/distributed/passes/auto_parallel_data_parallel_optimization.py +++ b/python/paddle/distributed/passes/auto_parallel_data_parallel_optimization.py @@ -19,14 +19,25 @@ from paddle.fluid import unique_name from paddle.fluid.framework import default_main_program from paddle.distributed.fleet.meta_optimizers.common import OP_ROLE_KEY, OpRole -from paddle.distributed.auto_parallel.operators.common import is_data_parallel_scale_op, is_data_parallel_reduce_op -from paddle.distributed.auto_parallel.utils import find_higher_order_backward_op, is_loss_grad_op, is_optimize_op, ring_id_to_process_group +from paddle.distributed.auto_parallel.operators.common import ( + is_data_parallel_scale_op, + is_data_parallel_reduce_op, +) +from paddle.distributed.auto_parallel.utils import ( + find_higher_order_backward_op, + is_loss_grad_op, + is_optimize_op, + ring_id_to_process_group, +) from .pass_base import PassBase, PassType, register_pass # add new optimizers supporting rescale_grad here __rescale_grad_supported_opts__ = [ - 'lars_momentum', 'sparse_momentum', 'dgc_momentum', 'momentum', - 'merge_momentum' + 'lars_momentum', + 'sparse_momentum', + 'dgc_momentum', + 'momentum', + 'merge_momentum', ] # a heuristic number @@ -62,8 +73,9 @@ def __init__(self): def _check_self(self): if self.get_attr("dist_context") is None: return False - if (not isinstance(self.get_attr("global_rank"), - int)) or self.get_attr("global_rank") < 0: + if (not isinstance(self.get_attr("global_rank"), int)) or self.get_attr( + "global_rank" + ) < 0: return False return True @@ -140,8 +152,11 @@ def _analyze_program(self): ), "Unexception: comm op [{}] has NOT ring id.".format(str(op)) group = ring_id_to_process_group(op.attr("ring_id")) - assert group is not None, "Unexception: data parallel group of [{}] from op [{}] is None".format( - grad_name, str(op)) + assert ( + group is not None + ), "Unexception: data parallel group of [{}] from op [{}] is None".format( + grad_name, str(op) + ) self._grad_name_to_group_map[grad_name] = group @@ -156,18 +171,21 @@ def _analyze_program(self): # TODO support multiple optimizers in on network in future. # here we assume that the optimizer is unique in network. - elif is_optimize_op( - op) and op.type in __rescale_grad_supported_opts__: + elif ( + is_optimize_op(op) + and op.type in __rescale_grad_supported_opts__ + ): self._support_rescale_grad = True not_synchronized_grads = [] for grad_name in scaled_grads: if grad_name not in self._grad_name_to_group_map: not_synchronized_grads.append(grad_name) - assert len( + assert ( + len(not_synchronized_grads) == 0 + ), "Unexception: gradients [{}] is scaled BUT NOT synchronized.".format( not_synchronized_grads - ) == 0, "Unexception: gradients [{}] is scaled BUT NOT synchronized.".format( - not_synchronized_grads) + ) def is_data_parallel_applied(self): return len(self._group_to_grad_name_map) > 0 @@ -175,14 +193,21 @@ def is_data_parallel_applied(self): def _could_be_prune(self): return self.dist_context.gradient_scale and ( - self._support_rescale_grad or self._all_dp_groups_same_degree()) + self._support_rescale_grad or self._all_dp_groups_same_degree() + ) def _all_dp_groups_same_degree(self): - return len( - set([ - len(group.ranks) - for group in self._group_to_grad_name_map.keys() - ])) == 1 + return ( + len( + set( + [ + len(group.ranks) + for group in self._group_to_grad_name_map.keys() + ] + ) + ) + == 1 + ) def _scale_backward_initial_grad(self): @@ -191,9 +216,10 @@ def _scale_backward_initial_grad(self): for idx, op in reversed(list(enumerate(block.ops))): if is_loss_grad_op(op): - assert op.type == 'fill_constant', \ - "loss_grad_op must be fill_constant op, " \ + assert op.type == 'fill_constant', ( + "loss_grad_op must be fill_constant op, " "but this op is {}".format(op.type) + ) assert op.has_attr('value') loss_scale = float(op.attr('value')) loss_scale = loss_scale / dp_degree @@ -215,28 +241,35 @@ def _update_opt_rescale_grad(self): scaled_grads = set() for idx, op in reversed(list(enumerate(block.ops))): - if is_optimize_op( - op) and op.type in __rescale_grad_supported_opts__: + if ( + is_optimize_op(op) + and op.type in __rescale_grad_supported_opts__ + ): assert op.has_attr( 'rescale_grad' ), "Unexception: op [{}] is supported to have [rescale_grad] attribute.".format( - str(op)) - assert len( - op.input("Grad") - ) == 1, "Unexception: op [{}] is supported to have only one input grad var.".format( - str(op)) + str(op) + ) + assert ( + len(op.input("Grad")) == 1 + ), "Unexception: op [{}] is supported to have only one input grad var.".format( + str(op) + ) grad_name = op.input("Grad")[0] dp_degree = len( - list(self._grad_name_to_group_map[grad_name].ranks)) + list(self._grad_name_to_group_map[grad_name].ranks) + ) scaled_grads.add(grad_name) rescale_grad = float(op.attr('rescale_grad')) / dp_degree op._set_attr('rescale_grad', rescale_grad) - assert scaled_grads == set(self._grad_name_to_group_map.keys( - )), "Unexception: gradients [{}] are unscaled.".format( - set(self._grad_name_to_group_map.keys()) - scaled_grads) + assert scaled_grads == set( + self._grad_name_to_group_map.keys() + ), "Unexception: gradients [{}] are unscaled.".format( + set(self._grad_name_to_group_map.keys()) - scaled_grads + ) def _could_be_overlap(self): # NOTE current different nccl comm will use different cuda stream @@ -266,14 +299,13 @@ def _comms_overlap_calc(self): op._set_attr('use_calc_stream', False) ring_id = op.attr("ring_id") - block._insert_op_without_sync(idx, - type='c_wait_compute', - inputs={'X': []}, - outputs={'Out': []}, - attrs={ - 'op_role': OpRole.Backward, - 'ring_id': ring_id - }) + block._insert_op_without_sync( + idx, + type='c_wait_compute', + inputs={'X': []}, + outputs={'Out': []}, + attrs={'op_role': OpRole.Backward, 'ring_id': ring_id}, + ) block._sync_with_cpp() @@ -307,8 +339,10 @@ def _calc_wait_comms(self): # other ops that might use communicating grad else: for input_var_name in op.input_arg_names: - for ring_id, unsync_grad_names in ring_id_to_un_sync_grad_map.items( - ): + for ( + ring_id, + unsync_grad_names, + ) in ring_id_to_un_sync_grad_map.items(): if input_var_name in unsync_grad_names: # need to sync before op_i if i in op_idx_to_sync_ring_id_map: @@ -328,14 +362,13 @@ def _calc_wait_comms(self): for i in sorted(indices, reverse=True): for ring_id in op_idx_to_sync_ring_id_map[i]: - block._insert_op_without_sync(i, - type='c_wait_comm', - inputs={'X': []}, - outputs={'Out': []}, - attrs={ - 'op_role': OpRole.Backward, - 'ring_id': ring_id - }) + block._insert_op_without_sync( + i, + type='c_wait_comm', + inputs={'X': []}, + outputs={'Out': []}, + attrs={'op_role': OpRole.Backward, 'ring_id': ring_id}, + ) def _could_be_fuse(self): # TODO support gradient fuse higher order gradient. @@ -423,36 +456,49 @@ def _update_program(self, grad_groups): for i, group in enumerate(grad_groups[::-1]): # create coalecse tensor - group.coalesce_var = block.create_var(name=unique_name.generate( - 'coalecse_grad_{}'.format(i)), - dtype=group.dtype, - persistable=False, - stop_gradient=True) + group.coalesce_var = block.create_var( + name=unique_name.generate('coalecse_grad_{}'.format(i)), + dtype=group.dtype, + persistable=False, + stop_gradient=True, + ) # update allreduce & scale op if group.scale_op_idx != -1: scale_op = block.ops[group.scale_op_idx] - assert scale_op.type == 'scale', "should found scale op but found {}".format( - str(scale_op)) - scale_op._rename_input(scale_op.input_arg_names[0], - group.coalesce_var.name) - scale_op._rename_output(scale_op.output_arg_names[0], - group.coalesce_var.name) + assert ( + scale_op.type == 'scale' + ), "should found scale op but found {}".format(str(scale_op)) + scale_op._rename_input( + scale_op.input_arg_names[0], group.coalesce_var.name + ) + scale_op._rename_output( + scale_op.output_arg_names[0], group.coalesce_var.name + ) allreduce_op = block.ops[group.allreduce_op_idx] - assert allreduce_op.type == 'c_allreduce_sum', "should found c_allreduce_sum op but found {}".format( - str(allreduce_op)) - allreduce_op._rename_input(allreduce_op.input_arg_names[0], - group.coalesce_var.name) - allreduce_op._rename_output(allreduce_op.output_arg_names[0], - group.coalesce_var.name) + assert ( + allreduce_op.type == 'c_allreduce_sum' + ), "should found c_allreduce_sum op but found {}".format( + str(allreduce_op) + ) + allreduce_op._rename_input( + allreduce_op.input_arg_names[0], group.coalesce_var.name + ) + allreduce_op._rename_output( + allreduce_op.output_arg_names[0], group.coalesce_var.name + ) # remvoe un-used op - remove_op_indices = group.remove_wait_op_indices + group.remove_allreduce_op_indices + group.remove_scale_op_indices + remove_op_indices = ( + group.remove_wait_op_indices + + group.remove_allreduce_op_indices + + group.remove_scale_op_indices + ) for idx in sorted(remove_op_indices, reverse=True): - assert block.ops[ - idx].type in remove_op_types, "Unexception: try to remove op {}".format( - str(op)) + assert ( + block.ops[idx].type in remove_op_types + ), "Unexception: try to remove op {}".format(str(op)) block._remove_op(idx) # insert coalecse op @@ -464,22 +510,23 @@ def _update_program(self, grad_groups): concated_ranks.append(len(shape)) grad_names = [grad.name for grad in group.gradients] - block._insert_op_without_sync(group.coalesce_op_idx, - type="coalesce_tensor", - inputs={"Input": grad_names}, - outputs={ - "Output": grad_names, - "FusedOutput": group.coalesce_var - }, - attrs={ - "copy_data": False, - "use_align": True, - "dtype": group.dtype, - "concated_shapes": - concated_shapes, - "concated_ranks": concated_ranks, - OP_ROLE_KEY: OpRole.Backward - }) + block._insert_op_without_sync( + group.coalesce_op_idx, + type="coalesce_tensor", + inputs={"Input": grad_names}, + outputs={ + "Output": grad_names, + "FusedOutput": group.coalesce_var, + }, + attrs={ + "copy_data": False, + "use_align": True, + "dtype": group.dtype, + "concated_shapes": concated_shapes, + "concated_ranks": concated_ranks, + OP_ROLE_KEY: OpRole.Backward, + }, + ) block._sync_with_cpp() # TODO update dist attr @@ -487,6 +534,7 @@ def _update_program(self, grad_groups): def summary(self, grad_groups=[]): # TODO: add logger module import logging + self._logger = logging.getLogger() self._logger.propagate = False if not self._logger.handlers: @@ -500,26 +548,31 @@ def summary(self, grad_groups=[]): if len(grad_groups) > 0: self._logger.info( - "origin {} allreduce ops are fused into {} coalecse allreduce ops." - .format(len(self._grad_name_to_group_map.keys()), - len(grad_groups))) + "origin {} allreduce ops are fused into {} coalecse allreduce ops.".format( + len(self._grad_name_to_group_map.keys()), len(grad_groups) + ) + ) self._logger.info("gradient fusing group are following: ") fused_grads = set() for i, group in enumerate(grad_groups): self._logger.info( "coalecse gradient [{}] is composed by: {}".format( - i, [grad.name for grad in group.gradients])) + i, [grad.name for grad in group.gradients] + ) + ) fused_grads.update([grad.name for grad in group.gradients]) - individual_grads = set( - self._grad_name_to_group_map.keys()) - set(fused_grads) + individual_grads = set(self._grad_name_to_group_map.keys()) - set( + fused_grads + ) self._logger.info( "the following [{}] gradients are not fused: ".format( - len(individual_grads))) + len(individual_grads) + ) + ) self._logger.info("individual gradient {}".format(individual_grads)) class GradientsGroup(object): - def __init__(self, ops, max_group_size): self.max_group_size = max_group_size self.ops = ops @@ -575,8 +628,11 @@ def add(self, grad_var, ring_id, i): grad_op_idx -= 1 grad_op = self.ops[grad_op_idx] - assert grad_var.name in grad_op.output_arg_names, "grad [{}] should be output of {}".format( - grad_var.name, str(grad_op)) + assert ( + grad_var.name in grad_op.output_arg_names + ), "grad [{}] should be output of {}".format( + grad_var.name, str(grad_op) + ) self.coalesce_op_idx = grad_op_idx def finalize(self): diff --git a/python/paddle/distributed/passes/auto_parallel_fp16.py b/python/paddle/distributed/passes/auto_parallel_fp16.py index 35a4831af3ace..cf1b2d45290bd 100644 --- a/python/paddle/distributed/passes/auto_parallel_fp16.py +++ b/python/paddle/distributed/passes/auto_parallel_fp16.py @@ -20,12 +20,31 @@ from paddle.fluid import unique_name from .pass_base import register_pass from paddle.fluid.data_feeder import check_variable_and_dtype, check_type -from paddle.distributed.auto_parallel.utils import set_var_dist_attr, naive_set_dist_op_attr_for_program_by_mesh_and_mapping -from paddle.distributed.auto_parallel.process_group import get_world_process_group -from paddle.fluid.contrib.mixed_precision.fp16_utils import AutoMixedPrecisionLists -from paddle.fluid.contrib.mixed_precision.fp16_utils import _keep_layer_norm_scale_bias_to_fp32, _need_keep_fp32, _valid_types, _dtype_to_str -from paddle.distributed.auto_parallel.dist_attribute import OperatorDistributedAttribute -from paddle.distributed.auto_parallel.utils import is_forward_op, is_backward_op, OP_ROLE_KEY, OpRole +from paddle.distributed.auto_parallel.utils import ( + set_var_dist_attr, + naive_set_dist_op_attr_for_program_by_mesh_and_mapping, +) +from paddle.distributed.auto_parallel.process_group import ( + get_world_process_group, +) +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + AutoMixedPrecisionLists, +) +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + _keep_layer_norm_scale_bias_to_fp32, + _need_keep_fp32, + _valid_types, + _dtype_to_str, +) +from paddle.distributed.auto_parallel.dist_attribute import ( + OperatorDistributedAttribute, +) +from paddle.distributed.auto_parallel.utils import ( + is_forward_op, + is_backward_op, + OP_ROLE_KEY, + OpRole, +) from .auto_parallel_amp import AMPPass world_process_group = get_world_process_group() @@ -39,11 +58,15 @@ def set_op_dtype_to_fp16(op): - if op.has_attr('in_dtype') and op.attr( - 'in_dtype') == core.VarDesc.VarType.FP32: + if ( + op.has_attr('in_dtype') + and op.attr('in_dtype') == core.VarDesc.VarType.FP32 + ): op._set_attr('in_dtype', core.VarDesc.VarType.FP16) - if op.has_attr('out_dtype') and op.attr( - 'out_dtype') == core.VarDesc.VarType.FP32: + if ( + op.has_attr('out_dtype') + and op.attr('out_dtype') == core.VarDesc.VarType.FP32 + ): op._set_attr('out_dtype', core.VarDesc.VarType.FP16) if op.has_attr('dtype') and op.attr('dtype') == core.VarDesc.VarType.FP32: op._set_attr('dtype', core.VarDesc.VarType.FP16) @@ -63,7 +86,12 @@ def _keep_fp32_input(op, in_name): return in_name not in {'X', 'FilterX', 'Z', 'FilterZ'} if op_type in ['fused_attention', 'fused_feedforward']: return in_name in { - 'LnScale', 'LnBias', 'Ln2Scale', 'Ln2Bias', "Ln1Scale", "Ln1Bias" + 'LnScale', + 'LnBias', + 'Ln2Scale', + 'Ln2Bias', + "Ln1Scale", + "Ln1Bias", } # backward if op_type in ['batch_norm_grad']: @@ -83,8 +111,12 @@ def _keep_fp32_output(op, out_name): return out_name not in {'Y', 'ConvX', 'ConvZ'} if op_type in ['fused_attention', 'fused_feedforward']: return out_name in { - 'LnMean', 'LnVariance', 'Ln2Mean', 'Ln2Variance', 'Ln1Mean', - 'Ln1Variance' + 'LnMean', + 'LnVariance', + 'Ln2Mean', + 'Ln2Variance', + 'Ln1Mean', + 'Ln1Variance', } # backward if op_type in ['layer_norm_grad']: @@ -95,24 +127,28 @@ def _keep_fp32_output(op, out_name): class FP16State(object): - - def __init__(self, - program, - amp_list, - dist_context, - use_fp16_guard, - input_data_var_names=None): + def __init__( + self, + program, + amp_list, + dist_context, + use_fp16_guard, + input_data_var_names=None, + ): self.program = program self.amp_list = amp_list self.use_fp16_guard = use_fp16_guard self.dist_context = dist_context - self.grad_op_to_op_map = self.dist_context.dist_op_context.grad_op_id_to_op_id + self.grad_op_to_op_map = ( + self.dist_context.dist_op_context.grad_op_id_to_op_id + ) if input_data_var_names: self.input_data_var_names = input_data_var_names else: self.input_data_var_names = [] - self._op_fp16_dict = { - } # op_id --> True/False. 'True' means that the op is should run in fp16 mode. + self._op_fp16_dict = ( + {} + ) # op_id --> True/False. 'True' means that the op is should run in fp16 mode. # a trick to determine leaf tensor node in program {varname: generator_op_id} self.forward_non_leaf_tensors = {} # record the cast ops that are inserted for a forward @@ -156,8 +192,9 @@ def _mark_op(self, op): if op.type == "assign" and "array_" in op.input_arg_names[0]: self._op_fp16_dict[op.desc.original_id()] = False return - if _need_keep_fp32(op, self.amp_list.unsupported_list, - self.use_fp16_guard): + if _need_keep_fp32( + op, self.amp_list.unsupported_list, self.use_fp16_guard + ): self._op_fp16_dict[op.desc.original_id()] = False else: self._op_fp16_dict[op.desc.original_id()] = True @@ -170,8 +207,9 @@ def _mark_op(self, op): if op.desc.original_id() in self.grad_op_to_op_map: fwd_op_id = self.grad_op_to_op_map[op.desc.original_id()] assert fwd_op_id in self._op_fp16_dict, "{}".format(str(op)) - self._op_fp16_dict[ - op.desc.original_id()] = self._op_fp16_dict[fwd_op_id] + self._op_fp16_dict[op.desc.original_id()] = self._op_fp16_dict[ + fwd_op_id + ] if int(op.attr('op_role')) == 257: self.is_train = True @@ -197,13 +235,18 @@ def resolute_tensor_dtype(self, block): for op in block.ops: if is_forward_op(op): # NOTE (JZ-LIANG) un-expected cast op when user call "+, -, *, /" in python - if self._is_fp16_op(op.desc.original_id()) == True \ - or op.type == "cast": + if ( + self._is_fp16_op(op.desc.original_id()) == True + or op.type == "cast" + ): for in_name in op.input_names: if _keep_fp32_input(op, in_name): continue for in_var_name in op.input(in_name): - if in_var_name not in self.forward_non_leaf_tensors and in_var_name not in self.input_data_var_names: + if ( + in_var_name not in self.forward_non_leaf_tensors + and in_var_name not in self.input_data_var_names + ): self.set_var_to_fp16(in_var_name, block) for out_name in op.output_names: if _keep_fp32_output(op, out_name): @@ -249,22 +292,42 @@ def cast_block(self, block): elif is_forward_op(op): if self._is_fp16_op(op.desc.original_id()) == False: num_cast_ops = self._insert_forward_cast_ops( - op, idx, block, core.VarDesc.VarType.FP16, - core.VarDesc.VarType.FP32, self.dist_context) + op, + idx, + block, + core.VarDesc.VarType.FP16, + core.VarDesc.VarType.FP32, + self.dist_context, + ) elif self._is_fp16_op(op.desc.original_id()) == True: num_cast_ops = self._insert_forward_cast_ops( - op, idx, block, core.VarDesc.VarType.FP32, - core.VarDesc.VarType.FP16, self.dist_context) + op, + idx, + block, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP16, + self.dist_context, + ) elif is_backward_op(op): if op.desc.original_id() in dist_op_context.grad_op_id_to_op_id: if self._is_fp16_op(op.desc.original_id()) == False: num_cast_ops = self._insert_backward_cast_ops( - op, idx, block, core.VarDesc.VarType.FP16, - core.VarDesc.VarType.FP32, self.dist_context) + op, + idx, + block, + core.VarDesc.VarType.FP16, + core.VarDesc.VarType.FP32, + self.dist_context, + ) elif self._is_fp16_op(op.desc.original_id()) == True: num_cast_ops = self._insert_backward_cast_ops( - op, idx, block, core.VarDesc.VarType.FP32, - core.VarDesc.VarType.FP16, self.dist_context) + op, + idx, + block, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP16, + self.dist_context, + ) elif op.type == "sum": # all inputs dtype of sum should be equal and output dtype should follow input out_var_name = op.output_arg_names[0] @@ -272,41 +335,51 @@ def cast_block(self, block): out_var = block.var(out_var_name) in_var = block._find_var_recursive(in_var_name) for in_var_name in op.input_arg_names: - assert in_var.dtype == block.var( - in_var_name).dtype, "{}, {}, {}".format( - in_var, block.var(in_var_name), str(op)) + assert ( + in_var.dtype == block.var(in_var_name).dtype + ), "{}, {}, {}".format( + in_var, block.var(in_var_name), str(op) + ) out_var.desc.set_dtype(in_var.dtype) idx += num_cast_ops + 1 block._sync_with_cpp() - def _insert_forward_cast_ops(self, op, idx, block, src_dtype, dst_dtype, - dist_context): + def _insert_forward_cast_ops( + self, op, idx, block, src_dtype, dst_dtype, dist_context + ): num_cast_ops = 0 for in_name in op.input_names: if src_dtype == core.VarDesc.VarType.FP32 and _keep_fp32_input( - op, in_name): + op, in_name + ): continue consume_op_attr = dist_context.get_op_dist_attr_for_program(op) assert consume_op_attr is not None for in_var_name in op.input(in_name): in_var = block._find_var_recursive(in_var_name) - if in_var is None or in_var.type not in _valid_types or in_var.dtype == dst_dtype: + if ( + in_var is None + or in_var.type not in _valid_types + or in_var.dtype == dst_dtype + ): continue if in_var.dtype == src_dtype: - cast_name = in_var.name + '.cast_' + _dtype_to_str( - dst_dtype) + cast_name = ( + in_var.name + '.cast_' + _dtype_to_str(dst_dtype) + ) cast_var = block.vars.get(cast_name) self.forward_input_cast_ops[op.desc.original_id()] += [ (cast_name, in_var.name, dst_dtype, src_dtype, in_name) ] in_var_dist_attr = consume_op_attr.get_input_dist_attr( - in_var.name) + in_var.name + ) assert in_var_dist_attr is not None # truly insert cast op if cast_var is None or cast_var.dtype != dst_dtype: @@ -320,9 +393,11 @@ def _insert_forward_cast_ops(self, op, idx, block, src_dtype, dst_dtype, name=cast_name, dtype=dst_dtype, persistable=False, - stop_gradient=in_var.stop_gradient) - set_var_dist_attr(dist_context, cast_var, ref_mapping, - ref_mesh) + stop_gradient=in_var.stop_gradient, + ) + set_var_dist_attr( + dist_context, cast_var, ref_mapping, ref_mesh + ) cast_op = block._insert_op_without_sync( idx, @@ -332,23 +407,27 @@ def _insert_forward_cast_ops(self, op, idx, block, src_dtype, dst_dtype, attrs={ "in_dtype": in_var.dtype, "out_dtype": cast_var.dtype, - OP_ROLE_KEY: OpRole.Forward - }) + OP_ROLE_KEY: OpRole.Forward, + }, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - cast_op, ref_mesh, ref_mapping, dist_context) + cast_op, ref_mesh, ref_mapping, dist_context + ) num_cast_ops += 1 op._rename_input(in_var.name, cast_name) - consume_op_attr.set_input_dist_attr(cast_name, - in_var_dist_attr) + consume_op_attr.set_input_dist_attr( + cast_name, in_var_dist_attr + ) if op.has_attr('out_dtype') and op.attr('out_dtype') != -1: assert op.attr('out_dtype') == dst_dtype return num_cast_ops - def _insert_backward_cast_ops(self, op, idx, block, src_dtype, dst_dtype, - dist_context): + def _insert_backward_cast_ops( + self, op, idx, block, src_dtype, dst_dtype, dist_context + ): num_cast_ops = 0 op_id = op.desc.id() @@ -364,10 +443,16 @@ def _insert_backward_cast_ops(self, op, idx, block, src_dtype, dst_dtype, if _keep_fp32_output(op, out_var.name): continue assert out_var.dtype == dst_dtype, "{}, {}".format( - str(out_var), dst_dtype) + str(out_var), dst_dtype + ) - for cast_name, src_name, dst_dtype, src_dtype, slot_name in self.forward_input_cast_ops[ - forward_op_id]: + for ( + cast_name, + src_name, + dst_dtype, + src_dtype, + slot_name, + ) in self.forward_input_cast_ops[forward_op_id]: # some forward output is not need by backward computation, e.g. logit in softmax_with_cross_entropy if slot_name not in op.input_names: @@ -375,8 +460,8 @@ def _insert_backward_cast_ops(self, op, idx, block, src_dtype, dst_dtype, # rename input assert src_name in op.input( - slot_name), "var: {} not in op's {}. {}".format( - src_name, slot_name, str(op)) + slot_name + ), "var: {} not in op's {}. {}".format(src_name, slot_name, str(op)) src_var_dist_attr = grad_op_attr.get_input_dist_attr(src_name) assert src_var_dist_attr is not None op._rename_input(src_name, cast_name) @@ -384,15 +469,16 @@ def _insert_backward_cast_ops(self, op, idx, block, src_dtype, dst_dtype, # create cast grad grad_slot_name = slot_name + "@GRAD" - assert grad_slot_name in op.output_names, "[{}], Current Op: {}".format( - grad_slot_name, str(op)) + assert ( + grad_slot_name in op.output_names + ), "[{}], Current Op: {}".format(grad_slot_name, str(op)) # some forward input maybe stop_gradient=True, e.g. input_mask if len(op.output(grad_slot_name)) == 0: continue - assert len( - op.output(grad_slot_name)) == 1, "[{}], Current Op: {}".format( - grad_slot_name, str(op)) + assert ( + len(op.output(grad_slot_name)) == 1 + ), "[{}], Current Op: {}".format(grad_slot_name, str(op)) grad_name = op.output(grad_slot_name)[0] grad = block.var(grad_name) grad_dist_attr = grad_op_attr.get_output_dist_attr(grad_name) @@ -401,15 +487,18 @@ def _insert_backward_cast_ops(self, op, idx, block, src_dtype, dst_dtype, ref_mapping = grad_dist_attr.dims_mapping cast_grad = block.create_var( - name=unique_name.generate_with_ignorable_key("".join( - [cast_name, '@GRAD'])), + name=unique_name.generate_with_ignorable_key( + "".join([cast_name, '@GRAD']) + ), dtype=dst_dtype, shape=grad.shape, type=grad.type, persistable=grad.persistable, - stop_gradient=grad.stop_gradient) + stop_gradient=grad.stop_gradient, + ) dist_context.set_tensor_dist_attr_for_program( - cast_grad, grad_dist_attr) + cast_grad, grad_dist_attr + ) op._rename_output(grad_name, cast_grad.name) grad_op_attr.set_output_dist_attr(cast_grad.name, grad_dist_attr) @@ -422,12 +511,14 @@ def _insert_backward_cast_ops(self, op, idx, block, src_dtype, dst_dtype, attrs={ "in_dtype": dst_dtype, "out_dtype": src_dtype, - OP_ROLE_KEY: OpRole.Backward - }) + OP_ROLE_KEY: OpRole.Backward, + }, + ) grad.desc.set_dtype(src_dtype) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - cast_op, ref_mesh, ref_mapping, dist_context) + cast_op, ref_mesh, ref_mapping, dist_context + ) num_cast_ops += 1 return num_cast_ops @@ -440,26 +531,34 @@ def _check_and_update_gradient(grads, loss_scaling, name, dist_context): check_type(grads, 'x', (tuple, list), 'check_finite_and_unscale') for e in grads: - check_variable_and_dtype(e, "x", ['float16', 'float32', 'float64'], - 'check_finite_and_unscale') + check_variable_and_dtype( + e, + "x", + ['float16', 'float32', 'float64'], + 'check_finite_and_unscale', + ) found_inf = main_block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - ['find_infinite_scale', name])), + name=unique_name.generate_with_ignorable_key( + ".".join(['find_infinite_scale', name]) + ), shape=[1], dtype='bool', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) set_var_dist_attr(dist_context, found_inf, [-1], world_process_group.ranks) inputs = {'X': grads, 'Scale': loss_scaling} outputs = {'Out': grads, 'FoundInfinite': found_inf} attrs = {'op_role': OpRole.Optimize} - new_op = main_block.append_op(type='check_finite_and_unscale', - inputs=inputs, - outputs=outputs, - attrs=attrs) + new_op = main_block.append_op( + type='check_finite_and_unscale', + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) new_op_dist_attr = OperatorDistributedAttribute() new_op_dist_attr.process_mesh = world_process_group.ranks @@ -469,10 +568,12 @@ def _check_and_update_gradient(grads, loss_scaling, name, dist_context): for g in grads: g_dist_attr = dist_context.get_tensor_dist_attr_for_program(g) assert g_dist_attr is not None - new_op_dist_attr.set_input_dims_mapping(g.name, - g_dist_attr.dims_mapping) - new_op_dist_attr.set_output_dims_mapping(g.name, - g_dist_attr.dims_mapping) + new_op_dist_attr.set_input_dims_mapping( + g.name, g_dist_attr.dims_mapping + ) + new_op_dist_attr.set_output_dims_mapping( + g.name, g_dist_attr.dims_mapping + ) dist_context.set_op_dist_attr_for_program(new_op, new_op_dist_attr) return grads, found_inf @@ -481,8 +582,9 @@ def _split_grads(params_grads): grads = [g for _, g in params_grads] fp32_grads = [g for g in grads if g.dtype == core.VarDesc.VarType.FP32] fp16_grads = [g for g in grads if g.dtype == core.VarDesc.VarType.FP16] - assert len(fp32_grads) + len(fp16_grads) == len(grads), \ - "Data types of all grads must be either fp16 or fp32." + assert len(fp32_grads) + len(fp16_grads) == len( + grads + ), "Data types of all grads must be either fp16 or fp32." return grads, fp32_grads, fp16_grads @@ -494,37 +596,45 @@ def _set_op_dist_attr_with_ranks(new_op, ranks, block, dist_context): var = block.var(var_name) var_dist_attr = dist_context.get_tensor_dist_attr_for_program(var) assert var_dist_attr is not None - new_op_dist_attr.set_input_dims_mapping(var_name, - var_dist_attr.dims_mapping) + new_op_dist_attr.set_input_dims_mapping( + var_name, var_dist_attr.dims_mapping + ) for var_name in new_op.output_arg_names: var = block.var(var_name) var_dist_attr = dist_context.get_tensor_dist_attr_for_program(var) assert var_dist_attr is not None - new_op_dist_attr.set_output_dims_mapping(var_name, - var_dist_attr.dims_mapping) + new_op_dist_attr.set_output_dims_mapping( + var_name, var_dist_attr.dims_mapping + ) dist_context.set_op_dist_attr_for_program(new_op, new_op_dist_attr) def _get_memcopy_idx(block, found_inf_var): # use reduce_any op for check_nan_inf as the anchor for now for idx, op in enumerate(block.ops): - if op.type == 'reduce_any' and op.output_arg_names[ - 0] == found_inf_var.name: + if ( + op.type == 'reduce_any' + and op.output_arg_names[0] == found_inf_var.name + ): return idx + 1 raise RuntimeError( - "not found the correct location for memcopy for found_inf_var.") + "not found the correct location for memcopy for found_inf_var." + ) def _insert_memcopy(block, idx, src_var, dist_context, direction="D2H"): src_name = src_var.name - output_var = block.create_var(name=unique_name.generate_with_ignorable_key( - src_name.join(['memcopy_'])), - dtype=src_var.dtype, - shape=src_var.shape, - type=core.VarDesc.VarType.LOD_TENSOR, - persistable=False, - stop_gradient=src_var.stop_gradient) + output_var = block.create_var( + name=unique_name.generate_with_ignorable_key( + src_name.join(['memcopy_']) + ), + dtype=src_var.dtype, + shape=src_var.shape, + type=core.VarDesc.VarType.LOD_TENSOR, + persistable=False, + stop_gradient=src_var.stop_gradient, + ) set_var_dist_attr(dist_context, output_var, [-1], world_process_group.ranks) @@ -535,16 +645,20 @@ def _insert_memcopy(block, idx, src_var, dist_context, direction="D2H"): dst_place_type = 1 else: raise NotImplementedError( - "direction [{}] is not supported yet.".format(direction)) + "direction [{}] is not supported yet.".format(direction) + ) attrs = {'dst_place_type': dst_place_type} - new_op = block._insert_op_without_sync(index=idx, - type='memcpy', - inputs={'X': [src_var]}, - outputs={'Out': [output_var]}, - attrs=attrs) - _set_op_dist_attr_with_ranks(new_op, world_process_group.ranks, block, - dist_context) + new_op = block._insert_op_without_sync( + index=idx, + type='memcpy', + inputs={'X': [src_var]}, + outputs={'Out': [output_var]}, + attrs=attrs, + ) + _set_op_dist_attr_with_ranks( + new_op, world_process_group.ranks, block, dist_context + ) block._sync_with_cpp() return output_var @@ -572,19 +686,21 @@ def is_initialization_op(op): for op in startup_program.global_block().ops: if is_initialization_op(op): output_name = op.output_arg_names[0] - if param_to_dtype.get(output_name, - None) == core.VarDesc.VarType.FP16: + if ( + param_to_dtype.get(output_name, None) + == core.VarDesc.VarType.FP16 + ): assert op.has_attr( 'dtype' ), "initialization op is supported to has dtype attribute but got {}.".format( - str(op)) + str(op) + ) if op.attr('dtype') == core.VarDesc.VarType.FP32: op._set_attr('dtype', core.VarDesc.VarType.FP16) @register_pass("auto_parallel_fp16") class FP16Pass(AMPPass): - def __init__(self): super(FP16Pass, self).__init__() @@ -597,16 +713,22 @@ def _apply_single_impl(self, main_program, startup_program, context): amp_list = AutoMixedPrecisionLists( set(self.get_attr("custom_white_list")), - set(self.get_attr("custom_black_list")), None) + set(self.get_attr("custom_black_list")), + None, + ) # NOTE don't not change input data dtype, since it is controled by dataloader # and which is out of control of FP16 Pass input_data_var_names = [var.name for var in self.get_attr("input_data")] with paddle.static.program_guard(main_program, startup_program): - fp16_state = FP16State(main_program, amp_list, self.dist_context, - self.get_attr("use_fp16_guard"), - input_data_var_names) + fp16_state = FP16State( + main_program, + amp_list, + self.dist_context, + self.get_attr("use_fp16_guard"), + input_data_var_names, + ) is_train = fp16_state._build_state() cast_startup_program() @@ -619,41 +741,63 @@ def _apply_single_impl(self, main_program, startup_program, context): grads, fp32_grads, fp16_grads = _split_grads(params_grads) - if self.get_attr("use_dynamic_loss_scaling" - ) or self.get_attr("init_loss_scaling") != 1.0: + if ( + self.get_attr("use_dynamic_loss_scaling") + or self.get_attr("init_loss_scaling") != 1.0 + ): found_infs = [] if fp32_grads: with main_program._optimized_guard([]): _, found_inf_fp32 = _check_and_update_gradient( - fp32_grads, self._loss_scaling, "@fp32", - self.dist_context) + fp32_grads, + self._loss_scaling, + "@fp32", + self.dist_context, + ) found_infs.append(found_inf_fp32) if fp16_grads: with main_program._optimized_guard([]): _, found_inf_fp16 = _check_and_update_gradient( - fp16_grads, self._loss_scaling, "@fp16", - self.dist_context) + fp16_grads, + self._loss_scaling, + "@fp16", + self.dist_context, + ) found_infs.append(found_inf_fp16) with main_program._optimized_guard([]): block = main_program.global_block() all_infs = paddle.fluid.layers.concat(found_infs) - set_var_dist_attr(self.dist_context, all_infs, [-1], - world_process_group.ranks) + set_var_dist_attr( + self.dist_context, + all_infs, + [-1], + world_process_group.ranks, + ) new_op = block.ops[-1] assert new_op.type == "concat" - _set_op_dist_attr_with_ranks(new_op, - world_process_group.ranks, - block, self.dist_context) + _set_op_dist_attr_with_ranks( + new_op, + world_process_group.ranks, + block, + self.dist_context, + ) found_inf = paddle.fluid.layers.reduce_any(all_infs) - set_var_dist_attr(self.dist_context, found_inf, [-1], - world_process_group.ranks) + set_var_dist_attr( + self.dist_context, + found_inf, + [-1], + world_process_group.ranks, + ) new_op = block.ops[-1] assert new_op.type == "reduce_any" - _set_op_dist_attr_with_ranks(new_op, - world_process_group.ranks, - block, self.dist_context) + _set_op_dist_attr_with_ranks( + new_op, + world_process_group.ranks, + block, + self.dist_context, + ) if self.get_attr("use_dynamic_loss_scaling"): with main_program._optimized_guard([]): @@ -668,14 +812,15 @@ def _apply_single_impl(self, main_program, startup_program, context): if self.get_attr("use_optimizer_fp16"): base_opt._multi_precision = False if isinstance( - base_opt, - (paddle.fluid.optimizer.Adam, paddle.optimizer.AdamW)): + base_opt, (paddle.fluid.optimizer.Adam, paddle.optimizer.AdamW) + ): with main_program._optimized_guard([]): # found_inf = paddle.tensor.creation._memcpy( # found_inf, paddle.CPUPlace()) insert_idx = _get_memcopy_idx(block, found_inf) - found_inf = _insert_memcopy(block, insert_idx, found_inf, - self.dist_context) + found_inf = _insert_memcopy( + block, insert_idx, found_inf, self.dist_context + ) base_opt._set_auxiliary_var('found_inf', found_inf.name) elif hasattr(base_opt, "_set_auxiliary_var"): base_opt._set_auxiliary_var('found_inf', found_inf.name) diff --git a/python/paddle/distributed/passes/auto_parallel_grad_clip.py b/python/paddle/distributed/passes/auto_parallel_grad_clip.py index 5108992ae55fe..ebb44bb0f1264 100644 --- a/python/paddle/distributed/passes/auto_parallel_grad_clip.py +++ b/python/paddle/distributed/passes/auto_parallel_grad_clip.py @@ -20,8 +20,17 @@ from .pass_base import PassBase, register_pass from ..auto_parallel.reshard import Resharder from ..auto_parallel.process_group import get_world_process_group -from ..auto_parallel.utils import is_gradient_clip_op, is_optimize_op, OP_ROLE_KEY, OpRole, _get_comm_group -from ..auto_parallel.dist_attribute import TensorDistributedAttribute, OperatorDistributedAttribute +from ..auto_parallel.utils import ( + is_gradient_clip_op, + is_optimize_op, + OP_ROLE_KEY, + OpRole, + _get_comm_group, +) +from ..auto_parallel.dist_attribute import ( + TensorDistributedAttribute, + OperatorDistributedAttribute, +) def _get_params_grads(block): @@ -52,7 +61,8 @@ def _get_dpmp_topology(origin_topology, sharding_group): """ sharding_axis = 1 dp_sharding_topology = [ - origin_topology[0] // sharding_group.nranks, sharding_group.nranks + origin_topology[0] // sharding_group.nranks, + sharding_group.nranks, ] if dp_sharding_topology[0] == 1: sharding_axis = 0 @@ -108,22 +118,24 @@ def _get_dpmp_process_mesh(rank_id, topology, processes, sharding_group): return dpmp_topology, list(dpmp_processes_in_sharding) -def _is_about_global_norm(rank_id, tensor_shape, topology, processes, - dims_mapping, sharding_group): +def _is_about_global_norm( + rank_id, tensor_shape, topology, processes, dims_mapping, sharding_group +): # get current process_mesh where the parameter exist. dpmp_topology, dpmp_processes = _get_dpmp_process_mesh( - rank_id, topology, processes, sharding_group) + rank_id, topology, processes, sharding_group + ) - complete_shape = Resharder.compute_complete_shape(tensor_shape, - dpmp_topology, - dims_mapping) + complete_shape = Resharder.compute_complete_shape( + tensor_shape, dpmp_topology, dims_mapping + ) complete_partitions = [] complete_param_ranks = [] for process in dpmp_processes: partition_index = Resharder.compute_partition_index( - process, complete_shape, dims_mapping, dpmp_topology, - dpmp_processes) + process, complete_shape, dims_mapping, dpmp_topology, dpmp_processes + ) if partition_index not in complete_partitions: complete_partitions.append(partition_index) complete_param_ranks.append(process) @@ -132,7 +144,6 @@ def _is_about_global_norm(rank_id, tensor_shape, topology, processes, class ClipHelper(object): - def __init__(self, params_grads, rank_id, block, dist_context): params, _ = zip(*params_grads) self.params = list(params) @@ -154,9 +165,14 @@ def _is_calcuate_norm(self, name): topology = dist_attr.process_mesh.topology processes = dist_attr.process_mesh.processes dims_mapping = dist_attr.dims_mapping - return _is_about_global_norm(self.rank_id, param.shape, topology, - processes, dims_mapping, - self.sharding_group) + return _is_about_global_norm( + self.rank_id, + param.shape, + topology, + processes, + dims_mapping, + self.sharding_group, + ) def _get_dist_attr(self, name): var = self.block.vars[name] @@ -181,7 +197,8 @@ def _init_dist_attr(self, op): in_dist_attr.process_mesh = self.world_ranks in_dist_attr.dims_mapping = [-1] self.dist_context.set_tensor_dist_attr_for_program( - in_var, in_dist_attr) + in_var, in_dist_attr + ) op_dist_attr.set_input_dist_attr(in_name, in_dist_attr) for out_name in op.output_arg_names: out_var = self.block.vars[out_name] @@ -189,7 +206,8 @@ def _init_dist_attr(self, op): out_dist_attr.process_mesh = self.world_ranks out_dist_attr.dims_mapping = [-1] self.dist_context.set_tensor_dist_attr_for_program( - out_var, out_dist_attr) + out_var, out_dist_attr + ) op_dist_attr.set_output_dist_attr(out_name, out_dist_attr) self.dist_context.set_op_dist_attr_for_program(op, op_dist_attr) @@ -228,14 +246,18 @@ def _apply_single_impl(self, main_program, startup_program, context): dist_params_grads = self.get_attr("params_grads", None) # dist_params_grads = _get_params_grads(block) - self.clip_helper = ClipHelper(dist_params_grads, rank_id, block, - dist_context) + self.clip_helper = ClipHelper( + dist_params_grads, rank_id, block, dist_context + ) self._remove_no_need_ops_vars(block) def _remove_no_need_ops_vars(self, block): removed_op_out_type = [ - 'clip_by_norm', 'squared_l2_norm', 'square', 'reduce_sum' + 'clip_by_norm', + 'squared_l2_norm', + 'square', + 'reduce_sum', ] removed_op_idx = set() @@ -248,12 +270,14 @@ def _remove_no_need_ops_vars(self, block): input_name = op.input("X")[0] if input_name.find("@GRAD") != -1: #'clip_by_norm', 'squared_l2_norm', 'square' - param_name = input_name[:input_name.find("@GRAD")] + param_name = input_name[: input_name.find("@GRAD")] is_local = self.clip_helper._is_local_param(param_name) is_calculate = self.clip_helper._is_calcuate_norm( - param_name) - if not is_local or (not is_calculate - and op.type != 'clip_by_norm'): + param_name + ) + if not is_local or ( + not is_calculate and op.type != 'clip_by_norm' + ): removed_op_idx.add(idx) removed_tmp_var.update(set(op.output_arg_names)) else: @@ -265,20 +289,23 @@ def _remove_no_need_ops_vars(self, block): elif op.type == 'elementwise_mul': input_name = op.input("X")[0] if input_name.find("@GRAD") != -1: - param_name = input_name[:input_name.find("@GRAD")] + param_name = input_name[: input_name.find("@GRAD")] is_local = self.clip_helper._is_local_param(param_name) if not is_local: removed_op_idx.add(idx) if block.ops[idx - 1].type == 'cast': removed_op_idx.add(idx - 1) removed_tmp_var.update( - set(block.ops[idx - 1].output_arg_names)) + set(block.ops[idx - 1].output_arg_names) + ) elif op.type == 'sum': reserved_vars = [] for input_name in op.input_arg_names: - if input_name not in removed_tmp_var and \ - self.clip_helper._is_local_var(input_name): + if ( + input_name not in removed_tmp_var + and self.clip_helper._is_local_var(input_name) + ): reserved_vars.append(input_name) if not reserved_vars: removed_op_idx.add(idx) @@ -286,7 +313,8 @@ def _remove_no_need_ops_vars(self, block): if block.ops[idx + 1].type == 'cast': removed_op_idx.add(idx + 1) removed_tmp_var.update( - set(block.ops[idx + 1].output_arg_names)) + set(block.ops[idx + 1].output_arg_names) + ) else: op.desc.set_input("X", reserved_vars) @@ -320,10 +348,12 @@ def _remove_no_need_ops_vars(self, block): 'dtype': input_var.dtype, 'value': 0, 'force_cpu': False, - OP_ROLE_KEY: OpRole.Optimize - }) - fill_constant_op._set_attr('op_namescope', - "/gradient_clip_pass") + OP_ROLE_KEY: OpRole.Optimize, + }, + ) + fill_constant_op._set_attr( + 'op_namescope', "/gradient_clip_pass" + ) offset += 1 self.clip_helper._init_dist_attr(fill_constant_op) @@ -336,9 +366,11 @@ def _remove_no_need_ops_vars(self, block): 'ring_id': 0, 'use_calc_stream': True, OP_ROLE_KEY: OpRole.Optimize, - }) - allreduce_op._set_attr('op_namescope', - "/gradient_clip_pass") + }, + ) + allreduce_op._set_attr( + 'op_namescope', "/gradient_clip_pass" + ) self.clip_helper._init_dist_attr(allreduce_op) for varname in removed_tmp_var: diff --git a/python/paddle/distributed/passes/auto_parallel_gradient_merge.py b/python/paddle/distributed/passes/auto_parallel_gradient_merge.py index 2c0af10e35d98..282f909987d41 100644 --- a/python/paddle/distributed/passes/auto_parallel_gradient_merge.py +++ b/python/paddle/distributed/passes/auto_parallel_gradient_merge.py @@ -19,9 +19,18 @@ from paddle.fluid import layers from paddle.fluid.framework import device_guard from .pass_base import PassBase, PassType, register_pass -from paddle.distributed.auto_parallel.utils import set_var_dist_attr, is_optimize_op, OpRole, OP_ROLE_KEY -from paddle.distributed.auto_parallel.utils import naive_set_dist_op_attr_for_program_by_mesh_and_mapping -from paddle.distributed.auto_parallel.process_group import get_world_process_group +from paddle.distributed.auto_parallel.utils import ( + set_var_dist_attr, + is_optimize_op, + OpRole, + OP_ROLE_KEY, +) +from paddle.distributed.auto_parallel.utils import ( + naive_set_dist_op_attr_for_program_by_mesh_and_mapping, +) +from paddle.distributed.auto_parallel.process_group import ( + get_world_process_group, +) world_process_group = get_world_process_group() @@ -56,79 +65,87 @@ def _remove_and_get_optimizer_op(main_program, dist_context): def _get_gm_cond_var(main_program, k_steps, dist_context): main_block = main_program.global_block() # Add const var - k_step_var = layers.create_global_var(name="gradient_merge_k", - shape=[1], - value=int(k_steps), - dtype='int32', - persistable=True, - force_cpu=True) + k_step_var = layers.create_global_var( + name="gradient_merge_k", + shape=[1], + value=int(k_steps), + dtype='int32', + persistable=True, + force_cpu=True, + ) set_var_dist_attr(dist_context, k_step_var, [-1], world_process_group.ranks) - zero_var = layers.create_global_var(name="gradient_merge_zero", - shape=[1], - value=int(0), - dtype='int32', - persistable=True, - force_cpu=True) + zero_var = layers.create_global_var( + name="gradient_merge_zero", + shape=[1], + value=int(0), + dtype='int32', + persistable=True, + force_cpu=True, + ) set_var_dist_attr(dist_context, zero_var, [-1], world_process_group.ranks) # Add step var & cond var - step_var = layers.create_global_var(name="gradient_merge_step", - shape=[1], - value=int(0), - dtype='int32', - persistable=True, - force_cpu=True) + step_var = layers.create_global_var( + name="gradient_merge_step", + shape=[1], + value=int(0), + dtype='int32', + persistable=True, + force_cpu=True, + ) set_var_dist_attr(dist_context, step_var, [-1], world_process_group.ranks) - cond_var = main_block.create_var(name="gradient_merge_cond", - shape=[1], - dtype='bool') + cond_var = main_block.create_var( + name="gradient_merge_cond", shape=[1], dtype='bool' + ) set_var_dist_attr(dist_context, cond_var, [-1], world_process_group.ranks) with device_guard("cpu"): # step_var += 1 - increment_op = main_block.append_op(type='increment', - inputs={'X': [step_var]}, - outputs={'Out': [step_var]}, - attrs={ - 'step': float(1.0), - OP_ROLE_KEY: OpRole.Backward - }) + increment_op = main_block.append_op( + type='increment', + inputs={'X': [step_var]}, + outputs={'Out': [step_var]}, + attrs={'step': float(1.0), OP_ROLE_KEY: OpRole.Backward}, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - increment_op, world_process_group.ranks, [-1], dist_context) + increment_op, world_process_group.ranks, [-1], dist_context + ) # step_var %= k_step - elementwise_mod_op = main_block.append_op(type='elementwise_mod', - inputs={ - 'X': step_var, - 'Y': k_step_var - }, - outputs={'Out': step_var}, - attrs={ - 'axis': -1, - 'use_mkldnn': False, - OP_ROLE_KEY: - OpRole.Backward - }) + elementwise_mod_op = main_block.append_op( + type='elementwise_mod', + inputs={'X': step_var, 'Y': k_step_var}, + outputs={'Out': step_var}, + attrs={ + 'axis': -1, + 'use_mkldnn': False, + OP_ROLE_KEY: OpRole.Backward, + }, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - elementwise_mod_op, world_process_group.ranks, [-1], dist_context) + elementwise_mod_op, world_process_group.ranks, [-1], dist_context + ) # cond_var = (step_var == 0) - equal_op = main_block.append_op(type='equal', - inputs={ - 'X': step_var, - 'Y': zero_var - }, - outputs={'Out': cond_var}, - attrs={OP_ROLE_KEY: OpRole.Backward}) + equal_op = main_block.append_op( + type='equal', + inputs={'X': step_var, 'Y': zero_var}, + outputs={'Out': cond_var}, + attrs={OP_ROLE_KEY: OpRole.Backward}, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - equal_op, world_process_group.ranks, [-1], dist_context) + equal_op, world_process_group.ranks, [-1], dist_context + ) return cond_var def _append_gradient_merge_backward_op( - main_program, startup_program, params_grads: List[Tuple[Any, Any]], - dist_context) -> Tuple[List[Tuple[Any, Any]], Dict[str, Any]]: + main_program, + startup_program, + params_grads: List[Tuple[Any, Any]], + dist_context, +) -> Tuple[List[Tuple[Any, Any]], Dict[str, Any]]: main_block = main_program.global_block() startup_block = startup_program.global_block() @@ -146,57 +163,66 @@ def _append_gradient_merge_backward_op( for param, grad in params_grads: param_name = param.name param_var = main_block.var(param_name) - assert (param_var is not None) + assert param_var is not None ref_dist_attr = dist_context.get_tensor_dist_attr_for_program(param_var) assert ref_dist_attr is not None - gradient_merge_var = main_block.create_var(name=param_name + - "@GRAD@GradientMerge", - shape=param_var.shape, - dtype=param_var.dtype, - persistable=True) + gradient_merge_var = main_block.create_var( + name=param_name + "@GRAD@GradientMerge", + shape=param_var.shape, + dtype=param_var.dtype, + persistable=True, + ) ref_process_mesh = ref_dist_attr.process_mesh ref_dims_mapping = ref_dist_attr.dims_mapping - set_var_dist_attr(dist_context, gradient_merge_var, ref_dims_mapping, - ref_process_mesh) + set_var_dist_attr( + dist_context, gradient_merge_var, ref_dims_mapping, ref_process_mesh + ) startup_gradient_merge_var = startup_block.create_var( name=param_name + "@GRAD@GradientMerge", shape=param_var.shape, dtype=param_var.dtype, - persistable=True) - startup_block.append_op(type="fill_constant", - outputs={"Out": startup_gradient_merge_var}, - attrs={ - "shape": param_var.shape, - "dtype": param_var.dtype, - "value": float(0), - }) + persistable=True, + ) + startup_block.append_op( + type="fill_constant", + outputs={"Out": startup_gradient_merge_var}, + attrs={ + "shape": param_var.shape, + "dtype": param_var.dtype, + "value": float(0), + }, + ) # grad_merge += grad - new_grad_op = main_block.append_op(type="elementwise_add", - inputs={ - 'X': grad, - 'Y': gradient_merge_var - }, - outputs={'Out': gradient_merge_var}, - attrs={ - 'axis': -1, - 'use_mkldnn': False, - OP_ROLE_KEY: OpRole.Backward - }) + new_grad_op = main_block.append_op( + type="elementwise_add", + inputs={'X': grad, 'Y': gradient_merge_var}, + outputs={'Out': gradient_merge_var}, + attrs={ + 'axis': -1, + 'use_mkldnn': False, + OP_ROLE_KEY: OpRole.Backward, + }, + ) new_params_to_grads.append([param, gradient_merge_var]) grad_to_gradient_merge[grad.name] = gradient_merge_var.name naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - new_grad_op, ref_process_mesh, ref_dims_mapping, dist_context) + new_grad_op, ref_process_mesh, ref_dims_mapping, dist_context + ) return new_params_to_grads, grad_to_gradient_merge def _create_cond_block_and_update_optimizer( - main_program, cond_var, new_params_to_grads: List[Tuple[Any, Any]], - grad_to_gradient_merge: Dict[str, str], optimize_ops_desc: List[Any], - k_steps, avg): - + main_program, + cond_var, + new_params_to_grads: List[Tuple[Any, Any]], + grad_to_gradient_merge: Dict[str, str], + optimize_ops_desc: List[Any], + k_steps, + avg, +): def true_apply_gradient(): cur_block_idx = main_program.current_block_idx cur_block = main_program.current_block() @@ -207,14 +233,16 @@ def true_apply_gradient(): if avg: for param, new_grad in new_params_to_grads: # grad /= k_steps - cur_block.append_op(type='scale', - inputs={'X': new_grad}, - outputs={'Out': new_grad}, - attrs={ - 'scale': 1.0 / k_steps, - 'bias': 0.0, - 'bias_after_scale': False - }) + cur_block.append_op( + type='scale', + inputs={'X': new_grad}, + outputs={'Out': new_grad}, + attrs={ + 'scale': 1.0 / k_steps, + 'bias': 0.0, + 'bias_after_scale': False, + }, + ) new_grad.op._set_attr(OP_ROLE_KEY, OpRole.Optimize) # append optimizer ops @@ -222,16 +250,18 @@ def true_apply_gradient(): new_op_desc = cur_block.desc.append_op() new_op_desc.copy_from(op_desc) - #update input/output + # update input/output for input_name in new_op_desc.input_arg_names(): if input_name in grad_to_gradient_merge: new_op_desc._rename_input( - input_name, grad_to_gradient_merge[input_name]) + input_name, grad_to_gradient_merge[input_name] + ) for output_name in new_op_desc.output_arg_names(): if output_name in grad_to_gradient_merge: new_op_desc._rename_output( - output_name, grad_to_gradient_merge[output_name]) + output_name, grad_to_gradient_merge[output_name] + ) # remove op_role_var if new_op_desc.has_attr(op_maker.kOpRoleVarAttrName()): @@ -249,10 +279,12 @@ def true_apply_gradient(): # clear gradient_merge_vars for param, new_grad in new_params_to_grads: - layers.fill_constant(shape=new_grad.shape, - dtype=new_grad.dtype, - value=0.0, - out=new_grad) + layers.fill_constant( + shape=new_grad.shape, + dtype=new_grad.dtype, + value=0.0, + out=new_grad, + ) new_grad.op._set_attr(OP_ROLE_KEY, op_maker.OpRole.Optimize) layers.cond(cond_var, true_fn=true_apply_gradient, false_fn=None) @@ -260,8 +292,9 @@ def true_apply_gradient(): cond_op._set_attr(OP_ROLE_KEY, OpRole.Optimize) -def parse_program(main_program, startup_program, params_grads, k_steps, avg, - dist_context): +def parse_program( + main_program, startup_program, params_grads, k_steps, avg, dist_context +): # 1 remove optimizer_op from main_program optimize_ops_desc = _remove_and_get_optimizer_op(main_program, dist_context) @@ -269,22 +302,30 @@ def parse_program(main_program, startup_program, params_grads, k_steps, avg, main_program._rollback() # 2 append gradient merge backward op to main_program - new_params_to_grads, grad_to_gradient_merge = _append_gradient_merge_backward_op( - main_program, startup_program, params_grads, dist_context) + ( + new_params_to_grads, + grad_to_gradient_merge, + ) = _append_gradient_merge_backward_op( + main_program, startup_program, params_grads, dist_context + ) # 3 create gradient_merge_cond cond_var = _get_gm_cond_var(main_program, k_steps, dist_context) # 4 create ConditionalBlock and append gradient merge optimizer ops - _create_cond_block_and_update_optimizer(main_program, cond_var, - new_params_to_grads, - grad_to_gradient_merge, - optimize_ops_desc, k_steps, avg) + _create_cond_block_and_update_optimizer( + main_program, + cond_var, + new_params_to_grads, + grad_to_gradient_merge, + optimize_ops_desc, + k_steps, + avg, + ) @register_pass("auto_parallel_gradient_merge_pass") class GradientMergePass(PassBase): - def __init__(self): super(GradientMergePass, self).__init__() self.set_attr("k_steps", -1) @@ -307,7 +348,13 @@ def _apply_single_impl(self, main_program, startup_program, context): dist_context = self.get_attr("dist_context") params_grads = self.get_attr("params_grads") with paddle.static.program_guard(main_program, startup_program): - parse_program(main_program, startup_program, params_grads, k_steps, - avg, dist_context) + parse_program( + main_program, + startup_program, + params_grads, + k_steps, + avg, + dist_context, + ) main_program._sync_with_cpp() diff --git a/python/paddle/distributed/passes/auto_parallel_quantization.py b/python/paddle/distributed/passes/auto_parallel_quantization.py index c0ac93d83939d..893b9d0d9e5cf 100644 --- a/python/paddle/distributed/passes/auto_parallel_quantization.py +++ b/python/paddle/distributed/passes/auto_parallel_quantization.py @@ -20,7 +20,10 @@ from paddle.fluid.contrib.slim.quantization import QuantizationTransformPassV2 from paddle.fluid.contrib.slim.quantization import AddQuantDequantPassV2 from paddle.fluid.contrib.slim.quantization import OutScaleForTrainingPass -from paddle.distributed.auto_parallel.dist_attribute import OperatorDistributedAttribute, TensorDistributedAttribute +from paddle.distributed.auto_parallel.dist_attribute import ( + OperatorDistributedAttribute, + TensorDistributedAttribute, +) from .pass_base import PassBase, register_pass @@ -34,7 +37,6 @@ def _node_id(node): @register_pass("auto_parallel_quantization") class QuantizationPass(PassBase): - def __init__(self): super(QuantizationPass, self).__init__() self.set_attr("dist_context", None) @@ -61,14 +63,19 @@ def _apply_single_impl(self, main_program, startup_program, context): place = paddle.fluid.CUDAPlace(ParallelEnv().dev_id) # 1. Program convert to Graph, and this pass is only for train mode - main_graph = framework.IrGraph(core.Graph(main_program.desc), - for_test=False) + main_graph = framework.IrGraph( + core.Graph(main_program.desc), for_test=False + ) # 2. Prepare inputs transform_pass_ops = [] quant_dequant_ops = [] quantize_op_types = [ - 'conv2d', 'depthwise_conv2d', 'mul', 'matmul', 'matmul_v2' + 'conv2d', + 'depthwise_conv2d', + 'mul', + 'matmul', + 'matmul_v2', ] for op_type in quantize_op_types: if op_type in TRANSFORM_PASS_OP_TYPES: @@ -76,8 +83,11 @@ def _apply_single_impl(self, main_program, startup_program, context): elif op_type in QUANT_DEQUANT_PASS_OP_TYPES: quant_dequant_ops.append(op_type) - weight_quantize_type = "channel_wise_abs_max" if self.get_attr( - 'channel_wise_abs_max') else "abs_max" + weight_quantize_type = ( + "channel_wise_abs_max" + if self.get_attr('channel_wise_abs_max') + else "abs_max" + ) # 3. Add quant op for ops which have parameters transform_pass = QuantizationTransformPassV2( @@ -94,7 +104,8 @@ def _apply_single_impl(self, main_program, startup_program, context): weight_preprocess_func=None, act_preprocess_func=None, optimizer_func=None, - executor=None) + executor=None, + ) transform_pass.apply(main_graph) # 4. Add quant op for ops which don't have parameter @@ -103,12 +114,14 @@ def _apply_single_impl(self, main_program, startup_program, context): place=place, quant_bits=self.get_attr('activation_bits'), skip_pattern=self.get_attr('not_quant_pattern'), - quantizable_op_type=quant_dequant_ops) + quantizable_op_type=quant_dequant_ops, + ) quant_dequant_pass.apply(main_graph) # 5. Gather quantitative information for the output - out_scale_training_pass = OutScaleForTrainingPass(scope=scope, - place=place) + out_scale_training_pass = OutScaleForTrainingPass( + scope=scope, place=place + ) out_scale_training_pass.apply(main_graph) # 6. Convert Graph back to Program @@ -132,34 +145,43 @@ def _apply_single_impl(self, main_program, startup_program, context): for ip, quant_op in enumerate(block.ops): quant_op_dist_attr = OperatorDistributedAttribute() - if "quantize" in quant_op.type or \ - quant_op.type == "moving_average_abs_max_scale": + if ( + "quantize" in quant_op.type + or quant_op.type == "moving_average_abs_max_scale" + ): input_name = quant_op.desc.input('X')[0] if "quantize" in input_name: - input_name = input_name[:input_name.index(".quantized")] + input_name = input_name[ + : input_name.index(".quantized") + ] if quant_op.type == "moving_average_abs_max_scale": consume_op = main_program.blocks[ib].vars[input_name].op else: - consume_op = main_program.blocks[ib].ops[ip - - qat_offset] + consume_op = main_program.blocks[ib].ops[ + ip - qat_offset + ] consume_op_dist_attr = dist_context.get_dist_op_for_program( - consume_op).dist_attr + consume_op + ).dist_attr ref_process_mesh = consume_op_dist_attr.process_mesh if input_name in consume_op_dist_attr.outputs_dist_attrs: - consume_input_dist_attr = consume_op_dist_attr.outputs_dist_attrs[ - input_name] + consume_input_dist_attr = ( + consume_op_dist_attr.outputs_dist_attrs[input_name] + ) else: - consume_input_dist_attr = consume_op_dist_attr.inputs_dist_attrs[ - input_name] + consume_input_dist_attr = ( + consume_op_dist_attr.inputs_dist_attrs[input_name] + ) quant_op_dist_attr.impl_idx = 0 quant_op_dist_attr.impl_type = "default" quant_op_dist_attr.process_mesh = ref_process_mesh quant_op_dist_attr.set_input_dist_attr( - quant_op.desc.input('X')[0], consume_input_dist_attr) + quant_op.desc.input('X')[0], consume_input_dist_attr + ) for slot_name in quant_op.desc.input_names(): if slot_name == "X": @@ -170,26 +192,32 @@ def _apply_single_impl(self, main_program, startup_program, context): tensor_dist_attr.process_mesh = ref_process_mesh tensor_dist_attr.dims_mapping = [-1] dist_context.set_tensor_dist_attr_for_program( - input_var, tensor_dist_attr) + input_var, tensor_dist_attr + ) quant_op_dist_attr.set_input_dist_attr( - in_name, tensor_dist_attr) + in_name, tensor_dist_attr + ) for slot_name in quant_op.desc.output_names(): output_name = quant_op.desc.output(slot_name)[0] output_var = block.vars[output_name] if slot_name == "Y": dist_context.set_tensor_dist_attr_for_program( - output_var, consume_input_dist_attr) + output_var, consume_input_dist_attr + ) quant_op_dist_attr.set_output_dist_attr( - output_name, consume_input_dist_attr) + output_name, consume_input_dist_attr + ) else: tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.process_mesh = ref_process_mesh tensor_dist_attr.dims_mapping = [-1] dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr) + output_var, tensor_dist_attr + ) quant_op_dist_attr.set_output_dist_attr( - output_name, tensor_dist_attr) + output_name, tensor_dist_attr + ) quant_op._set_attr("op_device", "") qat_offset += 1 @@ -199,59 +227,86 @@ def _apply_single_impl(self, main_program, startup_program, context): origin_op = main_program.blocks[ib].ops[ip - qat_offset] quant_op.desc.set_original_id(origin_op.desc.original_id()) dist_origin_op = dist_context.get_dist_op_for_program( - origin_op) - assert dist_origin_op is not None, "origin op must have dist attr." + origin_op + ) + assert ( + dist_origin_op is not None + ), "origin op must have dist attr." origin_op_dist_attr = dist_origin_op.dist_attr quant_op_dist_attr.impl_idx = origin_op_dist_attr.impl_idx quant_op_dist_attr.impl_type = origin_op_dist_attr.impl_type - quant_op_dist_attr.process_mesh = origin_op_dist_attr.process_mesh + quant_op_dist_attr.process_mesh = ( + origin_op_dist_attr.process_mesh + ) for idx, input_name in enumerate(quant_op.input_arg_names): origin_input_name = origin_op.input_arg_names[idx] - origin_input_dist_attr = origin_op_dist_attr.inputs_dist_attrs[ - origin_input_name] + origin_input_dist_attr = ( + origin_op_dist_attr.inputs_dist_attrs[ + origin_input_name + ] + ) quant_op_dist_attr.set_input_dist_attr( - input_name, origin_input_dist_attr) + input_name, origin_input_dist_attr + ) if input_name not in main_program.blocks[ib].vars: origin_input_var = main_program.blocks[ib].vars[ - origin_input_name] - origin_in_tensor_dist_attr = dist_context.get_dist_tensor_for_program( - origin_input_var).dist_attr + origin_input_name + ] + origin_in_tensor_dist_attr = ( + dist_context.get_dist_tensor_for_program( + origin_input_var + ).dist_attr + ) quant_input_var = block.vars[input_name] dist_context.set_tensor_dist_attr_for_program( - quant_input_var, origin_in_tensor_dist_attr) + quant_input_var, origin_in_tensor_dist_attr + ) for idx, output_name in enumerate( - quant_op.output_arg_names): + quant_op.output_arg_names + ): origin_output_name = origin_op.output_arg_names[idx] - origin_output_dist_attr = origin_op_dist_attr.outputs_dist_attrs[ - origin_output_name] + origin_output_dist_attr = ( + origin_op_dist_attr.outputs_dist_attrs[ + origin_output_name + ] + ) quant_op_dist_attr.set_output_dist_attr( - output_name, origin_output_dist_attr) + output_name, origin_output_dist_attr + ) if output_name not in main_program.blocks[ib].vars: origin_output_var = main_program.blocks[ib].vars[ - origin_output_name] - origin_out_tensor_dist_attr = dist_context.get_dist_tensor_for_program( - origin_output_var).dist_attr + origin_output_name + ] + origin_out_tensor_dist_attr = ( + dist_context.get_dist_tensor_for_program( + origin_output_var + ).dist_attr + ) quant_output_var = block.vars[output_name] dist_context.set_tensor_dist_attr_for_program( - quant_output_var, origin_out_tensor_dist_attr) + quant_output_var, origin_out_tensor_dist_attr + ) dist_context.set_op_dist_attr_for_program( - quant_op, quant_op_dist_attr) + quant_op, quant_op_dist_attr + ) # recover vars' dist_attr for name, dst_var in block.vars.items(): if name in main_program.blocks[ib].vars: src_var = main_program.blocks[ib].vars[name] dist_tensor = dist_context.get_dist_tensor_for_program( - src_var) + src_var + ) if not dist_tensor: continue dist_context.set_tensor_dist_attr_for_program( - dst_var, dist_tensor.dist_attr) + dst_var, dist_tensor.dist_attr + ) context.set_attr("main_program", quant_program) context.set_attr("startup_program", startup_program) diff --git a/python/paddle/distributed/passes/auto_parallel_recompute.py b/python/paddle/distributed/passes/auto_parallel_recompute.py index 81bda0d3d3955..26568fbe72755 100644 --- a/python/paddle/distributed/passes/auto_parallel_recompute.py +++ b/python/paddle/distributed/passes/auto_parallel_recompute.py @@ -20,13 +20,20 @@ from paddle.fluid.framework import Variable from paddle.fluid.backward import _append_grad_suffix_, _get_no_grad_set_name from paddle.fluid.backward import ProgramStats, _rename_arg_, _find_op_path_ -from paddle.distributed.auto_parallel.dist_attribute import OperatorDistributedAttribute -from paddle.distributed.auto_parallel.utils import get_loss_op, set_var_dist_attr, set_dist_op_desc_original_id -from paddle.distributed.auto_parallel.utils import naive_set_dist_op_attr_for_program_by_mesh_and_mapping +from paddle.distributed.auto_parallel.dist_attribute import ( + OperatorDistributedAttribute, +) +from paddle.distributed.auto_parallel.utils import ( + get_loss_op, + set_var_dist_attr, + set_dist_op_desc_original_id, +) +from paddle.distributed.auto_parallel.utils import ( + naive_set_dist_op_attr_for_program_by_mesh_and_mapping, +) class RecomputeState(ProgramStats): - def __init__(self, block, ops): super(RecomputeState, self).__init__(block=block, ops=ops) self._block = block @@ -52,7 +59,7 @@ def build_stats(self): self.var_op_deps[name]["var_as_output_ops"] = [i] def get_recompute_segments(self, checkpoints): - """ get recompute segments from checkpoints """ + """get recompute segments from checkpoints""" segments = [] start_idx = -1 pre_segment_end_idx = -1 @@ -67,27 +74,37 @@ def get_recompute_segments(self, checkpoints): segments.append([0, max(op_idx_list) + 1]) else: flag, min_idx, max_idx = self.is_subgraph( - [checkpoints[start_idx]], [checkpoints[start_idx + 1]]) + [checkpoints[start_idx]], [checkpoints[start_idx + 1]] + ) if flag: min_idx = self._update_segment_start( - min_idx, pre_segment_end_idx) + min_idx, pre_segment_end_idx + ) segments.append([min_idx, max_idx + 1]) else: logging.info( "Could not recompute op range [{}] - [{}] ".format( - min_idx, max_idx + 1)) + min_idx, max_idx + 1 + ) + ) start_idx += 1 for i, (idx1, idx2) in enumerate(segments): logging.info("recompute segment[{}]".format(i)) - logging.info("segment start op: [{}]: [{}] [{}]".format( - self._ops[idx1].desc.type(), - self._ops[idx1].desc.input_arg_names(), - self._ops[idx1].desc.output_arg_names())) - logging.info("segment end op: [{}]: [{}] [{}]".format( - self._ops[idx2 - 1].desc.type(), - self._ops[idx2 - 1].desc.input_arg_names(), - self._ops[idx2 - 1].desc.output_arg_names())) + logging.info( + "segment start op: [{}]: [{}] [{}]".format( + self._ops[idx1].desc.type(), + self._ops[idx1].desc.input_arg_names(), + self._ops[idx1].desc.output_arg_names(), + ) + ) + logging.info( + "segment end op: [{}]: [{}] [{}]".format( + self._ops[idx2 - 1].desc.type(), + self._ops[idx2 - 1].desc.input_arg_names(), + self._ops[idx2 - 1].desc.output_arg_names(), + ) + ) return segments @@ -115,44 +132,49 @@ def modify_forward_desc_for_recompute(self, dist_context): cur_op_dist_attr = dist_context.get_op_dist_attr_for_program(cur_op) # insert seed op to guarantee that two dropout op have the same outputs op_unique_name = unique_name.generate("seed") - var_unique_name = unique_name.generate_with_ignorable_key(".".join( - [op_unique_name, 'tmp'])) + var_unique_name = unique_name.generate_with_ignorable_key( + ".".join([op_unique_name, 'tmp']) + ) seed_var = self._block.create_var( name=var_unique_name, dtype='int32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) # set new seed_var's dist_attr ref_dims_mapping = [-1] ref_process_mesh = cur_op_dist_attr.process_mesh - seed_var_dist_attr = set_var_dist_attr(dist_context, seed_var, - ref_dims_mapping, - ref_process_mesh) - - seed = 0 if cur_op.attr("fix_seed") is False else int( - cur_op.attr("seed")) + seed_var_dist_attr = set_var_dist_attr( + dist_context, seed_var, ref_dims_mapping, ref_process_mesh + ) + + seed = ( + 0 + if cur_op.attr("fix_seed") is False + else int(cur_op.attr("seed")) + ) seed_op = self._block._insert_op_without_sync( index=cur_op.idx, type="seed", inputs={}, outputs={"Out": seed_var}, - attrs={ - "seed": seed, - "force_cpu": True - }) + attrs={"seed": seed, "force_cpu": True}, + ) # set new seed op's dist_attr naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - seed_op, ref_process_mesh, ref_dims_mapping, dist_context) + seed_op, ref_process_mesh, ref_dims_mapping, dist_context + ) # modify dropout op's desc self._ops.insert(op_idx, seed_op) cur_op.desc.set_input("Seed", [var_unique_name]) cur_op._remove_attr("fix_seed") cur_op._remove_attr("seed") - cur_op_dist_attr.set_input_dist_attr(seed_var.name, - seed_var_dist_attr) + cur_op_dist_attr.set_input_dist_attr( + seed_var.name, seed_var_dist_attr + ) op_idx += 2 self._block._sync_with_cpp() @@ -166,7 +188,7 @@ def _find_op_index(block, cur_op): def _get_stop_gradients(program, no_grad_set): - """ get no grad var """ + """get no grad var""" if no_grad_set is None: no_grad_set = set() else: @@ -183,8 +205,9 @@ def _get_stop_gradients(program, no_grad_set): return no_grad_set_name -def _add_needed_descs_to_block(descs, block, main_block, in_memory_vars, - dist_context): +def _add_needed_descs_to_block( + descs, block, main_block, in_memory_vars, dist_context +): """ Get the recomputed ops which will insert the backward part """ @@ -215,7 +238,6 @@ def _add_needed_descs_to_block(descs, block, main_block, in_memory_vars, @register_pass("auto_parallel_recompute") class RecomputePass(PassBase): - def __init__(self): super(RecomputePass, self).__init__() self.set_attr("checkpoints", None) @@ -259,12 +281,15 @@ def _apply_single_impl(self, main_program, startup_program, context): vars_should_be_hold = [] for segment in segments: vars_should_be_hold.extend( - rc_state.get_out_of_subgraph_vars(segment[0], segment[1])) + rc_state.get_out_of_subgraph_vars(segment[0], segment[1]) + ) cross_vars = set(vars_should_be_hold) - set(checkpoints) logging.info( "found [{}] vars which cross recompute segment: [{}]," "better checkpoints might be set to reduce those vars".format( - len(cross_vars), cross_vars)) + len(cross_vars), cross_vars + ) + ) vars_should_be_hold.extend(rc_state.get_reserved_vars()) vars_should_be_hold.extend(rc_state.get_input_nodes()) vars_should_be_hold = list(set(vars_should_be_hold)) @@ -275,14 +300,15 @@ def _apply_single_impl(self, main_program, startup_program, context): ckpt_ops_dict = {} buffer_block = main_block.program._create_block() for i, segment in enumerate(segments[::-1]): - fwd_ops = op_path[segment[0]:segment[1]] + fwd_ops = op_path[segment[0] : segment[1]] var_suffix = ".subprog_%d" % i for op in fwd_ops: input_and_output_names = [] input_and_output_names.extend(op.desc.input_arg_names()) input_and_output_names.extend(op.desc.output_arg_names()) - cur_op_dist_attr = self._dist_context.get_op_dist_attr_for_program( - op) + cur_op_dist_attr = ( + self._dist_context.get_op_dist_attr_for_program(op) + ) assert cur_op_dist_attr is not None for name in input_and_output_names: if main_block.var(name).persistable or name in checkpoints: @@ -292,11 +318,13 @@ def _apply_single_impl(self, main_program, startup_program, context): if name not in var_name_dict: ref_process_mesh = cur_op_dist_attr.process_mesh if name in op.desc.input_arg_names(): - ref_dims_mapping = cur_op_dist_attr.get_input_dims_mapping( - name) + ref_dims_mapping = ( + cur_op_dist_attr.get_input_dims_mapping(name) + ) else: - ref_dims_mapping = cur_op_dist_attr.get_output_dims_mapping( - name) + ref_dims_mapping = ( + cur_op_dist_attr.get_output_dims_mapping(name) + ) # record recomputed var's old_name and new_name (old_name.subprog_XXX) # create new var with new name var_name_dict[name] = name + var_suffix @@ -307,15 +335,23 @@ def _apply_single_impl(self, main_program, startup_program, context): dtype=ref_var.dtype, type=ref_var.type, persistable=ref_var.persistable, - stop_gradient=ref_var.stop_gradient) + stop_gradient=ref_var.stop_gradient, + ) # set new recomputed var's dist attr - set_var_dist_attr(self._dist_context, rc_var, - ref_dims_mapping, ref_process_mesh) + set_var_dist_attr( + self._dist_context, + rc_var, + ref_dims_mapping, + ref_process_mesh, + ) # get recomputed segment's descs - segment_descs = _add_needed_descs_to_block(fwd_ops, buffer_block, - main_block, - vars_in_memory, - self._dist_context) + segment_descs = _add_needed_descs_to_block( + fwd_ops, + buffer_block, + main_block, + vars_in_memory, + self._dist_context, + ) # rename recomputed ops' input and output var name for key in var_name_dict: _rename_arg_(segment_descs, key, var_name_dict[key]) @@ -343,7 +379,10 @@ def _apply_single_impl(self, main_program, startup_program, context): # rename grad op's var_name which is not in 'vars_in_memory' for key in var_name_dict: - if key not in grad_op.input_arg_names + grad_op.output_arg_names: + if ( + key + not in grad_op.input_arg_names + grad_op.output_arg_names + ): continue self.reset_op_dist_attr(grad_op, var_name_dict) _rename_arg_([grad_op.desc], key, var_name_dict[key]) @@ -358,17 +397,20 @@ def _apply_single_impl(self, main_program, startup_program, context): idx -= 1 segment_descs = ckpt_ops_dict[fwd_op_id][1] for _, op_desc in reversed(list(enumerate(segment_descs))): - rc_op = main_block._insert_op_without_sync(idx, - type='nop') + rc_op = main_block._insert_op_without_sync( + idx, type='nop' + ) rc_desc = rc_op.desc rc_desc.copy_from(op_desc) rc_desc.set_original_id(rc_desc.id()) # set recomputed ops' dist attr fwd_op_dist_attr = self._dist_context.get_op_dist_attr_for_program_with_id( - op_desc.original_id()) + op_desc.original_id() + ) assert fwd_op_dist_attr is not None - self.set_op_dist_attr(rc_op, fwd_op_dist_attr, - var_name_dict) + self.set_op_dist_attr( + rc_op, fwd_op_dist_attr, var_name_dict + ) ckpt_ops_dict[fwd_op_id][0] = False @@ -380,13 +422,15 @@ def reset_op_dist_attr(self, op, var_name_dict): for input in op.desc.input_arg_names(): if input in var_name_dict.keys(): in_dist_attr = op_dist_attr.get_input_dist_attr(input) - op_dist_attr.set_input_dist_attr(var_name_dict[input], - in_dist_attr) + op_dist_attr.set_input_dist_attr( + var_name_dict[input], in_dist_attr + ) for output in op.desc.output_arg_names(): if output in var_name_dict.keys(): out_dist_attr = op_dist_attr.get_output_dist_attr(output) - op_dist_attr.set_output_dist_attr(var_name_dict[output], - out_dist_attr) + op_dist_attr.set_output_dist_attr( + var_name_dict[output], out_dist_attr + ) def set_op_dist_attr(self, op, old_dist_attr, var_name_dict): new_dist_attr = OperatorDistributedAttribute() @@ -397,16 +441,18 @@ def set_op_dist_attr(self, op, old_dist_attr, var_name_dict): for input in old_dist_attr.inputs_dist_attrs.keys(): if input in var_name_dict.keys(): in_dist_attr = old_dist_attr.inputs_dist_attrs[input] - new_dist_attr.set_input_dist_attr(var_name_dict[input], - in_dist_attr) + new_dist_attr.set_input_dist_attr( + var_name_dict[input], in_dist_attr + ) else: in_dist_attr = old_dist_attr.inputs_dist_attrs[input] new_dist_attr.set_input_dist_attr(input, in_dist_attr) for output in old_dist_attr.outputs_dist_attrs.keys(): if output in var_name_dict.keys(): out_dist_attr = old_dist_attr.outputs_dist_attrs[output] - new_dist_attr.set_output_dist_attr(var_name_dict[output], - out_dist_attr) + new_dist_attr.set_output_dist_attr( + var_name_dict[output], out_dist_attr + ) else: out_dist_attr = old_dist_attr.outputs_dist_attrs[output] new_dist_attr.set_output_dist_attr(output, out_dist_attr) diff --git a/python/paddle/distributed/passes/auto_parallel_sharding.py b/python/paddle/distributed/passes/auto_parallel_sharding.py index 171188618efe2..5126250659866 100644 --- a/python/paddle/distributed/passes/auto_parallel_sharding.py +++ b/python/paddle/distributed/passes/auto_parallel_sharding.py @@ -17,27 +17,51 @@ from paddle.framework import core from paddle.fluid import unique_name from .pass_base import PassBase, register_pass -from paddle.distributed.fleet.meta_optimizers.common import is_backward_op, is_optimizer_op +from paddle.distributed.fleet.meta_optimizers.common import ( + is_backward_op, + is_optimizer_op, +) from paddle.distributed.auto_parallel.process_group import new_process_group -from paddle.distributed.auto_parallel.operators.common import is_parameter_related, is_data_parallel_reduce_op -from paddle.distributed.auto_parallel.utils import _get_comm_group, naive_set_dist_op_attr_for_program_by_mesh_and_mapping, set_var_dist_attr +from paddle.distributed.auto_parallel.operators.common import ( + is_parameter_related, + is_data_parallel_reduce_op, +) +from paddle.distributed.auto_parallel.utils import ( + _get_comm_group, + naive_set_dist_op_attr_for_program_by_mesh_and_mapping, + set_var_dist_attr, +) OpRole = core.op_proto_and_checker_maker.OpRole OP_ROLE_KEY = core.op_proto_and_checker_maker.kOpRoleAttrName() _skip_ops = [ - 'create_py_reader', 'create_double_buffer_reader', 'read', 'slice', 'split', - 'assign', "send_v2" + 'create_py_reader', + 'create_double_buffer_reader', + 'read', + 'slice', + 'split', + 'assign', + "send_v2", ] # update here to support new optimizers _supported_optimizer_type = [ - "adam", "adamax", "adamw", "decayed_adagrad", "momentum", "dgc_momentum", - "lars_momentum", "merged_momentum", "lamb", "sgd" + "adam", + "adamax", + "adamw", + "decayed_adagrad", + "momentum", + "dgc_momentum", + "lars_momentum", + "merged_momentum", + "lamb", + "sgd", ] def _is_reshard_op(op): - return op.desc.has_attr("op_namescope") and \ - "/auto_parallel/reshard" in op.desc.attr('op_namescope') + return op.desc.has_attr( + "op_namescope" + ) and "/auto_parallel/reshard" in op.desc.attr('op_namescope') # NOTE we add the "auto_parallel" prefix to the pass in order to @@ -46,7 +70,6 @@ def _is_reshard_op(op): # should use dist op instead of custom comm op @register_pass("auto_parallel_sharding") class ShardingPass(PassBase): - def __init__(self): super(ShardingPass, self).__init__() self.set_attr("dist_context", None) @@ -69,19 +92,22 @@ def _check_self(self): if self.get_attr("stage") not in [1, 2, 3]: return False if self.get_attr("sharding_degree") is not None: - if (not isinstance(self.get_attr("sharding_degree"), int)) \ - or self.get_attr("sharding_degree") <= 1: + if ( + not isinstance(self.get_attr("sharding_degree"), int) + ) or self.get_attr("sharding_degree") <= 1: return False elif self.get_attr("degree") is not None: - if (not isinstance(self.get_attr("degree"), int)) \ - or self.get_attr("degree") <= 1: + if (not isinstance(self.get_attr("degree"), int)) or self.get_attr( + "degree" + ) <= 1: return False else: return False if len(self.get_attr("params_grads")) <= 0: return False - if (not isinstance(self.get_attr("global_rank"), - int)) or self.get_attr("global_rank") < 0: + if (not isinstance(self.get_attr("global_rank"), int)) or self.get_attr( + "global_rank" + ) < 0: return False return True @@ -92,12 +118,15 @@ def _check_conflict(self, other_pass): def _apply_single_impl(self, main_program, startup_program, context): self._dist_context = self.get_attr("dist_context") self.sharding_world_size = int( - self.get_attr("sharding_degree") or self.get_attr("degree")) + self.get_attr("sharding_degree") or self.get_attr("degree") + ) self.stage = int(self.get_attr("stage")) self.global_rank = int(self.get_attr("global_rank")) params_grads = self.get_attr("params_grads") - main_block, startup_block = main_program.global_block( - ), startup_program.global_block() + main_block, startup_block = ( + main_program.global_block(), + startup_program.global_block(), + ) self._build_sharding_groups(main_block, params_grads) self._shard_optimizer(main_block, startup_block, params_grads, context) @@ -119,7 +148,8 @@ def _collective_data_parallel_groups(self, main_block): if _is_reshard_op(op): continue group = _inference_data_parallel_group_for_operator( - self.global_rank, op, self._dist_context) + self.global_rank, op, self._dist_context + ) if group is not None: self.dp_groups.add(group) @@ -127,32 +157,45 @@ def _collective_data_parallel_groups(self, main_block): # genetated by auto search if len(self.dp_groups) != 1: raise NotImplementedError( - "So far Only and Exactly one data parallel group in network are supported, but got [{}] different data parallel groups" - .format(len(self.dp_groups))) + "So far Only and Exactly one data parallel group in network are supported, but got [{}] different data parallel groups".format( + len(self.dp_groups) + ) + ) def _build_sharding_infos(self, params_grads): for dp_group in self.dp_groups: - assert dp_group.nranks >= self.sharding_world_size, "sharding world size [{}] should not larger than dp world size [{}]".format( - self.sharding_world_size, dp_group.nranks) - assert dp_group.nranks % self.sharding_world_size == 0, "sharding world size [{}] should be divisible by dp world size [{}]".format( - self.sharding_world_size, dp_group.nranks) - assert self.global_rank in dp_group.ranks, "current ranks [{}] does NOT belong to the data parallel group [{}]".format( - self.global_rank, dp_group.ranks) - assert len( - params_grads - ) >= self.sharding_world_size, "number of parameters [{}] is not enough to be shard among [{}] ranks".format( - len(params_grads), self.sharding_world_size) + assert ( + dp_group.nranks >= self.sharding_world_size + ), "sharding world size [{}] should not larger than dp world size [{}]".format( + self.sharding_world_size, dp_group.nranks + ) + assert ( + dp_group.nranks % self.sharding_world_size == 0 + ), "sharding world size [{}] should be divisible by dp world size [{}]".format( + self.sharding_world_size, dp_group.nranks + ) + assert ( + self.global_rank in dp_group.ranks + ), "current ranks [{}] does NOT belong to the data parallel group [{}]".format( + self.global_rank, dp_group.ranks + ) + assert ( + len(params_grads) >= self.sharding_world_size + ), "number of parameters [{}] is not enough to be shard among [{}] ranks".format( + len(params_grads), self.sharding_world_size + ) # sharding hybrid data parallel: partial sharding param within if dp_group.nranks > self.sharding_world_size: self.partial_sharding = True - assert len( - self.dp_groups - ) == 1, "hybrid sharding and data parallelism are supported only when there is excatly one data parallel group in the network" + assert ( + len(self.dp_groups) == 1 + ), "hybrid sharding and data parallelism are supported only when there is excatly one data parallel group in the network" outer_dp_group, sharding_group = _get_dp_and_sharding_groups( - dp_group.ranks, self.sharding_world_size, self.global_rank) + dp_group.ranks, self.sharding_world_size, self.global_rank + ) sharding_group = new_process_group(sharding_group) self.outer_dp_group = new_process_group(outer_dp_group) else: @@ -160,14 +203,16 @@ def _build_sharding_infos(self, params_grads): self._dist_context._sharding_group = sharding_group # TODO(JZ-LIANG) when support multiple dp groups in future, should group param and bind them to corresponding dp group - sharding_info = ShardingInfo(sharding_group, self.global_rank, - params_grads) + sharding_info = ShardingInfo( + sharding_group, self.global_rank, params_grads + ) self.sharding_infos.append(sharding_info) for param in sharding_info.params: self.varname_to_sharding_info[param.name] = sharding_info - def _shard_optimizer(self, main_block, startup_block, params_grads, - pass_context): + def _shard_optimizer( + self, main_block, startup_block, params_grads, pass_context + ): """ sharding all optimizer related ops and vars, include: gradient clip ops & vars @@ -189,7 +234,7 @@ def _shard_amp_related_op_and_vars(self, main_block, pass_context): # shard amp related param_grad cast if _is_param_grad_fp32_cast_op(main_block, op): output_name = op.output_arg_names[0] - param_name = output_name[:output_name.find("@")] + param_name = output_name[: output_name.find("@")] if not self._is_parameter_in_local_shard(param_name): main_block._remove_op(idx, sync=False) main_block._remove_var(output_name, sync=False) @@ -198,7 +243,7 @@ def _shard_amp_related_op_and_vars(self, main_block, pass_context): elif op.type in ["check_finite_and_unscale", "update_loss_scaling"]: reversed_x = [] for input_name in op.desc.input('X'): - param_name = input_name[:input_name.find("@")] + param_name = input_name[: input_name.find("@")] if self._is_parameter_in_local_shard(param_name): reversed_x.append(input_name) @@ -223,7 +268,8 @@ def _shard_amp_related_op_and_vars(self, main_block, pass_context): "dtype": out_var.dtype, "value": 0, OP_ROLE_KEY: op_role, - }) + }, + ) else: main_block._remove_op(idx, sync=False) @@ -245,7 +291,7 @@ def _shard_gradient_clip(self, main_block): if op.type in removed_op_type: input_name = op.input("X")[0] - param_name = input_name[:input_name.find("@GRAD")] + param_name = input_name[: input_name.find("@GRAD")] if not self._is_parameter_in_local_shard(param_name): removed_op_idx.add(idx) if op.type in ['squared_l2_norm', 'clip_by_norm']: @@ -283,9 +329,13 @@ def _shard_gradient_clip(self, main_block): 'op_namescope': "/gradient_clip_model_parallelism", 'use_calc_stream': True, OP_ROLE_KEY: OpRole.Optimize, - }) - dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - main_block.var(sum_op_output)) + }, + ) + dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + main_block.var(sum_op_output) + ) + ) # assert dist_attr is not None # naive_set_dist_op_attr_for_program_by_mesh_and_mapping( # new_op, dist_attr.process_mesh, dist_attr.dims_mapping, @@ -304,7 +354,8 @@ def _shard_weight_decay(self, main_block): continue else: raise NotImplementedError( - "weight decay is NOT supported by now") + "weight decay is NOT supported by now" + ) main_block._sync_with_cpp() def _shard_optimizer_ops_and_states(self, main_block, startup_block): @@ -319,18 +370,24 @@ def _shard_optimizer_ops_and_states(self, main_block, startup_block): assert len(op.input("Param")) == 1 param_name = op.input("Param")[0] if not self._is_parameter_in_local_shard(param_name): - should_removed_optimizer_states.extend([ - varname for varname in op.output_arg_names - if varname != param_name - ]) + should_removed_optimizer_states.extend( + [ + varname + for varname in op.output_arg_names + if varname != param_name + ] + ) main_block._remove_op(idx, sync=False) else: self.shared_params_grads.append( - self._get_param_grad(param_name)) + self._get_param_grad(param_name) + ) for idx, op in reversed(list(enumerate(startup_block.ops))): - if len(op.output_arg_names) == 1 and op.output_arg_names[ - 0] in should_removed_optimizer_states: + if ( + len(op.output_arg_names) == 1 + and op.output_arg_names[0] in should_removed_optimizer_states + ): startup_block._remove_op(idx, sync=False) for varname in should_removed_optimizer_states: @@ -352,26 +409,27 @@ def _insert_optimizer_broadcasts(self, main_block, startup_block): assert main_block.has_var(param.name) assert startup_block.has_var(param.name) - new_op = main_block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': - sharding_info.group.id, - 'root': - sharding_info.get_var_rank( - param.name), - 'use_calc_stream': - True, - OP_ROLE_KEY: - OpRole.Optimize - }) - param_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - param) + new_op = main_block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': sharding_info.group.id, + 'root': sharding_info.get_var_rank(param.name), + 'use_calc_stream': True, + OP_ROLE_KEY: OpRole.Optimize, + }, + ) + param_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program(param) + ) assert param_dist_attr is not None naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - new_op, param_dist_attr.process_mesh, - param_dist_attr.dims_mapping, self._dist_context) + new_op, + param_dist_attr.process_mesh, + param_dist_attr.dims_mapping, + self._dist_context, + ) main_block._sync_with_cpp() def _is_parameter_in_local_shard(self, param_name): @@ -397,12 +455,18 @@ def _shard_gradient_synchronization(self, main_block): input_name = op.input_arg_names[0] base_name = _get_base_name_from_grad_name(input_name) sharding_info = self.varname_to_sharding_info[base_name] - _insert_reduce_op(main_block, idx, input_name, - sharding_info.group.id, - sharding_info.get_var_rank(base_name), - self._dist_context) - if not self.partial_sharding or not sharding_info.is_in_local_shard( - base_name): + _insert_reduce_op( + main_block, + idx, + input_name, + sharding_info.group.id, + sharding_info.get_var_rank(base_name), + self._dist_context, + ) + if ( + not self.partial_sharding + or not sharding_info.is_in_local_shard(base_name) + ): main_block._remove_op(idx + 1, sync=False) else: op._set_attr("ring_id", self.outer_dp_group.id) @@ -427,12 +491,17 @@ def _shard_parameter(self, main_block, startup_block): dp_ring_ids = [group.id for group in self.dp_groups] for sharding_info in self.sharding_infos: - need_broadcast_vars, param_usage = sharding_info.get_broadcast_vars_and_param_usage( - main_block) + ( + need_broadcast_vars, + param_usage, + ) = sharding_info.get_broadcast_vars_and_param_usage(main_block) not_used_param_nane = [] for param_name in param_usage: - if param_usage[param_name] == 0 and sharding_info.get_var_rank( - param_name) != sharding_info.local_rank: + if ( + param_usage[param_name] == 0 + and sharding_info.get_var_rank(param_name) + != sharding_info.local_rank + ): not_used_param_nane.append(param_name) for idx, op in reversed(list(enumerate(main_block.ops))): @@ -442,8 +511,9 @@ def _shard_parameter(self, main_block, startup_block): for input_name in op.desc.input_arg_names(): # NOTE hack for embedding op when AMP 02-3 # paddle amp force embedding (lookup table) to be run on fp32 - if _is_param_fp16_cast_op(main_block, op, - sharding_info.param_names): + if _is_param_fp16_cast_op( + main_block, op, sharding_info.param_names + ): continue if input_name not in need_broadcast_vars: continue @@ -451,28 +521,39 @@ def _shard_parameter(self, main_block, startup_block): if root_rank == sharding_info.local_rank: broadcast_varname = input_name else: - broadcast_varname = unique_name.generate(input_name + - "@BroadCast") + broadcast_varname = unique_name.generate( + input_name + "@BroadCast" + ) input_var = main_block.var(input_name) - new_var = main_block.create_var(name=broadcast_varname, - shape=input_var.shape, - dtype=input_var.dtype, - persistable=False) - ref_dist_attr = self._dist_context.get_tensor_dist_attr_for_program( - input_var) + new_var = main_block.create_var( + name=broadcast_varname, + shape=input_var.shape, + dtype=input_var.dtype, + persistable=False, + ) + ref_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program( + input_var + ) + ) out_var_dist_attr = set_var_dist_attr( - self._dist_context, new_var, + self._dist_context, + new_var, ref_dist_attr.dims_mapping, - ref_dist_attr.process_mesh) + ref_dist_attr.process_mesh, + ) op._rename_input(input_name, broadcast_varname) - _insert_init_and_broadcast_op(main_block, idx, - broadcast_varname, - sharding_info.local_rank, - root_rank, - sharding_info.group.id, - op.attr('op_role'), - self._dist_context) + _insert_init_and_broadcast_op( + main_block, + idx, + broadcast_varname, + sharding_info.local_rank, + root_rank, + sharding_info.group.id, + op.attr('op_role'), + self._dist_context, + ) for idx, op in reversed(list(enumerate(main_block.ops))): if op.type != "cast": @@ -487,22 +568,33 @@ def _shard_parameter(self, main_block, startup_block): assert len(op.output_arg_names) == 1 output_name = op.output_arg_names[0] - if op.type == "c_broadcast" and op.attr( - "ring_id") in dp_ring_ids: - if self.outer_dp_group and sharding_info.get_var_rank( - output_name) == sharding_info.local_rank: + if ( + op.type == "c_broadcast" + and op.attr("ring_id") in dp_ring_ids + ): + if ( + self.outer_dp_group + and sharding_info.get_var_rank(output_name) + == sharding_info.local_rank + ): op._set_attr("ring_id", self.outer_dp_group.id) else: startup_block._remove_op(idx, sync=False) continue - if op.type != "c_broadcast" and output_name in param_usage and sharding_info.get_var_rank( - output_name) != sharding_info.local_rank: + if ( + op.type != "c_broadcast" + and output_name in param_usage + and sharding_info.get_var_rank(output_name) + != sharding_info.local_rank + ): startup_block._remove_op(idx, sync=False) for param_name in param_usage: - if sharding_info.get_var_rank( - param_name) != sharding_info.local_rank: + if ( + sharding_info.get_var_rank(param_name) + != sharding_info.local_rank + ): main_block._remove_var(param_name, sync=False) startup_block._remove_var(param_name, sync=False) @@ -510,28 +602,42 @@ def _shard_parameter(self, main_block, startup_block): startup_block._sync_with_cpp() -def _insert_init_and_broadcast_op(block, insert_idx, varname, local_rank, - root_rank, ring_id, op_role, dist_context): +def _insert_init_and_broadcast_op( + block, + insert_idx, + varname, + local_rank, + root_rank, + ring_id, + op_role, + dist_context, +): """ empty op for initialization """ broadcast_var = block.var(varname) broadcast_var_dist_attr = dist_context.get_tensor_dist_attr_for_program( - broadcast_var) - - new_op = block._insert_op_without_sync(insert_idx, - type='c_broadcast', - inputs={'X': varname}, - outputs={'Out': varname}, - attrs={ - 'ring_id': ring_id, - 'root': root_rank, - 'use_calc_stream': True, - OP_ROLE_KEY: op_role - }) + broadcast_var + ) + + new_op = block._insert_op_without_sync( + insert_idx, + type='c_broadcast', + inputs={'X': varname}, + outputs={'Out': varname}, + attrs={ + 'ring_id': ring_id, + 'root': root_rank, + 'use_calc_stream': True, + OP_ROLE_KEY: op_role, + }, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - new_op, broadcast_var_dist_attr.process_mesh, - broadcast_var_dist_attr.dims_mapping, dist_context) + new_op, + broadcast_var_dist_attr.process_mesh, + broadcast_var_dist_attr.dims_mapping, + dist_context, + ) if local_rank != root_rank: new_op = block._insert_op_without_sync( @@ -541,40 +647,50 @@ def _insert_init_and_broadcast_op(block, insert_idx, varname, local_rank, attrs={ "shape": broadcast_var.shape, "dtype": broadcast_var.dtype, - OP_ROLE_KEY: op_role - }) + OP_ROLE_KEY: op_role, + }, + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - new_op, broadcast_var_dist_attr.process_mesh, - broadcast_var_dist_attr.dims_mapping, dist_context) + new_op, + broadcast_var_dist_attr.process_mesh, + broadcast_var_dist_attr.dims_mapping, + dist_context, + ) return -def _insert_reduce_op(block, - insert_idx, - reduce_var, - ring_id, - root_id, - dist_context, - op_role=OpRole.Backward, - use_calc_stream=True): - assert root_id >= 0, "root id should be a positive int, but now root id is {}".format( - root_id) - new_op = block._insert_op_without_sync(insert_idx, - type='c_reduce_sum', - inputs={'X': [reduce_var]}, - outputs={'Out': [reduce_var]}, - attrs={ - 'ring_id': ring_id, - 'root_id': root_id, - 'use_calc_stream': - use_calc_stream, - OP_ROLE_KEY: op_role - }) +def _insert_reduce_op( + block, + insert_idx, + reduce_var, + ring_id, + root_id, + dist_context, + op_role=OpRole.Backward, + use_calc_stream=True, +): + assert ( + root_id >= 0 + ), "root id should be a positive int, but now root id is {}".format(root_id) + new_op = block._insert_op_without_sync( + insert_idx, + type='c_reduce_sum', + inputs={'X': [reduce_var]}, + outputs={'Out': [reduce_var]}, + attrs={ + 'ring_id': ring_id, + 'root_id': root_id, + 'use_calc_stream': use_calc_stream, + OP_ROLE_KEY: op_role, + }, + ) dist_attr = dist_context.get_tensor_dist_attr_for_program( - block.var(reduce_var)) + block.var(reduce_var) + ) naive_set_dist_op_attr_for_program_by_mesh_and_mapping( - new_op, dist_attr.process_mesh, dist_attr.dims_mapping, dist_context) + new_op, dist_attr.process_mesh, dist_attr.dims_mapping, dist_context + ) def _get_dp_and_sharding_groups(origin_group, sharding_group_size, rank): @@ -589,23 +705,26 @@ def _get_dp_and_sharding_groups(origin_group, sharding_group_size, rank): def _is_gradient_clip_op(op): - return op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/gradient_clip") + return op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/gradient_clip") def _is_weight_decay_op(op): - return op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/regularization") + return op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/regularization") def _is_param_grad_fp32_cast_op(block, op): if not is_backward_op(op): return False - if not _is_desired_cast_op(block, op, core.VarDesc.VarType.FP16, - core.VarDesc.VarType.FP32): + if not _is_desired_cast_op( + block, op, core.VarDesc.VarType.FP16, core.VarDesc.VarType.FP32 + ): return False output_name = op.desc.output_arg_names()[0] - base_name = output_name[:output_name.find("@")] + base_name = output_name[: output_name.find("@")] if not block.has_var(base_name): return False return block.var(base_name).is_parameter @@ -623,19 +742,20 @@ def _is_param_fp16_cast_op(block, op, params): return True -def _is_desired_cast_op(block, - op, - src_var_type=core.VarDesc.VarType.FP32, - dst_var_type=core.VarDesc.VarType.FP16): +def _is_desired_cast_op( + block, + op, + src_var_type=core.VarDesc.VarType.FP32, + dst_var_type=core.VarDesc.VarType.FP16, +): if op.type != "cast": return False - assert (len(op.desc.input_arg_names()) == 1) - assert (len(op.desc.output_arg_names()) == 1) + assert len(op.desc.input_arg_names()) == 1 + assert len(op.desc.output_arg_names()) == 1 input_var = block.var(op.desc.input_arg_names()[0]) output_var = block.var(op.desc.output_arg_names()[0]) - if input_var.dtype != src_var_type or \ - output_var.dtype != dst_var_type: + if input_var.dtype != src_var_type or output_var.dtype != dst_var_type: return False return True @@ -644,9 +764,9 @@ def _is_desired_cast_op(block, def _get_base_name_from_grad_name(grad_name): base_name = None if ".cast_fp16@GRAD" in grad_name: - base_name = grad_name[:grad_name.find(".cast_fp16@GRAD")] + base_name = grad_name[: grad_name.find(".cast_fp16@GRAD")] elif "@GRAD" in grad_name: - base_name = grad_name[:grad_name.find("@GRAD")] + base_name = grad_name[: grad_name.find("@GRAD")] return base_name @@ -696,9 +816,12 @@ def _inference_data_parallel_group_for_operator(rank_id, op, dist_context): # TODO(JZ-LIANG) replace with specific batch size dimension batch_size_axis = input_dim_mapping[0] if batch_size_axis > -1 and mesh_shape[batch_size_axis] > 1: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, - batch_size_axis, rank_id) + group_ranks = _get_comm_group( + process_mesh.processes, + process_mesh.topology, + batch_size_axis, + rank_id, + ) dp_group = new_process_group(group_ranks) break @@ -717,20 +840,23 @@ def shard_parameters(params, group_size): rank = sizes.index(min(sizes)) mapping[rank].append(param) numel = reduce(lambda x, y: x * y, param.shape) - assert numel > 0, "param [{}] should larger than 0, but it is [{}]".format( - param.name, numel) + assert ( + numel > 0 + ), "param [{}] should larger than 0, but it is [{}]".format( + param.name, numel + ) sizes[rank] += numel return mapping class ShardingInfo(object): - def __init__(self, group, rank, params_grads): self.group = group self.params_grads = dict([(p.name, (p, g)) for p, g in params_grads]) - assert len(self.params_grads) == len(set( - self.params_grads)), "found duplicated param in params_grads" + assert len(self.params_grads) == len( + set(self.params_grads) + ), "found duplicated param in params_grads" self.params = [p for p, _ in params_grads] self.param_names = [p.name for p in self.params] @@ -796,7 +922,8 @@ def get_broadcast_vars_and_param_usage(self, block): def get_param_grad(self, param_name): if not self.is_in_local_shard(param_name): raise ValueError( - "param[{}] not in current rank.".format(param_name)) + "param[{}] not in current rank.".format(param_name) + ) if param_name not in self.params_grads: raise ValueError('param[{}] not in params_grads'.format(param_name)) return self.params_grads.get(param_name, None) diff --git a/python/paddle/distributed/passes/cpp_pass.py b/python/paddle/distributed/passes/cpp_pass.py index c729a919c1a33..17d54186dadb8 100644 --- a/python/paddle/distributed/passes/cpp_pass.py +++ b/python/paddle/distributed/passes/cpp_pass.py @@ -19,7 +19,6 @@ @register_pass("fuse_elewise_add_act") class FuseElementwiseAddActPass(CPPPassWrapper): - def __init__(self): super(FuseElementwiseAddActPass, self).__init__() @@ -33,7 +32,6 @@ def _type(self): @register_pass("fuse_bn_act") class FuseBatchNormActPass(CPPPassWrapper): - def __init__(self): super(FuseBatchNormActPass, self).__init__() @@ -47,7 +45,6 @@ def _type(self): @register_pass("fuse_bn_add_act") class FuseBatchNormAddActPass(CPPPassWrapper): - def __init__(self): super(FuseBatchNormAddActPass, self).__init__() @@ -61,7 +58,6 @@ def _type(self): @register_pass("fuse_relu_depthwise_conv") class FuseReluDepthwiseConvPass(CPPPassWrapper): - def __init__(self): super(FuseReluDepthwiseConvPass, self).__init__() @@ -75,14 +71,15 @@ def _type(self): @register_pass("fuse_optimizer") class FuseOptimizerPass(CPPPassWrapper): - def __init__(self): super(FuseOptimizerPass, self).__init__() @property def cpp_name(self): return [ - "fuse_adam_op_pass", "fuse_sgd_op_pass", "fuse_momentum_op_pass" + "fuse_adam_op_pass", + "fuse_sgd_op_pass", + "fuse_momentum_op_pass", ] def _type(self): @@ -91,7 +88,6 @@ def _type(self): @register_pass("inplace_addto_op") class InplaceAddtoOpPass(CPPPassWrapper): - def __init__(self): super(InplaceAddtoOpPass, self).__init__() @@ -112,7 +108,6 @@ def _set_cinn_op_flag(flag_name, extra_ops): @register_pass("build_cinn") class BuildCINNPass(CPPPassWrapper): - def __init__(self): super(BuildCINNPass, self).__init__() self.set_attr("allow_ops", []) @@ -127,13 +122,15 @@ def _type(self): def _apply_single_impl(self, main_program, startup_program, context): - assert 'FLAGS_allow_cinn_ops' in core.globals( + assert ( + 'FLAGS_allow_cinn_ops' in core.globals() ), "PaddlePaddle is not compiled with CINN support" old_allow_ops = core.globals()['FLAGS_allow_cinn_ops'] old_deny_ops = core.globals()['FLAGS_deny_cinn_ops'] try: - _set_cinn_op_flag('FLAGS_allow_cinn_ops', - self.get_attr("allow_ops")) + _set_cinn_op_flag( + 'FLAGS_allow_cinn_ops', self.get_attr("allow_ops") + ) _set_cinn_op_flag('FLAGS_deny_cinn_ops', self.get_attr("deny_ops")) feed = self.get_attr('feed', []) @@ -142,18 +139,26 @@ def _apply_single_impl(self, main_program, startup_program, context): if prune_program: tmp_main_program = Executor._prune_program( - main_program, feed, fetch_list, []) + main_program, feed, fetch_list, [] + ) tmp_main_program = Executor._add_fetch_ops( - tmp_main_program, fetch_list, 'fetch') + tmp_main_program, fetch_list, 'fetch' + ) else: tmp_main_program = Executor._add_fetch_ops( - main_program, fetch_list, 'fetch') - - _apply_cpp_pass(tmp_main_program, startup_program, self.cpp_name, - {}, self.cpp_attr_types) + main_program, fetch_list, 'fetch' + ) + + _apply_cpp_pass( + tmp_main_program, + startup_program, + self.cpp_name, + {}, + self.cpp_attr_types, + ) tmp_main_program = Executor._remove_fetch_ops(tmp_main_program) diff --git a/python/paddle/distributed/passes/fuse_all_reduce.py b/python/paddle/distributed/passes/fuse_all_reduce.py index 628caa0696a96..dcd1976c89c61 100644 --- a/python/paddle/distributed/passes/fuse_all_reduce.py +++ b/python/paddle/distributed/passes/fuse_all_reduce.py @@ -18,9 +18,9 @@ import numpy as np -def find_adjacent_match_sequences(iterable, - filter_func, - adjacent_filter_func=None): +def find_adjacent_match_sequences( + iterable, filter_func, adjacent_filter_func=None +): n = len(iterable) match_sequences = [] if adjacent_filter_func is None: @@ -30,8 +30,11 @@ def find_adjacent_match_sequences(iterable, while i < n and not filter_func(iterable[i]): i += 1 j = i + 1 - while j < n and filter_func(iterable[j]) and adjacent_filter_func( - iterable[i], iterable[j]): + while ( + j < n + and filter_func(iterable[j]) + and adjacent_filter_func(iterable[i], iterable[j]) + ): j += 1 if i < n and j <= n: match_sequences.append((i, j)) @@ -41,11 +44,13 @@ def find_adjacent_match_sequences(iterable, return match_sequences -def insert_fuse_all_reduce_ops(block, reversed_op_indices, input_var_names, - output_var_names, dtype, attrs): - fused_var = block.create_var(name=unique_name.generate( - "FusedOutput_{}".format(input_var_names[0])), - dtype=dtype) +def insert_fuse_all_reduce_ops( + block, reversed_op_indices, input_var_names, output_var_names, dtype, attrs +): + fused_var = block.create_var( + name=unique_name.generate("FusedOutput_{}".format(input_var_names[0])), + dtype=dtype, + ) # FIXME(zengjinle): here we assume that we use # c_sync_calc_stream/c_sync_comm_stream to do sync. @@ -56,16 +61,20 @@ def insert_fuse_all_reduce_ops(block, reversed_op_indices, input_var_names, for i, op_idx in enumerate(reversed_op_indices): prev_op_idx = op_idx - 1 - while prev_op_idx >= 0 and block.ops[ - prev_op_idx].type == "c_sync_calc_stream": + while ( + prev_op_idx >= 0 + and block.ops[prev_op_idx].type == "c_sync_calc_stream" + ): new_op_indices.append(prev_op_idx) prev_op_idx -= 1 if i > 0: next_op_idx = op_idx + 1 n = len(block.ops) - while next_op_idx < n and block.ops[ - next_op_idx].type == "c_sync_comm_stream": + while ( + next_op_idx < n + and block.ops[next_op_idx].type == "c_sync_comm_stream" + ): assert block.ops[next_op_idx].attr("ring_id") == ring_id new_op_indices.append(next_op_idx) @@ -102,21 +111,22 @@ def insert_fuse_all_reduce_ops(block, reversed_op_indices, input_var_names, } if not attrs["use_calc_stream"]: - block._insert_op_without_sync(insert_idx, - type="c_sync_calc_stream", - inputs={"X": fused_var}, - outputs={ - "Out": fused_var, - op_role_key: attrs[op_role_key] - }) + block._insert_op_without_sync( + insert_idx, + type="c_sync_calc_stream", + inputs={"X": fused_var}, + outputs={"Out": fused_var, op_role_key: attrs[op_role_key]}, + ) insert_idx += 1 # c_allreduce_sum should insert - block._insert_op_without_sync(insert_idx, - type="c_allreduce_sum", - inputs={"X": fused_var}, - outputs={"Out": fused_var}, - attrs=attrs) + block._insert_op_without_sync( + insert_idx, + type="c_allreduce_sum", + inputs={"X": fused_var}, + outputs={"Out": fused_var}, + attrs=attrs, + ) for op_idx in reversed_op_indices: block._remove_op(op_idx) @@ -185,9 +195,9 @@ def is_same_adjacent_op(ref_op, new_op): return False return True - match_seqs = find_adjacent_match_sequences(collective_ops, - is_valid_allreduce_op, - is_same_adjacent_op) + match_seqs = find_adjacent_match_sequences( + collective_ops, is_valid_allreduce_op, is_same_adjacent_op + ) new_match_seqs = [] for i, j in match_seqs: new_match_seqs.append([collective_op_indices[k] for k in range(i, j)]) @@ -301,8 +311,13 @@ def insert_fuse_all_reduce_by_memory_size(block, groups, max_memory_size): if len(recorded_op_indices) > 1: attrs[op_role_var_key] = op_role_vars coalesce_op_kwargs = insert_fuse_all_reduce_ops( - block, recorded_op_indices, in_var_names, out_var_names, - dtype, attrs) + block, + recorded_op_indices, + in_var_names, + out_var_names, + dtype, + attrs, + ) coalesce_ops_kwargs.append(coalesce_op_kwargs) cur_mem_size = 0 @@ -321,8 +336,13 @@ def insert_fuse_all_reduce_by_memory_size(block, groups, max_memory_size): if len(recorded_op_indices) > 1: attrs[op_role_var_key] = op_role_vars coalesce_op_kwargs = insert_fuse_all_reduce_ops( - block, recorded_op_indices, in_var_names, out_var_names, dtype, - attrs) + block, + recorded_op_indices, + in_var_names, + out_var_names, + dtype, + attrs, + ) coalesce_ops_kwargs.append(coalesce_op_kwargs) block._sync_with_cpp() insert_coalesce_tensor_ops(block, coalesce_ops_kwargs) @@ -330,7 +350,6 @@ def insert_fuse_all_reduce_by_memory_size(block, groups, max_memory_size): @register_pass("fuse_all_reduce") class FuseAllReducePass(PassBase): - def __init__(self): super(FuseAllReducePass, self).__init__() self.set_attr("max_memory_size", -1) @@ -358,7 +377,9 @@ def _apply_single_impl(self, main_program, startup_program, context): block = main_program.block(i) groups = find_all_fuse_all_reduce_groups(block) groups = split_fuse_all_reduce_groups_by_deps( - block, groups, op_deps[i]) - insert_fuse_all_reduce_by_memory_size(block, groups, - max_memory_size) + block, groups, op_deps[i] + ) + insert_fuse_all_reduce_by_memory_size( + block, groups, max_memory_size + ) main_program._sync_with_cpp() diff --git a/python/paddle/distributed/passes/pass_base.py b/python/paddle/distributed/passes/pass_base.py index e042ce4a15e7d..56cc262ca0310 100644 --- a/python/paddle/distributed/passes/pass_base.py +++ b/python/paddle/distributed/passes/pass_base.py @@ -17,7 +17,6 @@ class PassContext: - def __init__(self): self._applied_passes = [] self._attrs = {} @@ -84,7 +83,8 @@ def _type(self): def _check_conflict_including_common_rules(self, other_pass): return self._check_conflict(other_pass) and all( - [r(other_pass, self) for r in PassBase._COMMON_RULES]) + [r(other_pass, self) for r in PassBase._COMMON_RULES] + ) def apply(self, main_programs, startup_programs, context=None): if context is None: @@ -93,10 +93,12 @@ def apply(self, main_programs, startup_programs, context=None): if not self._check_self(): return context - if not all([ + if not all( + [ self._check_conflict_including_common_rules(p) for p in context.passes - ]): + ] + ): return context assert isinstance(main_programs, list) @@ -107,8 +109,9 @@ def apply(self, main_programs, startup_programs, context=None): return context def _apply_impl(self, main_programs, startup_programs, context): - for main_program, startup_program in zip(main_programs, - startup_programs): + for main_program, startup_program in zip( + main_programs, startup_programs + ): self._apply_single_impl(main_program, startup_program, context) @abstractmethod @@ -117,7 +120,6 @@ def _apply_single_impl(self, main_program, startup_program, context): def register_pass(name): - def impl(cls): PassBase._register(name, cls) cls.name = name @@ -136,7 +138,6 @@ def new_pass(name, pass_attrs={}): class CPPPassWrapper(PassBase): - def __init__(self): super(CPPPassWrapper, self).__init__() @@ -155,21 +156,28 @@ def _check_conflict(self, other_pass): return True def _apply_single_impl(self, main_program, startup_program, context): - _apply_cpp_pass(main_program, startup_program, self.cpp_name, - self._attrs, self.cpp_attr_types) + _apply_cpp_pass( + main_program, + startup_program, + self.cpp_name, + self._attrs, + self.cpp_attr_types, + ) def _fusion_opt_last_rule(pass_before, pass_after): - if pass_before._type( - ) == PassType.FUSION_OPT and pass_after._type() != PassType.FUSION_OPT: + if ( + pass_before._type() == PassType.FUSION_OPT + and pass_after._type() != PassType.FUSION_OPT + ): return False else: return True -def _make_rule_from_white_lists_dict(before_white_lists_dict, - after_white_lists_dict): - +def _make_rule_from_white_lists_dict( + before_white_lists_dict, after_white_lists_dict +): def collect_pass_names(white_lists_dict, result): for k, v in white_lists_dict.items(): result.add(k) @@ -196,7 +204,10 @@ def collect_pass_names(white_lists_dict, result): def rule(pass_before, pass_after): all_passes_after = compatible_pass_dict.get(pass_before.name) - if all_passes_after is None or pass_after.name not in compatible_pass_dict: + if ( + all_passes_after is None + or pass_after.name not in compatible_pass_dict + ): return True else: return pass_after.name in all_passes_after @@ -220,8 +231,9 @@ def rule(pass_before, pass_after): PassBase._COMMON_RULES = [ _fusion_opt_last_rule, lambda pass_before, pass_after: type(pass_before) != type(pass_after), - _make_rule_from_white_lists_dict(PassBase._BEFORE_WHITE_LISTS_DICT, - PassBase._AFTER_WHITE_LISTS_DICT), + _make_rule_from_white_lists_dict( + PassBase._BEFORE_WHITE_LISTS_DICT, PassBase._AFTER_WHITE_LISTS_DICT + ), # Add more common rules here ] @@ -270,10 +282,12 @@ def _solve_pass_conflict(passes, context): old_passes = passes passes = [] for p in old_passes: - if all([ + if all( + [ p._check_conflict_including_common_rules(applied_p) for applied_p in context.passes - ]): + ] + ): passes.append(p) if not passes: @@ -287,14 +301,14 @@ def _solve_pass_conflict(passes, context): for i in range(n): for j in range(n): adjacent_matrix[i][j] = passes[ - j]._check_conflict_including_common_rules(passes[i]) + j + ]._check_conflict_including_common_rules(passes[i]) longest_path = _find_longest_path(adjacent_matrix) return [passes[idx] for idx in longest_path] class PassManager: - def __init__(self, passes, context=None, auto_solve_conflict=True): if context is None: context = PassContext() diff --git a/python/paddle/distributed/passes/pass_utils.py b/python/paddle/distributed/passes/pass_utils.py index e779048c56b12..8e608bd9ed64b 100644 --- a/python/paddle/distributed/passes/pass_utils.py +++ b/python/paddle/distributed/passes/pass_utils.py @@ -108,8 +108,9 @@ def split_program(program, op_indices): op_indices.append(op_num) for idx in range(len(op_indices) - 1): - assert op_indices[idx] < op_indices[ - idx + 1], "op_indices must be strictly sorted" + assert ( + op_indices[idx] < op_indices[idx + 1] + ), "op_indices must be strictly sorted" splitted_programs = [] for idx in range(len(op_indices) - 1): diff --git a/python/paddle/distributed/passes/ps_server_pass.py b/python/paddle/distributed/passes/ps_server_pass.py index 27c10fd1f5bf7..37e5622ea8e7f 100755 --- a/python/paddle/distributed/passes/ps_server_pass.py +++ b/python/paddle/distributed/passes/ps_server_pass.py @@ -15,16 +15,30 @@ import logging import paddle.fluid as fluid -from ..ps.utils.public import get_optimize_ops, get_ps_endpoint, get_role_id, get_trainers +from ..ps.utils.public import ( + get_optimize_ops, + get_ps_endpoint, + get_role_id, + get_trainers, +) from .pass_base import PassBase, register_pass from paddle.optimizer.lr import LRScheduler -from paddle.optimizer.lr import ExponentialDecay, InverseTimeDecay, NaturalExpDecay, NoamDecay -from paddle.fluid.layers.learning_rate_scheduler import exponential_decay, inverse_time_decay, natural_exp_decay, noam_decay +from paddle.optimizer.lr import ( + ExponentialDecay, + InverseTimeDecay, + NaturalExpDecay, + NoamDecay, +) +from paddle.fluid.layers.learning_rate_scheduler import ( + exponential_decay, + inverse_time_decay, + natural_exp_decay, + noam_decay, +) @register_pass("add_lr_decay_table_pass") class AddLrDecayTablePass(PassBase): - def __init__(self): super(AddLrDecayTablePass, self).__init__() @@ -34,13 +48,15 @@ def _check_self(self): def _check_conflict(self, other_pass): return True - def _add_tensor_table(self, - attrs, - feed_var_name, - fetch_var_name="", - startup_program=None, - main_program=None, - tensor_table_class=""): + def _add_tensor_table( + self, + attrs, + feed_var_name, + fetch_var_name="", + startup_program=None, + main_program=None, + tensor_table_class="", + ): tensor_table_dict = {} tensor_table_dict[feed_var_name] = {} tensor_table_dict[feed_var_name]["feed_var_name"] = feed_var_name @@ -48,13 +64,16 @@ def _add_tensor_table(self, tensor_table_dict[feed_var_name]["startup_program"] = startup_program tensor_table_dict[feed_var_name]["main_program"] = main_program tensor_table_dict[feed_var_name][ - "tensor_table_class"] = tensor_table_class + "tensor_table_class" + ] = tensor_table_class attrs['tensor_table'] = tensor_table_dict def _get_lr_sheduler_program(self, lr_sheduler, lr_decay_steps): schedler_decay = [ - 'NoamDecay', 'NaturalExpDecay', 'InverseTimeDecay', - 'ExponentialDecay' + 'NoamDecay', + 'NaturalExpDecay', + 'InverseTimeDecay', + 'ExponentialDecay', ] decay_main_program = fluid.framework.Program() @@ -63,48 +82,59 @@ def _get_lr_sheduler_program(self, lr_sheduler, lr_decay_steps): if isinstance(lr_sheduler, ExponentialDecay): with fluid.program_guard(decay_main_program, decay_startup_program): - lr = exponential_decay(1.0, lr_decay_steps, lr_sheduler.gamma, - True) + lr = exponential_decay( + 1.0, lr_decay_steps, lr_sheduler.gamma, True + ) lr_name = lr.name logging.warn( "ExponentialDecay is set, staircase = True, global learning rate decay step is [ %d ], Change decay steps as follow: \n" "\t strategy = paddle.distributed.fleet.DistributedStrategy() \n " "\t strategy.a_sync = True \n" "\t strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } \n" - % lr_decay_steps) + % lr_decay_steps + ) elif isinstance(lr_sheduler, NoamDecay): with fluid.program_guard(decay_main_program, decay_startup_program): - lr = noam_decay(lr_sheduler.d_model, lr_sheduler.warmup_steps, - 1.0) + lr = noam_decay( + lr_sheduler.d_model, lr_sheduler.warmup_steps, 1.0 + ) lr_name = lr.name - logging.warn("NoamDecay is set, warmup steps is [ %d ]" % - lr_sheduler.warmup_steps) + logging.warn( + "NoamDecay is set, warmup steps is [ %d ]" + % lr_sheduler.warmup_steps + ) elif isinstance(lr_sheduler, NaturalExpDecay): with fluid.program_guard(decay_main_program, decay_startup_program): - lr = natural_exp_decay(1.0, lr_decay_steps, lr_sheduler.gamma, - True) + lr = natural_exp_decay( + 1.0, lr_decay_steps, lr_sheduler.gamma, True + ) lr_name = lr.name logging.warn( "NaturalExpDecay is set, staircase = True, global learning rate decay step is [ %d ], Change decay steps as follow: \n" "\t strategy = paddle.distributed.fleet.DistributedStrategy() \n " "\t strategy.a_sync = True \n" "\t strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } \n" - % lr_decay_steps) + % lr_decay_steps + ) elif isinstance(lr_sheduler, InverseTimeDecay): with fluid.program_guard(decay_main_program, decay_startup_program): - lr = inverse_time_decay(1.0, lr_decay_steps, lr_sheduler.gamma, - True) + lr = inverse_time_decay( + 1.0, lr_decay_steps, lr_sheduler.gamma, True + ) lr_name = lr.name logging.warn( "InverseTimeDecay is set, staircase = True, global learning rate decay step is [ %d ], Change decay steps as follow: \n" "\t strategy = paddle.distributed.fleet.DistributedStrategy() \n " "\t strategy.a_sync = True \n" "\t strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } \n" - % lr_decay_steps) + % lr_decay_steps + ) else: raise ValueError( - "Not supported current LearningRate strategy, please use follow decay strategy: {}" - .format(schedler_decay)) + "Not supported current LearningRate strategy, please use follow decay strategy: {}".format( + schedler_decay + ) + ) return decay_main_program, decay_startup_program, lr_name @@ -113,21 +143,31 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): if hasattr(attrs['origin_main_program'], 'lr_sheduler') == False: return - assert isinstance(attrs['origin_main_program'].lr_sheduler, - LRScheduler), "must be LRScheduler" + assert isinstance( + attrs['origin_main_program'].lr_sheduler, LRScheduler + ), "must be LRScheduler" ops = get_optimize_ops(attrs['origin_main_program']) - lr_decay_main_program, lr_decay_startup_program, lr_name = self._get_lr_sheduler_program( - attrs['origin_main_program'].lr_sheduler, attrs['lr_decay_steps']) - self._add_tensor_table(attrs, "@LR_DECAY_COUNTER@", lr_name, - lr_decay_startup_program, lr_decay_main_program, - "GlobalStepTable") + ( + lr_decay_main_program, + lr_decay_startup_program, + lr_name, + ) = self._get_lr_sheduler_program( + attrs['origin_main_program'].lr_sheduler, attrs['lr_decay_steps'] + ) + self._add_tensor_table( + attrs, + "@LR_DECAY_COUNTER@", + lr_name, + lr_decay_startup_program, + lr_decay_main_program, + "GlobalStepTable", + ) return @register_pass("add_listen_and_serv_pass") class AddListenAndServPass(PassBase): - def __init__(self): super(AddListenAndServPass, self).__init__() @@ -145,7 +185,6 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): "lr_decay_block_id": None, "dense_optimize_blocks": None, "sparse_optimize_blocks": None, - # runtime attribute "endpoint": get_ps_endpoint(attrs['role_maker']), "pserver_id": get_role_id(attrs['role_maker']), @@ -153,17 +192,15 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): "distributed_mode": attrs['ps_mode'], "rpc_get_thread_num": -1, "rpc_send_thread_num": -1, - "rpc_prefetch_thread_num": -1 + "rpc_prefetch_thread_num": -1, } - main_program.global_block().append_op(type="listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs=opt) + main_program.global_block().append_op( + type="listen_and_serv", inputs={'X': []}, outputs={}, attrs=opt + ) @register_pass("add_rpc_global_flags_pass") class AddRpcGlobalFlagsPass(PassBase): - def __init__(self): super(AddRpcGlobalFlagsPass, self).__init__() @@ -179,7 +216,6 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): @register_pass("add_optimizer_pass") class AddOptimizerPass(PassBase): - def __init__(self): super(AddOptimizerPass, self).__init__() @@ -195,7 +231,6 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): @register_pass("add_geo_optimizer_pass") class AddGeoOptimizerPass(PassBase): - def __init__(self): super(AddGeoOptimizerPass, self).__init__() @@ -211,7 +246,6 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): @register_pass("build_pserver_startup_program_pass") class BuildPserverStartupProgramPass(PassBase): - def __init__(self): super(BuildPserverStartupProgramPass, self).__init__() @@ -227,7 +261,6 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): @register_pass("delete_unused_in_startup_pass") class DeleteUnusedInStartupPass(PassBase): - def __init__(self): super(DeleteUnusedInStartupPass, self).__init__() diff --git a/python/paddle/distributed/passes/ps_trainer_pass.py b/python/paddle/distributed/passes/ps_trainer_pass.py index 1b5eb1e442ff1..f99d9f316d462 100755 --- a/python/paddle/distributed/passes/ps_trainer_pass.py +++ b/python/paddle/distributed/passes/ps_trainer_pass.py @@ -25,7 +25,6 @@ @register_pass("append_send_ops_pass") class AppendSendOpsPass(PassBase): # 该 pass 被多种模式复用 - def __init__(self): super(AppendSendOpsPass, self).__init__() @@ -35,8 +34,9 @@ def _check_self(self): def _check_conflict(self, other_pass): return True - def _append_send_op(self, program, union_vars, queue, is_sparse, table_id, - ps_mode): + def _append_send_op( + self, program, union_vars, queue, is_sparse, table_id, ps_mode + ): if queue == STEP_COUNTER: send_input_vars = [] else: @@ -48,43 +48,42 @@ def _append_send_op(self, program, union_vars, queue, is_sparse, table_id, dummy_output = [] if ps_mode in [DistributedMode.SYNC, DistributedMode.HALF_ASYNC]: dummy_output = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) - program.global_block().append_op(type="send", - inputs={"X": send_input_vars}, - outputs={"Out": dummy_output}, - attrs={ - "send_varnames": [queue], - "is_sparse": - is_sparse, - "table_id": - table_id, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) + name=framework.generate_control_dev_var_name() + ) + program.global_block().append_op( + type="send", + inputs={"X": send_input_vars}, + outputs={"Out": dummy_output}, + attrs={ + "send_varnames": [queue], + "is_sparse": is_sparse, + "table_id": table_id, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return dummy_output def _append_barrier_op(self, program, dummys, trainer_id): - program.global_block().append_op(type="send_barrier", - inputs={"X": dummys}, - outputs={"Out": []}, - attrs={ - "trainer_id": - trainer_id, - "half_async": - True, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) + program.global_block().append_op( + type="send_barrier", + inputs={"X": dummys}, + outputs={"Out": []}, + attrs={ + "trainer_id": trainer_id, + "half_async": True, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) def _apply_single_impl(self, main_program, startup_program, pass_ctx): attrs = pass_ctx._attrs ps_mode = attrs['ps_mode'] - #if ps_mode == DistributedMode.GEO: + # if ps_mode == DistributedMode.GEO: # send_ctx = get_geo_trainer_send_context(attrs) # geo 模式, 没必要 send_ctx = get_the_one_send_context( - attrs, - split_dense_table=attrs['is_heter_ps_mode']) # async、sync 等各种模式 + attrs, split_dense_table=attrs['is_heter_ps_mode'] + ) # async、sync 等各种模式 dummys = [] for merged_name, send in send_ctx.items(): # embedding_0.w_0@GRAD @@ -99,9 +98,15 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): is_sparse = 1 if send.is_sparse() else 0 is_sparse = 2 if send.is_distributed() else is_sparse dummys.append( - self._append_send_op(main_program, send.origin_varnames(), - merged_name, is_sparse, send.table_id(), - ps_mode)) + self._append_send_op( + main_program, + send.origin_varnames(), + merged_name, + is_sparse, + send.table_id(), + ps_mode, + ) + ) if ps_mode in [DistributedMode.SYNC, DistributedMode.HALF_ASYNC]: trainer_id = get_role_id(attrs['role_maker']) self._append_barrier_op(main_program, dummys, trainer_id) @@ -109,7 +114,6 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): @register_pass("distributed_ops_pass") class DistributedOpsPass(PassBase): - def __init__(self): super(DistributedOpsPass, self).__init__() self.w_2_table_id = {} @@ -138,8 +142,10 @@ def _push_sparse_fuse(self, _program, push_sparse_ops, attrs, use_cvm_op): if len(entry) == 3 and entry[0] == 'show_click_entry': show_var_name = entry[1] click_var_name = entry[2] - if show_var_name in _program.global_block( - ).vars and click_var_name in _program.global_block().vars: + if ( + show_var_name in _program.global_block().vars + and click_var_name in _program.global_block().vars + ): show = _program.global_block().vars[show_var_name] clk = _program.global_block().vars[click_var_name] use_entry = True @@ -154,31 +160,37 @@ def _push_sparse_fuse(self, _program, push_sparse_ops, attrs, use_cvm_op): name="show", dtype=core.VarDesc.VarType.FP32, persistable=False, - stop_gradient=True) - _program.global_block()._insert_op(index=0, - type='fill_constant', - inputs={}, - outputs={'Out': show}, - attrs={ - 'shape': [1], - 'dtype': show.dtype, - 'value': 1, - }) + stop_gradient=True, + ) + _program.global_block()._insert_op( + index=0, + type='fill_constant', + inputs={}, + outputs={'Out': show}, + attrs={ + 'shape': [1], + 'dtype': show.dtype, + 'value': 1, + }, + ) clk = _program.global_block().create_var( name="clk", dtype=core.VarDesc.VarType.FP32, persistable=False, - stop_gradient=True) - _program.global_block()._insert_op(index=0, - type='fill_constant', - inputs={}, - outputs={'Out': clk}, - attrs={ - 'shape': [1], - 'dtype': clk.dtype, - 'value': 0, - }) + stop_gradient=True, + ) + _program.global_block()._insert_op( + index=0, + type='fill_constant', + inputs={}, + outputs={'Out': clk}, + attrs={ + 'shape': [1], + 'dtype': clk.dtype, + 'value': 0, + }, + ) for param, ops in push_sparse_ops.items(): all_ops = _program.global_block().ops @@ -203,27 +215,27 @@ def _push_sparse_fuse(self, _program, push_sparse_ops, attrs, use_cvm_op): for idx in op_idxs[::-1]: _program.global_block()._remove_op(idx) - _program.global_block().append_op(type="distributed_push_sparse", - inputs={ - "Ids": inputs, - 'W': w, - "Outputs": outputs, - "Shows": show, - "Clicks": clk, - }, - outputs={"Outputs": outputs}, - attrs={ - "is_distributed": - is_distributed, - "padding_idx": padding_idx, - "table_id": table_id, - "size": self.emb_size[param], - "use_cvm_op": use_cvm_op, - "slots": slots - }) + _program.global_block().append_op( + type="distributed_push_sparse", + inputs={ + "Ids": inputs, + 'W': w, + "Outputs": outputs, + "Shows": show, + "Clicks": clk, + }, + outputs={"Outputs": outputs}, + attrs={ + "is_distributed": is_distributed, + "padding_idx": padding_idx, + "table_id": table_id, + "size": self.emb_size[param], + "use_cvm_op": use_cvm_op, + "slots": slots, + }, + ) def _pull_sparse_fuse(self, _program, pull_sparse_ops, attrs, send_ctx): - def dag_check_up_and_reorder(program, inputs, outputs): global_block = program.global_block() min_output_index = len(global_block.ops) @@ -348,7 +360,8 @@ def dag_check_up_and_reorder(program, inputs, outputs): if table_id == -1: raise ValueError( - "can not find suitable sparse table, please check") + "can not find suitable sparse table, please check" + ) self.w_2_table_id[param] = table_id padding_idx = ops[0].attr("padding_idx") @@ -379,8 +392,9 @@ def dag_check_up_and_reorder(program, inputs, outputs): ins = op.input(op.input_names[i]) for out_id, out_var in enumerate(outputs): if out_var.name in ins: - outputs_idxs[out_id] = min(idx, - outputs_idxs[out_id]) + outputs_idxs[out_id] = min( + idx, outputs_idxs[out_id] + ) if attrs['use_ps_gpu']: gpups_inputs_idxs.extend(inputs_idxs) @@ -388,8 +402,9 @@ def dag_check_up_and_reorder(program, inputs, outputs): gpups_inputs.extend(inputs) gpups_outputs.extend(outputs) gpups_w_size.extend([w.shape[1]] * len(inputs)) - gpups_min_distributed_idx = min(min(op_idxs), - gpups_min_distributed_idx) + gpups_min_distributed_idx = min( + min(op_idxs), gpups_min_distributed_idx + ) continue if min(outputs_idxs) - max(inputs_idxs) >= 1: @@ -401,18 +416,16 @@ def dag_check_up_and_reorder(program, inputs, outputs): _program.global_block()._insert_op( index=distributed_idx, type="distributed_lookup_table", - inputs={ - "Ids": inputs, - 'W': w - }, + inputs={"Ids": inputs, 'W': w}, outputs={"Outputs": outputs}, attrs={ "is_distributed": is_distributed, "padding_idx": padding_idx, "table_id": table_id, "lookup_table_version": op_type, - "op_device": op_device - }) + "op_device": op_device, + }, + ) else: for i in range(len(inputs_idxs)): distributed_idx = op_idxs[i] @@ -420,34 +433,34 @@ def dag_check_up_and_reorder(program, inputs, outputs): _program.global_block()._insert_op( index=distributed_idx, type="distributed_lookup_table", - inputs={ - "Ids": [inputs[i]], - 'W': w - }, + inputs={"Ids": [inputs[i]], 'W': w}, outputs={"Outputs": [outputs[i]]}, attrs={ "is_distributed": is_distributed, "padding_idx": padding_idx, "table_id": table_id, "lookup_table_version": op_type, - "op_device": op_device - }) + "op_device": op_device, + }, + ) if attrs['use_ps_gpu'] and len(gpups_inputs) > 0: if max(gpups_inputs_idxs) > 0: raise ValueError("There can't be ops before embedding in gpups") - _program.global_block()._insert_op(index=gpups_min_distributed_idx, - type="pull_gpups_sparse", - inputs={ - "Ids": gpups_inputs, - }, - outputs={"Out": gpups_outputs}, - attrs={ - "size": gpups_w_size, - "is_distributed": True, - "is_sparse": True - }) + _program.global_block()._insert_op( + index=gpups_min_distributed_idx, + type="pull_gpups_sparse", + inputs={ + "Ids": gpups_inputs, + }, + outputs={"Out": gpups_outputs}, + attrs={ + "size": gpups_w_size, + "is_distributed": True, + "is_sparse": True, + }, + ) PSGPU = paddle.fluid.core.PSGPU() try: gpu_slot = [int(var.name) for var in gpups_inputs] @@ -466,8 +479,10 @@ def _get_pull_sparse_ops(self, _program, attrs): ops = {} use_cvm_op = False for op in _program.global_block().ops: - if op.type in SPARSE_OP_TYPE_DICT.keys() \ - and op.attr('remote_prefetch') is True: + if ( + op.type in SPARSE_OP_TYPE_DICT.keys() + and op.attr('remote_prefetch') is True + ): param_name = op.input(SPARSE_OP_TYPE_DICT[op.type])[0] if attrs['is_heter_ps_mode'] and not attrs['is_fl_ps_mode']: # TODO: trick for matchnet, need to modify for heter_ps @@ -486,8 +501,10 @@ def _get_pull_sparse_ops(self, _program, attrs): for op in _program.global_block().ops: if op.type in SPARSE_GRAD_OP_TYPE_DICT.keys(): param_name = op.input(SPARSE_GRAD_OP_TYPE_DICT[op.type])[0] - if param_name in pull_sparse_ids and op.input( - "Ids")[0] in pull_sparse_ids[param_name]: + if ( + param_name in pull_sparse_ids + and op.input("Ids")[0] in pull_sparse_ids[param_name] + ): ops = push_sparse_ops.get(param_name, []) ops.append(op) push_sparse_ops[param_name] = ops @@ -496,19 +513,25 @@ def _get_pull_sparse_ops(self, _program, attrs): def _apply_single_impl(self, main_program, startup_program, pass_ctx): attrs = pass_ctx._attrs - pull_sparse_ops, push_sparse_ops, use_cvm_op = self._get_pull_sparse_ops( - main_program, attrs) - print("is_heter_ps_mode in distributed_ops_pass {}?".format( - attrs['is_heter_ps_mode'])) + ( + pull_sparse_ops, + push_sparse_ops, + use_cvm_op, + ) = self._get_pull_sparse_ops(main_program, attrs) + print( + "is_heter_ps_mode in distributed_ops_pass {}?".format( + attrs['is_heter_ps_mode'] + ) + ) send_ctx = get_the_one_send_context( - attrs, split_dense_table=attrs['is_heter_ps_mode']) + attrs, split_dense_table=attrs['is_heter_ps_mode'] + ) self._pull_sparse_fuse(main_program, pull_sparse_ops, attrs, send_ctx) self._push_sparse_fuse(main_program, push_sparse_ops, attrs, use_cvm_op) @register_pass("delete_optimizer_pass") class DeleteOptimizesPass(PassBase): - def __init__(self): super(DeleteOptimizesPass, self).__init__() @@ -518,8 +541,9 @@ def _check_self(self): def _check_conflict(self, other_pass): return True - def _delete_optimizer_op_and_vars(self, _program, remote_optimize_ops, - local_optimize_ops): + def _delete_optimizer_op_and_vars( + self, _program, remote_optimize_ops, local_optimize_ops + ): local_optimize_vars = [] remote_optimize_vars = [] remote_optimize_op_role_vars = [] @@ -533,14 +557,18 @@ def _delete_optimizer_op_and_vars(self, _program, remote_optimize_ops, remote_optimize_op_role_vars.extend(op.attr("op_role_var")) remote_optimize_vars = list( - set(remote_optimize_vars - )) # param + grad + optimizer_state + learning_rate + set(remote_optimize_vars) + ) # param + grad + optimizer_state + learning_rate remote_optimize_op_role_vars = list( - set(remote_optimize_op_role_vars)) # param + grad + set(remote_optimize_op_role_vars) + ) # param + grad print( - "remote_optimize_vars: {}, remote_optimize_op_role_vars: {}, local_optimize_vars: {}" - .format(remote_optimize_vars, remote_optimize_op_role_vars, - local_optimize_vars)) + "remote_optimize_vars: {}, remote_optimize_op_role_vars: {}, local_optimize_vars: {}".format( + remote_optimize_vars, + remote_optimize_op_role_vars, + local_optimize_vars, + ) + ) for var in remote_optimize_vars: if var in local_optimize_vars: continue @@ -555,26 +583,32 @@ def _delete_optimizer_op_and_vars(self, _program, remote_optimize_ops, def _add_lr_var(self, main_program, attrs): # Todo: hard code for pe - lr_var = attrs['origin_main_program'].global_block( - ).vars["learning_rate_0"] - main_program.global_block().create_var(name=lr_var.name, - shape=lr_var.shape, - dtype=lr_var.dtype, - type=lr_var.type, - lod_level=lr_var.lod_level, - persistable=True) + lr_var = ( + attrs['origin_main_program'].global_block().vars["learning_rate_0"] + ) + main_program.global_block().create_var( + name=lr_var.name, + shape=lr_var.shape, + dtype=lr_var.dtype, + type=lr_var.type, + lod_level=lr_var.lod_level, + persistable=True, + ) def _apply_single_impl(self, main_program, startup_program, pass_ctx): attrs = pass_ctx._attrs all_optimize_ops = get_optimize_ops(main_program) - remote_optimize_ops = get_optimize_ops(main_program, - attrs['remote_sparse']) + remote_optimize_ops = get_optimize_ops( + main_program, attrs['remote_sparse'] + ) lr_ops = get_lr_ops(main_program) remote_optimize_ops.extend(lr_ops) local_optimize_ops = list( - set(all_optimize_ops) - set(remote_optimize_ops)) - self._delete_optimizer_op_and_vars(main_program, remote_optimize_ops, - local_optimize_ops) + set(all_optimize_ops) - set(remote_optimize_ops) + ) + self._delete_optimizer_op_and_vars( + main_program, remote_optimize_ops, local_optimize_ops + ) if hasattr(attrs['origin_main_program'], 'lr_sheduler'): self._add_lr_var(main_program, attrs) @@ -582,7 +616,6 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): @register_pass("delete_extra_optimizer_pass") class DeleteExtraOptimizerPass(PassBase): - def __init__(self): super(DeleteExtraOptimizerPass, self).__init__() @@ -598,10 +631,12 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): remote_optimize_op_role_vars = [] optimize_need_delete_vars = [] all_optimize_ops = get_optimize_ops(main_program) - remote_optimize_ops = get_optimize_ops(main_program, - attrs['remote_sparse']) + remote_optimize_ops = get_optimize_ops( + main_program, attrs['remote_sparse'] + ) local_optimize_ops = list( - set(all_optimize_ops) - set(remote_optimize_ops)) + set(all_optimize_ops) - set(remote_optimize_ops) + ) local_optimize_vars = [] for op in local_optimize_ops: @@ -638,7 +673,6 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): @register_pass("fake_init_ops_pass") class FakeInitOpsPass(PassBase): - def __init__(self): super(FakeInitOpsPass, self).__init__() @@ -649,19 +683,24 @@ def _check_conflict(self, other_pass): return True def _get_sparse_table_names(self, attrs): - dist_varnames = get_sparse_tablenames(attrs['origin_main_programs'], - True) - sparse_varnames = get_sparse_tablenames(attrs['origin_main_programs'], - False) + dist_varnames = get_sparse_tablenames( + attrs['origin_main_programs'], True + ) + sparse_varnames = get_sparse_tablenames( + attrs['origin_main_programs'], False + ) return list(set(dist_varnames + sparse_varnames)) - def _fake_init_sparsetable(self, startup_program, sparse_table_names, - attrs): + def _fake_init_sparsetable( + self, startup_program, sparse_table_names, attrs + ): # delete table init op for table_name in sparse_table_names: table_var = startup_program.global_block().vars[table_name] - if str(table_var).split( - ":")[0].strip().split()[-1] in attrs['local_sparse']: + if ( + str(table_var).split(":")[0].strip().split()[-1] + in attrs['local_sparse'] + ): continue table_param_init_op = [] for op in startup_program.global_block().ops: @@ -669,14 +708,16 @@ def _fake_init_sparsetable(self, startup_program, sparse_table_names, table_param_init_op.append(op) init_op_num = len(table_param_init_op) if init_op_num != 1: - raise ValueError("table init op num should be 1, now is " + - str(init_op_num)) + raise ValueError( + "table init op num should be 1, now is " + str(init_op_num) + ) table_init_op = table_param_init_op[0] startup_program.global_block().append_op( type="fake_init", inputs={}, outputs={"Out": table_var}, - attrs={"shape": table_init_op.attr('shape')}) + attrs={"shape": table_init_op.attr('shape')}, + ) delete_ops(startup_program.global_block(), table_param_init_op) def _apply_single_impl(self, main_program, startup_program, pass_ctx): @@ -687,7 +728,6 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): @register_pass("ps_gpu_pass") class PsGpuPass(PassBase): - def __init__(self): super(PsGpuPass, self).__init__() @@ -706,14 +746,17 @@ def _add_push_box_sparse_op(self, program): if op.type != "pull_box_sparse" and op.type != "pull_gpups_sparse": continue grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - op.desc, set(), []) + op.desc, set(), [] + ) for op_desc in grad_op_desc: new_op_desc = program.global_block().desc._insert_op( - insert_index + 1) + insert_index + 1 + ) new_op_desc.copy_from(op_desc) new_op_desc._set_attr(op_role_attr_name, backward) new_op = paddle.fluid.framework.Operator( - program.global_block(), new_op_desc) + program.global_block(), new_op_desc + ) program.global_block().ops.insert(insert_index + 1, new_op) program.global_block()._sync_with_cpp() @@ -787,7 +830,6 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): @register_pass("ps_transpile_pass") class PsTranspilePass(PassBase): - def __init__(self): super(PsTranspilePass, self).__init__() @@ -801,17 +843,18 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): attrs = pass_ctx._attrs t = SingleProcessMultiThread() env = get_dist_env() - t.transpile(startup_program=startup_program, - main_program=main_program, - rank=env["trainer_id"], - endpoints=env["trainer_endpoints"], - current_endpoint=env['current_endpoint'], - wait_port=False) + t.transpile( + startup_program=startup_program, + main_program=main_program, + rank=env["trainer_id"], + endpoints=env["trainer_endpoints"], + current_endpoint=env['current_endpoint'], + wait_port=False, + ) @register_pass("split_heter_worker_ops_pass") class SplitHeterWorkerOpsPass(PassBase): - def __init__(self): super(SplitHeterWorkerOpsPass, self).__init__() @@ -821,9 +864,15 @@ def _check_self(self): def _check_conflict(self, other_pass): return True - def _create_heter_program(self, program, attrs, heter_program, - program_block_ops_list, heter_ops, - block_var_detail): + def _create_heter_program( + self, + program, + attrs, + heter_program, + program_block_ops_list, + heter_ops, + block_var_detail, + ): # This function mainly includes the following contents: # 1. For every heter block: # a) copy heter device op from origin program @@ -849,10 +898,12 @@ def _create_heter_program(self, program, attrs, heter_program, current_device = role_maker._heter_device_type().lower() stage_id = int(role_maker._get_stage_id()) - heter_block_ops_forward = program_block_ops_list[stage_id - - 1]["forward"] - heter_block_ops_backward = program_block_ops_list[stage_id - - 1]["backward"] + heter_block_ops_forward = program_block_ops_list[stage_id - 1][ + "forward" + ] + heter_block_ops_backward = program_block_ops_list[stage_id - 1][ + "backward" + ] heter_block = heter_program._create_block(pre_block_idx) optimizer_block.append(heter_block) @@ -874,59 +925,82 @@ def _create_heter_program(self, program, attrs, heter_program, for _, op in enumerate(heter_block_ops_backward): block_append_op(heter_program, program, heter_block_bp, op) - bp_entrance_vars = block_var_detail[stage_id - - 1]["backward"]["entrance"] - add_vars_by_var_list(bp_entrance_vars, program, heter_program, - heter_block_bp) + bp_entrance_vars = block_var_detail[stage_id - 1]["backward"][ + "entrance" + ] + add_vars_by_var_list( + bp_entrance_vars, program, heter_program, heter_block_bp + ) bp_exit_vars = block_var_detail[stage_id - 1]["backward"]["exit"] - add_vars_by_var_list(bp_exit_vars, program, heter_program, - heter_block_bp) - backward_comm_info = get_communicate_var_info(program, - stage_id, - bp_entrance_vars, - type="backward") + add_vars_by_var_list( + bp_exit_vars, program, heter_program, heter_block_bp + ) + backward_comm_info = get_communicate_var_info( + program, stage_id, bp_entrance_vars, type="backward" + ) - grad_to_block_id.append(backward_comm_info["block_input_var_name"] + - ":" + str(heter_block_bp.idx)) + grad_to_block_id.append( + backward_comm_info["block_input_var_name"] + + ":" + + str(heter_block_bp.idx) + ) else: for _, op in enumerate(heter_block_ops_backward): block_append_op(heter_program, program, heter_block, op) - bp_entrance_vars = block_var_detail[stage_id - - 1]["backward"]["entrance"] - add_vars_by_var_list(bp_entrance_vars, program, heter_program, - heter_block) + bp_entrance_vars = block_var_detail[stage_id - 1]["backward"][ + "entrance" + ] + add_vars_by_var_list( + bp_entrance_vars, program, heter_program, heter_block + ) bp_exit_vars = block_var_detail[stage_id - 1]["backward"]["exit"] - add_vars_by_var_list(bp_exit_vars, program, heter_program, - heter_block) + add_vars_by_var_list( + bp_exit_vars, program, heter_program, heter_block + ) heter_block_bp = heter_block - forward_comm_info = get_communicate_var_info(program, - stage_id, - entrance_vars, - type="forward") + forward_comm_info = get_communicate_var_info( + program, stage_id, entrance_vars, type="forward" + ) - grad_to_block_id.append(forward_comm_info["block_input_var_name"] + - ":" + str(heter_block.idx)) + grad_to_block_id.append( + forward_comm_info["block_input_var_name"] + + ":" + + str(heter_block.idx) + ) first_op_index_bp = len(heter_block_bp.ops) if stage_id <= len(block_var_detail) - 1: - static_var = insert_communicate_op(program, role_maker, heter_block, - stage_id, first_op_index_fp, - block_var_detail, current_device) - static_var_bp = insert_communicate_op(program, role_maker, - heter_block_bp, stage_id, - first_op_index_bp, - block_var_detail, current_device, - False) + static_var = insert_communicate_op( + program, + role_maker, + heter_block, + stage_id, + first_op_index_fp, + block_var_detail, + current_device, + ) + static_var_bp = insert_communicate_op( + program, + role_maker, + heter_block_bp, + stage_id, + first_op_index_bp, + block_var_detail, + current_device, + False, + ) # add send op send_grad_var_list = add_send_op( - program, heter_block_bp, - block_var_detail[stage_id - 1]["backward"]["persistables"]) + program, + heter_block_bp, + block_var_detail[stage_id - 1]["backward"]["persistables"], + ) # add step conter send_input_vars = [] @@ -941,13 +1015,15 @@ def _create_heter_program(self, program, attrs, heter_program, "pserver_id": get_role_id(role_maker), "distributed_mode": attrs['ps_mode'], "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, } # append the listen_and_serv op - heter_program.global_block().append_op(type="heter_listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs=attrs) + heter_program.global_block().append_op( + type="heter_listen_and_serv", + inputs={'X': []}, + outputs={}, + attrs=attrs, + ) # TODO check heter program def _apply_single_impl(self, main_program, startup_program, pass_ctx): @@ -960,7 +1036,8 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): attrs = pass_ctx._attrs default_deveice = "cpu" program, heter_ops, _, program_block_ops = find_heter_ops( - main_program, default_deveice) + main_program, default_deveice + ) if len(heter_ops) == 0: warnings.warn( "Currently running in Heter Parameter Server mode, but no OP running on heterogeneous devices, Please check your code." @@ -969,18 +1046,23 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): return program_block_ops = union_forward_gradient_op(program_block_ops) - block_vars_detail = find_block_joints(program, program_block_ops, - heter_ops) + block_vars_detail = find_block_joints( + program, program_block_ops, heter_ops + ) heter_program = framework.Program() - self._create_heter_program(program, attrs, heter_program, - program_block_ops, heter_ops, - block_vars_detail) + self._create_heter_program( + program, + attrs, + heter_program, + program_block_ops, + heter_ops, + block_vars_detail, + ) main_program = heter_program @register_pass("split_trainer_ops_pass") class SplitTrainerOpsPass(PassBase): - def __init__(self): super(SplitTrainerOpsPass, self).__init__() @@ -990,8 +1072,9 @@ def _check_self(self): def _check_conflict(self, other_pass): return True - def _replace_ops_by_communicate_op(self, program, attrs, heter_block_index, - ops_list, block_var_detail): + def _replace_ops_by_communicate_op( + self, program, attrs, heter_block_index, ops_list, block_var_detail + ): all_op = program.global_block().ops start_op = ops_list[0] first_op_idx = -1 @@ -1008,10 +1091,12 @@ def _replace_ops_by_communicate_op(self, program, attrs, heter_block_index, next_heter_worker_endpoints = get_next_stage_trainers(role_maker) entrance_var = block_var_detail[heter_block_index]["forward"][ - "entrance"] + "entrance" + ] - comm_info = get_communicate_var_info(program, heter_block_index + 1, - entrance_var) + comm_info = get_communicate_var_info( + program, heter_block_index + 1, entrance_var + ) program.global_block()._insert_op( index=first_op_idx, type="send_and_recv", @@ -1025,8 +1110,9 @@ def _replace_ops_by_communicate_op(self, program, attrs, heter_block_index, "next_endpoints": next_heter_worker_endpoints, "previous_endpoints": [], "trainer_id": get_role_id(role_maker), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return entrance_var @@ -1051,18 +1137,22 @@ def _remove_var_pair_by_grad(self, var_name, attrs): del attrs['merged_sparse_pairs'][index] return - def _remove_trainer_send_op(self, program, attrs, heter_block_index, - block_var_detail): + def _remove_trainer_send_op( + self, program, attrs, heter_block_index, block_var_detail + ): # if trainer do FF->BP->SEND, it has follow vars: var, var@GRAD # if trainer only do SEND, it has one var: var@GRAD # Delete Send op ,if trainer doesn't has pair var (var<->var@GRAD) - persistables = block_var_detail[heter_block_index]["forward"]["persistables"] + \ - block_var_detail[heter_block_index]["backward"]["persistables"] + persistables = ( + block_var_detail[heter_block_index]["forward"]["persistables"] + + block_var_detail[heter_block_index]["backward"]["persistables"] + ) need_remove_send_op = [] need_remove_grad_var = [] for op in find_send_op(program): - input_list, _ = find_op_input_output(program, - program.global_block(), op) + input_list, _ = find_op_input_output( + program, program.global_block(), op + ) for var_name in input_list: origin_var_name = var_name.split("@GRAD")[0] if origin_var_name in persistables: @@ -1073,8 +1163,14 @@ def _remove_trainer_send_op(self, program, attrs, heter_block_index, for grad_var_name in need_remove_grad_var: self._remove_var_pair_by_grad(grad_var_name, attrs) - def _create_trainer_program(self, program, origin_program, attrs, - program_block_ops_list, block_var_detail): + def _create_trainer_program( + self, + program, + origin_program, + attrs, + program_block_ops_list, + block_var_detail, + ): # This function mainly includes the following contents: # 1. For every heter block in origin program # a) delete heter op and related variables @@ -1087,13 +1183,16 @@ def _create_trainer_program(self, program, origin_program, attrs, # 2. check every op's device static_var = [] for heter_block_index in range(1, len(program_block_ops_list)): - ops_list = program_block_ops_list[heter_block_index][ - "forward"] + program_block_ops_list[heter_block_index][ - "backward"] + ops_list = ( + program_block_ops_list[heter_block_index]["forward"] + + program_block_ops_list[heter_block_index]["backward"] + ) static_var += self._replace_ops_by_communicate_op( - program, attrs, heter_block_index, ops_list, block_var_detail) - self._remove_trainer_send_op(program, attrs, heter_block_index, - block_var_detail) + program, attrs, heter_block_index, ops_list, block_var_detail + ) + self._remove_trainer_send_op( + program, attrs, heter_block_index, block_var_detail + ) optimizer_block = [] grad_to_block_id = [] @@ -1101,40 +1200,46 @@ def _create_trainer_program(self, program, origin_program, attrs, bp_ops_list = program_block_ops_list[0]["backward"] delete_same_ops(program.global_block(), bp_ops_list) delete_trainer_useless_var(program, static_var) - backward_block = create_backward_block(program, origin_program, - bp_ops_list, block_var_detail) + backward_block = create_backward_block( + program, origin_program, bp_ops_list, block_var_detail + ) bp_entrance_vars = block_var_detail[0]["backward"]["entrance"] - backward_comm_info = get_communicate_var_info(origin_program, - 1, - bp_entrance_vars, - type="backward") - - grad_to_block_id.append(backward_comm_info["block_input_var_name"] + - ":" + str(backward_block.idx)) + backward_comm_info = get_communicate_var_info( + origin_program, 1, bp_entrance_vars, type="backward" + ) + + grad_to_block_id.append( + backward_comm_info["block_input_var_name"] + + ":" + + str(backward_block.idx) + ) optimizer_block.append(backward_block) role_maker = attrs['role_maker'] attrs = { "message_to_block_id": grad_to_block_id, "optimize_blocks": optimizer_block, # runtime attribute - "endpoint": - get_trainer_endpoint(role_maker), ## get trainer endpoint + "endpoint": get_trainer_endpoint( + role_maker + ), ## get trainer endpoint "fanin": 0, ## get heter worker "pserver_id": get_role_id(role_maker), "distributed_mode": attrs['ps_mode'], "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, } # append the listen_and_serv op - program.global_block()._insert_op(index=0, - type="heter_listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs=attrs) + program.global_block()._insert_op( + index=0, + type="heter_listen_and_serv", + inputs={'X': []}, + outputs={}, + attrs=attrs, + ) ## TODO add check for bp block - #check_op_device(program.global_block(), DEFAULT_DEVICE) + # check_op_device(program.global_block(), DEFAULT_DEVICE) def _apply_single_impl(self, main_program, startup_program, pass_ctx): """ @@ -1146,20 +1251,26 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): attrs = pass_ctx._attrs default_device_ = 'cpu' program, heter_ops, default_ops, program_block_ops = find_heter_ops( - main_program, default_device_) + main_program, default_device_ + ) program_block_ops = union_forward_gradient_op(program_block_ops) - block_vars_detail = find_block_joints(program, program_block_ops, - heter_ops) + block_vars_detail = find_block_joints( + program, program_block_ops, heter_ops + ) trainer_program = program.clone() - self._create_trainer_program(trainer_program, program, attrs, - program_block_ops, block_vars_detail) + self._create_trainer_program( + trainer_program, + program, + attrs, + program_block_ops, + block_vars_detail, + ) main_program = trainer_program @register_pass("set_heter_pipeline_opt_pass") class SetHeterPipelineOptPass(PassBase): - def __init__(self): super(SetHeterPipelineOptPass, self).__init__() @@ -1173,32 +1284,31 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): attrs = pass_ctx._attrs role_maker = attrs['role_maker'] num_microbatches = attrs['user_defined_strategy'].pipeline_configs[ - 'accumulate_steps'] + 'accumulate_steps' + ] startup_program._heter_pipeline_opt = { "startup_program": startup_program, "pipeline_stage": int(role_maker._get_stage_id()) - 1, "heter_place": role_maker._heter_device(), - "is_fl_mode": 1 + "is_fl_mode": 1, } main_program._heter_pipeline_opt = { "trainer": "HeterPipelineTrainer", "device_worker": "HeterSection", - "trainers": - role_maker._get_stage_trainers(), ## trainer num in each stage + "trainers": role_maker._get_stage_trainers(), ## trainer num in each stage "trainer_id": int(role_maker._role_id()), "pipeline_stage": int(role_maker._get_stage_id()) - 1, "num_pipeline_stages": int(role_maker._get_num_stage()), "section_program": main_program, "num_microbatches": num_microbatches, "heter_place": role_maker._heter_device(), - "is_fl_mode": 1 + "is_fl_mode": 1, } @register_pass("split_fl_ops_pass") class SplitFlOpsPass(PassBase): - def __init__(self): super(SplitFlOpsPass, self).__init__() self.PART_A_DEVICE_FlAG = 'gpu:0' @@ -1231,10 +1341,17 @@ def _split_fl_program(self): block = self.ori_main_program.block(0) for op in block.ops: device = op.attr(OP_DEVICE_KEY) - if device == self.PART_A_DEVICE_FlAG or device == '' or device == self.PART_A_JOINT_OP_DEVICE_FlAG: + if ( + device == self.PART_A_DEVICE_FlAG + or device == '' + or device == self.PART_A_JOINT_OP_DEVICE_FlAG + ): program = party_program_map['a'] self.partA_ops.append(op) - elif device == self.PART_B_DEVICE_FlAG or device == self.PART_B_JOINT_OP_DEVICE_FlAG: + elif ( + device == self.PART_B_DEVICE_FlAG + or device == self.PART_B_JOINT_OP_DEVICE_FlAG + ): program = party_program_map['b'] self.partB_ops.append(op) op_desc = op.desc @@ -1257,21 +1374,24 @@ def _insert_partA_communicate_op(self, block, idx): outputs={'Out': []}, attrs={ 'mode': 'forward', # mode 直接关联前向和反向 channel 选择 - 'send_var_name': - self.partA_to_partB_tensor_name + ["microbatch_id"], + 'send_var_name': self.partA_to_partB_tensor_name + + ["microbatch_id"], 'recv_var_name': [], 'message_name': comm_info, - 'next_endpoints': - get_next_stage_trainers(self.role_maker), # partB_endpoints - 'previous_endpoints': - get_previous_stage_trainers(self.role_maker), + 'next_endpoints': get_next_stage_trainers( + self.role_maker + ), # partB_endpoints + 'previous_endpoints': get_previous_stage_trainers( + self.role_maker + ), 'trainer_id': get_role_id(self.role_maker), # global id - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return def _insert_partB_communicate_op(self, block, idx): - comm_info = ("backward_joint_{}_{}@fl_ps".format(2, 1)) + comm_info = "backward_joint_{}_{}@fl_ps".format(2, 1) block._insert_op( idx, type='send_and_recv', @@ -1279,17 +1399,20 @@ def _insert_partB_communicate_op(self, block, idx): outputs={'Out': []}, attrs={ 'mode': 'backward', - 'send_var_name': - self.partB_to_partA_grad_name + ["microbatch_id"], + 'send_var_name': self.partB_to_partA_grad_name + + ["microbatch_id"], 'recv_var_name': [], 'message_name': comm_info, - 'next_endpoints': - get_next_stage_trainers(self.role_maker), # partA_endpoints - 'previous_endpoints': - get_previous_stage_trainers(self.role_maker), + 'next_endpoints': get_next_stage_trainers( + self.role_maker + ), # partA_endpoints + 'previous_endpoints': get_previous_stage_trainers( + self.role_maker + ), 'trainer_id': get_role_id(self.role_maker), # global id - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return def _create_var_for_block(self, vars, block): @@ -1308,7 +1431,8 @@ def _create_var_for_block(self, vars, block): trainable=source_var.trainable, optimize_attr=source_var.optimize_attr, regularizer=source_var.regularizer, - error_clip=source_var.error_clip) + error_clip=source_var.error_clip, + ) else: dest_var = block._clone_variable(source_var, False) dest_var.stop_gradient = source_var.stop_gradient @@ -1362,15 +1486,18 @@ def _get_partB_to_partA_grad(self, block, flag): def _find_dense_grad_vars(self, bp_op_list): program = self.ori_main_program bp_op_input, bp_op_output = find_ops_list_input_output( - program, bp_op_list) - return (screen_persistables(program, bp_op_input) + - screen_persistables(program, bp_op_output)) + program, bp_op_list + ) + return screen_persistables(program, bp_op_input) + screen_persistables( + program, bp_op_output + ) def _get_partA_program(self, block): # 1. create block 0 # 1.1 insert send op - op_idx = self._find_joint_forward_op(block, - self.PART_A_JOINT_OP_DEVICE_FlAG) + op_idx = self._find_joint_forward_op( + block, self.PART_A_JOINT_OP_DEVICE_FlAG + ) op_list = [] for i in range(len(block.ops)): op = block.ops[i] @@ -1388,8 +1515,9 @@ def _get_partA_program(self, block): bp_op_list = get_bp_op_list(block) push_sparse_op_list = get_distributed_push_sparse_op_list(block) # logger.info('bp_op_list: {}'.format(bp_op_list)) - second_block = self._get_block_by_idx(bp_op_list + push_sparse_op_list, - self.partA_program, 1) + second_block = self._get_block_by_idx( + bp_op_list + push_sparse_op_list, self.partA_program, 1 + ) # 2.1. insert partA recv op block_input_flag = "backward_joint_{}_{}@fl_ps".format(2, 1) grad_to_block_id = block_input_flag + ":" + str(second_block.idx) @@ -1401,13 +1529,15 @@ def _get_partA_program(self, block): "pserver_id": get_role_id(self.role_maker), "distributed_mode": self.ps_mode, "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, } - second_block._insert_op(index=0, - type='heter_listen_and_serv', - inputs={'X': []}, - outputs={}, - attrs=attrs) + second_block._insert_op( + index=0, + type='heter_listen_and_serv', + inputs={'X': []}, + outputs={}, + attrs=attrs, + ) # 2.2 insert push dense grad op send_ops = find_send_op(self.ori_main_program) # push dense delete_same_ops(block, send_ops) @@ -1417,9 +1547,11 @@ def _get_partA_program(self, block): def _get_partB_program(self, block): op_idx1 = self._find_joint_forward_op( - block, self.PART_B_JOINT_OP_DEVICE_FlAG) # elementwise_add op - op_idx2 = self._find_joint_backward_op(block, - self.PART_B_JOINT_OP_DEVICE_FlAG) + block, self.PART_B_JOINT_OP_DEVICE_FlAG + ) # elementwise_add op + op_idx2 = self._find_joint_backward_op( + block, self.PART_B_JOINT_OP_DEVICE_FlAG + ) op_cnt = 0 op_list1 = [] op_list2 = [] @@ -1458,16 +1590,18 @@ def _get_partB_program(self, block): "pserver_id": 1, # TODO "distributed_mode": self.ps_mode, "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, } - first_block._insert_op(index=len(op_list1), - type="heter_listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs=attrs) + first_block._insert_op( + index=len(op_list1), + type="heter_listen_and_serv", + inputs={'X': []}, + outputs={}, + attrs=attrs, + ) - #logger.info('partB-first_block:{}'.format(first_block)) - #logger.info('partB-second_block:{}'.format(second_block)) + # logger.info('partB-first_block:{}'.format(first_block)) + # logger.info('partB-second_block:{}'.format(second_block)) def _apply_single_impl(self, main_program, startup_program, pass_ctx): attrs = pass_ctx._attrs @@ -1482,8 +1616,9 @@ def _apply_single_impl(self, main_program, startup_program, pass_ctx): prog_a = party_program_map['a'] _main_file = ps_log_root_dir + '6_fl_A_main_program.prototxt' debug_program(_main_file, prog_a) - self._get_partB_to_partA_grad(prog_a.global_block(), - self.PART_A_JOINT_OP_DEVICE_FlAG) + self._get_partB_to_partA_grad( + prog_a.global_block(), self.PART_A_JOINT_OP_DEVICE_FlAG + ) prog_b = party_program_map['b'] _main_file = ps_log_root_dir + '6_fl_B_main_program.prototxt' diff --git a/python/paddle/distributed/ps/coordinator.py b/python/paddle/distributed/ps/coordinator.py index a012f338a514f..3a6e0756df435 100755 --- a/python/paddle/distributed/ps/coordinator.py +++ b/python/paddle/distributed/ps/coordinator.py @@ -26,7 +26,8 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) formatter = logging.Formatter( - fmt='%(asctime)s %(levelname)-2s [%(filename)s:%(lineno)d] %(message)s') + fmt='%(asctime)s %(levelname)-2s [%(filename)s:%(lineno)d] %(message)s' +) ch = logging.StreamHandler() ch.setFormatter(formatter) logger.addHandler(ch) @@ -46,7 +47,6 @@ class FLStrategy: class ClientSelectorBase(abc.ABC): - def __init__(self, fl_clients_info_mp): self.fl_clients_info_mp = fl_clients_info_mp self.clients_info = {} @@ -58,17 +58,19 @@ def parse_from_string(self): for client_id, info in self.fl_clients_info_mp.items(): self.fl_client_info_desc = the_one_ps_pb2.FLClientInfo() - text_format.Parse(bytes(info, encoding="utf8"), - self.fl_client_info_desc) + text_format.Parse( + bytes(info, encoding="utf8"), self.fl_client_info_desc + ) self.clients_info[client_id] = {} self.clients_info[client_id][ - ClientInfoAttr. - DEVICE_TYPE] = self.fl_client_info_desc.device_type + ClientInfoAttr.DEVICE_TYPE + ] = self.fl_client_info_desc.device_type self.clients_info[client_id][ - ClientInfoAttr. - COMPUTE_CAPACITY] = self.fl_client_info_desc.compute_capacity + ClientInfoAttr.COMPUTE_CAPACITY + ] = self.fl_client_info_desc.compute_capacity self.clients_info[client_id][ - ClientInfoAttr.BANDWIDTH] = self.fl_client_info_desc.bandwidth + ClientInfoAttr.BANDWIDTH + ] = self.fl_client_info_desc.bandwidth @abc.abstractmethod def select(self): @@ -76,7 +78,6 @@ def select(self): class ClientSelector(ClientSelectorBase): - def __init__(self, fl_clients_info_mp): super().__init__(fl_clients_info_mp) self.__fl_strategy = {} @@ -84,8 +85,11 @@ def __init__(self, fl_clients_info_mp): def select(self): self.parse_from_string() for client_id in self.clients_info: - logger.info("fl-ps > client {} info : {}".format( - client_id, self.clients_info[client_id])) + logger.info( + "fl-ps > client {} info : {}".format( + client_id, self.clients_info[client_id] + ) + ) # ......... to implement ...... # fl_strategy_desc = the_one_ps_pb2.FLStrategy() fl_strategy_desc.iteration_num = 99 @@ -97,7 +101,6 @@ def select(self): class FLClientBase(abc.ABC): - def __init__(self): pass @@ -113,8 +116,9 @@ def set_basic_config(self, role_maker, config, metrics): self.startup_program = paddle.static.default_startup_program() self._client_ptr = fleet.get_fl_client() self._coordinators = self.role_maker._get_coordinator_endpoints() - logger.info("fl-ps > coordinator enpoints: {}".format( - self._coordinators)) + logger.info( + "fl-ps > coordinator enpoints: {}".format(self._coordinators) + ) self.strategy_handlers = dict() self.exe = None self.use_cuda = int(self.config.get("runner.use_gpu")) @@ -129,8 +133,11 @@ def set_basic_config(self, role_maker, config, metrics): def set_train_dataset_info(self, train_dataset, train_file_list): self.train_dataset = train_dataset self.train_file_list = train_file_list - logger.info("fl-ps > {}, data_feed_desc:\n {}".format( - type(self.train_dataset), self.train_dataset._desc())) + logger.info( + "fl-ps > {}, data_feed_desc:\n {}".format( + type(self.train_dataset), self.train_dataset._desc() + ) + ) def set_test_dataset_info(self, test_dataset, test_file_list): self.test_dataset = test_dataset @@ -161,7 +168,8 @@ def set_dump_fields(self): if self.config.get("runner.need_dump"): self.debug = True dump_fields_path = "{}/epoch_{}".format( - self.config.get("runner.dump_fields_path"), self.epoch_idx) + self.config.get("runner.dump_fields_path"), self.epoch_idx + ) dump_fields = self.config.get("runner.dump_fields", []) dump_param = self.config.get("runner.dump_param", []) persist_vars_list = self.main_program.all_parameters() @@ -170,11 +178,13 @@ def set_dump_fields(self): for param in persist_vars_list ] logger.info( - "fl-ps > persist_vars_list: {}".format(persist_vars_name)) + "fl-ps > persist_vars_list: {}".format(persist_vars_name) + ) if dump_fields_path is not None: self.main_program._fleet_opt[ - 'dump_fields_path'] = dump_fields_path + 'dump_fields_path' + ] = dump_fields_path if dump_fields is not None: self.main_program._fleet_opt["dump_fields"] = dump_fields if dump_param is not None: @@ -186,7 +196,6 @@ def set_metrics(self, metrics): class FLClient(FLClientBase): - def __init__(self): super(FLClient, self).__init__() @@ -195,12 +204,13 @@ def __build_fl_client_info_desc(self, state_info): state_info = { ClientInfoAttr.DEVICE_TYPE: "Andorid", ClientInfoAttr.COMPUTE_CAPACITY: 10, - ClientInfoAttr.BANDWIDTH: 100 + ClientInfoAttr.BANDWIDTH: 100, } client_info = the_one_ps_pb2.FLClientInfo() client_info.device_type = state_info[ClientInfoAttr.DEVICE_TYPE] client_info.compute_capacity = state_info[ - ClientInfoAttr.COMPUTE_CAPACITY] + ClientInfoAttr.COMPUTE_CAPACITY + ] client_info.bandwidth = state_info[ClientInfoAttr.BANDWIDTH] str_msg = text_format.MessageToString(client_info) return str_msg @@ -224,7 +234,7 @@ def train_loop(self): state_info = { "client id": self.worker_index, "auc": 0.9, - "epoch": self.epoch_idx + "epoch": self.epoch_idx, } self.push_fl_client_info_sync(state_info) strategy_dict = self.pull_fl_strategy() @@ -242,13 +252,18 @@ def push_fl_client_info_sync(self, state_info): def pull_fl_strategy(self): strategy_dict = {} - fl_strategy_str = self._client_ptr.pull_fl_strategy( + fl_strategy_str = ( + self._client_ptr.pull_fl_strategy() ) # block: wait for coordinator's strategy arrived - logger.info("fl-ps > fl client recved fl_strategy(str):\n{}".format( - fl_strategy_str)) + logger.info( + "fl-ps > fl client recved fl_strategy(str):\n{}".format( + fl_strategy_str + ) + ) fl_strategy_desc = the_one_ps_pb2.FLStrategy() - text_format.Parse(bytes(fl_strategy_str, encoding="utf8"), - fl_strategy_desc) + text_format.Parse( + bytes(fl_strategy_str, encoding="utf8"), fl_strategy_desc + ) strategy_dict["next_state"] = fl_strategy_desc.next_state return strategy_dict @@ -262,8 +277,9 @@ def register_default_handlers(self): self.register_handlers('train', self.callback_train) self.register_handlers('infer', self.callback_infer) self.register_handlers('finish', self.callback_finish) - self.register_handlers('initialize_model_params', - self.callback_initialize_model_params) + self.register_handlers( + 'initialize_model_params', self.callback_initialize_model_params + ) self.register_handlers('init_worker', self.callback_init_worker) self.register_handlers('save_model', self.callback_save_model) @@ -282,12 +298,14 @@ def callback_train(self): "Epoch {} Var {}".format(self.epoch_idx, var_name) for var_name in self.metrics ] - self.exe.train_from_dataset(program=self.main_program, - dataset=self.train_dataset, - fetch_list=self.fetch_vars, - fetch_info=fetch_info, - print_period=self.print_step, - debug=self.debug) + self.exe.train_from_dataset( + program=self.main_program, + dataset=self.train_dataset, + fetch_list=self.fetch_vars, + fetch_info=fetch_info, + print_period=self.print_step, + debug=self.debug, + ) self.epoch_idx += 1 epoch_time = time.time() - epoch_start_time epoch_speed = self.train_example_nums / epoch_time @@ -299,12 +317,14 @@ def callback_infer(self): "Epoch {} Var {}".format(self.epoch_idx, var_name) for var_name in self.metrics ] - self.exe.infer_from_dataset(program=self.main_program, - dataset=self.test_dataset, - fetch_list=self.fetch_vars, - fetch_info=fetch_info, - print_period=self.print_step, - debug=self.debug) + self.exe.infer_from_dataset( + program=self.main_program, + dataset=self.test_dataset, + fetch_list=self.fetch_vars, + fetch_info=fetch_info, + print_period=self.print_step, + debug=self.debug, + ) def callback_save_model(self): model_dir = "{}/{}".format(self.save_model_path, self.epoch_idx) @@ -318,12 +338,14 @@ def callback_finish(self): fleet.stop_worker() def print_program(self): - with open("./{}_worker_main_program.prototxt".format(self.worker_index), - 'w+') as f: + with open( + "./{}_worker_main_program.prototxt".format(self.worker_index), 'w+' + ) as f: f.write(str(self.main_program)) with open( - "./{}_worker_startup_program.prototxt".format( - self.worker_index), 'w+') as f: + "./{}_worker_startup_program.prototxt".format(self.worker_index), + 'w+', + ) as f: f.write(str(self.startup_program)) def print_train_statical_info(self): @@ -332,7 +354,6 @@ def print_train_statical_info(self): class Coordinator(object): - def __init__(self, ps_hosts): self._communicator = FLCommunicator(ps_hosts) self._client_selector = None @@ -344,7 +365,8 @@ def make_fl_strategy(self): logger.info("fl-ps > running make_fl_strategy(loop) in coordinator\n") while True: # 1. get all fl clients reported info - str_map = self._communicator.query_fl_clients_info( + str_map = ( + self._communicator.query_fl_clients_info() ) # block: wait for all fl clients info reported # 2. generate fl strategy self._client_selector = ClientSelector(str_map) diff --git a/python/paddle/distributed/ps/the_one_ps.py b/python/paddle/distributed/ps/the_one_ps.py index 78ecef22b63e6..86766d60ae8c0 100755 --- a/python/paddle/distributed/ps/the_one_ps.py +++ b/python/paddle/distributed/ps/the_one_ps.py @@ -24,15 +24,21 @@ from paddle.fluid.executor import Executor from paddle.fluid.parallel_executor import ParallelExecutor from paddle.distributed.fleet.runtime.runtime_base import RuntimeBase -from paddle.distributed.fleet.base.private_helper_function import wait_server_ready +from paddle.distributed.fleet.base.private_helper_function import ( + wait_server_ready, +) from paddle.distributed.fleet.proto import the_one_ps_pb2 from paddle.fluid.communicator import Communicator, HeterClient from google.protobuf import text_format from paddle.distributed.ps.coordinator import Coordinator __all__ = [ - 'Table', 'SparseTable', 'GeoSparseTable', 'BarrierTable', 'TensorTable', - 'DenseTable' + 'Table', + 'SparseTable', + 'GeoSparseTable', + 'BarrierTable', + 'TensorTable', + 'DenseTable', ] @@ -52,7 +58,11 @@ def parse_table_class(varname, program_id, context): param_name = op.input("W")[0] - if param_name == varname and op.type == "lookup_table" or op.type == "lookup_table_v2": + if ( + param_name == varname + and op.type == "lookup_table" + or op.type == "lookup_table_v2" + ): if op.has_attr('table_class') and op.attr("table_class") != "none": return op.attr('table_class') else: @@ -65,37 +75,47 @@ def check_embedding_dim(accessor_proto, varname, program_id, context): for var in main_program.list_vars(): if var.name == varname: embedding_dim = var.shape[1] - print('new var: {}, {}, {}'.format(var, embedding_dim, - accessor_proto.fea_dim)) + print( + 'new var: {}, {}, {}'.format( + var, embedding_dim, accessor_proto.fea_dim + ) + ) break fea_dim = accessor_proto.fea_dim if accessor_proto.accessor_class == "SparseAccessor": if fea_dim != embedding_dim + 2: raise ValueError( - "The fea_dim is wrong, it will be sparse_embedding_dim + 2: {}, but got {}" - .format(embedding_dim + 2, fea_dim)) + "The fea_dim is wrong, it will be sparse_embedding_dim + 2: {}, but got {}".format( + embedding_dim + 2, fea_dim + ) + ) else: if fea_dim != embedding_dim: raise ValueError( - "The fea_dim is wrong, it will be sparse_embedding_dim: {}, but got {}" - .format(embedding_dim, fea_dim)) + "The fea_dim is wrong, it will be sparse_embedding_dim: {}, but got {}".format( + embedding_dim, fea_dim + ) + ) embedx_dim = accessor_proto.embedx_dim if accessor_proto.accessor_class == "SparseAccessor": if embedx_dim != embedding_dim - 1: raise ValueError( - "The embedx_dim is wrong, it will be sparse_embedding_dim - 1: {}, but got {}" - .format(embedding_dim - 1, embedx_dim)) + "The embedx_dim is wrong, it will be sparse_embedding_dim - 1: {}, but got {}".format( + embedding_dim - 1, embedx_dim + ) + ) else: if embedx_dim != embedding_dim - 3: raise ValueError( - "The embedx_dim is wrong, it will be sparse_embedding_dim - 3: {}, but got {}" - .format(embedding_dim - 3, embedx_dim)) + "The embedx_dim is wrong, it will be sparse_embedding_dim - 3: {}, but got {}".format( + embedding_dim - 3, embedx_dim + ) + ) class Service: - def __init__(self): pass @@ -108,7 +128,6 @@ def _set(self, service_proto): class GpuService(Service): - def __init__(self): super(GpuService, self).__init__() @@ -118,7 +137,6 @@ def _set(self, service_proto): class Accessor: - def __init__(self): self.accessor_class = "" self.optimizer = None @@ -126,10 +144,12 @@ def __init__(self): self.embedding_dim = 0 # TableAccessorParameter accessor - def _set(self, accessor_proto, varname, program_id, context, - common_accessor): + def _set( + self, accessor_proto, varname, program_id, context, common_accessor + ): main_program, startup_program, idx = get_program_by_id( - context, program_id) + context, program_id + ) embedding_dim = 0 for var in main_program.list_vars(): if var.name == varname: @@ -184,7 +204,8 @@ def _set(self, accessor_proto, varname, program_id, context, ctr_accessor_param.ssd_unseenday_threshold = 1 for sgd_param in [ - accessor_proto.embed_sgd_param, accessor_proto.embedx_sgd_param + accessor_proto.embed_sgd_param, + accessor_proto.embedx_sgd_param, ]: if not sgd_param.HasField("name"): if common_accessor.accessor_class == "sgd": @@ -194,7 +215,10 @@ def _set(self, accessor_proto, varname, program_id, context, else: # for fl-ps, because geo accessor is 'sum' sgd_param.name = "SparseAdamSGDRule" - if sgd_param.name == "SparseAdaGradSGDRule" or sgd_param.name == "StdAdaGradSGDRule": + if ( + sgd_param.name == "SparseAdaGradSGDRule" + or sgd_param.name == "StdAdaGradSGDRule" + ): if not sgd_param.adagrad.HasField("learning_rate"): sgd_param.adagrad.learning_rate = 0.05 if not sgd_param.adagrad.HasField("initial_g2sum"): @@ -206,42 +230,52 @@ def _set(self, accessor_proto, varname, program_id, context, if sgd_param.name == "SparseNaiveSGDRule": if not sgd_param.naive.HasField("learning_rate"): - learning_rate = common_accessor.initializers[-1].split( - "&")[1] + learning_rate = common_accessor.initializers[-1].split("&")[ + 1 + ] sgd_param.naive.learning_rate = float(learning_rate) if not sgd_param.naive.HasField("initial_range"): - initial_range = common_accessor.initializers[0].split( - "&")[-1] + initial_range = common_accessor.initializers[0].split("&")[ + -1 + ] sgd_param.naive.initial_range = float(initial_range) if len(sgd_param.naive.weight_bounds) == 0: sgd_param.naive.weight_bounds.extend([-10.0, 10.0]) - if sgd_param.name == "SparseAdamSGDRule" or sgd_param.name == "SparseSharedAdamSGDRule": + if ( + sgd_param.name == "SparseAdamSGDRule" + or sgd_param.name == "SparseSharedAdamSGDRule" + ): if not sgd_param.adam.HasField("learning_rate"): - learning_rate = common_accessor.initializers[-1].split( - "&")[1] + learning_rate = common_accessor.initializers[-1].split("&")[ + 1 + ] sgd_param.adam.learning_rate = float(learning_rate) if not sgd_param.adam.HasField("initial_range"): - initial_range = common_accessor.initializers[0].split( - "&")[-1] + initial_range = common_accessor.initializers[0].split("&")[ + -1 + ] sgd_param.adam.initial_range = float(initial_range) attr_list = [x.split("&") for x in common_accessor.attrs] - if not sgd_param.adam.HasField( - "beta1_decay_rate" - ) and common_accessor.accessor_class == "adam": + if ( + not sgd_param.adam.HasField("beta1_decay_rate") + and common_accessor.accessor_class == "adam" + ): sgd_param.adam.beta1_decay_rate = float(attr_list[0][1]) else: sgd_param.adam.beta1_decay_rate = 0.9 - if not sgd_param.adam.HasField( - "beta2_decay_rate" - ) and common_accessor.accessor_class == "adam": + if ( + not sgd_param.adam.HasField("beta2_decay_rate") + and common_accessor.accessor_class == "adam" + ): sgd_param.adam.beta2_decay_rate = float(attr_list[1][1]) else: sgd_param.adam.beta2_decay_rate = 0.999 - if not sgd_param.adam.HasField( - "ada_epsilon" - ) and common_accessor.accessor_class == "adam": + if ( + not sgd_param.adam.HasField("ada_epsilon") + and common_accessor.accessor_class == "adam" + ): sgd_param.adam.ada_epsilon = float(attr_list[2][1]) else: sgd_param.adam.ada_epsilon = 1e-08 @@ -250,7 +284,6 @@ def _set(self, accessor_proto, varname, program_id, context, class CommonAccessor(Accessor): - def __init__(self): super(CommonAccessor, self).__init__() self.table_name = '' @@ -269,27 +302,46 @@ def __init__(self): def define_optimize_map(self): opt_input_map = {} opt_input_map["sgd"] = [("Param", None), ("LearningRate", 1)] - opt_input_map["adam"] = [("Param", None), ("Moment1", None), - ("Moment2", None), ("Beta1Pow", 1), - ("Beta2Pow", 1), ("LearningRate", 1)] - opt_input_map["adam_d2sum"] = [("Param", None), ("D2Sum", None), - ("G2Sum", None), ("Moment", None), - ("MomentDecayRate", 1), - ("AdaDecayRate", 1), ("AdaEpsilon", 1), - ("LearningRate", 1)] + opt_input_map["adam"] = [ + ("Param", None), + ("Moment1", None), + ("Moment2", None), + ("Beta1Pow", 1), + ("Beta2Pow", 1), + ("LearningRate", 1), + ] + opt_input_map["adam_d2sum"] = [ + ("Param", None), + ("D2Sum", None), + ("G2Sum", None), + ("Moment", None), + ("MomentDecayRate", 1), + ("AdaDecayRate", 1), + ("AdaEpsilon", 1), + ("LearningRate", 1), + ] opt_input_map["sum"] = [("Param", None)] - opt_input_map["naive_adagrad"] = [("Param", None), ("G2Sum", 1), - ("LearningRate", 1)] + opt_input_map["naive_adagrad"] = [ + ("Param", None), + ("G2Sum", 1), + ("LearningRate", 1), + ] opt_input_map["summary"] = [("Param", None), ("SummaryDecayRate", 1)] opt_attr_map = {} opt_attr_map["sgd"] = [] opt_attr_map["sum"] = [] opt_attr_map["naive_adagrad"] = [] - opt_attr_map["adam"] = [("beta1", "f"), ("beta2", "f"), - ("epsilon", "f")] - opt_attr_map["adam_d2sum"] = [("beta1", "f"), ("beta2", "f"), - ("epsilon", "f")] + opt_attr_map["adam"] = [ + ("beta1", "f"), + ("beta2", "f"), + ("epsilon", "f"), + ] + opt_attr_map["adam_d2sum"] = [ + ("beta1", "f"), + ("beta2", "f"), + ("epsilon", "f"), + ] opt_attr_map["summary"] = [("summary_decay_rate", "f")] opt_init_map = {} @@ -304,7 +356,8 @@ def define_optimize_map(self): def parse_entry(self, varname, program_id, context): main_program, startup_program, idx = get_program_by_id( - context, program_id) + context, program_id + ) for op in main_program.global_block().ops: if not is_distributed_sparse_op(op) and not is_sparse_op(op): continue @@ -337,8 +390,10 @@ def get_initializer_attr(self, value_name, o_startup_program): origin_var_name = value_name # print("get_initializer_attr param name:", value_name) for op in o_startup_program.global_block().ops: - if op.type in self.opt_init_map.keys( - ) and origin_var_name == op.output("Out")[0]: + if ( + op.type in self.opt_init_map.keys() + and origin_var_name == op.output("Out")[0] + ): init_attr = [op.type] # print("get_initializer_attr op type:", op.type) for attr in self.opt_init_map[op.type]: @@ -359,7 +414,8 @@ def parse_by_optimizer(self, ctx, context): # ctx.table_id(), ctx.is_datanorm_table())) main_program, startup_program, idx = get_program_by_id( - context, ctx.program_id()) + context, ctx.program_id() + ) pserver_id = get_role_id(context['role_maker']) pserver_num = len(get_ps_endpoints(context['role_maker'])) optimizer_ops = get_optimize_ops(main_program) @@ -369,8 +425,9 @@ def parse_by_optimizer(self, ctx, context): for op in optimizer_ops: if ("Param" in op.input_names) and ( - op.input("Param")[0] - == context['grad_name_to_param_name'][grad_name]): + op.input("Param")[0] + == context['grad_name_to_param_name'][grad_name] + ): oop = op break @@ -409,7 +466,8 @@ def parse_by_optimizer(self, ctx, context): else: if oop.type != 'sgd' and oop.type != 'adam': raise ValueError( - "The dense optimizer in PS is only supported SGD or Adam!") + "The dense optimizer in PS is only supported SGD or Adam!" + ) param_varnames = self.opt_input_map[oop.type] attr_varnames = self.opt_attr_map[oop.type] self.accessor_class = oop.type @@ -417,7 +475,7 @@ def parse_by_optimizer(self, ctx, context): for (formal_name, shape) in param_varnames: params.append(formal_name) if self.accessor_class == "adam_d2sum": - #for dims + # for dims if shape is None: if is_sparse: shape = single_dim @@ -425,19 +483,24 @@ def parse_by_optimizer(self, ctx, context): shape = self.get_shard(size, pserver_num, pserver_id) dims.append(shape) - #for initializers + # for initializers if formal_name == "Param" or formal_name == "LearningRate": - param = main_program.global_block().vars[oop.input( - formal_name)[0]] - #TODO: for dense learning_rate, can be different from sparse lr - if formal_name == "LearningRate" and param.name != "learning_rate_" + str( - idx): + param = main_program.global_block().vars[ + oop.input(formal_name)[0] + ] + # TODO: for dense learning_rate, can be different from sparse lr + if ( + formal_name == "LearningRate" + and param.name != "learning_rate_" + str(idx) + ): warnings.warn("will support decay soon") param = main_program.global_block().vars[ - "learning_rate_" + str(idx)] + "learning_rate_" + str(idx) + ] initializer = self.get_initializer_attr( - param.name, startup_program) + param.name, startup_program + ) elif formal_name == "MomentDecayRate": initializer = "fill_constant&0.99" elif formal_name == "AdaDecayRate": @@ -448,7 +511,7 @@ def parse_by_optimizer(self, ctx, context): initializer = "fill_constant&0" initializers.append(initializer) elif self.accessor_class == "summary": - #for dims + # for dims if shape is None: if is_sparse: shape = single_dim @@ -456,13 +519,15 @@ def parse_by_optimizer(self, ctx, context): shape = self.get_shard(size, pserver_num, pserver_id) dims.append(shape) - #for initializers + # for initializers if formal_name == "Param": - param = main_program.global_block().vars[oop.input( - formal_name)[0]] + param = main_program.global_block().vars[ + oop.input(formal_name)[0] + ] initializer = self.get_initializer_attr( - param.name, startup_program) + param.name, startup_program + ) elif formal_name == "SummaryDecayRate": initializer = "fill_constant&0.999999" else: @@ -474,32 +539,39 @@ def parse_by_optimizer(self, ctx, context): initializer = "fill_constant&0" initializers.append(initializer) else: - param = main_program.global_block().vars[oop.input( - formal_name)[0]] - if formal_name == "LearningRate" and param.name != "learning_rate_" + str( - idx): + param = main_program.global_block().vars[ + oop.input(formal_name)[0] + ] + if ( + formal_name == "LearningRate" + and param.name != "learning_rate_" + str(idx) + ): warnings.warn("will support decay soon") param = main_program.global_block().vars[ - "learning_rate_" + str(idx)] + "learning_rate_" + str(idx) + ] if shape is None: if is_sparse: shape = single_dim else: - shape = self.get_shard(size, pserver_num, - pserver_id) + shape = self.get_shard( + size, pserver_num, pserver_id + ) dims.append(shape) initializer = self.get_initializer_attr( - param.name, startup_program) + param.name, startup_program + ) initializers.append(initializer) if self.accessor_class == 'summary': datanorm_ops = get_datanorm_ops(main_program) for op in datanorm_ops: if ("BatchSize" in op.input_names) and ( - op.input("BatchSize")[0] - == context['grad_name_to_param_name'][grad_name]): + op.input("BatchSize")[0] + == context['grad_name_to_param_name'][grad_name] + ): oop = op break @@ -528,23 +600,24 @@ def _set(self, proto): class Tensor: - def __init__(self, tesnor_dcit): self.tensor_dict = tesnor_dcit def _set(self, tensor_proto): tensor_proto.main_program_id = self.tensor_dict.get( - "main_program_id", 0) + "main_program_id", 0 + ) tensor_proto.startup_program_id = self.tensor_dict.get( - "startup_program_id", 0) + "startup_program_id", 0 + ) tensor_proto.feed_var_name = self.tensor_dict.get("feed_var_name", '') tensor_proto.fetch_var_name = self.tensor_dict.get("fetch_var_name", '') tensor_proto.tensor_table_class = self.tensor_dict.get( - "tensor_table_class", '') + "tensor_table_class", '' + ) class Table: - def __init__(self): self.table_class = None self.shard_num = -1 @@ -559,7 +632,6 @@ def _set(self, table_proto): class BarrierTable(Table): - def __init__(self, context, idx): super(BarrierTable, self).__init__() self.type = None @@ -595,7 +667,6 @@ def _set(self, table_proto): class TensorTable(Table): - def __init__(self, idx, tensor_dict, role_maker): super(TensorTable, self).__init__() self.idx = idx @@ -610,7 +681,8 @@ def _set(self, table_proto): table_proto.accessor.accessor_class = "CommMergeAccessor" table_proto.common.table_name = self.tensor_dict.get( - "feed_var_name", '') + "feed_var_name", '' + ) table_proto.common.trainer_num = get_trainers(self.role_maker) tensor = Tensor(self.tensor_dict) @@ -618,7 +690,6 @@ def _set(self, table_proto): class SparseTable(Table): - def __init__(self, context, send_ctx): super(SparseTable, self).__init__() self.context = context @@ -629,31 +700,39 @@ def __init__(self, context, send_ctx): def _set(self, table_proto): ctx = self.ctx - if ctx.is_tensor_table() or len( - ctx.origin_varnames()) < 1 or (ctx.is_sparse() == False): + if ( + ctx.is_tensor_table() + or len(ctx.origin_varnames()) < 1 + or (ctx.is_sparse() == False) + ): return table_proto.table_id = ctx.table_id() table_proto.table_class = self.table_class table_proto.type = the_one_ps_pb2.PS_SPARSE_TABLE table_proto.shard_num = self.shard_num if table_proto.sparse_table_cache_file_num > len( - get_ps_endpoints(self.context['role_maker'])): + get_ps_endpoints(self.context['role_maker']) + ): table_proto.sparse_table_cache_file_num = len( - get_ps_endpoints(self.context['role_maker'])) + get_ps_endpoints(self.context['role_maker']) + ) self.common.table_name = self.context['grad_name_to_param_name'][ - ctx.origin_varnames()[0]] + ctx.origin_varnames()[0] + ] self.common.parse_by_optimizer(ctx, self.context) - self.common.parse_entry(self.common.table_name, ctx.program_id(), - self.context) + self.common.parse_entry( + self.common.table_name, ctx.program_id(), self.context + ) self.common.sync = True if self.context['is_sync'] else False self.common._set(table_proto.common) print('new table_name: {}'.format(self.common.table_name)) all_table_proto = self.context[ - "user_defined_strategy"].sparse_table_configs + "user_defined_strategy" + ].sparse_table_configs usr_table_proto = all_table_proto.add() for proto in all_table_proto: if proto.table_name == self.common.table_name: @@ -679,11 +758,17 @@ def _set(self, table_proto): ) if usr_table_proto.HasField("enable_sparse_table_cache"): - table_proto.enable_sparse_table_cache = usr_table_proto.enable_sparse_table_cache + table_proto.enable_sparse_table_cache = ( + usr_table_proto.enable_sparse_table_cache + ) if usr_table_proto.HasField("sparse_table_cache_rate"): - table_proto.sparse_table_cache_rate = usr_table_proto.sparse_table_cache_rate + table_proto.sparse_table_cache_rate = ( + usr_table_proto.sparse_table_cache_rate + ) if usr_table_proto.HasField("sparse_table_cache_file_num"): - table_proto.sparse_table_cache_file_num = usr_table_proto.sparse_table_cache_file_num + table_proto.sparse_table_cache_file_num = ( + usr_table_proto.sparse_table_cache_file_num + ) if usr_table_proto.HasField("enable_revert"): table_proto.enable_revert = usr_table_proto.enable_revert if usr_table_proto.HasField("shard_merge_rate"): @@ -691,19 +776,29 @@ def _set(self, table_proto): if usr_table_proto.accessor.ByteSize() == 0: warnings.warn( - "The accessor of sparse table is not set, use default value.") + "The accessor of sparse table is not set, use default value." + ) table_proto.accessor.ParseFromString( - usr_table_proto.accessor.SerializeToString()) - self.accessor._set(table_proto.accessor, self.common.table_name, - ctx.program_id(), self.context, self.common) + usr_table_proto.accessor.SerializeToString() + ) + self.accessor._set( + table_proto.accessor, + self.common.table_name, + ctx.program_id(), + self.context, + self.common, + ) - check_embedding_dim(table_proto.accessor, self.common.table_name, - ctx.program_id(), self.context) + check_embedding_dim( + table_proto.accessor, + self.common.table_name, + ctx.program_id(), + self.context, + ) class GeoSparseTable(SparseTable): - def __init__(self, context, send_ctx): super(GeoSparseTable, self).__init__(context, send_ctx) self.table_class = "MemorySparseGeoTable" @@ -712,8 +807,11 @@ def __init__(self, context, send_ctx): def _set(self, table_proto): ctx = self.ctx - if ctx.is_tensor_table() or len( - ctx.origin_varnames()) < 1 or (ctx.is_sparse() == False): + if ( + ctx.is_tensor_table() + or len(ctx.origin_varnames()) < 1 + or (ctx.is_sparse() == False) + ): return table_proto.table_id = ctx.table_id() table_proto.table_class = self.table_class @@ -725,16 +823,17 @@ def _set(self, table_proto): table_proto.accessor.embedx_dim = ctx.sections()[1] self.common.table_name = self.context['grad_name_to_param_name'][ - ctx.origin_varnames()[0]] + ctx.origin_varnames()[0] + ] self.common.parse_by_optimizer(ctx, self.context) - self.common.parse_entry(self.common.table_name, ctx.program_id(), - self.context) + self.common.parse_entry( + self.common.table_name, ctx.program_id(), self.context + ) self.common.sync = False self.common._set(table_proto.common) class DenseTable(Table): - def __init__(self, context, send_ctx): super(DenseTable, self).__init__() self.context = context @@ -743,8 +842,11 @@ def __init__(self, context, send_ctx): def _set(self, table_proto): ctx = self.ctx - if ctx.is_tensor_table() or len( - ctx.origin_varnames()) < 1 or (ctx.is_sparse() == True): + if ( + ctx.is_tensor_table() + or len(ctx.origin_varnames()) < 1 + or (ctx.is_sparse() == True) + ): return table_proto.table_id = ctx.table_id() @@ -759,15 +861,15 @@ def _set(self, table_proto): self.common.table_name = "MergedDense" self.common.parse_by_optimizer(ctx, self.context) - self.common.parse_entry(self.common.table_name, ctx.program_id(), - self.context) + self.common.parse_entry( + self.common.table_name, ctx.program_id(), self.context + ) self.common.sync = True if self.context['is_sync'] else False self.common._set(table_proto.common) class Server: - def __init__(self): pass @@ -776,7 +878,6 @@ def _set(self): class DownpourServer(Server): - def __init__(self): super(DownpourServer, self).__init__() @@ -785,7 +886,6 @@ def _set(self): class Worker: - def __init__(self): pass @@ -794,7 +894,6 @@ def _set(self): class DownpourWorker(Worker): - def __init__(self): super(DownpourWorker, self).__init__() @@ -803,7 +902,6 @@ def _set(self): class fsClient: - def __init__(self, fs_client_param): self.fs_client_param = fs_client_param @@ -817,7 +915,6 @@ def _set(self, proto): class PsDescBuilder(object): - def __init__(self, context): self.context = context self.is_sync = context['is_sync'] @@ -827,7 +924,8 @@ def __init__(self, context): self.barrier_table_id = None self.send_ctx = get_the_one_send_context( - self.context, split_dense_table=self.is_heter_ps_mode) + self.context, split_dense_table=self.is_heter_ps_mode + ) self.tensor_table_dict = {} # TODO self._server_sub_program = [] @@ -846,8 +944,11 @@ def _get_tensor_tables(self): self._server_sub_program.append(Program().desc) tables = [] for table_name in self.tensor_table_dict: - tables.append(globals()['TensorTable'](len(tables), tensor_dict, - self.context['role_maker'])) + tables.append( + globals()['TensorTable']( + len(tables), tensor_dict, self.context['role_maker'] + ) + ) program_idx += 1 return tables @@ -857,14 +958,17 @@ def _get_tables(self): print("idx, name, ctx:", idx, name, ctx) if ctx.is_sparse(): if self.ps_mode == DistributedMode.GEO: - if (self.context['local_sparse'] - and name[:-5] in self.context['local_sparse']) or ( - not self.context['local_sparse']): - tables.append(globals()['GeoSparseTable'](self.context, - ctx)) + if ( + self.context['local_sparse'] + and name[:-5] in self.context['local_sparse'] + ) or (not self.context['local_sparse']): + tables.append( + globals()['GeoSparseTable'](self.context, ctx) + ) else: - tables.append(globals()['SparseTable'](self.context, - ctx)) + tables.append( + globals()['SparseTable'](self.context, ctx) + ) else: tables.append(globals()['SparseTable'](self.context, ctx)) else: @@ -888,37 +992,45 @@ def build_fl_client_desc(self, client_info): def build_worker_desc(self): for table in self.tables: - table_proto = self.ps_desc.worker_param.downpour_worker_param.downpour_table_param.add( + table_proto = ( + self.ps_desc.worker_param.downpour_worker_param.downpour_table_param.add() ) table._set(table_proto) - table_proto = self.ps_desc.server_param.downpour_server_param.downpour_table_param.add( + table_proto = ( + self.ps_desc.server_param.downpour_server_param.downpour_table_param.add() ) table._set(table_proto) if type(table) == BarrierTable and self.barrier_table_id is None: self.barrier_table_id = table.idx self.service._set( - self.ps_desc.server_param.downpour_server_param.service_param) + self.ps_desc.server_param.downpour_server_param.service_param + ) self.fs_client._set(self.ps_desc.fs_client_param) return text_format.MessageToString(self.ps_desc) def build_server_desc(self): self.sparse_table_maps = {} for table in self.tables: - table_proto = self.ps_desc.server_param.downpour_server_param.downpour_table_param.add( + table_proto = ( + self.ps_desc.server_param.downpour_server_param.downpour_table_param.add() ) table._set(table_proto) - if table_proto.type == the_one_ps_pb2.PS_SPARSE_TABLE and table_proto.common is not None: + if ( + table_proto.type == the_one_ps_pb2.PS_SPARSE_TABLE + and table_proto.common is not None + ): self.sparse_table_maps[ - table_proto.common.table_name] = table_proto.table_id + table_proto.common.table_name + ] = table_proto.table_id self.service._set( - self.ps_desc.server_param.downpour_server_param.service_param) + self.ps_desc.server_param.downpour_server_param.service_param + ) self.fs_client._set(self.ps_desc.fs_client_param) return text_format.MessageToString(self.ps_desc) class TheOnePSRuntime(RuntimeBase): - def __init__(self): super(TheOnePSRuntime, self).__init__() self._communicator = None @@ -936,30 +1048,41 @@ def _set_basic_info(self, context): self.debug = bool(int(os.getenv("PSERVER_DEBUG", "0"))) self.origin_main_program = context["origin_main_program"] - self.origin_main_programs = context.get("origin_main_programs", - [self.origin_main_program]) + self.origin_main_programs = context.get( + "origin_main_programs", [self.origin_main_program] + ) self.context["origin_main_programs"] = self.origin_main_programs self.context["origin_startup_programs"] = context.get( - 'origin_startup_programs', [context['origin_startup_program']]) + 'origin_startup_programs', [context['origin_startup_program']] + ) self.context[ - 'is_heter_ps_mode'] = self.role_maker._is_heter_parameter_server_mode + 'is_heter_ps_mode' + ] = self.role_maker._is_heter_parameter_server_mode self.is_heter_ps_mode = self.context['is_heter_ps_mode'] self.context['trainer'] = TrainerRuntimeConfig( - context['valid_strategy']) + context['valid_strategy'] + ) self.context['ps_mode'] = self.context['trainer'].mode self.context['use_ps_gpu'] = context['valid_strategy'].a_sync_configs[ - 'use_ps_gpu'] - self.context['is_sync'] = True if self.context[ - 'ps_mode'] == DistributedMode.SYNC else False + 'use_ps_gpu' + ] + self.context['is_sync'] = ( + True if self.context['ps_mode'] == DistributedMode.SYNC else False + ) self.context['grad_name_to_param_name'] = {} self.context['tensor_table'] = {} # FL self.context['local_sparse'] = context[ - "user_defined_strategy"].trainer_desc_configs["local_sparse"] + "user_defined_strategy" + ].trainer_desc_configs["local_sparse"] self.context['remote_sparse'] = context[ - "user_defined_strategy"].trainer_desc_configs["remote_sparse"] - print("fl-ps > local_sparse: {}, remote_sparse: {}".format( - self.context['local_sparse'], self.context['remote_sparse'])) + "user_defined_strategy" + ].trainer_desc_configs["remote_sparse"] + print( + "fl-ps > local_sparse: {}, remote_sparse: {}".format( + self.context['local_sparse'], self.context['remote_sparse'] + ) + ) build_var_distributed(self.context) @@ -993,7 +1116,7 @@ def _init_all_params(self, scopes, send_ctx, recv_map): scope = scopes[idx] table_id = ctx.table_id() var_names = recv_map[table_id] - #print("init params:", idx, table_id, var_names) + # print("init params:", idx, table_id, var_names) self._worker.push_dense_params(scope, table_id, var_names) all_var_names.extend(var_names) return all_var_names @@ -1007,7 +1130,7 @@ def _pull_all_dense(self, scopes, send_ctx, recv_map): scope = scopes[idx] table_id = ctx.table_id() var_names = recv_map[table_id] - #print("pull all dense:", idx, table_id, var_names) + # print("pull all dense:", idx, table_id, var_names) self._worker.pull_dense_params(scope, table_id, var_names) all_var_names.extend(var_names) return all_var_names @@ -1056,16 +1179,19 @@ def _init_worker(self, scopes=None): def sync_strategy_envs(): kwargs = {} kwargs[ - "pserver_endpoints"] = self.role_maker._get_pserver_endpoints() + "pserver_endpoints" + ] = self.role_maker._get_pserver_endpoints() kwargs["trainer_id"] = self.role_maker._worker_index() return kwargs dense_map = get_the_one_recv_context( - self.context, split_dense_table=self.is_heter_ps_mode) + self.context, split_dense_table=self.is_heter_ps_mode + ) send_ctx = get_the_one_send_context( self.context, split_dense_table=self.is_heter_ps_mode, - ep_list=self.endpoints) + ep_list=self.endpoints, + ) self._send_ctx = send_ctx trainer_config = self.context['trainer'] @@ -1097,24 +1223,34 @@ def sync_strategy_envs(): print("fl-ps > with_coordinator? {}".format(self.with_coordinator)) print("fl-ps > coordinator addr: {}".format(self.coordinator_hosts)) if self.with_coordinator: - self._worker.init_fl_worker(self.coordinator_hosts, self.role_id, - self.trainer_endpoint) + self._worker.init_fl_worker( + self.coordinator_hosts, self.role_id, self.trainer_endpoint + ) - if self.context[ - 'ps_mode'] == DistributedMode.GEO or self.is_heter_ps_mode: + if ( + self.context['ps_mode'] == DistributedMode.GEO + or self.is_heter_ps_mode + ): self._communicator = Communicator( - trainer_config.mode, kwargs, - trainer_config.get_communicator_flags()) - self._communicator.init_with_ctx(send_ctx, dense_map, worker_desc, - self.string_hosts, - fluid.global_scope()) + trainer_config.mode, + kwargs, + trainer_config.get_communicator_flags(), + ) + self._communicator.init_with_ctx( + send_ctx, + dense_map, + worker_desc, + self.string_hosts, + fluid.global_scope(), + ) fleet.util.barrier() # info = self._communicator.get_client_info() info = self._worker.get_client_info() if isinstance(info, list) and len(info) > 0: all_info = self.role_maker._all_gather( - info[0]) # 收集其他 client 的 service 地址 + info[0] + ) # 收集其他 client 的 service 地址 # for unittest if not isinstance(all_info, list): warnings.warn("gloo may not initialize correctly") @@ -1143,8 +1279,10 @@ def sync_strategy_envs(): self.scopes = scopes if not is_test: - if self.context[ - 'ps_mode'] == DistributedMode.GEO or self.is_heter_ps_mode == True: + if ( + self.context['ps_mode'] == DistributedMode.GEO + or self.is_heter_ps_mode == True + ): self._communicator.init_params(dense_map) else: if not self.context['use_ps_gpu']: @@ -1158,8 +1296,10 @@ def sync_strategy_envs(): self._pull_all_dense(scopes, send_ctx, dense_map) fleet.util.barrier() - if self.context[ - 'ps_mode'] == DistributedMode.GEO or self.is_heter_ps_mode == True: + if ( + self.context['ps_mode'] == DistributedMode.GEO + or self.is_heter_ps_mode == True + ): if not self._communicator.is_running(): self._communicator.start() else: @@ -1169,8 +1309,10 @@ def sync_strategy_envs(): launch_barrier_flag = int(os.getenv("FLAGS_LAUNCH_BARRIER", "1")) if launch_barrier and launch_barrier_flag: wait_server_ready(self.role_maker._get_pserver_endpoints()) - if self.is_heter_ps_mode and self.role_maker._get_next_trainers( - ) != []: + if ( + self.is_heter_ps_mode + and self.role_maker._get_next_trainers() != [] + ): wait_server_ready(self.role_maker._get_next_trainers()) if self.is_heter_ps_mode: previous_trainers = [] @@ -1180,8 +1322,8 @@ def sync_strategy_envs(): if self.role_maker._get_next_trainers() != []: next_trainers = self.role_maker._get_next_trainers() self._heter_client = HeterClient( - next_trainers, previous_trainers, - self.role_maker._role_id()) # --> HeterClient::GetInstance + next_trainers, previous_trainers, self.role_maker._role_id() + ) # --> HeterClient::GetInstance def _init_coordinator(self, scopes=None): if self._coordinator == None: @@ -1189,12 +1331,13 @@ def _init_coordinator(self, scopes=None): print(">>> curr node ip: {}".format(self.coordinator_hosts[0])) print(">>> all trainer endpoints: {}".format(self.trainer_endpoints)) - self._coordinator.start_coordinator(self.coordinator_hosts[0], - self.trainer_endpoints) + self._coordinator.start_coordinator( + self.coordinator_hosts[0], self.trainer_endpoints + ) def _make_fl_strategy(self): if self._coordinator == None: - assert ("Coordinator py object is null!") + assert "Coordinator py object is null!" else: self._coordinator.make_fl_strategy() @@ -1208,12 +1351,18 @@ def _init_server(self, dirname=None, var_names=None, **kwargs): print("server_desc: \n{}".format(server_desc)) self._server = fluid.core.DistFleetWrapper() - self._server.init_server(server_desc, self.string_hosts, self.role_id, - trainers, self._server_sub_program) + self._server.init_server( + server_desc, + self.string_hosts, + self.role_id, + trainers, + self._server_sub_program, + ) dist_varnames = get_sparse_tablenames(self.origin_main_programs, True) - sparse_varnames = get_sparse_tablenames(self.origin_main_programs, - False) + sparse_varnames = get_sparse_tablenames( + self.origin_main_programs, False + ) distributed_varnames = dist_varnames + sparse_varnames @@ -1223,8 +1372,10 @@ def _init_server(self, dirname=None, var_names=None, **kwargs): for var_name in var_names: if var_name not in distributed_varnames: raise ValueError( - "fleet.init server can only load sparse variables in {}" - .format(distributed_varnames)) + "fleet.init server can only load sparse variables in {}".format( + distributed_varnames + ) + ) load_varnames = var_names if dirname is None or not load_varnames: @@ -1249,17 +1400,19 @@ def _stop_worker(self): self._communicator.stop() self._worker.stop_worker() if self.is_heter_ps_mode: - assert self._heter_client != None, "heter client should not be None in heterps mode" + assert ( + self._heter_client != None + ), "heter client should not be None in heterps mode" self._heter_client.stop() @staticmethod def __exclude_vars(exclude_var_names=[]): - def is_valid(var): if var.name in exclude_var_names: return False from .utils.public import _get_varname_parts + origin_varname, _, _ = _get_varname_parts(var.name) if origin_varname.endswith("@GRAD"): return False @@ -1267,9 +1420,11 @@ def is_valid(var): if origin_varname.startswith("learning_rate_"): return False - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable @@ -1282,35 +1437,35 @@ def _get_inference_model_path(self, dirname): model_path = os.path.join(dirname, "dnn_plugin") return model_path - def _ps_save_dense_params(self, - executor, - dirname, - scope, - program, - var_names=None): + def _ps_save_dense_params( + self, executor, dirname, scope, program, var_names=None + ): dense_map = get_the_one_recv_context( - self.context, split_dense_table=self.is_heter_ps_mode) + self.context, split_dense_table=self.is_heter_ps_mode + ) send_ctx = get_the_one_send_context( self.context, split_dense_table=self.is_heter_ps_mode, - ep_list=self.endpoints) + ep_list=self.endpoints, + ) if program is None or len(self.origin_main_programs) == 1: program = self.origin_main_programs[0] dense_var_names = self._pull_dense(program, scope, send_ctx, dense_map) save_var_names = dense_var_names if var_names is None else var_names vars = [program.global_block().var(i) for i in save_var_names] import paddle + with paddle.static.scope_guard(scope): - paddle.static.save_vars(executor, - "./", - program, - vars=vars, - filename=dirname) - - def _save_sparse_params(self, executor, dirname, context, main_program, - mode): - distributed_varnames = get_sparse_tablenames(self.origin_main_programs, - True) + paddle.static.save_vars( + executor, "./", program, vars=vars, filename=dirname + ) + + def _save_sparse_params( + self, executor, dirname, context, main_program, mode + ): + distributed_varnames = get_sparse_tablenames( + self.origin_main_programs, True + ) values = [] model_path = self._get_inference_model_path(dirname) for id, names in context.items(): @@ -1326,12 +1481,9 @@ def _save_sparse_params(self, executor, dirname, context, main_program, # self._worker.save_all_model(dirname, mode) return values - def _save_distributed_persistables(self, - executor, - dirname, - main_program=None, - mode=0, - **kwargs): + def _save_distributed_persistables( + self, executor, dirname, main_program=None, mode=0, **kwargs + ): """ This function filters out all variables with `persistable==True` from the give `main_program` and then saves these variables to the folder `dirname` @@ -1350,7 +1502,8 @@ def _save_distributed_persistables(self, if not isinstance(executor, Executor): raise TypeError( - "in fleet.save() function, executor must be as Executor type") + "in fleet.save() function, executor must be as Executor type" + ) if main_program is None: main_program = self.context['origin_main_program'] @@ -1362,14 +1515,16 @@ def _save_distributed_persistables(self, self._worker.save_all_model(dirname, mode) - def _ps_inference_save_inference_model(self, - executor, - dirname, - feeded_var_names, - target_vars, - main_program=None, - export_for_deployment=True, - mode=0): + def _ps_inference_save_inference_model( + self, + executor, + dirname, + feeded_var_names, + target_vars, + main_program=None, + export_for_deployment=True, + mode=0, + ): """ Prune the given `main_program` to build a new program especially for inference, and then save it and all related parameters to given `dirname` by the `executor`. @@ -1382,11 +1537,16 @@ def _ps_inference_save_inference_model(self, if not isinstance(executor, Executor): raise TypeError( - "in fleet.save() function, executor must be as Executor type") + "in fleet.save() function, executor must be as Executor type" + ) import paddle - program = self.origin_main_programs[ - 0] if main_program is None else main_program + + program = ( + self.origin_main_programs[0] + if main_program is None + else main_program + ) _, _, idx = get_program_by_id(self.context, id(program)) scope = self.scopes[idx] print("save inference model scope idx:", idx) @@ -1400,8 +1560,9 @@ def _ps_inference_save_inference_model(self, program.global_block().var(name) for name in feeded_var_names ] - infer_program = paddle.static.normalize_program(program, feed_vars, - target_vars) + infer_program = paddle.static.normalize_program( + program, feed_vars, target_vars + ) infer_program._copy_dist_param_info_from(program) @@ -1413,30 +1574,40 @@ def _ps_inference_save_inference_model(self, sparses = get_the_one_recv_context( self.context, is_dense=False, - split_dense_table=self.is_heter_ps_mode) - sparse_names = self._save_sparse_params(executor, dirname, sparses, - main_program, mode) + split_dense_table=self.is_heter_ps_mode, + ) + sparse_names = self._save_sparse_params( + executor, dirname, sparses, main_program, mode + ) dense_map = get_the_one_recv_context( - self.context, split_dense_table=self.is_heter_ps_mode) + self.context, split_dense_table=self.is_heter_ps_mode + ) send_ctx = get_the_one_send_context( self.context, split_dense_table=self.is_heter_ps_mode, - ep_list=self.endpoints) + ep_list=self.endpoints, + ) self._pull_dense(program, scope, send_ctx, dense_map) generate_vars = self.context[ - "user_defined_strategy"].trainer_desc_configs["stat_var_names"] + "user_defined_strategy" + ].trainer_desc_configs["stat_var_names"] generate_vars = [var for var in generate_vars] remaining_vars = list( - filter(TheOnePSRuntime.__exclude_vars(sparse_names), - infer_program.list_vars())) + filter( + TheOnePSRuntime.__exclude_vars(sparse_names), + infer_program.list_vars(), + ) + ) for var in remaining_vars: tensor = var.get_value(scope) - paddle.save(tensor, - os.path.join(model_path, var.name), - use_binary_format=True) + paddle.save( + tensor, + os.path.join(model_path, var.name), + use_binary_format=True, + ) def _save_cache_model(self, dirname, **kwargs): mode = kwargs.get("mode", 1) @@ -1447,7 +1618,7 @@ def _save_cache_model(self, dirname, **kwargs): if self.role_maker._is_first_worker(): cache_threshold = self._worker.get_cache_threshold(table_id) - #check cache threshold right or not + # check cache threshold right or not fleet.util.barrier() if self.role_maker._is_first_worker(): @@ -1469,8 +1640,9 @@ def _check_save_pre_patch_done(self): fleet.util.barrier() def _load_sparse_params(self, dirname, context, main_program, mode): - distributed_varnames = get_sparse_tablenames(self.origin_main_programs, - True) + distributed_varnames = get_sparse_tablenames( + self.origin_main_programs, True + ) values = [] for id, names in context.items(): if names[0] not in distributed_varnames: @@ -1481,12 +1653,14 @@ def _load_sparse_params(self, dirname, context, main_program, mode): values.extend(names) return values - def _ps_inference_load_inference_model(self, - dirname, - mode=0, - main_program=None): - main_program = self.origin_main_programs[ - 0] if main_program is None else main_program + def _ps_inference_load_inference_model( + self, dirname, mode=0, main_program=None + ): + main_program = ( + self.origin_main_programs[0] + if main_program is None + else main_program + ) _, _, idx = get_program_by_id(self.context, id(main_program)) scope = self.scopes[idx] print("load inference model scope idx:", idx) @@ -1499,17 +1673,21 @@ def _ps_inference_load_inference_model(self, sparses = get_the_one_recv_context( self.context, is_dense=False, - split_dense_table=self.is_heter_ps_mode) + split_dense_table=self.is_heter_ps_mode, + ) - sparse_varnames = self._load_sparse_params(dirname, sparses, - main_program, mode) + sparse_varnames = self._load_sparse_params( + dirname, sparses, main_program, mode + ) dense_map = get_the_one_recv_context( - self.context, split_dense_table=self.is_heter_ps_mode) + self.context, split_dense_table=self.is_heter_ps_mode + ) send_ctx = get_the_one_send_context( self.context, split_dense_table=self.is_heter_ps_mode, - ep_list=self.endpoints) + ep_list=self.endpoints, + ) recv_dense_varnames = [] for _, names in dense_map.items(): @@ -1518,11 +1696,15 @@ def _ps_inference_load_inference_model(self, loaded_varnames = sparse_varnames remaining_vars = list( - filter(TheOnePSRuntime.__exclude_vars(loaded_varnames), - main_program.list_vars())) + filter( + TheOnePSRuntime.__exclude_vars(loaded_varnames), + main_program.list_vars(), + ) + ) model_path = self._get_inference_model_path(dirname) import paddle + for var in remaining_vars: if var.name not in recv_dense_varnames: continue @@ -1586,8 +1768,8 @@ def _shrink(self, threshold=None): sparses = get_the_one_recv_context( self.context, is_dense=False, - split_dense_table=self.role_maker. - _is_heter_parameter_server_mode) + split_dense_table=self.role_maker._is_heter_parameter_server_mode, + ) for id, names in sparses.items(): self._worker.shrink_sparse_table(id, threshold) diff --git a/python/paddle/distributed/ps/utils/ps_factory.py b/python/paddle/distributed/ps/utils/ps_factory.py index 1f522ec27f82c..0726fe15dc4cd 100755 --- a/python/paddle/distributed/ps/utils/ps_factory.py +++ b/python/paddle/distributed/ps/utils/ps_factory.py @@ -16,14 +16,18 @@ from .public import * # noqa: F403 __all__ = [ - 'PsProgramBuilder', 'GeoPsProgramBuilder', 'CpuSyncPsProgramBuilder', - 'CpuAsyncPsProgramBuilder', 'GpuPsProgramBuilder', - 'HeterAsyncPsProgramBuilder', 'FlPsProgramBuilder', 'NuPsProgramBuilder' + 'PsProgramBuilder', + 'GeoPsProgramBuilder', + 'CpuSyncPsProgramBuilder', + 'CpuAsyncPsProgramBuilder', + 'GpuPsProgramBuilder', + 'HeterAsyncPsProgramBuilder', + 'FlPsProgramBuilder', + 'NuPsProgramBuilder', ] class PsProgramBuilderFactory(object): - def __init__(self): pass diff --git a/python/paddle/distributed/ps/utils/public.py b/python/paddle/distributed/ps/utils/public.py index c5835c730eeb3..75182a497213c 100755 --- a/python/paddle/distributed/ps/utils/public.py +++ b/python/paddle/distributed/ps/utils/public.py @@ -22,9 +22,9 @@ from paddle.fluid import core import paddle.fluid.framework as framework -#logging.basicConfig( +# logging.basicConfig( # format='%(levelname)s - %(asctime)s - %(pathname)s: %(lineno)s - %(message)s', level=logging.INFO) -#logger = logging.getLogger(__name__) +# logger = logging.getLogger(__name__) OP_NAME_SCOPE = "op_namescope" CLIP_OP_NAME_SCOPE = "gradient_clip" @@ -47,7 +47,7 @@ SPARSE_OP_TYPE_DICT = {"lookup_table": "W", "lookup_table_v2": "W"} SPARSE_GRAD_OP_TYPE_DICT = { "lookup_table_grad": "W", - "lookup_table_v2_grad": "W" + "lookup_table_v2_grad": "W", } DEFAULT_DEVICE = 'cpu' @@ -58,13 +58,13 @@ def logger_config(log_path, logging_name): logger = logging.getLogger(logging_name) logger.setLevel(level=logging.WARNING) - handler = logging.FileHandler(log_path, - mode='a', - encoding='UTF-8', - delay=True) + handler = logging.FileHandler( + log_path, mode='a', encoding='UTF-8', delay=True + ) handler.setLevel(logging.INFO) formatter = logging.Formatter( - '%(levelname)s - %(asctime)s - %(pathname)s: %(lineno)s - %(message)s') + '%(levelname)s - %(asctime)s - %(pathname)s: %(lineno)s - %(message)s' + ) handler.setFormatter(formatter) console = logging.StreamHandler() console.setLevel(logging.DEBUG) @@ -74,8 +74,9 @@ def logger_config(log_path, logging_name): ps_log_root_dir = './ps_log/' -logger = logger_config(log_path='./ps_usr_print_log', - logging_name='ps_usr_print_log') +logger = logger_config( + log_path='./ps_usr_print_log', logging_name='ps_usr_print_log' +) class DistributedMode: @@ -88,7 +89,6 @@ class DistributedMode: class TrainerRuntimeConfig(object): - def __init__(self, valid_strategy): self.mode = None num_threads = os.getenv("CPU_NUM", "1") @@ -107,21 +107,28 @@ def __init__(self, valid_strategy): self.runtime_configs = {} self.runtime_configs['communicator_max_merge_var_num'] = os.getenv( - "FLAGS_communicator_max_merge_var_num", send_queue_size) + "FLAGS_communicator_max_merge_var_num", send_queue_size + ) self.runtime_configs['communicator_send_queue_size'] = os.getenv( - "FLAGS_communicator_send_queue_size", send_queue_size) + "FLAGS_communicator_send_queue_size", send_queue_size + ) self.runtime_configs[ - 'communicator_independent_recv_thread'] = os.getenv( - "FLAGS_communicator_independent_recv_thread", "1") + 'communicator_independent_recv_thread' + ] = os.getenv("FLAGS_communicator_independent_recv_thread", "1") self.runtime_configs[ - 'communicator_min_send_grad_num_before_recv'] = os.getenv( - "FLAGS_communicator_min_send_grad_num_before_recv", num_threads) + 'communicator_min_send_grad_num_before_recv' + ] = os.getenv( + "FLAGS_communicator_min_send_grad_num_before_recv", num_threads + ) self.runtime_configs['communicator_thread_pool_size'] = os.getenv( - "FLAGS_communicator_thread_pool_size", "5") + "FLAGS_communicator_thread_pool_size", "5" + ) self.runtime_configs['communicator_send_wait_times'] = os.getenv( - "FLAGS_communicator_send_wait_times", "5") + "FLAGS_communicator_send_wait_times", "5" + ) self.runtime_configs['communicator_is_sgd_optimizer'] = os.getenv( - "FLAGS_communicator_is_sgd_optimizer", "1") + "FLAGS_communicator_is_sgd_optimizer", "1" + ) def get_communicator_flags(self): need_keys = [] @@ -130,46 +137,62 @@ def get_communicator_flags(self): if self.mode is None or self.mode == DistributedMode.ASYNC: need_keys = self.runtime_configs.keys() mode_str = "async" - elif self.mode == DistributedMode.SYNC or self.mode == DistributedMode.HALF_ASYNC: + elif ( + self.mode == DistributedMode.SYNC + or self.mode == DistributedMode.HALF_ASYNC + ): mode_str = "sync or half_async" need_keys = [ 'communicator_max_merge_var_num', - 'communicator_send_wait_times', 'communicator_thread_pool_size', - 'communicator_send_queue_size' + 'communicator_send_wait_times', + 'communicator_thread_pool_size', + 'communicator_send_queue_size', ] elif self.mode == DistributedMode.GEO: mode_str = "GEO" need_keys = [ - 'communicator_thread_pool_size', 'communicator_send_wait_times', - 'communicator_max_merge_var_num', 'communicator_send_queue_size' + 'communicator_thread_pool_size', + 'communicator_send_wait_times', + 'communicator_max_merge_var_num', + 'communicator_send_queue_size', ] else: raise ValueError("Unsupported Mode") - if self.mode == DistributedMode.SYNC or self.mode == DistributedMode.HALF_ASYNC: + if ( + self.mode == DistributedMode.SYNC + or self.mode == DistributedMode.HALF_ASYNC + ): max_merge_var_num = self.runtime_configs[ - 'communicator_max_merge_var_num'] + 'communicator_max_merge_var_num' + ] send_queue_size = self.runtime_configs[ - 'communicator_send_queue_size'] + 'communicator_send_queue_size' + ] if max_merge_var_num != num_threads: print( 'WARNING: In {} mode, communicator_max_merge_var_num ' 'must be equal to CPU_NUM. But received, ' 'communicator_max_merge_var_num = {}, CPU_NUM = ' - '{}. communicator_max_merge_var_num will be forced to {}.'. - format(mode_str, max_merge_var_num, num_threads, - num_threads)) + '{}. communicator_max_merge_var_num will be forced to {}.'.format( + mode_str, max_merge_var_num, num_threads, num_threads + ) + ) self.runtime_configs[ - 'communicator_max_merge_var_num'] = num_threads + 'communicator_max_merge_var_num' + ] = num_threads if send_queue_size != num_threads: - print('WARNING: In {} mode, communicator_send_queue_size ' - 'must be equal to CPU_NUM. But received, ' - 'communicator_send_queue_size = {}, CPU_NUM = ' - '{}. communicator_send_queue_size will be forced to {}.'. - format(mode_str, send_queue_size, num_threads, - num_threads)) + print( + 'WARNING: In {} mode, communicator_send_queue_size ' + 'must be equal to CPU_NUM. But received, ' + 'communicator_send_queue_size = {}, CPU_NUM = ' + '{}. communicator_send_queue_size will be forced to {}.'.format( + mode_str, send_queue_size, num_threads, num_threads + ) + ) self.runtime_configs[ - 'communicator_send_queue_size'] = num_threads + 'communicator_send_queue_size' + ] = num_threads return dict((key, str(self.runtime_configs[key])) for key in need_keys) @@ -178,9 +201,9 @@ def get_lr_ops(program): lr_ops = [] for index, op in enumerate(program.global_block().ops): role_id = int(op.attr(RPC_OP_ROLE_ATTR_NAME)) - if role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) or \ - role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) | \ - int(OPT_OP_ROLE_ATTR_VALUE): + if role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) or role_id == int( + LR_SCHED_OP_ROLE_ATTR_VALUE + ) | int(OPT_OP_ROLE_ATTR_VALUE): lr_ops.append(op) return lr_ops @@ -190,16 +213,20 @@ def get_optimize_ops(_program, remote_sparse=[]): opt_ops = [] for op in block.ops: if _is_opt_role_op(op): - if len(remote_sparse) > 0 and op.input( - "Param" - )[0] not in remote_sparse: # for fl: only delete remote sparse optimize + if ( + len(remote_sparse) > 0 + and op.input("Param")[0] not in remote_sparse + ): # for fl: only delete remote sparse optimize continue # delete clip op from opt_ops when run in Parameter Server mode - if OP_NAME_SCOPE in op.all_attrs() \ - and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE): + if ( + OP_NAME_SCOPE in op.all_attrs() + and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE) + ): op._set_attr( "op_role", - int(core.op_proto_and_checker_maker.OpRole.Backward)) + int(core.op_proto_and_checker_maker.OpRole.Backward), + ) continue opt_ops.append(op) return opt_ops @@ -228,7 +255,7 @@ def get_dist_env(): 'trainer_id': trainer_id, 'num_trainers': num_trainers, 'current_endpoint': current_endpoint, - 'trainer_endpoints': trainer_endpoints + 'trainer_endpoints': trainer_endpoints, } @@ -276,8 +303,10 @@ def is_distributed_sparse_op(op): if op.type in SPARSE_OP_LIST and op.attr('is_distributed') is True: return True - if op.type == "distributed_lookup_table" and op.attr( - 'is_distributed') is True: + if ( + op.type == "distributed_lookup_table" + and op.attr('is_distributed') is True + ): return True return False @@ -288,12 +317,17 @@ def get_sparse_tablename(op): def is_sparse_op(op): - if op.type in SPARSE_OP_LIST and op.attr('is_sparse') is True and op.attr( - 'is_distributed') is False: + if ( + op.type in SPARSE_OP_LIST + and op.attr('is_sparse') is True + and op.attr('is_distributed') is False + ): return True - if op.type == "distributed_lookup_table" and op.attr( - 'is_distributed') is False: + if ( + op.type == "distributed_lookup_table" + and op.attr('is_distributed') is False + ): return True return False @@ -320,12 +354,14 @@ def get_trainers(role_maker): return role_maker.worker_num() -def get_dense_send_context(program, - send_ctx, - idx, - merged_dense_pairs, - trainer_id, - split_dense_table=False): +def get_dense_send_context( + program, + send_ctx, + idx, + merged_dense_pairs, + trainer_id, + split_dense_table=False, +): if len(merged_dense_pairs) < 1: return idx if not split_dense_table: @@ -356,10 +392,23 @@ def get_dense_send_context(program, # print("public get_dense_send_context dense_table:", grad_name, # var_numel, origin_varnames) from paddle.fluid.core import CommContext - dense_ctx = CommContext(grad_name, [grad_name], ["127.0.0.1:6071"], - [var_numel], origin_varnames, trainer_id, - aggregate, False, False, idx, False, False, - id(program), []) + + dense_ctx = CommContext( + grad_name, + [grad_name], + ["127.0.0.1:6071"], + [var_numel], + origin_varnames, + trainer_id, + aggregate, + False, + False, + idx, + False, + False, + id(program), + [], + ) send_ctx[grad_name] = dense_ctx idx += 1 @@ -379,10 +428,23 @@ def get_dense_send_context(program, # print("public get_dense_send_context data_norm table:", grad_name, # var_numel, origin_varnames) from paddle.fluid.core import CommContext - data_norm_ctx = CommContext(grad_name, [grad_name], ["127.0.0.1:6071"], - [var_numel], origin_varnames, trainer_id, - aggregate, False, False, idx, False, True, - id(program), []) + + data_norm_ctx = CommContext( + grad_name, + [grad_name], + ["127.0.0.1:6071"], + [var_numel], + origin_varnames, + trainer_id, + aggregate, + False, + False, + idx, + False, + True, + id(program), + [], + ) send_ctx[grad_name] = data_norm_ctx idx += 1 else: @@ -394,10 +456,23 @@ def get_dense_send_context(program, grad_name = origin_varname aggregate = True from paddle.fluid.core import CommContext - dense_ctx = CommContext(grad_name, [grad_name], ["127.0.0.1:6071"], - [var_numel], [origin_varname], trainer_id, - aggregate, False, False, idx, False, False, - id(program), []) + + dense_ctx = CommContext( + grad_name, + [grad_name], + ["127.0.0.1:6071"], + [var_numel], + [origin_varname], + trainer_id, + aggregate, + False, + False, + idx, + False, + False, + id(program), + [], + ) send_ctx[grad_name] = dense_ctx idx += 1 return idx @@ -405,8 +480,10 @@ def get_dense_send_context(program, def get_geo_trainer_send_context(attrs): if attrs['ps_mode'] != DistributedMode.GEO: - raise ValueError("ps mode: {} not matched {}", - format(ps_mode, "get_geo_trainer_send_context")) + raise ValueError( + "ps mode: {} not matched {}", + format(ps_mode, "get_geo_trainer_send_context"), + ) send_ctx = {} trainer_id = get_role_id(attrs['role_maker']) origin_programs = attrs['origin_main_programs'] @@ -422,16 +499,32 @@ def get_geo_trainer_send_context(attrs): if param_name in attrs['remote_sparse']: # for recall/ncf model continue - is_distributed = True if param_name in distibuted_varnames else False + is_distributed = ( + True if param_name in distibuted_varnames else False + ) var = program.global_block().vars[grad.merged_var.name] var_numel = reduce(lambda x, y: x * y, var.shape[1:]) from paddle.fluid.core import CommContext - print("public get_the_geo_send_context sparse: ", grad_name, - var_numel) - sparse_ctx = CommContext(grad_name, [grad_name], ["127.0.0.1:6071"], - [var_numel], [grad_name], trainer_id, True, - True, is_distributed, idx, False, False, - id(program), []) + + print( + "public get_the_geo_send_context sparse: ", grad_name, var_numel + ) + sparse_ctx = CommContext( + grad_name, + [grad_name], + ["127.0.0.1:6071"], + [var_numel], + [grad_name], + trainer_id, + True, + True, + is_distributed, + idx, + False, + False, + id(program), + [], + ) idx += 1 send_ctx[sparse_ctx.var_name()] = sparse_ctx @@ -452,8 +545,23 @@ def _step_ctx(idx, role_maker): sections = [1] * len(endpoints) names = [name] * len(endpoints) from paddle.fluid.core import CommContext - ctx = CommContext(name, names, endpoints, sections, [name], trainer_id, - True, False, False, idx, True, False, -1, []) + + ctx = CommContext( + name, + names, + endpoints, + sections, + [name], + trainer_id, + True, + False, + False, + idx, + True, + False, + -1, + [], + ) return name, ctx @@ -483,7 +591,9 @@ def get_the_one_send_context(attrs, split_dense_table=False, ep_list=None): for i in range(len(ep_list)): splited_varname.append("{}.block{}".format(param_name, i)) - is_distributed = True if param_name in distibuted_varnames else False + is_distributed = ( + True if param_name in distibuted_varnames else False + ) var = program.global_block().vars[grad.merged_var.name] @@ -493,20 +603,43 @@ def get_the_one_send_context(attrs, split_dense_table=False, ep_list=None): if grad_name in send_ctx: continue from paddle.fluid.core import CommContext - print("public get_the_one_send_context sparse: ", grad_name, - splited_varname, shape) - sparse_ctx = CommContext(grad_name, splited_varname, ep_list, shape, - [grad_name], trainer_id, True, True, - is_distributed, idx, False, False, - id(program), remote_sparse_ids) + + print( + "public get_the_one_send_context sparse: ", + grad_name, + splited_varname, + shape, + ) + sparse_ctx = CommContext( + grad_name, + splited_varname, + ep_list, + shape, + [grad_name], + trainer_id, + True, + True, + is_distributed, + idx, + False, + False, + id(program), + remote_sparse_ids, + ) idx += 1 send_ctx[sparse_ctx.var_name()] = sparse_ctx for i, program in enumerate(origin_programs): merged_dense_pairs = attrs['merged_dense_pairs'][i] - idx = get_dense_send_context(program, send_ctx, idx, merged_dense_pairs, - trainer_id, split_dense_table) + idx = get_dense_send_context( + program, + send_ctx, + idx, + merged_dense_pairs, + trainer_id, + split_dense_table, + ) if len(attrs['tensor_table']) > 0 and attrs['is_worker']: name, ctx = _step_ctx(idx, attrs['role_maker']) @@ -517,8 +650,11 @@ def get_the_one_send_context(attrs, split_dense_table=False, ep_list=None): def find_heter_ops(program, default_device="cpu"): if default_device not in DEVICE_LIST: - raise ValueError("Given device {} is not in device list {}".format( - default_device, DEVICE_LIST)) + raise ValueError( + "Given device {} is not in device list {}".format( + default_device, DEVICE_LIST + ) + ) def _is_heter_op(op, current_heter_device, default_device="cpu"): heter_devices = list(DEVICE_LIST) @@ -527,10 +663,13 @@ def _is_heter_op(op, current_heter_device, default_device="cpu"): op_type = op.type if op_device in heter_devices: return True - elif op_type in COMMUNICATE_OPS_TYPE and current_heter_device != default_device: + elif ( + op_type in COMMUNICATE_OPS_TYPE + and current_heter_device != default_device + ): # for distributed communciate ops: send & recv & barrier etc. # Todo: need update this method - #op._set_attr('op_device', current_heter_device) + # op._set_attr('op_device', current_heter_device) return True elif op_device == None or op_device == default_device: op._set_attr('op_device', default_device) @@ -565,16 +704,20 @@ def _append_heter_op(op, current_heter_block_ops, heter_ops): op = op_list[i] if "_grad" in op.type: forward_op_type = op.type.split("_grad")[0] - if forward_op_type in SPARSE_OP_TYPE_DICT.keys() \ - and op.attr('remote_prefetch') is True: + if ( + forward_op_type in SPARSE_OP_TYPE_DICT.keys() + and op.attr('remote_prefetch') is True + ): param_name = op.input(SPARSE_OP_TYPE_DICT[forward_op_type])[0] if param_name in var2idx: ## insert sum op & remove sum op from var2idx and origin place op_list = list(block.ops) sum_op = op_list[var2idx[param_name]] sum_op_inputs = { - sum_op.input_names[0]: - [block.vars[input] for input in sum_op.input_arg_names] + sum_op.input_names[0]: [ + block.vars[input] + for input in sum_op.input_arg_names + ] } sum_op_outputs = { sum_op.output_names[0]: [ @@ -582,11 +725,13 @@ def _append_heter_op(op, current_heter_block_ops, heter_ops): for output in sum_op.output_arg_names ] } - block._insert_op(index=i + 1, - type=sum_op.type, - inputs=sum_op_inputs, - outputs=sum_op_outputs, - attrs=sum_op.all_attrs()) + block._insert_op( + index=i + 1, + type=sum_op.type, + inputs=sum_op_inputs, + outputs=sum_op_outputs, + attrs=sum_op.all_attrs(), + ) block._remove_op(var2idx[param_name] + 1) var2idx.pop(param_name) for var_ in var2idx: @@ -607,9 +752,9 @@ def _append_heter_op(op, current_heter_block_ops, heter_ops): for no_grad_var in output_vars_no_grad: if no_grad_var in var2idx: """ - insert sum op & remove sum op from var2idx and origin place + insert sum op & remove sum op from var2idx and origin place - """ + """ op_list = list(block.ops) sum_op = op_list[var2idx[no_grad_var]] sum_op_inputs = { @@ -624,11 +769,13 @@ def _append_heter_op(op, current_heter_block_ops, heter_ops): for output in sum_op.output_arg_names ] } - block._insert_op(index=i + 1, - type=sum_op.type, - inputs=sum_op_inputs, - outputs=sum_op_outputs, - attrs=sum_op.all_attrs()) + block._insert_op( + index=i + 1, + type=sum_op.type, + inputs=sum_op_inputs, + outputs=sum_op_outputs, + attrs=sum_op.all_attrs(), + ) block._remove_op(var2idx[no_grad_var] + 1) var2idx.pop(no_grad_var) for var_ in var2idx: @@ -641,12 +788,16 @@ def _append_heter_op(op, current_heter_block_ops, heter_ops): pre_op = op_list[i - 1] if "_grad" in pre_op.type: forward_op_type = pre_op.type.split("_grad")[0] - if forward_op_type in SPARSE_OP_TYPE_DICT.keys() \ - and pre_op.attr('remote_prefetch') is True: + if ( + forward_op_type in SPARSE_OP_TYPE_DICT.keys() + and pre_op.attr('remote_prefetch') is True + ): param_name = pre_op.input( - SPARSE_OP_TYPE_DICT[forward_op_type])[0] + SPARSE_OP_TYPE_DICT[forward_op_type] + )[0] if param_name == origin_var and op.attr( - "op_device") == pre_op.attr("op_device"): + "op_device" + ) == pre_op.attr("op_device"): continue else: var2idx[origin_var] = i @@ -699,7 +850,8 @@ def _append_heter_op(op, current_heter_block_ops, heter_ops): # for cpu-op block append if len(current_default_block_ops) > 1: default_ops[default_device][ - block_index] = current_default_block_ops + block_index + ] = current_default_block_ops program_block_ops.append(current_default_block_ops) current_default_block_ops = [] block_index += 1 @@ -755,8 +907,10 @@ def _append_heter_op(op, current_heter_block_ops, heter_ops): for _, heter_block in heter_block_dict.items(): total_heter_ops += len(heter_block) print( - "There are {} OPs in your main_program, and contains {} heter-OPs which is made up of {} heter-blocks." - .format(len(block.ops), total_heter_ops, heter_blocks)) + "There are {} OPs in your main_program, and contains {} heter-OPs which is made up of {} heter-blocks.".format( + len(block.ops), total_heter_ops, heter_blocks + ) + ) return origin_porgram, heter_ops, default_ops, program_block_ops @@ -773,11 +927,14 @@ def union_forward_gradient_op(program_block_ops_list): """ block_length = len(program_block_ops_list) union_program_block_ops_list = [] - assert block_length % 2 != 0, "the length of program_block_ops_list should be odd" + assert ( + block_length % 2 != 0 + ), "the length of program_block_ops_list should be odd" for i in range(0, block_length // 2): block_op_list = {"forward": program_block_ops_list[i]} block_op_list.update( - {"backward": program_block_ops_list[block_length - 1 - i]}) + {"backward": program_block_ops_list[block_length - 1 - i]} + ) union_program_block_ops_list.append(block_op_list) block_op_list = {"forward": [], "backward": []} @@ -791,13 +948,15 @@ def union_forward_gradient_op(program_block_ops_list): def find_block_joints(program, program_block_ops_list, heter_ops): - block_var_detail = find_entrance_exit_private(program, - program_block_ops_list) - block_var_detail = entrance_exit_check(program, program_block_ops_list, - block_var_detail, heter_ops) - block_var_detail = delete_block_useless_exit(program, - program_block_ops_list, - block_var_detail) + block_var_detail = find_entrance_exit_private( + program, program_block_ops_list + ) + block_var_detail = entrance_exit_check( + program, program_block_ops_list, block_var_detail, heter_ops + ) + block_var_detail = delete_block_useless_exit( + program, program_block_ops_list, block_var_detail + ) return block_var_detail @@ -822,9 +981,11 @@ def find_entrance_exit_private(program, program_block_ops_list): for index, block_op_list in enumerate(program_block_ops_list): ## forward block_input, block_output = find_ops_list_input_output( - program, block_op_list["forward"]) + program, block_op_list["forward"] + ) persistables = screen_persistables( - program, block_input) + screen_persistables(program, block_output) + program, block_input + ) + screen_persistables(program, block_output) # find entrance & exit block_private_vars = list(set(block_input) & set(block_output)) block_entrance = list(set(block_input) - set(block_private_vars)) @@ -834,35 +995,40 @@ def find_entrance_exit_private(program, program_block_ops_list): "entrance": block_entrance, "exit": block_exit, "private": block_private_vars, - "persistables": persistables + "persistables": persistables, } } ## backward bp_block_input, bp_block_output = find_ops_list_input_output( - program, block_op_list["backward"]) + program, block_op_list["backward"] + ) bp_persistables = screen_persistables( - program, bp_block_input) + screen_persistables( - program, bp_block_output) + program, bp_block_input + ) + screen_persistables(program, bp_block_output) # find entrance & exit bp_block_private_vars = list(set(bp_block_input) & set(bp_block_output)) bp_block_entrance = list( - set(bp_block_input) - set(bp_block_private_vars)) + set(bp_block_input) - set(bp_block_private_vars) + ) bp_block_exit = list(set(bp_block_output) - set(bp_block_private_vars)) - detail.update({ - "backward": { - "entrance": bp_block_entrance, - "exit": bp_block_exit, - "private": bp_block_private_vars, - "persistables": bp_persistables + detail.update( + { + "backward": { + "entrance": bp_block_entrance, + "exit": bp_block_exit, + "private": bp_block_private_vars, + "persistables": bp_persistables, + } } - }) + ) block_var_detail.append(detail) return block_var_detail -def entrance_exit_check(program, program_block_ops_list, block_var_detail, - heter_ops): +def entrance_exit_check( + program, program_block_ops_list, block_var_detail, heter_ops +): for index in range(len(block_var_detail) - 1, -1, -1): if index - 1 < 0: break @@ -872,9 +1038,11 @@ def entrance_exit_check(program, program_block_ops_list, block_var_detail, backward_entrance = block_var_detail[index]["backward"]["entrance"] - forward_all = block_var_detail[index]["forward"][ - "entrance"] + block_var_detail[index]["forward"][ - "private"] + block_var_detail[index]["forward"]["exit"] + forward_all = ( + block_var_detail[index]["forward"]["entrance"] + + block_var_detail[index]["forward"]["private"] + + block_var_detail[index]["forward"]["exit"] + ) for var in backward_entrance: if not ("@GRAD" in var) and not (var in forward_all): @@ -885,18 +1053,24 @@ def entrance_exit_check(program, program_block_ops_list, block_var_detail, if previous_block_exit == current_block_entrance: continue exist_vars = list( - set(previous_block_exit) & set(current_block_entrance)) + set(previous_block_exit) & set(current_block_entrance) + ) need_add_vars = list(set(current_block_entrance) - set(exist_vars)) # var in different stage should not be ignored, since they are not placed in the same program & device - #need_add_vars = find_need_var_from_previous_block( + # need_add_vars = find_need_var_from_previous_block( # need_add_vars, block_var_detail, index, heter_ops) - previous_block_private = block_var_detail[index - - 1]["forward"]["private"] - previous_block_entrance = block_var_detail[index - - 1]["forward"]["entrance"] + previous_block_private = block_var_detail[index - 1]["forward"][ + "private" + ] + previous_block_entrance = block_var_detail[index - 1]["forward"][ + "entrance" + ] for var in need_add_vars: - if var not in previous_block_private and var not in previous_block_entrance: + if ( + var not in previous_block_private + and var not in previous_block_entrance + ): previous_block_entrance.append(var) previous_block_exit.append(var) if var not in current_block_entrance: @@ -912,27 +1086,35 @@ def entrance_exit_check(program, program_block_ops_list, block_var_detail, if previous_block_exit == current_block_entrance: continue exist_vars = list( - set(previous_block_exit) & set(current_block_entrance)) + set(previous_block_exit) & set(current_block_entrance) + ) need_add_vars = list(set(current_block_entrance) - set(exist_vars)) need_ignore_vars = [] for var in need_add_vars: if "@GRAD" not in var: need_ignore_vars.append(var) need_add_vars = list( - set(need_add_vars).difference(set(need_ignore_vars))) - previous_block_private = block_var_detail[index + - 1]["backward"]["private"] - previous_block_entrance = block_var_detail[index + - 1]["backward"]["entrance"] + set(need_add_vars).difference(set(need_ignore_vars)) + ) + previous_block_private = block_var_detail[index + 1]["backward"][ + "private" + ] + previous_block_entrance = block_var_detail[index + 1]["backward"][ + "entrance" + ] for var in need_add_vars: - if var not in previous_block_private and var not in previous_block_entrance: + if ( + var not in previous_block_private + and var not in previous_block_entrance + ): previous_block_entrance.append(var) previous_block_exit.append(var) return block_var_detail -def delete_block_useless_exit(program, program_block_ops_list, - block_var_detail): +def delete_block_useless_exit( + program, program_block_ops_list, block_var_detail +): ## forward for index in range(len(block_var_detail)): if index == len(block_var_detail) - 1: @@ -951,8 +1133,9 @@ def delete_block_useless_exit(program, program_block_ops_list, if index - 1 < 0: break current_block_exit = block_var_detail[index]["backward"]["exit"] - next_block_entrance = block_var_detail[index - - 1]["backward"]["entrance"] + next_block_entrance = block_var_detail[index - 1]["backward"][ + "entrance" + ] need_delete_var = [] for var in current_block_exit: if var not in next_block_entrance: @@ -963,19 +1146,20 @@ def delete_block_useless_exit(program, program_block_ops_list, return block_var_detail -def get_communicate_var_info(program, - block_index, - entrance_var_list, - type="forward"): +def get_communicate_var_info( + program, block_index, entrance_var_list, type="forward" +): input_var_reshape_dim = [] input_var_reshape_name = [] if type == "forward": block_input_var_name = "forward_joint_{}_{}@Heter".format( - block_index - 1, block_index) + block_index - 1, block_index + ) else: block_input_var_name = "backward_joint_{}_{}@Heter".format( - block_index + 1, block_index) + block_index + 1, block_index + ) entrance_var_list.sort() # input @@ -998,12 +1182,15 @@ def get_communicate_var_info(program, def add_vars_by_var_list(var_name_list, origin_program, program, block): for var_name in var_name_list: - if var_name not in program.global_block( - ).vars and var_name not in block.vars: + if ( + var_name not in program.global_block().vars + and var_name not in block.vars + ): var = origin_program.global_block().vars[var_name] if var.persistable: - program.global_block()._clone_variable(var, - force_persistable=False) + program.global_block()._clone_variable( + var, force_persistable=False + ) else: block._clone_variable(var, force_persistable=False) @@ -1082,11 +1269,14 @@ def block_append_op(program, origin_program, block, op): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: - if var.name not in program.global_block( - ).vars and var.name not in block.vars: + if ( + var.name not in program.global_block().vars + and var.name not in block.vars + ): if var.persistable: program.global_block()._clone_variable( - var, force_persistable=False) + var, force_persistable=False + ) else: block._clone_variable(var, force_persistable=False) @@ -1095,20 +1285,22 @@ def block_append_op(program, origin_program, block, op): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: - if var.name not in program.global_block( - ).vars and var.name not in block.vars: + if ( + var.name not in program.global_block().vars + and var.name not in block.vars + ): if var.persistable: program.global_block()._clone_variable( - var, force_persistable=False) + var, force_persistable=False + ) else: block._clone_variable(var, force_persistable=False) if "_grad" not in op.type: # for forward op - return block.append_op(type=op.type, - inputs=inputs, - outputs=outputs, - attrs=op.all_attrs()) + return block.append_op( + type=op.type, inputs=inputs, outputs=outputs, attrs=op.all_attrs() + ) else: # for grad op op_desc = op.desc @@ -1134,49 +1326,54 @@ def get_next_stage_trainers(role_maker): return role_maker.get_next_trainers() -def insert_communicate_op(orign_program, - role_maker, - heter_block, - stage_id, - first_op_index, - block_var_detail, - device, - is_forward=True): +def insert_communicate_op( + orign_program, + role_maker, + heter_block, + stage_id, + first_op_index, + block_var_detail, + device, + is_forward=True, +): if is_forward: next_heter_worker_endpoints = get_next_stage_trainers(role_maker) previous_heter_worker_endpoints = get_previous_stage_trainers( - role_maker) + role_maker + ) entrance_var = block_var_detail[stage_id]["forward"]["entrance"] - comm_info = get_communicate_var_info(orign_program, stage_id + 1, - entrance_var) + comm_info = get_communicate_var_info( + orign_program, stage_id + 1, entrance_var + ) else: next_heter_worker_endpoints = get_next_stage_trainers(role_maker) previous_heter_worker_endpoints = get_previous_stage_trainers( - role_maker) + role_maker + ) entrance_var = block_var_detail[stage_id - 1]["backward"]["exit"] - comm_info = get_communicate_var_info(orign_program, stage_id - 1, - entrance_var, "backward") - - heter_block._insert_op(index=first_op_index, - type="send_and_recv", - inputs={"X": heter_block.vars[entrance_var[0]]}, - outputs={"Out": []}, - attrs={ - "mode": "forward" if is_forward else "backward", - "send_var_name": - entrance_var + ["microbatch_id"], - "recv_var_name": [], - "message_name": - comm_info["block_input_var_name"], - "next_endpoints": next_heter_worker_endpoints, - "previous_endpoints": - previous_heter_worker_endpoints, - "trainer_id": get_role_id(role_maker), - "op_device": device, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + comm_info = get_communicate_var_info( + orign_program, stage_id - 1, entrance_var, "backward" + ) + + heter_block._insert_op( + index=first_op_index, + type="send_and_recv", + inputs={"X": heter_block.vars[entrance_var[0]]}, + outputs={"Out": []}, + attrs={ + "mode": "forward" if is_forward else "backward", + "send_var_name": entrance_var + ["microbatch_id"], + "recv_var_name": [], + "message_name": comm_info["block_input_var_name"], + "next_endpoints": next_heter_worker_endpoints, + "previous_endpoints": previous_heter_worker_endpoints, + "trainer_id": get_role_id(role_maker), + "op_device": device, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return entrance_var @@ -1185,8 +1382,9 @@ def get_the_one_recv_context(context, is_dense=True, split_dense_table=False): recv_id_maps = {} grad_name_to_param_name = {} if is_dense: - send_ctx = get_the_one_send_context(context, - split_dense_table=split_dense_table) + send_ctx = get_the_one_send_context( + context, split_dense_table=split_dense_table + ) for idx, (name, ctx) in enumerate(send_ctx.items()): if ctx.is_sparse(): continue @@ -1201,9 +1399,9 @@ def get_the_one_recv_context(context, is_dense=True, split_dense_table=False): param_names.append(param_name) recv_id_maps[ctx.table_id()] = param_names else: - send_ctx = get_the_one_send_context(context, - split_dense_table=False, - ep_list=None) + send_ctx = get_the_one_send_context( + context, split_dense_table=False, ep_list=None + ) for idx, (name, ctx) in enumerate(send_ctx.items()): if not ctx.is_sparse(): continue @@ -1225,15 +1423,15 @@ def _get_varname_parts(varname): block_part = "" trainer_idx = varname.find(".trainer_") if trainer_idx >= 0: - trainer_part = varname[trainer_idx + 1:] + trainer_part = varname[trainer_idx + 1 :] else: trainer_idx = len(varname) block_index = varname.find(".block") if block_index >= 0: - block_part = varname[block_index + 1:trainer_idx] + block_part = varname[block_index + 1 : trainer_idx] else: block_index = len(varname) - orig_var_name = varname[0:min(block_index, trainer_idx)] + orig_var_name = varname[0 : min(block_index, trainer_idx)] return orig_var_name, block_part, trainer_part @@ -1256,7 +1454,6 @@ def get_var_mem_size(var): class MergedVariable: - def __init__(self, merged, ordered, offsets): self.merged_var = merged self.ordered_vars = ordered @@ -1276,8 +1473,8 @@ def build_var_distributed(context): context["merged_variable_map"] = {} for origin_program in origin_programs: sparse_pairs, dense_pairs = get_param_grads(origin_program) - #print("public build_var_distributed sparse_pairs:", sparse_pairs) - #print("public build_var_distributed dense_pairs:", dense_pairs) + # print("public build_var_distributed sparse_pairs:", sparse_pairs) + # print("public build_var_distributed dense_pairs:", dense_pairs) origin_for_sparse = [] origin_for_dense = [] merged_sparse_pairs = [] @@ -1297,7 +1494,7 @@ def build_var_distributed(context): m_grad = MergedVariable(grad, [grad], [0]) merged_variables_pairs.append((m_param, m_grad)) merged_dense_pairs.append((m_param, m_grad)) - #print("public build_var_distributed merged_dense_pairs:", + # print("public build_var_distributed merged_dense_pairs:", # merged_dense_pairs) for sparse_pair in origin_for_sparse: @@ -1307,15 +1504,17 @@ def build_var_distributed(context): m_grad = MergedVariable(grad, [grad], [0]) merged_variables_pairs.append((m_param, m_grad)) merged_sparse_pairs.append((m_param, m_grad)) - #print("public build_var_distributed merged_sparse_pairs:", + # print("public build_var_distributed merged_sparse_pairs:", # merged_sparse_pairs) for merged in merged_variables_pairs: m_param, m_grad = merged context["merged_variable_map"][ - m_param.merged_var.name] = m_param.merged_var + m_param.merged_var.name + ] = m_param.merged_var context["merged_variable_map"][ - m_grad.merged_var.name] = m_grad.merged_var + m_grad.merged_var.name + ] = m_grad.merged_var param_merges = [] param_merges.extend(origin_for_sparse) @@ -1353,14 +1552,14 @@ def _is_opt_role_op(op): # optimize op_maker = core.op_proto_and_checker_maker optimize_role = core.op_proto_and_checker_maker.OpRole.Optimize - if op_maker.kOpRoleAttrName() in op.attr_names and \ - int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(optimize_role): + if op_maker.kOpRoleAttrName() in op.attr_names and int( + op.all_attrs()[op_maker.kOpRoleAttrName()] + ) == int(optimize_role): return True return False def get_param_grads(origin_program): - def _get_params_grads(sparse_varnames): block = origin_program.global_block() @@ -1373,8 +1572,10 @@ def _get_params_grads(sparse_varnames): for op in block.ops: if _is_opt_role_op(op): # delete clip op from opt_ops when run in Parameter Server mode - if OP_NAME_SCOPE in op.all_attrs() \ - and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE): + if ( + OP_NAME_SCOPE in op.all_attrs() + and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE) + ): op._set_attr("op_role", role_id) continue if op.attr(OP_ROLE_VAR_ATTR_NAME): @@ -1382,8 +1583,10 @@ def _get_params_grads(sparse_varnames): grad_name = op.attr(OP_ROLE_VAR_ATTR_NAME)[1] if param_name not in optimize_params: optimize_params.add(param_name) - param_grad = (origin_var_dict[param_name], - origin_var_dict[grad_name]) + param_grad = ( + origin_var_dict[param_name], + origin_var_dict[grad_name], + ) if param_name in sparse_varnames: sparse_param_grads.append(param_grad) @@ -1394,8 +1597,10 @@ def _get_params_grads(sparse_varnames): def _get_sparse_varnames(): varnames = [] for op in origin_program.global_block().ops: - if op.type in SPARSE_OP_TYPE_DICT.keys() \ - and op.attr('remote_prefetch') is True: + if ( + op.type in SPARSE_OP_TYPE_DICT.keys() + and op.attr('remote_prefetch') is True + ): param_name = op.input(SPARSE_OP_TYPE_DICT[op.type])[0] varnames.append(param_name) @@ -1437,13 +1642,13 @@ def find_op_input_output(program, block, op): def add_send_op(program, block, _vars): - def _get_send_op_dict(): send_op_dict = {} send_op_list = find_send_op(program) for op in send_op_list: - input_list, _ = find_op_input_output(program, - program.global_block(), op) + input_list, _ = find_op_input_output( + program, program.global_block(), op + ) for var in input_list: send_op_dict[var] = op return send_op_dict @@ -1472,21 +1677,23 @@ def _get_send_op_dict(): for table_id in table_dict: dummy_output = block.create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) send_input_vars = [ block.vars[union_var] for union_var in table_dict[table_id]['var_list'] ] - block.append_op(type="send", - inputs={"X": send_input_vars}, - outputs={"Out": dummy_output}, - attrs={ - "send_varnames": - table_dict[table_id]['send_varnames'], - "is_sparse": is_sparse, - "table_id": table_id, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + block.append_op( + type="send", + inputs={"X": send_input_vars}, + outputs={"Out": dummy_output}, + attrs={ + "send_varnames": table_dict[table_id]['send_varnames'], + "is_sparse": is_sparse, + "table_id": table_id, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return send_grad_var_list @@ -1503,21 +1710,26 @@ def delete_trainer_useless_var(program, static_var): program_useful_var_list = [] for op in program.global_block().ops: input_var_list, output_var_list = find_op_input_output( - program, program.global_block(), op) + program, program.global_block(), op + ) op_var_list = list(set(input_var_list).union(set(output_var_list))) program_useful_var_list = list( - set(program_useful_var_list).union(set(op_var_list))) + set(program_useful_var_list).union(set(op_var_list)) + ) program_useful_var_list += static_var program_useless_var_list = list( set(get_vars_name_in_block(program.global_block())).difference( - set(program_useful_var_list))) + set(program_useful_var_list) + ) + ) for var in program_useless_var_list: program.global_block()._remove_var(var) return program_useless_var_list -def create_backward_block(program, origin_program, bp_ops_list, - block_var_detail): +def create_backward_block( + program, origin_program, bp_ops_list, block_var_detail +): pre_block_idx = program.num_blocks - 1 heter_block = program._create_block(pre_block_idx) @@ -1526,8 +1738,10 @@ def create_backward_block(program, origin_program, bp_ops_list, send_varnames = op.attr('send_varnames') is_skip = False for varname in send_varnames: - if varname not in program.global_block( - ).vars and varname not in heter_block.vars: + if ( + varname not in program.global_block().vars + and varname not in heter_block.vars + ): is_skip = True break if is_skip == True: @@ -1543,12 +1757,14 @@ def create_backward_block(program, origin_program, bp_ops_list, def is_backward_op(op): return op_role_attr_name in op.attr_names and ( - int(op.attr(op_role_attr_name)) & int(op_role.Backward)) + int(op.attr(op_role_attr_name)) & int(op_role.Backward) + ) def is_forward_op(op): - return op_role_attr_name in op.attr_names and (int( - op.attr(op_role_attr_name)) == int(op_role.Forward)) + return op_role_attr_name in op.attr_names and ( + int(op.attr(op_role_attr_name)) == int(op_role.Forward) + ) def is_push_sparse_op(op): @@ -1591,11 +1807,13 @@ def check_program(program): for op in block.ops: input_var_names = op.desc.input_arg_names() output_var_names = op.desc.output_arg_names() - for var_name in (input_var_names + output_var_names): + for var_name in input_var_names + output_var_names: if not block._find_var_recursive(str(var_name)): raise ValueError( 'var: {} needed by op is not found in block: {}'.format( - str(var_name), block_idx)) + str(var_name), block_idx + ) + ) block_idx += 1 print('program checked valid') diff --git a/python/paddle/distributed/rpc/rpc.py b/python/paddle/distributed/rpc/rpc.py index c61b1a805b093..6a93f27517c21 100644 --- a/python/paddle/distributed/rpc/rpc.py +++ b/python/paddle/distributed/rpc/rpc.py @@ -26,7 +26,7 @@ WorkerInfo = namedtuple("WorkerInfo", ["name", "rank", "ip", "port"]) _DEFAULT_RPC_TIMEOUT = -1 -_MAX_RPC_TIMEOUT_MS = 0x7fffffff +_MAX_RPC_TIMEOUT_MS = 0x7FFFFFFF _BARRIER_TIMEOUT_MAX_DAYS = 99999999 # tcp store for `_barrier_never_timeout` _barrier_store = None @@ -55,8 +55,9 @@ def _exchange_all_service_infos(world_size): s = set() for rank in range(world_size): info = pickle.loads(_barrier_store.get(str(rank))) - assert (info.name not in s - ), "The Worker name must be unique, but name `{}` is repeated." + assert ( + info.name not in s + ), "The Worker name must be unique, but name `{}` is repeated." s.add(info.name) all_infos.append(info) return all_infos @@ -94,22 +95,30 @@ def init_rpc(name, rank=None, world_size=None, master_endpoint=None): """ rank = int(os.environ["PADDLE_TRAINER_ID"]) if rank is None else rank - world_size = int( - os.environ["PADDLE_TRAINERS_NUM"]) if world_size is None else world_size + world_size = ( + int(os.environ["PADDLE_TRAINERS_NUM"]) + if world_size is None + else world_size + ) worker_endpoint = os.getenv("PADDLE_WORKER_ENDPOINT", None) if worker_endpoint is None: worker_endpoint = _gen_endpoint() logger.info("Trainer {}: worker endpoint: {}".format(rank, worker_endpoint)) - master_endpoint = (master_endpoint if master_endpoint != None else - os.environ["PADDLE_MASTER_ENDPOINT"]) + master_endpoint = ( + master_endpoint + if master_endpoint != None + else os.environ["PADDLE_MASTER_ENDPOINT"] + ) master_addr, master_port = master_endpoint.split(":") master_port = int(master_port) stop_check_timeout = int(os.getenv("FLAGS_stop_check_timeout", "900")) - store = core.TCPStore(master_addr, - master_port, - rank == 0, - world_size, - timeout=stop_check_timeout) + store = core.TCPStore( + master_addr, + master_port, + rank == 0, + world_size, + timeout=stop_check_timeout, + ) _set_barrier_store(store) ip, port = worker_endpoint.split(":") port = int(port) @@ -117,8 +126,9 @@ def init_rpc(name, rank=None, world_size=None, master_endpoint=None): all_infos = _exchange_all_service_infos(world_size) c_infos = [] for node_info in all_infos: - info = core.WorkerInfo(node_info.name, node_info.rank, node_info.ip, - node_info.port) + info = core.WorkerInfo( + node_info.name, node_info.rank, node_info.ip, node_info.port + ) c_infos.append(info) core.init_and_set_agent_instance(name, c_infos) core.rpc_start_worker() @@ -226,7 +236,7 @@ def _barrier_never_timeout(global_rank, global_world_size): global _barrier_count barrier_prefix = "Barrier/" + str(_barrier_count) + "/" _barrier_count += 1 - is_master = (global_rank == 0) + is_master = global_rank == 0 def _check_keys_ready(wait_keys): start_time = time.time() @@ -235,11 +245,13 @@ def _check_keys_ready(wait_keys): elapse_time = time.time() - start_time if datetime.timedelta(seconds=elapse_time) > timeout: raise RuntimeError( - "Keys {} are not ready sinck rank {} is waiting them.". - format(wait_keys, global_rank)) + "Keys {} are not ready sinck rank {} is waiting them.".format( + wait_keys, global_rank + ) + ) wait_keys = list( - filter(lambda key: int(_barrier_store.get(key)) != 1, - wait_keys)) + filter(lambda key: int(_barrier_store.get(key)) != 1, wait_keys) + ) if is_master: # the master will add key, wait for all workers'exiting key and exit in the end. diff --git a/python/paddle/distributed/sharding/__init__.py b/python/paddle/distributed/sharding/__init__.py index e938c12d5af0e..3a710ca805942 100644 --- a/python/paddle/distributed/sharding/__init__.py +++ b/python/paddle/distributed/sharding/__init__.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .group_sharded import group_sharded_parallel, save_group_sharded_model # noqa: F401 +from .group_sharded import ( + group_sharded_parallel, + save_group_sharded_model, +) # noqa: F401 __all__ = ['group_sharded_parallel', 'save_group_sharded_model'] diff --git a/python/paddle/distributed/sharding/group_sharded.py b/python/paddle/distributed/sharding/group_sharded.py index 1474f639547fb..4137075c3f902 100644 --- a/python/paddle/distributed/sharding/group_sharded.py +++ b/python/paddle/distributed/sharding/group_sharded.py @@ -22,31 +22,49 @@ from paddle.fluid.framework import in_dygraph_mode # Old version -from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ShardingOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ShardingStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage3 import ShardingStage3 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ShardingScaler +from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ( + ShardingOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ( + ShardingStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage3 import ( + ShardingStage3, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ( + ShardingScaler, +) # New version -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import GroupShardedOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import GroupShardedStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import GroupShardedStage3 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import GroupShardedScaler +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import ( + GroupShardedOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import ( + GroupShardedStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import ( + GroupShardedStage3, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import ( + GroupShardedScaler, +) logger_ = get_logger(logging.WARNING) -def group_sharded_parallel(model, - optimizer, - level, - scaler=None, - group=None, - offload=False, - sync_buffers=False, - buffer_max_size=2**23, - segment_size=2**20, - sync_comm=False, - dp_group=None): +def group_sharded_parallel( + model, + optimizer, + level, + scaler=None, + group=None, + offload=False, + sync_buffers=False, + buffer_max_size=2**23, + segment_size=2**20, + sync_comm=False, + dp_group=None, +): """ Use group_sharded_parallel can perform group shared configuration on the model, optimizer and GradScaler. Level has three string options, 'os', 'os_g' and 'p_g_os' corresponds to three different usage scenarios: optimizer state segmentation, optimizer state + gradient segmentation, and parameter + gradient + optimizer state segmentation. Usually, optimizer state + gradient segmentation is actually a re optimization of optimizer state segmentation, so optimizer state + gradient segmentation can be used to realize optimizer state segmentation. @@ -101,13 +119,16 @@ def group_sharded_parallel(model, """ # check optition type assert isinstance( - model, - paddle.nn.Layer), "The model must be the instance of paddle.nn.Layer." + model, paddle.nn.Layer + ), "The model must be the instance of paddle.nn.Layer." assert isinstance( optimizer, Optimizer ), "The optimizer must be the instance of paddle.optimizer.Optimizer." - assert level in ['os', 'os_g', - 'p_g_os'], "The level must be os, os_g or p_g_os." + assert level in [ + 'os', + 'os_g', + 'p_g_os', + ], "The level must be os, os_g or p_g_os." def check_dtype(param): return param.dtype == paddle.float16 @@ -126,40 +147,51 @@ def check_dtype(param): optim=optimizer, group=group, offload=offload, - dp_group=dp_group) - model = GroupShardedStage2(model, - optimizer, - group=group, - sync_buffers=sync_buffers, - buffer_max_size=buffer_max_size, - dp_group=dp_group) + dp_group=dp_group, + ) + model = GroupShardedStage2( + model, + optimizer, + group=group, + sync_buffers=sync_buffers, + buffer_max_size=buffer_max_size, + dp_group=dp_group, + ) else: - optimizer = ShardingOptimizerStage2(params=model.parameters(), - optim=optimizer, - group=group, - offload=offload) - model = ShardingStage2(model, - optimizer, - group=group, - sync_buffers=sync_buffers, - buffer_max_size=buffer_max_size) + optimizer = ShardingOptimizerStage2( + params=model.parameters(), + optim=optimizer, + group=group, + offload=offload, + ) + model = ShardingStage2( + model, + optimizer, + group=group, + sync_buffers=sync_buffers, + buffer_max_size=buffer_max_size, + ) elif level == 'p_g_os': if in_dygraph_mode(): - model = GroupShardedStage3(model, - optimizer=optimizer, - group=group, - sync_buffers=sync_buffers, - segment_size=segment_size, - offload=offload, - sync_comm=sync_comm) + model = GroupShardedStage3( + model, + optimizer=optimizer, + group=group, + sync_buffers=sync_buffers, + segment_size=segment_size, + offload=offload, + sync_comm=sync_comm, + ) else: - model = ShardingStage3(model, - optimizer=optimizer, - group=group, - sync_buffers=sync_buffers, - segment_size=segment_size, - offload=offload, - sync_comm=sync_comm) + model = ShardingStage3( + model, + optimizer=optimizer, + group=group, + sync_buffers=sync_buffers, + segment_size=segment_size, + offload=offload, + sync_comm=sync_comm, + ) else: raise ValueError("Please enter the correct level.") if isinstance(scaler, paddle.amp.GradScaler): @@ -222,7 +254,8 @@ def save_group_sharded_model(model, output, optimizer=None): save_group_sharded_model(model, optimizer, output=output_dir) """ logger_.info( - "==========Begin to save group sharded model and optimizer==========") + "==========Begin to save group sharded model and optimizer==========" + ) assert not os.path.isfile( output ), "Saving directory ({}) should be a directory, not a file".format(output) @@ -246,4 +279,5 @@ def save_group_sharded_model(model, output, optimizer=None): output_opt = os.path.join(output, "model.pdopt") paddle.save(optimizer._optim.state_dict(), output_opt) logger_.info( - "==========End to save group sharded model and optimizer==========") + "==========End to save group sharded model and optimizer==========" + ) diff --git a/python/paddle/distributed/spawn.py b/python/paddle/distributed/spawn.py index 7deebdf549eba..2df9118ac4e87 100644 --- a/python/paddle/distributed/spawn.py +++ b/python/paddle/distributed/spawn.py @@ -18,11 +18,22 @@ import sys import warnings -from paddle.distributed.utils.launch_utils import _print_arguments, _prepare_trainer_env, get_host_name_ip -from paddle.distributed.cloud_utils import get_cluster_and_pod, _get_trainers_num +from paddle.distributed.utils.launch_utils import ( + _print_arguments, + _prepare_trainer_env, + get_host_name_ip, +) +from paddle.distributed.cloud_utils import ( + get_cluster_and_pod, + _get_trainers_num, +) from paddle.distributed.fleet.launch import get_cluster_from_args from paddle.distributed.fleet.cloud_utils import use_paddlecloud -from paddle.distributed.fleet.launch_utils import DeviceMode, check_backend, block_windows_and_macos +from paddle.distributed.fleet.launch_utils import ( + DeviceMode, + check_backend, + block_windows_and_macos, +) from paddle.device import get_device # deprecated module import @@ -33,7 +44,6 @@ class ParallelEnvArgs(object): - def __init__(self): # Paddle cluster nodes ips, such as 192.168.0.16,192.168.0.17.. self.cluster_node_ips = None @@ -64,17 +74,27 @@ def _py_supported_check(): "Use `paddle.distributed.spawn` to start parallel training " "requires python version greater than 3.4, if your python " "is lower than this version, please use " - "`paddle.distributed.launch` instead.") + "`paddle.distributed.launch` instead." + ) def _options_valid_check(options): # `print_config` keeped as a debug options, not show to users supported_options = [ - 'start_method', 'ips', 'gpus', 'xpus', 'mlus', 'print_config', 'backend' + 'start_method', + 'ips', + 'gpus', + 'xpus', + 'mlus', + 'print_config', + 'backend', ] deprecated_options = [ - 'selected_devices', 'started_port', 'cluster_node_ips', 'node_ip', - 'use_paddlecloud' + 'selected_devices', + 'started_port', + 'cluster_node_ips', + 'node_ip', + 'use_paddlecloud', ] for key in options: if key not in supported_options: @@ -82,11 +102,14 @@ def _options_valid_check(options): warnings.warn( "The config option (%s) of `paddle.distributed.spawn` is deprecated. " "Please use the latest config options stated in the `spawn` API documentation." - % key, DeprecationWarning) + % key, + DeprecationWarning, + ) else: raise ValueError( "The config option (%s) of `paddle.distributed.spawn` is not supported." - % key) + % key + ) def _get_default_nprocs(): @@ -101,8 +124,10 @@ def _get_default_nprocs(): return multiprocessing.cpu_count() else: raise RuntimeError( - "`paddle.distributed.spawn` does not support parallel training on device `{}` now." - .format(device)) + "`paddle.distributed.spawn` does not support parallel training on device `{}` now.".format( + device + ) + ) def _get_default_backend(): @@ -117,8 +142,10 @@ def _get_default_backend(): return 'gloo' else: raise RuntimeError( - "`paddle.distributed.spawn` does not support parallel training on device `{}` now." - .format(device)) + "`paddle.distributed.spawn` does not support parallel training on device `{}` now.".format( + device + ) + ) def _get_node_ip(ips): @@ -180,23 +207,28 @@ def _get_subprocess_env_list(nprocs, options): "the number of visible devices(%d) is less than the number " "of spawn processes(%d), please ensure that the correct " "`nprocs` argument is passed or the environment variable " - "`CUDA_VISIBLE_DEVICES` is correctly configured." % - (len(env_devices_list), nprocs)) + "`CUDA_VISIBLE_DEVICES` is correctly configured." + % (len(env_devices_list), nprocs) + ) args.selected_devices = ",".join( - [str(env_devices_list[x]) for x in range(0, nprocs)]) + [str(env_devices_list[x]) for x in range(0, nprocs)] + ) else: selected_device_list = args.selected_devices.split(',') if len(selected_device_list) != nprocs: raise ValueError( "The number of selected devices(%s) is not equal to " "the number of spawn processes(%d), please ensure that the " - "correct `nprocs` and `gpus` arguments are passed." % - (len(selected_device_list), nprocs)) + "correct `nprocs` and `gpus` arguments are passed." + % (len(selected_device_list), nprocs) + ) for card_id in selected_device_list: if card_id not in env_devices_list: - raise ValueError("The selected gpu card %s cannot found in " - "CUDA_VISIBLE_DEVICES (%s)." % - (card_id, ",".join(env_devices_list))) + raise ValueError( + "The selected gpu card %s cannot found in " + "CUDA_VISIBLE_DEVICES (%s)." + % (card_id, ",".join(env_devices_list)) + ) elif options['backend'] == 'bkcl': args.selected_devices = options.get('xpus', None) @@ -215,23 +247,28 @@ def _get_subprocess_env_list(nprocs, options): "the number of visible devices(%d) is less than the number " "of spawn processes(%d), please ensure that the correct " "`nprocs` argument is passed or the environment variable " - "`XPU_VISIBLE_DEVICES` is correctly configured." % - (len(env_devices_list), nprocs)) + "`XPU_VISIBLE_DEVICES` is correctly configured." + % (len(env_devices_list), nprocs) + ) args.selected_devices = ",".join( - [str(env_devices_list[x]) for x in range(0, nprocs)]) + [str(env_devices_list[x]) for x in range(0, nprocs)] + ) else: selected_device_list = args.selected_devices.split(',') if len(selected_device_list) != nprocs: raise ValueError( "The number of selected devices(%s) is not equal to " "the number of spawn processes(%d), please ensure that the " - "correct `nprocs` and `xpus` arguments are passed." % - (len(selected_device_list), nprocs)) + "correct `nprocs` and `xpus` arguments are passed." + % (len(selected_device_list), nprocs) + ) for card_id in selected_device_list: if card_id not in env_devices_list: - raise ValueError("The selected xpu card %s cannot found in " - "XPU_VISIBLE_DEVICES (%s)." % - (card_id, ",".join(env_devices_list))) + raise ValueError( + "The selected xpu card %s cannot found in " + "XPU_VISIBLE_DEVICES (%s)." + % (card_id, ",".join(env_devices_list)) + ) elif options['backend'] == 'cncl': args.selected_devices = options.get('mlus', None) if args.selected_devices is None: @@ -249,23 +286,28 @@ def _get_subprocess_env_list(nprocs, options): "the number of visible devices(%d) is less than the number " "of spawn processes(%d), please ensure that the correct " "`nprocs` argument is passed or the environment variable " - "`MLU_VISIBLE_DEVICES` is correctly configured." % - (len(env_devices_list), nprocs)) + "`MLU_VISIBLE_DEVICES` is correctly configured." + % (len(env_devices_list), nprocs) + ) args.selected_devices = ",".join( - [str(env_devices_list[x]) for x in range(0, nprocs)]) + [str(env_devices_list[x]) for x in range(0, nprocs)] + ) else: selected_device_list = args.selected_devices.split(',') if len(selected_device_list) != nprocs: raise ValueError( "The number of selected devices(%s) is not equal to " "the number of spawn processes(%d), please ensure that the " - "correct `nprocs` and `mlus` arguments are passed." % - (len(selected_device_list), nprocs)) + "correct `nprocs` and `mlus` arguments are passed." + % (len(selected_device_list), nprocs) + ) for card_id in selected_device_list: if card_id not in env_devices_list: - raise ValueError("The selected mlu card %s cannot found in " - "MLU_VISIBLE_DEVICES (%s)." % - (card_id, ",".join(env_devices_list))) + raise ValueError( + "The selected mlu card %s cannot found in " + "MLU_VISIBLE_DEVICES (%s)." + % (card_id, ",".join(env_devices_list)) + ) elif options['backend'] == 'gloo': # TODO check gpu / xpu flag must not exist warnings.warn( @@ -275,14 +317,15 @@ def _get_subprocess_env_list(nprocs, options): args.paddle_cpuonly = True args.selected_devices = None args.ips = args.cluster_node_ips - assert options.get( - 'use_paddlecloud', - None) is None, "CPUONLY spawn doesn't support use paddle cloud" - assert len( - args.cluster_node_ips.split(',') - ) <= 1, "CPUONLY spawn only support single trainer, that is len(ips)=1, but got %s." - assert _get_trainers_num( - ) == 1, "CPUONLY spawn doesn't support multi-trainer" + assert ( + options.get('use_paddlecloud', None) is None + ), "CPUONLY spawn doesn't support use paddle cloud" + assert ( + len(args.cluster_node_ips.split(',')) <= 1 + ), "CPUONLY spawn only support single trainer, that is len(ips)=1, but got %s." + assert ( + _get_trainers_num() == 1 + ), "CPUONLY spawn doesn't support multi-trainer" # set other inner args args.node_ip = options.get('node_ip', None) @@ -298,15 +341,17 @@ def _get_subprocess_env_list(nprocs, options): # get cluster and pod config if options['backend'] == 'gloo': devices_per_proc = [x for x in range(0, nprocs)] - cluster, pod = get_cluster_from_args(args, DeviceMode.CPU, - devices_per_proc) + cluster, pod = get_cluster_from_args( + args, DeviceMode.CPU, devices_per_proc + ) else: cluster, pod = get_cluster_and_pod(args) # prepare subprocess env list for trainer in pod.trainers: processes_env_list.append( - _prepare_trainer_env(cluster, trainer, options['backend'])) + _prepare_trainer_env(cluster, trainer, options['backend']) + ) # [Debug] print config args.print_config = options.get('print_config', False) @@ -340,7 +385,7 @@ def _set_trainer_env(env_dict, backend): elif backend == 'cncl': set_flags({'FLAGS_selected_mlus': env_dict['FLAGS_selected_mlus']}) else: - #NOTE(xiongkun) why not raise Error ? + # NOTE(xiongkun) why not raise Error ? # So far, we added support for CPU parallel, and will be applied when paddle is not # compiled with cuda or xp. just do nothing. pass @@ -362,12 +407,12 @@ def _func_wrapper(func, args, error_queue, return_queue, env_dict, backend): pass except Exception: import traceback + error_queue.put(traceback.format_exc()) sys.exit(1) class MultiprocessContext(object): - def __init__(self, processes, error_queues, return_queues): _py_supported_check() self.error_queues = error_queues @@ -379,16 +424,16 @@ def __init__(self, processes, error_queues, return_queues): self.return_queues = return_queues self.processes = processes self.sentinels = { - process.sentinel: index - for index, process in enumerate(processes) + process.sentinel: index for index, process in enumerate(processes) } def join(self, timeout=None): if len(self.sentinels) == 0: return True - ready = multiprocessing.connection.wait(self.sentinels.keys(), - timeout=timeout) + ready = multiprocessing.connection.wait( + self.sentinels.keys(), timeout=timeout + ) error_index = None for sentinel in ready: @@ -414,16 +459,22 @@ def _throw_exception(self, error_index): exitcode = self.processes[error_index].exitcode if exitcode < 0: name = signal.Signals(-exitcode).name - raise Exception("Process %d terminated with signal %s." % - (error_index, name)) + raise Exception( + "Process %d terminated with signal %s." + % (error_index, name) + ) else: - raise Exception("Process %d terminated with exit code %d." % - (error_index, exitcode)) + raise Exception( + "Process %d terminated with exit code %d." + % (error_index, exitcode) + ) original_trace = self.error_queues[error_index].get() - msg = "\n\n----------------------------------------------\n" \ - "Process %d terminated with the following error:\n" \ - "----------------------------------------------\n\n" % error_index + msg = ( + "\n\n----------------------------------------------\n" + "Process %d terminated with the following error:\n" + "----------------------------------------------\n\n" % error_index + ) msg += original_trace raise Exception(msg) @@ -584,9 +635,17 @@ def train(print_result=False): for i in range(nprocs): error_queue = mp.SimpleQueue() return_queue = mp.SimpleQueue() - process = mp.Process(target=_func_wrapper, - args=(func, args, error_queue, return_queue, - procs_env_list[i], options['backend'])) + process = mp.Process( + target=_func_wrapper, + args=( + func, + args, + error_queue, + return_queue, + procs_env_list[i], + options['backend'], + ), + ) process.daemon = daemon process.start() error_queues.append(error_queue) diff --git a/python/paddle/distributed/utils/launch_utils.py b/python/paddle/distributed/utils/launch_utils.py index a542523e60536..88acc643ead83 100644 --- a/python/paddle/distributed/utils/launch_utils.py +++ b/python/paddle/distributed/utils/launch_utils.py @@ -33,12 +33,18 @@ def get_cluster_from_args(args, selected_gpus): node_ip = args.node_ip node_rank = node_ips.index(node_ip) - logger.debug("parsed from args:node_ips:{} node_ip:{} node_rank:{}".format( - node_ips, node_ip, node_rank)) + logger.debug( + "parsed from args:node_ips:{} node_ip:{} node_rank:{}".format( + node_ips, node_ip, node_rank + ) + ) free_ports = None - if not args.use_paddlecloud and len( - node_ips) <= 1 and args.started_port is None: + if ( + not args.use_paddlecloud + and len(node_ips) <= 1 + and args.started_port is None + ): free_ports = find_free_ports(len(selected_gpus)) if free_ports is not None: free_ports = list(free_ports) @@ -60,6 +66,7 @@ def get_cluster_from_args(args, selected_gpus): def get_gpus(selected_gpus): if selected_gpus is None: from paddle.fluid import core + gpus_num = core.get_cuda_device_count() gpus = [str(x) for x in range(0, gpus_num)] else: @@ -72,48 +79,56 @@ def get_gpus(selected_gpus): # therefore selected_gpus=0,1,2,3 cuda_visible_devices_list = cuda_visible_devices.split(',') for x in selected_gpus.split(','): - assert x in cuda_visible_devices_list, "Can't find "\ - "your selected_gpus %s in CUDA_VISIBLE_DEVICES[%s]."\ - % (x, cuda_visible_devices) + assert x in cuda_visible_devices_list, ( + "Can't find " + "your selected_gpus %s in CUDA_VISIBLE_DEVICES[%s]." + % (x, cuda_visible_devices) + ) gpus = [ cuda_visible_devices_list.index(x.strip()) for x in selected_gpus.split(',') ] - logger.info("Change selected_gpus into reletive values. --ips:{} " - "will change into relative_ips:{} according to your " - "CUDA_VISIBLE_DEVICES:{}".format( - selected_gpus, gpus, cuda_visible_devices_list)) + logger.info( + "Change selected_gpus into reletive values. --ips:{} " + "will change into relative_ips:{} according to your " + "CUDA_VISIBLE_DEVICES:{}".format( + selected_gpus, gpus, cuda_visible_devices_list + ) + ) return gpus class Hdfs(object): - def __init__(self): self.hdfs_ugi = None self.hdfs_name = None self.hdfs_path = None def is_valid(self): - return self.hdfs_ugi is not None and \ - self.hdfs_name is not None and \ - self.hdfs_path is not None + return ( + self.hdfs_ugi is not None + and self.hdfs_name is not None + and self.hdfs_path is not None + ) def __str__(self): return "hdfs_ugi:{} hdfs_name:{} hdfs_path{}".format( - self.hdfs_ugi, self.hdfs_name, self.hdfs_path) + self.hdfs_ugi, self.hdfs_name, self.hdfs_path + ) def __eq__(self, n): - return self.hdfs_ugi == n.hdfs_ugi and \ - self.hdfs_name == n.hdfs_name and \ - self.hdfs_path == n.hdfs_path + return ( + self.hdfs_ugi == n.hdfs_ugi + and self.hdfs_name == n.hdfs_name + and self.hdfs_path == n.hdfs_path + ) def __ne__(self, n): return not self == n class Cluster(object): - def __init__(self, hdfs): self.job_server = None self.pods = [] @@ -122,8 +137,11 @@ def __init__(self, hdfs): def __str__(self): return "job_server:{} pods:{} job_stage_flag:{} hdfs:{}".format( - self.job_server, [str(pod) for pod in self.pods], - self.job_stage_flag, self.hdfs) + self.job_server, + [str(pod) for pod in self.pods], + self.job_stage_flag, + self.hdfs, + ) def __eq__(self, cluster): if len(self.pods) != len(cluster.pods): @@ -161,8 +179,9 @@ def pods_endpoints(self): r = [] for pod in self.pods: ep = "{}:{}".format(pod.addr, pod.port) - assert pod.port != None and pod.addr != None, "{} not a valid endpoint".format( - ep) + assert ( + pod.port != None and pod.addr != None + ), "{} not a valid endpoint".format(ep) r.append(ep) return r @@ -176,7 +195,6 @@ def get_pod_by_id(self, pod_id): class JobServer(object): - def __init__(self): self.endpoint = None @@ -191,22 +209,21 @@ def __ne__(self, j): class Trainer(object): - def __init__(self): self.gpus = [] self.endpoint = None self.rank = None def __str__(self): - return "gpu:{} endpoint:{} rank:{}".format(self.gpus, self.endpoint, - self.rank) + return "gpu:{} endpoint:{} rank:{}".format( + self.gpus, self.endpoint, self.rank + ) def __eq__(self, t): if len(self.gpus) != len(t.gpus): return False - if self.endpoint != t.endpoint or \ - self.rank != t.rank: + if self.endpoint != t.endpoint or self.rank != t.rank: return False for a, b in zip(self.gpus, t.gpus): @@ -223,7 +240,6 @@ def get_rank(self): class Pod(object): - def __init__(self): self.rank = None self.id = None @@ -233,27 +249,38 @@ def __init__(self): self.gpus = [] def __str__(self): - return "rank:{} id:{} addr:{} port:{} visible_gpu:{} trainers:{}".format( - self.rank, self.id, self.addr, self.port, self.gpus, - [str(t) for t in self.trainers]) + return ( + "rank:{} id:{} addr:{} port:{} visible_gpu:{} trainers:{}".format( + self.rank, + self.id, + self.addr, + self.port, + self.gpus, + [str(t) for t in self.trainers], + ) + ) def __eq__(self, pod): - if self.rank != pod.rank or \ - self.id != pod.id or \ - self.addr != pod.addr or \ - self.port != pod.port: + if ( + self.rank != pod.rank + or self.id != pod.id + or self.addr != pod.addr + or self.port != pod.port + ): logger.debug("pod {} != {}".format(self, pod)) return False if len(self.trainers) != len(pod.trainers): - logger.debug("trainers {} != {}".format(self.trainers, - pod.trainers)) + logger.debug( + "trainers {} != {}".format(self.trainers, pod.trainers) + ) return False for i in range(len(self.trainers)): if self.trainers[i] != pod.trainers[i]: - logger.debug("trainer {} != {}".format(self.trainers[i], - pod.trainers[i])) + logger.debug( + "trainer {} != {}".format(self.trainers[i], pod.trainers[i]) + ) return False return True @@ -310,7 +337,7 @@ def terminate_local_procs(procs): p.log_fn.close() logger.debug("terminate process id:{}".format(p.proc.pid)) - #wait all process terminiated + # wait all process terminiated time.sleep(3) for step in range(0, 50): alive = False @@ -347,15 +374,16 @@ def add_arguments(argname, type, default, help, argparser, **kwargs): args = parser.parse_args() """ type = strtobool if type == bool else type - argparser.add_argument("--" + argname, - default=default, - type=type, - help=help + ' Default: %(default)s.', - **kwargs) + argparser.add_argument( + "--" + argname, + default=default, + type=type, + help=help + ' Default: %(default)s.', + **kwargs + ) def find_free_ports(num): - def __free_port(): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: s.bind(('', 0)) @@ -386,30 +414,30 @@ def _prepare_trainer_env(cluster, trainer, backend=None): backend = get_backend_by_compile_flag() # for compatibility if backend == 'bkcl': proc_env = { - "FLAGS_selected_xpus": - "%s" % ",".join([str(g) for g in trainer.gpus]), + "FLAGS_selected_xpus": "%s" + % ",".join([str(g) for g in trainer.gpus]), "PADDLE_TRAINER_ID": "%d" % trainer.rank, "PADDLE_CURRENT_ENDPOINT": "%s" % trainer.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), - "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()) + "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } elif backend == 'nccl': proc_env = { - "FLAGS_selected_gpus": - "%s" % ",".join([str(g) for g in trainer.gpus]), + "FLAGS_selected_gpus": "%s" + % ",".join([str(g) for g in trainer.gpus]), "PADDLE_TRAINER_ID": "%d" % trainer.rank, "PADDLE_CURRENT_ENDPOINT": "%s" % trainer.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), - "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()) + "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } elif backend == 'cncl': proc_env = { - "FLAGS_selected_mlus": - "%s" % ",".join([str(g) for g in trainer.gpus]), + "FLAGS_selected_mlus": "%s" + % ",".join([str(g) for g in trainer.gpus]), "PADDLE_TRAINER_ID": "%d" % trainer.rank, "PADDLE_CURRENT_ENDPOINT": "%s" % trainer.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), - "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()) + "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } elif backend == 'gloo': # NOTE (xiongkun) default fall back into cpu only @@ -418,8 +446,7 @@ def _prepare_trainer_env(cluster, trainer, backend=None): "PADDLE_CURRENT_ENDPOINT": "%s" % trainer.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), - "PADDLE_DISTRI_BACKEND": - backend, # only add here, other will be auto + "PADDLE_DISTRI_BACKEND": backend, # only add here, other will be auto } else: raise ValueError("backend must be one of 'gloo, nccl, bkcl'") @@ -428,7 +455,6 @@ def _prepare_trainer_env(cluster, trainer, backend=None): class TrainerProc(object): - def __init__(self): self.proc = None self.log_fn = None @@ -438,16 +464,14 @@ def __init__(self): self.cmd = None -def start_local_trainers(cluster, - pod, - training_script, - training_script_args, - log_dir=None): +def start_local_trainers( + cluster, pod, training_script, training_script_args, log_dir=None +): current_env = copy.copy(os.environ.copy()) - #paddle broadcast ncclUniqueId use socket, and - #proxy maybe make trainers unreachable, so delete them. - #if we set them to "", grpc will log error message "bad uri" - #so just delete them. + # paddle broadcast ncclUniqueId use socket, and + # proxy maybe make trainers unreachable, so delete them. + # if we set them to "", grpc will log error message "bad uri" + # so just delete them. current_env.pop("http_proxy", None) current_env.pop("https_proxy", None) @@ -493,8 +517,9 @@ def pull_worker_log(tp): except UnicodeEncodeError: sys.stdout.write( 'UnicodeEncodeError occurs at this line. ' - 'Please refer to the original log file "%s"\n' % - tp.log_fn.name) + 'Please refer to the original log file "%s"\n' + % tp.log_fn.name + ) tp.log_offset = fin.tell() @@ -525,14 +550,18 @@ def watch_local_trainers(procs, nranks): raise except SystemExit: logger.error( - "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log." - .format(nranks, error_rank)) + "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log.".format( + nranks, error_rank + ) + ) terminate_local_procs(procs) raise except: logger.error( - "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log." - .format(nranks, error_rank)) + "ABORT!!! Out of all {} trainers, the trainer process with rank={} was aborted. Please check its log.".format( + nranks, error_rank + ) + ) terminate_local_procs(procs) raise diff --git a/python/paddle/distributed/utils/moe_utils.py b/python/paddle/distributed/utils/moe_utils.py index d6dbfdfab58c0..cd7c0e758d4e0 100644 --- a/python/paddle/distributed/utils/moe_utils.py +++ b/python/paddle/distributed/utils/moe_utils.py @@ -18,11 +18,9 @@ from paddle import _legacy_C_ops -def global_scatter(x, - local_count, - global_count, - group=None, - use_calc_stream=True): +def global_scatter( + x, local_count, global_count, group=None, use_calc_stream=True +): """ The global_scatter operator distributes the data of x to n_expert * world_size experts according to local_count, and then receives data according to global_count. The expert refers to a user-defined expert network, @@ -110,42 +108,49 @@ def global_scatter(x, ring_id = 0 if group is None else group.id if _non_static_mode(): - return _legacy_C_ops.global_scatter(x, local_count, \ - global_count, \ - 'use_calc_stream', use_calc_stream, \ - 'ring_id', ring_id) + return _legacy_C_ops.global_scatter( + x, + local_count, + global_count, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + ) else: op_type = 'global_scatter' check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'global_scatter') - check_variable_and_dtype(local_count, 'local_count', ['int64'], - 'global_scatter') - check_variable_and_dtype(global_count, 'global_count', ['int64'], - 'global_scatter') + x, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'global_scatter', + ) + check_variable_and_dtype( + local_count, 'local_count', ['int64'], 'global_scatter' + ) + check_variable_and_dtype( + global_count, 'global_count', ['int64'], 'global_scatter' + ) helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=op_type, - inputs={ - 'X': [x], - 'local_count': [local_count], - 'global_count': [global_count], - }, - outputs={'Out': [out]}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream - }) + helper.append_op( + type=op_type, + inputs={ + 'X': [x], + 'local_count': [local_count], + 'global_count': [global_count], + }, + outputs={'Out': [out]}, + attrs={'ring_id': ring_id, 'use_calc_stream': use_calc_stream}, + ) return out -def global_gather(x, - local_count, - global_count, - group=None, - use_calc_stream=True): +def global_gather( + x, local_count, global_count, group=None, use_calc_stream=True +): """ The global_gather operator gathers the data of x into n_expert * world_size experts according to global_count, and then receives data according to local_count. The expert refers to a user-defined expert network, n_expert refers to the number of expert networks owned by each card, and world_size refers to the number of graphics cards running the network. @@ -223,33 +228,45 @@ def global_gather(x, ring_id = 0 if group is None else group.id if _non_static_mode(): - return _legacy_C_ops.global_gather(x, local_count, \ - global_count, \ - 'use_calc_stream', use_calc_stream, \ - 'ring_id', ring_id) + return _legacy_C_ops.global_gather( + x, + local_count, + global_count, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + ) else: op_type = 'global_gather' check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'global_gather') + x, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'global_gather', + ) - check_variable_and_dtype(local_count, 'local_count', ['int64'], - 'global_gather') + check_variable_and_dtype( + local_count, 'local_count', ['int64'], 'global_gather' + ) - check_variable_and_dtype(global_count, 'global_count', ['int64'], - 'global_gather') + check_variable_and_dtype( + global_count, 'global_count', ['int64'], 'global_gather' + ) helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=op_type, - inputs={ - 'X': [x], - 'local_count': [local_count], - 'global_count': [global_count] - }, - outputs={'Out': [out]}, - attrs={ - 'ring_id': group, - 'use_calc_stream': use_calc_stream, - }) + helper.append_op( + type=op_type, + inputs={ + 'X': [x], + 'local_count': [local_count], + 'global_count': [global_count], + }, + outputs={'Out': [out]}, + attrs={ + 'ring_id': group, + 'use_calc_stream': use_calc_stream, + }, + ) return out diff --git a/python/paddle/distribution/__init__.py b/python/paddle/distribution/__init__.py index 805675a0f2373..77b83fa6a94c5 100644 --- a/python/paddle/distribution/__init__.py +++ b/python/paddle/distribution/__init__.py @@ -25,15 +25,26 @@ from paddle.distribution.multinomial import Multinomial from paddle.distribution.normal import Normal from paddle.distribution.transform import * # noqa: F403 -from paddle.distribution.transformed_distribution import \ - TransformedDistribution +from paddle.distribution.transformed_distribution import TransformedDistribution from paddle.distribution.uniform import Uniform from paddle.distribution.laplace import Laplace __all__ = [ # noqa - 'Beta', 'Categorical', 'Dirichlet', 'Distribution', 'ExponentialFamily', - 'Multinomial', 'Normal', 'Uniform', 'kl_divergence', 'register_kl', - 'Independent', 'TransformedDistribution', 'Laplace', 'LogNormal', 'Gumbel' + 'Beta', + 'Categorical', + 'Dirichlet', + 'Distribution', + 'ExponentialFamily', + 'Multinomial', + 'Normal', + 'Uniform', + 'kl_divergence', + 'register_kl', + 'Independent', + 'TransformedDistribution', + 'Laplace', + 'LogNormal', + 'Gumbel', ] __all__.extend(transform.__all__) diff --git a/python/paddle/distribution/beta.py b/python/paddle/distribution/beta.py index 3474ee87b4ca6..f0eaa1dfe17b0 100644 --- a/python/paddle/distribution/beta.py +++ b/python/paddle/distribution/beta.py @@ -92,20 +92,19 @@ def __init__(self, alpha, beta): self.alpha, self.beta = paddle.broadcast_tensors([alpha, beta]) self._dirichlet = dirichlet.Dirichlet( - paddle.stack([self.alpha, self.beta], -1)) + paddle.stack([self.alpha, self.beta], -1) + ) super(Beta, self).__init__(self._dirichlet._batch_shape) @property def mean(self): - """Mean of beta distribution. - """ + """Mean of beta distribution.""" return self.alpha / (self.alpha + self.beta) @property def variance(self): - """Variance of beat distribution - """ + """Variance of beat distribution""" sum = self.alpha + self.beta return self.alpha * self.beta / (sum.pow(2) * (sum + 1)) diff --git a/python/paddle/distribution/categorical.py b/python/paddle/distribution/categorical.py index 269e2b5567196..bd6c570ca1f10 100644 --- a/python/paddle/distribution/categorical.py +++ b/python/paddle/distribution/categorical.py @@ -15,9 +15,9 @@ import numpy as np import paddle from paddle.distribution import distribution -from paddle.fluid.data_feeder import (check_type, convert_dtype) +from paddle.fluid.data_feeder import check_type, convert_dtype from paddle.fluid.framework import _non_static_mode -from paddle.fluid.layers import (ops, tensor) +from paddle.fluid.layers import ops, tensor from paddle.tensor import multinomial @@ -90,9 +90,12 @@ def __init__(self, logits, name=None): name(str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. """ if not _non_static_mode(): - check_type(logits, 'logits', - (np.ndarray, tensor.Variable, list, tuple), - 'Categorical') + check_type( + logits, + 'logits', + (np.ndarray, tensor.Variable, list, tuple), + 'Categorical', + ) self.name = name if name is not None else 'Categorical' self.dtype = 'float32' @@ -101,8 +104,10 @@ def __init__(self, logits, name=None): self.logits = logits self.dtype = convert_dtype(logits.dtype) else: - if isinstance(logits, np.ndarray) and str( - logits.dtype) in ['float32', 'float64']: + if isinstance(logits, np.ndarray) and str(logits.dtype) in [ + 'float32', + 'float64', + ]: self.dtype = logits.dtype self.logits = self._to_tensor(logits)[0] if self.dtype != convert_dtype(self.logits.dtype): @@ -149,13 +154,15 @@ def sample(self, shape): if len(logits_shape) > 1: sample_shape = shape + logits_shape[:-1] logits = paddle.reshape( - self.logits, [np.prod(logits_shape[:-1]), logits_shape[-1]]) + self.logits, [np.prod(logits_shape[:-1]), logits_shape[-1]] + ) else: sample_shape = shape logits = self.logits - sample_index = multinomial(self._logits_to_probs(logits), num_samples, - True) + sample_index = multinomial( + self._logits_to_probs(logits), num_samples, True + ) # multinomial sample shape is (logits.shape[:-1], num_samples), need to # tanspose to (num_samples, logits.shape[:-1]) @@ -203,21 +210,22 @@ def kl_divergence(self, other): if not _non_static_mode(): check_type(other, 'other', Categorical, 'kl_divergence') - logits = self.logits - \ - paddle.max(self.logits, axis=-1, keepdim=True) + logits = self.logits - paddle.max(self.logits, axis=-1, keepdim=True) other_logits = other.logits - paddle.max( - other.logits, axis=-1, keepdim=True) + other.logits, axis=-1, keepdim=True + ) e_logits = ops.exp(logits) other_e_logits = ops.exp(other_logits) z = paddle.sum(e_logits, axis=-1, keepdim=True) other_z = paddle.sum(other_e_logits, axis=-1, keepdim=True) prob = e_logits / z kl = paddle.sum( - prob * - (logits - paddle.log(z) - other_logits + paddle.log(other_z)), + prob + * (logits - paddle.log(z) - other_logits + paddle.log(other_z)), axis=-1, keepdim=True, - name=name) + name=name, + ) return kl @@ -246,8 +254,7 @@ def entropy(self): """ name = self.name + '_entropy' - logits = self.logits - \ - paddle.max(self.logits, axis=-1, keepdim=True) + logits = self.logits - paddle.max(self.logits, axis=-1, keepdim=True) e_logits = ops.exp(logits) z = paddle.sum(e_logits, axis=-1, keepdim=True) prob = e_logits / z @@ -293,17 +300,20 @@ def probs(self, value): """ name = self.name + '_probs' if len(self._prob.shape) == 1: # batch_shape is empty - return paddle.gather(self._prob, - value.reshape([-1], name=name), - name=name).reshape(value.shape, name=name) + return paddle.gather( + self._prob, value.reshape([-1], name=name), name=name + ).reshape(value.shape, name=name) else: if len(value.shape) == 1: return paddle.take_along_axis( self._prob, - paddle.reshape(value, - (len(self._prob.shape) - 1) * [1] + [-1], - name=name), - axis=-1) + paddle.reshape( + value, + (len(self._prob.shape) - 1) * [1] + [-1], + name=name, + ), + axis=-1, + ) else: return paddle.take_along_axis(self._prob, value, axis=-1) diff --git a/python/paddle/distribution/constraint.py b/python/paddle/distribution/constraint.py index 4cde3d30a565c..1891d20177913 100644 --- a/python/paddle/distribution/constraint.py +++ b/python/paddle/distribution/constraint.py @@ -15,21 +15,18 @@ class Constraint(object): - """Constraint condition for random variable. - """ + """Constraint condition for random variable.""" def __call__(self, value): raise NotImplementedError class Real(Constraint): - def __call__(self, value): return value == value class Range(Constraint): - def __init__(self, lower, upper): self._lower = lower self._upper = upper @@ -40,16 +37,15 @@ def __call__(self, value): class Positive(Constraint): - def __call__(self, value): - return value >= 0. + return value >= 0.0 class Simplex(Constraint): - def __call__(self, value): - return paddle.all(value >= 0, - axis=-1) and ((value.sum(-1) - 1).abs() < 1e-6) + return paddle.all(value >= 0, axis=-1) and ( + (value.sum(-1) - 1).abs() < 1e-6 + ) real = Real() diff --git a/python/paddle/distribution/dirichlet.py b/python/paddle/distribution/dirichlet.py index 7a8e918096821..c57bc41c01832 100644 --- a/python/paddle/distribution/dirichlet.py +++ b/python/paddle/distribution/dirichlet.py @@ -73,11 +73,13 @@ class Dirichlet(exponential_family.ExponentialFamily): def __init__(self, concentration): if concentration.dim() < 1: raise ValueError( - "`concentration` parameter must be at least one dimensional") + "`concentration` parameter must be at least one dimensional" + ) self.concentration = concentration - super(Dirichlet, self).__init__(concentration.shape[:-1], - concentration.shape[-1:]) + super(Dirichlet, self).__init__( + concentration.shape[:-1], concentration.shape[-1:] + ) @property def mean(self): @@ -97,7 +99,8 @@ def variance(self): """ concentration0 = self.concentration.sum(-1, keepdim=True) return (self.concentration * (concentration0 - self.concentration)) / ( - concentration0.pow(2) * (concentration0 + 1)) + concentration0.pow(2) * (concentration0 + 1) + ) def sample(self, shape=()): """Sample from dirichlet distribution. @@ -125,9 +128,11 @@ def log_prob(self, value): Args: value (Tensor): Value to be evaluated. """ - return ((paddle.log(value) * (self.concentration - 1.0)).sum(-1) + - paddle.lgamma(self.concentration.sum(-1)) - - paddle.lgamma(self.concentration).sum(-1)) + return ( + (paddle.log(value) * (self.concentration - 1.0)).sum(-1) + + paddle.lgamma(self.concentration.sum(-1)) + - paddle.lgamma(self.concentration).sum(-1) + ) def entropy(self): """Entropy of Dirichlet distribution. @@ -137,15 +142,18 @@ def entropy(self): """ concentration0 = self.concentration.sum(-1) k = self.concentration.shape[-1] - return (paddle.lgamma(self.concentration).sum(-1) - - paddle.lgamma(concentration0) - - (k - concentration0) * paddle.digamma(concentration0) - - ((self.concentration - 1.0) * - paddle.digamma(self.concentration)).sum(-1)) + return ( + paddle.lgamma(self.concentration).sum(-1) + - paddle.lgamma(concentration0) + - (k - concentration0) * paddle.digamma(concentration0) + - ( + (self.concentration - 1.0) * paddle.digamma(self.concentration) + ).sum(-1) + ) @property def _natural_parameters(self): - return (self.concentration, ) + return (self.concentration,) def _log_normalizer(self, x): return x.lgamma().sum(-1) - paddle.lgamma(x.sum(-1)) @@ -154,8 +162,9 @@ def _log_normalizer(self, x): def _dirichlet(concentration, name=None): op_type = 'dirichlet' - check_variable_and_dtype(concentration, 'concentration', - ['float32', 'float64'], op_type) + check_variable_and_dtype( + concentration, 'concentration', ['float32', 'float64'], op_type + ) if in_dygraph_mode(): return paddle._C_ops.dirichlet(concentration) @@ -164,9 +173,12 @@ def _dirichlet(concentration, name=None): else: helper = LayerHelper(op_type, **locals()) out = helper.create_variable_for_type_inference( - dtype=concentration.dtype) - helper.append_op(type=op_type, - inputs={"Alpha": concentration}, - outputs={'Out': out}, - attrs={}) + dtype=concentration.dtype + ) + helper.append_op( + type=op_type, + inputs={"Alpha": concentration}, + outputs={'Out': out}, + attrs={}, + ) return out diff --git a/python/paddle/distribution/distribution.py b/python/paddle/distribution/distribution.py index 14861d3794e32..65a899c0fac68 100644 --- a/python/paddle/distribution/distribution.py +++ b/python/paddle/distribution/distribution.py @@ -24,8 +24,12 @@ import numpy as np import paddle from paddle import _C_ops, _legacy_C_ops -from paddle.fluid.data_feeder import (check_variable_and_dtype, convert_dtype) -from paddle.fluid.framework import _non_static_mode, in_dygraph_mode, _in_legacy_dygraph +from paddle.fluid.data_feeder import check_variable_and_dtype, convert_dtype +from paddle.fluid.framework import ( + _non_static_mode, + in_dygraph_mode, + _in_legacy_dygraph, +) from paddle.fluid.layers import tensor @@ -45,10 +49,16 @@ class Distribution(object): def __init__(self, batch_shape=(), event_shape=()): - self._batch_shape = batch_shape if isinstance( - batch_shape, tuple) else tuple(batch_shape) - self._event_shape = event_shape if isinstance( - event_shape, tuple) else tuple(event_shape) + self._batch_shape = ( + batch_shape + if isinstance(batch_shape, tuple) + else tuple(batch_shape) + ) + self._event_shape = ( + event_shape + if isinstance(event_shape, tuple) + else tuple(event_shape) + ) super(Distribution, self).__init__() @@ -147,7 +157,8 @@ def _validate_args(self, *args): if is_variable and is_number: raise ValueError( - 'if one argument is Tensor, all arguments should be Tensor') + 'if one argument is Tensor, all arguments should be Tensor' + ) return is_variable @@ -162,15 +173,17 @@ def _to_tensor(self, *args): """ numpy_args = [] variable_args = [] - tmp = 0. + tmp = 0.0 for arg in args: if isinstance(arg, float): arg = [arg] if not isinstance(arg, (list, tuple, np.ndarray, tensor.Variable)): raise TypeError( - "Type of input args must be float, list, numpy.ndarray or Tensor, but received type {}" - .format(type(arg))) + "Type of input args must be float, list, numpy.ndarray or Tensor, but received type {}".format( + type(arg) + ) + ) arg_np = np.array(arg) arg_dtype = arg_np.dtype @@ -208,20 +221,24 @@ def _check_values_dtype_in_probs(self, param, value): value (Tensor): Change value's dtype if value's dtype is different from param. """ if _non_static_mode(): - if value.dtype != param.dtype and convert_dtype( - value.dtype) in ['float32', 'float64']: + if value.dtype != param.dtype and convert_dtype(value.dtype) in [ + 'float32', + 'float64', + ]: warnings.warn( "dtype of input 'value' needs to be the same as parameters of distribution class. dtype of 'value' will be converted." ) if in_dygraph_mode(): return _C_ops.cast(value, param.dtype) if _in_legacy_dygraph(): - return _legacy_C_ops.cast(value, 'in_dtype', value.dtype, - 'out_dtype', param.dtype) + return _legacy_C_ops.cast( + value, 'in_dtype', value.dtype, 'out_dtype', param.dtype + ) return value - check_variable_and_dtype(value, 'value', ['float32', 'float64'], - 'log_prob') + check_variable_and_dtype( + value, 'value', ['float32', 'float64'], 'log_prob' + ) if value.dtype != param.dtype: warnings.warn( "dtype of input 'value' needs to be the same as parameters of distribution class. dtype of 'value' will be converted." @@ -236,8 +253,11 @@ def _probs_to_logits(self, probs, is_binary=False): multi-dimensional, values of last axis denote the probabilities of occurrence of each of the events. """ - return (paddle.log(probs) - paddle.log1p(-probs)) \ - if is_binary else paddle.log(probs) + return ( + (paddle.log(probs) - paddle.log1p(-probs)) + if is_binary + else paddle.log(probs) + ) def _logits_to_probs(self, logits, is_binary=False): r""" @@ -245,5 +265,8 @@ def _logits_to_probs(self, logits, is_binary=False): log odds, whereas for the multi-dimensional case, the values along the last dimension denote the log probabilities of the events. """ - return paddle.nn.functional.sigmoid(logits) \ - if is_binary else paddle.nn.functional.softmax(logits, axis=-1) + return ( + paddle.nn.functional.sigmoid(logits) + if is_binary + else paddle.nn.functional.softmax(logits, axis=-1) + ) diff --git a/python/paddle/distribution/exponential_family.py b/python/paddle/distribution/exponential_family.py index 301c0cea13a58..0939e73f72643 100644 --- a/python/paddle/distribution/exponential_family.py +++ b/python/paddle/distribution/exponential_family.py @@ -62,9 +62,9 @@ def entropy(self): log_norm = self._log_normalizer(*natural_parameters) if _non_static_mode(): - grads = paddle.grad(log_norm.sum(), - natural_parameters, - create_graph=True) + grads = paddle.grad( + log_norm.sum(), natural_parameters, create_graph=True + ) else: grads = paddle.static.gradients(log_norm.sum(), natural_parameters) diff --git a/python/paddle/distribution/gumbel.py b/python/paddle/distribution/gumbel.py index 225e327cc91e9..1f16fd5529b69 100644 --- a/python/paddle/distribution/gumbel.py +++ b/python/paddle/distribution/gumbel.py @@ -70,7 +70,8 @@ def __init__(self, loc, scale): if not isinstance(loc, (numbers.Real, framework.Variable)): raise TypeError( - f"Expected type of loc is Real|Variable, but got {type(loc)}") + f"Expected type of loc is Real|Variable, but got {type(loc)}" + ) if not isinstance(scale, (numbers.Real, framework.Variable)): raise TypeError( f"Expected type of scale is Real|Variable, but got {type(scale)}" @@ -90,7 +91,8 @@ def __init__(self, loc, scale): finfo = np.finfo(dtype='float32') self.base_dist = paddle.distribution.Uniform( paddle.full_like(self.loc, float(finfo.tiny)), - paddle.full_like(self.loc, float(1 - finfo.eps))) + paddle.full_like(self.loc, float(1 - finfo.eps)), + ) self.transforms = () @@ -136,9 +138,11 @@ def variance(self): Tensor: The variance value. """ - temp = paddle.full(shape=self.loc.shape, - fill_value=math.pi * math.pi, - dtype=self.scale.dtype) + temp = paddle.full( + shape=self.loc.shape, + fill_value=math.pi * math.pi, + dtype=self.scale.dtype, + ) return paddle.pow(self.scale, 2) * temp / 6 @@ -230,13 +234,19 @@ def rsample(self, shape): """ exp_trans = paddle.distribution.ExpTransform() affine_trans_1 = paddle.distribution.AffineTransform( - paddle.full(shape=self.scale.shape, - fill_value=0, - dtype=self.loc.dtype), -paddle.ones_like(self.scale)) + paddle.full( + shape=self.scale.shape, fill_value=0, dtype=self.loc.dtype + ), + -paddle.ones_like(self.scale), + ) affine_trans_2 = paddle.distribution.AffineTransform( - self.loc, -self.scale) + self.loc, -self.scale + ) return affine_trans_2.forward( exp_trans.inverse( affine_trans_1.forward( - exp_trans.inverse(self._base.sample(shape))))) + exp_trans.inverse(self._base.sample(shape)) + ) + ) + ) diff --git a/python/paddle/distribution/independent.py b/python/paddle/distribution/independent.py index 9f02d802fc88f..a12ab32b85a4e 100644 --- a/python/paddle/distribution/independent.py +++ b/python/paddle/distribution/independent.py @@ -61,11 +61,14 @@ def __init__(self, base, reinterpreted_batch_rank): self._reinterpreted_batch_rank = reinterpreted_batch_rank shape = base.batch_shape + base.event_shape - super(Independent, - self).__init__(batch_shape=shape[:len(base.batch_shape) - - reinterpreted_batch_rank], - event_shape=shape[len(base.batch_shape) - - reinterpreted_batch_rank:]) + super(Independent, self).__init__( + batch_shape=shape[ + : len(base.batch_shape) - reinterpreted_batch_rank + ], + event_shape=shape[ + len(base.batch_shape) - reinterpreted_batch_rank : + ], + ) @property def mean(self): @@ -79,15 +82,17 @@ def sample(self, shape=()): return self._base.sample(shape) def log_prob(self, value): - return self._sum_rightmost(self._base.log_prob(value), - self._reinterpreted_batch_rank) + return self._sum_rightmost( + self._base.log_prob(value), self._reinterpreted_batch_rank + ) def prob(self, value): return self.log_prob(value).exp() def entropy(self): - return self._sum_rightmost(self._base.entropy(), - self._reinterpreted_batch_rank) + return self._sum_rightmost( + self._base.entropy(), self._reinterpreted_batch_rank + ) def _sum_rightmost(self, value, n): return value.sum(list(range(-n, 0))) if n > 0 else value diff --git a/python/paddle/distribution/kl.py b/python/paddle/distribution/kl.py index aa288a57ba0b7..4f95366391a15 100644 --- a/python/paddle/distribution/kl.py +++ b/python/paddle/distribution/kl.py @@ -85,8 +85,9 @@ def register_kl(cls_p, cls_q): def kl_beta_beta(): pass # insert implementation here """ - if (not issubclass(cls_p, Distribution) - or not issubclass(cls_q, Distribution)): + if not issubclass(cls_p, Distribution) or not issubclass( + cls_q, Distribution + ): raise TypeError('cls_p and cls_q must be subclass of Distribution') def decorator(f): @@ -100,8 +101,11 @@ def _dispatch(cls_p, cls_q): """Multiple dispatch into concrete implement function.""" # find all matched super class pair of p and q - matchs = [(super_p, super_q) for super_p, super_q in _REGISTER_TABLE - if issubclass(cls_p, super_p) and issubclass(cls_q, super_q)] + matchs = [ + (super_p, super_q) + for super_p, super_q in _REGISTER_TABLE + if issubclass(cls_p, super_p) and issubclass(cls_q, super_q) + ] if not matchs: raise NotImplementedError @@ -110,16 +114,20 @@ def _dispatch(cls_p, cls_q): if _REGISTER_TABLE[left_p, left_q] is not _REGISTER_TABLE[right_p, right_q]: warnings.warn( - 'Ambiguous kl_divergence({}, {}). Please register_kl({}, {})'. - format(cls_p.__name__, cls_q.__name__, left_p.__name__, - right_q.__name__), RuntimeWarning) + 'Ambiguous kl_divergence({}, {}). Please register_kl({}, {})'.format( + cls_p.__name__, + cls_q.__name__, + left_p.__name__, + right_q.__name__, + ), + RuntimeWarning, + ) return _REGISTER_TABLE[left_p, left_q] @functools.total_ordering class _Compare(object): - def __init__(self, *classes): self.classes = classes @@ -137,22 +145,33 @@ def __le__(self, other): @register_kl(Beta, Beta) def _kl_beta_beta(p, q): - return ((q.alpha.lgamma() + q.beta.lgamma() + (p.alpha + p.beta).lgamma()) - - (p.alpha.lgamma() + p.beta.lgamma() + (q.alpha + q.beta).lgamma()) + - ((p.alpha - q.alpha) * p.alpha.digamma()) + - ((p.beta - q.beta) * p.beta.digamma()) + - (((q.alpha + q.beta) - (p.alpha + p.beta)) * - (p.alpha + p.beta).digamma())) + return ( + (q.alpha.lgamma() + q.beta.lgamma() + (p.alpha + p.beta).lgamma()) + - (p.alpha.lgamma() + p.beta.lgamma() + (q.alpha + q.beta).lgamma()) + + ((p.alpha - q.alpha) * p.alpha.digamma()) + + ((p.beta - q.beta) * p.beta.digamma()) + + ( + ((q.alpha + q.beta) - (p.alpha + p.beta)) + * (p.alpha + p.beta).digamma() + ) + ) @register_kl(Dirichlet, Dirichlet) def _kl_dirichlet_dirichlet(p, q): return ( - (p.concentration.sum(-1).lgamma() - q.concentration.sum(-1).lgamma()) - - ((p.concentration.lgamma() - q.concentration.lgamma()).sum(-1)) + - (((p.concentration - q.concentration) * - (p.concentration.digamma() - - p.concentration.sum(-1).digamma().unsqueeze(-1))).sum(-1))) + (p.concentration.sum(-1).lgamma() - q.concentration.sum(-1).lgamma()) + - ((p.concentration.lgamma() - q.concentration.lgamma()).sum(-1)) + + ( + ( + (p.concentration - q.concentration) + * ( + p.concentration.digamma() + - p.concentration.sum(-1).digamma().unsqueeze(-1) + ) + ).sum(-1) + ) + ) @register_kl(Categorical, Categorical) @@ -177,8 +196,7 @@ def _kl_laplace_laplace(p, q): @register_kl(ExponentialFamily, ExponentialFamily) def _kl_expfamily_expfamily(p, q): - """Compute kl-divergence using `Bregman divergences `_ - """ + """Compute kl-divergence using `Bregman divergences `_""" if not type(p) == type(q): raise NotImplementedError @@ -194,19 +212,22 @@ def _kl_expfamily_expfamily(p, q): try: if _non_static_mode(): - p_grads = paddle.grad(p_log_norm, - p_natural_params, - create_graph=True) + p_grads = paddle.grad( + p_log_norm, p_natural_params, create_graph=True + ) else: p_grads = paddle.static.gradients(p_log_norm, p_natural_params) except RuntimeError as e: raise TypeError( - "Cann't compute kl_divergence({cls_p}, {cls_q}) use bregman divergence. Please register_kl({cls_p}, {cls_q})." - .format(cls_p=type(p).__name__, cls_q=type(q).__name__)) from e + "Cann't compute kl_divergence({cls_p}, {cls_q}) use bregman divergence. Please register_kl({cls_p}, {cls_q}).".format( + cls_p=type(p).__name__, cls_q=type(q).__name__ + ) + ) from e kl = q._log_normalizer(*q_natural_params) - p_log_norm - for p_param, q_param, p_grad in zip(p_natural_params, q_natural_params, - p_grads): + for p_param, q_param, p_grad in zip( + p_natural_params, q_natural_params, p_grads + ): term = (q_param - p_param) * p_grad kl -= _sum_rightmost(term, len(q.event_shape)) diff --git a/python/paddle/distribution/laplace.py b/python/paddle/distribution/laplace.py index 2b117f93bacec..a7a9301724ca4 100644 --- a/python/paddle/distribution/laplace.py +++ b/python/paddle/distribution/laplace.py @@ -55,7 +55,8 @@ class Laplace(distribution.Distribution): def __init__(self, loc, scale): if not isinstance(loc, (numbers.Real, framework.Variable)): raise TypeError( - f"Expected type of loc is Real|Variable, but got {type(loc)}") + f"Expected type of loc is Real|Variable, but got {type(loc)}" + ) if not isinstance(scale, (numbers.Real, framework.Variable)): raise TypeError( @@ -68,8 +69,9 @@ def __init__(self, loc, scale): if isinstance(scale, numbers.Real): scale = paddle.full(shape=(), fill_value=scale) - if (len(scale.shape) > 0 or len(loc.shape) > 0) and (loc.dtype - == scale.dtype): + if (len(scale.shape) > 0 or len(loc.shape) > 0) and ( + loc.dtype == scale.dtype + ): self.loc, self.scale = paddle.broadcast_tensors([loc, scale]) else: self.loc, self.scale = loc, scale @@ -135,10 +137,14 @@ def _validate_value(self, value): value = paddle.full(shape=(), fill_value=value) if value.dtype != self.scale.dtype: value = paddle.cast(value, self.scale.dtype) - if len(self.scale.shape) > 0 or len(self.loc.shape) > 0 or len( - value.shape) > 0: + if ( + len(self.scale.shape) > 0 + or len(self.loc.shape) > 0 + or len(value.shape) > 0 + ): loc, scale, value = paddle.broadcast_tensors( - [self.loc, self.scale, value]) + [self.loc, self.scale, value] + ) else: loc, scale = self.loc, self.scale @@ -178,7 +184,7 @@ def log_prob(self, value): loc, scale, value = self._validate_value(value) log_scale = -paddle.log(2 * scale) - return (log_scale - paddle.abs(value - loc) / scale) + return log_scale - paddle.abs(value - loc) / scale def entropy(self): r"""Entropy of Laplace distribution. @@ -238,8 +244,11 @@ def cdf(self, value): # [0.54758132]) """ loc, scale, value = self._validate_value(value) - iterm = (0.5 * (value - loc).sign() * - paddle.expm1(-(value - loc).abs() / scale)) + iterm = ( + 0.5 + * (value - loc).sign() + * paddle.expm1(-(value - loc).abs() / scale) + ) return 0.5 - iterm @@ -276,7 +285,7 @@ def icdf(self, value): loc, scale, value = self._validate_value(value) term = value - 0.5 - return (loc - scale * (term).sign() * paddle.log1p(-2 * term.abs())) + return loc - scale * (term).sign() * paddle.log1p(-2 * term.abs()) def sample(self, shape=()): r"""Generate samples of the specified shape. @@ -299,7 +308,8 @@ def sample(self, shape=()): """ if not isinstance(shape, tuple): raise TypeError( - f'Expected shape should be tuple[int], but got {type(shape)}') + f'Expected shape should be tuple[int], but got {type(shape)}' + ) with paddle.no_grad(): return self.rsample(shape) @@ -325,19 +335,22 @@ def rsample(self, shape): """ eps = self._get_eps() - shape = self._extend_shape(shape) or (1, ) - uniform = paddle.uniform(shape=shape, - min=float(np.nextafter(-1, 1)) + eps / 2, - max=1. - eps / 2, - dtype=self.loc.dtype) + shape = self._extend_shape(shape) or (1,) + uniform = paddle.uniform( + shape=shape, + min=float(np.nextafter(-1, 1)) + eps / 2, + max=1.0 - eps / 2, + dtype=self.loc.dtype, + ) if len(self.scale.shape) == 0 and len(self.loc.shape) == 0: loc, scale, uniform = paddle.broadcast_tensors( - [self.loc, self.scale, uniform]) + [self.loc, self.scale, uniform] + ) else: loc, scale = self.loc, self.scale - return (loc - scale * uniform.sign() * paddle.log1p(-uniform.abs())) + return loc - scale * uniform.sign() * paddle.log1p(-uniform.abs()) def _get_eps(self): """ @@ -351,8 +364,10 @@ def _get_eps(self): Float: An eps value by different data types. """ eps = 1.19209e-07 - if (self.loc.dtype == paddle.float64 - or self.loc.dtype == paddle.complex128): + if ( + self.loc.dtype == paddle.float64 + or self.loc.dtype == paddle.complex128 + ): eps = 2.22045e-16 return eps @@ -400,7 +415,7 @@ def kl_divergence(self, other): var_ratio = other.scale / self.scale t = paddle.abs(self.loc - other.loc) - term1 = ((self.scale * paddle.exp(-t / self.scale) + t) / other.scale) + term1 = (self.scale * paddle.exp(-t / self.scale) + t) / other.scale term2 = paddle.log(var_ratio) return term1 + term2 - 1 diff --git a/python/paddle/distribution/lognormal.py b/python/paddle/distribution/lognormal.py index b171e1ecbc61e..f49eee4d2e01a 100644 --- a/python/paddle/distribution/lognormal.py +++ b/python/paddle/distribution/lognormal.py @@ -15,8 +15,7 @@ import paddle from paddle.distribution.normal import Normal from paddle.distribution.transform import ExpTransform -from paddle.distribution.transformed_distribution import \ - TransformedDistribution +from paddle.distribution.transformed_distribution import TransformedDistribution class LogNormal(TransformedDistribution): @@ -104,8 +103,9 @@ def variance(self): Returns: Tensor: variance value. """ - return (paddle.expm1(self._base.variance) * - paddle.exp(2 * self._base.mean + self._base.variance)) + return paddle.expm1(self._base.variance) * paddle.exp( + 2 * self._base.mean + self._base.variance + ) def entropy(self): r"""Shannon entropy in nats. diff --git a/python/paddle/distribution/multinomial.py b/python/paddle/distribution/multinomial.py index 22f8aa820f993..56bb3da44817f 100644 --- a/python/paddle/distribution/multinomial.py +++ b/python/paddle/distribution/multinomial.py @@ -14,6 +14,7 @@ import paddle from paddle.distribution import categorical, distribution + try: from collections.abc import Iterable except: @@ -77,12 +78,14 @@ def __init__(self, total_count, probs): if probs.dim() < 1: raise ValueError( - 'probs parameter shoule not be none and over one dimension') + 'probs parameter shoule not be none and over one dimension' + ) self.probs = probs / probs.sum(-1, keepdim=True) self.total_count = total_count self._categorical = categorical.Categorical( - logits=self._probs_to_logits(probs)) + logits=self._probs_to_logits(probs) + ) super(Multinomial, self).__init__(probs.shape[:-1], probs.shape[-1:]) @@ -128,11 +131,15 @@ def log_prob(self, value): value = paddle.cast(value, self.probs.dtype) logits, value = paddle.broadcast_tensors( - [paddle.log(self.probs), value]) + [paddle.log(self.probs), value] + ) logits[(value == 0) & (paddle.isinf(logits))] = 0 - return (paddle.lgamma(value.sum(-1) + 1) - - paddle.lgamma(value + 1).sum(-1) + (value * logits).sum(-1)) + return ( + paddle.lgamma(value.sum(-1) + 1) + - paddle.lgamma(value + 1).sum(-1) + + (value * logits).sum(-1) + ) def sample(self, shape=()): """draw sample data from multinomial distribution @@ -143,11 +150,17 @@ def sample(self, shape=()): if not isinstance(shape, Iterable): raise TypeError('sample shape must be Iterable object.') - samples = self._categorical.sample([ - self.total_count, - ] + list(shape)) - return paddle.nn.functional.one_hot(samples, self.probs.shape[-1]).cast( - self.probs.dtype).sum(0) + samples = self._categorical.sample( + [ + self.total_count, + ] + + list(shape) + ) + return ( + paddle.nn.functional.one_hot(samples, self.probs.shape[-1]) + .cast(self.probs.dtype) + .sum(0) + ) def entropy(self): """entropy of multinomial distribution @@ -155,18 +168,18 @@ def entropy(self): Returns: Tensor: entropy value """ - n = paddle.full(shape=[1], - fill_value=self.total_count, - dtype=self.probs.dtype) + n = paddle.full( + shape=[1], fill_value=self.total_count, dtype=self.probs.dtype + ) support = paddle.arange( - self.total_count + 1, - dtype=self.probs.dtype).reshape((-1, ) + - (1, ) * len(self.probs.shape))[1:] + self.total_count + 1, dtype=self.probs.dtype + ).reshape((-1,) + (1,) * len(self.probs.shape))[1:] binomial_pmf = paddle.exp(self._binomial_logpmf(n, support)) - return ((n * self._categorical.entropy() - paddle.lgamma(n + 1)) + - ((binomial_pmf * paddle.lgamma(support + 1)).sum([0, -1]))) + return (n * self._categorical.entropy() - paddle.lgamma(n + 1)) + ( + (binomial_pmf * paddle.lgamma(support + 1)).sum([0, -1]) + ) def _binomial_logpmf(self, count, value): logits = self._probs_to_logits(self.probs, is_binary=True) @@ -175,9 +188,11 @@ def _binomial_logpmf(self, count, value): factor_k = paddle.lgamma(value + 1) factor_nmk = paddle.lgamma(count - value + 1) - norm = (count * _clip_by_zero(logits) + - count * paddle.log1p(paddle.exp(-paddle.abs(logits))) - - factor_n) + norm = ( + count * _clip_by_zero(logits) + + count * paddle.log1p(paddle.exp(-paddle.abs(logits))) + - factor_n + ) return value * logits - factor_k - factor_nmk - norm diff --git a/python/paddle/distribution/normal.py b/python/paddle/distribution/normal.py index f76f13b250e4c..a05c8e4703b65 100644 --- a/python/paddle/distribution/normal.py +++ b/python/paddle/distribution/normal.py @@ -16,10 +16,17 @@ import numpy as np import paddle from paddle.distribution import distribution -from paddle.fluid.data_feeder import (check_type, convert_dtype) +from paddle.fluid.data_feeder import check_type, convert_dtype from paddle.fluid.framework import _non_static_mode -from paddle.fluid.layers import (elementwise_add, elementwise_div, - elementwise_sub, nn, ops, tensor) +from paddle.fluid.layers import ( + elementwise_add, + elementwise_div, + elementwise_sub, + nn, + ops, + tensor, +) + try: from collections.abc import Iterable except: @@ -89,12 +96,18 @@ class Normal(distribution.Distribution): def __init__(self, loc, scale, name=None): if not _non_static_mode(): - check_type(loc, 'loc', - (int, float, np.ndarray, tensor.Variable, list, tuple), - 'Normal') - check_type(scale, 'scale', - (int, float, np.ndarray, tensor.Variable, list, tuple), - 'Normal') + check_type( + loc, + 'loc', + (int, float, np.ndarray, tensor.Variable, list, tuple), + 'Normal', + ) + check_type( + scale, + 'scale', + (int, float, np.ndarray, tensor.Variable, list, tuple), + 'Normal', + ) self.batch_size_unknown = False self.all_arg_is_float = False @@ -114,11 +127,15 @@ def __init__(self, loc, scale, name=None): else: if isinstance(loc, float) and isinstance(scale, float): self.all_arg_is_float = True - if isinstance(loc, np.ndarray) and str( - loc.dtype) in ['float32', 'float64']: + if isinstance(loc, np.ndarray) and str(loc.dtype) in [ + 'float32', + 'float64', + ]: self.dtype = loc.dtype - elif isinstance(scale, np.ndarray) and str( - scale.dtype) in ['float32', 'float64']: + elif isinstance(scale, np.ndarray) and str(scale.dtype) in [ + 'float32', + 'float64', + ]: self.dtype = scale.dtype # pylint: disable=unbalanced-tuple-unpacking self.loc, self.scale = self._to_tensor(loc, scale) @@ -169,22 +186,21 @@ def sample(self, shape=(), seed=0): if self.batch_size_unknown: output_shape = shape + batch_shape zero_tmp = tensor.fill_constant_batch_size_like( - self.loc + self.scale, batch_shape + shape, self.dtype, 0.) + self.loc + self.scale, batch_shape + shape, self.dtype, 0.0 + ) zero_tmp_reshape = nn.reshape(zero_tmp, output_shape) zero_tmp_shape = nn.shape(zero_tmp_reshape) - normal_random_tmp = nn.gaussian_random(zero_tmp_shape, - mean=0., - std=1., - seed=seed, - dtype=self.dtype) + normal_random_tmp = nn.gaussian_random( + zero_tmp_shape, mean=0.0, std=1.0, seed=seed, dtype=self.dtype + ) output = normal_random_tmp * (zero_tmp_reshape + self.scale) output = elementwise_add(output, self.loc, name=name) return output else: output_shape = shape + batch_shape output = nn.gaussian_random( - output_shape, mean=0., std=1., seed=seed, dtype=self.dtype) * ( - tensor.zeros(output_shape, dtype=self.dtype) + self.scale) + output_shape, mean=0.0, std=1.0, seed=seed, dtype=self.dtype + ) * (tensor.zeros(output_shape, dtype=self.dtype) + self.scale) output = elementwise_add(output, self.loc, name=name) if self.all_arg_is_float: return nn.reshape(output, shape, name=name) @@ -206,7 +222,7 @@ def rsample(self, shape=()): shape = self._extend_shape(tuple(shape)) eps = paddle.normal(shape=shape) - return (self.loc + eps * self.scale) + return self.loc + eps * self.scale def entropy(self): r"""Shannon entropy in nats. @@ -227,13 +243,14 @@ def entropy(self): """ name = self.name + '_entropy' batch_shape = list((self.loc + self.scale).shape) - zero_tmp = tensor.fill_constant_batch_size_like(self.loc + self.scale, - batch_shape, self.dtype, - 0.) - return elementwise_add(0.5 + zero_tmp, - 0.5 * math.log(2 * math.pi) + nn.log( - (self.scale + zero_tmp)), - name=name) + zero_tmp = tensor.fill_constant_batch_size_like( + self.loc + self.scale, batch_shape, self.dtype, 0.0 + ) + return elementwise_add( + 0.5 + zero_tmp, + 0.5 * math.log(2 * math.pi) + nn.log((self.scale + zero_tmp)), + name=name, + ) def log_prob(self, value): """Log probability density/mass function. @@ -250,10 +267,11 @@ def log_prob(self, value): var = self.scale * self.scale log_scale = nn.log(self.scale) - return elementwise_sub(-1. * ((value - self.loc) * (value - self.loc)) / - (2. * var), - log_scale + math.log(math.sqrt(2. * math.pi)), - name=name) + return elementwise_sub( + -1.0 * ((value - self.loc) * (value - self.loc)) / (2.0 * var), + log_scale + math.log(math.sqrt(2.0 * math.pi)), + name=name, + ) def probs(self, value): """Probability density/mass function. @@ -269,10 +287,13 @@ def probs(self, value): value = self._check_values_dtype_in_probs(self.loc, value) var = self.scale * self.scale - return elementwise_div(ops.exp(-1. * ((value - self.loc) * - (value - self.loc)) / (2. * var)), - (math.sqrt(2 * math.pi) * self.scale), - name=name) + return elementwise_div( + ops.exp( + -1.0 * ((value - self.loc) * (value - self.loc)) / (2.0 * var) + ), + (math.sqrt(2 * math.pi) * self.scale), + name=name, + ) def kl_divergence(self, other): r"""The KL-divergence between two normal distributions. @@ -312,9 +333,9 @@ def kl_divergence(self, other): name = self.name + '_kl_divergence' var_ratio = self.scale / other.scale - var_ratio = (var_ratio * var_ratio) + var_ratio = var_ratio * var_ratio t1 = (self.loc - other.loc) / other.scale - t1 = (t1 * t1) - return elementwise_add(0.5 * var_ratio, - 0.5 * (t1 - 1. - nn.log(var_ratio)), - name=name) + t1 = t1 * t1 + return elementwise_add( + 0.5 * var_ratio, 0.5 * (t1 - 1.0 - nn.log(var_ratio)), name=name + ) diff --git a/python/paddle/distribution/transform.py b/python/paddle/distribution/transform.py index 6cb0e048bab7e..f31dfe1b154c6 100644 --- a/python/paddle/distribution/transform.py +++ b/python/paddle/distribution/transform.py @@ -20,20 +20,33 @@ import paddle import paddle.nn.functional as F -from paddle.distribution import (constraint, distribution, - transformed_distribution, variable) +from paddle.distribution import ( + constraint, + distribution, + transformed_distribution, + variable, +) __all__ = [ # noqa - 'Transform', 'AbsTransform', 'AffineTransform', 'ChainTransform', - 'ExpTransform', 'IndependentTransform', 'PowerTransform', - 'ReshapeTransform', 'SigmoidTransform', 'SoftmaxTransform', - 'StackTransform', 'StickBreakingTransform', 'TanhTransform' + 'Transform', + 'AbsTransform', + 'AffineTransform', + 'ChainTransform', + 'ExpTransform', + 'IndependentTransform', + 'PowerTransform', + 'ReshapeTransform', + 'SigmoidTransform', + 'SoftmaxTransform', + 'StackTransform', + 'StickBreakingTransform', + 'TanhTransform', ] class Type(enum.Enum): - """Mapping type of a transformation. - """ + """Mapping type of a transformation.""" + BIJECTION = 'bijection' # bijective(injective and surjective) INJECTION = 'injection' # injective-only SURJECTION = 'surjection' # surjective-only @@ -41,8 +54,7 @@ class Type(enum.Enum): @classmethod def is_injective(cls, _type): - """Both bijection and injection are injective mapping. - """ + """Both bijection and injection are injective mapping.""" return _type in (cls.BIJECTION, cls.INJECTION) @@ -138,7 +150,8 @@ def __call__(self, input): """ if isinstance(input, distribution.Distribution): return transformed_distribution.TransformedDistribution( - input, [self]) + input, [self] + ) if isinstance(input, Transform): return ChainTransform([self, input]) return self.forward(input) @@ -157,11 +170,13 @@ def forward(self, x): """ if not isinstance(x, paddle.fluid.framework.Variable): raise TypeError( - f"Expected 'x' is a Tensor or Real, but got {type(x)}.") + f"Expected 'x' is a Tensor or Real, but got {type(x)}." + ) if x.dim() < self._domain.event_rank: raise ValueError( f'The dimensions of x({x.dim()}) should be ' - f'grater than or equal to {self._domain.event_rank}') + f'grater than or equal to {self._domain.event_rank}' + ) return self._forward(x) def inverse(self, y): @@ -176,11 +191,13 @@ def inverse(self, y): """ if not isinstance(y, paddle.fluid.framework.Variable): raise TypeError( - f"Expected 'y' is a Tensor or Real, but got {type(y)}.") + f"Expected 'y' is a Tensor or Real, but got {type(y)}." + ) if y.dim() < self._codomain.event_rank: raise ValueError( f'The dimensions of y({y.dim()}) should be ' - f'grater than or equal to {self._codomain.event_rank}') + f'grater than or equal to {self._codomain.event_rank}' + ) return self._inverse(y) def forward_log_det_jacobian(self, x): @@ -196,16 +213,21 @@ def forward_log_det_jacobian(self, x): """ if not isinstance(x, paddle.fluid.framework.Variable): raise TypeError( - f"Expected 'y' is a Tensor or Real, but got {type(x)}.") - if isinstance(x, paddle.fluid.framework.Variable - ) and x.dim() < self._domain.event_rank: + f"Expected 'y' is a Tensor or Real, but got {type(x)}." + ) + if ( + isinstance(x, paddle.fluid.framework.Variable) + and x.dim() < self._domain.event_rank + ): raise ValueError( f'The dimensions of x({x.dim()}) should be ' - f'grater than or equal to {self._domain.event_rank}') + f'grater than or equal to {self._domain.event_rank}' + ) if not self._is_injective(): raise NotImplementedError( "forward_log_det_jacobian can't be implemented for non-injective" - "transforms.") + "transforms." + ) return self._call_forward_log_det_jacobian(x) @@ -226,7 +248,8 @@ def inverse_log_det_jacobian(self, y): if y.dim() < self._codomain.event_rank: raise ValueError( f'The dimensions of y({y.dim()}) should be ' - f'grater than or equal to {self._codomain.event_rank}') + f'grater than or equal to {self._codomain.event_rank}' + ) return self._call_inverse_log_det_jacobian(y) def forward_shape(self, shape): @@ -240,7 +263,8 @@ def forward_shape(self, shape): """ if not isinstance(shape, typing.Sequence): raise TypeError( - f"Expected shape is Sequence[int] type, but got {type(shape)}.") + f"Expected shape is Sequence[int] type, but got {type(shape)}." + ) return self._forward_shape(shape) def inverse_shape(self, shape): @@ -254,7 +278,8 @@ def inverse_shape(self, shape): """ if not isinstance(shape, typing.Sequence): raise TypeError( - f"Expected shape is Sequence[int] type, but got {type(shape)}.") + f"Expected shape is Sequence[int] type, but got {type(shape)}." + ) return self._inverse_shape(shape) @property @@ -287,7 +312,8 @@ def _call_forward_log_det_jacobian(self, x): return -self._inverse_log_det_jacobian(self.forward(x)) raise NotImplementedError( 'Neither _forward_log_det_jacobian nor _inverse_log_det_jacobian' - 'is implemented. One of them is required.') + 'is implemented. One of them is required.' + ) def _call_inverse_log_det_jacobian(self, y): """Inner method called by ``inverse_log_det_jacobian``""" @@ -297,7 +323,8 @@ def _call_inverse_log_det_jacobian(self, y): return -self._forward_log_det_jacobian(self._inverse(y)) raise NotImplementedError( 'Neither _forward_log_det_jacobian nor _inverse_log_det_jacobian ' - 'is implemented. One of them is required') + 'is implemented. One of them is required' + ) def _forward_shape(self, shape): """Inner method called by ``forward_shape``, which is used to infer the @@ -420,7 +447,8 @@ def __init__(self, loc, scale): raise TypeError(f"Expected 'loc' is a Tensor, but got {type(loc)}") if not isinstance(scale, paddle.fluid.framework.Variable): raise TypeError( - f"Expected scale is a Tensor, but got {type(scale)}") + f"Expected scale is a Tensor, but got {type(scale)}" + ) self._loc = loc self._scale = scale super(AffineTransform, self).__init__() @@ -446,13 +474,17 @@ def _forward_shape(self, shape): return tuple( paddle.broadcast_shape( paddle.broadcast_shape(shape, self._loc.shape), - self._scale.shape)) + self._scale.shape, + ) + ) def _inverse_shape(self, shape): return tuple( paddle.broadcast_shape( paddle.broadcast_shape(shape, self._loc.shape), - self._scale.shape)) + self._scale.shape, + ) + ) @property def _domain(self): @@ -504,7 +536,8 @@ def __init__(self, transforms): ) if not all(isinstance(t, Transform) for t in transforms): raise TypeError( - "All elements of transforms should be Transform type.") + "All elements of transforms should be Transform type." + ) self.transforms = transforms super(ChainTransform, self).__init__() @@ -523,11 +556,12 @@ def _inverse(self, y): return y def _forward_log_det_jacobian(self, x): - value = 0. + value = 0.0 event_rank = self._domain.event_rank for t in self.transforms: - value += self._sum_rightmost(t.forward_log_det_jacobian(x), - event_rank - t._domain.event_rank) + value += self._sum_rightmost( + t.forward_log_det_jacobian(x), event_rank - t._domain.event_rank + ) x = t.forward(x) event_rank += t._codomain.event_rank - t._domain.event_rank return value @@ -682,7 +716,8 @@ class IndependentTransform(Transform): def __init__(self, base, reinterpreted_batch_rank): if not isinstance(base, Transform): raise TypeError( - f"Expected 'base' is Transform type, but get {type(base)}") + f"Expected 'base' is Transform type, but get {type(base)}" + ) if reinterpreted_batch_rank <= 0: raise ValueError( f"Expected 'reinterpreted_batch_rank' is grater than zero, but got {reinterpreted_batch_rank}" @@ -707,7 +742,8 @@ def _inverse(self, y): def _forward_log_det_jacobian(self, x): return self._base.forward_log_det_jacobian(x).sum( - list(range(-self._reinterpreted_batch_rank, 0))) + list(range(-self._reinterpreted_batch_rank, 0)) + ) def _forward_shape(self, shape): return self._base.forward_shape(shape) @@ -717,13 +753,15 @@ def _inverse_shape(self, shape): @property def _domain(self): - return variable.Independent(self._base._domain, - self._reinterpreted_batch_rank) + return variable.Independent( + self._base._domain, self._reinterpreted_batch_rank + ) @property def _codomain(self): - return variable.Independent(self._base._codomain, - self._reinterpreted_batch_rank) + return variable.Independent( + self._base._codomain, self._reinterpreted_batch_rank + ) class PowerTransform(Transform): @@ -757,7 +795,8 @@ class PowerTransform(Transform): def __init__(self, power): if not isinstance(power, paddle.fluid.framework.Variable): raise TypeError( - f"Expected 'power' is a tensor, but got {type(power)}") + f"Expected 'power' is a tensor, but got {type(power)}" + ) self._power = power super(PowerTransform, self).__init__() @@ -826,13 +865,16 @@ class ReshapeTransform(Transform): def __init__(self, in_event_shape, out_event_shape): if not isinstance(in_event_shape, typing.Sequence) or not isinstance( - out_event_shape, typing.Sequence): + out_event_shape, typing.Sequence + ): raise TypeError( f"Expected type of 'in_event_shape' and 'out_event_shape' is " f"Squence[int], but got 'in_event_shape': {in_event_shape}, " - f"'out_event_shape': {out_event_shape}") + f"'out_event_shape': {out_event_shape}" + ) if functools.reduce(operator.mul, in_event_shape) != functools.reduce( - operator.mul, out_event_shape): + operator.mul, out_event_shape + ): raise ValueError( f"The numel of 'in_event_shape' should be 'out_event_shape', " f"but got {functools.reduce(operator.mul, in_event_shape)}!={functools.reduce(operator.mul, out_event_shape)}" @@ -860,39 +902,45 @@ def _codomain(self): def _forward(self, x): return x.reshape( - tuple(x.shape)[:x.dim() - len(self._in_event_shape)] + - self._out_event_shape) + tuple(x.shape)[: x.dim() - len(self._in_event_shape)] + + self._out_event_shape + ) def _inverse(self, y): return y.reshape( - tuple(y.shape)[:y.dim() - len(self._out_event_shape)] + - self._in_event_shape) + tuple(y.shape)[: y.dim() - len(self._out_event_shape)] + + self._in_event_shape + ) def _forward_shape(self, shape): if len(shape) < len(self._in_event_shape): raise ValueError( f"Expected length of 'shape' is not less than {len(self._in_event_shape)}, but got {len(shape)}" ) - if shape[-len(self._in_event_shape):] != self._in_event_shape: + if shape[-len(self._in_event_shape) :] != self._in_event_shape: raise ValueError( f"Event shape mismatch, expected: {self._in_event_shape}, but got {shape[-len(self._in_event_shape):]}" ) - return tuple(shape[:-len(self._in_event_shape)]) + self._out_event_shape + return ( + tuple(shape[: -len(self._in_event_shape)]) + self._out_event_shape + ) def _inverse_shape(self, shape): if len(shape) < len(self._out_event_shape): raise ValueError( f"Expected 'shape' length is not less than {len(self._out_event_shape)}, but got {len(shape)}" ) - if shape[-len(self._out_event_shape):] != self._out_event_shape: + if shape[-len(self._out_event_shape) :] != self._out_event_shape: raise ValueError( f"Event shape mismatch, expected: {self._out_event_shape}, but got {shape[-len(self._out_event_shape):]}" ) - return tuple(shape[:-len(self._out_event_shape)]) + self._in_event_shape + return ( + tuple(shape[: -len(self._out_event_shape)]) + self._in_event_shape + ) def _forward_log_det_jacobian(self, x): # paddle.zeros not support zero dimension Tensor. - shape = x.shape[:x.dim() - len(self._in_event_shape)] or [1] + shape = x.shape[: x.dim() - len(self._in_event_shape)] or [1] return paddle.zeros(shape, dtype=x.dtype) @@ -927,7 +975,7 @@ def _domain(self): @property def _codomain(self): - return variable.Variable(False, 0, constraint.Range(0., 1.)) + return variable.Variable(False, 0, constraint.Range(0.0, 1.0)) def _forward(self, x): return F.sigmoid(x) @@ -996,7 +1044,7 @@ def _inverse_shape(self, shape): class StackTransform(Transform): - r""" ``StackTransform`` applies a sequence of transformations along the + r"""``StackTransform`` applies a sequence of transformations along the specific axis. Args: @@ -1043,7 +1091,8 @@ def __init__(self, transforms, axis=0): ) if not all(isinstance(t, Transform) for t in transforms): raise TypeError( - 'Expected all element in transforms is Transform Type.') + 'Expected all element in transforms is Transform Type.' + ) if not isinstance(axis, int): raise TypeError(f"Expected 'axis' is int, but got{type(axis)}.") @@ -1063,34 +1112,45 @@ def axis(self): def _forward(self, x): self._check_size(x) - return paddle.stack([ - t.forward(v) - for v, t in zip(paddle.unstack(x, self._axis), self._transforms) - ], self._axis) + return paddle.stack( + [ + t.forward(v) + for v, t in zip(paddle.unstack(x, self._axis), self._transforms) + ], + self._axis, + ) def _inverse(self, y): self._check_size(y) - return paddle.stack([ - t.inverse(v) - for v, t in zip(paddle.unstack(y, self._axis), self._transforms) - ], self._axis) + return paddle.stack( + [ + t.inverse(v) + for v, t in zip(paddle.unstack(y, self._axis), self._transforms) + ], + self._axis, + ) def _forward_log_det_jacobian(self, x): self._check_size(x) - return paddle.stack([ - t.forward_log_det_jacobian(v) - for v, t in zip(paddle.unstack(x, self._axis), self._transforms) - ], self._axis) + return paddle.stack( + [ + t.forward_log_det_jacobian(v) + for v, t in zip(paddle.unstack(x, self._axis), self._transforms) + ], + self._axis, + ) def _check_size(self, v): if not (-v.dim() <= self._axis < v.dim()): raise ValueError( f'Input dimensions {v.dim()} should be grater than stack ' - f'transform axis {self._axis}.') + f'transform axis {self._axis}.' + ) if v.shape[self._axis] != len(self._transforms): raise ValueError( f'Input size along {self._axis} should be equal to the ' - f'length of transforms.') + f'length of transforms.' + ) @property def _domain(self): @@ -1098,8 +1158,9 @@ def _domain(self): @property def _codomain(self): - return variable.Stack([t._codomain for t in self._transforms], - self._axis) + return variable.Stack( + [t._codomain for t in self._transforms], self._axis + ) class StickBreakingTransform(Transform): @@ -1132,8 +1193,9 @@ def _forward(self, x): offset = x.shape[-1] + 1 - paddle.ones([x.shape[-1]]).cumsum(-1) z = F.sigmoid(x - offset.log()) z_cumprod = (1 - z).cumprod(-1) - return F.pad(z, [0] * 2 * (len(x.shape) - 1) + [0, 1], value=1) * \ - F.pad(z_cumprod, [0] * 2 * (len(x.shape) - 1) + [1, 0], value=1) + return F.pad(z, [0] * 2 * (len(x.shape) - 1) + [0, 1], value=1) * F.pad( + z_cumprod, [0] * 2 * (len(x.shape) - 1) + [1, 0], value=1 + ) def _inverse(self, y): y_crop = y[..., :-1] @@ -1151,12 +1213,12 @@ def _forward_log_det_jacobian(self, x): def _forward_shape(self, shape): if not shape: raise ValueError(f"Expected 'shape' is not empty, but got {shape}") - return shape[:-1] + (shape[-1] + 1, ) + return shape[:-1] + (shape[-1] + 1,) def _inverse_shape(self, shape): if not shape: raise ValueError(f"Expected 'shape' is not empty, but got {shape}") - return shape[:-1] + (shape[-1] - 1, ) + return shape[:-1] + (shape[-1] - 1,) @property def _domain(self): @@ -1220,4 +1282,4 @@ def _forward_log_det_jacobian(self, x): See details: https://github.com/tensorflow/probability/blob/master/tensorflow_probability/python/bijectors/tanh.py#L69-L80 """ - return 2. * (math.log(2.) - x - F.softplus(-2. * x)) + return 2.0 * (math.log(2.0) - x - F.softplus(-2.0 * x)) diff --git a/python/paddle/distribution/transformed_distribution.py b/python/paddle/distribution/transformed_distribution.py index 8433ed0c91098..1f97150cc9aa7 100644 --- a/python/paddle/distribution/transformed_distribution.py +++ b/python/paddle/distribution/transformed_distribution.py @@ -65,8 +65,9 @@ def __init__(self, base, transforms): self._base = base self._transforms = transforms if not transforms: - super(TransformedDistribution, - self).__init__(base.batch_shape, base.event_shape) + super(TransformedDistribution, self).__init__( + base.batch_shape, base.event_shape + ) return if len(base.batch_shape + base.event_shape) < chain._domain.event_rank: raise ValueError( @@ -74,15 +75,23 @@ def __init__(self, base, transforms): ) if chain._domain.event_rank > len(base.event_shape): base = independent.Independent( - (base, chain._domain.event_rank - len(base.event_shape))) + (base, chain._domain.event_rank - len(base.event_shape)) + ) - transformed_shape = chain.forward_shape(base.batch_shape + - base.event_shape) - transformed_event_rank = chain._codomain.event_rank + \ - max(len(base.event_shape) - chain._domain.event_rank, 0) + transformed_shape = chain.forward_shape( + base.batch_shape + base.event_shape + ) + transformed_event_rank = chain._codomain.event_rank + max( + len(base.event_shape) - chain._domain.event_rank, 0 + ) super(TransformedDistribution, self).__init__( - transformed_shape[:len(transformed_shape) - transformed_event_rank], - transformed_shape[len(transformed_shape) - transformed_event_rank:]) + transformed_shape[ + : len(transformed_shape) - transformed_event_rank + ], + transformed_shape[ + len(transformed_shape) - transformed_event_rank : + ], + ) def sample(self, shape=()): """Sample from ``TransformedDistribution``. @@ -127,12 +136,13 @@ def log_prob(self, value): for t in reversed(self._transforms): x = t.inverse(y) event_rank += t._domain.event_rank - t._codomain.event_rank - log_prob = log_prob - \ - _sum_rightmost(t.forward_log_det_jacobian( - x), event_rank - t._domain.event_rank) + log_prob = log_prob - _sum_rightmost( + t.forward_log_det_jacobian(x), event_rank - t._domain.event_rank + ) y = x - log_prob += _sum_rightmost(self._base.log_prob(y), - event_rank - len(self._base.event_shape)) + log_prob += _sum_rightmost( + self._base.log_prob(y), event_rank - len(self._base.event_shape) + ) return log_prob diff --git a/python/paddle/distribution/uniform.py b/python/paddle/distribution/uniform.py index 5f9dc6c3d37df..4e48dc606223f 100644 --- a/python/paddle/distribution/uniform.py +++ b/python/paddle/distribution/uniform.py @@ -15,10 +15,19 @@ import numpy as np from paddle import _C_ops, _legacy_C_ops from paddle.distribution import distribution -from paddle.fluid.data_feeder import (check_type, convert_dtype) -from paddle.fluid.framework import _non_static_mode, in_dygraph_mode, _in_legacy_dygraph -from paddle.fluid.layers import (elementwise_add, elementwise_div, - elementwise_sub, nn, tensor) +from paddle.fluid.data_feeder import check_type, convert_dtype +from paddle.fluid.framework import ( + _non_static_mode, + in_dygraph_mode, + _in_legacy_dygraph, +) +from paddle.fluid.layers import ( + elementwise_add, + elementwise_div, + elementwise_sub, + nn, + tensor, +) class Uniform(distribution.Distribution): @@ -86,12 +95,18 @@ class Uniform(distribution.Distribution): def __init__(self, low, high, name=None): if not _non_static_mode(): - check_type(low, 'low', - (int, float, np.ndarray, tensor.Variable, list, tuple), - 'Uniform') - check_type(high, 'high', - (int, float, np.ndarray, tensor.Variable, list, tuple), - 'Uniform') + check_type( + low, + 'low', + (int, float, np.ndarray, tensor.Variable, list, tuple), + 'Uniform', + ) + check_type( + high, + 'high', + (int, float, np.ndarray, tensor.Variable, list, tuple), + 'Uniform', + ) self.all_arg_is_float = False self.batch_size_unknown = False @@ -111,11 +126,15 @@ def __init__(self, low, high, name=None): else: if isinstance(low, float) and isinstance(high, float): self.all_arg_is_float = True - if isinstance(low, np.ndarray) and str( - low.dtype) in ['float32', 'float64']: + if isinstance(low, np.ndarray) and str(low.dtype) in [ + 'float32', + 'float64', + ]: self.dtype = low.dtype - elif isinstance(high, np.ndarray) and str( - high.dtype) in ['float32', 'float64']: + elif isinstance(high, np.ndarray) and str(high.dtype) in [ + 'float32', + 'float64', + ]: self.dtype = high.dtype # pylint: disable=unbalanced-tuple-unpacking self.low, self.high = self._to_tensor(low, high) @@ -145,27 +164,33 @@ def sample(self, shape, seed=0): if self.batch_size_unknown: output_shape = shape + batch_shape zero_tmp = tensor.fill_constant_batch_size_like( - self.low + self.high, batch_shape + shape, self.dtype, 0.) + self.low + self.high, batch_shape + shape, self.dtype, 0.0 + ) uniform_random_tmp = nn.uniform_random_batch_size_like( zero_tmp, zero_tmp.shape, dtype=self.dtype, - min=0., - max=1., - seed=seed) + min=0.0, + max=1.0, + seed=seed, + ) zero_tmp_reshape = nn.reshape(zero_tmp, output_shape) - uniform_random_tmp_reshape = nn.reshape(uniform_random_tmp, - output_shape) - output = uniform_random_tmp_reshape * (zero_tmp_reshape + - self.high - self.low) + uniform_random_tmp_reshape = nn.reshape( + uniform_random_tmp, output_shape + ) + output = uniform_random_tmp_reshape * ( + zero_tmp_reshape + self.high - self.low + ) output = elementwise_add(output, self.low, name=name) return output else: output_shape = shape + batch_shape output = nn.uniform_random( - output_shape, dtype=self.dtype, min=0., max=1., - seed=seed) * (tensor.zeros(output_shape, dtype=self.dtype) + - (self.high - self.low)) + output_shape, dtype=self.dtype, min=0.0, max=1.0, seed=seed + ) * ( + tensor.zeros(output_shape, dtype=self.dtype) + + (self.high - self.low) + ) output = elementwise_add(output, self.low, name=name) if self.all_arg_is_float: return nn.reshape(output, shape, name=name) @@ -194,10 +219,12 @@ def log_prob(self, value): return nn.log(lb * ub) - nn.log(self.high - self.low) if _in_legacy_dygraph(): - lb = _legacy_C_ops.cast(lb_bool, 'in_dtype', lb_bool.dtype, - 'out_dtype', value.dtype) - ub = _legacy_C_ops.cast(ub_bool, 'in_dtype', ub_bool.dtype, - 'out_dtype', value.dtype) + lb = _legacy_C_ops.cast( + lb_bool, 'in_dtype', lb_bool.dtype, 'out_dtype', value.dtype + ) + ub = _legacy_C_ops.cast( + ub_bool, 'in_dtype', ub_bool.dtype, 'out_dtype', value.dtype + ) return nn.log(lb * ub) - nn.log(self.high - self.low) name = self.name + '_log_prob' @@ -205,9 +232,9 @@ def log_prob(self, value): ub_bool = value < self.high lb = tensor.cast(lb_bool, dtype=value.dtype) ub = tensor.cast(ub_bool, dtype=value.dtype) - return elementwise_sub(nn.log(lb * ub), - nn.log(self.high - self.low), - name=name) + return elementwise_sub( + nn.log(lb * ub), nn.log(self.high - self.low), name=name + ) def probs(self, value): """Probability density/mass function. @@ -230,10 +257,12 @@ def probs(self, value): return (lb * ub) / (self.high - self.low) if _in_legacy_dygraph(): - lb = _legacy_C_ops.cast(lb_bool, 'in_dtype', lb_bool.dtype, - 'out_dtype', value.dtype) - ub = _legacy_C_ops.cast(ub_bool, 'in_dtype', ub_bool.dtype, - 'out_dtype', value.dtype) + lb = _legacy_C_ops.cast( + lb_bool, 'in_dtype', lb_bool.dtype, 'out_dtype', value.dtype + ) + ub = _legacy_C_ops.cast( + ub_bool, 'in_dtype', ub_bool.dtype, 'out_dtype', value.dtype + ) return (lb * ub) / (self.high - self.low) name = self.name + '_probs' diff --git a/python/paddle/distribution/variable.py b/python/paddle/distribution/variable.py index f6cced041645b..2bde6c79aeb98 100644 --- a/python/paddle/distribution/variable.py +++ b/python/paddle/distribution/variable.py @@ -43,13 +43,11 @@ def constraint(self, value): class Real(Variable): - def __init__(self, event_rank=0): super(Real, self).__init__(False, event_rank, constraint.real) class Positive(Variable): - def __init__(self, event_rank=0): super(Positive, self).__init__(False, event_rank, constraint.positive) @@ -66,23 +64,24 @@ class Independent(Variable): def __init__(self, base, reinterpreted_batch_rank): self._base = base self._reinterpreted_batch_rank = reinterpreted_batch_rank - super(Independent, - self).__init__(base.is_discrete, - base.event_rank + reinterpreted_batch_rank) + super(Independent, self).__init__( + base.is_discrete, base.event_rank + reinterpreted_batch_rank + ) def constraint(self, value): ret = self._base.constraint(value) if ret.dim() < self._reinterpreted_batch_rank: raise ValueError( "Input dimensions must be equal or grater than {}".format( - self._reinterpreted_batch_rank)) - return ret.reshape(ret.shape[:ret.dim() - - self.reinterpreted_batch_rank] + - (-1, )).all(-1) + self._reinterpreted_batch_rank + ) + ) + return ret.reshape( + ret.shape[: ret.dim() - self.reinterpreted_batch_rank] + (-1,) + ).all(-1) class Stack(Variable): - def __init__(self, vars, axis=0): self._vars = vars self._axis = axis @@ -102,12 +101,18 @@ def constraint(self, value): if not (-value.dim() <= self._axis < value.dim()): raise ValueError( f'Input dimensions {value.dim()} should be grater than stack ' - f'constraint axis {self._axis}.') - - return paddle.stack([ - var.check(value) - for var, value in zip(self._vars, paddle.unstack(value, self._axis)) - ], self._axis) + f'constraint axis {self._axis}.' + ) + + return paddle.stack( + [ + var.check(value) + for var, value in zip( + self._vars, paddle.unstack(value, self._axis) + ) + ], + self._axis, + ) real = Real() diff --git a/python/paddle/fft.py b/python/paddle/fft.py index b873d7b1df610..8bc95cd37e9f2 100644 --- a/python/paddle/fft.py +++ b/python/paddle/fft.py @@ -51,61 +51,76 @@ def _check_normalization(norm): if norm not in ['forward', 'backward', 'ortho']: raise ValueError( - "Unexpected norm: {}. Norm should be forward, backward or ortho". - format(norm)) + "Unexpected norm: {}. Norm should be forward, backward or ortho".format( + norm + ) + ) def _check_fft_n(n): if not isinstance(n, int): raise ValueError( - "Invalid FFT argument n({}), it shoule be an integer.".format(n)) + "Invalid FFT argument n({}), it shoule be an integer.".format(n) + ) if n <= 0: raise ValueError( - "Invalid FFT argument n({}), it should be positive.".format(n)) + "Invalid FFT argument n({}), it should be positive.".format(n) + ) def _check_fft_shape(x, s): ndim = x.ndim if not isinstance(s, Sequence): raise ValueError( - "Invaid FFT argument s({}), it should be a sequence of integers.") + "Invaid FFT argument s({}), it should be a sequence of integers." + ) if len(s) > ndim: raise ValueError( "Length of FFT argument s should not be larger than the rank of input. " - "Received s: {}, rank of x: {}".format(s, ndim)) + "Received s: {}, rank of x: {}".format(s, ndim) + ) for size in s: if not isinstance(size, int) or size <= 0: - raise ValueError("FFT sizes {} contains invalid value ({})".format( - s, size)) + raise ValueError( + "FFT sizes {} contains invalid value ({})".format(s, size) + ) def _check_fft_axis(x, axis): ndim = x.ndim if not isinstance(axis, int): raise ValueError( - "Invalid FFT axis ({}), it shoule be an integer.".format(axis)) + "Invalid FFT axis ({}), it shoule be an integer.".format(axis) + ) if axis < -ndim or axis >= ndim: raise ValueError( "Invalid FFT axis ({}), it should be in range [-{}, {})".format( - axis, ndim, ndim)) + axis, ndim, ndim + ) + ) def _check_fft_axes(x, axes): ndim = x.ndim if not isinstance(axes, Sequence): raise ValueError( - "Invalid FFT axes ({}), it should be a sequence of integers.". - format(axes)) + "Invalid FFT axes ({}), it should be a sequence of integers.".format( + axes + ) + ) if len(axes) > ndim: raise ValueError( "Length of fft axes should not be larger than the rank of input. " - "Received, len of axes: {}, rank of x: {}".format(len(axes), ndim)) + "Received, len of axes: {}, rank of x: {}".format(len(axes), ndim) + ) for axis in axes: if not isinstance(axis, int) or axis < -ndim or axis >= ndim: raise ValueError( - "FFT axes {} contains invalid value ({}), it should be in range [-{}, {})" - .format(axes, axis, ndim, ndim)) + "FFT axes {} contains invalid value ({}), it should be in range [-{}, {})".format( + axes, axis, ndim, ndim + ) + ) def _resize_fft_input(x, s, axes): @@ -127,10 +142,12 @@ def _resize_fft_input(x, s, axes): slices.append((0, s[i])) if axes_to_slice: - x = paddle.slice(x, - axes_to_slice, - starts=[item[0] for item in slices], - ends=[item[1] for item in slices]) + x = paddle.slice( + x, + axes_to_slice, + starts=[item[0] for item in slices], + ends=[item[1] for item in slices], + ) if axes_to_pad: padding_widths = [0] * (2 * ndim) for axis, pad in zip(axes_to_pad, paddings): @@ -146,8 +163,9 @@ def _normalize_axes(x, axes): def _check_at_least_ndim(x, rank): if x.ndim < rank: - raise ValueError("The rank of the input ({}) should >= {}".format( - x.ndim, rank)) + raise ValueError( + "The rank of the input ({}) should >= {}".format(x.ndim, rank) + ) # public APIs 1d @@ -197,13 +215,9 @@ def fft(x, n=None, axis=-1, norm="backward", name=None): """ if is_integer(x) or is_floating_point(x): - return fft_r2c(x, - n, - axis, - norm, - forward=True, - onesided=False, - name=name) + return fft_r2c( + x, n, axis, norm, forward=True, onesided=False, name=name + ) else: return fft_c2c(x, n, axis, norm, forward=True, name=name) @@ -266,13 +280,9 @@ def ifft(x, n=None, axis=-1, norm="backward", name=None): """ if is_integer(x) or is_floating_point(x): - return fft_r2c(x, - n, - axis, - norm, - forward=False, - onesided=False, - name=name) + return fft_r2c( + x, n, axis, norm, forward=False, onesided=False, name=name + ) else: return fft_c2c(x, n, axis, norm, forward=False, name=name) @@ -536,13 +546,9 @@ def fftn(x, s=None, axes=None, norm="backward", name=None): # [-8.-8.j 0.+0.j 0.+0.j 0.-0.j]]] """ if is_integer(x) or is_floating_point(x): - return fftn_r2c(x, - s, - axes, - norm, - forward=True, - onesided=False, - name=name) + return fftn_r2c( + x, s, axes, norm, forward=True, onesided=False, name=name + ) else: return fftn_c2c(x, s, axes, norm, forward=True, name=name) @@ -608,13 +614,9 @@ def ifftn(x, s=None, axes=None, norm="backward", name=None): # (-0.1666666716337204+0.28867512941360474j)]]) """ if is_integer(x) or is_floating_point(x): - return fftn_r2c(x, - s, - axes, - norm, - forward=False, - onesided=False, - name=name) + return fftn_r2c( + x, s, axes, norm, forward=False, onesided=False, name=name + ) else: return fftn_c2c(x, s, axes, norm, forward=False, name=name) @@ -914,13 +916,17 @@ def fft2(x, s=None, axes=(-2, -1), norm="backward", name=None): if s is not None: if not isinstance(s, Sequence) or len(s) != 2: raise ValueError( - "Invalid FFT argument s ({}), it should be a sequence of 2 integers." - .format(s)) + "Invalid FFT argument s ({}), it should be a sequence of 2 integers.".format( + s + ) + ) if axes is not None: if not isinstance(axes, Sequence) or len(axes) != 2: raise ValueError( - "Invalid FFT argument axes ({}), it should be a sequence of 2 integers." - .format(axes)) + "Invalid FFT argument axes ({}), it should be a sequence of 2 integers.".format( + axes + ) + ) return fftn(x, s, axes, norm, name) @@ -979,13 +985,17 @@ def ifft2(x, s=None, axes=(-2, -1), norm="backward", name=None): if s is not None: if not isinstance(s, Sequence) or len(s) != 2: raise ValueError( - "Invalid FFT argument s ({}), it should be a sequence of 2 integers." - .format(s)) + "Invalid FFT argument s ({}), it should be a sequence of 2 integers.".format( + s + ) + ) if axes is not None: if not isinstance(axes, Sequence) or len(axes) != 2: raise ValueError( - "Invalid FFT argument axes ({}), it should be a sequence of 2 integers." - .format(axes)) + "Invalid FFT argument axes ({}), it should be a sequence of 2 integers.".format( + axes + ) + ) return ifftn(x, s, axes, norm, name) @@ -1038,13 +1048,17 @@ def rfft2(x, s=None, axes=(-2, -1), norm="backward", name=None): if s is not None: if not isinstance(s, Sequence) or len(s) != 2: raise ValueError( - "Invalid FFT argument s ({}), it should be a sequence of 2 integers." - .format(s)) + "Invalid FFT argument s ({}), it should be a sequence of 2 integers.".format( + s + ) + ) if axes is not None: if not isinstance(axes, Sequence) or len(axes) != 2: raise ValueError( - "Invalid FFT argument axes ({}), it should be a sequence of 2 integers." - .format(axes)) + "Invalid FFT argument axes ({}), it should be a sequence of 2 integers.".format( + axes + ) + ) return rfftn(x, s, axes, norm, name) @@ -1090,13 +1104,17 @@ def irfft2(x, s=None, axes=(-2, -1), norm="backward", name=None): if s is not None: if not isinstance(s, Sequence) or len(s) != 2: raise ValueError( - "Invalid FFT argument s ({}), it should be a sequence of 2 integers." - .format(s)) + "Invalid FFT argument s ({}), it should be a sequence of 2 integers.".format( + s + ) + ) if axes is not None: if not isinstance(axes, Sequence) or len(axes) != 2: raise ValueError( - "Invalid FFT argument axes ({}), it should be a sequence of 2 integers." - .format(axes)) + "Invalid FFT argument axes ({}), it should be a sequence of 2 integers.".format( + axes + ) + ) return irfftn(x, s, axes, norm, name) @@ -1135,13 +1153,17 @@ def hfft2(x, s=None, axes=(-2, -1), norm="backward", name=None): if s is not None: if not isinstance(s, Sequence) or len(s) != 2: raise ValueError( - "Invalid FFT argument s ({}), it should be a sequence of 2 integers." - .format(s)) + "Invalid FFT argument s ({}), it should be a sequence of 2 integers.".format( + s + ) + ) if axes is not None: if not isinstance(axes, Sequence) or len(axes) != 2: raise ValueError( - "Invalid FFT argument axes ({}), it should be a sequence of 2 integers." - .format(axes)) + "Invalid FFT argument axes ({}), it should be a sequence of 2 integers.".format( + axes + ) + ) return hfftn(x, s, axes, norm, name) @@ -1187,13 +1209,17 @@ def ihfft2(x, s=None, axes=(-2, -1), norm="backward", name=None): if s is not None: if not isinstance(s, Sequence) or len(s) != 2: raise ValueError( - "Invalid FFT argument s ({}), it should be a sequence of 2 integers." - .format(s)) + "Invalid FFT argument s ({}), it should be a sequence of 2 integers.".format( + s + ) + ) if axes is not None: if not isinstance(axes, Sequence) or len(axes) != 2: raise ValueError( - "Invalid FFT argument axes ({}), it should be a sequence of 2 integers." - .format(axes)) + "Invalid FFT argument axes ({}), it should be a sequence of 2 integers.".format( + axes + ) + ) return ihfftn(x, s, axes, norm, name) @@ -1417,10 +1443,9 @@ def fft_c2c(x, n, axis, norm, forward, name): dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) outputs = {"Out": [out]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -1442,8 +1467,16 @@ def fft_r2c(x, n, axis, norm, forward, onesided, name): if in_dygraph_mode(): out = _C_ops.fft_r2c(x, axes, norm, forward, onesided) elif _in_legacy_dygraph(): - attrs = ('axes', axes, 'normalization', norm, 'forward', forward, - 'onesided', onesided) + attrs = ( + 'axes', + axes, + 'normalization', + norm, + 'forward', + forward, + 'onesided', + onesided, + ) out = getattr(_legacy_C_ops, op_type)(x, *attrs) else: inputs = { @@ -1458,12 +1491,12 @@ def fft_r2c(x, n, axis, norm, forward, onesided, name): helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference( - _real_to_complex_dtype(dtype)) + _real_to_complex_dtype(dtype) + ) outputs = {"Out": [out]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -1491,8 +1524,16 @@ def fft_c2r(x, n, axis, norm, forward, name): out = _C_ops.fft_c2r(x, axes, norm, forward, 0) elif _in_legacy_dygraph(): if n is not None: - attrs = ('axes', axes, 'normalization', norm, 'forward', forward, - 'last_dim_size', n) + attrs = ( + 'axes', + axes, + 'normalization', + norm, + 'forward', + forward, + 'last_dim_size', + n, + ) else: attrs = ('axes', axes, 'normalization', norm, 'forward', forward) out = getattr(_legacy_C_ops, op_type)(x, *attrs) @@ -1506,12 +1547,12 @@ def fft_c2r(x, n, axis, norm, forward, name): helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference( - _complex_to_real_dtype(dtype)) + _complex_to_real_dtype(dtype) + ) outputs = {"Out": [out]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -1539,8 +1580,10 @@ def fftn_c2c(x, s, axes, norm, forward, name): if s is not None: if len(s) != len(axes): raise ValueError( - "Length of s ({}) and length of axes ({}) does not match.". - format(len(s), len(axes))) + "Length of s ({}) and length of axes ({}) does not match.".format( + len(s), len(axes) + ) + ) s = [s[i] for i in axes_argsoft] if s is not None: @@ -1562,10 +1605,9 @@ def fftn_c2c(x, s, axes, norm, forward, name): dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) outputs = {"Out": [out]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -1591,8 +1633,10 @@ def fftn_r2c(x, s, axes, norm, forward, onesided, name): if s is not None: if len(s) != len(axes): raise ValueError( - "Length of s ({}) and length of axes ({}) does not match.". - format(len(s), len(axes))) + "Length of s ({}) and length of axes ({}) does not match.".format( + len(s), len(axes) + ) + ) s = [s[i] for i in axes_argsoft] + [s[-1]] if s is not None: @@ -1604,8 +1648,16 @@ def fftn_r2c(x, s, axes, norm, forward, onesided, name): if in_dygraph_mode(): out = _C_ops.fft_r2c(x, axes, norm, forward, onesided) elif _in_legacy_dygraph(): - attrs = ('axes', axes, 'normalization', norm, 'forward', forward, - 'onesided', onesided) + attrs = ( + 'axes', + axes, + 'normalization', + norm, + 'forward', + forward, + 'onesided', + onesided, + ) out = getattr(_legacy_C_ops, op_type)(x, *attrs) else: inputs = { @@ -1620,12 +1672,12 @@ def fftn_r2c(x, s, axes, norm, forward, onesided, name): helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference( - _real_to_complex_dtype(dtype)) + _real_to_complex_dtype(dtype) + ) outputs = {"Out": [out]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -1654,8 +1706,10 @@ def fftn_c2r(x, s, axes, norm, forward, name): if s is not None: if len(s) != len(axes): raise ValueError( - "Length of s ({}) and length of axes ({}) does not match.". - format(len(s), len(axes))) + "Length of s ({}) and length of axes ({}) does not match.".format( + len(s), len(axes) + ) + ) s = [s[i] for i in axes_argsoft] + [s[-1]] if s is not None: @@ -1673,8 +1727,16 @@ def fftn_c2r(x, s, axes, norm, forward, name): out = _C_ops.fft_c2r(x, axes, norm, forward, 0) elif _in_legacy_dygraph(): if s: - attrs = ('axes', axes, 'normalization', norm, 'forward', forward, - 'last_dim_size', s[-1]) + attrs = ( + 'axes', + axes, + 'normalization', + norm, + 'forward', + forward, + 'last_dim_size', + s[-1], + ) else: attrs = ('axes', axes, 'normalization', norm, 'forward', forward) out = getattr(_legacy_C_ops, op_type)(x, *attrs) @@ -1688,10 +1750,10 @@ def fftn_c2r(x, s, axes, norm, forward, name): helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference( - _complex_to_real_dtype(dtype)) + _complex_to_real_dtype(dtype) + ) outputs = {"Out": [out]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) return out diff --git a/python/paddle/fluid/__init__.py b/python/paddle/fluid/__init__.py index 00f222b3e8b20..53d50a8b4a3ed 100644 --- a/python/paddle/fluid/__init__.py +++ b/python/paddle/fluid/__init__.py @@ -22,8 +22,9 @@ if os.name == 'nt': core_suffix = 'pyd' -legacy_core = os.path.abspath( - os.path.dirname(__file__)) + os.sep + 'core.' + core_suffix +legacy_core = ( + os.path.abspath(os.path.dirname(__file__)) + os.sep + 'core.' + core_suffix +) if os.path.exists(legacy_core): sys.stderr.write('Deleting legacy file ' + legacy_core + '\n') try: @@ -34,6 +35,7 @@ # import all class inside framework into fluid module from . import framework from .framework import * + # import all class inside executor into fluid module from . import executor from .executor import * @@ -70,10 +72,23 @@ from .data_feeder import DataFeeder from .core import LoDTensor, LoDTensorArray, Scope, _Scope -from .core import CPUPlace, XPUPlace, CUDAPlace, CUDAPinnedPlace, NPUPlace, IPUPlace, MLUPlace, CustomPlace +from .core import ( + CPUPlace, + XPUPlace, + CUDAPlace, + CUDAPinnedPlace, + NPUPlace, + IPUPlace, + MLUPlace, + CustomPlace, +) from .incubate import fleet -from .transpiler import DistributeTranspiler, \ - memory_optimize, release_memory, DistributeTranspilerConfig +from .transpiler import ( + DistributeTranspiler, + memory_optimize, + release_memory, + DistributeTranspilerConfig, +) from .lod_tensor import create_lod_tensor, create_random_int_lodtensor from . import clip from . import profiler @@ -93,7 +108,14 @@ from . import generator from .core import _cuda_synchronize from .generator import Generator -from .trainer_desc import TrainerDesc, DistMultiTrainer, PipelineTrainer, HeterPipelineTrainer, MultiTrainer, HeterXpuTrainer +from .trainer_desc import ( + TrainerDesc, + DistMultiTrainer, + PipelineTrainer, + HeterPipelineTrainer, + MultiTrainer, + HeterXpuTrainer, +) from .transpiler import HashName, RoundRobin from .backward import append_backward @@ -101,10 +123,18 @@ enable_imperative = enable_dygraph disable_imperative = disable_dygraph -__all__ = framework.__all__ + executor.__all__ + \ - trainer_desc.__all__ + transpiler.__all__ + \ - parallel_executor.__all__ + lod_tensor.__all__ + \ - data_feed_desc.__all__ + compiler.__all__ + backward.__all__ + generator.__all__ + [ +__all__ = ( + framework.__all__ + + executor.__all__ + + trainer_desc.__all__ + + transpiler.__all__ + + parallel_executor.__all__ + + lod_tensor.__all__ + + data_feed_desc.__all__ + + compiler.__all__ + + backward.__all__ + + generator.__all__ + + [ 'io', 'initializer', 'embedding', @@ -142,8 +172,9 @@ 'install_check', 'save', 'load', - '_cuda_synchronize' + '_cuda_synchronize', ] +) def __bootstrap__(): @@ -169,18 +200,21 @@ def __bootstrap__(): num_threads = 1 if num_threads > 1: - print('WARNING: OMP_NUM_THREADS set to {0}, not 1. The computation ' - 'speed will not be optimized if you use data parallel. It will ' - 'fail if this PaddlePaddle binary is compiled with OpenBlas since' - ' OpenBlas does not support multi-threads.'.format(num_threads), - file=sys.stderr) + print( + 'WARNING: OMP_NUM_THREADS set to {0}, not 1. The computation ' + 'speed will not be optimized if you use data parallel. It will ' + 'fail if this PaddlePaddle binary is compiled with OpenBlas since' + ' OpenBlas does not support multi-threads.'.format(num_threads), + file=sys.stderr, + ) print('PLEASE USE OMP_NUM_THREADS WISELY.', file=sys.stderr) os.environ['OMP_NUM_THREADS'] = str(num_threads) flag_prefix = "FLAGS_" read_env_flags = [ - key[len(flag_prefix):] for key in core.globals().keys() + key[len(flag_prefix) :] + for key in core.globals().keys() if key.startswith(flag_prefix) ] diff --git a/python/paddle/fluid/average.py b/python/paddle/fluid/average.py index d31b4d496483c..2d83246df7f63 100644 --- a/python/paddle/fluid/average.py +++ b/python/paddle/fluid/average.py @@ -14,6 +14,7 @@ import numpy as np import warnings + """ Class of all kinds of Average. @@ -27,8 +28,11 @@ def _is_number_(var): - return isinstance(var, int) or isinstance( - var, float) or (isinstance(var, np.ndarray) and var.shape == (1, )) + return ( + isinstance(var, int) + or isinstance(var, float) + or (isinstance(var, np.ndarray) and var.shape == (1,)) + ) def _is_number_or_matrix_(var): @@ -59,8 +63,10 @@ class WeightedAverage(object): def __init__(self): warnings.warn( - "The %s is deprecated, please use fluid.metrics.Accuracy instead." % - (self.__class__.__name__), Warning) + "The %s is deprecated, please use fluid.metrics.Accuracy instead." + % (self.__class__.__name__), + Warning, + ) self.reset() def reset(self): @@ -70,7 +76,8 @@ def reset(self): def add(self, value, weight): if not _is_number_or_matrix_(value): raise ValueError( - "The 'value' must be a number(int, float) or a numpy ndarray.") + "The 'value' must be a number(int, float) or a numpy ndarray." + ) if not _is_number_(weight): raise ValueError("The 'weight' must be a number(int, float).") @@ -84,5 +91,6 @@ def add(self, value, weight): def eval(self): if self.numerator is None or self.denominator is None: raise ValueError( - "There is no data to be averaged in WeightedAverage.") + "There is no data to be averaged in WeightedAverage." + ) return self.numerator / self.denominator diff --git a/python/paddle/fluid/backward.py b/python/paddle/fluid/backward.py index 8da8faeefb119..16708268bd5a0 100755 --- a/python/paddle/fluid/backward.py +++ b/python/paddle/fluid/backward.py @@ -26,6 +26,7 @@ import paddle.fluid from .data_feeder import check_type import warnings + try: from collections.abc import Sequence except: @@ -36,13 +37,12 @@ 'gradients', ] -_logger = log_helper.get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = log_helper.get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class ProgramStats(object): - def __init__(self, block, ops): self.block = block self.ops = ops @@ -52,8 +52,10 @@ def __init__(self, block, ops): def get_input_nodes(self): input_names = [] for name in self.var_op_deps: - if len(self.var_op_deps[name]["var_as_output_ops"]) == 0 and \ - len(self.var_op_deps[name]["var_as_input_ops"]) > 0: + if ( + len(self.var_op_deps[name]["var_as_output_ops"]) == 0 + and len(self.var_op_deps[name]["var_as_input_ops"]) > 0 + ): if self.block.var(name).persistable: continue input_names.append(name) @@ -115,16 +117,21 @@ def _update_segment_start(self, min_idx, pre_segment_end_idx): """ def is_amp_cast(op): - return op.desc.type() == 'cast' and self.block.var( - op.desc.input_arg_names()[0]).persistable + return ( + op.desc.type() == 'cast' + and self.block.var(op.desc.input_arg_names()[0]).persistable + ) idx_ = min_idx - 1 updated_min_idx = min_idx while idx_ > pre_segment_end_idx: if is_amp_cast(self.ops[idx_]): - _logger.info("found amp-cast op: {}, : {}".format( - self.ops[idx_].desc.type(), - self.ops[idx_].desc.input_arg_names()[0])) + _logger.info( + "found amp-cast op: {}, : {}".format( + self.ops[idx_].desc.type(), + self.ops[idx_].desc.input_arg_names()[0], + ) + ) updated_min_idx = idx_ idx_ -= 1 else: @@ -138,7 +145,8 @@ def build_stats(self): for j, name in enumerate(op.desc.input_arg_names()): if name in self.var_op_deps: self.op_deps[i]["in_ops"].extend( - self.var_op_deps[name]["var_as_output_ops"]) + self.var_op_deps[name]["var_as_output_ops"] + ) for j, name in enumerate(op.desc.input_arg_names()): if name in self.var_op_deps: self.var_op_deps[name]["var_as_input_ops"].extend([i]) @@ -164,13 +172,15 @@ def sort_checkpoints(self, checkpoints_name): if name not in self.var_op_deps: _logger.info( "Recompute Optimizer: deleted %s from checkpoints, because it is not used in paddle program." - % name) + % name + ) elif self.var_op_deps[name]["var_as_output_ops"] == []: # input nodes sorted_checkpoints.append((name, -1)) else: sorted_checkpoints.append( - (name, max(self.var_op_deps[name]["var_as_output_ops"]))) + (name, max(self.var_op_deps[name]["var_as_output_ops"])) + ) sorted_checkpoints = sorted(sorted_checkpoints, key=lambda x: x[1]) return [x[0] for x in sorted_checkpoints] @@ -191,17 +201,20 @@ def modify_forward_desc_for_recompute(self): continue # add a seed op so that the two dropout op can generate same output op_unique_name = unique_name.generate("seed") - var_unique_name = unique_name.generate_with_ignorable_key(".".join( - [op_unique_name, 'tmp'])) + var_unique_name = unique_name.generate_with_ignorable_key( + ".".join([op_unique_name, 'tmp']) + ) added_var = self.block.create_var( name=var_unique_name, dtype='int32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) seed = 0 if op.attr("fix_seed") is False else int(op.attr("seed")) - op_device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName( + op_device_attr_name = ( + core.op_proto_and_checker_maker.kOpDeviceAttrName() ) op_device = "" if op.desc.has_attr(op_device_attr_name): @@ -209,15 +222,13 @@ def modify_forward_desc_for_recompute(self): # Setting the force_cpu of seed to true will make the output of seed in cpu memory, # reduce the synchronous copy from GPU to CPU in dropout, and reduce the communication hang - added_op = self.block._insert_op(index=op.idx, - type='seed', - inputs={}, - outputs={'Out': [added_var]}, - attrs={ - 'seed': seed, - 'op_device': op_device, - 'force_cpu': True - }) + added_op = self.block._insert_op( + index=op.idx, + type='seed', + inputs={}, + outputs={'Out': [added_var]}, + attrs={'seed': seed, 'op_device': op_device, 'force_cpu': True}, + ) self.ops.insert(op_idx, added_op) # modify dropout op desc so that it accept a seed var as input op.desc.set_input("Seed", [var_unique_name]) @@ -228,22 +239,24 @@ def modify_forward_desc_for_recompute(self): def _pretty_op_desc_(op_desc, prefix): - out_s = "%s\tname:[%s]\n%s \tinputs:[%s]\n%s \toutputs:[%s]" % \ - (prefix + "_op", str(op_desc.type()), prefix + "_input", " ".join(op_desc.input_arg_names()), - prefix + "_output", " ".join(op_desc.output_arg_names())) + out_s = "%s\tname:[%s]\n%s \tinputs:[%s]\n%s \toutputs:[%s]" % ( + prefix + "_op", + str(op_desc.type()), + prefix + "_input", + " ".join(op_desc.input_arg_names()), + prefix + "_output", + " ".join(op_desc.output_arg_names()), + ) return out_s -def _add_needed_descs_to_block(descs, - block, - main_block, - in_memory_vars, - grad_op_id_to_fwd_op=None): +def _add_needed_descs_to_block( + descs, block, main_block, in_memory_vars, grad_op_id_to_fwd_op=None +): if len(descs) == 0: return [] result_descs = [] - op_role_attr_name = \ - core.op_proto_and_checker_maker.kOpRoleAttrName() + op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName() backward = core.op_proto_and_checker_maker.OpRole.Backward for desc in descs: origin_desc = desc @@ -275,8 +288,7 @@ def _add_descs_to_block(descs, block, grad_op_id_to_fwd_op=None): if len(descs) == 0: return [] result_descs = [] - op_role_attr_name = \ - core.op_proto_and_checker_maker.kOpRoleAttrName() + op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName() backward = core.op_proto_and_checker_maker.OpRole.Backward for desc in descs: if isinstance(desc, framework.Operator): @@ -298,8 +310,10 @@ def _add_descs_to_block(descs, block, grad_op_id_to_fwd_op=None): def _find_loss_op_(loss): for op in reversed(loss.block.ops): assert isinstance(op, framework.Operator) - if len(op.output_arg_names - ) == 1 and op.output_arg_names[0] == loss.name: + if ( + len(op.output_arg_names) == 1 + and op.output_arg_names[0] == loss.name + ): loss.op = op break if loss.op is None: @@ -340,21 +354,30 @@ def _create_op_desc_(op_type, inputs, outputs, attrs): op_desc.set_input( para, list( - map(lambda arg: arg.decode() - if isinstance(arg, bytes) else arg, args))) + map( + lambda arg: arg.decode() if isinstance(arg, bytes) else arg, + args, + ) + ), + ) for para, args in outputs.items(): op_desc.set_output( para, list( - map(lambda arg: arg.decode() - if isinstance(arg, bytes) else arg, args))) + map( + lambda arg: arg.decode() if isinstance(arg, bytes) else arg, + args, + ) + ), + ) op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName() op_device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName() if op_role_attr_name not in attrs: attrs[ - op_role_attr_name] = core.op_proto_and_checker_maker.OpRole.Backward + op_role_attr_name + ] = core.op_proto_and_checker_maker.OpRole.Backward if op_device_attr_name not in attrs: attrs[op_device_attr_name] = "" for name, val in attrs.items(): @@ -367,20 +390,23 @@ def _create_op_desc_(op_type, inputs, outputs, attrs): def _create_loss_op_desc_(loss): op_desc = _create_op_desc_( - "fill_constant", {}, {"Out": [_append_grad_suffix_(loss.name)]}, { + "fill_constant", + {}, + {"Out": [_append_grad_suffix_(loss.name)]}, + { "shape": [1], - "value": - 1.0, - "dtype": - loss.dtype, - "force_cpu": - False, - core.op_proto_and_checker_maker.kOpRoleAttrName(): - int(core.op_proto_and_checker_maker.OpRole.Backward) + "value": 1.0, + "dtype": loss.dtype, + "force_cpu": False, + core.op_proto_and_checker_maker.kOpRoleAttrName(): int( + core.op_proto_and_checker_maker.OpRole.Backward + ) | int(core.op_proto_and_checker_maker.OpRole.Loss), - core.op_proto_and_checker_maker.kOpDeviceAttrName(): - loss.op.attr(core.op_proto_and_checker_maker.kOpDeviceAttrName()) - }) + core.op_proto_and_checker_maker.kOpDeviceAttrName(): loss.op.attr( + core.op_proto_and_checker_maker.kOpDeviceAttrName() + ), + }, + ) return op_desc @@ -397,8 +423,10 @@ def _infer_var_data_type_shape_(grad_var_name, block): else: # TODO(jiabin): Maybe we should not to this to cause some unexpected error on dtype warnings.warn( - "Set grad var: {} dtype to default FP32, since we can't find its related forward var" - .format(grad_var_name)) + "Set grad var: {} dtype to default FP32, since we can't find its related forward var".format( + grad_var_name + ) + ) grad_var.set_dtype(core.VarDesc.VarType.FP32) @@ -435,7 +463,7 @@ def _strip_grad_suffix_(name): pos = name.find(core.grad_var_suffix()) new_name = name[:pos] if pos != -1 else name new_pos = name.rfind('grad/') - return new_name[new_pos + 5:] if new_pos != -1 else new_name + return new_name[new_pos + 5 :] if new_pos != -1 else new_name def _append_grad_suffix_(name): @@ -446,30 +474,28 @@ def _append_grad_suffix_(name): return name + core.grad_var_suffix() -def _accumulate_gradients_by_sum_op_(var_name, - renamed_vars, - pending_sum_ops, - op_idx, - op_device=""): +def _accumulate_gradients_by_sum_op_( + var_name, renamed_vars, pending_sum_ops, op_idx, op_device="" +): """ Use sum op to accumulate_gradients, the gradients are stored in renamed_vars. """ if op_idx not in pending_sum_ops.keys(): pending_sum_ops[op_idx] = [] pending_sum_ops[op_idx].append( - _create_op_desc_("sum", {"X": renamed_vars[var_name]}, - {"Out": [var_name]}, { - "use_mkldnn": False, - "op_device": op_device - })) + _create_op_desc_( + "sum", + {"X": renamed_vars[var_name]}, + {"Out": [var_name]}, + {"use_mkldnn": False, "op_device": op_device}, + ) + ) renamed_vars[var_name] = [var_name] -def _accumulate_gradients_by_add_ops_(var_name, - renamed_vars, - pending_sum_ops, - op_idx, - op_device=""): +def _accumulate_gradients_by_add_ops_( + var_name, renamed_vars, pending_sum_ops, op_idx, op_device="" +): """ Use several inplace add op to accumulate_gradients, the gradients are stored in renamed_vars. """ @@ -484,20 +510,19 @@ def _accumulate_gradients_by_add_ops_(var_name, else: out_name = var_name pending_sum_ops[op_idx].append( - _create_op_desc_("grad_add", { - "X": [x_name], - "Y": [y_name] - }, {"Out": [out_name]}, { - "use_mkldnn": False, - "op_device": op_device - })) + _create_op_desc_( + "grad_add", + {"X": [x_name], "Y": [y_name]}, + {"Out": [out_name]}, + {"use_mkldnn": False, "op_device": op_device}, + ) + ) renamed_vars[var_name] = [var_name] -def _addup_repetitive_outputs_(op_descs, - block_idx, - grad_var_to_var=None, - grad_op_id_to_fwd_op=None): +def _addup_repetitive_outputs_( + op_descs, block_idx, grad_var_to_var=None, grad_op_id_to_fwd_op=None +): """ In backward part, an variable may be the output of more than one ops. And one op may yield its multiple outputs to the same variable. @@ -510,14 +535,15 @@ def _addup_repetitive_outputs_(op_descs, """ _MAX_ADD_NUM_ = framework._global_flags()['FLAGS_max_inplace_grad_add'] - #pending_sum_ops = [] + # pending_sum_ops = [] pending_sum_ops = collections.OrderedDict() var_rename_count = collections.defaultdict(int) renamed_vars = collections.defaultdict(list) renamed_var_start_idx = collections.defaultdict(list) var_device = collections.defaultdict(str) for idx, op_desc in enumerate(op_descs): - op_device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName( + op_device_attr_name = ( + core.op_proto_and_checker_maker.kOpDeviceAttrName() ) op_device = "" if op_desc.has_attr(op_device_attr_name): @@ -527,13 +553,21 @@ def _addup_repetitive_outputs_(op_descs, continue if len(renamed_vars[var_name]) > 1: if len(renamed_vars[var_name]) > _MAX_ADD_NUM_: - _accumulate_gradients_by_sum_op_(var_name, renamed_vars, - pending_sum_ops, idx, - var_device[var_name]) + _accumulate_gradients_by_sum_op_( + var_name, + renamed_vars, + pending_sum_ops, + idx, + var_device[var_name], + ) else: - _accumulate_gradients_by_add_ops_(var_name, renamed_vars, - pending_sum_ops, idx, - var_device[var_name]) + _accumulate_gradients_by_add_ops_( + var_name, + renamed_vars, + pending_sum_ops, + idx, + var_device[var_name], + ) for param_idx, param_name in enumerate(op_desc.output_names()): arg_names = op_desc.output(param_name) @@ -542,8 +576,10 @@ def _addup_repetitive_outputs_(op_descs, continue # if "@RENAME@" in var_name: # continue - if var_name == core.empty_var_name( - ) or var_name in op_desc.input_arg_names(): + if ( + var_name == core.empty_var_name() + or var_name in op_desc.input_arg_names() + ): # empty variable or inplace op continue if len(renamed_vars[var_name]) == 0: @@ -552,14 +588,20 @@ def _addup_repetitive_outputs_(op_descs, renamed_var_start_idx[var_name] = idx else: if len(renamed_vars[var_name]) == 1: - new_name = var_name + "@RENAME@block" + str(block_idx) + "@" + \ - str(var_rename_count[var_name]) + new_name = ( + var_name + + "@RENAME@block" + + str(block_idx) + + "@" + + str(var_rename_count[var_name]) + ) var_rename_count[var_name] += 1 # Build the mapping between the new_name and var_name (Only for auto parallel) if grad_var_to_var is not None: if var_name in grad_var_to_var: grad_var_to_var[new_name] = grad_var_to_var[ - var_name] + var_name + ] else: grad_var_to_var[new_name] = var_name # rename original var_name @@ -568,31 +610,45 @@ def _addup_repetitive_outputs_(op_descs, # new_name, 0, idx) # rename arg from idx of the first appearance # in backward, not always from 0 - _rename_arg_(op_descs, var_name, new_name, - renamed_var_start_idx[var_name], idx) + _rename_arg_( + op_descs, + var_name, + new_name, + renamed_var_start_idx[var_name], + idx, + ) _rename_arg_(pending_sum_ops, var_name, new_name) for p in op_desc.output_names()[:param_idx]: p_arg_names = op_desc.output(p) if var_name in p_arg_names: - op_desc.set_output(p, [ - new_name if x == var_name else x - for x in p_arg_names - ]) + op_desc.set_output( + p, + [ + new_name if x == var_name else x + for x in p_arg_names + ], + ) arg_names = [ new_name if x == var_name else x for x in arg_names[:arg_idx] ] + arg_names[arg_idx:] - new_name = var_name + "@RENAME@block" + str(block_idx) + "@" + \ - str(var_rename_count[var_name]) + new_name = ( + var_name + + "@RENAME@block" + + str(block_idx) + + "@" + + str(var_rename_count[var_name]) + ) var_rename_count[var_name] += 1 # Build the mapping between the new_name and var_name (Only for auto parallel) if grad_var_to_var is not None: if var_name in grad_var_to_var: grad_var_to_var[new_name] = grad_var_to_var[ - var_name] + var_name + ] else: grad_var_to_var[new_name] = var_name arg_names[arg_idx] = new_name @@ -604,19 +660,27 @@ def _addup_repetitive_outputs_(op_descs, for var_name, inputs in renamed_vars.items(): if len(renamed_vars[var_name]) > 1: if len(renamed_vars[var_name]) > _MAX_ADD_NUM_: - _accumulate_gradients_by_sum_op_(var_name, renamed_vars, - pending_sum_ops, len(op_descs), - var_device[var_name]) + _accumulate_gradients_by_sum_op_( + var_name, + renamed_vars, + pending_sum_ops, + len(op_descs), + var_device[var_name], + ) else: - _accumulate_gradients_by_add_ops_(var_name, - renamed_vars, pending_sum_ops, - len(op_descs), - var_device[var_name]) + _accumulate_gradients_by_add_ops_( + var_name, + renamed_vars, + pending_sum_ops, + len(op_descs), + var_device[var_name], + ) op_descs_len = len(op_descs) # sum_op descs are sorted according to their insert position for key, value in collections.OrderedDict( - reversed(list(pending_sum_ops.items()))).items(): + reversed(list(pending_sum_ops.items())) + ).items(): # NOTE(zhiqiu): Since reversed, the idx of op_descs to be inserted will remains correct. # For example, [0, 1, 2], and we want to insert 'a' at idx 1, 'b' at idx 2, and the expected result is [0, 1, 'a', 2, 'b']. @@ -626,19 +690,24 @@ def _addup_repetitive_outputs_(op_descs, for i, op in enumerate(value): # update the mapping between fwd and bwd target_idx = idx - 1 if idx == op_descs_len else idx + i - if grad_op_id_to_fwd_op is not None and grad_op_id_to_fwd_op.get( - op_descs[target_idx].original_id(), None) is not None: + if ( + grad_op_id_to_fwd_op is not None + and grad_op_id_to_fwd_op.get( + op_descs[target_idx].original_id(), None + ) + is not None + ): grad_op_id_to_fwd_op[op.original_id()] = grad_op_id_to_fwd_op[ - op_descs[target_idx].original_id()] + op_descs[target_idx].original_id() + ] op_descs.insert(idx + i, op) return op_descs -def _remove_no_grad_branch_(op_descs, - no_grad_set, - grad_op_id_to_fwd_op=None, - target_vars=[]): +def _remove_no_grad_branch_( + op_descs, no_grad_set, grad_op_id_to_fwd_op=None, target_vars=[] +): """ Remove unnecessary grad ops A grad op can be removed in two cases: @@ -651,19 +720,25 @@ def _op_can_be_removed_(op_desc, no_grad_set): out_arg_names = op_desc.output_arg_names() if len(out_arg_names) == 0 or _all_in_set_(out_arg_names, no_grad_set): return True - if _all_in_set_([ - name for name in op_desc.input_arg_names() + if _all_in_set_( + [ + name + for name in op_desc.input_arg_names() if name.find(core.grad_var_suffix()) != -1 - ], no_grad_set): + ], + no_grad_set, + ): no_grad_set.update(set(out_arg_names) - target_grad_var_names) return True return False # Remove ops whose outputs are all in no_grad_dict target_grad_var_names = set( - [var.name + core.grad_var_suffix() for var in target_vars]) + [var.name + core.grad_var_suffix() for var in target_vars] + ) op_descs = [ - op_desc for op_desc in op_descs + op_desc + for op_desc in op_descs if not _op_can_be_removed_(op_desc, no_grad_set) ] # Insert fill_any_like_op with value 0 @@ -675,16 +750,21 @@ def _op_can_be_removed_(op_desc, no_grad_set): x_in = _strip_grad_suffix_(arg) # the reason should be: arg can be input of another grad op # and the op is a not-to-remove op - new_op_desc = _create_op_desc_("fill_any_like", {"X": [x_in]}, - {"Out": [arg]}, { - 'value': 0, - 'dtype': -1 - }) + new_op_desc = _create_op_desc_( + "fill_any_like", + {"X": [x_in]}, + {"Out": [arg]}, + {'value': 0, 'dtype': -1}, + ) # update the mapping between fwd and bwd - if grad_op_id_to_fwd_op is not None and grad_op_id_to_fwd_op.get( - op_desc.original_id(), None) is not None: - grad_op_id_to_fwd_op[new_op_desc.original_id( - )] = grad_op_id_to_fwd_op[op_desc.original_id()] + if ( + grad_op_id_to_fwd_op is not None + and grad_op_id_to_fwd_op.get(op_desc.original_id(), None) + is not None + ): + grad_op_id_to_fwd_op[ + new_op_desc.original_id() + ] = grad_op_id_to_fwd_op[op_desc.original_id()] to_insert.append((new_op_desc, idx)) list([op_descs.insert(p[1], p[0]) for p in reversed(to_insert)]) @@ -711,7 +791,6 @@ def _find_not_need_ops(grad_op_descs, forward_ops, input_grad_names_set): """ class Var(object): - def __init__(self, var_name): self.var_name = var_name self.gen_op = None @@ -727,7 +806,6 @@ def add_pending_op(self, op): self.pendding_ops.append(op) class Op(object): - def __init__(self, op_desc): self.op_desc = op_desc self.inputs = [] @@ -768,8 +846,9 @@ def _create_op_node(op_desc): return op_node # Record the forward vars - forward_vars_set = set() if input_grad_names_set is None else set( - input_grad_names_set) + forward_vars_set = ( + set() if input_grad_names_set is None else set(input_grad_names_set) + ) for op in forward_ops: forward_vars_set.update(op.desc.input_arg_names()) forward_vars_set.update(op.desc.output_arg_names()) @@ -824,14 +903,16 @@ def serialize_op_decs(op_desc): return proto.__str__() -def _append_backward_ops_with_checkpoints_(block, - ops, - target_vars, - target_block, - no_grad_dict, - grad_to_var, - checkpoints, - grad_op_id_to_fwd_op=None): +def _append_backward_ops_with_checkpoints_( + block, + ops, + target_vars, + target_block, + no_grad_dict, + grad_to_var, + checkpoints, + grad_op_id_to_fwd_op=None, +): """ Create grad ops with forward ops, and insert them into given block @@ -898,16 +979,20 @@ def _append_backward_ops_with_checkpoints_(block, # min_idx: checkpoint_1' s input op # max_idx: checkpoint_2' s output op flag, min_idx, max_idx = program_stat.is_subgraph( - [checkpoints_name[start_idx]], - [checkpoints_name[start_idx + 1]]) + [checkpoints_name[start_idx]], [checkpoints_name[start_idx + 1]] + ) if flag: # max_idx + 1 since the exact and used segment end idx is max_idx min_idx = program_stat._update_segment_start( - min_idx, pre_segment_end_idx) + min_idx, pre_segment_end_idx + ) segments.append([min_idx, max_idx + 1]) else: - _logger.info("Could not recompute op range [{}] - [{}] ".format( - min_idx, max_idx + 1)) + _logger.info( + "Could not recompute op range [{}] - [{}] ".format( + min_idx, max_idx + 1 + ) + ) start_idx += 1 @@ -918,26 +1003,42 @@ def _append_backward_ops_with_checkpoints_(block, for i, (idx1, idx2) in enumerate(recompute_segments): _logger.info("recompute segment[{}]".format(i)) - _logger.info("segment start op: [{}]: [{}]".format( - ops[idx1].desc.type(), ops[idx1].desc.input_arg_names())) - _logger.info("segment end op: [{}]: [{}]".format( - ops[idx2 - 1].desc.type(), ops[idx2 - 1].desc.input_arg_names())) + _logger.info( + "segment start op: [{}]: [{}]".format( + ops[idx1].desc.type(), ops[idx1].desc.input_arg_names() + ) + ) + _logger.info( + "segment end op: [{}]: [{}]".format( + ops[idx2 - 1].desc.type(), ops[idx2 - 1].desc.input_arg_names() + ) + ) _logger.info("recompute segment[{}]".format(i)) - _logger.info("segment start op: [{}]: [{}]".format( - ops[idx1].desc.type(), ops[idx1].desc.input_arg_names())) - _logger.info("segment end op: [{}]: [{}]".format( - ops[idx2 - 1].desc.type(), ops[idx2 - 1].desc.input_arg_names())) + _logger.info( + "segment start op: [{}]: [{}]".format( + ops[idx1].desc.type(), ops[idx1].desc.input_arg_names() + ) + ) + _logger.info( + "segment end op: [{}]: [{}]".format( + ops[idx2 - 1].desc.type(), ops[idx2 - 1].desc.input_arg_names() + ) + ) # 2) go through all forward ops and induct all variables that will be hold in memory vars_should_be_hold = [] # a. variables that are used across segments will be held in memory for segment in recompute_segments: vars_should_be_hold.extend( - program_stat.get_out_of_subgraph_vars(segment[0], segment[1])) + program_stat.get_out_of_subgraph_vars(segment[0], segment[1]) + ) cross_vars = set(vars_should_be_hold) - set(checkpoints_name) - _logger.info("found [{}] vars which cross recompute segment: [{}], better checkpoints might be set to reduce those vars".format( \ - len(cross_vars), cross_vars)) + _logger.info( + "found [{}] vars which cross recompute segment: [{}], better checkpoints might be set to reduce those vars".format( + len(cross_vars), cross_vars + ) + ) # b. output of seed op should be kept in memory vars_should_be_hold.extend(program_stat.get_reserved_vars()) @@ -957,11 +1058,14 @@ def _append_backward_ops_with_checkpoints_(block, gap_ops = ops[0:max_calculated_op_position] for op in reversed(gap_ops): if op.has_attr("sub_block"): - raise Exception("Recompute don't support ops with sub_block" - "invoke op: %s" % - _pretty_op_desc_(op.desc, "with_sub_block")) + raise Exception( + "Recompute don't support ops with sub_block" + "invoke op: %s" + % _pretty_op_desc_(op.desc, "with_sub_block") + ) grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - op.desc, no_grad_dict[block.idx], []) + op.desc, no_grad_dict[block.idx], [] + ) # record the mapping between fwd and bwd if grad_op_id_to_fwd_op is not None: @@ -973,21 +1077,25 @@ def _append_backward_ops_with_checkpoints_(block, op_device = op.desc.attr(device_attr_name) for op_desc in grad_op_desc: op_desc._set_attr(device_attr_name, op_device) - added_descs = _add_descs_to_block(grad_op_desc, local_block, - grad_op_id_to_fwd_op) + added_descs = _add_descs_to_block( + grad_op_desc, local_block, grad_op_id_to_fwd_op + ) grad_op_descs.extend(added_descs) grad_to_var.update(op_grad_to_var) for i, segment in enumerate(recompute_segments[::-1]): - gap_ops = ops[segment[1]:max_calculated_op_position] + gap_ops = ops[segment[1] : max_calculated_op_position] max_calculated_op_position = segment[0] for op in reversed(gap_ops): if op.has_attr("sub_block"): - raise Exception("Recompute don't support ops with sub_block" - "invoke op: %s" % - _pretty_op_desc_(op.desc, "with_sub_block")) + raise Exception( + "Recompute don't support ops with sub_block" + "invoke op: %s" + % _pretty_op_desc_(op.desc, "with_sub_block") + ) grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - op.desc, no_grad_dict[block.idx], []) + op.desc, no_grad_dict[block.idx], [] + ) # record the mapping between fwd and bwd if grad_op_id_to_fwd_op is not None: @@ -999,19 +1107,22 @@ def _append_backward_ops_with_checkpoints_(block, op_device = op.desc.attr(device_attr_name) for op_desc in grad_op_desc: op_desc._set_attr(device_attr_name, op_device) - added_descs = _add_descs_to_block(grad_op_desc, local_block, - grad_op_id_to_fwd_op) + added_descs = _add_descs_to_block( + grad_op_desc, local_block, grad_op_id_to_fwd_op + ) grad_op_descs.extend(added_descs) grad_to_var.update(op_grad_to_var) - ff_ops = ops[segment[0]:segment[1]] + ff_ops = ops[segment[0] : segment[1]] var_suffix = ".subprog_%d" % i for op in ff_ops: if op.has_attr("sub_block"): - raise Exception("Recompute don't support ops with sub_block" - "invoke op: %s" % - _pretty_op_desc_(op.desc, "with_sub_block")) + raise Exception( + "Recompute don't support ops with sub_block" + "invoke op: %s" + % _pretty_op_desc_(op.desc, "with_sub_block") + ) input_and_output_names = [] input_and_output_names.extend(op.desc.input_arg_names()) input_and_output_names.extend(op.desc.output_arg_names()) @@ -1025,19 +1136,22 @@ def _append_backward_ops_with_checkpoints_(block, # we should create the rename var in subprog, otherwise its VarType will be BOOL ref_var = block.program.global_block().var(name) - block.create_var(name=var_name_dict[name], - shape=ref_var.shape, - dtype=ref_var.dtype, - type=ref_var.type, - persistable=ref_var.persistable, - stop_gradient=ref_var.stop_gradient) + block.create_var( + name=var_name_dict[name], + shape=ref_var.shape, + dtype=ref_var.dtype, + type=ref_var.type, + persistable=ref_var.persistable, + stop_gradient=ref_var.stop_gradient, + ) # 3.a. add ops in current recompute_segment as forward recomputation ops - buffer_descs = _add_needed_descs_to_block(ff_ops, buffer_block, block, - vars_in_memory, - grad_op_id_to_fwd_op) - added_descs = _add_descs_to_block(ff_ops, local_block, - grad_op_id_to_fwd_op) + buffer_descs = _add_needed_descs_to_block( + ff_ops, buffer_block, block, vars_in_memory, grad_op_id_to_fwd_op + ) + added_descs = _add_descs_to_block( + ff_ops, local_block, grad_op_id_to_fwd_op + ) # 3.b. rename all non-checkpoint variables in recomputation ops for key in var_name_dict: @@ -1049,13 +1163,15 @@ def _append_backward_ops_with_checkpoints_(block, # 3.c. add backward ops for all ops in current segment for op_desc in reversed(added_descs): grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - op_desc, no_grad_dict[block.idx], []) + op_desc, no_grad_dict[block.idx], [] + ) # record the mapping between fwd and bwd if grad_op_id_to_fwd_op is not None: for g_op_desc in grad_op_desc: - grad_op_id_to_fwd_op[g_op_desc.original_id( - )] = grad_op_id_to_fwd_op[op_desc.original_id()] + grad_op_id_to_fwd_op[ + g_op_desc.original_id() + ] = grad_op_id_to_fwd_op[op_desc.original_id()] # Set device for grad_op according to forward Op if op_desc.has_attr(device_attr_name): @@ -1070,21 +1186,33 @@ def _append_backward_ops_with_checkpoints_(block, # 3.d. add sum op for repetitive_outputs grad_op_descs = _addup_repetitive_outputs_( - grad_op_descs, block.idx, grad_op_id_to_fwd_op=grad_op_id_to_fwd_op) + grad_op_descs, block.idx, grad_op_id_to_fwd_op=grad_op_id_to_fwd_op + ) # 4) remove no grad branch as it is in _remove_no_grad_branch_ - grad_op_descs = _remove_no_grad_branch_(grad_op_descs, - no_grad_dict[block.idx], - grad_op_id_to_fwd_op, target_vars) - added_descs = _add_descs_to_block(grad_op_descs, target_block, - grad_op_id_to_fwd_op) - return program_stat, checkpoints_name, vars_should_be_hold, recompute_segments - - -def _get_sub_block_path(sub_block, - sub_block_op_desc, - no_grad_set, - op_path_dict, - sub_block_target_names=None): + grad_op_descs = _remove_no_grad_branch_( + grad_op_descs, + no_grad_dict[block.idx], + grad_op_id_to_fwd_op, + target_vars, + ) + added_descs = _add_descs_to_block( + grad_op_descs, target_block, grad_op_id_to_fwd_op + ) + return ( + program_stat, + checkpoints_name, + vars_should_be_hold, + recompute_segments, + ) + + +def _get_sub_block_path( + sub_block, + sub_block_op_desc, + no_grad_set, + op_path_dict, + sub_block_target_names=None, +): """ Get output vars in subblock which will be assigned to parent block. It is used to find the grad path in subblock. @@ -1102,8 +1230,8 @@ def _get_sub_block_path(sub_block, """ assert sub_block_op_desc.has_attr( - "sub_block") and sub_block.idx == sub_block_op_desc._block_attr_id( - "sub_block") + "sub_block" + ) and sub_block.idx == sub_block_op_desc._block_attr_id("sub_block") assert isinstance(sub_block_target_names, (set, type(None))) if sub_block_target_names is None: @@ -1123,8 +1251,9 @@ def _get_sub_block_path(sub_block, # Step2: find op path of sub-block is_while = sub_block_op_desc.type in ["while"] - sub_block_op_path = _find_op_path_(sub_block, sub_outputs, [], - no_grad_set, op_path_dict, is_while) + sub_block_op_path = _find_op_path_( + sub_block, sub_outputs, [], no_grad_set, op_path_dict, is_while + ) return sub_block_op_path return sub_block.ops @@ -1132,8 +1261,9 @@ def _get_sub_block_path(sub_block, def _is_grad_op_(op): op_maker = core.op_proto_and_checker_maker backward = core.op_proto_and_checker_maker.OpRole.Backward - if op_maker.kOpRoleVarAttrName() in op.attr_names and \ - int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(backward): + if op_maker.kOpRoleVarAttrName() in op.attr_names and int( + op.all_attrs()[op_maker.kOpRoleAttrName()] + ) == int(backward): return True return False @@ -1142,18 +1272,20 @@ def _rename_grad_name_(name, grad_order): return 'grad/' * grad_order + name -def _append_backward_ops_(block, - ops, - target_vars, - target_block, - no_grad_dict, - grad_to_var, - callbacks=None, - input_grad_names_set=None, - op_path_dict=None, - distop_context=None, - rename_var_map=None, - grad_op_id_to_fwd_op=None): +def _append_backward_ops_( + block, + ops, + target_vars, + target_block, + no_grad_dict, + grad_to_var, + callbacks=None, + input_grad_names_set=None, + op_path_dict=None, + distop_context=None, + rename_var_map=None, + grad_op_id_to_fwd_op=None, +): """ Create all grad ops, and insert them into given block @@ -1180,18 +1312,22 @@ def _append_backward_ops_(block, """ # Build the mapping between the forward op and backward op (Only for auto parallel) - def update_distop_context(distop_context, op_grad_to_var, - appending_grad_times): + def update_distop_context( + distop_context, op_grad_to_var, appending_grad_times + ): distop_context.grad_var_to_var[appending_grad_times].update( - op_grad_to_var) + op_grad_to_var + ) for op_desc in grad_op_desc: - assert op_desc.original_id( - ) not in distop_context.grad_op_id_to_op_id + assert ( + op_desc.original_id() not in distop_context.grad_op_id_to_op_id + ) distop_context.grad_op_id_to_op_id[ - op_desc.original_id()] = op.desc.original_id() + op_desc.original_id() + ] = op.desc.original_id() if callbacks is not None: - assert (isinstance(callbacks, (list, tuple))) + assert isinstance(callbacks, (list, tuple)) for cb in callbacks: if not hasattr(cb, '__call__'): raise ValueError("'callback' must be a callable object.") @@ -1216,16 +1352,18 @@ def update_distop_context(distop_context, op_grad_to_var, pre_input_grad_names_set = copy.copy(input_grad_names_set) input_grad_names_set = None sub_block_path = op_path_dict[op._block_attr_id("sub_block")] - _append_backward_ops_(sub_block, - sub_block_path, - target_vars, - grad_sub_block, - no_grad_dict, - grad_to_var, - callbacks, - input_grad_names_set, - op_path_dict, - grad_op_id_to_fwd_op=grad_op_id_to_fwd_op) + _append_backward_ops_( + sub_block, + sub_block_path, + target_vars, + grad_sub_block, + no_grad_dict, + grad_to_var, + callbacks, + input_grad_names_set, + op_path_dict, + grad_op_id_to_fwd_op=grad_op_id_to_fwd_op, + ) input_grad_names_set = pre_input_grad_names_set program._rollback() @@ -1233,7 +1371,8 @@ def update_distop_context(distop_context, op_grad_to_var, # Getting op's corresponding grad_op grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - op.desc, no_grad_dict[block.idx], grad_sub_block_list) + op.desc, no_grad_dict[block.idx], grad_sub_block_list + ) # record the mapping between fwd and bwd if grad_op_id_to_fwd_op is not None: @@ -1242,15 +1381,22 @@ def update_distop_context(distop_context, op_grad_to_var, # Build the mapping between the forward op and backward op (Only for auto parallel) if distop_context is not None: - update_distop_context(distop_context, op_grad_to_var, - program._appending_grad_times) + update_distop_context( + distop_context, op_grad_to_var, program._appending_grad_times + ) else: - default_ctx = getattr(paddle.distributed.auto_parallel.dist_context, - '_g_default_distributed_context', None) + default_ctx = getattr( + paddle.distributed.auto_parallel.dist_context, + '_g_default_distributed_context', + None, + ) if default_ctx is not None: distop_context = default_ctx.dist_op_context - update_distop_context(distop_context, op_grad_to_var, - program._appending_grad_times) + update_distop_context( + distop_context, + op_grad_to_var, + program._appending_grad_times, + ) # Set device for grad_op according to forward Op device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName() @@ -1278,7 +1424,8 @@ def update_distop_context(distop_context, op_grad_to_var, continue if block.desc.find_var(name.encode("ascii")): new_name = _rename_grad_name_( - name, program._appending_grad_times) + name, program._appending_grad_times + ) op_desc._rename_output(name, new_name) rename_var_map[name] = new_name @@ -1286,8 +1433,8 @@ def update_distop_context(distop_context, op_grad_to_var, # Build the mapping between the grad var name and var name (Only for auto parallel) if distop_context is not None: distop_context.grad_var_to_var[ - program._appending_grad_times][ - new_name] = op_grad_to_var[name] + program._appending_grad_times + ][new_name] = op_grad_to_var[name] op_grad_to_var[new_name] = op_grad_to_var[name] op_grad_to_var.pop(name) @@ -1296,12 +1443,15 @@ def update_distop_context(distop_context, op_grad_to_var, # But this strategy is not suited for while op for some control flow, # for example, for while op, the grads maybe generated in next loop. if input_grad_names_set is not None: - is_grad_name = lambda name: name.find(core.grad_var_suffix( - )) != -1 or name in input_grad_names_set + is_grad_name = ( + lambda name: name.find(core.grad_var_suffix()) != -1 + or name in input_grad_names_set + ) is_append_grad = False for op_desc in grad_op_desc: input_grad_names = [ - name for name in op_desc.input_arg_names() + name + for name in op_desc.input_arg_names() if is_grad_name(name) ] # some code of gradient ops, like increment, are not very @@ -1325,19 +1475,24 @@ def update_distop_context(distop_context, op_grad_to_var, grad_var_to_var = None if distop_context is not None: grad_var_to_var = distop_context.grad_var_to_var[ - program._appending_grad_times] + program._appending_grad_times + ] # sum parameter's gradients' var given multiple var gradient grad_op_descs = _addup_repetitive_outputs_( grad_op_descs, block.idx, grad_var_to_var, - grad_op_id_to_fwd_op=grad_op_id_to_fwd_op) + grad_op_id_to_fwd_op=grad_op_id_to_fwd_op, + ) # if all outputs of the grad op are in no_grad_set, then just remove and fill zero # if all inputs of the grad op are in no_grad_set, just remove this op - grad_op_descs = _remove_no_grad_branch_(grad_op_descs, - no_grad_dict[block.idx], - grad_op_id_to_fwd_op, target_vars) + grad_op_descs = _remove_no_grad_branch_( + grad_op_descs, + no_grad_dict[block.idx], + grad_op_id_to_fwd_op, + target_vars, + ) # remove some backward ops not_need_ops = _find_not_need_ops(grad_op_descs, ops, input_grad_names_set) @@ -1355,7 +1510,7 @@ def update_distop_context(distop_context, op_grad_to_var, new_op_desc._set_attr(op_role_attr_name, backward) grad_to_var["__current_op_desc__"] = new_op_desc if callbacks is not None: - assert (isinstance(callbacks, (list, tuple))) + assert isinstance(callbacks, (list, tuple)) for cb in callbacks: cb(block=target_block, context=grad_to_var) @@ -1376,8 +1531,10 @@ def _find_parent_op_(sub_block): block_desc = program.block(block_id).desc for op_idx in range(block_desc.op_size()): op = block_desc.op(op_idx) - if op.has_attr("sub_block") and op._block_attr_id( - "sub_block") == sub_block_id: + if ( + op.has_attr("sub_block") + and op._block_attr_id("sub_block") == sub_block_id + ): return op # NOTE(paddle-dev): When optimizer is added in conditional block, @@ -1431,11 +1588,13 @@ def _append_backward_vars_(block, start_op_idx, grad_to_var, grad_info_map): ] inputs = [ - var for var in op_desc.input_arg_names() + var + for var in op_desc.input_arg_names() if var != core.empty_var_name() ] outputs = [ - var for var in op_desc.output_arg_names() + var + for var in op_desc.output_arg_names() if var != core.empty_var_name() ] @@ -1450,7 +1609,8 @@ def _append_backward_vars_(block, start_op_idx, grad_to_var, grad_info_map): ''' if grad_var_ins: existing_grad_var_ins = [ - var for var in grad_var_ins + var + for var in grad_var_ins if block.desc.has_var_recursive(var.encode()) or var in parent_op_vars ] @@ -1470,8 +1630,10 @@ def _append_backward_vars_(block, start_op_idx, grad_to_var, grad_info_map): new_vars = set() # create new gradient variables for grad_var_name in op_desc.output_arg_names(): - if block.desc.has_var_recursive(grad_var_name.encode( - )) or grad_var_name == core.empty_var_name(): + if ( + block.desc.has_var_recursive(grad_var_name.encode()) + or grad_var_name == core.empty_var_name() + ): continue block.desc.var(grad_var_name.encode()) new_vars.add(grad_var_name) @@ -1550,21 +1712,26 @@ def _get_no_grad_set_name(no_grad_set): else: raise TypeError( "The type of no_grad_set's member must be paddle.fluid.Variable or str, but received %s." - % (type(no_grad_var))) + % (type(no_grad_var)) + ) else: raise TypeError( - "The type of no_grad_set should be set or list or tuple, but received {}" - .format(type(no_grad_set))) + "The type of no_grad_set should be set or list or tuple, but received {}".format( + type(no_grad_set) + ) + ) return no_grad_set_name @framework.static_only -def append_backward(loss, - parameter_list=None, - no_grad_set=None, - callbacks=None, - checkpoints=None, - distop_context=None): +def append_backward( + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + checkpoints=None, + distop_context=None, +): """ :api_attr: Static Graph @@ -1665,11 +1832,13 @@ def append_backward(loss, p_g_list6 = paddle.static.append_backward(loss=avg_loss, parameter_list=all_weights, no_grad_set=set(all_weights)) """ - grad_op_id_to_fwd_op = { - } # for cuda graph usage, recording the mapping between grad op original id to fwd op + grad_op_id_to_fwd_op = ( + {} + ) # for cuda graph usage, recording the mapping between grad op original id to fwd op - check_type(loss, 'loss', framework.Variable, - 'paddle.static.append_backward') + check_type( + loss, 'loss', framework.Variable, 'paddle.static.append_backward' + ) if loss.op is None: # the loss is from a cloned program. Find loss op manually. @@ -1678,11 +1847,16 @@ def append_backward(loss, loss.op._set_attr( core.op_proto_and_checker_maker.kOpRoleAttrName(), int(core.op_proto_and_checker_maker.OpRole.Forward) - | int(core.op_proto_and_checker_maker.OpRole.Loss)) + | int(core.op_proto_and_checker_maker.OpRole.Loss), + ) if callbacks is not None: - check_type(callbacks, 'callbacks', (list, tuple), - 'paddle.static.append_backward') + check_type( + callbacks, + 'callbacks', + (list, tuple), + 'paddle.static.append_backward', + ) program = loss.block.program root_block = program.block(0) @@ -1712,14 +1886,16 @@ def append_backward(loss, if is_in_control_flow: # create grad block if in switch control flow. target_grad_block = program._create_block( - parent_idx=current_block.parent_idx) + parent_idx=current_block.parent_idx + ) target_grad_block._set_forward_block_idx(current_block_idx) # after _create_block, program.current_block changes else: target_grad_block = root_block son_parent_block_idx_dict = _get_son_parent_block_idx_dict( - program, current_block_idx) + program, current_block_idx + ) block_fwd_op_num_dict = {} # block_id: fwd_op_num for idx in son_parent_block_idx_dict: @@ -1736,18 +1912,22 @@ def append_backward(loss, block = program.block(block_idx) block_no_grad_set = set( - map(_strip_grad_suffix_, no_grad_dict[block_idx])) + map(_strip_grad_suffix_, no_grad_dict[block_idx]) + ) op_path_dict = dict() - op_path = _find_op_path_(block, [loss], [], block_no_grad_set, - op_path_dict) + op_path = _find_op_path_( + block, [loss], [], block_no_grad_set, op_path_dict + ) - no_grad_vars = _find_no_grad_vars(block, op_path, [loss], - block_no_grad_set) + no_grad_vars = _find_no_grad_vars( + block, op_path, [loss], block_no_grad_set + ) block_no_grad_set.update(no_grad_vars) no_grad_dict[block_idx].update( - list(map(_append_grad_suffix_, block_no_grad_set))) + list(map(_append_grad_suffix_, block_no_grad_set)) + ) input_grad_names_set = None # For double backward, input_grad_names is used for filtering @@ -1762,22 +1942,27 @@ def append_backward(loss, # TODO: support _append_backward_ops_with_checkpoints_ in # sub-block (control flow) is_recompute = False - if checkpoints != None and \ - isinstance(checkpoints, list) and \ - len(checkpoints) > 0: + if ( + checkpoints != None + and isinstance(checkpoints, list) + and len(checkpoints) > 0 + ): is_recompute = True - program_stat, checkpoint_names, \ - vars_should_be_hold, \ - recompute_segments = \ - _append_backward_ops_with_checkpoints_( - root_block, - op_path, - [loss], - root_block, - no_grad_dict, - grad_to_var, - checkpoints, - grad_op_id_to_fwd_op) + ( + program_stat, + checkpoint_names, + vars_should_be_hold, + recompute_segments, + ) = _append_backward_ops_with_checkpoints_( + root_block, + op_path, + [loss], + root_block, + no_grad_dict, + grad_to_var, + checkpoints, + grad_op_id_to_fwd_op, + ) else: _append_backward_ops_( block, # the block where forward ops are in @@ -1790,22 +1975,27 @@ def append_backward(loss, input_grad_names_set=input_grad_names_set, op_path_dict=op_path_dict, distop_context=distop_context, - grad_op_id_to_fwd_op=grad_op_id_to_fwd_op) + grad_op_id_to_fwd_op=grad_op_id_to_fwd_op, + ) grad_info_map = dict() # if in control flow, target_grad_block is a created new block which only contains grad ops, # so fwd_op_num is set to 0. - fwd_op_num = block_fwd_op_num_dict[ - current_block_idx] if not is_in_control_flow else 0 + fwd_op_num = ( + block_fwd_op_num_dict[current_block_idx] + if not is_in_control_flow + else 0 + ) # Because append_backward may be called multiple times, # we need rename the internal gradient variables so that they have # different names. _rename_grad_(target_grad_block, fwd_op_num, grad_to_var, {}) - _append_backward_vars_(target_grad_block, fwd_op_num, grad_to_var, - grad_info_map) + _append_backward_vars_( + target_grad_block, fwd_op_num, grad_to_var, grad_info_map + ) program.current_block_idx = current_block_idx program._sync_with_cpp() @@ -1817,13 +2007,20 @@ def append_backward(loss, op._cuda_graph_attr = fwd_op._cuda_graph_attr if parameter_list is not None: - check_type(parameter_list, 'parameter_list', (list, tuple, set), - 'fluid.backward.append_backward') + check_type( + parameter_list, + 'parameter_list', + (list, tuple, set), + 'fluid.backward.append_backward', + ) parameters = [] for i, param in enumerate(parameter_list): - check_type(param, 'parameter_list[%s]' % i, - (framework.Variable, str), - 'fluid.backward.append_backward') + check_type( + param, + 'parameter_list[%s]' % i, + (framework.Variable, str), + 'fluid.backward.append_backward', + ) if isinstance(param, framework.Variable): parameters.append(param.name) elif isinstance(param, str): @@ -1840,8 +2037,11 @@ def append_backward(loss, grad_info = grad_info_map[param] grad_block = grad_info[1] if not grad_block.has_var(grad_info[0]): - raise ValueError("grad block[{0}] did not have grad var {1}".format( - grad_info[1], grad_info[0])) + raise ValueError( + "grad block[{0}] did not have grad var {1}".format( + grad_info[1], grad_info[0] + ) + ) # Get the param var from the global block param_var = program.global_block().var(param) grad_var = grad_block.var(grad_info[0]) @@ -1856,8 +2056,9 @@ def append_backward(loss, for p, g in params_and_grads: if g is None: continue - ops = grad_block.ops if is_in_control_flow else program.global_block( - ).ops + ops = ( + grad_block.ops if is_in_control_flow else program.global_block().ops + ) for op in reversed(ops): assert isinstance(op, framework.Operator) if g.name in op.output_arg_names: @@ -1924,8 +2125,9 @@ def _get_output_names(cur_block, targets): if _some_in_set_(op.desc.output_arg_names(), current_output_names): for name in op.desc.input_arg_names(): current_output_names.add(name) - if not block.desc.find_var(name.encode()) \ - and parent_block.desc.find_var(name.encode()): + if not block.desc.find_var( + name.encode() + ) and parent_block.desc.find_var(name.encode()): parent_block_output_names.add(name) block = parent_block @@ -1945,8 +2147,11 @@ def _find_no_grad_vars(block, op_path, targets, no_grad_set): # If the op has sub_block, it is too complicated to find the correct no_grad_var. if not op.has_attr("sub_block"): for out_var in op.desc.output_arg_names(): - if out_var not in output_names and out_var not in op.desc.input_arg_names( - ) and not block.vars[out_var].stop_gradient: + if ( + out_var not in output_names + and out_var not in op.desc.input_arg_names() + and not block.vars[out_var].stop_gradient + ): no_grad_var.append(out_var) for name in op.desc.input_arg_names(): if name not in no_grad_set: @@ -1954,12 +2159,9 @@ def _find_no_grad_vars(block, op_path, targets, no_grad_set): return set(no_grad_var) -def _find_op_path_(block, - targets, - inputs, - no_grad_set, - op_path_dict=None, - is_while=False): +def _find_op_path_( + block, targets, inputs, no_grad_set, op_path_dict=None, is_while=False +): """ It is used to find the grad path in `block`. @@ -1986,9 +2188,9 @@ def _find_op_path_(block, # All the inputs of the block are used if inputs is empty, if inputs: for i, op in enumerate(block.ops): - if _some_in_set_(op.desc.input_arg_names(), - input_names) and core.has_non_empty_grad_op_maker( - op.type): + if _some_in_set_( + op.desc.input_arg_names(), input_names + ) and core.has_non_empty_grad_op_maker(op.type): for name in op.desc.output_arg_names(): if name not in no_grad_set: input_names.add(name) @@ -2000,14 +2202,14 @@ def _find_op_path_(block, sub_block_id = op._block_attr_id("sub_block") sub_block = block.program.block(sub_block_id) sub_block_target_names = output_names & set(op.output_arg_names) - sub_block_path = _get_sub_block_path(sub_block, op, set(), - op_path_dict, - sub_block_target_names) + sub_block_path = _get_sub_block_path( + sub_block, op, set(), op_path_dict, sub_block_target_names + ) op_path_dict[sub_block_id] = sub_block_path - if _some_in_set_(op.desc.output_arg_names(), - output_names) and core.has_non_empty_grad_op_maker( - op.type): + if _some_in_set_( + op.desc.output_arg_names(), output_names + ) and core.has_non_empty_grad_op_maker(op.type): for name in op.desc.input_arg_names(): if name not in no_grad_set: output_names.add(name) @@ -2018,8 +2220,9 @@ def _find_op_path_(block, # If block is while block, dealing with op specifically again. # TODO(liym27): Consider special types of ops. for i, op in reversed(list(enumerate(block.ops))): - if relevant_op_flags[i] == False \ - and _some_in_set_(op.desc.output_arg_names(), output_names): + if relevant_op_flags[i] == False and _some_in_set_( + op.desc.output_arg_names(), output_names + ): relevant_op_flags[i] = True op_path = [ @@ -2072,7 +2275,8 @@ def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None): if len(targets) != len(target_gradients): raise ValueError( - "Should have the same number of target_gradients as targets") + "Should have the same number of target_gradients as targets" + ) if no_grad_set is None: no_grad_set = set() @@ -2093,16 +2297,24 @@ def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None): if grad is None: target_shape = target.name + '_shape' block.desc.append_op().copy_from( - _create_op_desc_("shape", {'Input': [target.name]}, - {"Out": [target_shape]}, {})) + _create_op_desc_( + "shape", + {'Input': [target.name]}, + {"Out": [target_shape]}, + {}, + ) + ) input_grad_names_set.add(target_shape) - op_desc = _create_op_desc_("fill_constant", - {"ShapeTensor": [target_shape]}, - {"Out": [grad_name]}, { - "shape": target.shape, - "value": 1.0, - "dtype": target.dtype, - }) + op_desc = _create_op_desc_( + "fill_constant", + {"ShapeTensor": [target_shape]}, + {"Out": [grad_name]}, + { + "shape": target.shape, + "value": 1.0, + "dtype": target.dtype, + }, + ) block.desc.append_op().copy_from(op_desc) input_grad_names_set.add(grad_name) @@ -2111,8 +2323,9 @@ def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None): raise ValueError("all targets must be in the same block") if target.shape != grad.shape: raise ValueError( - "The shapes of target and grad are different: %s %s" % - (target.name, grad.name)) + "The shapes of target and grad are different: %s %s" + % (target.name, grad.name) + ) target_grad_map[_append_grad_suffix_(target.name)] = grad.name input_grad_names_set.add(grad.name) rename_var_map[grad_name] = grad.name @@ -2131,26 +2344,30 @@ def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None): block_no_grad_set = set(map(_strip_grad_suffix_, no_grad_dict[0])) op_path_dict = dict() - op_path = _find_op_path_(block, targets, inputs, block_no_grad_set, - op_path_dict) + op_path = _find_op_path_( + block, targets, inputs, block_no_grad_set, op_path_dict + ) # find no grad var by op_path - no_grad_vars = _find_no_grad_vars(block, op_path, targets, - block_no_grad_set) + no_grad_vars = _find_no_grad_vars( + block, op_path, targets, block_no_grad_set + ) block_no_grad_set.update(no_grad_vars) no_grad_dict[0].update(list(map(_append_grad_suffix_, block_no_grad_set))) grad_to_var = dict() grad_info_map = dict() - _append_backward_ops_(block, - op_path, - targets, - block, - no_grad_dict, - grad_to_var, - input_grad_names_set=input_grad_names_set, - op_path_dict=op_path_dict, - rename_var_map=rename_var_map) + _append_backward_ops_( + block, + op_path, + targets, + block, + no_grad_dict, + grad_to_var, + input_grad_names_set=input_grad_names_set, + op_path_dict=op_path_dict, + rename_var_map=rename_var_map, + ) # Because calc_gradient may be called multiple times, # we need rename the internal gradient variables so that they have @@ -2214,13 +2431,24 @@ def gradients(targets, inputs, target_gradients=None, no_grad_set=None): z = paddle.static.gradients([y], x) print(z) # [var x@GRAD : LOD_TENSOR.shape(-1, 2, 8, 8).dtype(float32).stop_gradient(False)] """ - check_type(targets, 'targets', (framework.Variable, list, tuple), - 'paddle.static.gradients') - check_type(inputs, 'inputs', (framework.Variable, list, tuple), - 'paddle.static.gradients') - check_type(target_gradients, 'target_gradients', - (framework.Variable, list, tuple, type(None)), - 'paddle.static.gradients') + check_type( + targets, + 'targets', + (framework.Variable, list, tuple), + 'paddle.static.gradients', + ) + check_type( + inputs, + 'inputs', + (framework.Variable, list, tuple), + 'paddle.static.gradients', + ) + check_type( + target_gradients, + 'target_gradients', + (framework.Variable, list, tuple, type(None)), + 'paddle.static.gradients', + ) outs = calc_gradient(targets, inputs, target_gradients, no_grad_set) return _as_list(outs) @@ -2263,10 +2491,18 @@ def gradients_with_optimizer(program, optimizer, inputs=None, outputs=None): print(opt_ops) """ - check_type(program, 'program', paddle.fluid.Program, - 'paddle.static.gradients_with_optimizer') - check_type(optimizer, 'optimizer', paddle.optimizer.Optimizer, - 'paddle.static.gradients_with_optimizer') + check_type( + program, + 'program', + paddle.fluid.Program, + 'paddle.static.gradients_with_optimizer', + ) + check_type( + optimizer, + 'optimizer', + paddle.optimizer.Optimizer, + 'paddle.static.gradients_with_optimizer', + ) if inputs is None or outputs is None: in_set = set() @@ -2285,9 +2521,12 @@ def gradients_with_optimizer(program, optimizer, inputs=None, outputs=None): grads = gradients(outputs, inputs) with program_guard(program, None): - pram_grads = [(pram, grad) for pram, grad in zip(inputs, grads) - if isinstance(pram, paddle.fluid.framework.Parameter) - and grad is not None] + pram_grads = [ + (pram, grad) + for pram, grad in zip(inputs, grads) + if isinstance(pram, paddle.fluid.framework.Parameter) + and grad is not None + ] optimize_ops = optimizer.apply_gradients(pram_grads) diff --git a/python/paddle/fluid/clip.py b/python/paddle/fluid/clip.py index fd5c457946884..93ad5564cb27b 100644 --- a/python/paddle/fluid/clip.py +++ b/python/paddle/fluid/clip.py @@ -29,8 +29,11 @@ from paddle import _C_ops, _legacy_C_ops __all__ = [ - 'set_gradient_clip', 'ErrorClipByValue', 'ClipGradByValue', - 'ClipGradByNorm', 'ClipGradByGlobalNorm' + 'set_gradient_clip', + 'ErrorClipByValue', + 'ClipGradByValue', + 'ClipGradByNorm', + 'ClipGradByGlobalNorm', ] _clip_by_global_norm_using_mp_type_flag = False @@ -49,9 +52,10 @@ def _clip_by_global_norm_using_mp_type(*args): def _cast_to_mp_type_if_enabled(x): - if (x.dtype == core.VarDesc.VarType.FP16 - or x.dtype == core.VarDesc.VarType.BF16 - ) and _clip_by_global_norm_using_mp_type(): + if ( + x.dtype == core.VarDesc.VarType.FP16 + or x.dtype == core.VarDesc.VarType.BF16 + ) and _clip_by_global_norm_using_mp_type(): return x.astype(core.VarDesc.VarType.FP32) else: return x @@ -63,8 +67,11 @@ def _squared_l2_norm(x): """ x = _cast_to_mp_type_if_enabled(x) - if core.is_compiled_with_xpu( - ) or x.dtype == core.VarDesc.VarType.FP16 or x.dtype == core.VarDesc.VarType.BF16: + if ( + core.is_compiled_with_xpu() + or x.dtype == core.VarDesc.VarType.FP16 + or x.dtype == core.VarDesc.VarType.BF16 + ): square = layers.square(x) sum_square = layers.reduce_sum(square) return sum_square @@ -86,7 +93,6 @@ def _squared_l2_norm(x): class BaseErrorClipAttr(object): - def __str__(self): raise NotImplementedError() @@ -161,8 +167,9 @@ def error_clip_callback(block, context): for grad_n in [n for n in op_desc.output_arg_names() if n in grad_to_var]: fwd_var = block._var_recursive(grad_to_var[grad_n]) error_clip = getattr(fwd_var, "error_clip", None) - if not (error_clip is None - or isinstance(error_clip, BaseErrorClipAttr)): + if not ( + error_clip is None or isinstance(error_clip, BaseErrorClipAttr) + ): raise TypeError( "Variable's error_clip should be an instance of BaseErrorClipAttr or None." ) @@ -171,7 +178,6 @@ def error_clip_callback(block, context): class ClipGradBase(object): - def __init__(self): super(ClipGradBase, self).__init__() @@ -194,7 +200,8 @@ def __call__(self, params_grads): warnings.warn( "'set_gradient_clip' will be ineffective, because you have " "set 'need_clip' in 'ParamAttr'. So, 'set_gradient_clip' " - "is redundant and you can remove it.") + "is redundant and you can remove it." + ) break return self._static_clip(params_grads) @@ -249,7 +256,7 @@ class ClipGradByValue(ClipGradBase): def __init__(self, max, min=None): super(ClipGradByValue, self).__init__() if min is None: - assert (max > 0.0) + assert max > 0.0 min = -max self.max = float(max) self.min = float(min) @@ -465,10 +472,9 @@ class ClipGradByGlobalNorm(ClipGradBase): sdg.step() """ - def __init__(self, - clip_norm, - group_name="default_group", - auto_skip_clip=False): + def __init__( + self, clip_norm, group_name="default_group", auto_skip_clip=False + ): super(ClipGradByGlobalNorm, self).__init__() self.clip_norm = float(clip_norm) self.group_name = group_name @@ -500,7 +506,10 @@ def _dygraph_clip(self, params_grads): merge_grad = layers.get_tensor_from_selected_rows(merge_grad) sum_square = _squared_l2_norm(merge_grad) - if sum_square.dtype == core.VarDesc.VarType.FP16 or sum_square.dtype == core.VarDesc.VarType.BF16: + if ( + sum_square.dtype == core.VarDesc.VarType.FP16 + or sum_square.dtype == core.VarDesc.VarType.BF16 + ): sum_square_list_fp16.append(sum_square) elif sum_square.dtype == core.VarDesc.VarType.FP32: sum_square_list_fp32.append(sum_square) @@ -508,8 +517,12 @@ def _dygraph_clip(self, params_grads): sum_square_list.append(sum_square) # all parameters have been filterd out - if len(sum_square_list) + len(sum_square_list_fp16) + len( - sum_square_list_fp32) == 0: + if ( + len(sum_square_list) + + len(sum_square_list_fp16) + + len(sum_square_list_fp32) + == 0 + ): return params_grads sum_dtype = 'float64' if len(sum_square_list) > 0 else "float32" @@ -528,22 +541,23 @@ def _dygraph_clip(self, params_grads): global_norm_var.append(global_norm_var_fp64) global_norm_var = paddle.add_n(global_norm_var) global_norm_var = layers.sqrt(global_norm_var) - max_global_norm = layers.fill_constant(shape=[1], - dtype=global_norm_var.dtype, - value=self.clip_norm) + max_global_norm = layers.fill_constant( + shape=[1], dtype=global_norm_var.dtype, value=self.clip_norm + ) need_clip = False if not self.auto_skip_clip: # always apply clip need_clip = True - clip_var = layers.elementwise_div(x=max_global_norm, - y=layers.elementwise_max( - x=global_norm_var, - y=max_global_norm)) + clip_var = layers.elementwise_div( + x=max_global_norm, + y=layers.elementwise_max(x=global_norm_var, y=max_global_norm), + ) elif global_norm_var > max_global_norm: # only when global_norm_var > max_global_norm, grad need clip need_clip = True - clip_var = layers.elementwise_div(x=max_global_norm, - y=global_norm_var) + clip_var = layers.elementwise_div( + x=max_global_norm, y=global_norm_var + ) for p, g in params_grads: if g is None: @@ -553,8 +567,11 @@ def _dygraph_clip(self, params_grads): continue # TODO(wangxi): use inplace elementwise_mul if need_clip: - clip_input = (clip_var.astype(g.dtype) - if clip_var.dtype != g.dtype else clip_var) + clip_input = ( + clip_var.astype(g.dtype) + if clip_var.dtype != g.dtype + else clip_var + ) new_grad = layers.elementwise_mul(g, clip_input) params_and_grads.append((p, new_grad)) else: @@ -578,7 +595,8 @@ def _static_clip(self, params_grads): if g.type == core.VarDesc.VarType.SELECTED_ROWS: merge_grad = layers.merge_selected_rows(g) merge_grad = layers.get_tensor_from_selected_rows( - merge_grad) + merge_grad + ) sum_square = _squared_l2_norm(merge_grad) if sum_square.dtype == core.VarDesc.VarType.FP16: sum_square_list_fp16.append(sum_square) @@ -588,8 +606,12 @@ def _static_clip(self, params_grads): sum_square_list.append(sum_square) # all parameters have been filterd out - if len(sum_square_list) + len(sum_square_list_fp16) + len( - sum_square_list_fp32) == 0: + if ( + len(sum_square_list) + + len(sum_square_list_fp16) + + len(sum_square_list_fp32) + == 0 + ): return params_grads with p.block.program._optimized_guard([p, g]): @@ -598,10 +620,14 @@ def _static_clip(self, params_grads): global_norm_var = [] if len(sum_square_list_fp16) > 0: global_norm_var_fp16 = layers.sums(sum_square_list_fp16) - if sum_square_list_fp32 or sum_square_list or not _allow_pure_fp16_global_norm_clip( + if ( + sum_square_list_fp32 + or sum_square_list + or not _allow_pure_fp16_global_norm_clip() ): global_norm_var.append( - global_norm_var_fp16.astype(sum_dtype)) + global_norm_var_fp16.astype(sum_dtype) + ) else: global_norm_var.append(global_norm_var_fp16) if len(sum_square_list_fp32) > 0: @@ -610,23 +636,28 @@ def _static_clip(self, params_grads): global_norm_var.append(global_norm_var_fp32) else: global_norm_var.append( - global_norm_var_fp32.astype(sum_dtype)) + global_norm_var_fp32.astype(sum_dtype) + ) if len(sum_square_list) > 0: # fp64 global_norm_var_other_dtype = layers.sums(sum_square_list) global_norm_var.append(global_norm_var_other_dtype) - global_norm_var = layers.sums(global_norm_var) if len( - global_norm_var) > 1 else global_norm_var[0] + global_norm_var = ( + layers.sums(global_norm_var) + if len(global_norm_var) > 1 + else global_norm_var[0] + ) global_norm_var = layers.sqrt(x=global_norm_var) max_global_norm = layers.fill_constant( - shape=[1], - dtype=global_norm_var.dtype, - value=self.clip_norm) - scale_var = layers.elementwise_div(x=max_global_norm, - y=layers.elementwise_max( - x=max_global_norm, - y=global_norm_var)) + shape=[1], dtype=global_norm_var.dtype, value=self.clip_norm + ) + scale_var = layers.elementwise_div( + x=max_global_norm, + y=layers.elementwise_max( + x=max_global_norm, y=global_norm_var + ), + ) param_new_grad_name_dict = dict() for p, g in params_grads: if g is None: @@ -638,29 +669,32 @@ def _static_clip(self, params_grads): with p.block.program._optimized_guard([p, g]): new_g = _cast_to_mp_type_if_enabled(g) # inplace - scale_input = (scale_var.astype('float16') if - new_g.dtype == core.VarDesc.VarType.FP16 and - scale_var.dtype != core.VarDesc.VarType.FP16 - else scale_var) + scale_input = ( + scale_var.astype('float16') + if new_g.dtype == core.VarDesc.VarType.FP16 + and scale_var.dtype != core.VarDesc.VarType.FP16 + else scale_var + ) # NOTE(Yuang Liu): For pure dp with gradient merge, the p and g # will be in different blocks with the gradient clip related ops. # We need to handle the correct block, otherwise will encounter # a 'NotFoundError' during compile time. block = default_main_program().current_block() - block.append_op(type='elementwise_mul', - inputs={ - 'X': new_g, - 'Y': scale_input - }, - outputs={'Out': new_g}) + block.append_op( + type='elementwise_mul', + inputs={'X': new_g, 'Y': scale_input}, + outputs={'Out': new_g}, + ) if new_g is not g: - block.append_op(type='cast', - inputs={'X': new_g}, - outputs={'Out': g}, - attrs={ - 'in_dtype': new_g.dtype, - 'out_dtype': g.dtype - }) + block.append_op( + type='cast', + inputs={'X': new_g}, + outputs={'Out': g}, + attrs={ + 'in_dtype': new_g.dtype, + 'out_dtype': g.dtype, + }, + ) param_new_grad_name_dict[p.name] = g.name params_and_grads.append((p, g)) @@ -673,7 +707,8 @@ def _process_context(self, context, param, grad): context[self.group_name] = [] context[self.group_name + "_clip_value"] = self.clip_norm context[self.group_name + "_clip"] = layers.fill_constant( - shape=[1], dtype=grad.dtype, value=self.clip_norm) + shape=[1], dtype=grad.dtype, value=self.clip_norm + ) else: if not self.clip_norm == context[self.group_name + "_clip_value"]: raise ValueError( @@ -696,20 +731,19 @@ def _create_operators(self, param, grad): group_norm_var = layers.sums(input=self.context[self.group_name]) group_norm_var = layers.sqrt(x=group_norm_var) clip_var = self.context[self.group_name + "_clip"] - group_scale_var = layers.elementwise_div(x=clip_var, - y=layers.elementwise_max( - x=clip_var, - y=group_norm_var)) - assert group_scale_var.shape == (1, ) + group_scale_var = layers.elementwise_div( + x=clip_var, + y=layers.elementwise_max(x=clip_var, y=group_norm_var), + ) + assert group_scale_var.shape == (1,) self.context[group_scale_name] = group_scale_var # inplace - param.block.append_op(type='elementwise_mul', - inputs={ - 'X': grad, - 'Y': self.context[group_scale_name] - }, - outputs={'Out': grad}) + param.block.append_op( + type='elementwise_mul', + inputs={'X': grad, 'Y': self.context[group_scale_name]}, + outputs={'Out': grad}, + ) return param, grad @@ -804,22 +838,26 @@ def network(): """ - warnings.warn("Caution! 'set_gradient_clip' is not recommended " - "and may be deprecated in future! " - "We recommend a new strategy: set 'grad_clip' " - "when initializing the 'optimizer'. " - "This method can reduce the mistakes, please " - "refer to documention of 'optimizer'.") + warnings.warn( + "Caution! 'set_gradient_clip' is not recommended " + "and may be deprecated in future! " + "We recommend a new strategy: set 'grad_clip' " + "when initializing the 'optimizer'. " + "This method can reduce the mistakes, please " + "refer to documention of 'optimizer'." + ) if not isinstance(clip, ClipGradBase): raise TypeError( - "'clip' should be an instance of ClipGradBase's derived class") + "'clip' should be an instance of ClipGradBase's derived class" + ) if program is None: program = framework.default_main_program() for op in program.block(0).ops: if 'op_namescope' in op.all_attrs() and "optimizer" in op.attr( - "op_namescope"): + "op_namescope" + ): warnings.warn( "'minimize' has been invoked before, this will make 'set_gradient_clip' " "be ineffective! Please invoke 'set_gradient_clip' before 'minimize'." @@ -844,14 +882,16 @@ def append_gradient_clip_ops(param_grads): for p, g in param_grads: if g is None: continue - with p.block.program._optimized_guard( - [p, g]), framework.name_scope('gradient_clip'): + with p.block.program._optimized_guard([p, g]), framework.name_scope( + 'gradient_clip' + ): clip_attr = getattr(p, 'gradient_clip_attr', None) if clip_attr is None: return param_grads if not isinstance(clip_attr, ClipGradBase): raise TypeError( - "clip attribute should be an instance of GradientClipBase") + "clip attribute should be an instance of GradientClipBase" + ) clip_attr._process_context(context=context, param=p, grad=g) @@ -860,8 +900,9 @@ def append_gradient_clip_ops(param_grads): for p, g in param_grads: if g is None: continue - with p.block.program._optimized_guard( - [p, g]), framework.name_scope('gradient_clip'): + with p.block.program._optimized_guard([p, g]), framework.name_scope( + 'gradient_clip' + ): param, new_grad = clip_attr._create_operators(param=p, grad=g) param_new_grad_name_dict[param.name] = new_grad.name res.append([param, new_grad]) @@ -885,12 +926,16 @@ def _correct_clip_op_role_var(params_grads, param_new_grad_name_dict): continue block_id_list.append(block_id) for op in param.block.program.global_block().ops: - if op.has_attr("op_namescope") and "gradient_clip" in op.attr( - "op_namescope") and op.attr('op_role_var'): + if ( + op.has_attr("op_namescope") + and "gradient_clip" in op.attr("op_namescope") + and op.attr('op_role_var') + ): param_name = op.attr('op_role_var')[0] if param_name in param_new_grad_name_dict: correct_p_g = [ - param_name, param_new_grad_name_dict[param_name] + param_name, + param_new_grad_name_dict[param_name], ] op._set_attr('op_role_var', correct_p_g) diff --git a/python/paddle/fluid/communicator.py b/python/paddle/fluid/communicator.py index 4704628f2aee8..be3fb5ea22deb 100755 --- a/python/paddle/fluid/communicator.py +++ b/python/paddle/fluid/communicator.py @@ -27,6 +27,7 @@ # limitations under the License. from .executor import global_scope + """ Communicator is used for async distribute training in distribute_transpiler mode. It's a wrapper of a cpp class Communicator and should be used inside fleet API. @@ -38,7 +39,6 @@ class Communicator(object): - def __init__(self, mode, kwargs=None, envs=None): """ Communicator is used for async distribute training in distribute_transpiler mode. @@ -69,7 +69,8 @@ def __init__(self, mode, kwargs=None, envs=None): else: if mode == DistributedMode.SYNC: envs["pserver_endpoints"] = ','.join( - kwargs["pserver_endpoints"]) + kwargs["pserver_endpoints"] + ) envs["trainers"] = str(kwargs["trainers"]) envs["trainer_id"] = str(kwargs["trainer_id"]) @@ -93,26 +94,32 @@ def __init__(self, mode, kwargs=None, envs=None): self.send_ctx_ = None self.recv_ctx_ = None - def init_with_ctx(self, - send_ctx, - recv_ctx, - proto_txt, - unit64_hosts, - scope=None): + def init_with_ctx( + self, send_ctx, recv_ctx, proto_txt, unit64_hosts, scope=None + ): if scope == None: scope = global_scope() - self.communicator_ = core.DistCommunicator(self.mode, proto_txt, - unit64_hosts, send_ctx, - recv_ctx, scope, self.envs) + self.communicator_ = core.DistCommunicator( + self.mode, + proto_txt, + unit64_hosts, + send_ctx, + recv_ctx, + scope, + self.envs, + ) self.send_ctx_ = send_ctx self.recv_ctx_ = recv_ctx - def create_client_to_client_connection(self, - pserver_timeout_ms=500000, - pserver_connect_timeout_ms=10000, - max_retry=3): + def create_client_to_client_connection( + self, + pserver_timeout_ms=500000, + pserver_connect_timeout_ms=10000, + max_retry=3, + ): self.communicator_.create_client_to_client_connection( - pserver_timeout_ms, pserver_connect_timeout_ms, max_retry) + pserver_timeout_ms, pserver_connect_timeout_ms, max_retry + ) def get_client_info(self): return self.communicator_.get_client_info() @@ -209,7 +216,6 @@ def push_sparse_param(self, var_name, table_id=-1, scope=None): class FLCommunicator(Communicator): ## only for coordinator - def __init__(self, ps_hosts, kwargs=None): mode = None super(FLCommunicator, self).__init__(mode, kwargs) @@ -221,8 +227,9 @@ def __init__(self, ps_hosts, kwargs=None): def start_coordinator(self, self_endpoint, trainer_endpoints): if self.communicator_ != None: - self.communicator_.start_coordinator(self_endpoint, - trainer_endpoints) + self.communicator_.start_coordinator( + self_endpoint, trainer_endpoints + ) return def save_fl_strategy(self, mp): @@ -240,7 +247,6 @@ def query_fl_clients_info(self): class LargeScaleKV(object): - def __init__(self): self.scale_kv = core.LargeScaleKV() @@ -255,10 +261,10 @@ def size(self, varname): class HeterClient(object): - def __init__(self, endpoint, previous_endpoint, trainer_id): - self.heter_client_ = core.HeterClient(endpoint, previous_endpoint, - trainer_id) + self.heter_client_ = core.HeterClient( + endpoint, previous_endpoint, trainer_id + ) def stop(self): self.heter_client_.stop() diff --git a/python/paddle/fluid/compiler.py b/python/paddle/fluid/compiler.py index 7ac8e4c30ab69..1adef41f86a35 100644 --- a/python/paddle/fluid/compiler.py +++ b/python/paddle/fluid/compiler.py @@ -22,8 +22,11 @@ from . import core __all__ = [ - 'CompiledProgram', 'ExecutionStrategy', 'BuildStrategy', - 'IpuCompiledProgram', 'IpuStrategy' + 'CompiledProgram', + 'ExecutionStrategy', + 'BuildStrategy', + 'IpuCompiledProgram', + 'IpuStrategy', ] ExecutionStrategy = core.ParallelExecutor.ExecutionStrategy @@ -40,8 +43,7 @@ def _place_obj(place): def _is_pserver_mode(main_program): - main = main_program if main_program \ - else framework.default_main_program() + main = main_program if main_program else framework.default_main_program() for op in main.global_block().ops: if op.type in ["send", "recv"]: return True @@ -50,8 +52,11 @@ def _is_pserver_mode(main_program): def _has_backward_op(graph): for node in graph.nodes(): - if node.is_op() and node.op() is not None and \ - node.op().type().endswith("_grad"): + if ( + node.is_op() + and node.op() is not None + and node.op().type().endswith("_grad") + ): return True return False @@ -60,7 +65,8 @@ def _prune_feed_ops(program): # prune the feed ops in the program. pop_idx = [] for i, op in enumerate(program.global_block().ops): - if op.type == "feed": pop_idx.append(i) + if op.type == "feed": + pop_idx.append(i) for index in pop_idx[::-1]: program.global_block()._remove_op(index) @@ -69,8 +75,9 @@ def _has_optimize_op(block): for op in block.ops: op_maker = core.op_proto_and_checker_maker optimize = core.op_proto_and_checker_maker.OpRole.Optimize - if op_maker.kOpRoleVarAttrName() in op.attr_names and \ - int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(optimize): + if op_maker.kOpRoleVarAttrName() in op.attr_names and int( + op.all_attrs()[op_maker.kOpRoleAttrName()] + ) == int(optimize): return True return False @@ -93,7 +100,8 @@ def _should_broadcast_or_not_exists(program, var_name): if var is None: return True is_distributed = getattr(var, '_is_distributed', False) or getattr( - var, 'is_distributed', False) + var, 'is_distributed', False + ) return not is_distributed @@ -159,7 +167,8 @@ def __init__(self, program_or_graph, build_strategy=None): else: raise TypeError( "The type of program_to_graph parameter is wrong, expected Graph or Program, but received %s" - % type(program_or_graph)) + % type(program_or_graph) + ) self._scope = None self._place = None @@ -173,12 +182,14 @@ def __init__(self, program_or_graph, build_strategy=None): self._build_strategy = build_strategy self._exec_strategy = None - def with_data_parallel(self, - loss_name=None, - build_strategy=None, - exec_strategy=None, - share_vars_from=None, - places=None): + def with_data_parallel( + self, + loss_name=None, + build_strategy=None, + exec_strategy=None, + share_vars_from=None, + places=None, + ): """ This interface is used to transform the input Program or Graph to a multi-graph to run the model in data parallel mode. Users can use the build_strategy and @@ -284,14 +295,19 @@ def with_data_parallel(self, feed={"X": test_data}, fetch_list=[loss.name]) """ - assert not self._is_data_parallel, "Already compiled with parallel, cannot be recompiled." - assert not self._is_inference, "Cannot compile with both data parallel and inference." + assert ( + not self._is_data_parallel + ), "Already compiled with parallel, cannot be recompiled." + assert ( + not self._is_inference + ), "Cannot compile with both data parallel and inference." self._is_data_parallel = True # FIXME(zcd): Currently, the build_strategy can be set during creating # CompiledProgram or calling with_data_parallel, and it may be confusing, # but in the long run, we should set up build_strategy only when creating # CompiledProgram, and exec_strategy should be deprecated. - if build_strategy is not None: self._build_strategy = build_strategy + if build_strategy is not None: + self._build_strategy = build_strategy self._exec_strategy = exec_strategy self._loss_name = loss_name self._share_vars_from = share_vars_from @@ -301,7 +317,9 @@ def with_data_parallel(self, self._places = _get_paddle_place(places) if _has_backward_op(self._graph): - assert self._loss_name is not None, "The loss name of CompiledProgram is None. The loss name should be set if CompiledProgram contains backward part." + assert ( + self._loss_name is not None + ), "The loss name of CompiledProgram is None. The loss name should be set if CompiledProgram contains backward part." if self._places is not None: if not isinstance(self._places, (list, tuple)): @@ -310,20 +328,26 @@ def with_data_parallel(self, return self def _with_inference_optimize(self, config): - """ Add inference optimize + """Add inference optimize Args: config: instance of `NativeConfig` or `AnalysisConfig` to create predictor Returns: self """ - assert not self._is_data_parallel, "Cannot compile with both data parallel and inference" - assert not self._is_inference, "Already compiled with inference, cannot be recompiled." - - assert any([ - isinstance(config, InferNativeConfig), - isinstance(config, InferAnalysisConfig) - ]) + assert ( + not self._is_data_parallel + ), "Cannot compile with both data parallel and inference" + assert ( + not self._is_inference + ), "Already compiled with inference, cannot be recompiled." + + assert any( + [ + isinstance(config, InferNativeConfig), + isinstance(config, InferAnalysisConfig), + ] + ) self._is_inference = True self._infer_config = config return self @@ -340,18 +364,23 @@ def _compile_data_parallel(self, places, use_device, scope=None): if not self._share_vars_from._is_data_parallel: raise ValueError( "The shared Program is not data parallel, cannot " - "share variables from it.") + "share variables from it." + ) if self._share_vars_from._executor is None: raise ValueError( "The shared Program is not compiled and executed, so there is no " - "variables to share.") + "variables to share." + ) self._local_scopes = self._share_vars_from._executor.local_scopes() else: assert scope is not None, "" self._local_scopes = [] - assert isinstance(places, tuple) or isinstance(places, list), \ - "Currently , The places type can only be list or tuple, but the input type is {}.".format(type(places)) + assert isinstance(places, tuple) or isinstance( + places, list + ), "Currently , The places type can only be list or tuple, but the input type is {}.".format( + type(places) + ) if self._build_strategy is None: self._build_strategy = BuildStrategy() @@ -372,41 +401,61 @@ def _compile_data_parallel(self, places, use_device, scope=None): else: self._exec_strategy.num_threads = len(places) * 2 - if "FLAGS_use_cinn" in core.globals() and core.globals( - )["FLAGS_use_cinn"] and self._exec_strategy.num_threads != 1: - warnings.warn("At present, when CINN is turned on, each process can " \ - "only contain one thread, so reset the number of threads to 1 here.") + if ( + "FLAGS_use_cinn" in core.globals() + and core.globals()["FLAGS_use_cinn"] + and self._exec_strategy.num_threads != 1 + ): + warnings.warn( + "At present, when CINN is turned on, each process can " + "only contain one thread, so reset the number of threads to 1 here." + ) self._exec_strategy.num_threads = 1 if self._build_strategy.num_trainers > 1: - assert self._is_data_parallel, \ - "If you use multi-trainer to train the model, you should use "\ + assert self._is_data_parallel, ( + "If you use multi-trainer to train the model, you should use " "the data parallel model, i.e. calling with_data_parallel function." + ) # TODO(wuyi): trainer endpoings should be passed in through # build_strategy, not program.xxx. # TODO(gongwb): let user to set them once. - if self._program and self._build_strategy.num_trainers > 1 and \ - self._program._trainers_endpoints: + if ( + self._program + and self._build_strategy.num_trainers > 1 + and self._program._trainers_endpoints + ): tps = self._program._trainers_endpoints assert self._build_strategy.num_trainers == len( - tps), "The trainer numbers is not equal to endpoint numbers." + tps + ), "The trainer numbers is not equal to endpoint numbers." self._build_strategy.trainers_endpoints = tps if self._program: self._build_strategy.nccl_comm_num = self._program._nccl_comm_num - self._build_strategy.use_hierarchical_allreduce = self._program._use_hierarchical_allreduce - self._build_strategy.hierarchical_allreduce_inter_nranks = self._program._hierarchical_allreduce_inter_nranks + self._build_strategy.use_hierarchical_allreduce = ( + self._program._use_hierarchical_allreduce + ) + self._build_strategy.hierarchical_allreduce_inter_nranks = ( + self._program._hierarchical_allreduce_inter_nranks + ) if self._build_strategy.sync_batch_norm: self._build_strategy.enable_sequential_execution = True if self._program is not None and self._program._enable_dgc: - assert self._exec_strategy._use_device == DeviceType.CUDA, "DGC only used under CUDA environment." - assert self._build_strategy.num_trainers * len( - places) > 1, "DGC is not avaliable for single card training." - assert self._build_strategy.reduce_strategy == BuildStrategy.ReduceStrategy.AllReduce, "DGC \ + assert ( + self._exec_strategy._use_device == DeviceType.CUDA + ), "DGC only used under CUDA environment." + assert ( + self._build_strategy.num_trainers * len(places) > 1 + ), "DGC is not avaliable for single card training." + assert ( + self._build_strategy.reduce_strategy + == BuildStrategy.ReduceStrategy.AllReduce + ), "DGC \ only can be used for AllReduce BuildStrategy." # DGC doesn't support fuse for now, close fuse. @@ -414,11 +463,17 @@ def _compile_data_parallel(self, places, use_device, scope=None): self._persistable_vars = [] for node in self._graph.nodes(): - if node.is_var() and node.var() is not None and node.var().persistable() and \ - node.var().type() != core.VarDesc.VarType.RAW: + if ( + node.is_var() + and node.var() is not None + and node.var().persistable() + and node.var().type() != core.VarDesc.VarType.RAW + ): name = node.name() - if self._program is not None and _should_broadcast_or_not_exists( - self._program, name): + if ( + self._program is not None + and _should_broadcast_or_not_exists(self._program, name) + ): self._persistable_vars.append(node.name()) places = list(map(_place_obj, places)) @@ -429,11 +484,16 @@ def _compile_data_parallel(self, places, use_device, scope=None): self._persistable_vars = list(set(self._persistable_vars)) self._persistable_vars.sort() - return core.ParallelExecutor(places, self._persistable_vars, - self._loss_name if self._loss_name else '', - self._scope, self._local_scopes, - self._exec_strategy, self._build_strategy, - self._graph) + return core.ParallelExecutor( + places, + self._persistable_vars, + self._loss_name if self._loss_name else '', + self._scope, + self._local_scopes, + self._exec_strategy, + self._build_strategy, + self._graph, + ) def _compile_inference(self): return core.create_paddle_predictor(self._infer_config) @@ -471,28 +531,31 @@ def _compile(self, scope, place): # Todo(liym27):If optimizer is used in control flow, # training on multi-places is not supported now, will # be supported later. - if len(self._places) > 1 and \ - _has_optimizer_in_control_flow(self._program): + if len(self._places) > 1 and _has_optimizer_in_control_flow( + self._program + ): raise NotImplementedError( "If optimizer is used in control flow, " - "training on multi-places is not supported now.") + "training on multi-places is not supported now." + ) if isinstance(self._place, core.CUDAPlace): use_device = DeviceType.CUDA elif isinstance(self._place, core.XPUPlace): use_device = DeviceType.XPU else: use_device = DeviceType.CPU - self._executor = self._compile_data_parallel(use_device=use_device, - scope=self._scope, - places=self._places) + self._executor = self._compile_data_parallel( + use_device=use_device, scope=self._scope, places=self._places + ) return self def _get_places(self, place, place_list): - has_set_place = (place_list is not None) + has_set_place = place_list is not None if has_set_place: for p in place_list: - assert p._type() == place._type(), \ - "Place type not match. You may set wrong type of places." + assert ( + p._type() == place._type() + ), "Place type not match. You may set wrong type of places." else: if isinstance(place, core.CUDAPlace): place_list = cuda_places() @@ -515,9 +578,9 @@ def __init__(self): pass @staticmethod - def convert_concrete_program(ipu_strategy, - concrete_program, - class_instance=None): + def convert_concrete_program( + ipu_strategy, concrete_program, class_instance=None + ): """ Convert the ConcreteProgram to IPUConcreteProgram. """ @@ -553,7 +616,8 @@ def append_backward_desc(): # copy the bias and filters for param_or_buffer in concrete_program.parameters: param_or_buffer_tensor = scope.var( - param_or_buffer.name).get_tensor() + param_or_buffer.name + ).get_tensor() src_tensor = param_or_buffer.value().get_tensor() param_or_buffer_tensor._share_data_with(src_tensor) @@ -576,13 +640,16 @@ def append_backward_desc(): dtype=var_tmp.dtype, type=var_tmp.type, shape=var_tmp.shape, - belong_to_optimizer=True) + belong_to_optimizer=True, + ) device = optimizer._get_device_for_param(param_name) with device_guard(device): optimizer.helper.set_variable_initializer( - var, initializer=Constant(value=0.0)) + var, initializer=Constant(value=0.0) + ) param_or_lr_tensor = scope.find_var( - var_tmp.name).get_tensor() + var_tmp.name + ).get_tensor() optim_tensor = var.value().get_tensor() param_or_lr_tensor._share_data_with(optim_tensor) optimizer._accumulators[k][param_name] = var @@ -595,16 +662,19 @@ def func_compile(): to_fp16_var_names = paddle.static.amp.cast_model_to_fp16( concrete_program.main_program, amp_list, - use_fp16_guard=False) + use_fp16_guard=False, + ) paddle.static.amp.cast_parameters_to_fp16( paddle.CPUPlace(), concrete_program.main_program, - to_fp16_var_names=to_fp16_var_names) - - program = IpuCompiledProgram(concrete_program.main_program, - ipu_strategy=ipu_strategy, - scope=scope).compile( - feed_list, fetch_list) + to_fp16_var_names=to_fp16_var_names, + ) + + program = IpuCompiledProgram( + concrete_program.main_program, + ipu_strategy=ipu_strategy, + scope=scope, + ).compile(feed_list, fetch_list) return program main_program = func_compile() @@ -613,7 +683,7 @@ def func_compile(): @staticmethod def patch_program_cache(ipu_strategy): - """ Monkey patch ProgramCache discriptor to support dynamic2static in IPU. + """Monkey patch ProgramCache discriptor to support dynamic2static in IPU. Args: ipu_strategy: The ipu_strategy used in dynamic graph. @@ -621,53 +691,70 @@ def patch_program_cache(ipu_strategy): Returns: None """ - from ..fluid.dygraph.dygraph_to_static.program_translator import ProgramCache - from ..fluid.dygraph.dygraph_to_static.program_translator import CacheKey + from ..fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramCache, + ) + from ..fluid.dygraph.dygraph_to_static.program_translator import ( + CacheKey, + ) from ..fluid.dygraph.dygraph_to_static import logging_utils - from ..fluid.dygraph.dygraph_to_static.program_translator import MAX_TRACED_PROGRAM_COUNT - from ..fluid.dygraph.dygraph_to_static.partial_program import partial_program_from + from ..fluid.dygraph.dygraph_to_static.program_translator import ( + MAX_TRACED_PROGRAM_COUNT, + ) + from ..fluid.dygraph.dygraph_to_static.partial_program import ( + partial_program_from, + ) old_getter = ProgramCache.__getitem__ def patch_getter(self, item): if not isinstance(item, CacheKey): raise ValueError( - 'type(item) should be CacheKey, but received %s' % - type(item).__name__) + 'type(item) should be CacheKey, but received %s' + % type(item).__name__ + ) item_id = hash(item) self._recent_key = item_id if item_id not in self._caches or ipu_strategy.need_compile: if item_id in self._caches: logging_utils.warn( - "ipu_strategy chances detected. Please sync weights.") + "ipu_strategy chances detected. Please sync weights." + ) if self._caches and not ipu_strategy.need_compile: logging_utils.warn( "dynamic2static on IPU doesn't support mutiple caches. Please make sure" - "dynamic inputs is not used.") + "dynamic inputs is not used." + ) concrete_program, _ = self._build_once(item) concrete_program = IpuDynamicPatcher.convert_concrete_program( - ipu_strategy, concrete_program, item.class_instance) + ipu_strategy, concrete_program, item.class_instance + ) - self._caches[item_id] = (concrete_program, - partial_program_from(concrete_program)) + self._caches[item_id] = ( + concrete_program, + partial_program_from(concrete_program), + ) # Note: raise warnings if number of traced program is more than `max_tracing_count` current_tracing_count = len(self._caches) if current_tracing_count > MAX_TRACED_PROGRAM_COUNT: logging_utils.warn( "Current traced program number: {} > `max_tracing_count`:{}. Too much cached programs will bring expensive overhead. " - "The reason may be: (1) passing tensors with different shapes, (2) passing python objects instead of tensors." - .format(current_tracing_count, - MAX_TRACED_PROGRAM_COUNT)) + "The reason may be: (1) passing tensors with different shapes, (2) passing python objects instead of tensors.".format( + current_tracing_count, MAX_TRACED_PROGRAM_COUNT + ) + ) return self._caches[item_id] setattr(ProgramCache, '__getitem__', patch_getter) IpuDynamicPatcher.patcher_cache.append( - [ProgramCache, '__getitem__', old_getter]) + [ProgramCache, '__getitem__', old_getter] + ) @staticmethod def patch_lr_scheduler(ipu_strategy): from paddle.optimizer.lr import LRScheduler + # For IPU dynamic graph usage, lr_var is not synced in executor as static mode do. # Manually set lr to ipu_strategy to update the lr. old_step = LRScheduler.step @@ -718,10 +805,8 @@ def __init__(self): 'on_chip': 0, 'use_replicated_tensor_sharding': 1, }, # set optimizer location - 'accumulation_and_replication_reduction_type': - 1, # popart::ReductionType::Mean - 'mean_accumulation_and_replication_reduction_strategy': - 1, # popart::MeanReductionStrategy::Post + 'accumulation_and_replication_reduction_type': 1, # popart::ReductionType::Mean + 'mean_accumulation_and_replication_reduction_strategy': 1, # popart::MeanReductionStrategy::Post } self._ipu_strategy.set_options(default_options) self.has_custom_ops = False @@ -732,6 +817,7 @@ def __init__(self): "Can not use IpuStrategy in non IPU compiled environment, please re-compile with WITH_IPU=ON." ) from paddle import in_dynamic_mode + if in_dynamic_mode(): self.register_patch() @@ -797,6 +883,7 @@ def set_optimizer(self, optimizer): ipu_strategy.set_optimizer(optimizer) """ from paddle import in_dynamic_mode + if in_dynamic_mode(): self._optimizer = optimizer optimizer_attrs = self.parse_optimizer(optimizer) @@ -831,6 +918,7 @@ def parse_optimizer(self, optimizer): def get_lr(): from paddle.optimizer.lr import LRScheduler + if isinstance(optimizer._learning_rate, float): return {"lr": optimizer._learning_rate} elif isinstance(optimizer._learning_rate, LRScheduler): @@ -842,11 +930,13 @@ def get_lr(): optimizer_attrs.update(fn()) return optimizer_attrs - def set_graph_config(self, - num_ipus=1, - is_training=True, - micro_batch_size=1, - enable_manual_shard=False): + def set_graph_config( + self, + num_ipus=1, + is_training=True, + micro_batch_size=1, + enable_manual_shard=False, + ): """ Set graph configuration to the IpuStrategy instance. @@ -889,11 +979,13 @@ def set_graph_config(self, } self.set_options(options) - def set_pipelining_config(self, - enable_pipelining=False, - batches_per_step=1, - enable_gradient_accumulation=False, - accumulation_factor=1): + def set_pipelining_config( + self, + enable_pipelining=False, + batches_per_step=1, + enable_gradient_accumulation=False, + accumulation_factor=1, + ): """ Set pipelining configuration to the IpuStrategy instance. Used to optimize the throughput performance. @@ -967,11 +1059,9 @@ def set_precision_config(self, enable_fp16=False): } self.set_options(options) - def add_custom_op(self, - paddle_op, - popart_op=None, - domain='custom.ops', - version=1): + def add_custom_op( + self, paddle_op, popart_op=None, domain='custom.ops', version=1 + ): """ Add a mapping to use popart custom ops running on the IPU. @@ -1201,8 +1291,9 @@ def __init__(self, program=None, scope=None, ipu_strategy=None): if not isinstance(program, framework.Program): raise TypeError( - "The type of program is wrong, expected Program, but got %s" % - type(program)) + "The type of program is wrong, expected Program, but got %s" + % type(program) + ) self._program = program self._compiled = False @@ -1212,6 +1303,7 @@ def __init__(self, program=None, scope=None, ipu_strategy=None): else: # import here to avoiding confused import paddle + self._scope = paddle.static.global_scope() if ipu_strategy is not None: diff --git a/python/paddle/fluid/contrib/decoder/beam_search_decoder.py b/python/paddle/fluid/contrib/decoder/beam_search_decoder.py index 10bc30d13d1a7..429feda47bb53 100644 --- a/python/paddle/fluid/contrib/decoder/beam_search_decoder.py +++ b/python/paddle/fluid/contrib/decoder/beam_search_decoder.py @@ -64,13 +64,15 @@ class InitState(object): See `StateCell`. """ - def __init__(self, - init=None, - shape=None, - value=0.0, - init_boot=None, - need_reorder=False, - dtype='float32'): + def __init__( + self, + init=None, + shape=None, + value=0.0, + init_boot=None, + need_reorder=False, + dtype='float32', + ): if init is not None: self._init = init elif init_boot is None: @@ -78,10 +80,9 @@ def __init__(self, 'init_boot must be provided to infer the shape of InitState .\n' ) else: - self._init = layers.fill_constant_batch_size_like(input=init_boot, - value=value, - shape=shape, - dtype=dtype) + self._init = layers.fill_constant_batch_size_like( + input=init_boot, value=value, shape=shape, dtype=dtype + ) self._shape = shape self._value = value @@ -98,12 +99,12 @@ def need_reorder(self): class _MemoryState(object): - def __init__(self, state_name, rnn_obj, init_state): self._state_name = state_name # each is a rnn.memory self._rnn_obj = rnn_obj self._state_mem = self._rnn_obj.memory( - init=init_state.value, need_reorder=init_state.need_reorder) + init=init_state.value, need_reorder=init_state.need_reorder + ) def get_state(self): return self._state_mem @@ -113,7 +114,6 @@ def update_state(self, state): class _ArrayState(object): - def __init__(self, state_name, block, init_state): self._state_name = state_name self._block = block @@ -121,33 +121,36 @@ def __init__(self, state_name, block, init_state): self._state_array = self._block.create_var( name=unique_name.generate('array_state_array'), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=init_state.value.dtype) + dtype=init_state.value.dtype, + ) self._counter = self._block.create_var( name=unique_name.generate('array_state_counter'), type=core.VarDesc.VarType.LOD_TENSOR, - dtype='int64') + dtype='int64', + ) # initialize counter - self._block.append_op(type='fill_constant', - inputs={}, - outputs={'Out': [self._counter]}, - attrs={ - 'shape': [1], - 'dtype': self._counter.dtype, - 'value': float(0.0), - 'force_cpu': True - }) + self._block.append_op( + type='fill_constant', + inputs={}, + outputs={'Out': [self._counter]}, + attrs={ + 'shape': [1], + 'dtype': self._counter.dtype, + 'value': float(0.0), + 'force_cpu': True, + }, + ) self._counter.stop_gradient = True # write initial state - block.append_op(type='write_to_array', - inputs={ - 'X': init_state.value, - 'I': self._counter - }, - outputs={'Out': self._state_array}) + block.append_op( + type='write_to_array', + inputs={'X': init_state.value, 'I': self._counter}, + outputs={'Out': self._state_array}, + ) def get_state(self): state = layers.array_read(array=self._state_array, i=self._counter) @@ -220,8 +223,9 @@ def _enter_decoder(self, decoder_obj): def _leave_decoder(self, decoder_obj): if not self._in_decoder: - raise ValueError('StateCell not in decoder, ' - 'invalid leaving operation.') + raise ValueError( + 'StateCell not in decoder, ' 'invalid leaving operation.' + ) if self._cur_decoder_obj != decoder_obj: raise ValueError('Inconsistent decoder object in StateCell.') @@ -242,29 +246,35 @@ def _switch_decoder(self): # lazy switch state = self._cur_states[state_name] if not isinstance(state, InitState): - raise ValueError('Current type of state is %s, should be ' - 'an InitState object.' % type(state)) + raise ValueError( + 'Current type of state is %s, should be ' + 'an InitState object.' % type(state) + ) self._states_holder[state_name] = {} if self._cur_decoder_obj.type == _DecoderType.TRAINING: - self._states_holder[state_name][id(self._cur_decoder_obj)] \ - = _MemoryState(state_name, - self._cur_decoder_obj.dynamic_rnn, - state) + self._states_holder[state_name][ + id(self._cur_decoder_obj) + ] = _MemoryState( + state_name, self._cur_decoder_obj.dynamic_rnn, state + ) elif self._cur_decoder_obj.type == _DecoderType.BEAM_SEARCH: - self._states_holder[state_name][id(self._cur_decoder_obj)] \ - = _ArrayState(state_name, - self._cur_decoder_obj._parent_block(), - state) + self._states_holder[state_name][ + id(self._cur_decoder_obj) + ] = _ArrayState( + state_name, self._cur_decoder_obj._parent_block(), state + ) else: - raise ValueError('Unknown decoder type, only support ' - '[TRAINING, BEAM_SEARCH]') + raise ValueError( + 'Unknown decoder type, only support ' + '[TRAINING, BEAM_SEARCH]' + ) # Read back, since current state should be LoDTensor - self._cur_states[state_name] = \ - self._states_holder[state_name][ - id(self._cur_decoder_obj)].get_state() + self._cur_states[state_name] = self._states_holder[state_name][ + id(self._cur_decoder_obj) + ].get_state() self._switched_decoder = True @@ -284,7 +294,8 @@ def get_state(self, state_name): if state_name not in self._cur_states: raise ValueError( 'Unknown state %s. Please make sure _switch_decoder() ' - 'invoked.' % state_name) + 'invoked.' % state_name + ) return self._cur_states[state_name] @@ -328,8 +339,10 @@ def state_updater(self, updater): def _decorator(state_cell): if state_cell == self: - raise TypeError('Updater should only accept a StateCell object ' - 'as argument.') + raise TypeError( + 'Updater should only accept a StateCell object ' + 'as argument.' + ) updater(state_cell) return _decorator @@ -353,9 +366,11 @@ def compute_state(self, inputs): for input_name, input_value in inputs.items(): if input_name not in self._inputs: - raise ValueError('Unknown input %s. ' - 'Please make sure %s in input ' - 'place holder.' % (input_name, input_name)) + raise ValueError( + 'Unknown input %s. ' + 'Please make sure %s in input ' + 'place holder.' % (input_name, input_name) + ) self._inputs[input_name] = input_value self._state_updater(self) @@ -368,10 +383,13 @@ def update_states(self): for state_name, decoder_state in self._states_holder.items(): if id(self._cur_decoder_obj) not in decoder_state: - raise ValueError('Unknown decoder object, please make sure ' - 'switch_decoder been invoked.') + raise ValueError( + 'Unknown decoder object, please make sure ' + 'switch_decoder been invoked.' + ) decoder_state[id(self._cur_decoder_obj)].update_state( - self._cur_states[state_name]) + self._cur_states[state_name] + ) def out_state(self): """ @@ -409,6 +427,7 @@ class TrainingDecoder(object): decoder.state_cell.update_states() decoder.output(current_score) """ + BEFORE_DECODER = 0 IN_DECODER = 1 AFTER_DECODER = 2 @@ -495,8 +514,10 @@ def __call__(self, *args, **kwargs): Variable: The specified output of the RNN cell. """ if self._status != TrainingDecoder.AFTER_DECODER: - raise ValueError('Output of training decoder can only be visited ' - 'outside the block.') + raise ValueError( + 'Output of training decoder can only be visited ' + 'outside the block.' + ) return self._dynamic_rnn(*args, **kwargs) def output(self, *outputs): @@ -520,8 +541,10 @@ def output(self, *outputs): def _assert_in_decoder_block(self, method): if self._status != TrainingDecoder.IN_DECODER: - raise ValueError('%s should be invoked inside block of ' - 'TrainingDecoder object.' % method) + raise ValueError( + '%s should be invoked inside block of ' + 'TrainingDecoder object.' % method + ) class BeamSearchDecoder(object): @@ -569,42 +592,44 @@ class BeamSearchDecoder(object): decoder.decode() translation_ids, translation_scores = decoder() """ + BEFORE_BEAM_SEARCH_DECODER = 0 IN_BEAM_SEARCH_DECODER = 1 AFTER_BEAM_SEARCH_DECODER = 2 - def __init__(self, - state_cell, - init_ids, - init_scores, - target_dict_dim, - word_dim, - input_var_dict={}, - topk_size=50, - sparse_emb=True, - max_len=100, - beam_size=1, - end_id=1, - name=None): + def __init__( + self, + state_cell, + init_ids, + init_scores, + target_dict_dim, + word_dim, + input_var_dict={}, + topk_size=50, + sparse_emb=True, + max_len=100, + beam_size=1, + end_id=1, + name=None, + ): self._helper = LayerHelper('beam_search_decoder', name=name) self._counter = layers.zeros(shape=[1], dtype='int64') self._counter.stop_gradient = True self._type = _DecoderType.BEAM_SEARCH - self._max_len = layers.fill_constant(shape=[1], - dtype='int64', - value=max_len) - self._cond = layers.less_than(x=self._counter, - y=layers.fill_constant(shape=[1], - dtype='int64', - value=max_len)) + self._max_len = layers.fill_constant( + shape=[1], dtype='int64', value=max_len + ) + self._cond = layers.less_than( + x=self._counter, + y=layers.fill_constant(shape=[1], dtype='int64', value=max_len), + ) self._while_op = layers.While(self._cond) self._state_cell = state_cell self._state_cell._enter_decoder(self) self._status = BeamSearchDecoder.BEFORE_BEAM_SEARCH_DECODER - self._zero_idx = layers.fill_constant(shape=[1], - value=0, - dtype='int64', - force_cpu=True) + self._zero_idx = layers.fill_constant( + shape=[1], value=0, dtype='int64', force_cpu=True + ) self._array_dict = {} self._array_link = [] self._ids_array = None @@ -637,13 +662,13 @@ def block(self): layers.increment(x=self._counter, value=1.0, in_place=True) for value, array in self._array_link: - layers.array_write(x=value, - i=self._counter, - array=array) + layers.array_write( + x=value, i=self._counter, array=array + ) - layers.less_than(x=self._counter, - y=self._max_len, - cond=self._cond) + layers.less_than( + x=self._counter, y=self._max_len, cond=self._cond + ) self._status = BeamSearchDecoder.AFTER_BEAM_SEARCH_DECODER self._state_cell._leave_decoder(self) @@ -656,11 +681,9 @@ def early_stop(self): """ Stop the generation process in advance. Could be used as "break". """ - layers.fill_constant(shape=[1], - value=0, - dtype='bool', - force_cpu=True, - out=self._cond) + layers.fill_constant( + shape=[1], value=0, dtype='bool', force_cpu=True, out=self._cond + ) def decode(self): """ @@ -675,32 +698,39 @@ def decode(self): """ with self.block(): prev_ids = self.read_array(init=self._init_ids, is_ids=True) - prev_scores = self.read_array(init=self._init_scores, - is_scores=True) + prev_scores = self.read_array( + init=self._init_scores, is_scores=True + ) prev_ids_embedding = layers.embedding( input=prev_ids, size=[self._target_dict_dim, self._word_dim], dtype='float32', - is_sparse=self._sparse_emb) + is_sparse=self._sparse_emb, + ) feed_dict = {} update_dict = {} for init_var_name, init_var in self._input_var_dict.items(): if init_var_name not in self.state_cell._inputs: - raise ValueError('Variable ' + init_var_name + - ' not found in StateCell!\n') + raise ValueError( + 'Variable ' + + init_var_name + + ' not found in StateCell!\n' + ) read_var = self.read_array(init=init_var) update_dict[init_var_name] = read_var feed_var_expanded = layers.sequence_expand( - read_var, prev_scores) + read_var, prev_scores + ) feed_dict[init_var_name] = feed_var_expanded for state_str in self._state_cell._state_names: prev_state = self.state_cell.get_state(state_str) prev_state_expanded = layers.sequence_expand( - prev_state, prev_scores) + prev_state, prev_scores + ) self.state_cell.set_state(state_str, prev_state_expanded) for i, input_name in enumerate(self._state_cell._inputs): @@ -709,23 +739,29 @@ def decode(self): self.state_cell.compute_state(inputs=feed_dict) current_state = self.state_cell.out_state() - current_state_with_lod = layers.lod_reset(x=current_state, - y=prev_scores) - scores = layers.fc(input=current_state_with_lod, - size=self._target_dict_dim, - act='softmax') + current_state_with_lod = layers.lod_reset( + x=current_state, y=prev_scores + ) + scores = layers.fc( + input=current_state_with_lod, + size=self._target_dict_dim, + act='softmax', + ) topk_scores, topk_indices = layers.topk(scores, k=self._topk_size) - accu_scores = layers.elementwise_add(x=layers.log(x=topk_scores), - y=layers.reshape(prev_scores, - shape=[-1]), - axis=0) - selected_ids, selected_scores = layers.beam_search(prev_ids, - prev_scores, - topk_indices, - accu_scores, - self._beam_size, - end_id=1, - level=0) + accu_scores = layers.elementwise_add( + x=layers.log(x=topk_scores), + y=layers.reshape(prev_scores, shape=[-1]), + axis=0, + ) + selected_ids, selected_scores = layers.beam_search( + prev_ids, + prev_scores, + topk_indices, + accu_scores, + self._beam_size, + end_id=1, + level=0, + ) with layers.Switch() as switch: with switch.case(layers.is_empty(selected_ids)): @@ -760,8 +796,10 @@ def read_array(self, init, is_ids=False, is_scores=False): self._assert_in_decoder_block('read_array') if is_ids and is_scores: - raise ValueError('Shouldn\'t mark current array be ids array and' - 'scores array at the same time.') + raise ValueError( + 'Shouldn\'t mark current array be ids array and' + 'scores array at the same time.' + ) if not isinstance(init, Variable): raise TypeError('The input argument `init` must be a Variable.') @@ -770,13 +808,13 @@ def read_array(self, init, is_ids=False, is_scores=False): array = parent_block.create_var( name=unique_name.generate('beam_search_decoder_array'), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=init.dtype) - parent_block.append_op(type='write_to_array', - inputs={ - 'X': init, - 'I': self._zero_idx - }, - outputs={'Out': array}) + dtype=init.dtype, + ) + parent_block.append_op( + type='write_to_array', + inputs={'X': init, 'I': self._zero_idx}, + outputs={'Out': array}, + ) if is_ids: self._ids_array = array @@ -800,7 +838,8 @@ def update_array(self, array, value): if not isinstance(array, Variable): raise TypeError( - 'The input argument `array` of must be a Variable.') + 'The input argument `array` of must be a Variable.' + ) if not isinstance(value, Variable): raise TypeError('The input argument `value` of must be a Variable.') @@ -819,12 +858,16 @@ def __call__(self): as id, holds the score for each generated token. """ if self._status != BeamSearchDecoder.AFTER_BEAM_SEARCH_DECODER: - raise ValueError('Output of BeamSearchDecoder object can ' - 'only be visited outside the block.') - return layers.beam_search_decode(ids=self._ids_array, - scores=self._scores_array, - beam_size=self._beam_size, - end_id=self._end_id) + raise ValueError( + 'Output of BeamSearchDecoder object can ' + 'only be visited outside the block.' + ) + return layers.beam_search_decode( + ids=self._ids_array, + scores=self._scores_array, + beam_size=self._beam_size, + end_id=self._end_id, + ) @property def state_cell(self): @@ -847,5 +890,7 @@ def _parent_block(self): def _assert_in_decoder_block(self, method): if self._status != BeamSearchDecoder.IN_BEAM_SEARCH_DECODER: - raise ValueError('%s should be invoked inside block of ' - 'BeamSearchDecoder object.' % method) + raise ValueError( + '%s should be invoked inside block of ' + 'BeamSearchDecoder object.' % method + ) diff --git a/python/paddle/fluid/contrib/extend_optimizer/extend_optimizer_with_weight_decay.py b/python/paddle/fluid/contrib/extend_optimizer/extend_optimizer_with_weight_decay.py index 6a87bb54d3f8b..59b6ac388844e 100644 --- a/python/paddle/fluid/contrib/extend_optimizer/extend_optimizer_with_weight_decay.py +++ b/python/paddle/fluid/contrib/extend_optimizer/extend_optimizer_with_weight_decay.py @@ -18,10 +18,10 @@ class DecoupledWeightDecay(object): - def __init__(self, coeff=0.0, apply_decay_param_fun=None, **kwargs): - if not isinstance(coeff, float) and \ - not isinstance(coeff, framework.Variable): + if not isinstance(coeff, float) and not isinstance( + coeff, framework.Variable + ): raise TypeError("coeff should be float or Variable.") self._params_name = set() self._apply_decay_param_fun = apply_decay_param_fun @@ -47,19 +47,28 @@ def _scale_parameters(self, params_and_grads): # If no gradient then we don't need to do anything if grad is None: continue - if self._apply_decay_param_fun is not None \ - and not self._apply_decay_param_fun(param.name): + if ( + self._apply_decay_param_fun is not None + and not self._apply_decay_param_fun(param.name) + ): continue if isinstance(self._coeff, float): - assert param.dtype is not paddle.fluid.core.VarDesc.VarType.FP32, \ - "the type of coeff(float) and parameter(%s) is not consistent."%(self._coeff.dtype) + assert ( + param.dtype is not paddle.fluid.core.VarDesc.VarType.FP32 + ), ( + "the type of coeff(float) and parameter(%s) is not consistent." + % (self._coeff.dtype) + ) else: - assert self._coeff.dtype == param.dtype, \ - "the type of coeff(%s) and parameter(%s) is not consistent."%(self._coeff.dtype, param.dtype) + assert self._coeff.dtype == param.dtype, ( + "the type of coeff(%s) and parameter(%s) is not consistent." + % (self._coeff.dtype, param.dtype) + ) with param.block.program._optimized_guard( - [param, grad]), framework.name_scope('weight decay'): + [param, grad] + ), framework.name_scope('weight decay'): assert param.name not in self._params_name scaled_params.append((param, grad, param * self._coeff)) self._params_name.add(param.name) @@ -71,27 +80,31 @@ def backward(self, **kargs): def apply_optimize(self, **kargs): return super(DecoupledWeightDecay, self).apply_optimize(**kargs) - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): - params_grads = self.backward(loss=loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): + params_grads = self.backward( + loss=loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) scaled_params = self._scale_parameters(params_grads) for p_grad_sgrad in scaled_params: param, grad, scaled_param = p_grad_sgrad with param.block.program._optimized_guard( - [param, grad]), framework.name_scope('weight decay'): + [param, grad] + ), framework.name_scope('weight decay'): updated_param = paddle.fluid.layers.elementwise_sub( - x=param, y=scaled_param) + x=param, y=scaled_param + ) paddle.fluid.layers.assign(input=updated_param, output=param) - optimize_ops = self.apply_optimize(loss=loss, - params_grads=params_grads, - startup_program=startup_program) + optimize_ops = self.apply_optimize( + loss=loss, + params_grads=params_grads, + startup_program=startup_program, + ) return optimize_ops, params_grads def __str__(self): @@ -126,10 +139,12 @@ def extend_with_decoupled_weight_decay(base_optimizer): """ if not issubclass(base_optimizer, paddle.fluid.optimizer.Optimizer): raise TypeError( - "The input(base_optimizer) should be a derived class of Optimizer.") + "The input(base_optimizer) should be a derived class of Optimizer." + ) - class OptimizerWithDecoupledWeightDecay(DecoupledWeightDecay, - base_optimizer): + class OptimizerWithDecoupledWeightDecay( + DecoupledWeightDecay, base_optimizer + ): """ OptimizerWithDecoupledWeightDecay is used to update the optimized parameters with the parameters before optimization. For more information, please refer: @@ -145,7 +160,8 @@ class OptimizerWithDecoupledWeightDecay(DecoupledWeightDecay, """ def __init__(self, weight_decay, apply_decay_param_fun=None, **kwargs): - super(OptimizerWithDecoupledWeightDecay, - self).__init__(weight_decay, apply_decay_param_fun, **kwargs) + super(OptimizerWithDecoupledWeightDecay, self).__init__( + weight_decay, apply_decay_param_fun, **kwargs + ) return OptimizerWithDecoupledWeightDecay diff --git a/python/paddle/fluid/contrib/layers/metric_op.py b/python/paddle/fluid/contrib/layers/metric_op.py index 82d1a145f5b23..414fcf5b6cd51 100755 --- a/python/paddle/fluid/contrib/layers/metric_op.py +++ b/python/paddle/fluid/contrib/layers/metric_op.py @@ -78,171 +78,195 @@ def ctr_metric_bundle(input, label, ins_tag_weight=None): """ if ins_tag_weight is None: - ins_tag_weight = tensor.fill_constant(shape=[1, 1], - dtype="float32", - value=1.0) + ins_tag_weight = tensor.fill_constant( + shape=[1, 1], dtype="float32", value=1.0 + ) assert input.shape == label.shape helper = LayerHelper("ctr_metric_bundle", **locals()) - local_abserr = helper.create_global_variable(persistable=True, - dtype='float32', - shape=[1]) - local_sqrerr = helper.create_global_variable(persistable=True, - dtype='float32', - shape=[1]) - local_prob = helper.create_global_variable(persistable=True, - dtype='float32', - shape=[1]) - local_q = helper.create_global_variable(persistable=True, - dtype='float32', - shape=[1]) - local_pos_num = helper.create_global_variable(persistable=True, - dtype='float32', - shape=[1]) - local_ins_num = helper.create_global_variable(persistable=True, - dtype='float32', - shape=[1]) - - tmp_res_elesub = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[-1]) - tmp_res_sigmoid = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[-1]) - tmp_ones = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[-1]) - - batch_prob = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[1]) - batch_abserr = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[1]) - batch_sqrerr = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[1]) - batch_q = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[1]) - batch_pos_num = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[1]) - batch_ins_num = helper.create_global_variable(persistable=False, - dtype='float32', - shape=[1]) + local_abserr = helper.create_global_variable( + persistable=True, dtype='float32', shape=[1] + ) + local_sqrerr = helper.create_global_variable( + persistable=True, dtype='float32', shape=[1] + ) + local_prob = helper.create_global_variable( + persistable=True, dtype='float32', shape=[1] + ) + local_q = helper.create_global_variable( + persistable=True, dtype='float32', shape=[1] + ) + local_pos_num = helper.create_global_variable( + persistable=True, dtype='float32', shape=[1] + ) + local_ins_num = helper.create_global_variable( + persistable=True, dtype='float32', shape=[1] + ) + + tmp_res_elesub = helper.create_global_variable( + persistable=False, dtype='float32', shape=[-1] + ) + tmp_res_sigmoid = helper.create_global_variable( + persistable=False, dtype='float32', shape=[-1] + ) + tmp_ones = helper.create_global_variable( + persistable=False, dtype='float32', shape=[-1] + ) + + batch_prob = helper.create_global_variable( + persistable=False, dtype='float32', shape=[1] + ) + batch_abserr = helper.create_global_variable( + persistable=False, dtype='float32', shape=[1] + ) + batch_sqrerr = helper.create_global_variable( + persistable=False, dtype='float32', shape=[1] + ) + batch_q = helper.create_global_variable( + persistable=False, dtype='float32', shape=[1] + ) + batch_pos_num = helper.create_global_variable( + persistable=False, dtype='float32', shape=[1] + ) + batch_ins_num = helper.create_global_variable( + persistable=False, dtype='float32', shape=[1] + ) for var in [ - local_abserr, batch_abserr, local_sqrerr, batch_sqrerr, local_prob, - batch_prob, local_q, batch_q, batch_pos_num, batch_ins_num, - local_pos_num, local_ins_num + local_abserr, + batch_abserr, + local_sqrerr, + batch_sqrerr, + local_prob, + batch_prob, + local_q, + batch_q, + batch_pos_num, + batch_ins_num, + local_pos_num, + local_ins_num, ]: - helper.set_variable_initializer(var, Constant(value=0.0, - force_cpu=True)) - - helper.append_op(type="elementwise_sub", - inputs={ - "X": [input], - "Y": [label] - }, - outputs={"Out": [tmp_res_elesub]}) - - helper.append_op(type="squared_l2_norm", - inputs={"X": [tmp_res_elesub]}, - outputs={"Out": [batch_sqrerr]}) - helper.append_op(type="elementwise_add", - inputs={ - "X": [batch_sqrerr], - "Y": [local_sqrerr] - }, - outputs={"Out": [local_sqrerr]}) - - helper.append_op(type="l1_norm", - inputs={"X": [tmp_res_elesub]}, - outputs={"Out": [batch_abserr]}) - helper.append_op(type="elementwise_add", - inputs={ - "X": [batch_abserr], - "Y": [local_abserr] - }, - outputs={"Out": [local_abserr]}) - - helper.append_op(type="reduce_sum", - inputs={"X": [input]}, - outputs={"Out": [batch_prob]}) - helper.append_op(type="elementwise_add", - inputs={ - "X": [batch_prob], - "Y": [local_prob] - }, - outputs={"Out": [local_prob]}) - helper.append_op(type="sigmoid", - inputs={"X": [input]}, - outputs={"Out": [tmp_res_sigmoid]}) - helper.append_op(type="reduce_sum", - inputs={"X": [tmp_res_sigmoid]}, - outputs={"Out": [batch_q]}) - - helper.append_op(type="reduce_sum", - inputs={"X": [label]}, - outputs={"Out": [batch_pos_num]}) - helper.append_op(type="elementwise_add", - inputs={ - "X": [batch_pos_num], - "Y": [local_pos_num] - }, - outputs={"Out": [local_pos_num]}) - - helper.append_op(type='fill_constant_batch_size_like', - inputs={"Input": label}, - outputs={'Out': [tmp_ones]}, - attrs={ - 'shape': [-1, 1], - 'dtype': tmp_ones.dtype, - 'value': float(1.0), - }) - helper.append_op(type="reduce_sum", - inputs={"X": [tmp_ones]}, - outputs={"Out": [batch_ins_num]}) - - #if data is fake, return 0 + helper.set_variable_initializer( + var, Constant(value=0.0, force_cpu=True) + ) + + helper.append_op( + type="elementwise_sub", + inputs={"X": [input], "Y": [label]}, + outputs={"Out": [tmp_res_elesub]}, + ) + + helper.append_op( + type="squared_l2_norm", + inputs={"X": [tmp_res_elesub]}, + outputs={"Out": [batch_sqrerr]}, + ) + helper.append_op( + type="elementwise_add", + inputs={"X": [batch_sqrerr], "Y": [local_sqrerr]}, + outputs={"Out": [local_sqrerr]}, + ) + + helper.append_op( + type="l1_norm", + inputs={"X": [tmp_res_elesub]}, + outputs={"Out": [batch_abserr]}, + ) + helper.append_op( + type="elementwise_add", + inputs={"X": [batch_abserr], "Y": [local_abserr]}, + outputs={"Out": [local_abserr]}, + ) + + helper.append_op( + type="reduce_sum", inputs={"X": [input]}, outputs={"Out": [batch_prob]} + ) + helper.append_op( + type="elementwise_add", + inputs={"X": [batch_prob], "Y": [local_prob]}, + outputs={"Out": [local_prob]}, + ) + helper.append_op( + type="sigmoid", + inputs={"X": [input]}, + outputs={"Out": [tmp_res_sigmoid]}, + ) + helper.append_op( + type="reduce_sum", + inputs={"X": [tmp_res_sigmoid]}, + outputs={"Out": [batch_q]}, + ) + + helper.append_op( + type="reduce_sum", + inputs={"X": [label]}, + outputs={"Out": [batch_pos_num]}, + ) + helper.append_op( + type="elementwise_add", + inputs={"X": [batch_pos_num], "Y": [local_pos_num]}, + outputs={"Out": [local_pos_num]}, + ) + + helper.append_op( + type='fill_constant_batch_size_like', + inputs={"Input": label}, + outputs={'Out': [tmp_ones]}, + attrs={ + 'shape': [-1, 1], + 'dtype': tmp_ones.dtype, + 'value': float(1.0), + }, + ) + helper.append_op( + type="reduce_sum", + inputs={"X": [tmp_ones]}, + outputs={"Out": [batch_ins_num]}, + ) + + # if data is fake, return 0 inputs_slice = {'Input': ins_tag_weight} attrs = {'axes': [0]} attrs['starts'] = [0] attrs['ends'] = [1] - helper.append_op(type="slice", - inputs=inputs_slice, - attrs=attrs, - outputs={"Out": ins_tag_weight}) + helper.append_op( + type="slice", + inputs=inputs_slice, + attrs=attrs, + outputs={"Out": ins_tag_weight}, + ) axis = helper.kwargs.get('axis', 0) - helper.append_op(type="elementwise_mul", - inputs={ - "X": [batch_ins_num], - "Y": [ins_tag_weight] - }, - outputs={"Out": [batch_ins_num]}, - attrs={'axis': axis}) - - helper.append_op(type="elementwise_add", - inputs={ - "X": [batch_ins_num], - "Y": [local_ins_num] - }, - outputs={"Out": [local_ins_num]}) - - helper.append_op(type="elementwise_mul", - inputs={ - "X": [batch_q], - "Y": [ins_tag_weight] - }, - outputs={"Out": [batch_q]}, - attrs={'axis': axis}) - helper.append_op(type="elementwise_add", - inputs={ - "X": [batch_q], - "Y": [local_q] - }, - outputs={"Out": [local_q]}) - - return local_sqrerr, local_abserr, local_prob, local_q, local_pos_num, local_ins_num + helper.append_op( + type="elementwise_mul", + inputs={"X": [batch_ins_num], "Y": [ins_tag_weight]}, + outputs={"Out": [batch_ins_num]}, + attrs={'axis': axis}, + ) + + helper.append_op( + type="elementwise_add", + inputs={"X": [batch_ins_num], "Y": [local_ins_num]}, + outputs={"Out": [local_ins_num]}, + ) + + helper.append_op( + type="elementwise_mul", + inputs={"X": [batch_q], "Y": [ins_tag_weight]}, + outputs={"Out": [batch_q]}, + attrs={'axis': axis}, + ) + helper.append_op( + type="elementwise_add", + inputs={"X": [batch_q], "Y": [local_q]}, + outputs={"Out": [local_q]}, + ) + + return ( + local_sqrerr, + local_abserr, + local_prob, + local_q, + local_pos_num, + local_ins_num, + ) diff --git a/python/paddle/fluid/contrib/layers/nn.py b/python/paddle/fluid/contrib/layers/nn.py index e09bdcb0d8fbd..08ab46dfac9eb 100644 --- a/python/paddle/fluid/contrib/layers/nn.py +++ b/python/paddle/fluid/contrib/layers/nn.py @@ -25,7 +25,12 @@ from paddle.fluid.layers import utils from ... import unique_name from paddle.fluid.initializer import Normal, Constant, NumpyArrayInitializer -from paddle.fluid.data_feeder import check_variable_and_dtype, check_type, check_dtype, convert_dtype +from paddle.fluid.data_feeder import ( + check_variable_and_dtype, + check_type, + check_dtype, + convert_dtype, +) from paddle.fluid import core from paddle.fluid.param_attr import ParamAttr @@ -36,21 +41,33 @@ from paddle import _C_ops, _legacy_C_ops __all__ = [ - 'fused_elemwise_activation', 'sequence_topk_avg_pooling', 'var_conv_2d', - 'match_matrix_tensor', 'tree_conv', 'fused_embedding_seq_pool', - 'multiclass_nms2', 'search_pyramid_hash', 'shuffle_batch', 'partial_concat', - 'sparse_embedding', 'partial_sum', 'tdm_child', 'rank_attention', - 'tdm_sampler', 'batch_fc', '_pull_box_extended_sparse', 'bilateral_slice', - 'correlation', 'fused_bn_add_act', 'fused_seqpool_cvm' + 'fused_elemwise_activation', + 'sequence_topk_avg_pooling', + 'var_conv_2d', + 'match_matrix_tensor', + 'tree_conv', + 'fused_embedding_seq_pool', + 'multiclass_nms2', + 'search_pyramid_hash', + 'shuffle_batch', + 'partial_concat', + 'sparse_embedding', + 'partial_sum', + 'tdm_child', + 'rank_attention', + 'tdm_sampler', + 'batch_fc', + '_pull_box_extended_sparse', + 'bilateral_slice', + 'correlation', + 'fused_bn_add_act', + 'fused_seqpool_cvm', ] -def fused_elemwise_activation(x, - y, - functor_list, - axis=-1, - scale=0.0, - save_intermediate_out=True): +def fused_elemwise_activation( + x, y, functor_list, axis=-1, scale=0.0, save_intermediate_out=True +): """ **Fused elementwise_add/mul and activation layers** @@ -89,104 +106,103 @@ def fused_elemwise_activation(x, if not isinstance(functor_list, list) or len(functor_list) != 2: raise ValueError( - 'functor_list should be a list of str, and the length should be 2.') + 'functor_list should be a list of str, and the length should be 2.' + ) helper = LayerHelper('fused_elemwise_activation', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) intermediate_out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='fused_elemwise_activation', - inputs={ - 'X': x, - 'Y': y - }, - outputs={ - 'Out': out, - 'IntermediateOut': intermediate_out - }, - attrs={ - 'axis': axis, - 'scale': scale, - 'save_intermediate_out': save_intermediate_out, - 'functor_list': functor_list - }) + helper.append_op( + type='fused_elemwise_activation', + inputs={'X': x, 'Y': y}, + outputs={'Out': out, 'IntermediateOut': intermediate_out}, + attrs={ + 'axis': axis, + 'scale': scale, + 'save_intermediate_out': save_intermediate_out, + 'functor_list': functor_list, + }, + ) return out -def var_conv_2d(input, - row, - col, - input_channel, - output_channel, - filter_size, - stride=1, - param_attr=None, - act=None, - dtype='float32', - name=None): +def var_conv_2d( + input, + row, + col, + input_channel, + output_channel, + filter_size, + stride=1, + param_attr=None, + act=None, + dtype='float32', + name=None, +): r""" - The var_conv_2d layer calculates the output base on the :attr:`input` with variable length, - row, col, input channel, filter size and strides. Both :attr:`input`, :attr:`row`, - and :attr:`col` are 1-level LodTensor. The convolution operation is same as conv2d layer with - padding. Besides, input.dims[1] should be 1. - - .. code-block:: text - - If input_channel is 2 and given row lodTensor and col lodTensor as follows: - row.lod = [[5, 4]] - col.lod = [[6, 7]] - input is a lodTensor: - input.lod = [[60, 56]] # where 60 = input_channel * 5 * 6 - input.dims = [116, 1] # where 116 = 60 + 56 - - If set output_channel is 3, filter_size is [3, 3], stride is [1, 1]: - # where 90 = output_channel * [(5-1)/stride + 1] * [(6-1)/stride + 1] - output.lod = [[90, 84]] - output.dims = [174, 1] # where 174 = 90 + 84 - - Args: - input (Variable): The input should be 1-level LodTensor with dims[1] equals 1. - row (Variable): The row should be 1-level LodTensor to provide height information. - col (Variable): The col should be 1-level LodTensor to provide width information. - input_channel (int): The number of input channel. - output_channel (int): The number of output channel. - filter_size (int|tuple|None): The filter size. If filter_size is a tuple, - it must contain two integers, (filter_size_H, filter_size_W). - Otherwise, the filter will be a square. - stride (int|tuple): The stride size. If stride is a tuple, it must - contain two integers, (stride_H, stride_W). Otherwise, the - stride_H = stride_W = stride. Default: stride = 1. - param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights - of var_conv2d. If it is set to None or one attribute of ParamAttr, var_conv2d - will create ParamAttr as param_attr. If the Initializer of the param_attr - is not set, the parameter is initialized with :math:`Normal(0.0, std)`, - and the :math:`std` is :math:`(\\frac{2.0 }{filter\_elem\_num})^{ - 0.5}`. Default: None. - act (str): Activation type, if it is set to None, activation is not appended. - Default: None - dtype ('float32'): The data type of parameter and output. - name (str|None): A name for this layer(optional). If set None, the layer - will be named automatically. Default: None - - Returns: - Variable: Output variable with LoD specified by this layer. - - Examples: - .. code-block:: python - - import numpy as np - from paddle.fluid import layers - from paddle.fluid import contrib - - x_lod_tensor = layers.data(name='x', shape=[1], lod_level=1) - row_lod_tensor = layers.data(name='row', shape=[6], lod_level=1) - col_lod_tensor = layers.data(name='col', shape=[6], lod_level=1) - out = contrib.var_conv_2d(input=x_lod_tensor, - row=row_lod_tensor, - col=col_lod_tensor, - input_channel=3, - output_channel=5, - filter_size=[3, 3], - stride=1) + The var_conv_2d layer calculates the output base on the :attr:`input` with variable length, + row, col, input channel, filter size and strides. Both :attr:`input`, :attr:`row`, + and :attr:`col` are 1-level LodTensor. The convolution operation is same as conv2d layer with + padding. Besides, input.dims[1] should be 1. + + .. code-block:: text + + If input_channel is 2 and given row lodTensor and col lodTensor as follows: + row.lod = [[5, 4]] + col.lod = [[6, 7]] + input is a lodTensor: + input.lod = [[60, 56]] # where 60 = input_channel * 5 * 6 + input.dims = [116, 1] # where 116 = 60 + 56 + + If set output_channel is 3, filter_size is [3, 3], stride is [1, 1]: + # where 90 = output_channel * [(5-1)/stride + 1] * [(6-1)/stride + 1] + output.lod = [[90, 84]] + output.dims = [174, 1] # where 174 = 90 + 84 + + Args: + input (Variable): The input should be 1-level LodTensor with dims[1] equals 1. + row (Variable): The row should be 1-level LodTensor to provide height information. + col (Variable): The col should be 1-level LodTensor to provide width information. + input_channel (int): The number of input channel. + output_channel (int): The number of output channel. + filter_size (int|tuple|None): The filter size. If filter_size is a tuple, + it must contain two integers, (filter_size_H, filter_size_W). + Otherwise, the filter will be a square. + stride (int|tuple): The stride size. If stride is a tuple, it must + contain two integers, (stride_H, stride_W). Otherwise, the + stride_H = stride_W = stride. Default: stride = 1. + param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights + of var_conv2d. If it is set to None or one attribute of ParamAttr, var_conv2d + will create ParamAttr as param_attr. If the Initializer of the param_attr + is not set, the parameter is initialized with :math:`Normal(0.0, std)`, + and the :math:`std` is :math:`(\\frac{2.0 }{filter\_elem\_num})^{ + 0.5}`. Default: None. + act (str): Activation type, if it is set to None, activation is not appended. + Default: None + dtype ('float32'): The data type of parameter and output. + name (str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Default: None + + Returns: + Variable: Output variable with LoD specified by this layer. + + Examples: + .. code-block:: python + + import numpy as np + from paddle.fluid import layers + from paddle.fluid import contrib + + x_lod_tensor = layers.data(name='x', shape=[1], lod_level=1) + row_lod_tensor = layers.data(name='row', shape=[6], lod_level=1) + col_lod_tensor = layers.data(name='col', shape=[6], lod_level=1) + out = contrib.var_conv_2d(input=x_lod_tensor, + row=row_lod_tensor, + col=col_lod_tensor, + input_channel=3, + output_channel=5, + filter_size=[3, 3], + stride=1) """ helper = LayerHelper('var_conv_2d', **locals()) x_shape = list(input.shape) @@ -197,7 +213,7 @@ def var_conv_2d(input, filter_shape = [ int(output_channel), - int(input_channel) * filter_size[0] * filter_size[1] + int(input_channel) * filter_size[0] * filter_size[1], ] filter_param = helper.create_parameter( attr=helper.param_attr, @@ -206,39 +222,35 @@ def var_conv_2d(input, ) conv_res = helper.create_variable_for_type_inference(dtype) - tmp_res = helper.create_variable_for_type_inference(dtype, - stop_gradient=True) - - helper.append_op(type='var_conv_2d', - inputs={ - 'X': input, - 'ROW': row, - 'COLUMN': col, - 'W': filter_param, - }, - outputs={ - "Out": conv_res, - "Col": tmp_res - }, - attrs={ - 'InputChannel': input_channel, - 'OutputChannel': output_channel, - 'StrideH': stride[0], - 'StrideW': stride[1], - 'KernelH': filter_size[0], - 'KernelW': filter_size[1], - }) + tmp_res = helper.create_variable_for_type_inference( + dtype, stop_gradient=True + ) + + helper.append_op( + type='var_conv_2d', + inputs={ + 'X': input, + 'ROW': row, + 'COLUMN': col, + 'W': filter_param, + }, + outputs={"Out": conv_res, "Col": tmp_res}, + attrs={ + 'InputChannel': input_channel, + 'OutputChannel': output_channel, + 'StrideH': stride[0], + 'StrideW': stride[1], + 'KernelH': filter_size[0], + 'KernelW': filter_size[1], + }, + ) return helper.append_activation(conv_res) -def match_matrix_tensor(x, - y, - channel_num, - act=None, - param_attr=None, - dtype='float32', - name=None): +def match_matrix_tensor( + x, y, channel_num, act=None, param_attr=None, dtype='float32', name=None +): """ Calculate the semantic matching matrix of two word sequences with variable length. Given a query A of length `n` and a title B of length `m`, the input shape are respectively @@ -296,28 +308,28 @@ def match_matrix_tensor(x, x_shape = list(x.shape) y_shape = list(y.shape) - assert len(x_shape) == 2 and len( - y_shape) == 2 and x_shape[-1] == y_shape[-1] + assert ( + len(x_shape) == 2 and len(y_shape) == 2 and x_shape[-1] == y_shape[-1] + ) weight_shape = [x_shape[-1], channel_num, y_shape[-1]] - w = helper.create_parameter(attr=helper.param_attr, - shape=weight_shape, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=helper.param_attr, shape=weight_shape, dtype=dtype, is_bias=False + ) mm_res = helper.create_variable_for_type_inference(dtype) - tmp_res = helper.create_variable_for_type_inference(dtype, - stop_gradient=True) - helper.append_op(type='match_matrix_tensor', - inputs={ - 'X': x, - 'Y': y, - 'W': w, - }, - outputs={ - "Out": mm_res, - "Tmp": tmp_res - }, - attrs={'dim_t': channel_num}) + tmp_res = helper.create_variable_for_type_inference( + dtype, stop_gradient=True + ) + helper.append_op( + type='match_matrix_tensor', + inputs={ + 'X': x, + 'Y': y, + 'W': w, + }, + outputs={"Out": mm_res, "Tmp": tmp_res}, + attrs={'dim_t': channel_num}, + ) return helper.append_activation(mm_res), tmp_res @@ -375,72 +387,67 @@ def sequence_topk_avg_pooling(input, row, col, topks, channel_num): """ helper = LayerHelper('sequence_topk_avg_pooling', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - pos = helper.create_variable_for_type_inference(dtype=helper.input_dtype(), - stop_gradient=True) - helper.append_op(type='sequence_topk_avg_pooling', - inputs={ - 'X': input, - 'ROW': row, - 'COLUMN': col - }, - outputs={ - 'Out': out, - 'pos': pos - }, - attrs={ - 'topks': topks, - 'channel_num': channel_num - }) + pos = helper.create_variable_for_type_inference( + dtype=helper.input_dtype(), stop_gradient=True + ) + helper.append_op( + type='sequence_topk_avg_pooling', + inputs={'X': input, 'ROW': row, 'COLUMN': col}, + outputs={'Out': out, 'pos': pos}, + attrs={'topks': topks, 'channel_num': channel_num}, + ) return out -def tree_conv(nodes_vector, - edge_set, - output_size, - num_filters=1, - max_depth=2, - act='tanh', - param_attr=None, - bias_attr=None, - name=None): +def tree_conv( + nodes_vector, + edge_set, + output_size, + num_filters=1, + max_depth=2, + act='tanh', + param_attr=None, + bias_attr=None, + name=None, +): """ - ${comment} -Args : nodes_vector(${nodes_vector_type}) : $ { nodes_vector_comment } -edge_set(${edge_set_type}) : $ { edge_set_comment } - output_size(int): output feature width - num_filters(int): number of filters, Default 1 - max_depth(int): max depth of filters, Default 2 - act(str): activation function, Default tanh - param_attr(ParamAttr): the parameter attribute for the filters, Default None - bias_attr(ParamAttr): the parameter attribute for the bias of this layer, Default None - name(str): a name of this layer(optional). If set None, the layer will be named automatically, Default None - - Returns: - out(${out_type}): ${ - out_comment - } - - Examples: - .. code-block:: python - - import paddle.fluid as fluid + ${comment} + Args : nodes_vector(${nodes_vector_type}) : $ { nodes_vector_comment } + edge_set(${edge_set_type}) : $ { edge_set_comment } + output_size(int): output feature width + num_filters(int): number of filters, Default 1 + max_depth(int): max depth of filters, Default 2 + act(str): activation function, Default tanh + param_attr(ParamAttr): the parameter attribute for the filters, Default None + bias_attr(ParamAttr): the parameter attribute for the bias of this layer, Default None + name(str): a name of this layer(optional). If set None, the layer will be named automatically, Default None + + Returns: + out(${out_type}): ${ + out_comment + } + + Examples: + .. code-block:: python - # 10 for max_node_size of dataset, 5 for vector width - nodes_vector = fluid.layers.data( - name='vectors', shape=[10, 5], dtype='float32') - # 10 for max_node_size of dataset, 2 for every edge has two nodes - # edges must be directional - edge_set = fluid.layers.data(name='edge_set', shape=[ - 10, 2], dtype='float32') - # the shape of output will be [10, 6, 1], - # 10 for max_node_size of dataset, 6 for output size, 1 for 1 filter - out_vector = fluid.layers.tree_conv(nodes_vector, edge_set, 6, 1, 2) -#After reshape, output tensor could be nodes_vector for next tree convolution - out_vector = fluid.layers.reshape(out_vector, shape=[-1, 10, 6]) - out_vector_2 = fluid.layers.tree_conv(out_vector, edge_set, 3, 4, 2) -#also output tensor could be pooling(the pooling in paper called global pooling) - pooled = fluid.layers.reduce_max(out_vector, dim=2) # global pooling + import paddle.fluid as fluid + + # 10 for max_node_size of dataset, 5 for vector width + nodes_vector = fluid.layers.data( + name='vectors', shape=[10, 5], dtype='float32') + # 10 for max_node_size of dataset, 2 for every edge has two nodes + # edges must be directional + edge_set = fluid.layers.data(name='edge_set', shape=[ + 10, 2], dtype='float32') + # the shape of output will be [10, 6, 1], + # 10 for max_node_size of dataset, 6 for output size, 1 for 1 filter + out_vector = fluid.layers.tree_conv(nodes_vector, edge_set, 6, 1, 2) + #After reshape, output tensor could be nodes_vector for next tree convolution + out_vector = fluid.layers.reshape(out_vector, shape=[-1, 10, 6]) + out_vector_2 = fluid.layers.tree_conv(out_vector, edge_set, 3, 4, 2) + #also output tensor could be pooling(the pooling in paper called global pooling) + pooled = fluid.layers.reduce_max(out_vector, dim=2) # global pooling """ check_type(nodes_vector, 'nodes_vector', (Variable), 'tree_conv') check_type(edge_set, 'edge_set', (Variable), 'tree_conv') @@ -449,21 +456,18 @@ def tree_conv(nodes_vector, dtype = helper.input_dtype('nodes_vector') feature_size = nodes_vector.shape[2] W_shape = [feature_size, 3, output_size, num_filters] - W = helper.create_parameter(attr=param_attr, - shape=W_shape, - dtype=dtype, - is_bias=False) + W = helper.create_parameter( + attr=param_attr, shape=W_shape, dtype=dtype, is_bias=False + ) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='tree_conv', - inputs={ - 'NodesVector': nodes_vector, - 'EdgeSet': edge_set, - 'Filter': W - }, - outputs={ - 'Out': out, - }, - attrs={'max_depth': max_depth}) + helper.append_op( + type='tree_conv', + inputs={'NodesVector': nodes_vector, 'EdgeSet': edge_set, 'Filter': W}, + outputs={ + 'Out': out, + }, + attrs={'max_depth': max_depth}, + ) if helper.bias_attr: pre_activation = helper.append_bias_op(out) else: @@ -471,13 +475,15 @@ def tree_conv(nodes_vector, return helper.append_activation(pre_activation) -def fused_embedding_seq_pool(input, - size, - is_sparse=False, - padding_idx=None, - combiner='sum', - param_attr=None, - dtype='float32'): +def fused_embedding_seq_pool( + input, + size, + is_sparse=False, + padding_idx=None, + combiner='sum', + param_attr=None, + dtype='float32', +): r""" **Embedding Sequence pool** @@ -520,33 +526,33 @@ def fused_embedding_seq_pool(input, is_sparse=False) """ helper = LayerHelper('fused_embedding_seq_pool', **locals()) - w = helper.create_parameter(attr=helper.param_attr, - shape=size, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=helper.param_attr, shape=size, dtype=dtype, is_bias=False + ) out = helper.create_variable_for_type_inference(dtype) - padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( - size[0] + padding_idx) - helper.append_op(type='fused_embedding_seq_pool', - inputs={ - 'Ids': input, - 'W': w - }, - outputs={'Out': out}, - attrs={ - 'is_sparse': is_sparse, - 'combiner': combiner, - 'padding_idx': padding_idx - }) + padding_idx = ( + -1 + if padding_idx is None + else padding_idx + if padding_idx >= 0 + else (size[0] + padding_idx) + ) + helper.append_op( + type='fused_embedding_seq_pool', + inputs={'Ids': input, 'W': w}, + outputs={'Out': out}, + attrs={ + 'is_sparse': is_sparse, + 'combiner': combiner, + 'padding_idx': padding_idx, + }, + ) return out -def fused_seqpool_cvm(input, - pool_type, - cvm, - pad_value=0.0, - use_cvm=True, - cvm_offset=2): +def fused_seqpool_cvm( + input, pool_type, cvm, pad_value=0.0, use_cvm=True, cvm_offset=2 +): """ :api_attr: Static Graph @@ -592,13 +598,15 @@ def fused_seqpool_cvm(input, if pool_type.upper() != 'SUM': raise ValueError( "fused_seqpool_cvm only support SUM pooling now, and your type is: " - + pool_type) + + pool_type + ) check_type(input, 'input', list, 'fused_seqpool_cvm') if isinstance(input, list): for _input in input: - check_variable_and_dtype(_input, 'input', ['float32'], - 'fused_seqpool_cvm') + check_variable_and_dtype( + _input, 'input', ['float32'], 'fused_seqpool_cvm' + ) dtype = helper.input_dtype() inputs = helper.multiple_input() @@ -607,33 +615,34 @@ def fused_seqpool_cvm(input, for i in range(len(inputs)) ] - helper.append_op(type="fused_seqpool_cvm", - inputs={ - "X": inputs, - "CVM": cvm - }, - outputs={"Out": outs}, - attrs={ - "pooltype": pool_type.upper(), - "pad_value": pad_value, - "use_cvm": use_cvm, - "cvm_offset": cvm_offset, - }) + helper.append_op( + type="fused_seqpool_cvm", + inputs={"X": inputs, "CVM": cvm}, + outputs={"Out": outs}, + attrs={ + "pooltype": pool_type.upper(), + "pad_value": pad_value, + "use_cvm": use_cvm, + "cvm_offset": cvm_offset, + }, + ) return outs -def multiclass_nms2(bboxes, - scores, - score_threshold, - nms_top_k, - keep_top_k, - nms_threshold=0.3, - normalized=True, - nms_eta=1., - background_label=0, - return_index=False, - name=None): +def multiclass_nms2( + bboxes, + scores, + score_threshold, + nms_top_k, + keep_top_k, + nms_threshold=0.3, + normalized=True, + nms_eta=1.0, + background_label=0, + return_index=False, + name=None, +): """ **Multiclass NMS2** @@ -729,24 +738,20 @@ class number output = helper.create_variable_for_type_inference(dtype=bboxes.dtype) index = helper.create_variable_for_type_inference(dtype='int') - helper.append_op(type="multiclass_nms2", - inputs={ - 'BBoxes': bboxes, - 'Scores': scores - }, - attrs={ - 'background_label': background_label, - 'score_threshold': score_threshold, - 'nms_top_k': nms_top_k, - 'nms_threshold': nms_threshold, - 'keep_top_k': keep_top_k, - 'nms_eta': nms_eta, - 'normalized': normalized - }, - outputs={ - 'Out': output, - 'Index': index - }) + helper.append_op( + type="multiclass_nms2", + inputs={'BBoxes': bboxes, 'Scores': scores}, + attrs={ + 'background_label': background_label, + 'score_threshold': score_threshold, + 'nms_top_k': nms_top_k, + 'nms_threshold': nms_threshold, + 'keep_top_k': keep_top_k, + 'nms_eta': nms_eta, + 'normalized': normalized, + }, + outputs={'Out': output, 'Index': index}, + ) output.stop_gradient = True index.stop_gradient = True @@ -755,24 +760,26 @@ class number return output -def search_pyramid_hash(input, - num_emb, - space_len, - pyramid_layer, - rand_len, - drop_out_percent, - is_training, - use_filter, - white_list_len, - black_list_len, - seed, - lr, - param_attr=None, - param_attr_wl=None, - param_attr_bl=None, - name=None, - distribute_update_vars=None, - dtype='float32'): +def search_pyramid_hash( + input, + num_emb, + space_len, + pyramid_layer, + rand_len, + drop_out_percent, + is_training, + use_filter, + white_list_len, + black_list_len, + seed, + lr, + param_attr=None, + param_attr_wl=None, + param_attr_bl=None, + name=None, + distribute_update_vars=None, + dtype='float32', +): """ **Pyramid hash embedding** @@ -809,28 +816,25 @@ def search_pyramid_hash(input, helper = LayerHelper('search_pyramid_hash', **locals()) w_shape = [space_len + rand_len, 1] - w = helper.create_parameter(attr=param_attr, - shape=w_shape, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=param_attr, shape=w_shape, dtype=dtype, is_bias=False + ) w.stop_gradient = True input_vars = {'X': input, 'W': w} if white_list_len > 0: wl_shape = [white_list_len, 1] - white_list = helper.create_parameter(attr=param_attr_wl, - shape=wl_shape, - dtype=dtype, - is_bias=False) + white_list = helper.create_parameter( + attr=param_attr_wl, shape=wl_shape, dtype=dtype, is_bias=False + ) white_list.stop_gradient = True input_vars['WhiteList'] = white_list if black_list_len >= 0: bl_shape = [black_list_len, 1] - black_list = helper.create_parameter(attr=param_attr_bl, - shape=bl_shape, - dtype=dtype, - is_bias=False) + black_list = helper.create_parameter( + attr=param_attr_bl, shape=bl_shape, dtype=dtype, is_bias=False + ) black_list.stop_gradient = True input_vars['BlackList'] = black_list @@ -847,33 +851,32 @@ def search_pyramid_hash(input, for param in distribute_update_vars: if param not in special_name_list: raise ValueError( - "Pyramid Hash layer didn't have parameter {}".format(param)) + "Pyramid Hash layer didn't have parameter {}".format(param) + ) distribute_update_vars_str = ",".join(distribute_update_vars) res = helper.create_variable_for_type_inference(dtype) drop_pos = helper.create_variable_for_type_inference(dtype) x_temp_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='pyramid_hash', - inputs=input_vars, - outputs={ - "Out": res, - "X_Temp_Out": x_temp_out, - 'DropPos': drop_pos - }, - attrs={ - 'num_emb': num_emb, - 'space_len': space_len, - 'pyramid_layer': pyramid_layer, - 'rand_len': rand_len, - 'drop_out_percent': drop_out_percent, - 'is_training': is_training, - 'use_filter': use_filter, - 'white_list_len': white_list_len, - 'black_list_len': black_list_len, - 'seed': seed, - 'lr': lr, - 'distribute_update_vars': distribute_update_vars_str - }) + helper.append_op( + type='pyramid_hash', + inputs=input_vars, + outputs={"Out": res, "X_Temp_Out": x_temp_out, 'DropPos': drop_pos}, + attrs={ + 'num_emb': num_emb, + 'space_len': space_len, + 'pyramid_layer': pyramid_layer, + 'rand_len': rand_len, + 'drop_out_percent': drop_out_percent, + 'is_training': is_training, + 'use_filter': use_filter, + 'white_list_len': white_list_len, + 'black_list_len': black_list_len, + 'seed': seed, + 'lr': lr, + 'distribute_update_vars': distribute_update_vars_str, + }, + ) return res @@ -930,18 +933,14 @@ def shuffle_batch(x, seed=None): seed = helper.create_variable( name=unique_name.generate("shuffle_batch_seed"), dtype="int64", - persistable=False) - helper.append_op(type='shuffle_batch', - inputs={ - 'X': x, - 'Seed': seed - }, - outputs={ - 'Out': out, - 'ShuffleIdx': shuffle_idx, - 'SeedOut': seed - }, - attrs=op_attrs) + persistable=False, + ) + helper.append_op( + type='shuffle_batch', + inputs={'X': x, 'Seed': seed}, + outputs={'Out': out, 'ShuffleIdx': shuffle_idx, 'SeedOut': seed}, + attrs=op_attrs, + ) return out @@ -987,23 +986,28 @@ def partial_concat(input, start_index=0, length=-1): if not isinstance(input, list): warnings.warn( "The type of input in partial_concat should be list, but received %s." - % (type(input))) + % (type(input)) + ) input = [input] for id, x in enumerate(input): check_variable_and_dtype( - x, 'input[' + str(id) + ']', + x, + 'input[' + str(id) + ']', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'partial_concat') + 'partial_concat', + ) check_type(start_index, 'start_index', (int), 'partial_concat') check_type(length, 'length', (int), 'partial_concat') inputs = {'X': input} attrs = {'start_index': start_index, 'length': length} helper = LayerHelper('partial_concat', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='partial_concat', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs) + helper.append_op( + type='partial_concat', + inputs=inputs, + outputs={'Out': [out]}, + attrs=attrs, + ) return out @@ -1046,9 +1050,12 @@ def partial_sum(input, start_index=0, length=-1): out = exe.run(feed={"x":xx, "y":yy}, fetch_list=[sum]) """ for id, x in enumerate(input): - check_variable_and_dtype(x, 'input[' + str(id) + ']', - ['float32', 'float64', 'int32', 'int64'], - 'partial_sum') + check_variable_and_dtype( + x, + 'input[' + str(id) + ']', + ['float32', 'float64', 'int32', 'int64'], + 'partial_sum', + ) inputs = {'X': input} attrs = {} @@ -1056,22 +1063,23 @@ def partial_sum(input, start_index=0, length=-1): attrs['length'] = length helper = LayerHelper('partial_sum', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='partial_sum', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs) + helper.append_op( + type='partial_sum', inputs=inputs, outputs={'Out': [out]}, attrs=attrs + ) return out -def sparse_embedding(input, - size, - padding_idx=None, - is_test=False, - entry=None, - table_class="MemorySparseTable", - param_attr=None, - dtype='float32', - slot=None): +def sparse_embedding( + input, + size, + padding_idx=None, + is_test=False, + entry=None, + table_class="MemorySparseTable", + param_attr=None, + dtype='float32', + slot=None, +): r""" :api_attr: Static Graph @@ -1183,25 +1191,39 @@ def sparse_embedding(input, helper = LayerHelper('sparse_embedding', **locals()) - check_variable_and_dtype(input, 'input', ['int64'], - 'fluid.contrib.layers.sparse_embedding') + check_variable_and_dtype( + input, 'input', ['int64'], 'fluid.contrib.layers.sparse_embedding' + ) - check_dtype(dtype, 'dtype', ['float32', 'float64'], - 'paddle.static.nn.sparse_embedding') + check_dtype( + dtype, + 'dtype', + ['float32', 'float64'], + 'paddle.static.nn.sparse_embedding', + ) - w = helper.create_parameter(attr=helper.param_attr, - shape=size, - type=core.VarDesc.VarType.SELECTED_ROWS, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=helper.param_attr, + shape=size, + type=core.VarDesc.VarType.SELECTED_ROWS, + dtype=dtype, + is_bias=False, + ) tmp = helper.create_variable_for_type_inference(dtype) - padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( - size[0] + padding_idx) + padding_idx = ( + -1 + if padding_idx is None + else padding_idx + if padding_idx >= 0 + else (size[0] + padding_idx) + ) if table_class not in [ - "CommonSparseTable", "SSDSparseTable", "MemorySparseTable" + "CommonSparseTable", + "SSDSparseTable", + "MemorySparseTable", ]: raise ValueError( "table_class must be in [CommonSparseTable, SSDSparseTable, MemorySparseTable]" @@ -1211,7 +1233,9 @@ def sparse_embedding(input, if entry is not None: if entry.__class__.__name__ not in [ - "ProbabilityEntry", "CountFilterEntry", "ShowClickEntry" + "ProbabilityEntry", + "CountFilterEntry", + "ShowClickEntry", ]: raise ValueError( "entry must be instance in [paddle.distributed.ProbabilityEntry, paddle.distributed.CountFilterEntry, paddle.distributed.ShowClickEntry]" @@ -1221,22 +1245,21 @@ def sparse_embedding(input, if slot == None: slot = 0 - helper.append_op(type='lookup_table', - inputs={ - 'Ids': input, - 'W': w - }, - outputs={'Out': tmp}, - attrs={ - 'padding_idx': padding_idx, - 'is_sparse': True, - 'is_distributed': True, - 'remote_prefetch': True, - 'is_test': is_test, - 'entry': entry_str, - 'table_class': table_class, - 'slot': slot - }) + helper.append_op( + type='lookup_table', + inputs={'Ids': input, 'W': w}, + outputs={'Out': tmp}, + attrs={ + 'padding_idx': padding_idx, + 'is_sparse': True, + 'is_distributed': True, + 'remote_prefetch': True, + 'is_test': is_test, + 'entry': entry_str, + 'table_class': table_class, + 'slot': slot, + }, + ) return tmp @@ -1298,48 +1321,46 @@ def tdm_child(x, node_nums, child_nums, param_attr=None, dtype='int32'): exe.run(fluid.default_startup_program()) xx = np.array([[2],[3]]).reshape((2,1)).astype("int32") child_res, leaf_mask_res = exe.run(feed={"x":xx}, fetch_list=[child, leaf_mask]) - """ + """ helper = LayerHelper("tdm_child", **locals()) - check_dtype(dtype, 'dtype', ['int32', 'int64'], - 'fluid.contrib.layers.tdm_child') + check_dtype( + dtype, 'dtype', ['int32', 'int64'], 'fluid.contrib.layers.tdm_child' + ) c_dtype = convert_np_dtype_to_dtype_(dtype) - tree_info = helper.create_parameter(attr=helper.param_attr, - shape=[node_nums, 3 + child_nums], - dtype=dtype, - default_initializer=Constant(0)) + tree_info = helper.create_parameter( + attr=helper.param_attr, + shape=[node_nums, 3 + child_nums], + dtype=dtype, + default_initializer=Constant(0), + ) tree_info.stop_gradient = True child = helper.create_variable_for_type_inference(dtype=dtype) leaf_mask = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='tdm_child', - inputs={ - 'X': x, - 'TreeInfo': tree_info - }, - outputs={ - 'Child': child, - 'LeafMask': leaf_mask - }, - attrs={ - 'child_nums': child_nums, - 'dtype': c_dtype - }, - stop_gradient=True) + helper.append_op( + type='tdm_child', + inputs={'X': x, 'TreeInfo': tree_info}, + outputs={'Child': child, 'LeafMask': leaf_mask}, + attrs={'child_nums': child_nums, 'dtype': c_dtype}, + stop_gradient=True, + ) return (child, leaf_mask) -def tdm_sampler(x, - neg_samples_num_list, - layer_node_num_list, - leaf_node_num, - tree_travel_attr=None, - tree_layer_attr=None, - output_positive=True, - output_list=True, - seed=0, - tree_dtype='int32', - dtype='int32'): +def tdm_sampler( + x, + neg_samples_num_list, + layer_node_num_list, + leaf_node_num, + tree_travel_attr=None, + tree_layer_attr=None, + output_positive=True, + output_list=True, + seed=0, + tree_dtype='int32', + dtype='int32', +): """ **Tdm Sampler** According to the input positive samples at leaf node(x), do negative sampling layer by layer on the given tree. @@ -1426,18 +1447,25 @@ def tdm_sampler(x, """ helper = LayerHelper("tdm_sampler", **locals()) - check_dtype(tree_dtype, 'tree_dtype', ['int32', 'int64'], - 'fluid.contrib.layers.tdm_sampler') - check_dtype(dtype, 'dtype', ['int32', 'int64'], - 'fluid.contrib.layers.tdm_sampler') + check_dtype( + tree_dtype, + 'tree_dtype', + ['int32', 'int64'], + 'fluid.contrib.layers.tdm_sampler', + ) + check_dtype( + dtype, 'dtype', ['int32', 'int64'], 'fluid.contrib.layers.tdm_sampler' + ) c_dtype = convert_np_dtype_to_dtype_(dtype) if len(neg_samples_num_list) != len(layer_node_num_list): raise ValueError( "The shape of negative samples list must match the shape of layers. " "But received len of neg_samples_num_list: {}," - "and len of layer_node_num_list: {}, please check your input.". - format(len(neg_samples_num_list), len(layer_node_num_list))) + "and len of layer_node_num_list: {}, please check your input.".format( + len(neg_samples_num_list), len(layer_node_num_list) + ) + ) assert leaf_node_num is not None, "leaf_node_num should not be None here." layer_nums = 0 @@ -1452,21 +1480,31 @@ def tdm_sampler(x, "The number of negative samples must be less than the number of nodes " "in the layer {}, But received negative nums {}, and num of node at layer {} " "is {}, please check your input.".format( - layer_idx, neg_samples_num_list[layer_idx], layer_idx, - layer_node_num_list[layer_idx])) - assert leaf_node_num < node_nums, "leaf_node_num must be less than total node nums." + layer_idx, + neg_samples_num_list[layer_idx], + layer_idx, + layer_node_num_list[layer_idx], + ) + ) + assert ( + leaf_node_num < node_nums + ), "leaf_node_num must be less than total node nums." travel_shape = [leaf_node_num, layer_nums] - travel = helper.create_parameter(attr=tree_travel_attr, - shape=travel_shape, - dtype=tree_dtype, - default_initializer=Constant(0)) + travel = helper.create_parameter( + attr=tree_travel_attr, + shape=travel_shape, + dtype=tree_dtype, + default_initializer=Constant(0), + ) layer_shape = [node_nums, 1] - layer = helper.create_parameter(attr=tree_layer_attr, - shape=layer_shape, - dtype=tree_dtype, - default_initializer=Constant(0)) + layer = helper.create_parameter( + attr=tree_layer_attr, + shape=layer_shape, + dtype=tree_dtype, + default_initializer=Constant(0), + ) out = helper.create_variable_for_type_inference(dtype=dtype) out.stop_gradient = True @@ -1477,24 +1515,18 @@ def tdm_sampler(x, mask = helper.create_variable_for_type_inference(dtype=dtype) mask.stop_gradient = True - helper.append_op(type='tdm_sampler', - inputs={ - "X": x, - "Travel": travel, - "Layer": layer - }, - outputs={ - 'Out': out, - 'Labels': labels, - 'Mask': mask - }, - attrs={ - 'neg_samples_num_list': neg_samples_num_list, - 'output_positive': output_positive, - 'layer_offset_lod': tree_layer_offset_lod, - 'seed': seed, - 'dtype': c_dtype - }) + helper.append_op( + type='tdm_sampler', + inputs={"X": x, "Travel": travel, "Layer": layer}, + outputs={'Out': out, 'Labels': labels, 'Mask': mask}, + attrs={ + 'neg_samples_num_list': neg_samples_num_list, + 'output_positive': output_positive, + 'layer_offset_lod': tree_layer_offset_lod, + 'seed': seed, + 'dtype': c_dtype, + }, + ) if output_list: output_list = [] @@ -1506,31 +1538,30 @@ def tdm_sampler(x, positive_flag = 0 for layer_sample_num in neg_samples_num_list: - end_offset = start_offset + \ - layer_sample_num + positive_flag - layer_samples = slice(out, - axes=[1], - starts=[start_offset], - ends=[end_offset]) - layer_labels = slice(labels, - axes=[1], - starts=[start_offset], - ends=[end_offset]) - layer_mask = slice(mask, - axes=[1], - starts=[start_offset], - ends=[end_offset]) - - layer_samples = reshape(layer_samples, - [-1, layer_sample_num + positive_flag, 1]) + end_offset = start_offset + layer_sample_num + positive_flag + layer_samples = slice( + out, axes=[1], starts=[start_offset], ends=[end_offset] + ) + layer_labels = slice( + labels, axes=[1], starts=[start_offset], ends=[end_offset] + ) + layer_mask = slice( + mask, axes=[1], starts=[start_offset], ends=[end_offset] + ) + + layer_samples = reshape( + layer_samples, [-1, layer_sample_num + positive_flag, 1] + ) layer_samples.stop_gradient = True - layer_labels = reshape(layer_labels, - [-1, layer_sample_num + positive_flag, 1]) + layer_labels = reshape( + layer_labels, [-1, layer_sample_num + positive_flag, 1] + ) layer_labels.stop_gradient = True - layer_mask = reshape(layer_mask, - [-1, layer_sample_num + positive_flag, 1]) + layer_mask = reshape( + layer_mask, [-1, layer_sample_num + positive_flag, 1] + ) layer_mask.stop_gradient = True output_list.append(layer_samples) @@ -1545,12 +1576,14 @@ def tdm_sampler(x, return (out, labels, mask) -def rank_attention(input, - rank_offset, - rank_param_shape, - rank_param_attr, - max_rank=3, - max_size=0): +def rank_attention( + input, + rank_offset, + rank_param_shape, + rank_param_attr, + max_rank=3, + max_size=0, +): """ **Rank Attention layer** This Op can calculate rank attention between input and rank_param, and @@ -1588,32 +1621,25 @@ def rank_attention(input, input_shape = input.shape assert input_shape[1] * max_rank * max_rank == rank_param_shape[0] - rank_param = helper.create_parameter(attr=rank_param_attr, - shape=rank_param_shape, - dtype=dtype) + rank_param = helper.create_parameter( + attr=rank_param_attr, shape=rank_param_shape, dtype=dtype + ) rank_param.stop_gradient = False output = helper.create_variable_for_type_inference(dtype) - input_help = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) - ins_rank = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) - - helper.append_op(type="rank_attention", - inputs={ - "X": input, - "RankOffset": rank_offset, - "RankParam": rank_param - }, - outputs={ - "Out": output, - "InputHelp": input_help, - "InsRank": ins_rank - }, - attrs={ - "MaxRank": max_rank, - "MaxSize": max_size - }) + input_help = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) + ins_rank = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) + + helper.append_op( + type="rank_attention", + inputs={"X": input, "RankOffset": rank_offset, "RankParam": rank_param}, + outputs={"Out": output, "InputHelp": input_help, "InsRank": ins_rank}, + attrs={"MaxRank": max_rank, "MaxSize": max_size}, + ) return output @@ -1666,22 +1692,18 @@ def batch_fc(input, param_size, param_attr, bias_size, bias_attr, act=None): dtype = helper.input_dtype() check_dtype(dtype, 'input', ['float32', 'float64'], 'batch_fc') - w = helper.create_parameter(attr=param_attr, - shape=param_size, - dtype=dtype, - is_bias=False) - b = helper.create_parameter(attr=bias_attr, - shape=bias_size, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=param_attr, shape=param_size, dtype=dtype, is_bias=False + ) + b = helper.create_parameter( + attr=bias_attr, shape=bias_size, dtype=dtype, is_bias=False + ) pre_act = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="batch_fc", - inputs={ - "Input": input, - "W": w, - "Bias": b - }, - outputs={"Out": pre_act}) + helper.append_op( + type="batch_fc", + inputs={"Input": input, "W": w, "Bias": b}, + outputs={"Out": pre_act}, + ) return helper.append_activation(pre_act) @@ -1720,16 +1742,12 @@ def _pull_box_extended_sparse(input, size, extend_size=64, dtype='float32'): helper.create_variable_for_type_inference(dtype) for i in range(len(inputs)) ] - helper.append_op(type='pull_box_extended_sparse', - inputs={'Ids': inputs}, - outputs={ - 'Out': outs, - 'OutExtend': outs_extend - }, - attrs={ - 'emb_size': size, - 'emb_extended_size': extend_size - }) + helper.append_op( + type='pull_box_extended_sparse', + inputs={'Ids': inputs}, + outputs={'Out': outs, 'OutExtend': outs_extend}, + attrs={'emb_size': size, 'emb_extended_size': extend_size}, + ) if len(outs) == 1: return outs[0], outs_extend[0] return outs, outs_extend @@ -1738,8 +1756,8 @@ def _pull_box_extended_sparse(input, size, extend_size=64, dtype='float32'): def bilateral_slice(x, guide, grid, has_offset, name=None): """ :alias_main: paddle.nn.functional.bilateral_slice - :alias: paddle.nn.functional.bilateral_slice,paddle.nn.functional.vision.bilateral_slice - :old_api: paddle.fluid.layers.bilateral_slice + :alias: paddle.nn.functional.bilateral_slice,paddle.nn.functional.vision.bilateral_slice + :old_api: paddle.fluid.layers.bilateral_slice This operation implements bilateral slicing on the input according to the guide map. For more information of bilateral slicing, please refer to Deep Bilateral Learning for Real-Time Image Enhancement _ @@ -1783,28 +1801,34 @@ def bilateral_slice(x, guide, grid, has_offset, name=None): return getattr(_legacy_C_ops, "bilateral_slice")(x, grid, guide, *attrs) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'bilateral_slice') - check_variable_and_dtype(guide, 'guide', ['float32', 'float64'], - 'bilateral_slice') - check_variable_and_dtype(grid, 'grid', ['float32', 'float64'], - 'bilateral_slice') + check_variable_and_dtype( + guide, 'guide', ['float32', 'float64'], 'bilateral_slice' + ) + check_variable_and_dtype( + grid, 'grid', ['float32', 'float64'], 'bilateral_slice' + ) helper = LayerHelper("bilateral_slice", **locals()) out = helper.create_variable_for_type_inference(x.dtype) inputs = {'X': x, 'Guide': guide, 'Grid': grid} - helper.append_op(type='bilateral_slice', - inputs=inputs, - attrs={'has_offset': has_offset}, - outputs={'Out': out}) + helper.append_op( + type='bilateral_slice', + inputs=inputs, + attrs={'has_offset': has_offset}, + outputs={'Out': out}, + ) return out -def correlation(x, - y, - pad_size, - kernel_size, - max_displacement, - stride1, - stride2, - corr_type_multiply=1): +def correlation( + x, + y, + pad_size, + kernel_size, + max_displacement, + stride1, + stride2, + corr_type_multiply=1, +): """ This operation compute correlation of two tensor. @@ -1852,40 +1876,52 @@ def correlation(x, """ if paddle.fluid._non_static_mode(): - attrs = ("pad_size", pad_size, "kernel_size", kernel_size, - "max_displacement", max_displacement, "stride1", stride1, - "stride2", stride2, "corr_type_multiply", corr_type_multiply) + attrs = ( + "pad_size", + pad_size, + "kernel_size", + kernel_size, + "max_displacement", + max_displacement, + "stride1", + stride1, + "stride2", + stride2, + "corr_type_multiply", + corr_type_multiply, + ) output = getattr(_legacy_C_ops, "correlation")(x, y, *attrs) else: helper = LayerHelper("correlation", **locals()) output = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="correlation", - inputs={ - "Input1": x, - "Input2": y - }, - attrs={ - "pad_size": pad_size, - "kernel_size": kernel_size, - "max_displacement": max_displacement, - "stride1": stride1, - "stride2": stride2, - "corr_type_multiply": corr_type_multiply - }, - outputs={"Output": output}) + helper.append_op( + type="correlation", + inputs={"Input1": x, "Input2": y}, + attrs={ + "pad_size": pad_size, + "kernel_size": kernel_size, + "max_displacement": max_displacement, + "stride1": stride1, + "stride2": stride2, + "corr_type_multiply": corr_type_multiply, + }, + outputs={"Output": output}, + ) return output -def fused_bn_add_act(x, - y, - momentum=0.9, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - moving_mean_name=None, - moving_variance_name=None, - act=None, - name=None): +def fused_bn_add_act( + x, + y, + momentum=0.9, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + moving_mean_name=None, + moving_variance_name=None, + act=None, + name=None, +): r""" This Op performs batch norm on input x, and adds the result to input y. Then it performs activation on the sum. The data format of inputs must be NHWC @@ -1906,14 +1942,14 @@ def fused_bn_add_act(x, numerical stability. Default is 1e-5. param_attr(ParamAttr, optional): The parameter attribute for Parameter `scale` of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm - will create ParamAttr as param_attr, the name of scale can be set in ParamAttr. - If the Initializer of the param_attr is not set, the parameter is initialized - with Xavier. Default: None. + will create ParamAttr as param_attr, the name of scale can be set in ParamAttr. + If the Initializer of the param_attr is not set, the parameter is initialized + with Xavier. Default: None. bias_attr(ParamAttr, optional): The parameter attribute for the bias of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm - will create ParamAttr as bias_attr, the name of bias can be set in ParamAttr. - If the Initializer of the bias_attr is not set, the bias is initialized zero. - Default: None. + will create ParamAttr as bias_attr, the name of bias can be set in ParamAttr. + If the Initializer of the bias_attr is not set, the bias is initialized zero. + Default: None. moving_mean_name(str, optional): The name of moving_mean which store the global Mean. If it is set to None, batch_norm will save global mean with a random name, otherwise, batch_norm will save global mean with the string. @@ -1988,10 +2024,12 @@ def build_program(main_program, startup_program): """ helper = LayerHelper('fused_bn_add_act', **locals()) - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - 'fused_bn_add_act') - check_variable_and_dtype(y, 'input', ['float16', 'float32', 'float64'], - 'fused_bn_add_act') + check_variable_and_dtype( + x, 'input', ['float16', 'float32', 'float64'], 'fused_bn_add_act' + ) + check_variable_and_dtype( + y, 'input', ['float16', 'float32', 'float64'], 'fused_bn_add_act' + ) bn_param_dtype = core.VarDesc.VarType.FP32 x_shape = x.shape @@ -1999,25 +2037,35 @@ def build_program(main_program, startup_program): param_shape = [channel_num] # create parameter - scale = helper.create_parameter(attr=helper.param_attr, - shape=param_shape, - dtype=bn_param_dtype, - default_initializer=Constant(1.0)) - bias = helper.create_parameter(attr=helper.bias_attr, - shape=param_shape, - dtype=bn_param_dtype, - is_bias=True) - mean = helper.create_parameter(attr=ParamAttr(name=moving_mean_name, - initializer=Constant(0.0), - trainable=False), - shape=param_shape, - dtype=bn_param_dtype) + scale = helper.create_parameter( + attr=helper.param_attr, + shape=param_shape, + dtype=bn_param_dtype, + default_initializer=Constant(1.0), + ) + bias = helper.create_parameter( + attr=helper.bias_attr, + shape=param_shape, + dtype=bn_param_dtype, + is_bias=True, + ) + mean = helper.create_parameter( + attr=ParamAttr( + name=moving_mean_name, initializer=Constant(0.0), trainable=False + ), + shape=param_shape, + dtype=bn_param_dtype, + ) mean.stop_gradient = True - variance = helper.create_parameter(attr=ParamAttr(name=moving_variance_name, - initializer=Constant(1.0), - trainable=False), - shape=param_shape, - dtype=bn_param_dtype) + variance = helper.create_parameter( + attr=ParamAttr( + name=moving_variance_name, + initializer=Constant(1.0), + trainable=False, + ), + shape=param_shape, + dtype=bn_param_dtype, + ) variance.stop_gradient = True # create output @@ -2025,14 +2073,18 @@ def build_program(main_program, startup_program): mean_out = mean # variance and variance out share the same memory variance_out = variance - saved_mean = helper.create_variable_for_type_inference(dtype=bn_param_dtype, - stop_gradient=True) + saved_mean = helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) saved_variance = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) reserve_space = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.FP16, stop_gradient=True) + dtype=core.VarDesc.VarType.FP16, stop_gradient=True + ) batch_norm_out = helper.create_variable_for_type_inference( - core.VarDesc.VarType.FP16) + core.VarDesc.VarType.FP16 + ) inputs = { "X": x, @@ -2048,51 +2100,50 @@ def build_program(main_program, startup_program): "VarianceOut": variance_out, "SavedMean": saved_mean, "SavedVariance": saved_variance, - "ReserveSpace": reserve_space + "ReserveSpace": reserve_space, } - helper.append_op(type="fused_bn_add_activation", - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type="fused_bn_add_activation", + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) return batch_norm_out -def pow2_decay_with_linear_warmup(warmup_steps, - total_steps, - base_lr, - end_lr, - dtype='float32', - name=None): +def pow2_decay_with_linear_warmup( + warmup_steps, total_steps, base_lr, end_lr, dtype='float32', name=None +): if paddle.fluid._non_static_mode(): raise NotImplementedError( - "pow2_decay_with_linear_warmup does not support dygraph mode yet.") + "pow2_decay_with_linear_warmup does not support dygraph mode yet." + ) helper = LayerHelper("pow2_decay_with_linear_warmup", **locals()) lr = helper.create_global_variable(persistable=True, dtype=dtype, shape=[1]) helper.set_variable_initializer( - lr, Constant(value=float(base_lr) / warmup_steps)) + lr, Constant(value=float(base_lr) / warmup_steps) + ) - step = helper.create_global_variable(persistable=True, - dtype='int64', - shape=[1]) + step = helper.create_global_variable( + persistable=True, dtype='int64', shape=[1] + ) helper.set_variable_initializer(step, Constant(value=0)) - assert warmup_steps <= total_steps, "warmup_steps cannot be larger than total_steps" - - helper.append_op(type="pow2_decay_with_linear_warmup", - inputs={ - "LearningRate": lr, - "Step": step - }, - outputs={ - "LearningRateOut": lr, - "StepOut": step - }, - attrs={ - "warmup_steps": warmup_steps, - "total_steps": total_steps, - "base_lr": base_lr, - "end_lr": end_lr, - }) + assert ( + warmup_steps <= total_steps + ), "warmup_steps cannot be larger than total_steps" + + helper.append_op( + type="pow2_decay_with_linear_warmup", + inputs={"LearningRate": lr, "Step": step}, + outputs={"LearningRateOut": lr, "StepOut": step}, + attrs={ + "warmup_steps": warmup_steps, + "total_steps": total_steps, + "base_lr": base_lr, + "end_lr": end_lr, + }, + ) return lr diff --git a/python/paddle/fluid/contrib/layers/rnn_impl.py b/python/paddle/fluid/contrib/layers/rnn_impl.py index 95ce0a6ba538a..668ce445f78b0 100644 --- a/python/paddle/fluid/contrib/layers/rnn_impl.py +++ b/python/paddle/fluid/contrib/layers/rnn_impl.py @@ -74,18 +74,21 @@ class BasicGRUUnit(Layer): """ - def __init__(self, - name_scope, - hidden_size, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - dtype='float32'): + def __init__( + self, + name_scope, + hidden_size, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + dtype='float32', + ): super(BasicGRUUnit, self).__init__(name_scope, dtype) # reserve old school _full_name and _helper for static graph save load - self._full_name = unique_name.generate(name_scope + "/" + - self.__class__.__name__) + self._full_name = unique_name.generate( + name_scope + "/" + self.__class__.__name__ + ) self._helper = LayerObjectHelper(self._full_name) self._name = name_scope @@ -98,7 +101,7 @@ def __init__(self, def _build_once(self, input, pre_hidden): self._input_size = input.shape[-1] - assert (self._input_size > 0) + assert self._input_size > 0 if self._param_attr is not None and self._param_attr.name is not None: gate_param_attr = copy.deepcopy(self._param_attr) @@ -112,12 +115,14 @@ def _build_once(self, input, pre_hidden): self._gate_weight = self.create_parameter( attr=gate_param_attr, shape=[self._input_size + self._hiden_size, 2 * self._hiden_size], - dtype=self._dtype) + dtype=self._dtype, + ) self._candidate_weight = self.create_parameter( attr=candidate_param_attr, shape=[self._input_size + self._hiden_size, self._hiden_size], - dtype=self._dtype) + dtype=self._dtype, + ) if self._bias_attr is not None and self._bias_attr.name is not None: gate_bias_attr = copy.deepcopy(self._bias_attr) @@ -128,14 +133,18 @@ def _build_once(self, input, pre_hidden): gate_bias_attr = self._bias_attr candidate_bias_attr = self._bias_attr - self._gate_bias = self.create_parameter(attr=gate_bias_attr, - shape=[2 * self._hiden_size], - dtype=self._dtype, - is_bias=True) - self._candidate_bias = self.create_parameter(attr=candidate_bias_attr, - shape=[self._hiden_size], - dtype=self._dtype, - is_bias=True) + self._gate_bias = self.create_parameter( + attr=gate_bias_attr, + shape=[2 * self._hiden_size], + dtype=self._dtype, + is_bias=True, + ) + self._candidate_bias = self.create_parameter( + attr=candidate_bias_attr, + shape=[self._hiden_size], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input, pre_hidden): concat_input_hidden = layers.concat([input, pre_hidden], 1) @@ -149,8 +158,9 @@ def forward(self, input, pre_hidden): r_hidden = r * pre_hidden - candidate = layers.matmul(layers.concat([input, r_hidden], 1), - self._candidate_weight) + candidate = layers.matmul( + layers.concat([input, r_hidden], 1), self._candidate_weight + ) candidate = layers.elementwise_add(candidate, self._candidate_bias) c = self._activation(candidate) @@ -159,20 +169,22 @@ def forward(self, input, pre_hidden): return new_hidden -def basic_gru(input, - init_hidden, - hidden_size, - num_layers=1, - sequence_length=None, - dropout_prob=0.0, - bidirectional=False, - batch_first=True, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - dtype='float32', - name='basic_gru'): +def basic_gru( + input, + init_hidden, + hidden_size, + num_layers=1, + sequence_length=None, + dropout_prob=0.0, + bidirectional=False, + batch_first=True, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + dtype='float32', + name='basic_gru', +): r""" GRU implementation using basic operator, supports multiple layers and bidirectional gru. @@ -273,8 +285,16 @@ def basic_gru(input, else: layer_bias_attr = bias_attr fw_unit_list.append( - BasicGRUUnit(new_name, hidden_size, layer_param_attr, - layer_bias_attr, gate_activation, activation, dtype)) + BasicGRUUnit( + new_name, + hidden_size, + layer_param_attr, + layer_bias_attr, + gate_activation, + activation, + dtype, + ) + ) if bidirectional: bw_unit_list = [] @@ -292,9 +312,16 @@ def basic_gru(input, layer_bias_attr = bias_attr bw_unit_list.append( - BasicGRUUnit(new_name, hidden_size, layer_param_attr, - layer_bias_attr, gate_activation, activation, - dtype)) + BasicGRUUnit( + new_name, + hidden_size, + layer_param_attr, + layer_bias_attr, + gate_activation, + activation, + dtype, + ) + ) if batch_first: input = layers.transpose(input, [1, 0, 2]) @@ -302,9 +329,9 @@ def basic_gru(input, mask = None if sequence_length: max_seq_len = layers.shape(input)[0] - mask = layers.sequence_mask(sequence_length, - maxlen=max_seq_len, - dtype='float32') + mask = layers.sequence_mask( + sequence_length, maxlen=max_seq_len, dtype='float32' + ) mask = layers.transpose(mask, [1, 0]) direc_num = 1 @@ -312,12 +339,12 @@ def basic_gru(input, direc_num = 2 if init_hidden: init_hidden = layers.reshape( - init_hidden, shape=[num_layers, direc_num, -1, hidden_size]) + init_hidden, shape=[num_layers, direc_num, -1, hidden_size] + ) - def get_single_direction_output(rnn_input, - unit_list, - mask=None, - direc_index=0): + def get_single_direction_output( + rnn_input, unit_list, mask=None, direc_index=0 + ): rnn = StaticRNN() with rnn.step(): step_input = rnn.step_input(rnn_input) @@ -329,16 +356,20 @@ def get_single_direction_output(rnn_input, if init_hidden: pre_hidden = rnn.memory(init=init_hidden[i, direc_index]) else: - pre_hidden = rnn.memory(batch_ref=rnn_input, - shape=[-1, hidden_size], - ref_batch_dim_idx=1) + pre_hidden = rnn.memory( + batch_ref=rnn_input, + shape=[-1, hidden_size], + ref_batch_dim_idx=1, + ) new_hidden = unit_list[i](step_input, pre_hidden) if mask: new_hidden = layers.elementwise_mul( - new_hidden, step_mask, axis=0) - layers.elementwise_mul( - pre_hidden, (step_mask - 1), axis=0) + new_hidden, step_mask, axis=0 + ) - layers.elementwise_mul( + pre_hidden, (step_mask - 1), axis=0 + ) rnn.update_memory(pre_hidden, new_hidden) rnn.step_output(new_hidden) @@ -362,26 +393,25 @@ def get_single_direction_output(rnn_input, last_hidden_array.append(last_hidden) last_hidden_output = layers.concat(last_hidden_array, axis=0) - last_hidden_output = layers.reshape(last_hidden_output, - shape=[num_layers, -1, hidden_size]) + last_hidden_output = layers.reshape( + last_hidden_output, shape=[num_layers, -1, hidden_size] + ) return rnn_output, last_hidden_output # seq_len, batch_size, hidden_size - fw_rnn_out, fw_last_hidden = get_single_direction_output(input, - fw_unit_list, - mask, - direc_index=0) + fw_rnn_out, fw_last_hidden = get_single_direction_output( + input, fw_unit_list, mask, direc_index=0 + ) if bidirectional: bw_input = layers.reverse(input, axis=[0]) bw_mask = None if mask: bw_mask = layers.reverse(mask, axis=[0]) - bw_rnn_out, bw_last_hidden = get_single_direction_output(bw_input, - bw_unit_list, - bw_mask, - direc_index=1) + bw_rnn_out, bw_last_hidden = get_single_direction_output( + bw_input, bw_unit_list, bw_mask, direc_index=1 + ) bw_rnn_out = layers.reverse(bw_rnn_out, axis=[0]) @@ -389,7 +419,8 @@ def get_single_direction_output(rnn_input, last_hidden = layers.concat([fw_last_hidden, bw_last_hidden], axis=1) last_hidden = layers.reshape( - last_hidden, shape=[num_layers * direc_num, -1, hidden_size]) + last_hidden, shape=[num_layers * direc_num, -1, hidden_size] + ) if batch_first: rnn_out = layers.transpose(rnn_out, [1, 0, 2]) @@ -405,22 +436,24 @@ def get_single_direction_output(rnn_input, return rnn_out, last_hidden -def basic_lstm(input, - init_hidden, - init_cell, - hidden_size, - num_layers=1, - sequence_length=None, - dropout_prob=0.0, - bidirectional=False, - batch_first=True, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - forget_bias=1.0, - dtype='float32', - name='basic_lstm'): +def basic_lstm( + input, + init_hidden, + init_cell, + hidden_size, + num_layers=1, + sequence_length=None, + dropout_prob=0.0, + bidirectional=False, + batch_first=True, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + forget_bias=1.0, + dtype='float32', + name='basic_lstm', +): r""" LSTM implementation using basic operators, supports multiple layers and bidirectional LSTM. @@ -535,14 +568,17 @@ def basic_lstm(input, else: layer_bias_attr = bias_attr fw_unit_list.append( - BasicLSTMUnit(new_name, - hidden_size, - param_attr=layer_param_attr, - bias_attr=layer_bias_attr, - gate_activation=gate_activation, - activation=activation, - forget_bias=forget_bias, - dtype=dtype)) + BasicLSTMUnit( + new_name, + hidden_size, + param_attr=layer_param_attr, + bias_attr=layer_bias_attr, + gate_activation=gate_activation, + activation=activation, + forget_bias=forget_bias, + dtype=dtype, + ) + ) if bidirectional: bw_unit_list = [] @@ -559,14 +595,17 @@ def basic_lstm(input, else: layer_bias_attr = param_attr bw_unit_list.append( - BasicLSTMUnit(new_name, - hidden_size, - param_attr=layer_param_attr, - bias_attr=layer_bias_attr, - gate_activation=gate_activation, - activation=activation, - forget_bias=forget_bias, - dtype=dtype)) + BasicLSTMUnit( + new_name, + hidden_size, + param_attr=layer_param_attr, + bias_attr=layer_bias_attr, + gate_activation=gate_activation, + activation=activation, + forget_bias=forget_bias, + dtype=dtype, + ) + ) if batch_first: input = layers.transpose(input, [1, 0, 2]) @@ -574,9 +613,9 @@ def basic_lstm(input, mask = None if sequence_length: max_seq_len = layers.shape(input)[0] - mask = layers.sequence_mask(sequence_length, - maxlen=max_seq_len, - dtype='float32') + mask = layers.sequence_mask( + sequence_length, maxlen=max_seq_len, dtype='float32' + ) mask = layers.transpose(mask, [1, 0]) @@ -586,15 +625,16 @@ def basic_lstm(input, # convert to [num_layers, 2, batch_size, hidden_size] if init_hidden: init_hidden = layers.reshape( - init_hidden, shape=[num_layers, direc_num, -1, hidden_size]) + init_hidden, shape=[num_layers, direc_num, -1, hidden_size] + ) init_cell = layers.reshape( - init_cell, shape=[num_layers, direc_num, -1, hidden_size]) + init_cell, shape=[num_layers, direc_num, -1, hidden_size] + ) # forward direction - def get_single_direction_output(rnn_input, - unit_list, - mask=None, - direc_index=0): + def get_single_direction_output( + rnn_input, unit_list, mask=None, direc_index=0 + ): rnn = StaticRNN() with rnn.step(): step_input = rnn.step_input(rnn_input) @@ -607,21 +647,28 @@ def get_single_direction_output(rnn_input, pre_hidden = rnn.memory(init=init_hidden[i, direc_index]) pre_cell = rnn.memory(init=init_cell[i, direc_index]) else: - pre_hidden = rnn.memory(batch_ref=rnn_input, - shape=[-1, hidden_size]) - pre_cell = rnn.memory(batch_ref=rnn_input, - shape=[-1, hidden_size]) + pre_hidden = rnn.memory( + batch_ref=rnn_input, shape=[-1, hidden_size] + ) + pre_cell = rnn.memory( + batch_ref=rnn_input, shape=[-1, hidden_size] + ) - new_hidden, new_cell = unit_list[i](step_input, pre_hidden, - pre_cell) + new_hidden, new_cell = unit_list[i]( + step_input, pre_hidden, pre_cell + ) if mask: new_hidden = layers.elementwise_mul( - new_hidden, step_mask, axis=0) - layers.elementwise_mul( - pre_hidden, (step_mask - 1), axis=0) + new_hidden, step_mask, axis=0 + ) - layers.elementwise_mul( + pre_hidden, (step_mask - 1), axis=0 + ) new_cell = layers.elementwise_mul( - new_cell, step_mask, axis=0) - layers.elementwise_mul( - pre_cell, (step_mask - 1), axis=0) + new_cell, step_mask, axis=0 + ) - layers.elementwise_mul( + pre_cell, (step_mask - 1), axis=0 + ) rnn.update_memory(pre_hidden, new_hidden) rnn.update_memory(pre_cell, new_cell) @@ -634,7 +681,8 @@ def get_single_direction_output(rnn_input, step_input = layers.dropout( step_input, dropout_prob=dropout_prob, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn.step_output(step_input) @@ -652,17 +700,20 @@ def get_single_direction_output(rnn_input, last_cell_array.append(last_cell) last_hidden_output = layers.concat(last_hidden_array, axis=0) - last_hidden_output = layers.reshape(last_hidden_output, - shape=[num_layers, -1, hidden_size]) + last_hidden_output = layers.reshape( + last_hidden_output, shape=[num_layers, -1, hidden_size] + ) last_cell_output = layers.concat(last_cell_array, axis=0) - last_cell_output = layers.reshape(last_cell_output, - shape=[num_layers, -1, hidden_size]) + last_cell_output = layers.reshape( + last_cell_output, shape=[num_layers, -1, hidden_size] + ) return rnn_output, last_hidden_output, last_cell_output # seq_len, batch_size, hidden_size fw_rnn_out, fw_last_hidden, fw_last_cell = get_single_direction_output( - input, fw_unit_list, mask, direc_index=0) + input, fw_unit_list, mask, direc_index=0 + ) if bidirectional: bw_input = layers.reverse(input, axis=[0]) @@ -670,18 +721,21 @@ def get_single_direction_output(rnn_input, if mask: bw_mask = layers.reverse(mask, axis=[0]) bw_rnn_out, bw_last_hidden, bw_last_cell = get_single_direction_output( - bw_input, bw_unit_list, bw_mask, direc_index=1) + bw_input, bw_unit_list, bw_mask, direc_index=1 + ) bw_rnn_out = layers.reverse(bw_rnn_out, axis=[0]) rnn_out = layers.concat([fw_rnn_out, bw_rnn_out], axis=2) last_hidden = layers.concat([fw_last_hidden, bw_last_hidden], axis=1) last_hidden = layers.reshape( - last_hidden, shape=[num_layers * direc_num, -1, hidden_size]) + last_hidden, shape=[num_layers * direc_num, -1, hidden_size] + ) last_cell = layers.concat([fw_last_cell, bw_last_cell], axis=1) last_cell = layers.reshape( - last_cell, shape=[num_layers * direc_num, -1, hidden_size]) + last_cell, shape=[num_layers * direc_num, -1, hidden_size] + ) if batch_first: rnn_out = layers.transpose(rnn_out, [1, 0, 2]) @@ -769,19 +823,22 @@ class BasicLSTMUnit(Layer): """ - def __init__(self, - name_scope, - hidden_size, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - forget_bias=1.0, - dtype='float32'): + def __init__( + self, + name_scope, + hidden_size, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + forget_bias=1.0, + dtype='float32', + ): super(BasicLSTMUnit, self).__init__(name_scope, dtype) # reserve old school _full_name and _helper for static graph save load - self._full_name = unique_name.generate(name_scope + "/" + - self.__class__.__name__) + self._full_name = unique_name.generate( + name_scope + "/" + self.__class__.__name__ + ) self._helper = LayerObjectHelper(self._full_name) self._name = name_scope @@ -790,25 +847,28 @@ def __init__(self, self._bias_attr = bias_attr self._gate_activation = gate_activation or layers.sigmoid self._activation = activation or layers.tanh - self._forget_bias = layers.fill_constant([1], - dtype=dtype, - value=forget_bias) + self._forget_bias = layers.fill_constant( + [1], dtype=dtype, value=forget_bias + ) self._forget_bias.stop_gradient = False self._dtype = dtype def _build_once(self, input, pre_hidden, pre_cell): self._input_size = input.shape[-1] - assert (self._input_size > 0) + assert self._input_size > 0 self._weight = self.create_parameter( attr=self._param_attr, shape=[self._input_size + self._hiden_size, 4 * self._hiden_size], - dtype=self._dtype) + dtype=self._dtype, + ) - self._bias = self.create_parameter(attr=self._bias_attr, - shape=[4 * self._hiden_size], - dtype=self._dtype, - is_bias=True) + self._bias = self.create_parameter( + attr=self._bias_attr, + shape=[4 * self._hiden_size], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input, pre_hidden, pre_cell): concat_input_hidden = layers.concat([input, pre_hidden], 1) @@ -819,8 +879,10 @@ def forward(self, input, pre_hidden, pre_cell): new_cell = layers.elementwise_add( layers.elementwise_mul( pre_cell, - layers.sigmoid(layers.elementwise_add(f, self._forget_bias))), - layers.elementwise_mul(layers.sigmoid(i), layers.tanh(j))) + layers.sigmoid(layers.elementwise_add(f, self._forget_bias)), + ), + layers.elementwise_mul(layers.sigmoid(i), layers.tanh(j)), + ) new_hidden = layers.tanh(new_cell) * layers.sigmoid(o) return new_hidden, new_cell diff --git a/python/paddle/fluid/contrib/memory_usage_calc.py b/python/paddle/fluid/contrib/memory_usage_calc.py index 607f8b45da1f1..8a8dcb55a00b7 100644 --- a/python/paddle/fluid/contrib/memory_usage_calc.py +++ b/python/paddle/fluid/contrib/memory_usage_calc.py @@ -66,7 +66,8 @@ def memory_usage(program, batch_size): if not isinstance(program, Program): raise TypeError( "Calculating Memory Usage requires Program as its Parameter." - "But you passed in %s" % (type(program))) + "But you passed in %s" % (type(program)) + ) if batch_size <= 0: raise ValueError("The batch size need to be positive.") @@ -88,8 +89,9 @@ def memory_usage(program, batch_size): if x < 0: if neg_dim_count >= 1: raise ValueError( - "Var %s has more than one negative dim." % - (var_name)) + "Var %s has more than one negative dim." + % (var_name) + ) neg_dim_count += 1 data_count *= batch_size * (-x) else: diff --git a/python/paddle/fluid/contrib/mixed_precision/amp_nn.py b/python/paddle/fluid/contrib/mixed_precision/amp_nn.py index f9552fa705c11..f336c87a11e10 100644 --- a/python/paddle/fluid/contrib/mixed_precision/amp_nn.py +++ b/python/paddle/fluid/contrib/mixed_precision/amp_nn.py @@ -39,8 +39,12 @@ def check_finite_and_unscale(x, scale, name=None, float_status=None): """ check_type(x, 'x', (tuple, list), 'check_finite_and_unscale') for e in x: - check_variable_and_dtype(e, "x", ['float16', 'float32', 'float64'], - 'check_finite_and_unscale') + check_variable_and_dtype( + e, + "x", + ['float16', 'float32', 'float64'], + 'check_finite_and_unscale', + ) helper = LayerHelper("check_finite_and_unscale", **locals()) @@ -52,29 +56,34 @@ def check_finite_and_unscale(x, scale, name=None, float_status=None): inputs = {'X': x, 'Scale': scale} if core.is_compiled_with_npu(): - check_variable_and_dtype(float_status, "float_status", - ['float16', 'float32'], - 'check_finite_and_unscale') + check_variable_and_dtype( + float_status, + "float_status", + ['float16', 'float32'], + 'check_finite_and_unscale', + ) inputs['FloatStatus'] = float_status outputs = {'Out': x, 'FoundInfinite': found_inf} - helper.append_op(type='check_finite_and_unscale', - inputs=inputs, - outputs=outputs) + helper.append_op( + type='check_finite_and_unscale', inputs=inputs, outputs=outputs + ) return x, found_inf -def update_loss_scaling(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - stop_update=False, - name=None): +def update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + stop_update=False, + name=None, +): """ Update loss scaling according to overall gradients. If all gradients is finite after incr_every_n_steps, loss scaling will increase by incr_ratio. @@ -102,23 +111,39 @@ def update_loss_scaling(x, loss scaling. """ - check_variable_and_dtype(prev_loss_scaling, "prev_loss_scaling", - ['float32', 'float64'], "update_loss_scaling") + check_variable_and_dtype( + prev_loss_scaling, + "prev_loss_scaling", + ['float32', 'float64'], + "update_loss_scaling", + ) check_type(x, 'x', (tuple, list), 'update_loss_scaling') for e in x: - check_variable_and_dtype(e, "x", ['float16', 'float32', 'float64'], - 'update_loss_scaling') + check_variable_and_dtype( + e, "x", ['float16', 'float32', 'float64'], 'update_loss_scaling' + ) if e.dtype == core.VarDesc.VarType.FP16: - assert prev_loss_scaling.dtype == core.VarDesc.VarType.FP32, \ - "The dtype of prev_loss_scaling should be float32 when the dtype of x is float16." + assert ( + prev_loss_scaling.dtype == core.VarDesc.VarType.FP32 + ), "The dtype of prev_loss_scaling should be float32 when the dtype of x is float16." else: - assert prev_loss_scaling.dtype == e.dtype, "The dtype of prev_loss_scaling should be equal to the dtype of x." + assert ( + prev_loss_scaling.dtype == e.dtype + ), "The dtype of prev_loss_scaling should be equal to the dtype of x." if in_dygraph_mode(): - _C_ops.update_loss_scaling_(x, found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps, - incr_every_n_steps, decr_every_n_nan_or_inf, - incr_ratio, decr_ratio, stop_update) + _C_ops.update_loss_scaling_( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + stop_update, + ) return x helper = LayerHelper("update_loss_scaling", **locals()) @@ -128,14 +153,14 @@ def update_loss_scaling(x, 'FoundInfinite': found_inf, 'PrevLossScaling': prev_loss_scaling, 'InGoodSteps': num_good_steps, - 'InBadSteps': num_bad_steps + 'InBadSteps': num_bad_steps, } outputs = { 'Out': x, 'LossScaling': prev_loss_scaling, 'OutGoodSteps': num_good_steps, - 'OutBadSteps': num_bad_steps + 'OutBadSteps': num_bad_steps, } attrs = { @@ -150,9 +175,8 @@ def update_loss_scaling(x, else: attrs['stop_update'] = stop_update - helper.append_op(type='update_loss_scaling', - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='update_loss_scaling', inputs=inputs, outputs=outputs, attrs=attrs + ) return x diff --git a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py index 3f799809af977..33694f4d127ee 100644 --- a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py +++ b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_lists.py @@ -15,8 +15,11 @@ import copy from paddle.fluid import core -from ..fp16_lists import white_list as white_list_fp16, black_list as black_list_fp16,\ - gray_list as gray_list_fp16 +from ..fp16_lists import ( + white_list as white_list_fp16, + black_list as black_list_fp16, + gray_list as gray_list_fp16, +) __all__ = ["AutoMixedPrecisionListsBF16"] @@ -40,10 +43,12 @@ class AutoMixedPrecisionListsBF16(object): paddle.static.amp.AutoMixedPrecisionListsBF16(custom_fp32_list={'lstm'}) """ - def __init__(self, - custom_bf16_list=None, - custom_fp32_list=None, - custom_fp32_varnames=None): + def __init__( + self, + custom_bf16_list=None, + custom_fp32_list=None, + custom_fp32_varnames=None, + ): self._custom_bf16_list = custom_bf16_list self._custom_fp32_list = custom_fp32_list self.bf16_list = copy.copy(bf16_list) @@ -61,8 +66,9 @@ def _update_list(self): if self._custom_bf16_list and self._custom_fp32_list: for op_name in self._custom_bf16_list: if op_name in self._custom_fp32_list: - raise ValueError("Custom bf16 list overlap " - "custom fp32 list") + raise ValueError( + "Custom bf16 list overlap " "custom fp32 list" + ) if self._custom_bf16_list: for op_name in self._custom_bf16_list: if op_name in self.fp32_list: @@ -92,13 +98,28 @@ def _update_list(self): # depends on the prev_op type gray_list = { - 'elementwise_add', 'elementwise_sub', 'elementwise_mul', 'elementwise_div', - 'relu', 'layer_norm', 'slice', 'concat', 'uniform_random', 'reshape2', - 'transpose2', 'pool2d', 'sigmoid', 'cast', 'scale', 'fill_constant', 'split' + 'elementwise_add', + 'elementwise_sub', + 'elementwise_mul', + 'elementwise_div', + 'relu', + 'layer_norm', + 'slice', + 'concat', + 'uniform_random', + 'reshape2', + 'transpose2', + 'pool2d', + 'sigmoid', + 'cast', + 'scale', + 'fill_constant', + 'split', } _, _, _sys_unsupported_bf16_list = core.op_supported_infos( - 'CPU', core.VarDesc.VarType.BF16) + 'CPU', core.VarDesc.VarType.BF16 +) unsupported_list = _sys_unsupported_bf16_list fp32_list = black_list_fp16.copy().copy() diff --git a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py index 94734a8088de5..25468b563aa12 100644 --- a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py +++ b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py @@ -19,8 +19,13 @@ from ....log_helper import get_logger from ....wrapped_decorator import signature_safe_contextmanager from .amp_lists import AutoMixedPrecisionListsBF16 -from ..fp16_utils import find_true_prev_op, find_true_post_op, _rename_arg, \ - find_op_index, _rename_op_input +from ..fp16_utils import ( + find_true_prev_op, + find_true_post_op, + _rename_arg, + find_op_index, + _rename_op_input, +) import collections import struct @@ -28,17 +33,21 @@ import numpy as np __all__ = [ - "bf16_guard", "rewrite_program_bf16", "cast_model_to_bf16", - "cast_parameters_to_bf16", "convert_float_to_uint16" + "bf16_guard", + "rewrite_program_bf16", + "cast_model_to_bf16", + "cast_parameters_to_bf16", + "convert_float_to_uint16", ] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) _valid_types = [ - core.VarDesc.VarType.LOD_TENSOR, core.VarDesc.VarType.SELECTED_ROWS, - core.VarDesc.VarType.LOD_TENSOR_ARRAY + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.SELECTED_ROWS, + core.VarDesc.VarType.LOD_TENSOR_ARRAY, ] _bf16_guard_pattern = "__use_bf16__" @@ -48,7 +57,8 @@ def convert_float_to_uint16(in_list): in_list = np.asarray(in_list) out = np.vectorize( lambda x: struct.unpack('> 16, - otypes=[np.uint16])(in_list.flat) + otypes=[np.uint16], + )(in_list.flat) return np.reshape(out, in_list.shape) @@ -83,7 +93,9 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): for in_name in op.input_names: if src_dtype == core.VarDesc.VarType.FP32 and op.type in [ - 'batch_norm', 'fused_bn_add_activation', 'layer_norm' + 'batch_norm', + 'fused_bn_add_activation', + 'layer_norm', ]: if in_name not in {'X', 'Z'}: continue @@ -99,26 +111,34 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): name=cast_name, dtype=dest_dtype, persistable=False, - stop_gradient=in_var.stop_gradient) - - block._insert_op(idx, - type="cast", - inputs={"X": in_var}, - outputs={"Out": out_var}, - attrs={ - "in_dtype": in_var.dtype, - "out_dtype": out_var.dtype - }) + stop_gradient=in_var.stop_gradient, + ) + + block._insert_op( + idx, + type="cast", + inputs={"X": in_var}, + outputs={"Out": out_var}, + attrs={ + "in_dtype": in_var.dtype, + "out_dtype": out_var.dtype, + }, + ) num_cast_ops += 1 _rename_arg(op, in_var.name, out_var.name) else: if op.has_attr('in_dtype'): op._set_attr('in_dtype', dest_dtype) - if src_dtype == core.VarDesc.VarType.FP32 and dest_dtype == core.VarDesc.VarType.BF16: + if ( + src_dtype == core.VarDesc.VarType.FP32 + and dest_dtype == core.VarDesc.VarType.BF16 + ): for out_name in op.output_names: - if op.type in [ - 'batch_norm', 'fused_bn_add_activation', 'layer_norm' - ] and out_name != 'Y': + if ( + op.type + in ['batch_norm', 'fused_bn_add_activation', 'layer_norm'] + and out_name != 'Y' + ): continue for out_var_name in op.output(out_name): out_var = block.var(out_var_name) @@ -131,31 +151,36 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): return num_cast_ops -def _insert_cast_post_op(block, op, idx, src_dtype, dest_dtype, target_name, - op_var_rename_map): +def _insert_cast_post_op( + block, op, idx, src_dtype, dest_dtype, target_name, op_var_rename_map +): num_cast_ops = 0 target_var = block.var(target_name) if target_var.type not in _valid_types or target_var.dtype == dest_dtype: return num_cast_ops - assert target_var.dtype == src_dtype, \ - "The real dtype({}) is not equal to the src dtype({})".format(_dtype_to_str(target_var.dtype), _dtype_to_str(src_dtype)) + assert ( + target_var.dtype == src_dtype + ), "The real dtype({}) is not equal to the src dtype({})".format( + _dtype_to_str(target_var.dtype), _dtype_to_str(src_dtype) + ) cast_name = target_var.name + '.cast_' + _dtype_to_str(dest_dtype) cast_var = block.vars.get(cast_name) if cast_var is None or cast_var.dtype != dest_dtype: - cast_var = block.create_var(name=cast_name, - dtype=dest_dtype, - persistable=False, - stop_gradient=target_var.stop_gradient) - block._insert_op(idx, - type="cast", - inputs={"X": target_var}, - outputs={"Out": cast_var}, - attrs={ - "in_dtype": target_var.dtype, - "out_dtype": cast_var.dtype - }) + cast_var = block.create_var( + name=cast_name, + dtype=dest_dtype, + persistable=False, + stop_gradient=target_var.stop_gradient, + ) + block._insert_op( + idx, + type="cast", + inputs={"X": target_var}, + outputs={"Out": cast_var}, + attrs={"in_dtype": target_var.dtype, "out_dtype": cast_var.dtype}, + ) num_cast_ops += 1 op_var_rename_map[block.idx][target_var.name] = cast_var.name @@ -192,8 +217,9 @@ def _need_keep_fp32(op, unsupported_op_list, use_bf16_guard): return True if use_bf16_guard: - if op.has_attr("op_namescope") and \ - (_bf16_guard_pattern in op.attr("op_namescope")): + if op.has_attr("op_namescope") and ( + _bf16_guard_pattern in op.attr("op_namescope") + ): # op in bf16 guard return False else: @@ -238,12 +264,14 @@ def are_post_ops_bf16(post_ops, keep_fp32_ops): return True -def cast_initializers_to_bf16(startup_prog, - amp_lists, - block, - all_ops, - keep_fp32_ops, - to_bf16_var_names=None): +def cast_initializers_to_bf16( + startup_prog, + amp_lists, + block, + all_ops, + keep_fp32_ops, + to_bf16_var_names=None, +): prepend_ops = startup_prog.global_block().ops for op in prepend_ops: if str(op.type) in amp_lists.bf16_initializer_list: @@ -265,17 +293,21 @@ def cast_initializers_to_bf16(startup_prog, for out_var in op_out_vars: if out_var.dtype == core.VarDesc.VarType.FP32: out_var.desc.set_dtype(core.VarDesc.VarType.BF16) - if to_bf16_var_names is not None and out_var.name in to_bf16_var_names: + if ( + to_bf16_var_names is not None + and out_var.name in to_bf16_var_names + ): to_bf16_var_names.remove(out_var.name) - if op.has_attr('dtype') and op.attr( - 'dtype') == core.VarDesc.VarType.FP32: + if ( + op.has_attr('dtype') + and op.attr('dtype') == core.VarDesc.VarType.FP32 + ): op._set_attr('dtype', core.VarDesc.VarType.BF16) -def cast_model_to_bf16(program, - startup_prog=None, - amp_lists=None, - use_bf16_guard=True): +def cast_model_to_bf16( + program, startup_prog=None, amp_lists=None, use_bf16_guard=True +): """ Traverse all ops in the whole model and set their inputs and outputs to the bf16 data type. This function will do some special processing for @@ -307,7 +339,9 @@ def cast_model_to_bf16(program, continue # processed below for in_name in op.input_names: if op.type in { - 'batch_norm', 'fused_bn_add_activation', 'layer_norm' + 'batch_norm', + 'fused_bn_add_activation', + 'layer_norm', } and in_name not in {'X', 'Z'}: continue for in_var_name in op.input(in_name): @@ -316,13 +350,17 @@ def cast_model_to_bf16(program, in_var = block.var(in_var_name) except ValueError as e: _logger.debug( - "-- {}, try to get it in the global block --". - format(e)) + "-- {}, try to get it in the global block --".format( + e + ) + ) in_var = global_block.var(in_var_name) if in_var is not None: _logger.debug( - "-- var {} is got in the global block --". - format(in_var_name)) + "-- var {} is got in the global block --".format( + in_var_name + ) + ) if in_var is None or in_var.type not in _valid_types: continue @@ -332,13 +370,17 @@ def cast_model_to_bf16(program, to_bf16_var_names.add(in_var_name) _logger.debug( - "-- op type: {}, in var name: {}, in var dtype: {} --". - format(op.type, in_var_name, in_var.dtype)) + "-- op type: {}, in var name: {}, in var dtype: {} --".format( + op.type, in_var_name, in_var.dtype + ) + ) for out_name in op.output_names: - if op.type in { - 'batch_norm', 'fused_bn_add_activation', 'layer_norm' - } and out_name != 'Y': + if ( + op.type + in {'batch_norm', 'fused_bn_add_activation', 'layer_norm'} + and out_name != 'Y' + ): continue for out_var_name in op.output(out_name): out_var = None @@ -346,13 +388,17 @@ def cast_model_to_bf16(program, out_var = block.var(out_var_name) except ValueError as e: _logger.debug( - "-- {}, try to get it in the global block --". - format(e)) + "-- {}, try to get it in the global block --".format( + e + ) + ) out_var = global_block.var(out_var_name) if out_var is not None: _logger.debug( - "-- var {} is got in the global block --". - format(out_var_name)) + "-- var {} is got in the global block --".format( + out_var_name + ) + ) if out_var is None or out_var.type not in _valid_types: continue @@ -361,11 +407,15 @@ def cast_model_to_bf16(program, out_var.desc.set_dtype(core.VarDesc.VarType.BF16) _logger.debug( - "-- op type: {}, out var name: {}, out var dtype: {} --" - .format(op.type, out_var_name, out_var.dtype)) + "-- op type: {}, out var name: {}, out var dtype: {} --".format( + op.type, out_var_name, out_var.dtype + ) + ) for attr_name in ['in_dtype', 'out_dtype', 'dtype']: - if op.has_attr(attr_name) and op.attr( - attr_name) == core.VarDesc.VarType.FP32: + if ( + op.has_attr(attr_name) + and op.attr(attr_name) == core.VarDesc.VarType.FP32 + ): op._set_attr(attr_name, core.VarDesc.VarType.BF16) if op.has_attr('use_mkldnn'): op._set_attr('use_mkldnn', True) @@ -373,8 +423,14 @@ def cast_model_to_bf16(program, op._set_attr('mkldnn_data_type', 'bfloat16') if startup_prog is not None: - cast_initializers_to_bf16(startup_prog, amp_lists, global_block, - ops, keep_fp32_ops, to_bf16_var_names) + cast_initializers_to_bf16( + startup_prog, + amp_lists, + global_block, + ops, + keep_fp32_ops, + to_bf16_var_names, + ) # process ops in keep_fp32_ops op_var_rename_map = [ @@ -388,14 +444,22 @@ def cast_model_to_bf16(program, num_cast_ops = 0 if op not in keep_fp32_ops: if op in to_bf16_pre_cast_ops: - in_var_cast_num = _insert_cast_op(block, op, idx, - core.VarDesc.VarType.FP32, - core.VarDesc.VarType.BF16) + in_var_cast_num = _insert_cast_op( + block, + op, + idx, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.BF16, + ) num_cast_ops += in_var_cast_num else: - pre_cast_num = _insert_cast_op(block, op, idx, - core.VarDesc.VarType.BF16, - core.VarDesc.VarType.FP32) + pre_cast_num = _insert_cast_op( + block, + op, + idx, + core.VarDesc.VarType.BF16, + core.VarDesc.VarType.FP32, + ) num_cast_ops += pre_cast_num for out_var_name in op.output_arg_names: out_var = block.vars.get(out_var_name) @@ -408,10 +472,14 @@ def cast_model_to_bf16(program, if post_op in keep_fp32_ops: continue post_cast_num = _insert_cast_post_op( - block, op, idx + pre_cast_num + 1, + block, + op, + idx + pre_cast_num + 1, core.VarDesc.VarType.FP32, - core.VarDesc.VarType.BF16, out_var_name, - op_var_rename_map) + core.VarDesc.VarType.BF16, + out_var_name, + op_var_rename_map, + ) num_cast_ops += post_cast_num idx += num_cast_ops + 1 @@ -482,7 +550,8 @@ def rewrite_program_bf16(main_prog, amp_lists=None): continue if amp_lists.fp32_varnames is not None and _is_in_fp32_varnames( - op, amp_lists): + op, amp_lists + ): fp32_op_set.add(op) continue @@ -508,11 +577,15 @@ def rewrite_program_bf16(main_prog, amp_lists=None): else: prev_op = in_var.op # if it's one of inputs - if prev_op in fp32_op_set or \ - prev_op.type in amp_lists.fp32_list: + if ( + prev_op in fp32_op_set + or prev_op.type in amp_lists.fp32_list + ): is_fp32_op = True - elif prev_op in bf16_op_set or \ - prev_op.type in amp_lists.bf16_list: + elif ( + prev_op in bf16_op_set + or prev_op.type in amp_lists.bf16_list + ): is_bf16_op = True if is_fp32_op: fp32_op_set.add(op) @@ -530,20 +603,30 @@ def rewrite_program_bf16(main_prog, amp_lists=None): op = ops[idx] num_cast_ops = 0 if op in fp32_op_set: - num_cast_ops = _insert_cast_op(block, op, idx, - core.VarDesc.VarType.BF16, - core.VarDesc.VarType.FP32) + num_cast_ops = _insert_cast_op( + block, + op, + idx, + core.VarDesc.VarType.BF16, + core.VarDesc.VarType.FP32, + ) elif op in bf16_op_set: if op.has_attr('use_mkldnn'): op._set_attr('use_mkldnn', True) op._set_attr('mkldnn_data_type', 'bfloat16') - elif op.has_attr('dtype') and op.attr( - 'dtype') == core.VarDesc.VarType.FP32: + elif ( + op.has_attr('dtype') + and op.attr('dtype') == core.VarDesc.VarType.FP32 + ): op._set_attr('dtype', core.VarDesc.VarType.BF16) - num_cast_ops = _insert_cast_op(block, op, idx, - core.VarDesc.VarType.FP32, - core.VarDesc.VarType.BF16) + num_cast_ops = _insert_cast_op( + block, + op, + idx, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.BF16, + ) else: pass diff --git a/python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py b/python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py index 0551cf1aace40..9110686582220 100644 --- a/python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py +++ b/python/paddle/fluid/contrib/mixed_precision/bf16/decorator.py @@ -12,10 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle.fluid import (core, default_main_program, layers, program_guard, - unique_name) -from .amp_utils import (rewrite_program_bf16, cast_model_to_bf16, - cast_parameters_to_bf16) +from paddle.fluid import ( + core, + default_main_program, + layers, + program_guard, + unique_name, +) +from .amp_utils import ( + rewrite_program_bf16, + cast_model_to_bf16, + cast_parameters_to_bf16, +) from .amp_lists import AutoMixedPrecisionListsBF16 import types import warnings @@ -58,20 +66,24 @@ def _init_amp_var(self): # Ensure the data type of learning rate vars is float32 (same as the # master parameter dtype) if isinstance(self._optimizer._learning_rate, float): - self._optimizer._learning_rate_map[default_main_program()] = \ - layers.create_global_var( - name=unique_name.generate("learning_rate"), - shape=[1], - value=float(self._optimizer._learning_rate), - dtype='float32', - persistable=True) - - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + self._optimizer._learning_rate_map[ + default_main_program() + ] = layers.create_global_var( + name=unique_name.generate("learning_rate"), + shape=[1], + value=float(self._optimizer._learning_rate), + dtype='float32', + persistable=True, + ) + + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): """ Backward propagation or auto differentiation for gradients' computation. @@ -96,24 +108,25 @@ def backward(self, if self._use_pure_bf16: self._to_bf16_var_names = cast_model_to_bf16( - self._train_program, startup_program, self._amp_lists, - self._use_bf16_guard) + self._train_program, + startup_program, + self._amp_lists, + self._use_bf16_guard, + ) else: rewrite_program_bf16(self._train_program, self._amp_lists) if loss.dtype != core.VarDesc.VarType.FP32: loss = loss.astype('float32') - params_grads = self._optimizer.backward(loss, startup_program, - parameter_list, no_grad_set, - callbacks) + params_grads = self._optimizer.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) return params_grads - def amp_init(self, - place, - scope=None, - test_program=None, - use_bf16_test=False): + def amp_init( + self, place, scope=None, test_program=None, use_bf16_test=False + ): """ Init the amp training, such as cast fp32 parameters to bf16 type. @@ -165,16 +178,20 @@ def run_example_code(): optimizer.amp_init(place, scope=paddle.static.global_scope()) """ - assert self._train_program is not None, \ - "Please call the minimize method first." + assert ( + self._train_program is not None + ), "Please call the minimize method first." if self._use_pure_bf16: - cast_parameters_to_bf16(place, self._train_program, scope, - self._to_bf16_var_names) + cast_parameters_to_bf16( + place, self._train_program, scope, self._to_bf16_var_names + ) if test_program is not None: if self._use_pure_bf16: - cast_model_to_bf16(test_program, - amp_lists=self._amp_lists, - use_bf16_guard=self._use_bf16_guard) + cast_model_to_bf16( + test_program, + amp_lists=self._amp_lists, + use_bf16_guard=self._use_bf16_guard, + ) elif use_bf16_test: rewrite_program_bf16(test_program, amp_lists=self._amp_lists) @@ -197,11 +214,9 @@ def apply_optimize(self, loss, startup_program, params_grads): optimize_ops = self.apply_gradients(params_grads) return optimize_ops - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): """ Perform optimization by minimizing the given loss. @@ -217,26 +232,28 @@ def minimize(self, list of scaled parameters and gradients. """ opt_dict = self._optimizer.__class__.__dict__ - if 'minimize' in opt_dict and isinstance(opt_dict['minimize'], - types.FunctionType): + if 'minimize' in opt_dict and isinstance( + opt_dict['minimize'], types.FunctionType + ): warnings.warn( "The decorated optimizer has its own `minimize` method, but it will not be executed." ) - params_grads = self.backward(loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) + params_grads = self.backward( + loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) optimize_ops = self.apply_optimize(loss, startup_program, params_grads) return optimize_ops, params_grads -def decorate_bf16(optimizer, - amp_lists=None, - use_pure_bf16=False, - use_bf16_guard=None): +def decorate_bf16( + optimizer, amp_lists=None, use_pure_bf16=False, use_bf16_guard=None +): """ Decorate the given optimizer to adapt to the mixed-precision training. @@ -252,7 +269,7 @@ def decorate_bf16(optimizer, enabled. Examples 1: - .. code-block:: python + .. code-block:: python # fp32&bf16 list based strategy example import paddle @@ -316,7 +333,8 @@ def run_example_code(): if use_bf16_guard is None: use_bf16_guard = use_pure_bf16 - mp_optimizer = OptimizerWithMixedPrecision(optimizer, amp_lists, - use_pure_bf16, use_bf16_guard) + mp_optimizer = OptimizerWithMixedPrecision( + optimizer, amp_lists, use_pure_bf16, use_bf16_guard + ) return mp_optimizer diff --git a/python/paddle/fluid/contrib/mixed_precision/decorator.py b/python/paddle/fluid/contrib/mixed_precision/decorator.py index 70aefac5a5996..75554ff9c8123 100644 --- a/python/paddle/fluid/contrib/mixed_precision/decorator.py +++ b/python/paddle/fluid/contrib/mixed_precision/decorator.py @@ -63,10 +63,19 @@ class OptimizerWithMixedPrecision(object): """ - def __init__(self, optimizer, amp_lists, init_loss_scaling, - use_dynamic_loss_scaling, incr_every_n_steps, - decr_every_n_nan_or_inf, incr_ratio, decr_ratio, use_pure_fp16, - use_fp16_guard): + def __init__( + self, + optimizer, + amp_lists, + init_loss_scaling, + use_dynamic_loss_scaling, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + use_pure_fp16, + use_fp16_guard, + ): self._optimizer = optimizer self._amp_lists = amp_lists self._param_grads = None @@ -97,9 +106,10 @@ def _set_distributed(self, flag): self._is_distributed = flag def get_loss_scaling(self): - """Return the real-time loss scaling factor. - """ - assert self._loss_scaling is not None, 'Please call minimize() before calling get_loss_scaling().' + """Return the real-time loss scaling factor.""" + assert ( + self._loss_scaling is not None + ), 'Please call minimize() before calling get_loss_scaling().' return self._loss_scaling def get_scaled_loss(self): @@ -117,7 +127,8 @@ def _init_amp_var(self): shape=[1], value=self._init_loss_scaling, dtype='float32', - persistable=True) + persistable=True, + ) if self._use_dynamic_loss_scaling: self._num_good_steps = layers.create_global_var( @@ -125,31 +136,37 @@ def _init_amp_var(self): shape=[1], value=0, dtype='int32', - persistable=True) + persistable=True, + ) self._num_bad_steps = layers.create_global_var( name=unique_name.generate("num_bad_steps"), shape=[1], value=0, dtype='int32', - persistable=True) + persistable=True, + ) # Ensure the data type of learning rate vars is float32 (same as the # master parameter dtype) if isinstance(self._optimizer._learning_rate, float): - self._optimizer._learning_rate_map[default_main_program()] = \ - layers.create_global_var( - name=unique_name.generate("learning_rate"), - shape=[1], - value=float(self._optimizer._learning_rate), - dtype='float32', - persistable=True) - - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + self._optimizer._learning_rate_map[ + default_main_program() + ] = layers.create_global_var( + name=unique_name.generate("learning_rate"), + shape=[1], + value=float(self._optimizer._learning_rate), + dtype='float32', + persistable=True, + ) + + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): """ Backward propagation or auto differentiation for gradients' computation. @@ -171,9 +188,9 @@ def backward(self, # NOTE(zhiqiu): _float_status is only used for NPU. if core.is_compiled_with_npu(): - float_status = paddle.static.data(name="float_status", - shape=[8], - dtype='float32') + float_status = paddle.static.data( + name="float_status", shape=[8], dtype='float32' + ) self._train_program.global_block().append_op( type="alloc_float_status", outputs={"FloatStatus": float_status}, @@ -192,7 +209,8 @@ def backward(self, if self._use_pure_fp16: self._to_fp16_var_names = cast_model_to_fp16( - self._train_program, self._amp_lists, self._use_fp16_guard) + self._train_program, self._amp_lists, self._use_fp16_guard + ) else: rewrite_program(self._train_program, self._amp_lists) @@ -205,10 +223,13 @@ def backward(self, else: self._scaled_loss = loss - params_grads = self._optimizer.backward(self._scaled_loss, - startup_program, - parameter_list, no_grad_set, - callbacks) + params_grads = self._optimizer.backward( + self._scaled_loss, + startup_program, + parameter_list, + no_grad_set, + callbacks, + ) if self._supports_check_nan_inf(): self._add_cast_ops_to_startup_program(startup_program) return params_grads @@ -216,8 +237,11 @@ def backward(self, def _add_cast_ops_to_startup_program(self, startup_program): names = list(self._to_fp16_var_names) if self._to_fp16_var_names else [] names.sort() - startup_program = default_startup_program( - ) if startup_program is None else startup_program + startup_program = ( + default_startup_program() + if startup_program is None + else startup_program + ) block = startup_program.global_block() param_names = [p.name for p in block.all_parameters()] for name in names: @@ -225,23 +249,23 @@ def _add_cast_ops_to_startup_program(self, startup_program): continue tmp = block.create_var(dtype=core.VarDesc.VarType.FP32) - block.append_op(type='assign', - inputs={'X': [name]}, - outputs={'Out': [tmp]}) - block.append_op(type='cast', - inputs={'X': [tmp]}, - outputs={'Out': [name]}, - attrs={ - 'in_dtype': core.VarDesc.VarType.FP32, - 'out_dtype': core.VarDesc.VarType.FP16, - }) + block.append_op( + type='assign', inputs={'X': [name]}, outputs={'Out': [tmp]} + ) + block.append_op( + type='cast', + inputs={'X': [tmp]}, + outputs={'Out': [name]}, + attrs={ + 'in_dtype': core.VarDesc.VarType.FP32, + 'out_dtype': core.VarDesc.VarType.FP16, + }, + ) self._to_fp16_var_names = None - def amp_init(self, - place, - scope=None, - test_program=None, - use_fp16_test=False): + def amp_init( + self, place, scope=None, test_program=None, use_fp16_test=False + ): """ Init the amp training, such as cast fp32 parameters to fp16 type. @@ -297,15 +321,18 @@ def run_example_code(): if paddle.is_compiled_with_cuda() and len(paddle.static.cuda_places()) > 0: run_example_code() """ - assert self._train_program is not None, \ - "Please call the minimize method first." + assert ( + self._train_program is not None + ), "Please call the minimize method first." if self._use_pure_fp16: - cast_parameters_to_fp16(place, self._train_program, scope, - self._to_fp16_var_names) + cast_parameters_to_fp16( + place, self._train_program, scope, self._to_fp16_var_names + ) if test_program is not None: if self._use_pure_fp16: - cast_model_to_fp16(test_program, self._amp_lists, - self._use_fp16_guard) + cast_model_to_fp16( + test_program, self._amp_lists, self._use_fp16_guard + ) elif use_fp16_test: rewrite_program(test_program, self._amp_lists) @@ -327,7 +354,10 @@ def apply_gradients(self, params_grads): # When not using dynamic loss scaling and the init loss scaling value is equal to 1.0, # the model can be optimized. - if not self._use_dynamic_loss_scaling and self._init_loss_scaling == 1.0: + if ( + not self._use_dynamic_loss_scaling + and self._init_loss_scaling == 1.0 + ): return self._optimizer.apply_gradients(params_grads) if self._supports_check_nan_inf(): @@ -346,13 +376,16 @@ def apply_gradients(self, params_grads): real_optimizer = self._optimizer while hasattr(real_optimizer, "inner_opt"): real_optimizer = real_optimizer.inner_opt - if isinstance(real_optimizer, - (paddle.fluid.optimizer.Adam, paddle.optimizer.AdamW)): + if isinstance( + real_optimizer, + (paddle.fluid.optimizer.Adam, paddle.optimizer.AdamW), + ): # NOTE(zhiqiu): Since found_inf needs to be on cpu in adam op, we # copy it in advance to avoid multiple time copies. with self._train_program._optimized_guard([]): found_inf = paddle.tensor.creation._memcpy( - found_inf, paddle.CPUPlace()) + found_inf, paddle.CPUPlace() + ) real_optimizer._set_auxiliary_var('found_inf', found_inf) elif hasattr(real_optimizer, "_set_auxiliary_var"): real_optimizer._set_auxiliary_var('found_inf', found_inf) @@ -363,8 +396,9 @@ def _split_grads(self, params_grads): grads = [g for _, g in params_grads] fp32_grads = [g for g in grads if g.dtype == core.VarDesc.VarType.FP32] fp16_grads = [g for g in grads if g.dtype == core.VarDesc.VarType.FP16] - assert len(fp32_grads) + len(fp16_grads) == len(grads), \ - "Data types of all grads must be either fp16 or fp32." + assert len(fp32_grads) + len(fp16_grads) == len( + grads + ), "Data types of all grads must be either fp16 or fp32." return grads, fp32_grads, fp16_grads def _check_finite_and_unscale(self, params_grads): @@ -380,7 +414,8 @@ def _check_finite_and_unscale(self, params_grads): grads, self._loss_scaling, name="find_infinite_scale", - float_status=self._float_status) + float_status=self._float_status, + ) found_infs.append(found_inf) else: for p, g in params_grads: @@ -391,7 +426,8 @@ def _check_finite_and_unscale(self, params_grads): ], self._loss_scaling, name="find_infinite_scale", - float_status=self._float_status) + float_status=self._float_status, + ) found_infs.append(found_inf) elif self._use_pure_fp16: if fp32_grads: @@ -400,7 +436,8 @@ def _check_finite_and_unscale(self, params_grads): fp32_grads, self._loss_scaling, name="find_infinite_scale_fp32", - float_status=self._float_status) + float_status=self._float_status, + ) found_infs.append(fp32_found_inf) if fp16_grads: with self._train_program._optimized_guard(fp16_grads): @@ -408,7 +445,8 @@ def _check_finite_and_unscale(self, params_grads): fp16_grads, self._loss_scaling, name="find_infinite_scale_fp16", - float_status=self._float_status) + float_status=self._float_status, + ) found_infs.append(fp16_found_inf) else: with self._train_program._optimized_guard(grads): @@ -416,7 +454,8 @@ def _check_finite_and_unscale(self, params_grads): grads, self._loss_scaling, name="find_infinite_scale", - float_status=self._float_status) + float_status=self._float_status, + ) if self._is_distributed or self._use_pure_fp16: with self._train_program._optimized_guard([]): @@ -439,7 +478,8 @@ def _add_dynamic_loss_scaling(self, params_grads, found_inf): self._incr_ratio, self._decr_ratio, stop_update=self._optimizer._get_stop_update_var(), - name="update_loss_scaling") + name="update_loss_scaling", + ) return grads, fp32_grads, fp16_grads = self._split_grads(params_grads) @@ -447,42 +487,48 @@ def _add_dynamic_loss_scaling(self, params_grads, found_inf): stop_update = False with self._train_program._optimized_guard([]): if fp32_grads: - update_loss_scaling(fp32_grads, - found_inf, - self._loss_scaling, - self._num_good_steps, - self._num_bad_steps, - self._incr_every_n_steps, - self._decr_every_n_nan_or_inf, - self._incr_ratio, - self._decr_ratio, - stop_update=stop_update, - name="update_loss_scaling_fp32") + update_loss_scaling( + fp32_grads, + found_inf, + self._loss_scaling, + self._num_good_steps, + self._num_bad_steps, + self._incr_every_n_steps, + self._decr_every_n_nan_or_inf, + self._incr_ratio, + self._decr_ratio, + stop_update=stop_update, + name="update_loss_scaling_fp32", + ) stop_update = True if fp16_grads: - update_loss_scaling(fp16_grads, - found_inf, - self._loss_scaling, - self._num_good_steps, - self._num_bad_steps, - self._incr_every_n_steps, - self._decr_every_n_nan_or_inf, - self._incr_ratio, - self._decr_ratio, - stop_update=stop_update, - name="update_loss_scaling_fp16") + update_loss_scaling( + fp16_grads, + found_inf, + self._loss_scaling, + self._num_good_steps, + self._num_bad_steps, + self._incr_every_n_steps, + self._decr_every_n_nan_or_inf, + self._incr_ratio, + self._decr_ratio, + stop_update=stop_update, + name="update_loss_scaling_fp16", + ) else: with self._train_program._optimized_guard([]): - update_loss_scaling(grads, - found_inf, - self._loss_scaling, - self._num_good_steps, - self._num_bad_steps, - self._incr_every_n_steps, - self._decr_every_n_nan_or_inf, - self._incr_ratio, - self._decr_ratio, - name="update_loss_scaling") + update_loss_scaling( + grads, + found_inf, + self._loss_scaling, + self._num_good_steps, + self._num_bad_steps, + self._incr_every_n_steps, + self._decr_every_n_nan_or_inf, + self._incr_ratio, + self._decr_ratio, + name="update_loss_scaling", + ) def apply_optimize(self, loss, startup_program, params_grads): program = loss.block.program @@ -490,11 +536,9 @@ def apply_optimize(self, loss, startup_program, params_grads): optimize_ops = self.apply_gradients(params_grads) return optimize_ops - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): """ Perform optimization by minimizing the given loss. @@ -511,33 +555,39 @@ def minimize(self, """ opt_dict = self._optimizer.__class__.__dict__ - if 'minimize' in opt_dict and isinstance(opt_dict['minimize'], - types.FunctionType): + if 'minimize' in opt_dict and isinstance( + opt_dict['minimize'], types.FunctionType + ): warnings.warn( "The decorated optimizer has its own `minimize` method, but it will not be executed." ) - scaled_params_grads = self.backward(loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) + scaled_params_grads = self.backward( + loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) - optimize_ops = self.apply_optimize(loss, startup_program, - scaled_params_grads) + optimize_ops = self.apply_optimize( + loss, startup_program, scaled_params_grads + ) return optimize_ops, scaled_params_grads -def decorate(optimizer, - amp_lists=None, - init_loss_scaling=2**15, - incr_every_n_steps=1000, - decr_every_n_nan_or_inf=2, - incr_ratio=2.0, - decr_ratio=0.8, - use_dynamic_loss_scaling=True, - use_pure_fp16=False, - use_fp16_guard=None): +def decorate( + optimizer, + amp_lists=None, + init_loss_scaling=2**15, + incr_every_n_steps=1000, + decr_every_n_nan_or_inf=2, + incr_ratio=2.0, + decr_ratio=0.8, + use_dynamic_loss_scaling=True, + use_pure_fp16=False, + use_fp16_guard=None, +): """ Decorate the given optimizer to adapt to the mixed-precision training. @@ -564,7 +614,7 @@ def decorate(optimizer, enabled. Examples 1: - .. code-block:: python + .. code-block:: python # black&white list based strategy example import paddle @@ -635,8 +685,16 @@ def run_example_code(): use_fp16_guard = use_pure_fp16 mp_optimizer = OptimizerWithMixedPrecision( - optimizer, amp_lists, init_loss_scaling, use_dynamic_loss_scaling, - incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, decr_ratio, - use_pure_fp16, use_fp16_guard) + optimizer, + amp_lists, + init_loss_scaling, + use_dynamic_loss_scaling, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + use_pure_fp16, + use_fp16_guard, + ) return mp_optimizer diff --git a/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py b/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py index b2767b1dd1cbf..ef8f222bac23b 100644 --- a/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py +++ b/python/paddle/fluid/contrib/mixed_precision/fp16_lists.py @@ -19,7 +19,10 @@ # lookup_table fp16 is slower than fp32, though fp16 is supported. _extra_unsupported_fp16_list = { - 'lookup_table', 'lookup_table_v2', 'scatter', 'scatter_grad' + 'lookup_table', + 'lookup_table_v2', + 'scatter', + 'scatter_grad', } @@ -36,10 +39,12 @@ class AutoMixedPrecisionLists(object): custom_black_varnames (set): Users' custom black varibles' names. """ - def __init__(self, - custom_white_list=None, - custom_black_list=None, - custom_black_varnames=None): + def __init__( + self, + custom_white_list=None, + custom_black_list=None, + custom_black_varnames=None, + ): self._custom_white_list = custom_white_list self._custom_black_list = custom_black_list self.white_list = copy.copy(white_list) @@ -56,8 +61,9 @@ def _update_list(self): if self._custom_white_list and self._custom_black_list: for op_name in self._custom_white_list: if op_name in self._custom_black_list: - raise ValueError("Custom white list overlap " - "custom black list") + raise ValueError( + "Custom white list overlap " "custom black list" + ) if self._custom_white_list: for op_name in self._custom_white_list: if op_name in self.black_list: @@ -175,17 +181,23 @@ def _update_list(self): _sys_unsupported_fp16_list = [] if core.is_compiled_with_xpu(): _, _, _sys_unsupported_fp16_list = core.op_supported_infos( - 'XPU', core.VarDesc.VarType.FP16) + 'XPU', core.VarDesc.VarType.FP16 + ) elif core.is_compiled_with_npu(): _, _, _sys_unsupported_fp16_list = core.op_supported_infos( - 'NPU', core.VarDesc.VarType.FP16) + 'NPU', core.VarDesc.VarType.FP16 + ) elif core.is_compiled_with_mlu(): _, _, _sys_unsupported_fp16_list = core.op_supported_infos( - 'MLU', core.VarDesc.VarType.FP16) + 'MLU', core.VarDesc.VarType.FP16 + ) else: _, _, _sys_unsupported_fp16_list = core.op_supported_infos( - 'GPU', core.VarDesc.VarType.FP16) + 'GPU', core.VarDesc.VarType.FP16 + ) -unsupported_fp16_list = _extra_unsupported_fp16_list | _sys_unsupported_fp16_list +unsupported_fp16_list = ( + _extra_unsupported_fp16_list | _sys_unsupported_fp16_list +) CustomOpLists = AutoMixedPrecisionLists diff --git a/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py b/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py index ad91e8ba799b6..e50a2457147e1 100644 --- a/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py +++ b/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py @@ -25,13 +25,14 @@ __all__ = ["fp16_guard", "cast_model_to_fp16", "cast_parameters_to_fp16"] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) _valid_types = [ - core.VarDesc.VarType.LOD_TENSOR, core.VarDesc.VarType.SELECTED_ROWS, - core.VarDesc.VarType.LOD_TENSOR_ARRAY + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.SELECTED_ROWS, + core.VarDesc.VarType.LOD_TENSOR_ARRAY, ] _fp16_guard_pattern = "__use_fp16__" @@ -106,7 +107,12 @@ def _keep_fp32_input(op, in_name): return in_name not in {'X', 'FilterX', 'Z', 'FilterZ'} if op_type in ['fused_attention', 'fused_feedforward']: return in_name in { - 'LnScale', 'LnBias', 'Ln2Scale', 'Ln2Bias', "Ln1Scale", "Ln1Bias" + 'LnScale', + 'LnBias', + 'Ln2Scale', + 'Ln2Bias', + "Ln1Scale", + "Ln1Bias", } if op_type == 'fused_multi_transformer': return in_name in {'LnScale', 'LnBias', 'FFNLnScale', 'FFNLnBias'} @@ -123,8 +129,12 @@ def _keep_fp32_output(op, out_name): return out_name not in {'Y', 'ConvX', 'ConvZ'} if op_type in ['fused_attention', 'fused_feedforward']: return out_name in { - 'LnMean', 'LnVariance', 'Ln2Mean', 'Ln2Variance', 'Ln1Mean', - 'Ln1Variance' + 'LnMean', + 'LnVariance', + 'Ln2Mean', + 'Ln2Variance', + 'Ln1Mean', + 'Ln1Variance', } return False @@ -147,7 +157,8 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): for in_name in op.input_names: if src_dtype == core.VarDesc.VarType.FP32 and _keep_fp32_input( - op, in_name): + op, in_name + ): continue for in_var_name in op.input(in_name): in_var = block._find_var_recursive(in_var_name) @@ -163,11 +174,15 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): # set cast_op device to `all`, can reduce send cast_var. # TODO: need remove this after we unified the dynamic # and static pipeline interface. - if src_dtype == core.VarDesc.VarType.FP32 and in_var.stop_gradient: + if ( + src_dtype == core.VarDesc.VarType.FP32 + and in_var.stop_gradient + ): prev_op = None if in_var.op is op: - prev_op = find_true_prev_op(block.ops, op, - in_var_name) + prev_op = find_true_prev_op( + block.ops, op, in_var_name + ) elif in_var.op is not None: prev_op = in_var.op @@ -175,33 +190,40 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): if prev_op is not None: prev_op_device = prev_op.attr('op_device') - if prev_op_device is not None and 'all' in prev_op_device: + if ( + prev_op_device is not None + and 'all' in prev_op_device + ): op_device = prev_op_device out_var = block.create_var( name=cast_name, dtype=dest_dtype, persistable=False, - stop_gradient=in_var.stop_gradient) - - block._insert_op_without_sync(idx, - type="cast", - inputs={"X": in_var}, - outputs={"Out": out_var}, - attrs={ - "in_dtype": in_var.dtype, - "out_dtype": - out_var.dtype, - "op_device": op_device, - "op_role": - op.attr("op_role"), - }) + stop_gradient=in_var.stop_gradient, + ) + + block._insert_op_without_sync( + idx, + type="cast", + inputs={"X": in_var}, + outputs={"Out": out_var}, + attrs={ + "in_dtype": in_var.dtype, + "out_dtype": out_var.dtype, + "op_device": op_device, + "op_role": op.attr("op_role"), + }, + ) num_cast_ops += 1 _rename_arg(op, in_var.name, out_var.name) else: if op.has_attr('in_dtype'): op._set_attr('in_dtype', dest_dtype) - if src_dtype == core.VarDesc.VarType.FP32 and dest_dtype == core.VarDesc.VarType.FP16: + if ( + src_dtype == core.VarDesc.VarType.FP32 + and dest_dtype == core.VarDesc.VarType.FP16 + ): for out_name in op.output_names: if _keep_fp32_output(op, out_name): continue @@ -216,35 +238,42 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): return num_cast_ops -def _insert_cast_post_op(block, op, idx, src_dtype, dest_dtype, target_name, - op_var_rename_map): +def _insert_cast_post_op( + block, op, idx, src_dtype, dest_dtype, target_name, op_var_rename_map +): num_cast_ops = 0 target_var = block.var(target_name) if target_var.type not in _valid_types or target_var.dtype == dest_dtype: return num_cast_ops - assert target_var.dtype == src_dtype, \ - "The real dtype({}) is not equal to the src dtype({})".format( - _dtype_to_str(target_var.dtype), _dtype_to_str(src_dtype)) + assert ( + target_var.dtype == src_dtype + ), "The real dtype({}) is not equal to the src dtype({})".format( + _dtype_to_str(target_var.dtype), _dtype_to_str(src_dtype) + ) cast_name = target_var.name + '.cast_' + _dtype_to_str(dest_dtype) cast_var = block.vars.get(cast_name) if cast_var is None or cast_var.dtype != dest_dtype: - cast_var = block.create_var(name=cast_name, - dtype=dest_dtype, - persistable=False, - stop_gradient=target_var.stop_gradient) - block._insert_op(idx, - type="cast", - inputs={"X": target_var}, - outputs={"Out": cast_var}, - attrs={ - "in_dtype": target_var.dtype, - "out_dtype": cast_var.dtype, - "op_device": op.attr("op_device"), - "op_role": op.attr("op_role"), - }) + cast_var = block.create_var( + name=cast_name, + dtype=dest_dtype, + persistable=False, + stop_gradient=target_var.stop_gradient, + ) + block._insert_op( + idx, + type="cast", + inputs={"X": target_var}, + outputs={"Out": cast_var}, + attrs={ + "in_dtype": target_var.dtype, + "out_dtype": cast_var.dtype, + "op_device": op.attr("op_device"), + "op_role": op.attr("op_role"), + }, + ) num_cast_ops += 1 op_var_rename_map[block.idx][target_var.name] = cast_var.name @@ -270,8 +299,10 @@ def find_true_prev_op(ops, cur_op, var_name): prev_op.append(op) if prev_op: if not len(prev_op) == 1: - raise ValueError("There must be only one previous op " - "that outputs {0} variable".format(var_name)) + raise ValueError( + "There must be only one previous op " + "that outputs {0} variable".format(var_name) + ) else: return prev_op[0] return None @@ -313,8 +344,7 @@ def find_true_post_op(ops, cur_op, var_name, search_all=False): def find_op_index(block_desc, cur_op_desc): - """ - """ + """ """ for idx in range(block_desc.op_size()): if cur_op_desc == block_desc.op(idx): return idx @@ -348,8 +378,9 @@ def _need_keep_fp32(op, unsupported_op_list, use_fp16_guard): return True if use_fp16_guard: - if op.has_attr("op_namescope") and \ - (_fp16_guard_pattern in op.attr("op_namescope")): + if op.has_attr("op_namescope") and ( + _fp16_guard_pattern in op.attr("op_namescope") + ): # op in fp16 guard return False else: @@ -419,7 +450,8 @@ def cast_model_to_fp16(program, amp_lists=None, use_fp16_guard=True): for in_name in op.input_names: # for ipu, all inputs must be converted to fp16 if not core.is_compiled_with_ipu() and _keep_fp32_input( - op, in_name): + op, in_name + ): continue for in_var_name in op.input(in_name): in_var = None @@ -427,13 +459,17 @@ def cast_model_to_fp16(program, amp_lists=None, use_fp16_guard=True): in_var = block.var(in_var_name) except ValueError as e: _logger.debug( - "-- {}, try to get it in the global block --". - format(e)) + "-- {}, try to get it in the global block --".format( + e + ) + ) in_var = global_block.var(in_var_name) if in_var is not None: _logger.debug( - "-- var {} is got in the global block --". - format(in_var_name)) + "-- var {} is got in the global block --".format( + in_var_name + ) + ) if in_var is None or in_var.type not in _valid_types: continue @@ -443,13 +479,16 @@ def cast_model_to_fp16(program, amp_lists=None, use_fp16_guard=True): to_fp16_var_names.add(in_var_name) _logger.debug( - "-- op type: {}, in var name: {}, in var dtype: {} --". - format(op.type, in_var_name, in_var.dtype)) + "-- op type: {}, in var name: {}, in var dtype: {} --".format( + op.type, in_var_name, in_var.dtype + ) + ) for out_name in op.output_names: # for ipu, all outputs must be converted to fp16 if not core.is_compiled_with_ipu() and _keep_fp32_output( - op, out_name): + op, out_name + ): continue for out_var_name in op.output(out_name): out_var = None @@ -457,13 +496,17 @@ def cast_model_to_fp16(program, amp_lists=None, use_fp16_guard=True): out_var = block.var(out_var_name) except ValueError as e: _logger.debug( - "-- {}, try to get it in the global block --". - format(e)) + "-- {}, try to get it in the global block --".format( + e + ) + ) out_var = global_block.var(out_var_name) if out_var is not None: _logger.debug( - "-- var {} is got in the global block --". - format(out_var_name)) + "-- var {} is got in the global block --".format( + out_var_name + ) + ) if out_var is None or out_var.type not in _valid_types: continue @@ -472,16 +515,24 @@ def cast_model_to_fp16(program, amp_lists=None, use_fp16_guard=True): out_var.desc.set_dtype(core.VarDesc.VarType.FP16) _logger.debug( - "-- op type: {}, out var name: {}, out var dtype: {} --" - .format(op.type, out_var_name, out_var.dtype)) - if op.has_attr('in_dtype') and op.attr( - 'in_dtype') == core.VarDesc.VarType.FP32: + "-- op type: {}, out var name: {}, out var dtype: {} --".format( + op.type, out_var_name, out_var.dtype + ) + ) + if ( + op.has_attr('in_dtype') + and op.attr('in_dtype') == core.VarDesc.VarType.FP32 + ): op._set_attr('in_dtype', core.VarDesc.VarType.FP16) - if op.has_attr('out_dtype') and op.attr( - 'out_dtype') == core.VarDesc.VarType.FP32: + if ( + op.has_attr('out_dtype') + and op.attr('out_dtype') == core.VarDesc.VarType.FP32 + ): op._set_attr('out_dtype', core.VarDesc.VarType.FP16) - if op.has_attr('dtype') and op.attr( - 'dtype') == core.VarDesc.VarType.FP32: + if ( + op.has_attr('dtype') + and op.attr('dtype') == core.VarDesc.VarType.FP32 + ): op._set_attr('dtype', core.VarDesc.VarType.FP16) # process ops in keep_fp32_ops @@ -495,9 +546,13 @@ def cast_model_to_fp16(program, amp_lists=None, use_fp16_guard=True): op = ops[idx] num_cast_ops = 0 if op in keep_fp32_ops: - pre_cast_num = _insert_cast_op(block, op, idx, - core.VarDesc.VarType.FP16, - core.VarDesc.VarType.FP32) + pre_cast_num = _insert_cast_op( + block, + op, + idx, + core.VarDesc.VarType.FP16, + core.VarDesc.VarType.FP32, + ) num_cast_ops += pre_cast_num for out_var_name in op.output_arg_names: out_var = block.vars.get(out_var_name) @@ -510,10 +565,14 @@ def cast_model_to_fp16(program, amp_lists=None, use_fp16_guard=True): if post_op in keep_fp32_ops: continue post_cast_num = _insert_cast_post_op( - block, op, idx + pre_cast_num + 1, + block, + op, + idx + pre_cast_num + 1, core.VarDesc.VarType.FP32, - core.VarDesc.VarType.FP16, out_var_name, - op_var_rename_map) + core.VarDesc.VarType.FP16, + out_var_name, + op_var_rename_map, + ) num_cast_ops += post_cast_num idx += num_cast_ops + 1 @@ -583,7 +642,8 @@ def rewrite_program(main_prog, amp_lists): continue if amp_lists.black_varnames is not None and _is_in_black_varnames( - op, amp_lists): + op, amp_lists + ): black_op_set.add(op) continue @@ -609,11 +669,15 @@ def rewrite_program(main_prog, amp_lists): else: prev_op = in_var.op # if it's one of inputs - if prev_op in black_op_set or \ - prev_op.type in amp_lists.black_list: + if ( + prev_op in black_op_set + or prev_op.type in amp_lists.black_list + ): is_black_op = True - elif prev_op in white_op_set or \ - prev_op.type in amp_lists.white_list: + elif ( + prev_op in white_op_set + or prev_op.type in amp_lists.white_list + ): is_white_op = True if is_black_op: black_op_set.add(op) @@ -631,13 +695,21 @@ def rewrite_program(main_prog, amp_lists): op = ops[idx] num_cast_ops = 0 if op in black_op_set: - num_cast_ops = _insert_cast_op(block, op, idx, - core.VarDesc.VarType.FP16, - core.VarDesc.VarType.FP32) + num_cast_ops = _insert_cast_op( + block, + op, + idx, + core.VarDesc.VarType.FP16, + core.VarDesc.VarType.FP32, + ) elif op in white_op_set: - num_cast_ops = _insert_cast_op(block, op, idx, - core.VarDesc.VarType.FP32, - core.VarDesc.VarType.FP16) + num_cast_ops = _insert_cast_op( + block, + op, + idx, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP16, + ) else: pass @@ -668,13 +740,16 @@ def update_role_var_grad(main_prog, params_grads): if role & int(BACKWARD) and op.has_attr('op_role_var'): op._remove_attr("op_role_var") else: - raise ValueError("The cast op {0} must be in BACKWARD role " - "and have op_role_var attr.".format(op)) + raise ValueError( + "The cast op {0} must be in BACKWARD role " + "and have op_role_var attr.".format(op) + ) fp16_grad_name = op.input(op.input_names[0])[0] op_for_fp16_grad = find_true_prev_op(block.ops, op, fp16_grad_name) - op_role_var_attr_name = \ + op_role_var_attr_name = ( core.op_proto_and_checker_maker.kOpRoleVarAttrName() + ) attr_val = [p.name, fp16_grad_name] if op_for_fp16_grad.has_attr(op_role_var_attr_name): attr_val.extend(op_for_fp16_grad.attr(op_role_var_attr_name)) @@ -688,18 +763,22 @@ def update_role_var_grad(main_prog, params_grads): continue post_ops = find_true_post_op(block.ops, op, g.name) if post_ops: - raise ValueError("The cast op {0}'s output should not be" - "used by a non-optimize op, however, it" - "is used by {1}".format(op, post_ops[0])) + raise ValueError( + "The cast op {0}'s output should not be" + "used by a non-optimize op, however, it" + "is used by {1}".format(op, post_ops[0]) + ) # add new op in the python and cpp at the same time new_op_desc = block.desc.append_op() new_op_desc.copy_from(op.desc) - new_op = framework.Operator(block=block, - desc=new_op_desc, - type=None, - inputs=None, - outputs=None, - attrs=None) + new_op = framework.Operator( + block=block, + desc=new_op_desc, + type=None, + inputs=None, + outputs=None, + attrs=None, + ) block.ops.append(new_op) op_idx = find_op_index(block.desc, op.desc) if op_idx == -1: diff --git a/python/paddle/fluid/contrib/model_stat.py b/python/paddle/fluid/contrib/model_stat.py index 3bcc330689545..0c18deec5da22 100644 --- a/python/paddle/fluid/contrib/model_stat.py +++ b/python/paddle/fluid/contrib/model_stat.py @@ -151,8 +151,10 @@ def _format_summary(collected_ops_list): _verify_dependent_package() from prettytable import PrettyTable + summary_table = PrettyTable( - ["No.", "TYPE", "INPUT", "OUTPUT", "PARAMs", "FLOPs"]) + ["No.", "TYPE", "INPUT", "OUTPUT", "PARAMs", "FLOPs"] + ) summary_table.align = 'r' total = {} @@ -200,8 +202,9 @@ def _print_summary(summary_table, total): parmas = total['params'] flops = total['flops'] print(summary_table) - print('Total PARAMs: {}({:.4f}M)'.format(sum(parmas), - sum(parmas) / (10**6))) + print( + 'Total PARAMs: {}({:.4f}M)'.format(sum(parmas), sum(parmas) / (10**6)) + ) print('Total FLOPs: {}({:.2f}G)'.format(sum(flops), sum(flops) / 10**9)) print( "Notice: \n now supported ops include [Conv, DepthwiseConv, FC(mul), BatchNorm, Pool, Activation(sigmoid, tanh, relu, leaky_relu, prelu)]" diff --git a/python/paddle/fluid/contrib/op_frequence.py b/python/paddle/fluid/contrib/op_frequence.py index 5203aa308e312..ee435ac657af3 100644 --- a/python/paddle/fluid/contrib/op_frequence.py +++ b/python/paddle/fluid/contrib/op_frequence.py @@ -43,8 +43,10 @@ def op_freq_statistic(program): """ if not isinstance(program, Program): - raise TypeError("The input type should be Porgram." - "But you passed in %s" % (type(program))) + raise TypeError( + "The input type should be Porgram." + "But you passed in %s" % (type(program)) + ) uni_op_freq = OrderedDict() adj_2_op_freq = OrderedDict() @@ -78,8 +80,10 @@ def op_freq_statistic(program): else: op_in_ops[op.type] = [var_gen_op[var_name][-1]] else: - print("Var's generate op is not found,%s, %s" % - (var_name, op.type)) + print( + "Var's generate op is not found,%s, %s" + % (var_name, op.type) + ) for var_name in op.output_arg_names: if var_gen_op.has_key(var_name): @@ -95,11 +99,11 @@ def op_freq_statistic(program): else: adj_2_op_freq[op_op] = 1 - uni_op_freq = sorted(uni_op_freq.items(), - key=lambda item: item[1], - reverse=True) - adj_2_op_freq = sorted(adj_2_op_freq.items(), - key=lambda item: item[1], - reverse=True) + uni_op_freq = sorted( + uni_op_freq.items(), key=lambda item: item[1], reverse=True + ) + adj_2_op_freq = sorted( + adj_2_op_freq.items(), key=lambda item: item[1], reverse=True + ) return uni_op_freq, adj_2_op_freq diff --git a/python/paddle/fluid/contrib/optimizer.py b/python/paddle/fluid/contrib/optimizer.py index dc6dc213a8f88..53386bc666640 100644 --- a/python/paddle/fluid/contrib/optimizer.py +++ b/python/paddle/fluid/contrib/optimizer.py @@ -104,31 +104,35 @@ class Momentum(Optimizer): """ _velocity_acc_str = "velocity" - def __init__(self, - learning_rate, - momentum, - parameter_list=None, - use_nesterov=False, - regularization=None, - grad_clip=None, - multi_precision=False, - rescale_grad=1.0, - name=None): + def __init__( + self, + learning_rate, + momentum, + parameter_list=None, + use_nesterov=False, + regularization=None, + grad_clip=None, + multi_precision=False, + rescale_grad=1.0, + name=None, + ): assert learning_rate is not None assert momentum is not None predicate = lambda regular: isinstance(regular, L2DecayRegularizer) py_regular = None if predicate(regularization) else regularization - super(Momentum, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=py_regular, - grad_clip=grad_clip, - name=name) + super(Momentum, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=py_regular, + grad_clip=grad_clip, + name=name, + ) self.type = "momentum" self._momentum = momentum self._use_nesterov = bool(use_nesterov) self._regularization_method = "" self._regularization_coeff = 0 - if (isinstance(regularization, L2DecayRegularizer)): + if isinstance(regularization, L2DecayRegularizer): self._regularization_method = "l2_decay" self._regularization_coeff = regularization._regularization_coeff self._multi_precision = multi_precision @@ -140,19 +144,23 @@ def _create_master_weight(self, param): var_name = param.name + "_fp32_master" var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -168,15 +176,22 @@ def _get_accumulator(self, name, param): """ if self._name is not None: name = self._name + "_" + name - find_master = self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 - target_param = self._master_weights[ - param.name] if find_master else param + find_master = ( + self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 + ) + target_param = ( + self._master_weights[param.name] if find_master else param + ) target_name = target_param.name - if (name not in self._accumulators - or target_name not in self._accumulators[name]): + if ( + name not in self._accumulators + or target_name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, target_name)) + name, target_name + ) + ) return self._accumulators[name][target_name] def _create_accumulators(self, block, parameters): @@ -187,7 +202,10 @@ def _create_accumulators(self, block, parameters): master_p = self._create_master_weight(p) self._add_accumulator(self._velocity_acc_str, master_p) continue - if p.dtype == core.VarDesc.VarType.FP16 and not self._multi_precision: + if ( + p.dtype == core.VarDesc.VarType.FP16 + and not self._multi_precision + ): warnings.warn( "Accumulating with FP16 in optimizer can lead to poor accuracy or slow convergence." "Consider using multi_precision=True option of the Momentum optimizer." @@ -197,23 +215,42 @@ def _create_accumulators(self, block, parameters): def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) - velocity_acc = self._get_accumulator(self._velocity_acc_str, - param_and_grad[0]) + velocity_acc = self._get_accumulator( + self._velocity_acc_str, param_and_grad[0] + ) lr = self._create_param_lr(param_and_grad) - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 - master_weight = (self._master_weights[param_and_grad[0].name] - if find_master else None) + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) + master_weight = ( + self._master_weights[param_and_grad[0].name] + if find_master + else None + ) if framework._non_static_mode(): _, _, _ = _legacy_C_ops.momentum( - param_and_grad[0], param_and_grad[1], velocity_acc, lr, - master_weight, param_and_grad[0], velocity_acc, master_weight, - 'mu', self._momentum, 'use_nesterov', self._use_nesterov, - 'regularization_method', self._regularization_method, - 'regularization_coeff', self._regularization_coeff, - 'multi_precision', find_master) + param_and_grad[0], + param_and_grad[1], + velocity_acc, + lr, + master_weight, + param_and_grad[0], + velocity_acc, + master_weight, + 'mu', + self._momentum, + 'use_nesterov', + self._use_nesterov, + 'regularization_method', + self._regularization_method, + 'regularization_coeff', + self._regularization_coeff, + 'multi_precision', + find_master, + ) return None attrs = { @@ -222,17 +259,17 @@ def _append_optimize_op(self, block, param_and_grad): "regularization_method": self._regularization_method, "regularization_coeff": self._regularization_coeff, "multi_precision": find_master, - "rescale_grad": self._rescale_grad + "rescale_grad": self._rescale_grad, } inputs = { "Param": [param_and_grad[0]], "Grad": [param_and_grad[1]], "Velocity": [velocity_acc], - "LearningRate": [lr] + "LearningRate": [lr], } outputs = { "ParamOut": [param_and_grad[0]], - "VelocityOut": [velocity_acc] + "VelocityOut": [velocity_acc], } if find_master: @@ -240,10 +277,12 @@ def _append_optimize_op(self, block, param_and_grad): outputs["MasterParamOut"] = master_weight # create the momentum optimize op - momentum_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + momentum_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return momentum_op diff --git a/python/paddle/fluid/contrib/quantize/quantize_transpiler.py b/python/paddle/fluid/contrib/quantize/quantize_transpiler.py index de4c10040862a..6e225fdbcc8e6 100644 --- a/python/paddle/fluid/contrib/quantize/quantize_transpiler.py +++ b/python/paddle/fluid/contrib/quantize/quantize_transpiler.py @@ -15,7 +15,11 @@ import collections import numpy as np -from paddle.fluid.framework import default_main_program, default_startup_program, program_guard +from paddle.fluid.framework import ( + default_main_program, + default_startup_program, + program_guard, +) from paddle.fluid.layer_helper import LayerHelper from paddle.fluid import unique_name from paddle.fluid import core @@ -57,13 +61,13 @@ def _original_var_name(var_name): Return the original variable name. """ if var_name.endswith('.quantized.dequantized'): - return var_name[:-len('.quantized.dequantized')] + return var_name[: -len('.quantized.dequantized')] if var_name.endswith('.quantized'): - return var_name[:-len('.quantized')] + return var_name[: -len('.quantized')] if var_name.endswith('.dequantized'): - return var_name[:-len('.dequantized')] + return var_name[: -len('.dequantized')] if var_name.endswith('.scale'): - return var_name[:-len('.scale')] + return var_name[: -len('.scale')] else: return var_name @@ -78,14 +82,15 @@ def quant(x, scale, num_bits): class QuantizeTranspiler(object): - - def __init__(self, - weight_bits=8, - activation_bits=8, - activation_quantize_type='abs_max', - weight_quantize_type='abs_max', - window_size=10000, - moving_rate=0.9): + def __init__( + self, + weight_bits=8, + activation_bits=8, + activation_quantize_type='abs_max', + weight_quantize_type='abs_max', + window_size=10000, + moving_rate=0.9, + ): """ Convert and rewrite the fluid Program according to weight and activation quantization type. @@ -123,12 +128,14 @@ def __init__(self, raise ValueError( "Unknown weight_quantize_type: '%s'. It can only be ", "'abs_max' or 'range_abs_max' or 'moving_average_abs_max'.", - str(weight_quantize_type)) + str(weight_quantize_type), + ) if activation_quantize_type not in quant_type: raise ValueError( "Unknown activation_quantize_type : '%s'. It can only be ", "'abs_max' or 'range_abs_max' or 'moving_average_abs_max'.", - str(activation_quantize_type)) + str(activation_quantize_type), + ) self.weight_quantize_type = weight_quantize_type self.activation_quantize_type = activation_quantize_type @@ -137,8 +144,9 @@ def __init__(self, self.moving_rate = moving_rate self.helper = LayerHelper(self.__class__.__name__) self.fake_quant_op_types = [ - 'fake_quantize_abs_max', 'fake_quantize_range_abs_max', - 'fake_quantize_moving_average_abs_max' + 'fake_quantize_abs_max', + 'fake_quantize_range_abs_max', + 'fake_quantize_moving_average_abs_max', ] self.fake_dequant_op_types = ['fake_dequantize_max_abs'] self.is_test = None @@ -157,8 +165,11 @@ def training_transpile(self, program=None, startup_program=None): """ self.is_test = False program = default_main_program() if program is None else program - startup_program = default_startup_program() if startup_program is \ - None else startup_program + startup_program = ( + default_startup_program() + if startup_program is None + else startup_program + ) # marked the variable which has been quantized and dequantized. dequanted_vars = [ @@ -173,20 +184,28 @@ def _transpile_forward(block, op): block_id = block.idx # insert quant op and dequant op for name in op.input_arg_names: - #if share input between ops + # if share input between ops if name in dequanted_vars[block_id]: dequant_var = dequanted_vars[block_id][name] else: var = block.var(name) - quant_bits = self.weight_bits if var.name in params \ - else self.activation_bits - quant_type = self.weight_quantize_type if var.name \ - in params else self.activation_quantize_type + quant_bits = ( + self.weight_bits + if var.name in params + else self.activation_bits + ) + quant_type = ( + self.weight_quantize_type + if var.name in params + else self.activation_quantize_type + ) quant_var, scale_var = self._insert_quant_op( - block, idx, var, quant_bits, quant_type) + block, idx, var, quant_bits, quant_type + ) dequant_var = self._insert_dequant_op( - block, idx + 1, quant_var, scale_var, quant_bits) + block, idx + 1, quant_var, scale_var, quant_bits + ) dequanted_vars[block_id][name] = dequant_var # rename the forward op inputs op._rename_input(name, dequant_var.name) @@ -200,8 +219,9 @@ def _transpile_backward(block, op): op._rename_input(name, dequant_var.name) no_dequanted_input_vars = False if no_dequanted_input_vars: - raise ValueError("There is no dequanted inputs for op %s." % - (op.type)) + raise ValueError( + "There is no dequanted inputs for op %s." % (op.type) + ) with program_guard(program, startup_program): self._create_global_step() @@ -217,8 +237,10 @@ def _transpile_backward(block, op): _transpile_backward(block, op) def _create_global_step(self): - if self.weight_quantize_type == 'range_abs_max' or \ - self.activation_quantize_type == 'range_abs_max': + if ( + self.weight_quantize_type == 'range_abs_max' + or self.activation_quantize_type == 'range_abs_max' + ): self.global_step = autoincreased_step_counter() def freeze_program(self, program, place, scope=None): @@ -263,7 +285,7 @@ def _insert_post_dequant_op(block, op): max_range = None scale_var = None for name in op.input_arg_names: - #rename input name of the op to the input name of last op which has be removed + # rename input name of the op to the input name of last op which has be removed if name in op_in_rename_map[block_id]: op._rename_input(name, op_in_rename_map[block_id][name]) @@ -278,23 +300,25 @@ def _insert_post_dequant_op(block, op): scale_var = scale_v if len(op.output_arg_names) != 1: - raise ValueError("Only support one output, but op %s has" - " more than one output." % (op.type)) + raise ValueError( + "Only support one output, but op %s has" + " more than one output." % (op.type) + ) out_var = block.var(op.output_arg_names[0]) - dequant_var = block.create_var(name=_dequantized_var_name( - out_var.name), - type=out_var.type, - shape=out_var.shape, - dtype=out_var.dtype) + dequant_var = block.create_var( + name=_dequantized_var_name(out_var.name), + type=out_var.type, + shape=out_var.shape, + dtype=out_var.dtype, + ) # insert fake_dequantize_op - dequant_op = block._insert_op(idx + 1, - type="fake_dequantize_max_abs", - attrs={'max_range': float(max_range)}, - inputs={ - "X": out_var, - 'Scale': scale_var - }, - outputs={"Out": dequant_var}) + dequant_op = block._insert_op( + idx + 1, + type="fake_dequantize_max_abs", + attrs={'max_range': float(max_range)}, + inputs={"X": out_var, 'Scale': scale_var}, + outputs={"Out": dequant_var}, + ) op_out_rename_map[block_id][out_var.name] = dequant_var.name return dequant_var @@ -315,8 +339,9 @@ def _restore_var(name, arr): # input of the followed ops(of fc/conv) to the dquant_op for name in op.input_arg_names: if name in op_out_rename_map[block_id]: - op._rename_input(name, - op_out_rename_map[block_id][name]) + op._rename_input( + name, op_out_rename_map[block_id][name] + ) if op_type in self.fake_quant_op_types: in_arg_name = op.input('X')[0] @@ -346,7 +371,7 @@ def _restore_var(name, arr): # remove the unused var in ProgramDesc self._remove_unused_var(program) - #program = program.clone() + # program = program.clone() def convert_to_int8(self, program, place, scope=None): scope = global_scope() if scope is None else scope @@ -363,7 +388,8 @@ def convert_to_int8(var): name=int8_var_name.encode('ascii'), type=var.type, dtype=core.VarDesc.VarType.INT8, - shape=var.shape) + shape=var.shape, + ) tensor = _load_var(var.name) @@ -392,7 +418,7 @@ def _remove_unused_var(self, program): for op in block.ops: args += op.input_arg_names args += op.output_arg_names - args = list(set(args)) #vals of all left ops + args = list(set(args)) # vals of all left ops var_names = block.vars.keys() # all vals sub_block_remove_vars = [] for var in var_names: @@ -406,39 +432,45 @@ def _remove_unused_var(self, program): block._remove_var(v) def _insert_quant_abs_max_op(self, block, idx, var, quant_bits): - """Insert fake_quantize_abs_max op. - """ - quant_var = block.create_var(name=_quantized_var_name(var.name), - type=var.type, - shape=var.shape, - dtype=var.dtype) - scale = block.create_var(name=_quantized_scale_name(var.name), - type=var.type, - shape=var.shape, - dtype=var.dtype) - quant_op = block._insert_op(idx, - type='fake_quantize_abs_max', - attrs={'bit_length': quant_bits}, - inputs={'X': var}, - outputs={ - 'Out': quant_var, - 'OutScale': scale - }) + """Insert fake_quantize_abs_max op.""" + quant_var = block.create_var( + name=_quantized_var_name(var.name), + type=var.type, + shape=var.shape, + dtype=var.dtype, + ) + scale = block.create_var( + name=_quantized_scale_name(var.name), + type=var.type, + shape=var.shape, + dtype=var.dtype, + ) + quant_op = block._insert_op( + idx, + type='fake_quantize_abs_max', + attrs={'bit_length': quant_bits}, + inputs={'X': var}, + outputs={'Out': quant_var, 'OutScale': scale}, + ) return quant_var, scale def _insert_quant_range_abs_max_op(self, block, idx, var, quant_bits): - """Insert fake_quantize_range_abs_max - """ - quant_var = block.create_var(name=_quantized_var_name(var.name), - type=var.type, - shape=var.shape, - dtype=var.dtype) - scale = self.helper.create_parameter(attr=ParamAttr( - name=_quantized_scale_name(var.name), - initializer=Constant(0.001), - trainable=False), - shape=[1], - dtype=var.dtype) + """Insert fake_quantize_range_abs_max""" + quant_var = block.create_var( + name=_quantized_var_name(var.name), + type=var.type, + shape=var.shape, + dtype=var.dtype, + ) + scale = self.helper.create_parameter( + attr=ParamAttr( + name=_quantized_scale_name(var.name), + initializer=Constant(0.001), + trainable=False, + ), + shape=[1], + dtype=var.dtype, + ) scale.stop_gradient = True ins = {'X': var, 'InScale': scale} @@ -449,9 +481,11 @@ def _insert_quant_range_abs_max_op(self, block, idx, var, quant_bits): name=unique_name.generate('scales'), persistable=True, dtype=var.dtype, - shape=[self.window_size]) - self.helper.set_variable_initializer(scales, - initializer=Constant(value=0)) + shape=[self.window_size], + ) + self.helper.set_variable_initializer( + scales, initializer=Constant(value=0) + ) ins['Iter'] = self.global_step outs['OutScales'] = scales @@ -459,45 +493,56 @@ def _insert_quant_range_abs_max_op(self, block, idx, var, quant_bits): attrs = { 'window_size': self.window_size, 'bit_length': quant_bits, - 'is_test': self.is_test + 'is_test': self.is_test, } - quant_op = block._insert_op(idx, - type='fake_quantize_range_abs_max', - attrs=attrs, - inputs=ins, - outputs=outs) + quant_op = block._insert_op( + idx, + type='fake_quantize_range_abs_max', + attrs=attrs, + inputs=ins, + outputs=outs, + ) return quant_var, scale - def _insert_quant_moving_average_abs_max_op(self, block, idx, var, - quant_bits): - """Insert fake_quantize_moving_average_abs_max - """ - quant_var = block.create_var(name=_quantized_var_name(var.name), - type=var.type, - shape=var.shape, - dtype=var.dtype) + def _insert_quant_moving_average_abs_max_op( + self, block, idx, var, quant_bits + ): + """Insert fake_quantize_moving_average_abs_max""" + quant_var = block.create_var( + name=_quantized_var_name(var.name), + type=var.type, + shape=var.shape, + dtype=var.dtype, + ) state = self.helper.create_global_variable( name=unique_name.generate('state'), persistable=True, dtype=var.dtype, - shape=[1]) - self.helper.set_variable_initializer(state, - initializer=Constant(value=1)) + shape=[1], + ) + self.helper.set_variable_initializer( + state, initializer=Constant(value=1) + ) accum = self.helper.create_global_variable( name=unique_name.generate('accum'), persistable=True, dtype=var.dtype, - shape=[1]) - self.helper.set_variable_initializer(accum, - initializer=Constant(value=1)) - scale = self.helper.create_parameter(attr=ParamAttr( - name=_quantized_scale_name(var.name), - initializer=Constant(0.001), - trainable=False), - shape=[1], - dtype=var.dtype) + shape=[1], + ) + self.helper.set_variable_initializer( + accum, initializer=Constant(value=1) + ) + scale = self.helper.create_parameter( + attr=ParamAttr( + name=_quantized_scale_name(var.name), + initializer=Constant(0.001), + trainable=False, + ), + shape=[1], + dtype=var.dtype, + ) scale.stop_gradient = True ins = {'X': var, 'InScale': scale} @@ -511,14 +556,16 @@ def _insert_quant_moving_average_abs_max_op(self, block, idx, var, attrs = { 'bit_length': quant_bits, 'moving_rate': self.moving_rate, - 'is_test': self.is_test + 'is_test': self.is_test, } - quant_op = block._insert_op(idx, - type='fake_quantize_moving_average_abs_max', - attrs=attrs, - inputs=ins, - outputs=outs) + quant_op = block._insert_op( + idx, + type='fake_quantize_moving_average_abs_max', + attrs=attrs, + inputs=ins, + outputs=outs, + ) return quant_var, scale @@ -529,28 +576,31 @@ def _insert_quant_op(self, block, idx, var, quant_bits, quant_type): if quant_type == 'abs_max': return self._insert_quant_abs_max_op(block, idx, var, quant_bits) elif quant_type == 'range_abs_max': - return self._insert_quant_range_abs_max_op(block, idx, var, - quant_bits) + return self._insert_quant_range_abs_max_op( + block, idx, var, quant_bits + ) elif quant_type == 'moving_average_abs_max': return self._insert_quant_moving_average_abs_max_op( - block, idx, var, quant_bits) + block, idx, var, quant_bits + ) def _insert_dequant_op(self, block, idx, var, scale, quant_bits): """ Insert fake_quantize_op """ - dequant_var = block.create_var(name=_dequantized_var_name(var.name), - type=var.type, - shape=var.shape, - dtype=var.dtype) + dequant_var = block.create_var( + name=_dequantized_var_name(var.name), + type=var.type, + shape=var.shape, + dtype=var.dtype, + ) # insert fake_dequantize_op max_range = (1 << (quant_bits - 1)) - 1 - dequant_op = block._insert_op(idx, - type="fake_dequantize_max_abs", - attrs={'max_range': float(max_range)}, - inputs={ - "X": var, - 'Scale': scale - }, - outputs={"Out": dequant_var}) + dequant_op = block._insert_op( + idx, + type="fake_dequantize_max_abs", + attrs={'max_range': float(max_range)}, + inputs={"X": var, 'Scale': scale}, + outputs={"Out": dequant_var}, + ) return dequant_var diff --git a/python/paddle/fluid/contrib/slim/quantization/adaround.py b/python/paddle/fluid/contrib/slim/quantization/adaround.py index 04d894b055dc6..9dd00ddadc6ac 100644 --- a/python/paddle/fluid/contrib/slim/quantization/adaround.py +++ b/python/paddle/fluid/contrib/slim/quantization/adaround.py @@ -20,44 +20,52 @@ import paddle.fluid as fluid from ....log_helper import get_logger -from .utils import load_variable_data, set_variable_data, stable_sigmoid, quant_tensor, dequant_tensor, _channelwise_quant_axis1_ops, calculate_quant_cos_error, bias_correction_w - -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +from .utils import ( + load_variable_data, + set_variable_data, + stable_sigmoid, + quant_tensor, + dequant_tensor, + _channelwise_quant_axis1_ops, + calculate_quant_cos_error, + bias_correction_w, +) + +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) GAMMA = -0.1 ZETA = 1.1 def compute_soft_rounding(alpha_v): - return fluid.layers.clip(fluid.layers.sigmoid(alpha_v) * (ZETA - GAMMA) + - GAMMA, - min=0, - max=1) + return fluid.layers.clip( + fluid.layers.sigmoid(alpha_v) * (ZETA - GAMMA) + GAMMA, min=0, max=1 + ) def compute_soft_rounding_np(alpha_v): - return np.clip(stable_sigmoid(alpha_v) * (ZETA - GAMMA) + GAMMA, - a_min=0, - a_max=1) + return np.clip( + stable_sigmoid(alpha_v) * (ZETA - GAMMA) + GAMMA, a_min=0, a_max=1 + ) class AdaRoundLoss(object): - def __init__(self, reg_param=0.01, default_beta_range=(20, 2)): self.default_reg_param = reg_param self.default_beta_range = default_beta_range def compute_recon_loss(self, ada_quantized_output, orig_output): - square_cost = fluid.layers.square_error_cost(ada_quantized_output, - orig_output) + square_cost = fluid.layers.square_error_cost( + ada_quantized_output, orig_output + ) recon_loss = fluid.layers.reduce_mean( - fluid.layers.reduce_sum(square_cost, dim=-1)) + fluid.layers.reduce_sum(square_cost, dim=-1) + ) return recon_loss def compute_round_loss(self, alpha_v, warm_start, beta): - def round_loss_fn(): # compute rectified sigmoid of parameter 'alpha' which maps it between zero and one h_v = compute_soft_rounding(alpha_v) @@ -65,8 +73,9 @@ def round_loss_fn(): # calculate regularization term - which ensures parameter to converge to exactly zeros and ones # at the end of optimization reg_term = fluid.layers.reduce_sum( - -fluid.layers.pow(fluid.layers.abs(2 * h_v - 1), factor=beta) + - 1) + -fluid.layers.pow(fluid.layers.abs(2 * h_v - 1), factor=beta) + + 1 + ) # calculate the rounding loss round_loss = self.default_reg_param * reg_term @@ -74,8 +83,12 @@ def round_loss_fn(): return round_loss round_loss = fluid.layers.cond( - warm_start, lambda: fluid.layers.fill_constant( - shape=[1], dtype='float32', value=0.0), round_loss_fn) + warm_start, + lambda: fluid.layers.fill_constant( + shape=[1], dtype='float32', value=0.0 + ), + round_loss_fn, + ) return round_loss @@ -88,29 +101,32 @@ def compute_beta(self, max_iter, cur_iter, warm_start): warm_start_end_iter = warm_start * max_iter # compute relative iteration of current iteration - rel_iter = (cur_iter - warm_start_end_iter) / (max_iter - - warm_start_end_iter) - beta = end_beta + 0.5 * (start_beta - - end_beta) * (1 + np.cos(rel_iter * np.pi)) + rel_iter = (cur_iter - warm_start_end_iter) / ( + max_iter - warm_start_end_iter + ) + beta = end_beta + 0.5 * (start_beta - end_beta) * ( + 1 + np.cos(rel_iter * np.pi) + ) return beta class AdaRound(object): - - def __init__(self, - scale, - weight_tensor, - scope=None, - weight_var_name=None, - weight_op_type=None, - is_train=True, - num_iterations=1000): + def __init__( + self, + scale, + weight_tensor, + scope=None, + weight_var_name=None, + weight_op_type=None, + is_train=True, + num_iterations=1000, + ): self.is_train = is_train self.num_iterations = num_iterations self.warm_start = 0.1 self.weight_bits = 8 - self.offset = 0. # zero-point offset + self.offset = 0.0 # zero-point offset self.adaround_loss = AdaRoundLoss() self.ori_weight_tensor = weight_tensor self.scale = scale @@ -134,19 +150,23 @@ def initialize_alpha(self, tensor, scale, var_name): shape=alpha.shape, dtype="float32", name=var_name + ".alpha", - default_initializer=fluid.initializer.NumpyArrayInitializer(alpha)) - - def _calculate_output_with_adarounded_weights(self, program, place, exe, - data, fp32_fetch_list, - weight_tensor_dequant): - set_variable_data(self.scope, place, self.weight_var_name, - weight_tensor_dequant) - - adaround_out_tensor = exe.run(program=program, - feed=data, - fetch_list=[fp32_fetch_list], - return_numpy=True, - scope=self.scope) + default_initializer=fluid.initializer.NumpyArrayInitializer(alpha), + ) + + def _calculate_output_with_adarounded_weights( + self, program, place, exe, data, fp32_fetch_list, weight_tensor_dequant + ): + set_variable_data( + self.scope, place, self.weight_var_name, weight_tensor_dequant + ) + + adaround_out_tensor = exe.run( + program=program, + feed=data, + fetch_list=[fp32_fetch_list], + return_numpy=True, + scope=self.scope, + ) return adaround_out_tensor def _calculate_quant_weight(self): @@ -154,9 +174,11 @@ def _calculate_quant_weight(self): h_alpha = compute_soft_rounding_np(np_alpha) # Scale the tensor - tensor_scale = quant_tensor(self.ori_weight_tensor.copy(), - self.scale, - quant_axis=self.quant_axis) + tensor_scale = quant_tensor( + self.ori_weight_tensor.copy(), + self.scale, + quant_axis=self.quant_axis, + ) weight_tensor = np.floor(tensor_scale) @@ -168,10 +190,11 @@ def _calculate_adarounded_weights(self): weight_tensor_quant = self._calculate_quant_weight() # Dequantize the tensor - weight_tensor_dequant = dequant_tensor(weight_tensor_quant + - self.offset, - self.scale, - quant_axis=self.quant_axis) + weight_tensor_dequant = dequant_tensor( + weight_tensor_quant + self.offset, + self.scale, + quant_axis=self.quant_axis, + ) return weight_tensor_dequant def update_final_weights(self): @@ -180,37 +203,42 @@ def update_final_weights(self): def get_loss(self, beta, warm_start, adaround_out_tensor, orig_out_tensor): round_loss = self.adaround_loss.compute_round_loss( - self.alpha_v, warm_start, beta) + self.alpha_v, warm_start, beta + ) recon_loss = self.adaround_loss.compute_recon_loss( - adaround_out_tensor, orig_out_tensor) + adaround_out_tensor, orig_out_tensor + ) loss = round_loss + recon_loss losses = { 'loss': loss, 'round_loss': round_loss, - 'recon_loss': recon_loss + 'recon_loss': recon_loss, } return losses def update_beta_warm(self, cur_iteration): warm_start = cur_iteration < self.num_iterations * self.warm_start - beta = self.adaround_loss.compute_beta(self.num_iterations, - cur_iteration, self.warm_start) + beta = self.adaround_loss.compute_beta( + self.num_iterations, cur_iteration, self.warm_start + ) return beta, warm_start -def run_adaround(data_loader, - fp32_program, - fetch_list, - exe, - scope, - place, - quantized_op_pairs, - weight_op_pairs, - scale_dict, - num_iterations=1000, - lr=0.001, - bias_correction=False, - fast_mode=True): +def run_adaround( + data_loader, + fp32_program, + fetch_list, + exe, + scope, + place, + quantized_op_pairs, + weight_op_pairs, + scale_dict, + num_iterations=1000, + lr=0.001, + bias_correction=False, + fast_mode=True, +): fetch_op_name = fetch_list[0].name final_weight_tensor_quant_dict = {} for weight_var_name, quant_op_out_name in quantized_op_pairs.items(): @@ -224,7 +252,8 @@ def run_adaround(data_loader, if _op.type == "fetch": _op._rename_input(fetch_op_name, quant_op_out_name) fp32_fetch_list = fp32_program.global_block().var( - quant_op_out_name) + quant_op_out_name + ) fetch_op_name = quant_op_out_name # build adaround program @@ -235,29 +264,37 @@ def run_adaround(data_loader, with fluid.program_guard(train_program, startup_program): with fluid.unique_name.guard(): # initialize adaround - adaround = AdaRound(scale, - weight_var_tensor, - scope=scope, - weight_var_name=weight_var_name, - weight_op_type=weight_op_type, - num_iterations=num_iterations) - orig_out_tensor = fluid.data(name='orig_out_tensor', - shape=fp32_fetch_list.shape, - dtype='float32') - adaround_out_tensor = fluid.data(name='adaround_out_tensor', - shape=fp32_fetch_list.shape, - dtype='float32') - beta_tensor = fluid.data(name='beta', - shape=[1], - dtype='float32') - warm_start_tensor = fluid.data(name='warm_start', - shape=[1], - dtype='bool') - - train_fetches_loss = adaround.get_loss(beta_tensor, - warm_start_tensor, - adaround_out_tensor, - orig_out_tensor) + adaround = AdaRound( + scale, + weight_var_tensor, + scope=scope, + weight_var_name=weight_var_name, + weight_op_type=weight_op_type, + num_iterations=num_iterations, + ) + orig_out_tensor = fluid.data( + name='orig_out_tensor', + shape=fp32_fetch_list.shape, + dtype='float32', + ) + adaround_out_tensor = fluid.data( + name='adaround_out_tensor', + shape=fp32_fetch_list.shape, + dtype='float32', + ) + beta_tensor = fluid.data( + name='beta', shape=[1], dtype='float32' + ) + warm_start_tensor = fluid.data( + name='warm_start', shape=[1], dtype='bool' + ) + + train_fetches_loss = adaround.get_loss( + beta_tensor, + warm_start_tensor, + adaround_out_tensor, + orig_out_tensor, + ) optimizer = fluid.optimizer.Adam(learning_rate=lr) loss = train_fetches_loss['loss'] optimizer.minimize(loss) @@ -269,21 +306,32 @@ def run_adaround(data_loader, prev_start_time = start_time start_time = time.time() # run fp32 model - np_orig_out_tensor = exe.run(program=fp32_program, - feed=data, - fetch_list=[fp32_fetch_list], - return_numpy=True, - scope=scope) + np_orig_out_tensor = exe.run( + program=fp32_program, + feed=data, + fetch_list=[fp32_fetch_list], + return_numpy=True, + scope=scope, + ) - adaround_weight_tensor_dequant = adaround._calculate_adarounded_weights( + adaround_weight_tensor_dequant = ( + adaround._calculate_adarounded_weights() + ) + np_adaround_out_tensor = ( + adaround._calculate_output_with_adarounded_weights( + fp32_program, + place, + exe, + data, + fp32_fetch_list, + adaround_weight_tensor_dequant, + ) ) - np_adaround_out_tensor = adaround._calculate_output_with_adarounded_weights( - fp32_program, place, exe, data, fp32_fetch_list, - adaround_weight_tensor_dequant) # If the cosine distance of the two tensor is small, skip training - cos_error = calculate_quant_cos_error(np_orig_out_tensor[0], - np_adaround_out_tensor[0]) + cos_error = calculate_quant_cos_error( + np_orig_out_tensor[0], np_adaround_out_tensor[0] + ) if fast_mode and cos_error > 0.99: _logger.info("The cosine error is small, skip training.") break @@ -292,22 +340,30 @@ def run_adaround(data_loader, 'orig_out_tensor': np_orig_out_tensor[0], 'adaround_out_tensor': np_adaround_out_tensor[0], 'beta': beta, - 'warm_start': warm_start + 'warm_start': warm_start, } out = exe.run( train_program, feed=feed_dict, fetch_list=[v.name for v in train_fetches_loss.values()], - return_numpy=True) + return_numpy=True, + ) _logger.info( - "Iter {:d}, lr {:.5f}, loss {:.5f}, loss_round {:.5f}, loss_recon {:.5f}, time {:.5f}s" - .format(i, lr, np.mean(out[0]), np.mean(out[1]), - np.mean(out[2]), start_time - prev_start_time)) + "Iter {:d}, lr {:.5f}, loss {:.5f}, loss_round {:.5f}, loss_recon {:.5f}, time {:.5f}s".format( + i, + lr, + np.mean(out[0]), + np.mean(out[1]), + np.mean(out[2]), + start_time - prev_start_time, + ) + ) sys.stdout.flush() if i == num_iterations: break final_weight_tensor_quant_dict[ - weight_var_name] = adaround.update_final_weights() + weight_var_name + ] = adaround.update_final_weights() if bias_correction: final_weight_tensor_quant_dict[weight_var_name] = bias_correction_w( @@ -315,11 +371,16 @@ def run_adaround(data_loader, final_weight_tensor_quant_dict[weight_var_name], scale, adaround.quant_axis, - weight_bits=adaround.weight_bits) + weight_bits=adaround.weight_bits, + ) del adaround # update adarounded calibrated weights for weight_var_name in quantized_op_pairs.keys(): - set_variable_data(scope, place, weight_var_name, - final_weight_tensor_quant_dict[weight_var_name]) + set_variable_data( + scope, + place, + weight_var_name, + final_weight_tensor_quant_dict[weight_var_name], + ) diff --git a/python/paddle/fluid/contrib/slim/quantization/cal_kl_threshold.py b/python/paddle/fluid/contrib/slim/quantization/cal_kl_threshold.py index 69cd3f6406162..ea3b1876f2346 100644 --- a/python/paddle/fluid/contrib/slim/quantization/cal_kl_threshold.py +++ b/python/paddle/fluid/contrib/slim/quantization/cal_kl_threshold.py @@ -17,9 +17,9 @@ import numpy as np from ....log_helper import get_logger -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) __all__ = ['cal_kl_threshold'] @@ -38,11 +38,13 @@ def expand_quantized_bins(quantized_bins, reference_bins): if zero_count == num_merged_bins: avg_bin_ele = 0 else: - avg_bin_ele = quantized_bins[idx] / (num_merged_bins - zero_count + - 0.0) + avg_bin_ele = quantized_bins[idx] / ( + num_merged_bins - zero_count + 0.0 + ) for idx1 in range(j_start, j_end): - expanded_quantized_bins[idx1] = (0 if reference_bins[idx1] == 0 else - avg_bin_ele) + expanded_quantized_bins[idx1] = ( + 0 if reference_bins[idx1] == 0 else avg_bin_ele + ) j_start += num_merged_bins j_end += num_merged_bins if (idx + 1) == len(quantized_bins) - 1: @@ -65,8 +67,12 @@ def safe_entropy(reference_distr_P, P_sum, candidate_distr_Q, Q_sum): tmp_sum2 += 0 else: if q_idx == 0: - _logger.error("Fatal error!, idx = " + str(idx) + - " qindex = 0! p_idx = " + str(p_idx)) + _logger.error( + "Fatal error!, idx = " + + str(idx) + + " qindex = 0! p_idx = " + + str(p_idx) + ) tmp_sum1 += p_idx * (math.log(Q_sum * p_idx)) tmp_sum2 += p_idx * (math.log(P_sum * q_idx)) return (tmp_sum1 - tmp_sum2) / P_sum @@ -84,7 +90,7 @@ def cal_kl_threshold(hist, bin_width, bits): assert hist.ndim == 1 hist_bins = hist.shape[0] starting_iter = int((hist_bins - 1) * 0.5) - quant_range = 2**(bits - 1) - 1 + quant_range = 2 ** (bits - 1) - 1 P_sum = np.sum(np.array(hist).ravel()) min_kl_divergence = 0 @@ -105,16 +111,19 @@ def cal_kl_threshold(hist, bin_width, bits): j_end = num_merged_bins for idx in range(quant_range): candidate_distr_Q_quantized[idx] = sum( - candidate_distr_Q[j_start:j_end]) + candidate_distr_Q[j_start:j_end] + ) j_start += num_merged_bins j_end += num_merged_bins if (idx + 1) == quant_range - 1: j_end = i - candidate_distr_Q = expand_quantized_bins(candidate_distr_Q_quantized, - reference_distr_bins) + candidate_distr_Q = expand_quantized_bins( + candidate_distr_Q_quantized, reference_distr_bins + ) Q_sum = sum(candidate_distr_Q) - kl_divergence = safe_entropy(reference_distr_P, P_sum, - candidate_distr_Q, Q_sum) + kl_divergence = safe_entropy( + reference_distr_P, P_sum, candidate_distr_Q, Q_sum + ) if not kl_inited: min_kl_divergence = kl_divergence min_kl_index = i diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/fuse_utils.py b/python/paddle/fluid/contrib/slim/quantization/imperative/fuse_utils.py index 4ae949bf0fe37..b01ceddd1cab0 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/fuse_utils.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/fuse_utils.py @@ -51,20 +51,20 @@ def fuse_conv_bn(model): def fuse_layers(model, layers_to_fuse, inplace=False): ''' - fuse layers in layers_to_fuse - - Args: - model(paddle.nn.Layer): The model to be fused. - layers_to_fuse(list): The layers' names to be fused. For - example,"fuse_list = [["conv1", "bn1"], ["conv2", "bn2"]]". - A TypeError would be raised if "fuse" was set as - True but "fuse_list" was None. - Default: None. - inplace(bool): Whether apply fusing to the input model. - Default: False. - - Return - fused_model(paddle.nn.Layer): The fused model. + fuse layers in layers_to_fuse + + Args: + model(paddle.nn.Layer): The model to be fused. + layers_to_fuse(list): The layers' names to be fused. For + example,"fuse_list = [["conv1", "bn1"], ["conv2", "bn2"]]". + A TypeError would be raised if "fuse" was set as + True but "fuse_list" was None. + Default: None. + inplace(bool): Whether apply fusing to the input model. + Default: False. + + Return + fused_model(paddle.nn.Layer): The fused model. ''' if inplace == False: model = copy.deepcopy(model) @@ -78,12 +78,14 @@ def _fuse_layers(model, layers_list): layer_list = [] for layer_name in layers_list: parent_layer, sub_name = utils.find_parent_layer_and_sub_name( - model, layer_name) + model, layer_name + ) layer_list.append(getattr(parent_layer, sub_name)) new_layers = _fuse_func(layer_list) for i, item in enumerate(layers_list): parent_layer, sub_name = utils.find_parent_layer_and_sub_name( - model, item) + model, item + ) setattr(parent_layer, sub_name, new_layers[i]) @@ -109,10 +111,13 @@ def _fuse_func(layer_list): def _fuse_conv_bn(conv, bn): '''fuse conv and bn for train or eval''' - assert(conv.training == bn.training),\ - "Conv and BN both must be in the same mode (train or eval)." + assert ( + conv.training == bn.training + ), "Conv and BN both must be in the same mode (train or eval)." if conv.training: - assert bn._num_features == conv._out_channels, 'Output channel of Conv2d must match num_features of BatchNorm2d' + assert ( + bn._num_features == conv._out_channels + ), 'Output channel of Conv2d must match num_features of BatchNorm2d' raise NotImplementedError else: return _fuse_conv_bn_eval(conv, bn) @@ -120,17 +125,23 @@ def _fuse_conv_bn(conv, bn): def _fuse_conv_bn_eval(conv, bn): '''fuse conv and bn for eval''' - assert (not (conv.training or bn.training)), "Fusion only for eval!" + assert not (conv.training or bn.training), "Fusion only for eval!" fused_conv = copy.deepcopy(conv) - fused_weight, fused_bias = _fuse_conv_bn_weights(fused_conv.weight, - fused_conv.bias, bn._mean, - bn._variance, bn._epsilon, - bn.weight, bn.bias) + fused_weight, fused_bias = _fuse_conv_bn_weights( + fused_conv.weight, + fused_conv.bias, + bn._mean, + bn._variance, + bn._epsilon, + bn.weight, + bn.bias, + ) fused_conv.weight.set_value(fused_weight) if fused_conv.bias is None: fused_conv.bias = paddle.create_parameter( - shape=[fused_conv._out_channels], is_bias=True, dtype=bn.bias.dtype) + shape=[fused_conv._out_channels], is_bias=True, dtype=bn.bias.dtype + ) fused_conv.bias.set_value(fused_bias) return fused_conv @@ -144,19 +155,22 @@ def _fuse_conv_bn_weights(conv_w, conv_b, bn_rm, bn_rv, bn_eps, bn_w, bn_b): if bn_b is None: bn_b = paddle.zeros_like(bn_rm) bn_var_rsqrt = paddle.rsqrt(bn_rv + bn_eps) - conv_w = conv_w * \ - (bn_w * bn_var_rsqrt).reshape([-1] + [1] * (len(conv_w.shape) - 1)) + conv_w = conv_w * (bn_w * bn_var_rsqrt).reshape( + [-1] + [1] * (len(conv_w.shape) - 1) + ) conv_b = (conv_b - bn_rm) * bn_var_rsqrt * bn_w + bn_b return conv_w, conv_b def _fuse_linear_bn(linear, bn): '''fuse linear and bn''' - assert (linear.training == bn.training),\ - "Linear and BN both must be in the same mode (train or eval)." + assert ( + linear.training == bn.training + ), "Linear and BN both must be in the same mode (train or eval)." if linear.training: - assert bn._num_features == linear.weight.shape[ - 1], 'Output channel of Linear must match num_features of BatchNorm' + assert ( + bn._num_features == linear.weight.shape[1] + ), 'Output channel of Linear must match num_features of BatchNorm' raise NotImplementedError else: return _fuse_linear_bn_eval(linear, bn) @@ -164,26 +178,32 @@ def _fuse_linear_bn(linear, bn): def _fuse_linear_bn_eval(linear, bn): '''fuse linear and bn for eval''' - assert (not (linear.training or bn.training)), "Fusion only for eval!" + assert not (linear.training or bn.training), "Fusion only for eval!" fused_linear = copy.deepcopy(linear) - fused_weight, fused_bias = _fuse_linear_bn_weights(fused_linear.weight, - fused_linear.bias, - bn._mean, bn._variance, - bn._epsilon, bn.weight, - bn.bias) + fused_weight, fused_bias = _fuse_linear_bn_weights( + fused_linear.weight, + fused_linear.bias, + bn._mean, + bn._variance, + bn._epsilon, + bn.weight, + bn.bias, + ) fused_linear.weight.set_value(fused_weight) if fused_linear.bias is None: fused_linear.bias = paddle.create_parameter( shape=[fused_linear.weight.shape[1]], is_bias=True, - dtype=bn.bias.dtype) + dtype=bn.bias.dtype, + ) fused_linear.bias.set_value(fused_bias) return fused_linear -def _fuse_linear_bn_weights(linear_w, linear_b, bn_rm, bn_rv, bn_eps, bn_w, - bn_b): +def _fuse_linear_bn_weights( + linear_w, linear_b, bn_rm, bn_rv, bn_eps, bn_w, bn_b +): '''fuse weights and bias of linear and bn''' if linear_b is None: linear_b = paddle.zeros_like(bn_rm) diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq.py b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq.py index 23affb658cc5b..9c028736d6826 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq.py @@ -31,9 +31,9 @@ __all__ = ['ImperativePTQ'] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class ImperativePTQ(object): @@ -75,17 +75,20 @@ def quantize(self, model, inplace=False, fuse=False, fuse_list=None): Return quantized_model(paddle.nn.Layer): The quantized model. """ - assert isinstance(model, paddle.nn.Layer), \ - "The model must be the instance of paddle.nn.Layer." + assert isinstance( + model, paddle.nn.Layer + ), "The model must be the instance of paddle.nn.Layer." if not inplace: model = copy.deepcopy(model) if fuse: model.eval() model = fuse_utils.fuse_layers(model, fuse_list) for name, layer in model.named_sublayers(): - if PTQRegistry.is_supported_layer(layer) \ - and utils.is_leaf_layer(layer) \ - and not self._is_skip_layer(layer): + if ( + PTQRegistry.is_supported_layer(layer) + and utils.is_leaf_layer(layer) + and not self._is_skip_layer(layer) + ): # Add quant config quant_config = copy.deepcopy(self._quant_config) @@ -98,7 +101,8 @@ def quantize(self, model, inplace=False, fuse=False, fuse_list=None): quant_hook_handle = layer.register_forward_post_hook(hook) quant_config.quant_hook_handle = quant_hook_handle layer._forward_post_hooks.move_to_end( - quant_hook_handle._hook_id, last=False) + quant_hook_handle._hook_id, last=False + ) return model @@ -133,8 +137,9 @@ def save_quantized_model(self, model, path, input_spec=None, **config): None """ - assert isinstance(model, paddle.nn.Layer), \ - "The model must be the instance of paddle.nn.Layer." + assert isinstance( + model, paddle.nn.Layer + ), "The model must be the instance of paddle.nn.Layer." # Convert and save dygraph quantized model self._convert(model) @@ -156,12 +161,16 @@ def save_quantized_model(self, model, path, input_spec=None, **config): model_filename = basename + INFER_MODEL_SUFFIX params_filename = basename + INFER_PARAMS_SUFFIX - [infer_program, feed_target_names, - fetch_targets] = (paddle.fluid.io.load_inference_model( - dirname=dirname, - executor=exe, - model_filename=model_filename, - params_filename=params_filename)) + [ + infer_program, + feed_target_names, + fetch_targets, + ] = paddle.fluid.io.load_inference_model( + dirname=dirname, + executor=exe, + model_filename=model_filename, + params_filename=params_filename, + ) # Process inference program self._clean_up(infer_program) @@ -169,13 +178,15 @@ def save_quantized_model(self, model, path, input_spec=None, **config): self._remove_scale_op(infer_program) # Save final program - paddle.fluid.io.save_inference_model(dirname=dirname, - feeded_var_names=feed_target_names, - target_vars=fetch_targets, - executor=exe, - main_program=infer_program.clone(), - model_filename=model_filename, - params_filename=params_filename) + paddle.fluid.io.save_inference_model( + dirname=dirname, + feeded_var_names=feed_target_names, + target_vars=fetch_targets, + executor=exe, + main_program=infer_program.clone(), + model_filename=model_filename, + params_filename=params_filename, + ) if is_dynamic_mode: paddle.disable_static() @@ -213,8 +224,9 @@ def _cal_thresholds(self, model): Returns: None """ - assert isinstance(model, paddle.nn.Layer), \ - "The input model must be the instance of paddle.nn.Layer." + assert isinstance( + model, paddle.nn.Layer + ), "The input model must be the instance of paddle.nn.Layer." total_num = 0 cur_num = 0 @@ -226,8 +238,9 @@ def _cal_thresholds(self, model): if self._is_quant_layer(sub_layer): cur_num += 1 if cur_num % 5 == 0: - _logger.info("Process the %s / %s layer" % - (cur_num, total_num)) + _logger.info( + "Process the %s / %s layer" % (cur_num, total_num) + ) quant_config = sub_layer._quant_config @@ -236,7 +249,7 @@ def _cal_thresholds(self, model): quant_config.out_act_quantizer.cal_thresholds() if PTQRegistry.is_simulated_quant_layer(sub_layer): - weights = (sub_layer.weight, ) + weights = (sub_layer.weight,) quant_config.wt_quantizer.sample_data(sub_layer, weights) quant_config.wt_quantizer.cal_thresholds() @@ -250,8 +263,9 @@ def _save_output_thresholds(self, sub_layer, quant_config): Returns: None """ - assert isinstance(sub_layer, paddle.nn.Layer), \ - "The input model must be the instance of paddle.nn.Layer." + assert isinstance( + sub_layer, paddle.nn.Layer + ), "The input model must be the instance of paddle.nn.Layer." layer_info = PTQRegistry.layer_info(sub_layer) @@ -272,12 +286,14 @@ def _wrap_simulated_layers(self, model): Returns: None """ - assert isinstance(model, paddle.nn.Layer), \ - "The input model must be the instance of paddle.nn.Layer." + assert isinstance( + model, paddle.nn.Layer + ), "The input model must be the instance of paddle.nn.Layer." for name, sub_layer in model.named_sublayers(): - if self._is_quant_layer(sub_layer) \ - and PTQRegistry.is_simulated_quant_layer(sub_layer): + if self._is_quant_layer( + sub_layer + ) and PTQRegistry.is_simulated_quant_layer(sub_layer): quant_config = sub_layer._quant_config assert quant_config.enable_in_act_quantizer == True @@ -303,15 +319,17 @@ def _wrap_simulated_layers(self, model): "activation_bits": in_act_quantizer.quant_bits, } - quant_layer = quant_layers.__dict__[quant_layer_name](sub_layer, - **kwargs) + quant_layer = quant_layers.__dict__[quant_layer_name]( + sub_layer, **kwargs + ) # save the input thresholds assert hasattr(quant_layer, "_fake_quant_input") assert hasattr(quant_layer._fake_quant_input, "_scale") assert len(in_act_quantizer.thresholds) == 1 - input_threshold = np.array([in_act_quantizer.thresholds[0]], - dtype=np.float32) + input_threshold = np.array( + [in_act_quantizer.thresholds[0]], dtype=np.float32 + ) quant_layer._fake_quant_input._scale.set_value(input_threshold) assert hasattr(quant_layer, "_fake_quant_weight") @@ -319,20 +337,24 @@ def _wrap_simulated_layers(self, model): assert len(wt_quantizer.thresholds) == 1 weight_threshold = wt_quantizer.thresholds[0] if isinstance(weight_threshold, list): - weight_threshold = np.array(weight_threshold, - dtype=np.float32) + weight_threshold = np.array( + weight_threshold, dtype=np.float32 + ) else: - weight_threshold = np.array([weight_threshold], - dtype=np.float32) + weight_threshold = np.array( + [weight_threshold], dtype=np.float32 + ) quant_layer._fake_quant_weight._scale.set_value( - weight_threshold) + weight_threshold + ) # save the output thresholds self._save_output_thresholds(quant_layer, quant_config) # replace the layer - parent_layer, sub_name = \ - utils.find_parent_layer_and_sub_name(model, name) + parent_layer, sub_name = utils.find_parent_layer_and_sub_name( + model, name + ) setattr(parent_layer, sub_name, quant_layer) def _gather_input_thresholds(self, program, scope): @@ -351,30 +373,37 @@ def _gather_input_thresholds(self, program, scope): if previous_op is None: continue - if "quantize_dequantize" in previous_op.type or \ - previous_op.type == "moving_average_abs_max_scale": + if ( + "quantize_dequantize" in previous_op.type + or previous_op.type == "moving_average_abs_max_scale" + ): attr_name = previous_op.output('OutScale')[0] in_threshold = utils.load_variable_data(scope, attr_name) in_threshold = utils.fp_numpy_to_naive(in_threshold) argname, index = utils._get_input_name_index( - op, in_var_name) - op._set_attr(argname + str(index) + "_threshold", - in_threshold) + op, in_var_name + ) + op._set_attr( + argname + str(index) + "_threshold", in_threshold + ) op._set_attr("with_quant_attr", True) else: for out_var_name in utils._get_op_output_var_names( - previous_op): + previous_op + ): if out_var_name != in_var_name: continue argname, index = utils._get_output_name_index( - previous_op, out_var_name) + previous_op, out_var_name + ) attr_name = argname + str(index) + "_threshold" if not previous_op.has_attr(attr_name): continue threshold = previous_op.attr(attr_name) argname, index = utils._get_input_name_index( - op, in_var_name) + op, in_var_name + ) attr_name = argname + str(index) + "_threshold" op._set_attr(attr_name, threshold) op._set_attr("with_quant_attr", True) @@ -390,8 +419,11 @@ def _clean_up(self, program): """ def _helper(op, next_op, old_attr_name, new_attr_name): - if op.has_attr(old_attr_name) and next_op.has_attr(old_attr_name) \ - and op.attr(old_attr_name) == next_op.attr(old_attr_name): + if ( + op.has_attr(old_attr_name) + and next_op.has_attr(old_attr_name) + and op.attr(old_attr_name) == next_op.attr(old_attr_name) + ): threshold = op.attr(old_attr_name) op._remove_attr(old_attr_name) next_op._remove_attr(old_attr_name) @@ -417,8 +449,8 @@ def _helper(op, next_op, old_attr_name, new_attr_name): old_attr_name = argname + str(index) + "_threshold" argname, index = utils._get_output_name_index( - next_op, - next_op.output("Out")[0]) + next_op, next_op.output("Out")[0] + ) new_attr_name = argname + str(index) + "_threshold" _helper(op, next_op, old_attr_name, new_attr_name) diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_hooks.py b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_hooks.py index 41c9b07195aef..319beee0ed73b 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_hooks.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_hooks.py @@ -23,10 +23,11 @@ def quant_forward_post_hook(layer, inputs, outputs): """ The forward_post_hook for PTQ. """ - assert hasattr(layer, '_quant_config'), \ - "The layer should have _quant_config attr" + assert hasattr( + layer, '_quant_config' + ), "The layer should have _quant_config attr" qc = layer._quant_config if qc.enable_in_act_quantizer: qc.in_act_quantizer.sample_data(layer, inputs) - qc.out_act_quantizer.sample_data(layer, (outputs, )) + qc.out_act_quantizer.sample_data(layer, (outputs,)) diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_quantizer.py b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_quantizer.py index 9b751a93a0fc6..ba881f88efc39 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_quantizer.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_quantizer.py @@ -24,9 +24,13 @@ from ..cal_kl_threshold import cal_kl_threshold __all__ = [ - 'BaseQuantizer', 'AbsmaxQuantizer', 'PerChannelAbsmaxQuantizer', - 'KLQuantizer', 'HistQuantizer', 'SUPPORT_ACT_QUANTIZERS', - 'SUPPORT_WT_QUANTIZERS' + 'BaseQuantizer', + 'AbsmaxQuantizer', + 'PerChannelAbsmaxQuantizer', + 'KLQuantizer', + 'HistQuantizer', + 'SUPPORT_ACT_QUANTIZERS', + 'SUPPORT_WT_QUANTIZERS', ] @@ -50,25 +54,25 @@ def merge_max_value(old, new): return new -def combine_abs_max_and_hist(tensor, origin_max, origin_hist, bins, - upsample_bins): - """ - """ +def combine_abs_max_and_hist( + tensor, origin_max, origin_hist, bins, upsample_bins +): + """ """ new_max = abs_max_value(tensor) if new_max == 0.0: return origin_max, origin_hist elif origin_max == 0.0: - new_hist, _ = np.histogram(paddle.abs(tensor).numpy(), - range=(0, new_max), - bins=bins) + new_hist, _ = np.histogram( + paddle.abs(tensor).numpy(), range=(0, new_max), bins=bins + ) new_hist = new_hist.astype(np.float32) return new_max, new_hist elif new_max <= origin_max: - new_hist, _ = np.histogram(paddle.abs(tensor).numpy(), - range=(0, origin_max), - bins=bins) + new_hist, _ = np.histogram( + paddle.abs(tensor).numpy(), range=(0, origin_max), bins=bins + ) new_hist = new_hist.astype(np.float32) new_hist += origin_hist return origin_max, new_hist @@ -81,17 +85,18 @@ def combine_abs_max_and_hist(tensor, origin_max, origin_hist, bins, upsampled_hist = np.repeat(origin_hist, upsample_bins) expanded_hist = np.zeros((bins * downsampe_bins), dtype=np.float32) - expanded_hist[0:bins * upsample_bins] = upsampled_hist - cumsumed_hist = np.cumsum( - expanded_hist, dtype=np.float64)[downsampe_bins - 1::downsampe_bins] + expanded_hist[0 : bins * upsample_bins] = upsampled_hist + cumsumed_hist = np.cumsum(expanded_hist, dtype=np.float64)[ + downsampe_bins - 1 :: downsampe_bins + ] shift_cumsumed_hist = np.zeros((bins), dtype=np.float64) shift_cumsumed_hist[1:] = cumsumed_hist[0:-1] sampled_hist = (cumsumed_hist - shift_cumsumed_hist) / upsample_bins sampled_hist = sampled_hist.astype(np.float32) - new_hist, _ = np.histogram(paddle.abs(tensor).numpy(), - range=(0, new_max), - bins=bins) + new_hist, _ = np.histogram( + paddle.abs(tensor).numpy(), range=(0, new_max), bins=bins + ) new_hist = new_hist.astype(np.float32) new_hist += sampled_hist @@ -165,16 +170,16 @@ def sample_data(self, layer, tensors): ] abs_max_vals_list.append(abs_max_vals) - self.abs_max_vals = merge_max_value(self.abs_max_vals, - abs_max_vals_list) + self.abs_max_vals = merge_max_value( + self.abs_max_vals, abs_max_vals_list + ) def cal_thresholds(self): self.thresholds = self.abs_max_vals class BaseHistQuantizer(BaseQuantizer, metaclass=abc.ABCMeta): - """ - """ + """ """ def __init__(self, quant_bits=8, bins=1024, upsample_bins=64): super(BaseHistQuantizer, self).__init__(quant_bits) @@ -194,9 +199,11 @@ def sample_data(self, layer, tensors): if abs_max_vals[idx] == 0.0: self.hists.append(None) else: - hist, _ = np.histogram(paddle.abs(tensor).numpy(), - range=(0., abs_max_vals[idx]), - bins=self.bins) + hist, _ = np.histogram( + paddle.abs(tensor).numpy(), + range=(0.0, abs_max_vals[idx]), + bins=self.bins, + ) hist = hist.astype(np.float32) self.hists.append(hist) else: @@ -205,8 +212,12 @@ def sample_data(self, layer, tensors): for idx, tensor in enumerate(tensors): new_abs_max, new_hist = combine_abs_max_and_hist( - tensor, self.abs_max_vals[idx], self.hists[idx], self.bins, - self.upsample_bins) + tensor, + self.abs_max_vals[idx], + self.hists[idx], + self.bins, + self.upsample_bins, + ) self.abs_max_vals[idx] = new_abs_max self.hists[idx] = new_hist @@ -216,19 +227,15 @@ def cal_thresholds(self): class HistQuantizer(BaseHistQuantizer): - """ - """ + """ """ - def __init__(self, - quant_bits=8, - bins=1024, - upsample_bins=64, - hist_percent=0.99999): + def __init__( + self, quant_bits=8, bins=1024, upsample_bins=64, hist_percent=0.99999 + ): super(HistQuantizer, self).__init__(quant_bits, bins, upsample_bins) self.hist_percent = hist_percent def cal_thresholds(self): - def _helper(abs_max, hist, percent): assert hist.ndim == 1 and percent < 1.0 hist = hist / np.sum(hist, dtype=np.float64) @@ -240,14 +247,14 @@ def _helper(abs_max, hist, percent): if self.hists[idx] is None: self.thresholds.append(self.abs_max_vals[idx]) else: - threshold = _helper(self.abs_max_vals[idx], self.hists[idx], - self.hist_percent) + threshold = _helper( + self.abs_max_vals[idx], self.hists[idx], self.hist_percent + ) self.thresholds.append(threshold) class KLQuantizer(BaseHistQuantizer): - """ - """ + """ """ def __init__(self, quant_bits=8, bins=1024, upsample_bins=64): super(KLQuantizer, self).__init__(quant_bits, bins, upsample_bins) diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_registry.py b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_registry.py index a6b8033bc78c9..5bc7fc0c6b351 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_registry.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/ptq_registry.py @@ -48,10 +48,15 @@ def __init__(self, layer, input_names, weight_names, output_names): ] QUANT_LAYERS_INFO = [ - LayerInfo(paddle.nn.quant.quant_layers.QuantizedConv2D, ['Input'], - ['Filter'], ['Output']), - LayerInfo(paddle.nn.quant.quant_layers.QuantizedLinear, ['X'], ['Y'], - ['Out']), + LayerInfo( + paddle.nn.quant.quant_layers.QuantizedConv2D, + ['Input'], + ['Filter'], + ['Output'], + ), + LayerInfo( + paddle.nn.quant.quant_layers.QuantizedLinear, ['X'], ['Y'], ['Out'] + ), ] SIMULATED_LAYERS = [paddle.nn.Conv2D, paddle.nn.Linear] @@ -61,6 +66,7 @@ class PTQRegistry(object): """ Register the supported layers for PTQ and provide layers info. """ + supported_layers_map = {} registered_layers_map = {} is_inited = False @@ -89,8 +95,9 @@ def is_supported_layer(cls, layer): flag(bool): Whther the layer is supported. """ cls._init() - return layer in cls.supported_layers_map or \ - isinstance(layer, tuple(cls.supported_layers_map.keys())) + return layer in cls.supported_layers_map or isinstance( + layer, tuple(cls.supported_layers_map.keys()) + ) @classmethod def is_registered_layer(cls, layer): @@ -102,8 +109,9 @@ def is_registered_layer(cls, layer): flag(bool): Wether the layer is register layer_info. """ cls._init() - return layer in cls.registered_layers_map or \ - isinstance(layer, tuple(cls.registered_layers_map.keys())) + return layer in cls.registered_layers_map or isinstance( + layer, tuple(cls.registered_layers_map.keys()) + ) @classmethod def is_simulated_quant_layer(cls, layer): @@ -114,8 +122,9 @@ def is_simulated_quant_layer(cls, layer): Returns: flag(bool): Whther the layer is supported. """ - return layer in SIMULATED_LAYERS or \ - isinstance(layer, tuple(SIMULATED_LAYERS)) + return layer in SIMULATED_LAYERS or isinstance( + layer, tuple(SIMULATED_LAYERS) + ) @classmethod def layer_info(cls, layer): @@ -126,8 +135,9 @@ def layer_info(cls, layer): Returns: layer_info(LayerInfo): The layer info of the input layer. """ - assert cls.is_registered_layer(layer), \ - "The input layer is not register." + assert cls.is_registered_layer( + layer + ), "The input layer is not register." for layer_key, layer_info in cls.registered_layers_map.items(): if layer == layer_key or isinstance(layer, layer_key): diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/qat.py b/python/paddle/fluid/contrib/slim/quantization/imperative/qat.py index 423fb0fcd52f3..255e8e03a20ab 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/qat.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/qat.py @@ -38,17 +38,20 @@ __all__ = ['ImperativeQuantAware'] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) def lazy_import_fleet(layer_name_map, fake_quant_input_layers): from paddle.distributed import fleet + layer_name_map[ - 'ColumnParallelLinear'] = fleet.meta_parallel.parallel_layers.mp_layers.ColumnParallelLinear + 'ColumnParallelLinear' + ] = fleet.meta_parallel.parallel_layers.mp_layers.ColumnParallelLinear layer_name_map[ - 'RowParallelLinear'] = fleet.meta_parallel.parallel_layers.mp_layers.RowParallelLinear + 'RowParallelLinear' + ] = fleet.meta_parallel.parallel_layers.mp_layers.RowParallelLinear fake_quant_input_layers.append(fleet.meta_parallel.RowParallelLinear) fake_quant_input_layers.append(fleet.meta_parallel.ColumnParallelLinear) return layer_name_map, fake_quant_input_layers @@ -59,22 +62,27 @@ class ImperativeQuantAware(object): Applying quantization aware training (QAT) to the dgraph model. """ - def __init__(self, - quantizable_layer_type=[ - 'Conv2D', 'Linear', 'Conv2DTranspose', - 'ColumnParallelLinear', 'RowParallelLinear' - ], - weight_quantize_type='abs_max', - activation_quantize_type='moving_average_abs_max', - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - fuse_conv_bn=False, - weight_preprocess_layer=None, - act_preprocess_layer=None, - weight_quantize_layer=None, - act_quantize_layer=None, - onnx_format=False): + def __init__( + self, + quantizable_layer_type=[ + 'Conv2D', + 'Linear', + 'Conv2DTranspose', + 'ColumnParallelLinear', + 'RowParallelLinear', + ], + weight_quantize_type='abs_max', + activation_quantize_type='moving_average_abs_max', + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + fuse_conv_bn=False, + weight_preprocess_layer=None, + act_preprocess_layer=None, + weight_quantize_layer=None, + act_quantize_layer=None, + onnx_format=False, + ): """ The constructor for ImperativeQuantAware. @@ -222,13 +230,14 @@ def forward(self, inputs): "weight_preprocess_layer": weight_preprocess_layer, "act_preprocess_layer": act_preprocess_layer, "weight_quantize_layer": weight_quantize_layer, - "act_quantize_layer": act_quantize_layer + "act_quantize_layer": act_quantize_layer, } self._quantize_inputs = ImperativeQuantizeInputs(**kwargs) self._quantize_outputs = ImperativeQuantizeOutputs( - moving_rate, activation_bits, onnx_format) + moving_rate, activation_bits, onnx_format + ) def quantize(self, model): """ @@ -278,8 +287,9 @@ def forward(self, inputs): # fake quant logical. imperative_qat.quantize(model) """ - assert isinstance(model, dygraph.Layer), \ - "The model must be the instance of dygraph.Layer." + assert isinstance( + model, dygraph.Layer + ), "The model must be the instance of dygraph.Layer." if self.fuse_conv_bn: fuse_utils.fuse_conv_bn(model) @@ -289,8 +299,9 @@ def forward(self, inputs): return model def save_quantized_model(self, layer, path, input_spec=None, **config): - self._quantize_outputs.save_quantized_model(layer, path, input_spec, - **config) + self._quantize_outputs.save_quantized_model( + layer, path, input_spec, **config + ) class ImperativeQuantizeInputs(object): @@ -299,17 +310,19 @@ class ImperativeQuantizeInputs(object): logic both for activation inputs and weight inputs. """ - def __init__(self, - quantizable_layer_type=['Conv2D', 'Linear', 'Conv2DTranspose'], - weight_quantize_type='abs_max', - activation_quantize_type='moving_average_abs_max', - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - weight_preprocess_layer=None, - act_preprocess_layer=None, - weight_quantize_layer=None, - act_quantize_layer=None): + def __init__( + self, + quantizable_layer_type=['Conv2D', 'Linear', 'Conv2DTranspose'], + weight_quantize_type='abs_max', + activation_quantize_type='moving_average_abs_max', + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + weight_preprocess_layer=None, + act_preprocess_layer=None, + weight_quantize_layer=None, + act_quantize_layer=None, + ): """ The constructor for ImperativeQuantizeInputs. @@ -317,48 +330,66 @@ def __init__(self, """ super(ImperativeQuantizeInputs, self).__init__() self.layer_name_map, self.fake_quant_input_layers = lazy_import_fleet( - utils.layer_name_map, utils.fake_quant_input_layers) + utils.layer_name_map, utils.fake_quant_input_layers + ) self._quantizable_layer_type = tuple( - self.layer_name_map[layer] if layer in - self.layer_name_map else layer for layer in quantizable_layer_type) + self.layer_name_map[layer] + if layer in self.layer_name_map + else layer + for layer in quantizable_layer_type + ) for layer in self._quantizable_layer_type: - assert not isinstance(layer, str) \ - and layer in self.fake_quant_input_layers, \ - "%s is unspported to be quantized." % layer + assert ( + not isinstance(layer, str) + and layer in self.fake_quant_input_layers + ), ("%s is unspported to be quantized." % layer) quantize_type = { - 'abs_max', 'moving_average_abs_max', 'channel_wise_abs_max', - 'lsq_weight', 'channel_wise_lsq_weight' + 'abs_max', + 'moving_average_abs_max', + 'channel_wise_abs_max', + 'lsq_weight', + 'channel_wise_lsq_weight', } act_quantize_type = {'moving_average_abs_max', 'lsq_act'} - assert weight_quantize_type != 'moving_average_abs_max' \ - and weight_quantize_type in quantize_type, \ - "Unsupported weight_quantize_type: %s. It can only " \ + assert ( + weight_quantize_type != 'moving_average_abs_max' + and weight_quantize_type in quantize_type + ), ( + "Unsupported weight_quantize_type: %s. It can only " "be abs_max or channel_wise_abs_max." % weight_quantize_type + ) # TODO (jc): activation_quantize_type supports range_abs_max - assert activation_quantize_type in act_quantize_type, \ - "Unsupported activation_quantize_type: %s. It can " \ - "only be moving_average_abs_max or lsq_act now." \ + assert activation_quantize_type in act_quantize_type, ( + "Unsupported activation_quantize_type: %s. It can " + "only be moving_average_abs_max or lsq_act now." % activation_quantize_type - - bits_check = lambda bits: isinstance(bits, int) \ - and bits >= 0 and bits <= 16 - assert bits_check(weight_bits), \ - "weight_bits should be 1, 2,... or 16." - assert bits_check(activation_bits), \ - "activation_bits should be 1, 2,... or 16." - - layer_check = lambda method: method is None or \ - issubclass(method, dygraph.layers.Layer) - assert layer_check(weight_preprocess_layer), \ - "weight_preprocess should be nn.Layer." - assert layer_check(act_preprocess_layer), \ - "act_preprocess should be nn.Layer." - assert layer_check(weight_quantize_layer), \ - "weight_quantize should be nn.Layer." - assert layer_check(act_quantize_layer), \ - "act_quantize should be nn.Layer." + ) + + bits_check = ( + lambda bits: isinstance(bits, int) and bits >= 0 and bits <= 16 + ) + assert bits_check(weight_bits), "weight_bits should be 1, 2,... or 16." + assert bits_check( + activation_bits + ), "activation_bits should be 1, 2,... or 16." + + layer_check = lambda method: method is None or issubclass( + method, dygraph.layers.Layer + ) + assert layer_check( + weight_preprocess_layer + ), "weight_preprocess should be nn.Layer." + assert layer_check( + act_preprocess_layer + ), "act_preprocess should be nn.Layer." + assert layer_check( + weight_quantize_layer + ), "weight_quantize should be nn.Layer." + assert layer_check( + act_quantize_layer + ), "act_quantize should be nn.Layer." self._kwargs = { "weight_quantize_type": weight_quantize_type, @@ -369,7 +400,7 @@ def __init__(self, "weight_pre_layer": weight_preprocess_layer, "act_pre_layer": act_preprocess_layer, "weight_quant_layer": weight_quantize_layer, - "act_quant_layer": act_quantize_layer + "act_quant_layer": act_quantize_layer, } def apply(self, model): @@ -385,17 +416,20 @@ def apply(self, model): None """ - assert isinstance(model, dygraph.Layer), \ - "The model must be the instance of dygraph.Layer." + assert isinstance( + model, dygraph.Layer + ), "The model must be the instance of dygraph.Layer." for name, cur_layer in model.named_sublayers(): - if not isinstance(cur_layer, self._quantizable_layer_type) \ - or (hasattr(cur_layer, "skip_quant") \ - and cur_layer.skip_quant == True): + if not isinstance(cur_layer, self._quantizable_layer_type) or ( + hasattr(cur_layer, "skip_quant") + and cur_layer.skip_quant == True + ): continue - parent_layer, sub_name = \ - utils.find_parent_layer_and_sub_name(model, name) + parent_layer, sub_name = utils.find_parent_layer_and_sub_name( + model, name + ) cur_quant_layer = self._get_input_quantized_layer(cur_layer) setattr(parent_layer, sub_name, cur_quant_layer) @@ -407,9 +441,9 @@ def _get_input_quantized_layer(self, layer): if isinstance(layer, value): quant_layer_name = 'Quantized' + key break - assert quant_layer_name is not None, \ - "The layer %s is unsupported to be quantized." \ - % layer.full_name() + assert quant_layer_name is not None, ( + "The layer %s is unsupported to be quantized." % layer.full_name() + ) return quant_layers.__dict__[quant_layer_name](layer, **self._kwargs) @@ -445,8 +479,9 @@ def apply(self, model): Returns: None """ - assert isinstance(model, dygraph.Layer), \ - "The model must be the instance of dygraph.Layer." + assert isinstance( + model, dygraph.Layer + ), "The model must be the instance of dygraph.Layer." for cur_name, cur_layer in model.named_sublayers(): if '_act_preprocess' in cur_name: @@ -454,17 +489,20 @@ def apply(self, model): if not self._is_target_layer(cur_layer): continue - parent_layer, sub_name = \ - utils.find_parent_layer_and_sub_name(model, cur_name) + parent_layer, sub_name = utils.find_parent_layer_and_sub_name( + model, cur_name + ) reduce_type = None if isinstance(cur_layer, tuple(utils.fake_quant_output_layers)): cur_quant_layer = quant_layers.FakeQuantMAOutputScaleLayer( - cur_layer, self._moving_rate, reduce_type=reduce_type) + cur_layer, self._moving_rate, reduce_type=reduce_type + ) else: cur_quant_layer = quant_layers.MAOutputScaleLayer( - cur_layer, self._moving_rate, reduce_type=reduce_type) + cur_layer, self._moving_rate, reduce_type=reduce_type + ) setattr(parent_layer, sub_name, cur_quant_layer) @@ -496,8 +534,9 @@ def save_quantized_model(self, model, path, input_spec=None, **config): Returns: None """ - assert isinstance(model, dygraph.Layer), \ - "The model must be the instance of dygraph.Layer." + assert isinstance( + model, dygraph.Layer + ), "The model must be the instance of dygraph.Layer." paddle.jit.save(layer=model, path=path, input_spec=input_spec, **config) @@ -515,11 +554,16 @@ def save_quantized_model(self, model, path, input_spec=None, **config): model_filename = basename + INFER_MODEL_SUFFIX params_filename = basename + INFER_PARAMS_SUFFIX - [infer_program, feed_target_names, fetch_targets - ] = (load_inference_model(dirname=dirname, - executor=exe, - model_filename=model_filename, - params_filename=params_filename)) + [ + infer_program, + feed_target_names, + fetch_targets, + ] = load_inference_model( + dirname=dirname, + executor=exe, + model_filename=model_filename, + params_filename=params_filename, + ) if not self._onnx_format: self._gather_scales(infer_program, scope, fetch_targets) @@ -539,7 +583,8 @@ def save_quantized_model(self, model, path, input_spec=None, **config): else: graph = IrGraph(core.Graph(infer_program.desc), for_test=False) transform_pass = ReplaceFakeQuantDequantPass( - scope, place, quant_bits=self._activation_bits) + scope, place, quant_bits=self._activation_bits + ) for sub_graph in graph.all_sub_graphs(): sub_graph._for_test = True transform_pass.apply(sub_graph) @@ -555,14 +600,16 @@ def save_quantized_model(self, model, path, input_spec=None, **config): move_persistable_var_to_global_block(infer_program) - save_inference_model(dirname=dirname, - feeded_var_names=feed_target_names, - target_vars=fetch_targets, - executor=exe, - main_program=infer_program.clone(), - model_filename=model_filename, - params_filename=params_filename, - clip_extra=clip_extra) + save_inference_model( + dirname=dirname, + feeded_var_names=feed_target_names, + target_vars=fetch_targets, + executor=exe, + main_program=infer_program.clone(), + model_filename=model_filename, + params_filename=params_filename, + clip_extra=clip_extra, + ) if is_dynamic_mode: paddle.disable_static() @@ -576,12 +623,16 @@ def _is_target_layer(self, layer): return False if self._onnx_format: - return True if isinstance(layer, tuple( - utils.fake_quant_wrap_layers)) else False + return ( + True + if isinstance(layer, tuple(utils.fake_quant_wrap_layers)) + else False + ) flag = False - if utils.is_leaf_layer(layer) and \ - not isinstance(layer, tuple(utils.fake_quant_leaf_layers)): + if utils.is_leaf_layer(layer) and not isinstance( + layer, tuple(utils.fake_quant_leaf_layers) + ): flag = True if isinstance(layer, tuple(utils.fake_quant_wrap_layers)): @@ -600,8 +651,9 @@ def _gather_scales(self, program, scope, fetch_targets): def _gather_input_scale(): target_ops = [] - skip_ops = utils.fake_quantize_dequantize_op_types + \ - ["moving_average_abs_max_scale"] + skip_ops = utils.fake_quantize_dequantize_op_types + [ + "moving_average_abs_max_scale" + ] for block in program.blocks: for op in block.ops: if op.type not in skip_ops: @@ -611,16 +663,19 @@ def _gather_input_scale(): for in_var_name in utils._get_op_input_var_names(op): previous_op = utils.find_previous_op(op.block, in_var_name) - if previous_op is not None and \ - ("quantize_dequantize" in previous_op.type or \ - previous_op.type == "moving_average_abs_max_scale"): + if previous_op is not None and ( + "quantize_dequantize" in previous_op.type + or previous_op.type == "moving_average_abs_max_scale" + ): scale_name = previous_op.output('OutScale')[0] in_scale = utils.load_variable_data(scope, scale_name) in_scale = utils.fp_numpy_to_naive(in_scale) argname, index = utils._get_input_name_index( - op, in_var_name) - op._set_attr(argname + str(index) + "_threshold", - in_scale) + op, in_var_name + ) + op._set_attr( + argname + str(index) + "_threshold", in_scale + ) op._set_attr("with_quant_attr", True) def _gather_output_scale(): @@ -646,7 +701,8 @@ def _gather_output_scale(): if res is not None: argname, index = res previous_op._set_attr( - argname + str(index) + "_threshold", out_scale) + argname + str(index) + "_threshold", out_scale + ) previous_op._set_attr("out_threshold", out_scale) previous_op._set_attr("with_quant_attr", True) @@ -678,12 +734,20 @@ def _is_skip_quant_op(self, block, in_op): 2. the previous ops of the input op are not fake_quantize_dequantize ops """ target_op_types = [ - "conv2d", "depthwise_conv2d", "matmul", "conv2d_transpose" + "conv2d", + "depthwise_conv2d", + "matmul", + "conv2d_transpose", ] if in_op.type not in target_op_types: return False - previous_ops = [utils.find_previous_op(block, arg_name) \ - for arg_name in in_op.input_arg_names] - return any(op is not None and op.type not in \ - utils.fake_quantize_dequantize_op_types for op in previous_ops) + previous_ops = [ + utils.find_previous_op(block, arg_name) + for arg_name in in_op.input_arg_names + ] + return any( + op is not None + and op.type not in utils.fake_quantize_dequantize_op_types + for op in previous_ops + ) diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/utils.py b/python/paddle/fluid/contrib/slim/quantization/imperative/utils.py index a30d775165e18..d771b51e09d11 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/utils.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/utils.py @@ -18,7 +18,12 @@ import paddle import paddle.nn.quant.quant_layers as quant_layers -from ..utils import _get_op_input_var_names, _get_op_output_var_names, _get_output_name_index, _get_input_name_index +from ..utils import ( + _get_op_input_var_names, + _get_op_output_var_names, + _get_output_name_index, + _get_input_name_index, +) layer_name_map = { 'Conv2DTranspose': paddle.nn.Conv2DTranspose, @@ -54,8 +59,10 @@ # TODO(jc): fix the problem of adding duplicate fake_quant ops # paddle.nn.AdaptiveAvgPool2D, paddle.nn.AvgPool2D, paddle.nn.ReLU,paddle.nn.LeakyReLU fake_quant_output_layers = [ - paddle.nn.quant.add, paddle.nn.quant.subtract, paddle.nn.quant.multiply, - paddle.nn.quant.divide + paddle.nn.quant.add, + paddle.nn.quant.subtract, + paddle.nn.quant.multiply, + paddle.nn.quant.divide, ] fake_quant_leaf_layers = [ @@ -66,24 +73,28 @@ ] fake_quant_wrap_layers = [ - quant_layers.QuantizedConv2D, quant_layers.QuantizedLinear, + quant_layers.QuantizedConv2D, + quant_layers.QuantizedLinear, quant_layers.QuantizedConv2DTranspose, quant_layers.QuantizedColumnParallelLinear, - quant_layers.QuantizedRowParallelLinear + quant_layers.QuantizedRowParallelLinear, ] # The weight format of these layers is Cin * Cout * H * W spec_channel_axis_layers = [paddle.nn.Conv2DTranspose, paddle.nn.Linear] weight_op_types = [ - "conv2d", "depthwise_conv2d", "matmul", "conv2d_transpose", - "depthwise_conv2d_transpose" + "conv2d", + "depthwise_conv2d", + "matmul", + "conv2d_transpose", + "depthwise_conv2d_transpose", ] fake_quantize_dequantize_op_types = [ "fake_quantize_dequantize_abs_max", "fake_channel_wise_quantize_dequantize_abs_max", - "fake_quantize_dequantize_moving_average_abs_max" + "fake_quantize_dequantize_moving_average_abs_max", ] @@ -92,8 +103,7 @@ def load_variable_data(scope, var_name): Load variable value from scope """ var_node = scope.find_var(var_name) - assert var_node is not None, \ - "Can not find " + var_name + " in the scope." + assert var_node is not None, "Can not find " + var_name + " in the scope." return np.array(var_node.get_tensor()) @@ -131,8 +141,9 @@ def find_parent_layer_and_sub_name(model, name): Returns: parent_layer, subname """ - assert isinstance(model, paddle.nn.Layer), \ - "The model must be the instance of paddle.nn.Layer." + assert isinstance( + model, paddle.nn.Layer + ), "The model must be the instance of paddle.nn.Layer." assert len(name) > 0, "The input (name) should not be empty." last_idx = 0 @@ -164,8 +175,7 @@ def is_leaf_layer(layer): """ Whether the layer is leaf layer. """ - return isinstance(layer, paddle.nn.Layer) \ - and len(layer.sublayers()) == 0 + return isinstance(layer, paddle.nn.Layer) and len(layer.sublayers()) == 0 def fp_numpy_to_naive(x_np): diff --git a/python/paddle/fluid/contrib/slim/quantization/post_training_quantization.py b/python/paddle/fluid/contrib/slim/quantization/post_training_quantization.py index 5c0fff9abe4e4..73d1e31159a6f 100644 --- a/python/paddle/fluid/contrib/slim/quantization/post_training_quantization.py +++ b/python/paddle/fluid/contrib/slim/quantization/post_training_quantization.py @@ -32,7 +32,14 @@ from ....executor import global_scope, Executor from ....framework import IrGraph from ....log_helper import get_logger -from .quantization_pass import QuantizationTransformPass, QuantizationTransformPassV2, QuantizationFreezePass, QuantWeightPass, AddQuantDequantPass, AddQuantDequantPassV2 +from .quantization_pass import ( + QuantizationTransformPass, + QuantizationTransformPassV2, + QuantizationFreezePass, + QuantWeightPass, + AddQuantDequantPass, + AddQuantDequantPassV2, +) from .cal_kl_threshold import cal_kl_threshold from .adaround import run_adaround from . import utils @@ -43,9 +50,9 @@ 'PostTrainingQuantizationProgram', ] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) def _all_persistable_var_names(program): @@ -68,8 +75,9 @@ def _remove_unused_var_nodes(graph): all_used_vars = {n.node for n in all_used_vars} all_unused_vars = { n - for n in filter(lambda node: node.node not in all_used_vars, - graph.all_var_nodes()) + for n in filter( + lambda node: node.node not in all_used_vars, graph.all_var_nodes() + ) } graph.safe_remove_nodes(all_unused_vars) return graph @@ -84,12 +92,9 @@ def _remove_ctrl_vars(graph): return graph -def _apply_pass(scope, - graph, - pass_name, - attrs=None, - attr_values=None, - debug=False): +def _apply_pass( + scope, graph, pass_name, attrs=None, attr_values=None, debug=False +): ir_pass = core.get_pass(pass_name) cpp_graph = graph.graph if not cpp_graph.has('__param_scope__'): @@ -114,37 +119,39 @@ class PostTrainingQuantization(object): quantized variables. """ - def __init__(self, - executor, - model_dir, - scope=None, - model_filename=None, - params_filename=None, - batch_generator=None, - sample_generator=None, - data_loader=None, - batch_size=10, - batch_nums=None, - algo="KL", - hist_percent=0.99999, - quantizable_op_type=["conv2d", "depthwise_conv2d", "mul"], - round_type='round', - learning_rate=0.001, - is_full_quantize=False, - bias_correction=False, - activation_bits=8, - weight_bits=8, - activation_quantize_type='range_abs_max', - weight_quantize_type='channel_wise_abs_max', - onnx_format=False, - freeze_model=True, - optimize_model=False, - is_use_cache_file=False, - skip_tensor_list=None, - same_scale_tensor_list=None, - cache_dir=None, - scale_dict=None, - return_graph=False): + def __init__( + self, + executor, + model_dir, + scope=None, + model_filename=None, + params_filename=None, + batch_generator=None, + sample_generator=None, + data_loader=None, + batch_size=10, + batch_nums=None, + algo="KL", + hist_percent=0.99999, + quantizable_op_type=["conv2d", "depthwise_conv2d", "mul"], + round_type='round', + learning_rate=0.001, + is_full_quantize=False, + bias_correction=False, + activation_bits=8, + weight_bits=8, + activation_quantize_type='range_abs_max', + weight_quantize_type='channel_wise_abs_max', + onnx_format=False, + freeze_model=True, + optimize_model=False, + is_use_cache_file=False, + skip_tensor_list=None, + same_scale_tensor_list=None, + cache_dir=None, + scale_dict=None, + return_graph=False, + ): ''' Constructor. @@ -274,38 +281,65 @@ def __init__(self, ''' self._support_activation_quantize_type = [ - 'range_abs_max', 'moving_average_abs_max', 'abs_max' + 'range_abs_max', + 'moving_average_abs_max', + 'abs_max', ] self._support_weight_quantize_type = ['abs_max', 'channel_wise_abs_max'] self._support_algo_type = [ - 'KL', 'hist', 'avg', 'mse', 'emd', 'abs_max', 'min_max', 'ptf' + 'KL', + 'hist', + 'avg', + 'mse', + 'emd', + 'abs_max', + 'min_max', + 'ptf', ] assert round_type in ['adaround', 'round'] self._round_type = round_type self._learning_rate = learning_rate self._dynamic_quantize_op_type = ['lstm'] - self._support_quantize_op_type = \ - list(set(utils._weight_supported_quantizable_op_type + - utils._act_supported_quantizable_op_type + - self._dynamic_quantize_op_type)) + self._support_quantize_op_type = list( + set( + utils._weight_supported_quantizable_op_type + + utils._act_supported_quantizable_op_type + + self._dynamic_quantize_op_type + ) + ) # Check inputs assert executor is not None, "The executor cannot be None." - assert any([gen is not None] for gen in [sample_generator, - batch_generator, data_loader]), "The sample_generator, batch_generator " \ + assert any( + [gen is not None] + for gen in [sample_generator, batch_generator, data_loader] + ), ( + "The sample_generator, batch_generator " "and data_loader cannot be None in the same time." + ) if data_loader is not None: - assert isinstance(data_loader, (io.DataLoader, type(isgeneratorfunction), reader.GeneratorLoader)), \ - "data_loader only accepts `paddle.io.DataLoader` or Generator instance." + assert isinstance( + data_loader, + ( + io.DataLoader, + type(isgeneratorfunction), + reader.GeneratorLoader, + ), + ), "data_loader only accepts `paddle.io.DataLoader` or Generator instance." assert batch_size > 0, "The batch_size should be greater than 0." - assert algo in self._support_algo_type, \ - "The algo should be KL, hist, mse, avg, abs_max, min_max or ptf." - assert activation_quantize_type in self._support_activation_quantize_type, \ - "The activation_quantize_type ({}) should in ({}).".format( - activation_quantize_type, self._support_activation_quantize_type) - assert weight_quantize_type in self._support_weight_quantize_type, \ - "The weight_quantize_type ({}) shoud in ({}).".format( - weight_quantize_type, self._support_weight_quantize_type) + assert ( + algo in self._support_algo_type + ), "The algo should be KL, hist, mse, avg, abs_max, min_max or ptf." + assert ( + activation_quantize_type in self._support_activation_quantize_type + ), "The activation_quantize_type ({}) should in ({}).".format( + activation_quantize_type, self._support_activation_quantize_type + ) + assert ( + weight_quantize_type in self._support_weight_quantize_type + ), "The weight_quantize_type ({}) shoud in ({}).".format( + weight_quantize_type, self._support_weight_quantize_type + ) # Save input params self._bias_correction = bias_correction @@ -333,8 +367,9 @@ def __init__(self, else: self._quantizable_op_type = quantizable_op_type for op_type in self._quantizable_op_type: - assert op_type in self._support_quantize_op_type, \ + assert op_type in self._support_quantize_op_type, ( op_type + " is not supported for quantization." + ) self._optimize_model = optimize_model # Define variables @@ -389,16 +424,18 @@ def quantize(self): if self._algo in ["KL", "hist"]: batch_id = 0 with tqdm( - total=self._batch_nums, - bar_format= - 'Preparation stage, Run batch:|{bar}| {n_fmt}/{total_fmt}', - ncols=80) as t: + total=self._batch_nums, + bar_format='Preparation stage, Run batch:|{bar}| {n_fmt}/{total_fmt}', + ncols=80, + ) as t: for data in self._data_loader(): - self._executor.run(program=self._program, - feed=data, - fetch_list=self._fetch_list, - return_numpy=False, - scope=self._scope) + self._executor.run( + program=self._program, + feed=data, + fetch_list=self._fetch_list, + return_numpy=False, + scope=self._scope, + ) self._collect_activation_abs_min_max() batch_id += 1 t.update() @@ -407,16 +444,19 @@ def quantize(self): self._init_sampling_act_histogram() batch_id = 0 - with tqdm(total=self._batch_nums, - bar_format= - 'Sampling stage, Run batch:|{bar}| {n_fmt}/{total_fmt}', - ncols=80) as t: + with tqdm( + total=self._batch_nums, + bar_format='Sampling stage, Run batch:|{bar}| {n_fmt}/{total_fmt}', + ncols=80, + ) as t: for data in self._data_loader(): - self._executor.run(program=self._program, - feed=data, - fetch_list=self._fetch_list, - return_numpy=False, - scope=self._scope) + self._executor.run( + program=self._program, + feed=data, + fetch_list=self._fetch_list, + return_numpy=False, + scope=self._scope, + ) self._sampling() batch_id += 1 t.update() @@ -425,8 +465,9 @@ def quantize(self): if self._algo == 'avg': for var_name in self._quantized_act_var_name: - self._quantized_threshold[var_name] = \ - np.array(self._quantized_var_avg[var_name]).mean() + self._quantized_threshold[var_name] = np.array( + self._quantized_var_avg[var_name] + ).mean() if self._algo in ["KL", "hist"]: self._calculate_kl_hist_threshold() @@ -444,10 +485,13 @@ def quantize(self): if not self.FLAG: self._save_output_threshold() - if any(op_type in self._quantizable_op_type - for op_type in self._dynamic_quantize_op_type): + if any( + op_type in self._quantizable_op_type + for op_type in self._dynamic_quantize_op_type + ): self._collect_dynamic_quantize_op_threshold( - self._dynamic_quantize_op_type) + self._dynamic_quantize_op_type + ) utils.move_persistable_var_to_global_block(self._program) @@ -463,23 +507,24 @@ def _adaround_apply(self): scale_dict = self._quantized_var_threshold else: scale_dict = self._quantized_threshold - run_adaround(self._data_loader, - self._program, - self._fetch_list, - self._executor, - self._scope, - self._place, - self._quantized_op_pairs, - self._weight_op_pairs, - scale_dict, - num_iterations=self._batch_nums, - bias_correction=self._bias_correction, - lr=self._learning_rate) - - def save_quantized_model(self, - save_model_path, - model_filename=None, - params_filename=None): + run_adaround( + self._data_loader, + self._program, + self._fetch_list, + self._executor, + self._scope, + self._place, + self._quantized_op_pairs, + self._weight_op_pairs, + scale_dict, + num_iterations=self._batch_nums, + bias_correction=self._bias_correction, + lr=self._learning_rate, + ) + + def save_quantized_model( + self, save_model_path, model_filename=None, params_filename=None + ): ''' Save the quantized model to the disk. @@ -494,14 +539,16 @@ def save_quantized_model(self, Returns: None ''' - io.save_inference_model(dirname=save_model_path, - model_filename=model_filename, - params_filename=params_filename, - feeded_var_names=self._feed_list, - target_vars=self._fetch_list, - executor=self._executor, - main_program=self._program, - clip_extra=self._clip_extra) + io.save_inference_model( + dirname=save_model_path, + model_filename=model_filename, + params_filename=params_filename, + feeded_var_names=self._feed_list, + target_vars=self._fetch_list, + executor=self._executor, + main_program=self._program, + clip_extra=self._clip_extra, + ) _logger.info("The quantized model is saved in " + save_model_path) def _load_model_data(self): @@ -510,36 +557,49 @@ def _load_model_data(self): ''' if self._program is None: _logger.info("Load model and set data loader ...") - [self._program, self._feed_list, self._fetch_list] = \ - io.load_inference_model(dirname=self._model_dir, - executor=self._executor, - model_filename=self._model_filename, - params_filename=self._params_filename) + [ + self._program, + self._feed_list, + self._fetch_list, + ] = io.load_inference_model( + dirname=self._model_dir, + executor=self._executor, + model_filename=self._model_filename, + params_filename=self._params_filename, + ) if self._optimize_model: self._optimize_fp32_model() - feed_vars = [framework._get_var(str(var_name), self._program) \ - for var_name in self._feed_list] + feed_vars = [ + framework._get_var(str(var_name), self._program) + for var_name in self._feed_list + ] if self._data_loader is not None: - self._batch_nums = self._batch_nums if self._batch_nums else len( - self._data_loader) + self._batch_nums = ( + self._batch_nums if self._batch_nums else len(self._data_loader) + ) return - self._data_loader = io.DataLoader.from_generator(feed_list=feed_vars, - capacity=3 * - self._batch_size, - iterable=True) + self._data_loader = io.DataLoader.from_generator( + feed_list=feed_vars, capacity=3 * self._batch_size, iterable=True + ) if self._sample_generator is not None: - self._data_loader.set_sample_generator(self._sample_generator, - batch_size=self._batch_size, - drop_last=True, - places=self._place) + self._data_loader.set_sample_generator( + self._sample_generator, + batch_size=self._batch_size, + drop_last=True, + places=self._place, + ) elif self._batch_generator is not None: - self._data_loader.set_batch_generator(self._batch_generator, - places=self._place) - self._batch_nums = self._batch_nums if self._batch_nums else len( - list(self._data_loader)) + self._data_loader.set_batch_generator( + self._batch_generator, places=self._place + ) + self._batch_nums = ( + self._batch_nums + if self._batch_nums + else len(list(self._data_loader)) + ) def _optimize_fp32_model(self): ''' @@ -552,8 +612,9 @@ def _optimize_fp32_model(self): graph = _apply_pass(self._scope, graph, 'depthwise_conv_bn_fuse_pass') graph = _apply_pass(self._scope, graph, 'conv_transpose_bn_fuse_pass') graph = _apply_pass(self._scope, graph, 'conv_eltwiseadd_bn_fuse_pass') - graph = _apply_pass(self._scope, graph, - 'depthwise_conv_eltwiseadd_bn_fuse_pass') + graph = _apply_pass( + self._scope, graph, 'depthwise_conv_eltwiseadd_bn_fuse_pass' + ) self._program = graph.to_program() @@ -584,26 +645,39 @@ def collect_var_name(var_name_list, persistable_var_names, op_type): op._set_attr("op_namescope", "skip_quant") op_type = op.type - if self._is_full_quantize and \ - op_type not in self._quantizable_op_type: - _logger.warning(op_type + - " is not supported for quantization.") + if ( + self._is_full_quantize + and op_type not in self._quantizable_op_type + ): + _logger.warning( + op_type + " is not supported for quantization." + ) # For quantized ops, sample inputs and outputs if op_type in self._quantizable_op_type: - collect_var_name(utils._get_op_input_var_names(op), - persistable_var_names, op_type) - collect_var_name(utils._get_op_output_var_names(op), - persistable_var_names, op_type) + collect_var_name( + utils._get_op_input_var_names(op), + persistable_var_names, + op_type, + ) + collect_var_name( + utils._get_op_output_var_names(op), + persistable_var_names, + op_type, + ) # collect quanted op output var name for out_var_name in utils._get_op_output_var_names(op): for in_var_name in utils._get_op_input_var_names(op): if in_var_name in persistable_var_names: self._quantized_op_pairs[ - in_var_name] = out_var_name + in_var_name + ] = out_var_name # For other op, only sample output scale elif op_type in self._out_scale_op_list: - collect_var_name(utils._get_op_output_var_names(op), - persistable_var_names, op_type) + collect_var_name( + utils._get_op_output_var_names(op), + persistable_var_names, + op_type, + ) def _set_activation_persistable(self): ''' @@ -651,15 +725,19 @@ def _sample_mse(self): abs_max_value = float(np.max(np.abs(var_tensor))) elif self._weight_quantize_type == "channel_wise_abs_max": abs_max_value = [] - if self._weight_op_pairs[ - var_name] in utils._channelwise_quant_axis1_ops: + if ( + self._weight_op_pairs[var_name] + in utils._channelwise_quant_axis1_ops + ): for i in range(var_tensor.shape[1]): abs_max_value.append( - float(np.max(np.abs(var_tensor[:, i])))) + float(np.max(np.abs(var_tensor[:, i]))) + ) else: for i in range(var_tensor.shape[0]): abs_max_value.append( - float(np.max(np.abs(var_tensor[i])))) + float(np.max(np.abs(var_tensor[i]))) + ) self._quantized_threshold[var_name] = abs_max_value _logger.info("MSE searching stage ...") for var_name in self._quantized_act_var_name: @@ -673,16 +751,19 @@ def _sample_mse(self): while s <= 1.0: scale = s * abs_max_value s += 0.02 - bins = 2**(self._activation_bits - 1) - 1 + bins = 2 ** (self._activation_bits - 1) - 1 if self._onnx_format: - quant_var = np.clip(np.round(var_tensor / scale * bins), - -bins - 1, bins) + quant_var = np.clip( + np.round(var_tensor / scale * bins), -bins - 1, bins + ) quant_dequant_var = quant_var / bins * scale else: - quant_dequant_var = np.round( - np.clip(var_tensor, 0.0, scale) / scale * - bins) / bins * scale - mse_loss = ((var_tensor - quant_dequant_var)**2).mean() + quant_dequant_var = ( + np.round(np.clip(var_tensor, 0.0, scale) / scale * bins) + / bins + * scale + ) + mse_loss = ((var_tensor - quant_dequant_var) ** 2).mean() if mse_loss <= self._best_calibration_loss[var_name]: self._best_calibration_loss[var_name] = mse_loss self._quantized_threshold[var_name] = scale @@ -695,15 +776,19 @@ def _sample_emd(self): abs_max_value = float(np.max(np.abs(var_tensor))) elif self._weight_quantize_type == "channel_wise_abs_max": abs_max_value = [] - if self._weight_op_pairs[ - var_name] in utils._channelwise_quant_axis1_ops: + if ( + self._weight_op_pairs[var_name] + in utils._channelwise_quant_axis1_ops + ): for i in range(var_tensor.shape[1]): abs_max_value.append( - float(np.max(np.abs(var_tensor[:, i])))) + float(np.max(np.abs(var_tensor[:, i]))) + ) else: for i in range(var_tensor.shape[0]): abs_max_value.append( - float(np.max(np.abs(var_tensor[i])))) + float(np.max(np.abs(var_tensor[i]))) + ) self._quantized_threshold[var_name] = abs_max_value _logger.info("EMD searching stage ...") for var_name in self._quantized_act_var_name: @@ -717,18 +802,21 @@ def _sample_emd(self): while s <= 1.0: scale = s * abs_max_value s += 0.02 - bins = 2**(self._activation_bits - 1) - 1 + bins = 2 ** (self._activation_bits - 1) - 1 if self._onnx_format: - quant_var = np.clip(np.round(var_tensor / scale * bins), - -bins - 1, bins) + quant_var = np.clip( + np.round(var_tensor / scale * bins), -bins - 1, bins + ) quant_dequant_var = quant_var / bins * scale else: - quant_dequant_var = np.round( - np.clip(var_tensor, 0.0, scale) / scale * - bins) / bins * scale + quant_dequant_var = ( + np.round(np.clip(var_tensor, 0.0, scale) / scale * bins) + / bins + * scale + ) emd_loss = np.abs( - np.mean(var_tensor) - np.mean(quant_dequant_var)) + np.abs( - np.std(var_tensor) - np.std(quant_dequant_var)) + np.mean(var_tensor) - np.mean(quant_dequant_var) + ) + np.abs(np.std(var_tensor) - np.std(quant_dequant_var)) if emd_loss <= self._best_calibration_loss[var_name]: self._best_calibration_loss[var_name] = emd_loss self._quantized_threshold[var_name] = scale @@ -741,24 +829,34 @@ def _sample_avg(self): abs_max_value = float(np.max(np.abs(var_tensor))) elif self._weight_quantize_type == "channel_wise_abs_max": abs_max_value = [] - if self._weight_op_pairs[ - var_name] in utils._channelwise_quant_axis1_ops: + if ( + self._weight_op_pairs[var_name] + in utils._channelwise_quant_axis1_ops + ): for i in range(var_tensor.shape[1]): abs_max_value.append( - float(np.max(np.abs(var_tensor[:, i])))) + float(np.max(np.abs(var_tensor[:, i]))) + ) else: for i in range(var_tensor.shape[0]): abs_max_value.append( - float(np.max(np.abs(var_tensor[i])))) + float(np.max(np.abs(var_tensor[i]))) + ) self._quantized_threshold[var_name] = abs_max_value for var_name in self._quantized_act_var_name: var_tensor = utils.load_variable_data(self._scope, var_name) abs_max_value = float(np.max(np.abs(var_tensor))) - if (var_name not in self._quantized_var_avg): + if var_name not in self._quantized_var_avg: self._quantized_var_avg[var_name] = [] - abs_avg_value = float(np.mean(np.max( \ - np.abs(var_tensor.reshape(var_tensor.shape[0], -1)), axis=(1)))) + abs_avg_value = float( + np.mean( + np.max( + np.abs(var_tensor.reshape(var_tensor.shape[0], -1)), + axis=(1), + ) + ) + ) self._quantized_var_avg[var_name].append(abs_avg_value) continue @@ -770,22 +868,27 @@ def _sample_abs_max(self): abs_max_value = float(np.max(np.abs(var_tensor))) elif self._weight_quantize_type == "channel_wise_abs_max": abs_max_value = [] - if self._weight_op_pairs[ - var_name] in utils._channelwise_quant_axis1_ops: + if ( + self._weight_op_pairs[var_name] + in utils._channelwise_quant_axis1_ops + ): for i in range(var_tensor.shape[1]): abs_max_value.append( - float(np.max(np.abs(var_tensor[:, i])))) + float(np.max(np.abs(var_tensor[:, i]))) + ) else: for i in range(var_tensor.shape[0]): abs_max_value.append( - float(np.max(np.abs(var_tensor[i])))) + float(np.max(np.abs(var_tensor[i]))) + ) self._quantized_threshold[var_name] = abs_max_value for var_name in self._quantized_act_var_name: var_tensor = utils.load_variable_data(self._scope, var_name) abs_max_value = float(np.max(np.abs(var_tensor))) - if (var_name not in self._quantized_threshold) or \ - (abs_max_value > self._quantized_threshold[var_name]): + if (var_name not in self._quantized_threshold) or ( + abs_max_value > self._quantized_threshold[var_name] + ): self._quantized_threshold[var_name] = abs_max_value def _sample_min_max(self): @@ -798,8 +901,10 @@ def _sample_min_max(self): elif self._weight_quantize_type == "channel_wise_abs_max": min_value = [] max_value = [] - if self._weight_op_pairs[ - var_name] in utils._channelwise_quant_axis1_ops: + if ( + self._weight_op_pairs[var_name] + in utils._channelwise_quant_axis1_ops + ): for i in range(var_tensor.shape[1]): min_value.append(float(np.min(var_tensor[:, i]))) max_value.append(float(np.max(var_tensor[:, i]))) @@ -814,11 +919,13 @@ def _sample_min_max(self): var_tensor = utils.load_variable_data(self._scope, var_name) min_value = float(np.min(var_tensor)) max_value = float(np.max(var_tensor)) - if (var_name not in self._quantized_var_min) or \ - (min_value < self._quantized_var_min[var_name]): + if (var_name not in self._quantized_var_min) or ( + min_value < self._quantized_var_min[var_name] + ): self._quantized_var_min[var_name] = min_value - if (var_name not in self._quantized_var_max) or \ - (max_value > self._quantized_var_max[var_name]): + if (var_name not in self._quantized_var_max) or ( + max_value > self._quantized_var_max[var_name] + ): self._quantized_var_max[var_name] = max_value def _sample_histogram(self): @@ -841,39 +948,47 @@ def _sample_ptf(self): abs_max_value = float(np.max(np.abs(var_tensor))) elif self._weight_quantize_type == "channel_wise_abs_max": abs_max_value = [] - if self._weight_op_pairs[ - var_name] in utils._channelwise_quant_axis1_ops: + if ( + self._weight_op_pairs[var_name] + in utils._channelwise_quant_axis1_ops + ): for i in range(var_tensor.shape[1]): abs_max_value.append( - float(np.max(np.abs(var_tensor[:, i])))) + float(np.max(np.abs(var_tensor[:, i]))) + ) else: for i in range(var_tensor.shape[0]): abs_max_value.append( - float(np.max(np.abs(var_tensor[i])))) + float(np.max(np.abs(var_tensor[i]))) + ) self._quantized_threshold[var_name] = abs_max_value for var_name in self._quantized_act_var_name: var_tensor = utils.load_variable_data(self._scope, var_name) abs_max_value = float(np.max(np.abs(var_tensor))) - q_max = 2**(self._activation_bits - 1) - 1 + q_max = 2 ** (self._activation_bits - 1) - 1 scale8 = abs_max_value / q_max scale4 = scale8 / 2 scale2 = scale4 / 2 scale1 = scale2 / 2 - quant_dequant_var_scale1 = np.clip(np.round(var_tensor / scale1), 0, - q_max) * scale1 - quant_dequant_var_scale2 = np.clip(np.round(var_tensor / scale2), 0, - q_max) * scale2 - quant_dequant_var_scale4 = np.clip(np.round(var_tensor / scale4), 0, - q_max) * scale4 - quant_dequant_var_scale8 = np.clip(np.round(var_tensor / scale8), 0, - q_max) * scale8 + quant_dequant_var_scale1 = ( + np.clip(np.round(var_tensor / scale1), 0, q_max) * scale1 + ) + quant_dequant_var_scale2 = ( + np.clip(np.round(var_tensor / scale2), 0, q_max) * scale2 + ) + quant_dequant_var_scale4 = ( + np.clip(np.round(var_tensor / scale4), 0, q_max) * scale4 + ) + quant_dequant_var_scale8 = ( + np.clip(np.round(var_tensor / scale8), 0, q_max) * scale8 + ) score1 = utils.l2_loss(var_tensor, quant_dequant_var_scale1) score2 = utils.l2_loss(var_tensor, quant_dequant_var_scale2) score4 = utils.l2_loss(var_tensor, quant_dequant_var_scale4) score8 = utils.l2_loss(var_tensor, quant_dequant_var_scale8) score = [score1, score2, score4, score8] - mask = 2**score.index(min(score)) + mask = 2 ** score.index(min(score)) scale = scale1 * mask threshold = q_max * scale self._quantized_threshold[var_name] = threshold @@ -882,18 +997,21 @@ def _save_input_threhold(self): ''' Save input threshold to the quantized op. ''' - assert self._algo == "min_max", \ - "The algo should be min_max to save input threshold." + assert ( + self._algo == "min_max" + ), "The algo should be min_max to save input threshold." for block_id in range(len(self._program.blocks)): for op in self._program.blocks[block_id].ops: if op.type in self._quantizable_op_type: for var_name in utils._get_op_input_var_names(op): assert var_name in self._quantized_var_min assert var_name in self._quantized_var_max - op._set_attr(var_name + ".min", - self._quantized_var_min[var_name]) - op._set_attr(var_name + ".max", - self._quantized_var_max[var_name]) + op._set_attr( + var_name + ".min", self._quantized_var_min[var_name] + ) + op._set_attr( + var_name + ".max", self._quantized_var_max[var_name] + ) op._set_attr("with_quant_attr", True) def _collect_activation_abs_min_max(self): @@ -908,7 +1026,8 @@ def _collect_activation_abs_min_max(self): max_value = float(np.max(var_tensor)) if var_name not in self._sampling_act_abs_min_max: self._sampling_act_abs_min_max[var_name] = [ - min_value, max_value + min_value, + max_value, ] else: if min_value < self._sampling_act_abs_min_max[var_name][0]: @@ -924,9 +1043,9 @@ def _init_sampling_act_histogram(self): if var_name not in self._sampling_act_histogram: min_val = self._sampling_act_abs_min_max[var_name][0] max_val = self._sampling_act_abs_min_max[var_name][1] - hist, hist_edeges = np.histogram([], - bins=self._histogram_bins, - range=(min_val, max_val)) + hist, hist_edeges = np.histogram( + [], bins=self._histogram_bins, range=(min_val, max_val) + ) self._sampling_act_histogram[var_name] = [hist, hist_edeges] def _calculate_kl_hist_threshold(self): @@ -943,26 +1062,32 @@ def _calculate_kl_hist_threshold(self): weight_threshold = float(np.max(np.abs(weight_data))) elif self._weight_quantize_type == "channel_wise_abs_max": weight_threshold = [] - if self._weight_op_pairs[ - var_name] in utils._channelwise_quant_axis1_ops: + if ( + self._weight_op_pairs[var_name] + in utils._channelwise_quant_axis1_ops + ): for i in range(weight_data.shape[1]): weight_threshold.append( - float(np.max(np.abs(weight_data[:, i])))) + float(np.max(np.abs(weight_data[:, i]))) + ) else: for i in range(weight_data.shape[0]): weight_threshold.append( - float(np.max(np.abs(weight_data[i])))) + float(np.max(np.abs(weight_data[i]))) + ) self._quantized_var_threshold[var_name] = weight_threshold for var_name in self._quantized_act_var_name: hist, hist_edeges = self._sampling_act_histogram[var_name] if self._algo == "KL": bin_width = hist_edeges[1] - hist_edeges[0] - self._quantized_var_threshold[var_name] = \ - cal_kl_threshold(hist, bin_width, self._activation_bits) + self._quantized_var_threshold[var_name] = cal_kl_threshold( + hist, bin_width, self._activation_bits + ) elif self._algo == "hist": - self._quantized_var_threshold[var_name] = \ - self._get_hist_scaling_factor(hist, hist_edeges) + self._quantized_var_threshold[ + var_name + ] = self._get_hist_scaling_factor(hist, hist_edeges) def _update_program(self): ''' @@ -986,7 +1111,8 @@ def _update_program(self): activation_bits=self._activation_bits, activation_quantize_type=self._activation_quantize_type, weight_quantize_type=self._weight_quantize_type, - quantizable_op_type=major_quantizable_op_types) + quantizable_op_type=major_quantizable_op_types, + ) else: transform_pass = QuantizationTransformPassV2( scope=self._scope, @@ -995,7 +1121,8 @@ def _update_program(self): activation_bits=self._activation_bits, activation_quantize_type=self._activation_quantize_type, weight_quantize_type=self._weight_quantize_type, - quantizable_op_type=major_quantizable_op_types) + quantizable_op_type=major_quantizable_op_types, + ) for sub_graph in graph.all_sub_graphs(): # Insert fake_quant/fake_dequantize op must in test graph, so @@ -1012,13 +1139,15 @@ def _update_program(self): add_quant_dequant_pass = AddQuantDequantPass( scope=self._scope, place=self._place, - quantizable_op_type=minor_quantizable_op_types) + quantizable_op_type=minor_quantizable_op_types, + ) else: add_quant_dequant_pass = AddQuantDequantPassV2( scope=self._scope, place=self._place, quantizable_op_type=minor_quantizable_op_types, - is_full_quantized=True) + is_full_quantized=True, + ) for sub_graph in graph.all_sub_graphs(): sub_graph._for_test = True @@ -1038,41 +1167,49 @@ def _update_program(self): for tensor_name in tensor_list: if '#' in tensor_name: real_tensor_name, opera, scalar = tensor_name.split( - '#') + '#' + ) if real_tensor_name not in scale_dict.keys(): continue if opera == '*': scale_dict[real_tensor_name] = float( - scale_dict[real_tensor_name]) * float( - scalar) + scale_dict[real_tensor_name] + ) * float(scalar) elif opera == '/': scale_dict[real_tensor_name] = float( - scale_dict[real_tensor_name]) / float( - scalar) - max_scale = scale_dict[ - real_tensor_name] if max_scale is None else max( - max_scale, scale_dict[real_tensor_name]) + scale_dict[real_tensor_name] + ) / float(scalar) + max_scale = ( + scale_dict[real_tensor_name] + if max_scale is None + else max( + max_scale, scale_dict[real_tensor_name] + ) + ) else: if tensor_name not in scale_dict.keys(): continue - max_scale = scale_dict[ - tensor_name] if max_scale is None else max( - max_scale, scale_dict[tensor_name]) + max_scale = ( + scale_dict[tensor_name] + if max_scale is None + else max(max_scale, scale_dict[tensor_name]) + ) for tensor_name in tensor_list: if '#' in tensor_name: real_tensor_name, opera, scalar = tensor_name.split( - '#') + '#' + ) if real_tensor_name not in scale_dict.keys(): continue if opera == '*': scale_dict[ - real_tensor_name] = max_scale / float( - scalar) + real_tensor_name + ] = max_scale / float(scalar) elif opera == '/': scale_dict[ - real_tensor_name] = max_scale * float( - scalar) + real_tensor_name + ] = max_scale * float(scalar) else: if tensor_name not in scale_dict.keys(): continue @@ -1080,11 +1217,18 @@ def _update_program(self): self._scale_dict = scale_dict for key, val in self._scale_dict.items(): - utils.set_variable_data(self._scope, self._place, key + "@scale", - np.array([val], dtype=np.float32)) - utils.set_variable_data(self._scope, self._place, - key + ".quant_dequant@scale", - np.array([val], dtype=np.float32)) + utils.set_variable_data( + self._scope, + self._place, + key + "@scale", + np.array([val], dtype=np.float32), + ) + utils.set_variable_data( + self._scope, + self._place, + key + ".quant_dequant@scale", + np.array([val], dtype=np.float32), + ) if not self._onnx_format: # apply QuantizationFreezePass, and obtain the final quant model @@ -1097,7 +1241,8 @@ def _update_program(self): round_type=self._round_type, activation_bits=self._activation_bits, weight_quantize_type=self._weight_quantize_type, - quantizable_op_type=major_quantizable_op_types) + quantizable_op_type=major_quantizable_op_types, + ) for sub_graph in graph.all_sub_graphs(): sub_graph._for_test = True @@ -1116,16 +1261,20 @@ def _save_output_threshold(self): ''' self._calibration_scales = {} - def save_info(op_node, out_var_name, threshold_map, out_info_name, - quantized_type): - assert out_var_name in threshold_map, \ - "The output ({}) of {} node does not have threshold.".format( - out_var_name, op_node.type) + def save_info( + op_node, out_var_name, threshold_map, out_info_name, quantized_type + ): + assert ( + out_var_name in threshold_map + ), "The output ({}) of {} node does not have threshold.".format( + out_var_name, op_node.type + ) if self._onnx_format: # For easy extension, every var_node set a dict to save parameters of quant. self._calibration_scales[var_name] = {} self._calibration_scales[var_name]['scale'] = threshold_map[ - var_name] + var_name + ] else: op_node._set_attr(out_info_name, threshold_map[var_name]) op_node._set_attr("with_quant_attr", True) @@ -1134,42 +1283,78 @@ def save_info(op_node, out_var_name, threshold_map, out_info_name, def analysis_and_save_info(op_node, out_var_name): argname_index = utils._get_output_name_index(op_node, out_var_name) - assert argname_index is not None, \ + assert argname_index is not None, ( out_var_name + " is not the output of the op" + ) if self._algo == "KL": # For compatibility, we save output threshold by two methods. - save_info(op_node, out_var_name, self._quantized_var_threshold, - "out_threshold", "post_kl") save_info( - op_node, out_var_name, self._quantized_var_threshold, + op_node, + out_var_name, + self._quantized_var_threshold, + "out_threshold", + "post_kl", + ) + save_info( + op_node, + out_var_name, + self._quantized_var_threshold, argname_index[0] + str(argname_index[1]) + "_threshold", - "post_kl") + "post_kl", + ) elif self._algo == "hist": # For compatibility, we save output threshold by two methods. - save_info(op_node, out_var_name, self._quantized_var_threshold, - "out_threshold", "post_hist") save_info( - op_node, out_var_name, self._quantized_var_threshold, + op_node, + out_var_name, + self._quantized_var_threshold, + "out_threshold", + "post_hist", + ) + save_info( + op_node, + out_var_name, + self._quantized_var_threshold, argname_index[0] + str(argname_index[1]) + "_threshold", - "post_hist") + "post_hist", + ) elif self._algo in ["avg", "abs_max", "mse", "emd", "ptf"]: - save_info(op_node, out_var_name, self._quantized_threshold, - "out_threshold", "post_" + str(self._algo)) save_info( - op_node, out_var_name, self._quantized_threshold, + op_node, + out_var_name, + self._quantized_threshold, + "out_threshold", + "post_" + str(self._algo), + ) + save_info( + op_node, + out_var_name, + self._quantized_threshold, argname_index[0] + str(argname_index[1]) + "_threshold", - "post_" + str(self._algo)) + "post_" + str(self._algo), + ) elif self._algo == "min_max": - save_info(op_node, out_var_name, self._quantized_var_min, - "out_min", "post_min_max") - save_info(op_node, out_var_name, self._quantized_var_max, - "out_max", "post_min_max") + save_info( + op_node, + out_var_name, + self._quantized_var_min, + "out_min", + "post_min_max", + ) + save_info( + op_node, + out_var_name, + self._quantized_var_max, + "out_max", + "post_min_max", + ) for block_id in range(len(self._program.blocks)): for op in self._program.blocks[block_id].ops: - if op.type in (self._quantizable_op_type + - self._out_scale_op_list): + if op.type in ( + self._quantizable_op_type + self._out_scale_op_list + ): out_var_names = utils._get_op_output_var_names(op) for var_name in out_var_names: analysis_and_save_info(op, var_name) @@ -1221,55 +1406,77 @@ def _get_hist_scaling_factor(self, hist, hist_edges): class PostTrainingQuantizationProgram(PostTrainingQuantization): - - def __init__(self, - executor, - program, - feed_list=None, - fetch_list=None, - scope=None, - batch_generator=None, - sample_generator=None, - data_loader=None, - batch_size=10, - batch_nums=None, - algo="KL", - hist_percent=0.99999, - quantizable_op_type=["conv2d", "depthwise_conv2d", "mul"], - round_type='round', - learning_rate=0.001, - is_full_quantize=False, - bias_correction=False, - activation_bits=8, - weight_bits=8, - activation_quantize_type='range_abs_max', - weight_quantize_type='channel_wise_abs_max', - onnx_format=False, - freeze_model=True, - optimize_model=False, - is_use_cache_file=False, - skip_tensor_list=None, - same_scale_tensor_list=None, - cache_dir=None, - scale_dict=None, - return_graph=True): - super().__init__(executor, scope, None, None, None, batch_generator, - sample_generator, data_loader, batch_size, batch_nums, - algo, hist_percent, quantizable_op_type, round_type, - learning_rate, is_full_quantize, bias_correction, - activation_bits, weight_bits, activation_quantize_type, - weight_quantize_type, onnx_format, freeze_model, - optimize_model, is_use_cache_file, skip_tensor_list, - same_scale_tensor_list, cache_dir, scale_dict, - return_graph) + def __init__( + self, + executor, + program, + feed_list=None, + fetch_list=None, + scope=None, + batch_generator=None, + sample_generator=None, + data_loader=None, + batch_size=10, + batch_nums=None, + algo="KL", + hist_percent=0.99999, + quantizable_op_type=["conv2d", "depthwise_conv2d", "mul"], + round_type='round', + learning_rate=0.001, + is_full_quantize=False, + bias_correction=False, + activation_bits=8, + weight_bits=8, + activation_quantize_type='range_abs_max', + weight_quantize_type='channel_wise_abs_max', + onnx_format=False, + freeze_model=True, + optimize_model=False, + is_use_cache_file=False, + skip_tensor_list=None, + same_scale_tensor_list=None, + cache_dir=None, + scale_dict=None, + return_graph=True, + ): + super().__init__( + executor, + scope, + None, + None, + None, + batch_generator, + sample_generator, + data_loader, + batch_size, + batch_nums, + algo, + hist_percent, + quantizable_op_type, + round_type, + learning_rate, + is_full_quantize, + bias_correction, + activation_bits, + weight_bits, + activation_quantize_type, + weight_quantize_type, + onnx_format, + freeze_model, + optimize_model, + is_use_cache_file, + skip_tensor_list, + same_scale_tensor_list, + cache_dir, + scale_dict, + return_graph, + ) self.FLAG = False self._program = program if self._program is not None: self.FLAG = True - assert feed_list is not None, \ - "Feed list should not be None." - assert fetch_list is not None, \ - "Fetch list should not be None." + assert feed_list is not None, "Feed list should not be None." + assert fetch_list is not None, "Fetch list should not be None." self._feed_list = feed_list self._fetch_list = fetch_list @@ -1298,15 +1505,17 @@ def __init__(self, model_dir, model_filename=None, params_filename=None): self._model_filename = model_filename self._params_filename = params_filename - def quantize_weight_to_int(self, - save_model_dir, - save_model_filename=None, - save_params_filename=None, - quantizable_op_type=["conv2d", "mul"], - weight_bits=8, - weight_quantize_type="channel_wise_abs_max", - generate_test_model=False, - threshold_rate=0.0): + def quantize_weight_to_int( + self, + save_model_dir, + save_model_filename=None, + save_params_filename=None, + quantizable_op_type=["conv2d", "mul"], + weight_bits=8, + weight_quantize_type="channel_wise_abs_max", + generate_test_model=False, + threshold_rate=0.0, + ): ''' In order to reduce the size of model, this api quantizes the weight of some ops from float32 to int8/16. In the inference stage, the @@ -1342,28 +1551,45 @@ def quantize_weight_to_int(self, value will be optimized. Default is 0.0. ''' for op_type in quantizable_op_type: - assert op_type in self._supported_quantizable_op_type, \ - "Input error:" + op_type + \ - " is not supported for weight quantization." - assert weight_bits in [8, 16], \ - "Input error: weight_bits should be 8 or 16." - assert weight_quantize_type in self._supported_weight_quantize_type, \ - "Input error: weight_quantize_type should in {}".format( - self._supported_weight_quantize_type) + assert op_type in self._supported_quantizable_op_type, ( + "Input error:" + + op_type + + " is not supported for weight quantization." + ) + assert weight_bits in [ + 8, + 16, + ], "Input error: weight_bits should be 8 or 16." + assert ( + weight_quantize_type in self._supported_weight_quantize_type + ), "Input error: weight_quantize_type should in {}".format( + self._supported_weight_quantize_type + ) quantized_model_dir = os.path.join(save_model_dir, "quantized_model") - self._quantize_weight_to_int(quantized_model_dir, save_model_filename, - save_params_filename, quantizable_op_type, - weight_bits, weight_quantize_type, False, - threshold_rate) + self._quantize_weight_to_int( + quantized_model_dir, + save_model_filename, + save_params_filename, + quantizable_op_type, + weight_bits, + weight_quantize_type, + False, + threshold_rate, + ) if generate_test_model: test_model_dir = os.path.join(save_model_dir, "test_model") - self._quantize_weight_to_int(test_model_dir, save_model_filename, - save_params_filename, - quantizable_op_type, weight_bits, - weight_quantize_type, True, - threshold_rate) + self._quantize_weight_to_int( + test_model_dir, + save_model_filename, + save_params_filename, + quantizable_op_type, + weight_bits, + weight_quantize_type, + True, + threshold_rate, + ) def convert_weight_to_fp16(self, save_model_dir): """ @@ -1379,11 +1605,12 @@ def convert_weight_to_fp16(self, save_model_dir): place = core.CPUPlace() exe = Executor(place) scope = global_scope() - [infer_program, feed_list, fetch_list] = \ - io.load_inference_model(dirname=self._model_dir, - executor=exe, - model_filename=self._model_filename, - params_filename=self._params_filename) + [infer_program, feed_list, fetch_list] = io.load_inference_model( + dirname=self._model_dir, + executor=exe, + model_filename=self._model_filename, + params_filename=self._params_filename, + ) # Clone and save fp16 weights save_program = framework.Program() @@ -1391,27 +1618,31 @@ def convert_weight_to_fp16(self, save_model_dir): save_var_map = {} for var in infer_program.list_vars(): - if (var.type == core.VarDesc.VarType.RAW) or \ - (not var.persistable) or (var.name in ['feed', 'fetch']) \ - or (var.dtype != core.VarDesc.VarType.FP32): + if ( + (var.type == core.VarDesc.VarType.RAW) + or (not var.persistable) + or (var.name in ['feed', 'fetch']) + or (var.dtype != core.VarDesc.VarType.FP32) + ): continue - #new_var = _clone_var_to_block_(var, save_block) + # new_var = _clone_var_to_block_(var, save_block) new_var = save_block._clone_variable(var) if self._params_filename is not None: save_var_map[new_var.name] = new_var else: - save_file_path = os.path.join(os.path.normpath(save_model_dir), - new_var.name) - save_block.append_op(type='save', - inputs={'X': [new_var]}, - outputs={}, - attrs={ - 'file_path': - os.path.normpath(save_file_path), - 'save_as_fp16': - True - }) + save_file_path = os.path.join( + os.path.normpath(save_model_dir), new_var.name + ) + save_block.append_op( + type='save', + inputs={'X': [new_var]}, + outputs={}, + attrs={ + 'file_path': os.path.normpath(save_file_path), + 'save_as_fp16': True, + }, + ) if self._params_filename is not None: save_var_list = [] @@ -1420,33 +1651,44 @@ def convert_weight_to_fp16(self, save_model_dir): saved_params_var = save_block.create_var( type=core.VarDesc.VarType.RAW, - name=unique_name.generate("saved_params")) + name=unique_name.generate("saved_params"), + ) saved_params_var.desc.set_persistable(True) - save_path = os.path.join(os.path.normpath(save_model_dir), - self._params_filename) - save_block.append_op(type='save_combine', - inputs={'X': save_var_list}, - outputs={'Y': saved_params_var}, - attrs={ - 'file_path': save_path, - 'save_as_fp16': True - }) + save_path = os.path.join( + os.path.normpath(save_model_dir), self._params_filename + ) + save_block.append_op( + type='save_combine', + inputs={'X': save_var_list}, + outputs={'Y': saved_params_var}, + attrs={'file_path': save_path, 'save_as_fp16': True}, + ) save_program._sync_with_cpp() exe.run(save_program) # Copy model - model_filename = "__model__" if self._model_filename is None \ - else self._model_filename + model_filename = ( + "__model__" + if self._model_filename is None + else self._model_filename + ) src_model = os.path.join(self._model_dir, model_filename) dest_model = os.path.join(save_model_dir, model_filename) shutil.copyfile(src_model, dest_model) - def _quantize_weight_to_int(self, save_model_dir, save_model_filename, - save_params_filename, quantizable_op_type, - weight_bits, weight_quantize_type, for_test, - threshold_rate): + def _quantize_weight_to_int( + self, + save_model_dir, + save_model_filename, + save_params_filename, + quantizable_op_type, + weight_bits, + weight_quantize_type, + for_test, + threshold_rate, + ): """ Generate quantized model or fake quantized model. """ @@ -1454,11 +1696,12 @@ def _quantize_weight_to_int(self, save_model_dir, save_model_filename, place = core.CPUPlace() exe = Executor(place) scope = global_scope() - [program, feed_list, fetch_list] = \ - io.load_inference_model(dirname=self._model_dir, - executor=exe, - model_filename=self._model_filename, - params_filename=self._params_filename) + [program, feed_list, fetch_list] = io.load_inference_model( + dirname=self._model_dir, + executor=exe, + model_filename=self._model_filename, + params_filename=self._params_filename, + ) quantized_ops = [] for index in range(program.num_blocks): @@ -1474,22 +1717,32 @@ def _quantize_weight_to_int(self, save_model_dir, save_model_filename, if var_name in persistable_var_names: if weight_quantize_type == "abs_max": self._weight_abs_max_quantization( - scope, place, weight_bits, threshold_rate, op, - var_name, for_test) + scope, + place, + weight_bits, + threshold_rate, + op, + var_name, + for_test, + ) elif weight_quantize_type == "channel_wise_abs_max": self._weight_channel_wise_abs_max_quantization( - scope, place, weight_bits, op, var_name, for_test) - - io.save_inference_model(dirname=save_model_dir, - feeded_var_names=feed_list, - target_vars=fetch_list, - executor=exe, - main_program=program, - model_filename=save_model_filename, - params_filename=save_params_filename) - - def _weight_abs_max_quantization(self, scope, place, weight_bits, - threshold_rate, op, var_name, for_test): + scope, place, weight_bits, op, var_name, for_test + ) + + io.save_inference_model( + dirname=save_model_dir, + feeded_var_names=feed_list, + target_vars=fetch_list, + executor=exe, + main_program=program, + model_filename=save_model_filename, + params_filename=save_params_filename, + ) + + def _weight_abs_max_quantization( + self, scope, place, weight_bits, threshold_rate, op, var_name, for_test + ): ''' Use abs_max method to quantize weight. ''' @@ -1501,23 +1754,28 @@ def _weight_abs_max_quantization(self, scope, place, weight_bits, if abs(threshold_rate) < 1e-10: threshold_value = np.max(np.abs(weight_data)) else: - threshold_value = self._calculate_threshold(\ - weight_data, threshold_rate) + threshold_value = self._calculate_threshold( + weight_data, threshold_rate + ) weight_data[weight_data > threshold_value] = threshold_value weight_data[weight_data < -threshold_value] = -threshold_value scale = threshold_value / quantize_range - quantized_weight_data = \ - np.around(weight_data / scale).astype(save_weight_dtype) + quantized_weight_data = np.around(weight_data / scale).astype( + save_weight_dtype + ) # Set weight data if not for_test: - utils.set_variable_data(scope, place, var_name, - quantized_weight_data) + utils.set_variable_data( + scope, place, var_name, quantized_weight_data + ) else: - dequantized_weight_data = \ - (quantized_weight_data * scale).astype(np.float32) - utils.set_variable_data(scope, place, var_name, - dequantized_weight_data) + dequantized_weight_data = (quantized_weight_data * scale).astype( + np.float32 + ) + utils.set_variable_data( + scope, place, var_name, dequantized_weight_data + ) # Save info op._set_attr('quantization_type', 'post_weight_abs_max') @@ -1525,9 +1783,9 @@ def _weight_abs_max_quantization(self, scope, place, weight_bits, op._set_attr(var_name + "_quant_scale", [scale]) # Save as list op._set_attr("with_quant_attr", True) - def _weight_channel_wise_abs_max_quantization(self, scope, place, - weight_bits, op, var_name, - for_test): + def _weight_channel_wise_abs_max_quantization( + self, scope, place, weight_bits, op, var_name, for_test + ): ''' Use channel_wise_abs_max method to quantize weight. ''' @@ -1537,32 +1795,42 @@ def _weight_channel_wise_abs_max_quantization(self, scope, place, # Get quantized scale and weight data weight_data = utils.load_variable_data(scope, var_name) if op.type == "mul": - scales, quantized_weight_data = \ - self._mul_channel_wise_quantization(weight_data, - quantize_range, save_weight_dtype) + scales, quantized_weight_data = self._mul_channel_wise_quantization( + weight_data, quantize_range, save_weight_dtype + ) elif op.type in ["conv2d", "depthwise_conv2d"]: - scales, quantized_weight_data = \ - self._conv_channel_wise_quantization(weight_data, - quantize_range, save_weight_dtype) + ( + scales, + quantized_weight_data, + ) = self._conv_channel_wise_quantization( + weight_data, quantize_range, save_weight_dtype + ) else: _logger.error(op.type + " is not supported by weight quantization") # Set weight data if not for_test: - utils.set_variable_data(scope, place, var_name, - quantized_weight_data) + utils.set_variable_data( + scope, place, var_name, quantized_weight_data + ) else: if op.type == "mul": - dequantized_weight_data = \ - self._mul_channel_wise_dequantization(quantized_weight_data, scales) + dequantized_weight_data = self._mul_channel_wise_dequantization( + quantized_weight_data, scales + ) elif op.type in ["conv2d", "depthwise_conv2d"]: - dequantized_weight_data = \ - self._conv_channel_wise_dequantization(quantized_weight_data, scales) + dequantized_weight_data = ( + self._conv_channel_wise_dequantization( + quantized_weight_data, scales + ) + ) else: - _logger.error(op.type + - " is not supported by weight quantization") - utils.set_variable_data(scope, place, var_name, - dequantized_weight_data) + _logger.error( + op.type + " is not supported by weight quantization" + ) + utils.set_variable_data( + scope, place, var_name, dequantized_weight_data + ) # Save info op._set_attr('quantization_type', 'post_weight_channel_wise_abs_max') @@ -1570,67 +1838,77 @@ def _weight_channel_wise_abs_max_quantization(self, scope, place, op._set_attr(var_name + "_quant_scale", scales) op._set_attr("with_quant_attr", True) - def _conv_channel_wise_quantization(self, weight_data, quantize_range, - save_weight_dtype): + def _conv_channel_wise_quantization( + self, weight_data, quantize_range, save_weight_dtype + ): ''' Get channel wise scale for the weights of conv2d and depthwise_conv2d, and quantize the weights. ''' scales = [] - quantized_weight_data = np.zeros_like(weight_data, - dtype=save_weight_dtype) + quantized_weight_data = np.zeros_like( + weight_data, dtype=save_weight_dtype + ) channel_num = weight_data.shape[0] for i in range(channel_num): scale = np.max(np.abs(weight_data[i])) / quantize_range scales.append(scale) - quantized_weight_data[i] = \ - np.around(weight_data[i] / scale).astype(save_weight_dtype) + quantized_weight_data[i] = np.around(weight_data[i] / scale).astype( + save_weight_dtype + ) return scales, quantized_weight_data def _conv_channel_wise_dequantization(self, quantized_weight_data, scales): ''' For conv2d and depthwise_conv2d, dequantize the weights to fp32. ''' - dequantized_weight_data = np.zeros_like(quantized_weight_data, - dtype=np.float32) + dequantized_weight_data = np.zeros_like( + quantized_weight_data, dtype=np.float32 + ) for i in range(len(scales)): - dequantized_weight_data[i] = \ - (quantized_weight_data[i] * scales[i]).astype(np.float32) + dequantized_weight_data[i] = ( + quantized_weight_data[i] * scales[i] + ).astype(np.float32) return dequantized_weight_data - def _mul_channel_wise_quantization(self, weight_data, quantize_range, - save_weight_dtype): + def _mul_channel_wise_quantization( + self, weight_data, quantize_range, save_weight_dtype + ): ''' Get channel wise scale for the weights of conv2d and depthwise_conv2d, and quantize the weights. ''' scales = [] - quantized_weight_data = np.zeros_like(weight_data, - dtype=save_weight_dtype) + quantized_weight_data = np.zeros_like( + weight_data, dtype=save_weight_dtype + ) channel_num = weight_data.shape[-1] for i in range(channel_num): scale = np.max(np.abs(weight_data[:, i])) / quantize_range scales.append(scale) - quantized_weight_data[:, i] = \ - np.around(weight_data[:, i] / scale).astype(save_weight_dtype) + quantized_weight_data[:, i] = np.around( + weight_data[:, i] / scale + ).astype(save_weight_dtype) return scales, quantized_weight_data def _mul_channel_wise_dequantization(self, quantized_weight_data, scales): ''' For mul, dequantize the weights to fp32. ''' - dequantized_weight_data = np.zeros_like(quantized_weight_data, - dtype=np.float32) + dequantized_weight_data = np.zeros_like( + quantized_weight_data, dtype=np.float32 + ) for i in range(len(scales)): - dequantized_weight_data[:, i] = \ - (quantized_weight_data[:, i] * scales[i]).astype(np.float32) + dequantized_weight_data[:, i] = ( + quantized_weight_data[:, i] * scales[i] + ).astype(np.float32) return dequantized_weight_data def _calculate_threshold(self, input, threshold_rate, histogram_bins=5000): input_abs = np.abs(input) - hist, hist_edeges = np.histogram(input_abs, - bins=histogram_bins, - range=(0, np.max(input_abs))) + hist, hist_edeges = np.histogram( + input_abs, bins=histogram_bins, range=(0, np.max(input_abs)) + ) hist = hist / float(sum(hist)) hist_sum = 0 hist_index = 0 diff --git a/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py b/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py index f8de55ee3caea..c7723097f4deb 100644 --- a/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py +++ b/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py @@ -36,13 +36,15 @@ class Quant2Int8MkldnnPass(object): passes (`cpu_quantize_pass`, `cpu_quantize_squash_pass`). """ - def __init__(self, - _ops_to_quantize, - _op_ids_to_skip=None, - _scope=None, - _place=None, - _core=None, - _debug=False): + def __init__( + self, + _ops_to_quantize, + _op_ids_to_skip=None, + _scope=None, + _place=None, + _core=None, + _debug=False, + ): self._scope = _scope self._place = _get_paddle_place(_place) self._core = _core @@ -52,19 +54,26 @@ def __init__(self, 'fake_quantize_range_abs_max', ] self._fake_dequantize_types = [ - 'fake_dequantize_max_abs', 'fake_channel_wise_dequantize_max_abs' + 'fake_dequantize_max_abs', + 'fake_channel_wise_dequantize_max_abs', ] self._fake_quantize_dequantize_types = [ 'fake_quantize_dequantize_abs_max', 'fake_quantize_dequantize_moving_average_abs_max', - 'fake_channel_wise_quantize_dequantize_abs_max' + 'fake_channel_wise_quantize_dequantize_abs_max', ] self._ops_to_quantize = _ops_to_quantize - self._op_ids_to_skip = _op_ids_to_skip if _op_ids_to_skip is not None else set( - [-1]) + self._op_ids_to_skip = ( + _op_ids_to_skip if _op_ids_to_skip is not None else set([-1]) + ) self._scale_immutable_ops = [ - 'transpose2', 'reshape2', 'pool2d', 'slice', 'shape', - 'nearest_interp', 'nearest_interp_v2' + 'transpose2', + 'reshape2', + 'pool2d', + 'slice', + 'shape', + 'nearest_interp', + 'nearest_interp_v2', ] self._scale_ops = ['scale'] self._conv_ops = ['conv2d', 'depthwise_conv2d'] @@ -84,8 +93,9 @@ def __init__(self, self._pass_group = 'int8' def apply(self, graph): - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' self._reset_pass_idx_and_group('int8') graph = self._label_skip_quantized_op(graph) @@ -105,8 +115,9 @@ def apply(self, graph): return graph def prepare_and_optimize_fp32(self, graph): - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' self._reset_pass_idx_and_group('fp32') graph = self._optimize_fp32_graph(graph) @@ -130,10 +141,10 @@ def _is_any_of_op_types_in_graph(self, op_types, graph): return any(op.name() in op_types for op in graph.all_op_nodes()) def _is_any_of_op_types_quantized(self, op_types, graph): - return self._is_any_of_op_types_in_graph( - op_types, graph) and (self._is_quantizing_all_ops() - or any(op_type in self._ops_to_quantize - for op_type in op_types)) + return self._is_any_of_op_types_in_graph(op_types, graph) and ( + self._is_quantizing_all_ops() + or any(op_type in self._ops_to_quantize for op_type in op_types) + ) def _is_conv_quantized(self, graph): return self._is_any_of_op_types_quantized(self._conv_ops, graph) @@ -153,8 +164,9 @@ def _label_skip_quantized_op(self, graph): """ target_ops = self._conv_ops + self._mul_ops + self._matmul_ops for op_node in graph.all_op_nodes(): - if op_node.name() in target_ops and \ - not op_node.op().has_attr("quantization_type"): + if op_node.name() in target_ops and not op_node.op().has_attr( + "quantization_type" + ): is_quantized_op = True for var_node in op_node.inputs: for front_op_node in var_node.inputs: @@ -181,21 +193,25 @@ def _gather_input_scales_from_fake(self, graph): for op in graph.all_op_nodes(): if op.name() in fake_ops: bit_length = op.op().attr("bit_length") - assert bit_length == 8, 'Unsupported number quantization bits ({}). Only 8 is supported now.'.format( - bit_length) + assert ( + bit_length == 8 + ), 'Unsupported number quantization bits ({}). Only 8 is supported now.'.format( + bit_length + ) input_name = op.input("X")[0] scale_name = op.input("InScale")[0] output_name = op.output("Out")[0] # Gather new weight scales after folding batchnorm in convolution scale = np.array( - 1.0 / self._load_param(self._scope, scale_name)[0]).astype( - np.float64) + 1.0 / self._load_param(self._scope, scale_name)[0] + ).astype(np.float64) scale[scale == np.Inf] = 0.0 lod_tensor = self._convert_scale2tensor(scale) use_unsigned_int = False - self._add_scale_for_vars([input_name, output_name], - use_unsigned_int, lod_tensor) + self._add_scale_for_vars( + [input_name, output_name], use_unsigned_int, lod_tensor + ) return graph @@ -205,15 +221,16 @@ def _gather_weight_thresholds_from_fake(self, graph): input_name = op.input("X")[0] if op.op().has_attr("max_range"): _max_range = np.array(op.op().attr("max_range")).astype( - np.float64) + np.float64 + ) self._weight_thresholds[input_name] = np.array( - self._s8_max * self._s8_max / _max_range).astype( - np.float64) + self._s8_max * self._s8_max / _max_range + ).astype(np.float64) else: scale_name = op.input("Scales")[0] self._weight_thresholds[input_name] = np.array( - self._load_param(self._scope, - scale_name)).astype(np.float64) + self._load_param(self._scope, scale_name) + ).astype(np.float64) return graph @@ -229,14 +246,13 @@ def _gather_output_scales_from_attr(self, graph): use_unsigned_int = False for output_name in op.op().outputs(): for out_var_name in op.op().output(output_name): - self._add_scale_for_vars([out_var_name], - use_unsigned_int, - scale_lod_tensor) + self._add_scale_for_vars( + [out_var_name], use_unsigned_int, scale_lod_tensor + ) return graph def _propagate_scales(self, graph): - def _update_scale_op_in_scale(op, input, output): unsigned, tensor = self._var_quant_scales[output] scale = np.array(tensor) * op.op().attr("scale") @@ -254,24 +270,28 @@ def _update_scales(graph): output_name = op.output("Out")[0] tensor_names = [input_name, output_name] - if all(name not in self._var_quant_scales - for name in tensor_names): + if all( + name not in self._var_quant_scales + for name in tensor_names + ): waiting_for_scale.update(tensor_names) continue elif input_name in self._var_quant_scales: self._var_quant_scales[ - output_name] = self._var_quant_scales[input_name] + output_name + ] = self._var_quant_scales[input_name] elif output_name in self._var_quant_scales: self._var_quant_scales[ - input_name] = self._var_quant_scales[output_name] + input_name + ] = self._var_quant_scales[output_name] elif op.name() == 'concat': output_name = op.output("Out")[0] if output_name in self._var_quant_scales: input_names = op.input("X") for input_name in input_names: self._var_quant_scales[ - input_name] = self._var_quant_scales[ - output_name] + input_name + ] = self._var_quant_scales[output_name] elif op.name() in self._scale_ops: input_name = op.input("X")[0] output_name = op.output("Out")[0] @@ -282,8 +302,10 @@ def _update_scales(graph): waiting_for_scale = _update_scales(graph) waiting_for_scale_prev = set() - while len(waiting_for_scale - ) != 0 and waiting_for_scale != waiting_for_scale_prev: + while ( + len(waiting_for_scale) != 0 + and waiting_for_scale != waiting_for_scale_prev + ): waiting_for_scale_prev = waiting_for_scale waiting_for_scale = _update_scales(graph) @@ -305,27 +327,31 @@ def _remove_fake_ops(self, graph): def _remove_fake_quantize(self, graph, op): fake_quant_in = graph._find_node_by_name(op.inputs, op.input("X")[0]) - fake_quant_in_scale = graph._find_node_by_name(op.inputs, - op.input("InScale")[0]) - fake_quant_out = graph._find_node_by_name(op.outputs, - op.output("Out")[0]) + fake_quant_in_scale = graph._find_node_by_name( + op.inputs, op.input("InScale")[0] + ) + fake_quant_out = graph._find_node_by_name( + op.outputs, op.output("Out")[0] + ) fake_quant_out_scale = graph._find_node_by_name( - op.outputs, - op.output("OutScale")[0]) + op.outputs, op.output("OutScale")[0] + ) next_ops = fake_quant_out.outputs for next_op in next_ops: self._swap_inputs(next_op, fake_quant_out, fake_quant_in) graph.link_to(fake_quant_in, next_op) graph.safe_remove_nodes( - {op, fake_quant_in_scale, fake_quant_out, fake_quant_out_scale}) + {op, fake_quant_in_scale, fake_quant_out, fake_quant_out_scale} + ) return graph def _remove_fake_dequantize(self, graph, op): fake_dequant_in = graph._find_node_by_name(op.inputs, op.input("X")[0]) - fake_dequant_out = graph._find_node_by_name(op.outputs, - op.output("Out")[0]) + fake_dequant_out = graph._find_node_by_name( + op.outputs, op.output("Out")[0] + ) next_ops = fake_dequant_out.outputs for next_op in next_ops: @@ -338,13 +364,15 @@ def _remove_fake_dequantize(self, graph, op): def _swap_inputs(self, op, old_input, new_input): for input_name in op.op().input_names(): if old_input.name() in op.input(input_name): - op.op().set_input(input_name, [ - new_input.name() if x == old_input.name() else x - for x in op.input(input_name) - ]) + op.op().set_input( + input_name, + [ + new_input.name() if x == old_input.name() else x + for x in op.input(input_name) + ], + ) def _dequantize_weights(self, graph): - def _is_int8_weights(op_node, weight_name): weight_var_name = op_node.input(weight_name)[0] if self._scope.find_var(weight_var_name) is None: @@ -373,8 +401,10 @@ def _dequantize_op_weights(self, graph, op_node, weight_name, output_name): w_fp32 = np.multiply(np.divide(weight, self._s8_max), scales) else: raise ValueError( - "The size of weight scales vector ({}) does not match the dimensions ({}) of the weights tensor {}." - .format(scales.size, weight.shape, weight_var_name)) + "The size of weight scales vector ({}) does not match the dimensions ({}) of the weights tensor {}.".format( + scales.size, weight.shape, weight_var_name + ) + ) w_fp32 = w_fp32.reshape(weight.shape).astype(np.float32) self._restore_var(weight_var_name, w_fp32) @@ -384,8 +414,9 @@ def _restore_var(self, name, array): def _update_activations(self, graph): for op in graph.all_op_nodes(): - if op.name( - ) in self._conv_ops and not op.op().has_attr("fuse_activation"): + if op.name() in self._conv_ops and not op.op().has_attr( + "fuse_activation" + ): activation = "" if op.op().has_attr("fuse_relu") and op.op().attr("fuse_relu"): activation = "relu" @@ -403,8 +434,9 @@ def _remove_ctrl_vars(self, graph): def _optimize_fp32_graph(self, graph): graph = self._update_activations(graph) graph = self._remove_ctrl_vars(graph) - graph = self._apply_pass(graph, 'mkldnn_placement_pass', - ['mkldnn_enabled_op_types'], [set()]) + graph = self._apply_pass( + graph, 'mkldnn_placement_pass', ['mkldnn_enabled_op_types'], [set()] + ) # remove dropout ops graph = self._apply_pass(graph, 'simplify_with_basic_ops_pass') graph = self._apply_pass(graph, 'layer_norm_fuse_pass') @@ -433,29 +465,34 @@ def _optimize_fp32_graph(self, graph): graph = self._apply_pass(graph, 'conv_eltwiseadd_bn_fuse_pass') graph = self._apply_pass(graph, 'conv_affine_channel_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'conv_transpose_bn_fuse_pass') - graph = self._apply_pass(graph, - 'conv_transpose_eltwiseadd_bn_fuse_pass') + graph = self._apply_pass( + graph, 'conv_transpose_eltwiseadd_bn_fuse_pass' + ) graph = self._apply_pass(graph, 'conv_bias_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'conv_transpose_bias_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'conv_elementwise_add_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'conv_activation_mkldnn_fuse_pass') - graph = self._apply_pass(graph, 'fc_fuse_pass', - ['use_gpu', 'use_fc_padding'], [False, False]) + graph = self._apply_pass( + graph, 'fc_fuse_pass', ['use_gpu', 'use_fc_padding'], [False, False] + ) graph = self._apply_pass(graph, 'repeated_fc_relu_fuse_pass') if self._is_fc_quantized(graph): # Disabled due to topology-dependent speed-up graph = self._apply_pass(graph, 'fc_mkldnn_pass') graph = self._apply_pass(graph, 'fc_act_mkldnn_fuse_pass') - graph = self._apply_pass(graph, - 'matmul_transpose_reshape_mkldnn_fuse_pass') - graph = self._apply_pass(graph, - 'matmul_elementwise_add_mkldnn_fuse_pass') + graph = self._apply_pass( + graph, 'matmul_transpose_reshape_mkldnn_fuse_pass' + ) + graph = self._apply_pass( + graph, 'matmul_elementwise_add_mkldnn_fuse_pass' + ) graph = self._apply_pass(graph, 'matmul_activation_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'batch_norm_act_fuse_pass') graph = self._apply_pass(graph, 'softplus_activation_mkldnn_fuse_pass') graph = self._apply_pass(graph, 'scale_matmul_fuse_pass') - graph = self._apply_pass(graph, - 'reshape_transpose_matmul_mkldnn_fuse_pass') + graph = self._apply_pass( + graph, 'reshape_transpose_matmul_mkldnn_fuse_pass' + ) # the following pass should be the last one since it will work on all fused ops. graph = self._apply_pass(graph, 'runtime_context_cache_pass') return graph @@ -474,8 +511,10 @@ def _apply_pass(self, graph, pass_name, attrs=None, attr_values=None): ir_pass.apply(cpp_graph) if self._debug: graph.draw( - '.', '{}_{}_{}'.format(self._pass_group, self._pass_idx, - pass_name), graph.all_op_nodes()) + '.', + '{}_{}_{}'.format(self._pass_group, self._pass_idx, pass_name), + graph.all_op_nodes(), + ) self._remove_unused_var_nodes(graph) self._pass_idx += 1 return graph @@ -502,8 +541,10 @@ def _remove_unused_var_nodes(self, graph): all_used_vars = {n.node for n in all_used_vars} all_unused_vars = { n - for n in filter(lambda node: node.node not in all_used_vars, - graph.all_var_nodes()) + for n in filter( + lambda node: node.node not in all_used_vars, + graph.all_var_nodes(), + ) } graph.safe_remove_nodes(all_unused_vars) return graph @@ -515,45 +556,60 @@ def _set_op_role_forward(self, graph): return graph def _compute_weight_scales(self, graph): - def _compute_var_scales(ops, w_name, axis): for op in graph.all_op_nodes(): if op.op().type() in ops: weight_var_name = op.input(w_name)[0] weights = np.array( - self._load_param(self._scope, weight_var_name)) - scales = 1.0 / np.amax(np.abs( - weights.reshape(weights.shape[0], -1)).astype( - np.float64), - axis=axis) + self._load_param(self._scope, weight_var_name) + ) + scales = 1.0 / np.amax( + np.abs(weights.reshape(weights.shape[0], -1)).astype( + np.float64 + ), + axis=axis, + ) scales[scales == np.Inf] = 0.0 lod_tensor = self._convert_scale2tensor(scales) use_unsigned_int = False - self._var_quant_scales[weight_var_name] = (use_unsigned_int, - lod_tensor) + self._var_quant_scales[weight_var_name] = ( + use_unsigned_int, + lod_tensor, + ) def _compute_single_gru_weight_scales(wx_var_name, wh_var_name): wx = np.array(self._load_param(self._scope, wx_var_name)) wh = np.array(self._load_param(self._scope, wh_var_name)) OC = wh.shape[0] - scale_ur = 1.0 / np.max(np.abs( - np.concatenate([ - wx[:, :2 * OC], - wh.flatten()[:2 * OC * OC].reshape(OC, 2 * OC) - ], - axis=0)), - axis=0) - scale_o = 1.0 / np.max(np.abs( - np.concatenate([ - wx[:, 2 * OC:], - wh.flatten()[2 * OC * OC:].reshape(OC, OC) - ], - axis=0)), - axis=0) - - gru_weights_scale = np.concatenate([scale_ur, - scale_o]).astype('float') + scale_ur = 1.0 / np.max( + np.abs( + np.concatenate( + [ + wx[:, : 2 * OC], + wh.flatten()[: 2 * OC * OC].reshape(OC, 2 * OC), + ], + axis=0, + ) + ), + axis=0, + ) + scale_o = 1.0 / np.max( + np.abs( + np.concatenate( + [ + wx[:, 2 * OC :], + wh.flatten()[2 * OC * OC :].reshape(OC, OC), + ], + axis=0, + ) + ), + axis=0, + ) + + gru_weights_scale = np.concatenate([scale_ur, scale_o]).astype( + 'float' + ) return self._convert_scale2tensor(gru_weights_scale) @@ -563,21 +619,26 @@ def _compute_gru_weight_scales(wx_name, wh_name): assert len(op.input(wx_name)) == len( op.input(wh_name) ), 'Mismatch in number of weights inputs ({} for WeightX vs. {} for WeightH).'.format( - len(op.input(wx_name)), len(op.input(wh_name))) + len(op.input(wx_name)), len(op.input(wh_name)) + ) for i, wx_var_name in enumerate(op.input(wx_name)): wh_var_name = op.input(wh_name)[i] use_unsigned_int = False lod_tensor = _compute_single_gru_weight_scales( - wx_var_name, wh_var_name) - self._var_quant_scales[wx_var_name] = (use_unsigned_int, - lod_tensor) + wx_var_name, wh_var_name + ) + self._var_quant_scales[wx_var_name] = ( + use_unsigned_int, + lod_tensor, + ) def _compute_single_lstm_weight_scales(wx_var_name, wh_var_name): wx = np.array(self._load_param(self._scope, wx_var_name)) wh = np.array(self._load_param(self._scope, wh_var_name)) lstm_weights_scale = 1.0 / np.max( - np.abs(np.concatenate([wx[:, :], wh[:, :]], axis=0)), axis=0) + np.abs(np.concatenate([wx[:, :], wh[:, :]], axis=0)), axis=0 + ) lstm_weights_scale = lstm_weights_scale.astype('float') return self._convert_scale2tensor(lstm_weights_scale) @@ -588,14 +649,18 @@ def _compute_lstm_weight_scales(wx_name, wh_name): assert len(op.input(wx_name)) == len( op.input(wh_name) ), 'Mismatch in number of weights inputs ({} for WeightX vs. {} for WeightH).'.format( - len(op.input(wx_name)), len(op.input(wh_name))) + len(op.input(wx_name)), len(op.input(wh_name)) + ) for i, wx_var_name in enumerate(op.input(wx_name)): wh_var_name = op.input(wh_name)[i] use_unsigned_int = False lod_tensor = _compute_single_lstm_weight_scales( - wx_var_name, wh_var_name) - self._var_quant_scales[wx_var_name] = (use_unsigned_int, - lod_tensor) + wx_var_name, wh_var_name + ) + self._var_quant_scales[wx_var_name] = ( + use_unsigned_int, + lod_tensor, + ) _compute_var_scales(self._conv_ops, "Filter", axis=1) _compute_var_scales(self._fc_ops, "W", axis=0) @@ -606,7 +671,6 @@ def _compute_lstm_weight_scales(wx_name, wh_name): return graph def _update_relu_output_scales(self, graph): - def _set_unsigned_scale(graph, ops, op_out_name, predicate): ''' Sets the type of an output scale of a passed op type(s) to 'unsigned int8' if the @@ -617,7 +681,8 @@ def _set_unsigned_scale(graph, ops, op_out_name, predicate): if op.name() in ops: out_name = op.output(op_out_name)[0] if out_name in self._var_quant_scales and predicate( - op.op()): + op.op() + ): is_unsigned, tensor = self._var_quant_scales[out_name] if is_unsigned is False: # If the variable is signed, it means that the scales for this var @@ -625,23 +690,26 @@ def _set_unsigned_scale(graph, ops, op_out_name, predicate): # to fill the entire range of uint8 scale = np.array(tensor) * 2 tensor = self._convert_scale2tensor( - scale.astype(np.float64)) + scale.astype(np.float64) + ) self._var_quant_scales[out_name] = (True, tensor) return graph def conv_predicate(op): return op.attr("fuse_activation") in self._relu_ops - graph = _set_unsigned_scale(graph, self._conv_ops, "Output", - conv_predicate) + graph = _set_unsigned_scale( + graph, self._conv_ops, "Output", conv_predicate + ) def fc_predicate(op): return op.attr("activation_type") in self._relu_ops graph = _set_unsigned_scale(graph, self._fc_ops, "Out", fc_predicate) - graph = _set_unsigned_scale(graph, self._relu_ops, 'Out', - lambda op: True) + graph = _set_unsigned_scale( + graph, self._relu_ops, 'Out', lambda op: True + ) return graph @@ -650,15 +718,21 @@ def _get_data_layout(self, graph): def _quantize_fp32_graph(self, graph): graph = self._apply_pass(graph, 'scale_matmul_fuse_pass') - graph = self._apply_pass(graph, - 'reshape_transpose_matmul_mkldnn_fuse_pass') - graph = self._apply_pass(graph, 'cpu_quantize_placement_pass', - ['quantize_enabled_op_types'], - [self._ops_to_quantize]) graph = self._apply_pass( - graph, 'cpu_quantize_pass', ['quant_var_scales', 'data_layout'], - [self._var_quant_scales, - self._get_data_layout(graph)]) + graph, 'reshape_transpose_matmul_mkldnn_fuse_pass' + ) + graph = self._apply_pass( + graph, + 'cpu_quantize_placement_pass', + ['quantize_enabled_op_types'], + [self._ops_to_quantize], + ) + graph = self._apply_pass( + graph, + 'cpu_quantize_pass', + ['quant_var_scales', 'data_layout'], + [self._var_quant_scales, self._get_data_layout(graph)], + ) graph = self._apply_pass(graph, 'cpu_quantize_squash_pass') graph = self._apply_pass(graph, 'int8_scale_calculation_mkldnn_pass') graph = self._apply_pass(graph, 'params_quantization_mkldnn_pass') diff --git a/python/paddle/fluid/contrib/slim/quantization/quant_int8_mkldnn_pass.py b/python/paddle/fluid/contrib/slim/quantization/quant_int8_mkldnn_pass.py index d56aeb79f3f7c..73c611db01206 100644 --- a/python/paddle/fluid/contrib/slim/quantization/quant_int8_mkldnn_pass.py +++ b/python/paddle/fluid/contrib/slim/quantization/quant_int8_mkldnn_pass.py @@ -66,7 +66,7 @@ def __init__(self, _scope=None, _place=None): self._quantize_type = [ 'fake_quantize_moving_average_abs_max', - 'fake_quantize_range_abs_max' + 'fake_quantize_range_abs_max', ] self._dequantize_type = ['fake_dequantize_max_abs'] self._quantize_dequantize_type = [ @@ -92,8 +92,9 @@ def apply(self, graph): graph(IrGraph): the applied graph. """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' ops = graph.all_op_nodes() persistable_vars = [p.name() for p in graph.all_persistable_nodes()] @@ -104,7 +105,8 @@ def apply(self, graph): input_name = op_node.input("X")[0] scale_name = op_node.input("Scale")[0] self._in_scale[input_name] = self._load_param( - self._scope, scale_name)[0] + self._scope, scale_name + )[0] self._max_range[input_name] = op_node.op().attr("max_range") self._new_output[input_name] = op_node.output("Out")[0] @@ -114,7 +116,8 @@ def apply(self, graph): input_name = op_node.input("X")[0] scale_name = op_node.input("InScale")[0] self._in_scale[input_name] = self._load_param( - self._scope, scale_name)[0] + self._scope, scale_name + )[0] # self._max_range[input_name] = op_node.op().attr("max_range") self._new_output[input_name] = op_node.output("Out")[0] @@ -142,29 +145,30 @@ def _transform_to_conv_mkldnn(self, graph, op_node): output_name = op_node.output("Output")[0] # Convert int8 range weights to fp32 range weights weight = self._load_param(self._scope, weight_name) - w_fp32 = np.divide(np.multiply(weight, self._s8_max), - self._max_range[output_name]) + w_fp32 = np.divide( + np.multiply(weight, self._s8_max), self._max_range[output_name] + ) w_fp32 = w_fp32.reshape(weight.shape) self._restore_var(weight_name, w_fp32) - input_var_node = graph._find_node_by_name(op_node.inputs, - op_node.input("Input")[0]) + input_var_node = graph._find_node_by_name( + op_node.inputs, op_node.input("Input")[0] + ) weight_var_node = graph._find_node_by_name(op_node.inputs, weight_name) # Set fake_dequantize_abs_max's output as new output of conv2d output_var_node = graph._find_node_by_name( - graph.all_var_nodes(), self._new_output[output_name]) + graph.all_var_nodes(), self._new_output[output_name] + ) attrs = { - name: op_node.op().attr(name) - for name in op_node.op().attr_names() + name: op_node.op().attr(name) for name in op_node.op().attr_names() } - conv_op_node = graph.create_op_node(op_type='conv2d', - attrs=attrs, - inputs={ - 'Input': input_var_node, - 'Filter': weight_var_node - }, - outputs={'Output': output_var_node}) + conv_op_node = graph.create_op_node( + op_type='conv2d', + attrs=attrs, + inputs={'Input': input_var_node, 'Filter': weight_var_node}, + outputs={'Output': output_var_node}, + ) # Based on the Quant's scales to calculate the scales of MKL-DNN INT8 conv2d scale_in = self._s8_max / self._in_scale[output_name] @@ -187,29 +191,30 @@ def _transform_to_mul_mkldnn(self, graph, op_node): output_name = op_node.output("Out")[0] # Convert int8 range weights to fp32 range weights weight = self._load_param(self._scope, weight_name) - w_fp32 = np.divide(np.multiply(weight, self._s8_max), - self._max_range[output_name]) + w_fp32 = np.divide( + np.multiply(weight, self._s8_max), self._max_range[output_name] + ) w_fp32 = w_fp32.reshape(weight.shape) self._restore_var(weight_name, w_fp32) - input_var_node = graph._find_node_by_name(op_node.inputs, - op_node.input("X")[0]) + input_var_node = graph._find_node_by_name( + op_node.inputs, op_node.input("X")[0] + ) weight_var_node = graph._find_node_by_name(op_node.inputs, weight_name) # Set fake_dequantize_abs_max's output as new output of mul output_var_node = graph._find_node_by_name( - graph.all_var_nodes(), self._new_output[output_name]) + graph.all_var_nodes(), self._new_output[output_name] + ) attrs = { - name: op_node.op().attr(name) - for name in op_node.op().attr_names() + name: op_node.op().attr(name) for name in op_node.op().attr_names() } - mul_op_node = graph.create_op_node(op_type='mul', - attrs=attrs, - inputs={ - 'X': input_var_node, - 'Y': weight_var_node - }, - outputs={'Out': output_var_node}) + mul_op_node = graph.create_op_node( + op_type='mul', + attrs=attrs, + inputs={'X': input_var_node, 'Y': weight_var_node}, + outputs={'Out': output_var_node}, + ) # Based on the Quant's scales to calculate MKL-DNN INT8 mul's scales scale_in = self._s8_max / self._in_scale[output_name] @@ -230,30 +235,35 @@ def _transform_to_quantize_mkldnn(self, graph, op_node): """ Transform fake_quantize_xx op to quantize mkldnn op in the graph. """ - input_var_node = graph._find_node_by_name(op_node.inputs, - op_node.input("X")[0]) - output_var_node = graph._find_node_by_name(op_node.outputs, - op_node.output("Out")[0]) - scale_in = self._s8_max / self._load_param( - self._scope, - op_node.input("InScale")[0])[0] + input_var_node = graph._find_node_by_name( + op_node.inputs, op_node.input("X")[0] + ) + output_var_node = graph._find_node_by_name( + op_node.outputs, op_node.output("Out")[0] + ) + scale_in = ( + self._s8_max + / self._load_param(self._scope, op_node.input("InScale")[0])[0] + ) quant_op_node = graph.create_op_node( op_type='quantize', attrs={ 'data_format': 'MKLDNNLAYOUT', 'use_mkldnn': 1, 'Scale': scale_in, - 'is_negative_input': 1 + 'is_negative_input': 1, }, inputs={'Input': input_var_node}, - outputs={'Output': output_var_node}) + outputs={'Output': output_var_node}, + ) graph.link_to(input_var_node, quant_op_node) graph.link_to(quant_op_node, output_var_node) graph.safe_remove_nodes(op_node) def _remove_fake_dequantize_op(self, graph, op_node): - input_var_node = graph._find_node_by_name(op_node.inputs, - op_node.input("X")[0]) + input_var_node = graph._find_node_by_name( + op_node.inputs, op_node.input("X")[0] + ) graph.safe_remove_nodes(op_node) def _load_param(self, scope, param_name): @@ -275,7 +285,9 @@ def _remove_unused_var_nodes(self, graph): all_used_vars = {n.node for n in all_used_vars} all_unused_vars = { n - for n in filter(lambda node: node.node not in all_used_vars, - graph.all_var_nodes()) + for n in filter( + lambda node: node.node not in all_used_vars, + graph.all_var_nodes(), + ) } graph.safe_remove_nodes(all_unused_vars) diff --git a/python/paddle/fluid/contrib/slim/quantization/quantization_pass.py b/python/paddle/fluid/contrib/slim/quantization/quantization_pass.py index abd6a7822d872..81aac8823d813 100644 --- a/python/paddle/fluid/contrib/slim/quantization/quantization_pass.py +++ b/python/paddle/fluid/contrib/slim/quantization/quantization_pass.py @@ -14,6 +14,7 @@ import collections import numpy as np + try: from tqdm import tqdm except: @@ -47,12 +48,15 @@ ] _fake_quant_op_list = [ - 'fake_quantize_abs_max', 'fake_quantize_range_abs_max', - 'fake_quantize_moving_average_abs_max', 'fake_channel_wise_quantize_abs_max' + 'fake_quantize_abs_max', + 'fake_quantize_range_abs_max', + 'fake_quantize_moving_average_abs_max', + 'fake_channel_wise_quantize_abs_max', ] _fake_dequant_op_list = [ - 'fake_dequantize_max_abs', 'fake_channel_wise_dequantize_max_abs' + 'fake_dequantize_max_abs', + 'fake_channel_wise_dequantize_max_abs', ] _fake_quant_dequant_op_list = [ @@ -67,12 +71,11 @@ def _init_var_node(var_node, value, scope, place): - assert isinstance(value, - np.ndarray), 'The type of value should be numpy array.' - assert scope is not None, \ - 'The scope cannot be set None.' - assert place is not None, \ - 'The place cannot be set None.' + assert isinstance( + value, np.ndarray + ), 'The type of value should be numpy array.' + assert scope is not None, 'The scope cannot be set None.' + assert place is not None, 'The place cannot be set None.' tensor = scope.var(var_node.name()).get_tensor() tensor.set(value, place) @@ -84,8 +87,9 @@ def _is_input_all_not_persistable(graph, op_node): is_input_all_not_persistable = True for var_name in utils._get_op_input_var_names(op_node): in_node = graph._find_node_by_name(op_node.inputs, var_name) - is_input_all_not_persistable = (is_input_all_not_persistable and \ - (not in_node.persistable())) + is_input_all_not_persistable = is_input_all_not_persistable and ( + not in_node.persistable() + ) return is_input_all_not_persistable @@ -109,24 +113,26 @@ class QuantizationTransformPass(object): the quantized ops's inputs. """ - def __init__(self, - scope=None, - place=None, - weight_bits=8, - activation_bits=8, - activation_quantize_type='abs_max', - weight_quantize_type='abs_max', - window_size=10000, - moving_rate=0.9, - skip_pattern=['skip_quant'], - quantizable_op_type=['conv2d', 'depthwise_conv2d', 'mul'], - weight_quantize_func=None, - act_quantize_func=None, - weight_preprocess_func=None, - act_preprocess_func=None, - optimizer_func=None, - executor=None, - is_test=None): + def __init__( + self, + scope=None, + place=None, + weight_bits=8, + activation_bits=8, + activation_quantize_type='abs_max', + weight_quantize_type='abs_max', + window_size=10000, + moving_rate=0.9, + skip_pattern=['skip_quant'], + quantizable_op_type=['conv2d', 'depthwise_conv2d', 'mul'], + weight_quantize_func=None, + act_quantize_func=None, + weight_preprocess_func=None, + act_preprocess_func=None, + optimizer_func=None, + executor=None, + is_test=None, + ): r""" Constructor. @@ -215,21 +221,26 @@ def __init__(self, self._optimizer = optimizer_func self._exe = executor quant_type = [ - 'abs_max', 'channel_wise_abs_max', 'range_abs_max', - 'moving_average_abs_max' + 'abs_max', + 'channel_wise_abs_max', + 'range_abs_max', + 'moving_average_abs_max', ] - assert activation_quantize_type != 'channel_wise_abs_max', \ - "The activation quantization type does not support 'channel_wise_abs_max'." + assert ( + activation_quantize_type != 'channel_wise_abs_max' + ), "The activation quantization type does not support 'channel_wise_abs_max'." if activation_quantize_type not in quant_type: raise ValueError( "Unknown activation_quantize_type : '%s'. It can only be " - "'abs_max' or 'range_abs_max' or 'moving_average_abs_max'." % - (str(activation_quantize_type))) + "'abs_max' or 'range_abs_max' or 'moving_average_abs_max'." + % (str(activation_quantize_type)) + ) if weight_quantize_type not in quant_type: raise ValueError( "Unknown weight_quantize_type: '%s'. It can only be " "'abs_max' or 'channel_wise_abs_max' or 'range_abs_max' " - "or 'moving_average_abs_max'." % (str(weight_quantize_type))) + "or 'moving_average_abs_max'." % (str(weight_quantize_type)) + ) self._activation_quantize_type = activation_quantize_type self._weight_quantize_type = weight_quantize_type @@ -238,8 +249,9 @@ def __init__(self, self._quantizable_ops = quantizable_op_type for op in self._quantizable_ops: - assert op in utils._weight_supported_quantizable_op_type, \ + assert op in utils._weight_supported_quantizable_op_type, ( op + " is not supported for quantization." + ) self._quantizable_grad_ops = [ '%s_grad' % (op) for op in self._quantizable_ops ] @@ -260,8 +272,9 @@ def apply(self, graph): Returns: None """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' if self._is_test is None: self._is_test = graph.is_test() # marked the variable which has been dequantized. @@ -272,13 +285,18 @@ def apply(self, graph): def _quant_preprocess(op_node): user_skipped = False if isinstance(self._skip_pattern, list): - user_skipped = op_node.op().has_attr("op_namescope") and \ - any(pattern in op_node.op().attr("op_namescope") \ - for pattern in self._skip_pattern) + user_skipped = op_node.op().has_attr("op_namescope") and any( + pattern in op_node.op().attr("op_namescope") + for pattern in self._skip_pattern + ) elif isinstance(self._skip_pattern, str): - user_skipped = op_node.op().has_attr("op_namescope") and \ - op_node.op().attr("op_namescope").find( - self._skip_pattern) != -1 + user_skipped = ( + op_node.op().has_attr("op_namescope") + and op_node.op() + .attr("op_namescope") + .find(self._skip_pattern) + != -1 + ) if user_skipped: op_node.op()._set_attr("skip_quant", True) @@ -297,8 +315,9 @@ def _transform_forward(graph, op): name = var_node.name() if name in processed_vars: continue - is_weight = True if var_node.name() in persistable_vars \ - else False + is_weight = ( + True if var_node.name() in persistable_vars else False + ) # if var node is weight and weight_preprocess_func is not None, # will insert weight preprocess func @@ -308,11 +327,14 @@ def _transform_forward(graph, op): # to preorocess activation before quantization if is_weight and self._weight_preprocess_func is not None: var_node = self._insert_func( - graph, self._weight_preprocess_func, var_node, op) - elif not is_weight and self._act_preprocess_func is not None: - var_node = self._insert_func(graph, - self._act_preprocess_func, - var_node, op) + graph, self._weight_preprocess_func, var_node, op + ) + elif ( + not is_weight and self._act_preprocess_func is not None + ): + var_node = self._insert_func( + graph, self._act_preprocess_func, var_node, op + ) # if var node is weight and weight_quantize_func is not None, # will insert weight quantize func to quantize and dequantize weight @@ -320,32 +342,55 @@ def _transform_forward(graph, op): # will insert act quantize func to quantize and dequantize activation if is_weight and self._weight_quantize_func is not None: target_out_node = self._insert_func( - graph, self._weight_quantize_func, var_node, op) + graph, self._weight_quantize_func, var_node, op + ) processed_vars.append(name) continue elif not is_weight and self._act_quantize_func is not None: target_out_node = self._insert_func( - graph, self._act_quantize_func, var_node, op) + graph, self._act_quantize_func, var_node, op + ) processed_vars.append(name) continue - quant_bits = self._weight_bits if var_node.name() in persistable_vars \ + quant_bits = ( + self._weight_bits + if var_node.name() in persistable_vars else self._activation_bits - quant_type = self._weight_quantize_type if is_weight \ + ) + quant_type = ( + self._weight_quantize_type + if is_weight else self._activation_quantize_type - if quant_type == 'channel_wise_abs_max': # Weight quantization - quant_axis = 1 if op.name() in \ - utils._channelwise_quant_axis1_ops else 0 - quant_var_node, scale_var_node = self._insert_channel_quant_op( - graph, var_node, name, quant_bits, quant_axis) + ) + if ( + quant_type == 'channel_wise_abs_max' + ): # Weight quantization + quant_axis = ( + 1 + if op.name() in utils._channelwise_quant_axis1_ops + else 0 + ) + ( + quant_var_node, + scale_var_node, + ) = self._insert_channel_quant_op( + graph, var_node, name, quant_bits, quant_axis + ) dequant_var_node = self._insert_channel_dequant_op( - graph, quant_var_node, [scale_var_node], - [quant_bits], quant_axis) + graph, + quant_var_node, + [scale_var_node], + [quant_bits], + quant_axis, + ) else: quant_var_node, scale_var_node = self._insert_quant_op( - graph, var_node, name, quant_bits, quant_type) + graph, var_node, name, quant_bits, quant_type + ) dequant_var_node = self._insert_dequant_op( - graph, quant_var_node, scale_var_node, quant_bits) + graph, quant_var_node, scale_var_node, quant_bits + ) dequantized_vars[name] = dequant_var_node graph.update_input_link(var_node, dequant_var_node, op) @@ -373,17 +418,20 @@ def _has_weight(op): # Do the preproccess of quantization, such as skipping some ops # for not being quantized. for op in ops: - if op.name() in self._quantizable_ops or \ - op.name() in self._quantizable_grad_ops: + if ( + op.name() in self._quantizable_ops + or op.name() in self._quantizable_grad_ops + ): _quant_preprocess(op) # Insert mapping table to solve the problem in saving inference model. graph.out_node_mapping_table = dict() # The process of _transform_forward and _transform_backward is needed in two for loops. # The loop for transforming the forward graph: - with tqdm(total=len(ops), - bar_format= - 'Adding quant op with weight:|{bar}| {n_fmt}/{total_fmt}', - ncols=80) as t: + with tqdm( + total=len(ops), + bar_format='Adding quant op with weight:|{bar}| {n_fmt}/{total_fmt}', + ncols=80, + ) as t: for op in ops: if op.name() in self._quantizable_ops: if not self._is_skip_quant(graph, op) and _has_weight(op): @@ -397,8 +445,10 @@ def _has_weight(op): return graph def _create_global_step(self, graph): - if self._weight_quantize_type == 'range_abs_max' or \ - self._activation_quantize_type == 'range_abs_max': + if ( + self._weight_quantize_type == 'range_abs_max' + or self._activation_quantize_type == 'range_abs_max' + ): counter_name = '@STEP_COUNTER@' for node in graph.all_var_nodes(): if node.name() == counter_name: @@ -408,21 +458,27 @@ def _create_global_step(self, graph): name=counter_name, var_type=core.VarDesc.VarType.LOD_TENSOR, shape=[1], - var_dtype=core.VarDesc.VarType.INT64) - _init_var_node(global_step_in, np.zeros([1], dtype='int64'), - self._scope, self._place) + var_dtype=core.VarDesc.VarType.INT64, + ) + _init_var_node( + global_step_in, + np.zeros([1], dtype='int64'), + self._scope, + self._place, + ) global_step_out = graph.create_var_node_from_desc( - global_step_in.var()) + global_step_in.var() + ) # The attribute of `op_role` is needed by ParallelExecutor. increment_op = graph.create_op_node( op_type='increment', attrs={ 'step': 1.0, - 'op_role': - core.op_proto_and_checker_maker.OpRole.Forward + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, }, inputs={'X': global_step_in}, - outputs={'Out': global_step_out}) + outputs={'Out': global_step_out}, + ) graph.link_to(global_step_in, increment_op) graph.link_to(increment_op, global_step_out) self._global_step = global_step_out @@ -432,14 +488,17 @@ def _insert_quant_op(self, graph, var_node, name, quant_bits, quant_type): Insert fake_quantize_op in the graph. """ if quant_type == 'abs_max': - return self._insert_quant_abs_max_op(graph, var_node, name, - quant_bits) + return self._insert_quant_abs_max_op( + graph, var_node, name, quant_bits + ) elif quant_type == 'range_abs_max': - return self._insert_quant_range_abs_max_op(graph, var_node, name, - quant_bits) + return self._insert_quant_range_abs_max_op( + graph, var_node, name, quant_bits + ) elif quant_type == 'moving_average_abs_max': return self._insert_quant_moving_average_abs_max_op( - graph, var_node, name, quant_bits) + graph, var_node, name, quant_bits + ) def _insert_quant_abs_max_op(self, graph, var_node, name, quant_bits): """ @@ -451,33 +510,37 @@ def _insert_quant_abs_max_op(self, graph, var_node, name, quant_bits): name=self._quantized_var_name(name), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) scale_name = self._quantized_scale_name(name) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) try: scale_value = np.array( - self._scope.find_var(scale_name).get_tensor()) + self._scope.find_var(scale_name).get_tensor() + ) except: scale_value = np.zeros([1], dtype=data_type) scale_var_node = graph.create_persistable_node( name=scale_name, var_type=var_node.type(), shape=[1], - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) _init_var_node(scale_var_node, scale_value, self._scope, self._place) quant_op_node = graph.create_op_node( op_type='fake_quantize_abs_max', attrs={ 'bit_length': quant_bits, - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, }, inputs={'X': var_node}, - outputs={ - 'Out': quant_var_node, - 'OutScale': scale_var_node - }) + outputs={'Out': quant_var_node, 'OutScale': scale_var_node}, + ) graph.link_to(var_node, quant_op_node) graph.link_to(quant_op_node, quant_var_node) graph.link_to(quant_op_node, scale_var_node) @@ -493,21 +556,27 @@ def _insert_quant_range_abs_max_op(self, graph, var_node, name, quant_bits): name=self._quantized_var_name(name), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) scale_name = self._quantized_scale_name(name) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) try: scale_value = np.array( - self._scope.find_var(scale_name).get_tensor()) + self._scope.find_var(scale_name).get_tensor() + ) except: scale_value = np.array([_SCALE_DEFAULT_VALUE], dtype=data_type) scale_in_node = graph.create_persistable_node( name=scale_name, var_type=core.VarDesc.VarType.LOD_TENSOR, shape=[1], - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) _init_var_node(scale_in_node, scale_value, self._scope, self._place) scale_out_node = graph.create_var_node_from_desc(scale_in_node.var()) @@ -520,12 +589,19 @@ def _insert_quant_range_abs_max_op(self, graph, var_node, name, quant_bits): name=unique_name.generate('scales'), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=[self._window_size], - var_dtype=var_node.dtype()) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' - _init_var_node(scales_node, - np.zeros([self._window_size], dtype=data_type), - self._scope, self._place) + var_dtype=var_node.dtype(), + ) + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) + _init_var_node( + scales_node, + np.zeros([self._window_size], dtype=data_type), + self._scope, + self._place, + ) inputs['Iter'] = self._global_step outputs['OutScales'] = scales_node @@ -533,13 +609,14 @@ def _insert_quant_range_abs_max_op(self, graph, var_node, name, quant_bits): 'window_size': self._window_size, 'bit_length': quant_bits, 'is_test': self._is_test, - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, } quant_op_node = graph.create_op_node( op_type='fake_quantize_range_abs_max', attrs=attrs, inputs=inputs, - outputs=outputs) + outputs=outputs, + ) graph.link_to(var_node, quant_op_node) graph.link_to(scale_in_node, quant_op_node) @@ -552,28 +629,34 @@ def _insert_quant_range_abs_max_op(self, graph, var_node, name, quant_bits): return quant_var_node, scale_out_node - def _insert_quant_moving_average_abs_max_op(self, graph, var_node, name, - quant_bits): - """Insert fake_quantize_moving_average_abs_max - """ + def _insert_quant_moving_average_abs_max_op( + self, graph, var_node, name, quant_bits + ): + """Insert fake_quantize_moving_average_abs_max""" quant_var_node = graph.create_var_node( name=self._quantized_var_name(name), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) scale_name = self._quantized_scale_name(name) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) try: scale_value = np.array( - self._scope.find_var(scale_name).get_tensor()) + self._scope.find_var(scale_name).get_tensor() + ) except: scale_value = np.array([_SCALE_DEFAULT_VALUE], dtype=data_type) scale_in_node = graph.create_persistable_node( name=scale_name, var_type=core.VarDesc.VarType.LOD_TENSOR, shape=[1], - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) _init_var_node(scale_in_node, scale_value, self._scope, self._place) scale_out_node = graph.create_var_node_from_desc(scale_in_node.var()) @@ -584,22 +667,37 @@ def _insert_quant_moving_average_abs_max_op(self, graph, var_node, name, name=unique_name.generate('state'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=var_node.dtype(), - shape=[1]) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' - _init_var_node(state_in_node, np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) + _init_var_node( + state_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) accum_in_node = graph.create_persistable_node( name=unique_name.generate('accum'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=var_node.dtype(), - shape=[1]) - _init_var_node(accum_in_node, np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + _init_var_node( + accum_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) state_out_node = graph.create_var_node_from_desc( - state_in_node.var()) + state_in_node.var() + ) accum_out_node = graph.create_var_node_from_desc( - accum_in_node.var()) + accum_in_node.var() + ) ins['InState'] = state_in_node ins['InAccum'] = accum_in_node @@ -610,14 +708,15 @@ def _insert_quant_moving_average_abs_max_op(self, graph, var_node, name, 'bit_length': quant_bits, 'moving_rate': self._moving_rate, 'is_test': self._is_test, - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, } quant_op_node = graph.create_op_node( op_type='fake_quantize_moving_average_abs_max', attrs=attrs, inputs=ins, - outputs=outs) + outputs=outs, + ) graph.link_to(var_node, quant_op_node) graph.link_to(scale_in_node, quant_op_node) @@ -632,8 +731,9 @@ def _insert_quant_moving_average_abs_max_op(self, graph, var_node, name, return quant_var_node, scale_out_node - def _insert_channel_quant_op(self, graph, var_node, name, quant_bits, - quant_axis): + def _insert_channel_quant_op( + self, graph, var_node, name, quant_bits, quant_axis + ): """ Insert fake_channel_wise_quantize_abs_max op in the graph. """ @@ -643,21 +743,28 @@ def _insert_channel_quant_op(self, graph, var_node, name, quant_bits, name=self._quantized_var_name(name), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) scale_name = self._quantized_scale_name(name) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) try: scale_value = np.array( - self._scope.find_var(scale_name).get_tensor()) + self._scope.find_var(scale_name).get_tensor() + ) except: - scale_value = np.zeros([var_node.shape()[quant_axis]], - dtype=data_type) + scale_value = np.zeros( + [var_node.shape()[quant_axis]], dtype=data_type + ) scale_var_node = graph.create_persistable_node( name=self._quantized_scale_name(name), var_type=var_node.type(), shape=[var_node.shape()[quant_axis]], - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) _init_var_node(scale_var_node, scale_value, self._scope, self._place) quant_op_node = graph.create_op_node( op_type='fake_channel_wise_quantize_abs_max', @@ -665,13 +772,11 @@ def _insert_channel_quant_op(self, graph, var_node, name, quant_bits, 'bit_length': quant_bits, 'quant_axis': quant_axis, 'is_test': self._is_test, - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, }, inputs={'X': var_node}, - outputs={ - 'Out': quant_var_node, - 'OutScale': scale_var_node - }) + outputs={'Out': quant_var_node, 'OutScale': scale_var_node}, + ) graph.link_to(var_node, quant_op_node) graph.link_to(quant_op_node, quant_var_node) graph.link_to(quant_op_node, scale_var_node) @@ -687,26 +792,26 @@ def _insert_dequant_op(self, graph, var_node, scale_var_node, quant_bits): name=self._dequantized_var_name(var_node.name()), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) max_range = (1 << (quant_bits - 1)) - 1 dequant_op_node = graph.create_op_node( op_type='fake_dequantize_max_abs', attrs={ 'max_range': float(max_range), - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward - }, - inputs={ - 'X': var_node, - 'Scale': scale_var_node + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, }, - outputs={'Out': dequant_var_node}) + inputs={'X': var_node, 'Scale': scale_var_node}, + outputs={'Out': dequant_var_node}, + ) graph.link_to(var_node, dequant_op_node) graph.link_to(scale_var_node, dequant_op_node) graph.link_to(dequant_op_node, dequant_var_node) return dequant_var_node - def _insert_channel_dequant_op(self, graph, var_node, scale_var_nodes, - quant_bits, quant_axis): + def _insert_channel_dequant_op( + self, graph, var_node, scale_var_nodes, quant_bits, quant_axis + ): """ Insert fake_channel_wise_dequantize_max_abs in the graph. """ @@ -716,19 +821,18 @@ def _insert_channel_dequant_op(self, graph, var_node, scale_var_nodes, name=self._dequantized_var_name(var_node.name()), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) dequant_op_node = graph.create_op_node( op_type='fake_channel_wise_dequantize_max_abs', attrs={ 'quant_bits': quant_bits, 'quant_axis': quant_axis, - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward - }, - inputs={ - 'X': var_node, - 'Scales': scale_var_nodes + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, }, - outputs={'Out': dequant_var_node}) + inputs={'X': var_node, 'Scales': scale_var_nodes}, + outputs={'Out': dequant_var_node}, + ) graph.link_to(var_node, dequant_op_node) for scale_n in scale_var_nodes: graph.link_to(scale_n, dequant_op_node) @@ -815,27 +919,34 @@ def _insert_func(self, graph, func, var_node, op): startup_program = Program() with program_guard(tmp_program, startup_program): with unique_name.guard(var_node.name() + "_"): - in_node = data(var_node.name() + '_tmp_input', - shape=var_node.shape(), - dtype='float32') + in_node = data( + var_node.name() + '_tmp_input', + shape=var_node.shape(), + dtype='float32', + ) out_node = func(in_node) graph.out_node_mapping_table[out_node.name] = var_node.name() # loss shape must be 1 when minimize loss = mean(out_node) if not graph._for_test: - assert self._optimizer, "optimizer_func must be set when graph is test graph" + assert ( + self._optimizer + ), "optimizer_func must be set when graph is test graph" in_node.stop_gradient = False optimizer = self._optimizer() optimizer.minimize(loss) with scope_guard(self._scope): self._exe.run(startup_program) - tmp_graph = IrGraph(core.Graph(tmp_program.desc), - for_test=graph._for_test) - in_node = tmp_graph._find_node_by_name(tmp_graph.all_var_nodes(), - in_node.name) - out_node = tmp_graph._find_node_by_name(tmp_graph.all_var_nodes(), - out_node.name) + tmp_graph = IrGraph( + core.Graph(tmp_program.desc), for_test=graph._for_test + ) + in_node = tmp_graph._find_node_by_name( + tmp_graph.all_var_nodes(), in_node.name + ) + out_node = tmp_graph._find_node_by_name( + tmp_graph.all_var_nodes(), out_node.name + ) in_node_params = [] in_op_node = [] @@ -854,10 +965,12 @@ def _insert_func(self, graph, func, var_node, op): for node in in_op_node: self._copy_graph(graph, tmp_graph, node) - target_in_node = graph._find_node_by_name(graph.all_var_nodes(), - in_node.name()) - target_out_node = graph._find_node_by_name(graph.all_var_nodes(), - out_node.name()) + target_in_node = graph._find_node_by_name( + graph.all_var_nodes(), in_node.name() + ) + target_out_node = graph._find_node_by_name( + graph.all_var_nodes(), out_node.name() + ) loss_node = graph._find_node_by_name(graph.all_var_nodes(), loss.name) outputs = target_in_node.outputs for node in outputs: @@ -867,16 +980,17 @@ def _insert_func(self, graph, func, var_node, op): # update grad if not graph._for_test: op_out = op.outputs[0] - op_out_grad = graph._find_node_by_name(graph.all_var_nodes(), - op_out.name() + "@GRAD") + op_out_grad = graph._find_node_by_name( + graph.all_var_nodes(), op_out.name() + "@GRAD" + ) # find op's gradient op, such as conv2d_grad op_grad = op_out_grad.outputs[0] target_out_grad_node = graph._find_node_by_name( - graph.all_var_nodes(), - target_out_node.name() + "@GRAD") + graph.all_var_nodes(), target_out_node.name() + "@GRAD" + ) in_node_grad = graph._find_node_by_name( - graph.all_var_nodes(), - target_in_node.name() + "@GRAD") + graph.all_var_nodes(), target_in_node.name() + "@GRAD" + ) in_node_grad_op = in_node_grad.inputs # update op_grad's input graph.update_input_link(var_node, target_out_node, op_grad) @@ -888,8 +1002,9 @@ def _insert_func(self, graph, func, var_node, op): op_grad_out = node # update op_grad's output if op_grad_out is not None: - graph.update_output_link(op_grad_out, target_out_grad_node, - op_grad) + graph.update_output_link( + op_grad_out, target_out_grad_node, op_grad + ) else: graph.link_to(op_grad, target_out_grad_node) @@ -934,31 +1049,37 @@ def _is_skip_quant(self, graph, op_node): Analyse whether the op node skips quantization. """ is_skip = False - if op_node.op().has_attr("skip_quant") and \ - op_node.op().attr("skip_quant"): + if op_node.op().has_attr("skip_quant") and op_node.op().attr( + "skip_quant" + ): is_skip = True # if the inputs of mul and matmul are not all persistable, use # AddQuantDequantPass to quantize them. - if op_node.name() in ["mul", "matmul"] and \ - _is_input_all_not_persistable(graph, op_node): + if op_node.name() in [ + "mul", + "matmul", + ] and _is_input_all_not_persistable(graph, op_node): is_skip = True - if op_node.op().has_attr("quantization_type") and \ - op_node.op().attr("quantization_type") == "qat_without_weight": + if ( + op_node.op().has_attr("quantization_type") + and op_node.op().attr("quantization_type") == "qat_without_weight" + ): is_skip = True return is_skip class QuantizationFreezePass(object): - - def __init__(self, - scope, - place, - bias_correction=False, - weight_bits=8, - activation_bits=8, - round_type='round', - weight_quantize_type='abs_max', - quantizable_op_type=None): + def __init__( + self, + scope, + place, + bias_correction=False, + weight_bits=8, + activation_bits=8, + round_type='round', + weight_quantize_type='abs_max', + quantizable_op_type=None, + ): """ The freeze pass is used to adjust the quantize operator order, for example: 1) `activation -> quant -> dequant -> conv2d` will be frozen into @@ -984,10 +1105,8 @@ def __init__(self, quantizable_op_type(list[str]): This input param will be removed latter. The pass will process all quantized op, so it is not necessary to set the input param. """ - assert scope is not None, \ - 'The scope cannot be set None.' - assert place is not None, \ - 'The place cannot be set None.' + assert scope is not None, 'The scope cannot be set None.' + assert place is not None, 'The place cannot be set None.' self._scope = scope self._bias_correction = bias_correction self._place = _get_paddle_place(place) @@ -1020,21 +1139,26 @@ def apply(self, graph): if hasattr(graph, 'out_node_mapping_table'): if input_arg_name in graph.out_node_mapping_table.keys(): input_arg_name = graph.out_node_mapping_table[ - input_arg_name] + input_arg_name + ] if input_arg_name not in persistable_vars: scale_v = graph._find_node_by_name( - op_node.outputs, - op_node.output('OutScale')[0]) + op_node.outputs, op_node.output('OutScale')[0] + ) self._quant_var_scale_map[input_arg_name] = scale_v else: # Obtain scale from OutScale var node scale_v = self._load_var(op_node.output('OutScale')[0]) assert scale_v.ndim in [ - 1, 2 + 1, + 2, ], "the dim of scale_v should be 1 or 2" if scale_v.ndim == 2: scale_v = scale_v[0] - if scale_v.size == 1 and self._weight_quantize_type == 'abs_max': + if ( + scale_v.size == 1 + and self._weight_quantize_type == 'abs_max' + ): scale_v = scale_v[0] else: scale_v = scale_v.tolist() @@ -1043,14 +1167,18 @@ def apply(self, graph): if self._round_type == 'round': param_v = self._load_var(input_arg_name) if any( - _check_grandchild_op_node(op_node, op) - for op in utils._channelwise_quant_axis1_ops): + _check_grandchild_op_node(op_node, op) + for op in utils._channelwise_quant_axis1_ops + ): quant_axis = 1 else: quant_axis = 0 quantized_param_v = utils.quant_tensor( - param_v.copy(), scale_v, quant_axis, - self._weight_bits) + param_v.copy(), + scale_v, + quant_axis, + self._weight_bits, + ) quantized_param_v = np.round(quantized_param_v) # Weight bias correction if self._bias_correction == True: @@ -1059,7 +1187,8 @@ def apply(self, graph): quantized_param_v, scale_v, quant_axis, - weight_bits=self._weight_bits) + weight_bits=self._weight_bits, + ) quantized_param_v = np.round(quantized_param_v) self._restore_var(input_arg_name, quantized_param_v) self._remove_fake_quant_and_dequant_op(graph, op_node) @@ -1075,13 +1204,19 @@ def apply(self, graph): ops = graph.all_op_nodes() for op_node in ops: op_node_desc = op_node.op() - if op_node_desc.has_attr("quantization_type") and \ - op_node_desc.attr("quantization_type") == "qat_with_weight": + if ( + op_node_desc.has_attr("quantization_type") + and op_node_desc.attr("quantization_type") == "qat_with_weight" + ): if self._weight_quantize_type == 'channel_wise_abs_max': - quant_axis = 1 if op_node.name() in \ - utils._channelwise_quant_axis1_ops else 0 + quant_axis = ( + 1 + if op_node.name() in utils._channelwise_quant_axis1_ops + else 0 + ) self._insert_post_channel_dequant_op( - graph, op_node, quant_axis) + graph, op_node, quant_axis + ) else: self._insert_post_dequant_op(graph, op_node) @@ -1105,7 +1240,8 @@ def _remove_fake_quant_and_dequant_op(self, graph, op_node): self._op_input_rename_map[k.node] = v else: self._op_input_rename_map[k.node] = self._op_input_rename_map[ - v.node] + v.node + ] graph.safe_remove_nodes(op_node) def _insert_post_channel_dequant_op(self, graph, op_node, quant_axis): @@ -1123,35 +1259,47 @@ def _insert_post_channel_dequant_op(self, graph, op_node, quant_axis): scale_v = self._quant_var_scale_map[original_var_name] if original_var_name in persistable_vars: assert isinstance( - scale_v, - list), 'The scale of parameter %s is not a list.' % ( - original_var_name) + scale_v, list + ), 'The scale of parameter %s is not a list.' % ( + original_var_name + ) channel_scale = np.array(scale_v) else: assert isinstance(scale_v, IrNode) scale_var_node = self._quant_var_scale_map[original_var_name] if len(op_node.output_arg_names()) != 1: - raise ValueError("Only support one output, but op %s has" - " more than one output." % (op_node.name())) + raise ValueError( + "Only support one output, but op %s has" + " more than one output." % (op_node.name()) + ) output_var_node = graph._find_node_by_name( - op_node.outputs, - op_node.output_arg_names()[0]) + op_node.outputs, op_node.output_arg_names()[0] + ) weight_scale_node = graph.create_persistable_node( name=unique_name.generate('channel_scale'), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=[channel_scale.shape[0]], - var_dtype=output_var_node.dtype()) - data_type = 'float64' if output_var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' - _init_var_node(weight_scale_node, channel_scale.astype(data_type), - self._scope, self._place) + var_dtype=output_var_node.dtype(), + ) + data_type = ( + 'float64' + if output_var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) + _init_var_node( + weight_scale_node, + channel_scale.astype(data_type), + self._scope, + self._place, + ) dequant_var_node = graph.create_var_node( name=self._dequantized_var_name(output_var_node.name()), var_type=output_var_node.type(), shape=output_var_node.shape(), - var_dtype=output_var_node.dtype()) + var_dtype=output_var_node.dtype(), + ) x_num_col_dims = 1 if op_node.name() in ['matmul', 'matmul_v2', 'mul']: x_num_col_dims = len(op_node.outputs[0].shape()) - 1 @@ -1163,13 +1311,14 @@ def _insert_post_channel_dequant_op(self, graph, op_node, quant_axis): 'quant_bits': [self._weight_bits, self._activation_bits], 'quant_axis': quant_axis, 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, - 'x_num_col_dims': x_num_col_dims + 'x_num_col_dims': x_num_col_dims, }, inputs={ 'X': output_var_node, - 'Scales': [weight_scale_node, scale_var_node] + 'Scales': [weight_scale_node, scale_var_node], }, - outputs={'Out': dequant_var_node}) + outputs={'Out': dequant_var_node}, + ) graph.link_to(output_var_node, dequant_op_node) graph.link_to(scale_var_node, dequant_op_node) graph.link_to(weight_scale_node, dequant_op_node) @@ -1195,8 +1344,10 @@ def _insert_post_dequant_op(self, graph, op_node): scale_v = self._quant_var_scale_map[original_var_name] if original_var_name in persistable_vars: assert self._is_float( - scale_v), 'The scale of parameter %s is not a float.' % ( - original_var_name) + scale_v + ), 'The scale of parameter %s is not a float.' % ( + original_var_name + ) scale_v = 1e-8 if scale_v == 0.0 else scale_v max_range *= param_range / scale_v else: @@ -1205,28 +1356,29 @@ def _insert_post_dequant_op(self, graph, op_node): scale_var_node = self._quant_var_scale_map[original_var_name] if len(op_node.output_arg_names()) != 1: - raise ValueError("Only support one output, but op %s has" - " more than one output." % (op_node.name())) + raise ValueError( + "Only support one output, but op %s has" + " more than one output." % (op_node.name()) + ) output_var_node = graph._find_node_by_name( - op_node.outputs, - op_node.output_arg_names()[0]) + op_node.outputs, op_node.output_arg_names()[0] + ) dequant_var_node = graph.create_var_node( name=self._dequantized_var_name(output_var_node.name()), var_type=output_var_node.type(), shape=output_var_node.shape(), - var_dtype=output_var_node.dtype()) + var_dtype=output_var_node.dtype(), + ) dequant_op_node = graph.create_op_node( op_type='fake_dequantize_max_abs', attrs={ 'max_range': float(max_range), - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward - }, - inputs={ - 'X': output_var_node, - 'Scale': scale_var_node + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, }, - outputs={'Out': dequant_var_node}) + inputs={'X': output_var_node, 'Scale': scale_var_node}, + outputs={'Out': dequant_var_node}, + ) graph.link_to(output_var_node, dequant_op_node) graph.link_to(scale_var_node, dequant_op_node) graph.link_to(dequant_op_node, dequant_var_node) @@ -1252,8 +1404,10 @@ def _remove_unused_var_nodes(self, graph): all_used_vars = {n.node for n in all_used_vars} all_unused_vars = { n - for n in filter(lambda node: node.node not in all_used_vars, - graph.all_var_nodes()) + for n in filter( + lambda node: node.node not in all_used_vars, + graph.all_var_nodes(), + ) } graph.safe_remove_nodes(all_unused_vars) @@ -1262,13 +1416,13 @@ def _original_var_name(self, var_name): Return the original variable name. """ if var_name.endswith('.quantized.dequantized'): - return var_name[:-len('.quantized.dequantized')] + return var_name[: -len('.quantized.dequantized')] if var_name.endswith('.quantized'): - return var_name[:-len('.quantized')] + return var_name[: -len('.quantized')] if var_name.endswith('.dequantized'): - return var_name[:-len('.dequantized')] + return var_name[: -len('.dequantized')] if var_name.endswith('@scale'): - return var_name[:-len('@scale')] + return var_name[: -len('@scale')] else: return var_name @@ -1279,12 +1433,14 @@ def _dequantized_var_name(self, var_name): return "%s.dequantized" % (var_name) def _is_float(self, v): - return isinstance(v, float) or isinstance(v, np.float32) \ + return ( + isinstance(v, float) + or isinstance(v, np.float32) or isinstance(v, np.float64) + ) class ConvertToInt8Pass(object): - def __init__(self, scope, place, quantizable_op_type=None): """ Convert the weights into int8_t type. @@ -1297,10 +1453,8 @@ def __init__(self, scope, place, quantizable_op_type=None): quantizable_op_type(list[str]): This input param will be removed latter. The pass will process all quantized op, so it is not necessary to set the input param. """ - assert scope is not None, \ - 'The scope cannot be set None.' - assert place is not None, \ - 'The place cannot be set None.' + assert scope is not None, 'The scope cannot be set None.' + assert place is not None, 'The place cannot be set None.' self._scope = scope self._place = _get_paddle_place(place) @@ -1318,17 +1472,21 @@ def apply(self, graph): ops = graph.all_op_nodes() input_map = {} for op_node in ops: - if op_node.op().has_attr("quantization_type") and \ - op_node.op().attr("quantization_type") == "qat_with_weight": + if ( + op_node.op().has_attr("quantization_type") + and op_node.op().attr("quantization_type") == "qat_with_weight" + ): for var_node in op_node.inputs: name = var_node.name() if name in persistable_vars: if name not in input_map: int8_var_node = self._convert_to_int8( - graph, var_node) + graph, var_node + ) input_map[name] = int8_var_node - graph.update_input_link(var_node, input_map[name], - op_node) + graph.update_input_link( + var_node, input_map[name], op_node + ) # remove the unused var node in the graph self._remove_unused_var_nodes(graph) @@ -1341,7 +1499,8 @@ def _convert_to_int8(self, graph, var_node): name=int8_var_node_name, var_type=var_node.type(), shape=var_node.shape(), - var_dtype=core.VarDesc.VarType.INT8) + var_dtype=core.VarDesc.VarType.INT8, + ) array = self._load_var(var_node.name()) self._scope.var(int8_var_node_name) self._store_var(int8_var_node_name, array, np.int8) @@ -1366,14 +1525,15 @@ def _remove_unused_var_nodes(self, graph): all_used_vars = {n.node for n in all_used_vars} all_unused_vars = { n - for n in filter(lambda node: node.node not in all_used_vars, - graph.all_var_nodes()) + for n in filter( + lambda node: node.node not in all_used_vars, + graph.all_var_nodes(), + ) } graph.safe_remove_nodes(all_unused_vars) class TransformForMobilePass(object): - def __init__(self): """ This pass is used to convert the frozen graph for paddle-mobile execution. @@ -1416,13 +1576,14 @@ def apply(self, graph): class OutScaleForTrainingPass(object): - - def __init__(self, - scope=None, - place=None, - moving_rate=0.9, - is_test=None, - scale_dict=None): + def __init__( + self, + scope=None, + place=None, + moving_rate=0.9, + is_test=None, + scale_dict=None, + ): """ This pass is used for calculating output scales of some operators. These output scales may be used by tensorRT or some other inference engines. @@ -1449,48 +1610,61 @@ def apply(self, graph): Args: graph(IrGraph): the target graph. """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' if self._is_test is None: self._is_test = graph.is_test() target_ops = [] for op in graph.all_op_nodes(): if op.name() in self._teller_set: target_ops.append(op) - with tqdm(total=len(target_ops), - bar_format='Adding OutScale op:|{bar}| {n_fmt}/{total_fmt}', - ncols=80) as t: + with tqdm( + total=len(target_ops), + bar_format='Adding OutScale op:|{bar}| {n_fmt}/{total_fmt}', + ncols=80, + ) as t: for op in target_ops: for output_var_name in utils._get_op_output_var_names(op): - in_node = graph._find_node_by_name(op.outputs, - output_var_name) - if in_node.dtype() not in \ - [core.VarDesc.VarType.FP64, core.VarDesc.VarType.FP32]: + in_node = graph._find_node_by_name( + op.outputs, output_var_name + ) + if in_node.dtype() not in [ + core.VarDesc.VarType.FP64, + core.VarDesc.VarType.FP32, + ]: continue - data_type = 'float64' if in_node.dtype() \ - == core.VarDesc.VarType.FP64 else 'float32' + data_type = ( + 'float64' + if in_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) try: graph._find_node_by_name( graph.all_var_nodes(), - self._scale_name(in_node.name())) + self._scale_name(in_node.name()), + ) continue except: scale_node = graph.create_persistable_node( name=self._scale_name(in_node.name()), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=[1], - var_dtype=in_node.dtype()) + var_dtype=in_node.dtype(), + ) if self._scale_dict is not None: try: scale_value = np.array( - [self._scale_dict[in_node.name()]]) + [self._scale_dict[in_node.name()]] + ) except: scale_value = np.ones([1], dtype=data_type) else: scale_value = np.ones([1], dtype=data_type) - _init_var_node(scale_node, scale_value, self._scope, - self._place) + _init_var_node( + scale_node, scale_value, self._scope, self._place + ) ins = {'X': in_node} outs = {'OutScale': scale_node} @@ -1499,22 +1673,32 @@ def apply(self, graph): name=unique_name.generate('scale_state@'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=in_node.dtype(), - shape=[1]) - _init_var_node(state_in_node, - np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + _init_var_node( + state_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) accum_in_node = graph.create_persistable_node( name=unique_name.generate('scale_accum@'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=in_node.dtype(), - shape=[1]) - _init_var_node(accum_in_node, - np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + _init_var_node( + accum_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) state_out_node = graph.create_var_node_from_desc( - state_in_node.var()) + state_in_node.var() + ) accum_out_node = graph.create_var_node_from_desc( - accum_in_node.var()) + accum_in_node.var() + ) ins['InState'] = state_in_node ins['InAccum'] = accum_in_node @@ -1524,14 +1708,14 @@ def apply(self, graph): attrs = { 'moving_rate': self._moving_rate, 'is_test': self._is_test, - 'op_role': - core.op_proto_and_checker_maker.OpRole.Forward + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, } scale_op_node = graph.create_op_node( op_type='moving_average_abs_max_scale', attrs=attrs, inputs=ins, - outputs=outs) + outputs=outs, + ) next_op_node = None if len(in_node.outputs) > 0: @@ -1558,7 +1742,6 @@ def _scale_name(self, var_name): class OutScaleForInferencePass(object): - def __init__(self, scope=None): """ This pass is used for setting output scales of some operators. @@ -1578,34 +1761,45 @@ def apply(self, graph): Args: graph(IrGraph): the target graph. """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' op_nodes = graph.all_op_nodes() for op_node in op_nodes: if op_node.name() in self._teller_set: var_names = utils._get_op_output_var_names(op_node) for var_name in var_names: - in_node = graph._find_node_by_name(op_node.outputs, - var_name) - if in_node.dtype() not in \ - [core.VarDesc.VarType.FP64, core.VarDesc.VarType.FP32]: + in_node = graph._find_node_by_name( + op_node.outputs, var_name + ) + if in_node.dtype() not in [ + core.VarDesc.VarType.FP64, + core.VarDesc.VarType.FP32, + ]: continue scale_name = self._scale_name(var_name) scale_var = self._scope.find_var(scale_name) - assert scale_var is not None, \ - "Can not find {} variable in the scope".format(scale_name) + assert ( + scale_var is not None + ), "Can not find {} variable in the scope".format( + scale_name + ) scale_value = np.array(scale_var.get_tensor())[0] # For compatibility, we save output threshold by two methods. op_node.op()._set_attr("out_threshold", float(scale_value)) argname_index = utils._get_output_name_index( - op_node, var_name) - assert argname_index is not None, \ + op_node, var_name + ) + assert argname_index is not None, ( var_name + " is not the output of the op" - op_node.op()._set_attr(argname_index[0] + str(argname_index[1]) \ - + "_threshold", float(scale_value)) + ) + op_node.op()._set_attr( + argname_index[0] + str(argname_index[1]) + "_threshold", + float(scale_value), + ) op_node.op()._set_attr("with_quant_attr", True) graph.resolve_hazard() return graph @@ -1626,16 +1820,18 @@ class AddQuantDequantPass(object): # To be compatible with PaddleSlim, not remove _activation_type for now _activation_type = ["relu", "relu6", "leaky_relu", "tanh", "swish"] - def __init__(self, - scope=None, - place=None, - moving_rate=0.9, - quant_bits=8, - skip_pattern=["skip_quant"], - quantizable_op_type=["elementwise_add", "pool2d"], - is_full_quantized=False, - is_test=None, - scale_dict=None): + def __init__( + self, + scope=None, + place=None, + moving_rate=0.9, + quant_bits=8, + skip_pattern=["skip_quant"], + quantizable_op_type=["elementwise_add", "pool2d"], + is_full_quantized=False, + is_test=None, + scale_dict=None, + ): """ Constructor. @@ -1671,8 +1867,9 @@ def __init__(self, else: self._quantizable_op_type = quantizable_op_type for op_type in quantizable_op_type: - assert op_type in utils._act_supported_quantizable_op_type, \ + assert op_type in utils._act_supported_quantizable_op_type, ( op_type + " is not supported for quantization." + ) self._quantizable_grad_op_type = [ '%s_grad' % (op) for op in self._quantizable_op_type ] @@ -1690,50 +1887,71 @@ def apply(self, graph): Returns: None """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' if self._is_test is None: self._is_test = graph.is_test() dequantized_vars_map = collections.OrderedDict() # Forward stage, insert quant_dequant op all_op_nodes = graph.all_op_nodes() - with tqdm(total=len(all_op_nodes), - bar_format= - 'Adding quant activation op:|{bar}| {n_fmt}/{total_fmt}', - ncols=80) as t: + with tqdm( + total=len(all_op_nodes), + bar_format='Adding quant activation op:|{bar}| {n_fmt}/{total_fmt}', + ncols=80, + ) as t: for op_node in all_op_nodes: if op_node.name() in self._quantizable_op_type: is_skip = False if isinstance(self._skip_pattern, list): - is_skip = op_node.op().has_attr("op_namescope") and \ - any(pattern in op_node.op().attr("op_namescope") for pattern in self._skip_pattern) + is_skip = op_node.op().has_attr("op_namescope") and any( + pattern in op_node.op().attr("op_namescope") + for pattern in self._skip_pattern + ) elif isinstance(self._skip_pattern, str): - is_skip = op_node.op().has_attr("op_namescope") and \ - op_node.op().attr("op_namescope").find(self._skip_pattern) != -1 - is_quantized = op_node.op().has_attr("quantization_type") and \ - op_node.op().attr("quantization_type") == "qat_with_weight" - if is_skip or is_quantized or \ - (not _is_input_all_not_persistable(graph, op_node)): + is_skip = ( + op_node.op().has_attr("op_namescope") + and op_node.op() + .attr("op_namescope") + .find(self._skip_pattern) + != -1 + ) + is_quantized = ( + op_node.op().has_attr("quantization_type") + and op_node.op().attr("quantization_type") + == "qat_with_weight" + ) + if ( + is_skip + or is_quantized + or (not _is_input_all_not_persistable(graph, op_node)) + ): continue - op_node.op()._set_attr("quantization_type", - "qat_without_weight") + op_node.op()._set_attr( + "quantization_type", "qat_without_weight" + ) op_node.op()._set_attr("activation_bits", self._quant_bits) op_node.op()._set_attr("with_quant_attr", True) arg_names = utils._get_op_input_var_names(op_node) for arg_name in arg_names: in_node = graph._find_node_by_name( - op_node.inputs, arg_name) + op_node.inputs, arg_name + ) if arg_name in dequantized_vars_map: quant_var_node = dequantized_vars_map[arg_name] else: - quant_var_node, _ = \ - self._inser_quant_dequant_moving_average_abs_max_op( - graph, in_node, self._quant_bits) + ( + quant_var_node, + _, + ) = self._inser_quant_dequant_moving_average_abs_max_op( + graph, in_node, self._quant_bits + ) dequantized_vars_map[arg_name] = quant_var_node - graph.update_input_link(in_node, quant_var_node, - op_node) + graph.update_input_link( + in_node, quant_var_node, op_node + ) t.update() # Backward stage, update input link @@ -1742,35 +1960,45 @@ def apply(self, graph): for input_name in op_node.input_arg_names(): if input_name in dequantized_vars_map: in_node = graph._find_node_by_name( - op_node.inputs, input_name) + op_node.inputs, input_name + ) dequant_var_node = dequantized_vars_map[input_name] - graph.update_input_link(in_node, dequant_var_node, - op_node) + graph.update_input_link( + in_node, dequant_var_node, op_node + ) graph.resolve_hazard() return graph - def _inser_quant_dequant_moving_average_abs_max_op(self, graph, var_node, - quant_bits): - """Insert fake_quantize_dequantize_moving_average_abs_max op. - """ - quant_var_node = graph.create_var_node(name="{}.quant_dequant".format( - var_node.name()), - var_type=var_node.type(), - shape=var_node.shape(), - var_dtype=var_node.dtype()) + def _inser_quant_dequant_moving_average_abs_max_op( + self, graph, var_node, quant_bits + ): + """Insert fake_quantize_dequantize_moving_average_abs_max op.""" + quant_var_node = graph.create_var_node( + name="{}.quant_dequant".format(var_node.name()), + var_type=var_node.type(), + shape=var_node.shape(), + var_dtype=var_node.dtype(), + ) scale_name = "{}.quant_dequant@scale".format(var_node.name()) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) try: - if self._scale_dict is not None and var_node.name( - ) in self._scale_dict.keys(): - scale_value = np.array([self._scale_dict[var_node.name()]], - dtype=data_type) + if ( + self._scale_dict is not None + and var_node.name() in self._scale_dict.keys() + ): + scale_value = np.array( + [self._scale_dict[var_node.name()]], dtype=data_type + ) else: scale_value = np.array( self._scope.find_var(scale_name).get_tensor(), - dtype=data_type) + dtype=data_type, + ) except: scale_value = np.array([_SCALE_DEFAULT_VALUE], dtype=data_type) @@ -1778,7 +2006,8 @@ def _inser_quant_dequant_moving_average_abs_max_op(self, graph, var_node, name="{}.quant_dequant@scale".format(var_node.name()), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=[1], - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) _init_var_node(scale_in_node, scale_value, self._scope, self._place) scale_out_node = graph.create_var_node_from_desc(scale_in_node.var()) @@ -1789,22 +2018,37 @@ def _inser_quant_dequant_moving_average_abs_max_op(self, graph, var_node, name=unique_name.generate('quant_dequant.state'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=var_node.dtype(), - shape=[1]) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' - _init_var_node(state_in_node, np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) + _init_var_node( + state_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) accum_in_node = graph.create_persistable_node( name=unique_name.generate('quant_dequant.accum'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=var_node.dtype(), - shape=[1]) - _init_var_node(accum_in_node, np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + _init_var_node( + accum_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) state_out_node = graph.create_var_node_from_desc( - state_in_node.var()) + state_in_node.var() + ) accum_out_node = graph.create_var_node_from_desc( - accum_in_node.var()) + accum_in_node.var() + ) ins['InState'] = state_in_node ins['InAccum'] = accum_in_node @@ -1815,14 +2059,15 @@ def _inser_quant_dequant_moving_average_abs_max_op(self, graph, var_node, 'bit_length': quant_bits, 'moving_rate': self._moving_rate, 'is_test': self._is_test, - 'op_role': core.op_proto_and_checker_maker.OpRole.Forward + 'op_role': core.op_proto_and_checker_maker.OpRole.Forward, } quant_op_node = graph.create_op_node( op_type='fake_quantize_dequantize_moving_average_abs_max', attrs=attrs, inputs=ins, - outputs=outs) + outputs=outs, + ) graph.link_to(var_node, quant_op_node) graph.link_to(scale_in_node, quant_op_node) @@ -1856,15 +2101,17 @@ class InsertQuantizeLinear(object): scale_dict(dict, optional): calibration ranges of tensors output. """ - def __init__(self, - place, - scope, - quant_bits=8, - quant_axis=-1, - channel_wise=False, - moving_rate=0.9, - is_test=True, - scale_dict=None): + def __init__( + self, + place, + scope, + quant_bits=8, + quant_axis=-1, + channel_wise=False, + moving_rate=0.9, + is_test=True, + scale_dict=None, + ): self._place = place self._scope = scope self.quant_bits = quant_bits @@ -1881,32 +2128,42 @@ def insert_quant_op(self, graph, var_node, var_name=None): name=self._quantized_var_name(var_name), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' + var_dtype=var_node.dtype(), + ) + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) scale_name = self._quantized_scale_name(var_name) if self.channel_wise: scale_var_shape = var_node.shape()[self.quant_axis] scale_var_type = core.VarDesc.VarType.LOD_TENSOR - init_scale_value = np.ones(scale_var_shape, - dtype=data_type) * _SCALE_DEFAULT_VALUE + init_scale_value = ( + np.ones(scale_var_shape, dtype=data_type) * _SCALE_DEFAULT_VALUE + ) else: scale_var_shape = 1 scale_var_type = var_node.type() init_scale_value = np.array([_SCALE_DEFAULT_VALUE], dtype=data_type) - if self._scale_dict is not None and var_node.name( - ) in self._scale_dict.keys(): - init_scale_value = np.array([self._scale_dict[var_node.name()]], - dtype=data_type) + if ( + self._scale_dict is not None + and var_node.name() in self._scale_dict.keys() + ): + init_scale_value = np.array( + [self._scale_dict[var_node.name()]], dtype=data_type + ) scale_var_node = graph.create_persistable_node( name=scale_name, var_type=scale_var_type, shape=[scale_var_shape], - var_dtype=var_node.dtype()) - _init_var_node(scale_var_node, init_scale_value, self._scope, - self._place) + var_dtype=var_node.dtype(), + ) + _init_var_node( + scale_var_node, init_scale_value, self._scope, self._place + ) zero_point_node = None if zero_point_node is None: @@ -1914,10 +2171,14 @@ def insert_quant_op(self, graph, var_node, var_name=None): name=self._zero_point_name(quant_var_node.name()), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=scale_var_node.shape(), - var_dtype=core.VarDesc.VarType.INT32) - _init_var_node(zero_point_node, - np.zeros(scale_var_node.shape(), dtype="int32"), - self._scope, self._place) + var_dtype=core.VarDesc.VarType.INT32, + ) + _init_var_node( + zero_point_node, + np.zeros(scale_var_node.shape(), dtype="int32"), + self._scope, + self._place, + ) inputs = {"X": var_node, "Scale": scale_var_node} if zero_point_node is not None: @@ -1928,27 +2189,43 @@ def insert_quant_op(self, graph, var_node, var_name=None): outputs = {"Y": quant_var_node} if not self._is_test: scale_out_node = graph.create_var_node_from_desc( - scale_var_node.var()) + scale_var_node.var() + ) state_in_node = graph.create_persistable_node( name=unique_name.generate('state'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=var_node.dtype(), - shape=[1]) - data_type = 'float64' if var_node.dtype( - ) == core.VarDesc.VarType.FP64 else 'float32' - _init_var_node(state_in_node, np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + data_type = ( + 'float64' + if var_node.dtype() == core.VarDesc.VarType.FP64 + else 'float32' + ) + _init_var_node( + state_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) accum_in_node = graph.create_persistable_node( name=unique_name.generate('accum'), var_type=core.VarDesc.VarType.LOD_TENSOR, var_dtype=var_node.dtype(), - shape=[1]) - _init_var_node(accum_in_node, np.ones([1], dtype=data_type), - self._scope, self._place) + shape=[1], + ) + _init_var_node( + accum_in_node, + np.ones([1], dtype=data_type), + self._scope, + self._place, + ) state_out_node = graph.create_var_node_from_desc( - state_in_node.var()) + state_in_node.var() + ) accum_out_node = graph.create_var_node_from_desc( - accum_in_node.var()) + accum_in_node.var() + ) outputs["OutScale"] = scale_out_node inputs['InState'] = state_in_node @@ -1958,10 +2235,12 @@ def insert_quant_op(self, graph, var_node, var_name=None): attrs["is_test"] = self._is_test attrs['moving_rate'] = self._moving_rate - quant_op_node = graph.create_op_node(op_type="quantize_linear", - attrs=attrs, - inputs=inputs, - outputs=outputs) + quant_op_node = graph.create_op_node( + op_type="quantize_linear", + attrs=attrs, + inputs=inputs, + outputs=outputs, + ) graph.link_to(var_node, quant_op_node) graph.link_to(scale_var_node, quant_op_node) @@ -1983,7 +2262,8 @@ def insert_dequant_op(self, graph, var_node, scale_var_node): name=self._dequantized_var_name(var_node.name()), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) + var_dtype=var_node.dtype(), + ) zero_point_node = None if zero_point_node is None: @@ -1991,10 +2271,14 @@ def insert_dequant_op(self, graph, var_node, scale_var_node): name=self._zero_point_name(dequant_var_node.name()), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=scale_var_node.shape(), - var_dtype=core.VarDesc.VarType.INT32) - _init_var_node(zero_point_node, - np.zeros(scale_var_node.shape(), dtype="int32"), - self._scope, self._place) + var_dtype=core.VarDesc.VarType.INT32, + ) + _init_var_node( + zero_point_node, + np.zeros(scale_var_node.shape(), dtype="int32"), + self._scope, + self._place, + ) inputs = {"X": var_node, "Scale": scale_var_node} if zero_point_node is not None: @@ -2003,10 +2287,12 @@ def insert_dequant_op(self, graph, var_node, scale_var_node): attrs = {"quant_axis": self.quant_axis, "bit_length": self.quant_bits} attrs["op_role"] = core.op_proto_and_checker_maker.OpRole.Forward - quant_op_node = graph.create_op_node(op_type="dequantize_linear", - attrs=attrs, - inputs=inputs, - outputs={"Y": dequant_var_node}) + quant_op_node = graph.create_op_node( + op_type="dequantize_linear", + attrs=attrs, + inputs=inputs, + outputs={"Y": dequant_var_node}, + ) graph.link_to(var_node, quant_op_node) graph.link_to(scale_var_node, quant_op_node) @@ -2046,24 +2332,26 @@ class QuantizationTransformPassV2(QuantizationTransformPass): the quantized ops's inputs. It is used in the new format of quantization. """ - def __init__(self, - scope=None, - place=None, - weight_bits=8, - activation_bits=8, - activation_quantize_type='abs_max', - weight_quantize_type='abs_max', - window_size=10000, - moving_rate=0.9, - skip_pattern=['skip_quant'], - quantizable_op_type=['conv2d', 'depthwise_conv2d', 'mul'], - weight_quantize_func=None, - act_quantize_func=None, - weight_preprocess_func=None, - act_preprocess_func=None, - optimizer_func=None, - executor=None, - is_test=None): + def __init__( + self, + scope=None, + place=None, + weight_bits=8, + activation_bits=8, + activation_quantize_type='abs_max', + weight_quantize_type='abs_max', + window_size=10000, + moving_rate=0.9, + skip_pattern=['skip_quant'], + quantizable_op_type=['conv2d', 'depthwise_conv2d', 'mul'], + weight_quantize_func=None, + act_quantize_func=None, + weight_preprocess_func=None, + act_preprocess_func=None, + optimizer_func=None, + executor=None, + is_test=None, + ): r""" Args: scope(paddle.Scope): When activation use 'range_abs_max' as the quantize @@ -2149,21 +2437,26 @@ def __init__(self, self._optimizer = optimizer_func self._exe = executor quant_type = [ - 'abs_max', 'channel_wise_abs_max', 'range_abs_max', - 'moving_average_abs_max' + 'abs_max', + 'channel_wise_abs_max', + 'range_abs_max', + 'moving_average_abs_max', ] - assert activation_quantize_type != 'channel_wise_abs_max', \ - "The activation quantization type does not support 'channel_wise_abs_max'." + assert ( + activation_quantize_type != 'channel_wise_abs_max' + ), "The activation quantization type does not support 'channel_wise_abs_max'." if activation_quantize_type not in quant_type: raise ValueError( "Unknown activation_quantize_type : '%s'. It can only be " - "'abs_max' or 'range_abs_max' or 'moving_average_abs_max'." % - (str(activation_quantize_type))) + "'abs_max' or 'range_abs_max' or 'moving_average_abs_max'." + % (str(activation_quantize_type)) + ) if weight_quantize_type not in quant_type: raise ValueError( "Unknown weight_quantize_type: '%s'. It can only be " "'abs_max' or 'channel_wise_abs_max' or 'range_abs_max' " - "or 'moving_average_abs_max'." % (str(weight_quantize_type))) + "or 'moving_average_abs_max'." % (str(weight_quantize_type)) + ) self._activation_quantize_type = activation_quantize_type self._weight_quantize_type = weight_quantize_type @@ -2172,8 +2465,9 @@ def __init__(self, self._quantizable_ops = quantizable_op_type for op in self._quantizable_ops: - assert op in utils._weight_supported_quantizable_op_type, \ + assert op in utils._weight_supported_quantizable_op_type, ( op + " is not supported for quantization." + ) self._quantizable_grad_ops = [ '%s_grad' % (op) for op in self._quantizable_ops ] @@ -2191,13 +2485,16 @@ def __init__(self, def _quant_preprocess(self, op_node): user_skipped = False if isinstance(self._skip_pattern, list): - user_skipped = op_node.op().has_attr("op_namescope") and \ - any(pattern in op_node.op().attr("op_namescope") \ - for pattern in self._skip_pattern) + user_skipped = op_node.op().has_attr("op_namescope") and any( + pattern in op_node.op().attr("op_namescope") + for pattern in self._skip_pattern + ) elif isinstance(self._skip_pattern, str): - user_skipped = op_node.op().has_attr("op_namescope") and \ - op_node.op().attr("op_namescope").find( - self._skip_pattern) != -1 + user_skipped = ( + op_node.op().has_attr("op_namescope") + and op_node.op().attr("op_namescope").find(self._skip_pattern) + != -1 + ) if user_skipped: op_node.op()._set_attr("skip_quant", True) @@ -2215,8 +2512,9 @@ def _transform_forward(self, graph, op): name = var_node.name() if name in self.processed_vars: continue - is_weight = True if var_node.name() in self.persistable_vars \ - else False + is_weight = ( + True if var_node.name() in self.persistable_vars else False + ) # if var node is weight and weight_preprocess_func is not None, # will insert weight preprocess func @@ -2225,13 +2523,13 @@ def _transform_forward(self, graph, op): # will insert activation preprocess func # to preorocess activation before quantization if is_weight and self._weight_preprocess_func is not None: - var_node = self._insert_func(graph, - self._weight_preprocess_func, - var_node, op) + var_node = self._insert_func( + graph, self._weight_preprocess_func, var_node, op + ) elif not is_weight and self._act_preprocess_func is not None: - var_node = self._insert_func(graph, - self._act_preprocess_func, - var_node, op) + var_node = self._insert_func( + graph, self._act_preprocess_func, var_node, op + ) # if var node is weight and weight_quantize_func is not None, # will insert weight quantize func to quantize and dequantize weight @@ -2239,26 +2537,36 @@ def _transform_forward(self, graph, op): # will insert act quantize func to quantize and dequantize activation if is_weight and self._weight_quantize_func is not None: target_out_node = self._insert_func( - graph, self._weight_quantize_func, var_node, op) + graph, self._weight_quantize_func, var_node, op + ) self.processed_vars.append(name) continue elif not is_weight and self._act_quantize_func is not None: - target_out_node = self._insert_func(graph, - self._act_quantize_func, - var_node, op) + target_out_node = self._insert_func( + graph, self._act_quantize_func, var_node, op + ) self.processed_vars.append(name) continue - quant_bits = self._weight_bits if var_node.name() in self.persistable_vars \ + quant_bits = ( + self._weight_bits + if var_node.name() in self.persistable_vars else self._activation_bits - quant_type = self._weight_quantize_type if is_weight \ + ) + quant_type = ( + self._weight_quantize_type + if is_weight else self._activation_quantize_type + ) quant_axis = -1 channel_wise = False if quant_type == 'channel_wise_abs_max': # Weight quantization channel_wise = True - quant_axis = 1 if op.name() in \ - utils._channelwise_quant_axis1_ops else 0 + quant_axis = ( + 1 + if op.name() in utils._channelwise_quant_axis1_ops + else 0 + ) insert_quant_pass = InsertQuantizeLinear( self._place, self._scope, @@ -2266,11 +2574,17 @@ def _transform_forward(self, graph, op): quant_axis=quant_axis, channel_wise=channel_wise, moving_rate=self._moving_rate, - is_test=self._is_test) - quant_var_node, scale_var_node = insert_quant_pass.insert_quant_op( - graph, var_node, var_name=name) + is_test=self._is_test, + ) + ( + quant_var_node, + scale_var_node, + ) = insert_quant_pass.insert_quant_op( + graph, var_node, var_name=name + ) dequant_var_node = insert_quant_pass.insert_dequant_op( - graph, quant_var_node, scale_var_node) + graph, quant_var_node, scale_var_node + ) self.dequantized_vars[name] = dequant_var_node graph.update_input_link(var_node, dequant_var_node, op) @@ -2304,8 +2618,9 @@ def apply(self, graph): Returns: None """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' if self._is_test is None: self._is_test = graph.is_test() @@ -2317,21 +2632,25 @@ def apply(self, graph): # Do the preproccess of quantization, such as skipping some ops # for not being quantized. for op in ops: - if op.name() in self._quantizable_ops or \ - op.name() in self._quantizable_grad_ops: + if ( + op.name() in self._quantizable_ops + or op.name() in self._quantizable_grad_ops + ): self._quant_preprocess(op) # Insert mapping table to solve the problem in saving inference model. graph.out_node_mapping_table = dict() # The process of _transform_forward and _transform_backward is needed in two for loops. # The loop for transforming the forward graph: - with tqdm(total=len(ops), - bar_format= - 'Adding quant op with weight:|{bar}| {n_fmt}/{total_fmt}', - ncols=80) as t: + with tqdm( + total=len(ops), + bar_format='Adding quant op with weight:|{bar}| {n_fmt}/{total_fmt}', + ncols=80, + ) as t: for op in ops: if op.name() in self._quantizable_ops: - if not self._is_skip_quant(graph, - op) and self._has_weight(op): + if not self._is_skip_quant(graph, op) and self._has_weight( + op + ): self._transform_forward(graph, op) t.update() # The loop for renaming the inputs of backward op. @@ -2350,16 +2669,18 @@ class AddQuantDequantPassV2(object): # To be compatible with PaddleSlim, not remove _activation_type for now _activation_type = ["relu", "relu6", "leaky_relu", "tanh", "swish"] - def __init__(self, - scope=None, - place=None, - moving_rate=0.9, - quant_bits=8, - skip_pattern=["skip_quant"], - quantizable_op_type=["elementwise_add", "pool2d"], - is_full_quantized=False, - is_test=None, - scale_dict=None): + def __init__( + self, + scope=None, + place=None, + moving_rate=0.9, + quant_bits=8, + skip_pattern=["skip_quant"], + quantizable_op_type=["elementwise_add", "pool2d"], + is_full_quantized=False, + is_test=None, + scale_dict=None, + ): """ Args: scope(paddle.Scope): The scope is used to initialize these new parameters. @@ -2409,8 +2730,9 @@ def __init__(self, else: self._quantizable_op_type = quantizable_op_type for op_type in quantizable_op_type: - assert op_type in utils._act_supported_quantizable_op_type, \ + assert op_type in utils._act_supported_quantizable_op_type, ( op_type + " is not supported for quantization." + ) self._quantizable_grad_op_type = [ '%s_grad' % (op) for op in self._quantizable_op_type ] @@ -2429,8 +2751,9 @@ def apply(self, graph): Returns: None """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' if self._is_test is None: self._is_test = graph.is_test() dequantized_vars_map = collections.OrderedDict() @@ -2441,28 +2764,40 @@ def apply(self, graph): # Forward stage, insert quant_dequant op all_op_nodes = graph.all_op_nodes() - with tqdm(total=len(all_op_nodes), - bar_format= - 'Adding quant activation op:|{bar}| {n_fmt}/{total_fmt}', - ncols=80) as t: + with tqdm( + total=len(all_op_nodes), + bar_format='Adding quant activation op:|{bar}| {n_fmt}/{total_fmt}', + ncols=80, + ) as t: for op_node in all_op_nodes: if op_node.name() in self._quantizable_op_type: is_skip = False if isinstance(self._skip_pattern, list): - is_skip = op_node.op().has_attr("op_namescope") and \ - any(pattern in op_node.op().attr("op_namescope") for pattern in self._skip_pattern) + is_skip = op_node.op().has_attr("op_namescope") and any( + pattern in op_node.op().attr("op_namescope") + for pattern in self._skip_pattern + ) elif isinstance(self._skip_pattern, str): - is_skip = op_node.op().has_attr("op_namescope") and \ - op_node.op().attr("op_namescope").find(self._skip_pattern) != -1 - is_quantized = op_node.op().has_attr("quantization_type") and \ - op_node.op().attr("quantization_type") == "qat_with_weight" + is_skip = ( + op_node.op().has_attr("op_namescope") + and op_node.op() + .attr("op_namescope") + .find(self._skip_pattern) + != -1 + ) + is_quantized = ( + op_node.op().has_attr("quantization_type") + and op_node.op().attr("quantization_type") + == "qat_with_weight" + ) if is_skip or is_quantized: continue arg_names = utils._get_op_input_var_names(op_node) for arg_name in arg_names: in_node = graph._find_node_by_name( - op_node.inputs, arg_name) + op_node.inputs, arg_name + ) if in_node.persistable(): continue if arg_name in dequantized_vars_map: @@ -2476,14 +2811,23 @@ def apply(self, graph): channel_wise=False, moving_rate=self._moving_rate, is_test=self._is_test, - scale_dict=self._scale_dict) - quant_var_node, scale_var_node = insert_quant_pass.insert_quant_op( - graph, in_node) - dequant_var_node = insert_quant_pass.insert_dequant_op( - graph, quant_var_node, scale_var_node) + scale_dict=self._scale_dict, + ) + ( + quant_var_node, + scale_var_node, + ) = insert_quant_pass.insert_quant_op( + graph, in_node + ) + dequant_var_node = ( + insert_quant_pass.insert_dequant_op( + graph, quant_var_node, scale_var_node + ) + ) dequantized_vars_map[arg_name] = dequant_var_node - graph.update_input_link(in_node, dequant_var_node, - op_node) + graph.update_input_link( + in_node, dequant_var_node, op_node + ) t.update() # Backward stage, update input link @@ -2492,10 +2836,12 @@ def apply(self, graph): for input_name in op_node.input_arg_names(): if input_name in dequantized_vars_map: in_node = graph._find_node_by_name( - op_node.inputs, input_name) + op_node.inputs, input_name + ) dequant_var_node = dequantized_vars_map[input_name] - graph.update_input_link(in_node, dequant_var_node, - op_node) + graph.update_input_link( + in_node, dequant_var_node, op_node + ) return graph @@ -2536,13 +2882,16 @@ def __init__(self, scope, place, quant_bits=8): assert self._place != None, "place must not be None." def apply(self, graph): - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' fake_quant_dequant_ops = [] for op in graph.all_op_nodes(): - if op.name() in _fake_quant_dequant_op_list or op.name( - ) == "moving_average_abs_max_scale": + if ( + op.name() in _fake_quant_dequant_op_list + or op.name() == "moving_average_abs_max_scale" + ): fake_quant_dequant_ops.append(op) for _op in fake_quant_dequant_ops: @@ -2555,13 +2904,18 @@ def apply(self, graph): def _replace_op(self, graph, op): x_node = graph._find_node_by_name(op.inputs, op.input("X")[0]) out_node = graph._find_node_by_name(op.outputs, op.output("Out")[0]) - scale_node = graph._find_node_by_name(op.outputs, - op.output("OutScale")[0]) - - quant_axis = op.op().attr("quant_axis") if op.op().has_attr( - "quant_axis") else -1 - bit_length = op.op().attr("bit_length") if op.op().has_attr( - "bit_length") else self._quant_bits + scale_node = graph._find_node_by_name( + op.outputs, op.output("OutScale")[0] + ) + + quant_axis = ( + op.op().attr("quant_axis") if op.op().has_attr("quant_axis") else -1 + ) + bit_length = ( + op.op().attr("bit_length") + if op.op().has_attr("bit_length") + else self._quant_bits + ) zero_point_node = None quanted_node = x_node @@ -2570,43 +2924,46 @@ def _replace_op(self, graph, op): name=self._zero_point_name(quanted_node.name()), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=scale_node.shape(), - var_dtype=core.VarDesc.VarType.INT32) - _init_var_node(zero_point_node, - np.zeros(scale_node.shape(), dtype="int32"), - self._scope, self._place) - - quant_var_node = graph.create_var_node(name=self._quantized_var_name( - x_node.name()), - var_type=x_node.type(), - shape=x_node.shape(), - var_dtype=x_node.dtype()) - quant_op_node = graph.create_op_node(op_type="quantize_linear", - attrs={ - "quant_axis": quant_axis, - "bit_length": bit_length - }, - inputs={ - "X": x_node, - "Scale": scale_node, - "ZeroPoint": zero_point_node - }, - outputs={"Y": quant_var_node}) + var_dtype=core.VarDesc.VarType.INT32, + ) + _init_var_node( + zero_point_node, + np.zeros(scale_node.shape(), dtype="int32"), + self._scope, + self._place, + ) + + quant_var_node = graph.create_var_node( + name=self._quantized_var_name(x_node.name()), + var_type=x_node.type(), + shape=x_node.shape(), + var_dtype=x_node.dtype(), + ) + quant_op_node = graph.create_op_node( + op_type="quantize_linear", + attrs={"quant_axis": quant_axis, "bit_length": bit_length}, + inputs={ + "X": x_node, + "Scale": scale_node, + "ZeroPoint": zero_point_node, + }, + outputs={"Y": quant_var_node}, + ) graph.link_to(x_node, quant_op_node) graph.link_to(scale_node, quant_op_node) if zero_point_node is not None: graph.link_to(zero_point_node, quant_op_node) graph.link_to(quant_op_node, quant_var_node) - dequant_op_node = graph.create_op_node(op_type="dequantize_linear", - attrs={ - "quant_axis": quant_axis, - "bit_length": bit_length - }, - inputs={ - "X": quant_var_node, - "Scale": scale_node, - "ZeroPoint": zero_point_node - }, - outputs={"Y": out_node}) + dequant_op_node = graph.create_op_node( + op_type="dequantize_linear", + attrs={"quant_axis": quant_axis, "bit_length": bit_length}, + inputs={ + "X": quant_var_node, + "Scale": scale_node, + "ZeroPoint": zero_point_node, + }, + outputs={"Y": out_node}, + ) graph.link_to(quant_var_node, dequant_op_node) graph.link_to(scale_node, dequant_op_node) if zero_point_node is not None: @@ -2657,12 +3014,14 @@ class QuantWeightPass(object): quant_weight_pass.apply(graph) """ - def __init__(self, - scope, - place, - bias_correction=False, - quant_bits=8, - save_int_weight=True): + def __init__( + self, + scope, + place, + bias_correction=False, + quant_bits=8, + save_int_weight=True, + ): self._place = _get_paddle_place(place) self._scope = scope self._bias_correction = bias_correction @@ -2672,8 +3031,9 @@ def __init__(self, assert self._place != None, "place must not be None." def apply(self, graph): - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' fake_quant_ops_for_weight = [] fake_quant_ops = [ @@ -2682,17 +3042,21 @@ def apply(self, graph): for _op in fake_quant_ops: x_node = graph._find_node_by_name(_op.inputs, _op.input("X")[0]) if x_node.persistable(): - scale_node = graph._find_node_by_name(_op.inputs, - _op.input("Scale")[0]) + scale_node = graph._find_node_by_name( + _op.inputs, _op.input("Scale")[0] + ) zero_point_node = graph._find_node_by_name( - _op.inputs, - _op.input("ZeroPoint")[0]) - out_node = graph._find_node_by_name(_op.outputs, - _op.output("Y")[0]) + _op.inputs, _op.input("ZeroPoint")[0] + ) + out_node = graph._find_node_by_name( + _op.outputs, _op.output("Y")[0] + ) scale_v = self._load_var(scale_node.name()) - assert scale_v.ndim in [1, 2 - ], "the dim of scale_v should be 1 or 2" + assert scale_v.ndim in [ + 1, + 2, + ], "the dim of scale_v should be 1 or 2" if scale_v.ndim == 2: scale_v = scale_v[0] if scale_v.size == 1 and _op.name() == 'abs_max': @@ -2702,24 +3066,28 @@ def apply(self, graph): param_v = self._load_var(x_node.name()) quant_axis = _op.op().attr("quant_axis") bits_length = _op.op().attr("bit_length") - quantized_param_v = utils.quant_tensor(param_v.copy(), - scale_v, - quant_axis, - bits_length, - onnx_format=True) + quantized_param_v = utils.quant_tensor( + param_v.copy(), + scale_v, + quant_axis, + bits_length, + onnx_format=True, + ) if self._bias_correction == True: quantized_param_v = utils.bias_correction_w( param_v, quantized_param_v, scale_v, quant_axis, - weight_bits=bits_length) + weight_bits=bits_length, + ) if self._save_int_weight: # cast weight type to int if self._quant_bits == 8: save_weight_dtype = np.int8 quantized_param_v = quantized_param_v.astype( - save_weight_dtype) + save_weight_dtype + ) self._restore_var(x_node.name(), quantized_param_v) for next_op_node in out_node.outputs: @@ -2739,8 +3107,10 @@ def _remove_unused_var_nodes(self, graph): all_used_vars = {n.node for n in all_used_vars} all_unused_vars = { n - for n in filter(lambda node: node.node not in all_used_vars, - graph.all_var_nodes()) + for n in filter( + lambda node: node.node not in all_used_vars, + graph.all_var_nodes(), + ) } graph.safe_remove_nodes(all_unused_vars) @@ -2775,24 +3145,29 @@ def apply(self, graph): Args: graph(IrGraph): the target graph. """ - assert isinstance(graph, - IrGraph), 'graph must be the instance of IrGraph.' + assert isinstance( + graph, IrGraph + ), 'graph must be the instance of IrGraph.' dequant_node_map = {} dequantized_vars_map = collections.OrderedDict() for op_node in graph.all_op_nodes(): if op_node.name() in self._teller_set: var_names = utils._get_op_output_var_names(op_node) for var_name in var_names: - out_node = graph._find_node_by_name(op_node.outputs, - var_name) - if out_node.dtype() not in \ - [core.VarDesc.VarType.FP64, core.VarDesc.VarType.FP32]: + out_node = graph._find_node_by_name( + op_node.outputs, var_name + ) + if out_node.dtype() not in [ + core.VarDesc.VarType.FP64, + core.VarDesc.VarType.FP32, + ]: continue if var_name in dequantized_vars_map: dequant_var_node = dequantized_vars_map[var_name] else: dequant_var_node = self._insert_quant_dequant_op( - graph, out_node) + graph, out_node + ) dequantized_vars_map[var_name] = dequant_var_node dequant_node_map[var_name] = dequant_var_node @@ -2805,10 +3180,11 @@ def apply(self, graph): for var_name in var_names: if var_name in dequant_node_map: in_node = graph._find_node_by_name( - op_node.inputs, var_name) - graph.update_input_link(in_node, - dequant_node_map[var_name], - op_node) + op_node.inputs, var_name + ) + graph.update_input_link( + in_node, dequant_node_map[var_name], op_node + ) return graph @@ -2826,22 +3202,29 @@ def _insert_quant_dequant_op(self, graph, var_node): name="{}.quantized".format(var_name), var_type=var_node.type(), shape=var_node.shape(), - var_dtype=var_node.dtype()) - scale_var_node = graph._find_node_by_name(graph.all_persistable_nodes(), - self._scale_name(var_name)) + var_dtype=var_node.dtype(), + ) + scale_var_node = graph._find_node_by_name( + graph.all_persistable_nodes(), self._scale_name(var_name) + ) try: zero_point_node = graph._find_node_by_name( graph.all_persistable_nodes(), - "{}@zero_point".format(quant_var_node.name())) + "{}@zero_point".format(quant_var_node.name()), + ) except: zero_point_node = graph.create_persistable_node( name="{}@zero_point".format(quant_var_node.name()), var_type=core.VarDesc.VarType.LOD_TENSOR, shape=scale_var_node.shape(), - var_dtype=core.VarDesc.VarType.INT32) - _init_var_node(zero_point_node, - np.zeros(scale_var_node.shape(), dtype="int32"), - self._scope, self._place) + var_dtype=core.VarDesc.VarType.INT32, + ) + _init_var_node( + zero_point_node, + np.zeros(scale_var_node.shape(), dtype="int32"), + self._scope, + self._place, + ) inputs = {"X": var_node, "Scale": scale_var_node} if zero_point_node is not None: @@ -2851,10 +3234,12 @@ def _insert_quant_dequant_op(self, graph, var_node): attrs["op_role"] = core.op_proto_and_checker_maker.OpRole.Forward outputs = {"Y": quant_var_node} - quant_op_node = graph.create_op_node(op_type="quantize_linear", - attrs=attrs, - inputs=inputs, - outputs=outputs) + quant_op_node = graph.create_op_node( + op_type="quantize_linear", + attrs=attrs, + inputs=inputs, + outputs=outputs, + ) graph.link_to(var_node, quant_op_node) graph.link_to(scale_var_node, quant_op_node) @@ -2867,7 +3252,8 @@ def _insert_quant_dequant_op(self, graph, var_node): name="{}.dequantized".format(quant_var_node.name()), var_type=quant_var_node.type(), shape=quant_var_node.shape(), - var_dtype=quant_var_node.dtype()) + var_dtype=quant_var_node.dtype(), + ) inputs = {"X": quant_var_node, "Scale": scale_var_node} if zero_point_node is not None: @@ -2876,10 +3262,12 @@ def _insert_quant_dequant_op(self, graph, var_node): attrs = {"quant_axis": -1, "bit_length": self._quant_bits} attrs["op_role"] = core.op_proto_and_checker_maker.OpRole.Forward - dequant_op_node = graph.create_op_node(op_type="dequantize_linear", - attrs=attrs, - inputs=inputs, - outputs={"Y": dequant_var_node}) + dequant_op_node = graph.create_op_node( + op_type="dequantize_linear", + attrs=attrs, + inputs=inputs, + outputs={"Y": dequant_var_node}, + ) graph.link_to(quant_var_node, dequant_op_node) graph.link_to(scale_var_node, dequant_op_node) diff --git a/python/paddle/fluid/contrib/slim/quantization/quantize_transpiler_v2.py b/python/paddle/fluid/contrib/slim/quantization/quantize_transpiler_v2.py index ce20d2e695ad2..12f6f3c53d262 100644 --- a/python/paddle/fluid/contrib/slim/quantization/quantize_transpiler_v2.py +++ b/python/paddle/fluid/contrib/slim/quantization/quantize_transpiler_v2.py @@ -24,9 +24,9 @@ from ....initializer import Constant from ....log_helper import get_logger -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) def find_next_ops(block, var_name): @@ -45,24 +45,24 @@ def load_variable_data(scope, var_name): Load variable value from scope ''' var_node = scope.find_var(var_name) - assert var_node is not None, \ - "Cannot find " + var_name + " in scope." + assert var_node is not None, "Cannot find " + var_name + " in scope." return np.array(var_node.get_tensor()) class QuantizeTranspilerV2(object): - - def __init__(self, - weight_bits=8, - activation_bits=8, - weight_quantize_type='abs_max', - activation_quantize_type='moving_average_abs_max', - quantizable_op_type=[ - 'conv2d', - 'depthwise_conv2d', - 'mul', - ], - skip_pattern=['skip_quant']): + def __init__( + self, + weight_bits=8, + activation_bits=8, + weight_quantize_type='abs_max', + activation_quantize_type='moving_average_abs_max', + quantizable_op_type=[ + 'conv2d', + 'depthwise_conv2d', + 'mul', + ], + skip_pattern=['skip_quant'], + ): """ Apply fake quant for the quantized ops. @@ -81,18 +81,26 @@ def __init__(self, self._weight_bits = weight_bits self._activation_bits = activation_bits - assert activation_quantize_type in \ - ["abs_max", "moving_average_abs_max"], \ - "activation_quantize_type should be abs_max " \ + assert activation_quantize_type in [ + "abs_max", + "moving_average_abs_max", + ], ( + "activation_quantize_type should be abs_max " "or moving_average_abs_max for now." - assert weight_quantize_type in ["abs_max", "channel_wise_abs_max"], \ - "weight_quantize_type should be abs_max or channel_wise_abs_max." + ) + assert weight_quantize_type in [ + "abs_max", + "channel_wise_abs_max", + ], "weight_quantize_type should be abs_max or channel_wise_abs_max." self._activation_quantize_type = activation_quantize_type self._weight_quantize_type = weight_quantize_type for op_type in quantizable_op_type: - assert op_type in ['conv2d', 'depthwise_conv2d', 'mul'], \ - "Quantize op should be ['conv2d', 'depthwise_conv2d', 'mul']" + assert op_type in [ + 'conv2d', + 'depthwise_conv2d', + 'mul', + ], "Quantize op should be ['conv2d', 'depthwise_conv2d', 'mul']" self._quantizable_ops = quantizable_op_type self._quantizable_grad_ops = [ '%s_grad' % (op) for op in self._quantizable_ops @@ -115,10 +123,12 @@ def apply(self, program, startup_program, is_test=False): Returns: None """ - assert isinstance(program, Program), \ - "program must be the instance of Program" - assert isinstance(startup_program, Program), \ - "startup_program must be the instance of Program" + assert isinstance( + program, Program + ), "program must be the instance of Program" + assert isinstance( + startup_program, Program + ), "startup_program must be the instance of Program" var_rename_map = [ collections.OrderedDict() for _ in range(len(program.blocks)) @@ -127,16 +137,19 @@ def apply(self, program, startup_program, is_test=False): for block in program.blocks: ops = list(block.ops) for op in ops: - if op.type in self._quantizable_ops and \ - (not self._is_skip_quant(op)): - self._transform_forward(block, op, var_rename_map, - is_test) + if op.type in self._quantizable_ops and ( + not self._is_skip_quant(op) + ): + self._transform_forward( + block, op, var_rename_map, is_test + ) for block in program.blocks: ops = list(block.ops) for op in ops: - if op.type in self._quantizable_grad_ops and \ - (not self._is_skip_quant(op)): + if op.type in self._quantizable_grad_ops and ( + not self._is_skip_quant(op) + ): self._transform_backward(block, op, var_rename_map) def convert(self, test_program, scope=None): @@ -153,8 +166,10 @@ def convert(self, test_program, scope=None): for block in test_program.blocks: for op in block.ops: - if op.has_attr("quantization_type") \ - and op.attr("quantization_type") == "qat_with_weight": + if ( + op.has_attr("quantization_type") + and op.attr("quantization_type") == "qat_with_weight" + ): # quant op -> var1 -> fake op -> var2 assert len(op.output_arg_names) == 1 var1_name = op.output_arg_names[0] @@ -188,29 +203,40 @@ def _transform_forward(self, block, op, var_rename_map, is_test): else: in_var = block.var(in_name) target_dtype = [ - core.VarDesc.VarType.FP32, core.VarDesc.VarType.FP16 + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP16, ] if in_var.dtype not in target_dtype: continue - quant_bits = self._weight_bits if in_var.persistable \ - else self._activation_bits - quant_type = self._weight_quantize_type if in_var.persistable \ - else self._activation_quantize_type + quant_bits = ( + self._weight_bits + if in_var.persistable + else self._activation_bits + ) + quant_type = ( + self._weight_quantize_type + if in_var.persistable + else self._activation_quantize_type + ) if quant_type == "abs_max": new_var = self._insert_abs_max_fq_op( - block, idx, in_var, quant_bits) + block, idx, in_var, quant_bits + ) elif quant_type == "moving_average_abs_max": new_var = self._insert_ma_abs_max_fq_op( - block, idx, in_var, quant_bits, is_test) + block, idx, in_var, quant_bits, is_test + ) elif quant_type == "channel_wise_abs_max": ch_axis = 1 if op.type in self._out_ch_axis1_ops else 0 new_var = self._insert_pc_abs_max_fq_op( - block, idx, in_var, quant_bits, ch_axis) + block, idx, in_var, quant_bits, ch_axis + ) else: - _logger.error("Don't support the quant_type: %s" % - quant_type) + _logger.error( + "Don't support the quant_type: %s" % quant_type + ) continue new_in_name = new_var.name @@ -225,7 +251,8 @@ def _transform_forward(self, block, op, var_rename_map, is_test): idx = block.ops.index(op) out_var = block.var(out_name) new_out_var = self._insert_ma_abs_max_scale_op( - block, idx + 1, out_var, is_test, True) + block, idx + 1, out_var, is_test, True + ) for next_op in next_ops: if "_grad" not in next_op.type: @@ -237,13 +264,15 @@ def _is_skip_quant(self, op): """ user_skipped = False if isinstance(self._skip_pattern, list): - user_skipped = op.has_attr("op_namescope") and \ - any(pattern in op.attr("op_namescope") \ - for pattern in self._skip_pattern) + user_skipped = op.has_attr("op_namescope") and any( + pattern in op.attr("op_namescope") + for pattern in self._skip_pattern + ) elif isinstance(self._skip_pattern, str): - user_skipped = op.has_attr("op_namescope") and \ - op.attr("op_namescope").find( - self._skip_pattern) != -1 + user_skipped = ( + op.has_attr("op_namescope") + and op.attr("op_namescope").find(self._skip_pattern) != -1 + ) return user_skipped def _transform_backward(self, block, op, var_rename_map): @@ -259,75 +288,92 @@ def _transform_backward(self, block, op, var_rename_map): op._rename_input(name, new_var_name) no_dequanted_input_vars = False if no_dequanted_input_vars: - raise ValueError("There is no dequanted inputs for op %s." % - (op.type)) + raise ValueError( + "There is no dequanted inputs for op %s." % (op.type) + ) def _insert_abs_max_fq_op(self, block, idx, in_var, quant_bits): """ Inset abs max fake quant op. """ - quant_dequant_var = block.create_var(type=in_var.type, - name="{}.quant_dequant".format( - in_var.name), - shape=in_var.shape, - dtype=in_var.dtype) - scale_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.quant_dequant.scale".format(in_var.name), - initializer=Constant(0.), - trainable=False), - shape=[1], - dtype=in_var.dtype) + quant_dequant_var = block.create_var( + type=in_var.type, + name="{}.quant_dequant".format(in_var.name), + shape=in_var.shape, + dtype=in_var.dtype, + ) + scale_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.quant_dequant.scale".format(in_var.name), + initializer=Constant(0.0), + trainable=False, + ), + shape=[1], + dtype=in_var.dtype, + ) scale_var.stop_gradient = True inputs = {'X': in_var} outputs = {'Out': quant_dequant_var, 'OutScale': scale_var} attrs = {'bit_length': quant_bits} - block._insert_op(idx, - type='fake_quantize_dequantize_abs_max', - attrs=attrs, - inputs=inputs, - outputs=outputs) + block._insert_op( + idx, + type='fake_quantize_dequantize_abs_max', + attrs=attrs, + inputs=inputs, + outputs=outputs, + ) return quant_dequant_var def _insert_ma_abs_max_fq_op(self, block, idx, in_var, quant_bits, is_test): """ Insert moving average abs max fake quant op. """ - quant_dequant_var = block.create_var(type=in_var.type, - name="{}.quant_dequant".format( - in_var.name), - shape=in_var.shape, - dtype=in_var.dtype) - - scale_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.quant_dequant.scale".format(in_var.name), - initializer=Constant(0.), - trainable=False), - shape=[1], - dtype=in_var.dtype) + quant_dequant_var = block.create_var( + type=in_var.type, + name="{}.quant_dequant".format(in_var.name), + shape=in_var.shape, + dtype=in_var.dtype, + ) + + scale_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.quant_dequant.scale".format(in_var.name), + initializer=Constant(0.0), + trainable=False, + ), + shape=[1], + dtype=in_var.dtype, + ) scale_var.stop_gradient = True if not is_test: - state_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.quant_dequant.state".format(in_var.name), - initializer=Constant(0), - trainable=False), - shape=[1], - dtype=in_var.dtype) + state_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.quant_dequant.state".format(in_var.name), + initializer=Constant(0), + trainable=False, + ), + shape=[1], + dtype=in_var.dtype, + ) state_var.stop_gradient = True - accum_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.quant_dequant.accum".format(in_var.name), - initializer=Constant(0), - trainable=False), - shape=[1], - dtype=in_var.dtype) + accum_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.quant_dequant.accum".format(in_var.name), + initializer=Constant(0), + trainable=False, + ), + shape=[1], + dtype=in_var.dtype, + ) accum_var.stop_gradient = True attrs = { 'moving_rate': self._moving_rate, 'bit_length': quant_bits, - 'is_test': is_test + 'is_test': is_test, } inputs = {'X': in_var, 'InScale': scale_var} outputs = {'Out': quant_dequant_var, 'OutScale': scale_var} @@ -337,56 +383,64 @@ def _insert_ma_abs_max_fq_op(self, block, idx, in_var, quant_bits, is_test): outputs['OutState'] = state_var outputs['OutAccum'] = accum_var - block._insert_op(idx, - type='fake_quantize_dequantize_moving_average_abs_max', - attrs=attrs, - inputs=inputs, - outputs=outputs) + block._insert_op( + idx, + type='fake_quantize_dequantize_moving_average_abs_max', + attrs=attrs, + inputs=inputs, + outputs=outputs, + ) return quant_dequant_var def _insert_pc_abs_max_fq_op(self, block, idx, in_var, quant_bits, ch_axis): """ Insert per channel abs max fake quant op. """ - quant_dequant_var = block.create_var(type=in_var.type, - name="{}.quant_dequant".format( - in_var.name), - shape=in_var.shape, - dtype=in_var.dtype) - - scale_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.quant_dequant.scale".format(in_var.name), - initializer=Constant(0.), - trainable=False), - shape=[in_var.shape[ch_axis]], - dtype=in_var.dtype) + quant_dequant_var = block.create_var( + type=in_var.type, + name="{}.quant_dequant".format(in_var.name), + shape=in_var.shape, + dtype=in_var.dtype, + ) + + scale_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.quant_dequant.scale".format(in_var.name), + initializer=Constant(0.0), + trainable=False, + ), + shape=[in_var.shape[ch_axis]], + dtype=in_var.dtype, + ) scale_var.stop_gradient = True inputs = {'X': in_var} outputs = {'Out': quant_dequant_var, 'OutScale': scale_var} attrs = {'bit_length': quant_bits, 'quant_axis': ch_axis} - block._insert_op(idx, - type='fake_channel_wise_quantize_dequantize_abs_max', - attrs=attrs, - inputs=inputs, - outputs=outputs) + block._insert_op( + idx, + type='fake_channel_wise_quantize_dequantize_abs_max', + attrs=attrs, + inputs=inputs, + outputs=outputs, + ) return quant_dequant_var - def _insert_ma_abs_max_scale_op(self, - block, - idx, - in_var, - is_test, - has_out_var=False): + def _insert_ma_abs_max_scale_op( + self, block, idx, in_var, is_test, has_out_var=False + ): """ Insert moving average abs max scale op. """ - scale_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.outscale.scale".format(in_var.name), - initializer=Constant(0.), - trainable=False), - shape=[1], - dtype=in_var.dtype) + scale_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.outscale.scale".format(in_var.name), + initializer=Constant(0.0), + trainable=False, + ), + shape=[1], + dtype=in_var.dtype, + ) scale_var.stop_gradient = True attrs = {'moving_rate': self._moving_rate, 'is_test': is_test} @@ -394,20 +448,26 @@ def _insert_ma_abs_max_scale_op(self, outputs = {'OutScale': scale_var} if not is_test: - state_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.outscale.state".format(in_var.name), - initializer=Constant(0), - trainable=False), - shape=[1], - dtype=in_var.dtype) + state_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.outscale.state".format(in_var.name), + initializer=Constant(0), + trainable=False, + ), + shape=[1], + dtype=in_var.dtype, + ) state_var.stop_gradient = True - accum_var = self._helper.create_parameter(attr=ParamAttr( - name="{}.outscale.accum".format(in_var.name), - initializer=Constant(0), - trainable=False), - shape=[1], - dtype=in_var.dtype) + accum_var = self._helper.create_parameter( + attr=ParamAttr( + name="{}.outscale.accum".format(in_var.name), + initializer=Constant(0), + trainable=False, + ), + shape=[1], + dtype=in_var.dtype, + ) accum_var.stop_gradient = True inputs['InState'] = state_var @@ -416,18 +476,22 @@ def _insert_ma_abs_max_scale_op(self, outputs['OutAccum'] = accum_var if has_out_var: - out_var = block.create_var(type=in_var.type, - name="{}.tmp".format(in_var.name), - shape=in_var.shape, - dtype=in_var.dtype) + out_var = block.create_var( + type=in_var.type, + name="{}.tmp".format(in_var.name), + shape=in_var.shape, + dtype=in_var.dtype, + ) outputs['Out'] = out_var - block._insert_op(idx, - type='moving_average_abs_max_scale', - attrs=attrs, - inputs=inputs, - outputs=outputs) + block._insert_op( + idx, + type='moving_average_abs_max_scale', + attrs=attrs, + inputs=inputs, + outputs=outputs, + ) if has_out_var: return out_var diff --git a/python/paddle/fluid/contrib/slim/quantization/utils.py b/python/paddle/fluid/contrib/slim/quantization/utils.py index 158f7e07a0d91..11e39116389c2 100644 --- a/python/paddle/fluid/contrib/slim/quantization/utils.py +++ b/python/paddle/fluid/contrib/slim/quantization/utils.py @@ -18,8 +18,12 @@ from ....framework import Operator _weight_supported_quantizable_op_type = [ - 'conv2d', 'depthwise_conv2d', 'conv2d_transpose', 'mul', 'matmul', - 'matmul_v2' + 'conv2d', + 'depthwise_conv2d', + 'conv2d_transpose', + 'mul', + 'matmul', + 'matmul_v2', ] _act_supported_quantizable_op_type = [ @@ -114,13 +118,19 @@ ] QUANT_SUPPORTED_OP_TYPE_LIST = list( - set(_weight_supported_quantizable_op_type + - _act_supported_quantizable_op_type)) + set( + _weight_supported_quantizable_op_type + + _act_supported_quantizable_op_type + ) +) _out_scale_op_list = QUANT_SUPPORTED_OP_TYPE_LIST _channelwise_quant_axis1_ops = [ - 'conv2d_transpose', 'mul', 'matmul', 'matmul_v2' + 'conv2d_transpose', + 'mul', + 'matmul', + 'matmul_v2', ] # list op real input and output names, to avoid processing input such as AxisTensor. @@ -230,11 +240,11 @@ def _get_op_input_var_names(op): Returns: input_var_names or None. """ - assert isinstance(op, (IrNode, Operator)), \ - "The input op should be IrNode or Operator." + assert isinstance( + op, (IrNode, Operator) + ), "The input op should be IrNode or Operator." var_names = [] - op_name = op.name() if isinstance(op, IrNode) \ - else op.type + op_name = op.name() if isinstance(op, IrNode) else op.type if op_name not in _op_real_in_out_name: return [] @@ -250,11 +260,11 @@ def _get_op_input_var_names(op): def _get_op_output_var_names(op): """ """ - assert isinstance(op, (IrNode, Operator)), \ - "The input op should be IrNode or Operator." + assert isinstance( + op, (IrNode, Operator) + ), "The input op should be IrNode or Operator." var_names = [] - op_name = op.name() if isinstance(op, IrNode) \ - else op.type + op_name = op.name() if isinstance(op, IrNode) else op.type if op_name not in _op_real_in_out_name: return [] @@ -270,10 +280,10 @@ def _get_op_output_var_names(op): def _get_input_name_index(op, input_var_name): """Get the input name and index of the var_name in the op""" - assert isinstance(op, (IrNode, Operator)), \ - "The input op should be IrNode or Operator." - op_name = op.name() if isinstance(op, IrNode) \ - else op.type + assert isinstance( + op, (IrNode, Operator) + ), "The input op should be IrNode or Operator." + op_name = op.name() if isinstance(op, IrNode) else op.type if op_name not in _op_real_in_out_name: return None @@ -288,10 +298,10 @@ def _get_input_name_index(op, input_var_name): def _get_output_name_index(op, output_var_name): """Get the output name and index of the var_name in the op""" - assert isinstance(op, (IrNode, Operator)), \ - "The input op should be IrNode or Operator." - op_name = op.name() if isinstance(op, IrNode) \ - else op.type + assert isinstance( + op, (IrNode, Operator) + ), "The input op should be IrNode or Operator." + op_name = op.name() if isinstance(op, IrNode) else op.type if op_name not in _op_real_in_out_name: return None @@ -310,8 +320,7 @@ def load_variable_data(scope, var_name): Load variable value from scope ''' var_node = scope.find_var(var_name) - assert var_node is not None, \ - "Cannot find " + var_name + " in scope." + assert var_node is not None, "Cannot find " + var_name + " in scope." return np.array(var_node.get_tensor()) @@ -319,8 +328,9 @@ def set_variable_data(scope, place, var_name, np_value): ''' Set the value of var node by name, if the node exits, ''' - assert isinstance(np_value, np.ndarray), \ - 'The type of value should be numpy array.' + assert isinstance( + np_value, np.ndarray + ), 'The type of value should be numpy array.' var_node = scope.find_var(var_name) if var_node != None: tensor = var_node.get_tensor() @@ -405,10 +415,12 @@ def bias_correction_w(x, x_quant, scale_v, quant_axis, weight_bits=8): x_dequant[:, i] = x_quant[:, i] * s / bnt quant_bias = x - x_dequant mean_bias = np.array( - [quant_bias[:, i].mean() for i in range(quant_bias.shape[1])]) + [quant_bias[:, i].mean() for i in range(quant_bias.shape[1])] + ) std_orig = np.array([x[:, i].std() for i in range(x.shape[1])]) std_quant = np.array( - [x_dequant[:, i].std() for i in range(x_dequant.shape[1])]) + [x_dequant[:, i].std() for i in range(x_dequant.shape[1])] + ) std_bias = std_orig / (std_quant + eps) else: x_dequant = x_quant * scale_v / bnt @@ -419,8 +431,9 @@ def bias_correction_w(x, x_quant, scale_v, quant_axis, weight_bits=8): mean_bias = np.resize(mean_bias, x.shape) x_dequant = (mean_bias + x_dequant) * std_bias - quantized_param_v = quant_tensor(x_dequant, scale_v, quant_axis, - weight_bits) + quantized_param_v = quant_tensor( + x_dequant, scale_v, quant_axis, weight_bits + ) return quantized_param_v @@ -430,8 +443,10 @@ def stable_sigmoid(x): def calculate_quant_cos_error(orig_tensor, qdq_tensor): - cos_sim = np.inner(orig_tensor.flatten(), qdq_tensor.flatten()) \ - / (np.linalg.norm(orig_tensor.flatten()) * np.linalg.norm(qdq_tensor.flatten())) + cos_sim = np.inner(orig_tensor.flatten(), qdq_tensor.flatten()) / ( + np.linalg.norm(orig_tensor.flatten()) + * np.linalg.norm(qdq_tensor.flatten()) + ) return cos_sim @@ -454,11 +469,10 @@ def move_persistable_var_to_global_block(program): def l2_loss(gt, pred): - return ((gt - pred)**2).mean() + return ((gt - pred) ** 2).mean() class tqdm(object): - def __init__(self, total, bar_format='Loading|{bar}', ncols=80): self.total = total self.bar_format = bar_format @@ -470,8 +484,9 @@ def update(self, n=1): a = "=" * round((self.n / self.total) * self.ncols) b = " " * (self.ncols - len(a)) prefix = self.bar_format.split('|')[0] - sys.stderr.write("\r{}|{}=>{}| {}/{}".format(prefix, a, b, self.n, - self.total)) + sys.stderr.write( + "\r{}|{}=>{}| {}/{}".format(prefix, a, b, self.n, self.total) + ) sys.stderr.flush() def __enter__(self): diff --git a/python/paddle/fluid/contrib/slim/tests/convert_model2dot.py b/python/paddle/fluid/contrib/slim/tests/convert_model2dot.py index 3573f53e22db7..7bb7de706bcac 100644 --- a/python/paddle/fluid/contrib/slim/tests/convert_model2dot.py +++ b/python/paddle/fluid/contrib/slim/tests/convert_model2dot.py @@ -26,20 +26,21 @@ def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('--model_path', - type=str, - default='', - help='A path to a model.') - parser.add_argument('--save_graph_dir', - type=str, - default='', - help='A path to save the graph.') + parser.add_argument( + '--model_path', type=str, default='', help='A path to a model.' + ) + parser.add_argument( + '--save_graph_dir', + type=str, + default='', + help='A path to save the graph.', + ) parser.add_argument( '--save_graph_name', type=str, default='', - help= - 'A name to save the graph. Default - name from model path will be used') + help='A name to save the graph. Default - name from model path will be used', + ) test_args, args = parser.parse_known_args(namespace=unittest) return test_args, sys.argv[:1] + args @@ -51,12 +52,19 @@ def generate_dot_for_model(model_path, save_graph_dir, save_graph_name): inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if os.path.exists(os.path.join(model_path, '__model__')): - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(model_path, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) else: - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(model_path, exe, 'model', - 'params') + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + model_path, exe, 'model', 'params' + ) graph = IrGraph(core.Graph(inference_program.desc), for_test=True) if not os.path.exists(save_graph_dir): os.makedirs(save_graph_dir) @@ -65,12 +73,17 @@ def generate_dot_for_model(model_path, save_graph_dir, save_graph_name): save_graph_name = model_name graph.draw(save_graph_dir, save_graph_name, graph.all_op_nodes()) print( - "Success! Generated dot and pdf files for {0} model, that can be found at {1} named {2}.\n" - .format(model_name, save_graph_dir, save_graph_name)) + "Success! Generated dot and pdf files for {0} model, that can be found at {1} named {2}.\n".format( + model_name, save_graph_dir, save_graph_name + ) + ) if __name__ == '__main__': global test_args test_args, remaining_args = parse_args() - generate_dot_for_model(test_args.model_path, test_args.save_graph_dir, - test_args.save_graph_name) + generate_dot_for_model( + test_args.model_path, + test_args.save_graph_dir, + test_args.save_graph_name, + ) diff --git a/python/paddle/fluid/contrib/slim/tests/imperative_test_utils.py b/python/paddle/fluid/contrib/slim/tests/imperative_test_utils.py index d69241d6cb982..43ace1b77000d 100644 --- a/python/paddle/fluid/contrib/slim/tests/imperative_test_utils.py +++ b/python/paddle/fluid/contrib/slim/tests/imperative_test_utils.py @@ -24,9 +24,9 @@ from paddle.fluid.log_helper import get_logger -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) def fix_model_dict(model): @@ -37,16 +37,18 @@ def fix_model_dict(model): if name.endswith("bias"): value = np.zeros_like(p_value).astype('float32') else: - value = np.random.normal( - loc=0.0, scale=0.01, - size=np.product(p_shape)).reshape(p_shape).astype('float32') + value = ( + np.random.normal(loc=0.0, scale=0.01, size=np.product(p_shape)) + .reshape(p_shape) + .astype('float32') + ) fixed_state[name] = value model.set_dict(fixed_state) return model def pre_hook(layer, input): - input_return = (input[0] * 2) + input_return = input[0] * 2 return input_return @@ -59,8 +61,9 @@ def train_lenet(lenet, reader, optimizer): lenet.train() for batch_id, data in enumerate(reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') + x_data = np.array([x[0].reshape(1, 28, 28) for x in data]).astype( + 'float32' + ) y_data = np.array([x[1] for x in data]).astype('int64').reshape(-1, 1) img = paddle.to_tensor(x_data) @@ -82,7 +85,6 @@ def train_lenet(lenet, reader, optimizer): class ImperativeLenet(fluid.dygraph.Layer): - def __init__(self, num_classes=10): super(ImperativeLenet, self).__init__() conv2d_w1_attr = fluid.ParamAttr(name="conv2d_w_1") @@ -95,36 +97,55 @@ def __init__(self, num_classes=10): fc_b2_attr = fluid.ParamAttr(name="fc_b_2") fc_b3_attr = fluid.ParamAttr(name="fc_b_3") self.features = Sequential( - Conv2D(in_channels=1, - out_channels=6, - kernel_size=3, - stride=1, - padding=1, - weight_attr=conv2d_w1_attr, - bias_attr=False), BatchNorm2D(6), ReLU(), + Conv2D( + in_channels=1, + out_channels=6, + kernel_size=3, + stride=1, + padding=1, + weight_attr=conv2d_w1_attr, + bias_attr=False, + ), + BatchNorm2D(6), + ReLU(), + MaxPool2D(kernel_size=2, stride=2), + Conv2D( + in_channels=6, + out_channels=16, + kernel_size=5, + stride=1, + padding=0, + weight_attr=conv2d_w2_attr, + bias_attr=conv2d_b2_attr, + ), + BatchNorm2D(16), + PReLU(), MaxPool2D(kernel_size=2, stride=2), - Conv2D(in_channels=6, - out_channels=16, - kernel_size=5, - stride=1, - padding=0, - weight_attr=conv2d_w2_attr, - bias_attr=conv2d_b2_attr), BatchNorm2D(16), PReLU(), - MaxPool2D(kernel_size=2, stride=2)) + ) self.fc = Sequential( - Linear(in_features=400, - out_features=120, - weight_attr=fc_w1_attr, - bias_attr=fc_b1_attr), LeakyReLU(), - Linear(in_features=120, - out_features=84, - weight_attr=fc_w2_attr, - bias_attr=fc_b2_attr), Sigmoid(), - Linear(in_features=84, - out_features=num_classes, - weight_attr=fc_w3_attr, - bias_attr=fc_b3_attr), Softmax()) + Linear( + in_features=400, + out_features=120, + weight_attr=fc_w1_attr, + bias_attr=fc_b1_attr, + ), + LeakyReLU(), + Linear( + in_features=120, + out_features=84, + weight_attr=fc_w2_attr, + bias_attr=fc_b2_attr, + ), + Sigmoid(), + Linear( + in_features=84, + out_features=num_classes, + weight_attr=fc_w3_attr, + bias_attr=fc_b3_attr, + ), + Softmax(), + ) self.add = paddle.nn.quant.add() self.quant_stub = paddle.nn.quant.QuantStub() @@ -139,7 +160,6 @@ def forward(self, inputs): class ImperativeLenetWithSkipQuant(fluid.dygraph.Layer): - def __init__(self, num_classes=10): super(ImperativeLenetWithSkipQuant, self).__init__() @@ -153,48 +173,58 @@ def __init__(self, num_classes=10): fc_b1_attr = fluid.ParamAttr(name="fc_b_1") fc_b2_attr = fluid.ParamAttr(name="fc_b_2") fc_b3_attr = fluid.ParamAttr(name="fc_b_3") - self.conv2d_0 = Conv2D(in_channels=1, - out_channels=6, - kernel_size=3, - stride=1, - padding=1, - weight_attr=conv2d_w1_attr, - bias_attr=conv2d_b1_attr) + self.conv2d_0 = Conv2D( + in_channels=1, + out_channels=6, + kernel_size=3, + stride=1, + padding=1, + weight_attr=conv2d_w1_attr, + bias_attr=conv2d_b1_attr, + ) self.conv2d_0.skip_quant = True self.batch_norm_0 = BatchNorm2D(6) self.relu_0 = ReLU() self.pool2d_0 = MaxPool2D(kernel_size=2, stride=2) - self.conv2d_1 = Conv2D(in_channels=6, - out_channels=16, - kernel_size=5, - stride=1, - padding=0, - weight_attr=conv2d_w2_attr, - bias_attr=conv2d_b2_attr) + self.conv2d_1 = Conv2D( + in_channels=6, + out_channels=16, + kernel_size=5, + stride=1, + padding=0, + weight_attr=conv2d_w2_attr, + bias_attr=conv2d_b2_attr, + ) self.conv2d_1.skip_quant = False self.batch_norm_1 = BatchNorm2D(16) self.relu6_0 = ReLU6() self.pool2d_1 = MaxPool2D(kernel_size=2, stride=2) - self.linear_0 = Linear(in_features=400, - out_features=120, - weight_attr=fc_w1_attr, - bias_attr=fc_b1_attr) + self.linear_0 = Linear( + in_features=400, + out_features=120, + weight_attr=fc_w1_attr, + bias_attr=fc_b1_attr, + ) self.linear_0.skip_quant = True self.leaky_relu_0 = LeakyReLU() - self.linear_1 = Linear(in_features=120, - out_features=84, - weight_attr=fc_w2_attr, - bias_attr=fc_b2_attr) + self.linear_1 = Linear( + in_features=120, + out_features=84, + weight_attr=fc_w2_attr, + bias_attr=fc_b2_attr, + ) self.linear_1.skip_quant = False self.sigmoid_0 = Sigmoid() - self.linear_2 = Linear(in_features=84, - out_features=num_classes, - weight_attr=fc_w3_attr, - bias_attr=fc_b3_attr) + self.linear_2 = Linear( + in_features=84, + out_features=num_classes, + weight_attr=fc_w3_attr, + bias_attr=fc_b3_attr, + ) self.linear_2.skip_quant = False self.softmax_0 = Softmax() @@ -221,24 +251,28 @@ def forward(self, inputs): class ImperativeLinearBn(fluid.dygraph.Layer): - def __init__(self): super(ImperativeLinearBn, self).__init__() fc_w_attr = paddle.ParamAttr( name="fc_weight", - initializer=paddle.nn.initializer.Constant(value=0.5)) + initializer=paddle.nn.initializer.Constant(value=0.5), + ) fc_b_attr = paddle.ParamAttr( name="fc_bias", - initializer=paddle.nn.initializer.Constant(value=1.0)) + initializer=paddle.nn.initializer.Constant(value=1.0), + ) bn_w_attr = paddle.ParamAttr( name="bn_weight", - initializer=paddle.nn.initializer.Constant(value=0.5)) - - self.linear = Linear(in_features=10, - out_features=10, - weight_attr=fc_w_attr, - bias_attr=fc_b_attr) + initializer=paddle.nn.initializer.Constant(value=0.5), + ) + + self.linear = Linear( + in_features=10, + out_features=10, + weight_attr=fc_w_attr, + bias_attr=fc_b_attr, + ) self.bn = BatchNorm1D(10, weight_attr=bn_w_attr) def forward(self, inputs): @@ -249,17 +283,17 @@ def forward(self, inputs): class ImperativeLinearBn_hook(fluid.dygraph.Layer): - def __init__(self): super(ImperativeLinearBn_hook, self).__init__() fc_w_attr = paddle.ParamAttr( name="linear_weight", - initializer=paddle.nn.initializer.Constant(value=0.5)) + initializer=paddle.nn.initializer.Constant(value=0.5), + ) - self.linear = Linear(in_features=10, - out_features=10, - weight_attr=fc_w_attr) + self.linear = Linear( + in_features=10, out_features=10, weight_attr=fc_w_attr + ) self.bn = BatchNorm1D(10) forward_pre = self.linear.register_forward_pre_hook(pre_hook) diff --git a/python/paddle/fluid/contrib/slim/tests/quant2_int8_image_classification_comparison.py b/python/paddle/fluid/contrib/slim/tests/quant2_int8_image_classification_comparison.py index c3d7f1a254eea..dcd5d6de31331 100644 --- a/python/paddle/fluid/contrib/slim/tests/quant2_int8_image_classification_comparison.py +++ b/python/paddle/fluid/contrib/slim/tests/quant2_int8_image_classification_comparison.py @@ -40,50 +40,50 @@ def parse_args(): '--skip_batch_num', type=int, default=0, - help='Number of the first minibatches to skip in performance statistics.' + help='Number of the first minibatches to skip in performance statistics.', + ) + parser.add_argument( + '--quant_model', type=str, default='', help='A path to a Quant model.' + ) + parser.add_argument( + '--fp32_model', type=str, default='', help='A path to an FP32 model.' ) - parser.add_argument('--quant_model', - type=str, - default='', - help='A path to a Quant model.') - parser.add_argument('--fp32_model', - type=str, - default='', - help='A path to an FP32 model.') parser.add_argument('--infer_data', type=str, default='', help='Data file.') parser.add_argument( '--batch_num', type=int, default=0, - help= - 'Number of batches to process. 0 or less means whole dataset. Default: 0.' + help='Number of batches to process. 0 or less means whole dataset. Default: 0.', + ) + parser.add_argument( + '--acc_diff_threshold', + type=float, + default=0.01, + help='Accepted accuracy difference threshold.', ) - parser.add_argument('--acc_diff_threshold', - type=float, - default=0.01, - help='Accepted accuracy difference threshold.') parser.add_argument( '--ops_to_quantize', type=str, default='', - help= - 'A comma separated list of operators to quantize. Only quantizable operators are taken into account. If the option is not used, an attempt to quantize all quantizable operators will be made.' + help='A comma separated list of operators to quantize. Only quantizable operators are taken into account. If the option is not used, an attempt to quantize all quantizable operators will be made.', ) parser.add_argument( '--op_ids_to_skip', type=str, default='', - help='A comma separated list of operator ids to skip in quantization.') + help='A comma separated list of operator ids to skip in quantization.', + ) parser.add_argument( '--targets', type=str, default='quant,int8,fp32', - help= - 'A comma separated list of inference types to run ("int8", "fp32", "quant"). Default: "quant,int8,fp32"' + help='A comma separated list of inference types to run ("int8", "fp32", "quant"). Default: "quant,int8,fp32"', + ) + parser.add_argument( + '--debug', + action='store_true', + help='If used, the graph of Quant model is drawn.', ) - parser.add_argument('--debug', - action='store_true', - help='If used, the graph of Quant model is drawn.') test_args, args = parser.parse_known_args(namespace=unittest) return test_args, sys.argv[:1] + args @@ -95,7 +95,6 @@ class Quant2Int8ImageClassificationComparisonTest(unittest.TestCase): """ def _reader_creator(self, data_file='data.bin'): - def reader(): with open(data_file, 'rb') as fp: num = fp.read(8) @@ -114,7 +113,8 @@ def reader(): fp.seek(imgs_offset + img_size * step) img = fp.read(img_size) img = struct.unpack_from( - '{}f'.format(img_ch * img_w * img_h), img) + '{}f'.format(img_ch * img_w * img_h), img + ) img = np.array(img) img.shape = (img_ch, img_w, img_h) fp.seek(labels_offset + label_size * step) @@ -148,14 +148,14 @@ def _prepare_for_fp32_mkldnn(self, graph): name = op_node.name() if name in ['depthwise_conv2d']: input_var_node = graph._find_node_by_name( - op_node.inputs, - op_node.input("Input")[0]) + op_node.inputs, op_node.input("Input")[0] + ) weight_var_node = graph._find_node_by_name( - op_node.inputs, - op_node.input("Filter")[0]) + op_node.inputs, op_node.input("Filter")[0] + ) output_var_node = graph._find_node_by_name( - graph.all_var_nodes(), - op_node.output("Output")[0]) + graph.all_var_nodes(), op_node.output("Output")[0] + ) attrs = { name: op_node.op().attr(name) for name in op_node.op().attr_names() @@ -164,11 +164,9 @@ def _prepare_for_fp32_mkldnn(self, graph): conv_op_node = graph.create_op_node( op_type='conv2d', attrs=attrs, - inputs={ - 'Input': input_var_node, - 'Filter': weight_var_node - }, - outputs={'Output': output_var_node}) + inputs={'Input': input_var_node, 'Filter': weight_var_node}, + outputs={'Output': output_var_node}, + ) graph.link_to(input_var_node, conv_op_node) graph.link_to(weight_var_node, conv_op_node) @@ -177,28 +175,37 @@ def _prepare_for_fp32_mkldnn(self, graph): return graph - def _predict(self, - test_reader=None, - model_path=None, - batch_size=1, - batch_num=1, - skip_batch_num=0, - target='quant'): + def _predict( + self, + test_reader=None, + model_path=None, + batch_size=1, + batch_num=1, + skip_batch_num=0, + target='quant', + ): assert target in ['quant', 'int8', 'fp32'] place = fluid.CPUPlace() exe = fluid.Executor(place) inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if os.path.exists(os.path.join(model_path, '__model__')): - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(model_path, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) else: - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(model_path, exe, 'model', - 'params') + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + model_path, exe, 'model', 'params' + ) graph = IrGraph(core.Graph(inference_program.desc), for_test=True) - if (self._debug): + if self._debug: graph.draw('.', 'quant_orig', graph.all_op_nodes()) quant_transform_pass = Quant2Int8MkldnnPass( self._quantized_ops, @@ -206,10 +213,11 @@ def _predict(self, _scope=inference_scope, _place=place, _core=core, - _debug=self._debug) - if (target == 'quant'): + _debug=self._debug, + ) + if target == 'quant': graph = self._prepare_for_fp32_mkldnn(graph) - elif (target == 'int8'): + elif target == 'int8': graph = quant_transform_pass.apply(graph) else: # target == fp32 graph = quant_transform_pass.prepare_and_optimize_fp32(graph) @@ -238,30 +246,35 @@ def _predict(self, images = np.array(images).astype('float32') labels = np.array([x[1] for x in data]).astype('int64') - if (target == 'fp32'): + if target == 'fp32': # FP32 models have accuracy measuring layers labels = labels.reshape([-1, 1]) start = time.time() - out = exe.run(inference_program, - feed={ - feed_target_names[0]: images, - feed_target_names[1]: labels - }, - fetch_list=fetch_targets) + out = exe.run( + inference_program, + feed={ + feed_target_names[0]: images, + feed_target_names[1]: labels, + }, + fetch_list=fetch_targets, + ) batch_time = (time.time() - start) * 1000 # in miliseconds batch_acc1, batch_acc5 = out[1][0], out[2][0] outputs.append(batch_acc1) else: # Quant INT8 models do not have accuracy measuring layers start = time.time() - out = exe.run(inference_program, - feed={feed_target_names[0]: images}, - fetch_list=fetch_targets) + out = exe.run( + inference_program, + feed={feed_target_names[0]: images}, + fetch_list=fetch_targets, + ) batch_time = (time.time() - start) * 1000 # in miliseconds outputs.append(out[0]) # Calculate accuracy result batch_acc1, batch_acc5 = self._get_batch_accuracy( - out[0], labels) + out[0], labels + ) infer_accs1.append(batch_acc1) infer_accs5.append(batch_acc5) @@ -272,10 +285,17 @@ def _predict(self, fpses.append(fps) iters += 1 appx = ' (warm-up)' if iters <= skip_batch_num else '' - _logger.info('batch {0}{5}, acc1: {1:.4f}, acc5: {2:.4f}, ' - 'latency: {3:.4f} ms, fps: {4:.2f}'.format( - iters, batch_acc1, batch_acc5, - batch_time / batch_size, fps, appx)) + _logger.info( + 'batch {0}{5}, acc1: {1:.4f}, acc5: {2:.4f}, ' + 'latency: {3:.4f} ms, fps: {4:.2f}'.format( + iters, + batch_acc1, + batch_acc5, + batch_time / batch_size, + fps, + appx, + ) + ) # Postprocess benchmark data batch_latencies = batch_times[skip_batch_num:] @@ -287,18 +307,24 @@ def _predict(self, acc1_avg = np.mean(infer_accs1) acc5_avg = np.mean(infer_accs5) _logger.info( - 'Total inference run time: {:.2f} s'.format(infer_total_time)) + 'Total inference run time: {:.2f} s'.format(infer_total_time) + ) return outputs, acc1_avg, acc5_avg, fps_avg, latency_avg def _print_performance(self, title, fps, lat): - _logger.info('{0}: avg fps: {1:.2f}, avg latency: {2:.4f} ms'.format( - title, fps, lat)) + _logger.info( + '{0}: avg fps: {1:.2f}, avg latency: {2:.4f} ms'.format( + title, fps, lat + ) + ) def _print_accuracy(self, title, acc1, acc5): _logger.info( - '{0}: avg top1 accuracy: {1:.4f}, avg top5 accuracy: {2:.4f}'. - format(title, acc1, acc5)) + '{0}: avg top1 accuracy: {1:.4f}, avg top5 accuracy: {2:.4f}'.format( + title, acc1, acc5 + ) + ) def _summarize_performance(self, int8_fps, int8_lat, fp32_fps, fp32_lat): _logger.info('--- Performance summary ---') @@ -306,8 +332,9 @@ def _summarize_performance(self, int8_fps, int8_lat, fp32_fps, fp32_lat): if fp32_lat >= 0: self._print_performance('FP32', fp32_fps, fp32_lat) - def _summarize_accuracy(self, quant_acc1, quant_acc5, int8_acc1, int8_acc5, - fp32_acc1, fp32_acc5): + def _summarize_accuracy( + self, quant_acc1, quant_acc5, int8_acc1, int8_acc5, fp32_acc1, fp32_acc5 + ): _logger.info('--- Accuracy summary ---') self._print_accuracy('Quant', quant_acc1, quant_acc5) self._print_accuracy('INT8', int8_acc1, int8_acc5) @@ -316,8 +343,10 @@ def _summarize_accuracy(self, quant_acc1, quant_acc5, int8_acc1, int8_acc5, def _compare_accuracy(self, threshold, quant_acc1, int8_acc1): _logger.info( - 'Accepted top1 accuracy drop threshold: {0}. (condition: (Quant_top1_acc - IN8_top1_acc) <= threshold && Quant_top1_acc > 0.5 && INT8_top1_acc > 0.5)' - .format(threshold)) + 'Accepted top1 accuracy drop threshold: {0}. (condition: (Quant_top1_acc - IN8_top1_acc) <= threshold && Quant_top1_acc > 0.5 && INT8_top1_acc > 0.5)'.format( + threshold + ) + ) # We assume valid accuracy to be at least 0.5 assert quant_acc1 > 0.5 assert int8_acc1 > 0.5 @@ -334,9 +363,13 @@ def test_graph_transformation(self): return quant_model_path = test_case_args.quant_model - assert quant_model_path, 'The Quant model path cannot be empty. Please, use the --quant_model option.' + assert ( + quant_model_path + ), 'The Quant model path cannot be empty. Please, use the --quant_model option.' data_path = test_case_args.infer_data - assert data_path, 'The dataset path cannot be empty. Please, use the --infer_data option.' + assert ( + data_path + ), 'The dataset path cannot be empty. Please, use the --infer_data option.' fp32_model_path = test_case_args.fp32_model batch_size = test_case_args.batch_size batch_num = test_case_args.batch_num @@ -347,12 +380,14 @@ def test_graph_transformation(self): self._quantized_ops = set() if test_case_args.ops_to_quantize: self._quantized_ops = self._strings_from_csv( - test_case_args.ops_to_quantize) + test_case_args.ops_to_quantize + ) self._op_ids_to_skip = set([-1]) if test_case_args.op_ids_to_skip: self._op_ids_to_skip = self._ints_from_csv( - test_case_args.op_ids_to_skip) + test_case_args.op_ids_to_skip + ) self._targets = self._strings_from_csv(test_case_args.targets) assert self._targets.intersection( @@ -368,61 +403,99 @@ def test_graph_transformation(self): _logger.info('Batch number: {}'.format(batch_num)) _logger.info('Accuracy drop threshold: {}.'.format(acc_diff_threshold)) _logger.info( - 'Quantized ops: {}.'.format(','.join(self._quantized_ops) if self. - _quantized_ops else 'all quantizable')) - _logger.info('Op ids to skip quantization: {}.'.format( - ','.join(map(str, self._op_ids_to_skip) - ) if test_case_args.op_ids_to_skip else 'none')) + 'Quantized ops: {}.'.format( + ','.join(self._quantized_ops) + if self._quantized_ops + else 'all quantizable' + ) + ) + _logger.info( + 'Op ids to skip quantization: {}.'.format( + ','.join(map(str, self._op_ids_to_skip)) + if test_case_args.op_ids_to_skip + else 'none' + ) + ) _logger.info('Targets: {}.'.format(','.join(self._targets))) if 'quant' in self._targets: _logger.info('--- Quant prediction start ---') - val_reader = paddle.batch(self._reader_creator(data_path), - batch_size=batch_size) - quant_output, quant_acc1, quant_acc5, quant_fps, quant_lat = self._predict( + val_reader = paddle.batch( + self._reader_creator(data_path), batch_size=batch_size + ) + ( + quant_output, + quant_acc1, + quant_acc5, + quant_fps, + quant_lat, + ) = self._predict( val_reader, quant_model_path, batch_size, batch_num, skip_batch_num, - target='quant') + target='quant', + ) self._print_performance('Quant', quant_fps, quant_lat) self._print_accuracy('Quant', quant_acc1, quant_acc5) if 'int8' in self._targets: _logger.info('--- INT8 prediction start ---') - val_reader = paddle.batch(self._reader_creator(data_path), - batch_size=batch_size) - int8_output, int8_acc1, int8_acc5, int8_fps, int8_lat = self._predict( + val_reader = paddle.batch( + self._reader_creator(data_path), batch_size=batch_size + ) + ( + int8_output, + int8_acc1, + int8_acc5, + int8_fps, + int8_lat, + ) = self._predict( val_reader, quant_model_path, batch_size, batch_num, skip_batch_num, - target='int8') + target='int8', + ) self._print_performance('INT8', int8_fps, int8_lat) self._print_accuracy('INT8', int8_acc1, int8_acc5) fp32_acc1 = fp32_acc5 = fp32_fps = fp32_lat = -1 if 'fp32' in self._targets and fp32_model_path: _logger.info('--- FP32 prediction start ---') - val_reader = paddle.batch(self._reader_creator(data_path), - batch_size=batch_size) - fp32_output, fp32_acc1, fp32_acc5, fp32_fps, fp32_lat = self._predict( + val_reader = paddle.batch( + self._reader_creator(data_path), batch_size=batch_size + ) + ( + fp32_output, + fp32_acc1, + fp32_acc5, + fp32_fps, + fp32_lat, + ) = self._predict( val_reader, fp32_model_path, batch_size, batch_num, skip_batch_num, - target='fp32') + target='fp32', + ) self._print_performance('FP32', fp32_fps, fp32_lat) self._print_accuracy('FP32', fp32_acc1, fp32_acc5) if {'int8', 'fp32'}.issubset(self._targets): self._summarize_performance(int8_fps, int8_lat, fp32_fps, fp32_lat) if {'int8', 'quant'}.issubset(self._targets): - self._summarize_accuracy(quant_acc1, quant_acc5, int8_acc1, - int8_acc5, fp32_acc1, fp32_acc5) + self._summarize_accuracy( + quant_acc1, + quant_acc5, + int8_acc1, + int8_acc5, + fp32_acc1, + fp32_acc5, + ) self._compare_accuracy(acc_diff_threshold, quant_acc1, int8_acc1) diff --git a/python/paddle/fluid/contrib/slim/tests/quant2_int8_lstm_model.py b/python/paddle/fluid/contrib/slim/tests/quant2_int8_lstm_model.py index 0a9abe61e0e4b..71bac0208e4cf 100644 --- a/python/paddle/fluid/contrib/slim/tests/quant2_int8_lstm_model.py +++ b/python/paddle/fluid/contrib/slim/tests/quant2_int8_lstm_model.py @@ -25,35 +25,33 @@ def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('--fp32_model', - type=str, - default='', - help='A path to a FP32 model.') - parser.add_argument('--quant_model', - type=str, - default='', - help='A path to a quant model.') + parser.add_argument( + '--fp32_model', type=str, default='', help='A path to a FP32 model.' + ) + parser.add_argument( + '--quant_model', type=str, default='', help='A path to a quant model.' + ) parser.add_argument('--infer_data', type=str, default='', help='Data file.') parser.add_argument( '--warmup_iter', type=int, default=1, - help='Number of the first iterations to skip in performance statistics.' + help='Number of the first iterations to skip in performance statistics.', + ) + parser.add_argument( + '--acc_diff_threshold', + type=float, + default=0.01, + help='Accepted accuracy difference threshold.', + ) + parser.add_argument( + '--num_threads', type=int, default=1, help='Number of threads.' ) - parser.add_argument('--acc_diff_threshold', - type=float, - default=0.01, - help='Accepted accuracy difference threshold.') - parser.add_argument('--num_threads', - type=int, - default=1, - help='Number of threads.') parser.add_argument( '--mkldnn_cache_capacity', type=int, default=0, - help= - 'Mkldnn cache capacity. The default value in Python API is 15, which can slow down int8 models. Default 0 means unlimited cache.' + help='Mkldnn cache capacity. The default value in Python API is 15, which can slow down int8 models. Default 0 means unlimited cache.', ) test_args, args = parser.parse_known_args(namespace=unittest) @@ -61,7 +59,6 @@ def parse_args(): class TestLstmModelPTQ(unittest.TestCase): - def get_warmup_tensor(self, data_path, place): data = [] with open(data_path, 'rb') as in_f: @@ -75,11 +72,13 @@ def get_warmup_tensor(self, data_path, place): seq_len = (alllen >> 16) & 0xFFFF label = in_f.read(4 * label_len) - label = np.frombuffer(label, - dtype=np.int32).reshape([len(label) // 4]) + label = np.frombuffer(label, dtype=np.int32).reshape( + [len(label) // 4] + ) feat = in_f.read(4 * seq_len * 8) feat = np.frombuffer(feat, dtype=np.float32).reshape( - [len(feat) // 4 // 8, 8]) + [len(feat) // 4 // 8, 8] + ) lod_feat = [feat.shape[0]] minputs = fluid.create_lod_tensor(feat, [lod_feat], place) @@ -97,13 +96,15 @@ def get_warmup_tensor(self, data_path, place): inputs = data[1:] return warmup_data, inputs - def set_config(self, - model_path, - num_threads, - mkldnn_cache_capacity, - warmup_data=None, - use_analysis=False, - enable_ptq=False): + def set_config( + self, + model_path, + num_threads, + mkldnn_cache_capacity, + warmup_data=None, + use_analysis=False, + enable_ptq=False, + ): config = AnalysisConfig(model_path) config.set_cpu_math_library_num_threads(num_threads) if use_analysis: @@ -120,19 +121,27 @@ def set_config(self, config.quantizer_config().set_quant_batch_size(1) return config - def run_program(self, - model_path, - data_path, - num_threads, - mkldnn_cache_capacity, - warmup_iter, - use_analysis=False, - enable_ptq=False): + def run_program( + self, + model_path, + data_path, + num_threads, + mkldnn_cache_capacity, + warmup_iter, + use_analysis=False, + enable_ptq=False, + ): place = fluid.CPUPlace() warmup_data, inputs = self.get_warmup_tensor(data_path, place) warmup_data = [item[0] for item in warmup_data] - config = self.set_config(model_path, num_threads, mkldnn_cache_capacity, - warmup_data, use_analysis, enable_ptq) + config = self.set_config( + model_path, + num_threads, + mkldnn_cache_capacity, + warmup_data, + use_analysis, + enable_ptq, + ) predictor = create_paddle_predictor(config) data = [item[0] for item in inputs] @@ -164,13 +173,17 @@ def run_program(self, if this_label_data[0] <= 6350: all_ctc_num += 1 - if np_ctc_out.shape[0] == 1 and np_ctc_out.all( - ) == this_label_data.all(): + if ( + np_ctc_out.shape[0] == 1 + and np_ctc_out.all() == this_label_data.all() + ): ok_ctc_num += 1 else: all_ctc_num += 1 - if np_ctc_out.shape[0] == this_label.shape[ - 0] and np_ctc_out.all() == this_label_data.all(): + if ( + np_ctc_out.shape[0] == this_label.shape[0] + and np_ctc_out.all() == this_label_data.all() + ): ok_ctc_num += 1 if all_ctc_num > 1000 or all_hz_num > 1000: @@ -187,44 +200,75 @@ def test_lstm_model(self): return fp32_model = test_case_args.fp32_model - assert fp32_model, 'The FP32 model path cannot be empty. Please, use the --fp32_model option.' + assert ( + fp32_model + ), 'The FP32 model path cannot be empty. Please, use the --fp32_model option.' quant_model = test_case_args.quant_model - assert quant_model, 'The quant model path cannot be empty. Please, use the --quant_model option.' + assert ( + quant_model + ), 'The quant model path cannot be empty. Please, use the --quant_model option.' infer_data = test_case_args.infer_data - assert infer_data, 'The dataset path cannot be empty. Please, use the --infer_data option.' + assert ( + infer_data + ), 'The dataset path cannot be empty. Please, use the --infer_data option.' num_threads = test_case_args.num_threads mkldnn_cache_capacity = test_case_args.mkldnn_cache_capacity warmup_iter = test_case_args.warmup_iter acc_diff_threshold = test_case_args.acc_diff_threshold - (fp32_hx_acc, fp32_ctc_acc, - fp32_fps) = self.run_program(fp32_model, infer_data, num_threads, - mkldnn_cache_capacity, warmup_iter, False, - False) - - (int8_hx_acc, int8_ctc_acc, - int8_fps) = self.run_program(fp32_model, infer_data, num_threads, - mkldnn_cache_capacity, warmup_iter, True, - True) + (fp32_hx_acc, fp32_ctc_acc, fp32_fps) = self.run_program( + fp32_model, + infer_data, + num_threads, + mkldnn_cache_capacity, + warmup_iter, + False, + False, + ) + + (int8_hx_acc, int8_ctc_acc, int8_fps) = self.run_program( + fp32_model, + infer_data, + num_threads, + mkldnn_cache_capacity, + warmup_iter, + True, + True, + ) quant_model_save_path = quant_model + "_int8" # transform model to quant2 - transform_and_save_int8_model(quant_model, quant_model_save_path, - "fusion_lstm,concat") - - (quant_hx_acc, quant_ctc_acc, - quant_fps) = self.run_program(quant_model_save_path, infer_data, - num_threads, mkldnn_cache_capacity, - warmup_iter, True, False) - - print("FP32: fps {0}, hx_acc {1}, ctc_acc {2}".format( - fp32_fps, fp32_hx_acc, fp32_ctc_acc)) - - print("PTQ_INT8: fps {0}, hx_acc {1}, ctc_acc {2}".format( - int8_fps, int8_hx_acc, int8_ctc_acc)) - - print("QUANT2_INT8: fps {0}, hx_acc {1}, ctc_acc {2}".format( - quant_fps, quant_hx_acc, quant_ctc_acc)) + transform_and_save_int8_model( + quant_model, quant_model_save_path, "fusion_lstm,concat" + ) + + (quant_hx_acc, quant_ctc_acc, quant_fps) = self.run_program( + quant_model_save_path, + infer_data, + num_threads, + mkldnn_cache_capacity, + warmup_iter, + True, + False, + ) + + print( + "FP32: fps {0}, hx_acc {1}, ctc_acc {2}".format( + fp32_fps, fp32_hx_acc, fp32_ctc_acc + ) + ) + + print( + "PTQ_INT8: fps {0}, hx_acc {1}, ctc_acc {2}".format( + int8_fps, int8_hx_acc, int8_ctc_acc + ) + ) + + print( + "QUANT2_INT8: fps {0}, hx_acc {1}, ctc_acc {2}".format( + quant_fps, quant_hx_acc, quant_ctc_acc + ) + ) sys.stdout.flush() diff --git a/python/paddle/fluid/contrib/slim/tests/quant2_int8_nlp_comparison.py b/python/paddle/fluid/contrib/slim/tests/quant2_int8_nlp_comparison.py index fecead6d6de03..3b997fa0d5000 100644 --- a/python/paddle/fluid/contrib/slim/tests/quant2_int8_nlp_comparison.py +++ b/python/paddle/fluid/contrib/slim/tests/quant2_int8_nlp_comparison.py @@ -39,57 +39,56 @@ def parse_args(): '--skip_batch_num', type=int, default=0, - help='Number of the first minibatches to skip in performance statistics.' + help='Number of the first minibatches to skip in performance statistics.', + ) + parser.add_argument( + '--quant_model', type=str, default='', help='A path to a Quant model.' ) - parser.add_argument('--quant_model', - type=str, - default='', - help='A path to a Quant model.') parser.add_argument( '--fp32_model', type=str, default='', - help= - 'A path to an FP32 model. If empty, the Quant model will be used for FP32 inference.' + help='A path to an FP32 model. If empty, the Quant model will be used for FP32 inference.', ) parser.add_argument('--infer_data', type=str, default='', help='Data file.') - parser.add_argument('--labels', - type=str, - default='', - help='File with labels.') + parser.add_argument( + '--labels', type=str, default='', help='File with labels.' + ) parser.add_argument( '--batch_num', type=int, default=0, - help= - 'Number of batches to process. 0 or less means whole dataset. Default: 0.' + help='Number of batches to process. 0 or less means whole dataset. Default: 0.', + ) + parser.add_argument( + '--acc_diff_threshold', + type=float, + default=0.01, + help='Accepted accuracy difference threshold.', ) - parser.add_argument('--acc_diff_threshold', - type=float, - default=0.01, - help='Accepted accuracy difference threshold.') parser.add_argument( '--ops_to_quantize', type=str, default='', - help= - 'A comma separated list of operators to quantize. Only quantizable operators are taken into account. If the option is not used, an attempt to quantize all quantizable operators will be made.' + help='A comma separated list of operators to quantize. Only quantizable operators are taken into account. If the option is not used, an attempt to quantize all quantizable operators will be made.', ) parser.add_argument( '--op_ids_to_skip', type=str, default='', - help='A comma separated list of operator ids to skip in quantization.') + help='A comma separated list of operator ids to skip in quantization.', + ) parser.add_argument( '--targets', type=str, default='quant,int8,fp32', - help= - 'A comma separated list of inference types to run ("int8", "fp32", "quant"). Default: "quant,int8,fp32"' + help='A comma separated list of inference types to run ("int8", "fp32", "quant"). Default: "quant,int8,fp32"', + ) + parser.add_argument( + '--debug', + action='store_true', + help='If used, the graph of Quant model is drawn.', ) - parser.add_argument('--debug', - action='store_true', - help='If used, the graph of Quant model is drawn.') test_args, args = parser.parse_known_args(namespace=unittest) @@ -116,16 +115,16 @@ def reader(): for i in range(len(data_lines)): data_fields = data_lines[i].split(';') - assert len( - data_fields - ) >= 2, "The number of data fields in the dataset is less than 2" + assert ( + len(data_fields) >= 2 + ), "The number of data fields in the dataset is less than 2" buffers = [] shape = [] for j in range(2): data = data_fields[j].split(':') - assert len( - data - ) >= 2, "Size of data in the dataset is less than 2" + assert ( + len(data) >= 2 + ), "Size of data in the dataset is less than 2" # Shape is stored under index 0, while data under 1 shape = data[0].split() shape.pop(0) @@ -149,42 +148,53 @@ def _get_batch_correct(self, batch_output=None, labels=None): correct += 1 return correct - def _predict(self, - test_reader=None, - model_path=None, - batch_size=1, - batch_num=1, - skip_batch_num=0, - target='quant'): + def _predict( + self, + test_reader=None, + model_path=None, + batch_size=1, + batch_num=1, + skip_batch_num=0, + target='quant', + ): assert target in ['quant', 'int8', 'fp32'] place = fluid.CPUPlace() exe = fluid.Executor(place) inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if os.path.exists(os.path.join(model_path, '__model__')): - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(model_path, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) else: - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(model_path, exe, 'model', - 'params') + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + model_path, exe, 'model', 'params' + ) graph = IrGraph(core.Graph(inference_program.desc), for_test=True) - if (self._debug): + if self._debug: graph.draw('.', 'quant_orig', graph.all_op_nodes()) - if (target != 'quant'): + if target != 'quant': quant_transform_pass = Quant2Int8MkldnnPass( self._quantized_ops, _op_ids_to_skip=self._op_ids_to_skip, _scope=inference_scope, _place=place, _core=core, - _debug=self._debug) - if (target == 'int8'): + _debug=self._debug, + ) + if target == 'int8': graph = quant_transform_pass.apply(graph) else: # target == fp32 graph = quant_transform_pass.prepare_and_optimize_fp32( - graph) + graph + ) inference_program = graph.to_program() @@ -205,12 +215,14 @@ def _predict(self, labels = np.array([x[2] for x in data]).astype('int64') start = time.time() - out = exe.run(inference_program, - feed={ - feed_target_names[0]: input0, - feed_target_names[1]: input1 - }, - fetch_list=fetch_targets) + out = exe.run( + inference_program, + feed={ + feed_target_names[0]: input0, + feed_target_names[1]: input1, + }, + fetch_list=fetch_targets, + ) batch_time = (time.time() - start) * 1000 # in miliseconds batch_times.append(batch_time) batch_correct = self._get_batch_correct(out, labels) @@ -224,8 +236,10 @@ def _predict(self, iters += 1 appx = ' (warm-up)' if iters <= skip_batch_num else '' _logger.info( - 'batch {0}{4}, acc: {1:.4f}, latency: {2:.4f} ms, predictions per sec: {3:.2f}' - .format(iters, batch_acc, latency, pps, appx)) + 'batch {0}{4}, acc: {1:.4f}, latency: {2:.4f} ms, predictions per sec: {3:.2f}'.format( + iters, batch_acc, latency, pps, appx + ) + ) # Postprocess benchmark data infer_total_time = time.time() - infer_start_time @@ -236,14 +250,17 @@ def _predict(self, pps_avg = np.average(ppses) acc_avg = float(np.sum(total_correct)) / float(total_samples) _logger.info( - 'Total inference run time: {:.2f} s'.format(infer_total_time)) + 'Total inference run time: {:.2f} s'.format(infer_total_time) + ) return acc_avg, pps_avg, latency_avg def _print_performance(self, title, pps, lat): _logger.info( - '{0}: avg predictions per sec: {1:.2f}, avg latency: {2:.4f} ms'. - format(title, pps, lat)) + '{0}: avg predictions per sec: {1:.2f}, avg latency: {2:.4f} ms'.format( + title, pps, lat + ) + ) def _print_accuracy(self, title, acc): _logger.info('{0}: avg accuracy: {1:.6f}'.format(title, acc)) @@ -263,8 +280,10 @@ def _summarize_accuracy(self, quant_acc, int8_acc, fp32_acc): def _compare_accuracy(self, threshold, quant_acc, int8_acc): _logger.info( - 'Accepted accuracy drop threshold: {0}. (condition: (Quant_acc - INT8_acc) <= threshold)' - .format(threshold)) + 'Accepted accuracy drop threshold: {0}. (condition: (Quant_acc - INT8_acc) <= threshold)'.format( + threshold + ) + ) # Random outputs give accuracy about 0.33, we assume valid accuracy to be at least 0.5 assert quant_acc > 0.5 assert int8_acc > 0.5 @@ -281,9 +300,13 @@ def test_graph_transformation(self): return quant_model_path = test_case_args.quant_model - assert quant_model_path, 'The Quant model path cannot be empty. Please, use the --quant_model option.' + assert ( + quant_model_path + ), 'The Quant model path cannot be empty. Please, use the --quant_model option.' data_path = test_case_args.infer_data - assert data_path, 'The dataset path cannot be empty. Please, use the --infer_data option.' + assert ( + data_path + ), 'The dataset path cannot be empty. Please, use the --infer_data option.' fp32_model_path = test_case_args.fp32_model labels_path = test_case_args.labels batch_size = test_case_args.batch_size @@ -295,12 +318,14 @@ def test_graph_transformation(self): self._quantized_ops = set() if test_case_args.ops_to_quantize: self._quantized_ops = self._strings_from_csv( - test_case_args.ops_to_quantize) + test_case_args.ops_to_quantize + ) self._op_ids_to_skip = set([-1]) if test_case_args.op_ids_to_skip: self._op_ids_to_skip = self._ints_from_csv( - test_case_args.op_ids_to_skip) + test_case_args.op_ids_to_skip + ) self._targets = self._strings_from_csv(test_case_args.targets) assert self._targets.intersection( @@ -317,53 +342,70 @@ def test_graph_transformation(self): _logger.info('Batch number: {}'.format(batch_num)) _logger.info('Accuracy drop threshold: {}.'.format(acc_diff_threshold)) _logger.info( - 'Quantized ops: {}.'.format(','.join(self._quantized_ops) if self. - _quantized_ops else 'all quantizable')) - _logger.info('Op ids to skip quantization: {}.'.format( - ','.join(map(str, self._op_ids_to_skip) - ) if test_case_args.op_ids_to_skip else 'none')) + 'Quantized ops: {}.'.format( + ','.join(self._quantized_ops) + if self._quantized_ops + else 'all quantizable' + ) + ) + _logger.info( + 'Op ids to skip quantization: {}.'.format( + ','.join(map(str, self._op_ids_to_skip)) + if test_case_args.op_ids_to_skip + else 'none' + ) + ) _logger.info('Targets: {}.'.format(','.join(self._targets))) if 'quant' in self._targets: _logger.info('--- Quant prediction start ---') - val_reader = paddle.batch(self._reader_creator( - data_path, labels_path), - batch_size=batch_size) - quant_acc, quant_pps, quant_lat = self._predict(val_reader, - quant_model_path, - batch_size, - batch_num, - skip_batch_num, - target='quant') + val_reader = paddle.batch( + self._reader_creator(data_path, labels_path), + batch_size=batch_size, + ) + quant_acc, quant_pps, quant_lat = self._predict( + val_reader, + quant_model_path, + batch_size, + batch_num, + skip_batch_num, + target='quant', + ) self._print_performance('Quant', quant_pps, quant_lat) self._print_accuracy('Quant', quant_acc) if 'int8' in self._targets: _logger.info('--- INT8 prediction start ---') - val_reader = paddle.batch(self._reader_creator( - data_path, labels_path), - batch_size=batch_size) - int8_acc, int8_pps, int8_lat = self._predict(val_reader, - quant_model_path, - batch_size, - batch_num, - skip_batch_num, - target='int8') + val_reader = paddle.batch( + self._reader_creator(data_path, labels_path), + batch_size=batch_size, + ) + int8_acc, int8_pps, int8_lat = self._predict( + val_reader, + quant_model_path, + batch_size, + batch_num, + skip_batch_num, + target='int8', + ) self._print_performance('INT8', int8_pps, int8_lat) self._print_accuracy('INT8', int8_acc) fp32_acc = fp32_pps = fp32_lat = -1 if 'fp32' in self._targets and fp32_model_path: _logger.info('--- FP32 prediction start ---') - val_reader = paddle.batch(self._reader_creator( - data_path, labels_path), - batch_size=batch_size) - fp32_acc, fp32_pps, fp32_lat = self._predict(val_reader, - fp32_model_path, - batch_size, - batch_num, - skip_batch_num, - target='fp32') + val_reader = paddle.batch( + self._reader_creator(data_path, labels_path), + batch_size=batch_size, + ) + fp32_acc, fp32_pps, fp32_lat = self._predict( + val_reader, + fp32_model_path, + batch_size, + batch_num, + skip_batch_num, + target='fp32', + ) self._print_performance('FP32', fp32_pps, fp32_lat) self._print_accuracy('FP32', fp32_acc) diff --git a/python/paddle/fluid/contrib/slim/tests/quant_int8_image_classification_comparison.py b/python/paddle/fluid/contrib/slim/tests/quant_int8_image_classification_comparison.py index ec88664c2edce..e3aecd48c34db 100644 --- a/python/paddle/fluid/contrib/slim/tests/quant_int8_image_classification_comparison.py +++ b/python/paddle/fluid/contrib/slim/tests/quant_int8_image_classification_comparison.py @@ -40,27 +40,29 @@ def parse_args(): '--skip_batch_num', type=int, default=0, - help='Number of the first minibatches to skip in performance statistics.' + help='Number of the first minibatches to skip in performance statistics.', + ) + parser.add_argument( + '--debug', + action='store_true', + help='If used, the graph of Quant model is drawn.', + ) + parser.add_argument( + '--quant_model', type=str, default='', help='A path to a Quant model.' ) - parser.add_argument('--debug', - action='store_true', - help='If used, the graph of Quant model is drawn.') - parser.add_argument('--quant_model', - type=str, - default='', - help='A path to a Quant model.') parser.add_argument('--infer_data', type=str, default='', help='Data file.') parser.add_argument( '--batch_num', type=int, default=0, - help= - 'Number of batches to process. 0 or less means whole dataset. Default: 0.' + help='Number of batches to process. 0 or less means whole dataset. Default: 0.', + ) + parser.add_argument( + '--acc_diff_threshold', + type=float, + default=0.01, + help='Accepted accuracy difference threshold.', ) - parser.add_argument('--acc_diff_threshold', - type=float, - default=0.01, - help='Accepted accuracy difference threshold.') test_args, args = parser.parse_known_args(namespace=unittest) return test_args, sys.argv[:1] + args @@ -72,7 +74,6 @@ class QuantInt8ImageClassificationComparisonTest(unittest.TestCase): """ def _reader_creator(self, data_file='data.bin'): - def reader(): with open(data_file, 'rb') as fp: num = fp.read(8) @@ -91,7 +92,8 @@ def reader(): fp.seek(imgs_offset + img_size * step) img = fp.read(img_size) img = struct.unpack_from( - '{}f'.format(img_ch * img_w * img_h), img) + '{}f'.format(img_ch * img_w * img_h), img + ) img = np.array(img) img.shape = (img_ch, img_w, img_h) fp.seek(labels_offset + label_size * step) @@ -125,14 +127,14 @@ def _prepare_for_fp32_mkldnn(self, graph): name = op_node.name() if name in ['depthwise_conv2d']: input_var_node = graph._find_node_by_name( - op_node.inputs, - op_node.input("Input")[0]) + op_node.inputs, op_node.input("Input")[0] + ) weight_var_node = graph._find_node_by_name( - op_node.inputs, - op_node.input("Filter")[0]) + op_node.inputs, op_node.input("Filter")[0] + ) output_var_node = graph._find_node_by_name( - graph.all_var_nodes(), - op_node.output("Output")[0]) + graph.all_var_nodes(), op_node.output("Output")[0] + ) attrs = { name: op_node.op().attr(name) for name in op_node.op().attr_names() @@ -141,11 +143,9 @@ def _prepare_for_fp32_mkldnn(self, graph): conv_op_node = graph.create_op_node( op_type='conv2d', attrs=attrs, - inputs={ - 'Input': input_var_node, - 'Filter': weight_var_node - }, - outputs={'Output': output_var_node}) + inputs={'Input': input_var_node, 'Filter': weight_var_node}, + outputs={'Output': output_var_node}, + ) graph.link_to(input_var_node, conv_op_node) graph.link_to(weight_var_node, conv_op_node) @@ -154,31 +154,41 @@ def _prepare_for_fp32_mkldnn(self, graph): return graph - def _predict(self, - test_reader=None, - model_path=None, - batch_size=1, - batch_num=1, - skip_batch_num=0, - transform_to_int8=False): + def _predict( + self, + test_reader=None, + model_path=None, + batch_size=1, + batch_num=1, + skip_batch_num=0, + transform_to_int8=False, + ): place = fluid.CPUPlace() exe = fluid.Executor(place) inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if os.path.exists(os.path.join(model_path, '__model__')): - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(model_path, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) else: - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(model_path, exe, 'model', - 'params') + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + model_path, exe, 'model', 'params' + ) graph = IrGraph(core.Graph(inference_program.desc), for_test=True) - if (self._debug): + if self._debug: graph.draw('.', 'quant_orig', graph.all_op_nodes()) - if (transform_to_int8): - mkldnn_int8_pass = QuantInt8MkldnnPass(_scope=inference_scope, - _place=place) + if transform_to_int8: + mkldnn_int8_pass = QuantInt8MkldnnPass( + _scope=inference_scope, _place=place + ) graph = mkldnn_int8_pass.apply(graph) else: graph = self._prepare_for_fp32_mkldnn(graph) @@ -205,13 +215,16 @@ def _predict(self, labels = np.array([x[1] for x in data]).astype('int64') start = time.time() - out = exe.run(inference_program, - feed={feed_target_names[0]: images}, - fetch_list=fetch_targets) + out = exe.run( + inference_program, + feed={feed_target_names[0]: images}, + fetch_list=fetch_targets, + ) batch_time = (time.time() - start) * 1000 # in miliseconds outputs.append(out[0]) batch_acc1, batch_acc5 = self._get_batch_accuracy( - out[0], labels) + out[0], labels + ) infer_accs1.append(batch_acc1) infer_accs5.append(batch_acc5) samples = len(data) @@ -221,10 +234,17 @@ def _predict(self, fpses.append(fps) iters += 1 appx = ' (warm-up)' if iters <= skip_batch_num else '' - _logger.info('batch {0}{5}, acc1: {1:.4f}, acc5: {2:.4f}, ' - 'latency: {3:.4f} ms, fps: {4:.2f}'.format( - iters, batch_acc1, batch_acc5, - batch_time / batch_size, fps, appx)) + _logger.info( + 'batch {0}{5}, acc1: {1:.4f}, acc5: {2:.4f}, ' + 'latency: {3:.4f} ms, fps: {4:.2f}'.format( + iters, + batch_acc1, + batch_acc5, + batch_time / batch_size, + fps, + appx, + ) + ) # Postprocess benchmark data batch_latencies = batch_times[skip_batch_num:] @@ -236,29 +256,43 @@ def _predict(self, acc1_avg = np.mean(infer_accs1) acc5_avg = np.mean(infer_accs5) _logger.info( - 'Total inference run time: {:.2f} s'.format(infer_total_time)) + 'Total inference run time: {:.2f} s'.format(infer_total_time) + ) return outputs, acc1_avg, acc5_avg, fps_avg, latency_avg def _summarize_performance(self, fp32_fps, fp32_lat, int8_fps, int8_lat): _logger.info('--- Performance summary ---') - _logger.info('FP32: avg fps: {0:.2f}, avg latency: {1:.4f} ms'.format( - fp32_fps, fp32_lat)) - _logger.info('INT8: avg fps: {0:.2f}, avg latency: {1:.4f} ms'.format( - int8_fps, int8_lat)) - - def _compare_accuracy(self, fp32_acc1, fp32_acc5, int8_acc1, int8_acc5, - threshold): + _logger.info( + 'FP32: avg fps: {0:.2f}, avg latency: {1:.4f} ms'.format( + fp32_fps, fp32_lat + ) + ) + _logger.info( + 'INT8: avg fps: {0:.2f}, avg latency: {1:.4f} ms'.format( + int8_fps, int8_lat + ) + ) + + def _compare_accuracy( + self, fp32_acc1, fp32_acc5, int8_acc1, int8_acc5, threshold + ): _logger.info('--- Accuracy summary ---') _logger.info( - 'Accepted top1 accuracy drop threshold: {0}. (condition: (FP32_top1_acc - IN8_top1_acc) <= threshold)' - .format(threshold)) + 'Accepted top1 accuracy drop threshold: {0}. (condition: (FP32_top1_acc - IN8_top1_acc) <= threshold)'.format( + threshold + ) + ) _logger.info( - 'FP32: avg top1 accuracy: {0:.4f}, avg top5 accuracy: {1:.4f}'. - format(fp32_acc1, fp32_acc5)) + 'FP32: avg top1 accuracy: {0:.4f}, avg top5 accuracy: {1:.4f}'.format( + fp32_acc1, fp32_acc5 + ) + ) _logger.info( - 'INT8: avg top1 accuracy: {0:.4f}, avg top5 accuracy: {1:.4f}'. - format(int8_acc1, int8_acc5)) + 'INT8: avg top1 accuracy: {0:.4f}, avg top5 accuracy: {1:.4f}'.format( + int8_acc1, int8_acc5 + ) + ) assert fp32_acc1 > 0.0 assert int8_acc1 > 0.0 assert fp32_acc1 - int8_acc1 <= threshold @@ -268,9 +302,13 @@ def test_graph_transformation(self): return quant_model_path = test_case_args.quant_model - assert quant_model_path, 'The Quant model path cannot be empty. Please, use the --quant_model option.' + assert ( + quant_model_path + ), 'The Quant model path cannot be empty. Please, use the --quant_model option.' data_path = test_case_args.infer_data - assert data_path, 'The dataset path cannot be empty. Please, use the --infer_data option.' + assert ( + data_path + ), 'The dataset path cannot be empty. Please, use the --infer_data option.' batch_size = test_case_args.batch_size batch_num = test_case_args.batch_num skip_batch_num = test_case_args.skip_batch_num @@ -285,29 +323,34 @@ def test_graph_transformation(self): _logger.info('Accuracy drop threshold: {0}.'.format(acc_diff_threshold)) _logger.info('--- Quant FP32 prediction start ---') - val_reader = paddle.batch(self._reader_creator(data_path), - batch_size=batch_size) + val_reader = paddle.batch( + self._reader_creator(data_path), batch_size=batch_size + ) fp32_output, fp32_acc1, fp32_acc5, fp32_fps, fp32_lat = self._predict( val_reader, quant_model_path, batch_size, batch_num, skip_batch_num, - transform_to_int8=False) + transform_to_int8=False, + ) _logger.info('--- Quant INT8 prediction start ---') - val_reader = paddle.batch(self._reader_creator(data_path), - batch_size=batch_size) + val_reader = paddle.batch( + self._reader_creator(data_path), batch_size=batch_size + ) int8_output, int8_acc1, int8_acc5, int8_fps, int8_lat = self._predict( val_reader, quant_model_path, batch_size, batch_num, skip_batch_num, - transform_to_int8=True) + transform_to_int8=True, + ) self._summarize_performance(fp32_fps, fp32_lat, int8_fps, int8_lat) - self._compare_accuracy(fp32_acc1, fp32_acc5, int8_acc1, int8_acc5, - acc_diff_threshold) + self._compare_accuracy( + fp32_acc1, fp32_acc5, int8_acc1, int8_acc5, acc_diff_threshold + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/save_quant_model.py b/python/paddle/fluid/contrib/slim/tests/save_quant_model.py index cb15b3da4735c..b743615f575aa 100644 --- a/python/paddle/fluid/contrib/slim/tests/save_quant_model.py +++ b/python/paddle/fluid/contrib/slim/tests/save_quant_model.py @@ -27,88 +27,102 @@ def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('--quant_model_path', - type=str, - default='', - help='A path to a Quant model.') - parser.add_argument('--int8_model_save_path', - type=str, - default='', - help='Saved optimized and quantized INT8 model') + parser.add_argument( + '--quant_model_path', + type=str, + default='', + help='A path to a Quant model.', + ) + parser.add_argument( + '--int8_model_save_path', + type=str, + default='', + help='Saved optimized and quantized INT8 model', + ) parser.add_argument( '--ops_to_quantize', type=str, default='', - help= - 'A comma separated list of operators to quantize. Only quantizable operators are taken into account. If the option is not used, an attempt to quantize all quantizable operators will be made.' + help='A comma separated list of operators to quantize. Only quantizable operators are taken into account. If the option is not used, an attempt to quantize all quantizable operators will be made.', ) parser.add_argument( '--op_ids_to_skip', type=str, default='', - help='A comma separated list of operator ids to skip in quantization.') - parser.add_argument('--debug', - action='store_true', - help='If used, the graph of Quant model is drawn.') + help='A comma separated list of operator ids to skip in quantization.', + ) + parser.add_argument( + '--debug', + action='store_true', + help='If used, the graph of Quant model is drawn.', + ) parser.add_argument( '--quant_model_filename', type=str, default="", - help= - 'The input model`s file name. If empty, search default `__model__` and separate parameter files and use them or in case if not found, attempt loading `model` and `params` files.' + help='The input model`s file name. If empty, search default `__model__` and separate parameter files and use them or in case if not found, attempt loading `model` and `params` files.', ) parser.add_argument( '--quant_params_filename', type=str, default="", - help= - 'If quant_model_filename is empty, this field is ignored. The input model`s all parameters file name. If empty load parameters from separate files.' + help='If quant_model_filename is empty, this field is ignored. The input model`s all parameters file name. If empty load parameters from separate files.', ) parser.add_argument( '--save_model_filename', type=str, default="__model__", - help= - 'The name of file to save the inference program itself. If is set None, a default filename __model__ will be used.' + help='The name of file to save the inference program itself. If is set None, a default filename __model__ will be used.', ) parser.add_argument( '--save_params_filename', type=str, default=None, - help= - 'The name of file to save all related parameters. If it is set None, parameters will be saved in separate files' + help='The name of file to save all related parameters. If it is set None, parameters will be saved in separate files', ) test_args, args = parser.parse_known_args(namespace=unittest) return test_args, sys.argv[:1] + args -def transform_and_save_int8_model(original_path, - save_path, - ops_to_quantize='', - op_ids_to_skip='', - debug=False, - quant_model_filename='', - quant_params_filename='', - save_model_filename="__model__", - save_params_filename=None): +def transform_and_save_int8_model( + original_path, + save_path, + ops_to_quantize='', + op_ids_to_skip='', + debug=False, + quant_model_filename='', + quant_params_filename='', + save_model_filename="__model__", + save_params_filename=None, +): place = fluid.CPUPlace() exe = fluid.Executor(place) inference_scope = fluid.executor.global_scope() with fluid.scope_guard(inference_scope): if not quant_model_filename: if os.path.exists(os.path.join(original_path, '__model__')): - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(original_path, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(original_path, exe) else: - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(original_path, exe, 'model', - 'params') + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + original_path, exe, 'model', 'params' + ) else: - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(original_path, exe, - quant_model_filename, - quant_params_filename) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + original_path, exe, quant_model_filename, quant_params_filename + ) ops_to_quantize_set = set() print(ops_to_quantize) @@ -121,7 +135,7 @@ def transform_and_save_int8_model(original_path, op_ids_to_skip_set = set(map(int, op_ids_to_skip.split(','))) graph = IrGraph(core.Graph(inference_program.desc), for_test=True) - if (debug): + if debug: graph.draw('.', 'quant_orig', graph.all_op_nodes()) transform_to_mkldnn_int8_pass = Quant2Int8MkldnnPass( ops_to_quantize_set, @@ -129,27 +143,38 @@ def transform_and_save_int8_model(original_path, _scope=inference_scope, _place=place, _core=core, - _debug=debug) + _debug=debug, + ) graph = transform_to_mkldnn_int8_pass.apply(graph) inference_program = graph.to_program() with fluid.scope_guard(inference_scope): - fluid.io.save_inference_model(save_path, - feed_target_names, - fetch_targets, - exe, - inference_program, - model_filename=save_model_filename, - params_filename=save_params_filename) + fluid.io.save_inference_model( + save_path, + feed_target_names, + fetch_targets, + exe, + inference_program, + model_filename=save_model_filename, + params_filename=save_params_filename, + ) print( - "Success! INT8 model obtained from the Quant model can be found at {}\n" - .format(save_path)) + "Success! INT8 model obtained from the Quant model can be found at {}\n".format( + save_path + ) + ) if __name__ == '__main__': global test_args test_args, remaining_args = parse_args() transform_and_save_int8_model( - test_args.quant_model_path, test_args.int8_model_save_path, - test_args.ops_to_quantize, test_args.op_ids_to_skip, test_args.debug, - test_args.quant_model_filename, test_args.quant_params_filename, - test_args.save_model_filename, test_args.save_params_filename) + test_args.quant_model_path, + test_args.int8_model_save_path, + test_args.ops_to_quantize, + test_args.op_ids_to_skip, + test_args.debug, + test_args.quant_model_filename, + test_args.quant_params_filename, + test_args.save_model_filename, + test_args.save_params_filename, + ) diff --git a/python/paddle/fluid/contrib/slim/tests/test_graph.py b/python/paddle/fluid/contrib/slim/tests/test_graph.py index dd08eb1cbde63..482c7237bfce8 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_graph.py +++ b/python/paddle/fluid/contrib/slim/tests/test_graph.py @@ -29,19 +29,23 @@ def conv_block(): img = fluid.layers.data(name='image', shape=[1, 28, 28], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) prediction = fluid.layers.fc(input=conv_pool_2, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) avg_loss = paddle.mean(loss) @@ -49,7 +53,6 @@ def conv_block(): class TestGraph(unittest.TestCase): - def graph_apis(self, use_cuda=False, for_ci=True): main = fluid.Program() startup = fluid.Program() @@ -65,25 +68,27 @@ def graph_apis(self, use_cuda=False, for_ci=True): build_strategy.memory_optimize = False build_strategy.enable_inplace = False origin_binary = fluid.CompiledProgram(graph.graph).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) backup_binary = fluid.CompiledProgram( - backup_graph.graph).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + backup_graph.graph + ).with_data_parallel(loss_name=loss.name, build_strategy=build_strategy) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup) iters = 5 batch_size = 8 - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) def _train(binary): for _ in range(iters): data = next(train_reader()) - loss_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss.name]) + loss_v = exe.run( + binary, feed=feeder.feed(data), fetch_list=[loss.name] + ) if not for_ci: print('{}: {}'.format('loss', loss_v)) @@ -98,18 +103,18 @@ def _set_zero(var_name, scope, place): var.set(var_array, place) sum_before = np.sum( - np.array( - fluid.global_scope().find_var('conv2d_1.w_0').get_tensor())) + np.array(fluid.global_scope().find_var('conv2d_1.w_0').get_tensor()) + ) fluid.io._save_persistable_nodes(exe, checkponit_dir, graph) _set_zero('conv2d_1.w_0', fluid.global_scope(), place) set_after = np.sum( - np.array( - fluid.global_scope().find_var('conv2d_1.w_0').get_tensor())) + np.array(fluid.global_scope().find_var('conv2d_1.w_0').get_tensor()) + ) self.assertEqual(set_after, 0) fluid.io._load_persistable_nodes(exe, checkponit_dir, graph) sum_after = np.sum( - np.array( - fluid.global_scope().find_var('conv2d_1.w_0').get_tensor())) + np.array(fluid.global_scope().find_var('conv2d_1.w_0').get_tensor()) + ) self.assertEqual(sum_before, sum_after) marked_nodes = set() diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_out_scale.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_out_scale.py index 743e45e927a19..5d990d4a9860b 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_out_scale.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_out_scale.py @@ -42,9 +42,9 @@ if core.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) def get_vaild_warning_num(warning, w): @@ -56,7 +56,6 @@ def get_vaild_warning_num(warning, w): class ImperativeLenet(fluid.dygraph.Layer): - def __init__(self, num_classes=10): super(ImperativeLenet, self).__init__() conv2d_w1_attr = fluid.ParamAttr(name="conv2d_w_1") @@ -69,36 +68,55 @@ def __init__(self, num_classes=10): fc_b2_attr = fluid.ParamAttr(name="fc_b_2") fc_b3_attr = fluid.ParamAttr(name="fc_b_3") self.features = Sequential( - Conv2D(in_channels=1, - out_channels=6, - kernel_size=3, - stride=1, - padding=1, - weight_attr=conv2d_w1_attr, - bias_attr=False), BatchNorm2D(6), ReLU(), + Conv2D( + in_channels=1, + out_channels=6, + kernel_size=3, + stride=1, + padding=1, + weight_attr=conv2d_w1_attr, + bias_attr=False, + ), + BatchNorm2D(6), + ReLU(), + MaxPool2D(kernel_size=2, stride=2), + Conv2D( + in_channels=6, + out_channels=16, + kernel_size=5, + stride=1, + padding=0, + weight_attr=conv2d_w2_attr, + bias_attr=conv2d_b2_attr, + ), + BatchNorm2D(16), + PReLU(), MaxPool2D(kernel_size=2, stride=2), - Conv2D(in_channels=6, - out_channels=16, - kernel_size=5, - stride=1, - padding=0, - weight_attr=conv2d_w2_attr, - bias_attr=conv2d_b2_attr), BatchNorm2D(16), PReLU(), - MaxPool2D(kernel_size=2, stride=2)) + ) self.fc = Sequential( - Linear(in_features=400, - out_features=120, - weight_attr=fc_w1_attr, - bias_attr=fc_b1_attr), LeakyReLU(), - Linear(in_features=120, - out_features=84, - weight_attr=fc_w2_attr, - bias_attr=fc_b2_attr), Sigmoid(), - Linear(in_features=84, - out_features=num_classes, - weight_attr=fc_w3_attr, - bias_attr=fc_b3_attr), Softmax()) + Linear( + in_features=400, + out_features=120, + weight_attr=fc_w1_attr, + bias_attr=fc_b1_attr, + ), + LeakyReLU(), + Linear( + in_features=120, + out_features=84, + weight_attr=fc_w2_attr, + bias_attr=fc_b2_attr, + ), + Sigmoid(), + Linear( + in_features=84, + out_features=num_classes, + weight_attr=fc_w3_attr, + bias_attr=fc_b3_attr, + ), + Softmax(), + ) def forward(self, inputs): x = self.features(inputs) @@ -109,13 +127,14 @@ def forward(self, inputs): class TestImperativeOutSclae(unittest.TestCase): - def setUp(self): self.root_path = tempfile.TemporaryDirectory() - self.param_save_path = os.path.join(self.root_path.name, - "lenet.pdparams") - self.save_path = os.path.join(self.root_path.name, - "lenet_dynamic_outscale_infer_model") + self.param_save_path = os.path.join( + self.root_path.name, "lenet.pdparams" + ) + self.save_path = os.path.join( + self.root_path.name, "lenet_dynamic_outscale_infer_model" + ) def tearDown(self): self.root_path.cleanup() @@ -128,7 +147,8 @@ def func_out_scale_acc(self): activation_quantize_type = 'moving_average_abs_max' imperative_out_scale = ImperativeQuantAware( weight_quantize_type=weight_quantize_type, - activation_quantize_type=activation_quantize_type) + activation_quantize_type=activation_quantize_type, + ) with fluid.dygraph.guard(): np.random.seed(seed) @@ -139,11 +159,12 @@ def func_out_scale_acc(self): lenet = fix_model_dict(lenet) imperative_out_scale.quantize(lenet) - reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=32, - drop_last=True) - adam = AdamOptimizer(learning_rate=lr, - parameter_list=lenet.parameters()) + reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=32, drop_last=True + ) + adam = AdamOptimizer( + learning_rate=lr, parameter_list=lenet.parameters() + ) loss_list = train_lenet(lenet, reader, adam) lenet.eval() @@ -151,8 +172,10 @@ def func_out_scale_acc(self): paddle.save(save_dict, self.param_save_path) for i in range(len(loss_list) - 1): - self.assertTrue(loss_list[i] > loss_list[i + 1], - msg='Failed to do the imperative qat.') + self.assertTrue( + loss_list[i] > loss_list[i + 1], + msg='Failed to do the imperative qat.', + ) with fluid.dygraph.guard(): lenet = ImperativeLenet() @@ -160,11 +183,12 @@ def func_out_scale_acc(self): imperative_out_scale.quantize(lenet) lenet.set_dict(load_dict) - reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=32, - drop_last=True) - adam = AdamOptimizer(learning_rate=lr, - parameter_list=lenet.parameters()) + reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=32, drop_last=True + ) + adam = AdamOptimizer( + learning_rate=lr, parameter_list=lenet.parameters() + ) loss_list = train_lenet(lenet, reader, adam) lenet.eval() @@ -172,13 +196,17 @@ def func_out_scale_acc(self): layer=lenet, path=self.save_path, input_spec=[ - paddle.static.InputSpec(shape=[None, 1, 28, 28], - dtype='float32') - ]) + paddle.static.InputSpec( + shape=[None, 1, 28, 28], dtype='float32' + ) + ], + ) for i in range(len(loss_list) - 1): - self.assertTrue(loss_list[i] > loss_list[i + 1], - msg='Failed to do the imperative qat.') + self.assertTrue( + loss_list[i] > loss_list[i + 1], + msg='Failed to do the imperative qat.', + ) def test_out_scale_acc(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_ptq.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_ptq.py index 6727356b327ac..759e74907e1bc 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_ptq.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_ptq.py @@ -30,12 +30,16 @@ from paddle.dataset.common import download from paddle.fluid.framework import _test_eager_guard -from imperative_test_utils import fix_model_dict, ImperativeLenet, ImperativeLinearBn +from imperative_test_utils import ( + fix_model_dict, + ImperativeLenet, + ImperativeLinearBn, +) from imperative_test_utils import ImperativeLinearBn_hook -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class TestFuseLinearBn(unittest.TestCase): @@ -54,27 +58,30 @@ def test_fuse(self): quant_h = ptq.quantize(model_h, fuse=True, fuse_list=f_l) for name, layer in quant_model.named_sublayers(): if name in f_l: - assert not (isinstance(layer, nn.BatchNorm1D) - or isinstance(layer, nn.BatchNorm2D)) + assert not ( + isinstance(layer, nn.BatchNorm1D) + or isinstance(layer, nn.BatchNorm2D) + ) out = model(inputs) out_h = model_h(inputs) out_quant = quant_model(inputs) out_quant_h = quant_h(inputs) cos_sim_func = nn.CosineSimilarity(axis=0) - print('fuse linear+bn', cos_sim_func(out.flatten(), - out_quant.flatten())) + print( + 'fuse linear+bn', cos_sim_func(out.flatten(), out_quant.flatten()) + ) print(cos_sim_func(out_h.flatten(), out_quant_h.flatten())) class TestImperativePTQ(unittest.TestCase): - """ - """ + """ """ @classmethod def setUpClass(cls): cls.download_path = 'dygraph_int8/download' - cls.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + - cls.download_path) + cls.cache_folder = os.path.expanduser( + '~/.cache/paddle/dataset/' + cls.download_path + ) cls.lenet_url = "https://paddle-inference-dist.cdn.bcebos.com/int8/unittest_model_data/lenet_pretrained.tar.gz" cls.lenet_md5 = "953b802fb73b52fae42896e3c24f0afb" @@ -87,7 +94,8 @@ def setUpClass(cls): def cache_unzipping(self, target_folder, zip_path): if not os.path.exists(target_folder): cmd = 'mkdir {0} && tar xf {1} -C {0}'.format( - target_folder, zip_path) + target_folder, zip_path + ) os.system(cmd) def download_model(self, data_url, data_md5, folder_name): @@ -114,23 +122,29 @@ def set_vars(self): 'batch_norm2d_0': [[0.37673383951187134], [0.44249194860458374]], 're_lu_0': [[0.44249194860458374], [0.25804123282432556]], 'max_pool2d_0': [[0.25804123282432556], [0.25804123282432556]], - 'linear_0': [[1.7058950662612915], [14.405526161193848], - [0.4373355209827423]], + 'linear_0': [ + [1.7058950662612915], + [14.405526161193848], + [0.4373355209827423], + ], 'add_0': [[1.7058950662612915, 0.0], [1.7058950662612915]], } def model_test(self, model, batch_num=-1, batch_size=8): model.eval() - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) eval_acc_top1_list = [] for batch_id, data in enumerate(test_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] - for x in data]).astype('int64').reshape(-1, 1) + x_data = np.array([x[0].reshape(1, 28, 28) for x in data]).astype( + 'float32' + ) + y_data = ( + np.array([x[1] for x in data]).astype('int64').reshape(-1, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) @@ -141,8 +155,11 @@ def model_test(self, model, batch_num=-1, batch_size=8): eval_acc_top1_list.append(float(acc_top1.numpy())) if batch_id % 50 == 0: - _logger.info("Test | At step {}: acc1 = {:}, acc5 = {:}".format( - batch_id, acc_top1.numpy(), acc_top5.numpy())) + _logger.info( + "Test | At step {}: acc1 = {:}, acc5 = {:}".format( + batch_id, acc_top1.numpy(), acc_top5.numpy() + ) + ) if batch_num > 0 and batch_id + 1 >= batch_num: break @@ -153,31 +170,39 @@ def model_test(self, model, batch_num=-1, batch_size=8): def program_test(self, program_path, batch_num=-1, batch_size=8): exe = paddle.static.Executor(paddle.CPUPlace()) - [inference_program, feed_target_names, fetch_targets - ] = (paddle.static.load_inference_model(program_path, exe)) - - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - - top1_correct_num = 0. - total_num = 0. + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(program_path, exe) + + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + + top1_correct_num = 0.0 + total_num = 0.0 for batch_id, data in enumerate(test_reader()): - img = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') + img = np.array([x[0].reshape(1, 28, 28) for x in data]).astype( + 'float32' + ) label = np.array([x[1] for x in data]).astype('int64') feed = {feed_target_names[0]: img} - results = exe.run(inference_program, - feed=feed, - fetch_list=fetch_targets) + results = exe.run( + inference_program, feed=feed, fetch_list=fetch_targets + ) pred = np.argmax(results[0], axis=1) top1_correct_num += np.sum(np.equal(pred, label)) total_num += len(img) if total_num % 50 == 49: - _logger.info("Test | Test num {}: acc1 = {:}".format( - total_num, top1_correct_num / total_num)) + _logger.info( + "Test | Test num {}: acc1 = {:}".format( + total_num, top1_correct_num / total_num + ) + ) if batch_num > 0 and batch_id + 1 >= batch_num: break @@ -189,8 +214,9 @@ def func_ptq(self): self.set_vars() # Load model - params_path = self.download_model(self.lenet_url, self.lenet_md5, - "lenet") + params_path = self.download_model( + self.lenet_url, self.lenet_md5, "lenet" + ) params_path += "/lenet_pretrained/lenet.pdparams" model = ImperativeLenet() @@ -198,25 +224,28 @@ def func_ptq(self): model.set_state_dict(model_state_dict) # Quantize, calibrate and save quant_model = self.ptq.quantize(model) - before_acc_top1 = self.model_test(quant_model, self.batch_num, - self.batch_size) + before_acc_top1 = self.model_test( + quant_model, self.batch_num, self.batch_size + ) input_spec = [ paddle.static.InputSpec(shape=[None, 1, 28, 28], dtype='float32') ] with tempfile.TemporaryDirectory(prefix="imperative_ptq_") as tmpdir: save_path = os.path.join(tmpdir, "model") - self.ptq.save_quantized_model(model=quant_model, - path=save_path, - input_spec=input_spec) + self.ptq.save_quantized_model( + model=quant_model, path=save_path, input_spec=input_spec + ) print('Quantized model saved in {%s}' % save_path) - after_acc_top1 = self.model_test(quant_model, self.batch_num, - self.batch_size) + after_acc_top1 = self.model_test( + quant_model, self.batch_num, self.batch_size + ) paddle.enable_static() - infer_acc_top1 = self.program_test(save_path, self.batch_num, - self.batch_size) + infer_acc_top1 = self.program_test( + save_path, self.batch_num, self.batch_size + ) paddle.disable_static() # Check @@ -224,11 +253,15 @@ def func_ptq(self): print('After converted acc_top1: %s' % after_acc_top1) print('Infer acc_top1: %s' % infer_acc_top1) - self.assertTrue(after_acc_top1 >= self.eval_acc_top1, - msg="The test acc {%f} is less than {%f}." % - (after_acc_top1, self.eval_acc_top1)) - self.assertTrue(infer_acc_top1 >= after_acc_top1, - msg='The acc is lower after converting model.') + self.assertTrue( + after_acc_top1 >= self.eval_acc_top1, + msg="The test acc {%f} is less than {%f}." + % (after_acc_top1, self.eval_acc_top1), + ) + self.assertTrue( + infer_acc_top1 >= after_acc_top1, + msg='The acc is lower after converting model.', + ) end_time = time.time() print("total time: %ss \n" % (end_time - start_time)) @@ -240,15 +273,15 @@ def test_ptq(self): class TestImperativePTQfuse(TestImperativePTQ): - def func_ptq(self): start_time = time.time() self.set_vars() # Load model - params_path = self.download_model(self.lenet_url, self.lenet_md5, - "lenet") + params_path = self.download_model( + self.lenet_url, self.lenet_md5, "lenet" + ) params_path += "/lenet_pretrained/lenet.pdparams" model = ImperativeLenet() @@ -259,27 +292,32 @@ def func_ptq(self): quant_model = self.ptq.quantize(model, fuse=True, fuse_list=f_l) for name, layer in quant_model.named_sublayers(): if name in f_l: - assert not (isinstance(layer, nn.BatchNorm1D) - or isinstance(layer, nn.BatchNorm2D)) - before_acc_top1 = self.model_test(quant_model, self.batch_num, - self.batch_size) + assert not ( + isinstance(layer, nn.BatchNorm1D) + or isinstance(layer, nn.BatchNorm2D) + ) + before_acc_top1 = self.model_test( + quant_model, self.batch_num, self.batch_size + ) input_spec = [ paddle.static.InputSpec(shape=[None, 1, 28, 28], dtype='float32') ] with tempfile.TemporaryDirectory(prefix="imperative_ptq_") as tmpdir: save_path = os.path.join(tmpdir, "model") - self.ptq.save_quantized_model(model=quant_model, - path=save_path, - input_spec=input_spec) + self.ptq.save_quantized_model( + model=quant_model, path=save_path, input_spec=input_spec + ) print('Quantized model saved in {%s}' % save_path) - after_acc_top1 = self.model_test(quant_model, self.batch_num, - self.batch_size) + after_acc_top1 = self.model_test( + quant_model, self.batch_num, self.batch_size + ) paddle.enable_static() - infer_acc_top1 = self.program_test(save_path, self.batch_num, - self.batch_size) + infer_acc_top1 = self.program_test( + save_path, self.batch_num, self.batch_size + ) paddle.disable_static() # Check @@ -287,15 +325,19 @@ def func_ptq(self): print('After converted acc_top1: %s' % after_acc_top1) print('Infer acc_top1: %s' % infer_acc_top1) - #Check whether the quant_model is correct after converting. - #The acc of quantized model should be higher than 0.95. - self.assertTrue(after_acc_top1 >= self.eval_acc_top1, - msg="The test acc {%f} is less than {%f}." % - (after_acc_top1, self.eval_acc_top1)) - #Check the saved infer_model.The acc of infer model - #should not be lower than the one of dygraph model. - self.assertTrue(infer_acc_top1 >= after_acc_top1, - msg='The acc is lower after converting model.') + # Check whether the quant_model is correct after converting. + # The acc of quantized model should be higher than 0.95. + self.assertTrue( + after_acc_top1 >= self.eval_acc_top1, + msg="The test acc {%f} is less than {%f}." + % (after_acc_top1, self.eval_acc_top1), + ) + # Check the saved infer_model.The acc of infer model + # should not be lower than the one of dygraph model. + self.assertTrue( + infer_acc_top1 >= after_acc_top1, + msg='The acc is lower after converting model.', + ) end_time = time.time() print("total time: %ss \n" % (end_time - start_time)) @@ -307,7 +349,6 @@ def test_ptq(self): class TestImperativePTQHist(TestImperativePTQ): - def set_vars(self): config = PTQConfig(HistQuantizer(), AbsmaxQuantizer()) self.ptq = ImperativePTQ(config) @@ -317,19 +358,24 @@ def set_vars(self): self.eval_acc_top1 = 0.98 self.gt_thresholds = { - 'conv2d_0': [[0.99853515625], [0.35732391771364225], - [0.10933732241392136]], + 'conv2d_0': [ + [0.99853515625], + [0.35732391771364225], + [0.10933732241392136], + ], 'batch_norm2d_0': [[0.35732391771364225], [0.4291427868761275]], 're_lu_0': [[0.4291427868761275], [0.2359918110742001]], 'max_pool2d_0': [[0.2359918110742001], [0.25665526917146053]], - 'linear_0': [[1.7037603475152991], [14.395224522473026], - [0.4373355209827423]], + 'linear_0': [ + [1.7037603475152991], + [14.395224522473026], + [0.4373355209827423], + ], 'add_0': [[1.7037603475152991, 0.0], [1.7037603475152991]], } class TestImperativePTQKL(TestImperativePTQ): - def set_vars(self): config = PTQConfig(KLQuantizer(), PerChannelAbsmaxQuantizer()) self.ptq = ImperativePTQ(config) @@ -339,13 +385,20 @@ def set_vars(self): self.eval_acc_top1 = 0.98 conv2d_1_wt_thresholds = [ - 0.18116560578346252, 0.17079241573810577, 0.1702047884464264, - 0.179476797580719, 0.1454375684261322, 0.22981858253479004 + 0.18116560578346252, + 0.17079241573810577, + 0.1702047884464264, + 0.179476797580719, + 0.1454375684261322, + 0.22981858253479004, ] self.gt_thresholds = { 'conv2d_0': [[0.99267578125], [0.37695913558696836]], - 'conv2d_1': [[0.19189296757394914], [0.24514256547263358], - [conv2d_1_wt_thresholds]], + 'conv2d_1': [ + [0.19189296757394914], + [0.24514256547263358], + [conv2d_1_wt_thresholds], + ], 'batch_norm2d_0': [[0.37695913558696836], [0.27462541429440535]], 're_lu_0': [[0.27462541429440535], [0.19189296757394914]], 'max_pool2d_0': [[0.19189296757394914], [0.19189296757394914]], diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py index a507fbe416177..e037cea8f558c 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py @@ -29,7 +29,10 @@ from paddle.nn import Linear, Conv2D, Softmax, Conv2DTranspose from paddle.fluid.log_helper import get_logger from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX -from paddle.nn.quant.quant_layers import QuantizedConv2D, QuantizedConv2DTranspose +from paddle.nn.quant.quant_layers import ( + QuantizedConv2D, + QuantizedConv2DTranspose, +) from paddle.fluid.framework import _test_eager_guard from imperative_test_utils import fix_model_dict, ImperativeLenet @@ -39,9 +42,9 @@ if core.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class TestImperativeQat(unittest.TestCase): @@ -67,26 +70,28 @@ def func_qat(self): weight_quantize_type=self.weight_quantize_type, activation_quantize_type=self.activation_quantize_type, fuse_conv_bn=self.fuse_conv_bn, - onnx_format=self.onnx_format) + onnx_format=self.onnx_format, + ) with fluid.dygraph.guard(): # For CI coverage - conv1 = Conv2D(in_channels=3, - out_channels=2, - kernel_size=3, - stride=1, - padding=1, - padding_mode='replicate') + conv1 = Conv2D( + in_channels=3, + out_channels=2, + kernel_size=3, + stride=1, + padding=1, + padding_mode='replicate', + ) quant_conv1 = QuantizedConv2D(conv1) data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32') quant_conv1(fluid.dygraph.to_variable(data)) conv_transpose = Conv2DTranspose(4, 6, (3, 3)) quant_conv_transpose = QuantizedConv2DTranspose(conv_transpose) - x_var = paddle.uniform((2, 4, 8, 8), - dtype='float32', - min=-1.0, - max=1.0) + x_var = paddle.uniform( + (2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0 + ) quant_conv_transpose(x_var) seed = 1 @@ -97,23 +102,29 @@ def func_qat(self): lenet = ImperativeLenet() lenet = fix_model_dict(lenet) imperative_qat.quantize(lenet) - adam = AdamOptimizer(learning_rate=0.001, - parameter_list=lenet.parameters()) + adam = AdamOptimizer( + learning_rate=0.001, parameter_list=lenet.parameters() + ) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=32, - drop_last=True) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=32) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=32, drop_last=True + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=32 + ) epoch_num = 1 for epoch in range(epoch_num): lenet.train() for batch_id, data in enumerate(train_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = fluid.dygraph.to_variable(x_data) label = fluid.dygraph.to_variable(y_data) @@ -126,52 +137,65 @@ def func_qat(self): lenet.clear_gradients() if batch_id % 100 == 0: _logger.info( - "Train | At epoch {} step {}: loss = {:}, acc= {:}". - format(epoch, batch_id, avg_loss.numpy(), - acc.numpy())) + "Train | At epoch {} step {}: loss = {:}, acc= {:}".format( + epoch, batch_id, avg_loss.numpy(), acc.numpy() + ) + ) if batch_id == 500: # For shortening CI time break lenet.eval() eval_acc_top1_list = [] for batch_id, data in enumerate(test_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = fluid.dygraph.to_variable(x_data) label = fluid.dygraph.to_variable(y_data) out = lenet(img) - acc_top1 = fluid.layers.accuracy(input=out, - label=label, - k=1) - acc_top5 = fluid.layers.accuracy(input=out, - label=label, - k=5) + acc_top1 = fluid.layers.accuracy( + input=out, label=label, k=1 + ) + acc_top5 = fluid.layers.accuracy( + input=out, label=label, k=5 + ) if batch_id % 100 == 0: eval_acc_top1_list.append(float(acc_top1.numpy())) _logger.info( - "Test | At epoch {} step {}: acc1 = {:}, acc5 = {:}" - .format(epoch, batch_id, acc_top1.numpy(), - acc_top5.numpy())) + "Test | At epoch {} step {}: acc1 = {:}, acc5 = {:}".format( + epoch, + batch_id, + acc_top1.numpy(), + acc_top5.numpy(), + ) + ) # check eval acc eval_acc_top1 = sum(eval_acc_top1_list) / len( - eval_acc_top1_list) + eval_acc_top1_list + ) print('eval_acc_top1', eval_acc_top1) - self.assertTrue(eval_acc_top1 > 0.9, - msg="The test acc {%f} is less than 0.9." % - eval_acc_top1) + self.assertTrue( + eval_acc_top1 > 0.9, + msg="The test acc {%f} is less than 0.9." % eval_acc_top1, + ) # test the correctness of `paddle.jit.save` data = next(test_reader()) - test_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] - for x in data]).astype('int64').reshape(-1, 1) + test_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]).astype('int64').reshape(-1, 1) + ) test_img = fluid.dygraph.to_variable(test_data) label = fluid.dygraph.to_variable(y_data) lenet.eval() @@ -184,9 +208,11 @@ def func_qat(self): layer=lenet, path=os.path.join(tmpdir, "lenet"), input_spec=[ - paddle.static.InputSpec(shape=[None, 1, 28, 28], - dtype='float32') - ]) + paddle.static.InputSpec( + shape=[None, 1, 28, 28], dtype='float32' + ) + ], + ) print('Quantized model saved in %s' % tmpdir) if core.is_compiled_with_cuda(): @@ -194,15 +220,21 @@ def func_qat(self): else: place = core.CPUPlace() exe = fluid.Executor(place) - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model( - dirname=tmpdir, - executor=exe, - model_filename="lenet" + INFER_MODEL_SUFFIX, - params_filename="lenet" + INFER_PARAMS_SUFFIX) - quant_out, = exe.run(inference_program, - feed={feed_target_names[0]: test_data}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + dirname=tmpdir, + executor=exe, + model_filename="lenet" + INFER_MODEL_SUFFIX, + params_filename="lenet" + INFER_PARAMS_SUFFIX, + ) + (quant_out,) = exe.run( + inference_program, + feed={feed_target_names[0]: test_data}, + fetch_list=fetch_targets, + ) paddle.disable_static() quant_out = fluid.dygraph.to_variable(quant_out) quant_acc = fluid.layers.accuracy(quant_out, label).numpy() @@ -217,7 +249,6 @@ def test_qat(self): class TestImperativeQatONNXFormat(unittest.TestCase): - def set_vars(self): self.weight_quantize_type = 'abs_max' self.activation_quantize_type = 'moving_average_abs_max' diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_amp.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_amp.py index b92d19cab2514..ee0edb445e708 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_amp.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_amp.py @@ -33,9 +33,9 @@ if paddle.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class TestImperativeQatAmp(unittest.TestCase): @@ -46,12 +46,14 @@ class TestImperativeQatAmp(unittest.TestCase): @classmethod def setUpClass(cls): cls.root_path = tempfile.TemporaryDirectory( - prefix="imperative_qat_amp_") + prefix="imperative_qat_amp_" + ) cls.save_path = os.path.join(cls.root_path.name, "model") cls.download_path = 'dygraph_int8/download' - cls.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + - cls.download_path) + cls.cache_folder = os.path.expanduser( + '~/.cache/paddle/dataset/' + cls.download_path + ) cls.lenet_url = "https://paddle-inference-dist.cdn.bcebos.com/int8/unittest_model_data/lenet_pretrained.tar.gz" cls.lenet_md5 = "953b802fb73b52fae42896e3c24f0afb" @@ -68,7 +70,8 @@ def tearDownClass(cls): def cache_unzipping(self, target_folder, zip_path): if not os.path.exists(target_folder): cmd = 'mkdir {0} && tar xf {1} -C {0}'.format( - target_folder, zip_path) + target_folder, zip_path + ) os.system(cmd) def download_model(self, data_url, data_md5, folder_name): @@ -93,17 +96,21 @@ def set_vars(self): def model_train(self, model, batch_num=-1, batch_size=32, use_amp=False): model.train() - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size) - adam = paddle.optimizer.Adam(learning_rate=0.001, - parameters=model.parameters()) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size + ) + adam = paddle.optimizer.Adam( + learning_rate=0.001, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=500) for batch_id, data in enumerate(train_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] - for x in data]).astype('int64').reshape(-1, 1) + x_data = np.array([x[0].reshape(1, 28, 28) for x in data]).astype( + 'float32' + ) + y_data = ( + np.array([x[1] for x in data]).astype('int64').reshape(-1, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) @@ -130,8 +137,11 @@ def model_train(self, model, batch_num=-1, batch_size=32, use_amp=False): model.clear_gradients() if batch_id % 100 == 0: - _logger.info("Train | step {}: loss = {:}, acc= {:}".format( - batch_id, avg_loss.numpy(), acc.numpy())) + _logger.info( + "Train | step {}: loss = {:}, acc= {:}".format( + batch_id, avg_loss.numpy(), acc.numpy() + ) + ) if batch_num > 0 and batch_id + 1 >= batch_num: break @@ -139,15 +149,18 @@ def model_train(self, model, batch_num=-1, batch_size=32, use_amp=False): def model_test(self, model, batch_num=-1, batch_size=32, use_amp=False): model.eval() - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) acc_top1_list = [] for batch_id, data in enumerate(test_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] - for x in data]).astype('int64').reshape(-1, 1) + x_data = np.array([x[0].reshape(1, 28, 28) for x in data]).astype( + 'float32' + ) + y_data = ( + np.array([x[1] for x in data]).astype('int64').reshape(-1, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) @@ -159,8 +172,11 @@ def model_test(self, model, batch_num=-1, batch_size=32, use_amp=False): acc_top1_list.append(float(acc_top1.numpy())) if batch_id % 100 == 0: - _logger.info("Test | At step {}: acc1 = {:}, acc5 = {:}".format( - batch_id, acc_top1.numpy(), acc_top5.numpy())) + _logger.info( + "Test | At step {}: acc1 = {:}, acc5 = {:}".format( + batch_id, acc_top1.numpy(), acc_top5.numpy() + ) + ) if batch_num > 0 and batch_id + 1 >= batch_num: break @@ -173,8 +189,9 @@ def ptq(self): self.set_vars() - params_path = self.download_model(self.lenet_url, self.lenet_md5, - "lenet") + params_path = self.download_model( + self.lenet_url, self.lenet_md5, "lenet" + ) params_path += "/lenet_pretrained/lenet.pdparams" with fluid.dygraph.guard(): @@ -183,24 +200,31 @@ def ptq(self): model.set_state_dict(model_state_dict) _logger.info("Test fp32 model") - fp32_acc_top1 = self.model_test(model, self.test_batch_num, - self.test_batch_size) + fp32_acc_top1 = self.model_test( + model, self.test_batch_num, self.test_batch_size + ) self.qat.quantize(model) use_amp = True - self.model_train(model, self.train_batch_num, self.train_batch_size, - use_amp) + self.model_train( + model, self.train_batch_num, self.train_batch_size, use_amp + ) _logger.info("Test int8 model") - int8_acc_top1 = self.model_test(model, self.test_batch_num, - self.test_batch_size, use_amp) - - _logger.info('fp32_acc_top1: %f, int8_acc_top1: %f' % - (fp32_acc_top1, int8_acc_top1)) - self.assertTrue(int8_acc_top1 > fp32_acc_top1 - 0.01, - msg='fp32_acc_top1: %f, int8_acc_top1: %f' % - (fp32_acc_top1, int8_acc_top1)) + int8_acc_top1 = self.model_test( + model, self.test_batch_num, self.test_batch_size, use_amp + ) + + _logger.info( + 'fp32_acc_top1: %f, int8_acc_top1: %f' + % (fp32_acc_top1, int8_acc_top1) + ) + self.assertTrue( + int8_acc_top1 > fp32_acc_top1 - 0.01, + msg='fp32_acc_top1: %f, int8_acc_top1: %f' + % (fp32_acc_top1, int8_acc_top1), + ) input_spec = [ paddle.static.InputSpec(shape=[None, 1, 28, 28], dtype='float32') diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_channelwise.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_channelwise.py index 30ea0346d8bbb..45ea756ce27ab 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_channelwise.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_channelwise.py @@ -31,13 +31,12 @@ if core.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class TestImperativeQatChannelWise(TestImperativeQat): - def set_vars(self): self.weight_quantize_type = 'channel_wise_abs_max' self.activation_quantize_type = 'moving_average_abs_max' @@ -48,7 +47,6 @@ def set_vars(self): class TestImperativeQatChannelWiseONNXFormat(TestImperativeQat): - def set_vars(self): self.weight_quantize_type = 'channel_wise_abs_max' self.activation_quantize_type = 'moving_average_abs_max' diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_fuse.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_fuse.py index 898d89f0142b4..dc1dee13cf81f 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_fuse.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_fuse.py @@ -31,13 +31,12 @@ if core.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class TestImperativeQatfuseBN(TestImperativeQat): - def set_vars(self): self.weight_quantize_type = 'abs_max' self.activation_quantize_type = 'moving_average_abs_max' diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_lsq.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_lsq.py index 8269abbab4dee..7e94c8dedd55a 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_lsq.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_lsq.py @@ -23,14 +23,21 @@ import paddle import paddle.fluid as fluid from paddle.fluid import core -from paddle.fluid.optimizer import SGDOptimizer, AdamOptimizer, MomentumOptimizer +from paddle.fluid.optimizer import ( + SGDOptimizer, + AdamOptimizer, + MomentumOptimizer, +) from paddle.fluid.contrib.slim.quantization import ImperativeQuantAware from paddle.fluid.dygraph.container import Sequential from paddle.nn import ReLU, ReLU6, LeakyReLU, Sigmoid, Softmax, PReLU from paddle.nn import Linear, Conv2D, Softmax, BatchNorm2D, MaxPool2D from paddle.fluid.log_helper import get_logger from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX -from paddle.nn.quant.quant_layers import QuantizedConv2D, QuantizedConv2DTranspose +from paddle.nn.quant.quant_layers import ( + QuantizedConv2D, + QuantizedConv2DTranspose, +) from paddle.fluid.framework import _test_eager_guard from imperative_test_utils import fix_model_dict @@ -40,13 +47,12 @@ if core.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class ImperativeLenet(fluid.dygraph.Layer): - def __init__(self, num_classes=10): super(ImperativeLenet, self).__init__() conv2d_w1_attr = fluid.ParamAttr(name="conv2d_w_1") @@ -59,36 +65,55 @@ def __init__(self, num_classes=10): fc_b2_attr = fluid.ParamAttr(name="fc_b_2") fc_b3_attr = fluid.ParamAttr(name="fc_b_3") self.features = Sequential( - Conv2D(in_channels=1, - out_channels=6, - kernel_size=3, - stride=1, - padding=1, - weight_attr=conv2d_w1_attr, - bias_attr=False), BatchNorm2D(6), ReLU(), + Conv2D( + in_channels=1, + out_channels=6, + kernel_size=3, + stride=1, + padding=1, + weight_attr=conv2d_w1_attr, + bias_attr=False, + ), + BatchNorm2D(6), + ReLU(), + MaxPool2D(kernel_size=2, stride=2), + Conv2D( + in_channels=6, + out_channels=16, + kernel_size=5, + stride=1, + padding=0, + weight_attr=conv2d_w2_attr, + bias_attr=conv2d_b2_attr, + ), + BatchNorm2D(16), + PReLU(), MaxPool2D(kernel_size=2, stride=2), - Conv2D(in_channels=6, - out_channels=16, - kernel_size=5, - stride=1, - padding=0, - weight_attr=conv2d_w2_attr, - bias_attr=conv2d_b2_attr), BatchNorm2D(16), PReLU(), - MaxPool2D(kernel_size=2, stride=2)) + ) self.fc = Sequential( - Linear(in_features=400, - out_features=120, - weight_attr=fc_w1_attr, - bias_attr=fc_b1_attr), LeakyReLU(), - Linear(in_features=120, - out_features=84, - weight_attr=fc_w2_attr, - bias_attr=fc_b2_attr), Sigmoid(), - Linear(in_features=84, - out_features=num_classes, - weight_attr=fc_w3_attr, - bias_attr=fc_b3_attr), Softmax()) + Linear( + in_features=400, + out_features=120, + weight_attr=fc_w1_attr, + bias_attr=fc_b1_attr, + ), + LeakyReLU(), + Linear( + in_features=120, + out_features=84, + weight_attr=fc_w2_attr, + bias_attr=fc_b2_attr, + ), + Sigmoid(), + Linear( + in_features=84, + out_features=num_classes, + weight_attr=fc_w3_attr, + bias_attr=fc_b3_attr, + ), + Softmax(), + ) def forward(self, inputs): x = self.features(inputs) @@ -98,7 +123,6 @@ def forward(self, inputs): class TestImperativeQatLSQ(unittest.TestCase): - def set_vars(self): self.weight_quantize_type = 'channel_wise_lsq_weight' self.activation_quantize_type = 'lsq_act' @@ -111,7 +135,8 @@ def func_qat(self): imperative_qat = ImperativeQuantAware( weight_quantize_type=self.weight_quantize_type, activation_quantize_type=self.activation_quantize_type, - fuse_conv_bn=self.fuse_conv_bn) + fuse_conv_bn=self.fuse_conv_bn, + ) seed = 100 np.random.seed(seed) @@ -121,22 +146,26 @@ def func_qat(self): lenet = ImperativeLenet() lenet = fix_model_dict(lenet) imperative_qat.quantize(lenet) - optimizer = MomentumOptimizer(learning_rate=0.1, - parameter_list=lenet.parameters(), - momentum=0.9) + optimizer = MomentumOptimizer( + learning_rate=0.1, parameter_list=lenet.parameters(), momentum=0.9 + ) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=64, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=64, drop_last=True + ) test_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size=32) epoch_num = 2 for epoch in range(epoch_num): lenet.train() for batch_id, data in enumerate(train_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = fluid.dygraph.to_variable(x_data) label = fluid.dygraph.to_variable(y_data) @@ -151,42 +180,53 @@ def func_qat(self): if batch_id % 100 == 0: _logger.info( - "Train | At epoch {} step {}: loss = {:}, acc= {:}". - format(epoch, batch_id, avg_loss.numpy(), acc.numpy())) + "Train | At epoch {} step {}: loss = {:}, acc= {:}".format( + epoch, batch_id, avg_loss.numpy(), acc.numpy() + ) + ) lenet.eval() eval_acc_top1_list = [] with paddle.no_grad(): for batch_id, data in enumerate(test_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = fluid.dygraph.to_variable(x_data) label = fluid.dygraph.to_variable(y_data) out = lenet(img) - acc_top1 = fluid.layers.accuracy(input=out, - label=label, - k=1) - acc_top5 = fluid.layers.accuracy(input=out, - label=label, - k=5) + acc_top1 = fluid.layers.accuracy( + input=out, label=label, k=1 + ) + acc_top5 = fluid.layers.accuracy( + input=out, label=label, k=5 + ) if batch_id % 100 == 0: eval_acc_top1_list.append(float(acc_top1.numpy())) _logger.info( - "Test | At epoch {} step {}: acc1 = {:}, acc5 = {:}" - .format(epoch, batch_id, acc_top1.numpy(), - acc_top5.numpy())) + "Test | At epoch {} step {}: acc1 = {:}, acc5 = {:}".format( + epoch, + batch_id, + acc_top1.numpy(), + acc_top5.numpy(), + ) + ) # check eval acc eval_acc_top1 = sum(eval_acc_top1_list) / len(eval_acc_top1_list) print('eval_acc_top1', eval_acc_top1) - self.assertTrue(eval_acc_top1 > 0.9, - msg="The test acc {%f} is less than 0.9." % - eval_acc_top1) + self.assertTrue( + eval_acc_top1 > 0.9, + msg="The test acc {%f} is less than 0.9." % eval_acc_top1, + ) def test_qat(self): self.func_qat() diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_user_defined.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_user_defined.py index 2ae75744cf8a8..89e2362b51dd7 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_user_defined.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_user_defined.py @@ -32,21 +32,21 @@ os.environ["CPU_NUM"] = "1" -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class PACT(nn.Layer): - def __init__(self, init_value=20): super(PACT, self).__init__() alpha_attr = paddle.ParamAttr( name=self.full_name() + ".pact", - initializer=paddle.nn.initializer.Constant(value=init_value)) - self.alpha = self.create_parameter(shape=[1], - attr=alpha_attr, - dtype='float32') + initializer=paddle.nn.initializer.Constant(value=init_value), + ) + self.alpha = self.create_parameter( + shape=[1], attr=alpha_attr, dtype='float32' + ) def forward(self, x): out_left = paddle.nn.functional.relu(x - self.alpha) @@ -56,31 +56,30 @@ def forward(self, x): class CustomQAT(nn.Layer): - def __init__(self): super(CustomQAT, self).__init__() - attr = paddle.ParamAttr(initializer=paddle.nn.initializer.Constant( - value=1.0)) - self.u_param = self.create_parameter(shape=[1], - attr=attr, - dtype='float32') - self.l_param = self.create_parameter(shape=[1], - attr=attr, - dtype='float32') - self.alpha_param = self.create_parameter(shape=[1], - attr=attr, - dtype='float32') - self.upper = self.create_parameter(shape=[1], - attr=attr, - dtype='float32') + attr = paddle.ParamAttr( + initializer=paddle.nn.initializer.Constant(value=1.0) + ) + self.u_param = self.create_parameter( + shape=[1], attr=attr, dtype='float32' + ) + self.l_param = self.create_parameter( + shape=[1], attr=attr, dtype='float32' + ) + self.alpha_param = self.create_parameter( + shape=[1], attr=attr, dtype='float32' + ) + self.upper = self.create_parameter( + shape=[1], attr=attr, dtype='float32' + ) self.upper.stop_gradient = True - self.lower = self.create_parameter(shape=[1], - attr=attr, - dtype='float32') + self.lower = self.create_parameter( + shape=[1], attr=attr, dtype='float32' + ) self.lower.stop_gradient = True def forward(self, x): - def clip(x, upper, lower): x = x + paddle.nn.functional.relu(lower - x) x = x - paddle.nn.functional.relu(x - upper) @@ -111,7 +110,6 @@ def dequantize(x, lower_bound, delta, interval): class ModelForConv2dT(nn.Layer): - def __init__(self, num_classes=10): super(ModelForConv2dT, self).__init__() self.features = nn.Conv2DTranspose(4, 6, (3, 3)) @@ -125,29 +123,34 @@ def forward(self, inputs): class ImperativeLenet(paddle.nn.Layer): - def __init__(self, num_classes=10, classifier_activation='softmax'): super(ImperativeLenet, self).__init__() self.features = Sequential( - Conv2D(num_channels=1, - num_filters=6, - filter_size=3, - stride=1, - padding=1), + Conv2D( + num_channels=1, + num_filters=6, + filter_size=3, + stride=1, + padding=1, + ), Pool2D(pool_size=2, pool_type='max', pool_stride=2), - Conv2D(num_channels=6, - num_filters=16, - filter_size=5, - stride=1, - padding=0), - Pool2D(pool_size=2, pool_type='max', pool_stride=2)) + Conv2D( + num_channels=6, + num_filters=16, + filter_size=5, + stride=1, + padding=0, + ), + Pool2D(pool_size=2, pool_type='max', pool_stride=2), + ) self.fc = Sequential( Linear(input_dim=400, output_dim=120), Linear(input_dim=120, output_dim=84), - Linear(input_dim=84, - output_dim=num_classes, - act=classifier_activation)) + Linear( + input_dim=84, output_dim=num_classes, act=classifier_activation + ), + ) def forward(self, inputs): x = self.features(inputs) @@ -158,7 +161,6 @@ def forward(self, inputs): class TestUserDefinedActPreprocess(unittest.TestCase): - def setUp(self): _logger.info("test act_preprocess") self.imperative_qat = ImperativeQuantAware(act_preprocess_layer=PACT) @@ -178,9 +180,13 @@ def func_quant_aware_training(self): if name.endswith("bias"): value = np.zeros_like(p_value).astype('float32') else: - value = np.random.normal( - loc=0.0, scale=0.01, - size=np.product(p_shape)).reshape(p_shape).astype('float32') + value = ( + np.random.normal( + loc=0.0, scale=0.01, size=np.product(p_shape) + ) + .reshape(p_shape) + .astype('float32') + ) fixed_state[name] = value param_init_map[param.name] = value lenet.set_dict(fixed_state) @@ -188,10 +194,10 @@ def func_quant_aware_training(self): imperative_qat.quantize(lenet) adam = Adam(learning_rate=0.001, parameters=lenet.parameters()) dynamic_loss_rec = [] - #for CI coverage + # for CI coverage conv_transpose = ModelForConv2dT() imperative_qat.quantize(conv_transpose) - x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.) + x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0) conv_transpose(x_var) def train(model): @@ -200,10 +206,14 @@ def train(model): for epoch in range(epoch_num): model.train() for batch_id, data in enumerate(train_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) @@ -216,19 +226,24 @@ def train(model): model.clear_gradients() if batch_id % 50 == 0: _logger.info( - "Train | At epoch {} step {}: loss = {:}, acc= {:}". - format(epoch, batch_id, avg_loss.numpy(), - acc.numpy())) + "Train | At epoch {} step {}: loss = {:}, acc= {:}".format( + epoch, batch_id, avg_loss.numpy(), acc.numpy() + ) + ) break def test(model): model.eval() avg_acc = [[], []] for batch_id, data in enumerate(test_reader()): - x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) @@ -241,11 +256,13 @@ def test(model): if batch_id % 100 == 0: _logger.info( "Test | step {}: acc1 = {:}, acc5 = {:}".format( - batch_id, acc_top1.numpy(), acc_top5.numpy())) + batch_id, acc_top1.numpy(), acc_top5.numpy() + ) + ) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=512, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=512, drop_last=True + ) test_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size=512) train(lenet) test(lenet) @@ -257,25 +274,23 @@ def test_quant_aware_training(self): class TestUserDefinedWeightPreprocess(TestUserDefinedActPreprocess): - def setUp(self): _logger.info("test weight_preprocess") self.imperative_qat = ImperativeQuantAware(weight_preprocess_layer=PACT) class TestUserDefinedActQuantize(TestUserDefinedActPreprocess): - def setUp(self): _logger.info("test act_quantize") self.imperative_qat = ImperativeQuantAware(act_quantize_layer=CustomQAT) class TestUserDefinedWeightQuantize(TestUserDefinedActPreprocess): - def setUp(self): _logger.info("test weight_quantize") self.imperative_qat = ImperativeQuantAware( - weight_quantize_layer=CustomQAT) + weight_quantize_layer=CustomQAT + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_skip_op.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_skip_op.py index f7608ee66b41e..131866095ad7b 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_skip_op.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_skip_op.py @@ -29,20 +29,23 @@ from paddle.fluid.dygraph.nn import Pool2D from paddle.fluid.log_helper import get_logger -from imperative_test_utils import fix_model_dict, train_lenet, ImperativeLenetWithSkipQuant +from imperative_test_utils import ( + fix_model_dict, + train_lenet, + ImperativeLenetWithSkipQuant, +) from paddle.fluid.framework import _test_eager_guard os.environ["CPU_NUM"] = "1" if core.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class TestImperativeOutSclae(unittest.TestCase): - def func_out_scale_acc(self): paddle.disable_static() seed = 1000 @@ -51,16 +54,17 @@ def func_out_scale_acc(self): qat = ImperativeQuantAware() np.random.seed(seed) - reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=512, - drop_last=True) + reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=512, drop_last=True + ) lenet = ImperativeLenetWithSkipQuant() lenet = fix_model_dict(lenet) qat.quantize(lenet) - adam = AdamOptimizer(learning_rate=lr, - parameter_list=lenet.parameters()) + adam = AdamOptimizer( + learning_rate=lr, parameter_list=lenet.parameters() + ) dynamic_loss_rec = [] lenet.train() loss_list = train_lenet(lenet, reader, adam) @@ -70,13 +74,15 @@ def func_out_scale_acc(self): path = "./save_dynamic_quant_infer_model/lenet" save_dir = "./save_dynamic_quant_infer_model" - qat.save_quantized_model(layer=lenet, - path=path, - input_spec=[ - paddle.static.InputSpec( - shape=[None, 1, 28, 28], - dtype='float32') - ]) + qat.save_quantized_model( + layer=lenet, + path=path, + input_spec=[ + paddle.static.InputSpec( + shape=[None, 1, 28, 28], dtype='float32' + ) + ], + ) paddle.enable_static() @@ -86,12 +92,16 @@ def func_out_scale_acc(self): place = core.CPUPlace() exe = fluid.Executor(place) - [inference_program, feed_target_names, - fetch_targets] = (fluid.io.load_inference_model( - dirname=save_dir, - executor=exe, - model_filename="lenet" + INFER_MODEL_SUFFIX, - params_filename="lenet" + INFER_PARAMS_SUFFIX)) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + dirname=save_dir, + executor=exe, + model_filename="lenet" + INFER_MODEL_SUFFIX, + params_filename="lenet" + INFER_PARAMS_SUFFIX, + ) model_ops = inference_program.global_block().ops conv2d_count, matmul_count = 0, 0 @@ -105,10 +115,12 @@ def func_out_scale_acc(self): conv2d_skip_count += 1 if conv2d_count > 0: self.assertTrue( - 'fake_quantize_dequantize' in model_ops[i - 1].type) + 'fake_quantize_dequantize' in model_ops[i - 1].type + ) else: self.assertTrue( - 'fake_quantize_dequantize' not in model_ops[i - 1].type) + 'fake_quantize_dequantize' not in model_ops[i - 1].type + ) conv2d_count += 1 if op.type == 'matmul': @@ -117,10 +129,12 @@ def func_out_scale_acc(self): matmul_skip_count += 1 if matmul_count > 0: self.assertTrue( - 'fake_quantize_dequantize' in model_ops[i - 1].type) + 'fake_quantize_dequantize' in model_ops[i - 1].type + ) else: self.assertTrue( - 'fake_quantize_dequantize' not in model_ops[i - 1].type) + 'fake_quantize_dequantize' not in model_ops[i - 1].type + ) matmul_count += 1 if find_conv2d: diff --git a/python/paddle/fluid/contrib/slim/tests/test_moving_average_abs_max_scale_op.py b/python/paddle/fluid/contrib/slim/tests/test_moving_average_abs_max_scale_op.py index 04814ed9fe129..1ec463192d919 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_moving_average_abs_max_scale_op.py +++ b/python/paddle/fluid/contrib/slim/tests/test_moving_average_abs_max_scale_op.py @@ -28,46 +28,51 @@ def init_data(batch_size=32, img_shape=[784], label_range=9): assert isinstance(img_shape, list) input_shape = [batch_size] + img_shape img = np.random.random(size=input_shape).astype(np.float32) - label = np.array( - [np.random.randint(0, label_range) for _ in range(batch_size)]).reshape( - (-1, 1)).astype("int64") + label = ( + np.array([np.random.randint(0, label_range) for _ in range(batch_size)]) + .reshape((-1, 1)) + .astype("int64") + ) return img, label class TestMovingAverageAbsMaxScaleOp(unittest.TestCase): - def check_backward(self, use_cuda): main_program = fluid.Program() startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): - image = fluid.layers.data(name='image', - shape=[784], - dtype='float32') + image = fluid.layers.data( + name='image', shape=[784], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') fc_tmp = fluid.layers.fc(image, size=10, act='softmax') out_scale = quant_layers.MovingAverageAbsMaxScale( - name=fc_tmp.name, dtype=fc_tmp.dtype) + name=fc_tmp.name, dtype=fc_tmp.dtype + ) fc_tmp_1 = out_scale(fc_tmp) cross_entropy = fluid.layers.softmax_with_cross_entropy( - fc_tmp, label) + fc_tmp, label + ) loss = fluid.layers.reduce_mean(cross_entropy) sgd = fluid.optimizer.SGD(learning_rate=1e-3) sgd.minimize(loss) moving_average_abs_max_scale_ops = [ - op for op in main_program.blocks[0].ops + op + for op in main_program.blocks[0].ops if op.type == u'moving_average_abs_max_scale' ] - assert len( - moving_average_abs_max_scale_ops - ) == 1, "The number of moving_average_abs_max_scale_ops should be 1." + assert ( + len(moving_average_abs_max_scale_ops) == 1 + ), "The number of moving_average_abs_max_scale_ops should be 1." place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup_program) binary = fluid.compiler.CompiledProgram( - main_program).with_data_parallel(loss_name=loss.name) + main_program + ).with_data_parallel(loss_name=loss.name) img, label = init_data() feed_dict = {"image": img, "label": label} diff --git a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_lstm_model.py b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_lstm_model.py index 4c55f58e05e05..7eb7f4d479e26 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_lstm_model.py +++ b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_lstm_model.py @@ -34,19 +34,23 @@ class TestPostTrainingQuantization(unittest.TestCase): - def setUp(self): self.download_path = 'int8/download' - self.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + - self.download_path) + self.cache_folder = os.path.expanduser( + '~/.cache/paddle/dataset/' + self.download_path + ) self.root_path = tempfile.TemporaryDirectory() - self.int8_model_path = os.path.join(self.root_path.name, - "post_training_quantization") + self.int8_model_path = os.path.join( + self.root_path.name, "post_training_quantization" + ) try: os.system("mkdir -p " + self.int8_model_path) except Exception as e: - print("Failed to create {} due to {}".format( - self.int8_model_path, str(e))) + print( + "Failed to create {} due to {}".format( + self.int8_model_path, str(e) + ) + ) sys.exit(-1) def tearDown(self): @@ -55,7 +59,8 @@ def tearDown(self): def cache_unzipping(self, target_folder, zip_path): if not os.path.exists(target_folder): cmd = 'mkdir {0} && tar xf {1} -C {0}'.format( - target_folder, zip_path) + target_folder, zip_path + ) os.system(cmd) def download_model(self, data_url, data_md5, folder_name): @@ -69,7 +74,6 @@ def download_model(self, data_url, data_md5, folder_name): return data_cache_folder def get_batch_reader(self, data_path, place): - def reader(): with open(data_path, 'rb') as in_file: while True: @@ -83,13 +87,15 @@ def reader(): label = in_file.read(4 * label_len) label = np.frombuffer(label, dtype=np.int32).reshape( - [len(label) // 4]) + [len(label) // 4] + ) if label.shape[0] != 1 or label[0] > 6350: continue feat = in_file.read(4 * seq_len * 8) feat = np.frombuffer(feat, dtype=np.float32).reshape( - [len(feat) // 4 // 8, 8]) + [len(feat) // 4 // 8, 8] + ) lod_feat = [feat.shape[0]] minputs = fluid.create_lod_tensor(feat, [lod_feat], place) @@ -98,7 +104,6 @@ def reader(): return reader def get_simple_reader(self, data_path, place): - def reader(): with open(data_path, 'rb') as in_file: while True: @@ -112,13 +117,15 @@ def reader(): label = in_file.read(4 * label_len) label = np.frombuffer(label, dtype=np.int32).reshape( - [len(label) // 4]) + [len(label) // 4] + ) if label.shape[0] != 1 or label[0] > 6350: continue feat = in_file.read(4 * seq_len * 8) feat = np.frombuffer(feat, dtype=np.float32).reshape( - [len(feat) // 4 // 8, 8]) + [len(feat) // 4 // 8, 8] + ) lod_feat = [feat.shape[0]] minputs = fluid.create_lod_tensor(feat, [lod_feat], place) @@ -130,8 +137,11 @@ def run_program(self, model_path, data_path, infer_iterations): print("test model path:" + model_path) place = fluid.CPUPlace() exe = fluid.Executor(place) - [infer_program, feed_dict, fetch_targets] = \ - fluid.io.load_inference_model(model_path, exe) + [ + infer_program, + feed_dict, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) val_reader = self.get_simple_reader(data_path, place) @@ -140,10 +150,12 @@ def run_program(self, model_path, data_path, infer_iterations): periods = [] for batch_id, (data, label) in enumerate(val_reader()): t1 = time.time() - cls_out, ctc_out = exe.run(infer_program, - feed={feed_dict[0]: data}, - fetch_list=fetch_targets, - return_numpy=False) + cls_out, ctc_out = exe.run( + infer_program, + feed={feed_dict[0]: data}, + fetch_list=fetch_targets, + return_numpy=False, + ) t2 = time.time() periods.append(t2 - t1) @@ -161,86 +173,117 @@ def run_program(self, model_path, data_path, infer_iterations): acc = right_num / all_num return (latency, acc) - def generate_quantized_model(self, - model_path, - data_path, - algo="KL", - round_type="round", - quantizable_op_type=["conv2d"], - is_full_quantize=False, - is_use_cache_file=False, - is_optimize_model=False, - batch_size=10, - batch_nums=10, - onnx_format=False): + def generate_quantized_model( + self, + model_path, + data_path, + algo="KL", + round_type="round", + quantizable_op_type=["conv2d"], + is_full_quantize=False, + is_use_cache_file=False, + is_optimize_model=False, + batch_size=10, + batch_nums=10, + onnx_format=False, + ): place = fluid.CPUPlace() exe = fluid.Executor(place) scope = fluid.global_scope() batch_generator = self.get_batch_reader(data_path, place) - ptq = PostTrainingQuantization(executor=exe, - model_dir=model_path, - batch_generator=batch_generator, - batch_nums=batch_nums, - algo=algo, - quantizable_op_type=quantizable_op_type, - round_type=round_type, - is_full_quantize=is_full_quantize, - optimize_model=is_optimize_model, - onnx_format=onnx_format, - is_use_cache_file=is_use_cache_file) + ptq = PostTrainingQuantization( + executor=exe, + model_dir=model_path, + batch_generator=batch_generator, + batch_nums=batch_nums, + algo=algo, + quantizable_op_type=quantizable_op_type, + round_type=round_type, + is_full_quantize=is_full_quantize, + optimize_model=is_optimize_model, + onnx_format=onnx_format, + is_use_cache_file=is_use_cache_file, + ) ptq.quantize() if onnx_format: ptq._clip_extra = False ptq.save_quantized_model(self.int8_model_path) - def run_test(self, - model_name, - model_url, - model_md5, - data_name, - data_url, - data_md5, - algo, - round_type, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - infer_iterations, - quant_iterations, - onnx_format=False): + def run_test( + self, + model_name, + model_url, + model_md5, + data_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + infer_iterations, + quant_iterations, + onnx_format=False, + ): fp32_model_path = self.download_model(model_url, model_md5, model_name) fp32_model_path = os.path.join(fp32_model_path, model_name) data_path = self.download_model(data_url, data_md5, data_name) data_path = os.path.join(data_path, data_name) - print("Start FP32 inference for {0} on {1} samples ...".format( - model_name, infer_iterations)) - (fp32_latency, fp32_acc) = self.run_program(fp32_model_path, data_path, - infer_iterations) - - print("Start post training quantization for {0} on {1} samples ...". - format(model_name, quant_iterations)) - self.generate_quantized_model(fp32_model_path, data_path, algo, - round_type, quantizable_op_type, - is_full_quantize, is_use_cache_file, - is_optimize_model, 10, quant_iterations, - onnx_format) - - print("Start INT8 inference for {0} on {1} samples ...".format( - model_name, infer_iterations)) - (int8_latency, int8_acc) = self.run_program(self.int8_model_path, - data_path, infer_iterations) + print( + "Start FP32 inference for {0} on {1} samples ...".format( + model_name, infer_iterations + ) + ) + (fp32_latency, fp32_acc) = self.run_program( + fp32_model_path, data_path, infer_iterations + ) + + print( + "Start post training quantization for {0} on {1} samples ...".format( + model_name, quant_iterations + ) + ) + self.generate_quantized_model( + fp32_model_path, + data_path, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + 10, + quant_iterations, + onnx_format, + ) + + print( + "Start INT8 inference for {0} on {1} samples ...".format( + model_name, infer_iterations + ) + ) + (int8_latency, int8_acc) = self.run_program( + self.int8_model_path, data_path, infer_iterations + ) print("---Post training quantization of {} method---".format(algo)) - print("FP32 {0}: batch_size {1}, latency {2} s, acc {3}.".format( - model_name, 1, fp32_latency, fp32_acc)) - print("INT8 {0}: batch_size {1}, latency {2} s, acc1 {3}.\n".format( - model_name, 1, int8_latency, int8_acc)) + print( + "FP32 {0}: batch_size {1}, latency {2} s, acc {3}.".format( + model_name, 1, fp32_latency, fp32_acc + ) + ) + print( + "INT8 {0}: batch_size {1}, latency {2} s, acc1 {3}.\n".format( + model_name, 1, int8_latency, int8_acc + ) + ) sys.stdout.flush() delta_value = fp32_acc - int8_acc @@ -248,7 +291,6 @@ def run_test(self, class TestPostTrainingAvgForLSTM(TestPostTrainingQuantization): - def test_post_training_avg(self): model_name = "nlp_lstm_fp32_model" model_url = "https://paddle-inference-dist.cdn.bcebos.com/int8/unittest_model_data/nlp_lstm_fp32_model.tar.gz" @@ -265,14 +307,26 @@ def test_post_training_avg(self): diff_threshold = 0.02 infer_iterations = 100 quant_iterations = 10 - self.run_test(model_name, model_url, model_md5, data_name, data_url, - data_md5, algo, round_type, quantizable_op_type, - is_full_quantize, is_use_cache_file, is_optimize_model, - diff_threshold, infer_iterations, quant_iterations) + self.run_test( + model_name, + model_url, + model_md5, + data_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + infer_iterations, + quant_iterations, + ) class TestPostTrainingAvgForLSTMONNXFormat(TestPostTrainingQuantization): - def not_test_post_training_avg_onnx_format(self): model_name = "nlp_lstm_fp32_model" model_url = "https://paddle-inference-dist.cdn.bcebos.com/int8/unittest_model_data/nlp_lstm_fp32_model.tar.gz" @@ -290,22 +344,24 @@ def not_test_post_training_avg_onnx_format(self): infer_iterations = 100 quant_iterations = 10 onnx_format = True - self.run_test(model_name, - model_url, - model_md5, - data_name, - data_url, - data_md5, - algo, - round_type, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - infer_iterations, - quant_iterations, - onnx_format=onnx_format) + self.run_test( + model_name, + model_url, + model_md5, + data_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + infer_iterations, + quant_iterations, + onnx_format=onnx_format, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mnist.py b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mnist.py index 807bdbf8a9ab1..6ff54f7c97076 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mnist.py +++ b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mnist.py @@ -33,19 +33,23 @@ class TestPostTrainingQuantization(unittest.TestCase): - def setUp(self): self.root_path = tempfile.TemporaryDirectory() - self.int8_model_path = os.path.join(self.root_path.name, - "post_training_quantization") + self.int8_model_path = os.path.join( + self.root_path.name, "post_training_quantization" + ) self.download_path = 'int8/download' - self.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + - self.download_path) + self.cache_folder = os.path.expanduser( + '~/.cache/paddle/dataset/' + self.download_path + ) try: os.system("mkdir -p " + self.int8_model_path) except Exception as e: - print("Failed to create {} due to {}".format( - self.int8_model_path, str(e))) + print( + "Failed to create {} due to {}".format( + self.int8_model_path, str(e) + ) + ) sys.exit(-1) def tearDown(self): @@ -54,7 +58,8 @@ def tearDown(self): def cache_unzipping(self, target_folder, zip_path): if not os.path.exists(target_folder): cmd = 'mkdir {0} && tar xf {1} -C {0}'.format( - target_folder, zip_path) + target_folder, zip_path + ) os.system(cmd) def download_model(self, data_url, data_md5, folder_name): @@ -71,8 +76,11 @@ def run_program(self, model_path, batch_size, infer_iterations): print("test model path:" + model_path) place = fluid.CPUPlace() exe = fluid.Executor(place) - [infer_program, feed_dict, fetch_targets] = \ - fluid.io.load_inference_model(model_path, exe) + [ + infer_program, + feed_dict, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) val_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size) img_shape = [1, 28, 28] @@ -80,14 +88,17 @@ def run_program(self, model_path, batch_size, infer_iterations): cnt = 0 periods = [] for batch_id, data in enumerate(val_reader()): - image = np.array([x[0].reshape(img_shape) - for x in data]).astype("float32") + image = np.array([x[0].reshape(img_shape) for x in data]).astype( + "float32" + ) input_label = np.array([x[1] for x in data]).astype("int64") t1 = time.time() - out = exe.run(infer_program, - feed={feed_dict[0]: image}, - fetch_list=fetch_targets) + out = exe.run( + infer_program, + feed={feed_dict[0]: image}, + fetch_list=fetch_targets, + ) t2 = time.time() period = t2 - t1 periods.append(period) @@ -105,91 +116,117 @@ def run_program(self, model_path, batch_size, infer_iterations): acc1 = np.sum(test_info) / cnt return (throughput, latency, acc1) - def generate_quantized_model(self, - model_path, - algo="KL", - round_type="round", - quantizable_op_type=["conv2d"], - is_full_quantize=False, - is_use_cache_file=False, - is_optimize_model=False, - batch_size=10, - batch_nums=10, - onnx_format=False, - skip_tensor_list=None, - bias_correction=False): + def generate_quantized_model( + self, + model_path, + algo="KL", + round_type="round", + quantizable_op_type=["conv2d"], + is_full_quantize=False, + is_use_cache_file=False, + is_optimize_model=False, + batch_size=10, + batch_nums=10, + onnx_format=False, + skip_tensor_list=None, + bias_correction=False, + ): place = fluid.CPUPlace() exe = fluid.Executor(place) val_reader = paddle.dataset.mnist.train() - ptq = PostTrainingQuantization(executor=exe, - model_dir=model_path, - sample_generator=val_reader, - batch_size=batch_size, - batch_nums=batch_nums, - algo=algo, - quantizable_op_type=quantizable_op_type, - round_type=round_type, - is_full_quantize=is_full_quantize, - optimize_model=is_optimize_model, - bias_correction=bias_correction, - onnx_format=onnx_format, - skip_tensor_list=skip_tensor_list, - is_use_cache_file=is_use_cache_file) + ptq = PostTrainingQuantization( + executor=exe, + model_dir=model_path, + sample_generator=val_reader, + batch_size=batch_size, + batch_nums=batch_nums, + algo=algo, + quantizable_op_type=quantizable_op_type, + round_type=round_type, + is_full_quantize=is_full_quantize, + optimize_model=is_optimize_model, + bias_correction=bias_correction, + onnx_format=onnx_format, + skip_tensor_list=skip_tensor_list, + is_use_cache_file=is_use_cache_file, + ) ptq.quantize() ptq.save_quantized_model(self.int8_model_path) - def run_test(self, - model_name, - data_url, - data_md5, - algo, - round_type, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_size=10, - infer_iterations=10, - quant_iterations=5, - bias_correction=False, - onnx_format=False, - skip_tensor_list=None): + def run_test( + self, + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size=10, + infer_iterations=10, + quant_iterations=5, + bias_correction=False, + onnx_format=False, + skip_tensor_list=None, + ): origin_model_path = self.download_model(data_url, data_md5, model_name) origin_model_path = os.path.join(origin_model_path, model_name) - print("Start FP32 inference for {0} on {1} images ...".format( - model_name, infer_iterations * batch_size)) - (fp32_throughput, fp32_latency, - fp32_acc1) = self.run_program(origin_model_path, batch_size, - infer_iterations) - - print("Start INT8 post training quantization for {0} on {1} images ...". - format(model_name, quant_iterations * batch_size)) - self.generate_quantized_model(origin_model_path, algo, round_type, - quantizable_op_type, is_full_quantize, - is_use_cache_file, is_optimize_model, - batch_size, quant_iterations, onnx_format, - skip_tensor_list, bias_correction) - - print("Start INT8 inference for {0} on {1} images ...".format( - model_name, infer_iterations * batch_size)) - (int8_throughput, int8_latency, - int8_acc1) = self.run_program(self.int8_model_path, batch_size, - infer_iterations) + print( + "Start FP32 inference for {0} on {1} images ...".format( + model_name, infer_iterations * batch_size + ) + ) + (fp32_throughput, fp32_latency, fp32_acc1) = self.run_program( + origin_model_path, batch_size, infer_iterations + ) + + print( + "Start INT8 post training quantization for {0} on {1} images ...".format( + model_name, quant_iterations * batch_size + ) + ) + self.generate_quantized_model( + origin_model_path, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + batch_size, + quant_iterations, + onnx_format, + skip_tensor_list, + bias_correction, + ) + + print( + "Start INT8 inference for {0} on {1} images ...".format( + model_name, infer_iterations * batch_size + ) + ) + (int8_throughput, int8_latency, int8_acc1) = self.run_program( + self.int8_model_path, batch_size, infer_iterations + ) print("---Post training quantization of {} method---".format(algo)) print( - "FP32 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}." - .format(model_name, batch_size, fp32_throughput, fp32_latency, - fp32_acc1)) + "FP32 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}.".format( + model_name, batch_size, fp32_throughput, fp32_latency, fp32_acc1 + ) + ) print( - "INT8 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}.\n" - .format(model_name, batch_size, int8_throughput, int8_latency, - int8_acc1)) + "INT8 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}.\n".format( + model_name, batch_size, int8_throughput, int8_latency, int8_acc1 + ) + ) sys.stdout.flush() delta_value = fp32_acc1 - int8_acc1 @@ -197,10 +234,11 @@ def run_test(self, class TestPostTrainingKLForMnist(TestPostTrainingQuantization): - def test_post_training_kl(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "KL" round_type = "round" @@ -212,17 +250,29 @@ def test_post_training_kl(self): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, round_type, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold, batch_size, - infer_iterations, quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTraininghistForMnist(TestPostTrainingQuantization): - def test_post_training_hist(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "hist" round_type = "round" @@ -234,17 +284,29 @@ def test_post_training_hist(self): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, round_type, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold, batch_size, - infer_iterations, quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingmseForMnist(TestPostTrainingQuantization): - def test_post_training_mse(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "mse" round_type = "round" @@ -256,17 +318,29 @@ def test_post_training_mse(self): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, round_type, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold, batch_size, - infer_iterations, quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingemdForMnist(TestPostTrainingQuantization): - def test_post_training_mse(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "emd" round_type = "round" @@ -278,17 +352,29 @@ def test_post_training_mse(self): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, round_type, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold, batch_size, - infer_iterations, quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingavgForMnist(TestPostTrainingQuantization): - def test_post_training_avg(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "avg" round_type = "round" @@ -300,17 +386,29 @@ def test_post_training_avg(self): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, round_type, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold, batch_size, - infer_iterations, quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingAbsMaxForMnist(TestPostTrainingQuantization): - def test_post_training_abs_max(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "abs_max" round_type = "round" @@ -322,17 +420,29 @@ def test_post_training_abs_max(self): batch_size = 10 infer_iterations = 50 quant_iterations = 10 - self.run_test(model_name, data_url, data_md5, algo, round_type, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold, batch_size, - infer_iterations, quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingmseAdaroundForMnist(TestPostTrainingQuantization): - def test_post_training_mse(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "mse" round_type = "adaround" @@ -345,27 +455,30 @@ def test_post_training_mse(self): infer_iterations = 50 quant_iterations = 5 bias_correction = True - self.run_test(model_name, - data_url, - data_md5, - algo, - round_type, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_size, - infer_iterations, - quant_iterations, - bias_correction=bias_correction) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + bias_correction=bias_correction, + ) class TestPostTrainingKLAdaroundForMnist(TestPostTrainingQuantization): - def test_post_training_kl(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "KL" round_type = "adaround" @@ -377,17 +490,29 @@ def test_post_training_kl(self): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, round_type, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold, batch_size, - infer_iterations, quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingmseForMnistONNXFormat(TestPostTrainingQuantization): - def test_post_training_mse_onnx_format(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "mse" round_type = "round" @@ -400,28 +525,32 @@ def test_post_training_mse_onnx_format(self): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, - data_url, - data_md5, - algo, - round_type, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_size, - infer_iterations, - quant_iterations, - onnx_format=onnx_format) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + onnx_format=onnx_format, + ) class TestPostTrainingmseForMnistONNXFormatFullQuant( - TestPostTrainingQuantization): - + TestPostTrainingQuantization +): def test_post_training_mse_onnx_format_full_quant(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "mse" round_type = "round" @@ -434,27 +563,30 @@ def test_post_training_mse_onnx_format_full_quant(self): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, - data_url, - data_md5, - algo, - round_type, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_size, - infer_iterations, - quant_iterations, - onnx_format=onnx_format) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + onnx_format=onnx_format, + ) class TestPostTrainingavgForMnistSkipOP(TestPostTrainingQuantization): - def test_post_training_avg_skip_op(self): model_name = "mnist_model" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_model.tar.gz" + ) data_md5 = "be71d3997ec35ac2a65ae8a145e2887c" algo = "avg" round_type = "round" @@ -467,20 +599,22 @@ def test_post_training_avg_skip_op(self): infer_iterations = 50 quant_iterations = 5 skip_tensor_list = ["fc_0.w_0"] - self.run_test(model_name, - data_url, - data_md5, - algo, - round_type, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_size, - infer_iterations, - quant_iterations, - skip_tensor_list=skip_tensor_list) + self.run_test( + model_name, + data_url, + data_md5, + algo, + round_type, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + skip_tensor_list=skip_tensor_list, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mobilenetv1.py b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mobilenetv1.py index 41add5e8b8fb1..37daeab118629 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mobilenetv1.py +++ b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_mobilenetv1.py @@ -77,13 +77,14 @@ def process_image(sample, mode, color_jitter, rotate): return img, sample[1] -def _reader_creator(file_list, - mode, - shuffle=False, - color_jitter=False, - rotate=False, - data_dir=DATA_DIR): - +def _reader_creator( + file_list, + mode, + shuffle=False, + color_jitter=False, + rotate=False, + data_dir=DATA_DIR, +): def reader(): with open(file_list) as flist: full_lines = [line.strip() for line in flist] @@ -98,10 +99,9 @@ def reader(): continue yield img_path, int(label) - mapper = functools.partial(process_image, - mode=mode, - color_jitter=color_jitter, - rotate=rotate) + mapper = functools.partial( + process_image, mode=mode, color_jitter=color_jitter, rotate=rotate + ) return paddle.reader.xmap_readers(mapper, reader, THREAD, BUF_SIZE) @@ -112,11 +112,11 @@ def val(data_dir=DATA_DIR): class TestPostTrainingQuantization(unittest.TestCase): - def setUp(self): self.int8_download = 'int8/download' - self.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + - self.int8_download) + self.cache_folder = os.path.expanduser( + '~/.cache/paddle/dataset/' + self.int8_download + ) self.data_cache_folder = '' data_urls = [] data_md5s = [] @@ -129,29 +129,34 @@ def setUp(self): 'https://paddle-inference-dist.bj.bcebos.com/int8/ILSVRC2012_img_val.tar.gz.partab' ) data_md5s.append('1e9f15f64e015e58d6f9ec3210ed18b5') - self.data_cache_folder = self.download_data(data_urls, data_md5s, - "full_data", False) + self.data_cache_folder = self.download_data( + data_urls, data_md5s, "full_data", False + ) else: data_urls.append( 'http://paddle-inference-dist.bj.bcebos.com/int8/calibration_test_data.tar.gz' ) data_md5s.append('1b6c1c434172cca1bf9ba1e4d7a3157d') - self.data_cache_folder = self.download_data(data_urls, data_md5s, - "small_data", False) + self.data_cache_folder = self.download_data( + data_urls, data_md5s, "small_data", False + ) # reader/decorator.py requires the relative path to the data folder if not os.path.exists("./data/ILSVRC2012"): - cmd = 'rm -rf {0} && ln -s {1} {0}'.format("data", - self.data_cache_folder) + cmd = 'rm -rf {0} && ln -s {1} {0}'.format( + "data", self.data_cache_folder + ) os.system(cmd) self.batch_size = 1 if os.environ.get('DATASET') == 'full' else 50 - self.infer_iterations = 50000 if os.environ.get( - 'DATASET') == 'full' else 2 + self.infer_iterations = ( + 50000 if os.environ.get('DATASET') == 'full' else 2 + ) self.root_path = tempfile.TemporaryDirectory() - self.int8_model = os.path.join(self.root_path.name, - "post_training_quantization") + self.int8_model = os.path.join( + self.root_path.name, "post_training_quantization" + ) def tearDown(self): self.root_path.cleanup() @@ -159,7 +164,8 @@ def tearDown(self): def cache_unzipping(self, target_folder, zip_path): if not os.path.exists(target_folder): cmd = 'mkdir {0} && tar xf {1} -C {0}'.format( - target_folder, zip_path) + target_folder, zip_path + ) os.system(cmd) def download_data(self, data_urls, data_md5s, folder_name, is_model=True): @@ -171,13 +177,15 @@ def download_data(self, data_urls, data_md5s, folder_name, is_model=True): download(data_urls[i], self.int8_download, data_md5s[i]) file_names.append(data_urls[i].split('/')[-1]) - zip_path = os.path.join(self.cache_folder, - 'full_imagenet_val.tar.gz') + zip_path = os.path.join( + self.cache_folder, 'full_imagenet_val.tar.gz' + ) if not os.path.exists(zip_path): cat_command = 'cat' for file_name in file_names: - cat_command += ' ' + os.path.join(self.cache_folder, - file_name) + cat_command += ' ' + os.path.join( + self.cache_folder, file_name + ) cat_command += ' > ' + zip_path os.system(cat_command) @@ -197,8 +205,11 @@ def run_program(self, model_path, batch_size, infer_iterations): image_shape = [3, 224, 224] place = fluid.CPUPlace() exe = fluid.Executor(place) - [infer_program, feed_dict, fetch_targets] = \ - fluid.io.load_inference_model(model_path, exe) + [ + infer_program, + feed_dict, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) val_reader = paddle.batch(val(), batch_size) iterations = infer_iterations @@ -206,18 +217,18 @@ def run_program(self, model_path, batch_size, infer_iterations): cnt = 0 periods = [] for batch_id, data in enumerate(val_reader()): - image = np.array([x[0].reshape(image_shape) - for x in data]).astype("float32") + image = np.array([x[0].reshape(image_shape) for x in data]).astype( + "float32" + ) label = np.array([x[1] for x in data]).astype("int64") label = label.reshape([-1, 1]) t1 = time.time() - _, acc1, _ = exe.run(infer_program, - feed={ - feed_dict[0]: image, - feed_dict[1]: label - }, - fetch_list=fetch_targets) + _, acc1, _ = exe.run( + infer_program, + feed={feed_dict[0]: image, feed_dict[1]: label}, + fetch_list=fetch_targets, + ) t2 = time.time() period = t2 - t1 periods.append(period) @@ -236,22 +247,25 @@ def run_program(self, model_path, batch_size, infer_iterations): acc1 = np.sum(test_info) / cnt return (throughput, latency, acc1) - def generate_quantized_model(self, - model_path, - quantizable_op_type, - batch_size, - algo="KL", - round_type="round", - is_full_quantize=False, - is_use_cache_file=False, - is_optimize_model=False, - batch_nums=10, - onnx_format=False): + def generate_quantized_model( + self, + model_path, + quantizable_op_type, + batch_size, + algo="KL", + round_type="round", + is_full_quantize=False, + is_use_cache_file=False, + is_optimize_model=False, + batch_nums=10, + onnx_format=False, + ): try: os.system("mkdir " + self.int8_model) except Exception as e: - print("Failed to create {} due to {}".format( - self.int8_model, str(e))) + print( + "Failed to create {} due to {}".format(self.int8_model, str(e)) + ) sys.exit(-1) place = fluid.CPUPlace() @@ -259,68 +273,92 @@ def generate_quantized_model(self, scope = fluid.global_scope() val_reader = val() - ptq = PostTrainingQuantization(executor=exe, - sample_generator=val_reader, - model_dir=model_path, - batch_size=batch_size, - batch_nums=batch_nums, - algo=algo, - quantizable_op_type=quantizable_op_type, - round_type=round_type, - is_full_quantize=is_full_quantize, - optimize_model=is_optimize_model, - onnx_format=onnx_format, - is_use_cache_file=is_use_cache_file) + ptq = PostTrainingQuantization( + executor=exe, + sample_generator=val_reader, + model_dir=model_path, + batch_size=batch_size, + batch_nums=batch_nums, + algo=algo, + quantizable_op_type=quantizable_op_type, + round_type=round_type, + is_full_quantize=is_full_quantize, + optimize_model=is_optimize_model, + onnx_format=onnx_format, + is_use_cache_file=is_use_cache_file, + ) ptq.quantize() ptq.save_quantized_model(self.int8_model) - def run_test(self, - model, - algo, - round_type, - data_urls, - data_md5s, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - onnx_format=False, - batch_nums=10): + def run_test( + self, + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + onnx_format=False, + batch_nums=10, + ): infer_iterations = self.infer_iterations batch_size = self.batch_size model_cache_folder = self.download_data(data_urls, data_md5s, model) - print("Start FP32 inference for {0} on {1} images ...".format( - model, infer_iterations * batch_size)) + print( + "Start FP32 inference for {0} on {1} images ...".format( + model, infer_iterations * batch_size + ) + ) (fp32_throughput, fp32_latency, fp32_acc1) = self.run_program( - os.path.join(model_cache_folder, "model"), batch_size, - infer_iterations) - - print("Start INT8 post training quantization for {0} on {1} images ...". - format(model, batch_nums * batch_size)) - self.generate_quantized_model(os.path.join(model_cache_folder, "model"), - quantizable_op_type, batch_size, algo, - round_type, is_full_quantize, - is_use_cache_file, is_optimize_model, - batch_nums, onnx_format) - - print("Start INT8 inference for {0} on {1} images ...".format( - model, infer_iterations * batch_size)) - (int8_throughput, int8_latency, - int8_acc1) = self.run_program(self.int8_model, batch_size, - infer_iterations) + os.path.join(model_cache_folder, "model"), + batch_size, + infer_iterations, + ) + + print( + "Start INT8 post training quantization for {0} on {1} images ...".format( + model, batch_nums * batch_size + ) + ) + self.generate_quantized_model( + os.path.join(model_cache_folder, "model"), + quantizable_op_type, + batch_size, + algo, + round_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + batch_nums, + onnx_format, + ) + + print( + "Start INT8 inference for {0} on {1} images ...".format( + model, infer_iterations * batch_size + ) + ) + (int8_throughput, int8_latency, int8_acc1) = self.run_program( + self.int8_model, batch_size, infer_iterations + ) print("---Post training quantization of {} method---".format(algo)) print( - "FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}." - .format(model, batch_size, fp32_throughput, fp32_latency, - fp32_acc1)) + "FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.".format( + model, batch_size, fp32_throughput, fp32_latency, fp32_acc1 + ) + ) print( - "INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.\n" - .format(model, batch_size, int8_throughput, int8_latency, - int8_acc1)) + "INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.\n".format( + model, batch_size, int8_throughput, int8_latency, int8_acc1 + ) + ) sys.stdout.flush() delta_value = fp32_acc1 - int8_acc1 @@ -328,7 +366,6 @@ def run_test(self, class TestPostTrainingKLForMobilenetv1(TestPostTrainingQuantization): - def test_post_training_kl_mobilenetv1(self): model = "MobileNet-V1" algo = "KL" @@ -348,13 +385,21 @@ def test_post_training_kl_mobilenetv1(self): is_optimize_model = True diff_threshold = 0.025 batch_nums = 3 - self.run_test(model, algo, round_type, data_urls, data_md5s, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + ) class TestPostTrainingavgForMobilenetv1(TestPostTrainingQuantization): - def test_post_training_avg_mobilenetv1(self): model = "MobileNet-V1" algo = "avg" @@ -372,13 +417,21 @@ def test_post_training_avg_mobilenetv1(self): is_use_cache_file = False is_optimize_model = True diff_threshold = 0.025 - self.run_test(model, algo, round_type, data_urls, data_md5s, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + ) class TestPostTraininghistForMobilenetv1(TestPostTrainingQuantization): - def test_post_training_hist_mobilenetv1(self): model = "MobileNet-V1" algo = "hist" @@ -397,21 +450,22 @@ def test_post_training_hist_mobilenetv1(self): is_optimize_model = True diff_threshold = 0.03 batch_nums = 3 - self.run_test(model, - algo, - round_type, - data_urls, - data_md5s, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_nums=batch_nums) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_nums=batch_nums, + ) class TestPostTrainingAbsMaxForMobilenetv1(TestPostTrainingQuantization): - def test_post_training_abs_max_mobilenetv1(self): model = "MobileNet-V1" algo = "abs_max" @@ -429,13 +483,21 @@ def test_post_training_abs_max_mobilenetv1(self): is_optimize_model = False # The accuracy diff of post-training quantization (abs_max) maybe bigger diff_threshold = 0.05 - self.run_test(model, algo, round_type, data_urls, data_md5s, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + ) class TestPostTrainingAvgONNXFormatForMobilenetv1(TestPostTrainingQuantization): - def test_post_training_onnx_format_mobilenetv1(self): model = "MobileNet-V1" algo = "emd" @@ -455,18 +517,20 @@ def test_post_training_onnx_format_mobilenetv1(self): onnx_format = True diff_threshold = 0.05 batch_nums = 3 - self.run_test(model, - algo, - round_type, - data_urls, - data_md5s, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - onnx_format=onnx_format, - batch_nums=batch_nums) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + onnx_format=onnx_format, + batch_nums=batch_nums, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_program_resnet50.py b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_program_resnet50.py index 5854d40529d58..345853636a41b 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_program_resnet50.py +++ b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_program_resnet50.py @@ -23,8 +23,12 @@ import numpy as np import paddle.fluid as fluid from PIL import Image, ImageEnhance -from paddle.fluid.contrib.slim.quantization import PostTrainingQuantizationProgram -from test_post_training_quantization_mobilenetv1 import TestPostTrainingQuantization +from paddle.fluid.contrib.slim.quantization import ( + PostTrainingQuantizationProgram, +) +from test_post_training_quantization_mobilenetv1 import ( + TestPostTrainingQuantization, +) paddle.enable_static() @@ -76,13 +80,14 @@ def process_image(sample, mode, color_jitter, rotate): return img, sample[1] -def _reader_creator(file_list, - mode, - shuffle=False, - color_jitter=False, - rotate=False, - data_dir=DATA_DIR): - +def _reader_creator( + file_list, + mode, + shuffle=False, + color_jitter=False, + rotate=False, + data_dir=DATA_DIR, +): def reader(): with open(file_list) as flist: full_lines = [line.strip() for line in flist] @@ -97,10 +102,9 @@ def reader(): continue yield img_path, int(label) - mapper = functools.partial(process_image, - mode=mode, - color_jitter=color_jitter, - rotate=rotate) + mapper = functools.partial( + process_image, mode=mode, color_jitter=color_jitter, rotate=rotate + ) return paddle.reader.xmap_readers(mapper, reader, THREAD, BUF_SIZE) @@ -111,31 +115,33 @@ def val(data_dir=DATA_DIR): class TestPostTrainingQuantizationProgram(TestPostTrainingQuantization): - def run_program(self, model_path, batch_size, infer_iterations): image_shape = [3, 224, 224] place = fluid.CPUPlace() exe = fluid.Executor(place) - [infer_program, feed_dict, fetch_targets] = \ - fluid.io.load_inference_model(model_path, exe) + [ + infer_program, + feed_dict, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) val_reader = paddle.batch(val(), batch_size) iterations = infer_iterations test_info = [] cnt = 0 periods = [] for batch_id, data in enumerate(val_reader()): - image = np.array([x[0].reshape(image_shape) - for x in data]).astype("float32") + image = np.array([x[0].reshape(image_shape) for x in data]).astype( + "float32" + ) label = np.array([x[1] for x in data]).astype("int64") label = label.reshape([-1, 1]) t1 = time.time() - _, acc1, _ = exe.run(infer_program, - feed={ - feed_dict[0]: image, - feed_dict[1]: label - }, - fetch_list=fetch_targets) + _, acc1, _ = exe.run( + infer_program, + feed={feed_dict[0]: image, feed_dict[1]: label}, + fetch_list=fetch_targets, + ) t2 = time.time() period = t2 - t1 periods.append(period) @@ -152,10 +158,19 @@ def run_program(self, model_path, batch_size, infer_iterations): throughput = cnt / np.sum(periods) latency = np.average(periods) acc1 = np.sum(test_info) / cnt - [infer_program, feed_dict, fetch_targets] = \ - fluid.io.load_inference_model(model_path, exe) - return (throughput, latency, acc1, infer_program, feed_dict, - fetch_targets) + [ + infer_program, + feed_dict, + fetch_targets, + ] = fluid.io.load_inference_model(model_path, exe) + return ( + throughput, + latency, + acc1, + infer_program, + feed_dict, + fetch_targets, + ) def generate_quantized_model( self, @@ -173,25 +188,27 @@ def generate_quantized_model( try: os.system("mkdir " + self.int8_model) except Exception as e: - print("Failed to create {} due to {}".format( - self.int8_model, str(e))) + print( + "Failed to create {} due to {}".format(self.int8_model, str(e)) + ) sys.exit(-1) place = fluid.CPUPlace() exe = fluid.Executor(place) scope = fluid.global_scope() val_reader = val() - same_scale_tensor_list = [[ - 'batch_norm_3.tmp_2#/#1', 'batch_norm_4.tmp_2#*#1' - ], ['batch_norm_27.tmp_2', 'batch_norm_26.tmp_2'], - [ - 'test_scale_name_not_in_scale_dict1', - 'test_scale_name_not_in_scale_dict2' - ], - [ - 'test_scale_name_not_in_scale_dict1#/#1', - 'test_scale_name_not_in_scale_dict2#/#1' - ]] + same_scale_tensor_list = [ + ['batch_norm_3.tmp_2#/#1', 'batch_norm_4.tmp_2#*#1'], + ['batch_norm_27.tmp_2', 'batch_norm_26.tmp_2'], + [ + 'test_scale_name_not_in_scale_dict1', + 'test_scale_name_not_in_scale_dict2', + ], + [ + 'test_scale_name_not_in_scale_dict1#/#1', + 'test_scale_name_not_in_scale_dict2#/#1', + ], + ] ptq = PostTrainingQuantizationProgram( executor=exe, program=program, @@ -206,56 +223,86 @@ def generate_quantized_model( is_use_cache_file=is_use_cache_file, feed_list=feed_list, fetch_list=fetch_list, - same_scale_tensor_list=same_scale_tensor_list) + same_scale_tensor_list=same_scale_tensor_list, + ) ptq.quantize() ptq.save_quantized_model(self.int8_model) - def run_test(self, - model, - algo, - round_type, - data_urls, - data_md5s, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - onnx_format=False): + def run_test( + self, + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + onnx_format=False, + ): infer_iterations = self.infer_iterations batch_size = self.batch_size sample_iterations = self.sample_iterations model_cache_folder = self.download_data(data_urls, data_md5s, model) - print("Start FP32 inference for {0} on {1} images ...".format( - model, infer_iterations * batch_size)) - (fp32_throughput, fp32_latency, fp32_acc1, infer_program, feed_dict, - fetch_targets) = self.run_program( - os.path.join(model_cache_folder, "model"), batch_size, - infer_iterations) - print("Start INT8 post training quantization for {0} on {1} images ...". - format(model, sample_iterations * batch_size)) - self.generate_quantized_model(infer_program, quantizable_op_type, - feed_dict, fetch_targets, algo, - round_type, is_full_quantize, - is_use_cache_file, is_optimize_model, - onnx_format) - - print("Start INT8 inference for {0} on {1} images ...".format( - model, infer_iterations * batch_size)) - (int8_throughput, int8_latency, int8_acc1, _, _, - _) = self.run_program(self.int8_model, batch_size, infer_iterations) + print( + "Start FP32 inference for {0} on {1} images ...".format( + model, infer_iterations * batch_size + ) + ) + ( + fp32_throughput, + fp32_latency, + fp32_acc1, + infer_program, + feed_dict, + fetch_targets, + ) = self.run_program( + os.path.join(model_cache_folder, "model"), + batch_size, + infer_iterations, + ) + print( + "Start INT8 post training quantization for {0} on {1} images ...".format( + model, sample_iterations * batch_size + ) + ) + self.generate_quantized_model( + infer_program, + quantizable_op_type, + feed_dict, + fetch_targets, + algo, + round_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + onnx_format, + ) + + print( + "Start INT8 inference for {0} on {1} images ...".format( + model, infer_iterations * batch_size + ) + ) + (int8_throughput, int8_latency, int8_acc1, _, _, _) = self.run_program( + self.int8_model, batch_size, infer_iterations + ) print("---Post training quantization of {} method---".format(algo)) print( - "FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}." - .format(model, batch_size, fp32_throughput, fp32_latency, - fp32_acc1)) + "FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.".format( + model, batch_size, fp32_throughput, fp32_latency, fp32_acc1 + ) + ) print( - "INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.\n" - .format(model, batch_size, int8_throughput, int8_latency, - int8_acc1)) + "INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.\n".format( + model, batch_size, int8_throughput, int8_latency, int8_acc1 + ) + ) sys.stdout.flush() delta_value = fp32_acc1 - int8_acc1 @@ -263,8 +310,8 @@ def run_test(self, class TestPostTrainingProgramAbsMaxForResnet50( - TestPostTrainingQuantizationProgram): - + TestPostTrainingQuantizationProgram +): def test_post_training_abs_max_resnet50(self): model = "ResNet-50" algo = "abs_max" @@ -278,9 +325,18 @@ def test_post_training_abs_max_resnet50(self): is_use_cache_file = False is_optimize_model = False diff_threshold = 0.025 - self.run_test(model, algo, round_type, data_urls, data_md5s, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_resnet50.py b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_resnet50.py index c79499100cee3..65e1d391399dd 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_resnet50.py +++ b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_resnet50.py @@ -14,14 +14,15 @@ import sys import unittest -from test_post_training_quantization_mobilenetv1 import TestPostTrainingQuantization +from test_post_training_quantization_mobilenetv1 import ( + TestPostTrainingQuantization, +) import paddle paddle.enable_static() class TestPostTrainingForResnet50(TestPostTrainingQuantization): - def test_post_training_resnet50(self): model = "ResNet-50" algo = "min_max" @@ -35,13 +36,21 @@ def test_post_training_resnet50(self): is_use_cache_file = False is_optimize_model = False diff_threshold = 0.025 - self.run_test(model, algo, round_type, data_urls, data_md5s, - quantizable_op_type, is_full_quantize, is_use_cache_file, - is_optimize_model, diff_threshold) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + ) class TestPostTrainingForResnet50ONNXFormat(TestPostTrainingQuantization): - def test_post_training_resnet50(self): model = "ResNet-50" algo = "min_max" @@ -56,17 +65,19 @@ def test_post_training_resnet50(self): is_optimize_model = False diff_threshold = 0.025 onnx_format = True - self.run_test(model, - algo, - round_type, - data_urls, - data_md5s, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - onnx_format=onnx_format) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + onnx_format=onnx_format, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_while.py b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_while.py index f4eaf5d9bc777..628d120f45ebd 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_while.py +++ b/python/paddle/fluid/contrib/slim/tests/test_post_training_quantization_while.py @@ -32,27 +32,34 @@ class TestPostTrainingQuantization(unittest.TestCase): - def setUp(self): self.download_path = 'int8/download' - self.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + - self.download_path) + self.cache_folder = os.path.expanduser( + '~/.cache/paddle/dataset/' + self.download_path + ) self.timestamp = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime()) - self.int8_model_path = os.path.join(os.getcwd(), - "post_training_" + self.timestamp) + self.int8_model_path = os.path.join( + os.getcwd(), "post_training_" + self.timestamp + ) try: os.system("mkdir -p " + self.int8_model_path) except Exception as e: - print("Failed to create {} due to {}".format( - self.int8_model_path, str(e))) + print( + "Failed to create {} due to {}".format( + self.int8_model_path, str(e) + ) + ) sys.exit(-1) def tearDown(self): try: os.system("rm -rf {}".format(self.int8_model_path)) except Exception as e: - print("Failed to delete {} due to {}".format( - self.int8_model_path, str(e))) + print( + "Failed to delete {} due to {}".format( + self.int8_model_path, str(e) + ) + ) def cache_unzipping(self, target_folder, zip_path): cmd = 'tar xf {0} -C {1}'.format(zip_path, target_folder) @@ -72,10 +79,16 @@ def run_program(self, model_path, batch_size, infer_iterations): print("test model path:" + model_path) place = fluid.CPUPlace() exe = fluid.Executor(place) - [infer_program, feed_dict, fetch_targets] = \ - fluid.io.load_inference_model(model_path, - model_filename='model.pdmodel', - params_filename='model.pdiparams', executor=exe) + [ + infer_program, + feed_dict, + fetch_targets, + ] = fluid.io.load_inference_model( + model_path, + model_filename='model.pdmodel', + params_filename='model.pdiparams', + executor=exe, + ) val_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size) img_shape = [1, 28, 28] @@ -83,14 +96,17 @@ def run_program(self, model_path, batch_size, infer_iterations): cnt = 0 periods = [] for batch_id, data in enumerate(val_reader()): - image = np.array([x[0].reshape(img_shape) - for x in data]).astype("float32") + image = np.array([x[0].reshape(img_shape) for x in data]).astype( + "float32" + ) input_label = np.array([x[1] for x in data]).astype("int64") t1 = time.time() - out = exe.run(infer_program, - feed={feed_dict[0]: image}, - fetch_list=fetch_targets) + out = exe.run( + infer_program, + feed={feed_dict[0]: image}, + fetch_list=fetch_targets, + ) t2 = time.time() period = t2 - t1 periods.append(period) @@ -108,16 +124,18 @@ def run_program(self, model_path, batch_size, infer_iterations): acc1 = np.sum(test_info) / cnt return (throughput, latency, acc1) - def generate_quantized_model(self, - model_path, - algo="KL", - quantizable_op_type=["conv2d"], - is_full_quantize=False, - is_use_cache_file=False, - is_optimize_model=False, - batch_size=10, - batch_nums=10, - is_data_loader=False): + def generate_quantized_model( + self, + model_path, + algo="KL", + quantizable_op_type=["conv2d"], + is_full_quantize=False, + is_use_cache_file=False, + is_optimize_model=False, + batch_size=10, + batch_nums=10, + is_data_loader=False, + ): place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -146,63 +164,81 @@ def val_data_generator(): quantizable_op_type=quantizable_op_type, is_full_quantize=is_full_quantize, optimize_model=is_optimize_model, - is_use_cache_file=is_use_cache_file) + is_use_cache_file=is_use_cache_file, + ) ptq.quantize() - ptq.save_quantized_model(self.int8_model_path, - model_filename='model.pdmodel', - params_filename='model.pdiparams') - - def run_test(self, - model_name, - data_url, - data_md5, - algo, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_size=10, - infer_iterations=10, - quant_iterations=5, - is_data_loader=False): + ptq.save_quantized_model( + self.int8_model_path, + model_filename='model.pdmodel', + params_filename='model.pdiparams', + ) + + def run_test( + self, + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size=10, + infer_iterations=10, + quant_iterations=5, + is_data_loader=False, + ): origin_model_path = self.download_model(data_url, data_md5, model_name) - #origin_model_path = os.path.join(origin_model_path, model_name) - - print("Start FP32 inference for {0} on {1} images ...".format( - model_name, infer_iterations * batch_size)) - (fp32_throughput, fp32_latency, - fp32_acc1) = self.run_program(origin_model_path, batch_size, - infer_iterations) - - print("Start INT8 post training quantization for {0} on {1} images ...". - format(model_name, quant_iterations * batch_size)) - self.generate_quantized_model(origin_model_path, - algo, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - batch_size, - quant_iterations, - is_data_loader=is_data_loader) - - print("Start INT8 inference for {0} on {1} images ...".format( - model_name, infer_iterations * batch_size)) - (int8_throughput, int8_latency, - int8_acc1) = self.run_program(self.int8_model_path, batch_size, - infer_iterations) + # origin_model_path = os.path.join(origin_model_path, model_name) + + print( + "Start FP32 inference for {0} on {1} images ...".format( + model_name, infer_iterations * batch_size + ) + ) + (fp32_throughput, fp32_latency, fp32_acc1) = self.run_program( + origin_model_path, batch_size, infer_iterations + ) + + print( + "Start INT8 post training quantization for {0} on {1} images ...".format( + model_name, quant_iterations * batch_size + ) + ) + self.generate_quantized_model( + origin_model_path, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + batch_size, + quant_iterations, + is_data_loader=is_data_loader, + ) + + print( + "Start INT8 inference for {0} on {1} images ...".format( + model_name, infer_iterations * batch_size + ) + ) + (int8_throughput, int8_latency, int8_acc1) = self.run_program( + self.int8_model_path, batch_size, infer_iterations + ) print("---Post training quantization of {} method---".format(algo)) print( - "FP32 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}." - .format(model_name, batch_size, fp32_throughput, fp32_latency, - fp32_acc1)) + "FP32 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}.".format( + model_name, batch_size, fp32_throughput, fp32_latency, fp32_acc1 + ) + ) print( - "INT8 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}.\n" - .format(model_name, batch_size, int8_throughput, int8_latency, - int8_acc1)) + "INT8 {0}: batch_size {1}, throughput {2} img/s, latency {3} s, acc1 {4}.\n".format( + model_name, batch_size, int8_throughput, int8_latency, int8_acc1 + ) + ) sys.stdout.flush() delta_value = fp32_acc1 - int8_acc1 @@ -210,10 +246,11 @@ def run_test(self, class TestPostTrainingKLForWhile(TestPostTrainingQuantization): - def test_post_training_kl(self): model_name = "mnist_while" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + ) data_md5 = "2387390beeb37b51dec041c27b8a681f" algo = "KL" quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"] @@ -224,17 +261,28 @@ def test_post_training_kl(self): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, quantizable_op_type, - is_full_quantize, is_use_cache_file, is_optimize_model, - diff_threshold, batch_size, infer_iterations, - quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTraininghistForWhile(TestPostTrainingQuantization): - def test_post_training_hist(self): model_name = "mnist_while" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + ) data_md5 = "2387390beeb37b51dec041c27b8a681f" algo = "hist" quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"] @@ -245,17 +293,28 @@ def test_post_training_hist(self): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, quantizable_op_type, - is_full_quantize, is_use_cache_file, is_optimize_model, - diff_threshold, batch_size, infer_iterations, - quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingmseForWhile(TestPostTrainingQuantization): - def test_post_training_mse(self): model_name = "mnist_while" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + ) data_md5 = "2387390beeb37b51dec041c27b8a681f" algo = "mse" quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"] @@ -266,17 +325,28 @@ def test_post_training_mse(self): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, quantizable_op_type, - is_full_quantize, is_use_cache_file, is_optimize_model, - diff_threshold, batch_size, infer_iterations, - quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingavgForWhile(TestPostTrainingQuantization): - def test_post_training_avg(self): model_name = "mnist_while" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + ) data_md5 = "2387390beeb37b51dec041c27b8a681f" algo = "avg" quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"] @@ -287,17 +357,28 @@ def test_post_training_avg(self): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, quantizable_op_type, - is_full_quantize, is_use_cache_file, is_optimize_model, - diff_threshold, batch_size, infer_iterations, - quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingMinMaxForWhile(TestPostTrainingQuantization): - def test_post_training_min_max(self): model_name = "mnist_while" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + ) data_md5 = "2387390beeb37b51dec041c27b8a681f" algo = "min_max" quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"] @@ -308,17 +389,28 @@ def test_post_training_min_max(self): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, quantizable_op_type, - is_full_quantize, is_use_cache_file, is_optimize_model, - diff_threshold, batch_size, infer_iterations, - quant_iterations) + self.run_test( + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) class TestPostTrainingAbsMaxForWhile(TestPostTrainingQuantization): - def test_post_training_abs_max(self): model_name = "mnist_while" - data_url = "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + data_url = ( + "http://paddle-inference-dist.bj.bcebos.com/int8/mnist_while.tar.gz" + ) data_md5 = "2387390beeb37b51dec041c27b8a681f" algo = "abs_max" quantizable_op_type = ["conv2d", "depthwise_conv2d", "mul"] @@ -329,23 +421,35 @@ def test_post_training_abs_max(self): batch_size = 10 infer_iterations = 50 quant_iterations = 5 - self.run_test(model_name, data_url, data_md5, algo, quantizable_op_type, - is_full_quantize, is_use_cache_file, is_optimize_model, - diff_threshold, batch_size, infer_iterations, - quant_iterations) - self.run_test(model_name, - data_url, - data_md5, - algo, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - batch_size, - infer_iterations, - quant_iterations, - is_data_loader=True) + self.run_test( + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + ) + self.run_test( + model_name, + data_url, + data_md5, + algo, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + batch_size, + infer_iterations, + quant_iterations, + is_data_loader=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_quant2_int8_mkldnn_pass.py b/python/paddle/fluid/contrib/slim/tests/test_quant2_int8_mkldnn_pass.py index dea0fcd489768..0f7a43ebebd68 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_quant2_int8_mkldnn_pass.py +++ b/python/paddle/fluid/contrib/slim/tests/test_quant2_int8_mkldnn_pass.py @@ -24,7 +24,6 @@ class TestQuant2Int8MkldnnPassMul(unittest.TestCase): - def op_name(self): return "mul" @@ -39,7 +38,8 @@ def setUp(self): self.mul_weights_size = [3, 5] self.mul_output_size = [1, 5] self.mul_input = np.random.random(self.mul_input_size).astype( - self.dtype) + self.dtype + ) self.mul_weights = np.ones(self.mul_weights_size, self.dtype) self.mul_weights_bad = np.ones([1, 1], self.dtype) self.mul_output = np.ndarray(self.mul_output_size).astype(self.dtype) @@ -49,23 +49,22 @@ def setUp(self): "mul_input": self.mul_input, "mul_weights": self.mul_weights, "mul_output": self.mul_output, - "mul_weights_bad": self.mul_weights_bad + "mul_weights_bad": self.mul_weights_bad, } def prepare_program_mul(self, program): block = program.global_block() for name in self.variables_mul: - block.create_var(name=name, - dtype="float32", - shape=self.variables_mul[name].shape) - - mul_op1 = block.append_op(type=self.op_name(), - inputs={ - "X": block.var('mul_input'), - "Y": block.var('mul_weights') - }, - outputs={"Out": block.var('mul_output')}, - attrs={'use_mkldnn': self.use_mkldnn}) + block.create_var( + name=name, dtype="float32", shape=self.variables_mul[name].shape + ) + + mul_op1 = block.append_op( + type=self.op_name(), + inputs={"X": block.var('mul_input'), "Y": block.var('mul_weights')}, + outputs={"Out": block.var('mul_output')}, + attrs={'use_mkldnn': self.use_mkldnn}, + ) def test_dequantize_op_weights(self): program = fluid.Program() @@ -80,11 +79,13 @@ def test_dequantize_op_weights(self): break assert op_node != "", "op of type %s not found" % self.op_name() - qpass = Quant2Int8MkldnnPass(self.quantized_ops, - _scope=self.scope, - _place=self.place, - _core=core, - _debug=False) + qpass = Quant2Int8MkldnnPass( + self.quantized_ops, + _scope=self.scope, + _place=self.place, + _core=core, + _debug=False, + ) qpass._weight_thresholds["mul_output"] = self.mul_output_scale param = self.scope.var("mul_weights").get_tensor() param.set(self.variables_mul["mul_weights"], self.place) @@ -92,9 +93,30 @@ def test_dequantize_op_weights(self): assert np.allclose( self.scope.find_var("mul_weights").get_tensor(), - [[1. / 127., 2. / 127., 3. / 127., 4. / 127., 5. / 127.], - [1. / 127., 2. / 127., 3. / 127., 4. / 127., 5. / 127.], - [1. / 127., 2. / 127., 3. / 127., 4. / 127., 5. / 127.]]) + [ + [ + 1.0 / 127.0, + 2.0 / 127.0, + 3.0 / 127.0, + 4.0 / 127.0, + 5.0 / 127.0, + ], + [ + 1.0 / 127.0, + 2.0 / 127.0, + 3.0 / 127.0, + 4.0 / 127.0, + 5.0 / 127.0, + ], + [ + 1.0 / 127.0, + 2.0 / 127.0, + 3.0 / 127.0, + 4.0 / 127.0, + 5.0 / 127.0, + ], + ], + ) param = self.scope.var("mul_weights").get_tensor() param.set(self.variables_mul["mul_weights_bad"], self.place) @@ -103,13 +125,11 @@ def test_dequantize_op_weights(self): class TestQuant2Int8MkldnnPassMatmulV2(TestQuant2Int8MkldnnPassMul): - def op_name(self): return "matmul_v2" class TestQuant2Int8MkldnnPassConv2D(unittest.TestCase): - def setUp(self): self.scope = fluid.Scope() self.place = fluid.CPUPlace() @@ -131,7 +151,8 @@ def setUp(self): self.filter2 = np.random.random(self.filter_size2).astype(self.dtype) self.conv_output = np.ndarray(self.conv_output_size).astype(self.dtype) self.conv_output2 = np.ndarray(self.conv_output2_size).astype( - self.dtype) + self.dtype + ) self.quantized_ops = 'conv2d' self.variables = { "input": self.input, @@ -144,15 +165,12 @@ def setUp(self): def prepare_program_conv2d(self, program): block = program.global_block() for name in self.variables: - block.create_var(name=name, - dtype="float32", - shape=self.variables[name].shape) + block.create_var( + name=name, dtype="float32", shape=self.variables[name].shape + ) conv2d_op1 = block.append_op( type="conv2d", - inputs={ - "Input": block.var('input'), - 'Filter': block.var('filter') - }, + inputs={"Input": block.var('input'), 'Filter': block.var('filter')}, outputs={"Output": block.var('conv_output')}, attrs={ 'strides': self.stride, @@ -162,13 +180,14 @@ def prepare_program_conv2d(self, program): 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, 'data_format': self.data_format, - 'fuse_relu': True - }) + 'fuse_relu': True, + }, + ) conv2d_op2 = block.append_op( type="conv2d", inputs={ "Input": block.var('conv_output'), - 'Filter': block.var('filter2') + 'Filter': block.var('filter2'), }, outputs={"Output": block.var('conv_output2')}, attrs={ @@ -178,8 +197,9 @@ def prepare_program_conv2d(self, program): 'dilations': self.dilations, 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, - 'data_format': self.data_format - }) + 'data_format': self.data_format, + }, + ) def remove_fuse_activation_attribute(self, graph): for op in graph.all_op_nodes(): @@ -204,16 +224,17 @@ def test_quant_update_activation(self): graph = IrGraph(core.Graph(program.desc), for_test=True) graph = self.remove_fuse_activation_attribute(graph) self.check_graph_before_pass(graph) - quant2_int8_mkldnn_pass = Quant2Int8MkldnnPass(self.quantized_ops, - _scope=self.scope, - _place=self.place, - _core=core, - _debug=False) + quant2_int8_mkldnn_pass = Quant2Int8MkldnnPass( + self.quantized_ops, + _scope=self.scope, + _place=self.place, + _core=core, + _debug=False, + ) graph = quant2_int8_mkldnn_pass._update_activations(graph) self.check_graph_after_pass(graph) class TestQuant2Int8MkldnnPassNearestInterp(unittest.TestCase): - def op_name(self): return "nearest_interp" @@ -236,7 +257,8 @@ def setUp(self): self.input = np.random.random(self.input_size).astype(self.dtype) self.filter = np.random.random(self.filter_size).astype(self.dtype) self.conv_output = np.ndarray(self.conv_output_size).astype( - self.dtype) + self.dtype + ) # nearest_interp self.out_h = 1 @@ -246,16 +268,20 @@ def setUp(self): self.data_layout = 'NCHW' self.nearest_interp_output_size = [1, 1, 2, 2] self.nearest_interp_output = np.ndarray( - self.nearest_interp_output_size).astype(self.dtype) + self.nearest_interp_output_size + ).astype(self.dtype) # dropout self.dropout_prob = 0.5 self.dropout_out = np.ndarray( - self.nearest_interp_output_size).astype(self.dtype) + self.nearest_interp_output_size + ).astype(self.dtype) self.dropout_mask = np.ndarray(self.nearest_interp_output_size) self.quantized_ops = { - "conv2d", "nearest_interp", "nearest_interp_v2" + "conv2d", + "nearest_interp", + "nearest_interp_v2", } self.variables = { "input": self.input, @@ -263,55 +289,61 @@ def setUp(self): "conv_output": self.conv_output, "nearest_interp_output": self.nearest_interp_output, "dropout_out": self.dropout_out, - 'dropout_mask': self.dropout_mask + 'dropout_mask': self.dropout_mask, } def prepare_program(self, program): block = program.global_block() for name in self.variables: - block.create_var(name=name, - dtype="float32", - shape=self.variables[name].shape) - block.append_op(type="conv2d", - inputs={ - "Input": block.var('input'), - 'Filter': block.var('filter') - }, - outputs={"Output": block.var('conv_output')}, - attrs={ - 'strides': self.stride, - 'paddings': self.pad, - 'groups': self.groups, - 'dilations': self.dilations, - 'use_cudnn': self.use_cudnn, - 'use_mkldnn': self.use_mkldnn, - 'data_format': self.data_format, - 'fuse_relu': True - }) - block.append_op(type=self.op_name(), - inputs={ - "X": block.var('conv_output'), - }, - outputs={"Out": block.var('nearest_interp_output')}, - attrs={ - 'interp_method': self.interp_method, - 'out_h': self.out_h, - 'out_w': self.out_w, - 'scale': self.scale, - 'data_layout': self.data_layout, - 'use_mkldnn': self.use_mkldnn - }) - block.append_op(type='dropout', - inputs={ - "X": block.var('nearest_interp_output'), - }, - outputs={ - 'Out': block.var('dropout_out'), - 'Mask': block.var('dropout_mask') - }, - attrs={ - 'dropout_prob': self.dropout_prob, - }) + block.create_var( + name=name, dtype="float32", shape=self.variables[name].shape + ) + block.append_op( + type="conv2d", + inputs={ + "Input": block.var('input'), + 'Filter': block.var('filter'), + }, + outputs={"Output": block.var('conv_output')}, + attrs={ + 'strides': self.stride, + 'paddings': self.pad, + 'groups': self.groups, + 'dilations': self.dilations, + 'use_cudnn': self.use_cudnn, + 'use_mkldnn': self.use_mkldnn, + 'data_format': self.data_format, + 'fuse_relu': True, + }, + ) + block.append_op( + type=self.op_name(), + inputs={ + "X": block.var('conv_output'), + }, + outputs={"Out": block.var('nearest_interp_output')}, + attrs={ + 'interp_method': self.interp_method, + 'out_h': self.out_h, + 'out_w': self.out_w, + 'scale': self.scale, + 'data_layout': self.data_layout, + 'use_mkldnn': self.use_mkldnn, + }, + ) + block.append_op( + type='dropout', + inputs={ + "X": block.var('nearest_interp_output'), + }, + outputs={ + 'Out': block.var('dropout_out'), + 'Mask': block.var('dropout_mask'), + }, + attrs={ + 'dropout_prob': self.dropout_prob, + }, + ) def check_graph_after_pass(self, graph): for op in graph.all_op_nodes(): @@ -329,12 +361,21 @@ def test_quant_update_activation(self): _scope=self.scope, _place=self.place, _core=core, - _debug=False) - - input_scale_tensor = quant2_int8_mkldnn_pass._convert_scale2tensor( - np.array(self.scale).astype(np.float64)) - output_scale_tensor = quant2_int8_mkldnn_pass._convert_scale2tensor( - np.array(1. / self.scale * self.scale).astype(np.float64)) + _debug=False, + ) + + input_scale_tensor = ( + quant2_int8_mkldnn_pass._convert_scale2tensor( + np.array(self.scale).astype(np.float64) + ) + ) + output_scale_tensor = ( + quant2_int8_mkldnn_pass._convert_scale2tensor( + np.array(1.0 / self.scale * self.scale).astype( + np.float64 + ) + ) + ) var_scale = { "input": (False, input_scale_tensor), "filter": (False, input_scale_tensor), @@ -347,7 +388,6 @@ def test_quant_update_activation(self): self.check_graph_after_pass(graph) class TestQuant2Int8MkldnnPassNearestInterpV2(unittest.TestCase): - def op_name(self): return "nearest_interp_v2" diff --git a/python/paddle/fluid/contrib/slim/tests/test_quantization_mkldnn_pass.py b/python/paddle/fluid/contrib/slim/tests/test_quantization_mkldnn_pass.py index da3b0139e024a..23b89512454a7 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_quantization_mkldnn_pass.py +++ b/python/paddle/fluid/contrib/slim/tests/test_quantization_mkldnn_pass.py @@ -29,19 +29,23 @@ def conv_net(img, label): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) prediction = fluid.layers.fc(input=conv_pool_2, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) avg_loss = paddle.mean(loss) @@ -49,12 +53,11 @@ def conv_net(img, label): class TestMKLDNNTransformBasedFreezePass(unittest.TestCase): - def setUp(self): self.quantizable_op_and_inputs = { 'conv2d': ['Input', 'Filter'], 'depthwise_conv2d': ['Input', 'Filter'], - 'mul': ['X', 'Y'] + 'mul': ['X', 'Y'], } def check_program(self, program): @@ -74,25 +77,27 @@ def build_program(self, main, startup, is_test, seed): startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - img = fluid.layers.data(name='image', - shape=[1, 28, 28], - dtype='float32') - label = fluid.layers.data(name='label', - shape=[1], - dtype='int64') + img = fluid.layers.data( + name='image', shape=[1, 28, 28], dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='int64' + ) loss = conv_net(img, label) if not is_test: opt = fluid.optimizer.Adam(learning_rate=0.001) opt.minimize(loss) return [img, label], loss - def mkldnn_based_freeze_graph(self, - use_cuda, - seed, - activation_quant_type, - weight_quant_type='abs_max', - quant_perf=False, - for_ci=False): + def mkldnn_based_freeze_graph( + self, + use_cuda, + seed, + activation_quant_type, + weight_quant_type='abs_max', + quant_perf=False, + for_ci=False, + ): random.seed(0) np.random.seed(0) @@ -115,42 +120,48 @@ def mkldnn_based_freeze_graph(self, scope=scope, place=place, activation_quantize_type=activation_quant_type, - weight_quantize_type=weight_quant_type) + weight_quantize_type=weight_quant_type, + ) transform_pass.apply(main_graph) transform_pass = QuantizationTransformPass( scope=scope, place=place, activation_quantize_type=activation_quant_type, - weight_quantize_type=weight_quant_type) + weight_quantize_type=weight_quant_type, + ) transform_pass.apply(test_graph) build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = False build_strategy.enable_inplace = False binary = fluid.CompiledProgram(main_graph.graph).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) quantized_test_program = test_graph.to_program() iters = 5 batch_size = 8 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=batch_size, + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) # Training the model to get the weights value with fluid.scope_guard(scope): for _ in range(iters): data = next(train_reader()) - loss_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss]) + loss_v = exe.run( + binary, feed=feeder.feed(data), fetch_list=[loss] + ) # Freeze graph for inference, but the weight of fc/conv is still float type. freeze_pass = QuantizationFreezePass( - scope=scope, place=place, weight_quantize_type=weight_quant_type) + scope=scope, place=place, weight_quantize_type=weight_quant_type + ) freeze_pass.apply(test_graph) # Transform quantized graph for MKL-DNN INT8 inference @@ -163,8 +174,14 @@ def mkldnn_based_freeze_graph(self, if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw( - '.', 'test_mkldnn' + dev_name + activation_quant_type + '_' + - weight_quant_type, marked_nodes) + '.', + 'test_mkldnn' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + marked_nodes, + ) mkldnn_program = test_graph.to_program() # Check the transformation weights of conv2d and mul @@ -178,9 +195,16 @@ def mkldnn_based_freeze_graph(self, # output self.check_program(mkldnn_program) if not for_ci: - print('{}: {}'.format( - 'w_mkldnn' + dev_name + activation_quant_type + '_' + - weight_quant_type, np.sum(w_mkldnn))) + print( + '{}: {}'.format( + 'w_mkldnn' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + np.sum(w_mkldnn), + ) + ) def test_mkldnn_graph_cpu_static(self): with fluid.unique_name.guard(): @@ -189,13 +213,15 @@ def test_mkldnn_graph_cpu_static(self): seed=2, activation_quant_type='range_abs_max', weight_quant_type='abs_max', - for_ci=True) + for_ci=True, + ) self.mkldnn_based_freeze_graph( False, seed=2, activation_quant_type='moving_average_abs_max', weight_quant_type='abs_max', - for_ci=True) + for_ci=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_quantization_pass.py b/python/paddle/fluid/contrib/slim/tests/test_quantization_pass.py index 4310c259ca5f4..be42ab5cf2e20 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_quantization_pass.py +++ b/python/paddle/fluid/contrib/slim/tests/test_quantization_pass.py @@ -45,50 +45,47 @@ def linear_fc(num): def residual_block(num, quant_skip_pattern=None): - - def conv_bn_layer(input, - ch_out, - filter_size, - stride, - padding, - act='relu', - bias_attr=False): - tmp = fluid.layers.conv2d(input=input, - filter_size=filter_size, - num_filters=ch_out, - stride=stride, - padding=padding, - act=None, - bias_attr=bias_attr) + def conv_bn_layer( + input, ch_out, filter_size, stride, padding, act='relu', bias_attr=False + ): + tmp = fluid.layers.conv2d( + input=input, + filter_size=filter_size, + num_filters=ch_out, + stride=stride, + padding=padding, + act=None, + bias_attr=bias_attr, + ) return fluid.layers.batch_norm(input=tmp, act=act) - data = fluid.layers.data(name='image', - shape=[1, 1, 32, 32], - dtype='float32', - append_batch_size=False) - label = fluid.layers.data(name='label', - shape=[1, 1], - dtype='int64', - append_batch_size=False) + data = fluid.layers.data( + name='image', + shape=[1, 1, 32, 32], + dtype='float32', + append_batch_size=False, + ) + label = fluid.layers.data( + name='label', shape=[1, 1], dtype='int64', append_batch_size=False + ) hidden = data for _ in range(num): conv = conv_bn_layer(hidden, 16, 3, 1, 1, act=None, bias_attr=True) short = conv_bn_layer(hidden, 16, 1, 1, 0, act=None) hidden = fluid.layers.elementwise_add(x=conv, y=short, act='relu') - matmul_weight = fluid.layers.create_parameter(shape=[1, 16, 32, 32], - dtype='float32') + matmul_weight = fluid.layers.create_parameter( + shape=[1, 16, 32, 32], dtype='float32' + ) hidden = fluid.layers.matmul(hidden, matmul_weight, True, True) if quant_skip_pattern: with fluid.name_scope(quant_skip_pattern): - pool = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='avg', - pool_stride=2) + pool = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='avg', pool_stride=2 + ) else: - pool = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='avg', - pool_stride=2) + pool = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='avg', pool_stride=2 + ) fc = fluid.layers.fc(input=pool, size=10) loss = fluid.layers.cross_entropy(input=fc, label=label) loss = paddle.mean(loss) @@ -96,21 +93,25 @@ def conv_bn_layer(input, def conv_net(img, label, quant_skip_pattern): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - pool_type='max', - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + pool_type='max', + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - pool_type='avg', - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + pool_type='avg', + act="relu", + ) hidden = fluid.layers.fc(input=conv_pool_2, size=100, act='relu') with fluid.name_scope(quant_skip_pattern): prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') @@ -120,17 +121,16 @@ def conv_net(img, label, quant_skip_pattern): class TestQuantizationTransformPass(unittest.TestCase): - def setUp(self): self.quantizable_op_and_inputs = { 'conv2d': ['Input', 'Filter'], 'depthwise_conv2d': ['Input', 'Filter'], - 'mul': ['X', 'Y'] + 'mul': ['X', 'Y'], } self.quantizable_grad_op_inputs = { 'conv2d_grad': ['Input', 'Filter'], 'depthwise_conv2d_grad': ['Input', 'Filter'], - 'mul_grad': ['X', 'Y'] + 'mul_grad': ['X', 'Y'], } def check_program(self, program): @@ -141,7 +141,8 @@ def check_program(self, program): if op.type in self.quantizable_op_and_inputs: for arg_name in op.input_arg_names: self.assertTrue( - arg_name.endswith('.quantized.dequantized')) + arg_name.endswith('.quantized.dequantized') + ) quantized_ops.add(arg_name) for op in block.ops: @@ -150,13 +151,13 @@ def check_program(self, program): for pname in self.quantizable_grad_op_inputs[op.type]: arg_name = op.input(pname)[0] self.assertTrue( - arg_name.endswith('.quantized.dequantized')) + arg_name.endswith('.quantized.dequantized') + ) self.assertTrue(arg_name in quantized_ops) - def linear_fc_quant(self, - activation_quant_type, - weight_quantize_type, - for_ci=True): + def linear_fc_quant( + self, activation_quant_type, weight_quantize_type, for_ci=True + ): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): @@ -169,15 +170,17 @@ def linear_fc_quant(self, scope=fluid.global_scope(), place=place, activation_quantize_type=activation_quant_type, - weight_quantize_type=weight_quantize_type) + weight_quantize_type=weight_quantize_type, + ) transform_pass.apply(graph) if not for_ci: marked_nodes = set() for op in graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) - graph.draw('.', 'quantize_fc_' + activation_quant_type, - marked_nodes) + graph.draw( + '.', 'quantize_fc_' + activation_quant_type, marked_nodes + ) program = graph.to_program() self.check_program(program) val_graph = IrGraph(core.Graph(program.desc), for_test=False) @@ -186,8 +189,9 @@ def linear_fc_quant(self, for op in val_graph.all_op_nodes(): if op.name().find('quantize') > -1: val_marked_nodes.add(op) - val_graph.draw('.', 'val_fc_' + activation_quant_type, - val_marked_nodes) + val_graph.draw( + '.', 'val_fc_' + activation_quant_type, val_marked_nodes + ) def test_linear_fc_quant_abs_max(self): self.linear_fc_quant('abs_max', 'abs_max', for_ci=True) @@ -196,15 +200,17 @@ def test_linear_fc_quant_range_abs_max(self): self.linear_fc_quant('range_abs_max', 'abs_max', for_ci=True) def test_linear_fc_quant_moving_average_abs_max(self): - self.linear_fc_quant('moving_average_abs_max', - 'channel_wise_abs_max', - for_ci=True) - - def residual_block_quant(self, - activation_quant_type, - weight_quantize_type, - quantizable_op_type, - for_ci=True): + self.linear_fc_quant( + 'moving_average_abs_max', 'channel_wise_abs_max', for_ci=True + ) + + def residual_block_quant( + self, + activation_quant_type, + weight_quantize_type, + quantizable_op_type, + for_ci=True, + ): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): @@ -218,15 +224,17 @@ def residual_block_quant(self, place=place, activation_quantize_type=activation_quant_type, weight_quantize_type=weight_quantize_type, - quantizable_op_type=quantizable_op_type) + quantizable_op_type=quantizable_op_type, + ) transform_pass.apply(graph) if not for_ci: marked_nodes = set() for op in graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) - graph.draw('.', 'quantize_residual_' + activation_quant_type, - marked_nodes) + graph.draw( + '.', 'quantize_residual_' + activation_quant_type, marked_nodes + ) program = graph.to_program() self.check_program(program) val_graph = IrGraph(core.Graph(program.desc), for_test=False) @@ -235,53 +243,54 @@ def residual_block_quant(self, for op in val_graph.all_op_nodes(): if op.name().find('quantize') > -1: val_marked_nodes.add(op) - val_graph.draw('.', 'val_residual_' + activation_quant_type, - val_marked_nodes) + val_graph.draw( + '.', 'val_residual_' + activation_quant_type, val_marked_nodes + ) def test_residual_block_abs_max(self): quantizable_op_type = ['conv2d', 'depthwise_conv2d', 'mul', 'matmul'] - self.residual_block_quant('abs_max', - 'abs_max', - quantizable_op_type, - for_ci=True) + self.residual_block_quant( + 'abs_max', 'abs_max', quantizable_op_type, for_ci=True + ) def test_residual_block_range_abs_max(self): quantizable_op_type = ['conv2d', 'depthwise_conv2d', 'mul', 'matmul'] - self.residual_block_quant('range_abs_max', - 'abs_max', - quantizable_op_type, - for_ci=True) + self.residual_block_quant( + 'range_abs_max', 'abs_max', quantizable_op_type, for_ci=True + ) def test_residual_block_moving_average_abs_max(self): quantizable_op_type = ['conv2d', 'depthwise_conv2d', 'mul', 'matmul'] - self.residual_block_quant('moving_average_abs_max', - 'channel_wise_abs_max', - quantizable_op_type, - for_ci=True) + self.residual_block_quant( + 'moving_average_abs_max', + 'channel_wise_abs_max', + quantizable_op_type, + for_ci=True, + ) class TestQuantizationFreezePass(unittest.TestCase): - - def freeze_graph(self, - use_cuda, - seed, - activation_quant_type, - bias_correction=False, - weight_quant_type='abs_max', - for_ci=True, - quant_skip_pattern='skip_quant'): - + def freeze_graph( + self, + use_cuda, + seed, + activation_quant_type, + bias_correction=False, + weight_quant_type='abs_max', + for_ci=True, + quant_skip_pattern='skip_quant', + ): def build_program(main, startup, is_test): main.random_seed = seed startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - img = fluid.layers.data(name='image', - shape=[1, 28, 28], - dtype='float32') - label = fluid.layers.data(name='label', - shape=[1], - dtype='int64') + img = fluid.layers.data( + name='image', shape=[1, 28, 28], dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='int64' + ) loss = conv_net(img, label, quant_skip_pattern) if not is_test: opt = fluid.optimizer.Adam(learning_rate=0.001) @@ -310,14 +319,16 @@ def build_program(main, startup, is_test): place=place, activation_quantize_type=activation_quant_type, weight_quantize_type=weight_quant_type, - skip_pattern=quant_skip_pattern) + skip_pattern=quant_skip_pattern, + ) transform_pass.apply(main_graph) transform_pass = QuantizationTransformPass( scope=scope, place=place, activation_quantize_type=activation_quant_type, weight_quantize_type=weight_quant_type, - skip_pattern=quant_skip_pattern) + skip_pattern=quant_skip_pattern, + ) transform_pass.apply(test_graph) dev_name = '_gpu_' if use_cuda else '_cpu_' if not for_ci: @@ -326,57 +337,85 @@ def build_program(main, startup, is_test): if op.name().find('quantize') > -1: marked_nodes.add(op) main_graph.draw( - '.', 'main' + dev_name + activation_quant_type + '_' + - weight_quant_type, marked_nodes) + '.', + 'main' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + marked_nodes, + ) marked_nodes = set() for op in test_graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw( - '.', 'test' + dev_name + activation_quant_type + '_' + - weight_quant_type, marked_nodes) + '.', + 'test' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + marked_nodes, + ) build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = False build_strategy.enable_inplace = False build_strategy.fuse_all_reduce_ops = False binary = fluid.CompiledProgram(main_graph.graph).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) quantized_test_program = test_graph.to_program() iters = 5 batch_size = 8 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=batch_size, + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) with fluid.scope_guard(scope): for _ in range(iters): data = next(train_reader()) - loss_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss]) + loss_v = exe.run( + binary, feed=feeder.feed(data), fetch_list=[loss] + ) if not for_ci: - print('{}: {}'.format( - 'loss' + dev_name + activation_quant_type + '_' + - weight_quant_type, loss_v)) + print( + '{}: {}'.format( + 'loss' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + loss_v, + ) + ) test_data = next(test_reader()) with fluid.program_guard(quantized_test_program): - w_var = fluid.framework._get_var('conv2d_1.w_0.quantized', - quantized_test_program) + w_var = fluid.framework._get_var( + 'conv2d_1.w_0.quantized', quantized_test_program + ) # Testing with fluid.scope_guard(scope): - test_loss1, w_quant = exe.run(program=quantized_test_program, - feed=feeder.feed(test_data), - fetch_list=[loss, w_var]) + test_loss1, w_quant = exe.run( + program=quantized_test_program, + feed=feeder.feed(test_data), + fetch_list=[loss, w_var], + ) # Freeze graph for inference, but the weight of fc/conv is still float type. freeze_pass = QuantizationFreezePass( - scope=scope, place=place, bias_correction=bias_correction, \ - weight_quantize_type=weight_quant_type) + scope=scope, + place=place, + bias_correction=bias_correction, + weight_quantize_type=weight_quant_type, + ) freeze_pass.apply(test_graph) if not for_ci: marked_nodes = set() @@ -384,32 +423,68 @@ def build_program(main, startup, is_test): if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw( - '.', 'test_freeze' + dev_name + activation_quant_type + '_' + - weight_quant_type, marked_nodes) + '.', + 'test_freeze' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + marked_nodes, + ) server_program = test_graph.to_program() with fluid.scope_guard(scope): - test_loss2, = exe.run(program=server_program, - feed=feeder.feed(test_data), - fetch_list=[loss]) + (test_loss2,) = exe.run( + program=server_program, + feed=feeder.feed(test_data), + fetch_list=[loss], + ) self.assertAlmostEqual(test_loss1, test_loss2, delta=5e-3) if not for_ci: - print('{}: {}'.format( - 'test_loss1' + dev_name + activation_quant_type + '_' + - weight_quant_type, test_loss1)) - print('{}: {}'.format( - 'test_loss2' + dev_name + activation_quant_type + '_' + - weight_quant_type, test_loss2)) + print( + '{}: {}'.format( + 'test_loss1' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + test_loss1, + ) + ) + print( + '{}: {}'.format( + 'test_loss2' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + test_loss2, + ) + ) w_freeze = np.array(scope.find_var('conv2d_1.w_0').get_tensor()) # Maybe failed, this is due to the calculation precision # self.assertAlmostEqual(np.sum(w_freeze), np.sum(w_quant)) if not for_ci: - print('{}: {}'.format( - 'w_freeze' + dev_name + activation_quant_type + '_' + - weight_quant_type, np.sum(w_freeze))) - print('{}: {}'.format( - 'w_quant' + dev_name + activation_quant_type + '_' + - weight_quant_type, np.sum(w_quant))) + print( + '{}: {}'.format( + 'w_freeze' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + np.sum(w_freeze), + ) + ) + print( + '{}: {}'.format( + 'w_quant' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + np.sum(w_quant), + ) + ) # Convert parameter to 8-bit. convert_int8_pass = ConvertToInt8Pass(scope=scope, place=place) @@ -420,30 +495,62 @@ def build_program(main, startup, is_test): if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw( - '.', 'test_int8' + dev_name + activation_quant_type + '_' + - weight_quant_type, marked_nodes) + '.', + 'test_int8' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + marked_nodes, + ) server_program_int8 = test_graph.to_program() # Save the 8-bit parameter and model file. with fluid.scope_guard(scope): fluid.io.save_inference_model( - 'server_int8' + dev_name + activation_quant_type + '_' + - weight_quant_type, ['image', 'label'], [loss], exe, - server_program_int8) + 'server_int8' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + ['image', 'label'], + [loss], + exe, + server_program_int8, + ) # Test whether the 8-bit parameter and model file can be loaded successfully. [infer, feed, fetch] = fluid.io.load_inference_model( - 'server_int8' + dev_name + activation_quant_type + '_' + - weight_quant_type, exe) + 'server_int8' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + exe, + ) # Check the loaded 8-bit weight. w_8bit = np.array(scope.find_var('conv2d_1.w_0.int8').get_tensor()) self.assertEqual(w_8bit.dtype, np.int8) self.assertEqual(np.sum(w_8bit), np.sum(w_freeze)) if not for_ci: - print('{}: {}'.format( - 'w_8bit' + dev_name + activation_quant_type + '_' + - weight_quant_type, np.sum(w_8bit))) - print('{}: {}'.format( - 'w_freeze' + dev_name + activation_quant_type + '_' + - weight_quant_type, np.sum(w_freeze))) + print( + '{}: {}'.format( + 'w_8bit' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + np.sum(w_8bit), + ) + ) + print( + '{}: {}'.format( + 'w_freeze' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + np.sum(w_freeze), + ) + ) mobile_pass = TransformForMobilePass() mobile_pass.apply(test_graph) @@ -453,129 +560,164 @@ def build_program(main, startup, is_test): if op.name().find('quantize') > -1: marked_nodes.add(op) test_graph.draw( - '.', 'test_mobile' + dev_name + activation_quant_type + '_' + - weight_quant_type, marked_nodes) + '.', + 'test_mobile' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + marked_nodes, + ) mobile_program = test_graph.to_program() with fluid.scope_guard(scope): fluid.io.save_inference_model( - 'mobile_int8' + dev_name + activation_quant_type + '_' + - weight_quant_type, ['image', 'label'], [loss], exe, - mobile_program) + 'mobile_int8' + + dev_name + + activation_quant_type + + '_' + + weight_quant_type, + ['image', 'label'], + [loss], + exe, + mobile_program, + ) def test_freeze_graph_cuda_dynamic(self): if fluid.core.is_compiled_with_cuda(): with fluid.unique_name.guard(): - self.freeze_graph(True, - seed=1, - activation_quant_type='abs_max', - weight_quant_type='abs_max', - for_ci=True) + self.freeze_graph( + True, + seed=1, + activation_quant_type='abs_max', + weight_quant_type='abs_max', + for_ci=True, + ) with fluid.unique_name.guard(): - self.freeze_graph(True, - seed=1, - activation_quant_type='abs_max', - weight_quant_type='channel_wise_abs_max', - for_ci=True) + self.freeze_graph( + True, + seed=1, + activation_quant_type='abs_max', + weight_quant_type='channel_wise_abs_max', + for_ci=True, + ) def test_freeze_graph_cpu_dynamic(self): with fluid.unique_name.guard(): - self.freeze_graph(False, - seed=2, - activation_quant_type='abs_max', - weight_quant_type='abs_max', - for_ci=True) - self.freeze_graph(False, - seed=2, - activation_quant_type='abs_max', - weight_quant_type='channel_wise_abs_max', - for_ci=True) + self.freeze_graph( + False, + seed=2, + activation_quant_type='abs_max', + weight_quant_type='abs_max', + for_ci=True, + ) + self.freeze_graph( + False, + seed=2, + activation_quant_type='abs_max', + weight_quant_type='channel_wise_abs_max', + for_ci=True, + ) def test_freeze_graph_cuda_static(self): if fluid.core.is_compiled_with_cuda(): with fluid.unique_name.guard(): - self.freeze_graph(True, - seed=1, - activation_quant_type='range_abs_max', - bias_correction=True, - weight_quant_type='abs_max', - for_ci=True) - self.freeze_graph(True, - seed=1, - activation_quant_type='range_abs_max', - weight_quant_type='abs_max', - for_ci=True) + self.freeze_graph( + True, + seed=1, + activation_quant_type='range_abs_max', + bias_correction=True, + weight_quant_type='abs_max', + for_ci=True, + ) + self.freeze_graph( + True, + seed=1, + activation_quant_type='range_abs_max', + weight_quant_type='abs_max', + for_ci=True, + ) self.freeze_graph( True, seed=1, activation_quant_type='moving_average_abs_max', weight_quant_type='abs_max', - for_ci=True) - self.freeze_graph(True, - seed=1, - activation_quant_type='range_abs_max', - weight_quant_type='channel_wise_abs_max', - for_ci=True) + for_ci=True, + ) + self.freeze_graph( + True, + seed=1, + activation_quant_type='range_abs_max', + weight_quant_type='channel_wise_abs_max', + for_ci=True, + ) self.freeze_graph( True, seed=1, activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', - for_ci=True) + for_ci=True, + ) self.freeze_graph( True, seed=1, activation_quant_type='moving_average_abs_max', bias_correction=True, weight_quant_type='channel_wise_abs_max', - for_ci=True) + for_ci=True, + ) def test_freeze_graph_cpu_static(self): with fluid.unique_name.guard(): - self.freeze_graph(False, - seed=2, - activation_quant_type='range_abs_max', - weight_quant_type='abs_max', - for_ci=True) - self.freeze_graph(False, - seed=2, - activation_quant_type='moving_average_abs_max', - weight_quant_type='abs_max', - for_ci=True) - self.freeze_graph(False, - seed=2, - activation_quant_type='range_abs_max', - weight_quant_type='channel_wise_abs_max', - for_ci=True) - self.freeze_graph(False, - seed=2, - activation_quant_type='moving_average_abs_max', - weight_quant_type='channel_wise_abs_max', - for_ci=True) + self.freeze_graph( + False, + seed=2, + activation_quant_type='range_abs_max', + weight_quant_type='abs_max', + for_ci=True, + ) + self.freeze_graph( + False, + seed=2, + activation_quant_type='moving_average_abs_max', + weight_quant_type='abs_max', + for_ci=True, + ) + self.freeze_graph( + False, + seed=2, + activation_quant_type='range_abs_max', + weight_quant_type='channel_wise_abs_max', + for_ci=True, + ) + self.freeze_graph( + False, + seed=2, + activation_quant_type='moving_average_abs_max', + weight_quant_type='channel_wise_abs_max', + for_ci=True, + ) def quant_dequant_residual_block(num, quant_skip_pattern=None): - - def conv_bn_layer(input, - ch_out, - filter_size, - stride, - padding, - act='relu', - bias_attr=False): - tmp = fluid.layers.conv2d(input=input, - filter_size=filter_size, - num_filters=ch_out, - stride=stride, - padding=padding, - act=None, - bias_attr=bias_attr) + def conv_bn_layer( + input, ch_out, filter_size, stride, padding, act='relu', bias_attr=False + ): + tmp = fluid.layers.conv2d( + input=input, + filter_size=filter_size, + num_filters=ch_out, + stride=stride, + padding=padding, + act=None, + bias_attr=bias_attr, + ) return fluid.layers.batch_norm(input=tmp, act=act) data1 = fluid.layers.data(name='image', shape=[1, 32, 32], dtype='float32') - data2 = fluid.layers.data(name='matmul_input', - shape=[16, 32, 32], - dtype='float32') + data2 = fluid.layers.data( + name='matmul_input', shape=[16, 32, 32], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') hidden = data1 for _ in range(num): @@ -585,43 +727,37 @@ def conv_bn_layer(input, hidden = fluid.layers.matmul(hidden, data2, True, True) if isinstance(quant_skip_pattern, str): with fluid.name_scope(quant_skip_pattern): - pool1 = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='avg', - pool_stride=2) - pool2 = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='max', - pool_stride=2) - pool_add = fluid.layers.elementwise_add(x=pool1, - y=pool2, - act='relu') + pool1 = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='avg', pool_stride=2 + ) + pool2 = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='max', pool_stride=2 + ) + pool_add = fluid.layers.elementwise_add( + x=pool1, y=pool2, act='relu' + ) elif isinstance(quant_skip_pattern, list): - assert len( - quant_skip_pattern - ) > 1, 'test config error: the len of quant_skip_pattern list should be greater than 1.' + assert ( + len(quant_skip_pattern) > 1 + ), 'test config error: the len of quant_skip_pattern list should be greater than 1.' with fluid.name_scope(quant_skip_pattern[0]): - pool1 = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='avg', - pool_stride=2) - pool2 = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='max', - pool_stride=2) + pool1 = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='avg', pool_stride=2 + ) + pool2 = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='max', pool_stride=2 + ) with fluid.name_scope(quant_skip_pattern[1]): - pool_add = fluid.layers.elementwise_add(x=pool1, - y=pool2, - act='relu') + pool_add = fluid.layers.elementwise_add( + x=pool1, y=pool2, act='relu' + ) else: - pool1 = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='avg', - pool_stride=2) - pool2 = fluid.layers.pool2d(input=hidden, - pool_size=2, - pool_type='max', - pool_stride=2) + pool1 = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='avg', pool_stride=2 + ) + pool2 = fluid.layers.pool2d( + input=hidden, pool_size=2, pool_type='max', pool_stride=2 + ) pool_add = fluid.layers.elementwise_add(x=pool1, y=pool2, act='relu') fc = fluid.layers.fc(input=pool_add, size=10) loss = fluid.layers.cross_entropy(input=fc, label=label) @@ -630,7 +766,6 @@ def conv_bn_layer(input, class TestAddQuantDequantPass(unittest.TestCase): - def setUp(self): self._target_ops = {'elementwise_add', 'pool2d'} self._target_grad_ops = {'elementwise_add_grad', 'pool2d_grad'} @@ -641,32 +776,40 @@ def check_graph(self, graph, skip_pattern=None): if op_node.name() in self._target_ops: user_skipped = False if isinstance(skip_pattern, list): - user_skipped = op_node.op().has_attr("op_namescope") and \ - any(pattern in op_node.op().attr("op_namescope") for pattern in skip_pattern) + user_skipped = op_node.op().has_attr( + "op_namescope" + ) and any( + pattern in op_node.op().attr("op_namescope") + for pattern in skip_pattern + ) elif isinstance(skip_pattern, str): - user_skipped = op_node.op().has_attr("op_namescope") and \ - op_node.op().attr("op_namescope").find(skip_pattern) != -1 + user_skipped = ( + op_node.op().has_attr("op_namescope") + and op_node.op().attr("op_namescope").find(skip_pattern) + != -1 + ) if user_skipped: continue in_nodes_all_not_persistable = True for input_name in op_node.input_arg_names(): - in_node = graph._find_node_by_name(op_node.inputs, - input_name) - in_nodes_all_not_persistable = (in_nodes_all_not_persistable - and - not in_node.persistable()) + in_node = graph._find_node_by_name( + op_node.inputs, input_name + ) + in_nodes_all_not_persistable = ( + in_nodes_all_not_persistable + and not in_node.persistable() + ) if not in_nodes_all_not_persistable: continue input_names = op_node.input_arg_names() for input_name in input_names: self.assertTrue(input_name.endswith('.quant_dequant')) - def residual_block_quant(self, - quantizable_op_type, - skip_pattern=None, - for_ci=True): + def residual_block_quant( + self, quantizable_op_type, skip_pattern=None, for_ci=True + ): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): @@ -679,7 +822,8 @@ def residual_block_quant(self, scope=fluid.global_scope(), place=place, skip_pattern=skip_pattern, - quantizable_op_type=quantizable_op_type) + quantizable_op_type=quantizable_op_type, + ) add_quant_dequant_pass.apply(graph) if not for_ci: marked_nodes = set() @@ -699,35 +843,36 @@ def residual_block_quant(self, def test_residual_block(self): quantizable_op_type = ['elementwise_add', 'pool2d', 'mul', 'matmul'] - self.residual_block_quant(quantizable_op_type, - skip_pattern=None, - for_ci=True) + self.residual_block_quant( + quantizable_op_type, skip_pattern=None, for_ci=True + ) def test_residual_block_skip_pattern(self): quantizable_op_type = ['elementwise_add', 'pool2d', 'mul', 'matmul'] - self.residual_block_quant(quantizable_op_type, - skip_pattern='skip_quant', - for_ci=True) + self.residual_block_quant( + quantizable_op_type, skip_pattern='skip_quant', for_ci=True + ) def test_residual_block_skip_pattern_1(self): quantizable_op_type = ['elementwise_add', 'pool2d', 'mul', 'matmul'] - self.residual_block_quant(quantizable_op_type, - skip_pattern=['skip_quant1', 'skip_quant2'], - for_ci=True) + self.residual_block_quant( + quantizable_op_type, + skip_pattern=['skip_quant1', 'skip_quant2'], + for_ci=True, + ) class TestQuantizationTransformPassV2(unittest.TestCase): - def setUp(self): self.quantizable_op_and_inputs = { 'conv2d': ['Input', 'Filter'], 'depthwise_conv2d': ['Input', 'Filter'], - 'mul': ['X', 'Y'] + 'mul': ['X', 'Y'], } self.quantizable_grad_op_inputs = { 'conv2d_grad': ['Input', 'Filter'], 'depthwise_conv2d_grad': ['Input', 'Filter'], - 'mul_grad': ['X', 'Y'] + 'mul_grad': ['X', 'Y'], } def check_program(self, program): @@ -738,7 +883,8 @@ def check_program(self, program): if op.type in self.quantizable_op_and_inputs: for arg_name in op.input_arg_names: self.assertTrue( - arg_name.endswith('.quantized.dequantized')) + arg_name.endswith('.quantized.dequantized') + ) quantized_ops.add(arg_name) for op in block.ops: @@ -747,13 +893,13 @@ def check_program(self, program): for pname in self.quantizable_grad_op_inputs[op.type]: arg_name = op.input(pname)[0] self.assertTrue( - arg_name.endswith('.quantized.dequantized')) + arg_name.endswith('.quantized.dequantized') + ) self.assertTrue(arg_name in quantized_ops) - def linear_fc_quant(self, - activation_quant_type, - weight_quantize_type, - for_ci=True): + def linear_fc_quant( + self, activation_quant_type, weight_quantize_type, for_ci=True + ): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): @@ -766,15 +912,17 @@ def linear_fc_quant(self, scope=fluid.global_scope(), place=place, activation_quantize_type=activation_quant_type, - weight_quantize_type=weight_quantize_type) + weight_quantize_type=weight_quantize_type, + ) transform_pass.apply(graph) if not for_ci: marked_nodes = set() for op in graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) - graph.draw('.', 'quantize_fc_' + activation_quant_type, - marked_nodes) + graph.draw( + '.', 'quantize_fc_' + activation_quant_type, marked_nodes + ) program = graph.to_program() self.check_program(program) val_graph = IrGraph(core.Graph(program.desc), for_test=False) @@ -783,8 +931,9 @@ def linear_fc_quant(self, for op in val_graph.all_op_nodes(): if op.name().find('quantize') > -1: val_marked_nodes.add(op) - val_graph.draw('.', 'val_fc_' + activation_quant_type, - val_marked_nodes) + val_graph.draw( + '.', 'val_fc_' + activation_quant_type, val_marked_nodes + ) def test_linear_fc_quant_abs_max(self): self.linear_fc_quant('abs_max', 'abs_max', for_ci=True) @@ -792,11 +941,13 @@ def test_linear_fc_quant_abs_max(self): def test_linear_fc_quant_channel_wise_abs_max(self): self.linear_fc_quant('abs_max', 'channel_wise_abs_max', for_ci=True) - def residual_block_quant(self, - activation_quant_type, - weight_quantize_type, - quantizable_op_type, - for_ci=True): + def residual_block_quant( + self, + activation_quant_type, + weight_quantize_type, + quantizable_op_type, + for_ci=True, + ): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): @@ -810,15 +961,17 @@ def residual_block_quant(self, place=place, activation_quantize_type=activation_quant_type, weight_quantize_type=weight_quantize_type, - quantizable_op_type=quantizable_op_type) + quantizable_op_type=quantizable_op_type, + ) transform_pass.apply(graph) if not for_ci: marked_nodes = set() for op in graph.all_op_nodes(): if op.name().find('quantize') > -1: marked_nodes.add(op) - graph.draw('.', 'quantize_residual_' + activation_quant_type, - marked_nodes) + graph.draw( + '.', 'quantize_residual_' + activation_quant_type, marked_nodes + ) program = graph.to_program() self.check_program(program) val_graph = IrGraph(core.Graph(program.desc), for_test=False) @@ -827,22 +980,21 @@ def residual_block_quant(self, for op in val_graph.all_op_nodes(): if op.name().find('quantize') > -1: val_marked_nodes.add(op) - val_graph.draw('.', 'val_residual_' + activation_quant_type, - val_marked_nodes) + val_graph.draw( + '.', 'val_residual_' + activation_quant_type, val_marked_nodes + ) def test_residual_block_abs_max(self): quantizable_op_type = ['conv2d', 'depthwise_conv2d', 'mul', 'matmul'] - self.residual_block_quant('abs_max', - 'abs_max', - quantizable_op_type, - for_ci=True) + self.residual_block_quant( + 'abs_max', 'abs_max', quantizable_op_type, for_ci=True + ) def test_residual_block_channel_wise_abs_max(self): quantizable_op_type = ['conv2d', 'depthwise_conv2d', 'mul', 'matmul'] - self.residual_block_quant('abs_max', - 'channel_wise_abs_max', - quantizable_op_type, - for_ci=True) + self.residual_block_quant( + 'abs_max', 'channel_wise_abs_max', quantizable_op_type, for_ci=True + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_quantization_scale_pass.py b/python/paddle/fluid/contrib/slim/tests/test_quantization_scale_pass.py index a8466ad7371c3..46e3700246037 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_quantization_scale_pass.py +++ b/python/paddle/fluid/contrib/slim/tests/test_quantization_scale_pass.py @@ -34,21 +34,25 @@ def conv_net(img, label): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - pool_type='max', - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + pool_type='max', + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - pool_type='avg', - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + pool_type='avg', + act="relu", + ) hidden = fluid.layers.fc(input=conv_pool_2, size=100, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) @@ -57,25 +61,25 @@ def conv_net(img, label): class TestQuantizationScalePass(unittest.TestCase): - - def quantization_scale(self, - use_cuda, - seed, - activation_quant_type, - weight_quant_type='abs_max', - for_ci=False): - + def quantization_scale( + self, + use_cuda, + seed, + activation_quant_type, + weight_quant_type='abs_max', + for_ci=False, + ): def build_program(main, startup, is_test): main.random_seed = seed startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - img = fluid.layers.data(name='image', - shape=[1, 28, 28], - dtype='float32') - label = fluid.layers.data(name='label', - shape=[1], - dtype='int64') + img = fluid.layers.data( + name='image', shape=[1, 28, 28], dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='int64' + ) loss = conv_net(img, label) if not is_test: opt = fluid.optimizer.Adam(learning_rate=0.0001) @@ -104,7 +108,8 @@ def build_program(main, startup, is_test): scope=scope, place=place, activation_quantize_type=activation_quant_type, - weight_quantize_type=weight_quant_type) + weight_quantize_type=weight_quant_type, + ) transform_pass.apply(main_graph) transform_pass.apply(test_graph) @@ -133,20 +138,22 @@ def build_program(main, startup, is_test): build_strategy.enable_inplace = False build_strategy.fuse_all_reduce_ops = False binary = fluid.CompiledProgram(main_graph.graph).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) iters = 5 batch_size = 8 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=batch_size, + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) with fluid.scope_guard(scope): for _ in range(iters): data = next(train_reader()) - loss_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss]) + loss_v = exe.run( + binary, feed=feeder.feed(data), fetch_list=[loss] + ) if not for_ci: print('{}: {}'.format('loss' + dev_name, loss_v)) @@ -155,7 +162,8 @@ def build_program(main, startup, is_test): # Freeze graph for inference, but the weight of fc/conv is still float type. freeze_pass = QuantizationFreezePass( - scope=scope, place=place, weight_quantize_type=weight_quant_type) + scope=scope, place=place, weight_quantize_type=weight_quant_type + ) freeze_pass.apply(test_graph) server_program = test_graph.to_program() @@ -168,16 +176,21 @@ def build_program(main, startup, is_test): tempdir = tempfile.TemporaryDirectory() mapping_table_path = os.path.join( - tempdir.name, 'quant_scale_model' + dev_name + '.txt') + tempdir.name, 'quant_scale_model' + dev_name + '.txt' + ) save_path = os.path.join(tempdir.name, 'quant_scale_model' + dev_name) with open(mapping_table_path, 'w') as f: f.write(str(server_program)) with fluid.scope_guard(scope): - fluid.io.save_inference_model(save_path, ['image', 'label'], [loss], - exe, - server_program, - clip_extra=True) + fluid.io.save_inference_model( + save_path, + ['image', 'label'], + [loss], + exe, + server_program, + clip_extra=True, + ) tempdir.cleanup() def test_quant_scale_cuda(self): @@ -188,7 +201,8 @@ def test_quant_scale_cuda(self): seed=1, activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', - for_ci=True) + for_ci=True, + ) def test_quant_scale_cpu(self): with fluid.unique_name.guard(): @@ -197,7 +211,8 @@ def test_quant_scale_cpu(self): seed=2, activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', - for_ci=True) + for_ci=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_quantize_transpiler_v2.py b/python/paddle/fluid/contrib/slim/tests/test_quantize_transpiler_v2.py index 3893177cbf033..3b487ae1fae2b 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_quantize_transpiler_v2.py +++ b/python/paddle/fluid/contrib/slim/tests/test_quantize_transpiler_v2.py @@ -19,7 +19,9 @@ import paddle.fluid as fluid import paddle from paddle.fluid.framework import IrGraph -from paddle.fluid.contrib.slim.quantization.quantize_transpiler_v2 import QuantizeTranspilerV2 +from paddle.fluid.contrib.slim.quantization.quantize_transpiler_v2 import ( + QuantizeTranspilerV2, +) from paddle.fluid import core paddle.enable_static() @@ -29,20 +31,24 @@ def conv_net(img, label): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - pool_type='max', - act="relu") - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - pool_type='avg', - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + pool_type='max', + act="relu", + ) + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + pool_type='avg', + act="relu", + ) with fluid.name_scope("skip_quant"): hidden = fluid.layers.fc(input=conv_pool_1, size=100, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') @@ -52,25 +58,25 @@ def conv_net(img, label): class TestQuantizeProgramPass(unittest.TestCase): - - def quantize_program(self, - use_cuda, - seed, - activation_quant_type='abs_max', - weight_quant_type='abs_max', - for_ci=False): - + def quantize_program( + self, + use_cuda, + seed, + activation_quant_type='abs_max', + weight_quant_type='abs_max', + for_ci=False, + ): def build_program(main, startup, is_test): main.random_seed = seed startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - img = fluid.layers.data(name='image', - shape=[1, 28, 28], - dtype='float32') - label = fluid.layers.data(name='label', - shape=[1], - dtype='int64') + img = fluid.layers.data( + name='image', shape=[1, 28, 28], dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='int64' + ) loss = conv_net(img, label) if not is_test: opt = fluid.optimizer.Adam(learning_rate=0.0001) @@ -89,8 +95,9 @@ def build_program(main, startup, is_test): test_program = test_program.clone(for_test=True) if not for_ci: - train_graph = IrGraph(core.Graph(train_program.desc), - for_test=False) + train_graph = IrGraph( + core.Graph(train_program.desc), for_test=False + ) train_graph.draw('.', 'train_program_1') test_graph = IrGraph(core.Graph(test_program.desc), for_test=True) test_graph.draw('.', 'test_program_1') @@ -98,7 +105,8 @@ def build_program(main, startup, is_test): # 2 Apply quantization qt = QuantizeTranspilerV2( activation_quantize_type=activation_quant_type, - weight_quantize_type=weight_quant_type) + weight_quantize_type=weight_quant_type, + ) qt.apply(train_program, startup_program, is_test=False) qt.apply(test_program, startup_program, is_test=True) @@ -109,8 +117,9 @@ def build_program(main, startup, is_test): with fluid.scope_guard(scope): exe.run(startup_program) if not for_ci: - train_graph = IrGraph(core.Graph(train_program.desc), - for_test=False) + train_graph = IrGraph( + core.Graph(train_program.desc), for_test=False + ) train_graph.draw('.', 'train_program_2') test_graph = IrGraph(core.Graph(test_program.desc), for_test=True) test_graph.draw('.', 'test_program_2') @@ -120,19 +129,21 @@ def build_program(main, startup, is_test): build_strategy.enable_inplace = False build_strategy.fuse_all_reduce_ops = False binary = fluid.CompiledProgram(train_program).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) iters = 5 batch_size = 8 - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) with fluid.scope_guard(scope): for idx in range(iters): data = next(train_reader()) - loss_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss]) + loss_v = exe.run( + binary, feed=feeder.feed(data), fetch_list=[loss] + ) if not for_ci and idx % 20 == 0: print('{}: {}'.format('loss', np.mean(loss_v))) @@ -142,19 +153,24 @@ def build_program(main, startup, is_test): qt.convert(test_program, scope) if not for_ci: with fluid.scope_guard(scope): - fluid.io.save_inference_model('./infer_model', - ['image', 'label'], [loss], - exe, - test_program, - clip_extra=True) + fluid.io.save_inference_model( + './infer_model', + ['image', 'label'], + [loss], + exe, + test_program, + clip_extra=True, + ) def test_gpu_1(self): if fluid.core.is_compiled_with_cuda(): - self.quantize_program(use_cuda=True, - seed=1, - activation_quant_type='abs_max', - weight_quant_type='abs_max', - for_ci=True) + self.quantize_program( + use_cuda=True, + seed=1, + activation_quant_type='abs_max', + weight_quant_type='abs_max', + for_ci=True, + ) def test_gpu_2(self): if fluid.core.is_compiled_with_cuda(): @@ -163,21 +179,26 @@ def test_gpu_2(self): seed=1, activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', - for_ci=True) + for_ci=True, + ) def test_cpu_1(self): - self.quantize_program(use_cuda=False, - seed=2, - activation_quant_type='abs_max', - weight_quant_type='abs_max', - for_ci=True) + self.quantize_program( + use_cuda=False, + seed=2, + activation_quant_type='abs_max', + weight_quant_type='abs_max', + for_ci=True, + ) def test_cpu_2(self): - self.quantize_program(use_cuda=False, - seed=2, - activation_quant_type='moving_average_abs_max', - weight_quant_type='channel_wise_abs_max', - for_ci=True) + self.quantize_program( + use_cuda=False, + seed=2, + activation_quant_type='moving_average_abs_max', + weight_quant_type='channel_wise_abs_max', + for_ci=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_user_defined_quantization.py b/python/paddle/fluid/contrib/slim/tests/test_user_defined_quantization.py index 361440f0238c7..96635700666e2 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_user_defined_quantization.py +++ b/python/paddle/fluid/contrib/slim/tests/test_user_defined_quantization.py @@ -36,21 +36,25 @@ def conv_net(img, label): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - pool_type='max', - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + pool_type='max', + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - pool_type='avg', - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + pool_type='avg', + act="relu", + ) hidden = fluid.layers.fc(input=conv_pool_2, size=100, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) @@ -66,41 +70,44 @@ def pact(x, name=None): name=x.name + '_pact', initializer=fluid.initializer.ConstantInitializer(value=init_thres), regularizer=fluid.regularizer.L2Decay(0.0001), - learning_rate=1) + learning_rate=1, + ) u_param = helper.create_parameter(attr=u_param_attr, shape=[1], dtype=dtype) x = fluid.layers.elementwise_sub( - x, fluid.layers.relu(fluid.layers.elementwise_sub(x, u_param))) + x, fluid.layers.relu(fluid.layers.elementwise_sub(x, u_param)) + ) x = fluid.layers.elementwise_add( - x, fluid.layers.relu(fluid.layers.elementwise_sub(-u_param, x))) + x, fluid.layers.relu(fluid.layers.elementwise_sub(-u_param, x)) + ) return x class TestUserDefinedQuantization(unittest.TestCase): - - def quantization_scale(self, - use_cuda, - seed, - activation_quant_type, - weight_quant_type='abs_max', - for_ci=False, - act_preprocess_func=None, - weight_preprocess_func=None, - act_quantize_func=None, - weight_quantize_func=None): - + def quantization_scale( + self, + use_cuda, + seed, + activation_quant_type, + weight_quant_type='abs_max', + for_ci=False, + act_preprocess_func=None, + weight_preprocess_func=None, + act_quantize_func=None, + weight_quantize_func=None, + ): def build_program(main, startup, is_test): main.random_seed = seed startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - img = fluid.layers.data(name='image', - shape=[1, 28, 28], - dtype='float32') + img = fluid.layers.data( + name='image', shape=[1, 28, 28], dtype='float32' + ) img.stop_gradient = False - label = fluid.layers.data(name='label', - shape=[1], - dtype='int64') + label = fluid.layers.data( + name='label', shape=[1], dtype='int64' + ) loss = conv_net(img, label) if not is_test: opt = fluid.optimizer.SGD(learning_rate=0.0001) @@ -149,7 +156,8 @@ def save_dict(Dict, mapping_table_path): act_quantize_func=act_quantize_func, weight_quantize_func=weight_quantize_func, optimizer_func=get_optimizer, - executor=exe) + executor=exe, + ) train_transform_pass.apply(main_graph) test_transform_pass = QuantizationTransformPass( scope=scope, @@ -161,7 +169,8 @@ def save_dict(Dict, mapping_table_path): act_quantize_func=act_quantize_func, weight_quantize_func=weight_quantize_func, optimizer_func=get_optimizer, - executor=exe) + executor=exe, + ) test_transform_pass.apply(test_graph) save_dict(test_graph.out_node_mapping_table, mapping_table_path) @@ -180,20 +189,22 @@ def save_dict(Dict, mapping_table_path): build_strategy.enable_inplace = False build_strategy.fuse_all_reduce_ops = False binary = fluid.CompiledProgram(main_graph.graph).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) iters = 5 batch_size = 8 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=batch_size, + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) with fluid.scope_guard(scope): for _ in range(iters): data = next(train_reader()) - loss_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss]) + loss_v = exe.run( + binary, feed=feeder.feed(data), fetch_list=[loss] + ) out_scale_infer_pass = OutScaleForInferencePass(scope=scope) out_scale_infer_pass.apply(test_graph) @@ -203,7 +214,8 @@ def save_dict(Dict, mapping_table_path): place=place, weight_bits=8, activation_bits=8, - weight_quantize_type=weight_quant_type) + weight_quantize_type=weight_quant_type, + ) mapping_table = load_dict(mapping_table_path) test_graph.out_node_mapping_table = mapping_table @@ -220,7 +232,8 @@ def test_act_preprocess_cuda(self): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - act_preprocess_func=pact) + act_preprocess_func=pact, + ) def test_act_preprocess_cpu(self): with fluid.unique_name.guard(): @@ -230,7 +243,8 @@ def test_act_preprocess_cpu(self): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - act_preprocess_func=pact) + act_preprocess_func=pact, + ) def test_weight_preprocess_cuda(self): if fluid.core.is_compiled_with_cuda(): @@ -241,7 +255,8 @@ def test_weight_preprocess_cuda(self): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - weight_preprocess_func=pact) + weight_preprocess_func=pact, + ) def test_weight_preprocess_cpu(self): with fluid.unique_name.guard(): @@ -251,7 +266,8 @@ def test_weight_preprocess_cpu(self): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - weight_preprocess_func=pact) + weight_preprocess_func=pact, + ) def test_act_quantize_cuda(self): if fluid.core.is_compiled_with_cuda(): @@ -262,7 +278,8 @@ def test_act_quantize_cuda(self): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - act_quantize_func=pact) + act_quantize_func=pact, + ) def test_act_quantize_cpu(self): with fluid.unique_name.guard(): @@ -272,7 +289,8 @@ def test_act_quantize_cpu(self): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - act_quantize_func=pact) + act_quantize_func=pact, + ) def test_weight_quantize_cuda(self): if fluid.core.is_compiled_with_cuda(): @@ -283,7 +301,8 @@ def test_weight_quantize_cuda(self): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - weight_quantize_func=pact) + weight_quantize_func=pact, + ) def test_weight_quantize_cpu(self): with fluid.unique_name.guard(): @@ -293,7 +312,8 @@ def test_weight_quantize_cpu(self): activation_quant_type='moving_average_abs_max', weight_quant_type='channel_wise_abs_max', for_ci=True, - weight_quantize_func=pact) + weight_quantize_func=pact, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/slim/tests/test_weight_quantization_mobilenetv1.py b/python/paddle/fluid/contrib/slim/tests/test_weight_quantization_mobilenetv1.py index e6af06c6ceac0..929eb34994b46 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_weight_quantization_mobilenetv1.py +++ b/python/paddle/fluid/contrib/slim/tests/test_weight_quantization_mobilenetv1.py @@ -28,8 +28,7 @@ def _load_variable_data(scope, var_name): Load variable value from scope ''' var_node = scope.find_var(var_name) - assert var_node is not None, \ - "Cannot find " + var_name + " in scope." + assert var_node is not None, "Cannot find " + var_name + " in scope." return np.array(var_node.get_tensor()) @@ -37,8 +36,9 @@ def _set_variable_data(scope, place, var_name, np_value): ''' Set the value of var node by name, if the node exits, ''' - assert isinstance(np_value, np.ndarray), \ - 'The type of value should be numpy array.' + assert isinstance( + np_value, np.ndarray + ), 'The type of value should be numpy array.' var_node = scope.find_var(var_name) if var_node != None: tensor = var_node.get_tensor() @@ -46,11 +46,11 @@ def _set_variable_data(scope, place, var_name, np_value): class TestWeightQuantization(unittest.TestCase): - def setUp(self): self.weight_quantization_dir = 'weight_quantization' - self.cache_folder = os.path.join(DATA_HOME, - self.weight_quantization_dir) + self.cache_folder = os.path.join( + DATA_HOME, self.weight_quantization_dir + ) def download_model(self, model_name, data_url, data_md5): download(data_url, self.weight_quantization_dir, data_md5) @@ -66,21 +66,32 @@ def download_model(self, model_name, data_url, data_md5): def cache_unzipping(self, target_folder, zip_path): if not os.path.exists(target_folder): cmd = 'mkdir {0} && tar xf {1} -C {0}'.format( - target_folder, zip_path) + target_folder, zip_path + ) os.system(cmd) - def quantize_to_int(self, model_name, model_data_url, model_data_md5, - weight_bits, quantizable_op_type, weight_quantize_type, - generate_test_model, threshold_rate): - - model_dir = self.download_model(model_name, model_data_url, - model_data_md5) + def quantize_to_int( + self, + model_name, + model_data_url, + model_data_md5, + weight_bits, + quantizable_op_type, + weight_quantize_type, + generate_test_model, + threshold_rate, + ): + + model_dir = self.download_model( + model_name, model_data_url, model_data_md5 + ) load_model_dir = os.path.join(model_dir, model_name) timestamp = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime()) save_model_dir = os.path.join( os.getcwd(), - model_name + "_wq_" + str(weight_bits) + "_" + timestamp) + model_name + "_wq_" + str(weight_bits) + "_" + timestamp, + ) weight_quant = WeightQuantization(model_dir=load_model_dir) weight_quant.quantize_weight_to_int( @@ -89,39 +100,54 @@ def quantize_to_int(self, model_name, model_data_url, model_data_md5, quantizable_op_type=quantizable_op_type, weight_quantize_type=weight_quantize_type, generate_test_model=generate_test_model, - threshold_rate=threshold_rate) + threshold_rate=threshold_rate, + ) print("finish weight quantization for " + model_name + "\n") try: os.system("rm -rf {}".format(save_model_dir)) except Exception as e: - print("Failed to delete {} due to {}".format( - save_model_dir, str(e))) - - def convert_to_fp16(self, model_name, model_data_url, model_data_md5, - model_filename, params_filename): - model_dir = self.download_model(model_name, model_data_url, - model_data_md5) + print( + "Failed to delete {} due to {}".format(save_model_dir, str(e)) + ) + + def convert_to_fp16( + self, + model_name, + model_data_url, + model_data_md5, + model_filename, + params_filename, + ): + model_dir = self.download_model( + model_name, model_data_url, model_data_md5 + ) load_model_dir = os.path.join(model_dir, model_name) timestamp = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime()) - save_model_dir = os.path.join(os.getcwd(), - model_name + "_wq_fp16_" + timestamp) + save_model_dir = os.path.join( + os.getcwd(), model_name + "_wq_fp16_" + timestamp + ) - weight_quant = WeightQuantization(load_model_dir, model_filename, - params_filename) + weight_quant = WeightQuantization( + load_model_dir, model_filename, params_filename + ) weight_quant.convert_weight_to_fp16(save_model_dir) - print("finish converting the data type of weights to fp16 for " + - model_name) + print( + "finish converting the data type of weights to fp16 for " + + model_name + ) print("fp16 model saved in " + save_model_dir + "\n") input_data = np.ones([1, 3, 224, 224], dtype=np.float32) - res_fp32 = self.run_models(load_model_dir, model_filename, - params_filename, input_data, False) - res_fp16 = self.run_models(save_model_dir, model_filename, - params_filename, input_data, True) + res_fp32 = self.run_models( + load_model_dir, model_filename, params_filename, input_data, False + ) + res_fp16 = self.run_models( + save_model_dir, model_filename, params_filename, input_data, True + ) np.testing.assert_allclose( res_fp32, @@ -129,40 +155,60 @@ def convert_to_fp16(self, model_name, model_data_url, model_data_md5, rtol=1e-05, atol=1e-08, equal_nan=True, - err_msg='Failed to test the accuracy of the fp32 and fp16 model.') + err_msg='Failed to test the accuracy of the fp32 and fp16 model.', + ) try: os.system("rm -rf {}".format(save_model_dir)) except Exception as e: - print("Failed to delete {} due to {}".format( - save_model_dir, str(e))) - - def run_models(self, model_dir, model_filename, params_filename, input_data, - is_fp16_model): + print( + "Failed to delete {} due to {}".format(save_model_dir, str(e)) + ) + + def run_models( + self, + model_dir, + model_filename, + params_filename, + input_data, + is_fp16_model, + ): print(model_dir) place = paddle.CPUPlace() exe = paddle.static.Executor(place) scope = paddle.static.Scope() with paddle.static.scope_guard(scope): - [inference_program, feed_target_names, fetch_targets] = \ - paddle.fluid.io.load_inference_model(model_dir, exe, - model_filename=model_filename, - params_filename=params_filename) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.fluid.io.load_inference_model( + model_dir, + exe, + model_filename=model_filename, + params_filename=params_filename, + ) if is_fp16_model: for var in inference_program.list_vars(): - if (var.type == paddle.fluid.core.VarDesc.VarType.RAW) or \ - (not var.persistable) or (var.name in ['feed', 'fetch']) \ - or (var.dtype != paddle.fluid.core.VarDesc.VarType.FP16): + if ( + (var.type == paddle.fluid.core.VarDesc.VarType.RAW) + or (not var.persistable) + or (var.name in ['feed', 'fetch']) + or (var.dtype != paddle.fluid.core.VarDesc.VarType.FP16) + ): continue tensor = _load_variable_data(scope, var.name) - _set_variable_data(scope, place, var.name, - tensor.astype(np.float32)) - - results = exe.run(inference_program, - feed={feed_target_names[0]: input_data}, - fetch_list=fetch_targets) + _set_variable_data( + scope, place, var.name, tensor.astype(np.float32) + ) + + results = exe.run( + inference_program, + feed={feed_target_names[0]: input_data}, + fetch_list=fetch_targets, + ) return np.array(results[0]) @@ -181,10 +227,16 @@ def test_weight_quantization_mobilenetv1_8bit_abs_max(self): weight_quantize_type = "abs_max" generate_test_model = True threshold_rate = 0.0 - self.quantize_to_int(self.nocomb_model_name, self.nocomb_model_data_url, - self.nocomb_model_data_md5, weight_bits, - quantizable_op_type, weight_quantize_type, - generate_test_model, threshold_rate) + self.quantize_to_int( + self.nocomb_model_name, + self.nocomb_model_data_url, + self.nocomb_model_data_md5, + weight_bits, + quantizable_op_type, + weight_quantize_type, + generate_test_model, + threshold_rate, + ) def test_weight_quantization_mobilenetv1_8bit_channel_wise_abs_max(self): weight_bits = 8 @@ -192,10 +244,16 @@ def test_weight_quantization_mobilenetv1_8bit_channel_wise_abs_max(self): weight_quantize_type = "channel_wise_abs_max" generate_test_model = True threshold_rate = 0.0 - self.quantize_to_int(self.nocomb_model_name, self.nocomb_model_data_url, - self.nocomb_model_data_md5, weight_bits, - quantizable_op_type, weight_quantize_type, - generate_test_model, threshold_rate) + self.quantize_to_int( + self.nocomb_model_name, + self.nocomb_model_data_url, + self.nocomb_model_data_md5, + weight_bits, + quantizable_op_type, + weight_quantize_type, + generate_test_model, + threshold_rate, + ) def test_weight_quantization_mobilenetv1_16bit_abs_max(self): weight_bits = 16 @@ -203,10 +261,16 @@ def test_weight_quantization_mobilenetv1_16bit_abs_max(self): weight_quantize_type = "abs_max" generate_test_model = False threshold_rate = 0 - self.quantize_to_int(self.nocomb_model_name, self.nocomb_model_data_url, - self.nocomb_model_data_md5, weight_bits, - quantizable_op_type, weight_quantize_type, - generate_test_model, threshold_rate) + self.quantize_to_int( + self.nocomb_model_name, + self.nocomb_model_data_url, + self.nocomb_model_data_md5, + weight_bits, + quantizable_op_type, + weight_quantize_type, + generate_test_model, + threshold_rate, + ) def test_weight_quantization_mobilenetv1_16bit_channel_wise_abs_max(self): weight_bits = 16 @@ -214,24 +278,38 @@ def test_weight_quantization_mobilenetv1_16bit_channel_wise_abs_max(self): weight_quantize_type = "channel_wise_abs_max" generate_test_model = False threshold_rate = 1e-9 - self.quantize_to_int(self.nocomb_model_name, self.nocomb_model_data_url, - self.nocomb_model_data_md5, weight_bits, - quantizable_op_type, weight_quantize_type, - generate_test_model, threshold_rate) + self.quantize_to_int( + self.nocomb_model_name, + self.nocomb_model_data_url, + self.nocomb_model_data_md5, + weight_bits, + quantizable_op_type, + weight_quantize_type, + generate_test_model, + threshold_rate, + ) def test_mobilenetv1_fp16_combined(self): model_filename = '__model__' params_filename = '__params__' - self.convert_to_fp16(self.comb_model_name, self.comb_model_data_url, - self.comb_model_data_md5, model_filename, - params_filename) + self.convert_to_fp16( + self.comb_model_name, + self.comb_model_data_url, + self.comb_model_data_md5, + model_filename, + params_filename, + ) def test_mobilenetv1_fp16_nocombined(self): model_filename = None params_filename = None - self.convert_to_fp16(self.nocomb_model_name, self.nocomb_model_data_url, - self.nocomb_model_data_md5, model_filename, - params_filename) + self.convert_to_fp16( + self.nocomb_model_name, + self.nocomb_model_data_url, + self.nocomb_model_data_md5, + model_filename, + params_filename, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/sparsity/__init__.py b/python/paddle/fluid/contrib/sparsity/__init__.py index 43c645057e8b5..fcb7acda377a0 100644 --- a/python/paddle/fluid/contrib/sparsity/__init__.py +++ b/python/paddle/fluid/contrib/sparsity/__init__.py @@ -30,8 +30,19 @@ from .supported_layer_list import add_supported_layer __all__ = [ - 'calculate_density', 'check_mask_1d', 'get_mask_1d', 'check_mask_2d', - 'get_mask_2d_greedy', 'get_mask_2d_best', 'create_mask', 'check_sparsity', - 'MaskAlgo', 'CheckMethod', 'decorate', 'prune_model', 'set_excluded_layers', - 'reset_excluded_layers', 'add_supported_layer' + 'calculate_density', + 'check_mask_1d', + 'get_mask_1d', + 'check_mask_2d', + 'get_mask_2d_greedy', + 'get_mask_2d_best', + 'create_mask', + 'check_sparsity', + 'MaskAlgo', + 'CheckMethod', + 'decorate', + 'prune_model', + 'set_excluded_layers', + 'reset_excluded_layers', + 'add_supported_layer', ] diff --git a/python/paddle/fluid/contrib/sparsity/asp.py b/python/paddle/fluid/contrib/sparsity/asp.py index ac97bcecf323f..d770bd36e3980 100644 --- a/python/paddle/fluid/contrib/sparsity/asp.py +++ b/python/paddle/fluid/contrib/sparsity/asp.py @@ -25,14 +25,19 @@ from paddle.fluid.initializer import ConstantInitializer from paddle.fluid.contrib import sparsity from paddle.fluid import core -from paddle.fluid.contrib.sparsity.supported_layer_list import supported_layers_and_prune_func_map +from paddle.fluid.contrib.sparsity.supported_layer_list import ( + supported_layers_and_prune_func_map, +) from paddle.fluid.contrib.sparsity.supported_layer_list import _default_pruning OpRole = core.op_proto_and_checker_maker.OpRole OP_ROLE_KEY = core.op_proto_and_checker_maker.kOpRoleAttrName() __all__ = [ - 'decorate', 'prune_model', 'set_excluded_layers', 'reset_excluded_layers' + 'decorate', + 'prune_model', + 'set_excluded_layers', + 'reset_excluded_layers', ] @@ -118,8 +123,9 @@ def forward(self, img): """ if main_program is None: main_program = paddle.static.default_main_program() - ASPHelper.set_excluded_layers(param_names=param_names, - main_program=main_program) + ASPHelper.set_excluded_layers( + param_names=param_names, main_program=main_program + ) def reset_excluded_layers(main_program=None): @@ -437,32 +443,39 @@ def forward(self, img): MaskAlgo_mapping = { 'mask_1d': sparsity.MaskAlgo.MASK_1D, 'mask_2d_greedy': sparsity.MaskAlgo.MASK_2D_GREEDY, - 'mask_2d_best': sparsity.MaskAlgo.MASK_2D_BEST + 'mask_2d_best': sparsity.MaskAlgo.MASK_2D_BEST, } - assert (mask_algo in MaskAlgo_mapping), \ - 'The "mask_algo" should be one of ["mask_1d", "mask_2d_greedy", "mask_2d_best"]' + assert ( + mask_algo in MaskAlgo_mapping + ), 'The "mask_algo" should be one of ["mask_1d", "mask_2d_greedy", "mask_2d_best"]' prune_func = None if isinstance(model, paddle.nn.Layer): prune_func = ASPHelper.prune_model_by_layer elif isinstance(model, paddle.static.Program): prune_func = ASPHelper.prune_model_by_program - if hasattr(model, "distributed_info_") and \ - model.distributed_info_["sharding_degree"] > 1 and \ - paddle.fluid.is_compiled_with_cuda(): + if ( + hasattr(model, "distributed_info_") + and model.distributed_info_["sharding_degree"] > 1 + and paddle.fluid.is_compiled_with_cuda() + ): gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0)) place = paddle.CUDAPlace(gpu_id) else: raise TypeError( - "model should be paddle.nn.Layer or paddle.static.Program, but got {}" - .format(type(model))) + "model should be paddle.nn.Layer or paddle.static.Program, but got {}".format( + type(model) + ) + ) - return prune_func(place, - model, - n=n, - m=m, - mask_algo=MaskAlgo_mapping[mask_algo], - with_mask=with_mask) + return prune_func( + place, + model, + n=n, + m=m, + mask_algo=MaskAlgo_mapping[mask_algo], + with_mask=with_mask, + ) class ProgramASPInfo(object): @@ -547,18 +560,21 @@ def decorate(optimizer): # default_main_program as the key. main_prog = paddle.static.default_main_program() startup_prog = paddle.static.default_startup_program() - ASPHelper._create_mask_variables(main_prog, startup_prog, - optimizer._parameter_list) + ASPHelper._create_mask_variables( + main_prog, startup_prog, optimizer._parameter_list + ) return OptimizerWithSparsityGuarantee(optimizer) @classmethod - def prune_model_by_program(cls, - place, - main_program=None, - n=2, - m=4, - mask_algo=sparsity.MaskAlgo.MASK_1D, - with_mask=True): + def prune_model_by_program( + cls, + place, + main_program=None, + n=2, + m=4, + mask_algo=sparsity.MaskAlgo.MASK_1D, + with_mask=True, + ): r""" This is the implementation of `sparsity.prune_model`, for details please see explanation in `sparsity.prune_model`. """ @@ -574,34 +590,43 @@ def prune_model_by_program(cls, prune_func = ASPHelper._get_prune_func_by_name(param.name) - weight_pruned_nparray, weight_sparse_mask = \ - prune_func(weight_nparray, m, n, mask_algo, param.name) + weight_pruned_nparray, weight_sparse_mask = prune_func( + weight_nparray, m, n, mask_algo, param.name + ) weight_pruned_nparray = weight_pruned_nparray.astype( - weight_nparray.dtype) + weight_nparray.dtype + ) weight_tensor.set(weight_pruned_nparray, place) if with_mask: weight_mask_param = global_scope().find_var( - ASPHelper._get_mask_name(param.name)) - assert weight_mask_param is not None, \ - 'Cannot find {} variable, please call optimizer.minimize (' \ - 'paddle.sparsity.decorate(optimizer).minimize(loss)' \ - ' and initialization (exe.run(startup_program)) first!'.format(ASPHelper._get_mask_name(param.name)) + ASPHelper._get_mask_name(param.name) + ) + assert weight_mask_param is not None, ( + 'Cannot find {} variable, please call optimizer.minimize (' + 'paddle.sparsity.decorate(optimizer).minimize(loss)' + ' and initialization (exe.run(startup_program)) first!'.format( + ASPHelper._get_mask_name(param.name) + ) + ) weight_mask_tensor = weight_mask_param.get_tensor() weight_sparse_mask = weight_sparse_mask.astype( - np.array(weight_mask_tensor).dtype) + np.array(weight_mask_tensor).dtype + ) weight_mask_tensor.set(weight_sparse_mask, place) asp_info.update_masks(param.name, weight_sparse_mask) return asp_info.masks.copy() @classmethod - def prune_model_by_layer(cls, - place, - layer, - n=2, - m=4, - mask_algo=sparsity.MaskAlgo.MASK_1D, - with_mask=True): + def prune_model_by_layer( + cls, + place, + layer, + n=2, + m=4, + mask_algo=sparsity.MaskAlgo.MASK_1D, + with_mask=True, + ): r""" This is the implementation of `sparsity.prune_model`, for details please see explanation in `sparsity.prune_model`. """ @@ -615,19 +640,25 @@ def prune_model_by_layer(cls, prune_func = ASPHelper._get_prune_func_by_name(param.name) - weight_pruned_nparray, weight_sparse_mask = \ - prune_func(weight_nparray, m, n, mask_algo, param.name) + weight_pruned_nparray, weight_sparse_mask = prune_func( + weight_nparray, m, n, mask_algo, param.name + ) weight_pruned_nparray = weight_pruned_nparray.astype( - weight_nparray.dtype) + weight_nparray.dtype + ) param.set_value(weight_pruned_nparray) if with_mask: weight_mask_param = asp_info.mask_vars.get( - param.name, None) - assert weight_mask_param is not None, \ - 'Cannot find {} variable, please call sparsity.decorate() to' \ - ' decorate your optimizer first!'.format(ASPHelper._get_mask_name(param.name)) + param.name, None + ) + assert weight_mask_param is not None, ( + 'Cannot find {} variable, please call sparsity.decorate() to' + ' decorate your optimizer first!'.format( + ASPHelper._get_mask_name(param.name) + ) + ) weight_mask_param.set_value(weight_sparse_mask) asp_info.update_masks(param.name, weight_sparse_mask) @@ -639,14 +670,17 @@ def prune_model_by_layer(cls, target_program = None for param in layer.parameters(): target_program = param.block.program - assert target_program is not None, \ - 'Cannot get paddle.static.Program from Paddle.nn.Layer.' - return ASPHelper.prune_model_by_program(place, - target_program, - n=n, - m=m, - mask_algo=mask_algo, - with_mask=with_mask) + assert ( + target_program is not None + ), 'Cannot get paddle.static.Program from Paddle.nn.Layer.' + return ASPHelper.prune_model_by_program( + place, + target_program, + n=n, + m=m, + mask_algo=mask_algo, + with_mask=with_mask, + ) @staticmethod def _get_mask_name(param_name): @@ -728,13 +762,16 @@ def _is_supported_layer(cls, main_program, param_name): param_name_no_weight_suffix = param_name_list[0] param_type_suffix = param_name_list[1] - layer_name = param_name_no_weight_suffix[:param_name_no_weight_suffix. - rfind('_')] + layer_name = param_name_no_weight_suffix[ + : param_name_no_weight_suffix.rfind('_') + ] if ASPHelper.PADDLE_WEIGHT_SUFFIX not in param_type_suffix: return False - if param_name_no_weight_suffix in supported_layers_and_prune_func_map or \ - layer_name in supported_layers_and_prune_func_map: + if ( + param_name_no_weight_suffix in supported_layers_and_prune_func_map + or layer_name in supported_layers_and_prune_func_map + ): return True return False @@ -745,23 +782,27 @@ def _get_prune_func_by_name(cls, param_name): param_name_no_weight_suffix = param_name.split('.')[0] if func is None: func = supported_layers_and_prune_func_map.get( - param_name_no_weight_suffix, None) + param_name_no_weight_suffix, None + ) if func is None: - layer_name = param_name_no_weight_suffix[: - param_name_no_weight_suffix - .rfind('_')] + layer_name = param_name_no_weight_suffix[ + : param_name_no_weight_suffix.rfind('_') + ] func = supported_layers_and_prune_func_map.get( - layer_name, _default_pruning) + layer_name, _default_pruning + ) return func @classmethod - def _minimize(cls, - optimizer, - loss, - main_program=None, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def _minimize( + cls, + optimizer, + loss, + main_program=None, + startup_program=None, + parameter_list=None, + no_grad_set=None, + ): r""" This function is a decorator of `minimize` function in `Optimizer`. There are three steps: @@ -792,7 +833,8 @@ def _minimize(cls, startup_program = paddle.static.default_startup_program() optimizer_ops, params_and_grads = optimizer.minimize( - loss, startup_program, parameter_list, no_grad_set=no_grad_set) + loss, startup_program, parameter_list, no_grad_set=no_grad_set + ) params_only = [pg[0] for pg in params_and_grads] cls._create_mask_variables(main_program, startup_program, params_only) @@ -819,8 +861,9 @@ def _step(cls, optimizer): optimizer.step() main_prog = paddle.static.default_main_program() with paddle.fluid.dygraph.no_grad(): - ASPHelper._insert_sparse_mask_ops(main_prog, - optimizer._parameter_list) + ASPHelper._insert_sparse_mask_ops( + main_prog, optimizer._parameter_list + ) @classmethod def _create_mask_variables(cls, main_program, startup_program, params): @@ -842,7 +885,8 @@ def _create_mask_variables(cls, main_program, startup_program, params): name=ASPHelper._get_mask_name(param.name), shape=param.shape, dtype=param.dtype, - default_initializer=ConstantInitializer(value=1.0)) + default_initializer=ConstantInitializer(value=1.0), + ) mask_param.stop_gradient = True mask_param.trainable = False asp_info.update_mask_vars(param.name, mask_param) @@ -861,17 +905,16 @@ def _insert_sparse_mask_ops(cls, main_program, params): asp_info = cls._get_program_asp_info(main_program) for param in params: if param.name in asp_info.mask_vars: - block.append_op(type='elementwise_mul', - inputs={ - "X": param, - 'Y': asp_info.mask_vars[param.name] - }, - outputs={'Out': param}, - attrs={ - 'axis': -1, - 'use_mkldnn': False, - OP_ROLE_KEY: int(OpRole.Optimize) - }) + block.append_op( + type='elementwise_mul', + inputs={"X": param, 'Y': asp_info.mask_vars[param.name]}, + outputs={'Out': param}, + attrs={ + 'axis': -1, + 'use_mkldnn': False, + OP_ROLE_KEY: int(OpRole.Optimize), + }, + ) class OptimizerWithSparsityGuarantee(object): @@ -889,11 +932,9 @@ def __init__(self, optimizer): def __getattr__(self, item): return getattr(self._optimizer, item) - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): r""" This function is to call `ASPHelper.minimize()` and return its return @@ -906,11 +947,13 @@ def minimize(self, list: operators from :attr:`optimizer`.minimize(:attr:`loss`). list: pairs of parameters and their gradients. """ - return ASPHelper._minimize(self._optimizer, - loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) + return ASPHelper._minimize( + self._optimizer, + loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) @dygraph_only def step(self): @@ -940,7 +983,8 @@ def state_dict(self): """ state_dict = self._optimizer.state_dict() asp_info = ASPHelper._get_program_asp_info( - paddle.static.default_main_program()) + paddle.static.default_main_program() + ) for param_name, var in asp_info.mask_vars.items(): state_dict.update({ASPHelper._get_mask_name(param_name): var}) return state_dict @@ -955,11 +999,13 @@ def set_state_dict(self, state_dict): None """ asp_info = ASPHelper._get_program_asp_info( - paddle.static.default_main_program()) + paddle.static.default_main_program() + ) for param_name, var in asp_info.mask_vars.items(): param_mask_name = ASPHelper._get_mask_name(param_name) - assert param_mask_name in state_dict, \ - "The {} is not found.".format(param_mask_name) + assert param_mask_name in state_dict, "The {} is not found.".format( + param_mask_name + ) var.set_value(state_dict[param_mask_name]) asp_info.update_masks(param_name, var.numpy()) return self._optimizer.set_state_dict(state_dict) diff --git a/python/paddle/fluid/contrib/sparsity/supported_layer_list.py b/python/paddle/fluid/contrib/sparsity/supported_layer_list.py index 38dd428e0f095..f55a877b4b7f3 100644 --- a/python/paddle/fluid/contrib/sparsity/supported_layer_list.py +++ b/python/paddle/fluid/contrib/sparsity/supported_layer_list.py @@ -23,9 +23,9 @@ __all__ = ['add_supported_layer'] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) def _default_pruning(weight_nparray, m, n, func_name, param_name): @@ -38,13 +38,17 @@ def _default_pruning(weight_nparray, m, n, func_name, param_name): exlude_cond_shape4 = len(shape) == 4 and shape[1] < m if exlude_cond_shape2: _logger.warning( - '{} is not pruned because the first dimension of {} is smaller than {}' - .format(param_name, shape, m)) + '{} is not pruned because the first dimension of {} is smaller than {}'.format( + param_name, shape, m + ) + ) return weight_pruned_nparray, weight_sparse_mask if exlude_cond_shape4: _logger.warning( - '{} is not pruned because the second dimension of {} is smaller than {}' - .format(param_name, shape, m)) + '{} is not pruned because the second dimension of {} is smaller than {}'.format( + param_name, shape, m + ) + ) return weight_pruned_nparray, weight_sparse_mask checked_func_name = sparsity.CheckMethod.get_checking_method(func_name) @@ -60,13 +64,13 @@ def _default_pruning(weight_nparray, m, n, func_name, param_name): # sparsity/utils is row-major pruning. That is the reason we have to transpose weight # matrices beforce invoking create_mask. Then we transpose the result mask to make # sure its shape to be the same as the input weight. - weight_sparse_mask = sparsity.create_mask(weight_nparray.T, - func_name=func_name, - n=n, - m=m).T + weight_sparse_mask = sparsity.create_mask( + weight_nparray.T, func_name=func_name, n=n, m=m + ).T weight_pruned_nparray = np.multiply(weight_nparray, weight_sparse_mask) - assert sparsity.check_sparsity(weight_pruned_nparray.T, n=n, m=m, func_name=checked_func_name), \ - 'Pruning {} weight matrix failure!!!'.format(param_name) + assert sparsity.check_sparsity( + weight_pruned_nparray.T, n=n, m=m, func_name=checked_func_name + ), 'Pruning {} weight matrix failure!!!'.format(param_name) return weight_pruned_nparray, weight_sparse_mask @@ -93,13 +97,18 @@ def add_supported_layer(layer, pruning_func=None): name = layer elif isinstance(layer, paddle.fluid.dygraph.layers.Layer): name = paddle.fluid.dygraph.layers._convert_camel_to_snake( - type(layer).__name__) + type(layer).__name__ + ) elif issubclass(layer, paddle.fluid.dygraph.layers.Layer): name = paddle.fluid.dygraph.layers._convert_camel_to_snake( - layer.__name__) + layer.__name__ + ) else: - assert "The type of layer should be string of Layer, but got {}!".format( - type(layer)) + assert ( + "The type of layer should be string of Layer, but got {}!".format( + type(layer) + ) + ) if pruning_func is None: pruning_func = _default_pruning _supported_layers_and_prune_func_map_lock.acquire() diff --git a/python/paddle/fluid/contrib/sparsity/utils.py b/python/paddle/fluid/contrib/sparsity/utils.py index 82846920031a4..b5be3887380ae 100644 --- a/python/paddle/fluid/contrib/sparsity/utils.py +++ b/python/paddle/fluid/contrib/sparsity/utils.py @@ -25,9 +25,16 @@ import threading __all__ = [ - 'calculate_density', 'check_mask_1d', 'get_mask_1d', 'check_mask_2d', - 'get_mask_2d_greedy', 'get_mask_2d_best', 'create_mask', 'check_sparsity', - 'MaskAlgo', 'CheckMethod' + 'calculate_density', + 'check_mask_1d', + 'get_mask_1d', + 'check_mask_2d', + 'get_mask_2d_greedy', + 'get_mask_2d_best', + 'create_mask', + 'check_sparsity', + 'MaskAlgo', + 'CheckMethod', ] @@ -74,8 +81,9 @@ def get_checking_method(mask_algo): CheckMethod.get_checking_method(MaskAlgo.MASK_2D_BEST) # CheckMethod.CHECK_2D """ - assert isinstance(mask_algo, MaskAlgo), \ - "mask_algo should be MaskAlgo type" + assert isinstance( + mask_algo, MaskAlgo + ), "mask_algo should be MaskAlgo type" if mask_algo == MaskAlgo.MASK_1D: return CheckMethod.CHECK_1D else: @@ -124,7 +132,7 @@ def _reshape_1d(mat, m): remainder = mat.shape[1] % m if mat.shape[1] % m > 0: mat_padded = np.zeros((mat.shape[0], mat.shape[1] + (m - remainder))) - mat_padded[:, :mat.shape[1]] = mat + mat_padded[:, : mat.shape[1]] = mat shape = mat_padded.shape return mat_padded.reshape(-1, m), shape else: @@ -211,7 +219,7 @@ def get_mask_1d(mat, n, m): min_order_indices = np.argsort(np.absolute(sub_mat)) mask_flattern[i, min_order_indices[:n].tolist()] = 0 mask_flattern = mask_flattern.reshape(shape) - mask[:, :] = mask_flattern[:, :mat.shape[1]] + mask[:, :] = mask_flattern[:, : mat.shape[1]] return mask @@ -237,12 +245,12 @@ def _reshape_2d(mat, m): remainder_0 = mat.shape[0] % m remainder_1 = mat.shape[1] % m - new_shape = (mat.shape[0] if remainder_0 == 0 \ - else mat.shape[0] + (m - remainder_0), - mat.shape[1] if remainder_1 == 0 \ - else mat.shape[1] + (m - remainder_1)) + new_shape = ( + mat.shape[0] if remainder_0 == 0 else mat.shape[0] + (m - remainder_0), + mat.shape[1] if remainder_1 == 0 else mat.shape[1] + (m - remainder_1), + ) mat_padded = np.zeros(new_shape) - mat_padded[:mat.shape[0], :mat.shape[1]] = mat + mat_padded[: mat.shape[0], : mat.shape[1]] = mat mat_flattern = np.empty(new_shape).reshape(-1, m * m) curr_idx = 0 @@ -250,9 +258,9 @@ def _reshape_2d(mat, m): row_end = row_start + m for col_start in range(0, mat_padded.shape[1], m): col_end = col_start + m - sub_mat = np.squeeze(mat_padded[row_start:row_end, \ - col_start:col_end] \ - .reshape(-1)) + sub_mat = np.squeeze( + mat_padded[row_start:row_end, col_start:col_end].reshape(-1) + ) mat_flattern[curr_idx] = sub_mat curr_idx += 1 return mat_flattern, mat_padded.shape @@ -302,8 +310,9 @@ def check_mask_2d(mat, n, m): mat_padded, shape = _reshape_2d(mat, m) for sub_mat in mat_padded: sub_mask = np.absolute(np.squeeze(sub_mat.reshape(m, m))) > 0 - if (np.sum(np.sum(sub_mask, axis=1) > (m-n)) != 0) and \ - (np.sum(np.sum(sub_mask, axis=0) > (m-n)) != 0): + if (np.sum(np.sum(sub_mask, axis=1) > (m - n)) != 0) and ( + np.sum(np.sum(sub_mask, axis=0) > (m - n)) != 0 + ): return False return True @@ -348,15 +357,17 @@ def get_mask_2d_greedy(mat, n, m): sub_mask = np.squeeze(mask_padded[idx]) min_order_1d_indices = np.argsort(sub_mat) - min_order_2d_indices = [(int(x / m), x % m) - for x in min_order_1d_indices] + min_order_2d_indices = [ + (int(x / m), x % m) for x in min_order_1d_indices + ] row_counter = collections.Counter() col_counter = collections.Counter() for i in range(len(min_order_1d_indices) - 1, -1, -1): matrix_entry = min_order_2d_indices[i] - if (row_counter[matrix_entry[0]] == n) or \ - (col_counter[matrix_entry[1]] == n): + if (row_counter[matrix_entry[0]] == n) or ( + col_counter[matrix_entry[1]] == n + ): continue sub_mask[matrix_entry[0], matrix_entry[1]] = 1.0 @@ -371,7 +382,7 @@ def get_mask_2d_greedy(mat, n, m): col_end = col_start + m mask[row_start:row_end, col_start:col_end] = mask_padded[curr_idx] curr_idx += 1 - return mask[:mat.shape[0], :mat.shape[1]] + return mask[: mat.shape[0], : mat.shape[1]] _valid_2d_patterns_lock = threading.Lock() @@ -404,8 +415,11 @@ def _compute_valid_2d_patterns(n, m): patterns = patterns + patterns patterns = np.asarray(list(set(permutations(patterns, m)))) - valid = ((patterns.sum(axis=1) <= n).sum( - axis=1) == m).nonzero()[0].reshape(-1) + valid = ( + ((patterns.sum(axis=1) <= n).sum(axis=1) == m) + .nonzero()[0] + .reshape(-1) + ) valid_patterns = np.empty((valid.shape[0], m, m)) valid_patterns[:] = patterns[valid[:]] @@ -452,9 +466,10 @@ def get_mask_2d_best(mat, n, m): mat_flattern, shape = _reshape_2d(mat, m) mask_flattern = np.ones_like(mat_flattern).reshape(-1, m, m) - pmax = np.argmax(np.matmul(mat_flattern, - patterns.reshape(patterns.shape[0], m * m).T), - axis=1) + pmax = np.argmax( + np.matmul(mat_flattern, patterns.reshape(patterns.shape[0], m * m).T), + axis=1, + ) mask_flattern[:] = patterns[pmax[:]] mask = np.empty(shape) @@ -466,7 +481,7 @@ def get_mask_2d_best(mat, n, m): col_end = col_start + m mask[row_start:row_end, col_start:col_end] = mask_flattern[curr_idx] curr_idx += 1 - return mask[:mat.shape[0], :mat.shape[1]] + return mask[: mat.shape[0], : mat.shape[1]] def create_mask(tensor, func_name=MaskAlgo.MASK_1D, n=2, m=4): @@ -506,9 +521,10 @@ def create_mask(tensor, func_name=MaskAlgo.MASK_1D, n=2, m=4): dtype = tensor.dtype t = tensor.astype(float) - assert isinstance(func_name, MaskAlgo), \ - "func_name argumet of create_mask is only accepted as type MaskAlgo. " \ - "But got {}".format(type(func_name)) + assert isinstance(func_name, MaskAlgo), ( + "func_name argumet of create_mask is only accepted as type MaskAlgo. " + "But got {}".format(type(func_name)) + ) func = getattr(sys.modules[__name__], func_name.value, None) if len(shape) == 1: t = t.reshape(1, shape[0]) @@ -518,14 +534,20 @@ def create_mask(tensor, func_name=MaskAlgo.MASK_1D, n=2, m=4): t = t.reshape(shape[0] * shape[1], shape[2]) # 4d-tensor conv (h, w, in, out) -> (h*w*out, in) in GemmConvKernel Op elif len(shape) == 4: - t = t.transpose([0, 1, 3, 2]).reshape(shape[0] * shape[1] * shape[3], - shape[2]) + t = t.transpose([0, 1, 3, 2]).reshape( + shape[0] * shape[1] * shape[3], shape[2] + ) mask = func(t, n=n, m=m) - return mask.reshape([shape[0], shape[1], shape[3], - shape[2]]).transpose([0, 1, 3, 2]).astype(dtype) + return ( + mask.reshape([shape[0], shape[1], shape[3], shape[2]]) + .transpose([0, 1, 3, 2]) + .astype(dtype) + ) else: - raise ValueError("The dimension of input tensor is not supported in create_mask, " \ - "Only dimension < 4 is supported but got {}".format(len(shape))) + raise ValueError( + "The dimension of input tensor is not supported in create_mask, " + "Only dimension < 4 is supported but got {}".format(len(shape)) + ) mask = func(t, n=n, m=m) return mask.reshape(shape).astype(dtype) @@ -564,9 +586,10 @@ def check_sparsity(tensor, func_name=CheckMethod.CHECK_1D, n=2, m=4): shape = tensor.shape t = tensor.astype(float) - assert type(func_name) == CheckMethod, \ - "func_name argumet of check_sparsity is only accepted as type CheckMethod. " \ - "But got {}".format(type(func_name)) + assert type(func_name) == CheckMethod, ( + "func_name argumet of check_sparsity is only accepted as type CheckMethod. " + "But got {}".format(type(func_name)) + ) func = getattr(sys.modules[__name__], func_name.value, None) if len(shape) == 1: t = t.reshape(1, shape[0]) @@ -576,10 +599,13 @@ def check_sparsity(tensor, func_name=CheckMethod.CHECK_1D, n=2, m=4): t = t.reshape(shape[0] * shape[1], shape[2]) # 4d-tensor conv (h, w, in, out) -> (h*w*out, in) in GemmConvKernel Op elif len(shape) == 4: - t = t.transpose([0, 1, 3, - 2]).reshape([shape[0] * shape[1] * shape[3], shape[2]]) + t = t.transpose([0, 1, 3, 2]).reshape( + [shape[0] * shape[1] * shape[3], shape[2]] + ) else: - raise ValueError("The dimension of input tensor is not supported in create_mask, " \ - "Only dimension < 4 is supported but got {}".format(len(shape))) + raise ValueError( + "The dimension of input tensor is not supported in create_mask, " + "Only dimension < 4 is supported but got {}".format(len(shape)) + ) return func(t, n=n, m=m) diff --git a/python/paddle/fluid/contrib/tests/test_amp_list.py b/python/paddle/fluid/contrib/tests/test_amp_list.py index 93c99b5ea1c8f..4a30185229a97 100644 --- a/python/paddle/fluid/contrib/tests/test_amp_list.py +++ b/python/paddle/fluid/contrib/tests/test_amp_list.py @@ -14,11 +14,12 @@ import paddle import unittest -from paddle.fluid.contrib.mixed_precision.fp16_lists import AutoMixedPrecisionLists +from paddle.fluid.contrib.mixed_precision.fp16_lists import ( + AutoMixedPrecisionLists, +) class TestAMPList(unittest.TestCase): - def test_main(self): custom_white_list = [ 'lookup_table', @@ -31,8 +32,11 @@ def test_main(self): self.assertTrue(op not in amp_list.unsupported_list) default_black_list = [ - 'linear_interp_v2', 'nearest_interp_v2', 'bilinear_interp_v2', - 'bicubic_interp_v2', 'trilinear_interp_v2' + 'linear_interp_v2', + 'nearest_interp_v2', + 'bilinear_interp_v2', + 'bicubic_interp_v2', + 'trilinear_interp_v2', ] for op in default_black_list: self.assertTrue(op in amp_list.black_list) diff --git a/python/paddle/fluid/contrib/tests/test_bf16_utils.py b/python/paddle/fluid/contrib/tests/test_bf16_utils.py index c456b1263ce23..5d528f226596c 100644 --- a/python/paddle/fluid/contrib/tests/test_bf16_utils.py +++ b/python/paddle/fluid/contrib/tests/test_bf16_utils.py @@ -22,7 +22,6 @@ class AMPTest(unittest.TestCase): - def setUp(self): self.bf16_list = copy.copy(amp.bf16.amp_lists.bf16_list) self.fp32_list = copy.copy(amp.bf16.amp_lists.fp32_list) @@ -63,7 +62,8 @@ def test_amp_lists_4(self): self.fp32_list.add('matmul_v2') self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_list={'matmul_v2'}) + custom_fp32_list={'matmul_v2'} + ) def test_amp_lists_5(self): # 5. w=None, b={'matmul_v2'} @@ -71,43 +71,47 @@ def test_amp_lists_5(self): self.bf16_list.remove('matmul_v2') self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_list={'matmul_v2'}) + custom_fp32_list={'matmul_v2'} + ) def test_amp_lists_6(self): # 6. w=None, b={'lstm'} self.fp32_list.add('lstm') self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_list={'lstm'}) + custom_fp32_list={'lstm'} + ) def test_amp_lists_7(self): self.fp32_list.add('reshape2') self.gray_list.remove('reshape2') self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_list={'reshape2'}) + custom_fp32_list={'reshape2'} + ) def test_amp_list_8(self): self.bf16_list.add('reshape2') self.gray_list.remove('reshape2') self.amp_lists_ = amp.bf16.AutoMixedPrecisionListsBF16( - custom_bf16_list={'reshape2'}) + custom_bf16_list={'reshape2'} + ) class AMPTest2(unittest.TestCase): - def test_amp_lists_(self): # 7. w={'lstm'} b={'lstm'} # raise ValueError - self.assertRaises(ValueError, amp.bf16.AutoMixedPrecisionListsBF16, - {'lstm'}, {'lstm'}) + self.assertRaises( + ValueError, amp.bf16.AutoMixedPrecisionListsBF16, {'lstm'}, {'lstm'} + ) def test_find_op_index(self): block = fluid.default_main_program().global_block() op_desc = core.OpDesc() idx = amp.bf16.amp_utils.find_op_index(block.desc, op_desc) - assert (idx == -1) + assert idx == -1 def test_is_in_fp32_varnames(self): block = fluid.default_main_program().global_block() @@ -115,17 +119,19 @@ def test_is_in_fp32_varnames(self): var1 = block.create_var(name="X", shape=[3], dtype='float32') var2 = block.create_var(name="Y", shape=[3], dtype='float32') var3 = block.create_var(name="Z", shape=[3], dtype='float32') - op1 = block.append_op(type="abs", - inputs={"X": [var1]}, - outputs={"Out": [var2]}) - op2 = block.append_op(type="abs", - inputs={"X": [var2]}, - outputs={"Out": [var3]}) + op1 = block.append_op( + type="abs", inputs={"X": [var1]}, outputs={"Out": [var2]} + ) + op2 = block.append_op( + type="abs", inputs={"X": [var2]}, outputs={"Out": [var3]} + ) amp_lists_1 = amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_varnames={'X'}) + custom_fp32_varnames={'X'} + ) assert amp.bf16.amp_utils._is_in_fp32_varnames(op1, amp_lists_1) amp_lists_2 = amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_varnames={'Y'}) + custom_fp32_varnames={'Y'} + ) assert amp.bf16.amp_utils._is_in_fp32_varnames(op2, amp_lists_2) assert amp.bf16.amp_utils._is_in_fp32_varnames(op1, amp_lists_2) @@ -136,14 +142,14 @@ def test_find_true_post_op(self): var1 = block.create_var(name="X", shape=[3], dtype='float32') var2 = block.create_var(name="Y", shape=[3], dtype='float32') var3 = block.create_var(name="Z", shape=[3], dtype='float32') - op1 = block.append_op(type="abs", - inputs={"X": [var1]}, - outputs={"Out": [var2]}) - op2 = block.append_op(type="abs", - inputs={"X": [var2]}, - outputs={"Out": [var3]}) + op1 = block.append_op( + type="abs", inputs={"X": [var1]}, outputs={"Out": [var2]} + ) + op2 = block.append_op( + type="abs", inputs={"X": [var2]}, outputs={"Out": [var3]} + ) res = amp.bf16.amp_utils.find_true_post_op(block.ops, op1, "Y") - assert (res == [op2]) + assert res == [op2] def test_find_true_post_op_with_search_all(self): program = fluid.Program() @@ -152,27 +158,23 @@ def test_find_true_post_op_with_search_all(self): var1 = block.create_var(name="X", shape=[3], dtype='float32') var2 = block.create_var(name="Y", shape=[3], dtype='float32') - inititializer_op = startup_block._prepend_op(type="fill_constant", - outputs={"Out": var1}, - attrs={ - "shape": var1.shape, - "dtype": var1.dtype, - "value": 1.0 - }) - - op1 = block.append_op(type="abs", - inputs={"X": [var1]}, - outputs={"Out": [var2]}) - result = amp.bf16.amp_utils.find_true_post_op(block.ops, - inititializer_op, - "X", - search_all=False) - assert (len(result) == 0) - result = amp.bf16.amp_utils.find_true_post_op(block.ops, - inititializer_op, - "X", - search_all=True) - assert (result == [op1]) + inititializer_op = startup_block._prepend_op( + type="fill_constant", + outputs={"Out": var1}, + attrs={"shape": var1.shape, "dtype": var1.dtype, "value": 1.0}, + ) + + op1 = block.append_op( + type="abs", inputs={"X": [var1]}, outputs={"Out": [var2]} + ) + result = amp.bf16.amp_utils.find_true_post_op( + block.ops, inititializer_op, "X", search_all=False + ) + assert len(result) == 0 + result = amp.bf16.amp_utils.find_true_post_op( + block.ops, inititializer_op, "X", search_all=True + ) + assert result == [op1] if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/tests/test_correlation.py b/python/paddle/fluid/contrib/tests/test_correlation.py index d3ab5dff601ec..553bb912bc045 100644 --- a/python/paddle/fluid/contrib/tests/test_correlation.py +++ b/python/paddle/fluid/contrib/tests/test_correlation.py @@ -21,22 +21,28 @@ paddle.enable_static() -def corr(x_1, - x_2, - pad_size=4, - kernel_size=1, - max_displacement=4, - stride1=1, - stride2=1, - corr_multiply=1): +def corr( + x_1, + x_2, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1, + corr_multiply=1, +): K = kernel_size - rinput1 = np.pad(x_1, ((0, 0), (0, 0), (pad_size, pad_size), - (pad_size, pad_size)), - mode='constant') - rinput2 = np.pad(x_2, ((0, 0), (0, 0), (pad_size, pad_size), - (pad_size, pad_size)), - mode='constant') + rinput1 = np.pad( + x_1, + ((0, 0), (0, 0), (pad_size, pad_size), (pad_size, pad_size)), + mode='constant', + ) + rinput2 = np.pad( + x_2, + ((0, 0), (0, 0), (pad_size, pad_size), (pad_size, pad_size)), + mode='constant', + ) rinput1 = np.transpose(rinput1, (0, 2, 3, 1)) rinput2 = np.transpose(rinput2, (0, 2, 3, 1)) B = int(rinput1.shape[0]) @@ -55,17 +61,23 @@ def corr(x_1, y1_index = j + pad_size x2_index = x1_index + k y2_index = y1_index + l - output[b, l + d + D * (k + d), i, - j] = np.mean(rinput1[b, x1_index:x1_index + K, - y1_index:y1_index + K] * - rinput2[b, x2_index:x2_index + K, - y2_index:y2_index + K]) + output[b, l + d + D * (k + d), i, j] = np.mean( + rinput1[ + b, + x1_index : x1_index + K, + y1_index : y1_index + K, + ] + * rinput2[ + b, + x2_index : x2_index + K, + y2_index : y2_index + K, + ] + ) return output class TestCorrelationOp(unittest.TestCase): - def test_check_output(self): if not fluid.core.is_compiled_with_cuda(): return @@ -73,34 +85,42 @@ def test_check_output(self): np.set_printoptions(threshold=np.inf) x_shape = (2, 10, 3, 3) x_type = 'float32' - x1 = fluid.layers.data(name='x1', - shape=x_shape, - dtype=x_type, - append_batch_size=False, - stop_gradient=False) - x2 = fluid.layers.data(name='x2', - shape=x_shape, - dtype=x_type, - append_batch_size=False, - stop_gradient=False) + x1 = fluid.layers.data( + name='x1', + shape=x_shape, + dtype=x_type, + append_batch_size=False, + stop_gradient=False, + ) + x2 = fluid.layers.data( + name='x2', + shape=x_shape, + dtype=x_type, + append_batch_size=False, + stop_gradient=False, + ) x1_np = np.random.randn(2, 3, 4, 5).astype(x_type) x2_np = np.random.randn(2, 3, 4, 5).astype(x_type) - out_np = corr(x1_np, - x2_np, - pad_size=4, - kernel_size=1, - max_displacement=4, - stride1=1, - stride2=1) - - out = fluid.contrib.correlation(x1, - x2, - pad_size=4, - kernel_size=1, - max_displacement=4, - stride1=1, - stride2=1) + out_np = corr( + x1_np, + x2_np, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1, + ) + + out = fluid.contrib.correlation( + x1, + x2, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1, + ) loss = fluid.layers.reduce_mean(out) optimizer = fluid.optimizer.Momentum(0.0001, 0.9) @@ -108,33 +128,31 @@ def test_check_output(self): place = fluid.CUDAPlace(0) exe = fluid.Executor(place) - res = exe.run(feed={ - 'x1': x1_np, - 'x2': x2_np - }, - fetch_list=[out.name, loss.name]) + res = exe.run( + feed={'x1': x1_np, 'x2': x2_np}, fetch_list=[out.name, loss.name] + ) np.testing.assert_allclose(res[0], out_np, rtol=1e-05, atol=1e-8) class Net(fluid.dygraph.Layer): - def __init__(self, name_scope): super(Net, self).__init__(name_scope) def forward(self, x1, x2): - y = fluid.contrib.correlation(x1, - x2, - pad_size=4, - kernel_size=1, - max_displacement=4, - stride1=1, - stride2=1) + y = fluid.contrib.correlation( + x1, + x2, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1, + ) return y class TestCorrelationOpDyGraph(unittest.TestCase): - def test_check_output(self): if not fluid.core.is_compiled_with_cuda(): return @@ -146,13 +164,15 @@ def test_check_output(self): with fluid.dygraph.guard(place): x1_np = np.random.randn(2, 3, 4, 5).astype(x_type) x2_np = np.random.randn(2, 3, 4, 5).astype(x_type) - out_np = corr(x1_np, - x2_np, - pad_size=4, - kernel_size=1, - max_displacement=4, - stride1=1, - stride2=1) + out_np = corr( + x1_np, + x2_np, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1, + ) x1 = to_variable(x1_np) x2 = to_variable(x2_np) diff --git a/python/paddle/fluid/contrib/tests/test_fp16_utils.py b/python/paddle/fluid/contrib/tests/test_fp16_utils.py index 54753ce4479a3..af245a30fed43 100644 --- a/python/paddle/fluid/contrib/tests/test_fp16_utils.py +++ b/python/paddle/fluid/contrib/tests/test_fp16_utils.py @@ -22,12 +22,11 @@ class AMPTest(unittest.TestCase): - def test_find_op_index(self): block = fluid.default_main_program().global_block() op_desc = core.OpDesc() idx = fp16_utils.find_op_index(block.desc, op_desc) - assert (idx == -1) + assert idx == -1 def test_find_true_post_op(self): block = fluid.default_main_program().global_block() @@ -35,14 +34,14 @@ def test_find_true_post_op(self): var1 = block.create_var(name="X", shape=[3], dtype='float32') var2 = block.create_var(name="Y", shape=[3], dtype='float32') var3 = block.create_var(name="Z", shape=[3], dtype='float32') - op1 = block.append_op(type="abs", - inputs={"X": [var1]}, - outputs={"Out": [var2]}) - op2 = block.append_op(type="abs", - inputs={"X": [var2]}, - outputs={"Out": [var3]}) + op1 = block.append_op( + type="abs", inputs={"X": [var1]}, outputs={"Out": [var2]} + ) + op2 = block.append_op( + type="abs", inputs={"X": [var2]}, outputs={"Out": [var3]} + ) res = fp16_utils.find_true_post_op(block.ops, op1, "Y") - assert (res == [op2]) + assert res == [op2] if __name__ == '__main__': diff --git a/python/paddle/fluid/contrib/tests/test_image_classification_fp16.py b/python/paddle/fluid/contrib/tests/test_image_classification_fp16.py index bf09a14545015..7edaeb2760bed 100644 --- a/python/paddle/fluid/contrib/tests/test_image_classification_fp16.py +++ b/python/paddle/fluid/contrib/tests/test_image_classification_fp16.py @@ -29,21 +29,18 @@ def resnet_cifar10(input, depth=32): - - def conv_bn_layer(input, - ch_out, - filter_size, - stride, - padding, - act='relu', - bias_attr=False): - tmp = fluid.layers.conv2d(input=input, - filter_size=filter_size, - num_filters=ch_out, - stride=stride, - padding=padding, - act=None, - bias_attr=bias_attr) + def conv_bn_layer( + input, ch_out, filter_size, stride, padding, act='relu', bias_attr=False + ): + tmp = fluid.layers.conv2d( + input=input, + filter_size=filter_size, + num_filters=ch_out, + stride=stride, + padding=padding, + act=None, + bias_attr=bias_attr, + ) return fluid.layers.batch_norm(input=tmp, act=act) def shortcut(input, ch_in, ch_out, stride): @@ -66,33 +63,31 @@ def layer_warp(block_func, input, ch_in, ch_out, count, stride): assert (depth - 2) % 6 == 0 n = (depth - 2) // 6 - conv1 = conv_bn_layer(input=input, - ch_out=16, - filter_size=3, - stride=1, - padding=1) + conv1 = conv_bn_layer( + input=input, ch_out=16, filter_size=3, stride=1, padding=1 + ) res1 = layer_warp(basicblock, conv1, 16, 16, n, 1) res2 = layer_warp(basicblock, res1, 16, 32, n, 2) res3 = layer_warp(basicblock, res2, 32, 64, n, 2) - pool = fluid.layers.pool2d(input=res3, - pool_size=8, - pool_type='avg', - pool_stride=1) + pool = fluid.layers.pool2d( + input=res3, pool_size=8, pool_type='avg', pool_stride=1 + ) return pool def vgg16_bn_drop(input): - def conv_block(input, num_filter, groups, dropouts): - return fluid.nets.img_conv_group(input=input, - pool_size=2, - pool_stride=2, - conv_num_filter=[num_filter] * groups, - conv_filter_size=3, - conv_act='relu', - conv_with_batchnorm=True, - conv_batchnorm_drop_rate=dropouts, - pool_type='max') + return fluid.nets.img_conv_group( + input=input, + pool_size=2, + pool_stride=2, + conv_num_filter=[num_filter] * groups, + conv_filter_size=3, + conv_act='relu', + conv_with_batchnorm=True, + conv_batchnorm_drop_rate=dropouts, + pool_type='max', + ) conv1 = conv_block(input, 64, 2, [0.3, 0]) conv2 = conv_block(conv1, 128, 2, [0.4, 0]) @@ -117,9 +112,9 @@ def train(net_type, use_cuda, save_dirname, is_local): train_program.random_seed = 123 startup_prog.random_seed = 456 with fluid.program_guard(train_program, startup_prog): - images = fluid.layers.data(name='pixel', - shape=data_shape, - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=data_shape, dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') if net_type == "vgg": @@ -133,7 +128,8 @@ def train(net_type, use_cuda, save_dirname, is_local): logits = fluid.layers.fc(input=net, size=classdim, act="softmax") cost, predict = fluid.layers.softmax_with_cross_entropy( - logits, label, return_softmax=True) + logits, label, return_softmax=True + ) avg_cost = paddle.mean(cost) acc = fluid.layers.accuracy(input=predict, label=label) @@ -143,11 +139,14 @@ def train(net_type, use_cuda, save_dirname, is_local): optimizer = fluid.optimizer.Lamb(learning_rate=0.001) amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - custom_black_varnames={"loss", "conv2d_0.w_0"}) - mp_optimizer = decorate(optimizer=optimizer, - amp_lists=amp_lists, - init_loss_scaling=8.0, - use_dynamic_loss_scaling=True) + custom_black_varnames={"loss", "conv2d_0.w_0"} + ) + mp_optimizer = decorate( + optimizer=optimizer, + amp_lists=amp_lists, + init_loss_scaling=8.0, + use_dynamic_loss_scaling=True, + ) mp_optimizer.minimize(avg_cost) loss_scaling = mp_optimizer.get_loss_scaling() @@ -157,11 +156,13 @@ def train(net_type, use_cuda, save_dirname, is_local): PASS_NUM = 1 # no shuffle for unit test - train_reader = paddle.batch(paddle.dataset.cifar.train10(), - batch_size=BATCH_SIZE) + train_reader = paddle.batch( + paddle.dataset.cifar.train10(), batch_size=BATCH_SIZE + ) - test_reader = paddle.batch(paddle.dataset.cifar.test10(), - batch_size=BATCH_SIZE) + test_reader = paddle.batch( + paddle.dataset.cifar.test10(), batch_size=BATCH_SIZE + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) @@ -175,18 +176,25 @@ def train_loop(main_program): np_scaled_loss, loss = exe.run( main_program, feed=feeder.feed(data), - fetch_list=[scaled_loss, avg_cost]) + fetch_list=[scaled_loss, avg_cost], + ) print( - 'PassID {0:1}, BatchID {1:04}, train loss {2:2.4}, scaled train closs {3:2.4}' - .format(pass_id, batch_id + 1, float(loss), - float(np_scaled_loss))) + 'PassID {0:1}, BatchID {1:04}, train loss {2:2.4}, scaled train closs {3:2.4}'.format( + pass_id, + batch_id + 1, + float(loss), + float(np_scaled_loss), + ) + ) if (batch_id % 10) == 0: acc_list = [] avg_loss_list = [] for tid, test_data in enumerate(test_reader()): - loss_t, acc_t = exe.run(program=test_program, - feed=feeder.feed(test_data), - fetch_list=[avg_cost, acc]) + loss_t, acc_t = exe.run( + program=test_program, + feed=feeder.feed(test_data), + fetch_list=[avg_cost, acc], + ) if math.isnan(float(loss_t)): sys.exit("got NaN loss, training failed.") acc_list.append(float(acc_t)) @@ -197,16 +205,23 @@ def train_loop(main_program): avg_loss_value = numpy.array(avg_loss_list).mean() print( - 'PassID {0:1}, BatchID {1:04}, test loss {2:2.2}, acc {3:2.2}' - .format(pass_id, batch_id + 1, float(avg_loss_value), - float(acc_value))) + 'PassID {0:1}, BatchID {1:04}, test loss {2:2.2}, acc {3:2.2}'.format( + pass_id, + batch_id + 1, + float(avg_loss_value), + float(acc_value), + ) + ) if acc_value > 0.08: # Low threshold for speeding up CI fluid.io.save_inference_model( - save_dirname, ["pixel"], [predict], + save_dirname, + ["pixel"], + [predict], exe, main_program=train_program, - clip_extra=True) + clip_extra=True, + ) return if is_local: @@ -226,8 +241,9 @@ def train_loop(main_program): t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -247,8 +263,11 @@ def infer(use_cuda, save_dirname=None): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(save_dirname, exe) # The input's dimension of conv should be 4-D or 5-D. # Use normilized image pixels as input data, which should be in the range [0, 1.0]. @@ -257,22 +276,25 @@ def infer(use_cuda, save_dirname=None): # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. - results = exe.run(inference_program, - feed={feed_target_names[0]: tensor_img}, - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed={feed_target_names[0]: tensor_img}, + fetch_list=fetch_targets, + ) print("infer results: ", results[0]) - fluid.io.save_inference_model(save_dirname, - feed_target_names, - fetch_targets, - exe, - inference_program, - clip_extra=True) + fluid.io.save_inference_model( + save_dirname, + feed_target_names, + fetch_targets, + exe, + inference_program, + clip_extra=True, + ) class TestImageClassification(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -286,18 +308,22 @@ def main(self, net_type, use_cuda, is_local=True): # Directory for saving the trained model save_dirname = os.path.join( self.temp_dir.name, - "image_classification_" + net_type + ".inference.model") + "image_classification_" + net_type + ".inference.model", + ) train(net_type, use_cuda, save_dirname, is_local) - #infer(use_cuda, save_dirname) + # infer(use_cuda, save_dirname) def test_amp_lists(self): white_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.white_list) + fluid.contrib.mixed_precision.fp16_lists.white_list + ) black_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.black_list) + fluid.contrib.mixed_precision.fp16_lists.black_list + ) gray_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.gray_list) + fluid.contrib.mixed_precision.fp16_lists.gray_list + ) amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists() self.assertEqual(amp_lists.white_list, white_list) @@ -306,106 +332,130 @@ def test_amp_lists(self): def test_amp_lists_1(self): white_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.white_list) + fluid.contrib.mixed_precision.fp16_lists.white_list + ) black_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.black_list) + fluid.contrib.mixed_precision.fp16_lists.black_list + ) gray_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.gray_list) + fluid.contrib.mixed_precision.fp16_lists.gray_list + ) # 1. w={'exp}, b=None white_list.add('exp') black_list.remove('exp') amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - {'exp'}) + {'exp'} + ) self.assertEqual(amp_lists.white_list, white_list) self.assertEqual(amp_lists.black_list, black_list) self.assertEqual(amp_lists.gray_list, gray_list) def test_amp_lists_2(self): white_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.white_list) + fluid.contrib.mixed_precision.fp16_lists.white_list + ) black_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.black_list) + fluid.contrib.mixed_precision.fp16_lists.black_list + ) gray_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.gray_list) + fluid.contrib.mixed_precision.fp16_lists.gray_list + ) # 2. w={'tanh'}, b=None white_list.add('tanh') gray_list.remove('tanh') amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - {'tanh'}) + {'tanh'} + ) self.assertEqual(amp_lists.white_list, white_list) self.assertEqual(amp_lists.black_list, black_list) self.assertEqual(amp_lists.gray_list, gray_list) def test_amp_lists_3(self): white_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.white_list) + fluid.contrib.mixed_precision.fp16_lists.white_list + ) black_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.black_list) + fluid.contrib.mixed_precision.fp16_lists.black_list + ) gray_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.gray_list) + fluid.contrib.mixed_precision.fp16_lists.gray_list + ) # 3. w={'lstm'}, b=None white_list.add('lstm') amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - {'lstm'}) + {'lstm'} + ) self.assertEqual(amp_lists.white_list, white_list) self.assertEqual(amp_lists.black_list, black_list) self.assertEqual(amp_lists.gray_list, gray_list) def test_amp_lists_4(self): white_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.white_list) + fluid.contrib.mixed_precision.fp16_lists.white_list + ) black_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.black_list) + fluid.contrib.mixed_precision.fp16_lists.black_list + ) gray_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.gray_list) + fluid.contrib.mixed_precision.fp16_lists.gray_list + ) # 4. w=None, b={'conv2d'} white_list.remove('conv2d') black_list.add('conv2d') amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - custom_black_list={'conv2d'}) + custom_black_list={'conv2d'} + ) self.assertEqual(amp_lists.white_list, white_list) self.assertEqual(amp_lists.black_list, black_list) self.assertEqual(amp_lists.gray_list, gray_list) def test_amp_lists_5(self): white_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.white_list) + fluid.contrib.mixed_precision.fp16_lists.white_list + ) black_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.black_list) + fluid.contrib.mixed_precision.fp16_lists.black_list + ) gray_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.gray_list) + fluid.contrib.mixed_precision.fp16_lists.gray_list + ) # 5. w=None, b={'tanh'} black_list.add('tanh') gray_list.remove('tanh') amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - custom_black_list={'tanh'}) + custom_black_list={'tanh'} + ) self.assertEqual(amp_lists.white_list, white_list) self.assertEqual(amp_lists.black_list, black_list) self.assertEqual(amp_lists.gray_list, gray_list) def test_amp_lists_6(self): white_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.white_list) + fluid.contrib.mixed_precision.fp16_lists.white_list + ) black_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.black_list) + fluid.contrib.mixed_precision.fp16_lists.black_list + ) gray_list = copy.copy( - fluid.contrib.mixed_precision.fp16_lists.gray_list) + fluid.contrib.mixed_precision.fp16_lists.gray_list + ) # 6. w=None, b={'lstm'} black_list.add('lstm') amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - custom_black_list={'lstm'}) + custom_black_list={'lstm'} + ) self.assertEqual(amp_lists.white_list, white_list) self.assertEqual(amp_lists.black_list, black_list) self.assertEqual(amp_lists.gray_list, gray_list) @@ -413,9 +463,12 @@ def test_amp_lists_6(self): def test_amp_lists_7(self): # 7. w={'lstm'} b={'lstm'} # raise ValueError - self.assertRaises(ValueError, - fluid.contrib.mixed_precision.AutoMixedPrecisionLists, - {'lstm'}, {'lstm'}) + self.assertRaises( + ValueError, + fluid.contrib.mixed_precision.AutoMixedPrecisionLists, + {'lstm'}, + {'lstm'}, + ) def test_vgg_cuda(self): with self.scope_prog_guard(): @@ -436,37 +489,43 @@ def scope_prog_guard(self): class TestAmpWithNonIterableDataLoader(unittest.TestCase): - def decorate_with_data_loader(self): main_prog = paddle.static.Program() start_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, start_prog): with paddle.fluid.unique_name.guard(): - image = fluid.layers.data(name='image', - shape=[3, 224, 224], - dtype='float32') - label = fluid.layers.data(name='label', - shape=[1], - dtype='int64') + image = fluid.layers.data( + name='image', shape=[3, 224, 224], dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='int64' + ) py_reader = fluid.io.DataLoader.from_generator( feed_list=[image, label], capacity=4, iterable=False, - use_double_buffer=False) + use_double_buffer=False, + ) net = vgg16_bn_drop(image) logits = fluid.layers.fc(input=net, size=10, act="softmax") cost, predict = fluid.layers.softmax_with_cross_entropy( - logits, label, return_softmax=True) + logits, label, return_softmax=True + ) avg_cost = paddle.mean(cost) optimizer = fluid.optimizer.Lamb(learning_rate=0.001) - amp_lists = fluid.contrib.mixed_precision.AutoMixedPrecisionLists( - custom_black_varnames={"loss", "conv2d_0.w_0"}) - mp_optimizer = decorate(optimizer=optimizer, - amp_lists=amp_lists, - init_loss_scaling=8.0, - use_dynamic_loss_scaling=True) + amp_lists = ( + fluid.contrib.mixed_precision.AutoMixedPrecisionLists( + custom_black_varnames={"loss", "conv2d_0.w_0"} + ) + ) + mp_optimizer = decorate( + optimizer=optimizer, + amp_lists=amp_lists, + init_loss_scaling=8.0, + use_dynamic_loss_scaling=True, + ) mp_optimizer.minimize(avg_cost) diff --git a/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py b/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py index ab79242ae0092..c000d55fccdfa 100644 --- a/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py +++ b/python/paddle/fluid/contrib/tests/test_model_cast_to_bf16.py @@ -30,7 +30,8 @@ def convert_uint16_to_float(in_list): in_list = np.asarray(in_list) out = np.vectorize( lambda x: struct.unpack(' 0 else: sys.stderr.write('Do not get AVX flag on %s\n' % sysstr) @@ -162,10 +187,10 @@ def asm_func(code_str, restype=ctypes.c_uint32, argtypes=()): def run_shell_command(cmd): import subprocess - out, err = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True).communicate() + + out, err = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True + ).communicate() if err: return None else: @@ -174,8 +199,9 @@ def run_shell_command(cmd): def get_dso_path(core_so, dso_name): if core_so and dso_name: - return run_shell_command("ldd %s|grep %s|awk '{print $3}'" % - (core_so, dso_name)) + return run_shell_command( + "ldd %s|grep %s|awk '{print $3}'" % (core_so, dso_name) + ) else: return None @@ -184,6 +210,7 @@ def load_dso(dso_absolute_path): if dso_absolute_path: try: from ctypes import cdll + cdll.LoadLibrary(dso_absolute_path) except: warnings.warn("Load {} failed".format(dso_absolute_path)) @@ -242,12 +269,14 @@ def to_list(s): try: from . import libpaddle + if avx_supported() and not libpaddle.is_compiled_with_avx(): sys.stderr.write( "Hint: Your machine support AVX, but the installed paddlepaddle doesn't have avx core. " "Hence, no-avx core with worse preformance will be imported.\nIf you like, you could " "reinstall paddlepaddle by 'python -m pip install --force-reinstall paddlepaddle-gpu[==version]' " - "to get better performance.\n") + "to get better performance.\n" + ) # assign tensor alias libpaddle.LoDTensor = libpaddle.Tensor @@ -278,6 +307,7 @@ def to_list(s): from .libpaddle import _Profiler, _ProfilerResult, _RecordEvent from .libpaddle import _set_current_stream from .libpaddle import _get_phi_kernel_name + if sys.platform != 'win32': from .libpaddle import _set_process_pids from .libpaddle import _erase_process_pids @@ -290,12 +320,18 @@ def to_list(s): except Exception as e: if has_paddle_dy_lib: sys.stderr.write( - 'Error: Can not import paddle core while this file exists: ' + - current_path + os.sep + 'libpaddle.' + dy_lib_suffix + '\n') + 'Error: Can not import paddle core while this file exists: ' + + current_path + + os.sep + + 'libpaddle.' + + dy_lib_suffix + + '\n' + ) if not avx_supported() and libpaddle.is_compiled_with_avx(): sys.stderr.write( "Error: Your machine doesn't support AVX, but the installed PaddlePaddle is avx core, " - "you should reinstall paddlepaddle with no-avx core.\n") + "you should reinstall paddlepaddle with no-avx core.\n" + ) raise e @@ -312,22 +348,26 @@ def set_paddle_custom_device_lib_path(lib_path): # set paddle lib path def set_paddle_lib_path(): - site_dirs = site.getsitepackages() if hasattr( - site, - 'getsitepackages') else [x for x in sys.path if 'site-packages' in x] + site_dirs = ( + site.getsitepackages() + if hasattr(site, 'getsitepackages') + else [x for x in sys.path if 'site-packages' in x] + ) for site_dir in site_dirs: lib_dir = os.path.sep.join([site_dir, 'paddle', 'libs']) if os.path.exists(lib_dir): _set_paddle_lib_path(lib_dir) set_paddle_custom_device_lib_path( - os.path.sep.join([lib_dir, '..', '..', 'paddle-plugins'])) + os.path.sep.join([lib_dir, '..', '..', 'paddle-plugins']) + ) return if hasattr(site, 'USER_SITE'): lib_dir = os.path.sep.join([site.USER_SITE, 'paddle', 'libs']) if os.path.exists(lib_dir): _set_paddle_lib_path(lib_dir) set_paddle_custom_device_lib_path( - os.path.sep.join([lib_dir, '..', '..', 'paddle-plugins'])) + os.path.sep.join([lib_dir, '..', '..', 'paddle-plugins']) + ) set_paddle_lib_path() diff --git a/python/paddle/fluid/data.py b/python/paddle/fluid/data.py index 99e38b9470b76..00173a29c28ac 100644 --- a/python/paddle/fluid/data.py +++ b/python/paddle/fluid/data.py @@ -114,11 +114,13 @@ def data(name, shape, dtype='float32', lod_level=0): if shape[i] is None: shape[i] = -1 - return helper.create_global_variable(name=name, - shape=shape, - dtype=dtype, - type=core.VarDesc.VarType.LOD_TENSOR, - stop_gradient=True, - lod_level=lod_level, - is_data=True, - need_check_feed=True) + return helper.create_global_variable( + name=name, + shape=shape, + dtype=dtype, + type=core.VarDesc.VarType.LOD_TENSOR, + stop_gradient=True, + lod_level=lod_level, + is_data=True, + need_check_feed=True, + ) diff --git a/python/paddle/fluid/data_feed_desc.py b/python/paddle/fluid/data_feed_desc.py index 0aa72cadaf581..7a58c5ef5e1b4 100644 --- a/python/paddle/fluid/data_feed_desc.py +++ b/python/paddle/fluid/data_feed_desc.py @@ -175,7 +175,8 @@ def set_dense_slots(self, dense_slots_name): ) for name in dense_slots_name: self.proto_desc.multi_slot_desc.slots[ - self.__name_to_index[name]].is_dense = True + self.__name_to_index[name] + ].is_dense = True def set_use_slots(self, use_slots_name): """ @@ -220,7 +221,8 @@ def set_use_slots(self, use_slots_name): ) for name in use_slots_name: self.proto_desc.multi_slot_desc.slots[ - self.__name_to_index[name]].is_used = True + self.__name_to_index[name] + ].is_used = True def desc(self): """ diff --git a/python/paddle/fluid/data_feeder.py b/python/paddle/fluid/data_feeder.py index a6bf7c6d4406d..e84d41e56204c 100644 --- a/python/paddle/fluid/data_feeder.py +++ b/python/paddle/fluid/data_feeder.py @@ -18,7 +18,13 @@ import multiprocessing import warnings -from .framework import Variable, default_main_program, _current_expected_place, _non_static_mode, _in_eager_without_dygraph_check +from .framework import ( + Variable, + default_main_program, + _current_expected_place, + _non_static_mode, + _in_eager_without_dygraph_check, +) from .framework import _cpu_num, _cuda_ids __all__ = ['DataFeeder'] @@ -45,18 +51,46 @@ def convert_dtype(dtype): return _PADDLE_DTYPE_2_NUMPY_DTYPE[dtype] elif isinstance(dtype, type): if dtype in [ - bool, np.float16, np.uint16, np.float32, np.float64, np.int8, - np.int16, np.int32, np.int64, np.uint8, np.complex64, - np.complex128 + bool, + np.float16, + np.uint16, + np.float32, + np.float64, + np.int8, + np.int16, + np.int32, + np.int64, + np.uint8, + np.complex64, + np.complex128, ]: return dtype.__name__ else: if dtype in [ - 'bool', 'float16', 'uint16', 'float32', 'float64', 'int8', - 'int16', 'int32', 'int64', 'uint8', 'complex64', 'complex128', - u'bool', u'float16', u'uint16', u'float32', u'float64', u'int8', - u'int16', u'int32', u'int64', u'uint8', u'complex64', - u'complex128' + 'bool', + 'float16', + 'uint16', + 'float32', + 'float64', + 'int8', + 'int16', + 'int32', + 'int64', + 'uint8', + 'complex64', + 'complex128', + u'bool', + u'float16', + u'uint16', + u'float32', + u'float64', + u'int8', + u'int16', + u'int32', + u'int64', + u'uint8', + u'complex64', + u'complex128', ]: # this code is a little bit dangerous, since error could happen # when casting no-ascii code to str in python2. @@ -70,14 +104,13 @@ def convert_dtype(dtype): raise TypeError( "dtype must be any of [bool, float16, uint16, float32, float64, int8, int16, " - "int32, int64, uint8, complex64, complex128], but received %s" % dtype) + "int32, int64, uint8, complex64, complex128], but received %s" % dtype + ) -def check_variable_and_dtype(input, - input_name, - expected_dtype, - op_name, - extra_message=''): +def check_variable_and_dtype( + input, input_name, expected_dtype, op_name, extra_message='' +): check_type(input, input_name, Variable, op_name, extra_message) check_dtype(input.dtype, input_name, expected_dtype, op_name, extra_message) @@ -97,60 +130,76 @@ def check_type(input, input_name, expected_type, op_name, extra_message=''): # @declarative in transformation from dygrah to static layer. We add VarBase in # expected_type to skip checking because varBase may be created and used in unusual way. from .dygraph.base import in_declarative_mode + # Need a better design to be fix this. if in_declarative_mode(): if not isinstance(expected_type, tuple): - expected_type = (expected_type, ) - expected_type += (core.VarBase, ) + expected_type = (expected_type,) + expected_type += (core.VarBase,) if _in_eager_without_dygraph_check(): - expected_type += (core.eager.Tensor, ) + expected_type += (core.eager.Tensor,) elif isinstance(input, core.VarBase): raise TypeError( "Please use `with fluid.dygraph.guard()` as context or `fluid.enable_dygraph()` to switch to imperative mode firstly. " "Because received '{}' in {} is a imperative Variable.".format( - input_name, op_name)) + input_name, op_name + ) + ) elif hasattr(core, "eager"): if isinstance(input, core.eager.Tensor): raise TypeError( "Please use `with fluid.dygraph.guard()` as context or `fluid.enable_dygraph()` to switch to imperative mode firstly. " "Because received '{}' in {} is a imperative Variable.".format( - input_name, op_name)) + input_name, op_name + ) + ) if not isinstance(input, expected_type): raise TypeError( - "The type of '%s' in %s must be %s, but received %s. %s" % - (input_name, op_name, expected_type, type(input), extra_message)) + "The type of '%s' in %s must be %s, but received %s. %s" + % (input_name, op_name, expected_type, type(input), extra_message) + ) -def check_dtype(input_dtype, - input_name, - expected_dtype, - op_name, - extra_message=''): +def check_dtype( + input_dtype, input_name, expected_dtype, op_name, extra_message='' +): # See NOTE [ Why skip dynamic graph check ] if _non_static_mode(): return if convert_dtype(input_dtype) in ['float16']: warnings.warn( - "The data type of '%s' in %s only support float16 in GPU now. %s" % - (input_name, op_name, extra_message)) + "The data type of '%s' in %s only support float16 in GPU now. %s" + % (input_name, op_name, extra_message) + ) if convert_dtype(input_dtype) in ['uint16'] and op_name not in [ - 'reshape', 'lookup_table', 'scale' + 'reshape', + 'lookup_table', + 'scale', ]: warnings.warn( "The data type of '%s' in %s only support bfloat16 in OneDNN now. %s" - % (input_name, op_name, extra_message)) + % (input_name, op_name, extra_message) + ) if convert_dtype(input_dtype) not in expected_dtype: raise TypeError( - "The data type of '%s' in %s must be %s, but received %s. %s" % - (input_name, op_name, expected_dtype, convert_dtype(input_dtype), - extra_message)) - - -def check_shape(shape, + "The data type of '%s' in %s must be %s, but received %s. %s" + % ( + input_name, op_name, - expected_shape_type=(list, tuple, Variable), - expected_element_type=(int, Variable), - expected_tensor_dtype=('int32', 'int64')): + expected_dtype, + convert_dtype(input_dtype), + extra_message, + ) + ) + + +def check_shape( + shape, + op_name, + expected_shape_type=(list, tuple, Variable), + expected_element_type=(int, Variable), + expected_tensor_dtype=('int32', 'int64'), +): # See NOTE [ Why skip dynamic graph check ] if _non_static_mode(): return @@ -160,16 +209,19 @@ def check_shape(shape, check_type(item, 'element of shape', expected_element_type, op_name) if expected_tensor_dtype is not None and isinstance(item, Variable): check_dtype( - item.dtype, 'element of shape', expected_tensor_dtype, + item.dtype, + 'element of shape', + expected_tensor_dtype, op_name, - 'If element of shape is Tensor, its data type should be {}'. - format(', '.join(expected_tensor_dtype))) + 'If element of shape is Tensor, its data type should be {}'.format( + ', '.join(expected_tensor_dtype) + ), + ) if expected_tensor_dtype is not None and isinstance(shape, Variable): check_dtype(shape.dtype, 'shape', expected_tensor_dtype, op_name) class DataToLoDTensorConverter(object): - def __init__(self, place, lod_level, shape, dtype): self.place = place self.lod_level = lod_level @@ -203,8 +255,10 @@ def _check_shape(self, shape): for s1, s2 in zip(self.shape, shape): if s1 != s2 and s1 >= 0 and s2 >= 0: raise ValueError( - "Shape not match. What is defined in data layer is {}, but receive {}" - .format(self.shape, shape)) + "Shape not match. What is defined in data layer is {}, but receive {}".format( + self.shape, shape + ) + ) def done(self): arr = np.array(self.data, dtype=self.dtype) @@ -214,8 +268,10 @@ def done(self): arr = arr.reshape(self.shape) except ValueError: raise ValueError( - "Reshape error. What is defined in data layer is {}, but receive {}" - .format(self.shape, arr.shape)) + "Reshape error. What is defined in data layer is {}, but receive {}".format( + self.shape, arr.shape + ) + ) t = core.LoDTensor() t.set(arr, self.place) if self.lod_level > 0: @@ -225,7 +281,6 @@ def done(self): class BatchedTensorProvider(object): - def __init__(self, feed_list, place, batch_size, generator, drop_last): self.place = place self.batch_size = batch_size @@ -236,10 +291,13 @@ def __init__(self, feed_list, place, batch_size, generator, drop_last): for var in feed_list: assert var.lod_level == 0, "lod_level must be 0" self.converters.append( - DataToLoDTensorConverter(place=self.place, - lod_level=0, - shape=var.shape, - dtype=var.dtype)) + DataToLoDTensorConverter( + place=self.place, + lod_level=0, + shape=var.shape, + dtype=var.dtype, + ) + ) def _done(self): return [c.done() for c in self.converters] @@ -380,18 +438,23 @@ def reader(limit=5): """ converter = [] - for lod_level, shape, dtype in zip(self.feed_lod_level, - self.feed_shapes, self.feed_dtypes): + for lod_level, shape, dtype in zip( + self.feed_lod_level, self.feed_shapes, self.feed_dtypes + ): converter.append( - DataToLoDTensorConverter(place=self.place, - lod_level=lod_level, - shape=shape, - dtype=dtype)) + DataToLoDTensorConverter( + place=self.place, + lod_level=lod_level, + shape=shape, + dtype=dtype, + ) + ) for each_sample in iterable: assert len(each_sample) == len(converter), ( - "The number of fields in data (%d) does not match " + - "len(feed_list) (%d)") % (len(each_sample), len(converter)) + "The number of fields in data (%d) does not match " + + "len(feed_list) (%d)" + ) % (len(each_sample), len(converter)) for each_converter, each_slot in zip(converter, each_sample): each_converter.feed(each_slot) ret_dict = {} @@ -465,10 +528,12 @@ def _reader(): ] if len(iterable) != len(places): - raise ValueError("feed_parallel takes multiple mini-batches. Each " - "mini-batch will be feed on each device. The " - "number of devices and number of mini-batches " - "must be same.") + raise ValueError( + "feed_parallel takes multiple mini-batches. Each " + "mini-batch will be feed on each device. The " + "number of devices and number of mini-batches " + "must be same." + ) place = self.place for p, batch in zip(places, iterable): @@ -484,11 +549,9 @@ def _get_number_of_places_(self, num_places): else: return _cpu_num() - def decorate_reader(self, - reader, - multi_devices, - num_places=None, - drop_last=True): + def decorate_reader( + self, reader, multi_devices, num_places=None, drop_last=True + ): """ Decorate the reader (generator) to fit multiple devices. The reader generate multiple mini-batches. Each mini-batch will be fed into a single device. @@ -566,6 +629,7 @@ def __reader_creator__(): raise ValueError( "The data batch which cannot fit for devices will be " "dropped is not implementation. Other strategies are " - "not implemented") + "not implemented" + ) return __reader_creator__ diff --git a/python/paddle/fluid/dataloader/__init__.py b/python/paddle/fluid/dataloader/__init__.py index 9136db3de80b3..c0b2052283b1c 100644 --- a/python/paddle/fluid/dataloader/__init__.py +++ b/python/paddle/fluid/dataloader/__init__.py @@ -24,7 +24,9 @@ from . import sampler from .sampler import * -__all__ = dataset.__all__ \ - + batch_sampler.__all__ \ - + dataloader_iter.__all__ \ - + sampler.__all__ +__all__ = ( + dataset.__all__ + + batch_sampler.__all__ + + dataloader_iter.__all__ + + sampler.__all__ +) diff --git a/python/paddle/fluid/dataloader/batch_sampler.py b/python/paddle/fluid/dataloader/batch_sampler.py index 7c87bd3d4cd61..624754ae286ad 100644 --- a/python/paddle/fluid/dataloader/batch_sampler.py +++ b/python/paddle/fluid/dataloader/batch_sampler.py @@ -96,36 +96,51 @@ def __len__(self): """ - def __init__(self, - dataset=None, - sampler=None, - shuffle=False, - batch_size=1, - drop_last=False): + def __init__( + self, + dataset=None, + sampler=None, + shuffle=False, + batch_size=1, + drop_last=False, + ): if dataset is None: - assert sampler is not None, \ - "either dataset or sampler should be set" - assert isinstance(sampler, Sampler), \ - "sampler should be a paddle.io.Sampler, but got {}".format(type(sampler)) + assert ( + sampler is not None + ), "either dataset or sampler should be set" + assert isinstance( + sampler, Sampler + ), "sampler should be a paddle.io.Sampler, but got {}".format( + type(sampler) + ) assert not shuffle, "shuffle should be False when sampler is set" self.sampler = sampler else: - assert not isinstance(dataset, IterableDataset), \ - "dataset should not be a paddle.io.IterableDataset" - assert sampler is None, \ - "should not set both dataset and sampler" - assert isinstance(shuffle, bool), \ - "shuffle should be a boolean value, but got {}".format(type(shuffle)) + assert not isinstance( + dataset, IterableDataset + ), "dataset should not be a paddle.io.IterableDataset" + assert sampler is None, "should not set both dataset and sampler" + assert isinstance( + shuffle, bool + ), "shuffle should be a boolean value, but got {}".format( + type(shuffle) + ) if shuffle: self.sampler = RandomSampler(dataset) else: self.sampler = SequenceSampler(dataset) - assert isinstance(batch_size, int) and batch_size > 0, \ - "batch_size should be a positive integer, but got {}".format(batch_size) + assert ( + isinstance(batch_size, int) and batch_size > 0 + ), "batch_size should be a positive integer, but got {}".format( + batch_size + ) self.batch_size = batch_size - assert isinstance(drop_last, bool), \ - "drop_last should be a boolean value, but got {}".format(type(drop_last)) + assert isinstance( + drop_last, bool + ), "drop_last should be a boolean value, but got {}".format( + type(drop_last) + ) self.drop_last = drop_last def __iter__(self): @@ -145,7 +160,6 @@ def __len__(self): class _InfiniteIterableSampler(object): - def __init__(self, dataset, batch_size=1): assert isinstance( dataset, IterableDataset @@ -214,36 +228,41 @@ def __len__(self): break """ - def __init__(self, - dataset, - batch_size, - num_replicas=None, - rank=None, - shuffle=False, - drop_last=False): + def __init__( + self, + dataset, + batch_size, + num_replicas=None, + rank=None, + shuffle=False, + drop_last=False, + ): self.dataset = dataset - assert isinstance(batch_size, int) and batch_size > 0, \ - "batch_size should be a positive integer" + assert ( + isinstance(batch_size, int) and batch_size > 0 + ), "batch_size should be a positive integer" self.batch_size = batch_size - assert isinstance(shuffle, bool), \ - "shuffle should be a boolean value" + assert isinstance(shuffle, bool), "shuffle should be a boolean value" self.shuffle = shuffle - assert isinstance(drop_last, bool), \ - "drop_last should be a boolean number" + assert isinstance( + drop_last, bool + ), "drop_last should be a boolean number" from paddle.fluid.dygraph.parallel import ParallelEnv if num_replicas is not None: - assert isinstance(num_replicas, int) and num_replicas > 0, \ - "num_replicas should be a positive integer" + assert ( + isinstance(num_replicas, int) and num_replicas > 0 + ), "num_replicas should be a positive integer" self.nranks = num_replicas else: self.nranks = ParallelEnv().nranks if rank is not None: - assert isinstance(rank, int) and rank >= 0, \ - "rank should be a non-negative integer" + assert ( + isinstance(rank, int) and rank >= 0 + ), "rank should be a non-negative integer" self.local_rank = rank else: self.local_rank = ParallelEnv().local_rank @@ -256,7 +275,7 @@ def __init__(self, def __iter__(self): num_samples = len(self.dataset) indices = np.arange(num_samples).tolist() - indices += indices[:(self.total_size - len(indices))] + indices += indices[: (self.total_size - len(indices))] assert len(indices) == self.total_size if self.shuffle: np.random.RandomState(self.epoch).shuffle(indices) @@ -269,16 +288,21 @@ def _get_indices_by_batch_size(indices): assert last_batch_size % self.nranks == 0 last_local_batch_size = last_batch_size // self.nranks - for i in range(self.local_rank * self.batch_size, - len(indices) - last_batch_size, - self.batch_size * self.nranks): - subsampled_indices.extend(indices[i:i + self.batch_size]) + for i in range( + self.local_rank * self.batch_size, + len(indices) - last_batch_size, + self.batch_size * self.nranks, + ): + subsampled_indices.extend(indices[i : i + self.batch_size]) - indices = indices[len(indices) - last_batch_size:] + indices = indices[len(indices) - last_batch_size :] subsampled_indices.extend( - indices[self.local_rank * - last_local_batch_size:(self.local_rank + 1) * - last_local_batch_size]) + indices[ + self.local_rank + * last_local_batch_size : (self.local_rank + 1) + * last_local_batch_size + ] + ) return subsampled_indices if self.nranks > 1: diff --git a/python/paddle/fluid/dataloader/collate.py b/python/paddle/fluid/dataloader/collate.py index 3c46b54156d78..50b86ca41e53b 100644 --- a/python/paddle/fluid/dataloader/collate.py +++ b/python/paddle/fluid/dataloader/collate.py @@ -66,18 +66,20 @@ def default_collate_fn(batch): return batch elif isinstance(sample, Mapping): return { - key: default_collate_fn([d[key] for d in batch]) - for key in sample + key: default_collate_fn([d[key] for d in batch]) for key in sample } elif isinstance(sample, Sequence): sample_fields_num = len(sample) if not all(len(sample) == sample_fields_num for sample in iter(batch)): raise RuntimeError( - "fileds number not same among samples in a batch") + "fileds number not same among samples in a batch" + ) return [default_collate_fn(fields) for fields in zip(*batch)] - raise TypeError("batch data con only contains: tensor, numpy.ndarray, " - "dict, list, number, but got {}".format(type(sample))) + raise TypeError( + "batch data con only contains: tensor, numpy.ndarray, " + "dict, list, number, but got {}".format(type(sample)) + ) def default_convert_fn(batch): diff --git a/python/paddle/fluid/dataloader/dataloader_iter.py b/python/paddle/fluid/dataloader/dataloader_iter.py index 71e4a9f83adb4..83d95c479250e 100644 --- a/python/paddle/fluid/dataloader/dataloader_iter.py +++ b/python/paddle/fluid/dataloader/dataloader_iter.py @@ -24,7 +24,11 @@ import numpy as np import multiprocessing from collections import namedtuple -from paddle.fluid.framework import _set_expected_place, _current_expected_place, set_flags +from paddle.fluid.framework import ( + _set_expected_place, + _current_expected_place, + set_flags, +) # NOTE: queue has a different name in python2 and python3 import queue @@ -34,13 +38,23 @@ from paddle.profiler.utils import in_profiler_mode from .. import core, layers from ..framework import _non_static_mode, in_dygraph_mode, _in_legacy_dygraph -from ..multiprocess_utils import _set_SIGCHLD_handler, MP_STATUS_CHECK_INTERVAL, CleanupFuncRegistrar +from ..multiprocess_utils import ( + _set_SIGCHLD_handler, + MP_STATUS_CHECK_INTERVAL, + CleanupFuncRegistrar, +) from .fetcher import _IterableDatasetFetcher, _MapDatasetFetcher from .batch_sampler import _InfiniteIterableSampler from .collate import default_collate_fn, default_convert_fn -from .worker import ParentWatchDog, get_worker_info, _worker_loop, \ - _DatasetKind, _IterableDatasetStopIteration, _WorkerException, \ - _ResumeIteration +from .worker import ( + ParentWatchDog, + get_worker_info, + _worker_loop, + _DatasetKind, + _IterableDatasetStopIteration, + _WorkerException, + _ResumeIteration, +) from .flat import _flatten_batch, _restore_batch from paddle.profiler.timer import benchmark @@ -98,7 +112,9 @@ def __init__(self, loader): self._use_buffer_reader = loader.use_buffer_reader self._prefetch_factor = loader.prefetch_factor self._use_shared_memory = loader.use_shared_memory - self._timeout = loader.timeout if loader.timeout > 0 else MP_STATUS_CHECK_INTERVAL + self._timeout = ( + loader.timeout if loader.timeout > 0 else MP_STATUS_CHECK_INTERVAL + ) self._worker_init_fn = loader.worker_init_fn self._dataset_kind = loader.dataset_kind self._pin_memory = loader.pin_memory @@ -155,8 +171,12 @@ def __init__(self, loader): super(_DataLoaderIterSingleProcess, self).__init__(loader) self._dataset_fetcher = _DatasetKind.create_fetcher( - self._dataset_kind, self._dataset, self._auto_collate_batch, - self._collate_fn, self._drop_last) + self._dataset_kind, + self._dataset, + self._auto_collate_batch, + self._collate_fn, + self._drop_last, + ) # NOTE: _structrue_infos used to record the data structure of # batch to restore batch structure after reading Tensor @@ -170,7 +190,8 @@ def __init__(self, loader): # iteration, set blocking_queue can cache "self._prefetch_factor" iteration datas # at most here self._blocking_queue_capacity = self._prefetch_factor * len( - self._places) + self._places + ) self._init_thread() self._shutdown = False @@ -187,20 +208,30 @@ def _init_thread(self): ] # if only 1 place, do not need to keep order self._blocking_queue = core.init_lod_tensor_blocking_queue( - core.Variable(), self._blocking_queue_capacity, - len(self._places) > 1) + core.Variable(), + self._blocking_queue_capacity, + len(self._places) > 1, + ) self._reader = core.create_py_reader( - self._blocking_queue, self._var_names, self._shapes, self._dtypes, - self._need_check_feed, self._places, self._use_buffer_reader, True, - self._pin_memory) - - self._thread = threading.Thread(target=self._thread_loop, - args=(_current_expected_place(), )) + self._blocking_queue, + self._var_names, + self._shapes, + self._dtypes, + self._need_check_feed, + self._places, + self._use_buffer_reader, + True, + self._pin_memory, + ) + + self._thread = threading.Thread( + target=self._thread_loop, args=(_current_expected_place(),) + ) self._thread.daemon = True self._thread.start() def _thread_loop(self, legacy_expected_place): - #NOTE(zhiqiu): Set the expected place for new thread as the same as father thread, + # NOTE(zhiqiu): Set the expected place for new thread as the same as father thread, # and it will call platform::SetDeviceId() in c++ internally. # If we do not set cudaDeviceId in new thread, the default cudaDeviceId will be 0, # Which may cost hundreds of MB of GPU memory on CUDAPlace(0) if calling some cuda @@ -215,19 +246,22 @@ def _thread_loop(self, legacy_expected_place): # read data from dataset in mini-batch # with paddle.fluid.dygraph.guard(place=paddle.CPUPlace()): # read data from dataset in mini-batch - batch = self._dataset_fetcher.fetch(indices, - self._thread_done_event) + batch = self._dataset_fetcher.fetch( + indices, self._thread_done_event + ) except StopIteration: self._exit_thread_expectedly() return - if batch is None or self._thread_done_event.is_set(): break + if batch is None or self._thread_done_event.is_set(): + break # flat batch and record structure infos batch, structure = _flatten_batch(batch) self._structure_infos.append(structure) - if self._thread_done_event.is_set(): break + if self._thread_done_event.is_set(): + break try: # pack as LoDTensorArray @@ -242,7 +276,8 @@ def _thread_loop(self, legacy_expected_place): array.append(slot) - if self._thread_done_event.is_set(): break + if self._thread_done_event.is_set(): + break try: self._blocking_queue.push(array) @@ -259,14 +294,16 @@ def __next__(self): if in_profiler_mode(): trace_event = profiler.RecordEvent( name="_DataLoaderIterSingleProcess", - event_type=profiler.TracerEventType.Dataloader) + event_type=profiler.TracerEventType.Dataloader, + ) trace_event.begin() try: benchmark().check_if_need_record(self) benchmark().before_reader() if in_dygraph_mode(): data = core.eager.read_next_tensor_list( - self._reader.read_next_list()[0]) + self._reader.read_next_list()[0] + ) data = _restore_batch(data, self._structure_infos.pop(0)) else: if _in_legacy_dygraph(): @@ -281,8 +318,9 @@ def __next__(self): self._structure_infos.pop(0) for _ in range(len(self._places)) ] - data = [_restore_batch(d, s) \ - for d, s in zip(data, structs)] + data = [ + _restore_batch(d, s) for d, s in zip(data, structs) + ] # static graph organized data on multi-device with list, if # place number is 1, there is only 1 device, extra the data # from list for devices to be compatible with dygraph mode @@ -341,15 +379,17 @@ def __del__(self): class _DataLoaderIterMultiProcess(_DataLoaderIterBase): - def __init__(self, loader): super(_DataLoaderIterMultiProcess, self).__init__(loader) self._persistent_workers = loader._persistent_workers self._resume_worker_cnt = 0 - assert self._num_workers > 0, "Multi-process DataLoader " \ - "invalid num_workers({})".format(self._num_workers) + assert ( + self._num_workers > 0 + ), "Multi-process DataLoader " "invalid num_workers({})".format( + self._num_workers + ) # subprocess wrokers' result queue self._data_queue = None @@ -370,7 +410,8 @@ def __init__(self, loader): # output data for at least "_prefetch_factor" iterations(Note that len(_places) # batches will be composed as an iteration output) self._outstanding_capacity = self._prefetch_factor * max( - self._num_workers, len(self._places)) + self._num_workers, len(self._places) + ) # see _try_put_indices self._thread_lock = threading.Lock() @@ -405,12 +446,22 @@ def _init_workers(self): self._indices_queues.append(indices_queue) worker = multiprocessing.Process( target=_worker_loop, - args=(self._dataset, self._dataset_kind, indices_queue, - self._data_queue, self._workers_done_event, - self._auto_collate_batch, self._collate_fn, - self._drop_last, self._worker_init_fn, i, - self._num_workers, self._use_shared_memory, - self._base_seed)) + args=( + self._dataset, + self._dataset_kind, + indices_queue, + self._data_queue, + self._workers_done_event, + self._auto_collate_batch, + self._collate_fn, + self._drop_last, + self._worker_init_fn, + i, + self._num_workers, + self._use_shared_memory, + self._base_seed, + ), + ) worker.daemon = True worker.start() self._workers.append(worker) @@ -438,17 +489,25 @@ def _init_thread(self): ] # if only 1 place, do not need to keep order self._blocking_queue = core.init_lod_tensor_blocking_queue( - core.Variable(), self._outstanding_capacity, - len(self._places) > 1) + core.Variable(), self._outstanding_capacity, len(self._places) > 1 + ) self._reader = core.create_py_reader( - self._blocking_queue, self._var_names, self._shapes, self._dtypes, - self._need_check_feed, self._places, self._use_buffer_reader, True, - self._pin_memory) + self._blocking_queue, + self._var_names, + self._shapes, + self._dtypes, + self._need_check_feed, + self._places, + self._use_buffer_reader, + True, + self._pin_memory, + ) self._thread_done_event = threading.Event() # thread event is only need in multi-processing mode - self._thread = threading.Thread(target=self._thread_loop, - args=(_current_expected_place(), )) + self._thread = threading.Thread( + target=self._thread_loop, args=(_current_expected_place(),) + ) self._thread.daemon = True self._thread.start() @@ -471,7 +530,8 @@ def _reset(self): while self._blocking_queue.size() >= len(self._places): if in_dygraph_mode(): data = core.eager.read_next_tensor_list( - self._reader.read_next_list()[0]) + self._reader.read_next_list()[0] + ) else: if _in_legacy_dygraph(): self._reader.read_next_var_list() @@ -497,8 +557,9 @@ def _reset(self): self._try_put_indices() def _shutdown_worker(self, worker_id, shutdown=False): - if self._worker_status[worker_id] or (self._persistent_workers - and shutdown): + if self._worker_status[worker_id] or ( + self._persistent_workers and shutdown + ): self._indices_queues[worker_id].put(None) self._worker_status[worker_id] = False @@ -526,7 +587,7 @@ def _try_shutdown_all(self, timeout=None): self._shutdown = True def _thread_loop(self, legacy_expected_place): - #NOTE(zhiqiu): Set the expected place for new thread as the same as father thread, + # NOTE(zhiqiu): Set the expected place for new thread as the same as father thread, # and it will call platform::SetDeviceId() in c++ internally. # If we do not set cudaDeviceId in new thread, the default cudaDeviceId will be 0, # Which may cost hundreds of MB of GPU memory on CUDAPlace(0) if calling some cuda @@ -555,8 +616,8 @@ def _thread_loop(self, legacy_expected_place): # serializable, cannot be create in workers for slot in batch: if isinstance( - slot, - (paddle.Tensor, core.eager.Tensor)): + slot, (paddle.Tensor, core.eager.Tensor) + ): slot = slot.value().get_tensor() elif not isinstance(slot, core.LoDTensor): tmp = core.LoDTensor() @@ -607,8 +668,10 @@ def _get_data(self): if self._batches_outstanding < len(self._places): return None - if self._rcvd_idx in self._task_infos and \ - len(self._task_infos[self._rcvd_idx]) == 3: + if ( + self._rcvd_idx in self._task_infos + and len(self._task_infos[self._rcvd_idx]) == 3 + ): info = self._task_infos.pop(self._rcvd_idx) self._structure_infos.append(info[2]) return info[1] @@ -637,8 +700,10 @@ def _get_data(self): if len(failed_workers) > 0: self._exit_thread_unexpectedly() pids = ', '.join(str(w.pid) for w in failed_workers) - raise RuntimeError("DataLoader {} workers exit unexpectedly, " \ - "pids: {}".format(len(failed_workers), pids)) + raise RuntimeError( + "DataLoader {} workers exit unexpectedly, " + "pids: {}".format(len(failed_workers), pids) + ) # get(timeout) will call _poll(timeout) and may raise IOError if isinstance(e, queue.Empty) or isinstance(e, IOError): @@ -646,12 +711,15 @@ def _get_data(self): continue self._exit_thread_unexpectedly() - logging.error("DataLoader reader thread failed({}) to read data from " \ - "workers' result queue.".format(e)) + logging.error( + "DataLoader reader thread failed({}) to read data from " + "workers' result queue.".format(e) + ) six.reraise(*sys.exc_info()) else: if self._dataset_kind == _DatasetKind.ITER and isinstance( - data, _IterableDatasetStopIteration): + data, _IterableDatasetStopIteration + ): # if a worker get StopIteraion, we shutdown this worker, # note that this batch indices to trigger StopIteration # is discard, outstanding batch number should be decrease @@ -667,8 +735,11 @@ def _get_data(self): idx, batch, structure = data - if isinstance(idx, _ResumeIteration) and batch is None \ - and structure is None: + if ( + isinstance(idx, _ResumeIteration) + and batch is None + and structure is None + ): return idx if isinstance(batch, _WorkerException): @@ -684,8 +755,9 @@ def _get_data(self): continue def _try_put_indices(self): - assert self._batches_outstanding <= self._outstanding_capacity, \ - "too many indices have been put to queue" + assert ( + self._batches_outstanding <= self._outstanding_capacity + ), "too many indices have been put to queue" # In multi-process mode for IterableDataset, _try_put_indices will # be called both in main process(for our implement has blocking queue, # and blocking queue read is in main process) and thread, which may @@ -709,7 +781,7 @@ def _try_put_indices(self): return self._indices_queues[worker_idx].put((self._send_idx, indices)) - self._task_infos[self._send_idx] = (worker_idx, ) + self._task_infos[self._send_idx] = (worker_idx,) self._batches_outstanding += 1 self._send_idx += 1 @@ -723,7 +795,8 @@ def __next__(self): if in_profiler_mode(): trace_event = profiler.RecordEvent( name="_DataLoaderIterMultiProcess", - event_type=profiler.TracerEventType.Dataloader) + event_type=profiler.TracerEventType.Dataloader, + ) trace_event.begin() try: benchmark().check_if_need_record(self) @@ -744,7 +817,8 @@ def __next__(self): if in_dygraph_mode(): data = core.eager.read_next_tensor_list( - self._reader.read_next_list()[0]) + self._reader.read_next_list()[0] + ) data = _restore_batch(data, self._structure_infos.pop(0)) else: if _in_legacy_dygraph(): @@ -759,8 +833,9 @@ def __next__(self): self._structure_infos.pop(0) for _ in range(len(self._places)) ] - data = [_restore_batch(d, s) \ - for d, s in zip(data, structs)] + data = [ + _restore_batch(d, s) for d, s in zip(data, structs) + ] # static graph organized data on multi-device with list, if # place number is 1, there is only 1 device, extra the data # from list for devices to be compatible with dygraph mode diff --git a/python/paddle/fluid/dataloader/dataset.py b/python/paddle/fluid/dataloader/dataset.py index eaab078d7e871..5fe52196cd88f 100755 --- a/python/paddle/fluid/dataloader/dataset.py +++ b/python/paddle/fluid/dataloader/dataset.py @@ -16,8 +16,13 @@ from .. import framework __all__ = [ - "Dataset", "IterableDataset", "TensorDataset", "ComposeDataset", - "ChainDataset", "random_split", "Subset" + "Dataset", + "IterableDataset", + "TensorDataset", + "ComposeDataset", + "ChainDataset", + "random_split", + "Subset", ] @@ -67,12 +72,16 @@ def __init__(self): pass def __getitem__(self, idx): - raise NotImplementedError("'{}' not implement in class "\ - "{}".format('__getitem__', self.__class__.__name__)) + raise NotImplementedError( + "'{}' not implement in class " + "{}".format('__getitem__', self.__class__.__name__) + ) def __len__(self): - raise NotImplementedError("'{}' not implement in class "\ - "{}".format('__len__', self.__class__.__name__)) + raise NotImplementedError( + "'{}' not implement in class " + "{}".format('__len__', self.__class__.__name__) + ) class IterableDataset(Dataset): @@ -210,16 +219,22 @@ def __init__(self): pass def __iter__(self): - raise NotImplementedError("'{}' not implement in class "\ - "{}".format('__iter__', self.__class__.__name__)) + raise NotImplementedError( + "'{}' not implement in class " + "{}".format('__iter__', self.__class__.__name__) + ) def __getitem__(self, idx): - raise RuntimeError("'{}' should not be called for IterableDataset" \ - "{}".format('__getitem__', self.__class__.__name__)) + raise RuntimeError( + "'{}' should not be called for IterableDataset" + "{}".format('__getitem__', self.__class__.__name__) + ) def __len__(self): - raise RuntimeError("'{}' should not be called for IterableDataset" \ - "{}".format('__len__', self.__class__.__name__)) + raise RuntimeError( + "'{}' should not be called for IterableDataset" + "{}".format('__len__', self.__class__.__name__) + ) class TensorDataset(Dataset): @@ -261,9 +276,11 @@ class TensorDataset(Dataset): def __init__(self, tensors): if not framework._non_static_mode(): raise RuntimeError( - "TensorDataset con only be used in imperative mode") - assert all([tensor.shape[0] == tensors[0].shape[0] for tensor in tensors]), \ - "tensors not have same shape of the 1st dimension" + "TensorDataset con only be used in imperative mode" + ) + assert all( + [tensor.shape[0] == tensors[0].shape[0] for tensor in tensors] + ), "tensors not have same shape of the 1st dimension" self.tensors = tensors def __getitem__(self, index): @@ -330,13 +347,16 @@ def __init__(self, datasets): self.datasets = list(datasets) assert len(self.datasets) > 0, "input datasets shoule not be empty" for i, dataset in enumerate(self.datasets): - assert isinstance(dataset, Dataset), \ - "each input dataset should be paddle.io.Dataset" - assert not isinstance(dataset, IterableDataset), \ - "paddle.io.IterableDataset not supported" + assert isinstance( + dataset, Dataset + ), "each input dataset should be paddle.io.Dataset" + assert not isinstance( + dataset, IterableDataset + ), "paddle.io.IterableDataset not supported" if i > 0: - assert len(dataset) == len(self.datasets[i-1]), \ - "lengths of datasets should be same" + assert len(dataset) == len( + self.datasets[i - 1] + ), "lengths of datasets should be same" def __len__(self): return len(self.datasets[0]) @@ -391,8 +411,9 @@ def __init__(self, datasets): self.datasets = list(datasets) assert len(self.datasets) > 0, "input datasets shoule not be empty" for i, dataset in enumerate(self.datasets): - assert isinstance(dataset, IterableDataset), \ - "ChainDataset only support paddle.io.IterableDataset" + assert isinstance( + dataset, IterableDataset + ), "ChainDataset only support paddle.io.IterableDataset" def __iter__(self): for dataset in self.datasets: @@ -492,7 +513,7 @@ def random_split(dataset, lengths, generator=None): # For example var.item() and var.tolist() indices = paddle.randperm(sum(lengths)).numpy().tolist() return [ - Subset(dataset, indices[offset - length:offset]) + Subset(dataset, indices[offset - length : offset]) for offset, length in zip(_accumulate(lengths), lengths) ] diff --git a/python/paddle/fluid/dataloader/fetcher.py b/python/paddle/fluid/dataloader/fetcher.py index 387032cdfbbd3..b4a2ff6b923e4 100644 --- a/python/paddle/fluid/dataloader/fetcher.py +++ b/python/paddle/fluid/dataloader/fetcher.py @@ -20,7 +20,6 @@ class _DatasetFetcher(object): - def __init__(self, dataset, auto_collate_batch, collate_fn, drop_last): self.dataset = dataset self.auto_collate_batch = auto_collate_batch @@ -37,48 +36,55 @@ def __init__(self, dataset, auto_collate_batch, collate_fn, drop_last): # done_event argument to check DataLoader exit status between # ecah sample processing in the batch def fetch(self, batch_indices, done_event=None): - raise NotImplementedError("'fetch' not implement for class {}".format( - self.__class__.__name__)) + raise NotImplementedError( + "'fetch' not implement for class {}".format(self.__class__.__name__) + ) def _log_warning(self): # only log warning on GPU 0 when distributed launch from ...distributed import get_world_size, get_rank + if get_world_size() >= 2 and get_rank() != 0: return - warn_str = "Detect dataset only contains single fileds, return format " \ - "changed since Paddle 2.1. In Paddle <= 2.0, DataLoader add " \ - "a list surround output data(e.g. return [data]), and in " \ - "Paddle >= 2.1, DataLoader return the single filed directly " \ - "(e.g. return data). For example, in following code: \n\n" - warn_str += \ - "import numpy as np\n" \ - "from paddle.io import DataLoader, Dataset\n\n" \ - "class RandomDataset(Dataset):\n" \ - " def __getitem__(self, idx):\n" \ - " data = np.random.random((2, 3)).astype('float32')\n\n" \ - " return data\n\n" \ - " def __len__(self):\n" \ - " return 10\n\n" \ - "dataset = RandomDataset()\n" \ - "loader = DataLoader(dataset, batch_size=1)\n" \ - "data = next(loader())\n\n" - - warn_str += "In Paddle <= 2.0, data is in format '[Tensor(shape=(1, 2, 3), " \ - "dtype=float32)]', and in Paddle >= 2.1, data is in format" \ - " 'Tensor(shape=(1, 2, 3), dtype=float32)'\n" - - logger = get_logger("DataLoader", - logging.INFO, - fmt='%(levelname)s: %(message)s') + warn_str = ( + "Detect dataset only contains single fileds, return format " + "changed since Paddle 2.1. In Paddle <= 2.0, DataLoader add " + "a list surround output data(e.g. return [data]), and in " + "Paddle >= 2.1, DataLoader return the single filed directly " + "(e.g. return data). For example, in following code: \n\n" + ) + warn_str += ( + "import numpy as np\n" + "from paddle.io import DataLoader, Dataset\n\n" + "class RandomDataset(Dataset):\n" + " def __getitem__(self, idx):\n" + " data = np.random.random((2, 3)).astype('float32')\n\n" + " return data\n\n" + " def __len__(self):\n" + " return 10\n\n" + "dataset = RandomDataset()\n" + "loader = DataLoader(dataset, batch_size=1)\n" + "data = next(loader())\n\n" + ) + + warn_str += ( + "In Paddle <= 2.0, data is in format '[Tensor(shape=(1, 2, 3), " + "dtype=float32)]', and in Paddle >= 2.1, data is in format" + " 'Tensor(shape=(1, 2, 3), dtype=float32)'\n" + ) + + logger = get_logger( + "DataLoader", logging.INFO, fmt='%(levelname)s: %(message)s' + ) logger.warning(warn_str) class _IterableDatasetFetcher(_DatasetFetcher): - def __init__(self, dataset, auto_collate_batch, collate_fn, drop_last): - super(_IterableDatasetFetcher, - self).__init__(dataset, auto_collate_batch, collate_fn, drop_last) + super(_IterableDatasetFetcher, self).__init__( + dataset, auto_collate_batch, collate_fn, drop_last + ) self.dataset_iter = iter(dataset) def fetch(self, batch_indices, done_event=None): @@ -94,13 +100,13 @@ def fetch(self, batch_indices, done_event=None): else: return None - if len(data) == 0 or (self.drop_last - and len(data) < len(batch_indices)): + if len(data) == 0 or ( + self.drop_last and len(data) < len(batch_indices) + ): raise StopIteration global _WARNING_TO_LOG - if not isinstance(data[0], (Sequence, Mapping)) \ - and _WARNING_TO_LOG: + if not isinstance(data[0], (Sequence, Mapping)) and _WARNING_TO_LOG: self._log_warning() _WARNING_TO_LOG = False else: @@ -112,10 +118,10 @@ def fetch(self, batch_indices, done_event=None): class _MapDatasetFetcher(_DatasetFetcher): - def __init__(self, dataset, auto_collate_batch, collate_fn, drop_last): - super(_MapDatasetFetcher, self).__init__(dataset, auto_collate_batch, - collate_fn, drop_last) + super(_MapDatasetFetcher, self).__init__( + dataset, auto_collate_batch, collate_fn, drop_last + ) def fetch(self, batch_indices, done_event=None): if self.auto_collate_batch: @@ -127,8 +133,7 @@ def fetch(self, batch_indices, done_event=None): return None global _WARNING_TO_LOG - if not isinstance(data[0], (Sequence, Mapping)) \ - and _WARNING_TO_LOG: + if not isinstance(data[0], (Sequence, Mapping)) and _WARNING_TO_LOG: self._log_warning() _WARNING_TO_LOG = False else: diff --git a/python/paddle/fluid/dataloader/flat.py b/python/paddle/fluid/dataloader/flat.py index 5baf4cc853e27..6f3c6edf0b0cc 100644 --- a/python/paddle/fluid/dataloader/flat.py +++ b/python/paddle/fluid/dataloader/flat.py @@ -36,39 +36,47 @@ def _flatten_batch(batch): def _flatten(batch, flat_batch, structure, field_idx): if isinstance(batch, Sequence): for field in batch: - if isinstance(field, (np.ndarray, paddle.Tensor, - paddle.fluid.core.eager.Tensor)): + if isinstance( + field, + (np.ndarray, paddle.Tensor, paddle.fluid.core.eager.Tensor), + ): structure.append('{}{}'.format(FIELD_PREFIX, field_idx)) flat_batch.append(field) field_idx += 1 elif isinstance(field, (str, bytes, numbers.Number)): structure.append(field) elif isinstance(field, Sequence): - field_struct, field_idx = _flatten(field, flat_batch, [], - field_idx) + field_struct, field_idx = _flatten( + field, flat_batch, [], field_idx + ) structure.append(field_struct) elif isinstance(field, Mapping): - field_struct, field_idx = _flatten(field, flat_batch, {}, - field_idx) + field_struct, field_idx = _flatten( + field, flat_batch, {}, field_idx + ) structure.append(field_struct) else: structure.append(field) elif isinstance(batch, Mapping): for k, field in batch.items(): - if isinstance(field, (np.ndarray, paddle.Tensor, - paddle.fluid.core.eager.Tensor)): + if isinstance( + field, + (np.ndarray, paddle.Tensor, paddle.fluid.core.eager.Tensor), + ): structure[k] = '{}{}'.format(FIELD_PREFIX, field_idx) flat_batch.append(field) field_idx += 1 elif isinstance(field, (str, bytes, numbers.Number)): structure[k] = field elif isinstance(field, Sequence): - field_struct, field_idx = _flatten(field, flat_batch, [], - field_idx) + field_struct, field_idx = _flatten( + field, flat_batch, [], field_idx + ) structure[k] = field_struct elif isinstance(field, Mapping): - field_struct, field_idx = _flatten(field, flat_batch, {}, - field_idx) + field_struct, field_idx = _flatten( + field, flat_batch, {}, field_idx + ) structure[k] = field_struct else: structure[k] = field @@ -100,8 +108,9 @@ def _restore(structure, field_idx): if isinstance(field, str) and field.startswith(FIELD_PREFIX): cur_field_idx = int(field.replace(FIELD_PREFIX, '')) field_idx = max(field_idx, cur_field_idx) - assert flat_batch[cur_field_idx] is not None, \ - "flat_batch[{}] parsed repeatly" + assert ( + flat_batch[cur_field_idx] is not None + ), "flat_batch[{}] parsed repeatly" structure[i] = flat_batch[cur_field_idx] flat_batch[cur_field_idx] = None elif isinstance(field, (str, bytes, numbers.Number)): @@ -113,8 +122,9 @@ def _restore(structure, field_idx): if isinstance(field, str) and field.startswith(FIELD_PREFIX): cur_field_idx = int(field.replace(FIELD_PREFIX, '')) field_idx = max(field_idx, cur_field_idx) - assert flat_batch[cur_field_idx] is not None, \ - "flat_batch[{}] parsed repeatly" + assert ( + flat_batch[cur_field_idx] is not None + ), "flat_batch[{}] parsed repeatly" structure[k] = flat_batch[cur_field_idx] flat_batch[cur_field_idx] = None elif isinstance(field, (str, bytes, numbers.Number)): @@ -126,8 +136,7 @@ def _restore(structure, field_idx): return field_idx - assert isinstance(flat_batch, Sequence), \ - "flat_batch is not a list or tuple" + assert isinstance(flat_batch, Sequence), "flat_batch is not a list or tuple" # no np.array in dataset, no output tensor from blocking queue # simply return structure @@ -136,8 +145,9 @@ def _restore(structure, field_idx): # sample only contains single fields if isinstance(structure, (str, bytes)): - assert structure == '{}{}'.format(FIELD_PREFIX, 0), \ - "invalid structure: {}".format(structure) + assert structure == '{}{}'.format( + FIELD_PREFIX, 0 + ), "invalid structure: {}".format(structure) return flat_batch[0] field_idx = _restore(structure, 0) assert field_idx + 1 == len(flat_batch), "Tensor parse incomplete" diff --git a/python/paddle/fluid/dataloader/sampler.py b/python/paddle/fluid/dataloader/sampler.py index 77b8ff09ba06f..3626ed63e521c 100644 --- a/python/paddle/fluid/dataloader/sampler.py +++ b/python/paddle/fluid/dataloader/sampler.py @@ -16,7 +16,10 @@ from .. import core __all__ = [ - "Sampler", "SequenceSampler", "RandomSampler", "WeightedRandomSampler" + "Sampler", + "SequenceSampler", + "RandomSampler", + "WeightedRandomSampler", ] @@ -185,19 +188,19 @@ def __len__(self): see `paddle.io.Sampler` """ - def __init__(self, - data_source, - replacement=False, - num_samples=None, - generator=None): + def __init__( + self, data_source, replacement=False, num_samples=None, generator=None + ): self.data_source = data_source self.replacement = replacement self._num_samples = num_samples self.generator = generator if not isinstance(self.replacement, bool): - raise TypeError("expect boolean value for replacement, but got " - "replacement={}".format(self.replacement)) + raise TypeError( + "expect boolean value for replacement, but got " + "replacement={}".format(self.replacement) + ) if self._num_samples is not None and not replacement: raise ValueError( @@ -205,8 +208,10 @@ def __init__(self, ) if not isinstance(self.num_samples, int) or self.num_samples <= 0: - raise ValueError("num_samples should be a positive integer, " - "but got num_samples={}".format(self.num_samples)) + raise ValueError( + "num_samples should be a positive integer, " + "but got num_samples={}".format(self.num_samples) + ) @property def num_samples(self): @@ -225,13 +230,14 @@ def __iter__(self): yield index else: if self.replacement: - for index in np.random.choice(np.arange(n), - self.num_samples, - replace=True).tolist(): + for index in np.random.choice( + np.arange(n), self.num_samples, replace=True + ).tolist(): yield index else: - for index in np.random.choice(np.arange(n), n, - replace=False).tolist(): + for index in np.random.choice( + np.arange(n), n, replace=False + ).tolist(): yield index def __len__(self): @@ -243,31 +249,29 @@ def _weighted_sample(weights, num_samples, replacement=True): weights = weights.numpy() if isinstance(weights, (list, tuple)): weights = np.array(weights) - assert isinstance(weights, np.ndarray), \ - "weights should be paddle.Tensor, numpy.ndarray, list or tuple" - assert len(weights.shape) <= 2, \ - "weights should be a 1-D or 2-D array" + assert isinstance( + weights, np.ndarray + ), "weights should be paddle.Tensor, numpy.ndarray, list or tuple" + assert len(weights.shape) <= 2, "weights should be a 1-D or 2-D array" weights = weights.reshape((-1, weights.shape[-1])) - assert np.all(weights >= 0.), \ - "weights should be positive value" - assert not np.any(weights == np.inf), \ - "weights shoule not be INF" - assert not np.any(weights == np.nan), \ - "weights shoule not be NaN" - - non_zeros = np.sum(weights > 0., axis=1) - assert np.all(non_zeros > 0), \ - "weights should have positive values" + assert np.all(weights >= 0.0), "weights should be positive value" + assert not np.any(weights == np.inf), "weights shoule not be INF" + assert not np.any(weights == np.nan), "weights shoule not be NaN" + + non_zeros = np.sum(weights > 0.0, axis=1) + assert np.all(non_zeros > 0), "weights should have positive values" if not replacement: - assert np.all(non_zeros >= num_samples), \ - "weights positive value number should not " \ + assert np.all(non_zeros >= num_samples), ( + "weights positive value number should not " "less than num_samples when replacement=False" + ) weights = weights / weights.sum(axis=1) rets = [] for i in range(weights.shape[0]): - ret = np.random.choice(weights.shape[1], num_samples, replacement, - weights[i]) + ret = np.random.choice( + weights.shape[1], num_samples, replacement, weights[i] + ) rets.append(ret) return np.array(rets) @@ -311,8 +315,9 @@ def __init__(self, weights, num_samples, replacement=True): self.replacement = replacement def __iter__(self): - idxs = _weighted_sample(self.weights, self.num_samples, - self.replacement) + idxs = _weighted_sample( + self.weights, self.num_samples, self.replacement + ) return iter(idxs.reshape((-1)).tolist()) def __len__(self): diff --git a/python/paddle/fluid/dataloader/worker.py b/python/paddle/fluid/dataloader/worker.py index 06ea7ef9d72a3..f0aa32e774522 100644 --- a/python/paddle/fluid/dataloader/worker.py +++ b/python/paddle/fluid/dataloader/worker.py @@ -21,7 +21,11 @@ from collections import namedtuple from .. import core from .fetcher import _IterableDatasetFetcher, _MapDatasetFetcher -from ..multiprocess_utils import _cleanup_mmap, CleanupFuncRegistrar, MP_STATUS_CHECK_INTERVAL +from ..multiprocess_utils import ( + _cleanup_mmap, + CleanupFuncRegistrar, + MP_STATUS_CHECK_INTERVAL, +) from ..framework import _non_static_mode, _in_eager_without_dygraph_check from .flat import _flatten_batch @@ -32,7 +36,6 @@ class _IterableDatasetStopIteration(object): - def __init__(self, worker_id): self.worker_id = worker_id @@ -46,20 +49,22 @@ class _DatasetKind(object): ITER = 1 @staticmethod - def create_fetcher(kind, dataset, auto_collate_batch, collate_fn, - drop_last): + def create_fetcher( + kind, dataset, auto_collate_batch, collate_fn, drop_last + ): if kind == _DatasetKind.MAP: - return _MapDatasetFetcher(dataset, auto_collate_batch, collate_fn, - drop_last) + return _MapDatasetFetcher( + dataset, auto_collate_batch, collate_fn, drop_last + ) elif kind == _DatasetKind.ITER: - return _IterableDatasetFetcher(dataset, auto_collate_batch, - collate_fn, drop_last) + return _IterableDatasetFetcher( + dataset, auto_collate_batch, collate_fn, drop_last + ) else: raise NotImplementedError("unknown Dataset kind {}".format(kind)) class ParentWatchDog(object): - def __init__(self): self._parent_pid = os.getppid() self._parent_alive = True @@ -151,13 +156,15 @@ def __init__(self, **kwargs): def __setattr__(self, key, val): if self.__initialized: - raise RuntimeError("Cannot assign attributes to {} objects".format( - self.__class__.__name__)) + raise RuntimeError( + "Cannot assign attributes to {} objects".format( + self.__class__.__name__ + ) + ) return super(WorkerInfo, self).__setattr__(key, val) class _WorkerException(object): - def __init__(self, worker_id, exc_info=None): self.worker_id = worker_id exc_info = exc_info or sys.exc_info() @@ -166,7 +173,8 @@ def __init__(self, worker_id, exc_info=None): def reraise(self): msg = "DataLoader worker({}) caught {} with message:\n{}".format( - self.worker_id, self.exc_type.__name__, self.exc_msg) + self.worker_id, self.exc_type.__name__, self.exc_msg + ) if getattr(self.exc_type, "message", None): raise self.exc_type(message=msg) raise self.exc_type(msg) @@ -204,12 +212,12 @@ def reraise(self): # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -INIT_A = 0x43b0d7e5 -MULT_A = 0x931e8875 -INIT_B = 0x8b51f9dd -MULT_B = 0x58f38ded -MIX_MULT_L = 0xca01f9dd -MIX_MULT_R = 0x4973f715 +INIT_A = 0x43B0D7E5 +MULT_A = 0x931E8875 +INIT_B = 0x8B51F9DD +MULT_B = 0x58F38DED +MIX_MULT_L = 0xCA01F9DD +MIX_MULT_R = 0x4973F715 XSHIFT = np.dtype(np.uint32).itemsize * 8 // 2 MASK32 = 0xFFFFFFFF @@ -255,9 +263,21 @@ def mix(x, y): return states -def _worker_loop(dataset, dataset_kind, indices_queue, out_queue, done_event, - auto_collate_batch, collate_fn, drop_last, init_fn, worker_id, - num_workers, use_shared_memory, base_seed): +def _worker_loop( + dataset, + dataset_kind, + indices_queue, + out_queue, + done_event, + auto_collate_batch, + collate_fn, + drop_last, + init_fn, + worker_id, + num_workers, + use_shared_memory, + base_seed, +): try: # NOTE: [ mmap files clear ] When the child process exits unexpectedly, # some shared memory objects may have been applied for but have not yet @@ -282,18 +302,20 @@ def _worker_loop(dataset, dataset_kind, indices_queue, out_queue, done_event, np.random.seed(_generate_states(base_seed, worker_id)) global _worker_info - _worker_info = WorkerInfo(id=worker_id, - num_workers=num_workers, - dataset=dataset, - seed=base_seed) + _worker_info = WorkerInfo( + id=worker_id, + num_workers=num_workers, + dataset=dataset, + seed=base_seed, + ) init_exception = None try: if init_fn is not None: init_fn(worker_id) - fetcher = _DatasetKind.create_fetcher(dataset_kind, dataset, - auto_collate_batch, - collate_fn, drop_last) + fetcher = _DatasetKind.create_fetcher( + dataset_kind, dataset, auto_collate_batch, collate_fn, drop_last + ) except: init_exception = _WorkerException(worker_id) @@ -309,15 +331,16 @@ def _worker_loop(dataset, dataset_kind, indices_queue, out_queue, done_event, if isinstance(data, _ResumeIteration): out_queue.put((data, None, None)) iterator_drained = False - fetcher = _DatasetKind.create_fetcher(dataset_kind, dataset, - auto_collate_batch, - collate_fn, True) + fetcher = _DatasetKind.create_fetcher( + dataset_kind, dataset, auto_collate_batch, collate_fn, True + ) continue # None as poison piil, so worker event should be set if data is None: - assert done_event.is_set() or iterator_drained, \ - "get None when worker done_event set" + assert ( + done_event.is_set() or iterator_drained + ), "get None when worker done_event set" break # If worker done event is set but get still get data in # indices_queue, remaining data should be get and skipped. @@ -338,8 +361,10 @@ def _worker_loop(dataset, dataset_kind, indices_queue, out_queue, done_event, with paddle.fluid.dygraph.guard(place=paddle.CPUPlace()): batch = fetcher.fetch(indices) except Exception as e: - if isinstance( - e, StopIteration) and dataset_kind == _DatasetKind.ITER: + if ( + isinstance(e, StopIteration) + and dataset_kind == _DatasetKind.ITER + ): out_queue.put(_IterableDatasetStopIteration(worker_id)) iterator_drained = True else: @@ -355,10 +380,12 @@ def tensor_share_memory(tensor): if _in_eager_without_dygraph_check(): return core._array_to_share_memory_tensor(tensor) return tensor._share_memory() + tensor_list = [ core._array_to_share_memory_tensor(b) - if isinstance(b, np.ndarray) \ - else tensor_share_memory(b) for b in batch + if isinstance(b, np.ndarray) + else tensor_share_memory(b) + for b in batch ] out_queue.put((idx, tensor_list, structure)) core._remove_tensor_list_mmap_fds(tensor_list) diff --git a/python/paddle/fluid/dataset.py b/python/paddle/fluid/dataset.py index 4c7e8bb5378be..ff68300d4a01b 100644 --- a/python/paddle/fluid/dataset.py +++ b/python/paddle/fluid/dataset.py @@ -36,7 +36,7 @@ class DatasetFactory(object): """ def __init__(self): - """ Init. """ + """Init.""" pass def create_dataset(self, datafeed_class="QueueDataset"): @@ -59,15 +59,16 @@ def create_dataset(self, datafeed_class="QueueDataset"): dataset = globals()[datafeed_class]() return dataset except: - raise ValueError("datafeed class %s does not exist" % - datafeed_class) + raise ValueError( + "datafeed class %s does not exist" % datafeed_class + ) class DatasetBase(object): - """ Base dataset class. """ + """Base dataset class.""" def __init__(self): - """ Init. """ + """Init.""" # define class name here # to decide whether we need create in memory instance self.proto_desc = data_feed_pb2.DataFeedDesc() @@ -374,7 +375,7 @@ class InMemoryDataset(DatasetBase): @deprecated(since="2.0.0", update_to="paddle.distributed.InMemoryDataset") def __init__(self): - """ Init. """ + """Init.""" super(InMemoryDataset, self).__init__() self.proto_desc.name = "MultiSlotInMemoryDataFeed" self.fleet_send_batch_size = None @@ -389,18 +390,22 @@ def __init__(self): self.fleet_send_sleep_seconds = None self.trainer_num = -1 - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._set_feed_type") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset._set_feed_type", + ) def set_feed_type(self, data_feed_type): """ Set data_feed_desc """ self.proto_desc.name = data_feed_type - if (self.proto_desc.name == "SlotRecordInMemoryDataFeed"): + if self.proto_desc.name == "SlotRecordInMemoryDataFeed": self.dataset = core.Dataset("SlotRecordDataset") - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._prepare_to_run") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset._prepare_to_run", + ) def _prepare_to_run(self): """ Set data_feed_desc before load or shuffle, @@ -423,8 +428,8 @@ def _prepare_to_run(self): @deprecated( since="2.0.0", - update_to= - "paddle.distributed.InMemoryDataset._dynamic_adjust_before_train") + update_to="paddle.distributed.InMemoryDataset._dynamic_adjust_before_train", + ) def _dynamic_adjust_before_train(self, thread_num): if not self.is_user_set_queue_num: if self.use_ps_gpu: @@ -435,7 +440,7 @@ def _dynamic_adjust_before_train(self, thread_num): @deprecated( since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._dynamic_adjust_after_train" + update_to="paddle.distributed.InMemoryDataset._dynamic_adjust_after_train", ) def _dynamic_adjust_after_train(self): if not self.is_user_set_queue_num: @@ -445,8 +450,10 @@ def _dynamic_adjust_after_train(self): self.dataset.dynamic_adjust_channel_num(self.thread_num, False) self.dataset.dynamic_adjust_readers_num(self.thread_num) - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._set_queue_num") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset._set_queue_num", + ) def set_queue_num(self, queue_num): """ Set Dataset output queue num, training threads get data from queues @@ -465,9 +472,10 @@ def set_queue_num(self, queue_num): self.is_user_set_queue_num = True self.queue_num = queue_num - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._set_parse_ins_id" - ) + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset._set_parse_ins_id", + ) def set_parse_ins_id(self, parse_ins_id): """ Set id Dataset need to parse insid @@ -487,7 +495,8 @@ def set_parse_ins_id(self, parse_ins_id): @deprecated( since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._set_parse_content") + update_to="paddle.distributed.InMemoryDataset._set_parse_content", + ) def set_parse_content(self, parse_content): """ Set if Dataset need to parse content @@ -539,9 +548,10 @@ def _set_trainer_num(self, trainer_num): """ self.trainer_num = trainer_num - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._set_merge_by_sid" - ) + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset._set_merge_by_sid", + ) def set_merge_by_sid(self, merge_by_sid): """ Set if Dataset need to merge sid. If not, one ins means one Pv. @@ -633,7 +643,7 @@ def postprocess_instance(self): @deprecated( since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._set_fleet_send_batch_size" + update_to="paddle.distributed.InMemoryDataset._set_fleet_send_batch_size", ) def set_fleet_send_batch_size(self, fleet_send_batch_size=1024): """ @@ -654,8 +664,8 @@ def set_fleet_send_batch_size(self, fleet_send_batch_size=1024): @deprecated( since="2.0.0", - update_to= - "paddle.distributed.InMemoryDataset._set_fleet_send_sleep_seconds") + update_to="paddle.distributed.InMemoryDataset._set_fleet_send_sleep_seconds", + ) def set_fleet_send_sleep_seconds(self, fleet_send_sleep_seconds=0): """ Set fleet send sleep time, default is 0 @@ -675,7 +685,8 @@ def set_fleet_send_sleep_seconds(self, fleet_send_sleep_seconds=0): @deprecated( since="2.0.0", - update_to="paddle.distributed.InMemoryDataset._set_merge_by_lineid") + update_to="paddle.distributed.InMemoryDataset._set_merge_by_lineid", + ) def set_merge_by_lineid(self, merge_size=2): """ Set merge by line id, instances of same line id will be merged after @@ -698,8 +709,8 @@ def set_merge_by_lineid(self, merge_size=2): @deprecated( since="2.0.0", - update_to= - "paddle.distributed.InMemoryDataset._set_generate_unique_feasigns") + update_to="paddle.distributed.InMemoryDataset._set_generate_unique_feasigns", + ) def set_generate_unique_feasigns(self, generate_uni_feasigns, shard_num): self.dataset.set_generate_unique_feasigns(generate_uni_feasigns) self.gen_uni_feasigns = generate_uni_feasigns @@ -707,13 +718,14 @@ def set_generate_unique_feasigns(self, generate_uni_feasigns, shard_num): @deprecated( since="2.0.0", - update_to= - "paddle.distributed.InMemoryDataset._generate_local_tables_unlock") - def generate_local_tables_unlock(self, table_id, fea_dim, read_thread_num, - consume_thread_num, shard_num): - self.dataset.generate_local_tables_unlock(table_id, fea_dim, - read_thread_num, - consume_thread_num, shard_num) + update_to="paddle.distributed.InMemoryDataset._generate_local_tables_unlock", + ) + def generate_local_tables_unlock( + self, table_id, fea_dim, read_thread_num, consume_thread_num, shard_num + ): + self.dataset.generate_local_tables_unlock( + table_id, fea_dim, read_thread_num, consume_thread_num, shard_num + ) def set_date(self, date): """ @@ -738,8 +750,10 @@ def set_date(self, date): if self.use_ps_gpu and core._is_compiled_with_heterps(): self.psgpu.set_date(year, month, day) - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.load_into_memory") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset.load_into_memory", + ) def load_into_memory(self, is_shuffle=False): """ Load data into memory @@ -766,7 +780,8 @@ def load_into_memory(self, is_shuffle=False): @deprecated( since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.preload_into_memory") + update_to="paddle.distributed.InMemoryDataset.preload_into_memory", + ) def preload_into_memory(self, thread_num=None): """ Load data into memory in async mode @@ -792,9 +807,10 @@ def preload_into_memory(self, thread_num=None): self.dataset.create_preload_readers() self.dataset.preload_into_memory() - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.wait_preload_done" - ) + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset.wait_preload_done", + ) def wait_preload_done(self): """ Wait preload_into_memory done @@ -813,8 +829,10 @@ def wait_preload_done(self): self.dataset.wait_preload_done() self.dataset.destroy_preload_readers() - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.local_shuffle") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset.local_shuffle", + ) def local_shuffle(self): """ Local shuffle @@ -832,8 +850,10 @@ def local_shuffle(self): """ self.dataset.local_shuffle() - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.global_shuffle") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset.global_shuffle", + ) def global_shuffle(self, fleet=None, thread_num=12): """ Global shuffle. @@ -893,8 +913,10 @@ def global_shuffle(self, fleet=None, thread_num=12): else: fleet._role_maker.barrier_worker() - @deprecated(since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.release_memory") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.InMemoryDataset.release_memory", + ) def release_memory(self): """ :api_attr: Static Graph @@ -946,7 +968,8 @@ def get_pv_data_size(self): @deprecated( since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.get_memory_data_size") + update_to="paddle.distributed.InMemoryDataset.get_memory_data_size", + ) def get_memory_data_size(self, fleet=None): """ Get memory data size, user can call this function to know the num @@ -975,18 +998,21 @@ def get_memory_data_size(self, fleet=None): """ import numpy as np + local_data_size = self.dataset.get_memory_data_size() local_data_size = np.array([local_data_size]) if fleet is not None: global_data_size = local_data_size * 0 - fleet._role_maker.all_reduce_worker(local_data_size, - global_data_size) + fleet._role_maker.all_reduce_worker( + local_data_size, global_data_size + ) return global_data_size[0] return local_data_size[0] @deprecated( since="2.0.0", - update_to="paddle.distributed.InMemoryDataset.get_shuffle_data_size") + update_to="paddle.distributed.InMemoryDataset.get_shuffle_data_size", + ) def get_shuffle_data_size(self, fleet=None): """ Get shuffle data size, user can call this function to know the num @@ -1017,6 +1043,7 @@ def get_shuffle_data_size(self, fleet=None): """ import numpy as np + local_data_size = self.dataset.get_shuffle_data_size() local_data_size = np.array([local_data_size]) print('global shuffle local_data_size: ', local_data_size) @@ -1025,8 +1052,9 @@ def get_shuffle_data_size(self, fleet=None): if hasattr(fleet, "util"): global_data_size = fleet.util.all_reduce(local_data_size) else: - fleet._role_maker.all_reduce_worker(local_data_size, - global_data_size) + fleet._role_maker.all_reduce_worker( + local_data_size, global_data_size + ) return global_data_size[0] return local_data_size[0] @@ -1070,16 +1098,20 @@ def set_graph_config(self, config): self.proto_desc.graph_config.walk_len = config.get("walk_len", 20) self.proto_desc.graph_config.window = config.get("window", 5) self.proto_desc.graph_config.once_sample_startid_len = config.get( - "once_sample_startid_len", 8000) + "once_sample_startid_len", 8000 + ) self.proto_desc.graph_config.sample_times_one_chunk = config.get( - "sample_times_one_chunk", 10) + "sample_times_one_chunk", 10 + ) self.proto_desc.graph_config.batch_size = config.get("batch_size", 1) self.proto_desc.graph_config.debug_mode = config.get("debug_mode", 0) self.proto_desc.graph_config.first_node_type = config.get( - "first_node_type", "") + "first_node_type", "" + ) self.proto_desc.graph_config.meta_path = config.get("meta_path", "") self.proto_desc.graph_config.gpu_graph_training = config.get( - "gpu_graph_training", True) + "gpu_graph_training", True + ) self.dataset.set_gpu_graph_mode(True) @@ -1103,8 +1135,10 @@ def __init__(self): super(QueueDataset, self).__init__() self.proto_desc.name = "MultiSlotDataFeed" - @deprecated(since="2.0.0", - update_to="paddle.distributed.QueueDataset._prepare_to_run") + @deprecated( + since="2.0.0", + update_to="paddle.distributed.QueueDataset._prepare_to_run", + ) def _prepare_to_run(self): """ Set data_feed_desc/thread num/filelist before run, @@ -1139,7 +1173,8 @@ def local_shuffle(self): """ raise NotImplementedError( "QueueDataset does not support local shuffle, " - "please use InMemoryDataset for local_shuffle") + "please use InMemoryDataset for local_shuffle" + ) def global_shuffle(self, fleet=None): """ @@ -1165,7 +1200,8 @@ def global_shuffle(self, fleet=None): """ raise NotImplementedError( "QueueDataset does not support global shuffle, " - "please use InMemoryDataset for global_shuffle") + "please use InMemoryDataset for global_shuffle" + ) class FileInstantDataset(DatasetBase): @@ -1194,7 +1230,8 @@ def local_shuffle(self): """ raise NotImplementedError( "FileInstantDataset does not support local shuffle, " - "please use InMemoryDataset for local_shuffle") + "please use InMemoryDataset for local_shuffle" + ) def global_shuffle(self, fleet=None): """ @@ -1203,7 +1240,8 @@ def global_shuffle(self, fleet=None): """ raise NotImplementedError( "FileInstantDataset does not support global shuffle, " - "please use InMemoryDataset for global_shuffle") + "please use InMemoryDataset for global_shuffle" + ) class BoxPSDataset(InMemoryDataset): @@ -1289,7 +1327,7 @@ def load_into_memory(self): filelist = ["a.txt", "b.txt"] dataset.set_filelist(filelist) dataset.load_into_memory() - """ + """ self._prepare_to_run() self.boxps.load_into_memory() diff --git a/python/paddle/fluid/debugger.py b/python/paddle/fluid/debugger.py index 0c2d61db2d295..36f38df38e356 100644 --- a/python/paddle/fluid/debugger.py +++ b/python/paddle/fluid/debugger.py @@ -52,8 +52,9 @@ def repr_data_type(type): def repr_tensor(proto): - return "tensor(type={}, shape={})".format(_dtype2str_[int(proto.data_type)], - str(proto.dims)) + return "tensor(type={}, shape={})".format( + _dtype2str_[int(proto.data_type)], str(proto.dims) + ) reprtpl = "{ttype} {name} ({reprs})" @@ -65,30 +66,37 @@ def repr_lodtensor(proto): level = proto.type.lod_tensor.lod_level reprs = repr_tensor(proto.type.lod_tensor.tensor) - return reprtpl.format(ttype="LoDTensor" if level > 0 else "Tensor", - name=proto.name, - reprs="level=%d, %s" % - (level, reprs) if level > 0 else reprs) + return reprtpl.format( + ttype="LoDTensor" if level > 0 else "Tensor", + name=proto.name, + reprs="level=%d, %s" % (level, reprs) if level > 0 else reprs, + ) def repr_selected_rows(proto): if proto.type.type != framework_pb2.VarType.SELECTED_ROWS: return - return reprtpl.format(ttype="SelectedRows", - name=proto.name, - reprs=repr_tensor(proto.type.selected_rows)) + return reprtpl.format( + ttype="SelectedRows", + name=proto.name, + reprs=repr_tensor(proto.type.selected_rows), + ) def repr_tensor_array(proto): if proto.type.type != framework_pb2.VarType.LOD_TENSOR_ARRAY: return - return reprtpl.format(ttype="TensorArray", - name=proto.name, - reprs="level=%d, %s" % - (proto.type.tensor_array.lod_level, - repr_tensor(proto.type.lod_tensor.tensor))) + return reprtpl.format( + ttype="TensorArray", + name=proto.name, + reprs="level=%d, %s" + % ( + proto.type.tensor_array.lod_level, + repr_tensor(proto.type.lod_tensor.tensor), + ), + ) type_handlers = [ @@ -115,19 +123,23 @@ def pprint_program_codes(program_desc): def pprint_block_codes(block_desc, show_backward=False): - def is_op_backward(op_desc): - if op_desc.type.endswith('_grad'): return True + if op_desc.type.endswith('_grad'): + return True def is_var_backward(var): - if "@GRAD" in var.parameter: return True + if "@GRAD" in var.parameter: + return True for arg in var.arguments: - if "@GRAD" in arg: return True + if "@GRAD" in arg: + return True for var in op_desc.inputs: - if is_var_backward(var): return True + if is_var_backward(var): + return True for var in op_desc.outputs: - if is_var_backward(var): return True + if is_var_backward(var): + return True return False def is_var_backward(var_desc): @@ -135,7 +147,8 @@ def is_var_backward(var_desc): if type(block_desc) is not framework_pb2.BlockDesc: block_desc = framework_pb2.BlockDesc.FromString( - block_desc.desc.serialize_to_string()) + block_desc.desc.serialize_to_string() + ) var_reprs = [] op_reprs = [] for var in block_desc.vars: @@ -144,7 +157,8 @@ def is_var_backward(var_desc): var_reprs.append(repr_var(var)) for op in block_desc.ops: - if not show_backward and is_op_backward(op): continue + if not show_backward and is_op_backward(op): + continue op_reprs.append(repr_op(op)) tpl = "// block-{idx} parent-{pidx}\n// variables\n{vars}\n\n// operators\n{ops}\n" @@ -182,7 +196,8 @@ def _repr_op_fill_constant(optype, inputs, outputs, attrs): return "{output} = {data} [shape={shape}]".format( output=','.join(outputs), data=attrs['value'], - shape=str(attrs['shape'])) + shape=str(attrs['shape']), + ) op_repr_handlers = [ @@ -216,13 +231,16 @@ def repr_op(opdesc): for handler in op_repr_handlers: res = handler(opdesc.type, inputs, outputs, attr_dict) - if res: return res + if res: + return res - return tpl.format(outputs=', '.join(outputs), - optype=opdesc.type, - inputs=', '.join(inputs), - attrs="{%s}" % ','.join(attrs), - is_target=", is_target" if is_target else "") + return tpl.format( + outputs=', '.join(outputs), + optype=opdesc.type, + inputs=', '.join(inputs), + attrs="{%s}" % ','.join(attrs), + is_target=", is_target" if is_target else "", + ) def draw_block_graphviz(block, highlights=None, path="./temp.dot"): @@ -237,7 +255,8 @@ def draw_block_graphviz(block, highlights=None, path="./temp.dot"): desc = framework_pb2.BlockDesc.FromString(bytes(protostr)) def need_highlight(name): - if highlights is None: return False + if highlights is None: + return False for pattern in highlights: assert type(pattern) is str if re.match(pattern, name): @@ -250,9 +269,11 @@ def need_highlight(name): # TODO(gongwb): format the var.type # create var if var.persistable: - varn = graph.add_param(var.name, - str(var.type).replace("\n", "
", 1), - highlight=need_highlight(var.name)) + varn = graph.add_param( + var.name, + str(var.type).replace("\n", "
", 1), + highlight=need_highlight(var.name), + ) else: varn = graph.add_arg(var.name, highlight=need_highlight(var.name)) vars[var.name] = varn @@ -264,7 +285,8 @@ def add_op_link_var(op, var, op2var=False): vars[arg] = graph.add_arg(arg, highlight=need_highlight(arg)) varn = vars[arg] highlight = need_highlight(op.description) or need_highlight( - varn.description) + varn.description + ) if op2var: graph.add_edge(op, varn, highlight=highlight) else: diff --git a/python/paddle/fluid/device_worker.py b/python/paddle/fluid/device_worker.py index bf264f5a64703..ee82c7ebbdc10 100644 --- a/python/paddle/fluid/device_worker.py +++ b/python/paddle/fluid/device_worker.py @@ -14,8 +14,12 @@ """Defination of device workers.""" __all__ = [ - 'DeviceWorker', 'Hogwild', 'DownpourSGD', 'Section', 'DownpourSGDOPT', - 'HeterSection' + 'DeviceWorker', + 'Hogwild', + 'DownpourSGD', + 'Section', + 'DownpourSGDOPT', + 'HeterSection', ] @@ -67,7 +71,8 @@ def _gen_worker_desc(self, trainer_desc): """ raise NotImplementedError( "DeviceWorker does not implement gen_worker_desc, " - "please use Hogwild or DownpourSGD, etc.") + "please use Hogwild or DownpourSGD, etc." + ) class Hogwild(DeviceWorker): @@ -90,10 +95,16 @@ def _gen_worker_desc(self, trainer_desc): trainer_desc.device_worker_name = "HogwildWorker" if self._infer: # just ignore feed op for inference model - trainer_desc.hogwild_param.skip_ops.extend([ - "feed", "push_sparse", "push_sparse_v2", "push_dense", - "distributed_push_sparse", "send" - ]) + trainer_desc.hogwild_param.skip_ops.extend( + [ + "feed", + "push_sparse", + "push_sparse_v2", + "push_dense", + "distributed_push_sparse", + "send", + ] + ) dense_table_set = set() program_id = str(id(self._program)) @@ -114,8 +125,11 @@ def _gen_worker_desc(self, trainer_desc): from paddle.fluid.incubate.fleet.parameter_server import version - if version.is_transpiler( - ) and "fleet_desc" not in opt_info and "program_configs" not in opt_info: + if ( + version.is_transpiler() + and "fleet_desc" not in opt_info + and "program_configs" not in opt_info + ): return program_configs = opt_info["program_configs"] @@ -126,8 +140,10 @@ def _gen_worker_desc(self, trainer_desc): if pid == program_id: pc = downpour.program_config.add() pc.program_id = program_id - print("device worker pull dense:", - program_configs[program_id]["pull_dense"]) + print( + "device worker pull dense:", + program_configs[program_id]["pull_dense"], + ) for i in program_configs[program_id]["push_sparse"]: pc.push_sparse_table_id.extend([i]) for i in program_configs[program_id]["push_dense"]: @@ -143,36 +159,43 @@ def _gen_worker_desc(self, trainer_desc): trainer_desc.device_worker_name = "HogwildWorker" pull_thread = trainer_desc.pull_dense_param pull_thread.device_num = trainer_desc.thread_num - if opt_info.get("program_id_to_worker") is None and opt_info.get( - "dense_table_config") is None: + if ( + opt_info.get("program_id_to_worker") is None + and opt_info.get("dense_table_config") is None + ): raise ValueError( - "opt_info must have program_id_to_worker or dense_table_config") + "opt_info must have program_id_to_worker or dense_table_config" + ) if opt_info.get("program_id_to_worker") is not None: prog_id_to_worker = opt_info["program_id_to_worker"] if prog_id_to_worker.get(program_id) is None: - raise ValueError("%s not found in program_id_to_worker" % - program_id) + raise ValueError( + "%s not found in program_id_to_worker" % program_id + ) worker = opt_info["program_id_to_worker"][program_id] for i in worker.get_desc().dense_table: if i.table_id in dense_table_set: dense_table = pull_thread.dense_table.add() dense_table.dense_value_name.extend(i.dense_variable_name) - dense_table.table_id = \ - i.table_id + dense_table.table_id = i.table_id sparse_len = len(worker.get_desc().sparse_table) for i in range(sparse_len): sparse_table = downpour.sparse_table.add() - sparse_table.table_id = worker.get_desc( - ).sparse_table[i].table_id + sparse_table.table_id = ( + worker.get_desc().sparse_table[i].table_id + ) sparse_table.sparse_key_name.extend( - worker.get_desc().sparse_table[i].slot_key) + worker.get_desc().sparse_table[i].slot_key + ) sparse_table.sparse_value_name.extend( - worker.get_desc().sparse_table[i].slot_value) + worker.get_desc().sparse_table[i].slot_value + ) sparse_table.sparse_grad_name.extend( - worker.get_desc().sparse_table[i].slot_gradient) - sparse_table.fea_dim = \ - self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ - i].accessor.fea_dim + worker.get_desc().sparse_table[i].slot_gradient + ) + sparse_table.fea_dim = self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ + i + ].accessor.fea_dim # not use emb_dim sparse_table.emb_dim = -1 # not use hard code click @@ -184,7 +207,8 @@ def _gen_worker_desc(self, trainer_desc): dense_table.table_id = i.table_id dense_table.dense_value_name.extend(i.dense_variable_name) dense_table.dense_grad_name.extend( - i.dense_gradient_variable_name) + i.dense_gradient_variable_name + ) hogwild.skip_ops.extend(worker.get_desc().skip_op) else: dense_table_config = opt_info.get("dense_table_config") @@ -196,7 +220,8 @@ def _gen_worker_desc(self, trainer_desc): if self._infer: hogwild.skip_ops.extend( - ["push_sparse", "push_sparse_v2", "push_dense"]) + ["push_sparse", "push_sparse_v2", "push_dense"] + ) class DownpourLite(DeviceWorker): @@ -220,10 +245,16 @@ def _gen_worker_desc(self, trainer_desc): trainer_desc.device_worker_name = "DownpourLiteWorker" if self._infer: # just ignore feed op for inference model - trainer_desc.downpour_param.skip_ops.extend([ - "feed", "push_sparse", "push_sparse_v2", "push_dense", - "distributed_push_sparse", "send" - ]) + trainer_desc.downpour_param.skip_ops.extend( + [ + "feed", + "push_sparse", + "push_sparse_v2", + "push_dense", + "distributed_push_sparse", + "send", + ] + ) dense_table_set = set() program_id = str(id(self._program)) @@ -242,8 +273,11 @@ def _gen_worker_desc(self, trainer_desc): from paddle.fluid.incubate.fleet.parameter_server import version - if version.is_transpiler( - ) and "fleet_desc" not in opt_info and "program_configs" not in opt_info: + if ( + version.is_transpiler() + and "fleet_desc" not in opt_info + and "program_configs" not in opt_info + ): return program_configs = opt_info["program_configs"] @@ -254,8 +288,10 @@ def _gen_worker_desc(self, trainer_desc): if pid == program_id: pc = downpour.program_config.add() pc.program_id = program_id - print("device worker pull dense:", - program_configs[program_id]["pull_dense"]) + print( + "device worker pull dense:", + program_configs[program_id]["pull_dense"], + ) for i in program_configs[program_id]["push_sparse"]: pc.push_sparse_table_id.extend([i]) for i in program_configs[program_id]["push_dense"]: @@ -270,36 +306,43 @@ def _gen_worker_desc(self, trainer_desc): pull_thread = trainer_desc.pull_dense_param pull_thread.device_num = trainer_desc.thread_num - if opt_info.get("program_id_to_worker") is None and opt_info.get( - "dense_table_config") is None: + if ( + opt_info.get("program_id_to_worker") is None + and opt_info.get("dense_table_config") is None + ): raise ValueError( - "opt_info must have program_id_to_worker or dense_table_config") + "opt_info must have program_id_to_worker or dense_table_config" + ) if opt_info.get("program_id_to_worker") is not None: prog_id_to_worker = opt_info["program_id_to_worker"] if prog_id_to_worker.get(program_id) is None: - raise ValueError("%s not found in program_id_to_worker" % - program_id) + raise ValueError( + "%s not found in program_id_to_worker" % program_id + ) worker = opt_info["program_id_to_worker"][program_id] for i in worker.get_desc().dense_table: if i.table_id in dense_table_set: dense_table = pull_thread.dense_table.add() dense_table.dense_value_name.extend(i.dense_variable_name) - dense_table.table_id = \ - i.table_id + dense_table.table_id = i.table_id sparse_len = len(worker.get_desc().sparse_table) for i in range(sparse_len): sparse_table = downpour.sparse_table.add() - sparse_table.table_id = worker.get_desc( - ).sparse_table[i].table_id + sparse_table.table_id = ( + worker.get_desc().sparse_table[i].table_id + ) sparse_table.sparse_key_name.extend( - worker.get_desc().sparse_table[i].slot_key) + worker.get_desc().sparse_table[i].slot_key + ) sparse_table.sparse_value_name.extend( - worker.get_desc().sparse_table[i].slot_value) + worker.get_desc().sparse_table[i].slot_value + ) sparse_table.sparse_grad_name.extend( - worker.get_desc().sparse_table[i].slot_gradient) - sparse_table.fea_dim = \ - self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ - i].accessor.fea_dim + worker.get_desc().sparse_table[i].slot_gradient + ) + sparse_table.fea_dim = self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ + i + ].accessor.fea_dim # not use emb_dim sparse_table.emb_dim = -1 # not use hard code click @@ -311,7 +354,8 @@ def _gen_worker_desc(self, trainer_desc): dense_table.table_id = i.table_id dense_table.dense_value_name.extend(i.dense_variable_name) dense_table.dense_grad_name.extend( - i.dense_gradient_variable_name) + i.dense_gradient_variable_name + ) downpour.skip_ops.extend(worker.get_desc().skip_op) else: dense_table_config = opt_info.get("dense_table_config") @@ -323,7 +367,8 @@ def _gen_worker_desc(self, trainer_desc): if self._infer: downpour.skip_ops.extend( - ["push_sparse", "push_sparse_v2", "push_dense"]) + ["push_sparse", "push_sparse_v2", "push_dense"] + ) class DownpourSGD(DeviceWorker): @@ -377,43 +422,53 @@ def _gen_worker_desc(self, trainer_desc): mc_map.value = value break - trainer_desc.device_worker_name = opt_info.get("worker_class", - "DownpourWorker") + trainer_desc.device_worker_name = opt_info.get( + "worker_class", "DownpourWorker" + ) pull_thread = trainer_desc.pull_dense_param pull_thread.device_num = trainer_desc.thread_num if opt_info.get("program_id_to_worker") is None: raise ValueError("opt_info must have program_id_to_worker") prog_id_to_worker = opt_info["program_id_to_worker"] if prog_id_to_worker.get(program_id) is None: - raise ValueError("%s not found in program_id_to_worker" % - program_id) + raise ValueError( + "%s not found in program_id_to_worker" % program_id + ) worker = opt_info["program_id_to_worker"][program_id] for i in worker.get_desc().dense_table: if i.table_id in dense_table_set: dense_table = pull_thread.dense_table.add() dense_table.dense_value_name.extend(i.dense_variable_name) - dense_table.table_id = \ - i.table_id + dense_table.table_id = i.table_id sparse_len = len(worker.get_desc().sparse_table) for i in range(sparse_len): sparse_table = downpour.sparse_table.add() sparse_table.table_id = worker.get_desc().sparse_table[i].table_id sparse_table.sparse_key_name.extend( - worker.get_desc().sparse_table[i].slot_key) + worker.get_desc().sparse_table[i].slot_key + ) sparse_table.sparse_value_name.extend( - worker.get_desc().sparse_table[i].slot_value) + worker.get_desc().sparse_table[i].slot_value + ) sparse_table.sparse_grad_name.extend( - worker.get_desc().sparse_table[i].slot_gradient) - if opt_info["use_cvm"] or "no_cvm" in opt_info and opt_info[ - "no_cvm"] == True: - sparse_table.emb_dim = \ - self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ - i].accessor.fea_dim + worker.get_desc().sparse_table[i].slot_gradient + ) + if ( + opt_info["use_cvm"] + or "no_cvm" in opt_info + and opt_info["no_cvm"] == True + ): + sparse_table.emb_dim = self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ + i + ].accessor.fea_dim sparse_table.fea_dim = sparse_table.emb_dim else: - sparse_table.emb_dim = \ + sparse_table.emb_dim = ( self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ - i].accessor.fea_dim - 2 + i + ].accessor.fea_dim + - 2 + ) sparse_table.fea_dim = sparse_table.emb_dim + 2 # TODO(guru4elephant): hard code here, need to improve sparse_table.label_var_name = "click" @@ -427,7 +482,8 @@ def _gen_worker_desc(self, trainer_desc): dense_table.table_id = i.table_id dense_table.dense_value_name.extend(i.dense_variable_name) dense_table.dense_grad_name.extend( - i.dense_gradient_variable_name) + i.dense_gradient_variable_name + ) downpour.skip_ops.extend(worker.get_desc().skip_op) if self._infer: downpour.push_dense = False @@ -485,43 +541,56 @@ def _gen_worker_desc(self, trainer_desc): raise ValueError("opt_info must have program_id_to_worker") prog_id_to_worker = opt_info["program_id_to_worker"] if prog_id_to_worker.get(program_id) is None: - raise ValueError("%s not found in program_id_to_worker" % - program_id) + raise ValueError( + "%s not found in program_id_to_worker" % program_id + ) worker = opt_info["program_id_to_worker"][program_id] for i in worker.get_desc().dense_table: if i.table_id in dense_table_set: dense_table = pull_thread.dense_table.add() dense_table.dense_value_name.extend(i.dense_variable_name) - dense_table.table_id = \ - i.table_id + dense_table.table_id = i.table_id sparse_len = len(worker.get_desc().sparse_table) for i in range(sparse_len): sparse_table = downpour.sparse_table.add() sparse_table.table_id = worker.get_desc().sparse_table[i].table_id sparse_table.sparse_key_name.extend( - worker.get_desc().sparse_table[i].slot_key) + worker.get_desc().sparse_table[i].slot_key + ) sparse_table.sparse_value_name.extend( - worker.get_desc().sparse_table[i].slot_value) + worker.get_desc().sparse_table[i].slot_value + ) sparse_table.sparse_grad_name.extend( - worker.get_desc().sparse_table[i].slot_gradient) - if opt_info["use_cvm"] or "no_cvm" in opt_info and opt_info[ - "no_cvm"] == True: - sparse_table.emb_dim = \ - self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ - i].accessor.fea_dim + worker.get_desc().sparse_table[i].slot_gradient + ) + if ( + opt_info["use_cvm"] + or "no_cvm" in opt_info + and opt_info["no_cvm"] == True + ): + sparse_table.emb_dim = self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ + i + ].accessor.fea_dim sparse_table.fea_dim = sparse_table.emb_dim else: - sparse_table.emb_dim = \ + sparse_table.emb_dim = ( self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ - i].accessor.fea_dim - 2 + i + ].accessor.fea_dim + - 2 + ) sparse_table.fea_dim = sparse_table.emb_dim + 2 # TODO(guru4elephant): hard code here, need to improve sparse_table.label_var_name = "click" - if "local_tables" in opt_info and sparse_table.table_id in opt_info[ - "local_tables"]: + if ( + "local_tables" in opt_info + and sparse_table.table_id in opt_info["local_tables"] + ): sparse_table.is_local = True - if "async_tables" in opt_info and sparse_table.table_id in opt_info[ - "async_tables"]: + if ( + "async_tables" in opt_info + and sparse_table.table_id in opt_info["async_tables"] + ): sparse_table.is_async = True if opt_info["stat_var_names"]: for i in opt_info["stat_var_names"]: @@ -533,7 +602,8 @@ def _gen_worker_desc(self, trainer_desc): dense_table.table_id = i.table_id dense_table.dense_value_name.extend(i.dense_variable_name) dense_table.dense_grad_name.extend( - i.dense_gradient_variable_name) + i.dense_gradient_variable_name + ) downpour.skip_ops.extend(worker.get_desc().skip_op) if self._infer: downpour.push_dense = False @@ -555,6 +625,7 @@ def _gen_worker_desc(self, trainer_desc): """ from google.protobuf import text_format from . import core + trainer_desc.device_worker_name = "SectionWorker" pipeline_opt = self._program._pipeline_opt section_param = trainer_desc.section_param @@ -567,16 +638,16 @@ def _gen_worker_desc(self, trainer_desc): # then runs Backward phase for all microbatches. # 1F1B scheduler, which runs forward phase and backward phase altertively # after startup phase. - assert schedule_mode_str in [ - "F-then-B", "1F1B" - ], ("The schedule mode " - "for pipeline must be one of F-then-B or 1F1B") + assert schedule_mode_str in ["F-then-B", "1F1B"], ( + "The schedule mode " "for pipeline must be one of F-then-B or 1F1B" + ) schedule_mode = 0 if schedule_mode_str == "F-then-B" else 1 section_param.schedule_mode = schedule_mode cfg = section_param.section_config program = pipeline_opt["section_program"] cfg.program_desc.ParseFromString( - program._get_desc().serialize_to_string()) + program._get_desc().serialize_to_string() + ) # TODO: why does not work # cfg.program_desc.CopyFrom(program.program._get_desc()) place = pipeline_opt["place"] @@ -604,23 +675,27 @@ def _gen_worker_desc(self, trainer_desc): """ from google.protobuf import text_format from . import core + trainer_desc.device_worker_name = "HeterSectionWorker" heter_pipeline_opt = self._program._heter_pipeline_opt heter_section_param = trainer_desc.heter_section_param heter_section_param.num_microbatches = heter_pipeline_opt[ - "num_microbatches"] + "num_microbatches" + ] heter_section_param.pipeline_stage = heter_pipeline_opt[ - "pipeline_stage"] + "pipeline_stage" + ] heter_section_param.num_pipeline_stages = heter_pipeline_opt[ - "num_pipeline_stages"] + "num_pipeline_stages" + ] cfg = heter_section_param.section_config program = heter_pipeline_opt["section_program"] cfg.program_desc.ParseFromString( - program._get_desc().serialize_to_string()) + program._get_desc().serialize_to_string() + ) class DeviceWorkerFactory(object): - def _create_device_worker(self, worker_type): classname = worker_type.capitalize() return globals()[classname]() diff --git a/python/paddle/fluid/distribute_lookup_table.py b/python/paddle/fluid/distribute_lookup_table.py index 74824f6832442..cff2388bfdbe0 100644 --- a/python/paddle/fluid/distribute_lookup_table.py +++ b/python/paddle/fluid/distribute_lookup_table.py @@ -70,8 +70,10 @@ def find_distributed_lookup_table(program): if table_name is None: table_name = op.input("W")[0] if table_name != op.input("W")[0]: - raise RuntimeError("all distributed lookup_table_ops" - " should have only one table") + raise RuntimeError( + "all distributed lookup_table_ops" + " should have only one table" + ) else: if table_name is not None: assert op.input("W")[0] != table_name diff --git a/python/paddle/fluid/distributed/downpour.py b/python/paddle/fluid/distributed/downpour.py index 69d592d0d5f72..c710b7337ccb0 100644 --- a/python/paddle/fluid/distributed/downpour.py +++ b/python/paddle/fluid/distributed/downpour.py @@ -16,8 +16,12 @@ from ..backward import append_backward import ps_pb2 as pslib from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table -from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table_inputs -from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table_outputs +from paddle.fluid.distribute_lookup_table import ( + find_distributed_lookup_table_inputs, +) +from paddle.fluid.distribute_lookup_table import ( + find_distributed_lookup_table_outputs, +) from google.protobuf import text_format @@ -47,15 +51,21 @@ def __init__(self, learning_rate=0.001, window=1): self.window_ = window self.type = "downpour" self.data_norm_name = [ - ".batch_size", ".batch_square_sum", ".batch_sum", - ".batch_size@GRAD", ".batch_square_sum@GRAD", ".batch_sum@GRAD" + ".batch_size", + ".batch_square_sum", + ".batch_sum", + ".batch_size@GRAD", + ".batch_square_sum@GRAD", + ".batch_sum@GRAD", ] - def minimize(self, - losses, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, + losses, + startup_program=None, + parameter_list=None, + no_grad_set=None, + ): """ DownpounSGD is a distributed optimizer so that user can call minimize to generate backward @@ -76,30 +86,44 @@ def minimize(self, raise ValueError('losses is a list, just lick [model.cost]') table_name = find_distributed_lookup_table(losses[0].block.program) prefetch_slots = find_distributed_lookup_table_inputs( - losses[0].block.program, table_name) + losses[0].block.program, table_name + ) prefetch_slots_emb = find_distributed_lookup_table_outputs( - losses[0].block.program, table_name) + losses[0].block.program, table_name + ) ps_param = pslib.PSParameter() server = DownpourServer() worker = DownpourWorker(self.window_) sparse_table_index = 0 - server.add_sparse_table(sparse_table_index, self.learning_rate_, - prefetch_slots, prefetch_slots_emb) - worker.add_sparse_table(sparse_table_index, self.learning_rate_, - prefetch_slots, prefetch_slots_emb) + server.add_sparse_table( + sparse_table_index, + self.learning_rate_, + prefetch_slots, + prefetch_slots_emb, + ) + worker.add_sparse_table( + sparse_table_index, + self.learning_rate_, + prefetch_slots, + prefetch_slots_emb, + ) dense_table_index = 1 program_configs = [] param_grads_list = [] for loss_index in range(len(losses)): program_config = ps_param.trainer_param.program_config.add() - program_config.program_id = str(id( - losses[loss_index].block.program)) + program_config.program_id = str( + id(losses[loss_index].block.program) + ) program_config.pull_sparse_table_id.extend([sparse_table_index]) program_config.push_sparse_table_id.extend([sparse_table_index]) - params_grads = sorted(append_backward(losses[loss_index], - parameter_list, no_grad_set), - key=lambda x: x[0].name) + params_grads = sorted( + append_backward( + losses[loss_index], parameter_list, no_grad_set + ), + key=lambda x: x[0].name, + ) param_grads_list.append(params_grads) params = [] grads = [] @@ -121,19 +145,28 @@ def minimize(self, data_norm_grads.append(i[1]) if not is_data_norm_data: grads.append(i[1]) - server.add_dense_table(dense_table_index, self.learning_rate_, - params, grads) - worker.add_dense_table(dense_table_index, self.learning_rate_, - params, grads) + server.add_dense_table( + dense_table_index, self.learning_rate_, params, grads + ) + worker.add_dense_table( + dense_table_index, self.learning_rate_, params, grads + ) program_config.pull_dense_table_id.extend([dense_table_index]) program_config.push_dense_table_id.extend([dense_table_index]) if len(data_norm_params) != 0 and len(data_norm_grads) != 0: dense_table_index += 1 - server.add_data_norm_table(dense_table_index, - self.learning_rate_, - data_norm_params, data_norm_grads) - worker.add_dense_table(dense_table_index, self.learning_rate_, - data_norm_params, data_norm_grads) + server.add_data_norm_table( + dense_table_index, + self.learning_rate_, + data_norm_params, + data_norm_grads, + ) + worker.add_dense_table( + dense_table_index, + self.learning_rate_, + data_norm_params, + data_norm_grads, + ) program_config.pull_dense_table_id.extend([dense_table_index]) program_config.push_dense_table_id.extend([dense_table_index]) dense_table_index += 1 diff --git a/python/paddle/fluid/distributed/fleet.py b/python/paddle/fluid/distributed/fleet.py index 9874492a41323..08b46ce04733a 100644 --- a/python/paddle/fluid/distributed/fleet.py +++ b/python/paddle/fluid/distributed/fleet.py @@ -19,9 +19,7 @@ class Fleet(object): - """ - - """ + """ """ def __init__(self): self.instance_ = ps_instance.PaddlePSInstance() @@ -38,11 +36,13 @@ def stop(self): def init_pserver(self, opt_info): if "fleet_desc" in opt_info: self.dist_desc_str_ = text_format.MessageToString( - opt_info["fleet_desc"]) + opt_info["fleet_desc"] + ) self.dist_desc_ = opt_info["fleet_desc"] else: print( - "You should run distributed optimization to get opt_info first") + "You should run distributed optimization to get opt_info first" + ) sys.exit(-1) self.fleet_.init_server(self.dist_desc_str_) ip = self.fleet_.start_server() @@ -55,17 +55,22 @@ def init_pserver(self, opt_info): def init_worker(self, opt_info): if "fleet_desc" in opt_info: self.dist_desc_str_ = text_format.MessageToString( - opt_info["fleet_desc"]) + opt_info["fleet_desc"] + ) self.dist_desc_ = opt_info["fleet_desc"] else: print( - "You should run distributed optimization to get opt_info first") + "You should run distributed optimization to get opt_info first" + ) sys.exit(-1) self.instance_.barrier_all() ips = self.instance.gather_ips() - self.fleet_.init_worker(self.dist_desc_str_, ips, - self.instance_.get_node_cnt(), - self.instance._rankid) + self.fleet_.init_worker( + self.dist_desc_str_, + ips, + self.instance_.get_node_cnt(), + self.instance._rankid, + ) self.instance.barrier_worker() def init_pserver_model(self): diff --git a/python/paddle/fluid/distributed/helper.py b/python/paddle/fluid/distributed/helper.py index 08e6fca6165a2..9511ce2db629f 100644 --- a/python/paddle/fluid/distributed/helper.py +++ b/python/paddle/fluid/distributed/helper.py @@ -26,23 +26,26 @@ class FileSystem(object): fs = FileSystm() """ - def __init__(self, - fs_type="afs", - uri="afs://xx", - user=None, - passwd=None, - hadoop_bin=""): + def __init__( + self, + fs_type="afs", + uri="afs://xx", + user=None, + passwd=None, + hadoop_bin="", + ): assert user != None assert passwd != None assert hadoop_bin != None import ps_pb2 as pslib + self.fs_client = pslib.FsClientParameter() self.fs_client.uri = uri self.fs_client.user = user self.fs_client.passwd = passwd - #self.fs_client.buffer_size = 0 + # self.fs_client.buffer_size = 0 self.fs_client.hadoop_bin = hadoop_bin - #self.fs_client.afs_conf = afs_conf if not afs_conf else "" + # self.fs_client.afs_conf = afs_conf if not afs_conf else "" def get_desc(self): """ @@ -63,6 +66,7 @@ class MPIHelper(object): def __init__(self): from mpi4py import MPI + self.comm = MPI.COMM_WORLD self.MPI = MPI @@ -74,11 +78,13 @@ def get_size(self): def get_ip(self): import socket + local_ip = socket.gethostbyname(socket.gethostname()) return local_ip def get_hostname(self): import socket + return socket.gethostname() def finalize(self): diff --git a/python/paddle/fluid/distributed/node.py b/python/paddle/fluid/distributed/node.py index 90ac44ada1459..793787d0fd7b6 100644 --- a/python/paddle/fluid/distributed/node.py +++ b/python/paddle/fluid/distributed/node.py @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and import ps_pb2 as pslib + # NOTE: reduce removed in fuctools in python3 from functools import reduce class Server(object): """ - A Server basic class. + A Server basic class. """ def __init__(self): @@ -27,7 +28,7 @@ def __init__(self): class Worker(object): """ - A Worker basic class. + A Worker basic class. """ def __init__(self): @@ -36,23 +37,30 @@ def __init__(self): class DownpourServer(Server): """ - DownpourServer class is used to generate server program_desc - Args: - server: it is pslib.ServerParameter() - Examples: - server = DownpourServer() + DownpourServer class is used to generate server program_desc + Args: + server: it is pslib.ServerParameter() + Examples: + server = DownpourServer() """ def __init__(self): self.server_ = pslib.ServerParameter() self.server_.downpour_server_param.service_param.start_server_port = 0 - self.server_.downpour_server_param.service_param.server_class = "DownpourBrpcPsServer" - self.server_.downpour_server_param.service_param.client_class = "DownpourBrpcPsClient" - self.server_.downpour_server_param.service_param.service_class = "DownpourPsService" + self.server_.downpour_server_param.service_param.server_class = ( + "DownpourBrpcPsServer" + ) + self.server_.downpour_server_param.service_param.client_class = ( + "DownpourBrpcPsClient" + ) + self.server_.downpour_server_param.service_param.service_class = ( + "DownpourPsService" + ) self.server_.downpour_server_param.service_param.server_thread_num = 12 - def add_sparse_table(self, table_id, learning_rate, slot_key_vars, - slot_value_var): + def add_sparse_table( + self, table_id, learning_rate, slot_key_vars, slot_value_var + ): r""" Args: table_id(int): id of sparse params table @@ -108,8 +116,9 @@ def add_dense_table(self, table_id, learning_rate, param_var, grad_var): table.accessor.dense_sgd_param.adam.mom_decay_rate = 0.99 table.accessor.dense_sgd_param.naive.learning_rate = 0.0002 fea_dim = 0 - for param in filter(lambda x: x.name.find("embedding") == -1, - param_var): + for param in filter( + lambda x: x.name.find("embedding") == -1, param_var + ): fea_dim += reduce(lambda x, y: x * y, param.shape, 1) table.accessor.fea_dim = fea_dim @@ -122,20 +131,21 @@ def get_desc(self): class DownpourWorker(Worker): """ - DownpourWorker class is used to generate worker program_desc - Args: - window (int): push params frequency - worker: it is pslib.DownpourTrainerParameter - Examples: - worker = DownpourWorker(1) + DownpourWorker class is used to generate worker program_desc + Args: + window (int): push params frequency + worker: it is pslib.DownpourTrainerParameter + Examples: + worker = DownpourWorker(1) """ def __init__(self, window): self.window = window self.worker_ = pslib.DownpourTrainerParameter() - def add_sparse_table(self, table_id, learning_rate, slot_key_vars, - slot_value_vars): + def add_sparse_table( + self, table_id, learning_rate, slot_key_vars, slot_value_vars + ): r""" Args: table_id(int): id of sparse params table @@ -151,7 +161,8 @@ def add_sparse_table(self, table_id, learning_rate, slot_key_vars, table.slot_key.extend([var.name for var in slot_key_vars]) table.slot_value.extend([var.name for var in slot_value_vars]) table.slot_gradient.extend( - [var.name + "@GRAD" for var in slot_value_vars]) + [var.name + "@GRAD" for var in slot_value_vars] + ) def add_dense_table(self, table_id, learning_rate, param_vars, grad_vars): r""" @@ -167,11 +178,16 @@ def add_dense_table(self, table_id, learning_rate, param_vars, grad_vars): table = self.worker_.dense_table.add() table.table_id = table_id table.dense_variable_name.extend( - filter(lambda x: x.find("embedding") == -1, - [p.name for p in param_vars])) + filter( + lambda x: x.find("embedding") == -1, + [p.name for p in param_vars], + ) + ) table.dense_gradient_variable_name.extend( - filter(lambda x: x.find("embedding") == -1, - [g.name for g in grad_vars])) + filter( + lambda x: x.find("embedding") == -1, [g.name for g in grad_vars] + ) + ) def get_desc(self): """ diff --git a/python/paddle/fluid/distributed/ps_instance.py b/python/paddle/fluid/distributed/ps_instance.py index 6b19d7ca62e40..370e1b1964724 100644 --- a/python/paddle/fluid/distributed/ps_instance.py +++ b/python/paddle/fluid/distributed/ps_instance.py @@ -16,12 +16,12 @@ class PaddlePSInstance(object): """ - PaddlePSInstance class is used to generate A instance of server or worker - Args: - server_worker_mode: is a value 0 or 1, default is 1 - proc_per_node: process per node, default is 2 - Examples: - instance = PaddlePSInstance(1, 2) + PaddlePSInstance class is used to generate A instance of server or worker + Args: + server_worker_mode: is a value 0 or 1, default is 1 + proc_per_node: process per node, default is 2 + Examples: + instance = PaddlePSInstance(1, 2) """ def __init__(self, server_worker_mode=1, proc_per_node=2): @@ -35,7 +35,7 @@ def __init__(self, server_worker_mode=1, proc_per_node=2): self._worker_num = self._nodes * self._proc_per_node / 2 self._server_num = self._nodes * self._proc_per_node / 2 self._total_server_worker = self._worker_num + self._server_num - self._node_type = None #IDLE=-1, WORKER=1, SERVER=0 + self._node_type = None # IDLE=-1, WORKER=1, SERVER=0 self._set_nodetype() self._comm = None self._split_comm() @@ -104,7 +104,7 @@ def is_first_worker(self): def set_ip(self, ip): """ - set server ip + set server ip """ self._ip = ip diff --git a/python/paddle/fluid/distributed/ps_pb2.py b/python/paddle/fluid/distributed/ps_pb2.py index f1262ebae12ff..8a131a2548c44 100644 --- a/python/paddle/fluid/distributed/ps_pb2.py +++ b/python/paddle/fluid/distributed/ps_pb2.py @@ -24,6 +24,7 @@ from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database from google.protobuf import descriptor_pb2 + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -34,7 +35,8 @@ syntax='proto2', serialized_pb=_b( '\n\x08ps.proto\x12\x06paddle\"\x9e\x02\n\x0bPSParameter\x12\x14\n\x0cworker_class\x18\x01 \x01(\t\x12\x14\n\x0cserver_class\x18\x02 \x01(\t\x12\x16\n\x0einstance_class\x18\x03 \x01(\t\x12-\n\x0cworker_param\x18\x65 \x01(\x0b\x32\x17.paddle.WorkerParameter\x12-\n\x0cserver_param\x18\x66 \x01(\x0b\x32\x17.paddle.ServerParameter\x12\x38\n\rtrainer_param\x18\xad\x02 \x01(\x0b\x32 .paddle.DownpourTrainerParameter\x12\x33\n\x0f\x66s_client_param\x18\xf5\x03 \x01(\x0b\x32\x19.paddle.FsClientParameter\"Q\n\x0fWorkerParameter\x12>\n\x15\x64ownpour_worker_param\x18\x01 \x01(\x0b\x32\x1f.paddle.DownpourWorkerParameter\"Q\n\x0fServerParameter\x12>\n\x15\x64ownpour_server_param\x18\x01 \x01(\x0b\x32\x1f.paddle.DownpourServerParameter\"O\n\x17\x44ownpourWorkerParameter\x12\x34\n\x14\x64ownpour_table_param\x18\x01 \x03(\x0b\x32\x16.paddle.TableParameter\"\xfd\x01\n\x18\x44ownpourTrainerParameter\x12\x30\n\x0b\x64\x65nse_table\x18\x01 \x03(\x0b\x32\x1b.paddle.DenseTableParameter\x12\x32\n\x0csparse_table\x18\x02 \x03(\x0b\x32\x1c.paddle.SparseTableParameter\x12\x1d\n\x15push_sparse_per_batch\x18\x03 \x01(\x05\x12\x1c\n\x14push_dense_per_batch\x18\x04 \x01(\x05\x12\x0f\n\x07skip_op\x18\x05 \x03(\t\x12-\n\x0eprogram_config\x18\x06 \x03(\x0b\x32\x15.paddle.ProgramConfig\"\x99\x01\n\rProgramConfig\x12\x12\n\nprogram_id\x18\x01 \x02(\t\x12\x1c\n\x14push_sparse_table_id\x18\x02 \x03(\x05\x12\x1b\n\x13push_dense_table_id\x18\x03 \x03(\x05\x12\x1c\n\x14pull_sparse_table_id\x18\x04 \x03(\x05\x12\x1b\n\x13pull_dense_table_id\x18\x05 \x03(\x05\"{\n\x13\x44\x65nseTableParameter\x12\x10\n\x08table_id\x18\x01 \x01(\x05\x12\x1b\n\x13\x64\x65nse_variable_name\x18\x02 \x03(\t\x12$\n\x1c\x64\x65nse_gradient_variable_name\x18\x03 \x03(\t\x12\x0f\n\x07\x66\x65\x61_dim\x18\x04 \x01(\x05\"z\n\x14SparseTableParameter\x12\x10\n\x08table_id\x18\x01 \x01(\x05\x12\x13\n\x0b\x66\x65\x61ture_dim\x18\x02 \x01(\x05\x12\x10\n\x08slot_key\x18\x03 \x03(\t\x12\x12\n\nslot_value\x18\x04 \x03(\t\x12\x15\n\rslot_gradient\x18\x05 \x03(\t\"\x86\x01\n\x17\x44ownpourServerParameter\x12\x34\n\x14\x64ownpour_table_param\x18\x01 \x03(\x0b\x32\x16.paddle.TableParameter\x12\x35\n\rservice_param\x18\x02 \x01(\x0b\x32\x1e.paddle.ServerServiceParameter\"\xd7\x01\n\x16ServerServiceParameter\x12*\n\x0cserver_class\x18\x01 \x01(\t:\x14\x44ownpourBrpcPsServer\x12*\n\x0c\x63lient_class\x18\x02 \x01(\t:\x14\x44ownpourBrpcPsClient\x12(\n\rservice_class\x18\x03 \x01(\t:\x11\x44ownpourPsService\x12\x1c\n\x11start_server_port\x18\x04 \x01(\r:\x01\x30\x12\x1d\n\x11server_thread_num\x18\x05 \x01(\r:\x02\x31\x32\"\xbf\x01\n\x0eTableParameter\x12\x10\n\x08table_id\x18\x01 \x01(\x04\x12\x13\n\x0btable_class\x18\x02 \x01(\t\x12\x12\n\nshared_num\x18\x03 \x01(\x04\x12\x30\n\x08\x61\x63\x63\x65ssor\x18\x04 \x01(\x0b\x32\x1e.paddle.TableAccessorParameter\x12\x1f\n\x04type\x18\x05 \x01(\x0e\x32\x11.paddle.TableType\x12\x1f\n\x10\x63ompress_in_save\x18\x06 \x01(\x08:\x05\x66\x61lse\"\xf1\x02\n\x16TableAccessorParameter\x12\x16\n\x0e\x61\x63\x63\x65ssor_class\x18\x01 \x01(\t\x12\x38\n\x10sparse_sgd_param\x18\x02 \x01(\x0b\x32\x1e.paddle.SparseSGDRuleParameter\x12\x36\n\x0f\x64\x65nse_sgd_param\x18\x03 \x01(\x0b\x32\x1d.paddle.DenseSGDRuleParameter\x12\x0f\n\x07\x66\x65\x61_dim\x18\x04 \x01(\r\x12\x12\n\nembedx_dim\x18\x05 \x01(\r\x12\x18\n\x10\x65mbedx_threshold\x18\x06 \x01(\r\x12G\n\x17\x64ownpour_accessor_param\x18\x07 \x01(\x0b\x32&.paddle.DownpourTableAccessorParameter\x12\x45\n\x19table_accessor_save_param\x18\x08 \x03(\x0b\x32\".paddle.TableAccessorSaveParameter\"\xce\x01\n\x1e\x44ownpourTableAccessorParameter\x12\x14\n\x0cnonclk_coeff\x18\x01 \x01(\x02\x12\x13\n\x0b\x63lick_coeff\x18\x02 \x01(\x02\x12\x16\n\x0e\x62\x61se_threshold\x18\x03 \x01(\x02\x12\x17\n\x0f\x64\x65lta_threshold\x18\x04 \x01(\x02\x12\x17\n\x0f\x64\x65lta_keep_days\x18\x05 \x01(\x02\x12\x1d\n\x15show_click_decay_rate\x18\x06 \x01(\x02\x12\x18\n\x10\x64\x65lete_threshold\x18\x07 \x01(\x02\"S\n\x1aTableAccessorSaveParameter\x12\r\n\x05param\x18\x01 \x01(\r\x12\x11\n\tconverter\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65\x63onverter\x18\x03 \x01(\t\"e\n\x10PsRequestMessage\x12\x0e\n\x06\x63md_id\x18\x01 \x02(\r\x12\x10\n\x08table_id\x18\x02 \x01(\r\x12\x0e\n\x06params\x18\x03 \x03(\x0c\x12\x11\n\tclient_id\x18\x04 \x01(\x05\x12\x0c\n\x04\x64\x61ta\x18\x05 \x01(\x0c\"w\n\x16SparseSGDRuleParameter\x12\x15\n\rlearning_rate\x18\x01 \x01(\x01\x12\x15\n\rinitial_g2sum\x18\x02 \x01(\x01\x12\x18\n\rinitial_range\x18\x03 \x01(\x01:\x01\x30\x12\x15\n\rweight_bounds\x18\x04 \x03(\x02\"\xe1\x01\n\x15\x44\x65nseSGDRuleParameter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\x04\x61\x64\x61m\x18\x02 \x01(\x0b\x32\x18.paddle.AdamSGDParameter\x12(\n\x05naive\x18\x03 \x01(\x0b\x32\x19.paddle.NaiveSGDParameter\x12,\n\x07summary\x18\x04 \x01(\x0b\x32\x1b.paddle.SummarySGDParameter\x12:\n\x0emoving_average\x18\x05 \x01(\x0b\x32\".paddle.MovingAverageRuleParameter\"\x86\x01\n\x10\x41\x64\x61mSGDParameter\x12\x15\n\rlearning_rate\x18\x01 \x01(\x01\x12\x16\n\x0e\x61vg_decay_rate\x18\x02 \x01(\x01\x12\x16\n\x0e\x61\x64\x61_decay_rate\x18\x03 \x01(\x01\x12\x13\n\x0b\x61\x64\x61_epsilon\x18\x04 \x01(\x01\x12\x16\n\x0emom_decay_rate\x18\x05 \x01(\x01\"B\n\x11NaiveSGDParameter\x12\x15\n\rlearning_rate\x18\x01 \x01(\x01\x12\x16\n\x0e\x61vg_decay_rate\x18\x02 \x01(\x01\";\n\x13SummarySGDParameter\x12$\n\x12summary_decay_rate\x18\x01 \x01(\x01:\x08\x30.999999\".\n\x1aMovingAverageRuleParameter\x12\x10\n\x08momentum\x18\x01 \x01(\x01\"I\n\x11PsResponseMessage\x12\x13\n\x08\x65rr_code\x18\x01 \x02(\x05:\x01\x30\x12\x11\n\x07\x65rr_msg\x18\x02 \x02(\t:\x00\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\xd5\x01\n\x11\x46sClientParameter\x12:\n\x07\x66s_type\x18\x01 \x01(\x0e\x32#.paddle.FsClientParameter.FsApiType:\x04HDFS\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12\x0c\n\x04user\x18\x03 \x01(\t\x12\x0e\n\x06passwd\x18\x04 \x01(\t\x12\x13\n\x0b\x62uffer_size\x18\x05 \x01(\x05\x12\x12\n\nhadoop_bin\x18\x33 \x01(\t\x12\x10\n\x08\x61\x66s_conf\x18\x65 \x01(\t\"\x1e\n\tFsApiType\x12\x08\n\x04HDFS\x10\x00\x12\x07\n\x03\x41\x46S\x10\x01*4\n\tTableType\x12\x13\n\x0fPS_SPARSE_TABLE\x10\x00\x12\x12\n\x0ePS_DENSE_TABLE\x10\x01*\xbd\x02\n\x07PsCmdID\x12\x17\n\x13PS_PULL_DENSE_TABLE\x10\x00\x12\x17\n\x13PS_PUSH_DENSE_TABLE\x10\x01\x12\x18\n\x14PS_PULL_SPARSE_TABLE\x10\x02\x12\x18\n\x14PS_PUSH_SPARSE_TABLE\x10\x03\x12\x13\n\x0fPS_SHRINK_TABLE\x10\x04\x12\x15\n\x11PS_SAVE_ONE_TABLE\x10\x05\x12\x15\n\x11PS_SAVE_ALL_TABLE\x10\x06\x12\x15\n\x11PS_LOAD_ONE_TABLE\x10\x07\x12\x15\n\x11PS_LOAD_ALL_TABLE\x10\x08\x12\x16\n\x12PS_CLEAR_ONE_TABLE\x10\t\x12\x16\n\x12PS_CLEAR_ALL_TABLE\x10\n\x12\x17\n\x13PS_PUSH_DENSE_PARAM\x10\x0b\x12\x12\n\x0ePS_STOP_SERVER\x10\x0c\x32K\n\tPsService\x12>\n\x07service\x12\x18.paddle.PsRequestMessage\x1a\x19.paddle.PsResponseMessageB\x03\x80\x01\x01' - )) + ), +) _sym_db.RegisterFileDescriptor(DESCRIPTOR) _TABLETYPE = _descriptor.EnumDescriptor( @@ -43,16 +45,12 @@ filename=None, file=DESCRIPTOR, values=[ - _descriptor.EnumValueDescriptor(name='PS_SPARSE_TABLE', - index=0, - number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_DENSE_TABLE', - index=1, - number=1, - options=None, - type=None), + _descriptor.EnumValueDescriptor( + name='PS_SPARSE_TABLE', index=0, number=0, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_DENSE_TABLE', index=1, number=1, options=None, type=None + ), ], containing_type=None, options=None, @@ -68,71 +66,73 @@ filename=None, file=DESCRIPTOR, values=[ - _descriptor.EnumValueDescriptor(name='PS_PULL_DENSE_TABLE', - index=0, - number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PUSH_DENSE_TABLE', - index=1, - number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PULL_SPARSE_TABLE', - index=2, - number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PUSH_SPARSE_TABLE', - index=3, - number=3, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_SHRINK_TABLE', - index=4, - number=4, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_SAVE_ONE_TABLE', - index=5, - number=5, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_SAVE_ALL_TABLE', - index=6, - number=6, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_LOAD_ONE_TABLE', - index=7, - number=7, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_LOAD_ALL_TABLE', - index=8, - number=8, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_CLEAR_ONE_TABLE', - index=9, - number=9, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_CLEAR_ALL_TABLE', - index=10, - number=10, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PUSH_DENSE_PARAM', - index=11, - number=11, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_STOP_SERVER', - index=12, - number=12, - options=None, - type=None), + _descriptor.EnumValueDescriptor( + name='PS_PULL_DENSE_TABLE', + index=0, + number=0, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PUSH_DENSE_TABLE', + index=1, + number=1, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PULL_SPARSE_TABLE', + index=2, + number=2, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PUSH_SPARSE_TABLE', + index=3, + number=3, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_SHRINK_TABLE', index=4, number=4, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_SAVE_ONE_TABLE', index=5, number=5, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_SAVE_ALL_TABLE', index=6, number=6, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_LOAD_ONE_TABLE', index=7, number=7, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_LOAD_ALL_TABLE', index=8, number=8, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_CLEAR_ONE_TABLE', + index=9, + number=9, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_CLEAR_ALL_TABLE', + index=10, + number=10, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PUSH_DENSE_PARAM', + index=11, + number=11, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_STOP_SERVER', index=12, number=12, options=None, type=None + ), ], containing_type=None, options=None, @@ -164,16 +164,12 @@ filename=None, file=DESCRIPTOR, values=[ - _descriptor.EnumValueDescriptor(name='HDFS', - index=0, - number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='AFS', - index=1, - number=1, - options=None, - type=None), + _descriptor.EnumValueDescriptor( + name='HDFS', index=0, number=0, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='AFS', index=1, number=1, options=None, type=None + ), ], containing_type=None, options=None, @@ -189,36 +185,40 @@ file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='worker_class', - full_name='paddle.PSParameter.worker_class', - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='server_class', - full_name='paddle.PSParameter.server_class', - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='worker_class', + full_name='paddle.PSParameter.worker_class', + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='server_class', + full_name='paddle.PSParameter.server_class', + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='instance_class', full_name='paddle.PSParameter.instance_class', @@ -234,37 +234,42 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='worker_param', - full_name='paddle.PSParameter.worker_param', - index=3, - number=101, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='server_param', - full_name='paddle.PSParameter.server_param', - index=4, - number=102, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='worker_param', + full_name='paddle.PSParameter.worker_param', + index=3, + number=101, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='server_param', + full_name='paddle.PSParameter.server_param', + index=4, + number=102, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='trainer_param', full_name='paddle.PSParameter.trainer_param', @@ -280,7 +285,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='fs_client_param', full_name='paddle.PSParameter.fs_client_param', @@ -296,7 +302,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -332,7 +339,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -368,7 +376,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -404,7 +413,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -440,7 +450,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='sparse_table', full_name='paddle.DownpourTrainerParameter.sparse_table', @@ -456,7 +467,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='push_sparse_per_batch', full_name='paddle.DownpourTrainerParameter.push_sparse_per_batch', @@ -472,7 +484,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='push_dense_per_batch', full_name='paddle.DownpourTrainerParameter.push_dense_per_batch', @@ -488,7 +501,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='skip_op', full_name='paddle.DownpourTrainerParameter.skip_op', @@ -504,7 +518,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='program_config', full_name='paddle.DownpourTrainerParameter.program_config', @@ -520,7 +535,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -541,21 +557,23 @@ file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='program_id', - full_name='paddle.ProgramConfig.program_id', - index=0, - number=1, - type=9, - cpp_type=9, - label=2, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='program_id', + full_name='paddle.ProgramConfig.program_id', + index=0, + number=1, + type=9, + cpp_type=9, + label=2, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='push_sparse_table_id', full_name='paddle.ProgramConfig.push_sparse_table_id', @@ -571,7 +589,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='push_dense_table_id', full_name='paddle.ProgramConfig.push_dense_table_id', @@ -587,7 +606,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='pull_sparse_table_id', full_name='paddle.ProgramConfig.pull_sparse_table_id', @@ -603,7 +623,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='pull_dense_table_id', full_name='paddle.ProgramConfig.pull_dense_table_id', @@ -619,7 +640,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -655,7 +677,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='dense_variable_name', full_name='paddle.DenseTableParameter.dense_variable_name', @@ -671,7 +694,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='dense_gradient_variable_name', full_name='paddle.DenseTableParameter.dense_gradient_variable_name', @@ -687,7 +711,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='fea_dim', full_name='paddle.DenseTableParameter.fea_dim', @@ -703,7 +728,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -739,7 +765,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='feature_dim', full_name='paddle.SparseTableParameter.feature_dim', @@ -755,7 +782,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='slot_key', full_name='paddle.SparseTableParameter.slot_key', @@ -771,7 +799,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='slot_value', full_name='paddle.SparseTableParameter.slot_value', @@ -787,7 +816,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='slot_gradient', full_name='paddle.SparseTableParameter.slot_gradient', @@ -803,7 +833,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -839,7 +870,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='service_param', full_name='paddle.DownpourServerParameter.service_param', @@ -855,7 +887,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -891,7 +924,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='client_class', full_name='paddle.ServerServiceParameter.client_class', @@ -907,7 +941,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='service_class', full_name='paddle.ServerServiceParameter.service_class', @@ -923,7 +958,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='start_server_port', full_name='paddle.ServerServiceParameter.start_server_port', @@ -939,7 +975,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='server_thread_num', full_name='paddle.ServerServiceParameter.server_thread_num', @@ -955,7 +992,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -976,21 +1014,23 @@ file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='table_id', - full_name='paddle.TableParameter.table_id', - index=0, - number=1, - type=4, - cpp_type=4, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='table_id', + full_name='paddle.TableParameter.table_id', + index=0, + number=1, + type=4, + cpp_type=4, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='table_class', full_name='paddle.TableParameter.table_class', @@ -1006,7 +1046,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='shared_num', full_name='paddle.TableParameter.shared_num', @@ -1022,37 +1063,42 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='accessor', - full_name='paddle.TableParameter.accessor', - index=3, - number=4, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='type', - full_name='paddle.TableParameter.type', - index=4, - number=5, - type=14, - cpp_type=8, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='accessor', + full_name='paddle.TableParameter.accessor', + index=3, + number=4, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='type', + full_name='paddle.TableParameter.type', + index=4, + number=5, + type=14, + cpp_type=8, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='compress_in_save', full_name='paddle.TableParameter.compress_in_save', @@ -1068,7 +1114,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1104,7 +1151,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='sparse_sgd_param', full_name='paddle.TableAccessorParameter.sparse_sgd_param', @@ -1120,7 +1168,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='dense_sgd_param', full_name='paddle.TableAccessorParameter.dense_sgd_param', @@ -1136,7 +1185,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='fea_dim', full_name='paddle.TableAccessorParameter.fea_dim', @@ -1152,7 +1202,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='embedx_dim', full_name='paddle.TableAccessorParameter.embedx_dim', @@ -1168,7 +1219,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='embedx_threshold', full_name='paddle.TableAccessorParameter.embedx_threshold', @@ -1184,7 +1236,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='downpour_accessor_param', full_name='paddle.TableAccessorParameter.downpour_accessor_param', @@ -1200,7 +1253,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='table_accessor_save_param', full_name='paddle.TableAccessorParameter.table_accessor_save_param', @@ -1216,7 +1270,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1252,7 +1307,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='click_coeff', full_name='paddle.DownpourTableAccessorParameter.click_coeff', @@ -1268,7 +1324,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='base_threshold', full_name='paddle.DownpourTableAccessorParameter.base_threshold', @@ -1284,7 +1341,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='delta_threshold', full_name='paddle.DownpourTableAccessorParameter.delta_threshold', @@ -1300,7 +1358,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='delta_keep_days', full_name='paddle.DownpourTableAccessorParameter.delta_keep_days', @@ -1316,11 +1375,11 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='show_click_decay_rate', - full_name= - 'paddle.DownpourTableAccessorParameter.show_click_decay_rate', + full_name='paddle.DownpourTableAccessorParameter.show_click_decay_rate', index=5, number=6, type=2, @@ -1333,7 +1392,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='delete_threshold', full_name='paddle.DownpourTableAccessorParameter.delete_threshold', @@ -1349,7 +1409,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1385,7 +1446,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='converter', full_name='paddle.TableAccessorSaveParameter.converter', @@ -1401,7 +1463,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='deconverter', full_name='paddle.TableAccessorSaveParameter.deconverter', @@ -1417,7 +1480,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1438,21 +1502,23 @@ file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='cmd_id', - full_name='paddle.PsRequestMessage.cmd_id', - index=0, - number=1, - type=13, - cpp_type=3, - label=2, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='cmd_id', + full_name='paddle.PsRequestMessage.cmd_id', + index=0, + number=1, + type=13, + cpp_type=3, + label=2, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='table_id', full_name='paddle.PsRequestMessage.table_id', @@ -1468,22 +1534,25 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='params', - full_name='paddle.PsRequestMessage.params', - index=2, - number=3, - type=12, - cpp_type=9, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='params', + full_name='paddle.PsRequestMessage.params', + index=2, + number=3, + type=12, + cpp_type=9, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='client_id', full_name='paddle.PsRequestMessage.client_id', @@ -1499,22 +1568,25 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='data', - full_name='paddle.PsRequestMessage.data', - index=4, - number=5, - type=12, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b(""), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='data', + full_name='paddle.PsRequestMessage.data', + index=4, + number=5, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), ], extensions=[], nested_types=[], @@ -1550,7 +1622,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_g2sum', full_name='paddle.SparseSGDRuleParameter.initial_g2sum', @@ -1566,7 +1639,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_range', full_name='paddle.SparseSGDRuleParameter.initial_range', @@ -1582,7 +1656,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='weight_bounds', full_name='paddle.SparseSGDRuleParameter.weight_bounds', @@ -1598,7 +1673,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1634,7 +1710,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='adam', full_name='paddle.DenseSGDRuleParameter.adam', @@ -1650,7 +1727,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='naive', full_name='paddle.DenseSGDRuleParameter.naive', @@ -1666,7 +1744,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='summary', full_name='paddle.DenseSGDRuleParameter.summary', @@ -1682,7 +1761,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='moving_average', full_name='paddle.DenseSGDRuleParameter.moving_average', @@ -1698,7 +1778,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1734,7 +1815,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='avg_decay_rate', full_name='paddle.AdamSGDParameter.avg_decay_rate', @@ -1750,7 +1832,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='ada_decay_rate', full_name='paddle.AdamSGDParameter.ada_decay_rate', @@ -1766,7 +1849,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='ada_epsilon', full_name='paddle.AdamSGDParameter.ada_epsilon', @@ -1782,7 +1866,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='mom_decay_rate', full_name='paddle.AdamSGDParameter.mom_decay_rate', @@ -1798,7 +1883,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1834,7 +1920,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='avg_decay_rate', full_name='paddle.NaiveSGDParameter.avg_decay_rate', @@ -1850,7 +1937,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1886,7 +1974,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1922,7 +2011,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1958,7 +2048,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='err_msg', full_name='paddle.PsResponseMessage.err_msg', @@ -1974,22 +2065,25 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='data', - full_name='paddle.PsResponseMessage.data', - index=2, - number=3, - type=12, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b(""), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='data', + full_name='paddle.PsResponseMessage.data', + index=2, + number=3, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), ], extensions=[], nested_types=[], @@ -2025,52 +2119,59 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='uri', - full_name='paddle.FsClientParameter.uri', - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='user', - full_name='paddle.FsClientParameter.user', - index=2, - number=3, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='passwd', - full_name='paddle.FsClientParameter.passwd', - index=3, - number=4, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='uri', + full_name='paddle.FsClientParameter.uri', + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='user', + full_name='paddle.FsClientParameter.user', + index=2, + number=3, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='passwd', + full_name='paddle.FsClientParameter.passwd', + index=3, + number=4, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='buffer_size', full_name='paddle.FsClientParameter.buffer_size', @@ -2086,7 +2187,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='hadoop_bin', full_name='paddle.FsClientParameter.hadoop_bin', @@ -2102,7 +2204,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='afs_conf', full_name='paddle.FsClientParameter.afs_conf', @@ -2118,7 +2221,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2137,75 +2241,102 @@ _PSPARAMETER.fields_by_name['worker_param'].message_type = _WORKERPARAMETER _PSPARAMETER.fields_by_name['server_param'].message_type = _SERVERPARAMETER _PSPARAMETER.fields_by_name[ - 'trainer_param'].message_type = _DOWNPOURTRAINERPARAMETER + 'trainer_param' +].message_type = _DOWNPOURTRAINERPARAMETER _PSPARAMETER.fields_by_name['fs_client_param'].message_type = _FSCLIENTPARAMETER _WORKERPARAMETER.fields_by_name[ - 'downpour_worker_param'].message_type = _DOWNPOURWORKERPARAMETER + 'downpour_worker_param' +].message_type = _DOWNPOURWORKERPARAMETER _SERVERPARAMETER.fields_by_name[ - 'downpour_server_param'].message_type = _DOWNPOURSERVERPARAMETER + 'downpour_server_param' +].message_type = _DOWNPOURSERVERPARAMETER _DOWNPOURWORKERPARAMETER.fields_by_name[ - 'downpour_table_param'].message_type = _TABLEPARAMETER + 'downpour_table_param' +].message_type = _TABLEPARAMETER _DOWNPOURTRAINERPARAMETER.fields_by_name[ - 'dense_table'].message_type = _DENSETABLEPARAMETER + 'dense_table' +].message_type = _DENSETABLEPARAMETER _DOWNPOURTRAINERPARAMETER.fields_by_name[ - 'sparse_table'].message_type = _SPARSETABLEPARAMETER + 'sparse_table' +].message_type = _SPARSETABLEPARAMETER _DOWNPOURTRAINERPARAMETER.fields_by_name[ - 'program_config'].message_type = _PROGRAMCONFIG + 'program_config' +].message_type = _PROGRAMCONFIG _DOWNPOURSERVERPARAMETER.fields_by_name[ - 'downpour_table_param'].message_type = _TABLEPARAMETER + 'downpour_table_param' +].message_type = _TABLEPARAMETER _DOWNPOURSERVERPARAMETER.fields_by_name[ - 'service_param'].message_type = _SERVERSERVICEPARAMETER + 'service_param' +].message_type = _SERVERSERVICEPARAMETER _TABLEPARAMETER.fields_by_name[ - 'accessor'].message_type = _TABLEACCESSORPARAMETER + 'accessor' +].message_type = _TABLEACCESSORPARAMETER _TABLEPARAMETER.fields_by_name['type'].enum_type = _TABLETYPE _TABLEACCESSORPARAMETER.fields_by_name[ - 'sparse_sgd_param'].message_type = _SPARSESGDRULEPARAMETER + 'sparse_sgd_param' +].message_type = _SPARSESGDRULEPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'dense_sgd_param'].message_type = _DENSESGDRULEPARAMETER + 'dense_sgd_param' +].message_type = _DENSESGDRULEPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'downpour_accessor_param'].message_type = _DOWNPOURTABLEACCESSORPARAMETER + 'downpour_accessor_param' +].message_type = _DOWNPOURTABLEACCESSORPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'table_accessor_save_param'].message_type = _TABLEACCESSORSAVEPARAMETER + 'table_accessor_save_param' +].message_type = _TABLEACCESSORSAVEPARAMETER _DENSESGDRULEPARAMETER.fields_by_name['adam'].message_type = _ADAMSGDPARAMETER _DENSESGDRULEPARAMETER.fields_by_name['naive'].message_type = _NAIVESGDPARAMETER _DENSESGDRULEPARAMETER.fields_by_name[ - 'summary'].message_type = _SUMMARYSGDPARAMETER + 'summary' +].message_type = _SUMMARYSGDPARAMETER _DENSESGDRULEPARAMETER.fields_by_name[ - 'moving_average'].message_type = _MOVINGAVERAGERULEPARAMETER + 'moving_average' +].message_type = _MOVINGAVERAGERULEPARAMETER _FSCLIENTPARAMETER.fields_by_name[ - 'fs_type'].enum_type = _FSCLIENTPARAMETER_FSAPITYPE + 'fs_type' +].enum_type = _FSCLIENTPARAMETER_FSAPITYPE _FSCLIENTPARAMETER_FSAPITYPE.containing_type = _FSCLIENTPARAMETER DESCRIPTOR.message_types_by_name['PSParameter'] = _PSPARAMETER DESCRIPTOR.message_types_by_name['WorkerParameter'] = _WORKERPARAMETER DESCRIPTOR.message_types_by_name['ServerParameter'] = _SERVERPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourWorkerParameter'] = _DOWNPOURWORKERPARAMETER + 'DownpourWorkerParameter' +] = _DOWNPOURWORKERPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourTrainerParameter'] = _DOWNPOURTRAINERPARAMETER + 'DownpourTrainerParameter' +] = _DOWNPOURTRAINERPARAMETER DESCRIPTOR.message_types_by_name['ProgramConfig'] = _PROGRAMCONFIG DESCRIPTOR.message_types_by_name['DenseTableParameter'] = _DENSETABLEPARAMETER DESCRIPTOR.message_types_by_name['SparseTableParameter'] = _SPARSETABLEPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourServerParameter'] = _DOWNPOURSERVERPARAMETER + 'DownpourServerParameter' +] = _DOWNPOURSERVERPARAMETER DESCRIPTOR.message_types_by_name[ - 'ServerServiceParameter'] = _SERVERSERVICEPARAMETER + 'ServerServiceParameter' +] = _SERVERSERVICEPARAMETER DESCRIPTOR.message_types_by_name['TableParameter'] = _TABLEPARAMETER DESCRIPTOR.message_types_by_name[ - 'TableAccessorParameter'] = _TABLEACCESSORPARAMETER + 'TableAccessorParameter' +] = _TABLEACCESSORPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourTableAccessorParameter'] = _DOWNPOURTABLEACCESSORPARAMETER + 'DownpourTableAccessorParameter' +] = _DOWNPOURTABLEACCESSORPARAMETER DESCRIPTOR.message_types_by_name[ - 'TableAccessorSaveParameter'] = _TABLEACCESSORSAVEPARAMETER + 'TableAccessorSaveParameter' +] = _TABLEACCESSORSAVEPARAMETER DESCRIPTOR.message_types_by_name['PsRequestMessage'] = _PSREQUESTMESSAGE DESCRIPTOR.message_types_by_name[ - 'SparseSGDRuleParameter'] = _SPARSESGDRULEPARAMETER + 'SparseSGDRuleParameter' +] = _SPARSESGDRULEPARAMETER DESCRIPTOR.message_types_by_name[ - 'DenseSGDRuleParameter'] = _DENSESGDRULEPARAMETER + 'DenseSGDRuleParameter' +] = _DENSESGDRULEPARAMETER DESCRIPTOR.message_types_by_name['AdamSGDParameter'] = _ADAMSGDPARAMETER DESCRIPTOR.message_types_by_name['NaiveSGDParameter'] = _NAIVESGDPARAMETER DESCRIPTOR.message_types_by_name['SummarySGDParameter'] = _SUMMARYSGDPARAMETER DESCRIPTOR.message_types_by_name[ - 'MovingAverageRuleParameter'] = _MOVINGAVERAGERULEPARAMETER + 'MovingAverageRuleParameter' +] = _MOVINGAVERAGERULEPARAMETER DESCRIPTOR.message_types_by_name['PsResponseMessage'] = _PSRESPONSEMESSAGE DESCRIPTOR.message_types_by_name['FsClientParameter'] = _FSCLIENTPARAMETER DESCRIPTOR.enum_types_by_name['TableType'] = _TABLETYPE @@ -2213,215 +2344,259 @@ PSParameter = _reflection.GeneratedProtocolMessageType( 'PSParameter', - (_message.Message, ), - dict(DESCRIPTOR=_PSPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.PSParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PSPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.PSParameter) + ), +) _sym_db.RegisterMessage(PSParameter) WorkerParameter = _reflection.GeneratedProtocolMessageType( 'WorkerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_WORKERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.WorkerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_WORKERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.WorkerParameter) + ), +) _sym_db.RegisterMessage(WorkerParameter) ServerParameter = _reflection.GeneratedProtocolMessageType( 'ServerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SERVERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.ServerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SERVERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.ServerParameter) + ), +) _sym_db.RegisterMessage(ServerParameter) DownpourWorkerParameter = _reflection.GeneratedProtocolMessageType( 'DownpourWorkerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DOWNPOURWORKERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DownpourWorkerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DOWNPOURWORKERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DownpourWorkerParameter) + ), +) _sym_db.RegisterMessage(DownpourWorkerParameter) DownpourTrainerParameter = _reflection.GeneratedProtocolMessageType( 'DownpourTrainerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DOWNPOURTRAINERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DownpourTrainerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DOWNPOURTRAINERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DownpourTrainerParameter) + ), +) _sym_db.RegisterMessage(DownpourTrainerParameter) ProgramConfig = _reflection.GeneratedProtocolMessageType( 'ProgramConfig', - (_message.Message, ), - dict(DESCRIPTOR=_PROGRAMCONFIG, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.ProgramConfig) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PROGRAMCONFIG, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.ProgramConfig) + ), +) _sym_db.RegisterMessage(ProgramConfig) DenseTableParameter = _reflection.GeneratedProtocolMessageType( 'DenseTableParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DENSETABLEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DenseTableParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DENSETABLEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DenseTableParameter) + ), +) _sym_db.RegisterMessage(DenseTableParameter) SparseTableParameter = _reflection.GeneratedProtocolMessageType( 'SparseTableParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SPARSETABLEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.SparseTableParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SPARSETABLEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.SparseTableParameter) + ), +) _sym_db.RegisterMessage(SparseTableParameter) DownpourServerParameter = _reflection.GeneratedProtocolMessageType( 'DownpourServerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DOWNPOURSERVERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DownpourServerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DOWNPOURSERVERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DownpourServerParameter) + ), +) _sym_db.RegisterMessage(DownpourServerParameter) ServerServiceParameter = _reflection.GeneratedProtocolMessageType( 'ServerServiceParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SERVERSERVICEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.ServerServiceParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SERVERSERVICEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.ServerServiceParameter) + ), +) _sym_db.RegisterMessage(ServerServiceParameter) TableParameter = _reflection.GeneratedProtocolMessageType( 'TableParameter', - (_message.Message, ), - dict(DESCRIPTOR=_TABLEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.TableParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_TABLEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.TableParameter) + ), +) _sym_db.RegisterMessage(TableParameter) TableAccessorParameter = _reflection.GeneratedProtocolMessageType( 'TableAccessorParameter', - (_message.Message, ), - dict(DESCRIPTOR=_TABLEACCESSORPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.TableAccessorParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_TABLEACCESSORPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.TableAccessorParameter) + ), +) _sym_db.RegisterMessage(TableAccessorParameter) DownpourTableAccessorParameter = _reflection.GeneratedProtocolMessageType( 'DownpourTableAccessorParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_DOWNPOURTABLEACCESSORPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.DownpourTableAccessorParameter) - )) + ), +) _sym_db.RegisterMessage(DownpourTableAccessorParameter) TableAccessorSaveParameter = _reflection.GeneratedProtocolMessageType( 'TableAccessorSaveParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_TABLEACCESSORSAVEPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.TableAccessorSaveParameter) - )) + ), +) _sym_db.RegisterMessage(TableAccessorSaveParameter) PsRequestMessage = _reflection.GeneratedProtocolMessageType( 'PsRequestMessage', - (_message.Message, ), - dict(DESCRIPTOR=_PSREQUESTMESSAGE, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.PsRequestMessage) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PSREQUESTMESSAGE, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.PsRequestMessage) + ), +) _sym_db.RegisterMessage(PsRequestMessage) SparseSGDRuleParameter = _reflection.GeneratedProtocolMessageType( 'SparseSGDRuleParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SPARSESGDRULEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.SparseSGDRuleParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SPARSESGDRULEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.SparseSGDRuleParameter) + ), +) _sym_db.RegisterMessage(SparseSGDRuleParameter) DenseSGDRuleParameter = _reflection.GeneratedProtocolMessageType( 'DenseSGDRuleParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DENSESGDRULEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DenseSGDRuleParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DENSESGDRULEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DenseSGDRuleParameter) + ), +) _sym_db.RegisterMessage(DenseSGDRuleParameter) AdamSGDParameter = _reflection.GeneratedProtocolMessageType( 'AdamSGDParameter', - (_message.Message, ), - dict(DESCRIPTOR=_ADAMSGDPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.AdamSGDParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_ADAMSGDPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.AdamSGDParameter) + ), +) _sym_db.RegisterMessage(AdamSGDParameter) NaiveSGDParameter = _reflection.GeneratedProtocolMessageType( 'NaiveSGDParameter', - (_message.Message, ), - dict(DESCRIPTOR=_NAIVESGDPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.NaiveSGDParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_NAIVESGDPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.NaiveSGDParameter) + ), +) _sym_db.RegisterMessage(NaiveSGDParameter) SummarySGDParameter = _reflection.GeneratedProtocolMessageType( 'SummarySGDParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SUMMARYSGDPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.SummarySGDParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SUMMARYSGDPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.SummarySGDParameter) + ), +) _sym_db.RegisterMessage(SummarySGDParameter) MovingAverageRuleParameter = _reflection.GeneratedProtocolMessageType( 'MovingAverageRuleParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_MOVINGAVERAGERULEPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.MovingAverageRuleParameter) - )) + ), +) _sym_db.RegisterMessage(MovingAverageRuleParameter) PsResponseMessage = _reflection.GeneratedProtocolMessageType( 'PsResponseMessage', - (_message.Message, ), - dict(DESCRIPTOR=_PSRESPONSEMESSAGE, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.PsResponseMessage) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PSRESPONSEMESSAGE, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.PsResponseMessage) + ), +) _sym_db.RegisterMessage(PsResponseMessage) FsClientParameter = _reflection.GeneratedProtocolMessageType( 'FsClientParameter', - (_message.Message, ), - dict(DESCRIPTOR=_FSCLIENTPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.FsClientParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_FSCLIENTPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.FsClientParameter) + ), +) _sym_db.RegisterMessage(FsClientParameter) DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), - _b('\200\001\001')) +DESCRIPTOR._options = _descriptor._ParseOptions( + descriptor_pb2.FileOptions(), _b('\200\001\001') +) # @@protoc_insertion_point(module_scope) diff --git a/python/paddle/fluid/dygraph/amp/auto_cast.py b/python/paddle/fluid/dygraph/amp/auto_cast.py index 682f14fab5b9a..bb1acc7c09bfc 100644 --- a/python/paddle/fluid/dygraph/amp/auto_cast.py +++ b/python/paddle/fluid/dygraph/amp/auto_cast.py @@ -12,10 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle.fluid.wrapped_decorator import signature_safe_contextmanager, wrap_decorator +from paddle.fluid.wrapped_decorator import ( + signature_safe_contextmanager, + wrap_decorator, +) from paddle.fluid import core import contextlib -from paddle.fluid.framework import Variable, _non_static_mode, OpProtoHolder, Parameter, _dygraph_tracer, dygraph_only, set_flags, get_flags +from paddle.fluid.framework import ( + Variable, + _non_static_mode, + OpProtoHolder, + Parameter, + _dygraph_tracer, + dygraph_only, + set_flags, + get_flags, +) import warnings import copy import functools @@ -103,12 +115,11 @@ def amp_state(): return _g_amp_state_ -#NOTE(zhiqiu): similar as paddle.fluid.contrib.mixed_precision.fp16_lists.AutoMixedPrecisionLists._update_list +# NOTE(zhiqiu): similar as paddle.fluid.contrib.mixed_precision.fp16_lists.AutoMixedPrecisionLists._update_list # The reason why not use AutoMixedPrecisionLists is that custom_black_varnames is not suitable for imperative mode. -def _update_list(custom_white_list, - custom_black_list, - level='O1', - dtype='float16'): +def _update_list( + custom_white_list, custom_black_list, level='O1', dtype='float16' +): """ Update black and white list according to users' custom list. """ @@ -129,8 +140,9 @@ def _update_list(custom_white_list, if custom_white_list and custom_black_list: for op_name in custom_white_list: if op_name in custom_black_list: - raise ValueError("Custom white list overlap " - "custom black list") + raise ValueError( + "Custom white list overlap " "custom black list" + ) if custom_white_list: for op_name in custom_white_list: if op_name in _black_list: @@ -190,17 +202,29 @@ def pure_fp16_initialize(models): for layer in models[idx].sublayers(include_self=True): layer._casted_by_pure_fp16 = True if (layer._dtype == 'float16') or isinstance( - layer, (paddle.nn.BatchNorm, paddle.nn.BatchNorm1D, - paddle.nn.BatchNorm2D, paddle.nn.BatchNorm3D, - paddle.nn.LayerNorm, paddle.nn.SyncBatchNorm)): + layer, + ( + paddle.nn.BatchNorm, + paddle.nn.BatchNorm1D, + paddle.nn.BatchNorm2D, + paddle.nn.BatchNorm3D, + paddle.nn.LayerNorm, + paddle.nn.SyncBatchNorm, + ), + ): continue - if isinstance(layer, (paddle.incubate.nn.FusedFeedForward, - paddle.incubate.nn.FusedMultiHeadAttention)): + if isinstance( + layer, + ( + paddle.incubate.nn.FusedFeedForward, + paddle.incubate.nn.FusedMultiHeadAttention, + ), + ): layer._amp_decorate(dtype='float16') continue - layer._to_impl(dtype='float16', - include_sublayers=False, - floating_only=True) + layer._to_impl( + dtype='float16', include_sublayers=False, floating_only=True + ) return models @@ -208,9 +232,9 @@ def pure_fp16_initialize(models): def pure_bf16_initialize(models): for idx in range(len(models)): for layer in models[idx].sublayers(include_self=True): - layer._to_impl(dtype='bfloat16', - include_sublayers=False, - floating_only=True) + layer._to_impl( + dtype='bfloat16', include_sublayers=False, floating_only=True + ) return models @@ -218,8 +242,10 @@ def check_models(models): for model in models: if not isinstance(model, paddle.nn.Layer): raise RuntimeError( - "Current train mode is pure fp16, models should be paddle.nn.Layer, but receive {}." - .format(type(model))) + "Current train mode is pure fp16, models should be paddle.nn.Layer, but receive {}.".format( + type(model) + ) + ) if isinstance(model, paddle.DataParallel): raise RuntimeError( "For distributed AMP training, you should first use paddle.amp.decorate() to decotate origin model, and then call paddle.DataParallel get distributed model." @@ -229,20 +255,25 @@ def check_models(models): def check_optimizers(optimizers): for optimizer in optimizers: if not isinstance( - optimizer, - (paddle.optimizer.Optimizer, paddle.fluid.optimizer.Optimizer)): + optimizer, + (paddle.optimizer.Optimizer, paddle.fluid.optimizer.Optimizer), + ): raise RuntimeError( - "Current train mode is pure fp16, optimizers should be paddle.optimizer.Optimizer or paddle.fluid.optimizer.Optimizer, but receive {}." - .format(type(optimizer))) + "Current train mode is pure fp16, optimizers should be paddle.optimizer.Optimizer or paddle.fluid.optimizer.Optimizer, but receive {}.".format( + type(optimizer) + ) + ) @signature_safe_contextmanager @dygraph_only -def amp_guard(enable=True, - custom_white_list=None, - custom_black_list=None, - level='O1', - dtype='float16'): +def amp_guard( + enable=True, + custom_white_list=None, + custom_black_list=None, + level='O1', + dtype='float16', +): """ :api_attr: imperative @@ -306,19 +337,23 @@ def amp_guard(enable=True, tracer = _dygraph_tracer() if not tracer: raise ValueError( - "current_tracer is None, maybe it is not in imperative mode.") + "current_tracer is None, maybe it is not in imperative mode." + ) # check device_type: # NOTE: Now, amp only support gpu for float16 and bfloat16, xpu for float16, mlu for float16, npu for float16. # Maybe we will support cpu for bfloat16. - if enable and not (tracer._expected_place.is_gpu_place() - or tracer._expected_place.is_xpu_place() - or tracer._expected_place.is_mlu_place() - or tracer._expected_place.is_npu_place() - or tracer._expected_place.is_custom_place()): + if enable and not ( + tracer._expected_place.is_gpu_place() + or tracer._expected_place.is_xpu_place() + or tracer._expected_place.is_mlu_place() + or tracer._expected_place.is_npu_place() + or tracer._expected_place.is_custom_place() + ): warnings.warn( 'amp_guard can only be enabled on CUDAPlace, XPUPlace, MLUPlace, NPUPlace, and CustomPlace, current place is %s, so it makes no effect.' - % tracer._expected_place) + % tracer._expected_place + ) enable = False # For npu: if tracer._expected_place.is_npu_place() and (dtype == 'bfloat16'): @@ -343,14 +378,20 @@ def amp_guard(enable=True, prop = paddle.device.cuda.get_device_capability() warnings.warn( "For float16, amp only support NVIDIA GPU with Compute Capability 7.0 or higher, current GPU is: %s, with Compute Capability: %d.%d." - % (paddle.device.cuda.get_device_name(), prop[0], prop[1])) + % (paddle.device.cuda.get_device_name(), prop[0], prop[1]) + ) elif (dtype == 'bfloat16') and not _is_gpu_bfloat16_supported(): prop = paddle.device.cuda.get_device_capability() cuda_version = paddle.version.cuda() warnings.warn( "For bfloat16, amp only support NVIDIA GPU with Compute Capability 8.0 or higher and CUDA Version 11.0 or higher, current GPU is: %s, with Compute Capability: %d.%d, current CUDA Version is: %s." - % (paddle.device.cuda.get_device_name(), prop[0], prop[1], - cuda_version)) + % ( + paddle.device.cuda.get_device_name(), + prop[0], + prop[1], + cuda_version, + ) + ) amp_dtype = dtype @@ -381,8 +422,9 @@ def amp_guard(enable=True, _black_list = BF16_BLACK_LIST if custom_white_list or custom_black_list: - _white_list, _black_list = _update_list(custom_white_list, - custom_black_list, level, dtype) + _white_list, _black_list = _update_list( + custom_white_list, custom_black_list, level, dtype + ) if not enable: amp_level = AMP_LEVEL.O0 @@ -422,7 +464,6 @@ def amp_guard(enable=True, class StateDictHook(object): - def __init__(self, save_dtype): self._save_dtype = save_dtype @@ -437,12 +478,14 @@ def __call__(self, state_dict): @dygraph_only -def amp_decorate(models, - optimizers=None, - level='O1', - dtype='float16', - master_weight=None, - save_dtype=None): +def amp_decorate( + models, + optimizers=None, + level='O1', + dtype='float16', + master_weight=None, + save_dtype=None, +): """ Decorate models and optimizers for auto-mixed-precision. When level is O1(amp), the decorate will do nothing. When level is O2(pure fp16), the decorate will cast all parameters of models to FP16, except BatchNorm and LayerNorm. @@ -527,7 +570,8 @@ def amp_decorate(models, models_is_list = True else: raise TypeError( - "models must be either a single model or a list of models.") + "models must be either a single model or a list of models." + ) if dtype == 'float16': models = pure_fp16_initialize(models=models) elif dtype == 'bfloat16': @@ -539,8 +583,9 @@ def amp_decorate(models, # check optimizers optimizers_is_list = False if isinstance( - optimizers, - (paddle.optimizer.Optimizer, paddle.fluid.optimizer.Optimizer)): + optimizers, + (paddle.optimizer.Optimizer, paddle.fluid.optimizer.Optimizer), + ): optimizers_is_list = False optimizers = [optimizers] check_optimizers(optimizers) @@ -563,7 +608,8 @@ def amp_decorate(models, if not (save_dtype in ['float16', 'bfloat16', 'float32', 'float64']): raise ValueError( "save_dtype can only be float16 float32 or float64, but your input save_dtype is %s." - % save_dtype) + % save_dtype + ) for idx in range(len(models)): for layer in models[idx].sublayers(include_self=True): layer.register_state_dict_hook(StateDictHook(save_dtype)) diff --git a/python/paddle/fluid/dygraph/amp/loss_scaler.py b/python/paddle/fluid/dygraph/amp/loss_scaler.py index ce2e766d436dd..6ab153e4a2e9a 100644 --- a/python/paddle/fluid/dygraph/amp/loss_scaler.py +++ b/python/paddle/fluid/dygraph/amp/loss_scaler.py @@ -14,7 +14,11 @@ from paddle.fluid import core from paddle.fluid.dygraph import to_variable -from paddle.fluid.framework import _varbase_creator, _dygraph_tracer, dygraph_only +from paddle.fluid.framework import ( + _varbase_creator, + _dygraph_tracer, + dygraph_only, +) from paddle.fluid.data_feeder import check_type from ...wrapped_decorator import signature_safe_contextmanager, wrap_decorator import warnings @@ -89,28 +93,34 @@ class AmpScaler(object): """ @dygraph_only - def __init__(self, - enable=True, - init_loss_scaling=2.**15, - incr_ratio=2.0, - decr_ratio=0.5, - incr_every_n_steps=1000, - decr_every_n_nan_or_inf=1, - use_dynamic_loss_scaling=True): + def __init__( + self, + enable=True, + init_loss_scaling=2.0**15, + incr_ratio=2.0, + decr_ratio=0.5, + incr_every_n_steps=1000, + decr_every_n_nan_or_inf=1, + use_dynamic_loss_scaling=True, + ): tracer = _dygraph_tracer() if not tracer: raise ValueError( - "current_tracer is None, maybe it is not in imperative mode.") + "current_tracer is None, maybe it is not in imperative mode." + ) - if enable and not (tracer._expected_place.is_gpu_place() - or tracer._expected_place.is_xpu_place() - or tracer._expected_place.is_mlu_place() - or tracer._expected_place.is_npu_place() - or tracer._expected_place.is_custom_place()): + if enable and not ( + tracer._expected_place.is_gpu_place() + or tracer._expected_place.is_xpu_place() + or tracer._expected_place.is_mlu_place() + or tracer._expected_place.is_npu_place() + or tracer._expected_place.is_custom_place() + ): warnings.warn( 'AmpScaler can only be enabled on CUDAPlace, XPUPlace, MLUPlace, NPUPlace and CustomPlace, current place is %s, so it makes no effect.' - % tracer._expected_place) + % tracer._expected_place + ) enable = False self._enable = enable @@ -130,13 +140,17 @@ def __init__(self, self._found_inf = to_variable(np.array([0]).astype(np.bool_)) self._temp_found_inf_fp16 = to_variable( - np.array([0]).astype(np.bool_)) + np.array([0]).astype(np.bool_) + ) self._temp_found_inf_bf16 = to_variable( - np.array([0]).astype(np.bool_)) + np.array([0]).astype(np.bool_) + ) self._temp_found_inf_fp32 = to_variable( - np.array([0]).astype(np.bool_)) + np.array([0]).astype(np.bool_) + ) self._scale = to_variable( - np.array([self._init_loss_scaling]).astype(np.float32)) + np.array([self._init_loss_scaling]).astype(np.float32) + ) self._cache_founf_inf = None self._optimizer_states = defaultdict(_refresh_optimizer_state) @@ -260,7 +274,8 @@ def _unscale(self, optimizer): raise RuntimeError("unscale_() is being called after step().") if getattr(optimizer, '_param_groups', None) and isinstance( - optimizer._param_groups[0], dict): + optimizer._param_groups[0], dict + ): param_grads = [] param_grads_fp16 = [] param_grads_bf16 = [] @@ -269,29 +284,37 @@ def _unscale(self, optimizer): for param in group['params']: if param._grad_ivar() is not None: param_grads.append(param._grad_ivar()) - if param._grad_ivar( - ).dtype == core.VarDesc.VarType.FP16: + if ( + param._grad_ivar().dtype + == core.VarDesc.VarType.FP16 + ): param_grads_fp16.append(param._grad_ivar()) - elif param._grad_ivar( - ).dtype == core.VarDesc.VarType.BF16: + elif ( + param._grad_ivar().dtype + == core.VarDesc.VarType.BF16 + ): param_grads_bf16.append(param._grad_ivar()) else: param_grads_fp32.append(param._grad_ivar()) else: param_grads = [ - param._grad_ivar() for param in optimizer._parameter_list + param._grad_ivar() + for param in optimizer._parameter_list if param._grad_ivar() is not None ] param_grads_fp16 = [ - param for param in param_grads + param + for param in param_grads if param.dtype == core.VarDesc.VarType.FP16 ] param_grads_bf16 = [ - param for param in param_grads + param + for param in param_grads if param.dtype == core.VarDesc.VarType.BF16 ] param_grads_fp32 = [ - param for param in param_grads + param + for param in param_grads if param.dtype == core.VarDesc.VarType.FP32 ] if core.is_compiled_with_npu(): @@ -300,31 +323,56 @@ def _unscale(self, optimizer): if len(param_grads_fp16): _legacy_C_ops.check_finite_and_unscale( - param_grads_fp16, self._scale, float_status, - param_grads_fp16, self._temp_found_inf_fp16) + param_grads_fp16, + self._scale, + float_status, + param_grads_fp16, + self._temp_found_inf_fp16, + ) if len(param_grads_bf16): _legacy_C_ops.check_finite_and_unscale( - param_grads_bf16, self._scale, float_status, - param_grads_bf16, self._temp_found_inf_bf16) + param_grads_bf16, + self._scale, + float_status, + param_grads_bf16, + self._temp_found_inf_bf16, + ) if len(param_grads_fp32): _legacy_C_ops.check_finite_and_unscale( - param_grads_fp32, self._scale, float_status, - param_grads_fp32, self._temp_found_inf_fp32) + param_grads_fp32, + self._scale, + float_status, + param_grads_fp32, + self._temp_found_inf_fp32, + ) else: if len(param_grads_fp16): _legacy_C_ops.check_finite_and_unscale( - param_grads_fp16, self._scale, param_grads_fp16, - self._temp_found_inf_fp16) + param_grads_fp16, + self._scale, + param_grads_fp16, + self._temp_found_inf_fp16, + ) if len(param_grads_bf16): _legacy_C_ops.check_finite_and_unscale( - param_grads_bf16, self._scale, param_grads_bf16, - self._temp_found_inf_bf16) + param_grads_bf16, + self._scale, + param_grads_bf16, + self._temp_found_inf_bf16, + ) if len(param_grads_fp32): _legacy_C_ops.check_finite_and_unscale( - param_grads_fp32, self._scale, param_grads_fp32, - self._temp_found_inf_fp32) - - self._found_inf = self._temp_found_inf_fp16 or self._temp_found_inf_bf16 or self._temp_found_inf_fp32 + param_grads_fp32, + self._scale, + param_grads_fp32, + self._temp_found_inf_fp32, + ) + + self._found_inf = ( + self._temp_found_inf_fp16 + or self._temp_found_inf_bf16 + or self._temp_found_inf_fp32 + ) optimizer_state["state"] = OptimizerState.UNSCALED @@ -340,9 +388,12 @@ def _update(self): self._decr_count = self._decr_count + 1 if self._decr_count == self._decr_every_n_nan_or_inf: print( - 'Found inf or nan, current scale is: {}, decrease to: {}*{}' - .format(float(self._scale), float(self._scale), - float(self._decr_ratio))) + 'Found inf or nan, current scale is: {}, decrease to: {}*{}'.format( + float(self._scale), + float(self._scale), + float(self._decr_ratio), + ) + ) self._scale = self._scale * self._decr_ratio self._decr_count = 0 else: @@ -390,7 +441,8 @@ def set_init_loss_scaling(self, new_init_loss_scaling): """ self._init_loss_scaling = new_init_loss_scaling self._scale = to_variable( - np.array([self._init_loss_scaling]).astype(np.float32)) + np.array([self._init_loss_scaling]).astype(np.float32) + ) def get_incr_ratio(self): """ @@ -481,16 +533,20 @@ def state_dict(self): decr_count(int): The number of recent consecutive skipped steps. use_dynamic_loss_scaling(bool): Whether to use dynamic loss scaling. If False, fixed loss_scaling is used. If True, the loss scaling is updated dynamicly. Default is True. """ - return { - "scale": self._scale.numpy(), - "incr_ratio": self._incr_ratio, - "decr_ratio": self._decr_ratio, - "incr_every_n_steps": self._incr_every_n_steps, - "decr_every_n_nan_or_inf": self._decr_every_n_nan_or_inf, - "incr_count": self._incr_count, - "decr_count": self._decr_count, - "use_dynamic_loss_scaling": self._use_dynamic_loss_scaling - } if self._enable else {} + return ( + { + "scale": self._scale.numpy(), + "incr_ratio": self._incr_ratio, + "decr_ratio": self._decr_ratio, + "incr_every_n_steps": self._incr_every_n_steps, + "decr_every_n_nan_or_inf": self._decr_every_n_nan_or_inf, + "incr_count": self._incr_count, + "decr_count": self._decr_count, + "use_dynamic_loss_scaling": self._use_dynamic_loss_scaling, + } + if self._enable + else {} + ) def load_state_dict(self, state_dict): """ @@ -505,11 +561,13 @@ def load_state_dict(self, state_dict): if len(state_dict) == 0: raise RuntimeError( "The input state dict is empty, possibly because it was saved " - "from a disabled instance of GradScaler.") + "from a disabled instance of GradScaler." + ) self._init_loss_scaling = state_dict["scale"][0] self._scale = to_variable( - np.array([self._init_loss_scaling]).astype(np.float32)) + np.array([self._init_loss_scaling]).astype(np.float32) + ) self._incr_ratio = state_dict["incr_ratio"] self._decr_ratio = state_dict["decr_ratio"] self._incr_every_n_steps = state_dict["incr_every_n_steps"] diff --git a/python/paddle/fluid/dygraph/base.py b/python/paddle/fluid/dygraph/base.py index 82ef806583743..593ce558f2178 100644 --- a/python/paddle/fluid/dygraph/base.py +++ b/python/paddle/fluid/dygraph/base.py @@ -25,13 +25,23 @@ import logging from ..data_feeder import convert_dtype import warnings -from ..framework import _get_paddle_place, _in_legacy_dygraph, _in_eager_without_dygraph_check +from ..framework import ( + _get_paddle_place, + _in_legacy_dygraph, + _in_eager_without_dygraph_check, +) import paddle import warnings __all__ = [ - 'no_grad', 'no_grad_', 'grad', 'guard', 'enable_dygraph', 'disable_dygraph', - 'enabled', 'to_variable' + 'no_grad', + 'no_grad_', + 'grad', + 'guard', + 'enable_dygraph', + 'disable_dygraph', + 'enabled', + 'to_variable', ] # Flag that indicates whether running code under `@declarative` @@ -46,8 +56,9 @@ def in_declarative_mode(): return _in_declarative_mode_ -def declarative_unsupport_argument_warning(func_name, input_names, inputs, - support_values): +def declarative_unsupport_argument_warning( + func_name, input_names, inputs, support_values +): """ Warning if inputs do not elementwisely equals to support_values. It's a utility function for dy2static when dygraph interface have @@ -56,12 +67,13 @@ def declarative_unsupport_argument_warning(func_name, input_names, inputs, """ for name, inp, sup in zip(input_names, inputs, support_values): if inp != sup: - warnings.warn(f"{func_name} has unsupported parameter in jit: " + - f"{name}, jit will discard it") + warnings.warn( + f"{func_name} has unsupported parameter in jit: " + + f"{name}, jit will discard it" + ) def _switch_to_static_graph_(func): - def __impl__(*args, **kwargs): with framework._dygraph_guard(None): return func(*args, **kwargs) @@ -101,8 +113,11 @@ def program_desc_tracing_guard(enable): @signature_safe_contextmanager def param_guard(parameters): # Note: parameters is a reference of self._parameters or self._buffers - if in_declarative_mode( - ) and not framework._non_static_mode() and parameters: + if ( + in_declarative_mode() + and not framework._non_static_mode() + and parameters + ): origin_parameters = parameters.copy() for name, var_base in parameters.items(): if isinstance(var_base, list): @@ -126,8 +141,9 @@ def _convert_into_variable(tensor): if new_var is not None: assert isinstance(new_var, framework.Variable) # Convert ParamBase into Parameter with same attributes in dy2stat. - elif isinstance(tensor, - (framework.EagerParamBase, framework.ParamBase)): + elif isinstance( + tensor, (framework.EagerParamBase, framework.ParamBase) + ): new_var = tensor._to_static_var(to_parameter=True) else: # Note(Aurelius84): Convert VarBase in self._buffers into Variable with @@ -140,8 +156,9 @@ def _convert_into_variable(tensor): # non-persistable. See case of `drop_state` in lstm api. is_persistable = len(tensor.shape) > 0 - new_var = tensor._to_static_var(to_parameter=False, - persistable=is_persistable) + new_var = tensor._to_static_var( + to_parameter=False, persistable=is_persistable + ) return new_var else: return tensor @@ -207,7 +224,8 @@ def enable_dygraph(place=None): global _functional_dygraph_context_manager if _functional_dygraph_context_manager is None: _functional_dygraph_context_manager = guard( - place=_get_paddle_place(place)) + place=_get_paddle_place(place) + ) _functional_dygraph_context_manager.__enter__() # call disable_dygraph when Python exit @@ -368,7 +386,6 @@ def test_layer(): """ def __call__(self, func): - @decorator.decorator def _decorate_function(func, *args, **kwargs): with self: @@ -448,14 +465,16 @@ def guard(place=None): @framework.non_static_only -def grad(outputs, - inputs, - grad_outputs=None, - retain_graph=None, - create_graph=False, - only_inputs=True, - allow_unused=False, - no_grad_vars=None): +def grad( + outputs, + inputs, + grad_outputs=None, + retain_graph=None, + create_graph=False, + only_inputs=True, + allow_unused=False, + no_grad_vars=None, +): ''' .. note:: **This API is ONLY available in imperative mode.** @@ -581,16 +600,18 @@ def test_dygraph_grad(grad_outputs=None): # dy1 = [3], dy2 = [4] grad_y1 = paddle.to_tensor(3.0) print(test_dygraph_grad([grad_y1, grad_value])) # [24.] - ''' + ''' if in_declarative_mode(): # In dy2static context, we call static interface `gradients` # to calculate grads. from paddle.static import gradients + declarative_unsupport_argument_warning( "paddle.grad", ["retain_graph", "create_grad", "only_inputs", "allow_unused"], [retain_graph, create_graph, only_inputs, allow_unused], - [None, False, True, False]) + [None, False, True, False], + ) return gradients(outputs, inputs, grad_outputs, no_grad_vars) def check_in_out(in_out_list, name): @@ -605,9 +626,8 @@ def check_in_out(in_out_list, name): ), "Elements of {} must be Tensor".format(name) else: assert isinstance( - each_var, - core.VarBase), "Elements of {} must be Variable".format( - name) + each_var, core.VarBase + ), "Elements of {} must be Variable".format(name) return in_out_list else: if _in_eager_without_dygraph_check(): @@ -642,7 +662,8 @@ def check_in_out(in_out_list, name): if len(grad_outputs) > 0: assert len(grad_outputs) == len( - outputs), "The length of grad_outputs must be equal to outputs" + outputs + ), "The length of grad_outputs must be equal to outputs" if no_grad_vars is None: no_grad_vars = [] @@ -655,12 +676,12 @@ def check_in_out(in_out_list, name): for var in no_grad_vars: if _in_eager_without_dygraph_check(): assert isinstance( - var, - core.eager.Tensor), "no_grad_vars can only contains Tensor" + var, core.eager.Tensor + ), "no_grad_vars can only contains Tensor" else: assert isinstance( - var, - core.VarBase), "no_grad_vars can only contains Variable" + var, core.VarBase + ), "no_grad_vars can only contains Variable" else: if _in_eager_without_dygraph_check(): raise AssertionError( @@ -676,8 +697,9 @@ def check_in_out(in_out_list, name): if retain_graph is None: retain_graph = create_graph - assert isinstance(retain_graph, - bool), "retain_graph must be None, True or False" + assert isinstance( + retain_graph, bool + ), "retain_graph must be None, True or False" assert isinstance(allow_unused, bool), "allow_unused must be True or False" @@ -685,17 +707,30 @@ def check_in_out(in_out_list, name): assert only_inputs, "only_inputs=False is not supported yet" if _in_eager_without_dygraph_check(): - return core.eager.run_partial_grad(outputs, inputs, grad_outputs, - retain_graph, create_graph, - only_inputs, allow_unused, - no_grad_vars) + return core.eager.run_partial_grad( + outputs, + inputs, + grad_outputs, + retain_graph, + create_graph, + only_inputs, + allow_unused, + no_grad_vars, + ) else: place = core.Place() place.set_place(framework._current_expected_place()) - return core.dygraph_partial_grad(inputs, outputs, grad_outputs, - no_grad_vars, place, create_graph, - retain_graph, allow_unused, - only_inputs) + return core.dygraph_partial_grad( + inputs, + outputs, + grad_outputs, + no_grad_vars, + place, + create_graph, + retain_graph, + allow_unused, + only_inputs, + ) @framework.dygraph_only @@ -755,20 +790,30 @@ def to_variable(value, name=None, zero_copy=None, dtype=None): y.shape # [3L, 2L] """ - support_type = (list, tuple, np.ndarray, core.eager.Tensor, core.VarBase, - framework.Variable, core.Tensor, core.LoDTensor) + support_type = ( + list, + tuple, + np.ndarray, + core.eager.Tensor, + core.VarBase, + framework.Variable, + core.Tensor, + core.LoDTensor, + ) if not isinstance(value, support_type): raise TypeError( "The type of 'value' in fluid.dygraph.to_variable must be %s, but received %s." - % (support_type, type(value))) + % (support_type, type(value)) + ) if isinstance(value, (core.eager.Tensor, core.VarBase, framework.Variable)): return value elif isinstance(value, (core.Tensor, core.LoDTensor)): return core.VarBase(value) else: - if isinstance(framework._current_expected_place(), - framework.core.CPUPlace): - #TODO(zhiqiu): we found two problems when enable zero_copy on CPUPlace. + if isinstance( + framework._current_expected_place(), framework.core.CPUPlace + ): + # TODO(zhiqiu): we found two problems when enable zero_copy on CPUPlace. # (1): eigen requires 16-bytes alignments, but the data of numpy array may not statisfy. # Details: https://eigen.tuxfamily.org/dox/group__TopicUnalignedArrayAssert.html # (2): when used in flask framework, it may result in hang. @@ -780,7 +825,9 @@ def to_variable(value, name=None, zero_copy=None, dtype=None): ) zero_copy = False else: - assert not zero_copy, "zero_copy mode can only be used with CPUPlace" + assert ( + not zero_copy + ), "zero_copy mode can only be used with CPUPlace" if not isinstance(value, np.ndarray): value = np.array(value) @@ -791,13 +838,20 @@ def to_variable(value, name=None, zero_copy=None, dtype=None): value = value.astype(dtype) if _in_eager_without_dygraph_check(): - return core.eager.Tensor(value, framework._current_expected_place(), - False, zero_copy, name if name else None, - True) + return core.eager.Tensor( + value, + framework._current_expected_place(), + False, + zero_copy, + name if name else None, + True, + ) else: - py_var = core.VarBase(value=value, - place=framework._current_expected_place(), - persistable=False, - zero_copy=zero_copy, - name=name if name else '') + py_var = core.VarBase( + value=value, + place=framework._current_expected_place(), + persistable=False, + zero_copy=zero_copy, + name=name if name else '', + ) return py_var diff --git a/python/paddle/fluid/dygraph/checkpoint.py b/python/paddle/fluid/dygraph/checkpoint.py index 8306e702598e2..f7a1db032f48b 100644 --- a/python/paddle/fluid/dygraph/checkpoint.py +++ b/python/paddle/fluid/dygraph/checkpoint.py @@ -15,14 +15,27 @@ import os import collections import functools -from ..framework import Variable, default_main_program, _non_static_mode, dygraph_only, Parameter, ParamBase, _varbase_creator, _dygraph_tracer, EagerParamBase +from ..framework import ( + Variable, + default_main_program, + _non_static_mode, + dygraph_only, + Parameter, + ParamBase, + _varbase_creator, + _dygraph_tracer, + EagerParamBase, +) import pickle from . import learning_rate_scheduler import warnings from .. import core from .base import guard from paddle.fluid.dygraph.jit import _SaveLoadConfig -from paddle.fluid.dygraph.io import _construct_program_holders, _construct_params_and_buffers +from paddle.fluid.dygraph.io import ( + _construct_program_holders, + _construct_params_and_buffers, +) __all__ = [ 'save_dygraph', @@ -38,7 +51,8 @@ def _parse_load_config(configs): if key not in supported_configs: raise ValueError( "The additional config (%s) of `paddle.fluid.load_dygraph` is not supported." - % (key)) + % (key) + ) # construct inner config inner_config = _SaveLoadConfig() @@ -85,7 +99,9 @@ def save_dygraph(state_dict, model_path): ''' base_name = os.path.basename(model_path) - assert base_name != "", "The input model_path MUST be format of dirname/filename [dirname\\filename in Windows system], but received filename is empty string." + assert ( + base_name != "" + ), "The input model_path MUST be format of dirname/filename [dirname\\filename in Windows system], but received filename is empty string." suffix = ".pdparams" assert len(state_dict) > 0, "state_dict is empty, no need to save" @@ -193,7 +209,10 @@ def load_dygraph(model_path, **configs): with open(params_file_path, 'rb') as f: para_dict = pickle.load(f, encoding='latin1') - if not config.keep_name_table and "StructuredToParameterName@@" in para_dict: + if ( + not config.keep_name_table + and "StructuredToParameterName@@" in para_dict + ): del para_dict["StructuredToParameterName@@"] if os.path.exists(opti_file_path): @@ -202,8 +221,9 @@ def load_dygraph(model_path, **configs): else: # check model path if not os.path.isdir(model_prefix): - raise ValueError("Model saved directory '%s' is not exists." % - model_prefix) + raise ValueError( + "Model saved directory '%s' is not exists." % model_prefix + ) # check whether model file exists if config.model_filename is None: @@ -222,8 +242,9 @@ def load_dygraph(model_path, **configs): # NOTE(chenweihang): `jit.save` doesn't save optimizer state # 1. load program desc & construct _ProgramHolder - programs = _construct_program_holders(model_path, - config.model_filename) + programs = _construct_program_holders( + model_path, config.model_filename + ) # 2. load layer parameters & buffers # NOTE: using fluid.dygraph.guard() here will cause import error in py2 @@ -232,7 +253,8 @@ def load_dygraph(model_path, **configs): model_prefix, programs, config.params_filename, - append_suffix=False) + append_suffix=False, + ) # 3. construct state_dict para_dict = dict() @@ -248,10 +270,15 @@ def load_dygraph(model_path, **configs): structured_para_dict = dict() for var_name in para_dict: structured_name = extra_var_info[var_name].get( - 'structured_name', None) - assert structured_name is not None, "Cannot find saved variable (%s)'s structured name in saved model." % var_name + 'structured_name', None + ) + assert structured_name is not None, ( + "Cannot find saved variable (%s)'s structured name in saved model." + % var_name + ) structured_para_dict[structured_name] = para_dict[ - var_name] + var_name + ] para_dict = structured_para_dict else: # load state dict by `io.save_params/persistables` save format @@ -282,7 +309,8 @@ def load_dygraph(model_path, **configs): type='load', inputs={}, outputs={'Out': new_var}, - attrs={'file_path': os.path.join(model_path, name)}) + attrs={'file_path': os.path.join(model_path, name)}, + ) load_var_list.append(new_var) # 3. construct state_dict diff --git a/python/paddle/fluid/dygraph/container.py b/python/paddle/fluid/dygraph/container.py index 854df39355748..db4ad35610249 100644 --- a/python/paddle/fluid/dygraph/container.py +++ b/python/paddle/fluid/dygraph/container.py @@ -222,8 +222,10 @@ def _get_abs_idx(self, idx): if isinstance(idx, int): if not (-len(self) <= idx < len(self)): raise IndexError( - 'index {} is out of range, should be an integer in range [{}, {})' - .format(idx, -len(self), len(self))) + 'index {} is out of range, should be an integer in range [{}, {})'.format( + idx, -len(self), len(self) + ) + ) if idx < 0: idx += len(self) return idx @@ -248,7 +250,8 @@ def __delitem__(self, idx): delattr(self, str(idx)) str_indices = [str(i) for i in range(len(self._sub_layers))] self._sub_layers = OrderedDict( - list(zip(str_indices, self._sub_layers.values()))) + list(zip(str_indices, self._sub_layers.values())) + ) def __len__(self): return len(self._sub_layers) @@ -297,9 +300,11 @@ def insert(self, index, sublayer): linears.insert(-1, another) print(linears[-2] is another) # True """ - assert isinstance(index, int) and \ - -len(self._sub_layers) <= index < len(self._sub_layers), \ - "index should be an integer in range [{}, {})".format(-len(self), len(self)) + assert isinstance(index, int) and -len(self._sub_layers) <= index < len( + self._sub_layers + ), "index should be an integer in range [{}, {})".format( + -len(self), len(self) + ) index = self._get_abs_idx(index) for i in range(len(self._sub_layers), index, -1): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py index a48258e30d5d5..ca6f1e652e944 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/assert_transformer.py @@ -14,9 +14,13 @@ from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) class AssertTransformer(BaseTransformer): @@ -35,8 +39,15 @@ def transform(self): self.visit(self.root) def visit_Assert(self, node): - convert_assert_node = gast.parse('_jst.Assert({test}, {msg})'.format( - test=ast_to_source_code(node.test), - msg=ast_to_source_code(node.msg) if node.msg else "")).body[0].value + convert_assert_node = ( + gast.parse( + '_jst.Assert({test}, {msg})'.format( + test=ast_to_source_code(node.test), + msg=ast_to_source_code(node.msg) if node.msg else "", + ) + ) + .body[0] + .value + ) return gast.Expr(value=convert_assert_node) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py index ff7a9a2a957b6..3ae91027f5aee 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/ast_transformer.py @@ -19,24 +19,60 @@ import os from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer -from paddle.fluid.dygraph.dygraph_to_static.early_return_transformer import EarlyReturnTransformer -from paddle.fluid.dygraph.dygraph_to_static.assert_transformer import AssertTransformer -from paddle.fluid.dygraph.dygraph_to_static.basic_api_transformer import BasicApiTransformer -from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import BreakContinueTransformer -from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import BreakTransformOptimizer -from paddle.fluid.dygraph.dygraph_to_static.call_transformer import CallTransformer -from paddle.fluid.dygraph.dygraph_to_static.cast_transformer import CastTransformer -from paddle.fluid.dygraph.dygraph_to_static.typehint_transformer import TypeHintTransformer -from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import IfElseTransformer -from paddle.fluid.dygraph.dygraph_to_static.logical_transformer import LogicalTransformer -from paddle.fluid.dygraph.dygraph_to_static.loop_transformer import LoopTransformer -from paddle.fluid.dygraph.dygraph_to_static.print_transformer import PrintTransformer -from paddle.fluid.dygraph.dygraph_to_static.return_transformer import ReturnTransformer -from paddle.fluid.dygraph.dygraph_to_static.create_variable_transformer import CreateVariableTransformer -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor -from paddle.fluid.dygraph.dygraph_to_static.tensor_shape_transformer import TensorShapeTransformer -from paddle.fluid.dygraph.dygraph_to_static.decorator_transformer import DecoratorTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.early_return_transformer import ( + EarlyReturnTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.assert_transformer import ( + AssertTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.basic_api_transformer import ( + BasicApiTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import ( + BreakContinueTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import ( + BreakTransformOptimizer, +) +from paddle.fluid.dygraph.dygraph_to_static.call_transformer import ( + CallTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.cast_transformer import ( + CastTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.typehint_transformer import ( + TypeHintTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import ( + IfElseTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.logical_transformer import ( + LogicalTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.loop_transformer import ( + LoopTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.print_transformer import ( + PrintTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.return_transformer import ( + ReturnTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.create_variable_transformer import ( + CreateVariableTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + StaticAnalysisVisitor, +) +from paddle.fluid.dygraph.dygraph_to_static.tensor_shape_transformer import ( + TensorShapeTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.decorator_transformer import ( + DecoratorTransformer, +) from paddle.fluid.dygraph.dygraph_to_static import logging_utils from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code @@ -51,8 +87,11 @@ def apply_optimization(transformers): And not all optimized transformations are applied by default. It's controlled by 'export FLAGS_optim_transformation=1' """ - flag = str( - os.environ.get('FLAGS_optim_transformation')) in ['1', 'True', 'true'] + flag = str(os.environ.get('FLAGS_optim_transformation')) in [ + '1', + 'True', + 'true', + ] if flag: transformers.insert(3, BreakTransformOptimizer) @@ -69,7 +108,8 @@ def get_static_ast(self, root): # save root for some analysis may need global AST self.root = root self.static_analysis_visitor = StaticAnalysisVisitor(root) - self.static_analysis_root = self.static_analysis_visitor.get_node_wrapper_root( + self.static_analysis_root = ( + self.static_analysis_visitor.get_node_wrapper_root() ) self.decorate_func_name = None self.transfer_from_node_type(self.static_analysis_root) @@ -77,12 +117,14 @@ def get_static_ast(self, root): def _apply(self, transformer, node_wrapper, log_level): transformer(node_wrapper).transform() - self.translator_logger.log_transformed_code(log_level, self.root, - transformer.__name__) + self.translator_logger.log_transformed_code( + log_level, self.root, transformer.__name__ + ) def transfer_from_node_type(self, node_wrapper): self.translator_logger.log( - 1, "Source code: \n{}".format(ast_to_source_code(self.root))) + 1, "Source code: \n{}".format(ast_to_source_code(self.root)) + ) # Generic transformation self.visit(node_wrapper.node) @@ -110,7 +152,8 @@ def transfer_from_node_type(self, node_wrapper): self._apply(transformer, node_wrapper, log_level=index + 1) self.translator_logger.log_transformed_code( - logging_utils.LOG_AllTransformer, self.root, "All Transformers") + logging_utils.LOG_AllTransformer, self.root, "All Transformers" + ) def visit_FunctionDef(self, node): if self.decorate_func_name is None: diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/base_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/base_transformer.py index b77c5b3c70174..cb8f1562b3efc 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/base_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/base_transformer.py @@ -20,16 +20,23 @@ from paddle.fluid.dygraph.dygraph_to_static.utils import ORIGI_INFO from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_INDEX_PREFIX from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_TUPLE_PREFIX -from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_TUPLE_INDEX_PREFIX +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FOR_ITER_TUPLE_INDEX_PREFIX, +) from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_VAR_LEN_PREFIX -from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_VAR_NAME_PREFIX -from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_ZIP_TO_LIST_PREFIX +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FOR_ITER_VAR_NAME_PREFIX, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FOR_ITER_ZIP_TO_LIST_PREFIX, +) from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_TARGET_PREFIX -from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_ITERATOR_PREFIX +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FOR_ITER_ITERATOR_PREFIX, +) class BaseTransformer(gast.NodeTransformer): - def visit(self, node): if not isinstance(node, gast.AST): msg = ('Expected "gast.AST", but got "{}".').format(type(node)) @@ -41,7 +48,7 @@ def visit(self, node): iter_result = result if iter_result is not node and iter_result is not None: if not isinstance(iter_result, (list, tuple)): - iter_result = (iter_result, ) + iter_result = (iter_result,) if origin_info is not None: for n in iter_result: setattr(n, ORIGI_INFO, origin_info) @@ -50,10 +57,10 @@ def visit(self, node): class RenameTransformer(BaseTransformer): - def __init__(self, node): assert isinstance( - node, gast.AST), "RenameTransformer only accepts gast.AST as input" + node, gast.AST + ), "RenameTransformer only accepts gast.AST as input" self.root = node self.old_name = "" self.new_name = "" @@ -113,7 +120,8 @@ def visit_Nonlocal(self, node): names = node.names def replace(s): - if s == self.target_name: return self.replace_node.id + if s == self.target_name: + return self.replace_node.id return s node.names = list(map(replace, names)) @@ -121,7 +129,7 @@ def replace(s): class ForLoopTuplePreTransformer(BaseTransformer): - """ pre-process of for loop. + """pre-process of for loop. >>> for A in B: >>> C @@ -152,24 +160,28 @@ def visit_For(self, node): assign_iterator_node = gast.parse( f"{tuple_iterator} = _jst.Indexable({ast_to_source_code(node.iter).strip()})" ).body[0] - node.target = gast.Name(id=tuple_target, - ctx=gast.Store(), - annotation=None, - type_comment=None) - node.iter = gast.Name(id=tuple_iterator, - ctx=gast.Load(), - annotation=None, - type_comment=None) + node.target = gast.Name( + id=tuple_target, + ctx=gast.Store(), + annotation=None, + type_comment=None, + ) + node.iter = gast.Name( + id=tuple_iterator, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ) node.body[0:0] = self.tuple_to_stmts(origin_tuple_node, tuple_target) # return a list will insert a list of node replace the original for node. return [assign_iterator_node, node] def tuple_node_to_unpack_structure(self, node): - """ Create a sequence to represents the structure of nest. - For example: `a, (b,c), [d,e,f]` is represented by - `[1, [1,1], [1,1,1]]`. the `1` is just a notation. + """Create a sequence to represents the structure of nest. + For example: `a, (b,c), [d,e,f]` is represented by + `[1, [1,1], [1,1,1]]`. the `1` is just a notation. - Specially, `a` is represented by `1`. + Specially, `a` is represented by `1`. """ ret = [] if not isinstance(node, (gast.Tuple, gast.List)): @@ -181,7 +193,9 @@ def tuple_node_to_unpack_structure(self, node): def tuple_to_stmts(self, node, tuple_name): structure_str = str(self.tuple_node_to_unpack_structure(node)) node_str = ast_to_source_code(node).strip() - assign_node_str = f"{node_str} = _jst.Unpack({tuple_name}, {structure_str})" + assign_node_str = ( + f"{node_str} = _jst.Unpack({tuple_name}, {structure_str})" + ) assign_node = gast.parse(assign_node_str).body[0] return [assign_node] @@ -213,8 +227,9 @@ def __init__(self, for_node): # 2. gast.For node main parts self.target = for_node.target # NOTE: type may be Node or list[Node] - self.iter_args = for_node.iter if self.is_for_iter( - ) else for_node.iter.args + self.iter_args = ( + for_node.iter if self.is_for_iter() else for_node.iter.args + ) self.body = for_node.body # 3. key shared node or names @@ -236,7 +251,8 @@ def __init__(self, for_node): self.iter_var_len_name = unique_name.generate(FOR_ITER_VAR_LEN_PREFIX) # - created zip to list var : __for_loop_iter_zip_0 self.iter_zip_to_list_name = unique_name.generate( - FOR_ITER_ZIP_TO_LIST_PREFIX) + FOR_ITER_ZIP_TO_LIST_PREFIX + ) # - var.numpy()/var # - for x in var|var.numpy() @@ -262,17 +278,22 @@ def parse(self): return None def is_for_range_iter(self): - return isinstance(self.node.iter, gast.Call) and isinstance( - self.node.iter.func, - gast.Name) and self.node.iter.func.id == "range" + return ( + isinstance(self.node.iter, gast.Call) + and isinstance(self.node.iter.func, gast.Name) + and self.node.iter.func.id == "range" + ) def is_for_iter(self): - if isinstance(self.node.iter, - (gast.Name, gast.Attribute, gast.List, gast.Tuple)): + if isinstance( + self.node.iter, (gast.Name, gast.Attribute, gast.List, gast.Tuple) + ): return True - elif isinstance(self.node.iter, gast.Call) and isinstance( - self.node.iter.func, - gast.Attribute) and self.node.iter.func.attr == 'numpy': + elif ( + isinstance(self.node.iter, gast.Call) + and isinstance(self.node.iter.func, gast.Attribute) + and self.node.iter.func.attr == 'numpy' + ): return True elif isinstance(self.node.iter, gast.Subscript): return True @@ -280,17 +301,23 @@ def is_for_iter(self): return False def is_for_enumerate_iter(self): - return isinstance(self.node.iter, gast.Call) and isinstance( - self.node.iter.func, - gast.Name) and self.node.iter.func.id == "enumerate" + return ( + isinstance(self.node.iter, gast.Call) + and isinstance(self.node.iter.func, gast.Name) + and self.node.iter.func.id == "enumerate" + ) def _args_check(self): if self.is_for_range_iter(): self.args_length = len(self.iter_args) - assert self.args_length >= 1 and self.args_length <= 3, "range() function takes 1 to 3 arguments" + assert ( + self.args_length >= 1 and self.args_length <= 3 + ), "range() function takes 1 to 3 arguments" elif self.is_for_enumerate_iter(): self.args_length = len(self.iter_args) - assert self.args_length >= 1 and self.args_length <= 2, "enumerate() function takes 1 to 2 arguments" + assert ( + self.args_length >= 1 and self.args_length <= 2 + ), "enumerate() function takes 1 to 2 arguments" else: self.args_length = None @@ -324,8 +351,9 @@ def _parse_for_stmts(self): target_node, assign_node = self._build_assign_var_slice_node() body_stmts[0:0] = [assign_node] for body_node in body_stmts: - NameNodeReplaceTransformer(body_node, self.iter_var_name, - target_node) + NameNodeReplaceTransformer( + body_node, self.iter_var_name, target_node + ) body_stmts.append(self._build_index_increase_node(step_node)) return init_stmts, cond_stmt, body_stmts @@ -346,8 +374,9 @@ def _parse_for_enumerate_stmts(self): target_node, assign_node = self._build_assign_var_slice_node() body_stmts[0:0] = [assign_node] for body_node in body_stmts: - NameNodeReplaceTransformer(body_node, self.iter_var_name, - target_node) + NameNodeReplaceTransformer( + body_node, self.iter_var_name, target_node + ) body_stmts.append(self._build_index_increase_node(step_node)) body_stmts.append(self._build_enum_increase_node()) @@ -360,7 +389,8 @@ def _build_index_init_node(self): index_init_value_str = '0' else: index_init_value_str = ast_to_source_code( - self.iter_args[0]).strip() + self.iter_args[0] + ).strip() index_init_var_name = self.iter_var_name else: @@ -368,7 +398,8 @@ def _build_index_init_node(self): index_init_var_name = self.iter_idx_name index_init_node_source_str = "{target} = {value}".format( - target=index_init_var_name, value=index_init_value_str) + target=index_init_var_name, value=index_init_value_str + ) index_init_node = gast.parse(index_init_node_source_str).body[0] @@ -376,16 +407,20 @@ def _build_index_init_node(self): def _build_var_len_assign_node(self): # get the length of iterable variable - if isinstance(self.iter_node, gast.Call) and isinstance( - self.iter_node.func, - gast.Attribute) and self.iter_node.func.attr == 'numpy': + if ( + isinstance(self.iter_node, gast.Call) + and isinstance(self.iter_node.func, gast.Attribute) + and self.iter_node.func.attr == 'numpy' + ): iter_var_name = ast_to_source_code( - self.iter_node.func.value).strip() + self.iter_node.func.value + ).strip() else: iter_var_name = ast_to_source_code(self.iter_node).strip() convert_len_node_source_str = '{} = _jst.Len({})'.format( - self.iter_var_len_name, iter_var_name) + self.iter_var_len_name, iter_var_name + ) convert_len_node = gast.parse(convert_len_node_source_str).body[0] @@ -403,18 +438,22 @@ def _build_iter_node(self): """ new_nodes = [] if isinstance(self.iter_node, gast.Call) and isinstance( - self.iter_node.func, gast.Name): + self.iter_node.func, gast.Name + ): if self.iter_node.func.id == 'zip': iter_var_name = ast_to_source_code(self.iter_node).strip() zip_to_list_str = "{target} = list({value})".format( - target=self.iter_zip_to_list_name, value=iter_var_name) + target=self.iter_zip_to_list_name, value=iter_var_name + ) zip_to_list_node = gast.parse(zip_to_list_str).body[0] new_nodes.append(zip_to_list_node) - self.iter_node = gast.Name(id=self.iter_zip_to_list_name, - ctx=gast.Load(), - annotation=None, - type_comment=None) + self.iter_node = gast.Name( + id=self.iter_zip_to_list_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ) return new_nodes @@ -424,27 +463,35 @@ def _build_enum_init_node(self): else: init_value_str = '0' - enum_init_node_source_str = "{} = {}".format(self.enum_idx_name, - init_value_str) + enum_init_node_source_str = "{} = {}".format( + self.enum_idx_name, init_value_str + ) enum_init_node = gast.parse(enum_init_node_source_str).body[0] return enum_init_node def _build_compare_node(self): if self.is_for_range_iter(): - compare_node = self.iter_args[ - 0] if self.args_length == 1 else self.iter_args[1] + compare_node = ( + self.iter_args[0] + if self.args_length == 1 + else self.iter_args[1] + ) else: - compare_node = gast.Name(id=self.iter_var_len_name, - ctx=gast.Load(), - annotation=None, - type_comment=None) + compare_node = gast.Name( + id=self.iter_var_len_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ) return compare_node def _build_step_node(self): if self.is_for_range_iter(): - step_node = self.iter_args[ - 2] if self.args_length == 3 else gast.Constant(value=1, - kind=None) + step_node = ( + self.iter_args[2] + if self.args_length == 3 + else gast.Constant(value=1, kind=None) + ) else: step_node = gast.Constant(value=1, kind=None) return step_node @@ -453,62 +500,82 @@ def _build_cond_stmt(self, step_node, compare_node): if not isinstance(step_node, (gast.Constant, gast.UnaryOp)): raise NotImplementedError( "Dynamic-to-Static only supports the step value is a constant or negative constant in 'for-range' statements, " - "such as '2', '-3'. But received: '{}'. Please fix code to be compatible with Dynamic-to-Static." - .format(ast_to_source_code(step_node).strip())) + "such as '2', '-3'. But received: '{}'. Please fix code to be compatible with Dynamic-to-Static.".format( + ast_to_source_code(step_node).strip() + ) + ) if isinstance(step_node, gast.UnaryOp) or step_node.value < 0: # eg: # range(max, min, -2) # -> # i > min - return gast.Compare(left=gast.Name( - id=self.iter_var_name - if self.is_for_range_iter() else self.iter_idx_name, - ctx=gast.Load(), - annotation=None, - type_comment=None), - ops=[gast.Gt()], - comparators=[compare_node]) + return gast.Compare( + left=gast.Name( + id=self.iter_var_name + if self.is_for_range_iter() + else self.iter_idx_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ), + ops=[gast.Gt()], + comparators=[compare_node], + ) else: # eg: # range(min, max, 2) # -> # i < max - return gast.Compare(left=gast.Name( - id=self.iter_var_name - if self.is_for_range_iter() else self.iter_idx_name, - ctx=gast.Load(), - annotation=None, - type_comment=None), - ops=[gast.Lt()], - comparators=[compare_node]) + return gast.Compare( + left=gast.Name( + id=self.iter_var_name + if self.is_for_range_iter() + else self.iter_idx_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ), + ops=[gast.Lt()], + comparators=[compare_node], + ) def _build_index_increase_node(self, step_node): - return gast.AugAssign(target=gast.Name( - id=self.iter_var_name - if self.is_for_range_iter() else self.iter_idx_name, - ctx=gast.Store(), - annotation=None, - type_comment=None), - op=gast.Add(), - value=step_node) + return gast.AugAssign( + target=gast.Name( + id=self.iter_var_name + if self.is_for_range_iter() + else self.iter_idx_name, + ctx=gast.Store(), + annotation=None, + type_comment=None, + ), + op=gast.Add(), + value=step_node, + ) def _build_assign_var_slice_node(self): var_slice_str = "{}[{}]".format( - ast_to_source_code(self.iter_node).strip(), self.iter_idx_name) + ast_to_source_code(self.iter_node).strip(), self.iter_idx_name + ) var_slice_node = gast.parse(var_slice_str).body[0].value new_iter_var_name = unique_name.generate(FOR_ITER_VAR_NAME_PREFIX) - target_node, assign_node = create_assign_node(new_iter_var_name, - var_slice_node) + target_node, assign_node = create_assign_node( + new_iter_var_name, var_slice_node + ) return target_node, assign_node def _build_enum_increase_node(self): - return gast.AugAssign(target=gast.Name(id=self.enum_idx_name, - ctx=gast.Store(), - annotation=None, - type_comment=None), - op=gast.Add(), - value=gast.Constant(value=1, kind=None)) + return gast.AugAssign( + target=gast.Name( + id=self.enum_idx_name, + ctx=gast.Store(), + annotation=None, + type_comment=None, + ), + op=gast.Add(), + value=gast.Constant(value=1, kind=None), + ) def _get_iter_var_name(self): if self.is_for_range_iter(): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py index 55afb7ae6d6de..d6c32a1fc2bd1 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/basic_api_transformer.py @@ -15,9 +15,13 @@ import astor from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) from paddle.fluid.dygraph.dygraph_to_static import utils -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code @@ -139,9 +143,11 @@ def __init__(self, node): assert isinstance( node, gast.AST ), "Input non-gast.AST node for the initialization of ToTensorTransformer." - self.interested_name = set([ - 'size', - ]) + self.interested_name = set( + [ + 'size', + ] + ) self.root = node def transform(self): @@ -151,12 +157,21 @@ def transform(self): def visit_Attribute(self, node): assert isinstance(node, gast.Attribute) assert isinstance(node.attr, str) - if isinstance(node.ctx, - gast.Load) and node.attr in self.interested_name: + if ( + isinstance(node.ctx, gast.Load) + and node.attr in self.interested_name + ): attr = node.attr value = node.value - node = gast.parse("_jst.Attr({}, \"{}\")".format( - ast_to_source_code(value).strip(), attr)).body[0].value + node = ( + gast.parse( + "_jst.Attr({}, \"{}\")".format( + ast_to_source_code(value).strip(), attr + ) + ) + .body[0] + .value + ) self.generic_visit(node) return node diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py index 8bfc7187ecab0..6726e5f14f45c 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/break_continue_transformer.py @@ -17,9 +17,15 @@ from paddle.fluid import unique_name from paddle.fluid.dygraph.dygraph_to_static.utils import index_in_list from paddle.fluid.dygraph.dygraph_to_static.utils import BaseNodeVisitor -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_bool_node -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ForNodeVisitor +from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + create_bool_node, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + ForNodeVisitor, +) __all__ = ['BreakContinueTransformer'] @@ -35,8 +41,8 @@ class ForToWhileTransformer(BaseTransformer): def __init__(self, parent_node, loop_node, condition_node): assert isinstance( - loop_node, - gast.For), "loop_node is not gast.For in ForToWhileTransformer" + loop_node, gast.For + ), "loop_node is not gast.For in ForToWhileTransformer" self.parent_node = parent_node self.loop_node = loop_node self.condition_node = condition_node @@ -47,7 +53,7 @@ def transform(self): i = index_in_list(body_list, self.loop_node) if i != -1: new_stmts = self.get_for_stmt_nodes(body_list[i]) - body_list[i:i + 1] = new_stmts + body_list[i : i + 1] = new_stmts i += len(new_stmts) return new_stmts if hasattr(self.parent_node, 'orelse'): @@ -55,7 +61,7 @@ def transform(self): i = index_in_list(body_list, self.loop_node) if i != -1: new_stmts = self.get_for_stmt_nodes(body_list[i]) - body_list[i:i + 1] = new_stmts + body_list[i : i + 1] = new_stmts i += len(new_stmts) return new_stmts raise ValueError( @@ -64,7 +70,8 @@ def transform(self): def get_for_stmt_nodes(self, node): assert isinstance( - node, gast.For), "Input node is NOT gast.For in get_for_stmt_nodes" + node, gast.For + ), "Input node is NOT gast.For in get_for_stmt_nodes" # 1. parse current gast.For node current_for_node_parser = ForNodeVisitor(node) @@ -74,13 +81,14 @@ def get_for_stmt_nodes(self, node): init_stmts, cond_stmt, body_stmts = stmts_tuple # 2. append break statement - new_cond_stmt = gast.BoolOp(op=gast.And(), - values=[cond_stmt, self.condition_node]) + new_cond_stmt = gast.BoolOp( + op=gast.And(), values=[cond_stmt, self.condition_node] + ) # 3. construct gast.While node - while_node = gast.While(test=new_cond_stmt, - body=body_stmts, - orelse=node.orelse) + while_node = gast.While( + test=new_cond_stmt, body=body_stmts, orelse=node.orelse + ) init_stmts.append(while_node) return init_stmts @@ -132,7 +140,8 @@ def visit_Break(self, node): # block can be a node containing stmt list. We should remove all stmts # after the 'break/continue' and set the V to True here. first_block_index = self._remove_stmts_after_break_continue( - node, variable_name, loop_node_index) + node, variable_name, loop_node_index + ) # 3. Add 'if not V' for stmts in ancestor blocks between the first one # (exclusive) and the ancestor loop (inclusive) @@ -142,19 +151,25 @@ def visit_Break(self, node): assign_false_node = create_bool_node(variable_name, False) self._add_stmt_before_cur_node(loop_node_index, assign_false_node) - cond_var_node = gast.UnaryOp(op=gast.Not(), - operand=gast.Name(id=variable_name, - ctx=gast.Load(), - annotation=None, - type_comment=None)) + cond_var_node = gast.UnaryOp( + op=gast.Not(), + operand=gast.Name( + id=variable_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ), + ) if isinstance(loop_node, gast.While): - loop_node.test = gast.BoolOp(op=gast.And(), - values=[loop_node.test, cond_var_node]) + loop_node.test = gast.BoolOp( + op=gast.And(), values=[loop_node.test, cond_var_node] + ) elif isinstance(loop_node, gast.For): parent_node = self.ancestor_nodes[loop_node_index - 1] - for_to_while = ForToWhileTransformer(parent_node, loop_node, - cond_var_node) + for_to_while = ForToWhileTransformer( + parent_node, loop_node, cond_var_node + ) for_to_while.transform() def visit_Continue(self, node): @@ -169,7 +184,8 @@ def visit_Continue(self, node): # block can be a node containing stmt list. We should remove all stmts # after the 'break/continue' and set the V to True here. first_block_index = self._remove_stmts_after_break_continue( - node, variable_name, loop_node_index) + node, variable_name, loop_node_index + ) # 3. Add 'if not V' for stmts in ancestor blocks between the first one # (exclusive) and the ancestor loop (inclusive) @@ -179,44 +195,51 @@ def visit_Continue(self, node): assign_false_node = create_bool_node(variable_name, False) loop_node.body.insert(0, assign_false_node) - def _remove_stmts_after_break_continue(self, break_continue_node, - break_continue_name, - loop_node_index): + def _remove_stmts_after_break_continue( + self, break_continue_node, break_continue_name, loop_node_index + ): for first_block_index in range( - len(self.ancestor_nodes) - 1, loop_node_index - 1, -1): + len(self.ancestor_nodes) - 1, loop_node_index - 1, -1 + ): first_block = self.ancestor_nodes[first_block_index] - if hasattr(first_block, - "body") and self._replace_break_continue_in_stmt_list( - first_block.body, break_continue_node, - break_continue_name): + if hasattr( + first_block, "body" + ) and self._replace_break_continue_in_stmt_list( + first_block.body, break_continue_node, break_continue_name + ): return first_block_index - if hasattr(first_block, - "orelse") and self._replace_break_continue_in_stmt_list( - first_block.orelse, break_continue_node, - break_continue_name): + if hasattr( + first_block, "orelse" + ) and self._replace_break_continue_in_stmt_list( + first_block.orelse, break_continue_node, break_continue_name + ): return first_block_index return first_block_index - def _replace_if_stmt(self, loop_node_index, first_block_index, - break_continue_name): + def _replace_if_stmt( + self, loop_node_index, first_block_index, break_continue_name + ): for i in range(first_block_index - 1, loop_node_index - 1, -1): cur_node = self.ancestor_nodes[i] son_node = self.ancestor_nodes[i + 1] - if hasattr(cur_node, - 'body') and self._replace_after_node_to_if_in_stmt_list( - cur_node.body, son_node, break_continue_name): + if hasattr( + cur_node, 'body' + ) and self._replace_after_node_to_if_in_stmt_list( + cur_node.body, son_node, break_continue_name + ): continue if hasattr( - cur_node, - 'orelse') and self._replace_after_node_to_if_in_stmt_list( - cur_node.orelse, son_node, break_continue_name): + cur_node, 'orelse' + ) and self._replace_after_node_to_if_in_stmt_list( + cur_node.orelse, son_node, break_continue_name + ): continue - def _replace_break_continue_in_stmt_list(self, stmt_list, - break_continue_node, - break_continue_name): + def _replace_break_continue_in_stmt_list( + self, stmt_list, break_continue_node, break_continue_name + ): i = index_in_list(stmt_list, break_continue_node) if i == -1: return False @@ -224,8 +247,9 @@ def _replace_break_continue_in_stmt_list(self, stmt_list, stmt_list[i:] = [assign_true_node] return True - def _replace_after_node_to_if_in_stmt_list(self, stmt_list, node, - break_continue_name): + def _replace_after_node_to_if_in_stmt_list( + self, stmt_list, node, break_continue_name + ): i = index_in_list(stmt_list, node) if i == -1: return False @@ -234,28 +258,37 @@ def _replace_after_node_to_if_in_stmt_list(self, stmt_list, node, # No need to add, we consider this as added successfully return True - if_stmt = gast.If(test=gast.UnaryOp(op=gast.Not(), - operand=gast.Name( - id=break_continue_name, - ctx=gast.Store(), - annotation=None, - type_comment=None)), - body=stmt_list[i + 1:], - orelse=[]) - stmt_list[i + 1:] = [] + if_stmt = gast.If( + test=gast.UnaryOp( + op=gast.Not(), + operand=gast.Name( + id=break_continue_name, + ctx=gast.Store(), + annotation=None, + type_comment=None, + ), + ), + body=stmt_list[i + 1 :], + orelse=[], + ) + stmt_list[i + 1 :] = [] stmt_list.append(if_stmt) return True def _add_stmt_before_cur_node(self, cur_node_index, stmt_node): cur_node = self.ancestor_nodes[cur_node_index] parent_node = self.ancestor_nodes[cur_node_index - 1] - if hasattr(parent_node, - "body") and self._add_stmt_into_list_before_node( - parent_node.body, cur_node, stmt_node): + if hasattr( + parent_node, "body" + ) and self._add_stmt_into_list_before_node( + parent_node.body, cur_node, stmt_node + ): return True - if hasattr(parent_node, - "orelse") and self._add_stmt_into_list_before_node( - parent_node.orelse, cur_node, stmt_node): + if hasattr( + parent_node, "orelse" + ) and self._add_stmt_into_list_before_node( + parent_node.orelse, cur_node, stmt_node + ): return True return False @@ -330,11 +363,13 @@ def visit_Break(self, node): if isinstance(loop_node, gast.While): loop_node.test = gast.BoolOp( - op=gast.And(), values=[loop_node.test, cond_var_node]) + op=gast.And(), values=[loop_node.test, cond_var_node] + ) elif isinstance(loop_node, gast.For): parent_node = self.ancestor_nodes[loop_node_index - 1] - for_to_while = ForToWhileTransformer(parent_node, loop_node, - cond_var_node) + for_to_while = ForToWhileTransformer( + parent_node, loop_node, cond_var_node + ) for_to_while.transform() def _is_break_cond_pattern(self, break_node, loop_node): @@ -351,8 +386,10 @@ def _is_break_cond_pattern(self, break_node, loop_node): is_matched = False if isinstance(parent_if_node, gast.If): # gast.If only contains `break` - break_first_in_if = parent_if_node.body[0] == break_node and len( - parent_if_node.orelse) == 0 + break_first_in_if = ( + parent_if_node.body[0] == break_node + and len(parent_if_node.orelse) == 0 + ) # gast.If is first node of loop_node if_first_in_loop = loop_node.body[0] == parent_if_node diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py index d6af6ffced743..a29fe8f9d6b1e 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/call_transformer.py @@ -14,10 +14,14 @@ from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code from paddle.fluid.dygraph.dygraph_to_static.utils import is_paddle_api -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) PDB_SET = "pdb.set_trace" @@ -47,7 +51,10 @@ def _no_need_convert_call(self, node): func_str = ast_to_source_code(node.func).strip() try: - from paddle.fluid.dygraph.dygraph_to_static.convert_call_func import is_builtin + from paddle.fluid.dygraph.dygraph_to_static.convert_call_func import ( + is_builtin, + ) + need_convert_builtin_func_list = { 'len', 'zip', diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py index 06c797033812f..7a5821e12f5bf 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/cast_transformer.py @@ -14,9 +14,13 @@ from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) class CastTransformer(BaseTransformer): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py b/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py index 3f13618283916..5adf810eef4a3 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/convert_call_func.py @@ -26,12 +26,26 @@ import builtins from paddle.fluid.dygraph.container import Sequential -from paddle.fluid.dygraph.dygraph_to_static.convert_operators import convert_len, convert_zip -from paddle.fluid.dygraph.dygraph_to_static.convert_operators import convert_range, convert_enumerate -from paddle.fluid.dygraph.dygraph_to_static.logging_utils import TranslatorLogger -from paddle.fluid.dygraph.dygraph_to_static.program_translator import StaticFunction -from paddle.fluid.dygraph.dygraph_to_static.program_translator import convert_to_static -from paddle.fluid.dygraph.dygraph_to_static.program_translator import unwrap_decorators +from paddle.fluid.dygraph.dygraph_to_static.convert_operators import ( + convert_len, + convert_zip, +) +from paddle.fluid.dygraph.dygraph_to_static.convert_operators import ( + convert_range, + convert_enumerate, +) +from paddle.fluid.dygraph.dygraph_to_static.logging_utils import ( + TranslatorLogger, +) +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + StaticFunction, +) +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + convert_to_static, +) +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + unwrap_decorators, +) from paddle.fluid.dygraph.dygraph_to_static.utils import is_paddle_func, unwrap from paddle.fluid.dygraph.layers import Layer @@ -39,7 +53,14 @@ # TODO(liym27): A better way to do this. BUILTIN_LIKELY_MODULES = [ - collections, pdb, copy, inspect, re, six, numpy, logging + collections, + pdb, + copy, + inspect, + re, + six, + numpy, + logging, ] # The api(s) should be considered as plain function and convert # them into static layer code. @@ -65,8 +86,8 @@ def __init__(self, not_convert=False): def is_builtin(func, name=None): - """ predict whether a function is a builtin function with name={name}. - if name == None, then any builtin function will return True + """predict whether a function is a builtin function with name={name}. + if name == None, then any builtin function will return True """ def name_judge(): @@ -93,8 +114,10 @@ def is_unsupported(func): if func_in_dict: translator_logger.log( 2, - "Whitelist: {} is part of built-in module and does not have to be transformed." - .format(func)) + "Whitelist: {} is part of built-in module and does not have to be transformed.".format( + func + ), + ) return True # NOTE: should be placed before `is_paddle_func` @@ -104,8 +127,10 @@ def is_unsupported(func): if is_paddle_func(func): translator_logger.log( 2, - "Whitelist: {} is part of Paddle module and does not have to be transformed." - .format(func)) + "Whitelist: {} is part of Paddle module and does not have to be transformed.".format( + func + ), + ) return True @@ -145,8 +170,9 @@ def dyfunc(x): # [1. 1. 1.]] """ - translator_logger.log(1, - "Convert callable object: convert {}.".format(func)) + translator_logger.log( + 1, "Convert callable object: convert {}.".format(func) + ) func_self = None converted_call = None @@ -158,8 +184,10 @@ def dyfunc(x): if options is not None and options.not_convert: translator_logger.log( 2, - "{} is not converted when it is decorated by 'paddle.jit.not_to_static'." - .format(func)) + "{} is not converted when it is decorated by 'paddle.jit.not_to_static'.".format( + func + ), + ) return func if is_builtin(func, "len"): @@ -183,9 +211,15 @@ def dyfunc(x): # occasion. number_of_stars = 30 translator_logger.warn( - "\n\n" + "*" * number_of_stars + - "\nYour function:`{}` doesn't support to transform to static function because it is a generator function, it will be run as-is." - .format(func.__name__) + "\n" + "*" * number_of_stars + "\n\n") + "\n\n" + + "*" * number_of_stars + + "\nYour function:`{}` doesn't support to transform to static function because it is a generator function, it will be run as-is.".format( + func.__name__ + ) + + "\n" + + "*" * number_of_stars + + "\n\n" + ) return func if inspect.isfunction(func): @@ -214,10 +248,12 @@ def dyfunc(x): _, fn = unwrap_decorators(fn) global_functions.add(fn) elif inspect.isclass(fn): - if isinstance(fn.__dict__.get(func.__name__, None), - staticmethod): + if isinstance( + fn.__dict__.get(func.__name__, None), staticmethod + ): global_functions.add( - func) # Add func to ensure that we will convert + func + ) # Add func to ensure that we will convert if func in global_functions: converted_call = convert_to_static(func) @@ -227,8 +263,10 @@ def dyfunc(x): # If func is not in __globals__, it does not need to be transformed # because it has been transformed before. translator_logger.warn( - "{} doesn't have to be transformed to static function because it has been transformed before, it will be run as-is." - .format(func)) + "{} doesn't have to be transformed to static function because it has been transformed before, it will be run as-is.".format( + func + ) + ) converted_call = func except AttributeError: # NOTE: @@ -274,12 +312,15 @@ def dyfunc(x): func_self = None if func_self else func_self else: raise NotImplementedError( - "Callable {} can not be transformed at present.".format(func)) + "Callable {} can not be transformed at present.".format(func) + ) if converted_call is None: translator_logger.warn( - "{} doesn't have to be transformed to static function, and it will be run as-is." - .format(func)) + "{} doesn't have to be transformed to static function, and it will be run as-is.".format( + func + ) + ) return func if func_self: diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py b/python/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py index 2e8aa40541ca5..0fa48c4260c46 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py @@ -15,16 +15,46 @@ import re import paddle from paddle.fluid.data_feeder import convert_dtype -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import to_static_variable +from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + to_static_variable, +) from paddle.fluid.framework import core, Variable from paddle.fluid.layers import Assert, Print from paddle.fluid.layers import range as paddle_range -from paddle.fluid.layers import array_length, array_read, array_write, create_array -from paddle.fluid.layers import assign, fill_constant, slice, reduce_all, reduce_any -from paddle.fluid.layers import cast, control_flow, logical_and, logical_not, logical_or, nn -from paddle.fluid.layers.control_flow import cond, while_loop, less_than, increment -from paddle.fluid.dygraph.dygraph_to_static.return_transformer import RETURN_NO_VALUE_VAR_NAME -from paddle.fluid.dygraph.dygraph_to_static.utils import UndefinedVar, Dygraph2StaticException +from paddle.fluid.layers import ( + array_length, + array_read, + array_write, + create_array, +) +from paddle.fluid.layers import ( + assign, + fill_constant, + slice, + reduce_all, + reduce_any, +) +from paddle.fluid.layers import ( + cast, + control_flow, + logical_and, + logical_not, + logical_or, + nn, +) +from paddle.fluid.layers.control_flow import ( + cond, + while_loop, + less_than, + increment, +) +from paddle.fluid.dygraph.dygraph_to_static.return_transformer import ( + RETURN_NO_VALUE_VAR_NAME, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + UndefinedVar, + Dygraph2StaticException, +) from paddle.fluid.dygraph.dygraph_to_static.utils import GetterSetterHelper from paddle.fluid.layers.utils import copy_mutable_vars @@ -37,8 +67,10 @@ def convert_attr(x, attr): def indexable(x, code=None): - if isinstance(x, Variable): return x - if hasattr(x, '__len__') and hasattr(x, '__getitem__'): return x + if isinstance(x, Variable): + return x + if hasattr(x, '__len__') and hasattr(x, '__getitem__'): + return x if hasattr(x, '__iter__'): return [i for i in x] else: @@ -46,8 +78,7 @@ def indexable(x, code=None): def unpack_by_structure(target, structure): - """ unified unpack interface for paddle and python. - """ + """unified unpack interface for paddle and python.""" if isinstance(target, Variable): return _unpack_by_structure_paddle(target, structure) else: @@ -55,8 +86,7 @@ def unpack_by_structure(target, structure): def _unpack_by_structure_python(target, structure): - """ TODO(xiongkun): analysis the differences between python and paddle unpack. - """ + """TODO(xiongkun): analysis the differences between python and paddle unpack.""" return _unpack_by_structure_paddle(target, structure) @@ -75,12 +105,9 @@ def _unpack_by_structure_paddle(target, structure): return ret -def convert_while_loop(cond, - body, - getter, - setter, - return_name_ids=None, - push_pop_names=None): +def convert_while_loop( + cond, body, getter, setter, return_name_ids=None, push_pop_names=None +): """ A function representation of a Python ``while`` statement. @@ -100,8 +127,9 @@ def convert_while_loop(cond, # If loop_vars is changed during cond callable, then it causes bug, but current logical_and/logical_not/... doesn't change the loop_vars. pred = cond() if isinstance(pred, Variable): - _run_paddle_while(cond, body, getter, setter, return_name_ids, - push_pop_names) + _run_paddle_while( + cond, body, getter, setter, return_name_ids, push_pop_names + ) else: _run_py_while(cond, body, getter, setter) @@ -117,19 +145,21 @@ def maybe_to_tensor_array(v): else: return v - setterhelper.set(push_pop_names, - [maybe_to_tensor_array(v) for v in push_pop_vars]) + setterhelper.set( + push_pop_names, [maybe_to_tensor_array(v) for v in push_pop_vars] + ) -def _run_paddle_while(cond, body, getter, setter, return_name_ids, - push_pop_names): +def _run_paddle_while( + cond, body, getter, setter, return_name_ids, push_pop_names +): # NOTE: loop_vars of Paddle op `control_flow.while_loop` must be Paddle Tensors. helper = GetterSetterHelper(getter, setter, return_name_ids, push_pop_names) _convert_tensor_arrray_if_necessary(helper, push_pop_names) def new_body_fn(*args): - """ wrap the body() and add return value for `while_loop` - the args may be differ from getter(). + """wrap the body() and add return value for `while_loop` + the args may be differ from getter(). """ mutable_loop_vars = args helper.set(return_name_ids, mutable_loop_vars) @@ -137,8 +167,8 @@ def new_body_fn(*args): return helper.get(return_name_ids) def new_cond_fn(*args): - """ cond is a zero-args function, which is not - compatible with `while_loop`. + """cond is a zero-args function, which is not + compatible with `while_loop`. """ return cond() @@ -147,8 +177,9 @@ def new_cond_fn(*args): to_static_variable(var) if not isinstance(var, UndefinedVar) else var for var in helper.get(return_name_ids) ] - helper.set(return_name_ids, - loop_vars) # change the non-local var to variable + helper.set( + return_name_ids, loop_vars + ) # change the non-local var to variable # variable maybe modified to inner var. change it into loop_vars = control_flow.while_loop(new_cond_fn, new_body_fn, loop_vars) helper.set(return_name_ids, loop_vars) @@ -160,8 +191,10 @@ def _run_py_while(cond, body, getter, setter): pred = cond() if isinstance(pred, Variable): raise Dygraph2StaticException( - "python while pred change from bool to variable.") - if not pred: break + "python while pred change from bool to variable." + ) + if not pred: + break body() @@ -293,13 +326,15 @@ def _run_py_logical_not(x): return not x -def convert_ifelse(pred, - true_fn, - false_fn, - get_args, - set_args, - return_name_ids, - push_pop_names=None): +def convert_ifelse( + pred, + true_fn, + false_fn, + get_args, + set_args, + return_name_ids, + push_pop_names=None, +): """ A function representation of a Python ``if/else`` statement. @@ -317,63 +352,84 @@ def convert_ifelse(pred, """ if isinstance(pred, Variable): - out = _run_paddle_cond(pred, true_fn, false_fn, get_args, set_args, - return_name_ids, push_pop_names) + out = _run_paddle_cond( + pred, + true_fn, + false_fn, + get_args, + set_args, + return_name_ids, + push_pop_names, + ) else: - out = _run_py_ifelse(pred, true_fn, false_fn, get_args, set_args, - return_name_ids) + out = _run_py_ifelse( + pred, true_fn, false_fn, get_args, set_args, return_name_ids + ) return out -def _run_paddle_cond(pred, true_fn, false_fn, get_args, set_args, - return_name_ids, push_pop_names): +def _run_paddle_cond( + pred, true_fn, false_fn, get_args, set_args, return_name_ids, push_pop_names +): """ Paddle cond API will evaluate both ture_fn and false_fn codes. """ - helper = GetterSetterHelper(get_args, set_args, return_name_ids, - push_pop_names) + helper = GetterSetterHelper( + get_args, set_args, return_name_ids, push_pop_names + ) _convert_tensor_arrray_if_necessary(helper, push_pop_names) pred = cast_bool_if_necessary(pred) init_args = helper.get(return_name_ids) def new_true_fn(): - #init args may contain mutable python container like [var, 2], we copy then like in while_loop + # init args may contain mutable python container like [var, 2], we copy then like in while_loop helper.set(return_name_ids, copy_mutable_vars(init_args)) ret = true_fn() # IfExpr will return a non-None return value, so we just return ret. # We assume normal return has no return value. - if ret is None: return helper.get(return_name_ids) - else: return ret + if ret is None: + return helper.get(return_name_ids) + else: + return ret def new_false_fn(): - #init args may contain mutable python container like [var, 2], we copy then like in while_loop + # init args may contain mutable python container like [var, 2], we copy then like in while_loop helper.set(return_name_ids, copy_mutable_vars(init_args)) ret = false_fn() - if ret is None: return helper.get(return_name_ids) - else: return ret + if ret is None: + return helper.get(return_name_ids) + else: + return ret try: - cond_outs = control_flow.cond(pred, new_true_fn, new_false_fn, None, - return_name_ids) + cond_outs = control_flow.cond( + pred, new_true_fn, new_false_fn, None, return_name_ids + ) except Exception as e: - if re.search("Unsupported return type of true_fn and false_fn in cond", - str(e)): + if re.search( + "Unsupported return type of true_fn and false_fn in cond", str(e) + ): raise Dygraph2StaticException( - "Your if/else have different return type. TODO: add link to modifty. {}" - .format(str(e))) + "Your if/else have different return type. TODO: add link to modifty. {}".format( + str(e) + ) + ) if re.search("Incompatible return values of", str(e)): raise Dygraph2StaticException( - "Your if/else have different number of return value. TODO: add link to modifty. {}" - .format(str(e))) + "Your if/else have different number of return value. TODO: add link to modifty. {}".format( + str(e) + ) + ) raise e get_args = lambda: helper.get(return_name_ids) set_args = lambda vs: helper.set(return_name_ids, vs) return _recover_args_state(cond_outs, get_args, set_args, return_name_ids) -def _run_py_ifelse(pred, true_fn, false_fn, get_args, set_args, - return_name_ids): +def _run_py_ifelse( + pred, true_fn, false_fn, get_args, set_args, return_name_ids +): """ Evaluate python original branch function if-else. """ @@ -387,20 +443,22 @@ def _remove_no_value_return_var(out): align_ret = out[0] if isinstance(align_ret, tuple): for index, item in enumerate(align_ret): - if isinstance(item, Variable) and (RETURN_NO_VALUE_VAR_NAME - in item.name): + if isinstance(item, Variable) and ( + RETURN_NO_VALUE_VAR_NAME in item.name + ): # return None if index == 0: - processed_out = (None, ) + out[1:] + processed_out = (None,) + out[1:] elif index == 1: processed_out = align_ret[:1] + out[1:] else: - processed_out = (align_ret[:index], ) + out[1:] + processed_out = (align_ret[:index],) + out[1:] break for index, item in enumerate(processed_out): - if isinstance(item, Variable) and (RETURN_NO_VALUE_VAR_NAME - in item.name): + if isinstance(item, Variable) and ( + RETURN_NO_VALUE_VAR_NAME in item.name + ): processed_out = processed_out[:index] if not processed_out: @@ -415,14 +473,17 @@ def _remove_no_value_return_var(out): def _check_no_undefined_var(outs, names, branch_name): - if names is None: return + if names is None: + return if not isinstance(outs, (list, tuple)): outs = [outs] for var, name in zip(list(outs), names): if isinstance(var, UndefinedVar): raise ValueError( - "Required '{}' must be initialized both in if-else branch, but found it not initialized in '{}'." - .format(name, branch_name)) + "Required '{}' must be initialized both in if-else branch, but found it not initialized in '{}'.".format( + name, branch_name + ) + ) def _recover_args_state(outs, get_args, set_args, return_name_ids): @@ -443,10 +504,11 @@ def _recover_args_state(outs, get_args, set_args, return_name_ids): assert num_outs <= num_args if num_args == 1: - final_outs = (outs, ) if not isinstance(outs, - (list, tuple)) else tuple(outs) + final_outs = ( + (outs,) if not isinstance(outs, (list, tuple)) else tuple(outs) + ) else: - outs = (outs, ) if num_outs == 1 else tuple(outs) + outs = (outs,) if num_outs == 1 else tuple(outs) final_outs = outs + init_args[num_outs:] set_args(final_outs) @@ -464,8 +526,8 @@ def convert_len(var): if isinstance(var, Variable): assert var.ndim > 0, "len() of a 0D tensor is wrong" if var.type in [ - core.VarDesc.VarType.LOD_TENSOR, - core.VarDesc.VarType.SELECTED_ROWS + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.SELECTED_ROWS, ]: # Note: Length of var may be known ahead of time in dygraph, # but it probably represents batch size which can be variant. @@ -478,7 +540,8 @@ def convert_len(var): else: raise TypeError( 'len(var) only supports LoDTensor/LoDTensorArray/SelectedRows, but received %s.' - % type(var)) + % type(var) + ) else: if isinstance(var, VariableTuple): return var.__len__() @@ -490,16 +553,17 @@ def convert_zip(*args): if isinstance(arg, Variable) and arg.shape[0] == -1: raise RuntimeError( "Not support zip(tensor, ...) when tensor.shape[0] == -1, " - "but found args[{}].shape[0] == -1 in 'zip'".format(str(i))) + "but found args[{}].shape[0] == -1 in 'zip'".format(str(i)) + ) return zip(*args) # TODO(xiongkun): delete when list is ready. class VariableTuple: """ - this class will cause enumerate can't be wrapped by other iterator change function. - this will be fixed when list is producted. - VariableTuple can only deal with variables which is fixed. + this class will cause enumerate can't be wrapped by other iterator change function. + this will be fixed when list is producted. + VariableTuple can only deal with variables which is fixed. """ def __init__(self, var, start=0): @@ -527,7 +591,8 @@ def convert_enumerate(*args): def convert_range(*args): has_variable = any(map(lambda x: isinstance(x, Variable), args)) if has_variable: - if len(args) == 1: return paddle_range(0, args[0], 1, paddle.int64) + if len(args) == 1: + return paddle_range(0, args[0], 1, paddle.int64) if len(args) == 2: return paddle_range(args[0], args[1], 1, paddle.int64) if len(args) == 3: @@ -581,8 +646,12 @@ def convert_shape_compare(left, *args): """ args_len = len(args) - assert args_len >= 2, "convert_shape_compare needs at least one right compare variable" - assert args_len % 2 == 0, "Illegal input for convert_shape_compare, *args should be op(str), var, op(str), var ..." + assert ( + args_len >= 2 + ), "convert_shape_compare needs at least one right compare variable" + assert ( + args_len % 2 == 0 + ), "Illegal input for convert_shape_compare, *args should be op(str), var, op(str), var ..." num_cmp = args_len // 2 if isinstance(left, Variable): @@ -590,7 +659,12 @@ def reduce_compare(x, op_str, y): element_wise_result = eval("x " + op_str + " y") if op_str == "!=": return reduce_any(element_wise_result) - elif op_str == "is" or op_str == "is not" or op_str == "in" or op_str == "not in": + elif ( + op_str == "is" + or op_str == "is not" + or op_str == "in" + or op_str == "not in" + ): return element_wise_result else: return reduce_all(element_wise_result) @@ -601,8 +675,9 @@ def reduce_compare(x, op_str, y): cmp_op = args[i * 2] cmp_right = args[i * 2 + 1] cur_result = reduce_compare(cmp_left, cmp_op, cmp_right) - final_result = convert_logical_and(lambda: final_result, - lambda: cur_result) + final_result = convert_logical_and( + lambda: final_result, lambda: cur_result + ) return final_result else: cmp_left = left @@ -633,13 +708,23 @@ def convert_var_dtype(var, dtype): if isinstance(var, Variable): src_dtype = convert_dtype(var.dtype) assert src_dtype in [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'uint8' + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'uint8', ], "The dtype of var {} is {}, which is not supported in the cast op.".format( - var.name, src_dtype) + var.name, src_dtype + ) assert dtype in [ - 'bool', 'int', 'float' + 'bool', + 'int', + 'float', ], "The casted target dtype is {}, which is not supported in type casting.".format( - dtype) + dtype + ) cast_map = { 'bool': 'bool', 'int': 'int32', @@ -733,7 +818,6 @@ def body(i, new_array): # TODO(liym27): A better way to slice tensor array. # Maybe support start == end for slice op. def _slice_tensor_array(array, start, end): - def true_fn(): null_array = create_array("float32") return null_array diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/create_variable_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/create_variable_transformer.py index 78ca606dcec42..3432765191fdd 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/create_variable_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/create_variable_transformer.py @@ -13,21 +13,28 @@ # limitations under the License. from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper -from paddle.fluid.dygraph.dygraph_to_static.utils import FunctionNameLivenessAnalysis -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_undefined_var -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FunctionNameLivenessAnalysis, +) +from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + create_undefined_var, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) class CreateVariableTransformer(BaseTransformer): - """ - """ + """ """ def __init__(self, wrapper_root): - assert isinstance( - wrapper_root, AstNodeWrapper - ), "Type of input node should be AstNodeWrapper, but received %s ." % type( - wrapper_root) + assert isinstance(wrapper_root, AstNodeWrapper), ( + "Type of input node should be AstNodeWrapper, but received %s ." + % type(wrapper_root) + ) self.root = wrapper_root.node FunctionNameLivenessAnalysis(self.root) @@ -38,7 +45,7 @@ def transform(self): self.visit(self.root) def visit_FunctionDef(self, node): - #attributes = set(filter(lambda x: '.' in x, node.pd_scope.modified_vars())) + # attributes = set(filter(lambda x: '.' in x, node.pd_scope.modified_vars())) self.generic_visit(node) bodys = node.body names = sorted(node.pd_scope.created_vars()) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/decorator_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/decorator_transformer.py index db88d1670e1b4..55ca1f9d9aca1 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/decorator_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/decorator_transformer.py @@ -14,17 +14,31 @@ # limitations under the License. from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer -from paddle.fluid.dygraph.dygraph_to_static.utils import create_funcDef_node, ast_to_source_code, is_paddle_api, Dygraph2StaticException +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + create_funcDef_node, + ast_to_source_code, + is_paddle_api, + Dygraph2StaticException, +) import warnings import re from paddle.fluid.dygraph.dygraph_to_static.utils import RE_PYNAME, RE_PYMODULE IGNORE_NAMES = [ - 'declarative', 'to_static', 'dygraph_to_static_func', 'wraps', - 'staticmethod', 'classmethod', 'decorator' + 'declarative', + 'to_static', + 'dygraph_to_static_func', + 'wraps', + 'staticmethod', + 'classmethod', + 'decorator', ] @@ -34,10 +48,10 @@ class DecoratorTransformer(BaseTransformer): """ def __init__(self, wrapper_root): - assert isinstance( - wrapper_root, AstNodeWrapper - ), "Type of input node should be AstNodeWrapper, but received %s ." % type( - wrapper_root) + assert isinstance(wrapper_root, AstNodeWrapper), ( + "Type of input node should be AstNodeWrapper, but received %s ." + % type(wrapper_root) + ) self.root = wrapper_root.node self.ancestor_nodes = [] @@ -70,16 +84,21 @@ def visit_FunctionDef(self, node): # 1: @_jst.Call(a.b.c.d.deco)() # 2: @q.w.e.r.deco() re_tmp = re.match( - r'({module})*({name}\(){{0,1}}({module})*({name})(\)){{0,1}}\(.*$' - .format(name=RE_PYNAME, module=RE_PYMODULE), deco_full_name) + r'({module})*({name}\(){{0,1}}({module})*({name})(\)){{0,1}}\(.*$'.format( + name=RE_PYNAME, module=RE_PYMODULE + ), + deco_full_name, + ) deco_name = re_tmp.group(4) else: # match case like: # @a.d.g.deco re_tmp = re.match( - r'({module})*({name})$'.format(name=RE_PYNAME, - module=RE_PYMODULE), - deco_full_name) + r'({module})*({name})$'.format( + name=RE_PYNAME, module=RE_PYMODULE + ), + deco_full_name, + ) deco_name = re_tmp.group(2) if deco_name in IGNORE_NAMES: continue @@ -95,25 +114,37 @@ def visit_FunctionDef(self, node): if '_jst.Call' in deco_full_name: # in this case , the deco_full_name will be like: # '_jst.Call(deco)(5)' - rematch = re.match(r'\_jst\.Call\((.+?)\)\((.*)\)', - deco_full_name) + rematch = re.match( + r'\_jst\.Call\((.+?)\)\((.*)\)', deco_full_name + ) re_name = rematch.group(1) re_args = rematch.group(2) re_args_with_func = deco_target + ', ' + re_args - decofun_str = 'try:\n\t{0} = _jst.Call({1})({2})\nexcept:\n\t{0} = _jst.Call({1})({3})({4})'\ - .format(decoed_func, re_name, re_args_with_func, re_args, deco_target) + decofun_str = 'try:\n\t{0} = _jst.Call({1})({2})\nexcept:\n\t{0} = _jst.Call({1})({3})({4})'.format( + decoed_func, + re_name, + re_args_with_func, + re_args, + deco_target, + ) else: # paddle api will not be transformed to '_jst.Call' rematch = re.match(r'(.+?)\((.*)\)', deco_full_name) re_name = rematch.group(1) re_args = rematch.group(2) re_args_with_func = deco_target + ', ' + re_args - decofun_str = 'try:\n\t{0} = {1}({2})\nexcept:\n\t{0} = {1}({3})({4})'\ - .format(decoed_func, re_name, re_args_with_func, re_args, deco_target) + decofun_str = 'try:\n\t{0} = {1}({2})\nexcept:\n\t{0} = {1}({3})({4})'.format( + decoed_func, + re_name, + re_args_with_func, + re_args, + deco_target, + ) else: decofun_str = '{} = _jst.Call({})({})'.format( - decoed_func, deco_full_name, deco_target) + decoed_func, deco_full_name, deco_target + ) decofun_nodes.extend(gast.parse(decofun_str).body) deco_target = decoed_func @@ -121,12 +152,14 @@ def visit_FunctionDef(self, node): if not decofun_nodes: return node - orig_func_node = gast.FunctionDef(name='_orig_' + node.name, - args=node.args, - body=node.body, - decorator_list=[], - returns=None, - type_comment=None) + orig_func_node = gast.FunctionDef( + name='_orig_' + node.name, + args=node.args, + body=node.body, + decorator_list=[], + returns=None, + type_comment=None, + ) args = [arg.id for arg in node.args.args] arg_str = ','.join(args) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/early_return_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/early_return_transformer.py index e20f50b609e92..1ce75b277864e 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/early_return_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/early_return_transformer.py @@ -13,8 +13,12 @@ # limitations under the License. from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) class EarlyReturnTransformer(BaseTransformer): @@ -23,10 +27,10 @@ class EarlyReturnTransformer(BaseTransformer): """ def __init__(self, wrapper_root): - assert isinstance( - wrapper_root, AstNodeWrapper - ), "Type of input node should be AstNodeWrapper, but received %s ." % type( - wrapper_root) + assert isinstance(wrapper_root, AstNodeWrapper), ( + "Type of input node should be AstNodeWrapper, but received %s ." + % type(wrapper_root) + ) self.root = wrapper_root.node def transform(self): @@ -39,7 +43,8 @@ def is_define_return_in_if(self, node): assert isinstance( node, gast.If ), "Type of input node should be gast.If, but received %s ." % type( - node) + node + ) for child in node.body: if isinstance(child, gast.Return): return True @@ -60,9 +65,11 @@ def visit_block_nodes(self, nodes): if isinstance(node, gast.If) and self.is_define_return_in_if(node): destination_nodes = node.orelse # handle stmt like `if/elif/elif` - while len(destination_nodes) > 0 and \ - isinstance(destination_nodes[0], gast.If) and \ - self.is_define_return_in_if(destination_nodes[0]): + while ( + len(destination_nodes) > 0 + and isinstance(destination_nodes[0], gast.If) + and self.is_define_return_in_if(destination_nodes[0]) + ): destination_nodes = destination_nodes[0].orelse return result_nodes diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/error.py b/python/paddle/fluid/dygraph/dygraph_to_static/error.py index d05a8ce4d8421..7e05b5db891f3 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/error.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/error.py @@ -19,8 +19,15 @@ import re import numpy as np -from paddle.fluid.dygraph.dygraph_to_static.origin_info import Location, OriginInfo, global_origin_info_map -from paddle.fluid.dygraph.dygraph_to_static.utils import _is_api_in_module_helper, RE_PYMODULE +from paddle.fluid.dygraph.dygraph_to_static.origin_info import ( + Location, + OriginInfo, + global_origin_info_map, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + _is_api_in_module_helper, + RE_PYMODULE, +) ERROR_DATA = "Error data about original source code information and traceback." @@ -72,10 +79,17 @@ def __init__(self, location, function_name, source_code): def formated_message(self): # self.source_code may be empty in some functions. # For example, decorator generated function - return ' ' * BLANK_COUNT_BEFORE_FILE_STR + 'File "{}", line {}, in {}\n\t{}'.format( - self.location.filepath, self.location.lineno, self.function_name, - self.source_code.lstrip() - if isinstance(self.source_code, str) else self.source_code) + return ( + ' ' * BLANK_COUNT_BEFORE_FILE_STR + + 'File "{}", line {}, in {}\n\t{}'.format( + self.location.filepath, + self.location.lineno, + self.function_name, + self.source_code.lstrip() + if isinstance(self.source_code, str) + else self.source_code, + ) + ) class TraceBackFrameRange(OriginInfo): @@ -118,26 +132,35 @@ def __init__(self, location, function_name): for i in range(len(self.source_code)): # if source_code[i] is empty line between two code line, dont add blank if self.source_code[i]: - self.source_code[i] = ' ' * ( - blank_count[i] - min_black_count + - BLANK_COUNT_BEFORE_FILE_STR * 2) + self.source_code[i] + self.source_code[i] = ( + ' ' + * ( + blank_count[i] + - min_black_count + + BLANK_COUNT_BEFORE_FILE_STR * 2 + ) + + self.source_code[i] + ) def formated_message(self): - msg = ' ' * BLANK_COUNT_BEFORE_FILE_STR + 'File "{}", line {}, in {}\n'.format( - self.location.filepath, self.location.lineno, self.function_name) + msg = ( + ' ' * BLANK_COUNT_BEFORE_FILE_STR + + 'File "{}", line {}, in {}\n'.format( + self.location.filepath, self.location.lineno, self.function_name + ) + ) # add empty line after range code return msg + '\n'.join(self.source_code) class SuggestionDict(object): - def __init__(self): # {(keywords): (suggestions)} self.suggestion_dict = { - ('is not initialized.', 'Hint:', 'IsInitialized'): - ("Please ensure all your sublayers are inheritted from nn.Layer.", - "Please ensure there is no tensor created explicitly depended on external data, we suggest to register it as buffer tensor. See https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/04_dygraph_to_static/export_model/principle_cn.html#parameters-buffers for details" - ) + ('is not initialized.', 'Hint:', 'IsInitialized'): ( + "Please ensure all your sublayers are inheritted from nn.Layer.", + "Please ensure there is no tensor created explicitly depended on external data, we suggest to register it as buffer tensor. See https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/04_dygraph_to_static/export_model/principle_cn.html#parameters-buffers for details", + ) } def keys(self): @@ -156,8 +179,9 @@ class ErrorData(object): Error data attached to an exception which is raised in un-transformed code. """ - def __init__(self, error_type, error_value, origin_traceback, - origin_info_map): + def __init__( + self, error_type, error_value, origin_traceback, origin_info_map + ): self.error_type = error_type self.error_value = error_value self.origin_traceback = origin_traceback @@ -182,24 +206,30 @@ def numpy_api_check(self, format_exception, error_line): func_str = None for frame in tb: searched_name = re.search( - r'({module})*{name}'.format(module=RE_PYMODULE, - name=frame.name), error_line) + r'({module})*{name}'.format( + module=RE_PYMODULE, name=frame.name + ), + error_line, + ) if searched_name: func_str = searched_name.group(0) break try: - module_result = eval("_is_api_in_module_helper({}, '{}')".format( - func_str, "numpy")) - is_numpy_api_err = module_result or (func_str.startswith("numpy.") - or func_str.startswith("np.")) + module_result = eval( + "_is_api_in_module_helper({}, '{}')".format(func_str, "numpy") + ) + is_numpy_api_err = module_result or ( + func_str.startswith("numpy.") or func_str.startswith("np.") + ) except Exception: is_numpy_api_err = False if is_numpy_api_err and func_str: return [ - "TypeError: Code '{}' called numpy API {}, please use Paddle API to replace it." - .format(error_line, func_str), - " values will be changed to variables by dy2static, numpy api can not handle variables" + "TypeError: Code '{}' called numpy API {}, please use Paddle API to replace it.".format( + error_line, func_str + ), + " values will be changed to variables by dy2static, numpy api can not handle variables", ] else: return format_exception @@ -219,45 +249,56 @@ def create_message(self): # Simplify error value to improve readability if error is raised in runtime if self.in_runtime: if int( - os.getenv(SIMPLIFY_ERROR_ENV_NAME, - DEFAULT_SIMPLIFY_NEW_ERROR)): + os.getenv(SIMPLIFY_ERROR_ENV_NAME, DEFAULT_SIMPLIFY_NEW_ERROR) + ): self._simplify_error_value() message_lines.append(str(self.error_value)) return '\n'.join(message_lines) # Step2: Optimizes stack information with source code information of dygraph from user. user_code_traceback_index = [] - for i, (filepath, lineno, funcname, - code) in enumerate(self.origin_traceback): - dygraph_func_info = self.origin_info_map.get((filepath, lineno), - None) + for i, (filepath, lineno, funcname, code) in enumerate( + self.origin_traceback + ): + dygraph_func_info = self.origin_info_map.get( + (filepath, lineno), None + ) if dygraph_func_info: user_code_traceback_index.append(i) # Add user code traceback for i in user_code_traceback_index: filepath, lineno, funcname, code = self.origin_traceback[i] - dygraph_func_info = self.origin_info_map.get((filepath, lineno), - None) + dygraph_func_info = self.origin_info_map.get( + (filepath, lineno), None + ) if i == user_code_traceback_index[-1]: traceback_frame = TraceBackFrameRange( - dygraph_func_info.location, dygraph_func_info.function_name) + dygraph_func_info.location, dygraph_func_info.function_name + ) else: traceback_frame = TraceBackFrame( - dygraph_func_info.location, dygraph_func_info.function_name, - dygraph_func_info.source_code) + dygraph_func_info.location, + dygraph_func_info.function_name, + dygraph_func_info.source_code, + ) message_lines.append(traceback_frame.formated_message()) error_line = traceback_frame.error_line message_lines.append("") # Add paddle traceback after user code traceback - paddle_traceback_start_index = user_code_traceback_index[ - -1] + 1 if user_code_traceback_index else 0 + paddle_traceback_start_index = ( + user_code_traceback_index[-1] + 1 + if user_code_traceback_index + else 0 + ) for filepath, lineno, funcname, code in self.origin_traceback[ - paddle_traceback_start_index:]: - traceback_frame = TraceBackFrame(Location(filepath, lineno), - funcname, code) + paddle_traceback_start_index: + ]: + traceback_frame = TraceBackFrame( + Location(filepath, lineno), funcname, code + ) message_lines.append(traceback_frame.formated_message()) message_lines.append("") @@ -265,10 +306,12 @@ def create_message(self): # NOTE: `format_exception` is a list, its length is 1 in most cases, but sometimes its length # is gather than 1, for example, the error_type is IndentationError. format_exception = traceback.format_exception_only( - self.error_type, self.error_value) + self.error_type, self.error_value + ) if error_line is not None: - format_exception = self.numpy_api_check(format_exception, - error_line) + format_exception = self.numpy_api_check( + format_exception, error_line + ) error_message = [ " " * BLANK_COUNT_BEFORE_FILE_STR + line @@ -280,17 +323,23 @@ def create_message(self): def _create_revise_suggestion(self, bottom_error_message): revise_suggestions = [ - '', ' ' * BLANK_COUNT_BEFORE_FILE_STR + 'Revise suggestion: ' + '', + ' ' * BLANK_COUNT_BEFORE_FILE_STR + 'Revise suggestion: ', ] for keywords in self.suggestion_dict.keys(): contain_keywords = [ True for i in keywords if i in ''.join(bottom_error_message) ] if len(contain_keywords) == len( - keywords): # all keywords should be in bottom_error_message + keywords + ): # all keywords should be in bottom_error_message for suggestion in self.suggestion_dict[keywords]: - suggestion_msg = ' ' * BLANK_COUNT_BEFORE_FILE_STR * 2 + '{}. {}'.format( - str(len(revise_suggestions) - 1), suggestion) + suggestion_msg = ( + ' ' * BLANK_COUNT_BEFORE_FILE_STR * 2 + + '{}. {}'.format( + str(len(revise_suggestions) - 1), suggestion + ) + ) revise_suggestions.append(suggestion_msg) return revise_suggestions if len(revise_suggestions) > 2 else [] @@ -313,12 +362,12 @@ def _simplify_error_value(self): start_trace = "outputs = static_func(*inputs)" start_idx = error_value_lines_strip.index(start_trace) - error_value_lines = error_value_lines[start_idx + 1:] - error_value_lines_strip = error_value_lines_strip[start_idx + 1:] + error_value_lines = error_value_lines[start_idx + 1 :] + error_value_lines_strip = error_value_lines_strip[start_idx + 1 :] # use empty line to locate the bottom_error_message empty_line_idx = error_value_lines_strip.index('') - bottom_error_message = error_value_lines[empty_line_idx + 1:] + bottom_error_message = error_value_lines[empty_line_idx + 1 :] revise_suggestion = self._create_revise_suggestion(bottom_error_message) error_traceback = [] @@ -336,14 +385,18 @@ def _simplify_error_value(self): if error_value_lines_strip[i].startswith("File "): re_result = re.search(pattern, error_value_lines_strip[i]) tmp_filepath, lineno_str, function_name = re_result.groups() - code = error_value_lines_strip[ - i + 1] if i + 1 < len(error_value_lines_strip) else '' + code = ( + error_value_lines_strip[i + 1] + if i + 1 < len(error_value_lines_strip) + else '' + ) if static_info_map.get((tmp_filepath, int(lineno_str))): user_code_traceback_index.append(len(error_traceback)) error_traceback.append( - (tmp_filepath, int(lineno_str), function_name, code)) + (tmp_filepath, int(lineno_str), function_name, code) + ) error_frame = [] # Add user code traceback @@ -351,20 +404,27 @@ def _simplify_error_value(self): filepath, lineno, funcname, code = error_traceback[i] if i == user_code_traceback_index[-1]: traceback_frame = TraceBackFrameRange( - Location(filepath, lineno), funcname) + Location(filepath, lineno), funcname + ) else: - traceback_frame = TraceBackFrame(Location(filepath, lineno), - funcname, code) + traceback_frame = TraceBackFrame( + Location(filepath, lineno), funcname, code + ) error_frame.append(traceback_frame.formated_message()) error_frame.append("") # Add paddle traceback after user code traceback - paddle_traceback_start_index = user_code_traceback_index[ - -1] + 1 if user_code_traceback_index else 0 + paddle_traceback_start_index = ( + user_code_traceback_index[-1] + 1 + if user_code_traceback_index + else 0 + ) for filepath, lineno, funcname, code in error_traceback[ - paddle_traceback_start_index:]: - traceback_frame = TraceBackFrame(Location(filepath, lineno), - funcname, code) + paddle_traceback_start_index: + ]: + traceback_frame = TraceBackFrame( + Location(filepath, lineno), funcname, code + ) error_frame.append(traceback_frame.formated_message()) error_frame.append("") diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/function_spec.py b/python/paddle/fluid/dygraph/dygraph_to_static/function_spec.py index 730c02fbc5e91..039df7e84e433 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/function_spec.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/function_spec.py @@ -48,8 +48,9 @@ def __init__(self, function, input_spec=None): self._arg_names, self._default_kwargs = parse_arg_and_kwargs(function) # parse *args self.varargs_name = parse_varargs_name(function) - if self.varargs_name is not None and isinstance(function.__self__, - TranslatedLayer): + if self.varargs_name is not None and isinstance( + function.__self__, TranslatedLayer + ): self._arg_names += function.__self__._input_args_names def unified_args_and_kwargs(self, args, kwargs): @@ -72,8 +73,12 @@ def unified_args_and_kwargs(self, args, kwargs): """ if len(self._arg_names) < len(args): error_msg = "The decorated function `{}` requires {} arguments: {}, but received {} with {}.".format( - self._dygraph_function.__name__, len(self._arg_names), - self._arg_names, len(args), args) + self._dygraph_function.__name__, + len(self._arg_names), + self._arg_names, + len(args), + args, + ) if args and inspect.isclass(args[0]): error_msg += "\n\tMaybe the function has more than one decorator, we don't support this for now." raise NotImplementedError(error_msg) @@ -90,9 +95,13 @@ def unified_args_and_kwargs(self, args, kwargs): else: if arg_name not in self._default_kwargs: raise ValueError( - "`{}()` requires `{}` arguments, but not found in input `args`: {} and `kwargs`: {}." - .format(self._dygraph_function.__name__, arg_name, args, - kwargs)) + "`{}()` requires `{}` arguments, but not found in input `args`: {} and `kwargs`: {}.".format( + self._dygraph_function.__name__, + arg_name, + args, + kwargs, + ) + ) args.append(self._default_kwargs[arg_name]) return tuple(args), kwargs @@ -135,16 +144,20 @@ def args_to_input_spec(self, args, kwargs): # So we don't support to deal this case while specificing `input_spec` currently. if kwargs: raise ValueError( - "{} got unexpected keyword arguments: {}. Cannot trace the function when `input_spec` is specificed." - .format(self._dygraph_function.__name__, kwargs)) + "{} got unexpected keyword arguments: {}. Cannot trace the function when `input_spec` is specificed.".format( + self._dygraph_function.__name__, kwargs + ) + ) # Note: The length of `input_spec` can be greater than `args`, # because `args` may contains non-tensor value merged form `kwargs` # after `unified_args_and_kwargs`. if len(args) < len(self._input_spec): raise ValueError( - "Requires len(arguments) >= len(input_spec), but received len(args):{} < len(InputSpec): {}" - .format(len(args), len(self._input_spec))) + "Requires len(arguments) >= len(input_spec), but received len(args):{} < len(InputSpec): {}".format( + len(args), len(self._input_spec) + ) + ) # replace argument with corresponding InputSpec. args_with_spec = convert_to_input_spec(args, self._input_spec) @@ -154,8 +167,9 @@ def args_to_input_spec(self, args, kwargs): # If without specificing name in input_spec, add default name # according to argument name from decorated function. - args_with_spec = replace_spec_empty_name(self._arg_names, - args_with_spec) + args_with_spec = replace_spec_empty_name( + self._arg_names, args_with_spec + ) return args_with_spec, kwargs_with_spec @@ -182,7 +196,8 @@ def to_static_inputs_with_spec(self, input_with_spec, main_program): dtype=var_spec.dtype, is_data=True, need_check_feed=False, - stop_gradient=stop_gradient) + stop_gradient=stop_gradient, + ) else: feed_layer = var_spec inputs.append(feed_layer) @@ -195,15 +210,19 @@ def _verify_input_spec(self, input_spec): """ if not isinstance(input_spec, (tuple, list)): raise TypeError( - "The type(input_spec) should be one of (tuple, list), but received {}." - .format(type_name(input_spec))) + "The type(input_spec) should be one of (tuple, list), but received {}.".format( + type_name(input_spec) + ) + ) return tuple(input_spec) def __repr__(self): return "function: {}({}), input_spec: {}".format( - self._dygraph_function.__name__, ','.join(self._arg_names), - self._input_spec) + self._dygraph_function.__name__, + ','.join(self._arg_names), + self._input_spec, + ) @property def dygraph_function(self): @@ -242,8 +261,10 @@ def get_parameters(layer_instance, include_sublayer=True): params = layer_instance._parameters else: raise TypeError( - "Type of `layer_instance` should be nn.Layer, but received {}". - format(type_name(layer_instance))) + "Type of `layer_instance` should be nn.Layer, but received {}".format( + type_name(layer_instance) + ) + ) return params @@ -264,8 +285,10 @@ def get_buffers(layer_instance, include_sublayer=True): buffers = layer_instance._buffers else: raise TypeError( - "Type of `layer_instance` should be nn.Layer, but received {}". - format(type_name(layer_instance))) + "Type of `layer_instance` should be nn.Layer, but received {}".format( + type_name(layer_instance) + ) + ) return buffers @@ -284,12 +307,17 @@ def convert_to_input_spec(inputs, input_spec): def check_type_and_len(input, spec, check_length=False): if type(input) is not type(spec): - raise TypeError('type(input) should be {}, but received {}.'.format( - type(spec), type(input))) + raise TypeError( + 'type(input) should be {}, but received {}.'.format( + type(spec), type(input) + ) + ) if check_length and len(input) < len(spec): raise ValueError( - 'Requires len(inputs) >= len(input_spec), but received len(inputs):{} < len(input_spec):{}' - .format(len(inputs), len(input_spec))) + 'Requires len(inputs) >= len(input_spec), but received len(inputs):{} < len(input_spec):{}'.format( + len(inputs), len(input_spec) + ) + ) if isinstance(input_spec, (tuple, list)): input_with_spec = [] @@ -302,13 +330,15 @@ def check_type_and_len(input, spec, check_length=False): # Note: If the rest inputs contain tensor or numpy.ndarray # without specific InputSpec, raise warning. if len(inputs) > len(input_spec): - for rest_input in inputs[len(input_spec):]: + for rest_input in inputs[len(input_spec) :]: if isinstance(rest_input, (core.VarBase, np.ndarray)): logging_utils.warn( "The inputs constain `{}` without specificing InputSpec, its shape and dtype will be treated immutable. " - "Please specific InputSpec information in `@to_static` if you expect them as mutable inputs." - .format(type_name(rest_input))) - input_with_spec.extend(inputs[len(input_spec):]) + "Please specific InputSpec information in `@to_static` if you expect them as mutable inputs.".format( + type_name(rest_input) + ) + ) + input_with_spec.extend(inputs[len(input_spec) :]) return input_with_spec elif isinstance(input_spec, dict): @@ -317,7 +347,8 @@ def check_type_and_len(input, spec, check_length=False): for name, input in inputs.items(): if name in input_spec: input_with_spec[name] = convert_to_input_spec( - input, input_spec[name]) + input, input_spec[name] + ) else: input_with_spec[name] = input return input_with_spec @@ -354,7 +385,7 @@ def replace_spec_empty_name(args_name, input_with_spec): print([in_var.name for in_var in foo.inputs]) # [x, y] """ input_with_spec = list(input_with_spec) - candidate_arg_names = args_name[:len(input_with_spec)] + candidate_arg_names = args_name[: len(input_with_spec)] for i, arg_name in enumerate(candidate_arg_names): input_spec = input_with_spec[i] @@ -405,11 +436,13 @@ def _hash_spec_names(args_specs, kwargs_specs): the former has one input ('x'), but the latter has two input ('x', 'y'). """ spec_names = [ - spec.name for spec in flatten(args_specs) + spec.name + for spec in flatten(args_specs) if isinstance(spec, paddle.static.InputSpec) ] spec_names += [ - spec.name for spec in flatten(kwargs_specs) + spec.name + for spec in flatten(kwargs_specs) if isinstance(spec, paddle.static.InputSpec) ] i, name_ids = 0, {} diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py index 5d51b0f48c274..8120e79c1da30 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/ifelse_transformer.py @@ -23,16 +23,47 @@ from paddle.utils import gast from paddle.fluid import unique_name -from paddle.fluid.dygraph.dygraph_to_static.utils import create_funcDef_node, ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.utils import create_assign_node, FunctionNameLivenessAnalysis -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_undefined_var -from paddle.fluid.dygraph.dygraph_to_static.utils import create_nonlocal_stmt_nodes -from paddle.fluid.dygraph.dygraph_to_static.utils import create_get_args_node, create_set_args_node -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer -from paddle.fluid.dygraph.dygraph_to_static.utils import FOR_ITER_INDEX_PREFIX, FOR_ITER_TUPLE_PREFIX, FOR_ITER_TUPLE_INDEX_PREFIX, FOR_ITER_VAR_LEN_PREFIX, FOR_ITER_VAR_NAME_PREFIX, FOR_ITER_ZIP_TO_LIST_PREFIX, FOR_ITER_TARGET_PREFIX, FOR_ITER_ITERATOR_PREFIX -from paddle.fluid.dygraph.dygraph_to_static.utils import GetterSetterHelper, create_name_str +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + create_funcDef_node, + ast_to_source_code, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + create_assign_node, + FunctionNameLivenessAnalysis, +) +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + StaticAnalysisVisitor, +) +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) +from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + create_undefined_var, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + create_nonlocal_stmt_nodes, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + create_get_args_node, + create_set_args_node, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FOR_ITER_INDEX_PREFIX, + FOR_ITER_TUPLE_PREFIX, + FOR_ITER_TUPLE_INDEX_PREFIX, + FOR_ITER_VAR_LEN_PREFIX, + FOR_ITER_VAR_NAME_PREFIX, + FOR_ITER_ZIP_TO_LIST_PREFIX, + FOR_ITER_TARGET_PREFIX, + FOR_ITER_ITERATOR_PREFIX, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + GetterSetterHelper, + create_name_str, +) TRUE_FUNC_PREFIX = 'true_fn' FALSE_FUNC_PREFIX = 'false_fn' @@ -47,13 +78,14 @@ class IfElseTransformer(BaseTransformer): """ def __init__(self, wrapper_root): - assert isinstance( - wrapper_root, AstNodeWrapper - ), "Type of input node should be AstNodeWrapper, but received %s ." % type( - wrapper_root) + assert isinstance(wrapper_root, AstNodeWrapper), ( + "Type of input node should be AstNodeWrapper, but received %s ." + % type(wrapper_root) + ) self.root = wrapper_root.node FunctionNameLivenessAnalysis( - self.root) # name analysis of current ast tree. + self.root + ) # name analysis of current ast tree. def transform(self): """ @@ -63,16 +95,31 @@ def transform(self): def visit_If(self, node): self.generic_visit(node) - true_func_node, false_func_node, get_args_node, set_args_node, return_name_ids, push_pop_ids = transform_if_else( - node, self.root) - - new_node = create_convert_ifelse_node(return_name_ids, push_pop_ids, - node.test, true_func_node, - false_func_node, get_args_node, - set_args_node) - - return [get_args_node, set_args_node, true_func_node, false_func_node - ] + [new_node] + ( + true_func_node, + false_func_node, + get_args_node, + set_args_node, + return_name_ids, + push_pop_ids, + ) = transform_if_else(node, self.root) + + new_node = create_convert_ifelse_node( + return_name_ids, + push_pop_ids, + node.test, + true_func_node, + false_func_node, + get_args_node, + set_args_node, + ) + + return [ + get_args_node, + set_args_node, + true_func_node, + false_func_node, + ] + [new_node] def visit_Call(self, node): # Remove `numpy()` statement, like `Tensor.numpy()[i]` -> `Tensor[i]` @@ -89,8 +136,9 @@ def visit_IfExp(self, node): """ self.generic_visit(node) - new_node = create_convert_ifelse_node(None, None, node.test, node.body, - node.orelse, None, None, True) + new_node = create_convert_ifelse_node( + None, None, node.test, node.body, node.orelse, None, None, True + ) # Note: A blank line will be added separately if transform gast.Expr # into source code. Using gast.Expr.value instead to avoid syntax error # in python. @@ -101,7 +149,6 @@ def visit_IfExp(self, node): class NameVisitor(gast.NodeVisitor): - def __init__(self, after_node=None, end_node=None): # The start node (exclusive) of the visitor self.after_node = after_node @@ -171,7 +218,8 @@ def visit_If(self, node): # Blocks the vars in `if.body` and only inserts the vars both created in 'if/else' branch # into name_ids. new_name_ids = self._find_new_name_ids( - body_name_ids, else_name_ids) + body_name_ids, else_name_ids + ) for new_name_id in new_name_ids: before_if_name_ids[new_name_id].append(gast.Store()) @@ -186,7 +234,8 @@ def visit_Name(self, node): self.generic_visit(node) return blacklist = {'True', 'False', 'None'} - if node.id in blacklist: return + if node.id in blacklist: + return if node.id in self._def_func_names: return if not self._is_call_func_name_node(node): @@ -205,7 +254,10 @@ def visit_FunctionDef(self, node): # NOTE: We skip to visit names of get_args and set_args, because they contains # nonlocal statement such as 'nonlocal x, self' where 'self' should not be # parsed as returned value in contron flow. - if GET_ARGS_FUNC_PREFIX in node.name or SET_ARGS_FUNC_PREFIX in node.name: + if ( + GET_ARGS_FUNC_PREFIX in node.name + or SET_ARGS_FUNC_PREFIX in node.name + ): return if not self._in_range: @@ -236,7 +288,6 @@ def _visit_child(self, node): return copy.deepcopy(self.name_ids) def _find_new_name_ids(self, body_name_ids, else_name_ids): - def is_required_ctx(ctxs, required_ctx): for ctx in ctxs: if isinstance(ctx, required_ctx): @@ -244,13 +295,14 @@ def is_required_ctx(ctxs, required_ctx): return False candidate_name_ids = set(body_name_ids.keys()) & set( - else_name_ids.keys()) + else_name_ids.keys() + ) store_ctx = gast.Store new_name_ids = set() for name_id in candidate_name_ids: - if is_required_ctx(body_name_ids[name_id], - store_ctx) and is_required_ctx( - else_name_ids[name_id], store_ctx): + if is_required_ctx( + body_name_ids[name_id], store_ctx + ) and is_required_ctx(else_name_ids[name_id], store_ctx): new_name_ids.add(name_id) return new_name_ids @@ -262,8 +314,10 @@ def _is_call_func_name_node(self, node): parent_node = self.ancestor_nodes[-2] if isinstance(parent_node, gast.Call) and parent_node.func == node: # e.g: var_list.append(elem), var_list is also a name_id. - should_skip = isinstance( - node, gast.Attribute) and node.attr in white_func_names + should_skip = ( + isinstance(node, gast.Attribute) + and node.attr in white_func_names + ) if not should_skip: return True return False @@ -290,8 +344,10 @@ def _valid_nonlocal_names(return_name_ids, nonlocal_names): for name in return_name_ids: if name not in nonlocal_names: raise ValueError( - "Required returned var '{}' must be in 'nonlocal' statement '', but not found." - .format(name)) + "Required returned var '{}' must be in 'nonlocal' statement '', but not found.".format( + name + ) + ) nonlocal_names.remove(name) return return_name_ids + nonlocal_names @@ -313,15 +369,21 @@ def transform_if_else(node, root): # TODO(dev): Need a better way to deal this. # LoopTransformer will create some special vars, which is not visiable by users. so we can sure it's safe to remove them. filter_names = [ - ARGS_NAME, FOR_ITER_INDEX_PREFIX, FOR_ITER_TUPLE_PREFIX, - FOR_ITER_TARGET_PREFIX, FOR_ITER_ITERATOR_PREFIX, - FOR_ITER_TUPLE_INDEX_PREFIX, FOR_ITER_VAR_LEN_PREFIX, - FOR_ITER_VAR_NAME_PREFIX, FOR_ITER_ZIP_TO_LIST_PREFIX + ARGS_NAME, + FOR_ITER_INDEX_PREFIX, + FOR_ITER_TUPLE_PREFIX, + FOR_ITER_TARGET_PREFIX, + FOR_ITER_ITERATOR_PREFIX, + FOR_ITER_TUPLE_INDEX_PREFIX, + FOR_ITER_VAR_LEN_PREFIX, + FOR_ITER_VAR_NAME_PREFIX, + FOR_ITER_ZIP_TO_LIST_PREFIX, ] def remove_if(x): for name in filter_names: - if x.startswith(name): return False + if x.startswith(name): + return False return True nonlocal_names = list(filter(remove_if, nonlocal_names)) @@ -329,40 +391,53 @@ def remove_if(x): nonlocal_stmt_node = create_nonlocal_stmt_nodes(nonlocal_names) - empty_arg_node = gast.arguments(args=[], - posonlyargs=[], - vararg=None, - kwonlyargs=[], - kw_defaults=None, - kwarg=None, - defaults=[]) + empty_arg_node = gast.arguments( + args=[], + posonlyargs=[], + vararg=None, + kwonlyargs=[], + kw_defaults=None, + kwarg=None, + defaults=[], + ) true_func_node = create_funcDef_node( nonlocal_stmt_node + node.body, name=unique_name.generate(TRUE_FUNC_PREFIX), input_args=empty_arg_node, - return_name_ids=[]) + return_name_ids=[], + ) false_func_node = create_funcDef_node( nonlocal_stmt_node + node.orelse, name=unique_name.generate(FALSE_FUNC_PREFIX), input_args=empty_arg_node, - return_name_ids=[]) + return_name_ids=[], + ) helper = GetterSetterHelper(None, None, nonlocal_names, push_pop_ids) get_args_node = create_get_args_node(helper.union()) set_args_node = create_set_args_node(helper.union()) - return true_func_node, false_func_node, get_args_node, set_args_node, return_name_ids, push_pop_ids - - -def create_convert_ifelse_node(return_name_ids, - push_pop_ids, - pred, - true_func, - false_func, - get_args_func, - set_args_func, - is_if_expr=False): + return ( + true_func_node, + false_func_node, + get_args_node, + set_args_node, + return_name_ids, + push_pop_ids, + ) + + +def create_convert_ifelse_node( + return_name_ids, + push_pop_ids, + pred, + true_func, + false_func, + get_args_func, + set_args_func, + is_if_expr=False, +): """ Create `paddle.jit.dy2static.convert_ifelse( pred, true_fn, false_fn, get_args, set_args, return_name_ids)` @@ -377,16 +452,19 @@ def create_convert_ifelse_node(return_name_ids, convert_ifelse_layer = gast.parse( '_jst.IfElse(' - '{pred}, {true_fn}, {false_fn}, {get_args}, {set_args}, {return_name_ids}, push_pop_names={push_pop_ids})' - .format( + '{pred}, {true_fn}, {false_fn}, {get_args}, {set_args}, {return_name_ids}, push_pop_names={push_pop_ids})'.format( pred=ast_to_source_code(pred), true_fn=true_func_source, false_fn=false_func_source, - get_args=get_args_func.name if not is_if_expr else - 'lambda: None', #TODO: better way to deal with this + get_args=get_args_func.name + if not is_if_expr + else 'lambda: None', # TODO: better way to deal with this set_args=set_args_func.name - if not is_if_expr else 'lambda args: None', + if not is_if_expr + else 'lambda args: None', return_name_ids=create_name_str(return_name_ids), - push_pop_ids=create_name_str(push_pop_ids))).body[0] + push_pop_ids=create_name_str(push_pop_ids), + ) + ).body[0] return convert_ifelse_layer diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/logging_utils.py b/python/paddle/fluid/dygraph/dygraph_to_static/logging_utils.py index f0a4672794c3c..6f73a23316d27 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/logging_utils.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/logging_utils.py @@ -29,7 +29,6 @@ def synchronized(func): - def wrapper(*args, **kwargs): with threading.Lock(): return func(*args, **kwargs) @@ -59,7 +58,8 @@ def __init__(self): self._logger = log_helper.get_logger( self.logger_name, 1, - fmt='%(asctime)s %(name)s %(levelname)s: %(message)s') + fmt='%(asctime)s %(name)s %(levelname)s: %(message)s', + ) self._verbosity_level = None self._transformed_code_level = None self._need_to_echo_log_to_stdout = None @@ -155,16 +155,19 @@ def log(self, level, msg, *args, **kwargs): if self.need_to_echo_log_to_stdout: self._output_to_stdout('INFO: ' + msg_with_level, *args) - def log_transformed_code(self, level, ast_node, transformer_name, *args, - **kwargs): + def log_transformed_code( + self, level, ast_node, transformer_name, *args, **kwargs + ): if self.has_code_level(level): source_code = ast_to_source_code(ast_node) if level == LOG_AllTransformer: - header_msg = "After the last level ast transformer: '{}', the transformed code:\n" \ - .format(transformer_name) + header_msg = "After the last level ast transformer: '{}', the transformed code:\n".format( + transformer_name + ) else: - header_msg = "After the level {} ast transformer: '{}', the transformed code:\n"\ - .format(level, transformer_name) + header_msg = "After the level {} ast transformer: '{}', the transformed code:\n".format( + level, transformer_name + ) msg = header_msg + source_code self.logger.info(msg, *args, **kwargs) @@ -271,5 +274,6 @@ def log(level, msg, *args, **kwargs): def log_transformed_code(level, ast_node, transformer_name, *args, **kwargs): - _TRANSLATOR_LOGGER.log_transformed_code(level, ast_node, transformer_name, - *args, **kwargs) + _TRANSLATOR_LOGGER.log_transformed_code( + level, ast_node, transformer_name, *args, **kwargs + ) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py index b7006d935ec9f..3ad623a8ff085 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/logical_transformer.py @@ -14,7 +14,9 @@ from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) cmpop_type_to_str = { gast.Eq: "==", @@ -26,7 +28,7 @@ gast.Is: "is", gast.IsNot: "is not", gast.In: "in", - gast.NotIn: "not in" + gast.NotIn: "not in", } @@ -70,7 +72,8 @@ def visit_BoolOp(self, node): new_node = self._create_bool_op_node(node.values, 'Or') else: raise TypeError( - "Only supports and/or syntax in control flow if statement.") + "Only supports and/or syntax in control flow if statement." + ) return new_node def _create_bool_op_node(self, nodes, api_type): @@ -80,10 +83,11 @@ def _create_bool_op_node(self, nodes, api_type): according to the actual order. In `convert_logical_and(lambda:x>1, lambda:y<1)`, `lambda:y<1` must be run after `lambda:x>1`, If `x>1` is False, `y<1` should NOT be run. ''' - assert len( - nodes - ) > 1, "The length of BoolOp should be at least 2, but received {}.".format( - len(nodes)) + assert ( + len(nodes) > 1 + ), "The length of BoolOp should be at least 2, but received {}.".format( + len(nodes) + ) if len(nodes) > 2: # Creates logic_and/logic_or node recursively. pre_logic_node = self._create_bool_op_node(nodes[:2], api_type) @@ -95,7 +99,8 @@ def _create_bool_op_node(self, nodes, api_type): args = [ast_to_source_code(child) for child in nodes] new_node_str = "_jst.{}(lambda:{}, lambda:{})".format( - api_type, args[0], args[1]) + api_type, args[0], args[1] + ) # NOTE: gast.parse return Module(body=[expr(...)]) new_node = gast.parse(new_node_str).body[0].value return new_node diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py index 38dc7468e0820..b61f9f6e43c20 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/loop_transformer.py @@ -17,21 +17,44 @@ from collections import defaultdict from paddle.fluid import unique_name -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) from paddle.fluid.dygraph.dygraph_to_static.static_analysis import NodeVarType -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import StaticAnalysisVisitor +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + StaticAnalysisVisitor, +) from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code from paddle.fluid.dygraph.dygraph_to_static.utils import generate_name_node from paddle.fluid.dygraph.dygraph_to_static.utils import get_attribute_full_name -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_undefined_var -from paddle.fluid.dygraph.dygraph_to_static.utils import create_nonlocal_stmt_nodes, create_get_args_node, create_set_args_node -from paddle.fluid.dygraph.dygraph_to_static.utils import FunctionNameLivenessAnalysis +from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + create_undefined_var, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + create_nonlocal_stmt_nodes, + create_get_args_node, + create_set_args_node, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FunctionNameLivenessAnalysis, +) from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import ARGS_NAME -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import RenameTransformer -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ForLoopTuplePreTransformer -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ForNodeVisitor -from paddle.fluid.dygraph.dygraph_to_static.utils import GetterSetterHelper, create_name_str +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + RenameTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + ForLoopTuplePreTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + ForNodeVisitor, +) +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + GetterSetterHelper, + create_name_str, +) __all__ = ['LoopTransformer', 'NameVisitor'] @@ -42,8 +65,14 @@ FOR_BODY_PREFIX = 'for_loop_body' -def create_while_nodes(condition_name, body_name, loop_var_names, - push_pop_names, getter_name, setter_name): +def create_while_nodes( + condition_name, + body_name, + loop_var_names, + push_pop_names, + getter_name, + setter_name, +): """ Returns a list of gast.Node which represents the calling of Paddle controlflow while_loop. @@ -79,13 +108,21 @@ def create_while_nodes(condition_name, body_name, loop_var_names, # set doesn't have order so we convert it to list loop_var_names = list(loop_var_names) assign_loop_var_names = [] - for name in (loop_var_names): + for name in loop_var_names: assign_loop_var_names.append(name) while_func_name = "_jst.While" - while_node_str = "{}({}, {}, {}, {}, return_name_ids={}, push_pop_names={})".format( - while_func_name, condition_name, body_name, getter_name, setter_name, - create_name_str(loop_var_names), create_name_str(push_pop_names)) + while_node_str = ( + "{}({}, {}, {}, {}, return_name_ids={}, push_pop_names={})".format( + while_func_name, + condition_name, + body_name, + getter_name, + setter_name, + create_name_str(loop_var_names), + create_name_str(push_pop_names), + ) + ) while_node = gast.parse(while_node_str).body[0] ret = [while_node] @@ -123,14 +160,16 @@ def __init__(self, root_node): self.type_vars = set() self.static_analysis_visitor = StaticAnalysisVisitor(root_node) - self.node_to_wrapper_map = self.static_analysis_visitor.get_node_to_wrapper_map( + self.node_to_wrapper_map = ( + self.static_analysis_visitor.get_node_to_wrapper_map() ) self.visit(root_node) def get_loop_var_names(self, node): assert isinstance( - node, (gast.While, gast.For)), "Input node is not gast loop node" + node, (gast.While, gast.For) + ), "Input node is not gast loop node" loop_var_names = set() create_var_names = set() read_context = {type(gast.Load()), type(gast.AugLoad())} @@ -141,7 +180,8 @@ def get_loop_var_names(self, node): var_name_to_ctxs = defaultdict(list) for var_node in in_loop_vars_list: var_name_to_ctxs[self._var_node_to_name(var_node)].append( - var_node.ctx) + var_node.ctx + ) in_loop_vars = set(in_loop_vars_list) in_loop_vars = self._remove_unnecessary_vars(in_loop_vars, node) @@ -149,13 +189,17 @@ def get_loop_var_names(self, node): before_loop_body_vars = self.before_loop_body_vars[node] before_loop_body_vars = self._remove_unnecessary_vars( - before_loop_body_vars, node) + before_loop_body_vars, node + ) before_loop_name_strs = self._var_nodes_to_names(before_loop_body_vars) - after_loop_vars = self.current_seen_vars - before_loop_body_vars - in_loop_vars + after_loop_vars = ( + self.current_seen_vars - before_loop_body_vars - in_loop_vars + ) after_loop_vars = self._remove_unnecessary_vars(after_loop_vars, node) - after_loop_name_strs = self._var_nodes_to_names(after_loop_vars, - read_context) + after_loop_name_strs = self._var_nodes_to_names( + after_loop_vars, read_context + ) condition_vars = self.condition_vars[node] condition_names = self._var_nodes_to_names(condition_vars) @@ -204,8 +248,10 @@ def get_loop_var_names(self, node): if isinstance(ctx, gast.Store): is_created = True - if isinstance(var_name_to_ctxs[name][0], - gast.Load) and is_created: + if ( + isinstance(var_name_to_ctxs[name][0], gast.Load) + and is_created + ): loop_var_names.add(name) create_var_names.add(name) @@ -223,7 +269,7 @@ def visit_Name(self, node): write_context = { type(gast.Store()), type(gast.AugStore()), - type(gast.Del()) + type(gast.Del()), } for loop_node in self.current_loop: @@ -322,8 +368,10 @@ def _var_node_to_name(self, node): def _node_var_type_is_basic(self, node_var_type): basic_types = { - NodeVarType.BOOLEAN, NodeVarType.INT, NodeVarType.FLOAT, - NodeVarType.STRING + NodeVarType.BOOLEAN, + NodeVarType.INT, + NodeVarType.FLOAT, + NodeVarType.STRING, } for t in node_var_type: if t in basic_types: @@ -411,7 +459,8 @@ def filter_name_nodes_from(root_node, target_var_names): target_vars = [target_node] vars_of_list_generator = vars_of_list_generator | set( - target_vars) + target_vars + ) # 1.2 vars from target vars used in elt_node target_var_names = {var.id for var in target_vars} @@ -424,7 +473,8 @@ def filter_name_nodes_from(root_node, target_var_names): for node in elt_nodes: vars_of_list_generator |= filter_name_nodes_from( - node, target_var_names) + node, target_var_names + ) # 2. Get target vars or vars from target vars used in for-loop but the for-loop is # 1) not the "loop_node" itself @@ -450,15 +500,18 @@ def filter_name_nodes_from(root_node, target_var_names): target_vars = [target_node] target_vars_of_for_node = target_vars_of_for_node | set( - target_vars) + target_vars + ) # 2.2 vars from target vars used in for-loop target_vars_name_strs = {var.id for var in target_vars_of_for_node} for var in loop_vars: if not isinstance(var, gast.Name): continue - if var.id in target_vars_name_strs and var not in self.condition_vars[ - loop_node]: + if ( + var.id in target_vars_name_strs + and var not in self.condition_vars[loop_node] + ): target_vars_of_for_node.add(var) removed_vars = target_vars_of_for_node | vars_of_list_generator @@ -506,11 +559,11 @@ def replace_stmt_list(self, body_list): while i < len(body_list): if isinstance(body_list[i], gast.While): new_stmts = self.get_while_stmt_nodes(body_list[i]) - body_list[i:i + 1] = new_stmts + body_list[i : i + 1] = new_stmts i += len(new_stmts) elif isinstance(body_list[i], gast.For): new_stmts = self.get_for_stmt_nodes(body_list[i]) - body_list[i:i + 1] = new_stmts + body_list[i : i + 1] = new_stmts i += len(new_stmts) else: i += 1 @@ -536,8 +589,10 @@ def get_for_stmt_nodes(self, node): return [node] init_stmts, cond_stmt, body_stmts = stmts_tuple # 2. get original loop vars - loop_var_names, create_var_names = node.pd_scope.modified_vars( - ), node.pd_scope.created_vars() + loop_var_names, create_var_names = ( + node.pd_scope.modified_vars(), + node.pd_scope.created_vars(), + ) push_pop_names = list(node.pd_scope.variadic_length_vars()) # TODO: Remove the bunch of code? We have the unique format `for A in B:` # NOTE: in 'for x in var' or 'for i, x in enumerate(var)' cases, @@ -577,53 +632,64 @@ def get_for_stmt_nodes(self, node): # 5. create & append condition function node condition_func_node = gast.FunctionDef( name=unique_name.generate(FOR_CONDITION_PREFIX), - args=gast.arguments(args=[], - posonlyargs=[], - vararg=None, - kwonlyargs=[], - kw_defaults=None, - kwarg=None, - defaults=[]), + args=gast.arguments( + args=[], + posonlyargs=[], + vararg=None, + kwonlyargs=[], + kw_defaults=None, + kwarg=None, + defaults=[], + ), body=nonlocal_stmt_node + [gast.Return(value=cond_stmt)], decorator_list=[], returns=None, - type_comment=None) + type_comment=None, + ) new_stmts.append(condition_func_node) # 6. create & append loop body function node # append return values for loop body body_func_node = gast.FunctionDef( name=unique_name.generate(FOR_BODY_PREFIX), - args=gast.arguments(args=[], - posonlyargs=[], - vararg=None, - kwonlyargs=[], - kw_defaults=None, - kwarg=None, - defaults=[]), + args=gast.arguments( + args=[], + posonlyargs=[], + vararg=None, + kwonlyargs=[], + kw_defaults=None, + kwarg=None, + defaults=[], + ), body=nonlocal_stmt_node + body_stmts, decorator_list=[], returns=None, - type_comment=None) + type_comment=None, + ) new_stmts.append(body_func_node) helper = GetterSetterHelper(None, None, nonlocal_names, push_pop_names) get_args_node = create_get_args_node(helper.union()) set_args_node = create_set_args_node(helper.union()) # 7. create & append while loop node - while_loop_nodes = create_while_nodes(condition_func_node.name, - body_func_node.name, - nonlocal_names, push_pop_names, - get_args_node.name, - set_args_node.name) + while_loop_nodes = create_while_nodes( + condition_func_node.name, + body_func_node.name, + nonlocal_names, + push_pop_names, + get_args_node.name, + set_args_node.name, + ) new_stmts.extend([get_args_node, set_args_node]) new_stmts.extend(while_loop_nodes) return new_stmts def get_while_stmt_nodes(self, node): - loop_var_names, create_var_names = node.pd_scope.modified_vars( - ), node.pd_scope.created_vars() + loop_var_names, create_var_names = ( + node.pd_scope.modified_vars(), + node.pd_scope.created_vars(), + ) push_pop_names = list(node.pd_scope.variadic_length_vars()) new_stmts = [] @@ -648,45 +714,54 @@ def get_while_stmt_nodes(self, node): condition_func_node = gast.FunctionDef( name=unique_name.generate(WHILE_CONDITION_PREFIX), - args=gast.arguments(args=[], - posonlyargs=[], - vararg=None, - kwonlyargs=[], - kw_defaults=None, - kwarg=None, - defaults=[]), + args=gast.arguments( + args=[], + posonlyargs=[], + vararg=None, + kwonlyargs=[], + kw_defaults=None, + kwarg=None, + defaults=[], + ), body=nonlocal_stmt_node + [gast.Return(value=node.test)], decorator_list=[], returns=None, - type_comment=None) + type_comment=None, + ) new_stmts.append(condition_func_node) new_body = node.body body_func_node = gast.FunctionDef( name=unique_name.generate(WHILE_BODY_PREFIX), - args=gast.arguments(args=[], - posonlyargs=[], - vararg=None, - kwonlyargs=[], - kw_defaults=None, - kwarg=None, - defaults=[]), + args=gast.arguments( + args=[], + posonlyargs=[], + vararg=None, + kwonlyargs=[], + kw_defaults=None, + kwarg=None, + defaults=[], + ), body=nonlocal_stmt_node + new_body, decorator_list=[], returns=None, - type_comment=None) + type_comment=None, + ) new_stmts.append(body_func_node) helper = GetterSetterHelper(None, None, nonlocal_names, push_pop_names) get_args_node = create_get_args_node(helper.union()) set_args_node = create_set_args_node(helper.union()) - while_loop_nodes = create_while_nodes(condition_func_node.name, - body_func_node.name, - nonlocal_names, push_pop_names, - get_args_node.name, - set_args_node.name) + while_loop_nodes = create_while_nodes( + condition_func_node.name, + body_func_node.name, + nonlocal_names, + push_pop_names, + get_args_node.name, + set_args_node.name, + ) new_stmts.extend([get_args_node, set_args_node]) new_stmts.extend(while_loop_nodes) return new_stmts diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py b/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py index 1399eeb2b6bed..4bfc73a32565f 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py @@ -20,6 +20,7 @@ from paddle.fluid.dygraph.dygraph_to_static.utils import unwrap from paddle.fluid.dygraph.dygraph_to_static.utils import ORIGI_INFO from paddle.fluid.framework import Program + try: from collections.abc import Sequence except: @@ -30,6 +31,7 @@ class Location(object): """ Location information of source code. """ + __slots__ = ( "filepath", "lineno", @@ -42,8 +44,9 @@ def __init__(self, filepath, lineno, col_offset=None): self.col_offset = col_offset def __str__(self): - return "location: {}:{}:{}".format(self.filepath, self.lineno, - self.col_offset) + return "location: {}:{}:{}".format( + self.filepath, self.lineno, self.col_offset + ) @property def line_location(self): @@ -54,6 +57,7 @@ class OriginInfo(object): """ Original information of source code. """ + __slots__ = ( "location", "function_name", @@ -67,17 +71,26 @@ def __init__(self, location, function_name, source_code): def __str__(self): return "{} \nsource_code: {} in function {}\n ".format( - self.location, self.source_code, self.function_name) + self.location, self.source_code, self.function_name + ) def formated_message(self): flag_for_origin_info = "(* user code *)" return ' File "{}", line {}, in {} {}\n\t{}'.format( - self.location.filepath, self.location.lineno, self.function_name, - flag_for_origin_info, self.source_code.lstrip()) + self.location.filepath, + self.location.lineno, + self.function_name, + flag_for_origin_info, + self.source_code.lstrip(), + ) def as_frame(self): - return (self.location.filepath, self.location.lineno, - self.function_name, self.source_code.lstrip()) + return ( + self.location.filepath, + self.location.lineno, + self.function_name, + self.source_code.lstrip(), + ) class OriginInfoAttacher(gast.NodeTransformer): @@ -142,9 +155,9 @@ def _abs_col_offset(self, node): global_origin_info_map = {} -def create_and_update_origin_info_map(transformed_node, - static_func, - is_global=True): +def create_and_update_origin_info_map( + transformed_node, static_func, is_global=True +): """ Creates a original information map between transformed static function and original dygraph function. @@ -162,9 +175,11 @@ def create_and_update_origin_info_map(transformed_node, static_node = attach_origin_info(static_node, static_func) for t_node, s_node in ast_walk(transformed_node, static_node): - assert type(t_node) == type(s_node), \ - "The node types should be the same, but received type(t_node) is {}, and type(s_node) is {}." \ - .format(type(t_node), type(s_node)) + assert type(t_node) == type( + s_node + ), "The node types should be the same, but received type(t_node) is {}, and type(s_node) is {}.".format( + type(t_node), type(s_node) + ) dygraph_info = getattr(t_node, ORIGI_INFO, None) static_info = getattr(s_node, ORIGI_INFO, None) @@ -174,9 +189,15 @@ def create_and_update_origin_info_map(transformed_node, exist_origin_info = origin_info_map.get(static_loc) if exist_origin_info is not None: - if exist_origin_info.location.lineno >= dygraph_info.location.lineno: + if ( + exist_origin_info.location.lineno + >= dygraph_info.location.lineno + ): continue - if exist_origin_info.location.col_offset <= dygraph_info.location.col_offset: + if ( + exist_origin_info.location.col_offset + <= dygraph_info.location.col_offset + ): continue origin_info_map[static_loc] = dygraph_info @@ -229,12 +250,15 @@ def _as_list(x): # Node types should be strictly required, but there is no strict distinction between gast.Load and gast.Param # in the ast transformation process. if isinstance(t_node, (gast.Load, gast.Param)) or isinstance( - s_node, (gast.Load, gast.Param)): + s_node, (gast.Load, gast.Param) + ): continue - assert type(t_node) == type(s_node), \ - "The node types should be the same, but received type(t_node) is {}, and type(s_node) is {}."\ - .format(type(t_node), type(s_node)) + assert type(t_node) == type( + s_node + ), "The node types should be the same, but received type(t_node) is {}, and type(s_node) is {}.".format( + type(t_node), type(s_node) + ) yield t_node, s_node @@ -289,11 +313,11 @@ def get_new_op_callstack(callstack): loc = Location(filepath, lineno) dygraph_func_info = global_origin_info_map.get(loc.line_location) if dygraph_func_info: - filepath, lineno, funcname, code = \ - dygraph_func_info.as_frame() + filepath, lineno, funcname, code = dygraph_func_info.as_frame() callstack[i] = ' File "{}", line {}, in {}'.format( - filepath, lineno, funcname) + filepath, lineno, funcname + ) callstack[i + 1] = ' {}'.format(code) return callstack diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py b/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py index ebc8b351f785a..e0129e8466995 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py @@ -16,19 +16,32 @@ import paddle from paddle.fluid import framework, backward, core, program_guard -from paddle.fluid.executor import _is_enable_standalone_executor, _is_dy2st_enable_standalone_executor +from paddle.fluid.executor import ( + _is_enable_standalone_executor, + _is_dy2st_enable_standalone_executor, +) from paddle.fluid.dygraph import layers from paddle.fluid.dygraph.base import switch_to_static_graph from paddle.fluid.dygraph.dygraph_to_static import logging_utils -from paddle.fluid.dygraph.dygraph_to_static.return_transformer import RETURN_NO_VALUE_MAGIC_NUM +from paddle.fluid.dygraph.dygraph_to_static.return_transformer import ( + RETURN_NO_VALUE_MAGIC_NUM, +) from paddle.fluid.layers.utils import flatten from paddle.fluid.layers.utils import pack_sequence_as from paddle.fluid.layers.utils import _hash_with_id from paddle.fluid.compiler import BuildStrategy from paddle.fluid.framework import _apply_pass -from paddle.fluid.contrib.mixed_precision.decorator import AutoMixedPrecisionLists -from paddle.fluid.contrib.mixed_precision.fp16_utils import rewrite_program, cast_model_to_fp16 -from paddle.fluid.dygraph.amp.auto_cast import _in_amp_guard, _in_pure_fp16_guard +from paddle.fluid.contrib.mixed_precision.decorator import ( + AutoMixedPrecisionLists, +) +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + rewrite_program, + cast_model_to_fp16, +) +from paddle.fluid.dygraph.amp.auto_cast import ( + _in_amp_guard, + _in_pure_fp16_guard, +) from paddle import _legacy_C_ops @@ -61,7 +74,8 @@ def _get_var_ids(self): var_ids = [] for idx, var in enumerate(self.__input_list): if isinstance( - var, (framework.Variable, core.VarBase, core.eager.Tensor)): + var, (framework.Variable, core.VarBase, core.eager.Tensor) + ): var_ids.append(idx) return var_ids @@ -74,15 +88,17 @@ def _check_non_variable(self, need_check): warning_types = set() for var in self.__input_list: if not isinstance( - var, - (framework.Variable, core.VarBase, core.eager.Tensor)): + var, (framework.Variable, core.VarBase, core.eager.Tensor) + ): warning_types.add(type(var)) if warning_types: logging_utils.warn( "Output of traced function contains non-tensor type values: {}. " "Currently, We don't support to update them while training and will return " - "what we first saw. Please try to return them as tensor.". - format(list(warning_types))) + "what we first saw. Please try to return them as tensor.".format( + list(warning_types) + ) + ) @property def var_ids(self): @@ -136,12 +152,9 @@ class PartialProgramLayer: Layer: A Layer object that run all ops internally in static mode. """ - def __init__(self, - main_program, - inputs, - outputs, - parameters=None, - **kwargs): + def __init__( + self, main_program, inputs, outputs, parameters=None, **kwargs + ): super(PartialProgramLayer, self).__init__() self._inputs = NestSequence(inputs) self._outputs = NestSequence(outputs, need_check=True) @@ -164,7 +177,8 @@ def __init__(self, # For AMP training self._amp_list = AutoMixedPrecisionLists( custom_white_list=custom_white_list, - custom_black_list=custom_black_list) + custom_black_list=custom_black_list, + ) # program_id -> list(scope) self._scope_cache = {} @@ -200,7 +214,8 @@ def _create_program(self, is_infer_mode=False): return self._origin_main_program.clone(for_test=is_infer_mode) else: train_program = self._append_backward_desc( - self._origin_main_program) + self._origin_main_program + ) # Note: Only set grad type once after initializing train program. So we put it here. self._set_grad_type(self._params, train_program) return train_program @@ -220,16 +235,18 @@ def _create_amp_program(self, is_infer_mode=False): @switch_to_static_graph def _create_pure_fp16_program(self, is_infer_mode=False): pure_fp16_program = self._origin_main_program.clone( - for_test=is_infer_mode) + for_test=is_infer_mode + ) with program_guard(pure_fp16_program): - cast_model_to_fp16(pure_fp16_program, - self._amp_list, - use_fp16_guard=False) + cast_model_to_fp16( + pure_fp16_program, self._amp_list, use_fp16_guard=False + ) if is_infer_mode: return pure_fp16_program else: train_pure_fp16_program = self._append_backward_desc( - pure_fp16_program) + pure_fp16_program + ) self._set_grad_type(self._params, train_pure_fp16_program) return train_pure_fp16_program @@ -237,23 +254,27 @@ def _create_pure_fp16_program(self, is_infer_mode=False): def _create_forward_backward_train_program(self): whole_program = self._create_program() forward_end_op_index = self._infer_program.desc.block(0).op_size() - return self._get_forward_backward_program_form(whole_program, - forward_end_op_index) + return self._get_forward_backward_program_form( + whole_program, forward_end_op_index + ) @switch_to_static_graph def _create_forward_backward_train_amp_program(self): whole_program = self._create_amp_program() forward_end_op_index = self._infer_amp_program.desc.block(0).op_size() - return self._get_forward_backward_program_form(whole_program, - forward_end_op_index) + return self._get_forward_backward_program_form( + whole_program, forward_end_op_index + ) @switch_to_static_graph def _create_forward_backward_train_pure_fp16_program(self): whole_program = self._create_pure_fp16_program() forward_end_op_index = self._infer_pure_fp16_program.desc.block( - 0).op_size() - return self._get_forward_backward_program_form(whole_program, - forward_end_op_index) + 0 + ).op_size() + return self._get_forward_backward_program_form( + whole_program, forward_end_op_index + ) @LazyInitialized def _train_program(self): @@ -349,8 +370,9 @@ def backward_program(self): @LazyInitialized def _train_program_id(self): program_id = _hash_with_id(self._train_program, self) - core._set_cached_executor_build_strategy(program_id, - self._build_strategy) + core._set_cached_executor_build_strategy( + program_id, self._build_strategy + ) return program_id @LazyInitialized @@ -360,8 +382,9 @@ def _infer_program_id(self): @LazyInitialized def _train_amp_program_id(self): program_id = _hash_with_id(self._train_amp_program, self) - core._set_cached_executor_build_strategy(program_id, - self._build_strategy) + core._set_cached_executor_build_strategy( + program_id, self._build_strategy + ) return program_id @LazyInitialized @@ -371,8 +394,9 @@ def _infer_amp_program_id(self): @LazyInitialized def _train_pure_fp16_program_id(self): program_id = _hash_with_id(self._train_pure_fp16_program, self) - core._set_cached_executor_build_strategy(program_id, - self._build_strategy) + core._set_cached_executor_build_strategy( + program_id, self._build_strategy + ) return program_id @LazyInitialized @@ -408,8 +432,9 @@ def _verify_program(self, main_program): return main_program - def prepare_gradient_aggregation(self, start_idx, main_program, - target_program): + def prepare_gradient_aggregation( + self, start_idx, main_program, target_program + ): """ Why we need add gradient aggregation operation ? In some cases, if non leaf nodes are used as output, gradient overwriting will occur, such as @@ -427,8 +452,8 @@ def _need_aggregation(var): if exist a op whose inputs is var, then return True """ if not isinstance(var, framework.Variable) or var.type not in [ - core.VarDesc.VarType.LOD_TENSOR, - core.VarDesc.VarType.SELECTED_ROWS + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.SELECTED_ROWS, ]: return False if var.dtype not in [paddle.float32, paddle.float64]: @@ -445,20 +470,28 @@ def _insert_aggregation_ops_for_var(target_program, var): new_grad_name = var.name + suffix + "@GRAD" finded_ops = list( filter( - lambda x: x[0] >= start_idx and any([ - out_arg == var_grad_name - for out_arg in x[1].output_arg_names - ]), enumerate(target_program.block(0).ops))) + lambda x: x[0] >= start_idx + and any( + [ + out_arg == var_grad_name + for out_arg in x[1].output_arg_names + ] + ), + enumerate(target_program.block(0).ops), + ) + ) # len(finded_ops) may equals zero when stop_gradient works. # len(finded_ops) may > 1, because we may have fill_constant op. if len(finded_ops) == 0: return None # step1: create a new var named var.name@GRAD - target_program.block(0).create_var(name=new_grad_name, - type=var.type, - dtype=var.dtype, - shape=var.shape) + target_program.block(0).create_var( + name=new_grad_name, + type=var.type, + dtype=var.dtype, + shape=var.shape, + ) # step2: rename the var.name@GRAD to var.name@GRAD@dy2static for idx, op in finded_ops: op._rename_input(var_grad_name, new_grad_name) @@ -469,11 +502,13 @@ def _insert_aggregation_ops_for_var(target_program, var): finded_ops[-1][0] + 1, type='sum', inputs={'X': [var_grad_name, new_grad_name]}, - outputs={"Out": var_grad_name}) + outputs={"Out": var_grad_name}, + ) return None to_processed_vars = list( - filter(_need_aggregation, self._outputs.tolist())) + filter(_need_aggregation, self._outputs.tolist()) + ) for _var in to_processed_vars: _insert_aggregation_ops_for_var(target_program, _var) @@ -489,8 +524,9 @@ def _append_backward_desc(self, main_program): if targets and self._params: backward.gradients(targets=targets, inputs=[]) - start_idx = len( - main_program.block(0).ops) + 2 * len(self._outputs.tolist()) + start_idx = len(main_program.block(0).ops) + 2 * len( + self._outputs.tolist() + ) self.prepare_gradient_aggregation(start_idx, main_program, program) @@ -509,7 +545,10 @@ def _prune_unused_params(self, program): found_param = False for block in program.blocks: for op in block.ops: - if param.name in op.input_arg_names or param.name in op.output_arg_names: + if ( + param.name in op.input_arg_names + or param.name in op.output_arg_names + ): required_params.append(param) found_param = True break @@ -526,15 +565,21 @@ def _get_double_grads(self, program): var_desc = block.vars[name].desc var_base = None if not framework._in_eager_mode_: - var_base = core.VarBase(var_desc.dtype(), - var_desc.shape(), - var_desc.name(), - var_desc.type(), False) + var_base = core.VarBase( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) else: - var_base = core.eager.Tensor(var_desc.dtype(), - var_desc.shape(), - var_desc.name(), - var_desc.type(), False) + var_base = core.eager.Tensor( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) double_grads.append(var_base) return self._valid_vars(double_grads) @@ -554,36 +599,62 @@ def __call__(self, inputs): attrs = [ 'global_block', - self.program.desc.block(0), 'start_op_index', 0, 'end_op_index', - self._get_end_op_index(), 'is_test', not self.training, - 'program_id', self.program_id + self.program.desc.block(0), + 'start_op_index', + 0, + 'end_op_index', + self._get_end_op_index(), + 'is_test', + not self.training, + 'program_id', + self.program_id, ] if self._cuda_graph_capture_mode: attrs.extend( - ('cuda_graph_capture_mode', self._cuda_graph_capture_mode, - 'cuda_graph_pool_id', self._cuda_graph_pool_id)) - - use_interpretorcore = _is_enable_standalone_executor( - ) and _is_dy2st_enable_standalone_executor() + ( + 'cuda_graph_capture_mode', + self._cuda_graph_capture_mode, + 'cuda_graph_pool_id', + self._cuda_graph_pool_id, + ) + ) + + use_interpretorcore = ( + _is_enable_standalone_executor() + and _is_dy2st_enable_standalone_executor() + ) attrs.extend(('use_interpretorcore', use_interpretorcore)) if use_interpretorcore: attrs.extend( - ('forward_global_block', self.forward_program.desc.block(0), - 'backward_global_block', self.backward_program.desc.block(0))) + ( + 'forward_global_block', + self.forward_program.desc.block(0), + 'backward_global_block', + self.backward_program.desc.block(0), + ) + ) _legacy_C_ops.run_program( - self._valid_vars(in_vars), self._valid_vars(self._params), + self._valid_vars(in_vars), + self._valid_vars(self._params), self._valid_vars(out_vars), - self._create_scope_vec(program_id=self.program_id, - use_scope_cache=True), - self._double_grads, self._cuda_graph_vec, *attrs) + self._create_scope_vec( + program_id=self.program_id, use_scope_cache=True + ), + self._double_grads, + self._cuda_graph_vec, + *attrs + ) else: - _legacy_C_ops.run_program(self._valid_vars(in_vars), - self._valid_vars(self._params), - self._valid_vars(out_vars), - self._create_scope_vec(), - self._double_grads, self._cuda_graph_vec, - *attrs) + _legacy_C_ops.run_program( + self._valid_vars(in_vars), + self._valid_vars(self._params), + self._valid_vars(out_vars), + self._create_scope_vec(), + self._double_grads, + self._cuda_graph_vec, + *attrs + ) restored_nest_out = self._restore_out(out_vars) return self._remove_no_value(restored_nest_out) @@ -591,9 +662,11 @@ def _cast_fp16_if_pure_fp16(self, in_vars): if _in_pure_fp16_guard(): for i, var in enumerate(in_vars): name = var.name - if (self.program.global_block().has_var(name) - and self.program.global_block().var(name).dtype - == paddle.float16): + if ( + self.program.global_block().has_var(name) + and self.program.global_block().var(name).dtype + == paddle.float16 + ): in_vars[i] = var.astype('float16') in_vars[i].name = name @@ -624,25 +697,32 @@ def infer_program(self): return self._infer_program @switch_to_static_graph - def _get_forward_backward_program_form(self, whole_program, - forward_end_op_index): + def _get_forward_backward_program_form( + self, whole_program, forward_end_op_index + ): forward_builded_program = add_build_strategy_for( - whole_program, 0, forward_end_op_index, self._build_strategy) + whole_program, 0, forward_end_op_index, self._build_strategy + ) backward_start_op_index = forward_end_op_index + 2 * len( - self._outputs.var_ids) + self._outputs.var_ids + ) backward_end_op_index = whole_program.desc.block(0).op_size() backward_builded_program = add_build_strategy_for( - whole_program, backward_start_op_index, backward_end_op_index, - self._build_strategy) - self._apply_inplace_pass(forward_builded_program, - backward_builded_program) + whole_program, + backward_start_op_index, + backward_end_op_index, + self._build_strategy, + ) + self._apply_inplace_pass( + forward_builded_program, backward_builded_program + ) return [forward_builded_program, backward_builded_program] def _apply_inplace_pass(self, forward_program, backward_program): attr_types = { "use_cuda": "bool", "mem_opt_skip_vars": "list[str]", - "for_partial_block": "bool" + "for_partial_block": "bool", } empty_startup_program = paddle.static.Program() use_cuda = True if core.is_compiled_with_cuda() else False @@ -664,22 +744,33 @@ def _apply_inplace_pass(self, forward_program, backward_program): forward_mem_opt_skip_vars.append(var.desc.name()) backward_mem_opt_skip_vars.append(var.desc.name()) for var_name in core.parse_safe_eager_deletion_skip_vars( - backward_program.desc): + backward_program.desc + ): forward_mem_opt_skip_vars.append(var_name) attrs = { "use_cuda": use_cuda, "mem_opt_skip_vars": forward_mem_opt_skip_vars, - "for_partial_block": True + "for_partial_block": True, } - _apply_pass(forward_program, empty_startup_program, - "buffer_shared_inplace_pass", attrs, attr_types) + _apply_pass( + forward_program, + empty_startup_program, + "buffer_shared_inplace_pass", + attrs, + attr_types, + ) attrs = { "use_cuda": use_cuda, "mem_opt_skip_vars": backward_mem_opt_skip_vars, - "for_partial_block": True + "for_partial_block": True, } - _apply_pass(backward_program, empty_startup_program, - "buffer_shared_inplace_pass", attrs, attr_types) + _apply_pass( + backward_program, + empty_startup_program, + "buffer_shared_inplace_pass", + attrs, + attr_types, + ) def _prepare(self, inputs): """ @@ -695,23 +786,28 @@ def _prepare(self, inputs): if isinstance(value, np.ndarray): var = None if not framework._in_eager_mode_: - var = core.VarBase(value=value, - name=self._inputs[i].desc.name(), - persistable=False, - place=expected_place, - zero_copy=True) + var = core.VarBase( + value=value, + name=self._inputs[i].desc.name(), + persistable=False, + place=expected_place, + zero_copy=True, + ) else: - var = core.eager.Tensor(value=value, - name=self._inputs[i].desc.name(), - persistable=False, - place=expected_place, - zero_copy=True) + var = core.eager.Tensor( + value=value, + name=self._inputs[i].desc.name(), + persistable=False, + place=expected_place, + zero_copy=True, + ) elif isinstance(value, (core.VarBase, core.eager.Tensor)): # NOTE(Aurelius84): If var is on CPUPlace, it will be transformed multi times # into CUDAPlace when it's as input of multi Ops. so we move it in advance # to avoid this problem. if value.stop_gradient and not value.place._equals( - expected_place): + expected_place + ): var = value._copy_to(expected_place, False) var.stop_gradient = True else: @@ -734,12 +830,21 @@ def create_out(var_id): return out_varbase_map[var_desc.name()] if not framework._in_eager_mode_: - var_base = core.VarBase(var_desc.dtype(), var_desc.shape(), - var_desc.name(), var_desc.type(), False) + var_base = core.VarBase( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) else: - var_base = core.eager.Tensor(var_desc.dtype(), var_desc.shape(), - var_desc.name(), var_desc.type(), - False) + var_base = core.eager.Tensor( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) var_base.stop_gradient = var.stop_gradient out_varbase_map[var_desc.name()] = var_base return var_base @@ -752,20 +857,30 @@ def create_out(var_id): def _create_scope_vec(self, program_id=None, use_scope_cache=False): # Hold forward variables tmp_scope_vec = None - inner_scope = self._get_scope(program_id=program_id, - use_scope_cache=use_scope_cache) + inner_scope = self._get_scope( + program_id=program_id, use_scope_cache=use_scope_cache + ) if not framework._in_eager_mode_: - tmp_scope_vec = core.VarBase(core.VarDesc.VarType.FP32, [], - "program_out_scope", - core.VarDesc.VarType.STEP_SCOPES, True) + tmp_scope_vec = core.VarBase( + core.VarDesc.VarType.FP32, + [], + "program_out_scope", + core.VarDesc.VarType.STEP_SCOPES, + True, + ) tmp_scope_vec.value().set_scope(inner_scope) else: tmp_scope_vec = [inner_scope] return tmp_scope_vec def _create_cuda_graph_vec(self): - var = core.VarBase(core.VarDesc.VarType.FP32, [], "cuda_graph", - core.VarDesc.VarType.RAW, True) + var = core.VarBase( + core.VarDesc.VarType.FP32, + [], + "cuda_graph", + core.VarDesc.VarType.RAW, + True, + ) var.stop_gradient = True return var @@ -788,8 +903,9 @@ def _clone_for_test(self, main_program): return main_program.clone(for_test=True) def _is_no_value(self, var): - if isinstance(var, - (core.VarBase, core.eager.Tensor)) and var.shape == [1]: + if isinstance(var, (core.VarBase, core.eager.Tensor)) and var.shape == [ + 1 + ]: # NOTE: .numpy() will insert MemcpySync operation, it hits performance. if var.numpy()[0] == RETURN_NO_VALUE_MAGIC_NUM: return True @@ -805,13 +921,14 @@ def _remove_no_value(self, out_vars): return out_vars elif isinstance(out_vars, (tuple, list)): if isinstance(out_vars, tuple): - res = tuple(var for var in out_vars - if not self._is_no_value(var)) + res = tuple( + var for var in out_vars if not self._is_no_value(var) + ) else: # isinstance(out_vars, list) res = [var for var in out_vars if not self._is_no_value(var)] - has_removed = (len(out_vars) > len(res)) + has_removed = len(out_vars) > len(res) # len(out_vars) > len(res) means we have removed var. This is # preventing out_vars is empty or just one element at the beginning if len(res) == 0 and has_removed: @@ -860,15 +977,18 @@ def _check_params_all_inited(self, main_program): if not isinstance(self._params, (list, tuple)): raise TypeError( "Type of self._params in PartialProgramLayer should be list or tuple, but received %s." - % type(self._params)) + % type(self._params) + ) param_and_buffer_names_set = set() for i, var in enumerate(self._params): # self._params constains parameters and buffers with persistable=True. if not isinstance(var, (core.VarBase, core.eager.Tensor)): raise TypeError( - 'Type of self._params[{}] in PartialProgramLayer should be Parameter or Variable, but received {}.' - .format(i, type(var))) + 'Type of self._params[{}] in PartialProgramLayer should be Parameter or Variable, but received {}.'.format( + i, type(var) + ) + ) param_and_buffer_names_set.add(var.name) for block in main_program.blocks: @@ -882,7 +1002,8 @@ def _check_params_all_inited(self, main_program): "\n\tRevise suggestion: " "\n\t\t1. Please ensure all your sublayers are inheritted from nn.Layer." "\n\t\t2. Please use nn.ParameterList and nn.LayerList as container instead of using a native Python container such as List" - % name) + % name + ) def _valid_vars(self, vars): """ @@ -899,13 +1020,23 @@ def _create_fake_var(): """ if not framework._in_eager_mode_: return [ - core.VarBase(core.VarDesc.VarType.FP32, [], "Fake_var", - core.VarDesc.VarType.RAW, False) + core.VarBase( + core.VarDesc.VarType.FP32, + [], + "Fake_var", + core.VarDesc.VarType.RAW, + False, + ) ] else: return [ - core.eager.Tensor(core.VarDesc.VarType.FP32, [], "Fake_var", - core.VarDesc.VarType.RAW, False) + core.eager.Tensor( + core.VarDesc.VarType.FP32, + [], + "Fake_var", + core.VarDesc.VarType.RAW, + False, + ) ] @@ -914,23 +1045,27 @@ def partial_program_from(concrete_program): if inputs and isinstance(inputs[0], layers.Layer): inputs = inputs[1:] - return PartialProgramLayer(concrete_program.main_program, inputs, - concrete_program.outputs, - concrete_program.parameters, - **concrete_program.kwargs) + return PartialProgramLayer( + concrete_program.main_program, + inputs, + concrete_program.outputs, + concrete_program.parameters, + **concrete_program.kwargs + ) @switch_to_static_graph -def add_build_strategy_for(program, - start_op_index, - end_op_index, - build_strategy=None): - if (start_op_index < end_op_index): +def add_build_strategy_for( + program, start_op_index, end_op_index, build_strategy=None +): + if start_op_index < end_op_index: compiled_program = paddle.static.CompiledProgram( core.Graph(program.desc, start_op_index, end_op_index), - build_strategy=build_strategy) - compiled_program._compile(core.Scope(), - framework._current_expected_place()) + build_strategy=build_strategy, + ) + compiled_program._compile( + core.Scope(), framework._current_expected_place() + ) ir_graph = framework.IrGraph(compiled_program._graph) builded_program = ir_graph.to_program() if hasattr(compiled_program._program, 'lr_sheduler'): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py index 93a16322b0b37..fdbd585a71bfb 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/print_transformer.py @@ -14,8 +14,13 @@ from paddle.utils import gast -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper, StaticAnalysisVisitor -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, + StaticAnalysisVisitor, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) class PrintTransformer(BaseTransformer): @@ -31,7 +36,8 @@ def __init__(self, wrapper_root): self.root = wrapper_root.node self.static_analysis_visitor = StaticAnalysisVisitor(self.root) - self.node_to_wrapper_map = self.static_analysis_visitor.get_node_to_wrapper_map( + self.node_to_wrapper_map = ( + self.static_analysis_visitor.get_node_to_wrapper_map() ) def transform(self): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py b/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py index ad327a7f2e75b..a8a0cd71212cc 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py @@ -29,19 +29,36 @@ from paddle.fluid.dygraph.dygraph_to_static import DygraphToStaticAst from paddle.fluid.dygraph.dygraph_to_static import error from paddle.fluid.dygraph.dygraph_to_static import logging_utils -from paddle.fluid.dygraph.dygraph_to_static.origin_info import attach_origin_info -from paddle.fluid.dygraph.dygraph_to_static.origin_info import create_and_update_origin_info_map -from paddle.fluid.dygraph.dygraph_to_static.origin_info import update_op_callstack_with_origin_info -from paddle.fluid.dygraph.dygraph_to_static.partial_program import partial_program_from +from paddle.fluid.dygraph.dygraph_to_static.origin_info import ( + attach_origin_info, +) +from paddle.fluid.dygraph.dygraph_to_static.origin_info import ( + create_and_update_origin_info_map, +) +from paddle.fluid.dygraph.dygraph_to_static.origin_info import ( + update_op_callstack_with_origin_info, +) +from paddle.fluid.dygraph.dygraph_to_static.partial_program import ( + partial_program_from, +) from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_func from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code from paddle.fluid.dygraph.dygraph_to_static.utils import func_to_source_code from paddle.fluid.dygraph.dygraph_to_static.utils import input_specs_compatible from paddle.fluid.dygraph.dygraph_to_static.utils import type_name from paddle.fluid.dygraph.dygraph_to_static.utils import unwrap -from paddle.fluid.dygraph.dygraph_to_static.utils import make_hashable, ALREADY_D2S -from paddle.fluid.dygraph.dygraph_to_static.function_spec import FunctionSpec, _hash_spec_names -from paddle.fluid.dygraph.dygraph_to_static.function_spec import get_buffers, get_parameters +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + make_hashable, + ALREADY_D2S, +) +from paddle.fluid.dygraph.dygraph_to_static.function_spec import ( + FunctionSpec, + _hash_spec_names, +) +from paddle.fluid.dygraph.dygraph_to_static.function_spec import ( + get_buffers, + get_parameters, +) from paddle.fluid.wrapped_decorator import signature_safe_contextmanager __all__ = ['ProgramTranslator', 'convert_to_static'] @@ -145,13 +162,24 @@ class CacheKey(object): """ Cached key for ProgramCache. """ + __slots__ = [ - 'function_spec', 'input_args_with_spec', 'input_kwargs_with_spec', - 'class_instance', 'kwargs', '_spec_names_id' + 'function_spec', + 'input_args_with_spec', + 'input_kwargs_with_spec', + 'class_instance', + 'kwargs', + '_spec_names_id', ] - def __init__(self, function_spec, input_args_with_spec, - input_kwargs_with_spec, class_instance, **kwargs): + def __init__( + self, + function_spec, + input_args_with_spec, + input_kwargs_with_spec, + class_instance, + **kwargs + ): """ Initializes a cache key. @@ -168,8 +196,9 @@ def __init__(self, function_spec, input_args_with_spec, self.class_instance = class_instance # NOTE: `kwargs` is usually not considered as basic member for `__hash__` self.kwargs = kwargs - self._spec_names_id = _hash_spec_names(input_args_with_spec, - input_kwargs_with_spec) + self._spec_names_id = _hash_spec_names( + input_args_with_spec, input_kwargs_with_spec + ) @classmethod def from_func_and_args(cls, function_spec, args, kwargs, class_instance): @@ -187,22 +216,34 @@ def from_func_and_args(cls, function_spec, args, kwargs, class_instance): args = args[1:] # 2. convert tensor and numpy array into InputSpec _args, _kwargs = function_spec.unified_args_and_kwargs(args, kwargs) - input_args_with_spec, input_kwargs_with_spec = function_spec.args_to_input_spec( - _args, _kwargs) + ( + input_args_with_spec, + input_kwargs_with_spec, + ) = function_spec.args_to_input_spec(_args, _kwargs) # 3. check whether hit the cache or build a new program for the input arguments - return CacheKey(function_spec, input_args_with_spec, - input_kwargs_with_spec, class_instance) + return CacheKey( + function_spec, + input_args_with_spec, + input_kwargs_with_spec, + class_instance, + ) def __hash__(self): error_msg = "Arguments to a `@paddle.jit.to_static` must be a hashable Python objects (or nested structures of these types)." with_hook = self.kwargs.get("with_hook", False) is_train = self.kwargs.get("is_train", False) - return hash((id(self.function_spec), - make_hashable(self.input_args_with_spec, error_msg), - make_hashable(self.input_kwargs_with_spec, - error_msg), self._spec_names_id, - self.class_instance, with_hook, is_train)) + return hash( + ( + id(self.function_spec), + make_hashable(self.input_args_with_spec, error_msg), + make_hashable(self.input_kwargs_with_spec, error_msg), + self._spec_names_id, + self.class_instance, + with_hook, + is_train, + ) + ) def __eq__(self, other): return (type(self) is type(other)) and hash(self) == hash(other) @@ -212,8 +253,11 @@ def __neq__(self, other): def __repr__(self): return "id(function_spec): {}, input_args_with_spec: {}, input_kwargs_with_spec: {}, class_instance: {}".format( - id(self.function_spec), self.input_args_with_spec, - self.input_kwargs_with_spec, self.class_instance) + id(self.function_spec), + self.input_args_with_spec, + self.input_kwargs_with_spec, + self.class_instance, + ) def unwrap_decorators(func): @@ -260,9 +304,11 @@ def __init__(self, function, input_spec=None, **kwargs): if not hasattr(self._class_instance, '_original_funcs'): raise TypeError( "When using 'to_static' to convert method of a class, " - "please ensure the class inherits from nn.Layer") + "please ensure the class inherits from nn.Layer" + ) self._class_instance._original_funcs[ - function.__name__] = self._dygraph_function + function.__name__ + ] = self._dygraph_function else: self._dygraph_function = function self._class_instance = None @@ -286,21 +332,29 @@ def is_property(self): return self._property def train(self): - if isinstance(self._class_instance, - layers.Layer) and self._class_instance.training == False: + if ( + isinstance(self._class_instance, layers.Layer) + and self._class_instance.training == False + ): raise RuntimeError( "Failed to switch train mode. {} is a Layer's method, " "please use Layer.train() to switch train mode.".format( - self.dygraph_function)) + self.dygraph_function + ) + ) self._training = True def eval(self): - if isinstance(self._class_instance, - layers.Layer) and self._class_instance.training == True: + if ( + isinstance(self._class_instance, layers.Layer) + and self._class_instance.training == True + ): raise RuntimeError( "Failed to switch eval mode. {} is a Layer's method, " "please use Layer.eval() to switch eval mode.".format( - self.dygraph_function)) + self.dygraph_function + ) + ) self._training = False def __get__(self, instance, owner): @@ -338,8 +392,9 @@ def forward(self, x, y): return self._descriptor_cache[instance] def _clone(self): - return self.__class__(self._dygraph_function, self._input_spec, - **self._kwargs) + return self.__class__( + self._dygraph_function, self._input_spec, **self._kwargs + ) def __call__(self, *args, **kwargs): """ @@ -364,7 +419,8 @@ def __call__(self, *args, **kwargs): logging_utils.warn( "The decorator '@paddle.jit.to_static' does NOT work when setting ProgramTranslator.enable to False. " "We will just return dygraph output. If you would like to get static graph output, please call API " - "ProgramTranslator.enable(True)") + "ProgramTranslator.enable(True)" + ) return self._call_dygraph_function(*args, **kwargs) if not _non_static_mode(): @@ -372,21 +428,26 @@ def __call__(self, *args, **kwargs): "Failed to run the callable object {} decorated by '@paddle.jit.to_static', " "because it is NOT in dynamic mode. Please disable the static mode to enter dynamic mode with the " "following API: paddle.disable_static().".format( - self.dygraph_function)) + self.dygraph_function + ) + ) # 2. trace ops from dygraph layers and cache the generated program. args, kwargs = self._function_spec.unified_args_and_kwargs(args, kwargs) try: concrete_program, partial_program_layer = self.get_concrete_program( - *args, **kwargs, is_train=self._is_train_mode()) + *args, **kwargs, is_train=self._is_train_mode() + ) # 3. synchronize self.training attribute. if isinstance(self._class_instance, layers.Layer): partial_program_layer.training = self._class_instance.training else: partial_program_layer.training = self._training - partial_program_layer._cuda_graph_capture_mode = self._cuda_graph_capture_mode + partial_program_layer._cuda_graph_capture_mode = ( + self._cuda_graph_capture_mode + ) partial_program_layer._cuda_graph_pool_id = self._cuda_graph_pool_id # 4. return outputs. @@ -404,7 +465,8 @@ def __call__(self, *args, **kwargs): else: logging_utils.warn( "Please file an issue at 'https://github.com/PaddlePaddle/Paddle/issues'" - " if you can't handle this {} yourself.".format(type(e))) + " if you can't handle this {} yourself.".format(type(e)) + ) raise e def _is_train_mode(self): @@ -412,7 +474,8 @@ def _is_train_mode(self): if not hasattr(self._class_instance, 'training'): raise TypeError( "When using 'to_static' to convert method of a class, " - "please ensure the class inherits from nn.Layer") + "please ensure the class inherits from nn.Layer" + ) return self._class_instance.training else: return self._training @@ -430,7 +493,8 @@ def _call_dygraph_function(self, *args, **kwargs): """ if self._class_instance is not None: dygraph_function = self._dygraph_function.__get__( - self._class_instance) + self._class_instance + ) else: dygraph_function = self._dygraph_function @@ -460,23 +524,30 @@ def get_concrete_program(self, *args, **kwargs): with_hook = kwargs.get("with_hook", False) is_train = kwargs.get("is_train", True) - if "is_train" in kwargs: kwargs.pop("is_train") - if "with_hook" in kwargs: kwargs.pop("with_hook") + if "is_train" in kwargs: + kwargs.pop("is_train") + if "with_hook" in kwargs: + kwargs.pop("with_hook") # 1. unify args/kwargs and replace Tensor with InputSpec if len(args) != len(self._function_spec.args_name): args, kwargs = self._function_spec.unified_args_and_kwargs( - args, kwargs) - input_args_with_spec, input_kwargs_with_spec = self._function_spec.args_to_input_spec( - args, kwargs) + args, kwargs + ) + ( + input_args_with_spec, + input_kwargs_with_spec, + ) = self._function_spec.args_to_input_spec(args, kwargs) # 2. generate cache key - cache_key = CacheKey(self._function_spec, - input_args_with_spec, - input_kwargs_with_spec, - self._class_instance, - **self._kwargs, - with_hook=with_hook, - is_train=is_train) + cache_key = CacheKey( + self._function_spec, + input_args_with_spec, + input_kwargs_with_spec, + self._class_instance, + **self._kwargs, + with_hook=with_hook, + is_train=is_train + ) # 3. check whether hit the cache or build a new program for the input arguments concrete_program, partial_program_layer = self._program_cache[cache_key] @@ -533,9 +604,9 @@ def foo(x, y): """ return self.concrete_program_specify_input_spec(input_spec=None) - def concrete_program_specify_input_spec(self, - input_spec=None, - with_hook=False): + def concrete_program_specify_input_spec( + self, input_spec=None, with_hook=False + ): """ Returns recent ConcreteProgram instance of decorated function while specifying input_spec. If the self._function_spec already has @@ -556,30 +627,37 @@ def concrete_program_specify_input_spec(self, desired_input_spec = input_spec if self._function_spec.input_spec is not None: if input_spec is not None and not input_specs_compatible( - flatten(input_spec), - flatten(self._function_spec.input_spec)): + flatten(input_spec), flatten(self._function_spec.input_spec) + ): raise ValueError( - "The `input_spec`: {} used to construct concrete_program is conflict with the `input_spec`: {} in `@paddle.jit.to_static`" - .format(input_spec, self._function_spec.input_spec)) + "The `input_spec`: {} used to construct concrete_program is conflict with the `input_spec`: {} in `@paddle.jit.to_static`".format( + input_spec, self._function_spec.input_spec + ) + ) # NOTE(chenweihang): we should always translated program based on the `input_spec` # decorated on forward if it is valid desired_input_spec = self._function_spec.input_spec if input_spec is not None: logging_utils.warn( - "\n\nYou have specified `input_spec` both in function definition (higher priority) and `paddle.jit.save` (will be ignored.)\n\n\t Using: {}\n\n\t Ignore: {}\n" - .format(desired_input_spec, input_spec)) + "\n\nYou have specified `input_spec` both in function definition (higher priority) and `paddle.jit.save` (will be ignored.)\n\n\t Using: {}\n\n\t Ignore: {}\n".format( + desired_input_spec, input_spec + ) + ) - has_input_spec = (desired_input_spec is not None) + has_input_spec = desired_input_spec is not None if has_input_spec: concrete_program, _ = self.get_concrete_program( *desired_input_spec, with_hook=with_hook, - is_train=self._is_train_mode()) + is_train=self._is_train_mode() + ) return concrete_program else: raise ValueError( - "No valid transformed program for {}.\n\t Please specific `input_spec` in `@paddle.jit.to_static` or feed input tensor to call the decorated function at once.\n" - .format(self._function_spec)) + "No valid transformed program for {}.\n\t Please specific `input_spec` in `@paddle.jit.to_static` or feed input tensor to call the decorated function at once.\n".format( + self._function_spec + ) + ) elif with_hook: cache_key = self._program_cache._recent_cache_key cache_key.kwargs["with_hook"] = True @@ -589,11 +667,15 @@ def concrete_program_specify_input_spec(self, # If more than one programs have been cached, return the recent converted program by default. elif cached_program_len > 1: logging_utils.warn( - "Current {} has more than one cached programs: {}, the last traced progam will be return by default." - .format(self._function_spec, cached_program_len)) + "Current {} has more than one cached programs: {}, the last traced progam will be return by default.".format( + self._function_spec, cached_program_len + ) + ) - cache_key, (concrete_program, - partial_layer) = self._program_cache.last() + cache_key, ( + concrete_program, + partial_layer, + ) = self._program_cache.last() return concrete_program def rollback(self): @@ -639,11 +721,15 @@ def rollback_impl(class_instance): # only rollback sub-functions on path of top _dygraph_function func_name = self._dygraph_function.__name__ - assert func_name in self._class_instance._original_funcs, "Not Found function '{}' in class '{}'.".format( - func_name, self._class_instance.__name__) + assert ( + func_name in self._class_instance._original_funcs + ), "Not Found function '{}' in class '{}'.".format( + func_name, self._class_instance.__name__ + ) func = self._class_instance._original_funcs[func_name] - setattr(self._class_instance, func_name, - func.__get__(self._class_instance)) + setattr( + self._class_instance, func_name, func.__get__(self._class_instance) + ) for sublayer in self._class_instance.sublayers(include_self=False): rollback_impl(sublayer) @@ -686,12 +772,15 @@ def forward(self, x, flag=True): net_name = type(self._class_instance).__name__ logging_utils.log( level=-1, - msg="Not recommend to deepcopy '{}' decorated with @to_static, it has side effect that will" \ - " rollback into original state before @to_static. Please deepcopy '{}' before applying @to_static." - .format(net_name, net_name)) + msg="Not recommend to deepcopy '{}' decorated with @to_static, it has side effect that will" + " rollback into original state before @to_static. Please deepcopy '{}' before applying @to_static.".format( + net_name, net_name + ), + ) self.rollback() - return self._dygraph_function.__get__(memo[id( - self._class_instance)]) + return self._dygraph_function.__get__( + memo[id(self._class_instance)] + ) else: return self._dygraph_function @@ -703,7 +792,8 @@ def inputs(self): self._raise_when_property() concrete_program = self.concrete_program inputs = [ - var for var in flatten(concrete_program.inputs) + var + for var in flatten(concrete_program.inputs) if isinstance(var, framework.Variable) ] return inputs @@ -716,7 +806,8 @@ def outputs(self): self._raise_when_property() concrete_program = self.concrete_program outputs = [ - var for var in flatten(concrete_program.outputs) + var + for var in flatten(concrete_program.outputs) if isinstance(var, framework.Variable) ] @@ -751,7 +842,9 @@ def _verify_init_in_dynamic_mode(class_instance): " `paddle.jit.to_static` is only available in dynamic mode. Please call `paddle.disable_static()` before " "initializing your Layer class `{}` . Because parameters of Layer class should be initialized firstly " "in dynamic mode while applying transformation.".format( - class_instance)) + class_instance + ) + ) class HookHelper(object): @@ -764,22 +857,25 @@ def __init__(self, func, class_instance, with_hook=False): self.func = func self.class_instance = class_instance self.with_hook = with_hook - self.need_apply_hook = with_hook and isinstance( - self.class_instance, layers.Layer) and getattr( - func, "__name__") == "forward" + self.need_apply_hook = ( + with_hook + and isinstance(self.class_instance, layers.Layer) + and getattr(func, "__name__") == "forward" + ) def apply_pre_hooks(self, inputs): """ Apply _forward_pre_hooks from outermost layer """ - if not self.need_apply_hook: return inputs + if not self.need_apply_hook: + return inputs inputs = inputs[1:] for forward_pre_hook in self.class_instance._forward_pre_hooks.values(): hook_result = forward_pre_hook(self.class_instance, inputs) if hook_result is not None: if not isinstance(hook_result, tuple): - hook_result = (hook_result, ) + hook_result = (hook_result,) inputs = hook_result return [self.class_instance] + list(inputs) @@ -788,13 +884,16 @@ def apply_post_hooks(self, inputs, outputs): """ Apply _forward_post_hooks from outermost layer """ - if not self.need_apply_hook: return outputs + if not self.need_apply_hook: + return outputs inputs = inputs[1:] - for forward_post_hook in self.class_instance._forward_post_hooks.values( - ): - hook_result = forward_post_hook(self.class_instance, inputs, - outputs) + for ( + forward_post_hook + ) in self.class_instance._forward_post_hooks.values(): + hook_result = forward_post_hook( + self.class_instance, inputs, outputs + ) if hook_result is not None: outputs = hook_result @@ -805,18 +904,25 @@ def apply_post_hooks(self, inputs, outputs): class ConcreteProgram(object): __slots__ = [ - 'inputs', 'outputs', 'main_program', "startup_program", "parameters", - "function", 'kwargs' + 'inputs', + 'outputs', + 'main_program', + "startup_program", + "parameters", + "function", + 'kwargs', ] - def __init__(self, - inputs, - outputs, - parameters, - function, - main_program, - startup_program=None, - **kwargs): + def __init__( + self, + inputs, + outputs, + parameters, + function, + main_program, + startup_program=None, + **kwargs + ): self.inputs = inputs self.outputs = outputs self.main_program = main_program @@ -827,8 +933,9 @@ def __init__(self, @staticmethod @switch_to_static_graph - def from_func_spec(func_spec, input_spec, input_kwargs_spec, class_instance, - **kwargs): + def from_func_spec( + func_spec, input_spec, input_kwargs_spec, class_instance, **kwargs + ): """ Builds the main_program with specialized inputs and returns outputs of program as fetch_list. @@ -844,38 +951,44 @@ def from_func_spec(func_spec, input_spec, input_kwargs_spec, class_instance, dygraph_function = func_spec.dygraph_function static_func = convert_to_static(dygraph_function) # apply pre\post hook for outermost layer - hook_helper = HookHelper(dygraph_function, class_instance, - kwargs.get("with_hook", False)) + hook_helper = HookHelper( + dygraph_function, class_instance, kwargs.get("with_hook", False) + ) main_program, startup_program = framework.Program(), framework.Program() # Note: The random seed should be synchronized into cached program # if set in `fluid.dygraph_guard` because some ops rely on it, such as # `fluid.layers.dropout`. main_program.random_seed = framework.default_main_program().random_seed - startup_program.random_seed = framework.default_startup_program( - ).random_seed + startup_program.random_seed = ( + framework.default_startup_program().random_seed + ) from paddle.fluid.dygraph.base import _switch_declarative_mode_guard_ + with framework.program_guard(main_program, startup_program): with _switch_declarative_mode_guard_(is_declarative=True): # 1. Adds `fluid.data` layers for input if needed static_inputs = func_spec.to_static_inputs_with_spec( - input_spec, main_program) + input_spec, main_program + ) _kwargs = func_spec.to_static_inputs_with_spec( - input_kwargs_spec, main_program) + input_kwargs_spec, main_program + ) if class_instance: - static_inputs = tuple([class_instance] + - list(static_inputs)) + static_inputs = tuple( + [class_instance] + list(static_inputs) + ) # 2. Gets all ParamBases and buffered VarBases in the function all_parameters_and_buffers = _extract_indeed_params_buffers( - class_instance) + class_instance + ) # 3. Builds program only once and returns the output Variables. - with param_guard(get_parameters( - class_instance, - False)), param_guard(get_buffers(class_instance, - False)): + with param_guard( + get_parameters(class_instance, False) + ), param_guard(get_buffers(class_instance, False)): try: # only for jit.save, do nothing while train and eval process inputs = hook_helper.apply_pre_hooks(static_inputs) @@ -893,20 +1006,24 @@ def from_func_spec(func_spec, input_spec, input_kwargs_spec, class_instance, raise if outputs is not None: - need_wrap_into_list = not isinstance( - outputs, (tuple, list)) or len(outputs) == 1 + need_wrap_into_list = ( + not isinstance(outputs, (tuple, list)) + or len(outputs) == 1 + ) if need_wrap_into_list: outputs = [outputs] main_program = update_op_callstack_with_origin_info(main_program) - return ConcreteProgram(inputs=static_inputs, - outputs=outputs, - parameters=all_parameters_and_buffers, - function=dygraph_function, - main_program=main_program, - startup_program=startup_program, - **kwargs) + return ConcreteProgram( + inputs=static_inputs, + outputs=outputs, + parameters=all_parameters_and_buffers, + function=dygraph_function, + main_program=main_program, + startup_program=startup_program, + **kwargs + ) def _extract_indeed_params_buffers(class_instance): @@ -938,13 +1055,16 @@ def _build_once(self, cache_key): input_spec=cache_key.input_args_with_spec, input_kwargs_spec=cache_key.input_kwargs_with_spec, class_instance=cache_key.class_instance, - **cache_key.kwargs) + **cache_key.kwargs + ) return concrete_program, partial_program_from(concrete_program) def __getitem__(self, item): if not isinstance(item, CacheKey): - raise ValueError('type(item) should be CacheKey, but received %s' % - type_name(item)) + raise ValueError( + 'type(item) should be CacheKey, but received %s' + % type_name(item) + ) item_id = hash(item) self._recent_cache_key = item self._recent_key = item_id @@ -955,16 +1075,19 @@ def __getitem__(self, item): if current_tracing_count > MAX_TRACED_PROGRAM_COUNT: logging_utils.warn( "Current traced program number: {} > `max_tracing_count`:{}. Too much cached programs will bring expensive overhead. " - "The reason may be: (1) passing tensors with different shapes, (2) passing python objects instead of tensors." - .format(current_tracing_count, MAX_TRACED_PROGRAM_COUNT)) + "The reason may be: (1) passing tensors with different shapes, (2) passing python objects instead of tensors.".format( + current_tracing_count, MAX_TRACED_PROGRAM_COUNT + ) + ) return self._caches[item_id] def get_program(self, item): if not isinstance(item, CacheKey): raise ValueError( - "Input item's type should be FunctionSpec, but received %s" % - type_name(item)) + "Input item's type should be FunctionSpec, but received %s" + % type_name(item) + ) item_id = hash(item) if item_id not in self._caches: raise RuntimeError( @@ -973,8 +1096,9 @@ def get_program(self, item): return self._caches[item_id] def last(self): - assert len( - self._caches) >= 1, "No valid cached program in ProgramCache." + assert ( + len(self._caches) >= 1 + ), "No valid cached program in ProgramCache." assert self._recent_key is not None return self._recent_key, self._caches[self._recent_key] @@ -1082,8 +1206,12 @@ def func(x): print(func(x)) # [[0. 0.]] """ - check_type(enable_to_static, "enable_to_static", bool, - "ProgramTranslator.enable") + check_type( + enable_to_static, + "enable_to_static", + bool, + "ProgramTranslator.enable", + ) self.enable_to_static = enable_to_static def get_output(self, dygraph_func, *args, **kwargs): @@ -1137,8 +1265,11 @@ def func(x): try: function_spec = FunctionSpec(dygraph_func) cache_key = CacheKey.from_func_and_args( - function_spec, args, kwargs, - getattr(dygraph_func, '__self__', None)) + function_spec, + args, + kwargs, + getattr(dygraph_func, '__self__', None), + ) _, partial_program_layer = self._program_cache[cache_key] if args and isinstance(args[0], layers.Layer): @@ -1162,7 +1293,8 @@ def func(x): else: logging_utils.warn( "Please file an issue at 'https://github.com/PaddlePaddle/Paddle/issues'" - " if you can't handle this {} yourself.".format(type(e))) + " if you can't handle this {} yourself.".format(type(e)) + ) raise e def get_func(self, dygraph_func): @@ -1267,24 +1399,28 @@ def func(x): function_spec = FunctionSpec(dygraph_func) cache_key = CacheKey.from_func_and_args( - function_spec, args, kwargs, getattr(dygraph_func, '__self__', - None)) + function_spec, args, kwargs, getattr(dygraph_func, '__self__', None) + ) concrete_program, partial_program_layer = self._program_cache[cache_key] # Note: concrete_program hold all input/output infos include non-Variable input_vars = [ - var for var in concrete_program.inputs + var + for var in concrete_program.inputs if isinstance(var, framework.Variable) ] output_vars = [ - var for var in concrete_program.outputs + var + for var in concrete_program.outputs if isinstance(var, framework.Variable) ] - return concrete_program.main_program, \ - concrete_program.startup_program, \ - input_vars, \ - output_vars + return ( + concrete_program.main_program, + concrete_program.startup_program, + input_vars, + output_vars, + ) def get_code(self, dygraph_func): """ diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py index 88277bc832fe7..80bebbf501e55 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/return_transformer.py @@ -16,15 +16,23 @@ from paddle.fluid import unique_name from paddle.fluid.dygraph.dygraph_to_static.utils import index_in_list -from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import ForToWhileTransformer -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_fill_constant_node +from paddle.fluid.dygraph.dygraph_to_static.break_continue_transformer import ( + ForToWhileTransformer, +) +from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + create_fill_constant_node, +) from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) from paddle.fluid.dygraph.dygraph_to_static.utils import Dygraph2StaticException from paddle.fluid.dygraph.dygraph_to_static.utils import ORIGI_INFO __all__ = [ - 'RETURN_NO_VALUE_MAGIC_NUM', 'RETURN_NO_VALUE_VAR_NAME', 'ReturnTransformer' + 'RETURN_NO_VALUE_MAGIC_NUM', + 'RETURN_NO_VALUE_VAR_NAME', + 'ReturnTransformer', ] # Constant for the name of the variable which stores the boolean state that we @@ -45,7 +53,7 @@ # should return. # Assign not support float64, use float32 value as magic number. -RETURN_NO_VALUE_MAGIC_NUM = 1.77113e+27 +RETURN_NO_VALUE_MAGIC_NUM = 1.77113e27 RETURN_NO_VALUE_VAR_NAME = "__no_value_return_var" @@ -91,7 +99,8 @@ class ReturnAnalysisVisitor(gast.NodeVisitor): def __init__(self, root_node): self.root = root_node assert isinstance( - self.root, gast.FunctionDef), "Input is not gast.FunctionDef node" + self.root, gast.FunctionDef + ), "Input is not gast.FunctionDef node" # the number of return statements self.count_return = 0 @@ -157,7 +166,8 @@ class SingleReturnTransformer(BaseTransformer): def __init__(self, root): self.root = root assert isinstance( - self.root, gast.FunctionDef), "Input is not gast.FunctionDef node" + self.root, gast.FunctionDef + ), "Input is not gast.FunctionDef node" self.ancestor_nodes = [] @@ -206,15 +216,18 @@ def visit_FunctionDef(self, node): self.generic_visit(node) return node - def append_assign_to_return_node(self, value, parent_node_of_return, - return_name, assign_nodes): + def append_assign_to_return_node( + self, value, parent_node_of_return, return_name, assign_nodes + ): self.assert_parent_is_not_while(parent_node_of_return) assert value in [True, False], "value must be True or False." if isinstance(parent_node_of_return, gast.If): # Prepend control flow boolean nodes such as '__return@1 = True' node_str = "{} = _jst.create_bool_as_type({}, {})".format( return_name, - ast_to_source_code(parent_node_of_return.test).strip(), value) + ast_to_source_code(parent_node_of_return.test).strip(), + value, + ) assign_node = gast.parse(node_str).body[0] assign_nodes.append(assign_node) @@ -236,18 +249,26 @@ def transform(self): value_name = self.return_value_name if value_name is not None: node.body.append( - gast.Return(value=gast.Name(id=value_name, - ctx=gast.Load(), - annotation=None, - type_comment=None))) - assign_return_value_node = gast.Assign(targets=[ - gast.Name(id=value_name, - ctx=gast.Store(), - annotation=None, - type_comment=None) - ], - value=gast.Constant( - kind=None, value=None)) + gast.Return( + value=gast.Name( + id=value_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ) + ) + ) + assign_return_value_node = gast.Assign( + targets=[ + gast.Name( + id=value_name, + ctx=gast.Store(), + annotation=None, + type_comment=None, + ) + ], + value=gast.Constant(kind=None, value=None), + ) node.body.insert(0, assign_return_value_node) # Prepend no value placeholders @@ -269,37 +290,52 @@ def _deal_branches(branch_name): if index_in_list(branch_node, cur_node) != -1: if cur_node == node: self._replace_return_in_stmt_list( - branch_node, cur_node, return_name, - max_return_length, parent_node_of_return) + branch_node, + cur_node, + return_name, + max_return_length, + parent_node_of_return, + ) self._replace_after_node_to_if_in_stmt_list( - branch_node, cur_node, return_name, - parent_node_of_return) + branch_node, + cur_node, + return_name, + parent_node_of_return, + ) _deal_branches("body") _deal_branches("orelse") # If return node in while loop, add `not return_name` in gast.While.test if isinstance(ancestor, gast.While): - cond_var_node = gast.UnaryOp(op=gast.Not(), - operand=gast.Name( - id=return_name, - ctx=gast.Load(), - annotation=None, - type_comment=None)) + cond_var_node = gast.UnaryOp( + op=gast.Not(), + operand=gast.Name( + id=return_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ), + ) ancestor.test = gast.BoolOp( - op=gast.And(), values=[ancestor.test, cond_var_node]) + op=gast.And(), values=[ancestor.test, cond_var_node] + ) continue # If return node in for loop, add `not return_name` in gast.While.test if isinstance(ancestor, gast.For): - cond_var_node = gast.UnaryOp(op=gast.Not(), - operand=gast.Name( - id=return_name, - ctx=gast.Load(), - annotation=None, - type_comment=None)) + cond_var_node = gast.UnaryOp( + op=gast.Not(), + operand=gast.Name( + id=return_name, + ctx=gast.Load(), + annotation=None, + type_comment=None, + ), + ) parent_node = self.ancestor_nodes[ancestor_index - 1] - for_to_while = ForToWhileTransformer(parent_node, ancestor, - cond_var_node) + for_to_while = ForToWhileTransformer( + parent_node, ancestor, cond_var_node + ) new_stmts = for_to_while.transform() while_node = new_stmts[-1] self.ancestor_nodes[ancestor_index] = while_node @@ -308,8 +344,14 @@ def _deal_branches(branch_name): break # return_node is replaced so we shouldn't return here - def _replace_return_in_stmt_list(self, stmt_list, return_node, return_name, - max_return_length, parent_node_of_return): + def _replace_return_in_stmt_list( + self, + stmt_list, + return_node, + return_name, + max_return_length, + parent_node_of_return, + ): assert max_return_length >= 0, "Input illegal max_return_length" i = index_in_list(stmt_list, return_node) @@ -317,24 +359,31 @@ def _replace_return_in_stmt_list(self, stmt_list, return_node, return_name, return False assign_nodes = [] - self.append_assign_to_return_node(True, parent_node_of_return, - return_name, assign_nodes) + self.append_assign_to_return_node( + True, parent_node_of_return, return_name, assign_nodes + ) return_length = get_return_size(return_node) # In this case we should NOT append RETURN_NO_VALUE placeholder if return_node.value is not None: if self.return_value_name is None: self.return_value_name = unique_name.generate( - RETURN_VALUE_PREFIX) + RETURN_VALUE_PREFIX + ) assign_nodes.append( - gast.Assign(targets=[ - gast.Name(id=self.return_value_name, - ctx=gast.Store(), - annotation=None, - type_comment=None) - ], - value=return_node.value)) + gast.Assign( + targets=[ + gast.Name( + id=self.return_value_name, + ctx=gast.Store(), + annotation=None, + type_comment=None, + ) + ], + value=return_node.value, + ) + ) return_origin_info = getattr(return_node, ORIGI_INFO, None) setattr(assign_nodes[-1], ORIGI_INFO, return_origin_info) @@ -343,9 +392,9 @@ def _replace_return_in_stmt_list(self, stmt_list, return_node, return_name, stmt_list[i:] = assign_nodes return True - def _replace_after_node_to_if_in_stmt_list(self, stmt_list, node, - return_name, - parent_node_of_return): + def _replace_after_node_to_if_in_stmt_list( + self, stmt_list, node, return_name, parent_node_of_return + ): i = index_in_list(stmt_list, node) if i < 0 or i >= len(stmt_list): return False @@ -353,20 +402,26 @@ def _replace_after_node_to_if_in_stmt_list(self, stmt_list, node, # No need to add, we consider this as added successfully return True - if_stmt = gast.If(test=gast.UnaryOp(op=gast.Not(), - operand=gast.Name( - id=return_name, - ctx=gast.Store(), - annotation=None, - type_comment=None)), - body=stmt_list[i + 1:], - orelse=[]) - - stmt_list[i + 1:] = [if_stmt] + if_stmt = gast.If( + test=gast.UnaryOp( + op=gast.Not(), + operand=gast.Name( + id=return_name, + ctx=gast.Store(), + annotation=None, + type_comment=None, + ), + ), + body=stmt_list[i + 1 :], + orelse=[], + ) + + stmt_list[i + 1 :] = [if_stmt] # Here assume that the parent node of return is gast.If assign_nodes = [] - self.append_assign_to_return_node(False, parent_node_of_return, - return_name, assign_nodes) + self.append_assign_to_return_node( + False, parent_node_of_return, return_name, assign_nodes + ) stmt_list[i:i] = assign_nodes return True diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py b/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py index 16723bea10eeb..bf07523e12dba 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/static_analysis.py @@ -14,7 +14,13 @@ from paddle.utils import gast from .logging_utils import warn -from .utils import is_paddle_api, is_dygraph_api, is_numpy_api, index_in_list, ast_to_source_code +from .utils import ( + is_paddle_api, + is_dygraph_api, + is_numpy_api, + index_in_list, + ast_to_source_code, +) __all__ = ['AstNodeWrapper', 'NodeVarType', 'StaticAnalysisVisitor'] @@ -26,6 +32,7 @@ class NodeVarType(object): tensor variable in if clause may lead to different conversion from dygraph to static graph. """ + ERROR = -1 # Returns when static analysis gets error UNKNOWN = 0 # Reserve for AST nodes have not known the type STATEMENT = 1 # For nodes representing statement (non-variable type) @@ -62,7 +69,7 @@ class NodeVarType(object): "int": INT, "float": FLOAT, "bool": BOOLEAN, - "str": STRING + "str": STRING, } @staticmethod @@ -76,9 +83,12 @@ def binary_op_output_type(in_type1, in_type2): return in_type1 supported_types = [ - NodeVarType.BOOLEAN, NodeVarType.INT, NodeVarType.FLOAT, - NodeVarType.NUMPY_NDARRAY, NodeVarType.TENSOR, - NodeVarType.PADDLE_RETURN_TYPES + NodeVarType.BOOLEAN, + NodeVarType.INT, + NodeVarType.FLOAT, + NodeVarType.NUMPY_NDARRAY, + NodeVarType.TENSOR, + NodeVarType.PADDLE_RETURN_TYPES, ] if in_type1 not in supported_types: @@ -122,14 +132,14 @@ class AstVarScope(object): AstVarScope is a class holding the map from current scope variable to its type. """ + SCOPE_TYPE_SCRIPT = 0 SCOPE_TYPE_FUNCTION = 1 SCOPE_TYPE_CLASS = 2 - def __init__(self, - scope_name='', - scope_type=SCOPE_TYPE_SCRIPT, - parent_scope=None): + def __init__( + self, scope_name='', scope_type=SCOPE_TYPE_SCRIPT, parent_scope=None + ): self.sub_scopes = [] self.name_to_id = {} self.id_to_type = {} @@ -158,8 +168,9 @@ def set_var_type(self, var_name, node_var_type): num_id = self.cur_id self.cur_id += 1 self.name_to_id[var_name] = num_id - self.id_to_type[num_id] = node_var_type if isinstance( - node_var_type, set) else {node_var_type} + self.id_to_type[num_id] = ( + node_var_type if isinstance(node_var_type, set) else {node_var_type} + ) def get_var_type(self, var_name): if var_name in self.name_to_id: @@ -179,20 +190,24 @@ def __init__(self): self.cur_scope = AstVarScope() def enter_scope(self, scope_name, scope_type): - self.cur_scope = AstVarScope(scope_name, - scope_type, - parent_scope=self.cur_scope) + self.cur_scope = AstVarScope( + scope_name, scope_type, parent_scope=self.cur_scope + ) return self.cur_scope def exit_scope(self): - assert self.cur_scope.parent_scope is not None, "Call exit_scope in "\ + assert self.cur_scope.parent_scope is not None, ( + "Call exit_scope in " "AstVarEnv when current scope doesn't have parent scope." + ) self.cur_scope = self.cur_scope.parent_scope return self.cur_scope def get_parent_scope(self): - assert self.cur_scope.parent_scope is not None, "Call parent_scope in "\ + assert self.cur_scope.parent_scope is not None, ( + "Call parent_scope in " "AstVarEnv when current scope doesn't have parent scope." + ) return self.cur_scope.parent_scope def add_var_type(self, var_name, node_var_type): @@ -252,11 +267,13 @@ def dfs_visit(self, node): self.ancestor_wrappers.append(cur_wrapper) for child in gast.iter_child_nodes(node): if isinstance(child, gast.FunctionDef) or isinstance( - child, gast.AsyncFunctionDef): + child, gast.AsyncFunctionDef + ): # TODO: current version is function name mapping to its type # consider complex case involving parameters - self.var_env.enter_scope(child.name, - AstVarScope.SCOPE_TYPE_FUNCTION) + self.var_env.enter_scope( + child.name, AstVarScope.SCOPE_TYPE_FUNCTION + ) func_type = self.dfs_visit(child) self.var_env.exit_scope() else: @@ -284,8 +301,10 @@ def is_tensor_node(self, node): return True def _get_constant_node_type(self, node): - assert isinstance(node, gast.Constant), \ - "Type of input node should be gast.Constant, but received %s" % type(node) + assert isinstance(node, gast.Constant), ( + "Type of input node should be gast.Constant, but received %s" + % type(node) + ) # singleton: None, True or False if node.value is None: return {NodeVarType.NONE} @@ -338,7 +357,8 @@ def _get_node_var_type(self, cur_wrapper): for sub_target in target.elts: if isinstance(sub_target, gast.Name): self.node_to_wrapper_map[ - sub_target].node_var_type = ret_type + sub_target + ].node_var_type = ret_type self.var_env.set_var_type(sub_target.id, ret_type) return ret_type @@ -349,9 +369,12 @@ def _get_node_var_type(self, cur_wrapper): # if annotation and value(Constant) are diffent type, we use value type if node.value: node_value_type = self.node_to_wrapper_map[ - node.value].node_var_type - if not (node_value_type - & {NodeVarType.UNKNOWN, NodeVarType.STATEMENT}): + node.value + ].node_var_type + if not ( + node_value_type + & {NodeVarType.UNKNOWN, NodeVarType.STATEMENT} + ): ret_type = node_value_type if isinstance(node.target, gast.Name): self.node_to_wrapper_map[node.target].node_var_type = ret_type @@ -365,8 +388,9 @@ def _get_node_var_type(self, cur_wrapper): return {NodeVarType.BOOLEAN} # If node is child of functionDef.arguments parent_node_wrapper = cur_wrapper.parent - if parent_node_wrapper and isinstance(parent_node_wrapper.node, - gast.arguments): + if parent_node_wrapper and isinstance( + parent_node_wrapper.node, gast.arguments + ): return self._get_func_argument_type(parent_node_wrapper, node) @@ -378,7 +402,10 @@ def _get_node_var_type(self, cur_wrapper): return {NodeVarType.NONE} return_type = self.node_to_wrapper_map[node.value].node_var_type - assert self.var_env.cur_scope.scope_type == AstVarScope.SCOPE_TYPE_FUNCTION, "Return at non-function scope" + assert ( + self.var_env.cur_scope.scope_type + == AstVarScope.SCOPE_TYPE_FUNCTION + ), "Return at non-function scope" func_name = self.var_env.cur_scope.scope_name parent_scope = self.var_env.get_parent_scope() parent_scope.add_var_type(func_name, return_type) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py index b11f18d2658a2..e7a882b28a296 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/tensor_shape_transformer.py @@ -15,8 +15,12 @@ from paddle.utils import gast from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) class TensorShapeTransformer(BaseTransformer): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/typehint_transformer.py b/python/paddle/fluid/dygraph/dygraph_to_static/typehint_transformer.py index f258b98b50711..3fddc0bcf58c2 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/typehint_transformer.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/typehint_transformer.py @@ -15,9 +15,13 @@ from paddle.utils import gast import warnings -from paddle.fluid.dygraph.dygraph_to_static.static_analysis import AstNodeWrapper +from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + AstNodeWrapper, +) from paddle.fluid.dygraph.dygraph_to_static import utils -from paddle.fluid.dygraph.dygraph_to_static.base_transformer import BaseTransformer +from paddle.fluid.dygraph.dygraph_to_static.base_transformer import ( + BaseTransformer, +) class TypeHintTransformer(BaseTransformer): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/utils.py b/python/paddle/fluid/dygraph/dygraph_to_static/utils.py index 8e1950b21fca7..8b4a7ac645ec0 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/utils.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/utils.py @@ -98,10 +98,18 @@ def visit(self, node): # FullArgSpec is valid from Python3. Defined a Namedtuple to # to make it available in Python2. -FullArgSpec = collections.namedtuple('FullArgSpec', [ - 'args', 'varargs', 'varkw', 'defaults', 'kwonlyargs', 'kwonlydefaults', - 'annotations' -]) +FullArgSpec = collections.namedtuple( + 'FullArgSpec', + [ + 'args', + 'varargs', + 'varkw', + 'defaults', + 'kwonlyargs', + 'kwonlydefaults', + 'annotations', + ], +) def data_layer_not_check(name, shape, dtype='float32', lod_level=0): @@ -139,20 +147,26 @@ def data_layer_not_check(name, shape, dtype='float32', lod_level=0): if shape[i] is None: shape[i] = -1 - return helper.create_global_variable(name=name, - shape=shape, - dtype=dtype, - type=core.VarDesc.VarType.LOD_TENSOR, - stop_gradient=True, - lod_level=lod_level, - is_data=True, - need_check_feed=False) + return helper.create_global_variable( + name=name, + shape=shape, + dtype=dtype, + type=core.VarDesc.VarType.LOD_TENSOR, + stop_gradient=True, + lod_level=lod_level, + is_data=True, + need_check_feed=False, + ) def create_undefined_variable(): - from paddle.fluid.dygraph.dygraph_to_static.return_transformer import RETURN_NO_VALUE_MAGIC_NUM - var = data_layer_not_check(unique_name.generate("undefined_var"), [1], - "float64") + from paddle.fluid.dygraph.dygraph_to_static.return_transformer import ( + RETURN_NO_VALUE_MAGIC_NUM, + ) + + var = data_layer_not_check( + unique_name.generate("undefined_var"), [1], "float64" + ) var.stop_gradient = False # the variable is created in block(0), we append assign in block(0) either. helper = LayerHelper('create_undefined_variable', **locals()) @@ -164,17 +178,16 @@ def create_undefined_variable(): class UndefinedVar: - def __init__(self, name): self.name = name def check(self): raise UnboundLocalError( - "local variable '{}' should be created before using it.") + "local variable '{}' should be created before using it." + ) class Dygraph2StaticException(Exception): - def __init__(self, message): super().__init__(message) @@ -191,13 +204,15 @@ def getfullargspec(target): return inspect.getfullargspec(target) else: argspec = inspect.getargspec(target) - return FullArgSpec(args=argspec.args, - varargs=argspec.varargs, - varkw=argspec.keywords, - defaults=argspec.defaults, - kwonlyargs=[], - kwonlydefaults=None, - annotations={}) + return FullArgSpec( + args=argspec.args, + varargs=argspec.varargs, + varkw=argspec.keywords, + defaults=argspec.defaults, + kwonlyargs=[], + kwonlydefaults=None, + annotations={}, + ) def parse_arg_and_kwargs(function): @@ -214,7 +229,7 @@ def parse_arg_and_kwargs(function): default_values = fullargspec.defaults if default_values: assert len(default_values) <= len(arg_names) - default_kwarg_names = arg_names[-len(default_values):] + default_kwarg_names = arg_names[-len(default_values) :] default_kwargs = dict(zip(default_kwarg_names, default_values)) return arg_names, default_kwargs @@ -288,8 +303,9 @@ def is_api_in_module(node, module_prefix): from paddle.fluid.dygraph import to_variable from paddle import to_tensor - return eval("_is_api_in_module_helper({}, '{}')".format( - func_str, module_prefix)) + return eval( + "_is_api_in_module_helper({}, '{}')".format(func_str, module_prefix) + ) except Exception: return False @@ -320,12 +336,15 @@ def is_numpy_api(node): func_str = astor.to_source(gast.gast_to_ast(node.func)) try: import numpy as np - module_result = eval("_is_api_in_module_helper({}, '{}')".format( - func_str, "numpy")) + + module_result = eval( + "_is_api_in_module_helper({}, '{}')".format(func_str, "numpy") + ) # BUG: np.random.uniform doesn't have module and cannot be analyzed # TODO: find a better way - return module_result or (func_str.startswith("numpy.") - or func_str.startswith("np.")) + return module_result or ( + func_str.startswith("numpy.") or func_str.startswith("np.") + ) except Exception: return False @@ -334,6 +353,7 @@ def _delete_keywords_from(node): assert isinstance(node, gast.Call) func_src = astor.to_source(gast.gast_to_ast(node.func)) import paddle.fluid as fluid + full_args = eval("inspect.getargspec({})".format(func_src)) full_args_name = full_args[0] @@ -347,7 +367,8 @@ def to_static_api(dygraph_class): else: raise NotImplementedError( "Paddle dygraph API {} cannot be converted " - "to static graph at present.".format(dygraph_class)) + "to static graph at present.".format(dygraph_class) + ) def _add_keywords_to(node, dygraph_api_name): @@ -358,8 +379,10 @@ def _add_keywords_to(node, dygraph_api_name): ast_keyword.arg = "size" node.keywords.append( - gast.keyword(arg="num_flatten_dims", - value=gast.Constant(value=-1, kind=None))) + gast.keyword( + arg="num_flatten_dims", value=gast.Constant(value=-1, kind=None) + ) + ) if dygraph_api_name == "BilinearTensorProduct": for ast_keyword in node.keywords: @@ -378,15 +401,17 @@ def to_static_ast(node, class_node): assert isinstance(class_node, gast.Call) static_api = to_static_api(class_node.func.attr) - node.func = gast.Attribute(attr=static_api, - ctx=gast.Load(), - value=gast.Attribute(attr='layers', - ctx=gast.Load(), - value=gast.Name( - ctx=gast.Load(), - id='fluid', - annotation=None, - type_comment=None))) + node.func = gast.Attribute( + attr=static_api, + ctx=gast.Load(), + value=gast.Attribute( + attr='layers', + ctx=gast.Load(), + value=gast.Name( + ctx=gast.Load(), id='fluid', annotation=None, type_comment=None + ), + ), + ) update_args_of_func(node, class_node, 'forward') @@ -409,10 +434,13 @@ def update_args_of_func(node, dygraph_node, method_name): class_src = astor.to_source(gast.gast_to_ast(dygraph_node.func)) import paddle.fluid as fluid + if method_name == "__init__" or eval( - "issubclass({}, fluid.dygraph.Layer)".format(class_src)): - full_args = eval("inspect.getargspec({}.{})".format( - class_src, method_name)) + "issubclass({}, fluid.dygraph.Layer)".format(class_src) + ): + full_args = eval( + "inspect.getargspec({}.{})".format(class_src, method_name) + ) full_args_name = [ arg_name for arg_name in full_args[0] if arg_name != "self" ] @@ -427,21 +455,24 @@ def update_args_of_func(node, dygraph_node, method_name): def create_api_shape_node(tensor_shape_node): - assert isinstance(tensor_shape_node, - (gast.Name, gast.Attribute, gast.Subscript)) + assert isinstance( + tensor_shape_node, (gast.Name, gast.Attribute, gast.Subscript) + ) if isinstance(tensor_shape_node, gast.Name): api_shape_node = gast.Call( func=gast.parse('paddle.shape').body[0].value, args=[tensor_shape_node], - keywords=[]) + keywords=[], + ) return api_shape_node if isinstance(tensor_shape_node, gast.Attribute): api_shape_node = gast.Call( func=gast.parse('paddle.shape').body[0].value, args=[tensor_shape_node.value], - keywords=[]) + keywords=[], + ) return api_shape_node if isinstance(tensor_shape_node, gast.Subscript): @@ -451,14 +482,15 @@ def create_api_shape_node(tensor_shape_node): def get_constant_variable_node(name, value, shape=[1], dtype='int64'): - return gast.parse('%s = paddle.full(%s, "%s", %s)' % - (name, str(shape), str(value), dtype)) + return gast.parse( + '%s = paddle.full(%s, "%s", %s)' % (name, str(shape), str(value), dtype) + ) def get_attribute_full_name(node): assert isinstance( - node, - gast.Attribute), "Input non-Attribute node to get attribute full name" + node, gast.Attribute + ), "Input non-Attribute node to get attribute full name" return astor.to_source(gast.gast_to_ast(node)).strip() @@ -476,15 +508,15 @@ def generate_name_node(name_ids, ctx=gast.Load(), gen_tuple_if_single=False): name_ids = [name_ids] if not isinstance(name_ids, (list, tuple, set)): raise TypeError( - 'name_ids must be list or tuple or set, but received %s' % - type(type(name_ids))) + 'name_ids must be list or tuple or set, but received %s' + % type(type(name_ids)) + ) def create_node_for_name(name): if '.' not in name: - return gast.Name(id=name, - ctx=ctx, - annotation=None, - type_comment=None) + return gast.Name( + id=name, ctx=ctx, annotation=None, type_comment=None + ) return gast.parse(name).body[0].value gast_names = [create_node_for_name(name_id) for name_id in name_ids] @@ -506,12 +538,14 @@ def create_funcDef_node(nodes, name, input_args, return_name_ids): nodes.append(gast.Return(value=generate_name_node(return_name_ids))) else: nodes.append(gast.Return(value=None)) - func_def_node = gast.FunctionDef(name=name, - args=input_args, - body=nodes, - decorator_list=[], - returns=None, - type_comment=None) + func_def_node = gast.FunctionDef( + name=name, + args=input_args, + body=nodes, + decorator_list=[], + returns=None, + type_comment=None, + ) return func_def_node @@ -571,12 +605,14 @@ def func_prefix(func): source = ast_to_source_code(ast_root) source = _inject_import_statements() + source temp_dir = get_temp_dir() - f = tempfile.NamedTemporaryFile(mode='w', - prefix=func_prefix(dyfunc), - suffix='.py', - delete=False, - dir=temp_dir, - encoding='utf-8') + f = tempfile.NamedTemporaryFile( + mode='w', + prefix=func_prefix(dyfunc), + suffix='.py', + delete=False, + dir=temp_dir, + encoding='utf-8', + ) with f: module_name = os.path.basename(f.name[:-3]) f.write(source) @@ -598,8 +634,9 @@ def func_prefix(func): callable_func = getattr(module, func_name) else: raise ValueError( - 'Function: %s doesn\'t exist in the Module transformed from AST.' % - func_name) + 'Function: %s doesn\'t exist in the Module transformed from AST.' + % func_name + ) # After transform dygraph function into callable_func saved in tmp file, # it lost the global variables from imported statements or defined in source file. # Recovers the necessary variables by `__globals__`. @@ -610,10 +647,14 @@ def func_prefix(func): def _inject_import_statements(): import_statements = [ - "import paddle", "from paddle import Tensor", - "import paddle.fluid as fluid", "import paddle.jit.dy2static as _jst", - "from typing import *", "import numpy as np", "import warnings", - "warnings.filterwarnings('ignore', category=DeprecationWarning)" + "import paddle", + "from paddle import Tensor", + "import paddle.fluid as fluid", + "import paddle.jit.dy2static as _jst", + "from typing import *", + "import numpy as np", + "import warnings", + "warnings.filterwarnings('ignore', category=DeprecationWarning)", ] return '\n'.join(import_statements) + '\n' @@ -636,8 +677,10 @@ def func_to_source_code(function, dedent=True): """ if not (inspect.isfunction(function) or inspect.ismethod(function)): raise TypeError( - "The type of 'function' should be a function or method, but received {}." - .format(type(function).__name__)) + "The type of 'function' should be a function or method, but received {}.".format( + type(function).__name__ + ) + ) source_code_list, _ = inspect.getsourcelines(function) # Replace comments with blank lines so that error messages are not misplaced source_code_list = [ @@ -657,8 +700,9 @@ def ast_to_source_code(ast_node): """ if not isinstance(ast_node, (gast.AST, ast.AST)): raise TypeError( - "Type of ast_root should be gast.AST or ast.AST, but received %s." % - type(ast_node)) + "Type of ast_root should be gast.AST or ast.AST, but received %s." + % type(ast_node) + ) if isinstance(ast_node, gast.AST): ast_node = gast.gast_to_ast(ast_node) @@ -674,8 +718,17 @@ def is_candidate_node(node): """ Nodes with specified type will be dependent on tensor. """ - is_compare_node = isinstance(node, (gast.Compare, gast.BoolOp, gast.UnaryOp, - gast.For, gast.If, gast.While)) + is_compare_node = isinstance( + node, + ( + gast.Compare, + gast.BoolOp, + gast.UnaryOp, + gast.For, + gast.If, + gast.While, + ), + ) # TODO(Aurelius84): `.numpy()` may be an customized function, # and should consider a more elegant way to solve this problem. has_numpy_attr = ".numpy()" in ast_to_source_code(node) @@ -691,9 +744,9 @@ def compare_with_none(node): # node.comparators is a list. if isinstance(child, list): child = child[0] - if (isinstance(child, gast.Constant) - and child.value is None) or (isinstance(child, gast.Name) - and child.id == 'None'): + if (isinstance(child, gast.Constant) and child.value is None) or ( + isinstance(child, gast.Name) and child.id == 'None' + ): return True return False @@ -728,20 +781,22 @@ class IsControlFlowVisitor(gast.NodeVisitor): because reshape_op may be called before this statement. """ - def __init__(self, - ast_node, - static_analysis_visitor=None, - node_var_type_map=None): + def __init__( + self, ast_node, static_analysis_visitor=None, node_var_type_map=None + ): assert isinstance( ast_node, gast.AST ), "Type of input node should be gast.AST, but received %s." % type( - ast_node) + ast_node + ) self.ast_root = ast_node if static_analysis_visitor is None: from .static_analysis import StaticAnalysisVisitor + static_analysis_visitor = StaticAnalysisVisitor(ast_node) self.static_analysis_visitor = static_analysis_visitor - self.node_to_wrapper_map = self.static_analysis_visitor.get_node_to_wrapper_map( + self.node_to_wrapper_map = ( + self.static_analysis_visitor.get_node_to_wrapper_map() ) self.node_var_type_map = node_var_type_map @@ -770,7 +825,10 @@ def _visit_For(self, node): if isinstance(node.iter, gast.Call): # for in range(var[0]|var.numpy()[0]) or for in enumerate(var|var.numpy()) if isinstance(node.iter.func, gast.Name): - if node.iter.func.id == "range" or node.iter.func.id == "enumerate": + if ( + node.iter.func.id == "range" + or node.iter.func.id == "enumerate" + ): for arg in node.iter.args: self.visit(arg) else: @@ -869,7 +927,9 @@ def visit_Constant(self, node): return node def _is_node_with_tensor(self, node, name_id): - from paddle.fluid.dygraph.dygraph_to_static.static_analysis import NodeVarType + from paddle.fluid.dygraph.dygraph_to_static.static_analysis import ( + NodeVarType, + ) # Look up the node_var_type_map by name_id. if self.node_var_type_map: @@ -899,7 +959,7 @@ def _is_wrapped(f): return hasattr(f, '__wrapped__') unwrapped_f = func - while (_is_wrapped(unwrapped_f)): + while _is_wrapped(unwrapped_f): unwrapped_f = unwrapped_f.__wrapped__ return unwrapped_f @@ -923,10 +983,12 @@ def input_specs_compatible(src_input_specs, desired_input_specs): if spec not in desired_input_specs: return False else: - for (src_spec, desired_spec) in zip(src_input_specs, - desired_input_specs): + for (src_spec, desired_spec) in zip( + src_input_specs, desired_input_specs + ): if isinstance(src_spec, paddle.static.InputSpec) or isinstance( - desired_spec, paddle.static.InputSpec): + desired_spec, paddle.static.InputSpec + ): if not _compatible_tensor_spec(src_spec, desired_spec): return False else: @@ -986,15 +1048,14 @@ def hash_value(spec): class NameScope: - def __init__(self): """ - A NameScope is a object which manager all the variable names. - only FunctionDef and Controlflow node will have a namescope property. + A NameScope is a object which manager all the variable names. + only FunctionDef and Controlflow node will have a namescope property. - type can be "function" and "controlflow" + type can be "function" and "controlflow" - we don't analyze the read only variable because they don't affect the analysis. + we don't analyze the read only variable because they don't affect the analysis. """ self.globals = set() self.nonlocals = set() @@ -1010,8 +1071,8 @@ def set_father(self, father): self.father = father def existed_vars(self): - """ vars existing in current scope. - they must not contain qualified names. + """vars existing in current scope. + they must not contain qualified names. """ local_vars = self.w_vars - self.globals - self.nonlocals - self.args return set(filter(lambda x: '.' not in x, local_vars)) @@ -1040,18 +1101,20 @@ def func(): f"Find variable `{var}` defined in global scope" f" and call `{var}.append() or {var}.pop()`" f", which will be ignored and never be transfered into" - f" tensor array.") + f" tensor array." + ) else: non_global_push_pop_names.append(var) return set(non_global_push_pop_names) def control_flow_vars(self): valid_names = self.w_vars - tmp = self.father.global_vars & valid_names, + tmp = (self.father.global_vars & valid_names,) return {"global": tmp, "nonlocal": self.w_vars - tmp} def _is_simple_name(self, name): - if '.' in name or '[' in name: return False + if '.' in name or '[' in name: + return False return True def is_global_var(self, name): @@ -1062,11 +1125,14 @@ def is_global_var(self, name): Only valid after FunctionNameLivenessAnalysis visitor. """ assert self._is_simple_name( - name), "is_global_var accept a simple name, but get `{name}`." + name + ), "is_global_var accept a simple name, but get `{name}`." ancestor = self while ancestor is not None: - if name in ancestor.globals: return True - if name in (ancestor.nonlocals | ancestor.w_vars): return False + if name in ancestor.globals: + return True + if name in (ancestor.nonlocals | ancestor.w_vars): + return False ancestor = ancestor.father return True @@ -1082,46 +1148,46 @@ def merge_from(self, name_scope): class FunctionNameLivenessAnalysis(gast.NodeVisitor): - """ analyze the liveness of a function. - - every variables stored in this scope will be collected, - in addition with global/nonlocal information and - push_pop information. - - 1. global variable is stored in node.var_globals. - 2. nonlocal variable is stored in node.var_nonlocals. - 3. arguments is stored in node.var_args. - 4. if a variable's push and pop attribute is called, - it will be collected in push_pop_vars. They are - used for transformation to tensor_array. - NOTE: push_pop_vars **may not** in w_vars. - a.push(0) don't modify the variable a, but the content - of a. - - For example: - - def func(*args, **kargs): - a = 12 - global i,j - nonlocal x,y - print(a) - i = k - b = [] - c = [1,2,3] - for m in range(10): - q = 12 - b.push(1) - c.pop() - - After this visitor we have: - # node is the FunctionDef node with name: "func" - node.pd_scope = NameScope( - globals = ['i', 'j'], - nonlocals = ['x', 'y'], - args = ['args', 'kargs'], - wr_vars = ['a', 'i', 'q', 'm', 'c', 'b'] - push_pop_vars = ['b', 'c'] - ) + """analyze the liveness of a function. + + every variables stored in this scope will be collected, + in addition with global/nonlocal information and + push_pop information. + + 1. global variable is stored in node.var_globals. + 2. nonlocal variable is stored in node.var_nonlocals. + 3. arguments is stored in node.var_args. + 4. if a variable's push and pop attribute is called, + it will be collected in push_pop_vars. They are + used for transformation to tensor_array. + NOTE: push_pop_vars **may not** in w_vars. + a.push(0) don't modify the variable a, but the content + of a. + + For example: + + def func(*args, **kargs): + a = 12 + global i,j + nonlocal x,y + print(a) + i = k + b = [] + c = [1,2,3] + for m in range(10): + q = 12 + b.push(1) + c.pop() + + After this visitor we have: + # node is the FunctionDef node with name: "func" + node.pd_scope = NameScope( + globals = ['i', 'j'], + nonlocals = ['x', 'y'], + args = ['args', 'kargs'], + wr_vars = ['a', 'i', 'q', 'm', 'c', 'b'] + push_pop_vars = ['b', 'c'] + ) """ def __init__(self, root_node): @@ -1141,25 +1207,26 @@ def _current_name_scope(self): return self._get_name_scope(self.scope_node_stack[-1]) def _father_name_scope(self): - if len(self.scope_node_stack) == 1: return None + if len(self.scope_node_stack) == 1: + return None return self._get_name_scope(self.scope_node_stack[-2]) def _nearest_function_scope(self): - if len(self.scope_node_stack) == 1: return None + if len(self.scope_node_stack) == 1: + return None for node in self.scope_node_stack[-2::-1]: if isinstance(node, gast.FunctionDef): return self._get_name_scope(node) def visit_ListComp(self, node): - """ [ i for i in range(10) ] - In this case, `i` will not created in FunctionScope. - We don't collect `i` by not calling generic_visit. + """[ i for i in range(10) ] + In this case, `i` will not created in FunctionScope. + We don't collect `i` by not calling generic_visit. """ pass def visit_DictComp(self, node): - """ the same as ListComp. - """ + """the same as ListComp.""" pass def visit_Name(self, node): @@ -1169,62 +1236,86 @@ def visit_Name(self, node): self._current_name_scope().w_vars.add(node.id) def visit_FunctionDef(self, node): - def pre_func(): self._current_name_scope().args |= set( - self._get_argument_names(node)) + self._get_argument_names(node) + ) def post_func(): - """ NOTE: why we need merge w_vars and push_pop_vars here ? - because we do ifelse_transformer after loop_transformer. Loops will changed into functioons. but we know this function will be called in if. so we add w_vars to father function scope. + """NOTE: why we need merge w_vars and push_pop_vars here ? + because we do ifelse_transformer after loop_transformer. Loops will changed into functioons. but we know this function will be called in if. so we add w_vars to father function scope. """ - from paddle.fluid.dygraph.dygraph_to_static.loop_transformer import WHILE_CONDITION_PREFIX, WHILE_BODY_PREFIX, FOR_CONDITION_PREFIX, FOR_BODY_PREFIX - from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import TRUE_FUNC_PREFIX, FALSE_FUNC_PREFIX + from paddle.fluid.dygraph.dygraph_to_static.loop_transformer import ( + WHILE_CONDITION_PREFIX, + WHILE_BODY_PREFIX, + FOR_CONDITION_PREFIX, + FOR_BODY_PREFIX, + ) + from paddle.fluid.dygraph.dygraph_to_static.ifelse_transformer import ( + TRUE_FUNC_PREFIX, + FALSE_FUNC_PREFIX, + ) + control_flow_function_def = [ - WHILE_BODY_PREFIX, WHILE_BODY_PREFIX, FOR_CONDITION_PREFIX, - FOR_BODY_PREFIX, TRUE_FUNC_PREFIX, FALSE_FUNC_PREFIX + WHILE_BODY_PREFIX, + WHILE_BODY_PREFIX, + FOR_CONDITION_PREFIX, + FOR_BODY_PREFIX, + TRUE_FUNC_PREFIX, + FALSE_FUNC_PREFIX, ] def is_control_flow_def_node(): for prefix in control_flow_function_def: - if node.name.startswith(prefix): return True + if node.name.startswith(prefix): + return True return False if self._father_name_scope() and is_control_flow_def_node(): - self._father_name_scope().w_vars |= self._current_name_scope( - ).w_vars - self._father_name_scope( - ).push_pop_vars |= self._current_name_scope().push_pop_vars + self._father_name_scope().w_vars |= ( + self._current_name_scope().w_vars + ) + self._father_name_scope().push_pop_vars |= ( + self._current_name_scope().push_pop_vars + ) self._visit_scope_node(node, pre_func, post_func) def _visit_scope_node(self, node, pre_func, post_func): - """ scope node main visit logic. - pre_func and post_func is callbacks + """scope node main visit logic. + pre_func and post_func is callbacks """ self._reset_name_scope(node) self.scope_node_stack.append(node) self._current_name_scope().set_father(self._nearest_function_scope()) - if pre_func: pre_func() + if pre_func: + pre_func() self.generic_visit(node) - if post_func: post_func() + if post_func: + post_func() self.scope_node_stack.pop() def _visit_controlflow_node(self, node): - def post_func(): self._father_name_scope().merge_from(self._current_name_scope()) self._nearest_function_scope().merge_from( - self._current_name_scope()) - self._current_name_scope().created = self._nearest_function_scope( - ).existed_vars() - node.before_created + self._current_name_scope() + ) + self._current_name_scope().created = ( + self._nearest_function_scope().existed_vars() + - node.before_created + ) # gather created vars into father and used in CreateUndefinedVarTransform - self._nearest_function_scope().created |= self._current_name_scope( - ).created + self._nearest_function_scope().created |= ( + self._current_name_scope().created + ) def pre_func(): - setattr(node, "before_created", - self._nearest_function_scope().existed_vars()) + setattr( + node, + "before_created", + self._nearest_function_scope().existed_vars(), + ) self._visit_scope_node(node, pre_func, post_func) @@ -1262,12 +1353,13 @@ def visit_Call(self, node): self._current_name_scope().push_pop_vars.add(name) def _get_argument_names(self, node): - """ get all arguments name in the functiondef node. - this node is local to the function and shouldn't - be created. + """get all arguments name in the functiondef node. + this node is local to the function and shouldn't + be created. """ assert isinstance( - node, gast.FunctionDef), "Input node is not function define node" + node, gast.FunctionDef + ), "Input node is not function define node" names = [a for a in node.args.args] names.append(node.args.vararg) names.append(node.args.kwarg) @@ -1288,7 +1380,9 @@ def empty_node(): func_def = """ def {func_name}(): return - """.format(func_name=unique_name.generate(GET_ARGS_FUNC_PREFIX)) + """.format( + func_name=unique_name.generate(GET_ARGS_FUNC_PREFIX) + ) return gast.parse(textwrap.dedent(func_def)).body[0] assert isinstance(names, (list, tuple)) @@ -1307,7 +1401,8 @@ def {func_name}(): func_def = template.format( func_name=unique_name.generate(GET_ARGS_FUNC_PREFIX), nonlocal_vars=nonlocal_vars, - vars=",".join(names)) + vars=",".join(names), + ) return gast.parse(textwrap.dedent(func_def)).body[0] @@ -1324,8 +1419,9 @@ def empty_node(): func_def = """ def {func_name}({args}): pass - """.format(func_name=unique_name.generate(SET_ARGS_FUNC_PREFIX), - args=ARGS_NAME) + """.format( + func_name=unique_name.generate(SET_ARGS_FUNC_PREFIX), args=ARGS_NAME + ) return gast.parse(textwrap.dedent(func_def)).body[0] assert isinstance(names, (list, tuple)) @@ -1345,7 +1441,8 @@ def {func_name}({args}): func_name=unique_name.generate(SET_ARGS_FUNC_PREFIX), args=ARGS_NAME, nonlocal_vars=nonlocal_vars, - vars=",".join(names)) + vars=",".join(names), + ) return gast.parse(textwrap.dedent(func_def)).body[0] @@ -1355,8 +1452,8 @@ def create_nonlocal_stmt_nodes(names): mapped = list(filter(lambda n: '.' not in n, names)) mapped = list(filter(lambda n: '[' not in n, mapped)) names = sorted( - mapped, - key=mapped.index) # to keep the order, we can't use set() to unique + mapped, key=mapped.index + ) # to keep the order, we can't use set() to unique if not names: return [] func_code = "nonlocal {}".format(','.join(names)) @@ -1364,10 +1461,10 @@ def create_nonlocal_stmt_nodes(names): class GetterSetterHelper: - """ we have two classes of names in setter and getter function: - w_vars(loop_vars) + push_pop_vars - To simplify the setter logic in convert_while and convert_cond, - we extract the helper class here. + """we have two classes of names in setter and getter function: + w_vars(loop_vars) + push_pop_vars + To simplify the setter logic in convert_while and convert_cond, + we extract the helper class here. """ def __init__(self, getter_func, setter_func, *name_lists): @@ -1383,22 +1480,33 @@ def union(self): return self._union def get(self, names): - if names is None: names = [] + if names is None: + names = [] vars = self.getter() - if vars is None: return tuple() + if vars is None: + return tuple() for n in names: - assert n in self.name2id, "the name `{}` not in name union set`{}`.".format( - n, self.name2id.keys()) + assert ( + n in self.name2id + ), "the name `{}` not in name union set`{}`.".format( + n, self.name2id.keys() + ) return tuple(map(lambda n: vars[self.name2id[n]], names)) def set(self, names, values): - if names is None: names = [] - if values is None: values = [] + if names is None: + names = [] + if values is None: + values = [] vars = self.getter() - if vars is None: return + if vars is None: + return for n in names: - assert n in self.name2id, "the name `{}` not in name union set`{}`.".format( - n, self.name2id.keys()) + assert ( + n in self.name2id + ), "the name `{}` not in name union set`{}`.".format( + n, self.name2id.keys() + ) vars = list(vars) indices = list(map(lambda n: self.name2id[n], names)) for i, v in zip(indices, values): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py b/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py index 8afe558ecfc2c..f28f1993621cc 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/variable_trans_func.py @@ -17,7 +17,10 @@ from paddle.utils import gast from paddle.fluid import unique_name from paddle.fluid.framework import Variable -from paddle.fluid.dygraph.dygraph_to_static.utils import UndefinedVar, create_undefined_variable +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + UndefinedVar, + create_undefined_variable, +) from paddle.fluid.layers.utils import map_structure, is_sequence __all__ = [ @@ -37,16 +40,19 @@ def create_fill_constant_node(name, value=0): func_code = "{} = paddle.full(shape=[1], ".format(name) if isinstance(value, bool): func_code += "dtype='bool', fill_value={}, name='{}')".format( - value, name) + value, name + ) return gast.parse(func_code).body[0] if isinstance(value, float): func_code += "dtype='float64', fill_value={}, name='{}')".format( - value, name) + value, name + ) return gast.parse(func_code).body[0] if isinstance(value, int): func_code += "dtype='int64', fill_value={}, name='{}')".format( - value, name) + value, name + ) return gast.parse(func_code).body[0] diff --git a/python/paddle/fluid/dygraph/inplace_utils.py b/python/paddle/fluid/dygraph/inplace_utils.py index 968a957b660d3..fb27a5674b7d8 100644 --- a/python/paddle/fluid/dygraph/inplace_utils.py +++ b/python/paddle/fluid/dygraph/inplace_utils.py @@ -23,13 +23,14 @@ # in dygraph mode. If static mode is used, the inplace mechanism will not be used, and the static method # of the original API will be called. def _inplace_apis_in_dygraph_only_(func): - def __impl__(*args, **kwargs): if not _non_static_mode(): origin_api_name = func.__name__[:-1] warnings.warn( - "In static mode, {}() is the same as {}() and does not perform inplace operation." - .format(func.__name__, origin_api_name)) + "In static mode, {}() is the same as {}() and does not perform inplace operation.".format( + func.__name__, origin_api_name + ) + ) origin_func = "{}.{}".format(func.__module__, origin_api_name) return eval(origin_func)(*args, **kwargs) return func(*args, **kwargs) diff --git a/python/paddle/fluid/dygraph/io.py b/python/paddle/fluid/dygraph/io.py index 9ddb46798cfb1..d2e7ada85a756 100644 --- a/python/paddle/fluid/dygraph/io.py +++ b/python/paddle/fluid/dygraph/io.py @@ -27,8 +27,14 @@ from paddle.fluid.layers.utils import _hash_with_id from paddle.fluid.dygraph.base import switch_to_static_graph from paddle.fluid.framework import _non_static_mode -from paddle.fluid.executor import _is_enable_standalone_executor, _is_dy2st_enable_standalone_executor -from paddle.fluid.dygraph.dygraph_to_static.partial_program import add_build_strategy_for, LazyInitialized +from paddle.fluid.executor import ( + _is_enable_standalone_executor, + _is_dy2st_enable_standalone_executor, +) +from paddle.fluid.dygraph.dygraph_to_static.partial_program import ( + add_build_strategy_for, + LazyInitialized, +) from paddle import _C_ops, _legacy_C_ops __all__ = ['TranslatedLayer'] @@ -50,17 +56,20 @@ def _load_program_desc(model_file_path): program_desc = core.ProgramDesc(program_desc_str) if not core._is_program_version_supported(program_desc._version()): - raise ValueError("Unsupported program version: %d\n" % - program_desc._version()) + raise ValueError( + "Unsupported program version: %d\n" % program_desc._version() + ) return program_desc def _is_persistable(var_desc): - if var_desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var_desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var_desc.type() == core.VarDesc.VarType.READER or \ - var_desc.type() == core.VarDesc.VarType.RAW: + if ( + var_desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var_desc.type() == core.VarDesc.VarType.FETCH_LIST + or var_desc.type() == core.VarDesc.VarType.READER + or var_desc.type() == core.VarDesc.VarType.RAW + ): return False return var_desc.persistable() @@ -204,9 +213,11 @@ def _rename_var_program_desc(program_desc, include=None, exclude=None): name_old = var.name() is_double_grad_var = "@GRAD" in name_old has_double_grad = has_double_grad or is_double_grad_var - should_rename = (include is None or name_old in include) and ( - exclude is None - or name_old not in exclude) and not is_double_grad_var + should_rename = ( + (include is None or name_old in include) + and (exclude is None or name_old not in exclude) + and not is_double_grad_var + ) if should_rename: temp_name = name_old.split('_') if len(temp_name) > 1 and temp_name[-1].isnumeric(): @@ -215,9 +226,12 @@ def _rename_var_program_desc(program_desc, include=None, exclude=None): temp_name = name_old while True: name_new = _generate_unique_var_name_sync_with_main_program( - temp_name) - if name_new not in old_names[:var_idx] + old_names[var_idx + - 1:]: + temp_name + ) + if ( + name_new + not in old_names[:var_idx] + old_names[var_idx + 1 :] + ): break else: name_new = name_old @@ -237,13 +251,16 @@ def _rename_var_program_desc(program_desc, include=None, exclude=None): var_name = var.name() if "@GRAD" in var_name and name_old in var_name: new_var_name = var_name.replace( - name_old, dict_rename_var_old_new[name_old]) + name_old, dict_rename_var_old_new[name_old] + ) double_grad_rename_dict[var_name] = new_var_name for var_name in double_grad_rename_dict: dict_rename_var_old_new[var_name] = double_grad_rename_dict[ - var_name] + var_name + ] dict_rename_var_new_old[ - double_grad_rename_dict[var_name]] = var_name + double_grad_rename_dict[var_name] + ] = var_name # Rename on program desc for b_idx in range(program_desc.num_blocks()): @@ -252,27 +269,38 @@ def _rename_var_program_desc(program_desc, include=None, exclude=None): op = cur_block.op(op_idx) for input_arg_name in op.input_arg_names(): if input_arg_name in dict_rename_var_old_new: - if input_arg_name != dict_rename_var_old_new[input_arg_name]: + if ( + input_arg_name + != dict_rename_var_old_new[input_arg_name] + ): op._rename_input( input_arg_name, - dict_rename_var_old_new[input_arg_name]) + dict_rename_var_old_new[input_arg_name], + ) if cur_block.has_var(input_arg_name.encode()): cur_block._rename_var( input_arg_name.encode(), - dict_rename_var_old_new[input_arg_name].encode( - )) + dict_rename_var_old_new[ + input_arg_name + ].encode(), + ) for output_arg_name in op.output_arg_names(): if output_arg_name in dict_rename_var_old_new: - if output_arg_name != dict_rename_var_old_new[ - output_arg_name]: + if ( + output_arg_name + != dict_rename_var_old_new[output_arg_name] + ): op._rename_output( output_arg_name, - dict_rename_var_old_new[output_arg_name]) + dict_rename_var_old_new[output_arg_name], + ) if cur_block.has_var(output_arg_name.encode()): cur_block._rename_var( output_arg_name.encode(), - dict_rename_var_old_new[output_arg_name].encode( - )) + dict_rename_var_old_new[ + output_arg_name + ].encode(), + ) program_desc.flush() return dict_rename_var_new_old, dict_rename_var_old_new @@ -327,7 +355,8 @@ def __init__(self, program_desc): self._infer_program_desc = self._preprocess(program_desc) # forward + backward program self._train_program_desc = self._append_backward_desc( - self._infer_program_desc) + self._infer_program_desc + ) # forward: @switch_to_static_graph @@ -348,11 +377,13 @@ def _forward_program_desc(self): def _create_backward_train_program(self): whole_program = _build_program_by_desc(self._train_program_desc) start_op_index = self._infer_program_desc.block(0).op_size() + 2 * len( - self._output_descs) + self._output_descs + ) end_op_index = whole_program.desc.block(0).op_size() - if (start_op_index < end_op_index): - return add_build_strategy_for(whole_program, start_op_index, - end_op_index) + if start_op_index < end_op_index: + return add_build_strategy_for( + whole_program, start_op_index, end_op_index + ) else: return paddle.static.Program() @@ -400,7 +431,8 @@ def _preprocess(self, program_desc): # rename persistable variables of 'program_desc' list_persistable_var = _get_persistable_var_names(program_desc) rename_new_old_dict, _ = _rename_var_program_desc( - program_desc, list_persistable_var) + program_desc, list_persistable_var + ) # 1. Prune original program # remove feed, fetch and scale-1 op, remove op_callstack attr ops_to_remove = [] @@ -412,14 +444,17 @@ def _preprocess(self, program_desc): feed_var_name = op.input('X')[0].encode() root_block._remove_var(feed_var_name) self._input_descs.append( - root_block.find_var(op.output('Out')[0].encode())) + root_block.find_var(op.output('Out')[0].encode()) + ) elif op.type() == 'scale' and op.output('Out')[0].startswith( - 'save_infer_model/scale_'): + 'save_infer_model/scale_' + ): ops_to_remove.append(i) out_var_name = op.output('Out')[0].encode() root_block._remove_var(out_var_name) self._output_descs.append( - root_block.find_var(op.input('X')[0].encode())) + root_block.find_var(op.input('X')[0].encode()) + ) elif op.type() == 'fetch': ops_to_remove.append(i) fetch_var_name = op.output('Out')[0].encode() @@ -427,7 +462,8 @@ def _preprocess(self, program_desc): # NOTE: some old pre-train models have no extra scale_op if not op.input('X')[0].startswith('save_infer_model/scale_'): self._output_descs.append( - root_block.find_var(op.input('X')[0].encode())) + root_block.find_var(op.input('X')[0].encode()) + ) else: if op.has_attr("op_callstack"): op.remove_attr("op_callstack") @@ -472,7 +508,8 @@ def _preprocess(self, program_desc): # there will be a problem of duplicate names, so here is unified # to add the LOADED suffix to the parameters of the model loaded self._suffix_varname_dict = _get_loaded_var_new_old( - program_desc, rename_new_old_dict) + program_desc, rename_new_old_dict + ) # - get persistable var self._persistable_names = _get_persistable_var_names(program_desc) @@ -486,9 +523,9 @@ def _append_scale_to_output(self, program): with framework.program_guard(program): for i, out in enumerate(self._output_descs): var = program.global_block().var(out.name()) - var = nn.scale(var, - 1., - name="translated_layer/scale_{}".format(i)) + var = nn.scale( + var, 1.0, name="translated_layer/scale_{}".format(i) + ) scale_output_vars.append(var) # 2. update output names & descs for i, var in enumerate(scale_output_vars): @@ -513,15 +550,19 @@ def _get_train_forward_program(self, infer_program_desc): block = program.block(block_idx) for op in block.ops: if op.type == "batch_norm": - if "ReserveSpace" not in op.output_names or len( - op.output("ReserveSpace")) == 0: + if ( + "ReserveSpace" not in op.output_names + or len(op.output("ReserveSpace")) == 0 + ): reserve_space = block.create_var( name=unique_name.generate_with_ignorable_key( - ".".join(["reserve_space", 'tmp'])), + ".".join(["reserve_space", 'tmp']) + ), dtype=block.var(op.input("X")[0]).dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) op.desc.set_output("ReserveSpace", [reserve_space.name]) return program @@ -567,9 +608,9 @@ def _append_backward_desc(self, infer_program_desc): # NOTE: [compatible] deal with model saved by save_inference_model, # which need get var info from program desc -def _load_persistable_vars_by_program(model_path, - program_holder, - params_filename=None): +def _load_persistable_vars_by_program( + model_path, program_holder, params_filename=None +): # make sure the path has been checked persistable_vars = _get_persistable_vars(program_holder.infer_program) load_var_dict = {} @@ -578,37 +619,43 @@ def _load_persistable_vars_by_program(model_path, if _is_parameter(each_var, program_holder.infer_program): # create output varbase if framework._in_eager_without_dygraph_check(): - new_var = framework.EagerParamBase(shape=each_var.shape(), - dtype=each_var.dtype(), - name=each_var.name(), - type=each_var.type(), - persistable=True) + new_var = framework.EagerParamBase( + shape=each_var.shape(), + dtype=each_var.dtype(), + name=each_var.name(), + type=each_var.type(), + persistable=True, + ) else: - new_var = framework.ParamBase(shape=each_var.shape(), - dtype=each_var.dtype(), - name=each_var.name(), - type=each_var.type(), - persistable=True) + new_var = framework.ParamBase( + shape=each_var.shape(), + dtype=each_var.dtype(), + name=each_var.name(), + type=each_var.type(), + persistable=True, + ) else: - new_var = framework._varbase_creator(type=each_var.type(), - name=each_var.name(), - shape=each_var.shape(), - dtype=each_var.dtype(), - persistable=True) + new_var = framework._varbase_creator( + type=each_var.type(), + name=each_var.name(), + shape=each_var.shape(), + dtype=each_var.dtype(), + persistable=True, + ) if params_filename is None: framework._dygraph_tracer().trace_op( type='load', inputs={}, outputs={'Out': new_var}, - attrs={'file_path': os.path.join(model_path, orig_each_name)}) + attrs={'file_path': os.path.join(model_path, orig_each_name)}, + ) new_var.stop_gradient = False load_var_dict[each_var.name()] = new_var if params_filename is not None: load_var_list = [] dict_name_old_new = { - v: k - for k, v in program_holder._suffix_varname_dict.items() + v: k for k, v in program_holder._suffix_varname_dict.items() } for name in sorted(dict_name_old_new.keys()): load_var_list.append(load_var_dict[dict_name_old_new[name]]) @@ -617,7 +664,8 @@ def _load_persistable_vars_by_program(model_path, type='load_combine', inputs={}, outputs={'Out': load_var_list}, - attrs={'file_path': os.path.join(model_path, params_filename)}) + attrs={'file_path': os.path.join(model_path, params_filename)}, + ) for each_var in persistable_vars: if not _is_parameter(each_var, program_holder.infer_program): @@ -639,8 +687,9 @@ def _load_persistable_vars_by_program(model_path, return load_var_dict -def _load_persistable_vars(model_path, var_info_path, program_holder, - params_filename): +def _load_persistable_vars( + model_path, var_info_path, program_holder, params_filename +): # 1. load extra var info with open(var_info_path, 'rb') as f: extra_var_info = pickle.load(f) @@ -649,8 +698,7 @@ def _load_persistable_vars(model_path, var_info_path, program_holder, load_var_dict = dict() load_var_list = [] inv_suffix_varname_dict = { - value: key - for key, value in program_holder._suffix_varname_dict.items() + value: key for key, value in program_holder._suffix_varname_dict.items() } # NOTE(chenweihang): we need load persistable vars based the program, @@ -661,7 +709,8 @@ def _load_persistable_vars(model_path, var_info_path, program_holder, raise RuntimeError( "The model to be loaded is not complete." "The variable `%s` of program cannot be found in loaded model.", - name) + name, + ) # get suffix var name, see [why need to append suffix to persistable vars] new_name = inv_suffix_varname_dict[name] # create output varbase @@ -674,7 +723,8 @@ def _load_persistable_vars(model_path, var_info_path, program_holder, ], # only to pass check, this shape is not meaningful dtype=core.VarDesc.VarType.FP32, name=new_name, - persistable=True) + persistable=True, + ) else: new_var = framework.ParamBase( shape=[ @@ -682,10 +732,12 @@ def _load_persistable_vars(model_path, var_info_path, program_holder, ], # only to pass check, this shape is not meaningful dtype=core.VarDesc.VarType.FP32, name=new_name, - persistable=True) + persistable=True, + ) else: - new_var = framework._varbase_creator(name=new_name, - persistable=True) + new_var = framework._varbase_creator( + name=new_name, persistable=True + ) new_var.stop_gradient = extra_var_info[name]['stop_gradient'] load_var_dict[new_name] = new_var @@ -698,10 +750,12 @@ def _load_persistable_vars(model_path, var_info_path, program_holder, if len(extra_var_info) != 0: raise ValueError("The model to be loaded is incomplete.") else: - framework._dygraph_tracer().trace_op(type='load_combine', - inputs={}, - outputs={'Out': load_var_list}, - attrs={'file_path': var_file_path}) + framework._dygraph_tracer().trace_op( + type='load_combine', + inputs={}, + outputs={'Out': load_var_list}, + attrs={'file_path': var_file_path}, + ) return load_var_dict @@ -723,17 +777,18 @@ def _construct_program_holders(model_path, model_filename=None): # [compatible] if assign model_filename, only can load one program as Layer.forward model_filename = os.path.basename(model_filename) model_file_path = os.path.join(model_path, model_filename) - model_name = model_filename[:-len(INFER_MODEL_SUFFIX)] - #Load every file that meets the requirements in the directory model_path. + model_name = model_filename[: -len(INFER_MODEL_SUFFIX)] + # Load every file that meets the requirements in the directory model_path. for filename in os.listdir(model_path): if model_filename == filename: func_name = 'forward' model_file_path = os.path.join(model_path, model_filename) elif filename.endswith(INFER_MODEL_SUFFIX) and filename.startswith( - model_name): - parsing_names = filename[len(model_name - ):-len(INFER_MODEL_SUFFIX) + - 1].split('.') + model_name + ): + parsing_names = filename[ + len(model_name) : -len(INFER_MODEL_SUFFIX) + 1 + ].split('.') if len(parsing_names) == 3 and len(parsing_names[1]) > 0: func_name = parsing_names[1] model_file_path = os.path.join(model_path, filename) @@ -742,7 +797,8 @@ def _construct_program_holders(model_path, model_filename=None): else: continue program_holder_dict[func_name] = _ProgramHolder( - _load_program_desc(model_file_path)) + _load_program_desc(model_file_path) + ) else: for _, _, file_names in os.walk(model_path): for name in file_names: @@ -754,30 +810,32 @@ def _construct_program_holders(model_path, model_filename=None): else: method_name.replace('model', '') program_holder_dict[method_name] = _ProgramHolder( - _load_program_desc(model_file_path)) + _load_program_desc(model_file_path) + ) return program_holder_dict -def _construct_params_and_buffers(model_path, - programs, - params_filename=None, - append_suffix=True): +def _construct_params_and_buffers( + model_path, programs, params_filename=None, append_suffix=True +): var_info_filename = str(params_filename) + ".info" var_info_path = os.path.join(model_path, var_info_filename) params_path = os.path.join(model_path, str(params_filename)) if os.path.exists(var_info_path): - var_dict = _load_persistable_vars(model_path, var_info_path, - programs['forward'], params_filename) - model_name = params_filename[:-len(INFER_PARAMS_SUFFIX)] - #Load every file that meets the requirements in the directory model_path. + var_dict = _load_persistable_vars( + model_path, var_info_path, programs['forward'], params_filename + ) + model_name = params_filename[: -len(INFER_PARAMS_SUFFIX)] + # Load every file that meets the requirements in the directory model_path. for file_name in os.listdir(model_path): if file_name.startswith(model_name) and file_name.endswith( - INFER_PARAMS_SUFFIX): - parsing_names = file_name[len(model_name - ):-len(INFER_PARAMS_SUFFIX) + - 1].split('.') + INFER_PARAMS_SUFFIX + ): + parsing_names = file_name[ + len(model_name) : -len(INFER_PARAMS_SUFFIX) + 1 + ].split('.') if len(parsing_names) == 3 and len(parsing_names[1]) > 0: func_name = parsing_names[1] else: @@ -786,15 +844,17 @@ def _construct_params_and_buffers(model_path, continue var_info_path = os.path.join(model_path, var_info_filename) var_dict.update( - _load_persistable_vars(model_path, var_info_path, - programs[func_name], file_name)) + _load_persistable_vars( + model_path, var_info_path, programs[func_name], file_name + ) + ) elif params_filename is not None and not os.path.exists(params_path): # When saving XX, there is only '*.pdmodel' return dict() else: - var_dict = _load_persistable_vars_by_program(model_path, - programs['forward'], - params_filename) + var_dict = _load_persistable_vars_by_program( + model_path, programs['forward'], params_filename + ) if not append_suffix: var_dict = _remove_varname_suffix(var_dict, programs['forward']) @@ -807,13 +867,23 @@ def _valid_vars(vars): return vars if framework._in_eager_without_dygraph_check(): return [ - core.eager.Tensor(core.VarDesc.VarType.FP32, [], "Fake_var", - core.VarDesc.VarType.RAW, False) + core.eager.Tensor( + core.VarDesc.VarType.FP32, + [], + "Fake_var", + core.VarDesc.VarType.RAW, + False, + ) ] else: return [ - core.VarBase(core.VarDesc.VarType.FP32, [], "Fake_var", - core.VarDesc.VarType.RAW, False) + core.VarBase( + core.VarDesc.VarType.FP32, + [], + "Fake_var", + core.VarDesc.VarType.RAW, + False, + ) ] @@ -825,7 +895,8 @@ def _run_dygraph(instance, input, program_holder): if not isinstance(value, (np.ndarray, core.VarBase, core.eager.Tensor)): raise TypeError( "The type of input in TranslatedLayer must be numpy array or Variable(VarBase), but received %s." - % type(value)) + % type(value) + ) # NOTE: In order to unify the API, firstly convert the input to VarBase if isinstance(value, np.ndarray): if framework._in_eager_without_dygraph_check(): @@ -834,13 +905,16 @@ def _run_dygraph(instance, input, program_holder): name=program_holder.input_descs[i].name(), persistable=False, place=framework._current_expected_place(), - zero_copy=True) + zero_copy=True, + ) else: - var = core.VarBase(value=value, - name=program_holder.input_descs[i].name(), - persistable=False, - place=framework._current_expected_place(), - zero_copy=True) + var = core.VarBase( + value=value, + name=program_holder.input_descs[i].name(), + persistable=False, + place=framework._current_expected_place(), + zero_copy=True, + ) else: var = value # NOTE: we changed var name here, @@ -862,67 +936,112 @@ def _run_dygraph(instance, input, program_holder): else: raise ValueError( "The persistable variable %s does not exist in current TranslatedLayer." - % var_name) + % var_name + ) output_vars = [] for var_desc in program_holder.output_descs: if framework._in_eager_without_dygraph_check(): - var = core.eager.Tensor(dtype=var_desc.dtype(), - dims=var_desc.shape(), - name=var_desc.name(), - type=var_desc.type(), - persistable=False) + var = core.eager.Tensor( + dtype=var_desc.dtype(), + dims=var_desc.shape(), + name=var_desc.name(), + type=var_desc.type(), + persistable=False, + ) else: - var = core.VarBase(var_desc.dtype(), var_desc.shape(), - var_desc.name(), var_desc.type(), False) + var = core.VarBase( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) output_vars.append(var) # hold forward variables if framework._in_eager_without_dygraph_check(): tmp_scope_vec = [program_holder.scope] else: - tmp_scope_vec = core.VarBase(core.VarDesc.VarType.FP32, [], - "program_out_scope", - core.VarDesc.VarType.STEP_SCOPES, True) + tmp_scope_vec = core.VarBase( + core.VarDesc.VarType.FP32, + [], + "program_out_scope", + core.VarDesc.VarType.STEP_SCOPES, + True, + ) tmp_scope_vec.value().set_scope(program_holder.scope) double_grad_vars = [] for var_desc in program_holder.double_grad_descs: if framework._in_eager_without_dygraph_check(): - var = core.eager.Tensor(dtype=var_desc.dtype(), - dims=var_desc.shape(), - name=var_desc.name(), - type=var_desc.type(), - persistable=False) + var = core.eager.Tensor( + dtype=var_desc.dtype(), + dims=var_desc.shape(), + name=var_desc.name(), + type=var_desc.type(), + persistable=False, + ) else: - var = core.VarBase(var_desc.dtype(), var_desc.shape(), - var_desc.name(), var_desc.type(), False) + var = core.VarBase( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) double_grad_vars.append(var) # 2. run program by op - trace_program = program_holder.infer_program if instance._is_test else program_holder.train_program - forward_program = program_holder._infer_program_desc if instance._is_test else program_holder.forward_program + trace_program = ( + program_holder.infer_program + if instance._is_test + else program_holder.train_program + ) + forward_program = ( + program_holder._infer_program_desc + if instance._is_test + else program_holder.forward_program + ) end_op_index = program_holder.infer_program.block(0).op_size() attrs = [ 'global_block', - trace_program.block(0), 'start_op_index', 0, 'end_op_index', - end_op_index, 'is_test', instance._is_test, 'program_id', - _hash_with_id(trace_program, instance) + trace_program.block(0), + 'start_op_index', + 0, + 'end_op_index', + end_op_index, + 'is_test', + instance._is_test, + 'program_id', + _hash_with_id(trace_program, instance), ] - use_interpretorcore = _is_enable_standalone_executor( - ) and _is_dy2st_enable_standalone_executor() + use_interpretorcore = ( + _is_enable_standalone_executor() + and _is_dy2st_enable_standalone_executor() + ) attrs.extend(('use_interpretorcore', use_interpretorcore)) if use_interpretorcore: attrs.extend( - ('forward_global_block', forward_program.block(0), - 'backward_global_block', program_holder.backward_program.block(0))) - - _legacy_C_ops.run_program(_valid_vars(input_vars), - _valid_vars(persistable_vars), - _valid_vars(output_vars), tmp_scope_vec, - _valid_vars(double_grad_vars), None, *attrs) + ( + 'forward_global_block', + forward_program.block(0), + 'backward_global_block', + program_holder.backward_program.block(0), + ) + ) + + _legacy_C_ops.run_program( + _valid_vars(input_vars), + _valid_vars(persistable_vars), + _valid_vars(output_vars), + tmp_scope_vec, + _valid_vars(double_grad_vars), + None, + *attrs + ) # NOTE: [ why need set param's gradient type here ] # if user set sparse gradient mode, the param's gradient @@ -951,8 +1070,11 @@ def _run_dygraph(instance, input, program_holder): def drop_scope_if_no_grad(instance, scope_vec): tracer = framework._dygraph_tracer() - scope = scope_vec.value().get_scope() if isinstance( - scope_vec, (core.VarBase)) else scope_vec[0] + scope = ( + scope_vec.value().get_scope() + if isinstance(scope_vec, (core.VarBase)) + else scope_vec[0] + ) if (not instance._is_test) and (not tracer._has_grad): scope.drop_kids() @@ -961,15 +1083,22 @@ def _run_static_graph(input, program_holder, trace_program): main_program = framework.default_main_program() param_var_names = _get_persistable_var_names(trace_program) _, dict_rename_var_old_new = _rename_var_program_desc( - trace_program, exclude=param_var_names) + trace_program, exclude=param_var_names + ) trace_program.flush() output_names = [var.name() for var in program_holder.output_descs] # append blocks from 'trace_program' - _append_block(main_program, trace_program, program_holder, input, - dict_rename_var_old_new) + _append_block( + main_program, + trace_program, + program_holder, + input, + dict_rename_var_old_new, + ) main_program._sync_with_cpp() - outs = _get_output_from_program(main_program, program_holder, - dict_rename_var_old_new) + outs = _get_output_from_program( + main_program, program_holder, dict_rename_var_old_new + ) if len(outs) == 1: outs = outs[0] return outs @@ -997,11 +1126,13 @@ def _collect_current_and_parent_var(program, block_idx): return vars -def _append_block(dest_program, - src_program_desc, - program_holder, - input_variables, - dict_rename_var_old_new=None): +def _append_block( + dest_program, + src_program_desc, + program_holder, + input_variables, + dict_rename_var_old_new=None, +): ''' Append Variables and Operators in 'src_program_desc' to dest_program. @@ -1015,28 +1146,35 @@ def _append_block(dest_program, ''' origin_block_idx = dest_program.current_block_idx - param_var_names = _collect_current_and_parent_var(dest_program, - origin_block_idx) - append_var_from_block_desc_static(dest_program.block(origin_block_idx), - src_program_desc.block(0), - exclude=param_var_names) + param_var_names = _collect_current_and_parent_var( + dest_program, origin_block_idx + ) + append_var_from_block_desc_static( + dest_program.block(origin_block_idx), + src_program_desc.block(0), + exclude=param_var_names, + ) name_inp_desc = [inp.name() for inp in program_holder.input_descs] input_names = [inp.name for inp in input_variables] if len(name_inp_desc) != len(input_names): raise ValueError( - "The number of input is invalid, expected {}, but received {}.". - format(len(name_inp_desc), len(input_names))) + "The number of input is invalid, expected {}, but received {}.".format( + len(name_inp_desc), len(input_names) + ) + ) for i, out_name in enumerate(name_inp_desc): if dict_rename_var_old_new: out_name = dict_rename_var_old_new[out_name] dest_program.block(origin_block_idx).append_op( type='assign', inputs={'X': [input_names[i]]}, - outputs={'Out': [out_name]}) + outputs={'Out': [out_name]}, + ) append_ops = append_op_from_block_desc_static( - dest_program.block(origin_block_idx), src_program_desc.block(0)) + dest_program.block(origin_block_idx), src_program_desc.block(0) + ) dest_program._sync_with_cpp() offset_block_idx = dest_program.num_blocks - 1 @@ -1050,11 +1188,12 @@ def _append_block(dest_program, else: parent_idx = origin_block_idx dest_block = dest_program._create_block(parent_idx=parent_idx) - append_var_from_block_desc_static(dest_block, - src_block, - exclude=param_var_names) + append_var_from_block_desc_static( + dest_block, src_block, exclude=param_var_names + ) append_ops += append_op_from_block_desc_static( - dest_block, src_block) + dest_block, src_block + ) dest_program._sync_with_cpp() for op in append_ops: @@ -1064,15 +1203,16 @@ def _append_block(dest_program, origin_id = sub.id if isinstance(sub, framework.Block): origin_id = sub.idx - op._set_attr('sub_block', - dest_program.block(offset_block_idx + origin_id)) + op._set_attr( + 'sub_block', dest_program.block(offset_block_idx + origin_id) + ) dest_program._sync_with_cpp() dest_program.current_block_idx = origin_block_idx -def _get_output_from_program(program, - program_holder, - dict_rename_var_old_new=None): +def _get_output_from_program( + program, program_holder, dict_rename_var_old_new=None +): """ Get output name of 'program' according to program_holder """ @@ -1121,20 +1261,21 @@ def append_op_from_desc_static(block, op_desc): op_type = op_desc.type() op_append = block.desc.append_op() op_append.copy_from(op_desc) - op = framework.Operator(block=block, - desc=op_append, - type=op_type, - inputs=None, - outputs=None, - attrs=None) + op = framework.Operator( + block=block, + desc=op_append, + type=op_type, + inputs=None, + outputs=None, + attrs=None, + ) block.ops.append(op) return op -def append_var_from_block_desc_static(block, - src_block_desc, - include=None, - exclude=None): +def append_var_from_block_desc_static( + block, src_block_desc, include=None, exclude=None +): """ Append Variables of 'src_block_desc' to current block. If 'include' is not `None`,variables that are not in include are not append. @@ -1153,13 +1294,14 @@ def append_var_from_block_desc_static(block, for var_desc in src_block_desc.all_vars(): var_desc_name = var_desc.name() should_append = (include is None or var_desc_name in include) and ( - exclude is None or var_desc_name not in exclude) + exclude is None or var_desc_name not in exclude + ) if not block.has_var(var_desc_name) and should_append: var_type = var_desc.type() if var_type in [ - core.VarDesc.VarType.SELECTED_ROWS, - core.VarDesc.VarType.LOD_TENSOR, - core.VarDesc.VarType.LOD_TENSOR_ARRAY + core.VarDesc.VarType.SELECTED_ROWS, + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.LOD_TENSOR_ARRAY, ]: data_type = var_desc.dtype() var_shape = var_desc.shape() @@ -1167,8 +1309,8 @@ def append_var_from_block_desc_static(block, data_type = None var_shape = None if var_type in [ - core.VarDesc.VarType.LOD_TENSOR, - core.VarDesc.VarType.LOD_TENSOR_ARRAY + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.LOD_TENSOR_ARRAY, ]: lod_level = var_desc.lod_level() else: @@ -1187,7 +1329,9 @@ def append_var_from_block_desc_static(block, shape=var_shape, lod_level=lod_level, persistable=var_desc.persistable(), - set_need_check_feed=var_desc.need_check_feed())) + set_need_check_feed=var_desc.need_check_feed(), + ) + ) return vars_append @@ -1312,8 +1456,9 @@ def __init__(self, programs, persistable_vars): # the TranslatedLayer object holded var names count started from 0 with unique_name.guard(): for name, var in persistable_vars.items(): - if isinstance(var, - (framework.ParamBase, framework.EagerParamBase)): + if isinstance( + var, (framework.ParamBase, framework.EagerParamBase) + ): dy_name = _generate_unique_var_name(PARAMETER_NAME_PREFIX) self._persistable_var_name_dict[name] = dy_name self.add_parameter(dy_name, var) @@ -1347,7 +1492,8 @@ def _construct(model_path, configs=None): # 2. load layer parameters & buffers persistable_vars = _construct_params_and_buffers( - model_path, programs, params_filename) + model_path, programs, params_filename + ) # 3. construct TranslatedLayer object translated_layer = TranslatedLayer(programs, persistable_vars) @@ -1359,9 +1505,12 @@ def _construct(model_path, configs=None): ins.name() for ins in program_holder.input_descs ] setattr( - TranslatedLayer, method_name, + TranslatedLayer, + method_name, TranslatedLayer._execution_method_creator( - method_name, program_holder)) + method_name, program_holder + ), + ) # 5. set TranslatedLayer's default mode to eval translated_layer.eval() @@ -1370,7 +1519,6 @@ def _construct(model_path, configs=None): @staticmethod def _execution_method_creator(method_name, program_holder): - def __i_m_p_l__(self, *input): program_holder = self._program_holder_dict[__i_m_p_l__.__name__] # When using jit.save, it runs in static graph mode. @@ -1383,7 +1531,8 @@ def __i_m_p_l__(self, *input): # because '_run_static_graph' modifies 'ProgramDesc', 'OpDesc.op_size()' will return a very large wrong number. # A Segmentation fault error may occur if used 'p=ProgramDesc(program_holder.infer_program)'. p = framework.Program._construct_from_desc( - core.ProgramDesc(program_holder.infer_program)) + core.ProgramDesc(program_holder.infer_program) + ) return _run_static_graph(input, program_holder, p.desc) __i_m_p_l__.__name__ = method_name @@ -1496,8 +1645,9 @@ def _get_program_holder(self, method_name='forward'): program_holder = self._program_holder_dict.get(method_name, None) if program_holder is None: raise ValueError( - "The method `%s` does not exist in loaded TranslatedLayer." % - method_name) + "The method `%s` does not exist in loaded TranslatedLayer." + % method_name + ) return program_holder def _input_spec(self, method_name='forward'): @@ -1507,9 +1657,11 @@ def _input_spec(self, method_name='forward'): # 2. build input spec by input desc input_spec = [] for var_desc in program_holder.input_descs: - spec = paddle.static.InputSpec(shape=var_desc.shape(), - dtype=var_desc.dtype(), - name=var_desc.name()) + spec = paddle.static.InputSpec( + shape=var_desc.shape(), + dtype=var_desc.dtype(), + name=var_desc.name(), + ) input_spec.append(spec) return input_spec @@ -1524,9 +1676,11 @@ def _output_spec(self, method_name='forward'): # NOTE(chenweihang): InputSpec describes a tensor, not just input. # Maybe the name is not good enough. Here we use InputSpec to # construct the description of Output tensor - spec = paddle.static.InputSpec(shape=var_desc.shape(), - dtype=var_desc.dtype(), - name=var_desc.name()) + spec = paddle.static.InputSpec( + shape=var_desc.shape(), + dtype=var_desc.dtype(), + name=var_desc.name(), + ) output_spec.append(spec) return output_spec diff --git a/python/paddle/fluid/dygraph/jit.py b/python/paddle/fluid/dygraph/jit.py index c659af4a7832f..acca96a937693 100644 --- a/python/paddle/fluid/dygraph/jit.py +++ b/python/paddle/fluid/dygraph/jit.py @@ -24,25 +24,65 @@ import paddle from paddle.fluid import core, dygraph -from paddle.fluid.compiler import BuildStrategy, CompiledProgram, ExecutionStrategy +from paddle.fluid.compiler import ( + BuildStrategy, + CompiledProgram, + ExecutionStrategy, +) from paddle.fluid.data_feeder import check_type from paddle.fluid.layers.utils import flatten, pack_sequence_as -from paddle.fluid.dygraph.base import program_desc_tracing_guard, switch_to_static_graph +from paddle.fluid.dygraph.base import ( + program_desc_tracing_guard, + switch_to_static_graph, +) from paddle.fluid.dygraph.dygraph_to_static import logging_utils -from paddle.fluid.dygraph.dygraph_to_static.convert_call_func import ConversionOptions, CONVERSION_OPTIONS -from paddle.fluid.dygraph.dygraph_to_static.logging_utils import set_code_level, set_verbosity -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator, StaticFunction, unwrap_decorators -from paddle.fluid.dygraph.io import TranslatedLayer, INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX, INFER_PARAMS_INFO_SUFFIX, INFER_PROPERTY_SUFFIX +from paddle.fluid.dygraph.dygraph_to_static.convert_call_func import ( + ConversionOptions, + CONVERSION_OPTIONS, +) +from paddle.fluid.dygraph.dygraph_to_static.logging_utils import ( + set_code_level, + set_verbosity, +) +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramTranslator, + StaticFunction, + unwrap_decorators, +) +from paddle.fluid.dygraph.io import ( + TranslatedLayer, + INFER_MODEL_SUFFIX, + INFER_PARAMS_SUFFIX, + INFER_PARAMS_INFO_SUFFIX, + INFER_PROPERTY_SUFFIX, +) from paddle.fluid.dygraph.layers import Layer from paddle.fluid.executor import Executor, scope_guard -from paddle.fluid.framework import Block, ParamBase, Program, Variable, Parameter, EagerParamBase -from paddle.fluid.framework import _current_expected_place, _dygraph_guard, _dygraph_tracer +from paddle.fluid.framework import ( + Block, + ParamBase, + Program, + Variable, + Parameter, + EagerParamBase, +) +from paddle.fluid.framework import ( + _current_expected_place, + _dygraph_guard, + _dygraph_tracer, +) from paddle.fluid.framework import dygraph_only, _non_static_mode from paddle.fluid.wrapped_decorator import wrap_decorator __all__ = [ - 'TracedLayer', 'declarative', 'dygraph_to_static_func', 'set_code_level', - 'set_verbosity', 'save', 'load', 'not_to_static' + 'TracedLayer', + 'declarative', + 'dygraph_to_static_func', + 'set_code_level', + 'set_verbosity', + 'save', + 'load', + 'not_to_static', ] @@ -62,8 +102,10 @@ def _extract_vars(inputs, result_list, err_tag='inputs'): _extract_vars(var, result_list, err_tag) else: raise TypeError( - "The type of 'each element of {}' in fluid.dygraph.jit.TracedLayer.trace must be fluid.Variable, but received {}." - .format(err_tag, type(inputs))) + "The type of 'each element of {}' in fluid.dygraph.jit.TracedLayer.trace must be fluid.Variable, but received {}.".format( + err_tag, type(inputs) + ) + ) def extract_vars(inputs, err_tag='inputs'): @@ -127,7 +169,8 @@ def __impl__(*args, **kwargs): logging_utils.warn( "The decorator 'dygraph_to_static_func' doesn't work in " "dygraph mode or set ProgramTranslator.enable to False. " - "We will just return dygraph output.") + "We will just return dygraph output." + ) return dygraph_func(*args, **kwargs) static_func = program_translator.get_func(dygraph_func) return static_func(*args, **kwargs) @@ -158,10 +201,9 @@ def copy_decorator_attrs(original_func, decorated_obj): return decorated_obj -def declarative(function=None, - input_spec=None, - build_strategy=None, - property=False): +def declarative( + function=None, input_spec=None, build_strategy=None, property=False +): """ Converts imperative dygraph APIs into declarative function APIs. Decorator @declarative handles the Program and Executor of static mode and returns @@ -213,20 +255,25 @@ def decorated(python_func): _, python_func = unwrap_decorators(python_func) # Step 2. copy some attributes from original python function. - static_layer = copy_decorator_attrs(original_func=python_func, - decorated_obj=StaticFunction( - function=python_func, - input_spec=input_spec, - build_strategy=build_strategy, - property=property)) + static_layer = copy_decorator_attrs( + original_func=python_func, + decorated_obj=StaticFunction( + function=python_func, + input_spec=input_spec, + build_strategy=build_strategy, + property=property, + ), + ) return static_layer build_strategy = build_strategy or BuildStrategy() if not isinstance(build_strategy, BuildStrategy): raise TypeError( - "Required type(build_strategy) shall be `paddle.static.BuildStrategy`, but received {}" - .format(type(build_strategy).__name__)) + "Required type(build_strategy) shall be `paddle.static.BuildStrategy`, but received {}".format( + type(build_strategy).__name__ + ) + ) # for usage: `declarative(foo, ...)` if function is not None: @@ -234,8 +281,10 @@ def decorated(python_func): if isinstance(function.forward, StaticFunction): class_name = function.__class__.__name__ logging_utils.warn( - "`{}.forward` has already been decorated somewhere. It will be redecorated to replace previous one." - .format(class_name)) + "`{}.forward` has already been decorated somewhere. It will be redecorated to replace previous one.".format( + class_name + ) + ) function.forward = decorated(function.forward) return function else: @@ -286,7 +335,6 @@ def func(x): class _SaveLoadConfig(object): - def __init__(self): self._output_spec = None self._model_filename = None @@ -321,12 +369,14 @@ def output_spec(self, spec): if not isinstance(spec, list): raise TypeError( "The config `output_spec` should be 'list', but received input type is %s." - % type(input)) + % type(input) + ) for var in spec: if not isinstance(var, core.VarBase): raise TypeError( "The element in config `output_spec` list should be 'Variable', but received element's type is %s." - % type(var)) + % type(var) + ) self._output_spec = spec @property @@ -340,7 +390,8 @@ def model_filename(self, filename): if not isinstance(filename, str): raise TypeError( "The config `model_filename` should be str, but received input's type is %s." - % type(filename)) + % type(filename) + ) if len(filename) == 0: raise ValueError("The config `model_filename` is empty string.") self._model_filename = filename @@ -356,7 +407,8 @@ def params_filename(self, filename): if not isinstance(filename, str): raise TypeError( "The config `params_filename` should be str, but received input's type is %s." - % type(filename)) + % type(filename) + ) if len(filename) == 0: raise ValueError("The config `params_filename` is empty string.") self._params_filename = filename @@ -372,14 +424,18 @@ def keep_name_table(self, value): if not isinstance(value, bool): raise TypeError( "The config `keep_name_table` should be bool value, but received input's type is %s." - % type(value)) + % type(value) + ) self._keep_name_table = value def _parse_save_configs(configs): supported_configs = [ - 'output_spec', "with_hook", "combine_params", "clip_extra", - "skip_forward" + 'output_spec', + "with_hook", + "combine_params", + "clip_extra", + "skip_forward", ] # input check @@ -387,7 +443,8 @@ def _parse_save_configs(configs): if key not in supported_configs: raise ValueError( "The additional config (%s) of `paddle.jit.save` is not supported." - % (key)) + % (key) + ) # construct inner config inner_config = _SaveLoadConfig() @@ -408,7 +465,8 @@ def _parse_load_config(configs): if key not in supported_configs: raise ValueError( "The additional config (%s) of `paddle.jit.load` is not supported." - % (key)) + % (key) + ) # construct inner config inner_config = _SaveLoadConfig() @@ -419,14 +477,18 @@ def _parse_load_config(configs): def _get_input_var_names(inputs, input_spec): - name_none_error = "The %s's name is None. " \ - "When using jit.save, please set InputSepc's name in " \ - "to_static(input_spec=[]) and jit.save(input_spec=[]) " \ + name_none_error = ( + "The %s's name is None. " + "When using jit.save, please set InputSepc's name in " + "to_static(input_spec=[]) and jit.save(input_spec=[]) " "and make sure they are consistent." - name_no_exists_error = "The tensor `%s` does not exists. " \ - "Please make sure the name of InputSpec or example Tensor " \ - "in input_spec is the same as the name of InputSpec in " \ + ) + name_no_exists_error = ( + "The tensor `%s` does not exists. " + "Please make sure the name of InputSpec or example Tensor " + "in input_spec is the same as the name of InputSpec in " "`to_static` decorated on the Layer.forward method." + ) result_list = [] input_var_names = [ var.name for var in flatten(inputs) if isinstance(var, Variable) @@ -437,7 +499,8 @@ def _get_input_var_names(inputs, input_spec): else: # fileter out non-tensor type spec infos. input_spec = [ - spec for spec in input_spec + spec + for spec in input_spec if isinstance(spec, paddle.static.InputSpec) ] @@ -469,10 +532,12 @@ def _get_input_var_names(inputs, input_spec): def _get_output_vars(outputs, output_spec, with_hook=False): - name_no_exists_error = "The tensor `%s` does not exists. " \ - "Please make sure the name of example Tensor " \ - "in configs.output_spec is the output tensor of " \ + name_no_exists_error = ( + "The tensor `%s` does not exists. " + "Please make sure the name of example Tensor " + "in configs.output_spec is the output tensor of " "Layer.forward method." + ) if output_spec and with_hook: raise RuntimeError( "Currently not support specify output_spec while founding pre/post hooks in your outermost layer." @@ -519,12 +584,14 @@ def _build_load_path_and_config(path, config): raise ValueError( "The %s.pdmodel and %s directory exist at the same time, " "don't know which one to load, please make sure that the specified target " - "of ``path`` is unique." % (path, path)) + "of ``path`` is unique." % (path, path) + ) elif not prefix_format_exist and not directory_format_exist: - raise ValueError("The ``path`` (%s) to load model not exists. " - "Please make sure that *.pdmodel exists or " - "don't using ``skip_forward=True`` to jit.save." % - path) + raise ValueError( + "The ``path`` (%s) to load model not exists. " + "Please make sure that *.pdmodel exists or " + "don't using ``skip_forward=True`` to jit.save." % path + ) else: if prefix_format_exist: file_prefix = os.path.basename(path) @@ -533,13 +600,15 @@ def _build_load_path_and_config(path, config): warnings.warn( "When loading the result saved with the " "specified file prefix, the ``model_filename`` config does " - "not take effect.") + "not take effect." + ) config.model_filename = file_prefix + INFER_MODEL_SUFFIX if config.params_filename is not None: warnings.warn( "When loading the result saved with the " "specified file prefix, the ``params_filename`` config does " - "not take effect.") + "not take effect." + ) config.params_filename = file_prefix + INFER_PARAMS_SUFFIX else: # Compatible with the old save_inference_model format @@ -553,7 +622,7 @@ def _build_load_path_and_config(path, config): class HookRemoveHelper(object): - """ A HookRemoveHelper that can be used to remove hook. """ + """A HookRemoveHelper that can be used to remove hook.""" def __init__(self, hook): self._hook = hook @@ -638,7 +707,6 @@ def _remove_save_pre_hook(hook): @wrap_decorator def _run_save_pre_hooks(func): - def wrapper(layer, path, input_spec=None, **configs): global _save_pre_hooks for hook in _save_pre_hooks: @@ -826,11 +894,15 @@ def fun(inputs): "The paddle.jit.save doesn't work when setting ProgramTranslator.enable to False." ) - if not (isinstance(layer, Layer) or inspect.isfunction(layer) - or isinstance(layer, StaticFunction)): + if not ( + isinstance(layer, Layer) + or inspect.isfunction(layer) + or isinstance(layer, StaticFunction) + ): raise TypeError( "The input of paddle.jit.save should be 'Layer' or 'Function', but received input type is %s." - % type(layer)) + % type(layer) + ) elif inspect.isfunction(layer) or isinstance(layer, StaticFunction): warnings.warn( 'What you save is a function, and `jit.save` will generate the name of the model file according to `path` you specify. When loading these files with `jit.load`, you get a `TranslatedLayer` whose inference result is the same as the inference result of the function you saved.' @@ -852,7 +924,8 @@ def fun(inputs): raise ValueError( "The input path MUST be format of dirname/file_prefix " "[dirname\\file_prefix in Windows system], but received " - "file_prefix is empty string.") + "file_prefix is empty string." + ) dirname = os.path.dirname(path) if dirname and not os.path.exists(dirname): @@ -864,23 +937,28 @@ def fun(inputs): if isinstance(layer, Layer): for attr_func in dir(inner_layer): static_func = getattr(inner_layer, attr_func, None) - if isinstance(static_func, - StaticFunction) and 'forward' != attr_func: + if ( + isinstance(static_func, StaticFunction) + and 'forward' != attr_func + ): raise ValueError( "If there are static functions other than 'forward' that need to be saved, the input 'input_spec' should be None, but received the type of 'input_spec' is %s." - % type(input_spec)) + % type(input_spec) + ) if not isinstance(input_spec, (list, tuple)): raise TypeError( "The input input_spec should be 'list', but received input_spec's type is %s." - % type(input_spec)) + % type(input_spec) + ) inner_input_spec = [] for var in flatten(input_spec): if isinstance(var, paddle.static.InputSpec): inner_input_spec.append(var) elif isinstance(var, (core.VarBase, core.eager.Tensor, Variable)): inner_input_spec.append( - paddle.static.InputSpec.from_tensor(var)) + paddle.static.InputSpec.from_tensor(var) + ) else: # NOTE(Aurelius84): Support non-Tensor type in `input_spec`. inner_input_spec.append(var) @@ -917,12 +995,18 @@ def fun(inputs): # property method to be exported immediate_val = static_func() property_vals.append( - (immediate_val, - layer.__class__.__name__ + '.' + attr_func)) + ( + immediate_val, + layer.__class__.__name__ + '.' + attr_func, + ) + ) continue - concrete_program = static_func.concrete_program_specify_input_spec( - inner_input_spec, with_hook=with_hook) + concrete_program = ( + static_func.concrete_program_specify_input_spec( + inner_input_spec, with_hook=with_hook + ) + ) elif 'forward' == attr_func: if configs.skip_forward: # do not jit.save forward function @@ -932,12 +1016,17 @@ def fun(inputs): # inner_input_spec is list[InputSpec], it should be packed with same structure # as original input_spec here. if inner_input_spec: - inner_input_spec = pack_sequence_as(input_spec, - inner_input_spec) - static_forward = declarative(inner_layer.forward, - input_spec=inner_input_spec) - concrete_program = static_forward.concrete_program_specify_input_spec( - with_hook=with_hook) + inner_input_spec = pack_sequence_as( + input_spec, inner_input_spec + ) + static_forward = declarative( + inner_layer.forward, input_spec=inner_input_spec + ) + concrete_program = ( + static_forward.concrete_program_specify_input_spec( + with_hook=with_hook + ) + ) # the input_spec has been used in declarative, which is equal to # @declarative with input_spec and jit.save without input_spec, # avoid needless warning @@ -953,20 +1042,27 @@ def fun(inputs): property_vals.append((immediate_val, attr_func)) continue - concrete_program = attr_func.concrete_program_specify_input_spec( - inner_input_spec) + concrete_program = ( + attr_func.concrete_program_specify_input_spec( + inner_input_spec + ) + ) else: if inner_input_spec: - inner_input_spec = pack_sequence_as(input_spec, - inner_input_spec) - static_function = declarative(attr_func, - input_spec=inner_input_spec) + inner_input_spec = pack_sequence_as( + input_spec, inner_input_spec + ) + static_function = declarative( + attr_func, input_spec=inner_input_spec + ) concrete_program = static_function.concrete_program if static_function._class_instance is None: warnings.warn( - '`jit.save` will only save the `Program`, not the parameters. If you have to save the parameters, please make sure that {} is a member function of `paddle.nn.Layer` and the saved parameters are in `state_dict`' - .format(layer)) + '`jit.save` will only save the `Program`, not the parameters. If you have to save the parameters, please make sure that {} is a member function of `paddle.nn.Layer` and the saved parameters are in `state_dict`'.format( + layer + ) + ) # when save multi `StaticFunction`, all `StaticFunction` share params. dygraph_state_dict = None @@ -974,7 +1070,8 @@ def fun(inputs): dygraph_state_dict = inner_layer.to_static_state_dict() elif isinstance(attr_func, StaticFunction): if attr_func._class_instance: - dygraph_state_dict = attr_func._class_instance.to_static_state_dict( + dygraph_state_dict = ( + attr_func._class_instance.to_static_state_dict() ) if dygraph_state_dict: @@ -998,19 +1095,25 @@ def fun(inputs): tgt_var.set_vocab(scr_tensor) else: param_or_buffer_tensor = scope.var( - param_or_buffer.name).get_tensor() - #src_tensor = param_or_buffer.value().get_tensor() - src_tensor = state_var_dict[ - param_or_buffer.name].value().get_tensor() + param_or_buffer.name + ).get_tensor() + # src_tensor = param_or_buffer.value().get_tensor() + src_tensor = ( + state_var_dict[param_or_buffer.name] + .value() + .get_tensor() + ) param_or_buffer_tensor._share_data_with(src_tensor) # record var info if param_or_buffer.name not in extra_var_info: extra_info_dict = dict() if param_or_buffer.name in state_names_dict: extra_info_dict['structured_name'] = state_names_dict[ - param_or_buffer.name] + param_or_buffer.name + ] extra_info_dict[ - 'stop_gradient'] = param_or_buffer.stop_gradient + 'stop_gradient' + ] = param_or_buffer.stop_gradient if isinstance(param_or_buffer, (ParamBase, EagerParamBase)): extra_info_dict['trainable'] = param_or_buffer.trainable extra_var_info[param_or_buffer.name] = extra_info_dict @@ -1024,8 +1127,9 @@ def fun(inputs): # - prune inputs: # - the input_spec length < len((concrete_program.inputs) - 1 # - the input_spec's name should be in concrete_program.inputs - input_var_names = _get_input_var_names(concrete_program.inputs, - inner_input_spec) + input_var_names = _get_input_var_names( + concrete_program.inputs, inner_input_spec + ) # NOTE(chenweihang): [ Get output variables ] # the rule is like [ Get input variables name ]. For output var, @@ -1033,8 +1137,9 @@ def fun(inputs): # var name of output, and we don't recommended to use output_spec # print(concrete_program.main_program) # print(concrete_program.outputs, configs.output_spec) - output_vars = _get_output_vars(concrete_program.outputs, - configs.output_spec, with_hook) + output_vars = _get_output_vars( + concrete_program.outputs, configs.output_spec, with_hook + ) # 5. save inference model from paddle.fluid.io import save_inference_model @@ -1048,7 +1153,9 @@ def fun(inputs): params_filename = file_prefix + INFER_PARAMS_SUFFIX else: model_filename = file_prefix + '.' + attr_func + INFER_MODEL_SUFFIX - params_filename = file_prefix + '.' + attr_func + INFER_PARAMS_SUFFIX + params_filename = ( + file_prefix + '.' + attr_func + INFER_PARAMS_SUFFIX + ) with scope_guard(scope): save_inference_model( @@ -1061,12 +1168,14 @@ def fun(inputs): params_filename=params_filename, export_for_deployment=configs._export_for_deployment, program_only=configs._program_only, - clip_extra=configs.clip_extra) + clip_extra=configs.clip_extra, + ) if combine_params: clone_main_program = concrete_program.main_program.clone() clone_main_program = clone_main_program._prune_with_input( - input_var_names, output_vars) + input_var_names, output_vars + ) for block in clone_main_program.blocks: combine_vars.update(block.vars) @@ -1080,15 +1189,16 @@ def fun(inputs): params_filename = file_prefix + INFER_PARAMS_SUFFIX with scope_guard(scope): - paddle.static.save_vars(Executor(_current_expected_place()), - dirname=model_path, - vars=list( - filter(paddle.fluid.io.is_persistable, - ordered_vars)), - filename=params_filename) + paddle.static.save_vars( + Executor(_current_expected_place()), + dirname=model_path, + vars=list(filter(paddle.fluid.io.is_persistable, ordered_vars)), + filename=params_filename, + ) # save property - property_save_path = os.path.join(os.path.normpath(model_path), - file_prefix + INFER_PROPERTY_SUFFIX) + property_save_path = os.path.join( + os.path.normpath(model_path), file_prefix + INFER_PROPERTY_SUFFIX + ) _save_property(property_save_path, property_vals) # NOTE(chenweihang): [ Save extra variable info ] @@ -1350,11 +1460,9 @@ def __len__(self): @dygraph_only -def _trace(layer, - inputs, - feed_prefix='feed_', - fetch_prefix='fetch_', - tmp_prefix='t_'): +def _trace( + layer, inputs, feed_prefix='feed_', fetch_prefix='fetch_', tmp_prefix='t_' +): assert isinstance(layer, Layer) if not isinstance(inputs, (list, tuple)): @@ -1372,8 +1480,14 @@ def _trace(layer, outputs = original_outputs out_vars = extract_vars(outputs, err_tag='outputs') - program_desc, feed_names, fetch_names, parameters = tracer.create_program_desc( - var_list, feed_prefix, out_vars, fetch_prefix, tmp_prefix) + ( + program_desc, + feed_names, + fetch_names, + parameters, + ) = tracer.create_program_desc( + var_list, feed_prefix, out_vars, fetch_prefix, tmp_prefix + ) tracer.reset() with _dygraph_guard(None): @@ -1483,7 +1597,8 @@ def forward(self, input): assert isinstance( layer, Layer ), "The type of 'layer' in fluid.dygraph.jit.TracedLayer.trace must be fluid.dygraph.Layer, but received {}.".format( - type(layer)) + type(layer) + ) outs, prog, feed, fetch, parameters = _trace(layer, inputs) traced = TracedLayer(prog, parameters, feed, fetch) return outs, traced @@ -1533,25 +1648,30 @@ def forward(self, input): assert isinstance( build_strategy, (type(None), BuildStrategy) ), "The type of 'build_strategy' in fluid.dygraph.jit.TracedLayer.set_strategy must be fluid.BuildStrategy, but received {}.".format( - type(build_strategy)) + type(build_strategy) + ) assert isinstance( exec_strategy, (type(None), ExecutionStrategy) ), "The type of 'exec_strategy' in fluid.dygraph.jit.TracedLayer.set_strategy must be fluid.ExecutionStrategy, but received {}.".format( - type(exec_strategy)) + type(exec_strategy) + ) self._build_strategy = build_strategy self._exec_strategy = exec_strategy @switch_to_static_graph def _compile(self): self._compiled_program = CompiledProgram( - self._program).with_data_parallel( - build_strategy=self._build_strategy, - exec_strategy=self._exec_strategy, - places=self._place) + self._program + ).with_data_parallel( + build_strategy=self._build_strategy, + exec_strategy=self._exec_strategy, + places=self._place, + ) def _build_feed(self, inputs): - assert isinstance(inputs, (list, tuple)), \ - "Inputs should be a list or tuple of variables" + assert isinstance( + inputs, (list, tuple) + ), "Inputs should be a list or tuple of variables" assert len(inputs) == len(self._feed_names) feed_dict = {} if _non_static_mode(): @@ -1565,9 +1685,9 @@ def _build_feed(self, inputs): @switch_to_static_graph def _run(self, feed): - return self._exe.run(self._compiled_program, - feed=feed, - fetch_list=self._fetch_names) + return self._exe.run( + self._compiled_program, feed=feed, fetch_list=self._fetch_names + ) def __call__(self, inputs): with scope_guard(self._scope): @@ -1631,22 +1751,40 @@ def forward(self, input): fetch, = exe.run(program, feed={feed_vars[0]: in_np}, fetch_list=fetch_vars) print(fetch.shape) # (2, 10) """ - check_type(path, "path", str, - "fluid.dygraph.jit.TracedLayer.save_inference_model") - check_type(feed, "feed", (type(None), list), - "fluid.dygraph.jit.TracedLayer.save_inference_model") + check_type( + path, + "path", + str, + "fluid.dygraph.jit.TracedLayer.save_inference_model", + ) + check_type( + feed, + "feed", + (type(None), list), + "fluid.dygraph.jit.TracedLayer.save_inference_model", + ) if isinstance(feed, list): for f in feed: check_type( - f, "each element of feed", int, - "fluid.dygraph.jit.TracedLayer.save_inference_model") - check_type(fetch, "fetch", (type(None), list), - "fluid.dygraph.jit.TracedLayer.save_inference_model") + f, + "each element of feed", + int, + "fluid.dygraph.jit.TracedLayer.save_inference_model", + ) + check_type( + fetch, + "fetch", + (type(None), list), + "fluid.dygraph.jit.TracedLayer.save_inference_model", + ) if isinstance(fetch, list): for f in fetch: check_type( - f, "each element of fetch", int, - "fluid.dygraph.jit.TracedLayer.save_inference_model") + f, + "each element of fetch", + int, + "fluid.dygraph.jit.TracedLayer.save_inference_model", + ) clip_extra = kwargs.get('clip_extra', True) # path check file_prefix = os.path.basename(path) @@ -1654,7 +1792,8 @@ def forward(self, input): raise ValueError( "The input path MUST be format of dirname/file_prefix " "[dirname\\file_prefix in Windows system], but received " - "file_prefix is empty string.") + "file_prefix is empty string." + ) dirname = os.path.dirname(path) if dirname and not os.path.exists(dirname): @@ -1680,11 +1819,13 @@ def get_feed_fetch(all_vars, partial_vars): model_filename = file_prefix + INFER_MODEL_SUFFIX params_filename = file_prefix + INFER_PARAMS_SUFFIX - save_inference_model(dirname=dirname, - feeded_var_names=feeded_var_names, - target_vars=target_vars, - executor=self._exe, - main_program=self._program.clone(), - model_filename=model_filename, - params_filename=params_filename, - clip_extra=clip_extra) + save_inference_model( + dirname=dirname, + feeded_var_names=feeded_var_names, + target_vars=target_vars, + executor=self._exe, + main_program=self._program.clone(), + model_filename=model_filename, + params_filename=params_filename, + clip_extra=clip_extra, + ) diff --git a/python/paddle/fluid/dygraph/layer_hooks.py b/python/paddle/fluid/dygraph/layer_hooks.py index a231e7cf8745a..8a373cd17c86d 100644 --- a/python/paddle/fluid/dygraph/layer_hooks.py +++ b/python/paddle/fluid/dygraph/layer_hooks.py @@ -37,13 +37,16 @@ def record_program_ops_pre_hook(layer, inputs): if not _non_static_mode(): if layer._op_recorder.start < 0: layer._op_recorder.start = len( - default_main_program().current_block().ops) + default_main_program().current_block().ops + ) layer._op_recorder.is_valid = True else: layer._op_recorder.is_valid = False warnings.warn( - "{} has recorded the op information before. Please check whether you call this layer twice." - .format(layer._full_name)) + "{} has recorded the op information before. Please check whether you call this layer twice.".format( + layer._full_name + ) + ) return None @@ -56,7 +59,7 @@ def set_op_customized_attrs_post_hook(layer, inputs, outputs): start = layer._op_recorder.start end = len(default_main_program().current_block().ops) - assert (start >= 0 and end >= start) + assert start >= 0 and end >= start ops = default_main_program().current_block().ops[start:end] layer._op_recorder.end = end diff --git a/python/paddle/fluid/dygraph/layer_object_helper.py b/python/paddle/fluid/dygraph/layer_object_helper.py index bdd34abd5a865..b07798b72e642 100644 --- a/python/paddle/fluid/dygraph/layer_object_helper.py +++ b/python/paddle/fluid/dygraph/layer_object_helper.py @@ -22,16 +22,17 @@ class LayerObjectHelper(LayerHelperBase): - def __init__(self, name): super(LayerObjectHelper, self).__init__(name, layer_type=name) - def append_op(self, - type=None, - inputs=None, - outputs=None, - attrs=None, - stop_gradient=None): + def append_op( + self, + type=None, + inputs=None, + outputs=None, + attrs=None, + stop_gradient=None, + ): """append an operator for this layer object. Args: @@ -48,7 +49,8 @@ def append_op(self, inputs=inputs, outputs=outputs, attrs=attrs, - stop_gradient=stop_gradient) + stop_gradient=stop_gradient, + ) def _multiple_input(self, inputs_in): inputs = inputs_in @@ -73,8 +75,9 @@ def _multiple_param_attr(self, length, param_attr_in=None): param_attr = [param_attr] if len(param_attr) != 1 and len(param_attr) != length: - raise ValueError("parameter number mismatch in {}".format( - self.name)) + raise ValueError( + "parameter number mismatch in {}".format(self.name) + ) elif len(param_attr) == 1 and length != 1: tmp = [None] * length for i in range(length): @@ -93,8 +96,9 @@ def iter_inputs_and_params(self, inputs_in, param_attr_in=None): """ param_attr_in = ParamAttr._to_attr(param_attr_in) if isinstance(param_attr_in, bool): - raise ValueError('Param_attr should not be False in {}'.format( - self.name)) + raise ValueError( + 'Param_attr should not be False in {}'.format(self.name) + ) inputs = inputs_in if (inputs_in is not None) else [] inputs = self._multiple_input(inputs) param_attrs = self._multiple_param_attr(len(inputs), param_attr_in) @@ -116,8 +120,10 @@ def input_dtype(self, inputs_in): if dtype is None: dtype = each.dtype elif dtype != each.dtype: - raise ValueError("Data Type mismatch: %d to %d in %s" % - (dtype, each.dtype, self.name)) + raise ValueError( + "Data Type mismatch: %d to %d in %s" + % (dtype, each.dtype, self.name) + ) return dtype def get_parameter(self, name): @@ -130,8 +136,9 @@ def get_parameter(self, name): """ param = self.main_program.global_block().var(name) if not isinstance(param, Parameter): - raise ValueError("no Parameter name %s found in %s" % - (name, self.name)) + raise ValueError( + "no Parameter name %s found in %s" % (name, self.name) + ) return param # TODO: this should not be called anymore after all activation func move to Layers @@ -153,7 +160,8 @@ def append_activation(self, input_var, act=None, use_cudnn=None): act = {'type': act} else: raise TypeError( - str(act) + " should be unicode or str in %s ", self.name) + str(act) + " should be unicode or str in %s ", self.name + ) if (use_cudnn is not None) and use_cudnn: act['use_cudnn'] = use_cudnn @@ -162,15 +170,18 @@ def append_activation(self, input_var, act=None, use_cudnn=None): act['use_mkldnn'] = use_mkldnn act_type = act.pop('type') if _non_static_mode(): - res = _append_activation_in_dygraph(input_var, act_type, use_cudnn, - use_mkldnn) + res = _append_activation_in_dygraph( + input_var, act_type, use_cudnn, use_mkldnn + ) return res else: tmp = self.create_variable_for_type_inference(dtype=input_var.dtype) - self.append_op(type=act_type, - inputs={"X": [input_var]}, - outputs={"Out": [tmp]}, - attrs=act) + self.append_op( + type=act_type, + inputs={"X": [input_var]}, + outputs={"Out": [tmp]}, + attrs=act, + ) return tmp def is_instance(self, param, cls): @@ -186,4 +197,8 @@ def is_instance(self, param, cls): if not isinstance(param, cls): raise TypeError( "The input {0} parameter of method {1} must be {2}, in layer {3}", - param, self.layer_type, cls.__name__, self.name) + param, + self.layer_type, + cls.__name__, + self.name, + ) diff --git a/python/paddle/fluid/dygraph/layers.py b/python/paddle/fluid/dygraph/layers.py index 29f2e186bd2e7..889a910d8b9ac 100644 --- a/python/paddle/fluid/dygraph/layers.py +++ b/python/paddle/fluid/dygraph/layers.py @@ -31,12 +31,25 @@ from .. import unique_name from paddle.fluid import core from .layer_object_helper import LayerObjectHelper -from .layer_hooks import record_program_ops_pre_hook, set_op_customized_attrs_post_hook, LayerOpsRecoder -from .base import program_desc_tracing_guard, param_guard, in_declarative_mode, _convert_into_variable +from .layer_hooks import ( + record_program_ops_pre_hook, + set_op_customized_attrs_post_hook, + LayerOpsRecoder, +) +from .base import ( + program_desc_tracing_guard, + param_guard, + in_declarative_mode, + _convert_into_variable, +) from paddle.fluid import framework from ..param_attr import ParamAttr from paddle.fluid.executor import Executor, global_scope -from paddle.fluid.framework import _non_static_mode, convert_np_dtype_to_dtype_, in_dygraph_mode +from paddle.fluid.framework import ( + _non_static_mode, + convert_np_dtype_to_dtype_, + in_dygraph_mode, +) from paddle.fluid.framework import Program, program_guard from paddle.fluid.framework import _current_expected_place as _get_device from paddle.fluid.core import VarDesc @@ -66,7 +79,7 @@ def _addindent(string, indent): class HookRemoveHelper(object): - """ A HookRemoveHelper that can be used to remove hook. """ + """A HookRemoveHelper that can be used to remove hook.""" next_hook_id = 0 @@ -397,12 +410,14 @@ def forward_pre_hook(layer, input): self._forward_pre_hooks[hook_remove_helper._hook_id] = hook return hook_remove_helper - def create_parameter(self, - shape, - attr=None, - dtype=None, - is_bias=False, - default_initializer=None): + def create_parameter( + self, + shape, + attr=None, + dtype=None, + is_bias=False, + default_initializer=None, + ): """Create parameters for this layer. Parameters: @@ -442,12 +457,15 @@ def forward(self, input): temp_attr = copy.deepcopy(attr) if isinstance(temp_attr, str) and temp_attr == "": temp_attr = None - return self._helper.create_parameter(temp_attr, shape, dtype, is_bias, - default_initializer) - - @deprecated(since="2.0.0", - update_to="paddle.nn.Layer.create_tensor", - reason="New api in create_tensor, easier to use.") + return self._helper.create_parameter( + temp_attr, shape, dtype, is_bias, default_initializer + ) + + @deprecated( + since="2.0.0", + update_to="paddle.nn.Layer.create_tensor", + reason="New api in create_tensor, easier to use.", + ) def create_variable(self, name=None, persistable=None, dtype=None): """ @@ -487,14 +505,16 @@ def forward(self, input): if name is not None: var_name = ".".join([self._full_name, name]) else: - var_name = unique_name.generate(".".join( - [self._full_name, "_generated_var"])) + var_name = unique_name.generate( + ".".join([self._full_name, "_generated_var"]) + ) return self._helper.main_program.current_block().create_var( name=var_name, persistable=persistable, dtype=dtype, - type=core.VarDesc.VarType.LOD_TENSOR) + type=core.VarDesc.VarType.LOD_TENSOR, + ) # TODO: Add more parameter list when we need them def create_tensor(self, name=None, persistable=None, dtype=None): @@ -537,14 +557,16 @@ def forward(self, input): if name is not None: var_name = ".".join([self._full_name, name]) else: - var_name = unique_name.generate(".".join( - [self._full_name, "_generated_var"])) + var_name = unique_name.generate( + ".".join([self._full_name, "_generated_var"]) + ) return self._helper.main_program.current_block().create_var( name=var_name, persistable=persistable, dtype=dtype, - type=core.VarDesc.VarType.LOD_TENSOR) + type=core.VarDesc.VarType.LOD_TENSOR, + ) def parameters(self, include_sublayers=True): """Returns a list of all Parameters from current layer and its sub-layers. @@ -562,8 +584,10 @@ def parameters(self, include_sublayers=True): """ ret = [ - param for _, param in self.named_parameters( - include_sublayers=include_sublayers) + param + for _, param in self.named_parameters( + include_sublayers=include_sublayers + ) ] return ret @@ -677,9 +701,11 @@ def named_parameters(self, prefix='', include_sublayers=True): """ params_set = set() - named_sublayers = self.named_sublayers( - prefix=prefix, include_self=True) if include_sublayers else zip( - [prefix], [self]) + named_sublayers = ( + self.named_sublayers(prefix=prefix, include_self=True) + if include_sublayers + else zip([prefix], [self]) + ) for layer_prefix, sublayer in named_sublayers: params = sublayer._parameters.items() for key, param in params: @@ -723,9 +749,9 @@ def named_sublayers(self, prefix='', include_self=False, layers_set=None): if layer is None: continue layer_prefix = prefix + ('.' if prefix else '') + key - for p, l in layer.named_sublayers(prefix=layer_prefix, - include_self=True, - layers_set=layers_set): + for p, l in layer.named_sublayers( + prefix=layer_prefix, include_self=True, layers_set=layers_set + ): yield p, l def register_buffer(self, name, tensor, persistable=True): @@ -768,25 +794,32 @@ def register_buffer(self, name, tensor, persistable=True): if '_buffers' not in self.__dict__: raise ValueError( - "super(YourLayer, self).__init__() should be called first") + "super(YourLayer, self).__init__() should be called first" + ) elif not isinstance(name, str): raise TypeError( - "The name of buffer should be a string, but received {}.". - format(type(name).__name__)) + "The name of buffer should be a string, but received {}.".format( + type(name).__name__ + ) + ) elif '.' in name: raise KeyError( "The name of buffer can not contain `.`, " "because when you access the newly added buffer in the " - "form of `self.**.**`, it will cause AttributeError.") + "form of `self.**.**`, it will cause AttributeError." + ) elif name == '': raise KeyError("The name of buffer can not be empty.") elif hasattr(self, name) and name not in self._buffers: raise KeyError("attribute '{}' already exists.".format(name)) - elif tensor is not None and not (type(tensor) == core.VarBase - or type(tensor) == core.eager.Tensor): + elif tensor is not None and not ( + type(tensor) == core.VarBase or type(tensor) == core.eager.Tensor + ): raise TypeError( - "The registered buffer should be a Paddle.Tensor, but received {}." - .format(type(tensor).__name__)) + "The registered buffer should be a Paddle.Tensor, but received {}.".format( + type(tensor).__name__ + ) + ) else: self._buffers[name] = tensor if persistable: @@ -819,8 +852,10 @@ def buffers(self, include_sublayers=True): """ ret = [ - buffer for _, buffer in self.named_buffers( - include_sublayers=include_sublayers) + buffer + for _, buffer in self.named_buffers( + include_sublayers=include_sublayers + ) ] return ret @@ -861,9 +896,11 @@ def named_buffers(self, prefix='', include_sublayers=True): """ buffers_set = set() - named_sublayers = self.named_sublayers( - prefix=prefix, include_self=True) if include_sublayers else zip( - [prefix], [self]) + named_sublayers = ( + self.named_sublayers(prefix=prefix, include_self=True) + if include_sublayers + else zip([prefix], [self]) + ) for layer_prefix, sublayer in named_sublayers: buffers = sublayer._buffers.items() for key, buffer in buffers: @@ -909,7 +946,7 @@ def _dygraph_call_func(self, *inputs, **kwargs): hook_result = forward_pre_hook(self, inputs) if hook_result is not None: if not isinstance(hook_result, tuple): - hook_result = (hook_result, ) + hook_result = (hook_result,) inputs = hook_result if not self._built: @@ -919,16 +956,20 @@ def _dygraph_call_func(self, *inputs, **kwargs): # TODO(liuyuhui) Only xpu broadcast parameters here. # The other device is to call _sync_params_buffers in DataParallel # to realize the parameter synchronization among multiply cards. - if parallel_helper._is_data_parallel_mode( - ) and paddle.is_compiled_with_xpu(): + if ( + parallel_helper._is_data_parallel_mode() + and paddle.is_compiled_with_xpu() + ): parallel_helper._broadcast_parameters( - self._parameters.values()) + self._parameters.values() + ) self._built = True if in_profiler_mode(): - with profiler.RecordEvent(self.__class__.__name__, - profiler.TracerEventType.Forward): + with profiler.RecordEvent( + self.__class__.__name__, profiler.TracerEventType.Forward + ): outputs = self.forward(*inputs, **kwargs) else: outputs = self.forward(*inputs, **kwargs) @@ -941,8 +982,14 @@ def _dygraph_call_func(self, *inputs, **kwargs): return outputs def __call__(self, *inputs, **kwargs): - if (not in_declarative_mode()) and (not self._forward_pre_hooks) \ - and (not self._forward_post_hooks) and (not self._built) and in_dygraph_mode() and (not in_profiler_mode()): + if ( + (not in_declarative_mode()) + and (not self._forward_pre_hooks) + and (not self._forward_post_hooks) + and (not self._built) + and in_dygraph_mode() + and (not in_profiler_mode()) + ): self._build_once(*inputs, **kwargs) return self.forward(*inputs, **kwargs) else: @@ -999,7 +1046,7 @@ def forward(self, input): for prefix, layer in model.named_sublayers(): print(prefix, layer) """ - assert (isinstance(sublayer, Layer) or sublayer == None) + assert isinstance(sublayer, Layer) or sublayer == None self._sub_layers[name] = sublayer return sublayer @@ -1036,32 +1083,42 @@ def forward(self, input): """ if '_parameters' not in self.__dict__: raise RuntimeError( - "super(YourLayer, self).__init__() should be called firstly.") + "super(YourLayer, self).__init__() should be called firstly." + ) elif not isinstance(name, str): raise TypeError( - "The name of parameter should be a string, but received {}.". - format(type(name).__name__)) + "The name of parameter should be a string, but received {}.".format( + type(name).__name__ + ) + ) elif '.' in name: raise KeyError( "The name of parameter can not contain `.`, " "because when you access the newly added parameter in the " - "form of `self.**.**`, it will cause AttributeError.") + "form of `self.**.**`, it will cause AttributeError." + ) elif name == '': raise KeyError("The name of parameter can not be empty.") elif hasattr(self, name) and name not in self._parameters: raise KeyError("The parameter '{}' already exists.".format(name)) - elif parameter is not None and not isinstance(parameter, - framework.Parameter): + elif parameter is not None and not isinstance( + parameter, framework.Parameter + ): raise TypeError( - "The parameter to be added should be a Parameter, but received {}." - .format(type(parameter).__name__)) + "The parameter to be added should be a Parameter, but received {}.".format( + type(parameter).__name__ + ) + ) else: if parameter is None: self._parameters[name] = None if len(self._loaddict_holder) > 0: - assert parameter.name in self._loaddict_holder, "Parameter not found, Can't not find [ {} ] in state_dict".format( - parameter.name) + assert ( + parameter.name in self._loaddict_holder + ), "Parameter not found, Can't not find [ {} ] in state_dict".format( + parameter.name + ) parameter.set_value(self._loaddict_holder[parameter.name]) @@ -1080,37 +1137,50 @@ def _set_op_attrs(self, attrs): """ def is_already_registered(is_pre_hook): - layers_hooks = self._forward_pre_hooks if is_pre_hook else self._forward_post_hooks - candidate_hook = record_program_ops_pre_hook if is_pre_hook else set_op_customized_attrs_post_hook + layers_hooks = ( + self._forward_pre_hooks + if is_pre_hook + else self._forward_post_hooks + ) + candidate_hook = ( + record_program_ops_pre_hook + if is_pre_hook + else set_op_customized_attrs_post_hook + ) already_registed = False if layers_hooks: last_key = next(reversed(layers_hooks)) - already_registed = (layers_hooks[last_key] == candidate_hook) + already_registed = layers_hooks[last_key] == candidate_hook return already_registed if not isinstance(attrs, dict): raise TypeError( "attrs should be type(dict), but received {}".format( - type(attrs).__name__)) + type(attrs).__name__ + ) + ) # NOTE: Overwrite behavior for same key. self._customized_attrs.update(attrs) if not is_already_registered(is_pre_hook=True): pre_hook_helper = self.register_forward_pre_hook( - record_program_ops_pre_hook) + record_program_ops_pre_hook + ) assert len(self._op_recorder.hooks) == 0 self._op_recorder.hooks = [pre_hook_helper] # manually register post_hook to ensure it is inserted into the head. if not is_already_registered(is_pre_hook=False): post_hook_helper = self.register_forward_post_hook( - set_op_customized_attrs_post_hook) + set_op_customized_attrs_post_hook + ) if len(self._forward_post_hooks) > 1: - self._forward_post_hooks.move_to_end(post_hook_helper._hook_id, - last=False) + self._forward_post_hooks.move_to_end( + post_hook_helper._hook_id, last=False + ) assert len(self._op_recorder.hooks) == 1 @@ -1143,7 +1213,6 @@ def __getattr__(self, name): return object.__getattribute__(self, name) def __setattr__(self, name, value): - def _remove_if_exist(*dicts): for d in dicts: if name in d: @@ -1155,10 +1224,14 @@ def _remove_if_exist(*dicts): if isinstance(value, framework.Parameter): if params is None: raise ValueError( - "super(YourLayer, self).__init__() should be called first") + "super(YourLayer, self).__init__() should be called first" + ) if len(self._loaddict_holder) > 0: - assert value.name in self._loaddict_holder, "Parameter not found, Can't not find [ {} ] in state_dict".format( - value.name) + assert ( + value.name in self._loaddict_holder + ), "Parameter not found, Can't not find [ {} ] in state_dict".format( + value.name + ) value.set_value(self._loaddict_holder[value.name]) @@ -1167,9 +1240,10 @@ def _remove_if_exist(*dicts): elif params is not None and name in params: if value is not None: raise TypeError( - "assignment to parameter '{}' should be of type Parameter or None, but got '{}'" - .format(name, - type(value).__name__)) + "assignment to parameter '{}' should be of type Parameter or None, but got '{}'".format( + name, type(value).__name__ + ) + ) params[name] = None else: layers = self.__dict__.get('_sub_layers', None) @@ -1184,9 +1258,10 @@ def _remove_if_exist(*dicts): elif layers is not None and name in layers: if value is not None: raise TypeError( - "assignment to sublayer '{}' should be of type Layer or None, but got '{}'" - .format(name, - type(value).__name__)) + "assignment to sublayer '{}' should be of type Layer or None, but got '{}'".format( + name, type(value).__name__ + ) + ) layers[name] = None else: _buffers = self.__dict__.get('_buffers', None) @@ -1195,8 +1270,9 @@ def _remove_if_exist(*dicts): raise ValueError( "super(YourLayer, self).__init__() should be called first" ) - _remove_if_exist(self.__dict__, self._parameters, - self._sub_layers) + _remove_if_exist( + self.__dict__, self._parameters, self._sub_layers + ) # Set persistable=False by default. Only `register_buffer` can # add a persistable buffer. if name not in self._buffers: @@ -1210,6 +1286,7 @@ def _remove_if_exist(*dicts): # value via `assign`. if type(value) == framework.Variable: from paddle import assign + # Note(zhhsplendid): the condition below happens in PaddleGan model, # but should all non-Variable _buffers[name] be re-assign? We # should consider it in the future. I current wrote this as @@ -1217,18 +1294,23 @@ def _remove_if_exist(*dicts): if in_declarative_mode() and _buffers[name] is None: raise RuntimeError( 'In Dy2stat, self.{0} is a buffer and self.{0} is ' - 'not allowed to be set to Variable when self.{0} is None.' - .format(name)) - elif _buffers[name] is None or type(getattr( - self, name)) == core.VarBase: + 'not allowed to be set to Variable when self.{0} is None.'.format( + name + ) + ) + elif ( + _buffers[name] is None + or type(getattr(self, name)) == core.VarBase + ): _buffers[name] = assign(value) else: assign(value, getattr(self, name)) elif value is not None: raise TypeError( - "assignment to buffers '{}' should be of type core.VarBase or None, but got '{}'" - .format(name, - type(value).__name__)) + "assignment to buffers '{}' should be of type core.VarBase or None, but got '{}'".format( + name, type(value).__name__ + ) + ) else: # Assigning None will remove the buffer, but if re-assign a new varBase to it, # it will be remarked as a buffer with same `persistable` attribute. @@ -1315,10 +1397,12 @@ def register_state_dict_hook(self, hook): self._state_dict_hooks[hook_remove_helper._hook_id] = hook return hook_remove_helper - def _obtain_parameters_buffers(self, - destination=None, - include_sublayers=True, - structured_name_prefix=""): + def _obtain_parameters_buffers( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + ): """ The difference from state_dict() is that state_dict_hook will not be called, but the original types of parameters and buffers will be maintained. @@ -1329,7 +1413,10 @@ def _obtain_parameters_buffers(self, if data is not None: destination[structured_name_prefix + name] = data for name, buffer in self._buffers.items(): - if buffer is not None and name not in self._non_persistable_buffer_names_set: + if ( + buffer is not None + and name not in self._non_persistable_buffer_names_set + ): destination[structured_name_prefix + name] = buffer if include_sublayers: @@ -1338,17 +1425,22 @@ def _obtain_parameters_buffers(self, destination_temp = destination.copy() destination_temp.update( layer_item._obtain_parameters_buffers( - destination_temp, include_sublayers, - structured_name_prefix + layer_name + ".")) + destination_temp, + include_sublayers, + structured_name_prefix + layer_name + ".", + ) + ) destination = destination_temp return destination - def _state_dict_impl(self, - destination=None, - include_sublayers=True, - structured_name_prefix="", - include_non_persistable_buffer=False, - use_hook=True): + def _state_dict_impl( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + include_non_persistable_buffer=False, + use_hook=True, + ): """ Get all parameters and persistable buffers of current layer and its sub-layers. And set them into a dict @@ -1366,7 +1458,10 @@ def _state_dict_impl(self, destination[structured_name_prefix + name] = data for name, buffer in self._buffers.items(): if not include_non_persistable_buffer: - if buffer is not None and name not in self._non_persistable_buffer_names_set: + if ( + buffer is not None + and name not in self._non_persistable_buffer_names_set + ): destination[structured_name_prefix + name] = buffer else: if buffer is not None: @@ -1378,9 +1473,13 @@ def _state_dict_impl(self, destination_temp = destination.copy() destination_temp.update( layer_item._state_dict_impl( - destination_temp, include_sublayers, + destination_temp, + include_sublayers, structured_name_prefix + layer_name + ".", - include_non_persistable_buffer, use_hook)) + include_non_persistable_buffer, + use_hook, + ) + ) destination = destination_temp if use_hook: for state_dict_hook in self._state_dict_hooks.values(): @@ -1390,11 +1489,13 @@ def _state_dict_impl(self, return destination - def to_static_state_dict(self, - destination=None, - include_sublayers=True, - structured_name_prefix="", - use_hook=True): + def to_static_state_dict( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + use_hook=True, + ): ''' Get all parameters and buffers of current layer and its sub-layers. And set them into a dict @@ -1422,13 +1523,16 @@ def to_static_state_dict(self, include_sublayers=include_sublayers, structured_name_prefix=structured_name_prefix, include_non_persistable_buffer=True, - use_hook=use_hook) - - def state_dict(self, - destination=None, - include_sublayers=True, - structured_name_prefix="", - use_hook=True): + use_hook=use_hook, + ) + + def state_dict( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + use_hook=True, + ): ''' Get all parameters and persistable buffers of current layer and its sub-layers. And set them into a dict @@ -1456,7 +1560,8 @@ def state_dict(self, include_sublayers=include_sublayers, structured_name_prefix=structured_name_prefix, include_non_persistable_buffer=False, - use_hook=use_hook) + use_hook=use_hook, + ) @framework.deprecate_stat_dict def set_state_dict(self, state_dict, use_structured_name=True): @@ -1488,22 +1593,31 @@ def _check_match(key, param): state = state_dict.get(key, None) if state is None: raise ValueError( - "{} is not found in the provided dict.".format(key)) - if (isinstance(state, dict) or isinstance(state, list)): - if (len(state) != len(param)): - raise ValueError("{} receieves the length of {}, " - "but the expected shape is {}".format( - key, len(state), len(param))) + "{} is not found in the provided dict.".format(key) + ) + if isinstance(state, dict) or isinstance(state, list): + if len(state) != len(param): + raise ValueError( + "{} receieves the length of {}, " + "but the expected shape is {}".format( + key, len(state), len(param) + ) + ) else: return param, state else: - state_shape = state.shape() if inspect.ismethod( - state.shape) else state.shape + state_shape = ( + state.shape() + if inspect.ismethod(state.shape) + else state.shape + ) if list(state_shape) != list(param.shape): raise ValueError( - "{} receives a shape {}, but the expected shape is {}.". - format(key, list(state_shape), list(param.shape))) + "{} receives a shape {}, but the expected shape is {}.".format( + key, list(state_shape), list(param.shape) + ) + ) return param, state matched_param_state = [] @@ -1542,7 +1656,9 @@ def _set_var(var, ndarray): # restore parameter states core._create_loaded_parameter( [param for param, state in matched_param_state], - global_scope(), executor) + global_scope(), + executor, + ) for param, state in matched_param_state: _set_var(param, state) except ValueError as e: @@ -1598,11 +1714,13 @@ def to(self, device=None, dtype=None, blocking=None): # [ 0.33960250, 0.96878713]]) ''' - return self._to_impl(device=device, - dtype=dtype, - blocking=blocking, - include_sublayers=True, - floating_only=False) + return self._to_impl( + device=device, + dtype=dtype, + blocking=blocking, + include_sublayers=True, + floating_only=False, + ) def _apply(self, func, device, dtype, blocking, include_sublayers=True): if include_sublayers: @@ -1616,8 +1734,9 @@ def _apply(self, func, device, dtype, blocking, include_sublayers=True): if param.grad is not None: with no_grad(): - grad_applied = func(param._grad_ivar(), device, dtype, - blocking) + grad_applied = func( + param._grad_ivar(), device, dtype, blocking + ) for key, buf in self._buffers.items(): if buf is not None: @@ -1641,12 +1760,14 @@ def _transform(self, t, device, dtype, blocking): # Note(zhangbo): Paddle GPU minimum memory allocation unit is 256 bytes, waiting_alloc_memory will comput ‘t’ occupied memory space. # Coefficient 1.2 is used to avoid OOM that may occur in this critical state when the memory is just enough. waiting_alloc_memory = ( - (np.prod(t.shape) * size_dtype) / 256 + 1) * 256 * 1.2 + ((np.prod(t.shape) * size_dtype) / 256 + 1) * 256 * 1.2 + ) gpu_memory_available = core.gpu_memory_available() if gpu_memory_available < waiting_alloc_memory: # Copy param / Tensor to cpu - t_used = t._copy_to(paddle.CPUPlace(), - blocking) # k-v type will error + t_used = t._copy_to( + paddle.CPUPlace(), blocking + ) # k-v type will error # Release mem of t t.value().get_tensor()._clear() else: @@ -1657,7 +1778,8 @@ def _transform(self, t, device, dtype, blocking): # 2. cast param / Tensor to dtype if dtype is not None and dtype != t_used.dtype: with paddle.fluid.framework._dygraph_place_guard( - place=t_used.place): + place=t_used.place + ): t_casted = t_used.cast(dtype=dtype) else: t_casted = t_used @@ -1675,12 +1797,14 @@ def _transform(self, t, device, dtype, blocking): return t - def _to_impl(self, - device=None, - dtype=None, - blocking=None, - include_sublayers=True, - floating_only=False): + def _to_impl( + self, + device=None, + dtype=None, + blocking=None, + include_sublayers=True, + floating_only=False, + ): ''' Cast the parameters and buffers of Layer by the give device, dtype and blocking. @@ -1709,20 +1833,28 @@ def _to_impl(self, if device is not None: if isinstance(device, str): device = paddle.device._convert_to_place(device) - elif isinstance(device, (core.CPUPlace, core.CUDAPlace, - core.CUDAPinnedPlace, core.XPUPlace)): + elif isinstance( + device, + ( + core.CPUPlace, + core.CUDAPlace, + core.CUDAPinnedPlace, + core.XPUPlace, + ), + ): pass else: raise ValueError( "device value error, must be str, paddle.CPUPlace(), paddle.CUDAPlace(), paddle.CUDAPinnedPlace() or paddle.XPUPlace(), but the type of device is " - + type(device).__name__) + + type(device).__name__ + ) if blocking is None: blocking = True else: assert isinstance( - blocking, - bool), "blocking value error, must be the True, False or None" + blocking, bool + ), "blocking value error, must be the True, False or None" def transform(t, device, dtype, blocking): if floating_only and (not paddle.is_floating_point(t)): diff --git a/python/paddle/fluid/dygraph/learning_rate_scheduler.py b/python/paddle/fluid/dygraph/learning_rate_scheduler.py index 0620425910e6b..65219a9b17663 100644 --- a/python/paddle/fluid/dygraph/learning_rate_scheduler.py +++ b/python/paddle/fluid/dygraph/learning_rate_scheduler.py @@ -20,9 +20,18 @@ from ..data_feeder import check_type __all__ = [ - 'NoamDecay', 'PiecewiseDecay', 'NaturalExpDecay', 'ExponentialDecay', - 'InverseTimeDecay', 'PolynomialDecay', 'CosineDecay', 'LinearLrWarmup', - 'ReduceLROnPlateau', 'StepDecay', 'MultiStepDecay', 'LambdaDecay' + 'NoamDecay', + 'PiecewiseDecay', + 'NaturalExpDecay', + 'ExponentialDecay', + 'InverseTimeDecay', + 'PolynomialDecay', + 'CosineDecay', + 'LinearLrWarmup', + 'ReduceLROnPlateau', + 'StepDecay', + 'MultiStepDecay', + 'LambdaDecay', ] @@ -57,12 +66,14 @@ def create_lr_var(self, lr): learning rate variable """ from .. import layers + lr = layers.create_global_var( name=unique_name.generate("learning_rate"), shape=[1], value=float(lr), dtype=self.dtype, - persistable=False) + persistable=False, + ) return lr # Note: If you want to change what optimizer.state_dict stores, just overwrite this functions, @@ -83,7 +94,8 @@ def state_dict(self): assert value.shape == [ 1 ], "shape of Variable in state_dict must be [1] {}".format( - value.shape) + value.shape + ) value = value.numpy()[0] state_dict[key] = value @@ -105,8 +117,10 @@ def set_state_dict(self, state_dict): self.__dict__[key] = state_dict[key] else: raise RuntimeError( - "Please check whether state_dict is correct for optimizer. Can't find [ {} ] in state_dict" - .format(key)) + "Please check whether state_dict is correct for optimizer. Can't find [ {} ] in state_dict".format( + key + ) + ) if len(state_dict) > len(self.keys): warnings.warn( "There are some unused values in state_dict. Maybe the optimizer have different 'LearningRateDecay' when invoking state_dict and set_dict" @@ -238,14 +252,16 @@ class NaturalExpDecay(LearningRateDecay): """ - def __init__(self, - learning_rate, - decay_steps, - decay_rate, - staircase=False, - begin=0, - step=1, - dtype='float32'): + def __init__( + self, + learning_rate, + decay_steps, + decay_rate, + staircase=False, + begin=0, + step=1, + dtype='float32', + ): super(NaturalExpDecay, self).__init__(begin, step, dtype) self.learning_rate = learning_rate self.decay_steps = decay_steps @@ -254,11 +270,13 @@ def __init__(self, def step(self): from .. import layers + div_res = self.create_lr_var(self.step_num / self.decay_steps) if self.staircase: div_res = layers.floor(div_res) decayed_lr = self.learning_rate * layers.exp( - -1 * self.decay_rate * div_res) + -1 * self.decay_rate * div_res + ) return decayed_lr @@ -312,22 +330,24 @@ class ExponentialDecay(LearningRateDecay): base_lr = 0.1 with fluid.dygraph.guard(): sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.dygraph.ExponentialDecay( - learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True)) + learning_rate=fluid.dygraph.ExponentialDecay( + learning_rate=base_lr, + decay_steps=10000, + decay_rate=0.5, + staircase=True)) """ - def __init__(self, - learning_rate, - decay_steps, - decay_rate, - staircase=False, - begin=0, - step=1, - dtype='float32'): + def __init__( + self, + learning_rate, + decay_steps, + decay_rate, + staircase=False, + begin=0, + step=1, + dtype='float32', + ): super(ExponentialDecay, self).__init__(begin, step, dtype) self.learning_rate = learning_rate self.decay_steps = decay_steps @@ -336,6 +356,7 @@ def __init__(self, def step(self): from .. import layers + div_res = self.create_lr_var(self.step_num / self.decay_steps) if self.staircase: div_res = layers.floor(div_res) @@ -389,23 +410,25 @@ class InverseTimeDecay(LearningRateDecay): with fluid.dygraph.guard(): emb = fluid.dygraph.Embedding([10, 10]) sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.dygraph.InverseTimeDecay( - learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True), + learning_rate=fluid.dygraph.InverseTimeDecay( + learning_rate=base_lr, + decay_steps=10000, + decay_rate=0.5, + staircase=True), parameter_list = emb.parameters()) """ - def __init__(self, - learning_rate, - decay_steps, - decay_rate, - staircase=False, - begin=0, - step=1, - dtype='float32'): + def __init__( + self, + learning_rate, + decay_steps, + decay_rate, + staircase=False, + begin=0, + step=1, + dtype='float32', + ): super(InverseTimeDecay, self).__init__(begin, step, dtype) self.learning_rate = learning_rate self.decay_steps = decay_steps @@ -414,6 +437,7 @@ def __init__(self, def step(self): from .. import layers + div_res = self.create_lr_var(self.step_num / self.decay_steps) if self.staircase: div_res = layers.floor(div_res) @@ -480,15 +504,17 @@ class PolynomialDecay(LearningRateDecay): """ - def __init__(self, - learning_rate, - decay_steps, - end_learning_rate=0.0001, - power=1.0, - cycle=False, - begin=0, - step=1, - dtype='float32'): + def __init__( + self, + learning_rate, + decay_steps, + end_learning_rate=0.0001, + power=1.0, + cycle=False, + begin=0, + step=1, + dtype='float32', + ): super(PolynomialDecay, self).__init__(begin, step, dtype) self.learning_rate = learning_rate self.decay_steps = decay_steps @@ -498,22 +524,27 @@ def __init__(self, def step(self): from .. import layers + tmp_step_num = self.step_num tmp_decay_steps = self.decay_steps if self.cycle: div_res = layers.ceil( - self.create_lr_var(tmp_step_num / float(self.decay_steps))) + self.create_lr_var(tmp_step_num / float(self.decay_steps)) + ) if tmp_step_num == 0: div_res = self.create_lr_var(1.0) tmp_decay_steps = self.decay_steps * div_res else: tmp_step_num = self.create_lr_var( - tmp_step_num if tmp_step_num < self.decay_steps else self. - decay_steps) + tmp_step_num + if tmp_step_num < self.decay_steps + else self.decay_steps + ) - decayed_lr = (self.learning_rate - self.end_learning_rate) * \ - ((1 - tmp_step_num / tmp_decay_steps) ** self.power) + self.end_learning_rate + decayed_lr = (self.learning_rate - self.end_learning_rate) * ( + (1 - tmp_step_num / tmp_decay_steps) ** self.power + ) + self.end_learning_rate return decayed_lr @@ -545,22 +576,24 @@ class CosineDecay(LearningRateDecay): None. Examples: - .. code-block:: python + .. code-block:: python - base_lr = 0.1 + base_lr = 0.1 with fluid.dygraph.guard(): optimizer = fluid.optimizer.SGD( - learning_rate = fluid.dygraph.CosineDecay( - base_lr, 10000, 120) ) + learning_rate = fluid.dygraph.CosineDecay( + base_lr, 10000, 120) ) """ - def __init__(self, - learning_rate, - step_each_epoch, - epochs, - begin=0, - step=1, - dtype='float32'): + def __init__( + self, + learning_rate, + step_each_epoch, + epochs, + begin=0, + step=1, + dtype='float32', + ): super(CosineDecay, self).__init__(begin, step, dtype) self.learning_rate = learning_rate self.step_each_epoch = step_each_epoch @@ -568,10 +601,15 @@ def __init__(self, def step(self): from .. import layers + cur_epoch = layers.floor( - self.create_lr_var(self.step_num / self.step_each_epoch)) - decayed_lr = self.learning_rate * 0.5 * ( - layers.cos(cur_epoch * math.pi / self.epochs) + 1) + self.create_lr_var(self.step_num / self.step_each_epoch) + ) + decayed_lr = ( + self.learning_rate + * 0.5 + * (layers.cos(cur_epoch * math.pi / self.epochs) + 1) + ) return decayed_lr @@ -621,13 +659,15 @@ class NoamDecay(LearningRateDecay): parameter_list = emb.parameters()) """ - def __init__(self, - d_model, - warmup_steps, - begin=1, - step=1, - dtype='float32', - learning_rate=1.0): + def __init__( + self, + d_model, + warmup_steps, + begin=1, + step=1, + dtype='float32', + learning_rate=1.0, + ): super(NoamDecay, self).__init__(begin, step, dtype) self.learning_rate = learning_rate self.d_model = d_model @@ -635,10 +675,14 @@ def __init__(self, def step(self): from .. import layers + a = self.create_lr_var(self.step_num**-0.5) b = self.create_lr_var((self.warmup_steps**-1.5) * self.step_num) - lr_value = self.learning_rate * (self.d_model** - -0.5) * layers.elementwise_min(a, b) + lr_value = ( + self.learning_rate + * (self.d_model**-0.5) + * layers.elementwise_min(a, b) + ) return lr_value @@ -698,28 +742,37 @@ class LinearLrWarmup(LearningRateDecay): """ - def __init__(self, - learning_rate, - warmup_steps, - start_lr, - end_lr, - begin=1, - step=1, - dtype='float32'): + def __init__( + self, + learning_rate, + warmup_steps, + start_lr, + end_lr, + begin=1, + step=1, + dtype='float32', + ): super(LinearLrWarmup, self).__init__(begin, step, dtype) - type_check = isinstance(learning_rate, float) or isinstance( - learning_rate, int) or isinstance(learning_rate, LearningRateDecay) + type_check = ( + isinstance(learning_rate, float) + or isinstance(learning_rate, int) + or isinstance(learning_rate, LearningRateDecay) + ) if not type_check: raise TypeError( - "the type of learning_rate should be [int, float or LearningRateDecay], the current type is {}" - .format(learning_rate)) + "the type of learning_rate should be [int, float or LearningRateDecay], the current type is {}".format( + learning_rate + ) + ) self.learning_rate = learning_rate self.warmup_steps = warmup_steps self.start_lr = start_lr - assert end_lr > start_lr, "end_lr {} must be greater than start_lr {}".format( - end_lr, start_lr) - self.lr_ratio_before_warmup = (float(end_lr) - - float(start_lr)) / float(warmup_steps) + assert ( + end_lr > start_lr + ), "end_lr {} must be greater than start_lr {}".format(end_lr, start_lr) + self.lr_ratio_before_warmup = (float(end_lr) - float(start_lr)) / float( + warmup_steps + ) def step(self): base_lr = self.learning_rate @@ -727,6 +780,7 @@ def step(self): base_lr = base_lr() from .. import layers + if self.step_num < self.warmup_steps: return self.lr_ratio_before_warmup * self.step_num + self.start_lr else: @@ -812,18 +866,20 @@ class ReduceLROnPlateau(LearningRateDecay): """ - def __init__(self, - learning_rate, - mode='min', - decay_rate=0.1, - patience=10, - verbose=False, - threshold=1e-4, - threshold_mode='rel', - cooldown=0, - min_lr=0, - eps=1e-8, - dtype='float32'): + def __init__( + self, + learning_rate, + mode='min', + decay_rate=0.1, + patience=10, + verbose=False, + threshold=1e-4, + threshold_mode='rel', + cooldown=0, + min_lr=0, + eps=1e-8, + dtype='float32', + ): super(ReduceLROnPlateau, self).__init__(dtype=dtype) mode = mode.lower() if mode not in ['min', 'max']: @@ -838,15 +894,21 @@ def __init__(self, threshold_mode = threshold_mode.lower() if threshold_mode not in ['rel', 'abs']: - raise ValueError('threshold mode ' + threshold_mode + - ' is unknown!') + raise ValueError( + 'threshold mode ' + threshold_mode + ' is unknown!' + ) self.threshold_mode = threshold_mode - check_type(learning_rate, 'learning_rate', (float, int, Variable), - 'ReduceLROnPlateau') + check_type( + learning_rate, + 'learning_rate', + (float, int, Variable), + 'ReduceLROnPlateau', + ) if not isinstance(learning_rate, (float, int, Variable)): raise TypeError( "The type of 'learning_rate' in 'ReduceLROnPlateau' must be 'float, int, Variable', but received %s." - % type(learning_rate)) + % type(learning_rate) + ) self.learning_rate = learning_rate self.verbose = verbose @@ -865,8 +927,11 @@ def __init__(self, # "cooldown_counter / best_loss / num_bad_epochs / epoch_num / learning_rate" will be stored. def _state_keys(self): self.keys = [ - 'cooldown_counter', 'best_loss', 'num_bad_epochs', 'epoch_num', - 'learning_rate' + 'cooldown_counter', + 'best_loss', + 'num_bad_epochs', + 'epoch_num', + 'learning_rate', ] def __call__(self): @@ -893,9 +958,13 @@ def step(self, loss): # loss must be 1-D Tensor with shape [1] check_type(loss, 'loss', Variable, 'ReduceLROnPlateau.step') - assert len(loss.shape) == 1 and loss.shape[0] == 1, "the loss.shape " \ - "should be (1L,), but the current loss.shape is {}. Maybe that " \ - "you should call paddle.mean to process it first.".format(loss.shape) + assert len(loss.shape) == 1 and loss.shape[0] == 1, ( + "the loss.shape " + "should be (1L,), but the current loss.shape is {}. Maybe that " + "you should call paddle.mean to process it first.".format( + loss.shape + ) + ) self.epoch_num += 1 if self.cooldown_counter > 0: @@ -909,18 +978,24 @@ def step(self, loss): if self.num_bad_epochs > self.patience: from .. import layers + self.cooldown_counter = self.cooldown self.num_bad_epochs = 0 new_lr = layers.elementwise_max( - self.learning_rate * self.decay_rate, self.min_lr) + self.learning_rate * self.decay_rate, self.min_lr + ) if self.learning_rate - new_lr > self.eps: if self.verbose: - old_lr = self.learning_rate.numpy()[0] if isinstance( - self.learning_rate, - Variable) else self.learning_rate - print('Epoch {}: reducing learning rate from {} to {}.'. - format(self.epoch_num, old_lr, - new_lr.numpy()[0])) + old_lr = ( + self.learning_rate.numpy()[0] + if isinstance(self.learning_rate, Variable) + else self.learning_rate + ) + print( + 'Epoch {}: reducing learning rate from {} to {}.'.format( + self.epoch_num, old_lr, new_lr.numpy()[0] + ) + ) self.learning_rate = new_lr def _is_better(self, current, best): @@ -952,7 +1027,8 @@ def __init__(self, learning_rate, dtype=None): if not isinstance(learning_rate, (float, int)): raise TypeError( "The type of 'learning_rate' must be 'float, int', but received %s." - % type(learning_rate)) + % type(learning_rate) + ) if learning_rate < 0: raise ValueError("Invalid learning rate: {}".format(learning_rate)) @@ -1057,8 +1133,9 @@ class StepDecay(_LearningRateEpochDecay): def __init__(self, learning_rate, step_size, decay_rate=0.1): if not isinstance(step_size, int): raise TypeError( - "The type of 'step_size' must be 'int', but received %s." % - type(step_size)) + "The type of 'step_size' must be 'int', but received %s." + % type(step_size) + ) if decay_rate >= 1.0: raise ValueError('decay_rate should be < 1.0.') @@ -1134,12 +1211,15 @@ def __init__(self, learning_rate, milestones, decay_rate=0.1): if not isinstance(milestones, (tuple, list)): raise TypeError( "The type of 'milestones' in 'MultiStepDecay' must be 'tuple, list', but received %s." - % type(milestones)) + % type(milestones) + ) - if not all([ + if not all( + [ milestones[i] < milestones[i + 1] for i in range(len(milestones) - 1) - ]): + ] + ): raise ValueError('The elements of milestones must be incremented') if decay_rate >= 1.0: raise ValueError('decay_rate should be < 1.0.') @@ -1154,7 +1234,7 @@ def get_lr(self): if self.epoch_num < self.milestones[i]: return self.base_lr * (decay_rate**i) - return self.base_lr * (decay_rate**len(self.milestones)) + return self.base_lr * (decay_rate ** len(self.milestones)) class LambdaDecay(_LearningRateEpochDecay): @@ -1213,7 +1293,8 @@ def __init__(self, learning_rate, lr_lambda): if not callable(lr_lambda): raise TypeError( "The type of 'lr_lambda' in 'LambdaDecay' must be 'function', but received %s." - % type(lr_lambda)) + % type(lr_lambda) + ) self.lr_lambda = lr_lambda super(LambdaDecay, self).__init__(learning_rate) diff --git a/python/paddle/fluid/dygraph/math_op_patch.py b/python/paddle/fluid/dygraph/math_op_patch.py index ac894a3acc7f2..de71f68c1fed9 100644 --- a/python/paddle/fluid/dygraph/math_op_patch.py +++ b/python/paddle/fluid/dygraph/math_op_patch.py @@ -13,7 +13,13 @@ # limitations under the License. from .. import core -from ..framework import Variable, convert_np_dtype_to_dtype_, _varbase_creator, _in_legacy_dygraph, in_dygraph_mode +from ..framework import ( + Variable, + convert_np_dtype_to_dtype_, + _varbase_creator, + _in_legacy_dygraph, + in_dygraph_mode, +) from ..layers.layer_function_generator import OpProtoHolder from . import no_grad from .. import framework @@ -70,13 +76,22 @@ def monkey_patch_math_varbase(): @no_grad def create_tensor(value, dtype, shape): if framework._in_eager_mode_: - out = _C_ops.full(shape, value, dtype, - framework._current_expected_place()) + out = _C_ops.full( + shape, value, dtype, framework._current_expected_place() + ) else: out = _varbase_creator(dtype=dtype) - out = _legacy_C_ops.fill_constant(out, 'dtype', dtype, 'shape', - shape, 'value', value, - 'force_cpu', False) + out = _legacy_C_ops.fill_constant( + out, + 'dtype', + dtype, + 'shape', + shape, + 'value', + value, + 'force_cpu', + False, + ) out.stop_gradient = True return out @@ -110,8 +125,9 @@ def astype(self, dtype): dtype = convert_np_dtype_to_dtype_(dtype) if _in_legacy_dygraph(): - return _legacy_C_ops.cast(self, 'in_dtype', self.dtype, 'out_dtype', - dtype) + return _legacy_C_ops.cast( + self, 'in_dtype', self.dtype, 'out_dtype', dtype + ) return _C_ops.cast(self, dtype) def _scalar_elementwise_op_(var, scale, bias): @@ -124,7 +140,9 @@ def _neg_(var): def _float_(var): numel = np.prod(var.shape) - assert numel == 1, "only one element variable can be converted to float." + assert ( + numel == 1 + ), "only one element variable can be converted to float." tensor = var.value().get_tensor() assert tensor._is_initialized(), "variable's tensor is not initialized" return float(var.numpy().flatten()[0]) @@ -154,7 +172,9 @@ def _len_(var): def _index_(var): numel = np.prod(var.shape) - assert numel == 1, "only one element variable can be converted to python index." + assert ( + numel == 1 + ), "only one element variable can be converted to python index." tensor = var.value().get_tensor() assert tensor._is_initialized(), "variable's tensor is not initialized" return int(var.numpy().flatten()[0]) @@ -196,12 +216,13 @@ def _scalar_div_(var, value): return _scalar_elementwise_op_(var, 1.0 / value, 0.0) # for binary operator such as elementwise, compare - def _binary_creator_(method_name, - op_type, - reverse=False, - scalar_method=None, - call_final_api=False): - + def _binary_creator_( + method_name, + op_type, + reverse=False, + scalar_method=None, + call_final_api=False, + ): def __impl__(self, other_var): # 1. scalar exists cases # we need combine the tensor.dtype and scalar.dtype, cast correct object @@ -224,8 +245,9 @@ def __impl__(self, other_var): # so the calculation result here and the calculation result of numpy are # different after 6 decimal point. If necessary, we can also use float64 here. # torch's behavior here is consistent with ours - if (op_type == "divide" or op_type == "elementwise_div" - ) and self.dtype in _supported_int_dtype_: + if ( + op_type == "divide" or op_type == "elementwise_div" + ) and self.dtype in _supported_int_dtype_: self = astype(self, 'float32') # here use `scale` replace `elementwise` to get better performance # but only +, -, *, / can use this method @@ -244,36 +266,47 @@ def __impl__(self, other_var): if not isinstance(other_var, other_var_should_be): if isinstance(other_var, complex): import paddle + other_var = paddle.to_tensor(other_var, dtype='complex64') else: if reverse: - other_var = create_tensor(other_var, - dtype=lhs_dtype, - shape=self.shape) + other_var = create_tensor( + other_var, dtype=lhs_dtype, shape=self.shape + ) else: # add fill_op - other_var = create_scalar(value=other_var, - dtype=lhs_dtype) + other_var = create_scalar( + value=other_var, dtype=lhs_dtype + ) # 3. promote types or unify right var type to left var rhs_dtype = other_var.dtype if lhs_dtype != rhs_dtype: if method_name in _supported_promote_complex_types_ and ( - lhs_dtype in _complex_dtypes - or rhs_dtype in _complex_dtypes): + lhs_dtype in _complex_dtypes or rhs_dtype in _complex_dtypes + ): # only when lhs_dtype or rhs_dtype is complex type, # the dtype will promote, in other cases, directly # use lhs_dtype, this is consistent will original rule promote_dtype = core._promote_types_if_complex_exists( - lhs_dtype, rhs_dtype) - self = self if lhs_dtype == promote_dtype else astype( - self, promote_dtype) - other_var = other_var if rhs_dtype == promote_dtype else astype( - other_var, promote_dtype) + lhs_dtype, rhs_dtype + ) + self = ( + self + if lhs_dtype == promote_dtype + else astype(self, promote_dtype) + ) + other_var = ( + other_var + if rhs_dtype == promote_dtype + else astype(other_var, promote_dtype) + ) else: warnings.warn( - 'The dtype of left and right variables are not the same, left dtype is {}, but right dtype is {}, the right dtype will convert to {}' - .format(lhs_dtype, rhs_dtype, lhs_dtype)) + 'The dtype of left and right variables are not the same, left dtype is {}, but right dtype is {}, the right dtype will convert to {}'.format( + lhs_dtype, rhs_dtype, lhs_dtype + ) + ) other_var = astype(other_var, lhs_dtype) if reverse: @@ -281,8 +314,9 @@ def __impl__(self, other_var): self = other_var other_var = tmp - if (op_type == "divide" or op_type == "elementwise_div" - ) and self.dtype in _supported_int_dtype_: + if ( + op_type == "divide" or op_type == "elementwise_div" + ) and self.dtype in _supported_int_dtype_: self = astype(self, 'float32') other_var = astype(other_var, 'float32') @@ -315,7 +349,9 @@ def __impl__(self, other_var): Returns: Tensor - """.format(comment) + """.format( + comment + ) __impl__.__name__ = method_name return __impl__ @@ -332,39 +368,78 @@ def __impl__(self, other_var): ('ndim', _ndim_), ('size', _size_), ('T', _T_), - ('__add__', - _binary_creator_('__add__', 'elementwise_add', False, _scalar_add_)), + ( + '__add__', + _binary_creator_('__add__', 'elementwise_add', False, _scalar_add_), + ), # a+b == b+a. Do not need to reverse explicitly - ('__radd__', - _binary_creator_('__radd__', 'elementwise_add', False, _scalar_add_)), - ('__sub__', - _binary_creator_('__sub__', 'elementwise_sub', False, _scalar_sub_)), - ('__rsub__', - _binary_creator_('__rsub__', 'elementwise_sub', True, _scalar_rsub_)), - ('__mul__', - _binary_creator_('__mul__', 'elementwise_mul', False, _scalar_mul_)), + ( + '__radd__', + _binary_creator_( + '__radd__', 'elementwise_add', False, _scalar_add_ + ), + ), + ( + '__sub__', + _binary_creator_('__sub__', 'elementwise_sub', False, _scalar_sub_), + ), + ( + '__rsub__', + _binary_creator_( + '__rsub__', 'elementwise_sub', True, _scalar_rsub_ + ), + ), + ( + '__mul__', + _binary_creator_('__mul__', 'elementwise_mul', False, _scalar_mul_), + ), ## a*b == b*a. Do not need to reverse explicitly - ('__rmul__', - _binary_creator_('__rmul__', 'elementwise_mul', False, _scalar_mul_)), - ('__div__', - _binary_creator_('__div__', 'elementwise_div', False, _scalar_div_)), - ('__truediv__', - _binary_creator_('__truediv__', 'elementwise_div', False, - _scalar_div_)), - ('__rdiv__', _binary_creator_('__rdiv__', 'elementwise_div', True, - None)), - ('__rtruediv__', - _binary_creator_('rtruediv__', 'elementwise_div', True, None)), - ('__pow__', _binary_creator_('__pow__', 'elementwise_pow', False, - None)), - ('__rpow__', _binary_creator_('__rpow__', 'elementwise_pow', True, - None)), - ('__floordiv__', - _binary_creator_('__floordiv__', 'elementwise_floordiv', False, None)), - ('__mod__', _binary_creator_('__mod__', 'elementwise_mod', False, - None)), - ('__matmul__', _binary_creator_('__matmul__', "matmul_v2", False, - None)), + ( + '__rmul__', + _binary_creator_( + '__rmul__', 'elementwise_mul', False, _scalar_mul_ + ), + ), + ( + '__div__', + _binary_creator_('__div__', 'elementwise_div', False, _scalar_div_), + ), + ( + '__truediv__', + _binary_creator_( + '__truediv__', 'elementwise_div', False, _scalar_div_ + ), + ), + ( + '__rdiv__', + _binary_creator_('__rdiv__', 'elementwise_div', True, None), + ), + ( + '__rtruediv__', + _binary_creator_('rtruediv__', 'elementwise_div', True, None), + ), + ( + '__pow__', + _binary_creator_('__pow__', 'elementwise_pow', False, None), + ), + ( + '__rpow__', + _binary_creator_('__rpow__', 'elementwise_pow', True, None), + ), + ( + '__floordiv__', + _binary_creator_( + '__floordiv__', 'elementwise_floordiv', False, None + ), + ), + ( + '__mod__', + _binary_creator_('__mod__', 'elementwise_mod', False, None), + ), + ( + '__matmul__', + _binary_creator_('__matmul__', "matmul_v2", False, None), + ), ## for logical compare ('__eq__', _binary_creator_('__eq__', 'equal', False, None)), ('__ne__', _binary_creator_('__ne__', 'not_equal', False, None)), @@ -372,7 +447,7 @@ def __impl__(self, other_var): ('__le__', _binary_creator_('__le__', 'less_equal', False, None)), ('__gt__', _binary_creator_('__gt__', 'greater_than', False, None)), ('__ge__', _binary_creator_('__ge__', 'greater_equal', False, None)), - ('__array_ufunc__', None) + ('__array_ufunc__', None), ] eager_methods = [ @@ -390,7 +465,7 @@ def __impl__(self, other_var): ('T', _T_), # for logical compare ('__eq__', _binary_creator_('__eq__', 'equal', False, None, True)), - ('__array_ufunc__', None) + ('__array_ufunc__', None), ] eager_cpp_level_patch = [ @@ -447,12 +522,16 @@ def __impl__(self, other_var): setattr(local_tensor, method_name, method_impl) else: import paddle.tensor + # Tensor method from module paddle.tensor for method_name in paddle.tensor.tensor_method_func: - if hasattr(local_tensor, method_name): continue + if hasattr(local_tensor, method_name): + continue method_impl = getattr(paddle.tensor, method_name, None) - if method_impl: setattr(local_tensor, method_name, method_impl) + if method_impl: + setattr(local_tensor, method_name, method_impl) for magic_method, origin_method in paddle.tensor.magic_method_func: impl = getattr(paddle.tensor, origin_method, None) - if impl: setattr(local_tensor, magic_method, impl) + if impl: + setattr(local_tensor, magic_method, impl) diff --git a/python/paddle/fluid/dygraph/nn.py b/python/paddle/fluid/dygraph/nn.py index 27ac082da79f5..60202c2a6b105 100644 --- a/python/paddle/fluid/dygraph/nn.py +++ b/python/paddle/fluid/dygraph/nn.py @@ -18,8 +18,24 @@ from ..layers import nn as F from .. import dygraph_utils from . import layers -from ..framework import Variable, _non_static_mode, OpProtoHolder, Parameter, _dygraph_tracer, _varbase_creator, default_main_program, _global_flags, in_dygraph_mode, _in_legacy_dygraph -from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..framework import ( + Variable, + _non_static_mode, + OpProtoHolder, + Parameter, + _dygraph_tracer, + _varbase_creator, + default_main_program, + _global_flags, + in_dygraph_mode, + _in_legacy_dygraph, +) +from ..data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) from ..param_attr import ParamAttr from ..initializer import Normal, Constant, NumpyArrayInitializer from .. import unique_name @@ -33,10 +49,25 @@ from paddle import _C_ops, _legacy_C_ops __all__ = [ - 'Conv2D', 'Conv3D', 'Pool2D', 'Linear', 'BatchNorm', 'Dropout', 'Embedding', - 'GRUUnit', 'InstanceNorm', 'LayerNorm', 'NCE', 'PRelu', - 'BilinearTensorProduct', 'Conv2DTranspose', 'Conv3DTranspose', 'GroupNorm', - 'SpectralNorm', 'TreeConv', 'Flatten' + 'Conv2D', + 'Conv3D', + 'Pool2D', + 'Linear', + 'BatchNorm', + 'Dropout', + 'Embedding', + 'GRUUnit', + 'InstanceNorm', + 'LayerNorm', + 'NCE', + 'PRelu', + 'BilinearTensorProduct', + 'Conv2DTranspose', + 'Conv3DTranspose', + 'GroupNorm', + 'SpectralNorm', + 'TreeConv', + 'Flatten', ] @@ -157,24 +188,30 @@ class Conv2D(layers.Layer): """ - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - padding=0, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - dtype='float32'): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + padding=0, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + dtype='float32', + ): assert param_attr is not False, "param_attr should not be False here." super(Conv2D, self).__init__() - if (core.is_compiled_with_cuda() and paddle.fluid.get_flags( - "FLAGS_conv2d_disable_cudnn")["FLAGS_conv2d_disable_cudnn"]): + if ( + core.is_compiled_with_cuda() + and paddle.fluid.get_flags("FLAGS_conv2d_disable_cudnn")[ + "FLAGS_conv2d_disable_cudnn" + ] + ): use_cudnn = False self._num_channels = num_channels @@ -193,17 +230,22 @@ def __init__(self, self._bias_attr = bias_attr self._dtype = dtype - if (self._num_channels == self._groups - and num_filters % self._num_channels == 0 - and not self._use_cudnn and not self._use_mkldnn): + if ( + self._num_channels == self._groups + and num_filters % self._num_channels == 0 + and not self._use_cudnn + and not self._use_mkldnn + ): self._l_type = 'depthwise_conv2d' else: self._l_type = 'conv2d' # NPU only supports depthwise_conv2d when "input_channel = output_channel = groups" if core.is_compiled_with_npu(): - if (self._num_channels == self._groups - and self._num_channels == self._num_filters): + if ( + self._num_channels == self._groups + and self._num_channels == self._num_filters + ): self._l_type = 'depthwise_conv2d' else: self._l_type = 'conv2d' @@ -219,48 +261,75 @@ def __init__(self, filter_shape = [self._num_filters, num_filter_channels] + filter_size def _get_default_param_initializer(): - filter_elem_num = filter_size[0] * filter_size[ - 1] * self._num_channels - std = (2.0 / filter_elem_num)**0.5 + filter_elem_num = ( + filter_size[0] * filter_size[1] * self._num_channels + ) + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std, 0) self.weight = self.create_parameter( attr=self._param_attr, shape=filter_shape, dtype=self._dtype, - default_initializer=_get_default_param_initializer()) + default_initializer=_get_default_param_initializer(), + ) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._num_filters], - dtype=self._dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[self._num_filters], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input): if in_dygraph_mode() and self._l_type == "conv2d": - pre_bias = _C_ops.conv2d(input, self.weight, self._stride, - self._padding, "EXPLICIT", - self._groups if self._groups else 1, - self._dilation, "NCHW", False, -1, False) + pre_bias = _C_ops.conv2d( + input, + self.weight, + self._stride, + self._padding, + "EXPLICIT", + self._groups if self._groups else 1, + self._dilation, + "NCHW", + False, + -1, + False, + ) if self.bias is not None: pre_act = F.elementwise_add(pre_bias, self.bias, axis=1) else: pre_act = pre_bias return dygraph_utils._append_activation_in_dygraph( - pre_act, self._act, use_mkldnn=self._use_mkldnn) - - if _non_static_mode() and (self._l_type == 'conv2d' - or self._l_type == 'depthwise_conv2d'): - attrs = ('strides', self._stride, 'paddings', self._padding, - 'dilations', self._dilation, 'groups', - self._groups if self._groups else 1, 'use_cudnn', - self._use_cudnn, 'use_mkldnn', self._use_mkldnn) + pre_act, self._act, use_mkldnn=self._use_mkldnn + ) + + if _non_static_mode() and ( + self._l_type == 'conv2d' or self._l_type == 'depthwise_conv2d' + ): + attrs = ( + 'strides', + self._stride, + 'paddings', + self._padding, + 'dilations', + self._dilation, + 'groups', + self._groups if self._groups else 1, + 'use_cudnn', + self._use_cudnn, + 'use_mkldnn', + self._use_mkldnn, + ) out = _legacy_C_ops.conv2d(input, self.weight, *attrs) pre_bias = out pre_act = dygraph_utils._append_bias_in_dygraph( - pre_bias, self.bias, 1, use_mkldnn=self._use_mkldnn) + pre_bias, self.bias, 1, use_mkldnn=self._use_mkldnn + ) return dygraph_utils._append_activation_in_dygraph( - pre_act, self._act, use_mkldnn=self._use_mkldnn) + pre_act, self._act, use_mkldnn=self._use_mkldnn + ) inputs = { 'Input': [input], 'Filter': [self.weight], @@ -274,32 +343,33 @@ def forward(self, input): 'use_mkldnn': self._use_mkldnn, } - check_variable_and_dtype(input, 'input', - ['float16', 'float32', 'float64'], 'Conv2D') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'Conv2D' + ) pre_bias = self._helper.create_variable_for_type_inference( - dtype=self._dtype) + dtype=self._dtype + ) - self._helper.append_op(type=self._l_type, - inputs={ - 'Input': input, - 'Filter': self.weight, - }, - outputs={"Output": pre_bias}, - attrs=attrs) + self._helper.append_op( + type=self._l_type, + inputs={ + 'Input': input, + 'Filter': self.weight, + }, + outputs={"Output": pre_bias}, + attrs=attrs, + ) if self.bias is not None: pre_act = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [self.bias] - }, - outputs={'Out': [pre_act]}, - attrs={ - 'axis': 1, - 'use_mkldnn': self._use_mkldnn - }) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [self.bias]}, + outputs={'Out': [pre_act]}, + attrs={'axis': 1, 'use_mkldnn': self._use_mkldnn}, + ) else: pre_act = pre_bias @@ -418,19 +488,21 @@ class Conv3D(layers.Layer): """ - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - padding=0, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - dtype='float32'): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + padding=0, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + dtype='float32', + ): assert param_attr is not False, "param_attr should not be False here." super(Conv3D, self).__init__() self._num_channels = num_channels @@ -457,52 +529,61 @@ def __init__(self, filter_shape = [self._num_filters, num_filter_channels] + filter_size def _get_default_param_initializer(): - filter_elem_num = filter_size[0] * filter_size[1] * filter_size[ - 2] * self._num_channels - std = (2.0 / filter_elem_num)**0.5 + filter_elem_num = ( + filter_size[0] + * filter_size[1] + * filter_size[2] + * self._num_channels + ) + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std, 0) self.weight = self.create_parameter( attr=self._param_attr, shape=filter_shape, dtype=self._dtype, - default_initializer=_get_default_param_initializer()) + default_initializer=_get_default_param_initializer(), + ) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._num_filters], - dtype=self._dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[self._num_filters], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input): pre_bias = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - - self._helper.append_op(type='conv3d', - inputs={ - 'Input': input, - 'Filter': self.weight, - }, - outputs={"Output": pre_bias}, - attrs={ - 'strides': self._stride, - 'paddings': self._padding, - 'dilations': self._dilation, - 'groups': - self._groups if self._groups else 1, - 'use_cudnn': self._use_cudnn, - 'use_mkldnn': False - }) + dtype=self._dtype + ) + + self._helper.append_op( + type='conv3d', + inputs={ + 'Input': input, + 'Filter': self.weight, + }, + outputs={"Output": pre_bias}, + attrs={ + 'strides': self._stride, + 'paddings': self._padding, + 'dilations': self._dilation, + 'groups': self._groups if self._groups else 1, + 'use_cudnn': self._use_cudnn, + 'use_mkldnn': False, + }, + ) if self.bias is not None: pre_act = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [self.bias] - }, - outputs={'Out': [pre_act]}, - attrs={'axis': 1}) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [self.bias]}, + outputs={'Out': [pre_act]}, + attrs={'axis': 1}, + ) else: pre_act = pre_bias @@ -651,23 +732,27 @@ class Conv3DTranspose(layers.Layer): """ - def __init__(self, - num_channels, - num_filters, - filter_size, - padding=0, - stride=1, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - dtype='float32'): + def __init__( + self, + num_channels, + num_filters, + filter_size, + padding=0, + stride=1, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + dtype='float32', + ): super(Conv3DTranspose, self).__init__() if not isinstance(use_cudnn, bool): raise ValueError("use_cudnn should be True or False") - assert param_attr is not False, "param_attr should not be False in conv3d_transpose." + assert ( + param_attr is not False + ), "param_attr should not be False in conv3d_transpose." self._padding = utils.convert_to_list(padding, 3, 'padding') self._stride = utils.convert_to_list(stride, 3, 'stride') self._dilation = utils.convert_to_list(dilation, 3, 'dilation') @@ -682,46 +767,50 @@ def __init__(self, self._dtype = dtype self._filter_size = utils.convert_to_list( - self._filter_size, 3, 'conv3d_transpose.filter_size') - - filter_shape = [self._num_channels, self._num_filters // self._groups - ] + self._filter_size - self.weight = self.create_parameter(dtype=self._dtype, - shape=filter_shape, - attr=self._param_attr) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._num_filters], - dtype=self._dtype, - is_bias=True) + self._filter_size, 3, 'conv3d_transpose.filter_size' + ) + + filter_shape = [ + self._num_channels, + self._num_filters // self._groups, + ] + self._filter_size + self.weight = self.create_parameter( + dtype=self._dtype, shape=filter_shape, attr=self._param_attr + ) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[self._num_filters], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input): pre_bias = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type="conv3d_transpose", - inputs={ - 'Input': [input], - 'Filter': [self.weight] - }, - outputs={'Output': pre_bias}, - attrs={ - 'strides': self._stride, - 'paddings': self._padding, - 'dilations': self._dilation, - 'groups': - self._groups if self._groups else 1, - 'use_cudnn': self._use_cudnn - }) + dtype=self._dtype + ) + self._helper.append_op( + type="conv3d_transpose", + inputs={'Input': [input], 'Filter': [self.weight]}, + outputs={'Output': pre_bias}, + attrs={ + 'strides': self._stride, + 'paddings': self._padding, + 'dilations': self._dilation, + 'groups': self._groups if self._groups else 1, + 'use_cudnn': self._use_cudnn, + }, + ) if self._bias_attr: pre_act = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [self.bias] - }, - outputs={'Out': [pre_act]}, - attrs={'axis': 1}) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [self.bias]}, + outputs={'Out': [pre_act]}, + attrs={'axis': 1}, + ) else: pre_act = pre_bias @@ -835,27 +924,31 @@ class Pool2D(layers.Layer): """ - def __init__(self, - pool_size=-1, - pool_type="max", - pool_stride=1, - pool_padding=0, - global_pooling=False, - use_cudnn=True, - ceil_mode=False, - exclusive=True, - data_format="NCHW"): + def __init__( + self, + pool_size=-1, + pool_type="max", + pool_stride=1, + pool_padding=0, + global_pooling=False, + use_cudnn=True, + ceil_mode=False, + exclusive=True, + data_format="NCHW", + ): data_format = data_format.upper() # supprt NHWC, nhwc, etc. pool_type = pool_type.lower() # supprt max, Max, etc. if pool_type not in ["max", "avg"]: raise ValueError( "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.", - str(pool_type)) + str(pool_type), + ) if global_pooling is False and pool_size == -1: raise ValueError( "When the global_pooling is False, pool_size must be passed " - "and be a valid value. Received pool_size: " + str(pool_size)) + "and be a valid value. Received pool_size: " + str(pool_size) + ) if not isinstance(use_cudnn, bool): raise ValueError("use_cudnn should be True or False") @@ -865,14 +958,16 @@ def __init__(self, if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) super(Pool2D, self).__init__() self._pool_type = pool_type self._pool_size = utils.convert_to_list(pool_size, 2, 'pool_size') - self._pool_padding = utils.convert_to_list(pool_padding, 2, - 'pool_padding') + self._pool_padding = utils.convert_to_list( + pool_padding, 2, 'pool_padding' + ) self._pool_stride = utils.convert_to_list(pool_stride, 2, 'pool_stride') self._global_pooling = global_pooling self._use_cudnn = use_cudnn @@ -884,23 +979,51 @@ def __init__(self, def forward(self, input): if _non_static_mode(): if not self._use_mkldnn and in_dygraph_mode(): - return _C_ops.pool2d(input, self._pool_size, self._pool_stride, - self._pool_padding, self._ceil_mode, - self._exclusive, self._data_format, - self._pool_type, self._global_pooling, - False, "EXPLICIT", self._use_cudnn) - - attrs = ('pooling_type', self._pool_type, 'ksize', self._pool_size, - 'global_pooling', self._global_pooling, 'strides', - self._pool_stride, 'paddings', self._pool_padding, - 'use_cudnn', self._use_cudnn, 'ceil_mode', self._ceil_mode, - 'use_mkldnn', self._use_mkldnn, 'exclusive', - self._exclusive, 'data_format', self._data_format) + return _C_ops.pool2d( + input, + self._pool_size, + self._pool_stride, + self._pool_padding, + self._ceil_mode, + self._exclusive, + self._data_format, + self._pool_type, + self._global_pooling, + False, + "EXPLICIT", + self._use_cudnn, + ) + + attrs = ( + 'pooling_type', + self._pool_type, + 'ksize', + self._pool_size, + 'global_pooling', + self._global_pooling, + 'strides', + self._pool_stride, + 'paddings', + self._pool_padding, + 'use_cudnn', + self._use_cudnn, + 'ceil_mode', + self._ceil_mode, + 'use_mkldnn', + self._use_mkldnn, + 'exclusive', + self._exclusive, + 'data_format', + self._data_format, + ) return _legacy_C_ops.pool2d(input, *attrs) check_variable_and_dtype( - input, 'input', ['int8', 'uint8', 'float16', 'float32', 'float64'], - 'Pool2D') + input, + 'input', + ['int8', 'uint8', 'float16', 'float32', 'float64'], + 'Pool2D', + ) attrs = { "pooling_type": self._pool_type, @@ -918,10 +1041,12 @@ def forward(self, input): pool_out = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type=self._l_type, - inputs={"X": input}, - outputs={"Out": pool_out}, - attrs=attrs) + self._helper.append_op( + type=self._l_type, + inputs={"X": input}, + outputs={"Out": pool_out}, + attrs=attrs, + ) return pool_out @@ -977,44 +1102,60 @@ class Linear(layers.Layer): res = linear(data) # [30, 10, 64] """ - def __init__(self, - input_dim, - output_dim, - param_attr=None, - bias_attr=None, - act=None, - dtype="float32"): + def __init__( + self, + input_dim, + output_dim, + param_attr=None, + bias_attr=None, + act=None, + dtype="float32", + ): super(Linear, self).__init__() self._act = act self._dtype = dtype - self.weight = self.create_parameter(shape=[input_dim, output_dim], - attr=param_attr, - dtype=dtype, - is_bias=False) - self.bias = self.create_parameter(shape=[output_dim], - attr=bias_attr, - dtype=dtype, - is_bias=True) + self.weight = self.create_parameter( + shape=[input_dim, output_dim], + attr=param_attr, + dtype=dtype, + is_bias=False, + ) + self.bias = self.create_parameter( + shape=[output_dim], attr=bias_attr, dtype=dtype, is_bias=True + ) self._use_mkldnn = _global_flags()["FLAGS_use_mkldnn"] def forward(self, input): if _non_static_mode(): pre_bias = _varbase_creator(dtype=input.dtype) - _legacy_C_ops.matmul(input, self.weight, pre_bias, 'transpose_X', - False, 'transpose_Y', False, "alpha", 1, - "use_mkldnn", self._use_mkldnn) + _legacy_C_ops.matmul( + input, + self.weight, + pre_bias, + 'transpose_X', + False, + 'transpose_Y', + False, + "alpha", + 1, + "use_mkldnn", + self._use_mkldnn, + ) pre_act = dygraph_utils._append_bias_in_dygraph( pre_bias, self.bias, axis=len(input.shape) - 1, - use_mkldnn=self._use_mkldnn) + use_mkldnn=self._use_mkldnn, + ) return dygraph_utils._append_activation_in_dygraph( - pre_act, self._act, use_mkldnn=self._use_mkldnn) + pre_act, self._act, use_mkldnn=self._use_mkldnn + ) - check_variable_and_dtype(input, 'input', - ['float16', 'float32', 'float64'], "Linear") + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], "Linear" + ) attrs = { "transpose_X": False, @@ -1025,23 +1166,22 @@ def forward(self, input): inputs = {"X": [input], "Y": [self.weight]} tmp = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type="matmul", - inputs=inputs, - outputs={"Out": tmp}, - attrs=attrs) + self._helper.append_op( + type="matmul", inputs=inputs, outputs={"Out": tmp}, attrs=attrs + ) if self.bias is not None: pre_activation = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [tmp], - 'Y': [self.bias] - }, - outputs={'Out': [pre_activation]}, - attrs={ - 'axis': len(input.shape) - 1, - 'use_mkldnn': self._use_mkldnn - }) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [tmp], 'Y': [self.bias]}, + outputs={'Out': [pre_activation]}, + attrs={ + 'axis': len(input.shape) - 1, + 'use_mkldnn': self._use_mkldnn, + }, + ) else: pre_activation = tmp return self._helper.append_activation(pre_activation, act=self._act) @@ -1115,16 +1255,20 @@ class InstanceNorm(layers.Layer): """ - def __init__(self, - num_channels, - epsilon=1e-5, - param_attr=None, - bias_attr=None, - dtype='float32'): + def __init__( + self, + num_channels, + epsilon=1e-5, + param_attr=None, + bias_attr=None, + dtype='float32', + ): super(InstanceNorm, self).__init__() if param_attr == False or bias_attr == False: - assert bias_attr == param_attr, "param_attr and bias_attr must be set to Fasle at the same time in InstanceNorm" + assert ( + bias_attr == param_attr + ), "param_attr and bias_attr must be set to Fasle at the same time in InstanceNorm" self._epsilon = epsilon self._param_attr = param_attr self._bias_attr = bias_attr @@ -1136,29 +1280,34 @@ def __init__(self, shape=[num_channels], dtype=self._dtype, default_initializer=Constant(1.0), - is_bias=False) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[num_channels], - dtype=self._dtype, - default_initializer=Constant(0.0), - is_bias=True) + is_bias=False, + ) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[num_channels], + dtype=self._dtype, + default_initializer=Constant(0.0), + is_bias=True, + ) else: self.scale = None self.bias = None def forward(self, input): if in_dygraph_mode(): - out = _C_ops.instance_norm(input, self.scale, self.bias, - self._epsilon) + out = _C_ops.instance_norm( + input, self.scale, self.bias, self._epsilon + ) return out if _in_legacy_dygraph(): - out, _, _ = _legacy_C_ops.instance_norm(input, self.scale, - self.bias, 'epsilon', - self._epsilon) + out, _, _ = _legacy_C_ops.instance_norm( + input, self.scale, self.bias, 'epsilon', self._epsilon + ) return out - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - "InstanceNorm") + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], "InstanceNorm" + ) attrs = {"epsilon": self._epsilon} @@ -1168,22 +1317,24 @@ def forward(self, input): inputs = {"X": [input]} saved_mean = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) saved_variance = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) instance_norm_out = self._helper.create_variable_for_type_inference( - self._dtype) + self._dtype + ) outputs = { "Y": [instance_norm_out], "SavedMean": [saved_mean], - "SavedVariance": [saved_variance] + "SavedVariance": [saved_variance], } - self._helper.append_op(type="instance_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + self._helper.append_op( + type="instance_norm", inputs=inputs, outputs=outputs, attrs=attrs + ) return instance_norm_out @@ -1285,29 +1436,33 @@ class BatchNorm(layers.Layer): hidden1 = batch_norm(x) """ - def __init__(self, - num_channels, - act=None, - is_test=False, - momentum=0.9, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - dtype='float32', - data_layout='NCHW', - in_place=False, - moving_mean_name=None, - moving_variance_name=None, - do_model_average_for_mean_and_var=True, - use_global_stats=False, - trainable_statistics=False): + def __init__( + self, + num_channels, + act=None, + is_test=False, + momentum=0.9, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + dtype='float32', + data_layout='NCHW', + in_place=False, + moving_mean_name=None, + moving_variance_name=None, + do_model_average_for_mean_and_var=True, + use_global_stats=False, + trainable_statistics=False, + ): super(BatchNorm, self).__init__() self._param_attr = param_attr self._bias_attr = bias_attr self._act = act self._use_mkldnn = _global_flags()["FLAGS_use_mkldnn"] - assert bias_attr is not False, "bias_attr should not be False in batch_norm." + assert ( + bias_attr is not False + ), "bias_attr should not be False in batch_norm." if dtype == "float16": self._dtype = "float32" @@ -1317,34 +1472,48 @@ def __init__(self, param_shape = [num_channels] # create parameter - self.weight = self.create_parameter(attr=self._param_attr, - shape=param_shape, - dtype=self._dtype, - default_initializer=Constant(1.0)) - self.weight.stop_gradient = use_global_stats and self._param_attr.learning_rate == 0. - - self.bias = self.create_parameter(attr=self._bias_attr, - shape=param_shape, - dtype=self._dtype, - is_bias=True) - self.bias.stop_gradient = use_global_stats and self._param_attr.learning_rate == 0. - - self._mean = self.create_parameter(attr=ParamAttr( - name=moving_mean_name, - initializer=Constant(0.0), - trainable=False, - do_model_average=do_model_average_for_mean_and_var), - shape=param_shape, - dtype=self._dtype) + self.weight = self.create_parameter( + attr=self._param_attr, + shape=param_shape, + dtype=self._dtype, + default_initializer=Constant(1.0), + ) + self.weight.stop_gradient = ( + use_global_stats and self._param_attr.learning_rate == 0.0 + ) + + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=param_shape, + dtype=self._dtype, + is_bias=True, + ) + self.bias.stop_gradient = ( + use_global_stats and self._param_attr.learning_rate == 0.0 + ) + + self._mean = self.create_parameter( + attr=ParamAttr( + name=moving_mean_name, + initializer=Constant(0.0), + trainable=False, + do_model_average=do_model_average_for_mean_and_var, + ), + shape=param_shape, + dtype=self._dtype, + ) self._mean.stop_gradient = True - self._variance = self.create_parameter(attr=ParamAttr( - name=moving_variance_name, - initializer=Constant(1.0), - trainable=False, - do_model_average=do_model_average_for_mean_and_var), - shape=param_shape, - dtype=self._dtype) + self._variance = self.create_parameter( + attr=ParamAttr( + name=moving_variance_name, + initializer=Constant(1.0), + trainable=False, + do_model_average=do_model_average_for_mean_and_var, + ), + shape=param_shape, + dtype=self._dtype, + ) self._variance.stop_gradient = True self._in_place = in_place @@ -1366,29 +1535,61 @@ def forward(self, input): if _non_static_mode(): if in_dygraph_mode(): batch_norm_out, t1, t2, t3, t4, _ = _C_ops.batch_norm( - input, self.weight, self.bias, self._mean, self._variance, - self._momentum, self._epsilon, self._data_layout, - not self.training, self._use_global_stats, - self._trainable_statistics, False) + input, + self.weight, + self.bias, + self._mean, + self._variance, + self._momentum, + self._epsilon, + self._data_layout, + not self.training, + self._use_global_stats, + self._trainable_statistics, + False, + ) return dygraph_utils._append_activation_in_dygraph( - batch_norm_out, act=self._act, use_mkldnn=self._use_mkldnn) + batch_norm_out, act=self._act, use_mkldnn=self._use_mkldnn + ) elif _in_legacy_dygraph(): - attrs = ("momentum", self._momentum, "epsilon", self._epsilon, - "is_test", not self.training, "data_layout", - self._data_layout, "use_mkldnn", self._use_mkldnn, - "fuse_with_relu", self._fuse_with_relu, - "use_global_stats", self._use_global_stats, - 'trainable_statistics', self._trainable_statistics) + attrs = ( + "momentum", + self._momentum, + "epsilon", + self._epsilon, + "is_test", + not self.training, + "data_layout", + self._data_layout, + "use_mkldnn", + self._use_mkldnn, + "fuse_with_relu", + self._fuse_with_relu, + "use_global_stats", + self._use_global_stats, + 'trainable_statistics', + self._trainable_statistics, + ) batch_norm_out, _, _, _, _, _ = _legacy_C_ops.batch_norm( - input, self.weight, self.bias, self._mean, self._variance, - None, mean_out, variance_out, *attrs) + input, + self.weight, + self.bias, + self._mean, + self._variance, + None, + mean_out, + variance_out, + *attrs + ) return dygraph_utils._append_activation_in_dygraph( - batch_norm_out, act=self._act, use_mkldnn=self._use_mkldnn) + batch_norm_out, act=self._act, use_mkldnn=self._use_mkldnn + ) - check_variable_and_dtype(input, 'input', - ['float16', 'float32', 'float64'], 'BatchNorm') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'BatchNorm' + ) attrs = { "momentum": self._momentum, @@ -1406,33 +1607,38 @@ def forward(self, input): "Scale": [self.weight], "Bias": [self.bias], "Mean": [self._mean], - "Variance": [self._variance] + "Variance": [self._variance], } saved_mean = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) saved_variance = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) reserve_space = self._helper.create_variable_for_type_inference( - dtype=self._helper.input_dtype(input), stop_gradient=True) + dtype=self._helper.input_dtype(input), stop_gradient=True + ) - batch_norm_out = input if self._in_place else self._helper.create_variable_for_type_inference( - self._dtype) + batch_norm_out = ( + input + if self._in_place + else self._helper.create_variable_for_type_inference(self._dtype) + ) outputs = { "Y": [batch_norm_out], "MeanOut": [mean_out], "VarianceOut": [variance_out], "SavedMean": [saved_mean], - "SavedVariance": [saved_variance] + "SavedVariance": [saved_variance], } if reserve_space is not None: outputs["ReserveSpace"] = [reserve_space] - self._helper.append_op(type="batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + self._helper.append_op( + type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs + ) # Currently, we don't support inplace in dygraph mode return self._helper.append_activation(batch_norm_out, self._act) @@ -1440,78 +1646,82 @@ def forward(self, input): class Dropout(layers.Layer): """ - This interface is used to construct a callable object of the ``Dropout`` class. - For more details, refer to code examples. + This interface is used to construct a callable object of the ``Dropout`` class. + For more details, refer to code examples. - Drop or keep each element of input independently. Dropout is a regularization - technique for reducing overfitting by preventing neuron co-adaption during - training. The dropout operator randomly sets (according to the given dropout - probability) the outputs of some units to zero, while others are remain - unchanged. + Drop or keep each element of input independently. Dropout is a regularization + technique for reducing overfitting by preventing neuron co-adaption during + training. The dropout operator randomly sets (according to the given dropout + probability) the outputs of some units to zero, while others are remain + unchanged. - Dropout layer can be removed for efficiency concern. + Dropout layer can be removed for efficiency concern. - Parameters: - p (float, optional): Probability of setting units to zero. Default: 0.5 - seed (int, optional): A Python integer used to create random seeds. If this - parameter is set to None, a random seed is used. - NOTE: If an integer seed is given, always the same output - units will be dropped. DO NOT use a fixed seed in training. Default: None. - dropout_implementation(string, optional): ['downgrade_in_infer'(default)|'upscale_in_train'] + Parameters: + p (float, optional): Probability of setting units to zero. Default: 0.5 + seed (int, optional): A Python integer used to create random seeds. If this + parameter is set to None, a random seed is used. + NOTE: If an integer seed is given, always the same output + units will be dropped. DO NOT use a fixed seed in training. Default: None. + dropout_implementation(string, optional): ['downgrade_in_infer'(default)|'upscale_in_train'] - 1. downgrade_in_infer(default), downgrade the outcome at inference + 1. downgrade_in_infer(default), downgrade the outcome at inference - - train: out = input * mask - - inference: out = input * (1.0 - p) + - train: out = input * mask + - inference: out = input * (1.0 - p) - (mask is a tensor same shape with input, value is 0 or 1 - ratio of 0 is dropout_prob) - 2. upscale_in_train, upscale the outcome at training time + (mask is a tensor same shape with input, value is 0 or 1 + ratio of 0 is dropout_prob) + 2. upscale_in_train, upscale the outcome at training time - - train: out = input * mask / ( 1.0 - p ) - - inference: out = input + - train: out = input * mask / ( 1.0 - p ) + - inference: out = input - (mask is a tensor same shape with input, value is 0 or 1 - ratio of 0 is p) - is_test (bool, optional): A flag indicating whether it is in test phrase or not. - This flag only has effect on static graph mode. For dygraph mode, please use ``eval()``. - Default: False. + (mask is a tensor same shape with input, value is 0 or 1 + ratio of 0 is p) + is_test (bool, optional): A flag indicating whether it is in test phrase or not. + This flag only has effect on static graph mode. For dygraph mode, please use ``eval()``. + Default: False. - Returns: - None + Returns: + None - Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - from paddle.fluid.dygraph.base import to_variable - import numpy as np - - x = np.random.random(size=(3, 10, 3, 7)).astype('float32') - with fluid.dygraph.guard(): - x = to_variable(x) - m = fluid.dygraph.Dropout(p=0.5) - droped_train = m(x) - # switch to eval mode - m.eval() - droped_eval = m(x) - """ - - def __init__(self, - p=0.5, - seed=None, - dropout_implementation="downgrade_in_infer", - is_test=False): + import paddle.fluid as fluid + from paddle.fluid.dygraph.base import to_variable + import numpy as np + + x = np.random.random(size=(3, 10, 3, 7)).astype('float32') + with fluid.dygraph.guard(): + x = to_variable(x) + m = fluid.dygraph.Dropout(p=0.5) + droped_train = m(x) + # switch to eval mode + m.eval() + droped_eval = m(x) + """ + + def __init__( + self, + p=0.5, + seed=None, + dropout_implementation="downgrade_in_infer", + is_test=False, + ): super(Dropout, self).__init__() assert isinstance(p, (float, int)), "p argument should be a number" assert 0 <= p <= 1, "p argument should between 0 and 1" self._dropout_prob = p assert seed is None or isinstance( - seed, int), "seed argument should be None or a integer" + seed, int + ), "seed argument should be None or a integer" self._seed = seed assert dropout_implementation in ( - 'downgrade_in_infer', 'upscale_in_train' + 'downgrade_in_infer', + 'upscale_in_train', ), "dropout_implementation argument should be 'downgrade_in_infer' or 'upscale_in_train'" self._dropout_implementation = dropout_implementation self._is_test = is_test @@ -1525,8 +1735,9 @@ def forward(self, input): self._seed = prog.random_seed attrs = { 'dropout_prob': self._dropout_prob, - 'is_test': - not self.training if _non_static_mode() else self._is_test, + 'is_test': not self.training + if _non_static_mode() + else self._is_test, 'fix_seed': self._seed is not None, 'seed': self._seed if self._seed is not None else 0, 'dropout_implementation': self._dropout_implementation, @@ -1539,23 +1750,23 @@ def forward(self, input): out = self._helper.create_variable_for_type_inference(dtype=input.dtype) mask = self._helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) - - self._helper.append_op(type='dropout', - inputs={'X': [input]}, - outputs={ - 'Out': [out], - 'Mask': [mask] - }, - attrs=attrs) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) + + self._helper.append_op( + type='dropout', + inputs={'X': [input]}, + outputs={'Out': [out], 'Mask': [mask]}, + attrs=attrs, + ) return out class Embedding(layers.Layer): r""" :alias_main: paddle.nn.Embedding - :alias: paddle.nn.Embedding,paddle.nn.layer.Embedding,paddle.nn.layer.common.Embedding - :old_api: paddle.fluid.dygraph.Embedding + :alias: paddle.nn.Embedding,paddle.nn.layer.Embedding,paddle.nn.layer.common.Embedding + :old_api: paddle.fluid.dygraph.Embedding **Embedding Layer** @@ -1659,19 +1870,26 @@ class Embedding(layers.Layer): static_rlt3 = emb(base.to_variable(inp_word)) """ - def __init__(self, - size, - is_sparse=False, - is_distributed=False, - padding_idx=None, - param_attr=None, - dtype='float32'): + def __init__( + self, + size, + is_sparse=False, + is_distributed=False, + padding_idx=None, + param_attr=None, + dtype='float32', + ): super(Embedding, self).__init__() self._size = size self._is_sparse = is_sparse self._is_distributed = is_distributed - self._padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( - size[0] + padding_idx) + self._padding_idx = ( + -1 + if padding_idx is None + else padding_idx + if padding_idx >= 0 + else (size[0] + padding_idx) + ) self._param_attr = param_attr self._dtype = dtype @@ -1679,36 +1897,48 @@ def __init__(self, if self._remote_prefetch: assert self._is_sparse is True and self._is_distributed is False - self.weight = self.create_parameter(attr=self._param_attr, - shape=self._size, - dtype=self._dtype, - is_bias=False) + self.weight = self.create_parameter( + attr=self._param_attr, + shape=self._size, + dtype=self._dtype, + is_bias=False, + ) def forward(self, input): if _non_static_mode(): return _legacy_C_ops.lookup_table_v2( - self.weight, input, 'is_sparse', self._is_sparse, - 'is_distributed', self._is_distributed, 'remote_prefetch', - self._remote_prefetch, 'padding_idx', self._padding_idx) + self.weight, + input, + 'is_sparse', + self._is_sparse, + 'is_distributed', + self._is_distributed, + 'remote_prefetch', + self._remote_prefetch, + 'padding_idx', + self._padding_idx, + ) - check_variable_and_dtype(input, 'input', - ['uint8', 'int8', 'int16', 'int32', 'int64'], - 'Embedding') + check_variable_and_dtype( + input, + 'input', + ['uint8', 'int8', 'int16', 'int32', 'int64'], + 'Embedding', + ) attrs = { 'is_sparse': self._is_sparse, 'is_distributed': self._is_distributed, 'remote_prefetch': self._remote_prefetch, - 'padding_idx': self._padding_idx + 'padding_idx': self._padding_idx, } out = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type='lookup_table_v2', - inputs={ - 'Ids': input, - 'W': self.weight - }, - outputs={'Out': out}, - attrs=attrs) + self._helper.append_op( + type='lookup_table_v2', + inputs={'Ids': input, 'W': self.weight}, + outputs={'Out': out}, + attrs=attrs, + ) return out @@ -1716,8 +1946,8 @@ def forward(self, input): class LayerNorm(layers.Layer): r""" :alias_main: paddle.nn.LayerNorm - :alias: paddle.nn.LayerNorm,paddle.nn.layer.LayerNorm,paddle.nn.layer.norm.LayerNorm - :old_api: paddle.fluid.dygraph.LayerNorm + :alias: paddle.nn.LayerNorm,paddle.nn.layer.LayerNorm,paddle.nn.layer.norm.LayerNorm + :old_api: paddle.fluid.dygraph.LayerNorm This interface is used to construct a callable object of the ``LayerNorm`` class. For more details, refer to code examples. @@ -1784,15 +2014,17 @@ class LayerNorm(layers.Layer): """ - def __init__(self, - normalized_shape, - scale=True, - shift=True, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - act=None, - dtype='float32'): + def __init__( + self, + normalized_shape, + scale=True, + shift=True, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + act=None, + dtype='float32', + ): super(LayerNorm, self).__init__() if isinstance(normalized_shape, numbers.Integral): normalized_shape = [normalized_shape] @@ -1811,7 +2043,8 @@ def __init__(self, attr=self._param_attr, shape=param_shape, dtype=self._dtype, - default_initializer=Constant(1.0)) + default_initializer=Constant(1.0), + ) else: if self._param_attr: logging.warn("param_attr are only available with scale is True") @@ -1819,10 +2052,12 @@ def __init__(self, if self._shift: assert self._bias_attr is not False - self.bias = self.create_parameter(attr=self._bias_attr, - shape=param_shape, - dtype=self._dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=param_shape, + dtype=self._dtype, + is_bias=True, + ) else: if self._bias_attr: logging.warn("bias_attr are only available with shift is True") @@ -1833,31 +2068,50 @@ def forward(self, input): input_ndim = len(input_shape) normalized_ndim = len(self._normalized_shape) self._begin_norm_axis = input_ndim - normalized_ndim - if input_ndim < normalized_ndim or input_shape[ - self._begin_norm_axis:] != self._normalized_shape: + if ( + input_ndim < normalized_ndim + or input_shape[self._begin_norm_axis :] != self._normalized_shape + ): str_normalized_shape = str(self._normalized_shape) - raise ValueError('Given normalized_shape is ' + - str_normalized_shape + - ', expected input with shape [*, ' + - str_normalized_shape[1:] + - ', but got input shape ' + str(input_shape)) + raise ValueError( + 'Given normalized_shape is ' + + str_normalized_shape + + ', expected input with shape [*, ' + + str_normalized_shape[1:] + + ', but got input shape ' + + str(input_shape) + ) if _non_static_mode(): if in_dygraph_mode(): - pre_act, _, _, = _C_ops.layer_norm(input, self.weight, - self.bias, self._epsilon, - self._begin_norm_axis, False) + pre_act, _, _, = _C_ops.layer_norm( + input, + self.weight, + self.bias, + self._epsilon, + self._begin_norm_axis, + False, + ) return dygraph_utils._append_activation_in_dygraph( - pre_act, act=self._act) + pre_act, act=self._act + ) else: pre_act, _, _ = _legacy_C_ops.layer_norm( - input, self.weight, self.bias, 'epsilon', self._epsilon, - 'begin_norm_axis', self._begin_norm_axis) + input, + self.weight, + self.bias, + 'epsilon', + self._epsilon, + 'begin_norm_axis', + self._begin_norm_axis, + ) return dygraph_utils._append_activation_in_dygraph( - pre_act, act=self._act) + pre_act, act=self._act + ) - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'LayerNorm') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'LayerNorm' + ) inputs = dict() inputs['X'] = [input] @@ -1867,28 +2121,33 @@ def forward(self, input): inputs['Bias'] = [self.bias] attrs = { "epsilon": self._epsilon, - "begin_norm_axis": self._begin_norm_axis + "begin_norm_axis": self._begin_norm_axis, } # create output mean_out = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) variance_out = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) layer_norm_out = self._helper.create_variable_for_type_inference( - self._dtype) - - self._helper.append_op(type="layer_norm", - inputs=inputs, - outputs={ - "Y": layer_norm_out, - "Mean": mean_out, - "Variance": variance_out, - }, - attrs={ - "epsilon": self._epsilon, - "begin_norm_axis": self._begin_norm_axis - }) + self._dtype + ) + + self._helper.append_op( + type="layer_norm", + inputs=inputs, + outputs={ + "Y": layer_norm_out, + "Mean": mean_out, + "Variance": variance_out, + }, + attrs={ + "epsilon": self._epsilon, + "begin_norm_axis": self._begin_norm_axis, + }, + ) return self._helper.append_activation(layer_norm_out, act=self._act) @@ -2002,14 +2261,16 @@ class GRUUnit(layers.Layer): """ - def __init__(self, - size, - param_attr=None, - bias_attr=None, - activation='tanh', - gate_activation='sigmoid', - origin_mode=False, - dtype='float32'): + def __init__( + self, + size, + param_attr=None, + bias_attr=None, + activation='tanh', + gate_activation='sigmoid', + origin_mode=False, + dtype='float32', + ): super(GRUUnit, self).__init__() self._bias_attr = bias_attr activation_dict = dict( @@ -2024,52 +2285,64 @@ def __init__(self, self._dtype = dtype size = size // 3 # create weight - self.weight = self.create_parameter(attr=param_attr, - shape=[size, 3 * size], - dtype=dtype) + self.weight = self.create_parameter( + attr=param_attr, shape=[size, 3 * size], dtype=dtype + ) # create bias bias_size = [1, 3 * size] self._bias_size = bias_size - self.bias = self.create_parameter(attr=bias_attr, - shape=bias_size, - dtype=dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=bias_attr, shape=bias_size, dtype=dtype, is_bias=True + ) def forward(self, input, hidden): if _non_static_mode(): gate, reset_hidden_pre, updated_hidden = _legacy_C_ops.gru_unit( - input, hidden, self.weight, self.bias, 'activation', - self.activation, 'gate_activation', self.gate_activation) + input, + hidden, + self.weight, + self.bias, + 'activation', + self.activation, + 'gate_activation', + self.gate_activation, + ) return updated_hidden, reset_hidden_pre, gate - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'GRUUnit') - check_variable_and_dtype(hidden, 'hidden', ['float32', 'float64'], - 'GRUUnit') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'GRUUnit' + ) + check_variable_and_dtype( + hidden, 'hidden', ['float32', 'float64'], 'GRUUnit' + ) inputs = { 'Input': [input], 'HiddenPrev': [hidden], - 'Weight': [self.weight] + 'Weight': [self.weight], } if self.bias is not None: inputs['Bias'] = [self.bias] gate = self._helper.create_variable_for_type_inference(self._dtype) reset_hidden_pre = self._helper.create_variable_for_type_inference( - self._dtype) + self._dtype + ) updated_hidden = self._helper.create_variable_for_type_inference( - self._dtype) - self._helper.append_op(type='gru_unit', - inputs=inputs, - outputs={ - 'Gate': gate, - 'ResetHiddenPrev': reset_hidden_pre, - 'Hidden': updated_hidden, - }, - attrs={ - 'activation': self.activation, - 'gate_activation': self.gate_activation, - }) + self._dtype + ) + self._helper.append_op( + type='gru_unit', + inputs=inputs, + outputs={ + 'Gate': gate, + 'ResetHiddenPrev': reset_hidden_pre, + 'Hidden': updated_hidden, + }, + attrs={ + 'activation': self.activation, + 'gate_activation': self.gate_activation, + }, + ) return updated_hidden, reset_hidden_pre, gate @@ -2161,25 +2434,29 @@ class NCE(layers.Layer): """ - def __init__(self, - num_total_classes, - dim, - sample_weight=None, - param_attr=None, - bias_attr=None, - num_neg_samples=None, - sampler="uniform", - custom_dist=None, - seed=0, - is_sparse=False, - dtype='float32'): + def __init__( + self, + num_total_classes, + dim, + sample_weight=None, + param_attr=None, + bias_attr=None, + num_neg_samples=None, + sampler="uniform", + custom_dist=None, + seed=0, + is_sparse=False, + dtype='float32', + ): super(NCE, self).__init__() self._param_attr = param_attr self._bias_attr = bias_attr self._num_total_classes = num_total_classes self._dtype = dtype self._inputs = dict() - self._inputs['SampleWeight'] = sample_weight if sample_weight is not None else [] + self._inputs['SampleWeight'] = ( + sample_weight if sample_weight is not None else [] + ) if sampler == "uniform": sampler = 0 elif sampler == "log_uniform": @@ -2235,16 +2512,20 @@ def _init_by_numpy_array(numpy_array): attr=ParamAttr(), shape=numpy_array.shape, dtype=numpy_array.dtype, - default_initializer=NumpyArrayInitializer(numpy_array)) + default_initializer=NumpyArrayInitializer(numpy_array), + ) ret.stop_gradient = True return ret self._inputs['CustomDistProbs'] = _init_by_numpy_array( - np.array(custom_dist).astype('float32')) + np.array(custom_dist).astype('float32') + ) self._inputs['CustomDistAlias'] = _init_by_numpy_array( - np.array(alias_).astype('int32')) + np.array(alias_).astype('int32') + ) self._inputs['CustomDistAliasProbs'] = _init_by_numpy_array( - np.array(alias_probs_).astype('float32')) + np.array(alias_probs_).astype('float32') + ) sampler = 2 else: raise Exception("Unsupported sampler type.") @@ -2264,64 +2545,88 @@ def _init_by_numpy_array(numpy_array): 'seed': seed, 'sampler': sampler, 'is_sparse': is_sparse, - 'remote_prefetch': remote_prefetch + 'remote_prefetch': remote_prefetch, } self.weight = self.create_parameter( attr=self._param_attr, shape=[self._num_total_classes, dim], is_bias=False, - dtype=self._dtype) + dtype=self._dtype, + ) if self._bias_attr: self.bias = self.create_parameter( attr=self._bias_attr, shape=[self._num_total_classes, 1], is_bias=True, - dtype=self._dtype) + dtype=self._dtype, + ) self._inputs['Bias'] = self.bias self._inputs['Weight'] = self.weight def forward(self, input, label, sample_weight=None): if _non_static_mode(): - attrs = ('num_total_classes', self._attrs['num_total_classes'], - 'num_neg_samples', self._attrs['num_neg_samples'], 'seed', - self._attrs['seed'], 'sampler', self._attrs['sampler'], - 'is_sparse', self._attrs['is_sparse'], 'remote_prefetch', - self._attrs['remote_prefetch']) - cost, _, _ = _legacy_C_ops.nce(input, label, self.weight, self.bias, - self._inputs['SampleWeight'], - self._inputs['CustomDistProbs'], - self._inputs['CustomDistAlias'], - self._inputs['CustomDistAliasProbs'], - *attrs) + attrs = ( + 'num_total_classes', + self._attrs['num_total_classes'], + 'num_neg_samples', + self._attrs['num_neg_samples'], + 'seed', + self._attrs['seed'], + 'sampler', + self._attrs['sampler'], + 'is_sparse', + self._attrs['is_sparse'], + 'remote_prefetch', + self._attrs['remote_prefetch'], + ) + cost, _, _ = _legacy_C_ops.nce( + input, + label, + self.weight, + self.bias, + self._inputs['SampleWeight'], + self._inputs['CustomDistProbs'], + self._inputs['CustomDistAlias'], + self._inputs['CustomDistAliasProbs'], + *attrs + ) return cost / (self._num_neg_samples + 1) check_variable_and_dtype(input, "input", ['float32', 'float64'], "NCE") check_variable_and_dtype(label, "label", ['int64'], "NCE") - check_type(sample_weight, 'sample_weight', (Variable, type(None)), - 'NCE') + check_type( + sample_weight, 'sample_weight', (Variable, type(None)), 'NCE' + ) assert isinstance(input, Variable) assert isinstance(label, Variable) self._inputs['Input'] = input self._inputs['Label'] = label - self._inputs['SampleWeight'] = sample_weight if sample_weight is not None else [] + self._inputs['SampleWeight'] = ( + sample_weight if sample_weight is not None else [] + ) cost = self._helper.create_variable_for_type_inference( - dtype=input.dtype) + dtype=input.dtype + ) sample_logits = self._helper.create_variable_for_type_inference( - dtype=input.dtype) + dtype=input.dtype + ) sample_labels = self._helper.create_variable_for_type_inference( - dtype=label.dtype) - - self._helper.append_op(type='nce', - inputs=self._inputs, - outputs={ - 'Cost': cost, - 'SampleLogits': sample_logits, - 'SampleLabels': sample_labels - }, - attrs=self._attrs) + dtype=label.dtype + ) + + self._helper.append_op( + type='nce', + inputs=self._inputs, + outputs={ + 'Cost': cost, + 'SampleLogits': sample_logits, + 'SampleLabels': sample_labels, + }, + attrs=self._attrs, + ) return cost / (self._num_neg_samples + 1) @@ -2385,12 +2690,14 @@ class PRelu(layers.Layer): """ - def __init__(self, - mode, - channel=None, - input_shape=None, - param_attr=None, - dtype='float32'): + def __init__( + self, + mode, + channel=None, + input_shape=None, + param_attr=None, + dtype='float32', + ): # need specify name_scope since snake-cased 'PRelu' is 'p_relu' super(PRelu, self).__init__(name_scope='prelu') self._mode = mode @@ -2400,26 +2707,27 @@ def __init__(self, self._alpha_shape = [1] elif mode == 'channel': assert isinstance( - channel, - int), "channel argument is required when mode is 'channel'." - #NOTE(zhiqiu): The _alpha_shape should be [1, channel] + [1] * len(input_shape[2:]), not [1, channel, 1, 1]. + channel, int + ), "channel argument is required when mode is 'channel'." + # NOTE(zhiqiu): The _alpha_shape should be [1, channel] + [1] * len(input_shape[2:]), not [1, channel, 1, 1]. # However, the suffix 1 in the list is useless, since the tensor is viewed as one demension array during kernel calculation. # And, input_shape is not required when mode is 'channel', so it is simplified. - #NOTE(zhiqiu): Revert shape to [1, channel, 1, 1] for compatibility with saved model of old version. + # NOTE(zhiqiu): Revert shape to [1, channel, 1, 1] for compatibility with saved model of old version. self._alpha_shape = [1, channel, 1, 1] elif mode == 'element': assert isinstance( - input_shape, - (list, tuple - )), "input_shape argument is required when mode is 'element'." + input_shape, (list, tuple) + ), "input_shape argument is required when mode is 'element'." self._alpha_shape = [1] + list(input_shape)[1:] else: raise ValueError('mode should be one of all, channel, element.') - self.weight = self.create_parameter(attr=self._param_attr, - shape=self._alpha_shape, - dtype='float32', - is_bias=False, - default_initializer=Constant(1.0)) + self.weight = self.create_parameter( + attr=self._param_attr, + shape=self._alpha_shape, + dtype='float32', + is_bias=False, + default_initializer=Constant(1.0), + ) def forward(self, input): if in_dygraph_mode(): @@ -2427,13 +2735,12 @@ def forward(self, input): check_variable_and_dtype(input, 'input', ['float32'], 'PRelu') out = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type="prelu", - inputs={ - "X": input, - 'Alpha': self.weight - }, - attrs={"mode": self._mode}, - outputs={"Out": out}) + self._helper.append_op( + type="prelu", + inputs={"X": input, 'Alpha': self.weight}, + attrs={"mode": self._mode}, + outputs={"Out": out}, + ) return out @@ -2492,15 +2799,17 @@ class BilinearTensorProduct(layers.Layer): """ - def __init__(self, - input1_dim, - input2_dim, - output_dim, - name=None, - act=None, - param_attr=None, - bias_attr=None, - dtype='float32'): + def __init__( + self, + input1_dim, + input2_dim, + output_dim, + name=None, + act=None, + param_attr=None, + bias_attr=None, + dtype='float32', + ): super(BilinearTensorProduct, self).__init__() self._param_attr = param_attr self._bias_attr = bias_attr @@ -2513,38 +2822,50 @@ def __init__(self, self._dtype = dtype param_shape = [self._output_dim, self._input1_dim, self._input2_dim] - self.weight = self.create_parameter(attr=self._param_attr, - shape=param_shape, - dtype=self._dtype, - is_bias=False) + self.weight = self.create_parameter( + attr=self._param_attr, + shape=param_shape, + dtype=self._dtype, + is_bias=False, + ) bias_size = [1, self._output_dim] - self.bias = self.create_parameter(attr=self._bias_attr, - shape=bias_size, - dtype=self._dtype, - is_bias=True) - - @deprecated(since="2.0.0", - update_to="paddle.nn.Bilinear", - reason="New name and new args in Bilinear, easier to use.") + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=bias_size, + dtype=self._dtype, + is_bias=True, + ) + + @deprecated( + since="2.0.0", + update_to="paddle.nn.Bilinear", + reason="New name and new args in Bilinear, easier to use.", + ) def forward(self, x, y): - check_variable_and_dtype(x, 'x', ['float32', 'float64'], - 'BilinearTensorProduct') - check_variable_and_dtype(y, 'y', ['float32', 'float64'], - 'BilinearTensorProduct') + check_variable_and_dtype( + x, 'x', ['float32', 'float64'], 'BilinearTensorProduct' + ) + check_variable_and_dtype( + y, 'y', ['float32', 'float64'], 'BilinearTensorProduct' + ) self._inputs = {"X": x, "Y": y, "Weight": self.weight} if self.bias is not None: self._inputs["Bias"] = self.bias if self._name is not None: - out = self._helper.create_variable(name=".".join( - [self.full_name(), self._name]), - dtype=self._dtype, - persistable=False) + out = self._helper.create_variable( + name=".".join([self.full_name(), self._name]), + dtype=self._dtype, + persistable=False, + ) else: - out = self._helper.create_variable(dtype=self._dtype, - persistable=False) - self._helper.append_op(type="bilinear_tensor_product", - inputs=self._inputs, - outputs={"Out": out}) + out = self._helper.create_variable( + dtype=self._dtype, persistable=False + ) + self._helper.append_op( + type="bilinear_tensor_product", + inputs=self._inputs, + outputs={"Out": out}, + ) # add activation return self._helper.append_activation(out, act=self._act) @@ -2668,22 +2989,26 @@ class Conv2DTranspose(layers.Layer): """ - def __init__(self, - num_channels, - num_filters, - filter_size, - output_size=None, - padding=0, - stride=1, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - dtype='float32'): + def __init__( + self, + num_channels, + num_filters, + filter_size, + output_size=None, + padding=0, + stride=1, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + dtype='float32', + ): super(Conv2DTranspose, self).__init__() - assert param_attr is not False, "param_attr should not be False in conv2d_transpose." + assert ( + param_attr is not False + ), "param_attr should not be False in conv2d_transpose." self._param_attr = param_attr self._bias_attr = bias_attr self._act = act @@ -2698,9 +3023,11 @@ def __init__(self, self._output_size = output_size self._dtype = dtype - if (self._num_channels == self._groups - and self._num_filters == self._num_channels - and not self._use_cudnn): + if ( + self._num_channels == self._groups + and self._num_filters == self._num_channels + and not self._use_cudnn + ): self._op_type = 'depthwise_conv2d_transpose' else: self._op_type = 'conv2d_transpose' @@ -2710,63 +3037,91 @@ def __init__(self, self._dilation = utils.convert_to_list(self._dilation, 2, 'dilation') self._filter_size = utils.convert_to_list( - self._filter_size, 2, 'conv2d_transpose.filter_size') + self._filter_size, 2, 'conv2d_transpose.filter_size' + ) if self._output_size is None: self._output_size = [] elif isinstance(self._output_size, list): if utils._contain_var(self._output_size): self._output_size = utils._convert_to_tensor_list( - self._output_size) + self._output_size + ) else: self._output_size = utils.convert_to_list( - self._output_size, 2, 'output_size') + self._output_size, 2, 'output_size' + ) elif isinstance(self._output_size, int): - self._output_size = utils.convert_to_list(self._output_size, 2, - 'output_size') + self._output_size = utils.convert_to_list( + self._output_size, 2, 'output_size' + ) elif isinstance(self._output_size, Variable): - check_dtype(self._output_size.dtype, 'output_size', - ['int32', 'int64'], 'Conv2DTranspose') + check_dtype( + self._output_size.dtype, + 'output_size', + ['int32', 'int64'], + 'Conv2DTranspose', + ) if len(self._output_size.shape) == 1 and ( - self._output_size.shape[0] == 1 - or self._output_size.shape[0] == 2): + self._output_size.shape[0] == 1 + or self._output_size.shape[0] == 2 + ): if self._output_size.shape[0] == 1: self._output_size = [self._output_size, self._output_size] else: raise ValueError( - "output_size must contain one or two integers.") + "output_size must contain one or two integers." + ) else: raise ValueError("output_size should be list or int or Tensor") self._padding = utils.convert_to_list(self._padding, 2, 'padding') self._groups = 1 if self._groups is None else self._groups - filter_shape = [self._num_channels, self._num_filters // self._groups - ] + self._filter_size + filter_shape = [ + self._num_channels, + self._num_filters // self._groups, + ] + self._filter_size - self.weight = self.create_parameter(dtype=self._dtype, - shape=filter_shape, - attr=self._param_attr) + self.weight = self.create_parameter( + dtype=self._dtype, shape=filter_shape, attr=self._param_attr + ) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._num_filters], - dtype=self._dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[self._num_filters], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input): if _non_static_mode(): op = getattr(_legacy_C_ops, self._op_type) - out = op(input, self.weight, 'output_size', self._output_size, - 'strides', self._stride, 'paddings', self._padding, - 'dilations', self._dilation, 'groups', self._groups, - 'use_cudnn', self._use_cudnn) + out = op( + input, + self.weight, + 'output_size', + self._output_size, + 'strides', + self._stride, + 'paddings', + self._padding, + 'dilations', + self._dilation, + 'groups', + self._groups, + 'use_cudnn', + self._use_cudnn, + ) pre_bias = out pre_act = dygraph_utils._append_bias_in_dygraph( - pre_bias, self.bias, 1) - return dygraph_utils._append_activation_in_dygraph(pre_act, - act=self._act) + pre_bias, self.bias, 1 + ) + return dygraph_utils._append_activation_in_dygraph( + pre_act, act=self._act + ) - check_variable_and_dtype(input, 'input', - ['float16', 'float32', 'float64'], - "Conv2DTranspose") + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], "Conv2DTranspose" + ) inputs = {'Input': [input], 'Filter': [self.weight]} attrs = { @@ -2775,26 +3130,29 @@ def forward(self, input): 'paddings': self._padding, 'dilations': self._dilation, 'groups': self._groups, - 'use_cudnn': self._use_cudnn + 'use_cudnn': self._use_cudnn, } pre_bias = self._helper.create_variable_for_type_inference( - dtype=input.dtype) - self._helper.append_op(type=self._op_type, - inputs=inputs, - outputs={'Output': pre_bias}, - attrs=attrs) + dtype=input.dtype + ) + self._helper.append_op( + type=self._op_type, + inputs=inputs, + outputs={'Output': pre_bias}, + attrs=attrs, + ) if self.bias is not None: pre_act = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [self.bias] - }, - outputs={'Out': [pre_act]}, - attrs={'axis': 1}) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [self.bias]}, + outputs={'Out': [pre_act]}, + attrs={'axis': 1}, + ) else: pre_act = pre_bias @@ -2834,16 +3192,19 @@ class SequenceConv(layers.Layer): Variable: output of sequence_conv """ - def __init__(self, - name_scope, - num_filters, - filter_size=3, - filter_stride=1, - padding=None, - bias_attr=None, - param_attr=None, - act=None): - assert not _non_static_mode( + def __init__( + self, + name_scope, + num_filters, + filter_size=3, + filter_stride=1, + padding=None, + bias_attr=None, + param_attr=None, + act=None, + ): + assert ( + not _non_static_mode() ), "SequenceConv is not supported by dynamic graph mode yet!" super(SequenceConv, self).__init__(name_scope) self._num_filters = num_filters @@ -2857,39 +3218,43 @@ def __init__(self, def _build_once(self, input): self._dtype = self._helper.input_dtype(input) filter_shape = [self._filter_size * input.shape[1], self._num_filters] - self.weight = self.create_parameter(attr=self._param_attr, - shape=filter_shape, - dtype=self._dtype) + self.weight = self.create_parameter( + attr=self._param_attr, shape=filter_shape, dtype=self._dtype + ) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._num_filters], - dtype=self._dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[self._num_filters], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input): pre_bias = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type='sequence_conv', - inputs={ - 'X': [input], - 'Filter': [self.weight], - }, - outputs={"Out": pre_bias}, - attrs={ - 'contextStride': self._filter_stride, - 'contextStart': -int(self._filter_size // 2), - 'contextLength': self._filter_size - }) + self._helper.append_op( + type='sequence_conv', + inputs={ + 'X': [input], + 'Filter': [self.weight], + }, + outputs={"Out": pre_bias}, + attrs={ + 'contextStride': self._filter_stride, + 'contextStart': -int(self._filter_size // 2), + 'contextLength': self._filter_size, + }, + ) if self.bias is not None: pre_act = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [self.bias] - }, - outputs={'Out': [pre_act]}, - attrs={'axis': 1}) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [self.bias]}, + outputs={'Out': [pre_act]}, + attrs={'axis': 1}, + ) else: pre_act = pre_bias @@ -2944,12 +3309,11 @@ class RowConv(layers.Layer): """ - def __init__(self, - name_scope, - future_context_size, - param_attr=None, - act=None): - assert not _non_static_mode( + def __init__( + self, name_scope, future_context_size, param_attr=None, act=None + ): + assert ( + not _non_static_mode() ), "RowConv is not supported by dynamic graph mode yet!" super(RowConv, self).__init__(name_scope) self._act = act @@ -2959,27 +3323,28 @@ def __init__(self, def _build_once(self, input): self._dtype = self._helper.input_dtype(input) filter_shape = [self._future_context_size + 1, input.shape[1]] - self.weight = self.create_parameter(attr=self._param_attr, - shape=filter_shape, - dtype=self._dtype, - is_bias=False) + self.weight = self.create_parameter( + attr=self._param_attr, + shape=filter_shape, + dtype=self._dtype, + is_bias=False, + ) def forward(self, input): out = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type='row_conv', - inputs={ - 'X': [input], - 'Filter': [self.weight] - }, - outputs={'Out': [out]}) + self._helper.append_op( + type='row_conv', + inputs={'X': [input], 'Filter': [self.weight]}, + outputs={'Out': [out]}, + ) return self._helper.append_activation(out, act=self._act) class GroupNorm(layers.Layer): """ :alias_main: paddle.nn.GroupNorm - :alias: paddle.nn.GroupNorm,paddle.nn.layer.GroupNorm,paddle.nn.layer.norm.GroupNorm - :old_api: paddle.fluid.dygraph.GroupNorm + :alias: paddle.nn.GroupNorm,paddle.nn.layer.GroupNorm,paddle.nn.layer.norm.GroupNorm + :old_api: paddle.fluid.dygraph.GroupNorm This interface is used to construct a callable object of the ``GroupNorm`` class. For more details, refer to code examples. @@ -3016,15 +3381,17 @@ class GroupNorm(layers.Layer): """ - def __init__(self, - channels, - groups, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - act=None, - data_layout='NCHW', - dtype='float32'): + def __init__( + self, + channels, + groups, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + act=None, + data_layout='NCHW', + dtype='float32', + ): super(GroupNorm, self).__init__() self._param_attr = param_attr self._bias_attr = bias_attr @@ -3038,31 +3405,44 @@ def __init__(self, param_shape = [self._channels] - self.weight = self.create_parameter(attr=self._param_attr or False, - shape=param_shape, - dtype=self._dtype, - default_initializer=Constant(1.0)) + self.weight = self.create_parameter( + attr=self._param_attr or False, + shape=param_shape, + dtype=self._dtype, + default_initializer=Constant(1.0), + ) - self.bias = self.create_parameter(attr=self._bias_attr or False, - shape=param_shape, - dtype=self._dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr or False, + shape=param_shape, + dtype=self._dtype, + is_bias=True, + ) def forward(self, input): mean_out = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) variance_out = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) if in_dygraph_mode(): - out = _C_ops.group_norm(input, self.weight, self.bias, - self._epsilon, self._groups, "NCHW") + out = _C_ops.group_norm( + input, + self.weight, + self.bias, + self._epsilon, + self._groups, + "NCHW", + ) return dygraph_utils._append_activation_in_dygraph(out, self._act) elif _in_legacy_dygraph(): attrs = ('epsilon', self._epsilon, 'groups', self._groups) - out, _, _ = _legacy_C_ops.group_norm(input, self.weight, self.bias, - mean_out, variance_out, *attrs) + out, _, _ = _legacy_C_ops.group_norm( + input, self.weight, self.bias, mean_out, variance_out, *attrs + ) return dygraph_utils._append_activation_in_dygraph(out, self._act) else: @@ -3074,19 +3454,19 @@ def forward(self, input): # create output group_norm_out = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - - self._helper.append_op(type="group_norm", - inputs=inputs, - outputs={ - "Y": group_norm_out, - "Mean": mean_out, - "Variance": variance_out, - }, - attrs={ - "epsilon": self._epsilon, - "groups": self._groups - }) + dtype=self._dtype + ) + + self._helper.append_op( + type="group_norm", + inputs=inputs, + outputs={ + "Y": group_norm_out, + "Mean": mean_out, + "Variance": variance_out, + }, + attrs={"epsilon": self._epsilon, "groups": self._groups}, + ) return self._helper.append_activation(group_norm_out, self._act) @@ -3150,12 +3530,9 @@ class SpectralNorm(layers.Layer): """ - def __init__(self, - weight_shape, - dim=0, - power_iters=1, - eps=1e-12, - dtype='float32'): + def __init__( + self, weight_shape, dim=0, power_iters=1, eps=1e-12, dtype='float32' + ): super(SpectralNorm, self).__init__() self._power_iters = power_iters self._eps = eps @@ -3163,48 +3540,61 @@ def __init__(self, self._dtype = dtype self._weight_shape = list(weight_shape) - assert np.prod(self._weight_shape) > 0,\ - "Any dimension of `weight_shape` cannot be equal to 0." - assert dim < len(self._weight_shape), \ - ("The input `dim` should be less than the " + assert ( + np.prod(self._weight_shape) > 0 + ), "Any dimension of `weight_shape` cannot be equal to 0." + assert dim < len(self._weight_shape), ( + "The input `dim` should be less than the " "length of `weight_shape`, but received dim=" - "{}".format(dim)) + "{}".format(dim) + ) h = self._weight_shape[self._dim] w = np.prod(self._weight_shape) // h - self.weight_u = self.create_parameter(attr=ParamAttr(), - shape=[h], - dtype=self._dtype, - default_initializer=Normal( - 0., 1.)) + self.weight_u = self.create_parameter( + attr=ParamAttr(), + shape=[h], + dtype=self._dtype, + default_initializer=Normal(0.0, 1.0), + ) self.weight_u.stop_gradient = True - self.weight_v = self.create_parameter(attr=ParamAttr(), - shape=[w], - dtype=self._dtype, - default_initializer=Normal( - 0., 1.)) + self.weight_v = self.create_parameter( + attr=ParamAttr(), + shape=[w], + dtype=self._dtype, + default_initializer=Normal(0.0, 1.0), + ) self.weight_v.stop_gradient = True def forward(self, weight): if in_dygraph_mode(): - return _C_ops.spectral_norm(weight, self.weight_u, self.weight_v, - self._dim, self._power_iters, self._eps) + return _C_ops.spectral_norm( + weight, + self.weight_u, + self.weight_v, + self._dim, + self._power_iters, + self._eps, + ) - check_variable_and_dtype(weight, "weight", ['float32', 'float64'], - 'SpectralNorm') + check_variable_and_dtype( + weight, "weight", ['float32', 'float64'], 'SpectralNorm' + ) inputs = {'Weight': weight, 'U': self.weight_u, 'V': self.weight_v} out = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type="spectral_norm", - inputs=inputs, - outputs={ - "Out": out, - }, - attrs={ - "dim": self._dim, - "power_iters": self._power_iters, - "eps": self._eps, - }) + self._helper.append_op( + type="spectral_norm", + inputs=inputs, + outputs={ + "Out": out, + }, + attrs={ + "dim": self._dim, + "power_iters": self._power_iters, + "eps": self._eps, + }, + ) return out @@ -3254,16 +3644,18 @@ class TreeConv(layers.Layer): ret = treeConv(fluid.dygraph.base.to_variable(nodes_vector), fluid.dygraph.base.to_variable(edge_set)) """ - def __init__(self, - feature_size, - output_size, - num_filters=1, - max_depth=2, - act='tanh', - param_attr=None, - bias_attr=None, - name=None, - dtype='float32'): + def __init__( + self, + feature_size, + output_size, + num_filters=1, + max_depth=2, + act='tanh', + param_attr=None, + bias_attr=None, + name=None, + dtype='float32', + ): super(TreeConv, self).__init__() self._name = name self._feature_size = feature_size @@ -3276,45 +3668,52 @@ def __init__(self, self._dtype = dtype w_shape = [self._feature_size, 3, self._output_size, self._num_filters] if self._bias_attr: - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._num_filters], - dtype=self._dtype, - is_bias=True) - self.weight = self.create_parameter(attr=self._param_attr, - shape=w_shape, - dtype=self._dtype, - is_bias=False) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[self._num_filters], + dtype=self._dtype, + is_bias=True, + ) + self.weight = self.create_parameter( + attr=self._param_attr, + shape=w_shape, + dtype=self._dtype, + is_bias=False, + ) def forward(self, nodes_vector, edge_set): check_type(nodes_vector, 'nodes_vector', (Variable), 'TreeConv') check_type(edge_set, 'edge_set', (Variable), 'TreeConv') if self._name: - out = self.create_variable(name=self._name, - dtype=self._dtype, - persistable=False) + out = self.create_variable( + name=self._name, dtype=self._dtype, persistable=False + ) else: out = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='tree_conv', - inputs={ - 'NodesVector': nodes_vector, - 'EdgeSet': edge_set, - 'Filter': self.weight - }, - outputs={ - 'Out': out, - }, - attrs={'max_depth': self._max_depth}) + dtype=self._dtype + ) + self._helper.append_op( + type='tree_conv', + inputs={ + 'NodesVector': nodes_vector, + 'EdgeSet': edge_set, + 'Filter': self.weight, + }, + outputs={ + 'Out': out, + }, + attrs={'max_depth': self._max_depth}, + ) if self._bias_attr: pre_activation = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [out], - 'Y': [self.bias] - }, - outputs={'Out': [pre_activation]}, - attrs={'axis': 1}) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [out], 'Y': [self.bias]}, + outputs={'Out': [pre_activation]}, + attrs={'axis': 1}, + ) else: pre_activation = out return self._helper.append_activation(pre_activation, act=self._act) @@ -3353,7 +3752,7 @@ def __init__(self, start_axis=1, stop_axis=-1): self.stop_axis = stop_axis def forward(self, input): - out = paddle.tensor.manipulation.flatten(input, - start_axis=self.start_axis, - stop_axis=self.stop_axis) + out = paddle.tensor.manipulation.flatten( + input, start_axis=self.start_axis, stop_axis=self.stop_axis + ) return out diff --git a/python/paddle/fluid/dygraph/parallel.py b/python/paddle/fluid/dygraph/parallel.py index bd0a4c9e5d221..51e0527e4fa99 100644 --- a/python/paddle/fluid/dygraph/parallel.py +++ b/python/paddle/fluid/dygraph/parallel.py @@ -30,7 +30,12 @@ from paddle.utils import deprecated from ..layers import collective from paddle.fluid.dygraph import base as imperative_base -from paddle.fluid.framework import ParamBase, _in_legacy_dygraph, _non_static_mode, in_dygraph_mode +from paddle.fluid.framework import ( + ParamBase, + _in_legacy_dygraph, + _non_static_mode, + in_dygraph_mode, +) __all__ = ["prepare_context", "ParallelEnv", "DataParallel"] @@ -50,24 +55,29 @@ def prepare_context(strategy=None): strategy.current_endpoint = Env().current_endpoint if strategy.nranks < 2: return - assert framework._non_static_mode() is True, \ - "dygraph.prepare_context should be used with dygraph mode." + assert ( + framework._non_static_mode() is True + ), "dygraph.prepare_context should be used with dygraph mode." place = framework._current_expected_place() - assert place is not None, \ - "dygraph.prepare_context should be used in fluid.dygraph.guard(place) guard." + assert ( + place is not None + ), "dygraph.prepare_context should be used in fluid.dygraph.guard(place) guard." if not parallel_helper._is_parallel_ctx_initialized(): if isinstance(place, core.CUDAPlace): parallel_helper._set_parallel_ctx( - core.NCCLParallelContext(strategy, place)) + core.NCCLParallelContext(strategy, place) + ) elif isinstance(place, core.XPUPlace): parallel_helper._set_parallel_ctx( - core.BKCLParallelContext(strategy, place)) + core.BKCLParallelContext(strategy, place) + ) elif isinstance(place, core.NPUPlace): parallel_helper._set_parallel_ctx( - core.HCCLParallelContext(strategy, place)) + core.HCCLParallelContext(strategy, place) + ) else: # TODO(Yancey1989): add Gloo Parallel Context to support CPU parallel computation - assert ("Only support CUDAPlace or XPUPlace or NPUPlace for now.") + assert "Only support CUDAPlace or XPUPlace or NPUPlace for now." parallel_helper._init_parallel_ctx() return strategy @@ -122,9 +132,11 @@ def __init__(self): # imperative only support one gpu or xpu if self._device_type != "": FLAGS_selected_custom_devices = 'FLAGS_selected_{}s'.format( - self._device_type) - selected_custom_devices = os.getenv(FLAGS_selected_custom_devices, - "0").split(",") + self._device_type + ) + selected_custom_devices = os.getenv( + FLAGS_selected_custom_devices, "0" + ).split(",") self._device_id = int(selected_custom_devices[0]) else: if core.is_compiled_with_cuda(): @@ -140,14 +152,17 @@ def __init__(self): selected_mlus = os.getenv("FLAGS_selected_mlus", "0").split(",") self._device_id = int(selected_mlus[0]) - self._trainer_endpoints = os.getenv("PADDLE_TRAINER_ENDPOINTS", - "").split(",") + self._trainer_endpoints = os.getenv( + "PADDLE_TRAINER_ENDPOINTS", "" + ).split(",") self._current_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT", "") self._nrings = int(os.getenv("FLAGS_nccl_nrings", "1")) - assert self._nrings > 0, \ - "nccl_nrings must be an integer greater than 0." - assert self._nrings < 9, \ - "nccl_nrings should be less than 9, which is enough in most scenarios." + assert ( + self._nrings > 0 + ), "nccl_nrings must be an integer greater than 0." + assert ( + self._nrings < 9 + ), "nccl_nrings should be less than 9, which is enough in most scenarios." @property def rank(self): @@ -297,6 +312,7 @@ def _build_default_parallel_strategy(): def _coalesce_tensors(var_groups): from ..layers import nn + coalesced_grads_and_grad_vars = [] for group_id, grad_vars in var_groups.items(): flattened_vars = [] @@ -304,43 +320,50 @@ def _coalesce_tensors(var_groups): for g_var in grad_vars: g_var_shapes.append(g_var.shape) flattened_vars.append( - nn.reshape(x=g_var, shape=[np.prod(g_var.shape)])) + nn.reshape(x=g_var, shape=[np.prod(g_var.shape)]) + ) coalesced_grad = nn.concat(flattened_vars) coalesced_grads_and_grad_vars.append( - [coalesced_grad, grad_vars, g_var_shapes]) + [coalesced_grad, grad_vars, g_var_shapes] + ) return coalesced_grads_and_grad_vars @framework.dygraph_only def _reshape_inplace(x, shape): x_shape = framework._varbase_creator(dtype=x.dtype) - framework._dygraph_tracer().trace_op(type="reshape2", - inputs={'X': x}, - outputs={ - 'Out': x, - 'XShape': x_shape - }, - attrs={'shape': shape}) + framework._dygraph_tracer().trace_op( + type="reshape2", + inputs={'X': x}, + outputs={'Out': x, 'XShape': x_shape}, + attrs={'shape': shape}, + ) @framework.dygraph_only def _split_tensors(coalesced_grads_and_grad_vars): if _in_legacy_dygraph(): - for coalesced_grad, origin_grad_vars, grad_shapes in coalesced_grads_and_grad_vars: + for ( + coalesced_grad, + origin_grad_vars, + grad_shapes, + ) in coalesced_grads_and_grad_vars: grad_var_len = [np.prod(g_shape) for g_shape in grad_shapes] framework._dygraph_tracer().trace_op( type='split', inputs={'X': coalesced_grad}, outputs={'Out': origin_grad_vars}, - attrs={ - 'sections': grad_var_len, - 'axis': 0 - }) + attrs={'sections': grad_var_len, 'axis': 0}, + ) for g_var, g_shape in zip(origin_grad_vars, grad_shapes): _reshape_inplace(x=g_var, shape=g_shape) assert g_var.shape == g_shape elif in_dygraph_mode(): - for coalesced_grad, origin_grad_vars, grad_shapes in coalesced_grads_and_grad_vars: + for ( + coalesced_grad, + origin_grad_vars, + grad_shapes, + ) in coalesced_grads_and_grad_vars: grad_var_len = [np.prod(g_shape) for g_shape in grad_shapes] attrs = () attrs += ('sections', grad_var_len) @@ -357,7 +380,8 @@ def scale_loss(loss): return loss loss_scale = to_variable( - np.array([ParallelEnv().world_size]).astype("float32")) + np.array([ParallelEnv().world_size]).astype("float32") + ) loss_scale.stop_gradient = True scaled_loss = loss / loss_scale return scaled_loss @@ -385,16 +409,16 @@ def build_groups(vars, group_size): @imperative_base.no_grad @framework.dygraph_only -def sync_params_buffers(model, - comm_group=None, - src_rank=0, - is_model_parallel=False): +def sync_params_buffers( + model, comm_group=None, src_rank=0, is_model_parallel=False +): model_vars = [] for _, param in model._obtain_parameters_buffers().items(): if not isinstance(param, (core.VarBase, core.eager.Tensor)): raise TypeError( - "The data type of '%s' must be Varbase or eager.Tensor" % - param.name) + "The data type of '%s' must be Varbase or eager.Tensor" + % param.name + ) # is_distributed param not need to sync when in mp mode if isinstance(param, (ParamBase, core.eager.Tensor)): @@ -416,10 +440,9 @@ def sync_params_buffers(model, coalesced_vars = build_groups(model_vars, 128 * 1024 * 1024) for coalesced_var, _, _ in coalesced_vars: - paddle.distributed.broadcast(coalesced_var, - src=src_rank, - group=comm_group, - sync_op=True) + paddle.distributed.broadcast( + coalesced_var, src=src_rank, group=comm_group, sync_op=True + ) for coalesced_var, origin_vars, var_shapes in coalesced_vars: var_len = [np.prod(v_shape) for v_shape in var_shapes] @@ -427,10 +450,8 @@ def sync_params_buffers(model, type='split', inputs={'X': coalesced_var}, outputs={'Out': origin_vars}, - attrs={ - 'sections': var_len, - 'axis': 0 - }) + attrs={'sections': var_len, 'axis': 0}, + ) class DataParallel(layers.Layer): @@ -594,25 +615,30 @@ def forward(self, inputs): """ - def __init__(self, - layers, - strategy=None, - comm_buffer_size=25, - last_comm_buffer_size=1, - find_unused_parameters=False, - group=None): - super(DataParallel, - self).__init__(layers.full_name() + "_data_parallel") - - assert _non_static_mode(), \ - "It's not supported to construct DataParallel in static mode." + def __init__( + self, + layers, + strategy=None, + comm_buffer_size=25, + last_comm_buffer_size=1, + find_unused_parameters=False, + group=None, + ): + super(DataParallel, self).__init__( + layers.full_name() + "_data_parallel" + ) + + assert ( + _non_static_mode() + ), "It's not supported to construct DataParallel in static mode." self._layers = layers self.find_unused_parameters = find_unused_parameters self.grad_need_sync = True self.group = group - self.var_dtype = core.eager.Tensor if in_dygraph_mode( - ) else core.VarBase + self.var_dtype = ( + core.eager.Tensor if in_dygraph_mode() else core.VarBase + ) # NOTE(chenweihang): The ParallelStrategy here is not strictly a strategy. # It just stores some environment variables, which can be constructed by @@ -625,16 +651,21 @@ def __init__(self, if self._strategy.nranks > 1: # check the environment - assert parallel_helper.__parallel_ctx__clz__ is not None, \ - "ParallelContext must be initialized before. You should use init_parallel_env() before" \ - "constructing the DataParallel." + assert parallel_helper.__parallel_ctx__clz__ is not None, ( + "ParallelContext must be initialized before. You should use init_parallel_env() before" + "constructing the DataParallel." + ) if in_dygraph_mode(): - self.group = paddle.distributed.collective._get_default_group( - ) if self.group is None else self.group + self.group = ( + paddle.distributed.collective._get_default_group() + if self.group is None + else self.group + ) - assert isinstance(self.group, paddle.distributed.collective.Group), \ - "ProcessGroup must be an instance of Group in DataParallel." + assert isinstance( + self.group, paddle.distributed.collective.Group + ), "ProcessGroup must be an instance of Group in DataParallel." # sync buffer and params # TODO(liuyuhui) Currently not support xpu. xpu is @@ -647,14 +678,17 @@ def __init__(self, # the size of the group, Default: 1MB. The role of this small group is: # when the last group allreduce, the overlap cannot work. Making the # the last group small is useful to improve performance. - self.last_comm_buffer_size = int(last_comm_buffer_size * 1024 * - 1024) + self.last_comm_buffer_size = int( + last_comm_buffer_size * 1024 * 1024 + ) self.init_reducer() else: - warnings.warn("The program will return to single-card operation. " - "Please check 1, whether you use spawn or fleetrun " - "to start the program. 2, Whether it is a multi-card " - "program. 3, Is the current environment multi-card.") + warnings.warn( + "The program will return to single-card operation. " + "Please check 1, whether you use spawn or fleetrun " + "to start the program. 2, Whether it is a multi-card " + "program. 3, Is the current environment multi-card." + ) def init_reducer(self): layers_param = [] @@ -665,16 +699,19 @@ def init_reducer(self): continue params_set.add(param) if not isinstance(param, self.var_dtype): - raise TypeError("The data type of '%s' must be '%s'" % - (param.name, self.var_dtype)) + raise TypeError( + "The data type of '%s' must be '%s'" + % (param.name, self.var_dtype) + ) if param.trainable: layers_param.append((sublayer, param)) trainable_parameters = [param for _, param in layers_param] - assert len(trainable_parameters) > 0, \ - "This model does not have any parameters to train, and " \ + assert len(trainable_parameters) > 0, ( + "This model does not have any parameters to train, and " "does not need to use DataParallel" + ) # NOTE(shenliang03): Here we can only use the attributes to judge whether # parameter is sparse(or SelectedRows). The reason is that the sparse message @@ -695,24 +732,34 @@ def check_layer_sparse(sublayer): if in_dygraph_mode(): self.group_indices = core.eager_assign_group_by_size( - trainable_parameters, is_sparse_gradient, - [self.last_comm_buffer_size, self.comm_buffer_size]) + trainable_parameters, + is_sparse_gradient, + [self.last_comm_buffer_size, self.comm_buffer_size], + ) self._reducer = core.EagerReducer( - trainable_parameters, list(reversed(self.group_indices)), - is_sparse_gradient, self.group.process_group, + trainable_parameters, + list(reversed(self.group_indices)), + is_sparse_gradient, + self.group.process_group, [self.last_comm_buffer_size, self.comm_buffer_size], - self.find_unused_parameters) + self.find_unused_parameters, + ) elif _in_legacy_dygraph(): self.group_indices = core.assign_group_by_size( - trainable_parameters, is_sparse_gradient, - [self.last_comm_buffer_size, self.comm_buffer_size]) + trainable_parameters, + is_sparse_gradient, + [self.last_comm_buffer_size, self.comm_buffer_size], + ) self._reducer = core.Reducer( - trainable_parameters, list(reversed(self.group_indices)), - is_sparse_gradient, parallel_helper.__parallel_ctx__clz__, + trainable_parameters, + list(reversed(self.group_indices)), + is_sparse_gradient, + parallel_helper.__parallel_ctx__clz__, [self.last_comm_buffer_size, self.comm_buffer_size], - self.find_unused_parameters) + self.find_unused_parameters, + ) def _find_varbase(self, obj): var_type = core.eager.Tensor if in_dygraph_mode() else core.VarBase @@ -771,14 +818,19 @@ def forward(self, x): def forward(self, *inputs, **kwargs): outputs = self._layers(*inputs, **kwargs) - if self._strategy.nranks > 1 and framework._dygraph_tracer( - )._has_grad and self.grad_need_sync: - self._reducer.prepare_for_backward(list( - self._find_varbase(outputs))) + if ( + self._strategy.nranks > 1 + and framework._dygraph_tracer()._has_grad + and self.grad_need_sync + ): + self._reducer.prepare_for_backward( + list(self._find_varbase(outputs)) + ) return outputs - @deprecated(since="2.0.0", - reason="This method does not need to be called anymore.") + @deprecated( + since="2.0.0", reason="This method does not need to be called anymore." + ) def scale_loss(self, loss): """ Deprecated method, now ``scale_loss`` is an empty method, @@ -786,8 +838,9 @@ def scale_loss(self, loss): """ return loss - @deprecated(since="2.0.0", - reason="This method does not need to be called anymore.") + @deprecated( + since="2.0.0", reason="This method does not need to be called anymore." + ) def apply_collective_grads(self): """ Deprecated method, now ``apply_collective_grads`` is an empty method, @@ -795,10 +848,12 @@ def apply_collective_grads(self): """ return - def state_dict(self, - destination=None, - include_sublayers=True, - structured_name_prefix=""): + def state_dict( + self, + destination=None, + include_sublayers=True, + structured_name_prefix="", + ): ''' Get all parameters and persistable buffers of current layer and its sub-layers. And set them into a dict @@ -828,7 +883,8 @@ def state_dict(self, return self._layers.state_dict( destination=destination, include_sublayers=include_sublayers, - structured_name_prefix=structured_name_prefix) + structured_name_prefix=structured_name_prefix, + ) @framework.deprecate_stat_dict def set_state_dict(self, state_dict, use_structured_name=True): @@ -861,8 +917,9 @@ def set_state_dict(self, state_dict, use_structured_name=True): ''' - self._layers.set_state_dict(state_dict, - use_structured_name=use_structured_name) + self._layers.set_state_dict( + state_dict, use_structured_name=use_structured_name + ) # [aliases] Compatible with old method names set_dict = set_state_dict diff --git a/python/paddle/fluid/dygraph/parallel_helper.py b/python/paddle/fluid/dygraph/parallel_helper.py index bc0bb4603525e..32d7974af31bb 100644 --- a/python/paddle/fluid/dygraph/parallel_helper.py +++ b/python/paddle/fluid/dygraph/parallel_helper.py @@ -20,8 +20,10 @@ def _is_data_parallel_mode(): global __parallel_ctx__clz__ - return __parallel_ctx__clz__ is not None and int( - os.getenv("PADDLE_TRAINERS_NUM", "1")) > 1 + return ( + __parallel_ctx__clz__ is not None + and int(os.getenv("PADDLE_TRAINERS_NUM", "1")) > 1 + ) def _is_parallel_ctx_initialized(): @@ -31,15 +33,17 @@ def _is_parallel_ctx_initialized(): def _set_parallel_ctx(ccl_parallel_context): global __parallel_ctx__clz__ - assert __parallel_ctx__clz__ is None, \ - "ParallelContext can only be initialized once." + assert ( + __parallel_ctx__clz__ is None + ), "ParallelContext can only be initialized once." __parallel_ctx__clz__ = ccl_parallel_context def _init_parallel_ctx(): global __parallel_ctx__clz__ - assert __parallel_ctx__clz__ is not None, \ - "ParallelContext should be initialized." + assert ( + __parallel_ctx__clz__ is not None + ), "ParallelContext should be initialized." __parallel_ctx__clz__.init() @@ -47,7 +51,8 @@ def _broadcast_parameters(parameters): for param in parameters: # In model parallel, some parameters are split into multiple devices, # so we could not broadcast these parameters. - if param.is_distributed: continue + if param.is_distributed: + continue if isinstance(param, Parameter) and param.trainable: collective._broadcast(param, 0, sync_mode=True) diff --git a/python/paddle/fluid/dygraph/rnn.py b/python/paddle/fluid/dygraph/rnn.py index e70533662976f..4ae9c5ba5fec4 100644 --- a/python/paddle/fluid/dygraph/rnn.py +++ b/python/paddle/fluid/dygraph/rnn.py @@ -13,7 +13,16 @@ # limitations under the License. from . import Layer -from ..layers import sigmoid, tanh, concat, fill_constant, matmul, elementwise_add, elementwise_mul, split +from ..layers import ( + sigmoid, + tanh, + concat, + fill_constant, + matmul, + elementwise_add, + elementwise_mul, + split, +) import copy __all__ = ['LSTMCell', 'GRUCell'] @@ -111,16 +120,18 @@ class LSTMCell(Layer): """ - def __init__(self, - hidden_size, - input_size, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - forget_bias=1.0, - use_cudnn_impl=True, - dtype='float64'): + def __init__( + self, + hidden_size, + input_size, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + forget_bias=1.0, + use_cudnn_impl=True, + dtype='float64', + ): super(LSTMCell, self).__init__(dtype) self._hidden_size = hidden_size @@ -134,7 +145,10 @@ def __init__(self, if self._use_cudnn_impl: - if self._param_attr is not None and self._param_attr.name is not None: + if ( + self._param_attr is not None + and self._param_attr.name is not None + ): weight_ih_param_attr = copy.deepcopy(self._param_attr) weight_hh_param_attr = copy.deepcopy(self._param_attr) weight_ih_param_attr.name += "_weight_ih" @@ -155,40 +169,50 @@ def __init__(self, self._weight_ih = self.create_parameter( attr=weight_ih_param_attr, shape=[4 * self._hidden_size, self._input_size], - dtype=self._dtype) + dtype=self._dtype, + ) self._weight_hh = self.create_parameter( attr=weight_hh_param_attr, shape=[4 * self._hidden_size, self._hidden_size], - dtype=self._dtype) - - self._bias_ih = self.create_parameter(attr=bias_ih_param_attr, - shape=[4 * self._hidden_size], - dtype=self._dtype, - is_bias=True) - self._bias_hh = self.create_parameter(attr=bias_hh_param_attr, - shape=[4 * self._hidden_size], - dtype=self._dtype, - is_bias=True) + dtype=self._dtype, + ) + + self._bias_ih = self.create_parameter( + attr=bias_ih_param_attr, + shape=[4 * self._hidden_size], + dtype=self._dtype, + is_bias=True, + ) + self._bias_hh = self.create_parameter( + attr=bias_hh_param_attr, + shape=[4 * self._hidden_size], + dtype=self._dtype, + is_bias=True, + ) else: - self._forget_bias = fill_constant([1], - dtype=dtype, - value=forget_bias) + self._forget_bias = fill_constant( + [1], dtype=dtype, value=forget_bias + ) self._forget_bias.stop_gradient = False self._weight = self.create_parameter( attr=self._param_attr, shape=[ - self._input_size + self._hidden_size, 4 * self._hidden_size + self._input_size + self._hidden_size, + 4 * self._hidden_size, ], - dtype=dtype) + dtype=dtype, + ) - self._bias = self.create_parameter(attr=self._bias_attr, - shape=[4 * self._hidden_size], - dtype=dtype, - is_bias=True) + self._bias = self.create_parameter( + attr=self._bias_attr, + shape=[4 * self._hidden_size], + dtype=dtype, + is_bias=True, + ) def forward(self, input, pre_hidden, pre_cell): @@ -226,9 +250,12 @@ def forward(self, input, pre_hidden, pre_cell): new_cell = elementwise_add( elementwise_mul( pre_cell, - self._gate_activation(elementwise_add(f, - self._forget_bias))), - elementwise_mul(sigmoid(i), tanh(j))) + self._gate_activation( + elementwise_add(f, self._forget_bias) + ), + ), + elementwise_mul(sigmoid(i), tanh(j)), + ) new_hidden = self._activation(new_cell) * self._gate_activation(o) return new_hidden, new_cell @@ -312,15 +339,17 @@ class GRUCell(Layer): """ - def __init__(self, - hidden_size, - input_size, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - use_cudnn_impl=True, - dtype='float64'): + def __init__( + self, + hidden_size, + input_size, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + use_cudnn_impl=True, + dtype='float64', + ): super(GRUCell, self).__init__() self._hidden_size = hidden_size @@ -334,7 +363,10 @@ def __init__(self, if self._use_cudnn_impl: - if self._param_attr is not None and self._param_attr.name is not None: + if ( + self._param_attr is not None + and self._param_attr.name is not None + ): weight_ih_param_attr = copy.deepcopy(self._param_attr) weight_hh_param_attr = copy.deepcopy(self._param_attr) weight_ih_param_attr.name += "_weight_ih" @@ -355,25 +387,34 @@ def __init__(self, self._weight_ih = self.create_parameter( attr=weight_ih_param_attr, shape=[3 * self._hidden_size, self._input_size], - dtype=self._dtype) + dtype=self._dtype, + ) self._weight_hh = self.create_parameter( attr=weight_hh_param_attr, shape=[3 * self._hidden_size, self._hidden_size], - dtype=self._dtype) - - self._bias_ih = self.create_parameter(attr=bias_ih_param_attr, - shape=[3 * self._hidden_size], - dtype=self._dtype, - is_bias=True) - self._bias_hh = self.create_parameter(attr=bias_hh_param_attr, - shape=[3 * self._hidden_size], - dtype=self._dtype, - is_bias=True) + dtype=self._dtype, + ) + + self._bias_ih = self.create_parameter( + attr=bias_ih_param_attr, + shape=[3 * self._hidden_size], + dtype=self._dtype, + is_bias=True, + ) + self._bias_hh = self.create_parameter( + attr=bias_hh_param_attr, + shape=[3 * self._hidden_size], + dtype=self._dtype, + is_bias=True, + ) else: - if self._param_attr is not None and self._param_attr.name is not None: + if ( + self._param_attr is not None + and self._param_attr.name is not None + ): gate_weight_param_attr = copy.deepcopy(self._param_attr) candidate_weight_param_attr = copy.deepcopy(self._param_attr) gate_weight_param_attr.name += "_gate_weight" @@ -394,25 +435,30 @@ def __init__(self, self._gate_weight = self.create_parameter( attr=gate_weight_param_attr, shape=[ - self._input_size + self._hidden_size, 2 * self._hidden_size + self._input_size + self._hidden_size, + 2 * self._hidden_size, ], - dtype=dtype) + dtype=dtype, + ) self._candidate_weight = self.create_parameter( attr=candidate_weight_param_attr, shape=[self._input_size + self._hidden_size, self._hidden_size], - dtype=dtype) + dtype=dtype, + ) self._gate_bias = self.create_parameter( attr=gate_bias_param_attr, shape=[2 * self._hidden_size], dtype=dtype, - is_bias=True) + is_bias=True, + ) self._candidate_bias = self.create_parameter( attr=candidate_bias_param_attr, shape=[self._hidden_size], dtype=dtype, - is_bias=True) + is_bias=True, + ) def forward(self, input, pre_hidden): @@ -450,8 +496,9 @@ def forward(self, input, pre_hidden): r_hidden = r * pre_hidden - candidate = matmul(concat([input, r_hidden], 1), - self._candidate_weight) + candidate = matmul( + concat([input, r_hidden], 1), self._candidate_weight + ) candidate = elementwise_add(candidate, self._candidate_bias) c = self._activation(candidate) diff --git a/python/paddle/fluid/dygraph/tracer.py b/python/paddle/fluid/dygraph/tracer.py index 1553efad41a25..a274296cce39e 100644 --- a/python/paddle/fluid/dygraph/tracer.py +++ b/python/paddle/fluid/dygraph/tracer.py @@ -24,7 +24,7 @@ "src_index": "Src_index", "dst_index": "Dst_index", "out": "Out", - "dst_count": "Dst_count" + "dst_count": "Dst_count", }, "matmul_v2": { "final_op_name": "matmul", @@ -106,13 +106,15 @@ def __init__(self): self._train_mode = True - def eager_legacy_trace_op(self, - op_type, - inputs, - outputs, - attrs, - stop_gradient=False, - inplace_map=None): + def eager_legacy_trace_op( + self, + op_type, + inputs, + outputs, + attrs, + stop_gradient=False, + inplace_map=None, + ): function_ptr = _legacy_C_ops.__dict__[op_type] core_ops_args_info = _legacy_C_ops.get_core_ops_args_info() @@ -186,14 +188,17 @@ def eager_legacy_trace_op(self, if isinstance(returns[i], list): for j in range(len(returns[i])): outputs[retname][j].reconstruct_from_( - returns[i][j], False) + returns[i][j], False + ) else: if isinstance(outputs[retname], list): outputs[retname][0].reconstruct_from_( - returns[i], False) + returns[i], False + ) else: outputs[retname].reconstruct_from_( - returns[i], False) + returns[i], False + ) elif isinstance(returns, list): assert len(outputs.keys()) == 1 key = list(outputs.keys())[0] @@ -207,13 +212,15 @@ def eager_legacy_trace_op(self, else: outputs[key].reconstruct_from_(returns, False) - def eager_trace_op(self, - op_type, - inputs, - outputs, - attrs, - stop_gradient=False, - inplace_map=None): + def eager_trace_op( + self, + op_type, + inputs, + outputs, + attrs, + stop_gradient=False, + inplace_map=None, + ): assert op_type in name_mapping.keys() op_type = name_mapping[op_type]["final_op_name"] @@ -273,7 +280,8 @@ def eager_trace_op(self, if isinstance(returns[i], list): for j in range(len(returns[i])): outputs[retname][j].reconstruct_from_( - returns[i][j], False) + returns[i][j], False + ) else: outputs[retname][0].reconstruct_from_(returns[i], False) elif isinstance(returns, list): @@ -289,13 +297,15 @@ def eager_trace_op(self, else: outputs[key].reconstruct_from_(returns, False) - def trace_op(self, - type, - inputs, - outputs, - attrs, - stop_gradient=False, - inplace_map=None): + def trace_op( + self, + type, + inputs, + outputs, + attrs, + stop_gradient=False, + inplace_map=None, + ): if not framework._in_legacy_dygraph(): # inputs : {"sum": [tensor], ...} # outputs : {"sum": [tensor], ...} @@ -303,16 +313,23 @@ def trace_op(self, type = name_mapping[type]["final_op_name"] assert type in _legacy_C_ops.__dict__ - self.eager_trace_op(type, inputs, outputs, attrs, stop_gradient, - inplace_map) + self.eager_trace_op( + type, inputs, outputs, attrs, stop_gradient, inplace_map + ) else: - self.eager_legacy_trace_op(type, inputs, outputs, attrs, - stop_gradient, inplace_map) + self.eager_legacy_trace_op( + type, inputs, outputs, attrs, stop_gradient, inplace_map + ) else: - self.trace(type, inputs, outputs, attrs, - framework._current_expected_place(), self._has_grad - and not stop_gradient, - inplace_map if inplace_map else {}) + self.trace( + type, + inputs, + outputs, + attrs, + framework._current_expected_place(), + self._has_grad and not stop_gradient, + inplace_map if inplace_map else {}, + ) def train_mode(self): self._train_mode = True diff --git a/python/paddle/fluid/dygraph/varbase_patch_methods.py b/python/paddle/fluid/dygraph/varbase_patch_methods.py index 618bcd540f0a7..7dbcbb6f98efd 100644 --- a/python/paddle/fluid/dygraph/varbase_patch_methods.py +++ b/python/paddle/fluid/dygraph/varbase_patch_methods.py @@ -23,7 +23,15 @@ from ..framework import convert_np_dtype_to_dtype_, _in_legacy_dygraph from .. import core from .. import unique_name -from ..framework import Variable, Parameter, ParamBase, _getitem_impl_, _setitem_impl_, EagerParamBase, in_dygraph_mode +from ..framework import ( + Variable, + Parameter, + ParamBase, + _getitem_impl_, + _setitem_impl_, + EagerParamBase, + in_dygraph_mode, +) from .base import switch_to_static_graph from .math_op_patch import monkey_patch_math_varbase from .parallel import scale_loss @@ -43,8 +51,9 @@ class TensorHookRemoveHelper(object): """ def __init__(self, tensor, hook_id): - self._tensor = tensor if framework._in_eager_mode_ else weakref.ref( - tensor) + self._tensor = ( + tensor if framework._in_eager_mode_ else weakref.ref(tensor) + ) self._hook_id = hook_id def remove(self): @@ -62,7 +71,9 @@ def remove(self): else: warnings.warn( "The backward hook (ID: %d) of Tensor `%s` you want to remove does not exist or has been removed." - % (self._hook_id, tensor.name), RuntimeWarning) + % (self._hook_id, tensor.name), + RuntimeWarning, + ) return False @@ -70,7 +81,6 @@ def remove(self): def monkey_patch_varbase(): - @switch_to_static_graph def _to_static_var(self, to_parameter=False, **kwargs): """ @@ -111,8 +121,9 @@ def _to_static_var(self, to_parameter=False, **kwargs): attr_names = [] for name in dir(self): if name not in attr_not_need_keys: - if not inspect.ismethod(getattr( - self, name)) and not name.startswith('_'): + if not inspect.ismethod( + getattr(self, name) + ) and not name.startswith('_'): attr_names.append(name) attr_kwargs = {name: getattr(self, name) for name in attr_names} @@ -169,38 +180,45 @@ def set_value(self, value): base_tensor = core.eager.Tensor else: base_tensor = core.VarBase - assert isinstance(value, (np.ndarray, base_tensor, dict, str)), \ - "Variable set_value function, arguments type only support Variable, numpy, VarBase, dict, string." + assert isinstance( + value, (np.ndarray, base_tensor, dict, str) + ), "Variable set_value function, arguments type only support Variable, numpy, VarBase, dict, string." if isinstance(value, (dict, str)): assert len(self) == len( value ), "Variable length not match, Variable [ {} ] need tensor with length {} but load set tensor with length {}".format( - self.name, len(self), len(value)) + self.name, len(self), len(value) + ) if isinstance(value, dict): self.value().set_vocab(value) else: self.value().set_string_list(value) else: - assert self.shape == list(value.shape), \ - "Variable Shape not match, Variable [ {} ] need tensor with shape {} but load set tensor with shape {}".format( - self.name, self.shape, value.shape) + assert self.shape == list( + value.shape + ), "Variable Shape not match, Variable [ {} ] need tensor with shape {} but load set tensor with shape {}".format( + self.name, self.shape, value.shape + ) if isinstance(value, base_tensor): dtype = value.dtype else: dtype = convert_np_dtype_to_dtype_(value.dtype) - assert self.dtype == dtype, \ - "Variable dtype not match, Variable [ {} ] need tensor with dtype {} but load tensor with dtype {}".format( - self.name, self.dtype, dtype) + assert ( + self.dtype == dtype + ), "Variable dtype not match, Variable [ {} ] need tensor with dtype {} but load tensor with dtype {}".format( + self.name, self.dtype, dtype + ) # NOTE(wuweilong): self could be VarBase or Tensor, the subsequent behavior are defined in different files # if self is VarBase, method value() return Variable that bindded in imperative.cc, get_tensor() bindded in pybind.cc # if self is Tensor, method value() return self that defined in this file, get_tensor() defined in eager_method.cc # this Interface behavior will be unifed in the future. - self.value().get_tensor().set(value, - framework._current_expected_place()) + self.value().get_tensor().set( + value, framework._current_expected_place() + ) @framework.dygraph_only def backward(self, grad_tensor=None, retain_graph=False): @@ -258,7 +276,8 @@ def backward(self, grad_tensor=None, retain_graph=False): if framework._non_static_mode(): if in_profiler_mode(): record_event = profiler.RecordEvent( - "Gradient Backward", profiler.TracerEventType.Backward) + "Gradient Backward", profiler.TracerEventType.Backward + ) record_event.begin() if grad_tensor is not None: if framework._in_eager_mode_: @@ -269,9 +288,11 @@ def backward(self, grad_tensor=None, retain_graph=False): assert isinstance( grad_tensor, paddle.Tensor ), "The type of grad_tensor must be paddle.Tensor" - assert grad_tensor.shape == self.shape, \ - "Tensor shape not match, Tensor of grad_tensor [ {} ] with shape {} mismatch Tensor [ {} ] with shape {}".format( - grad_tensor.name, grad_tensor.shape, self.name, self.shape) + assert ( + grad_tensor.shape == self.shape + ), "Tensor shape not match, Tensor of grad_tensor [ {} ] with shape {} mismatch Tensor [ {} ] with shape {}".format( + grad_tensor.name, grad_tensor.shape, self.name, self.shape + ) if framework._in_eager_mode_: if grad_tensor is None: @@ -281,36 +302,46 @@ def backward(self, grad_tensor=None, retain_graph=False): if _grad_scalar: # When using amp with Fleet DistributedStrategy, we do loss scaling implicitly. self = _grad_scalar.scale(self) - if paddle.is_compiled_with_xpu() or paddle.is_compiled_with_npu( - ) or paddle.is_compiled_with_mlu(): + if ( + paddle.is_compiled_with_xpu() + or paddle.is_compiled_with_npu() + or paddle.is_compiled_with_mlu() + ): # TODO(liuyuhui): Currently only for xpu. Will be removed in the future. scaled_loss = scale_loss(self) if framework._in_eager_mode_: - core.eager.run_backward([scaled_loss], grad_tensor, - retain_graph) + core.eager.run_backward( + [scaled_loss], grad_tensor, retain_graph + ) else: - core.dygraph_run_backward([scaled_loss], [grad_tensor], - retain_graph, - framework._dygraph_tracer()) + core.dygraph_run_backward( + [scaled_loss], + [grad_tensor], + retain_graph, + framework._dygraph_tracer(), + ) else: if framework._in_eager_mode_: core.eager.run_backward([self], grad_tensor, retain_graph) else: - core.dygraph_run_backward([self], [grad_tensor], - retain_graph, - framework._dygraph_tracer()) + core.dygraph_run_backward( + [self], + [grad_tensor], + retain_graph, + framework._dygraph_tracer(), + ) if in_profiler_mode(): record_event.end() else: raise ValueError( - "Variable.backward() is only available in DyGraph mode") + "Variable.backward() is only available in DyGraph mode" + ) @framework.dygraph_only @deprecated( since="2.1.0", level=1, - reason= - "Please use tensor.grad, which returns the tensor value of the gradient." + reason="Please use tensor.grad, which returns the tensor value of the gradient.", ) def gradient(self): """ @@ -347,9 +378,10 @@ def gradient(self): new_ivar = self._grad_ivar()._copy_to(core.CPUPlace(), True) if self._grad_ivar().type == core.VarDesc.VarType.SELECTED_ROWS: - return (np.array( - new_ivar.value().get_selected_rows().get_tensor()), - np.array(new_ivar.value().get_selected_rows().rows())) + return ( + np.array(new_ivar.value().get_selected_rows().get_tensor()), + np.array(new_ivar.value().get_selected_rows().rows()), + ) else: return np.array(new_ivar.value().get_tensor()) @@ -414,7 +446,8 @@ def double_hook_fn(grad): """ if self.stop_gradient is True: raise RuntimeError( - "Cannot register hook on a tensor that stop gradient.") + "Cannot register hook on a tensor that stop gradient." + ) hook_id = self._register_grad_hook(hook) helper = TensorHookRemoveHelper(self, hook_id) @@ -430,21 +463,28 @@ def _to(self, device=None, dtype=None, blocking=None): if isinstance(device, str): device = paddle.device._convert_to_place(device) elif isinstance( - device, - (core.CPUPlace, core.CUDAPlace, core.CUDAPinnedPlace, - core.XPUPlace, core.CustomPlace)): + device, + ( + core.CPUPlace, + core.CUDAPlace, + core.CUDAPinnedPlace, + core.XPUPlace, + core.CustomPlace, + ), + ): pass else: raise ValueError( "device value error, must be str, paddle.CPUPlace(), paddle.CUDAPlace(), paddle.CUDAPinnedPlace(), paddle.XPUPlace() or paddle.CustomPlace(), but the type of device is " - + type(device).__name__) + + type(device).__name__ + ) if blocking is None: blocking = True else: assert isinstance( - blocking, - bool), "blocking value error, must be the True, False or None" + blocking, bool + ), "blocking value error, must be the True, False or None" def transform(t, device, dtype, blocking): if device is None: @@ -461,7 +501,8 @@ def transform(t, device, dtype, blocking): # waiting_alloc_memory will compute the memory space occupied by 't'. # Coefficient 1.2 is used to avoid OOM that may occur in this critical state when the memory is just enough. waiting_alloc_memory = ( - (t._numel() * size_dtype) / 256 + 1) * 256 * 1.2 + ((t._numel() * size_dtype) / 256 + 1) * 256 * 1.2 + ) gpu_memory_available = core.gpu_memory_available() if gpu_memory_available < waiting_alloc_memory: # Copy Tensor to cpu @@ -477,7 +518,8 @@ def transform(t, device, dtype, blocking): # 2. cast Tensor to dtype if dtype is not None and dtype != t_used.dtype: with paddle.fluid.framework._dygraph_place_guard( - place=t_used.place): + place=t_used.place + ): t_casted = t_used.cast(dtype=dtype) else: t_casted = t_used @@ -523,10 +565,12 @@ def grad(self): # Tensor(shape=[1], dtype=float32, place=CUDAPlace(0), stop_gradient=False, [500.]) """ - msg = 'tensor.grad will return the tensor value of the gradient.' \ - ' This is an incompatible upgrade for tensor.grad API. ' \ - ' It\'s return type changes from numpy.ndarray in version 2.0 to paddle.Tensor in version 2.1.0. ' \ + msg = ( + 'tensor.grad will return the tensor value of the gradient.' + ' This is an incompatible upgrade for tensor.grad API. ' + ' It\'s return type changes from numpy.ndarray in version 2.0 to paddle.Tensor in version 2.1.0. ' ' If you want to get the numpy value of the gradient, you can use :code:`x.grad.numpy()`' + ) warning_msg = "\033[93m\nWarning:\n%s \033[0m" % (msg) # ensure ANSI escape sequences print correctly in cmd and powershell if sys.platform.lower() == 'win32': @@ -622,9 +666,11 @@ def __str__(self): """ if framework._in_eager_mode_: from paddle.tensor.to_string import tensor_to_string + return tensor_to_string(self) else: from paddle.tensor.to_string import to_string + return to_string(self) def __deepcopy__(self, memo): @@ -667,7 +713,9 @@ def block(self): def __nonzero__(self): numel = np.prod(self.shape) - assert numel == 1, "When Variable is used as the condition of if/while , Variable can only contain one element." + assert ( + numel == 1 + ), "When Variable is used as the condition of if/while , Variable can only contain one element." if framework._in_eager_mode_: assert self._is_initialized(), "tensor not initialized" return bool(np.all(self.numpy() > 0)) @@ -711,21 +759,22 @@ def contain_tensor(item): for slice_item in item: if isinstance(slice_item, slice): - if isinstance(slice_item.start, Variable) \ - or isinstance(slice_item.stop, Variable) \ - or isinstance(slice_item.step, Variable): + if ( + isinstance(slice_item.start, Variable) + or isinstance(slice_item.stop, Variable) + or isinstance(slice_item.step, Variable) + ): return True else: - if isinstance( - slice_item, - (Variable, np.ndarray)) and Variable.dtype != paddle.bool: + if ( + isinstance(slice_item, (Variable, np.ndarray)) + and Variable.dtype != paddle.bool + ): return True return False def __getitem__(self, item): - def is_list_tuple(index, contain_type): - def _is_list_tuple(item): if isinstance(item, (tuple, list)): for s in item: @@ -753,7 +802,6 @@ def _is_list_tuple(item): return self._getitem_index_not_tensor(item) def __setitem__(self, item, value): - def contain_tensor_or_list(item): if not isinstance(item, tuple): item = [item] @@ -813,7 +861,8 @@ def _set_grad_ivar(self, value): self._unset_fake_empty() else: raise TypeError( - "_set_grad_ivar is only supported for Parameter Tensor") + "_set_grad_ivar is only supported for Parameter Tensor" + ) @framework.dygraph_only def value(self): @@ -971,25 +1020,30 @@ def to_sparse_coo(self, sparse_dim): if framework._in_eager_mode_ and not hasattr(core, "eager"): return - for method_name, method in (("__bool__", __bool__), ("__nonzero__", - __nonzero__), - ("_to_static_var", - _to_static_var), ("set_value", set_value), - ("block", block), ("backward", backward), - ("clear_grad", clear_grad), ("inplace_version", - inplace_version), - ("gradient", gradient), ("register_hook", - register_hook), - ("__str__", __str__), ("__repr__", __str__), - ("__deepcopy__", __deepcopy__), ("__module__", - "paddle"), - ("__array__", - __array__), ("__getitem__", - __getitem__), ("item", item), - ("__setitem__", - __setitem__), ("_to", _to), ("values", values), - ("to_dense", to_dense), ("to_sparse_coo", - to_sparse_coo)): + for method_name, method in ( + ("__bool__", __bool__), + ("__nonzero__", __nonzero__), + ("_to_static_var", _to_static_var), + ("set_value", set_value), + ("block", block), + ("backward", backward), + ("clear_grad", clear_grad), + ("inplace_version", inplace_version), + ("gradient", gradient), + ("register_hook", register_hook), + ("__str__", __str__), + ("__repr__", __str__), + ("__deepcopy__", __deepcopy__), + ("__module__", "paddle"), + ("__array__", __array__), + ("__getitem__", __getitem__), + ("item", item), + ("__setitem__", __setitem__), + ("_to", _to), + ("values", values), + ("to_dense", to_dense), + ("to_sparse_coo", to_sparse_coo), + ): if framework._in_eager_mode_: setattr(core.eager.Tensor, method_name, method) else: diff --git a/python/paddle/fluid/dygraph_utils.py b/python/paddle/fluid/dygraph_utils.py index d93915e8bb5ff..e89f8f591082a 100644 --- a/python/paddle/fluid/dygraph_utils.py +++ b/python/paddle/fluid/dygraph_utils.py @@ -18,10 +18,9 @@ @dygraph_only -def _append_activation_in_dygraph(input, - act=None, - use_cudnn=None, - use_mkldnn=None): +def _append_activation_in_dygraph( + input, act=None, use_cudnn=None, use_mkldnn=None +): """Append activation in dygraph mode. Args: @@ -60,5 +59,6 @@ def _append_bias_in_dygraph(input, bias=None, axis=1, use_mkldnn=False): if bias is None: return input - return _legacy_C_ops.elementwise_add(input, bias, 'axis', axis, - 'use_mkldnn', use_mkldnn) + return _legacy_C_ops.elementwise_add( + input, bias, 'axis', axis, 'use_mkldnn', use_mkldnn + ) diff --git a/python/paddle/fluid/entry_attr.py b/python/paddle/fluid/entry_attr.py index 4d481943229ed..51e9f5a6d51d0 100644 --- a/python/paddle/fluid/entry_attr.py +++ b/python/paddle/fluid/entry_attr.py @@ -37,7 +37,6 @@ def _to_attr(self): class ProbabilityEntry(EntryAttr): - def __init__(self, probability): super(ProbabilityEntry, self).__init__() @@ -55,17 +54,18 @@ def _to_attr(self): class CountFilterEntry(EntryAttr): - def __init__(self, count_filter): super(CountFilterEntry, self).__init__() if not isinstance(count_filter, int): raise ValueError( - "count_filter must be a valid integer greater than 0") + "count_filter must be a valid integer greater than 0" + ) if count_filter < 0: raise ValueError( - "count_filter must be a valid integer greater or equal than 0") + "count_filter must be a valid integer greater or equal than 0" + ) self._name = "count_filter_entry" self._count_filter = count_filter diff --git a/python/paddle/fluid/evaluator.py b/python/paddle/fluid/evaluator.py index 6c3ab7c497e5b..06f01ecdf5e68 100644 --- a/python/paddle/fluid/evaluator.py +++ b/python/paddle/fluid/evaluator.py @@ -31,12 +31,14 @@ def _clone_var_(block, var): assert isinstance(var, Variable) - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=True) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=True, + ) class Evaluator(object): @@ -66,7 +68,9 @@ class Evaluator(object): def __init__(self, name, **kwargs): warnings.warn( "The %s is deprecated, because maintain a modified program inside evaluator cause bug easily, please use fluid.metrics.%s instead." - % (self.__class__.__name__, self.__class__.__name__), Warning) + % (self.__class__.__name__, self.__class__.__name__), + Warning, + ) self.states = [] self.metrics = [] self.helper = LayerHelper(name, **kwargs) @@ -86,10 +90,9 @@ def reset(self, executor, reset_program=None): for var in self.states: assert isinstance(var, Variable) g_var = _clone_var_(reset_program.current_block(), var) - layers.fill_constant(shape=g_var.shape, - value=0.0, - dtype=g_var.dtype, - out=g_var) + layers.fill_constant( + shape=g_var.shape, value=0.0, dtype=g_var.dtype, out=g_var + ) executor.run(reset_program) @@ -114,11 +117,12 @@ def _create_state(self, suffix, dtype, shape): Returns: State variable """ - state = self.helper.create_variable(name="_".join( - [unique_name.generate(self.helper.name), suffix]), - persistable=True, - dtype=dtype, - shape=shape) + state = self.helper.create_variable( + name="_".join([unique_name.generate(self.helper.name), suffix]), + persistable=True, + dtype=dtype, + shape=shape, + ) self.states.append(state) return state @@ -169,27 +173,41 @@ def __init__( if main_program.current_block().idx != 0: raise ValueError("You can only invoke Evaluator in root block") - self.num_infer_chunks = self._create_state(dtype='int64', - shape=[1], - suffix='num_infer_chunks') - self.num_label_chunks = self._create_state(dtype='int64', - shape=[1], - suffix='num_label_chunks') + self.num_infer_chunks = self._create_state( + dtype='int64', shape=[1], suffix='num_infer_chunks' + ) + self.num_label_chunks = self._create_state( + dtype='int64', shape=[1], suffix='num_label_chunks' + ) self.num_correct_chunks = self._create_state( - dtype='int64', shape=[1], suffix='num_correct_chunks') - precision, recall, f1_score, num_infer_chunks, num_label_chunks, num_correct_chunks = layers.chunk_eval( + dtype='int64', shape=[1], suffix='num_correct_chunks' + ) + ( + precision, + recall, + f1_score, + num_infer_chunks, + num_label_chunks, + num_correct_chunks, + ) = layers.chunk_eval( input=input, label=label, chunk_scheme=chunk_scheme, num_chunk_types=num_chunk_types, excluded_chunk_types=excluded_chunk_types, ) - layers.sums(input=[self.num_infer_chunks, num_infer_chunks], - out=self.num_infer_chunks) - layers.sums(input=[self.num_label_chunks, num_label_chunks], - out=self.num_label_chunks) - layers.sums(input=[self.num_correct_chunks, num_correct_chunks], - out=self.num_correct_chunks) + layers.sums( + input=[self.num_infer_chunks, num_infer_chunks], + out=self.num_infer_chunks, + ) + layers.sums( + input=[self.num_label_chunks, num_label_chunks], + out=self.num_label_chunks, + ) + layers.sums( + input=[self.num_correct_chunks, num_correct_chunks], + out=self.num_correct_chunks, + ) self.metrics.extend([precision, recall, f1_score]) @@ -199,18 +217,31 @@ def eval(self, executor, eval_program=None): block = eval_program.current_block() num_infer_chunks, num_label_chunks, num_correct_chunks = executor.run( eval_program, - fetch_list=[_clone_var_(block, state) for state in self.states]) + fetch_list=[_clone_var_(block, state) for state in self.states], + ) num_infer_chunks = num_infer_chunks[0] num_label_chunks = num_label_chunks[0] num_correct_chunks = num_correct_chunks[0] - precision = float( - num_correct_chunks) / num_infer_chunks if num_infer_chunks else 0 - recall = float( - num_correct_chunks) / num_label_chunks if num_label_chunks else 0 - f1_score = float(2 * precision * recall) / ( - precision + recall) if num_correct_chunks else 0 - return np.array([precision], dtype='float32'), np.array( - [recall], dtype='float32'), np.array([f1_score], dtype='float32') + precision = ( + float(num_correct_chunks) / num_infer_chunks + if num_infer_chunks + else 0 + ) + recall = ( + float(num_correct_chunks) / num_label_chunks + if num_label_chunks + else 0 + ) + f1_score = ( + float(2 * precision * recall) / (precision + recall) + if num_correct_chunks + else 0 + ) + return ( + np.array([precision], dtype='float32'), + np.array([recall], dtype='float32'), + np.array([f1_score], dtype='float32'), + ) class EditDistance(Evaluator): @@ -250,31 +281,35 @@ def __init__(self, input, label, ignored_tokens=None, **kwargs): if main_program.current_block().idx != 0: raise ValueError("You can only invoke Evaluator in root block") - self.total_distance = self._create_state(dtype='float32', - shape=[1], - suffix='total_distance') - self.seq_num = self._create_state(dtype='int64', - shape=[1], - suffix='seq_num') - self.instance_error = self._create_state(dtype='int64', - shape=[1], - suffix='instance_error') - distances, seq_num = layers.edit_distance(input=input, - label=label, - ignored_tokens=ignored_tokens) + self.total_distance = self._create_state( + dtype='float32', shape=[1], suffix='total_distance' + ) + self.seq_num = self._create_state( + dtype='int64', shape=[1], suffix='seq_num' + ) + self.instance_error = self._create_state( + dtype='int64', shape=[1], suffix='instance_error' + ) + distances, seq_num = layers.edit_distance( + input=input, label=label, ignored_tokens=ignored_tokens + ) zero = layers.fill_constant(shape=[1], value=0.0, dtype='float32') compare_result = layers.equal(distances, zero) compare_result_int = layers.cast(x=compare_result, dtype='int64') seq_right_count = layers.reduce_sum(compare_result_int) - instance_error_count = layers.elementwise_sub(x=seq_num, - y=seq_right_count) + instance_error_count = layers.elementwise_sub( + x=seq_num, y=seq_right_count + ) total_distance = layers.reduce_sum(distances) - layers.sums(input=[self.total_distance, total_distance], - out=self.total_distance) + layers.sums( + input=[self.total_distance, total_distance], out=self.total_distance + ) layers.sums(input=[self.seq_num, seq_num], out=self.seq_num) - layers.sums(input=[self.instance_error, instance_error_count], - out=self.instance_error) + layers.sums( + input=[self.instance_error, instance_error_count], + out=self.instance_error, + ) self.metrics.append(total_distance) self.metrics.append(instance_error_count) @@ -289,10 +324,12 @@ def eval(self, executor, eval_program=None): seq_num = layers.cast(x=seq_num, dtype='float32') instance_error = layers.cast(x=instance_error, dtype='float32') avg_distance = layers.elementwise_div(x=total_distance, y=seq_num) - avg_instance_error = layers.elementwise_div(x=instance_error, - y=seq_num) - result = executor.run(eval_program, - fetch_list=[avg_distance, avg_instance_error]) + avg_instance_error = layers.elementwise_div( + x=instance_error, y=seq_num + ) + result = executor.run( + eval_program, fetch_list=[avg_distance, avg_instance_error] + ) return np.array(result[0]), np.array(result[1]) @@ -355,16 +392,18 @@ class DetectionMAP(Evaluator): 'accum_map_v' is the accumulative mAP of one pass. """ - def __init__(self, - input, - gt_label, - gt_box, - gt_difficult=None, - class_num=None, - background_label=0, - overlap_threshold=0.5, - evaluate_difficult=True, - ap_version='integral'): + def __init__( + self, + input, + gt_label, + gt_box, + gt_difficult=None, + class_num=None, + background_label=0, + overlap_threshold=0.5, + evaluate_difficult=True, + ap_version='integral', + ): super(DetectionMAP, self).__init__("map_eval") gt_label = layers.cast(x=gt_label, dtype=gt_box.dtype) @@ -375,26 +414,29 @@ def __init__(self, label = layers.concat([gt_label, gt_box], axis=1) # calculate mean average precision (mAP) of current mini-batch - map = detection.detection_map(input, - label, - class_num, - background_label, - overlap_threshold=overlap_threshold, - evaluate_difficult=evaluate_difficult, - ap_version=ap_version) + map = detection.detection_map( + input, + label, + class_num, + background_label, + overlap_threshold=overlap_threshold, + evaluate_difficult=evaluate_difficult, + ap_version=ap_version, + ) self._create_state(dtype='int32', shape=None, suffix='accum_pos_count') self._create_state(dtype='float32', shape=None, suffix='accum_true_pos') - self._create_state(dtype='float32', - shape=None, - suffix='accum_false_pos') + self._create_state( + dtype='float32', shape=None, suffix='accum_false_pos' + ) self.has_state = None - var = self.helper.create_variable(persistable=True, - dtype='int32', - shape=[1]) - self.helper.set_variable_initializer(var, - initializer=Constant(value=int(0))) + var = self.helper.create_variable( + persistable=True, dtype='int32', shape=[1] + ) + self.helper.set_variable_initializer( + var, initializer=Constant(value=int(0)) + ) self.has_state = var # calculate accumulative mAP @@ -408,12 +450,15 @@ def __init__(self, has_state=self.has_state, input_states=self.states, out_states=self.states, - ap_version=ap_version) + ap_version=ap_version, + ) - layers.fill_constant(shape=self.has_state.shape, - value=1, - dtype=self.has_state.dtype, - out=self.has_state) + layers.fill_constant( + shape=self.has_state.shape, + value=1, + dtype=self.has_state.dtype, + out=self.has_state, + ) self.cur_map = map self.accum_map = accum_map @@ -426,8 +471,7 @@ def reset(self, executor, reset_program=None): reset_program = Program() with program_guard(main_program=reset_program): var = _clone_var_(reset_program.current_block(), self.has_state) - layers.fill_constant(shape=var.shape, - value=0, - dtype=var.dtype, - out=var) + layers.fill_constant( + shape=var.shape, value=0, dtype=var.dtype, out=var + ) executor.run(reset_program) diff --git a/python/paddle/fluid/executor.py b/python/paddle/fluid/executor.py index 489ed0b476a56..ba8221bb2f0fb 100755 --- a/python/paddle/fluid/executor.py +++ b/python/paddle/fluid/executor.py @@ -144,10 +144,12 @@ def as_numpy(tensor, copy=False): assert isinstance(tensor, core.LoDTensor) lod = tensor.lod() if len(lod) > 0: - raise RuntimeError("Some of your fetched tensors hold LoD information. \ + raise RuntimeError( + "Some of your fetched tensors hold LoD information. \ They can not be completely cast to Python ndarray. \ Please set the parameter 'return_numpy' as 'False' to \ - return LoDTensor itself directly.") + return LoDTensor itself directly." + ) if tensor._is_initialized(): if copy: return np.array(tensor) @@ -237,21 +239,29 @@ def check_feed_shape_type(var, feed, num_places=1): if diff_shape is not None: raise ValueError( 'The fed Variable %r should have dimensions = %d, shape = ' - '%r, but received fed shape %r on each device' % - (var.name, len(var.shape), var.shape, diff_shape)) + '%r, but received fed shape %r on each device' + % (var.name, len(var.shape), var.shape, diff_shape) + ) if not dtype_is_compatible_with(feed._dtype(), var.dtype): - var_dtype_format = convert_dtype(var.dtype) if isinstance( - var.dtype, core.VarDesc.VarType) else var.dtype - feed_dtype_format = convert_dtype(feed._dtype()) if isinstance( - feed._dtype(), core.VarDesc.VarType) else feed._dtype() + var_dtype_format = ( + convert_dtype(var.dtype) + if isinstance(var.dtype, core.VarDesc.VarType) + else var.dtype + ) + feed_dtype_format = ( + convert_dtype(feed._dtype()) + if isinstance(feed._dtype(), core.VarDesc.VarType) + else feed._dtype() + ) raise ValueError( - 'The data type of fed Variable %r must be %r, but received %r' % - (var.name, var_dtype_format, feed_dtype_format)) + 'The data type of fed Variable %r must be %r, but received %r' + % (var.name, var_dtype_format, feed_dtype_format) + ) return True def has_feed_operators(block, feed_targets, feed_holder_name): - """ Check whether the block already has feed operators. + """Check whether the block already has feed operators. Return false if the block does not have any feed operators. If some feed operators have been prepended to the block, check that @@ -280,20 +290,22 @@ def has_feed_operators(block, feed_targets, feed_holder_name): if feed_target_name not in feed_targets: raise Exception( "'feed_targets' does not have {} variable".format( - feed_target_name)) + feed_target_name + ) + ) else: break if feed_count > 0 and feed_count != len(feed_targets): raise Exception( - "Feed operators in program desc do not match 'feed_targets'") + "Feed operators in program desc do not match 'feed_targets'" + ) return feed_count > 0 -def has_fetch_operators(block, - fetch_targets, - fetch_holder_name, - fetch_op='fetch'): - """ Check whether the block already has fetch operators. +def has_fetch_operators( + block, fetch_targets, fetch_holder_name, fetch_op='fetch' +): + """Check whether the block already has fetch operators. Return false if the block does not have any fetch operators. If some fetch operators have been appended to the block, check that @@ -321,25 +333,25 @@ def has_fetch_operators(block, assert op.desc.output('Out')[0] == fetch_holder_name fetch_target_name = op.desc.input('X')[0] if fetch_target_name not in [ - var.desc.name() for var in fetch_targets + var.desc.name() for var in fetch_targets ]: raise Exception( "'fetch_targets' does not have {} variable".format( - fetch_target_name)) + fetch_target_name + ) + ) idx = op.desc.attr('col') assert fetch_target_name == fetch_targets[idx].desc.name() if fetch_count > 0 and fetch_count != len(fetch_targets): raise Exception( - "Fetch operators in program desc do not match 'fetch_targets'") + "Fetch operators in program desc do not match 'fetch_targets'" + ) return fetch_count > 0 -def _add_feed_fetch_ops(program, - feed, - fetch_list, - feed_var_name, - fetch_var_name, - use_fetch_v2=False): +def _add_feed_fetch_ops( + program, feed, fetch_list, feed_var_name, fetch_var_name, use_fetch_v2=False +): tmp_program = program.clone() global_block = tmp_program.global_block() @@ -350,7 +362,8 @@ def _add_feed_fetch_ops(program, feed_var = global_block.create_var( name=feed_var_name, type=core.VarDesc.VarType.FEED_MINIBATCH, - persistable=True) + persistable=True, + ) if fetch_var_name in global_block.vars: fetch_var = global_block.var(fetch_var_name) @@ -358,21 +371,25 @@ def _add_feed_fetch_ops(program, fetch_var = global_block.create_var( name=fetch_var_name, type=core.VarDesc.VarType.FETCH_LIST, - persistable=True) + persistable=True, + ) # prepend feed operators if not has_feed_operators(global_block, feed, feed_var_name): for i, name in enumerate(feed): if global_block.has_var(name): out = global_block.var(name) - global_block._prepend_op(type='feed', - inputs={'X': [feed_var]}, - outputs={'Out': [out]}, - attrs={'col': i}) + global_block._prepend_op( + type='feed', + inputs={'X': [feed_var]}, + outputs={'Out': [out]}, + attrs={'col': i}, + ) else: warnings.warn( "The variable %s is not found in program. It is not declared or is pruned." - % name) + % name + ) if use_fetch_v2: fetch_op = 'fetch_v2' @@ -380,22 +397,26 @@ def _add_feed_fetch_ops(program, fetch_op = 'fetch' # append fetch_operators - if not has_fetch_operators(global_block, fetch_list, fetch_var_name, - fetch_op): + if not has_fetch_operators( + global_block, fetch_list, fetch_var_name, fetch_op + ): for i, var in enumerate(fetch_list): assert isinstance(var, Variable) or isinstance( - var, str), ("Wrong type for fetch_list[%s]: %s" % - (i, type(var))) - global_block.append_op(type=fetch_op, - inputs={'X': [var]}, - outputs={'Out': [fetch_var]}, - attrs={'col': i}) + var, str + ), "Wrong type for fetch_list[%s]: %s" % (i, type(var)) + global_block.append_op( + type=fetch_op, + inputs={'X': [var]}, + outputs={'Out': [fetch_var]}, + attrs={'col': i}, + ) return tmp_program -def _apply_inplace_addto_pass(program, enable_inplace, enable_addto, - skip_var_names): +def _apply_inplace_addto_pass( + program, enable_inplace, enable_addto, skip_var_names +): use_cuda = True if core.is_compiled_with_cuda() else False attrs = {"use_cuda": use_cuda, "mem_opt_skip_vars": skip_var_names} @@ -404,12 +425,14 @@ def _apply_inplace_addto_pass(program, enable_inplace, enable_addto, empty_startup_program = Program() if enable_inplace: pass_name = "buffer_shared_inplace_pass" - _apply_pass(program, empty_startup_program, pass_name, attrs, - attr_types) + _apply_pass( + program, empty_startup_program, pass_name, attrs, attr_types + ) if enable_addto and use_cuda: pass_name = "inplace_addto_op_pass" - _apply_pass(program, empty_startup_program, pass_name, attrs, - attr_types) + _apply_pass( + program, empty_startup_program, pass_name, attrs, attr_types + ) def _fetch_var(name, scope=None, return_numpy=True): @@ -438,7 +461,8 @@ def _fetch_var(name, scope=None, return_numpy=True): assert var is not None, ( "Cannot find " + name + " in scope. Perhaps you need to make the" " variable persistable by using var.persistable = True in your" - " program.") + " program." + ) tensor = var.get_tensor() if return_numpy: tensor = as_numpy(tensor, copy=True) @@ -446,7 +470,6 @@ def _fetch_var(name, scope=None, return_numpy=True): def _to_name_str(var): - def _to_str(var): if isinstance(var, Variable): return var.desc.name() @@ -471,19 +494,26 @@ def _to_str(var): def _is_enable_standalone_executor(): - return framework._enable_standalone_executor_ is None or framework._enable_standalone_executor_ in [ - 1, '1', True, 'True', 'true' - ] + return ( + framework._enable_standalone_executor_ is None + or framework._enable_standalone_executor_ + in [1, '1', True, 'True', 'true'] + ) def _is_dy2st_enable_standalone_executor(): return framework._dy2st_enable_standalone_executor_ in [ - 1, '1', True, 'True', 'true' + 1, + '1', + True, + 'True', + 'true', ] def _prepare_fleet_executor(): from ..distributed.fleet.proto import fleet_executor_desc_pb2 + trainer_endpoints_str = os.getenv("PADDLE_TRAINER_ENDPOINTS", "") trainer_endpoints = trainer_endpoints_str.split(',') fleet_exe_desc = fleet_executor_desc_pb2.FleetExecutorDesc() @@ -501,7 +531,8 @@ def _prepare_fleet_executor(): def _get_strong_program_cache_key_for_new_exe(program, feed, fetch_list): return program.desc.cached_hash_str() + _get_program_cache_key( - feed, fetch_list) + feed, fetch_list + ) def _get_strong_program_cache_key(program, feed, fetch_list): @@ -512,10 +543,16 @@ def _get_varname_from_block(block): block_str.append(var_name) return "\n".join(block_str) - inner_program = program._program if isinstance( - program, compiler.CompiledProgram) else program - return _get_varname_from_block(inner_program.blocks[0]) + str( - id(program)) + _get_program_cache_key(feed, fetch_list) + inner_program = ( + program._program + if isinstance(program, compiler.CompiledProgram) + else program + ) + return ( + _get_varname_from_block(inner_program.blocks[0]) + + str(id(program)) + + _get_program_cache_key(feed, fetch_list) + ) def _get_program_cache_key(feed, fetch_list): @@ -531,30 +568,35 @@ def _get_program_cache_key(feed, fetch_list): def _as_lodtensor(data, place, dtype=None): """ - Convert numpy.ndarray to Tensor, its only support Tensor without LoD information. - For higher dimensional sequence data, please use LoDTensor directly. + Convert numpy.ndarray to Tensor, its only support Tensor without LoD information. + For higher dimensional sequence data, please use LoDTensor directly. - Examples: - >>> import paddle.fluid as fluid - >>> place = fluid.CPUPlace() - >>> exe = fluid.executor(place) - >>> data = np.array(size=(100, 200, 300)) - >>> np_outs = map(lambda x: fluid.executor._as_lodtensor(x, place), data) - >>> ... + Examples: + >>> import paddle.fluid as fluid + >>> place = fluid.CPUPlace() + >>> exe = fluid.executor(place) + >>> data = np.array(size=(100, 200, 300)) + >>> np_outs = map(lambda x: fluid.executor._as_lodtensor(x, place), data) + >>> ... - Args: - data(numpy.ndarray|list|tuple|scalar): a instance of array, scalar, list or tuple - data(core.Place): the place of created tensor - dtype(core.VarDesc.VarType|str): the expected data type of created tensor + Args: + data(numpy.ndarray|list|tuple|scalar): a instance of array, scalar, list or tuple + data(core.Place): the place of created tensor + dtype(core.VarDesc.VarType|str): the expected data type of created tensor - Returns: - LoDTensor - """ - #NOTE(zhiqiu): convert python builtin, like float, int, and list, to numpy ndarray + Returns: + LoDTensor + """ + # NOTE(zhiqiu): convert python builtin, like float, int, and list, to numpy ndarray if not isinstance(data, np.ndarray): - assert dtype is not None, 'The dtype should be given when feed data is not np.ndarray' - dtype = convert_dtype(dtype) if isinstance( - dtype, core.VarDesc.VarType) else dtype + assert ( + dtype is not None + ), 'The dtype should be given when feed data is not np.ndarray' + dtype = ( + convert_dtype(dtype) + if isinstance(dtype, core.VarDesc.VarType) + else dtype + ) if np.isscalar(data): data = np.array([data]).astype(dtype) elif isinstance(data, (list, tuple)): @@ -569,7 +611,9 @@ def _as_lodtensor(data, place, dtype=None): else: raise TypeError( "Convert data of type {} to Tensor is not supported".format( - type(data))) + type(data) + ) + ) # convert numpy.ndarray to tensor tensor = core.LoDTensor() @@ -578,7 +622,6 @@ def _as_lodtensor(data, place, dtype=None): class FetchHandler(object): - def __init__(self, var_dict=None, period_secs=60): assert var_dict != None self.var_dict = var_dict @@ -592,7 +635,8 @@ def handler(self, res_dict): @staticmethod def help(): - print(""" + print( + """ class FetchHandlerExample(FetchHandler): def handler(self, res_dict): print(res_dict["auc"]) @@ -601,11 +645,11 @@ def handler(self, res_dict): auc = Variable() var_dict = {"auc": auc} handler = FetchHandlerExample(var_dict=var_dict) -""") +""" + ) class _StandaloneExecutor(object): - def __init__(self, place, main_program, scope): self._place = core.Place() self._place.set_place(place) @@ -625,8 +669,9 @@ def run(self, scope, feed_names, fetch_list, return_numpy=True): """ fetch_list = self._check_fetch(fetch_list) - tensors = self._new_exe.run(scope, feed_names, - fetch_list)._move_to_list() + tensors = self._new_exe.run( + scope, feed_names, fetch_list + )._move_to_list() if return_numpy: return as_numpy(tensors, copy=True) else: @@ -658,8 +703,9 @@ def _update_feed(self, feed): if not isinstance(feed, dict): raise TypeError( - "feed requires dict as its Parameter. But you passed in %s" % - (type(feed))) + "feed requires dict as its Parameter. But you passed in %s" + % (type(feed)) + ) global_block = self._main_program.global_block() for feed_name in list(feed.keys()): @@ -667,7 +713,8 @@ def _update_feed(self, feed): feed.pop(feed_name) warnings.warn( "The variable %s is not found in program. It is not declared or is pruned." - % feed_name) + % feed_name + ) return feed @@ -681,19 +728,27 @@ def _check_fetch(self, fetch_list): fetch_var = fetch_var.name elif not isinstance(fetch_var, str): raise TypeError( - "Required fetch_var shall be str|Variable, but received {}". - format(type(fetch_var).__name__)) + "Required fetch_var shall be str|Variable, but received {}".format( + type(fetch_var).__name__ + ) + ) res.append(fetch_var) return res class _ExecutorCache(object): - class _CachedData(object): - - def __init__(self, program, feed, fetch_list, feed_var_name, - fetch_var_name, place, scope): + def __init__( + self, + program, + feed, + fetch_list, + feed_var_name, + fetch_var_name, + place, + scope, + ): self.program = program self.feed = feed self.fetch_list = fetch_list @@ -709,18 +764,25 @@ def __init__(self, program, feed, fetch_list, feed_var_name, # The program holds no _program, maybe it is constructed by graph. # Convert graph to program in order to generate key. self.program._program = framework.IrGraph( - self.program._graph).to_program() + self.program._graph + ).to_program() self.key = hash( _get_strong_program_cache_key_for_new_exe( - self.program._program, feed, fetch_list)) + self.program._program, feed, fetch_list + ) + ) else: self.key = hash( _get_strong_program_cache_key_for_new_exe( - self.program, feed, fetch_list)) + self.program, feed, fetch_list + ) + ) def __eq__(self, other): - return isinstance( - other, _ExecutorCache._CachedData) and self.key == other.key + return ( + isinstance(other, _ExecutorCache._CachedData) + and self.key == other.key + ) def __hash__(self): return self.key @@ -730,21 +792,41 @@ def __init__(self): # the _ExecutorCache instance, otherwise a global cache may not be released after # the Executor instance deleted self._get_cached_program_and_executor = lru_cache(maxsize=8)( - self._get_program_and_executor) + self._get_program_and_executor + ) def clear(self): self._get_cached_program_and_executor.cache_clear() - def get_program_and_executor(self, program, feed, fetch_list, feed_var_name, - fetch_var_name, place, scope): + def get_program_and_executor( + self, + program, + feed, + fetch_list, + feed_var_name, + fetch_var_name, + place, + scope, + ): return self._get_cached_program_and_executor( - self._CachedData(program, feed, fetch_list, feed_var_name, - fetch_var_name, place, scope)) + self._CachedData( + program, + feed, + fetch_list, + feed_var_name, + fetch_var_name, + place, + scope, + ) + ) def _get_program_and_executor(self, cached_data): program = cached_data.program - inner_program = program._program if isinstance( - program, compiler.CompiledProgram) else program + inner_program = ( + program._program + if isinstance(program, compiler.CompiledProgram) + else program + ) feed = cached_data.feed fetch_list = cached_data.fetch_list feed_var_name = cached_data.feed_var_name @@ -754,9 +836,13 @@ def _get_program_and_executor(self, cached_data): # To apply IR pass, compile the Program to IrGraph and convert it back to Program if isinstance(program, compiler.CompiledProgram) or isinstance( - program._graph, compiler.CompiledProgram): - compiled_program = program if isinstance( - program, compiler.CompiledProgram) else program._graph + program._graph, compiler.CompiledProgram + ): + compiled_program = ( + program + if isinstance(program, compiler.CompiledProgram) + else program._graph + ) build_strategy = compiled_program._build_strategy # print(f"Program before convert:\n {inner_program}", flush=True) compiled_program._compile(scope, place) @@ -771,21 +857,26 @@ def _get_program_and_executor(self, cached_data): else: build_strategy = None from paddle.incubate.autograd import prim_enabled, prim2orig + if prim_enabled() and program == default_main_program(): prim2orig() inner_program = program - program = _add_feed_fetch_ops(program=inner_program, - feed=feed, - fetch_list=fetch_list, - feed_var_name=feed_var_name, - fetch_var_name=fetch_var_name, - use_fetch_v2=True) + program = _add_feed_fetch_ops( + program=inner_program, + feed=feed, + fetch_list=fetch_list, + feed_var_name=feed_var_name, + fetch_var_name=fetch_var_name, + use_fetch_v2=True, + ) - if os.environ.get('FLAGS_CONVERT_GRAPH_TO_PROGRAM', None) in [ - 1, '1', True, 'True', 'true' - ] and not program._is_start_up_program_: + if ( + os.environ.get('FLAGS_CONVERT_GRAPH_TO_PROGRAM', None) + in [1, '1', True, 'True', 'true'] + and not program._is_start_up_program_ + ): if program.num_blocks > 1: # If there are multiple blocks in the program, subblock will not be executed with the new executor in temporary logging.warning("There are more than 1 block in program.") @@ -796,13 +887,22 @@ def _get_program_and_executor(self, cached_data): # standalone executor will apply buffer_shared_inplace_pass and # inplace_addto_op_pass to program according to build_strategy - enable_inplace = True if build_strategy is None or build_strategy.enable_inplace else False - enable_addto = True if build_strategy is not None and build_strategy.enable_addto else False + enable_inplace = ( + True + if build_strategy is None or build_strategy.enable_inplace + else False + ) + enable_addto = ( + True + if build_strategy is not None and build_strategy.enable_addto + else False + ) if enable_inplace or enable_addto: # inplace should skip feed and fetch var skip_var_names = eval(_get_program_cache_key(feed, fetch_list)) - _apply_inplace_addto_pass(program, enable_inplace, enable_addto, - skip_var_names) + _apply_inplace_addto_pass( + program, enable_inplace, enable_addto, skip_var_names + ) new_program = program.clone() new_exe = _StandaloneExecutor(place, new_program, scope) @@ -906,7 +1006,8 @@ def __init__(self, place=None): self._prepare_to_run_called = False self._auto_checkpoint_name = unique_name.generate( - "__auto_checkpoint_executor__") + "__auto_checkpoint_executor__" + ) # NOTE: Whether to use experimental executor `StandaloneExecutor`. self._enable_interpreter_core = _is_enable_standalone_executor() @@ -976,8 +1077,9 @@ def _feed_data(self, program, feed, feed_var_name, scope): var = global_block.var(feed_target_name) if var.dtype != core.VarDesc.VarType.STRINGS: if not isinstance(cur_feed, core.LoDTensor): - cur_feed = _as_lodtensor(cur_feed, self.place, - var.dtype) + cur_feed = _as_lodtensor( + cur_feed, self.place, var.dtype + ) check_feed_shape_type(var, cur_feed) idx = op.desc.attr('col') core.set_feed_variable(scope, cur_feed, feed_var_name, idx) @@ -1015,14 +1117,19 @@ def _get_targets(_optimize_ops, _fetch_list, item): _optimize_ops.append(item) else: raise TypeError( - "The operator in fetch_list is not an optimize_op") - elif isinstance(item, Variable) or isinstance( - item, str) or isinstance(item, str): + "The operator in fetch_list is not an optimize_op" + ) + elif ( + isinstance(item, Variable) + or isinstance(item, str) + or isinstance(item, str) + ): _fetch_list.append(item) else: raise TypeError( "The item in fetch_list should be str, variable or optimize_op, but received %s.", - type(item)) + type(item), + ) for index, item in enumerate(fetch_list): # NOTE(zhiqiu): to support (optimizer_ops, param_and_grads) and optimizer_ops in fetch_list @@ -1034,9 +1141,10 @@ def _get_targets(_optimize_ops, _fetch_list, item): elif isinstance(item, tuple): if not isinstance(item[0], (list, tuple)): raise TypeError( - "Requires fetch_list[{}][0] shall be one of (list, tuple) when type(fetch_list[{}]) is `tuple`, but received fetch_list[{}][0]'s type is `{}`." - .format(index, index, index, - type(item[0]).__name__)) + "Requires fetch_list[{}][0] shall be one of (list, tuple) when type(fetch_list[{}]) is `tuple`, but received fetch_list[{}][0]'s type is `{}`.".format( + index, index, index, type(item[0]).__name__ + ) + ) for i in item[0]: _get_targets(_optimize_ops, _fetch_list, i) else: @@ -1045,11 +1153,9 @@ def _get_targets(_optimize_ops, _fetch_list, item): return _fetch_list, _optimize_ops @classmethod - def _prune_program(cls, - program, - feed=None, - fetch_list=None, - optimize_ops=None): + def _prune_program( + cls, program, feed=None, fetch_list=None, optimize_ops=None + ): """ Prune operators and variables which are not needed to generate :code:`fetch_list` and optimize operators. @@ -1141,7 +1247,8 @@ def _update_feed(cls, program, feed): feed.pop(feed_name) warnings.warn( "The variable %s is not found in program. It is not declared or is pruned." - % feed_name) + % feed_name + ) elif isinstance(feed, list) or isinstance(feed, tuple): for i, each in enumerate(feed): @@ -1150,7 +1257,8 @@ def _update_feed(cls, program, feed): each.pop(feed_name) warnings.warn( "The variable %s is not found in program. It is not declared or is pruned." - % feed_name) + % feed_name + ) return feed ''' @@ -1185,9 +1293,18 @@ def close(self): del trainer_instance self._default_executor.close() - def _run_parallel(self, program, scope, feed, fetch_list, fetch_var_name, - return_numpy, return_merged): + def _run_parallel( + self, + program, + scope, + feed, + fetch_list, + fetch_var_name, + return_numpy, + return_merged, + ): from paddle.optimizer.lr import LRScheduler + exe = program._executor # TODO(zhenghuihuang): quantization uses Graph in CompiledProgram # instead of program. We will add support for checking Vars in Graph @@ -1202,9 +1319,11 @@ def _run_parallel(self, program, scope, feed, fetch_list, fetch_var_name, if not isinstance(feed_tensor, core.LoDTensor): # always set to CPU place, since the tensor need to be split # it is fast in CPU - feed_tensor = _as_lodtensor(feed[feed_name], - core.CPUPlace(), - var.dtype if var else None) + feed_tensor = _as_lodtensor( + feed[feed_name], + core.CPUPlace(), + var.dtype if var else None, + ) if need_check_feed: check_feed_shape_type(var, feed_tensor, exe.device_count()) feed_tensor_dict[feed_name] = feed_tensor @@ -1215,16 +1334,20 @@ def _run_parallel(self, program, scope, feed, fetch_list, fetch_var_name, for i, each in enumerate(feed): if not isinstance(each, dict): raise TypeError( - "Each element of feed list should be a dict") + "Each element of feed list should be a dict" + ) res_dict = dict() for feed_name in each: tensor = each[feed_name] - var = global_block.var( - feed_name) if need_check_feed else None + var = ( + global_block.var(feed_name) if need_check_feed else None + ) if not isinstance(tensor, core.LoDTensor): - tensor = _as_lodtensor(each[feed_name], - program._places[i], - var.dtype if var else None) + tensor = _as_lodtensor( + each[feed_name], + program._places[i], + var.dtype if var else None, + ) if need_check_feed: check_feed_shape_type(var, tensor) res_dict[feed_name] = tensor @@ -1245,23 +1368,26 @@ def _run_parallel(self, program, scope, feed, fetch_list, fetch_var_name, ) else: exe.feed_and_split_tensor_into_local_scopes( - {lr_sheduler._var_name: lr_tensor}) + {lr_sheduler._var_name: lr_tensor} + ) fetch_var_names = list(map(_to_name_str, fetch_list)) tensors = exe.run(fetch_var_names, return_merged)._move_to_list() return as_numpy(tensors) if return_numpy else tensors - def run(self, - program=None, - feed=None, - fetch_list=None, - feed_var_name='feed', - fetch_var_name='fetch', - scope=None, - return_numpy=True, - use_program_cache=False, - return_merged=True, - use_prune=False): + def run( + self, + program=None, + feed=None, + fetch_list=None, + feed_var_name='feed', + fetch_var_name='fetch', + scope=None, + return_numpy=True, + use_program_cache=False, + return_merged=True, + use_prune=False, + ): """ Run the specified :code:`Program` or :code:`CompiledProgram`. It should be noted that the executor will execute all the operators in :code:`Program` or :code:`CompiledProgram` without pruning some @@ -1436,32 +1562,49 @@ def run(self, """ # Temporary FLAGS, just for testing the performance of program cache force_use_program_cache = os.environ.get( - 'FLAGS_FORCE_USE_PROGRAM_CACHE', None) + 'FLAGS_FORCE_USE_PROGRAM_CACHE', None + ) if force_use_program_cache is not None: use_program_cache = force_use_program_cache in [ - 1, '1', True, 'True', 'true' + 1, + '1', + True, + 'True', + 'true', ] self._log_force_set_program_cache(use_program_cache) try: - res = self._run_impl(program=program, - feed=feed, - fetch_list=fetch_list, - feed_var_name=feed_var_name, - fetch_var_name=fetch_var_name, - scope=scope, - return_numpy=return_numpy, - use_program_cache=use_program_cache, - use_prune=use_prune, - return_merged=return_merged) + res = self._run_impl( + program=program, + feed=feed, + fetch_list=fetch_list, + feed_var_name=feed_var_name, + fetch_var_name=fetch_var_name, + scope=scope, + return_numpy=return_numpy, + use_program_cache=use_program_cache, + use_prune=use_prune, + return_merged=return_merged, + ) core.update_autotune_status() return res except Exception as e: six.reraise(*sys.exc_info()) - def _run_impl(self, program, feed, fetch_list, feed_var_name, - fetch_var_name, scope, return_numpy, use_program_cache, - return_merged, use_prune): + def _run_impl( + self, + program, + feed, + fetch_list, + feed_var_name, + fetch_var_name, + scope, + return_numpy, + use_program_cache, + return_merged, + use_prune, + ): if self._closed: raise RuntimeError("Attempted to use a closed Executor") @@ -1480,17 +1623,19 @@ def _run_impl(self, program, feed, fetch_list, feed_var_name, program=program, feed=feed, fetch_list=fetch_list, - with_standalone_executor=self. - _fleet_executor_with_standalone) + with_standalone_executor=self._fleet_executor_with_standalone, + ) if "startup_program" in program._pipeline_opt: program = program._pipeline_opt["startup_program"] else: - return self._run_pipeline(program, - fetch_list=fetch_list, - use_program_cache=use_program_cache) + return self._run_pipeline( + program, + fetch_list=fetch_list, + use_program_cache=use_program_cache, + ) if isinstance(program, Program) and program._heter_pipeline_opt: - #print("program._heter_pipeline_opt: {}".format( + # print("program._heter_pipeline_opt: {}".format( # program._heter_pipeline_opt)) ## change default executor heter_place = program._heter_pipeline_opt["heter_place"] @@ -1500,20 +1645,26 @@ def _run_impl(self, program, feed, fetch_list, feed_var_name, self._default_executor = core.Executor(p) # TODO(zhangminxu): support heterps pipeline training using exe.run if "startup_program" in program._heter_pipeline_opt: - #print("get startup_program from _pipeline_opt") + # print("get startup_program from _pipeline_opt") program = program._heter_pipeline_opt["startup_program"] - if isinstance(program, Program) and \ - len(program.global_block().ops) == 0: + if ( + isinstance(program, Program) + and len(program.global_block().ops) == 0 + ): if use_default_main_program: - error_info = "Now you are using default_main_program, "\ - "but there are no operators in the program to be executed. "\ - "Please ensure you create model correctly or you can pass "\ + error_info = ( + "Now you are using default_main_program, " + "but there are no operators in the program to be executed. " + "Please ensure you create model correctly or you can pass " "the Program or the CompiledProgram manually." + ) else: - error_info = "There are no operators in the program to be executed. "\ - "If you pass Program manually, please use fluid.program_guard "\ + error_info = ( + "There are no operators in the program to be executed. " + "If you pass Program manually, please use fluid.program_guard " "to ensure the current Program is being used." + ) warnings.warn(error_info) if scope is None: @@ -1523,27 +1674,36 @@ def _run_impl(self, program, feed, fetch_list, feed_var_name, _origin_fetch_list = fetch_list _origin_program = program fetch_list, optimize_ops = self._split_optimize_ops_in_fetch_list( - fetch_list) + fetch_list + ) if optimize_ops: use_prune = True if use_prune: - cache_key = _get_strong_program_cache_key(program, feed, - _origin_fetch_list) + cache_key = _get_strong_program_cache_key( + program, feed, _origin_fetch_list + ) cached_pruned_program = self._get_pruned_program_cache(cache_key) if cached_pruned_program is None: if isinstance(program, compiler.CompiledProgram): program_scope_cache = self._get_pruned_program_scope_cache( - str(id(_origin_program))) + str(id(_origin_program)) + ) # copy the original program, so it can be cached. program = copy.copy(program) # share the local scopes for same original CompiledProgram. program._share_vars_from = program_scope_cache - if self._get_pruned_program_scope_cache( - str(id(_origin_program))) is None: + if ( + self._get_pruned_program_scope_cache( + str(id(_origin_program)) + ) + is None + ): self._add_pruned_program_scope_cache( - str(id(_origin_program)), program) - pruned_program = self._prune_program(program, feed, fetch_list, - optimize_ops) + str(id(_origin_program)), program + ) + pruned_program = self._prune_program( + program, feed, fetch_list, optimize_ops + ) self._add_pruned_program_cache(cache_key, pruned_program) else: pruned_program = cached_pruned_program @@ -1556,68 +1716,97 @@ def _can_use_interpreter_core(program, place): return False use_standalone_executor_for_distribution = os.environ.get( - 'FLAGS_CONVERT_GRAPH_TO_PROGRAM', - None) in [1, '1', True, 'True', 'true'] + 'FLAGS_CONVERT_GRAPH_TO_PROGRAM', None + ) in [1, '1', True, 'True', 'true'] - compiled = isinstance(program, - compiler.CompiledProgram) or isinstance( - program._graph, compiler.CompiledProgram) + compiled = isinstance( + program, compiler.CompiledProgram + ) or isinstance(program._graph, compiler.CompiledProgram) if compiled: - compiled_program = program if isinstance( - program, compiler.CompiledProgram) else program._graph + compiled_program = ( + program + if isinstance(program, compiler.CompiledProgram) + else program._graph + ) # Unsupported case 1: data parallel - if compiled_program._is_data_parallel and len( + if ( + compiled_program._is_data_parallel + and len( compiled_program._get_places( - place, compiled_program._places)) != 1: + place, compiled_program._places + ) + ) + != 1 + ): warnings.warn( "Standalone executor is not used for data parallel", - UserWarning) + UserWarning, + ) return False # Unsupported case 2: parallel graph if core.globals()['FLAGS_enable_parallel_graph'] in [ - 1, '1', True, 'True', 'true' + 1, + '1', + True, + 'True', + 'true', ]: warnings.warn( "Standalone executor is not used for parallel graph", - UserWarning) + UserWarning, + ) return False # Unsupported case 3: inference if compiled_program._is_inference: warnings.warn( "Standalone executor is not used for inference", - UserWarning) + UserWarning, + ) return False # Unsupported case 4: CUDA Graph - if compiled_program._build_strategy is not None and compiled_program._build_strategy.allow_cuda_graph_capture: + if ( + compiled_program._build_strategy is not None + and compiled_program._build_strategy.allow_cuda_graph_capture + ): warnings.warn( "Standalone executor is not used for CUDA Graph", - UserWarning) + UserWarning, + ) return False # Unsupported case 5: async mode - if compiled_program._build_strategy is not None and compiled_program._build_strategy.async_mode: + if ( + compiled_program._build_strategy is not None + and compiled_program._build_strategy.async_mode + ): warnings.warn( "Standalone executor is not used for async mode", - UserWarning) + UserWarning, + ) return False # delete this code after supporting fleet from paddle.distributed.fleet import fleet + if fleet._role_maker is not None: - warnings.warn("Standalone executor is not used for fleet", - UserWarning) + warnings.warn( + "Standalone executor is not used for fleet", UserWarning + ) return use_standalone_executor_for_distribution return True # NOTE: This is an experimental feature. If `export FLAGS_USE_STANDALONE_EXECUTOR=1 `, # use StandaloneExecutor to run the program. - if return_merged and self._enable_interpreter_core and _can_use_interpreter_core( - program, self.place): + if ( + return_merged + and self._enable_interpreter_core + and _can_use_interpreter_core(program, self.place) + ): if feed is None: feed = {} @@ -1627,18 +1816,27 @@ def _can_use_interpreter_core(program, place): if not isinstance(feed, dict): raise TypeError( "feed requires dict as its Parameter. But you passed in %s" - % (type(feed))) + % (type(feed)) + ) feed = self._update_feed(program, feed) program, new_exe = self._executor_cache.get_program_and_executor( - program, feed, fetch_list, feed_var_name, fetch_var_name, - self.place, scope) + program, + feed, + fetch_list, + feed_var_name, + fetch_var_name, + self.place, + scope, + ) self._feed_data(program, feed, feed_var_name, scope) if hasattr(program, 'lr_sheduler'): from paddle.optimizer.lr import LRScheduler - assert isinstance(program.lr_sheduler, - LRScheduler), "must be LRScheduler" + + assert isinstance( + program.lr_sheduler, LRScheduler + ), "must be LRScheduler" lr_sheduler = program.lr_sheduler lr_value = lr_sheduler() lr_var = program.global_block().vars[lr_sheduler._var_name] @@ -1652,8 +1850,9 @@ def _can_use_interpreter_core(program, place): else: tensor._copy_from(cpu_tensor, self.place) - return new_exe.run(scope, list(feed.keys()), fetch_list, - return_numpy) + return new_exe.run( + scope, list(feed.keys()), fetch_list, return_numpy + ) compiled = isinstance(program, compiler.CompiledProgram) @@ -1668,13 +1867,15 @@ def _can_use_interpreter_core(program, place): varobj = global_block.vars[varname] # Can not check var build by fluid.layers.data(), bucause fluid.layers.data() had not set need_check_feed - if vardesc.persistable() == False and \ - vardesc.type() == core.VarDesc.VarType.LOD_TENSOR and \ - vardesc.need_check_feed() == True and \ - varobj.stop_gradient == True and \ - varobj.is_data == True and \ - varobj.belong_to_optimizer == False and \ - varname not in feed: + if ( + vardesc.persistable() == False + and vardesc.type() == core.VarDesc.VarType.LOD_TENSOR + and vardesc.need_check_feed() == True + and varobj.stop_gradient == True + and varobj.is_data == True + and varobj.belong_to_optimizer == False + and varname not in feed + ): raise ValueError('Need feed data for variable %s' % varname) acp._auto_checkpoint(self, program) @@ -1682,46 +1883,63 @@ def _can_use_interpreter_core(program, place): # For backward compatibility, run directly. if not compiled: # In distributed training, the compiled program is saved in Program._graph - has_compiled_graph = isinstance(program._graph, - compiler.CompiledProgram) + has_compiled_graph = isinstance( + program._graph, compiler.CompiledProgram + ) if has_compiled_graph: program._graph._compile(scope, self.place) # _graph in program does not support inference since the _graph is optimized # through optimizer.minimize function and should not be used as inference graph # assert not program._graph._is_inference - return self._run_parallel(program._graph, - scope=scope, - feed=feed, - fetch_list=fetch_list, - fetch_var_name=fetch_var_name, - return_numpy=return_numpy, - return_merged=return_merged) - - return self._run_program(program, - feed=feed, - fetch_list=fetch_list, - feed_var_name=feed_var_name, - fetch_var_name=fetch_var_name, - scope=scope, - return_numpy=return_numpy, - use_program_cache=use_program_cache) + return self._run_parallel( + program._graph, + scope=scope, + feed=feed, + fetch_list=fetch_list, + fetch_var_name=fetch_var_name, + return_numpy=return_numpy, + return_merged=return_merged, + ) + + return self._run_program( + program, + feed=feed, + fetch_list=fetch_list, + feed_var_name=feed_var_name, + fetch_var_name=fetch_var_name, + scope=scope, + return_numpy=return_numpy, + use_program_cache=use_program_cache, + ) program._compile(scope, self.place) if program._is_inference: return self._run_inference(program._executor, feed) else: - return self._run_parallel(program, - scope=scope, - feed=feed, - fetch_list=fetch_list, - fetch_var_name=fetch_var_name, - return_numpy=return_numpy, - return_merged=return_merged) - - def _run_program(self, program, feed, fetch_list, feed_var_name, - fetch_var_name, scope, return_numpy, use_program_cache): + return self._run_parallel( + program, + scope=scope, + feed=feed, + fetch_list=fetch_list, + fetch_var_name=fetch_var_name, + return_numpy=return_numpy, + return_merged=return_merged, + ) + + def _run_program( + self, + program, + feed, + fetch_list, + feed_var_name, + fetch_var_name, + scope, + return_numpy, + use_program_cache, + ): from paddle.optimizer.lr import LRScheduler + if feed is None: feed = {} elif isinstance(feed, (list, tuple)): @@ -1730,19 +1948,22 @@ def _run_program(self, program, feed, fetch_list, feed_var_name, if not isinstance(feed, dict): raise TypeError( - "feed requires dict as its Parameter. But you passed in %s" % - (type(feed))) + "feed requires dict as its Parameter. But you passed in %s" + % (type(feed)) + ) assert program is not None, "The program should not be Empty" if not isinstance(program, Program): raise TypeError( "Executor requires Program as its Parameter. But you passed in %s" - % (type(program))) + % (type(program)) + ) if not isinstance(fetch_var_name, str): raise TypeError( "The name of fetch variable requires string as its Parameter. But you passed in %s" - % (type(fetch_var_name))) + % (type(fetch_var_name)) + ) if use_program_cache: cache_key = _get_strong_program_cache_key(program, feed, fetch_list) @@ -1755,35 +1976,41 @@ def _run_program(self, program, feed, fetch_list, feed_var_name, feed=feed, fetch_list=fetch_list, feed_var_name=feed_var_name, - fetch_var_name=fetch_var_name) + fetch_var_name=fetch_var_name, + ) self._add_program_cache(cache_key, cached_program) fetch_list_str = list(map(_to_name_str, fetch_list)) cached_ctx = self._default_executor.prepare( - cached_program.desc, 0, fetch_list_str, False) + cached_program.desc, 0, fetch_list_str, False + ) # currently, we cache program, vars, sub_scope here # we suppose that in a life cycle of training, a user # will not create many programs. So, here the basic # rule of caching is to cache all unseen (program, var, scope) # when a user use use_program_cache. cached_scope = scope.new_scope() - self._default_executor.create_variables(cached_program.desc, - cached_scope, 0) + self._default_executor.create_variables( + cached_program.desc, cached_scope, 0 + ) self._add_ctx_cache(cache_key, cached_ctx) self._add_scope_cache(cache_key, cached_scope) program = cached_program ctx = cached_ctx scope = cached_scope else: - program = _add_feed_fetch_ops(program=program, - feed=feed, - fetch_list=fetch_list, - feed_var_name=feed_var_name, - fetch_var_name=fetch_var_name) + program = _add_feed_fetch_ops( + program=program, + feed=feed, + fetch_list=fetch_list, + feed_var_name=feed_var_name, + fetch_var_name=fetch_var_name, + ) self._feed_data(program, feed, feed_var_name, scope) if hasattr(program, 'lr_sheduler'): - assert isinstance(program.lr_sheduler, - LRScheduler), "must be LRScheduler" + assert isinstance( + program.lr_sheduler, LRScheduler + ), "must be LRScheduler" lr_sheduler = program.lr_sheduler lr_value = lr_sheduler() lr_var = program.global_block().vars[lr_sheduler._var_name] @@ -1792,11 +2019,13 @@ def _run_program(self, program, feed, fetch_list, feed_var_name, tensor.set(data, self.place) if not use_program_cache: - self._default_executor.run(program.desc, scope, 0, True, True, - [fetch_var_name]) + self._default_executor.run( + program.desc, scope, 0, True, True, [fetch_var_name] + ) else: - self._default_executor.run_prepared_ctx(ctx, scope, False, False, - False) + self._default_executor.run_prepared_ctx( + ctx, scope, False, False, False + ) arr = scope.find_var(fetch_var_name).get_fetch_list() tensors = arr._move_to_list() if return_numpy: @@ -1811,13 +2040,16 @@ def _check_fetch_list(self, fetch_list): is_fetch_var = lambda var: isinstance(var, (Variable, str)) is_tuple_list = lambda var: isinstance(var, (tuple, list)) - if fetch_list is None: return [] - if is_fetch_var(fetch_list): return [fetch_list] + if fetch_list is None: + return [] + if is_fetch_var(fetch_list): + return [fetch_list] - assert is_tuple_list(fetch_list), \ - "Currently , The fetch_list type only should be list or tuple, \n"\ - "but the input type is {}. For more information please refer to \n"\ + assert is_tuple_list(fetch_list), ( + "Currently , The fetch_list type only should be list or tuple, \n" + "but the input type is {}. For more information please refer to \n" "the executor.run(...).".format(type(fetch_list)) + ) res = [] for i, var in enumerate(fetch_list): @@ -1831,9 +2063,10 @@ def _check_fetch_list(self, fetch_list): res.append(var) else: raise TypeError( - "Require fetch_list[{}] 's type shall be one of (Variable, str), but received {}." - .format(i, - type(var).__name__)) + "Require fetch_list[{}] 's type shall be one of (Variable, str), but received {}.".format( + i, type(var).__name__ + ) + ) return res @@ -1850,25 +2083,30 @@ def _adjust_pipeline_resource(self, pipeline_opt, dataset, pipeline_num): pipeline_num = filelist_length print( "Pipeline training: setting the pipeline num to %d is enough because there are only %d files" - % (filelist_length, filelist_length)) + % (filelist_length, filelist_length) + ) if filelist_length < pipeline_num * pipeline_opt["concurrency_list"][0]: print( "Pipeline training: setting the 1st element in concurrency_list to %d is enough because there are only %d files" - % (filelist_length // pipeline_num, filelist_length)) - pipeline_opt["concurrency_list"][ - 0] = filelist_length // pipeline_num + % (filelist_length // pipeline_num, filelist_length) + ) + pipeline_opt["concurrency_list"][0] = ( + filelist_length // pipeline_num + ) dataset.set_thread(pipeline_opt["concurrency_list"][0] * pipeline_num) return pipeline_num - def _prepare_trainer(self, - program=None, - dataset=None, - scope=None, - thread=0, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100): + def _prepare_trainer( + self, + program=None, + dataset=None, + scope=None, + thread=0, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + ): is_heter = 0 use_ps_gpu = 0 if not program._fleet_opt is None: @@ -1889,16 +2127,19 @@ def _prepare_trainer(self, if is_heter: from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.utils.fleet_util import FleetUtil + fu = FleetUtil() ret = fu.split_program_by_device(program) if not compiled: # TODO: Need a better way to distinguish and specify different execution mode if program._pipeline_opt: trainer = TrainerFactory()._create_trainer( - program._pipeline_opt) + program._pipeline_opt + ) elif program._heter_pipeline_opt: trainer = TrainerFactory()._create_trainer( - program._heter_pipeline_opt) + program._heter_pipeline_opt + ) else: trainer = TrainerFactory()._create_trainer(program._fleet_opt) trainer._set_thread_barrier(program._is_distributed) @@ -1908,13 +2149,16 @@ def _prepare_trainer(self, else: if program._pipeline_opt: trainer = TrainerFactory()._create_trainer( - program.program._pipeline_opt) + program.program._pipeline_opt + ) elif program._heter_pipeline_opt: trainer = TrainerFactory()._create_trainer( - program.program._heter_pipeline_opt) + program.program._heter_pipeline_opt + ) else: trainer = TrainerFactory()._create_trainer( - program.program._fleet_opt) + program.program._fleet_opt + ) trainer._set_program(program.program) if thread <= 0: @@ -1923,7 +2167,8 @@ def _prepare_trainer(self, elif dataset.thread_num <= 0: raise RuntimeError( "You should set thread num first, either in Dataset" - "or in Executor.train_from_dataset") + "or in Executor.train_from_dataset" + ) else: trainer._set_thread(dataset.thread_num) else: @@ -1933,19 +2178,22 @@ def _prepare_trainer(self, trainer._set_fetch_var_and_info(fetch_list, fetch_info, print_period) return scope, trainer - def _run_from_dataset(self, - program=None, - dataset=None, - scope=None, - thread=0, - is_infer=False, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None): + def _run_from_dataset( + self, + program=None, + dataset=None, + scope=None, + thread=0, + is_infer=False, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + ): if program._pipeline_opt is not None: import paddle + if dataset is not None: raise RuntimeError("dataset should be None for pipeline mode") # The following fake dataset is created to call @@ -1956,24 +2204,28 @@ def _run_from_dataset(self, data_vars.append(var) if core.is_compiled_with_npu(): dataset = paddle.fluid.DatasetFactory().create_dataset( - 'InMemoryDataset') + 'InMemoryDataset' + ) else: dataset = paddle.fluid.DatasetFactory().create_dataset( - 'FileInstantDataset') + 'FileInstantDataset' + ) dataset.set_batch_size(1) dataset.set_thread(1) dataset.set_filelist(['None']) dataset.set_use_var(data_vars) elif program._heter_pipeline_opt is not None: stage_id = program._heter_pipeline_opt["pipeline_stage"] - #print("test_fl_stage_id: {}".format(stage_id)) + # print("test_fl_stage_id: {}".format(stage_id)) heter_place = program._heter_pipeline_opt["heter_place"] if stage_id != 0: if "is_fl_mode" not in program._heter_pipeline_opt: import paddle + if dataset is not None: raise RuntimeError( - "dataset should be None for heter pipeline mode") + "dataset should be None for heter pipeline mode" + ) # The following fake dataset is created to call # the _prepare_trainer api, and it is meaningless. data_vars = [] @@ -1981,7 +2233,8 @@ def _run_from_dataset(self, if var.is_data: data_vars.append(var) dataset = paddle.fluid.DatasetFactory().create_dataset( - 'InMemoryDataset') + 'InMemoryDataset' + ) dataset.set_batch_size(1) dataset.set_thread(1) dataset.set_filelist(['None']) @@ -1989,7 +2242,8 @@ def _run_from_dataset(self, else: if dataset is None: raise RuntimeError( - "dataset is need and should be initialized") + "dataset is need and should be initialized" + ) ## change default executor heter_place = framework._get_paddle_place(heter_place) p = core.Place() @@ -2016,7 +2270,8 @@ def _run_from_dataset(self, feed=[], fetch_list=real_fetch_list, feed_var_name='feed', - fetch_var_name='fetch') + fetch_var_name='fetch', + ) main_block = program._pipeline_opt["section_program"].block(0) for op in main_block.ops: # set the op_role of fetch op to Optimize to avoid @@ -2024,16 +2279,19 @@ def _run_from_dataset(self, if op.type == 'fetch': op._set_attr( 'op_role', - core.op_proto_and_checker_maker.OpRole.Optimize) + core.op_proto_and_checker_maker.OpRole.Optimize, + ) fetch_list = None - scope, trainer = self._prepare_trainer(program=program, - dataset=dataset, - scope=scope, - thread=thread, - debug=debug, - fetch_list=fetch_list, - fetch_info=fetch_info, - print_period=print_period) + scope, trainer = self._prepare_trainer( + program=program, + dataset=dataset, + scope=scope, + thread=thread, + debug=debug, + fetch_list=fetch_list, + fetch_info=fetch_info, + print_period=print_period, + ) trainer._set_infer(is_infer) trainer._gen_trainer_desc() @@ -2048,8 +2306,11 @@ def _run_from_dataset(self, dataset._dynamic_adjust_before_train(trainer.proto_desc.thread_num) if program._heter_pipeline_opt is None: - trainer_instance = self._default_executor.init_for_dataset( # -->InitForDataset - program.desc, trainer._desc(), scope, dataset.dataset) + trainer_instance = ( + self._default_executor.init_for_dataset( # -->InitForDataset + program.desc, trainer._desc(), scope, dataset.dataset + ) + ) else: # cache trainer instance for heterps pipeline training if fetch_list == None: @@ -2058,8 +2319,9 @@ def _run_from_dataset(self, trainer_instance = self._get_trainer_cache(cache_key) if trainer_instance is None: trainer_instance = self._default_executor.init_for_dataset( - program.desc, trainer._desc(), scope, dataset.dataset) - #print("test_fl_ps - trainer_desc: {}\n".format(trainer)) + program.desc, trainer._desc(), scope, dataset.dataset + ) + # print("test_fl_ps - trainer_desc: {}\n".format(trainer)) self._add_trainer_cache(cache_key, trainer_instance) else: trainer_instance.ResetDataset(dataset.dataset) @@ -2086,18 +2348,20 @@ def _run_from_dataset(self, return None - def _prepare_pipeline_ctx(self, - program=None, - dataset=None, - scope=None, - thread=0, - is_infer=False, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None, - use_program_cache=False): + def _prepare_pipeline_ctx( + self, + program=None, + dataset=None, + scope=None, + thread=0, + is_infer=False, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + use_program_cache=False, + ): assert program._pipeline_opt is not None assert dataset is None, "dataset should be None for pipeline mode" @@ -2117,10 +2381,12 @@ def _get_dataset(): data_vars.append(var) if core.is_compiled_with_npu(): dataset = paddle.fluid.DatasetFactory().create_dataset( - 'InMemoryDataset') + 'InMemoryDataset' + ) else: dataset = paddle.fluid.DatasetFactory().create_dataset( - 'FileInstantDataset') + 'FileInstantDataset' + ) dataset.set_batch_size(1) dataset.set_thread(1) dataset.set_filelist(['None']) @@ -2141,11 +2407,13 @@ def _get_real_program_fetch_list(): if fetch_var_name in real_program.global_block().vars: real_fetch_list.append(fetch_var) - real_program = _add_feed_fetch_ops(program=real_program, - feed=[], - fetch_list=real_fetch_list, - feed_var_name='feed', - fetch_var_name='fetch') + real_program = _add_feed_fetch_ops( + program=real_program, + feed=[], + fetch_list=real_fetch_list, + feed_var_name='feed', + fetch_var_name='fetch', + ) main_block = real_program.block(0) for op in main_block.ops: # set the op_role of fetch op to Optimize to avoid @@ -2153,7 +2421,8 @@ def _get_real_program_fetch_list(): if op.type == 'fetch': op._set_attr( 'op_role', - core.op_proto_and_checker_maker.OpRole.Optimize) + core.op_proto_and_checker_maker.OpRole.Optimize, + ) return real_program, real_fetch_list real_program, real_fetch_list = _get_real_program_fetch_list() @@ -2161,14 +2430,16 @@ def _get_real_program_fetch_list(): program._pipeline_opt["section_program"] = real_program fetch_list = None - scope, trainer = self._prepare_trainer(program=program, - dataset=dataset, - scope=scope, - thread=thread, - debug=debug, - fetch_list=fetch_list, - fetch_info=fetch_info, - print_period=print_period) + scope, trainer = self._prepare_trainer( + program=program, + dataset=dataset, + scope=scope, + thread=thread, + debug=debug, + fetch_list=fetch_list, + fetch_info=fetch_info, + print_period=print_period, + ) trainer._set_infer(is_infer) trainer._gen_trainer_desc() @@ -2183,59 +2454,84 @@ def _get_real_program_fetch_list(): trainer_desc = trainer._desc() # slow, cache trainer_instance = self._default_executor.init_for_dataset( - program.desc, trainer_desc, scope, dataset.dataset) + program.desc, trainer_desc, scope, dataset.dataset + ) ctx = [scope, real_fetch_list, trainer_instance] - if use_program_cache: self._add_ctx_cache(cache_key, ctx) + if use_program_cache: + self._add_ctx_cache(cache_key, ctx) return ctx - def _prepare_fleet_executor_carrier(self, - carrier_id="", - program=None, - scope=None, - fleet_opt=None, - with_standalone_executor=False): - num_micro_batches = fleet_opt[ - "num_micro_batches"] if "num_micro_batches" in fleet_opt else 1 + def _prepare_fleet_executor_carrier( + self, + carrier_id="", + program=None, + scope=None, + fleet_opt=None, + with_standalone_executor=False, + ): + num_micro_batches = ( + fleet_opt["num_micro_batches"] + if "num_micro_batches" in fleet_opt + else 1 + ) cur_rank = int(os.getenv("PADDLE_TRAINER_ID", 0)) trainer_endpoints = os.getenv("PADDLE_TRAINER_ENDPOINTS", "").split(',') nrank = len(trainer_endpoints) - assert 'scheduler' in fleet_opt or 'tasks' in fleet_opt, \ - "Fleet executor need configuration for scheduler, you can choose from 1F1B or Origin. " \ + assert 'scheduler' in fleet_opt or 'tasks' in fleet_opt, ( + "Fleet executor need configuration for scheduler, you can choose from 1F1B or Origin. " "Or you can provide a list of task nodes to init fleet executor directly." + ) if 'tasks' in fleet_opt: - assert 'task_id_to_rank' in fleet_opt, "If you provide tasks to init fleet executor," \ - " task_id_to_rank should also be provided." + assert 'task_id_to_rank' in fleet_opt, ( + "If you provide tasks to init fleet executor," + " task_id_to_rank should also be provided." + ) print('fleet executor will use user defined task nodes') tasks = [task.task_node() for task in fleet_opt['tasks']] task_id_to_rank = fleet_opt['task_id_to_rank'] else: scheduler = fleet_opt['scheduler'] if scheduler == '1F1B': - from paddle.distributed.fleet.fleet_executor_utils import run1f1b - if "dist_strategy" not in fleet_opt or \ - "pp_degree" not in fleet_opt["dist_strategy"] or \ - fleet_opt["dist_strategy"]["pp_degree"] == 1: + from paddle.distributed.fleet.fleet_executor_utils import ( + run1f1b, + ) + + if ( + "dist_strategy" not in fleet_opt + or "pp_degree" not in fleet_opt["dist_strategy"] + or fleet_opt["dist_strategy"]["pp_degree"] == 1 + ): warnings.warn("Using 1F1B scheduler with pp_degree == 1.") tasks, task_id_to_rank = run1f1b( - program, cur_rank, fleet_opt.get('num_micro_batches', 1), - fleet_opt.get('dist_strategy', {}), nrank, - with_standalone_executor) + program, + cur_rank, + fleet_opt.get('num_micro_batches', 1), + fleet_opt.get('dist_strategy', {}), + nrank, + with_standalone_executor, + ) elif scheduler == 'Origin': from paddle.distributed.fleet.fleet_executor_utils import origin - if "dist_strategy" in fleet_opt and \ - "pp_degree" in fleet_opt["dist_strategy"]: - assert fleet_opt["dist_strategy"]["pp_degree"] == 1, \ - "For pipeline mode, the scheduler should be 1F1B instead of Origin." + + if ( + "dist_strategy" in fleet_opt + and "pp_degree" in fleet_opt["dist_strategy"] + ): + assert ( + fleet_opt["dist_strategy"]["pp_degree"] == 1 + ), "For pipeline mode, the scheduler should be 1F1B instead of Origin." if "num_micro_batches" in fleet_opt: - assert fleet_opt["num_micro_batches"] == 1, \ - "For origin scheduler mode, the num micro batches should be 1." + assert ( + fleet_opt["num_micro_batches"] == 1 + ), "For origin scheduler mode, the num micro batches should be 1." tasks, task_id_to_rank = origin(program, cur_rank) else: - raise "Fleet_executor only supports 1F1B and Origin scheduler, " \ - "but received " + str(scheduler) + "." + raise "Fleet_executor only supports 1F1B and Origin scheduler, " "but received " + str( + scheduler + ) + "." # NOTE: have to hold these vars, otherwise will be destructed fleet_opt['tasks'] = tasks fleet_opt['task_id_to_rank'] = task_id_to_rank @@ -2243,16 +2539,26 @@ def _prepare_fleet_executor_carrier(self, place.set_place(self.place) # NOTE: the last argument is used to force create some vars in root scope, # won't be used during train. - self._fleet_executor.init(carrier_id, program.desc, scope, place, - num_micro_batches, tasks, task_id_to_rank, []) - - def _run_using_fleet_executor(self, - program=None, - feed=None, - feed_var_name="feed", - fetch_var_name="fetch", - fetch_list=None, - with_standalone_executor=False): + self._fleet_executor.init( + carrier_id, + program.desc, + scope, + place, + num_micro_batches, + tasks, + task_id_to_rank, + [], + ) + + def _run_using_fleet_executor( + self, + program=None, + feed=None, + feed_var_name="feed", + fetch_var_name="fetch", + fetch_list=None, + with_standalone_executor=False, + ): cache_key = _get_strong_program_cache_key(program, feed, fetch_list) cached_program = self._get_program_cache(cache_key) cached_scope = self._get_scope_cache(cache_key) @@ -2260,16 +2566,20 @@ def _run_using_fleet_executor(self, cached_scope = global_scope() self._add_scope_cache(cache_key, cached_scope) if cached_program is None: - assert program._pipeline_opt, "program should have _pipeline_opt to start carrier" + assert ( + program._pipeline_opt + ), "program should have _pipeline_opt to start carrier" real_feed = [] if feed is None else feed real_program = program if "section_program" in program._pipeline_opt: real_program = program._pipeline_opt["section_program"] - cached_program = _add_feed_fetch_ops(program=real_program, - feed=real_feed, - fetch_list=fetch_list, - feed_var_name=feed_var_name, - fetch_var_name=fetch_var_name) + cached_program = _add_feed_fetch_ops( + program=real_program, + feed=real_feed, + fetch_list=fetch_list, + feed_var_name=feed_var_name, + fetch_var_name=fetch_var_name, + ) main_block = cached_program.block(0) for op in main_block.ops: # set the op_role of fetch op to Optimize to avoid @@ -2277,7 +2587,8 @@ def _run_using_fleet_executor(self, if op.type == 'fetch': op._set_attr( 'op_role', - core.op_proto_and_checker_maker.OpRole.Optimize) + core.op_proto_and_checker_maker.OpRole.Optimize, + ) self._add_program_cache(cache_key, cached_program) fleet_opt = program._pipeline_opt["fleet_opt"] if 'tasks' in fleet_opt: @@ -2291,9 +2602,11 @@ def _run_using_fleet_executor(self, feed_task = fleet_opt['tasks'][0] print("Inserting feed ops for task", feed_task.task_id()) feed_program = feed_task.get_program() - feed_program = self._add_feed_ops(program=feed_program, - feed=real_feed, - feed_var_name=feed_var_name) + feed_program = self._add_feed_ops( + program=feed_program, + feed=real_feed, + feed_var_name=feed_var_name, + ) feed_task.set_program(feed_program) # Insert fetch ops @@ -2303,7 +2616,8 @@ def _run_using_fleet_executor(self, fetch_program = self._add_fetch_ops( program=fetch_program, fetch_list=fetch_list, - fetch_var_name=fetch_var_name) + fetch_var_name=fetch_var_name, + ) main_block = fetch_program.block(0) for op in main_block.ops: # set the op_role of fetch op to Optimize to avoid @@ -2311,7 +2625,8 @@ def _run_using_fleet_executor(self, if op.type == 'fetch': op._set_attr( 'op_role', - core.op_proto_and_checker_maker.OpRole.Optimize) + core.op_proto_and_checker_maker.OpRole.Optimize, + ) fetch_task.set_program(fetch_program) self._prepare_fleet_executor_carrier( @@ -2319,7 +2634,8 @@ def _run_using_fleet_executor(self, program=cached_program, scope=cached_scope, fleet_opt=fleet_opt, - with_standalone_executor=with_standalone_executor) + with_standalone_executor=with_standalone_executor, + ) if feed: # NOTE: don't have to traverse programs in task nodes, @@ -2328,14 +2644,16 @@ def _run_using_fleet_executor(self, self._feed_data(cached_program, feed, feed_var_name, cached_scope) from paddle.optimizer.lr import LRScheduler + if hasattr(program, 'lr_sheduler'): lr_sheduler = program.lr_sheduler assert isinstance(lr_sheduler, LRScheduler), "must be LRScheduler" lr_value = lr_sheduler() lr_var = program.global_block().vars[lr_sheduler._var_name] data = np.array([lr_value]).astype(convert_dtype(lr_var.dtype)) - tensor = core.get_variable_tensor(cached_scope, - lr_sheduler._var_name) + tensor = core.get_variable_tensor( + cached_scope, lr_sheduler._var_name + ) tensor.set(data, self.place) self._fleet_executor.run(cache_key) @@ -2357,30 +2675,32 @@ def _add_feed_ops(self, program, feed, feed_var_name): feed_var = global_block.create_var( name=feed_var_name, type=core.VarDesc.VarType.FEED_MINIBATCH, - persistable=True) + persistable=True, + ) # prepend feed operators if not has_feed_operators(global_block, feed, feed_var_name): for i, name in enumerate(feed): if global_block.has_var(name): out = global_block.var(name) - global_block._prepend_op(type='feed', - inputs={'X': [feed_var]}, - outputs={'Out': [out]}, - attrs={'col': i}) + global_block._prepend_op( + type='feed', + inputs={'X': [feed_var]}, + outputs={'Out': [out]}, + attrs={'col': i}, + ) else: warnings.warn( "The variable %s is not found in program. It is not declared or is pruned." - % name) + % name + ) return tmp_program @classmethod - def _add_fetch_ops(cls, - program, - fetch_list, - fetch_var_name, - use_fetch_v2=False): + def _add_fetch_ops( + cls, program, fetch_list, fetch_var_name, use_fetch_v2=False + ): tmp_program = program.clone() global_block = tmp_program.global_block() @@ -2391,7 +2711,8 @@ def _add_fetch_ops(cls, fetch_var = global_block.create_var( name=fetch_var_name, type=core.VarDesc.VarType.FETCH_LIST, - persistable=True) + persistable=True, + ) if use_fetch_v2: fetch_op = 'fetch_v2' @@ -2399,16 +2720,19 @@ def _add_fetch_ops(cls, fetch_op = 'fetch' # append fetch_operators - if not has_fetch_operators(global_block, fetch_list, fetch_var_name, - fetch_op): + if not has_fetch_operators( + global_block, fetch_list, fetch_var_name, fetch_op + ): for i, var in enumerate(fetch_list): assert isinstance(var, Variable) or isinstance( - var, str), ("Wrong type for fetch_list[%s]: %s" % - (i, type(var))) - global_block.append_op(type=fetch_op, - inputs={'X': [var]}, - outputs={'Out': [fetch_var]}, - attrs={'col': i}) + var, str + ), "Wrong type for fetch_list[%s]: %s" % (i, type(var)) + global_block.append_op( + type=fetch_op, + inputs={'X': [var]}, + outputs={'Out': [fetch_var]}, + attrs={'col': i}, + ) return tmp_program @@ -2423,25 +2747,36 @@ def _remove_fetch_ops(cls, program, fetch_op_name='fetch'): return tmp_program - def _run_pipeline(self, - program=None, - dataset=None, - scope=None, - thread=0, - is_infer=False, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None, - use_program_cache=False): - scope, real_fetch_list, trainer_instance = \ - self._prepare_pipeline_ctx(program, dataset, scope, thread, - is_infer, debug, fetch_list, fetch_info, - print_period, fetch_handler, - use_program_cache) + def _run_pipeline( + self, + program=None, + dataset=None, + scope=None, + thread=0, + is_infer=False, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + use_program_cache=False, + ): + scope, real_fetch_list, trainer_instance = self._prepare_pipeline_ctx( + program, + dataset, + scope, + thread, + is_infer, + debug, + fetch_list, + fetch_info, + print_period, + fetch_handler, + use_program_cache, + ) from paddle.optimizer.lr import LRScheduler + if hasattr(program, 'lr_sheduler'): lr_sheduler = program.lr_sheduler assert isinstance(lr_sheduler, LRScheduler), "must be LRScheduler" @@ -2463,16 +2798,18 @@ def _run_pipeline(self, return None - def infer_from_dataset(self, - program=None, - dataset=None, - scope=None, - thread=0, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None): + def infer_from_dataset( + self, + program=None, + dataset=None, + scope=None, + thread=0, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + ): """ Infer from a pre-defined Dataset. Dataset is defined in paddle.fluid.dataset. Given a program, either a program or compiled program, infer_from_dataset will @@ -2528,26 +2865,39 @@ def infer_from_dataset(self, dataset=dataset) """ - return self._run_from_dataset(program, dataset, scope, thread, True, - debug, fetch_list, fetch_info, - print_period, fetch_handler) - - def start_heter_trainer(self, - program=None, - scope=None, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None): - scope, trainer = self._prepare_trainer(program=program, - dataset=None, - scope=scope, - thread=1, - debug=debug, - fetch_list=fetch_list, - fetch_info=fetch_info, - print_period=print_period) + return self._run_from_dataset( + program, + dataset, + scope, + thread, + True, + debug, + fetch_list, + fetch_info, + print_period, + fetch_handler, + ) + + def start_heter_trainer( + self, + program=None, + scope=None, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + ): + scope, trainer = self._prepare_trainer( + program=program, + dataset=None, + scope=scope, + thread=1, + debug=debug, + fetch_list=fetch_list, + fetch_info=fetch_info, + print_period=print_period, + ) trainer._set_infer(False) trainer._gen_trainer_desc() @@ -2555,32 +2905,35 @@ def start_heter_trainer(self, self._dump_debug_info(program=program, trainer=trainer) trainer_instance = self._default_executor.init_for_dataset( - program.desc, trainer._desc(), scope, None) + program.desc, trainer._desc(), scope, None + ) - #if fetch_handler is not None: + # if fetch_handler is not None: # scope0 = trainer_instance.get_worker_scope(0) # fetch_monitor = FetchHandlerMonitor(scope0, fetch_handler) # fetch_monitor.start() # self._default_executor.run_from_dataset(trainer_instance) # fetch_monitor.stop() # self._default_executor.release_trainer(trainer_instance) - #else: + # else: self._default_executor.run_from_dataset(trainer_instance) - #self._default_executor.release_trainer(trainer_instance) + # self._default_executor.release_trainer(trainer_instance) return trainer_instance - def train_from_dataset(self, - program=None, - dataset=None, - scope=None, - thread=0, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None): + def train_from_dataset( + self, + program=None, + dataset=None, + scope=None, + thread=0, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + ): """ Train from a pre-defined Dataset. Dataset is defined in paddle.fluid.dataset. Given a program, either a program or compiled program, train_from_dataset will @@ -2635,6 +2988,15 @@ def train_from_dataset(self, dataset=dataset) """ - return self._run_from_dataset(program, dataset, scope, thread, False, - debug, fetch_list, fetch_info, - print_period, fetch_handler) + return self._run_from_dataset( + program, + dataset, + scope, + thread, + False, + debug, + fetch_list, + fetch_info, + print_period, + fetch_handler, + ) diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 99abb76d21388..46bd46d706ae2 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -72,7 +72,7 @@ CONTROL_DEP_VAR_PREFIX = core.kControlDepVarName() _dygraph_tracer_ = None -_in_eager_mode_ = (os.environ.get('FLAGS_enable_eager_mode', '1') == '1') +_in_eager_mode_ = os.environ.get('FLAGS_enable_eager_mode', '1') == '1' _global_expected_place_ = None _current_device = None global_prog_seed = 0 @@ -81,10 +81,12 @@ _already_patch_varbase = False _current_cuda_graph_mode = None _global_flags_ = core.globals() -_enable_standalone_executor_ = (os.environ.get('FLAGS_USE_STANDALONE_EXECUTOR', - None)) -_dy2st_enable_standalone_executor_ = (os.environ.get( - 'FLAGS_DY2ST_USE_STANDALONE_EXECUTOR', 1)) +_enable_standalone_executor_ = os.environ.get( + 'FLAGS_USE_STANDALONE_EXECUTOR', None +) +_dy2st_enable_standalone_executor_ = os.environ.get( + 'FLAGS_DY2ST_USE_STANDALONE_EXECUTOR', 1 +) # Some explanation of our execution system 2022.03 # For now we have 3 kinds of execution system, since we refactored dygraph mode to @@ -145,6 +147,7 @@ def _update_monkey_methods(is_eager): def _switch_tensor_bind_type(is_eager): import paddle + if is_eager: paddle.Tensor = core.eager.Tensor else: @@ -179,8 +182,11 @@ def _fallback_legacy_dygraph(): global _is_first_import_ need_fallback = False # Only enable eager on CPU/GPU/XPU - is_not_support = core.is_compiled_with_npu() or core.is_compiled_with_ipu( - ) or core.is_compiled_with_mlu() + is_not_support = ( + core.is_compiled_with_npu() + or core.is_compiled_with_ipu() + or core.is_compiled_with_mlu() + ) if _in_eager_mode_ and is_not_support: # switch into legacy dygraph mode @@ -358,7 +364,6 @@ def set_ipu_shard(call_func, index=-1, stage=-1): """ def decorate(func): - def wrapper(*args, **kwargs): with ipu_shard_guard(index=index, stage=stage): return func(*args, **kwargs) @@ -366,16 +371,17 @@ def wrapper(*args, **kwargs): return wrapper from .dygraph.layers import Layer + if not isinstance(call_func, Layer): if callable(call_func): return decorate(call_func) else: raise TypeError( - "Unsupported type. Only accept paddle.nn.Layer or function.") + "Unsupported type. Only accept paddle.nn.Layer or function." + ) # patch paddle.nn.Layer class BlockFn(type(call_func)): - def __call__(self, *args, **kwargs): with ipu_shard_guard(index=index, stage=stage): return super().__call__(*args, **kwargs) @@ -387,62 +393,68 @@ def __call__(self, *args, **kwargs): def require_version(min_version, max_version=None): """ - Check if the installed version of PaddlePaddle is in [min_version, max_version], - if the installed version is lower than ``min_version`` or higher than ``max_version``, - an exception will be thrown, NO returns if the installed version is satisfied. + Check if the installed version of PaddlePaddle is in [min_version, max_version], + if the installed version is lower than ``min_version`` or higher than ``max_version``, + an exception will be thrown, NO returns if the installed version is satisfied. - Args: - min_version (str): the minimum version required (like '1.4.0'). - max_version (str, optional): the max version required (like '1.6.0'), default is None, - meaning any version equal or higher than ``min_version`` is acceptable. + Args: + min_version (str): the minimum version required (like '1.4.0'). + max_version (str, optional): the max version required (like '1.6.0'), default is None, + meaning any version equal or higher than ``min_version`` is acceptable. - Returns: - None. + Returns: + None. - Raises: - TypeError: if the type of ``min_version`` is not str. - TypeError: if the type of ``max_version`` is not str or type(None). - ValueError: if the value of ``min_version`` is not in version format. - ValueError: if the value of ``max_version`` is not in version format or None. - Exception: if the installed version is lower than ``min_version`` or higher than ``max_version``. + Raises: + TypeError: if the type of ``min_version`` is not str. + TypeError: if the type of ``max_version`` is not str or type(None). + ValueError: if the value of ``min_version`` is not in version format. + ValueError: if the value of ``max_version`` is not in version format or None. + Exception: if the installed version is lower than ``min_version`` or higher than ``max_version``. - Examples: - .. code-block:: python + Examples: + .. code-block:: python - import paddle.fluid as fluid + import paddle.fluid as fluid - # any version >= 0.1.0 is acceptable. - fluid.require_version('0.1.0') + # any version >= 0.1.0 is acceptable. + fluid.require_version('0.1.0') - # if 0.1.0 <= version <= 10.0.0, it is acceptable. - fluid.require_version(min_version='0.1.0', max_version='10.0.0') - """ + # if 0.1.0 <= version <= 10.0.0, it is acceptable. + fluid.require_version(min_version='0.1.0', max_version='10.0.0') + """ if not isinstance(min_version, str): raise TypeError( "The type of 'min_version' in require_version must be str, but received %s." - % (type(min_version))) + % (type(min_version)) + ) if not isinstance(max_version, (str, type(None))): raise TypeError( "The type of 'max_version' in require_version must be str or type(None), but received %s." - % (type(max_version))) + % (type(max_version)) + ) check_format = re.match(r'\d+(\.\d+){0,3}', min_version) if check_format is None or check_format.group() != min_version: raise ValueError( "The value of 'min_version' in require_version must be in format '\\d+(\\.\\d+){0,3}', " - "like '1.5.2.0', but received %s" % min_version) + "like '1.5.2.0', but received %s" % min_version + ) if max_version is not None: check_format = re.match(r'\d+(\.\d+){0,3}', max_version) if check_format is None or check_format.group() != max_version: raise ValueError( "The value of 'max_version' in require_version must be in format '\\d+(\\.\\d+){0,3}', " - "like '1.5.2.0', but received %s" % max_version) + "like '1.5.2.0', but received %s" % max_version + ) version_installed = [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] zero_version = ['0', '0', '0', '0'] @@ -459,75 +471,86 @@ def version_cmp(ver_a, ver_b): warnings.warn( "PaddlePaddle version in [%s, %s] required, but %s installed. " "Maybe you are using a develop version, " - "please make sure the version is good with your code." % - (min_version, max_version, fluid_version.full_version)) + "please make sure the version is good with your code." + % (min_version, max_version, fluid_version.full_version) + ) else: warnings.warn( "PaddlePaddle version %s or higher is required, but %s installed, " "Maybe you are using a develop version, " - "please make sure the version is good with your code." % - (min_version, fluid_version.full_version)) + "please make sure the version is good with your code." + % (min_version, fluid_version.full_version) + ) return min_version_split = min_version.split('.') - min_version_to_check = min_version_split + zero_version[ - len(min_version_split):] + min_version_to_check = ( + min_version_split + zero_version[len(min_version_split) :] + ) if max_version is not None: max_version_split = max_version.split('.') - max_version_to_check = max_version_split + zero_version[ - len(max_version_split):] + max_version_to_check = ( + max_version_split + zero_version[len(max_version_split) :] + ) - if version_cmp(version_installed, - max_version_to_check) > 0 or version_cmp( - version_installed, min_version_to_check) < 0: + if ( + version_cmp(version_installed, max_version_to_check) > 0 + or version_cmp(version_installed, min_version_to_check) < 0 + ): raise Exception( "VersionError: PaddlePaddle version in [%s, %s] required, but %s installed." - % (min_version, max_version, fluid_version.full_version)) + % (min_version, max_version, fluid_version.full_version) + ) else: if version_cmp(version_installed, min_version_to_check) < 0: raise Exception( "VersionError: PaddlePaddle version %s or higher is required, but %s installed, " "please upgrade your PaddlePaddle to %s or other higher version." - % (min_version, fluid_version.full_version, min_version)) + % (min_version, fluid_version.full_version, min_version) + ) def _dygraph_not_support_(func): - def __impl__(*args, **kwargs): - assert not _non_static_mode( - ), "We don't support %s in dynamic graph mode" % func.__name__ + assert not _non_static_mode(), ( + "We don't support %s in dynamic graph mode" % func.__name__ + ) return func(*args, **kwargs) return __impl__ def _dygraph_only_(func): - def __impl__(*args, **kwargs): - assert _non_static_mode( - ), "We only support '%s()' in dynamic graph mode, please call 'paddle.disable_static()' to enter dynamic graph mode." % func.__name__ + assert _non_static_mode(), ( + "We only support '%s()' in dynamic graph mode, please call 'paddle.disable_static()' to enter dynamic graph mode." + % func.__name__ + ) return func(*args, **kwargs) return __impl__ def _non_static_only_(func): - def __impl__(*args, **kwargs): from .dygraph.base import in_declarative_mode - assert _non_static_mode() or in_declarative_mode( - ), "We only support '%s()' in dynamic graph mode, please call 'paddle.disable_static()' to enter dynamic graph mode." % func.__name__ + + assert _non_static_mode() or in_declarative_mode(), ( + "We only support '%s()' in dynamic graph mode, please call 'paddle.disable_static()' to enter dynamic graph mode." + % func.__name__ + ) return func(*args, **kwargs) return __impl__ def _static_only_(func): - def __impl__(*args, **kwargs): - assert not _non_static_mode( - ), "In PaddlePaddle 2.x, we turn on dynamic graph mode by default, and '%s()' is only supported in static graph mode. So if you want to use this api, please call 'paddle.enable_static()' before this api to enter static graph mode." % func.__name__ + assert not _non_static_mode(), ( + "In PaddlePaddle 2.x, we turn on dynamic graph mode by default, and '%s()' is only supported in static graph mode. So if you want to use this api, please call 'paddle.enable_static()' before this api to enter static graph mode." + % func.__name__ + ) return func(*args, **kwargs) return __impl__ @@ -546,14 +569,14 @@ def _set_pipeline_stage(stage): # TODO(zhiqiu): We should make VarBase consistent with Variable in future, for example, by inheritting # same base class. def _fake_interface_only_(func): - def __impl__(*args, **kwargs): raise AssertionError( "'%s' only can be called by `paddle.Tensor` in dynamic graph mode. Suggestions:\n" " 1. If you are in static graph mode, you can switch to dynamic graph mode by turning off `paddle.enable_static()` or calling `paddle.disable_static()`.\n" " 2. If you are using `@paddle.jit.to_static`, you can turn off ProgramTranslator by calling `paddle.jit.ProgramTranslator().enable(False)`. " "If you have to translate dynamic graph to static graph, please use other API to replace '%s'." - % (func.__name__, func.__name__)) + % (func.__name__, func.__name__) + ) return __impl__ @@ -564,13 +587,13 @@ def __impl__(*args, **kwargs): # NOTE(chenweihang): not using `wrap_decorator` here is because `wrap_decorator` will # move kwargs to args, which doesn't work in this decorate case def deprecate_stat_dict(func): - @functools.wraps(func) def wrapper(*args, **kwargs): if 'stat_dict' in kwargs: warnings.warn( "The argument `stat_dict` has deprecated, please change it to `state_dict`.", - DeprecationWarning) + DeprecationWarning, + ) kwargs['state_dict'] = kwargs['stat_dict'] kwargs.pop('stat_dict') return func(*args, **kwargs) @@ -676,7 +699,9 @@ def _cpu_num(): 'And if this parameter are set as N (equal to the number of physical CPU core) the program may be faster.\n\n' 'export CPU_NUM={} # for example, set CPU_NUM as number of physical CPU core which is {}.\n\n' '!!! The default number of CPU_NUM=1.\n'.format( - multiprocessing.cpu_count(), multiprocessing.cpu_count())) + multiprocessing.cpu_count(), multiprocessing.cpu_count() + ) + ) os.environ['CPU_NUM'] = str(1) cpu_num = os.environ.get('CPU_NUM') return int(cpu_num) @@ -858,8 +883,7 @@ def cuda_places(device_ids=None): cuda_places = static.cuda_places() """ - assert core.is_compiled_with_cuda(), \ - "Not compiled with CUDA" + assert core.is_compiled_with_cuda(), "Not compiled with CUDA" if device_ids is None: device_ids = _cuda_ids() elif not isinstance(device_ids, (list, tuple)): @@ -898,8 +922,7 @@ def xpu_places(device_ids=None): paddle.enable_static() xpu_places = static.xpu_places() """ - assert core.is_compiled_with_xpu(), \ - "Not compiled with XPU" + assert core.is_compiled_with_xpu(), "Not compiled with XPU" if device_ids is None: device_ids = _xpu_ids() elif not isinstance(device_ids, (list, tuple)): @@ -939,8 +962,7 @@ def npu_places(device_ids=None): paddle.enable_static() npu_places = static.npu_places() """ - assert core.is_compiled_with_npu(), \ - "Not compiled with NPU" + assert core.is_compiled_with_npu(), "Not compiled with NPU" if device_ids is None: device_ids = _npu_ids() elif not isinstance(device_ids, (list, tuple)): @@ -1008,8 +1030,7 @@ def cuda_pinned_places(device_count=None): cuda_pinned_places = fluid.cuda_pinned_places(1) """ - assert core.is_compiled_with_cuda(), \ - "Not compiled with CUDA" + assert core.is_compiled_with_cuda(), "Not compiled with CUDA" if device_count is None: device_count = len(_cuda_ids()) return [core.CUDAPinnedPlace()] * device_count @@ -1049,8 +1070,7 @@ def mlu_places(device_ids=None): paddle.enable_static() mlu_places = static.mlu_places() """ - assert core.is_compiled_with_mlu(), \ - "Not compiled with MLU" + assert core.is_compiled_with_mlu(), "Not compiled with MLU" if device_ids is None: device_ids = _mlu_ids() elif not isinstance(device_ids, (list, tuple)): @@ -1059,7 +1079,6 @@ def mlu_places(device_ids=None): class NameScope(object): - def __init__(self, name="", parent=None): self._children = dict() self._name = name @@ -1070,8 +1089,9 @@ def child(self, prefix): new_child = NameScope(prefix, self) self._children[prefix] = [new_child] else: - new_child = NameScope(prefix + "_%d" % len(self._children[prefix]), - self) + new_child = NameScope( + prefix + "_%d" % len(self._children[prefix]), self + ) self._children[prefix].append(new_child) return new_child @@ -1161,6 +1181,7 @@ def _full_name_scope(): def generate_control_dev_var_name(): import random + return CONTROL_DEP_VAR_PREFIX + "@" + str(random.random()) @@ -1234,8 +1255,9 @@ def dtype_is_floating(dtype): dtype = convert_np_dtype_to_dtype_(dtype) return dtype in [ - core.VarDesc.VarType.FP16, core.VarDesc.VarType.FP32, - core.VarDesc.VarType.FP64 + core.VarDesc.VarType.FP16, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP64, ] @@ -1255,16 +1277,20 @@ def _debug_string_(proto, throw_on_error=True): if not proto.IsInitialized(error_fields) and throw_on_error: raise ValueError( "{0} are not initialized.\nThe message is {1}:\n".format( - error_fields, proto)) + error_fields, proto + ) + ) return proto.__str__() -def _varbase_creator(type=core.VarDesc.VarType.LOD_TENSOR, - name=None, - shape=None, - dtype=None, - persistable=None, - **kwargs): +def _varbase_creator( + type=core.VarDesc.VarType.LOD_TENSOR, + name=None, + shape=None, + dtype=None, + persistable=None, + **kwargs +): if dtype is not None: if not isinstance(dtype, core.VarDesc.VarType): dtype = convert_np_dtype_to_dtype_(dtype) @@ -1272,16 +1298,21 @@ def _varbase_creator(type=core.VarDesc.VarType.LOD_TENSOR, if _in_eager_mode_: eager_tensor = core.eager.Tensor( dtype if dtype else core.VarDesc.VarType.FP32, - list(shape) if shape else [], name, + list(shape) if shape else [], + name, type if type else core.VarDesc.VarType.LOD_TENSOR, - True if persistable else False) + True if persistable else False, + ) eager_tensor.retain_grads() return eager_tensor else: - return core.VarBase(dtype if dtype else core.VarDesc.VarType.FP32, - list(shape) if shape else [], name, - type if type else core.VarDesc.VarType.LOD_TENSOR, - True if persistable else False) + return core.VarBase( + dtype if dtype else core.VarDesc.VarType.FP32, + list(shape) if shape else [], + name, + type if type else core.VarDesc.VarType.LOD_TENSOR, + True if persistable else False, + ) def _all_is_type(vals, expected_type): @@ -1291,12 +1322,12 @@ def _all_is_type(vals, expected_type): NOTE: BuiltIn all() will always return True if vals is empty. """ assert isinstance(vals, (list, tuple)) - if not vals: return False + if not vals: + return False return all(isinstance(v, expected_type) for v in vals) class VariableMetaClass(type): - @classmethod def __instancecheck__(cls, instance): t = type(instance) @@ -1309,7 +1340,6 @@ def __instancecheck__(cls, instance): class ParameterMetaClass(VariableMetaClass): - @classmethod def __instancecheck__(cls, instance): t = type(instance) @@ -1365,21 +1395,23 @@ class Variable(metaclass=VariableMetaClass): """ - def __init__(self, - block, - type=core.VarDesc.VarType.LOD_TENSOR, - name=None, - shape=None, - dtype=None, - lod_level=None, - capacity=None, - persistable=None, - error_clip=None, - stop_gradient=False, - is_data=False, - need_check_feed=False, - belong_to_optimizer=False, - **kwargs): + def __init__( + self, + block, + type=core.VarDesc.VarType.LOD_TENSOR, + name=None, + shape=None, + dtype=None, + lod_level=None, + capacity=None, + persistable=None, + error_clip=None, + stop_gradient=False, + is_data=False, + need_check_feed=False, + belong_to_optimizer=False, + **kwargs + ): self.block = block if name is None: name = unique_name.generate('_generated_var') @@ -1409,10 +1441,11 @@ def __init__(self, if is_new_var: self.desc.set_type(type) elif self.desc.type() != type: - raise ValueError("Variable '{0}' has been created before. The " - "previous type is {1}, the new type is {2}. They" - " are not matched".format(self.name, - self.desc.type(), type)) + raise ValueError( + "Variable '{0}' has been created before. The " + "previous type is {1}, the new type is {2}. They" + " are not matched".format(self.name, self.desc.type(), type) + ) if shape is not None: if is_new_var: @@ -1424,29 +1457,32 @@ def __init__(self, raise ValueError( "Variable '{0}' has been created before. The previous " "shape is {1}, the new shape is {2}. They are not " - "matched.".format(self.name, old_shape, shape)) + "matched.".format(self.name, old_shape, shape) + ) if dtype is not None: if is_new_var: self.desc.set_dtype(dtype) else: old_dtype = self.dtype if dtype != old_dtype: - raise ValueError("Variable '{0}' has been created before. " - "The previous data type is {1}, the new " - "data type is {2}. They are not " - "matched.".format(self.name, old_dtype, - dtype)) + raise ValueError( + "Variable '{0}' has been created before. " + "The previous data type is {1}, the new " + "data type is {2}. They are not " + "matched.".format(self.name, old_dtype, dtype) + ) if lod_level is not None: if is_new_var: self.desc.set_lod_level(lod_level) else: if lod_level != self.lod_level: - raise ValueError("Variable '{0}' has been created before. " - "The previous lod_level is {1}, the new " - "lod_level is {2}. They are not " - "matched".format(self.name, self.lod_level, - lod_level)) + raise ValueError( + "Variable '{0}' has been created before. " + "The previous lod_level is {1}, the new " + "lod_level is {2}. They are not " + "matched".format(self.name, self.lod_level, lod_level) + ) if persistable is not None: if is_new_var: self.desc.set_persistable(persistable) @@ -1456,7 +1492,9 @@ def __init__(self, "Variable '{0}' has been created before." "The previous persistable is {1}, the new " "persistable is {2}. They are not matched".format( - self.name, self.persistable, persistable)) + self.name, self.persistable, persistable + ) + ) if need_check_feed and is_new_var: self.desc.set_need_check_feed(need_check_feed) @@ -1497,20 +1535,22 @@ def detach(self): y = x.detach() """ - assert self.type == core.VarDesc.VarType.SELECTED_ROWS or \ - self.type == core.VarDesc.VarType.LOD_TENSOR, \ - "only support a variable with SELECTED_ROWS or LOD_TENSOR to be detached" + assert ( + self.type == core.VarDesc.VarType.SELECTED_ROWS + or self.type == core.VarDesc.VarType.LOD_TENSOR + ), "only support a variable with SELECTED_ROWS or LOD_TENSOR to be detached" output = self.block.create_var( name=unique_name.generate_with_ignorable_key("detach_" + self.name), dtype=self.dtype, type=self.type, persistable=self.persistable, - stop_gradient=True) + stop_gradient=True, + ) - self.block.append_op(type='share_data', - inputs={'X': [self]}, - outputs={'Out': [output]}) + self.block.append_op( + type='share_data', inputs={'X': [self]}, outputs={'Out': [output]} + ) return output @fake_interface_only @@ -1700,14 +1740,20 @@ def _to_readable_code(self): """ # VarType.LOD_TENSOR -> LOD_TENSOR type_str = str(self.type).split('.')[1] - if self.type == core.VarDesc.VarType.SELECTED_ROWS or self.type == core.VarDesc.VarType.LOD_TENSOR: + if ( + self.type == core.VarDesc.VarType.SELECTED_ROWS + or self.type == core.VarDesc.VarType.LOD_TENSOR + ): dtype_str = str(self.dtype).split('.')[1] - var_str = "{name} : {type}.shape{shape}.dtype({dtype}).stop_gradient({stop_gradient})".\ - format(name=self.name, type=type_str, shape=self.shape, - dtype=dtype_str, stop_gradient=self.stop_gradient) + var_str = "{name} : {type}.shape{shape}.dtype({dtype}).stop_gradient({stop_gradient})".format( + name=self.name, + type=type_str, + shape=self.shape, + dtype=dtype_str, + stop_gradient=self.stop_gradient, + ) else: - var_str = "{name} : {type})".\ - format(name=self.name, type=type_str) + var_str = "{name} : {type})".format(name=self.name, type=type_str) if self.is_parameter: if self.trainable: @@ -1720,12 +1766,16 @@ def _to_readable_code(self): if self.persistable: var_str = "persist " + var_str - from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context + from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, + ) + dist_context = get_default_distributed_context() dist_tensor = dist_context.get_dist_tensor_for_program(self) if dist_tensor is not None: - var_str += ", {name} = {value}".format(name="dist_attr", - value=dist_tensor) + var_str += ", {name} = {value}".format( + name="dist_attr", value=dist_tensor + ) return var_str @@ -1759,12 +1809,13 @@ def to_string(self, throw_on_error, with_details=False): print(new_variable.to_string(True, True)) """ assert isinstance(throw_on_error, bool) and isinstance( - with_details, bool) + with_details, bool + ) protostr = self.desc.serialize_to_string() proto = framework_pb2.VarDesc.FromString(bytes(protostr)) res_str = _debug_string_(proto, throw_on_error) if with_details: - additional_attr = ("error_clip", ) + additional_attr = ("error_clip",) for attr_name in additional_attr: res_str += "%s: %s\n" % (attr_name, getattr(self, attr_name)) @@ -2061,21 +2112,22 @@ def T(self): dtype=self.dtype, type=self.type, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) input_shape = self.block.create_var( name=unique_name.generate_with_ignorable_key(self.name + '.tmp'), dtype=self.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - - self.block.append_op(type='transpose2', - inputs={'X': [self]}, - outputs={ - 'Out': [out], - 'XShape': [input_shape] - }, - attrs={'axis': perm}) + stop_gradient=False, + ) + + self.block.append_op( + type='transpose2', + inputs={'X': [self]}, + outputs={'Out': [out], 'XShape': [input_shape]}, + attrs={'axis': perm}, + ) return out def clone(self): @@ -2106,11 +2158,12 @@ def clone(self): dtype=self.dtype, type=self.type, persistable=self.persistable, - stop_gradient=self.stop_gradient) + stop_gradient=self.stop_gradient, + ) - self.block.append_op(type='assign', - inputs={'X': [self]}, - outputs={'Out': [output]}) + self.block.append_op( + type='assign', inputs={'X': [self]}, outputs={'Out': [output]} + ) return output def _set_error_clip(self, error_clip): @@ -2176,8 +2229,9 @@ def _slice_indices(self, slice, length): start = upper if step < 0 else lower else: start = slice.start - start = max(start + - length, lower) if start < 0 else min(start, upper) + start = ( + max(start + length, lower) if start < 0 else min(start, upper) + ) # Compute stop. if slice.stop is None: @@ -2223,11 +2277,15 @@ def _detectContinuesSlice(self, item): for index, o in enumerate(item): if isinstance(o, int): start = int(o) - if (index > 0 and index >= self.shape[index]) \ - or (index < 0 and (index + self.shape[index]) < 0): + if (index > 0 and index >= self.shape[index]) or ( + index < 0 and (index + self.shape[index]) < 0 + ): raise IndexError("invalid index") - start = max(start + self.shape[index], 0) if start < 0 else min( - start, self.shape[index]) + start = ( + max(start + self.shape[index], 0) + if start < 0 + else min(start, self.shape[index]) + ) starts.append(start) ends.append(start + 1) elif isinstance(o, slice): @@ -2245,30 +2303,31 @@ def _cloneVar(self, copy=False): if not copy: return self.block.create_var( name=unique_name.generate_with_ignorable_key(self.name), - dtype=self.dtype) + dtype=self.dtype, + ) else: return self def _sliceVar(self, axes, starts, ends): new_var = self._cloneVar() - self.block.append_op(type="slice", - inputs={'Input': [self]}, - outputs={'Out': [new_var]}, - attrs={ - 'axes': axes, - 'starts': starts, - 'ends': ends - }) + self.block.append_op( + type="slice", + inputs={'Input': [self]}, + outputs={'Out': [new_var]}, + attrs={'axes': axes, 'starts': starts, 'ends': ends}, + ) return new_var def _concatVar(self, inputs, axis): new_var = self._cloneVar() - self.block.append_op(type="concat", - inputs={'X': inputs}, - outputs={'Out': [new_var]}, - attrs={ - 'axis': axis, - }) + self.block.append_op( + type="concat", + inputs={'X': inputs}, + outputs={'Out': [new_var]}, + attrs={ + 'axis': axis, + }, + ) return new_var def _sliceAndConcatVar(self, item, axis): @@ -2282,21 +2341,24 @@ def _sliceAndConcatVar(self, item, axis): vars = [] if step > 0: while start < stop: - vars.append(self._sliceVar([axis], [start], - [start + 1])) + vars.append( + self._sliceVar([axis], [start], [start + 1]) + ) start += step else: while start > stop: - vars.append(self._sliceVar([axis], [start], - [start + 1])) + vars.append( + self._sliceVar([axis], [start], [start + 1]) + ) start += step return self._concatVar(vars, axis) elif isinstance(item, int): if self.shape[axis] < 0: return self._cloneVar(True) index = int(item) - if (index > 0 and index >= self.shape[axis]) \ - or (index < 0 and (index + self.shape[axis]) < 0): + if (index > 0 and index >= self.shape[axis]) or ( + index < 0 and (index + self.shape[axis]) < 0 + ): raise IndexError("invalid index") return self._sliceVar([axis], [index], [index + 1]) else: @@ -2353,17 +2415,21 @@ def get_value(self, scope=None): # can not be imported at the begainning of this file. # Therefore, the above two modules are dynamically imported. from .executor import global_scope + if scope is not None and not isinstance(scope, core._Scope): raise TypeError( - "`scope` should be None or `paddle.static.Scope` type, but received {}." - .format(type(scope))) + "`scope` should be None or `paddle.static.Scope` type, but received {}.".format( + type(scope) + ) + ) if scope is None: scope = global_scope() var_temp = scope.find_var(self.name) if var_temp is None: - raise ValueError("Can not find Variable '{}' in the Scope.".format( - self.name)) + raise ValueError( + "Can not find Variable '{}' in the Scope.".format(self.name) + ) t = var_temp.get_tensor() return t @@ -2417,21 +2483,26 @@ def set_value(self, value, scope=None): if not (isinstance(value, np.ndarray) or hasattr(value, '__array__')): raise TypeError( - "`value` should be `numpy.ndarray` or `LoDTensor`, but received {}." - .format(type(value))) + "`value` should be `numpy.ndarray` or `LoDTensor`, but received {}.".format( + type(value) + ) + ) if scope is not None and not isinstance(scope, core._Scope): raise TypeError( - "`scope` should be None or `paddle.static.Scope` type, but received {}." - .format(type(scope))) + "`scope` should be None or `paddle.static.Scope` type, but received {}.".format( + type(scope) + ) + ) if scope is None: scope = global_scope() var_temp = scope.find_var(self.name) if var_temp is None: - raise ValueError("Can not find Variable '{}' in the Scope.".format( - self.name)) + raise ValueError( + "Can not find Variable '{}' in the Scope.".format(self.name) + ) t = var_temp.get_tensor() @@ -2442,8 +2513,10 @@ def set_value(self, value, scope=None): value_shape = value.shape if list(t.shape()) != list(value_shape): raise ValueError( - "{} expected a shape {}, but the received shape is {}.". - format(self.name, list(t.shape()), list(value_shape))) + "{} expected a shape {}, but the received shape is {}.".format( + self.name, list(t.shape()), list(value_shape) + ) + ) p = t._place() if p.is_cpu_place(): @@ -2492,11 +2565,12 @@ def size(self): output = self.block.create_var( name=unique_name.generate_with_ignorable_key(self.name + "_size"), - dtype=core.VarDesc.VarType.INT64) + dtype=core.VarDesc.VarType.INT64, + ) - self.block.append_op(type='size', - inputs={'Input': [self]}, - outputs={'Out': [output]}) + self.block.append_op( + type='size', inputs={'Input': [self]}, outputs={'Out': [output]} + ) return output def _set_attr(self, name, val): @@ -2595,8 +2669,8 @@ def instance(cls): def __init__(self): assert not hasattr( - self.__class__, - '_instance'), 'Please use `instance()` to get OpProtoHolder object!' + self.__class__, '_instance' + ), 'Please use `instance()` to get OpProtoHolder object!' op_protos = get_all_op_protos() self.op_proto_map = {} for proto in op_protos: @@ -2632,7 +2706,7 @@ def generated_op_attr_names(): core.op_proto_and_checker_maker.kOpRoleVarAttrName(), core.op_proto_and_checker_maker.kOpNameScopeAttrName(), core.op_proto_and_checker_maker.kOpCreationCallstackAttrName(), - core.op_proto_and_checker_maker.kOpDeviceAttrName() + core.op_proto_and_checker_maker.kOpDeviceAttrName(), } @@ -2679,24 +2753,44 @@ class Operator(object): inputs={"X": [var1, var2, var3]}, outputs={"Out": [var1]}) """ + OP_WITHOUT_KERNEL_SET = { - 'feed', 'fetch', 'recurrent', 'go', 'rnn_memory_helper_grad', - 'conditional_block', 'while', 'send', 'recv', 'listen_and_serv', - 'fl_listen_and_serv', 'ncclInit', 'select', 'checkpoint_notify', - 'gen_bkcl_id', 'c_gen_bkcl_id', 'gen_nccl_id', 'c_gen_nccl_id', - 'c_comm_init', 'c_sync_calc_stream', 'c_sync_comm_stream', - 'queue_generator', 'dequeue', 'enqueue', 'heter_listen_and_serv', - 'c_wait_comm', 'c_wait_compute', 'c_gen_hccl_id', 'c_comm_init_hccl', - 'copy_cross_scope', 'c_gen_cncl_id' + 'feed', + 'fetch', + 'recurrent', + 'go', + 'rnn_memory_helper_grad', + 'conditional_block', + 'while', + 'send', + 'recv', + 'listen_and_serv', + 'fl_listen_and_serv', + 'ncclInit', + 'select', + 'checkpoint_notify', + 'gen_bkcl_id', + 'c_gen_bkcl_id', + 'gen_nccl_id', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_sync_calc_stream', + 'c_sync_comm_stream', + 'queue_generator', + 'dequeue', + 'enqueue', + 'heter_listen_and_serv', + 'c_wait_comm', + 'c_wait_compute', + 'c_gen_hccl_id', + 'c_comm_init_hccl', + 'copy_cross_scope', + 'c_gen_cncl_id', } - def __init__(self, - block, - desc, - type=None, - inputs=None, - outputs=None, - attrs=None): + def __init__( + self, block, desc, type=None, inputs=None, outputs=None, attrs=None + ): # read attr type index from op proto to avoid unexpected type # conversions, e.g. narrowing conversion like double to float try: @@ -2710,7 +2804,8 @@ def __init__(self, if _non_static_mode(): if type is None: raise ValueError( - "`type` to initialized an Operator can not be None.") + "`type` to initialized an Operator can not be None." + ) self._type = type self.attrs = attrs if attrs else {} else: @@ -2730,11 +2825,14 @@ def __init__(self, if op_maker.kOpRoleAttrName() not in op_attrs: op_attrs[ - op_maker.kOpRoleAttrName()] = self.block.program._op_role + op_maker.kOpRoleAttrName() + ] = self.block.program._op_role role_var_name = op_maker.kOpRoleVarAttrName() - if len(self.block.program._op_role_var - ) != 0 and role_var_name not in op_attrs: + if ( + len(self.block.program._op_role_var) != 0 + and role_var_name not in op_attrs + ): op_attrs[role_var_name] = self.block.program._op_role_var if role_var_name in op_attrs and len(op_attrs[role_var_name]) == 0: @@ -2749,16 +2847,20 @@ def __init__(self, return if type is None: raise ValueError( - "`type` to initialized an Operator can not be None.") + "`type` to initialized an Operator can not be None." + ) else: callstack_var_name = op_maker.kOpCreationCallstackAttrName() op_attrs[callstack_var_name] = [] for frame in traceback.extract_stack(): op_attrs[callstack_var_name].append( ' File "{}", line {}, in {}'.format( - frame[0], frame[1], frame[2])) - op_attrs[callstack_var_name].append(' {}'.format( - frame[3])) + frame[0], frame[1], frame[2] + ) + ) + op_attrs[callstack_var_name].append( + ' {}'.format(frame[3]) + ) self.desc.set_type(type) proto = OpProtoHolder.instance().get_op_proto(type) @@ -2774,20 +2876,25 @@ def __init__(self, op_device = op_maker.kOpDeviceAttrName() op_attrs[op_device] = _current_device else: - warnings.warn("The Op(%s) is not support to set device." % - type) + warnings.warn( + "The Op(%s) is not support to set device." % type + ) if 'force_cpu' in op_attrs: - if (type == 'less_than' and op_attrs['force_cpu'] != None - ) or op_attrs['force_cpu'] != False: + if ( + type == 'less_than' and op_attrs['force_cpu'] != None + ) or op_attrs['force_cpu'] != False: warnings.warn( "The Attr(force_cpu) of Op(%s) will be deprecated in the future, " "please use 'device_guard' instead. 'device_guard' has higher priority when they are " - "used at the same time." % type) + "used at the same time." % type + ) if _current_pipeline_stage is not None: - pipeline_attr_name = 'pipeline_stage' + core.kAutoParallelSuffix( + pipeline_attr_name = ( + 'pipeline_stage' + core.kAutoParallelSuffix() + ) + self._update_desc_attr( + pipeline_attr_name, _current_pipeline_stage ) - self._update_desc_attr(pipeline_attr_name, - _current_pipeline_stage) def find_name(var_list, name): for var_name in var_list: @@ -2798,8 +2905,9 @@ def find_name(var_list, name): if inputs is not None: for in_proto in proto.inputs: found = find_name(inputs, in_proto.name) - assert found or in_proto.dispensable, "Input {} not found".format( - in_proto.name) + assert ( + found or in_proto.dispensable + ), "Input {} not found".format(in_proto.name) if found: in_args = inputs[in_proto.name] if not isinstance(in_args, (list, tuple)): @@ -2807,7 +2915,8 @@ def find_name(var_list, name): if not in_proto.duplicable and len(in_args) > 1: raise ValueError( "Input %s expects only one input, but %d are given." - % (in_proto.name, len(in_args))) + % (in_proto.name, len(in_args)) + ) in_arg_names = [] for index, arg in enumerate(in_args): if isinstance(arg, str): @@ -2821,8 +2930,9 @@ def find_name(var_list, name): "The type of '%s' in operator %s should be " "one of [basestring(), str, Varibale] in python2, " "or one of [str, bytes, Variable] in python3." - "but received : %s" % - (in_proto.name, type, arg)) + "but received : %s" + % (in_proto.name, type, arg) + ) self.desc.set_input(in_proto.name, in_arg_names) else: self.desc.set_input(in_proto.name, []) @@ -2833,9 +2943,12 @@ def find_name(var_list, name): continue if not ((m.name in outputs) or m.dispensable): raise ValueError( - ("Incorrect setting for output(s) of " - "operator \"%s\", should set: [%s].") % - (type, m.name)) + ( + "Incorrect setting for output(s) of " + "operator \"%s\", should set: [%s]." + ) + % (type, m.name) + ) for out_proto in proto.outputs: if out_proto.name not in outputs: continue @@ -2845,7 +2958,8 @@ def find_name(var_list, name): if not out_proto.duplicable and len(out_args) > 1: raise ValueError( "Output %s expects only one output, but %d are given." - % (out_proto.name, len(out_args))) + % (out_proto.name, len(out_args)) + ) out_arg_names = [] for arg in out_args: if isinstance(arg, str): @@ -2866,27 +2980,32 @@ def find_name(var_list, name): raise TypeError("'attrs' should be a dict.") for attr in proto.attrs: attr_name = attr.name - if (attr_name - not in op_attrs) or (op_attrs[attr_name] is None): + if (attr_name not in op_attrs) or ( + op_attrs[attr_name] is None + ): continue attr_val = op_attrs[attr_name] self._update_desc_attr(attr_name, attr_val) for attr_name in extra_attrs_map.keys(): - if (attr_name - not in op_attrs) or (op_attrs[attr_name] is None): - self._update_desc_attr(attr_name, - extra_attrs_map[attr_name]) + if (attr_name not in op_attrs) or ( + op_attrs[attr_name] is None + ): + self._update_desc_attr( + attr_name, extra_attrs_map[attr_name] + ) else: self._update_desc_attr(attr_name, op_attrs[attr_name]) # proto.attrs doesn't include ipu_index if core.is_compiled_with_ipu(): if global_ipu_index >= 0: - self._update_desc_attr(ipu_index_attr_name, - global_ipu_index) + self._update_desc_attr( + ipu_index_attr_name, global_ipu_index + ) if global_ipu_stage >= 0: - self._update_desc_attr(ipu_stage_attr_name, - global_ipu_stage) + self._update_desc_attr( + ipu_stage_attr_name, global_ipu_stage + ) self.desc.check_attrs() if self._has_kernel(type): @@ -2945,7 +3064,8 @@ def _to_readable_code(self, skip_op_callstack=True): assert isinstance( skip_op_callstack, bool ), "skip_op_callstack parameter's type is error, expect bool, received {}".format( - type(skip_op_callstack)) + type(skip_op_callstack) + ) outputs_str = "{" for i in range(0, len(self.output_names)): outputs_str += "{name}=".format(name=self.output_names[i]) @@ -2975,9 +3095,9 @@ def _to_readable_code(self, skip_op_callstack=True): attr_type = self.desc.attr_type(name, True) if attr_type == core.AttrType.VAR: attr_var_name = self.desc.attr(name, True).name() - a = "{name} = Var['{value}']".format(name=name, - type=attr_type, - value=attr_var_name) + a = "{name} = Var['{value}']".format( + name=name, type=attr_type, value=attr_var_name + ) attrs_str += a if i != len(attr_names) - 1: attrs_str += ", " @@ -2988,7 +3108,8 @@ def _to_readable_code(self, skip_op_callstack=True): "'%s'" % var.name() for var in self.desc.attr(name, True) ] a = "{name} = Vars[{value}]".format( - name=name, type=attr_type, value=','.join(attr_var_names)) + name=name, type=attr_type, value=','.join(attr_var_names) + ) attrs_str += a if i != len(attr_names) - 1: attrs_str += ", " @@ -2996,7 +3117,8 @@ def _to_readable_code(self, skip_op_callstack=True): if attr_type == core.AttrType.BLOCK: a = "{name} = block[{value}]".format( - name=name, type=attr_type, value=self._block_attr_id(name)) + name=name, type=attr_type, value=self._block_attr_id(name) + ) attrs_str += a if i != len(attr_names) - 1: attrs_str += ", " @@ -3004,17 +3126,19 @@ def _to_readable_code(self, skip_op_callstack=True): if attr_type == core.AttrType.BLOCKS: a = "{name} = blocks{value}".format( - name=name, - type=attr_type, - value=self._blocks_attr_ids(name)) + name=name, type=attr_type, value=self._blocks_attr_ids(name) + ) attrs_str += a if i != len(attr_names) - 1: attrs_str += ", " continue # it is bytes of serialized protobuf - if is_compiled_with_cinn( - ) and self.type == 'cinn_launch' and name == 'compilation_key': + if ( + is_compiled_with_cinn() + and self.type == 'cinn_launch' + and name == 'compilation_key' + ): key = self.desc.attr(name) v = core.get_serialize_comile_key(key) prog = Program() @@ -3026,28 +3150,36 @@ def _to_readable_code(self, skip_op_callstack=True): else: value = self.desc.attr(name) - a = "{name} = {value}".format(name=name, - type=attr_type, - value=value) + a = "{name} = {value}".format( + name=name, type=attr_type, value=value + ) attrs_str += a if i != len(attr_names) - 1: attrs_str += ", " - from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context + from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, + ) + dist_context = get_default_distributed_context() dist_op = dist_context.get_dist_op_for_program(self) if dist_op is not None: - attrs_str += ", {name} = {value}".format(name="dist_attr", - value=dist_op) + attrs_str += ", {name} = {value}".format( + name="dist_attr", value=dist_op + ) if outputs_str != "{}": - op_str = "{outputs} = {op_type}(inputs={inputs}, {attrs})".\ - format(outputs=outputs_str, op_type=self.type, - inputs=inputs_str, attrs=attrs_str) + op_str = "{outputs} = {op_type}(inputs={inputs}, {attrs})".format( + outputs=outputs_str, + op_type=self.type, + inputs=inputs_str, + attrs=attrs_str, + ) else: - op_str = "{op_type}(inputs={inputs}, {attrs})".\ - format(op_type=self.type, inputs=inputs_str, attrs=attrs_str) + op_str = "{op_type}(inputs={inputs}, {attrs})".format( + op_type=self.type, inputs=inputs_str, attrs=attrs_str + ) return op_str def __str__(self): @@ -3133,7 +3265,8 @@ def idx(self): if op == self: return i raise ValueError( - "Can't find op itself in it's block. It could be a bug of Paddle.") + "Can't find op itself in it's block. It could be a bug of Paddle." + ) def has_attr(self, name): """ @@ -3195,8 +3328,9 @@ def _update_desc_attr(self, name, val): self.desc.set_block_attr(name, val.desc) elif isinstance(val, list) and val and _all_is_type(val, Block): self.desc.set_blocks_attr(name, [v.desc for v in val]) - elif isinstance(val, core.BlockDesc) or \ - isinstance(val, core.ProgramDesc): + elif isinstance(val, core.BlockDesc) or isinstance( + val, core.ProgramDesc + ): self.desc.set_serialized_attr(name, val.serialize_to_string()) else: self._update_desc_plain_attr(name, val) @@ -3277,7 +3411,7 @@ def _block_attr(self, name): """ id = self._block_attr_id(name) - assert (id >= 0 and id < len(self.block.program.blocks)) + assert id >= 0 and id < len(self.block.program.blocks) return self.block.program.blocks[id] def _blocks_attr(self, name): @@ -3292,7 +3426,7 @@ def _blocks_attr(self, name): """ attrs = [] for i in self._blocks_attr_ids(name): - assert (i >= 0 and i < len(self.block.program.blocks)) + assert i >= 0 and i < len(self.block.program.blocks) attrs.append(self.block.program.blocks[i]) return attrs @@ -3321,8 +3455,11 @@ def _var_attr(self, name): Variable: the Variable attribute. """ attr_type = self.desc.attr_type(name, True) - assert attr_type == core.AttrType.VAR, "Required type attr({}) is Variable, but received {}".format( - name, attr_type) + assert ( + attr_type == core.AttrType.VAR + ), "Required type attr({}) is Variable, but received {}".format( + name, attr_type + ) attr_var_name = self.desc.attr(name, True).name() return self.block._var_recursive(attr_var_name) @@ -3337,8 +3474,11 @@ def _vars_attr(self, name): Variables: the Variables attribute. """ attr_type = self.desc.attr_type(name, True) - assert attr_type == core.AttrType.VARS, "Required type attr({}) is list[Variable], but received {}".format( - name, attr_type) + assert ( + attr_type == core.AttrType.VARS + ), "Required type attr({}) is list[Variable], but received {}".format( + name, attr_type + ) attr_vars = [ self.block._var_recursive(var.name()) for var in self.desc.attr(name, True) @@ -3485,7 +3625,8 @@ def _to_readable_code(self, skip_op_callstack=True): assert isinstance( skip_op_callstack, bool ), "skip_op_callstack parameter's type is error, expect bool, received {}".format( - type(skip_op_callstack)) + type(skip_op_callstack) + ) block_str = "{ // block " block_str += "{}\n".format(self.idx) for var in list(self.vars.values()): @@ -3493,7 +3634,8 @@ def _to_readable_code(self, skip_op_callstack=True): block_str += "\n" for op in self.ops: block_str += " {}\n".format( - op._to_readable_code(skip_op_callstack)) + op._to_readable_code(skip_op_callstack) + ) block_str += "}" return block_str @@ -3512,17 +3654,22 @@ def to_string(self, throw_on_error, with_details=False): str: The debug string. """ assert isinstance(throw_on_error, bool) and isinstance( - with_details, bool) + with_details, bool + ) if with_details: re_add_indent = re.compile(r"\n(.)") res_str = "blocks {\n idx: %d\n parent_idx: %d" % ( - self.idx, self.parent_idx) + self.idx, + self.parent_idx, + ) for var in list(self.vars.values()): res_str += "\n vars {\n %s }" % re_add_indent.sub( - r"\n \1", var.to_string(throw_on_error, with_details)) + r"\n \1", var.to_string(throw_on_error, with_details) + ) for op in self.ops: res_str += "\n ops {\n %s }" % re_add_indent.sub( - r"\n \1", op.to_string(throw_on_error)) + r"\n \1", op.to_string(throw_on_error) + ) res_str += "\n}" else: protostr = self.desc.serialize_to_string() @@ -3580,8 +3727,9 @@ def var(self, name): """ if not isinstance(name, str): raise TypeError( - "var require string as parameter, but get %s instead." % - (type(name))) + "var require string as parameter, but get %s instead." + % (type(name)) + ) v = self.vars.get(name, None) if v is None: raise ValueError("var %s not in this block" % name) @@ -3647,8 +3795,11 @@ def all_parameters(self): return list(self.iter_parameters()) def iter_parameters(self): - return (item[1] for item in self.vars.items() - if isinstance(item[1], Parameter)) + return ( + item[1] + for item in self.vars.items() + if isinstance(item[1], Parameter) + ) def create_var(self, *args, **kwargs): if _non_static_mode(): @@ -3680,8 +3831,9 @@ def _rename_var(self, name, new_name): """ # Ensure the type of name and new_name is str name = name.decode() if isinstance(name, bytes) else name - new_name = new_name.decode() if isinstance(new_name, - bytes) else new_name + new_name = ( + new_name.decode() if isinstance(new_name, bytes) else new_name + ) if not self.has_var(name): raise ValueError("var %s is not in current block" % name) @@ -3705,43 +3857,51 @@ def _rename_var(self, name, new_name): d = self.desc.find_var(new_name.encode()) if var_type == "Parameter": if in_dygraph_mode(): - var = EagerParamBase(d.shape(), - d.dtype(), - type=orig_var_type, - name=new_name, - stop_gradient=stop_gradient, - trainable=trainable, - optimize_attr=optimize_attr, - regularizer=regularizer, - error_clip=error_clip) + var = EagerParamBase( + d.shape(), + d.dtype(), + type=orig_var_type, + name=new_name, + stop_gradient=stop_gradient, + trainable=trainable, + optimize_attr=optimize_attr, + regularizer=regularizer, + error_clip=error_clip, + ) else: if _in_legacy_dygraph(): - var = ParamBase(d.shape(), - d.dtype(), - type=orig_var_type, - name=new_name, - stop_gradient=stop_gradient, - trainable=trainable, - optimize_attr=optimize_attr, - regularizer=regularizer, - error_clip=error_clip) + var = ParamBase( + d.shape(), + d.dtype(), + type=orig_var_type, + name=new_name, + stop_gradient=stop_gradient, + trainable=trainable, + optimize_attr=optimize_attr, + regularizer=regularizer, + error_clip=error_clip, + ) else: - var = Parameter(self, - d.shape(), - d.dtype(), - type=orig_var_type, - name=new_name, - stop_gradient=stop_gradient, - trainable=trainable, - optimize_attr=optimize_attr, - regularizer=regularizer, - error_clip=error_clip) + var = Parameter( + self, + d.shape(), + d.dtype(), + type=orig_var_type, + name=new_name, + stop_gradient=stop_gradient, + trainable=trainable, + optimize_attr=optimize_attr, + regularizer=regularizer, + error_clip=error_clip, + ) elif var_type == "Variable": - var = Variable(self, - type=orig_var_type, - name=new_name, - error_clip=error_clip, - stop_gradient=stop_gradient) + var = Variable( + self, + type=orig_var_type, + name=new_name, + error_clip=error_clip, + stop_gradient=stop_gradient, + ) # rename the python side, _sync_with_cpp will only add # new vars/ops to python side. @@ -3778,8 +3938,9 @@ def _is_inited_by(block, var): # Think of "c_broadcast" and "c_sync_comm_stream" as a special case here. # NOTE: "coalesce_tensor" is a special case for rnn with cudnn support if op.type in [ - "c_broadcast", "c_sync_comm_stream", - "coalesce_tensor" + "c_broadcast", + "c_sync_comm_stream", + "coalesce_tensor", ]: continue init_ops.append(op) @@ -3789,9 +3950,12 @@ def _is_inited_by(block, var): init_ops = _is_inited_by(global_block, param) init_ops_len = len(init_ops) if init_ops_len > 1: - raise RuntimeError("param " + param.name + - " is inited by multiple init ops " + - str(init_ops)) + raise RuntimeError( + "param " + + param.name + + " is inited by multiple init ops " + + str(init_ops) + ) elif init_ops_len == 1: # TODO already inited, do nothing, should log a warning pass @@ -3813,24 +3977,31 @@ def append_op(self, *args, **kwargs): warnings.warn( "Op `%s` is executed through `append_op` under the dynamic mode, " "the corresponding API implementation needs to be upgraded to " - "using `_C_ops` method." % type, DeprecationWarning) - op = Operator(block=self, - desc=None, - type=type, - inputs=None, - outputs=None, - attrs=attrs) + "using `_C_ops` method." % type, + DeprecationWarning, + ) + op = Operator( + block=self, + desc=None, + type=type, + inputs=None, + outputs=None, + attrs=attrs, + ) # record ops in tracer rather than blocks # # TODO(minqiyang): add op stop_gradient support in static mode too. # currently, we only support stop_gradient in dygraph mode. - _dygraph_tracer().trace_op(type, kwargs.get("inputs", {}), - kwargs.get("outputs", - {}), attrs if attrs else {}, - kwargs.get("stop_gradient", False), - inplace_map) + _dygraph_tracer().trace_op( + type, + kwargs.get("inputs", {}), + kwargs.get("outputs", {}), + attrs if attrs else {}, + kwargs.get("stop_gradient", False), + inplace_map, + ) else: from paddle.fluid.dygraph.base import param_guard @@ -3841,12 +4012,14 @@ def append_op(self, *args, **kwargs): inputs = kwargs.get("inputs", None) outputs = kwargs.get("outputs", None) with param_guard(inputs), param_guard(outputs): - op = Operator(block=self, - desc=op_desc, - type=kwargs.get("type", None), - inputs=inputs, - outputs=outputs, - attrs=kwargs.get("attrs", None)) + op = Operator( + block=self, + desc=op_desc, + type=kwargs.get("type", None), + inputs=inputs, + outputs=outputs, + attrs=kwargs.get("attrs", None), + ) self.ops.append(op) @@ -3913,25 +4086,27 @@ def _prepend_op(self, *args, **kwargs): if _non_static_mode(): type = kwargs.get("type", None) attrs = kwargs.get("attrs", {}) - op = Operator(self, - None, - type=type, - inputs=None, - outputs=None, - attrs=attrs) - - _dygraph_tracer().trace_op(type, kwargs.get("inputs", {}), - kwargs.get("outputs", {}), - attrs if attrs else {}, - kwargs.get("stop_gradient", False)) + op = Operator( + self, None, type=type, inputs=None, outputs=None, attrs=attrs + ) + + _dygraph_tracer().trace_op( + type, + kwargs.get("inputs", {}), + kwargs.get("outputs", {}), + attrs if attrs else {}, + kwargs.get("stop_gradient", False), + ) else: op_desc = self.desc._prepend_op() - op = Operator(self, - op_desc, - type=kwargs.get("type", None), - inputs=kwargs.get("inputs", None), - outputs=kwargs.get("outputs", None), - attrs=kwargs.get("attrs", None)) + op = Operator( + self, + op_desc, + type=kwargs.get("type", None), + inputs=kwargs.get("inputs", None), + outputs=kwargs.get("outputs", None), + attrs=kwargs.get("attrs", None), + ) self.ops.insert(0, op) return op @@ -3948,17 +4123,21 @@ def _sync_with_cpp(self): if var.has_stop_gradient(): is_stop_gradient = var.stop_gradient() if var.has_is_parameter() and var.is_parameter(): - self.create_parameter(name=var.name(), - desc=var, - type=var.type(), - shape=var.shape(), - dtype=var.dtype(), - stop_gradient=is_stop_gradient) + self.create_parameter( + name=var.name(), + desc=var, + type=var.type(), + shape=var.shape(), + dtype=var.dtype(), + stop_gradient=is_stop_gradient, + ) else: - self.create_var(name=var.name(), - desc=var, - type=var.type(), - stop_gradient=is_stop_gradient) + self.create_var( + name=var.name(), + desc=var, + type=var.type(), + stop_gradient=is_stop_gradient, + ) # sync variables removed from c++ end for var in list(self.vars.keys()): @@ -4004,9 +4183,12 @@ def _sync_with_cpp(self): ops_in_cpp_index = 0 ops_in_python_index = 0 while ops_in_python_index < len( - self.ops) and ops_in_cpp_index < len(ops_in_cpp): - if self.ops[ops_in_python_index].desc != ops_in_cpp[ - ops_in_cpp_index]: + self.ops + ) and ops_in_cpp_index < len(ops_in_cpp): + if ( + self.ops[ops_in_python_index].desc + != ops_in_cpp[ops_in_cpp_index] + ): del self.ops[ops_in_python_index] else: ops_in_cpp_index += 1 @@ -4032,7 +4214,8 @@ def _copy_param_info_from(self, other): """ if not isinstance(other, Block): raise TypeError( - "_copy_param_info_from should be invoked with Block") + "_copy_param_info_from should be invoked with Block" + ) for p in other.iter_parameters(): assert isinstance(p, Parameter) v = self.vars.get(p.name, None) @@ -4042,28 +4225,32 @@ def _copy_param_info_from(self, other): assert isinstance(v, Variable) new_p = None if in_dygraph_mode(): - new_p = EagerParamBase(shape=v.shape, - dtype=v.dtype, - type=v.type, - lod_level=v.lod_level, - stop_gradient=p.stop_gradient, - trainable=p.trainable, - optimize_attr=p.optimize_attr, - regularizer=p.regularizer, - error_clip=p.error_clip, - name=v.name) + new_p = EagerParamBase( + shape=v.shape, + dtype=v.dtype, + type=v.type, + lod_level=v.lod_level, + stop_gradient=p.stop_gradient, + trainable=p.trainable, + optimize_attr=p.optimize_attr, + regularizer=p.regularizer, + error_clip=p.error_clip, + name=v.name, + ) else: if _in_legacy_dygraph(): - new_p = ParamBase(shape=v.shape, - dtype=v.dtype, - type=v.type, - lod_level=v.lod_level, - stop_gradient=p.stop_gradient, - trainable=p.trainable, - optimize_attr=p.optimize_attr, - regularizer=p.regularizer, - error_clip=p.error_clip, - name=v.name) + new_p = ParamBase( + shape=v.shape, + dtype=v.dtype, + type=v.type, + lod_level=v.lod_level, + stop_gradient=p.stop_gradient, + trainable=p.trainable, + optimize_attr=p.optimize_attr, + regularizer=p.regularizer, + error_clip=p.error_clip, + name=v.name, + ) else: new_p = Parameter( block=self, @@ -4071,13 +4258,15 @@ def _copy_param_info_from(self, other): dtype=v.dtype, type=v.type, lod_level=v.lod_level - if v.type == core.VarDesc.VarType.LOD_TENSOR else None, + if v.type == core.VarDesc.VarType.LOD_TENSOR + else None, stop_gradient=p.stop_gradient, trainable=p.trainable, optimize_attr=p.optimize_attr, regularizer=p.regularizer, error_clip=p.error_clip, - name=v.name) + name=v.name, + ) self.vars[new_p.name] = new_p def _clone_variable(self, var, force_persistable=True): @@ -4097,13 +4286,13 @@ def _clone_variable(self, var, force_persistable=True): ret_var = None # make STEP_SCOPES var can be safely cloned. if var.type == core.VarDesc.VarType.STEP_SCOPES: - ret_var = self.create_var(name=var.name, - persistable=var.persistable, - type=var.type) + ret_var = self.create_var( + name=var.name, persistable=var.persistable, type=var.type + ) elif var.type == core.VarDesc.VarType.RAW: - ret_var = self.create_var(name=var.name, - persistable=var.persistable, - type=var.type) + ret_var = self.create_var( + name=var.name, persistable=var.persistable, type=var.type + ) elif var.type == core.VarDesc.VarType.SELECTED_ROWS: ret_var = self.create_var( name=var.name, @@ -4112,7 +4301,8 @@ def _clone_variable(self, var, force_persistable=True): type=var.type, persistable=True if force_persistable else var.persistable, is_data=var.is_data, - need_check_feed=var.desc.need_check_feed()) + need_check_feed=var.desc.need_check_feed(), + ) else: ret_var = self.create_var( name=var.name, @@ -4122,7 +4312,8 @@ def _clone_variable(self, var, force_persistable=True): lod_level=var.lod_level, persistable=True if force_persistable else var.persistable, is_data=var.is_data, - need_check_feed=var.desc.need_check_feed()) + need_check_feed=var.desc.need_check_feed(), + ) return ret_var @@ -4132,17 +4323,20 @@ def _clone_variable(self, var, force_persistable=True): # re-constructed inside this method. The underlying VarDesc(OpDesc) # of some old Python Variables(all old Python Operators) may have # been destructed. -def _apply_pass(main_program, - startup_program, - pass_name, - pass_attrs={}, - pass_attr_types={}): +def _apply_pass( + main_program, startup_program, pass_name, pass_attrs={}, pass_attr_types={} +): assert isinstance(pass_attrs, dict), "pass_attrs must be dict" assert isinstance(pass_attr_types, dict), "pass_attr_types must be dict" tmp_main_program = core.ProgramDesc(main_program.desc) tmp_startup_program = core.ProgramDesc(startup_program.desc) - attrs = core.apply_pass(tmp_main_program, tmp_startup_program, pass_name, - pass_attrs, pass_attr_types) + attrs = core.apply_pass( + tmp_main_program, + tmp_startup_program, + pass_name, + pass_attrs, + pass_attr_types, + ) main_program._rebuild_from_desc(tmp_main_program) startup_program._rebuild_from_desc(tmp_startup_program) return attrs @@ -4160,8 +4354,9 @@ def __init__(self, node): Args: node(core.Node): C++ Node. """ - assert isinstance(node, - core.Node), 'node must be the instance of core.Node.' + assert isinstance( + node, core.Node + ), 'node must be the instance of core.Node.' self.node = node def name(self): @@ -4337,8 +4532,9 @@ def __init__(self, node): Args: node(core.Node): C++ Node. """ - assert isinstance(node, core.Node) and node.is_var(), \ - 'node must be the instance of core.Node and it must be a variable node.' + assert ( + isinstance(node, core.Node) and node.is_var() + ), 'node must be the instance of core.Node and it must be a variable node.' super(IrVarNode, self).__init__(node) self.node = node @@ -4349,8 +4545,9 @@ def set_shape(self, shape): Args: shape(list): shape to be set. """ - assert self.node.var() is not None, \ - "The node variable description can not be None." + assert ( + self.node.var() is not None + ), "The node variable description can not be None." self.node.var().set_shape(shape) def persistable(self): @@ -4360,8 +4557,9 @@ def persistable(self): Returns: bool: indicate whether the variable is persistable. """ - assert self.node.var() is not None, \ - "The node variable description can not be None." + assert ( + self.node.var() is not None + ), "The node variable description can not be None." return self.node.var().persistable() def type(self): @@ -4371,8 +4569,9 @@ def type(self): Returns: core.VarDesc.VarType: the variable type. """ - assert self.node.var() is not None, \ - "The node variable description can not be None." + assert ( + self.node.var() is not None + ), "The node variable description can not be None." return self.node.var().type() def dtype(self): @@ -4382,8 +4581,9 @@ def dtype(self): Returns: core.VarDesc.VarType: the variable data type. """ - assert self.node.var() is not None, \ - "The node variable description can not be None." + assert ( + self.node.var() is not None + ), "The node variable description can not be None." return self.node.var().dtype() def shape(self): @@ -4393,8 +4593,9 @@ def shape(self): Returns: list: the variable shape. """ - assert self.node.var() is not None, \ - "The node variable description can not be None." + assert ( + self.node.var() is not None + ), "The node variable description can not be None." return self.node.var().shape() @property @@ -4430,8 +4631,9 @@ def __init__(self, node): Args: node(core.Node): C++ Node. """ - assert isinstance(node, core.Node) and node.is_op(), \ - 'node must be the instance of core.Node and it must be a operator node.' + assert ( + isinstance(node, core.Node) and node.is_op() + ), 'node must be the instance of core.Node and it must be a operator node.' super(IrOpNode, self).__init__(node) self.node = node @@ -4443,8 +4645,9 @@ def rename_input(self, old_input_name, new_input_name): old_input_name(str): the old input name. new_input_name(str): the new input name. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." self.node.op()._rename_input(old_input_name, new_input_name) def rename_output(self, old_output_name, new_output_name): @@ -4455,8 +4658,9 @@ def rename_output(self, old_output_name, new_output_name): old_output_name(str): the old output name. new_output_name(str): the new output name. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." self.node.op()._rename_output(old_output_name, new_output_name) def input(self, name): @@ -4469,8 +4673,9 @@ def input(self, name): Returns: list(str): the argument name list. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." return self.node.op().input(name) def output(self, name): @@ -4483,8 +4688,9 @@ def output(self, name): Returns: list(str): the argument name list. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." return self.node.op().output(name) def set_type(self, new_type): @@ -4494,8 +4700,9 @@ def set_type(self, new_type): Args: new_type(str): new operator type to be set. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." return self.node.op().set_type(new_type) def set_attr(self, name, val): @@ -4512,8 +4719,9 @@ def _update_desc_attr(self, name, val): """ Update the value of the op desc's attribute by attribute's name. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." desc = self.node.op() if isinstance(val, Variable): desc.set_var_attr(name, val.desc) @@ -4523,8 +4731,9 @@ def _update_desc_attr(self, name, val): desc.set_block_attr(name, val.desc) elif isinstance(val, list) and val and _all_is_type(val, Block): desc.set_blocks_attr(name, [v.desc for v in val]) - elif isinstance(val, core.BlockDesc) or \ - isinstance(val, core.ProgramDesc): + elif isinstance(val, core.BlockDesc) or isinstance( + val, core.ProgramDesc + ): desc.set_serialized_attr(name, val.serialize_to_string()) else: desc._set_attr(name, val) @@ -4536,8 +4745,9 @@ def input_arg_names(self): Returns: list(str): input arguments' names of this op node. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." return self.node.op().input_arg_names() def output_arg_names(self): @@ -4547,8 +4757,9 @@ def output_arg_names(self): Returns: list(str): output arguments' names of this op node. """ - assert self.node.op() is not None, \ - "The node operator description can not be None." + assert ( + self.node.op() is not None + ), "The node operator description can not be None." return self.node.op().output_arg_names() @property @@ -4589,7 +4800,8 @@ def __init__(self, graph, for_test=False): for_test(bool): True for the test graph and false for the train graph. """ assert isinstance( - graph, core.Graph), 'graph must be the instance of core.Graph.' + graph, core.Graph + ), 'graph must be the instance of core.Graph.' self.graph = graph self._for_test = for_test @@ -4630,8 +4842,11 @@ def all_persistable_nodes(self): """ persistable_nodes = set() for node in self.graph.nodes(): - if node.is_var() and node.var() is not None and node.var( - ).persistable(): + if ( + node.is_var() + and node.var() is not None + and node.var().persistable() + ): persistable_nodes.add(node) return {IrVarNode(p) for p in persistable_nodes} @@ -4738,13 +4953,15 @@ def create_op_node(self, op_type, attrs, inputs, outputs): for input_name, var_nodes in inputs.items(): if not isinstance(var_nodes, list): var_nodes = [var_nodes] - op_desc.set_input(input_name, - [var_node.name() for var_node in var_nodes]) + op_desc.set_input( + input_name, [var_node.name() for var_node in var_nodes] + ) for output_name, var_nodes in outputs.items(): if not isinstance(var_nodes, list): var_nodes = [var_nodes] - op_desc.set_output(output_name, - [var_node.name() for var_node in var_nodes]) + op_desc.set_output( + output_name, [var_node.name() for var_node in var_nodes] + ) return IrOpNode(self.graph.create_op_node(op_desc)) def create_op_node_from_desc(self, op_desc): @@ -4768,9 +4985,11 @@ def update_input_link(self, old_input_node, new_input_node, op_node): new_input_node(IrNode): the new input node of the giving op_node. op_node(IrOpNode): the operator node that is needed to update input's link. """ - assert old_input_node.node in self.graph.nodes() and new_input_node.node in \ - self.graph.nodes() and op_node.node in self.graph.nodes(), \ - 'The three arguments(old_input_node&new_input_node&op_node) must be in the graph nodes.' + assert ( + old_input_node.node in self.graph.nodes() + and new_input_node.node in self.graph.nodes() + and op_node.node in self.graph.nodes() + ), 'The three arguments(old_input_node&new_input_node&op_node) must be in the graph nodes.' old_input_node.remove_output(op_node) op_node.remove_input(old_input_node) new_input_node.append_output(op_node) @@ -4786,9 +5005,11 @@ def update_output_link(self, old_output_node, new_output_node, op_node): new_output_node(IrNode): the new output node of the giving op_node. op_node(IrOpNode): the operator node that is needed to update input's link. """ - assert old_output_node.node in self.graph.nodes() and new_output_node.node in \ - self.graph.nodes() and op_node.node in self.graph.nodes(), \ - 'The three arguments(old_output_node &new_output_node &op_node) must be in the graph nodes.' + assert ( + old_output_node.node in self.graph.nodes() + and new_output_node.node in self.graph.nodes() + and op_node.node in self.graph.nodes() + ), 'The three arguments(old_output_node &new_output_node &op_node) must be in the graph nodes.' old_output_node.remove_input(op_node) op_node.remove_output(old_output_node) new_output_node.append_input(op_node) @@ -4804,9 +5025,11 @@ def link_to(self, node_in, node_out): node_out(IrNode): the output node. """ assert node_in.node in self.graph.nodes(), ( - 'node_in(%s) must be in the graph nodes.' % node_in.node.name()) + 'node_in(%s) must be in the graph nodes.' % node_in.node.name() + ) assert node_out.node in self.graph.nodes(), ( - 'node_out(%s) must be in the graph nodes.' % node_out.node.name()) + 'node_out(%s) must be in the graph nodes.' % node_out.node.name() + ) node_in.append_output(node_out) node_out.append_input(node_in) @@ -4843,8 +5066,8 @@ def resolve_hazard(self): ] else: var_nodes[each_var_name].append( - self._find_node_by_name(node.outputs, - each_var_name)) + self._find_node_by_name(node.outputs, each_var_name) + ) self.graph.resolve_hazard(var_nodes) def has_circle(self): @@ -4906,13 +5129,15 @@ def draw(self, save_path, name, marked_nodes=None, remove_ctr_var=True): def _convert_to_pdf(dot_file_path): pdf_save_path = os.path.splitext(dot_file_path)[0] + '.pdf' - exited_code = subprocess.call('dot -Tpdf ' + dot_file_path + - ' -o ' + pdf_save_path, - shell=True) + exited_code = subprocess.call( + 'dot -Tpdf ' + dot_file_path + ' -o ' + pdf_save_path, + shell=True, + ) if exited_code != 0: print('The dot command is needed for creating pdf files.') - print('The {} is saved as the dot filetype.'.format( - dot_file_path)) + print( + 'The {} is saved as the dot filetype.'.format(dot_file_path) + ) remove_ctr_vars = set() if remove_ctr_var: @@ -4969,7 +5194,8 @@ def _find_node_by_name(self, nodes, node_name): if n.name() == node_name: target_node = n assert target_node is not None, ( - "Cannot find the target node (%s)in the giving set." % node_name) + "Cannot find the target node (%s)in the giving set." % node_name + ) return target_node def _update_desc_attr(self, desc, name, val): @@ -4984,8 +5210,9 @@ def _update_desc_attr(self, desc, name, val): desc.set_block_attr(name, val.desc) elif isinstance(val, list) and val and _all_is_type(val, Block): desc.set_blocks_attr(name, [v.desc for v in val]) - elif isinstance(val, core.BlockDesc) or \ - isinstance(val, core.ProgramDesc): + elif isinstance(val, core.BlockDesc) or isinstance( + val, core.ProgramDesc + ): desc.set_serialized_attr(name, val.serialize_to_string()) else: desc._set_attr(name, val) @@ -5087,7 +5314,8 @@ def __init__(self): # identifier for auto checkpoint self._auto_checkpoint_name = unique_name.generate( - "__auto_checkpoint_program__") + "__auto_checkpoint_program__" + ) # compiled program, i.e. Graph self._graph = None @@ -5107,7 +5335,7 @@ def get_var_desc_attr_or_none(var_desc, attr_name, allowed_types): all_new_vars = [] block_num = new_desc.num_blocks() for idx in range(block_num): - if (idx > (len(self.blocks) - 1)): + if idx > (len(self.blocks) - 1): self._create_block() new_block_desc = new_desc.block(idx) all_new_vars.append([]) @@ -5119,60 +5347,75 @@ def get_var_desc_attr_or_none(var_desc, attr_name, allowed_types): old_var = None kwargs = { - 'type': - new_var_desc.type(), - 'name': - new_var_desc.name(), - 'shape': - get_var_desc_attr_or_none(new_var_desc, "shape", [ - core.VarDesc.VarType.LOD_TENSOR, - core.VarDesc.VarType.SELECTED_ROWS, - core.VarDesc.VarType.LOD_TENSOR_ARRAY, - ]), - 'dtype': - get_var_desc_attr_or_none(new_var_desc, "dtype", [ - core.VarDesc.VarType.LOD_TENSOR, - core.VarDesc.VarType.SELECTED_ROWS, - core.VarDesc.VarType.LOD_TENSOR_ARRAY, - ]), - 'lod_level': - get_var_desc_attr_or_none(new_var_desc, "lod_level", [ - core.VarDesc.VarType.LOD_TENSOR, - core.VarDesc.VarType.LOD_TENSOR_ARRAY, - ]), - 'error_clip': - old_var.error_clip if old_var is not None else None, - 'stop_gradient': - old_var.stop_gradient if old_var is not None else False, - 'is_data': - old_var.is_data if old_var is not None else False, - 'need_check_feed': - new_var_desc.need_check_feed(), - 'belong_to_optimizer': - old_var.belong_to_optimizer - if old_var is not None else False, + 'type': new_var_desc.type(), + 'name': new_var_desc.name(), + 'shape': get_var_desc_attr_or_none( + new_var_desc, + "shape", + [ + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.SELECTED_ROWS, + core.VarDesc.VarType.LOD_TENSOR_ARRAY, + ], + ), + 'dtype': get_var_desc_attr_or_none( + new_var_desc, + "dtype", + [ + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.SELECTED_ROWS, + core.VarDesc.VarType.LOD_TENSOR_ARRAY, + ], + ), + 'lod_level': get_var_desc_attr_or_none( + new_var_desc, + "lod_level", + [ + core.VarDesc.VarType.LOD_TENSOR, + core.VarDesc.VarType.LOD_TENSOR_ARRAY, + ], + ), + 'error_clip': old_var.error_clip + if old_var is not None + else None, + 'stop_gradient': old_var.stop_gradient + if old_var is not None + else False, + 'is_data': old_var.is_data + if old_var is not None + else False, + 'need_check_feed': new_var_desc.need_check_feed(), + 'belong_to_optimizer': old_var.belong_to_optimizer + if old_var is not None + else False, } if isinstance(old_var, Parameter): - kwargs.update({ - 'trainable': old_var.trainable, - 'optimize_attr': old_var.optimize_attr, - 'regularizer': old_var.regularizer, - 'do_model_average': old_var.do_model_average, - 'need_clip': old_var.need_clip, - 'is_distributed': old_var.is_distributed, - 'is_parameter': old_var.is_parameter, - }) - block_new_vars.append({ - 'class': Parameter, - 'kwargs': copy.deepcopy(kwargs), - }) + kwargs.update( + { + 'trainable': old_var.trainable, + 'optimize_attr': old_var.optimize_attr, + 'regularizer': old_var.regularizer, + 'do_model_average': old_var.do_model_average, + 'need_clip': old_var.need_clip, + 'is_distributed': old_var.is_distributed, + 'is_parameter': old_var.is_parameter, + } + ) + block_new_vars.append( + { + 'class': Parameter, + 'kwargs': copy.deepcopy(kwargs), + } + ) else: kwargs['persistable'] = new_var_desc.persistable() - block_new_vars.append({ - 'class': Variable, - 'kwargs': copy.deepcopy(kwargs), - }) + block_new_vars.append( + { + 'class': Variable, + 'kwargs': copy.deepcopy(kwargs), + } + ) return all_new_vars @@ -5404,7 +5647,8 @@ def _to_readable_code(self, skip_op_callstack=True): assert isinstance( skip_op_callstack, bool ), "skip_op_callstack parameter's type is error, expect bool, received {}".format( - type(skip_op_callstack)) + type(skip_op_callstack) + ) program_str = "" for block in self.blocks: program_str += block._to_readable_code(skip_op_callstack) @@ -5446,11 +5690,13 @@ def to_string(self, throw_on_error, with_details=False): assert isinstance( throw_on_error, bool ), "The type of throw_on_error parameter is wrong, expected bool, but received {}.".format( - type(throw_on_error)) + type(throw_on_error) + ) assert isinstance( with_details, bool ), "The type of with_details parameter is wrong, expected bool, but received {}.".format( - type(with_details)) + type(with_details) + ) if with_details: res_str = "" @@ -5654,7 +5900,8 @@ def network(): if for_test: forward_prog = Program() forward_prog.desc, pruned_origin_block_id_map = core.prune_backward( - self.desc) + self.desc + ) forward_prog.blocks = [ Block(forward_prog, i) for i in range(forward_prog.desc.num_blocks()) @@ -5732,7 +5979,8 @@ def _prune_with_input(self, feeded_var_names, targets): if not isinstance(var, str): raise ValueError( "All feeded_var_names of Program._prune_with_input() can only be " - "str, but received %s." % type(var)) + "str, but received %s." % type(var) + ) # find out all variables that can be generated or updated with given feed generatable_vars = set() @@ -5760,7 +6008,8 @@ def _prune_with_input(self, feeded_var_names, targets): else: raise ValueError( "All targets of Program._prune_with_input() can only be " - "Variable or Operator, but received %s." % type(t)) + "Variable or Operator, but received %s." % type(t) + ) # NOTEZ(zhiqiu): For variable to be fed in fetch_list, there two cases: # (1) the variable is leaf, it has no op that generates it; @@ -5794,7 +6043,8 @@ def _prune_with_input(self, feeded_var_names, targets): res = Program() res.desc, pruned_origin_block_id_map = core.prune( - self.desc, set(feeded_var_names), targets_idx) + self.desc, set(feeded_var_names), targets_idx + ) res.blocks = [Block(res, i) for i in range(res.desc.num_blocks())] res._sync_with_cpp() @@ -5833,8 +6083,10 @@ def _inference_optimize(self, prune_read_op=True): root_block = res.desc.block(0) if prune_read_op: while True: - if read_op_idx >= root_block.op_size() or root_block.op( - read_op_idx).type() == 'read': + if ( + read_op_idx >= root_block.op_size() + or root_block.op(read_op_idx).type() == 'read' + ): break read_op_idx += 1 if read_op_idx < root_block.op_size(): @@ -5926,14 +6178,22 @@ def _remove_training_info(self, clip_extra=True): # for name in remove_output_list: # op.remove_output(name) - op_quant_name = core.op_proto_and_checker_maker.kOpWithQuantAttrName( + op_quant_name = ( + core.op_proto_and_checker_maker.kOpWithQuantAttrName() + ) + quant = ( + bool(op.attr(op_quant_name)) + if op_quant_name in op.attr_names() + else False ) - quant = bool(op.attr(op_quant_name) - ) if op_quant_name in op.attr_names() else False quant_attrs = [ - op_quant_name, "quantization_type", "skip_quant", - "activation_bits", "bit_length", "quantize_weight_bits", - "weight_quant_scale" + op_quant_name, + "quantization_type", + "skip_quant", + "activation_bits", + "bit_length", + "quantize_weight_bits", + "weight_quant_scale", ] for extra_attr_name in extra_attrs_map.keys(): op.remove_attr(extra_attr_name) @@ -6096,7 +6356,8 @@ def random_seed(self, seed): if not isinstance(seed, int): raise ValueError( "Program.random_seed's input seed must be an integer, but received %s." - % type(seed)) + % type(seed) + ) self._seed = seed def __repr__(self): @@ -6193,8 +6454,11 @@ def _create_block(self, parent_idx=None): Block: The new block. """ new_block_idx = len(self.blocks) - parent = self.current_block() if parent_idx is None else self.block( - parent_idx) + parent = ( + self.current_block() + if parent_idx is None + else self.block(parent_idx) + ) self.desc.append_block(parent.desc) self.current_block_idx = new_block_idx self.blocks.append(Block(self, self.current_block_idx)) @@ -6240,7 +6504,8 @@ def _copy_param_info_from(self, other): if not isinstance(other, Program): raise TypeError( "Function Program._copy_param_info_from() needs to pass in a source Program, but received %s" - % type(other)) + % type(other) + ) self.global_block()._copy_param_info_from(other.global_block()) @@ -6257,7 +6522,8 @@ def _copy_dist_param_info_from(self, other): if not isinstance(other, Program): raise TypeError( "Function Program._copy_param_info_from() needs to pass in a source Program, but received %s" - % type(other)) + % type(other) + ) self._is_distributed = other._is_distributed self._is_chief = other._is_chief self._parameters_on_pservers = other._parameters_on_pservers @@ -6285,12 +6551,12 @@ def _copy_data_info_from(self, other, pruned_origin_block_id_map=None): if not isinstance(other, Program): raise TypeError( "Function Program._copy_param_info_from() needs to pass in a source Program, but received %s" - % type(other)) + % type(other) + ) if not pruned_origin_block_id_map: pruned_origin_block_id_map = { - i: i - for i in range(self.desc.num_blocks()) + i: i for i in range(self.desc.num_blocks()) } # NOTE(zhiqiu): All vars in cloned program exist in original program. @@ -6418,10 +6684,13 @@ def state_dict(self, mode='all', scope=None): # can not be imported at the begainning of this file. # Therefore, the above two modules are dynamically imported. from .executor import global_scope + if scope is not None and not isinstance(scope, core._Scope): raise TypeError( - "`scope` should be None or `paddle.static.Scope'` type, but received {}." - .format(type(scope))) + "`scope` should be None or `paddle.static.Scope'` type, but received {}.".format( + type(scope) + ) + ) if scope is None: scope = global_scope() @@ -6429,15 +6698,19 @@ def state_dict(self, mode='all', scope=None): if not isinstance(mode, str): raise TypeError( "Type of `mode` should be string, but received {}.".format( - type(mode))) + type(mode) + ) + ) def is_parameter(var): return isinstance(var, Parameter) def is_persistable(var): - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable @@ -6456,8 +6729,10 @@ def condition(var): return is_parameter(var) or is_belong_to_optimizer(var) else: raise ValueError( - "`mode` string should be 'param', 'opt' or 'all', but received {}." - .format(mode)) + "`mode` string should be 'param', 'opt' or 'all', but received {}.".format( + mode + ) + ) var_list = filter(condition, self.list_vars()) @@ -6466,8 +6741,10 @@ def condition(var): var_temp = scope.find_var(var.name) if var_temp is None: raise ValueError( - "Can not find Variable '{}' in the scope. Make sure it is initialized" - .format(var.name)) + "Can not find Variable '{}' in the scope. Make sure it is initialized".format( + var.name + ) + ) state_dict[var.name] = var_temp.get_tensor() return state_dict @@ -6517,10 +6794,14 @@ def set_state_dict(self, state_dict, scope=None): if not isinstance(state_dict, dict): raise TypeError( "Type of `state_dict` should be dict, but received {}.".format( - type(state_dict))) + type(state_dict) + ) + ) vars_dict = {var.name: var for var in self.list_vars()} - condition = True if 'StructuredToParameterName@@' in state_dict else False + condition = ( + True if 'StructuredToParameterName@@' in state_dict else False + ) for name, value in state_dict.items(): if condition: if name == "StructuredToParameterName@@": @@ -6532,14 +6813,20 @@ def set_state_dict(self, state_dict, scope=None): vars_dict[name].set_value(value, scope) except ValueError as err: warnings.warn( - ("Skip loading for '{}'. ".format(name) + str(err))) + ("Skip loading for '{}'. ".format(name) + str(err)) + ) except TypeError as err: warnings.warn( - ("Skip loading for '{}'. ".format(name) + str(err))) + ("Skip loading for '{}'. ".format(name) + str(err)) + ) else: warnings.warn( - ("Skip loading for '{0}'. Because '{0}' not in the program." - .format(name))) + ( + "Skip loading for '{0}'. Because '{0}' not in the program.".format( + name + ) + ) + ) class Parameter(Variable, metaclass=ParameterMetaClass): @@ -6566,12 +6853,14 @@ class Parameter(Variable, metaclass=ParameterMetaClass): in optimizer. Default is True. """ - def __init__(self, - block, - shape, - dtype, - type=core.VarDesc.VarType.LOD_TENSOR, - **kwargs): + def __init__( + self, + block, + shape, + dtype, + type=core.VarDesc.VarType.LOD_TENSOR, + **kwargs + ): if shape is None: raise ValueError("The shape of Parameter should not be None") if dtype is None: @@ -6581,15 +6870,18 @@ def __init__(self, if each < 0: raise ValueError( "Each dimension of shape for Parameter must be greater than 0, but received %s" - % list(shape)) - - Variable.__init__(self, - block, - persistable=True, - shape=shape, - dtype=dtype, - type=type, - **kwargs) + % list(shape) + ) + + Variable.__init__( + self, + block, + persistable=True, + shape=shape, + dtype=dtype, + type=type, + **kwargs + ) self.trainable = kwargs.get('trainable', True) self.optimize_attr = kwargs.get('optimize_attr', {'learning_rate': 1.0}) @@ -6630,11 +6922,17 @@ def to_string(self, throw_on_error, with_details=False): print(debug_str) """ assert isinstance(throw_on_error, bool) and isinstance( - with_details, bool) + with_details, bool + ) if with_details: res_str = Variable.to_string(self, throw_on_error, True) - additional_attr = ("trainable", "optimize_attr", "regularizer", - "do_model_average", "need_clip") + additional_attr = ( + "trainable", + "optimize_attr", + "regularizer", + "do_model_average", + "need_clip", + ) for attr_name in additional_attr: res_str += "%s: %s\n" % (attr_name, getattr(self, attr_name)) else: @@ -6680,7 +6978,8 @@ def __init__(self, shape, dtype, **kwargs): if each < 0: raise ValueError( "Each dimension of shape for Parameter must be greater than 0, but received %s" - % list(shape)) + % list(shape) + ) if dtype is not None: if not isinstance(dtype, core.VarDesc.VarType): @@ -6688,10 +6987,13 @@ def __init__(self, shape, dtype, **kwargs): name = kwargs.get('name', unique_name.generate('_param_base')) - super(ParamBase, - self).__init__(dtype if dtype else core.VarDesc.VarType.FP32, - list(shape) if shape else [], name, - core.VarDesc.VarType.LOD_TENSOR, True) + super(ParamBase, self).__init__( + dtype if dtype else core.VarDesc.VarType.FP32, + list(shape) if shape else [], + name, + core.VarDesc.VarType.LOD_TENSOR, + True, + ) trainable = kwargs.get('trainable', True) self.stop_gradient = not trainable @@ -6718,7 +7020,8 @@ def trainable(self, trainable): else: raise ValueError( "The type of trainable MUST be bool, but the type is ", - type(trainable)) + type(trainable), + ) def __str__(self): """ @@ -6739,7 +7042,8 @@ def __str__(self): # [-0.54217887, 0.48439729, 0.34082305]]) """ return "Parameter containing:\n{tensor}".format( - tensor=super(ParamBase, self).__str__()) + tensor=super(ParamBase, self).__str__() + ) def __deepcopy__(self, memo): """ @@ -6822,7 +7126,8 @@ def __init__(self, shape, dtype, **kwargs): if each < 0: raise ValueError( "Each dimension of shape for Parameter must be greater than 0, but received %s" - % list(shape)) + % list(shape) + ) if dtype is not None: if not isinstance(dtype, core.VarDesc.VarType): @@ -6833,10 +7138,13 @@ def __init__(self, shape, dtype, **kwargs): if isinstance(shape, core.eager.Tensor): shape = shape.numpy() - super(EagerParamBase, - self).__init__(dtype if dtype else core.VarDesc.VarType.FP32, - list(shape) if shape else [], name, - core.VarDesc.VarType.LOD_TENSOR, True) + super(EagerParamBase, self).__init__( + dtype if dtype else core.VarDesc.VarType.FP32, + list(shape) if shape else [], + name, + core.VarDesc.VarType.LOD_TENSOR, + True, + ) self.retain_grads() trainable = kwargs.get('trainable', True) @@ -6860,7 +7168,9 @@ def set_init_func(self, obj): @dygraph_only def initialize(self): - assert self._init_func is not None, "Required self._init_func is not None, but received None." + assert ( + self._init_func is not None + ), "Required self._init_func is not None, but received None." self._init_func() # clear function handle to release resource self._init_func = None @@ -6876,13 +7186,16 @@ def trainable(self, trainable): else: raise ValueError( "The type of trainable MUST be bool, but the type is ", - type(trainable)) + type(trainable), + ) def _create_init_op(self, block): """ Call init_op_creator function to create initializer operation in block. """ - assert self._init_op_creator is not None, "Required self._init_op_creator is not None, but received None." + assert ( + self._init_op_creator is not None + ), "Required self._init_op_creator is not None, but received None." self._init_op_creator(block) def __str__(self): @@ -6904,7 +7217,8 @@ def __str__(self): # [-0.54217887, 0.48439729, 0.34082305]]) """ return "Parameter containing:\n{tensor}".format( - tensor=super(EagerParamBase, self).__str__()) + tensor=super(EagerParamBase, self).__str__() + ) def __deepcopy__(self, memo): """ @@ -7091,12 +7405,18 @@ def program_guard(main_program, startup_program=None): """ from .data_feeder import check_type - check_type(main_program, 'main_program', Program, - 'paddle.static.program_guard') + + check_type( + main_program, 'main_program', Program, 'paddle.static.program_guard' + ) main_program = switch_main_program(main_program) if startup_program is not None: - check_type(startup_program, 'startup_program', Program, - 'paddle.static.program_guard') + check_type( + startup_program, + 'startup_program', + Program, + 'paddle.static.program_guard', + ) # Tag the program __is_start_up as True startup_program._is_start_up_program_ = True startup_program = switch_startup_program(startup_program) @@ -7219,7 +7539,8 @@ def device_guard(device=None): if device not in ['cpu', 'gpu', 'npu', 'xpu', 'mlu', '', None]: raise ValueError( "The Attr(device) should be 'cpu' 'npu' 'xpu' 'mlu' or 'gpu', and it can also be empty string or None " - "when there is no need to specify device. But received %s" % device) + "when there is no need to specify device. But received %s" % device + ) if index: device = ":".join([device, index]) pre_device = switch_device(device) @@ -7249,9 +7570,11 @@ def _cuda_graph_guard(cuda_graph_attr=None): cuda_graph_attr(str|None): The cuda graph attr with the format of: cuda_graph_capture_mode;memory_pool_id;cuda_graph_id """ - assert not _non_static_mode( + assert ( + not _non_static_mode() ), "cuda_graph_guard only works under static mode" - assert core.is_compiled_with_cuda( + assert ( + core.is_compiled_with_cuda() ), "cuda_graph_guard context can be only used when Paddle is compiled with cuda" pre_mode = _switch_cuda_graph_mode(cuda_graph_attr) try: @@ -7281,7 +7604,8 @@ def set_flags(flags): _global_flags()[key] = value else: raise ValueError( - "Flag %s cannot set its value through this function." % (key)) + "Flag %s cannot set its value through this function." % (key) + ) def get_flags(flags): @@ -7308,22 +7632,24 @@ def get_flags(flags): flags_value = {} if isinstance(flags, (list, tuple)): for key in flags: - if (_global_flags().is_public(key)): + if _global_flags().is_public(key): value = _global_flags()[key] temp = {key: value} flags_value.update(temp) else: raise ValueError( - 'Flag %s cannot get its value through this function.' % - (key)) + 'Flag %s cannot get its value through this function.' + % (key) + ) elif isinstance(flags, str): - if (_global_flags().is_public(flags)): + if _global_flags().is_public(flags): value = _global_flags()[flags] temp = {flags: value} flags_value.update(temp) else: raise ValueError( - 'Flag %s cannot get its value through this function.' % (flags)) + 'Flag %s cannot get its value through this function.' % (flags) + ) else: raise TypeError('Flags in get_flags should be a list, tuple or string.') return flags_value @@ -7333,20 +7659,32 @@ def _get_paddle_place(place): "convert the string to paddle Place" if place is None: return place - if isinstance(place, (core.Place, core.XPUPlace, core.CPUPlace, - core.CUDAPinnedPlace, core.CUDAPlace, core.NPUPlace, - core.IPUPlace, core.MLUPlace, core.CustomPlace)): + if isinstance( + place, + ( + core.Place, + core.XPUPlace, + core.CPUPlace, + core.CUDAPinnedPlace, + core.CUDAPlace, + core.NPUPlace, + core.IPUPlace, + core.MLUPlace, + core.CustomPlace, + ), + ): return place if not isinstance(place, str): raise ValueError( - "place only support string which is 'Place' and so on.") + "place only support string which is 'Place' and so on." + ) place = place.lower() - if (place == "cpu"): + if place == "cpu": return core.CPUPlace() - if (place == "device"): + if place == "device": return core.Place() # GPU @@ -7354,8 +7692,9 @@ def _get_paddle_place(place): if place == "gpu_pinned" or place == "gpu" or avaliable_gpu_place: if not core.is_compiled_with_cuda(): raise ValueError( - "The device should not be {}, since PaddlePaddle is " \ - "not compiled with CUDA".format(avaliable_gpu_place)) + "The device should not be {}, since PaddlePaddle is " + "not compiled with CUDA".format(avaliable_gpu_place) + ) if place == "gpu_pinned": return core.CUDAPinnedPlace() elif place == "gpu": @@ -7371,8 +7710,9 @@ def _get_paddle_place(place): if avaliable_xpu_place: if not core.is_compiled_with_xpu(): raise ValueError( - "The device should not be {}, since PaddlePaddle is " \ - "not compiled with XPU".format(avaliable_xpu_place)) + "The device should not be {}, since PaddlePaddle is " + "not compiled with XPU".format(avaliable_xpu_place) + ) place_info_list = place.split(':', 1) device_id = place_info_list[1] device_id = int(device_id) @@ -7383,8 +7723,9 @@ def _get_paddle_place(place): if avaliable_npu_place: if not core.is_compiled_with_npu(): raise ValueError( - "The device should not be {}, since PaddlePaddle is " \ - "not compiled with NPU".format(avaliable_npu_place)) + "The device should not be {}, since PaddlePaddle is " + "not compiled with NPU".format(avaliable_npu_place) + ) place_info_list = place.split(':', 1) device_id = place_info_list[1] device_id = int(device_id) @@ -7395,8 +7736,9 @@ def _get_paddle_place(place): if avaliable_ipu_place: if not core.is_compiled_with_ipu(): raise ValueError( - "The device should not be {}, since PaddlePaddle is " \ - "not compiled with IPU".format(avaliable_ipu_place)) + "The device should not be {}, since PaddlePaddle is " + "not compiled with IPU".format(avaliable_ipu_place) + ) place_info_list = place.split(':', 1) device_id = place_info_list[1] device_id = int(device_id) @@ -7407,16 +7749,19 @@ def _get_paddle_place(place): if avaliable_mlu_place: if not core.is_compiled_with_mlu(): raise ValueError( - "The device should not be {}, since PaddlePaddle is " \ - "not compiled with MLU".format(avaliable_mlu_place)) + "The device should not be {}, since PaddlePaddle is " + "not compiled with MLU".format(avaliable_mlu_place) + ) place_info_list = place.split(':', 1) device_id = place_info_list[1] device_id = int(device_id) return core.MLUPlace(device_id) raise ValueError( - "Paddle supports CPUPlace, CUDAPlace,CUDAPinnedPlace, XPUPlace, IPUPlace, MLUPlace and NPUPlace, but received {}." - .format(place)) + "Paddle supports CPUPlace, CUDAPlace,CUDAPinnedPlace, XPUPlace, IPUPlace, MLUPlace and NPUPlace, but received {}.".format( + place + ) + ) def _get_paddle_place_list(places): diff --git a/python/paddle/fluid/generator.py b/python/paddle/fluid/generator.py index 5bbe7a0e12c3f..4982de95f498c 100644 --- a/python/paddle/fluid/generator.py +++ b/python/paddle/fluid/generator.py @@ -43,4 +43,5 @@ def __init__(self, place=None): else: raise ValueError( "Generator class with %s does is not supported yet, currently only support generator with CPUPlace " - % place) + % place + ) diff --git a/python/paddle/fluid/graphviz.py b/python/paddle/fluid/graphviz.py index 9579bff409efe..b7ef1c7531fc0 100644 --- a/python/paddle/fluid/graphviz.py +++ b/python/paddle/fluid/graphviz.py @@ -26,7 +26,6 @@ def crepr(v): class Rank(object): - def __init__(self, kind, name, priority): ''' kind: str @@ -42,8 +41,12 @@ def __str__(self): if not self.nodes: return '' - return '{' + 'rank={};'.format(self.kind) + \ - ','.join([node.name for node in self.nodes]) + '}' + return ( + '{' + + 'rank={};'.format(self.kind) + + ','.join([node.name for node in self.nodes]) + + '}' + ) class Graph(object): @@ -84,28 +87,36 @@ def edge(self, source, target, **attrs): def compile(self, dot_path): file = open(dot_path, 'w') file.write(self.__str__()) - image_path = os.path.join(os.path.dirname(dot_path), - dot_path[:-3] + "pdf") + image_path = os.path.join( + os.path.dirname(dot_path), dot_path[:-3] + "pdf" + ) cmd = ["dot", "-Tpdf", dot_path, "-o", image_path] - subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) logging.warning("write block debug graph to {}".format(image_path)) return image_path def show(self, dot_path): image = self.compile(dot_path) cmd = ["open", image] - subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) def _rank_repr(self): - ranks = sorted(self.rank_groups.items(), - key=functools.cmp_to_key( - lambda a, b: a[1].priority > b[1].priority)) + ranks = sorted( + self.rank_groups.items(), + key=functools.cmp_to_key( + lambda a, b: a[1].priority > b[1].priority + ), + ) repr = [] for x in ranks: repr.append(str(x[1])) @@ -118,8 +129,9 @@ def __str__(self): ] for attr in self.attrs: - reprs.append("{key}={value};".format(key=attr, - value=crepr(self.attrs[attr]))) + reprs.append( + "{key}={value};".format(key=attr, value=crepr(self.attrs[attr])) + ) reprs.append(self._rank_repr()) @@ -147,14 +159,18 @@ def __str__(self): reprs = '{name} [label={label} {extra} ];'.format( name=self.name, label=self.label, - extra=',' + ','.join("%s=%s" % (key, crepr(value)) - for key, value in self.attrs.items()) - if self.attrs else "") + extra=',' + + ','.join( + "%s=%s" % (key, crepr(value)) + for key, value in self.attrs.items() + ) + if self.attrs + else "", + ) return reprs class Edge(object): - def __init__(self, source, target, **attrs): ''' Link source to target. @@ -171,9 +187,15 @@ def __str__(self): repr = "{source} -> {target} {extra}".format( source=self.source.name, target=self.target.name, - extra="" if not self.attrs else "[" + - ','.join("{}={}".format(attr[0], crepr(attr[1])) - for attr in self.attrs.items()) + "]") + extra="" + if not self.attrs + else "[" + + ','.join( + "{}={}".format(attr[0], crepr(attr[1])) + for attr in self.attrs.items() + ) + + "]", + ) return repr @@ -202,31 +224,34 @@ def __call__(self, path='temp.dot', show=False): self.graph.show(path) def add_param(self, name, data_type, highlight=False): - label = '\n'.join([ - '<', - ' ', - ' ', - ' ', - ' ', - ' ' - ' ', - '
', - ' ', - name, - ' ', - '
', - str(data_type), - '
>', - ]) - return self.graph.node(label, - prefix="param", - description=name, - shape="none", - style="rounded,filled,bold", - width="1.3", - color="#148b97" if not highlight else "orange", - fontcolor="#ffffff", - fontname="Arial") + label = '\n'.join( + [ + '<', + ' ', + ' ', + ' ', + ' ', + ' ' ' ', + '
', + ' ', + name, + ' ', + '
', + str(data_type), + '
>', + ] + ) + return self.graph.node( + label, + prefix="param", + description=name, + shape="none", + style="rounded,filled,bold", + width="1.3", + color="#148b97" if not highlight else "orange", + fontcolor="#ffffff", + fontname="Arial", + ) def add_op(self, opType, **kwargs): highlight = False @@ -247,21 +272,25 @@ def add_op(self, opType, **kwargs): ) def add_arg(self, name, highlight=False): - return self.graph.node(crepr(name), - prefix="arg", - description=name, - shape="box", - style="rounded,filled,bold", - fontname="Arial", - fontcolor="#999999", - color="#dddddd" if not highlight else "orange") + return self.graph.node( + crepr(name), + prefix="arg", + description=name, + shape="box", + style="rounded,filled,bold", + fontname="Arial", + fontcolor="#999999", + color="#dddddd" if not highlight else "orange", + ) def add_edge(self, source, target, **kwargs): highlight = False if 'highlight' in kwargs: highlight = kwargs['highlight'] del kwargs['highlight'] - return self.graph.edge(source, - target, - color="#00000" if not highlight else "orange", - **kwargs) + return self.graph.edge( + source, + target, + color="#00000" if not highlight else "orange", + **kwargs + ) diff --git a/python/paddle/fluid/incubate/checkpoint/auto_checkpoint.py b/python/paddle/fluid/incubate/checkpoint/auto_checkpoint.py index e0d8062877ca6..134723cdbc0c7 100644 --- a/python/paddle/fluid/incubate/checkpoint/auto_checkpoint.py +++ b/python/paddle/fluid/incubate/checkpoint/auto_checkpoint.py @@ -55,7 +55,8 @@ def _get_logger(log_level, name="auto_checkpoint"): log_handler = logging.StreamHandler() log_format = logging.Formatter( - '%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s') + '%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s' + ) log_handler.setFormatter(log_format) logger.addHandler(log_handler) @@ -63,12 +64,12 @@ def _get_logger(log_level, name="auto_checkpoint"): def _thread_checker(): - assert current_thread().name == "MainThread", \ - "auto checkpoint must run under main thread" + assert ( + current_thread().name == "MainThread" + ), "auto checkpoint must run under main thread" class AutoCheckpointChecker(object): - def __init__(self): self._run_env = None self._platform = None @@ -91,35 +92,43 @@ def __init__(self): self._hdfs_name = os.environ["PADDLE_EDL_HDFS_NAME"] self._hdfs_ugi = os.environ["PADDLE_EDL_HDFS_UGI"] self._hdfs_checkpoint_path = os.environ[ - "PADDLE_EDL_HDFS_CHECKPOINT_PATH"] + "PADDLE_EDL_HDFS_CHECKPOINT_PATH" + ] self._trainer_id = int(os.environ["PADDLE_TRAINER_ID"]) self._ce_test = int(os.getenv("PADDLE_EDL_ONLY_FOR_CE_TEST", "0")) self._fs_cache = os.getenv("PADDLE_EDL_FS_CACHE", ".cache") self._save_checkpoint_inter = int( - os.getenv("PADDLE_EDL_SAVE_CHECKPOINT_INTER", "900")) # s + os.getenv("PADDLE_EDL_SAVE_CHECKPOINT_INTER", "900") + ) # s if not self._ce_test: - assert len(self._hdfs_home) > 3 and \ - len(self._hdfs_name) > 6 and \ - len(self._hdfs_ugi) > 3 and \ - len(self._hdfs_checkpoint_path) > 0, "hdfs environ must set" + assert ( + len(self._hdfs_home) > 3 + and len(self._hdfs_name) > 6 + and len(self._hdfs_ugi) > 3 + and len(self._hdfs_checkpoint_path) > 0 + ), "hdfs environ must set" else: - assert len(self._hdfs_home) > 3 and \ - len(self._hdfs_checkpoint_path) > 0, "hdfs environ must set" + assert ( + len(self._hdfs_home) > 3 + and len(self._hdfs_checkpoint_path) > 0 + ), "hdfs environ must set" except Exception as e: logger.fatal("exception:{}".format(e)) sys.exit(1) def get_range_checkpoint_path(self, name): - return "{}/{}/range/{}".format(self.hdfs_checkpoint_path, self.job_id, - name) + return "{}/{}/range/{}".format( + self.hdfs_checkpoint_path, self.job_id, name + ) def get_exe_checkpoint_path(self, name): - return "{}/{}/exe/{}".format(self.hdfs_checkpoint_path, self.job_id, - name) + return "{}/{}/exe/{}".format( + self.hdfs_checkpoint_path, self.job_id, name + ) def get_job_path(self): return "{}/{}".format(self.hdfs_checkpoint_path, self.job_id) @@ -132,22 +141,30 @@ def valid(self): if _non_static_mode(): return False - return self._run_env is not None and \ - self._platform is not None and \ - self._job_id is not None and \ - self._hdfs_home is not None and \ - self._hdfs_name is not None and \ - self._hdfs_ugi is not None and \ - self._hdfs_checkpoint_path is not None and \ - self._trainer_id is not None + return ( + self._run_env is not None + and self._platform is not None + and self._job_id is not None + and self._hdfs_home is not None + and self._hdfs_name is not None + and self._hdfs_ugi is not None + and self._hdfs_checkpoint_path is not None + and self._trainer_id is not None + ) def __str__(self): return "run_env:{} platform:{} job_id:{} \ hdfs_home:{} hdfs_name:{} hdfs_ugi:{} \ hdfs_checkpoint_path:{} trainer_id:{} ce_test".format( - self._run_env, self._platform, self._hdfs_home, self._hdfs_name, - self._hdfs_ugi, self._hdfs_checkpoint_path, self._trainer_id, - self._ce_test) + self._run_env, + self._platform, + self._hdfs_home, + self._hdfs_name, + self._hdfs_ugi, + self._hdfs_checkpoint_path, + self._trainer_id, + self._ce_test, + ) @property def trainer_id(self): @@ -191,7 +208,6 @@ def generate_range_name(): class ExeTrainStatus(SerializableBase): - def __init__(self): self._epoch_no = -1 # start epoch_no self._hash_key = None @@ -207,13 +223,15 @@ def __init__(self): self._file_name = "exe_train_status" def __eq__(self, t): - return self._epoch_no == t._epoch_no and \ - self._hash_key == t._hash_key and \ - self._key == t._key and \ - self._checkpoint_path == t._checkpoint_path and \ - self._checkpoint_no == t._checkpoint_no and \ - self._exe_name == t._exe_name and \ - self._program_name == t._program_name + return ( + self._epoch_no == t._epoch_no + and self._hash_key == t._hash_key + and self._key == t._key + and self._checkpoint_path == t._checkpoint_path + and self._checkpoint_no == t._checkpoint_no + and self._exe_name == t._exe_name + and self._program_name == t._program_name + ) def __ne__(self, t): return not self == t @@ -256,7 +274,7 @@ def _to_dict(self): "restored_from": self._restored_from, "exe_name": self._exe_name, "program_name": self._program_name, - "checkpoint_no": self._checkpoint_no + "checkpoint_no": self._checkpoint_no, } def __str__(self): @@ -264,12 +282,9 @@ def __str__(self): class TrainEpochRange(SerializableBase): - - def __init__(self, - max_epoch_num, - name, - checkpoint_inter=None, - restored=True): + def __init__( + self, max_epoch_num, name, checkpoint_inter=None, restored=True + ): self._max_epoch_num = max_epoch_num self._epoch_no = -1 # current epoch_no self._name = name @@ -282,8 +297,9 @@ def __init__(self, self._save_checkpoint_inter = checkpoint_inter else: self._save_checkpoint_inter = self._checker.save_checkpoint_inter - assert self._save_checkpoint_inter >= 0, "checkpointer:{} must >=0".format( - self._save_checkpoint_inter) + assert ( + self._save_checkpoint_inter >= 0 + ), "checkpointer:{} must >=0".format(self._save_checkpoint_inter) self._last_checkpoint_time = time.time() self._load_cp_nos = None @@ -301,13 +317,14 @@ def __init__(self, config = { "fs.default.name": self._checker.hdfs_name, - "hadoop.job.ugi": self._checker.hdfs_ugi + "hadoop.job.ugi": self._checker.hdfs_ugi, } if self._checker.ce_test: config = None from paddle.distributed.fleet.utils.fs import HDFSClient + self._hdfs = HDFSClient(self._checker.hdfs_home, config) self._cper = CheckpointSaver(self._hdfs) @@ -321,10 +338,13 @@ def _look_for_valid(self, cp_nos): epoch_no = -1 for i in cp_nos[::-1]: t = TrainEpochRange(self._max_epoch_num, self.name, restored=False) - self._cper.load_checkpoint(self._checkpoint_path, [t], - self._checker.trainer_id, - checkpoint_no=i, - local_cache_path=self._checker._fs_cache) + self._cper.load_checkpoint( + self._checkpoint_path, + [t], + self._checker.trainer_id, + checkpoint_no=i, + local_cache_path=self._checker._fs_cache, + ) cps.append(t) logger.debug("look for valid:{} t:{}".format(i, t._serialize())) if epoch_no < 0: @@ -344,14 +364,18 @@ def _get_last_valid_checkpoint(self): if g_acp_type == CONST_ACP_TYPE: # get the last one - self._cper.load_checkpoint(self._checkpoint_path, [self], - self._checker.trainer_id, - local_cache_path=self._checker._fs_cache) + self._cper.load_checkpoint( + self._checkpoint_path, + [self], + self._checker.trainer_id, + local_cache_path=self._checker._fs_cache, + ) self._restored_from = CONST_CHECKPOINT self._checkpoint_epoch_no = self._epoch_no - logger.info("load tain_epoch_range checkpoint:{}".format( - self._serialize())) + logger.info( + "load tain_epoch_range checkpoint:{}".format(self._serialize()) + ) elif g_acp_type == CONST_DACP_TYPE: t, i = self._look_for_valid(self._load_cp_nos) @@ -359,15 +383,19 @@ def _get_last_valid_checkpoint(self): self._restored_from = CONST_MEMORYINIT return - self._cper.load_checkpoint(self._checkpoint_path, [self], - self._checker.trainer_id, - checkpoint_no=i, - local_cache_path=self._checker._fs_cache) + self._cper.load_checkpoint( + self._checkpoint_path, + [self], + self._checker.trainer_id, + checkpoint_no=i, + local_cache_path=self._checker._fs_cache, + ) self._restored_from = CONST_CHECKPOINT self._checkpoint_epoch_no = self._epoch_no - logger.info("load tain_epoch_range checkpoint:{}".format( - self._serialize())) + logger.info( + "load tain_epoch_range checkpoint:{}".format(self._serialize()) + ) else: assert False, "not supported acp_type:{}".format(g_acp_type) @@ -378,7 +406,7 @@ def _to_dict(self): "name": self._name, "checkpoint_path": self._checkpoint_path, "restored_from": self._restored_from, - "checkpoint_epoch_no": self._checkpoint_epoch_no + "checkpoint_epoch_no": self._checkpoint_epoch_no, } return d @@ -438,12 +466,16 @@ def next(self): self._max_epoch_num = sys.maxint assert self._epoch_no >= -1, "self._epoch_no:{} must >=-1".format( - self._epoch_no) + self._epoch_no + ) self._last_checkpoint_time = time.time() start = self._epoch_no + 1 - logger.info("started epoch_no:{} max_epoch_num:{}".format( - start, self._max_epoch_num)) + logger.info( + "started epoch_no:{} max_epoch_num:{}".format( + start, self._max_epoch_num + ) + ) for i in range(start, self._max_epoch_num): self._epoch_no = i @@ -458,11 +490,16 @@ def save_checkpoint(self): # not save last one because exe and program can't be restored. if self._checker.trainer_id == 0: - if time.time() - self._last_checkpoint_time >= \ - self._save_checkpoint_inter: + if ( + time.time() - self._last_checkpoint_time + >= self._save_checkpoint_inter + ): if g_acp_type == CONST_ACP_TYPE: # not save the last one - if self._max_epoch_num > 0 and self._epoch_no != self._max_epoch_num - 1: + if ( + self._max_epoch_num > 0 + and self._epoch_no != self._max_epoch_num - 1 + ): self._save_checkpoint() elif g_acp_type == CONST_DACP_TYPE: self._save_checkpoint() @@ -484,9 +521,11 @@ def _save_checkpoint(self): p = self._checker.get_exe_checkpoint_path(t._hash_key) t._epoch_no = self.get() path, checkpoint_no = self._cper.save_checkpoint( - p, [m], + p, + [m], self._checker.trainer_id, - local_cache_path=self._checker._fs_cache) + local_cache_path=self._checker._fs_cache, + ) # index info t._checkpoint_path = path t._checkpoint_no = checkpoint_no @@ -496,10 +535,14 @@ def _save_checkpoint(self): logger.debug("save executor checkpoint:{}".format(t._serialize())) if len(self._exe_status) > 0: - self._cper.save_checkpoint(self._checkpoint_path, [self], - local_cache_path=self._checker._fs_cache) - logger.info("save train_epoch_range checkpoint:{}".format( - self._serialize())) + self._cper.save_checkpoint( + self._checkpoint_path, + [self], + local_cache_path=self._checker._fs_cache, + ) + logger.info( + "save train_epoch_range checkpoint:{}".format(self._serialize()) + ) self._generate_flag() @@ -538,13 +581,13 @@ def _check_program_oprole(program): def _can_auto_checkpoint(prog): - if not isinstance(prog, compiler.CompiledProgram) and \ - not isinstance(prog, Program): + if not isinstance(prog, compiler.CompiledProgram) and not isinstance( + prog, Program + ): return False if isinstance(prog, compiler.CompiledProgram): - if prog._program is None or \ - prog._program._is_distributed: + if prog._program is None or prog._program._is_distributed: return False else: if prog._is_distributed: @@ -564,8 +607,11 @@ def _can_auto_checkpoint(prog): g_program_attr[program._auto_checkpoint_name] = ret if not ret: - logger.debug("program {} need't to auto checkpoint".format( - program._auto_checkpoint_name)) + logger.debug( + "program {} need't to auto checkpoint".format( + program._auto_checkpoint_name + ) + ) return False return g_checker.valid() and g_train_epoch_range is not None @@ -597,7 +643,8 @@ def train_epoch_range(max_epoch_num, save_checkpoint_inter=None): global g_acp_type if not _get_checker().valid(): logger.warning( - "auto checkpoint will take effect automaticly on PaddleCloud") + "auto checkpoint will take effect automaticly on PaddleCloud" + ) for i in _normal_yield(max_epoch_num): yield i @@ -617,7 +664,8 @@ def train_epoch_range(max_epoch_num, save_checkpoint_inter=None): g_train_epoch_range = TrainEpochRange( max_epoch_num, g_checker.generate_range_name(), - checkpoint_inter=save_checkpoint_inter) + checkpoint_inter=save_checkpoint_inter, + ) for i in g_train_epoch_range.next(): yield i @@ -643,12 +691,16 @@ def _auto_checkpoint(exe, prog): assert program._auto_checkpoint_name != None exe_status = g_train_epoch_range._exe_status - key = _get_running_key(exe._auto_checkpoint_name, - program._auto_checkpoint_name) + key = _get_running_key( + exe._auto_checkpoint_name, program._auto_checkpoint_name + ) if g_train_epoch_range.restored_from == CONST_CHECKPOINT: - assert key in exe_status, "when restored key:{} must be in train_epoch_range:{}".format( - key, g_train_epoch_range) + assert ( + key in exe_status + ), "when restored key:{} must be in train_epoch_range:{}".format( + key, g_train_epoch_range + ) t = None if key in exe_status: @@ -656,10 +708,13 @@ def _auto_checkpoint(exe, prog): if t._restored_from is None: a = CheckpointSaver(g_train_epoch_range._hdfs) m = PaddleModel(exe, program) - a.load_checkpoint(g_checker.get_exe_checkpoint_path(key), [m], - trainer_id=g_checker.trainer_id, - checkpoint_no=t._checkpoint_no, - local_cache_path=g_checker._fs_cache) + a.load_checkpoint( + g_checker.get_exe_checkpoint_path(key), + [m], + trainer_id=g_checker.trainer_id, + checkpoint_no=t._checkpoint_no, + local_cache_path=g_checker._fs_cache, + ) t._restored_from = CONST_CHECKPOINT logger.info("load executor checkpoint {}".format(t)) t._exe = exe diff --git a/python/paddle/fluid/incubate/checkpoint/checkpoint_saver.py b/python/paddle/fluid/incubate/checkpoint/checkpoint_saver.py index c8aeb50f157c0..21e305afc16bc 100644 --- a/python/paddle/fluid/incubate/checkpoint/checkpoint_saver.py +++ b/python/paddle/fluid/incubate/checkpoint/checkpoint_saver.py @@ -16,7 +16,6 @@ class SerializableBase(object): - def serialize(self, path): raise NotImplementedError @@ -25,7 +24,6 @@ def deserialize(self, path): class PaddleModel(SerializableBase): - def __init__(self, exe, program): self._exe = exe self._origin_program = program @@ -37,30 +35,33 @@ def __init__(self, exe, program): def serialize(self, path): from ...io import save_persistables - save_persistables(executor=self._exe, - dirname=path, - main_program=self._program, - filename=self._file_name) + + save_persistables( + executor=self._exe, + dirname=path, + main_program=self._program, + filename=self._file_name, + ) def deserialize(self, path): from ...io import load_persistables - load_persistables(executor=self._exe, - dirname=path, - main_program=self._program, - filename=self._file_name) + load_persistables( + executor=self._exe, + dirname=path, + main_program=self._program, + filename=self._file_name, + ) -class CheckpointSaver(object): +class CheckpointSaver(object): def __init__(self, fs): self._fs = fs self._checkpoint_prefix = "__paddle_checkpoint__" - def save_checkpoint(self, - path, - slists, - trainer_id=None, - local_cache_path=".cache"): + def save_checkpoint( + self, path, slists, trainer_id=None, local_cache_path=".cache" + ): """ Serialize objects in slists to path Return really saved path and checkpoint_no @@ -69,7 +70,8 @@ def save_checkpoint(self, self._fs.mkdirs(path) else: assert self._fs.is_dir(path), "path:{} must be a directory".format( - path) + path + ) max_no = self._get_last_checkpoint_no(path) if max_no < 0: @@ -81,13 +83,14 @@ def save_checkpoint(self, saved_path = tmp_path from paddle.distributed.fleet.utils.fs import LocalFS + local_fs = LocalFS() cache_path = None if self._fs.need_upload_download(): - cache_path = "{}/{}.{}.saved_cache".format(local_cache_path, - self._checkpoint_prefix, - max_no) + cache_path = "{}/{}.{}.saved_cache".format( + local_cache_path, self._checkpoint_prefix, max_no + ) if trainer_id is not None: cache_path = "{}.{}".format(cache_path, trainer_id) @@ -95,8 +98,9 @@ def save_checkpoint(self, if not local_fs.is_exist(cache_path): local_fs.mkdirs(cache_path) else: - assert local_fs.is_dir(cache_path), \ - "cache path:{} must be a directory".format(cache_path) + assert local_fs.is_dir( + cache_path + ), "cache path:{} must be a directory".format(cache_path) saved_path = cache_path @@ -111,13 +115,15 @@ def save_checkpoint(self, return real_path, max_no - def load_checkpoint(self, - path, - slists, - trainer_id, - local_cache_path=".cache", - checkpoint_no=None, - ignore_empty=True): + def load_checkpoint( + self, + path, + slists, + trainer_id, + local_cache_path=".cache", + checkpoint_no=None, + ignore_empty=True, + ): """ Deserialize objects in slists from path Return really load path @@ -137,11 +143,12 @@ def load_checkpoint(self, assert checkpoint_no >= 0 from paddle.distributed.fleet.utils.fs import LocalFS + local_fs = LocalFS() if self._fs.need_upload_download(): - cache_path = "{}/{}.{}.load_cache".format(local_cache_path, - self._checkpoint_prefix, - checkpoint_no) + cache_path = "{}/{}.{}.load_cache".format( + local_cache_path, self._checkpoint_prefix, checkpoint_no + ) if trainer_id is not None: cache_path = "{}.{}".format(cache_path, trainer_id) @@ -151,8 +158,9 @@ def load_checkpoint(self, if local_fs.is_exist(cache_path): local_fs.delete(cache_path) - real_path = "{}/{}.{}".format(path, self._checkpoint_prefix, - checkpoint_no) + real_path = "{}/{}.{}".format( + path, self._checkpoint_prefix, checkpoint_no + ) load_path = real_path if self._fs.need_upload_download(): self._fs.download(real_path, cache_path) @@ -217,8 +225,9 @@ def clean_redundant_checkpoints(self, root_path, reserved=[]): try: n = int(g[1]) if n not in s: - path = "{}/{}.{}".format(root_path, self._checkpoint_prefix, - n) + path = "{}/{}.{}".format( + root_path, self._checkpoint_prefix, n + ) self._fs.delete(path) except Exception as e: print(e) diff --git a/python/paddle/fluid/incubate/data_generator/__init__.py b/python/paddle/fluid/incubate/data_generator/__init__.py index 60bfa9eb1108a..10e4fba92dd5a 100644 --- a/python/paddle/fluid/incubate/data_generator/__init__.py +++ b/python/paddle/fluid/incubate/data_generator/__init__.py @@ -31,8 +31,9 @@ def __init__(self): def _set_line_limit(self, line_limit): if not isinstance(line_limit, int): - raise ValueError("line_limit%s must be in int type" % - type(line_limit)) + raise ValueError( + "line_limit%s must be in int type" % type(line_limit) + ) if line_limit < 1: raise ValueError("line_limit can not less than 1") self._line_limit = line_limit @@ -140,7 +141,8 @@ def _gen_str(self, line): Return a string data that can be read directly by the datafeed. ''' raise NotImplementedError( - "pls use MultiSlotDataGenerator or PairWiseDataGenerator") + "pls use MultiSlotDataGenerator or PairWiseDataGenerator" + ) def generate_sample(self, line): ''' @@ -172,8 +174,9 @@ def local_iter(): return local_iter ''' raise NotImplementedError( - "Please rewrite this function to return a list or tuple: " + - "[(name, [feasign, ...]), ...] or ((name, [feasign, ...]), ...)") + "Please rewrite this function to return a list or tuple: " + + "[(name, [feasign, ...]), ...] or ((name, [feasign, ...]), ...)" + ) def generate_batch(self, samples): ''' @@ -214,7 +217,6 @@ def local_iter(): # add more generalized DataGenerator that can adapt user-defined slot # for example, [(name, float_list), (name, str_list), (name, int_list)] class MultiSlotStringDataGenerator(DataGenerator): - def _gen_str(self, line): ''' Further processing the output of the process() function rewritten by @@ -238,7 +240,8 @@ def _gen_str(self, line): if not isinstance(line, list) and not isinstance(line, tuple): raise ValueError( "the output of process() must be in list or tuple type" - "Examples: [('words', ['1926', '08', '17']), ('label', ['1'])]") + "Examples: [('words', ['1926', '08', '17']), ('label', ['1'])]" + ) output = "" for index, item in enumerate(line): name, elements = item @@ -252,7 +255,6 @@ def _gen_str(self, line): class MultiSlotDataGenerator(DataGenerator): - def _gen_str(self, line): ''' Further processing the output of the process() function rewritten by @@ -281,7 +283,8 @@ def _gen_str(self, line): if not isinstance(line, list) and not isinstance(line, tuple): raise ValueError( "the output of process() must be in list or tuple type" - "Example: [('words', [1926, 08, 17]), ('label', [1])]") + "Example: [('words', [1926, 08, 17]), ('label', [1])]" + ) output = "" if self._proto_info is None: @@ -291,8 +294,9 @@ def _gen_str(self, line): if not isinstance(name, str): raise ValueError("name%s must be in str type" % type(name)) if not isinstance(elements, list): - raise ValueError("elements%s must be in list type" % - type(elements)) + raise ValueError( + "elements%s must be in list type" % type(elements) + ) if not elements: raise ValueError( "the elements of each field can not be empty, you need padding it in process()." @@ -305,10 +309,12 @@ def _gen_str(self, line): if isinstance(elem, float): self._proto_info[-1] = (name, "float") elif not isinstance(elem, int) and not isinstance( - elem, long): + elem, long + ): raise ValueError( - "the type of element%s must be in int or float" % - type(elem)) + "the type of element%s must be in int or float" + % type(elem) + ) output += " " + str(elem) else: if len(line) != len(self._proto_info): @@ -320,8 +326,9 @@ def _gen_str(self, line): if not isinstance(name, str): raise ValueError("name%s must be in str type" % type(name)) if not isinstance(elements, list): - raise ValueError("elements%s must be in list type" % - type(elements)) + raise ValueError( + "elements%s must be in list type" % type(elements) + ) if not elements: raise ValueError( "the elements of each field can not be empty, you need padding it in process()." @@ -329,7 +336,8 @@ def _gen_str(self, line): if name != self._proto_info[index][0]: raise ValueError( "the field name of two given line are not match: require<%s>, get<%s>." - % (self._proto_info[index][0], name)) + % (self._proto_info[index][0], name) + ) if output: output += " " output += str(len(elements)) @@ -338,9 +346,11 @@ def _gen_str(self, line): if isinstance(elem, float): self._proto_info[index] = (name, "float") elif not isinstance(elem, int) and not isinstance( - elem, long): + elem, long + ): raise ValueError( "the type of element%s must be in int or float" - % type(elem)) + % type(elem) + ) output += " " + str(elem) return output + "\n" diff --git a/python/paddle/fluid/incubate/fleet/base/fleet_base.py b/python/paddle/fluid/incubate/fleet/base/fleet_base.py index f49dbafb5547a..ea03e9305605a 100644 --- a/python/paddle/fluid/incubate/fleet/base/fleet_base.py +++ b/python/paddle/fluid/incubate/fleet/base/fleet_base.py @@ -21,7 +21,9 @@ from paddle.fluid.incubate.fleet.base.mode import Mode from paddle.distributed.fleet.base.role_maker import RoleMakerBase -from paddle.fluid.contrib.mixed_precision.decorator import OptimizerWithMixedPrecision +from paddle.fluid.contrib.mixed_precision.decorator import ( + OptimizerWithMixedPrecision, +) from . import mode __all__ = ['Fleet', 'DistributedOptimizer'] @@ -38,6 +40,7 @@ class Fleet(object): Returns: None """ + __metaclass__ = abc.ABCMeta def __init__(self, mode): @@ -179,7 +182,7 @@ def split_files(self, files): trainer_files = [[]] * trainers begin = 0 for i in range(trainers): - trainer_files[i] = files[begin:begin + blocks[i]] + trainer_files[i] = files[begin : begin + blocks[i]] begin += blocks[i] return trainer_files[trainer_id] @@ -199,10 +202,14 @@ def init(self, role_maker=None): self._executor = Executor(fluid.CPUPlace()) if role_maker and not isinstance(role_maker, RoleMakerBase): - from paddle.fluid.incubate.fleet.base.role_maker import RoleMakerBase as RoleMakerBaseIncubate + from paddle.fluid.incubate.fleet.base.role_maker import ( + RoleMakerBase as RoleMakerBaseIncubate, + ) + if role_maker and not isinstance(role_maker, RoleMakerBaseIncubate): raise TypeError( - "role_maker must be an instance of RoleMakerBase") + "role_maker must be an instance of RoleMakerBase" + ) self._role_maker = role_maker self._role_maker.generate_role() @@ -245,13 +252,15 @@ def distributed_optimizer(self, optimizer, strategy=None): pass @abc.abstractmethod - def save_inference_model(self, - executor, - dirname, - feeded_var_names, - target_vars, - main_program=None, - export_for_deployment=True): + def save_inference_model( + self, + executor, + dirname, + feeded_var_names, + target_vars, + main_program=None, + export_for_deployment=True, + ): pass @abc.abstractmethod @@ -277,24 +286,29 @@ class DistributedOptimizer(object): None """ + __metaclass__ = abc.ABCMeta def __init__(self, optimizer, strategy=None): - if not isinstance(optimizer, SGD.__bases__) \ - and not isinstance(optimizer, OptimizerWithMixedPrecision) \ - and not isinstance(optimizer, SGD_v2.__base__): + if ( + not isinstance(optimizer, SGD.__bases__) + and not isinstance(optimizer, OptimizerWithMixedPrecision) + and not isinstance(optimizer, SGD_v2.__base__) + ): raise TypeError("optimizer must be an instance of Optimizer") self._optimizer = optimizer self._strategy = strategy @abc.abstractmethod - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): """ First part of `minimize`, do auto-diff to append backward ops for the current program. @@ -341,12 +355,14 @@ def apply_gradients(self, params_grads): pass @abc.abstractmethod - def minimize(self, - losses, - scopes=None, - startup_programs=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, + losses, + scopes=None, + startup_programs=None, + parameter_list=None, + no_grad_set=None, + ): """ Add operations to minimize `loss` by updating `parameter_list`. diff --git a/python/paddle/fluid/incubate/fleet/base/mode.py b/python/paddle/fluid/incubate/fleet/base/mode.py index a940630d4c6ca..80bd7e6b4b51d 100644 --- a/python/paddle/fluid/incubate/fleet/base/mode.py +++ b/python/paddle/fluid/incubate/fleet/base/mode.py @@ -19,6 +19,7 @@ class Mode: """ There are various mode for fleet, each of them is designed for different model. """ + TRANSPILER = 1 PSLIB = 2 COLLECTIVE = 3 diff --git a/python/paddle/fluid/incubate/fleet/base/role_maker.py b/python/paddle/fluid/incubate/fleet/base/role_maker.py index 31a20a409b6d8..3349dcf275b3f 100644 --- a/python/paddle/fluid/incubate/fleet/base/role_maker.py +++ b/python/paddle/fluid/incubate/fleet/base/role_maker.py @@ -19,8 +19,13 @@ import time __all__ = [ - 'Role', 'RoleMakerBase', 'MPISymetricRoleMaker', 'UserDefinedRoleMaker', - 'UserDefinedCollectiveRoleMaker', 'PaddleCloudRoleMaker', 'GeneralRoleMaker' + 'Role', + 'RoleMakerBase', + 'MPISymetricRoleMaker', + 'UserDefinedRoleMaker', + 'UserDefinedCollectiveRoleMaker', + 'PaddleCloudRoleMaker', + 'GeneralRoleMaker', ] @@ -145,8 +150,11 @@ def get_pserver_endpoints(self): def to_string(self): return "role: {}, current_id: {}, worker_endpoints: {}, server_endpoints: {}".format( - self._role, self._current_id, self._worker_endpoints, - self._server_endpoints) + self._role, + self._current_id, + self._worker_endpoints, + self._server_endpoints, + ) def all_gather(self, input): """ @@ -196,6 +204,7 @@ def __init__(self): """Init.""" super(MPIRoleMaker, self).__init__() from mpi4py import MPI + self.MPI = MPI self._comm = MPI.COMM_WORLD self._node_type_comm = None @@ -251,6 +260,7 @@ def _get_ips(self): def get_local_ip(self): """Return get local ip.""" import socket + self._ip = socket.gethostbyname(socket.gethostname()) return self._ip @@ -347,6 +357,7 @@ def get_pserver_endpoints(self): """ if self._pserver_rand_port <= 0: import random + random.seed(self._server_num()) # port will be randomly generated from 60001 to 63999 # random seed is server num so that all nodes will get @@ -495,7 +506,8 @@ def generate_role(self): # Environment variable PADDLE_PSERVERS_IP_PORT_LIST must be set # format: string(ip:port), eg. 127.0.0.1:6001 eplist = os.environ["PADDLE_PSERVERS_IP_PORT_LIST"].split( - ",") + "," + ) # note that, we usually assign the same port to different ips # if we run parameter server training in local mode # port should be different in environment variables @@ -505,7 +517,8 @@ def generate_role(self): if training_role not in ["TRAINER", "PSERVER"]: raise ValueError( - "TRAINING_ROLE must be PSERVER or TRAINER") + "TRAINING_ROLE must be PSERVER or TRAINER" + ) if training_role == "TRAINER": role = Role.WORKER @@ -518,7 +531,8 @@ def generate_role(self): current_id = eplist.index(curr_endpoint) else: raise ValueError( - "TRAINING_ROLE must be PSERVER or TRAINER") + "TRAINING_ROLE must be PSERVER or TRAINER" + ) except ValueError as ve: raise ValueError( "something wrong with PaddleCloud, please check environment" @@ -530,12 +544,15 @@ def generate_role(self): self._current_id = current_id else: self._current_id = int(os.getenv("PADDLE_TRAINER_ID", "0")) - self._training_role = os.getenv("PADDLE_TRAINING_ROLE", - "TRAINER") - assert (self._training_role == "TRAINER") + self._training_role = os.getenv( + "PADDLE_TRAINING_ROLE", "TRAINER" + ) + assert self._training_role == "TRAINER" self._worker_endpoints = os.getenv("PADDLE_TRAINER_ENDPOINTS") self._current_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT") - assert self._worker_endpoints is not None, "can't find PADDLE_TRAINER_ENDPOINTS" + assert ( + self._worker_endpoints is not None + ), "can't find PADDLE_TRAINER_ENDPOINTS" self._worker_endpoints = self._worker_endpoints.split(",") self._trainers_num = len(self._worker_endpoints) @@ -630,7 +647,8 @@ def generate_role(self): self._is_barrier_all = 1 if "PADDLE_IS_BARRIER_ALL_ROLE" in os.environ: self._is_barrier_all = int( - os.environ["PADDLE_IS_BARRIER_ALL_ROLE"]) + os.environ["PADDLE_IS_BARRIER_ALL_ROLE"] + ) if training_role == "TRAINER": role = Role.WORKER current_id = int(os.environ["PADDLE_TRAINER_ID"]) @@ -638,12 +656,13 @@ def generate_role(self): size_d = { "trainer": len(worker_endpoints), "pserver": len(eplist), - "all": len(worker_endpoints) + len(eplist) + "all": len(worker_endpoints) + len(eplist), } # child process for http server - self._http_server = Process(target=self.__start_kv_server, - args=(self._http_server_d, - size_d)) + self._http_server = Process( + target=self.__start_kv_server, + args=(self._http_server_d, size_d), + ) self._http_server.daemon = True # set running status to True self._http_server_d["running"] = True @@ -657,15 +676,21 @@ def generate_role(self): gloo.set_size(len(worker_endpoints)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) if len(self._http_ip_port) != 0: - gloo.set_http_store(self._http_ip_port[0], - int(self._http_ip_port[1]), - "trainer") + gloo.set_http_store( + self._http_ip_port[0], + int(self._http_ip_port[1]), + "trainer", + ) else: - gloo.set_hdfs_store(self._hdfs_path + "/trainer", - self._hdfs_name, self._hdfs_ugi) + gloo.set_hdfs_store( + self._hdfs_path + "/trainer", + self._hdfs_name, + self._hdfs_ugi, + ) gloo.init() self._node_type_comm = gloo if self._use_ps_gpu or self._use_metric: @@ -676,7 +701,9 @@ def generate_role(self): Gloo_strategy.ip_port = int(self._http_ip_port[1]) Default_init_timeout_seconds = 3600 Default_run_timeout_seconds = 9999999 - Gloo_strategy.init_seconds = Default_init_timeout_seconds + Gloo_strategy.init_seconds = ( + Default_init_timeout_seconds + ) Gloo_strategy.run_seconds = Default_run_timeout_seconds Gloo = fluid.core.GlooParallelContext(Gloo_strategy) Gloo.init() @@ -700,14 +727,21 @@ def generate_role(self): gloo.set_size(len(eplist)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) if len(self._http_ip_port) != 0: - gloo.set_http_store(self._http_ip_port[0], - int(self._http_ip_port[1]), "pserver") + gloo.set_http_store( + self._http_ip_port[0], + int(self._http_ip_port[1]), + "pserver", + ) else: - gloo.set_hdfs_store(self._hdfs_path + "/pserver", - self._hdfs_name, self._hdfs_ugi) + gloo.set_hdfs_store( + self._hdfs_path + "/pserver", + self._hdfs_name, + self._hdfs_ugi, + ) gloo.init() self._node_type_comm = gloo @@ -717,14 +751,17 @@ def generate_role(self): gloo.set_size(len(all_list)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) if len(self._http_ip_port) != 0: - gloo.set_http_store(self._http_ip_port[0], - int(self._http_ip_port[1]), "all") + gloo.set_http_store( + self._http_ip_port[0], int(self._http_ip_port[1]), "all" + ) else: - gloo.set_hdfs_store(self._hdfs_path + "/all", self._hdfs_name, - self._hdfs_ugi) + gloo.set_hdfs_store( + self._hdfs_path + "/all", self._hdfs_name, self._hdfs_ugi + ) gloo.init() self._all_comm = gloo self._trainers_num = trainers_num @@ -977,8 +1014,12 @@ def __get_default_iface_from_gateway(self): gateway = None if len(item) > gateway_idx: gateway = item[gateway_idx] - if gateway and gateway != '*' and gateway != "0.0.0.0" and len( - item) > iface_idx: + if ( + gateway + and gateway != '*' + and gateway != "0.0.0.0" + and len(item) > iface_idx + ): return item[iface_idx] return "lo" @@ -986,8 +1027,9 @@ def __get_default_iface_from_interfaces(self): """ get default physical interface """ - res = os.popen("ip -f inet addr | awk NR%3==1").read().strip().split( - "\n") + res = ( + os.popen("ip -f inet addr | awk NR%3==1").read().strip().split("\n") + ) for item in res: if "BROADCAST" in item: return item.split(":")[1].strip() @@ -995,6 +1037,7 @@ def __get_default_iface_from_interfaces(self): def __start_kv_server(self, http_server_d, size_d): from paddle.fluid.incubate.fleet.utils.http_server import KVServer + http_server = KVServer(int(self._http_ip_port[1]), size_d) http_server.start() wait_seconds = 5 @@ -1028,7 +1071,8 @@ def generate_role(self): xpu_num = len(xpu_endpoints) if training_role not in ["TRAINER", "PSERVER", "XPU"]: raise ValueError( - "TRAINING_ROLE must be PSERVER or TRAINER or XPU") + "TRAINING_ROLE must be PSERVER or TRAINER or XPU" + ) if training_role == "TRAINER": role = Role.WORKER current_id = int(os.environ["PADDLE_TRAINER_ID"]) @@ -1040,11 +1084,14 @@ def generate_role(self): gloo.set_size(len(worker_endpoints)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_hdfs_store( - self._hdfs_path.rstrip("/") + "/trainer", self._hdfs_name, - self._hdfs_ugi) + self._hdfs_path.rstrip("/") + "/trainer", + self._hdfs_name, + self._hdfs_ugi, + ) gloo.init() self._node_type_comm = gloo elif training_role == "XPU": @@ -1058,11 +1105,14 @@ def generate_role(self): gloo.set_size(len(xpu_endpoints)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_hdfs_store( - self._hdfs_path.rstrip("/") + "/xpu", self._hdfs_name, - self._hdfs_ugi) + self._hdfs_path.rstrip("/") + "/xpu", + self._hdfs_name, + self._hdfs_ugi, + ) gloo.init() self._node_type_comm = gloo elif training_role == "PSERVER": @@ -1083,11 +1133,14 @@ def generate_role(self): gloo.set_size(len(eplist)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_hdfs_store( - self._hdfs_path.rstrip("/") + "/pserver", self._hdfs_name, - self._hdfs_ugi) + self._hdfs_path.rstrip("/") + "/pserver", + self._hdfs_name, + self._hdfs_ugi, + ) gloo.init() self._node_type_comm = gloo @@ -1099,11 +1152,14 @@ def generate_role(self): gloo.set_size(len(heter_list)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_hdfs_store( - self._hdfs_path.rstrip("/") + "/heter", self._hdfs_name, - self._hdfs_ugi) + self._hdfs_path.rstrip("/") + "/heter", + self._hdfs_name, + self._hdfs_ugi, + ) gloo.init() self._heter_comm = gloo @@ -1114,11 +1170,14 @@ def generate_role(self): gloo.set_size(len(all_list)) gloo.set_prefix(self._prefix) gloo.set_iface(self._iface) - gloo.set_timeout_seconds(self._init_timeout_seconds, - self._run_timeout_seconds) + gloo.set_timeout_seconds( + self._init_timeout_seconds, self._run_timeout_seconds + ) gloo.set_hdfs_store( - self._hdfs_path.rstrip("/") + "/all", self._hdfs_name, - self._hdfs_ugi) + self._hdfs_path.rstrip("/") + "/all", + self._hdfs_name, + self._hdfs_ugi, + ) gloo.init() self._all_comm = gloo @@ -1167,8 +1226,7 @@ def _barrier_heter(self): self._heter_comm.barrier() def xpu_num(self): - """ - """ + """ """ if not self._role_is_generated: self.generate_role() return len(self._xpu_endpoints) @@ -1181,18 +1239,21 @@ class UserDefinedRoleMaker(RoleMakerBase): on each physical node, It can be assign by user. """ - def __init__(self, - current_id=0, - role=Role.WORKER, - worker_num=0, - server_endpoints=None): + def __init__( + self, + current_id=0, + role=Role.WORKER, + worker_num=0, + server_endpoints=None, + ): super(UserDefinedRoleMaker, self).__init__() if not isinstance(server_endpoints, list): raise TypeError("server_endpoints must be as string list") elif len(server_endpoints) <= 0: raise ValueError( - "the length of server_endpoints list must be greater than 0") + "the length of server_endpoints list must be greater than 0" + ) elif len(server_endpoints) != len(set(server_endpoints)): raise ValueError("server_endpoints can't have duplicate elements") else: @@ -1213,9 +1274,11 @@ def __init__(self, else: if current_id < 0: raise ValueError( - "current_id must be greater than or equal to 0") + "current_id must be greater than or equal to 0" + ) elif self._role == Role.SERVER and current_id >= len( - server_endpoints): + server_endpoints + ): raise ValueError( "if role is Role.SERVER, current_id must be less than or equal to len(server_endpoints) - 1" ) @@ -1263,7 +1326,8 @@ def __init__(self, current_id=0, worker_endpoints=None): raise TypeError("worker_endpoints must be as string list") elif len(worker_endpoints) <= 0: raise ValueError( - "the length of worker_endpoints list must be greater than 0") + "the length of worker_endpoints list must be greater than 0" + ) elif len(worker_endpoints) != len(set(worker_endpoints)): raise ValueError("worker_endpoints can't have duplicate elements") else: @@ -1279,7 +1343,8 @@ def __init__(self, current_id=0, worker_endpoints=None): else: if current_id < 0: raise ValueError( - "current_id must be greater than or equal to 0") + "current_id must be greater than or equal to 0" + ) elif current_id >= len(worker_endpoints): raise ValueError( "current_id must be less than or equal to len(worker_endpoints) - 1" diff --git a/python/paddle/fluid/incubate/fleet/collective/__init__.py b/python/paddle/fluid/incubate/fleet/collective/__init__.py index 8887a75f1d348..229d6e44bd621 100644 --- a/python/paddle/fluid/incubate/fleet/collective/__init__.py +++ b/python/paddle/fluid/incubate/fleet/collective/__init__.py @@ -26,7 +26,10 @@ from paddle.fluid.incubate.fleet.base.fleet_base import DistributedOptimizer from paddle.fluid import compiler -from paddle.fluid.incubate.checkpoint.checkpoint_saver import PaddleModel, CheckpointSaver +from paddle.fluid.incubate.checkpoint.checkpoint_saver import ( + PaddleModel, + CheckpointSaver, +) import paddle @@ -38,19 +41,16 @@ class LambConfig(object): - def __init__(self): pass class DistFCConfig(object): - def __init__(self): pass class Collective(Fleet): - def __init__(self): super(Collective, self).__init__(Mode.COLLECTIVE) self._local_ip = 0 @@ -64,60 +64,73 @@ def __init__(self): def init_worker(self): logging.warn( - "You should not call 'init_worker' method for collective mode.") + "You should not call 'init_worker' method for collective mode." + ) def run_worker(self, main_programs=None, scopes=None): logging.warn( - "You should not call 'run_worker' method for collective mode.") + "You should not call 'run_worker' method for collective mode." + ) def init_server(self, model_dir=None): logging.warn( - "You should not call 'init_server' method for collective mode.") + "You should not call 'init_server' method for collective mode." + ) def run_server(self): logging.warn( - "You should not call 'run_server' method for collective mode.") + "You should not call 'run_server' method for collective mode." + ) def stop_worker(self): logging.warn( - "You should not call 'stop_worker' method for collective mode.") + "You should not call 'stop_worker' method for collective mode." + ) def distributed_optimizer(self, optimizer, strategy=None): - self._optimizer = \ - CollectiveOptimizer(optimizer, strategy) + self._optimizer = CollectiveOptimizer(optimizer, strategy) return self._optimizer - def save_inference_model(self, - executor, - dirname, - feeded_var_names=None, - target_vars=None, - main_program=None, - export_for_deployment=True): + def save_inference_model( + self, + executor, + dirname, + feeded_var_names=None, + target_vars=None, + main_program=None, + export_for_deployment=True, + ): """ Prune the given `main_program` to build a new program especially for inference, and then save it and all related parameters to given `dirname` by the `executor`. """ - assert isinstance(executor, Executor), \ - "In fleet.save_inference_model() function, executor must be as" \ + assert isinstance(executor, Executor), ( + "In fleet.save_inference_model() function, executor must be as" " Executor type." + ) if main_program is None: main_program = self._origin_program - assert isinstance(main_program, Program), \ - "In fleet.save_inference_model() function, main_program " \ + assert isinstance(main_program, Program), ( + "In fleet.save_inference_model() function, main_program " "must be as Program type." - - io.save_inference_model(dirname, feeded_var_names, target_vars, - executor, main_program, None, None, - export_for_deployment) - - def save_persistables(self, - executor, - dirname, - main_program=None, - filename=None): + ) + + io.save_inference_model( + dirname, + feeded_var_names, + target_vars, + executor, + main_program, + None, + None, + export_for_deployment, + ) + + def save_persistables( + self, executor, dirname, main_program=None, filename=None + ): """ This function filters out all variables with `persistable==True` from the give `main_program` and then saves these variables to the folder @@ -128,28 +141,32 @@ def save_persistables(self, files, set `filename` None; if you would like to save all variables in a single file, use `filename` to specify the file name. """ - assert isinstance(executor, Executor), \ - "In fleet.save_inference_model() function, executor must be as" \ + assert isinstance(executor, Executor), ( + "In fleet.save_inference_model() function, executor must be as" " Executor type." + ) if main_program is None: main_program = self._origin_program - assert isinstance(main_program, Program), \ - "In fleet.save_inference_model() function, main_program " \ + assert isinstance(main_program, Program), ( + "In fleet.save_inference_model() function, main_program " "must be as Program type." + ) io.save_persistables(executor, dirname, main_program, filename=filename) - def save_checkpoint(self, - executor, - path, - trainer_id, - train_status, - fs, - main_program=None, - local_cache_path=".cache", - remain_all_checkpoint=True): + def save_checkpoint( + self, + executor, + path, + trainer_id, + train_status, + fs, + main_program=None, + local_cache_path=".cache", + remain_all_checkpoint=True, + ): """ This function save persistables and current epoch num to path. """ @@ -163,22 +180,25 @@ def save_checkpoint(self, path=path, slists=[m, t], trainer_id=trainer_id, - local_cache_path=local_cache_path) + local_cache_path=local_cache_path, + ) if not remain_all_checkpoint: c.clean_redundant_checkpoints(path) return real_path, checkpoint_no - def load_checkpoint(self, - executor, - path, - trainer_id, - train_status, - fs, - main_program=None, - local_cache_path=".cache", - ignore_empty=True): + def load_checkpoint( + self, + executor, + path, + trainer_id, + train_status, + fs, + main_program=None, + local_cache_path=".cache", + ignore_empty=True, + ): """ This function load persistables and current epoch num from path. """ @@ -188,10 +208,13 @@ def load_checkpoint(self, m = PaddleModel(executor, main_program) c = CheckpointSaver(fs) - return c.load_checkpoint(path, [m, train_status], - trainer_id=trainer_id, - ignore_empty=ignore_empty, - local_cache_path=local_cache_path) + return c.load_checkpoint( + path, + [m, train_status], + trainer_id=trainer_id, + ignore_empty=ignore_empty, + local_cache_path=local_cache_path, + ) fleet = Collective() @@ -230,18 +253,21 @@ class CollectiveOpBasedOptimizer(DistributedOptimizer): def __init__(self, optimizer, strategy=None): assert isinstance( - strategy, - DistributedStrategy), "strategy must be DistributedStrategy" + strategy, DistributedStrategy + ), "strategy must be DistributedStrategy" super(CollectiveOpBasedOptimizer, self).__init__(optimizer, strategy) - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): - return self._optimizer.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): + return self._optimizer.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) def apply_gradients(self, params_grads): return self._optimizer.apply_gradients(params_grads) @@ -263,22 +289,26 @@ def __init__(self, optimizer, strategy=DistributedStrategy()): strategy = DistributedStrategy() super(CollectiveOptimizer, self).__init__(optimizer, strategy) self._forward_recompute = strategy.forward_recompute - if (not isinstance(strategy.recompute_checkpoints, list)): - raise ValueError("DistStrategy.recompute_checkpoints should" - "be a List") + if not isinstance(strategy.recompute_checkpoints, list): + raise ValueError( + "DistStrategy.recompute_checkpoints should" "be a List" + ) self._recompute_checkpoints = strategy.recompute_checkpoints self._use_amp = strategy.use_amp self._amp_loss_scaling = strategy.amp_loss_scaling self.print_config = False - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): - return self._optimizer.backward(loss, startup_program, parameter_list, - no_grad_set, callbacks) + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): + return self._optimizer.backward( + loss, startup_program, parameter_list, no_grad_set, callbacks + ) def apply_gradients(self, params_grads): return self._optimizer.apply_gradients(params_grads) @@ -295,29 +325,40 @@ def _check_collective_mode(self, main_program, optimizer, strategy): if strategy.use_local_sgd: strategy.mode = "collective" strategy.collective_mode = "local_sgd" - self._check_condition("use_local_sgd", - use_dgc=main_program._enable_dgc, - use_dist_fc=strategy.use_dist_fc, - use_lamb=main_program._use_lamb) + self._check_condition( + "use_local_sgd", + use_dgc=main_program._enable_dgc, + use_dist_fc=strategy.use_dist_fc, + use_lamb=main_program._use_lamb, + ) if strategy.use_dist_fc: - self._check_condition("use_dist_fc", - use_dgc=main_program._enable_dgc, - use_local_sgd=strategy.use_local_sgd, - use_lamb=main_program._use_lamb) - assert strategy.dist_fc_config is not None, "DistributedStrategy.dist_fc_config should be set" + self._check_condition( + "use_dist_fc", + use_dgc=main_program._enable_dgc, + use_local_sgd=strategy.use_local_sgd, + use_lamb=main_program._use_lamb, + ) + assert ( + strategy.dist_fc_config is not None + ), "DistributedStrategy.dist_fc_config should be set" if strategy._ut4grad_allreduce: strategy.mode = "collective" strategy.collective_mode = "grad_allreduce" - self._check_condition("_ut4grad_allreduce", - use_dgc=main_program._enable_dgc, - use_lamb=main_program._use_lamb) + self._check_condition( + "_ut4grad_allreduce", + use_dgc=main_program._enable_dgc, + use_lamb=main_program._use_lamb, + ) - if self._strategy.collective_mode=="local_sgd" \ - or self._strategy.collective_mode == "grad_allreduce": - assert self._strategy.mode == "collective", \ - "local_sgd and grad_allreduce can be used under collective mode" + if ( + self._strategy.collective_mode == "local_sgd" + or self._strategy.collective_mode == "grad_allreduce" + ): + assert ( + self._strategy.mode == "collective" + ), "local_sgd and grad_allreduce can be used under collective mode" def _transpile(self, startup_program, main_program): """ @@ -330,9 +371,12 @@ def _transpile(self, startup_program, main_program): trainers_num = fleet.worker_num() if self.print_config: - print("worker_endpoints:{} trainers_num:{} current_endpoint:{} \ - trainer_id:{}".format(worker_endpoints, trainers_num, - current_endpoint, trainer_id)) + print( + "worker_endpoints:{} trainers_num:{} current_endpoint:{} \ + trainer_id:{}".format( + worker_endpoints, trainers_num, current_endpoint, trainer_id + ) + ) # call transpiler config = dist_transpiler.DistributeTranspilerConfig() @@ -340,15 +384,21 @@ def _transpile(self, startup_program, main_program): config.collective_mode = self._strategy.collective_mode config.nccl_comm_num = self._strategy.nccl_comm_num - config.use_hierarchical_allreduce = self._strategy.use_hierarchical_allreduce - config.hierarchical_allreduce_inter_nranks = self._strategy.hierarchical_allreduce_inter_nranks + config.use_hierarchical_allreduce = ( + self._strategy.use_hierarchical_allreduce + ) + config.hierarchical_allreduce_inter_nranks = ( + self._strategy.hierarchical_allreduce_inter_nranks + ) t = dist_transpiler.DistributeTranspiler(config=config) - t.transpile(trainer_id=trainer_id, - trainers=worker_endpoints_env, - startup_program=startup_program, - program=main_program, - current_endpoint=current_endpoint) + t.transpile( + trainer_id=trainer_id, + trainers=worker_endpoints_env, + startup_program=startup_program, + program=main_program, + current_endpoint=current_endpoint, + ) def _get_node_ips_from_endpoints(self, endpoints): ss = set() @@ -415,11 +465,18 @@ def _try_to_compile(self, startup_program, main_program): ) if self.print_config: - print("node_num:", node_num, "num_threads:", - exec_strategy.num_threads, "use_hierarchical_allreduce:", - self._strategy.use_hierarchical_allreduce, "nccl_comm_num:", - self._strategy.nccl_comm_num, "FLAGS_sync_nccl_allreduce:", - sync_allreduce) + print( + "node_num:", + node_num, + "num_threads:", + exec_strategy.num_threads, + "use_hierarchical_allreduce:", + self._strategy.use_hierarchical_allreduce, + "nccl_comm_num:", + self._strategy.nccl_comm_num, + "FLAGS_sync_nccl_allreduce:", + sync_allreduce, + ) self._transpile(startup_program, main_program) @@ -437,19 +494,20 @@ def _try_to_compile(self, startup_program, main_program): loss_name=self._loss.name, build_strategy=self._strategy, exec_strategy=self._strategy.exec_strategy, - share_vars_from=None) + share_vars_from=None, + ) return self._compiled_program def raiseOptimizeError(self, strategy_name, optimize_name): - raise ValueError("can not use {0} when you set DistStrategy.{1} " - "as True".format(optimize_name, strategy_name)) - - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + raise ValueError( + "can not use {0} when you set DistStrategy.{1} " + "as True".format(optimize_name, strategy_name) + ) + + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): """ minimize a program through loss Args: @@ -470,28 +528,36 @@ def minimize(self, # check optimizer conflicts if self._forward_recompute: if self._recompute_checkpoints == []: - raise ValueError("please set strategy.recompute_checkpoints" - "when set strategy.forward_recompute as True") + raise ValueError( + "please set strategy.recompute_checkpoints" + "when set strategy.forward_recompute as True" + ) if self._optimizer.__class__.__name__ in [ - "RecomputeOptimizer", "OptimizerWithMixedPrecision" + "RecomputeOptimizer", + "OptimizerWithMixedPrecision", ]: - self.raiseOptimizeError("forward_recompute", - self._optimizer.__class__.__name__) + self.raiseOptimizeError( + "forward_recompute", self._optimizer.__class__.__name__ + ) - self._optimizer = \ - fluid.optimizer.RecomputeOptimizer(self._optimizer) + self._optimizer = fluid.optimizer.RecomputeOptimizer( + self._optimizer + ) self._optimizer._set_checkpoints(self._recompute_checkpoints) if self._use_amp: if self._optimizer.__class__.__name__ in [ - "OptimizerWithMixedPrecision", "DGCMomentumOptimizer" + "OptimizerWithMixedPrecision", + "DGCMomentumOptimizer", ]: - self.raiseOptimizeError("mixed_precision", - self._optimizer.__class__.__name__) + self.raiseOptimizeError( + "mixed_precision", self._optimizer.__class__.__name__ + ) self._optimizer = fluid.contrib.mixed_precision.decorate( self._optimizer, init_loss_scaling=self._amp_loss_scaling, - use_dynamic_loss_scaling=True) + use_dynamic_loss_scaling=True, + ) main_program = loss.block.program if startup_program is None: @@ -500,11 +566,13 @@ def minimize(self, self._loss = loss - self._check_collective_mode(main_program, self._optimizer, - self._strategy) + self._check_collective_mode( + main_program, self._optimizer, self._strategy + ) optimize_ops, param_grads = self._optimizer.minimize( - loss, startup_program, parameter_list, no_grad_set=no_grad_set) + loss, startup_program, parameter_list, no_grad_set=no_grad_set + ) fleet._origin_program = main_program.clone(for_test=False) fleet._transpiled_program = main_program diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/__init__.py b/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/__init__.py index 67c1a0f0f8b47..5ba4f92732229 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/__init__.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/__init__.py @@ -29,26 +29,43 @@ from paddle.fluid.parallel_executor import ParallelExecutor from paddle.fluid.optimizer import Optimizer -from paddle.fluid.transpiler.distribute_transpiler import DistributeTranspilerConfig +from paddle.fluid.transpiler.distribute_transpiler import ( + DistributeTranspilerConfig, +) from paddle.fluid.incubate.fleet.base.fleet_base import Fleet from paddle.fluid.incubate.fleet.base.mode import Mode from paddle.fluid.incubate.fleet.base.role_maker import MPISymetricRoleMaker from paddle.fluid.incubate.fleet.parameter_server import version -from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablenames +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablenames, +) from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_lr_ops -from paddle.fluid.incubate.fleet.parameter_server.ir.public import _has_global_step -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import TrainerRuntimeConfig, DistributedStrategy, \ - SyncStrategy, AsyncStrategy, HalfAsyncStrategy, GeoStrategy, StrategyFactory +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _has_global_step, +) +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + TrainerRuntimeConfig, + DistributedStrategy, + SyncStrategy, + AsyncStrategy, + HalfAsyncStrategy, + GeoStrategy, + StrategyFactory, +) from paddle.fluid.transpiler.details.checkport import wait_server_ready from paddle.fluid.incubate.fleet.parameter_server.mode import PSMode from paddle.fluid.incubate.fleet.base.fleet_base import DistributedOptimizer -from paddle.fluid.incubate.fleet.parameter_server.ir import trainer_pass as worker -from paddle.fluid.incubate.fleet.parameter_server.ir import pserver_pass as server +from paddle.fluid.incubate.fleet.parameter_server.ir import ( + trainer_pass as worker, +) +from paddle.fluid.incubate.fleet.parameter_server.ir import ( + pserver_pass as server, +) from paddle.fluid.incubate.fleet.parameter_server.ir import public as public @@ -105,25 +122,29 @@ def _init_transpiler_worker(self): def sync_strategy_envs(): kwargs = {} kwargs[ - "pserver_endpoints"] = self._role_maker.get_pserver_endpoints() + "pserver_endpoints" + ] = self._role_maker.get_pserver_endpoints() kwargs["trainer_id"] = self._role_maker.worker_index() return kwargs def geo_strategy_envs(): - def get_sparse_attrs(): opt_init_map = {} opt_init_map["gaussian_random"] = ["seed", "mean", "std"] opt_init_map["fill_constant"] = ["value"] opt_init_map["uniform_random"] = ["seed", "min", "max"] opt_init_map["truncated_gaussian_random"] = [ - "seed", "mean", "std" + "seed", + "mean", + "std", ] - dist_varnames = get_sparse_tablenames(self._origin_main_program, - True) + dist_varnames = get_sparse_tablenames( + self._origin_main_program, True + ) sparse_varnames = get_sparse_tablenames( - self._origin_main_program, False) + self._origin_main_program, False + ) if len(dist_varnames) != 0: raise ValueError( @@ -132,15 +153,18 @@ def get_sparse_attrs(): init_attrs = [] for value_name in sparse_varnames: - value_var = self._origin_main_program.global_block( - ).vars[value_name] + value_var = self._origin_main_program.global_block().vars[ + value_name + ] value_attr = [ value_name, - ",".join([str(dim) for dim in value_var.shape]) + ",".join([str(dim) for dim in value_var.shape]), ] for op in self._origin_startup_program.global_block().ops: - if op.type in opt_init_map.keys( - ) and value_name == op.output("Out")[0]: + if ( + op.type in opt_init_map.keys() + and value_name == op.output("Out")[0] + ): init_attr = [op.type] for attr in opt_init_map[op.type]: init_attr.append(str(op.attr(attr))) @@ -185,15 +209,18 @@ def get_sparse_attrs(): if self.compiled_config.is_geo_mode(): recv_ctx = fleet.compiled_config.get_communicator_recv_context( - recv_type=4) + recv_type=4 + ) else: recv_ctx = fleet.compiled_config.get_communicator_recv_context( - recv_type=1) + recv_type=1 + ) from paddle.fluid.communicator import Communicator + self._communicator = Communicator( - trainer_config.mode, kwargs, - trainer_config.get_communicator_flags()) + trainer_config.mode, kwargs, trainer_config.get_communicator_flags() + ) self._communicator.init_with_ctx(send_ctx, recv_ctx) @@ -201,7 +228,8 @@ def get_sparse_attrs(): self._communicator.start() else: raise ValueError( - "Communicator can only be inited once, please check") + "Communicator can only be inited once, please check" + ) def init_worker(self): """ @@ -231,23 +259,31 @@ def _init_transpiler_server(self, model_dir=None): raise ValueError("There is no directory named '%s'", model_dir) sparse_varnames = self.compiled_config.get_sparse_varname_on_ps( - True) - distribtued_varnames = self.compiled_config.get_sparse_varname_on_ps( - False) + True + ) + distribtued_varnames = ( + self.compiled_config.get_sparse_varname_on_ps(False) + ) remaining_vars = list( filter( - FleetTranspiler.__exclude_vars(sparse_varnames + - distribtued_varnames), - self.main_program.list_vars())) + FleetTranspiler.__exclude_vars( + sparse_varnames + distribtued_varnames + ), + self.main_program.list_vars(), + ) + ) - fluid.io.load_vars(self._executor, - main_program=self.main_program, - dirname=model_dir, - vars=remaining_vars) + fluid.io.load_vars( + self._executor, + main_program=self.main_program, + dirname=model_dir, + vars=remaining_vars, + ) - self._load_sparse_params(dirname=model_dir, - varnames=sparse_varnames) + self._load_sparse_params( + dirname=model_dir, varnames=sparse_varnames + ) # todo(tangwei12) load distributed vars # self._load_sparse_params(dirname=model_dir, varnames=distribtued_varnames) @@ -366,13 +402,15 @@ def distributed_optimizer(self, optimizer, strategy=None): self._optimizer = ParameterServerOptimizer(optimizer, _strategy) return self._optimizer - def save_inference_model(self, - executor, - dirname, - feeded_var_names, - target_vars, - main_program=None, - export_for_deployment=True): + def save_inference_model( + self, + executor, + dirname, + feeded_var_names, + target_vars, + main_program=None, + export_for_deployment=True, + ): """ Prune the given `main_program` to build a new program especially for inference, and then save it and all related parameters to given `dirname` by the `executor`. @@ -402,14 +440,28 @@ def save_inference_model(self, raise TypeError( "in fleet.save_inference_model() function, main_program must be as Program type, CompiledProgram is not allowed" ) - fluid.io.save_inference_model(dirname, feeded_var_names, - target_vars, executor, main_program, - None, None, export_for_deployment) + fluid.io.save_inference_model( + dirname, + feeded_var_names, + target_vars, + executor, + main_program, + None, + None, + export_for_deployment, + ) else: - fluid.io.save_inference_model(dirname, feeded_var_names, - target_vars, executor, - self._origin_main_program, None, None, - export_for_deployment, True) + fluid.io.save_inference_model( + dirname, + feeded_var_names, + target_vars, + executor, + self._origin_main_program, + None, + None, + export_for_deployment, + True, + ) model_basename = "__model__" model_filename = os.path.join(dirname, model_basename) @@ -423,6 +475,7 @@ def save_inference_model(self, def _load_sparse_params(self, dirname, varnames): from paddle.fluid.communicator import LargeScaleKV + scale_kv = LargeScaleKV() for varname in varnames: origin_varname, _, _ = public._get_varname_parts(varname) @@ -431,8 +484,15 @@ def _load_sparse_params(self, dirname, varnames): def _get_optimizer_status(self, op, param_name): supported_opts = [ - "sgd", "adam", "adagrad", "adamax", "momentum", "lars_momentum", - "rmsprop", "decayed_adagrad", "ftrl" + "sgd", + "adam", + "adagrad", + "adamax", + "momentum", + "lars_momentum", + "rmsprop", + "decayed_adagrad", + "ftrl", ] reshaped_val_map = {} @@ -443,7 +503,9 @@ def _get_optimizer_status(self, op, param_name): reshaped_val_map["momentum"] = ["velocity_0"] reshaped_val_map["lars_momentum"] = ["velocity_0"] reshaped_val_map["rmsprop"] = [ - "momentum_0", "mean_square_0", "mean_grad_0" + "momentum_0", + "mean_square_0", + "mean_grad_0", ] reshaped_val_map["decayed_adagrad"] = ["moment_0"] reshaped_val_map["ftrl"] = ["squared_0", "linear_0"] @@ -454,8 +516,10 @@ def _get_optimizer_status(self, op, param_name): if op not in supported_opts: raise ValueError( - "fleet can not support optimizer: {}, only this can be supported: {}" - .format(op, supported_opts)) + "fleet can not support optimizer: {}, only this can be supported: {}".format( + op, supported_opts + ) + ) reshaped_names = [ param_name + "_" + val for val in reshaped_val_map[op] @@ -472,8 +536,11 @@ def _get_optimizer_status(self, op, param_name): def _get_optimizer_op(self, param_name): opts = public._get_optimize_ops(self._origin_main_program) for op in opts: - if "Param" in op.input_names and \ - "LearningRate" in op.input_names and op.input("Param")[0] == param_name: + if ( + "Param" in op.input_names + and "LearningRate" in op.input_names + and op.input("Param")[0] == param_name + ): return op def _save_dense_params(self, executor, dirname, context, main_program): @@ -492,27 +559,24 @@ def _save_dense_params(self, executor, dirname, context, main_program): optimizer = self._get_optimizer_op(varname) reshaped_varnames, origin_varnames = self._get_optimizer_status( - optimizer.type, varname) + optimizer.type, varname + ) for var_name in [varname] + reshaped_varnames + origin_varnames: var = self._origin_main_program.global_block().vars[var_name] - block.append_op(type='recv_save', - attrs={ - "trainer_id": - self._role_maker.worker_index(), - "shape": - var.shape, - "slice_shapes": - [",".join([str(i) for i in var.shape])], - "slice_varnames": [var.name], - "remote_varnames": [var.name], - "is_sparse": - False, - "endpoints": - var_ctx.split_endpoints(), - "file_path": - os.path.join(dirname, var.name) - }) + block.append_op( + type='recv_save', + attrs={ + "trainer_id": self._role_maker.worker_index(), + "shape": var.shape, + "slice_shapes": [",".join([str(i) for i in var.shape])], + "slice_varnames": [var.name], + "remote_varnames": [var.name], + "is_sparse": False, + "endpoints": var_ctx.split_endpoints(), + "file_path": os.path.join(dirname, var.name), + }, + ) executor.run(prog) return local_vars @@ -531,7 +595,8 @@ def _save_sparse_params(self, executor, dirname, context, main_program): optimizer = self._get_optimizer_op(varname) reshaped_varnames, origin_varnames = self._get_optimizer_status( - optimizer.type, varname) + optimizer.type, varname + ) var = self._origin_main_program.global_block().vars[varname] slice_shapes = [] @@ -540,37 +605,34 @@ def _save_sparse_params(self, executor, dirname, context, main_program): for section in var_ctx.sections(): slice_shapes.append(str(section) + dims1) - block.append_op(type='recv_save', - attrs={ - "trainer_id": - self._role_maker.worker_index(), - "shape": - var.shape, - "slice_shapes": - slice_shapes, - "slice_varnames": - var_ctx.split_varnames(), - "remote_varnames": - var_ctx.split_varnames(), - "is_sparse": - True, - "endpoints": - var_ctx.split_endpoints(), - "pserver_num": - len(self._role_maker.get_pserver_endpoints()), - "file_path": - os.path.join(dirname, var.name) - }) + block.append_op( + type='recv_save', + attrs={ + "trainer_id": self._role_maker.worker_index(), + "shape": var.shape, + "slice_shapes": slice_shapes, + "slice_varnames": var_ctx.split_varnames(), + "remote_varnames": var_ctx.split_varnames(), + "is_sparse": True, + "endpoints": var_ctx.split_endpoints(), + "pserver_num": len( + self._role_maker.get_pserver_endpoints() + ), + "file_path": os.path.join(dirname, var.name), + }, + ) for reshaped_varname in reshaped_varnames: - var = self._origin_main_program.global_block( - ).vars[reshaped_varname] + var = self._origin_main_program.global_block().vars[ + reshaped_varname + ] slice_varnames = [] remote_varnames = [] for i in range(len(var_ctx.split_varnames())): - slice_varnames.append("{}.block{}".format( - reshaped_varname, i)) + slice_varnames.append( + "{}.block{}".format(reshaped_varname, i) + ) remote_varnames.append(reshaped_varname) block.append_op( @@ -583,97 +645,112 @@ def _save_sparse_params(self, executor, dirname, context, main_program): "remote_varnames": remote_varnames, "is_sparse": True, "endpoints": var_ctx.split_endpoints(), - "pserver_num": - len(self._role_maker.get_pserver_endpoints()), - "file_path": os.path.join(dirname, var.name) - }) + "pserver_num": len( + self._role_maker.get_pserver_endpoints() + ), + "file_path": os.path.join(dirname, var.name), + }, + ) for origin_varname in origin_varnames: - var = self._origin_main_program.global_block( - ).vars[origin_varname] - - block.append_op(type='recv_save', - attrs={ - "trainer_id": - self._role_maker.worker_index(), - "shape": - var.shape, - "slice_shapes": - [",".join([str(i) for i in var.shape])], - "slice_varnames": [origin_varname], - "remote_varnames": [origin_varname], - "is_sparse": - False, - "endpoints": - var_ctx.split_endpoints()[:1], - "file_path": - os.path.join(dirname, var.name) - }) + var = self._origin_main_program.global_block().vars[ + origin_varname + ] + + block.append_op( + type='recv_save', + attrs={ + "trainer_id": self._role_maker.worker_index(), + "shape": var.shape, + "slice_shapes": [",".join([str(i) for i in var.shape])], + "slice_varnames": [origin_varname], + "remote_varnames": [origin_varname], + "is_sparse": False, + "endpoints": var_ctx.split_endpoints()[:1], + "file_path": os.path.join(dirname, var.name), + }, + ) executor.run(prog) return context.keys() - def _save_distributed_params(self, executor, dirname, context, - main_program): + def _save_distributed_params( + self, executor, dirname, context, main_program + ): prog = Program() block = prog.global_block() for name, var_ctx in context.items(): - block.append_op(type='checkpoint_notify', - attrs={ - "varname": name, - "is_slice": True, - "slice_varnames": var_ctx.split_varnames(), - "remote_varnames": var_ctx.split_varnames(), - "endpoints": var_ctx.split_endpoints(), - "dirname": dirname - }) + block.append_op( + type='checkpoint_notify', + attrs={ + "varname": name, + "is_slice": True, + "slice_varnames": var_ctx.split_varnames(), + "remote_varnames": var_ctx.split_varnames(), + "endpoints": var_ctx.split_endpoints(), + "dirname": dirname, + }, + ) executor.run(prog) return context.keys() def _save_distributed_persistables(self, executor, dirname, main_program): dense_ctx = fleet.compiled_config.get_communicator_recv_context( - recv_type=1) + recv_type=1 + ) sparse_ctx = fleet.compiled_config.get_communicator_recv_context( - recv_type=2) + recv_type=2 + ) distributed_ctx = fleet.compiled_config.get_communicator_recv_context( - recv_type=3) + recv_type=3 + ) - recv_dense_varnames = self._save_dense_params(executor, dirname, - dense_ctx, main_program) + recv_dense_varnames = self._save_dense_params( + executor, dirname, dense_ctx, main_program + ) - recv_sparse_varnames = self._save_sparse_params(executor, dirname, - sparse_ctx, - main_program) + recv_sparse_varnames = self._save_sparse_params( + executor, dirname, sparse_ctx, main_program + ) recv_distributed_varnames = self._save_distributed_params( - executor, dirname, distributed_ctx, main_program) + executor, dirname, distributed_ctx, main_program + ) - saved_varnames = recv_dense_varnames + list( - recv_sparse_varnames) + list(recv_distributed_varnames) + saved_varnames = ( + recv_dense_varnames + + list(recv_sparse_varnames) + + list(recv_distributed_varnames) + ) remaining_vars = list( - filter(FleetTranspiler.__exclude_vars(saved_varnames), - main_program.list_vars())) + filter( + FleetTranspiler.__exclude_vars(saved_varnames), + main_program.list_vars(), + ) + ) - fluid.io.save_vars(executor, - main_program=main_program, - dirname=dirname, - vars=remaining_vars) + fluid.io.save_vars( + executor, + main_program=main_program, + dirname=dirname, + vars=remaining_vars, + ) def save_persistables(self, executor, dirname, main_program=None, **kwargs): """ - This function filters out all variables with `persistable==True` from the - give `main_program` and then saves these variables to the folder `dirname` - or file `filename`. - - The `dirname` is used to specify the folder where persistable variables - are going to be saved. If you would like to save variables in separate - files, set `filename` None; -if you would like to save all variables in a - single file, use `filename` to specify the file name. + This function filters out all variables with `persistable==True` from the + give `main_program` and then saves these variables to the folder `dirname` + or file `filename`. + + The `dirname` is used to specify the folder where persistable variables + are going to be saved. If you would like to save variables in separate + files, set `filename` None; + if you would like to save all variables in a + single file, use `filename` to specify the file name. """ if self._inner_mode == PSMode.PSLIB: @@ -702,12 +779,12 @@ def save_persistables(self, executor, dirname, main_program=None, **kwargs): "in fleet.save_persistables() function, main_program must be as Program type, CompiledProgram is not allowed" ) - self._save_distributed_persistables(save_executor, dirname, - main_program) + self._save_distributed_persistables( + save_executor, dirname, main_program + ) @staticmethod def __exclude_vars(exclude_var_names=[]): - def is_valid(var): if var.name in exclude_var_names: return False @@ -719,9 +796,11 @@ def is_valid(var): if origin_varname == "learning_rate_0": return False - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable @@ -760,7 +839,8 @@ def __init__(self, optimizer, strategy, mode=PSMode.TRANSPILER): "Currently, distributed optimizer only support Adam" "Will config built-in adam for you." "We will support more functions in DistributedOptimizer", - sys.stderr) + sys.stderr, + ) self._optimizer_name = "DistributedAdam" self._optimizer = globals()[self._optimizer_name](optimizer) @@ -770,16 +850,22 @@ def __init__(self, optimizer, strategy, mode=PSMode.TRANSPILER): self._window = 1 self.type = "downpour" self.data_norm_name = [ - ".batch_size", ".batch_square_sum", ".batch_sum", - ".batch_size@GRAD", ".batch_square_sum@GRAD", ".batch_sum@GRAD" + ".batch_size", + ".batch_square_sum", + ".batch_sum", + ".batch_size@GRAD", + ".batch_square_sum@GRAD", + ".batch_sum@GRAD", ] - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): raise NotImplementedError() def apply_gradients(self, params_grads): @@ -799,7 +885,8 @@ def _build_trainer_programs(self, compiled_config): _startup = worker.fake_init_ops_pass(_startup, compiled_config) _startup = worker.init_from_server_pass(_startup, compiled_config) _startup = worker.delet_extra_optimizes_pass( - _startup, compiled_config) + _startup, compiled_config + ) else: _main = worker.append_send_ops_pass(_main, compiled_config) _startup = _startup @@ -814,57 +901,76 @@ def _build_pserver_programs(self, compiled_config): _main = server.add_listen_and_serv_pass(_main, compiled_config) _main = server.add_rpc_global_flags_pass(_main, compiled_config) _main = server.add_optimizer_pass(_main, compiled_config) - _main = server.large_scale_sparse_pass(_main, _main, - compiled_config, False) + _main = server.large_scale_sparse_pass( + _main, _main, compiled_config, False + ) _startup = server.build_pserver_startup_program_pass( - _startup, _main, compiled_config) - _startup = server.large_scale_sparse_pass(_startup, _main, - compiled_config, True) + _startup, _main, compiled_config + ) + _startup = server.large_scale_sparse_pass( + _startup, _main, compiled_config, True + ) if not compiled_config.is_sync_mode(): _main = server.delete_unused_in_main_pass( - _main, compiled_config) + _main, compiled_config + ) _startup = server.delete_unused_in_startup_pass( - _startup, _main, compiled_config) + _startup, _main, compiled_config + ) else: _main = server.add_listen_and_serv_pass(_main, compiled_config) _main = server.add_rpc_global_flags_pass(_main, compiled_config) _main = server.add_geo_optimizer_pass(_main, compiled_config) - _main = server.large_scale_sparse_pass(_main, _main, - compiled_config, False) + _main = server.large_scale_sparse_pass( + _main, _main, compiled_config, False + ) _startup = server.build_pserver_startup_program_pass( - _startup, _main, compiled_config) - _startup = server.large_scale_sparse_pass(_startup, _main, - compiled_config, True) + _startup, _main, compiled_config + ) + _startup = server.large_scale_sparse_pass( + _startup, _main, compiled_config, True + ) _startup = server.delete_unused_in_startup_pass( - _startup, _main, compiled_config) + _startup, _main, compiled_config + ) return _main, _startup - def minimize(self, - losses, - scopes=None, - startup_programs=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, + losses, + scopes=None, + startup_programs=None, + parameter_list=None, + no_grad_set=None, + ): if isinstance(losses, list): raise ValueError("need implement later") - self._optimizer.minimize(losses, startup_programs, parameter_list, - no_grad_set) + self._optimizer.minimize( + losses, startup_programs, parameter_list, no_grad_set + ) fleet._origin_main_program = default_main_program().clone( - for_test=False) + for_test=False + ) fleet._origin_startup_program = default_startup_program().clone( - for_test=False) + for_test=False + ) compiled_config = public.CompileTimeStrategy( - fleet._origin_main_program, fleet._origin_startup_program, - self._strategy, fleet._role_maker) + fleet._origin_main_program, + fleet._origin_startup_program, + self._strategy, + fleet._role_maker, + ) fleet.compiled_config = compiled_config - fleet.main_program, fleet.startup_program = \ - self._build_trainer_programs(compiled_config) if fleet.is_worker() \ + fleet.main_program, fleet.startup_program = ( + self._build_trainer_programs(compiled_config) + if fleet.is_worker() else self._build_pserver_programs(compiled_config) + ) diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/distributed_strategy.py b/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/distributed_strategy.py index 8e40fa81ebbc4..e035b9136c2dd 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/distributed_strategy.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/distributed_strategy.py @@ -13,45 +13,61 @@ # limitations under the License. __all__ = [ - "TrainerRuntimeConfig", "DistributedStrategy", "SyncStrategy", - "AsyncStrategy", "HalfAsyncStrategy", "GeoStrategy", "StrategyFactory" + "TrainerRuntimeConfig", + "DistributedStrategy", + "SyncStrategy", + "AsyncStrategy", + "HalfAsyncStrategy", + "GeoStrategy", + "StrategyFactory", ] import os import paddle.fluid as fluid -from paddle.fluid.transpiler.distribute_transpiler import DistributeTranspilerConfig, ServerRuntimeConfig +from paddle.fluid.transpiler.distribute_transpiler import ( + DistributeTranspilerConfig, + ServerRuntimeConfig, +) from paddle.fluid.incubate.fleet.parameter_server.mode import DistributedMode class TrainerRuntimeConfig(object): - def __init__(self): self.mode = None num_threads = os.getenv("CPU_NUM", "1") self.runtime_configs = {} self.runtime_configs['communicator_max_merge_var_num'] = os.getenv( - "FLAGS_communicator_max_merge_var_num", num_threads) + "FLAGS_communicator_max_merge_var_num", num_threads + ) self.runtime_configs['communicator_send_queue_size'] = os.getenv( - "FLAGS_communicator_send_queue_size", num_threads) + "FLAGS_communicator_send_queue_size", num_threads + ) self.runtime_configs[ - 'communicator_independent_recv_thread'] = os.getenv( - "FLAGS_communicator_independent_recv_thread", "1") + 'communicator_independent_recv_thread' + ] = os.getenv("FLAGS_communicator_independent_recv_thread", "1") self.runtime_configs[ - 'communicator_min_send_grad_num_before_recv'] = os.getenv( - "FLAGS_communicator_min_send_grad_num_before_recv", num_threads) + 'communicator_min_send_grad_num_before_recv' + ] = os.getenv( + "FLAGS_communicator_min_send_grad_num_before_recv", num_threads + ) self.runtime_configs['communicator_thread_pool_size'] = os.getenv( - "FLAGS_communicator_thread_pool_size", "5") + "FLAGS_communicator_thread_pool_size", "5" + ) self.runtime_configs['communicator_send_wait_times'] = os.getenv( - "FLAGS_communicator_send_wait_times", "5") + "FLAGS_communicator_send_wait_times", "5" + ) self.runtime_configs['communicator_is_sgd_optimizer'] = os.getenv( - "FLAGS_communicator_is_sgd_optimizer", "1") + "FLAGS_communicator_is_sgd_optimizer", "1" + ) # not used self.runtime_configs['rpc_deadline'] = os.getenv( - "FLAGS_rpc_deadline", "180000") + "FLAGS_rpc_deadline", "180000" + ) self.runtime_configs['rpc_retry_times'] = os.getenv( - "FLAGS_rpc_retry_times", "3") + "FLAGS_rpc_retry_times", "3" + ) def get_communicator_flags(self): need_keys = [] @@ -60,45 +76,62 @@ def get_communicator_flags(self): if self.mode is None or self.mode == DistributedMode.ASYNC: need_keys = self.runtime_configs.keys() mode_str = "async" - elif self.mode == DistributedMode.SYNC or self.mode == DistributedMode.HALF_ASYNC: + elif ( + self.mode == DistributedMode.SYNC + or self.mode == DistributedMode.HALF_ASYNC + ): mode_str = "sync or half_async" need_keys = [ 'communicator_max_merge_var_num', - 'communicator_send_wait_times', 'communicator_thread_pool_size', - 'communicator_send_queue_size' + 'communicator_send_wait_times', + 'communicator_thread_pool_size', + 'communicator_send_queue_size', ] elif self.mode == DistributedMode.GEO: mode_str = "GEO" need_keys = [ - 'communicator_thread_pool_size', 'communicator_send_wait_times', - 'communicator_max_merge_var_num', 'communicator_send_queue_size' + 'communicator_thread_pool_size', + 'communicator_send_wait_times', + 'communicator_max_merge_var_num', + 'communicator_send_queue_size', ] else: raise ValueError("Unsupported Mode") - if self.mode == DistributedMode.SYNC or self.mode == DistributedMode.HALF_ASYNC: + if ( + self.mode == DistributedMode.SYNC + or self.mode == DistributedMode.HALF_ASYNC + ): max_merge_var_num = self.runtime_configs[ - 'communicator_max_merge_var_num'] + 'communicator_max_merge_var_num' + ] send_queue_size = self.runtime_configs[ - 'communicator_send_queue_size'] + 'communicator_send_queue_size' + ] if max_merge_var_num != num_threads: - print('WARNING: In {} mode, communicator_max_merge_var_num ' - 'must be equal to CPU_NUM. But received, ' - 'communicator_max_merge_var_num = {}, CPU_NUM = ' - '{}. communicator_max_merge_var_num will be fored to {}.'. - format(mode_str, max_merge_var_num, num_threads, - num_threads)) + print( + 'WARNING: In {} mode, communicator_max_merge_var_num ' + 'must be equal to CPU_NUM. But received, ' + 'communicator_max_merge_var_num = {}, CPU_NUM = ' + '{}. communicator_max_merge_var_num will be fored to {}.'.format( + mode_str, max_merge_var_num, num_threads, num_threads + ) + ) self.runtime_configs[ - 'communicator_max_merge_var_num'] = num_threads + 'communicator_max_merge_var_num' + ] = num_threads if send_queue_size != num_threads: - print('WARNING: In {} mode, communicator_send_queue_size ' - 'must be equal to CPU_NUM. But received, ' - 'communicator_send_queue_size = {}, CPU_NUM = ' - '{}. communicator_send_queue_size will be fored to {}.'. - format(mode_str, send_queue_size, num_threads, - num_threads)) + print( + 'WARNING: In {} mode, communicator_send_queue_size ' + 'must be equal to CPU_NUM. But received, ' + 'communicator_send_queue_size = {}, CPU_NUM = ' + '{}. communicator_send_queue_size will be fored to {}.'.format( + mode_str, send_queue_size, num_threads, num_threads + ) + ) self.runtime_configs[ - 'communicator_send_queue_size'] = num_threads + 'communicator_send_queue_size' + ] = num_threads return dict((key, str(self.runtime_configs[key])) for key in need_keys) @@ -128,7 +161,6 @@ def __repr__(self): class PSLibRuntimeConfig(object): - def __init__(self): self.runtime_configs = {} @@ -137,7 +169,6 @@ def get_runtime_configs(self): class DistributedStrategy(object): - def __init__(self): self._program_config = DistributeTranspilerConfig() self._trainer_runtime_config = TrainerRuntimeConfig() @@ -150,7 +181,9 @@ def __init__(self): self._execute_strategy.num_threads = num_threads if num_threads > 1: - self._build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce + self._build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + ) self.debug_opt = None self.use_ps_gpu = False @@ -162,11 +195,13 @@ def get_debug_opt(self): if self.debug_opt is not None and isinstance(self.debug_opt, dict): opt_info["dump_slot"] = bool(self.debug_opt.get("dump_slot", 0)) opt_info["dump_converter"] = str( - self.debug_opt.get("dump_converter", "")) + self.debug_opt.get("dump_converter", "") + ) opt_info["dump_fields"] = self.debug_opt.get("dump_fields", []) opt_info["dump_file_num"] = self.debug_opt.get("dump_file_num", 16) opt_info["dump_fields_path"] = self.debug_opt.get( - "dump_fields_path", "") + "dump_fields_path", "" + ) opt_info["dump_param"] = self.debug_opt.get("dump_param", []) return opt_info @@ -182,8 +217,10 @@ def set_program_config(self, config): setattr(self._program_config, key, config[key]) else: raise ValueError( - "DistributeTranspilerConfig doesn't have key: {}". - format(key)) + "DistributeTranspilerConfig doesn't have key: {}".format( + key + ) + ) else: raise TypeError( "program_config only accept input type: dict or DistributeTranspilerConfig" @@ -207,7 +244,8 @@ def set_trainer_runtime_config(self, config): self._trainer_runtime_config.runtime_configs[key] = Value else: raise ValueError( - "TrainerRuntimeConfig doesn't have key: {}".format(key)) + "TrainerRuntimeConfig doesn't have key: {}".format(key) + ) else: raise TypeError( "trainer_runtime_config only accept input type: dict or TrainerRuntimeConfig" @@ -237,7 +275,8 @@ def set_server_runtime_config(self, config): setattr(self._server_runtime_config, key, config[key]) else: raise ValueError( - "ServerRuntimeConfig doesn't have key: {}".format(key)) + "ServerRuntimeConfig doesn't have key: {}".format(key) + ) else: raise TypeError( "server_runtime_config only accept input type: dict or ServerRuntimeConfig" @@ -261,7 +300,8 @@ def set_execute_strategy(self, config): setattr(self._execute_strategy, key, config[key]) else: raise ValueError( - "ExecutionStrategy doesn't have key: {}".format(key)) + "ExecutionStrategy doesn't have key: {}".format(key) + ) else: raise TypeError( "execute_strategy only accept input type: dict or ExecutionStrategy" @@ -285,10 +325,12 @@ def set_build_strategy(self, config): setattr(self._build_strategy, key, config[key]) else: raise ValueError( - "BuildStrategy doesn't have key: {}".format(key)) + "BuildStrategy doesn't have key: {}".format(key) + ) else: raise TypeError( - "build_strategy only accept input type: dict or BuildStrategy") + "build_strategy only accept input type: dict or BuildStrategy" + ) self.check_build_strategy() def check_build_strategy(self): @@ -298,7 +340,6 @@ def check_build_strategy(self): class SyncStrategy(DistributedStrategy): - def __init__(self): super(SyncStrategy, self).__init__() self.check_program_config() @@ -327,7 +368,6 @@ def check_build_strategy(self): class AsyncStrategy(DistributedStrategy): - def __init__(self): super(AsyncStrategy, self).__init__() self.check_program_config() @@ -354,7 +394,6 @@ def check_build_strategy(self): class HalfAsyncStrategy(DistributedStrategy): - def __init__(self): super(HalfAsyncStrategy, self).__init__() self.check_program_config() @@ -382,7 +421,6 @@ def check_build_strategy(self): class GeoStrategy(DistributedStrategy): - def __init__(self, update_frequency=100): super(GeoStrategy, self).__init__() self._program_config.geo_sgd_need_push_nums = update_frequency @@ -401,10 +439,12 @@ def check_trainer_runtime_config(self): self._trainer_runtime_config.mode = DistributedMode.GEO self._trainer_runtime_config.runtime_configs[ - 'communicator_send_queue_size'] = self._program_config.geo_sgd_need_push_nums + 'communicator_send_queue_size' + ] = self._program_config.geo_sgd_need_push_nums self._trainer_runtime_config.runtime_configs[ - 'communicator_max_merge_var_num'] = self._program_config.geo_sgd_need_push_nums + 'communicator_max_merge_var_num' + ] = self._program_config.geo_sgd_need_push_nums def check_server_runtime_config(self): pass @@ -417,7 +457,6 @@ def check_build_strategy(self): class StrategyFactory(object): - def __init_(self): pass diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/ir/heter_trainer_pass.py b/python/paddle/fluid/incubate/fleet/parameter_server/ir/heter_trainer_pass.py index bcb83a56370c8..f799a69e2c8e0 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/ir/heter_trainer_pass.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/ir/heter_trainer_pass.py @@ -18,13 +18,27 @@ import paddle.fluid.framework as framework from paddle.fluid.transpiler.details.program_utils import delete_ops -from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import find_heter_ops -from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import union_forward_gradient_op -from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import create_heter_program -from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import create_trainer_program -from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import find_block_joints -from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import find_op_input_output -from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import get_vars_name_in_block +from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import ( + find_heter_ops, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import ( + union_forward_gradient_op, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import ( + create_heter_program, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import ( + create_trainer_program, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import ( + find_block_joints, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import ( + find_op_input_output, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.trainer_pass import ( + get_vars_name_in_block, +) def split_heter_worker_ops_pass(program, config, stage_id, device): @@ -36,7 +50,8 @@ def split_heter_worker_ops_pass(program, config, stage_id, device): """ default_deveice = "cpu" program, heter_ops, _, program_block_ops = find_heter_ops( - program, default_deveice) + program, default_deveice + ) if len(heter_ops) == 0: warnings.warn( "Currently running in Heter Parameter Server mode, but no OP running on heterogeneous devices, Please check your code." @@ -46,8 +61,16 @@ def split_heter_worker_ops_pass(program, config, stage_id, device): program_block_ops = union_forward_gradient_op(program_block_ops) block_vars_detail = find_block_joints(program, program_block_ops, heter_ops) heter_program = framework.Program() - create_heter_program(program, config, heter_program, program_block_ops, - heter_ops, block_vars_detail, device, stage_id) + create_heter_program( + program, + config, + heter_program, + program_block_ops, + heter_ops, + block_vars_detail, + device, + stage_id, + ) return heter_program @@ -61,11 +84,13 @@ def split_trainer_ops_pass(program, config, default_device="cpu"): # Todo: support user define default_device (MrChengmo) default_device_ = default_device program, heter_ops, default_ops, program_block_ops = find_heter_ops( - program, default_device_) + program, default_device_ + ) program_block_ops = union_forward_gradient_op(program_block_ops) block_vars_detail = find_block_joints(program, program_block_ops, heter_ops) trainer_program = program.clone() - create_trainer_program(trainer_program, program, config, program_block_ops, - block_vars_detail) + create_trainer_program( + trainer_program, program, config, program_block_ops, block_vars_detail + ) return trainer_program diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/ir/pserver_pass.py b/python/paddle/fluid/incubate/fleet/parameter_server/ir/pserver_pass.py index 65d186c855ced..1d10bc1516821 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/ir/pserver_pass.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/ir/pserver_pass.py @@ -17,12 +17,22 @@ from paddle.fluid import core from paddle.fluid.framework import Block -from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_optimize_ops +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_optimize_ops, +) from paddle.fluid.incubate.fleet.parameter_server.ir.public import _orig_varname -from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_varname_parts -from paddle.fluid.incubate.fleet.parameter_server.ir.public import is_distributed_sparse_op -from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablename -from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablenames +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_varname_parts, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + is_distributed_sparse_op, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablename, +) +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablenames, +) from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_lr_ops LEARNING_RATE_DECAY_COUNTER = "@LR_DECAY_COUNTER@" @@ -33,8 +43,7 @@ def _is_optimizer_op(op): - if "Param" in op.input_names and \ - "LearningRate" in op.input_names: + if "Param" in op.input_names and "LearningRate" in op.input_names: return True return False @@ -75,12 +84,12 @@ def _get_optimizer_input_shape(op_type, varkey, orig_shape, param_shape): pass else: raise ValueError( - "Not supported optimizer for distributed training: %s" % op_type) + "Not supported optimizer for distributed training: %s" % op_type + ) return orig_shape def _append_pserver_non_opt_ops(optimize_block, opt_op, origin_program, config): - def _get_pserver_grad_param_var(var, var_dict): """ Return pserver side grad/param variable, return None @@ -120,8 +129,8 @@ def _get_pserver_grad_param_var(var, var_dict): # for ops like clipping and weight decay, get the split var(xxx.block0) # for inputs / outputs grad_block = _get_pserver_grad_param_var( - var, - program.global_block().vars) + var, program.global_block().vars + ) if grad_block: varlist[i] = grad_block elif var.name not in program.global_block().vars: @@ -131,16 +140,17 @@ def _get_pserver_grad_param_var(var, var_dict): varlist[i] = program.global_block().vars[var.name] inputs[key] = varlist - outputs = _get_output_map_from_op(origin_program.global_block().vars, - opt_op) + outputs = _get_output_map_from_op( + origin_program.global_block().vars, opt_op + ) for key, varlist in outputs.items(): if not isinstance(varlist, list): varlist = [varlist] for i in range(len(varlist)): var = varlist[i] grad_block = _get_pserver_grad_param_var( - var, - program.global_block().vars) + var, program.global_block().vars + ) if grad_block: varlist[i] = grad_block elif var.name not in program.global_block().vars: @@ -150,15 +160,24 @@ def _get_pserver_grad_param_var(var, var_dict): varlist[i] = program.global_block().vars[var.name] outputs[key] = varlist - return optimize_block.append_op(type=opt_op.type, - inputs=inputs, - outputs=outputs, - attrs=opt_op.all_attrs()) - - -def _append_pserver_ops(optimize_block, opt_op, endpoint, grad_to_block_id, - origin_program, merged_var, sparse_grad_to_param, - config): + return optimize_block.append_op( + type=opt_op.type, + inputs=inputs, + outputs=outputs, + attrs=opt_op.all_attrs(), + ) + + +def _append_pserver_ops( + optimize_block, + opt_op, + endpoint, + grad_to_block_id, + origin_program, + merged_var, + sparse_grad_to_param, + config, +): program = optimize_block.program pserver_block = program.global_block() new_inputs = collections.OrderedDict() @@ -180,8 +199,10 @@ def _get_param_block(opt_op): for pairs in config.merged_variables_pairs: merged_p = pairs[0] if merged_p.merged_var.name == orig_varname: - if merged_p.merged_var.name == merged_p.ordered_vars[ - 0].name: + if ( + merged_p.merged_var.name + == merged_p.ordered_vars[0].name + ): unmerged_vars.append(merged_p.ordered_vars[0]) else: merged_vars.append(merged_p.merged_var) @@ -209,8 +230,10 @@ def _get_param_block(opt_op): # because it will create a new tensor for # decayed gradient but not inplace modify the origin one origin_grad_name = opt_op.input(key)[0] - if core.kNewGradSuffix( - ) in origin_grad_name and pserver_block.has_var(origin_grad_name): + if ( + core.kNewGradSuffix() in origin_grad_name + and pserver_block.has_var(origin_grad_name) + ): new_grad = pserver_block.var(origin_grad_name) new_inputs[key] = new_grad else: @@ -220,10 +243,12 @@ def _get_param_block(opt_op): if not param_block: return - tmpvar = pserver_block.create_var(name=param_block.name, - persistable=True, - dtype=param_block.dtype, - shape=param_block.shape) + tmpvar = pserver_block.create_var( + name=param_block.name, + persistable=True, + dtype=param_block.dtype, + shape=param_block.shape, + ) new_inputs[key] = tmpvar elif key == "LearningRate": @@ -238,40 +263,52 @@ def _get_param_block(opt_op): name=origin_var.name, persistable=origin_var.persistable, dtype=origin_var.dtype, - shape=origin_var.shape) + shape=origin_var.shape, + ) new_inputs[key] = tmpvar for key in opt_op.input_names: new_shape = None if key in [ - "Param", "Grad", "LearningRate", "MasterParam", "Beta1Tensor", - "Beta2Tensor" + "Param", + "Grad", + "LearningRate", + "MasterParam", + "Beta1Tensor", + "Beta2Tensor", ]: continue var = origin_program.global_block().vars[opt_op.input(key)[0]] param_var = new_inputs["Param"] # update accumulator variable shape - new_shape = _get_optimizer_input_shape(opt_op.type, key, var.shape, - param_var.shape) - tmpvar = pserver_block.create_var(name=var.name, - persistable=var.persistable, - dtype=var.dtype, - shape=new_shape) + new_shape = _get_optimizer_input_shape( + opt_op.type, key, var.shape, param_var.shape + ) + tmpvar = pserver_block.create_var( + name=var.name, + persistable=var.persistable, + dtype=var.dtype, + shape=new_shape, + ) new_inputs[key] = tmpvar # change output's ParamOut variable - outputs = _get_output_map_from_op(origin_program.global_block().vars, - opt_op) + outputs = _get_output_map_from_op( + origin_program.global_block().vars, opt_op + ) outputs["ParamOut"] = new_inputs["Param"] - optimize_block.append_op(type=opt_op.type, - inputs=new_inputs, - outputs=outputs, - attrs=opt_op.all_attrs()) + optimize_block.append_op( + type=opt_op.type, + inputs=new_inputs, + outputs=outputs, + attrs=opt_op.all_attrs(), + ) # record sparse grad to param name if new_inputs["Grad"].type == core.VarDesc.VarType.SELECTED_ROWS: sparse_grad_to_param.append( - str(new_inputs["Grad"].name) + ":" + str(new_inputs["Param"].name)) + str(new_inputs["Grad"].name) + ":" + str(new_inputs["Param"].name) + ) def _get_input_map_from_op(varmap, op): @@ -316,7 +353,6 @@ def add_listen_and_serv_pass(program, config): "lr_decay_block_id": None, "dense_optimize_blocks": None, "sparse_optimize_blocks": None, - # runtime attribute "endpoint": config.get_ps_endpoint(), "pserver_id": config.get_role_id(), @@ -324,14 +360,13 @@ def add_listen_and_serv_pass(program, config): "distributed_mode": config.get_distributed_mode(), "rpc_get_thread_num": -1, "rpc_send_thread_num": -1, - "rpc_prefetch_thread_num": -1 + "rpc_prefetch_thread_num": -1, } # step5 append the listen_and_serv op - program.global_block().append_op(type="listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs=attrs) + program.global_block().append_op( + type="listen_and_serv", inputs={'X': []}, outputs={}, attrs=attrs + ) return program @@ -346,7 +381,8 @@ def add_rpc_global_flags_pass(program, config): if get_threads < 1 or send_threads < 1 or pull_threads < 1: raise ValueError( - "error arguments in get_threads/send_threads/pull_threads") + "error arguments in get_threads/send_threads/pull_threads" + ) op._set_attr("rpc_get_thread_num", get_threads) op._set_attr("rpc_send_thread_num", send_threads) @@ -356,18 +392,20 @@ def add_rpc_global_flags_pass(program, config): def _clone_var(block, var, persistable=True): - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=persistable) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=persistable, + ) def add_optimizer_pass(program, config): - - def _append_pserver_grad_merge_ops(optimize_block, grad_varname_for_block, - endpoint, grad_to_block_id): + def _append_pserver_grad_merge_ops( + optimize_block, grad_varname_for_block, endpoint, grad_to_block_id + ): trainers = config.get_trainers() program = optimize_block.program @@ -375,8 +413,7 @@ def _append_pserver_grad_merge_ops(optimize_block, grad_varname_for_block, grad_block = None for g in config.param_grad_ep_mapping[endpoint]["grads"]: - if _orig_varname(g.name) == \ - _orig_varname(grad_varname_for_block): + if _orig_varname(g.name) == _orig_varname(grad_varname_for_block): grad_block = g break @@ -386,41 +423,48 @@ def _append_pserver_grad_merge_ops(optimize_block, grad_varname_for_block, return None orig_varname, block_name, trainer_name = _get_varname_parts( - grad_block.name) + grad_block.name + ) if block_name: merged_var_name = '.'.join([orig_varname, block_name]) else: merged_var_name = orig_varname - merged_var = pserver_block.create_var(name=grad_block.name, - persistable=True, - type=grad_block.type, - dtype=grad_block.dtype, - shape=grad_block.shape) + merged_var = pserver_block.create_var( + name=grad_block.name, + persistable=True, + type=grad_block.type, + dtype=grad_block.dtype, + shape=grad_block.shape, + ) grad_to_block_id.append(merged_var.name + ":" + str(optimize_block.idx)) if config.is_sync_mode() and trainers > 1: vars2merge = [] for i in range(trainers): - per_trainer_name = "%s.trainer_%d" % \ - (merged_var_name, i) + per_trainer_name = "%s.trainer_%d" % (merged_var_name, i) per_trainer_var = pserver_block.create_var( name=per_trainer_name, persistable=False, type=grad_block.type, dtype=grad_block.dtype, - shape=grad_block.shape) + shape=grad_block.shape, + ) vars2merge.append(per_trainer_var) - optimize_block.append_op(type="sum", - inputs={"X": vars2merge}, - outputs={"Out": merged_var}, - attrs={"use_mkldnn": False}) - optimize_block.append_op(type="scale", - inputs={"X": merged_var}, - outputs={"Out": merged_var}, - attrs={"scale": 1.0 / float(trainers)}) + optimize_block.append_op( + type="sum", + inputs={"X": vars2merge}, + outputs={"Out": merged_var}, + attrs={"use_mkldnn": False}, + ) + optimize_block.append_op( + type="scale", + inputs={"X": merged_var}, + outputs={"Out": merged_var}, + attrs={"scale": 1.0 / float(trainers)}, + ) return merged_var origin_program = config.get_origin_main_program() @@ -450,8 +494,10 @@ def _is_opt_op_on_pserver(endpoint, op): for pairs in config.merged_variables_pairs: merged_p = pairs[0] if merged_p.merged_var.name == orig_varname: - if merged_p.merged_var.name == merged_p.ordered_vars[ - 0].name: + if ( + merged_p.merged_var.name + == merged_p.ordered_vars[0].name + ): unmerged_varnames.append(merged_p.ordered_vars[0].name) else: merged_varnames.append(merged_p.merged_var.name) @@ -473,9 +519,16 @@ def _is_opt_op_on_pserver(endpoint, op): def __append_optimize_op__(op, block, grad_to_block_id, merged_var, lr_ops): if _is_optimizer_op(op): - _append_pserver_ops(block, op, ps_endpoint, grad_to_block_id, - origin_program, merged_var, - sparse_grad_to_param, config) + _append_pserver_ops( + block, + op, + ps_endpoint, + grad_to_block_id, + origin_program, + merged_var, + sparse_grad_to_param, + config, + ) elif op not in lr_ops: _append_pserver_non_opt_ops(block, op, origin_program, config) @@ -506,8 +559,9 @@ def __append_optimize_op__(op, block, grad_to_block_id, merged_var, lr_ops): lr_decay_block = program._create_block(program.num_blocks - 1) optimize_blocks.append(lr_decay_block) for op in lr_ops: - cloned_op = _append_pserver_non_opt_ops(lr_decay_block, op, - origin_program, config) + cloned_op = _append_pserver_non_opt_ops( + lr_decay_block, op, origin_program, config + ) # append sub blocks to pserver_program in lr_decay_op # todo(tangwei12): __clone_lr_op_sub_block__ lr_decay_block_id = lr_decay_block.idx @@ -529,18 +583,25 @@ def __append_optimize_op__(op, block, grad_to_block_id, merged_var, lr_ops): grad_varname_for_block = op.attr(OP_ROLE_VAR_ATTR_NAME)[1] if op.attr(OP_ROLE_VAR_ATTR_NAME)[0] == optimize_target_param_name: merged_var = _append_pserver_grad_merge_ops( - per_opt_block, grad_varname_for_block, ps_endpoint, - grad_to_block_id) + per_opt_block, + grad_varname_for_block, + ps_endpoint, + grad_to_block_id, + ) if merged_var: break # append optimize op once then append other ops. if merged_var: for _, op in enumerate(optimize_ops): # optimizer is connected to itself - if op.attr(OP_ROLE_VAR_ATTR_NAME)[0] == optimize_target_param_name and \ - op not in global_ops: - __append_optimize_op__(op, per_opt_block, grad_to_block_id, - merged_var, lr_ops) + if ( + op.attr(OP_ROLE_VAR_ATTR_NAME)[0] + == optimize_target_param_name + and op not in global_ops + ): + __append_optimize_op__( + op, per_opt_block, grad_to_block_id, merged_var, lr_ops + ) # dedup grad to ids list grad_to_block_id = list(set(grad_to_block_id)) @@ -549,8 +610,9 @@ def __append_optimize_op__(op, block, grad_to_block_id, merged_var, lr_ops): opt_state_block = program._create_block(program.num_blocks - 1) optimize_blocks.append(opt_state_block) for glb_op in global_ops: - __append_optimize_op__(glb_op, opt_state_block, grad_to_block_id, - None, lr_ops) + __append_optimize_op__( + glb_op, opt_state_block, grad_to_block_id, None, lr_ops + ) if len(optimize_blocks) == 0: pre_block_idx = program.num_blocks - 1 @@ -590,8 +652,10 @@ def get_entry_attr(param_name): origin_name = _orig_varname(param_name) o_main_program = config.get_origin_main_program() for op in o_main_program.global_block().ops: - if is_distributed_sparse_op(op) and get_sparse_tablename( - op) == origin_name: + if ( + is_distributed_sparse_op(op) + and get_sparse_tablename(op) == origin_name + ): entry = op.attr("entry") return entry @@ -604,8 +668,10 @@ def get_initializer_attrs(acture_value_names): for value_name in acture_value_names: origin_var_name = _orig_varname(value_name) for op in o_startup_program.global_block().ops: - if op.type in opt_init_map.keys( - ) and origin_var_name == op.output("Out")[0]: + if ( + op.type in opt_init_map.keys() + and origin_var_name == op.output("Out")[0] + ): init_attr = [op.type] for attr in opt_init_map[op.type]: init_attr.append(str(op.attr(attr))) @@ -646,8 +712,16 @@ def get_optimizer_values(block): break return grad, opt_idx, value_names, value_dims, acture_names, fuse - def add_fuse_large_scale_op(block, global_block, table_name, value_names, - acture_names, grad, is_entry, opt_idx): + def add_fuse_large_scale_op( + block, + global_block, + table_name, + value_names, + acture_names, + grad, + is_entry, + opt_idx, + ): op = block.ops[opt_idx] @@ -655,99 +729,108 @@ def add_fuse_large_scale_op(block, global_block, table_name, value_names, grad = main_program.global_block().vars[op.input("Grad")[0]] lr = main_program.global_block().vars[op.input("LearningRate")[0]] - block._insert_op(opt_idx, - type="lookup_sparse_table_fuse_sgd", - inputs={ - "Grad": grad, - "LearningRate": lr - }, - attrs={ - "is_entry": is_entry, - "tablename": table_name, - "value_names": value_names - }) + block._insert_op( + opt_idx, + type="lookup_sparse_table_fuse_sgd", + inputs={"Grad": grad, "LearningRate": lr}, + attrs={ + "is_entry": is_entry, + "tablename": table_name, + "value_names": value_names, + }, + ) elif op.type == "adam": grad = main_program.global_block().vars[op.input("Grad")[0]] lr = main_program.global_block().vars[op.input("LearningRate")[0]] - beta1_pow = main_program.global_block().vars[op.input("Beta1Pow") - [0]] - beta2_pow = main_program.global_block().vars[op.input("Beta2Pow") - [0]] - beta1_pow_o = main_program.global_block().vars[op.output( - "Beta1PowOut")[0]] - beta2_pow_o = main_program.global_block().vars[op.output( - "Beta2PowOut")[0]] + beta1_pow = main_program.global_block().vars[ + op.input("Beta1Pow")[0] + ] + beta2_pow = main_program.global_block().vars[ + op.input("Beta2Pow")[0] + ] + beta1_pow_o = main_program.global_block().vars[ + op.output("Beta1PowOut")[0] + ] + beta2_pow_o = main_program.global_block().vars[ + op.output("Beta2PowOut")[0] + ] beta1 = op.attr('beta1') beta2 = op.attr('beta2') epsilon = op.attr('epsilon') - block._insert_op(opt_idx, - type="lookup_sparse_table_fuse_adam", - inputs={ - "Grad": grad, - "LearningRate": lr, - "Beta1Pow": beta1_pow, - "Beta2Pow": beta2_pow - }, - outputs={ - "Beta1PowOut": beta1_pow_o, - "Beta2PowOut": beta2_pow_o - }, - attrs={ - "beta1": beta1, - "beta2": beta2, - "epsilon": epsilon, - "is_entry": is_entry, - "tablename": table_name, - "value_names": value_names - }) + block._insert_op( + opt_idx, + type="lookup_sparse_table_fuse_adam", + inputs={ + "Grad": grad, + "LearningRate": lr, + "Beta1Pow": beta1_pow, + "Beta2Pow": beta2_pow, + }, + outputs={ + "Beta1PowOut": beta1_pow_o, + "Beta2PowOut": beta2_pow_o, + }, + attrs={ + "beta1": beta1, + "beta2": beta2, + "epsilon": epsilon, + "is_entry": is_entry, + "tablename": table_name, + "value_names": value_names, + }, + ) else: raise ValueError("only support sgd/adam optimizer now") - def add_large_scale_op(block, global_block, table_name, value_names, - acture_names, grad, is_entry, opt_idx): - ids = global_block.create_var(name="kSparseIDs@{}".format(table_name), - persistable=False, - dtype="int64", - shape=[1, 1], - lod_level=0) + def add_large_scale_op( + block, + global_block, + table_name, + value_names, + acture_names, + grad, + is_entry, + opt_idx, + ): + ids = global_block.create_var( + name="kSparseIDs@{}".format(table_name), + persistable=False, + dtype="int64", + shape=[1, 1], + lod_level=0, + ) # insert grad split to ids and tensor op - block._insert_op(opt_idx, - type="lookup_sparse_table_grad_split", - inputs={"Grad": grad}, - outputs={ - "Row": ids, - "Value": grad - }, - attrs={ - "tablename": table_name, - "is_entry": is_entry - }) + block._insert_op( + opt_idx, + type="lookup_sparse_table_grad_split", + inputs={"Grad": grad}, + outputs={"Row": ids, "Value": grad}, + attrs={"tablename": table_name, "is_entry": is_entry}, + ) # insert read at first vars = [global_block.vars[acture_name] for acture_name in acture_names] - block._insert_op(opt_idx + 1, - type="lookup_sparse_table_read", - inputs={"Ids": ids}, - outputs={"Out": vars}, - attrs={ - "tablename": table_name, - "value_names": value_names - }) + block._insert_op( + opt_idx + 1, + type="lookup_sparse_table_read", + inputs={"Ids": ids}, + outputs={"Out": vars}, + attrs={"tablename": table_name, "value_names": value_names}, + ) # append write at last inputs = {"Ids": ids, "In": vars} - block.append_op(type="lookup_sparse_table_write", - inputs=inputs, - outputs={}, - attrs={ - "tablename": table_name, - "value_names": value_names - }) + block.append_op( + type="lookup_sparse_table_write", + inputs=inputs, + outputs={}, + attrs={"tablename": table_name, "value_names": value_names}, + ) op = get_op_by_type(main_program.global_block(), "listen_and_serv") @@ -775,27 +858,53 @@ def add_large_scale_op(block, global_block, table_name, value_names, for param, blockid in param_blockid_map.items(): opt_block = program.block(blockid) - grad, opt_idx, value_names, value_dims, acture_names, fuse = \ - get_optimizer_values(opt_block) + ( + grad, + opt_idx, + value_names, + value_dims, + acture_names, + fuse, + ) = get_optimizer_values(opt_block) entry_attr = get_entry_attr(param) is_entry = False if entry_attr == "none" else True if fuse: - add_fuse_large_scale_op(opt_block, program.global_block(), - param, value_names, acture_names, grad, - is_entry, opt_idx) + add_fuse_large_scale_op( + opt_block, + program.global_block(), + param, + value_names, + acture_names, + grad, + is_entry, + opt_idx, + ) else: - add_large_scale_op(opt_block, program.global_block(), param, - value_names, acture_names, grad, is_entry, - opt_idx) + add_large_scale_op( + opt_block, + program.global_block(), + param, + value_names, + acture_names, + grad, + is_entry, + opt_idx, + ) else: large_scale_kv_metas = [] for param, blockid in param_blockid_map.items(): opt_block = main_program.block(blockid) - grad, opt_idx, value_names, value_dims, acture_names, fuse = \ - get_optimizer_values(opt_block) + ( + grad, + opt_idx, + value_names, + value_dims, + acture_names, + fuse, + ) = get_optimizer_values(opt_block) entry_attr = get_entry_attr(param) @@ -811,10 +920,18 @@ def add_large_scale_op(block, global_block, table_name, value_names, cached_str = ",".join(acture_names + [ids_name]) init_attr_str = get_initializer_attrs(acture_names) - meta_str = ":".join([ - param, names_str, dims_str, mode, grad.name, cached_str, - init_attr_str, entry_attr - ]) + meta_str = ":".join( + [ + param, + names_str, + dims_str, + mode, + grad.name, + cached_str, + init_attr_str, + entry_attr, + ] + ) print("large_scale_metas: {}".format(meta_str)) large_scale_kv_metas.append(meta_str) @@ -822,7 +939,8 @@ def add_large_scale_op(block, global_block, table_name, value_names, type="lookup_sparse_table_init", inputs=None, outputs=None, - attrs={"large_scale_metas": large_scale_kv_metas}) + attrs={"large_scale_metas": large_scale_kv_metas}, + ) # todo: need delete unused var. return program @@ -843,7 +961,8 @@ def get_distributed_from_listen_and_serv(program, origin_program): def delete_unused_in_main_pass(program, config): origin_program = config.get_origin_main_program() sparse_params = get_distributed_from_listen_and_serv( - program, origin_program) + program, origin_program + ) for var in sparse_params: if program.global_block().has_var(var): @@ -854,7 +973,8 @@ def delete_unused_in_main_pass(program, config): def delete_unused_in_startup_pass(program, main_program, config): origin_program = config.get_origin_main_program() sparse_params = get_distributed_from_listen_and_serv( - main_program, origin_program) + main_program, origin_program + ) remove_ops = [] for op in program.global_block().ops: @@ -937,15 +1057,19 @@ def _get_splited_name_and_shape(varname): new_inputs = _get_input_map_from_op(pserver_vars, op) if op.type in [ - "gaussian_random", "fill_constant", "uniform_random", - "truncated_gaussian_random" + "gaussian_random", + "fill_constant", + "uniform_random", + "truncated_gaussian_random", ]: op._set_attr("shape", list(new_outputs["Out"].shape)) - program.global_block().append_op(type=op.type, - inputs=new_inputs, - outputs=new_outputs, - attrs=op.all_attrs()) + program.global_block().append_op( + type=op.type, + inputs=new_inputs, + outputs=new_outputs, + attrs=op.all_attrs(), + ) return program @@ -954,8 +1078,9 @@ def add_geo_optimizer_pass(program, config): endpoint = config.get_ps_endpoint() params = [p for p in config.param_grad_ep_mapping[endpoint]["params"]] - sparse_tablenames = get_sparse_tablenames(config.get_origin_main_program(), - False) + sparse_tablenames = get_sparse_tablenames( + config.get_origin_main_program(), False + ) for param in params: _clone_var(program.global_block(), param) @@ -978,15 +1103,17 @@ def add_geo_optimizer_pass(program, config): if origin_varname in sparse_tablenames: sparse_grad_to_param.append(":".join([delta_var_name, param.name])) - delta_var = pserver_block.create_var(name=delta_var_name, - persistable=False, - type=param.type, - dtype=param.dtype, - shape=param.shape) - - per_opt_block.append_op(type="sum", - inputs={"X": [param, delta_var]}, - outputs={"Out": param}) + delta_var = pserver_block.create_var( + name=delta_var_name, + persistable=False, + type=param.type, + dtype=param.dtype, + shape=param.shape, + ) + + per_opt_block.append_op( + type="sum", inputs={"X": [param, delta_var]}, outputs={"Out": param} + ) param_to_block_id.append(delta_var_name + ":" + str(per_opt_block.idx)) diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/ir/public.py b/python/paddle/fluid/incubate/fleet/parameter_server/ir/public.py index 1778ace9c39b5..9b762bb9ee7f8 100755 --- a/python/paddle/fluid/incubate/fleet/parameter_server/ir/public.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/ir/public.py @@ -26,7 +26,10 @@ import paddle.fluid.framework as framework from paddle.fluid.incubate.fleet.parameter_server.mode import DistributedMode from paddle.fluid.incubate.fleet.parameter_server.ir import vars_metatools -from paddle.fluid.incubate.fleet.parameter_server.ir.ps_dispatcher import RoundRobin, PSDispatcher +from paddle.fluid.incubate.fleet.parameter_server.ir.ps_dispatcher import ( + RoundRobin, + PSDispatcher, +) from paddle.fluid.transpiler.details.program_utils import delete_ops OP_NAME_SCOPE = "op_namescope" @@ -49,9 +52,9 @@ def _get_lr_ops(program): lr_ops = [] for index, op in enumerate(program.global_block().ops): role_id = int(op.attr(RPC_OP_ROLE_ATTR_NAME)) - if role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) or \ - role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) | \ - int(OPT_OP_ROLE_ATTR_VALUE): + if role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) or role_id == int( + LR_SCHED_OP_ROLE_ATTR_VALUE + ) | int(OPT_OP_ROLE_ATTR_VALUE): lr_ops.append(op) return lr_ops @@ -68,12 +71,17 @@ def _has_global_step(lr_ops): def is_sparse_op(op): - if op.type in SPARSE_OP_LIST and op.attr('is_sparse') is True and op.attr( - 'is_distributed') is False: + if ( + op.type in SPARSE_OP_LIST + and op.attr('is_sparse') is True + and op.attr('is_distributed') is False + ): return True - if op.type == "distributed_lookup_table" and op.attr( - 'is_distributed') is False: + if ( + op.type == "distributed_lookup_table" + and op.attr('is_distributed') is False + ): return True return False @@ -83,8 +91,10 @@ def is_distributed_sparse_op(op): if op.type in SPARSE_OP_LIST and op.attr('is_distributed') is True: return True - if op.type == "distributed_lookup_table" and op.attr( - 'is_distributed') is True: + if ( + op.type == "distributed_lookup_table" + and op.attr('is_distributed') is True + ): return True return False @@ -108,7 +118,6 @@ def get_sparse_tablenames(program, is_distributed): class MergedVariable: - def __init__(self, merged, ordered, offsets): self.merged_var = merged self.ordered_vars = ordered @@ -128,7 +137,6 @@ def _singleton(*args, **kargs): @Singleton class CompileTimeStrategy(object): - def __init__(self, main_program, startup_program, strategy, role_maker): self.min_block_size = 81920 @@ -268,20 +276,24 @@ def get_origin_ps_main_program(self): def get_origin_ps_startup_program(self): return self.origin_ps_startup_program - def add_tensor_table(self, - feed_var_name, - fetch_var_name="", - startup_program=None, - main_program=None, - tensor_table_class=""): + def add_tensor_table( + self, + feed_var_name, + fetch_var_name="", + startup_program=None, + main_program=None, + tensor_table_class="", + ): self.tensor_table_dict[feed_var_name] = {} self.tensor_table_dict[feed_var_name]["feed_var_name"] = feed_var_name self.tensor_table_dict[feed_var_name]["fetch_var_name"] = fetch_var_name self.tensor_table_dict[feed_var_name][ - "startup_program"] = startup_program + "startup_program" + ] = startup_program self.tensor_table_dict[feed_var_name]["main_program"] = main_program self.tensor_table_dict[feed_var_name][ - "tensor_table_class"] = tensor_table_class + "tensor_table_class" + ] = tensor_table_class def get_tensor_table_dict(self): return self.tensor_table_dict @@ -289,8 +301,9 @@ def get_tensor_table_dict(self): def get_sparse_varname_on_ps(self, is_distributed, endpoint=None): if not endpoint: endpoint = self.get_ps_endpoint() - varnames = get_sparse_tablenames(self.get_origin_main_program(), - is_distributed) + varnames = get_sparse_tablenames( + self.get_origin_main_program(), is_distributed + ) ps_sparse_varnames = [] for varname in varnames: @@ -313,13 +326,19 @@ def get_optimize_varname_on_ps(self, param_name): # check all input for key in op.input_names: if key in [ - "Param", "Grad", "LearningRate", "Beta1Tensor", - "Beta2Tensor" + "Param", + "Grad", + "LearningRate", + "Beta1Tensor", + "Beta2Tensor", ]: continue # check varibale shape related param, e.g: Moment1 - optimize_var_names += self._get_optimizer_param_related_var_name( - op, op.type, key) + optimize_var_names += ( + self._get_optimizer_param_related_var_name( + op, op.type, key + ) + ) return optimize_var_names def _get_optimizer_param_related_var_name(self, op, op_type, varkey): @@ -346,18 +365,13 @@ def _get_optimizer_param_related_var_name(self, op, op_type, varkey): pass else: raise ValueError( - "Not supported optimizer for distributed training: %s" % - op_type) + "Not supported optimizer for distributed training: %s" % op_type + ) return related_var_names - def build_ctx(self, - vars, - mapping, - is_grad, - is_sparse, - is_send, - is_distributed=False): - + def build_ctx( + self, vars, mapping, is_grad, is_sparse, is_send, is_distributed=False + ): def get_grad_var_ep(slices): names = [] eps = [] @@ -369,10 +383,12 @@ def get_grad_var_ep(slices): names.append("{}.delta".format(slice.name)) else: names.append(slice.name) - elif is_grad and self.is_sync_mode( - ) and self.get_trainers() > 1: - names.append("{}.trainer_{}".format(slice.name, - self.get_role_id())) + elif ( + is_grad and self.is_sync_mode() and self.get_trainers() > 1 + ): + names.append( + "{}.trainer_{}".format(slice.name, self.get_role_id()) + ) else: names.append(slice.name) @@ -400,21 +416,33 @@ def get_grad_var_ep(slices): trainer_id = self.get_role_id() aggregate = True - ctx = CommContext(name, names, eps, sections, origin_varnames, - trainer_id, aggregate, is_sparse, is_distributed, []) + ctx = CommContext( + name, + names, + eps, + sections, + origin_varnames, + trainer_id, + aggregate, + is_sparse, + is_distributed, + [], + ) return ctx def get_trainer_send_context(self): send_ctx = {} - distibuted_varnames = get_sparse_tablenames(self.origin_main_program, - True) + distibuted_varnames = get_sparse_tablenames( + self.origin_main_program, True + ) idx = 0 if not self.is_geo_mode(): for merged in self.merged_dense_pairs: grad = merged[1] - ctx = self.build_ctx(grad, self.grad_var_mapping, True, False, - True) + ctx = self.build_ctx( + grad, self.grad_var_mapping, True, False, True + ) send_ctx[ctx.var_name()] = ctx for merged in self.merged_sparse_pairs: @@ -423,10 +451,18 @@ def get_trainer_send_context(self): param_name = param.merged_var.name - is_distributed = True if param_name in distibuted_varnames else False - - ctx = self.build_ctx(grad, self.grad_var_mapping, True, True, - True, is_distributed) + is_distributed = ( + True if param_name in distibuted_varnames else False + ) + + ctx = self.build_ctx( + grad, + self.grad_var_mapping, + True, + True, + True, + is_distributed, + ) send_ctx[ctx.var_name()] = ctx idx += 1 @@ -437,21 +473,39 @@ def get_trainer_send_context(self): for pairs in self.origin_sparse_pairs: param, grad = pairs param_name = param.name - is_distributed = True if param_name in distibuted_varnames else False - - param_ctx = self.build_ctx(param, self.param_var_mapping, False, - True, True, is_distributed) - grad_ctx = self.build_ctx(grad, self.grad_var_mapping, True, - True, True, is_distributed) - - ctx = CommContext(param_ctx.var_name(), - param_ctx.split_varnames(), - param_ctx.split_endpoints(), - param_ctx.sections(), - grad_ctx.origin_varnames(), - param_ctx.trainer_id(), param_ctx.aggregate(), - param_ctx.is_sparse(), - param_ctx.is_distributed(), []) + is_distributed = ( + True if param_name in distibuted_varnames else False + ) + + param_ctx = self.build_ctx( + param, + self.param_var_mapping, + False, + True, + True, + is_distributed, + ) + grad_ctx = self.build_ctx( + grad, + self.grad_var_mapping, + True, + True, + True, + is_distributed, + ) + + ctx = CommContext( + param_ctx.var_name(), + param_ctx.split_varnames(), + param_ctx.split_endpoints(), + param_ctx.sections(), + grad_ctx.origin_varnames(), + param_ctx.trainer_id(), + param_ctx.aggregate(), + param_ctx.is_sparse(), + param_ctx.is_distributed(), + [], + ) send_ctx[ctx.var_name()] = ctx idx += 1 @@ -461,24 +515,34 @@ def get_trainer_send_context(self): def get_communicator_send_context(self): send_ctx = {} - distibuted_varnames = get_sparse_tablenames(self.origin_main_program, - True) + distibuted_varnames = get_sparse_tablenames( + self.origin_main_program, True + ) idx = 0 if self.is_geo_mode(): for pairs in self.merged_dense_pairs: param = pairs[0] - ctx = self.build_ctx(param, self.param_var_mapping, False, - False, True) + ctx = self.build_ctx( + param, self.param_var_mapping, False, False, True + ) send_ctx[ctx.var_name()] = ctx for pairs in self.merged_sparse_pairs: param = pairs[0] param_name = param.merged_var.name - is_distributed = True if param_name in distibuted_varnames else False - - ctx = self.build_ctx(param, self.param_var_mapping, False, True, - True, is_distributed) + is_distributed = ( + True if param_name in distibuted_varnames else False + ) + + ctx = self.build_ctx( + param, + self.param_var_mapping, + False, + True, + True, + is_distributed, + ) send_ctx[ctx.var_name()] = ctx idx += 1 name, ctx = self._step_ctx(idx) @@ -486,18 +550,27 @@ def get_communicator_send_context(self): else: for merged in self.merged_dense_pairs: grad = merged[1] - ctx = self.build_ctx(grad, self.grad_var_mapping, True, False, - True) + ctx = self.build_ctx( + grad, self.grad_var_mapping, True, False, True + ) send_ctx[ctx.var_name()] = ctx for merged in self.merged_sparse_pairs: param, grad = merged param_name = param.merged_var.name - is_distributed = True if param_name in distibuted_varnames else False - - ctx = self.build_ctx(grad, self.grad_var_mapping, True, True, - True, is_distributed) + is_distributed = ( + True if param_name in distibuted_varnames else False + ) + + ctx = self.build_ctx( + grad, + self.grad_var_mapping, + True, + True, + True, + is_distributed, + ) send_ctx[ctx.var_name()] = ctx idx += 1 @@ -505,13 +578,14 @@ def get_communicator_send_context(self): send_ctx[name] = ctx return send_ctx - def get_communicator_recv_context(self, - recv_type=1, - use_origin_program=False): + def get_communicator_recv_context( + self, recv_type=1, use_origin_program=False + ): # recv_type # 1 : DENSE 2. SPARSE 3. DISTRIBUTED 4. ALL - distibuted_varnames = get_sparse_tablenames(self.origin_main_program, - True) + distibuted_varnames = get_sparse_tablenames( + self.origin_main_program, True + ) sparse_varnames = [] for pairs in self.origin_sparse_pairs: param, grad = pairs @@ -521,26 +595,33 @@ def get_communicator_recv_context(self, sparse_recv_ctx = {} distributed_recv_ctx = {} - variables_pairs = self.merged_variables_pairs if not use_origin_program else self.origin_merged_variables_pairs + variables_pairs = ( + self.merged_variables_pairs + if not use_origin_program + else self.origin_merged_variables_pairs + ) for merged in variables_pairs: params = merged[0] if params.merged_var.name in sparse_varnames: continue - ctx = self.build_ctx(params, self.param_var_mapping, False, False, - False, False) + ctx = self.build_ctx( + params, self.param_var_mapping, False, False, False, False + ) dense_recv_ctx[ctx.var_name()] = ctx for pairs in self.origin_sparse_pairs: param, grad = pairs if param.name in distibuted_varnames: - ctx = self.build_ctx(param, self.param_var_mapping, False, True, - False, True) + ctx = self.build_ctx( + param, self.param_var_mapping, False, True, False, True + ) distributed_recv_ctx[ctx.var_name()] = ctx else: - ctx = self.build_ctx(param, self.param_var_mapping, False, True, - False, False) + ctx = self.build_ctx( + param, self.param_var_mapping, False, True, False, False + ) sparse_recv_ctx[ctx.var_name()] = ctx if recv_type == 1: @@ -564,28 +645,44 @@ def get_the_one_trainer_send_context(self, split_dense_table): idx = 0 distibuted_varnames = get_sparse_tablenames( - self.origin_main_program, True) + self.origin_main_program, True + ) for merged in self.merged_sparse_pairs: param, grad = merged grad_name = grad.merged_var.name param_name = param.merged_var.name - is_distributed = True if param_name in distibuted_varnames else False + is_distributed = ( + True if param_name in distibuted_varnames else False + ) var = self.origin_main_program.global_block().vars[ - grad.merged_var.name] + grad.merged_var.name + ] var_numel = reduce(lambda x, y: x * y, var.shape[1:]) - sparse_ctx = CommContext(grad_name, [grad_name], - ["127.0.0.1:6071"], [var_numel], - [grad_name], trainer_id, True, True, - is_distributed, idx, False, False, -1, - []) + sparse_ctx = CommContext( + grad_name, + [grad_name], + ["127.0.0.1:6071"], + [var_numel], + [grad_name], + trainer_id, + True, + True, + is_distributed, + idx, + False, + False, + -1, + [], + ) idx += 1 send_ctx[sparse_ctx.var_name()] = sparse_ctx if len(send_ctx) == 0: raise ValueError( - "GeoSGD require sparse parameters in your net.") + "GeoSGD require sparse parameters in your net." + ) if len(self.tensor_table_dict) > 0 and self.role_maker._is_worker(): name, ctx = self._step_ctx(idx) @@ -595,12 +692,14 @@ def get_the_one_trainer_send_context(self, split_dense_table): else: return self.get_the_one_send_context(split_dense_table) - def get_dense_send_context(self, - send_ctx, - idx, - merged_dense_pairs, - trainer_id, - split_dense_table=False): + def get_dense_send_context( + self, + send_ctx, + idx, + merged_dense_pairs, + trainer_id, + split_dense_table=False, + ): if len(merged_dense_pairs) < 1: return idx if not split_dense_table: @@ -610,52 +709,87 @@ def get_dense_send_context(self, grad = merged[1] origin_varnames.append(grad.merged_var.name) var = self.origin_main_program.global_block().vars[ - grad.merged_var.name] + grad.merged_var.name + ] var_numel += reduce(lambda x, y: x * y, var.shape) grad_name = "Dense@Grad" trainer_id = self.get_role_id() aggregate = True - dense_ctx = CommContext(grad_name, [grad_name], ["127.0.0.1:6071"], - [var_numel], origin_varnames, trainer_id, - aggregate, False, False, idx, False, False, - -1, []) + dense_ctx = CommContext( + grad_name, + [grad_name], + ["127.0.0.1:6071"], + [var_numel], + origin_varnames, + trainer_id, + aggregate, + False, + False, + idx, + False, + False, + -1, + [], + ) send_ctx[grad_name] = dense_ctx idx += 1 else: for merged in merged_dense_pairs: grad = merged[1] origin_varname = grad.merged_var.name - var = self.origin_main_program.global_block( - ).vars[origin_varname] + var = self.origin_main_program.global_block().vars[ + origin_varname + ] var_numel = reduce(lambda x, y: x * y, var.shape) grad_name = origin_varname aggregate = True - dense_ctx = CommContext(grad_name, [grad_name], - ["127.0.0.1:6071"], [var_numel], - [origin_varname], trainer_id, aggregate, - False, False, idx, False, False, -1, []) + dense_ctx = CommContext( + grad_name, + [grad_name], + ["127.0.0.1:6071"], + [var_numel], + [origin_varname], + trainer_id, + aggregate, + False, + False, + idx, + False, + False, + -1, + [], + ) send_ctx[grad_name] = dense_ctx idx += 1 return idx - def get_the_one_send_context(self, - split_dense_table=False, - use_origin_program=False, - ep_list=None): + def get_the_one_send_context( + self, split_dense_table=False, use_origin_program=False, ep_list=None + ): if ep_list is None: ep_list = ["127.0.0.1:6071"] send_ctx = {} trainer_id = self.get_role_id() idx = 0 - merged_dense_pairs = self.origin_merged_dense_pairs if use_origin_program else self.merged_dense_pairs - merged_sparse_pairs = self.origin_merged_sparse_pairs if use_origin_program else self.merged_sparse_pairs + merged_dense_pairs = ( + self.origin_merged_dense_pairs + if use_origin_program + else self.merged_dense_pairs + ) + merged_sparse_pairs = ( + self.origin_merged_sparse_pairs + if use_origin_program + else self.merged_sparse_pairs + ) - idx += self.get_dense_send_context(send_ctx, idx, merged_dense_pairs, - trainer_id, split_dense_table) + idx += self.get_dense_send_context( + send_ctx, idx, merged_dense_pairs, trainer_id, split_dense_table + ) - distibuted_varnames = get_sparse_tablenames(self.origin_main_program, - True) + distibuted_varnames = get_sparse_tablenames( + self.origin_main_program, True + ) for merged in merged_sparse_pairs: param, grad = merged grad_name = grad.merged_var.name @@ -665,17 +799,33 @@ def get_the_one_send_context(self, for i in range(len(ep_list)): splited_varname.append("{}.block{}".format(param_name, i)) - is_distributed = True if param_name in distibuted_varnames else False + is_distributed = ( + True if param_name in distibuted_varnames else False + ) var = self.origin_main_program.global_block().vars[ - grad.merged_var.name] + grad.merged_var.name + ] shape = list(var.shape) shape[0] = 0 if is_distributed else shape[0] - sparse_ctx = CommContext(grad_name, splited_varname, ep_list, shape, - [grad_name], trainer_id, True, True, - is_distributed, idx, False, False, -1, []) + sparse_ctx = CommContext( + grad_name, + splited_varname, + ep_list, + shape, + [grad_name], + trainer_id, + True, + True, + is_distributed, + idx, + False, + False, + -1, + [], + ) idx += 1 send_ctx[sparse_ctx.var_name()] = sparse_ctx @@ -686,15 +836,15 @@ def get_the_one_send_context(self, return send_ctx - def get_the_one_recv_context(self, - is_dense=True, - split_dense_table=False, - use_origin_program=False): + def get_the_one_recv_context( + self, is_dense=True, split_dense_table=False, use_origin_program=False + ): recv_id_maps = {} if is_dense: send_ctx = self.get_the_one_send_context( split_dense_table=split_dense_table, - use_origin_program=use_origin_program) + use_origin_program=use_origin_program, + ) for idx, (name, ctx) in enumerate(send_ctx.items()): if ctx.is_sparse(): continue @@ -752,8 +902,22 @@ def _step_ctx(self, idx): endpoints = self.get_ps_endpoints() sections = [1] * len(endpoints) names = [name] * len(endpoints) - ctx = CommContext(name, names, endpoints, sections, [name], trainer_id, - True, False, False, idx, True, False, -1, []) + ctx = CommContext( + name, + names, + endpoints, + sections, + [name], + trainer_id, + True, + False, + False, + idx, + True, + False, + -1, + [], + ) return name, ctx def _create_vars_from_blocklist(self, block_list): @@ -784,20 +948,23 @@ def _create_vars_from_blocklist(self, block_list): if len(split) == 1: var_mapping[varname] = [orig_var] - self.var_distributed.add_distributed_var(origin_var=orig_var, - slice_var=orig_var, - block_id=0, - offset=0, - is_slice=False, - vtype="Param") + self.var_distributed.add_distributed_var( + origin_var=orig_var, + slice_var=orig_var, + block_id=0, + offset=0, + is_slice=False, + vtype="Param", + ) else: var_mapping[varname] = [] orig_shape = orig_var.shape orig_dim1_flatten = 1 if len(orig_shape) >= 2: - orig_dim1_flatten = reduce(lambda x, y: x * y, - orig_shape[1:]) + orig_dim1_flatten = reduce( + lambda x, y: x * y, orig_shape[1:] + ) for i, block in enumerate(split): size = block[1] @@ -813,7 +980,8 @@ def _create_vars_from_blocklist(self, block_list): dtype=orig_var.dtype, type=orig_var.type, lod_level=orig_var.lod_level, - persistable=False) + persistable=False, + ) var_mapping[varname].append(slice_var) self.var_distributed.add_distributed_var( @@ -822,7 +990,8 @@ def _create_vars_from_blocklist(self, block_list): block_id=i, offset=-1, is_slice=False, - vtype="Param") + vtype="Param", + ) return var_mapping @@ -871,11 +1040,9 @@ def _dispatcher(self): self.param_grad_ep_mapping[ep]["params"].append(recv_vars[i]) self.param_grad_ep_mapping[ep]["grads"].append(send_vars[i]) - def _slice_variable(self, - var_list, - slice_count, - min_block_size, - uniform=False): + def _slice_variable( + self, var_list, slice_count, min_block_size, uniform=False + ): """ We may need to split dense tensor to one or more blocks and put them equally onto parameter server. One block is a sub-tensor @@ -906,7 +1073,8 @@ def _slice_variable(self, else: split_count = slice_count max_pserver_count = int( - math.floor(var_numel / float(min_block_size))) + math.floor(var_numel / float(min_block_size)) + ) if max_pserver_count == 0: max_pserver_count = 1 if max_pserver_count < slice_count: @@ -922,10 +1090,12 @@ def _slice_variable(self, # update split_count after aligning split_count = int(math.ceil(var_numel / float(block_size))) for block_id in range(split_count): - curr_block_size = min(block_size, - var_numel - ((block_id) * block_size)) - block = vars_metatools.VarBlock(var.name, block_id, - curr_block_size) + curr_block_size = min( + block_size, var_numel - ((block_id) * block_size) + ) + block = vars_metatools.VarBlock( + var.name, block_id, curr_block_size + ) blocks.append(str(block)) else: block_size = var.shape[0] / slice_count @@ -966,13 +1136,13 @@ def _get_param_grad_blocks(self, pairs, min_block_size, uniform=False): # when we slice var up into blocks, we will slice the var according to # pserver services' count. A pserver may have two or more listening ports. - grad_blocks = self._slice_variable(grad_list, - len(self.get_ps_endpoints()), - min_block_size, uniform) + grad_blocks = self._slice_variable( + grad_list, len(self.get_ps_endpoints()), min_block_size, uniform + ) - param_blocks = self._slice_variable(param_list, - len(self.get_ps_endpoints()), - min_block_size, uniform) + param_blocks = self._slice_variable( + param_list, len(self.get_ps_endpoints()), min_block_size, uniform + ) return param_blocks, grad_blocks def _var_slice_and_distribute(self): @@ -982,15 +1152,17 @@ def _var_slice_and_distribute(self): # 3. grad_param_mapping : grad.blockx->param.blockx # 4. param_grad_ep_mapping : ep->{"params" : [], "grads" : [] } - dps, dgs = self._get_param_grad_blocks(self.merged_dense_pairs, - self.min_block_size, False) - sps, sgs = self._get_param_grad_blocks(self.merged_sparse_pairs, - self.min_block_size, True) + dps, dgs = self._get_param_grad_blocks( + self.merged_dense_pairs, self.min_block_size, False + ) + sps, sgs = self._get_param_grad_blocks( + self.merged_sparse_pairs, self.min_block_size, True + ) param_blocks = dps + sps grad_blocks = dgs + sgs - assert (len(grad_blocks) == len(param_blocks)) + assert len(grad_blocks) == len(param_blocks) # origin_param_name->[splited_param_vars] self.param_var_mapping = self._create_vars_from_blocklist(param_blocks) @@ -1001,8 +1173,9 @@ def _var_slice_and_distribute(self): for g, p in zip(grad_blocks, param_blocks): g_name, g_bid, _ = g.split(":") p_name, p_bid, _ = p.split(":") - self.grad_param_mapping[self.grad_var_mapping[g_name][int(g_bid)]] = \ - self.param_var_mapping[p_name][int(p_bid)] + self.grad_param_mapping[ + self.grad_var_mapping[g_name][int(g_bid)] + ] = self.param_var_mapping[p_name][int(p_bid)] print_maps = {} for k, v in self.grad_param_mapping.items(): @@ -1011,10 +1184,8 @@ def _var_slice_and_distribute(self): # create mapping of endpoint->split var to create pserver side program self.param_grad_ep_mapping = collections.OrderedDict() [ - self.param_grad_ep_mapping.update({ep: { - "params": [], - "grads": [] - }}) for ep in self.get_ps_endpoints() + self.param_grad_ep_mapping.update({ep: {"params": [], "grads": []}}) + for ep in self.get_ps_endpoints() ] def _build_var_distributed(self): @@ -1055,7 +1226,8 @@ def _build_var_distributed(self): for merged in self.merged_variables_pairs: m_param, m_grad = merged self.merged_variable_map[ - m_param.merged_var.name] = m_param.merged_var + m_param.merged_var.name + ] = m_param.merged_var self.merged_variable_map[m_grad.merged_var.name] = m_grad.merged_var param_merges = [] @@ -1096,8 +1268,10 @@ def _get_params_grads(sparse_varnames): for op in block.ops: if _is_opt_role_op(op): # delete clip op from opt_ops when run in Parameter Server mode - if OP_NAME_SCOPE in op.all_attrs() \ - and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE): + if ( + OP_NAME_SCOPE in op.all_attrs() + and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE) + ): op._set_attr("op_role", role_id) continue if op.attr(OP_ROLE_VAR_ATTR_NAME): @@ -1105,8 +1279,10 @@ def _get_params_grads(sparse_varnames): grad_name = op.attr(OP_ROLE_VAR_ATTR_NAME)[1] if param_name not in optimize_params: optimize_params.add(param_name) - param_grad = (origin_var_dict[param_name], - origin_var_dict[grad_name]) + param_grad = ( + origin_var_dict[param_name], + origin_var_dict[grad_name], + ) if param_name in sparse_varnames: sparse_param_grads.append(param_grad) @@ -1117,8 +1293,10 @@ def _get_params_grads(sparse_varnames): def _get_sparse_varnames(): varnames = [] for op in origin_program.global_block().ops: - if op.type in SPARSE_OP_TYPE_DICT.keys() \ - and op.attr('remote_prefetch') is True: + if ( + op.type in SPARSE_OP_TYPE_DICT.keys() + and op.attr('remote_prefetch') is True + ): param_name = op.input(SPARSE_OP_TYPE_DICT[op.type])[0] varnames.append(param_name) @@ -1126,7 +1304,8 @@ def _get_sparse_varnames(): sparse_varnames = _get_sparse_varnames() sparse_param_grads, dense_param_grads = _get_params_grads( - sparse_varnames) + sparse_varnames + ) return sparse_param_grads, dense_param_grads @@ -1160,8 +1339,9 @@ def _is_opt_role_op(op): # optimize op_maker = core.op_proto_and_checker_maker optimize_role = core.op_proto_and_checker_maker.OpRole.Optimize - if op_maker.kOpRoleAttrName() in op.attr_names and \ - int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(optimize_role): + if op_maker.kOpRoleAttrName() in op.attr_names and int( + op.all_attrs()[op_maker.kOpRoleAttrName()] + ) == int(optimize_role): return True return False @@ -1172,11 +1352,14 @@ def _get_optimize_ops(_program): for op in block.ops: if _is_opt_role_op(op): # delete clip op from opt_ops when run in Parameter Server mode - if OP_NAME_SCOPE in op.all_attrs() \ - and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE): + if ( + OP_NAME_SCOPE in op.all_attrs() + and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE) + ): op._set_attr( "op_role", - int(core.op_proto_and_checker_maker.OpRole.Backward)) + int(core.op_proto_and_checker_maker.OpRole.Backward), + ) continue opt_ops.append(op) return opt_ops @@ -1185,17 +1368,28 @@ def _get_optimize_ops(_program): def _add_lr_decay_table_pass(main_program, compiled_config, lr_decay_steps): if hasattr(compiled_config.origin_main_program, 'lr_sheduler'): from paddle.optimizer.lr import LRScheduler - assert isinstance(compiled_config.origin_main_program.lr_sheduler, - LRScheduler), "must be LRScheduler" + + assert isinstance( + compiled_config.origin_main_program.lr_sheduler, LRScheduler + ), "must be LRScheduler" ops = _get_optimize_ops(compiled_config.origin_main_program) lr_param_dict = _get_lr_param_dict(ops) - lr_decay_main_program, lr_decay_startup_program, lr_name = _get_lr_sheduler_program( - compiled_config.origin_main_program.lr_sheduler, lr_param_dict, - lr_decay_steps) - compiled_config.add_tensor_table("@LR_DECAY_COUNTER@", lr_name, - lr_decay_startup_program, - lr_decay_main_program, - "GlobalStepTable") + ( + lr_decay_main_program, + lr_decay_startup_program, + lr_name, + ) = _get_lr_sheduler_program( + compiled_config.origin_main_program.lr_sheduler, + lr_param_dict, + lr_decay_steps, + ) + compiled_config.add_tensor_table( + "@LR_DECAY_COUNTER@", + lr_name, + lr_decay_startup_program, + lr_decay_main_program, + "GlobalStepTable", + ) def _get_lr_param_dict(opt_ops): @@ -1211,11 +1405,26 @@ def _get_lr_param_dict(opt_ops): def _get_lr_sheduler_program(lr_sheduler, lr_param_dict, lr_decay_steps): schedler_decay = [ - 'NoamDecay', 'NaturalExpDecay', 'InverseTimeDecay', 'ExponentialDecay' + 'NoamDecay', + 'NaturalExpDecay', + 'InverseTimeDecay', + 'ExponentialDecay', ] - from paddle.optimizer.lr import ExponentialDecay, NoamDecay, PiecewiseDecay, NaturalExpDecay, InverseTimeDecay - from paddle.fluid.layers.learning_rate_scheduler import exponential_decay, noam_decay, piecewise_decay, natural_exp_decay, inverse_time_decay + from paddle.optimizer.lr import ( + ExponentialDecay, + NoamDecay, + PiecewiseDecay, + NaturalExpDecay, + InverseTimeDecay, + ) + from paddle.fluid.layers.learning_rate_scheduler import ( + exponential_decay, + noam_decay, + piecewise_decay, + natural_exp_decay, + inverse_time_decay, + ) decay_main_program = fluid.framework.Program() decay_startup_program = fluid.framework.Program() @@ -1230,13 +1439,16 @@ def _get_lr_sheduler_program(lr_sheduler, lr_param_dict, lr_decay_steps): "\t strategy = paddle.distributed.fleet.DistributedStrategy() \n " "\t strategy.a_sync = True \n" "\t strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } \n" - % lr_decay_steps) + % lr_decay_steps + ) elif isinstance(lr_sheduler, NoamDecay): with fluid.program_guard(decay_main_program, decay_startup_program): lr = noam_decay(lr_sheduler.d_model, lr_sheduler.warmup_steps, 1.0) lr_name = lr.name - logging.warn("NoamDecay is set, warmup steps is [ %d ]" % - lr_sheduler.warmup_steps) + logging.warn( + "NoamDecay is set, warmup steps is [ %d ]" + % lr_sheduler.warmup_steps + ) elif isinstance(lr_sheduler, NaturalExpDecay): with fluid.program_guard(decay_main_program, decay_startup_program): lr = natural_exp_decay(1.0, lr_decay_steps, lr_sheduler.gamma, True) @@ -1246,22 +1458,27 @@ def _get_lr_sheduler_program(lr_sheduler, lr_param_dict, lr_decay_steps): "\t strategy = paddle.distributed.fleet.DistributedStrategy() \n " "\t strategy.a_sync = True \n" "\t strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } \n" - % lr_decay_steps) + % lr_decay_steps + ) elif isinstance(lr_sheduler, InverseTimeDecay): with fluid.program_guard(decay_main_program, decay_startup_program): - lr = inverse_time_decay(1.0, lr_decay_steps, lr_sheduler.gamma, - True) + lr = inverse_time_decay( + 1.0, lr_decay_steps, lr_sheduler.gamma, True + ) lr_name = lr.name logging.warn( "InverseTimeDecay is set, staircase = True, global learning rate decay step is [ %d ], Change decay steps as follow: \n" "\t strategy = paddle.distributed.fleet.DistributedStrategy() \n " "\t strategy.a_sync = True \n" "\t strategy.a_sync_configs= { 'lr_decay_steps' : YOUR_DECAY_STEP } \n" - % lr_decay_steps) + % lr_decay_steps + ) else: raise ValueError( - "Not supported current LearningRate strategy, please use follow decay strategy: {}" - .format(schedler_decay)) + "Not supported current LearningRate strategy, please use follow decay strategy: {}".format( + schedler_decay + ) + ) return decay_main_program, decay_startup_program, lr_name @@ -1273,15 +1490,15 @@ def _get_varname_parts(varname): block_part = "" trainer_idx = varname.find(".trainer_") if trainer_idx >= 0: - trainer_part = varname[trainer_idx + 1:] + trainer_part = varname[trainer_idx + 1 :] else: trainer_idx = len(varname) block_index = varname.find(".block") if block_index >= 0: - block_part = varname[block_index + 1:trainer_idx] + block_part = varname[block_index + 1 : trainer_idx] else: block_index = len(varname) - orig_var_name = varname[0:min(block_index, trainer_idx)] + orig_var_name = varname[0 : min(block_index, trainer_idx)] return orig_var_name, block_part, trainer_part diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/ir/trainer_pass.py b/python/paddle/fluid/incubate/fleet/parameter_server/ir/trainer_pass.py index acd337720ca91..fa818e3c41300 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/ir/trainer_pass.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/ir/trainer_pass.py @@ -24,9 +24,13 @@ import paddle.fluid.framework as framework from paddle.fluid.transpiler.details.program_utils import delete_ops -from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_optimize_ops +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + _get_optimize_ops, +) from paddle.fluid.incubate.fleet.parameter_server.ir.public import _get_lr_ops -from paddle.fluid.incubate.fleet.parameter_server.ir.public import get_sparse_tablenames +from paddle.fluid.incubate.fleet.parameter_server.ir.public import ( + get_sparse_tablenames, +) from paddle.fluid.incubate.fleet.parameter_server.mode import DistributedMode OP_NAME_SCOPE = "op_namescope" @@ -42,7 +46,7 @@ SPARSE_OP_TYPE_DICT = {"lookup_table": "W", "lookup_table_v2": "W"} SPARSE_GRAD_OP_TYPE_DICT = { "lookup_table_grad": "W", - "lookup_table_v2_grad": "W" + "lookup_table_v2_grad": "W", } DEVICE_LIST = ["cpu", "gpu", "xpu"] COMMUNICATE_OPS_TYPE = ["send", "recv", "fetch_barrier", "send_barrier"] @@ -50,7 +54,6 @@ def delete_optimizer_pass(program, config): - def _delete_optimizer_op_and_vars(_program, optimize_ops): optimize_vars = [] optimize_op_role_vars = [] @@ -75,14 +78,17 @@ def _delete_optimizer_op_and_vars(_program, optimize_ops): def _add_lr_var(main_program, compiled_config): # Todo: hard code for pe - lr_var = compiled_config.origin_main_program.global_block( - ).vars["learning_rate_0"] - main_program.global_block().create_var(name=lr_var.name, - shape=lr_var.shape, - dtype=lr_var.dtype, - type=lr_var.type, - lod_level=lr_var.lod_level, - persistable=True) + lr_var = compiled_config.origin_main_program.global_block().vars[ + "learning_rate_0" + ] + main_program.global_block().create_var( + name=lr_var.name, + shape=lr_var.shape, + dtype=lr_var.dtype, + type=lr_var.type, + lod_level=lr_var.lod_level, + persistable=True, + ) optimizer_ops = _get_optimize_ops(program) lr_ops = _get_lr_ops(program) @@ -98,7 +104,8 @@ def _add_lr_var(main_program, compiled_config): def distributed_ops_pass(program, config, use_ps_gpu=False): trainer_id = config.get_role_id() send_ctx = config.get_the_one_send_context( - split_dense_table=config.is_heter_ps_mode) + split_dense_table=config.is_heter_ps_mode + ) w_2_table_id = {} emb_size = {} @@ -108,8 +115,10 @@ def _get_pull_sparse_ops(_program): push_sparse_ops = {} ops = {} for op in _program.global_block().ops: - if op.type in SPARSE_OP_TYPE_DICT.keys() \ - and op.attr('remote_prefetch') is True: + if ( + op.type in SPARSE_OP_TYPE_DICT.keys() + and op.attr('remote_prefetch') is True + ): param_name = op.input(SPARSE_OP_TYPE_DICT[op.type])[0] if config.is_heter_ps_mode: # trick for matchnet, need to modify @@ -123,15 +132,16 @@ def _get_pull_sparse_ops(_program): for op in _program.global_block().ops: if op.type in SPARSE_GRAD_OP_TYPE_DICT.keys(): param_name = op.input(SPARSE_GRAD_OP_TYPE_DICT[op.type])[0] - if param_name in pull_sparse_ids and op.input( - "Ids")[0] in pull_sparse_ids[param_name]: + if ( + param_name in pull_sparse_ids + and op.input("Ids")[0] in pull_sparse_ids[param_name] + ): ops = push_sparse_ops.get(param_name, []) ops.append(op) push_sparse_ops[param_name] = ops return pull_sparse_ops, push_sparse_ops def _pull_sparse_fuse(_program, pull_sparse_ops, use_ps_gpu): - def dag_check_up_and_reorder(program, inputs, outputs): global_block = program.global_block() min_output_index = len(global_block.ops) @@ -248,7 +258,8 @@ def dag_check_up_and_reorder(program, inputs, outputs): if table_id == -1: raise ValueError( - "can not find suitable sparse table, please check") + "can not find suitable sparse table, please check" + ) w_2_table_id[param] = table_id padding_idx = ops[0].attr("padding_idx") @@ -279,8 +290,9 @@ def dag_check_up_and_reorder(program, inputs, outputs): ins = op.input(op.input_names[i]) for out_id, out_var in enumerate(outputs): if out_var.name in ins: - outputs_idxs[out_id] = min(idx, - outputs_idxs[out_id]) + outputs_idxs[out_id] = min( + idx, outputs_idxs[out_id] + ) if min(outputs_idxs) - max(inputs_idxs) >= 1: if max(inputs_idxs) == -1: @@ -292,32 +304,28 @@ def dag_check_up_and_reorder(program, inputs, outputs): program.global_block()._insert_op( index=distributed_idx, type="pull_gpups_sparse", - inputs={ - "Ids": inputs, - 'W': w - }, + inputs={"Ids": inputs, 'W': w}, outputs={"Out": outputs}, attrs={ "size": [w.shape[1] for i in inputs], "is_distributed": True, - "is_sparse": True - }) + "is_sparse": True, + }, + ) else: program.global_block()._insert_op( index=distributed_idx, type="distributed_lookup_table", - inputs={ - "Ids": inputs, - 'W': w - }, + inputs={"Ids": inputs, 'W': w}, outputs={"Outputs": outputs}, attrs={ "is_distributed": is_distributed, "padding_idx": padding_idx, "table_id": table_id, "lookup_table_version": op_type, - "op_device": op_device - }) + "op_device": op_device, + }, + ) else: for i in range(len(inputs_idxs)): distributed_idx = op_idxs[i] @@ -325,18 +333,16 @@ def dag_check_up_and_reorder(program, inputs, outputs): program.global_block()._insert_op( index=distributed_idx, type="distributed_lookup_table", - inputs={ - "Ids": [inputs[i]], - 'W': w - }, + inputs={"Ids": [inputs[i]], 'W': w}, outputs={"Outputs": [outputs[i]]}, attrs={ "is_distributed": is_distributed, "padding_idx": padding_idx, "table_id": table_id, "lookup_table_version": op_type, - "op_device": op_device - }) + "op_device": op_device, + }, + ) def _push_sparse_fuse(_program, push_sparse_ops, use_ps_gpu): if use_ps_gpu: @@ -357,8 +363,10 @@ def _push_sparse_fuse(_program, push_sparse_ops, use_ps_gpu): if len(entry) == 3 and entry[0] == 'show_click_entry': show_var_name = entry[1] click_var_name = entry[2] - if show_var_name in program.global_block( - ).vars and click_var_name in program.global_block().vars: + if ( + show_var_name in program.global_block().vars + and click_var_name in program.global_block().vars + ): show = program.global_block().vars[show_var_name] clk = program.global_block().vars[click_var_name] use_entry = True @@ -373,7 +381,8 @@ def _push_sparse_fuse(_program, push_sparse_ops, use_ps_gpu): name="show", dtype=core.VarDesc.VarType.INT64, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) program.global_block()._insert_op( index=0, type='fill_constant', @@ -383,14 +392,16 @@ def _push_sparse_fuse(_program, push_sparse_ops, use_ps_gpu): 'shape': [1], 'dtype': show.dtype, 'value': 1, - #OP_ROLE_KEY: OpRole.Forward - }) + # OP_ROLE_KEY: OpRole.Forward + }, + ) clk = program.global_block().create_var( name="clk", dtype=core.VarDesc.VarType.INT64, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) program.global_block()._insert_op( index=0, type='fill_constant', @@ -400,8 +411,9 @@ def _push_sparse_fuse(_program, push_sparse_ops, use_ps_gpu): 'shape': [1], 'dtype': clk.dtype, 'value': 0, - #OP_ROLE_KEY: OpRole.Forward - }) + # OP_ROLE_KEY: OpRole.Forward + }, + ) for param, ops in push_sparse_ops.items(): all_ops = program.global_block().ops @@ -423,35 +435,35 @@ def _push_sparse_fuse(_program, push_sparse_ops, use_ps_gpu): for idx in op_idxs[::-1]: program.global_block()._remove_op(idx) - -# if use_ps_gpu: -# program.global_block().append_op( -# type="push_box_sparse", -# inputs={"Ids": inputs, -# 'Out': outputs}, -# outputs={"Out": outputs}, -# attrs={ -# "size": w.shape[1], -# "is_distributed": True, -# "is_sparse": True -# }) -# else: - program.global_block().append_op(type="distributed_push_sparse", - inputs={ - "Ids": inputs, - 'W': w, - "Outputs": outputs, - "Shows": show, - "Clicks": clk - }, - outputs={"Outputs": outputs}, - attrs={ - "is_distributed": - is_distributed, - "padding_idx": padding_idx, - "table_id": table_id, - "size": emb_size[param] - }) + # if use_ps_gpu: + # program.global_block().append_op( + # type="push_box_sparse", + # inputs={"Ids": inputs, + # 'Out': outputs}, + # outputs={"Out": outputs}, + # attrs={ + # "size": w.shape[1], + # "is_distributed": True, + # "is_sparse": True + # }) + # else: + program.global_block().append_op( + type="distributed_push_sparse", + inputs={ + "Ids": inputs, + 'W': w, + "Outputs": outputs, + "Shows": show, + "Clicks": clk, + }, + outputs={"Outputs": outputs}, + attrs={ + "is_distributed": is_distributed, + "padding_idx": padding_idx, + "table_id": table_id, + "size": emb_size[param], + }, + ) pull_sparse_ops, push_sparse_ops = _get_pull_sparse_ops(program) _pull_sparse_fuse(program, pull_sparse_ops, use_ps_gpu) @@ -476,40 +488,40 @@ def _append_send_op(union_vars, queue, is_sparse, table_id): dummy_output = [] if mode in [DistributedMode.SYNC, DistributedMode.HALF_ASYNC]: dummy_output = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) - - program.global_block().append_op(type="send", - inputs={"X": send_input_vars}, - outputs={"Out": dummy_output}, - attrs={ - "send_varnames": [queue], - "is_sparse": - is_sparse, - "table_id": - table_id, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) + name=framework.generate_control_dev_var_name() + ) + + program.global_block().append_op( + type="send", + inputs={"X": send_input_vars}, + outputs={"Out": dummy_output}, + attrs={ + "send_varnames": [queue], + "is_sparse": is_sparse, + "table_id": table_id, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return dummy_output def _append_barrier_op(dummys): - program.global_block().append_op(type="send_barrier", - inputs={"X": dummys}, - outputs={"Out": []}, - attrs={ - "trainer_id": - trainer_id, - "half_async": - True, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) + program.global_block().append_op( + type="send_barrier", + inputs={"X": dummys}, + outputs={"Out": []}, + attrs={ + "trainer_id": trainer_id, + "half_async": True, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) dummys = [] sends = config.get_the_one_trainer_send_context( - split_dense_table=config.is_heter_ps_mode) + split_dense_table=config.is_heter_ps_mode + ) for merged_name, send in sends.items(): if send.is_sparse() and not config.is_geo_mode(): @@ -517,8 +529,10 @@ def _append_barrier_op(dummys): is_sparse = 1 if send.is_sparse() else 0 is_sparse = 2 if send.is_distributed() else is_sparse dummys.append( - _append_send_op(send.origin_varnames(), merged_name, is_sparse, - send.table_id())) + _append_send_op( + send.origin_varnames(), merged_name, is_sparse, send.table_id() + ) + ) if mode in [DistributedMode.SYNC, DistributedMode.HALF_ASYNC]: _append_barrier_op(dummys) @@ -532,19 +546,19 @@ def init_from_server_pass(program, config): return program fetch_barrier_out = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) - - program.global_block().append_op(type="fetch_barrier", - inputs={}, - outputs={"Out": fetch_barrier_out}, - attrs={ - "endpoints": - config.get_ps_endpoints(), - "trainer_id": - config.get_role_id(), - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) + name=framework.generate_control_dev_var_name() + ) + + program.global_block().append_op( + type="fetch_barrier", + inputs={}, + outputs={"Out": fetch_barrier_out}, + attrs={ + "endpoints": config.get_ps_endpoints(), + "trainer_id": config.get_role_id(), + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return program @@ -566,14 +580,16 @@ def _fake_init_sparsetable(sparse_table_names): table_param_init_op.append(op) init_op_num = len(table_param_init_op) if init_op_num != 1: - raise ValueError("table init op num should be 1, now is " + - str(init_op_num)) + raise ValueError( + "table init op num should be 1, now is " + str(init_op_num) + ) table_init_op = table_param_init_op[0] program.global_block().append_op( type="fake_init", inputs={}, outputs={"Out": table_var}, - attrs={"shape": table_init_op.attr('shape')}) + attrs={"shape": table_init_op.attr('shape')}, + ) delete_ops(program.global_block(), table_param_init_op) sparse_tables = _get_sparse_table_names() @@ -583,7 +599,6 @@ def _fake_init_sparsetable(sparse_table_names): def ps_gpu_pass(program): - def _add_push_box_sparse_op(program): op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName() backward = core.op_proto_and_checker_maker.OpRole.Backward @@ -591,7 +606,8 @@ def _add_push_box_sparse_op(program): if op.type != "pull_box_sparse" and op.type != "pull_gpups_sparse": continue grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - op.desc, set(), []) + op.desc, set(), [] + ) for op_desc in grad_op_desc: new_op_desc = program.global_block().desc.append_op() new_op_desc.copy_from(op_desc) @@ -700,8 +716,11 @@ def delete_extra_optimizes_pass(program, config): def find_heter_ops(program, default_device="cpu"): if default_device not in DEVICE_LIST: - raise ValueError("Given device {} is not in device list {}".format( - default_device, DEVICE_LIST)) + raise ValueError( + "Given device {} is not in device list {}".format( + default_device, DEVICE_LIST + ) + ) def _is_heter_op(op, current_heter_device, default_device="cpu"): heter_devices = list(DEVICE_LIST) @@ -710,10 +729,13 @@ def _is_heter_op(op, current_heter_device, default_device="cpu"): op_type = op.type if op_device in heter_devices: return True - elif op_type in COMMUNICATE_OPS_TYPE and current_heter_device != default_device: + elif ( + op_type in COMMUNICATE_OPS_TYPE + and current_heter_device != default_device + ): # for distributed communciate ops: send & recv & barrier etc. # Todo: need update this method - #op._set_attr('op_device', current_heter_device) + # op._set_attr('op_device', current_heter_device) return True elif op_device == None or op_device == default_device: op._set_attr('op_device', default_device) @@ -748,16 +770,20 @@ def _append_heter_op(op, current_heter_block_ops, heter_ops): op = op_list[i] if "_grad" in op.type: forward_op_type = op.type.split("_grad")[0] - if forward_op_type in SPARSE_OP_TYPE_DICT.keys() \ - and op.attr('remote_prefetch') is True: + if ( + forward_op_type in SPARSE_OP_TYPE_DICT.keys() + and op.attr('remote_prefetch') is True + ): param_name = op.input(SPARSE_OP_TYPE_DICT[forward_op_type])[0] if param_name in var2idx: ## insert sum op & remove sum op from var2idx and origin place op_list = list(block.ops) sum_op = op_list[var2idx[param_name]] sum_op_inputs = { - sum_op.input_names[0]: - [block.vars[input] for input in sum_op.input_arg_names] + sum_op.input_names[0]: [ + block.vars[input] + for input in sum_op.input_arg_names + ] } sum_op_outputs = { sum_op.output_names[0]: [ @@ -765,11 +791,13 @@ def _append_heter_op(op, current_heter_block_ops, heter_ops): for output in sum_op.output_arg_names ] } - block._insert_op(index=i + 1, - type=sum_op.type, - inputs=sum_op_inputs, - outputs=sum_op_outputs, - attrs=sum_op.all_attrs()) + block._insert_op( + index=i + 1, + type=sum_op.type, + inputs=sum_op_inputs, + outputs=sum_op_outputs, + attrs=sum_op.all_attrs(), + ) block._remove_op(var2idx[param_name] + 1) var2idx.pop(param_name) for var_ in var2idx: @@ -790,9 +818,9 @@ def _append_heter_op(op, current_heter_block_ops, heter_ops): for no_grad_var in output_vars_no_grad: if no_grad_var in var2idx: """ - insert sum op & remove sum op from var2idx and origin place + insert sum op & remove sum op from var2idx and origin place - """ + """ op_list = list(block.ops) sum_op = op_list[var2idx[no_grad_var]] sum_op_inputs = { @@ -807,11 +835,13 @@ def _append_heter_op(op, current_heter_block_ops, heter_ops): for output in sum_op.output_arg_names ] } - block._insert_op(index=i + 1, - type=sum_op.type, - inputs=sum_op_inputs, - outputs=sum_op_outputs, - attrs=sum_op.all_attrs()) + block._insert_op( + index=i + 1, + type=sum_op.type, + inputs=sum_op_inputs, + outputs=sum_op_outputs, + attrs=sum_op.all_attrs(), + ) block._remove_op(var2idx[no_grad_var] + 1) var2idx.pop(no_grad_var) for var_ in var2idx: @@ -824,12 +854,16 @@ def _append_heter_op(op, current_heter_block_ops, heter_ops): pre_op = op_list[i - 1] if "_grad" in pre_op.type: forward_op_type = pre_op.type.split("_grad")[0] - if forward_op_type in SPARSE_OP_TYPE_DICT.keys() \ - and pre_op.attr('remote_prefetch') is True: + if ( + forward_op_type in SPARSE_OP_TYPE_DICT.keys() + and pre_op.attr('remote_prefetch') is True + ): param_name = pre_op.input( - SPARSE_OP_TYPE_DICT[forward_op_type])[0] + SPARSE_OP_TYPE_DICT[forward_op_type] + )[0] if param_name == origin_var and op.attr( - "op_device") == pre_op.attr("op_device"): + "op_device" + ) == pre_op.attr("op_device"): continue else: var2idx[origin_var] = i @@ -882,7 +916,8 @@ def _append_heter_op(op, current_heter_block_ops, heter_ops): # for cpu-op block append if len(current_default_block_ops) > 1: default_ops[default_device][ - block_index] = current_default_block_ops + block_index + ] = current_default_block_ops program_block_ops.append(current_default_block_ops) current_default_block_ops = [] block_index += 1 @@ -938,14 +973,24 @@ def _append_heter_op(op, current_heter_block_ops, heter_ops): for _, heter_block in heter_block_dict.items(): total_heter_ops += len(heter_block) print( - "There are {} OPs in your main_program, and contains {} heter-OPs which is made up of {} heter-blocks." - .format(len(block.ops), total_heter_ops, heter_blocks)) + "There are {} OPs in your main_program, and contains {} heter-OPs which is made up of {} heter-blocks.".format( + len(block.ops), total_heter_ops, heter_blocks + ) + ) return origin_porgram, heter_ops, default_ops, program_block_ops -def create_heter_program(program, config, heter_program, program_block_ops_list, - heter_ops, block_var_detail, current_device, stage_id): +def create_heter_program( + program, + config, + heter_program, + program_block_ops_list, + heter_ops, + block_var_detail, + current_device, + stage_id, +): # This function mainly includes the following contents: # 1. For every heter block: # a) copy heter device op from origin program @@ -993,57 +1038,76 @@ def create_heter_program(program, config, heter_program, program_block_ops_list, for _, op in enumerate(heter_block_ops_backward): block_append_op(heter_program, program, heter_block_bp, op) - bp_entrance_vars = block_var_detail[stage_id - - 1]["backward"]["entrance"] - add_vars_by_var_list(bp_entrance_vars, program, heter_program, - heter_block_bp) + bp_entrance_vars = block_var_detail[stage_id - 1]["backward"][ + "entrance" + ] + add_vars_by_var_list( + bp_entrance_vars, program, heter_program, heter_block_bp + ) bp_exit_vars = block_var_detail[stage_id - 1]["backward"]["exit"] - add_vars_by_var_list(bp_exit_vars, program, heter_program, - heter_block_bp) - backward_comm_info = get_communicate_var_info(program, - stage_id, - bp_entrance_vars, - type="backward") + add_vars_by_var_list( + bp_exit_vars, program, heter_program, heter_block_bp + ) + backward_comm_info = get_communicate_var_info( + program, stage_id, bp_entrance_vars, type="backward" + ) - grad_to_block_id.append(backward_comm_info["block_input_var_name"] + - ":" + str(heter_block_bp.idx)) + grad_to_block_id.append( + backward_comm_info["block_input_var_name"] + + ":" + + str(heter_block_bp.idx) + ) else: for _, op in enumerate(heter_block_ops_backward): block_append_op(heter_program, program, heter_block, op) - bp_entrance_vars = block_var_detail[stage_id - - 1]["backward"]["entrance"] - add_vars_by_var_list(bp_entrance_vars, program, heter_program, - heter_block) + bp_entrance_vars = block_var_detail[stage_id - 1]["backward"][ + "entrance" + ] + add_vars_by_var_list( + bp_entrance_vars, program, heter_program, heter_block + ) bp_exit_vars = block_var_detail[stage_id - 1]["backward"]["exit"] add_vars_by_var_list(bp_exit_vars, program, heter_program, heter_block) heter_block_bp = heter_block - forward_comm_info = get_communicate_var_info(program, - stage_id, - entrance_vars, - type="forward") + forward_comm_info = get_communicate_var_info( + program, stage_id, entrance_vars, type="forward" + ) - grad_to_block_id.append(forward_comm_info["block_input_var_name"] + ":" + - str(heter_block.idx)) + grad_to_block_id.append( + forward_comm_info["block_input_var_name"] + ":" + str(heter_block.idx) + ) first_op_index_bp = len(heter_block_bp.ops) if stage_id <= len(block_var_detail) - 1: - static_var = insert_communicate_op(program, config, heter_block, - stage_id, first_op_index_fp, - block_var_detail, current_device) - static_var_bp = insert_communicate_op(program, config, heter_block_bp, - stage_id, first_op_index_bp, - block_var_detail, current_device, - False) + static_var = insert_communicate_op( + program, + config, + heter_block, + stage_id, + first_op_index_fp, + block_var_detail, + current_device, + ) + static_var_bp = insert_communicate_op( + program, + config, + heter_block_bp, + stage_id, + first_op_index_bp, + block_var_detail, + current_device, + False, + ) # add send op - send_grad_var_list = add_heter_send_op(program, heter_program, - heter_block_bp, - block_var_detail[stage_id - 1]) + send_grad_var_list = add_heter_send_op( + program, heter_program, heter_block_bp, block_var_detail[stage_id - 1] + ) # --------------- # add step conter @@ -1064,9 +1128,9 @@ def create_heter_program(program, config, heter_program, program_block_ops_list, # add info in listen&serv attrs = { - #"mode": "sync", - #"trainers": config.get_trainers(), - #"trainer_id": config.get_role_id() + config.get_trainers(), + # "mode": "sync", + # "trainers": config.get_trainers(), + # "trainer_id": config.get_role_id() + config.get_trainers(), "message_to_block_id": grad_to_block_id, "optimize_blocks": optimizer_block, # runtime attribute @@ -1075,13 +1139,12 @@ def create_heter_program(program, config, heter_program, program_block_ops_list, "pserver_id": config.get_role_id(), "distributed_mode": config.get_distributed_mode(), "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, } # append the listen_and_serv op - heter_program.global_block().append_op(type="heter_listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs=attrs) + heter_program.global_block().append_op( + type="heter_listen_and_serv", inputs={'X': []}, outputs={}, attrs=attrs + ) check_heter_compile_time_strategy(program, config, send_grad_var_list) @@ -1093,14 +1156,16 @@ def check_heter_compile_time_strategy(program, config, send_grad_var_list): origin_grad_var_list = list(set(origin_grad_var_list)) send_grad_var_list = list(set(send_grad_var_list)) useless_grad_var_list = list( - set(origin_grad_var_list) - set(send_grad_var_list)) + set(origin_grad_var_list) - set(send_grad_var_list) + ) for useless_grad_var in useless_grad_var_list: config.remove_var_pair_by_grad(useless_grad_var) -def create_trainer_program(program, origin_program, config, - program_block_ops_list, block_var_detail): +def create_trainer_program( + program, origin_program, config, program_block_ops_list, block_var_detail +): # This function mainly includes the following contents: # 1. For every heter block in origin program # a) delete heter op and related variables @@ -1113,13 +1178,16 @@ def create_trainer_program(program, origin_program, config, # 2. check every op's device static_var = [] for heter_block_index in range(1, len(program_block_ops_list)): - ops_list = program_block_ops_list[heter_block_index][ - "forward"] + program_block_ops_list[heter_block_index]["backward"] - static_var += replace_ops_by_communicate_op(program, config, - heter_block_index, ops_list, - block_var_detail) - remove_trainer_send_op(program, config, heter_block_index, - block_var_detail) + ops_list = ( + program_block_ops_list[heter_block_index]["forward"] + + program_block_ops_list[heter_block_index]["backward"] + ) + static_var += replace_ops_by_communicate_op( + program, config, heter_block_index, ops_list, block_var_detail + ) + remove_trainer_send_op( + program, config, heter_block_index, block_var_detail + ) optimizer_block = [] grad_to_block_id = [] @@ -1127,23 +1195,26 @@ def create_trainer_program(program, origin_program, config, bp_ops_list = program_block_ops_list[0]["backward"] delete_same_ops(program.global_block(), bp_ops_list) delete_trainer_useless_var(config, program, static_var) - backward_block = create_backward_block(program, origin_program, config, - bp_ops_list, block_var_detail) + backward_block = create_backward_block( + program, origin_program, config, bp_ops_list, block_var_detail + ) bp_entrance_vars = block_var_detail[0]["backward"]["entrance"] - backward_comm_info = get_communicate_var_info(origin_program, - 1, - bp_entrance_vars, - type="backward") - - grad_to_block_id.append(backward_comm_info["block_input_var_name"] + ":" + - str(backward_block.idx)) + backward_comm_info = get_communicate_var_info( + origin_program, 1, bp_entrance_vars, type="backward" + ) + + grad_to_block_id.append( + backward_comm_info["block_input_var_name"] + + ":" + + str(backward_block.idx) + ) optimizer_block.append(backward_block) attrs = { - #"mode": "sync", - #"trainers": config.get_trainers(), - #"trainer_id": config.get_role_id(), + # "mode": "sync", + # "trainers": config.get_trainers(), + # "trainer_id": config.get_role_id(), "message_to_block_id": grad_to_block_id, "optimize_blocks": optimizer_block, # runtime attribute @@ -1152,68 +1223,74 @@ def create_trainer_program(program, origin_program, config, "pserver_id": config.get_role_id(), "distributed_mode": config.get_distributed_mode(), "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, } # append the listen_and_serv op - program.global_block()._insert_op(index=0, - type="heter_listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs=attrs) + program.global_block()._insert_op( + index=0, + type="heter_listen_and_serv", + inputs={'X': []}, + outputs={}, + attrs=attrs, + ) ## TODO add check for bp block check_op_device(program.global_block(), DEFAULT_DEVICE) -def insert_communicate_op(orign_program, - config, - heter_block, - stage_id, - first_op_index, - block_var_detail, - device, - is_forward=True): +def insert_communicate_op( + orign_program, + config, + heter_block, + stage_id, + first_op_index, + block_var_detail, + device, + is_forward=True, +): if is_forward: next_heter_worker_endpoints = config.get_next_stage_trainers() previous_heter_worker_endpoints = config.get_previous_stage_trainers() entrance_var = block_var_detail[stage_id]["forward"]["entrance"] - comm_info = get_communicate_var_info(orign_program, stage_id + 1, - entrance_var) + comm_info = get_communicate_var_info( + orign_program, stage_id + 1, entrance_var + ) else: next_heter_worker_endpoints = config.get_next_stage_trainers() - #if next_heter_worker_endpoints == "": + # if next_heter_worker_endpoints == "": # next_heter_worker_endpoints = [] previous_heter_worker_endpoints = config.get_previous_stage_trainers() entrance_var = block_var_detail[stage_id - 1]["backward"]["exit"] - comm_info = get_communicate_var_info(orign_program, stage_id - 1, - entrance_var, "backward") - - heter_block._insert_op(index=first_op_index, - type="send_and_recv", - inputs={"X": heter_block.vars[entrance_var[0]]}, - outputs={"Out": []}, - attrs={ - "mode": "forward" if is_forward else "backward", - "send_var_name": - entrance_var + ["microbatch_id"], - "recv_var_name": [], - "message_name": - comm_info["block_input_var_name"], - "next_endpoints": next_heter_worker_endpoints, - "previous_endpoints": - previous_heter_worker_endpoints, - "trainer_id": config.get_role_id(), - "op_device": device, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + comm_info = get_communicate_var_info( + orign_program, stage_id - 1, entrance_var, "backward" + ) + + heter_block._insert_op( + index=first_op_index, + type="send_and_recv", + inputs={"X": heter_block.vars[entrance_var[0]]}, + outputs={"Out": []}, + attrs={ + "mode": "forward" if is_forward else "backward", + "send_var_name": entrance_var + ["microbatch_id"], + "recv_var_name": [], + "message_name": comm_info["block_input_var_name"], + "next_endpoints": next_heter_worker_endpoints, + "previous_endpoints": previous_heter_worker_endpoints, + "trainer_id": config.get_role_id(), + "op_device": device, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return entrance_var -def create_backward_block(program, origin_program, config, bp_ops_list, - block_var_detail): +def create_backward_block( + program, origin_program, config, bp_ops_list, block_var_detail +): pre_block_idx = program.num_blocks - 1 heter_block = program._create_block(pre_block_idx) @@ -1222,8 +1299,10 @@ def create_backward_block(program, origin_program, config, bp_ops_list, send_varnames = op.attr('send_varnames') is_skip = False for varname in send_varnames: - if varname not in program.global_block( - ).vars and varname not in heter_block.vars: + if ( + varname not in program.global_block().vars + and varname not in heter_block.vars + ): is_skip = True break if is_skip == True: @@ -1237,8 +1316,9 @@ def create_backward_block(program, origin_program, config, bp_ops_list, return heter_block -def replace_ops_by_communicate_op(program, config, heter_block_index, ops_list, - block_var_detail): +def replace_ops_by_communicate_op( + program, config, heter_block_index, ops_list, block_var_detail +): all_op = program.global_block().ops start_op = ops_list[0] first_op_idx = -1 @@ -1256,10 +1336,12 @@ def replace_ops_by_communicate_op(program, config, heter_block_index, ops_list, next_heter_worker_endpoints = config.get_next_stage_trainers() entrance_var = block_var_detail[heter_block_index]["forward"][ - "entrance"] + "entrance" + ] - comm_info = get_communicate_var_info(program, heter_block_index + 1, - entrance_var) + comm_info = get_communicate_var_info( + program, heter_block_index + 1, entrance_var + ) program.global_block()._insert_op( index=first_op_idx, type="send_and_recv", @@ -1273,25 +1355,30 @@ def replace_ops_by_communicate_op(program, config, heter_block_index, ops_list, "next_endpoints": next_heter_worker_endpoints, "previous_endpoints": [], "trainer_id": config.get_role_id(), - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return entrance_var -def remove_trainer_send_op(program, config, heter_block_index, - block_var_detail): +def remove_trainer_send_op( + program, config, heter_block_index, block_var_detail +): # if trainer do FF->BP->SEND, it has follow vars: var, var@GRAD # if trainer only do SEND, it has one var: var@GRAD # Delete Send op ,if trainer doesn't has pair var (var<->var@GRAD) - persistables = block_var_detail[heter_block_index]["forward"]["persistables"] + \ - block_var_detail[heter_block_index]["backward"]["persistables"] + persistables = ( + block_var_detail[heter_block_index]["forward"]["persistables"] + + block_var_detail[heter_block_index]["backward"]["persistables"] + ) need_remove_send_op = [] need_remove_grad_var = [] for op in find_send_op(program): - input_list, _ = find_op_input_output(program, program.global_block(), - op) + input_list, _ = find_op_input_output( + program, program.global_block(), op + ) for var_name in input_list: origin_var_name = var_name.split("@GRAD")[0] if origin_var_name in persistables: @@ -1304,13 +1391,13 @@ def remove_trainer_send_op(program, config, heter_block_index, def add_heter_send_op(program, heter_program, block, block_var_detail): - def _get_send_op_dict(): send_op_dict = {} send_op_list = find_send_op(program) for op in send_op_list: - input_list, _ = find_op_input_output(program, - program.global_block(), op) + input_list, _ = find_op_input_output( + program, program.global_block(), op + ) for var in input_list: send_op_dict[var] = op return send_op_dict @@ -1345,21 +1432,23 @@ def _get_send_op_dict(): for table_id in table_dict: dummy_output = block.create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) send_input_vars = [ block.vars[union_var] for union_var in table_dict[table_id]['var_list'] ] - block.append_op(type="send", - inputs={"X": send_input_vars}, - outputs={"Out": dummy_output}, - attrs={ - "send_varnames": - table_dict[table_id]['send_varnames'], - "is_sparse": is_sparse, - "table_id": table_id, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + block.append_op( + type="send", + inputs={"X": send_input_vars}, + outputs={"Out": dummy_output}, + attrs={ + "send_varnames": table_dict[table_id]['send_varnames'], + "is_sparse": is_sparse, + "table_id": table_id, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) return send_grad_var_list @@ -1372,19 +1461,20 @@ def find_send_op(program): return send_op_list -def get_communicate_var_info(program, - block_index, - entrance_var_list, - type="forward"): +def get_communicate_var_info( + program, block_index, entrance_var_list, type="forward" +): input_var_reshape_dim = [] input_var_reshape_name = [] if type == "forward": block_input_var_name = "forward_joint_{}_{}@Heter".format( - block_index - 1, block_index) + block_index - 1, block_index + ) else: block_input_var_name = "backward_joint_{}_{}@Heter".format( - block_index + 1, block_index) + block_index + 1, block_index + ) entrance_var_list.sort() # input @@ -1402,7 +1492,7 @@ def get_communicate_var_info(program, # output # var -> reshape -> var@Heter_SERVER_BLOCK@INPUT_RESHAPE_VAR -> concat -> Heter_SERVER_BLOCK_index@JOINT_VAR - #for var_name in exit_var_list: + # for var_name in exit_var_list: # var = program.global_block().vars[var_name] # shape = var.shape # # if len(shape) < 2 or shape[0] != -1: @@ -1473,11 +1563,14 @@ def union_forward_gradient_op(program_block_ops_list): ''' union_program_block_ops_list = [] - assert block_length % 2 != 0, "the length of program_block_ops_list should be odd" + assert ( + block_length % 2 != 0 + ), "the length of program_block_ops_list should be odd" for i in range(0, block_length // 2): block_op_list = {"forward": program_block_ops_list[i]} block_op_list.update( - {"backward": program_block_ops_list[block_length - 1 - i]}) + {"backward": program_block_ops_list[block_length - 1 - i]} + ) union_program_block_ops_list.append(block_op_list) block_op_list = {"forward": [], "backward": []} @@ -1491,13 +1584,15 @@ def union_forward_gradient_op(program_block_ops_list): def find_block_joints(program, program_block_ops_list, heter_ops): - block_var_detail = find_entrance_exit_private(program, - program_block_ops_list) - block_var_detail = entrance_exit_check(program, program_block_ops_list, - block_var_detail, heter_ops) - block_var_detail = delete_block_useless_exit(program, - program_block_ops_list, - block_var_detail) + block_var_detail = find_entrance_exit_private( + program, program_block_ops_list + ) + block_var_detail = entrance_exit_check( + program, program_block_ops_list, block_var_detail, heter_ops + ) + block_var_detail = delete_block_useless_exit( + program, program_block_ops_list, block_var_detail + ) return block_var_detail @@ -1508,9 +1603,11 @@ def find_entrance_exit_private(program, program_block_ops_list): for index, block_op_list in enumerate(program_block_ops_list): ## forward block_input, block_output = find_ops_list_input_output( - program, block_op_list["forward"]) + program, block_op_list["forward"] + ) persistables = screen_persistables( - program, block_input) + screen_persistables(program, block_output) + program, block_input + ) + screen_persistables(program, block_output) # find entrance & exit block_private_vars = list(set(block_input) & set(block_output)) block_entrance = list(set(block_input) - set(block_private_vars)) @@ -1520,35 +1617,40 @@ def find_entrance_exit_private(program, program_block_ops_list): "entrance": block_entrance, "exit": block_exit, "private": block_private_vars, - "persistables": persistables + "persistables": persistables, } } ## backward bp_block_input, bp_block_output = find_ops_list_input_output( - program, block_op_list["backward"]) + program, block_op_list["backward"] + ) bp_persistables = screen_persistables( - program, bp_block_input) + screen_persistables( - program, bp_block_output) + program, bp_block_input + ) + screen_persistables(program, bp_block_output) # find entrance & exit bp_block_private_vars = list(set(bp_block_input) & set(bp_block_output)) bp_block_entrance = list( - set(bp_block_input) - set(bp_block_private_vars)) + set(bp_block_input) - set(bp_block_private_vars) + ) bp_block_exit = list(set(bp_block_output) - set(bp_block_private_vars)) - detail.update({ - "backward": { - "entrance": bp_block_entrance, - "exit": bp_block_exit, - "private": bp_block_private_vars, - "persistables": bp_persistables + detail.update( + { + "backward": { + "entrance": bp_block_entrance, + "exit": bp_block_exit, + "private": bp_block_private_vars, + "persistables": bp_persistables, + } } - }) + ) block_var_detail.append(detail) return block_var_detail -def entrance_exit_check(program, program_block_ops_list, block_var_detail, - heter_ops): +def entrance_exit_check( + program, program_block_ops_list, block_var_detail, heter_ops +): for index in range(len(block_var_detail) - 1, -1, -1): if index - 1 < 0: break @@ -1558,9 +1660,11 @@ def entrance_exit_check(program, program_block_ops_list, block_var_detail, backward_entrance = block_var_detail[index]["backward"]["entrance"] - forward_all = block_var_detail[index]["forward"][ - "entrance"] + block_var_detail[index]["forward"][ - "private"] + block_var_detail[index]["forward"]["exit"] + forward_all = ( + block_var_detail[index]["forward"]["entrance"] + + block_var_detail[index]["forward"]["private"] + + block_var_detail[index]["forward"]["exit"] + ) for var in backward_entrance: if not ("@GRAD" in var) and not (var in forward_all): @@ -1571,18 +1675,24 @@ def entrance_exit_check(program, program_block_ops_list, block_var_detail, if previous_block_exit == current_block_entrance: continue exist_vars = list( - set(previous_block_exit) & set(current_block_entrance)) + set(previous_block_exit) & set(current_block_entrance) + ) need_add_vars = list(set(current_block_entrance) - set(exist_vars)) # var in different stage should not be ignored, since they are not placed in the same program & device - #need_add_vars = find_need_var_from_previous_block( + # need_add_vars = find_need_var_from_previous_block( # need_add_vars, block_var_detail, index, heter_ops) - previous_block_private = block_var_detail[index - - 1]["forward"]["private"] - previous_block_entrance = block_var_detail[index - - 1]["forward"]["entrance"] + previous_block_private = block_var_detail[index - 1]["forward"][ + "private" + ] + previous_block_entrance = block_var_detail[index - 1]["forward"][ + "entrance" + ] for var in need_add_vars: - if var not in previous_block_private and var not in previous_block_entrance: + if ( + var not in previous_block_private + and var not in previous_block_entrance + ): previous_block_entrance.append(var) previous_block_exit.append(var) if not var in current_block_entrance: @@ -1598,27 +1708,35 @@ def entrance_exit_check(program, program_block_ops_list, block_var_detail, if previous_block_exit == current_block_entrance: continue exist_vars = list( - set(previous_block_exit) & set(current_block_entrance)) + set(previous_block_exit) & set(current_block_entrance) + ) need_add_vars = list(set(current_block_entrance) - set(exist_vars)) need_ignore_vars = [] for var in need_add_vars: if not "@GRAD" in var: need_ignore_vars.append(var) need_add_vars = list( - set(need_add_vars).difference(set(need_ignore_vars))) - previous_block_private = block_var_detail[index + - 1]["backward"]["private"] - previous_block_entrance = block_var_detail[index + - 1]["backward"]["entrance"] + set(need_add_vars).difference(set(need_ignore_vars)) + ) + previous_block_private = block_var_detail[index + 1]["backward"][ + "private" + ] + previous_block_entrance = block_var_detail[index + 1]["backward"][ + "entrance" + ] for var in need_add_vars: - if var not in previous_block_private and var not in previous_block_entrance: + if ( + var not in previous_block_private + and var not in previous_block_entrance + ): previous_block_entrance.append(var) previous_block_exit.append(var) return block_var_detail -def find_need_var_from_previous_block(need_add_vars, block_var_detail, - current_index, heter_ops): +def find_need_var_from_previous_block( + need_add_vars, block_var_detail, current_index, heter_ops +): # create index_device_map index_device_map = {} for index in range(len(block_var_detail)): @@ -1633,15 +1751,21 @@ def find_need_var_from_previous_block(need_add_vars, block_var_detail, # if need_add_var in current device, no need communicate for var in need_add_vars: - while (pre_index >= 0): + while pre_index >= 0: previous_block_private = block_var_detail[pre_index]["private"] previous_block_exit = block_var_detail[pre_index]["exit"] previous_block_entrance = block_var_detail[pre_index]["entrance"] - total_var = previous_block_private + previous_block_exit + previous_block_entrance + total_var = ( + previous_block_private + + previous_block_exit + + previous_block_entrance + ) if var in total_var: - if index_device_map[current_index] == index_device_map[ - pre_index] and index_device_map[ - current_index] == DEFAULT_DEVICE: + if ( + index_device_map[current_index] + == index_device_map[pre_index] + and index_device_map[current_index] == DEFAULT_DEVICE + ): need_ignore_var.append(var) break pre_index -= 1 @@ -1650,8 +1774,9 @@ def find_need_var_from_previous_block(need_add_vars, block_var_detail, return need_add_vars -def delete_block_useless_exit(program, program_block_ops_list, - block_var_detail): +def delete_block_useless_exit( + program, program_block_ops_list, block_var_detail +): ## forward for index in range(len(block_var_detail)): if index == len(block_var_detail) - 1: @@ -1670,8 +1795,9 @@ def delete_block_useless_exit(program, program_block_ops_list, if index - 1 < 0: break current_block_exit = block_var_detail[index]["backward"]["exit"] - next_block_entrance = block_var_detail[index - - 1]["backward"]["entrance"] + next_block_entrance = block_var_detail[index - 1]["backward"][ + "entrance" + ] need_delete_var = [] for var in current_block_exit: if var not in next_block_entrance: @@ -1706,71 +1832,82 @@ def screen_persistables(program, var_list): return need_remove -def insert_reshape_op(program, - block, - index, - var_name, - new_var_name, - new_var_shape=None): +def insert_reshape_op( + program, block, index, var_name, new_var_name, new_var_shape=None +): input_var = block.vars[var_name] if new_var_name not in block.vars: - out = block.create_var(name=new_var_name, - shape=new_var_shape, - dtype=input_var.dtype, - type=input_var.type) + out = block.create_var( + name=new_var_name, + shape=new_var_shape, + dtype=input_var.dtype, + type=input_var.type, + ) else: out = block.vars[new_var_name] new_var_shape = out.shape - x_shape = block.create_var(name="{}.xshape@Heter".format(var_name), - dtype=input_var.dtype) - block._insert_op(index=index, - type="reshape2", - inputs={"X": input_var}, - attrs={'shape': new_var_shape}, - outputs={ - "Out": out, - "XShape": x_shape - }) - - -def insert_send_concat_op(program, block, index, var_name_list, new_var_name, - new_var_shape): + x_shape = block.create_var( + name="{}.xshape@Heter".format(var_name), dtype=input_var.dtype + ) + block._insert_op( + index=index, + type="reshape2", + inputs={"X": input_var}, + attrs={'shape': new_var_shape}, + outputs={"Out": out, "XShape": x_shape}, + ) + + +def insert_send_concat_op( + program, block, index, var_name_list, new_var_name, new_var_shape +): input_var_list = [block.vars[var_name] for var_name in var_name_list] - out = program.global_block().create_var(name=new_var_name, - shape=new_var_shape, - dtype=input_var_list[0].dtype, - type=input_var_list[0].type) - - block._insert_op(index=index, - type='concat', - inputs={"X": input_var_list}, - outputs={'Out': [out]}, - attrs={ - 'axis': -1, - 'use_stack': False - }) - - -def insert_recv_slice_op(program, block, index, var_name, var_shape, dtype, - type, new_var_name_list, new_var_shape_list): + out = program.global_block().create_var( + name=new_var_name, + shape=new_var_shape, + dtype=input_var_list[0].dtype, + type=input_var_list[0].type, + ) + + block._insert_op( + index=index, + type='concat', + inputs={"X": input_var_list}, + outputs={'Out': [out]}, + attrs={'axis': -1, 'use_stack': False}, + ) + + +def insert_recv_slice_op( + program, + block, + index, + var_name, + var_shape, + dtype, + type, + new_var_name_list, + new_var_shape_list, +): if var_name not in program.global_block().vars: - input_var = program.global_block().create_var(name=var_name, - shape=var_shape, - dtype=dtype, - type=type) + input_var = program.global_block().create_var( + name=var_name, shape=var_shape, dtype=dtype, type=type + ) else: input_var = program.global_block().vars[var_name] out_list = [] for i in range(len(new_var_name_list)): if new_var_name_list[i] not in block.vars: - out = block.create_var(name=new_var_name_list[i], - shape=new_var_shape_list[i], - dtype=input_var.dtype, - type=input_var.type) + out = block.create_var( + name=new_var_name_list[i], + shape=new_var_shape_list[i], + dtype=input_var.dtype, + type=input_var.type, + ) else: out = block.vars[new_var_name_list[i]] out_list.append(out) @@ -1787,25 +1924,31 @@ def insert_recv_slice_op(program, block, index, var_name, var_shape, dtype, attrs['starts'] = starts attrs['ends'] = ends - block._insert_op(index=index, - type='slice', - inputs={'Input': input_var}, - attrs=attrs, - outputs={'Out': out_list[i]}) + block._insert_op( + index=index, + type='slice', + inputs={'Input': input_var}, + attrs=attrs, + outputs={'Out': out_list[i]}, + ) start_index = end_index index += 1 -def add_heter_trainer_useful_vars(config, program, heter_program, heter_block, - static_var): +def add_heter_trainer_useful_vars( + config, program, heter_program, heter_block, static_var +): static_var = list(set(static_var)) for var_name in static_var: - if var_name not in heter_program.global_block( - ).vars and var_name not in heter_block.vars: + if ( + var_name not in heter_program.global_block().vars + and var_name not in heter_block.vars + ): var = program.global_block().vars[var_name] if var.persistable: heter_program.global_block()._clone_variable( - var, force_persistable=False) + var, force_persistable=False + ) else: heter_block._clone_variable(var, force_persistable=False) @@ -1815,14 +1958,18 @@ def delete_trainer_useless_var(config, program, static_var): program_useful_var_list = [] for op in program.global_block().ops: input_var_list, output_var_list = find_op_input_output( - program, program.global_block(), op) + program, program.global_block(), op + ) op_var_list = list(set(input_var_list).union(set(output_var_list))) program_useful_var_list = list( - set(program_useful_var_list).union(set(op_var_list))) + set(program_useful_var_list).union(set(op_var_list)) + ) program_useful_var_list += static_var program_useless_var_list = list( set(get_vars_name_in_block(program.global_block())).difference( - set(program_useful_var_list))) + set(program_useful_var_list) + ) + ) for var in program_useless_var_list: program.global_block()._remove_var(var) return program_useless_var_list @@ -1837,11 +1984,14 @@ def block_append_op(program, origin_program, block, op): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: - if var.name not in program.global_block( - ).vars and var.name not in block.vars: + if ( + var.name not in program.global_block().vars + and var.name not in block.vars + ): if var.persistable: program.global_block()._clone_variable( - var, force_persistable=False) + var, force_persistable=False + ) else: block._clone_variable(var, force_persistable=False) @@ -1850,20 +2000,22 @@ def block_append_op(program, origin_program, block, op): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: - if var.name not in program.global_block( - ).vars and var.name not in block.vars: + if ( + var.name not in program.global_block().vars + and var.name not in block.vars + ): if var.persistable: program.global_block()._clone_variable( - var, force_persistable=False) + var, force_persistable=False + ) else: block._clone_variable(var, force_persistable=False) if "_grad" not in op.type: # for forward op - return block.append_op(type=op.type, - inputs=inputs, - outputs=outputs, - attrs=op.all_attrs()) + return block.append_op( + type=op.type, inputs=inputs, outputs=outputs, attrs=op.all_attrs() + ) else: # for grad op op_desc = op.desc @@ -1885,12 +2037,15 @@ def block_append_op(program, origin_program, block, op): def add_vars_by_var_list(var_name_list, origin_program, program, block): for var_name in var_name_list: - if var_name not in program.global_block( - ).vars and var_name not in block.vars: + if ( + var_name not in program.global_block().vars + and var_name not in block.vars + ): var = origin_program.global_block().vars[var_name] if var.persistable: - program.global_block()._clone_variable(var, - force_persistable=False) + program.global_block()._clone_variable( + var, force_persistable=False + ) else: block._clone_variable(var, force_persistable=False) diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/ir/ufind.py b/python/paddle/fluid/incubate/fleet/parameter_server/ir/ufind.py index 0e30d0e3f9c57..fef6f24570c17 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/ir/ufind.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/ir/ufind.py @@ -14,7 +14,7 @@ class UnionFind(object): - """ Union-find data structure. + """Union-find data structure. Union-find is a data structure that keeps track of a set of elements partitioned into a number of disjoint (non-overlapping) subsets. diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/ir/vars_metatools.py b/python/paddle/fluid/incubate/fleet/parameter_server/ir/vars_metatools.py index b1beecd66d5fa..745c05d986a6e 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/ir/vars_metatools.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/ir/vars_metatools.py @@ -29,7 +29,6 @@ class VarBlock: - def __init__(self, varname, offset, size): self.varname = varname # NOTE: real offset is offset * size @@ -48,8 +47,9 @@ def create_var_struct(var): else: raise ValueError("can only support SELECTED_ROWS/LOD_TENSOR now") - return VarStruct(var.name, var.shape, var.dtype, var.type, lod_level, - var.persistable) + return VarStruct( + var.name, var.shape, var.dtype, var.type, lod_level, var.persistable + ) class VarStruct(object): @@ -70,8 +70,14 @@ def __init__(self, name, shape, dtype, type, lod_level, persistable): def __str__(self): return "N: {}, S: {}, D: {}, T: {}, LL: {}, P: {}, M: {}".format( - self.name, self.shape, self.dtype, self.type, self.lod_level, - self.persistable, self.m_size) + self.name, + self.shape, + self.dtype, + self.type, + self.lod_level, + self.persistable, + self.m_size, + ) class VarDistributed(object): @@ -81,14 +87,16 @@ class VarDistributed(object): the slice var's properties, such as type/shape/offset/endpoint. """ - def __init__(self, - origin_var, - slice_var, - is_slice=None, - block_id=None, - offset=None, - vtype=None, - endpoint=None): + def __init__( + self, + origin_var, + slice_var, + is_slice=None, + block_id=None, + offset=None, + vtype=None, + endpoint=None, + ): """ Args: origin_var(Variable|VarStruct): origin var properties @@ -138,26 +146,45 @@ def equal(var1, var2): """ assert isinstance(var1, VarStruct) and isinstance(var2, VarStruct) - return var1.name == var2.name and \ - var1.type == var2.type and \ - var1.shape == var2.shape and \ - var1.dtype == var2.dtype and \ - var1.lod_level == var2.lod_level and \ - var1.persistable == var2.persistable + return ( + var1.name == var2.name + and var1.type == var2.type + and var1.shape == var2.shape + and var1.dtype == var2.dtype + and var1.lod_level == var2.lod_level + and var1.persistable == var2.persistable + ) def __str__(self): - origin_var_str = "{name} : fluid.{type}.shape{shape}.astype({dtype})". \ - format(i="{", e="}", name=self.origin.name, type=self.origin.type, - shape=self.origin.shape, dtype=self.origin.dtype) - - slice_var_str = "{name} : fluid.{type}.shape{shape}.astype({dtype})" \ - ".slice({is_slice}).block({block_id}).offset({offset})". \ - format(i="{", e="}", name=self.slice.name, type=self.slice.type, - shape=self.slice.shape, dtype=self.slice.dtype, - is_slice=self.is_slice, block_id=self.block_id, offset=self.offset) + origin_var_str = ( + "{name} : fluid.{type}.shape{shape}.astype({dtype})".format( + i="{", + e="}", + name=self.origin.name, + type=self.origin.type, + shape=self.origin.shape, + dtype=self.origin.dtype, + ) + ) + + slice_var_str = ( + "{name} : fluid.{type}.shape{shape}.astype({dtype})" + ".slice({is_slice}).block({block_id}).offset({offset})".format( + i="{", + e="}", + name=self.slice.name, + type=self.slice.type, + shape=self.slice.shape, + dtype=self.slice.dtype, + is_slice=self.is_slice, + block_id=self.block_id, + offset=self.offset, + ) + ) return "var owned: {}, origin var: ( {} ), slice var: ( {} ), endpoint: {} ".format( - self.vtype, origin_var_str, slice_var_str, self.endpoint) + self.vtype, origin_var_str, slice_var_str, self.endpoint + ) class VarsDistributed(object): @@ -171,14 +198,16 @@ class VarsDistributed(object): def __init__(self): self.distributed_vars = [] - def add_distributed_var(self, - origin_var, - slice_var, - is_slice=None, - block_id=None, - offset=None, - vtype=None, - endpoint=None): + def add_distributed_var( + self, + origin_var, + slice_var, + is_slice=None, + block_id=None, + offset=None, + vtype=None, + endpoint=None, + ): """ add distributed var in this. @@ -194,5 +223,13 @@ def add_distributed_var(self, None """ self.distributed_vars.append( - VarDistributed(origin_var, slice_var, is_slice, block_id, offset, - vtype, endpoint)) + VarDistributed( + origin_var, + slice_var, + is_slice, + block_id, + offset, + vtype, + endpoint, + ) + ) diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/mode.py b/python/paddle/fluid/incubate/fleet/parameter_server/mode.py index 0733f9b8a23e4..623e919ba35a8 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/mode.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/mode.py @@ -17,6 +17,7 @@ class PSMode: """ There are various mode for fleet, each of them is designed for different model. """ + TRANSPILER = 1 PSLIB = 2 diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py index ef6c34b23fdb7..39d6d3ff1c738 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py @@ -49,8 +49,9 @@ def init(self, role_maker=None): if isinstance(role_maker, HeterRoleMaker): self._heter_ptr = fluid.core.Heter() - def _set_client_communication_config(self, request_timeout_ms, - connect_timeout_ms, max_retry): + def _set_client_communication_config( + self, request_timeout_ms, connect_timeout_ms, max_retry + ): self._client2client_request_timeout_ms = request_timeout_ms self._client2client_connect_timeout_ms = connect_timeout_ms self._client2client_max_retry = max_retry @@ -70,33 +71,41 @@ def init_worker(self): if len(self._main_programs) == 0: raise ValueError( - "You should run DistributedOptimizer.minimize() first") + "You should run DistributedOptimizer.minimize() first" + ) if self._opt_info: if "fleet_desc" in self._opt_info: self._dist_desc_str = text_format.MessageToString( - self._opt_info["fleet_desc"]) + self._opt_info["fleet_desc"] + ) self._dist_desc = self._opt_info["fleet_desc"] else: raise Exception( - "You should run DistributedOptimizer.minimize() first") + "You should run DistributedOptimizer.minimize() first" + ) # barrier_all for init_server, wait for server starts if isinstance(self._role_maker, HeterRoleMaker): if self._role_maker.is_xpu(): local_endpoint = self._role_maker.get_local_endpoint() local_endpoint = local_endpoint.split(":") - self._heter_ptr.start_xpu_service(str(local_endpoint[0]), - int(local_endpoint[1])) + self._heter_ptr.start_xpu_service( + str(local_endpoint[0]), int(local_endpoint[1]) + ) self._role_maker._barrier_all() self.all_ips_ = self._role_maker._all_gather(self._local_ip) # worker_index * 2 is for compatible with older versions of pslib - self._fleet_ptr.init_worker(self._dist_desc_str, self.all_ips_, - self._role_maker._get_size(), - self._role_maker.worker_index() * 2) + self._fleet_ptr.init_worker( + self._dist_desc_str, + self.all_ips_, + self._role_maker._get_size(), + self._role_maker.worker_index() * 2, + ) if isinstance(self._role_maker, HeterRoleMaker): if self._role_maker.is_worker(): self._heter_ptr.set_xpu_list( - self._role_maker._xpu_endpoints) + self._role_maker._xpu_endpoints + ) self._heter_ptr.create_client2xpu_connection() # barrier_all for init_worker self._role_maker._barrier_all() @@ -110,7 +119,8 @@ def init_worker(self): self._fleet_ptr.set_client2client_config( self._client2client_request_timeout_ms, self._client2client_connect_timeout_ms, - self._client2client_max_retry) + self._client2client_max_retry, + ) self._fleet_ptr.create_client2client_connection() # barrier for init model self._role_maker._barrier_worker() @@ -136,19 +146,22 @@ def init_worker(self): var_name = table.dense_variable_name[i] if scope.find_var(var_name) is None: raise ValueError( - "var " + var_name + - " not found in scope, " + - "you should run startup program first") + "var " + + var_name + + " not found in scope, " + + "you should run startup program first" + ) var_name_list.append(var_name) if not self._opt_info["use_ps_gpu"]: - self._fleet_ptr.init_model(scope, - int(table.table_id), - var_name_list) + self._fleet_ptr.init_model( + scope, int(table.table_id), var_name_list + ) # barrier for init model done self._role_maker._barrier_worker() else: raise NameError( - "You should run DistributedOptimizer.minimize() first") + "You should run DistributedOptimizer.minimize() first" + ) def init_server(self, model_dir=None, **kwargs): """ @@ -177,83 +190,102 @@ def init_server(self, model_dir=None, **kwargs): def run_server(self): """ - init_pserver(): will be called by user. When a user knows current process is_worker(), he/she - should call init_pserver() to initialize global information about parameter server + init_pserver(): will be called by user. When a user knows current process is_worker(), he/she + should call init_pserver() to initialize global information about parameter server """ if self._opt_info: if "fleet_desc" in self._opt_info: self._dist_desc_str = text_format.MessageToString( - self._opt_info["fleet_desc"]) + self._opt_info["fleet_desc"] + ) self._dist_desc = self._opt_info["fleet_desc"] else: raise Exception( - "You should run DistributedOptimizer.minimize() first") + "You should run DistributedOptimizer.minimize() first" + ) # server_index * 2 is for compatible with older versions of pslib - self._fleet_ptr.init_server(self._dist_desc_str, - self._role_maker.server_index() * 2) + self._fleet_ptr.init_server( + self._dist_desc_str, self._role_maker.server_index() * 2 + ) if isinstance(self._role_maker, MPISymetricRoleMaker): self._local_ip = self._fleet_ptr.run_server() else: local_endpoint = self._role_maker.get_local_endpoint() local_endpoint = local_endpoint.split(":") self._local_ip = self._fleet_ptr.run_server( - str(local_endpoint[0]), int(local_endpoint[1])) + str(local_endpoint[0]), int(local_endpoint[1]) + ) # barrier_all for init_server self._role_maker._barrier_all() self.all_ips_ = self._role_maker._all_gather(self._local_ip) - self._fleet_ptr.gather_servers(self.all_ips_, - self._role_maker._get_size()) + self._fleet_ptr.gather_servers( + self.all_ips_, self._role_maker._get_size() + ) # barrier_all for init_worker, wait all workers start self._role_maker._barrier_all() else: raise Exception( - "You should run DistributedOptimizer.minimize() first") + "You should run DistributedOptimizer.minimize() first" + ) def end_pass(self, scope): if self._role_maker.worker_index() < self._role_maker.xpu_num(): self._heter_ptr.end_pass(scope, self._role_maker.worker_index()) self._heter_ptr.stop_xpu_service(self._role_maker.worker_index()) - def train_from_dataset(self, - executor, - program=None, - dataset=None, - scope=None, - thread=0, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None): - """ - - """ + def train_from_dataset( + self, + executor, + program=None, + dataset=None, + scope=None, + thread=0, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + ): + """ """ if self._role_maker.is_worker(): self._role_maker._barrier_heter() - executor.train_from_dataset(program, dataset, scope, thread, debug, - fetch_list, fetch_info, print_period, - fetch_handler) - - def start_heter_trainer(self, - executor, - program=None, - scope=None, - debug=False, - fetch_list=None, - fetch_info=None, - print_period=100, - fetch_handler=None): - """ - - """ - - trainer_instance = executor.start_heter_trainer(program, scope, debug, - fetch_list, fetch_info, - print_period, - fetch_handler) + executor.train_from_dataset( + program, + dataset, + scope, + thread, + debug, + fetch_list, + fetch_info, + print_period, + fetch_handler, + ) + + def start_heter_trainer( + self, + executor, + program=None, + scope=None, + debug=False, + fetch_list=None, + fetch_info=None, + print_period=100, + fetch_handler=None, + ): + """ """ + + trainer_instance = executor.start_heter_trainer( + program, + scope, + debug, + fetch_list, + fetch_info, + print_period, + fetch_handler, + ) if self._role_maker.is_xpu(): print("barrier heter") self._role_maker._barrier_heter() @@ -293,13 +325,15 @@ def distributed_optimizer(self, optimizer, strategy={}): self._optimizer = DownpourOptimizer(optimizer, strategy) return self._optimizer - def save_inference_model(self, - executor, - dirname, - feeded_var_names=None, - target_vars=None, - main_program=None, - export_for_deployment=True): + def save_inference_model( + self, + executor, + dirname, + feeded_var_names=None, + target_vars=None, + main_program=None, + export_for_deployment=True, + ): """ save pserver model called from a worker Args: @@ -369,12 +403,9 @@ def save_persistables(self, executor, dirname, main_program=None, **kwargs): self._fleet_ptr.save_model(dirname, mode) self._role_maker._barrier_worker() - def save_model_with_whitelist(self, - executor, - dirname, - whitelist_path, - main_program=None, - **kwargs): + def save_model_with_whitelist( + self, executor, dirname, whitelist_path, main_program=None, **kwargs + ): """ save whitelist, mode is consistent with fleet.save_persistables, when using fleet, it will save sparse and dense feature @@ -400,8 +431,9 @@ def save_model_with_whitelist(self, self._fleet_ptr.client_flush() self._role_maker._barrier_worker() if self._role_maker.is_first_worker(): - self._fleet_ptr.save_model_with_whitelist(table_id, dirname, mode, - whitelist_path) + self._fleet_ptr.save_model_with_whitelist( + table_id, dirname, mode, whitelist_path + ) self._role_maker._barrier_worker() def save_multi_table_one_path(self, table_ids, model_dir, **kwargs): @@ -422,8 +454,9 @@ def save_multi_table_one_path(self, table_ids, model_dir, **kwargs): mode = kwargs.get("mode", 0) self._role_maker._barrier_worker() if self._role_maker.is_first_worker(): - self._fleet_ptr.save_multi_table_one_path(table_ids, model_dir, - mode) + self._fleet_ptr.save_multi_table_one_path( + table_ids, model_dir, mode + ) self._role_maker._barrier_worker() def save_cache_model(self, executor, dirname, main_program=None, **kwargs): @@ -452,12 +485,13 @@ def save_cache_model(self, executor, dirname, main_program=None, **kwargs): if self._role_maker.is_first_worker(): cache_threshold = self._fleet_ptr.get_cache_threshold(table_id) - #check cache threshold right or not + # check cache threshold right or not self._role_maker._barrier_worker() if self._role_maker.is_first_worker(): - self._fleet_ptr.cache_shuffle(table_id, dirname, mode, - cache_threshold) + self._fleet_ptr.cache_shuffle( + table_id, dirname, mode, cache_threshold + ) self._role_maker._barrier_worker() @@ -516,8 +550,9 @@ def shrink_dense_table(self, decay, emb_dim=11, scope=None, table_id=None): break if skip: continue - self._fleet_ptr.shrink_dense_table(i.table_id, scope, - var_list, decay, emb_dim) + self._fleet_ptr.shrink_dense_table( + i.table_id, scope, var_list, decay, emb_dim + ) self._role_maker._barrier_worker() def clear_one_table(self, table_id): @@ -587,8 +622,9 @@ def load_pslib_whitelist(self, table_id, model_path, **kwargs): self._role_maker._barrier_worker() mode = kwargs.get("mode", 0) if self._role_maker.is_first_worker(): - self._fleet_ptr.load_table_with_whitelist(table_id, model_path, - mode) + self._fleet_ptr.load_table_with_whitelist( + table_id, model_path, mode + ) self._role_maker._barrier_worker() def load_one_table(self, table_id, model_path, **kwargs): @@ -631,20 +667,27 @@ def load_one_table(self, table_id, model_path, **kwargs): load_combine = kwargs.get("load_combine", False) self._role_maker._barrier_worker() if scope is not None and model_proto_file is not None: - self._load_one_table_from_paddle_model(scope, table_id, model_path, - model_proto_file, var_names, - load_combine) + self._load_one_table_from_paddle_model( + scope, + table_id, + model_path, + model_proto_file, + var_names, + load_combine, + ) elif self._role_maker.is_first_worker(): self._fleet_ptr.load_model_one_table(table_id, model_path, mode) self._role_maker._barrier_worker() - def _load_one_table_from_paddle_model(self, - scope, - table_id, - model_path, - model_proto_file, - var_names=None, - load_combine=False): + def _load_one_table_from_paddle_model( + self, + scope, + table_id, + model_path, + model_proto_file, + var_names=None, + load_combine=False, + ): """ load params from paddle model, and push params to pserver Args: @@ -660,26 +703,46 @@ def _load_one_table_from_paddle_model(self, if self._role_maker.is_first_worker(): # get fs config from fleet_desc fs_name = self._opt_info["fleet_desc"].fs_client_param.uri - fs_ugi = self._opt_info["fleet_desc"].fs_client_param.user + "," + \ - self._opt_info["fleet_desc"].fs_client_param.passwd + fs_ugi = ( + self._opt_info["fleet_desc"].fs_client_param.user + + "," + + self._opt_info["fleet_desc"].fs_client_param.passwd + ) hadoop_bin = self._opt_info["fleet_desc"].fs_client_param.hadoop_bin # download model_path if it's hdfs/afs if model_path.startswith("hdfs:") or model_path.startswith("afs:"): dest = "./model_for_load_table_%s" % table_id - cmd = hadoop_bin + " fs -D fs.default.name=" + fs_name + \ - " -D hadoop.job.ugi=" + fs_ugi + " -get " + model_path + \ - " " + dest + cmd = ( + hadoop_bin + + " fs -D fs.default.name=" + + fs_name + + " -D hadoop.job.ugi=" + + fs_ugi + + " -get " + + model_path + + " " + + dest + ) ret = os.system(cmd) if ret != 0: raise RuntimeError("download model failed") model_path = dest # download model_proto_file if it's hdfs/afs - if model_proto_file.startswith("hdfs:") or \ - model_proto_file.startswith("afs:"): + if model_proto_file.startswith( + "hdfs:" + ) or model_proto_file.startswith("afs:"): dest = "./model_proto_file_for_load_table_%s" % table_id - cmd = hadoop_bin + " fs -D fs.default.name=" + fs_name + \ - " -D hadoop.job.ugi=" + fs_ugi + " -get " + \ - model_proto_file + " " + dest + cmd = ( + hadoop_bin + + " fs -D fs.default.name=" + + fs_name + + " -D hadoop.job.ugi=" + + fs_ugi + + " -get " + + model_proto_file + + " " + + dest + ) ret = os.system(cmd) if ret != 0: raise RuntimeError("download model proto file failed") @@ -697,8 +760,14 @@ def _load_one_table_from_paddle_model(self, if skip: continue self._fleet_ptr.load_from_paddle_model( - scope, table_id, var_names, model_path, - model_proto_file, table_var_names, load_combine) + scope, + table_id, + var_names, + model_path, + model_proto_file, + table_var_names, + load_combine, + ) self._role_maker._barrier_worker() def confirm(self): @@ -784,7 +853,8 @@ def save_one_table(self, table_id, model_dir, **kwargs): if self._role_maker.is_first_worker(): if prefix is not None: self._fleet_ptr.save_model_one_table_with_prefix( - table_id, model_dir, mode, prefix) + table_id, model_dir, mode, prefix + ) else: self._fleet_ptr.save_model_one_table(table_id, model_dir, mode) self._role_maker._barrier_worker() @@ -808,13 +878,15 @@ def _set_opt_info(self, opt_info): fleet = PSLib() -def _prepare_params(input, - size, - is_sparse=False, - is_distributed=False, - padding_idx=None, - param_attr=None, - dtype='float32'): +def _prepare_params( + input, + size, + is_sparse=False, + is_distributed=False, + padding_idx=None, + param_attr=None, + dtype='float32', +): """ preprocess params, this interface is not for users. Args: @@ -844,16 +916,18 @@ def _prepare_params(input, if d_size.get(name) is None: d_size[name] = size elif d_size[name] != size: - raise ValueError("embedding size error: %s vs %s" % - (size, d_size[name])) + raise ValueError( + "embedding size error: %s vs %s" % (size, d_size[name]) + ) # check embedding accessor accessor = FLEET_GLOBAL_DICT["cur_accessor"] if d_accessor.get(name) is None: d_accessor[name] = accessor elif d_accessor[name] != accessor: - raise ValueError("embedding size error: %s vs %s" % - (d_accessor[name], accessor)) + raise ValueError( + "embedding size error: %s vs %s" % (d_accessor[name], accessor) + ) # check embedding table id if d_table.get(name) is None: @@ -869,13 +943,15 @@ def _prepare_params(input, raise ValueError("dtype must be float32") -def _fleet_embedding(input, - size, - is_sparse=False, - is_distributed=False, - padding_idx=None, - param_attr=None, - dtype='float32'): +def _fleet_embedding( + input, + size, + is_sparse=False, + is_distributed=False, + padding_idx=None, + param_attr=None, + dtype='float32', +): """ add fleet embedding, this interface is not for users. Args: @@ -888,8 +964,9 @@ def _fleet_embedding(input, dtype(str): data type of output """ # check and set params - _prepare_params(input, size, is_sparse, is_distributed, padding_idx, - param_attr, dtype) + _prepare_params( + input, size, is_sparse, is_distributed, padding_idx, param_attr, dtype + ) name = param_attr.name size = size[-1] if padding_idx is None: @@ -904,16 +981,19 @@ def _fleet_embedding(input, ctr_label_name=FLEET_GLOBAL_DICT["click_name"], padding_id=padding_idx, dtype=dtype, - scale_sparse_grad=FLEET_GLOBAL_DICT["scale_sparse_grad"]) - - -def _fleet_embedding_v2(input, - size, - is_sparse=False, - is_distributed=False, - padding_idx=None, - param_attr=None, - dtype='float32'): + scale_sparse_grad=FLEET_GLOBAL_DICT["scale_sparse_grad"], + ) + + +def _fleet_embedding_v2( + input, + size, + is_sparse=False, + is_distributed=False, + padding_idx=None, + param_attr=None, + dtype='float32', +): """ add fleet embedding v2, this interface is not for users. Args: @@ -926,8 +1006,9 @@ def _fleet_embedding_v2(input, dtype(str): data type of output """ # check and set params - _prepare_params(input, size, is_sparse, is_distributed, padding_idx, - param_attr, dtype) + _prepare_params( + input, size, is_sparse, is_distributed, padding_idx, param_attr, dtype + ) name = param_attr.name size = size[-1] if padding_idx is None: @@ -942,7 +1023,8 @@ def _fleet_embedding_v2(input, ctr_label_name=FLEET_GLOBAL_DICT["click_name"], padding_id=padding_idx, dtype=dtype, - scale_sparse_grad=FLEET_GLOBAL_DICT["scale_sparse_grad"]) + scale_sparse_grad=FLEET_GLOBAL_DICT["scale_sparse_grad"], + ) class fleet_embedding(object): @@ -1012,17 +1094,20 @@ def __init__(self, optimizer, strategy=None): "Currently, distributed optimizer only support Adam" "Will config built-in adam for you." "We will support more functions in DistributedOptimizer", - sys.stderr) + sys.stderr, + ) self._optimizer_name = "DistributedAdam" self._distributed_optimizer = globals()[self._optimizer_name](optimizer) - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): """ Currently, backward function can not be called through DistributedOptimizer """ @@ -1050,7 +1135,8 @@ def get_dist_env(self): current_endpoint = '' num_trainers = 0 if os.getenv('PADDLE_TRAINER_ENDPOINTS') and os.getenv( - 'PADDLE_CURRENT_ENDPOINT'): + 'PADDLE_CURRENT_ENDPOINT' + ): trainer_endpoints = os.getenv('PADDLE_TRAINER_ENDPOINTS') current_endpoint = os.getenv('PADDLE_CURRENT_ENDPOINT') num_trainers = len(trainer_endpoints.split(',')) @@ -1059,7 +1145,7 @@ def get_dist_env(self): 'trainer_id': trainer_id, 'num_trainers': num_trainers, 'current_endpoint': current_endpoint, - 'trainer_endpoints': trainer_endpoints + 'trainer_endpoints': trainer_endpoints, } def _remove_collective_op_for_embedding(self, loss, table_name): @@ -1086,13 +1172,15 @@ def _remove_collective_op_for_embedding(self, loss, table_name): for index in need_remove_op_index: block._remove_op(index) - def minimize(self, - losses, - scopes=None, - startup_programs=None, - parameter_list=None, - no_grad_set=None, - program_mode="all_reduce"): + def minimize( + self, + losses, + scopes=None, + startup_programs=None, + parameter_list=None, + no_grad_set=None, + program_mode="all_reduce", + ): """ minimize a program through loss, loss can be a list in DistributedOptimizer. Note that in parameter server mode, a worker will not get anything about optimize_os @@ -1115,13 +1203,17 @@ def minimize(self, if not isinstance(losses, list): losses = [losses] - optimize_ops, param_grads, opt_info = \ - self._distributed_optimizer._minimize( - losses, - startup_programs, - parameter_list, - no_grad_set, - self._strategy) + ( + optimize_ops, + param_grads, + opt_info, + ) = self._distributed_optimizer._minimize( + losses, + startup_programs, + parameter_list, + no_grad_set, + self._strategy, + ) opt_info["mpi_rank"] = fleet.worker_index() opt_info["mpi_size"] = fleet.worker_num() fleet._set_opt_info(opt_info) @@ -1140,13 +1232,18 @@ def minimize(self, fleet._scopes = scopes if opt_info["use_ps_gpu"]: from paddle.fluid.transpiler.collective import MultiThread + # check start program if program_mode not in [ - "all_reduce", "fuse_all_reduce", "all_gather", - "all_reduce_xpu" + "all_reduce", + "fuse_all_reduce", + "all_gather", + "all_reduce_xpu", ]: - raise ValueError("You should set program_mode in [ all_reduce, \ - fuse_all_reduce, all_gather, all_reduce_xpu ]") + raise ValueError( + "You should set program_mode in [ all_reduce, \ + fuse_all_reduce, all_gather, all_reduce_xpu ]" + ) env = self.get_dist_env() if not isinstance(losses, list): startup_programs = [startup_programs] @@ -1155,19 +1252,23 @@ def minimize(self, t = MultiThread(trans_mode=program_mode) start_program = startup_programs[i] main_program = programs[i] - t.transpile(startup_program=start_program, - main_program=main_program, - rank=env["trainer_id"], - endpoints=env["trainer_endpoints"], - current_endpoint=env['current_endpoint'], - wait_port=False) + t.transpile( + startup_program=start_program, + main_program=main_program, + rank=env["trainer_id"], + endpoints=env["trainer_endpoints"], + current_endpoint=env['current_endpoint'], + wait_port=False, + ) if i > 0: - self._remove_collective_ops(start_program, - "c_comm_init_all") + self._remove_collective_ops( + start_program, "c_comm_init_all" + ) for i in range(0, len(losses)): loss = losses[i] embedding_table = self._distributed_optimizer._find_multi_distributed_lookup_table( - [loss]) + [loss] + ) self._remove_collective_op_for_embedding(loss, embedding_table) return [optimize_ops, param_grads] diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/node.py b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/node.py index 308261cea0676..d1c9fae5ccbeb 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/node.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/node.py @@ -13,14 +13,15 @@ """Defination of Server and Worker.""" from . import ps_pb2 as pslib + # NOTE: reduce removed in fuctools in python3 from functools import reduce class Server(object): """ - A Server basic class - it's a base class, does not have implementation + A Server basic class + it's a base class, does not have implementation """ def __init__(self): @@ -29,8 +30,8 @@ def __init__(self): class Worker(object): """ - A Worker basic class. - it's a base class, does not have implementation + A Worker basic class. + it's a base class, does not have implementation """ def __init__(self): @@ -39,18 +40,24 @@ def __init__(self): class DownpourServer(Server): """ - DownpourServer class is used to generate server program_desc - Args: - server: it is pslib.ServerParameter() - Examples: - server = DownpourServer() + DownpourServer class is used to generate server program_desc + Args: + server: it is pslib.ServerParameter() + Examples: + server = DownpourServer() """ def __init__(self): self._server = pslib.ServerParameter() - self._server.downpour_server_param.service_param.server_class = "DownpourBrpcPsServer" - self._server.downpour_server_param.service_param.client_class = "DownpourBrpcPsClient" - self._server.downpour_server_param.service_param.service_class = "DownpourPsService" + self._server.downpour_server_param.service_param.server_class = ( + "DownpourBrpcPsServer" + ) + self._server.downpour_server_param.service_param.client_class = ( + "DownpourBrpcPsClient" + ) + self._server.downpour_server_param.service_param.service_class = ( + "DownpourPsService" + ) self._server.downpour_server_param.service_param.start_server_port = 0 self._server.downpour_server_param.service_param.server_thread_num = 12 @@ -68,27 +75,60 @@ def add_sparse_table(self, table_id, strategy): if table.type == pslib.PS_SPARSE_TABLE: return else: - raise ValueError("expect table %s type=%s, but actual type=%s" \ - %(table_id, pslib.PS_SPARSE_TABLE, table.type)) + raise ValueError( + "expect table %s type=%s, but actual type=%s" + % (table_id, pslib.PS_SPARSE_TABLE, table.type) + ) if strategy is None: strategy = dict() table = self._server.downpour_server_param.downpour_table_param.add() table.table_id = table_id table.type = pslib.PS_SPARSE_TABLE - support_sparse_key_list = ['sparse_table_class', 'sparse_compress_in_save', 'sparse_shard_num', \ - 'sparse_accessor_class', 'sparse_learning_rate', 'sparse_initial_g2sum', 'sparse_initial_range', \ - 'sparse_weight_bounds', 'sparse_embedx_dim', 'sparse_embedx_threshold', 'sparse_nonclk_coeff', \ - 'sparse_click_coeff', 'sparse_base_threshold', 'sparse_delta_threshold', 'sparse_delta_keep_days', \ - 'sparse_delete_after_unseen_days', 'sparse_show_click_decay_rate', 'sparse_delete_threshold', \ - 'sparse_converter', 'sparse_deconverter', 'sparse_enable_cache', 'sparse_cache_rate', \ - 'sparse_cache_file_num', 'sparse_beta1_decay_rate', 'sparse_beta2_decay_rate', \ - 'sparse_ada_epsilon', 'sparse_optimizer', 'sparse_ssd_unseenday_threshold', \ - 'embed_sparse_optimizer', 'embed_sparse_learning_rate', 'embed_sparse_weight_bounds', \ - 'embed_sparse_initial_range', 'embed_sparse_initial_g2sum', 'embed_sparse_beta1_decay_rate', \ - 'embed_sparse_beta2_decay_rate', 'embedx_sparse_optimizer', 'embedx_sparse_learning_rate', \ - 'embedx_sparse_weight_bounds', 'embedx_sparse_initial_range', 'embedx_sparse_initial_g2sum', \ - 'embedx_sparse_beta1_decay_rate', 'embedx_sparse_beta2_decay_rate'] + support_sparse_key_list = [ + 'sparse_table_class', + 'sparse_compress_in_save', + 'sparse_shard_num', + 'sparse_accessor_class', + 'sparse_learning_rate', + 'sparse_initial_g2sum', + 'sparse_initial_range', + 'sparse_weight_bounds', + 'sparse_embedx_dim', + 'sparse_embedx_threshold', + 'sparse_nonclk_coeff', + 'sparse_click_coeff', + 'sparse_base_threshold', + 'sparse_delta_threshold', + 'sparse_delta_keep_days', + 'sparse_delete_after_unseen_days', + 'sparse_show_click_decay_rate', + 'sparse_delete_threshold', + 'sparse_converter', + 'sparse_deconverter', + 'sparse_enable_cache', + 'sparse_cache_rate', + 'sparse_cache_file_num', + 'sparse_beta1_decay_rate', + 'sparse_beta2_decay_rate', + 'sparse_ada_epsilon', + 'sparse_optimizer', + 'sparse_ssd_unseenday_threshold', + 'embed_sparse_optimizer', + 'embed_sparse_learning_rate', + 'embed_sparse_weight_bounds', + 'embed_sparse_initial_range', + 'embed_sparse_initial_g2sum', + 'embed_sparse_beta1_decay_rate', + 'embed_sparse_beta2_decay_rate', + 'embedx_sparse_optimizer', + 'embedx_sparse_learning_rate', + 'embedx_sparse_weight_bounds', + 'embedx_sparse_initial_range', + 'embedx_sparse_initial_g2sum', + 'embedx_sparse_beta1_decay_rate', + 'embedx_sparse_beta2_decay_rate', + ] for key in strategy: if key not in support_sparse_key_list: @@ -100,21 +140,30 @@ def add_sparse_table(self, table_id, strategy): if table_class not in support_table_calss: raise ValueError( "support sparse_table_class: [ 'DownpourSparseTable', 'DownpourSparseSSDTable'], \ - but actual %s" % (table_class)) + but actual %s" + % (table_class) + ) else: table_class = 'DownpourSparseTable' table.table_class = table_class - if table_class == 'DownpourSparseTable' or table_class == 'DownpourSparseSSDTable': + if ( + table_class == 'DownpourSparseTable' + or table_class == 'DownpourSparseSSDTable' + ): table.enable_sparse_table_cache = strategy.get( - 'sparse_enable_cache', True) + 'sparse_enable_cache', True + ) table.sparse_table_cache_rate = strategy.get( - 'sparse_cache_rate', 0.00055) + 'sparse_cache_rate', 0.00055 + ) table.sparse_table_cache_file_num = strategy.get( - 'sparse_cache_file_num', 16) - table.compress_in_save = strategy.get('sparse_compress_in_save', - True) + 'sparse_cache_file_num', 16 + ) + table.compress_in_save = strategy.get( + 'sparse_compress_in_save', True + ) table.shard_num = strategy.get('sparse_shard_num', 1000) # DownpourFeatureValueAccessor: for ctr task, has cvm, embedding and sgd info # DownpourCtrAccessor : for ctr task, has cvm, slot, embedding and sgd info @@ -123,10 +172,13 @@ def add_sparse_table(self, table_id, strategy): # DownpourUnitAccessor : for ctr task, has cvm, slot, embedding and sgd info support_accessor_class = [ - 'DownpourFeatureValueAccessor', 'DownpourCtrAccessor', - 'DownpourCtrDymfAccessor', 'DownpourSparseValueAccessor', - 'DownpourCtrDoubleAccessor', 'DownpourUnitAccessor', - 'DownpourDoubleUnitAccessor' + 'DownpourFeatureValueAccessor', + 'DownpourCtrAccessor', + 'DownpourCtrDymfAccessor', + 'DownpourSparseValueAccessor', + 'DownpourCtrDoubleAccessor', + 'DownpourUnitAccessor', + 'DownpourDoubleUnitAccessor', ] if strategy.get('sparse_accessor_class') is not None: accessor_class = strategy.get('sparse_accessor_class') @@ -134,56 +186,76 @@ def add_sparse_table(self, table_id, strategy): raise ValueError( "support sparse_accessor_class: ['DownpourFeatureValueAccessor', 'DownpourCtrAccessor', 'DownpourCtrDymfAccessor', \ 'DownpourSparseValueAccessor', 'DownpourCtrDoubleAccessor'], \ - but actual %s" % (accessor_class)) + but actual %s" + % (accessor_class) + ) else: accessor_class = 'DownpourCtrAccessor' table.accessor.accessor_class = accessor_class - if accessor_class == 'DownpourFeatureValueAccessor' \ - or accessor_class == 'DownpourCtrAccessor' \ - or accessor_class == 'DownpourCtrDymfAccessor' \ - or accessor_class == 'DownpourCtrDoubleAccessor': + if ( + accessor_class == 'DownpourFeatureValueAccessor' + or accessor_class == 'DownpourCtrAccessor' + or accessor_class == 'DownpourCtrDymfAccessor' + or accessor_class == 'DownpourCtrDoubleAccessor' + ): table.accessor.sparse_sgd_param.learning_rate = strategy.get( - 'sparse_learning_rate', 0.05) + 'sparse_learning_rate', 0.05 + ) table.accessor.sparse_sgd_param.initial_g2sum = strategy.get( - 'sparse_initial_g2sum', 3) + 'sparse_initial_g2sum', 3 + ) table.accessor.sparse_sgd_param.initial_range = strategy.get( - 'sparse_initial_range', 1e-4) + 'sparse_initial_range', 1e-4 + ) if strategy.get('sparse_weight_bounds') is None: table.accessor.sparse_sgd_param.weight_bounds.extend( - [-10, 10]) + [-10, 10] + ) else: table.accessor.sparse_sgd_param.weight_bounds.extend( - strategy.get('sparse_weight_bounds')) + strategy.get('sparse_weight_bounds') + ) table.accessor.embedx_dim = strategy.get('sparse_embedx_dim', 8) table.accessor.embedx_threshold = strategy.get( - 'sparse_embedx_threshold', 10) + 'sparse_embedx_threshold', 10 + ) table.accessor.fea_dim = int(table.accessor.embedx_dim) + 3 - table.accessor.downpour_accessor_param.nonclk_coeff = strategy.get( - 'sparse_nonclk_coeff', 0.1) - table.accessor.downpour_accessor_param.click_coeff = strategy.get( - 'sparse_click_coeff', 1) - table.accessor.downpour_accessor_param.base_threshold = strategy.get( - 'sparse_base_threshold', 1.5) - table.accessor.downpour_accessor_param.delta_threshold = strategy.get( - 'sparse_delta_threshold', 0.25) - table.accessor.downpour_accessor_param.delta_keep_days = strategy.get( - 'sparse_delta_keep_days', 16) + table.accessor.downpour_accessor_param.nonclk_coeff = ( + strategy.get('sparse_nonclk_coeff', 0.1) + ) + table.accessor.downpour_accessor_param.click_coeff = ( + strategy.get('sparse_click_coeff', 1) + ) + table.accessor.downpour_accessor_param.base_threshold = ( + strategy.get('sparse_base_threshold', 1.5) + ) + table.accessor.downpour_accessor_param.delta_threshold = ( + strategy.get('sparse_delta_threshold', 0.25) + ) + table.accessor.downpour_accessor_param.delta_keep_days = ( + strategy.get('sparse_delta_keep_days', 16) + ) table.accessor.downpour_accessor_param.delete_after_unseen_days = strategy.get( - 'sparse_delete_after_unseen_days', 30) + 'sparse_delete_after_unseen_days', 30 + ) table.accessor.downpour_accessor_param.ssd_unseenday_threshold = strategy.get( - 'sparse_ssd_unseenday_threshold', 1) - table.accessor.downpour_accessor_param.show_click_decay_rate = strategy.get( - 'sparse_show_click_decay_rate', 0.98) - table.accessor.downpour_accessor_param.delete_threshold = strategy.get( - 'sparse_delete_threshold', 0.8) + 'sparse_ssd_unseenday_threshold', 1 + ) + table.accessor.downpour_accessor_param.show_click_decay_rate = ( + strategy.get('sparse_show_click_decay_rate', 0.98) + ) + table.accessor.downpour_accessor_param.delete_threshold = ( + strategy.get('sparse_delete_threshold', 0.8) + ) converter = strategy.get( 'sparse_converter', - "(scripts/xbox_compressor_mf.py | bin/xbox_pb_converter)") + "(scripts/xbox_compressor_mf.py | bin/xbox_pb_converter)", + ) deconverter = strategy.get( 'sparse_deconverter', - "(bin/xbox_pb_deconverter | scripts/xbox_decompressor_mf.awk)" + "(bin/xbox_pb_deconverter | scripts/xbox_decompressor_mf.awk)", ) table1 = table.accessor.table_accessor_save_param.add() @@ -201,52 +273,69 @@ def add_sparse_table(self, table_id, strategy): table.accessor.embedx_dim = strategy.get('sparse_embedx_dim', 8) table.accessor.fea_dim = int(table.accessor.embedx_dim) if optimizer_name == "naive": - table.accessor.sparse_commonsgd_param.naive.learning_rate = \ - strategy.get('sparse_learning_rate', 0.05) - table.accessor.sparse_commonsgd_param.naive.initial_range = \ - strategy.get('sparse_initial_range', 1e-4) + table.accessor.sparse_commonsgd_param.naive.learning_rate = strategy.get( + 'sparse_learning_rate', 0.05 + ) + table.accessor.sparse_commonsgd_param.naive.initial_range = strategy.get( + 'sparse_initial_range', 1e-4 + ) if strategy.get('sparse_weight_bounds') is None: table.accessor.sparse_commonsgd_param.naive.weight_bounds.extend( - [-10, 10]) + [-10, 10] + ) else: table.accessor.sparse_commonsgd_param.naive.weight_bounds.extend( - strategy.get('sparse_weight_bounds')) + strategy.get('sparse_weight_bounds') + ) elif optimizer_name == "adagrad": - table.accessor.sparse_commonsgd_param.adagrad.learning_rate = \ - strategy.get('sparse_learning_rate', 0.05) - table.accessor.sparse_commonsgd_param.adagrad.initial_range = \ - strategy.get('sparse_initial_range', 1e-4) + table.accessor.sparse_commonsgd_param.adagrad.learning_rate = strategy.get( + 'sparse_learning_rate', 0.05 + ) + table.accessor.sparse_commonsgd_param.adagrad.initial_range = strategy.get( + 'sparse_initial_range', 1e-4 + ) table.accessor.sparse_commonsgd_param.adagrad.initial_g2sum = strategy.get( - 'sparse_initial_g2sum', 3) + 'sparse_initial_g2sum', 3 + ) if strategy.get('sparse_weight_bounds') is None: table.accessor.sparse_commonsgd_param.adagrad.weight_bounds.extend( - [-10, 10]) + [-10, 10] + ) else: table.accessor.sparse_commonsgd_param.adagrad.weight_bounds.extend( - strategy.get('sparse_weight_bounds')) + strategy.get('sparse_weight_bounds') + ) elif optimizer_name == "adam": - table.accessor.sparse_commonsgd_param.adam.learning_rate = \ + table.accessor.sparse_commonsgd_param.adam.learning_rate = ( strategy.get('sparse_learning_rate', 0.001) - table.accessor.sparse_commonsgd_param.adam.initial_range = \ + ) + table.accessor.sparse_commonsgd_param.adam.initial_range = ( strategy.get('sparse_initial_range', 1e-4) + ) table.accessor.sparse_commonsgd_param.adam.beta1_decay_rate = strategy.get( - 'sparse_beta1_decay_rate', 0.9) + 'sparse_beta1_decay_rate', 0.9 + ) table.accessor.sparse_commonsgd_param.adam.beta2_decay_rate = strategy.get( - 'sparse_beta2_decay_rate', 0.999) - table.accessor.sparse_commonsgd_param.adam.ada_epsilon = strategy.get( - 'sparse_ada_epsilon', 1e-8) + 'sparse_beta2_decay_rate', 0.999 + ) + table.accessor.sparse_commonsgd_param.adam.ada_epsilon = ( + strategy.get('sparse_ada_epsilon', 1e-8) + ) if strategy.get('sparse_weight_bounds') is None: table.accessor.sparse_commonsgd_param.adam.weight_bounds.extend( - [-10, 10]) + [-10, 10] + ) else: table.accessor.sparse_commonsgd_param.adam.weight_bounds.extend( - strategy.get('sparse_weight_bounds')) + strategy.get('sparse_weight_bounds') + ) converter = strategy.get( 'sparse_converter', - "(scripts/xbox_compressor_mf.py | bin/xbox_pb_converter)") + "(scripts/xbox_compressor_mf.py | bin/xbox_pb_converter)", + ) deconverter = strategy.get( 'sparse_deconverter', - "(bin/xbox_pb_deconverter | scripts/xbox_decompressor_mf.awk)" + "(bin/xbox_pb_deconverter | scripts/xbox_decompressor_mf.awk)", ) table1 = table.accessor.table_accessor_save_param.add() @@ -258,15 +347,21 @@ def add_sparse_table(self, table_id, strategy): table2.param = 2 table2.converter = converter table2.deconverter = deconverter - elif accessor_class == 'DownpourUnitAccessor' or accessor_class == 'DownpourDoubleUnitAccessor': + elif ( + accessor_class == 'DownpourUnitAccessor' + or accessor_class == 'DownpourDoubleUnitAccessor' + ): self.add_sparse_table_common_config(table, strategy) - self.add_sparse_optimizer(table.accessor.embed_sgd_param, - strategy, "embed_") - self.add_sparse_optimizer(table.accessor.embedx_sgd_param, - strategy, "embedx_") + self.add_sparse_optimizer( + table.accessor.embed_sgd_param, strategy, "embed_" + ) + self.add_sparse_optimizer( + table.accessor.embedx_sgd_param, strategy, "embedx_" + ) - def add_dense_table(self, table_id, param_var, grad_var, strategy, - sparse_table_names): + def add_dense_table( + self, table_id, param_var, grad_var, strategy, sparse_table_names + ): """ Args: table_id(int): id of sparse params table @@ -292,45 +387,72 @@ def add_dense_table(self, table_id, param_var, grad_var, strategy, table.accessor.fea_dim = fea_dim return else: - raise ValueError("expect table %s type=%s, but actual type=%s" \ - %(table_id, pslib.PS_DENSE_TABLE, table.type)) + raise ValueError( + "expect table %s type=%s, but actual type=%s" + % (table_id, pslib.PS_DENSE_TABLE, table.type) + ) if strategy is None: strategy = dict() table = self._server.downpour_server_param.downpour_table_param.add() table.table_id = table_id - support_dense_key_list = ['dense_table_class', 'dense_compress_in_save', 'dense_accessor_class', \ - 'dense_optimizer', 'dense_learning_rate', 'dense_avg_decay', 'dense_ada_decay', \ - 'dense_ada_epsilon', 'dense_mom_decay', 'dense_naive_lr'] + support_dense_key_list = [ + 'dense_table_class', + 'dense_compress_in_save', + 'dense_accessor_class', + 'dense_optimizer', + 'dense_learning_rate', + 'dense_avg_decay', + 'dense_ada_decay', + 'dense_ada_epsilon', + 'dense_mom_decay', + 'dense_naive_lr', + ] for key in strategy: if key not in support_dense_key_list: raise ValueError("strategy key '%s' not support" % (key)) - table.table_class = strategy.get('dense_table_class', - "DownpourDenseTable") + table.table_class = strategy.get( + 'dense_table_class', "DownpourDenseTable" + ) table.type = pslib.PS_DENSE_TABLE table.compress_in_save = strategy.get('dense_compress_in_save', True) table.accessor.accessor_class = strategy.get( - 'dense_accessor_class', "DownpourDenseValueAccessor") + 'dense_accessor_class', "DownpourDenseValueAccessor" + ) table.accessor.dense_sgd_param.name = strategy.get( - 'dense_optimizer', "adam") + 'dense_optimizer', "adam" + ) table.accessor.dense_sgd_param.adam.learning_rate = strategy.get( - 'dense_learning_rate', 5e-06) + 'dense_learning_rate', 5e-06 + ) table.accessor.dense_sgd_param.adam.avg_decay_rate = strategy.get( - 'dense_avg_decay', 0.999993) + 'dense_avg_decay', 0.999993 + ) table.accessor.dense_sgd_param.adam.ada_decay_rate = strategy.get( - 'dense_ada_decay', 0.9999) + 'dense_ada_decay', 0.9999 + ) table.accessor.dense_sgd_param.adam.ada_epsilon = strategy.get( - 'dense_ada_epsilon', 1e-8) + 'dense_ada_epsilon', 1e-8 + ) table.accessor.dense_sgd_param.adam.mom_decay_rate = strategy.get( - 'dense_mom_decay', 0.99) + 'dense_mom_decay', 0.99 + ) table.accessor.dense_sgd_param.naive.learning_rate = strategy.get( - 'dense_naive_lr', 0.0002) + 'dense_naive_lr', 0.0002 + ) table.accessor.fea_dim = fea_dim - def add_data_norm_table(self, table_id, learning_rate, param_var, grad_var, - strategy, sparse_table_names): + def add_data_norm_table( + self, + table_id, + learning_rate, + param_var, + grad_var, + strategy, + sparse_table_names, + ): """ Args: table_id(int): id of datanorm table @@ -357,13 +479,20 @@ def add_data_norm_table(self, table_id, learning_rate, param_var, grad_var, table.accessor.fea_dim = fea_dim return else: - raise ValueError("expect table %s type=%s, but actual type=%s" \ - %(table_id, pslib.PS_DENSE_TABLE, table.type)) + raise ValueError( + "expect table %s type=%s, but actual type=%s" + % (table_id, pslib.PS_DENSE_TABLE, table.type) + ) if strategy is None: strategy = dict() - support_datanorm_key_list = ['datanorm_table_class', 'datanorm_compress_in_save', \ - 'datanorm_accessor_class', 'datanorm_operation', 'datanorm_decay_rate'] + support_datanorm_key_list = [ + 'datanorm_table_class', + 'datanorm_compress_in_save', + 'datanorm_accessor_class', + 'datanorm_operation', + 'datanorm_decay_rate', + ] for key in strategy: if key not in support_datanorm_key_list: @@ -371,91 +500,119 @@ def add_data_norm_table(self, table_id, learning_rate, param_var, grad_var, table = self._server.downpour_server_param.downpour_table_param.add() table.table_id = table_id - table.table_class = strategy.get('datanorm_table_class', - 'DownpourDenseTable') + table.table_class = strategy.get( + 'datanorm_table_class', 'DownpourDenseTable' + ) table.type = pslib.PS_DENSE_TABLE table.compress_in_save = strategy.get('datanorm_compress_in_save', True) table.accessor.accessor_class = strategy.get( - 'datanorm_accessor_class', 'DownpourDenseValueAccessor') + 'datanorm_accessor_class', 'DownpourDenseValueAccessor' + ) table.accessor.dense_sgd_param.name = strategy.get( - 'datanorm_operation', 'summary') - table.accessor.dense_sgd_param.summary.summary_decay_rate = strategy.get( - 'datanorm_decay_rate', 0.999999) + 'datanorm_operation', 'summary' + ) + table.accessor.dense_sgd_param.summary.summary_decay_rate = ( + strategy.get('datanorm_decay_rate', 0.999999) + ) table.accessor.fea_dim = fea_dim def add_sparse_optimizer(self, sgd, strategy, prefix): optimizer_name = strategy.get(prefix + "sparse_optimizer", "adagrad") sgd.name = optimizer_name if optimizer_name == "naive": - sgd.naive.learning_rate = \ - strategy.get(prefix + 'sparse_learning_rate', 0.05) - sgd.naive.initial_range = \ - strategy.get(prefix + 'sparse_initial_range', 1e-4) + sgd.naive.learning_rate = strategy.get( + prefix + 'sparse_learning_rate', 0.05 + ) + sgd.naive.initial_range = strategy.get( + prefix + 'sparse_initial_range', 1e-4 + ) bounds = strategy.get(prefix + 'sparse_weight_bounds', [-10, 10]) sgd.naive.weight_bounds.extend(bounds) elif optimizer_name == "adagrad": - sgd.adagrad.learning_rate = \ - strategy.get(prefix + 'sparse_learning_rate', 0.05) - sgd.adagrad.initial_range = \ - strategy.get(prefix + 'sparse_initial_range', 1e-4) + sgd.adagrad.learning_rate = strategy.get( + prefix + 'sparse_learning_rate', 0.05 + ) + sgd.adagrad.initial_range = strategy.get( + prefix + 'sparse_initial_range', 1e-4 + ) if prefix == "embed_": sgd.adagrad.initial_range = 0 sgd.adagrad.initial_g2sum = strategy.get( - prefix + 'sparse_initial_g2sum', 3) + prefix + 'sparse_initial_g2sum', 3 + ) bounds = strategy.get(prefix + 'sparse_weight_bounds', [-10, 10]) sgd.adagrad.weight_bounds.extend(bounds) elif optimizer_name == "std_adagrad": - sgd.adagrad.learning_rate = \ - strategy.get(prefix + 'sparse_learning_rate', 0.05) - sgd.adagrad.initial_range = \ - strategy.get(prefix + 'sparse_initial_range', 1e-4) + sgd.adagrad.learning_rate = strategy.get( + prefix + 'sparse_learning_rate', 0.05 + ) + sgd.adagrad.initial_range = strategy.get( + prefix + 'sparse_initial_range', 1e-4 + ) if prefix == "embed_": sgd.adagrad.initial_range = 0 sgd.adagrad.initial_g2sum = strategy.get( - prefix + 'sparse_initial_g2sum', 3) + prefix + 'sparse_initial_g2sum', 3 + ) bounds = strategy.get(prefix + 'sparse_weight_bounds', [-10, 10]) sgd.adagrad.weight_bounds.extend(bounds) elif optimizer_name == "adam": - sgd.adam.learning_rate = \ - strategy.get(prefix + 'sparse_learning_rate', 0.001) - sgd.adam.initial_range = \ - strategy.get(prefix + 'sparse_initial_range', 1e-4) + sgd.adam.learning_rate = strategy.get( + prefix + 'sparse_learning_rate', 0.001 + ) + sgd.adam.initial_range = strategy.get( + prefix + 'sparse_initial_range', 1e-4 + ) sgd.adam.beta1_decay_rate = strategy.get( - prefix + 'sparse_beta1_decay_rate', 0.9) + prefix + 'sparse_beta1_decay_rate', 0.9 + ) sgd.adam.beta2_decay_rate = strategy.get( - prefix + 'sparse_beta2_decay_rate', 0.999) - sgd.adam.ada_epsilon = strategy.get(prefix + 'sparse_ada_epsilon', - 1e-8) + prefix + 'sparse_beta2_decay_rate', 0.999 + ) + sgd.adam.ada_epsilon = strategy.get( + prefix + 'sparse_ada_epsilon', 1e-8 + ) bounds = strategy.get(prefix + 'sparse_weight_bounds', [-10, 10]) sgd.adam.weight_bounds.extend(bounds) def add_sparse_table_common_config(self, table, strategy): table.accessor.embedx_dim = strategy.get('sparse_embedx_dim', 8) table.accessor.embedx_threshold = strategy.get( - 'sparse_embedx_threshold', 10) + 'sparse_embedx_threshold', 10 + ) table.accessor.fea_dim = int(table.accessor.embedx_dim) + 3 table.accessor.downpour_accessor_param.nonclk_coeff = strategy.get( - 'sparse_nonclk_coeff', 0.1) + 'sparse_nonclk_coeff', 0.1 + ) table.accessor.downpour_accessor_param.click_coeff = strategy.get( - 'sparse_click_coeff', 1) + 'sparse_click_coeff', 1 + ) table.accessor.downpour_accessor_param.base_threshold = strategy.get( - 'sparse_base_threshold', 1.5) + 'sparse_base_threshold', 1.5 + ) table.accessor.downpour_accessor_param.delta_threshold = strategy.get( - 'sparse_delta_threshold', 0.25) + 'sparse_delta_threshold', 0.25 + ) table.accessor.downpour_accessor_param.delta_keep_days = strategy.get( - 'sparse_delta_keep_days', 16) - table.accessor.downpour_accessor_param.delete_after_unseen_days = strategy.get( - 'sparse_delete_after_unseen_days', 30) - table.accessor.downpour_accessor_param.show_click_decay_rate = strategy.get( - 'sparse_show_click_decay_rate', 0.98) + 'sparse_delta_keep_days', 16 + ) + table.accessor.downpour_accessor_param.delete_after_unseen_days = ( + strategy.get('sparse_delete_after_unseen_days', 30) + ) + table.accessor.downpour_accessor_param.show_click_decay_rate = ( + strategy.get('sparse_show_click_decay_rate', 0.98) + ) table.accessor.downpour_accessor_param.delete_threshold = strategy.get( - 'sparse_delete_threshold', 0.8) + 'sparse_delete_threshold', 0.8 + ) converter = strategy.get( 'sparse_converter', - "(scripts/xbox_compressor_mf.py | bin/xbox_pb_converter)") + "(scripts/xbox_compressor_mf.py | bin/xbox_pb_converter)", + ) deconverter = strategy.get( 'sparse_deconverter', - "(bin/xbox_pb_deconverter | scripts/xbox_decompressor_mf.awk)") + "(bin/xbox_pb_deconverter | scripts/xbox_decompressor_mf.awk)", + ) table1 = table.accessor.table_accessor_save_param.add() table1.param = 1 @@ -476,23 +633,21 @@ def get_desc(self): class DownpourWorker(Worker): """ - DownpourWorker class is used to generate worker program_desc - Args: - window (int): push params frequency - worker: it is pslib.DownpourTrainerParameter - Examples: - worker = DownpourWorker(1) + DownpourWorker class is used to generate worker program_desc + Args: + window (int): push params frequency + worker: it is pslib.DownpourTrainerParameter + Examples: + worker = DownpourWorker(1) """ def __init__(self, window): self.window = window self._worker = pslib.DownpourTrainerParameter() - def add_sparse_table(self, - table_id, - slot_key_vars, - slot_value_vars, - slot_value_grads=None): + def add_sparse_table( + self, table_id, slot_key_vars, slot_value_vars, slot_value_grads=None + ): """ Args: table_id(int): id of sparse params table @@ -503,8 +658,9 @@ def add_sparse_table(self, return None """ if slot_value_grads is None: - slot_value_grad_names = \ - [var.name + "@GRAD" for var in slot_value_vars] + slot_value_grad_names = [ + var.name + "@GRAD" for var in slot_value_vars + ] else: value_to_key = {} for i in range(len(slot_key_vars)): @@ -514,12 +670,19 @@ def add_sparse_table(self, for var in slot_value_vars: if var.name + "@GRAD" in all_grad_names: slot_value_grad_names.append(var.name + "@GRAD") - sorted_slot_value_vars = [i for i in slot_value_vars if \ - i.name + "@GRAD" in slot_value_grad_names] - sorted_slot_value_vars += [i for i in slot_value_vars if \ - i.name + "@GRAD" not in slot_value_grad_names] - sorted_slot_key_vars = \ - [value_to_key[v.name] for v in sorted_slot_value_vars] + sorted_slot_value_vars = [ + i + for i in slot_value_vars + if i.name + "@GRAD" in slot_value_grad_names + ] + sorted_slot_value_vars += [ + i + for i in slot_value_vars + if i.name + "@GRAD" not in slot_value_grad_names + ] + sorted_slot_key_vars = [ + value_to_key[v.name] for v in sorted_slot_value_vars + ] target_table = None for table in self._worker.sparse_table: @@ -528,8 +691,9 @@ def add_sparse_table(self, key_names = [var.name for var in sorted_slot_key_vars] for key_name in key_names: if key_name not in keys: - raise ValueError("sparse table %s slot_key error" % - table_id) + raise ValueError( + "sparse table %s slot_key error" % table_id + ) target_table = table break @@ -542,8 +706,15 @@ def add_sparse_table(self, table.slot_value.extend([var.name for var in sorted_slot_value_vars]) table.slot_gradient.extend(slot_value_grad_names) - def add_dense_table(self, table_id, learning_rate, param_vars, grad_vars, - dense_start_table_id, sparse_table_names): + def add_dense_table( + self, + table_id, + learning_rate, + param_vars, + grad_vars, + dense_start_table_id, + sparse_table_names, + ): r""" Args: table_id(int): id of sparse params table @@ -580,22 +751,25 @@ def add_dense_table(self, table_id, learning_rate, param_vars, grad_vars, if dense_param_name == desc_dense_param_name: desc_dense_grad_name = list( - table.dense_gradient_variable_name) + table.dense_gradient_variable_name + ) desc_dense_grad_name.sort() if dense_grad_name == desc_dense_grad_name: return else: raise ValueError( "dense table %s dense_gradient_variable_name " - "error" % table_id) + "error" % table_id + ) else: raise ValueError( - "dense table %s dense_variable_name error" % table_id) + "dense table %s dense_variable_name error" % table_id + ) table = self._worker.dense_table.add() table.table_id = table_id - #def cmp_fc(x, y): + # def cmp_fc(x, y): # if x.startswith("fc_") and y.startswith("fc_"): # index_x = x.find('.') # index_y = y.find('.') @@ -616,8 +790,8 @@ def add_dense_table(self, table_id, learning_rate, param_vars, grad_vars, # else: # return 1 - #table.dense_variable_name.extend(sorted(dense_param_name, cmp_fc)) - #table.dense_gradient_variable_name.extend( + # table.dense_variable_name.extend(sorted(dense_param_name, cmp_fc)) + # table.dense_gradient_variable_name.extend( # sorted(dense_grad_name, cmp_fc)) table.dense_variable_name.extend(dense_param_name) table.dense_gradient_variable_name.extend(dense_grad_name) diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/optimizer_factory.py b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/optimizer_factory.py index 35cda4c34b009..ed9db255146e5 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/optimizer_factory.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/optimizer_factory.py @@ -17,8 +17,12 @@ import paddle.fluid as fluid from paddle.fluid import core from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table -from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table_inputs -from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table_outputs +from paddle.fluid.distribute_lookup_table import ( + find_distributed_lookup_table_inputs, +) +from paddle.fluid.distribute_lookup_table import ( + find_distributed_lookup_table_outputs, +) from google.protobuf import text_format from collections import OrderedDict import copy @@ -61,11 +65,13 @@ def __init__(self, optimizer): self._learning_rate = optimizer._learning_rate self._regularization = optimizer.regularization - def minimize(self, - losses, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, + losses, + startup_program=None, + parameter_list=None, + no_grad_set=None, + ): """ Args: losses(Variable): loss variable defined by user @@ -90,15 +96,24 @@ def __init__(self, optimizer): self._window = 1 self.type = "downpour" self.data_norm_name = [ - ".batch_size", ".batch_square_sum", ".batch_sum", - ".batch_size@GRAD", ".batch_square_sum@GRAD", ".batch_sum@GRAD" + ".batch_size", + ".batch_square_sum", + ".batch_sum", + ".batch_size@GRAD", + ".batch_square_sum@GRAD", + ".batch_sum@GRAD", ] self.supported_embedding_types = [ - "lookup_table", "pull_sparse", "pull_sparse_v2", "pull_box_sparse", - "pull_gpups_sparse" + "lookup_table", + "pull_sparse", + "pull_sparse_v2", + "pull_box_sparse", + "pull_gpups_sparse", ] self.supported_embedding_grad_types = [ - "lookup_table_grad", "push_sparse", "push_sparse_v2" + "lookup_table_grad", + "push_sparse", + "push_sparse_v2", ] op_maker = core.op_proto_and_checker_maker self.op_role_key = op_maker.kOpRoleAttrName() @@ -122,7 +137,8 @@ def _find_distributed_lookup_table_inputs(self, program, table_names): if op.type in self.supported_embedding_types: if op.input("W")[0] in table_names: inputs_dict[op.input("W")[0]].extend( - [local_vars[name] for name in op.input("Ids")]) + [local_vars[name] for name in op.input("Ids")] + ) return inputs_dict def _find_distributed_lookup_table_outputs(self, program, table_names): @@ -144,7 +160,8 @@ def _find_distributed_lookup_table_outputs(self, program, table_names): if op.type in self.supported_embedding_types: if op.input("W")[0] in table_names: outputs_dict[op.input("W")[0]].extend( - [local_vars[name] for name in op.output("Out")]) + [local_vars[name] for name in op.output("Out")] + ) return outputs_dict def _find_distributed_lookup_table_grads(self, program, table_names): @@ -157,12 +174,14 @@ def _find_distributed_lookup_table_grads(self, program, table_names): if op.type in self.supported_embedding_grad_types: if op.input("W")[0] in table_names: grads_dict[op.input("W")[0]].extend( - [local_vars[name] for name in op.input("Out@GRAD")]) + [local_vars[name] for name in op.input("Out@GRAD")] + ) return grads_dict def _is_optimizer_op(self, op): - return self.op_role_key in op.attr_names and \ - int(op.all_attrs()[self.op_role_key]) & int(OpRole.Optimize) + return self.op_role_key in op.attr_names and int( + op.all_attrs()[self.op_role_key] + ) & int(OpRole.Optimize) def _remove_optimize_op_for_embedding(self, loss, table_name): """ @@ -217,8 +236,9 @@ def _if_last_block(self, op, _equal_dict): return False return True - def _generte_cond_para_map(self, op, _fill_value_dict, _equal_fill_dict, - _now_program, _all_params): + def _generte_cond_para_map( + self, op, _fill_value_dict, _equal_fill_dict, _now_program, _all_params + ): # generate cond value to parameter map recursively cond_str = op.input('Cond')[0] vars_ = op.input('Input') @@ -241,9 +261,13 @@ def _generte_cond_para_map(self, op, _fill_value_dict, _equal_fill_dict, ops_cond = _now_program.block(int(op.attr('sub_block').id)).ops for op in ops_cond: if op.type == 'conditional_block': - self._generte_cond_para_map(op, _fill_value_dict, - _equal_fill_dict, _now_program, - _all_params) + self._generte_cond_para_map( + op, + _fill_value_dict, + _equal_fill_dict, + _now_program, + _all_params, + ) def _has_conditional_block(self, loss): now_program = loss.block.program @@ -256,8 +280,10 @@ def _has_conditional_block(self, loss): def _check_params_grads(self, params, grads): if len(params) != len(grads): - raise ValueError("params size != grads size, %s vs %s" % - (len(params), len(grads))) + raise ValueError( + "params size != grads size, %s vs %s" + % (len(params), len(grads)) + ) pname2grad = dict() for i in range(len(params)): @@ -269,13 +295,15 @@ def _check_params_grads(self, params, grads): return pname2grad - def _generate_multi_dense_table(self, - params, - grads, - cond_params, - other_params, - sparse_table_names, - dense_table_id=0): + def _generate_multi_dense_table( + self, + params, + grads, + cond_params, + other_params, + sparse_table_names, + dense_table_id=0, + ): # generate multi dense table by cond value pname2grad = self._check_params_grads(params, grads) root_params_list = [] @@ -307,7 +335,14 @@ def _generate_multi_dense_table(self, lists_params[name2key[p.name]].append(p) lists_grads[name2key[p.name]].append(pname2grad[p.name]) - return dense_tables, cond2denseid, lists_params, lists_grads, root_params_list, root_grads_list + return ( + dense_tables, + cond2denseid, + lists_params, + lists_grads, + root_params_list, + root_grads_list, + ) def _gen_distributed_emb_to_size_dict(self, program): d_size = dict() @@ -321,13 +356,16 @@ def _gen_distributed_emb_to_size_dict(self, program): if d_size.get(table_name) is None: d_size[table_name] = emb_size elif d_size[table_name] != emb_size: - raise ValueError("embedding size error: %s vs %s" % - (emb_size, d_size[table_name])) + raise ValueError( + "embedding size error: %s vs %s" + % (emb_size, d_size[table_name]) + ) return d_size - def _check_config_fleet_with_program_op(self, strategy, table_name, - emb_to_size): + def _check_config_fleet_with_program_op( + self, strategy, table_name, emb_to_size + ): if strategy.get(table_name) is None: strategy[table_name] = dict() st = strategy[table_name] @@ -337,67 +375,93 @@ def _check_config_fleet_with_program_op(self, strategy, table_name, accessor = st["sparse_accessor_class"] # set sparse_embedx_dim in the strategy according to accessor and use_cvm config - if accessor == "DownpourFeatureValueAccessor" \ - or accessor == "DownpourCtrAccessor" \ - or accessor == "DownpourCtrDymfAccessor" \ - or accessor == "DownpourDoubleUnitAccessor" \ - or accessor == "DownpourUnitAccessor": - if st.get("sparse_embedx_dim") is not None \ - and strategy.get("use_cvm") == True \ - and st["sparse_embedx_dim"] != emb_to_size[table_name] - 3: + if ( + accessor == "DownpourFeatureValueAccessor" + or accessor == "DownpourCtrAccessor" + or accessor == "DownpourCtrDymfAccessor" + or accessor == "DownpourDoubleUnitAccessor" + or accessor == "DownpourUnitAccessor" + ): + if ( + st.get("sparse_embedx_dim") is not None + and strategy.get("use_cvm") == True + and st["sparse_embedx_dim"] != emb_to_size[table_name] - 3 + ): raise ValueError( "fleet config sparse_embedx_dim=%s not" - " equal to embedding dim - 3 = %s" % - (st["sparse_embedx_dim"], emb_to_size[table_name] - 3)) - if st.get("sparse_embedx_dim") is not None \ - and strategy.get("use_cvm") == False \ - and st["sparse_embedx_dim"] != emb_to_size[table_name] - 1: + " equal to embedding dim - 3 = %s" + % (st["sparse_embedx_dim"], emb_to_size[table_name] - 3) + ) + if ( + st.get("sparse_embedx_dim") is not None + and strategy.get("use_cvm") == False + and st["sparse_embedx_dim"] != emb_to_size[table_name] - 1 + ): raise ValueError( "fleet config sparse_embedx_dim=%s not" - " equal to embedding dim - 1 = %s" % - (st["sparse_embedx_dim"], emb_to_size[table_name] - 1)) - if st.get("sparse_embedx_dim") is None \ - and strategy.get("use_cvm") == True: + " equal to embedding dim - 1 = %s" + % (st["sparse_embedx_dim"], emb_to_size[table_name] - 1) + ) + if ( + st.get("sparse_embedx_dim") is None + and strategy.get("use_cvm") == True + ): logger.warning( "sparse embedding dim for table name '{}' is: {}, while sparse_embedx_dim " "with same sparse table name is not set in config_fleet.py. " - "Hence automatically set sparse_embedx_dim = {} - 3.". - format(table_name, emb_to_size[table_name], - emb_to_size[table_name])) + "Hence automatically set sparse_embedx_dim = {} - 3.".format( + table_name, + emb_to_size[table_name], + emb_to_size[table_name], + ) + ) st["sparse_embedx_dim"] = emb_to_size[table_name] - 3 - if st.get("sparse_embedx_dim") is None \ - and strategy.get("use_cvm") == False: + if ( + st.get("sparse_embedx_dim") is None + and strategy.get("use_cvm") == False + ): logger.warning( "sparse embedding dim for table name '{}' is: {}, while sparse_embedx_dim " "with same sparse table name is not set in config_fleet.py. " - "Hence automatically set sparse_embedx_dim = {} - 1.". - format(table_name, emb_to_size[table_name], - emb_to_size[table_name])) + "Hence automatically set sparse_embedx_dim = {} - 1.".format( + table_name, + emb_to_size[table_name], + emb_to_size[table_name], + ) + ) st["sparse_embedx_dim"] = emb_to_size[table_name] - 1 elif accessor == "DownpourSparseValueAccessor": - if st.get("sparse_embedx_dim") is not None \ - and st["sparse_embedx_dim"] != emb_to_size[table_name]: + if ( + st.get("sparse_embedx_dim") is not None + and st["sparse_embedx_dim"] != emb_to_size[table_name] + ): raise ValueError( "fleet config sparse_embedx_dim=%s not" - " equal to embedding dim = %s" % - (st["sparse_embedx_dim"], emb_to_size[table_name])) + " equal to embedding dim = %s" + % (st["sparse_embedx_dim"], emb_to_size[table_name]) + ) if st.get("sparse_embedx_dim") is None: logger.warning( "sparse embedding dim for table name '{}' is: {}, while sparse_embedx_dim " "with same sparse table name is not set in config_fleet.py. " "Hence automatically set sparse_embedx_dim = {}.".format( - table_name, emb_to_size[table_name], - emb_to_size[table_name])) + table_name, + emb_to_size[table_name], + emb_to_size[table_name], + ) + ) st["sparse_embedx_dim"] = emb_to_size[table_name] return strategy - def _minimize(self, - losses, - startup_program=None, - parameter_list=None, - no_grad_set=None, - strategy={}): + def _minimize( + self, + losses, + startup_program=None, + parameter_list=None, + no_grad_set=None, + strategy={}, + ): """ DownpounSGD is a distributed optimizer so that user can call minimize to generate backward @@ -439,9 +503,10 @@ def _minimize(self, parameters = parameter_list[num] prog_id = str(id(loss.block.program)) # param_grads of program - params_grads = sorted(fluid.backward.append_backward( - loss, parameters, no_grad_set), - key=lambda x: x[0].name) + params_grads = sorted( + fluid.backward.append_backward(loss, parameters, no_grad_set), + key=lambda x: x[0].name, + ) flag_use_ps_gpu = strategy.get("use_ps_gpu", False) if flag_use_ps_gpu: @@ -451,9 +516,11 @@ def _minimize(self, optimize_ops = optimizer.apply_optimize( loss, startup_program=startup_program[num], - params_grads=params_grads) + params_grads=params_grads, + ) embedding_table = self._find_multi_distributed_lookup_table( - [loss]) + [loss] + ) self._remove_optimize_op_for_embedding(loss, embedding_table) # has condition_block op means multi-task flag_multi_task = self._has_conditional_block(loss) @@ -476,9 +543,13 @@ def _minimize(self, if op.type == 'equal': equal_fill_dict[op.output('Out')[0]] = op.input('Y')[0] if op.type == 'conditional_block': - self._generte_cond_para_map(op, fill_value_dict, - equal_fill_dict, - now_program, all_params) + self._generte_cond_para_map( + op, + fill_value_dict, + equal_fill_dict, + now_program, + all_params, + ) if prog_id not in program_id_set: program_id_set.add(prog_id) @@ -493,28 +564,32 @@ def _minimize(self, # get {table_name: emb_size} dict from program ops emb_to_size = self._gen_distributed_emb_to_size_dict( - loss.block.program) + loss.block.program + ) # get inputs_dict inputs_dict = self._find_distributed_lookup_table_inputs( - loss.block.program, sparse_table) + loss.block.program, sparse_table + ) prog_id_to_inputs_dict[prog_id] = inputs_dict # get outputs_dict outputs_dict = self._find_distributed_lookup_table_outputs( - loss.block.program, sparse_table) + loss.block.program, sparse_table + ) prog_id_to_outputs_dict[prog_id] = outputs_dict prog_id_to_worker[prog_id] = DownpourWorker(self._window) grads_dict = self._find_distributed_lookup_table_grads( - loss.block.program, sparse_table) + loss.block.program, sparse_table + ) prog_id_to_sparse_grads[prog_id] = grads_dict if prog_id not in prog_id_to_param_grads: prog_id_to_param_grads[prog_id] = [] prog_id_to_param_grads[prog_id].append(params_grads) - #if strategy.get("parallel_compute") + # if strategy.get("parallel_compute") # if user specify a fleet_desc.prototxt file, then load the file # instead of creating default fleet_desc.prototxt. @@ -527,17 +602,20 @@ def _minimize(self, if len(ps_param.trainer_param) == 1: for k in prog_id_to_worker: prog_id_to_worker[k].get_desc().CopyFrom( - ps_param.trainer_param[0]) + ps_param.trainer_param[0] + ) else: if len(ps_param.trainer_param) != len(prog_id_to_worker): raise ValueError( - "trainer param size != program size, %s vs %s" % - (len(ps_param.trainer_param), len(prog_id_to_worker))) + "trainer param size != program size, %s vs %s" + % (len(ps_param.trainer_param), len(prog_id_to_worker)) + ) idx = 0 # prog_id_to_worker is OrderedDict for k in prog_id_to_worker: prog_id_to_worker[k].get_desc().CopyFrom( - ps_param.trainer_param[idx]) + ps_param.trainer_param[idx] + ) idx += 1 # check config in op defination and fleet config @@ -550,10 +628,13 @@ def _minimize(self, if len(sparse_table_to_index) != len(emb_to_table): raise ValueError( "sparse tables from program != sparse tables from op: %s " - "vs %s" % (len(sparse_table_to_index), len(emb_to_table))) + "vs %s" % (len(sparse_table_to_index), len(emb_to_table)) + ) for key in sparse_table_to_index: - if key not in emb_to_table or \ - sparse_table_to_index[key] != emb_to_table[key]: + if ( + key not in emb_to_table + or sparse_table_to_index[key] != emb_to_table[key] + ): print("sparse_table_to_index ", sparse_table_to_index) print("emb_to_table ", emb_to_table) raise ValueError("key error: %s" % key) @@ -564,8 +645,9 @@ def _minimize(self, accessor = None if st.get("sparse_accessor_class") is not None: accessor = st["sparse_accessor_class"] - tables = \ + tables = ( server.get_desc().downpour_server_param.downpour_table_param + ) for table in tables: if table.table_id == sparse_table_to_index[key]: accessor = table.accessor.accessor_class @@ -574,44 +656,57 @@ def _minimize(self, for loss in losses: for op in loss.block.program.global_block().ops: if op.type in self.supported_embedding_types: - if accessor is not None \ - and op.has_attr("AccessorClass"): + if accessor is not None and op.has_attr( + "AccessorClass" + ): op._set_attr("AccessorClass", accessor) if one_slot is None: - one_slot = loss.block.program. \ - global_block().var(op.input("Ids")[0]) + one_slot = ( + loss.block.program.global_block().var( + op.input("Ids")[0] + ) + ) # if accessor is None, use default accessor in op definition if accessor is None: accessor = emb_to_accessor[key] # set sparse_embedx_dim in strategy, # user do not have to set it in config_fleet - if accessor == "DownpourFeatureValueAccessor" \ - or accessor == "DownpourCtrDymfAccessor" \ - or accessor == "DownpourCtrAccessor" \ - or accessor == "DownpourDoubleUnitAccessor" \ - or accessor == "DownpourUnitAccessor": - if st.get("sparse_embedx_dim") is not None \ - and st["sparse_embedx_dim"] != emb_to_size[key] - 3: + if ( + accessor == "DownpourFeatureValueAccessor" + or accessor == "DownpourCtrDymfAccessor" + or accessor == "DownpourCtrAccessor" + or accessor == "DownpourDoubleUnitAccessor" + or accessor == "DownpourUnitAccessor" + ): + if ( + st.get("sparse_embedx_dim") is not None + and st["sparse_embedx_dim"] != emb_to_size[key] - 3 + ): raise ValueError( "fleet config sparse_embedx_dim=%s not" - " equal to embedding size - 3 = %s" % - (st["sparse_embedx_dim"], emb_to_size[key] - 3)) + " equal to embedding size - 3 = %s" + % (st["sparse_embedx_dim"], emb_to_size[key] - 3) + ) st["sparse_embedx_dim"] = emb_to_size[key] - 3 elif accessor == "DownpourSparseValueAccessor": - if st.get("sparse_embedx_dim") is not None \ - and st["sparse_embedx_dim"] != emb_to_size[key]: + if ( + st.get("sparse_embedx_dim") is not None + and st["sparse_embedx_dim"] != emb_to_size[key] + ): raise ValueError( "fleet config sparse_embedx_dim=%s not" - " equal to embedding size = %s" % - (st["sparse_embedx_dim"], emb_to_size[key])) + " equal to embedding size = %s" + % (st["sparse_embedx_dim"], emb_to_size[key]) + ) st["sparse_embedx_dim"] = emb_to_size[key] # ServerParameter add all sparse tables for tn in sparse_table_to_index: sparse_table_index = sparse_table_to_index[tn] st = self._check_config_fleet_with_program_op( - strategy, tn, emb_to_size) + strategy, tn, emb_to_size + ) if st.get(tn) is not None: server.add_sparse_table(sparse_table_index, st[tn]) else: @@ -629,8 +724,12 @@ def _minimize(self, for tn in prog_id_to_sparse_table[prog_id]: sparse_table_index = sparse_table_to_index[tn] grads_dict = prog_id_to_sparse_grads[prog_id] - worker.add_sparse_table(sparse_table_index, inputs_dict[tn], - outputs_dict[tn], grads_dict[tn]) + worker.add_sparse_table( + sparse_table_index, + inputs_dict[tn], + outputs_dict[tn], + grads_dict[tn], + ) dense_start_table_id = len(sparse_table_to_index) dense_table_index = len(sparse_table_to_index) @@ -644,12 +743,13 @@ def _minimize(self, program_id_set.add(program_id) worker = prog_id_to_worker[program_id] sparse_table_names = prog_id_to_sparse_table[program_id] - sparse_table_index = \ - [sparse_table_to_index[i] for i in sparse_table_names] + sparse_table_index = [ + sparse_table_to_index[i] for i in sparse_table_names + ] program_configs[program_id] = { "pull_sparse": [t_index for t_index in sparse_table_index], - "push_sparse": [t_index for t_index in sparse_table_index] + "push_sparse": [t_index for t_index in sparse_table_index], } params_grads = prog_id_to_param_grads[program_id] @@ -679,12 +779,24 @@ def _minimize(self, multi_task_dense_tables_push = [] multi_task_dense_tables_pull = [] if flag_multi_task: - dense_tables, cond2denseid, lists_params, lists_grads, root_params_list, root_grads_list = self._generate_multi_dense_table( - params, grads, self._cond_params, - self._other_params, sparse_table_names, - dense_table_index) + ( + dense_tables, + cond2denseid, + lists_params, + lists_grads, + root_params_list, + root_grads_list, + ) = self._generate_multi_dense_table( + params, + grads, + self._cond_params, + self._other_params, + sparse_table_names, + dense_table_index, + ) program_configs[program_id][ - 'cond2denseid'] = cond2denseid + 'cond2denseid' + ] = cond2denseid multi_task_dense_tables_push = dense_tables multi_task_dense_tables_pull = dense_tables[:] @@ -692,56 +804,76 @@ def _minimize(self, if flag_multi_task: server_dense_table_index = dense_table_index if len(root_params_list) > 0: - server.add_dense_table(server_dense_table_index, - root_params_list, - root_grads_list, - strategy['dense_table'], - sparse_table_names) + server.add_dense_table( + server_dense_table_index, + root_params_list, + root_grads_list, + strategy['dense_table'], + sparse_table_names, + ) server_dense_table_index += 1 for i in range(len(lists_params)): - server.add_dense_table(server_dense_table_index, - lists_params[i], - lists_grads[i], - strategy['dense_table'], - sparse_table_names) + server.add_dense_table( + server_dense_table_index, + lists_params[i], + lists_grads[i], + strategy['dense_table'], + sparse_table_names, + ) server_dense_table_index += 1 else: - server.add_dense_table(dense_table_index, params, - grads, - strategy['dense_table'], - sparse_table_names) + server.add_dense_table( + dense_table_index, + params, + grads, + strategy['dense_table'], + sparse_table_names, + ) else: - server.add_dense_table(dense_table_index, params, grads, - None, sparse_table_names) + server.add_dense_table( + dense_table_index, + params, + grads, + None, + sparse_table_names, + ) if flag_multi_task: if len(root_params_list) > 0: - worker.add_dense_table(dense_table_index, - self._learning_rate, - root_params_list, - root_grads_list, - dense_start_table_id, - sparse_table_names) + worker.add_dense_table( + dense_table_index, + self._learning_rate, + root_params_list, + root_grads_list, + dense_start_table_id, + sparse_table_names, + ) dense_table_index += 1 for i in range(len(lists_params)): - worker.add_dense_table(dense_table_index, - self._learning_rate, - lists_params[i], - lists_grads[i], - dense_start_table_id, - sparse_table_names) + worker.add_dense_table( + dense_table_index, + self._learning_rate, + lists_params[i], + lists_grads[i], + dense_start_table_id, + sparse_table_names, + ) dense_table_index += 1 dense_table_index -= 1 else: - worker.add_dense_table(dense_table_index, - self._learning_rate, params, - grads, dense_start_table_id, - sparse_table_names) + worker.add_dense_table( + dense_table_index, + self._learning_rate, + params, + grads, + dense_start_table_id, + sparse_table_names, + ) if FLEET_GLOBAL_DICT["enable"]: cur_prog = losses[loss_index].block.program @@ -751,30 +883,39 @@ def _minimize(self, attrs={ "InputNames": [i.name for i in grads], "TableId": dense_table_index, - "ScaleDataNorm": - strategy.get("scale_datanorm", -1) - }) - - if "pull_dense" in program_configs[ - program_id] and "push_dense" in program_configs[ - program_id] and len(program_configs[program_id] - ["pull_dense"]) > 0: + "ScaleDataNorm": strategy.get( + "scale_datanorm", -1 + ), + }, + ) + + if ( + "pull_dense" in program_configs[program_id] + and "push_dense" in program_configs[program_id] + and len(program_configs[program_id]["pull_dense"]) > 0 + ): if flag_multi_task: program_configs[program_id]["pull_dense"].extend( - multi_task_dense_tables_pull) + multi_task_dense_tables_pull + ) program_configs[program_id]["push_dense"].extend( - multi_task_dense_tables_push) + multi_task_dense_tables_push + ) else: program_configs[program_id]["pull_dense"].extend( - [dense_table_index]) + [dense_table_index] + ) program_configs[program_id]["push_dense"].extend( - [dense_table_index]) + [dense_table_index] + ) else: if flag_multi_task: program_configs[program_id][ - "pull_dense"] = multi_task_dense_tables_pull + "pull_dense" + ] = multi_task_dense_tables_pull program_configs[program_id][ - "push_dense"] = multi_task_dense_tables_push + "push_dense" + ] = multi_task_dense_tables_push else: program_configs[program_id]["pull_dense"] = [ dense_table_index @@ -787,22 +928,31 @@ def _minimize(self, dense_table_index += 1 if strategy.get('datanorm_table') is not None: server.add_data_norm_table( - dense_table_index, self._learning_rate, - data_norm_params, data_norm_grads, - strategy['datanorm_table'], sparse_table_names) + dense_table_index, + self._learning_rate, + data_norm_params, + data_norm_grads, + strategy['datanorm_table'], + sparse_table_names, + ) else: - server.add_data_norm_table(dense_table_index, - self._learning_rate, - data_norm_params, - data_norm_grads, None, - sparse_table_names) - - worker.add_dense_table(dense_table_index, - self._learning_rate, - data_norm_params, - data_norm_grads, - dense_start_table_id, - sparse_table_names) + server.add_data_norm_table( + dense_table_index, + self._learning_rate, + data_norm_params, + data_norm_grads, + None, + sparse_table_names, + ) + + worker.add_dense_table( + dense_table_index, + self._learning_rate, + data_norm_params, + data_norm_grads, + dense_start_table_id, + sparse_table_names, + ) if FLEET_GLOBAL_DICT["enable"]: cur_prog = losses[loss_index].block.program @@ -810,18 +960,22 @@ def _minimize(self, type="push_dense", inputs={"Ids": one_slot}, attrs={ - "InputNames": - [i.name for i in data_norm_grads], - "TableId": - dense_table_index, - "ScaleDataNorm": - strategy.get("scale_datanorm", -1) - }) + "InputNames": [ + i.name for i in data_norm_grads + ], + "TableId": dense_table_index, + "ScaleDataNorm": strategy.get( + "scale_datanorm", -1 + ), + }, + ) program_configs[program_id]["pull_dense"].extend( - [dense_table_index]) + [dense_table_index] + ) program_configs[program_id]["push_dense"].extend( - [dense_table_index]) + [dense_table_index] + ) dense_table_index += 1 # Todo(guru4elephant): figure out how to support more sparse parameters @@ -865,36 +1019,47 @@ def _minimize(self, opt_info["use_cvm"] = strategy.get("use_cvm", False) opt_info["no_cvm"] = strategy.get("no_cvm", False) opt_info["scale_sparse_gradient_with_batch_size"] = strategy.get( - "scale_sparse_gradient_with_batch_size", True) - opt_info["worker_class"] = strategy.get("worker_class", - "DownpourWorker") + "scale_sparse_gradient_with_batch_size", True + ) + opt_info["worker_class"] = strategy.get( + "worker_class", "DownpourWorker" + ) opt_info["stat_var_names"] = strategy.get("stat_var_names", []) opt_info["local_tables"] = strategy.get("local_tables", []) opt_info["async_tables"] = strategy.get("async_tables", []) opt_info["async_tables"] = strategy.get("async_tables", []) opt_info["scale_datanorm"] = strategy.get("scale_datanorm", -1) - opt_info["check_nan_var_names"] = strategy.get("check_nan_var_names", - []) + opt_info["check_nan_var_names"] = strategy.get( + "check_nan_var_names", [] + ) opt_info["dump_slot"] = False opt_info["dump_converter"] = "" opt_info["dump_fields"] = strategy.get("dump_fields", []) opt_info["dump_file_num"] = strategy.get("dump_file_num", 16) opt_info["user_define_dump_filename"] = strategy.get( - "user_define_dump_filename", "") + "user_define_dump_filename", "" + ) opt_info["dump_fields_path"] = strategy.get("dump_fields_path", "") opt_info["dump_param"] = strategy.get("dump_param", []) gpus_env = os.getenv("FLAGS_selected_gpus", "0") opt_info["worker_places"] = [int(s) for s in gpus_env.split(",")] opt_info["use_ps_gpu"] = strategy.get("use_ps_gpu", False) if server._server.downpour_server_param.downpour_table_param[ - 0].accessor.accessor_class in [ - "DownpourCtrAccessor", "DownpourCtrDoubleAccessor", - "DownpourUnitAccessor", "DownpourDoubleUnitAccessor", - "DownpourCtrDymfAccessor" - ]: + 0 + ].accessor.accessor_class in [ + "DownpourCtrAccessor", + "DownpourCtrDoubleAccessor", + "DownpourUnitAccessor", + "DownpourDoubleUnitAccessor", + "DownpourCtrDymfAccessor", + ]: opt_info["dump_slot"] = True - elif server._server.downpour_server_param.downpour_table_param[ - 0].accessor.accessor_class == "DownpourSparseValueAccessor": + elif ( + server._server.downpour_server_param.downpour_table_param[ + 0 + ].accessor.accessor_class + == "DownpourSparseValueAccessor" + ): opt_info["no_cvm"] = True opt_info["adjust_ins_weight"] = strategy.get("adjust_ins_weight", {}) opt_info["copy_table"] = strategy.get("copy_table", {}) diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/ps_pb2.py b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/ps_pb2.py index eec51ef827c57..4291115b0bcbb 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/ps_pb2.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/ps_pb2.py @@ -24,6 +24,7 @@ from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database from google.protobuf import descriptor_pb2 + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -34,7 +35,8 @@ syntax='proto2', serialized_pb=_b( '\n\x08ps.proto\x12\x06paddle\"\xb5\x02\n\x0bPSParameter\x12\x14\n\x0cworker_class\x18\x01 \x01(\t\x12\x14\n\x0cserver_class\x18\x02 \x01(\t\x12\x16\n\x0einstance_class\x18\x03 \x01(\t\x12\x15\n\x0binit_gflags\x18\x04 \x01(\t:\x00\x12-\n\x0cworker_param\x18\x65 \x01(\x0b\x32\x17.paddle.WorkerParameter\x12-\n\x0cserver_param\x18\x66 \x01(\x0b\x32\x17.paddle.ServerParameter\x12\x38\n\rtrainer_param\x18\xad\x02 \x03(\x0b\x32 .paddle.DownpourTrainerParameter\x12\x33\n\x0f\x66s_client_param\x18\xf5\x03 \x01(\x0b\x32\x19.paddle.FsClientParameter\"Q\n\x0fWorkerParameter\x12>\n\x15\x64ownpour_worker_param\x18\x01 \x01(\x0b\x32\x1f.paddle.DownpourWorkerParameter\"Q\n\x0fServerParameter\x12>\n\x15\x64ownpour_server_param\x18\x01 \x01(\x0b\x32\x1f.paddle.DownpourServerParameter\"O\n\x17\x44ownpourWorkerParameter\x12\x34\n\x14\x64ownpour_table_param\x18\x01 \x03(\x0b\x32\x16.paddle.TableParameter\"\xfd\x01\n\x18\x44ownpourTrainerParameter\x12\x30\n\x0b\x64\x65nse_table\x18\x01 \x03(\x0b\x32\x1b.paddle.DenseTableParameter\x12\x32\n\x0csparse_table\x18\x02 \x03(\x0b\x32\x1c.paddle.SparseTableParameter\x12\x1d\n\x15push_sparse_per_batch\x18\x03 \x01(\x05\x12\x1c\n\x14push_dense_per_batch\x18\x04 \x01(\x05\x12\x0f\n\x07skip_op\x18\x05 \x03(\t\x12-\n\x0eprogram_config\x18\x06 \x03(\x0b\x32\x15.paddle.ProgramConfig\"\x99\x01\n\rProgramConfig\x12\x12\n\nprogram_id\x18\x01 \x02(\t\x12\x1c\n\x14push_sparse_table_id\x18\x02 \x03(\x05\x12\x1b\n\x13push_dense_table_id\x18\x03 \x03(\x05\x12\x1c\n\x14pull_sparse_table_id\x18\x04 \x03(\x05\x12\x1b\n\x13pull_dense_table_id\x18\x05 \x03(\x05\"{\n\x13\x44\x65nseTableParameter\x12\x10\n\x08table_id\x18\x01 \x01(\x05\x12\x1b\n\x13\x64\x65nse_variable_name\x18\x02 \x03(\t\x12$\n\x1c\x64\x65nse_gradient_variable_name\x18\x03 \x03(\t\x12\x0f\n\x07\x66\x65\x61_dim\x18\x04 \x01(\x05\"z\n\x14SparseTableParameter\x12\x10\n\x08table_id\x18\x01 \x01(\x05\x12\x13\n\x0b\x66\x65\x61ture_dim\x18\x02 \x01(\x05\x12\x10\n\x08slot_key\x18\x03 \x03(\t\x12\x12\n\nslot_value\x18\x04 \x03(\t\x12\x15\n\rslot_gradient\x18\x05 \x03(\t\"\x86\x01\n\x17\x44ownpourServerParameter\x12\x34\n\x14\x64ownpour_table_param\x18\x01 \x03(\x0b\x32\x16.paddle.TableParameter\x12\x35\n\rservice_param\x18\x02 \x01(\x0b\x32\x1e.paddle.ServerServiceParameter\"\xd7\x01\n\x16ServerServiceParameter\x12*\n\x0cserver_class\x18\x01 \x01(\t:\x14\x44ownpourBrpcPsServer\x12*\n\x0c\x63lient_class\x18\x02 \x01(\t:\x14\x44ownpourBrpcPsClient\x12(\n\rservice_class\x18\x03 \x01(\t:\x11\x44ownpourPsService\x12\x1c\n\x11start_server_port\x18\x04 \x01(\r:\x01\x30\x12\x1d\n\x11server_thread_num\x18\x05 \x01(\r:\x02\x31\x32\"\xc0\x02\n\x0eTableParameter\x12\x10\n\x08table_id\x18\x01 \x01(\x04\x12\x13\n\x0btable_class\x18\x02 \x01(\t\x12\x17\n\tshard_num\x18\x03 \x01(\x04:\x04\x31\x30\x30\x30\x12\x30\n\x08\x61\x63\x63\x65ssor\x18\x04 \x01(\x0b\x32\x1e.paddle.TableAccessorParameter\x12\x1f\n\x04type\x18\x05 \x01(\x0e\x32\x11.paddle.TableType\x12\x1f\n\x10\x63ompress_in_save\x18\x06 \x01(\x08:\x05\x66\x61lse\x12\'\n\x19\x65nable_sparse_table_cache\x18\x07 \x01(\x08:\x04true\x12(\n\x17sparse_table_cache_rate\x18\x08 \x01(\x01:\x07\x30.00055\x12\'\n\x1bsparse_table_cache_file_num\x18\t \x01(\r:\x02\x31\x36\"\xc1\x04\n\x16TableAccessorParameter\x12\x16\n\x0e\x61\x63\x63\x65ssor_class\x18\x01 \x01(\t\x12\x38\n\x10sparse_sgd_param\x18\x02 \x01(\x0b\x32\x1e.paddle.SparseSGDRuleParameter\x12\x36\n\x0f\x64\x65nse_sgd_param\x18\x03 \x01(\x0b\x32\x1d.paddle.DenseSGDRuleParameter\x12\x13\n\x07\x66\x65\x61_dim\x18\x04 \x01(\r:\x02\x31\x31\x12\x15\n\nembedx_dim\x18\x05 \x01(\r:\x01\x38\x12\x1c\n\x10\x65mbedx_threshold\x18\x06 \x01(\r:\x02\x31\x30\x12G\n\x17\x64ownpour_accessor_param\x18\x07 \x01(\x0b\x32&.paddle.DownpourTableAccessorParameter\x12\x45\n\x19table_accessor_save_param\x18\x08 \x03(\x0b\x32\".paddle.TableAccessorSaveParameter\x12\x44\n\x16sparse_commonsgd_param\x18\t \x01(\x0b\x32$.paddle.SparseCommonSGDRuleParameter\x12=\n\x0f\x65mbed_sgd_param\x18\n \x01(\x0b\x32$.paddle.SparseCommonSGDRuleParameter\x12>\n\x10\x65mbedx_sgd_param\x18\x0b \x01(\x0b\x32$.paddle.SparseCommonSGDRuleParameter\"\xba\x02\n\x1e\x44ownpourTableAccessorParameter\x12\x19\n\x0cnonclk_coeff\x18\x01 \x01(\x02:\x03\x30.1\x12\x16\n\x0b\x63lick_coeff\x18\x02 \x01(\x02:\x01\x31\x12\x1b\n\x0e\x62\x61se_threshold\x18\x03 \x01(\x02:\x03\x31.5\x12\x1d\n\x0f\x64\x65lta_threshold\x18\x04 \x01(\x02:\x04\x30.25\x12\x1b\n\x0f\x64\x65lta_keep_days\x18\x05 \x01(\x02:\x02\x31\x36\x12#\n\x15show_click_decay_rate\x18\x06 \x01(\x02:\x04\x30.98\x12\x1d\n\x10\x64\x65lete_threshold\x18\x07 \x01(\x02:\x03\x30.8\x12$\n\x18\x64\x65lete_after_unseen_days\x18\x08 \x01(\x02:\x02\x33\x30\x12\"\n\x17ssd_unseenday_threshold\x18\t \x01(\x05:\x01\x31\"S\n\x1aTableAccessorSaveParameter\x12\r\n\x05param\x18\x01 \x01(\r\x12\x11\n\tconverter\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65\x63onverter\x18\x03 \x01(\t\"e\n\x10PsRequestMessage\x12\x0e\n\x06\x63md_id\x18\x01 \x02(\r\x12\x10\n\x08table_id\x18\x02 \x01(\r\x12\x0e\n\x06params\x18\x03 \x03(\x0c\x12\x11\n\tclient_id\x18\x04 \x01(\x05\x12\x0c\n\x04\x64\x61ta\x18\x05 \x01(\x0c\"\x85\x01\n\x16SparseSGDRuleParameter\x12\x1b\n\rlearning_rate\x18\x01 \x01(\x01:\x04\x30.05\x12\x18\n\rinitial_g2sum\x18\x02 \x01(\x01:\x01\x33\x12\x1d\n\rinitial_range\x18\x03 \x01(\x01:\x06\x30.0001\x12\x15\n\rweight_bounds\x18\x04 \x03(\x02\"\xc6\x01\n\x1cSparseCommonSGDRuleParameter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x32\n\x05naive\x18\x02 \x01(\x0b\x32#.paddle.SparseNaiveSGDRuleParameter\x12\x36\n\x07\x61\x64\x61grad\x18\x03 \x01(\x0b\x32%.paddle.SparseAdagradSGDRuleParameter\x12,\n\x04\x61\x64\x61m\x18\x04 \x01(\x0b\x32\x1e.paddle.SparseAdamSGDParameter\"p\n\x1bSparseNaiveSGDRuleParameter\x12\x1b\n\rlearning_rate\x18\x01 \x01(\x01:\x04\x30.05\x12\x1d\n\rinitial_range\x18\x02 \x01(\x01:\x06\x30.0001\x12\x15\n\rweight_bounds\x18\x03 \x03(\x02\"\x8c\x01\n\x1dSparseAdagradSGDRuleParameter\x12\x1b\n\rlearning_rate\x18\x01 \x01(\x01:\x04\x30.05\x12\x18\n\rinitial_g2sum\x18\x02 \x01(\x01:\x01\x33\x12\x1d\n\rinitial_range\x18\x03 \x01(\x01:\x06\x30.0001\x12\x15\n\rweight_bounds\x18\x04 \x03(\x02\"\xc8\x01\n\x16SparseAdamSGDParameter\x12\x1c\n\rlearning_rate\x18\x01 \x01(\x01:\x05\x30.001\x12\x1d\n\rinitial_range\x18\x02 \x01(\x01:\x06\x30.0001\x12\x1d\n\x10\x62\x65ta1_decay_rate\x18\x03 \x01(\x01:\x03\x30.9\x12\x1f\n\x10\x62\x65ta2_decay_rate\x18\x04 \x01(\x01:\x05\x30.999\x12\x1a\n\x0b\x61\x64\x61_epsilon\x18\x05 \x01(\x01:\x05\x31\x65-08\x12\x15\n\rweight_bounds\x18\x06 \x03(\x02\"\xe1\x01\n\x15\x44\x65nseSGDRuleParameter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\x04\x61\x64\x61m\x18\x02 \x01(\x0b\x32\x18.paddle.AdamSGDParameter\x12(\n\x05naive\x18\x03 \x01(\x0b\x32\x19.paddle.NaiveSGDParameter\x12,\n\x07summary\x18\x04 \x01(\x0b\x32\x1b.paddle.SummarySGDParameter\x12:\n\x0emoving_average\x18\x05 \x01(\x0b\x32\".paddle.MovingAverageRuleParameter\"\xac\x01\n\x10\x41\x64\x61mSGDParameter\x12\x1c\n\rlearning_rate\x18\x01 \x01(\x01:\x05\x35\x65-06\x12 \n\x0e\x61vg_decay_rate\x18\x02 \x01(\x01:\x08\x30.999993\x12\x1e\n\x0e\x61\x64\x61_decay_rate\x18\x03 \x01(\x01:\x06\x30.9999\x12\x1a\n\x0b\x61\x64\x61_epsilon\x18\x04 \x01(\x01:\x05\x31\x65-08\x12\x1c\n\x0emom_decay_rate\x18\x05 \x01(\x01:\x04\x30.99\"J\n\x11NaiveSGDParameter\x12\x1d\n\rlearning_rate\x18\x01 \x01(\x01:\x06\x30.0002\x12\x16\n\x0e\x61vg_decay_rate\x18\x02 \x01(\x01\";\n\x13SummarySGDParameter\x12$\n\x12summary_decay_rate\x18\x01 \x01(\x01:\x08\x30.999999\".\n\x1aMovingAverageRuleParameter\x12\x10\n\x08momentum\x18\x01 \x01(\x01\"I\n\x11PsResponseMessage\x12\x13\n\x08\x65rr_code\x18\x01 \x02(\x05:\x01\x30\x12\x11\n\x07\x65rr_msg\x18\x02 \x02(\t:\x00\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\xd5\x01\n\x11\x46sClientParameter\x12:\n\x07\x66s_type\x18\x01 \x01(\x0e\x32#.paddle.FsClientParameter.FsApiType:\x04HDFS\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12\x0c\n\x04user\x18\x03 \x01(\t\x12\x0e\n\x06passwd\x18\x04 \x01(\t\x12\x13\n\x0b\x62uffer_size\x18\x05 \x01(\x05\x12\x12\n\nhadoop_bin\x18\x33 \x01(\t\x12\x10\n\x08\x61\x66s_conf\x18\x65 \x01(\t\"\x1e\n\tFsApiType\x12\x08\n\x04HDFS\x10\x00\x12\x07\n\x03\x41\x46S\x10\x01*4\n\tTableType\x12\x13\n\x0fPS_SPARSE_TABLE\x10\x00\x12\x12\n\x0ePS_DENSE_TABLE\x10\x01*\xba\x04\n\x07PsCmdID\x12\x17\n\x13PS_PULL_DENSE_TABLE\x10\x00\x12\x17\n\x13PS_PUSH_DENSE_TABLE\x10\x01\x12\x18\n\x14PS_PULL_SPARSE_TABLE\x10\x02\x12\x18\n\x14PS_PUSH_SPARSE_TABLE\x10\x03\x12\x13\n\x0fPS_SHRINK_TABLE\x10\x04\x12\x15\n\x11PS_SAVE_ONE_TABLE\x10\x05\x12\x15\n\x11PS_SAVE_ALL_TABLE\x10\x06\x12\x15\n\x11PS_LOAD_ONE_TABLE\x10\x07\x12\x15\n\x11PS_LOAD_ALL_TABLE\x10\x08\x12\x16\n\x12PS_CLEAR_ONE_TABLE\x10\t\x12\x16\n\x12PS_CLEAR_ALL_TABLE\x10\n\x12\x17\n\x13PS_PUSH_DENSE_PARAM\x10\x0b\x12\x12\n\x0ePS_STOP_SERVER\x10\x0c\x12\x1b\n\x17PS_SAVE_ONE_CACHE_TABLE\x10\r\x12\x1a\n\x16PS_GET_CACHE_THRESHOLD\x10\x0e\x12\x14\n\x10PS_CACHE_SHUFFLE\x10\x0f\x12\x11\n\rPS_COPY_TABLE\x10\x10\x12\x1c\n\x18PS_COPY_TABLE_BY_FEASIGN\x10\x11\x12(\n$PS_PULL_SPARSE_TABLE_WITH_DEPENDENCY\x10\x12\x12(\n$PS_PUSH_SPARSE_TABLE_WITH_DEPENDENCY\x10\x13\x12\x17\n\x13PS_PRINT_TABLE_STAT\x10\x14\x12\x0e\n\nPS_S2S_MSG\x10\x65\x32K\n\tPsService\x12>\n\x07service\x12\x18.paddle.PsRequestMessage\x1a\x19.paddle.PsResponseMessageB\x06\x80\x01\x01\xf8\x01\x01' - )) + ), +) _sym_db.RegisterFileDescriptor(DESCRIPTOR) _TABLETYPE = _descriptor.EnumDescriptor( @@ -43,16 +45,12 @@ filename=None, file=DESCRIPTOR, values=[ - _descriptor.EnumValueDescriptor(name='PS_SPARSE_TABLE', - index=0, - number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_DENSE_TABLE', - index=1, - number=1, - options=None, - type=None), + _descriptor.EnumValueDescriptor( + name='PS_SPARSE_TABLE', index=0, number=0, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_DENSE_TABLE', index=1, number=1, options=None, type=None + ), ], containing_type=None, options=None, @@ -68,118 +66,128 @@ filename=None, file=DESCRIPTOR, values=[ - _descriptor.EnumValueDescriptor(name='PS_PULL_DENSE_TABLE', - index=0, - number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PUSH_DENSE_TABLE', - index=1, - number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PULL_SPARSE_TABLE', - index=2, - number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PUSH_SPARSE_TABLE', - index=3, - number=3, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_SHRINK_TABLE', - index=4, - number=4, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_SAVE_ONE_TABLE', - index=5, - number=5, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_SAVE_ALL_TABLE', - index=6, - number=6, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_LOAD_ONE_TABLE', - index=7, - number=7, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_LOAD_ALL_TABLE', - index=8, - number=8, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_CLEAR_ONE_TABLE', - index=9, - number=9, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_CLEAR_ALL_TABLE', - index=10, - number=10, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PUSH_DENSE_PARAM', - index=11, - number=11, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_STOP_SERVER', - index=12, - number=12, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_SAVE_ONE_CACHE_TABLE', - index=13, - number=13, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_GET_CACHE_THRESHOLD', - index=14, - number=14, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_CACHE_SHUFFLE', - index=15, - number=15, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_COPY_TABLE', - index=16, - number=16, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_COPY_TABLE_BY_FEASIGN', - index=17, - number=17, - options=None, - type=None), + _descriptor.EnumValueDescriptor( + name='PS_PULL_DENSE_TABLE', + index=0, + number=0, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PUSH_DENSE_TABLE', + index=1, + number=1, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PULL_SPARSE_TABLE', + index=2, + number=2, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PUSH_SPARSE_TABLE', + index=3, + number=3, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_SHRINK_TABLE', index=4, number=4, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_SAVE_ONE_TABLE', index=5, number=5, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_SAVE_ALL_TABLE', index=6, number=6, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_LOAD_ONE_TABLE', index=7, number=7, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_LOAD_ALL_TABLE', index=8, number=8, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_CLEAR_ONE_TABLE', + index=9, + number=9, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_CLEAR_ALL_TABLE', + index=10, + number=10, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PUSH_DENSE_PARAM', + index=11, + number=11, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_STOP_SERVER', index=12, number=12, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_SAVE_ONE_CACHE_TABLE', + index=13, + number=13, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_GET_CACHE_THRESHOLD', + index=14, + number=14, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_CACHE_SHUFFLE', + index=15, + number=15, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_COPY_TABLE', index=16, number=16, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='PS_COPY_TABLE_BY_FEASIGN', + index=17, + number=17, + options=None, + type=None, + ), _descriptor.EnumValueDescriptor( name='PS_PULL_SPARSE_TABLE_WITH_DEPENDENCY', index=18, number=18, options=None, - type=None), + type=None, + ), _descriptor.EnumValueDescriptor( name='PS_PUSH_SPARSE_TABLE_WITH_DEPENDENCY', index=19, number=19, options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_PRINT_TABLE_STAT', - index=20, - number=20, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='PS_S2S_MSG', - index=21, - number=101, - options=None, - type=None), + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_PRINT_TABLE_STAT', + index=20, + number=20, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name='PS_S2S_MSG', index=21, number=101, options=None, type=None + ), ], containing_type=None, options=None, @@ -220,16 +228,12 @@ filename=None, file=DESCRIPTOR, values=[ - _descriptor.EnumValueDescriptor(name='HDFS', - index=0, - number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor(name='AFS', - index=1, - number=1, - options=None, - type=None), + _descriptor.EnumValueDescriptor( + name='HDFS', index=0, number=0, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name='AFS', index=1, number=1, options=None, type=None + ), ], containing_type=None, options=None, @@ -245,36 +249,40 @@ file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='worker_class', - full_name='paddle.PSParameter.worker_class', - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='server_class', - full_name='paddle.PSParameter.server_class', - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='worker_class', + full_name='paddle.PSParameter.worker_class', + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='server_class', + full_name='paddle.PSParameter.server_class', + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='instance_class', full_name='paddle.PSParameter.instance_class', @@ -290,52 +298,59 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='init_gflags', - full_name='paddle.PSParameter.init_gflags', - index=3, - number=4, - type=9, - cpp_type=9, - label=1, - has_default_value=True, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='worker_param', - full_name='paddle.PSParameter.worker_param', - index=4, - number=101, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='server_param', - full_name='paddle.PSParameter.server_param', - index=5, - number=102, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='init_gflags', + full_name='paddle.PSParameter.init_gflags', + index=3, + number=4, + type=9, + cpp_type=9, + label=1, + has_default_value=True, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='worker_param', + full_name='paddle.PSParameter.worker_param', + index=4, + number=101, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='server_param', + full_name='paddle.PSParameter.server_param', + index=5, + number=102, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='trainer_param', full_name='paddle.PSParameter.trainer_param', @@ -351,7 +366,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='fs_client_param', full_name='paddle.PSParameter.fs_client_param', @@ -367,7 +383,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -403,7 +420,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -439,7 +457,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -475,7 +494,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -511,7 +531,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='sparse_table', full_name='paddle.DownpourTrainerParameter.sparse_table', @@ -527,7 +548,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='push_sparse_per_batch', full_name='paddle.DownpourTrainerParameter.push_sparse_per_batch', @@ -543,7 +565,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='push_dense_per_batch', full_name='paddle.DownpourTrainerParameter.push_dense_per_batch', @@ -559,7 +582,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='skip_op', full_name='paddle.DownpourTrainerParameter.skip_op', @@ -575,7 +599,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='program_config', full_name='paddle.DownpourTrainerParameter.program_config', @@ -591,7 +616,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -612,21 +638,23 @@ file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='program_id', - full_name='paddle.ProgramConfig.program_id', - index=0, - number=1, - type=9, - cpp_type=9, - label=2, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='program_id', + full_name='paddle.ProgramConfig.program_id', + index=0, + number=1, + type=9, + cpp_type=9, + label=2, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='push_sparse_table_id', full_name='paddle.ProgramConfig.push_sparse_table_id', @@ -642,7 +670,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='push_dense_table_id', full_name='paddle.ProgramConfig.push_dense_table_id', @@ -658,7 +687,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='pull_sparse_table_id', full_name='paddle.ProgramConfig.pull_sparse_table_id', @@ -674,7 +704,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='pull_dense_table_id', full_name='paddle.ProgramConfig.pull_dense_table_id', @@ -690,7 +721,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -726,7 +758,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='dense_variable_name', full_name='paddle.DenseTableParameter.dense_variable_name', @@ -742,7 +775,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='dense_gradient_variable_name', full_name='paddle.DenseTableParameter.dense_gradient_variable_name', @@ -758,7 +792,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='fea_dim', full_name='paddle.DenseTableParameter.fea_dim', @@ -774,7 +809,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -810,7 +846,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='feature_dim', full_name='paddle.SparseTableParameter.feature_dim', @@ -826,7 +863,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='slot_key', full_name='paddle.SparseTableParameter.slot_key', @@ -842,7 +880,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='slot_value', full_name='paddle.SparseTableParameter.slot_value', @@ -858,7 +897,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='slot_gradient', full_name='paddle.SparseTableParameter.slot_gradient', @@ -874,7 +914,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -910,7 +951,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='service_param', full_name='paddle.DownpourServerParameter.service_param', @@ -926,7 +968,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -962,7 +1005,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='client_class', full_name='paddle.ServerServiceParameter.client_class', @@ -978,7 +1022,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='service_class', full_name='paddle.ServerServiceParameter.service_class', @@ -994,7 +1039,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='start_server_port', full_name='paddle.ServerServiceParameter.start_server_port', @@ -1010,7 +1056,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='server_thread_num', full_name='paddle.ServerServiceParameter.server_thread_num', @@ -1026,7 +1073,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1047,21 +1095,23 @@ file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='table_id', - full_name='paddle.TableParameter.table_id', - index=0, - number=1, - type=4, - cpp_type=4, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='table_id', + full_name='paddle.TableParameter.table_id', + index=0, + number=1, + type=4, + cpp_type=4, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='table_class', full_name='paddle.TableParameter.table_class', @@ -1077,52 +1127,59 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='shard_num', - full_name='paddle.TableParameter.shard_num', - index=2, - number=3, - type=4, - cpp_type=4, - label=1, - has_default_value=True, - default_value=1000, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='accessor', - full_name='paddle.TableParameter.accessor', - index=3, - number=4, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='type', - full_name='paddle.TableParameter.type', - index=4, - number=5, - type=14, - cpp_type=8, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='shard_num', + full_name='paddle.TableParameter.shard_num', + index=2, + number=3, + type=4, + cpp_type=4, + label=1, + has_default_value=True, + default_value=1000, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='accessor', + full_name='paddle.TableParameter.accessor', + index=3, + number=4, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='type', + full_name='paddle.TableParameter.type', + index=4, + number=5, + type=14, + cpp_type=8, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='compress_in_save', full_name='paddle.TableParameter.compress_in_save', @@ -1138,7 +1195,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='enable_sparse_table_cache', full_name='paddle.TableParameter.enable_sparse_table_cache', @@ -1154,7 +1212,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='sparse_table_cache_rate', full_name='paddle.TableParameter.sparse_table_cache_rate', @@ -1170,7 +1229,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='sparse_table_cache_file_num', full_name='paddle.TableParameter.sparse_table_cache_file_num', @@ -1186,7 +1246,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1222,7 +1283,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='sparse_sgd_param', full_name='paddle.TableAccessorParameter.sparse_sgd_param', @@ -1238,7 +1300,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='dense_sgd_param', full_name='paddle.TableAccessorParameter.dense_sgd_param', @@ -1254,7 +1317,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='fea_dim', full_name='paddle.TableAccessorParameter.fea_dim', @@ -1270,7 +1334,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='embedx_dim', full_name='paddle.TableAccessorParameter.embedx_dim', @@ -1286,7 +1351,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='embedx_threshold', full_name='paddle.TableAccessorParameter.embedx_threshold', @@ -1302,7 +1368,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='downpour_accessor_param', full_name='paddle.TableAccessorParameter.downpour_accessor_param', @@ -1318,7 +1385,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='table_accessor_save_param', full_name='paddle.TableAccessorParameter.table_accessor_save_param', @@ -1334,7 +1402,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='sparse_commonsgd_param', full_name='paddle.TableAccessorParameter.sparse_commonsgd_param', @@ -1350,7 +1419,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='embed_sgd_param', full_name='paddle.TableAccessorParameter.embed_sgd_param', @@ -1366,7 +1436,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='embedx_sgd_param', full_name='paddle.TableAccessorParameter.embedx_sgd_param', @@ -1382,7 +1453,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1418,7 +1490,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='click_coeff', full_name='paddle.DownpourTableAccessorParameter.click_coeff', @@ -1434,7 +1507,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='base_threshold', full_name='paddle.DownpourTableAccessorParameter.base_threshold', @@ -1450,7 +1524,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='delta_threshold', full_name='paddle.DownpourTableAccessorParameter.delta_threshold', @@ -1466,7 +1541,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='delta_keep_days', full_name='paddle.DownpourTableAccessorParameter.delta_keep_days', @@ -1482,11 +1558,11 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='show_click_decay_rate', - full_name= - 'paddle.DownpourTableAccessorParameter.show_click_decay_rate', + full_name='paddle.DownpourTableAccessorParameter.show_click_decay_rate', index=5, number=6, type=2, @@ -1499,7 +1575,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='delete_threshold', full_name='paddle.DownpourTableAccessorParameter.delete_threshold', @@ -1515,11 +1592,11 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='delete_after_unseen_days', - full_name= - 'paddle.DownpourTableAccessorParameter.delete_after_unseen_days', + full_name='paddle.DownpourTableAccessorParameter.delete_after_unseen_days', index=7, number=8, type=2, @@ -1532,11 +1609,11 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='ssd_unseenday_threshold', - full_name= - 'paddle.DownpourTableAccessorParameter.ssd_unseenday_threshold', + full_name='paddle.DownpourTableAccessorParameter.ssd_unseenday_threshold', index=8, number=9, type=5, @@ -1549,7 +1626,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1585,7 +1663,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='converter', full_name='paddle.TableAccessorSaveParameter.converter', @@ -1601,7 +1680,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='deconverter', full_name='paddle.TableAccessorSaveParameter.deconverter', @@ -1617,7 +1697,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1638,21 +1719,23 @@ file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor(name='cmd_id', - full_name='paddle.PsRequestMessage.cmd_id', - index=0, - number=1, - type=13, - cpp_type=3, - label=2, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + _descriptor.FieldDescriptor( + name='cmd_id', + full_name='paddle.PsRequestMessage.cmd_id', + index=0, + number=1, + type=13, + cpp_type=3, + label=2, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='table_id', full_name='paddle.PsRequestMessage.table_id', @@ -1668,22 +1751,25 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='params', - full_name='paddle.PsRequestMessage.params', - index=2, - number=3, - type=12, - cpp_type=9, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='params', + full_name='paddle.PsRequestMessage.params', + index=2, + number=3, + type=12, + cpp_type=9, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='client_id', full_name='paddle.PsRequestMessage.client_id', @@ -1699,22 +1785,25 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='data', - full_name='paddle.PsRequestMessage.data', - index=4, - number=5, - type=12, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b(""), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='data', + full_name='paddle.PsRequestMessage.data', + index=4, + number=5, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), ], extensions=[], nested_types=[], @@ -1750,7 +1839,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_g2sum', full_name='paddle.SparseSGDRuleParameter.initial_g2sum', @@ -1766,7 +1856,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_range', full_name='paddle.SparseSGDRuleParameter.initial_range', @@ -1782,7 +1873,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='weight_bounds', full_name='paddle.SparseSGDRuleParameter.weight_bounds', @@ -1798,7 +1890,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1834,7 +1927,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='naive', full_name='paddle.SparseCommonSGDRuleParameter.naive', @@ -1850,7 +1944,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='adagrad', full_name='paddle.SparseCommonSGDRuleParameter.adagrad', @@ -1866,7 +1961,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='adam', full_name='paddle.SparseCommonSGDRuleParameter.adam', @@ -1882,7 +1978,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1918,7 +2015,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_range', full_name='paddle.SparseNaiveSGDRuleParameter.initial_range', @@ -1934,7 +2032,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='weight_bounds', full_name='paddle.SparseNaiveSGDRuleParameter.weight_bounds', @@ -1950,7 +2049,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -1986,7 +2086,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_g2sum', full_name='paddle.SparseAdagradSGDRuleParameter.initial_g2sum', @@ -2002,7 +2103,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_range', full_name='paddle.SparseAdagradSGDRuleParameter.initial_range', @@ -2018,7 +2120,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='weight_bounds', full_name='paddle.SparseAdagradSGDRuleParameter.weight_bounds', @@ -2034,7 +2137,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2070,7 +2174,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='initial_range', full_name='paddle.SparseAdamSGDParameter.initial_range', @@ -2086,7 +2191,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='beta1_decay_rate', full_name='paddle.SparseAdamSGDParameter.beta1_decay_rate', @@ -2102,7 +2208,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='beta2_decay_rate', full_name='paddle.SparseAdamSGDParameter.beta2_decay_rate', @@ -2118,7 +2225,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='ada_epsilon', full_name='paddle.SparseAdamSGDParameter.ada_epsilon', @@ -2134,7 +2242,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='weight_bounds', full_name='paddle.SparseAdamSGDParameter.weight_bounds', @@ -2150,7 +2259,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2186,7 +2296,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='adam', full_name='paddle.DenseSGDRuleParameter.adam', @@ -2202,7 +2313,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='naive', full_name='paddle.DenseSGDRuleParameter.naive', @@ -2218,7 +2330,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='summary', full_name='paddle.DenseSGDRuleParameter.summary', @@ -2234,7 +2347,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='moving_average', full_name='paddle.DenseSGDRuleParameter.moving_average', @@ -2250,7 +2364,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2286,7 +2401,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='avg_decay_rate', full_name='paddle.AdamSGDParameter.avg_decay_rate', @@ -2302,7 +2418,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='ada_decay_rate', full_name='paddle.AdamSGDParameter.ada_decay_rate', @@ -2318,7 +2435,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='ada_epsilon', full_name='paddle.AdamSGDParameter.ada_epsilon', @@ -2334,7 +2452,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='mom_decay_rate', full_name='paddle.AdamSGDParameter.mom_decay_rate', @@ -2350,7 +2469,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2386,7 +2506,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='avg_decay_rate', full_name='paddle.NaiveSGDParameter.avg_decay_rate', @@ -2402,7 +2523,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2438,7 +2560,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2474,7 +2597,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2510,7 +2634,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='err_msg', full_name='paddle.PsResponseMessage.err_msg', @@ -2526,22 +2651,25 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='data', - full_name='paddle.PsResponseMessage.data', - index=2, - number=3, - type=12, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b(""), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='data', + full_name='paddle.PsResponseMessage.data', + index=2, + number=3, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), ], extensions=[], nested_types=[], @@ -2577,52 +2705,59 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='uri', - full_name='paddle.FsClientParameter.uri', - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='user', - full_name='paddle.FsClientParameter.user', - index=2, - number=3, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), - _descriptor.FieldDescriptor(name='passwd', - full_name='paddle.FsClientParameter.passwd', - index=3, - number=4, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode('utf-8'), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None), + options=None, + ), + _descriptor.FieldDescriptor( + name='uri', + full_name='paddle.FsClientParameter.uri', + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='user', + full_name='paddle.FsClientParameter.user', + index=2, + number=3, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name='passwd', + full_name='paddle.FsClientParameter.passwd', + index=3, + number=4, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode('utf-8'), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), _descriptor.FieldDescriptor( name='buffer_size', full_name='paddle.FsClientParameter.buffer_size', @@ -2638,7 +2773,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='hadoop_bin', full_name='paddle.FsClientParameter.hadoop_bin', @@ -2654,7 +2790,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), _descriptor.FieldDescriptor( name='afs_conf', full_name='paddle.FsClientParameter.afs_conf', @@ -2670,7 +2807,8 @@ containing_type=None, is_extension=False, extension_scope=None, - options=None), + options=None, + ), ], extensions=[], nested_types=[], @@ -2689,95 +2827,132 @@ _PSPARAMETER.fields_by_name['worker_param'].message_type = _WORKERPARAMETER _PSPARAMETER.fields_by_name['server_param'].message_type = _SERVERPARAMETER _PSPARAMETER.fields_by_name[ - 'trainer_param'].message_type = _DOWNPOURTRAINERPARAMETER + 'trainer_param' +].message_type = _DOWNPOURTRAINERPARAMETER _PSPARAMETER.fields_by_name['fs_client_param'].message_type = _FSCLIENTPARAMETER _WORKERPARAMETER.fields_by_name[ - 'downpour_worker_param'].message_type = _DOWNPOURWORKERPARAMETER + 'downpour_worker_param' +].message_type = _DOWNPOURWORKERPARAMETER _SERVERPARAMETER.fields_by_name[ - 'downpour_server_param'].message_type = _DOWNPOURSERVERPARAMETER + 'downpour_server_param' +].message_type = _DOWNPOURSERVERPARAMETER _DOWNPOURWORKERPARAMETER.fields_by_name[ - 'downpour_table_param'].message_type = _TABLEPARAMETER + 'downpour_table_param' +].message_type = _TABLEPARAMETER _DOWNPOURTRAINERPARAMETER.fields_by_name[ - 'dense_table'].message_type = _DENSETABLEPARAMETER + 'dense_table' +].message_type = _DENSETABLEPARAMETER _DOWNPOURTRAINERPARAMETER.fields_by_name[ - 'sparse_table'].message_type = _SPARSETABLEPARAMETER + 'sparse_table' +].message_type = _SPARSETABLEPARAMETER _DOWNPOURTRAINERPARAMETER.fields_by_name[ - 'program_config'].message_type = _PROGRAMCONFIG + 'program_config' +].message_type = _PROGRAMCONFIG _DOWNPOURSERVERPARAMETER.fields_by_name[ - 'downpour_table_param'].message_type = _TABLEPARAMETER + 'downpour_table_param' +].message_type = _TABLEPARAMETER _DOWNPOURSERVERPARAMETER.fields_by_name[ - 'service_param'].message_type = _SERVERSERVICEPARAMETER + 'service_param' +].message_type = _SERVERSERVICEPARAMETER _TABLEPARAMETER.fields_by_name[ - 'accessor'].message_type = _TABLEACCESSORPARAMETER + 'accessor' +].message_type = _TABLEACCESSORPARAMETER _TABLEPARAMETER.fields_by_name['type'].enum_type = _TABLETYPE _TABLEACCESSORPARAMETER.fields_by_name[ - 'sparse_sgd_param'].message_type = _SPARSESGDRULEPARAMETER + 'sparse_sgd_param' +].message_type = _SPARSESGDRULEPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'dense_sgd_param'].message_type = _DENSESGDRULEPARAMETER + 'dense_sgd_param' +].message_type = _DENSESGDRULEPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'downpour_accessor_param'].message_type = _DOWNPOURTABLEACCESSORPARAMETER + 'downpour_accessor_param' +].message_type = _DOWNPOURTABLEACCESSORPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'table_accessor_save_param'].message_type = _TABLEACCESSORSAVEPARAMETER + 'table_accessor_save_param' +].message_type = _TABLEACCESSORSAVEPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'sparse_commonsgd_param'].message_type = _SPARSECOMMONSGDRULEPARAMETER + 'sparse_commonsgd_param' +].message_type = _SPARSECOMMONSGDRULEPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'embed_sgd_param'].message_type = _SPARSECOMMONSGDRULEPARAMETER + 'embed_sgd_param' +].message_type = _SPARSECOMMONSGDRULEPARAMETER _TABLEACCESSORPARAMETER.fields_by_name[ - 'embedx_sgd_param'].message_type = _SPARSECOMMONSGDRULEPARAMETER + 'embedx_sgd_param' +].message_type = _SPARSECOMMONSGDRULEPARAMETER _SPARSECOMMONSGDRULEPARAMETER.fields_by_name[ - 'naive'].message_type = _SPARSENAIVESGDRULEPARAMETER + 'naive' +].message_type = _SPARSENAIVESGDRULEPARAMETER _SPARSECOMMONSGDRULEPARAMETER.fields_by_name[ - 'adagrad'].message_type = _SPARSEADAGRADSGDRULEPARAMETER + 'adagrad' +].message_type = _SPARSEADAGRADSGDRULEPARAMETER _SPARSECOMMONSGDRULEPARAMETER.fields_by_name[ - 'adam'].message_type = _SPARSEADAMSGDPARAMETER + 'adam' +].message_type = _SPARSEADAMSGDPARAMETER _DENSESGDRULEPARAMETER.fields_by_name['adam'].message_type = _ADAMSGDPARAMETER _DENSESGDRULEPARAMETER.fields_by_name['naive'].message_type = _NAIVESGDPARAMETER _DENSESGDRULEPARAMETER.fields_by_name[ - 'summary'].message_type = _SUMMARYSGDPARAMETER + 'summary' +].message_type = _SUMMARYSGDPARAMETER _DENSESGDRULEPARAMETER.fields_by_name[ - 'moving_average'].message_type = _MOVINGAVERAGERULEPARAMETER + 'moving_average' +].message_type = _MOVINGAVERAGERULEPARAMETER _FSCLIENTPARAMETER.fields_by_name[ - 'fs_type'].enum_type = _FSCLIENTPARAMETER_FSAPITYPE + 'fs_type' +].enum_type = _FSCLIENTPARAMETER_FSAPITYPE _FSCLIENTPARAMETER_FSAPITYPE.containing_type = _FSCLIENTPARAMETER DESCRIPTOR.message_types_by_name['PSParameter'] = _PSPARAMETER DESCRIPTOR.message_types_by_name['WorkerParameter'] = _WORKERPARAMETER DESCRIPTOR.message_types_by_name['ServerParameter'] = _SERVERPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourWorkerParameter'] = _DOWNPOURWORKERPARAMETER + 'DownpourWorkerParameter' +] = _DOWNPOURWORKERPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourTrainerParameter'] = _DOWNPOURTRAINERPARAMETER + 'DownpourTrainerParameter' +] = _DOWNPOURTRAINERPARAMETER DESCRIPTOR.message_types_by_name['ProgramConfig'] = _PROGRAMCONFIG DESCRIPTOR.message_types_by_name['DenseTableParameter'] = _DENSETABLEPARAMETER DESCRIPTOR.message_types_by_name['SparseTableParameter'] = _SPARSETABLEPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourServerParameter'] = _DOWNPOURSERVERPARAMETER + 'DownpourServerParameter' +] = _DOWNPOURSERVERPARAMETER DESCRIPTOR.message_types_by_name[ - 'ServerServiceParameter'] = _SERVERSERVICEPARAMETER + 'ServerServiceParameter' +] = _SERVERSERVICEPARAMETER DESCRIPTOR.message_types_by_name['TableParameter'] = _TABLEPARAMETER DESCRIPTOR.message_types_by_name[ - 'TableAccessorParameter'] = _TABLEACCESSORPARAMETER + 'TableAccessorParameter' +] = _TABLEACCESSORPARAMETER DESCRIPTOR.message_types_by_name[ - 'DownpourTableAccessorParameter'] = _DOWNPOURTABLEACCESSORPARAMETER + 'DownpourTableAccessorParameter' +] = _DOWNPOURTABLEACCESSORPARAMETER DESCRIPTOR.message_types_by_name[ - 'TableAccessorSaveParameter'] = _TABLEACCESSORSAVEPARAMETER + 'TableAccessorSaveParameter' +] = _TABLEACCESSORSAVEPARAMETER DESCRIPTOR.message_types_by_name['PsRequestMessage'] = _PSREQUESTMESSAGE DESCRIPTOR.message_types_by_name[ - 'SparseSGDRuleParameter'] = _SPARSESGDRULEPARAMETER + 'SparseSGDRuleParameter' +] = _SPARSESGDRULEPARAMETER DESCRIPTOR.message_types_by_name[ - 'SparseCommonSGDRuleParameter'] = _SPARSECOMMONSGDRULEPARAMETER + 'SparseCommonSGDRuleParameter' +] = _SPARSECOMMONSGDRULEPARAMETER DESCRIPTOR.message_types_by_name[ - 'SparseNaiveSGDRuleParameter'] = _SPARSENAIVESGDRULEPARAMETER + 'SparseNaiveSGDRuleParameter' +] = _SPARSENAIVESGDRULEPARAMETER DESCRIPTOR.message_types_by_name[ - 'SparseAdagradSGDRuleParameter'] = _SPARSEADAGRADSGDRULEPARAMETER + 'SparseAdagradSGDRuleParameter' +] = _SPARSEADAGRADSGDRULEPARAMETER DESCRIPTOR.message_types_by_name[ - 'SparseAdamSGDParameter'] = _SPARSEADAMSGDPARAMETER + 'SparseAdamSGDParameter' +] = _SPARSEADAMSGDPARAMETER DESCRIPTOR.message_types_by_name[ - 'DenseSGDRuleParameter'] = _DENSESGDRULEPARAMETER + 'DenseSGDRuleParameter' +] = _DENSESGDRULEPARAMETER DESCRIPTOR.message_types_by_name['AdamSGDParameter'] = _ADAMSGDPARAMETER DESCRIPTOR.message_types_by_name['NaiveSGDParameter'] = _NAIVESGDPARAMETER DESCRIPTOR.message_types_by_name['SummarySGDParameter'] = _SUMMARYSGDPARAMETER DESCRIPTOR.message_types_by_name[ - 'MovingAverageRuleParameter'] = _MOVINGAVERAGERULEPARAMETER + 'MovingAverageRuleParameter' +] = _MOVINGAVERAGERULEPARAMETER DESCRIPTOR.message_types_by_name['PsResponseMessage'] = _PSRESPONSEMESSAGE DESCRIPTOR.message_types_by_name['FsClientParameter'] = _FSCLIENTPARAMETER DESCRIPTOR.enum_types_by_name['TableType'] = _TABLETYPE @@ -2785,254 +2960,303 @@ PSParameter = _reflection.GeneratedProtocolMessageType( 'PSParameter', - (_message.Message, ), - dict(DESCRIPTOR=_PSPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.PSParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PSPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.PSParameter) + ), +) _sym_db.RegisterMessage(PSParameter) WorkerParameter = _reflection.GeneratedProtocolMessageType( 'WorkerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_WORKERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.WorkerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_WORKERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.WorkerParameter) + ), +) _sym_db.RegisterMessage(WorkerParameter) ServerParameter = _reflection.GeneratedProtocolMessageType( 'ServerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SERVERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.ServerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SERVERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.ServerParameter) + ), +) _sym_db.RegisterMessage(ServerParameter) DownpourWorkerParameter = _reflection.GeneratedProtocolMessageType( 'DownpourWorkerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DOWNPOURWORKERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DownpourWorkerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DOWNPOURWORKERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DownpourWorkerParameter) + ), +) _sym_db.RegisterMessage(DownpourWorkerParameter) DownpourTrainerParameter = _reflection.GeneratedProtocolMessageType( 'DownpourTrainerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DOWNPOURTRAINERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DownpourTrainerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DOWNPOURTRAINERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DownpourTrainerParameter) + ), +) _sym_db.RegisterMessage(DownpourTrainerParameter) ProgramConfig = _reflection.GeneratedProtocolMessageType( 'ProgramConfig', - (_message.Message, ), - dict(DESCRIPTOR=_PROGRAMCONFIG, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.ProgramConfig) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PROGRAMCONFIG, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.ProgramConfig) + ), +) _sym_db.RegisterMessage(ProgramConfig) DenseTableParameter = _reflection.GeneratedProtocolMessageType( 'DenseTableParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DENSETABLEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DenseTableParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DENSETABLEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DenseTableParameter) + ), +) _sym_db.RegisterMessage(DenseTableParameter) SparseTableParameter = _reflection.GeneratedProtocolMessageType( 'SparseTableParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SPARSETABLEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.SparseTableParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SPARSETABLEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.SparseTableParameter) + ), +) _sym_db.RegisterMessage(SparseTableParameter) DownpourServerParameter = _reflection.GeneratedProtocolMessageType( 'DownpourServerParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DOWNPOURSERVERPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DownpourServerParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DOWNPOURSERVERPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DownpourServerParameter) + ), +) _sym_db.RegisterMessage(DownpourServerParameter) ServerServiceParameter = _reflection.GeneratedProtocolMessageType( 'ServerServiceParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SERVERSERVICEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.ServerServiceParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SERVERSERVICEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.ServerServiceParameter) + ), +) _sym_db.RegisterMessage(ServerServiceParameter) TableParameter = _reflection.GeneratedProtocolMessageType( 'TableParameter', - (_message.Message, ), - dict(DESCRIPTOR=_TABLEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.TableParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_TABLEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.TableParameter) + ), +) _sym_db.RegisterMessage(TableParameter) TableAccessorParameter = _reflection.GeneratedProtocolMessageType( 'TableAccessorParameter', - (_message.Message, ), - dict(DESCRIPTOR=_TABLEACCESSORPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.TableAccessorParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_TABLEACCESSORPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.TableAccessorParameter) + ), +) _sym_db.RegisterMessage(TableAccessorParameter) DownpourTableAccessorParameter = _reflection.GeneratedProtocolMessageType( 'DownpourTableAccessorParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_DOWNPOURTABLEACCESSORPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.DownpourTableAccessorParameter) - )) + ), +) _sym_db.RegisterMessage(DownpourTableAccessorParameter) TableAccessorSaveParameter = _reflection.GeneratedProtocolMessageType( 'TableAccessorSaveParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_TABLEACCESSORSAVEPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.TableAccessorSaveParameter) - )) + ), +) _sym_db.RegisterMessage(TableAccessorSaveParameter) PsRequestMessage = _reflection.GeneratedProtocolMessageType( 'PsRequestMessage', - (_message.Message, ), - dict(DESCRIPTOR=_PSREQUESTMESSAGE, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.PsRequestMessage) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PSREQUESTMESSAGE, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.PsRequestMessage) + ), +) _sym_db.RegisterMessage(PsRequestMessage) SparseSGDRuleParameter = _reflection.GeneratedProtocolMessageType( 'SparseSGDRuleParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SPARSESGDRULEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.SparseSGDRuleParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SPARSESGDRULEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.SparseSGDRuleParameter) + ), +) _sym_db.RegisterMessage(SparseSGDRuleParameter) SparseCommonSGDRuleParameter = _reflection.GeneratedProtocolMessageType( 'SparseCommonSGDRuleParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_SPARSECOMMONSGDRULEPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.SparseCommonSGDRuleParameter) - )) + ), +) _sym_db.RegisterMessage(SparseCommonSGDRuleParameter) SparseNaiveSGDRuleParameter = _reflection.GeneratedProtocolMessageType( 'SparseNaiveSGDRuleParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_SPARSENAIVESGDRULEPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.SparseNaiveSGDRuleParameter) - )) + ), +) _sym_db.RegisterMessage(SparseNaiveSGDRuleParameter) SparseAdagradSGDRuleParameter = _reflection.GeneratedProtocolMessageType( 'SparseAdagradSGDRuleParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_SPARSEADAGRADSGDRULEPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.SparseAdagradSGDRuleParameter) - )) + ), +) _sym_db.RegisterMessage(SparseAdagradSGDRuleParameter) SparseAdamSGDParameter = _reflection.GeneratedProtocolMessageType( 'SparseAdamSGDParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SPARSEADAMSGDPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.SparseAdamSGDParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SPARSEADAMSGDPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.SparseAdamSGDParameter) + ), +) _sym_db.RegisterMessage(SparseAdamSGDParameter) DenseSGDRuleParameter = _reflection.GeneratedProtocolMessageType( 'DenseSGDRuleParameter', - (_message.Message, ), - dict(DESCRIPTOR=_DENSESGDRULEPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.DenseSGDRuleParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_DENSESGDRULEPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.DenseSGDRuleParameter) + ), +) _sym_db.RegisterMessage(DenseSGDRuleParameter) AdamSGDParameter = _reflection.GeneratedProtocolMessageType( 'AdamSGDParameter', - (_message.Message, ), - dict(DESCRIPTOR=_ADAMSGDPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.AdamSGDParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_ADAMSGDPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.AdamSGDParameter) + ), +) _sym_db.RegisterMessage(AdamSGDParameter) NaiveSGDParameter = _reflection.GeneratedProtocolMessageType( 'NaiveSGDParameter', - (_message.Message, ), - dict(DESCRIPTOR=_NAIVESGDPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.NaiveSGDParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_NAIVESGDPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.NaiveSGDParameter) + ), +) _sym_db.RegisterMessage(NaiveSGDParameter) SummarySGDParameter = _reflection.GeneratedProtocolMessageType( 'SummarySGDParameter', - (_message.Message, ), - dict(DESCRIPTOR=_SUMMARYSGDPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.SummarySGDParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_SUMMARYSGDPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.SummarySGDParameter) + ), +) _sym_db.RegisterMessage(SummarySGDParameter) MovingAverageRuleParameter = _reflection.GeneratedProtocolMessageType( 'MovingAverageRuleParameter', - (_message.Message, ), + (_message.Message,), dict( DESCRIPTOR=_MOVINGAVERAGERULEPARAMETER, __module__='ps_pb2' # @@protoc_insertion_point(class_scope:paddle.MovingAverageRuleParameter) - )) + ), +) _sym_db.RegisterMessage(MovingAverageRuleParameter) PsResponseMessage = _reflection.GeneratedProtocolMessageType( 'PsResponseMessage', - (_message.Message, ), - dict(DESCRIPTOR=_PSRESPONSEMESSAGE, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.PsResponseMessage) - )) + (_message.Message,), + dict( + DESCRIPTOR=_PSRESPONSEMESSAGE, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.PsResponseMessage) + ), +) _sym_db.RegisterMessage(PsResponseMessage) FsClientParameter = _reflection.GeneratedProtocolMessageType( 'FsClientParameter', - (_message.Message, ), - dict(DESCRIPTOR=_FSCLIENTPARAMETER, - __module__='ps_pb2' - # @@protoc_insertion_point(class_scope:paddle.FsClientParameter) - )) + (_message.Message,), + dict( + DESCRIPTOR=_FSCLIENTPARAMETER, + __module__='ps_pb2' + # @@protoc_insertion_point(class_scope:paddle.FsClientParameter) + ), +) _sym_db.RegisterMessage(FsClientParameter) DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), - _b('\200\001\001\370\001\001')) +DESCRIPTOR._options = _descriptor._ParseOptions( + descriptor_pb2.FileOptions(), _b('\200\001\001\370\001\001') +) # @@protoc_insertion_point(module_scope) diff --git a/python/paddle/fluid/incubate/fleet/tests/ctr_dataset_reader.py b/python/paddle/fluid/incubate/fleet/tests/ctr_dataset_reader.py index 2530a611336c0..97914fe16123b 100644 --- a/python/paddle/fluid/incubate/fleet/tests/ctr_dataset_reader.py +++ b/python/paddle/fluid/incubate/fleet/tests/ctr_dataset_reader.py @@ -20,9 +20,9 @@ import paddle.distributed.fleet as fleet from paddle.fluid.log_helper import get_logger -logger = get_logger("paddle", - logging.INFO, - fmt='%(asctime)s - %(levelname)s - %(message)s') +logger = get_logger( + "paddle", logging.INFO, fmt='%(asctime)s - %(levelname)s - %(message)s' +) DATA_URL = "http://paddle-ctr-data.bj.bcebos.com/avazu_ctr_data.tgz" DATA_MD5 = "c11df99fbd14e53cd4bfa6567344b26e" @@ -59,17 +59,16 @@ def load_lr_input_record(sent): class DatasetCtrReader(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def iter(): fs = line.strip().split('\t') dnn_input = load_dnn_input_record(fs[0]) lr_input = load_lr_input_record(fs[1]) click = [int(fs[2])] - yield ("dnn_data", dnn_input), \ - ("lr_data", lr_input), \ - ("click", click) + yield ("dnn_data", dnn_input), ("lr_data", lr_input), ( + "click", + click, + ) return iter @@ -85,7 +84,9 @@ def prepare_data(): lines = f.readlines() err_info = "wrong meta format" assert len(lines) == 2, err_info - assert 'dnn_input_dim:' in lines[0] and 'lr_input_dim:' in lines[1], err_info + assert ( + 'dnn_input_dim:' in lines[0] and 'lr_input_dim:' in lines[1] + ), err_info res = map(int, [_.split(':')[1] for _ in lines]) res = list(res) dnn_input_dim = res[0] diff --git a/python/paddle/fluid/incubate/fleet/tests/fleet_deep_ctr.py b/python/paddle/fluid/incubate/fleet/tests/fleet_deep_ctr.py index 1b763c6ed5952..d2c843ca4d0b5 100644 --- a/python/paddle/fluid/incubate/fleet/tests/fleet_deep_ctr.py +++ b/python/paddle/fluid/incubate/fleet/tests/fleet_deep_ctr.py @@ -19,16 +19,20 @@ import paddle import paddle.fluid as fluid import paddle.fluid.incubate.fleet.base.role_maker as role_maker -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, +) +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + StrategyFactory, +) from paddle.fluid.log_helper import get_logger import ctr_dataset_reader -logger = get_logger("fluid", - logging.INFO, - fmt='%(asctime)s - %(levelname)s - %(message)s') +logger = get_logger( + "fluid", logging.INFO, fmt='%(asctime)s - %(levelname)s - %(message)s' +) def parse_args(): @@ -39,48 +43,64 @@ def parse_args(): '--role', type=str, default='pserver', # trainer or pserver - help='The path for model to store (default: models)') + help='The path for model to store (default: models)', + ) parser.add_argument( '--endpoints', type=str, default='127.0.0.1:6000', - help='The pserver endpoints, like: 127.0.0.1:6000,127.0.0.1:6001') + help='The pserver endpoints, like: 127.0.0.1:6000,127.0.0.1:6001', + ) parser.add_argument( '--current_endpoint', type=str, default='127.0.0.1:6000', - help='The path for model to store (default: 127.0.0.1:6000)') - parser.add_argument('--trainer_id', - type=int, - default=0, - help='The path for model to store (default: models)') - parser.add_argument('--trainers', - type=int, - default=1, - help='The num of trainers, (default: 1)') + help='The path for model to store (default: 127.0.0.1:6000)', + ) + parser.add_argument( + '--trainer_id', + type=int, + default=0, + help='The path for model to store (default: models)', + ) + parser.add_argument( + '--trainers', + type=int, + default=1, + help='The num of trainers, (default: 1)', + ) return parser.parse_args() def model(): - dnn_input_dim, lr_input_dim, train_file_path = ctr_dataset_reader.prepare_data( - ) + ( + dnn_input_dim, + lr_input_dim, + train_file_path, + ) = ctr_dataset_reader.prepare_data() """ network definition """ - dnn_data = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - lr_data = fluid.layers.data(name="lr_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="int64", - lod_level=0, - append_batch_size=False) + dnn_data = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + lr_data = fluid.layers.data( + name="lr_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=0, + append_batch_size=False, + ) datas = [dnn_data, lr_data, label] @@ -92,8 +112,10 @@ def model(): size=[dnn_input_dim, dnn_layer_dims[0]], param_attr=fluid.ParamAttr( name="deep_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=True) + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) dnn_pool = fluid.layers.sequence_pool(input=dnn_embedding, pool_type="sum") dnn_out = dnn_pool for i, dim in enumerate(dnn_layer_dims[1:]): @@ -101,9 +123,11 @@ def model(): input=dnn_out, size=dim, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01)), - name='dnn-fc-%d' % i) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + name='dnn-fc-%d' % i, + ) dnn_out = fc # build lr model @@ -113,16 +137,19 @@ def model(): size=[lr_input_dim, 1], param_attr=fluid.ParamAttr( name="wide_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=True) + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) lr_pool = fluid.layers.sequence_pool(input=lr_embbding, pool_type="sum") merge_layer = fluid.layers.concat(input=[dnn_out, lr_pool], axis=1) predict = fluid.layers.fc(input=merge_layer, size=2, act='softmax') acc = fluid.layers.accuracy(input=predict, label=label) - auc_var, batch_auc_var, auc_states = fluid.layers.auc(input=predict, - label=label) + auc_var, batch_auc_var, auc_states = fluid.layers.auc( + input=predict, label=label + ) cost = fluid.layers.cross_entropy(input=predict, label=label) avg_cost = paddle.mean(x=cost) @@ -140,9 +167,11 @@ def train(args): role = role_maker.UserDefinedRoleMaker( current_id=current_id, role=role_maker.Role.WORKER - if args.role.upper() == "TRAINER" else role_maker.Role.SERVER, + if args.role.upper() == "TRAINER" + else role_maker.Role.SERVER, worker_num=args.trainers, - server_endpoints=endpoints) + server_endpoints=endpoints, + ) exe = fluid.Executor(fluid.CPUPlace()) fleet.init(role) @@ -183,15 +212,18 @@ def train(args): logger.info("epoch {} start".format(epoch_id)) pass_start = time.time() dataset.set_filelist(filelist) - exe.train_from_dataset(program=fleet.main_program, - dataset=dataset, - fetch_list=[avg_cost], - fetch_info=["cost"], - print_period=100, - debug=False) + exe.train_from_dataset( + program=fleet.main_program, + dataset=dataset, + fetch_list=[avg_cost], + fetch_info=["cost"], + print_period=100, + debug=False, + ) pass_time = time.time() - pass_start - logger.info("epoch {} finished, pass_time {}".format( - epoch_id, pass_time)) + logger.info( + "epoch {} finished, pass_time {}".format(epoch_id, pass_time) + ) fleet.stop_worker() diff --git a/python/paddle/fluid/incubate/fleet/utils/fleet_util.py b/python/paddle/fluid/incubate/fleet/utils/fleet_util.py index 365043f92d007..b15892e0f5ec0 100644 --- a/python/paddle/fluid/incubate/fleet/utils/fleet_util.py +++ b/python/paddle/fluid/incubate/fleet/utils/fleet_util.py @@ -32,9 +32,9 @@ __all__ = ["FleetUtil", "GPUPSUtil"] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s %(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s %(levelname)s: %(message)s' +) fleet = None @@ -57,14 +57,21 @@ def __init__(self, mode="pslib"): op_maker = core.op_proto_and_checker_maker self.op_role_key = op_maker.kOpRoleAttrName() if mode == "pslib": - from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet as fleet_pslib + from paddle.fluid.incubate.fleet.parameter_server.pslib import ( + fleet as fleet_pslib, + ) + fleet = fleet_pslib elif mode == "transpiler": - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet as fleet_transpiler + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet as fleet_transpiler, + ) + fleet = fleet_transpiler else: raise ValueError( - "Please choose one mode from [\"pslib\", \"transpiler\"]") + "Please choose one mode from [\"pslib\", \"transpiler\"]" + ) def rank0_print(self, s): """ @@ -124,11 +131,13 @@ def rank0_error(self, s): return _logger.error(s) - def set_zero(self, - var_name, - scope=fluid.global_scope(), - place=fluid.CPUPlace(), - param_type="int64"): + def set_zero( + self, + var_name, + scope=fluid.global_scope(), + place=fluid.CPUPlace(), + param_type="int64", + ): """ Set tensor of a Variable to zero. @@ -150,11 +159,13 @@ def set_zero(self, param_array = np.zeros(param._get_dims()).astype(param_type) param.set(param_array, place) - def print_global_auc(self, - scope=fluid.global_scope(), - stat_pos="_generated_var_2", - stat_neg="_generated_var_3", - print_prefix=""): + def print_global_auc( + self, + scope=fluid.global_scope(), + stat_pos="_generated_var_2", + stat_neg="_generated_var_3", + print_prefix="", + ): r""" Print global auc of all distributed workers. @@ -189,10 +200,12 @@ def print_global_auc(self, auc_value = self.get_global_auc(scope, stat_pos, stat_neg) self.rank0_print(print_prefix + " global auc = %s" % auc_value) - def get_global_auc(self, - scope=fluid.global_scope(), - stat_pos="_generated_var_2", - stat_neg="_generated_var_3"): + def get_global_auc( + self, + scope=fluid.global_scope(), + stat_pos="_generated_var_2", + stat_neg="_generated_var_3", + ): """ Get global auc of all distributed workers. @@ -324,15 +337,17 @@ def save_fleet_model(self, path, mode=0): """ fleet.save_persistables(None, path, mode=mode) - def _get_xbox_str(self, - output_path, - day, - model_path, - xbox_base_key, - data_path, - hadoop_fs_name, - monitor_data={}, - mode="patch"): + def _get_xbox_str( + self, + output_path, + day, + model_path, + xbox_base_key, + data_path, + hadoop_fs_name, + monitor_data={}, + mode="patch", + ): xbox_dict = collections.OrderedDict() if mode == "base": xbox_dict["id"] = str(xbox_base_key) @@ -344,7 +359,7 @@ def _get_xbox_str(self, xbox_dict["id"] = str(int(time.time())) xbox_dict["key"] = str(xbox_base_key) if model_path.startswith("hdfs:") or model_path.startswith("afs:"): - model_path = model_path[model_path.find(":") + 1:] + model_path = model_path[model_path.find(":") + 1 :] xbox_dict["input"] = hadoop_fs_name + model_path.rstrip("/") + "/000" xbox_dict["record_count"] = "111111" xbox_dict["partition_type"] = "2" @@ -360,20 +375,23 @@ def _get_xbox_str(self, xbox_dict["job_id"] = job_id_with_host # currently hard code here, set monitor_data empty string xbox_dict["monitor_data"] = "" - xbox_dict["monitor_path"] = output_path.rstrip("/") + "/monitor/" \ - + day + ".txt" + xbox_dict["monitor_path"] = ( + output_path.rstrip("/") + "/monitor/" + day + ".txt" + ) xbox_dict["mpi_size"] = str(fleet.worker_num()) return json.dumps(xbox_dict) - def write_model_donefile(self, - output_path, - day, - pass_id, - xbox_base_key, - hadoop_fs_name, - hadoop_fs_ugi, - hadoop_home="$HADOOP_HOME", - donefile_name="donefile.txt"): + def write_model_donefile( + self, + output_path, + day, + pass_id, + xbox_base_key, + hadoop_fs_name, + hadoop_fs_ugi, + hadoop_home="$HADOOP_HOME", + donefile_name="donefile.txt", + ): """ write donefile when save model @@ -414,11 +432,16 @@ def write_model_donefile(self, if fleet.worker_index() == 0: donefile_path = output_path + "/" + donefile_name - content = "%s\t%lu\t%s\t%s\t%d" % (day, xbox_base_key,\ - model_path, pass_id, 0) + content = "%s\t%lu\t%s\t%s\t%d" % ( + day, + xbox_base_key, + model_path, + pass_id, + 0, + ) configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if client.is_file(donefile_path): @@ -428,8 +451,9 @@ def write_model_donefile(self, pass_list = [i.split("\t")[3] for i in pre_content_list] exist = False for i in range(len(day_list)): - if int(day) == int(day_list[i]) and \ - int(pass_id) == int(pass_list[i]): + if int(day) == int(day_list[i]) and int(pass_id) == int( + pass_list[i] + ): exist = True break if not exist: @@ -438,30 +462,36 @@ def write_model_donefile(self, f.write(content + "\n") client.delete(donefile_path) client.upload(donefile_name, output_path) - self.rank0_error("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) + self.rank0_error( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) else: - self.rank0_error("not write %s because %s/%s already " - "exists" % (donefile_name, day, pass_id)) + self.rank0_error( + "not write %s because %s/%s already " + "exists" % (donefile_name, day, pass_id) + ) else: with open(donefile_name, "w") as f: f.write(content + "\n") client.upload(donefile_name, output_path) - self.rank0_error("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) + self.rank0_error( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) fleet._role_maker._barrier_worker() - def write_xbox_donefile(self, - output_path, - day, - pass_id, - xbox_base_key, - data_path, - hadoop_fs_name, - hadoop_fs_ugi, - monitor_data={}, - hadoop_home="$HADOOP_HOME", - donefile_name=None): + def write_xbox_donefile( + self, + output_path, + day, + pass_id, + xbox_base_key, + data_path, + hadoop_fs_name, + hadoop_fs_ugi, + monitor_data={}, + hadoop_home="$HADOOP_HOME", + donefile_name=None, + ): """ write delta donefile or xbox base donefile @@ -518,12 +548,19 @@ def write_xbox_donefile(self, if fleet.worker_index() == 0: donefile_path = output_path + "/" + donefile_name - xbox_str = self._get_xbox_str(output_path, day, model_path, \ - xbox_base_key, data_path, hadoop_fs_name, monitor_data={}, - mode=mode) + xbox_str = self._get_xbox_str( + output_path, + day, + model_path, + xbox_base_key, + data_path, + hadoop_fs_name, + monitor_data={}, + mode=mode, + ) configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if client.is_file(donefile_path): @@ -532,9 +569,11 @@ def write_xbox_donefile(self, last_day = last_dict["input"].split("/")[-3] last_pass = last_dict["input"].split("/")[-2].split("-")[-1] exist = False - if int(day) < int(last_day) or \ - int(day) == int(last_day) and \ - int(pass_id) <= int(last_pass): + if ( + int(day) < int(last_day) + or int(day) == int(last_day) + and int(pass_id) <= int(last_pass) + ): exist = True if not exist: with open(donefile_name, "w") as f: @@ -542,29 +581,35 @@ def write_xbox_donefile(self, f.write(xbox_str + "\n") client.delete(donefile_path) client.upload(donefile_name, output_path) - self.rank0_error("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) + self.rank0_error( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) else: - self.rank0_error("not write %s because %s/%s already " - "exists" % (donefile_name, day, pass_id)) + self.rank0_error( + "not write %s because %s/%s already " + "exists" % (donefile_name, day, pass_id) + ) else: with open(donefile_name, "w") as f: f.write(xbox_str + "\n") client.upload(donefile_name, output_path) - self.rank0_error("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) + self.rank0_error( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) fleet._role_maker._barrier_worker() - def write_cache_donefile(self, - output_path, - day, - pass_id, - key_num, - hadoop_fs_name, - hadoop_fs_ugi, - hadoop_home="$HADOOP_HOME", - donefile_name="sparse_cache.meta", - **kwargs): + def write_cache_donefile( + self, + output_path, + day, + pass_id, + key_num, + hadoop_fs_name, + hadoop_fs_ugi, + hadoop_home="$HADOOP_HOME", + donefile_name="sparse_cache.meta", + **kwargs + ): """ write cache donefile @@ -613,15 +658,18 @@ def write_cache_donefile(self, donefile_path = model_path + "/" + donefile_name configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if client.is_file(donefile_path): - self.rank0_error( \ - "not write because %s already exists" % donefile_path) + self.rank0_error( + "not write because %s already exists" % donefile_path + ) else: - meta_str = "file_prefix:part\npart_num:%s\nkey_num:%d\n" \ - % (file_num, key_num) + meta_str = "file_prefix:part\npart_num:%s\nkey_num:%d\n" % ( + file_num, + key_num, + ) with open(donefile_name, "w") as f: f.write(meta_str) client.upload(donefile_name, model_path) @@ -779,10 +827,9 @@ def save_cache_model(self, output_path, day, pass_id, mode=1, **kwargs): suffix_name = "/%s/delta-%s" % (day, pass_id) model_path = output_path.rstrip("/") + suffix_name self.rank0_print("going to save_cache_model %s" % model_path) - key_num = fleet.save_cache_model(None, - model_path, - mode=mode, - table_id=table_id) + key_num = fleet.save_cache_model( + None, model_path, mode=mode, table_id=table_id + ) self.rank0_print("save_cache_model done") return key_num @@ -813,10 +860,9 @@ def save_cache_base_model(self, output_path, day, **kwargs): suffix_name = "/%s/base" % day model_path = output_path.rstrip("/") + suffix_name self.rank0_print("going to save_cache_base_model %s" % model_path) - key_num = fleet.save_cache_model(None, - model_path, - mode=2, - table_id=table_id) + key_num = fleet.save_cache_model( + None, model_path, mode=2, table_id=table_id + ) self.rank0_print("save_cache_base_model done") return key_num @@ -839,8 +885,11 @@ def pull_all_dense_params(self, scope, program): fleet._role_maker._barrier_worker() if fleet._role_maker.is_first_worker(): prog_id = str(id(program)) - tables = fleet._opt_info["program_id_to_worker"][prog_id].\ - get_desc().dense_table + tables = ( + fleet._opt_info["program_id_to_worker"][prog_id] + .get_desc() + .dense_table + ) prog_conf = fleet._opt_info['program_configs'][prog_id] prog_tables = {} for key in prog_conf: @@ -855,27 +904,33 @@ def pull_all_dense_params(self, scope, program): for i in range(0, len(table.dense_variable_name)): var_name = table.dense_variable_name[i] if scope.find_var(var_name) is None: - raise ValueError("var " + var_name + - " not found in scope " + - "when pull dense") + raise ValueError( + "var " + + var_name + + " not found in scope " + + "when pull dense" + ) var_name_list.append(var_name) - fleet._fleet_ptr.pull_dense(scope, int(table.table_id), - var_name_list) + fleet._fleet_ptr.pull_dense( + scope, int(table.table_id), var_name_list + ) fleet._role_maker._barrier_worker() - def save_paddle_inference_model(self, - executor, - scope, - program, - feeded_vars, - target_vars, - output_path, - day, - pass_id, - hadoop_fs_name, - hadoop_fs_ugi, - hadoop_home="$HADOOP_HOME", - save_combine=True): + def save_paddle_inference_model( + self, + executor, + scope, + program, + feeded_vars, + target_vars, + output_path, + day, + pass_id, + hadoop_fs_name, + hadoop_fs_ugi, + hadoop_home="$HADOOP_HOME", + save_combine=True, + ): """ save paddle inference model, and upload to hdfs dnn_plugin path @@ -925,26 +980,31 @@ def save_paddle_inference_model(self, target_vars=target_vars, executor=executor, main_program=program.clone(), - params_filename="params") + params_filename="params", + ) else: fluid.io.save_inference_model( dirname=model_name, feeded_var_names=feeded_var_names, target_vars=target_vars, executor=executor, - main_program=program.clone()) + main_program=program.clone(), + ) configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if pass_id == "-1": dest = "%s/%s/base/dnn_plugin/" % (output_path, day) else: - dest = "%s/%s/delta-%s/dnn_plugin/" % (output_path, day, - pass_id) + dest = "%s/%s/delta-%s/dnn_plugin/" % ( + output_path, + day, + pass_id, + ) if not client.is_exist(dest): client.makedirs(dest) @@ -952,19 +1012,21 @@ def save_paddle_inference_model(self, fleet._role_maker._barrier_worker() - def save_paddle_params(self, - executor, - scope, - program, - model_name, - output_path, - day, - pass_id, - hadoop_fs_name, - hadoop_fs_ugi, - hadoop_home="$HADOOP_HOME", - var_names=None, - save_combine=True): + def save_paddle_params( + self, + executor, + scope, + program, + model_name, + output_path, + day, + pass_id, + hadoop_fs_name, + hadoop_fs_ugi, + hadoop_home="$HADOOP_HOME", + var_names=None, + save_combine=True, + ): """ save paddle model, and upload to hdfs dnn_plugin path @@ -1028,36 +1090,39 @@ def save_paddle_params(self, vars = [program.global_block().var(i) for i in var_names] with fluid.scope_guard(scope): if save_combine: - fluid.io.save_vars(executor, - "./", - program, - vars=vars, - filename=model_name) + fluid.io.save_vars( + executor, "./", program, vars=vars, filename=model_name + ) else: fluid.io.save_vars(executor, model_name, program, vars=vars) configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if pass_id == "-1": dest = "%s/%s/base/dnn_plugin/" % (output_path, day) else: - dest = "%s/%s/delta-%s/dnn_plugin/" % (output_path, day, - pass_id) + dest = "%s/%s/delta-%s/dnn_plugin/" % ( + output_path, + day, + pass_id, + ) if not client.is_exist(dest): client.mkdirs(dest) client.upload(model_name, dest, multi_processes=5, overwrite=True) fleet._role_maker._barrier_worker() - def get_last_save_xbox_base(self, - output_path, - hadoop_fs_name, - hadoop_fs_ugi, - hadoop_home="$HADOOP_HOME"): + def get_last_save_xbox_base( + self, + output_path, + hadoop_fs_name, + hadoop_fs_ugi, + hadoop_home="$HADOOP_HOME", + ): r""" get last saved base xbox info from xbox_base_done.txt @@ -1086,7 +1151,7 @@ def get_last_save_xbox_base(self, donefile_path = output_path + "/xbox_base_done.txt" configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if not client.is_file(donefile_path): @@ -1098,11 +1163,13 @@ def get_last_save_xbox_base(self, xbox_base_key = int(last_dict["key"]) return [last_day, last_path, xbox_base_key] - def get_last_save_xbox(self, - output_path, - hadoop_fs_name, - hadoop_fs_ugi, - hadoop_home="$HADOOP_HOME"): + def get_last_save_xbox( + self, + output_path, + hadoop_fs_name, + hadoop_fs_ugi, + hadoop_home="$HADOOP_HOME", + ): r""" get last saved xbox info from xbox_patch_done.txt @@ -1131,7 +1198,7 @@ def get_last_save_xbox(self, donefile_path = output_path + "/xbox_patch_done.txt" configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if not client.is_file(donefile_path): @@ -1144,11 +1211,13 @@ def get_last_save_xbox(self, xbox_base_key = int(last_dict["key"]) return [last_day, last_pass, last_path, xbox_base_key] - def get_last_save_model(self, - output_path, - hadoop_fs_name, - hadoop_fs_ugi, - hadoop_home="$HADOOP_HOME"): + def get_last_save_model( + self, + output_path, + hadoop_fs_name, + hadoop_fs_ugi, + hadoop_home="$HADOOP_HOME", + ): r""" get last saved model info from donefile.txt @@ -1180,7 +1249,7 @@ def get_last_save_model(self, donefile_path = output_path + "/donefile.txt" configs = { "fs.default.name": hadoop_fs_name, - "hadoop.job.ugi": hadoop_fs_ugi + "hadoop.job.ugi": hadoop_fs_ugi, } client = HDFSClient(hadoop_home, configs) if not client.is_file(donefile_path): @@ -1193,8 +1262,9 @@ def get_last_save_model(self, xbox_base_key = int(content[1]) return [last_save_day, last_save_pass, last_path, xbox_base_key] - def get_online_pass_interval(self, days, hours, split_interval, - split_per_pass, is_data_hourly_placed): + def get_online_pass_interval( + self, days, hours, split_interval, split_per_pass, is_data_hourly_placed + ): """ get online pass interval @@ -1254,16 +1324,18 @@ def get_online_pass_interval(self, days, hours, split_interval, return online_pass_interval - def get_global_metrics(self, - scope=fluid.global_scope(), - stat_pos_name="_generated_var_2", - stat_neg_name="_generated_var_3", - sqrerr_name="sqrerr", - abserr_name="abserr", - prob_name="prob", - q_name="q", - pos_ins_num_name="pos", - total_ins_num_name="total"): + def get_global_metrics( + self, + scope=fluid.global_scope(), + stat_pos_name="_generated_var_2", + stat_neg_name="_generated_var_3", + sqrerr_name="sqrerr", + abserr_name="abserr", + prob_name="prob", + q_name="q", + pos_ins_num_name="pos", + total_ins_num_name="total", + ): r""" get global metrics, including auc, bucket_error, mae, rmse, actual_ctr, predicted_ctr, copc, mean_predict_qvalue, total_ins_num. @@ -1317,8 +1389,10 @@ def get_global_metrics(self, similarity_norm, label) """ - if scope.find_var(stat_pos_name) is None or \ - scope.find_var(stat_neg_name) is None: + if ( + scope.find_var(stat_pos_name) is None + or scope.find_var(stat_neg_name) is None + ): self.rank0_print("not found auc bucket") return [None] * 9 elif scope.find_var(sqrerr_name) is None: @@ -1337,8 +1411,9 @@ def get_global_metrics(self, self.rank0_print("not found pos_ins_num_name=%s" % pos_ins_num_name) return [None] * 9 elif scope.find_var(total_ins_num_name) is None: - self.rank0_print("not found total_ins_num_name=%s" % \ - total_ins_num_name) + self.rank0_print( + "not found total_ins_num_name=%s" % total_ins_num_name + ) return [None] * 9 # barrier worker to ensure all workers finished training @@ -1427,8 +1502,9 @@ def get_metric(name): adjust_ctr = ctr_sum / impression_sum if adjust_ctr == 0: continue - relative_error = \ - math.sqrt((1 - adjust_ctr) / (adjust_ctr * impression_sum)) + relative_error = math.sqrt( + (1 - adjust_ctr) / (adjust_ctr * impression_sum) + ) if relative_error < k_relative_error_bound: actual_ctr = click_sum / impression_sum relative_ctr_error = abs(actual_ctr / adjust_ctr - 1) @@ -1439,22 +1515,30 @@ def get_metric(name): bucket_error = error_sum / error_count if error_count > 0 else 0.0 return [ - auc, bucket_error, mae, rmse, return_actual_ctr, predicted_ctr, - copc, mean_predict_qvalue, - int(total_ins_num) + auc, + bucket_error, + mae, + rmse, + return_actual_ctr, + predicted_ctr, + copc, + mean_predict_qvalue, + int(total_ins_num), ] - def print_global_metrics(self, - scope=fluid.global_scope(), - stat_pos_name="_generated_var_2", - stat_neg_name="_generated_var_3", - sqrerr_name="sqrerr", - abserr_name="abserr", - prob_name="prob", - q_name="q", - pos_ins_num_name="pos", - total_ins_num_name="total", - print_prefix=""): + def print_global_metrics( + self, + scope=fluid.global_scope(), + stat_pos_name="_generated_var_2", + stat_neg_name="_generated_var_3", + sqrerr_name="sqrerr", + abserr_name="abserr", + prob_name="prob", + q_name="q", + pos_ins_num_name="pos", + total_ins_num_name="total", + print_prefix="", + ): r""" print global metrics, including auc, bucket_error, mae, rmse, actual_ctr, predicted_ctr, copc, mean_predict_qvalue, total_ins_num. @@ -1505,8 +1589,10 @@ def print_global_metrics(self, similarity_norm, label) """ - if scope.find_var(stat_pos_name) is None or \ - scope.find_var(stat_neg_name) is None: + if ( + scope.find_var(stat_pos_name) is None + or scope.find_var(stat_neg_name) is None + ): self.rank0_print("not found auc bucket") return elif scope.find_var(sqrerr_name) is None: @@ -1525,26 +1611,56 @@ def print_global_metrics(self, self.rank0_print("not found pos_ins_num_name=%s" % pos_ins_num_name) return elif scope.find_var(total_ins_num_name) is None: - self.rank0_print("not found total_ins_num_name=%s" % \ - total_ins_num_name) + self.rank0_print( + "not found total_ins_num_name=%s" % total_ins_num_name + ) return - auc, bucket_error, mae, rmse, actual_ctr, predicted_ctr, copc,\ - mean_predict_qvalue, total_ins_num = self.get_global_metrics(\ - scope, stat_pos_name, stat_neg_name, sqrerr_name, abserr_name,\ - prob_name, q_name, pos_ins_num_name, total_ins_num_name) + ( + auc, + bucket_error, + mae, + rmse, + actual_ctr, + predicted_ctr, + copc, + mean_predict_qvalue, + total_ins_num, + ) = self.get_global_metrics( + scope, + stat_pos_name, + stat_neg_name, + sqrerr_name, + abserr_name, + prob_name, + q_name, + pos_ins_num_name, + total_ins_num_name, + ) self.rank0_print( "%s global AUC=%.6f BUCKET_ERROR=%.6f MAE=%.6f " "RMSE=%.6f Actural_CTR=%.6f Predicted_CTR=%.6f " - "COPC=%.6f MEAN Q_VALUE=%.6f Ins number=%s" % - (print_prefix, auc, bucket_error, mae, rmse, actual_ctr, - predicted_ctr, copc, mean_predict_qvalue, total_ins_num)) + "COPC=%.6f MEAN Q_VALUE=%.6f Ins number=%s" + % ( + print_prefix, + auc, + bucket_error, + mae, + rmse, + actual_ctr, + predicted_ctr, + copc, + mean_predict_qvalue, + total_ins_num, + ) + ) def program_type_trans(self, prog_dir, prog_fn, is_text): return utils.program_type_trans(prog_dir, prog_fn, is_text) - def draw_from_program_file(self, model_filename, is_text, output_dir, - output_filename): + def draw_from_program_file( + self, model_filename, is_text, output_dir, output_filename + ): """draw program from file""" program = utils.load_program(model_filename, is_text) utils.graphviz(program.global_block(), output_dir, output_filename) @@ -1554,14 +1670,17 @@ def draw_from_program(self, program, output_dir, output_name): utils.graphviz(program.global_block(), output_dir, output_name) def check_two_programs(self, config): - train_prog = utils.load_program(config.train_prog_path, - config.is_text_train_program) - pruned_prog = utils.load_program(config.pruned_prog_path, - config.is_text_pruned_program) + train_prog = utils.load_program( + config.train_prog_path, config.is_text_train_program + ) + pruned_prog = utils.load_program( + config.pruned_prog_path, config.is_text_pruned_program + ) if config.draw: pruned_dir = os.path.dirname(config.pruned_prog_path) - self.draw_from_program(pruned_prog, pruned_dir, - config.draw_out_name) + self.draw_from_program( + pruned_prog, pruned_dir, config.draw_out_name + ) res = utils.check_pruned_program_vars(train_prog, pruned_prog) if res: _logger.info("check_programs succeed.") @@ -1574,9 +1693,14 @@ def check_two_programs(self, config): def check_vars_and_dump(self, config): _logger.info("start check_vars_and_dump.") results = utils.check_saved_vars_try_dump( - config.dump_model_dir, config.dump_program_filename, - config.is_text_dump_program, config.feed_config, - config.fetch_config, config.batch_size, config.save_params_filename) + config.dump_model_dir, + config.dump_program_filename, + config.is_text_dump_program, + config.feed_config, + config.fetch_config, + config.batch_size, + config.save_params_filename, + ) _logger.info("check_vars_and_dump succeed.") return results @@ -1607,8 +1731,9 @@ def parse_program_proto(self, prog_path, is_text, output_dir): utils.parse_program(program, output_dir) def _is_optimizer_op(self, op): - return self.op_role_key in op.attr_names and \ - int(op.all_attrs()[self.op_role_key]) & int(OpRole.Optimize) + return self.op_role_key in op.attr_names and int( + op.all_attrs()[self.op_role_key] + ) & int(OpRole.Optimize) def split_program_by_device(self, program): ops_list = [] @@ -1619,8 +1744,11 @@ def split_program_by_device(self, program): if self._is_optimizer_op(op): break if op.has_attr("op_device"): - cur_attr = op.attr( - "op_device") if op.attr("op_device") != "" else type_cpu + cur_attr = ( + op.attr("op_device") + if op.attr("op_device") != "" + else type_cpu + ) if pre is None or pre != cur_attr: ops_list.append([]) type_list.append(cur_attr) @@ -1710,8 +1838,9 @@ def split_program_by_device(self, program): send_list[i].extend(list(in_from_pre[i + 1])) prog = program.clone() if merged_type_list[i] != type_cpu: - prog = prog._prune_with_input(list(in_from_pre[i]), - list(send_list[i])) + prog = prog._prune_with_input( + list(in_from_pre[i]), list(send_list[i]) + ) program_list.append(prog) else: program_list.append(prog) @@ -1729,8 +1858,13 @@ def split_program_by_device(self, program): print("warning: non heter program") return None else: - return [start_list[heter_index], end_list[heter_index], send_list[heter_index], \ - recv_list[heter_index], program_list[heter_index]] + return [ + start_list[heter_index], + end_list[heter_index], + send_list[heter_index], + recv_list[heter_index], + program_list[heter_index], + ] class GPUPSUtil(FleetUtil): @@ -1921,12 +2055,14 @@ def get_last_save_model(self, output_path): os.remove("donefile.txt") return [last_save_day, last_save_pass, last_path, xbox_base_key] - def write_model_donefile(self, - output_path, - day, - pass_id, - xbox_base_key, - donefile_name="donefile.txt"): + def write_model_donefile( + self, + output_path, + day, + pass_id, + xbox_base_key, + donefile_name="donefile.txt", + ): """ write donefile when save model @@ -1965,8 +2101,13 @@ def write_model_donefile(self, if fleet.worker_index() == 0: donefile_path = output_path + "/" + donefile_name - content = "%s\t%lu\t%s\t%s\t%d" % (day, xbox_base_key,\ - model_path, pass_id, 0) + content = "%s\t%lu\t%s\t%s\t%d" % ( + day, + xbox_base_key, + model_path, + pass_id, + 0, + ) if self._afs.is_file(donefile_path): self._afs.download(donefile_path, donefile_name) pre_content = "" @@ -1978,8 +2119,9 @@ def write_model_donefile(self, os.remove(donefile_name) exist = False for i in range(len(day_list)): - if int(day) == int(day_list[i]) and \ - int(pass_id) == int(pass_list[i]): + if int(day) == int(day_list[i]) and int(pass_id) == int( + pass_list[i] + ): exist = True break if not exist: @@ -1988,29 +2130,35 @@ def write_model_donefile(self, f.write(content + "\n") self._afs.delete(donefile_path) self._afs.upload(donefile_name, donefile_path) - self.rank0_error("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) + self.rank0_error( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) else: - self.rank0_error("not write %s because %s/%s already " - "exists" % (donefile_name, day, pass_id)) + self.rank0_error( + "not write %s because %s/%s already " + "exists" % (donefile_name, day, pass_id) + ) else: with open(donefile_name, "w") as f: f.write(content + "\n") self._afs.upload(donefile_name, donefile_path) - self.rank0_error("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) - - def write_xbox_donefile(self, - output_path, - day, - pass_id, - xbox_base_key, - data_path, - hadoop_fs_name, - hadoop_fs_ugi, - monitor_data={}, - hadoop_home="$HADOOP_HOME", - donefile_name=None): + self.rank0_error( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) + + def write_xbox_donefile( + self, + output_path, + day, + pass_id, + xbox_base_key, + data_path, + hadoop_fs_name, + hadoop_fs_ugi, + monitor_data={}, + hadoop_home="$HADOOP_HOME", + donefile_name=None, + ): """ write delta donefile or xbox base donefile @@ -2063,9 +2211,16 @@ def write_xbox_donefile(self, data_path = ",".join(data_path) if fleet.worker_index() == 0: donefile_path = output_path + "/" + donefile_name - xbox_str = self._get_xbox_str(output_path, day, model_path, \ - xbox_base_key, data_path, hadoop_fs_name, monitor_data={}, - mode=mode) + xbox_str = self._get_xbox_str( + output_path, + day, + model_path, + xbox_base_key, + data_path, + hadoop_fs_name, + monitor_data={}, + mode=mode, + ) if self._afs.is_exist(donefile_path): self.rank0_info("exist %s succeed" % (donefile_path)) @@ -2080,9 +2235,11 @@ def write_xbox_donefile(self, os.remove(donefile_name) self.rank0_info("remove %s succeed" % (donefile_name)) exist = False - if int(day) < int(last_day) or \ - int(day) == int(last_day) and \ - int(pass_id) <= int(last_pass): + if ( + int(day) < int(last_day) + or int(day) == int(last_day) + and int(pass_id) <= int(last_pass) + ): exist = True if not exist: with open(donefile_name, "w") as f: @@ -2090,25 +2247,31 @@ def write_xbox_donefile(self, f.write(xbox_str + "\n") self._afs.delete(donefile_path) self._afs.upload(donefile_name, donefile_path) - self.rank0_info("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) + self.rank0_info( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) else: - self.rank0_info("not write %s because %s/%s already " - "exists" % (donefile_name, day, pass_id)) + self.rank0_info( + "not write %s because %s/%s already " + "exists" % (donefile_name, day, pass_id) + ) else: with open(donefile_name, "w") as f: f.write(xbox_str + "\n") self._afs.upload(donefile_name, donefile_path) - self.rank0_error("write %s/%s %s succeed" % \ - (day, pass_id, donefile_name)) - - def write_cache_donefile(self, - output_path, - day, - pass_id, - key_num, - donefile_name="sparse_cache.meta", - **kwargs): + self.rank0_error( + "write %s/%s %s succeed" % (day, pass_id, donefile_name) + ) + + def write_cache_donefile( + self, + output_path, + day, + pass_id, + key_num, + donefile_name="sparse_cache.meta", + **kwargs + ): """ write cache donefile @@ -2154,25 +2317,30 @@ def write_cache_donefile(self, donefile_path = model_path + "/" + donefile_name if self._afs.is_file(donefile_path): - self.rank0_error( \ - "not write because %s already exists" % donefile_path) + self.rank0_error( + "not write because %s already exists" % donefile_path + ) else: - meta_str = "file_prefix:part\npart_num:%s\nkey_num:%d\n" \ - % (file_num, key_num) + meta_str = "file_prefix:part\npart_num:%s\nkey_num:%d\n" % ( + file_num, + key_num, + ) with open(donefile_name, "w") as f: f.write(meta_str) self._afs.upload(donefile_name, donefile_path) self.rank0_error("write %s succeed" % donefile_path) - def _get_xbox_str(self, - output_path, - day, - model_path, - xbox_base_key, - data_path, - hadoop_fs_name, - monitor_data={}, - mode="patch"): + def _get_xbox_str( + self, + output_path, + day, + model_path, + xbox_base_key, + data_path, + hadoop_fs_name, + monitor_data={}, + mode="patch", + ): xbox_dict = collections.OrderedDict() if mode == "base": xbox_dict["id"] = str(xbox_base_key) @@ -2184,7 +2352,7 @@ def _get_xbox_str(self, xbox_dict["id"] = str(int(time.time())) xbox_dict["key"] = str(xbox_base_key) if model_path.startswith("hdfs:") or model_path.startswith("afs:"): - model_path = model_path[model_path.find(":") + 1:] + model_path = model_path[model_path.find(":") + 1 :] xbox_dict["input"] = hadoop_fs_name + model_path.rstrip("/") + "/000" xbox_dict["record_count"] = "111111" xbox_dict["partition_type"] = "2" @@ -2194,7 +2362,8 @@ def _get_xbox_str(self, xbox_dict["job_id"] = os.environ.get("PADDLE_JOB_ID", "") # currently hard code here, set monitor_data empty string xbox_dict["monitor_data"] = "" - xbox_dict["monitor_path"] = output_path.rstrip("/") + "/monitor/" \ - + day + ".txt" + xbox_dict["monitor_path"] = ( + output_path.rstrip("/") + "/monitor/" + day + ".txt" + ) xbox_dict["mpi_size"] = str(fleet.worker_num()) return json.dumps(xbox_dict) diff --git a/python/paddle/fluid/incubate/fleet/utils/hdfs.py b/python/paddle/fluid/incubate/fleet/utils/hdfs.py index 41aff381959e2..c35e266357e0e 100644 --- a/python/paddle/fluid/incubate/fleet/utils/hdfs.py +++ b/python/paddle/fluid/incubate/fleet/utils/hdfs.py @@ -24,8 +24,17 @@ import errno import time import logging -#from . import fs -from paddle.distributed.fleet.utils.fs import FS, LocalFS, FSFileExistsError, FSFileNotExistsError, ExecuteError, FSTimeOut, FSShellCmdAborted + +# from . import fs +from paddle.distributed.fleet.utils.fs import ( + FS, + LocalFS, + FSFileExistsError, + FSFileNotExistsError, + ExecuteError, + FSTimeOut, + FSShellCmdAborted, +) from paddle.fluid import core import functools @@ -35,9 +44,7 @@ def _handle_errors(max_time_out=None): - def decorator(f): - @functools.wraps(f) def handler(*args, **kwargs): o = args[0] @@ -53,19 +60,23 @@ def handler(*args, **kwargs): while True: try: return f(*args, **kwargs) - #important: only ExecuteError need to retry + # important: only ExecuteError need to retry except ExecuteError as e: if time.time() - start >= time_out: - raise FSTimeOut("args:{} timeout:{}".format( - args, - time.time() - start)) + raise FSTimeOut( + "args:{} timeout:{}".format( + args, time.time() - start + ) + ) time.sleep(inter) if time.time() - last_print_time > 30: - print("hadoop operator timeout:args:{} timeout:{}".format( - args, - time.time() - start)) + print( + "hadoop operator timeout:args:{} timeout:{}".format( + args, time.time() - start + ) + ) last_print_time = time.time() return handler @@ -74,13 +85,13 @@ def handler(*args, **kwargs): class HDFSClient(FS): - def __init__( - self, - hadoop_home, - configs, - time_out=5 * 60 * 1000, #ms - sleep_inter=1000): #ms + self, + hadoop_home, + configs, + time_out=5 * 60 * 1000, # ms + sleep_inter=1000, + ): # ms # Raise exception if JAVA_HOME not exists. self.pre_commands = [] @@ -98,7 +109,8 @@ def __init__( self._sleep_inter = sleep_inter self._base_cmd = " ".join(self.pre_commands) self._bd_err_re = re.compile( - r'\s?responseErrorMsg\s?\:.*, errorCode\:\s?[0-9]+, path\:') + r'\s?responseErrorMsg\s?\:.*, errorCode\:\s?[0-9]+, path\:' + ) def _run_cmd(self, cmd, redirect_stderr=False): exe_cmd = "{} -{}".format(self._base_cmd, cmd) @@ -269,7 +281,8 @@ def mv(self, fs_src_path, fs_dst_path, overwrite=False, test_exists=True): if test_exists: if not self.is_exist(fs_src_path): raise FSFileNotExistsError( - "{} is not exists".format(fs_src_path)) + "{} is not exists".format(fs_src_path) + ) if self.is_exist(fs_dst_path): raise FSFileExistsError("{} exists already".format(fs_dst_path)) @@ -285,8 +298,7 @@ def _try_mv(self, fs_src_path, fs_dst_path): if ret != 0: raise ExecuteError(cmd) except Exception as e: - if not self.is_exist(fs_src_path) and \ - self.is_exist(fs_dst_path): + if not self.is_exist(fs_src_path) and self.is_exist(fs_dst_path): return raise e diff --git a/python/paddle/fluid/incubate/fleet/utils/http_server.py b/python/paddle/fluid/incubate/fleet/utils/http_server.py index 685228f07490e..269f912073e85 100644 --- a/python/paddle/fluid/incubate/fleet/utils/http_server.py +++ b/python/paddle/fluid/incubate/fleet/utils/http_server.py @@ -14,6 +14,7 @@ """Http Server.""" import logging + # NOTE: HTTPServer has a different name in python2 and python3 from http.server import HTTPServer import http.server as SimpleHTTPServer @@ -32,9 +33,9 @@ def get_logger(name, level, fmt): return logger -_http_server_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_http_server_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class KVHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): @@ -164,7 +165,8 @@ def start(self): start server until user calls stop to let it quit. """ self.listen_thread = threading.Thread( - target=lambda: self.http_server.serve_forever()) + target=lambda: self.http_server.serve_forever() + ) self.listen_thread.start() def stop(self): diff --git a/python/paddle/fluid/incubate/fleet/utils/utils.py b/python/paddle/fluid/incubate/fleet/utils/utils.py index c154fcb2f1dfb..c675fea39bc9c 100644 --- a/python/paddle/fluid/incubate/fleet/utils/utils.py +++ b/python/paddle/fluid/incubate/fleet/utils/utils.py @@ -28,9 +28,13 @@ from paddle.fluid.proto import framework_pb2 __all__ = [ - "load_program", "save_program", "program_type_trans", - "check_saved_vars_try_dump", "parse_program", "check_pruned_program_vars", - "graphviz" + "load_program", + "save_program", + "program_type_trans", + "check_saved_vars_try_dump", + "parse_program", + "check_pruned_program_vars", + "graphviz", ] logger = logging.getLogger(__name__) @@ -45,7 +49,8 @@ ops_out_fn = "ops.log" feed_fetch_type_list = [ - core.VarDesc.VarType.FEED_MINIBATCH, core.VarDesc.VarType.FETCH_LIST + core.VarDesc.VarType.FEED_MINIBATCH, + core.VarDesc.VarType.FETCH_LIST, ] not_expected_op_types = ["lookup_table"] @@ -85,12 +90,16 @@ def save_program(program, model_filename='__model__', is_text=False): def check_pruned_program_vars(train_prog, pruned_prog): is_match = True - pruned_vars = [(v.name, v) for v in pruned_prog.list_vars() - if fluid.io.is_persistable(v)] + pruned_vars = [ + (v.name, v) + for v in pruned_prog.list_vars() + if fluid.io.is_persistable(v) + ] pruned_vars = OrderedDict(pruned_vars) pruned_vars_name = [name for name in pruned_vars] logger.info( - "persistable vars in pruned program: {}".format(pruned_vars_name)) + "persistable vars in pruned program: {}".format(pruned_vars_name) + ) for var_name in pruned_vars: var = pruned_vars[var_name] @@ -102,14 +111,23 @@ def check_pruned_program_vars(train_prog, pruned_prog): except ValueError as e: logger.error( "not find variable '%s' in train program. please check pruning." - % var_name) + % var_name + ) logger.error(e) continue - if var.shape != train_prog_var.shape or var.dtype != train_prog_var.dtype: + if ( + var.shape != train_prog_var.shape + or var.dtype != train_prog_var.dtype + ): logger.error( - "variable: {} not match. in pruned program shape: {} dtype:{}, in train program shape: {} dtype: {}" - .format(var_name, var.shape, var.dtype, train_prog_var.shape, - train_prog_var.dtype)) + "variable: {} not match. in pruned program shape: {} dtype:{}, in train program shape: {} dtype: {}".format( + var_name, + var.shape, + var.dtype, + train_prog_var.shape, + train_prog_var.dtype, + ) + ) is_match = False return is_match @@ -119,10 +137,12 @@ def graphviz(block, output_dir="", filename='debug'): pdf_path = os.path.join(output_dir, filename + '.pdf') debugger.draw_block_graphviz(block, path=dot_path) cmd = ["dot", "-Tpdf", dot_path, "-o", pdf_path] - p = subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) p.wait() @@ -134,17 +154,18 @@ def program_type_trans(prog_dir, prog_fn, is_text): def append_save_op(block, var, path): - block.append_op(type='save', - inputs={'X': [var]}, - outputs={}, - attrs={'file_path': path}) + block.append_op( + type='save', inputs={'X': [var]}, outputs={}, attrs={'file_path': path} + ) def append_load_op(block, var, path): - block.append_op(type='load', - inputs={}, - outputs={'Out': [var]}, - attrs={'file_path': path}) + block.append_op( + type='load', + inputs={}, + outputs={'Out': [var]}, + attrs={'file_path': path}, + ) def save_var(np_array, var_name, shape_list, dtype, save_path): @@ -199,19 +220,34 @@ def feed_gen(batch_size, feeded_vars_dims, feeded_vars_filelist): return batch_feed -def try_load_model_vars(dump_dir, dump_prog_fn, is_text_dump_program, - batch_size, feed_config, fetch_config, save_filename, - saved_params): +def try_load_model_vars( + dump_dir, + dump_prog_fn, + is_text_dump_program, + batch_size, + feed_config, + fetch_config, + save_filename, + saved_params, +): place = fluid.CPUPlace() exe = fluid.Executor(place) scope = fluid.core.Scope() with fluid.scope_guard(scope): if is_text_dump_program: - dump_prog_fn = program_type_trans(dump_dir, dump_prog_fn, - is_text_dump_program) - inference_program, feed_target_names, fetch_targets = \ - fluid.io.load_inference_model(dump_dir, exe, model_filename=dump_prog_fn, - params_filename=save_filename) + dump_prog_fn = program_type_trans( + dump_dir, dump_prog_fn, is_text_dump_program + ) + ( + inference_program, + feed_target_names, + fetch_targets, + ) = fluid.io.load_inference_model( + dump_dir, + exe, + model_filename=dump_prog_fn, + params_filename=save_filename, + ) # check program vars and saved vars shape orig_para_shape = { @@ -222,13 +258,17 @@ def try_load_model_vars(dump_dir, dump_prog_fn, is_text_dump_program, var_temp = fluid.global_scope().find_var(each_var.name) assert var_temp != None, "can't not find var: " + each_var.name new_shape = (np.array(var_temp.get_tensor())).shape - assert each_var.name in orig_para_shape, each_var.name + "MUST in var list" + assert each_var.name in orig_para_shape, ( + each_var.name + "MUST in var list" + ) orig_shape = orig_para_shape.get(each_var.name) if new_shape != orig_shape: raise RuntimeError( "Shape not matching: the Program requires a parameter with a shape of ({}), " - "while the loaded parameter (namely [ {} ]) has a shape of ({})." - .format(orig_shape, each_var.name, new_shape)) + "while the loaded parameter (namely [ {} ]) has a shape of ({}).".format( + orig_shape, each_var.name, new_shape + ) + ) # check feed/fetch vars in program and config fetch_targets_names = [v.name for v in fetch_targets] @@ -238,10 +278,15 @@ def try_load_model_vars(dump_dir, dump_prog_fn, is_text_dump_program, logger.warning("no fetch targets in program.") fetch_list = fetch_targets feed_name_list = feed_target_names - if feed_config.feeded_vars_names is not None and feed_target_names != feed_config.feeded_vars_names: + if ( + feed_config.feeded_vars_names is not None + and feed_target_names != feed_config.feeded_vars_names + ): logger.warning( - "feed vars in program and config are diff: feed in program: {}. feed in config {}." - .format(feed_target_names, feed_config.feeded_vars_names)) + "feed vars in program and config are diff: feed in program: {}. feed in config {}.".format( + feed_target_names, feed_config.feeded_vars_names + ) + ) feed_name_list = feed_config.feeded_vars_names # remove feed op in inference_program. new feed op will be added in exe.run global_block = inference_program.global_block() @@ -252,10 +297,15 @@ def try_load_model_vars(dump_dir, dump_prog_fn, is_text_dump_program, need_to_remove_op_index.append(i) for index in need_to_remove_op_index[::-1]: global_block._remove_op(index) - if fetch_config.fetch_vars_names is not None and fetch_targets_names != fetch_config.fetch_vars_names: + if ( + fetch_config.fetch_vars_names is not None + and fetch_targets_names != fetch_config.fetch_vars_names + ): logger.warning( - "fetch vars in program and config are diff: fetch in program: {}. fetch in config {}." - .format(fetch_targets_names, fetch_config.fetch_vars_names)) + "fetch vars in program and config are diff: fetch in program: {}. fetch in config {}.".format( + fetch_targets_names, fetch_config.fetch_vars_names + ) + ) fetch_list = [ inference_program.global_block().var(i) for i in fetch_config.fetch_vars_names @@ -275,70 +325,101 @@ def try_load_model_vars(dump_dir, dump_prog_fn, is_text_dump_program, # try dump fetch_targets feed_tensors = [] - assert len(feed_config.feeded_vars_names) == len( - feed_config.feeded_vars_dims) == len(feed_config.feeded_vars_types) + assert ( + len(feed_config.feeded_vars_names) + == len(feed_config.feeded_vars_dims) + == len(feed_config.feeded_vars_types) + ) # check program vars and feed tensor shape in config for i in range(len(feed_config.feeded_vars_names)): var = inference_program.global_block().var( - feed_config.feeded_vars_names[i]) + feed_config.feeded_vars_names[i] + ) if not isinstance(feed_config.feeded_vars_dims[i], (list, tuple)): - tensor_shape = (feed_config.feeded_vars_dims[i], ) + tensor_shape = (feed_config.feeded_vars_dims[i],) else: tensor_shape = tuple(feed_config.feeded_vars_dims[i]) feed_config.feeded_vars_dims[i] = tensor_shape var_shape = var.shape[1:] if tensor_shape != var_shape: raise RuntimeError( - "feed variable '{}' shape not match. infer program shape: {}. feed tensor shape: {}" - .format(feed_config.feeded_vars_names[i], var_shape, - tensor_shape)) + "feed variable '{}' shape not match. infer program shape: {}. feed tensor shape: {}".format( + feed_config.feeded_vars_names[i], + var_shape, + tensor_shape, + ) + ) if not feed_config.feeded_vars_filelist: logger.info("generate random feed vars.") for i in range(len(feed_config.feeded_vars_names)): var = inference_program.global_block().var( - feed_config.feeded_vars_names[i]) + feed_config.feeded_vars_names[i] + ) # create fake feed tensor. if lod_level > 1, should create_lod_tensor() if var.lod_level == 0: feed_tensors.append( - np.array(np.random.random( - tuple([batch_size] + - list(feed_config.feeded_vars_dims[i]))), - dtype=feed_config.feeded_vars_types[i])) + np.array( + np.random.random( + tuple( + [batch_size] + + list(feed_config.feeded_vars_dims[i]) + ) + ), + dtype=feed_config.feeded_vars_types[i], + ) + ) elif var.lod_level == 1: - t = np.array(np.random.random( - tuple([batch_size] + - list(feed_config.feeded_vars_dims[i]))), - dtype=feed_config.feeded_vars_types[i]) + t = np.array( + np.random.random( + tuple( + [batch_size] + + list(feed_config.feeded_vars_dims[i]) + ) + ), + dtype=feed_config.feeded_vars_types[i], + ) feed_tensors.append( - fluid.create_lod_tensor(t, [[1] * batch_size], place)) + fluid.create_lod_tensor(t, [[1] * batch_size], place) + ) else: raise RuntimeError( "vars with lod_level >= 2 is not supported now in this infer program check tool." ) - results = exe.run(inference_program, - feed={ - name: feed_tensors[i] - for i, name in enumerate(feed_name_list) - }, - fetch_list=fetch_list, - return_numpy=return_numpy) + results = exe.run( + inference_program, + feed={ + name: feed_tensors[i] + for i, name in enumerate(feed_name_list) + }, + fetch_list=fetch_list, + return_numpy=return_numpy, + ) else: - logger.info("load feed vars from files: {}.".format( - feed_config.feeded_vars_filelist)) + logger.info( + "load feed vars from files: {}.".format( + feed_config.feeded_vars_filelist + ) + ) feed_vars = [ inference_program.global_block().var( - feed_config.feeded_vars_names[i]) + feed_config.feeded_vars_names[i] + ) for i in range(len(feed_config.feeded_vars_names)) ] feeder = fluid.DataFeeder(feed_list=feed_vars, place=place) - batch_feed = feed_gen(batch_size, feed_config.feeded_vars_dims, - feed_config.feeded_vars_filelist) + batch_feed = feed_gen( + batch_size, + feed_config.feeded_vars_dims, + feed_config.feeded_vars_filelist, + ) slots = [batch_feed] - results = exe.run(inference_program, - feed=feeder.feed(slots), - fetch_list=fetch_list, - return_numpy=return_numpy) + results = exe.run( + inference_program, + feed=feeder.feed(slots), + fetch_list=fetch_list, + return_numpy=return_numpy, + ) for i, v in enumerate(fetch_list): logger.info("fetch_targets name: %s" % v.name) logger.info("fetch_targets: {}".format(results[i])) @@ -350,31 +431,46 @@ def check_not_expected_ops(prog): for op in prog.global_block().ops: if op.type in not_expected_op_types and op.type not in op_types_set: logger.warning( - "find op type '{}' in program, please check if your program is pruned correctly !" - .format(op.type)) + "find op type '{}' in program, please check if your program is pruned correctly !".format( + op.type + ) + ) op_types_set.add(op.type) -def check_saved_vars_try_dump(dump_dir, - dump_prog_fn, - is_text_dump_program, - feed_config, - fetch_config, - batch_size=1, - save_filename=None): - dump_prog = load_program(os.path.join(dump_dir, dump_prog_fn), - is_text_dump_program) +def check_saved_vars_try_dump( + dump_dir, + dump_prog_fn, + is_text_dump_program, + feed_config, + fetch_config, + batch_size=1, + save_filename=None, +): + dump_prog = load_program( + os.path.join(dump_dir, dump_prog_fn), is_text_dump_program + ) saved_params = [ v for v in dump_prog.list_vars() if fluid.io.is_persistable(v) ] - logger.info("persistable vars in dump program: {}".format( - [v.name for v in saved_params])) + logger.info( + "persistable vars in dump program: {}".format( + [v.name for v in saved_params] + ) + ) check_not_expected_ops(dump_prog) - return try_load_model_vars(dump_dir, dump_prog_fn, is_text_dump_program, - batch_size, feed_config, fetch_config, - save_filename, saved_params) + return try_load_model_vars( + dump_dir, + dump_prog_fn, + is_text_dump_program, + batch_size, + feed_config, + fetch_config, + save_filename, + saved_params, + ) def parse_program(program, output_dir): @@ -383,13 +479,16 @@ def parse_program(program, output_dir): persistable_vars = [ v for v in program.list_vars() if fluid.io.is_persistable(v) ] - output["persistable_vars"] = [{ - 'name': str(v.name), - 'shape': str(v.shape), - 'lod_level': int(v.lod_level), - 'dtype': str(v.dtype), - 'type': str(v.type) - } for v in persistable_vars] + output["persistable_vars"] = [ + { + 'name': str(v.name), + 'shape': str(v.shape), + 'lod_level': int(v.lod_level), + 'dtype': str(v.dtype), + 'type': str(v.type), + } + for v in persistable_vars + ] with open(os.path.join(output_dir, persistable_vars_out_fn), 'w') as f: f.write("persistable vars:\n") for var in output["persistable_vars"]: @@ -398,15 +497,17 @@ def parse_program(program, output_dir): # all vars all_vars = [v for v in program.list_vars()] - output["all_vars"] = [{ - 'name': str(v.name), - 'shape': str(v.shape), - 'lod_level': int(v.lod_level), - 'dtype': str(v.dtype) - } if v.type not in feed_fetch_type_list else { - 'name': str(v.name), - 'type': str(v.type) - } for v in all_vars] + output["all_vars"] = [ + { + 'name': str(v.name), + 'shape': str(v.shape), + 'lod_level': int(v.lod_level), + 'dtype': str(v.dtype), + } + if v.type not in feed_fetch_type_list + else {'name': str(v.name), 'type': str(v.type)} + for v in all_vars + ] with open(os.path.join(output_dir, all_vars_out_fn), 'w') as f: f.write("all vars:\n") for var in output["all_vars"]: @@ -415,11 +516,14 @@ def parse_program(program, output_dir): # ops ops = program.global_block().ops - output["ops"] = [{ - 'type': op.type, - 'input_arg_names': str(op.input_arg_names), - 'output_arg_names': str(op.output_arg_names) - } for op in ops] + output["ops"] = [ + { + 'type': op.type, + 'input_arg_names': str(op.input_arg_names), + 'output_arg_names': str(op.output_arg_names), + } + for op in ops + ] with open(os.path.join(output_dir, ops_out_fn), 'w') as f: f.write("ops:\n") for op in output["ops"]: diff --git a/python/paddle/fluid/inference/__init__.py b/python/paddle/fluid/inference/__init__.py index d6b8b10248792..51127de403f7f 100644 --- a/python/paddle/fluid/inference/__init__.py +++ b/python/paddle/fluid/inference/__init__.py @@ -12,7 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .wrapper import Config, DataType, PlaceType, PrecisionType, Tensor, Predictor +from .wrapper import ( + Config, + DataType, + PlaceType, + PrecisionType, + Tensor, + Predictor, +) from .wrapper import convert_to_mixed_precision -from ..core import create_predictor, get_version, _get_phi_kernel_name, get_num_bytes_of_data_type, PredictorPool, get_trt_compile_version, get_trt_runtime_version +from ..core import ( + create_predictor, + get_version, + _get_phi_kernel_name, + get_num_bytes_of_data_type, + PredictorPool, + get_trt_compile_version, + get_trt_runtime_version, +) diff --git a/python/paddle/fluid/inference/wrapper.py b/python/paddle/fluid/inference/wrapper.py index 83811012e5294..ffad69335a427 100644 --- a/python/paddle/fluid/inference/wrapper.py +++ b/python/paddle/fluid/inference/wrapper.py @@ -33,8 +33,9 @@ def tensor_copy_from_cpu(self, data): ''' Support input type check based on tensor.copy_from_cpu. ''' - if isinstance(data, np.ndarray) or (isinstance(data, list) and len(data) > 0 - and isinstance(data[0], str)): + if isinstance(data, np.ndarray) or ( + isinstance(data, list) and len(data) > 0 and isinstance(data[0], str) + ): self.copy_from_cpu_bind(data) else: raise TypeError( @@ -50,17 +51,20 @@ def tensor_share_external_data(self, data): self.share_external_data_bind(data) else: raise TypeError( - "In share_external_data, we only support LoDTensor data type.") + "In share_external_data, we only support LoDTensor data type." + ) -def convert_to_mixed_precision(model_file: str, - params_file: str, - mixed_model_file: str, - mixed_params_file: str, - mixed_precision: PrecisionType, - backend: PlaceType, - keep_io_types: bool = True, - black_list: Set = set()): +def convert_to_mixed_precision( + model_file: str, + params_file: str, + mixed_model_file: str, + mixed_params_file: str, + mixed_precision: PrecisionType, + backend: PlaceType, + keep_io_types: bool = True, + black_list: Set = set(), +): ''' Convert a fp32 model to mixed precision model. @@ -80,9 +84,16 @@ def convert_to_mixed_precision(model_file: str, os.makedirs(mixed_model_dirname) if not os.path.exists(mixed_params_dirname): os.makedirs(mixed_params_dirname) - convert_to_mixed_precision_bind(model_file, params_file, mixed_model_file, - mixed_params_file, mixed_precision, backend, - keep_io_types, black_list) + convert_to_mixed_precision_bind( + model_file, + params_file, + mixed_model_file, + mixed_params_file, + mixed_precision, + backend, + keep_io_types, + black_list, + ) Tensor.copy_from_cpu = tensor_copy_from_cpu diff --git a/python/paddle/fluid/initializer.py b/python/paddle/fluid/initializer.py index 540ea40833d8d..3e6c9ef3674ac 100644 --- a/python/paddle/fluid/initializer.py +++ b/python/paddle/fluid/initializer.py @@ -16,7 +16,13 @@ import functools from . import framework from . import core -from .framework import _non_static_mode, in_dygraph_mode, _in_legacy_dygraph, default_main_program, _current_expected_place +from .framework import ( + _non_static_mode, + in_dygraph_mode, + _in_legacy_dygraph, + default_main_program, + _current_expected_place, +) from .lazy_init import lazy_init_helper from .framework import program_guard import numpy as np @@ -27,10 +33,22 @@ import paddle __all__ = [ - 'Constant', 'Uniform', 'Normal', 'TruncatedNormal', 'Xavier', 'Bilinear', - 'MSRA', 'ConstantInitializer', 'UniformInitializer', 'NormalInitializer', - 'TruncatedNormalInitializer', 'XavierInitializer', 'BilinearInitializer', - 'MSRAInitializer', 'NumpyArrayInitializer', 'set_global_initializer' + 'Constant', + 'Uniform', + 'Normal', + 'TruncatedNormal', + 'Xavier', + 'Bilinear', + 'MSRA', + 'ConstantInitializer', + 'UniformInitializer', + 'NormalInitializer', + 'TruncatedNormalInitializer', + 'XavierInitializer', + 'BilinearInitializer', + 'MSRAInitializer', + 'NumpyArrayInitializer', + 'set_global_initializer', ] _global_weight_initializer_ = None @@ -56,8 +74,7 @@ def __call__(self, param, block=None): return self._lazy_init(param, block) def forward(self, param, block=None): - """Add corresponding initialization operations to the network - """ + """Add corresponding initialization operations to the network""" raise NotImplementedError() def _lazy_init(self, param, block=None): @@ -74,8 +91,9 @@ def init_op_creator(forward, param, block): # Add hook function for initializing param in dygraph mode param.set_init_func(functools.partial(self.forward, param, block)) - param._init_op_creator = functools.partial(init_op_creator, - self.forward, param) + param._init_op_creator = functools.partial( + init_op_creator, self.forward, param + ) return param @@ -160,35 +178,47 @@ def forward(self, var, block=None): """ block = self._check_block(block) - assert (isinstance(var, framework.Variable) - or isinstance(var, framework.EagerParamBase)) + assert isinstance(var, framework.Variable) or isinstance( + var, framework.EagerParamBase + ) assert isinstance(block, framework.Block) if in_dygraph_mode(): place = _current_expected_place() if self._force_cpu: place = core.CPUPlace() - _C_ops.full_(var, var.shape, str(float(self._value)), var.dtype, - place) + _C_ops.full_( + var, var.shape, str(float(self._value)), var.dtype, place + ) return None elif _in_legacy_dygraph(): - _legacy_C_ops.fill_constant(var, 'value', float(self._value), - 'force_cpu', self._force_cpu, 'dtype', - int(var.dtype), 'str_value', - str(float(self._value)), 'shape', - var.shape) + _legacy_C_ops.fill_constant( + var, + 'value', + float(self._value), + 'force_cpu', + self._force_cpu, + 'dtype', + int(var.dtype), + 'str_value', + str(float(self._value)), + 'shape', + var.shape, + ) return None else: - op = block.append_op(type="fill_constant", - outputs={"Out": var}, - attrs={ - "shape": var.shape, - "dtype": int(var.dtype), - "value": float(self._value), - 'str_value': str(float(self._value)), - 'force_cpu': self._force_cpu - }, - stop_gradient=True) + op = block.append_op( + type="fill_constant", + outputs={"Out": var}, + attrs={ + "shape": var.shape, + "dtype": int(var.dtype), + "value": float(self._value), + 'str_value': str(float(self._value)), + 'force_cpu': self._force_cpu, + }, + stop_gradient=True, + ) var.op = op return op @@ -214,16 +244,12 @@ class UniformInitializer(Initializer): import paddle.fluid as fluid x = fluid.data(name='x', shape=[None, 1], dtype='float32') fc = fluid.layers.fc(input=x, size=10, - param_attr=fluid.initializer.Uniform(low=-0.5, high=0.5)) + param_attr=fluid.initializer.Uniform(low=-0.5, high=0.5)) """ - def __init__(self, - low=-1.0, - high=1.0, - seed=0, - diag_num=0, - diag_step=0, - diag_val=1.0): + def __init__( + self, low=-1.0, high=1.0, seed=0, diag_num=0, diag_step=0, diag_val=1.0 + ): assert low is not None assert high is not None assert high >= low @@ -232,7 +258,7 @@ def __init__(self, assert diag_step is not None assert diag_val is not None if diag_num > 0 or diag_step > 0: - assert (diag_num > 0 and diag_step > 0) + assert diag_num > 0 and diag_step > 0 super(UniformInitializer, self).__init__() self._low = low self._high = high @@ -255,9 +281,12 @@ def forward(self, var, block=None): block = self._check_block(block) assert isinstance(block, framework.Block) - check_variable_and_dtype(var, "Out", - ["uint16", "float16", "float32", "float64"], - "uniform_random") + check_variable_and_dtype( + var, + "Out", + ["uint16", "float16", "float32", "float64"], + "uniform_random", + ) if self._seed == 0: self._seed = block.program.random_seed @@ -265,62 +294,88 @@ def forward(self, var, block=None): # to be compatible of fp16 initializers if var.dtype == VarDesc.VarType.FP16: out_dtype = VarDesc.VarType.FP32 - out_var = block.create_var(name=unique_name.generate(".".join( - ['uniform_random', var.name, 'tmp'])), - shape=var.shape, - dtype=out_dtype, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False) + out_var = block.create_var( + name=unique_name.generate( + ".".join(['uniform_random', var.name, 'tmp']) + ), + shape=var.shape, + dtype=out_dtype, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + ) else: out_dtype = var.dtype out_var = var if framework._non_static_mode(): if in_dygraph_mode(): - out_var = _C_ops.uniform_random(var.shape, out_dtype, self._low, - self._high, self._seed, - _current_expected_place()) + out_var = _C_ops.uniform_random( + var.shape, + out_dtype, + self._low, + self._high, + self._seed, + _current_expected_place(), + ) elif _in_legacy_dygraph(): out_var = _legacy_C_ops.uniform_random( - 'shape', var.shape, 'min', self._low, 'max', self._high, - 'seed', self._seed, 'dtype', out_dtype, 'diag_num', - self._diag_num, 'diag_step', self._diag_step, 'diag_val', - self._diag_val) + 'shape', + var.shape, + 'min', + self._low, + 'max', + self._high, + 'seed', + self._seed, + 'dtype', + out_dtype, + 'diag_num', + self._diag_num, + 'diag_step', + self._diag_step, + 'diag_val', + self._diag_val, + ) if var.dtype == VarDesc.VarType.FP16: if in_dygraph_mode(): var_tmp = _C_ops.cast(out_var, var.dtype) elif _in_legacy_dygraph(): - var_tmp = _legacy_C_ops.cast(out_var, 'in_dtype', - out_var.dtype, 'out_dtype', - var.dtype) + var_tmp = _legacy_C_ops.cast( + out_var, + 'in_dtype', + out_var.dtype, + 'out_dtype', + var.dtype, + ) var_tmp._share_underline_tensor_to(var) else: out_var._share_underline_tensor_to(var) return None else: - op = block.append_op(type="uniform_random", - inputs={}, - outputs={"Out": out_var}, - attrs={ - "shape": var.shape, - "dtype": out_dtype, - "min": self._low, - "max": self._high, - "seed": self._seed, - "diag_num": self._diag_num, - "diag_step": self._diag_step, - "diag_val": self._diag_val - }, - stop_gradient=True) + op = block.append_op( + type="uniform_random", + inputs={}, + outputs={"Out": out_var}, + attrs={ + "shape": var.shape, + "dtype": out_dtype, + "min": self._low, + "max": self._high, + "seed": self._seed, + "diag_num": self._diag_num, + "diag_step": self._diag_step, + "diag_val": self._diag_val, + }, + stop_gradient=True, + ) if var.dtype == VarDesc.VarType.FP16: - block.append_op(type="cast", - inputs={"X": out_var}, - outputs={"Out": var}, - attrs={ - "in_dtype": out_var.dtype, - "out_dtype": var.dtype - }) + block.append_op( + type="cast", + inputs={"X": out_var}, + outputs={"Out": var}, + attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}, + ) var.op = op return op @@ -368,40 +423,61 @@ def forward(self, var, block=None): assert isinstance(block, framework.Block) - check_variable_and_dtype(var, "Out", - ["uint16", "float16", "float32", "float64"], - "guassian_random") + check_variable_and_dtype( + var, + "Out", + ["uint16", "float16", "float32", "float64"], + "guassian_random", + ) if self._seed == 0: self._seed = block.program.random_seed if in_dygraph_mode(): place = _current_expected_place() - out_var = _C_ops.gaussian_random(var.shape, self._mean, - self._std_dev, self._seed, - var.dtype, place) + out_var = _C_ops.gaussian_random( + var.shape, + self._mean, + self._std_dev, + self._seed, + var.dtype, + place, + ) out_var._share_underline_tensor_to(var) return None if _in_legacy_dygraph(): out_var = _legacy_C_ops.gaussian_random( - 'shape', var.shape, 'dtype', var.dtype, 'mean', self._mean, - 'std', self._std_dev, 'seed', self._seed, 'use_mkldnn', False) + 'shape', + var.shape, + 'dtype', + var.dtype, + 'mean', + self._mean, + 'std', + self._std_dev, + 'seed', + self._seed, + 'use_mkldnn', + False, + ) out_var._share_underline_tensor_to(var) return None else: - op = block.append_op(type="gaussian_random", - outputs={"Out": var}, - attrs={ - "shape": var.shape, - "dtype": var.dtype, - "mean": self._mean, - "std": self._std_dev, - "seed": self._seed, - "use_mkldnn": False - }, - stop_gradient=True) + op = block.append_op( + type="gaussian_random", + outputs={"Out": var}, + attrs={ + "shape": var.shape, + "dtype": var.dtype, + "mean": self._mean, + "std": self._std_dev, + "seed": self._seed, + "use_mkldnn": False, + }, + stop_gradient=True, + ) var.op = op return op @@ -454,20 +530,28 @@ def forward(self, var, block=None): # to be compatible of fp16 initalizers if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: out_dtype = VarDesc.VarType.FP32 - out_var = block.create_var(name=unique_name.generate(".".join( - ['truncated_gaussian_random', var.name, 'tmp'])), - shape=var.shape, - dtype=out_dtype, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False) + out_var = block.create_var( + name=unique_name.generate( + ".".join(['truncated_gaussian_random', var.name, 'tmp']) + ), + shape=var.shape, + dtype=out_dtype, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + ) else: out_dtype = var.dtype out_var = var if in_dygraph_mode(): out_var = _C_ops.truncated_gaussian_random( - var.shape, self._mean, self._std_dev, self._seed, out_dtype, - _current_expected_place()) + var.shape, + self._mean, + self._std_dev, + self._seed, + out_dtype, + _current_expected_place(), + ) if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: var_tmp = _C_ops.cast(out_var, var.dtype) var_tmp._share_underline_tensor_to(var) @@ -477,35 +561,46 @@ def forward(self, var, block=None): if _in_legacy_dygraph(): out_var = _legacy_C_ops.truncated_gaussian_random( - 'shape', var.shape, 'dtype', out_dtype, 'mean', self._mean, - 'std', self._std_dev, 'seed', self._seed) + 'shape', + var.shape, + 'dtype', + out_dtype, + 'mean', + self._mean, + 'std', + self._std_dev, + 'seed', + self._seed, + ) if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: - var_tmp = _legacy_C_ops.cast(out_var, 'in_dtype', out_var.dtype, - 'out_dtype', var.dtype) + var_tmp = _legacy_C_ops.cast( + out_var, 'in_dtype', out_var.dtype, 'out_dtype', var.dtype + ) var_tmp._share_underline_tensor_to(var) else: out_var._share_underline_tensor_to(var) return None else: - op = block.append_op(type="truncated_gaussian_random", - outputs={"Out": out_var}, - attrs={ - "shape": var.shape, - "dtype": out_dtype, - "mean": self._mean, - "std": self._std_dev, - "seed": self._seed - }, - stop_gradient=True) + op = block.append_op( + type="truncated_gaussian_random", + outputs={"Out": out_var}, + attrs={ + "shape": var.shape, + "dtype": out_dtype, + "mean": self._mean, + "std": self._std_dev, + "seed": self._seed, + }, + stop_gradient=True, + ) if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: - block.append_op(type="cast", - inputs={"X": out_var}, - outputs={"Out": var}, - attrs={ - "in_dtype": out_var.dtype, - "out_dtype": var.dtype - }) + block.append_op( + type="cast", + inputs={"X": out_var}, + outputs={"Out": var}, + attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}, + ) var.op = op return op @@ -578,9 +673,12 @@ def forward(self, var, block=None): block = self._check_block(block) assert isinstance(block, framework.Block) - check_variable_and_dtype(var, "Out", - ["uint16", "float16", "float32", "float64"], - "xavier_init") + check_variable_and_dtype( + var, + "Out", + ["uint16", "float16", "float32", "float64"], + "xavier_init", + ) f_in, f_out = self._compute_fans(var) @@ -593,14 +691,18 @@ def forward(self, var, block=None): # to be compatible of fp16 initalizers if var.dtype == VarDesc.VarType.FP16 or ( - var.dtype == VarDesc.VarType.BF16 and not self._uniform): + var.dtype == VarDesc.VarType.BF16 and not self._uniform + ): out_dtype = VarDesc.VarType.FP32 - out_var = block.create_var(name=unique_name.generate(".".join( - ['xavier_init', var.name, 'tmp'])), - shape=var.shape, - dtype=out_dtype, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False) + out_var = block.create_var( + name=unique_name.generate( + ".".join(['xavier_init', var.name, 'tmp']) + ), + shape=var.shape, + dtype=out_dtype, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + ) else: out_dtype = var.dtype out_var = var @@ -609,34 +711,62 @@ def forward(self, var, block=None): if self._uniform: limit = math.sqrt(6.0 / float(fan_in + fan_out)) if in_dygraph_mode(): - out_var = _C_ops.uniform_random(out_var.shape, out_dtype, - -limit, limit, self._seed, - _current_expected_place()) + out_var = _C_ops.uniform_random( + out_var.shape, + out_dtype, + -limit, + limit, + self._seed, + _current_expected_place(), + ) elif _in_legacy_dygraph(): out_var = _legacy_C_ops.uniform_random( - 'shape', out_var.shape, 'min', -limit, 'max', limit, - 'seed', self._seed, 'dtype', out_dtype) + 'shape', + out_var.shape, + 'min', + -limit, + 'max', + limit, + 'seed', + self._seed, + 'dtype', + out_dtype, + ) else: std = math.sqrt(2.0 / float(fan_in + fan_out)) if in_dygraph_mode(): place = _current_expected_place() - out_var = _C_ops.gaussian_random(out_var.shape, 0.0, std, - self._seed, out_dtype, - place) + out_var = _C_ops.gaussian_random( + out_var.shape, 0.0, std, self._seed, out_dtype, place + ) else: out_var = _legacy_C_ops.gaussian_random( - 'shape', out_var.shape, 'dtype', out_dtype, 'mean', 0.0, - 'std', std, 'seed', self._seed) + 'shape', + out_var.shape, + 'dtype', + out_dtype, + 'mean', + 0.0, + 'std', + std, + 'seed', + self._seed, + ) if var.dtype == VarDesc.VarType.FP16 or ( - var.dtype == VarDesc.VarType.BF16 and not self._uniform): + var.dtype == VarDesc.VarType.BF16 and not self._uniform + ): if in_dygraph_mode(): var_tmp = _C_ops.cast(out_var, var.dtype) elif _in_legacy_dygraph(): - var_tmp = _legacy_C_ops.cast(out_var, 'in_dtype', - out_var.dtype, 'out_dtype', - var.dtype) + var_tmp = _legacy_C_ops.cast( + out_var, + 'in_dtype', + out_var.dtype, + 'out_dtype', + var.dtype, + ) var_tmp._share_underline_tensor_to(var) else: out_var._share_underline_tensor_to(var) @@ -644,39 +774,43 @@ def forward(self, var, block=None): else: if self._uniform: limit = math.sqrt(6.0 / float(fan_in + fan_out)) - op = block.append_op(type="uniform_random", - inputs={}, - outputs={"Out": out_var}, - attrs={ - "shape": out_var.shape, - "dtype": out_dtype, - "min": -limit, - "max": limit, - "seed": self._seed - }, - stop_gradient=True) + op = block.append_op( + type="uniform_random", + inputs={}, + outputs={"Out": out_var}, + attrs={ + "shape": out_var.shape, + "dtype": out_dtype, + "min": -limit, + "max": limit, + "seed": self._seed, + }, + stop_gradient=True, + ) else: std = math.sqrt(2.0 / float(fan_in + fan_out)) - op = block.append_op(type="gaussian_random", - outputs={"Out": out_var}, - attrs={ - "shape": out_var.shape, - "dtype": out_var.dtype, - "mean": 0.0, - "std": std, - "seed": self._seed - }, - stop_gradient=True) + op = block.append_op( + type="gaussian_random", + outputs={"Out": out_var}, + attrs={ + "shape": out_var.shape, + "dtype": out_var.dtype, + "mean": 0.0, + "std": std, + "seed": self._seed, + }, + stop_gradient=True, + ) if var.dtype == VarDesc.VarType.FP16 or ( - var.dtype == VarDesc.VarType.BF16 and not self._uniform): - block.append_op(type="cast", - inputs={"X": out_var}, - outputs={"Out": var}, - attrs={ - "in_dtype": out_var.dtype, - "out_dtype": var.dtype - }) + var.dtype == VarDesc.VarType.BF16 and not self._uniform + ): + block.append_op( + type="cast", + inputs={"X": out_var}, + outputs={"Out": var}, + attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}, + ) var.op = op return op @@ -725,14 +859,15 @@ class MSRAInitializer(Initializer): """ - def __init__(self, - uniform=True, - fan_in=None, - seed=0, - negative_slope=0, - nonlinearity='relu'): - """Constructor for MSRAInitializer - """ + def __init__( + self, + uniform=True, + fan_in=None, + seed=0, + negative_slope=0, + nonlinearity='relu', + ): + """Constructor for MSRAInitializer""" assert uniform is not None assert seed is not None super(MSRAInitializer, self).__init__() @@ -767,14 +902,18 @@ def forward(self, var, block=None): # to be compatible of fp16 initalizers if var.dtype == VarDesc.VarType.FP16 or ( - var.dtype == VarDesc.VarType.BF16 and not self._uniform): + var.dtype == VarDesc.VarType.BF16 and not self._uniform + ): out_dtype = VarDesc.VarType.FP32 - out_var = block.create_var(name=unique_name.generate(".".join( - ['masra_init', var.name, 'tmp'])), - shape=var.shape, - dtype=out_dtype, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False) + out_var = block.create_var( + name=unique_name.generate( + ".".join(['masra_init', var.name, 'tmp']) + ), + shape=var.shape, + dtype=out_dtype, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + ) else: out_dtype = var.dtype out_var = var @@ -784,34 +923,62 @@ def forward(self, var, block=None): gain = calculate_gain(self._nonlinearity, self._negative_slope) limit = gain * math.sqrt(3.0 / float(fan_in)) if in_dygraph_mode(): - out_var = _C_ops.uniform_random(var.shape, out_dtype, - -limit, limit, self._seed, - _current_expected_place()) + out_var = _C_ops.uniform_random( + var.shape, + out_dtype, + -limit, + limit, + self._seed, + _current_expected_place(), + ) else: out_var = _legacy_C_ops.uniform_random( - 'shape', out_var.shape, 'min', -limit, 'max', limit, - 'seed', self._seed, 'dtype', int(out_dtype)) + 'shape', + out_var.shape, + 'min', + -limit, + 'max', + limit, + 'seed', + self._seed, + 'dtype', + int(out_dtype), + ) else: gain = calculate_gain(self._nonlinearity, self._negative_slope) std = gain / math.sqrt(float(fan_in)) if in_dygraph_mode(): place = _current_expected_place() - out_var = _C_ops.gaussian_random(out_var.shape, 0.0, std, - self._seed, out_dtype, - place) + out_var = _C_ops.gaussian_random( + out_var.shape, 0.0, std, self._seed, out_dtype, place + ) else: out_var = _legacy_C_ops.gaussian_random( - 'shape', out_var.shape, 'dtype', int(out_dtype), 'mean', - 0.0, 'std', std, 'seed', self._seed) + 'shape', + out_var.shape, + 'dtype', + int(out_dtype), + 'mean', + 0.0, + 'std', + std, + 'seed', + self._seed, + ) if var.dtype == VarDesc.VarType.FP16 or ( - var.dtype == VarDesc.VarType.BF16 and not self._uniform): + var.dtype == VarDesc.VarType.BF16 and not self._uniform + ): if in_dygraph_mode(): var_tmp = _C_ops.cast(out_var, var.dtype) elif _in_legacy_dygraph(): - var_tmp = _legacy_C_ops.cast(out_var, 'in_dtype', - out_var.dtype, 'out_dtype', - var.dtype) + var_tmp = _legacy_C_ops.cast( + out_var, + 'in_dtype', + out_var.dtype, + 'out_dtype', + var.dtype, + ) var_tmp._share_underline_tensor_to(var) else: out_var._share_underline_tensor_to(var) @@ -820,41 +987,45 @@ def forward(self, var, block=None): if self._uniform: gain = calculate_gain(self._nonlinearity, self._negative_slope) limit = gain * math.sqrt(3.0 / float(fan_in)) - op = block.append_op(type="uniform_random", - inputs={}, - outputs={"Out": out_var}, - attrs={ - "shape": out_var.shape, - "dtype": int(out_dtype), - "min": -limit, - "max": limit, - "seed": self._seed - }, - stop_gradient=True) + op = block.append_op( + type="uniform_random", + inputs={}, + outputs={"Out": out_var}, + attrs={ + "shape": out_var.shape, + "dtype": int(out_dtype), + "min": -limit, + "max": limit, + "seed": self._seed, + }, + stop_gradient=True, + ) else: gain = calculate_gain(self._nonlinearity, self._negative_slope) std = gain / math.sqrt(float(fan_in)) - op = block.append_op(type="gaussian_random", - outputs={"Out": out_var}, - attrs={ - "shape": out_var.shape, - "dtype": int(out_dtype), - "mean": 0.0, - "std": std, - "seed": self._seed - }, - stop_gradient=True) + op = block.append_op( + type="gaussian_random", + outputs={"Out": out_var}, + attrs={ + "shape": out_var.shape, + "dtype": int(out_dtype), + "mean": 0.0, + "std": std, + "seed": self._seed, + }, + stop_gradient=True, + ) if var.dtype == VarDesc.VarType.FP16 or ( - var.dtype == VarDesc.VarType.BF16 and not self._uniform): - block.append_op(type="cast", - inputs={"X": out_var}, - outputs={"Out": var}, - attrs={ - "in_dtype": out_var.dtype, - "out_dtype": var.dtype - }) + var.dtype == VarDesc.VarType.BF16 and not self._uniform + ): + block.append_op( + type="cast", + inputs={"X": out_var}, + outputs={"Out": var}, + attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}, + ) var.op = op return op @@ -905,8 +1076,7 @@ class BilinearInitializer(Initializer): """ def __init__(self): - """Constructor for BilinearInitializer. - """ + """Constructor for BilinearInitializer.""" super(BilinearInitializer, self).__init__() def forward(self, var, block=None): @@ -937,9 +1107,9 @@ def forward(self, var, block=None): weight = np.zeros(np.prod(var.shape), dtype='float32') size = shape[3] # factor - f = np.ceil(size / 2.) + f = np.ceil(size / 2.0) # center - c = (2 * f - 1 - f % 2) / (2. * f) + c = (2 * f - 1 - f % 2) / (2.0 * f) for i in range(np.prod(shape)): x = i % size y = (i / size) % size @@ -948,15 +1118,20 @@ def forward(self, var, block=None): # to be compatible of fp16 initalizers if var.dtype in [ - VarDesc.VarType.FP16, VarDesc.VarType.BF16, VarDesc.VarType.FP64 + VarDesc.VarType.FP16, + VarDesc.VarType.BF16, + VarDesc.VarType.FP64, ]: out_dtype = VarDesc.VarType.FP32 - out_var = block.create_var(name=unique_name.generate(".".join( - ['bilinear_init', var.name, 'tmp'])), - shape=var.shape, - dtype=out_dtype, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False) + out_var = block.create_var( + name=unique_name.generate( + ".".join(['bilinear_init', var.name, 'tmp']) + ), + shape=var.shape, + dtype=out_dtype, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + ) else: out_dtype = var.dtype out_var = var @@ -972,46 +1147,64 @@ def forward(self, var, block=None): if framework._non_static_mode(): if in_dygraph_mode(): - _C_ops.assign_value_(out_var, list(shape), out_dtype, values, - _current_expected_place()) + _C_ops.assign_value_( + out_var, + list(shape), + out_dtype, + values, + _current_expected_place(), + ) elif _in_legacy_dygraph(): - _legacy_C_ops.assign_value(out_var, 'shape', list(shape), - 'dtype', out_dtype, value_name, - values) + _legacy_C_ops.assign_value( + out_var, + 'shape', + list(shape), + 'dtype', + out_dtype, + value_name, + values, + ) if var.dtype in [ - VarDesc.VarType.FP16, VarDesc.VarType.BF16, - VarDesc.VarType.FP64 + VarDesc.VarType.FP16, + VarDesc.VarType.BF16, + VarDesc.VarType.FP64, ]: if in_dygraph_mode(): var_tmp = _C_ops.cast(out_var, var.dtype) elif _in_legacy_dygraph(): - var_tmp = _legacy_C_ops.cast(out_var, 'in_dtype', - out_var.dtype, 'out_dtype', - var.dtype) + var_tmp = _legacy_C_ops.cast( + out_var, + 'in_dtype', + out_var.dtype, + 'out_dtype', + var.dtype, + ) var_tmp._share_underline_tensor_to(var) else: out_var._share_underline_tensor_to(var) return None else: - op = block.append_op(type='assign_value', - outputs={'Out': [out_var]}, - attrs={ - 'dtype': out_dtype, - 'shape': list(shape), - value_name: values - }) + op = block.append_op( + type='assign_value', + outputs={'Out': [out_var]}, + attrs={ + 'dtype': out_dtype, + 'shape': list(shape), + value_name: values, + }, + ) if var.dtype in [ - VarDesc.VarType.FP16, VarDesc.VarType.BF16, - VarDesc.VarType.FP64 + VarDesc.VarType.FP16, + VarDesc.VarType.BF16, + VarDesc.VarType.FP64, ]: - block.append_op(type="cast", - inputs={"X": out_var}, - outputs={"Out": var}, - attrs={ - "in_dtype": out_var.dtype, - "out_dtype": var.dtype - }) + block.append_op( + type="cast", + inputs={"X": out_var}, + outputs={"Out": var}, + attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}, + ) var.op = op return op @@ -1039,6 +1232,7 @@ class NumpyArrayInitializer(Initializer): def __init__(self, value): import numpy + assert isinstance(value, numpy.ndarray) super(NumpyArrayInitializer, self).__init__() self._value = value @@ -1063,12 +1257,15 @@ def forward(self, var, block=None): if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: out_dtype = VarDesc.VarType.FP32 np_value = self._value.astype("float32") - out_var = block.create_var(name=unique_name.generate(".".join( - ['numpy_array_init', var.name, 'tmp'])), - shape=var.shape, - dtype=out_dtype, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False) + out_var = block.create_var( + name=unique_name.generate( + ".".join(['numpy_array_init', var.name, 'tmp']) + ), + shape=var.shape, + dtype=out_dtype, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + ) else: out_var = var out_dtype = var.dtype @@ -1083,47 +1280,64 @@ def forward(self, var, block=None): else: raise ValueError("Unsupported dtype %s", self._value.dtype) if self._value.size > 1024 * 1024 * 1024: - raise ValueError("The size of input is too big. Please consider " - "saving it to file and 'load_op' to load it") + raise ValueError( + "The size of input is too big. Please consider " + "saving it to file and 'load_op' to load it" + ) if framework._non_static_mode(): if in_dygraph_mode(): - _C_ops.assign_value_(out_var, - list(self._value.shape), out_dtype, values, - _current_expected_place()) + _C_ops.assign_value_( + out_var, + list(self._value.shape), + out_dtype, + values, + _current_expected_place(), + ) elif _in_legacy_dygraph(): - _legacy_C_ops.assign_value(out_var, 'shape', - list(self._value.shape), 'dtype', - out_dtype, value_name, values) + _legacy_C_ops.assign_value( + out_var, + 'shape', + list(self._value.shape), + 'dtype', + out_dtype, + value_name, + values, + ) if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: if in_dygraph_mode(): var_tmp = _C_ops.cast(out_var, var.dtype) elif _in_legacy_dygraph(): - var_tmp = _legacy_C_ops.cast(out_var, 'in_dtype', - out_var.dtype, 'out_dtype', - var.dtype) + var_tmp = _legacy_C_ops.cast( + out_var, + 'in_dtype', + out_var.dtype, + 'out_dtype', + var.dtype, + ) var_tmp._share_underline_tensor_to(var) else: out_var._share_underline_tensor_to(var) return None else: - op = block.append_op(type='assign_value', - outputs={'Out': out_var}, - attrs={ - 'dtype': out_dtype, - 'shape': list(self._value.shape), - value_name: values - }, - stop_gradient=True) + op = block.append_op( + type='assign_value', + outputs={'Out': out_var}, + attrs={ + 'dtype': out_dtype, + 'shape': list(self._value.shape), + value_name: values, + }, + stop_gradient=True, + ) if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: - block.append_op(type="cast", - inputs={"X": out_var}, - outputs={"Out": var}, - attrs={ - "in_dtype": out_var.dtype, - "out_dtype": var.dtype - }) + block.append_op( + type="cast", + inputs={"X": out_var}, + outputs={"Out": var}, + attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}, + ) var.op = op return op @@ -1179,13 +1393,21 @@ def set_global_initializer(weight_init, bias_init=None): nn.initializer.set_global_initializer(None) """ - check_type(weight_init, 'weight_init', (Initializer, type(None)), - 'set_global_initializer') + check_type( + weight_init, + 'weight_init', + (Initializer, type(None)), + 'set_global_initializer', + ) global _global_weight_initializer_ _global_weight_initializer_ = weight_init - check_type(bias_init, 'bias_init', (Initializer, type(None)), - 'set_global_initializer') + check_type( + bias_init, + 'bias_init', + (Initializer, type(None)), + 'set_global_initializer', + ) global _global_bias_initializer_ _global_bias_initializer_ = bias_init @@ -1244,14 +1466,16 @@ def calculate_gain(nonlinearity, param=None): 'tanh': 5.0 / 3, 'relu': math.sqrt(2.0), 'leaky_relu': math.sqrt(2.0 / (1 + param**2)), - 'selu': 3.0 / 4 + 'selu': 3.0 / 4, } if nonlinearity in recommended_gain.keys(): return recommended_gain[nonlinearity] else: raise ValueError( "nonlinearity function {} is not suppported now.".format( - nonlinearity)) + nonlinearity + ) + ) # We short the class name, since users will use the initializer with the package diff --git a/python/paddle/fluid/input.py b/python/paddle/fluid/input.py index bfab41f86a8e6..255a17d6483c4 100644 --- a/python/paddle/fluid/input.py +++ b/python/paddle/fluid/input.py @@ -25,8 +25,8 @@ def one_hot(input, depth, allow_out_of_range=False): """ :alias_main: paddle.nn.functional.one_hot - :alias: paddle.nn.functional.one_hot,paddle.nn.functional.common.one_hot - :old_api: paddle.fluid.one_hot + :alias: paddle.nn.functional.one_hot,paddle.nn.functional.common.one_hot + :old_api: paddle.fluid.one_hot The operator converts each id in the input to an one-hot vector with a depth length. The value in the vector dimension corresponding to the id @@ -122,23 +122,27 @@ def one_hot(input, depth, allow_out_of_range=False): depth.stop_gradient = True inputs = {'X': input, 'depth_tensor': depth} attrs = {'allow_out_of_range': allow_out_of_range} - helper.append_op(type="one_hot_v2", - inputs=inputs, - attrs=attrs, - outputs={'Out': one_hot_out}, - stop_gradient=True) + helper.append_op( + type="one_hot_v2", + inputs=inputs, + attrs=attrs, + outputs={'Out': one_hot_out}, + stop_gradient=True, + ) return one_hot_out @static_only @deprecated(since='2.0.0', update_to='paddle.nn.functional.embedding') -def embedding(input, - size, - is_sparse=False, - is_distributed=False, - padding_idx=None, - param_attr=None, - dtype='float32'): +def embedding( + input, + size, + is_sparse=False, + is_distributed=False, + padding_idx=None, + param_attr=None, + dtype='float32', +): r""" :api_attr: Static Graph @@ -310,28 +314,35 @@ def embedding(input, helper = LayerHelper('embedding', **locals()) check_variable_and_dtype(input, 'input', ['int64'], 'fluid.embedding') - check_dtype(dtype, 'dtype', ['float16', 'float32', 'float64', 'uint16'], - 'fluid.embedding') + check_dtype( + dtype, + 'dtype', + ['float16', 'float32', 'float64', 'uint16'], + 'fluid.embedding', + ) remote_prefetch = is_sparse and (not is_distributed) if remote_prefetch: assert is_sparse is True and is_distributed is False - w = helper.create_parameter(attr=helper.param_attr, - shape=size, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=helper.param_attr, shape=size, dtype=dtype, is_bias=False + ) tmp = helper.create_variable_for_type_inference(dtype) - padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( - size[0] + padding_idx) - helper.append_op(type='lookup_table_v2', - inputs={ - 'Ids': input, - 'W': w - }, - outputs={'Out': tmp}, - attrs={ - 'is_sparse': is_sparse, - 'is_distributed': is_distributed, - 'remote_prefetch': remote_prefetch, - 'padding_idx': padding_idx - }) + padding_idx = ( + -1 + if padding_idx is None + else padding_idx + if padding_idx >= 0 + else (size[0] + padding_idx) + ) + helper.append_op( + type='lookup_table_v2', + inputs={'Ids': input, 'W': w}, + outputs={'Out': tmp}, + attrs={ + 'is_sparse': is_sparse, + 'is_distributed': is_distributed, + 'remote_prefetch': remote_prefetch, + 'padding_idx': padding_idx, + }, + ) return tmp diff --git a/python/paddle/fluid/install_check.py b/python/paddle/fluid/install_check.py index 299275be5e2c3..74bc8df61bbcb 100644 --- a/python/paddle/fluid/install_check.py +++ b/python/paddle/fluid/install_check.py @@ -14,7 +14,13 @@ import os import paddle -from .framework import Program, program_guard, unique_name, cuda_places, cpu_places +from .framework import ( + Program, + program_guard, + unique_name, + cuda_places, + cpu_places, +) from .param_attr import ParamAttr from .initializer import Constant from . import layers @@ -31,13 +37,11 @@ class SimpleLayer(Layer): - def __init__(self, input_size): super(SimpleLayer, self).__init__() self._linear1 = nn.Linear( - input_size, - 3, - param_attr=ParamAttr(initializer=Constant(value=0.1))) + input_size, 3, param_attr=ParamAttr(initializer=Constant(value=0.1)) + ) def forward(self, inputs): x = self._linear1(inputs) @@ -74,7 +78,8 @@ def run_check(): except Exception as e: logging.warning( "You are using GPU version Paddle Fluid, But Your CUDA Device is not set properly" - "\n Original Error is {}".format(e)) + "\n Original Error is {}".format(e) + ) return 0 device_list = cuda_places() else: @@ -100,22 +105,29 @@ def test_parallerl_exe(): simple_layer = SimpleLayer(input_size=2) out = simple_layer(inp) exe = executor.Executor( - core.CUDAPlace(0) if core.is_compiled_with_cuda() and - (core.get_cuda_device_count() > 0) else core.CPUPlace()) + core.CUDAPlace(0) + if core.is_compiled_with_cuda() + and (core.get_cuda_device_count() > 0) + else core.CPUPlace() + ) loss = paddle.mean(out) loss.persistable = True optimizer.SGD(learning_rate=0.01).minimize(loss) startup_prog.random_seed = 1 compiled_prog = compiler.CompiledProgram( - train_prog).with_data_parallel( - build_strategy=build_strategy, - loss_name=loss.name, - places=device_list) + train_prog + ).with_data_parallel( + build_strategy=build_strategy, + loss_name=loss.name, + places=device_list, + ) exe.run(startup_prog) - exe.run(compiled_prog, - feed={inp.name: np_inp_muti}, - fetch_list=[loss.name]) + exe.run( + compiled_prog, + feed={inp.name: np_inp_muti}, + fetch_list=[loss.name], + ) def test_simple_exe(): train_prog = Program() @@ -124,20 +136,26 @@ def test_simple_exe(): with executor.scope_guard(scope): with program_guard(train_prog, startup_prog): with unique_name.guard(): - inp0 = layers.data(name="inp", - shape=[2, 2], - append_batch_size=False) + inp0 = layers.data( + name="inp", shape=[2, 2], append_batch_size=False + ) simple_layer0 = SimpleLayer(input_size=2) out0 = simple_layer0(inp0) param_grads = backward.append_backward( out0, - parameter_list=[simple_layer0._linear1.weight.name])[0] + parameter_list=[simple_layer0._linear1.weight.name], + )[0] exe0 = executor.Executor( - core.CUDAPlace(0) if core.is_compiled_with_cuda() and - (core.get_cuda_device_count() > 0) else core.CPUPlace()) + core.CUDAPlace(0) + if core.is_compiled_with_cuda() + and (core.get_cuda_device_count() > 0) + else core.CPUPlace() + ) exe0.run(startup_prog) - exe0.run(feed={inp0.name: np_inp_single}, - fetch_list=[out0.name, param_grads[1].name]) + exe0.run( + feed={inp0.name: np_inp_single}, + fetch_list=[out0.name, param_grads[1].name], + ) test_simple_exe() @@ -160,6 +178,7 @@ def test_simple_exe(): print("\n Original Error is: {}".format(e)) print( "Your Paddle Fluid is installed successfully ONLY for SINGLE GPU or CPU! " - "\n Let's start deep Learning with Paddle Fluid now") + "\n Let's start deep Learning with Paddle Fluid now" + ) paddle.disable_static() diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py index 1af1d0a606d8e..c5e55a95186a2 100644 --- a/python/paddle/fluid/io.py +++ b/python/paddle/fluid/io.py @@ -28,10 +28,28 @@ from paddle.fluid import layers from paddle.fluid.executor import Executor, global_scope from paddle.fluid.evaluator import Evaluator -from paddle.fluid.framework import Program, Parameter, default_main_program, default_startup_program, Variable, \ - program_guard, dygraph_not_support, static_only -from paddle.reader import cache, map_readers, buffered, compose, chain, shuffle, \ - ComposeNotAligned, firstn, xmap_readers, multiprocess_reader +from paddle.fluid.framework import ( + Program, + Parameter, + default_main_program, + default_startup_program, + Variable, + program_guard, + dygraph_not_support, + static_only, +) +from paddle.reader import ( + cache, + map_readers, + buffered, + compose, + chain, + shuffle, + ComposeNotAligned, + firstn, + xmap_readers, + multiprocess_reader, +) from .wrapped_decorator import signature_safe_contextmanager from paddle.fluid.compiler import CompiledProgram from paddle.fluid.log_helper import get_logger @@ -64,13 +82,12 @@ 'get_program_persistable_vars', ] + reader.__all__ -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class _open_buffer(object): - def __init__(self, buffer): self.buffer = buffer @@ -79,7 +96,6 @@ def __enter__(self): class _buffer_reader(_open_buffer): - def __init__(self, buffer): super(_buffer_reader, self).__init__(buffer) self.initial_tell = self.buffer.tell() @@ -91,7 +107,6 @@ def __exit__(self, *args): class _buffer_writer(_open_buffer): - def __exit__(self, *args): self.buffer.flush() @@ -111,7 +126,8 @@ def _open_file_buffer(path_or_buffer, mode): return _buffer_reader(path_or_buffer) else: raise ValueError( - "Expected 'r' or 'w' in mode but got {}".format(mode)) + "Expected 'r' or 'w' in mode but got {}".format(mode) + ) def _is_memory_buffer(buffer): @@ -163,9 +179,11 @@ def is_persistable(var): param = fluid.default_main_program().global_block().var('fc.b') res = fluid.io.is_persistable(param) """ - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable @@ -232,18 +250,22 @@ def get_program_persistable_vars(program): def _clone_var_in_block_(block, var): assert isinstance(var, Variable) if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR: - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=True) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=True, + ) else: - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - persistable=True) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + persistable=True, + ) @signature_safe_contextmanager @@ -268,21 +290,25 @@ def _get_valid_program(main_program=None): "The type of input main_program is invalid, expected tyep is Program, but received None" ) warnings.warn( - "The input is a CompiledProgram, this is not recommended.") + "The input is a CompiledProgram, this is not recommended." + ) if not isinstance(main_program, Program): raise TypeError( "The type of input main_program is invalid, expected type is fluid.Program, but received %s" - % type(main_program)) + % type(main_program) + ) return main_program @dygraph_not_support -def save_vars(executor, - dirname, - main_program=None, - vars=None, - predicate=None, - filename=None): +def save_vars( + executor, + dirname, + main_program=None, + vars=None, + predicate=None, + filename=None, +): """ Save specific variables in the `Program` to files. @@ -360,11 +386,13 @@ def name_has_fc(var): main_program = _get_valid_program(main_program) if vars is None: - return save_vars(executor, - main_program=main_program, - dirname=dirname, - vars=list(filter(predicate, main_program.list_vars())), - filename=filename) + return save_vars( + executor, + main_program=main_program, + dirname=dirname, + vars=list(filter(predicate, main_program.list_vars())), + filename=filename, + ) else: params_var_name = "saved_params" # give warning when there is no var in model @@ -384,13 +412,15 @@ def name_has_fc(var): continue new_var = _clone_var_in_block_(save_block, each_var) if filename is None and save_to_memory is False: - save_file_path = os.path.join(os.path.normpath(dirname), - new_var.name) + save_file_path = os.path.join( + os.path.normpath(dirname), new_var.name + ) save_block.append_op( type='save', inputs={'X': [new_var]}, outputs={}, - attrs={'file_path': os.path.normpath(save_file_path)}) + attrs={'file_path': os.path.normpath(save_file_path)}, + ) else: save_var_map[new_var.name] = new_var @@ -403,16 +433,19 @@ def name_has_fc(var): if save_to_memory is False: save_path = os.path.join(os.path.normpath(dirname), filename) - saved_params = save_block.create_var(type=core.VarDesc.VarType.RAW, - name=params_var_name) + saved_params = save_block.create_var( + type=core.VarDesc.VarType.RAW, name=params_var_name + ) saved_params.desc.set_persistable(True) - save_block.append_op(type='save_combine', - inputs={'X': save_var_list}, - outputs={'Y': saved_params}, - attrs={ - 'file_path': save_path, - 'save_to_memory': save_to_memory - }) + save_block.append_op( + type='save_combine', + inputs={'X': save_var_list}, + outputs={'Y': saved_params}, + attrs={ + 'file_path': save_path, + 'save_to_memory': save_to_memory, + }, + ) # NOTE(zhiqiu): save op will add variable kLookupTablePath in save_program.desc, # which leads to diff on save_program and its desc. Call _sync_with_cpp @@ -489,12 +522,14 @@ def save_params(executor, dirname, main_program=None, filename=None): # The parameters weights and bias of the fc layer in the network are going to # be saved in different files in the path "./my_paddle_model" """ - return save_vars(executor, - dirname=dirname, - main_program=main_program, - vars=None, - predicate=is_parameter, - filename=filename) + return save_vars( + executor, + dirname=dirname, + main_program=main_program, + vars=None, + predicate=is_parameter, + filename=filename, + ) def _save_distributed_persistables(executor, dirname, main_program): @@ -568,21 +603,24 @@ def __save_remote_params(executor, dirname, remote_params_map): tmp = [str(dim) for dim in slice.shape] slice_shapes.append(",".join(tmp)) - block.append_op(type='recv_save', - attrs={ - "trainer_id": 0, - "shape": origin.shape, - "slice_shapes": slice_shapes, - "slice_varnames": slice_varnames, - "remote_varnames": remote_varnames, - "endpoints": endpoints, - "file_path": os.path.join(dirname, origin.name) - }) + block.append_op( + type='recv_save', + attrs={ + "trainer_id": 0, + "shape": origin.shape, + "slice_shapes": slice_shapes, + "slice_varnames": slice_varnames, + "remote_varnames": remote_varnames, + "endpoints": endpoints, + "file_path": os.path.join(dirname, origin.name), + }, + ) executor.run(prog) - def __save_distributed_lookup_tables(executor, dirname, - distributed_lookup_table, endpoints): + def __save_distributed_lookup_tables( + executor, dirname, distributed_lookup_table, endpoints + ): """ because the distributed lookup table may too huge to merge and save at one place, it will be saved at parameter server independent respectively. @@ -599,20 +637,20 @@ def __save_distributed_lookup_tables(executor, dirname, attrs['epmap'] = endpoints attrs['dir'] = lookup_table_filename attrs['lookup_table'] = distributed_lookup_table - block.append_op(type='checkpoint_notify', - inputs={}, - outputs={}, - attrs=attrs) + block.append_op( + type='checkpoint_notify', inputs={}, outputs={}, attrs=attrs + ) executor.run(prog) def __exclude_vars(exclude_var_names=[]): - def is_valid(var): if var.name in exclude_var_names: return False - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable @@ -626,8 +664,11 @@ def is_valid(var): "'_save_distributed_persistables' just be designed for distributed training." ) - remote_params_map = main_program._parameters_on_pservers.get_distributed_vars_by_vtypes( - ["Optimizer", "RemotePrefetch"], groupby=True) + remote_params_map = ( + main_program._parameters_on_pservers.get_distributed_vars_by_vtypes( + ["Optimizer", "RemotePrefetch"], groupby=True + ) + ) exclude_var_names = [] if remote_params_map: @@ -640,19 +681,22 @@ def is_valid(var): exclude_var_names.append(main_program._distributed_lookup_table) local_vars = list( - filter(__exclude_vars(exclude_var_names), main_program.list_vars())) - save_vars(executor, - main_program=main_program, - dirname=dirname, - vars=local_vars) + filter(__exclude_vars(exclude_var_names), main_program.list_vars()) + ) + save_vars( + executor, main_program=main_program, dirname=dirname, vars=local_vars + ) if main_program._is_chief: if remote_params_map: __save_remote_params(executor, dirname, remote_params_map) if main_program._distributed_lookup_table: __save_distributed_lookup_tables( - executor, dirname, main_program._distributed_lookup_table, - main_program._endpoints) + executor, + dirname, + main_program._distributed_lookup_table, + main_program._endpoints, + ) @dygraph_not_support @@ -714,24 +758,28 @@ def save_persistables(executor, dirname, main_program=None, filename=None): # "./my_paddle_model" """ if main_program and main_program._is_distributed: - return _save_distributed_persistables(executor, - dirname=dirname, - main_program=main_program) + return _save_distributed_persistables( + executor, dirname=dirname, main_program=main_program + ) else: - return save_vars(executor, - dirname=dirname, - main_program=main_program, - vars=None, - predicate=is_persistable, - filename=filename) - - -def load_vars(executor, - dirname, - main_program=None, - vars=None, - predicate=None, - filename=None): + return save_vars( + executor, + dirname=dirname, + main_program=main_program, + vars=None, + predicate=is_persistable, + filename=filename, + ) + + +def load_vars( + executor, + dirname, + main_program=None, + vars=None, + predicate=None, + filename=None, +): """ :api_attr: Static Graph @@ -820,13 +868,16 @@ def name_has_fc(var): if not isinstance(main_program, Program): raise TypeError( "The type of input main_program is invalid, expected type is fluid.Program, but received %s" - % type(main_program)) + % type(main_program) + ) - load_vars(executor, - dirname=dirname, - main_program=main_program, - vars=list(filter(predicate, main_program.list_vars())), - filename=filename) + load_vars( + executor, + dirname=dirname, + main_program=main_program, + vars=list(filter(predicate, main_program.list_vars())), + filename=filename, + ) else: load_prog = Program() load_block = load_prog.global_block() @@ -837,7 +888,8 @@ def name_has_fc(var): if not isinstance(main_program, Program): raise TypeError( "The type of input main_program is invalid, expected type is fluid.Program, but received %s" - % type(main_program)) + % type(main_program) + ) # save origin param shape orig_para_shape = {} @@ -854,7 +906,8 @@ def name_has_fc(var): if isinstance(each_var, Parameter): orig_para_shape[each_var.name] = tuple( - each_var.desc.get_shape()) + each_var.desc.get_shape() + ) if each_var.type == core.VarDesc.VarType.SELECTED_ROWS: sparse_vars.append(each_var) @@ -872,7 +925,8 @@ def name_has_fc(var): type='load', inputs={}, outputs={'Out': [new_var]}, - attrs={'file_path': os.path.join(dirname, new_var.name)}) + attrs={'file_path': os.path.join(dirname, new_var.name)}, + ) else: load_var_map[new_var.name] = new_var @@ -881,7 +935,8 @@ def name_has_fc(var): if filename is not None: raise ValueError( - "SelectedRows can not be load with load_combine") + "SelectedRows can not be load with load_combine" + ) new_var = _clone_var_in_block_(load_block, each_var) @@ -889,14 +944,17 @@ def name_has_fc(var): if not os.path.exists(var_path): raise ValueError( "SelectedRows var {} can not find at {}".format( - new_var.name, var_path)) + new_var.name, var_path + ) + ) if os.path.isfile(var_path): load_block.append_op( type='load', inputs={}, outputs={'Out': [new_var]}, - attrs={'file_path': os.path.join(dirname, new_var.name)}) + attrs={'file_path': os.path.join(dirname, new_var.name)}, + ) else: blocks = [] block_paths = os.listdir(var_path) @@ -907,23 +965,29 @@ def name_has_fc(var): slices = [] for block in blocks: - slice = load_block.create_var(name=block, - type=new_var.type, - shape=new_var.shape, - dtype=new_var.dtype, - persistable=False) + slice = load_block.create_var( + name=block, + type=new_var.type, + shape=new_var.shape, + dtype=new_var.dtype, + persistable=False, + ) slices.append(slice) file_path = os.path.join(var_path, block, "Param") - load_block.append_op(type='load', - inputs={}, - outputs={'Out': [slice]}, - attrs={'file_path': file_path}) + load_block.append_op( + type='load', + inputs={}, + outputs={'Out': [slice]}, + attrs={'file_path': file_path}, + ) - load_block.append_op(type='lookup_sparse_table_merge', - inputs={'X': slices}, - outputs={'Out': new_var}, - attrs={}) + load_block.append_op( + type='lookup_sparse_table_merge', + inputs={'X': slices}, + outputs={'Out': new_var}, + attrs={}, + ) if filename is not None: load_var_list = [] @@ -933,13 +997,15 @@ def name_has_fc(var): if vars_from_memory is False: filename = os.path.join(dirname, filename) - load_block.append_op(type='load_combine', - inputs={}, - outputs={"Out": load_var_list}, - attrs={ - 'file_path': filename, - 'model_from_memory': vars_from_memory - }) + load_block.append_op( + type='load_combine', + inputs={}, + outputs={"Out": load_var_list}, + attrs={ + 'file_path': filename, + 'model_from_memory': vars_from_memory, + }, + ) executor.run(load_prog) # check var shape @@ -949,13 +1015,17 @@ def name_has_fc(var): var_temp = paddle.fluid.global_scope().find_var(each_var.name) assert var_temp != None, "can't not find var: " + each_var.name new_shape = (np.array(var_temp.get_tensor())).shape - assert each_var.name in orig_para_shape, each_var.name + "MUST in var list" + assert each_var.name in orig_para_shape, ( + each_var.name + "MUST in var list" + ) orig_shape = orig_para_shape.get(each_var.name) if new_shape != orig_shape: raise RuntimeError( "Variable's shape does not match, the Program requires a parameter with the shape of ({}), " - "while the loaded parameter (namely [ {} ]) has a shape of ({})." - .format(orig_shape, each_var.name, new_shape)) + "while the loaded parameter (namely [ {} ]) has a shape of ({}).".format( + orig_shape, each_var.name, new_shape + ) + ) @dygraph_not_support @@ -1012,11 +1082,13 @@ def load_params(executor, dirname, main_program=None, filename=None): fluid.io.load_params(executor=exe, dirname=param_path, main_program=None) """ - load_vars(executor, - dirname=dirname, - main_program=main_program, - predicate=is_parameter, - filename=filename) + load_vars( + executor, + dirname=dirname, + main_program=main_program, + predicate=is_parameter, + filename=filename, + ) @dygraph_not_support @@ -1064,15 +1136,17 @@ def load_persistables(executor, dirname, main_program=None, filename=None): """ if main_program and main_program._is_distributed: - _load_distributed_persistables(executor, - dirname=dirname, - main_program=main_program) + _load_distributed_persistables( + executor, dirname=dirname, main_program=main_program + ) else: - load_vars(executor, - dirname=dirname, - main_program=main_program, - predicate=is_persistable, - filename=filename) + load_vars( + executor, + dirname=dirname, + main_program=main_program, + predicate=is_persistable, + filename=filename, + ) def _load_distributed_persistables(executor, dirname, main_program=None): @@ -1122,35 +1196,38 @@ def __load_persistable_vars(executor, dirname, need_load_vars): offset = param.offset if is_slice: - slice = load_block.create_var(name=slice_var.name, - type=slice_var.type, - shape=slice_var.shape, - dtype=slice_var.dtype, - persistable=True) - - load_block.append_op(type='load', - inputs={}, - outputs={'Out': [slice]}, - attrs={ - 'file_path': - os.path.join(dirname, origin_var.name), - 'seek': - offset, - 'shape': - slice.shape - }) + slice = load_block.create_var( + name=slice_var.name, + type=slice_var.type, + shape=slice_var.shape, + dtype=slice_var.dtype, + persistable=True, + ) + + load_block.append_op( + type='load', + inputs={}, + outputs={'Out': [slice]}, + attrs={ + 'file_path': os.path.join(dirname, origin_var.name), + 'seek': offset, + 'shape': slice.shape, + }, + ) else: - origin = load_block.create_var(name="{}".format( - origin_var.name), - type=origin_var.type, - shape=origin_var.shape, - dtype=origin_var.dtype, - persistable=True) + origin = load_block.create_var( + name="{}".format(origin_var.name), + type=origin_var.type, + shape=origin_var.shape, + dtype=origin_var.dtype, + persistable=True, + ) load_block.append_op( type='load', inputs={}, outputs={'Out': [origin]}, - attrs={'file_path': os.path.join(dirname, origin_var.name)}) + attrs={'file_path': os.path.join(dirname, origin_var.name)}, + ) load_block.append_op( type='delete_var', @@ -1172,63 +1249,78 @@ def __load_persistable_vars(executor, dirname, need_load_vars): "'_load_distributed_persistables' need current_endpoint set in DistributeTranspiler.transpile" ) - need_load_vars = main_program._parameters_on_pservers.get_distributed_vars_by_ep( - main_program._ps_endpoint) + need_load_vars = ( + main_program._parameters_on_pservers.get_distributed_vars_by_ep( + main_program._ps_endpoint + ) + ) __load_persistable_vars(executor, dirname, need_load_vars) -def prepend_feed_ops(inference_program, - feed_target_names, - feed_holder_name='feed'): +def prepend_feed_ops( + inference_program, feed_target_names, feed_holder_name='feed' +): if len(feed_target_names) == 0: return global_block = inference_program.global_block() - feed_var = global_block.create_var(name=feed_holder_name, - type=core.VarDesc.VarType.FEED_MINIBATCH, - persistable=True) + feed_var = global_block.create_var( + name=feed_holder_name, + type=core.VarDesc.VarType.FEED_MINIBATCH, + persistable=True, + ) for i, name in enumerate(feed_target_names): if not global_block.has_var(name): raise ValueError( "The feeded_var_names[{i}]: '{name}' doesn't exist in pruned inference program. " "Please check whether '{name}' is a valid feed_var name, or remove it from feeded_var_names " - "if '{name}' is not involved in the target_vars calculation.". - format(i=i, name=name)) + "if '{name}' is not involved in the target_vars calculation.".format( + i=i, name=name + ) + ) out = global_block.var(name) - global_block._prepend_op(type='feed', - inputs={'X': [feed_var]}, - outputs={'Out': [out]}, - attrs={'col': i}) + global_block._prepend_op( + type='feed', + inputs={'X': [feed_var]}, + outputs={'Out': [out]}, + attrs={'col': i}, + ) -def append_fetch_ops(inference_program, - fetch_target_names, - fetch_holder_name='fetch'): +def append_fetch_ops( + inference_program, fetch_target_names, fetch_holder_name='fetch' +): global_block = inference_program.global_block() - fetch_var = global_block.create_var(name=fetch_holder_name, - type=core.VarDesc.VarType.FETCH_LIST, - persistable=True) + fetch_var = global_block.create_var( + name=fetch_holder_name, + type=core.VarDesc.VarType.FETCH_LIST, + persistable=True, + ) for i, name in enumerate(fetch_target_names): - global_block.append_op(type='fetch', - inputs={'X': [name]}, - outputs={'Out': [fetch_var]}, - attrs={'col': i}) + global_block.append_op( + type='fetch', + inputs={'X': [name]}, + outputs={'Out': [fetch_var]}, + attrs={'col': i}, + ) @static_only @deprecated(since="2.0.0", update_to="paddle.static.save_inference_model") -def save_inference_model(dirname, - feeded_var_names, - target_vars, - executor, - main_program=None, - model_filename=None, - params_filename=None, - export_for_deployment=True, - program_only=False, - clip_extra=True): +def save_inference_model( + dirname, + feeded_var_names, + target_vars, + executor, + main_program=None, + model_filename=None, + params_filename=None, + export_for_deployment=True, + program_only=False, + clip_extra=True, +): """ Prune the given `main_program` to build a new program especially for inference, and then save it and all related parameters to given `dirname` . @@ -1314,16 +1406,19 @@ def save_inference_model(dirname, elif export_for_deployment: if len(feeded_var_names) > 0: # TODO(paddle-dev): polish these code blocks - if not (bool(feeded_var_names) - and all(isinstance(name, str) - for name in feeded_var_names)): + if not ( + bool(feeded_var_names) + and all(isinstance(name, str) for name in feeded_var_names) + ): raise ValueError("'feed_var_names' should be a list of str.") if isinstance(target_vars, Variable): target_vars = [target_vars] elif export_for_deployment: - if not (bool(target_vars) - and all(isinstance(var, Variable) for var in target_vars)): + if not ( + bool(target_vars) + and all(isinstance(var, Variable) for var in target_vars) + ): raise ValueError("'target_vars' should be a list of Variable.") main_program = _get_valid_program(main_program) @@ -1384,7 +1479,8 @@ def save_inference_model(dirname, main_program.desc.flush() main_program = main_program._prune_with_input( - feeded_var_names=feeded_var_names, targets=target_vars) + feeded_var_names=feeded_var_names, targets=target_vars + ) main_program = main_program._inference_optimize(prune_read_op=True) fetch_var_names = [v.name for v in target_vars] @@ -1394,7 +1490,8 @@ def save_inference_model(dirname, name=target_v.name, shape=target_v.shape, dtype=target_v.dtype, - persistable=target_v.persistable) + persistable=target_v.persistable, + ) prepend_feed_ops(main_program, feeded_var_names) append_fetch_ops(main_program, fetch_var_names) @@ -1404,14 +1501,18 @@ def save_inference_model(dirname, with open(model_basename, "wb") as f: f.write( main_program._remove_training_info( - clip_extra=clip_extra).desc.serialize_to_string()) + clip_extra=clip_extra + ).desc.serialize_to_string() + ) else: # TODO(panyx0718): Save more information so that it can also be used # for training and more flexible post-processing. with open(model_basename + ".main_program", "wb") as f: f.write( main_program._remove_training_info( - clip_extra=clip_extra).desc.serialize_to_string()) + clip_extra=clip_extra + ).desc.serialize_to_string() + ) if program_only: warnings.warn( @@ -1430,11 +1531,13 @@ def save_inference_model(dirname, @static_only @deprecated(since="2.0.0", update_to="paddle.static.load_inference_model") -def load_inference_model(dirname, - executor, - model_filename=None, - params_filename=None, - pserver_endpoints=None): +def load_inference_model( + dirname, + executor, + model_filename=None, + params_filename=None, + pserver_endpoints=None, +): """ Load the inference model from a given directory. By this API, you can get the model structure(Inference Program) and model parameters. If you just want to load @@ -1534,8 +1637,9 @@ def load_inference_model(dirname, if model_filename is None: model_filename = '__model__' - model_filename = os.path.join(load_dirname, - os.path.basename(model_filename)) + model_filename = os.path.join( + load_dirname, os.path.basename(model_filename) + ) if params_filename is not None: params_filename = os.path.basename(params_filename) @@ -1554,8 +1658,9 @@ def load_inference_model(dirname, program = Program.parse_from_string(program_desc_str) if not core._is_program_version_supported(program._version()): - raise ValueError("Unsupported program version: %d\n" % - program._version()) + raise ValueError( + "Unsupported program version: %d\n" % program._version() + ) # Binary data also need versioning. load_persistables(executor, load_dirname, program, params_filename) @@ -1666,8 +1771,10 @@ def _save_persistable_nodes(executor, dirname, graph): var_list = [] for node in persistable_nodes: var_desc = node.var() - if var_desc.type() == core.VarDesc.VarType.RAW or \ - var_desc.type() == core.VarDesc.VarType.READER: + if ( + var_desc.type() == core.VarDesc.VarType.RAW + or var_desc.type() == core.VarDesc.VarType.READER + ): continue var = program.global_block().create_var( name=var_desc.name(), @@ -1675,7 +1782,8 @@ def _save_persistable_nodes(executor, dirname, graph): dtype=var_desc.dtype(), type=var_desc.type(), lod_level=var_desc.lod_level(), - persistable=var_desc.persistable()) + persistable=var_desc.persistable(), + ) var_list.append(var) save_vars(executor=executor, dirname=dirname, vars=var_list) @@ -1705,8 +1813,10 @@ def _exist(var): for node in persistable_nodes: var_desc = node.var() - if var_desc.type() == core.VarDesc.VarType.RAW or \ - var_desc.type() == core.VarDesc.VarType.READER: + if ( + var_desc.type() == core.VarDesc.VarType.RAW + or var_desc.type() == core.VarDesc.VarType.READER + ): continue var = program.global_block().create_var( name=var_desc.name(), @@ -1714,7 +1824,8 @@ def _exist(var): dtype=var_desc.dtype(), type=var_desc.type(), lod_level=var_desc.lod_level(), - persistable=var_desc.persistable()) + persistable=var_desc.persistable(), + ) if _exist(var): var_list.append(var) else: @@ -1731,7 +1842,8 @@ def _unpack_saved_dict(saved_obj, protocol): for key, value in saved_obj.items(): if isinstance(value, np.ndarray): MAX_NUMBER_OF_ELEMENT = int( - (2**30 - 1) / value.dtype.itemsize) + (2**30 - 1) / value.dtype.itemsize + ) num_element = np.prod(value.shape) if num_element > MAX_NUMBER_OF_ELEMENT: unpack_infor[key] = {} @@ -1739,15 +1851,19 @@ def _unpack_saved_dict(saved_obj, protocol): unpack_infor[key]["slices"] = [] value = value.flatten() for i in range( - int( - math.ceil(num_element * 1.0 / - MAX_NUMBER_OF_ELEMENT))): + int( + math.ceil( + num_element * 1.0 / MAX_NUMBER_OF_ELEMENT + ) + ) + ): part_name = key + "@@." + str(i) unpack_infor[key]["slices"].append(part_name) temp_saved_obj[part_name] = value[ - i * - MAX_NUMBER_OF_ELEMENT:MAX_NUMBER_OF_ELEMENT * - (i + 1)] + i + * MAX_NUMBER_OF_ELEMENT : MAX_NUMBER_OF_ELEMENT + * (i + 1) + ] if unpack_infor: for key, value in unpack_infor.items(): @@ -1767,7 +1883,8 @@ def _pack_loaded_dict(load_obj): for key, value in load_obj[unpack_info].items(): slices = [load_obj[part] for part in value["slices"]] load_obj[key] = np.concatenate(slices).reshape( - value["OriginShape"]) + value["OriginShape"] + ) removes += value["slices"] for key in removes: load_obj.pop(key) @@ -1778,7 +1895,6 @@ def _pack_loaded_dict(load_obj): @static_only def _legacy_save(param_dict, model_path, protocol=2): - def get_tensor(var): if isinstance(var, (core.VarBase, core.eager.Tensor)): return var.numpy() @@ -1789,14 +1905,16 @@ def get_tensor(var): param_dict = {name: get_tensor(param_dict[name]) for name in param_dict} # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' - if _is_file_path( - model_path - ) and sys.platform == 'darwin' and sys.version_info.major == 3: + if ( + _is_file_path(model_path) + and sys.platform == 'darwin' + and sys.version_info.major == 3 + ): pickle_bytes = pickle.dumps(param_dict, protocol=protocol) with open(model_path, 'wb') as f: max_bytes = 2**30 for i in range(0, len(pickle_bytes), max_bytes): - f.write(pickle_bytes[i:i + max_bytes]) + f.write(pickle_bytes[i : i + max_bytes]) else: with _open_file_buffer(model_path, 'wb') as f: pickle.dump(param_dict, f, protocol=protocol) @@ -1843,8 +1961,9 @@ def save(program, model_path, protocol=4, **configs): """ base_name = os.path.basename(model_path) - assert base_name != "", \ - "The input model_path MUST be format of dirname/filename [dirname\\filename in Windows system], but received model_path is empty string." + assert ( + base_name != "" + ), "The input model_path MUST be format of dirname/filename [dirname\\filename in Windows system], but received model_path is empty string." if 'pickle_protocol' in configs: protocol = configs['pickle_protocol'] warnings.warn( @@ -1852,13 +1971,16 @@ def save(program, model_path, protocol=4, **configs): ) if not isinstance(protocol, int): - raise ValueError("The 'protocol' MUST be `int`, but received {}".format( - type(protocol))) + raise ValueError( + "The 'protocol' MUST be `int`, but received {}".format( + type(protocol) + ) + ) if protocol < 2 or protocol > 4: raise ValueError( - "Expected 1<'protocol'<5, but received protocol={}".format( - protocol)) + "Expected 1<'protocol'<5, but received protocol={}".format(protocol) + ) dir_name = os.path.dirname(model_path) if dir_name and not os.path.exists(dir_name): @@ -1879,13 +2001,14 @@ def get_tensor(var): with open(model_path + ".pdparams", 'wb') as f: max_bytes = 2**30 for i in range(0, len(pickle_bytes), max_bytes): - f.write(pickle_bytes[i:i + max_bytes]) + f.write(pickle_bytes[i : i + max_bytes]) else: with open(model_path + ".pdparams", 'wb') as f: pickle.dump(param_dict, f, protocol=protocol) optimizer_var_list = list( - filter(is_belong_to_optimizer, program.list_vars())) + filter(is_belong_to_optimizer, program.list_vars()) + ) opt_dict = {p.name: get_tensor(p) for p in optimizer_var_list} with open(model_path + ".pdopt", 'wb') as f: @@ -1971,8 +2094,10 @@ def load(program, model_path, executor=None, var_list=None): # model file save by fluid.save not found, try to load model file saved with # [save_vars, save_params, save_persistables] _logger.debug( - "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]" - .format(parameter_file_name)) + "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format( + parameter_file_name + ) + ) if executor is None: raise ValueError( "executor is required when loading model file saved with [ save_params, save_persistables, save_vars ]" @@ -1988,30 +2113,36 @@ def load(program, model_path, executor=None, var_list=None): for root, dirs, files in os.walk(model_path, topdown=False): for f in files: binary_file_set.add( - os.path.join(root, f).replace("\\", "/")) + os.path.join(root, f).replace("\\", "/") + ) program_var_list = list(program.list_vars()) loaded_var_list = [] for var in program_var_list: var_path = os.path.join(model_path, var.name).replace("\\", "/") - load_condition = var_list_names is None or var.name in var_list_names + load_condition = ( + var_list_names is None or var.name in var_list_names + ) if var_path in binary_file_set and load_condition: loaded_var_list.append(var) binary_file_set.remove(var_path) if len(binary_file_set) > 0: unused_var_list = " ".join(list(binary_file_set)) - _logger.warning("variable file [ %s ] not used" % - (" ".join(list(binary_file_set)))) + _logger.warning( + "variable file [ %s ] not used" + % (" ".join(list(binary_file_set))) + ) try: - load_vars(executor=executor, - dirname=model_path, - vars=loaded_var_list) + load_vars( + executor=executor, dirname=model_path, vars=loaded_var_list + ) except RuntimeError as e: _logger.error(e) raise e except: raise RuntimeError( "Failed to load model file, please make sure model file is saved with the " - "following APIs: save_params, save_persistables, save_vars") + "following APIs: save_params, save_persistables, save_vars" + ) return elif os.path.isfile(model_path): @@ -2026,21 +2157,26 @@ def load(program, model_path, executor=None, var_list=None): for var in var_list: if var.name not in program_var_name_set: raise LookupError( - "loaded var [{}] is not in program variable list") + "loaded var [{}] is not in program variable list" + ) dir_name, file_name = os.path.split(model_path) try: - load_vars(executor=executor, - dirname=dir_name, - vars=var_list, - filename=file_name) + load_vars( + executor=executor, + dirname=dir_name, + vars=var_list, + filename=file_name, + ) except RuntimeError as e: _logger.error(e) raise e except: - raise RuntimeError("Failed to load model file , please make sure model file is saved with the " \ - "the following APIs: [ save_params, save_persistables, save_vars ]. " \ - "When these API called, filename CANNOT be None") + raise RuntimeError( + "Failed to load model file , please make sure model file is saved with the " + "the following APIs: [ save_params, save_persistables, save_vars ]. " + "When these API called, filename CANNOT be None" + ) return @@ -2073,9 +2209,9 @@ def set_var(var, ndarray): parameter_list = list(filter(is_parameter, program.list_vars())) if executor: - paddle.fluid.core._create_loaded_parameter(parameter_list, - global_scope(), - executor._default_executor) + paddle.fluid.core._create_loaded_parameter( + parameter_list, global_scope(), executor._default_executor + ) with open(parameter_file_name, 'rb') as f: # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' @@ -2085,29 +2221,36 @@ def set_var(var, ndarray): load_dict = pickle.load(f, encoding='latin1') load_dict = _pack_loaded_dict(load_dict) for v in parameter_list: - assert v.name in load_dict, \ - "Can not find [{}] in model file [{}]".format( - v.name, parameter_file_name) + assert ( + v.name in load_dict + ), "Can not find [{}] in model file [{}]".format( + v.name, parameter_file_name + ) set_var(v, load_dict[v.name]) optimizer_var_list = list( - filter(is_belong_to_optimizer, program.list_vars())) + filter(is_belong_to_optimizer, program.list_vars()) + ) if len(optimizer_var_list) > 0: opt_file_name = model_prefix + ".pdopt" - assert os.path.exists(opt_file_name), \ - "Optimizer file [{}] not exits".format(opt_file_name) + assert os.path.exists( + opt_file_name + ), "Optimizer file [{}] not exits".format(opt_file_name) if executor: paddle.fluid.core._create_loaded_parameter( - optimizer_var_list, global_scope(), executor._default_executor) + optimizer_var_list, global_scope(), executor._default_executor + ) with open(opt_file_name, 'rb') as f: load_dict = pickle.load(f, encoding='latin1') for v in optimizer_var_list: - assert v.name in load_dict, \ - "Can not find [{}] in model file [{}]".format( - v.name, opt_file_name) + assert ( + v.name in load_dict + ), "Can not find [{}] in model file [{}]".format( + v.name, opt_file_name + ) set_var(v, load_dict[v.name]) @@ -2160,13 +2303,16 @@ def load_program_state(model_path, var_list=None): # model file saved with fluid.save is not found, try to load model file saved with # [save_vars, save_params, save_persistables] _logger.debug( - "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]" - .format(parameter_file_name)) + "{} not found, try to load model file saved with [ save_params, save_persistables, save_vars ]".format( + parameter_file_name + ) + ) var_name_list = [] if var_list is None and os.path.isfile(model_path): raise ValueError( - "var_list can not be None when model_path is a file type") + "var_list can not be None when model_path is a file type" + ) for root, dirs, files in os.walk(model_path, topdown=False): for f in files: @@ -2187,27 +2333,34 @@ def clone_var_to_block(block, var): shape=var.shape, dtype=var.dtype, type=var.type, - lod_level=var.lod_level if var.desc.type() - == core.VarDesc.VarType.LOD_TENSOR else None, - persistable=True) - - def _load_vars_with_try_catch(exe, - dirname, - vars, - filename, - raise_error=True): + lod_level=var.lod_level + if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR + else None, + persistable=True, + ) + + def _load_vars_with_try_catch( + exe, dirname, vars, filename, raise_error=True + ): try: - load_vars(executor=exe, - dirname=dirname, - vars=vars, - filename=filename) + load_vars( + executor=exe, + dirname=dirname, + vars=vars, + filename=filename, + ) return True except: - error_str = "Failed to load model/variables `%s`, please make sure " \ - "model/variables file is saved with the following APIs: " \ - "save_params, save_persistables, save_vars." - filenames = [var.name for var in vars - ] if filename is None else filename + error_str = ( + "Failed to load model/variables `%s`, please make sure " + "model/variables file is saved with the following APIs: " + "save_params, save_persistables, save_vars." + ) + filenames = ( + [var.name for var in vars] + if filename is None + else filename + ) if raise_error: raise RuntimeError(error_str % filenames) else: @@ -2224,16 +2377,19 @@ def _load_vars_with_try_catch(exe, dir_name, file_name = os.path.split(model_path) for var in var_list: loaded_var_list.append(clone_var_to_block(load_block, var)) - _load_vars_with_try_catch(exe, dir_name, loaded_var_list, - file_name) + _load_vars_with_try_catch( + exe, dir_name, loaded_var_list, file_name + ) else: # var_list can be None or not None if var_list is not None: for var in var_list: loaded_var_list.append( - clone_var_to_block(load_block, var)) - _load_vars_with_try_catch(exe, model_path, loaded_var_list, - None) + clone_var_to_block(load_block, var) + ) + _load_vars_with_try_catch( + exe, model_path, loaded_var_list, None + ) else: for var_name in var_name_list: # NOTE(chenweihang): If identify which files the user wants @@ -2241,21 +2397,25 @@ def _load_vars_with_try_catch(exe, # If a file does not exist, we only warn the user that the # file may be an irrelevant file, but does not throw an error # to ensure that other legal variables can be loaded. - temp_var = load_block.create_var(name=var_name, - persistable=True) - if _load_vars_with_try_catch(exe, model_path, - [temp_var], None, False): + temp_var = load_block.create_var( + name=var_name, persistable=True + ) + if _load_vars_with_try_catch( + exe, model_path, [temp_var], None, False + ): loaded_var_list.append(temp_var) res_dict = {} for var in loaded_var_list: res_dict[var.name] = np.asarray( - paddle.fluid.global_scope().find_var(var.name).get_tensor()) + paddle.fluid.global_scope().find_var(var.name).get_tensor() + ) return res_dict - assert os.path.exists(parameter_file_name), \ - "Parameter file [{}] not exits".format(parameter_file_name) + assert os.path.exists( + parameter_file_name + ), "Parameter file [{}] not exits".format(parameter_file_name) with open(parameter_file_name, 'rb') as f: # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' @@ -2318,25 +2478,32 @@ def set_program_state(program, state_dict): used_para_list = {} for para in parameter_list: var_temp = paddle.fluid.global_scope().find_var(para.name) - assert var_temp != None, \ - "Variable [ {} ] Not found, Please make sure run startup program".format(para.name) + assert ( + var_temp != None + ), "Variable [ {} ] Not found, Please make sure run startup program".format( + para.name + ) if para.name in state_dict: # set value from state dict orig_para_np = np.array(var_temp.get_tensor()) new_para_np = state_dict[para.name] - assert orig_para_np.shape == new_para_np.shape, \ - "Parameter's shape does not match, the Program requires a parameter with the shape of ({}), " \ - "while the loaded parameter (namely [ {} ]) has a shape of ({})." \ - .format(orig_para_np.shape, para.name, new_para_np.shape) - assert orig_para_np.dtype == new_para_np.dtype, \ - "Parameter's data type does not match, the Program requires a parameter with a dtype of ({}), " \ - "while the loaded parameter (namely [ {} ]) has a dtype of ({})." \ - .format(orig_para_np.dtype, para.name, new_para_np.dtype) + assert orig_para_np.shape == new_para_np.shape, ( + "Parameter's shape does not match, the Program requires a parameter with the shape of ({}), " + "while the loaded parameter (namely [ {} ]) has a shape of ({}).".format( + orig_para_np.shape, para.name, new_para_np.shape + ) + ) + assert orig_para_np.dtype == new_para_np.dtype, ( + "Parameter's data type does not match, the Program requires a parameter with a dtype of ({}), " + "while the loaded parameter (namely [ {} ]) has a dtype of ({}).".format( + orig_para_np.dtype, para.name, new_para_np.dtype + ) + ) ten = var_temp.get_tensor() ten_place = ten._place() - #assert ten_place.is_gpu_place() or ten_place.is_cpu_place(), \ + # assert ten_place.is_gpu_place() or ten_place.is_cpu_place(), \ # "Place not support, only support CPUPlace and GPUPlace, now is {}".format(str(ten_place)) py_place = paddle.fluid.CPUPlace() if ten_place.is_cuda_pinned_place(): @@ -2368,5 +2535,7 @@ def set_program_state(program, state_dict): unused_para_list.append(k) if len(unused_para_list) > 0: warnings.warn( - "This list is not set, Because of Paramerter not found in program. There are: {}" - .format(" ".join(unused_para_list))) + "This list is not set, Because of Paramerter not found in program. There are: {}".format( + " ".join(unused_para_list) + ) + ) diff --git a/python/paddle/fluid/ir.py b/python/paddle/fluid/ir.py index 3e58af416d506..7dbe815b5e0e0 100644 --- a/python/paddle/fluid/ir.py +++ b/python/paddle/fluid/ir.py @@ -20,10 +20,12 @@ from .framework import _apply_pass, OpProtoHolder from .proto import framework_pb2 + try: from .proto import pass_desc_pb2 except ModuleNotFoundError: import sys + sys.path.append(path.join(path.dirname(__file__), 'proto')) from .proto import pass_desc_pb2 @@ -65,9 +67,9 @@ def _update_grad_persistable(main_program): g_var.persistable = True -def apply_build_strategy(main_program, startup_program, build_strategy, - pass_attrs): - +def apply_build_strategy( + main_program, startup_program, build_strategy, pass_attrs +): def update_attr(attrs, attr_types, name, value, typ=None): if name not in attrs: attrs[name] = value @@ -80,8 +82,13 @@ def apply_pass(name): update_attr(attrs, attr_types, "nranks", 1, "size_t") update_attr(attrs, attr_types, "use_cuda", False, "bool") # TODO(zjl): how to skip fetch variables ? - update_attr(attrs, attr_types, "mem_opt_skip_vars", - get_data_vars(main_program), "list[str]") + update_attr( + attrs, + attr_types, + "mem_opt_skip_vars", + get_data_vars(main_program), + "list[str]", + ) _apply_pass(main_program, startup_program, name, attrs, attr_types) _update_grad_persistable(main_program) @@ -109,12 +116,14 @@ def apply_pass(name): apply_pass("fuse_elewise_add_act_pass") build_strategy.fuse_elewise_add_act_ops = False if build_strategy.fuse_all_optimizer_ops: - apply_pass([ - "coalesce_grad_tensor_pass", - "fuse_adam_op_pass", - "fuse_sgd_op_pass", - "fuse_momentum_op_pass", - ]) + apply_pass( + [ + "coalesce_grad_tensor_pass", + "fuse_adam_op_pass", + "fuse_sgd_op_pass", + "fuse_momentum_op_pass", + ] + ) build_strategy.fuse_all_optimizer_ops = False # TODO(zjl): support fuse all reduce ops if build_strategy.cache_runtime_context: @@ -147,8 +156,10 @@ def _get_args_from_func(self, func): input_spec = self._input_specs.get(arg_name) if isinstance(input_spec, paddle.static.InputSpec): args.append( - PassDesc.VarHelper(arg_name, input_spec.shape, - input_spec.dtype)) + PassDesc.VarHelper( + arg_name, input_spec.shape, input_spec.dtype + ) + ) elif isinstance(input_spec, paddle.ParamAttr): args.append(paddle.ParamAttr(arg_name)) else: @@ -158,12 +169,15 @@ def _get_args_from_func(self, func): def _prune_program_desc(self, ops): for op_desc in ops: default_attrs = core.get_op_attrs_default_value( - op_desc.type.encode()) + op_desc.type.encode() + ) remove_attrs = list() for attr in op_desc.attrs: # attr must not in if attr.name not in [ - "op_namescope", "op_callstack", "op_device" + "op_namescope", + "op_callstack", + "op_device", ]: attr_list_fields = attr.ListFields() # attr format must be: name, type, value @@ -192,8 +206,10 @@ def _func_to_program_desc(self, func, ops): op_outs = out.Outputs() if len(op_outs) != 1: raise ValueError( - "Operator '{}' has multiple outputs, please specify one output variable." - .format(out._type)) + "Operator '{}' has multiple outputs, please specify one output variable.".format( + out._type + ) + ) for op_out in op_outs.values(): vars.extend(op_out) else: @@ -205,7 +221,6 @@ def _func_to_program_desc(self, func, ops): return vars, program.current_block().ops def _convert_vars_to_pass_desc(self, patterns, replaces, desc): - def _add_element_conditions(conditions, elements): for element in elements: if element._condition: @@ -257,11 +272,14 @@ def SerializeMultiPassDesc(self): pass_desc = multi_pass_desc.pass_descs.add() # Convert ProgramDescs of pattern and replace subgraphs. pattern_vars, pattern_ops = self._func_to_program_desc( - pattern, pass_desc.pattern) + pattern, pass_desc.pattern + ) replace_vars, replace_ops = self._func_to_program_desc( - replace, pass_desc.replace) - self._convert_vars_to_pass_desc(pattern_vars, replace_vars, - pass_desc) + replace, pass_desc.replace + ) + self._convert_vars_to_pass_desc( + pattern_vars, replace_vars, pass_desc + ) self._convert_ops_to_pass_desc(pattern_ops, replace_ops, pass_desc) if switch_static_mode: paddle.disable_static() @@ -269,9 +287,7 @@ def SerializeMultiPassDesc(self): class PassDesc(object): - class AttrHelper(object): - def __init__(self, obj, name, element_index=None): self._obj = obj self._name = name @@ -283,9 +299,9 @@ def __init__(self, obj, name, element_index=None): self._mapped = None def __getitem__(self, index): - element = PassDesc.AttrHelper(self._obj, - self._name, - element_index=index) + element = PassDesc.AttrHelper( + self._obj, self._name, element_index=index + ) self._elements.append(element) return element @@ -311,8 +327,9 @@ def _to_op_desc_attr(self, value, op_desc_attr): raise NotImplementedError("Unimplemented transform operation.") def _clone_with_operation(self, type, value=None): - attr = PassDesc.AttrHelper(self._obj, self._name, - self._element_index) + attr = PassDesc.AttrHelper( + self._obj, self._name, self._element_index + ) self._elements.append(attr) if value is None: attr._operation_type = type @@ -329,19 +346,23 @@ def _clone_with_operation(self, type, value=None): def __sub__(self, value): return self._clone_with_operation( - pass_desc_pb2.PassDesc.OperationType.kSub, value) + pass_desc_pb2.PassDesc.OperationType.kSub, value + ) def __add__(self, value): return self._clone_with_operation( - pass_desc_pb2.PassDesc.OperationType.kAdd, value) + pass_desc_pb2.PassDesc.OperationType.kAdd, value + ) def Mod(self, value): return self._clone_with_operation( - pass_desc_pb2.PassDesc.OperationType.kMod, value) + pass_desc_pb2.PassDesc.OperationType.kMod, value + ) def Size(self): return self._clone_with_operation( - pass_desc_pb2.PassDesc.OperationType.kSize) + pass_desc_pb2.PassDesc.OperationType.kSize + ) def _set_with_condition(self, type, value): condition = pass_desc_pb2.PassDesc.AttrCondition() @@ -356,36 +377,36 @@ def _set_with_condition(self, type, value): self._condition = condition def EQ(self, value): - self._set_with_condition(pass_desc_pb2.PassDesc.ConditionType.kEQ, - value) - - def MappedPattern(self, - var=None, - op=None, - index=0, - name=None, - element_index=None): + self._set_with_condition( + pass_desc_pb2.PassDesc.ConditionType.kEQ, value + ) + + def MappedPattern( + self, var=None, op=None, index=0, name=None, element_index=None + ): if all([var, op]): raise ValueError("Only mapped one of which var or op.") def mapped_var(pattern_ops): raise NotImplementedError( - "Mapping to variable is not implemented.") + "Mapping to variable is not implemented." + ) def mapped_op(pattern_ops): ops = [o for o in pattern_ops if o._type == op] if len(ops) <= index: raise ValueError( "Index '{}' of operator '{}' is incorrect.".format( - index, op)) - return PassDesc.AttrHelper(ops[index], - name, - element_index=element_index) + index, op + ) + ) + return PassDesc.AttrHelper( + ops[index], name, element_index=element_index + ) self._mapped = mapped_op if var is None else mapped_var class VarHelper(paddle.static.Variable): - def __init__(self, *args, **kwargs): block = paddle.static.default_main_program().current_block() self._var = paddle.static.data(*args, **kwargs) @@ -402,7 +423,6 @@ def Attr(self, name): return attr class OpHelper(object): - def __init__(self, type=None): self._type = type @@ -414,18 +434,23 @@ def __getattr__(self, name): def __call__(self, *args, **kwargs): if len(args) > 0: raise ValueError( - "Each input argument needs to specify a parameter name.") + "Each input argument needs to specify a parameter name." + ) for (in_name, in_args) in kwargs.items(): op_input = self._inputs.get(in_name) if op_input is None: raise ValueError( "Operator '{}' does not have input named '{}'.".format( - self._type, in_name)) + self._type, in_name + ) + ) if isinstance(in_args, (list, tuple)): if len(in_args) == 0: raise ValueError( - "Input '{}' of operator '{}' cannot be empty.". - format(in_name, self._type)) + "Input '{}' of operator '{}' cannot be empty.".format( + in_name, self._type + ) + ) else: in_args = [in_args] for in_arg in in_args: @@ -433,8 +458,10 @@ def __call__(self, *args, **kwargs): op_outs = in_arg.Outputs() if len(op_outs) != 1: raise ValueError( - "The size of outputs of operator '{}' is not equal 1, please specify one output variable." - .format(in_arg._type)) + "The size of outputs of operator '{}' is not equal 1, please specify one output variable.".format( + in_arg._type + ) + ) for op_out in op_outs.values(): op_input.extend(op_out) else: @@ -453,7 +480,9 @@ def Init(self): if self._proto is None: raise AttributeError( "type object 'OpHelper' has no attribute '{}'".format( - self._type)) + self._type + ) + ) self._index = len(block.ops) self._desc = block.desc.append_op() self._desc.set_type(self._type) @@ -480,7 +509,9 @@ def Output(self, name): if output is None: raise ValueError( "Operator '{}' does not have output named '{}'.".format( - self._type, name)) + self._type, name + ) + ) return output def Outputs(self): @@ -541,7 +572,8 @@ def decorated(python_func): signature = inspect.signature(python_func) if len(signature.parameters) > 0: raise NotImplementedError( - "Pass function with parameter is not supported now.") + "Pass function with parameter is not supported now." + ) elif len(signature.parameters) == 0: pass_pairs = python_func() if _is_pass_pair(pass_pairs): diff --git a/python/paddle/fluid/layer_helper.py b/python/paddle/fluid/layer_helper.py index fc63bbea4e5aa..595aa656f2946 100644 --- a/python/paddle/fluid/layer_helper.py +++ b/python/paddle/fluid/layer_helper.py @@ -14,7 +14,13 @@ import copy -from .framework import Parameter, dtype_is_floating, _non_static_mode, OpProtoHolder, _global_flags +from .framework import ( + Parameter, + dtype_is_floating, + _non_static_mode, + OpProtoHolder, + _global_flags, +) from . import unique_name from paddle.fluid.initializer import Constant, Xavier from .param_attr import ParamAttr @@ -25,7 +31,6 @@ class LayerHelper(LayerHelperBase): - def __init__(self, layer_type, **kwargs): self.kwargs = kwargs name = self.kwargs.get('name', None) @@ -35,8 +40,9 @@ def __init__(self, layer_type, **kwargs): if name is None: self.kwargs['name'] = unique_name.generate(layer_type) - super(LayerHelper, self).__init__(self.kwargs['name'], - layer_type=layer_type) + super(LayerHelper, self).__init__( + self.kwargs['name'], layer_type=layer_type + ) def append_op(self, *args, **kwargs): return self.main_program.current_block().append_op(*args, **kwargs) @@ -65,7 +71,7 @@ def param_attr(self): def bias_attr(self): return ParamAttr._to_attr(self.kwargs.get('bias_attr', None)) - #TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of param_attr + # TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of param_attr def multiple_param_attr(self, length): param_attr = self.param_attr if isinstance(param_attr, ParamAttr): @@ -93,8 +99,9 @@ def input_dtype(self, input_param_name='input'): if dtype is None: dtype = each.dtype elif dtype != each.dtype: - raise ValueError("Data Type mismatch: %d to %d" % - (dtype, each.dtype)) + raise ValueError( + "Data Type mismatch: %d to %d" % (dtype, each.dtype) + ) return dtype def get_parameter(self, name): @@ -103,7 +110,7 @@ def get_parameter(self, name): raise ValueError("no Parameter name %s found" % name) return param - #TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of bias_attr + # TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of bias_attr def append_bias_op(self, input_var, dim_start=1, dim_end=None): """ Append bias operator and return its output. If the user does not set @@ -123,21 +130,19 @@ def append_bias_op(self, input_var, dim_start=1, dim_end=None): if not bias_attr: return input_var - b = self.create_parameter(attr=bias_attr, - shape=size, - dtype=input_var.dtype, - is_bias=True) + b = self.create_parameter( + attr=bias_attr, shape=size, dtype=input_var.dtype, is_bias=True + ) tmp = self.create_variable_for_type_inference(dtype=input_var.dtype) - self.append_op(type='elementwise_add', - inputs={ - 'X': [input_var], - 'Y': [b] - }, - outputs={'Out': [tmp]}, - attrs={'axis': dim_start}) + self.append_op( + type='elementwise_add', + inputs={'X': [input_var], 'Y': [b]}, + outputs={'Out': [tmp]}, + attrs={'axis': dim_start}, + ) return tmp - #TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of act + # TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of act def append_activation(self, input_var): act = self.kwargs.get('act', None) if act is None: @@ -152,24 +157,27 @@ def append_activation(self, input_var): use_cudnn = self.kwargs.get('use_cudnn') act['use_cudnn'] = use_cudnn use_mkldnn = self.kwargs.get( - 'use_mkldnn', - _global_flags().get("FLAGS_use_mkldnn", False)) + 'use_mkldnn', _global_flags().get("FLAGS_use_mkldnn", False) + ) if use_mkldnn: act['use_mkldnn'] = use_mkldnn act_type = act.pop('type') if _non_static_mode(): - res = _append_activation_in_dygraph(input_var, act_type, use_cudnn, - use_mkldnn) + res = _append_activation_in_dygraph( + input_var, act_type, use_cudnn, use_mkldnn + ) return res else: tmp = self.create_variable_for_type_inference(dtype=input_var.dtype) - self.append_op(type=act_type, - inputs={"X": [input_var]}, - outputs={"Out": [tmp]}, - attrs=act) + self.append_op( + type=act_type, + inputs={"X": [input_var]}, + outputs={"Out": [tmp]}, + attrs=act, + ) return tmp - #TODO (jiabin): should we remove this since it has never be used + # TODO (jiabin): should we remove this since it has never be used def _get_default_initializer(self, dtype): if dtype is None or dtype_is_floating(dtype) is True: return Xavier() @@ -177,9 +185,13 @@ def _get_default_initializer(self, dtype): # For integer and boolean types, initialize with all zeros return Constant() - #TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of kwargs + # TODO (jiabin): reconstruct this in LayerObjHelper and avoid dependency of kwargs def is_instance(self, param_name, cls): param = self.kwargs.get(param_name, None) if not isinstance(param, cls): - raise TypeError("The input {0} parameter of method {1} must be {2}", - param_name, self.layer_type, cls.__name__) + raise TypeError( + "The input {0} parameter of method {1} must be {2}", + param_name, + self.layer_type, + cls.__name__, + ) diff --git a/python/paddle/fluid/layer_helper_base.py b/python/paddle/fluid/layer_helper_base.py index 3a688cf9f444f..1ea8d504add89 100644 --- a/python/paddle/fluid/layer_helper_base.py +++ b/python/paddle/fluid/layer_helper_base.py @@ -15,7 +15,14 @@ import copy import numpy as np -from .framework import Variable, default_main_program, default_startup_program, _non_static_mode, _current_expected_place, _in_eager_without_dygraph_check +from .framework import ( + Variable, + default_main_program, + default_startup_program, + _non_static_mode, + _current_expected_place, + _in_eager_without_dygraph_check, +) from . import unique_name from .param_attr import ParamAttr, WeightNormParamAttr from . import core @@ -81,123 +88,151 @@ def to_variable(self, value, name=None): """ if isinstance(value, np.ndarray): if _in_eager_without_dygraph_check(): - return core.eager.Tensor(value, _current_expected_place(), - False, False, name if name else None, - True) + return core.eager.Tensor( + value, + _current_expected_place(), + False, + False, + name if name else None, + True, + ) else: - py_var = core.VarBase(value=value, - name=name if name else '', - persistable=False, - place=_current_expected_place(), - zero_copy=False) + py_var = core.VarBase( + value=value, + name=name if name else '', + persistable=False, + place=_current_expected_place(), + zero_copy=False, + ) return py_var elif isinstance(value, (core.VarBase, Variable, core.eager.Tensor)): return value else: raise TypeError( "The type of input value is invalid, expected type is 'ndarray' or 'Variable', but received %s" - % type(value)) + % type(value) + ) def _create_weight_normalize(self, attr, shape, dtype): from .layers import elementwise_mul, elementwise_div, reshape # Remove these ops when LayerHelper and layers support indicating # program and block. - def __norm_op(x, - out=None, - p=2, - dim=None, - keep_dim=False, - block=self.startup_program.global_block()): + def __norm_op( + x, + out=None, + p=2, + dim=None, + keep_dim=False, + block=self.startup_program.global_block(), + ): if out is None: out = block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'weight_norm_norm'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'weight_norm_norm']) + ), dtype=dtype, - persistable=False) + persistable=False, + ) abs_out = block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'weight_norm_abs'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'weight_norm_abs']) + ), dtype=dtype, - persistable=False) - block.append_op(type='abs', - inputs={'X': x}, - outputs={'Out': abs_out}) + persistable=False, + ) + block.append_op( + type='abs', inputs={'X': x}, outputs={'Out': abs_out} + ) pow_out = block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'weight_norm_pow'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'weight_norm_pow']) + ), dtype=dtype, - persistable=False) - block.append_op(type='pow', - inputs={'X': abs_out}, - outputs={'Out': pow_out}, - attrs={'factor': float(p)}) + persistable=False, + ) + block.append_op( + type='pow', + inputs={'X': abs_out}, + outputs={'Out': pow_out}, + attrs={'factor': float(p)}, + ) sum_out = block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'weight_norm_sum'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'weight_norm_sum']) + ), dtype=dtype, - persistable=False) - block.append_op(type='reduce_sum', - inputs={'X': pow_out}, - outputs={'Out': sum_out}, - attrs={ - 'dim': dim, - 'keep_dim': keep_dim, - 'reduce_all': True if dim is None else False - }) - block.append_op(type='pow', - inputs={'X': sum_out}, - outputs={'Out': out}, - attrs={'factor': 1. / p}) + persistable=False, + ) + block.append_op( + type='reduce_sum', + inputs={'X': pow_out}, + outputs={'Out': sum_out}, + attrs={ + 'dim': dim, + 'keep_dim': keep_dim, + 'reduce_all': True if dim is None else False, + }, + ) + block.append_op( + type='pow', + inputs={'X': sum_out}, + outputs={'Out': out}, + attrs={'factor': 1.0 / p}, + ) return out - def __reshape_op(x, - shape, - out=None, - block=self.startup_program.global_block()): + def __reshape_op( + x, shape, out=None, block=self.startup_program.global_block() + ): if out is None: out = block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'weight_norm_reshape'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'weight_norm_reshape']) + ), dtype=dtype, - persistable=False) + persistable=False, + ) x_shape = block.create_var(name="Xshape", dtype=x.dtype) - block.append_op(type="reshape2", - inputs={'X': x}, - attrs={'shape': shape}, - outputs={ - "Out": out, - "XShape": x_shape - }) + block.append_op( + type="reshape2", + inputs={'X': x}, + attrs={'shape': shape}, + outputs={"Out": out, "XShape": x_shape}, + ) return out - def __transpose_op(x, - axis, - out=None, - block=self.startup_program.global_block()): + def __transpose_op( + x, axis, out=None, block=self.startup_program.global_block() + ): if out is None: out = block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'weight_norm_transpose'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'weight_norm_transpose']) + ), dtype=dtype, - persistable=False) - block.append_op(type='transpose', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'axis': axis}) + persistable=False, + ) + block.append_op( + type='transpose', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'axis': axis}, + ) return out - def __norm_except_dim(x, - out=None, - dim=None, - block=self.startup_program.global_block()): + def __norm_except_dim( + x, out=None, dim=None, block=self.startup_program.global_block() + ): """Computes the norm over all dimensions except dim""" if out is None: out = block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'weight_norm_norm'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'weight_norm_norm']) + ), dtype=dtype, - persistable=False) + persistable=False, + ) if dim is None: __norm_op(x, out, dim=dim, block=block) elif dim == 0: @@ -214,11 +249,12 @@ def __norm_except_dim(x, perm = list(range(len(x.shape))) perm[0], perm[dim] = dim, 0 transpose = __transpose_op(x, perm, block=block) - out_shape = [transpose.shape[0] - ] + [1] * (len(transpose.shape) - 1) - reshape = __reshape_op(transpose, - shape=[transpose.shape[0], -1], - block=block) + out_shape = [transpose.shape[0]] + [1] * ( + len(transpose.shape) - 1 + ) + reshape = __reshape_op( + transpose, shape=[transpose.shape[0], -1], block=block + ) norm = __norm_op(reshape, dim=[1], block=block) reshape2 = __reshape_op(norm, shape=out_shape, block=block) __transpose_op(reshape2, perm, out=out, block=block) @@ -226,18 +262,22 @@ def __norm_except_dim(x, def __weight_normalize(g, v, dim): """Calculations for weight normalization""" - norm = __norm_except_dim(v, - dim=dim, - block=self.main_program.current_block()) + norm = __norm_except_dim( + v, dim=dim, block=self.main_program.current_block() + ) scale = elementwise_div( - x=g, y=norm) # The shapes of g and norm are the same. + x=g, y=norm + ) # The shapes of g and norm are the same. # Currently, elementwise_mul only support broadcast when the shape # of y is a subset of the shape of x. Thus, we reshape y to squeeze # to achieve the subset. - w = elementwise_mul(x=v, - y=scale if dim is None else reshape( - x=scale, shape=[v.shape[dim]]), - axis=-1 if dim is None else dim) + w = elementwise_mul( + x=v, + y=scale + if dim is None + else reshape(x=scale, shape=[v.shape[dim]]), + axis=-1 if dim is None else dim, + ) # To serialize the original parameter for inference, maybe a # parameter rather than a variable should be returned. return w @@ -261,39 +301,49 @@ def __weight_normalize(g, v, dim): g_param = self.startup_program.global_block().create_parameter( dtype=dtype, shape=g_param_shape, - **g_param_attr._to_kwargs(with_initializer=False)) + **g_param_attr._to_kwargs(with_initializer=False) + ) v_param = self.startup_program.global_block().create_parameter( dtype=dtype, shape=v_param_shape, - **v_param_attr._to_kwargs(with_initializer=True)) - __norm_except_dim(x=v_param, - out=g_param, - dim=attr.dim, - block=self.startup_program.global_block()) + **v_param_attr._to_kwargs(with_initializer=True) + ) + __norm_except_dim( + x=v_param, + out=g_param, + dim=attr.dim, + block=self.startup_program.global_block(), + ) # keep g_param shape to be consistent with that in main_program - __reshape_op(g_param, - g_param_shape, - out=g_param, - block=self.startup_program.global_block()) + __reshape_op( + g_param, + g_param_shape, + out=g_param, + block=self.startup_program.global_block(), + ) # Add weight normalization to main_program g_param = self.main_program.global_block().create_parameter( - dtype=dtype, shape=g_param_shape, **g_param_attr._to_kwargs()) + dtype=dtype, shape=g_param_shape, **g_param_attr._to_kwargs() + ) v_param = self.main_program.global_block().create_parameter( - dtype=dtype, shape=v_param_shape, **v_param_attr._to_kwargs()) + dtype=dtype, shape=v_param_shape, **v_param_attr._to_kwargs() + ) w_param = __weight_normalize(g_param, v_param, dim=attr.dim) return w_param # TODO: hide the func after we move the layers to Layers - def create_parameter(self, - attr, - shape, - dtype=None, - is_bias=False, - default_initializer=None, - stop_gradient=False, - type=core.VarDesc.VarType.LOD_TENSOR): + def create_parameter( + self, + attr, + shape, + dtype=None, + is_bias=False, + default_initializer=None, + stop_gradient=False, + type=core.VarDesc.VarType.LOD_TENSOR, + ): """Create parameters for this layers. Args: @@ -312,36 +362,45 @@ def create_parameter(self, return None assert isinstance(attr, ParamAttr) for i, size in enumerate(shape): - assert size > 0, ("Expected every dim's size to be larger than 0, " - "but the size of the {}-th dim is {}".format( - i, size)) + assert size > 0, ( + "Expected every dim's size to be larger than 0, " + "but the size of the {}-th dim is {}".format(i, size) + ) # set global dtype if not dtype: dtype = self.__dtype if is_bias: suffix = 'b' - default_initializer = _global_bias_initializer( - ) if _global_bias_initializer() is not None else default_initializer + default_initializer = ( + _global_bias_initializer() + if _global_bias_initializer() is not None + else default_initializer + ) else: suffix = 'w' - default_initializer = _global_weight_initializer( - ) if _global_weight_initializer( - ) is not None else default_initializer + default_initializer = ( + _global_weight_initializer() + if _global_weight_initializer() is not None + else default_initializer + ) if attr.name is None: attr.name = unique_name.generate(".".join([self.name, suffix])) if default_initializer is None and attr.initializer is None: if isinstance(dtype, core.VarDesc.VarType): - if dtype != core.VarDesc.VarType.FP32 and \ - dtype != core.VarDesc.VarType.FP64 and \ - dtype != core.VarDesc.VarType.FP16 and \ - dtype != core.VarDesc.VarType.BF16: + if ( + dtype != core.VarDesc.VarType.FP32 + and dtype != core.VarDesc.VarType.FP64 + and dtype != core.VarDesc.VarType.FP16 + and dtype != core.VarDesc.VarType.BF16 + ): raise TypeError( "Can not create parameter with default initializer when dtype is not float type. Set default_initializer to fit the parameter dtype!" ) else: - if not (dtype.startswith("float") - or dtype in ["double", "uint16"]): + if not ( + dtype.startswith("float") or dtype in ["double", "uint16"] + ): raise TypeError( "Can not create parameter with default initializer when dtype is not float type. Set default_initializer to fit the parameter dtype!" ) @@ -368,26 +427,29 @@ def create_parameter(self, "parameter name [{}] have be been used. " "In dygraph mode, the name of parameter can't be same." "Please check the parameter attr value passed to self.create_parameter or " - "constructor of dygraph Layers".format(attr.name)) + "constructor of dygraph Layers".format(attr.name) + ) return self.main_program.global_block().create_parameter( dtype=dtype, shape=shape, type=type, stop_gradient=stop_gradient, - **attr._to_kwargs(with_initializer=True)) + **attr._to_kwargs(with_initializer=True) + ) else: self.startup_program.global_block().create_parameter( dtype=dtype, shape=shape, type=type, - **attr._to_kwargs(with_initializer=True)) + **attr._to_kwargs(with_initializer=True) + ) return self.main_program.global_block().create_parameter( - dtype=dtype, shape=shape, type=type, **attr._to_kwargs()) + dtype=dtype, shape=shape, type=type, **attr._to_kwargs() + ) - def create_variable_for_type_inference(self, - dtype, - stop_gradient=False, - shape=None): + def create_variable_for_type_inference( + self, dtype, stop_gradient=False, shape=None + ): """Create a temporary variable that should be type inferred layer. Note: @@ -400,18 +462,19 @@ def create_variable_for_type_inference(self, if not dtype: dtype = self.__dtype return self.main_program.current_block().create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'tmp']) + ), dtype=dtype, shape=shape, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=stop_gradient) + stop_gradient=stop_gradient, + ) - def create_sparse_variable_for_type_inference(self, - dtype, - stop_gradient=False, - shape=None): + def create_sparse_variable_for_type_inference( + self, dtype, stop_gradient=False, shape=None + ): """Create a temporary sparse variable that should be type inferred layer. Note: @@ -424,13 +487,15 @@ def create_sparse_variable_for_type_inference(self, if not dtype: dtype = self.__dtype return self.main_program.current_block().create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [self.name, 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join([self.name, 'tmp']) + ), dtype=dtype, shape=shape, type=core.VarDesc.VarType.SPARSE_COO, persistable=False, - stop_gradient=stop_gradient) + stop_gradient=stop_gradient, + ) def create_variable(self, *args, **kwargs): """Create Variable for this layers. @@ -449,7 +514,8 @@ def create_global_variable(self, persistable=False, *args, **kwargs): Returns(Variable): the created variable. """ return self.main_program.global_block().create_var( - *args, persistable=persistable, **kwargs) + *args, persistable=persistable, **kwargs + ) def create_or_get_global_variable(self, name, *args, **kwargs): """ @@ -464,9 +530,9 @@ def create_or_get_global_variable(self, name, *args, **kwargs): def set_variable_initializer(self, var, initializer): """Set target Variable's initializer - Args: - var: target Variable - initializer: initializer to use + Args: + var: target Variable + initializer: initializer to use """ assert isinstance(var, Variable) if _non_static_mode(): @@ -478,4 +544,5 @@ def set_variable_initializer(self, var, initializer): dtype=var.dtype, shape=var.shape, persistable=True, - initializer=initializer) + initializer=initializer, + ) diff --git a/python/paddle/fluid/layers/collective.py b/python/paddle/fluid/layers/collective.py index df3147a27cdcc..7764235dc2f3f 100644 --- a/python/paddle/fluid/layers/collective.py +++ b/python/paddle/fluid/layers/collective.py @@ -35,40 +35,38 @@ def _allreduce(x, out=None, reduce_type="sum", sync_mode=False): if out is None: out = helper.create_variable( - name=unique_name.generate_with_ignorable_key(".".join( - [x.name, 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join([x.name, 'tmp']) + ), shape=x.shape, dtype=x.dtype, type=x.type, persistable=x.persistable, - stop_gradient=True) - helper.append_op(type='allreduce', - inputs={'X': [x]}, - outputs={'Out': [out]}, - attrs={ - "reduce_type": red_typ_int, - "sync_mode": sync_mode - }) + stop_gradient=True, + ) + helper.append_op( + type='allreduce', + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={"reduce_type": red_typ_int, "sync_mode": sync_mode}, + ) return out def _broadcast(x, root, sync_mode=False): helper = LayerHelper("broadcast", **locals()) - helper.append_op(type='broadcast', - inputs={'X': [x]}, - outputs={'Out': [x]}, - attrs={ - "sync_mode": sync_mode, - "root": root - }) + helper.append_op( + type='broadcast', + inputs={'X': [x]}, + outputs={'Out': [x]}, + attrs={"sync_mode": sync_mode, "root": root}, + ) return x -def _c_allreduce(x, - out=None, - reduce_type='sum', - ring_id=0, - use_calc_stream=False): +def _c_allreduce( + x, out=None, reduce_type='sum', ring_id=0, use_calc_stream=False +): helper = LayerHelper('c_allreduce', **locals()) if reduce_type not in ['sum', 'prob', 'max', 'min']: @@ -77,34 +75,37 @@ def _c_allreduce(x, op_type = 'c_allreduce_' + reduce_type if out is None: out = helper.create_variable( - name=unique_name.generate_with_ignorable_key('.'.join( - [x.name, op_type])), + name=unique_name.generate_with_ignorable_key( + '.'.join([x.name, op_type]) + ), shape=x.shape, dtype=x.dtype, type=x.type, - persistable=x.persistable) - - helper.append_op(type=op_type, - inputs={'X': [x]}, - outputs={'Out': [out]}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream - }) + persistable=x.persistable, + ) + + helper.append_op( + type=op_type, + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={'ring_id': ring_id, 'use_calc_stream': use_calc_stream}, + ) return out def _c_broadcast(x, root=0, ring_id=0, use_calc_stream=False): op_type = 'c_broadcast' helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [x]}, - outputs={'Out': [x]}, - attrs={ - 'root': root, - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream - }) + helper.append_op( + type=op_type, + inputs={'X': [x]}, + outputs={'Out': [x]}, + attrs={ + 'root': root, + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + }, + ) return x @@ -121,28 +122,39 @@ def _c_allgather(x, nranks, ring_id=0, use_calc_stream=False): return out if _in_legacy_dygraph(): - attrs = ('nranks', nranks, 'ring_id', ring_id, 'use_calc_stream', - use_calc_stream) + attrs = ( + 'nranks', + nranks, + 'ring_id', + ring_id, + 'use_calc_stream', + use_calc_stream, + ) return _legacy_C_ops.c_allgather(x, *attrs) helper = LayerHelper(op_type, **locals()) out_shape = list(x.shape[:]) if out_shape[0] > 0: out_shape[0] *= nranks - out = helper.create_variable(name=unique_name.generate_with_ignorable_key( - '.'.join([x.name, op_type])), - shape=out_shape, - dtype=x.dtype, - type=x.type, - persistable=x.persistable) - helper.append_op(type=op_type, - inputs={'X': [x]}, - outputs={'Out': [out]}, - attrs={ - 'nranks': nranks, - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream - }) + out = helper.create_variable( + name=unique_name.generate_with_ignorable_key( + '.'.join([x.name, op_type]) + ), + shape=out_shape, + dtype=x.dtype, + type=x.type, + persistable=x.persistable, + ) + helper.append_op( + type=op_type, + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={ + 'nranks': nranks, + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + }, + ) return out @@ -152,28 +164,34 @@ def _c_reducescatter(x, nranks, ring_id=0, use_calc_stream=False): if x.shape[0] > 0 and x.shape[0] % nranks != 0: raise ValueError( - 'x.shape[0](%d) cannot be evenly divided by nranks(%d)' % - (x.shape[0], nranks)) + 'x.shape[0](%d) cannot be evenly divided by nranks(%d)' + % (x.shape[0], nranks) + ) op_type = 'c_reducescatter' helper = LayerHelper(op_type, **locals()) out_shape = list(x.shape[:]) if out_shape[0] > 0: out_shape[0] //= nranks - out = helper.create_variable(name=unique_name.generate_with_ignorable_key( - '.'.join([x.name, op_type])), - shape=out_shape, - dtype=x.dtype, - type=x.type, - persistable=x.persistable) - helper.append_op(type=op_type, - inputs={'X': [x]}, - outputs={'Out': [out]}, - attrs={ - 'nranks': nranks, - 'ring_id': ring_id, - 'use_calc_stream': use_calc_stream - }) + out = helper.create_variable( + name=unique_name.generate_with_ignorable_key( + '.'.join([x.name, op_type]) + ), + shape=out_shape, + dtype=x.dtype, + type=x.type, + persistable=x.persistable, + ) + helper.append_op( + type=op_type, + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={ + 'nranks': nranks, + 'ring_id': ring_id, + 'use_calc_stream': use_calc_stream, + }, + ) return out @@ -187,8 +205,10 @@ def _c_sync_calc_stream(x): def _c_sync_comm_stream(x, ring_id): op_type = 'c_sync_comm_stream' helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [x]}, - outputs={'Out': [x]}, - attrs={'ring_id': ring_id}) + helper.append_op( + type=op_type, + inputs={'X': [x]}, + outputs={'Out': [x]}, + attrs={'ring_id': ring_id}, + ) return x diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index b800e67fe70f6..eb654458fbf29 100755 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -17,24 +17,66 @@ from .layer_function_generator import autodoc, templatedoc from .tensor import assign, cast, fill_constant from .. import core -from ..framework import Program, Variable, Operator, _non_static_mode, static_only, _in_legacy_dygraph, in_dygraph_mode +from ..framework import ( + Program, + Variable, + Operator, + _non_static_mode, + static_only, + _in_legacy_dygraph, + in_dygraph_mode, +) from ..layer_helper import LayerHelper, unique_name from .nn import logical_and, logical_not, logical_or -from .utils import assert_same_structure, map_structure, hold_mutable_vars, copy_mutable_vars, padding_to_same_structure, is_sequence, pack_sequence_as, flatten, to_sequence +from .utils import ( + assert_same_structure, + map_structure, + hold_mutable_vars, + copy_mutable_vars, + padding_to_same_structure, + is_sequence, + pack_sequence_as, + flatten, + to_sequence, +) import numpy import warnings from functools import reduce, partial -from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) from ... import compat as cpt from ..backward import _infer_var_data_type_shape_ from paddle import _C_ops, _legacy_C_ops __all__ = [ - 'While', 'Switch', 'increment', 'array_write', 'create_array', 'less_than', - 'less_equal', 'greater_than', 'greater_equal', 'equal', 'not_equal', - 'array_read', 'array_length', 'cond', 'IfElse', 'DynamicRNN', 'StaticRNN', - 'reorder_lod_tensor_by_rank', 'Print', 'Assert', 'is_empty', 'case', - 'switch_case', 'while_loop' + 'While', + 'Switch', + 'increment', + 'array_write', + 'create_array', + 'less_than', + 'less_equal', + 'greater_than', + 'greater_equal', + 'equal', + 'not_equal', + 'array_read', + 'array_length', + 'cond', + 'IfElse', + 'DynamicRNN', + 'StaticRNN', + 'reorder_lod_tensor_by_rank', + 'Print', + 'Assert', + 'is_empty', + 'case', + 'switch_case', + 'while_loop', ] @@ -59,12 +101,11 @@ def select_output(input, outputs, mask): check_variable_and_dtype(mask, 'mask', ['int32'], 'select_output') check_type(outputs, 'outputs', (list, tuple), 'select_output') - helper.append_op(type='select_output', - inputs={ - 'X': input, - 'Mask': mask - }, - outputs={'Out': outputs}) + helper.append_op( + type='select_output', + inputs={'X': input, 'Mask': mask}, + outputs={'Out': outputs}, + ) return outputs @@ -83,7 +124,8 @@ def _select_input_infer_shape(first_shape, second_shape): ) return second_shape out_shape = list( - map(lambda a, b: a if a == b else -1, first_shape, second_shape)) + map(lambda a, b: a if a == b else -1, first_shape, second_shape) + ) return out_shape @@ -107,32 +149,34 @@ def select_input(inputs, mask): check_variable_and_dtype(mask, 'mask', ['int32'], 'select_input') # Select input should expand the shape. If it is - 1 and valid number, use - 1 first. If the dim is different, an error will be reported directly - #assert inputs[0].dtype == inputs[1].dtype, f"Expect the inputs should have the same dtype, but get {inputs[0].dtype} and {inputs[1].dtype}" + # assert inputs[0].dtype == inputs[1].dtype, f"Expect the inputs should have the same dtype, but get {inputs[0].dtype} and {inputs[1].dtype}" output_shape = _select_input_infer_shape(inputs[0].shape, inputs[1].shape) output_dtype = inputs[1].dtype output_type = inputs[1].type - out = helper.create_variable(dtype=output_dtype, - shape=output_shape, - type=output_type) - helper.append_op(type='select_input', - inputs={ - 'X': inputs, - 'Mask': mask - }, - outputs={'Out': out}) + out = helper.create_variable( + dtype=output_dtype, shape=output_shape, type=output_type + ) + helper.append_op( + type='select_input', + inputs={'X': inputs, 'Mask': mask}, + outputs={'Out': out}, + ) return out def select_input_with_buildin_type(inputs, mask, name): - from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import to_static_variable + from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + to_static_variable, + ) from paddle.fluid.dygraph.dygraph_to_static.utils import UndefinedVar + false_var, true_var = inputs if isinstance(false_var, UndefinedVar) and isinstance( - true_var, UndefinedVar): - """ None -> UndefinedVar, so the real value is a [None, UndefinedVar] or [None, None], we just return None. - """ + true_var, UndefinedVar + ): + """None -> UndefinedVar, so the real value is a [None, UndefinedVar] or [None, None], we just return None.""" return None if isinstance(false_var, Variable) and isinstance(true_var, Variable): @@ -140,50 +184,63 @@ def select_input_with_buildin_type(inputs, mask, name): return select_input(inputs, mask) except Exception as e: raise RuntimeError( - f"Exceptions throwed while doing select_input on {name}:\n{e}") + f"Exceptions throwed while doing select_input on {name}:\n{e}" + ) - elif (isinstance(false_var, support_ret_buildin_type) - and isinstance(false_var, type(true_var))): + elif isinstance(false_var, support_ret_buildin_type) and isinstance( + false_var, type(true_var) + ): if false_var == true_var: return false_var else: inputs = [ to_static_variable(false_var), - to_static_variable(true_var) + to_static_variable(true_var), ] # Deal with the situations like this: false_var is int and true_var is Variable - elif ((isinstance(false_var, support_ret_buildin_type) - and isinstance(true_var, Variable)) - or (isinstance(true_var, support_ret_buildin_type) - and isinstance(false_var, Variable))): + elif ( + isinstance(false_var, support_ret_buildin_type) + and isinstance(true_var, Variable) + ) or ( + isinstance(true_var, support_ret_buildin_type) + and isinstance(false_var, Variable) + ): inputs = [to_static_variable(false_var), to_static_variable(true_var)] warnings.warn( "Return results from different branches in cond are not same type: " "false_var returned by fasle_fn is '{}' and true_var of true_fn is " - "'{}'".format(type(false_var), type(true_var))) - elif ((isinstance(false_var, UndefinedVar) - and isinstance(true_var, (Variable, ) + support_ret_buildin_type)) - or (isinstance(true_var, UndefinedVar) - and isinstance(false_var, - (Variable, ) + support_ret_buildin_type))): + "'{}'".format(type(false_var), type(true_var)) + ) + elif ( + isinstance(false_var, UndefinedVar) + and isinstance(true_var, (Variable,) + support_ret_buildin_type) + ) or ( + isinstance(true_var, UndefinedVar) + and isinstance(false_var, (Variable,) + support_ret_buildin_type) + ): def create_var_if_not_undefined_var(a): - if isinstance(a, UndefinedVar): return a + if isinstance(a, UndefinedVar): + return a return to_static_variable(a) true_var, false_var = to_static_variable(true_var), to_static_variable( - false_var) + false_var + ) inputs = [false_var, true_var] else: raise TypeError( "Unsupported return type of true_fn and false_fn in cond: false_var " - "returned by fasle_fn is '{}' and true_var of true_fn is '{}'". - format(type(false_var), type(true_var))) + "returned by fasle_fn is '{}' and true_var of true_fn is '{}'".format( + type(false_var), type(true_var) + ) + ) try: return select_input(inputs, mask) except Exception as e: raise RuntimeError( - f"Exceptions throwed while doing select_input on {name}:\n{e}") + f"Exceptions throwed while doing select_input on {name}:\n{e}" + ) def split_lod_tensor(input, mask, level=0): @@ -220,23 +277,26 @@ def split_lod_tensor(input, mask, level=0): input=x, mask=y, level=level) """ - check_type(input, 'input', (Variable, list, tuple, type(None)), - 'fluid.layers.split_lod_tensor') + check_type( + input, + 'input', + (Variable, list, tuple, type(None)), + 'fluid.layers.split_lod_tensor', + ) check_type(mask, 'mask', (Variable, list), 'fluid.layers.split_lod_tensor') check_type(level, 'level', int, 'fluid.layers.split_lod_tensor') helper = LayerHelper('split_lod_tensor', **locals()) out_true = helper.create_variable_for_type_inference(dtype=input.dtype) out_false = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='split_lod_tensor', - inputs={ - 'X': input, - 'Mask': mask, - }, - outputs={ - 'OutTrue': out_true, - 'OutFalse': out_false - }, - attrs={'level': level}) + helper.append_op( + type='split_lod_tensor', + inputs={ + 'X': input, + 'Mask': mask, + }, + outputs={'OutTrue': out_true, 'OutFalse': out_false}, + attrs={'level': level}, + ) return out_true, out_false @@ -278,37 +338,48 @@ def merge_lod_tensor(in_true, in_false, x, mask, level=0): in_true=out_true, in_false=out_false, mask=y, x=x, level=level) """ helper = LayerHelper('merge_lod_tensor', **locals()) - check_type(x, 'x', (Variable, list, tuple, type(None)), - 'fluid.layers.merge_lod_tensor') + check_type( + x, + 'x', + (Variable, list, tuple, type(None)), + 'fluid.layers.merge_lod_tensor', + ) check_type(mask, 'mask', (Variable, list), 'fluid.layers.merge_lod_tensor') - check_type(in_true, 'in_true', (Variable, list, tuple, type(None)), - 'fluid.layers.merge_lod_tensor') - check_type(in_false, 'in_false', (Variable, list, tuple, type(None)), - 'fluid.layers.merge_lod_tensor') + check_type( + in_true, + 'in_true', + (Variable, list, tuple, type(None)), + 'fluid.layers.merge_lod_tensor', + ) + check_type( + in_false, + 'in_false', + (Variable, list, tuple, type(None)), + 'fluid.layers.merge_lod_tensor', + ) out = helper.create_variable_for_type_inference(dtype=in_true.dtype) - helper.append_op(type='merge_lod_tensor', - inputs={ - 'X': x, - 'Mask': mask, - 'InTrue': in_true, - 'InFalse': in_false - }, - outputs={'Out': out}, - attrs={'level': level}) + helper.append_op( + type='merge_lod_tensor', + inputs={'X': x, 'Mask': mask, 'InTrue': in_true, 'InFalse': in_false}, + outputs={'Out': out}, + attrs={'level': level}, + ) return out @static_only -def Print(input, - first_n=-1, - message=None, - summarize=20, - print_tensor_name=True, - print_tensor_type=True, - print_tensor_shape=True, - print_tensor_layout=True, - print_tensor_lod=True, - print_phase='both'): +def Print( + input, + first_n=-1, + message=None, + summarize=20, + print_tensor_name=True, + print_tensor_type=True, + print_tensor_shape=True, + print_tensor_layout=True, + print_tensor_lod=True, + print_phase='both', +): ''' :api_attr: Static Graph @@ -366,26 +437,31 @@ def Print(input, # - dtype: long # - data: [3 3 3 3 3 3] ''' - check_variable_and_dtype(input, 'input', - ['float32', 'float64', 'int32', 'int64', 'bool'], - 'fluid.layers.Print') + check_variable_and_dtype( + input, + 'input', + ['float32', 'float64', 'int32', 'int64', 'bool'], + 'fluid.layers.Print', + ) helper = LayerHelper('print' + "_" + input.name, **locals()) output = helper.create_variable_for_type_inference(input.dtype) - helper.append_op(type='print', - inputs={'In': input}, - outputs={'Out': output}, - attrs={ - 'first_n': first_n, - 'summarize': summarize, - 'message': message or "", - 'print_tensor_name': print_tensor_name, - 'print_tensor_type': print_tensor_type, - 'print_tensor_shape': print_tensor_shape, - 'print_tensor_layout': print_tensor_layout, - 'print_tensor_lod': print_tensor_lod, - 'print_phase': print_phase.upper() - }) + helper.append_op( + type='print', + inputs={'In': input}, + outputs={'Out': output}, + attrs={ + 'first_n': first_n, + 'summarize': summarize, + 'message': message or "", + 'print_tensor_name': print_tensor_name, + 'print_tensor_type': print_tensor_type, + 'print_tensor_shape': print_tensor_shape, + 'print_tensor_layout': print_tensor_layout, + 'print_tensor_lod': print_tensor_lod, + 'print_phase': print_phase.upper(), + }, + ) return output @@ -452,12 +528,11 @@ def Assert(cond, data=None, summarize=20, name=None): layer_name = name if name else ('assert_' + cond.name) helper = LayerHelper(layer_name, **locals()) - op = helper.append_op(type="assert", - inputs={ - "Cond": cond, - "Data": [] if data is None else list(data) - }, - attrs={"summarize": summarize}) + op = helper.append_op( + type="assert", + inputs={"Cond": cond, "Data": [] if data is None else list(data)}, + attrs={"summarize": summarize}, + ) return op @@ -507,8 +582,9 @@ def __exit__(self, exc_type, exc_val, exc_tb): return False self.rnn.status = StaticRNN.AFTER_RNN_BLOCK self.rnn._complete_op() - return super(BlockGuardWithCompletion, - self).__exit__(exc_type, exc_val, exc_tb) + return super(BlockGuardWithCompletion, self).__exit__( + exc_type, exc_val, exc_tb + ) class StaticRNNMemoryLink(object): @@ -580,6 +656,7 @@ class StaticRNN(object): result = rnn() """ + BEFORE_RNN_BLOCK = 0 IN_RNN_BLOCK = 1 AFTER_RNN_BLOCK = 2 @@ -605,13 +682,15 @@ def _assert_in_rnn_block_(self, method): if self.status != StaticRNN.IN_RNN_BLOCK: raise ValueError("You must invoke {0} in rnn block".format(method)) - def memory(self, - init=None, - shape=None, - batch_ref=None, - init_value=0.0, - init_batch_dim_idx=0, - ref_batch_dim_idx=1): + def memory( + self, + init=None, + shape=None, + batch_ref=None, + init_value=0.0, + init_batch_dim_idx=0, + ref_batch_dim_idx=1, + ): """ Create a memory variable for static rnn. If the :code:`init` is not None, :code:`memory` will be initialized by @@ -637,97 +716,118 @@ def memory(self, Examples 1: .. code-block:: python - import paddle.fluid as fluid - import paddle.fluid.layers as layers - - vocab_size, hidden_size=10000, 200 - x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') - # create word sequence - x_emb = layers.embedding( - input=x, - size=[vocab_size, hidden_size], - dtype='float32', - is_sparse=False) - # transform batch size to dim 1 - x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) - - rnn = fluid.layers.StaticRNN() - with rnn.step(): - # mark created x_emb as input, each step process a word - word = rnn.step_input(x_emb) - # create prev memory parameter, batch size comes from word - prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) - hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') - # use hidden to update prev - rnn.update_memory(prev, hidden) + import paddle.fluid as fluid + import paddle.fluid.layers as layers + + vocab_size, hidden_size=10000, 200 + x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') + # create word sequence + x_emb = layers.embedding( + input=x, + size=[vocab_size, hidden_size], + dtype='float32', + is_sparse=False) + # transform batch size to dim 1 + x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) + + rnn = fluid.layers.StaticRNN() + with rnn.step(): + # mark created x_emb as input, each step process a word + word = rnn.step_input(x_emb) + # create prev memory parameter, batch size comes from word + prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) + hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') + # use hidden to update prev + rnn.update_memory(prev, hidden) Examples 2: .. code-block:: python - import paddle.fluid as fluid - import paddle.fluid.layers as layers - vocab_size, hidden_size=10000, 200 - x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') - # create word sequence - x_emb = layers.embedding( - input=x, - size=[vocab_size, hidden_size], - dtype='float32', - is_sparse=False) - # transform batch size to dim 1 - x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) - boot_memory = fluid.layers.data(name='boot', shape=[hidden_size], dtype='float32', lod_level=1) - rnn = fluid.layers.StaticRNN() - with rnn.step(): - # mark created x_emb as input, each step process a word - word = rnn.step_input(x_emb) - # init memory - prev = rnn.memory(init=boot_memory) - hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') - # update hidden with prev - rnn.update_memory(prev, hidden) + import paddle.fluid as fluid + import paddle.fluid.layers as layers + vocab_size, hidden_size=10000, 200 + x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') + # create word sequence + x_emb = layers.embedding( + input=x, + size=[vocab_size, hidden_size], + dtype='float32', + is_sparse=False) + # transform batch size to dim 1 + x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) + boot_memory = fluid.layers.data(name='boot', shape=[hidden_size], dtype='float32', lod_level=1) + rnn = fluid.layers.StaticRNN() + with rnn.step(): + # mark created x_emb as input, each step process a word + word = rnn.step_input(x_emb) + # init memory + prev = rnn.memory(init=boot_memory) + hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') + # update hidden with prev + rnn.update_memory(prev, hidden) """ self._assert_in_rnn_block_('memory') - check_type(init, "init", (Variable, type(None)), - "fluid.layers.StaticRNN.memory") - check_type(shape, "shape", (list, tuple, type(None)), - "fluid.layers.StaticRNN.memory") - check_type(batch_ref, "batch_ref", (Variable, type(None)), - "fluid.layers.StaticRNN.memory") + check_type( + init, + "init", + (Variable, type(None)), + "fluid.layers.StaticRNN.memory", + ) + check_type( + shape, + "shape", + (list, tuple, type(None)), + "fluid.layers.StaticRNN.memory", + ) + check_type( + batch_ref, + "batch_ref", + (Variable, type(None)), + "fluid.layers.StaticRNN.memory", + ) if init is None: if shape is None or batch_ref is None: raise ValueError( - "if init is None, memory at least need shape and batch_ref") + "if init is None, memory at least need shape and batch_ref" + ) parent_block = self._parent_block() - var_name = unique_name.generate_with_ignorable_key("@".join( - [self.helper.name, "memory_boot"])) - boot_var = parent_block.create_var(name=var_name, - shape=shape, - dtype=batch_ref.dtype, - persistable=False) - - parent_block.append_op(type="fill_constant_batch_size_like", - inputs={'Input': [batch_ref]}, - outputs={'Out': [boot_var]}, - attrs={ - 'value': init_value, - 'shape': boot_var.shape, - 'dtype': boot_var.dtype, - 'input_dim_idx': ref_batch_dim_idx, - 'output_dim_idx': init_batch_dim_idx - }) + var_name = unique_name.generate_with_ignorable_key( + "@".join([self.helper.name, "memory_boot"]) + ) + boot_var = parent_block.create_var( + name=var_name, + shape=shape, + dtype=batch_ref.dtype, + persistable=False, + ) + + parent_block.append_op( + type="fill_constant_batch_size_like", + inputs={'Input': [batch_ref]}, + outputs={'Out': [boot_var]}, + attrs={ + 'value': init_value, + 'shape': boot_var.shape, + 'dtype': boot_var.dtype, + 'input_dim_idx': ref_batch_dim_idx, + 'output_dim_idx': init_batch_dim_idx, + }, + ) return self.memory(init=boot_var) else: pre_mem = self.helper.create_variable( - name=unique_name.generate_with_ignorable_key("@".join( - [self.helper.name, "mem"])), + name=unique_name.generate_with_ignorable_key( + "@".join([self.helper.name, "mem"]) + ), dtype=init.dtype, - shape=init.shape) - self.memories[pre_mem.name] = StaticRNNMemoryLink(init=init, - pre_mem=pre_mem) + shape=init.shape, + ) + self.memories[pre_mem.name] = StaticRNNMemoryLink( + init=init, pre_mem=pre_mem + ) return pre_mem def step_input(self, x): @@ -744,29 +844,29 @@ def step_input(self, x): Examples: .. code-block:: python - import paddle.fluid as fluid - import paddle.fluid.layers as layers - - vocab_size, hidden_size=10000, 200 - x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') - # create word sequence - x_emb = layers.embedding( - input=x, - size=[vocab_size, hidden_size], - dtype='float32', - is_sparse=False) - # transform batch size to dim 1 - x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) - - rnn = fluid.layers.StaticRNN() - with rnn.step(): - # mark created x_emb as input, each step process a word - word = rnn.step_input(x_emb) - # create prev memory parameter, batch size comes from word - prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) - hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') - # use hidden to update prev - rnn.update_memory(prev, hidden) + import paddle.fluid as fluid + import paddle.fluid.layers as layers + + vocab_size, hidden_size=10000, 200 + x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') + # create word sequence + x_emb = layers.embedding( + input=x, + size=[vocab_size, hidden_size], + dtype='float32', + is_sparse=False) + # transform batch size to dim 1 + x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) + + rnn = fluid.layers.StaticRNN() + with rnn.step(): + # mark created x_emb as input, each step process a word + word = rnn.step_input(x_emb) + # create prev memory parameter, batch size comes from word + prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) + hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') + # use hidden to update prev + rnn.update_memory(prev, hidden) """ self._assert_in_rnn_block_('step_input') @@ -776,10 +876,9 @@ def step_input(self, x): elif x.shape[0] != -1 and self.seq_len != x.shape[0]: raise ValueError("Static RNN only take fix seq_len input") - ipt = self.helper.create_variable(name=x.name, - dtype=x.dtype, - shape=list(x.shape[1:]), - type=x.type) + ipt = self.helper.create_variable( + name=x.name, dtype=x.dtype, shape=list(x.shape[1:]), type=x.type + ) self.inputs.append(ipt) return ipt @@ -796,47 +895,50 @@ def step_output(self, o): Examples: .. code-block:: python - import paddle.fluid as fluid - import paddle.fluid.layers as layers - - vocab_size, hidden_size=10000, 200 - x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') - # create word sequence - x_emb = layers.embedding( - input=x, - size=[vocab_size, hidden_size], - dtype='float32', - is_sparse=False) - # transform batch size to dim 1 - x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) - - rnn = fluid.layers.StaticRNN() - with rnn.step(): - # mark created x_emb as input, each step process a word - word = rnn.step_input(x_emb) - # create prev memory parameter, batch size comes from word - prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) - hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') - # use hidden to update prev - rnn.update_memory(prev, hidden) - rnn.step_output(hidden) - - result = rnn() + import paddle.fluid as fluid + import paddle.fluid.layers as layers + + vocab_size, hidden_size=10000, 200 + x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') + # create word sequence + x_emb = layers.embedding( + input=x, + size=[vocab_size, hidden_size], + dtype='float32', + is_sparse=False) + # transform batch size to dim 1 + x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) + + rnn = fluid.layers.StaticRNN() + with rnn.step(): + # mark created x_emb as input, each step process a word + word = rnn.step_input(x_emb) + # create prev memory parameter, batch size comes from word + prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) + hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') + # use hidden to update prev + rnn.update_memory(prev, hidden) + rnn.step_output(hidden) + + result = rnn() """ self._assert_in_rnn_block_('step_output') check_type(o, "o", Variable, "fluid.layers.StaticRNN.step_output") tmp_o = self.helper.create_variable_for_type_inference(dtype=o.dtype) - self.helper.append_op(type='rnn_memory_helper', - inputs={'X': [o]}, - outputs={'Out': tmp_o}, - attrs={'dtype': o.dtype}) + self.helper.append_op( + type='rnn_memory_helper', + inputs={'X': [o]}, + outputs={'Out': tmp_o}, + attrs={'dtype': o.dtype}, + ) - out_var = self._parent_block().create_var(name=tmp_o.name, - shape=[self.seq_len] + - list(tmp_o.shape), - dtype=tmp_o.dtype) + out_var = self._parent_block().create_var( + name=tmp_o.name, + shape=[self.seq_len] + list(tmp_o.shape), + dtype=tmp_o.dtype, + ) self.outputs.append(out_var) @@ -853,33 +955,33 @@ def output(self, *outputs): Examples: .. code-block:: python - import paddle.fluid as fluid - import paddle.fluid.layers as layers - - vocab_size, hidden_size=10000, 200 - x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') - # create word sequence - x_emb = layers.embedding( - input=x, - size=[vocab_size, hidden_size], - dtype='float32', - is_sparse=False) - # transform batch size to dim 1 - x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) - - rnn = fluid.layers.StaticRNN() - with rnn.step(): - # mark created x_emb as input, each step process a word - word = rnn.step_input(x_emb) - # create prev memory parameter, batch size comes from word - prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) - hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') - # use hidden to update prev - rnn.update_memory(prev, hidden) - # mark each step's hidden and word as output - rnn.output(hidden, word) - - result = rnn() + import paddle.fluid as fluid + import paddle.fluid.layers as layers + + vocab_size, hidden_size=10000, 200 + x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64') + # create word sequence + x_emb = layers.embedding( + input=x, + size=[vocab_size, hidden_size], + dtype='float32', + is_sparse=False) + # transform batch size to dim 1 + x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) + + rnn = fluid.layers.StaticRNN() + with rnn.step(): + # mark created x_emb as input, each step process a word + word = rnn.step_input(x_emb) + # create prev memory parameter, batch size comes from word + prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) + hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') + # use hidden to update prev + rnn.update_memory(prev, hidden) + # mark each step's hidden and word as output + rnn.output(hidden, word) + + result = rnn() """ for each in outputs: self.step_output(each) @@ -952,7 +1054,8 @@ def _complete_op(self): ] step_scope = parent_block.create_var( - type=core.VarDesc.VarType.STEP_SCOPES) + type=core.VarDesc.VarType.STEP_SCOPES + ) inlinks = [parent_block.var(i.name) for i in self.inputs] outlinks = self.outputs @@ -964,39 +1067,41 @@ def _complete_op(self): for _, mem in self.memories.items(): boot_memories.append(mem.init) pre_memories.append(mem.pre_mem.name) - assert mem.mem is not None, "%s should be updated in every step." % ( - mem.init.name) + assert ( + mem.mem is not None + ), "%s should be updated in every step." % (mem.init.name) mem_var = rnn_block.var(mem.mem.name) assert isinstance(mem_var, Variable) new_mem = self.helper.create_variable_for_type_inference( - dtype=mem_var.dtype) - rnn_block.append_op(type='rnn_memory_helper', - inputs={'X': [mem_var]}, - outputs={'Out': [new_mem]}, - attrs={'dtype': mem_var.dtype}) + dtype=mem_var.dtype + ) + rnn_block.append_op( + type='rnn_memory_helper', + inputs={'X': [mem_var]}, + outputs={'Out': [new_mem]}, + attrs={'dtype': mem_var.dtype}, + ) memories.append(new_mem.name) - parent_block.append_op(type='recurrent', - inputs={ - 'inputs': inlinks, - 'initial_states': boot_memories, - 'parameters': parameters - }, - outputs={ - 'outputs': outlinks, - 'step_scopes': [step_scope] - }, - attrs={ - 'has_states': len(pre_memories) > 0, - 'ex_states': pre_memories, - 'states': memories, - 'sub_block': rnn_block - }) + parent_block.append_op( + type='recurrent', + inputs={ + 'inputs': inlinks, + 'initial_states': boot_memories, + 'parameters': parameters, + }, + outputs={'outputs': outlinks, 'step_scopes': [step_scope]}, + attrs={ + 'has_states': len(pre_memories) > 0, + 'ex_states': pre_memories, + 'states': memories, + 'sub_block': rnn_block, + }, + ) class WhileGuard(BlockGuard): - def __init__(self, while_op): if not isinstance(while_op, While): raise TypeError("WhileGuard takes a while op") @@ -1015,8 +1120,9 @@ def __exit__(self, exc_type, exc_val, exc_tb): return super(WhileGuard, self).__exit__(exc_type, exc_val, exc_tb) -def get_inputs_outputs_in_block(current_block, inner_inputs, inner_outputs, - helper): +def get_inputs_outputs_in_block( + current_block, inner_inputs, inner_outputs, helper +): """ Find inputs and outputs in current control flow block. :param current_block: Current control flow block. @@ -1047,7 +1153,8 @@ def is_ignore_vars(op, var_name): for iname in op.input_names: for in_var_name in op.input(iname): if in_var_name not in inner_outputs and not is_ignore_vars( - op, in_var_name): + op, in_var_name + ): inner_inputs.add(in_var_name) for oname in op.output_names: @@ -1063,8 +1170,11 @@ def is_ignore_vars(op, var_name): current_block_var = None if current_block.has_var(in_var_name): current_block_var = current_block.var(in_var_name) - if not parent_block_var and current_block_var and \ - current_block_var.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + not parent_block_var + and current_block_var + and current_block_var.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): remove_inner_inputs.add(in_var_name) inner_inputs = inner_inputs - remove_inner_inputs @@ -1154,8 +1264,10 @@ def __init__(self, cond, is_test=False, name=None): check_variable_and_dtype(cond, 'cond', ['bool'], 'fluid.layers.While') if reduce(lambda a, b: a * b, cond.shape, 1) != 1: raise TypeError( - "condition expected shape as [1], but given shape as {0}.". - format(list(cond.shape))) + "condition expected shape as [1], but given shape as {0}.".format( + list(cond.shape) + ) + ) self.cond_var = cond self.is_test = is_test @@ -1166,12 +1278,14 @@ def _complete(self): main_program = self.helper.main_program while_block = main_program.current_block() parent_block = main_program.block( - main_program.current_block().parent_idx) + main_program.current_block().parent_idx + ) inner_outputs = {self.cond_var.name} x_name_list = set() x_name_list, inner_outputs = get_inputs_outputs_in_block( - while_block, x_name_list, inner_outputs, self.helper) + while_block, x_name_list, inner_outputs, self.helper + ) out_vars = [] for inner_out_name in inner_outputs: @@ -1185,23 +1299,21 @@ def _complete(self): x_name_list -= {self.cond_var.name} step_scope = parent_block.create_var( - type=core.VarDesc.VarType.STEP_SCOPES) + type=core.VarDesc.VarType.STEP_SCOPES + ) parent_block.append_op( type='while', inputs={ - 'X': - [parent_block._var_recursive(x_name) for x_name in x_name_list], - 'Condition': [self.cond_var] - }, - outputs={ - 'Out': out_vars, - 'StepScopes': [step_scope] + 'X': [ + parent_block._var_recursive(x_name) + for x_name in x_name_list + ], + 'Condition': [self.cond_var], }, - attrs={ - 'sub_block': while_block, - "is_test": self.is_test - }) + outputs={'Out': out_vars, 'StepScopes': [step_scope]}, + attrs={'sub_block': while_block, "is_test": self.is_test}, + ) support_ret_buildin_type = (bool, float, int) @@ -1213,14 +1325,17 @@ def assign_skip_lod_tensor_array(input, output): """ def has_shape_diff(x_var, y_var): - if len(x_var.shape) != len(y_var.shape): return True + if len(x_var.shape) != len(y_var.shape): + return True for x_dim, y_dim in zip(x_var.shape, y_var.shape): - if x_dim != y_dim and -1 not in [x_dim, y_dim]: return True + if x_dim != y_dim and -1 not in [x_dim, y_dim]: + return True return False if not isinstance(input, (Variable, core.VarBase)): if isinstance(output, Variable) and isinstance( - input, support_ret_buildin_type): + input, support_ret_buildin_type + ): assign(input, output) else: output = input @@ -1229,15 +1344,21 @@ def has_shape_diff(x_var, y_var): if input.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY: main_program = input.block.program parent_block = main_program.block( - main_program.current_block().parent_idx) + main_program.current_block().parent_idx + ) if parent_block and not parent_block._find_var_recursive(input.name): assign(input, output) else: - if isinstance(output, Variable) and isinstance( - input, Variable) and has_shape_diff(input, output): + if ( + isinstance(output, Variable) + and isinstance(input, Variable) + and has_shape_diff(input, output) + ): warnings.warn( - "In dy2static mode, we attemp to assign a variable with shape {} into a variable with shape{}, which is not always right." - .format(input.shape, output.shape)) + "In dy2static mode, we attemp to assign a variable with shape {} into a variable with shape{}, which is not always right.".format( + input.shape, output.shape + ) + ) assign(input, output) @@ -1253,7 +1374,7 @@ def while_loop(cond, body, loop_vars, is_test=False, name=None): Args: cond(Callable): A callable returning a boolean tensor controlling whether to continue looping. And ``cond`` takes - as many arguments as ``loop_vars`` . + as many arguments as ``loop_vars`` . body(Callable): A callable returning a tuple or list of tensors or LoDTensorArrays of the same arity (length and structure) and types as ``loops_vars`` . And ``body`` takes as many arguments as ``loop_vars`` . loop_vars(list|tuple): A list or tuple of tensors or LoDTensorArrays that is passed to both ``cond`` and ``body`` . @@ -1299,23 +1420,26 @@ def body(i, ten): raise ValueError("loop_vars in while_loop should not be empty") pre_cond = cond(*loop_vars) - check_variable_and_dtype(pre_cond, 'var of cond returned', ['bool'], - 'fluid.layers.while_loop') + check_variable_and_dtype( + pre_cond, 'var of cond returned', ['bool'], 'fluid.layers.while_loop' + ) if reduce(lambda a, b: a * b, pre_cond.shape, 1) != 1: raise TypeError( "the shape of the variable returned by cond should be [1]," - "but given shape as {0}.".format(list(pre_cond.shape))) + "but given shape as {0}.".format(list(pre_cond.shape)) + ) if _non_static_mode(): now_cond = pre_cond.numpy()[0] - while (now_cond): + while now_cond: output_vars = body(*loop_vars) if not isinstance(output_vars, (list, tuple)): output_vars = [output_vars] if len(output_vars) != len(loop_vars): raise ValueError( "body in while_loop should return the same arity " - "(length and structure) and types as loop_vars") + "(length and structure) and types as loop_vars" + ) now_cond = cond(*output_vars).numpy()[0] map_structure(assign_skip_lod_tensor_array, output_vars, loop_vars) return loop_vars @@ -1340,7 +1464,8 @@ def body(i, ten): except ValueError as e: raise ValueError( "body in while_loop should return the same arity " - "(length and structure) as loop_vars: {0}".format(e)) + "(length and structure) as loop_vars: {0}".format(e) + ) now_cond = cond(*output_vars) map_structure(assign_skip_lod_tensor_array, output_vars, loop_vars) assign(now_cond, pre_cond) @@ -1348,19 +1473,24 @@ def body(i, ten): def _deal_with_undefined_var(output_vars, loop_vars): - """ Deal with undefined var cases, We create undefined variable based on the results of body(). - In Dy2Static, we use undefined var to represent the var created in control flow. This function - expand the loop_vars and replace original loop_vars. - 1. UndefinedVar = Variable # create a variable - 2. UndefinedVar = None # create a undefined var with RETURN_NO_VALUE_MAGIC_NUM - 3. UndefinedVar = List(int) # create a list of variable - 4. UndefinedVar = value # create a variable + """Deal with undefined var cases, We create undefined variable based on the results of body(). + In Dy2Static, we use undefined var to represent the var created in control flow. This function + expand the loop_vars and replace original loop_vars. + 1. UndefinedVar = Variable # create a variable + 2. UndefinedVar = None # create a undefined var with RETURN_NO_VALUE_MAGIC_NUM + 3. UndefinedVar = List(int) # create a list of variable + 4. UndefinedVar = value # create a variable """ - from paddle.fluid.dygraph.dygraph_to_static.utils import UndefinedVar, create_undefined_variable + from paddle.fluid.dygraph.dygraph_to_static.utils import ( + UndefinedVar, + create_undefined_variable, + ) def create_var_like(o_var): - if isinstance(o_var, - (Variable, ) + support_ret_buildin_type) or o_var is None: + if ( + isinstance(o_var, (Variable,) + support_ret_buildin_type) + or o_var is None + ): return create_undefined_variable() if is_sequence(o_var): """ @@ -1431,16 +1561,21 @@ def lod_rank_table(x, level=0): check_type(x, 'x', (Variable, list), 'lod_rank_table') if isinstance(x, (list)): for i, input_x in enumerate(x): - check_type(input_x, 'input[' + str(i) + ']', Variable, - 'lod_rank_table') + check_type( + input_x, 'input[' + str(i) + ']', Variable, 'lod_rank_table' + ) helper = LayerHelper("lod_rank_table", **locals()) - table = helper.create_variable(type=core.VarDesc.VarType.LOD_RANK_TABLE, - name=unique_name.generate("lod_rank_table")) - helper.append_op(type='lod_rank_table', - inputs={'X': x}, - outputs={'Out': table}, - attrs={'level': level}) + table = helper.create_variable( + type=core.VarDesc.VarType.LOD_RANK_TABLE, + name=unique_name.generate("lod_rank_table"), + ) + helper.append_op( + type='lod_rank_table', + inputs={'X': x}, + outputs={'Out': table}, + attrs={'level': level}, + ) return table @@ -1463,9 +1598,11 @@ def max_sequence_len(rank_table): """ helper = LayerHelper("max_seqence_len", **locals()) res = helper.create_variable_for_type_inference(dtype="int64") - helper.append_op(type="max_sequence_len", - inputs={"RankTable": rank_table}, - outputs={"Out": res}) + helper.append_op( + type="max_sequence_len", + inputs={"RankTable": rank_table}, + outputs={"Out": res}, + ) return res @@ -1501,24 +1638,32 @@ def lod_tensor_to_array(x, table): check_type(x, 'x', (Variable, list), 'lod_tensor_to_array') if isinstance(x, (list)): for i, input_x in enumerate(x): - check_type(input_x, 'input[' + str(i) + ']', Variable, - 'lod_tensor_to_array') + check_type( + input_x, + 'input[' + str(i) + ']', + Variable, + 'lod_tensor_to_array', + ) check_type(table, 'table', (Variable, list), 'lod_tensor_to_array') if isinstance(table, (list)): for i, table_x in enumerate(table): - check_type(table_x, 'table[' + str(i) + ']', Variable, - 'lod_tensor_to_array') + check_type( + table_x, + 'table[' + str(i) + ']', + Variable, + 'lod_tensor_to_array', + ) helper = LayerHelper("lod_tensor_to_array", **locals()) array = helper.create_variable( name=unique_name.generate("lod_tensor_to_array"), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=x.dtype) - helper.append_op(type='lod_tensor_to_array', - inputs={ - 'X': x, - 'RankTable': table - }, - outputs={'Out': array}) + dtype=x.dtype, + ) + helper.append_op( + type='lod_tensor_to_array', + inputs={'X': x, 'RankTable': table}, + outputs={'Out': array}, + ) return array @@ -1547,22 +1692,29 @@ def array_to_lod_tensor(x, table): check_type(x, 'x', (Variable, list), 'array_to_lod_tensor') if isinstance(x, (list)): for i, input_x in enumerate(x): - check_type(input_x, 'input[' + str(i) + ']', Variable, - 'array_to_lod_tensor') + check_type( + input_x, + 'input[' + str(i) + ']', + Variable, + 'array_to_lod_tensor', + ) check_type(table, 'table', (Variable, list), 'array_to_lod_tensor') if isinstance(table, (list)): for i, table_x in enumerate(table): - check_type(table_x, 'table[' + str(i) + ']', Variable, - 'array_to_lod_tensor') + check_type( + table_x, + 'table[' + str(i) + ']', + Variable, + 'array_to_lod_tensor', + ) helper = LayerHelper("array_to_lod_tensor", **locals()) tmp = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="array_to_lod_tensor", - inputs={ - 'X': x, - 'RankTable': table - }, - outputs={'Out': tmp}) + helper.append_op( + type="array_to_lod_tensor", + inputs={'X': x, 'RankTable': table}, + outputs={'Out': tmp}, + ) return tmp @@ -1590,17 +1742,20 @@ def increment(x, value=1.0, in_place=True): if in_dygraph_mode(): return _C_ops.increment_(x, value) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'increment') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'increment' + ) helper = LayerHelper("increment", **locals()) if not in_place: out = helper.create_variable_for_type_inference(dtype=x.dtype) else: out = x - helper.append_op(type='increment', - inputs={'X': [x]}, - outputs={'Out': [out]}, - attrs={'step': float(value)}) + helper.append_op( + type='increment', + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={'step': float(value)}, + ) return out @@ -1668,8 +1823,8 @@ def array_write(x, i, array=None): if array is None: array = create_array(x.dtype) assert isinstance( - array, - list), "The 'array' in array_write must be a list in dygraph mode" + array, list + ), "The 'array' in array_write must be a list in dygraph mode" assert i <= len( array ), "The index 'i' should not be greater than the length of 'array' in dygraph mode" @@ -1683,22 +1838,24 @@ def array_write(x, i, array=None): check_type(x, 'x', (Variable), 'array_write') helper = LayerHelper('array_write', **locals()) if array is not None: - if not isinstance( - array, Variable - ) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + not isinstance(array, Variable) + or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): raise TypeError( - "array should be tensor array vairable in array_write Op") + "array should be tensor array vairable in array_write Op" + ) if array is None: array = helper.create_variable( name="{0}.out".format(helper.name), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=x.dtype) - helper.append_op(type='write_to_array', - inputs={ - 'X': [x], - 'I': [i] - }, - outputs={'Out': [array]}) + dtype=x.dtype, + ) + helper.append_op( + type='write_to_array', + inputs={'X': [x], 'I': [i]}, + outputs={'Out': [array]}, + ) return array @@ -1729,16 +1886,20 @@ def create_array(dtype, initialized_list=None): if initialized_list is not None: if not isinstance(initialized_list, (list, tuple)): raise TypeError( - "Require type(initialized_list) should be list/tuple, but received {}" - .format(type(initialized_list))) + "Require type(initialized_list) should be list/tuple, but received {}".format( + type(initialized_list) + ) + ) array = list(initialized_list) # NOTE: Only support plain list like [x, y,...], not support nested list in static mode. for val in array: if not isinstance(val, Variable): raise TypeError( - "All values in `initialized_list` should be Variable, but recevied {}." - .format(type(val))) + "All values in `initialized_list` should be Variable, but recevied {}.".format( + type(val) + ) + ) if _non_static_mode(): return array @@ -1747,7 +1908,8 @@ def create_array(dtype, initialized_list=None): tensor_array = helper.create_variable( name="{0}.out".format(helper.name), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=dtype) + dtype=dtype, + ) for val in array: array_write(x=val, i=array_length(tensor_array), array=tensor_array) @@ -1784,10 +1946,12 @@ def less_than(x, y, force_cpu=None, cond=None, name=None): print(result) # [True, False, False, False] """ - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "less_than") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "less_than") + check_variable_and_dtype( + x, "x", ["float32", "float64", "int32", "int64"], "less_than" + ) + check_variable_and_dtype( + y, "y", ["float32", "float64", "int32", "int64"], "less_than" + ) if cond is not None: check_type(cond, "cond", Variable, "less_than") if force_cpu != None: @@ -1802,13 +1966,12 @@ def less_than(x, y, force_cpu=None, cond=None, name=None): if force_cpu is not None: attrs['force_cpu'] = force_cpu - helper.append_op(type='less_than', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [cond]}, - attrs=attrs) + helper.append_op( + type='less_than', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [cond]}, + attrs=attrs, + ) return cond @@ -1816,8 +1979,8 @@ def less_than(x, y, force_cpu=None, cond=None, name=None): def less_equal(x, y, cond=None, name=None): """ :alias_main: paddle.less_equal - :alias: paddle.less_equal,paddle.tensor.less_equal,paddle.tensor.logic.less_equal - :old_api: paddle.fluid.layers.less_equal + :alias: paddle.less_equal,paddle.tensor.less_equal,paddle.tensor.logic.less_equal + :old_api: paddle.fluid.layers.less_equal This OP returns the truth value of :math:`x <= y` elementwise, which is equivalent function to the overloaded operator `<=`. @@ -1843,10 +2006,12 @@ def less_equal(x, y, cond=None, name=None): out1 = label<= limit #out1=[True, False] """ - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "less_equal") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "less_equal") + check_variable_and_dtype( + x, "x", ["float32", "float64", "int32", "int64"], "less_equal" + ) + check_variable_and_dtype( + y, "y", ["float32", "float64", "int32", "int64"], "less_equal" + ) if cond is not None: check_type(cond, "cond", Variable, "less_equal") @@ -1857,13 +2022,12 @@ def less_equal(x, y, cond=None, name=None): attrs = dict() - helper.append_op(type='less_equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [cond]}, - attrs=attrs) + helper.append_op( + type='less_equal', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [cond]}, + attrs=attrs, + ) return cond @@ -1871,8 +2035,8 @@ def less_equal(x, y, cond=None, name=None): def greater_than(x, y, cond=None, name=None): """ :alias_main: paddle.greater_than - :alias: paddle.greater_than,paddle.tensor.greater_than,paddle.tensor.logic.greater_than - :old_api: paddle.fluid.layers.greater_than + :alias: paddle.greater_than,paddle.tensor.greater_than,paddle.tensor.logic.greater_than + :old_api: paddle.fluid.layers.greater_than This OP returns the truth value of :math:`x > y` elementwise, which is equivalent function to the overloaded operator `>`. @@ -1897,10 +2061,12 @@ def greater_than(x, y, cond=None, name=None): out = fluid.layers.greater_than(x=label, y=limit) #out=[False, True] out1 = label > limit #out1=[False, True] """ - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "greater_than") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "greater_than") + check_variable_and_dtype( + x, "x", ["float32", "float64", "int32", "int64"], "greater_than" + ) + check_variable_and_dtype( + y, "y", ["float32", "float64", "int32", "int64"], "greater_than" + ) if cond is not None: check_type(cond, "cond", Variable, "greater_than") @@ -1914,13 +2080,12 @@ def greater_than(x, y, cond=None, name=None): if in_dygraph_mode(): return _C_ops.greater_than(x, y, -1) else: - helper.append_op(type='greater_than', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [cond]}, - attrs=attrs) + helper.append_op( + type='greater_than', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [cond]}, + attrs=attrs, + ) return cond @@ -1928,8 +2093,8 @@ def greater_than(x, y, cond=None, name=None): def greater_equal(x, y, cond=None, name=None): """ :alias_main: paddle.greater_equal - :alias: paddle.greater_equal,paddle.tensor.greater_equal,paddle.tensor.logic.greater_equal - :old_api: paddle.fluid.layers.greater_equal + :alias: paddle.greater_equal,paddle.tensor.greater_equal,paddle.tensor.logic.greater_equal + :old_api: paddle.fluid.layers.greater_equal This OP returns the truth value of :math:`x >= y` elementwise, which is equivalent function to the overloaded operator `>=`. @@ -1956,10 +2121,12 @@ def greater_equal(x, y, cond=None, name=None): out_1 = label >= limit #out1=[True, False] """ - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "greater_equal") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "greater_equal") + check_variable_and_dtype( + x, "x", ["float32", "float64", "int32", "int64"], "greater_equal" + ) + check_variable_and_dtype( + y, "y", ["float32", "float64", "int32", "int64"], "greater_equal" + ) if cond is not None: check_type(cond, "cond", Variable, "greater_equal") @@ -1970,13 +2137,12 @@ def greater_equal(x, y, cond=None, name=None): attrs = dict() - helper.append_op(type='greater_equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [cond]}, - attrs=attrs) + helper.append_op( + type='greater_equal', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [cond]}, + attrs=attrs, + ) return cond @@ -2013,10 +2179,12 @@ def equal(x, y, cond=None, name=None): default_axis = -1 return _C_ops.equal(x, y, default_axis) - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "equal") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "equal") + check_variable_and_dtype( + x, "x", ["float32", "float64", "int32", "int64"], "equal" + ) + check_variable_and_dtype( + y, "y", ["float32", "float64", "int32", "int64"], "equal" + ) if cond is not None: check_type(cond, "cond", Variable, "equal") @@ -2025,20 +2193,17 @@ def equal(x, y, cond=None, name=None): cond = helper.create_variable_for_type_inference(dtype='bool') cond.stop_gradient = True - helper.append_op(type='equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [cond]}) + helper.append_op( + type='equal', inputs={'X': [x], 'Y': [y]}, outputs={'Out': [cond]} + ) return cond def not_equal(x, y, cond=None, name=None): """ :alias_main: paddle.not_equal - :alias: paddle.not_equal,paddle.tensor.not_equal,paddle.tensor.logic.not_equal - :old_api: paddle.fluid.layers.not_equal + :alias: paddle.not_equal,paddle.tensor.not_equal,paddle.tensor.logic.not_equal + :old_api: paddle.fluid.layers.not_equal This OP returns the truth value of :math:`x != y` elementwise, which is equivalent function to the overloaded operator `!=`. @@ -2062,10 +2227,12 @@ def not_equal(x, y, cond=None, name=None): limit = fluid.layers.fill_constant(shape=[1], value=1, dtype='int64') out = fluid.layers.not_equal(x=label, y=limit) """ - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "not_equal") - check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"], - "not_equal") + check_variable_and_dtype( + x, "x", ["float32", "float64", "int32", "int64"], "not_equal" + ) + check_variable_and_dtype( + y, "y", ["float32", "float64", "int32", "int64"], "not_equal" + ) if cond is not None: check_type(cond, "cond", Variable, "not_equal") @@ -2074,12 +2241,9 @@ def not_equal(x, y, cond=None, name=None): cond = helper.create_variable_for_type_inference(dtype='bool') cond.stop_gradient = True - helper.append_op(type='not_equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [cond]}) + helper.append_op( + type='not_equal', inputs={'X': [x], 'Y': [y]}, outputs={'Out': [cond]} + ) return cond @@ -2146,8 +2310,8 @@ def array_read(array, i): """ if _non_static_mode(): assert isinstance( - array, - list), "The 'array' in array_read must be list in dygraph mode" + array, list + ), "The 'array' in array_read must be list in dygraph mode" assert isinstance( i, Variable ), "The index 'i' in array_read must be Variable in dygraph mode" @@ -2159,17 +2323,17 @@ def array_read(array, i): check_variable_and_dtype(i, 'i', ['int64'], 'array_read') helper = LayerHelper('array_read', **locals()) - if not isinstance( - array, - Variable) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + not isinstance(array, Variable) + or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): raise TypeError("array should be tensor array vairable") out = helper.create_variable_for_type_inference(dtype=array.dtype) - helper.append_op(type='read_from_array', - inputs={ - 'X': [array], - 'I': [i] - }, - outputs={'Out': [out]}) + helper.append_op( + type='read_from_array', + inputs={'X': [array], 'I': [i]}, + outputs={'Out': [out]}, + ) return out @@ -2203,14 +2367,12 @@ def shrink_memory(x, i, table): check_type(i, 'i', Variable, 'shrink_memory') check_type(table, 'table', Variable, 'shrink_memory') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='shrink_rnn_memory', - inputs={ - 'X': [x], - 'I': [i], - 'RankTable': [table] - }, - outputs={'Out': [out]}, - attrs={}) + helper.append_op( + type='shrink_rnn_memory', + inputs={'X': [x], 'I': [i], 'RankTable': [table]}, + outputs={'Out': [out]}, + attrs={}, + ) return out @@ -2262,22 +2424,24 @@ def array_length(array): if _non_static_mode(): assert isinstance( - array, - list), "The 'array' in array_write must be a list in dygraph mode" + array, list + ), "The 'array' in array_write must be a list in dygraph mode" return len(array) - if not isinstance( - array, - Variable) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + not isinstance(array, Variable) + or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): raise TypeError( - "array should be tensor array vairable in array_length Op") + "array should be tensor array vairable in array_length Op" + ) helper = LayerHelper('array_length', **locals()) tmp = helper.create_variable_for_type_inference(dtype='int64') tmp.stop_gradient = True - helper.append_op(type='lod_array_length', - inputs={'X': [array]}, - outputs={'Out': [tmp]}) + helper.append_op( + type='lod_array_length', inputs={'X': [array]}, outputs={'Out': [tmp]} + ) return tmp @@ -2299,8 +2463,9 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.block.complete() - return super(ConditionalBlockGuard, - self).__exit__(exc_type, exc_val, exc_tb) + return super(ConditionalBlockGuard, self).__exit__( + exc_type, exc_val, exc_tb + ) class ConditionalBlock(object): @@ -2346,10 +2511,9 @@ def complete(self): intermediate = set() params = set() - params, intermediate = get_inputs_outputs_in_block(inside_block, - params, - intermediate, - helper=self.helper) + params, intermediate = get_inputs_outputs_in_block( + inside_block, params, intermediate, helper=self.helper + ) # Todo(liym27) Here assume that all params are in recursive parent block # but when minimize() called in control flow, some params may be in @@ -2365,25 +2529,25 @@ def complete(self): out_list.append(inner_var) step_scope = parent_block.create_var( - type=core.VarDesc.VarType.STEP_SCOPES) + type=core.VarDesc.VarType.STEP_SCOPES + ) conditional_block_op = parent_block.append_op( type='conditional_block', inputs={ 'Cond': self.inputs, 'Input': param_list, }, - outputs={ - 'Out': out_list, - 'Scope': [step_scope] - }, + outputs={'Out': out_list, 'Scope': [step_scope]}, attrs={ 'sub_block': inside_block, - 'is_scalar_condition': self.is_scalar_condition - }) + 'is_scalar_condition': self.is_scalar_condition, + }, + ) if self.need_append_conditional_block_grad(inside_block): - self.append_conditional_block_grad(parent_block, inside_block, - conditional_block_op) + self.append_conditional_block_grad( + parent_block, inside_block, conditional_block_op + ) def need_append_conditional_block_grad(self, inside_block): grad_sub_block_idx = inside_block.backward_block_idx @@ -2391,10 +2555,13 @@ def need_append_conditional_block_grad(self, inside_block): # if inside_block have grad_block and grad_block is not itself, # we will append conditional block grad. - return grad_sub_block_idx != -1 and grad_sub_block_idx != inside_block_idx + return ( + grad_sub_block_idx != -1 and grad_sub_block_idx != inside_block_idx + ) - def append_conditional_block_grad(self, parent_block, inside_block, - conditional_block_op): + def append_conditional_block_grad( + self, parent_block, inside_block, conditional_block_op + ): ''' Append op `conditional_block_grad` manually. When `optimizer.minimize/append_backward` is called in Paddle control flow, @@ -2433,7 +2600,8 @@ def append_conditional_block_grad(self, parent_block, inside_block, param_list.append(inner_var.name) grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - conditional_block_op.desc, set(), [grad_sub_block.desc]) + conditional_block_op.desc, set(), [grad_sub_block.desc] + ) # append op_desc in grad_op_descs to target_block op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName() @@ -2443,13 +2611,16 @@ def append_conditional_block_grad(self, parent_block, inside_block, new_op_desc._set_attr(op_role_attr_name, backward) # set input and output manually new_op_desc.set_input('Input', param_list) - new_op_desc.set_output('Input@GRAD', - [param + "@GRAD" for param in param_list]) + new_op_desc.set_output( + 'Input@GRAD', [param + "@GRAD" for param in param_list] + ) new_vars = set() for grad_var_name in new_op_desc.output_arg_names(): - if grad_sub_block.desc.has_var_recursive(grad_var_name.encode( - )) or grad_var_name == core.empty_var_name(): + if ( + grad_sub_block.desc.has_var_recursive(grad_var_name.encode()) + or grad_var_name == core.empty_var_name() + ): continue grad_sub_block.desc.var(grad_var_name.encode()) new_vars.add(grad_var_name) @@ -2472,16 +2643,20 @@ def copy_var_to_parent_block(var, layer_helper): return var prog = layer_helper.main_program parent_idx = prog.current_block().parent_idx - assert parent_idx >= 0, "Got wrong parent block index when assigning var to parent scope in control_flow" + assert ( + parent_idx >= 0 + ), "Got wrong parent block index when assigning var to parent scope in control_flow" parent_block = prog.block(parent_idx) - if var.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY \ - and parent_block._find_var_recursive(var.name): + if ( + var.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + and parent_block._find_var_recursive(var.name) + ): parent_block_var = var else: - parent_block_var = parent_block.create_var(dtype=var.dtype, - shape=var.shape, - type=var.type) + parent_block_var = parent_block.create_var( + dtype=var.dtype, shape=var.shape, type=var.type + ) assign(var, parent_block_var) return parent_block_var @@ -2594,15 +2769,19 @@ def false_func(): if true_fn is not None: if not callable(true_fn): raise TypeError( - "The true_fn in cond must be callable, but received {}". - format(type(true_fn).__name__)) + "The true_fn in cond must be callable, but received {}".format( + type(true_fn).__name__ + ) + ) return true_fn() else: if false_fn is not None: if not callable(false_fn): raise TypeError( - "The false_fn in cond must be callable, but received {}" - .format(type(false_fn).__name__)) + "The false_fn in cond must be callable, but received {}".format( + type(false_fn).__name__ + ) + ) return false_fn() return None @@ -2616,25 +2795,32 @@ def false_func(): if not callable(true_fn): raise TypeError( "The true_fn in cond must be callable, but received {}".format( - type(true_fn).__name__)) + type(true_fn).__name__ + ) + ) true_cond_block = ConditionalBlock([pred], is_scalar_condition=True) with true_cond_block.block(): origin_true_output = true_fn() if origin_true_output is not None: - true_output = map_structure(copy_to_parent_func, - origin_true_output) + true_output = map_structure( + copy_to_parent_func, origin_true_output + ) if false_fn is not None: if not callable(false_fn): raise TypeError( "The false_fn in cond must be callable, but received {}".format( - type(false_fn).__name__)) - false_cond_block = ConditionalBlock([logical_not(pred)], - is_scalar_condition=True) + type(false_fn).__name__ + ) + ) + false_cond_block = ConditionalBlock( + [logical_not(pred)], is_scalar_condition=True + ) with false_cond_block.block(): origin_false_output = false_fn() if origin_false_output is not None: - false_output = map_structure(copy_to_parent_func, - origin_false_output) + false_output = map_structure( + copy_to_parent_func, origin_false_output + ) if true_output is None and false_output is None: return None @@ -2642,11 +2828,13 @@ def false_func(): if true_output is None: raise ValueError( "Incompatible return values of true_fn and false_fn in cond: " - "true_fn returns None while false_fn returns non-None") + "true_fn returns None while false_fn returns non-None" + ) if false_output is None: raise ValueError( "Incompatible return values of true_fn and false_fn in cond: " - "true_fn returns non-None while false_fn returns None") + "true_fn returns non-None while false_fn returns None" + ) # Merge ture and false output if they are not None if return_names is None: @@ -2664,22 +2852,28 @@ def false_func(): # a = 1 # Because we can not use variable to express 'None' true_output, false_output = expand_undefined_var( - true_output, false_output, return_names) + true_output, false_output, return_names + ) if len(to_sequence(true_output)) != len(to_sequence(false_output)): raise ValueError( - "true fn returns {} vars, but false fn returns {} vars, which is not equals" - .format(len(to_sequence(true_output)), - len(to_sequence(false_output)))) - for true_out, false_out, return_name in zip(to_sequence(true_output), - to_sequence(false_output), - to_sequence(return_names)): + "true fn returns {} vars, but false fn returns {} vars, which is not equals".format( + len(to_sequence(true_output)), len(to_sequence(false_output)) + ) + ) + for true_out, false_out, return_name in zip( + to_sequence(true_output), + to_sequence(false_output), + to_sequence(return_names), + ): try: assert_same_structure(true_out, false_out, check_types=False) except ValueError as e: raise ValueError( - "Incompatible return values of `{}` in true_fn and false_fn in cond: {}" - .format(return_name, e)) + "Incompatible return values of `{}` in true_fn and false_fn in cond: {}".format( + return_name, e + ) + ) def check_ret_none(seq_true, seq_false, seq_names): length = len(seq_true) @@ -2687,32 +2881,53 @@ def check_ret_none(seq_true, seq_false, seq_names): f_true = flatten(seq_true[i]) f_false = flatten(seq_false[i]) for idx in range(len(f_true)): - if f_true[idx] is None and f_false[idx] is not None or f_false[ - idx] is None and f_true[idx] is not None: + if ( + f_true[idx] is None + and f_false[idx] is not None + or f_false[idx] is None + and f_true[idx] is not None + ): warnings.warn( "In cond : Var '{}' or part of it is set differently in ifelse branchs, " "<{}, {}> in true branch and <{}, {}> in false branch. Set var to " "'None' in ifelse block might lead to error.".format( - seq_names[i], type(f_true[idx]), f_true[idx], - type(f_false[idx]), f_false[idx])) - - check_ret_none(to_sequence(true_output), to_sequence(false_output), - to_sequence(return_names)) + seq_names[i], + type(f_true[idx]), + f_true[idx], + type(f_false[idx]), + f_false[idx], + ) + ) + + check_ret_none( + to_sequence(true_output), + to_sequence(false_output), + to_sequence(return_names), + ) if is_dy2staic: true_output, false_output = change_none_to_undefinedvar( - true_output, false_output) + true_output, false_output + ) mask = cast(pred, dtype='int32') - merge_func = lambda name, false_var, true_var: select_input_with_buildin_type( - [false_var, true_var], mask, name) + merge_func = ( + lambda name, false_var, true_var: select_input_with_buildin_type( + [false_var, true_var], mask, name + ) + ) def merge_every_var_list(false_vars, true_vars, name): return map_structure(partial(merge_func, name), false_vars, true_vars) merged_output = list( - map(merge_every_var_list, to_sequence(false_output), - to_sequence(true_output), to_sequence(return_names))) + map( + merge_every_var_list, + to_sequence(false_output), + to_sequence(true_output), + to_sequence(return_names), + ) + ) merged_output = pack_sequence_as(false_output, flatten(merged_output)) return merged_output @@ -2721,7 +2936,8 @@ def change_none_to_undefinedvar(nest1, nest2): from paddle.fluid.dygraph.dygraph_to_static.utils import UndefinedVar def map_fn(x): - if x is None: return UndefinedVar("padding") + if x is None: + return UndefinedVar("padding") return x nest1_out = pack_sequence_as(nest1, list(map(map_fn, flatten(nest1)))) @@ -2730,56 +2946,81 @@ def map_fn(x): def expand_undefined_var(nest1, nest2, names): - """ TODO: make this function recursively. - nest1: Var1, (UndefinedVar, [1,2,3]) - nest2: Var2, ([1,2,3,4], UndefinedVar) - In this case, we should not expand recursively. + """TODO: make this function recursively. + nest1: Var1, (UndefinedVar, [1,2,3]) + nest2: Var2, ([1,2,3,4], UndefinedVar) + In this case, we should not expand recursively. """ from paddle.fluid.dygraph.dygraph_to_static.utils import UndefinedVar - from paddle.fluid.dygraph.dygraph_to_static.return_transformer import RETURN_VALUE_PREFIX + from paddle.fluid.dygraph.dygraph_to_static.return_transformer import ( + RETURN_VALUE_PREFIX, + ) def pack_undefined_var_as(seq): - return pack_sequence_as(seq, - [UndefinedVar("padding") for i in flatten(seq)]) + return pack_sequence_as( + seq, [UndefinedVar("padding") for i in flatten(seq)] + ) def map_fn(n1, n2, name, order): - if not name.startswith(RETURN_VALUE_PREFIX) and (isinstance( - n1, UndefinedVar) or n1 is None): + if not name.startswith(RETURN_VALUE_PREFIX) and ( + isinstance(n1, UndefinedVar) or n1 is None + ): if n1 is None and n2 is not None: if order == 0: warnings.warn( "In cond : Var '{}' or part of it is set differently in ifelse branchs, " "<{}, {}> in true branch and <{}, {}> in false branch. Set var to " "'None' in ifelse block might lead to error.".format( - name, type(n1), n1, type(n2), n2)) + name, type(n1), n1, type(n2), n2 + ) + ) else: warnings.warn( "In cond : Var '{}' or part of it is set differently in ifelse branchs, " "<{}, {}> in true branch and <{}, {}> in false branch. Set var to " "'None' in ifelse block might lead to error.".format( - name, type(n2), n2, type(n1), n1)) + name, type(n2), n2, type(n1), n1 + ) + ) return pack_undefined_var_as(n2) return n1 nest1_out = list( - map(map_fn, to_sequence(nest1), to_sequence(nest2), to_sequence(names), - [0 for i in to_sequence(names)])) + map( + map_fn, + to_sequence(nest1), + to_sequence(nest2), + to_sequence(names), + [0 for i in to_sequence(names)], + ) + ) nest2_out = list( - map(map_fn, to_sequence(nest2), to_sequence(nest1), to_sequence(names), - [1 for i in to_sequence(names)])) - if not is_sequence(nest1): nest1_out = nest1_out[0] - if not is_sequence(nest2): nest2_out = nest2_out[0] + map( + map_fn, + to_sequence(nest2), + to_sequence(nest1), + to_sequence(names), + [1 for i in to_sequence(names)], + ) + ) + if not is_sequence(nest1): + nest1_out = nest1_out[0] + if not is_sequence(nest2): + nest2_out = nest2_out[0] return nest1_out, nest2_out def _error_message(what, arg_name, op_name, right_value, error_value): - error_message = "{what} of '{arg_name}' in {op_name} must be " \ + error_message = ( + "{what} of '{arg_name}' in {op_name} must be " "{right_value}, but received: {error_value}.".format( - what=what, - arg_name=arg_name, - op_name=op_name, - right_value=right_value, - error_value=error_value) + what=what, + arg_name=arg_name, + op_name=op_name, + right_value=right_value, + error_value=error_value, + ) + ) return error_message @@ -2860,24 +3101,42 @@ def _case_check_args(pred_fn_pairs, default): for pred_fn in pred_fn_pairs: if not isinstance(pred_fn, tuple): raise TypeError( - _error_message("The elements' type", "pred_fn_pairs", - "case", tuple, type(pred_fn))) + _error_message( + "The elements' type", + "pred_fn_pairs", + "case", + tuple, + type(pred_fn), + ) + ) if len(pred_fn) != 2: raise TypeError( - _error_message("The tuple's size", "pred_fn_pairs", "case", - "2", - str(len(pred_fn)) + "-tuple")) + _error_message( + "The tuple's size", + "pred_fn_pairs", + "case", + "2", + str(len(pred_fn)) + "-tuple", + ) + ) pred, fn = pred_fn if not isinstance(pred, Variable): raise TypeError( - _error_message("The pred's type", "pred_fn_pairs", "case", - "boolean Variable", type(pred))) + _error_message( + "The pred's type", + "pred_fn_pairs", + "case", + "boolean Variable", + type(pred), + ) + ) if not callable(fn): raise TypeError( "The fn for {} of pred_fn_pairs in Op(case) must" - " be callable.".format(pred.name)) + " be callable.".format(pred.name) + ) if default is None: default_index = len(pred_fn_pairs) - 1 # pick the last one @@ -2979,8 +3238,11 @@ def case(self, condition): raise ValueError("case should be called inside with") check_variable_and_dtype( - condition, 'condition', ['bool'], - 'the member function case of fluid.layers.Switch') + condition, + 'condition', + ['bool'], + 'the member function case of fluid.layers.Switch', + ) if len(self.pre_not_conditions) == 0: cond_block = ConditionalBlock([condition], is_scalar_condition=True) @@ -2989,12 +3251,14 @@ def case(self, condition): else: pre_cond_num = len(self.pre_not_conditions) pre_not_cond = self.pre_not_conditions[pre_cond_num - 1] - new_not_cond = logical_and(x=pre_not_cond, - y=logical_not(x=condition)) + new_not_cond = logical_and( + x=pre_not_cond, y=logical_not(x=condition) + ) self.pre_not_conditions.append(new_not_cond) cond_block = ConditionalBlock( [logical_and(x=pre_not_cond, y=condition)], - is_scalar_condition=True) + is_scalar_condition=True, + ) return ConditionalBlockGuard(cond_block) @@ -3004,7 +3268,8 @@ def default(self): raise ValueError("there should be at least one condition") cond_block = ConditionalBlock( [self.pre_not_conditions[pre_cond_num - 1]], - is_scalar_condition=True) + is_scalar_condition=True, + ) return ConditionalBlockGuard(cond_block) def __enter__(self): @@ -3024,7 +3289,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): class IfElseBlockGuard(object): - def __init__(self, is_true, ifelse): if not isinstance(ifelse, IfElse): raise TypeError("ifelse must be an instance of IfElse class") @@ -3045,7 +3309,11 @@ def __init__(self, is_true, ifelse): self.cond_block = self.cond_block.block() def __enter__(self): - self.ie.status = IfElse.IN_IF_ELSE_TRUE_BLOCKS if self.is_true else IfElse.IN_IF_ELSE_FALSE_BLOCKS + self.ie.status = ( + IfElse.IN_IF_ELSE_TRUE_BLOCKS + if self.is_true + else IfElse.IN_IF_ELSE_FALSE_BLOCKS + ) self.cond_block.__enter__() def __exit__(self, exc_type, exc_val, exc_tb): @@ -3132,6 +3400,7 @@ class IfElse(object): There is a ``call ()`` function inside the object, that is, by calling ``output = ie ()``, all the outputs inside the block of False are fused as the whole output, the output type is a list, and the type of each element in the list is Variable. """ + OUT_IF_ELSE_BLOCKS = 0 IN_IF_ELSE_TRUE_BLOCKS = 1 IN_IF_ELSE_FALSE_BLOCKS = 2 @@ -3153,24 +3422,27 @@ def input(self, x): if id(x) not in self.input_table: parent_block = self._parent_block() out_true = parent_block.create_var( - name=unique_name.generate_with_ignorable_key('ifelse_input' + - self.helper.name), - dtype=x.dtype) + name=unique_name.generate_with_ignorable_key( + 'ifelse_input' + self.helper.name + ), + dtype=x.dtype, + ) out_false = parent_block.create_var( - name=unique_name.generate_with_ignorable_key('ifelse_input' + - self.helper.name), - dtype=x.dtype) - parent_block.append_op(type='split_lod_tensor', - inputs={ - 'X': x, - 'Mask': self.cond, - }, - outputs={ - 'OutTrue': out_true, - 'OutFalse': out_false - }, - attrs={'level': 0}) + name=unique_name.generate_with_ignorable_key( + 'ifelse_input' + self.helper.name + ), + dtype=x.dtype, + ) + parent_block.append_op( + type='split_lod_tensor', + inputs={ + 'X': x, + 'Mask': self.cond, + }, + outputs={'OutTrue': out_true, 'OutFalse': out_false}, + attrs={'level': 0}, + ) self.input_table[id(x)] = (out_true, out_false) else: out_true, out_false = self.input_table[id(x)] @@ -3194,17 +3466,21 @@ def output(self, *outs): if self.status == self.OUT_IF_ELSE_BLOCKS: raise ValueError("output can only be invoked in the sub-block") - out_table = self.output_table[1 if self.status == - self.IN_IF_ELSE_TRUE_BLOCKS else 0] + out_table = self.output_table[ + 1 if self.status == self.IN_IF_ELSE_TRUE_BLOCKS else 0 + ] parent_block = self._parent_block() for each_out in outs: - check_type(each_out, "each output", Variable, - "fluid.layers.IfElse.output") + check_type( + each_out, "each output", Variable, "fluid.layers.IfElse.output" + ) # create outside tensor outside_out = parent_block.create_var( - name=unique_name.generate_with_ignorable_key("_".join( - [self.helper.name, 'output'])), - dtype=each_out.dtype) + name=unique_name.generate_with_ignorable_key( + "_".join([self.helper.name, 'output']) + ), + dtype=each_out.dtype, + ) out_table.append(outside_out) # assign local var to outside @@ -3215,8 +3491,9 @@ def __call__(self): raise ValueError("IfElse::__call__ must be out of sub-block") false_len, true_len = list(map(len, self.output_table)) if false_len == 0 and true_len == 0: - raise ValueError("Must invoke true_block/false_block before " - "__call__") + raise ValueError( + "Must invoke true_block/false_block before " "__call__" + ) elif false_len != true_len and false_len != 0 and true_len != 0: raise ValueError("The output side must be same") elif false_len == 0 or true_len == 0: @@ -3227,11 +3504,14 @@ def __call__(self): rlist = [] for false_var, true_var in zip(*self.output_table): rlist.append( - merge_lod_tensor(in_true=true_var, - in_false=false_var, - mask=self.cond, - x=self.cond, - level=0)) + merge_lod_tensor( + in_true=true_var, + in_false=false_var, + mask=self.cond, + x=self.cond, + level=0, + ) + ) return rlist @@ -3302,6 +3582,7 @@ class DynamicRNN(object): # Get RNN's result of the last time step last = fluid.layers.sequence_last_step(out) """ + BEFORE_RNN = 0 IN_RNN = 1 AFTER_RNN = 2 @@ -3419,39 +3700,44 @@ def step_input(self, x, level=0): if self.lod_rank_table is None: self.lod_rank_table = parent_block.create_var( name=unique_name.generate('lod_rank_table'), - type=core.VarDesc.VarType.LOD_RANK_TABLE) + type=core.VarDesc.VarType.LOD_RANK_TABLE, + ) self.lod_rank_table.stop_gradient = True - parent_block.append_op(type='lod_rank_table', - inputs={"X": x}, - outputs={"Out": self.lod_rank_table}, - attrs={"level": level}) + parent_block.append_op( + type='lod_rank_table', + inputs={"X": x}, + outputs={"Out": self.lod_rank_table}, + attrs={"level": level}, + ) self.max_seq_len = parent_block.create_var( name=unique_name.generate('dynamic_rnn_max_seq_len'), - dtype='int64') + dtype='int64', + ) self.max_seq_len.stop_gradient = False - parent_block.append_op(type='max_sequence_len', - inputs={'RankTable': self.lod_rank_table}, - outputs={"Out": self.max_seq_len}) + parent_block.append_op( + type='max_sequence_len', + inputs={'RankTable': self.lod_rank_table}, + outputs={"Out": self.max_seq_len}, + ) self.cond.stop_gradient = True - parent_block.append_op(type='less_than', - inputs={ - 'X': self.step_idx, - 'Y': self.max_seq_len - }, - outputs={'Out': self.cond}, - attrs={'force_cpu': True}) + parent_block.append_op( + type='less_than', + inputs={'X': self.step_idx, 'Y': self.max_seq_len}, + outputs={'Out': self.cond}, + attrs={'force_cpu': True}, + ) input_array = parent_block.create_var( name=unique_name.generate('dynamic_rnn_input_array'), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=x.dtype) + dtype=x.dtype, + ) self.input_array.append((input_array, x.dtype)) - parent_block.append_op(type='lod_tensor_to_array', - inputs={ - 'X': x, - 'RankTable': self.lod_rank_table - }, - outputs={'Out': input_array}) + parent_block.append_op( + type='lod_tensor_to_array', + inputs={'X': x, 'RankTable': self.lod_rank_table}, + outputs={'Out': input_array}, + ) return array_read(array=input_array, i=self.step_idx) def static_input(self, x): @@ -3584,18 +3870,19 @@ def static_input(self, x): check_type(x, 'x', Variable, 'fluid.layers.DynamicRNN.static_input()') if self.lod_rank_table is None: raise RuntimeError( - "static_input() must be called after step_input().") + "static_input() must be called after step_input()." + ) parent_block = self._parent_block_() x_reordered = parent_block.create_var( name=unique_name.generate("dynamic_rnn_static_input_reordered"), type=core.VarDesc.VarType.LOD_TENSOR, - dtype=x.dtype) - parent_block.append_op(type='reorder_lod_tensor_by_rank', - inputs={ - 'X': [x], - 'RankTable': [self.lod_rank_table] - }, - outputs={'Out': [x_reordered]}) + dtype=x.dtype, + ) + parent_block.append_op( + type='reorder_lod_tensor_by_rank', + inputs={'X': [x], 'RankTable': [self.lod_rank_table]}, + outputs={'Out': [x_reordered]}, + ) return shrink_memory(x_reordered, self.step_idx, self.lod_rank_table) @signature_safe_contextmanager @@ -3610,10 +3897,9 @@ def block(self): """ if self.status != DynamicRNN.BEFORE_RNN: raise ValueError("rnn.block() can only be invoke once") - self.step_idx = fill_constant(shape=[1], - dtype='int64', - value=0, - force_cpu=True) + self.step_idx = fill_constant( + shape=[1], dtype='int64', value=0, force_cpu=True + ) self.step_idx.stop_gradient = False self.status = DynamicRNN.IN_RNN with self.while_op.block(): @@ -3623,15 +3909,18 @@ def block(self): for new_mem, mem_array in self.mem_link: array_write(x=new_mem, i=self.step_idx, array=mem_array) - less_than(x=self.step_idx, - y=self.max_seq_len, - force_cpu=True, - cond=self.cond) + less_than( + x=self.step_idx, + y=self.max_seq_len, + force_cpu=True, + cond=self.cond, + ) self.status = DynamicRNN.AFTER_RNN for each_array in self.output_array: self.outputs.append( - array_to_lod_tensor(x=each_array, table=self.lod_rank_table)) + array_to_lod_tensor(x=each_array, table=self.lod_rank_table) + ) def __call__(self, *args, **kwargs): """ @@ -3647,19 +3936,25 @@ def __call__(self, *args, **kwargs): ValueError: When :code:`__call__()` is called before :code:`block()` . """ if self.status != DynamicRNN.AFTER_RNN: - raise ValueError(("Output of the dynamic RNN can only be visited " - "outside the rnn block.")) + raise ValueError( + ( + "Output of the dynamic RNN can only be visited " + "outside the rnn block." + ) + ) if len(self.outputs) == 1: return self.outputs[0] else: return self.outputs - def memory(self, - init=None, - shape=None, - value=0.0, - need_reorder=False, - dtype='float32'): + def memory( + self, + init=None, + shape=None, + value=0.0, + need_reorder=False, + dtype='float32', + ): r""" Create a memory Variable for DynamicRNN to deliver data cross time steps. It can be initialized by an existing Tensor or a constant Tensor of given @@ -3748,11 +4043,16 @@ def memory(self, self._assert_in_rnn_block_('memory') self._init_zero_idx_() if shape is not None: - check_type(shape, 'shape', (list, tuple), - 'fluid.layers.DynamicRNN.memory()') + check_type( + shape, + 'shape', + (list, tuple), + 'fluid.layers.DynamicRNN.memory()', + ) if init is not None: - check_type(init, 'init', Variable, - 'fluid.layers.DynamicRNN.memory()') + check_type( + init, 'init', Variable, 'fluid.layers.DynamicRNN.memory()' + ) parent_block = self._parent_block_() init_tensor = init if need_reorder == True: @@ -3760,32 +4060,36 @@ def memory(self, raise ValueError( 'If set need_reorder to True, make sure step_input be ' 'invoked before ' - 'memory(init=init, need_reordered=True, ...).') + 'memory(init=init, need_reordered=True, ...).' + ) init_reordered = parent_block.create_var( name=unique_name.generate('dynamic_rnn_mem_init_reordered'), type=core.VarDesc.VarType.LOD_TENSOR, - dtype=init.dtype) - parent_block.append_op(type='reorder_lod_tensor_by_rank', - inputs={ - 'X': [init_tensor], - 'RankTable': [self.lod_rank_table] - }, - outputs={'Out': [init_reordered]}) + dtype=init.dtype, + ) + parent_block.append_op( + type='reorder_lod_tensor_by_rank', + inputs={ + 'X': [init_tensor], + 'RankTable': [self.lod_rank_table], + }, + outputs={'Out': [init_reordered]}, + ) init_tensor = init_reordered mem_array = parent_block.create_var( name=unique_name.generate('dynamic_rnn_mem_array'), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=init.dtype) - parent_block.append_op(type='write_to_array', - inputs={ - 'X': init_tensor, - 'I': self.zero_idx - }, - outputs={'Out': mem_array}) + dtype=init.dtype, + ) + parent_block.append_op( + type='write_to_array', + inputs={'X': init_tensor, 'I': self.zero_idx}, + outputs={'Out': mem_array}, + ) retv = array_read(array=mem_array, i=self.step_idx) - retv = shrink_memory(x=retv, - i=self.step_idx, - table=self.lod_rank_table) + retv = shrink_memory( + x=retv, i=self.step_idx, table=self.lod_rank_table + ) self.mem_dict[retv.name] = mem_array return retv else: @@ -3795,24 +4099,27 @@ def memory(self, ) parent_block = self._parent_block_() init = parent_block.create_var( - name=unique_name.generate('mem_init'), dtype=dtype) + name=unique_name.generate('mem_init'), dtype=dtype + ) arr, dtype = self.input_array[0] - in0 = parent_block.create_var(name=unique_name.generate('in0'), - dtype=dtype) - parent_block.append_op(type='read_from_array', - inputs={ - 'X': [arr], - 'I': [self.zero_idx] - }, - outputs={'Out': [in0]}) - parent_block.append_op(type='fill_constant_batch_size_like', - inputs={'Input': [in0]}, - outputs={'Out': [init]}, - attrs={ - 'shape': [-1] + shape, - 'value': float(value), - 'dtype': init.dtype - }) + in0 = parent_block.create_var( + name=unique_name.generate('in0'), dtype=dtype + ) + parent_block.append_op( + type='read_from_array', + inputs={'X': [arr], 'I': [self.zero_idx]}, + outputs={'Out': [in0]}, + ) + parent_block.append_op( + type='fill_constant_batch_size_like', + inputs={'Input': [in0]}, + outputs={'Out': [init]}, + attrs={ + 'shape': [-1] + shape, + 'value': float(value), + 'dtype': init.dtype, + }, + ) return self.memory(init=init) def update_memory(self, ex_mem, new_mem): @@ -3834,10 +4141,18 @@ def update_memory(self, ex_mem, new_mem): ValueError: When :code:`update_memory()` is called before :code:`step_input()` . """ self._assert_in_rnn_block_('update_memory') - check_type(ex_mem, 'ex_mem', Variable, - 'fluid.layers.DynamicRNN.update_memory()') - check_type(new_mem, 'new_mem', Variable, - 'fluid.layers.DynamicRNN.update_memory()') + check_type( + ex_mem, + 'ex_mem', + Variable, + 'fluid.layers.DynamicRNN.update_memory()', + ) + check_type( + new_mem, + 'new_mem', + Variable, + 'fluid.layers.DynamicRNN.update_memory()', + ) mem_array = self.mem_dict.get(ex_mem.name, None) if mem_array is None: @@ -3864,13 +4179,16 @@ def output(self, *outputs): self._assert_in_rnn_block_('output') parent_block = self._parent_block_() for each in outputs: - check_type(each, "outputs", Variable, - "fluid.layers.DynamicRNN.output") + check_type( + each, "outputs", Variable, "fluid.layers.DynamicRNN.output" + ) outside_array = parent_block.create_var( - name=unique_name.generate_with_ignorable_key("_".join( - [self.helper.name, "output_array", each.name])), + name=unique_name.generate_with_ignorable_key( + "_".join([self.helper.name, "output_array", each.name]) + ), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=each.dtype) + dtype=each.dtype, + ) array_write(x=each, i=self.step_idx, array=outside_array) self.output_array.append(outside_array) @@ -3878,16 +4196,19 @@ def _init_zero_idx_(self): if self.zero_idx is None: parent_block = self._parent_block_() self.zero_idx = parent_block.create_var( - name=unique_name.generate('zero_idx'), dtype='int64') - parent_block.append_op(type='fill_constant', - inputs={}, - outputs={'Out': [self.zero_idx]}, - attrs={ - 'shape': [1], - 'dtype': self.zero_idx.dtype, - 'value': float(0), - 'force_cpu': True - }) + name=unique_name.generate('zero_idx'), dtype='int64' + ) + parent_block.append_op( + type='fill_constant', + inputs={}, + outputs={'Out': [self.zero_idx]}, + attrs={ + 'shape': [1], + 'dtype': self.zero_idx.dtype, + 'value': float(0), + 'force_cpu': True, + }, + ) def _parent_block_(self): prog = self.helper.main_program @@ -3900,7 +4221,8 @@ def _parent_block_(self): def _assert_in_rnn_block_(self, method): if self.status != DynamicRNN.IN_RNN: raise ValueError( - "{0} can only be invoked inside rnn block.".format(method)) + "{0} can only be invoked inside rnn block.".format(method) + ) def switch_case(branch_index, branch_fns, default=None, name=None): @@ -3977,44 +4299,71 @@ def fn_3(): def _check_args(branch_index, branch_fns, default): - check_variable_and_dtype(branch_index, 'branch_index', - ['uint8', 'int32', 'int64'], 'switch_case') + check_variable_and_dtype( + branch_index, + 'branch_index', + ['uint8', 'int32', 'int64'], + 'switch_case', + ) if convert_dtype(branch_index.dtype) != "int64": branch_index = cast(branch_index, "int64") check_type(branch_fns, 'branch_fns', (list, tuple, dict), 'switch_case') - branch_fns = branch_fns.items() if isinstance(branch_fns, - dict) else branch_fns + branch_fns = ( + branch_fns.items() if isinstance(branch_fns, dict) else branch_fns + ) - branch_fns = list(enumerate(branch_fns)) if all( - callable(fn) for fn in branch_fns) else branch_fns + branch_fns = ( + list(enumerate(branch_fns)) + if all(callable(fn) for fn in branch_fns) + else branch_fns + ) keys_of_fns = [] for index_fn_pair in branch_fns: if not isinstance(index_fn_pair, tuple): raise TypeError( - _error_message("The elements' type", "branch_fns", - "switch_case", tuple, type(branch_fns))) + _error_message( + "The elements' type", + "branch_fns", + "switch_case", + tuple, + type(branch_fns), + ) + ) if len(index_fn_pair) != 2: raise TypeError( - _error_message("The tuple's size", "branch_fns", - "switch_case", "2", - str(len(index_fn_pair)) + "-tuple")) + _error_message( + "The tuple's size", + "branch_fns", + "switch_case", + "2", + str(len(index_fn_pair)) + "-tuple", + ) + ) key, fn = index_fn_pair if not isinstance(key, int): raise TypeError( - _error_message("The key's type", "branch_fns", - "switch_case", int, type(key))) + _error_message( + "The key's type", + "branch_fns", + "switch_case", + int, + type(key), + ) + ) if key in keys_of_fns: raise ValueError( - "The key in 'branch_fns' must be unique, but '{}' appears more than once." - .format(key)) + "The key in 'branch_fns' must be unique, but '{}' appears more than once.".format( + key + ) + ) else: keys_of_fns.append(key) @@ -4022,7 +4371,12 @@ def _check_args(branch_index, branch_fns, default): raise TypeError( _error_message( "The type of function for key {}".format(key), - "branch_fns", "switch_case", "callable", type(fn))) + "branch_fns", + "switch_case", + "callable", + type(fn), + ) + ) if default is None: default = sorted(branch_fns)[-1][1] @@ -4073,20 +4427,20 @@ def reorder_lod_tensor_by_rank(x, rank_table): """ check_type(x, 'x', (Variable), 'reorder_lod_tensor_by_rank') - check_type(rank_table, 'rank_table', (Variable), - 'reorder_lod_tensor_by_rank') + check_type( + rank_table, 'rank_table', (Variable), 'reorder_lod_tensor_by_rank' + ) if rank_table.type != core.VarDesc.VarType.LOD_RANK_TABLE: raise TypeError("The type of rank_table should be LOD_RANK_TABLE.") helper = LayerHelper('reorder_lod_tensor_by_rank', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reorder_lod_tensor_by_rank', - inputs={ - 'X': [x], - 'RankTable': [rank_table] - }, - outputs={'Out': [out]}) + helper.append_op( + type='reorder_lod_tensor_by_rank', + inputs={'X': [x], 'RankTable': [rank_table]}, + outputs={'Out': [out]}, + ) return out @@ -4125,14 +4479,15 @@ def is_empty(x, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.is_empty(x) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'is_empty') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'is_empty' + ) check_type(name, "name", (str, type(None)), "is_empty") helper = LayerHelper("is_empty", **locals()) cond = helper.create_variable_for_type_inference(dtype='bool') cond.stop_gradient = True - helper.append_op(type='is_empty', - inputs={'X': [x]}, - outputs={'Out': [cond]}) + helper.append_op( + type='is_empty', inputs={'X': [x]}, outputs={'Out': [cond]} + ) return cond diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index 80754898e72ca..d577459237472 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -31,7 +31,12 @@ import math import numpy as np from functools import reduce -from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) from paddle.utils import deprecated from paddle import _C_ops, _legacy_C_ops from ..framework import in_dygraph_mode @@ -68,17 +73,19 @@ ] -def retinanet_target_assign(bbox_pred, - cls_logits, - anchor_box, - anchor_var, - gt_boxes, - gt_labels, - is_crowd, - im_info, - num_classes=1, - positive_overlap=0.5, - negative_overlap=0.4): +def retinanet_target_assign( + bbox_pred, + cls_logits, + anchor_box, + anchor_var, + gt_boxes, + gt_labels, + is_crowd, + im_info, + num_classes=1, + positive_overlap=0.5, + negative_overlap=0.4, +): r""" **Target Assign Layer for the detector RetinaNet.** @@ -245,22 +252,42 @@ def retinanet_target_assign(bbox_pred, """ - check_variable_and_dtype(bbox_pred, 'bbox_pred', ['float32', 'float64'], - 'retinanet_target_assign') - check_variable_and_dtype(cls_logits, 'cls_logits', ['float32', 'float64'], - 'retinanet_target_assign') - check_variable_and_dtype(anchor_box, 'anchor_box', ['float32', 'float64'], - 'retinanet_target_assign') - check_variable_and_dtype(anchor_var, 'anchor_var', ['float32', 'float64'], - 'retinanet_target_assign') - check_variable_and_dtype(gt_boxes, 'gt_boxes', ['float32', 'float64'], - 'retinanet_target_assign') - check_variable_and_dtype(gt_labels, 'gt_labels', ['int32'], - 'retinanet_target_assign') - check_variable_and_dtype(is_crowd, 'is_crowd', ['int32'], - 'retinanet_target_assign') - check_variable_and_dtype(im_info, 'im_info', ['float32', 'float64'], - 'retinanet_target_assign') + check_variable_and_dtype( + bbox_pred, + 'bbox_pred', + ['float32', 'float64'], + 'retinanet_target_assign', + ) + check_variable_and_dtype( + cls_logits, + 'cls_logits', + ['float32', 'float64'], + 'retinanet_target_assign', + ) + check_variable_and_dtype( + anchor_box, + 'anchor_box', + ['float32', 'float64'], + 'retinanet_target_assign', + ) + check_variable_and_dtype( + anchor_var, + 'anchor_var', + ['float32', 'float64'], + 'retinanet_target_assign', + ) + check_variable_and_dtype( + gt_boxes, 'gt_boxes', ['float32', 'float64'], 'retinanet_target_assign' + ) + check_variable_and_dtype( + gt_labels, 'gt_labels', ['int32'], 'retinanet_target_assign' + ) + check_variable_and_dtype( + is_crowd, 'is_crowd', ['int32'], 'retinanet_target_assign' + ) + check_variable_and_dtype( + im_info, 'im_info', ['float32', 'float64'], 'retinanet_target_assign' + ) helper = LayerHelper('retinanet_target_assign', **locals()) # Assign target label to anchors @@ -268,30 +295,34 @@ def retinanet_target_assign(bbox_pred, score_index = helper.create_variable_for_type_inference(dtype='int32') target_label = helper.create_variable_for_type_inference(dtype='int32') target_bbox = helper.create_variable_for_type_inference( - dtype=anchor_box.dtype) + dtype=anchor_box.dtype + ) bbox_inside_weight = helper.create_variable_for_type_inference( - dtype=anchor_box.dtype) + dtype=anchor_box.dtype + ) fg_num = helper.create_variable_for_type_inference(dtype='int32') - helper.append_op(type="retinanet_target_assign", - inputs={ - 'Anchor': anchor_box, - 'GtBoxes': gt_boxes, - 'GtLabels': gt_labels, - 'IsCrowd': is_crowd, - 'ImInfo': im_info - }, - outputs={ - 'LocationIndex': loc_index, - 'ScoreIndex': score_index, - 'TargetLabel': target_label, - 'TargetBBox': target_bbox, - 'BBoxInsideWeight': bbox_inside_weight, - 'ForegroundNumber': fg_num - }, - attrs={ - 'positive_overlap': positive_overlap, - 'negative_overlap': negative_overlap - }) + helper.append_op( + type="retinanet_target_assign", + inputs={ + 'Anchor': anchor_box, + 'GtBoxes': gt_boxes, + 'GtLabels': gt_labels, + 'IsCrowd': is_crowd, + 'ImInfo': im_info, + }, + outputs={ + 'LocationIndex': loc_index, + 'ScoreIndex': score_index, + 'TargetLabel': target_label, + 'TargetBBox': target_bbox, + 'BBoxInsideWeight': bbox_inside_weight, + 'ForegroundNumber': fg_num, + }, + attrs={ + 'positive_overlap': positive_overlap, + 'negative_overlap': negative_overlap, + }, + ) loc_index.stop_gradient = True score_index.stop_gradient = True @@ -305,22 +336,31 @@ def retinanet_target_assign(bbox_pred, predicted_cls_logits = nn.gather(cls_logits, score_index) predicted_bbox_pred = nn.gather(bbox_pred, loc_index) - return predicted_cls_logits, predicted_bbox_pred, target_label, target_bbox, bbox_inside_weight, fg_num - - -def rpn_target_assign(bbox_pred, - cls_logits, - anchor_box, - anchor_var, - gt_boxes, - is_crowd, - im_info, - rpn_batch_size_per_im=256, - rpn_straddle_thresh=0.0, - rpn_fg_fraction=0.5, - rpn_positive_overlap=0.7, - rpn_negative_overlap=0.3, - use_random=True): + return ( + predicted_cls_logits, + predicted_bbox_pred, + target_label, + target_bbox, + bbox_inside_weight, + fg_num, + ) + + +def rpn_target_assign( + bbox_pred, + cls_logits, + anchor_box, + anchor_var, + gt_boxes, + is_crowd, + im_info, + rpn_batch_size_per_im=256, + rpn_straddle_thresh=0.0, + rpn_fg_fraction=0.5, + rpn_positive_overlap=0.7, + rpn_negative_overlap=0.3, + use_random=True, +): """ **Target Assign Layer for region proposal network (RPN) in Faster-RCNN detection.** @@ -410,51 +450,62 @@ def rpn_target_assign(bbox_pred, helper = LayerHelper('rpn_target_assign', **locals()) - check_variable_and_dtype(bbox_pred, 'bbox_pred', ['float32', 'float64'], - 'rpn_target_assign') - check_variable_and_dtype(cls_logits, 'cls_logits', ['float32', 'float64'], - 'rpn_target_assign') - check_variable_and_dtype(anchor_box, 'anchor_box', ['float32', 'float64'], - 'rpn_target_assign') - check_variable_and_dtype(anchor_var, 'anchor_var', ['float32', 'float64'], - 'rpn_target_assign') - check_variable_and_dtype(gt_boxes, 'gt_boxes', ['float32', 'float64'], - 'rpn_target_assign') - check_variable_and_dtype(is_crowd, 'is_crowd', ['int32'], - 'rpn_target_assign') - check_variable_and_dtype(im_info, 'im_info', ['float32', 'float64'], - 'rpn_target_assign') + check_variable_and_dtype( + bbox_pred, 'bbox_pred', ['float32', 'float64'], 'rpn_target_assign' + ) + check_variable_and_dtype( + cls_logits, 'cls_logits', ['float32', 'float64'], 'rpn_target_assign' + ) + check_variable_and_dtype( + anchor_box, 'anchor_box', ['float32', 'float64'], 'rpn_target_assign' + ) + check_variable_and_dtype( + anchor_var, 'anchor_var', ['float32', 'float64'], 'rpn_target_assign' + ) + check_variable_and_dtype( + gt_boxes, 'gt_boxes', ['float32', 'float64'], 'rpn_target_assign' + ) + check_variable_and_dtype( + is_crowd, 'is_crowd', ['int32'], 'rpn_target_assign' + ) + check_variable_and_dtype( + im_info, 'im_info', ['float32', 'float64'], 'rpn_target_assign' + ) # Assign target label to anchors loc_index = helper.create_variable_for_type_inference(dtype='int32') score_index = helper.create_variable_for_type_inference(dtype='int32') target_label = helper.create_variable_for_type_inference(dtype='int32') target_bbox = helper.create_variable_for_type_inference( - dtype=anchor_box.dtype) + dtype=anchor_box.dtype + ) bbox_inside_weight = helper.create_variable_for_type_inference( - dtype=anchor_box.dtype) - helper.append_op(type="rpn_target_assign", - inputs={ - 'Anchor': anchor_box, - 'GtBoxes': gt_boxes, - 'IsCrowd': is_crowd, - 'ImInfo': im_info - }, - outputs={ - 'LocationIndex': loc_index, - 'ScoreIndex': score_index, - 'TargetLabel': target_label, - 'TargetBBox': target_bbox, - 'BBoxInsideWeight': bbox_inside_weight - }, - attrs={ - 'rpn_batch_size_per_im': rpn_batch_size_per_im, - 'rpn_straddle_thresh': rpn_straddle_thresh, - 'rpn_positive_overlap': rpn_positive_overlap, - 'rpn_negative_overlap': rpn_negative_overlap, - 'rpn_fg_fraction': rpn_fg_fraction, - 'use_random': use_random - }) + dtype=anchor_box.dtype + ) + helper.append_op( + type="rpn_target_assign", + inputs={ + 'Anchor': anchor_box, + 'GtBoxes': gt_boxes, + 'IsCrowd': is_crowd, + 'ImInfo': im_info, + }, + outputs={ + 'LocationIndex': loc_index, + 'ScoreIndex': score_index, + 'TargetLabel': target_label, + 'TargetBBox': target_bbox, + 'BBoxInsideWeight': bbox_inside_weight, + }, + attrs={ + 'rpn_batch_size_per_im': rpn_batch_size_per_im, + 'rpn_straddle_thresh': rpn_straddle_thresh, + 'rpn_positive_overlap': rpn_positive_overlap, + 'rpn_negative_overlap': rpn_negative_overlap, + 'rpn_fg_fraction': rpn_fg_fraction, + 'use_random': use_random, + }, + ) loc_index.stop_gradient = True score_index.stop_gradient = True @@ -467,7 +518,13 @@ def rpn_target_assign(bbox_pred, predicted_cls_logits = nn.gather(cls_logits, score_index) predicted_bbox_pred = nn.gather(bbox_pred, loc_index) - return predicted_cls_logits, predicted_bbox_pred, target_label, target_bbox, bbox_inside_weight + return ( + predicted_cls_logits, + predicted_bbox_pred, + target_label, + target_bbox, + bbox_inside_weight, + ) def sigmoid_focal_loss(x, label, fg_num, gamma=2.0, alpha=0.25): @@ -597,8 +654,9 @@ def build_model(mode='train'): print(outs) """ - check_variable_and_dtype(x, 'x', ['float32', 'float64'], - 'sigmoid_focal_loss') + check_variable_and_dtype( + x, 'x', ['float32', 'float64'], 'sigmoid_focal_loss' + ) check_variable_and_dtype(label, 'label', ['int32'], 'sigmoid_focal_loss') check_variable_and_dtype(fg_num, 'fg_num', ['int32'], 'sigmoid_focal_loss') @@ -606,31 +664,28 @@ def build_model(mode='train'): out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="sigmoid_focal_loss", - inputs={ - "X": x, - "Label": label, - "FgNum": fg_num - }, - attrs={ - "gamma": gamma, - 'alpha': alpha - }, - outputs={"Out": out}) + helper.append_op( + type="sigmoid_focal_loss", + inputs={"X": x, "Label": label, "FgNum": fg_num}, + attrs={"gamma": gamma, 'alpha': alpha}, + outputs={"Out": out}, + ) return out -def detection_output(loc, - scores, - prior_box, - prior_box_var, - background_label=0, - nms_threshold=0.3, - nms_top_k=400, - keep_top_k=200, - score_threshold=0.01, - nms_eta=1.0, - return_index=False): +def detection_output( + loc, + scores, + prior_box, + prior_box_var, + background_label=0, + nms_threshold=0.3, + nms_top_k=400, + keep_top_k=200, + score_threshold=0.01, + nms_eta=1.0, + return_index=False, +): """ Given the regression locations, classification confidences and prior boxes, @@ -715,50 +770,48 @@ class number, M is number of bounding boxes. return_index=True) """ helper = LayerHelper("detection_output", **locals()) - decoded_box = box_coder(prior_box=prior_box, - prior_box_var=prior_box_var, - target_box=loc, - code_type='decode_center_size') + decoded_box = box_coder( + prior_box=prior_box, + prior_box_var=prior_box_var, + target_box=loc, + code_type='decode_center_size', + ) scores = nn.softmax(input=scores) scores = nn.transpose(scores, perm=[0, 2, 1]) scores.stop_gradient = True nmsed_outs = helper.create_variable_for_type_inference( - dtype=decoded_box.dtype) + dtype=decoded_box.dtype + ) if return_index: index = helper.create_variable_for_type_inference(dtype='int') - helper.append_op(type="multiclass_nms2", - inputs={ - 'Scores': scores, - 'BBoxes': decoded_box - }, - outputs={ - 'Out': nmsed_outs, - 'Index': index - }, - attrs={ - 'background_label': 0, - 'nms_threshold': nms_threshold, - 'nms_top_k': nms_top_k, - 'keep_top_k': keep_top_k, - 'score_threshold': score_threshold, - 'nms_eta': 1.0, - }) + helper.append_op( + type="multiclass_nms2", + inputs={'Scores': scores, 'BBoxes': decoded_box}, + outputs={'Out': nmsed_outs, 'Index': index}, + attrs={ + 'background_label': 0, + 'nms_threshold': nms_threshold, + 'nms_top_k': nms_top_k, + 'keep_top_k': keep_top_k, + 'score_threshold': score_threshold, + 'nms_eta': 1.0, + }, + ) index.stop_gradient = True else: - helper.append_op(type="multiclass_nms", - inputs={ - 'Scores': scores, - 'BBoxes': decoded_box - }, - outputs={'Out': nmsed_outs}, - attrs={ - 'background_label': 0, - 'nms_threshold': nms_threshold, - 'nms_top_k': nms_top_k, - 'keep_top_k': keep_top_k, - 'score_threshold': score_threshold, - 'nms_eta': 1.0, - }) + helper.append_op( + type="multiclass_nms", + inputs={'Scores': scores, 'BBoxes': decoded_box}, + outputs={'Out': nmsed_outs}, + attrs={ + 'background_label': 0, + 'nms_threshold': nms_threshold, + 'nms_top_k': nms_top_k, + 'keep_top_k': keep_top_k, + 'score_threshold': score_threshold, + 'nms_eta': 1.0, + }, + ) nmsed_outs.stop_gradient = True if return_index: return nmsed_outs, index @@ -768,9 +821,9 @@ class number, M is number of bounding boxes. @templatedoc() def iou_similarity(x, y, box_normalized=True, name=None): """ - :alias_main: paddle.nn.functional.iou_similarity - :alias: paddle.nn.functional.iou_similarity,paddle.nn.functional.loss.iou_similarity - :old_api: paddle.fluid.layers.iou_similarity + :alias_main: paddle.nn.functional.iou_similarity + :alias: paddle.nn.functional.iou_similarity,paddle.nn.functional.loss.iou_similarity + :old_api: paddle.fluid.layers.iou_similarity ${comment} @@ -810,24 +863,25 @@ def iou_similarity(x, y, box_normalized=True, name=None): helper = LayerHelper("iou_similarity", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="iou_similarity", - inputs={ - "X": x, - "Y": y - }, - attrs={"box_normalized": box_normalized}, - outputs={"Out": out}) + helper.append_op( + type="iou_similarity", + inputs={"X": x, "Y": y}, + attrs={"box_normalized": box_normalized}, + outputs={"Out": out}, + ) return out @templatedoc() -def box_coder(prior_box, - prior_box_var, - target_box, - code_type="encode_center_size", - box_normalized=True, - name=None, - axis=0): +def box_coder( + prior_box, + prior_box_var, + target_box, + code_type="encode_center_size", + box_normalized=True, + name=None, + axis=0, +): r""" **Box Coder Layer** @@ -942,33 +996,49 @@ def box_coder(prior_box, box_normalized=False, axis=1) """ - check_variable_and_dtype(prior_box, 'prior_box', ['float32', 'float64'], - 'box_coder') - check_variable_and_dtype(target_box, 'target_box', ['float32', 'float64'], - 'box_coder') + check_variable_and_dtype( + prior_box, 'prior_box', ['float32', 'float64'], 'box_coder' + ) + check_variable_and_dtype( + target_box, 'target_box', ['float32', 'float64'], 'box_coder' + ) if in_dygraph_mode(): if isinstance(prior_box_var, Variable): - box_coder_op = _C_ops.box_coder(prior_box, prior_box_var, - target_box, code_type, - box_normalized, axis, []) + box_coder_op = _C_ops.box_coder( + prior_box, + prior_box_var, + target_box, + code_type, + box_normalized, + axis, + [], + ) elif isinstance(prior_box_var, list): - box_coder_op = _C_ops.box_coder(prior_box, None, target_box, - code_type, box_normalized, axis, - prior_box_var) + box_coder_op = _C_ops.box_coder( + prior_box, + None, + target_box, + code_type, + box_normalized, + axis, + prior_box_var, + ) else: raise TypeError( - "Input variance of box_coder must be Variable or lisz") + "Input variance of box_coder must be Variable or lisz" + ) return box_coder_op helper = LayerHelper("box_coder", **locals()) output_box = helper.create_variable_for_type_inference( - dtype=prior_box.dtype) + dtype=prior_box.dtype + ) inputs = {"PriorBox": prior_box, "TargetBox": target_box} attrs = { "code_type": code_type, "box_normalized": box_normalized, - "axis": axis + "axis": axis, } if isinstance(prior_box_var, Variable): inputs['PriorBoxVar'] = prior_box_var @@ -976,10 +1046,12 @@ def box_coder(prior_box, attrs['variance'] = prior_box_var else: raise TypeError("Input variance of box_coder must be Variable or lisz") - helper.append_op(type="box_coder", - inputs=inputs, - attrs=attrs, - outputs={"OutputBox": output_box}) + helper.append_op( + type="box_coder", + inputs=inputs, + attrs=attrs, + outputs={"OutputBox": output_box}, + ) return output_box @@ -1004,32 +1076,37 @@ def polygon_box_transform(input, name=None): input = fluid.data(name='input', shape=[4, 10, 5, 5], dtype='float32') out = fluid.layers.polygon_box_transform(input) """ - check_variable_and_dtype(input, "input", ['float32', 'float64'], - 'polygon_box_transform') + check_variable_and_dtype( + input, "input", ['float32', 'float64'], 'polygon_box_transform' + ) helper = LayerHelper("polygon_box_transform", **locals()) output = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type="polygon_box_transform", - inputs={"Input": input}, - attrs={}, - outputs={"Output": output}) + helper.append_op( + type="polygon_box_transform", + inputs={"Input": input}, + attrs={}, + outputs={"Output": output}, + ) return output @deprecated(since="2.0.0", update_to="paddle.vision.ops.yolo_loss") @templatedoc(op_type="yolov3_loss") -def yolov3_loss(x, - gt_box, - gt_label, - anchors, - anchor_mask, - class_num, - ignore_thresh, - downsample_ratio, - gt_score=None, - use_label_smooth=True, - name=None, - scale_x_y=1.): +def yolov3_loss( + x, + gt_box, + gt_label, + anchors, + anchor_mask, + class_num, + ignore_thresh, + downsample_ratio, + gt_score=None, + use_label_smooth=True, + name=None, + scale_x_y=1.0, +): """ ${comment} @@ -1105,18 +1182,33 @@ def yolov3_loss(x, raise TypeError("Attr class_num of yolov3_loss must be an integer") if not isinstance(ignore_thresh, float): raise TypeError( - "Attr ignore_thresh of yolov3_loss must be a float number") + "Attr ignore_thresh of yolov3_loss must be a float number" + ) if not isinstance(use_label_smooth, bool): raise TypeError( - "Attr use_label_smooth of yolov3_loss must be a bool value") + "Attr use_label_smooth of yolov3_loss must be a bool value" + ) if _non_static_mode(): - attrs = ("anchors", anchors, "anchor_mask", anchor_mask, "class_num", - class_num, "ignore_thresh", ignore_thresh, "downsample_ratio", - downsample_ratio, "use_label_smooth", use_label_smooth, - "scale_x_y", scale_x_y) - loss, _, _ = _legacy_C_ops.yolov3_loss(x, gt_box, gt_label, gt_score, - *attrs) + attrs = ( + "anchors", + anchors, + "anchor_mask", + anchor_mask, + "class_num", + class_num, + "ignore_thresh", + ignore_thresh, + "downsample_ratio", + downsample_ratio, + "use_label_smooth", + use_label_smooth, + "scale_x_y", + scale_x_y, + ) + loss, _, _ = _legacy_C_ops.yolov3_loss( + x, gt_box, gt_label, gt_score, *attrs + ) return loss helper = LayerHelper('yolov3_loss', **locals()) @@ -1142,30 +1234,34 @@ def yolov3_loss(x, "scale_x_y": scale_x_y, } - helper.append_op(type='yolov3_loss', - inputs=inputs, - outputs={ - 'Loss': loss, - 'ObjectnessMask': objectness_mask, - 'GTMatchMask': gt_match_mask - }, - attrs=attrs) + helper.append_op( + type='yolov3_loss', + inputs=inputs, + outputs={ + 'Loss': loss, + 'ObjectnessMask': objectness_mask, + 'GTMatchMask': gt_match_mask, + }, + attrs=attrs, + ) return loss @deprecated(since="2.0.0", update_to="paddle.vision.ops.yolo_box") @templatedoc(op_type="yolo_box") -def yolo_box(x, - img_size, - anchors, - class_num, - conf_thresh, - downsample_ratio, - clip_bbox=True, - name=None, - scale_x_y=1., - iou_aware=False, - iou_aware_factor=0.5): +def yolo_box( + x, + img_size, + anchors, + class_num, + conf_thresh, + downsample_ratio, + clip_bbox=True, + name=None, + scale_x_y=1.0, + iou_aware=False, + iou_aware_factor=0.5, +): """ ${comment} @@ -1233,33 +1329,37 @@ def yolo_box(x, "clip_bbox": clip_bbox, "scale_x_y": scale_x_y, "iou_aware": iou_aware, - "iou_aware_factor": iou_aware_factor + "iou_aware_factor": iou_aware_factor, } - helper.append_op(type='yolo_box', - inputs={ - "X": x, - "ImgSize": img_size, - }, - outputs={ - 'Boxes': boxes, - 'Scores': scores, - }, - attrs=attrs) + helper.append_op( + type='yolo_box', + inputs={ + "X": x, + "ImgSize": img_size, + }, + outputs={ + 'Boxes': boxes, + 'Scores': scores, + }, + attrs=attrs, + ) return boxes, scores @templatedoc() -def detection_map(detect_res, - label, - class_num, - background_label=0, - overlap_threshold=0.3, - evaluate_difficult=True, - has_state=None, - input_states=None, - out_states=None, - ap_version='integral'): +def detection_map( + detect_res, + label, + class_num, + background_label=0, + overlap_threshold=0.3, + evaluate_difficult=True, + has_state=None, + input_states=None, + out_states=None, + ap_version='integral', +): """ ${comment} @@ -1307,45 +1407,49 @@ def __create_var(type): return helper.create_variable_for_type_inference(dtype=type) map_out = __create_var('float32') - accum_pos_count_out = out_states[ - 0] if out_states is not None else __create_var('int32') - accum_true_pos_out = out_states[ - 1] if out_states is not None else __create_var('float32') - accum_false_pos_out = out_states[ - 2] if out_states is not None else __create_var('float32') + accum_pos_count_out = ( + out_states[0] if out_states is not None else __create_var('int32') + ) + accum_true_pos_out = ( + out_states[1] if out_states is not None else __create_var('float32') + ) + accum_false_pos_out = ( + out_states[2] if out_states is not None else __create_var('float32') + ) pos_count = input_states[0] if input_states is not None else None true_pos = input_states[1] if input_states is not None else None false_pos = input_states[2] if input_states is not None else None - helper.append_op(type="detection_map", - inputs={ - 'Label': label, - 'DetectRes': detect_res, - 'HasState': has_state, - 'PosCount': pos_count, - 'TruePos': true_pos, - 'FalsePos': false_pos - }, - outputs={ - 'MAP': map_out, - 'AccumPosCount': accum_pos_count_out, - 'AccumTruePos': accum_true_pos_out, - 'AccumFalsePos': accum_false_pos_out - }, - attrs={ - 'overlap_threshold': overlap_threshold, - 'evaluate_difficult': evaluate_difficult, - 'ap_type': ap_version, - 'class_num': class_num, - }) + helper.append_op( + type="detection_map", + inputs={ + 'Label': label, + 'DetectRes': detect_res, + 'HasState': has_state, + 'PosCount': pos_count, + 'TruePos': true_pos, + 'FalsePos': false_pos, + }, + outputs={ + 'MAP': map_out, + 'AccumPosCount': accum_pos_count_out, + 'AccumTruePos': accum_true_pos_out, + 'AccumFalsePos': accum_false_pos_out, + }, + attrs={ + 'overlap_threshold': overlap_threshold, + 'evaluate_difficult': evaluate_difficult, + 'ap_type': ap_version, + 'class_num': class_num, + }, + ) return map_out -def bipartite_match(dist_matrix, - match_type=None, - dist_threshold=None, - name=None): +def bipartite_match( + dist_matrix, match_type=None, dist_threshold=None, name=None +): """ This operator implements a greedy bipartite matching algorithm, which is @@ -1417,25 +1521,30 @@ def bipartite_match(dist_matrix, helper = LayerHelper('bipartite_match', **locals()) match_indices = helper.create_variable_for_type_inference(dtype='int32') match_distance = helper.create_variable_for_type_inference( - dtype=dist_matrix.dtype) - helper.append_op(type='bipartite_match', - inputs={'DistMat': dist_matrix}, - attrs={ - 'match_type': match_type, - 'dist_threshold': dist_threshold, - }, - outputs={ - 'ColToRowMatchIndices': match_indices, - 'ColToRowMatchDist': match_distance - }) + dtype=dist_matrix.dtype + ) + helper.append_op( + type='bipartite_match', + inputs={'DistMat': dist_matrix}, + attrs={ + 'match_type': match_type, + 'dist_threshold': dist_threshold, + }, + outputs={ + 'ColToRowMatchIndices': match_indices, + 'ColToRowMatchDist': match_distance, + }, + ) return match_indices, match_distance -def target_assign(input, - matched_indices, - negative_indices=None, - mismatch_value=None, - name=None): +def target_assign( + input, + matched_indices, + negative_indices=None, + mismatch_value=None, + name=None, +): """ This operator can be, for given the target bounding boxes or labels, @@ -1524,36 +1633,37 @@ def target_assign(input, helper = LayerHelper('target_assign', **locals()) out = helper.create_variable_for_type_inference(dtype=input.dtype) out_weight = helper.create_variable_for_type_inference(dtype='float32') - helper.append_op(type='target_assign', - inputs={ - 'X': input, - 'MatchIndices': matched_indices, - 'NegIndices': negative_indices - }, - outputs={ - 'Out': out, - 'OutWeight': out_weight - }, - attrs={'mismatch_value': mismatch_value}) + helper.append_op( + type='target_assign', + inputs={ + 'X': input, + 'MatchIndices': matched_indices, + 'NegIndices': negative_indices, + }, + outputs={'Out': out, 'OutWeight': out_weight}, + attrs={'mismatch_value': mismatch_value}, + ) return out, out_weight -def ssd_loss(location, - confidence, - gt_box, - gt_label, - prior_box, - prior_box_var=None, - background_label=0, - overlap_threshold=0.5, - neg_pos_ratio=3.0, - neg_overlap=0.5, - loc_loss_weight=1.0, - conf_loss_weight=1.0, - match_type='per_prediction', - mining_type='max_negative', - normalize=True, - sample_size=None): +def ssd_loss( + location, + confidence, + gt_box, + gt_label, + prior_box, + prior_box_var=None, + background_label=0, + overlap_threshold=0.5, + neg_pos_ratio=3.0, + neg_overlap=0.5, + loc_loss_weight=1.0, + conf_loss_weight=1.0, + match_type='per_prediction', + mining_type='max_negative', + normalize=True, + sample_size=None, +): r""" :alias_main: paddle.nn.functional.ssd_loss :alias: paddle.nn.functional.ssd_loss,paddle.nn.functional.loss.ssd_loss @@ -1685,17 +1795,19 @@ def __reshape_to_2d(var): # 1.1 Compute IOU similarity between ground-truth boxes and prior boxes. iou = iou_similarity(x=gt_box, y=prior_box) # 1.2 Compute matched bounding box by bipartite matching algorithm. - matched_indices, matched_dist = bipartite_match(iou, match_type, - overlap_threshold) + matched_indices, matched_dist = bipartite_match( + iou, match_type, overlap_threshold + ) # 2. Compute confidence for mining hard examples # 2.1. Get the target label based on matched indices - gt_label = nn.reshape(x=gt_label, - shape=(len(gt_label.shape) - 1) * (0, ) + (-1, 1)) + gt_label = nn.reshape( + x=gt_label, shape=(len(gt_label.shape) - 1) * (0,) + (-1, 1) + ) gt_label.stop_gradient = True - target_label, _ = target_assign(gt_label, - matched_indices, - mismatch_value=background_label) + target_label, _ = target_assign( + gt_label, matched_indices, mismatch_value=background_label + ) # 2.2. Compute confidence loss. # Reshape confidence to 2D tensor. confidence = __reshape_to_2d(confidence) @@ -1708,47 +1820,54 @@ def __reshape_to_2d(var): actual_shape.stop_gradient = True # shape=(-1, 0) is set for compile-time, the correct shape is set by # actual_shape in runtime. - conf_loss = nn.reshape(x=conf_loss, - shape=(-1, 0), - actual_shape=actual_shape) + conf_loss = nn.reshape( + x=conf_loss, shape=(-1, 0), actual_shape=actual_shape + ) conf_loss.stop_gradient = True neg_indices = helper.create_variable_for_type_inference(dtype='int32') dtype = matched_indices.dtype updated_matched_indices = helper.create_variable_for_type_inference( - dtype=dtype) - helper.append_op(type='mine_hard_examples', - inputs={ - 'ClsLoss': conf_loss, - 'LocLoss': None, - 'MatchIndices': matched_indices, - 'MatchDist': matched_dist, - }, - outputs={ - 'NegIndices': neg_indices, - 'UpdatedMatchIndices': updated_matched_indices - }, - attrs={ - 'neg_pos_ratio': neg_pos_ratio, - 'neg_dist_threshold': neg_overlap, - 'mining_type': mining_type, - 'sample_size': sample_size, - }) + dtype=dtype + ) + helper.append_op( + type='mine_hard_examples', + inputs={ + 'ClsLoss': conf_loss, + 'LocLoss': None, + 'MatchIndices': matched_indices, + 'MatchDist': matched_dist, + }, + outputs={ + 'NegIndices': neg_indices, + 'UpdatedMatchIndices': updated_matched_indices, + }, + attrs={ + 'neg_pos_ratio': neg_pos_ratio, + 'neg_dist_threshold': neg_overlap, + 'mining_type': mining_type, + 'sample_size': sample_size, + }, + ) # 4. Assign classification and regression targets # 4.1. Encoded bbox according to the prior boxes. - encoded_bbox = box_coder(prior_box=prior_box, - prior_box_var=prior_box_var, - target_box=gt_box, - code_type='encode_center_size') + encoded_bbox = box_coder( + prior_box=prior_box, + prior_box_var=prior_box_var, + target_box=gt_box, + code_type='encode_center_size', + ) # 4.2. Assign regression targets target_bbox, target_loc_weight = target_assign( - encoded_bbox, updated_matched_indices, mismatch_value=background_label) + encoded_bbox, updated_matched_indices, mismatch_value=background_label + ) # 4.3. Assign classification targets target_label, target_conf_weight = target_assign( gt_label, updated_matched_indices, negative_indices=neg_indices, - mismatch_value=background_label) + mismatch_value=background_label, + ) # 5. Compute loss. # 5.1 Compute confidence loss. @@ -1794,7 +1913,7 @@ def prior_box( image, min_sizes, max_sizes=None, - aspect_ratios=[1.], + aspect_ratios=[1.0], variance=[0.1, 0.1, 0.2, 0.2], flip=False, clip=False, @@ -1840,66 +1959,66 @@ def prior_box( Tuple: A tuple with two Variable (boxes, variances) boxes(Variable): the output prior boxes of PriorBox. - 4-D tensor, the layout is [H, W, num_priors, 4]. + 4-D tensor, the layout is [H, W, num_priors, 4]. H is the height of input, W is the width of input, num_priors is the total box count of each position of input. variances(Variable): the expanded variances of PriorBox. - 4-D tensor, the layput is [H, W, num_priors, 4]. + 4-D tensor, the layput is [H, W, num_priors, 4]. H is the height of input, W is the width of input num_priors is the total box count of each position of input Examples: .. code-block:: python - #declarative mode - import paddle.fluid as fluid - import numpy as np + #declarative mode + import paddle.fluid as fluid + import numpy as np import paddle paddle.enable_static() - input = fluid.data(name="input", shape=[None,3,6,9]) - image = fluid.data(name="image", shape=[None,3,9,12]) - box, var = fluid.layers.prior_box( + input = fluid.data(name="input", shape=[None,3,6,9]) + image = fluid.data(name="image", shape=[None,3,9,12]) + box, var = fluid.layers.prior_box( input=input, image=image, - min_sizes=[100.], + min_sizes=[100.], clip=True, flip=True) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) - # prepare a batch of data - input_data = np.random.rand(1,3,6,9).astype("float32") - image_data = np.random.rand(1,3,9,12).astype("float32") + # prepare a batch of data + input_data = np.random.rand(1,3,6,9).astype("float32") + image_data = np.random.rand(1,3,9,12).astype("float32") - box_out, var_out = exe.run(fluid.default_main_program(), + box_out, var_out = exe.run(fluid.default_main_program(), feed={"input":input_data,"image":image_data}, fetch_list=[box,var], return_numpy=True) - # print(box_out.shape) - # (6, 9, 1, 4) - # print(var_out.shape) - # (6, 9, 1, 4) - - # imperative mode - import paddle.fluid.dygraph as dg - - with dg.guard(place) as g: - input = dg.to_variable(input_data) - image = dg.to_variable(image_data) - box, var = fluid.layers.prior_box( - input=input, - image=image, - min_sizes=[100.], - clip=True, - flip=True) - # print(box.shape) - # [6L, 9L, 1L, 4L] + # print(box_out.shape) + # (6, 9, 1, 4) + # print(var_out.shape) + # (6, 9, 1, 4) + + # imperative mode + import paddle.fluid.dygraph as dg + + with dg.guard(place) as g: + input = dg.to_variable(input_data) + image = dg.to_variable(image_data) + box, var = fluid.layers.prior_box( + input=input, + image=image, + min_sizes=[100.], + clip=True, + flip=True) + # print(box.shape) + # [6L, 9L, 1L, 4L] # print(var.shape) - # [6L, 9L, 1L, 4L] + # [6L, 9L, 1L, 4L] """ @@ -1907,25 +2026,38 @@ def prior_box( step_w, step_h = steps if max_sizes == None: max_sizes = [] - return _C_ops.prior_box(input, image, min_sizes, aspect_ratios, - variance, max_sizes, flip, clip, step_w, step_h, - offset, min_max_aspect_ratios_order) + return _C_ops.prior_box( + input, + image, + min_sizes, + aspect_ratios, + variance, + max_sizes, + flip, + clip, + step_w, + step_h, + offset, + min_max_aspect_ratios_order, + ) helper = LayerHelper("prior_box", **locals()) dtype = helper.input_dtype() - check_variable_and_dtype(input, 'input', - ['uint8', 'int8', 'float32', 'float64'], - 'prior_box') + check_variable_and_dtype( + input, 'input', ['uint8', 'int8', 'float32', 'float64'], 'prior_box' + ) def _is_list_or_tuple_(data): - return (isinstance(data, list) or isinstance(data, tuple)) + return isinstance(data, list) or isinstance(data, tuple) if not _is_list_or_tuple_(min_sizes): min_sizes = [min_sizes] if not _is_list_or_tuple_(aspect_ratios): aspect_ratios = [aspect_ratios] if not (_is_list_or_tuple_(steps) and len(steps) == 2): - raise ValueError('steps should be a list or tuple ', - 'with length 2, (step_width, step_height).') + raise ValueError( + 'steps should be a list or tuple ', + 'with length 2, (step_width, step_height).', + ) min_sizes = list(map(float, min_sizes)) aspect_ratios = list(map(float, aspect_ratios)) @@ -1940,7 +2072,7 @@ def _is_list_or_tuple_(data): 'step_w': steps[0], 'step_h': steps[1], 'offset': offset, - 'min_max_aspect_ratios_order': min_max_aspect_ratios_order + 'min_max_aspect_ratios_order': min_max_aspect_ratios_order, } if max_sizes is not None and len(max_sizes) > 0 and max_sizes[0] > 0: if not _is_list_or_tuple_(max_sizes): @@ -1951,14 +2083,8 @@ def _is_list_or_tuple_(data): var = helper.create_variable_for_type_inference(dtype) helper.append_op( type="prior_box", - inputs={ - "Input": input, - "Image": image - }, - outputs={ - "Boxes": box, - "Variances": var - }, + inputs={"Input": input, "Image": image}, + outputs={"Boxes": box, "Variances": var}, attrs=attrs, ) box.stop_gradient = True @@ -1966,17 +2092,19 @@ def _is_list_or_tuple_(data): return box, var -def density_prior_box(input, - image, - densities=None, - fixed_sizes=None, - fixed_ratios=None, - variance=[0.1, 0.1, 0.2, 0.2], - clip=False, - steps=[0.0, 0.0], - offset=0.5, - flatten_to_2d=False, - name=None): +def density_prior_box( + input, + image, + densities=None, + fixed_sizes=None, + fixed_ratios=None, + variance=[0.1, 0.1, 0.2, 0.2], + clip=False, + steps=[0.0, 0.0], + offset=0.5, + flatten_to_2d=False, + name=None, +): r""" This op generates density prior boxes for SSD(Single Shot MultiBox Detector) @@ -2100,11 +2228,12 @@ def density_prior_box(input, """ helper = LayerHelper("density_prior_box", **locals()) dtype = helper.input_dtype() - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'density_prior_box') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'density_prior_box' + ) def _is_list_or_tuple_(data): - return (isinstance(data, list) or isinstance(data, tuple)) + return isinstance(data, list) or isinstance(data, tuple) check_type(densities, 'densities', (list, tuple), 'density_prior_box') check_type(fixed_sizes, 'fixed_sizes', (list, tuple), 'density_prior_box') @@ -2113,8 +2242,10 @@ def _is_list_or_tuple_(data): raise ValueError('densities and fixed_sizes length should be euqal.') if not (_is_list_or_tuple_(steps) and len(steps) == 2): - raise ValueError('steps should be a list or tuple ', - 'with length 2, (step_width, step_height).') + raise ValueError( + 'steps should be a list or tuple ', + 'with length 2, (step_width, step_height).', + ) densities = list(map(int, densities)) fixed_sizes = list(map(float, fixed_sizes)) @@ -2136,14 +2267,8 @@ def _is_list_or_tuple_(data): var = helper.create_variable_for_type_inference(dtype) helper.append_op( type="density_prior_box", - inputs={ - "Input": input, - "Image": image - }, - outputs={ - "Boxes": box, - "Variances": var - }, + inputs={"Input": input, "Image": image}, + outputs={"Boxes": box, "Variances": var}, attrs=attrs, ) box.stop_gradient = True @@ -2152,29 +2277,31 @@ def _is_list_or_tuple_(data): @static_only -def multi_box_head(inputs, - image, - base_size, - num_classes, - aspect_ratios, - min_ratio=None, - max_ratio=None, - min_sizes=None, - max_sizes=None, - steps=None, - step_w=None, - step_h=None, - offset=0.5, - variance=[0.1, 0.1, 0.2, 0.2], - flip=True, - clip=False, - kernel_size=1, - pad=0, - stride=1, - name=None, - min_max_aspect_ratios_order=False): +def multi_box_head( + inputs, + image, + base_size, + num_classes, + aspect_ratios, + min_ratio=None, + max_ratio=None, + min_sizes=None, + max_sizes=None, + steps=None, + step_w=None, + step_h=None, + offset=0.5, + variance=[0.1, 0.1, 0.2, 0.2], + flip=True, + clip=False, + kernel_size=1, + pad=0, + stride=1, + name=None, + min_max_aspect_ratios_order=False, +): """ - :api_attr: Static Graph + :api_attr: Static Graph Base on SSD ((Single Shot MultiBox Detector) algorithm, generate prior boxes, regression location and classification confidence on multiple input feature @@ -2319,7 +2446,7 @@ def _reshape_with_axis_(input, axis=1): return out def _is_list_or_tuple_(data): - return (isinstance(data, list) or isinstance(data, tuple)) + return isinstance(data, list) or isinstance(data, tuple) def _is_list_or_tuple_and_equal(data, length, err_info): if not (_is_list_or_tuple_(data) and len(data) == length): @@ -2338,31 +2465,39 @@ def _is_list_or_tuple_and_equal(data, length, err_info): max_sizes = [] step = int(math.floor(((max_ratio - min_ratio)) / (num_layer - 2))) for ratio in range(min_ratio, max_ratio + 1, step): - min_sizes.append(base_size * ratio / 100.) - max_sizes.append(base_size * (ratio + step) / 100.) - min_sizes = [base_size * .10] + min_sizes - max_sizes = [base_size * .20] + max_sizes + min_sizes.append(base_size * ratio / 100.0) + max_sizes.append(base_size * (ratio + step) / 100.0) + min_sizes = [base_size * 0.10] + min_sizes + max_sizes = [base_size * 0.20] + max_sizes if aspect_ratios: _is_list_or_tuple_and_equal( - aspect_ratios, num_layer, + aspect_ratios, + num_layer, 'aspect_ratios should be list or tuple, and the length of inputs ' - 'and aspect_ratios should be the same.') + 'and aspect_ratios should be the same.', + ) if step_h is not None: _is_list_or_tuple_and_equal( - step_h, num_layer, + step_h, + num_layer, 'step_h should be list or tuple, and the length of inputs and ' - 'step_h should be the same.') + 'step_h should be the same.', + ) if step_w is not None: _is_list_or_tuple_and_equal( - step_w, num_layer, + step_w, + num_layer, 'step_w should be list or tuple, and the length of inputs and ' - 'step_w should be the same.') + 'step_w should be the same.', + ) if steps is not None: _is_list_or_tuple_and_equal( - steps, num_layer, + steps, + num_layer, 'steps should be list or tuple, and the length of inputs and ' - 'step_w should be the same.') + 'step_w should be the same.', + ) step_w = steps step_h = steps @@ -2386,9 +2521,20 @@ def _is_list_or_tuple_and_equal(data, length, err_info): aspect_ratio = [aspect_ratio] step = [step_w[i] if step_w else 0.0, step_h[i] if step_w else 0.0] - box, var = prior_box(input, image, min_size, max_size, aspect_ratio, - variance, flip, clip, step, offset, None, - min_max_aspect_ratios_order) + box, var = prior_box( + input, + image, + min_size, + max_size, + aspect_ratio, + variance, + flip, + clip, + step, + offset, + None, + min_max_aspect_ratios_order, + ) box_results.append(box) var_results.append(var) @@ -2397,11 +2543,13 @@ def _is_list_or_tuple_and_equal(data, length, err_info): # get loc num_loc_output = num_boxes * 4 - mbox_loc = nn.conv2d(input=input, - num_filters=num_loc_output, - filter_size=kernel_size, - padding=pad, - stride=stride) + mbox_loc = nn.conv2d( + input=input, + num_filters=num_loc_output, + filter_size=kernel_size, + padding=pad, + stride=stride, + ) mbox_loc = nn.transpose(mbox_loc, perm=[0, 2, 3, 1]) mbox_loc_flatten = nn.flatten(mbox_loc, axis=1) @@ -2409,11 +2557,13 @@ def _is_list_or_tuple_and_equal(data, length, err_info): # get conf num_conf_output = num_boxes * num_classes - conf_loc = nn.conv2d(input=input, - num_filters=num_conf_output, - filter_size=kernel_size, - padding=pad, - stride=stride) + conf_loc = nn.conv2d( + input=input, + num_filters=num_conf_output, + filter_size=kernel_size, + padding=pad, + stride=stride, + ) conf_loc = nn.transpose(conf_loc, perm=[0, 2, 3, 1]) conf_loc_flatten = nn.flatten(conf_loc, axis=1) mbox_confs.append(conf_loc_flatten) @@ -2435,21 +2585,24 @@ def _is_list_or_tuple_and_equal(data, length, err_info): mbox_locs_concat = tensor.concat(mbox_locs, axis=1) mbox_locs_concat = nn.reshape(mbox_locs_concat, shape=[0, -1, 4]) mbox_confs_concat = tensor.concat(mbox_confs, axis=1) - mbox_confs_concat = nn.reshape(mbox_confs_concat, - shape=[0, -1, num_classes]) + mbox_confs_concat = nn.reshape( + mbox_confs_concat, shape=[0, -1, num_classes] + ) box.stop_gradient = True var.stop_gradient = True return mbox_locs_concat, mbox_confs_concat, box, var -def anchor_generator(input, - anchor_sizes=None, - aspect_ratios=None, - variance=[0.1, 0.1, 0.2, 0.2], - stride=None, - offset=0.5, - name=None): +def anchor_generator( + input, + anchor_sizes=None, + aspect_ratios=None, + variance=[0.1, 0.1, 0.2, 0.2], + stride=None, + offset=0.5, + name=None, +): """ **Anchor generator operator** @@ -2513,15 +2666,17 @@ def anchor_generator(input, dtype = helper.input_dtype() def _is_list_or_tuple_(data): - return (isinstance(data, list) or isinstance(data, tuple)) + return isinstance(data, list) or isinstance(data, tuple) if not _is_list_or_tuple_(anchor_sizes): anchor_sizes = [anchor_sizes] if not _is_list_or_tuple_(aspect_ratios): aspect_ratios = [aspect_ratios] if not (_is_list_or_tuple_(stride) and len(stride) == 2): - raise ValueError('stride should be a list or tuple ', - 'with length 2, (stride_width, stride_height).') + raise ValueError( + 'stride should be a list or tuple ', + 'with length 2, (stride_width, stride_height).', + ) anchor_sizes = list(map(float, anchor_sizes)) aspect_ratios = list(map(float, aspect_ratios)) @@ -2532,7 +2687,7 @@ def _is_list_or_tuple_(data): 'aspect_ratios': aspect_ratios, 'variances': variance, 'stride': stride, - 'offset': offset + 'offset': offset, } anchor = helper.create_variable_for_type_inference(dtype) @@ -2540,10 +2695,7 @@ def _is_list_or_tuple_(data): helper.append_op( type="anchor_generator", inputs={"Input": input}, - outputs={ - "Anchors": anchor, - "Variances": var - }, + outputs={"Anchors": anchor, "Variances": var}, attrs=attrs, ) anchor.stop_gradient = True @@ -2551,12 +2703,14 @@ def _is_list_or_tuple_(data): return anchor, var -def roi_perspective_transform(input, - rois, - transformed_height, - transformed_width, - spatial_scale=1.0, - name=None): +def roi_perspective_transform( + input, + rois, + transformed_height, + transformed_width, + spatial_scale=1.0, + name=None, +): """ **The** `rois` **of this op should be a LoDTensor.** @@ -2606,16 +2760,24 @@ def roi_perspective_transform(input, rois = fluid.data(name='rois', shape=[None, 8], lod_level=1, dtype='float32') out, mask, transform_matrix = fluid.layers.roi_perspective_transform(x, rois, 7, 7, 1.0) """ - check_variable_and_dtype(input, 'input', ['float32'], - 'roi_perspective_transform') - check_variable_and_dtype(rois, 'rois', ['float32'], - 'roi_perspective_transform') - check_type(transformed_height, 'transformed_height', int, - 'roi_perspective_transform') - check_type(transformed_width, 'transformed_width', int, - 'roi_perspective_transform') - check_type(spatial_scale, 'spatial_scale', float, - 'roi_perspective_transform') + check_variable_and_dtype( + input, 'input', ['float32'], 'roi_perspective_transform' + ) + check_variable_and_dtype( + rois, 'rois', ['float32'], 'roi_perspective_transform' + ) + check_type( + transformed_height, + 'transformed_height', + int, + 'roi_perspective_transform', + ) + check_type( + transformed_width, 'transformed_width', int, 'roi_perspective_transform' + ) + check_type( + spatial_scale, 'spatial_scale', float, 'roi_perspective_transform' + ) helper = LayerHelper('roi_perspective_transform', **locals()) dtype = helper.input_dtype() @@ -2624,43 +2786,44 @@ def roi_perspective_transform(input, transform_matrix = helper.create_variable_for_type_inference(dtype) out2in_idx = helper.create_variable_for_type_inference(dtype="int32") out2in_w = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="roi_perspective_transform", - inputs={ - "X": input, - "ROIs": rois - }, - outputs={ - "Out": out, - "Out2InIdx": out2in_idx, - "Out2InWeights": out2in_w, - "Mask": mask, - "TransformMatrix": transform_matrix - }, - attrs={ - "transformed_height": transformed_height, - "transformed_width": transformed_width, - "spatial_scale": spatial_scale - }) + helper.append_op( + type="roi_perspective_transform", + inputs={"X": input, "ROIs": rois}, + outputs={ + "Out": out, + "Out2InIdx": out2in_idx, + "Out2InWeights": out2in_w, + "Mask": mask, + "TransformMatrix": transform_matrix, + }, + attrs={ + "transformed_height": transformed_height, + "transformed_width": transformed_width, + "spatial_scale": spatial_scale, + }, + ) return out, mask, transform_matrix -def generate_proposal_labels(rpn_rois, - gt_classes, - is_crowd, - gt_boxes, - im_info, - batch_size_per_im=256, - fg_fraction=0.25, - fg_thresh=0.25, - bg_thresh_hi=0.5, - bg_thresh_lo=0.0, - bbox_reg_weights=[0.1, 0.1, 0.2, 0.2], - class_nums=None, - use_random=True, - is_cls_agnostic=False, - is_cascade_rcnn=False, - max_overlap=None, - return_max_overlap=False): +def generate_proposal_labels( + rpn_rois, + gt_classes, + is_crowd, + gt_boxes, + im_info, + batch_size_per_im=256, + fg_fraction=0.25, + fg_thresh=0.25, + bg_thresh_hi=0.5, + bg_thresh_lo=0.0, + bbox_reg_weights=[0.1, 0.1, 0.2, 0.2], + class_nums=None, + use_random=True, + is_cls_agnostic=False, + is_cascade_rcnn=False, + max_overlap=None, + return_max_overlap=False, +): """ **Generate Proposal Labels of Faster-RCNN** @@ -2730,26 +2893,36 @@ def generate_proposal_labels(rpn_rois, helper = LayerHelper('generate_proposal_labels', **locals()) - check_variable_and_dtype(rpn_rois, 'rpn_rois', ['float32', 'float64'], - 'generate_proposal_labels') - check_variable_and_dtype(gt_classes, 'gt_classes', ['int32'], - 'generate_proposal_labels') - check_variable_and_dtype(is_crowd, 'is_crowd', ['int32'], - 'generate_proposal_labels') + check_variable_and_dtype( + rpn_rois, 'rpn_rois', ['float32', 'float64'], 'generate_proposal_labels' + ) + check_variable_and_dtype( + gt_classes, 'gt_classes', ['int32'], 'generate_proposal_labels' + ) + check_variable_and_dtype( + is_crowd, 'is_crowd', ['int32'], 'generate_proposal_labels' + ) if is_cascade_rcnn: - assert max_overlap is not None, "Input max_overlap of generate_proposal_labels should not be None if is_cascade_rcnn is True" + assert ( + max_overlap is not None + ), "Input max_overlap of generate_proposal_labels should not be None if is_cascade_rcnn is True" rois = helper.create_variable_for_type_inference(dtype=rpn_rois.dtype) labels_int32 = helper.create_variable_for_type_inference( - dtype=gt_classes.dtype) + dtype=gt_classes.dtype + ) bbox_targets = helper.create_variable_for_type_inference( - dtype=rpn_rois.dtype) + dtype=rpn_rois.dtype + ) bbox_inside_weights = helper.create_variable_for_type_inference( - dtype=rpn_rois.dtype) + dtype=rpn_rois.dtype + ) bbox_outside_weights = helper.create_variable_for_type_inference( - dtype=rpn_rois.dtype) + dtype=rpn_rois.dtype + ) max_overlap_with_gt = helper.create_variable_for_type_inference( - dtype=rpn_rois.dtype) + dtype=rpn_rois.dtype + ) inputs = { 'RpnRois': rpn_rois, @@ -2760,28 +2933,30 @@ def generate_proposal_labels(rpn_rois, } if max_overlap is not None: inputs['MaxOverlap'] = max_overlap - helper.append_op(type="generate_proposal_labels", - inputs=inputs, - outputs={ - 'Rois': rois, - 'LabelsInt32': labels_int32, - 'BboxTargets': bbox_targets, - 'BboxInsideWeights': bbox_inside_weights, - 'BboxOutsideWeights': bbox_outside_weights, - 'MaxOverlapWithGT': max_overlap_with_gt - }, - attrs={ - 'batch_size_per_im': batch_size_per_im, - 'fg_fraction': fg_fraction, - 'fg_thresh': fg_thresh, - 'bg_thresh_hi': bg_thresh_hi, - 'bg_thresh_lo': bg_thresh_lo, - 'bbox_reg_weights': bbox_reg_weights, - 'class_nums': class_nums, - 'use_random': use_random, - 'is_cls_agnostic': is_cls_agnostic, - 'is_cascade_rcnn': is_cascade_rcnn - }) + helper.append_op( + type="generate_proposal_labels", + inputs=inputs, + outputs={ + 'Rois': rois, + 'LabelsInt32': labels_int32, + 'BboxTargets': bbox_targets, + 'BboxInsideWeights': bbox_inside_weights, + 'BboxOutsideWeights': bbox_outside_weights, + 'MaxOverlapWithGT': max_overlap_with_gt, + }, + attrs={ + 'batch_size_per_im': batch_size_per_im, + 'fg_fraction': fg_fraction, + 'fg_thresh': fg_thresh, + 'bg_thresh_hi': bg_thresh_hi, + 'bg_thresh_lo': bg_thresh_lo, + 'bbox_reg_weights': bbox_reg_weights, + 'class_nums': class_nums, + 'use_random': use_random, + 'is_cls_agnostic': is_cls_agnostic, + 'is_cascade_rcnn': is_cascade_rcnn, + }, + ) rois.stop_gradient = True labels_int32.stop_gradient = True @@ -2791,12 +2966,33 @@ def generate_proposal_labels(rpn_rois, max_overlap_with_gt.stop_gradient = True if return_max_overlap: - return rois, labels_int32, bbox_targets, bbox_inside_weights, bbox_outside_weights, max_overlap_with_gt - return rois, labels_int32, bbox_targets, bbox_inside_weights, bbox_outside_weights + return ( + rois, + labels_int32, + bbox_targets, + bbox_inside_weights, + bbox_outside_weights, + max_overlap_with_gt, + ) + return ( + rois, + labels_int32, + bbox_targets, + bbox_inside_weights, + bbox_outside_weights, + ) -def generate_mask_labels(im_info, gt_classes, is_crowd, gt_segms, rois, - labels_int32, num_classes, resolution): +def generate_mask_labels( + im_info, + gt_classes, + is_crowd, + gt_segms, + rois, + labels_int32, + num_classes, + resolution, +): r""" **Generate Mask Labels for Mask-RCNN** @@ -2912,28 +3108,29 @@ def generate_mask_labels(im_info, gt_classes, is_crowd, gt_segms, rois, mask_rois = helper.create_variable_for_type_inference(dtype=rois.dtype) roi_has_mask_int32 = helper.create_variable_for_type_inference( - dtype=gt_classes.dtype) + dtype=gt_classes.dtype + ) mask_int32 = helper.create_variable_for_type_inference( - dtype=gt_classes.dtype) - - helper.append_op(type="generate_mask_labels", - inputs={ - 'ImInfo': im_info, - 'GtClasses': gt_classes, - 'IsCrowd': is_crowd, - 'GtSegms': gt_segms, - 'Rois': rois, - 'LabelsInt32': labels_int32 - }, - outputs={ - 'MaskRois': mask_rois, - 'RoiHasMaskInt32': roi_has_mask_int32, - 'MaskInt32': mask_int32 - }, - attrs={ - 'num_classes': num_classes, - 'resolution': resolution - }) + dtype=gt_classes.dtype + ) + + helper.append_op( + type="generate_mask_labels", + inputs={ + 'ImInfo': im_info, + 'GtClasses': gt_classes, + 'IsCrowd': is_crowd, + 'GtSegms': gt_segms, + 'Rois': rois, + 'LabelsInt32': labels_int32, + }, + outputs={ + 'MaskRois': mask_rois, + 'RoiHasMaskInt32': roi_has_mask_int32, + 'MaskInt32': mask_int32, + }, + attrs={'num_classes': num_classes, 'resolution': resolution}, + ) mask_rois.stop_gradient = True roi_has_mask_int32.stop_gradient = True @@ -2942,18 +3139,20 @@ def generate_mask_labels(im_info, gt_classes, is_crowd, gt_segms, rois, return mask_rois, roi_has_mask_int32, mask_int32 -def generate_proposals(scores, - bbox_deltas, - im_info, - anchors, - variances, - pre_nms_top_n=6000, - post_nms_top_n=1000, - nms_thresh=0.5, - min_size=0.1, - eta=1.0, - return_rois_num=False, - name=None): +def generate_proposals( + scores, + bbox_deltas, + im_info, + anchors, + variances, + pre_nms_top_n=6000, + post_nms_top_n=1000, + nms_thresh=0.5, + min_size=0.1, + eta=1.0, + return_rois_num=False, + name=None, +): """ **Generate proposal Faster-RCNN** @@ -3031,18 +3230,20 @@ def generate_proposals(scores, im_info, anchors, variances) """ - return paddle.vision.ops.generate_proposals(scores=scores, - bbox_deltas=bbox_deltas, - img_size=im_info[:2], - anchors=anchors, - variances=variances, - pre_nms_top_n=pre_nms_top_n, - post_nms_top_n=post_nms_top_n, - nms_thresh=nms_thresh, - min_size=min_size, - eta=eta, - return_rois_num=return_rois_num, - name=name) + return paddle.vision.ops.generate_proposals( + scores=scores, + bbox_deltas=bbox_deltas, + img_size=im_info[:2], + anchors=anchors, + variances=variances, + pre_nms_top_n=pre_nms_top_n, + post_nms_top_n=post_nms_top_n, + nms_thresh=nms_thresh, + min_size=min_size, + eta=eta, + return_rois_num=return_rois_num, + name=name, + ) def box_clip(input, im_info, name=None): @@ -3097,8 +3298,9 @@ def box_clip(input, im_info, name=None): """ check_variable_and_dtype(input, 'input', ['float32', 'float64'], 'box_clip') - check_variable_and_dtype(im_info, 'im_info', ['float32', 'float64'], - 'box_clip') + check_variable_and_dtype( + im_info, 'im_info', ['float32', 'float64'], 'box_clip' + ) helper = LayerHelper("box_clip", **locals()) output = helper.create_variable_for_type_inference(dtype=input.dtype) @@ -3108,15 +3310,17 @@ def box_clip(input, im_info, name=None): return output -def retinanet_detection_output(bboxes, - scores, - anchors, - im_info, - score_threshold=0.05, - nms_top_k=1000, - keep_top_k=100, - nms_threshold=0.3, - nms_eta=1.0): +def retinanet_detection_output( + bboxes, + scores, + anchors, + im_info, + score_threshold=0.05, + nms_top_k=1000, + keep_top_k=100, + nms_threshold=0.3, + nms_eta=1.0, +): """ **Detection Output Layer for the detector RetinaNet.** @@ -3225,54 +3429,69 @@ def retinanet_detection_output(bboxes, check_type(bboxes, 'bboxes', (list), 'retinanet_detection_output') for i, bbox in enumerate(bboxes): - check_variable_and_dtype(bbox, 'bbox{}'.format(i), - ['float32', 'float64'], - 'retinanet_detection_output') + check_variable_and_dtype( + bbox, + 'bbox{}'.format(i), + ['float32', 'float64'], + 'retinanet_detection_output', + ) check_type(scores, 'scores', (list), 'retinanet_detection_output') for i, score in enumerate(scores): - check_variable_and_dtype(score, 'score{}'.format(i), - ['float32', 'float64'], - 'retinanet_detection_output') + check_variable_and_dtype( + score, + 'score{}'.format(i), + ['float32', 'float64'], + 'retinanet_detection_output', + ) check_type(anchors, 'anchors', (list), 'retinanet_detection_output') for i, anchor in enumerate(anchors): - check_variable_and_dtype(anchor, 'anchor{}'.format(i), - ['float32', 'float64'], - 'retinanet_detection_output') - check_variable_and_dtype(im_info, 'im_info', ['float32', 'float64'], - 'retinanet_detection_output') + check_variable_and_dtype( + anchor, + 'anchor{}'.format(i), + ['float32', 'float64'], + 'retinanet_detection_output', + ) + check_variable_and_dtype( + im_info, 'im_info', ['float32', 'float64'], 'retinanet_detection_output' + ) helper = LayerHelper('retinanet_detection_output', **locals()) output = helper.create_variable_for_type_inference( - dtype=helper.input_dtype('scores')) - helper.append_op(type="retinanet_detection_output", - inputs={ - 'BBoxes': bboxes, - 'Scores': scores, - 'Anchors': anchors, - 'ImInfo': im_info - }, - attrs={ - 'score_threshold': score_threshold, - 'nms_top_k': nms_top_k, - 'nms_threshold': nms_threshold, - 'keep_top_k': keep_top_k, - 'nms_eta': 1., - }, - outputs={'Out': output}) + dtype=helper.input_dtype('scores') + ) + helper.append_op( + type="retinanet_detection_output", + inputs={ + 'BBoxes': bboxes, + 'Scores': scores, + 'Anchors': anchors, + 'ImInfo': im_info, + }, + attrs={ + 'score_threshold': score_threshold, + 'nms_top_k': nms_top_k, + 'nms_threshold': nms_threshold, + 'keep_top_k': keep_top_k, + 'nms_eta': 1.0, + }, + outputs={'Out': output}, + ) output.stop_gradient = True return output -def multiclass_nms(bboxes, - scores, - score_threshold, - nms_top_k, - keep_top_k, - nms_threshold=0.3, - normalized=True, - nms_eta=1., - background_label=0, - name=None): +def multiclass_nms( + bboxes, + scores, + score_threshold, + nms_top_k, + keep_top_k, + nms_threshold=0.3, + normalized=True, + nms_eta=1.0, + background_label=0, + name=None, +): """ **Multiclass NMS** @@ -3384,10 +3603,12 @@ class number. The data type is float32 or float64. keep_top_k=200, normalized=False) """ - check_variable_and_dtype(bboxes, 'BBoxes', ['float32', 'float64'], - 'multiclass_nms') - check_variable_and_dtype(scores, 'Scores', ['float32', 'float64'], - 'multiclass_nms') + check_variable_and_dtype( + bboxes, 'BBoxes', ['float32', 'float64'], 'multiclass_nms' + ) + check_variable_and_dtype( + scores, 'Scores', ['float32', 'float64'], 'multiclass_nms' + ) check_type(score_threshold, 'score_threshold', float, 'multicalss_nms') check_type(nms_top_k, 'nums_top_k', int, 'multiclass_nms') check_type(keep_top_k, 'keep_top_k', int, 'mutliclass_nms') @@ -3398,36 +3619,37 @@ class number. The data type is float32 or float64. helper = LayerHelper('multiclass_nms', **locals()) output = helper.create_variable_for_type_inference(dtype=bboxes.dtype) - helper.append_op(type="multiclass_nms", - inputs={ - 'BBoxes': bboxes, - 'Scores': scores - }, - attrs={ - 'background_label': background_label, - 'score_threshold': score_threshold, - 'nms_top_k': nms_top_k, - 'nms_threshold': nms_threshold, - 'nms_eta': nms_eta, - 'keep_top_k': keep_top_k, - 'normalized': normalized - }, - outputs={'Out': output}) + helper.append_op( + type="multiclass_nms", + inputs={'BBoxes': bboxes, 'Scores': scores}, + attrs={ + 'background_label': background_label, + 'score_threshold': score_threshold, + 'nms_top_k': nms_top_k, + 'nms_threshold': nms_threshold, + 'nms_eta': nms_eta, + 'keep_top_k': keep_top_k, + 'normalized': normalized, + }, + outputs={'Out': output}, + ) output.stop_gradient = True return output -def locality_aware_nms(bboxes, - scores, - score_threshold, - nms_top_k, - keep_top_k, - nms_threshold=0.3, - normalized=True, - nms_eta=1., - background_label=-1, - name=None): +def locality_aware_nms( + bboxes, + scores, + score_threshold, + nms_top_k, + keep_top_k, + nms_threshold=0.3, + normalized=True, + nms_eta=1.0, + background_label=-1, + name=None, +): """ **Local Aware NMS** @@ -3506,10 +3728,12 @@ def locality_aware_nms(bboxes, keep_top_k=200, normalized=False) """ - check_variable_and_dtype(bboxes, 'bboxes', ['float32', 'float64'], - 'locality_aware_nms') - check_variable_and_dtype(scores, 'scores', ['float32', 'float64'], - 'locality_aware_nms') + check_variable_and_dtype( + bboxes, 'bboxes', ['float32', 'float64'], 'locality_aware_nms' + ) + check_variable_and_dtype( + scores, 'scores', ['float32', 'float64'], 'locality_aware_nms' + ) check_type(background_label, 'background_label', int, 'locality_aware_nms') check_type(score_threshold, 'score_threshold', float, 'locality_aware_nms') check_type(nms_top_k, 'nms_top_k', int, 'locality_aware_nms') @@ -3520,47 +3744,49 @@ def locality_aware_nms(bboxes, shape = scores.shape assert len(shape) == 3, "dim size of scores must be 3" - assert shape[ - 1] == 1, "locality_aware_nms only support one class, Tensor score shape must be [N, 1, M]" + assert ( + shape[1] == 1 + ), "locality_aware_nms only support one class, Tensor score shape must be [N, 1, M]" helper = LayerHelper('locality_aware_nms', **locals()) output = helper.create_variable_for_type_inference(dtype=bboxes.dtype) out = {'Out': output} - helper.append_op(type="locality_aware_nms", - inputs={ - 'BBoxes': bboxes, - 'Scores': scores - }, - attrs={ - 'background_label': background_label, - 'score_threshold': score_threshold, - 'nms_top_k': nms_top_k, - 'nms_threshold': nms_threshold, - 'nms_eta': nms_eta, - 'keep_top_k': keep_top_k, - 'nms_eta': nms_eta, - 'normalized': normalized - }, - outputs={'Out': output}) + helper.append_op( + type="locality_aware_nms", + inputs={'BBoxes': bboxes, 'Scores': scores}, + attrs={ + 'background_label': background_label, + 'score_threshold': score_threshold, + 'nms_top_k': nms_top_k, + 'nms_threshold': nms_threshold, + 'nms_eta': nms_eta, + 'keep_top_k': keep_top_k, + 'nms_eta': nms_eta, + 'normalized': normalized, + }, + outputs={'Out': output}, + ) output.stop_gradient = True return output -def matrix_nms(bboxes, - scores, - score_threshold, - post_threshold, - nms_top_k, - keep_top_k, - use_gaussian=False, - gaussian_sigma=2., - background_label=0, - normalized=True, - return_index=False, - name=None): +def matrix_nms( + bboxes, + scores, + score_threshold, + post_threshold, + nms_top_k, + keep_top_k, + use_gaussian=False, + gaussian_sigma=2.0, + background_label=0, + normalized=True, + return_index=False, + name=None, +): """ **Matrix NMS** @@ -3637,8 +3863,16 @@ def matrix_nms(bboxes, normalized=False) """ if in_dygraph_mode(): - attrs = (score_threshold, nms_top_k, keep_top_k, post_threshold, - use_gaussian, gaussian_sigma, background_label, normalized) + attrs = ( + score_threshold, + nms_top_k, + keep_top_k, + post_threshold, + use_gaussian, + gaussian_sigma, + background_label, + normalized, + ) out, index = _C_ops.matrix_nms(bboxes, scores, *attrs) if return_index: @@ -3646,10 +3880,12 @@ def matrix_nms(bboxes, else: return out - check_variable_and_dtype(bboxes, 'BBoxes', ['float32', 'float64'], - 'matrix_nms') - check_variable_and_dtype(scores, 'Scores', ['float32', 'float64'], - 'matrix_nms') + check_variable_and_dtype( + bboxes, 'BBoxes', ['float32', 'float64'], 'matrix_nms' + ) + check_variable_and_dtype( + scores, 'Scores', ['float32', 'float64'], 'matrix_nms' + ) check_type(score_threshold, 'score_threshold', float, 'matrix_nms') check_type(post_threshold, 'post_threshold', float, 'matrix_nms') check_type(nms_top_k, 'nums_top_k', int, 'matrix_nms') @@ -3662,25 +3898,21 @@ def matrix_nms(bboxes, helper = LayerHelper('matrix_nms', **locals()) output = helper.create_variable_for_type_inference(dtype=bboxes.dtype) index = helper.create_variable_for_type_inference(dtype='int') - helper.append_op(type="matrix_nms", - inputs={ - 'BBoxes': bboxes, - 'Scores': scores - }, - attrs={ - 'score_threshold': score_threshold, - 'post_threshold': post_threshold, - 'nms_top_k': nms_top_k, - 'keep_top_k': keep_top_k, - 'use_gaussian': use_gaussian, - 'gaussian_sigma': gaussian_sigma, - 'background_label': background_label, - 'normalized': normalized - }, - outputs={ - 'Out': output, - 'Index': index - }) + helper.append_op( + type="matrix_nms", + inputs={'BBoxes': bboxes, 'Scores': scores}, + attrs={ + 'score_threshold': score_threshold, + 'post_threshold': post_threshold, + 'nms_top_k': nms_top_k, + 'keep_top_k': keep_top_k, + 'use_gaussian': use_gaussian, + 'gaussian_sigma': gaussian_sigma, + 'background_label': background_label, + 'normalized': normalized, + }, + outputs={'Out': output, 'Index': index}, + ) output.stop_gradient = True if return_index: @@ -3689,13 +3921,15 @@ def matrix_nms(bboxes, return output -def distribute_fpn_proposals(fpn_rois, - min_level, - max_level, - refer_level, - refer_scale, - rois_num=None, - name=None): +def distribute_fpn_proposals( + fpn_rois, + min_level, + max_level, + refer_level, + refer_scale, + rois_num=None, + name=None, +): r""" **This op only takes LoDTensor as input.** In Feature Pyramid Networks @@ -3763,22 +3997,21 @@ def distribute_fpn_proposals(fpn_rois, refer_level=4, refer_scale=224) """ - return paddle.vision.ops.distribute_fpn_proposals(fpn_rois=fpn_rois, - min_level=min_level, - max_level=max_level, - refer_level=refer_level, - refer_scale=refer_scale, - rois_num=rois_num, - name=name) + return paddle.vision.ops.distribute_fpn_proposals( + fpn_rois=fpn_rois, + min_level=min_level, + max_level=max_level, + refer_level=refer_level, + refer_scale=refer_scale, + rois_num=rois_num, + name=name, + ) @templatedoc() -def box_decoder_and_assign(prior_box, - prior_box_var, - target_box, - box_score, - box_clip, - name=None): +def box_decoder_and_assign( + prior_box, prior_box_var, target_box, box_score, box_clip, name=None +): """ ${comment} @@ -3818,41 +4051,53 @@ def box_decoder_and_assign(prior_box, pb, pbv, loc, scores, 4.135) """ - check_variable_and_dtype(prior_box, 'prior_box', ['float32', 'float64'], - 'box_decoder_and_assign') - check_variable_and_dtype(target_box, 'target_box', ['float32', 'float64'], - 'box_decoder_and_assign') - check_variable_and_dtype(box_score, 'box_score', ['float32', 'float64'], - 'box_decoder_and_assign') + check_variable_and_dtype( + prior_box, 'prior_box', ['float32', 'float64'], 'box_decoder_and_assign' + ) + check_variable_and_dtype( + target_box, + 'target_box', + ['float32', 'float64'], + 'box_decoder_and_assign', + ) + check_variable_and_dtype( + box_score, 'box_score', ['float32', 'float64'], 'box_decoder_and_assign' + ) helper = LayerHelper("box_decoder_and_assign", **locals()) decoded_box = helper.create_variable_for_type_inference( - dtype=prior_box.dtype) + dtype=prior_box.dtype + ) output_assign_box = helper.create_variable_for_type_inference( - dtype=prior_box.dtype) - - helper.append_op(type="box_decoder_and_assign", - inputs={ - "PriorBox": prior_box, - "PriorBoxVar": prior_box_var, - "TargetBox": target_box, - "BoxScore": box_score - }, - attrs={"box_clip": box_clip}, - outputs={ - "DecodeBox": decoded_box, - "OutputAssignBox": output_assign_box - }) + dtype=prior_box.dtype + ) + + helper.append_op( + type="box_decoder_and_assign", + inputs={ + "PriorBox": prior_box, + "PriorBoxVar": prior_box_var, + "TargetBox": target_box, + "BoxScore": box_score, + }, + attrs={"box_clip": box_clip}, + outputs={ + "DecodeBox": decoded_box, + "OutputAssignBox": output_assign_box, + }, + ) return decoded_box, output_assign_box -def collect_fpn_proposals(multi_rois, - multi_scores, - min_level, - max_level, - post_nms_top_n, - rois_num_per_level=None, - name=None): +def collect_fpn_proposals( + multi_rois, + multi_scores, + min_level, + max_level, + post_nms_top_n, + rois_num_per_level=None, + name=None, +): """ **This OP only supports LoDTensor as input**. Concat multi-level RoIs @@ -3922,17 +4167,21 @@ def collect_fpn_proposals(multi_rois, input_scores = multi_scores[:num_lvl] if _non_static_mode(): - assert rois_num_per_level is not None, "rois_num_per_level should not be None in dygraph mode." + assert ( + rois_num_per_level is not None + ), "rois_num_per_level should not be None in dygraph mode." attrs = ('post_nms_topN', post_nms_top_n) output_rois, rois_num = _legacy_C_ops.collect_fpn_proposals( - input_rois, input_scores, rois_num_per_level, *attrs) + input_rois, input_scores, rois_num_per_level, *attrs + ) check_type(multi_rois, 'multi_rois', list, 'collect_fpn_proposals') check_type(multi_scores, 'multi_scores', list, 'collect_fpn_proposals') helper = LayerHelper('collect_fpn_proposals', **locals()) dtype = helper.input_dtype('multi_rois') - check_dtype(dtype, 'multi_rois', ['float32', 'float64'], - 'collect_fpn_proposals') + check_dtype( + dtype, 'multi_rois', ['float32', 'float64'], 'collect_fpn_proposals' + ) output_rois = helper.create_variable_for_type_inference(dtype) output_rois.stop_gradient = True @@ -3946,10 +4195,12 @@ def collect_fpn_proposals(multi_rois, rois_num = helper.create_variable_for_type_inference(dtype='int32') rois_num.stop_gradient = True outputs['RoisNum'] = rois_num - helper.append_op(type='collect_fpn_proposals', - inputs=inputs, - outputs=outputs, - attrs={'post_nms_topN': post_nms_top_n}) + helper.append_op( + type='collect_fpn_proposals', + inputs=inputs, + outputs=outputs, + attrs={'post_nms_topN': post_nms_top_n}, + ) if rois_num_per_level is not None: return output_rois, rois_num return output_rois diff --git a/python/paddle/fluid/layers/device.py b/python/paddle/fluid/layers/device.py index a5c2e8c3e200d..ac352ef52b18b 100644 --- a/python/paddle/fluid/layers/device.py +++ b/python/paddle/fluid/layers/device.py @@ -28,15 +28,16 @@ def get_places(device_count=None, device_type=None): helper = LayerHelper('get_places', **locals()) out_places = helper.create_variable( - name=unique_name.generate_with_ignorable_key(helper.name + ".out")) + name=unique_name.generate_with_ignorable_key(helper.name + ".out") + ) attrs = dict() if device_count is not None: attrs['device_count'] = int(device_count) if device_type is not None: attrs['device_type'] = str(device_type) - helper.append_op(type='get_places', - outputs={"Out": [out_places]}, - attrs=attrs) + helper.append_op( + type='get_places', outputs={"Out": [out_places]}, attrs=attrs + ) return out_places diff --git a/python/paddle/fluid/layers/distributions.py b/python/paddle/fluid/layers/distributions.py index e776346323281..f955b9299c74e 100644 --- a/python/paddle/fluid/layers/distributions.py +++ b/python/paddle/fluid/layers/distributions.py @@ -20,7 +20,12 @@ import numpy as np import warnings -from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) __all__ = ['Uniform', 'Normal', 'Categorical', 'MultivariateNormalDiag'] @@ -64,7 +69,8 @@ def _validate_args(self, *args): if is_variable and is_number: raise ValueError( - 'if one argument is Variable, all arguments should be Variable') + 'if one argument is Variable, all arguments should be Variable' + ) return is_variable @@ -79,7 +85,7 @@ def _to_variable(self, *args): """ numpy_args = [] variable_args = [] - tmp = 0. + tmp = 0.0 for arg in args: valid_arg = False @@ -87,7 +93,9 @@ def _to_variable(self, *args): if isinstance(arg, cls): valid_arg = True break - assert valid_arg, "type of input args must be float, list, numpy.ndarray or Variable." + assert ( + valid_arg + ), "type of input args must be float, list, numpy.ndarray or Variable." if isinstance(arg, float): arg = np.zeros(1) + arg arg_np = np.array(arg) @@ -175,10 +183,12 @@ class Uniform(Distribution): """ def __init__(self, low, high): - check_type(low, 'low', (float, np.ndarray, tensor.Variable, list), - 'Uniform') - check_type(high, 'high', (float, np.ndarray, tensor.Variable, list), - 'Uniform') + check_type( + low, 'low', (float, np.ndarray, tensor.Variable, list), 'Uniform' + ) + check_type( + high, 'high', (float, np.ndarray, tensor.Variable, list), 'Uniform' + ) self.all_arg_is_float = False self.batch_size_unknown = False @@ -209,17 +219,26 @@ def sample(self, shape, seed=0): if self.batch_size_unknown: output_shape = shape + batch_shape zero_tmp = tensor.fill_constant_batch_size_like( - self.low + self.high, batch_shape + shape, self.low.dtype, 0.) + self.low + self.high, batch_shape + shape, self.low.dtype, 0.0 + ) uniform_random_tmp = nn.uniform_random_batch_size_like( - zero_tmp, zero_tmp.shape, min=0., max=1., seed=seed) - output = uniform_random_tmp * (zero_tmp + self.high - - self.low) + self.low + zero_tmp, zero_tmp.shape, min=0.0, max=1.0, seed=seed + ) + output = ( + uniform_random_tmp * (zero_tmp + self.high - self.low) + + self.low + ) return nn.reshape(output, output_shape) else: output_shape = shape + batch_shape - output = nn.uniform_random(output_shape, seed=seed) * ( - tensor.zeros(output_shape, dtype=self.low.dtype) + - (self.high - self.low)) + self.low + output = ( + nn.uniform_random(output_shape, seed=seed) + * ( + tensor.zeros(output_shape, dtype=self.low.dtype) + + (self.high - self.low) + ) + + self.low + ) if self.all_arg_is_float: return nn.reshape(output, shape) else: @@ -235,8 +254,9 @@ def log_prob(self, value): Variable: log probability.The data type is same with value. """ - check_variable_and_dtype(value, 'value', ['float32', 'float64'], - 'log_prob') + check_variable_and_dtype( + value, 'value', ['float32', 'float64'], 'log_prob' + ) lb_bool = control_flow.less_than(self.low, value) ub_bool = control_flow.less_than(value, self.high) @@ -317,10 +337,12 @@ class Normal(Distribution): """ def __init__(self, loc, scale): - check_type(loc, 'loc', (float, np.ndarray, tensor.Variable, list), - 'Normal') - check_type(scale, 'scale', (float, np.ndarray, tensor.Variable, list), - 'Normal') + check_type( + loc, 'loc', (float, np.ndarray, tensor.Variable, list), 'Normal' + ) + check_type( + scale, 'scale', (float, np.ndarray, tensor.Variable, list), 'Normal' + ) self.batch_size_unknown = False self.all_arg_is_float = False @@ -353,18 +375,24 @@ def sample(self, shape, seed=0): if self.batch_size_unknown: output_shape = shape + batch_shape zero_tmp = tensor.fill_constant_batch_size_like( - self.loc + self.scale, batch_shape + shape, self.loc.dtype, 0.) + self.loc + self.scale, batch_shape + shape, self.loc.dtype, 0.0 + ) zero_tmp_shape = nn.shape(zero_tmp) - normal_random_tmp = nn.gaussian_random(zero_tmp_shape, - mean=0., - std=1., - seed=seed) + normal_random_tmp = nn.gaussian_random( + zero_tmp_shape, mean=0.0, std=1.0, seed=seed + ) output = normal_random_tmp * (zero_tmp + self.scale) + self.loc return nn.reshape(output, output_shape) else: output_shape = shape + batch_shape - output = nn.gaussian_random(output_shape, mean=0., std=1., seed=seed) * \ - (tensor.zeros(output_shape, dtype=self.loc.dtype) + self.scale) + self.loc + output = ( + nn.gaussian_random(output_shape, mean=0.0, std=1.0, seed=seed) + * ( + tensor.zeros(output_shape, dtype=self.loc.dtype) + + self.scale + ) + + self.loc + ) if self.all_arg_is_float: return nn.reshape(output, shape) else: @@ -378,11 +406,12 @@ def entropy(self): """ batch_shape = list((self.loc + self.scale).shape) - zero_tmp = tensor.fill_constant_batch_size_like(self.loc + self.scale, - batch_shape, - self.loc.dtype, 0.) - return 0.5 + 0.5 * math.log(2 * math.pi) + nn.log( - (self.scale + zero_tmp)) + zero_tmp = tensor.fill_constant_batch_size_like( + self.loc + self.scale, batch_shape, self.loc.dtype, 0.0 + ) + return ( + 0.5 + 0.5 * math.log(2 * math.pi) + nn.log((self.scale + zero_tmp)) + ) def log_prob(self, value): """Log probability density/mass function. @@ -394,14 +423,17 @@ def log_prob(self, value): Variable: log probability.The data type is same with value. """ - check_variable_and_dtype(value, 'value', ['float32', 'float64'], - 'log_prob') + check_variable_and_dtype( + value, 'value', ['float32', 'float64'], 'log_prob' + ) var = self.scale * self.scale log_scale = nn.log(self.scale) - return -1. * ((value - self.loc) * - (value - self.loc)) / (2. * var) - log_scale - math.log( - math.sqrt(2. * math.pi)) + return ( + -1.0 * ((value - self.loc) * (value - self.loc)) / (2.0 * var) + - log_scale + - math.log(math.sqrt(2.0 * math.pi)) + ) def kl_divergence(self, other): """The KL-divergence between two normal distributions. @@ -417,10 +449,10 @@ def kl_divergence(self, other): check_type(other, 'other', Normal, 'kl_divergence') var_ratio = self.scale / other.scale - var_ratio = (var_ratio * var_ratio) + var_ratio = var_ratio * var_ratio t1 = (self.loc - other.loc) / other.scale - t1 = (t1 * t1) - return 0.5 * (var_ratio + t1 - 1. - nn.log(var_ratio)) + t1 = t1 * t1 + return 0.5 * (var_ratio + t1 - 1.0 - nn.log(var_ratio)) class Categorical(Distribution): @@ -477,8 +509,9 @@ def __init__(self, logits): Args: logits(list|numpy.ndarray|Variable): The logits input of categorical distribution. The data type is float32. """ - check_type(logits, 'logits', (np.ndarray, tensor.Variable, list), - 'Categorical') + check_type( + logits, 'logits', (np.ndarray, tensor.Variable, list), 'Categorical' + ) if self._validate_args(logits): self.logits = logits @@ -499,7 +532,8 @@ def kl_divergence(self, other): logits = self.logits - nn.reduce_max(self.logits, dim=-1, keep_dim=True) other_logits = other.logits - nn.reduce_max( - other.logits, dim=-1, keep_dim=True) + other.logits, dim=-1, keep_dim=True + ) e_logits = ops.exp(logits) other_e_logits = ops.exp(other_logits) z = nn.reduce_sum(e_logits, dim=-1, keep_dim=True) @@ -508,7 +542,8 @@ def kl_divergence(self, other): kl = nn.reduce_sum( prob * (logits - nn.log(z) - other_logits + nn.log(other_z)), dim=-1, - keep_dim=True) + keep_dim=True, + ) return kl @@ -524,7 +559,8 @@ def entropy(self): z = nn.reduce_sum(e_logits, dim=-1, keep_dim=True) prob = e_logits / z entropy = -1.0 * nn.reduce_sum( - prob * (logits - nn.log(z)), dim=-1, keep_dim=True) + prob * (logits - nn.log(z)), dim=-1, keep_dim=True + ) return entropy @@ -598,10 +634,18 @@ class MultivariateNormalDiag(Distribution): """ def __init__(self, loc, scale): - check_type(loc, 'loc', (np.ndarray, tensor.Variable, list), - 'MultivariateNormalDiag') - check_type(scale, 'scale', (np.ndarray, tensor.Variable, list), - 'MultivariateNormalDiag') + check_type( + loc, + 'loc', + (np.ndarray, tensor.Variable, list), + 'MultivariateNormalDiag', + ) + check_type( + scale, + 'scale', + (np.ndarray, tensor.Variable, list), + 'MultivariateNormalDiag', + ) if self._validate_args(loc, scale): self.loc = loc @@ -614,7 +658,8 @@ def _det(self, value): batch_shape = list(value.shape) one_all = tensor.ones(shape=batch_shape, dtype=self.loc.dtype) one_diag = tensor.diag( - tensor.ones(shape=[batch_shape[0]], dtype=self.loc.dtype)) + tensor.ones(shape=[batch_shape[0]], dtype=self.loc.dtype) + ) det_diag = nn.reduce_prod(value + one_all - one_diag) return det_diag @@ -624,7 +669,8 @@ def _inv(self, value): batch_shape = list(value.shape) one_all = tensor.ones(shape=batch_shape, dtype=self.loc.dtype) one_diag = tensor.diag( - tensor.ones(shape=[batch_shape[0]], dtype=self.loc.dtype)) + tensor.ones(shape=[batch_shape[0]], dtype=self.loc.dtype) + ) inv_diag = nn.elementwise_pow(value, (one_all - 2 * one_diag)) return inv_diag @@ -636,8 +682,10 @@ def entropy(self): Variable: Shannon entropy of Multivariate Normal distribution. The data type is float32. """ - entropy = 0.5 * (self.scale.shape[0] * (1.0 + math.log(2 * math.pi)) + - nn.log(self._det(self.scale))) + entropy = 0.5 * ( + self.scale.shape[0] * (1.0 + math.log(2 * math.pi)) + + nn.log(self._det(self.scale)) + ) return entropy @@ -654,8 +702,9 @@ def kl_divergence(self, other): check_type(other, 'other', MultivariateNormalDiag, 'kl_divergence') tr_cov_matmul = nn.reduce_sum(self._inv(other.scale) * self.scale) - loc_matmul_cov = nn.matmul((other.loc - self.loc), - self._inv(other.scale)) + loc_matmul_cov = nn.matmul( + (other.loc - self.loc), self._inv(other.scale) + ) tri_matmul = nn.matmul(loc_matmul_cov, (other.loc - self.loc)) k = list(self.scale.shape)[0] ln_cov = nn.log(self._det(other.scale)) - nn.log(self._det(self.scale)) diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 5c62dc6d61d66..86654bbf66958 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -23,30 +23,46 @@ from .layer_function_generator import templatedoc from .. import core from ..executor import global_scope -from ..framework import convert_np_dtype_to_dtype_, default_main_program, \ - default_startup_program, program_guard, Program, Variable +from ..framework import ( + convert_np_dtype_to_dtype_, + default_main_program, + default_startup_program, + program_guard, + Program, + Variable, +) from ..layer_helper import LayerHelper from ..unique_name import generate as unique_name import logging from ..data_feeder import check_dtype, check_type from paddle.fluid.framework import static_only -from ..framework import _get_paddle_place, _current_expected_place, _set_expected_place +from ..framework import ( + _get_paddle_place, + _current_expected_place, + _set_expected_place, +) __all__ = [ - 'data', 'read_file', 'double_buffer', 'py_reader', - 'create_py_reader_by_data', 'load' + 'data', + 'read_file', + 'double_buffer', + 'py_reader', + 'create_py_reader_by_data', + 'load', ] @static_only -def data(name, - shape, - append_batch_size=True, - dtype='float32', - lod_level=0, - type=core.VarDesc.VarType.LOD_TENSOR, - stop_gradient=True): +def data( + name, + shape, + append_batch_size=True, + dtype='float32', + lod_level=0, + type=core.VarDesc.VarType.LOD_TENSOR, + stop_gradient=True, +): """ **Data Layer** @@ -124,13 +140,15 @@ def data(name, if append_batch_size: shape = [-1] + shape # append batch size as -1 - data_var = helper.create_global_variable(name=name, - shape=shape, - dtype=dtype, - type=type, - stop_gradient=stop_gradient, - lod_level=lod_level, - is_data=True) + data_var = helper.create_global_variable( + name=name, + shape=shape, + dtype=dtype, + type=type, + stop_gradient=stop_gradient, + lod_level=lod_level, + is_data=True, + ) return data_var @@ -245,13 +263,13 @@ def complete_op(self): attrs={ 'endpoint': self.endpoint, 'Fanin': self.fan_in, - 'optimize_blocks': - [current_block - ], # did not support multiple optimize blocks in layers - 'distributed_mode': - DistributedMode.SYNC, # did not support async now in layers - 'grad_to_block_id': [""] - }) + 'optimize_blocks': [ + current_block + ], # did not support multiple optimize blocks in layers + 'distributed_mode': DistributedMode.SYNC, # did not support async now in layers + 'grad_to_block_id': [""], + }, + ) def Send(endpoints, send_vars, dummy_output=None, sync=True): @@ -266,14 +284,14 @@ def Send(endpoints, send_vars, dummy_output=None, sync=True): sync (bool): whether to wait the request finish """ - assert (type(send_vars) == list) + assert type(send_vars) == list if dummy_output is None: dummy_output = [] elif isinstance(dummy_output, Variable): dummy_output = [dummy_output] - assert (type(dummy_output) == list) + assert type(dummy_output) == list epmap = endpoints.split(",") endpoints = list(set(epmap)) @@ -281,22 +299,23 @@ def Send(endpoints, send_vars, dummy_output=None, sync=True): helper = LayerHelper("Send", **locals()) rpc_op_role_name = core.op_proto_and_checker_maker.kOpRoleAttrName() - helper.append_op(type="send", - inputs={"X": send_vars}, - outputs={"Out": dummy_output}, - attrs={ - "endpoints": - endpoints, - "epmap": - epmap, - rpc_op_role_name: - core.op_proto_and_checker_maker.OpRole.RPC - }) + helper.append_op( + type="send", + inputs={"X": send_vars}, + outputs={"Out": dummy_output}, + attrs={ + "endpoints": endpoints, + "epmap": epmap, + rpc_op_role_name: core.op_proto_and_checker_maker.OpRole.RPC, + }, + ) if sync: - helper.append_op(type="send_barrier", - inputs={"X": dummy_output}, - outputs={"Out": []}, - attrs={"endpoints": endpoints}) + helper.append_op( + type="send_barrier", + inputs={"X": dummy_output}, + outputs={"Out": []}, + attrs={"endpoints": endpoints}, + ) def Recv(endpoints, get_vars, dummy_input=None, sync=True): @@ -312,35 +331,35 @@ def Recv(endpoints, get_vars, dummy_input=None, sync=True): Returns: list: list of received variables """ - assert (type(get_vars) == list) + assert type(get_vars) == list if dummy_input is None: dummy_input = [] elif isinstance(dummy_input, Variable): dummy_input = [dummy_input] - assert (type(dummy_input) == list) + assert type(dummy_input) == list epmap = endpoints.split(",") endpoints = list(set(epmap)) helper = LayerHelper("Recv", **locals()) - helper.append_op(type="recv", - inputs={"X": dummy_input}, - outputs={"Out": get_vars}, - attrs={ - "endpoints": endpoints, - "epmap": epmap - }) + helper.append_op( + type="recv", + inputs={"X": dummy_input}, + outputs={"Out": get_vars}, + attrs={"endpoints": endpoints, "epmap": epmap}, + ) if sync: - helper.append_op(type="fetch_barrier", - outputs={"Out": get_vars}, - attrs={"endpoints": endpoints}) + helper.append_op( + type="fetch_barrier", + outputs={"Out": get_vars}, + attrs={"endpoints": endpoints}, + ) return get_vars def monkey_patch_reader_methods(reader): - def __get_reader__(): scope = global_scope() var = scope.find_var(reader.name) @@ -381,24 +400,30 @@ def _copy_reader_create_op_(block, op): for arg_name in arg_names: new_output_map[param_name].append(block.var(arg_name)) - new_op = block.append_op(type=op.type, - inputs=new_input_map, - outputs=new_output_map, - attrs=op.all_attrs()) + new_op = block.append_op( + type=op.type, + inputs=new_input_map, + outputs=new_output_map, + attrs=op.all_attrs(), + ) return new_op -def _py_reader(capacity, - shapes, - dtypes, - lod_levels=None, - name=None, - use_double_buffer=True, - feed_list=None): +def _py_reader( + capacity, + shapes, + dtypes, + lod_levels=None, + name=None, + use_double_buffer=True, + feed_list=None, +): if feed_list is not None: if not isinstance(feed_list, list): - raise TypeError("feed_list should be a list of Variable" - " instead of " + str(type(feed_list))) + raise TypeError( + "feed_list should be a list of Variable" + " instead of " + str(type(feed_list)) + ) lod_levels = [] dtypes = [] shape_concat = [] @@ -440,22 +465,25 @@ def _py_reader(capacity, startup_blk = default_startup_program().current_block() startup_var = startup_blk.create_var(name=reader_name) - startup_blk.append_op(type='create_py_reader', - inputs={'blocking_queue': [queue_name]}, - outputs={'Out': [startup_var]}, - attrs={ - 'shape_concat': shape_concat, - 'lod_levels': lod_levels, - 'dtypes': dtype_int, - 'need_check_feed': need_check_feed, - 'ranks': ranks - }) + startup_blk.append_op( + type='create_py_reader', + inputs={'blocking_queue': [queue_name]}, + outputs={'Out': [startup_var]}, + attrs={ + 'shape_concat': shape_concat, + 'lod_levels': lod_levels, + 'dtypes': dtype_int, + 'need_check_feed': need_check_feed, + 'ranks': ranks, + }, + ) startup_var.desc.set_dtypes(dtypes) startup_var.persistable = True - main_prog_var = _copy_reader_var_(default_main_program().current_block(), - startup_var) + main_prog_var = _copy_reader_var_( + default_main_program().current_block(), startup_var + ) reader = monkey_patch_reader_methods(main_prog_var) if use_double_buffer: @@ -473,7 +501,6 @@ def _py_reader(capacity, reader.exited = False def start_provide_thread(func): - def __provider_thread__(legacy_expected_place): try: # See _DataLoaderIterSingleProcess._thread_loop() for why set expected place here. @@ -501,8 +528,9 @@ def __provider_thread__(legacy_expected_place): logging.warn('Your decorated reader has raised an exception!') six.reraise(*sys.exc_info()) - reader.thread = threading.Thread(target=__provider_thread__, - args=(_current_expected_place(), )) + reader.thread = threading.Thread( + target=__provider_thread__, args=(_current_expected_place(),) + ) reader.thread.daemon = True reader.thread.start() @@ -518,17 +546,22 @@ def __set_paddle_reader__(paddle_reader): for dtype, shape, lod_level in zip(dtypes, shapes, lod_levels): name = str(counter) actual_feed_list.append( - data(name=name, - dtype=dtype, - shape=shape, - lod_level=lod_level)) + data( + name=name, + dtype=dtype, + shape=shape, + lod_level=lod_level, + ) + ) counter += 1 data_names = [feed_data.name for feed_data in actual_feed_list] - feeder = DataFeeder(feed_list=actual_feed_list, - place=core.CPUPlace()) - paddle_reader = feeder.decorate_reader(paddle_reader, - multi_devices=False) + feeder = DataFeeder( + feed_list=actual_feed_list, place=core.CPUPlace() + ) + paddle_reader = feeder.decorate_reader( + paddle_reader, multi_devices=False + ) def __tensor_provider__(): for slots in paddle_reader(): @@ -557,14 +590,11 @@ def __start__(): return reader -def py_reader(capacity, - shapes, - dtypes, - lod_levels=None, - name=None, - use_double_buffer=True): +def py_reader( + capacity, shapes, dtypes, lod_levels=None, name=None, use_double_buffer=True +): """ - :api_attr: Static Graph + :api_attr: Static Graph Create a Python reader for data feeding in Python @@ -718,21 +748,23 @@ def network(reader): """ logging.warn( 'paddle.fluid.layers.py_reader() may be deprecated in the near future. ' - 'Please use paddle.fluid.io.DataLoader.from_generator() instead.') - return _py_reader(capacity=capacity, - shapes=shapes, - dtypes=dtypes, - lod_levels=lod_levels, - name=name, - use_double_buffer=use_double_buffer) - - -def create_py_reader_by_data(capacity, - feed_list, - name=None, - use_double_buffer=True): + 'Please use paddle.fluid.io.DataLoader.from_generator() instead.' + ) + return _py_reader( + capacity=capacity, + shapes=shapes, + dtypes=dtypes, + lod_levels=lod_levels, + name=name, + use_double_buffer=use_double_buffer, + ) + + +def create_py_reader_by_data( + capacity, feed_list, name=None, use_double_buffer=True +): """ - :api_attr: Static Graph + :api_attr: Static Graph The OP creates a Python reader for data feeding in Python, it is similar to :ref:`api_fluid_layers_py_reader` except that it can read data from @@ -799,24 +831,29 @@ def network(img, label): """ logging.warn( 'paddle.fluid.layers.create_py_reader_by_data() may be deprecated in the near future. ' - 'Please use paddle.fluid.io.DataLoader.from_generator() instead.') - return _py_reader(capacity=capacity, - shapes=None, - dtypes=None, - lod_levels=None, - name=name, - use_double_buffer=use_double_buffer, - feed_list=feed_list) + 'Please use paddle.fluid.io.DataLoader.from_generator() instead.' + ) + return _py_reader( + capacity=capacity, + shapes=None, + dtypes=None, + lod_levels=None, + name=name, + use_double_buffer=use_double_buffer, + feed_list=feed_list, + ) def __create_shared_decorated_reader__(op_type, reader, attrs): var_name = unique_name(op_type) startup_blk = default_startup_program().current_block() startup_var = startup_blk.create_var(name=var_name) - startop_op = startup_blk.append_op(type=op_type, - inputs={'UnderlyingReader': reader}, - outputs={'Out': [startup_var]}, - attrs=attrs) + startop_op = startup_blk.append_op( + type=op_type, + inputs={'UnderlyingReader': reader}, + outputs={'Out': [startup_var]}, + attrs=attrs, + ) startup_var.persistable = True main_prog_block = default_main_program().current_block() main_prog_var = _copy_reader_var_(main_prog_block, startup_var) @@ -828,10 +865,12 @@ def __create_unshared_decorated_reader__(op_type, reader, attrs, name=None): new_reader_name = name if name is not None else unique_name(op_type) main_blk = default_main_program().current_block() new_reader = main_blk.create_var(name=new_reader_name) - main_blk.append_op(type=op_type, - inputs={'UnderlyingReader': reader}, - outputs={'Out': [new_reader]}, - attrs=attrs) + main_blk.append_op( + type=op_type, + inputs={'UnderlyingReader': reader}, + outputs={'Out': [new_reader]}, + attrs=attrs, + ) return monkey_patch_reader_methods(new_reader) @@ -864,15 +903,14 @@ def double_buffer(reader, place=None, name=None): if place is not None: attrs['place'] = str(_get_paddle_place(place)).upper() - return __create_unshared_decorated_reader__('create_double_buffer_reader', - reader, - attrs, - name=name) + return __create_unshared_decorated_reader__( + 'create_double_buffer_reader', reader, attrs, name=name + ) def read_file(reader): """ - :api_attr: Static Graph + :api_attr: Static Graph Execute the given reader and get data via it. @@ -898,13 +936,14 @@ def read_file(reader): """ helper = LayerHelper('read_file') out = [ - helper.create_variable_for_type_inference(stop_gradient=True, - dtype='float32') + helper.create_variable_for_type_inference( + stop_gradient=True, dtype='float32' + ) for _ in range(len(reader.desc.shapes())) ] - helper.append_op(type='read', - inputs={'Reader': [reader]}, - outputs={'Out': out}) + helper.append_op( + type='read', inputs={'Reader': [reader]}, outputs={'Out': out} + ) if len(out) == 1: return out[0] else: diff --git a/python/paddle/fluid/layers/layer_function_generator.py b/python/paddle/fluid/layers/layer_function_generator.py index 3beb3bae352c1..b75e628a37a76 100755 --- a/python/paddle/fluid/layers/layer_function_generator.py +++ b/python/paddle/fluid/layers/layer_function_generator.py @@ -19,15 +19,26 @@ from io import StringIO from ..proto import framework_pb2 -from ..framework import OpProtoHolder, Variable, core, convert_np_dtype_to_dtype_, _non_static_mode, in_dygraph_mode, _in_legacy_dygraph +from ..framework import ( + OpProtoHolder, + Variable, + core, + convert_np_dtype_to_dtype_, + _non_static_mode, + in_dygraph_mode, + _in_legacy_dygraph, +) from ..layer_helper import LayerHelper from ..data_feeder import check_variable_and_dtype from paddle.fluid.framework import in_dygraph_mode, _in_legacy_dygraph from paddle import _C_ops, _legacy_C_ops __all__ = [ - 'generate_layer_fn', 'generate_activation_fn', 'generate_inplace_fn', - 'autodoc', 'templatedoc' + 'generate_layer_fn', + 'generate_activation_fn', + 'generate_inplace_fn', + 'autodoc', + 'templatedoc', ] @@ -57,16 +68,16 @@ def _type_to_str_(tp): def escape_math(text): - #return _two_bang_pattern_.sub( + # return _two_bang_pattern_.sub( # r'$$\1$$', # _single_dollar_pattern_.sub(r':math:\n`\1`', # _two_dollar_pattern_.sub(r"!!\1!!", text))) return _two_dollar_pattern_.sub(r':math:`\1`', text) -def _generate_doc_string_(op_proto, - additional_args_lines=None, - skip_attrs_set=None): +def _generate_doc_string_( + op_proto, additional_args_lines=None, skip_attrs_set=None +): """ Generate docstring by OpProto @@ -146,23 +157,30 @@ def generate_layer_fn(op_type): """ op_proto = OpProtoHolder.instance().get_op_proto(op_type) - not_intermediate_outputs = \ - [output for output in op_proto.outputs if not output.intermediate] - intermediate_outputs = \ - [output for output in op_proto.outputs if output.intermediate] + not_intermediate_outputs = [ + output for output in op_proto.outputs if not output.intermediate + ] + intermediate_outputs = [ + output for output in op_proto.outputs if output.intermediate + ] if len(not_intermediate_outputs) != 1: - raise ValueError("Only one non intermediate output operator can be", - "automatically generated. {0}".format(op_type)) + raise ValueError( + "Only one non intermediate output operator can be", + "automatically generated. {0}".format(op_type), + ) if not_intermediate_outputs[0].duplicable: raise ValueError( - "Only non duplicable op can be automatically generated.") + "Only non duplicable op can be automatically generated." + ) for output in intermediate_outputs: if output.duplicable: - raise ValueError("The op can be automatically generated only when ", - "all intermediate ops are not duplicable.") + raise ValueError( + "The op can be automatically generated only when ", + "all intermediate ops are not duplicable.", + ) o_name = not_intermediate_outputs[0].name intermediate_output_names = [output.name for output in intermediate_outputs] @@ -187,14 +205,17 @@ def infer_and_check_dtype(op_proto, *args, **kwargs): for each in val: if not isinstance(each, Variable): raise ValueError( - "input of {0} must be variable".format(op_type)) + "input of {0} must be variable".format(op_type) + ) if dtype is None: dtype = each.dtype elif dtype != each.dtype: raise ValueError( "operator {0} must input same dtype. {1} vs {2}".format( - op_type, dtype, each.dtype)) + op_type, dtype, each.dtype + ) + ) if dtype is None: arg_dtype = kwargs.get("dtype") @@ -226,8 +247,11 @@ def func(*args, **kwargs): outputs = dict() out = kwargs.pop(_convert_(o_name), []) if out: - out_var = out[0] if (isinstance(out, list) - or isinstance(out, tuple)) else out + out_var = ( + out[0] + if (isinstance(out, list) or isinstance(out, tuple)) + else out + ) else: out_var = helper.create_variable_for_type_inference(dtype=dtype) outputs[o_name] = [out_var] @@ -235,10 +259,9 @@ def func(*args, **kwargs): outputs[name] = [ helper.create_variable_for_type_inference(dtype=dtype) ] - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=kwargs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=kwargs + ) return helper.append_activation(out_var) func.__name__ = op_type @@ -269,14 +292,25 @@ def func(x, name=None): return op(x) if op_type not in ["abs", "exp", "square"]: - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - op_type) + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], op_type + ) else: # abs exp square ops support dtype(int32, int64, float16, float32, float64) - check_variable_and_dtype(x, 'x', [ - 'int32', 'int64', 'float16', 'float32', 'float64', 'complex64', - 'complex128' - ], op_type) + check_variable_and_dtype( + x, + 'x', + [ + 'int32', + 'int64', + 'float16', + 'float32', + 'float64', + 'complex64', + 'complex128', + ], + op_type, + ) helper = LayerHelper(op_type, **locals()) @@ -289,7 +323,8 @@ def func(x, name=None): op_proto, additional_args_lines=[ "name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`." - ]) + ], + ) return func @@ -309,24 +344,31 @@ def func(x, name=None): op = getattr(_legacy_C_ops, inplace_op_type) return op(x) warnings.warn( - "In static mode, {}() is the same as {}() and does not perform inplace operation." - .format(inplace_op_type, origin_op_type)) + "In static mode, {}() is the same as {}() and does not perform inplace operation.".format( + inplace_op_type, origin_op_type + ) + ) return generate_activation_fn(origin_op_type)(x, name) func.__name__ = inplace_op_type func.__doc__ = """ Inplace version of ``{0}`` API, the output Tensor will be inplaced with input ``x``. Please refer to :ref:`api_fluid_layers_{1}`. -""".format(origin_op_type, origin_op_type) +""".format( + origin_op_type, origin_op_type + ) return func def autodoc(comment=""): - def __impl__(func): - func.__doc__ = _generate_doc_string_( - OpProtoHolder.instance().get_op_proto(func.__name__)) + comment + func.__doc__ = ( + _generate_doc_string_( + OpProtoHolder.instance().get_op_proto(func.__name__) + ) + + comment + ) return func return __impl__ @@ -371,18 +413,21 @@ def __impl__(func): for each_input in op_proto.inputs: input_name = _convert_(each_input.name) args["{0}_comment".format(input_name)] = trim_ending_dot( - each_input.comment) + each_input.comment + ) args["{0}_type".format(input_name)] = "Variable" for each_attr in op_proto.attrs: input_name = _convert_(each_attr.name) args["{0}_comment".format(input_name)] = trim_ending_dot( - each_attr.comment) + each_attr.comment + ) args["{0}_type".format(input_name)] = _type_to_str_(each_attr.type) for each_opt in op_proto.outputs: output_name = _convert_(each_opt.name) args["{0}_comment".format(output_name)] = trim_ending_dot( - each_opt.comment) + each_opt.comment + ) args["{0}_type".format(output_name)] = "Variable" func.__doc__ = tmpl.substitute(args) return func diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py index 3b1147979e1d7..737d9e75e11cb 100644 --- a/python/paddle/fluid/layers/learning_rate_scheduler.py +++ b/python/paddle/fluid/layers/learning_rate_scheduler.py @@ -34,16 +34,22 @@ from ..data_feeder import check_variable_and_dtype, check_type __all__ = [ - 'exponential_decay', 'natural_exp_decay', 'inverse_time_decay', - 'polynomial_decay', 'piecewise_decay', 'noam_decay', 'cosine_decay', - 'linear_lr_warmup' + 'exponential_decay', + 'natural_exp_decay', + 'inverse_time_decay', + 'polynomial_decay', + 'piecewise_decay', + 'noam_decay', + 'cosine_decay', + 'linear_lr_warmup', ] def _decay_step_counter(begin=0): # the first global step is zero in learning rate decay global_step = nn.autoincreased_step_counter( - counter_name='@LR_DECAY_COUNTER@', begin=begin, step=1) + counter_name='@LR_DECAY_COUNTER@', begin=begin, step=1 + ) global_step = tensor.cast(global_step, 'float32') return global_step @@ -94,17 +100,18 @@ def noam_decay(d_model, warmup_steps, learning_rate=1.0): """ with default_main_program()._lr_schedule_guard(): if _non_static_mode(): - decay = imperate_lr.NoamDecay(d_model, - warmup_steps, - learning_rate=learning_rate) + decay = imperate_lr.NoamDecay( + d_model, warmup_steps, learning_rate=learning_rate + ) return decay else: global_step = _decay_step_counter(1) a = global_step**-0.5 b = (warmup_steps**-1.5) * global_step - lr_value = learning_rate * (d_model**-0.5) * nn.elementwise_min( - a, b) + lr_value = ( + learning_rate * (d_model**-0.5) * nn.elementwise_min(a, b) + ) return lr_value @@ -147,17 +154,18 @@ def exponential_decay(learning_rate, decay_steps, decay_rate, staircase=False): paddle.enable_static() base_lr = 0.1 sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.exponential_decay( - learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True)) + learning_rate=fluid.layers.exponential_decay( + learning_rate=base_lr, + decay_steps=10000, + decay_rate=0.5, + staircase=True)) """ with default_main_program()._lr_schedule_guard(): if _non_static_mode(): - decay = imperate_lr.ExponentialDecay(learning_rate, decay_steps, - decay_rate, staircase) + decay = imperate_lr.ExponentialDecay( + learning_rate, decay_steps, decay_rate, staircase + ) return decay else: global_step = _decay_step_counter() @@ -173,52 +181,53 @@ def exponential_decay(learning_rate, decay_steps, decay_rate, staircase=False): def natural_exp_decay(learning_rate, decay_steps, decay_rate, staircase=False): """ -Applies natural exponential decay to the initial learning rate. + Applies natural exponential decay to the initial learning rate. - When training a model, it is often recommended to lower the learning rate as the - training progresses. By using this function, the learning rate will be decayed by - natural exponential power 'decay_rate' every 'decay_steps' steps. + When training a model, it is often recommended to lower the learning rate as the + training progresses. By using this function, the learning rate will be decayed by + natural exponential power 'decay_rate' every 'decay_steps' steps. - Decayed learning rate calculates as follows: + Decayed learning rate calculates as follows: - >>> if not staircase: - >>> decayed_learning_rate = learning_rate * exp(- decay_rate * (global_step / decay_steps)) - >>> else: - >>> decayed_learning_rate = learning_rate * exp(- decay_rate * floor(global_step / decay_steps)) + >>> if not staircase: + >>> decayed_learning_rate = learning_rate * exp(- decay_rate * (global_step / decay_steps)) + >>> else: + >>> decayed_learning_rate = learning_rate * exp(- decay_rate * floor(global_step / decay_steps)) - Args: - learning_rate(Variable|float): The initial learning rate. It should be a Variable - or a float - decay_steps(int): The learning rate decay steps. See the decay computation above. - decay_rate(float): The learning rate decay rate. See the decay computation above. - staircase(bool): If True, decay the learning rate at discrete intervals, which - means the learning rate will be decayed by natural exponential power - `decay_rate` every `decay_steps`. If False, learning rate will be - decayed continuously and following the formula above. Default: False + Args: + learning_rate(Variable|float): The initial learning rate. It should be a Variable + or a float + decay_steps(int): The learning rate decay steps. See the decay computation above. + decay_rate(float): The learning rate decay rate. See the decay computation above. + staircase(bool): If True, decay the learning rate at discrete intervals, which + means the learning rate will be decayed by natural exponential power + `decay_rate` every `decay_steps`. If False, learning rate will be + decayed continuously and following the formula above. Default: False - Returns: - The decayed learning rate. The data type is float32. + Returns: + The decayed learning rate. The data type is float32. - Examples: - .. code-block:: python + Examples: + .. code-block:: python - import paddle.fluid as fluid - import paddle + import paddle.fluid as fluid + import paddle - paddle.enable_static() - base_lr = 0.1 - sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.natural_exp_decay( - learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True)) + paddle.enable_static() + base_lr = 0.1 + sgd_optimizer = fluid.optimizer.SGD( + learning_rate=fluid.layers.natural_exp_decay( + learning_rate=base_lr, + decay_steps=10000, + decay_rate=0.5, + staircase=True)) """ with default_main_program()._lr_schedule_guard(): if _non_static_mode(): - decay = imperate_lr.NaturalExpDecay(learning_rate, decay_steps, - decay_rate, staircase) + decay = imperate_lr.NaturalExpDecay( + learning_rate, decay_steps, decay_rate, staircase + ) return decay else: global_step = _decay_step_counter() @@ -268,16 +277,17 @@ def inverse_time_decay(learning_rate, decay_steps, decay_rate, staircase=False): paddle.enable_static() base_lr = 0.1 sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.inverse_time_decay( - learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True)) + learning_rate=fluid.layers.inverse_time_decay( + learning_rate=base_lr, + decay_steps=10000, + decay_rate=0.5, + staircase=True)) """ with default_main_program()._lr_schedule_guard(): if _non_static_mode(): - decay = imperate_lr.InverseTimeDecay(learning_rate, decay_steps, - decay_rate, staircase) + decay = imperate_lr.InverseTimeDecay( + learning_rate, decay_steps, decay_rate, staircase + ) return decay else: global_step = _decay_step_counter() @@ -291,11 +301,9 @@ def inverse_time_decay(learning_rate, decay_steps, decay_rate, staircase=False): return decayed_lr -def polynomial_decay(learning_rate, - decay_steps, - end_learning_rate=0.0001, - power=1.0, - cycle=False): +def polynomial_decay( + learning_rate, decay_steps, end_learning_rate=0.0001, power=1.0, cycle=False +): """ Applies polynomial decay to the initial learning rate. @@ -332,74 +340,77 @@ def polynomial_decay(learning_rate, """ with default_main_program()._lr_schedule_guard(): if _non_static_mode(): - decay = imperate_lr.PolynomialDecay(learning_rate, decay_steps, - end_learning_rate, power, cycle) + decay = imperate_lr.PolynomialDecay( + learning_rate, decay_steps, end_learning_rate, power, cycle + ) return decay else: global_step = _decay_step_counter() if cycle: div_res = ops.ceil(global_step / decay_steps) - zero_var = tensor.fill_constant(shape=[1], - dtype='float32', - value=0.0) - one_var = tensor.fill_constant(shape=[1], - dtype='float32', - value=1.0) + zero_var = tensor.fill_constant( + shape=[1], dtype='float32', value=0.0 + ) + one_var = tensor.fill_constant( + shape=[1], dtype='float32', value=1.0 + ) with control_flow.Switch() as switch: with switch.case(global_step == zero_var): tensor.assign(input=one_var, output=div_res) decay_steps = decay_steps * div_res else: - decay_steps_var = tensor.fill_constant(shape=[1], - dtype='float32', - value=float(decay_steps)) - global_step = nn.elementwise_min(x=global_step, - y=decay_steps_var) - - decayed_lr = (learning_rate - end_learning_rate) * \ - ((1 - global_step / decay_steps) ** power) + end_learning_rate + decay_steps_var = tensor.fill_constant( + shape=[1], dtype='float32', value=float(decay_steps) + ) + global_step = nn.elementwise_min( + x=global_step, y=decay_steps_var + ) + + decayed_lr = (learning_rate - end_learning_rate) * ( + (1 - global_step / decay_steps) ** power + ) + end_learning_rate return decayed_lr def piecewise_decay(boundaries, values): """ -Applies piecewise decay to the initial learning rate. - - The algorithm can be described as the code below. + Applies piecewise decay to the initial learning rate. - .. code-block:: text + The algorithm can be described as the code below. - boundaries = [10000, 20000] - values = [1.0, 0.5, 0.1] - if step < 10000: - learning_rate = 1.0 - elif 10000 <= step < 20000: - learning_rate = 0.5 - else: - learning_rate = 0.1 - Args: - boundaries: A list of steps numbers. - values: A list of learning rate values that will be picked during - different step boundaries. + .. code-block:: text - Returns: - The decayed learning rate. - - Examples: - .. code-block:: python - - import paddle.fluid as fluid - import paddle - paddle.enable_static() boundaries = [10000, 20000] values = [1.0, 0.5, 0.1] - optimizer = fluid.optimizer.Momentum( - momentum=0.9, - learning_rate=fluid.layers.piecewise_decay(boundaries=boundaries, values=values), - regularization=fluid.regularizer.L2Decay(1e-4)) + if step < 10000: + learning_rate = 1.0 + elif 10000 <= step < 20000: + learning_rate = 0.5 + else: + learning_rate = 0.1 + Args: + boundaries: A list of steps numbers. + values: A list of learning rate values that will be picked during + different step boundaries. + + Returns: + The decayed learning rate. + + Examples: + .. code-block:: python + + import paddle.fluid as fluid + import paddle + paddle.enable_static() + boundaries = [10000, 20000] + values = [1.0, 0.5, 0.1] + optimizer = fluid.optimizer.Momentum( + momentum=0.9, + learning_rate=fluid.layers.piecewise_decay(boundaries=boundaries, values=values), + regularization=fluid.regularizer.L2Decay(1e-4)) """ @@ -413,29 +424,36 @@ def piecewise_decay(boundaries, values): else: global_step = _decay_step_counter() - lr = tensor.create_global_var(shape=[1], - value=0.0, - dtype='float32', - persistable=True, - name="learning_rate") + lr = tensor.create_global_var( + shape=[1], + value=0.0, + dtype='float32', + persistable=True, + name="learning_rate", + ) with control_flow.Switch() as switch: for i in range(len(boundaries)): - boundary_val = tensor.fill_constant(shape=[1], - dtype='float32', - value=float( - boundaries[i]), - force_cpu=True) + boundary_val = tensor.fill_constant( + shape=[1], + dtype='float32', + value=float(boundaries[i]), + force_cpu=True, + ) with switch.case(global_step < boundary_val): - tensor.fill_constant(shape=[1], - dtype="float32", - value=float(values[i]), - out=lr) + tensor.fill_constant( + shape=[1], + dtype="float32", + value=float(values[i]), + out=lr, + ) with switch.default(): - tensor.fill_constant(shape=[1], - dtype="float32", - value=float(values[len(values) - 1]), - out=lr) + tensor.fill_constant( + shape=[1], + dtype="float32", + value=float(values[len(values) - 1]), + out=lr, + ) return lr @@ -469,20 +487,25 @@ def cosine_decay(learning_rate, step_each_epoch, epochs): lr = fluid.layers.cosine_decay( learning_rate = base_lr, step_each_epoch=10000, epochs=120) """ - check_type(learning_rate, 'learning_rate', (float, tensor.Variable), - 'cosine_decay') + check_type( + learning_rate, 'learning_rate', (float, tensor.Variable), 'cosine_decay' + ) with default_main_program()._lr_schedule_guard(): if _non_static_mode(): - decay = imperate_lr.CosineDecay(learning_rate, step_each_epoch, - epochs) + decay = imperate_lr.CosineDecay( + learning_rate, step_each_epoch, epochs + ) return decay else: global_step = _decay_step_counter() cur_epoch = ops.floor(global_step / step_each_epoch) - decayed_lr = learning_rate * 0.5 * ( - ops.cos(cur_epoch * math.pi / epochs) + 1) + decayed_lr = ( + learning_rate + * 0.5 + * (ops.cos(cur_epoch * math.pi / epochs) + 1) + ) return decayed_lr @@ -551,26 +574,31 @@ def linear_lr_warmup(learning_rate, warmup_steps, start_lr, end_lr): with default_main_program()._lr_schedule_guard(): if _non_static_mode(): - lr = imperate_lr.LinearLrWarmup(learning_rate, warmup_steps, - start_lr, end_lr) + lr = imperate_lr.LinearLrWarmup( + learning_rate, warmup_steps, start_lr, end_lr + ) return lr else: - lr = tensor.create_global_var(shape=[1], - value=0.0, - dtype=dtype, - persistable=True, - name="learning_rate_warmup") + lr = tensor.create_global_var( + shape=[1], + value=0.0, + dtype=dtype, + persistable=True, + name="learning_rate_warmup", + ) global_step = _decay_step_counter() with control_flow.Switch() as switch: with switch.case(global_step < warmup_steps): - decayed_lr = start_lr + linear_step * (global_step / - float(warmup_steps)) + decayed_lr = start_lr + linear_step * ( + global_step / float(warmup_steps) + ) tensor.assign(decayed_lr, lr) with switch.default(): if not isinstance(learning_rate, Variable): learning_rate = tensor.fill_constant( - shape=[1], dtype=dtype, value=float(learning_rate)) + shape=[1], dtype=dtype, value=float(learning_rate) + ) tensor.assign(learning_rate, lr) return lr diff --git a/python/paddle/fluid/layers/loss.py b/python/paddle/fluid/layers/loss.py index 23835d7e7a01f..6dbbb7338a701 100644 --- a/python/paddle/fluid/layers/loss.py +++ b/python/paddle/fluid/layers/loss.py @@ -19,7 +19,13 @@ from . import nn from .layer_function_generator import templatedoc from ..layer_helper import LayerHelper -from ..framework import Variable, _non_static_mode, static_only, _in_legacy_dygraph, in_dygraph_mode +from ..framework import ( + Variable, + _non_static_mode, + static_only, + _in_legacy_dygraph, + in_dygraph_mode, +) from .. import core from ..data_feeder import check_variable_and_dtype, check_type from ..param_attr import ParamAttr @@ -52,12 +58,9 @@ kIgnoreIndex = -100 -def center_loss(input, - label, - num_classes, - alpha, - param_attr, - update_center=True): +def center_loss( + input, label, num_classes, alpha, param_attr, update_center=True +): r""" :api_attr: Static Graph @@ -107,20 +110,22 @@ def center_loss(input, """ helper = LayerHelper('center_loss', **locals()) dtype = helper.input_dtype() - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'center_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'center_loss' + ) check_variable_and_dtype(label, 'label', ['int32', 'int64'], 'center_loss') centers_shape = [num_classes, input.shape[1]] - centers_param = helper.create_parameter(attr=param_attr, - shape=centers_shape, - dtype=dtype) + centers_param = helper.create_parameter( + attr=param_attr, shape=centers_shape, dtype=dtype + ) centers_param.stop_gradient = True if isinstance(alpha, Variable): alpha_param = alpha - check_variable_and_dtype(alpha, 'alpha', ['float32', 'float64'], - 'center_loss') + check_variable_and_dtype( + alpha, 'alpha', ['float32', 'float64'], 'center_loss' + ) else: assert isinstance(alpha, float) alpha_param = helper.create_variable( @@ -130,26 +135,26 @@ def center_loss(input, type=core.VarDesc.VarType.LOD_TENSOR, persistable=True, stop_gradient=True, - initializer=Constant(alpha)) + initializer=Constant(alpha), + ) centersdiff = helper.create_variable_for_type_inference(dtype=input.dtype) loss = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='center_loss', - inputs={ - 'X': [input], - 'Label': [label], - 'Centers': [centers_param], - 'CenterUpdateRate': [alpha_param] - }, - outputs={ - 'SampleCenterDiff': [centersdiff], - 'Loss': [loss], - 'CentersOut': [centers_param] - }, - attrs={ - 'cluster_num': num_classes, - 'need_update': update_center - }) + helper.append_op( + type='center_loss', + inputs={ + 'X': [input], + 'Label': [label], + 'Centers': [centers_param], + 'CenterUpdateRate': [alpha_param], + }, + outputs={ + 'SampleCenterDiff': [centersdiff], + 'Loss': [loss], + 'CentersOut': [centers_param], + }, + attrs={'cluster_num': num_classes, 'need_update': update_center}, + ) return loss @@ -195,22 +200,22 @@ def bpr_loss(input, label, name=None): """ helper = LayerHelper('bpr_loss', **locals()) out = helper.create_variable_for_type_inference(dtype=input.dtype) - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'bpr_loss') - helper.append_op(type='bpr_loss', - inputs={ - 'X': [input], - 'Label': [label] - }, - outputs={'Y': [out]}) + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'bpr_loss' + ) + helper.append_op( + type='bpr_loss', + inputs={'X': [input], 'Label': [label]}, + outputs={'Y': [out]}, + ) return out def cross_entropy(input, label, soft_label=False, ignore_index=kIgnoreIndex): r""" :alias_main: paddle.nn.functional.cross_entropy - :alias: paddle.nn.functional.cross_entropy,paddle.nn.functional.loss.cross_entropy - :old_api: paddle.fluid.layers.cross_entropy + :alias: paddle.nn.functional.cross_entropy,paddle.nn.functional.loss.cross_entropy + :old_api: paddle.fluid.layers.cross_entropy This operator computes the cross entropy between input and label. It supports both hard-label and and soft-label cross entropy computation. @@ -264,46 +269,46 @@ def cross_entropy(input, label, soft_label=False, ignore_index=kIgnoreIndex): return cross_entropy2(input, label, ignore_index) if _non_static_mode(): - return _legacy_C_ops.cross_entropy(input, label, "soft_label", - soft_label, "ignore_index", - ignore_index) + return _legacy_C_ops.cross_entropy( + input, label, "soft_label", soft_label, "ignore_index", ignore_index + ) inputs = {'X': [input], 'Label': [label]} attrs = {"soft_label": soft_label, "ignore_index": ignore_index} - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'cross_entropy') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'cross_entropy' + ) helper = LayerHelper('cross_entropy', **locals()) out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='cross_entropy', - inputs=inputs, - outputs={'Y': [out]}, - attrs=attrs) + helper.append_op( + type='cross_entropy', inputs=inputs, outputs={'Y': [out]}, attrs=attrs + ) return out def cross_entropy2(input, label, ignore_index=kIgnoreIndex): if _non_static_mode(): - loss, _, _ = _legacy_C_ops.cross_entropy2(input, label, 'ignore_index', - ignore_index) + loss, _, _ = _legacy_C_ops.cross_entropy2( + input, label, 'ignore_index', ignore_index + ) return loss inputs = {'X': [input], 'Label': [label]} attrs = {'ignore_index': ignore_index} - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'cross_entropy2') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'cross_entropy2' + ) helper = LayerHelper('cross_entropy2', **locals()) out = helper.create_variable_for_type_inference(dtype=input.dtype) xshape = helper.create_variable_for_type_inference(dtype=input.dtype) match_x = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='cross_entropy2', - inputs=inputs, - outputs={ - 'Y': [out], - 'MatchX': [match_x], - 'XShape': [xshape] - }, - attrs=attrs) + helper.append_op( + type='cross_entropy2', + inputs=inputs, + outputs={'Y': [out], 'MatchX': [match_x], 'XShape': [xshape]}, + attrs=attrs, + ) return out @@ -342,12 +347,14 @@ def square_error_cost(input, label): return paddle.nn.functional.square_error_cost(input, label) -def edit_distance(input, - label, - normalized=True, - ignored_tokens=None, - input_length=None, - label_length=None): +def edit_distance( + input, + label, + normalized=True, + ignored_tokens=None, + input_length=None, + label_length=None, +): """ This op computes the edit distances, also called Levenshtein distance, between a batch of hypothesis strings and their references. It measures how dissimilar two strings are by counting @@ -382,7 +389,7 @@ def edit_distance(input, NOTE: This Api is different from fluid.metrics.EditDistance Returns: - Tuple: + Tuple: distance(Tensor): edit distance result, its data type is float32, and its shape is (batch_size, 1). sequence_num(Tensor): sequence number, its data type is float32, and its shape is (1,). @@ -415,17 +422,19 @@ def edit_distance(input, # [4] """ - return paddle.nn.functional.loss.edit_distance(input, label, normalized, - ignored_tokens, input_length, - label_length) + return paddle.nn.functional.loss.edit_distance( + input, label, normalized, ignored_tokens, input_length, label_length + ) -def warpctc(input, - label, - blank=0, - norm_by_times=False, - input_length=None, - label_length=None): +def warpctc( + input, + label, + blank=0, + norm_by_times=False, + input_length=None, + label_length=None, +): """ An operator integrating the open source Warp-CTC library (https://github.com/baidu-research/warp-ctc) @@ -548,8 +557,9 @@ def warpctc(input, raise ValueError( "input_length and label_length must not be None in dygraph mode!" ) - loss_out = _C_ops.warpctc(input, label, input_length, label_length, - blank, norm_by_times) + loss_out = _C_ops.warpctc( + input, label, input_length, label_length, blank, norm_by_times + ) return loss_out if _non_static_mode(): if input_length is None or label_length is None: @@ -572,26 +582,27 @@ def warpctc(input, check_variable_and_dtype(label, 'label', ['int32'], "warpctc") this_inputs = {'Logits': [input], 'Label': [label]} if input_length is not None and label_length is not None: - check_variable_and_dtype(input_length, 'LogitsLength', ['int64'], - "warpctc") - check_variable_and_dtype(label_length, 'LabelLength', ['int64'], - "warpctc") + check_variable_and_dtype( + input_length, 'LogitsLength', ['int64'], "warpctc" + ) + check_variable_and_dtype( + label_length, 'LabelLength', ['int64'], "warpctc" + ) this_inputs['LogitsLength'] = [input_length] this_inputs['LabelLength'] = [label_length] loss_out = helper.create_variable_for_type_inference(dtype=input.dtype) grad_out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='warpctc', - inputs=this_inputs, - outputs={ - 'WarpCTCGrad': [grad_out], - 'Loss': [loss_out] - }, - attrs={ - 'blank': blank, - 'norm_by_times': norm_by_times, - }) + helper.append_op( + type='warpctc', + inputs=this_inputs, + outputs={'WarpCTCGrad': [grad_out], 'Loss': [loss_out]}, + attrs={ + 'blank': blank, + 'norm_by_times': norm_by_times, + }, + ) return loss_out @@ -600,18 +611,20 @@ def warpctc(input, # the type is often "Variable", and arguments may vary. @static_only @templatedoc(op_type="nce") -def nce(input, - label, - num_total_classes, - sample_weight=None, - param_attr=None, - bias_attr=None, - num_neg_samples=None, - name=None, - sampler="uniform", - custom_dist=None, - seed=0, - is_sparse=False): +def nce( + input, + label, + num_total_classes, + sample_weight=None, + param_attr=None, + bias_attr=None, + num_neg_samples=None, + name=None, + sampler="uniform", + custom_dist=None, + seed=0, + is_sparse=False, +): """ :api_attr: Static Graph @@ -696,16 +709,20 @@ def nce(input, dim = input.shape[1] num_true_class = label.shape[1] - w = helper.create_parameter(attr=helper.param_attr, - shape=[num_total_classes, dim], - is_bias=False, - dtype=input.dtype) + w = helper.create_parameter( + attr=helper.param_attr, + shape=[num_total_classes, dim], + is_bias=False, + dtype=input.dtype, + ) inputs = {} if helper.bias_attr: - b = helper.create_parameter(attr=helper.bias_attr, - shape=[num_total_classes, 1], - is_bias=True, - dtype=input.dtype) + b = helper.create_parameter( + attr=helper.bias_attr, + shape=[num_total_classes, 1], + is_bias=True, + dtype=input.dtype, + ) inputs['Bias'] = b cost = helper.create_variable_for_type_inference(dtype=input.dtype) sample_logits = helper.create_variable_for_type_inference(dtype=input.dtype) @@ -770,16 +787,20 @@ def _init_by_numpy_array(numpy_array): attr=ParamAttr(), shape=numpy_array.shape, dtype=numpy_array.dtype, - default_initializer=NumpyArrayInitializer(numpy_array)) + default_initializer=NumpyArrayInitializer(numpy_array), + ) ret.stop_gradient = True return ret inputs['CustomDistProbs'] = _init_by_numpy_array( - np.array(custom_dist).astype('float32')) + np.array(custom_dist).astype('float32') + ) inputs['CustomDistAlias'] = _init_by_numpy_array( - np.array(alias_).astype('int32')) + np.array(alias_).astype('int32') + ) inputs['CustomDistAliasProbs'] = _init_by_numpy_array( - np.array(alias_probs_).astype('float32')) + np.array(alias_probs_).astype('float32') + ) sampler = 2 else: raise Exception("Unsupported sampler type.") @@ -800,30 +821,34 @@ def _init_by_numpy_array(numpy_array): 'seed': seed, 'sampler': sampler, 'is_sparse': is_sparse, - 'remote_prefetch': remote_prefetch + 'remote_prefetch': remote_prefetch, } - helper.append_op(type='nce', - inputs=inputs, - outputs={ - 'Cost': cost, - 'SampleLogits': sample_logits, - 'SampleLabels': sample_labels - }, - attrs=attrs) + helper.append_op( + type='nce', + inputs=inputs, + outputs={ + 'Cost': cost, + 'SampleLogits': sample_logits, + 'SampleLabels': sample_labels, + }, + attrs=attrs, + ) return cost / (num_neg_samples + 1) -def hsigmoid(input, - label, - num_classes, - param_attr=None, - bias_attr=None, - name=None, - path_table=None, - path_code=None, - is_custom=False, - is_sparse=False): +def hsigmoid( + input, + label, + num_classes, + param_attr=None, + bias_attr=None, + name=None, + path_table=None, + path_code=None, + is_custom=False, + is_sparse=False, +): """ :api_attr: Static Graph @@ -906,16 +931,19 @@ def hsigmoid(input, dim = input.shape[1] if ((num_classes is None) or (num_classes < 2)) and (not is_custom): raise ValueError( - "num_classes must not be less than 2 with default tree") + "num_classes must not be less than 2 with default tree" + ) if (not is_custom) and (is_sparse): print("Sparse mode should not be used without custom tree") is_sparse = False - if (not is_custom) and ((path_table is not None) or - (path_code is not None)): + if (not is_custom) and ( + (path_table is not None) or (path_code is not None) + ): raise ValueError( - "only num_classes should be passed without custom tree") + "only num_classes should be passed without custom tree" + ) if (is_custom) and (path_code is None): raise ValueError("path_code should not be None with custom tree") @@ -932,59 +960,67 @@ def hsigmoid(input, "With sparse mode, if your models has only small parameter prefetch may cause speed down" ) if not is_custom: - weights = helper.create_parameter(attr=helper.param_attr, - shape=[num_classes - 1, dim], - is_bias=False, - dtype=input.dtype) + weights = helper.create_parameter( + attr=helper.param_attr, + shape=[num_classes - 1, dim], + is_bias=False, + dtype=input.dtype, + ) else: - weights = helper.create_parameter(attr=helper.param_attr, - shape=[num_classes, dim], - is_bias=False, - dtype=input.dtype) + weights = helper.create_parameter( + attr=helper.param_attr, + shape=[num_classes, dim], + is_bias=False, + dtype=input.dtype, + ) inputs = { "X": input, "W": weights, "PathTable": path_table, "PathCode": path_code, - "Label": label + "Label": label, } if helper.bias_attr: if not is_custom: - bias = helper.create_parameter(attr=helper.bias_attr, - shape=[num_classes - 1, 1], - is_bias=True, - dtype=input.dtype) + bias = helper.create_parameter( + attr=helper.bias_attr, + shape=[num_classes - 1, 1], + is_bias=True, + dtype=input.dtype, + ) inputs['Bias'] = bias else: - bias = helper.create_parameter(attr=helper.bias_attr, - shape=[num_classes, 1], - is_bias=True, - dtype=input.dtype) + bias = helper.create_parameter( + attr=helper.bias_attr, + shape=[num_classes, 1], + is_bias=True, + dtype=input.dtype, + ) inputs['Bias'] = bias - helper.append_op(type="hierarchical_sigmoid", - inputs=inputs, - outputs={ - "Out": out, - "PreOut": pre_out, - "W_Out": weights - }, - attrs={ - "num_classes": num_classes, - "is_sparse": is_sparse, - "remote_prefetch": remote_prefetch - }) + helper.append_op( + type="hierarchical_sigmoid", + inputs=inputs, + outputs={"Out": out, "PreOut": pre_out, "W_Out": weights}, + attrs={ + "num_classes": num_classes, + "is_sparse": is_sparse, + "remote_prefetch": remote_prefetch, + }, + ) return out -def sampled_softmax_with_cross_entropy(logits, - label, - num_samples, - num_true=1, - remove_accidental_hits=True, - use_customized_samples=False, - customized_samples=None, - customized_probabilities=None, - seed=0): +def sampled_softmax_with_cross_entropy( + logits, + label, + num_samples, + num_true=1, + remove_accidental_hits=True, + use_customized_samples=False, + customized_samples=None, + customized_probabilities=None, + seed=0, +): """ **Sampled Softmax With Cross Entropy Operator.** @@ -1052,90 +1088,121 @@ def sampled_softmax_with_cross_entropy(logits, logits=fc, label=label, num_samples=25) """ if _non_static_mode(): - sample_logits_attrs = ('use_customized_samples', use_customized_samples, - 'uniq', True, 'remove_accidental_hits', - remove_accidental_hits, 'num_samples', - num_samples, 'seed', seed) - _, _, _, _, sampled_logits_out, sampled_label_out = _legacy_C_ops.sample_logits( - logits, label, *sample_logits_attrs) + sample_logits_attrs = ( + 'use_customized_samples', + use_customized_samples, + 'uniq', + True, + 'remove_accidental_hits', + remove_accidental_hits, + 'num_samples', + num_samples, + 'seed', + seed, + ) + ( + _, + _, + _, + _, + sampled_logits_out, + sampled_label_out, + ) = _legacy_C_ops.sample_logits(logits, label, *sample_logits_attrs) depth = num_samples + 1 - sampled_softlabel_out = _legacy_C_ops.one_hot(sampled_label_out, - 'depth', depth) + sampled_softlabel_out = _legacy_C_ops.one_hot( + sampled_label_out, 'depth', depth + ) - softmax_with_cross_entropy_attrs = ('soft_label', True, - 'numeric_stable_mode', False) + softmax_with_cross_entropy_attrs = ( + 'soft_label', + True, + 'numeric_stable_mode', + False, + ) _, loss = _legacy_C_ops.softmax_with_cross_entropy( - sampled_logits_out, sampled_softlabel_out, - *softmax_with_cross_entropy_attrs) + sampled_logits_out, + sampled_softlabel_out, + *softmax_with_cross_entropy_attrs + ) return loss / num_true helper = LayerHelper('sample_logits', **locals()) - samples = customized_samples if use_customized_samples else helper.create_variable_for_type_inference( - dtype='int64') - probabilities = customized_probabilities if use_customized_samples else helper.create_variable_for_type_inference( - dtype=logits.dtype) - sampled_logits \ - = helper.create_variable_for_type_inference(dtype=logits.dtype) + samples = ( + customized_samples + if use_customized_samples + else helper.create_variable_for_type_inference(dtype='int64') + ) + probabilities = ( + customized_probabilities + if use_customized_samples + else helper.create_variable_for_type_inference(dtype=logits.dtype) + ) + sampled_logits = helper.create_variable_for_type_inference( + dtype=logits.dtype + ) sampled_label = helper.create_variable_for_type_inference(dtype='int64') sampled_softlabel = helper.create_variable_for_type_inference( - dtype=logits.dtype) + dtype=logits.dtype + ) logits_dim = helper.create_variable_for_type_inference(dtype=logits.dtype) labels_dim = helper.create_variable_for_type_inference(dtype=label.type) - helper.append_op(type='sample_logits', - inputs={ - 'Logits': logits, - 'Labels': label, - 'CustomizedSamples': customized_samples, - 'CustomizedProbabilities': customized_probabilities - }, - outputs={ - 'Samples': samples, - 'Probabilities': probabilities, - 'SampledLabels': sampled_label, - 'SampledLogits': sampled_logits, - 'LogitsDim': logits_dim, - 'LabelsDim': labels_dim - }, - attrs={ - 'use_customized_samples': use_customized_samples, - 'uniq': True, - 'remove_accidental_hits': remove_accidental_hits, - 'num_samples': num_samples, - 'seed': seed - }) + helper.append_op( + type='sample_logits', + inputs={ + 'Logits': logits, + 'Labels': label, + 'CustomizedSamples': customized_samples, + 'CustomizedProbabilities': customized_probabilities, + }, + outputs={ + 'Samples': samples, + 'Probabilities': probabilities, + 'SampledLabels': sampled_label, + 'SampledLogits': sampled_logits, + 'LogitsDim': logits_dim, + 'LabelsDim': labels_dim, + }, + attrs={ + 'use_customized_samples': use_customized_samples, + 'uniq': True, + 'remove_accidental_hits': remove_accidental_hits, + 'num_samples': num_samples, + 'seed': seed, + }, + ) loss = helper.create_variable_for_type_inference(dtype=logits.dtype) softmax = helper.create_variable_for_type_inference(dtype=logits.dtype) - helper.append_op(type='one_hot', - inputs={'X': sampled_label}, - attrs={'depth': num_samples + 1}, - outputs={'Out': sampled_softlabel}) - - helper.append_op(type='softmax_with_cross_entropy', - inputs={ - 'Logits': sampled_logits, - 'Label': sampled_softlabel - }, - outputs={ - 'Softmax': softmax, - 'Loss': loss - }, - attrs={ - 'soft_label': True, - 'ignore_index': False, - 'numeric_stable_mode': False - }) + helper.append_op( + type='one_hot', + inputs={'X': sampled_label}, + attrs={'depth': num_samples + 1}, + outputs={'Out': sampled_softlabel}, + ) + + helper.append_op( + type='softmax_with_cross_entropy', + inputs={'Logits': sampled_logits, 'Label': sampled_softlabel}, + outputs={'Softmax': softmax, 'Loss': loss}, + attrs={ + 'soft_label': True, + 'ignore_index': False, + 'numeric_stable_mode': False, + }, + ) return loss / num_true -def softmax_with_cross_entropy(logits, - label, - soft_label=False, - ignore_index=kIgnoreIndex, - numeric_stable_mode=True, - return_softmax=False, - axis=-1): +def softmax_with_cross_entropy( + logits, + label, + soft_label=False, + ignore_index=kIgnoreIndex, + numeric_stable_mode=True, + return_softmax=False, + axis=-1, +): r""" This operator implements the cross entropy loss function with softmax. This function @@ -1231,8 +1298,14 @@ def softmax_with_cross_entropy(logits, print(out) """ return paddle.nn.functional.loss.fluid_softmax_with_cross_entropy( - logits, label, soft_label, ignore_index, numeric_stable_mode, - return_softmax, axis) + logits, + label, + soft_label, + ignore_index, + numeric_stable_mode, + return_softmax, + axis, + ) def identity_loss(x, reduction="none"): @@ -1285,10 +1358,9 @@ def identity_loss(x, reduction="none"): helper = LayerHelper('identity_loss', **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="identity_loss", - inputs={"X": x}, - outputs={"Out": out}, - attrs=attrs) + helper.append_op( + type="identity_loss", inputs={"X": x}, outputs={"Out": out}, attrs=attrs + ) return out @@ -1346,13 +1418,11 @@ def rank_loss(label, left, right, name=None): out = helper.create_variable_for_type_inference("float32") - helper.append_op(type='rank_loss', - inputs={ - "Label": label, - "Left": left, - "Right": right - }, - outputs={'Out': out}) + helper.append_op( + type='rank_loss', + inputs={"Label": label, "Left": left, "Right": right}, + outputs={'Out': out}, + ) return out @@ -1397,26 +1467,19 @@ def margin_rank_loss(label, left, right, margin=0.1, name=None): check_variable_and_dtype(label, 'right', ['float32'], 'margin_rank_loss') out = helper.create_variable_for_type_inference(left.dtype) act = helper.create_variable_for_type_inference(left.dtype) - helper.append_op(type='margin_rank_loss', - inputs={ - "Label": label, - "X1": left, - "X2": right - }, - outputs={ - 'Out': out, - 'Activated': act - }, - attrs={'margin': margin}) + helper.append_op( + type='margin_rank_loss', + inputs={"Label": label, "X1": left, "X2": right}, + outputs={'Out': out, 'Activated': act}, + attrs={'margin': margin}, + ) return out @templatedoc() -def sigmoid_cross_entropy_with_logits(x, - label, - ignore_index=kIgnoreIndex, - name=None, - normalize=False): +def sigmoid_cross_entropy_with_logits( + x, label, ignore_index=kIgnoreIndex, name=None, normalize=False +): """ ${comment} @@ -1453,32 +1516,32 @@ def sigmoid_cross_entropy_with_logits(x, """ if in_dygraph_mode(): - return _C_ops.sigmoid_cross_entropy_with_logits(x, label, normalize, - int(ignore_index)) - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - 'sigmoid_cross_entropy_with_logits') + return _C_ops.sigmoid_cross_entropy_with_logits( + x, label, normalize, int(ignore_index) + ) + check_variable_and_dtype( + x, + 'input', + ['float16', 'float32', 'float64'], + 'sigmoid_cross_entropy_with_logits', + ) helper = LayerHelper("sigmoid_cross_entropy_with_logits", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="sigmoid_cross_entropy_with_logits", - inputs={ - "X": x, - "Label": label - }, - attrs={ - "ignore_index": ignore_index, - 'normalize': normalize - }, - outputs={"Out": out}) + helper.append_op( + type="sigmoid_cross_entropy_with_logits", + inputs={"X": x, "Label": label}, + attrs={"ignore_index": ignore_index, 'normalize': normalize}, + outputs={"Out": out}, + ) return out -def teacher_student_sigmoid_loss(input, - label, - soft_max_up_bound=15.0, - soft_max_lower_bound=-15.0): +def teacher_student_sigmoid_loss( + input, label, soft_max_up_bound=15.0, soft_max_lower_bound=-15.0 +): """ **Teacher Student Log Loss Layer** @@ -1517,22 +1580,30 @@ def teacher_student_sigmoid_loss(input, cost = fluid.layers.teacher_student_sigmoid_loss(input=similarity, label=label) """ - check_variable_and_dtype(input, "input", - ['float32', 'float64', 'int32', 'int64'], - 'teacher_student_sigmoid_loss') - check_variable_and_dtype(label, "label", - ['float32', 'float64', 'int32', 'int64'], - 'teacher_student_sigmoid_loss') + check_variable_and_dtype( + input, + "input", + ['float32', 'float64', 'int32', 'int64'], + 'teacher_student_sigmoid_loss', + ) + check_variable_and_dtype( + label, + "label", + ['float32', 'float64', 'int32', 'int64'], + 'teacher_student_sigmoid_loss', + ) helper = LayerHelper('teacher_student_sigmoid_loss', **locals()) out = helper.create_variable(dtype=input.dtype) helper.append_op( type='teacher_student_sigmoid_loss', - inputs={'X': [input], - 'Label': [label]}, + inputs={'X': [input], 'Label': [label]}, outputs={'Y': [out]}, - attrs={"soft_max_lower_bound": float(soft_max_lower_bound), \ - "soft_max_up_bound": float(soft_max_up_bound)}) + attrs={ + "soft_max_lower_bound": float(soft_max_lower_bound), + "soft_max_up_bound": float(soft_max_up_bound), + }, + ) return out @@ -1587,23 +1658,22 @@ def huber_loss(input, label, delta): return out helper = LayerHelper('huber_loss', **locals()) - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'huber_loss') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'huber_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'huber_loss' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'huber_loss' + ) residual = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='huber_loss', - inputs={ - 'X': input, - 'Y': label - }, - outputs={ - 'Out': out, - 'Residual': residual - }, - attrs={'delta': delta}) + helper.append_op( + type='huber_loss', + inputs={'X': input, 'Y': label}, + outputs={'Out': out, 'Residual': residual}, + attrs={'delta': delta}, + ) return out @@ -1653,17 +1723,17 @@ def kldiv_loss(x, target, reduction='mean', name=None): """ helper = LayerHelper('kldiv_loss', **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'kldiv_loss') - check_variable_and_dtype(target, 'target', ['float32', 'float64'], - 'kldiv_loss') + check_variable_and_dtype( + target, 'target', ['float32', 'float64'], 'kldiv_loss' + ) check_type(reduction, 'reduction', str, 'kldiv_loss') loss = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='kldiv_loss', - inputs={ - 'X': x, - 'Target': target - }, - outputs={'Loss': loss}, - attrs={'reduction': reduction}) + helper.append_op( + type='kldiv_loss', + inputs={'X': x, 'Target': target}, + outputs={'Loss': loss}, + attrs={'reduction': reduction}, + ) return loss diff --git a/python/paddle/fluid/layers/math_op_patch.py b/python/paddle/fluid/layers/math_op_patch.py index a0fd6782db633..fc71321f14732 100644 --- a/python/paddle/fluid/layers/math_op_patch.py +++ b/python/paddle/fluid/layers/math_op_patch.py @@ -53,14 +53,13 @@ "__lt__": "A < B", "__le__": "A <= B", "__gt__": "A > B", - "__ge__": "A >= B" + "__ge__": "A >= B", } _already_patch_variable = False def monkey_patch_variable(): - def unique_tmp_name(): return unique_name.generate("tmp") @@ -85,15 +84,17 @@ def create_new_tmp_sparse_var(block, dtype, type): def create_tensor(block, value, dtype, shape): value = float(value) var = create_new_tmp_var(block, dtype) - block.append_op(type="fill_constant", - outputs={'Out': [var]}, - attrs={ - 'dtype': var.dtype, - 'shape': shape, - 'value': value, - 'force_cpu': False - }, - stop_gradient=True) + block.append_op( + type="fill_constant", + outputs={'Out': [var]}, + attrs={ + 'dtype': var.dtype, + 'shape': shape, + 'value': value, + 'force_cpu': False, + }, + stop_gradient=True, + ) var.stop_gradient = True return var @@ -117,16 +118,18 @@ def create_tensor_with_batchsize(ref_var, value, dtype): else: out_shape.append(d) assert batch_dim != -1 - block.append_op(type='fill_constant_batch_size_like', - outputs={'Out': [var]}, - inputs={'Input': [ref_var]}, - attrs={ - 'shape': out_shape, - 'value': value, - 'input_dim_idx': batch_dim, - 'output_dim_idx': batch_dim - }, - stop_gradient=True) + block.append_op( + type='fill_constant_batch_size_like', + outputs={'Out': [var]}, + inputs={'Input': [ref_var]}, + attrs={ + 'shape': out_shape, + 'value': value, + 'input_dim_idx': batch_dim, + 'output_dim_idx': batch_dim, + }, + stop_gradient=True, + ) var.stop_gradient = True return var @@ -208,37 +211,40 @@ def astype(self, dtype): """ block = current_block(self) out = create_new_tmp_var(block, dtype) - block.append_op(type="cast", - inputs={"X": [self]}, - outputs={"Out": [out]}, - attrs={ - "in_dtype": self.dtype, - "out_dtype": out.dtype - }) + block.append_op( + type="cast", + inputs={"X": [self]}, + outputs={"Out": [out]}, + attrs={"in_dtype": self.dtype, "out_dtype": out.dtype}, + ) out.stop_gradient = self.stop_gradient return out @static_only def append(self, var): """ - **Notes**: - **The type variable must be LoD Tensor Array. + **Notes**: + **The type variable must be LoD Tensor Array. """ if not isinstance(var, Variable): if in_declarative_mode(): - """ in dy2static mode, x may be tensorable values such as int, float, np.array - """ + """in dy2static mode, x may be tensorable values such as int, float, np.array""" from paddle.tensor.creation import to_tensor + var = to_tensor(var) else: raise TypeError( - "Required input var should be Variable, but received {}". - format(type(var))) + "Required input var should be Variable, but received {}".format( + type(var) + ) + ) if self.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: raise TypeError( - "Only Variable with VarType.LOD_TENSOR_ARRAY support `append` method, but received type: {}" - .format(self.type)) + "Only Variable with VarType.LOD_TENSOR_ARRAY support `append` method, but received type: {}".format( + self.type + ) + ) array_write(x=var, i=array_length(self), array=self) @static_only @@ -249,8 +255,10 @@ def _item(self): """ if len(self.shape) > 1: raise TypeError( - "Required input var should be 1-D Variable, but received {}". - format(self.shape)) + "Required input var should be 1-D Variable, but received {}".format( + self.shape + ) + ) return self @static_only @@ -266,23 +274,27 @@ def pop(self, *args): Returns: Variable: self[index] """ - from paddle.fluid.dygraph.dygraph_to_static.convert_operators import _run_paddle_pop + from paddle.fluid.dygraph.dygraph_to_static.convert_operators import ( + _run_paddle_pop, + ) + if self.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: raise TypeError( - "Only Variable with VarType.LOD_TENSOR_ARRAY support `append` method, but received type: {}" - .format(self.type)) + "Only Variable with VarType.LOD_TENSOR_ARRAY support `append` method, but received type: {}".format( + self.type + ) + ) return _run_paddle_pop(self, *args) def _scalar_op_(var, scale, bias): block = current_block(var) out = create_new_tmp_var(block, var.dtype) - block.append_op(type="scale", - inputs={"X": [var]}, - outputs={"Out": [out]}, - attrs={ - "scale": scale, - "bias": bias - }) + block.append_op( + type="scale", + inputs={"X": [var]}, + outputs={"Out": [out]}, + attrs={"scale": scale, "bias": bias}, + ) return out def _neg_(var): @@ -325,11 +337,9 @@ def _scalar_mul_(var, value): def _scalar_div_(var, value): return _scalar_op_(var, 1.0 / value, 0.0) - def _binary_creator_(method_name, - op_type, - reverse=False, - scalar_method=None): - + def _binary_creator_( + method_name, op_type, reverse=False, scalar_method=None + ): def __impl__(self, other_var): # 1. scalar exists cases # we need combine the tensor.dtype and scalar.dtype, cast correct object @@ -352,7 +362,10 @@ def __impl__(self, other_var): # so the calculation result here and the calculation result of numpy are # different after 6 decimal point. If necessary, we can also use float64 here. # torch's behavior here is consistent with ours - if op_type == 'elementwise_div' and self.dtype in _supported_int_dtype_: + if ( + op_type == 'elementwise_div' + and self.dtype in _supported_int_dtype_ + ): self = astype(self, 'float32') # here use `scale` replace `elementwise` to get better performance # but only +, -, *, / can use this method @@ -372,18 +385,21 @@ def __impl__(self, other_var): has_batch_size = True break if not has_batch_size: - other_var = create_tensor(current_block(self), - other_var, - dtype=lhs_dtype, - shape=self.shape) + other_var = create_tensor( + current_block(self), + other_var, + dtype=lhs_dtype, + shape=self.shape, + ) else: other_var = create_tensor_with_batchsize( - self, other_var, lhs_dtype) + self, other_var, lhs_dtype + ) else: # add fill_op to current_block - other_var = create_scalar(current_block(self), - value=other_var, - dtype=lhs_dtype) + other_var = create_scalar( + current_block(self), value=other_var, dtype=lhs_dtype + ) # 3. unify right var type to left var rhs_dtype = safe_get_dtype(other_var) @@ -409,16 +425,22 @@ def __impl__(self, other_var): "%s:%s\nThe behavior of expression %s has been unified with %s(X, Y, axis=-1) from Paddle 2.0. " "If your code works well in the older versions but crashes in this version, try to use " "%s(X, Y, axis=0) instead of %s. This transitional warning will be dropped in the future." - % (file_name, line_num, EXPRESSION_MAP[method_name], - op_type, op_type, EXPRESSION_MAP[method_name]), - category=DeprecationWarning) - current_block(self).append_op(type=op_type, - inputs={ - 'X': [self], - 'Y': [other_var] - }, - outputs={'Out': out}, - attrs={'axis': axis}) + % ( + file_name, + line_num, + EXPRESSION_MAP[method_name], + op_type, + op_type, + EXPRESSION_MAP[method_name], + ), + category=DeprecationWarning, + ) + current_block(self).append_op( + type=op_type, + inputs={'X': [self], 'Y': [other_var]}, + outputs={'Out': out}, + attrs={'axis': axis}, + ) return out comment = OpProtoHolder.instance().get_op_proto(op_type).comment @@ -431,35 +453,43 @@ def __impl__(self, other_var): Returns: Variable - """.format(comment) + """.format( + comment + ) __impl__.__name__ = method_name return __impl__ def values(var): block = current_block(var) out = create_new_tmp_var(block, var.dtype) - block.append_op(type="sparse_values", - inputs={"x": [var]}, - outputs={"out": [out]}, - attrs={}) + block.append_op( + type="sparse_values", + inputs={"x": [var]}, + outputs={"out": [out]}, + attrs={}, + ) return out def indices(var): block = current_block(var) out = create_new_tmp_var(block, var.dtype) - block.append_op(type="sparse_indices", - inputs={"x": [var]}, - outputs={"out": [out]}, - attrs={}) + block.append_op( + type="sparse_indices", + inputs={"x": [var]}, + outputs={"out": [out]}, + attrs={}, + ) return out def to_dense(var): block = current_block(var) out = create_new_tmp_var(block, var.dtype) - block.append_op(type="sparse_to_dense", - inputs={"x": [var]}, - outputs={"out": [out]}, - attrs={}) + block.append_op( + type="sparse_to_dense", + inputs={"x": [var]}, + outputs={"out": [out]}, + attrs={}, + ) return out variable_methods = [ @@ -475,39 +505,78 @@ def to_dense(var): ('dim', lambda x: len(x.shape)), ('ndimension', lambda x: len(x.shape)), ('ndim', _ndim_), - ('__add__', - _binary_creator_('__add__', 'elementwise_add', False, _scalar_add_)), + ( + '__add__', + _binary_creator_('__add__', 'elementwise_add', False, _scalar_add_), + ), # a+b == b+a. Do not need to reverse explicitly - ('__radd__', - _binary_creator_('__radd__', 'elementwise_add', False, _scalar_add_)), - ('__sub__', - _binary_creator_('__sub__', 'elementwise_sub', False, _scalar_sub_)), - ('__rsub__', - _binary_creator_('__rsub__', 'elementwise_sub', True, _scalar_rsub_)), - ('__mul__', - _binary_creator_('__mul__', 'elementwise_mul', False, _scalar_mul_)), + ( + '__radd__', + _binary_creator_( + '__radd__', 'elementwise_add', False, _scalar_add_ + ), + ), + ( + '__sub__', + _binary_creator_('__sub__', 'elementwise_sub', False, _scalar_sub_), + ), + ( + '__rsub__', + _binary_creator_( + '__rsub__', 'elementwise_sub', True, _scalar_rsub_ + ), + ), + ( + '__mul__', + _binary_creator_('__mul__', 'elementwise_mul', False, _scalar_mul_), + ), # a*b == b*a. Do not need to reverse explicitly - ('__rmul__', - _binary_creator_('__rmul__', 'elementwise_mul', False, _scalar_mul_)), - ('__div__', - _binary_creator_('__div__', 'elementwise_div', False, _scalar_div_)), - ('__truediv__', - _binary_creator_('__truediv__', 'elementwise_div', False, - _scalar_div_)), - ('__rdiv__', _binary_creator_('__rdiv__', 'elementwise_div', True, - None)), - ('__rtruediv__', - _binary_creator_('__rtruediv__', 'elementwise_div', True, None)), - ('__pow__', _binary_creator_('__pow__', 'elementwise_pow', False, - None)), - ('__rpow__', _binary_creator_('__rpow__', 'elementwise_pow', True, - None)), - ('__floordiv__', - _binary_creator_('__floordiv__', 'elementwise_floordiv', False, None)), - ('__mod__', _binary_creator_('__mod__', 'elementwise_mod', False, - None)), - ('__matmul__', _binary_creator_('__matmul__', "matmul_v2", False, - None)), + ( + '__rmul__', + _binary_creator_( + '__rmul__', 'elementwise_mul', False, _scalar_mul_ + ), + ), + ( + '__div__', + _binary_creator_('__div__', 'elementwise_div', False, _scalar_div_), + ), + ( + '__truediv__', + _binary_creator_( + '__truediv__', 'elementwise_div', False, _scalar_div_ + ), + ), + ( + '__rdiv__', + _binary_creator_('__rdiv__', 'elementwise_div', True, None), + ), + ( + '__rtruediv__', + _binary_creator_('__rtruediv__', 'elementwise_div', True, None), + ), + ( + '__pow__', + _binary_creator_('__pow__', 'elementwise_pow', False, None), + ), + ( + '__rpow__', + _binary_creator_('__rpow__', 'elementwise_pow', True, None), + ), + ( + '__floordiv__', + _binary_creator_( + '__floordiv__', 'elementwise_floordiv', False, None + ), + ), + ( + '__mod__', + _binary_creator_('__mod__', 'elementwise_mod', False, None), + ), + ( + '__matmul__', + _binary_creator_('__matmul__', "matmul_v2", False, None), + ), # for logical compare ('__eq__', _binary_creator_('__eq__', 'equal', False, None)), ('__ne__', _binary_creator_('__ne__', 'not_equal', False, None)), @@ -528,13 +597,17 @@ def to_dense(var): setattr(Variable, method_name, method_impl) else: import paddle.tensor + for method_name in paddle.tensor.tensor_method_func: - if hasattr(Variable, method_name): continue + if hasattr(Variable, method_name): + continue method_impl = getattr(paddle.tensor, method_name, None) - if method_impl: setattr(Variable, method_name, method_impl) + if method_impl: + setattr(Variable, method_name, method_impl) for magic_method, origin_method in paddle.tensor.magic_method_func: impl = getattr(paddle.tensor, origin_method, None) - if impl: setattr(Variable, magic_method, impl) + if impl: + setattr(Variable, magic_method, impl) _already_patch_variable = True diff --git a/python/paddle/fluid/layers/metric_op.py b/python/paddle/fluid/layers/metric_op.py index 36679f13b5a43..8a63b55089e8b 100755 --- a/python/paddle/fluid/layers/metric_op.py +++ b/python/paddle/fluid/layers/metric_op.py @@ -18,7 +18,13 @@ import warnings from ..layer_helper import LayerHelper from ..initializer import Normal, Constant -from ..framework import Variable, _non_static_mode, _varbase_creator, _in_legacy_dygraph, in_dygraph_mode +from ..framework import ( + Variable, + _non_static_mode, + _varbase_creator, + _in_legacy_dygraph, + in_dygraph_mode, +) from .. import core from ..param_attr import ParamAttr from . import nn @@ -74,15 +80,18 @@ def accuracy(input, label, k=1, correct=None, total=None): total = _varbase_creator(dtype="int32") _k = k.numpy().item(0) if isinstance(k, Variable) else k - topk_out, topk_indices = _legacy_C_ops.top_k_v2(input, 'k', _k, - 'sorted', False) - _acc, _, _ = _legacy_C_ops.accuracy(topk_out, topk_indices, label, - correct, total) + topk_out, topk_indices = _legacy_C_ops.top_k_v2( + input, 'k', _k, 'sorted', False + ) + _acc, _, _ = _legacy_C_ops.accuracy( + topk_out, topk_indices, label, correct, total + ) return _acc helper = LayerHelper("accuracy", **locals()) - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'accuracy') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'accuracy' + ) topk_out = helper.create_variable_for_type_inference(dtype=input.dtype) topk_indices = helper.create_variable_for_type_inference(dtype="int64") inputs = {"X": [input]} @@ -91,39 +100,38 @@ def accuracy(input, label, k=1, correct=None, total=None): else: attrs = {'k': k} attrs['sorted'] = False - helper.append_op(type="top_k_v2", - inputs=inputs, - attrs=attrs, - outputs={ - "Out": [topk_out], - "Indices": [topk_indices] - }) + helper.append_op( + type="top_k_v2", + inputs=inputs, + attrs=attrs, + outputs={"Out": [topk_out], "Indices": [topk_indices]}, + ) acc_out = helper.create_variable_for_type_inference(dtype="float32") if correct is None: correct = helper.create_variable_for_type_inference(dtype="int32") if total is None: total = helper.create_variable_for_type_inference(dtype="int32") - helper.append_op(type="accuracy", - inputs={ - "Out": [topk_out], - "Indices": [topk_indices], - "Label": [label] - }, - outputs={ - "Accuracy": [acc_out], - "Correct": [correct], - "Total": [total], - }) + helper.append_op( + type="accuracy", + inputs={"Out": [topk_out], "Indices": [topk_indices], "Label": [label]}, + outputs={ + "Accuracy": [acc_out], + "Correct": [correct], + "Total": [total], + }, + ) return acc_out -def auc(input, - label, - curve='ROC', - num_thresholds=2**12 - 1, - topk=1, - slide_steps=1, - ins_tag_weight=None): +def auc( + input, + label, + curve='ROC', + num_thresholds=2**12 - 1, + topk=1, + slide_steps=1, + ins_tag_weight=None, +): """ **Area Under the Curve (AUC) Layer** @@ -214,13 +222,14 @@ def auc(input, helper = LayerHelper("auc", **locals()) if ins_tag_weight is None: - ins_tag_weight = tensor.fill_constant(shape=[1, 1], - dtype="float32", - value=1.0) + ins_tag_weight = tensor.fill_constant( + shape=[1, 1], dtype="float32", value=1.0 + ) check_variable_and_dtype(input, 'input', ['float32', 'float64'], 'auc') check_variable_and_dtype(label, 'label', ['int32', 'int64'], 'auc') - check_variable_and_dtype(ins_tag_weight, 'ins_tag_weight', - ['float32', 'float64'], 'auc') + check_variable_and_dtype( + ins_tag_weight, 'ins_tag_weight', ['float32', 'float64'], 'auc' + ) auc_out = helper.create_variable_for_type_inference(dtype="float64") batch_auc_out = helper.create_variable_for_type_inference(dtype="float64") # make tp, tn, fp, fn persistable, so that can accumulate all batches. @@ -234,62 +243,71 @@ def auc(input, batch_stat_pos = helper.create_global_variable( persistable=True, dtype='int64', - shape=[(1 + slide_steps) * (num_thresholds + 1) + 1]) + shape=[(1 + slide_steps) * (num_thresholds + 1) + 1], + ) batch_stat_neg = helper.create_global_variable( persistable=True, dtype='int64', - shape=[(1 + slide_steps) * (num_thresholds + 1) + 1]) + shape=[(1 + slide_steps) * (num_thresholds + 1) + 1], + ) # for global auc # Needn't maintain the batch id - stat_pos = helper.create_global_variable(persistable=True, - dtype='int64', - shape=[1, num_thresholds + 1]) - stat_neg = helper.create_global_variable(persistable=True, - dtype='int64', - shape=[1, num_thresholds + 1]) + stat_pos = helper.create_global_variable( + persistable=True, dtype='int64', shape=[1, num_thresholds + 1] + ) + stat_neg = helper.create_global_variable( + persistable=True, dtype='int64', shape=[1, num_thresholds + 1] + ) for var in [batch_stat_pos, batch_stat_neg, stat_pos, stat_neg]: - helper.set_variable_initializer(var, Constant(value=0.0, - force_cpu=False)) + helper.set_variable_initializer( + var, Constant(value=0.0, force_cpu=False) + ) - #"InsTagWeight": [ins_tag_weight] + # "InsTagWeight": [ins_tag_weight] # Batch AUC - helper.append_op(type="auc", - inputs={ - "Predict": [input], - "Label": [label], - "StatPos": [batch_stat_pos], - "StatNeg": [batch_stat_neg] - }, - attrs={ - "curve": curve, - "num_thresholds": num_thresholds, - "slide_steps": slide_steps - }, - outputs={ - "AUC": [batch_auc_out], - "StatPosOut": [batch_stat_pos], - "StatNegOut": [batch_stat_neg] - }) + helper.append_op( + type="auc", + inputs={ + "Predict": [input], + "Label": [label], + "StatPos": [batch_stat_pos], + "StatNeg": [batch_stat_neg], + }, + attrs={ + "curve": curve, + "num_thresholds": num_thresholds, + "slide_steps": slide_steps, + }, + outputs={ + "AUC": [batch_auc_out], + "StatPosOut": [batch_stat_pos], + "StatNegOut": [batch_stat_neg], + }, + ) # Global AUC - helper.append_op(type="auc", - inputs={ - "Predict": [input], - "Label": [label], - "StatPos": [stat_pos], - "StatNeg": [stat_neg] - }, - attrs={ - "curve": curve, - "num_thresholds": num_thresholds, - "slide_steps": 0 - }, - outputs={ - "AUC": [auc_out], - "StatPosOut": [stat_pos], - "StatNegOut": [stat_neg] - }) - return auc_out, batch_auc_out, [ - batch_stat_pos, batch_stat_neg, stat_pos, stat_neg - ] + helper.append_op( + type="auc", + inputs={ + "Predict": [input], + "Label": [label], + "StatPos": [stat_pos], + "StatNeg": [stat_neg], + }, + attrs={ + "curve": curve, + "num_thresholds": num_thresholds, + "slide_steps": 0, + }, + outputs={ + "AUC": [auc_out], + "StatPosOut": [stat_pos], + "StatNegOut": [stat_neg], + }, + ) + return ( + auc_out, + batch_auc_out, + [batch_stat_pos, batch_stat_neg, stat_pos, stat_neg], + ) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 38d1b30f0a471..9f7cbb1141193 100755 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -25,18 +25,39 @@ from ..layer_helper import LayerHelper from paddle.fluid.framework import _in_legacy_dygraph from ..initializer import Normal, Constant, NumpyArrayInitializer -from ..framework import Variable, OpProtoHolder, _non_static_mode, dygraph_only, _dygraph_tracer, default_main_program, _varbase_creator, static_only, _global_flags, _in_legacy_dygraph, in_dygraph_mode +from ..framework import ( + Variable, + OpProtoHolder, + _non_static_mode, + dygraph_only, + _dygraph_tracer, + default_main_program, + _varbase_creator, + static_only, + _global_flags, + _in_legacy_dygraph, + in_dygraph_mode, +) from ..framework import _current_expected_place from .. import dygraph_utils from ..param_attr import ParamAttr -from .layer_function_generator import autodoc, templatedoc, _generate_doc_string_ +from .layer_function_generator import ( + autodoc, + templatedoc, + _generate_doc_string_, +) from .tensor import concat, assign, fill_constant, zeros, tensor_array_to_tensor from . import utils from .. import unique_name from functools import reduce from .. import core from ...utils import deprecated -from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) import paddle from paddle.utils import deprecated from paddle import _C_ops, _legacy_C_ops @@ -219,8 +240,10 @@ def _get_reduce_dim(dim, input): dim = [dim] else: raise TypeError( - "The type of dim must be int, list, tuple or range, but received {}" - .format(type(axis))) + "The type of dim must be int, list, tuple or range, but received {}".format( + type(axis) + ) + ) if dim is None: dim = [] if dim == [] or len(dim) == len(input.shape): @@ -232,13 +255,9 @@ def _get_reduce_dim(dim, input): @dygraph_only -def _elementwise_op_in_dygraph(x, - y, - axis=-1, - act=None, - use_mkldnn=False, - op_name=None): - +def _elementwise_op_in_dygraph( + x, y, axis=-1, act=None, use_mkldnn=False, op_name=None +): def is_inplace(op_name): return op_name[-1] == "_" @@ -249,24 +268,27 @@ def is_inplace(op_name): if in_dygraph_mode(): op = getattr( _C_ops, - OP_NAMEMAPPING[op_name] if not is_inplace(op_name) else op_name) + OP_NAMEMAPPING[op_name] if not is_inplace(op_name) else op_name, + ) out = op(x, y) if _in_legacy_dygraph(): op = getattr(_legacy_C_ops, op_name) out = op(x, y, 'axis', axis, 'use_mkldnn', use_mkldnn) - return dygraph_utils._append_activation_in_dygraph(out, - act, - use_mkldnn=use_mkldnn) - - -def fc(input, - size, - num_flatten_dims=1, - param_attr=None, - bias_attr=None, - act=None, - name=None): + return dygraph_utils._append_activation_in_dygraph( + out, act, use_mkldnn=use_mkldnn + ) + + +def fc( + input, + size, + num_flatten_dims=1, + param_attr=None, + bias_attr=None, + act=None, + name=None, +): r""" :api_attr: Static Graph @@ -384,8 +406,9 @@ def fc(input, for i, input_x in enumerate(input): check_type(input_x, 'input[' + str(i) + ']', Variable, 'fc') dtype = helper.input_dtype() - check_dtype(dtype, 'input', ['float16', 'uint16', 'float32', 'float64'], - 'fc') + check_dtype( + dtype, 'input', ['float16', 'uint16', 'float32', 'float64'], 'fc' + ) mul_results = [] for input_var, param_attr in helper.iter_inputs_and_params(): input_shape = input_var.shape @@ -395,31 +418,28 @@ def fc(input, reduce(lambda a, b: a * b, input_shape[num_flatten_dims:], 1) ] + [size] - w = helper.create_parameter(attr=param_attr, - shape=param_shape, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=param_attr, shape=param_shape, dtype=dtype, is_bias=False + ) tmp = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="mul", - inputs={ - "X": input_var, - "Y": w - }, - outputs={"Out": tmp}, - attrs={ - "x_num_col_dims": num_flatten_dims, - "y_num_col_dims": 1 - }) + helper.append_op( + type="mul", + inputs={"X": input_var, "Y": w}, + outputs={"Out": tmp}, + attrs={"x_num_col_dims": num_flatten_dims, "y_num_col_dims": 1}, + ) mul_results.append(tmp) if len(mul_results) == 1: pre_bias = mul_results[0] else: pre_bias = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="sum", - inputs={"X": mul_results}, - outputs={"Out": pre_bias}, - attrs={"use_mkldnn": False}) + helper.append_op( + type="sum", + inputs={"X": mul_results}, + outputs={"Out": pre_bias}, + attrs={"use_mkldnn": False}, + ) # add bias pre_activation = helper.append_bias_op(pre_bias, dim_start=num_flatten_dims) # add activation @@ -427,13 +447,15 @@ def fc(input, @deprecated(since="2.0.0", update_to="paddle.nn.functional.embedding") -def embedding(input, - size, - is_sparse=False, - is_distributed=False, - padding_idx=None, - param_attr=None, - dtype='float32'): +def embedding( + input, + size, + is_sparse=False, + is_distributed=False, + padding_idx=None, + param_attr=None, + dtype='float32', +): r""" :api_attr: Static Graph @@ -546,10 +568,15 @@ def embedding(input, """ helper = LayerHelper('embedding', **locals()) - check_variable_and_dtype(input, 'input', ['int64'], - 'fluid.layers.embedding') - check_dtype(dtype, 'dtype', ['uint16', 'float16', 'float32', 'float64'], - 'fluid.layers.embedding') + check_variable_and_dtype( + input, 'input', ['int64'], 'fluid.layers.embedding' + ) + check_dtype( + dtype, + 'dtype', + ['uint16', 'float16', 'float32', 'float64'], + 'fluid.layers.embedding', + ) if is_distributed: is_distributed = False @@ -559,37 +586,42 @@ def embedding(input, remote_prefetch = True if is_sparse else False - w = helper.create_parameter(attr=helper.param_attr, - shape=size, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=helper.param_attr, shape=size, dtype=dtype, is_bias=False + ) tmp = helper.create_variable_for_type_inference(dtype) - padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( - size[0] + padding_idx) - helper.append_op(type='lookup_table', - inputs={ - 'Ids': input, - 'W': w - }, - outputs={'Out': tmp}, - attrs={ - 'is_sparse': is_sparse, - 'is_distributed': is_distributed, - 'remote_prefetch': remote_prefetch, - 'padding_idx': padding_idx - }) + padding_idx = ( + -1 + if padding_idx is None + else padding_idx + if padding_idx >= 0 + else (size[0] + padding_idx) + ) + helper.append_op( + type='lookup_table', + inputs={'Ids': input, 'W': w}, + outputs={'Out': tmp}, + attrs={ + 'is_sparse': is_sparse, + 'is_distributed': is_distributed, + 'remote_prefetch': remote_prefetch, + 'padding_idx': padding_idx, + }, + ) return tmp -def _pull_sparse(input, - size, - table_id, - accessor_class, - name="embedding", - ctr_label_name="", - padding_id=0, - dtype='float32', - scale_sparse_grad=True): +def _pull_sparse( + input, + size, + table_id, + accessor_class, + name="embedding", + ctr_label_name="", + padding_id=0, + dtype='float32', + scale_sparse_grad=True, +): r""" **Pull Fleet Sparse Layer** @@ -636,35 +668,34 @@ def _pull_sparse(input, 'ScaleSparseGrad': scale_sparse_grad, 'InputNames': input_names, # this is only for compatible with embedding op - 'is_distributed': True + 'is_distributed': True, } # this is only for compatible with embedding op - w, _ = helper.create_or_get_global_variable(name=name, - shape=[size], - dtype=dtype, - is_bias=False, - persistable=True) - helper.append_op(type='pull_sparse', - inputs={ - 'Ids': inputs, - 'W': w - }, - outputs={'Out': outs}, - attrs=attrs) + w, _ = helper.create_or_get_global_variable( + name=name, shape=[size], dtype=dtype, is_bias=False, persistable=True + ) + helper.append_op( + type='pull_sparse', + inputs={'Ids': inputs, 'W': w}, + outputs={'Out': outs}, + attrs=attrs, + ) if len(outs) == 1: return outs[0] return outs -def _pull_sparse_v2(input, - size, - table_id, - accessor_class, - name="embedding", - ctr_label_name="", - padding_id=0, - dtype='float32', - scale_sparse_grad=True): +def _pull_sparse_v2( + input, + size, + table_id, + accessor_class, + name="embedding", + ctr_label_name="", + padding_id=0, + dtype='float32', + scale_sparse_grad=True, +): r""" **Pull Fleet Sparse Layer** @@ -711,31 +742,26 @@ def _pull_sparse_v2(input, 'ScaleSparseGrad': scale_sparse_grad, 'InputNames': input_names, # this is only for compatible with embedding op - 'is_distributed': True + 'is_distributed': True, } # this is only for compatible with embedding op - w, _ = helper.create_or_get_global_variable(name=name, - shape=[size], - dtype=dtype, - is_bias=False, - persistable=True) - helper.append_op(type='pull_sparse_v2', - inputs={ - 'Ids': inputs, - 'W': w - }, - outputs={'Out': outs}, - attrs=attrs) + w, _ = helper.create_or_get_global_variable( + name=name, shape=[size], dtype=dtype, is_bias=False, persistable=True + ) + helper.append_op( + type='pull_sparse_v2', + inputs={'Ids': inputs, 'W': w}, + outputs={'Out': outs}, + attrs=attrs, + ) if len(outs) == 1: return outs[0] return outs -def _pull_gpups_sparse(input, - size, - dtype='float32', - is_distributed=False, - is_sparse=False): +def _pull_gpups_sparse( + input, size, dtype='float32', is_distributed=False, is_sparse=False +): r""" **Pull GpuPS Sparse Layer** @@ -769,39 +795,36 @@ def _pull_gpups_sparse(input, helper = LayerHelper('pull_gpups_sparse', **locals()) if dtype != 'float32': raise ValueError( - "GpuPS only support float type embedding now, and your type is: " + - dtype) + "GpuPS only support float type embedding now, and your type is: " + + dtype + ) helper.input_dtype() inputs = helper.multiple_input() outs = [ helper.create_variable_for_type_inference(dtype) for i in range(len(inputs)) ] - w = helper.create_parameter(attr=helper.param_attr, - shape=[size[0]], - dtype=dtype, - is_bias=False) - helper.append_op(type='pull_gpups_sparse', - inputs={ - 'Ids': inputs, - 'W': w - }, - outputs={'Out': outs}, - attrs={ - 'size': size, - 'is_distributed': is_distributed, - 'is_sparse': is_sparse - }) + w = helper.create_parameter( + attr=helper.param_attr, shape=[size[0]], dtype=dtype, is_bias=False + ) + helper.append_op( + type='pull_gpups_sparse', + inputs={'Ids': inputs, 'W': w}, + outputs={'Out': outs}, + attrs={ + 'size': size, + 'is_distributed': is_distributed, + 'is_sparse': is_sparse, + }, + ) if len(outs) == 1: return outs[0] return outs -def _pull_box_sparse(input, - size, - dtype='float32', - is_distributed=False, - is_sparse=False): +def _pull_box_sparse( + input, size, dtype='float32', is_distributed=False, is_sparse=False +): r""" **Pull Box Sparse Layer** @@ -831,29 +854,28 @@ def _pull_box_sparse(input, helper = LayerHelper('pull_box_sparse', **locals()) if dtype != 'float32': raise ValueError( - "BoxPS only support float type embedding now, and your type is: " + - dtype) + "BoxPS only support float type embedding now, and your type is: " + + dtype + ) helper.input_dtype() inputs = helper.multiple_input() outs = [ helper.create_variable_for_type_inference(dtype) for i in range(len(inputs)) ] - w = helper.create_parameter(attr=helper.param_attr, - shape=[size], - dtype=dtype, - is_bias=False) - helper.append_op(type='pull_box_sparse', - inputs={ - 'Ids': inputs, - 'W': w - }, - outputs={'Out': outs}, - attrs={ - 'size': size, - 'is_distributed': is_distributed, - 'is_sparse': is_sparse - }) + w = helper.create_parameter( + attr=helper.param_attr, shape=[size], dtype=dtype, is_bias=False + ) + helper.append_op( + type='pull_box_sparse', + inputs={'Ids': inputs, 'W': w}, + outputs={'Out': outs}, + attrs={ + 'size': size, + 'is_distributed': is_distributed, + 'is_sparse': is_sparse, + }, + ) if len(outs) == 1: return outs[0] return outs @@ -949,37 +971,46 @@ def linear_chain_crf(input, label, param_attr=None, length=None): print(transition) """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'linear_chain_crf') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'linear_chain_crf' + ) check_variable_and_dtype(label, 'label', ['int64'], 'linear_chain_crf') helper = LayerHelper('linear_chain_crf', **locals()) size = input.shape[2] if length else input.shape[1] - transition = helper.create_parameter(attr=helper.param_attr, - shape=[size + 2, size], - dtype=helper.input_dtype()) + transition = helper.create_parameter( + attr=helper.param_attr, + shape=[size + 2, size], + dtype=helper.input_dtype(), + ) alpha = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) emission_exps = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) transition_exps = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) log_likelihood = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) this_inputs = { "Emission": [input], "Transition": transition, - "Label": [label] + "Label": [label], } if length: this_inputs['Length'] = [length] - helper.append_op(type='linear_chain_crf', - inputs=this_inputs, - outputs={ - "Alpha": [alpha], - "EmissionExps": [emission_exps], - "TransitionExps": transition_exps, - "LogLikelihood": log_likelihood - }) + helper.append_op( + type='linear_chain_crf', + inputs=this_inputs, + outputs={ + "Alpha": [alpha], + "EmissionExps": [emission_exps], + "TransitionExps": transition_exps, + "LogLikelihood": log_likelihood, + }, + ) return log_likelihood @@ -1035,18 +1066,22 @@ def crf_decoding(input, param_attr, label=None, length=None): crf_decode = paddle.static.nn.crf_decoding(input=emission, length=length, param_attr=paddle.ParamAttr(name="crfw_pad")) """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'crf_decoding') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'crf_decoding' + ) helper = LayerHelper('crf_decoding', **locals()) transition = helper.get_parameter(param_attr.name) viterbi_path = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.INT64) + dtype=core.VarDesc.VarType.INT64 + ) inputs = {"Emission": [input], "Transition": transition, "Label": label} if length: inputs['Length'] = length - helper.append_op(type='crf_decoding', - inputs=inputs, - outputs={"ViterbiPath": [viterbi_path]}) + helper.append_op( + type='crf_decoding', + inputs=inputs, + outputs={"ViterbiPath": [viterbi_path]}, + ) return viterbi_path @@ -1080,26 +1115,23 @@ def cos_sim(X, Y): out = helper.create_variable_for_type_inference(dtype=X.dtype) xnorm = helper.create_variable_for_type_inference(dtype=X.dtype) ynorm = helper.create_variable_for_type_inference(dtype=X.dtype) - helper.append_op(type='cos_sim', - inputs={ - 'X': [X], - 'Y': [Y] - }, - outputs={ - 'Out': [out], - 'XNorm': [xnorm], - 'YNorm': [ynorm] - }) + helper.append_op( + type='cos_sim', + inputs={'X': [X], 'Y': [Y]}, + outputs={'Out': [out], 'XNorm': [xnorm], 'YNorm': [ynorm]}, + ) return out @deprecated(since="2.0.0", update_to="paddle.nn.functional.dropout") -def dropout(x, - dropout_prob, - is_test=None, - seed=None, - name=None, - dropout_implementation="downgrade_in_infer"): +def dropout( + x, + dropout_prob, + is_test=None, + seed=None, + name=None, + dropout_implementation="downgrade_in_infer", +): """ Computes dropout. @@ -1157,23 +1189,32 @@ def dropout(x, """ if not isinstance(dropout_prob, (float, int, Variable)): raise TypeError( - "dropout_prob argument should be a number(int|float) or Variable") + "dropout_prob argument should be a number(int|float) or Variable" + ) # fast return for p == 0 if isinstance(dropout_prob, (int, float)) and dropout_prob == 0: return x if _non_static_mode(): - if (seed is None - or seed == 0) and default_main_program().random_seed != 0: + if ( + seed is None or seed == 0 + ) and default_main_program().random_seed != 0: seed = default_main_program().random_seed if is_test is None: is_test = not _dygraph_tracer()._train_mode - out, mask = _legacy_C_ops.dropout(x, 'dropout_prob', dropout_prob, - 'is_test', is_test, 'fix_seed', seed - is not None, 'seed', - seed if seed is not None else 0, - 'dropout_implementation', - dropout_implementation) + out, mask = _legacy_C_ops.dropout( + x, + 'dropout_prob', + dropout_prob, + 'is_test', + is_test, + 'fix_seed', + seed is not None, + 'seed', + seed if seed is not None else 0, + 'dropout_implementation', + dropout_implementation, + ) return out def get_attrs(prog, dropout_prob, is_test, seed): @@ -1181,8 +1222,10 @@ def get_attrs(prog, dropout_prob, is_test, seed): seed = prog.random_seed if isinstance(dropout_prob, Variable) and not dropout_prob.shape != [1]: raise TypeError( - "Required dropout_prob.shape == [1] if type(dropout_prob) is Variable, but received dropout_prob.shape = {}" - .format(dropout_prob.shape)) + "Required dropout_prob.shape == [1] if type(dropout_prob) is Variable, but received dropout_prob.shape = {}".format( + dropout_prob.shape + ) + ) attrs = { 'dropout_prob': dropout_prob, 'is_test': is_test, @@ -1193,32 +1236,35 @@ def get_attrs(prog, dropout_prob, is_test, seed): return attrs helper = LayerHelper('dropout', **locals()) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'dropout') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'dropout' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) mask = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) attrs = get_attrs(helper.main_program, dropout_prob, is_test, seed) - helper.append_op(type='dropout', - inputs={'X': [x]}, - outputs={ - 'Out': [out], - 'Mask': [mask] - }, - attrs=attrs) + helper.append_op( + type='dropout', + inputs={'X': [x]}, + outputs={'Out': [out], 'Mask': [mask]}, + attrs=attrs, + ) return out @templatedoc() -def chunk_eval(input, - label, - chunk_scheme, - num_chunk_types, - excluded_chunk_types=None, - seq_length=None): +def chunk_eval( + input, + label, + chunk_scheme, + num_chunk_types, + excluded_chunk_types=None, + seq_length=None, +): r""" This operator computes the precision, recall and F1-score for chunk detection. It is often used in sequence tagging tasks, such as Named Entity Recognition(NER). @@ -1337,30 +1383,39 @@ def chunk_eval(input, num_infer_chunks = helper.create_variable_for_type_inference(dtype="int64") num_label_chunks = helper.create_variable_for_type_inference(dtype="int64") num_correct_chunks = helper.create_variable_for_type_inference( - dtype="int64") + dtype="int64" + ) this_input = {"Inference": [input], "Label": [label]} if seq_length is not None: this_input["SeqLength"] = [seq_length] - helper.append_op(type="chunk_eval", - inputs=this_input, - outputs={ - "Precision": [precision], - "Recall": [recall], - "F1-Score": [f1_score], - "NumInferChunks": [num_infer_chunks], - "NumLabelChunks": [num_label_chunks], - "NumCorrectChunks": [num_correct_chunks] - }, - attrs={ - "num_chunk_types": num_chunk_types, - "chunk_scheme": chunk_scheme, - "excluded_chunk_types": excluded_chunk_types or [] - }) - return (precision, recall, f1_score, num_infer_chunks, num_label_chunks, - num_correct_chunks) + helper.append_op( + type="chunk_eval", + inputs=this_input, + outputs={ + "Precision": [precision], + "Recall": [recall], + "F1-Score": [f1_score], + "NumInferChunks": [num_infer_chunks], + "NumLabelChunks": [num_label_chunks], + "NumCorrectChunks": [num_correct_chunks], + }, + attrs={ + "num_chunk_types": num_chunk_types, + "chunk_scheme": chunk_scheme, + "excluded_chunk_types": excluded_chunk_types or [], + }, + ) + return ( + precision, + recall, + f1_score, + num_infer_chunks, + num_label_chunks, + num_correct_chunks, + ) @deprecated(since="2.0.0", update_to="paddle.nn.functional.softmax") @@ -1481,38 +1536,44 @@ def softmax(input, use_cudnn=True, name=None, axis=-1): return _C_ops.softmax(input, axis) if _non_static_mode(): - return _legacy_C_ops.softmax(input, 'axis', axis, 'use_cudnn', - use_cudnn) + return _legacy_C_ops.softmax( + input, 'axis', axis, 'use_cudnn', use_cudnn + ) inputs = {"X": [input]} attrs = {"axis": axis, "use_cudnn": use_cudnn} helper = LayerHelper('softmax', **locals()) - check_variable_and_dtype(input, 'input/x', - ['float16', 'float32', 'float64'], 'softmax') + check_variable_and_dtype( + input, 'input/x', ['float16', 'float32', 'float64'], 'softmax' + ) dtype = helper.input_dtype() softmax_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="softmax", - inputs={"X": input}, - outputs={"Out": softmax_out}, - attrs=attrs) + helper.append_op( + type="softmax", + inputs={"X": input}, + outputs={"Out": softmax_out}, + attrs=attrs, + ) return softmax_out -def conv2d(input, - num_filters, - filter_size, - stride=1, - padding=0, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - name=None, - data_format="NCHW"): +def conv2d( + input, + num_filters, + filter_size, + stride=1, + padding=0, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + name=None, + data_format="NCHW", +): r""" :api_attr: Static Graph @@ -1648,27 +1709,34 @@ def conv2d(input, print(conv2d.shape) # [-1, 2, 30, 30] """ - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'conv2d') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'conv2d' + ) if len(input.shape) != 4: - raise ValueError("Input size should be 4, " - "but received {}".format(len(input.shape))) + raise ValueError( + "Input size should be 4, " + "but received {}".format(len(input.shape)) + ) num_channels = input.shape[1] if not isinstance(use_cudnn, bool): - raise ValueError("Attr(use_cudnn) should be True or False. Received " - "Attr(use_cudnn): %s. " % str(use_cudnn)) + raise ValueError( + "Attr(use_cudnn) should be True or False. Received " + "Attr(use_cudnn): %s. " % str(use_cudnn) + ) if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) - channel_last = (data_format == "NHWC") + channel_last = data_format == "NHWC" num_channels = input.shape[3] if channel_last else input.shape[1] if num_channels < 0: raise ValueError( "The channel dimmention of the input(%s) should be defined. " - "Received: %s." % (str(input.shape), str(num_channels))) + "Received: %s." % (str(input.shape), str(num_channels)) + ) assert param_attr is not False, "param_attr should not be False here." if groups is None: @@ -1676,27 +1744,35 @@ def conv2d(input, elif groups <= 0: raise ValueError( "the groups of input must be greater than 0, " - "but received the groups of input is {}".format(groups)) + "but received the groups of input is {}".format(groups) + ) else: if num_channels % groups != 0: raise ValueError( "the channel of input must be divisible by groups," "received: the channel of input is {}, the shape of input is {}" - ", the groups is {}".format(num_channels, input.shape, groups)) + ", the groups is {}".format(num_channels, input.shape, groups) + ) num_filter_channels = num_channels // groups l_type = 'conv2d' - if (num_channels == groups and num_filters % num_channels == 0 - and not use_cudnn): + if ( + num_channels == groups + and num_filters % num_channels == 0 + and not use_cudnn + ): l_type = 'depthwise_conv2d' - if (num_channels == groups and num_filters % num_channels == 0 - and core.is_compiled_with_rocm()): + if ( + num_channels == groups + and num_filters % num_channels == 0 + and core.is_compiled_with_rocm() + ): l_type = 'depthwise_conv2d' # NPU only supports depthwise_conv2d when "input_channel = output_channel = groups" if core.is_compiled_with_npu(): - if (num_channels == groups and num_channels == num_filters): + if num_channels == groups and num_channels == num_filters: l_type = 'depthwise_conv2d' else: l_type = 'conv2d' @@ -1710,7 +1786,6 @@ def conv2d(input, # padding def _update_padding(padding, data_format): - def is_list_or_tuple(ele): if isinstance(ele, list) or isinstance(ele, tuple): return True @@ -1721,14 +1796,16 @@ def is_list_or_tuple(ele): if not (padding[0] == [0, 0] and padding[1] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[2:4] padding = [ele for a_list in padding for ele in a_list] elif is_list_or_tuple(padding[0]) and (data_format == "NHWC"): if not (padding[0] == [0, 0] and padding[3] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[1:3] padding = [ele for a_list in padding for ele in a_list] padding = utils.convert_to_list(padding, 4, 'padding') @@ -1745,8 +1822,9 @@ def is_list_or_tuple(ele): padding = padding.upper() if padding not in ["SAME", "VALID"]: raise ValueError( - "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." % - str(padding)) + "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." + % str(padding) + ) if padding == "VALID": padding_algorithm = "VALID" padding = [0, 0] @@ -1764,39 +1842,47 @@ def _get_default_param_initializer(): raise ValueError( "Invalid filter number, excepted number is larger than 0, but" " received {}, please check the input shape and " - "filter size.".format(filter_elem_num)) - std = (2.0 / filter_elem_num)**0.5 + "filter size.".format(filter_elem_num) + ) + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std, 0) filter_param = helper.create_parameter( attr=helper.param_attr, shape=filter_shape, dtype=dtype, - default_initializer=_get_default_param_initializer()) + default_initializer=_get_default_param_initializer(), + ) pre_bias = helper.create_variable_for_type_inference(dtype) - if (core.is_compiled_with_cuda() and paddle.fluid.get_flags( - "FLAGS_conv2d_disable_cudnn")["FLAGS_conv2d_disable_cudnn"]): + if ( + core.is_compiled_with_cuda() + and paddle.fluid.get_flags("FLAGS_conv2d_disable_cudnn")[ + "FLAGS_conv2d_disable_cudnn" + ] + ): use_cudnn = False - helper.append_op(type=l_type, - inputs={ - 'Input': input, - 'Filter': filter_param, - }, - outputs={"Output": pre_bias}, - attrs={ - 'strides': stride, - 'paddings': padding, - 'dilations': dilation, - 'groups': groups, - 'use_cudnn': use_cudnn, - 'use_mkldnn': False, - 'fuse_relu_before_depthwise_conv': False, - "padding_algorithm": padding_algorithm, - "data_format": data_format, - }) + helper.append_op( + type=l_type, + inputs={ + 'Input': input, + 'Filter': filter_param, + }, + outputs={"Output": pre_bias}, + attrs={ + 'strides': stride, + 'paddings': padding, + 'dilations': dilation, + 'groups': groups, + 'use_cudnn': use_cudnn, + 'use_mkldnn': False, + 'fuse_relu_before_depthwise_conv': False, + "padding_algorithm": padding_algorithm, + "data_format": data_format, + }, + ) if data_format == 'NCHW': pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2) @@ -1806,19 +1892,21 @@ def _get_default_param_initializer(): return helper.append_activation(pre_act) -def conv3d(input, - num_filters, - filter_size, - stride=1, - padding=0, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - name=None, - data_format="NCDHW"): +def conv3d( + input, + num_filters, + filter_size, + stride=1, + padding=0, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + name=None, + data_format="NCDHW", +): r""" :api_attr: Static Graph @@ -1961,37 +2049,46 @@ def conv3d(input, dtype = helper.input_dtype() if not isinstance(use_cudnn, bool): - raise ValueError("Attr(use_cudnn) should be True or False. Received " - "Attr(use_cudnn): %s. " % str(use_cudnn)) + raise ValueError( + "Attr(use_cudnn) should be True or False. Received " + "Attr(use_cudnn): %s. " % str(use_cudnn) + ) if data_format not in ["NCDHW", "NDHWC"]: raise ValueError( "Attr(data_format) should be 'NCDHW' or 'NDHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) - channel_last = (data_format == "NDHWC") + channel_last = data_format == "NDHWC" if len(input.shape) != 5: raise ValueError( - "Input should be 5D tensor, but received input with the shape of {}" - .format(input.shape)) + "Input should be 5D tensor, but received input with the shape of {}".format( + input.shape + ) + ) num_channels = input.shape[4] if channel_last else input.shape[1] if num_channels < 0: raise ValueError( "The channel dimmention of the input(%s) should be defined. " - "Received: %s." % (str(input.shape), str(num_channels))) + "Received: %s." % (str(input.shape), str(num_channels)) + ) if groups is None: num_filter_channels = num_channels elif groups <= 0: raise ValueError( - "the groups of conv3d should be greater than 0. Received groups: {}" - .format(groups)) + "the groups of conv3d should be greater than 0. Received groups: {}".format( + groups + ) + ) else: if num_channels % groups != 0: raise ValueError( "The number of input channels must be divisible by Attr(groups). " - "Received: number of channels(%s), groups(%s)." % - (str(num_channels), str(groups))) + "Received: number of channels(%s), groups(%s)." + % (str(num_channels), str(groups)) + ) num_filter_channels = num_channels // groups filter_size = utils.convert_to_list(filter_size, 3, 'filter_size') @@ -1999,7 +2096,6 @@ def conv3d(input, dilation = utils.convert_to_list(dilation, 3, 'dilation') def _update_padding(padding, data_format): - def is_list_or_tuple(ele): if isinstance(ele, list) or isinstance(ele, tuple): return True @@ -2010,14 +2106,16 @@ def is_list_or_tuple(ele): if not (padding[0] == [0, 0] and padding[1] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[2:5] padding = [ele for a_list in padding for ele in a_list] elif is_list_or_tuple(padding[0]) and (data_format == "NDHWC"): if not (padding[0] == [0, 0] and padding[4] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[1:4] padding = [ele for a_list in padding for ele in a_list] padding = utils.convert_to_list(padding, 6, 'padding') @@ -2037,8 +2135,9 @@ def is_list_or_tuple(ele): padding = padding.upper() if padding not in ["SAME", "VALID"]: raise ValueError( - "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." % - str(padding)) + "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." + % str(padding) + ) if padding == "VALID": padding_algorithm = "VALID" padding = [0, 0, 0] @@ -2052,41 +2151,46 @@ def is_list_or_tuple(ele): filter_shape = [num_filters, num_filter_channels] + filter_size def _get_default_param_initializer(): - filter_elem_num = filter_size[0] * filter_size[1] * filter_size[ - 2] * num_channels + filter_elem_num = ( + filter_size[0] * filter_size[1] * filter_size[2] * num_channels + ) if filter_elem_num <= 0: raise ValueError( "Invalid filter number, excepted number is larger than 0, but" " received {}, please check the input shape and " - "filter size.".format(filter_elem_num)) + "filter size.".format(filter_elem_num) + ) - std = (2.0 / filter_elem_num)**0.5 + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std, 0) filter_param = helper.create_parameter( attr=helper.param_attr, shape=filter_shape, dtype=dtype, - default_initializer=_get_default_param_initializer()) + default_initializer=_get_default_param_initializer(), + ) pre_bias = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=l_type, - inputs={ - 'Input': input, - 'Filter': filter_param, - }, - outputs={"Output": pre_bias}, - attrs={ - 'strides': stride, - 'paddings': padding, - 'dilations': dilation, - 'groups': groups, - 'use_cudnn': use_cudnn, - 'use_mkldnn': False, - "padding_algorithm": padding_algorithm, - "data_format": data_format, - }) + helper.append_op( + type=l_type, + inputs={ + 'Input': input, + 'Filter': filter_param, + }, + outputs={"Output": pre_bias}, + attrs={ + 'strides': stride, + 'paddings': padding, + 'dilations': dilation, + 'groups': groups, + 'use_cudnn': use_cudnn, + 'use_mkldnn': False, + "padding_algorithm": padding_algorithm, + "data_format": data_format, + }, + ) if data_format == 'NCDHW': pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2) @@ -2097,17 +2201,19 @@ def _get_default_param_initializer(): @templatedoc() -def pool2d(input, - pool_size=-1, - pool_type="max", - pool_stride=1, - pool_padding=0, - global_pooling=False, - use_cudnn=True, - ceil_mode=False, - name=None, - exclusive=True, - data_format="NCHW"): +def pool2d( + input, + pool_size=-1, + pool_type="max", + pool_stride=1, + pool_padding=0, + global_pooling=False, + use_cudnn=True, + ceil_mode=False, + name=None, + exclusive=True, + data_format="NCHW", +): """ ${comment} @@ -2218,27 +2324,31 @@ def pool2d(input, if pool_type not in ["max", "avg"]: raise ValueError( "Unknown Attr(pool_type): '%s'. It can only be 'max' or 'avg'.", - str(pool_type)) + str(pool_type), + ) if global_pooling is False and pool_size == -1: raise ValueError( "When Attr(global_pooling) is False, Attr(pool_size) must be passed " - "and be a valid value. Received pool_size: %s." % str(pool_size)) + "and be a valid value. Received pool_size: %s." % str(pool_size) + ) if not isinstance(use_cudnn, bool): - raise TypeError("Attr(use_cudnn) should be True or False. Received " - "Attr(use_cudnn): %s." % str(use_cudnn)) + raise TypeError( + "Attr(use_cudnn) should be True or False. Received " + "Attr(use_cudnn): %s." % str(use_cudnn) + ) if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) pool_size = utils.convert_to_list(pool_size, 2, 'pool_size') pool_stride = utils.convert_to_list(pool_stride, 2, 'pool_stride') def update_padding(padding, data_format): - def is_list_or_tuple(ele): if isinstance(ele, list) or isinstance(ele, tuple): return True @@ -2249,14 +2359,16 @@ def is_list_or_tuple(ele): if not (padding[0] == [0, 0] and padding[1] == [0, 0]): raise ValueError( "Non-zero pool_padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[2:4] padding = [ele for a_list in padding for ele in a_list] elif is_list_or_tuple(padding[0]) and (data_format == "NHWC"): if not (padding[0] == [0, 0] and padding[3] == [0, 0]): raise ValueError( "Non-zero pool_padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[1:3] padding = [ele for a_list in padding for ele in a_list] padding = utils.convert_to_list(padding, 4, 'padding') @@ -2274,61 +2386,77 @@ def is_list_or_tuple(ele): if pool_padding not in ["SAME", "VALID"]: raise ValueError( "Unknown Attr(pool_padding): '%s'. It can only be 'SAME' or 'VALID'." - % str(pool_padding)) + % str(pool_padding) + ) if pool_padding == "VALID": padding_algorithm = "VALID" pool_padding = [0, 0] if ceil_mode != False: raise ValueError( "When Attr(pool_padding) is \"VALID\", Attr(ceil_mode) must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) elif pool_padding == "SAME": padding_algorithm = "SAME" pool_padding = [0, 0] pool_padding = update_padding(pool_padding, data_format) if in_dygraph_mode(): - return _C_ops.pool2d(input, pool_size, pool_stride, pool_padding, - ceil_mode, exclusive, data_format, pool_type, - global_pooling, False, padding_algorithm, - use_cudnn) + return _C_ops.pool2d( + input, + pool_size, + pool_stride, + pool_padding, + ceil_mode, + exclusive, + data_format, + pool_type, + global_pooling, + False, + padding_algorithm, + use_cudnn, + ) op_type = 'pool2d' helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype() pool_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs={"X": input}, - outputs={"Out": pool_out}, - attrs={ - "pooling_type": pool_type, - "ksize": pool_size, - "global_pooling": global_pooling, - "strides": pool_stride, - "paddings": pool_padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": use_cudnn, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": exclusive, - "data_format": data_format, - }) + helper.append_op( + type=op_type, + inputs={"X": input}, + outputs={"Out": pool_out}, + attrs={ + "pooling_type": pool_type, + "ksize": pool_size, + "global_pooling": global_pooling, + "strides": pool_stride, + "paddings": pool_padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": use_cudnn, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": exclusive, + "data_format": data_format, + }, + ) return pool_out @templatedoc() -def pool3d(input, - pool_size=-1, - pool_type="max", - pool_stride=1, - pool_padding=0, - global_pooling=False, - use_cudnn=True, - ceil_mode=False, - name=None, - exclusive=True, - data_format="NCDHW"): +def pool3d( + input, + pool_size=-1, + pool_type="max", + pool_stride=1, + pool_padding=0, + global_pooling=False, + use_cudnn=True, + ceil_mode=False, + name=None, + exclusive=True, + data_format="NCDHW", +): """ ${comment} @@ -2445,28 +2573,32 @@ def pool3d(input, if pool_type not in ["max", "avg"]: raise ValueError( "Unknown Attr(pool_type): '%s'. It can only be 'max' or 'avg'.", - str(pool_type)) + str(pool_type), + ) if global_pooling is False and pool_size == -1: raise ValueError( "When Attr(global_pooling) is False, Attr(pool_size) must be passed " - "and be a valid value. Received Attr(pool_size): %s." % - str(pool_size)) + "and be a valid value. Received Attr(pool_size): %s." + % str(pool_size) + ) if not isinstance(use_cudnn, bool): - raise TypeError("Attr(use_cudnn) should be True or False. Received " - "Attr(use_cudnn): %s. " % str(use_cudnn)) + raise TypeError( + "Attr(use_cudnn) should be True or False. Received " + "Attr(use_cudnn): %s. " % str(use_cudnn) + ) if data_format not in ["NCDHW", "NDHWC"]: raise ValueError( "Attr(data_format) should be 'NCDHW' or 'NDHWC'. Received " - "Attr(data_format): %s" % str(data_format)) + "Attr(data_format): %s" % str(data_format) + ) pool_size = utils.convert_to_list(pool_size, 3, 'pool_size') pool_stride = utils.convert_to_list(pool_stride, 3, 'pool_stride') def update_padding(padding, data_format): - def is_list_or_tuple(ele): if isinstance(ele, (list, tuple)): return True @@ -2477,14 +2609,16 @@ def is_list_or_tuple(ele): if not (padding[0] == [0, 0] and padding[1] == [0, 0]): raise ValueError( "Non-zero pool_padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[2:5] padding = [ele for a_list in padding for ele in a_list] elif is_list_or_tuple(padding[0]) and (data_format == "NDHWC"): if not (padding[0] == [0, 0] and padding[4] == [0, 0]): raise ValueError( "Non-zero pool_padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[1:4] padding = [ele for a_list in padding for ele in a_list] padding = utils.convert_to_list(padding, 6, 'padding') @@ -2506,14 +2640,16 @@ def is_list_or_tuple(ele): if pool_padding not in ["SAME", "VALID"]: raise ValueError( "Unknown Attr(pool_padding): '%s'. It can only be 'SAME' or 'VALID'." - % str(pool_padding)) + % str(pool_padding) + ) if pool_padding == "VALID": padding_algorithm = "VALID" pool_padding = [0, 0, 0] if ceil_mode != False: raise ValueError( "When Attr(pool_padding) is \"VALID\", ceil_mode must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) elif pool_padding == "SAME": padding_algorithm = "SAME" pool_padding = [0, 0, 0] @@ -2525,33 +2661,33 @@ def is_list_or_tuple(ele): dtype = helper.input_dtype() pool_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs={"X": input}, - outputs={"Out": pool_out}, - attrs={ - "pooling_type": pool_type, - "ksize": pool_size, - "global_pooling": global_pooling, - "strides": pool_stride, - "paddings": pool_padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": use_cudnn, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": exclusive, - "data_format": data_format, - }) + helper.append_op( + type=op_type, + inputs={"X": input}, + outputs={"Out": pool_out}, + attrs={ + "pooling_type": pool_type, + "ksize": pool_size, + "global_pooling": global_pooling, + "strides": pool_stride, + "paddings": pool_padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": use_cudnn, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": exclusive, + "data_format": data_format, + }, + ) return pool_out @deprecated(since="2.0.0") @templatedoc(op_type="pool2d") -def adaptive_pool2d(input, - pool_size, - pool_type="max", - require_index=False, - name=None): +def adaptive_pool2d( + input, pool_size, pool_type="max", require_index=False, name=None +): r""" This operation calculates the output based on the input, pool_size, @@ -2648,19 +2784,24 @@ def adaptive_pool2d(input, pool_type='max') """ check_variable_and_dtype( - input, 'input', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'adaptive_pool2d') + input, + 'input', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'adaptive_pool2d', + ) check_type(pool_type, 'pool_type', str, 'adaptive_pool2d') check_type(pool_size, 'pool_size', (int, list, tuple), 'adaptive_pool2d') check_type(require_index, 'require_index', bool, 'adaptive_pool2d') if pool_type not in ["max", "avg"]: raise ValueError( "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.", - str(pool_type)) + str(pool_type), + ) if pool_type == "avg" and require_index: raise ValueError( - "invalid setting 'require_index' true when 'pool_type' is 'avg'.") + "invalid setting 'require_index' true when 'pool_type' is 'avg'." + ) pool_size = utils.convert_to_list(pool_size, 2, 'pool_size') @@ -2678,25 +2819,25 @@ def adaptive_pool2d(input, mask = helper.create_variable_for_type_inference(dtype) outputs["Mask"] = mask - helper.append_op(type=l_type, - inputs={"X": input}, - outputs=outputs, - attrs={ - "pooling_type": pool_type, - "ksize": pool_size, - "adaptive": True, - }) + helper.append_op( + type=l_type, + inputs={"X": input}, + outputs=outputs, + attrs={ + "pooling_type": pool_type, + "ksize": pool_size, + "adaptive": True, + }, + ) return (pool_out, mask) if require_index else pool_out @deprecated(since="2.0.0") @templatedoc(op_type="pool3d") -def adaptive_pool3d(input, - pool_size, - pool_type="max", - require_index=False, - name=None): +def adaptive_pool3d( + input, pool_size, pool_type="max", require_index=False, name=None +): r""" This operation calculates the output based on the input, pool_size, @@ -2807,19 +2948,24 @@ def adaptive_pool3d(input, pool_type='max') """ check_variable_and_dtype( - input, 'input', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'adaptive_pool3d') + input, + 'input', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'adaptive_pool3d', + ) check_type(pool_type, 'pool_type', str, 'adaptive_pool3d') check_type(pool_size, 'pool_size', (int, list, tuple), 'adaptive_pool3d') check_type(require_index, 'require_index', bool, 'adaptive_pool3d') if pool_type not in ["max", "avg"]: raise ValueError( "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.", - str(pool_type)) + str(pool_type), + ) if pool_type == "avg" and require_index: raise ValueError( - "invalid setting 'require_index' true when 'pool_type' is 'avg'.") + "invalid setting 'require_index' true when 'pool_type' is 'avg'." + ) pool_size = utils.convert_to_list(pool_size, 3, 'pool_size') @@ -2837,32 +2983,36 @@ def adaptive_pool3d(input, mask = helper.create_variable_for_type_inference(dtype) outputs["Mask"] = mask - helper.append_op(type=l_type, - inputs={"X": input}, - outputs=outputs, - attrs={ - "pooling_type": pool_type, - "ksize": pool_size, - "adaptive": True, - }) + helper.append_op( + type=l_type, + inputs={"X": input}, + outputs=outputs, + attrs={ + "pooling_type": pool_type, + "ksize": pool_size, + "adaptive": True, + }, + ) return (pool_out, mask) if require_index else pool_out -def batch_norm(input, - act=None, - is_test=False, - momentum=0.9, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - data_layout='NCHW', - in_place=False, - name=None, - moving_mean_name=None, - moving_variance_name=None, - do_model_average_for_mean_and_var=True, - use_global_stats=False): +def batch_norm( + input, + act=None, + is_test=False, + momentum=0.9, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + data_layout='NCHW', + in_place=False, + name=None, + moving_mean_name=None, + moving_variance_name=None, + do_model_average_for_mean_and_var=True, + use_global_stats=False, +): r""" :api_attr: Static Graph @@ -2977,11 +3127,14 @@ def batch_norm(input, print(hidden2.shape) # [3, 200] """ - assert bias_attr is not False, "bias_attr should not be False in batch_norm." + assert ( + bias_attr is not False + ), "bias_attr should not be False in batch_norm." helper = LayerHelper('batch_norm', **locals()) - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'batch_norm') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'batch_norm' + ) dtype = helper.input_dtype() # use fp32 for bn parameter @@ -3000,31 +3153,38 @@ def batch_norm(input, param_shape = [channel_num] # create parameter - scale = helper.create_parameter(attr=helper.param_attr, - shape=param_shape, - dtype=dtype, - default_initializer=Constant(1.0)) - bias = helper.create_parameter(attr=helper.bias_attr, - shape=param_shape, - dtype=dtype, - is_bias=True) - - mean = helper.create_parameter(attr=ParamAttr( - name=moving_mean_name, - initializer=Constant(0.0), - trainable=False, - do_model_average=do_model_average_for_mean_and_var), - shape=param_shape, - dtype=dtype) + scale = helper.create_parameter( + attr=helper.param_attr, + shape=param_shape, + dtype=dtype, + default_initializer=Constant(1.0), + ) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=param_shape, dtype=dtype, is_bias=True + ) + + mean = helper.create_parameter( + attr=ParamAttr( + name=moving_mean_name, + initializer=Constant(0.0), + trainable=False, + do_model_average=do_model_average_for_mean_and_var, + ), + shape=param_shape, + dtype=dtype, + ) mean.stop_gradient = True - variance = helper.create_parameter(attr=ParamAttr( - name=moving_variance_name, - initializer=Constant(1.0), - trainable=False, - do_model_average=do_model_average_for_mean_and_var), - shape=param_shape, - dtype=dtype) + variance = helper.create_parameter( + attr=ParamAttr( + name=moving_variance_name, + initializer=Constant(1.0), + trainable=False, + do_model_average=do_model_average_for_mean_and_var, + ), + shape=param_shape, + dtype=dtype, + ) variance.stop_gradient = True # create output @@ -3044,38 +3204,81 @@ def batch_norm(input, attrs_ = () if attrs_has_momentum: - attrs_ = ('momentum', momentum, 'epsilon', epsilon, 'is_test', - is_test, 'data_layout', data_layout, 'use_mkldnn', False, - 'fuse_with_relu', False, 'use_global_stats', - use_global_stats) + attrs_ = ( + 'momentum', + momentum, + 'epsilon', + epsilon, + 'is_test', + is_test, + 'data_layout', + data_layout, + 'use_mkldnn', + False, + 'fuse_with_relu', + False, + 'use_global_stats', + use_global_stats, + ) else: - attrs_ = ('epsilon', epsilon, 'is_test', is_test, 'data_layout', - data_layout, 'use_mkldnn', False, 'fuse_with_relu', False, - 'use_global_stats', use_global_stats) + attrs_ = ( + 'epsilon', + epsilon, + 'is_test', + is_test, + 'data_layout', + data_layout, + 'use_mkldnn', + False, + 'fuse_with_relu', + False, + 'use_global_stats', + use_global_stats, + ) if inputs_has_MomemtumTensor: batch_norm_out, _, _, _, _, _ = _legacy_C_ops.batch_norm( - input, scale, bias, mean, variance, momentum, mean_out, - variance_out, *attrs_) + input, + scale, + bias, + mean, + variance, + momentum, + mean_out, + variance_out, + *attrs_, + ) else: batch_norm_out, _, _, _, _, _ = _legacy_C_ops.batch_norm( - input, scale, bias, mean, variance, None, mean_out, - variance_out, *attrs_) + input, + scale, + bias, + mean, + variance, + None, + mean_out, + variance_out, + *attrs_, + ) - return dygraph_utils._append_activation_in_dygraph(batch_norm_out, - act=act, - use_mkldnn=False) + return dygraph_utils._append_activation_in_dygraph( + batch_norm_out, act=act, use_mkldnn=False + ) - saved_mean = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) + saved_mean = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) saved_variance = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) reserve_space = None if not is_test: reserve_space = helper.create_variable_for_type_inference( - dtype=helper.input_dtype(), stop_gradient=True) + dtype=helper.input_dtype(), stop_gradient=True + ) - batch_norm_out = input if in_place else \ - helper.create_variable_for_type_inference(dtype) + batch_norm_out = ( + input if in_place else helper.create_variable_for_type_inference(dtype) + ) inputs = { "X": input, @@ -3084,7 +3287,7 @@ def batch_norm(input, "Mean": mean, "Variance": variance, "MeanOut": mean_out, - "VarianceOut": variance_out + "VarianceOut": variance_out, } attrs = { "epsilon": epsilon, @@ -3092,7 +3295,7 @@ def batch_norm(input, "data_layout": data_layout, "use_mkldnn": False, "fuse_with_relu": False, - "use_global_stats": use_global_stats + "use_global_stats": use_global_stats, } if isinstance(momentum, Variable): inputs['MomemtumTensor'] = momentum @@ -3104,33 +3307,34 @@ def batch_norm(input, "MeanOut": mean_out, "VarianceOut": variance_out, "SavedMean": saved_mean, - "SavedVariance": saved_variance + "SavedVariance": saved_variance, } if reserve_space is not None: outputs["ReserveSpace"] = reserve_space - helper.append_op(type="batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs + ) return helper.append_activation(batch_norm_out) -def inplace_abn(input, - act=None, - is_test=False, - momentum=0.9, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - data_layout='NCHW', - name=None, - moving_mean_name=None, - moving_variance_name=None, - do_model_average_for_mean_and_var=True, - use_global_stats=False, - act_alpha=1.0): +def inplace_abn( + input, + act=None, + is_test=False, + momentum=0.9, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + data_layout='NCHW', + name=None, + moving_mean_name=None, + moving_variance_name=None, + do_model_average_for_mean_and_var=True, + use_global_stats=False, + act_alpha=1.0, +): r""" **In-place Activation Batch Normalization Layer** @@ -3164,14 +3368,14 @@ def inplace_abn(input, numerical stability. Default is 1e-5. param_attr(ParamAttr|None): The parameter attribute for Parameter `scale` of inplace_abn. If it is set to None or one attribute of ParamAttr, inplace_abn - will create ParamAttr as param_attr, the name of scale can be set in ParamAttr. - If the Initializer of the param_attr is not set, the parameter is initialized - with Xavier. Default: None. + will create ParamAttr as param_attr, the name of scale can be set in ParamAttr. + If the Initializer of the param_attr is not set, the parameter is initialized + with Xavier. Default: None. bias_attr(ParamAttr|None): The parameter attribute for the bias of inplace_abn. If it is set to None or one attribute of ParamAttr, inplace_abn - will create ParamAttr as bias_attr, the name of bias can be set in ParamAttr. - If the Initializer of the bias_attr is not set, the bias is initialized zero. - Default: None. + will create ParamAttr as bias_attr, the name of bias can be set in ParamAttr. + If the Initializer of the bias_attr is not set, the bias is initialized zero. + Default: None. data_layout (str, optional): Specify the data format of the input, and the data format of the output will be consistent with that of the input. An optional string from: `"NCHW"`, `"NHWC"`. The default is `"NCHW"`. When it is `"NCHW"`, the data is stored in the order of: @@ -3209,14 +3413,18 @@ def inplace_abn(input, hidden3 = fluid.layers.inplace_abn(input=hidden2, act='leaky_relu', act_alpha=0.2) """ - assert act in [None, 'identity', 'leaky_relu', 'elu'], \ - "inplace_abn only support act as None, 'identity', " \ + assert act in [None, 'identity', 'leaky_relu', 'elu'], ( + "inplace_abn only support act as None, 'identity', " "'leaky_relu', 'elu' currently" - assert bias_attr is not False, "bias_attr should not be False in inplace_abn." + ) + assert ( + bias_attr is not False + ), "bias_attr should not be False in inplace_abn." helper = LayerHelper('inplace_abn', **locals()) - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'inplace_abn') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'inplace_abn' + ) dtype = helper.input_dtype() input_shape = input.shape @@ -3231,31 +3439,38 @@ def inplace_abn(input, param_shape = [channel_num] # create parameter - scale = helper.create_parameter(attr=helper.param_attr, - shape=param_shape, - dtype=dtype, - default_initializer=Constant(1.0)) - bias = helper.create_parameter(attr=helper.bias_attr, - shape=param_shape, - dtype=dtype, - is_bias=True) - - mean = helper.create_parameter(attr=ParamAttr( - name=moving_mean_name, - initializer=Constant(0.0), - trainable=False, - do_model_average=do_model_average_for_mean_and_var), - shape=param_shape, - dtype=dtype) + scale = helper.create_parameter( + attr=helper.param_attr, + shape=param_shape, + dtype=dtype, + default_initializer=Constant(1.0), + ) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=param_shape, dtype=dtype, is_bias=True + ) + + mean = helper.create_parameter( + attr=ParamAttr( + name=moving_mean_name, + initializer=Constant(0.0), + trainable=False, + do_model_average=do_model_average_for_mean_and_var, + ), + shape=param_shape, + dtype=dtype, + ) mean.stop_gradient = True - variance = helper.create_parameter(attr=ParamAttr( - name=moving_variance_name, - initializer=Constant(1.0), - trainable=False, - do_model_average=do_model_average_for_mean_and_var), - shape=param_shape, - dtype=dtype) + variance = helper.create_parameter( + attr=ParamAttr( + name=moving_variance_name, + initializer=Constant(1.0), + trainable=False, + do_model_average=do_model_average_for_mean_and_var, + ), + shape=param_shape, + dtype=dtype, + ) variance.stop_gradient = True # create output @@ -3277,39 +3492,88 @@ def inplace_abn(input, attrs__ = () if attrs_has_momentum: - attrs__ = ('momentum', momentum, 'epsilon', epsilon, 'is_test', - is_test, 'data_layout', data_layout, 'use_mkldnn', False, - 'fuse_with_relu', False, 'use_global_stats', - use_global_stats, 'activation', act, 'alpha', act_alpha) + attrs__ = ( + 'momentum', + momentum, + 'epsilon', + epsilon, + 'is_test', + is_test, + 'data_layout', + data_layout, + 'use_mkldnn', + False, + 'fuse_with_relu', + False, + 'use_global_stats', + use_global_stats, + 'activation', + act, + 'alpha', + act_alpha, + ) else: - attrs__ = ('epsilon', epsilon, 'is_test', is_test, 'data_layout', - data_layout, 'use_mkldnn', False, 'fuse_with_relu', - False, 'use_global_stats', use_global_stats, - 'activation', act, 'alpha', act_alpha) + attrs__ = ( + 'epsilon', + epsilon, + 'is_test', + is_test, + 'data_layout', + data_layout, + 'use_mkldnn', + False, + 'fuse_with_relu', + False, + 'use_global_stats', + use_global_stats, + 'activation', + act, + 'alpha', + act_alpha, + ) if inputs_has_MomemtumTensor: batch_norm_out, _, _, _, _, _ = _legacy_C_ops.inplace_abn_( - input, scale, bias, mean, variance, momentum, mean_out, - variance_out, *attrs__) + input, + scale, + bias, + mean, + variance, + momentum, + mean_out, + variance_out, + *attrs__, + ) return batch_norm_out else: batch_norm_out, _, _, _, _, _ = _legacy_C_ops.inplace_abn_( - input, scale, bias, mean, variance, None, mean_out, - variance_out, *attrs__) + input, + scale, + bias, + mean, + variance, + None, + mean_out, + variance_out, + *attrs__, + ) return batch_norm_out - saved_mean = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) + saved_mean = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) saved_variance = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) reserve_space = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) inputs = { "X": input, "Scale": scale, "Bias": bias, "Mean": mean, - "Variance": variance + "Variance": variance, } attrs = { "epsilon": epsilon, @@ -3330,24 +3594,21 @@ def inplace_abn(input, "MeanOut": mean_out, "VarianceOut": variance_out, "SavedMean": saved_mean, - "SavedVariance": saved_variance + "SavedVariance": saved_variance, } if reserve_space is not None: outputs["ReserveSpace"] = reserve_space - helper.append_op(type="inplace_abn", - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type="inplace_abn", inputs=inputs, outputs=outputs, attrs=attrs + ) return batch_norm_out -def instance_norm(input, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - name=None): +def instance_norm( + input, epsilon=1e-05, param_attr=None, bias_attr=None, name=None +): r""" :api_attr: Static Graph @@ -3411,10 +3672,13 @@ def instance_norm(input, hidden1 = paddle.static.nn.fc(x, size=200) hidden2 = paddle.static.nn.instance_norm(hidden1) """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'instance_norm') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'instance_norm' + ) if param_attr is False: - assert bias_attr is False, "param_attr and bias_attr must be set to Fasle at the same time in instance_norm" + assert ( + bias_attr is False + ), "param_attr and bias_attr must be set to Fasle at the same time in instance_norm" helper = LayerHelper('instance_norm', **locals()) dtype = helper.input_dtype() @@ -3426,29 +3690,37 @@ def instance_norm(input, input_shape = input.shape if len(input.shape) < 2 or len(input.shape) > 5: raise ValueError( - 'expected 2D or 3D or 4D or 5D input (got {}D input, input shape is: {})' - .format(len(input.shape), input_shape)) + 'expected 2D or 3D or 4D or 5D input (got {}D input, input shape is: {})'.format( + len(input.shape), input_shape + ) + ) channel_num = input_shape[1] param_shape = [channel_num] if param_attr != False and bias_attr != False: # create parameter - scale = helper.create_parameter(attr=helper.param_attr, - shape=param_shape, - dtype=dtype, - default_initializer=Constant(1.0)) - bias = helper.create_parameter(attr=helper.bias_attr, - shape=param_shape, - dtype=dtype, - is_bias=True, - default_initializer=Constant(0.0)) + scale = helper.create_parameter( + attr=helper.param_attr, + shape=param_shape, + dtype=dtype, + default_initializer=Constant(1.0), + ) + bias = helper.create_parameter( + attr=helper.bias_attr, + shape=param_shape, + dtype=dtype, + is_bias=True, + default_initializer=Constant(0.0), + ) # create output - saved_mean = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) + saved_mean = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) saved_variance = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) instance_norm_out = helper.create_variable_for_type_inference(dtype) @@ -3457,35 +3729,39 @@ def instance_norm(input, inputs["Scale"] = scale inputs["Bias"] = bias - helper.append_op(type="instance_norm", - inputs=inputs, - outputs={ - "Y": instance_norm_out, - "SavedMean": saved_mean, - "SavedVariance": saved_variance - }, - attrs={ - "epsilon": epsilon, - }) + helper.append_op( + type="instance_norm", + inputs=inputs, + outputs={ + "Y": instance_norm_out, + "SavedMean": saved_mean, + "SavedVariance": saved_variance, + }, + attrs={ + "epsilon": epsilon, + }, + ) return instance_norm_out @static_only -def data_norm(input, - act=None, - epsilon=1e-05, - param_attr=None, - data_layout='NCHW', - in_place=False, - name=None, - moving_mean_name=None, - moving_variance_name=None, - do_model_average_for_mean_and_var=True, - slot_dim=-1, - sync_stats=False, - summary_decay_rate=0.9999999, - enable_scale_and_shift=False): +def data_norm( + input, + act=None, + epsilon=1e-05, + param_attr=None, + data_layout='NCHW', + in_place=False, + name=None, + moving_mean_name=None, + moving_variance_name=None, + do_model_average_for_mean_and_var=True, + slot_dim=-1, + sync_stats=False, + summary_decay_rate=0.9999999, + enable_scale_and_shift=False, +): r""" :api_attr: Static Graph @@ -3583,39 +3859,54 @@ def data_norm(input, if name == None: name = "dn" if enable_scale_and_shift: - scale_w = helper.create_parameter(attr=ParamAttr( - name=name + '.scale_w', - initializer=Constant(value=float(scale_w_default)), - trainable=True), - shape=param_shape, - dtype=input.dtype) - bias = helper.create_parameter(attr=ParamAttr( - name=name + '.bias', - initializer=Constant(value=float(bias_default)), - trainable=True), - shape=param_shape, - dtype=input.dtype) + scale_w = helper.create_parameter( + attr=ParamAttr( + name=name + '.scale_w', + initializer=Constant(value=float(scale_w_default)), + trainable=True, + ), + shape=param_shape, + dtype=input.dtype, + ) + bias = helper.create_parameter( + attr=ParamAttr( + name=name + '.bias', + initializer=Constant(value=float(bias_default)), + trainable=True, + ), + shape=param_shape, + dtype=input.dtype, + ) # create parameter - batch_size = helper.create_parameter(attr=ParamAttr( - name=name + '.batch_size', - initializer=Constant(value=float(batch_size_default)), - trainable=True), - shape=param_shape, - dtype=input.dtype) - - batch_sum = helper.create_parameter(attr=ParamAttr( - name=name + '.batch_sum', - initializer=Constant(value=float(batch_sum_default)), - trainable=True), - shape=param_shape, - dtype=input.dtype) - - batch_square_sum = helper.create_parameter(attr=ParamAttr( - name=name + '.batch_square_sum', - initializer=Constant(value=float(batch_square_sum_default)), - trainable=True), - shape=param_shape, - dtype=input.dtype) + batch_size = helper.create_parameter( + attr=ParamAttr( + name=name + '.batch_size', + initializer=Constant(value=float(batch_size_default)), + trainable=True, + ), + shape=param_shape, + dtype=input.dtype, + ) + + batch_sum = helper.create_parameter( + attr=ParamAttr( + name=name + '.batch_sum', + initializer=Constant(value=float(batch_sum_default)), + trainable=True, + ), + shape=param_shape, + dtype=input.dtype, + ) + + batch_square_sum = helper.create_parameter( + attr=ParamAttr( + name=name + '.batch_square_sum', + initializer=Constant(value=float(batch_square_sum_default)), + trainable=True, + ), + shape=param_shape, + dtype=input.dtype, + ) means = helper.create_variable(dtype=dtype, stop_gradient=True) scales = helper.create_variable(dtype=dtype, stop_gradient=True) @@ -3626,7 +3917,7 @@ def data_norm(input, "X": input, "BatchSize": batch_size, "BatchSum": batch_sum, - "BatchSquareSum": batch_square_sum + "BatchSquareSum": batch_square_sum, } attrs = { "epsilon": epsilon, @@ -3641,31 +3932,35 @@ def data_norm(input, if enable_scale_and_shift: inputs["scale_w"] = scale_w inputs["bias"] = bias - helper.append_op(type="data_norm", - inputs=inputs, - outputs={ - "Y": data_norm_out, - "Means": means, - "Scales": scales, - "BatchSize": batch_size, - "BatchSum": batch_sum, - "BatchSquareSum": batch_square_sum - }, - attrs=attrs) + helper.append_op( + type="data_norm", + inputs=inputs, + outputs={ + "Y": data_norm_out, + "Means": means, + "Scales": scales, + "BatchSize": batch_size, + "BatchSum": batch_sum, + "BatchSquareSum": batch_square_sum, + }, + attrs=attrs, + ) return helper.append_activation(data_norm_out) @templatedoc() -def layer_norm(input, - scale=True, - shift=True, - begin_norm_axis=1, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - act=None, - name=None): +def layer_norm( + input, + scale=True, + shift=True, + begin_norm_axis=1, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + act=None, + name=None, +): r""" :api_attr: Static Graph @@ -3728,11 +4023,13 @@ def layer_norm(input, output = paddle.static.nn.layer_norm(input=x, begin_norm_axis=1) print(output.shape) # [8, 32, 32] """ - assert _non_static_mode( - ) is not True, "please use LayerNorm instead of layer_norm in dygraph mode!" + assert ( + _non_static_mode() is not True + ), "please use LayerNorm instead of layer_norm in dygraph mode!" helper = LayerHelper('layer_norm', **locals()) - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'layer_norm') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'layer_norm' + ) dtype = helper.input_dtype() # create intput and parameters @@ -3740,57 +4037,65 @@ def layer_norm(input, input_shape = input.shape param_shape = [reduce(lambda x, y: x * y, input_shape[begin_norm_axis:])] if scale: - assert param_attr is not False, "param_attr should not be False when using scale." - scale = helper.create_parameter(attr=helper.param_attr, - shape=param_shape, - dtype=dtype, - default_initializer=Constant(1.0)) + assert ( + param_attr is not False + ), "param_attr should not be False when using scale." + scale = helper.create_parameter( + attr=helper.param_attr, + shape=param_shape, + dtype=dtype, + default_initializer=Constant(1.0), + ) inputs['Scale'] = scale else: if param_attr: warnings.warn("param_attr is only available with scale is True.") if shift: - assert bias_attr is not False, "bias_attr should not be False when using shift." - bias = helper.create_parameter(attr=helper.bias_attr, - shape=param_shape, - dtype=dtype, - is_bias=True) + assert ( + bias_attr is not False + ), "bias_attr should not be False when using shift." + bias = helper.create_parameter( + attr=helper.bias_attr, shape=param_shape, dtype=dtype, is_bias=True + ) inputs['Bias'] = bias else: if bias_attr: warnings.warn("bias_attr is only available with shift is True.") # create output - mean_out = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) - variance_out = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) + mean_out = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) + variance_out = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) layer_norm_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="layer_norm", - inputs=inputs, - outputs={ - "Y": layer_norm_out, - "Mean": mean_out, - "Variance": variance_out, - }, - attrs={ - "epsilon": epsilon, - "begin_norm_axis": begin_norm_axis - }) + helper.append_op( + type="layer_norm", + inputs=inputs, + outputs={ + "Y": layer_norm_out, + "Mean": mean_out, + "Variance": variance_out, + }, + attrs={"epsilon": epsilon, "begin_norm_axis": begin_norm_axis}, + ) return helper.append_activation(layer_norm_out) @templatedoc() -def group_norm(input, - groups, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - act=None, - data_layout='NCHW', - name=None): +def group_norm( + input, + groups, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + act=None, + data_layout='NCHW', + name=None, +): """ :api_attr: Static Graph @@ -3835,8 +4140,9 @@ def group_norm(input, """ helper = LayerHelper('group_norm', **locals()) dtype = helper.input_dtype() - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'group_norm') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'group_norm' + ) # create intput and parameters inputs = {'X': input} input_shape = input.shape @@ -3847,20 +4153,23 @@ def group_norm(input, if data_layout != 'NCHW' and data_layout != 'NHWC': raise ValueError( "Param(data_layout) of Op(fluid.layers.group_norm) got wrong value: received " - + data_layout + " but only NCHW or NHWC supported.") + + data_layout + + " but only NCHW or NHWC supported." + ) channel_num = input_shape[1] if data_layout == 'NCHW' else input_shape[-1] param_shape = [channel_num] if param_attr: - scale = helper.create_parameter(attr=helper.param_attr, - shape=param_shape, - dtype=dtype, - default_initializer=Constant(1.0)) + scale = helper.create_parameter( + attr=helper.param_attr, + shape=param_shape, + dtype=dtype, + default_initializer=Constant(1.0), + ) inputs['Scale'] = scale if bias_attr: - bias = helper.create_parameter(attr=helper.bias_attr, - shape=param_shape, - dtype=dtype, - is_bias=True) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=param_shape, dtype=dtype, is_bias=True + ) inputs['Bias'] = bias # create output @@ -3868,18 +4177,20 @@ def group_norm(input, variance_out = helper.create_variable(dtype=dtype, stop_gradient=True) group_norm_out = helper.create_variable(dtype=dtype) - helper.append_op(type="group_norm", - inputs=inputs, - outputs={ - "Y": group_norm_out, - "Mean": mean_out, - "Variance": variance_out, - }, - attrs={ - "epsilon": epsilon, - "groups": groups, - "data_layout": data_layout - }) + helper.append_op( + type="group_norm", + inputs=inputs, + outputs={ + "Y": group_norm_out, + "Mean": mean_out, + "Variance": variance_out, + }, + attrs={ + "epsilon": epsilon, + "groups": groups, + "data_layout": data_layout, + }, + ) return helper.append_activation(group_norm_out) @@ -3948,8 +4259,9 @@ def spectral_norm(weight, dim=0, power_iters=1, eps=1e-12, name=None): print(x.shape) # [2, 8, 32, 32] """ helper = LayerHelper('spectral_norm', **locals()) - check_variable_and_dtype(weight, 'weight', ['float32', 'float64'], - 'spectral_norm') + check_variable_and_dtype( + weight, 'weight', ['float32', 'float64'], 'spectral_norm' + ) check_type(dim, 'dim', int, 'spectral_norm') check_type(power_iters, 'power_iters', int, 'spectral_norm') check_type(eps, 'eps', float, 'spectral_norm') @@ -3958,21 +4270,27 @@ def spectral_norm(weight, dim=0, power_iters=1, eps=1e-12, name=None): # create intput and parameters input_shape = weight.shape assert weight.numel() > 0, "Any dimension of input cannot be equal to 0." - assert dim < len(input_shape), ("The input `dim` should be less than the " - "rank of `weight`, but received dim=" - "{}".format(dim)) + assert dim < len(input_shape), ( + "The input `dim` should be less than the " + "rank of `weight`, but received dim=" + "{}".format(dim) + ) h = input_shape[dim] w = np.prod(input_shape) // h - u = helper.create_parameter(attr=ParamAttr(), - shape=[h], - dtype=dtype, - default_initializer=Normal(0., 1.)) + u = helper.create_parameter( + attr=ParamAttr(), + shape=[h], + dtype=dtype, + default_initializer=Normal(0.0, 1.0), + ) u.stop_gradient = True - v = helper.create_parameter(attr=ParamAttr(), - shape=[w], - dtype=dtype, - default_initializer=Normal(0., 1.)) + v = helper.create_parameter( + attr=ParamAttr(), + shape=[w], + dtype=dtype, + default_initializer=Normal(0.0, 1.0), + ) v.stop_gradient = True if in_dygraph_mode(): @@ -3985,34 +4303,38 @@ def spectral_norm(weight, dim=0, power_iters=1, eps=1e-12, name=None): # create output out = helper.create_variable(dtype=dtype) - helper.append_op(type="spectral_norm", - inputs=inputs, - outputs={ - "Out": out, - }, - attrs={ - "dim": dim, - "power_iters": power_iters, - "eps": eps, - }) + helper.append_op( + type="spectral_norm", + inputs=inputs, + outputs={ + "Out": out, + }, + attrs={ + "dim": dim, + "power_iters": power_iters, + "eps": eps, + }, + ) return out -def conv2d_transpose(input, - num_filters, - output_size=None, - filter_size=None, - padding=0, - stride=1, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - name=None, - data_format='NCHW'): +def conv2d_transpose( + input, + num_filters, + output_size=None, + filter_size=None, + padding=0, + stride=1, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + name=None, + data_format='NCHW', +): r""" :api_attr: Static Graph @@ -4168,20 +4490,29 @@ def conv2d_transpose(input, conv2d_transpose = paddle.static.nn.conv2d_transpose(input=data, num_filters=2, filter_size=3) print(conv2d_transpose.shape) # [-1, 2, 34, 34] """ - assert param_attr is not False, "param_attr should not be False in conv2d_transpose." + assert ( + param_attr is not False + ), "param_attr should not be False in conv2d_transpose." if len(input.shape) != 4: - raise ValueError("Input size should be 4, " - "but received {}".format(len(input.shape))) + raise ValueError( + "Input size should be 4, " + "but received {}".format(len(input.shape)) + ) if data_format not in ['NCHW', 'NHWC']: raise ValueError( "Attr(data_format) of Op(fluid.layers.conv2d_transpose) got wrong value: received " - + data_format + " but only NCHW or NHWC supported.") + + data_format + + " but only NCHW or NHWC supported." + ) input_channel = input.shape[1] if data_format == 'NCHW' else input.shape[-1] op_type = 'conv2d_transpose' - if (input_channel == groups and num_filters == input_channel - and not use_cudnn): + if ( + input_channel == groups + and num_filters == input_channel + and not use_cudnn + ): op_type = 'depthwise_conv2d_transpose' helper = LayerHelper(op_type, **locals()) @@ -4195,7 +4526,6 @@ def conv2d_transpose(input, raise ValueError("use_cudnn should be True or False") def _update_padding(padding, data_format): - def is_list_or_tuple(ele): if isinstance(ele, list) or isinstance(ele, tuple): return True @@ -4206,14 +4536,16 @@ def is_list_or_tuple(ele): if not (padding[0] == [0, 0] and padding[1] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[2:4] padding = [ele for a_list in padding for ele in a_list] elif is_list_or_tuple(padding[0]) and (data_format == "NHWC"): if not (padding[0] == [0, 0] and padding[3] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[1:3] padding = [ele for a_list in padding for ele in a_list] padding = utils.convert_to_list(padding, 4, 'padding') @@ -4227,8 +4559,9 @@ def is_list_or_tuple(ele): padding = padding.upper() if padding not in ["SAME", "VALID"]: raise ValueError( - "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." % - str(padding)) + "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." + % str(padding) + ) if padding == "VALID": padding_algorithm = "VALID" padding = [0, 0, 0, 0] @@ -4248,44 +4581,63 @@ def is_list_or_tuple(ele): elif isinstance(output_size, int): output_size = utils.convert_to_list(output_size, 2, 'output_size') elif isinstance(output_size, Variable): - check_dtype(output_size.dtype, 'output_size', ['int32', 'int64'], - 'conv2d_transpose') - if len(output_size.shape) == 1 and (output_size.shape[0] == 1 - or output_size.shape[0] == 2): + check_dtype( + output_size.dtype, + 'output_size', + ['int32', 'int64'], + 'conv2d_transpose', + ) + if len(output_size.shape) == 1 and ( + output_size.shape[0] == 1 or output_size.shape[0] == 2 + ): if output_size.shape[0] == 1: output_size = [output_size, output_size] else: raise ValueError("output_size must contain one or two integers.") else: raise ValueError( - "output_size should be int, list[int] or tuple[int] or Tensor") + "output_size should be int, list[int] or tuple[int] or Tensor" + ) if filter_size is None: if output_size is []: raise ValueError("output_size must be set when filter_size is None") if not _non_static_mode(): - if isinstance(output_size, - Variable) or utils._contain_var(output_size): + if isinstance(output_size, Variable) or utils._contain_var( + output_size + ): raise ValueError( "filter_size should not be None when output_size is Variable or contain Variable in static mode." ) else: output_size = utils.convert_shape_to_list(output_size) if len(output_size) == 1: - output_size = utils.convert_to_list(output_size[0], 2, - 'output_size') + output_size = utils.convert_to_list( + output_size[0], 2, 'output_size' + ) h_in = input.shape[2] if data_format == 'NCHW' else input.shape[1] w_in = input.shape[3] if data_format == 'NCHW' else input.shape[2] - filter_size_h = (output_size[0] - (h_in - 1) * stride[0] + padding[0] + - padding[1] - 1) // dilation[0] + 1 - filter_size_w = (output_size[1] - (w_in - 1) * stride[1] + padding[2] + - padding[3] - 1) // dilation[1] + 1 + filter_size_h = ( + output_size[0] + - (h_in - 1) * stride[0] + + padding[0] + + padding[1] + - 1 + ) // dilation[0] + 1 + filter_size_w = ( + output_size[1] + - (w_in - 1) * stride[1] + + padding[2] + + padding[3] + - 1 + ) // dilation[1] + 1 filter_size = [filter_size_h, filter_size_w] else: - filter_size = utils.convert_to_list(filter_size, 2, - 'conv2d_transpose.filter_size') + filter_size = utils.convert_to_list( + filter_size, 2, 'conv2d_transpose.filter_size' + ) if len(padding) == 4 and utils._is_symmetric_padding(padding, 2): padding = [padding[0], padding[2]] @@ -4295,31 +4647,31 @@ def is_list_or_tuple(ele): elif groups <= 0: raise ValueError( "the groups of input must be greater than 0, " - "but received the groups of input is {}".format(groups)) + "but received the groups of input is {}".format(groups) + ) filter_shape = [input_channel, num_filters // groups] + filter_size - img_filter = helper.create_parameter(dtype=input.dtype, - shape=filter_shape, - attr=helper.param_attr) + img_filter = helper.create_parameter( + dtype=input.dtype, shape=filter_shape, attr=helper.param_attr + ) pre_bias = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type=op_type, - inputs={ - 'Input': [input], - 'Filter': [img_filter] - }, - outputs={'Output': pre_bias}, - attrs={ - 'output_size': output_size, - 'strides': stride, - 'paddings': padding, - 'padding_algorithm': padding_algorithm, - 'dilations': dilation, - 'groups': groups, - 'use_cudnn': use_cudnn, - 'data_format': data_format - }) + helper.append_op( + type=op_type, + inputs={'Input': [input], 'Filter': [img_filter]}, + outputs={'Output': pre_bias}, + attrs={ + 'output_size': output_size, + 'strides': stride, + 'paddings': padding, + 'padding_algorithm': padding_algorithm, + 'dilations': dilation, + 'groups': groups, + 'use_cudnn': use_cudnn, + 'data_format': data_format, + }, + ) if data_format == 'NCHW': pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2) @@ -4329,20 +4681,22 @@ def is_list_or_tuple(ele): return out -def conv3d_transpose(input, - num_filters, - output_size=None, - filter_size=None, - padding=0, - stride=1, - dilation=1, - groups=None, - param_attr=None, - bias_attr=None, - use_cudnn=True, - act=None, - name=None, - data_format='NCDHW'): +def conv3d_transpose( + input, + num_filters, + output_size=None, + filter_size=None, + padding=0, + stride=1, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + name=None, + data_format='NCDHW', +): r""" :api_attr: Static Graph @@ -4506,11 +4860,15 @@ def conv3d_transpose(input, output = exe.run(feed={"data": x}, fetch_list=[res]) print(output) """ - assert param_attr is not False, "param_attr should not be False in conv3d_transpose." + assert ( + param_attr is not False + ), "param_attr should not be False in conv3d_transpose." if data_format not in ['NCDHW', 'NDHWC']: raise ValueError( "Param(data_format) of Op(fluid.layers.conv3d_transpose) got wrong value: received " - + data_format + " but only NCDHW or NDHWC supported.") + + data_format + + " but only NCDHW or NDHWC supported." + ) l_type = "conv3d_transpose" helper = LayerHelper(l_type, **locals()) @@ -4518,9 +4876,13 @@ def conv3d_transpose(input, raise TypeError("Input of conv3d_transpose must be Variable") if len(input.shape) != 5: raise ValueError( - "Input should be 5D tensor, but received input with the shape of {}" - .format(input.shape)) - input_channel = input.shape[1] if data_format == 'NCDHW' else input.shape[-1] + "Input should be 5D tensor, but received input with the shape of {}".format( + input.shape + ) + ) + input_channel = ( + input.shape[1] if data_format == 'NCDHW' else input.shape[-1] + ) stride = utils.convert_to_list(stride, 3, 'stride') dilation = utils.convert_to_list(dilation, 3, 'dilation') @@ -4529,7 +4891,6 @@ def conv3d_transpose(input, raise ValueError("use_cudnn should be True or False") def _update_padding(padding, data_format): - def is_list_or_tuple(ele): if isinstance(ele, list) or isinstance(ele, tuple): return True @@ -4540,14 +4901,16 @@ def is_list_or_tuple(ele): if not (padding[0] == [0, 0] and padding[1] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[2:5] padding = [ele for a_list in padding for ele in a_list] elif is_list_or_tuple(padding[0]) and (data_format == "NDHWC"): if not (padding[0] == [0, 0] and padding[4] == [0, 0]): raise ValueError( "Non-zero padding(%s) in the batch or channel dimensions " - "is not supported." % str(padding)) + "is not supported." % str(padding) + ) padding = padding[1:4] padding = [ele for a_list in padding for ele in a_list] padding = utils.convert_to_list(padding, 6, 'padding') @@ -4558,8 +4921,12 @@ def is_list_or_tuple(ele): else: padding = utils.convert_to_list(padding, 3, 'padding') padding = [ - padding[0], padding[0], padding[1], padding[1], padding[2], - padding[2] + padding[0], + padding[0], + padding[1], + padding[1], + padding[2], + padding[2], ] return padding @@ -4568,8 +4935,9 @@ def is_list_or_tuple(ele): padding = padding.upper() if padding not in ["SAME", "VALID"]: raise ValueError( - "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." % - str(padding)) + "Unknown padding: '%s'. It can only be 'SAME' or 'VALID'." + % str(padding) + ) if padding == "VALID": padding_algorithm = "VALID" padding = [0, 0, 0, 0, 0, 0] @@ -4589,16 +4957,32 @@ def is_list_or_tuple(ele): h_in = input.shape[3] if data_format == 'NCDHW' else input.shape[2] w_in = input.shape[4] if data_format == 'NCDHW' else input.shape[3] - filter_size_d = (output_size[0] - (d_in - 1) * stride[0] + padding[0] + - padding[1] - 1) // dilation[0] + 1 - filter_size_h = (output_size[1] - (h_in - 1) * stride[1] + padding[2] + - padding[3] - 1) // dilation[1] + 1 - filter_size_w = (output_size[2] - (w_in - 1) * stride[2] + padding[4] + - padding[5] - 1) // dilation[2] + 1 + filter_size_d = ( + output_size[0] + - (d_in - 1) * stride[0] + + padding[0] + + padding[1] + - 1 + ) // dilation[0] + 1 + filter_size_h = ( + output_size[1] + - (h_in - 1) * stride[1] + + padding[2] + + padding[3] + - 1 + ) // dilation[1] + 1 + filter_size_w = ( + output_size[2] + - (w_in - 1) * stride[2] + + padding[4] + + padding[5] + - 1 + ) // dilation[2] + 1 filter_size = [filter_size_d, filter_size_h, filter_size_w] else: - filter_size = utils.convert_to_list(filter_size, 3, - 'conv3d_transpose.filter_size') + filter_size = utils.convert_to_list( + filter_size, 3, 'conv3d_transpose.filter_size' + ) if len(padding) == 6 and utils._is_symmetric_padding(padding, 3): padding = [padding[0], padding[2], padding[4]] @@ -4613,18 +4997,22 @@ def is_list_or_tuple(ele): groups = 1 if groups is None else groups if groups <= 0: raise ValueError( - "the groups of conv3d_transpose should be greater than 0. Received groups: {}" - .format(groups)) + "the groups of conv3d_transpose should be greater than 0. Received groups: {}".format( + groups + ) + ) if num_filters % groups != 0: raise ValueError( "Attr(num_filters) must be divisible by groups," "Received: Attr(num_filters) is {}, the groups is {}".format( - num_filters, groups)) + num_filters, groups + ) + ) filter_shape = [input_channel, num_filters // groups] + filter_size - img_filter = helper.create_parameter(dtype=input.dtype, - shape=filter_shape, - attr=helper.param_attr) + img_filter = helper.create_parameter( + dtype=input.dtype, shape=filter_shape, attr=helper.param_attr + ) if data_format == 'NCDHW': data_format = 'NCHW' @@ -4632,22 +5020,21 @@ def is_list_or_tuple(ele): data_format = 'NHWC' pre_bias = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type=l_type, - inputs={ - 'Input': [input], - 'Filter': [img_filter] - }, - outputs={'Output': pre_bias}, - attrs={ - 'output_size': output_size, - 'strides': stride, - 'paddings': padding, - 'padding_algorithm': padding_algorithm, - 'dilations': dilation, - 'groups': groups, - 'use_cudnn': use_cudnn, - 'data_format': data_format - }) + helper.append_op( + type=l_type, + inputs={'Input': [input], 'Filter': [img_filter]}, + outputs={'Output': pre_bias}, + attrs={ + 'output_size': output_size, + 'strides': stride, + 'paddings': padding, + 'padding_algorithm': padding_algorithm, + 'dilations': dilation, + 'groups': groups, + 'use_cudnn': use_cudnn, + 'data_format': data_format, + }, + ) if data_format == 'NCHW': pre_act = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2) @@ -4717,18 +5104,24 @@ def reduce_sum(input, dim=None, keep_dim=False, name=None): if in_dygraph_mode(): return _C_ops.sum(input, dim, None, keep_dim) elif _in_legacy_dygraph(): - return _legacy_C_ops.reduce_sum(input, 'dim', dim, 'keep_dim', keep_dim, - 'reduce_all', reduce_all) + return _legacy_C_ops.reduce_sum( + input, 'dim', dim, 'keep_dim', keep_dim, 'reduce_all', reduce_all + ) attrs = {'dim': dim, 'keep_dim': keep_dim, 'reduce_all': reduce_all} check_variable_and_dtype( - input, 'input', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'reduce_sum') + input, + 'input', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'reduce_sum', + ) helper = LayerHelper('reduce_sum', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='reduce_sum', - inputs={'X': input}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='reduce_sum', + inputs={'X': input}, + outputs={'Out': out}, + attrs=attrs, + ) return out @@ -4846,18 +5239,18 @@ def reduce_max(input, dim=None, keep_dim=False, name=None): if in_dygraph_mode(): return _C_ops.max(input, dim if dim != None else [], keep_dim) - helper.append_op(type='reduce_max', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'dim': - dim if dim != None and dim != [] else [0], - 'keep_dim': - keep_dim, - 'reduce_all': - True if dim == None or dim == [] - or len(dim) == len(input.shape) else False - }) + helper.append_op( + type='reduce_max', + inputs={'X': input}, + outputs={'Out': out}, + attrs={ + 'dim': dim if dim != None and dim != [] else [0], + 'keep_dim': keep_dim, + 'reduce_all': True + if dim == None or dim == [] or len(dim) == len(input.shape) + else False, + }, + ) return out @@ -4918,18 +5311,18 @@ def reduce_min(input, dim=None, keep_dim=False, name=None): if in_dygraph_mode(): return _C_ops.min(input, dim if dim != None else [], keep_dim) - helper.append_op(type='reduce_min', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'dim': - dim if dim != None and dim != [] else [0], - 'keep_dim': - keep_dim, - 'reduce_all': - True if dim == None or dim == [] - or len(dim) == len(input.shape) else False - }) + helper.append_op( + type='reduce_min', + inputs={'X': input}, + outputs={'Out': out}, + attrs={ + 'dim': dim if dim != None and dim != [] else [0], + 'keep_dim': keep_dim, + 'reduce_all': True + if dim == None or dim == [] or len(dim) == len(input.shape) + else False, + }, + ) return out @@ -4990,30 +5383,37 @@ def reduce_prod(input, dim=None, keep_dim=False, name=None): dim = [dim] else: raise TypeError( - "The type of axis must be int, list or tuple, but received {}". - format(type(dim))) + "The type of axis must be int, list or tuple, but received {}".format( + type(dim) + ) + ) if in_dygraph_mode(): return _C_ops.reduce_prod( - input, dim if dim != None and dim != [] else [0], keep_dim, True if - dim == None or dim == [] or len(dim) == len(input.shape) else False) + input, + dim if dim != None and dim != [] else [0], + keep_dim, + True + if dim == None or dim == [] or len(dim) == len(input.shape) + else False, + ) helper = LayerHelper('reduce_prod', **locals()) - check_variable_and_dtype(input, 'input', - ['float32', 'float64', 'int32', 'int64'], - 'reduce_prod') + check_variable_and_dtype( + input, 'input', ['float32', 'float64', 'int32', 'int64'], 'reduce_prod' + ) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='reduce_prod', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'dim': - dim if dim != None and dim != [] else [0], - 'keep_dim': - keep_dim, - 'reduce_all': - True if dim == None or dim == [] - or len(dim) == len(input.shape) else False - }) + helper.append_op( + type='reduce_prod', + inputs={'X': input}, + outputs={'Out': out}, + attrs={ + 'dim': dim if dim != None and dim != [] else [0], + 'keep_dim': keep_dim, + 'reduce_all': True + if dim == None or dim == [] or len(dim) == len(input.shape) + else False, + }, + ) return out @@ -5070,18 +5470,18 @@ def reduce_all(input, dim=None, keep_dim=False, name=None): check_variable_and_dtype(input, 'input', ('bool'), 'reduce_all') helper = LayerHelper('reduce_all', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='reduce_all', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'dim': - dim if dim != None and dim != [] else [0], - 'keep_dim': - keep_dim, - 'reduce_all': - True if dim == None or dim == [] - or len(dim) == len(input.shape) else False - }) + helper.append_op( + type='reduce_all', + inputs={'X': input}, + outputs={'Out': out}, + attrs={ + 'dim': dim if dim != None and dim != [] else [0], + 'keep_dim': keep_dim, + 'reduce_all': True + if dim == None or dim == [] or len(dim) == len(input.shape) + else False, + }, + ) return out @@ -5133,18 +5533,18 @@ def reduce_any(input, dim=None, keep_dim=False, name=None): out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) if dim is not None and not isinstance(dim, list): dim = [dim] - helper.append_op(type='reduce_any', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'dim': - dim if dim != None and dim != [] else [0], - 'keep_dim': - keep_dim, - 'reduce_all': - True if dim == None or dim == [] - or len(dim) == len(input.shape) else False - }) + helper.append_op( + type='reduce_any', + inputs={'X': input}, + outputs={'Out': out}, + attrs={ + 'dim': dim if dim != None and dim != [] else [0], + 'keep_dim': keep_dim, + 'reduce_all': True + if dim == None or dim == [] or len(dim) == len(input.shape) + else False, + }, + ) return out @@ -5219,15 +5619,17 @@ def split(input, num_or_sections, dim=-1, name=None): if utils._contain_var(num_or_sections): for index, item in enumerate(num_or_sections): if isinstance(item, Variable): - num_or_sections[index] = num_or_sections[index].numpy( - )[0] + num_or_sections[index] = num_or_sections[index].numpy()[ + 0 + ] attrs += ('sections', list(num_or_sections)) else: attrs += ('sections', list(num_or_sections)) else: raise TypeError( "The type of 'num_or_sections' in split must be int, list or tuple in imperative mode, but " - "received %s." % (type(num_or_sections))) + "received %s." % (type(num_or_sections)) + ) if in_dygraph_mode(): if isinstance(num_or_sections, int): return _C_ops.split_with_num(input, num_or_sections, dim) @@ -5239,8 +5641,11 @@ def split(input, num_or_sections, dim=-1, name=None): return out check_variable_and_dtype( - input, 'input', - ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], 'split') + input, + 'input', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'split', + ) check_type(num_or_sections, 'num_or_sections', (list, int, tuple), 'split') check_type(dim, 'dim', (int, Variable), 'split') if isinstance(dim, Variable): @@ -5260,19 +5665,18 @@ def _get_SectionsTensorList(one_list): dim_size.stop_gradient = True tensor_list.append(dim_size) else: - assert (isinstance(dim_size, int)) + assert isinstance(dim_size, int) if dim_size == -1: assert unk_dim_idx == -1, ( "Only one value of 'num_or_section' in split can " - "be -1. But received num_or_section[%d] is also -1." % - idx) + "be -1. But received num_or_section[%d] is also -1." + % idx + ) unk_dim_idx = idx temp_out = helper.create_variable_for_type_inference('int32') - fill_constant([1], - 'int32', - dim_size, - force_cpu=True, - out=temp_out) + fill_constant( + [1], 'int32', dim_size, force_cpu=True, out=temp_out + ) tensor_list.append(temp_out) return tensor_list @@ -5287,31 +5691,37 @@ def _get_SectionsTensorList(one_list): if isinstance(num_or_sections, int): assert num_or_sections > 1, 'num_or_sections must be more than 1.' if isinstance(dim, int) and input_shape[dim] > 0: - assert input_shape[dim] % num_or_sections ==0, \ - "The input's size along the split dimension " \ - "must be evenly divisible by Attr(num_or_sections). " \ - "But %d is not evenly divisible by %d. " % (num_or_sections,input_shape[dim]) + assert input_shape[dim] % num_or_sections == 0, ( + "The input's size along the split dimension " + "must be evenly divisible by Attr(num_or_sections). " + "But %d is not evenly divisible by %d. " + % (num_or_sections, input_shape[dim]) + ) num = num_or_sections else: if isinstance(dim, int) and input_shape[dim] > 0: - assert len(num_or_sections) <= input_shape[ - dim], 'len(num_or_sections) must not be more than input.shape[dim].' + assert ( + len(num_or_sections) <= input_shape[dim] + ), 'len(num_or_sections) must not be more than input.shape[dim].' num = len(num_or_sections) attrs['sections'] = list( - map(lambda ele: -1 - if isinstance(ele, Variable) else ele, num_or_sections)) + map( + lambda ele: -1 if isinstance(ele, Variable) else ele, + num_or_sections, + ) + ) if utils._contain_var(num_or_sections): inputs['SectionsTensorList'] = _get_SectionsTensorList( - num_or_sections) + num_or_sections + ) outs = [ helper.create_variable_for_type_inference(dtype=helper.input_dtype()) for i in range(num) ] - helper.append_op(type='split', - inputs=inputs, - outputs={'Out': outs}, - attrs=attrs) + helper.append_op( + type='split', inputs=inputs, outputs={'Out': outs}, attrs=attrs + ) return outs @@ -5362,8 +5772,9 @@ def l2_normalize(x, axis, epsilon=1e-12, name=None): if in_dygraph_mode(): out, _ = _C_ops.norm(x, 1 if axis is None else axis, epsilon, False) elif _in_legacy_dygraph(): - _, out = _legacy_C_ops.norm(x, 'axis', 1 if axis is None else axis, - 'epsilon', epsilon) + _, out = _legacy_C_ops.norm( + x, 'axis', 1 if axis is None else axis, 'epsilon', epsilon + ) return out check_variable_and_dtype(x, "X", ("float16", "float32", "float64"), "norm") @@ -5371,16 +5782,15 @@ def l2_normalize(x, axis, epsilon=1e-12, name=None): helper = LayerHelper("l2_normalize", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) norm = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="norm", - inputs={"X": x}, - outputs={ - "Out": out, - "Norm": norm - }, - attrs={ - "axis": 1 if axis is None else axis, - "epsilon": epsilon, - }) + helper.append_op( + type="norm", + inputs={"X": x}, + outputs={"Out": out, "Norm": norm}, + attrs={ + "axis": 1 if axis is None else axis, + "epsilon": epsilon, + }, + ) return out @@ -5461,16 +5871,25 @@ def matmul(x, y, transpose_x=False, transpose_y=False, alpha=1.0, name=None): """ if _non_static_mode(): out = _varbase_creator(dtype=x.dtype) - _legacy_C_ops.matmul(x, y, out, 'transpose_X', transpose_x, - 'transpose_Y', transpose_y, 'alpha', float(alpha)) + _legacy_C_ops.matmul( + x, + y, + out, + 'transpose_X', + transpose_x, + 'transpose_Y', + transpose_y, + 'alpha', + float(alpha), + ) return out def __check_input(x, y): var_names = {'x': x, 'y': y} for name, val in var_names.items(): - check_variable_and_dtype(val, name, - ['float16', 'float32', 'float64'], - 'matmul') + check_variable_and_dtype( + val, name, ['float16', 'float32', 'float64'], 'matmul' + ) x_shape = list(x.shape) y_shape = list(y.shape) if len(x_shape) == 1: @@ -5484,11 +5903,12 @@ def __check_input(x, y): if transpose_y: y_shape[-2], y_shape[-1] = y_shape[-1], y_shape[-2] if x_shape[-1] != y_shape[-2]: - assert (x_shape[-1] == -1) or (y_shape[-2] == -1), \ - "After performing an optional transpose, Input X's width should be " \ - "equal to Y's width for multiplication " \ - "prerequisites. But received X's shape: %s, Y's shape: %s\n" % \ - (x_shape, y_shape) + assert (x_shape[-1] == -1) or (y_shape[-2] == -1), ( + "After performing an optional transpose, Input X's width should be " + "equal to Y's width for multiplication " + "prerequisites. But received X's shape: %s, Y's shape: %s\n" + % (x_shape, y_shape) + ) if len(y_shape) > 2 and len(x_shape) > 2: for i, dim_x in enumerate(x_shape[:-2]): @@ -5500,7 +5920,8 @@ def __check_input(x, y): "When the matrix is larger than 2 dimensions, the higher " "dimensional values of the two matrices need to be equal. " "But received x_shape[%d] != y_shape[%d]. X's shape: %s, " - "Y's shape: %s.\n" % (i, i, x_shape, y_shape)) + "Y's shape: %s.\n" % (i, i, x_shape, y_shape) + ) attrs = { 'transpose_X': transpose_x, @@ -5512,21 +5933,20 @@ def __check_input(x, y): helper = LayerHelper('matmul', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='matmul', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='matmul', + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs=attrs, + ) return out def topk(input, k, name=None): """ :alias_main: paddle.topk - :alias: paddle.topk,paddle.tensor.topk,paddle.tensor.search.topk - :old_api: paddle.fluid.layers.topk + :alias: paddle.topk,paddle.tensor.topk,paddle.tensor.search.topk + :old_api: paddle.fluid.layers.topk This OP is used to find values and indices of the k largest entries for the last dimension. @@ -5611,23 +6031,20 @@ def topk(input, k, name=None): values = helper.create_variable_for_type_inference(dtype=input.dtype) indices = helper.create_variable_for_type_inference(dtype="int64") - helper.append_op(type="top_k", - inputs=inputs, - outputs={ - "Out": [values], - "Indices": [indices] - }, - attrs=attrs) + helper.append_op( + type="top_k", + inputs=inputs, + outputs={"Out": [values], "Indices": [indices]}, + attrs=attrs, + ) values.stop_gradient = True indices.stop_gradient = True return values, indices -def ctc_greedy_decoder(input, - blank, - input_length=None, - padding_value=0, - name=None): +def ctc_greedy_decoder( + input, blank, input_length=None, padding_value=0, name=None +): r""" This op is used to decode sequences by greedy policy by the following steps: @@ -5753,8 +6170,9 @@ def ctc_greedy_decoder(input, input_length=x_pad_len) """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'ctc_greedy_decoder') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'ctc_greedy_decoder' + ) helper = LayerHelper("ctc_greedy_decoder", **locals()) _, topk_indices = topk(input, k=1) @@ -5763,32 +6181,27 @@ def ctc_greedy_decoder(input, ctc_out = helper.create_variable_for_type_inference(dtype="int64") if input_length is None: - helper.append_op(type="ctc_align", - inputs={"Input": [topk_indices]}, - outputs={"Output": [ctc_out]}, - attrs={ - "merge_repeated": True, - "blank": blank - }) + helper.append_op( + type="ctc_align", + inputs={"Input": [topk_indices]}, + outputs={"Output": [ctc_out]}, + attrs={"merge_repeated": True, "blank": blank}, + ) return ctc_out else: ctc_out_len = helper.create_variable_for_type_inference(dtype="int64") ctc_input = squeeze(topk_indices, [2]) - helper.append_op(type="ctc_align", - inputs={ - "Input": [ctc_input], - "InputLength": [input_length] - }, - outputs={ - "Output": [ctc_out], - "OutputLength": [ctc_out_len] - }, - attrs={ - "merge_repeated": True, - "blank": blank, - "padding_value": padding_value - }) + helper.append_op( + type="ctc_align", + inputs={"Input": [ctc_input], "InputLength": [input_length]}, + outputs={"Output": [ctc_out], "OutputLength": [ctc_out_len]}, + attrs={ + "merge_repeated": True, + "blank": blank, + "padding_value": padding_value, + }, + ) return ctc_out, ctc_out_len @@ -5849,10 +6262,21 @@ def transpose(x, perm, name=None): out, _ = _legacy_C_ops.transpose2(x, 'axis', perm) return out - check_variable_and_dtype(x, 'x', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], 'transpose') + check_variable_and_dtype( + x, + 'x', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'transpose', + ) check_type(perm, 'perm', (list, tuple), 'transpose') if isinstance(perm, tuple): perm = list(perm) @@ -5861,34 +6285,37 @@ def transpose(x, perm, name=None): "Input(perm) is the permutation of dimensions of Input(x), " "its length should be equal to dimensions of Input(x), " "but received dimension of Input(x) is %s, " - "the length of Input(perm) is %s." % (len(x.shape), len(perm))) + "the length of Input(perm) is %s." % (len(x.shape), len(perm)) + ) for idx, dim in enumerate(perm): if dim >= len(x.shape): raise ValueError( "Each element in Input(perm) should be less than Input(x)'s dimension, " "but %d-th element in Input(perm) is %d which exceeds Input(x)'s " - "dimension %d." % (idx, perm[idx], len(x.shape))) + "dimension %d." % (idx, perm[idx], len(x.shape)) + ) helper = LayerHelper('transpose', **locals()) out = helper.create_variable_for_type_inference(x.dtype) x_shape = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='transpose2', - inputs={'X': [x]}, - outputs={ - 'Out': [out], - 'XShape': [x_shape] - }, - attrs={'axis': perm}) + helper.append_op( + type='transpose2', + inputs={'X': [x]}, + outputs={'Out': [out], 'XShape': [x_shape]}, + attrs={'axis': perm}, + ) return out -def im2sequence(input, - filter_size=1, - stride=1, - padding=0, - input_image_size=None, - out_stride=1, - name=None): +def im2sequence( + input, + filter_size=1, + stride=1, + padding=0, + input_image_size=None, + out_stride=1, + name=None, +): r""" :api_attr: Static Graph @@ -6002,8 +6429,9 @@ def im2sequence(input, """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." check_variable_and_dtype(input, 'input', ['float32'], 'im2sequence') @@ -6025,10 +6453,9 @@ def im2sequence(input, attrs["out_stride"] = out_stride helper = LayerHelper('im2sequence', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='im2sequence', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='im2sequence', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -6068,16 +6495,15 @@ def row_conv(input, future_context_size, param_attr=None, act=None): check_variable_and_dtype(input, 'input', ['float32'], 'row_conv') dtype = helper.input_dtype() filter_shape = [future_context_size + 1, input.shape[-1]] - filter_param = helper.create_parameter(attr=helper.param_attr, - shape=filter_shape, - dtype=dtype) + filter_param = helper.create_parameter( + attr=helper.param_attr, shape=filter_shape, dtype=dtype + ) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='row_conv', - inputs={ - 'X': [input], - 'Filter': [filter_param] - }, - outputs={'Out': [out]}) + helper.append_op( + type='row_conv', + inputs={'X': [input], 'Filter': [filter_param]}, + outputs={'Out': [out]}, + ) return helper.append_activation(out) @@ -6143,20 +6569,23 @@ def multiplex(inputs, index, name=None): check_type(inputs, 'inputs', (list), 'multiplex') if len(inputs) < 2: raise ValueError( - "inputs should be a list object with at least 2 elements.") + "inputs should be a list object with at least 2 elements." + ) for id, x in enumerate(inputs): - check_variable_and_dtype(x, 'input[' + str(id) + ']', - ['float32', 'float64', 'int32', 'int64'], - 'multiplex') + check_variable_and_dtype( + x, + 'input[' + str(id) + ']', + ['float32', 'float64', 'int32', 'int64'], + 'multiplex', + ) check_variable_and_dtype(index, "index", ['int32', 'int64'], 'multiplex') out = helper.create_variable_for_type_inference(inputs[0].dtype) - helper.append_op(type='multiplex', - inputs={ - 'X': inputs, - 'Ids': index - }, - outputs={'Out': [out]}) + helper.append_op( + type='multiplex', + inputs={'X': inputs, 'Ids': index}, + outputs={'Out': [out]}, + ) return out @@ -6223,18 +6652,17 @@ def smooth_l1(x, y, inside_weight=None, outside_weight=None, sigma=None): diff = helper.create_variable_for_type_inference(dtype=x.dtype) loss = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='smooth_l1_loss', - inputs={ - 'X': x, - 'Y': y, - 'InsideWeight': inside_weight, - 'OutsideWeight': outside_weight - }, - outputs={ - 'Diff': diff, - 'Out': loss - }, - attrs={'sigma': sigma if sigma is not None else 1.0}) + helper.append_op( + type='smooth_l1_loss', + inputs={ + 'X': x, + 'Y': y, + 'InsideWeight': inside_weight, + 'OutsideWeight': outside_weight, + }, + outputs={'Diff': diff, 'Out': loss}, + attrs={'sigma': sigma if sigma is not None else 1.0}, + ) return loss @@ -6326,10 +6754,12 @@ def one_hot(input, depth, allow_out_of_range=False): if isinstance(depth, Variable): depth = depth.numpy() assert depth.shape == ( - 1, ), "depth of type Variable should have shape [1]" + 1, + ), "depth of type Variable should have shape [1]" depth = depth.item(0) - out = _legacy_C_ops.one_hot(input, 'depth', depth, 'allow_out_of_range', - allow_out_of_range) + out = _legacy_C_ops.one_hot( + input, 'depth', depth, 'allow_out_of_range', allow_out_of_range + ) out.stop_gradient = True return out @@ -6346,10 +6776,9 @@ def one_hot(input, depth, allow_out_of_range=False): depth.stop_gradient = True inputs = {'X': input, 'depth_tensor': depth} attrs = {'allow_out_of_range': allow_out_of_range} - helper.append_op(type="one_hot", - inputs=inputs, - attrs=attrs, - outputs={'Out': one_hot_out}) + helper.append_op( + type="one_hot", inputs=inputs, attrs=attrs, outputs={'Out': one_hot_out} + ) one_hot_out.stop_gradient = True return one_hot_out @@ -6387,16 +6816,18 @@ def autoincreased_step_counter(counter_name=None, begin=1, step=1): dtype='int64', shape=[1], persistable=True, - belong_to_optimizer=True) + belong_to_optimizer=True, + ) if is_new_var: - helper.set_variable_initializer(counter, - initializer=Constant(value=begin - 1, - force_cpu=True)) + helper.set_variable_initializer( + counter, initializer=Constant(value=begin - 1, force_cpu=True) + ) helper.main_program.global_block()._prepend_op( type='increment', inputs={'X': [counter]}, outputs={'Out': [counter]}, - attrs={'step': float(step)}) + attrs={'step': float(step)}, + ) counter.stop_gradient = True return counter @@ -6500,7 +6931,7 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=False, name=None): """ if in_dygraph_mode(): tmp_tensor_type = core.eager.Tensor - #TODO(zhiqiu): enable inplace in dygraph mode. + # TODO(zhiqiu): enable inplace in dygraph mode. if inplace: warnings.warn( "Inplace on reshape is not allowed and will be discarded in dygraph mode currently." @@ -6518,7 +6949,8 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=False, name=None): else: raise ValueError( "shape must be an instance of `list`, `tuple` or `Variable`," - " got '{}.'".format(type(shape))) + " got '{}.'".format(type(shape)) + ) return dygraph_utils._append_activation_in_dygraph(out, act) else: @@ -6540,14 +6972,26 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=False, name=None): else: raise ValueError( "shape must be an instance of `list`, `tuple` or `Variable`," - " got '{}.'".format(type(shape))) + " got '{}.'".format(type(shape)) + ) return dygraph_utils._append_activation_in_dygraph(out, act) - check_variable_and_dtype(x, 'x', [ - 'float16', 'float32', 'float64', 'int16', 'int32', 'int64', 'bool', - 'uint16' - ], 'reshape') + check_variable_and_dtype( + x, + 'x', + [ + 'float16', + 'float32', + 'float64', + 'int16', + 'int32', + 'int64', + 'bool', + 'uint16', + ], + 'reshape', + ) check_type(shape, 'shape', (list, tuple, Variable), 'reshape') check_type(actual_shape, 'actual_shape', (Variable, type(None)), 'reshape') @@ -6571,20 +7015,23 @@ def get_attr_shape(list_shape): "\t# z.shape is [-1, -1, 4]\n\n" " If your target shape in Reshape represents dynamic shape, " "please turn it into a Tensor under @to_static. See above example for details." - % dim_idx) + % dim_idx + ) unk_dim_idx = dim_idx elif dim_size == 0: assert dim_idx < len(x.shape), ( "The index of 0 in `shape` must be less than " "the input tensor X's dimensions. " - "But received shape[%d] = 0, X's dimensions = %d." % - (dim_idx, len(x.shape))) + "But received shape[%d] = 0, X's dimensions = %d." + % (dim_idx, len(x.shape)) + ) else: assert dim_size > 0, ( "Each dimension value of 'shape' in reshape must not " "be negative except one unknown dimension. " - "But received shape[%d] = %s." % - (dim_idx, str(dim_size))) + "But received shape[%d] = %s." + % (dim_idx, str(dim_size)) + ) return attrs_shape inputs = {"X": x} @@ -6593,8 +7040,10 @@ def get_attr_shape(list_shape): shape.stop_gradient = True inputs["Shape"] = shape elif isinstance(shape, (list, tuple)): - assert len(shape) > 0, ("The size of 'shape' in reshape can't be zero, " - "but received %s." % len(shape)) + assert len(shape) > 0, ( + "The size of 'shape' in reshape can't be zero, " + "but received %s." % len(shape) + ) attrs["shape"] = get_attr_shape(shape) if utils._contain_var(shape): inputs['ShapeTensor'] = utils._convert_to_tensor_list(shape) @@ -6602,16 +7051,18 @@ def get_attr_shape(list_shape): actual_shape.stop_gradient = True inputs["Shape"] = actual_shape - out = x if inplace else helper.create_variable_for_type_inference( - dtype=x.dtype) + out = ( + x + if inplace + else helper.create_variable_for_type_inference(dtype=x.dtype) + ) x_shape = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="reshape2", - inputs=inputs, - attrs=attrs, - outputs={ - "Out": out, - "XShape": x_shape - }) + helper.append_op( + type="reshape2", + inputs=inputs, + attrs=attrs, + outputs={"Out": out, "XShape": x_shape}, + ) return helper.append_activation(out) @@ -6676,10 +7127,22 @@ def squeeze(input, axes, name=None): return out helper = LayerHelper("squeeze", **locals()) - check_variable_and_dtype(input, 'input', [ - 'float16', 'float32', 'float64', 'bool', 'int8', 'int32', 'int64', - 'complex64', 'complex128' - ], 'squeeze') + check_variable_and_dtype( + input, + 'input', + [ + 'float16', + 'float32', + 'float64', + 'bool', + 'int8', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'squeeze', + ) check_type(axes, 'axis/axes', (list, tuple, Variable), 'squeeze') attrs = {} @@ -6693,13 +7156,12 @@ def squeeze(input, axes, name=None): attrs["axes"] = axes out = helper.create_variable_for_type_inference(dtype=input.dtype) x_shape = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type="squeeze2", - inputs={"X": input}, - attrs=attrs, - outputs={ - "Out": out, - "XShape": x_shape - }) + helper.append_op( + type="squeeze2", + inputs={"X": input}, + attrs=attrs, + outputs={"Out": out, "XShape": x_shape}, + ) return out @@ -6749,18 +7211,23 @@ def unsqueeze(input, axes, name=None): return _C_ops.unsqueeze(input, axes) check_type(axes, 'axis/axes', (int, list, tuple, Variable), 'unsqueeze') - check_variable_and_dtype(input, 'input', [ - 'float16', - 'float32', - 'float64', - 'bool', - 'int8', - 'int16', - 'int32', - 'int64', - 'complex64', - 'complex128', - ], 'unsqueeze') + check_variable_and_dtype( + input, + 'input', + [ + 'float16', + 'float32', + 'float64', + 'bool', + 'int8', + 'int16', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'unsqueeze', + ) helper = LayerHelper("unsqueeze2", **locals()) inputs = {"X": input} attrs = {} @@ -6778,13 +7245,12 @@ def unsqueeze(input, axes, name=None): out = helper.create_variable_for_type_inference(dtype=input.dtype) x_shape = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type="unsqueeze2", - inputs=inputs, - attrs=attrs, - outputs={ - "Out": out, - "XShape": x_shape - }) + helper.append_op( + type="unsqueeze2", + inputs=inputs, + attrs=attrs, + outputs={"Out": out, "XShape": x_shape}, + ) return out @@ -6870,24 +7336,24 @@ def lod_reset(x, y=None, target_lod=None): y = fluid.layers.data(name='y', shape=[10, 20], lod_level=2) out = fluid.layers.lod_reset(x=x, y=y) """ - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'lod_reset') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'lod_reset' + ) helper = LayerHelper("lod_reset", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) if y is not None: check_type(y, 'y', (Variable), 'lod_reset') - #TODO: check y.lod_level = 0 dtype - helper.append_op(type="lod_reset", - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}) + # TODO: check y.lod_level = 0 dtype + helper.append_op( + type="lod_reset", inputs={'X': x, 'Y': y}, outputs={'Out': out} + ) elif target_lod is not None: - helper.append_op(type="lod_reset", - inputs={'X': x}, - attrs={'target_lod': target_lod}, - outputs={'Out': out}) + helper.append_op( + type="lod_reset", + inputs={'X': x}, + attrs={'target_lod': target_lod}, + outputs={'Out': out}, + ) else: raise ValueError("y and target_lod should not be both none.") return out @@ -6941,8 +7407,9 @@ def lod_append(x, level): if (not isinstance(level, Iterable)) and (not isinstance(level, Variable)): raise ValueError("Input(level) must be list, tuple or Variable.") - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'lod_append') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'lod_append' + ) helper = LayerHelper("lod_append", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) @@ -6952,27 +7419,22 @@ def lod_append(x, level): if isinstance(level, Variable): inputs['Y'] = level - #TODO: check y.lod_level = 0 dtype + # TODO: check y.lod_level = 0 dtype else: attrs['target_lod'] = level - helper.append_op(type="lod_reset", - inputs=inputs, - attrs=attrs, - outputs={'Out': out}) + helper.append_op( + type="lod_reset", inputs=inputs, attrs=attrs, outputs={'Out': out} + ) return out -def lrn(input, - n=5, - k=1.0, - alpha=1e-4, - beta=0.75, - name=None, - data_format='NCHW'): +def lrn( + input, n=5, k=1.0, alpha=1e-4, beta=0.75, name=None, data_format='NCHW' +): r""" :alias_main: paddle.nn.functional.lrn - :alias: paddle.nn.functional.lrn,paddle.nn.functional.norm.lrn - :old_api: paddle.fluid.layers.lrn + :alias: paddle.nn.functional.lrn,paddle.nn.functional.norm.lrn + :old_api: paddle.fluid.layers.lrn This operator implements the Local Response Normalization Layer. This layer performs a type of "lateral inhibition" by normalizing over local input regions. @@ -7030,38 +7492,44 @@ def lrn(input, if dims != 4: raise ValueError( - "Input's dimension size of Op(lrn) must be 4, but received %d." % - (dims)) + "Input's dimension size of Op(lrn) must be 4, but received %d." + % (dims) + ) if data_format not in ['NCHW', 'NHWC']: raise ValueError( - "Attr(data_format) of Op(lrn) got wrong value: received " + - data_format + " but only NCHW or NHWC supported.") + "Attr(data_format) of Op(lrn) got wrong value: received " + + data_format + + " but only NCHW or NHWC supported." + ) - mid_out = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) + mid_out = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) lrn_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="lrn", - inputs={"X": input}, - outputs={ - "Out": lrn_out, - "MidOut": mid_out, - }, - attrs={ - "n": n, - "k": k, - "alpha": alpha, - "beta": beta, - "data_format": data_format - }) + helper.append_op( + type="lrn", + inputs={"X": input}, + outputs={ + "Out": lrn_out, + "MidOut": mid_out, + }, + attrs={ + "n": n, + "k": k, + "alpha": alpha, + "beta": beta, + "data_format": data_format, + }, + ) return lrn_out -def pad(x, paddings, pad_value=0., name=None): +def pad(x, paddings, pad_value=0.0, name=None): r""" :alias_main: paddle.nn.functional.pad - :alias: paddle.nn.functional.pad,paddle.nn.functional.common.pad - :old_api: paddle.fluid.layers.pad + :alias: paddle.nn.functional.pad,paddle.nn.functional.common.pad + :old_api: paddle.fluid.layers.pad This op will pad a tensor with a constant value given by :attr:`pad_value`, and the padded shape is specified by :attr:`paddings`. @@ -7112,10 +7580,20 @@ def pad(x, paddings, pad_value=0., name=None): x = fluid.data(name='data', shape=[300, 300], dtype='float32') out = fluid.layers.pad(x=x, paddings=[0, 1, 1, 2], pad_value=0.) """ - check_variable_and_dtype(x, 'x', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], "pad") + check_variable_and_dtype( + x, + 'x', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + "pad", + ) check_type(pad_value, 'pad_value', (float, int, Variable), 'pad') if isinstance(pad_value, int): @@ -7124,17 +7602,16 @@ def pad(x, paddings, pad_value=0., name=None): helper = LayerHelper('pad', **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='pad', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'paddings': paddings, - 'pad_value': pad_value - }) + helper.append_op( + type='pad', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'paddings': paddings, 'pad_value': pad_value}, + ) return out -def pad_constant_like(x, y, pad_value=0., name=None): +def pad_constant_like(x, y, pad_value=0.0, name=None): r""" Pad :attr:`y` with :attr:`pad_value`, the number of values padded to the edges of each axis is specified by the difference of the shape @@ -7214,31 +7691,29 @@ def pad_constant_like(x, y, pad_value=0., name=None): # out is a rank 4 tensor variable, and out.shape = [2, 3 ,2 , 3] """ check_type(x, 'x', (Variable), 'pad_constant_like') - check_variable_and_dtype(y, 'y', ['float32', 'float64', 'int32', 'int64'], - "pad_constant_like") + check_variable_and_dtype( + y, 'y', ['float32', 'float64', 'int32', 'int64'], "pad_constant_like" + ) helper = LayerHelper('pad_constant_like', **locals()) dtype = helper.input_dtype(input_param_name='y') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='pad_constant_like', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs={'pad_value': float(pad_value)}) + helper.append_op( + type='pad_constant_like', + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs={'pad_value': float(pad_value)}, + ) return out -def label_smooth(label, - prior_dist=None, - epsilon=0.1, - dtype="float32", - name=None): +def label_smooth( + label, prior_dist=None, epsilon=0.1, dtype="float32", name=None +): r""" :alias_main: paddle.nn.functional.label_smooth - :alias: paddle.nn.functional.label_smooth,paddle.nn.functional.common.label_smooth - :old_api: paddle.fluid.layers.label_smooth + :alias: paddle.nn.functional.label_smooth,paddle.nn.functional.common.label_smooth + :old_api: paddle.fluid.layers.label_smooth Label smoothing is a mechanism to regularize the classifier layer and is called label-smoothing regularization (LSR). @@ -7295,37 +7770,42 @@ def label_smooth(label, if in_dygraph_mode(): return _C_ops.label_smooth(label, prior_dist, float(epsilon)) - if epsilon > 1. or epsilon < 0.: + if epsilon > 1.0 or epsilon < 0.0: raise ValueError("The value of epsilon must be between 0 and 1.") if _non_static_mode(): - return _legacy_C_ops.label_smooth(label, prior_dist, 'epsilon', - float(epsilon)) + return _legacy_C_ops.label_smooth( + label, prior_dist, 'epsilon', float(epsilon) + ) - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'label_smooth') + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'label_smooth' + ) helper = LayerHelper("label_smooth", **locals()) label.stop_gradient = True smooth_label = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="label_smooth", - inputs={ - "X": label, - "PriorDist": prior_dist - } if prior_dist else {"X": label}, - outputs={"Out": smooth_label}, - attrs={"epsilon": float(epsilon)}) + helper.append_op( + type="label_smooth", + inputs={"X": label, "PriorDist": prior_dist} + if prior_dist + else {"X": label}, + outputs={"Out": smooth_label}, + attrs={"epsilon": float(epsilon)}, + ) return smooth_label @templatedoc() -def roi_pool(input, - rois, - pooled_height=1, - pooled_width=1, - spatial_scale=1.0, - rois_num=None, - name=None): +def roi_pool( + input, + rois, + pooled_height=1, + pooled_width=1, + spatial_scale=1.0, + rois_num=None, + name=None, +): """ This operator implements the roi_pooling layer. @@ -7391,10 +7871,20 @@ def roi_pool(input, print(np.array(out).shape) # (2, 1, 1, 1) """ if _non_static_mode(): - assert rois_num is not None, "rois_num should not be None in dygraph mode." + assert ( + rois_num is not None + ), "rois_num should not be None in dygraph mode." pool_out, argmaxes = _legacy_C_ops.roi_pool( - input, rois, rois_num, "pooled_height", pooled_height, - "pooled_width", pooled_width, "spatial_scale", spatial_scale) + input, + rois, + rois_num, + "pooled_height", + pooled_height, + "pooled_width", + pooled_width, + "spatial_scale", + spatial_scale, + ) return pool_out, argmaxes check_variable_and_dtype(input, 'input', ['float32'], 'roi_pool') @@ -7410,29 +7900,30 @@ def roi_pool(input, } if rois_num is not None: inputs['RoisNum'] = rois_num - helper.append_op(type="roi_pool", - inputs=inputs, - outputs={ - "Out": pool_out, - "Argmax": argmaxes - }, - attrs={ - "pooled_height": pooled_height, - "pooled_width": pooled_width, - "spatial_scale": spatial_scale - }) + helper.append_op( + type="roi_pool", + inputs=inputs, + outputs={"Out": pool_out, "Argmax": argmaxes}, + attrs={ + "pooled_height": pooled_height, + "pooled_width": pooled_width, + "spatial_scale": spatial_scale, + }, + ) return pool_out @templatedoc() -def roi_align(input, - rois, - pooled_height=1, - pooled_width=1, - spatial_scale=1.0, - sampling_ratio=-1, - rois_num=None, - name=None): +def roi_align( + input, + rois, + pooled_height=1, + pooled_width=1, + spatial_scale=1.0, + sampling_ratio=-1, + rois_num=None, + name=None, +): """ ${comment} @@ -7480,21 +7971,41 @@ def roi_align(input, rois_num=rois_num) """ if in_dygraph_mode(): - assert rois_num is not None, "rois_num should not be None in dygraph mode." - return _C_ops.roi_align(input, rois, rois_num, pooled_height, - pooled_width, spatial_scale, sampling_ratio, - False) + assert ( + rois_num is not None + ), "rois_num should not be None in dygraph mode." + return _C_ops.roi_align( + input, + rois, + rois_num, + pooled_height, + pooled_width, + spatial_scale, + sampling_ratio, + False, + ) if _in_legacy_dygraph(): - assert rois_num is not None, "rois_num should not be None in dygraph mode." - align_out = _legacy_C_ops.roi_align(input, rois, rois_num, - "pooled_height", pooled_height, - "pooled_width", pooled_width, - "spatial_scale", spatial_scale, - "sampling_ratio", sampling_ratio) + assert ( + rois_num is not None + ), "rois_num should not be None in dygraph mode." + align_out = _legacy_C_ops.roi_align( + input, + rois, + rois_num, + "pooled_height", + pooled_height, + "pooled_width", + pooled_width, + "spatial_scale", + spatial_scale, + "sampling_ratio", + sampling_ratio, + ) return align_out - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'roi_align') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'roi_align' + ) check_variable_and_dtype(rois, 'rois', ['float32', 'float64'], 'roi_align') helper = LayerHelper('roi_align', **locals()) dtype = helper.input_dtype() @@ -7505,15 +8016,17 @@ def roi_align(input, } if rois_num is not None: inputs['RoisNum'] = rois_num - helper.append_op(type="roi_align", - inputs=inputs, - outputs={"Out": align_out}, - attrs={ - "pooled_height": pooled_height, - "pooled_width": pooled_width, - "spatial_scale": spatial_scale, - "sampling_ratio": sampling_ratio - }) + helper.append_op( + type="roi_align", + inputs=inputs, + outputs={"Out": align_out}, + attrs={ + "pooled_height": pooled_height, + "pooled_width": pooled_width, + "spatial_scale": spatial_scale, + "sampling_ratio": sampling_ratio, + }, + ) return align_out @@ -7559,21 +8072,22 @@ def dice_loss(input, label, epsilon=0.00001, name=None): predictions = F.softmax(x) loss = F.dice_loss(input=predictions, label=label) """ - return paddle.nn.functional.dice_loss(input, - label, - epsilon=epsilon, - name=name) - - -def image_resize(input, - out_shape=None, - scale=None, - name=None, - resample='BILINEAR', - actual_shape=None, - align_corners=True, - align_mode=1, - data_format='NCHW'): + return paddle.nn.functional.dice_loss( + input, label, epsilon=epsilon, name=name + ) + + +def image_resize( + input, + out_shape=None, + scale=None, + name=None, + resample='BILINEAR', + actual_shape=None, + align_corners=True, + align_mode=1, + data_format='NCHW', +): """ This op resizes a batch of images. @@ -7813,65 +8327,65 @@ def image_resize(input, Examples: .. code-block:: python - #declarative mode - import paddle - import paddle.fluid as fluid - import numpy as np - paddle.enable_static() - input = fluid.data(name="input", shape=[None,3,6,10]) + #declarative mode + import paddle + import paddle.fluid as fluid + import numpy as np + paddle.enable_static() + input = fluid.data(name="input", shape=[None,3,6,10]) - #1 - output = fluid.layers.image_resize(input=input,out_shape=[12,12]) + #1 + output = fluid.layers.image_resize(input=input,out_shape=[12,12]) - #2 - #x = np.array([2]).astype("int32") - #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") - #fluid.layers.assign(input=x, output=dim1) - #output = fluid.layers.image_resize(input=input,out_shape=[12,dim1]) + #2 + #x = np.array([2]).astype("int32") + #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") + #fluid.layers.assign(input=x, output=dim1) + #output = fluid.layers.image_resize(input=input,out_shape=[12,dim1]) - #3 - #x = np.array([3,12]).astype("int32") - #shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") - #fluid.layers.assign(input=x, output=shape_tensor) - #output = fluid.layers.image_resize(input=input,out_shape=shape_tensor) + #3 + #x = np.array([3,12]).astype("int32") + #shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") + #fluid.layers.assign(input=x, output=shape_tensor) + #output = fluid.layers.image_resize(input=input,out_shape=shape_tensor) - #4 - #x = np.array([0.5]).astype("float32") - #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") - #fluid.layers.assign(x,scale_tensor) - #output = fluid.layers.image_resize(input=input,scale=scale_tensor) + #4 + #x = np.array([0.5]).astype("float32") + #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") + #fluid.layers.assign(x,scale_tensor) + #output = fluid.layers.image_resize(input=input,scale=scale_tensor) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) - input_data = np.random.rand(2,3,6,10).astype("float32") + input_data = np.random.rand(2,3,6,10).astype("float32") - output_data = exe.run(fluid.default_main_program(), + output_data = exe.run(fluid.default_main_program(), feed={"input":input_data}, fetch_list=[output], return_numpy=True) - print(output_data[0].shape) + print(output_data[0].shape) - #1 - # (2, 3, 12, 12) - #2 - # (2, 3, 12, 2) - #3 - # (2, 3, 3, 12) - #4 - # (2, 3, 3, 5) + #1 + # (2, 3, 12, 12) + #2 + # (2, 3, 12, 2) + #3 + # (2, 3, 3, 12) + #4 + # (2, 3, 3, 5) - #imperative mode - import paddle.fluid.dygraph as dg + #imperative mode + import paddle.fluid.dygraph as dg - with dg.guard(place) as g: - input = dg.to_variable(input_data) - output = fluid.layers.image_resize(input=input, out_shape=[12,12]) - print(output.shape) + with dg.guard(place) as g: + input = dg.to_variable(input_data) + output = fluid.layers.image_resize(input=input, out_shape=[12,12]) + print(output.shape) - # [2L, 3L, 12L, 12L] + # [2L, 3L, 12L, 12L] """ resample_methods = { @@ -7885,7 +8399,8 @@ def image_resize(input, if resample not in resample_methods: raise ValueError( "The 'resample' of image_resize can only be 'LINEAR', 'BILINEAR', 'TRILINEAR' " - "or 'NEAREST' currently.") + "or 'NEAREST' currently." + ) resample_type = resample_methods[resample] if resample == 'LINEAR' and len(input.shape) != 3: @@ -7907,19 +8422,25 @@ def image_resize(input, if len(input.shape) == 3 and data_format not in ['NCW', 'NWC']: raise ValueError( - "Got wrong value for param `data_format`: " + data_format + - " received but only `NCW` or `NWC` supported for 3-D input.") + "Got wrong value for param `data_format`: " + + data_format + + " received but only `NCW` or `NWC` supported for 3-D input." + ) elif len(input.shape) == 4 and data_format not in ['NCHW', 'NHWC']: raise ValueError( - "Got wrong value for param `data_format`: " + data_format + - " received but only `NCHW` or `NHWC` supported for 4-D input.") + "Got wrong value for param `data_format`: " + + data_format + + " received but only `NCHW` or `NHWC` supported for 4-D input." + ) elif len(input.shape) == 5 and data_format not in ['NCDHW', 'NDHWC']: raise ValueError( - "Got wrong value for param `data_format`: " + data_format + - " received but only `NCDHW` or `NDHWC` supported for 5-D input.") + "Got wrong value for param `data_format`: " + + data_format + + " received but only `NCDHW` or `NDHWC` supported for 5-D input." + ) def _is_list_or_turple_(data): - return (isinstance(data, list) or isinstance(data, tuple)) + return isinstance(data, list) or isinstance(data, tuple) if data_format == 'NCHW' or data_format == 'NCDHW' or data_format == 'NCW': data_layout = 'NCHW' @@ -7934,7 +8455,7 @@ def _is_list_or_turple_(data): "interp_method": resample_type, "align_corners": align_corners, "align_mode": align_mode, - "data_layout": data_layout + "data_layout": data_layout, } if out_shape is not None: @@ -7952,16 +8473,17 @@ def _is_list_or_turple_(data): out_shape[i] = dim.numpy()[0] if not (_is_list_or_turple_(out_shape)): raise TypeError( - "out_shape should be a list or tuple or Variable.") + "out_shape should be a list or tuple or Variable." + ) # Validate the shape contain_var = False for dim_idx, dim_size in enumerate(out_shape): if isinstance(dim_size, Variable): contain_var = True continue - assert dim_size > 0, ( - "Each dimension size given in out_shape must be greater than 0." - ) + assert ( + dim_size > 0 + ), "Each dimension size given in out_shape must be greater than 0." if contain_var: new_size_tensor = [] @@ -7972,22 +8494,22 @@ def _is_list_or_turple_(data): new_size_tensor.append(dim) size_list.append(-1) else: - assert (isinstance(dim, int)) + assert isinstance(dim, int) temp_out = helper.create_variable_for_type_inference( - 'int32') - fill_constant([1], - 'int32', - dim, - force_cpu=True, - out=temp_out) + 'int32' + ) + fill_constant( + [1], 'int32', dim, force_cpu=True, out=temp_out + ) new_size_tensor.append(temp_out) size_list.append(dim) inputs['SizeTensor'] = new_size_tensor if len(input.shape) == 3: if len(out_shape) != 1: - raise ValueError("out_shape length should be 1 for " - "input 3-D tensor.") + raise ValueError( + "out_shape length should be 1 for " "input 3-D tensor." + ) if contain_var: attrs['out_w'] = size_list[0] else: @@ -7995,8 +8517,9 @@ def _is_list_or_turple_(data): attrs['out_w'] = out_shape[0] elif len(input.shape) == 4: if len(out_shape) != 2: - raise ValueError("out_shape length should be 2 for " - "input 4-D tensor.") + raise ValueError( + "out_shape length should be 2 for " "input 4-D tensor." + ) if contain_var: attrs['out_h'] = size_list[0] attrs['out_w'] = size_list[1] @@ -8006,8 +8529,9 @@ def _is_list_or_turple_(data): attrs['out_w'] = out_shape[1] if len(input.shape) == 5: if len(out_shape) != 3: - raise ValueError("out_shape length should be 3 for " - "input 5-D tensor.") + raise ValueError( + "out_shape length should be 3 for " "input 5-D tensor." + ) if contain_var: attrs['out_d'] = size_list[0] attrs['out_h'] = size_list[1] @@ -8030,7 +8554,8 @@ def _is_list_or_turple_(data): attrs['scale'] = float(scale) else: raise TypeError( - "Attr(scale)'s type should be float, int or Variable.") + "Attr(scale)'s type should be float, int or Variable." + ) if isinstance(actual_shape, Variable): warnings.warn( @@ -8062,22 +8587,26 @@ def _is_list_or_turple_(data): return out out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='{}_interp'.format(resample_type), - inputs=inputs, - outputs={"Out": out}, - attrs=attrs) + helper.append_op( + type='{}_interp'.format(resample_type), + inputs=inputs, + outputs={"Out": out}, + attrs=attrs, + ) return out @templatedoc(op_type="linear_interp") -def resize_linear(input, - out_shape=None, - scale=None, - name=None, - actual_shape=None, - align_corners=True, - align_mode=1, - data_format='NCW'): +def resize_linear( + input, + out_shape=None, + scale=None, + name=None, + actual_shape=None, + align_corners=True, + align_mode=1, + data_format='NCW', +): """ This op resizes the input by performing linear interpolation based on given output shape which specified by actual_shape, out_shape and scale @@ -8153,58 +8682,69 @@ def resize_linear(input, For more information, please refer to :ref:`api_guide_Name` Returns: - Variable: 3-D tensor(NCW or NWC). + Variable: 3-D tensor(NCW or NWC). Examples: .. code-block:: python - #declarative mode - import paddle.fluid as fluid - import numpy as np - input = fluid.data(name="input", shape=[None,3,100]) + #declarative mode + import paddle.fluid as fluid + import numpy as np + input = fluid.data(name="input", shape=[None,3,100]) - output = fluid.layers.resize_linear(input=input,out_shape=[50,]) + output = fluid.layers.resize_linear(input=input,out_shape=[50,]) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) - input_data = np.random.rand(1,3,100).astype("float32") + input_data = np.random.rand(1,3,100).astype("float32") - output_data = exe.run(fluid.default_main_program(), + output_data = exe.run(fluid.default_main_program(), feed={"input":input_data}, fetch_list=[output], return_numpy=True) - print(output_data[0].shape) + print(output_data[0].shape) - # (1, 3, 50) + # (1, 3, 50) - #imperative mode - import paddle.fluid.dygraph as dg + #imperative mode + import paddle.fluid.dygraph as dg - with dg.guard(place) as g: - input = dg.to_variable(input_data) - output = fluid.layers.resize_linear(input=input, out_shape=[50,]) - print(output.shape) + with dg.guard(place) as g: + input = dg.to_variable(input_data) + output = fluid.layers.resize_linear(input=input, out_shape=[50,]) + print(output.shape) - # [1L, 3L, 50L] + # [1L, 3L, 50L] """ - return image_resize(input, out_shape, scale, name, 'LINEAR', actual_shape, - align_corners, align_mode, data_format) + return image_resize( + input, + out_shape, + scale, + name, + 'LINEAR', + actual_shape, + align_corners, + align_mode, + data_format, + ) @templatedoc(op_type="bilinear_interp") -def resize_bilinear(input, - out_shape=None, - scale=None, - name=None, - actual_shape=None, - align_corners=True, - align_mode=1, - data_format='NCHW'): +def resize_bilinear( + input, + out_shape=None, + scale=None, + name=None, + actual_shape=None, + align_corners=True, + align_mode=1, + data_format='NCHW', +): """ This op resizes the input by performing bilinear interpolation based on given @@ -8291,86 +8831,97 @@ def resize_bilinear(input, name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` Returns: - Variable: 4-D tensor(NCHW or NHWC). + Variable: 4-D tensor(NCHW or NHWC). Examples: .. code-block:: python - #declarative mode - import paddle.fluid as fluid - import numpy as np - import paddle - paddle.enable_static() - input = fluid.data(name="input", shape=[None,3,6,10]) + #declarative mode + import paddle.fluid as fluid + import numpy as np + import paddle + paddle.enable_static() + input = fluid.data(name="input", shape=[None,3,6,10]) - #1 - output = fluid.layers.resize_bilinear(input=input,out_shape=[12,12]) + #1 + output = fluid.layers.resize_bilinear(input=input,out_shape=[12,12]) - #2 - #x = np.array([2]).astype("int32") - #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") - #fluid.layers.assign(input=x, output=dim1) - #output = fluid.layers.resize_bilinear(input=input,out_shape=[12,dim1]) + #2 + #x = np.array([2]).astype("int32") + #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") + #fluid.layers.assign(input=x, output=dim1) + #output = fluid.layers.resize_bilinear(input=input,out_shape=[12,dim1]) - #3 - #x = np.array([3,12]).astype("int32") - #shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") - #fluid.layers.assign(input=x, output=shape_tensor) - #output = fluid.layers.resize_bilinear(input=input,out_shape=shape_tensor) + #3 + #x = np.array([3,12]).astype("int32") + #shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") + #fluid.layers.assign(input=x, output=shape_tensor) + #output = fluid.layers.resize_bilinear(input=input,out_shape=shape_tensor) - #4 - #x = np.array([0.5]).astype("float32") - #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") - #fluid.layers.assign(x,scale_tensor) - #output = fluid.layers.resize_bilinear(input=input,scale=scale_tensor) + #4 + #x = np.array([0.5]).astype("float32") + #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") + #fluid.layers.assign(x,scale_tensor) + #output = fluid.layers.resize_bilinear(input=input,scale=scale_tensor) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) - input_data = np.random.rand(2,3,6,10).astype("float32") + input_data = np.random.rand(2,3,6,10).astype("float32") - output_data = exe.run(fluid.default_main_program(), + output_data = exe.run(fluid.default_main_program(), feed={"input":input_data}, fetch_list=[output], return_numpy=True) - print(output_data[0].shape) + print(output_data[0].shape) - #1 - # (2, 3, 12, 12) - #2 - # (2, 3, 12, 2) - #3 - # (2, 3, 3, 12) - #4 - # (2, 3, 3, 5) + #1 + # (2, 3, 12, 12) + #2 + # (2, 3, 12, 2) + #3 + # (2, 3, 3, 12) + #4 + # (2, 3, 3, 5) - #imperative mode - import paddle.fluid.dygraph as dg + #imperative mode + import paddle.fluid.dygraph as dg - with dg.guard(place) as g: - input = dg.to_variable(input_data) - output = fluid.layers.resize_bilinear(input=input, out_shape=[12,12]) - print(output.shape) + with dg.guard(place) as g: + input = dg.to_variable(input_data) + output = fluid.layers.resize_bilinear(input=input, out_shape=[12,12]) + print(output.shape) - # [2L, 3L, 12L, 12L] + # [2L, 3L, 12L, 12L] """ - return image_resize(input, out_shape, scale, name, 'BILINEAR', actual_shape, - align_corners, align_mode, data_format) + return image_resize( + input, + out_shape, + scale, + name, + 'BILINEAR', + actual_shape, + align_corners, + align_mode, + data_format, + ) @templatedoc(op_type="trilinear_interp") -def resize_trilinear(input, - out_shape=None, - scale=None, - name=None, - actual_shape=None, - align_corners=True, - align_mode=1, - data_format='NCDHW'): +def resize_trilinear( + input, + out_shape=None, + scale=None, + name=None, + actual_shape=None, + align_corners=True, + align_mode=1, + data_format='NCDHW', +): """ This op resizes the input by performing trilinear interpolation based on given @@ -8462,82 +9013,93 @@ def resize_trilinear(input, Examples: .. code-block:: python - #declarative mode - import paddle.fluid as fluid - import paddle - import numpy as np - paddle.enable_static() - input = fluid.data(name="input", shape=[None,3,6,8,10]) + #declarative mode + import paddle.fluid as fluid + import paddle + import numpy as np + paddle.enable_static() + input = fluid.data(name="input", shape=[None,3,6,8,10]) - #1 - output = fluid.layers.resize_trilinear(input=input,out_shape=[12,12,12]) + #1 + output = fluid.layers.resize_trilinear(input=input,out_shape=[12,12,12]) - #2 - #x = np.array([2]).astype("int32") - #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") - #fluid.layers.assign(input=x, output=dim1) - #output = fluid.layers.resize_trilinear(input=input,out_shape=[12,dim1,4]) + #2 + #x = np.array([2]).astype("int32") + #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") + #fluid.layers.assign(input=x, output=dim1) + #output = fluid.layers.resize_trilinear(input=input,out_shape=[12,dim1,4]) - #3 - #x = np.array([3,12,12]).astype("int32") - #shape_tensor = fluid.data(name="shape_tensor", shape=[3], dtype="int32") - #fluid.layers.assign(input=x, output=shape_tensor) - #output = fluid.layers.resize_trilinear(input=input,out_shape=shape_tensor) + #3 + #x = np.array([3,12,12]).astype("int32") + #shape_tensor = fluid.data(name="shape_tensor", shape=[3], dtype="int32") + #fluid.layers.assign(input=x, output=shape_tensor) + #output = fluid.layers.resize_trilinear(input=input,out_shape=shape_tensor) - #4 - #x = np.array([0.5]).astype("float32") - #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") - #fluid.layers.assign(x,scale_tensor) - #output = fluid.layers.resize_trilinear(input=input,scale=scale_tensor) + #4 + #x = np.array([0.5]).astype("float32") + #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") + #fluid.layers.assign(x,scale_tensor) + #output = fluid.layers.resize_trilinear(input=input,scale=scale_tensor) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) - input_data = np.random.rand(2,3,6,8,10).astype("float32") + input_data = np.random.rand(2,3,6,8,10).astype("float32") - output_data = exe.run(fluid.default_main_program(), + output_data = exe.run(fluid.default_main_program(), feed={"input":input_data}, fetch_list=[output], return_numpy=True) - print(output_data[0].shape) + print(output_data[0].shape) - #1 - # (2, 3, 12, 12, 12) - #2 - # (2, 3, 12, 2, 4) - #3 - # (2, 3, 3, 12, 12) - #4 - # (2, 3, 3, 4, 5) + #1 + # (2, 3, 12, 12, 12) + #2 + # (2, 3, 12, 2, 4) + #3 + # (2, 3, 3, 12, 12) + #4 + # (2, 3, 3, 4, 5) - #imperative mode - import paddle.fluid.dygraph as dg + #imperative mode + import paddle.fluid.dygraph as dg - with dg.guard(place) as g: - input = dg.to_variable(input_data) - output = fluid.layers.resize_trilinear(input=input, out_shape=[12,12,12]) - print(output.shape) + with dg.guard(place) as g: + input = dg.to_variable(input_data) + output = fluid.layers.resize_trilinear(input=input, out_shape=[12,12,12]) + print(output.shape) - # [2L, 3L, 12L, 12L, 12L] + # [2L, 3L, 12L, 12L, 12L] """ - return image_resize(input, out_shape, scale, name, 'TRILINEAR', - actual_shape, align_corners, align_mode, data_format) + return image_resize( + input, + out_shape, + scale, + name, + 'TRILINEAR', + actual_shape, + align_corners, + align_mode, + data_format, + ) @templatedoc(op_type="nearest_interp") -def resize_nearest(input, - out_shape=None, - scale=None, - name=None, - actual_shape=None, - align_corners=True, - data_format='NCHW'): +def resize_nearest( + input, + out_shape=None, + scale=None, + name=None, + actual_shape=None, + align_corners=True, + data_format='NCHW', +): """ This op resizes the input by performing nearest neighbor interpolation in both the @@ -8593,7 +9155,7 @@ def resize_nearest(input, And :attr:`out_shape` has a higher priority than :attr:`scale`. Default: None. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` - actual_shape(Variable): An optional input to specify output shape + actual_shape(Variable): An optional input to specify output shape dynamically. If provided, image resize according to this given shape rather than :attr:`out_shape` and :attr:`scale` specifying @@ -8613,85 +9175,87 @@ def resize_nearest(input, `[batch_size, input_channels, input_height, input_width]`. Returns: - Variable: 4-D tensor(NCHW or NHWC). + Variable: 4-D tensor(NCHW or NHWC). Examples: .. code-block:: python - #declarative mode - import paddle.fluid as fluid - import numpy as np - import paddle - paddle.enable_static() + #declarative mode + import paddle.fluid as fluid + import numpy as np + import paddle + paddle.enable_static() - input = fluid.data(name="input", shape=[None,3,6,10]) + input = fluid.data(name="input", shape=[None,3,6,10]) - #1 - output = fluid.layers.resize_nearest(input=input,out_shape=[12,12]) + #1 + output = fluid.layers.resize_nearest(input=input,out_shape=[12,12]) - #2 - #x = np.array([2]).astype("int32") - #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") - #fluid.layers.assign(input=x, output=dim1) - #output = fluid.layers.resize_nearest(input=input,out_shape=[12,dim1]) + #2 + #x = np.array([2]).astype("int32") + #dim1 = fluid.data(name="dim1", shape=[1], dtype="int32") + #fluid.layers.assign(input=x, output=dim1) + #output = fluid.layers.resize_nearest(input=input,out_shape=[12,dim1]) - #3 - #x = np.array([3,12]).astype("int32") - #shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") - #fluid.layers.assign(input=x, output=shape_tensor) - #output = fluid.layers.resize_nearest(input=input,out_shape=shape_tensor) + #3 + #x = np.array([3,12]).astype("int32") + #shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") + #fluid.layers.assign(input=x, output=shape_tensor) + #output = fluid.layers.resize_nearest(input=input,out_shape=shape_tensor) - #4 - #x = np.array([0.5]).astype("float32") - #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") - #fluid.layers.assign(x,scale_tensor) - #output = fluid.layers.resize_nearest(input=input,scale=scale_tensor) + #4 + #x = np.array([0.5]).astype("float32") + #scale_tensor = fluid.data(name="scale", shape=[1], dtype="float32") + #fluid.layers.assign(x,scale_tensor) + #output = fluid.layers.resize_nearest(input=input,scale=scale_tensor) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) - input_data = np.random.rand(2,3,6,10).astype("float32") + input_data = np.random.rand(2,3,6,10).astype("float32") - output_data = exe.run(fluid.default_main_program(), + output_data = exe.run(fluid.default_main_program(), feed={"input":input_data}, fetch_list=[output], return_numpy=True) - print(output_data[0].shape) + print(output_data[0].shape) - #1 - # (2, 3, 12, 12) - #2 - # (2, 3, 12, 2) - #3 - # (2, 3, 3, 12) - #4 - # (2, 3, 3, 5) + #1 + # (2, 3, 12, 12) + #2 + # (2, 3, 12, 2) + #3 + # (2, 3, 3, 12) + #4 + # (2, 3, 3, 5) - #imperative mode - import paddle.fluid.dygraph as dg + #imperative mode + import paddle.fluid.dygraph as dg - with dg.guard(place) as g: - input = dg.to_variable(input_data) - output = fluid.layers.resize_nearest(input=input, out_shape=[12,12]) - print(output.shape) + with dg.guard(place) as g: + input = dg.to_variable(input_data) + output = fluid.layers.resize_nearest(input=input, out_shape=[12,12]) + print(output.shape) - # [2L, 3L, 12L, 12L] + # [2L, 3L, 12L, 12L] """ - return image_resize(input, - out_shape, - scale, - name, - 'NEAREST', - actual_shape, - align_corners, - align_mode=1, - data_format=data_format) + return image_resize( + input, + out_shape, + scale, + name, + 'NEAREST', + actual_shape, + align_corners, + align_mode=1, + data_format=data_format, + ) def image_resize_short(input, out_short_len, resample='BILINEAR'): @@ -8719,15 +9283,18 @@ def image_resize_short(input, out_short_len, resample='BILINEAR'): in_shape = input.shape if len(in_shape) != 4: raise ValueError( - "The rank of input must be 4 (num_batches, channels, in_h, in_w).") + "The rank of input must be 4 (num_batches, channels, in_h, in_w)." + ) hw = in_shape[2:4] short_idx = hw.index(min(hw)) long_idx = 1 - short_idx out_shape = list(hw) out_shape[short_idx] = out_short_len out_shape[long_idx] = int( - float(out_shape[long_idx]) * - (float(out_short_len) / float(hw[short_idx])) + 0.5) + float(out_shape[long_idx]) + * (float(out_short_len) / float(hw[short_idx])) + + 0.5 + ) return image_resize(input=input, out_shape=out_shape, resample=resample) @@ -8766,8 +9333,8 @@ def gather(input, index, overwrite=True): index (Tensor): The index input tensor with rank=1. Data type is int32 or int64. overwrite (bool, optional): The mode that updating the grad when has same index. If True, use the overwrite mode to update the grad of the same index, - if False, use the accumulate mode to update the grad of the same index. - Default value is True. + if False, use the accumulate mode to update the grad of the same index. + Default value is True. Returns: output (Tensor): The output is a tensor with the same rank as input. @@ -8788,19 +9355,21 @@ def gather(input, index, overwrite=True): return _legacy_C_ops.gather(input, index, None, 'overwrite', overwrite) check_variable_and_dtype( - input, 'x', - ['float16', 'float32', 'float64', 'int32', 'int64', 'uint8'], 'gather') + input, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64', 'uint8'], + 'gather', + ) check_variable_and_dtype(index, 'index', ['int32', 'int64'], 'gather') helper = LayerHelper('gather', **locals()) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="gather", - inputs={ - "X": input, - "Index": index - }, - outputs={"Out": out}, - attrs={'overwrite': overwrite}) + helper.append_op( + type="gather", + inputs={"X": input, "Index": index}, + outputs={"Out": out}, + attrs={'overwrite': overwrite}, + ) return out @@ -8885,18 +9454,20 @@ def gather_nd(input, index, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.gather_nd(input, index) check_variable_and_dtype( - input, 'input', - ['bool', 'float32', 'float64', 'int16', 'int32', 'int64'], 'gather_np') + input, + 'input', + ['bool', 'float32', 'float64', 'int16', 'int32', 'int64'], + 'gather_np', + ) check_variable_and_dtype(index, 'index', ['int32', 'int64'], 'gather_np') helper = LayerHelper('gather_nd', **locals()) dtype = helper.input_dtype() output = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="gather_nd", - inputs={ - "X": input, - "Index": index - }, - outputs={"Out": output}) + helper.append_op( + type="gather_nd", + inputs={"X": input, "Index": index}, + outputs={"Out": output}, + ) return output @@ -8904,8 +9475,8 @@ def gather_nd(input, index, name=None): def scatter(input, index, updates, name=None, overwrite=True): """ :alias_main: paddle.scatter - :alias: paddle.scatter,paddle.tensor.scatter,paddle.tensor.manipulation.scatter - :old_api: paddle.fluid.layers.scatter + :alias: paddle.scatter,paddle.tensor.scatter,paddle.tensor.manipulation.scatter + :old_api: paddle.fluid.layers.scatter **Scatter Layer** @@ -8944,8 +9515,8 @@ def scatter(input, index, updates, name=None, overwrite=True): name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` . overwrite (bool): The mode that updating the output when there are same indices. If True, use the overwrite mode to update the output of the same index, - if False, use the accumulate mode to update the output of the same index. - Default value is True. + if False, use the accumulate mode to update the output of the same index. + Default value is True. Returns: Variable(Tensor|LoDTensor): The output is a Tensor with the same shape as input. @@ -8981,14 +9552,12 @@ def scatter(input, index, updates, name=None, overwrite=True): helper = LayerHelper('scatter', **locals()) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="scatter", - inputs={ - "X": input, - "Ids": index, - "Updates": updates - }, - attrs={'overwrite': overwrite}, - outputs={"Out": out}) + helper.append_op( + type="scatter", + inputs={"X": input, "Ids": index, "Updates": updates}, + attrs={'overwrite': overwrite}, + outputs={"Out": out}, + ) return out @@ -9073,13 +9642,11 @@ def scatter_nd_add(ref, index, updates, name=None): helper = LayerHelper('scatter_nd_add', **locals()) dtype = helper.input_dtype(input_param_name='ref') output = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="scatter_nd_add", - inputs={ - "X": ref, - "Index": index, - "Updates": updates - }, - outputs={"Out": output}) + helper.append_op( + type="scatter_nd_add", + inputs={"X": ref, "Index": index, "Updates": updates}, + outputs={"Out": output}, + ) return output @@ -9156,9 +9723,9 @@ def random_crop(x, shape, seed=None): """ helper = LayerHelper("random_crop", **locals()) - check_variable_and_dtype(x, 'x', - ['float32', 'float64', 'uint8', 'int16', 'int32'], - 'random_crop') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'uint8', 'int16', 'int32'], 'random_crop' + ) check_type(shape, 'shape', (list, Variable), 'random_crop') dtype = x.dtype out = helper.create_variable_for_type_inference(dtype) @@ -9170,19 +9737,16 @@ def random_crop(x, shape, seed=None): seed = helper.create_variable( name=unique_name.generate("random_crop_seed"), dtype="int64", - persistable=True) + persistable=True, + ) elif not isinstance(seed, Variable): raise ValueError("'seed' must be a Variable or an int.") - helper.append_op(type="random_crop", - inputs={ - "X": x, - "Seed": seed - }, - outputs={ - "Out": out, - "SeedOut": seed - }, - attrs=op_attrs) + helper.append_op( + type="random_crop", + inputs={"X": x, "Seed": seed}, + outputs={"Out": out, "SeedOut": seed}, + attrs=op_attrs, + ) return out @@ -9253,8 +9817,7 @@ def relu(x, name=None): out1 = fluid.layers.relu(x1) print(out1.numpy()) # [[0. 0. ] - # [1. 2.6]] -""" + # [1. 2.6]]""" if in_dygraph_mode(): return _C_ops.relu(x) @@ -9267,9 +9830,9 @@ def relu(x, name=None): helper = LayerHelper('relu', **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="relu", - inputs={"X": helper.input('x')}, - outputs={"Out": out}) + helper.append_op( + type="relu", inputs={"X": helper.input('x')}, outputs={"Out": out} + ) return out @@ -9339,10 +9902,9 @@ def selu(x, scale=None, alpha=None, name=None): if alpha is not None: attrs["alpha"] = alpha - helper.append_op(type="selu", - inputs={"X": x}, - outputs={"Out": out}, - attrs=attrs) + helper.append_op( + type="selu", inputs={"X": x}, outputs={"Out": out}, attrs=attrs + ) return out @@ -9393,23 +9955,23 @@ def mean_iou(input, label, num_classes): return _legacy_C_ops.mean_iou(input, label, 'num_classes', num_classes) helper = LayerHelper('mean_iou', **locals()) - check_variable_and_dtype(input, 'Predictions', ['int32', 'int64'], - 'mean_iou') + check_variable_and_dtype( + input, 'Predictions', ['int32', 'int64'], 'mean_iou' + ) check_variable_and_dtype(label, 'Labels', ['int32', 'int64'], 'mean_iou') out_mean_iou = helper.create_variable_for_type_inference(dtype='float32') out_wrong = helper.create_variable_for_type_inference(dtype='int32') out_correct = helper.create_variable_for_type_inference(dtype='int32') - helper.append_op(type="mean_iou", - inputs={ - "Predictions": input, - "Labels": label - }, - outputs={ - "OutMeanIou": out_mean_iou, - "OutWrong": out_wrong, - "OutCorrect": out_correct - }, - attrs={"num_classes": num_classes}) + helper.append_op( + type="mean_iou", + inputs={"Predictions": input, "Labels": label}, + outputs={ + "OutMeanIou": out_mean_iou, + "OutWrong": out_wrong, + "OutCorrect": out_correct, + }, + attrs={"num_classes": num_classes}, + ) return out_mean_iou, out_wrong, out_correct @@ -9506,10 +10068,12 @@ def crop(x, shape=None, offsets=None, name=None): else: attrs['offsets'] = offsets - helper.append_op(type='crop', - inputs=ipts, - outputs={'Out': out}, - attrs=None if len(attrs) == 0 else attrs) + helper.append_op( + type='crop', + inputs=ipts, + outputs={'Out': out}, + attrs=None if len(attrs) == 0 else attrs, + ) return out @@ -9602,11 +10166,13 @@ def crop_tensor(x, shape=None, offsets=None, name=None): """ helper = LayerHelper('crop_tensor', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'crop_tensor') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'crop_tensor' + ) check_type(shape, 'shape', (list, tuple, Variable), 'crop_tensor') - check_type(offsets, 'offsets', (list, tuple, Variable, type(None)), - 'crop_tensor') + check_type( + offsets, 'offsets', (list, tuple, Variable, type(None)), 'crop_tensor' + ) if offsets is None: offsets = [0] * len(x.shape) @@ -9619,25 +10185,30 @@ def _attr_shape_check(shape_val): if not isinstance(shape_val, int): raise TypeError( "Attr(shape)'s dtype of Op(crop_tensor) should be int32, but received: %s." - % type(shape_val)) + % type(shape_val) + ) if shape_val == 0: raise ValueError( "Attr(shape) of Op(crop_tensor) should not be zero, but received: %s." - % str(shape_val)) + % str(shape_val) + ) if shape_val < -1: raise ValueError( "When the element in Attr(shape) of Op(crop_tensor) is negative, only -1 is supported, but received: %s." - % str(shape_val)) + % str(shape_val) + ) def _attr_offsets_check(offset_val): if not isinstance(offset_val, int): raise TypeError( "Attr(offsets)'s dtype of Op(crop_tensor) should be int32, but received: %s." - % type(offset_val)) + % type(offset_val) + ) if offset_val < 0: raise ValueError( "Attr(offsets) of Op(crop_tensor) should be greater or equal to zero, but received: %s." - % str(offset_val)) + % str(offset_val) + ) if isinstance(offsets, Variable): offsets.stop_gradient = True @@ -9678,11 +10249,9 @@ def _attr_offsets_check(offset_val): else: _attr_shape_check(dim_size) temp_out = helper.create_variable_for_type_inference('int32') - fill_constant([1], - 'int32', - dim_size, - force_cpu=True, - out=temp_out) + fill_constant( + [1], 'int32', dim_size, force_cpu=True, out=temp_out + ) new_shape_tensor.append(temp_out) shape_attr.append(dim_size) ipts['ShapeTensor'] = new_shape_tensor @@ -9692,18 +10261,20 @@ def _attr_offsets_check(offset_val): _attr_shape_check(dim_size) attrs['shape'] = shape - helper.append_op(type='crop_tensor', - inputs=ipts, - outputs={'Out': out}, - attrs=None if len(attrs) == 0 else attrs) + helper.append_op( + type='crop_tensor', + inputs=ipts, + outputs={'Out': out}, + attrs=None if len(attrs) == 0 else attrs, + ) return out def affine_grid(theta, out_shape, name=None): """ :alias_main: paddle.nn.functional.affine_grid - :alias: paddle.nn.functional.affine_grid,paddle.nn.functional.vision.affine_grid - :old_api: paddle.fluid.layers.affine_grid + :alias: paddle.nn.functional.affine_grid,paddle.nn.functional.vision.affine_grid + :old_api: paddle.fluid.layers.affine_grid It generates a grid of (x,y) coordinates using the parameters of the affine transformation that correspond to a set of points where @@ -9746,11 +10317,15 @@ def affine_grid(theta, out_shape, name=None): """ helper = LayerHelper('affine_grid') - check_variable_and_dtype(theta, 'theta', ['float32', 'float64'], - 'affine_grid') - - if not (isinstance(out_shape, list) or isinstance(out_shape, tuple) or \ - isinstance(out_shape, Variable)): + check_variable_and_dtype( + theta, 'theta', ['float32', 'float64'], 'affine_grid' + ) + + if not ( + isinstance(out_shape, list) + or isinstance(out_shape, tuple) + or isinstance(out_shape, Variable) + ): raise ValueError("The out_shape should be a list, tuple or Variable.") if not isinstance(theta, Variable): @@ -9761,27 +10336,32 @@ def affine_grid(theta, out_shape, name=None): attrs = {} if isinstance(out_shape, Variable): ipts['OutputShape'] = out_shape - check_variable_and_dtype(out_shape, 'out_shape', ['int32'], - 'affine_grid') + check_variable_and_dtype( + out_shape, 'out_shape', ['int32'], 'affine_grid' + ) else: attrs['output_shape'] = out_shape if core.is_compiled_with_rocm(): # ROCM platform do not have MIOPEN kernel for affine_grid attrs['use_cudnn'] = False - helper.append_op(type='affine_grid', - inputs=ipts, - outputs={'Output': out}, - attrs=None if len(attrs) == 0 else attrs) + helper.append_op( + type='affine_grid', + inputs=ipts, + outputs={'Output': out}, + attrs=None if len(attrs) == 0 else attrs, + ) return out -def pad2d(input, - paddings=[0, 0, 0, 0], - mode='constant', - pad_value=0.0, - data_format="NCHW", - name=None): +def pad2d( + input, + paddings=[0, 0, 0, 0], + mode='constant', + pad_value=0.0, + data_format="NCHW", + name=None, +): """ Pad 2-d images according to 'paddings' and 'mode'. @@ -9795,10 +10375,10 @@ def pad2d(input, Otherwise, it is a 1-D Tensor with shape [4]. Data type is int32. Default is [0, 0, 0, 0]. mode (str): Three modes: 'constant' (default), 'reflect', 'edge' . - When in 'constant' mode, this op uses a constant value to pad the input tensor. - When in 'reflect' mode, uses reflection of the input boundaries to pad the input tensor. - When in 'edge' mode, uses input boundaries to pad the input tensor. - Default is 'constant' + When in 'constant' mode, this op uses a constant value to pad the input tensor. + When in 'reflect' mode, uses reflection of the input boundaries to pad the input tensor. + When in 'edge' mode, uses input boundaries to pad the input tensor. + Default is 'constant' pad_value (float32): The value to fill the padded areas in 'constant' mode . Default is 0.0 data_format (str): An string from: "NHWC", "NCHW". Specify the data format of the input data. @@ -9869,15 +10449,29 @@ def pad2d(input, # [2. 1. 2. 3. 2.]]]] """ if _non_static_mode(): - _paddings = paddings.numpy().tolist() if isinstance( - paddings, Variable) else paddings - return _legacy_C_ops.pad2d(input, 'mode', mode, 'pad_value', pad_value, - 'data_format', data_format, 'paddings', - _paddings) + _paddings = ( + paddings.numpy().tolist() + if isinstance(paddings, Variable) + else paddings + ) + return _legacy_C_ops.pad2d( + input, + 'mode', + mode, + 'pad_value', + pad_value, + 'data_format', + data_format, + 'paddings', + _paddings, + ) check_variable_and_dtype( - input, 'input', ['float16', 'float32', 'float64', 'int32', 'int64'], - "pad2d") + input, + 'input', + ['float16', 'float32', 'float64', 'int32', 'int64'], + "pad2d", + ) attrs = {'mode': mode, 'pad_value': pad_value, 'data_format': data_format} inputs = {'X': [input]} @@ -9889,16 +10483,18 @@ def pad2d(input, helper = LayerHelper('pad2d', **locals()) - assert mode in ['reflect', 'edge', 'constant' - ], "mode should be one of constant, reflect, edge." + assert mode in [ + 'reflect', + 'edge', + 'constant', + ], "mode should be one of constant, reflect, edge." dtype = helper.input_dtype(input_param_name='input') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='pad2d', - inputs=inputs, - outputs={"Out": out}, - attrs=attrs) + helper.append_op( + type='pad2d', inputs=inputs, outputs={"Out": out}, attrs=attrs + ) return out @@ -9907,8 +10503,8 @@ def pad2d(input, def elu(x, alpha=1.0, name=None): """ :alias_main: paddle.nn.functional.elu - :alias: paddle.nn.functional.elu,paddle.nn.functional.activation.elu - :old_api: paddle.fluid.layers.elu + :alias: paddle.nn.functional.elu,paddle.nn.functional.activation.elu + :old_api: paddle.fluid.layers.elu ${comment} Args: @@ -9937,10 +10533,12 @@ def elu(x, alpha=1.0, name=None): helper = LayerHelper('elu', **locals()) check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'elu') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='elu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'alpha': alpha}) + helper.append_op( + type='elu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'alpha': alpha}, + ) return out @@ -9978,13 +10576,15 @@ def relu6(x, threshold=6.0, name=None): helper = LayerHelper('relu6', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='relu6', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'threshold': threshold, - 'use_mkldnn': _global_flags()["FLAGS_use_mkldnn"] - }) + helper.append_op( + type='relu6', + inputs={'X': x}, + outputs={'Out': out}, + attrs={ + 'threshold': threshold, + 'use_mkldnn': _global_flags()["FLAGS_use_mkldnn"], + }, + ) return out @@ -10021,7 +10621,8 @@ def pow(x, factor=1.0, name=None): # y_2 is x^{3.0} """ check_variable_and_dtype( - x, 'x', ['int32', 'int64', 'float16', 'float32', 'float64'], 'pow') + x, 'x', ['int32', 'int64', 'float16', 'float32', 'float64'], 'pow' + ) helper = LayerHelper('pow', **locals()) inputs = {'X': x} @@ -10034,10 +10635,9 @@ def pow(x, factor=1.0, name=None): attrs['factor'] = factor out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='pow', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='pow', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -10077,13 +10677,12 @@ def stanh(x, scale_a=0.67, scale_b=1.7159, name=None): helper = LayerHelper('stanh', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='stanh', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'scale_a': scale_a, - 'scale_b': scale_b - }) + helper.append_op( + type='stanh', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'scale_a': scale_a, 'scale_b': scale_b}, + ) return out @@ -10116,18 +10715,18 @@ def hard_sigmoid(x, slope=0.2, offset=0.5, name=None): if _non_static_mode(): return _legacy_C_ops.hard_sigmoid(x, 'slope', slope, 'offset', offset) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'hard_sigmoid') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'hard_sigmoid' + ) helper = LayerHelper('hard_sigmoid', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='hard_sigmoid', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'slope': slope, - 'offset': offset - }) + helper.append_op( + type='hard_sigmoid', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'slope': slope, 'offset': offset}, + ) return out @@ -10135,8 +10734,8 @@ def hard_sigmoid(x, slope=0.2, offset=0.5, name=None): def swish(x, beta=1.0, name=None): r""" :alias_main: paddle.nn.functional.swish - :alias: paddle.nn.functional.swish,paddle.nn.functional.activation.swish - :old_api: paddle.fluid.layers.swish + :alias: paddle.nn.functional.swish,paddle.nn.functional.activation.swish + :old_api: paddle.fluid.layers.swish Elementwise swish activation function. See `Searching for Activation Functions `_ for more details. @@ -10208,10 +10807,12 @@ def swish(x, beta=1.0, name=None): helper = LayerHelper('swish', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='swish', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'slope': beta}) + helper.append_op( + type='swish', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'slope': beta}, + ) return out @@ -10272,52 +10873,57 @@ def prelu(x, mode, param_attr=None, data_format="NCHW", name=None): if mode == 'channel': true_data_format = [ - 'NC', 'NCL', 'NCHW', 'NCDHW', 'NLC', 'NHWC', 'NDHWC' + 'NC', + 'NCL', + 'NCHW', + 'NCDHW', + 'NLC', + 'NHWC', + 'NDHWC', ] if data_format not in true_data_format: raise ValueError( "data_format must be one of 'NC', 'NCL', 'NCHW', 'NCDHW', " - "'NLC', 'NHWC', 'NDHWC' but receive {}".format(data_format)) + "'NLC', 'NHWC', 'NDHWC' but receive {}".format(data_format) + ) data_format = 'NCHW' if data_format[1] == 'C' else 'NHWC' - assert len( - x.shape - ) >= 2, "The size of input shape should be equal or larger than 2 in prelu() when mode is 'channel'" - #NOTE(zhiqiu): The alpha_shape should be [1, channel] + [1] * len(x.shape[2:]). + assert ( + len(x.shape) >= 2 + ), "The size of input shape should be equal or larger than 2 in prelu() when mode is 'channel'" + # NOTE(zhiqiu): The alpha_shape should be [1, channel] + [1] * len(x.shape[2:]). # To be consistent with Prelu, it is simplified. - #NOTE(zhiqiu): Revert shape to [1, channel, 1, 1] for compatibility with saved model of old version. - #NOTE(GuoxiaWang): support NHWC data format + # NOTE(zhiqiu): Revert shape to [1, channel, 1, 1] for compatibility with saved model of old version. + # NOTE(GuoxiaWang): support NHWC data format if data_format == 'NHWC': alpha_shape = [1, 1, 1, x.shape[-1]] else: alpha_shape = [1, x.shape[1], 1, 1] elif mode == 'element': - assert len( - x.shape - ) >= 1, "The size of input shape should be equal or larger than 1 in prelu() when mode is 'element'" + assert ( + len(x.shape) >= 1 + ), "The size of input shape should be equal or larger than 1 in prelu() when mode is 'element'" alpha_shape = [1] + list(x.shape)[1:] dtype = helper.input_dtype(input_param_name='x') - alpha = helper.create_parameter(attr=helper.param_attr, - shape=alpha_shape, - dtype=dtype, - is_bias=False, - default_initializer=Constant(0.25)) + alpha = helper.create_parameter( + attr=helper.param_attr, + shape=alpha_shape, + dtype=dtype, + is_bias=False, + default_initializer=Constant(0.25), + ) if in_dygraph_mode(): return _C_ops.prelu(x, alpha, data_format, mode) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="prelu", - inputs={ - "X": x, - 'Alpha': alpha - }, - attrs={ - "mode": mode, - "data_format": data_format - }, - outputs={"Out": out}) + helper.append_op( + type="prelu", + inputs={"X": x, 'Alpha': alpha}, + attrs={"mode": mode, "data_format": data_format}, + outputs={"Out": out}, + ) return out @@ -10358,13 +10964,12 @@ def brelu(x, t_min=0.0, t_max=24.0, name=None): helper = LayerHelper('brelu', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='brelu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 't_min': t_min, - 't_max': t_max - }) + helper.append_op( + type='brelu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'t_min': t_min, 't_max': t_max}, + ) return out @@ -10431,15 +11036,18 @@ def soft_relu(x, threshold=40.0, name=None): res = exe.run(fluid.default_main_program(), feed={'x':img}, fetch_list=[output]) print(res) # [array([[0.6931472, 1.3132616], [2.126928 , 3.0485873]], dtype=float32)] """ - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'soft_relu') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'soft_relu' + ) helper = LayerHelper('soft_relu', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='soft_relu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'threshold': threshold}) + helper.append_op( + type='soft_relu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'threshold': threshold}, + ) return out @@ -10508,8 +11116,11 @@ def flatten(x, axis=1, name=None): # out shape is [16, 3] """ check_variable_and_dtype( - x, 'x', ['float32', 'float64', 'int8', 'int32', 'int64', 'uint8'], - 'flatten') + x, + 'x', + ['float32', 'float64', 'int8', 'int32', 'int64', 'uint8'], + 'flatten', + ) if _non_static_mode(): return _legacy_C_ops.flatten2(x, 'axis', axis)[0] @@ -10523,13 +11134,12 @@ def flatten(x, axis=1, name=None): out = helper.create_variable_for_type_inference(x.dtype) x_shape = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='flatten2', - inputs={"X": x}, - outputs={ - 'Out': out, - 'XShape': x_shape - }, - attrs={"axis": axis}) + helper.append_op( + type='flatten2', + inputs={"X": x}, + outputs={'Out': out, 'XShape': x_shape}, + attrs={"axis": axis}, + ) return out @@ -10623,42 +11233,53 @@ def stack(x, axis=0, name=None): if not isinstance(x, list) and not isinstance(x, tuple): # NOTE:(zhiqiu) Only support Variable as input if the Variable is a LOD_TENSOR_ARRAY create by create_array, array_write, array_read, etc. # In that case, Variable is array of tensors indeed. - if isinstance(x, Variable) and x.desc.type( - ) == core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + isinstance(x, Variable) + and x.desc.type() == core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): x = [x] else: raise TypeError( - "The type of '%s' in %s must be %s, but received %s" % - ('x', 'stack', 'list[Tensor], tuple[Tensor] or TensorArray', - type(x))) + "The type of '%s' in %s must be %s, but received %s" + % ( + 'x', + 'stack', + 'list[Tensor], tuple[Tensor] or TensorArray', + type(x), + ) + ) helper = LayerHelper('stack', **locals()) out = helper.create_variable_for_type_inference(x[0].dtype) if x[0].desc.type() == core.VarDesc.VarType.LOD_TENSOR_ARRAY: - assert len(x) == 1, "If the elements of 'x' in stack are Variable(LoDTensorArray), " \ - "number of the elements must be 1, but received %s." % len(x) + assert len(x) == 1, ( + "If the elements of 'x' in stack are Variable(LoDTensorArray), " + "number of the elements must be 1, but received %s." % len(x) + ) out_index = helper.create_variable_for_type_inference(dtype="int32") for i in x: - check_variable_and_dtype(i, 'x', \ - ['float16', 'float32', 'float64', 'int32', 'int64'], 'stack') - - helper.append_op(type='tensor_array_to_tensor', - inputs={'X': x[0]}, - outputs={ - 'Out': [out], - 'OutIndex': [out_index] - }, - attrs={ - 'axis': axis, - 'use_stack': True - }) + check_variable_and_dtype( + i, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'stack', + ) + + helper.append_op( + type='tensor_array_to_tensor', + inputs={'X': x[0]}, + outputs={'Out': [out], 'OutIndex': [out_index]}, + attrs={'axis': axis, 'use_stack': True}, + ) else: - helper.append_op(type='stack', - inputs={'X': x}, - outputs={'Y': out}, - attrs={'axis': axis}) + helper.append_op( + type='stack', + inputs={'X': x}, + outputs={'Y': out}, + attrs={'axis': axis}, + ) return out @@ -10722,21 +11343,12 @@ def filter_by_instag(ins, ins_tag, filter_tag, is_lod, out_val_if_empty=0): out = helper.create_variable_for_type_inference(dtype=ins.dtype) loss_weight = helper.create_variable_for_type_inference(dtype=np.float64) mmap = helper.create_variable_for_type_inference(dtype=ins_tag.dtype) - helper.append_op(type='filter_by_instag', - inputs={ - 'Ins': ins, - 'Ins_tag': ins_tag, - 'Filter_tag': filter_tag - }, - outputs={ - 'Out': out, - 'LossWeight': loss_weight, - 'IndexMap': mmap - }, - attrs={ - 'is_lod': is_lod, - 'out_val_if_empty': out_val_if_empty - }) + helper.append_op( + type='filter_by_instag', + inputs={'Ins': ins, 'Ins_tag': ins_tag, 'Filter_tag': filter_tag}, + outputs={'Out': out, 'LossWeight': loss_weight, 'IndexMap': mmap}, + attrs={'is_lod': is_lod, 'out_val_if_empty': out_val_if_empty}, + ) return [out, loss_weight] @@ -10744,8 +11356,8 @@ def filter_by_instag(ins, ins_tag, filter_tag, is_lod, out_val_if_empty=0): def unstack(x, axis=0, num=None): """ :alias_main: paddle.unstack - :alias: paddle.unstack,paddle.tensor.unstack,paddle.tensor.manipulation.unstack - :old_api: paddle.fluid.layers.unstack + :alias: paddle.unstack,paddle.tensor.unstack,paddle.tensor.manipulation.unstack + :old_api: paddle.fluid.layers.unstack **UnStack Layer** @@ -10794,13 +11406,12 @@ def unstack(x, axis=0, num=None): for _ in range(num): outs.append(helper.create_variable_for_type_inference(x.dtype)) - helper.append_op(type='unstack', - inputs={'X': [x]}, - outputs={'Y': outs}, - attrs={ - 'axis': axis, - 'num': num - }) + helper.append_op( + type='unstack', + inputs={'X': [x]}, + outputs={'Y': outs}, + attrs={'axis': axis, 'num': num}, + ) return outs @@ -10808,8 +11419,8 @@ def unstack(x, axis=0, num=None): def expand(x, expand_times, name=None): """ :alias_main: paddle.expand - :alias: paddle.expand,paddle.tensor.expand,paddle.tensor.manipulation.expand - :old_api: paddle.fluid.layers.expand + :alias: paddle.expand,paddle.tensor.expand,paddle.tensor.manipulation.expand + :old_api: paddle.fluid.layers.expand This operation tiles ``x`` multiple times according to the parameter ``expand_times``. The times number for each dimension of ``x`` is set by the parameter ``expand_times``. @@ -10883,12 +11494,16 @@ def expand(x, expand_times, name=None): inputs = {"X": [x]} attrs = {} check_variable_and_dtype( - x, 'x', ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'expand') + x, + 'x', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'expand', + ) check_type(expand_times, 'expand_times', (list, tuple, Variable), 'expand') if convert_dtype(x.dtype) == 'bool' and x.stop_gradient == True: raise ValueError( - "expand op bool date type must set the stop_gradient to be False") + "expand op bool date type must set the stop_gradient to be False" + ) helper = LayerHelper('expand', input=x, **locals()) @@ -10899,8 +11514,9 @@ def get_attr_expand_times(list_expand_times): attrs_expand_times.append(-1) else: attrs_expand_times.append(times) - assert times > 0, ( - "Each element given in expand_times must not be negative.") + assert ( + times > 0 + ), "Each element given in expand_times must not be negative." return attrs_expand_times if isinstance(expand_times, Variable): @@ -10910,14 +11526,14 @@ def get_attr_expand_times(list_expand_times): attrs['expand_times'] = get_attr_expand_times(expand_times) if utils._contain_var(expand_times): inputs['expand_times_tensor'] = utils._convert_to_tensor_list( - expand_times) + expand_times + ) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='expand', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='expand', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -10925,8 +11541,8 @@ def get_attr_expand_times(list_expand_times): def expand_as(x, target_tensor, name=None): """ :alias_main: paddle.expand_as - :alias: paddle.expand_as,paddle.tensor.expand_as,paddle.tensor.manipulation.expand_as - :old_api: paddle.fluid.layers.expand_as + :alias: paddle.expand_as,paddle.tensor.expand_as,paddle.tensor.manipulation.expand_as + :old_api: paddle.fluid.layers.expand_as expand_as operator tiles to the input by given expand tensor. You should set expand tensor for each dimension by providing tensor 'target_tensor'. The rank of X @@ -10992,12 +11608,15 @@ def expand_as(x, target_tensor, name=None): if _non_static_mode(): return _legacy_C_ops.expand_as(x, target_tensor) - check_variable_and_dtype(x, 'x', - ['float32', 'float64', 'int32', 'int64', 'bool'], - 'expand_as') - check_variable_and_dtype(target_tensor, 'target_tensor', - ['float32', 'float64', 'int32', 'int64', 'bool'], - 'expand_as') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64', 'bool'], 'expand_as' + ) + check_variable_and_dtype( + target_tensor, + 'target_tensor', + ['float32', 'float64', 'int32', 'int64', 'bool'], + 'expand_as', + ) helper = LayerHelper('expand_as', input=x, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) @@ -11011,14 +11630,16 @@ def expand_as(x, target_tensor, name=None): @deprecated(since='1.8.0', update_to="paddle.uniform") @templatedoc() -def uniform_random_batch_size_like(input, - shape, - dtype='float32', - input_dim_idx=0, - output_dim_idx=0, - min=-1.0, - max=1.0, - seed=0): +def uniform_random_batch_size_like( + input, + shape, + dtype='float32', + input_dim_idx=0, + output_dim_idx=0, + min=-1.0, + max=1.0, + seed=0, +): """ This OP initializes a variable with random values sampled from a uniform distribution in the range [min, max). The input_dim_idx used to get the input dimension value which will be used to resize the output dimension. @@ -11081,39 +11702,46 @@ def uniform_random_batch_size_like(input, """ - check_variable_and_dtype(input, 'Input', ("float32", 'float64', "uint16"), - 'uniform_random_batch_size_like') + check_variable_and_dtype( + input, + 'Input', + ("float32", 'float64', "uint16"), + 'uniform_random_batch_size_like', + ) check_type(shape, 'shape', (list, tuple), 'uniform_random_batch_size_like') - check_dtype(dtype, 'dtype', ('float32', 'float64', "uint16"), - 'uniform_random_batch_size_like') + check_dtype( + dtype, + 'dtype', + ('float32', 'float64', "uint16"), + 'uniform_random_batch_size_like', + ) helper = LayerHelper('uniform_random_batch_size_like', **locals()) out = helper.create_variable_for_type_inference(dtype) c_dtype = convert_np_dtype_to_dtype_(dtype) - helper.append_op(type='uniform_random_batch_size_like', - inputs={'Input': input}, - outputs={'Out': out}, - attrs={ - 'shape': shape, - 'input_dim_idx': input_dim_idx, - 'output_dim_idx': output_dim_idx, - 'min': min, - 'max': max, - 'seed': seed, - 'dtype': c_dtype - }) + helper.append_op( + type='uniform_random_batch_size_like', + inputs={'Input': input}, + outputs={'Out': out}, + attrs={ + 'shape': shape, + 'input_dim_idx': input_dim_idx, + 'output_dim_idx': output_dim_idx, + 'min': min, + 'max': max, + 'seed': seed, + 'dtype': c_dtype, + }, + ) return out @deprecated(since="2.0.0", update_to="paddle.normal") @templatedoc() -def gaussian_random(shape, - mean=0.0, - std=1.0, - seed=0, - dtype='float32', - name=None): +def gaussian_random( + shape, mean=0.0, std=1.0, seed=0, dtype='float32', name=None +): """ This OP returns a Tensor filled with random values sampled from a Gaussian distribution, with ``shape`` and ``dtype``. @@ -11212,15 +11840,24 @@ def gaussian_random(shape, if in_dygraph_mode(): shape = utils.convert_shape_to_list(shape) place = _current_expected_place() - return _C_ops.gaussian_random(shape, float(mean), float(std), seed, - dtype, place) + return _C_ops.gaussian_random( + shape, float(mean), float(std), seed, dtype, place + ) if _in_legacy_dygraph(): shape = utils.convert_shape_to_list(shape) - return _legacy_C_ops.gaussian_random('shape', shape, - 'mean', float(mean), 'std', - float(std), 'seed', seed, 'dtype', - dtype) + return _legacy_C_ops.gaussian_random( + 'shape', + shape, + 'mean', + float(mean), + 'std', + float(std), + 'seed', + seed, + 'dtype', + dtype, + ) check_type(shape, 'shape', (list, tuple, Variable), 'gaussian_random/randn') check_dtype(dtype, 'dtype', ['float32', 'float64'], 'gaussian_random/randn') @@ -11231,19 +11868,17 @@ def gaussian_random(shape, 'std': std, 'seed': seed, 'dtype': dtype, - 'use_mkldnn': False + 'use_mkldnn': False, } - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type='gaussian_random/randn') + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type='gaussian_random/randn' + ) helper = LayerHelper('gaussian_random', **locals()) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='gaussian_random', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='gaussian_random', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -11277,28 +11912,28 @@ def sampling_id(x, min=0.0, max=1.0, seed=0, dtype='float32'): helper = LayerHelper('sampling_id', **locals()) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='sampling_id', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'min': min, - 'max': max, - 'seed': seed - }) + helper.append_op( + type='sampling_id', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'min': min, 'max': max, 'seed': seed}, + ) return out @deprecated(since='1.8.0', update_to="paddle.normal") @templatedoc() -def gaussian_random_batch_size_like(input, - shape, - input_dim_idx=0, - output_dim_idx=0, - mean=0.0, - std=1.0, - seed=0, - dtype='float32'): +def gaussian_random_batch_size_like( + input, + shape, + input_dim_idx=0, + output_dim_idx=0, + mean=0.0, + std=1.0, + seed=0, + dtype='float32', +): """ ${comment} @@ -11329,26 +11964,40 @@ def gaussian_random_batch_size_like(input, """ helper = LayerHelper('gaussian_random_batch_size_like', **locals()) - check_type(input, 'input', (Variable), - 'fluid.layers.gaussian_random_batch_size_like') - check_type(shape, 'shape', (list, tuple), - 'fluid.layers.gaussian_random_batch_size_like') - check_dtype(dtype, 'dtype', ['float16', 'float32', 'int'], - 'fluid.layers.gaussian_random_batch_size_like') + check_type( + input, + 'input', + (Variable), + 'fluid.layers.gaussian_random_batch_size_like', + ) + check_type( + shape, + 'shape', + (list, tuple), + 'fluid.layers.gaussian_random_batch_size_like', + ) + check_dtype( + dtype, + 'dtype', + ['float16', 'float32', 'int'], + 'fluid.layers.gaussian_random_batch_size_like', + ) out = helper.create_variable_for_type_inference(dtype) c_dtype = convert_np_dtype_to_dtype_(dtype) - helper.append_op(type='gaussian_random_batch_size_like', - inputs={'Input': input}, - outputs={'Out': out}, - attrs={ - 'shape': shape, - 'input_dim_idx': input_dim_idx, - 'output_dim_idx': output_dim_idx, - 'mean': mean, - 'std': std, - 'seed': seed, - 'dtype': c_dtype - }) + helper.append_op( + type='gaussian_random_batch_size_like', + inputs={'Input': input}, + outputs={'Out': out}, + attrs={ + 'shape': shape, + 'input_dim_idx': input_dim_idx, + 'output_dim_idx': output_dim_idx, + 'mean': mean, + 'std': std, + 'seed': seed, + 'dtype': c_dtype, + }, + ) return out @@ -11507,7 +12156,8 @@ def slice(input, axes, starts, ends): axes = list(axes) if len(axes) == 0: raise ValueError( - "Input axes should not be an empty list/tuple.") + "Input axes should not be an empty list/tuple." + ) for i in range(len(axes)): if axes[i] < 0: axes[i] = max(0, axes[i] + len(input.shape)) @@ -11516,8 +12166,10 @@ def slice(input, axes, starts, ends): else: raise ValueError( - "Input axes must be a python list or tuple, but reveived {}". - format(type(axes))) + "Input axes must be a python list or tuple, but reveived {}".format( + type(axes) + ) + ) infer_flags = list(1 for i in range(len(axes))) @@ -11525,7 +12177,8 @@ def slice(input, axes, starts, ends): if isinstance(starts, (list, tuple)): starts = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item + if isinstance(item, tmp_tensor_type) + else item for item in starts ] elif isinstance(starts, tmp_tensor_type): @@ -11535,7 +12188,9 @@ def slice(input, axes, starts, ends): if isinstance(ends, (list, tuple)): ends = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item for item in ends + if isinstance(item, tmp_tensor_type) + else item + for item in ends ] attrs += ('ends', ends) elif isinstance(ends, tmp_tensor_type): @@ -11553,7 +12208,8 @@ def slice(input, axes, starts, ends): axes = list(axes) if len(axes) == 0: raise ValueError( - "Input axes should not be an empty list/tuple.") + "Input axes should not be an empty list/tuple." + ) for i in range(len(axes)): if axes[i] < 0: axes[i] = max(0, axes[i] + len(input.shape)) @@ -11562,8 +12218,10 @@ def slice(input, axes, starts, ends): else: raise ValueError( - "Input axes must be a python list or tuple, but reveived {}" - .format(type(axes))) + "Input axes must be a python list or tuple, but reveived {}".format( + type(axes) + ) + ) infer_flags = list(1 for i in range(len(axes))) @@ -11572,7 +12230,8 @@ def slice(input, axes, starts, ends): if isinstance(starts, (list, tuple)): starts = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item + if isinstance(item, tmp_tensor_type) + else item for item in starts ] attrs += ('starts', starts) @@ -11584,7 +12243,8 @@ def slice(input, axes, starts, ends): if isinstance(ends, (list, tuple)): ends = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item + if isinstance(item, tmp_tensor_type) + else item for item in ends ] attrs += ('ends', ends) @@ -11593,16 +12253,27 @@ def slice(input, axes, starts, ends): ends_tensor.stop_gradient = True infer_flags = list(-1 for i in range(len(axes))) - return _legacy_C_ops.slice(input, starts_tensor, ends_tensor, None, - None, 'axes', axes, 'infer_flags', - infer_flags, *attrs) + return _legacy_C_ops.slice( + input, + starts_tensor, + ends_tensor, + None, + None, + 'axes', + axes, + 'infer_flags', + infer_flags, + *attrs, + ) if not isinstance(starts, (list, tuple, Variable)): raise ValueError( - "Input starts must be an Variable, python list or tuple.") + "Input starts must be an Variable, python list or tuple." + ) if not isinstance(ends, (list, tuple, Variable)): raise ValueError( - "Input ends must be an Variable, python list or tuple.") + "Input ends must be an Variable, python list or tuple." + ) helper = LayerHelper('slice', **locals()) @@ -11649,11 +12320,11 @@ def slice(input, axes, starts, ends): # infer_flags attrs['infer_flags'] = infer_flags out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype('input')) - helper.append_op(type='slice', - inputs=inputs, - attrs=attrs, - outputs={'Out': out}) + dtype=helper.input_dtype('input') + ) + helper.append_op( + type='slice', inputs=inputs, attrs=attrs, outputs={'Out': out} + ) return out @@ -11662,8 +12333,8 @@ def slice(input, axes, starts, ends): def strided_slice(input, axes, starts, ends, strides): """ :alias_main: paddle.strided_slice - :alias: paddle.strided_slice,paddle.tensor.strided_slice,paddle.tensor.manipulation.strided_slice - :old_api: paddle.fluid.layers.strided_slice + :alias: paddle.strided_slice,paddle.tensor.strided_slice,paddle.tensor.manipulation.strided_slice + :old_api: paddle.fluid.layers.strided_slice This operator produces a slice of ``input`` along multiple axes. Similar to numpy: https://docs.scipy.org/doc/numpy/reference/arrays.indexing.html @@ -11764,9 +12435,12 @@ def strided_slice(input, axes, starts, ends, strides): helper = LayerHelper('strided_slice', **locals()) - check_variable_and_dtype(input, 'input', - ['bool', 'float32', 'float64', 'int32', 'int64'], - 'strided_slice') + check_variable_and_dtype( + input, + 'input', + ['bool', 'float32', 'float64', 'int32', 'int64'], + 'strided_slice', + ) check_type(axes, 'axes', (list, tuple), 'strided_slice') check_type(starts, 'starts', (list, tuple, Variable), 'strided_slice') check_type(ends, 'ends', (list, tuple, Variable), 'strided_slice') @@ -11774,8 +12448,9 @@ def strided_slice(input, axes, starts, ends, strides): def check_list_elements_dtype(list_input, input_name): if isinstance(list_input, Variable): - check_dtype(list_input.dtype, input_name, ['int32'], - 'strided_slice') + check_dtype( + list_input.dtype, input_name, ['int32'], 'strided_slice' + ) else: for i, var in enumerate(list_input): var_name = input_name + '[' + str(i) + ']' @@ -11794,7 +12469,7 @@ def get_new_list_tensor(old_list): dim.stop_gradient = True new_list_tensor.append(dim) else: - assert (isinstance(dim, int)) + assert isinstance(dim, int) temp_out = helper.create_variable_for_type_inference('int32') fill_constant([1], 'int32', dim, force_cpu=True, out=temp_out) new_list_tensor.append(temp_out) @@ -11811,7 +12486,7 @@ def get_new_list_tensor(old_list): 'starts': starts, 'ends': ends, 'strides': strides, - 'infer_flags': infer_flags + 'infer_flags': infer_flags, } else: # starts @@ -11866,11 +12541,11 @@ def get_new_list_tensor(old_list): attrs['strides'] = strides attrs['infer_flags'] = infer_flags out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype('input')) - helper.append_op(type='strided_slice', - inputs=inputs, - attrs=attrs, - outputs={'Out': out}) + dtype=helper.input_dtype('input') + ) + helper.append_op( + type='strided_slice', inputs=inputs, attrs=attrs, outputs={'Out': out} + ) return out @@ -11878,8 +12553,8 @@ def get_new_list_tensor(old_list): def shape(input): """ :alias_main: paddle.shape - :alias: paddle.shape,paddle.tensor.shape,paddle.tensor.attribute.shape - :old_api: paddle.fluid.layers.shape + :alias: paddle.shape,paddle.tensor.shape,paddle.tensor.attribute.shape + :old_api: paddle.fluid.layers.shape **Shape Layer** @@ -11937,16 +12612,29 @@ def shape(input): out.stop_gradient = True return out - check_variable_and_dtype(input, 'input', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], 'shape') + check_variable_and_dtype( + input, + 'input', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'shape', + ) helper = LayerHelper('shape', **locals()) out = helper.create_variable_for_type_inference(dtype='int32') - helper.append_op(type='shape', - inputs={'Input': input}, - outputs={'Out': out}, - stop_gradient=True) + helper.append_op( + type='shape', + inputs={'Input': input}, + outputs={'Out': out}, + stop_gradient=True, + ) return out @@ -12014,8 +12702,11 @@ def size(input): return _legacy_C_ops.size(input) check_variable_and_dtype( - input, 'input', - ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], "size") + input, + 'input', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + "size", + ) helper = LayerHelper('size', **locals()) out = helper.create_variable_for_type_inference(dtype='int64') helper.append_op(type='size', inputs={'Input': input}, outputs={'Out': out}) @@ -12031,27 +12722,29 @@ def _elementwise_op(helper): assert x is not None, 'x cannot be None in {}'.format(op_type) assert y is not None, 'y cannot be None in {}'.format(op_type) check_variable_and_dtype( - x, 'x', ['float16', 'uint16', 'float32', 'float64', 'int32', 'int64'], - op_type) + x, + 'x', + ['float16', 'uint16', 'float32', 'float64', 'int32', 'int64'], + op_type, + ) check_variable_and_dtype( - y, 'y', ['float16', 'uint16', 'float32', 'float64', 'int32', 'int64'], - op_type) + y, + 'y', + ['float16', 'uint16', 'float32', 'float64', 'int32', 'int64'], + op_type, + ) axis = helper.kwargs.get('axis', -1) use_mkldnn = helper.kwargs.get('use_mkldnn', False) name = helper.kwargs.get('name', None) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=op_type, - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs={ - 'axis': axis, - 'use_mkldnn': use_mkldnn - }) + helper.append_op( + type=op_type, + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs={'axis': axis, 'use_mkldnn': use_mkldnn}, + ) return helper.append_activation(out) @@ -12107,15 +12800,33 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): return dygraph_utils._append_activation_in_dygraph(out) if _non_static_mode(): _scale = scale.numpy().item(0) if isinstance(scale, Variable) else scale - out = _legacy_C_ops.scale(x, 'scale', float(_scale), 'bias', - float(bias), 'bias_after_scale', - bias_after_scale) + out = _legacy_C_ops.scale( + x, + 'scale', + float(_scale), + 'bias', + float(bias), + 'bias_after_scale', + bias_after_scale, + ) return dygraph_utils._append_activation_in_dygraph(out) - check_variable_and_dtype(x, "x", [ - 'float16', 'uint16', 'float32', 'float64', 'int8', 'int16', 'int32', - 'int64', 'uint8' - ], "scale") + check_variable_and_dtype( + x, + "x", + [ + 'float16', + 'uint16', + 'float32', + 'float64', + 'int8', + 'int16', + 'int32', + 'int64', + 'uint8', + ], + "scale", + ) inputs = {'X': [x]} attrs = { 'bias': float(bias), @@ -12128,91 +12839,90 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): helper = LayerHelper('scale', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='scale', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='scale', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return helper.append_activation(out) def elementwise_add(x, y, axis=-1, act=None, name=None): """ -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle - def gen_data(): - return { - "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='float32') - y = fluid.data(name="y", shape=[3], dtype='float32') - z = fluid.layers.elementwise_add(x, y) - # z = x + y + import paddle.fluid as fluid + import numpy as np + import paddle + def gen_data(): + return { + "x": np.array([2, 3, 4]).astype('float32'), + "y": np.array([1, 5, 2]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='float32') + y = fluid.data(name="y", shape=[3], dtype='float32') + z = fluid.layers.elementwise_add(x, y) + # z = x + y - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # [3., 8., 6.] + print(z_value) # [3., 8., 6.] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.ones((2, 3, 4, 5)).astype('float32'), - "y": np.zeros((3, 4)).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[3,4], dtype='float32') - z = fluid.layers.elementwise_add(x, y, axis=1) - # z = x + y + def gen_data(): + return { + "x": np.ones((2, 3, 4, 5)).astype('float32'), + "y": np.zeros((3, 4)).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[3,4], dtype='float32') + z = fluid.layers.elementwise_add(x, y, axis=1) + # z = x + y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + print(z_value) # z.shape=[2,3,4,5] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), - "y": np.random.randint(1, 5, size=[5]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[5], dtype='float32') - z = fluid.layers.elementwise_add(x, y, axis=3) - # z = x + y + def gen_data(): + return { + "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), + "y": np.random.randint(1, 5, size=[5]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[5], dtype='float32') + z = fluid.layers.elementwise_add(x, y, axis=3) + # z = x + y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) + print(z_value) # z.shape=[2,3,4,5] """ if _non_static_mode(): @@ -12222,7 +12932,8 @@ def gen_data(): axis=axis, act=act, op_name='elementwise_add', - use_mkldnn=_global_flags()["FLAGS_use_mkldnn"]) + use_mkldnn=_global_flags()["FLAGS_use_mkldnn"], + ) return _elementwise_op(LayerHelper('elementwise_add', **locals())) @@ -12231,90 +12942,88 @@ def gen_data(): def elementwise_div(x, y, axis=-1, act=None, name=None): """ -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='float32') - y = fluid.data(name="y", shape=[3], dtype='float32') - z = fluid.layers.elementwise_div(x, y) - # z = x / y + def gen_data(): + return { + "x": np.array([2, 3, 4]).astype('float32'), + "y": np.array([1, 5, 2]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='float32') + y = fluid.data(name="y", shape=[3], dtype='float32') + z = fluid.layers.elementwise_div(x, y) + # z = x / y - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # [2., 0.6, 2.] + print(z_value) # [2., 0.6, 2.] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.ones((2, 3, 4, 5)).astype('float32'), - "y": np.zeros((3, 4)).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[3,4], dtype='float32') - z = fluid.layers.elementwise_div(x, y, axis=1) - # z = x / y + def gen_data(): + return { + "x": np.ones((2, 3, 4, 5)).astype('float32'), + "y": np.zeros((3, 4)).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[3,4], dtype='float32') + z = fluid.layers.elementwise_div(x, y, axis=1) + # z = x / y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + print(z_value) # z.shape=[2,3,4,5] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), - "y": np.random.randint(1, 5, size=[5]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[5], dtype='float32') - z = fluid.layers.elementwise_div(x, y, axis=3) - # z = x / y + def gen_data(): + return { + "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), + "y": np.random.randint(1, 5, size=[5]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[5], dtype='float32') + z = fluid.layers.elementwise_div(x, y, axis=3) + # z = x / y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) + print(z_value) # z.shape=[2,3,4,5] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_div') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_div' + ) return _elementwise_op(LayerHelper('elementwise_div', **locals())) @@ -12322,90 +13031,88 @@ def gen_data(): def elementwise_sub(x, y, axis=-1, act=None, name=None): """ -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='float32') - y = fluid.data(name="y", shape=[3], dtype='float32') - z = fluid.layers.elementwise_sub(x, y) - # z = x - y + def gen_data(): + return { + "x": np.array([2, 3, 4]).astype('float32'), + "y": np.array([1, 5, 2]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='float32') + y = fluid.data(name="y", shape=[3], dtype='float32') + z = fluid.layers.elementwise_sub(x, y) + # z = x - y - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # [1., -2., 2.] + print(z_value) # [1., -2., 2.] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.ones((2, 3, 4, 5)).astype('float32'), - "y": np.zeros((3, 4)).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[3,4], dtype='float32') - z = fluid.layers.elementwise_sub(x, y, axis=1) - # z = x - y + def gen_data(): + return { + "x": np.ones((2, 3, 4, 5)).astype('float32'), + "y": np.zeros((3, 4)).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[3,4], dtype='float32') + z = fluid.layers.elementwise_sub(x, y, axis=1) + # z = x - y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + print(z_value) # z.shape=[2,3,4,5] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), - "y": np.random.randint(1, 5, size=[5]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[5], dtype='float32') - z = fluid.layers.elementwise_sub(x, y, axis=3) - # z = x - y + def gen_data(): + return { + "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), + "y": np.random.randint(1, 5, size=[5]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[5], dtype='float32') + z = fluid.layers.elementwise_sub(x, y, axis=3) + # z = x - y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) + print(z_value) # z.shape=[2,3,4,5] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_sub') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_sub' + ) return _elementwise_op(LayerHelper('elementwise_sub', **locals())) @@ -12414,222 +13121,216 @@ def gen_data(): def elementwise_mul(x, y, axis=-1, act=None, name=None): """ -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='float32') - y = fluid.data(name="y", shape=[3], dtype='float32') - z = fluid.layers.elementwise_mul(x, y) - # z = x * y + def gen_data(): + return { + "x": np.array([2, 3, 4]).astype('float32'), + "y": np.array([1, 5, 2]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='float32') + y = fluid.data(name="y", shape=[3], dtype='float32') + z = fluid.layers.elementwise_mul(x, y) + # z = x * y - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # [2., 15., 8.] + print(z_value) # [2., 15., 8.] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.ones((2, 3, 4, 5)).astype('float32'), - "y": np.zeros((3, 4)).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[3,4], dtype='float32') - z = fluid.layers.elementwise_mul(x, y, axis=1) - # z = x * y + def gen_data(): + return { + "x": np.ones((2, 3, 4, 5)).astype('float32'), + "y": np.zeros((3, 4)).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[3,4], dtype='float32') + z = fluid.layers.elementwise_mul(x, y, axis=1) + # z = x * y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + print(z_value) # z.shape=[2,3,4,5] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), - "y": np.random.randint(1, 5, size=[5]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[5], dtype='float32') - z = fluid.layers.elementwise_mul(x, y, axis=3) - # z = x * y + def gen_data(): + return { + "x": np.random.randint(1, 5, size=[2, 3, 4, 5]).astype('float32'), + "y": np.random.randint(1, 5, size=[5]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[5], dtype='float32') + z = fluid.layers.elementwise_mul(x, y, axis=3) + # z = x * y - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) - print(z_value) # z.shape=[2,3,4,5] + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) + print(z_value) # z.shape=[2,3,4,5] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_mul') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_mul' + ) return _elementwise_op(LayerHelper('elementwise_mul', **locals())) def elementwise_max(x, y, axis=-1, act=None, name=None): """ - :alias_main: paddle.elementwise_max - :alias: paddle.elementwise_max,paddle.tensor.elementwise_max,paddle.tensor.math.elementwise_max - :old_api: paddle.fluid.layers.elementwise_max + :alias_main: paddle.elementwise_max + :alias: paddle.elementwise_max,paddle.tensor.elementwise_max,paddle.tensor.math.elementwise_max + :old_api: paddle.fluid.layers.elementwise_max -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='float32') - y = fluid.data(name="y", shape=[3], dtype='float32') - z = fluid.layers.elementwise_max(x, y) + def gen_data(): + return { + "x": np.array([2, 3, 4]).astype('float32'), + "y": np.array([1, 5, 2]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='float32') + y = fluid.data(name="y", shape=[3], dtype='float32') + z = fluid.layers.elementwise_max(x, y) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) #[2, 5, 4] + print(z_value) #[2, 5, 4] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.ones((2, 3, 4, 5)).astype('float32'), - "y": np.zeros((3, 4)).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[3,4], dtype='float32') - z = fluid.layers.elementwise_max(x, y, axis=1) + def gen_data(): + return { + "x": np.ones((2, 3, 4, 5)).astype('float32'), + "y": np.zeros((3, 4)).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[3,4], dtype='float32') + z = fluid.layers.elementwise_max(x, y, axis=1) - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value)#[[[[1., 1., 1., 1., 1.] .... [1., 1., 1., 1., 1.]]]] + print(z_value)#[[[[1., 1., 1., 1., 1.] .... [1., 1., 1., 1., 1.]]]] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_max') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_max' + ) return _elementwise_op(LayerHelper('elementwise_max', **locals())) def elementwise_min(x, y, axis=-1, act=None, name=None): """ - :alias_main: paddle.elementwise_min - :alias: paddle.elementwise_min,paddle.tensor.elementwise_min,paddle.tensor.math.elementwise_min - :old_api: paddle.fluid.layers.elementwise_min + :alias_main: paddle.elementwise_min + :alias: paddle.elementwise_min,paddle.tensor.elementwise_min,paddle.tensor.math.elementwise_min + :old_api: paddle.fluid.layers.elementwise_min -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='float32') - y = fluid.data(name="y", shape=[3], dtype='float32') - z = fluid.layers.elementwise_min(x, y) + def gen_data(): + return { + "x": np.array([2, 3, 4]).astype('float32'), + "y": np.array([1, 5, 2]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='float32') + y = fluid.data(name="y", shape=[3], dtype='float32') + z = fluid.layers.elementwise_min(x, y) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) #[1, 3, 2] + print(z_value) #[1, 3, 2] - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.ones((2, 3, 4, 5)).astype('float32'), - "y": np.zeros((3, 4)).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') - y = fluid.data(name="y", shape=[3,4], dtype='float32') - z = fluid.layers.elementwise_min(x, y, axis=1) + def gen_data(): + return { + "x": np.ones((2, 3, 4, 5)).astype('float32'), + "y": np.zeros((3, 4)).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[2,3,4,5], dtype='float32') + y = fluid.data(name="y", shape=[3,4], dtype='float32') + z = fluid.layers.elementwise_min(x, y, axis=1) - place = fluid.CPUPlace() - exe = fluid.Executor(place) + place = fluid.CPUPlace() + exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value)#[[[[0., 0., 0., 0., 0.] .... [0., 0., 0., 0., 0.]]]] + print(z_value)#[[[[0., 0., 0., 0., 0.] .... [0., 0., 0., 0., 0.]]]] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_min') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_min' + ) return _elementwise_op(LayerHelper('elementwise_min', **locals())) @@ -12637,37 +13338,35 @@ def gen_data(): def elementwise_pow(x, y, axis=-1, act=None, name=None): """ -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='float32') - y = fluid.data(name="y", shape=[3], dtype='float32') - z = fluid.layers.elementwise_pow(x, y) + def gen_data(): + return { + "x": np.array([2, 3, 4]).astype('float32'), + "y": np.array([1, 5, 2]).astype('float32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='float32') + y = fluid.data(name="y", shape=[3], dtype='float32') + z = fluid.layers.elementwise_pow(x, y) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) #[2, 243, 16] + print(z_value) #[2, 243, 16] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_pow') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_pow' + ) return _elementwise_op(LayerHelper('elementwise_pow', **locals())) @@ -12675,37 +13374,35 @@ def gen_data(): def elementwise_mod(x, y, axis=-1, act=None, name=None): """ -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([10, 15, 8]).astype('int32'), - "y": np.array([3, 6, 5]).astype('int32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='int32') - y = fluid.data(name="y", shape=[3], dtype='int32') - z = fluid.layers.elementwise_mod(x, y) + def gen_data(): + return { + "x": np.array([10, 15, 8]).astype('int32'), + "y": np.array([3, 6, 5]).astype('int32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='int32') + y = fluid.data(name="y", shape=[3], dtype='int32') + z = fluid.layers.elementwise_mod(x, y) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) #[1, 3, 3] + print(z_value) #[1, 3, 3] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_mod') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_mod' + ) return _elementwise_op(LayerHelper('elementwise_mod', **locals())) @@ -12714,78 +13411,89 @@ def gen_data(): def elementwise_floordiv(x, y, axis=-1, act=None, name=None): """ -Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle.fluid as fluid - import numpy as np - import paddle + import paddle.fluid as fluid + import numpy as np + import paddle - def gen_data(): - return { - "x": np.array([10, 15, 8]).astype('int32'), - "y": np.array([3, 7, 5]).astype('int32') - } - paddle.enable_static() - x = fluid.data(name="x", shape=[3], dtype='int32') - y = fluid.data(name="y", shape=[3], dtype='int32') - z = fluid.layers.elementwise_floordiv(x, y) + def gen_data(): + return { + "x": np.array([10, 15, 8]).astype('int32'), + "y": np.array([3, 7, 5]).astype('int32') + } + paddle.enable_static() + x = fluid.data(name="x", shape=[3], dtype='int32') + y = fluid.data(name="y", shape=[3], dtype='int32') + z = fluid.layers.elementwise_floordiv(x, y) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - z_value = exe.run(feed=gen_data(), - fetch_list=[z.name]) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + z_value = exe.run(feed=gen_data(), + fetch_list=[z.name]) - print(z_value) #[3, 2, 1] + print(z_value) #[3, 2, 1] """ if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_floordiv') + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_floordiv' + ) return _elementwise_op(LayerHelper('elementwise_floordiv', **locals())) for func in [ - elementwise_add, - elementwise_div, - elementwise_sub, - elementwise_mul, - elementwise_max, - elementwise_pow, - elementwise_min, - elementwise_mod, - elementwise_floordiv, + elementwise_add, + elementwise_div, + elementwise_sub, + elementwise_mul, + elementwise_max, + elementwise_pow, + elementwise_min, + elementwise_mod, + elementwise_floordiv, ]: op_proto = OpProtoHolder.instance().get_op_proto(func.__name__) # insert the c++ doc string on top of python doc string - func.__doc__ = _generate_doc_string_( - op_proto, - additional_args_lines=[ - "axis (int32, optional): If X.dimension != Y.dimension, \ + func.__doc__ = ( + _generate_doc_string_( + op_proto, + additional_args_lines=[ + "axis (int32, optional): If X.dimension != Y.dimension, \ Y.dimension must be a subsequence of x.dimension. \ And axis is the start dimension index for broadcasting Y onto X. ", - "act (string, optional): Activation applied to the output. \ + "act (string, optional): Activation applied to the output. \ Default is None. Details: :ref:`api_guide_activations_en` ", - "name (string, optional): Name of the output. \ + "name (string, optional): Name of the output. \ Default is None. It's used to print debug info for developers. Details: \ - :ref:`api_guide_Name` " - ], - skip_attrs_set={ - "x_data_format", "y_data_format", "axis", "use_quantizer", - "mkldnn_data_type", "Scale_x", "Scale_y", "Scale_out" - }) + """\n""" + str(func.__doc__) + :ref:`api_guide_Name` ", + ], + skip_attrs_set={ + "x_data_format", + "y_data_format", + "axis", + "use_quantizer", + "mkldnn_data_type", + "Scale_x", + "Scale_y", + "Scale_out", + }, + ) + + """\n""" + + str(func.__doc__) + ) doc_list = func.__doc__.splitlines() for idx, val in enumerate(doc_list): - if val.startswith("Warning: ") and val.endswith( - " instead." - ) and "and will be removed in future versions." in val: + if ( + val.startswith("Warning: ") + and val.endswith(" instead.") + and "and will be removed in future versions." in val + ): doc_list.insert(0, doc_list.pop(idx)) func.__doc__ = "\n" + "\n".join(i for i in doc_list) break @@ -12796,9 +13504,12 @@ def gen_data(): op_proto, additional_args_lines=[ "act (basestring|None): Activation applied to the output.", - "name (basestring|None): Name of the output." - ]) - func.__doc__ = func.__doc__ + """ + "name (basestring|None): Name of the output.", + ], + ) + func.__doc__ = ( + func.__doc__ + + """ Examples: .. code-block:: python @@ -12833,8 +13544,16 @@ def gen_data(): x5 = fluid.layers.data(name="x5", shape=[2, 3, 4, 5], dtype='float32') y5 = fluid.layers.data(name="y5", shape=[2], dtype='float32') z5 = fluid.layers.%s(x5, y5, axis=0) - """ % (func.__name__, func.__name__, func.__name__, func.__name__, - func.__name__, func.__name__) + """ + % ( + func.__name__, + func.__name__, + func.__name__, + func.__name__, + func.__name__, + func.__name__, + ) + ) def _logical_op(op_name, x, y, out=None, name=None, binary_op=True): @@ -12845,14 +13564,18 @@ def _logical_op(op_name, x, y, out=None, name=None, binary_op=True): else: return op(x) check_variable_and_dtype( - x, "x", + x, + "x", ["bool", "int8", "int16", "int32", "int64", "float32", "float64"], - op_name) + op_name, + ) if y is not None: check_variable_and_dtype( - y, "y", + y, + "y", ["bool", "int8", "int16", "int32", "int64", "float32", "float64"], - op_name) + op_name, + ) if out is not None: check_type(out, "out", Variable, op_name) @@ -12861,18 +13584,16 @@ def _logical_op(op_name, x, y, out=None, name=None, binary_op=True): if binary_op and x.dtype != y.dtype: raise ValueError( "(InvalidArgument) The DataType of %s Op's Variable must be consistent, but received %s and %s." - % (op_name, x.dtype, y.dtype)) + % (op_name, x.dtype, y.dtype) + ) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) if binary_op: - helper.append_op(type=op_name, - inputs={ - "X": x, - "Y": y - }, - outputs={"Out": out}) + helper.append_op( + type=op_name, inputs={"X": x, "Y": y}, outputs={"Out": out} + ) else: helper.append_op(type=op_name, inputs={"X": x}, outputs={"Out": out}) @@ -12914,12 +13635,9 @@ def logical_and(x, y, out=None, name=None): if in_dygraph_mode(): return _C_ops.logical_and(x, y) - return _logical_op(op_name="logical_and", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _logical_op( + op_name="logical_and", x=x, y=y, name=name, out=out, binary_op=True + ) def logical_or(x, y, out=None, name=None): @@ -12959,12 +13677,9 @@ def logical_or(x, y, out=None, name=None): """ if in_dygraph_mode(): return _C_ops.logical_or(x, y) - return _logical_op(op_name="logical_or", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _logical_op( + op_name="logical_or", x=x, y=y, name=name, out=out, binary_op=True + ) def logical_xor(x, y, out=None, name=None): @@ -13005,12 +13720,9 @@ def logical_xor(x, y, out=None, name=None): if in_dygraph_mode(): return _C_ops.logical_xor(x, y) - return _logical_op(op_name="logical_xor", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _logical_op( + op_name="logical_xor", x=x, y=y, name=name, out=out, binary_op=True + ) @templatedoc() @@ -13043,18 +13755,15 @@ def logical_not(x, out=None, name=None): """ if in_dygraph_mode(): return _C_ops.logical_not(x) - return _logical_op(op_name="logical_not", - x=x, - y=None, - name=name, - out=out, - binary_op=False) + return _logical_op( + op_name="logical_not", x=x, y=None, name=name, out=out, binary_op=False + ) @templatedoc() def clip(x, min, max, name=None): """ - :old_api: paddle.fluid.layers.clip + :old_api: paddle.fluid.layers.clip ${comment} @@ -13085,21 +13794,20 @@ def clip(x, min, max, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'clip') if name is None: - name = unique_name.generate_with_ignorable_key(".".join( - [helper.name, 'tmp'])) - - out = helper.create_variable(type=x.type, - name=name, - dtype=x.dtype, - persistable=False) - - helper.append_op(type="clip", - inputs={"X": x}, - attrs={ - "min": min, - "max": max - }, - outputs={"Out": out}) + name = unique_name.generate_with_ignorable_key( + ".".join([helper.name, 'tmp']) + ) + + out = helper.create_variable( + type=x.type, name=name, dtype=x.dtype, persistable=False + ) + + helper.append_op( + type="clip", + inputs={"X": x}, + attrs={"min": min, "max": max}, + outputs={"Out": out}, + ) return out @@ -13143,18 +13851,20 @@ def clip_by_norm(x, max_norm, name=None): check_type(max_norm, 'max_norm', (float), 'clip_by_norm') if name is None: - name = unique_name.generate_with_ignorable_key(".".join( - [helper.name, 'tmp'])) + name = unique_name.generate_with_ignorable_key( + ".".join([helper.name, 'tmp']) + ) - out = helper.create_variable(type=x.type, - name=name, - dtype=x.dtype, - persistable=False) + out = helper.create_variable( + type=x.type, name=name, dtype=x.dtype, persistable=False + ) - helper.append_op(type="clip_by_norm", - inputs={"X": x}, - attrs={"max_norm": max_norm}, - outputs={"Out": out}) + helper.append_op( + type="clip_by_norm", + inputs={"X": x}, + attrs={"max_norm": max_norm}, + outputs={"Out": out}, + ) return out @@ -13193,10 +13903,9 @@ def mean(x, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'mean') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="mean", - inputs={"X": x}, - attrs={}, - outputs={"Out": out}) + helper.append_op( + type="mean", inputs={"X": x}, attrs={}, outputs={"Out": out} + ) return out @@ -13231,10 +13940,12 @@ def merge_selected_rows(x, name=None): helper = LayerHelper("merge_selected_rows", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="merge_selected_rows", - inputs={"X": x}, - attrs={}, - outputs={"Out": out}) + helper.append_op( + type="merge_selected_rows", + inputs={"X": x}, + attrs={}, + outputs={"Out": out}, + ) return out @@ -13274,8 +13985,14 @@ def mul(x, y, x_num_col_dims=1, y_num_col_dims=1, name=None): """ if _non_static_mode(): - return _legacy_C_ops.mul(x, y, 'x_num_col_dims', x_num_col_dims, - 'y_num_col_dims', y_num_col_dims) + return _legacy_C_ops.mul( + x, + y, + 'x_num_col_dims', + x_num_col_dims, + 'y_num_col_dims', + y_num_col_dims, + ) inputs = {"X": [x], "Y": [y]} attrs = {"x_num_col_dims": x_num_col_dims, "y_num_col_dims": y_num_col_dims} @@ -13284,13 +14001,9 @@ def mul(x, y, x_num_col_dims=1, y_num_col_dims=1, name=None): check_variable_and_dtype(y, 'y', ['float16', 'float32', 'float64'], 'mul') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="mul", - inputs={ - "X": x, - "Y": y - }, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="mul", inputs={"X": x, "Y": y}, attrs=attrs, outputs={"Out": out} + ) return out @@ -13420,24 +14133,27 @@ def space_to_depth(x, blocksize, name=None): if not (isinstance(blocksize, int)): raise ValueError("blocksize must be a python Int") - check_variable_and_dtype(x, 'x', \ - ['float16', 'float32', 'float64', 'int32', 'int64'], 'space_to_depth') + check_variable_and_dtype( + x, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'space_to_depth', + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="space_to_depth", - inputs={"X": x}, - attrs={"blocksize": blocksize}, - outputs={"Out": out}) + helper.append_op( + type="space_to_depth", + inputs={"X": x}, + attrs={"blocksize": blocksize}, + outputs={"Out": out}, + ) return out -def affine_channel(x, - scale=None, - bias=None, - data_layout='NCHW', - name=None, - act=None): +def affine_channel( + x, scale=None, bias=None, data_layout='NCHW', name=None, act=None +): """ Applies a separate affine transformation to each channel of the input. @@ -13504,14 +14220,12 @@ def affine_channel(x, check_type(bias, 'bias', (Variable, type(None)), 'affine_channel') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="affine_channel", - inputs={ - "X": x, - 'Scale': scale, - 'Bias': bias - }, - attrs={"data_layout": data_layout}, - outputs={"Out": out}) + helper.append_op( + type="affine_channel", + inputs={"X": x, 'Scale': scale, 'Bias': bias}, + attrs={"data_layout": data_layout}, + outputs={"Out": out}, + ) return helper.append_activation(out) @@ -13610,8 +14324,9 @@ def similarity_focus(input, axis, indexes, name=None): """ helper = LayerHelper('similarity_focus', **locals()) # check attrs - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - "similarity_focus") + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], "similarity_focus" + ) check_type(axis, 'axis', int, "similarity_focus") check_type(indexes, 'indexes', list, "similarity_focus") if axis != 1 and axis != 2 and axis != 3: @@ -13620,13 +14335,12 @@ def similarity_focus(input, axis, indexes, name=None): raise ValueError("indexes can not be empty.") out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='similarity_focus', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - "axis": axis, - "indexes": indexes - }) + helper.append_op( + type='similarity_focus', + inputs={'X': input}, + outputs={'Out': out}, + attrs={"axis": axis, "indexes": indexes}, + ) return out @@ -13681,15 +14395,15 @@ def hash(input, hash_size, num_hash=1, name=None): check_type(hash_size, 'hash_size', int, 'hash') check_type(num_hash, 'num_hash', int, 'hash') helper = LayerHelper('hash', **locals()) - out = helper.create_variable_for_type_inference(helper.input_dtype(), - stop_gradient=True) - helper.append_op(type='hash', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'num_hash': num_hash, - 'mod_by': hash_size - }) + out = helper.create_variable_for_type_inference( + helper.input_dtype(), stop_gradient=True + ) + helper.append_op( + type='hash', + inputs={'X': input}, + outputs={'Out': out}, + attrs={'num_hash': num_hash, 'mod_by': hash_size}, + ) return out @@ -13783,8 +14497,9 @@ def grid_sampler(x, grid, name=None): helper = LayerHelper("grid_sampler", **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'grid_sampler') - check_variable_and_dtype(grid, 'grid', ['float32', 'float64'], - 'grid_sampler') + check_variable_and_dtype( + grid, 'grid', ['float32', 'float64'], 'grid_sampler' + ) if not isinstance(x, Variable): return ValueError("The x should be a Variable") @@ -13796,10 +14511,9 @@ def grid_sampler(x, grid, name=None): attrs = {'use_cudnn': False} if core.is_compiled_with_rocm() else {} - helper.append_op(type='grid_sampler', - inputs=ipts, - outputs={'Output': out}, - attrs=attrs) + helper.append_op( + type='grid_sampler', inputs=ipts, outputs={'Output': out}, attrs=attrs + ) return out @@ -13892,33 +14606,30 @@ def add_position_encoding(input, alpha, beta, name=None): """ if _non_static_mode(): - return _legacy_C_ops.add_position_encoding(input, "alpha", alpha, - "beta", beta) + return _legacy_C_ops.add_position_encoding( + input, "alpha", alpha, "beta", beta + ) helper = LayerHelper('add_position_encoding', **locals()) - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - "add_position_encoding") + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], "add_position_encoding" + ) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type="add_position_encoding", - inputs={"X": input}, - outputs={"Out": out}, - attrs={ - "alpha": alpha, - "beta": beta - }) + helper.append_op( + type="add_position_encoding", + inputs={"X": input}, + outputs={"Out": out}, + attrs={"alpha": alpha, "beta": beta}, + ) return out -def bilinear_tensor_product(x, - y, - size, - act=None, - name=None, - param_attr=None, - bias_attr=None): +def bilinear_tensor_product( + x, y, size, act=None, name=None, param_attr=None, bias_attr=None +): r""" :api_attr: Static Graph @@ -13969,23 +14680,21 @@ def bilinear_tensor_product(x, param_shape = [size, x.shape[1], y.shape[1]] - w = helper.create_parameter(attr=helper.param_attr, - shape=param_shape, - dtype=dtype, - is_bias=False) + w = helper.create_parameter( + attr=helper.param_attr, shape=param_shape, dtype=dtype, is_bias=False + ) out = helper.create_variable_for_type_inference(dtype=dtype) inputs = {"X": x, "Y": y, "Weight": w} if helper.bias_attr: bias_size = [1, size] - bias = helper.create_parameter(attr=helper.bias_attr, - shape=bias_size, - dtype=dtype, - is_bias=True) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True + ) inputs["Bias"] = bias - helper.append_op(type="bilinear_tensor_product", - inputs=inputs, - outputs={"Out": out}) + helper.append_op( + type="bilinear_tensor_product", inputs=inputs, outputs={"Out": out} + ) # add activation return helper.append_activation(out) @@ -14035,10 +14744,12 @@ def get_tensor_from_selected_rows(x, name=None): ) helper = LayerHelper('get_tensor_from_selected_rows', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='get_tensor_from_selected_rows', - inputs={'X': x}, - outputs={'Out': out}, - attrs={}) + helper.append_op( + type='get_tensor_from_selected_rows', + inputs={'X': x}, + outputs={'Out': out}, + attrs={}, + ) return out @@ -14108,10 +14819,12 @@ def shuffle_channel(x, group, name=None): if not isinstance(group, int): raise TypeError("group must be int type") - helper.append_op(type="shuffle_channel", - inputs={"X": x}, - outputs={"Out": out}, - attrs={"group": group}) + helper.append_op( + type="shuffle_channel", + inputs={"X": x}, + outputs={"Out": out}, + attrs={"group": group}, + ) return out @@ -14149,8 +14862,9 @@ def temporal_shift(x, seg_num, shift_ratio=0.25, name=None, data_format="NCHW"): input = paddle.randn([6, 4, 2, 2]) out = F.temporal_shift(x=input, seg_num=2, shift_ratio=0.2) """ - return paddle.nn.functional.temporal_shift(x, seg_num, shift_ratio, name, - data_format) + return paddle.nn.functional.temporal_shift( + x, seg_num, shift_ratio, name, data_format + ) class PyFuncRegistry(object): @@ -14209,7 +14923,7 @@ def __call__(self, *args): func_ret = self._func(*args[idx:], **kwargs) if not isinstance(func_ret, (list, tuple)): - func_ret = (func_ret, ) + func_ret = (func_ret,) ret = [] for each_ret in func_ret: @@ -14425,11 +15139,13 @@ def py_func_demo(): out_list = out else: raise TypeError( - 'Output must be Variable/list(Variable)/tuple(Variable)') + 'Output must be Variable/list(Variable)/tuple(Variable)' + ) fwd_func_id = PyFuncRegistry(func).id - bwd_func_id = PyFuncRegistry( - backward_func).id if backward_func is not None else -1 + bwd_func_id = ( + PyFuncRegistry(backward_func).id if backward_func is not None else -1 + ) for each_out in out_list: if len(each_out.shape) == 0: @@ -14449,18 +15165,22 @@ def py_func_demo(): for v in skip_vars_in_backward_input: if not v.name in fwd_in_out: raise ValueError( - 'Variable {} is not found in forward inputs and outputs'. - format(v.name)) + 'Variable {} is not found in forward inputs and outputs'.format( + v.name + ) + ) backward_skip_vars.add(v.name) - helper.append_op(type='py_func', - inputs={'X': x}, - outputs={'Out': out_list}, - attrs={ - 'forward_callable_id': fwd_func_id, - 'backward_callable_id': bwd_func_id, - 'backward_skip_vars': list(backward_skip_vars) - }) + helper.append_op( + type='py_func', + inputs={'X': x}, + outputs={'Out': out_list}, + attrs={ + 'forward_callable_id': fwd_func_id, + 'backward_callable_id': bwd_func_id, + 'backward_skip_vars': list(backward_skip_vars), + }, + ) return out @@ -14470,13 +15190,15 @@ def py_func_demo(): @templatedoc() -def psroi_pool(input, - rois, - output_channels, - spatial_scale, - pooled_height, - pooled_width, - name=None): +def psroi_pool( + input, + rois, + output_channels, + spatial_scale, + pooled_height, + pooled_width, + name=None, +): """ ${comment} @@ -14524,29 +15246,30 @@ def psroi_pool(input, raise TypeError("pooled_width must be int type") dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='psroi_pool', - inputs={ - 'X': input, - 'ROIs': rois - }, - outputs={'Out': out}, - attrs={ - 'output_channels': output_channels, - 'spatial_scale': spatial_scale, - 'pooled_height': pooled_height, - 'pooled_width': pooled_width - }) + helper.append_op( + type='psroi_pool', + inputs={'X': input, 'ROIs': rois}, + outputs={'Out': out}, + attrs={ + 'output_channels': output_channels, + 'spatial_scale': spatial_scale, + 'pooled_height': pooled_height, + 'pooled_width': pooled_width, + }, + ) return out @templatedoc() -def prroi_pool(input, - rois, - spatial_scale=1.0, - pooled_height=1, - pooled_width=1, - batch_roi_nums=None, - name=None): +def prroi_pool( + input, + rois, + spatial_scale=1.0, + pooled_height=1, + pooled_width=1, + batch_roi_nums=None, + name=None, +): """ The precise roi pooling implementation for paddle. Reference: https://arxiv.org/pdf/1807.11590.pdf @@ -14609,14 +15332,16 @@ def prroi_pool(input, inputs_op = {'X': input, 'ROIs': rois} if batch_roi_nums is not None: inputs_op['BatchRoINums'] = batch_roi_nums - helper.append_op(type='prroi_pool', - inputs=inputs_op, - outputs={'Out': out}, - attrs={ - 'spatial_scale': spatial_scale, - 'pooled_height': pooled_height, - 'pooled_width': pooled_width - }) + helper.append_op( + type='prroi_pool', + inputs=inputs_op, + outputs={'Out': out}, + attrs={ + 'spatial_scale': spatial_scale, + 'pooled_height': pooled_height, + 'pooled_width': pooled_width, + }, + ) return out @@ -14645,23 +15370,23 @@ def pixel_shuffle(x, upscale_factor): Examples: .. code-block:: python - # declarative mode - import paddle.fluid as fluid - import numpy as np - input = fluid.data(name="input", shape=[2,9,4,4]) - output = fluid.layers.pixel_shuffle(x=input, upscale_factor=3) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) + # declarative mode + import paddle.fluid as fluid + import numpy as np + input = fluid.data(name="input", shape=[2,9,4,4]) + output = fluid.layers.pixel_shuffle(x=input, upscale_factor=3) + place = fluid.CPUPlace() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) - input_data = np.random.rand(2,9,4,4).astype("float32") - output_data = exe.run(fluid.default_main_program(), + input_data = np.random.rand(2,9,4,4).astype("float32") + output_data = exe.run(fluid.default_main_program(), feed={"input":input_data}, fetch_list=[output], return_numpy=True) - # print(output.shape) - # (2L, 1L, 12L, 12L) + # print(output.shape) + # (2L, 1L, 12L, 12L) """ @@ -14673,10 +15398,12 @@ def pixel_shuffle(x, upscale_factor): if not isinstance(upscale_factor, int): raise TypeError("upscale factor must be int type") - helper.append_op(type="pixel_shuffle", - inputs={"X": x}, - outputs={"Out": out}, - attrs={"upscale_factor": upscale_factor}) + helper.append_op( + type="pixel_shuffle", + inputs={"X": x}, + outputs={"Out": out}, + attrs={"upscale_factor": upscale_factor}, + ) return out @@ -14726,8 +15453,9 @@ def fsp_matrix(x, y): check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'fsp_matrix') check_variable_and_dtype(y, 'y', ['float32', 'float64'], 'fsp_matrix') helper = LayerHelper('fsp_matrix', **locals()) - out = helper.create_variable_for_type_inference(dtype=helper.input_dtype( - input_param_name='x')) + out = helper.create_variable_for_type_inference( + dtype=helper.input_dtype(input_param_name='x') + ) helper.append_op(type='fsp', inputs={'X': x, 'Y': y}, outputs={'Out': out}) return out @@ -14777,15 +15505,15 @@ def continuous_value_model(input, cvm, use_cvm=True): """ helper = LayerHelper('cvm', **locals()) out = helper.create_variable(dtype=input.dtype) - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'cvm') - helper.append_op(type='cvm', - inputs={ - 'X': [input], - 'CVM': [cvm] - }, - outputs={'Y': [out]}, - attrs={"use_cvm": use_cvm}) + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'cvm' + ) + helper.append_op( + type='cvm', + inputs={'X': [input], 'CVM': [cvm]}, + outputs={'Y': [out]}, + attrs={"use_cvm": use_cvm}, + ) return out @@ -14831,11 +15559,14 @@ def where(condition): helper = LayerHelper("where_index", **locals()) out = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.INT64) - - helper.append_op(type='where_index', - inputs={'Condition': condition}, - outputs={'Out': [out]}) + dtype=core.VarDesc.VarType.INT64 + ) + + helper.append_op( + type='where_index', + inputs={'Condition': condition}, + outputs={'Out': [out]}, + ) return out @@ -14894,21 +15625,21 @@ def unique(x, dtype='int32'): out, index = fluid.layers.unique(x) # out is [2, 3, 1, 5]; index is [0, 1, 1, 2, 3, 1] """ - check_variable_and_dtype(x, "x", ['float32', 'float64', 'int32', 'int64'], - "unique") + check_variable_and_dtype( + x, "x", ['float32', 'float64', 'int32', 'int64'], "unique" + ) helper = LayerHelper("unique", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) index = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='unique', - inputs={'X': x}, - attrs={'dtype': convert_np_dtype_to_dtype_(dtype)}, - outputs={ - 'Out': [out], - 'Index': [index] - }) + helper.append_op( + type='unique', + inputs={'X': x}, + attrs={'dtype': convert_np_dtype_to_dtype_(dtype)}, + outputs={'Out': [out], 'Index': [index]}, + ) return out, index @@ -14941,11 +15672,13 @@ def unique_with_counts(x, dtype='int32'): # count is [1, 3, 1, 1] # x.shape=(6,) out.shape=(4,), index.shape=(6,), count.shape=(4,) """ - check_variable_and_dtype(x, "x", ['float32', 'float64', 'int32', 'int64'], - "unique_with_counts") + check_variable_and_dtype( + x, "x", ['float32', 'float64', 'int32', 'int64'], "unique_with_counts" + ) if not (dtype == 'int32' or dtype == 'int64'): raise TypeError( - "Op unique_with_counts, index dtype must be int32 or int64") + "Op unique_with_counts, index dtype must be int32 or int64" + ) if x is None or len(x.shape) != 1: raise ValueError( @@ -14960,33 +15693,33 @@ def unique_with_counts(x, dtype='int32'): count = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='unique_with_counts', - inputs={'X': x}, - attrs={'dtype': convert_np_dtype_to_dtype_(dtype)}, - outputs={ - 'Out': [out], - 'Index': [index], - 'Count': [count] - }) + helper.append_op( + type='unique_with_counts', + inputs={'X': x}, + attrs={'dtype': convert_np_dtype_to_dtype_(dtype)}, + outputs={'Out': [out], 'Index': [index], 'Count': [count]}, + ) return out, index, count -def deformable_conv(input, - offset, - mask, - num_filters, - filter_size, - stride=1, - padding=0, - dilation=1, - groups=None, - deformable_groups=None, - im2col_step=None, - param_attr=None, - bias_attr=None, - modulated=True, - name=None): +def deformable_conv( + input, + offset, + mask, + num_filters, + filter_size, + stride=1, + padding=0, + dilation=1, + groups=None, + deformable_groups=None, + im2col_step=None, + param_attr=None, + bias_attr=None, + modulated=True, + name=None, +): r""" :api_attr: Static Graph @@ -15117,10 +15850,12 @@ def deformable_conv(input, num_filters=2, filter_size=filter_size, padding=1, modulated=False) """ - check_variable_and_dtype(input, "input", ['float32', 'float64'], - 'deformable_conv') - check_variable_and_dtype(offset, "offset", ['float32', 'float64'], - 'deformable_conv') + check_variable_and_dtype( + input, "input", ['float32', 'float64'], 'deformable_conv' + ) + check_variable_and_dtype( + offset, "offset", ['float32', 'float64'], 'deformable_conv' + ) check_type(mask, 'mask', (Variable, type(None)), 'deformable_conv') num_channels = input.shape[1] @@ -15155,52 +15890,58 @@ def _get_default_param_initializer(): raise ValueError( "Invalid filter number, excepted number is larger than 0, but" " received {}, please check the input shape and " - "filter size.".format(filter_elem_num)) - std = (2.0 / filter_elem_num)**0.5 + "filter size.".format(filter_elem_num) + ) + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std, 0) filter_param = helper.create_parameter( attr=helper.param_attr, shape=filter_shape, dtype=dtype, - default_initializer=_get_default_param_initializer()) + default_initializer=_get_default_param_initializer(), + ) pre_bias = helper.create_variable_for_type_inference(dtype) if modulated: - helper.append_op(type='deformable_conv', - inputs={ - 'Input': input, - 'Filter': filter_param, - 'Offset': offset, - 'Mask': mask, - }, - outputs={"Output": pre_bias}, - attrs={ - 'strides': stride, - 'paddings': padding, - 'dilations': dilation, - 'groups': groups, - 'deformable_groups': deformable_groups, - 'im2col_step': im2col_step, - }) + helper.append_op( + type='deformable_conv', + inputs={ + 'Input': input, + 'Filter': filter_param, + 'Offset': offset, + 'Mask': mask, + }, + outputs={"Output": pre_bias}, + attrs={ + 'strides': stride, + 'paddings': padding, + 'dilations': dilation, + 'groups': groups, + 'deformable_groups': deformable_groups, + 'im2col_step': im2col_step, + }, + ) else: - helper.append_op(type='deformable_conv_v1', - inputs={ - 'Input': input, - 'Filter': filter_param, - 'Offset': offset, - }, - outputs={"Output": pre_bias}, - attrs={ - 'strides': stride, - 'paddings': padding, - 'dilations': dilation, - 'groups': groups, - 'deformable_groups': deformable_groups, - 'im2col_step': im2col_step, - }) + helper.append_op( + type='deformable_conv_v1', + inputs={ + 'Input': input, + 'Filter': filter_param, + 'Offset': offset, + }, + outputs={"Output": pre_bias}, + attrs={ + 'strides': stride, + 'paddings': padding, + 'dilations': dilation, + 'groups': groups, + 'deformable_groups': deformable_groups, + 'im2col_step': im2col_step, + }, + ) output = helper.append_bias_op(pre_bias, dim_start=1, dim_end=2) return output @@ -15276,23 +16017,26 @@ def unfold(x, kernel_sizes, strides=1, paddings=0, dilations=1, name=None): y = F.unfold(x, [3, 3], 1, 1, 1) """ - return paddle.nn.functional.unfold(x, kernel_sizes, strides, paddings, - dilations, name) - - -def deformable_roi_pooling(input, - rois, - trans, - no_trans=False, - spatial_scale=1.0, - group_size=[1, 1], - pooled_height=1, - pooled_width=1, - part_size=None, - sample_per_part=1, - trans_std=0.1, - position_sensitive=False, - name=None): + return paddle.nn.functional.unfold( + x, kernel_sizes, strides, paddings, dilations, name + ) + + +def deformable_roi_pooling( + input, + rois, + trans, + no_trans=False, + spatial_scale=1.0, + group_size=[1, 1], + pooled_height=1, + pooled_width=1, + part_size=None, + sample_per_part=1, + trans_std=0.1, + position_sensitive=False, + name=None, +): r""" Deformable ROI Pooling Layer @@ -15397,17 +16141,22 @@ def deformable_roi_pooling(input, position_sensitive=False) """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'deformable_roi_pooling') - check_variable_and_dtype(rois, 'rois', ['float32', 'float64'], - 'deformable_roi_pooling') - check_variable_and_dtype(trans, 'trans', ['float32', 'float64'], - 'deformable_roi_pooling') - check_type(group_size, 'group_size', (list, tuple), - 'deformable_roi_pooling') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'deformable_roi_pooling' + ) + check_variable_and_dtype( + rois, 'rois', ['float32', 'float64'], 'deformable_roi_pooling' + ) + check_variable_and_dtype( + trans, 'trans', ['float32', 'float64'], 'deformable_roi_pooling' + ) + check_type( + group_size, 'group_size', (list, tuple), 'deformable_roi_pooling' + ) if part_size is not None: - check_type(part_size, 'part_size', (list, tuple), - 'deformable_roi_pooling') + check_type( + part_size, 'part_size', (list, tuple), 'deformable_roi_pooling' + ) input_channels = input.shape[1] if position_sensitive == False: @@ -15425,27 +16174,22 @@ def deformable_roi_pooling(input, dtype = helper.input_dtype() output = helper.create_variable_for_type_inference(dtype) top_count = helper.create_variable_for_type_inference(dtype='int32') - helper.append_op(type="deformable_psroi_pooling", - inputs={ - "Input": input, - "ROIs": rois, - "Trans": trans - }, - outputs={ - "Output": output, - "TopCount": top_count - }, - attrs={ - "no_trans": no_trans, - "spatial_scale": spatial_scale, - "output_dim": output_channels, - "group_size": group_size, - "pooled_height": pooled_height, - "pooled_width": pooled_width, - "part_size": part_size, - "sample_per_part": sample_per_part, - "trans_std": trans_std - }) + helper.append_op( + type="deformable_psroi_pooling", + inputs={"Input": input, "ROIs": rois, "Trans": trans}, + outputs={"Output": output, "TopCount": top_count}, + attrs={ + "no_trans": no_trans, + "spatial_scale": spatial_scale, + "output_dim": output_channels, + "group_size": group_size, + "pooled_height": pooled_height, + "pooled_width": pooled_width, + "part_size": part_size, + "sample_per_part": sample_per_part, + "trans_std": trans_std, + }, + ) return output @@ -15497,27 +16241,31 @@ def shard_index(input, index_num, nshards, shard_id, ignore_value=-1): # [[-1], [1]] """ if in_dygraph_mode(): - return _C_ops.shard_index(input, index_num, nshards, shard_id, - ignore_value) + return _C_ops.shard_index( + input, index_num, nshards, shard_id, ignore_value + ) check_variable_and_dtype(input, 'input', ['int64', 'int32'], 'shard_index') op_type = 'shard_index' helper = LayerHelper(op_type, **locals()) if shard_id < 0 or shard_id >= nshards: - raise ValueError('The shard_id(%d) should be in [0, %d)' % - (shard_id, nshards)) + raise ValueError( + 'The shard_id(%d) should be in [0, %d)' % (shard_id, nshards) + ) out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type=op_type, - inputs={'X': [input]}, - outputs={'Out': out}, - attrs={ - 'index_num': index_num, - 'nshards': nshards, - 'shard_id': shard_id, - 'ignore_value': ignore_value - }, - stop_gradient=True) + helper.append_op( + type=op_type, + inputs={'X': [input]}, + outputs={'Out': out}, + attrs={ + 'index_num': index_num, + 'nshards': nshards, + 'shard_id': shard_id, + 'ignore_value': ignore_value, + }, + stop_gradient=True, + ) return out @@ -15572,22 +16320,22 @@ def hard_swish(x, threshold=6.0, scale=6.0, offset=3.0, name=None): print(out) # [[0.66666667, 1.66666667,3., 4.]] """ if _non_static_mode(): - return _legacy_C_ops.hard_swish(x, 'threshold', threshold, 'scale', - scale, 'offset', offset) + return _legacy_C_ops.hard_swish( + x, 'threshold', threshold, 'scale', scale, 'offset', offset + ) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'hard_swish') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'hard_swish' + ) helper = LayerHelper('hard_swish', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='hard_swish', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'threshold': threshold, - 'scale': scale, - 'offset': offset - }) + helper.append_op( + type='hard_swish', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'threshold': threshold, 'scale': scale, 'offset': offset}, + ) return out @@ -15658,15 +16406,20 @@ def mish(x, threshold=20, name=None): check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'mish') check_type(threshold, 'threshold', (float, int), 'mish') - assert threshold > 0, "threshold of mish should be greater than 0, " \ - "but got {}".format(threshold) + assert ( + threshold > 0 + ), "threshold of mish should be greater than 0, " "but got {}".format( + threshold + ) helper = LayerHelper('mish', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='mish', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'threshold': threshold}) + helper.append_op( + type='mish', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'threshold': threshold}, + ) return out @@ -15736,12 +16489,9 @@ def gather_tree(ids, parents): @deprecated(since="2.0.0", update_to="paddle.uniform") @templatedoc() -def uniform_random(shape, - dtype='float32', - min=-1.0, - max=1.0, - seed=0, - name=None): +def uniform_random( + shape, dtype='float32', min=-1.0, max=1.0, seed=0, name=None +): """ This OP returns a Tensor filled with random values sampled from a uniform distribution in the range [``min``, ``max``), with ``shape`` and ``dtype``. @@ -15821,34 +16571,47 @@ def uniform_random(shape, if in_dygraph_mode(): shape = utils.convert_shape_to_list(shape) - return _C_ops.uniform_random(shape, dtype, float(min), float(max), seed, - _current_expected_place()) + return _C_ops.uniform_random( + shape, + dtype, + float(min), + float(max), + seed, + _current_expected_place(), + ) elif _in_legacy_dygraph(): shape = utils.convert_shape_to_list(shape) - return _legacy_C_ops.uniform_random('shape', - shape, 'min', float(min), 'max', - float(max), 'seed', seed, 'dtype', - dtype) + return _legacy_C_ops.uniform_random( + 'shape', + shape, + 'min', + float(min), + 'max', + float(max), + 'seed', + seed, + 'dtype', + dtype, + ) check_type(shape, 'shape', (list, tuple, Variable), 'uniform_random/rand') - check_dtype(dtype, 'dtype', ('float32', 'float64', 'uint16'), - 'uniform_random/rand') + check_dtype( + dtype, 'dtype', ('float32', 'float64', 'uint16'), 'uniform_random/rand' + ) check_type(min, 'min', (float, int, Variable), 'uniform_random/rand') check_type(max, 'max', (float, int, Variable), 'uniform_random/rand') inputs = dict() attrs = {'seed': seed, 'min': min, 'max': max, 'dtype': dtype} - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type='uniform_random/rand') + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type='uniform_random/rand' + ) helper = LayerHelper("uniform_random", **locals()) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="uniform_random", - inputs=inputs, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="uniform_random", inputs=inputs, attrs=attrs, outputs={"Out": out} + ) utils.try_set_static_shape_tensor(out, shape) return out @@ -15884,11 +16647,13 @@ def unbind(input, axis=0): helper = LayerHelper("unbind", **locals()) check_type(input, 'input', (Variable), 'unbind') dtype = helper.input_dtype() - check_dtype(dtype, 'unbind', ['float32', 'float64', 'int32', 'int64'], - 'unbind') + check_dtype( + dtype, 'unbind', ['float32', 'float64', 'int32', 'int64'], 'unbind' + ) if not isinstance(axis, (int)): - raise TypeError("The type of 'axis' must be int, but received %s." % - (type(axis))) + raise TypeError( + "The type of 'axis' must be int, but received %s." % (type(axis)) + ) if isinstance(axis, np.generic): axis = np.asscalar(axis) input_shape = input.shape @@ -15899,8 +16664,10 @@ def unbind(input, axis=0): for i in range(num) ] - helper.append_op(type="unbind", - inputs={"X": input}, - outputs={"Out": outs}, - attrs={"axis": axis}) + helper.append_op( + type="unbind", + inputs={"X": input}, + outputs={"Out": outs}, + attrs={"axis": axis}, + ) return outs diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index b1b4a1a55f9bc..61e12a98a0127 100755 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -13,17 +13,27 @@ # limitations under the License. import os -from .layer_function_generator import generate_layer_fn, generate_activation_fn, generate_inplace_fn, add_sample_code +from .layer_function_generator import ( + generate_layer_fn, + generate_activation_fn, + generate_inplace_fn, + add_sample_code, +) from .. import core from ..framework import convert_np_dtype_to_dtype_, Variable, in_dygraph_mode -from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) from paddle.utils import deprecated from paddle import _C_ops, _legacy_C_ops import paddle __deprecated_func_name__ = { 'tanh_shrink': 'tanhshrink', - 'logsigmoid': 'log_sigmoid' + 'logsigmoid': 'log_sigmoid', } __activations_noattr__ = [ @@ -36,9 +46,28 @@ ] __unary_func__ = [ - 'exp', 'expm1', 'atan', 'sqrt', 'rsqrt', 'abs', 'ceil', 'floor', 'cos', - 'tan', 'acos', 'sin', 'sinh', 'asin', 'cosh', 'round', 'reciprocal', - 'square', 'acosh', 'asinh', 'atanh', 'lgamma' + 'exp', + 'expm1', + 'atan', + 'sqrt', + 'rsqrt', + 'abs', + 'ceil', + 'floor', + 'cos', + 'tan', + 'acos', + 'sin', + 'sinh', + 'asin', + 'cosh', + 'round', + 'reciprocal', + 'square', + 'acosh', + 'asinh', + 'atanh', + 'lgamma', ] __inplace_unary_func__ = [ @@ -80,8 +109,9 @@ if _OP in __deprecated_func_name__: _new_OP = __deprecated_func_name__[_OP] _func = generate_activation_fn(_OP) - _func = deprecated(since="2.0.0", - update_to="paddle.nn.functional.%s" % (_new_OP))(_func) + _func = deprecated( + since="2.0.0", update_to="paddle.nn.functional.%s" % (_new_OP) + )(_func) globals()[_OP] = _func for _OP in set(__unary_func__): @@ -101,7 +131,8 @@ globals()[_OP] = _func add_sample_code( - globals()["sigmoid"], r""" + globals()["sigmoid"], + r""" Examples: .. code-block:: python @@ -113,10 +144,12 @@ print(out) # [0.40131234 0.450166 0.52497919 0.57444252] -""") +""", +) add_sample_code( - globals()["silu"], r""" + globals()["silu"], + r""" Examples: .. code-block:: python @@ -128,10 +161,12 @@ print(out) # [ 0.7310586 1.7615942 2.8577224, 3.9280552 ] -""") +""", +) add_sample_code( - globals()["logsigmoid"], r""" + globals()["logsigmoid"], + r""" Examples: .. code-block:: python @@ -143,10 +178,12 @@ print(out) # [-0.91301525 -0.79813887 -0.64439666 -0.55435524] -""") +""", +) add_sample_code( - globals()["exp"], r""" + globals()["exp"], + r""" Examples: .. code-block:: python @@ -157,10 +194,12 @@ print(out) # [0.67032005 0.81873075 1.10517092 1.34985881] -""") +""", +) add_sample_code( - globals()["expm1"], r""" + globals()["expm1"], + r""" Examples: .. code-block:: python @@ -171,10 +210,12 @@ print(out) # [-0.32967997, -0.18126924, 0.10517092, 0.34985882] -""") +""", +) add_sample_code( - globals()["tanh"], r""" + globals()["tanh"], + r""" Examples: .. code-block:: python @@ -185,10 +226,12 @@ print(out) # [-0.37994896 -0.19737532 0.09966799 0.29131261] -""") +""", +) add_sample_code( - globals()["atan"], r""" + globals()["atan"], + r""" Examples: .. code-block:: python @@ -199,10 +242,12 @@ print(out) # [-0.38050638 -0.19739556 0.09966865 0.29145679] -""") +""", +) add_sample_code( - globals()["tanh_shrink"], r""" + globals()["tanh_shrink"], + r""" Examples: .. code-block:: python @@ -214,10 +259,12 @@ print(out) # [-0.020051, -0.00262468, 0.000332005, 0.00868739] -""") +""", +) add_sample_code( - globals()["sqrt"], r""" + globals()["sqrt"], + r""" Examples: .. code-block:: python @@ -228,10 +275,12 @@ print(out) # [0.31622777 0.4472136 0.54772256 0.63245553] -""") +""", +) add_sample_code( - globals()["rsqrt"], r""" + globals()["rsqrt"], + r""" Examples: .. code-block:: python @@ -242,10 +291,12 @@ print(out) # [3.16227766 2.23606798 1.82574186 1.58113883] -""") +""", +) add_sample_code( - globals()["abs"], r""" + globals()["abs"], + r""" Examples: .. code-block:: python @@ -256,10 +307,12 @@ print(out) # [0.4 0.2 0.1 0.3] -""") +""", +) add_sample_code( - globals()["ceil"], r""" + globals()["ceil"], + r""" Examples: .. code-block:: python @@ -270,10 +323,12 @@ print(out) # [-0. -0. 1. 1.] -""") +""", +) add_sample_code( - globals()["floor"], r""" + globals()["floor"], + r""" Examples: .. code-block:: python @@ -284,10 +339,12 @@ print(out) # [-1. -1. 0. 0.] -""") +""", +) add_sample_code( - globals()["cos"], r""" + globals()["cos"], + r""" Examples: .. code-block:: python @@ -298,10 +355,12 @@ print(out) # [0.92106099 0.98006658 0.99500417 0.95533649] -""") +""", +) add_sample_code( - globals()["tan"], r""" + globals()["tan"], + r""" Examples: .. code-block:: python @@ -312,10 +371,12 @@ print(out) # [-0.42279324, -0.20271005, 0.10033467, 0.30933627] -""") +""", +) add_sample_code( - globals()["acos"], r""" + globals()["acos"], + r""" Examples: .. code-block:: python @@ -326,10 +387,12 @@ print(out) # [1.98231317 1.77215425 1.47062891 1.26610367] -""") +""", +) add_sample_code( - globals()["sin"], r""" + globals()["sin"], + r""" Examples: .. code-block:: python @@ -340,10 +403,12 @@ print(out) # [-0.38941834 -0.19866933 0.09983342 0.29552021] -""") +""", +) add_sample_code( - globals()["asin"], r""" + globals()["asin"], + r""" Examples: .. code-block:: python @@ -354,10 +419,12 @@ print(out) # [-0.41151685 -0.20135792 0.10016742 0.30469265] -""") +""", +) add_sample_code( - globals()["cosh"], r""" + globals()["cosh"], + r""" Examples: .. code-block:: python @@ -368,10 +435,12 @@ print(out) # [1.08107237 1.02006676 1.00500417 1.04533851] -""") +""", +) add_sample_code( - globals()["sinh"], r""" + globals()["sinh"], + r""" Examples: .. code-block:: python @@ -382,10 +451,12 @@ print(out) # [-0.41075233 -0.201336 0.10016675 0.30452029] -""") +""", +) add_sample_code( - globals()["asinh"], r""" + globals()["asinh"], + r""" Examples: .. code-block:: python @@ -396,10 +467,12 @@ print(out) # [-0.39003533, -0.19869010, 0.09983408, 0.29567307] -""") +""", +) add_sample_code( - globals()["acosh"], r""" + globals()["acosh"], + r""" Examples: .. code-block:: python @@ -410,10 +483,12 @@ print(out) # [0. , 1.76274729, 2.06343699, 2.29243159] -""") +""", +) add_sample_code( - globals()["atanh"], r""" + globals()["atanh"], + r""" Examples: .. code-block:: python @@ -424,10 +499,12 @@ print(out) # [-0.42364895, -0.20273256, 0.10033535, 0.30951962] -""") +""", +) add_sample_code( - globals()["round"], r""" + globals()["round"], + r""" Examples: .. code-block:: python @@ -438,10 +515,12 @@ print(out) # [-1. -0. 1. 2.] -""") +""", +) add_sample_code( - globals()["reciprocal"], r""" + globals()["reciprocal"], + r""" Examples: .. code-block:: python @@ -452,10 +531,12 @@ print(out) # [-2.5 -5. 10. 3.33333333] -""") +""", +) add_sample_code( - globals()["square"], r""" + globals()["square"], + r""" Examples: .. code-block:: python @@ -466,7 +547,8 @@ print(out) # [0.16 0.04 0.01 0.09] -""") +""", +) _softplus_ = generate_layer_fn('softplus') @@ -515,7 +597,8 @@ def softplus(x, beta: float = 1.0, threshold: float = 20.0, name=None): """ add_sample_code( - globals()["softsign"], r""" + globals()["softsign"], + r""" Examples: .. code-block:: python @@ -527,14 +610,16 @@ def softplus(x, beta: float = 1.0, threshold: float = 20.0, name=None): print(out) # [-0.285714, -0.166667, 0.0909091, 0.230769] -""") +""", +) _softshrink_ = generate_layer_fn('softshrink') def softshrink(x, alpha=None): - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'softshrink') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'softshrink' + ) locals_var = locals().copy() kwargs = dict() @@ -582,8 +667,9 @@ def softshrink(x, alpha=None): @deprecated(since="2.0.0", update_to="paddle.nn.functional.hardshrink") def hard_shrink(x, threshold=None): - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'hard_shrink') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'hard_shrink' + ) locals_var = locals().copy() kwargs = dict() @@ -593,20 +679,25 @@ def hard_shrink(x, threshold=None): return _hard_shrink_(**kwargs) -hard_shrink.__doc__ = _hard_shrink_.__doc__ + """ +hard_shrink.__doc__ = ( + _hard_shrink_.__doc__ + + """ Examples: >>> import paddle.fluid as fluid >>> data = fluid.layers.data(name="input", shape=[784]) >>> result = fluid.layers.hard_shrink(x=data, threshold=0.3) """ +) _cum_sum_ = generate_layer_fn('cumsum') -@deprecated(since="2.0.0", - update_to="paddle.cumsum", - reason="New APIs for Paddle 2.0 are coming.") +@deprecated( + since="2.0.0", + update_to="paddle.cumsum", + reason="New APIs for Paddle 2.0 are coming.", +) def cumsum(x, axis=None, exclusive=None, reverse=None): check_type(x, 'x', (Variable), 'cumsum') locals_var = locals().copy() @@ -645,8 +736,9 @@ def cumsum(x, axis=None, exclusive=None, reverse=None): def thresholded_relu(x, threshold=None): - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'thresholded_relu') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'thresholded_relu' + ) locals_var = locals().copy() kwargs = dict() diff --git a/python/paddle/fluid/layers/rnn.py b/python/paddle/fluid/layers/rnn.py index 087cffcfe3b32..f2ab7da4c84cb 100644 --- a/python/paddle/fluid/layers/rnn.py +++ b/python/paddle/fluid/layers/rnn.py @@ -31,6 +31,7 @@ from ..framework import _non_static_mode from ..param_attr import ParamAttr from ..data_feeder import check_variable_and_dtype, check_type, check_dtype + try: from collections.abc import Sequence except: @@ -63,7 +64,7 @@ class RNNCell(object): """ - :api_attr: Static Graph + :api_attr: Static Graph RNNCell is the base class for abstraction representing the calculations mapping the input and state to the output and new state. It is suitable to @@ -95,12 +96,14 @@ def call(self, inputs, states, **kwargs): def __call__(self, inputs, states, **kwargs): return self.call(inputs, states, **kwargs) - def get_initial_states(self, - batch_ref, - shape=None, - dtype='float32', - init_value=0, - batch_dim_idx=0): + def get_initial_states( + self, + batch_ref, + shape=None, + dtype='float32', + init_value=0, + batch_dim_idx=0, + ): r""" Generate initialized states according to provided shape, data type and value. @@ -126,9 +129,12 @@ def get_initial_states(self, Variable: tensor variable[s] packed in the same structure provided \ by shape, representing the initialized states. """ - check_variable_and_dtype(batch_ref, 'batch_ref', - ['float32', 'float64', 'int32', 'int64'], - 'RNNCell') + check_variable_and_dtype( + batch_ref, + 'batch_ref', + ['float32', 'float64', 'int32', 'int64'], + 'RNNCell', + ) check_type(shape, 'shape', (list, tuple, type(None), int), 'RNNCell') if isinstance(shape, (list, tuple)): shapes = map_structure(lambda x: x, shape) @@ -144,17 +150,17 @@ def get_initial_states(self, def _is_shape_sequence(seq): """For shape, list/tuple of integer is the finest-grained objection""" - if (isinstance(seq, list) or isinstance(seq, tuple)): - if reduce(lambda flag, x: isinstance(x, int) and flag, seq, - True): + if isinstance(seq, list) or isinstance(seq, tuple): + if reduce( + lambda flag, x: isinstance(x, int) and flag, seq, True + ): return False # TODO: Add check for the illegal if isinstance(seq, dict): return True - return (isinstance(seq, Sequence) and not isinstance(seq, str)) + return isinstance(seq, Sequence) and not isinstance(seq, str) class Shape(object): - def __init__(self, shape): self.shape = shape if shape[0] == -1 else ([-1] + list(shape)) @@ -180,7 +186,11 @@ def __init__(self, shape): shape=shape.shape, dtype=dtype, value=init_value, - input_dim_idx=batch_dim_idx), states_shapes, states_dtypes) + input_dim_idx=batch_dim_idx, + ), + states_shapes, + states_dtypes, + ) return init_states @property @@ -196,7 +206,8 @@ def state_shape(self): `get_initial_states`. """ raise NotImplementedError( - "Please add implementaion for `state_shape` in the used cell.") + "Please add implementaion for `state_shape` in the used cell." + ) @property def state_dtype(self): @@ -211,12 +222,13 @@ def state_dtype(self): `get_initial_states`. """ raise NotImplementedError( - "Please add implementaion for `state_dtype` in the used cell.") + "Please add implementaion for `state_dtype` in the used cell." + ) class GRUCell(RNNCell): r""" - :api_attr: Static Graph + :api_attr: Static Graph Gated Recurrent Unit cell. It is a wrapper for `fluid.contrib.layers.rnn_impl.BasicGRUUnit` to make it adapt to RNNCell. @@ -244,14 +256,16 @@ class GRUCell(RNNCell): cell = layers.GRUCell(hidden_size=256) """ - def __init__(self, - hidden_size, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - dtype="float32", - name="GRUCell"): + def __init__( + self, + hidden_size, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + dtype="float32", + name="GRUCell", + ): """ Constructor of GRUCell. @@ -272,9 +286,16 @@ def __init__(self, check_dtype(dtype, 'dtype', ['float32', 'float64'], 'GRUCell') self.hidden_size = hidden_size from .. import contrib # TODO: resolve recurrent import + self.gru_unit = contrib.layers.rnn_impl.BasicGRUUnit( - name, hidden_size, param_attr, bias_attr, gate_activation, - activation, dtype) + name, + hidden_size, + param_attr, + bias_attr, + gate_activation, + activation, + dtype, + ) def call(self, inputs, states): r""" @@ -295,10 +316,12 @@ def call(self, inputs, states): tensor is same as that of `states`. """ - check_variable_and_dtype(inputs, 'inputs', ['float32', 'float64'], - 'GRUCell') - check_variable_and_dtype(states, 'states', ['float32', 'float64'], - 'GRUCell') + check_variable_and_dtype( + inputs, 'inputs', ['float32', 'float64'], 'GRUCell' + ) + check_variable_and_dtype( + states, 'states', ['float32', 'float64'], 'GRUCell' + ) new_hidden = self.gru_unit(inputs, states) return new_hidden, new_hidden @@ -314,7 +337,7 @@ def state_shape(self): class LSTMCell(RNNCell): r""" - :api_attr: Static Graph + :api_attr: Static Graph Long-Short Term Memory cell. It is a wrapper for `fluid.contrib.layers.rnn_impl.BasicLSTMUnit` to make it adapt to RNNCell. @@ -343,15 +366,17 @@ class LSTMCell(RNNCell): cell = layers.LSTMCell(hidden_size=256) """ - def __init__(self, - hidden_size, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - forget_bias=1.0, - dtype="float32", - name="LSTMCell"): + def __init__( + self, + hidden_size, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + forget_bias=1.0, + dtype="float32", + name="LSTMCell", + ): """ Constructor of LSTMCell. @@ -375,9 +400,17 @@ def __init__(self, check_dtype(dtype, 'dtype', ['float32', 'float64'], 'LSTMCell') self.hidden_size = hidden_size from .. import contrib # TODO: resolve recurrent import + self.lstm_unit = contrib.layers.rnn_impl.BasicLSTMUnit( - name, hidden_size, param_attr, bias_attr, gate_activation, - activation, forget_bias, dtype) + name, + hidden_size, + param_attr, + bias_attr, + gate_activation, + activation, + forget_bias, + dtype, + ) def call(self, inputs, states): r""" @@ -400,13 +433,18 @@ def call(self, inputs, states): tensors all is same as that of `states`. """ - check_variable_and_dtype(inputs, 'inputs', ['float32', 'float64'], - 'LSTMCell') + check_variable_and_dtype( + inputs, 'inputs', ['float32', 'float64'], 'LSTMCell' + ) check_type(states, 'states', list, 'LSTMCell') if isinstance(states, list): for i, state in enumerate(states): - check_variable_and_dtype(state, 'state[' + str(i) + ']', - ['float32', 'float64'], 'LSTMCell') + check_variable_and_dtype( + state, + 'state[' + str(i) + ']', + ['float32', 'float64'], + 'LSTMCell', + ) pre_hidden, pre_cell = states new_hidden, new_cell = self.lstm_unit(inputs, pre_hidden, pre_cell) @@ -422,13 +460,15 @@ def state_shape(self): return [[self.hidden_size], [self.hidden_size]] -def rnn(cell, - inputs, - initial_states=None, - sequence_length=None, - time_major=False, - is_reverse=False, - **kwargs): +def rnn( + cell, + inputs, + initial_states=None, + sequence_length=None, + time_major=False, + is_reverse=False, + **kwargs +): """ rnn creates a recurrent neural network specified by RNNCell `cell`, which performs :code:`cell.call()` (for dygraph mode :code:`cell.forward`) @@ -483,15 +523,28 @@ def rnn(cell, """ if _non_static_mode(): - return _rnn_dynamic_graph(cell, inputs, initial_states, sequence_length, - time_major, is_reverse, **kwargs) + return _rnn_dynamic_graph( + cell, + inputs, + initial_states, + sequence_length, + time_major, + is_reverse, + **kwargs + ) else: - return _rnn_static_graph(cell, inputs, initial_states, sequence_length, - time_major, is_reverse, **kwargs) + return _rnn_static_graph( + cell, + inputs, + initial_states, + sequence_length, + time_major, + is_reverse, + **kwargs + ) class ArrayWrapper(object): - def __init__(self, x): self.array = [x] @@ -505,8 +558,9 @@ def __getitem__(self, item): def _maybe_copy(state, new_state, step_mask): """update rnn state or just pass the old state through""" - new_state = nn.elementwise_mul(new_state, step_mask, axis=0) \ - + nn.elementwise_mul(state, (1 - step_mask), axis=0) + new_state = nn.elementwise_mul( + new_state, step_mask, axis=0 + ) + nn.elementwise_mul(state, (1 - step_mask), axis=0) return new_state @@ -515,34 +569,40 @@ def _transpose_batch_time(x): return nn.transpose(x, perm) -def _rnn_dynamic_graph(cell, - inputs, - initial_states=None, - sequence_length=None, - time_major=False, - is_reverse=False, - **kwargs): +def _rnn_dynamic_graph( + cell, + inputs, + initial_states=None, + sequence_length=None, + time_major=False, + is_reverse=False, + **kwargs +): time_step_index = 0 if time_major else 1 flat_inputs = flatten(inputs) time_steps = flat_inputs[0].shape[time_step_index] if initial_states is None: initial_states = cell.get_initial_states( - batch_ref=inputs, batch_dim_idx=1 if time_major else 0) + batch_ref=inputs, batch_dim_idx=1 if time_major else 0 + ) if not time_major: inputs = map_structure(_transpose_batch_time, inputs) if sequence_length is not None: - mask = sequence_lod.sequence_mask(sequence_length, - maxlen=time_steps, - dtype=inputs.dtype) + mask = sequence_lod.sequence_mask( + sequence_length, maxlen=time_steps, dtype=inputs.dtype + ) mask = nn.transpose(mask, [1, 0]) if is_reverse: inputs = map_structure(lambda x: tensor.reverse(x, axis=[0]), inputs) - mask = tensor.reverse(mask, axis=[0]) \ - if sequence_length is not None else None + mask = ( + tensor.reverse(mask, axis=[0]) + if sequence_length is not None + else None + ) states = initial_states outputs = [] @@ -550,42 +610,56 @@ def _rnn_dynamic_graph(cell, step_inputs = map_structure(lambda x: x[i], inputs) step_outputs, new_states = cell(step_inputs, states, **kwargs) if sequence_length is not None: - new_states = map_structure(partial(_maybe_copy, step_mask=mask[i]), - states, new_states) + new_states = map_structure( + partial(_maybe_copy, step_mask=mask[i]), states, new_states + ) states = new_states - outputs = map_structure(lambda x: ArrayWrapper(x), - step_outputs) if i == 0 else map_structure( - lambda x, x_array: x_array.append(x), - step_outputs, outputs) + outputs = ( + map_structure(lambda x: ArrayWrapper(x), step_outputs) + if i == 0 + else map_structure( + lambda x, x_array: x_array.append(x), step_outputs, outputs + ) + ) final_outputs = map_structure( - lambda x: nn.stack(x.array, axis=time_step_index), outputs) + lambda x: nn.stack(x.array, axis=time_step_index), outputs + ) if is_reverse: final_outputs = map_structure( - lambda x: tensor.reverse(x, axis=time_step_index), final_outputs) + lambda x: tensor.reverse(x, axis=time_step_index), final_outputs + ) final_states = new_states return final_outputs, final_states -def _rnn_static_graph(cell, - inputs, - initial_states=None, - sequence_length=None, - time_major=False, - is_reverse=False, - **kwargs): +def _rnn_static_graph( + cell, + inputs, + initial_states=None, + sequence_length=None, + time_major=False, + is_reverse=False, + **kwargs +): check_type(inputs, 'inputs', (Variable, list, tuple), 'rnn') if isinstance(inputs, (list, tuple)): for i, input_x in enumerate(inputs): - check_variable_and_dtype(input_x, 'inputs[' + str(i) + ']', - ['float32', 'float64'], 'rnn') - check_type(initial_states, 'initial_states', - (Variable, list, tuple, type(None)), 'rnn') + check_variable_and_dtype( + input_x, 'inputs[' + str(i) + ']', ['float32', 'float64'], 'rnn' + ) + check_type( + initial_states, + 'initial_states', + (Variable, list, tuple, type(None)), + 'rnn', + ) - check_type(sequence_length, 'sequence_length', (Variable, type(None)), - 'rnn') + check_type( + sequence_length, 'sequence_length', (Variable, type(None)), 'rnn' + ) def _switch_grad(x, stop=False): x.stop_gradient = stop @@ -593,7 +667,8 @@ def _switch_grad(x, stop=False): if initial_states is None: initial_states = cell.get_initial_states( - batch_ref=inputs, batch_dim_idx=1 if time_major else 0) + batch_ref=inputs, batch_dim_idx=1 if time_major else 0 + ) initial_states = map_structure(_switch_grad, initial_states) if not time_major: @@ -604,7 +679,8 @@ def _switch_grad(x, stop=False): mask = sequence_lod.sequence_mask( sequence_length, maxlen=max_seq_len, - dtype=flatten(initial_states)[0].dtype) + dtype=flatten(initial_states)[0].dtype, + ) mask = nn.transpose(mask, [1, 0]) if is_reverse: inputs = map_structure(lambda x: tensor.reverse(x, axis=[0]), inputs) @@ -621,7 +697,8 @@ def _switch_grad(x, stop=False): if sequence_length: step_mask = rnn.step_input(mask) new_states = map_structure( - partial(_maybe_copy, step_mask=step_mask), states, new_states) + partial(_maybe_copy, step_mask=step_mask), states, new_states + ) map_structure(rnn.update_memory, states, new_states) flat_outputs = flatten(outputs) @@ -629,14 +706,15 @@ def _switch_grad(x, stop=False): map_structure(rnn.step_output, new_states) rnn_out = rnn() - final_outputs = rnn_out[:len(flat_outputs)] + final_outputs = rnn_out[: len(flat_outputs)] final_outputs = pack_sequence_as(outputs, final_outputs) - final_states = map_structure(lambda x: x[-1], rnn_out[len(flat_outputs):]) + final_states = map_structure(lambda x: x[-1], rnn_out[len(flat_outputs) :]) final_states = pack_sequence_as(new_states, final_states) if is_reverse: - final_outputs = map_structure(lambda x: tensor.reverse(x, axis=[0]), - final_outputs) + final_outputs = map_structure( + lambda x: tensor.reverse(x, axis=[0]), final_outputs + ) if not time_major: final_outputs = map_structure(_transpose_batch_time, final_outputs) @@ -644,13 +722,15 @@ def _switch_grad(x, stop=False): return (final_outputs, final_states) -def birnn(cell_fw, - cell_bw, - inputs, - initial_states=None, - sequence_length=None, - time_major=False, - **kwargs): +def birnn( + cell_fw, + cell_bw, + inputs, + initial_states=None, + sequence_length=None, + time_major=False, + **kwargs +): """ birnn creates a bidirectional recurrent neural network specified by RNNCell `cell_fw` and `cell_bw`, which performs :code:`cell.call()` @@ -709,28 +789,35 @@ def birnn(cell_fw, """ if initial_states is None: states_fw = cell_fw.get_initial_states( - batch_ref=inputs, batch_dim_idx=1 if time_major else 0) + batch_ref=inputs, batch_dim_idx=1 if time_major else 0 + ) states_bw = cell_fw.get_initial_states( - batch_ref=inputs, batch_dim_idx=1 if time_major else 0) + batch_ref=inputs, batch_dim_idx=1 if time_major else 0 + ) else: states_fw, states_bw = initial_states - outputs_fw, states_fw = rnn(cell_fw, - inputs, - states_fw, - sequence_length, - time_major=time_major, - **kwargs) - - outputs_bw, states_bw = rnn(cell_bw, - inputs, - states_bw, - sequence_length, - time_major=time_major, - is_reverse=True, - **kwargs) - - outputs = map_structure(lambda x, y: tensor.concat([x, y], -1), outputs_fw, - outputs_bw) + outputs_fw, states_fw = rnn( + cell_fw, + inputs, + states_fw, + sequence_length, + time_major=time_major, + **kwargs + ) + + outputs_bw, states_bw = rnn( + cell_bw, + inputs, + states_bw, + sequence_length, + time_major=time_major, + is_reverse=True, + **kwargs + ) + + outputs = map_structure( + lambda x, y: tensor.concat([x, y], -1), outputs_fw, outputs_bw + ) final_states = (states_fw, states_bw) return outputs, final_states @@ -738,7 +825,7 @@ def birnn(cell_fw, class Decoder(object): """ - :api_attr: Static Graph + :api_attr: Static Graph Decoder is the base class for any decoder instance used in `dynamic_decode`. It provides interface for output generation for one time step, which can be @@ -890,13 +977,15 @@ class BeamSearchDecoder(Decoder): """ - def __init__(self, - cell, - start_token, - end_token, - beam_size, - embedding_fn=None, - output_fn=None): + def __init__( + self, + cell, + start_token, + end_token, + beam_size, + embedding_fn=None, + output_fn=None, + ): """ Constructor of BeamSearchDecoder. @@ -939,22 +1028,23 @@ def tile_beam_merge_with_batch(x, beam_size): Variable: A tensor with shape `[batch_size * beam_size, ...]`, whose \ data type is same as `x`. """ - check_type(x, 'x', (Variable), - 'BeamSearchDecoder.tile_beam_merge_with_batch') + check_type( + x, 'x', (Variable), 'BeamSearchDecoder.tile_beam_merge_with_batch' + ) x = nn.unsqueeze(x, [1]) # [batch_size, 1, ...] expand_times = [1] * len(x.shape) expand_times[1] = beam_size x = paddle.tile(x, expand_times) # [batch_size, beam_size, ...] - x = nn.transpose(x, - list(range(2, len(x.shape))) + - [0, 1]) # [..., batch_size, beam_size] + x = nn.transpose( + x, list(range(2, len(x.shape))) + [0, 1] + ) # [..., batch_size, beam_size] # use 0 to copy to avoid wrong shape - x = nn.reshape(x, shape=[0] * (len(x.shape) - 2) + - [-1]) # [..., batch_size * beam_size] + x = nn.reshape( + x, shape=[0] * (len(x.shape) - 2) + [-1] + ) # [..., batch_size * beam_size] x = nn.transpose( - x, [len(x.shape) - 1] + - list(range(0, - len(x.shape) - 1))) # [batch_size * beam_size, ...] + x, [len(x.shape) - 1] + list(range(0, len(x.shape) - 1)) + ) # [batch_size * beam_size, ...] return x def _split_batch_beams(self, x): @@ -1032,14 +1122,16 @@ def _mask_probs(self, probs, finished): replaced with a tensor with all probability on the EOS token. """ check_type(probs, 'probs', (Variable), 'BeamSearchDecoder._mask_probs') - check_type(finished, 'finished', (Variable), - 'BeamSearchDecoder._mask_probs') + check_type( + finished, 'finished', (Variable), 'BeamSearchDecoder._mask_probs' + ) # TODO: use where_op finished = tensor.cast(finished, dtype=probs.dtype) probs = nn.elementwise_mul( paddle.tile(nn.unsqueeze(finished, [2]), [1, 1, self.vocab_size]), self.noend_mask_tensor, - axis=-1) - nn.elementwise_mul(probs, (finished - 1), axis=0) + axis=-1, + ) - nn.elementwise_mul(probs, (finished - 1), axis=0) return probs def _gather(self, x, indices, batch_size): @@ -1059,37 +1151,48 @@ def _gather(self, x, indices, batch_size): """ check_type(x, 'x', (Variable), 'BeamSearchDecoder._gather') check_type(indices, 'indices', (Variable), 'BeamSearchDecoder._gather') - check_type(batch_size, 'batch_size', (Variable), - 'BeamSearchDecoder._gather') + check_type( + batch_size, 'batch_size', (Variable), 'BeamSearchDecoder._gather' + ) # TODO: compatibility of int32 and int64 - batch_size = tensor.cast( - batch_size, - indices.dtype) if batch_size.dtype != indices.dtype else batch_size + batch_size = ( + tensor.cast(batch_size, indices.dtype) + if batch_size.dtype != indices.dtype + else batch_size + ) batch_size.stop_gradient = True # TODO: remove this batch_pos = paddle.tile( - nn.unsqueeze(tensor.range(0, batch_size, 1, dtype=indices.dtype), - [1]), [1, self.beam_size]) + nn.unsqueeze( + tensor.range(0, batch_size, 1, dtype=indices.dtype), [1] + ), + [1, self.beam_size], + ) topk_coordinates = nn.stack([batch_pos, indices], axis=2) topk_coordinates.stop_gradient = True return nn.gather_nd(x, topk_coordinates) class OutputWrapper( - collections.namedtuple("OutputWrapper", - ("scores", "predicted_ids", "parent_ids"))): + collections.namedtuple( + "OutputWrapper", ("scores", "predicted_ids", "parent_ids") + ) + ): """ The structure for the returned value `outputs` of `decoder.step`. A namedtuple includes scores, predicted_ids, parent_ids as fields. """ + pass class StateWrapper( - collections.namedtuple( - "StateWrapper", - ("cell_states", "log_probs", "finished", "lengths"))): + collections.namedtuple( + "StateWrapper", ("cell_states", "log_probs", "finished", "lengths") + ) + ): """ The structure for the argument `states` of `decoder.step`. A namedtuple includes cell_states, log_probs, finished, lengths as fields. """ + pass def initialize(self, initial_cell_states): @@ -1117,22 +1220,30 @@ def initialize(self, initial_cell_states): state = flatten(initial_cell_states)[0] self.batch_size = nn.shape(state)[0] - self.start_token_tensor = tensor.fill_constant(shape=[1], - dtype="int64", - value=self.start_token) - self.end_token_tensor = tensor.fill_constant(shape=[1], - dtype="int64", - value=self.end_token) - - init_cell_states = map_structure(self._expand_to_beam_size, - initial_cell_states) - init_inputs = paddle.full(shape=[self.batch_size, self.beam_size], - fill_value=self.start_token_tensor, - dtype=self.start_token_tensor.dtype) + self.start_token_tensor = tensor.fill_constant( + shape=[1], dtype="int64", value=self.start_token + ) + self.end_token_tensor = tensor.fill_constant( + shape=[1], dtype="int64", value=self.end_token + ) + + init_cell_states = map_structure( + self._expand_to_beam_size, initial_cell_states + ) + init_inputs = paddle.full( + shape=[self.batch_size, self.beam_size], + fill_value=self.start_token_tensor, + dtype=self.start_token_tensor.dtype, + ) log_probs = paddle.tile( tensor.assign( - np.array([[0.] + [-self.kinf] * (self.beam_size - 1)], - dtype="float32")), [self.batch_size, 1]) + np.array( + [[0.0] + [-self.kinf] * (self.beam_size - 1)], + dtype="float32", + ) + ), + [self.batch_size, 1], + ) if paddle.get_default_dtype() == "float64": log_probs = tensor.cast(log_probs, "float64") # TODO: remove the restriction of force_cpu @@ -1141,13 +1252,19 @@ def initialize(self, initial_cell_states): shape=[-1, self.beam_size], dtype="bool", value=False, - force_cpu=True) + force_cpu=True, + ) init_lengths = tensor.zeros_like(init_inputs) - init_inputs = self.embedding_fn( - init_inputs) if self.embedding_fn else init_inputs - return init_inputs, self.StateWrapper(init_cell_states, log_probs, - init_finished, - init_lengths), init_finished + init_inputs = ( + self.embedding_fn(init_inputs) if self.embedding_fn else init_inputs + ) + return ( + init_inputs, + self.StateWrapper( + init_cell_states, log_probs, init_finished, init_lengths + ), + init_finished, + ) def _beam_search_step(self, time, logits, next_cell_states, beam_state): r""" @@ -1178,50 +1295,61 @@ def _beam_search_step(self, time, logits, next_cell_states, beam_state): """ self.vocab_size = logits.shape[-1] - self.vocab_size_tensor = tensor.fill_constant(shape=[1], - dtype="int64", - value=self.vocab_size) + self.vocab_size_tensor = tensor.fill_constant( + shape=[1], dtype="int64", value=self.vocab_size + ) noend_array = [-self.kinf] * self.vocab_size noend_array[self.end_token] = 0 self.noend_mask_tensor = tensor.assign(np.array(noend_array, "float32")) if paddle.get_default_dtype() == "float64": - self.noend_mask_tensor = tensor.cast(self.noend_mask_tensor, - "float64") + self.noend_mask_tensor = tensor.cast( + self.noend_mask_tensor, "float64" + ) step_log_probs = nn.log(nn.softmax(logits)) step_log_probs = self._mask_probs(step_log_probs, beam_state.finished) - log_probs = nn.elementwise_add(x=step_log_probs, - y=beam_state.log_probs, - axis=0) + log_probs = nn.elementwise_add( + x=step_log_probs, y=beam_state.log_probs, axis=0 + ) # TODO: length penalty scores = log_probs scores = nn.reshape(scores, [-1, self.beam_size * self.vocab_size]) # TODO: add grad for topk then this beam search can be used to train topk_scores, topk_indices = paddle.topk(x=scores, k=self.beam_size) - beam_indices = nn.elementwise_floordiv(topk_indices, - self.vocab_size_tensor) + beam_indices = nn.elementwise_floordiv( + topk_indices, self.vocab_size_tensor + ) token_indices = nn.elementwise_mod(topk_indices, self.vocab_size_tensor) next_log_probs = self._gather( nn.reshape(log_probs, [-1, self.beam_size * self.vocab_size]), - topk_indices, self.batch_size) + topk_indices, + self.batch_size, + ) next_cell_states = map_structure( lambda x: self._gather(x, beam_indices, self.batch_size), - next_cell_states) - next_finished = self._gather(beam_state.finished, beam_indices, - self.batch_size) - next_lengths = self._gather(beam_state.lengths, beam_indices, - self.batch_size) - next_lengths = next_lengths + tensor.cast(nn.logical_not(next_finished), - beam_state.lengths.dtype) + next_cell_states, + ) + next_finished = self._gather( + beam_state.finished, beam_indices, self.batch_size + ) + next_lengths = self._gather( + beam_state.lengths, beam_indices, self.batch_size + ) + next_lengths = next_lengths + tensor.cast( + nn.logical_not(next_finished), beam_state.lengths.dtype + ) next_finished = control_flow.logical_or( next_finished, - control_flow.equal(token_indices, self.end_token_tensor)) - - beam_search_output = self.OutputWrapper(topk_scores, token_indices, - beam_indices) - beam_search_state = self.StateWrapper(next_cell_states, next_log_probs, - next_finished, next_lengths) + control_flow.equal(token_indices, self.end_token_tensor), + ) + + beam_search_output = self.OutputWrapper( + topk_scores, token_indices, beam_indices + ) + beam_search_state = self.StateWrapper( + next_cell_states, next_log_probs, next_finished, next_lengths + ) return beam_search_output, beam_search_state def step(self, time, inputs, states, **kwargs): @@ -1254,11 +1382,13 @@ def step(self, time, inputs, states, **kwargs): """ inputs = map_structure(self._merge_batch_beams, inputs) cell_states = map_structure(self._merge_batch_beams, states.cell_states) - cell_outputs, next_cell_states = self.cell(inputs, cell_states, - **kwargs) + cell_outputs, next_cell_states = self.cell( + inputs, cell_states, **kwargs + ) cell_outputs = map_structure(self._split_batch_beams, cell_outputs) - next_cell_states = map_structure(self._split_batch_beams, - next_cell_states) + next_cell_states = map_structure( + self._split_batch_beams, next_cell_states + ) if self.output_fn is not None: cell_outputs = self.output_fn(cell_outputs) @@ -1267,12 +1397,14 @@ def step(self, time, inputs, states, **kwargs): time=time, logits=cell_outputs, next_cell_states=next_cell_states, - beam_state=states) + beam_state=states, + ) finished = beam_search_state.finished sample_ids = beam_search_output.predicted_ids sample_ids.stop_gradient = True - next_inputs = self.embedding_fn( - sample_ids) if self.embedding_fn else sample_ids + next_inputs = ( + self.embedding_fn(sample_ids) if self.embedding_fn else sample_ids + ) return (beam_search_output, beam_search_state, next_inputs, finished) @@ -1300,8 +1432,9 @@ def finalize(self, outputs, final_states, sequence_lengths): `[time_step, batch_size, beam_size]`. `final_states` is the same \ as the input argument `final_states`. """ - predicted_ids = nn.gather_tree(outputs.predicted_ids, - outputs.parent_ids) + predicted_ids = nn.gather_tree( + outputs.predicted_ids, outputs.parent_ids + ) # TODO: use FinalBeamSearchDecoderOutput as output return predicted_ids, final_states @@ -1319,15 +1452,16 @@ def tracks_own_finished(self): return True -def _dynamic_decode_imperative(decoder, - inits=None, - max_step_num=None, - output_time_major=False, - impute_finished=False, - is_test=False, - return_length=False, - **kwargs): - +def _dynamic_decode_imperative( + decoder, + inits=None, + max_step_num=None, + output_time_major=False, + impute_finished=False, + is_test=False, + return_length=False, + **kwargs +): def _maybe_copy(state, new_state, step_mask): # TODO: use where_op state_dtype = state.dtype @@ -1340,28 +1474,30 @@ def _maybe_copy(state, new_state, step_mask): # to sum(bool) error. step_mask.stop_gradient = True new_state = nn.elementwise_mul( - state, step_mask, axis=0) - nn.elementwise_mul(new_state, - (step_mask - 1), - axis=0) + state, step_mask, axis=0 + ) - nn.elementwise_mul(new_state, (step_mask - 1), axis=0) if convert_dtype(state_dtype) in ["bool"]: new_state = tensor.cast(new_state, dtype=state_dtype) return new_state initial_inputs, initial_states, initial_finished = decoder.initialize(inits) - inputs, states, finished = (initial_inputs, initial_states, - initial_finished) + inputs, states, finished = ( + initial_inputs, + initial_states, + initial_finished, + ) cond = control_flow.logical_not((nn.reduce_all(initial_finished))) sequence_lengths = tensor.cast(tensor.zeros_like(initial_finished), "int64") outputs = None step_idx = 0 - step_idx_tensor = tensor.fill_constant(shape=[1], - dtype="int64", - value=step_idx) + step_idx_tensor = tensor.fill_constant( + shape=[1], dtype="int64", value=step_idx + ) while cond.numpy(): - (step_outputs, next_states, next_inputs, - next_finished) = decoder.step(step_idx_tensor, inputs, states, - **kwargs) + (step_outputs, next_states, next_inputs, next_finished) = decoder.step( + step_idx_tensor, inputs, states, **kwargs + ) if not decoder.tracks_own_finished: # BeamSearchDecoder would track it own finished, since # beams would be reordered and the finished status of each @@ -1373,26 +1509,37 @@ def _maybe_copy(state, new_state, step_mask): tensor.assign(next_finished, finished) next_sequence_lengths = nn.elementwise_add( sequence_lengths, - tensor.cast(control_flow.logical_not(finished), - sequence_lengths.dtype)) + tensor.cast( + control_flow.logical_not(finished), sequence_lengths.dtype + ), + ) if impute_finished: # rectify the states for the finished. next_states = map_structure( - lambda x, y: _maybe_copy(x, y, finished), states, - next_states) + lambda x, y: _maybe_copy(x, y, finished), + states, + next_states, + ) else: warnings.warn( "`next_states` has no `lengths` attribute, the returned `sequence_lengths` would be all zeros." ) if not hasattr(next_states, "lengths") else None - next_sequence_lengths = getattr(next_states, "lengths", - sequence_lengths) - - outputs = map_structure( - lambda x: ArrayWrapper(x), - step_outputs) if step_idx == 0 else map_structure( - lambda x, x_array: x_array.append(x), step_outputs, outputs) - inputs, states, finished, sequence_lengths = (next_inputs, next_states, - next_finished, - next_sequence_lengths) + next_sequence_lengths = getattr( + next_states, "lengths", sequence_lengths + ) + + outputs = ( + map_structure(lambda x: ArrayWrapper(x), step_outputs) + if step_idx == 0 + else map_structure( + lambda x, x_array: x_array.append(x), step_outputs, outputs + ) + ) + inputs, states, finished, sequence_lengths = ( + next_inputs, + next_states, + next_finished, + next_sequence_lengths, + ) control_flow.increment(x=step_idx_tensor, value=1.0, in_place=True) step_idx += 1 @@ -1405,42 +1552,49 @@ def _maybe_copy(state, new_state, step_mask): final_states = states try: - final_outputs, final_states = decoder.finalize(final_outputs, - final_states, - sequence_lengths) + final_outputs, final_states = decoder.finalize( + final_outputs, final_states, sequence_lengths + ) except NotImplementedError: pass if not output_time_major: final_outputs = map_structure( lambda x: nn.transpose(x, [1, 0] + list(range(2, len(x.shape)))), - final_outputs) + final_outputs, + ) - return (final_outputs, final_states, - sequence_lengths) if return_length else (final_outputs, - final_states) + return ( + (final_outputs, final_states, sequence_lengths) + if return_length + else (final_outputs, final_states) + ) -def _dynamic_decode_declarative(decoder, - inits=None, - max_step_num=None, - output_time_major=False, - impute_finished=False, - is_test=False, - return_length=False, - **kwargs): +def _dynamic_decode_declarative( + decoder, + inits=None, + max_step_num=None, + output_time_major=False, + impute_finished=False, + is_test=False, + return_length=False, + **kwargs +): initial_inputs, initial_states, initial_finished = decoder.initialize(inits) - global_inputs, global_states, global_finished = (initial_inputs, - initial_states, - initial_finished) + global_inputs, global_states, global_finished = ( + initial_inputs, + initial_states, + initial_finished, + ) global_finished.stop_gradient = True step_idx = tensor.fill_constant(shape=[1], dtype="int64", value=0) cond = control_flow.logical_not((nn.reduce_all(initial_finished))) if max_step_num is not None: - max_step_num = tensor.fill_constant(shape=[1], - dtype="int64", - value=max_step_num) + max_step_num = tensor.fill_constant( + shape=[1], dtype="int64", value=max_step_num + ) while_op = control_flow.While(cond, is_test=is_test) sequence_lengths = tensor.cast(tensor.zeros_like(initial_finished), "int64") @@ -1453,9 +1607,11 @@ def _dynamic_decode_declarative(decoder, else: # inputs and states of all steps must be saved for backward and training inputs_arrays = map_structure( - lambda x: control_flow.array_write(x, step_idx), initial_inputs) + lambda x: control_flow.array_write(x, step_idx), initial_inputs + ) states_arrays = map_structure( - lambda x: control_flow.array_write(x, step_idx), initial_states) + lambda x: control_flow.array_write(x, step_idx), initial_states + ) def _maybe_copy(state, new_state, step_mask): # TODO: use where_op @@ -1469,9 +1625,8 @@ def _maybe_copy(state, new_state, step_mask): # to sum(bool) error. step_mask.stop_gradient = True new_state = nn.elementwise_mul( - state, step_mask, axis=0) - nn.elementwise_mul(new_state, - (step_mask - 1), - axis=0) + state, step_mask, axis=0 + ) - nn.elementwise_mul(new_state, (step_mask - 1), axis=0) if convert_dtype(state_dtype) in ["bool"]: new_state = tensor.cast(new_state, dtype=state_dtype) return new_state @@ -1481,8 +1636,9 @@ def _transpose_batch_time(x): def _create_array_out_of_while(dtype): current_block_idx = default_main_program().current_block_idx - default_main_program().current_block_idx = default_main_program( - ).current_block().parent_idx + default_main_program().current_block_idx = ( + default_main_program().current_block().parent_idx + ) tensor_array = control_flow.create_array(dtype) default_main_program().current_block_idx = current_block_idx return tensor_array @@ -1492,23 +1648,30 @@ def _create_array_out_of_while(dtype): if not is_test: inputs = map_structure( lambda array: control_flow.array_read(array, step_idx), - inputs_arrays) + inputs_arrays, + ) states = map_structure( lambda array: control_flow.array_read(array, step_idx), - states_arrays) - (outputs, next_states, next_inputs, - next_finished) = decoder.step(step_idx, inputs, states, **kwargs) + states_arrays, + ) + (outputs, next_states, next_inputs, next_finished) = decoder.step( + step_idx, inputs, states, **kwargs + ) if not decoder.tracks_own_finished: # BeamSearchDecoder would track it own finished, since beams would # be reordered and the finished status of each entry might change. # Otherwise, perform logical OR which would not change the already # finished. - next_finished = control_flow.logical_or(next_finished, - global_finished) + next_finished = control_flow.logical_or( + next_finished, global_finished + ) next_sequence_lengths = nn.elementwise_add( sequence_lengths, - tensor.cast(control_flow.logical_not(global_finished), - sequence_lengths.dtype)) + tensor.cast( + control_flow.logical_not(global_finished), + sequence_lengths.dtype, + ), + ) if impute_finished: # rectify the states for the finished. next_states = map_structure( lambda x, y: _maybe_copy(x, y, global_finished), @@ -1519,16 +1682,22 @@ def _create_array_out_of_while(dtype): warnings.warn( "`next_states` has no `lengths` attribute, the returned `sequence_lengths` would be all zeros." ) if not hasattr(next_states, "lengths") else None - next_sequence_lengths = getattr(next_states, "lengths", - sequence_lengths) + next_sequence_lengths = getattr( + next_states, "lengths", sequence_lengths + ) # create tensor array in global block after dtype[s] of outputs can be got outputs_arrays = map_structure( - lambda x: _create_array_out_of_while(x.dtype), outputs) + lambda x: _create_array_out_of_while(x.dtype), outputs + ) map_structure( lambda x, x_array: control_flow.array_write( - x, i=step_idx, array=x_array), outputs, outputs_arrays) + x, i=step_idx, array=x_array + ), + outputs, + outputs_arrays, + ) control_flow.increment(x=step_idx, value=1.0, in_place=True) # update the global_finished first, since it might be also in states of # decoder, which otherwise would write a stale finished status to array @@ -1540,50 +1709,68 @@ def _create_array_out_of_while(dtype): else: map_structure( lambda x, x_array: control_flow.array_write( - x, i=step_idx, array=x_array), next_inputs, inputs_arrays) + x, i=step_idx, array=x_array + ), + next_inputs, + inputs_arrays, + ) map_structure( lambda x, x_array: control_flow.array_write( - x, i=step_idx, array=x_array), next_states, states_arrays) + x, i=step_idx, array=x_array + ), + next_states, + states_arrays, + ) if max_step_num is not None: control_flow.logical_and( control_flow.logical_not(nn.reduce_all(global_finished)), - control_flow.less_equal(step_idx, max_step_num), cond) + control_flow.less_equal(step_idx, max_step_num), + cond, + ) else: control_flow.logical_not(nn.reduce_all(global_finished), cond) final_outputs = map_structure( lambda array: tensor.tensor_array_to_tensor( - array, axis=0, use_stack=True)[0], outputs_arrays) + array, axis=0, use_stack=True + )[0], + outputs_arrays, + ) if is_test: final_states = global_states else: final_states = map_structure( lambda array: control_flow.array_read(array, step_idx), - states_arrays) + states_arrays, + ) try: - final_outputs, final_states = decoder.finalize(final_outputs, - final_states, - sequence_lengths) + final_outputs, final_states = decoder.finalize( + final_outputs, final_states, sequence_lengths + ) except NotImplementedError: pass if not output_time_major: final_outputs = map_structure(_transpose_batch_time, final_outputs) - return (final_outputs, final_states, - sequence_lengths) if return_length else (final_outputs, - final_states) + return ( + (final_outputs, final_states, sequence_lengths) + if return_length + else (final_outputs, final_states) + ) -def dynamic_decode(decoder, - inits=None, - max_step_num=None, - output_time_major=False, - impute_finished=False, - is_test=False, - return_length=False, - **kwargs): +def dynamic_decode( + decoder, + inits=None, + max_step_num=None, + output_time_major=False, + impute_finished=False, + is_test=False, + return_length=False, + **kwargs +): r""" Dynamic decoding performs :code:`decoder.step()` repeatedly until the returned Tensor indicating finished status contains all True values or the number of @@ -1659,13 +1846,27 @@ def dynamic_decode(decoder, max_step_num=10) """ if _non_static_mode(): - return _dynamic_decode_imperative(decoder, inits, max_step_num, - output_time_major, impute_finished, - is_test, return_length, **kwargs) + return _dynamic_decode_imperative( + decoder, + inits, + max_step_num, + output_time_major, + impute_finished, + is_test, + return_length, + **kwargs + ) else: - return _dynamic_decode_declarative(decoder, inits, max_step_num, - output_time_major, impute_finished, - is_test, return_length, **kwargs) + return _dynamic_decode_declarative( + decoder, + inits, + max_step_num, + output_time_major, + impute_finished, + is_test, + return_length, + **kwargs + ) class DecodeHelper(object): @@ -1792,11 +1993,14 @@ def __init__(self, inputs, sequence_length, time_major=False): # extend inputs to avoid to slice out of range in `next_inputs` # may be easier and have better performance than condition_op self.inputs_ = map_structure( - lambda x: nn.pad(x, - paddings=([0, 1] + [0, 0] * (len(x.shape) - 1)) - if time_major else ([0, 0, 0, 1] + [0, 0] * - (len(x.shape) - 2))), - self.inputs) + lambda x: nn.pad( + x, + paddings=([0, 1] + [0, 0] * (len(x.shape) - 1)) + if time_major + else ([0, 0, 0, 1] + [0, 0] * (len(x.shape) - 2)), + ), + self.inputs, + ) def initialize(self): r""" @@ -1813,12 +2017,14 @@ def initialize(self): """ init_finished = control_flow.equal( self.sequence_length, - tensor.fill_constant(shape=[1], - dtype=self.sequence_length.dtype, - value=0)) + tensor.fill_constant( + shape=[1], dtype=self.sequence_length.dtype, value=0 + ), + ) # TODO: support zero length init_inputs = map_structure( - lambda x: x[0] if self.time_major else x[:, 0], self.inputs) + lambda x: x[0] if self.time_major else x[:, 0], self.inputs + ) return init_inputs, init_finished def sample(self, time, outputs, states): @@ -1872,8 +2078,11 @@ def next_inputs(self, time, outputs, states, sample_ids): shape `[batch_size]`. """ # TODO: compatibility of int32 and int64 - time = tensor.cast(time, "int32") if convert_dtype( - time.dtype) not in ["int32"] else time + time = ( + tensor.cast(time, "int32") + if convert_dtype(time.dtype) not in ["int32"] + else time + ) if self.sequence_length.dtype != time.dtype: self.sequence_length = tensor.cast(self.sequence_length, time.dtype) next_time = time + 1 @@ -1881,11 +2090,12 @@ def next_inputs(self, time, outputs, states, sample_ids): def _slice(x): # TODO: use Variable.__getitem__ axes = [0 if self.time_major else 1] - return nn.squeeze(nn.slice(x, - axes=axes, - starts=[next_time], - ends=[next_time + 1]), - axes=axes) + return nn.squeeze( + nn.slice( + x, axes=axes, starts=[next_time], ends=[next_time + 1] + ), + axes=axes, + ) next_inputs = map_structure(_slice, self.inputs_) return finished, next_inputs, states @@ -1943,9 +2153,9 @@ def __init__(self, embedding_fn, start_tokens, end_token): """ self.embedding_fn = embedding_fn self.start_tokens = start_tokens - self.end_token = tensor.fill_constant(shape=[1], - dtype="int64", - value=end_token) + self.end_token = tensor.fill_constant( + shape=[1], dtype="int64", value=end_token + ) def initialize(self): r""" @@ -1966,7 +2176,8 @@ def initialize(self): shape=[-1], dtype="bool", value=False, - force_cpu=True) + force_cpu=True, + ) init_inputs = self.embedding_fn(self.start_tokens) return init_inputs, init_finished @@ -2054,12 +2265,14 @@ class SampleEmbeddingHelper(GreedyEmbeddingHelper): decoder=decoder, inits=decoder_cell.get_initial_states(encoder_output)) """ - def __init__(self, - embedding_fn, - start_tokens, - end_token, - softmax_temperature=None, - seed=None): + def __init__( + self, + embedding_fn, + start_tokens, + end_token, + softmax_temperature=None, + seed=None, + ): r""" Constructor of SampleEmbeddingHelper. @@ -2086,11 +2299,16 @@ def __init__(self, structure of) tensor variable[s], and `finished` is a tensor with \ bool data type. """ - super(SampleEmbeddingHelper, self).__init__(embedding_fn, start_tokens, - end_token) - self.softmax_temperature = tensor.fill_constant( - shape=[1], dtype="float32", value=softmax_temperature - ) if softmax_temperature is not None else None + super(SampleEmbeddingHelper, self).__init__( + embedding_fn, start_tokens, end_token + ) + self.softmax_temperature = ( + tensor.fill_constant( + shape=[1], dtype="float32", value=softmax_temperature + ) + if softmax_temperature is not None + else None + ) self.seed = seed def sample(self, time, outputs, states): @@ -2112,16 +2330,19 @@ def sample(self, time, outputs, states): Variable: An `int64` tensor with shape `[batch_size]`, representing \ the sampled ids. """ - logits = (outputs / self.softmax_temperature - ) if self.softmax_temperature is not None else outputs + logits = ( + (outputs / self.softmax_temperature) + if self.softmax_temperature is not None + else outputs + ) probs = nn.softmax(logits) # TODO: remove this stop_gradient. The stop_gradient of sample_ids can # not pass to probs, since sampling_id op does not have corresponding # grad op and thus can not pass. probs.stop_gradient = True - sample_ids = nn.sampling_id(probs, - seed=self.seed, - dtype=self.start_tokens.dtype) + sample_ids = nn.sampling_id( + probs, seed=self.seed, dtype=self.start_tokens.dtype + ) return sample_ids @@ -2201,12 +2422,13 @@ def initialize(self, initial_cell_states): return initial_inputs, initial_cell_states, initial_finished class OutputWrapper( - collections.namedtuple("OutputWrapper", - ("cell_outputs", "sample_ids"))): + collections.namedtuple("OutputWrapper", ("cell_outputs", "sample_ids")) + ): """ The structure for the returned value `outputs` of `decoder.step`. A namedtuple includes cell_outputs, sample_ids as fields. """ + pass def step(self, time, inputs, states, **kwargs): @@ -2248,32 +2470,35 @@ def step(self, time, inputs, states, **kwargs): cell_outputs, cell_states = self.cell(inputs, states, **kwargs) if self.output_fn is not None: cell_outputs = self.output_fn(cell_outputs) - sample_ids = self.helper.sample(time=time, - outputs=cell_outputs, - states=cell_states) + sample_ids = self.helper.sample( + time=time, outputs=cell_outputs, states=cell_states + ) sample_ids.stop_gradient = True - (finished, next_inputs, - next_states) = self.helper.next_inputs(time=time, - outputs=cell_outputs, - states=cell_states, - sample_ids=sample_ids) + (finished, next_inputs, next_states) = self.helper.next_inputs( + time=time, + outputs=cell_outputs, + states=cell_states, + sample_ids=sample_ids, + ) outputs = self.OutputWrapper(cell_outputs, sample_ids) return (outputs, next_states, next_inputs, finished) -def dynamic_lstm(input, - size, - h_0=None, - c_0=None, - param_attr=None, - bias_attr=None, - use_peepholes=True, - is_reverse=False, - gate_activation='sigmoid', - cell_activation='tanh', - candidate_activation='tanh', - dtype='float32', - name=None): +def dynamic_lstm( + input, + size, + h_0=None, + c_0=None, + param_attr=None, + bias_attr=None, + use_peepholes=True, + is_reverse=False, + gate_activation='sigmoid', + cell_activation='tanh', + candidate_activation='tanh', + dtype='float32', + name=None, +): r""" :api_attr: Static Graph @@ -2373,35 +2598,40 @@ def dynamic_lstm(input, forward.shape # (-1, 512) cell.shape # (-1, 512) """ - assert _non_static_mode( - ) is not True, "please use lstm instead of dynamic_lstm in dygraph mode!" - assert bias_attr is not False, "bias_attr should not be False in dynamic_lstm." - - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'dynamic_lstm') + assert ( + _non_static_mode() is not True + ), "please use lstm instead of dynamic_lstm in dygraph mode!" + assert ( + bias_attr is not False + ), "bias_attr should not be False in dynamic_lstm." + + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'dynamic_lstm' + ) check_type(h_0, 'h_0', (Variable, type(None)), 'dynamic_lstm') if isinstance(h_0, Variable): - check_variable_and_dtype(h_0, 'h_0', ['float32', 'float64'], - 'dynamic_lstm') + check_variable_and_dtype( + h_0, 'h_0', ['float32', 'float64'], 'dynamic_lstm' + ) check_type(c_0, 'c_0', (Variable, type(None)), 'dynamic_lstm') if isinstance(c_0, Variable): - check_variable_and_dtype(c_0, 'c_0', ['float32', 'float64'], - 'dynamic_lstm') + check_variable_and_dtype( + c_0, 'c_0', ['float32', 'float64'], 'dynamic_lstm' + ) helper = LayerHelper('lstm', **locals()) size = size // 4 - weight = helper.create_parameter(attr=helper.param_attr, - shape=[size, 4 * size], - dtype=dtype) + weight = helper.create_parameter( + attr=helper.param_attr, shape=[size, 4 * size], dtype=dtype + ) bias_size = [1, 7 * size] if not use_peepholes: bias_size[1] = 4 * size - bias = helper.create_parameter(attr=helper.bias_attr, - shape=bias_size, - dtype=dtype, - is_bias=True) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True + ) hidden = helper.create_variable_for_type_inference(dtype) cell = helper.create_variable_for_type_inference(dtype) @@ -2410,47 +2640,55 @@ def dynamic_lstm(input, inputs = {'Input': input, 'Weight': weight, 'Bias': bias} batch_size = input.shape[0] if h_0: - assert h_0.shape == (batch_size, size), \ + assert h_0.shape == (batch_size, size), ( 'The shape of h0 should be (batch_size, %d)' % size + ) inputs['H0'] = h_0 if c_0: - assert c_0.shape == (batch_size, size), \ + assert c_0.shape == (batch_size, size), ( 'The shape of c0 should be (batch_size, %d)' % size + ) inputs['C0'] = c_0 - helper.append_op(type='lstm', - inputs=inputs, - outputs={ - 'Hidden': hidden, - 'Cell': cell, - 'BatchGate': batch_gate, - 'BatchCellPreAct': batch_cell_pre_act - }, - attrs={ - 'use_peepholes': use_peepholes, - 'is_reverse': is_reverse, - 'gate_activation': gate_activation, - 'cell_activation': cell_activation, - 'candidate_activation': candidate_activation - }) + helper.append_op( + type='lstm', + inputs=inputs, + outputs={ + 'Hidden': hidden, + 'Cell': cell, + 'BatchGate': batch_gate, + 'BatchCellPreAct': batch_cell_pre_act, + }, + attrs={ + 'use_peepholes': use_peepholes, + 'is_reverse': is_reverse, + 'gate_activation': gate_activation, + 'cell_activation': cell_activation, + 'candidate_activation': candidate_activation, + }, + ) return hidden, cell -@deprecated(since='2.0.0', - update_to='paddle.nn.LSTM', - reason="This API may occur CUDNN errors.") -def lstm(input, - init_h, - init_c, - max_len, - hidden_size, - num_layers, - dropout_prob=0.0, - is_bidirec=False, - is_test=False, - name=None, - default_initializer=None, - seed=-1): +@deprecated( + since='2.0.0', + update_to='paddle.nn.LSTM', + reason="This API may occur CUDNN errors.", +) +def lstm( + input, + init_h, + init_c, + max_len, + hidden_size, + num_layers, + dropout_prob=0.0, + is_bidirec=False, + is_test=False, + name=None, + default_initializer=None, + seed=-1, +): r""" :api_attr: Static Graph @@ -2577,63 +2815,71 @@ def lstm(input, weight_size += input_weight_size + hidden_weight_size weight_size += hidden_size * 8 * num_dirrection - weight = helper.create_parameter(attr=helper.param_attr, - shape=[weight_size], - dtype=dtype, - default_initializer=default_initializer) + weight = helper.create_parameter( + attr=helper.param_attr, + shape=[weight_size], + dtype=dtype, + default_initializer=default_initializer, + ) out = helper.create_variable_for_type_inference(dtype) last_h = helper.create_variable_for_type_inference(dtype) last_c = helper.create_variable_for_type_inference(dtype) reserve = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) state_out = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) state_out.persistable = True - helper.append_op(type='cudnn_lstm', - inputs={ - 'Input': input, - 'InitH': init_h, - 'InitC': init_c, - 'W': weight, - }, - outputs={ - 'Out': out, - 'LastH': last_h, - 'LastC': last_c, - 'Reserve': reserve, - 'StateOut': state_out, - }, - attrs={ - 'is_bidirec': is_bidirec, - 'input_size': input_size, - 'hidden_size': hidden_size, - 'num_layers': num_layers, - 'is_test': is_test, - 'dropout_prob': dropout_prob, - 'seed': seed, - }) + helper.append_op( + type='cudnn_lstm', + inputs={ + 'Input': input, + 'InitH': init_h, + 'InitC': init_c, + 'W': weight, + }, + outputs={ + 'Out': out, + 'LastH': last_h, + 'LastC': last_c, + 'Reserve': reserve, + 'StateOut': state_out, + }, + attrs={ + 'is_bidirec': is_bidirec, + 'input_size': input_size, + 'hidden_size': hidden_size, + 'num_layers': num_layers, + 'is_test': is_test, + 'dropout_prob': dropout_prob, + 'seed': seed, + }, + ) return out, last_h, last_c -def dynamic_lstmp(input, - size, - proj_size, - param_attr=None, - bias_attr=None, - use_peepholes=True, - is_reverse=False, - gate_activation='sigmoid', - cell_activation='tanh', - candidate_activation='tanh', - proj_activation='tanh', - dtype='float32', - name=None, - h_0=None, - c_0=None, - cell_clip=None, - proj_clip=None): +def dynamic_lstmp( + input, + size, + proj_size, + param_attr=None, + bias_attr=None, + use_peepholes=True, + is_reverse=False, + gate_activation='sigmoid', + cell_activation='tanh', + candidate_activation='tanh', + proj_activation='tanh', + dtype='float32', + name=None, + h_0=None, + c_0=None, + cell_clip=None, + proj_clip=None, +): r""" :api_attr: Static Graph @@ -2756,39 +3002,44 @@ def dynamic_lstmp(input, last_c.shape # (-1, 512) """ - assert _non_static_mode( - ) is not True, "please use lstm instead of dynamic_lstmp in dygraph mode!" + assert ( + _non_static_mode() is not True + ), "please use lstm instead of dynamic_lstmp in dygraph mode!" - assert bias_attr is not False, "bias_attr should not be False in dynamic_lstmp." + assert ( + bias_attr is not False + ), "bias_attr should not be False in dynamic_lstmp." - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'dynamic_lstmp') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'dynamic_lstmp' + ) check_type(h_0, 'h_0', (Variable, type(None)), 'dynamic_lstmp') if isinstance(h_0, Variable): - check_variable_and_dtype(h_0, 'h_0', ['float32', 'float64'], - 'dynamic_lstmp') + check_variable_and_dtype( + h_0, 'h_0', ['float32', 'float64'], 'dynamic_lstmp' + ) check_type(c_0, 'c_0', (Variable, type(None)), 'dynamic_lstmp') if isinstance(c_0, Variable): - check_variable_and_dtype(c_0, 'c_0', ['float32', 'float64'], - 'dynamic_lstmp') + check_variable_and_dtype( + c_0, 'c_0', ['float32', 'float64'], 'dynamic_lstmp' + ) helper = LayerHelper('lstmp', **locals()) size = size // 4 - weight = helper.create_parameter(attr=helper.param_attr, - shape=[proj_size, 4 * size], - dtype=dtype) - proj_weight = helper.create_parameter(attr=helper.param_attr, - shape=[size, proj_size], - dtype=dtype) + weight = helper.create_parameter( + attr=helper.param_attr, shape=[proj_size, 4 * size], dtype=dtype + ) + proj_weight = helper.create_parameter( + attr=helper.param_attr, shape=[size, proj_size], dtype=dtype + ) bias_size = [1, 7 * size] if not use_peepholes: bias_size[1] = 4 * size - bias = helper.create_parameter(attr=helper.bias_attr, - shape=bias_size, - dtype=dtype, - is_bias=True) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True + ) projection = helper.create_variable_for_type_inference(dtype) cell = helper.create_variable_for_type_inference(dtype) @@ -2800,16 +3051,18 @@ def dynamic_lstmp(input, 'Input': input, 'Weight': weight, 'ProjWeight': proj_weight, - 'Bias': bias + 'Bias': bias, } batch_size = input.shape[0] if h_0: - assert h_0.shape == (batch_size, proj_size), \ + assert h_0.shape == (batch_size, proj_size), ( 'The shape of h0 should be (batch_size, %d)' % proj_size + ) inputs['H0'] = h_0 if c_0: - assert c_0.shape == (batch_size, size), \ + assert c_0.shape == (batch_size, size), ( 'The shape of c0 should be (batch_size, %d)' % size + ) inputs['C0'] = c_0 if cell_clip: @@ -2817,37 +3070,41 @@ def dynamic_lstmp(input, if proj_clip: assert proj_clip >= 0, "proj_clip should not be negative." - helper.append_op(type='lstmp', - inputs=inputs, - outputs={ - 'Projection': projection, - 'Cell': cell, - 'BatchHidden': batch_hidden, - 'BatchGate': batch_gate, - 'BatchCellPreAct': batch_cell_pre_act - }, - attrs={ - 'use_peepholes': use_peepholes, - 'cell_clip': cell_clip, - 'proj_clip': proj_clip, - 'is_reverse': is_reverse, - 'gate_activation': gate_activation, - 'cell_activation': cell_activation, - 'candidate_activation': candidate_activation, - 'proj_activation': proj_activation - }) + helper.append_op( + type='lstmp', + inputs=inputs, + outputs={ + 'Projection': projection, + 'Cell': cell, + 'BatchHidden': batch_hidden, + 'BatchGate': batch_gate, + 'BatchCellPreAct': batch_cell_pre_act, + }, + attrs={ + 'use_peepholes': use_peepholes, + 'cell_clip': cell_clip, + 'proj_clip': proj_clip, + 'is_reverse': is_reverse, + 'gate_activation': gate_activation, + 'cell_activation': cell_activation, + 'candidate_activation': candidate_activation, + 'proj_activation': proj_activation, + }, + ) return projection, cell -def dynamic_gru(input, - size, - param_attr=None, - bias_attr=None, - is_reverse=False, - gate_activation='sigmoid', - candidate_activation='tanh', - h_0=None, - origin_mode=False): +def dynamic_gru( + input, + size, + param_attr=None, + bias_attr=None, + is_reverse=False, + gate_activation='sigmoid', + candidate_activation='tanh', + h_0=None, + origin_mode=False, +): r""" :api_attr: Static Graph @@ -2953,33 +3210,35 @@ def dynamic_gru(input, hidden = fluid.layers.dynamic_gru(input=x, size=hidden_dim) """ - assert _non_static_mode( - ) is not True, "please use gru instead of dynamic_gru in dygraph mode!" + assert ( + _non_static_mode() is not True + ), "please use gru instead of dynamic_gru in dygraph mode!" - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'dynamic_gru') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'dynamic_gru' + ) check_type(h_0, 'h_0', (Variable, type(None)), 'dynamic_gru') if isinstance(h_0, Variable): - check_variable_and_dtype(h_0, 'h_0', ['float32', 'float64'], - 'dynamic_gru') + check_variable_and_dtype( + h_0, 'h_0', ['float32', 'float64'], 'dynamic_gru' + ) helper = LayerHelper('gru', **locals()) dtype = helper.input_dtype() - weight = helper.create_parameter(attr=helper.param_attr, - shape=[size, 3 * size], - dtype=dtype) - bias = helper.create_parameter(attr=helper.bias_attr, - shape=[1, 3 * size], - dtype=dtype, - is_bias=True) + weight = helper.create_parameter( + attr=helper.param_attr, shape=[size, 3 * size], dtype=dtype + ) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=[1, 3 * size], dtype=dtype, is_bias=True + ) batch_size = input.shape[0] inputs = {'Input': input, 'Weight': weight, 'Bias': bias} if h_0: - assert h_0.shape == ( - batch_size, - size), 'The shape of h0 should be(batch_size, %d)' % size + assert h_0.shape == (batch_size, size), ( + 'The shape of h0 should be(batch_size, %d)' % size + ) inputs['H0'] = h_0 hidden = helper.create_variable_for_type_inference(dtype) @@ -2987,31 +3246,35 @@ def dynamic_gru(input, batch_reset_hidden_prev = helper.create_variable_for_type_inference(dtype) batch_hidden = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='gru', - inputs=inputs, - outputs={ - 'Hidden': hidden, - 'BatchGate': batch_gate, - 'BatchResetHiddenPrev': batch_reset_hidden_prev, - 'BatchHidden': batch_hidden - }, - attrs={ - 'is_reverse': is_reverse, - 'gate_activation': gate_activation, - 'activation': candidate_activation, - 'origin_mode': origin_mode - }) + helper.append_op( + type='gru', + inputs=inputs, + outputs={ + 'Hidden': hidden, + 'BatchGate': batch_gate, + 'BatchResetHiddenPrev': batch_reset_hidden_prev, + 'BatchHidden': batch_hidden, + }, + attrs={ + 'is_reverse': is_reverse, + 'gate_activation': gate_activation, + 'activation': candidate_activation, + 'origin_mode': origin_mode, + }, + ) return hidden -def gru_unit(input, - hidden, - size, - param_attr=None, - bias_attr=None, - activation='tanh', - gate_activation='sigmoid', - origin_mode=False): +def gru_unit( + input, + hidden, + size, + param_attr=None, + bias_attr=None, + activation='tanh', + gate_activation='sigmoid', + origin_mode=False, +): r""" :api_attr: Static Graph @@ -3113,8 +3376,9 @@ def gru_unit(input, """ check_variable_and_dtype(input, 'input', ['float32', 'float64'], 'gru_unit') - check_variable_and_dtype(hidden, 'hidden', ['float32', 'float64'], - 'gru_unit') + check_variable_and_dtype( + hidden, 'hidden', ['float32', 'float64'], 'gru_unit' + ) check_type(size, 'size', (int), 'gru_unit') activation_dict = dict( identity=0, @@ -3130,9 +3394,9 @@ def gru_unit(input, size = size // 3 # create weight - weight = helper.create_parameter(attr=helper.param_attr, - shape=[size, 3 * size], - dtype=dtype) + weight = helper.create_parameter( + attr=helper.param_attr, shape=[size, 3 * size], dtype=dtype + ) gate = helper.create_variable_for_type_inference(dtype) reset_hidden_pre = helper.create_variable_for_type_inference(dtype) @@ -3141,10 +3405,9 @@ def gru_unit(input, # create bias if helper.bias_attr: bias_size = [1, 3 * size] - bias = helper.create_parameter(attr=helper.bias_attr, - shape=bias_size, - dtype=dtype, - is_bias=True) + bias = helper.create_parameter( + attr=helper.bias_attr, shape=bias_size, dtype=dtype, is_bias=True + ) inputs['Bias'] = bias helper.append_op( @@ -3158,22 +3421,25 @@ def gru_unit(input, attrs={ 'activation': 2, # tanh 'gate_activation': 1, # sigmoid - 'origin_mode': origin_mode - }) + 'origin_mode': origin_mode, + }, + ) return updated_hidden, reset_hidden_pre, gate -def beam_search(pre_ids, - pre_scores, - ids, - scores, - beam_size, - end_id, - level=0, - is_accumulated=True, - name=None, - return_parent_idx=False): +def beam_search( + pre_ids, + pre_scores, + ids, + scores, + beam_size, + end_id, + level=0, + is_accumulated=True, + name=None, + return_parent_idx=False, +): r""" Beam search is a classical algorithm for selecting candidate words in a @@ -3280,11 +3546,13 @@ def beam_search(pre_ids, end_id=end_id) """ check_variable_and_dtype(pre_ids, 'pre_ids', ['int64'], 'beam_search') - check_variable_and_dtype(pre_scores, 'pre_scores', ['float32', 'float64'], - 'beam_search') + check_variable_and_dtype( + pre_scores, 'pre_scores', ['float32', 'float64'], 'beam_search' + ) check_type(ids, 'ids', (Variable, type(None)), 'beam_search') - check_variable_and_dtype(scores, 'scores', ['float32', 'float64'], - 'beam_search') + check_variable_and_dtype( + scores, 'scores', ['float32', 'float64'], 'beam_search' + ) helper = LayerHelper('beam_search', **locals()) score_type = pre_scores.dtype id_type = pre_ids.dtype @@ -3294,7 +3562,8 @@ def beam_search(pre_ids, inputs["ids"] = ids selected_scores = helper.create_variable_for_type_inference( - dtype=score_type) + dtype=score_type + ) selected_ids = helper.create_variable_for_type_inference(dtype=id_type) # parent_idx is a tensor used to gather cell states at the next time # step. Though lod in selected_ids can also be used to gather by @@ -3308,7 +3577,7 @@ def beam_search(pre_ids, outputs={ 'selected_ids': selected_ids, 'selected_scores': selected_scores, - 'parent_idx': parent_idx + 'parent_idx': parent_idx, }, attrs={ # TODO(ChunweiYan) to assure other value support @@ -3316,7 +3585,8 @@ def beam_search(pre_ids, 'beam_size': beam_size, 'end_id': end_id, 'is_accumulated': is_accumulated, - }) + }, + ) if return_parent_idx: return selected_ids, selected_scores, parent_idx else: @@ -3381,37 +3651,37 @@ def beam_search_decode(ids, scores, beam_size, end_id, name=None): ids, scores, beam_size=5, end_id=0) """ check_variable_and_dtype(ids, 'ids', ['int64'], 'beam_search_encode') - check_variable_and_dtype(scores, 'scores', ['float32'], - 'beam_search_encode') + check_variable_and_dtype( + scores, 'scores', ['float32'], 'beam_search_encode' + ) helper = LayerHelper('beam_search_decode', **locals()) sentence_ids = helper.create_variable_for_type_inference(dtype=ids.dtype) sentence_scores = helper.create_variable_for_type_inference( - dtype=scores.dtype) - - helper.append_op(type="beam_search_decode", - inputs={ - "Ids": ids, - "Scores": scores - }, - outputs={ - "SentenceIds": sentence_ids, - "SentenceScores": sentence_scores - }, - attrs={ - "beam_size": beam_size, - "end_id": end_id - }) + dtype=scores.dtype + ) + + helper.append_op( + type="beam_search_decode", + inputs={"Ids": ids, "Scores": scores}, + outputs={ + "SentenceIds": sentence_ids, + "SentenceScores": sentence_scores, + }, + attrs={"beam_size": beam_size, "end_id": end_id}, + ) return sentence_ids, sentence_scores -def lstm_unit(x_t, - hidden_t_prev, - cell_t_prev, - forget_bias=0.0, - param_attr=None, - bias_attr=None, - name=None): +def lstm_unit( + x_t, + hidden_t_prev, + cell_t_prev, + forget_bias=0.0, + param_attr=None, + bias_attr=None, + name=None, +): r""" :api_attr: Static Graph @@ -3496,10 +3766,12 @@ def lstm_unit(x_t, """ helper = LayerHelper('lstm_unit', **locals()) check_variable_and_dtype(x_t, 'x_t', ['float32', 'float64'], 'lstm_unit') - check_variable_and_dtype(hidden_t_prev, 'hidden_t_prev', - ['float32', 'float64'], 'lstm_unit') - check_variable_and_dtype(cell_t_prev, 'cell_t_prev', ['float32', 'float64'], - 'lstm_unit') + check_variable_and_dtype( + hidden_t_prev, 'hidden_t_prev', ['float32', 'float64'], 'lstm_unit' + ) + check_variable_and_dtype( + cell_t_prev, 'cell_t_prev', ['float32', 'float64'], 'lstm_unit' + ) if len(x_t.shape) != 2: raise ValueError("Rank of x_t must be 2.") @@ -3509,37 +3781,41 @@ def lstm_unit(x_t, if len(cell_t_prev.shape) != 2: raise ValueError("Rank of cell_t_prev must be 2.") - if x_t.shape[0] != hidden_t_prev.shape[0] or x_t.shape[ - 0] != cell_t_prev.shape[0]: - raise ValueError("The 1st dimensions of x_t, hidden_t_prev and " - "cell_t_prev must be the same.") + if ( + x_t.shape[0] != hidden_t_prev.shape[0] + or x_t.shape[0] != cell_t_prev.shape[0] + ): + raise ValueError( + "The 1st dimensions of x_t, hidden_t_prev and " + "cell_t_prev must be the same." + ) if hidden_t_prev.shape[1] != cell_t_prev.shape[1]: - raise ValueError("The 2nd dimensions of hidden_t_prev and " - "cell_t_prev must be the same.") + raise ValueError( + "The 2nd dimensions of hidden_t_prev and " + "cell_t_prev must be the same." + ) if bias_attr is None: bias_attr = ParamAttr() size = cell_t_prev.shape[1] concat_out = nn.concat(input=[x_t, hidden_t_prev], axis=1) - fc_out = nn.fc(input=concat_out, - size=4 * size, - param_attr=param_attr, - bias_attr=bias_attr) + fc_out = nn.fc( + input=concat_out, + size=4 * size, + param_attr=param_attr, + bias_attr=bias_attr, + ) dtype = x_t.dtype c = helper.create_variable_for_type_inference(dtype) h = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='lstm_unit', - inputs={ - "X": fc_out, - "C_prev": cell_t_prev - }, - outputs={ - "C": c, - "H": h - }, - attrs={"forget_bias": forget_bias}) + helper.append_op( + type='lstm_unit', + inputs={"X": fc_out, "C_prev": cell_t_prev}, + outputs={"C": c, "H": h}, + attrs={"forget_bias": forget_bias}, + ) return h, c diff --git a/python/paddle/fluid/layers/sequence_lod.py b/python/paddle/fluid/layers/sequence_lod.py index 5deac6ff952b8..1ea5e2cf5e5ef 100644 --- a/python/paddle/fluid/layers/sequence_lod.py +++ b/python/paddle/fluid/layers/sequence_lod.py @@ -14,7 +14,14 @@ import paddle from .layer_function_generator import templatedoc -from ..framework import core, Variable, _non_static_mode, in_dygraph_mode, _in_legacy_dygraph, convert_np_dtype_to_dtype_ +from ..framework import ( + core, + Variable, + _non_static_mode, + in_dygraph_mode, + _in_legacy_dygraph, + convert_np_dtype_to_dtype_, +) from ..layer_helper import LayerHelper from ..data_feeder import check_variable_and_dtype, check_type, check_dtype from ..core import VarDesc @@ -41,20 +48,22 @@ @templatedoc() -def sequence_conv(input, - num_filters, - filter_size=3, - filter_stride=1, - padding=True, - padding_start=None, - bias_attr=None, - param_attr=None, - act=None, - name=None): +def sequence_conv( + input, + num_filters, + filter_size=3, + filter_stride=1, + padding=True, + padding_start=None, + bias_attr=None, + param_attr=None, + act=None, + name=None, +): r""" Note: - Only receives LoDTensor as input. If your input is Tensor, please use conv2d Op.(fluid.layers.** :ref:`api_fluid_layers_conv2d` ). + Only receives LoDTensor as input. If your input is Tensor, please use conv2d Op.(fluid.layers.** :ref:`api_fluid_layers_conv2d` ). This operator receives input sequences with variable length and other convolutional configuration parameters(num_filters, filter_size) to apply the convolution operation. @@ -146,31 +155,35 @@ def sequence_conv(input, x_conved = paddle.static.nn.sequence_conv(input=x, num_filters=2, filter_size=3, padding_start=-1) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'sequence_conv') + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'sequence_conv' + ) helper = LayerHelper('sequence_conv', **locals()) dtype = helper.input_dtype() filter_shape = [filter_size * input.shape[1], num_filters] - filter_param = helper.create_parameter(attr=helper.param_attr, - shape=filter_shape, - dtype=dtype) + filter_param = helper.create_parameter( + attr=helper.param_attr, shape=filter_shape, dtype=dtype + ) pre_bias = helper.create_variable_for_type_inference(dtype) if padding_start is None: padding_start = -int(filter_size // 2) - helper.append_op(type='sequence_conv', - inputs={ - 'X': [input], - 'Filter': [filter_param], - }, - outputs={"Out": pre_bias}, - attrs={ - 'contextStride': filter_stride, - 'contextStart': padding_start, - 'contextLength': filter_size, - }) + helper.append_op( + type='sequence_conv', + inputs={ + 'X': [input], + 'Filter': [filter_param], + }, + outputs={"Out": pre_bias}, + attrs={ + 'contextStride': filter_stride, + 'contextStart': padding_start, + 'contextLength': filter_size, + }, + ) pre_act = helper.append_bias_op(pre_bias) return helper.append_activation(pre_act) @@ -244,17 +257,21 @@ def sequence_softmax(input, use_cudnn=False, name=None): dtype='float32', lod_level=1) x_sequence_softmax_2 = paddle.static.nn.sequence_softmax(input=y) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper('sequence_softmax', **locals()) - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'sequence_softmax') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'sequence_softmax' + ) dtype = helper.input_dtype() softmax_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="sequence_softmax", - inputs={"X": input}, - outputs={"Out": softmax_out}, - attrs={"use_cudnn": use_cudnn}) + helper.append_op( + type="sequence_softmax", + inputs={"X": input}, + outputs={"Out": softmax_out}, + attrs={"use_cudnn": use_cudnn}, + ) return softmax_out @@ -345,26 +362,27 @@ def sequence_pool(input, pool_type, is_test=False, pad_value=0.0): last_x = paddle.static.nn.sequence_pool(input=x, pool_type='last') first_x = paddle.static.nn.sequence_pool(input=x, pool_type='first') """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'sequence_pool') + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'sequence_pool' + ) helper = LayerHelper('sequence_pool', **locals()) dtype = helper.input_dtype() pool_out = helper.create_variable_for_type_inference(dtype) max_index = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="sequence_pool", - inputs={"X": input}, - outputs={ - "Out": pool_out, - "MaxIndex": max_index - }, - attrs={ - "pooltype": pool_type.upper(), - "is_test": is_test, - "pad_value": pad_value - }) + helper.append_op( + type="sequence_pool", + inputs={"X": input}, + outputs={"Out": pool_out, "MaxIndex": max_index}, + attrs={ + "pooltype": pool_type.upper(), + "is_test": is_test, + "pad_value": pad_value, + }, + ) # when pool_type is max, variable max_index is initialized, # so we stop the gradient explicitly here @@ -422,20 +440,24 @@ def sequence_concat(input, name=None): y = paddle.static.data(name='y', shape=[-1, 10], dtype='float32', lod_level=1) out = paddle.static.nn.sequence_concat(input=[x, y]) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper('sequence_concat', **locals()) check_type(input, 'input', list, 'fluid.layers.sequence_concat') for i, input_x in enumerate(input): - check_variable_and_dtype(input_x, 'input[' + str(i) + ']', - ['int64', 'float32', 'float64'], - 'fluid.layers.sequence_concat') + check_variable_and_dtype( + input_x, + 'input[' + str(i) + ']', + ['int64', 'float32', 'float64'], + 'fluid.layers.sequence_concat', + ) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) - helper.append_op(type='sequence_concat', - inputs={'X': input}, - outputs={'Out': [out]}) + helper.append_op( + type='sequence_concat', inputs={'X': input}, outputs={'Out': [out]} + ) return out @@ -491,8 +513,9 @@ def sequence_first_step(input): x = paddle.static.data(name='x', shape=[None, 10], dtype='float32', lod_level=1) x_first_step = paddle.static.nn.sequence_first_step(input=x) """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'sequence_first_step') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'sequence_first_step' + ) return sequence_pool(input=input, pool_type="first") @@ -549,8 +572,9 @@ def sequence_last_step(input): x = paddle.static.data(name='x', shape=[None, 10], dtype='float32', lod_level=1) x_last_step = paddle.static.nn.sequence_last_step(input=x) """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'sequence_last_step') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'sequence_last_step' + ) return sequence_pool(input=input, pool_type="last") @@ -615,17 +639,23 @@ def sequence_slice(input, offset, length, name=None): subseqs = paddle.static.nn.sequence_slice(input=seqs, offset=offset, length=length) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper("sequence_slice", **locals()) - check_variable_and_dtype(input, 'input', - ['float32', 'float64', 'int32', 'int64'], - 'sequence_slice') - check_variable_and_dtype(offset, 'offset', ['int32', 'int64'], - 'sequence_slice') - check_variable_and_dtype(length, 'length', ['int32', 'int64'], - 'sequence_slice') + check_variable_and_dtype( + input, + 'input', + ['float32', 'float64', 'int32', 'int64'], + 'sequence_slice', + ) + check_variable_and_dtype( + offset, 'offset', ['int32', 'int64'], 'sequence_slice' + ) + check_variable_and_dtype( + length, 'length', ['int32', 'int64'], 'sequence_slice' + ) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) @@ -633,13 +663,11 @@ def sequence_slice(input, offset, length, name=None): offset.stop_gradient = True length.stop_gradient = True - helper.append_op(type="sequence_slice", - inputs={ - "X": input, - "Offset": offset, - "Length": length - }, - outputs={"Out": out}) + helper.append_op( + type="sequence_slice", + inputs={"X": input, "Offset": offset, "Length": length}, + outputs={"Out": out}, + ) return out @@ -765,20 +793,21 @@ def sequence_expand(x, y, ref_level=-1, name=None): # dtype: float # data: [1 2 1 2 3 4 3 4] """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'sequence_expand') + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'sequence_expand' + ) helper = LayerHelper('sequence_expand', **locals()) dtype = helper.input_dtype(input_param_name='x') tmp = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='sequence_expand', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': tmp}, - attrs={'ref_level': ref_level}) + helper.append_op( + type='sequence_expand', + inputs={'X': x, 'Y': y}, + outputs={'Out': tmp}, + attrs={'ref_level': ref_level}, + ) return tmp @@ -886,20 +915,19 @@ def sequence_expand_as(x, y, name=None): # dtype: float # data: [1 1 1 2 2 2 3 4] """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'sequence_expand_as') + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'sequence_expand_as' + ) check_type(y, 'y', Variable, 'sequence_expand_as') helper = LayerHelper('sequence_expand_as', **locals()) dtype = helper.input_dtype(input_param_name='x') tmp = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='sequence_expand_as', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': tmp}) + helper.append_op( + type='sequence_expand_as', inputs={'X': x, 'Y': y}, outputs={'Out': tmp} + ) return tmp @@ -990,14 +1018,22 @@ def sequence_pad(x, pad_value, maxlen=None, name=None): out = paddle.static.nn.sequence_pad(x=x, pad_value=pad_value) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper('sequence_pad', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'fluid.layers.sequence_pad') - check_variable_and_dtype(pad_value, 'pad_value', - ['float32', 'float64', 'int32', 'int64'], - 'fluid.layers.sequence_pad') + check_variable_and_dtype( + x, + 'x', + ['float32', 'float64', 'int32', 'int64'], + 'fluid.layers.sequence_pad', + ) + check_variable_and_dtype( + pad_value, + 'pad_value', + ['float32', 'float64', 'int32', 'int64'], + 'fluid.layers.sequence_pad', + ) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) length = helper.create_variable_for_type_inference(VarDesc.VarType.INT64) @@ -1007,16 +1043,12 @@ def sequence_pad(x, pad_value, maxlen=None, name=None): if maxlen is None: maxlen = -1 - helper.append_op(type='sequence_pad', - inputs={ - 'X': x, - 'PadValue': pad_value - }, - outputs={ - 'Out': out, - 'Length': length - }, - attrs={'padded_length': maxlen}) + helper.append_op( + type='sequence_pad', + inputs={'X': x, 'PadValue': pad_value}, + outputs={'Out': out, 'Length': length}, + attrs={'padded_length': maxlen}, + ) return out, length @@ -1030,22 +1062,22 @@ def sequence_unpad(x, length, name=None): .. code-block:: text - Case 1: + Case 1: - Given input Variable **x**: - x.data = [[ 1.0, 2.0, 3.0, 4.0, 5.0], - [ 6.0, 7.0, 8.0, 9.0, 10.0], - [11.0, 12.0, 13.0, 14.0, 15.0]], + Given input Variable **x**: + x.data = [[ 1.0, 2.0, 3.0, 4.0, 5.0], + [ 6.0, 7.0, 8.0, 9.0, 10.0], + [11.0, 12.0, 13.0, 14.0, 15.0]], - in which there are 3 sequences padded to length 5, and the actual length - specified by input Variable **length**: + in which there are 3 sequences padded to length 5, and the actual length + specified by input Variable **length**: - length.data = [2, 3, 4], + length.data = [2, 3, 4], - after unpadding, the output Variable will be: + after unpadding, the output Variable will be: - out.data = [[1.0, 2.0, 6.0, 7.0, 8.0, 11.0, 12.0, 13.0, 14.0]] - out.lod = [[0, 2, 5, 9]] + out.data = [[1.0, 2.0, 6.0, 7.0, 8.0, 11.0, 12.0, 13.0, 14.0]] + out.lod = [[0, 2, 5, 9]] Args: x(Variable): A Tensor which contains padding data, and its shape size can not be less than 2. @@ -1075,24 +1107,29 @@ def sequence_unpad(x, length, name=None): unpad_data = paddle.static.nn.sequence_unpad(x=pad_data, length=len) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper('sequence_unpad', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'fluid.layers.sequence_unpad') - check_variable_and_dtype(length, 'length', ['int64'], - 'fluid.layers.sequence_unpad') + check_variable_and_dtype( + x, + 'x', + ['float32', 'float64', 'int32', 'int64'], + 'fluid.layers.sequence_unpad', + ) + check_variable_and_dtype( + length, 'length', ['int64'], 'fluid.layers.sequence_unpad' + ) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) length.stop_gradient = True - helper.append_op(type='sequence_unpad', - inputs={ - 'X': x, - 'Length': length - }, - outputs={'Out': out}) + helper.append_op( + type='sequence_unpad', + inputs={'X': x, 'Length': length}, + outputs={'Out': out}, + ) return out @@ -1145,17 +1182,23 @@ def sequence_reshape(input, new_dim): x = paddle.static.data(name='x', shape=[None, 16], dtype='float32', lod_level=1) x_reshaped = paddle.static.nn.sequence_reshape(input=x, new_dim=4) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper('sequence_reshape', **locals()) - check_variable_and_dtype(input, 'input', - ['float32', 'float64', 'int32', 'int64'], - 'fluid.layers.sequence_reshape') + check_variable_and_dtype( + input, + 'input', + ['float32', 'float64', 'int32', 'int64'], + 'fluid.layers.sequence_reshape', + ) out = helper.create_variable_for_type_inference(helper.input_dtype()) - helper.append_op(type='sequence_reshape', - inputs={'X': [input]}, - outputs={'Out': [out]}, - attrs={'new_dim': new_dim}) + helper.append_op( + type='sequence_reshape', + inputs={'X': [input]}, + outputs={'Out': [out]}, + attrs={'new_dim': new_dim}, + ) return out @@ -1224,28 +1267,34 @@ def sequence_scatter(input, index, updates, name=None): output = paddle.static.nn.sequence_scatter(input, index, updates) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper('sequence_scatter', **locals()) - check_variable_and_dtype(input, 'input', - ['float32', 'float64', 'int32', 'int64'], - 'sequence_scatter') - check_variable_and_dtype(index, 'index', ['int32', 'int64'], - 'sequence_scatter') - check_variable_and_dtype(updates, 'updates', - ['float32', 'float64', 'int32', 'int64'], - 'sequence_scatter') + check_variable_and_dtype( + input, + 'input', + ['float32', 'float64', 'int32', 'int64'], + 'sequence_scatter', + ) + check_variable_and_dtype( + index, 'index', ['int32', 'int64'], 'sequence_scatter' + ) + check_variable_and_dtype( + updates, + 'updates', + ['float32', 'float64', 'int32', 'int64'], + 'sequence_scatter', + ) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="sequence_scatter", - inputs={ - "X": input, - "Ids": index, - "Updates": updates - }, - outputs={"Out": out}) + helper.append_op( + type="sequence_scatter", + inputs={"X": input, "Ids": index, "Updates": updates}, + outputs={"Out": out}, + ) return out @@ -1300,20 +1349,22 @@ def sequence_enumerate(input, win_size, pad_value=0, name=None): x = paddle.static.data(name='x', shape=[-1, 1], dtype='int32', lod_level=1) out = paddle.static.nn.sequence_enumerate(input=x, win_size=3, pad_value=0) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") - check_variable_and_dtype(input, 'input', ['int32', 'int64'], - 'sequence_enumerate') + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." + check_variable_and_dtype( + input, 'input', ['int32', 'int64'], 'sequence_enumerate' + ) helper = LayerHelper('sequence_enumerate', **locals()) - out = helper.create_variable_for_type_inference(helper.input_dtype(), - stop_gradient=True) - helper.append_op(type='sequence_enumerate', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'win_size': win_size, - 'pad_value': pad_value - }) + out = helper.create_variable_for_type_inference( + helper.input_dtype(), stop_gradient=True + ) + helper.append_op( + type='sequence_enumerate', + inputs={'X': input}, + outputs={'Out': out}, + attrs={'win_size': win_size, 'pad_value': pad_value}, + ) return out @@ -1427,16 +1478,22 @@ def sequence_reverse(x, name=None): x = paddle.static.data(name='x', shape=[None, 10], dtype='float32', lod_level=1) x_reversed = paddle.static.nn.sequence_reverse(x) """ - assert not _non_static_mode(), ( - "sequence layer is not supported in dygraph mode yet.") + assert ( + not _non_static_mode() + ), "sequence layer is not supported in dygraph mode yet." helper = LayerHelper("sequence_reverse", **locals()) - check_variable_and_dtype(x, 'x', - ['float32', 'float64', 'int8', 'int32', 'int64'], - 'fluid.layers.sequence_reverse') + check_variable_and_dtype( + x, + 'x', + ['float32', 'float64', 'int8', 'int32', 'int64'], + 'fluid.layers.sequence_reverse', + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="sequence_reverse", - inputs={"X": x}, - outputs={"Y": out}, - attrs=dict()) + helper.append_op( + type="sequence_reverse", + inputs={"X": x}, + outputs={"Y": out}, + attrs=dict(), + ) return out diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 5e82bef749bb3..a282fae027f24 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -19,14 +19,28 @@ from ..layer_helper import LayerHelper from ..param_attr import ParamAttr from ..initializer import Initializer -from ..framework import _current_expected_place, convert_np_dtype_to_dtype_, _non_static_mode, _varbase_creator, device_guard, _in_legacy_dygraph, in_dygraph_mode, _get_paddle_place +from ..framework import ( + _current_expected_place, + convert_np_dtype_to_dtype_, + _non_static_mode, + _varbase_creator, + device_guard, + _in_legacy_dygraph, + in_dygraph_mode, + _get_paddle_place, +) from ..framework import Variable from ..initializer import Constant from ..core import VarDesc from .. import core from .layer_function_generator import templatedoc from . import utils -from ..data_feeder import check_variable_and_dtype, check_type, check_dtype, convert_dtype +from ..data_feeder import ( + check_variable_and_dtype, + check_type, + check_dtype, + convert_dtype, +) from paddle.utils import deprecated from .utils import check_shape @@ -83,24 +97,32 @@ def create_tensor(dtype, name=None, persistable=False): import paddle.fluid as fluid tensor = fluid.layers.create_tensor(dtype='float32') """ - check_dtype(dtype, 'dtype', [ - 'bool', 'float16', 'float32', 'float64', 'int8', 'int32', 'int32', - 'int64' - ], 'create_tensor') + check_dtype( + dtype, + 'dtype', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int8', + 'int32', + 'int32', + 'int64', + ], + 'create_tensor', + ) helper = LayerHelper("create_tensor", **locals()) - return helper.create_variable(name=helper.name, - dtype=dtype, - persistable=persistable) + return helper.create_variable( + name=helper.name, dtype=dtype, persistable=persistable + ) -def create_parameter(shape, - dtype, - name=None, - attr=None, - is_bias=False, - default_initializer=None): +def create_parameter( + shape, dtype, name=None, attr=None, is_bias=False, default_initializer=None +): """ - :api_attr: Static Graph + :api_attr: Static Graph This function creates a parameter. The parameter is a learnable variable, which can have gradient, and can be optimized. @@ -132,31 +154,55 @@ def create_parameter(shape, """ check_type(shape, 'shape', (list, tuple, numpy.ndarray), 'create_parameter') for item in shape: - check_type(item, 'item of shape', - (int, numpy.uint8, numpy.int8, numpy.int16, numpy.int32, - numpy.int64), 'create_parameter') - - check_dtype(dtype, 'dtype', [ - 'bool', 'float16', 'float32', 'float64', 'int8', 'int16', 'int32', - 'int64', 'uint8' - ], 'create_parameter') + check_type( + item, + 'item of shape', + ( + int, + numpy.uint8, + numpy.int8, + numpy.int16, + numpy.int32, + numpy.int64, + ), + 'create_parameter', + ) + + check_dtype( + dtype, + 'dtype', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int8', + 'int16', + 'int32', + 'int64', + 'uint8', + ], + 'create_parameter', + ) check_type(attr, 'attr', (type(None), ParamAttr), 'create_parameter') - check_type(default_initializer, 'default_initializer', - (type(None), Initializer), 'create_parameter') + check_type( + default_initializer, + 'default_initializer', + (type(None), Initializer), + 'create_parameter', + ) helper = LayerHelper("create_parameter", **locals()) if attr is None: attr = ParamAttr(name=name) - return helper.create_parameter(attr, shape, convert_dtype(dtype), is_bias, - default_initializer) + return helper.create_parameter( + attr, shape, convert_dtype(dtype), is_bias, default_initializer + ) -def create_global_var(shape, - value, - dtype, - persistable=False, - force_cpu=False, - name=None): +def create_global_var( + shape, value, dtype, persistable=False, force_cpu=False, name=None +): """ This function creates a new tensor variable with value in the global block(block 0). @@ -183,35 +229,53 @@ def create_global_var(shape, var = paddle.static.create_global_var(shape=[2,3], value=1.0, dtype='float32', persistable=True, force_cpu=True, name='new_var') """ - check_type(shape, 'shape', (list, tuple, numpy.ndarray), - 'create_global_var') + check_type( + shape, 'shape', (list, tuple, numpy.ndarray), 'create_global_var' + ) for item in shape: - check_type(item, 'item of shape', - (int, numpy.uint8, numpy.int8, numpy.int16, numpy.int32, - numpy.int64), 'create_global_var') - - check_dtype(dtype, 'dtype', [ - 'bool', - 'float16', - 'float32', - 'float64', - 'int8', - 'int16', - 'int32', - 'int64', - 'uint8', - 'uint16', - ], 'create_global_var') + check_type( + item, + 'item of shape', + ( + int, + numpy.uint8, + numpy.int8, + numpy.int16, + numpy.int32, + numpy.int64, + ), + 'create_global_var', + ) + + check_dtype( + dtype, + 'dtype', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int8', + 'int16', + 'int32', + 'int64', + 'uint8', + 'uint16', + ], + 'create_global_var', + ) helper = LayerHelper("global_var", **locals()) - var = helper.create_global_variable(dtype=dtype, - shape=shape, - persistable=persistable, - name=name, - stop_gradient=True) - helper.set_variable_initializer(var, - initializer=Constant(value=float(value), - force_cpu=force_cpu)) + var = helper.create_global_variable( + dtype=dtype, + shape=shape, + persistable=persistable, + name=name, + stop_gradient=True, + ) + helper.set_variable_initializer( + var, initializer=Constant(value=float(value), force_cpu=force_cpu) + ) return var @@ -251,25 +315,50 @@ def cast(x, dtype): out = _legacy_C_ops.cast(x, 'in_dtype', x.dtype, 'out_dtype', dtype) return out - check_variable_and_dtype(x, 'x', [ - 'bool', 'float16', 'float32', 'float64', 'int16', 'int32', 'int64', - 'uint8', 'uint16' - ], 'cast') - check_dtype(dtype, 'dtype', [ - 'bool', 'float16', 'float32', 'float64', 'int8', 'int16', 'int32', - 'int64', 'uint8', 'uint16' - ], 'cast') + check_variable_and_dtype( + x, + 'x', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int16', + 'int32', + 'int64', + 'uint8', + 'uint16', + ], + 'cast', + ) + check_dtype( + dtype, + 'dtype', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int8', + 'int16', + 'int32', + 'int64', + 'uint8', + 'uint16', + ], + 'cast', + ) helper = LayerHelper('cast', **locals()) out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=x.stop_gradient) - helper.append_op(type='cast', - inputs={'X': [x]}, - outputs={'Out': [out]}, - attrs={ - 'in_dtype': x.dtype, - 'out_dtype': out.dtype - }) + dtype=dtype, stop_gradient=x.stop_gradient + ) + helper.append_op( + type='cast', + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={'in_dtype': x.dtype, 'out_dtype': out.dtype}, + ) return out @@ -344,9 +433,11 @@ def concat(input, axis=0, name=None): if not isinstance(input, Variable): for id, x in enumerate(input): check_variable_and_dtype( - x, 'input[' + str(id) + ']', + x, + 'input[' + str(id) + ']', ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'concat') + 'concat', + ) if x.dtype != input[0].dtype: raise TypeError( "All the Tensors in the input must have the same data type." @@ -357,8 +448,11 @@ def concat(input, axis=0, name=None): if isinstance(axis, Variable): check_dtype( - axis.dtype, 'axis', ['int32', 'int64'], 'concat', - "The data type of axis must be int32 or int64 when axis is a Tensor" + axis.dtype, + 'axis', + ['int32', 'int64'], + 'concat', + "The data type of axis must be int32 or int64 when axis is a Tensor", ) helper = LayerHelper('concat', **locals()) @@ -369,19 +463,17 @@ def concat(input, axis=0, name=None): # This feature is supported for Dynamic-to-Static, because after transformed, the type of inputs[0] # is LOD_TENSOR_ARRAY in some scenarios. And this feature can be used in static mode. - assert len(input) == 1, "If the elements of 'input' in concat are Variable(LoDTensorArray), " \ - "number of the elements must be 1, but received %s." % len(input) + assert len(input) == 1, ( + "If the elements of 'input' in concat are Variable(LoDTensorArray), " + "number of the elements must be 1, but received %s." % len(input) + ) out_index = helper.create_variable_for_type_inference(dtype="int32") - helper.append_op(type='tensor_array_to_tensor', - inputs={'X': input[0]}, - outputs={ - 'Out': [out], - 'OutIndex': [out_index] - }, - attrs={ - 'axis': axis, - 'use_stack': False - }) + helper.append_op( + type='tensor_array_to_tensor', + inputs={'X': input[0]}, + outputs={'Out': [out], 'OutIndex': [out_index]}, + attrs={'axis': axis, 'use_stack': False}, + ) else: inputs = {'X': input} attrs = {} @@ -389,10 +481,9 @@ def concat(input, axis=0, name=None): axis.stop_gradient = True attrs['axis'] = axis - helper.append_op(type='concat', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs) + helper.append_op( + type='concat', inputs=inputs, outputs={'Out': [out]}, attrs=attrs + ) return out @@ -478,33 +569,36 @@ def tensor_array_to_tensor(input, axis=1, name=None, use_stack=False): """ if _non_static_mode(): assert isinstance( - input, list), "The 'input' in tensor_array_to_tensor must be list" + input, list + ), "The 'input' in tensor_array_to_tensor must be list" from .nn import stack, concat from ..dygraph import to_variable + op = stack if use_stack else concat res = op(input, axis=axis) sizes = to_variable( - numpy.array(list(map(lambda x: int(x.shape[axis]), input)))) + numpy.array(list(map(lambda x: int(x.shape[axis]), input))) + ) return res, sizes check_type(input, 'input', (list, Variable), 'tensor_array_to_tensor') if isinstance(input, list): for i, input_x in enumerate(input): - check_type(input_x, 'input[' + str(i) + ']', Variable, - 'tensor_array_to_tensor') + check_type( + input_x, + 'input[' + str(i) + ']', + Variable, + 'tensor_array_to_tensor', + ) helper = LayerHelper('tensor_array_to_tensor', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) out_index = helper.create_variable_for_type_inference(dtype="int32") - helper.append_op(type='tensor_array_to_tensor', - inputs={'X': input}, - outputs={ - 'Out': [out], - 'OutIndex': [out_index] - }, - attrs={ - 'axis': axis, - 'use_stack': use_stack - }) + helper.append_op( + type='tensor_array_to_tensor', + inputs={'X': input}, + outputs={'Out': [out], 'OutIndex': [out_index]}, + attrs={'axis': axis, 'use_stack': use_stack}, + ) return out, out_index @@ -561,25 +655,36 @@ def sums(input, out=None): check_type(input, 'input', (Variable, tuple, list), 'sums') if isinstance(input, list) or isinstance(input, tuple): for input_section in input: - check_variable_and_dtype(input_section, "input", \ - ['float16', 'float32', 'float64', 'int32', 'int64'], 'sums') + check_variable_and_dtype( + input_section, + "input", + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'sums', + ) else: - check_variable_and_dtype(input, "input", \ - ['float16', 'float32', 'float64', 'int32', 'int64'], 'sums') + check_variable_and_dtype( + input, + "input", + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'sums', + ) helper = LayerHelper('sum', **locals()) if out is None: out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) else: - check_variable_and_dtype(out, "out", - ['float32', 'float64', 'int32', 'int64'], - 'sums') - - helper.append_op(type='sum', - inputs={'X': input}, - outputs={'Out': out}, - attrs={'use_mkldnn': False}) + check_variable_and_dtype( + out, "out", ['float32', 'float64', 'int32', 'int64'], 'sums' + ) + + helper.append_op( + type='sum', + inputs={'X': input}, + outputs={'Out': out}, + attrs={'use_mkldnn': False}, + ) return out @@ -614,9 +719,12 @@ def assign(input, output=None): result3 = paddle.assign(np.array([[2.5, 2.5], [2.5, 2.5], [2.5, 2.5]], dtype='float32')) # result3 = [[2.5, 2.5], [2.5, 2.5], [2.5, 2.5]] """ helper = LayerHelper('assign', **locals()) - check_type(input, 'input', - (Variable, numpy.ndarray, list, tuple, float, int, bool), - 'assign') + check_type( + input, + 'input', + (Variable, numpy.ndarray, list, tuple, float, int, bool), + 'assign', + ) is_inplace = True if output is not None else False if numpy.isscalar(input) and not isinstance(input, str): @@ -642,16 +750,29 @@ def assign(input, output=None): output = core.eager.Tensor() _legacy_C_ops.assign(input, output) else: - check_dtype(input.dtype, 'input', [ - 'float16', 'uint16', 'float32', 'float64', 'int32', 'int64', - 'uint8', 'bool' - ], 'assign', '(When the type of input in assign is Variable.)') + check_dtype( + input.dtype, + 'input', + [ + 'float16', + 'uint16', + 'float32', + 'float64', + 'int32', + 'int64', + 'uint8', + 'bool', + ], + 'assign', + '(When the type of input in assign is Variable.)', + ) if output is None: output = helper.create_variable_for_type_inference( - dtype=input.dtype) - helper.append_op(type='assign', - inputs={'X': [input]}, - outputs={'Out': [output]}) + dtype=input.dtype + ) + helper.append_op( + type='assign', inputs={'X': [input]}, outputs={'Out': [output]} + ) elif isinstance(input, numpy.ndarray): # Not support [var, var, ...] currently. if len(input.shape) > 0 and any(isinstance(x, Variable) for x in input): @@ -665,7 +786,8 @@ def assign(input, output=None): warnings.warn( "paddle.assign doesn't support float64 input now due " "to current platform protobuf data limitation, we convert " - "it to float32") + "it to float32" + ) dtype = VarDesc.VarType.FP32 if dtype == VarDesc.VarType.BOOL: value_name = "bool_values" @@ -683,31 +805,49 @@ def assign(input, output=None): raise TypeError( "When the type of 'input' in assign is numpy.ndarray, " "the data type of 'input' must be bool, float32, int32 or int64, but " - "received %s." % convert_dtype(dtype)) + "received %s." % convert_dtype(dtype) + ) if input.size > 1024 * 1024: - raise ValueError("The size of input is too big. Please consider " - "saving it to file and 'load_op' to load it") + raise ValueError( + "The size of input is too big. Please consider " + "saving it to file and 'load_op' to load it" + ) if in_dygraph_mode(): if output is None: output = zeros(list(input.shape), dtype) - _C_ops.assign_value_(output, list(input.shape), dtype, values, - _current_expected_place()) + _C_ops.assign_value_( + output, + list(input.shape), + dtype, + values, + _current_expected_place(), + ) elif _in_legacy_dygraph(): if output is None: output = core.VarBase() - _legacy_C_ops.assign_value(output, 'shape', list(input.shape), - 'dtype', dtype, value_name, values) + _legacy_C_ops.assign_value( + output, + 'shape', + list(input.shape), + 'dtype', + dtype, + value_name, + values, + ) else: if output is None: output = helper.create_variable_for_type_inference( - dtype=input.dtype) - helper.append_op(type='assign_value', - outputs={'Out': [output]}, - attrs={ - 'dtype': dtype, - 'shape': list(input.shape), - value_name: values - }) + dtype=input.dtype + ) + helper.append_op( + type='assign_value', + outputs={'Out': [output]}, + attrs={ + 'dtype': dtype, + 'shape': list(input.shape), + value_name: values, + }, + ) if is_inplace and _non_static_mode(): output._bump_inplace_version() @@ -783,7 +923,11 @@ def fill_constant(shape, dtype, value, force_cpu=False, out=None, name=None): shape = list( map( lambda x: x.numpy().flat[0] - if isinstance(x, Variable) else x, shape)) + if isinstance(x, Variable) + else x, + shape, + ) + ) break if not isinstance(dtype, core.VarDesc.VarType): @@ -811,9 +955,19 @@ def fill_constant(shape, dtype, value, force_cpu=False, out=None, name=None): else: attrs['str_value'] = str(float(value.numpy().item(0))) - _legacy_C_ops.fill_constant(out, 'value', float(value), 'force_cpu', - force_cpu, 'dtype', out.dtype, 'str_value', - attrs['str_value'], 'shape', shape) + _legacy_C_ops.fill_constant( + out, + 'value', + float(value), + 'force_cpu', + force_cpu, + 'dtype', + out.dtype, + 'str_value', + attrs['str_value'], + 'shape', + shape, + ) out.stop_gradient = True return out @@ -825,43 +979,60 @@ def fill_constant(shape, dtype, value, force_cpu=False, out=None, name=None): inputs['ValueTensor'] = value check_shape(shape) - check_dtype(dtype, 'dtype', [ - 'bool', 'float16', 'float32', 'float64', 'uint8', 'int16', 'int32', - 'int64', 'complex64', 'complex128' - ], 'fill_constant') + check_dtype( + dtype, + 'dtype', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'uint8', + 'int16', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'fill_constant', + ) check_type(shape, 'shape', (Variable, list, tuple), 'fill_constant') if out is not None: - check_variable_and_dtype(out, 'out', [convert_dtype(dtype)], - 'fill_constant') + check_variable_and_dtype( + out, 'out', [convert_dtype(dtype)], 'fill_constant' + ) helper = LayerHelper("fill_constant", **locals()) - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type='fill_constant') + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type='fill_constant' + ) if out is None: out = helper.create_variable_for_type_inference(dtype=dtype) attrs['dtype'] = out.dtype - helper.append_op(type='fill_constant', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs, - stop_gradient=True) + helper.append_op( + type='fill_constant', + inputs=inputs, + outputs={'Out': [out]}, + attrs=attrs, + stop_gradient=True, + ) out.stop_gradient = True return out @deprecated(since='1.8.0', update_to="paddle.fluid.layers.fill_constant") @templatedoc() -def fill_constant_batch_size_like(input, - shape, - dtype, - value, - input_dim_idx=0, - output_dim_idx=0, - force_cpu=False): +def fill_constant_batch_size_like( + input, + shape, + dtype, + value, + input_dim_idx=0, + output_dim_idx=0, + force_cpu=False, +): """ This OP creates a Tesnor according the shape and dtype, and initializes the Tensor with the constants provided in ``value``. When the input is LoDTensor @@ -903,8 +1074,9 @@ def fill_constant_batch_size_like(input, place = _current_expected_place() if force_cpu: place = core.CPUPlace() - out = _C_ops.full_batch_size_like(input, shape, dtype, value, - input_dim_idx, output_dim_idx, place) + out = _C_ops.full_batch_size_like( + input, shape, dtype, value, input_dim_idx, output_dim_idx, place + ) out.stop_gradient = True return out @@ -916,25 +1088,27 @@ def fill_constant_batch_size_like(input, 'value': float(value), 'input_dim_idx': input_dim_idx, 'output_dim_idx': output_dim_idx, - 'force_cpu': force_cpu + 'force_cpu': force_cpu, } if convert_dtype(dtype) in ['int64', 'int32']: attrs['str_value'] = str(int(value)) else: attrs['str_value'] = str(float(value)) - helper.append_op(type='fill_constant_batch_size_like', - inputs={'Input': input}, - outputs={'Out': [out]}, - attrs=attrs) + helper.append_op( + type='fill_constant_batch_size_like', + inputs={'Input': input}, + outputs={'Out': [out]}, + attrs=attrs, + ) out.stop_gradient = True return out def argmin(x, axis=0): """ - :alias_main: paddle.argmin - :alias: paddle.argmin,paddle.tensor.argmin,paddle.tensor.search.argmin - :old_api: paddle.fluid.layers.argmin + :alias_main: paddle.argmin + :alias: paddle.argmin,paddle.tensor.argmin,paddle.tensor.search.argmin + :old_api: paddle.fluid.layers.argmin **argmin** @@ -984,14 +1158,19 @@ def argmin(x, axis=0): # [1 0 2]] """ check_variable_and_dtype( - x, 'x', ['float32', 'float64', 'uint8', 'int16', 'int32', 'int64'], - 'argmin') + x, + 'x', + ['float32', 'float64', 'uint8', 'int16', 'int32', 'int64'], + 'argmin', + ) helper = LayerHelper("arg_min", **locals()) out = helper.create_variable_for_type_inference(VarDesc.VarType.INT64) - helper.append_op(type='arg_min', - inputs={'X': x}, - outputs={'Out': [out]}, - attrs={'axis': axis}) + helper.append_op( + type='arg_min', + inputs={'X': x}, + outputs={'Out': [out]}, + attrs={'axis': axis}, + ) out.stop_gradient = True return out @@ -1046,23 +1225,28 @@ def argmax(x, axis=0): # [0 3 1]] """ check_variable_and_dtype( - x, 'x', ['float32', 'float64', 'uint8', 'int16', 'int32', 'int64'], - 'argmax') + x, + 'x', + ['float32', 'float64', 'uint8', 'int16', 'int32', 'int64'], + 'argmax', + ) helper = LayerHelper("arg_max", **locals()) out = helper.create_variable_for_type_inference(VarDesc.VarType.INT64) - helper.append_op(type='arg_max', - inputs={'X': x}, - outputs={'Out': [out]}, - attrs={'axis': axis}) + helper.append_op( + type='arg_max', + inputs={'X': x}, + outputs={'Out': [out]}, + attrs={'axis': axis}, + ) out.stop_gradient = True return out def argsort(input, axis=-1, descending=False, name=None): """ - :alias_main: paddle.argsort - :alias: paddle.argsort,paddle.tensor.argsort,paddle.tensor.search.argsort - :old_api: paddle.fluid.layers.argsort + :alias_main: paddle.argsort + :alias: paddle.argsort,paddle.tensor.argsort,paddle.tensor.search.argsort + :old_api: paddle.fluid.layers.argsort This OP sorts the input along the given axis, and returns sorted output data Varibale and its corresponding index Variable with the same shape as @@ -1133,23 +1317,24 @@ def argsort(input, axis=-1, descending=False, name=None): # [5. 7. 7. 9.]]] """ check_variable_and_dtype( - input, 'input', - ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], 'argsort') + input, + 'input', + ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], + 'argsort', + ) helper = LayerHelper("argsort", **locals()) - out = helper.create_variable_for_type_inference(dtype=input.dtype, - stop_gradient=True) - ids = helper.create_variable_for_type_inference(VarDesc.VarType.INT64, - stop_gradient=True) - helper.append_op(type='argsort', - inputs={'X': input}, - outputs={ - 'Out': out, - 'Indices': ids - }, - attrs={ - 'axis': axis, - 'descending': descending - }) + out = helper.create_variable_for_type_inference( + dtype=input.dtype, stop_gradient=True + ) + ids = helper.create_variable_for_type_inference( + VarDesc.VarType.INT64, stop_gradient=True + ) + helper.append_op( + type='argsort', + inputs={'X': input}, + outputs={'Out': out, 'Indices': ids}, + attrs={'axis': axis, 'descending': descending}, + ) return out, ids @@ -1215,9 +1400,9 @@ def zeros(shape, dtype, force_cpu=False, name=None): def reverse(x, axis): """ - :alias_main: paddle.reverse - :alias: paddle.reverse,paddle.tensor.reverse,paddle.tensor.manipulation.reverse - :old_api: paddle.fluid.layers.reverse + :alias_main: paddle.reverse + :alias: paddle.reverse,paddle.tensor.reverse,paddle.tensor.manipulation.reverse + :old_api: paddle.fluid.layers.reverse The OP reverses the tensor :attr:`x` along the given :attr:`axis`. @@ -1275,9 +1460,9 @@ def reverse(x, axis): reversed_tensor_array = fluid.layers.reverse(tensor_array, 0) # {[[3, 4, 5]], [[0, 1, 2]]} """ - check_variable_and_dtype(x, 'x', - ('float32', 'float64', 'int32', 'int64', 'uint8'), - 'reverse') + check_variable_and_dtype( + x, 'x', ('float32', 'float64', 'int32', 'int64', 'uint8'), 'reverse' + ) check_type(axis, 'axis', (int, tuple, list, Variable), 'reverse') if isinstance(axis, int): axis = [axis] @@ -1285,10 +1470,12 @@ def reverse(x, axis): return _C_ops.reverse(x, axis) helper = LayerHelper("reverse", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reverse', - inputs={'X': x}, - outputs={'Out': [out]}, - attrs={'axis': axis}) + helper.append_op( + type='reverse', + inputs={'X': x}, + outputs={'Out': [out]}, + attrs={'axis': axis}, + ) return out @@ -1304,13 +1491,12 @@ def save(x, file_path, overwrite=True): error will be thrown. """ helper = LayerHelper("save", **locals()) - helper.append_op(type="save", - inputs={"input": x}, - outputs={}, - args={ - "file_path": file_path, - "overwrite": overwrite - }) + helper.append_op( + type="save", + inputs={"input": x}, + outputs={}, + args={"file_path": file_path, "overwrite": overwrite}, + ) def save_combine(x, file_path, overwrite=True): @@ -1342,13 +1528,12 @@ def save_combine(x, file_path, overwrite=True): normed = fluid.layers.save_combine([v1, v2], file_path="output") """ helper = LayerHelper("save_combine", **locals()) - helper.append_op(type="save_combine", - inputs={"input": x}, - outputs={}, - args={ - "file_path": file_path, - "overwrite": overwrite - }) + helper.append_op( + type="save_combine", + inputs={"input": x}, + outputs={}, + args={"file_path": file_path, "overwrite": overwrite}, + ) def load_combine(out, file_path): @@ -1360,10 +1545,12 @@ def load_combine(out, file_path): file_path(str): The path of the disk file. """ helper = LayerHelper("load_combine", **locals()) - helper.append_op(type="load_combine", - inputs={}, - output={"Out": out}, - args={"file_path": file_path}) + helper.append_op( + type="load_combine", + inputs={}, + output={"Out": out}, + args={"file_path": file_path}, + ) def has_inf(x): @@ -1447,8 +1634,9 @@ def isfinite(x): print(y) """ - check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"], - "isfinite") + check_variable_and_dtype( + x, "x", ["float32", "float64", "int32", "int64"], "isfinite" + ) helper = LayerHelper("isfinite", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') @@ -1506,8 +1694,11 @@ def range(start, end, step, dtype, name=None): """ out_shape = None - if not isinstance(start, Variable) and not isinstance( - end, Variable) and not isinstance(step, Variable): + if ( + not isinstance(start, Variable) + and not isinstance(end, Variable) + and not isinstance(step, Variable) + ): out_shape = [int(math.ceil((end - start) / step))] if not isinstance(dtype, core.VarDesc.VarType): @@ -1539,17 +1730,16 @@ def range(start, end, step, dtype, name=None): out.stop_gradient = True return out - check_dtype(dtype, 'dtype', ['float32', 'float64', 'int32', 'int64'], - 'range/arange') + check_dtype( + dtype, 'dtype', ['float32', 'float64', 'int32', 'int64'], 'range/arange' + ) helper = LayerHelper('range', **locals()) out = helper.create_variable_for_type_inference(dtype, shape=out_shape) - helper.append_op(type='range', - inputs={ - 'Start': start, - 'End': end, - 'Step': step - }, - outputs={'Out': out}) + helper.append_op( + type='range', + inputs={'Start': start, 'End': end, 'Step': step}, + outputs={'Out': out}, + ) out.stop_gradient = True if out_shape is not None: out.desc.set_shape(out_shape) @@ -1604,52 +1794,70 @@ def linspace(start, stop, num, dtype=None, name=None): with device_guard("cpu"): tensor_num = fill_constant([1], 'int32', num) if in_dygraph_mode(): - return _C_ops.linspace(tensor_start, tensor_stop, tensor_num, dtype, - _current_expected_place()) + return _C_ops.linspace( + tensor_start, + tensor_stop, + tensor_num, + dtype, + _current_expected_place(), + ) if _in_legacy_dygraph(): - return _legacy_C_ops.linspace(tensor_start, tensor_stop, tensor_num, - 'dtype', dtype) + return _legacy_C_ops.linspace( + tensor_start, tensor_stop, tensor_num, 'dtype', dtype + ) helper = LayerHelper("linspace", **locals()) start_dtype = convert_dtype(tensor_start.dtype) stop_dtype = convert_dtype(tensor_stop.dtype) out_dtype = convert_dtype(dtype) if isinstance(start, Variable): - check_dtype(start.dtype, 'start', - ['float32', 'float64', 'int32', 'int64'], 'linspace') + check_dtype( + start.dtype, + 'start', + ['float32', 'float64', 'int32', 'int64'], + 'linspace', + ) else: check_type(start, 'start', (int, float), 'linspace') if isinstance(stop, Variable): - check_dtype(stop.dtype, 'stop', - ['float32', 'float64', 'int32', 'int64'], 'linspace') + check_dtype( + stop.dtype, + 'stop', + ['float32', 'float64', 'int32', 'int64'], + 'linspace', + ) else: check_type(stop, 'stop', (int, float), 'linspace') if isinstance(num, Variable): check_dtype(num.dtype, 'num', ['int32'], 'linspace') - check_dtype(dtype, 'dtype', ['int32', 'int64', 'float32', 'float64'], - 'linspace') - if ((stop_dtype == "float64" or start_dtype == "float64") - and out_dtype in ["float32", "int32"]) or ( - (stop_dtype == "int64" or start_dtype == "int64") - and out_dtype == "int32"): + check_dtype( + dtype, 'dtype', ['int32', 'int64', 'float32', 'float64'], 'linspace' + ) + if ( + (stop_dtype == "float64" or start_dtype == "float64") + and out_dtype in ["float32", "int32"] + ) or ( + (stop_dtype == "int64" or start_dtype == "int64") + and out_dtype == "int32" + ): raise ValueError( "The dtype of start/stop is {}/{} but the attr(dtype) of linspace is {}, " - "which may cause data type overflows. Please reset attr(dtype) of linspace." - .format(start_dtype, stop_dtype, dtype)) + "which may cause data type overflows. Please reset attr(dtype) of linspace.".format( + start_dtype, stop_dtype, dtype + ) + ) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='linspace', - inputs={ - 'Start': tensor_start, - 'Stop': tensor_stop, - 'Num': tensor_num - }, - attrs={'dtype': dtype}, - outputs={'Out': [out]}) + helper.append_op( + type='linspace', + inputs={'Start': tensor_start, 'Stop': tensor_stop, 'Num': tensor_num}, + attrs={'dtype': dtype}, + outputs={'Out': [out]}, + ) if isinstance(num, int): - out.desc.set_shape((num, )) + out.desc.set_shape((num,)) return out @@ -1679,23 +1887,25 @@ def zeros_like(x, out=None): data = fluid.layers.zeros_like(x) # [0.0, 0.0, 0.0] """ - check_variable_and_dtype(x, "x", - ['bool', 'float32', 'float64', 'int32', 'int64'], - 'zeros_like') + check_variable_and_dtype( + x, "x", ['bool', 'float32', 'float64', 'int32', 'int64'], 'zeros_like' + ) helper = LayerHelper("zeros_like", **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) else: check_variable_and_dtype( - out, "out", ['bool', 'float32', 'float64', 'int32', 'int64'], - 'zeros_like') - helper.append_op(type='fill_any_like', - inputs={'X': [x]}, - attrs={ - 'value': 0, - "dtype": x.dtype - }, - outputs={'Out': [out]}) + out, + "out", + ['bool', 'float32', 'float64', 'int32', 'int64'], + 'zeros_like', + ) + helper.append_op( + type='fill_any_like', + inputs={'X': [x]}, + attrs={'value': 0, "dtype": x.dtype}, + outputs={'Out': [out]}, + ) out.stop_gradient = True return out @@ -1732,8 +1942,12 @@ def diag(diagonal): """ check_type(diagonal, 'diagonal', (Variable, numpy.ndarray), 'diag') - check_dtype(diagonal.dtype, 'diagonal', - ['float32', 'float64', 'int32', 'int64'], 'diag') + check_dtype( + diagonal.dtype, + 'diagonal', + ['float32', 'float64', 'int32', 'int64'], + 'diag', + ) helper = LayerHelper("diag", **locals()) if not isinstance(diagonal, Variable): @@ -1741,19 +1955,17 @@ def diag(diagonal): out = helper.create_variable_for_type_inference(dtype=diagonal.dtype) - helper.append_op(type='diag', - inputs={'Diagonal': [diagonal]}, - outputs={'Out': [out]}) + helper.append_op( + type='diag', inputs={'Diagonal': [diagonal]}, outputs={'Out': [out]} + ) out.stop_gradient = True return out -def eye(num_rows, - num_columns=None, - batch_shape=None, - dtype='float32', - name=None): +def eye( + num_rows, num_columns=None, batch_shape=None, dtype='float32', name=None +): """ This function constructs a or a batch of 2-D tensor with ones on the diagonal and zeros elsewhere. @@ -1806,25 +2018,33 @@ def _check_attr(attr, message): num_columns = num_rows if in_dygraph_mode(): - out = _C_ops.eye(num_rows, num_columns, dtype, - _current_expected_place()) + out = _C_ops.eye( + num_rows, num_columns, dtype, _current_expected_place() + ) elif _in_legacy_dygraph(): - out = _legacy_C_ops.eye('dtype', dtype, 'num_rows', num_rows, - 'num_columns', num_columns) + out = _legacy_C_ops.eye( + 'dtype', dtype, 'num_rows', num_rows, 'num_columns', num_columns + ) else: helper = LayerHelper("eye", **locals()) - check_dtype(dtype, 'dtype', - ['float16', 'float32', 'float64', 'int32', 'int64'], 'eye') + check_dtype( + dtype, + 'dtype', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'eye', + ) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='eye', - inputs={}, - outputs={'Out': [out]}, - attrs={ - 'num_rows': num_rows, - 'num_columns': num_columns, - 'dtype': dtype - }, - stop_gradient=True) + helper.append_op( + type='eye', + inputs={}, + outputs={'Out': [out]}, + attrs={ + 'num_rows': num_rows, + 'num_columns': num_columns, + 'dtype': dtype, + }, + stop_gradient=True, + ) if batch_shape is not None: re_shape = [1] * len(batch_shape) @@ -1836,11 +2056,12 @@ def _check_attr(attr, message): if not isinstance(batch_shape, list): raise TypeError("batch_shape should be a list") - for batch_val in (batch_shape): + for batch_val in batch_shape: if batch_val <= 0: raise TypeError("batch_shape should be a positive int list") from .nn import reshape, expand + out = reshape(x=out, shape=re_shape) out = expand(x=out, expand_times=expand_times) @@ -1871,25 +2092,31 @@ def ones_like(x, out=None): data = fluid.layers.ones_like(x) # [1.0, 1.0, 1.0] """ - check_variable_and_dtype(x, "x", - ['bool', 'float32', 'float64', 'int32', 'int64'], - 'ones_like') + check_variable_and_dtype( + x, "x", ['bool', 'float32', 'float64', 'int32', 'int64'], 'ones_like' + ) helper = LayerHelper("ones_like", **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) else: check_variable_and_dtype( - out, "out", ['bool', 'float32', 'float64', 'int32', 'int64'], - 'ones_like') - helper.append_op(type='fill_any_like', - inputs={'X': [x]}, - attrs={'value': 1.0}, - outputs={'Out': [out]}) + out, + "out", + ['bool', 'float32', 'float64', 'int32', 'int64'], + 'ones_like', + ) + helper.append_op( + type='fill_any_like', + inputs={'X': [x]}, + attrs={'value': 1.0}, + outputs={'Out': [out]}, + ) return out @deprecated(since="2.0.0", update_to="paddle.triu") def triu(input, diagonal=0, name=None): import paddle + return paddle.tensor.triu(x=input, diagonal=diagonal, name=name) diff --git a/python/paddle/fluid/layers/utils.py b/python/paddle/fluid/layers/utils.py index 28d00161b26eb..e96e105c141df 100644 --- a/python/paddle/fluid/layers/utils.py +++ b/python/paddle/fluid/layers/utils.py @@ -17,9 +17,15 @@ import six import numpy as np from ..framework import Block, Variable, _non_static_mode -from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) from ..layer_helper import LayerHelper from sys import version_info + try: from collections.abc import Sequence except: @@ -54,25 +60,45 @@ def convert_to_list(value, n, name, dtype=int): try: value_list = list(value) except TypeError: - raise ValueError("The " + name + - "'s type must be list or tuple. Received: " + - str(value)) + raise ValueError( + "The " + + name + + "'s type must be list or tuple. Received: " + + str(value) + ) if len(value_list) != n: - raise ValueError("The " + name + "'s length must be " + str(n) + - ". Received: " + str(value)) + raise ValueError( + "The " + + name + + "'s length must be " + + str(n) + + ". Received: " + + str(value) + ) for single_value in value_list: - assert not isinstance( - single_value, Variable - ), "Required numerical type with '%s', but received Tensor." % dtype + assert not isinstance(single_value, Variable), ( + "Required numerical type with '%s', but received Tensor." + % dtype + ) try: dtype(single_value) except (ValueError, TypeError): - raise ValueError("The " + name + - "'s type must be a list or tuple of " + - str(n) + " " + str(dtype) + " . Received: " + - str(value) + " " - "including element " + str(single_value) + - " of type" + " " + str(type(single_value))) + raise ValueError( + "The " + + name + + "'s type must be a list or tuple of " + + str(n) + + " " + + str(dtype) + + " . Received: " + + str(value) + + " " + "including element " + + str(single_value) + + " of type" + + " " + + str(type(single_value)) + ) return value_list @@ -82,7 +108,7 @@ def is_sequence(seq): """ if isinstance(seq, dict): return True - return (isinstance(seq, Sequence) and not isinstance(seq, str)) + return isinstance(seq, Sequence) and not isinstance(seq, str) def _hash_with_id(*args): @@ -91,7 +117,7 @@ def _hash_with_id(*args): """ assert len(args) > 0 info = tuple([id(v) for v in args]) - return hash(info) & 0xfffffff + return hash(info) & 0xFFFFFFF def _sorted(dict_): @@ -136,9 +162,9 @@ def to_sequence(nest): def flatten(nest): """ - :alias_main: paddle.flatten - :alias: paddle.flatten,paddle.tensor.flatten,paddle.tensor.manipulation.flatten - :old_api: paddle.fluid.layers.flatten + :alias_main: paddle.flatten + :alias: paddle.flatten,paddle.tensor.flatten,paddle.tensor.manipulation.flatten + :old_api: paddle.fluid.layers.flatten Traverse all entries in the nested structure and put them into an list. """ @@ -160,9 +186,12 @@ def _sequence_like(instance, args): # corresponding `OrderedDict` to pack it back). result = dict(zip(_sorted(instance), args)) return type(instance)((key, result[key]) for key in instance.keys()) - elif (isinstance(instance, tuple) and hasattr(instance, "_fields") - and isinstance(instance._fields, Sequence) - and all(isinstance(f, str) for f in instance._fields)): + elif ( + isinstance(instance, tuple) + and hasattr(instance, "_fields") + and isinstance(instance._fields, Sequence) + and all(isinstance(f, str) for f in instance._fields) + ): # This is a namedtuple return type(instance)(*args) else: @@ -195,15 +224,22 @@ def pack_sequence_as(structure, flat_sequence): if not is_sequence(structure): if len(flat_sequence) != 1: raise ValueError( - "Structure is a scalar but len(flat_sequence) == %d > 1" % - len(flat_sequence)) + "Structure is a scalar but len(flat_sequence) == %d > 1" + % len(flat_sequence) + ) return flat_sequence[0] flat_structure = flatten(structure) if len(flat_structure) != len(flat_sequence): raise ValueError( "Could not pack sequence. Structure had %d elements, but flat_sequence " - "had %d elements. Structure: %s, flat_sequence: %s." % - (len(flat_structure), len(flat_sequence), structure, flat_sequence)) + "had %d elements. Structure: %s, flat_sequence: %s." + % ( + len(flat_structure), + len(flat_sequence), + structure, + flat_sequence, + ) + ) _, packed = _packed_nest_with_indices(structure, flat_sequence, 0) return _sequence_like(structure, packed) @@ -243,7 +279,8 @@ def _recursive_assert_same_structure(nest1, nest2, check_types): if is_sequence_nest1 != is_sequence(nest2): raise ValueError( "The two structures don't have the same nested structure.\n\n" - "First structure: %s\n\nSecond structure: %s." % (nest1, nest2)) + "First structure: %s\n\nSecond structure: %s." % (nest1, nest2) + ) if not is_sequence_nest1: return # finished checking if check_types: @@ -252,16 +289,19 @@ def _recursive_assert_same_structure(nest1, nest2, check_types): if type_nest1 != type_nest2: raise TypeError( "The two structures don't have the same sequence type. First " - "structure has type %s, while second structure has type %s." % - (type_nest1, type_nest2)) + "structure has type %s, while second structure has type %s." + % (type_nest1, type_nest2) + ) if isinstance(nest1, dict): keys1 = set(nest1.keys()) keys2 = set(nest2.keys()) if keys1 != keys2: raise ValueError( "The two dictionaries don't have the same set of keys. First " - "structure has keys {}, while second structure has keys {}." - .format(keys1, keys2)) + "structure has keys {}, while second structure has keys {}.".format( + keys1, keys2 + ) + ) nest1_as_sequence = [n for n in _yield_value(nest1)] nest2_as_sequence = [n for n in _yield_value(nest2)] for n1, n2 in zip(nest1_as_sequence, nest2_as_sequence): @@ -269,16 +309,16 @@ def _recursive_assert_same_structure(nest1, nest2, check_types): def padding_to_same_structure(nest1, nest2, obj=None): - def _padding_to_same_structure_single(value, obj): - def change_none_to_obj(x): - if x is None: return obj + if x is None: + return obj return x if is_sequence(value): value = pack_sequence_as( - value, [change_none_to_obj(item) for item in flatten(value)]) + value, [change_none_to_obj(item) for item in flatten(value)] + ) else: value = change_none_to_obj(value) return value @@ -295,10 +335,12 @@ def assert_same_structure(nest1, nest2, check_types=True): len_nest1 = len(flatten(nest1)) if is_sequence(nest1) else 1 len_nest2 = len(flatten(nest2)) if is_sequence(nest2) else 1 if len_nest1 != len_nest2: - raise ValueError("The two structures don't have the same number of " - "elements.\n\nFirst structure (%i elements): %s\n\n" - "Second structure (%i elements): %s" % - (len_nest1, nest1, len_nest2, nest2)) + raise ValueError( + "The two structures don't have the same number of " + "elements.\n\nFirst structure (%i elements): %s\n\n" + "Second structure (%i elements): %s" + % (len_nest1, nest1, len_nest2, nest2) + ) _recursive_assert_same_structure(nest1, nest2, check_types) @@ -343,9 +385,12 @@ def _get_shape_tensor(list_shape): if isinstance(dim, Variable): dim.stop_gradient = True check_dtype( - dim.dtype, 'shape[' + str(idx) + ']', ['int32', 'int64'], + dim.dtype, + 'shape[' + str(idx) + ']', + ['int32', 'int64'], op_type, - '(When type of shape in' + op_type + 'is list or tuple.)') + '(When type of shape in' + op_type + 'is list or tuple.)', + ) if convert_dtype(dim.dtype) == 'int64': dim = cast(x=dim, dtype='int32') shape_tensor_list.append(dim) @@ -356,9 +401,14 @@ def _get_shape_tensor(list_shape): if isinstance(shape, Variable): shape.stop_gradient = True - check_dtype(shape.dtype, 'shape', ['int32', 'int64'], 'fill_constant', - '(When type of shape in' + op_type + ' is Variable.)') - if (convert_dtype(shape.dtype) == 'int64'): + check_dtype( + shape.dtype, + 'shape', + ['int32', 'int64'], + 'fill_constant', + '(When type of shape in' + op_type + ' is Variable.)', + ) + if convert_dtype(shape.dtype) == 'int64': shape = cast(shape, 'int32') inputs["ShapeTensor"] = shape elif isinstance(shape, (list, tuple)): @@ -374,6 +424,7 @@ def _convert_to_tensor_list(old_list, dtype="int32"): Converts all elements of a list to Variable. """ from .tensor import fill_constant + new_list_tensor = [] for ele in old_list: @@ -393,8 +444,11 @@ def convert_shape_to_list(shape): """ if isinstance(shape, (list, tuple)): shape = list( - map(lambda x: x.numpy().flat[0] - if isinstance(x, Variable) else x, shape)) + map( + lambda x: x.numpy().flat[0] if isinstance(x, Variable) else x, + shape, + ) + ) else: shape = shape.numpy().astype(int).tolist() return shape @@ -462,7 +516,8 @@ def try_get_constant_shape_from_tensor(shape_tensor): generate_op = shape_tensor.op if generate_op.type == 'shape': var = shape_tensor.block.vars[ - generate_op.input_arg_names[0]] + generate_op.input_arg_names[0] + ] return var.shape except: return None @@ -476,9 +531,11 @@ def get_inputs_outputs_in_block(block): created in this block. """ assert isinstance( - block, - Block), "input non-Block argument for get_inputs_outputs_in_block." - assert block.parent_idx != -1, "input block should be a sub-block, not main block." + block, Block + ), "input non-Block argument for get_inputs_outputs_in_block." + assert ( + block.parent_idx != -1 + ), "input block should be a sub-block, not main block." # Find input/output var names of all ops in block inner_inputs = set() diff --git a/python/paddle/fluid/lazy_init.py b/python/paddle/fluid/lazy_init.py index 8fc175ae3efeb..d2118259d03cf 100644 --- a/python/paddle/fluid/lazy_init.py +++ b/python/paddle/fluid/lazy_init.py @@ -36,7 +36,8 @@ def enable(self): """ if self._state: return - assert framework._non_static_mode( + assert ( + framework._non_static_mode() ), "LazyInit.enable() is only available in dygraph mode." self._state = True @@ -56,7 +57,8 @@ def __enter__(self): dygraph mode into static mode. """ self.enable() - if self._in_guard: return + if self._in_guard: + return self._tracer = framework._dygraph_tracer_ framework._dygraph_tracer_ = None self._in_guard = True @@ -66,7 +68,8 @@ def __exit__(self, *args, **kwargs): Exit from lazy mode and recover _dygraph_tracer_. """ self.disable() - if not self._in_guard: return + if not self._in_guard: + return assert self._tracer is not None framework._dygraph_tracer_ = self._tracer self._tracer = None diff --git a/python/paddle/fluid/lod_tensor.py b/python/paddle/fluid/lod_tensor.py index 1b1e74f1fec99..d099536fcd784 100644 --- a/python/paddle/fluid/lod_tensor.py +++ b/python/paddle/fluid/lod_tensor.py @@ -72,10 +72,12 @@ def create_lod_tensor(data, recursive_seq_lens, place): elif isinstance(data, list): # dtype and shape are not important here, # we only want to reuse code of DataToLoDTensorConverter - converter = DataToLoDTensorConverter(place=place, - lod_level=len(recursive_seq_lens), - shape=[], - dtype=core.VarDesc.VarType.FP32) + converter = DataToLoDTensorConverter( + place=place, + lod_level=len(recursive_seq_lens), + shape=[], + dtype=core.VarDesc.VarType.FP32, + ) new_recursive_seq_lens = [] for seq in data: @@ -91,7 +93,7 @@ def create_lod_tensor(data, recursive_seq_lens, place): # FIXME(zjl): the original logic of create_lod_tensor would append # 1 to the shape. Maybe it is not a right way? Currently, we only # follow the previous logic - arr = arr.reshape(arr.shape + (1, )) + arr = arr.reshape(arr.shape + (1,)) tensor = core.LoDTensor() tensor.set(arr, place) tensor.set_recursive_sequence_lengths(recursive_seq_lens) @@ -100,18 +102,21 @@ def create_lod_tensor(data, recursive_seq_lens, place): tensor = core.LoDTensor() tensor.set(data, place) tensor.set_recursive_sequence_lengths(recursive_seq_lens) - assert tensor.has_valid_recursive_sequence_lengths( + assert ( + tensor.has_valid_recursive_sequence_lengths() ), "the provided lod info is invalid" return tensor else: raise TypeError( - "data should be either a LoDTensor, a Numpy array or a list") + "data should be either a LoDTensor, a Numpy array or a list" + ) -def create_random_int_lodtensor(recursive_seq_lens, base_shape, place, low, - high): +def create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low, high +): """ - :api_attr: Static Graph + :api_attr: Static Graph Create a LoDTensor containing random integers. diff --git a/python/paddle/fluid/memory_analysis.py b/python/paddle/fluid/memory_analysis.py index de9a260ada89e..9d33927b98bbe 100644 --- a/python/paddle/fluid/memory_analysis.py +++ b/python/paddle/fluid/memory_analysis.py @@ -19,8 +19,9 @@ def get_var_and_memory_size(block, var_name, batch_size=None): var = block._find_var_recursive(var_name) assert var is not None, "Variable {} cannot be found".format(var_name) - assert var.type == core.VarDesc.VarType.LOD_TENSOR, "Variable {} is not Tensor".format( - var_name) + assert ( + var.type == core.VarDesc.VarType.LOD_TENSOR + ), "Variable {} is not Tensor".format(var_name) shape = list(var.shape) if not shape: return var, 0 @@ -31,8 +32,9 @@ def get_var_and_memory_size(block, var_name, batch_size=None): assert not has_none shape[i] = batch_size has_none = True - assert all([s >= 0 - for s in shape]), "shape {} is not deterministic".format(shape) + assert all([s >= 0 for s in shape]), "shape {} is not deterministic".format( + shape + ) mem_size = int(np.prod(shape)) * core.size_of_dtype(var.dtype) return var, mem_size @@ -46,7 +48,9 @@ def pre_allocate_memory(size, place): # NOTE: does not consider inplace yet. def get_max_memory_info(program, batch_size=None): - assert program.num_blocks == 1, "only support to analysis program with only one block" + assert ( + program.num_blocks == 1 + ), "only support to analysis program with only one block" cur_tmp_mem = 0 max_tmp_mem = 0 max_persistable_mem = 0 diff --git a/python/paddle/fluid/metrics.py b/python/paddle/fluid/metrics.py index f14692f79ebb7..8f15508e699c1 100644 --- a/python/paddle/fluid/metrics.py +++ b/python/paddle/fluid/metrics.py @@ -43,8 +43,12 @@ def _is_numpy_(var): def _is_number_(var): - return isinstance(var, int) or isinstance(var, np.int64) or isinstance( - var, float) or (isinstance(var, np.ndarray) and var.shape == (1, )) + return ( + isinstance(var, int) + or isinstance(var, np.int64) + or isinstance(var, float) + or (isinstance(var, np.ndarray) and var.shape == (1,)) + ) def _is_number_or_matrix_(var): @@ -116,13 +120,14 @@ def reset(self): """ states = { attr: value - for attr, value in self.__dict__.items() if not attr.startswith("_") + for attr, value in self.__dict__.items() + if not attr.startswith("_") } for attr, value in states.items(): if isinstance(value, int): setattr(self, attr, 0) elif isinstance(value, float): - setattr(self, attr, .0) + setattr(self, attr, 0.0) elif isinstance(value, (np.ndarray, np.generic)): setattr(self, attr, np.zeros_like(value)) else: @@ -144,7 +149,8 @@ def get_config(self): """ states = { attr: value - for attr, value in self.__dict__.items() if not attr.startswith("_") + for attr, value in self.__dict__.items() + if not attr.startswith("_") } config = {} config.update({"name": self._name, "states": copy.deepcopy(states)}) @@ -170,7 +176,8 @@ def update(self, preds, labels): """ raise NotImplementedError( - "Should not use it directly, please extend it.") + "Should not use it directly, please extend it." + ) def eval(self): """ @@ -187,7 +194,8 @@ def eval(self): float|list(float)|numpy.array: the metrics via Python. """ raise NotImplementedError( - "Should not use it directly, please extend it.") + "Should not use it directly, please extend it." + ) class CompositeMetric(MetricBase): @@ -340,7 +348,7 @@ def eval(self): float: Results of the calculated Precision. Scalar output with float dtype. """ ap = self.tp + self.fp - return float(self.tp) / ap if ap != 0 else .0 + return float(self.tp) / ap if ap != 0 else 0.0 class Recall(MetricBase): @@ -423,7 +431,7 @@ def eval(self): float: results of the calculated Recall. Scalar output with float dtype. """ recall = self.tp + self.fn - return float(self.tp) / recall if recall != 0 else .0 + return float(self.tp) / recall if recall != 0 else 0.0 class Accuracy(MetricBase): @@ -465,8 +473,8 @@ class Accuracy(MetricBase): def __init__(self, name=None): super(Accuracy, self).__init__(name) - self.value = .0 - self.weight = .0 + self.value = 0.0 + self.weight = 0.0 def update(self, value, weight): r""" @@ -482,7 +490,8 @@ def update(self, value, weight): """ if not _is_number_or_matrix_(value): raise ValueError( - "The 'value' must be a number(int, float) or a numpy ndarray.") + "The 'value' must be a number(int, float) or a numpy ndarray." + ) if not _is_number_(weight): raise ValueError("The 'weight' must be a number(int, float).") if _is_number_(weight) and weight < 0: @@ -499,8 +508,10 @@ def eval(self): """ if self.weight == 0: - raise ValueError("There is no data in Accuracy Metrics. \ - Please check layers.accuracy output has added to Accuracy.") + raise ValueError( + "There is no data in Accuracy Metrics. \ + Please check layers.accuracy output has added to Accuracy." + ) return self.value / self.weight @@ -592,13 +603,21 @@ def eval(self): float: mean precision, recall and f1 score. """ - precision = float( - self.num_correct_chunks - ) / self.num_infer_chunks if self.num_infer_chunks else 0 - recall = float(self.num_correct_chunks - ) / self.num_label_chunks if self.num_label_chunks else 0 - f1_score = float(2 * precision * recall) / ( - precision + recall) if self.num_correct_chunks else 0 + precision = ( + float(self.num_correct_chunks) / self.num_infer_chunks + if self.num_infer_chunks + else 0 + ) + recall = ( + float(self.num_correct_chunks) / self.num_label_chunks + if self.num_label_chunks + else 0 + ) + f1_score = ( + float(2 * precision * recall) / (precision + recall) + if self.num_correct_chunks + else 0 + ) return precision, recall, f1_score @@ -653,7 +672,7 @@ class EditDistance(MetricBase): def __init__(self, name): super(EditDistance, self).__init__(name) - self.total_distance = .0 + self.total_distance = 0.0 self.seq_num = 0 self.instance_error = 0 @@ -789,11 +808,14 @@ def eval(self): tot_neg_prev = tot_neg tot_pos += self._stat_pos[idx] tot_neg += self._stat_neg[idx] - auc += self.trapezoid_area(tot_neg, tot_neg_prev, tot_pos, - tot_pos_prev) + auc += self.trapezoid_area( + tot_neg, tot_neg_prev, tot_pos, tot_pos_prev + ) idx -= 1 - return auc / tot_pos / tot_neg if tot_pos > 0.0 and tot_neg > 0.0 else 0.0 + return ( + auc / tot_pos / tot_neg if tot_pos > 0.0 and tot_neg > 0.0 else 0.0 + ) class DetectionMAP(object): @@ -879,16 +901,18 @@ class DetectionMAP(object): """ - def __init__(self, - input, - gt_label, - gt_box, - gt_difficult=None, - class_num=None, - background_label=0, - overlap_threshold=0.5, - evaluate_difficult=True, - ap_version='integral'): + def __init__( + self, + input, + gt_label, + gt_box, + gt_difficult=None, + class_num=None, + background_label=0, + overlap_threshold=0.5, + evaluate_difficult=True, + ap_version='integral', + ): self.helper = LayerHelper('map_eval') gt_label = layers.cast(x=gt_label, dtype=gt_box.dtype) @@ -899,30 +923,36 @@ def __init__(self, label = layers.concat([gt_label, gt_box], axis=1) # calculate mean average precision (mAP) of current mini-batch - map = detection.detection_map(input, - label, - class_num, - background_label, - overlap_threshold=overlap_threshold, - evaluate_difficult=evaluate_difficult, - ap_version=ap_version) + map = detection.detection_map( + input, + label, + class_num, + background_label, + overlap_threshold=overlap_threshold, + evaluate_difficult=evaluate_difficult, + ap_version=ap_version, + ) states = [] states.append( - self._create_state(dtype='int32', - shape=None, - suffix='accum_pos_count')) + self._create_state( + dtype='int32', shape=None, suffix='accum_pos_count' + ) + ) states.append( - self._create_state(dtype='float32', - shape=None, - suffix='accum_true_pos')) + self._create_state( + dtype='float32', shape=None, suffix='accum_true_pos' + ) + ) states.append( - self._create_state(dtype='float32', - shape=None, - suffix='accum_false_pos')) + self._create_state( + dtype='float32', shape=None, suffix='accum_false_pos' + ) + ) var = self._create_state(dtype='int32', shape=[1], suffix='has_state') - self.helper.set_variable_initializer(var, - initializer=Constant(value=int(0))) + self.helper.set_variable_initializer( + var, initializer=Constant(value=int(0)) + ) self.has_state = var # calculate accumulative mAP @@ -936,12 +966,15 @@ def __init__(self, has_state=self.has_state, input_states=states, out_states=states, - ap_version=ap_version) + ap_version=ap_version, + ) - layers.fill_constant(shape=self.has_state.shape, - value=1, - dtype=self.has_state.dtype, - out=self.has_state) + layers.fill_constant( + shape=self.has_state.shape, + value=1, + dtype=self.has_state.dtype, + out=self.has_state, + ) self.cur_map = map self.accum_map = accum_map @@ -955,11 +988,12 @@ def _create_state(self, suffix, dtype, shape): shape(tuple|list): the shape of state Returns: State variable """ - state = self.helper.create_variable(name="_".join( - [unique_name.generate(self.helper.name), suffix]), - persistable=True, - dtype=dtype, - shape=shape) + state = self.helper.create_variable( + name="_".join([unique_name.generate(self.helper.name), suffix]), + persistable=True, + dtype=dtype, + shape=shape, + ) return state def get_map_var(self): @@ -981,19 +1015,20 @@ def reset(self, executor, reset_program=None): def _clone_var_(block, var): assert isinstance(var, Variable) - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=var.persistable) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=var.persistable, + ) if reset_program is None: reset_program = Program() with program_guard(main_program=reset_program): var = _clone_var_(reset_program.current_block(), self.has_state) - layers.fill_constant(shape=var.shape, - value=0, - dtype=var.dtype, - out=var) + layers.fill_constant( + shape=var.shape, value=0, dtype=var.dtype, out=var + ) executor.run(reset_program) diff --git a/python/paddle/fluid/multiprocess_utils.py b/python/paddle/fluid/multiprocess_utils.py index 73bba0069cdd2..deca95457e61c 100644 --- a/python/paddle/fluid/multiprocess_utils.py +++ b/python/paddle/fluid/multiprocess_utils.py @@ -23,7 +23,7 @@ # multi-process worker check indices queue interval, avoid # hanging in subprocess data loading -MP_STATUS_CHECK_INTERVAL = 5. +MP_STATUS_CHECK_INTERVAL = 5.0 # NOTE: [ mmap files clear ] If there is still data in the multiprocess queue when the main process finishes reading, # the data in the queue needs to be popped. Then the LoDTensor read by the main process @@ -56,7 +56,7 @@ def _cleanup_mmap(): # NOTE used for register a function to be executed at interpreter exit. -class CleanupFuncRegistrar(): +class CleanupFuncRegistrar: # Record the cleanup functions that have been executed _executed_func_set = set() # Record the cleanup functions that have been registered @@ -64,7 +64,6 @@ class CleanupFuncRegistrar(): @classmethod def register(cls, function, signals=[]): - def _func_exectuor(): if function not in cls._executed_func_set: try: @@ -93,8 +92,10 @@ def _signal_register(signals): for sig in signals: orig_handler = signal.signal(sig, _signal_handler) if orig_handler not in (signal.SIG_DFL, signal.SIG_IGN): - if (sig == signal.SIGINT - and orig_handler is signal.default_int_handler): + if ( + sig == signal.SIGINT + and orig_handler is signal.default_int_handler + ): continue if orig_handler not in cls._registered_func_set: atexit.register(orig_handler) diff --git a/python/paddle/fluid/net_drawer.py b/python/paddle/fluid/net_drawer.py index b6a4c28b49953..cf5cbf60ea3d8 100644 --- a/python/paddle/fluid/net_drawer.py +++ b/python/paddle/fluid/net_drawer.py @@ -30,16 +30,19 @@ 'Cannot import graphviz, which is required for drawing a network. This ' 'can usually be installed in python with "pip install graphviz". Also, ' 'pydot requires graphviz to convert dot files to pdf: in ubuntu, this ' - 'can usually be installed with "sudo apt-get install graphviz".') - print('net_drawer will not run correctly. Please install the correct ' - 'dependencies.') + 'can usually be installed with "sudo apt-get install graphviz".' + ) + print( + 'net_drawer will not run correctly. Please install the correct ' + 'dependencies.' + ) exit(0) OP_STYLE = { 'shape': 'oval', 'color': '#0F9D58', 'style': 'filled', - 'fontcolor': '#FFFFFF' + 'fontcolor': '#FFFFFF', } VAR_STYLE = {} @@ -52,7 +55,6 @@ def unique_id(): - def generator(): GRAPH_ID += 1 return GRAPH_ID @@ -85,7 +87,8 @@ def parse_graph(program, graph, var_dict, **kwargs): temp_id = 0 proto = framework_pb2.ProgramDesc.FromString( - program.desc.serialize_to_string()) + program.desc.serialize_to_string() + ) for block in proto.blocks: for op in block.ops: op.type = op.type + "_" + str(temp_id) @@ -113,12 +116,14 @@ def draw_graph(startup_program, main_program, **kwargs): filename = kwargs.get("filename") if filename == None: filename = str(graph_id) + ".gv" - g = Graph(name=str(graph_id), - filename=filename, - graph_attr=GRAPH_STYLE, - node_attr=OP_STYLE, - edge_attr=VAR_STYLE, - **kwargs) + g = Graph( + name=str(graph_id), + filename=filename, + graph_attr=GRAPH_STYLE, + node_attr=OP_STYLE, + edge_attr=VAR_STYLE, + **kwargs + ) var_dict = {} parse_graph(startup_program, g, var_dict) diff --git a/python/paddle/fluid/nets.py b/python/paddle/fluid/nets.py index a61487ee71d4c..bfff9c12e9241 100644 --- a/python/paddle/fluid/nets.py +++ b/python/paddle/fluid/nets.py @@ -25,24 +25,26 @@ ] -def simple_img_conv_pool(input, - num_filters, - filter_size, - pool_size, - pool_stride, - pool_padding=0, - pool_type='max', - global_pooling=False, - conv_stride=1, - conv_padding=0, - conv_dilation=1, - conv_groups=1, - param_attr=None, - bias_attr=None, - act=None, - use_cudnn=True): +def simple_img_conv_pool( + input, + num_filters, + filter_size, + pool_size, + pool_stride, + pool_padding=0, + pool_type='max', + global_pooling=False, + conv_stride=1, + conv_padding=0, + conv_dilation=1, + conv_groups=1, + param_attr=None, + bias_attr=None, + act=None, + use_cudnn=True, +): r""" - :api_attr: Static Graph + :api_attr: Static Graph The simple_img_conv_pool api is composed of :ref:`api_fluid_layers_conv2d` and :ref:`api_fluid_layers_pool2d` . @@ -115,42 +117,48 @@ def simple_img_conv_pool(input, pool_stride=2, act="relu") """ - conv_out = layers.conv2d(input=input, - num_filters=num_filters, - filter_size=filter_size, - stride=conv_stride, - padding=conv_padding, - dilation=conv_dilation, - groups=conv_groups, - param_attr=param_attr, - bias_attr=bias_attr, - act=act, - use_cudnn=use_cudnn) - - pool_out = layers.pool2d(input=conv_out, - pool_size=pool_size, - pool_type=pool_type, - pool_stride=pool_stride, - pool_padding=pool_padding, - global_pooling=global_pooling, - use_cudnn=use_cudnn) + conv_out = layers.conv2d( + input=input, + num_filters=num_filters, + filter_size=filter_size, + stride=conv_stride, + padding=conv_padding, + dilation=conv_dilation, + groups=conv_groups, + param_attr=param_attr, + bias_attr=bias_attr, + act=act, + use_cudnn=use_cudnn, + ) + + pool_out = layers.pool2d( + input=conv_out, + pool_size=pool_size, + pool_type=pool_type, + pool_stride=pool_stride, + pool_padding=pool_padding, + global_pooling=global_pooling, + use_cudnn=use_cudnn, + ) return pool_out -def img_conv_group(input, - conv_num_filter, - pool_size, - conv_padding=1, - conv_filter_size=3, - conv_act=None, - param_attr=None, - conv_with_batchnorm=False, - conv_batchnorm_drop_rate=0.0, - pool_stride=1, - pool_type="max", - use_cudnn=True): +def img_conv_group( + input, + conv_num_filter, + pool_size, + conv_padding=1, + conv_filter_size=3, + conv_act=None, + param_attr=None, + conv_with_batchnorm=False, + conv_batchnorm_drop_rate=0.0, + pool_stride=1, + pool_type="max", + use_cudnn=True, +): """ - :api_attr: Static Graph + :api_attr: Static Graph The Image Convolution Group is composed of Convolution2d, BatchNorm, DropOut, and Pool2D. According to the input arguments, img_conv_group will do serials of @@ -210,8 +218,9 @@ def img_conv_group(input, pool_stride=2) """ tmp = input - assert isinstance(conv_num_filter, list) or \ - isinstance(conv_num_filter, tuple) + assert isinstance(conv_num_filter, list) or isinstance( + conv_num_filter, tuple + ) def __extend_list__(obj): if not hasattr(obj, '__len__'): @@ -231,13 +240,15 @@ def __extend_list__(obj): if conv_with_batchnorm[i]: local_conv_act = None - tmp = layers.conv2d(input=tmp, - num_filters=conv_num_filter[i], - filter_size=conv_filter_size[i], - padding=conv_padding[i], - param_attr=param_attr[i], - act=local_conv_act, - use_cudnn=use_cudnn) + tmp = layers.conv2d( + input=tmp, + num_filters=conv_num_filter[i], + filter_size=conv_filter_size[i], + padding=conv_padding[i], + param_attr=param_attr[i], + act=local_conv_act, + use_cudnn=use_cudnn, + ) if conv_with_batchnorm[i]: tmp = layers.batch_norm(input=tmp, act=conv_act) @@ -245,23 +256,27 @@ def __extend_list__(obj): if abs(drop_rate) > 1e-5: tmp = layers.dropout(x=tmp, dropout_prob=drop_rate) - pool_out = layers.pool2d(input=tmp, - pool_size=pool_size, - pool_type=pool_type, - pool_stride=pool_stride, - use_cudnn=use_cudnn) + pool_out = layers.pool2d( + input=tmp, + pool_size=pool_size, + pool_type=pool_type, + pool_stride=pool_stride, + use_cudnn=use_cudnn, + ) return pool_out -def sequence_conv_pool(input, - num_filters, - filter_size, - param_attr=None, - act="sigmoid", - pool_type="max", - bias_attr=None): +def sequence_conv_pool( + input, + num_filters, + filter_size, + param_attr=None, + act="sigmoid", + pool_type="max", + bias_attr=None, +): """ - :api_attr: Static Graph + :api_attr: Static Graph **This api takes input as an LoDTensor. If input is a Tensor, please use** :ref:`api_fluid_nets_simple_img_conv_pool` **instead** @@ -315,12 +330,14 @@ def sequence_conv_pool(input, """ check_variable_and_dtype(input, 'input', ['float32', 'float64'], 'input') - conv_out = layers.sequence_conv(input=input, - num_filters=num_filters, - filter_size=filter_size, - param_attr=param_attr, - bias_attr=bias_attr, - act=act) + conv_out = layers.sequence_conv( + input=input, + num_filters=num_filters, + filter_size=filter_size, + param_attr=param_attr, + bias_attr=bias_attr, + act=act, + ) pool_out = layers.sequence_pool(input=conv_out, pool_type=pool_type) return pool_out @@ -329,7 +346,7 @@ def sequence_conv_pool(input, @deprecated(since="2.0.0", update_to="paddle.nn.functional.glu") def glu(input, dim=-1): r""" - :api_attr: Static Graph + :api_attr: Static Graph The Gated Linear Units(GLU) composed by :ref:`api_fluid_layers_split` , :ref:`api_fluid_layers_sigmoid` and :ref:`api_fluid_layers_elementwise_mul` . @@ -366,19 +383,18 @@ def glu(input, dim=-1): # shape of output: [-1, 3, 3, 9] output = fluid.nets.glu(input=data, dim=1) """ - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - "glu") + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], "glu" + ) a, b = layers.split(input, num_or_sections=2, dim=dim) act_b = layers.sigmoid(x=b) out = layers.elementwise_mul(x=a, y=act_b) return out -def scaled_dot_product_attention(queries, - keys, - values, - num_heads=1, - dropout_rate=0.): +def scaled_dot_product_attention( + queries, keys, values, num_heads=1, dropout_rate=0.0 +): r""" :api_attr: Static Graph @@ -449,46 +465,63 @@ def scaled_dot_product_attention(queries, contexts = fluid.nets.scaled_dot_product_attention(queries, keys, values) contexts.shape # [3, 5, 10] """ - check_variable_and_dtype(queries, 'queries', ['float32', 'float64'], - "scaled_dot_product_attention") - check_variable_and_dtype(keys, 'keys', ['float32', 'float64'], - "scaled_dot_product_attention") - check_variable_and_dtype(values, 'values', ['float32', 'float64'], - "scaled_dot_product_attention") + check_variable_and_dtype( + queries, + 'queries', + ['float32', 'float64'], + "scaled_dot_product_attention", + ) + check_variable_and_dtype( + keys, 'keys', ['float32', 'float64'], "scaled_dot_product_attention" + ) + check_variable_and_dtype( + values, 'values', ['float32', 'float64'], "scaled_dot_product_attention" + ) if not (queries.dtype == keys.dtype == values.dtype): raise TypeError( "The dtype of keys, values and queries should be the same." "But received queries.dtype = %s, " - " keys.dtype = %s, values.dtype) = %s." % - (convert_dtype(queries.dtype), convert_dtype( - keys.dtype), convert_dtype(values.dtype))) + " keys.dtype = %s, values.dtype) = %s." + % ( + convert_dtype(queries.dtype), + convert_dtype(keys.dtype), + convert_dtype(values.dtype), + ) + ) if not (len(queries.shape) == len(keys.shape) == len(values.shape) == 3): raise ValueError( "Inputs queries, keys and values should all be 3-D tensors." "But received len(queries.shape) = %d, " - "len(keys.shape) = %d, len(values.shape) = %d." % - (len(queries.shape), len(keys.shape), len(values.shape))) + "len(keys.shape) = %d, len(values.shape) = %d." + % (len(queries.shape), len(keys.shape), len(values.shape)) + ) if queries.shape[-1] != keys.shape[-1]: raise ValueError( "The hidden size of queries and keys should be the same." "But received queries' hidden size = %d and keys' hidden size = %d." - % (queries.shape[-1], keys.shape[-1])) + % (queries.shape[-1], keys.shape[-1]) + ) if keys.shape[-2] != values.shape[-2]: raise ValueError( "The max sequence length in value batch and in key batch " "should be the same. But received max sequence length in value batch " - "= %d, in key batch = %d." % (values.shape[-2], keys.shape[-2])) + "= %d, in key batch = %d." % (values.shape[-2], keys.shape[-2]) + ) if keys.shape[-1] % num_heads != 0: - raise ValueError("The hidden size of keys (%d) must be divisible " - "by the number of attention heads (%d)." % - (keys.shape[-1], num_heads)) + raise ValueError( + "The hidden size of keys (%d) must be divisible " + "by the number of attention heads (%d)." + % (keys.shape[-1], num_heads) + ) if values.shape[-1] % num_heads != 0: - raise ValueError("The hidden size of values (%d) must be divisible " - "by the number of attention heads (%d)." % - (values.shape[-1], num_heads)) + raise ValueError( + "The hidden size of values (%d) must be divisible " + "by the number of attention heads (%d)." + % (values.shape[-1], num_heads) + ) def __compute_qkv(queries, keys, values, num_heads): """ @@ -535,9 +568,10 @@ def __split_heads(x, num_heads): # reshape the 3-D input: [batch_size, max_sequence_length, hidden_dim] # into a 4-D output: # [batch_size, max_sequence_length, num_heads, hidden_size_per_head]. - reshaped = layers.reshape(x=x, - shape=list(x.shape[:-1]) + - [num_heads, hidden_size // num_heads]) + reshaped = layers.reshape( + x=x, + shape=list(x.shape[:-1]) + [num_heads, hidden_size // num_heads], + ) # permute the dimensions into: # [batch_size, num_heads, max_sequence_len, hidden_size_per_head] @@ -557,17 +591,25 @@ def __combine_heads(x): [bs, max_sequence_length, num_heads * hidden_dim]. """ - if len(x.shape) == 3: return x + if len(x.shape) == 3: + return x if len(x.shape) != 4: raise ValueError("Input(x) should be a 4-D Tensor.") trans_x = layers.transpose(x, perm=[0, 2, 1, 3]) - return layers.reshape(x=trans_x, - shape=list( - map(int, [ - trans_x.shape[0], trans_x.shape[1], - trans_x.shape[2] * trans_x.shape[3] - ]))) + return layers.reshape( + x=trans_x, + shape=list( + map( + int, + [ + trans_x.shape[0], + trans_x.shape[1], + trans_x.shape[2] * trans_x.shape[3], + ], + ) + ), + ) q, k, v = __compute_qkv(queries, keys, values, num_heads) @@ -579,13 +621,15 @@ def __combine_heads(x): scaled_q = layers.scale(x=q, scale=key_dim_per_head**-0.5) product = layers.matmul(x=scaled_q, y=k, transpose_y=True) - weights = layers.reshape(x=layers.reshape(x=product, - shape=[-1, product.shape[-1]], - act="softmax"), - shape=product.shape) + weights = layers.reshape( + x=layers.reshape( + x=product, shape=[-1, product.shape[-1]], act="softmax" + ), + shape=product.shape, + ) if dropout_rate: - weights = layers.dropout(weights, - dropout_prob=dropout_rate, - is_test=False) + weights = layers.dropout( + weights, dropout_prob=dropout_rate, is_test=False + ) ctx_multiheads = layers.matmul(weights, v) return __combine_heads(ctx_multiheads) diff --git a/python/paddle/fluid/op.py b/python/paddle/fluid/op.py index bdcdad1218e41..c8b118127cc13 100644 --- a/python/paddle/fluid/op.py +++ b/python/paddle/fluid/op.py @@ -47,7 +47,8 @@ class OpDescCreationMethod(object): def __init__(self, op_proto): if not isinstance(op_proto, framework_pb2.OpProto): raise TypeError( - "Type of op_proto should be OpProto in PaddlePaddle.") + "Type of op_proto should be OpProto in PaddlePaddle." + ) self.__op_proto__ = op_proto self.__extra_attrs__ = core.get_op_extra_attrs(op_proto.type) @@ -67,8 +68,9 @@ def __call__(self, *args, **kwargs): if not input_parameter.duplicable and len(input_arguments) > 1: raise ValueError( - "Input %s expects only one input, but %d are given." % - (input_parameter.name, len(input_arguments))) + "Input %s expects only one input, but %d are given." + % (input_parameter.name, len(input_arguments)) + ) ipt = op_desc.inputs.add() ipt.parameter = input_parameter.name @@ -81,8 +83,9 @@ def __call__(self, *args, **kwargs): if not output_parameter.duplicable and len(output_arguments) > 1: raise ValueError( - "Output %s expects only one output, but %d are given." % - (output_parameter.name, len(output_arguments))) + "Output %s expects only one output, but %d are given." + % (output_parameter.name, len(output_arguments)) + ) out = op_desc.outputs.add() out.parameter = output_parameter.name @@ -126,13 +129,14 @@ def __call__(self, *args, **kwargs): new_attr.float64 = user_defined_attr else: raise NotImplementedError( - "A not supported attribute type: %s." % - (str(attr.type))) + "A not supported attribute type: %s." % (str(attr.type)) + ) for attr_name, defalut_val in self.__extra_attrs__.items(): user_defined_attr = kwargs.get(attr_name, None) if user_defined_attr is not None: attr_type = int( - core.get_attrtibute_type(op_desc.type, attr_name)) + core.get_attrtibute_type(op_desc.type, attr_name) + ) new_attr = op_desc.attrs.add() new_attr.name = attr_name new_attr.type = attr_type @@ -160,8 +164,8 @@ def __call__(self, *args, **kwargs): new_attr.longs.extend(user_defined_attr) else: raise NotImplementedError( - "A not supported attribute type: %s." % - (str(attr_type))) + "A not supported attribute type: %s." % (str(attr_type)) + ) return op_desc @@ -178,7 +182,6 @@ def any_is_true(generator): class OpInfo(object): - def __init__(self, name, method, inputs, outputs, attrs, extra_attrs): self.name = name self.method = method @@ -200,18 +203,17 @@ def __impl__(*args, **kwargs): extra_attrs_map = core.get_op_extra_attrs(op_proto.type) - return OpInfo(method=__impl__, - name=op_proto.type, - inputs=[(var.name, var.duplicable) - for var in op_proto.inputs], - outputs=[(var.name, var.duplicable) - for var in op_proto.outputs], - attrs=[attr.name for attr in op_proto.attrs], - extra_attrs=[item for item in extra_attrs_map.keys()]) + return OpInfo( + method=__impl__, + name=op_proto.type, + inputs=[(var.name, var.duplicable) for var in op_proto.inputs], + outputs=[(var.name, var.duplicable) for var in op_proto.outputs], + attrs=[attr.name for attr in op_proto.attrs], + extra_attrs=[item for item in extra_attrs_map.keys()], + ) class OperatorFactory(object): - def __init__(self): self.op_methods = dict() @@ -224,13 +226,15 @@ def __call__(self, *args, **kwargs): if len(args) != 0: raise ValueError( "Except the argument \"type\"," - "all of the other arguments should be keyword arguments.") + "all of the other arguments should be keyword arguments." + ) t = kwargs.pop("type") else: if len(args) != 1: raise ValueError( "Except the argument \"type\"," - "all of the other arguments should be keyword arguments.") + "all of the other arguments should be keyword arguments." + ) t = args[0] return self.get_op_info(t).method(**kwargs) diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index 4471eb9dcbc75..ce58c0cfe02fb 100755 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -19,20 +19,42 @@ import paddle from paddle.fluid.distribute_lookup_table import find_distributed_lookup_table -from paddle.fluid.framework import Program, Variable, Parameter, name_scope, default_main_program, default_startup_program, device_guard +from paddle.fluid.framework import ( + Program, + Variable, + Parameter, + name_scope, + default_main_program, + default_startup_program, + device_guard, +) from . import framework from . import layers from . import unique_name -from .backward import append_backward, _some_in_set_, _append_grad_suffix_, _get_no_grad_set_name -from .clip import GradientClipBase, GradientClipByNorm, error_clip_callback, append_gradient_clip_ops, ClipGradByGlobalNorm +from .backward import ( + append_backward, + _some_in_set_, + _append_grad_suffix_, + _get_no_grad_set_name, +) +from .clip import ( + GradientClipBase, + GradientClipByNorm, + error_clip_callback, + append_gradient_clip_ops, + ClipGradByGlobalNorm, +) from .framework import program_guard from .initializer import Constant from .layer_helper import LayerHelper from .layers import ops from .dygraph import base as imperative_base from .dygraph import no_grad -from .dygraph.learning_rate_scheduler import LearningRateDecay, _LearningRateEpochDecay +from .dygraph.learning_rate_scheduler import ( + LearningRateDecay, + _LearningRateEpochDecay, +) from paddle.fluid import core from paddle.fluid.layers import tensor from functools import reduce @@ -41,16 +63,40 @@ from .. import compat as cpt import warnings from paddle import _C_ops, _legacy_C_ops -from ..fluid.framework import _in_legacy_dygraph, in_dygraph_mode, _current_expected_place +from ..fluid.framework import ( + _in_legacy_dygraph, + in_dygraph_mode, + _current_expected_place, +) __all__ = [ - 'SGD', 'Momentum', 'Adagrad', 'Adam', 'Adamax', 'Dpsgd', 'DecayedAdagrad', - 'Ftrl', 'SGDOptimizer', 'MomentumOptimizer', 'AdagradOptimizer', - 'AdamOptimizer', 'AdamaxOptimizer', 'DpsgdOptimizer', - 'DecayedAdagradOptimizer', 'RMSPropOptimizer', 'FtrlOptimizer', 'Adadelta', - 'AdadeltaOptimizer', 'ModelAverage', 'LarsMomentum', - 'LarsMomentumOptimizer', 'LambOptimizer', 'ExponentialMovingAverage', - 'PipelineOptimizer', 'LookaheadOptimizer', 'RecomputeOptimizer' + 'SGD', + 'Momentum', + 'Adagrad', + 'Adam', + 'Adamax', + 'Dpsgd', + 'DecayedAdagrad', + 'Ftrl', + 'SGDOptimizer', + 'MomentumOptimizer', + 'AdagradOptimizer', + 'AdamOptimizer', + 'AdamaxOptimizer', + 'DpsgdOptimizer', + 'DecayedAdagradOptimizer', + 'RMSPropOptimizer', + 'FtrlOptimizer', + 'Adadelta', + 'AdadeltaOptimizer', + 'ModelAverage', + 'LarsMomentum', + 'LarsMomentumOptimizer', + 'LambOptimizer', + 'ExponentialMovingAverage', + 'PipelineOptimizer', + 'LookaheadOptimizer', + 'RecomputeOptimizer', ] @@ -63,14 +109,16 @@ class Optimizer(object): """ @imperative_base.no_grad - def __init__(self, - learning_rate, - parameter_list=None, - regularization=None, - grad_clip=None, - flatten_param_grads=False, - align_size=-1, - name=None): + def __init__( + self, + learning_rate, + parameter_list=None, + regularization=None, + grad_clip=None, + flatten_param_grads=False, + align_size=-1, + name=None, + ): """ Args: flatten_param_grads (bool, optional): Whether to flatten all the parameters and grads. @@ -79,15 +127,19 @@ def __init__(self, """ # Because of the loop import, so place it in the function body from paddle.optimizer.lr import LRScheduler - self._parameter_list = list( - parameter_list) if parameter_list is not None else None + + self._parameter_list = ( + list(parameter_list) if parameter_list is not None else None + ) self._name = name if framework._non_static_mode(): - if not isinstance(learning_rate, - (float, LearningRateDecay, LRScheduler)): + if not isinstance( + learning_rate, (float, LearningRateDecay, LRScheduler) + ): raise TypeError( "learning rate should be float or LRScheduler, got %s here" - % type(learning_rate)) + % type(learning_rate) + ) if self._parameter_list is None: raise AttributeError( "parameter_list argument given to the Optimizer should not be None in dygraph mode." @@ -98,14 +150,17 @@ def __init__(self, logging.info( "If regularizer of a Parameter has been set by 'fluid.ParamAttr' or 'fluid.WeightNormParamAttr' already. " "The Regularization[%s] in Optimizer will not take effect, and it will only be applied to other Parameters!" - % regularization.__str__()) + % regularization.__str__() + ) break else: - if not isinstance(learning_rate, - (float, framework.Variable, LRScheduler)): + if not isinstance( + learning_rate, (float, framework.Variable, LRScheduler) + ): raise TypeError( "learning rate should be float or LRScheduler, got %s here" - % type(learning_rate)) + % type(learning_rate) + ) if grad_clip is not None: if not isinstance(grad_clip, GradientClipBase): @@ -128,7 +183,8 @@ def __init__(self, self._learning_rate_map = dict() if isinstance(self._learning_rate, framework.Variable): self._learning_rate_map[ - framework.default_main_program()] = self._learning_rate + framework.default_main_program() + ] = self._learning_rate # Dictionary of accumulators. Some optimizer subclasses need to # allocate and manage extra variables associated with the parameters # to train. These variables are called accumulators. @@ -169,6 +225,7 @@ def state_dict(self): ''' from paddle.optimizer.lr import LRScheduler + state_dict = {} for k, v in self._accumulators.items(): for para_name, var_tmp in v.items(): @@ -184,14 +241,13 @@ def state_dict(self): if not isinstance(self._learning_rate, _LearningRateEpochDecay): var_tmp = None - var_temp = framework._varbase_creator(None, - name='global_step', - dtype='int32') + var_temp = framework._varbase_creator( + None, name='global_step', dtype='int32' + ) - tensor.fill_constant([1], - "int32", - self._learning_rate.step_num, - out=var_temp) + tensor.fill_constant( + [1], "int32", self._learning_rate.step_num, out=var_temp + ) state_dict['global_step'] = var_temp return state_dict @@ -230,6 +286,7 @@ def set_state_dict(self, state_dict): para_state_dict, opti_state_dict = fluid.load_dygraph("paddle_dy") ''' from paddle.optimizer.lr import LRScheduler + if isinstance(self._learning_rate, LRScheduler): self._learning_rate.set_dict(state_dict["LR_Scheduler"]) @@ -237,25 +294,33 @@ def set_state_dict(self, state_dict): self._learning_rate.set_dict(state_dict["LR_Scheduler"]) if not isinstance(self._learning_rate, _LearningRateEpochDecay): - assert 'global_step' in state_dict, \ - 'Global step not in state dict, Dygraph use LearningRateDecay, global_step must in state_dict' + assert ( + 'global_step' in state_dict + ), 'Global step not in state dict, Dygraph use LearningRateDecay, global_step must in state_dict' global_step = state_dict['global_step'] if isinstance(global_step, Variable): step_np = global_step step_np = np.array(step_np.value().get_tensor()) - assert step_np.shape == (1,), \ - "global step shape is (1,), the shape is {}".format( step_np.shape ) + assert step_np.shape == ( + 1, + ), "global step shape is (1,), the shape is {}".format( + step_np.shape + ) self._learning_rate.step_num = int(step_np[0]) elif isinstance(global_step, np.ndarray): - assert global_step.shape == (1,), \ - "global step shape is (1,), the shape is {}".format( global_step.shape ) + assert global_step.shape == ( + 1, + ), "global step shape is (1,), the shape is {}".format( + global_step.shape + ) self._learning_rate.step_num = global_step[0] else: raise RuntimeError( "Type not supprt, value in state dict must be [VarBase, Variable, numpy], the type is ", - type(global_step)) + type(global_step), + ) def _load_state_para(state_dict, param): var = param.value() @@ -269,29 +334,36 @@ def _load_state_para(state_dict, param): elif isinstance(load_para, np.ndarray): load_para_np = load_para else: - raise RuntimeError("State dict type {} not supprt".format( - str(type(load_para)))) + raise RuntimeError( + "State dict type {} not supprt".format(str(type(load_para))) + ) - assert model_np.shape == load_para_np.shape, \ - "Parameter shape not match, Dygraph Parameter [ {} ] need tensor with shape {} but load tensor with shape {}".format( - param.name, model_np.shape, load_para_np.shape) + assert ( + model_np.shape == load_para_np.shape + ), "Parameter shape not match, Dygraph Parameter [ {} ] need tensor with shape {} but load tensor with shape {}".format( + param.name, model_np.shape, load_para_np.shape + ) - assert model_np.dtype == load_para_np.dtype, \ - "Parameter dtype not match, Dygraph Parameter [ {} ] need tensor with dtype {} but load tensor with dtype {}".format( - param.name, model_np.dtype, load_para_np.dtype) + assert ( + model_np.dtype == load_para_np.dtype + ), "Parameter dtype not match, Dygraph Parameter [ {} ] need tensor with dtype {} but load tensor with dtype {}".format( + param.name, model_np.dtype, load_para_np.dtype + ) tensor.set(load_para_np, framework._current_expected_place()) self._accumulators_holder = state_dict for k, v in self._accumulators.items(): for para_name, var_tmp in v.items(): - assert var_tmp.name in state_dict, \ - "optimizer variable {} not found".format( var_tmp.name ) + assert ( + var_tmp.name in state_dict + ), "optimizer variable {} not found".format(var_tmp.name) _load_state_para(state_dict, var_tmp) for k, v in self._global_accumulators.items(): - assert v.name in state_dict, \ - "optimizer variable {} not found".format( v.name ) + assert ( + v.name in state_dict + ), "optimizer variable {} not found".format(v.name) _load_state_para(state_dict, v) # [aliases] Compatible with old method names @@ -311,6 +383,7 @@ def _get_auxiliary_var(self, key): def _create_global_learning_rate(self): from paddle.optimizer.lr import LRScheduler + if isinstance(self._learning_rate, LRScheduler): lr_var = self._global_learning_rate() # only create global lr_var once @@ -322,16 +395,19 @@ def _create_global_learning_rate(self): shape=[1], persistable=True, stop_gradient=True, - dtype='float32' if self._dtype is None else self._dtype) + dtype='float32' if self._dtype is None else self._dtype, + ) main_prog = framework.default_main_program() main_prog.lr_sheduler = self._learning_rate main_prog.lr_var = lr_var self._learning_rate_map[ - framework.default_main_program()] = lr_var + framework.default_main_program() + ] = lr_var lr_value = float(self._learning_rate()) self.helper.set_variable_initializer( - lr_var, initializer=Constant(value=lr_value)) + lr_var, initializer=Constant(value=lr_value) + ) return if imperative_base.enabled(): @@ -342,17 +418,20 @@ def _create_global_learning_rate(self): if isinstance(lr, framework.Variable): return else: - self._learning_rate_map[framework.default_main_program( - )] = layers.create_global_var( + self._learning_rate_map[ + framework.default_main_program() + ] = layers.create_global_var( name=unique_name.generate("learning_rate"), shape=[1], value=float(self._learning_rate), dtype='float32' if self._dtype is None else self._dtype, - persistable=True) + persistable=True, + ) # get learning rate Variable from LearningRateDecay elif isinstance(self._learning_rate, LearningRateDecay): self._learning_rate_map[ - framework.default_main_program()] = self._learning_rate() + framework.default_main_program() + ] = self._learning_rate() else: raise TypeError( "optimizer's learning rate must be float or LearningRateDecay" @@ -371,12 +450,14 @@ def _create_global_learning_rate(self): # create learning rate in the current main program self._learning_rate_map[ - framework.default_main_program()] = layers.create_global_var( - name=unique_name.generate("learning_rate"), - shape=[1], - value=float(self._learning_rate), - dtype='float32' if self._dtype is None else self._dtype, - persistable=True) + framework.default_main_program() + ] = layers.create_global_var( + name=unique_name.generate("learning_rate"), + shape=[1], + value=float(self._learning_rate), + dtype='float32' if self._dtype is None else self._dtype, + persistable=True, + ) @framework.dygraph_only def set_lr(self, value): @@ -431,7 +512,8 @@ def set_lr(self, value): if not isinstance(value, (framework.Variable, float)): raise TypeError( "The type of 'value' in optimizer.set_lr must be (float, Variable), but received %s." - % (type(value))) + % (type(value)) + ) if isinstance(self._learning_rate, LearningRateDecay): raise RuntimeError( "optimizer's learning rate can't be LearningRateDecay when invoke this API, because this will lead to conflict." @@ -442,28 +524,42 @@ def set_lr(self, value): if current_lr is not None: if in_dygraph_mode(): place = _current_expected_place() - _C_ops.full_(current_lr, list(current_lr.shape), - float(value), current_lr.dtype, place) + _C_ops.full_( + current_lr, + list(current_lr.shape), + float(value), + current_lr.dtype, + place, + ) elif _in_legacy_dygraph(): - _legacy_C_ops.fill_constant(current_lr, 'value', - float(value), 'dtype', - current_lr.dtype, 'shape', - list(current_lr.shape)) + _legacy_C_ops.fill_constant( + current_lr, + 'value', + float(value), + 'dtype', + current_lr.dtype, + 'shape', + list(current_lr.shape), + ) else: - global_block = framework.default_main_program( - ).global_block() - global_block.append_op(type='fill_constant', - outputs={'Out': [current_lr]}, - attrs={ - 'dtype': current_lr.dtype, - 'shape': list(current_lr.shape), - 'value': float(value) - }, - stop_gradient=True) + global_block = ( + framework.default_main_program().global_block() + ) + global_block.append_op( + type='fill_constant', + outputs={'Out': [current_lr]}, + attrs={ + 'dtype': current_lr.dtype, + 'shape': list(current_lr.shape), + 'value': float(value), + }, + stop_gradient=True, + ) else: - assert len(value.shape) == 1 and value.shape[ - 0] == 1, "optimizer's learning rate must be 1-D Tensor with shape[1]" + assert ( + len(value.shape) == 1 and value.shape[0] == 1 + ), "optimizer's learning rate must be 1-D Tensor with shape[1]" self._learning_rate_map[framework.default_main_program()] = value @framework.dygraph_only @@ -540,8 +636,7 @@ def _global_learning_rate(self, program=None): return self._learning_rate_map.get(program, None) def _append_optimize_op(self, block, param_and_grad): - """ append optimize operator to block and return all the added optimize_op - """ + """append optimize operator to block and return all the added optimize_op""" raise NotImplementedError() def _create_param_lr(self, param_and_grad): @@ -555,8 +650,8 @@ def _create_param_lr(self, param_and_grad): return self._global_learning_rate() else: with default_main_program()._lr_schedule_guard( - is_with_opt=True), framework.name_scope( - 'scale_with_param_lr'): + is_with_opt=True + ), framework.name_scope('scale_with_param_lr'): return self._global_learning_rate() * param_lr def _create_accumulators(self, block, parameters): @@ -581,14 +676,16 @@ def _finish_update(self, block, parameters_and_grads): """ pass - def _add_accumulator(self, - name, - param, - dtype=None, - fill_value=0.0, - shape=None, - type=None, - device=None): + def _add_accumulator( + self, + name, + param, + dtype=None, + fill_value=0.0, + shape=None, + type=None, + device=None, + ): """Utility function to add an accumulator for a parameter Args: @@ -600,13 +697,17 @@ def _add_accumulator(self, """ if self._name is not None: name = self._name + "_" + name - if (name in self._accumulators - and param.name in self._accumulators[name]): + if ( + name in self._accumulators + and param.name in self._accumulators[name] + ): if framework._non_static_mode(): return self._accumulators[name][param.name] raise Exception( "Accumulator {} already exists for parameter {}".format( - name, param.name)) + name, param.name + ) + ) if shape == None: shape = param.shape assert isinstance(self.helper, LayerHelper) @@ -620,32 +721,39 @@ def _add_accumulator(self, persistable=True, dtype=dtype or param.dtype, type=core.VarDesc.VarType.LOD_TENSOR - if framework._non_static_mode() else - (param.type if type is None else type), + if framework._non_static_mode() + else (param.type if type is None else type), shape=shape, - belong_to_optimizer=True) + belong_to_optimizer=True, + ) if device is None: device = self._get_device_for_param(param.name) with device_guard(device): self.helper.set_variable_initializer( - var, initializer=Constant(value=float(fill_value))) + var, initializer=Constant(value=float(fill_value)) + ) if framework._non_static_mode(): if len(self._accumulators_holder) > 0: - assert var_name in self._accumulators_holder, \ - "Optimizer set error, {} should in state dict".format( var_name ) + assert ( + var_name in self._accumulators_holder + ), "Optimizer set error, {} should in state dict".format( + var_name + ) var.set_value(self._accumulators_holder[var_name]) self._accumulators[name][param.name] = var return var - def _add_global_accumulator(self, - name, - dtype=None, - fill_value=0.0, - shape=None, - type=None, - device=None): + def _add_global_accumulator( + self, + name, + dtype=None, + fill_value=0.0, + shape=None, + type=None, + device=None, + ): """Utility function to add a global accumulator for all parameters in the model Args: @@ -659,7 +767,7 @@ def _add_global_accumulator(self, """ if self._name is not None: name = self._name + "_" + name - if (name in self._global_accumulators): + if name in self._global_accumulators: if framework._non_static_mode(): return self._global_accumulators[name] raise Exception("Global accumulator {} already exists".format(name)) @@ -677,17 +785,22 @@ def _add_global_accumulator(self, dtype=dtype if dtype else self._dtype, type=type, shape=shape, - belong_to_optimizer=True) + belong_to_optimizer=True, + ) if device is None: device = 'cpu' with device_guard(device): self.helper.set_variable_initializer( - var, initializer=Constant(value=float(fill_value))) + var, initializer=Constant(value=float(fill_value)) + ) if framework._non_static_mode(): if len(self._accumulators_holder) > 0: - assert var_name in self._accumulators_holder, \ - "Optimizer set error, {} should in state dict".format( var_name ) + assert ( + var_name in self._accumulators_holder + ), "Optimizer set error, {} should in state dict".format( + var_name + ) var.set_value(self._accumulators_holder[var_name]) self._global_accumulators[name] = var @@ -705,11 +818,15 @@ def _get_accumulator(self, name, param): """ if self._name is not None: name = self._name + "_" + name - if (name not in self._accumulators - or param.name not in self._accumulators[name]): + if ( + name not in self._accumulators + or param.name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, param.name)) + name, param.name + ) + ) return self._accumulators[name][param.name] def _get_global_accumulator(self, name): @@ -723,7 +840,7 @@ def _get_global_accumulator(self, name): """ if self._name is not None: name = self._name + "_" + name - if (name not in self._global_accumulators): + if name not in self._global_accumulators: raise Exception("Global accumulator {} does not exist".format(name)) return self._global_accumulators[name] @@ -732,13 +849,15 @@ def _update_param_device_map(self, parameters_and_grads, target_block): if param_and_grad[0].trainable is True: param_name = param_and_grad[0].name ops = target_block.ops - device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName( + device_attr_name = ( + core.op_proto_and_checker_maker.kOpDeviceAttrName() ) for op in ops: input_arg_names = op.input_arg_names if param_name in input_arg_names: self._param_device_map[param_name] = op.attr( - device_attr_name) + device_attr_name + ) break def _get_device_for_param(self, param_name): @@ -775,17 +894,19 @@ def _create_optimization_pass(self, parameters_and_grads): target_block = global_block current_block = framework.default_main_program().current_block() if current_block.idx != global_block.idx: - assert current_block.backward_block_idx != -1, \ - "current block is not global_block, but it doesn't have backward block." + assert ( + current_block.backward_block_idx != -1 + ), "current block is not global_block, but it doesn't have backward block." target_block = framework.default_main_program().blocks[ - current_block.backward_block_idx] + current_block.backward_block_idx + ] start = len(target_block.ops) self._update_param_device_map(parameters_and_grads, target_block) self._create_accumulators( - target_block, - [p[0] for p in parameters_and_grads if p[0].trainable]) + target_block, [p[0] for p in parameters_and_grads if p[0].trainable] + ) self._create_global_learning_rate() if framework._non_static_mode(): @@ -799,13 +920,16 @@ def _create_optimization_pass(self, parameters_and_grads): if param_and_grad[1] is None: continue with param_and_grad[0].block.program._optimized_guard( - param_and_grad), name_scope("optimizer"): + param_and_grad + ), name_scope("optimizer"): if param_and_grad[0].trainable is True: device = self._get_device_for_param( - param_and_grad[0].name) + param_and_grad[0].name + ) with device_guard(device): optimize_op = self._append_optimize_op( - target_block, param_and_grad) + target_block, param_and_grad + ) # Get custom finish ops for subclasses # FIXME: Need to fix this once we figure out how to handle dependencies @@ -834,7 +958,8 @@ def _process_distribute_lookuptable(self, param_grads): if p.name == table_name: if table_param is not None: raise RuntimeError( - "multi dist table var found, only support one now!") + "multi dist table var found, only support one now!" + ) table_param = p table_grad = g else: @@ -842,8 +967,9 @@ def _process_distribute_lookuptable(self, param_grads): sgd_op = None if table_param is not None: param_and_grad = [table_param, table_grad] - with table_param.block.program._optimized_guard(param_and_grad), \ - framework.name_scope("optimizer"): + with table_param.block.program._optimized_guard( + param_and_grad + ), framework.name_scope("optimizer"): self._create_global_learning_rate() # create the optimize op sgd_op = global_block.append_op( @@ -851,17 +977,20 @@ def _process_distribute_lookuptable(self, param_grads): inputs={ "Param": table_param, "Grad": table_grad, - "LearningRate": self._create_param_lr(param_and_grad) + "LearningRate": self._create_param_lr(param_and_grad), }, - outputs={"ParamOut": param_and_grad[0]}) + outputs={"ParamOut": param_and_grad[0]}, + ) return new_param_grads, (table_param, table_grad), sgd_op - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): """ The first part of ``minimize``, do auto-diff to append backward operations for the current program. @@ -897,8 +1026,9 @@ def backward(self, self._dtype = loss.dtype if framework._non_static_mode(): - parameter_list = parameter_list if parameter_list \ - else self._parameter_list + parameter_list = ( + parameter_list if parameter_list else self._parameter_list + ) params_grads = [] for param in parameter_list: @@ -912,29 +1042,36 @@ def backward(self, if callbacks is None: callbacks = [error_clip_callback] else: - assert (isinstance(callbacks, list)) + assert isinstance(callbacks, list) program = loss.block.program - assert len(loss.shape) == 1 and loss.shape[0] == 1, \ - "The loss.shape should be (1L,), but the current loss.shape is {}. " \ + assert len(loss.shape) == 1 and loss.shape[0] == 1, ( + "The loss.shape should be (1L,), but the current loss.shape is {}. " "Maybe that you should call paddle.mean to process the current loss.".format( - loss.shape) - parameter_list = parameter_list if parameter_list \ - else self._parameter_list + loss.shape + ) + ) + parameter_list = ( + parameter_list if parameter_list else self._parameter_list + ) with program_guard(program, startup_program): - params_grads = append_backward(loss, parameter_list, - act_no_grad_set, callbacks) + params_grads = append_backward( + loss, parameter_list, act_no_grad_set, callbacks + ) return params_grads def _create_regularization_of_grad(self, param, grad, regularization=None): - """ Create and add backward regularization Operators + """Create and add backward regularization Operators Function helper of append_regularization_ops. """ # If no gradient or no regularization is specified, then we don't need to do anything if grad is None or ( - (not hasattr(param, 'regularizer') or - (hasattr(param, 'regularizer') and param.regularizer is None)) - and regularization is None): + ( + not hasattr(param, 'regularizer') + or (hasattr(param, 'regularizer') and param.regularizer is None) + ) + and regularization is None + ): return grad regularization_term = None if hasattr(param, 'regularizer') and param.regularizer is not None: @@ -959,7 +1096,8 @@ def _create_regularization_of_grad(self, param, grad, regularization=None): dtype=param.dtype, shape=param.shape, lod_level=param.lod_level, - type=core.VarDesc.VarType.LOD_TENSOR) + type=core.VarDesc.VarType.LOD_TENSOR, + ) inputs = {"X": [grad, regularization_term]} outputs = {"Out": [new_grad]} @@ -967,9 +1105,9 @@ def _create_regularization_of_grad(self, param, grad, regularization=None): return new_grad - def append_regularization_ops(self, - parameters_and_grads, - regularization=None): + def append_regularization_ops( + self, parameters_and_grads, regularization=None + ): r"""Create and add backward regularization Operators Creates and adds backward regularization operators in the BlockDesc. @@ -994,23 +1132,28 @@ def append_regularization_ops(self, if framework._non_static_mode(): for param, grad in parameters_and_grads: new_grad = self._create_regularization_of_grad( - param, grad, regularization) + param, grad, regularization + ) params_and_grads.append((param, new_grad)) else: repeate_regularizer = False with framework.name_scope('regularization'): for param, grad in parameters_and_grads: - if not repeate_regularizer and getattr( - param, 'regularizer', - None) is not None and regularization is not None: + if ( + not repeate_regularizer + and getattr(param, 'regularizer', None) is not None + and regularization is not None + ): repeate_regularizer = True logging.info( "If regularizer of a Parameter has been set by 'fluid.ParamAttr' or 'fluid.WeightNormParamAttr' already. " "The Regularization[%s] in Optimizer will not take effect, and it will only be applied to other Parameters!" - % regularization.__str__()) + % regularization.__str__() + ) with param.block.program._optimized_guard([param, grad]): new_grad = self._create_regularization_of_grad( - param, grad, regularization) + param, grad, regularization + ) params_and_grads.append((param, new_grad)) return params_and_grads @@ -1021,11 +1164,14 @@ def flatten_param_grads(self, params_grads): if g is None: continue g.persistable = True - if getattr(p, 'need_clip', True) is False or getattr( - p, 'regularizer', None) is not None: + if ( + getattr(p, 'need_clip', True) is False + or getattr(p, 'regularizer', None) is not None + ): warnings.warn( "flatten_param_grads=True will be discarded since paramter '{}''s need_clip is False or " - "the regularizer is set".format(p.name)) + "the regularizer is set".format(p.name) + ) self._flatten_param_grads = False return params_grads @@ -1040,7 +1186,8 @@ def flatten_param_grads(self, params_grads): persistable=True, dtype=need_flatten_params[0].dtype, shape=[np.sum(shape)], - belong_to_optimizer=True) + belong_to_optimizer=True, + ) flatten_param.trainable = True flatten_param.optimize_attr = need_flatten_params[0].optimize_attr @@ -1051,41 +1198,48 @@ def flatten_param_grads(self, params_grads): persistable=True, dtype=need_flatten_grads[0].dtype, shape=[np.sum(shape)], - belong_to_optimizer=True) + belong_to_optimizer=True, + ) with program_guard(default_main_program()): - block.append_op(type="coalesce_tensor", - inputs={"Input": need_flatten_params}, - outputs={ - "Output": need_flatten_params, - "FusedOutput": flatten_param - }, - attrs={ - "copy_data": True, - "use_align": True, - "align_size": self._align_size, - "dtype": need_flatten_params[0].dtype - }) - - block.append_op(type="coalesce_tensor", - inputs={"Input": need_flatten_grads}, - outputs={ - "Output": need_flatten_grads, - "FusedOutput": flatten_grad - }, - attrs={ - "copy_data": True, - "use_align": True, - "align_size": self._align_size, - "dtype": need_flatten_grads[0].dtype - }) + block.append_op( + type="coalesce_tensor", + inputs={"Input": need_flatten_params}, + outputs={ + "Output": need_flatten_params, + "FusedOutput": flatten_param, + }, + attrs={ + "copy_data": True, + "use_align": True, + "align_size": self._align_size, + "dtype": need_flatten_params[0].dtype, + }, + ) + + block.append_op( + type="coalesce_tensor", + inputs={"Input": need_flatten_grads}, + outputs={ + "Output": need_flatten_grads, + "FusedOutput": flatten_grad, + }, + attrs={ + "copy_data": True, + "use_align": True, + "align_size": self._align_size, + "dtype": need_flatten_grads[0].dtype, + }, + ) - #NOTE(zhiqiu): the initializer should be set after coalesce_tensor op, + # NOTE(zhiqiu): the initializer should be set after coalesce_tensor op, # so the shape of flatten_param and flatten_grad will be inferred. - self.helper.set_variable_initializer(flatten_param, - initializer=Constant(0.0)) - self.helper.set_variable_initializer(flatten_grad, - initializer=Constant(0.0)) + self.helper.set_variable_initializer( + flatten_param, initializer=Constant(0.0) + ) + self.helper.set_variable_initializer( + flatten_grad, initializer=Constant(0.0) + ) return [(flatten_param, flatten_grad)] @@ -1115,8 +1269,9 @@ def apply_gradients(self, params_grads): # NOTE(zhiqiu): currently, only support ClipGradByGlobalNorm and without regularization. if self._flatten_param_grads and self.regularization is None: - if self._grad_clip == None or isinstance(self._grad_clip, - ClipGradByGlobalNorm): + if self._grad_clip == None or isinstance( + self._grad_clip, ClipGradByGlobalNorm + ): params_grads = self.flatten_param_grads(params_grads) # 'optimizer(grad_clip)' or 'set_gradient_clip' @@ -1126,8 +1281,9 @@ def apply_gradients(self, params_grads): params_grads = append_gradient_clip_ops(params_grads) # Add regularization if any - params_grads = self.append_regularization_ops(params_grads, - self.regularization) + params_grads = self.append_regularization_ops( + params_grads, self.regularization + ) optimize_ops = self._create_optimization_pass(params_grads) return optimize_ops @@ -1145,12 +1301,15 @@ def apply_optimize(self, loss, startup_program, params_grads): list: A list of operators appended to the current program. """ if framework._non_static_mode(): - with program_guard(framework.default_main_program(), - framework.default_startup_program()): + with program_guard( + framework.default_main_program(), + framework.default_startup_program(), + ): if self._grad_clip is not None: params_grads = self._grad_clip(params_grads) params_grads = self.append_regularization_ops( - params_grads, self.regularization) + params_grads, self.regularization + ) optimize_ops = self._create_optimization_pass(params_grads) else: program = loss.block.program @@ -1162,7 +1321,8 @@ def _get_no_grad_set(self, loss, no_grad_set=None): no_grad_set = _get_no_grad_set_name(no_grad_set) parameters = loss.block.program.global_block().all_parameters() param_no_trainable = set( - [param.name for param in parameters if param.trainable is False]) + [param.name for param in parameters if param.trainable is False] + ) # If the parameter is no trainable, it should not have a gradient. no_grad_set.update(param_no_trainable) @@ -1202,11 +1362,9 @@ def clear_gradients(self): p.clear_gradient() @imperative_base.no_grad - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): """ Add operations to minimize ``loss`` by updating ``parameter_list``. @@ -1234,17 +1392,20 @@ def minimize(self, """ assert isinstance(loss, Variable), "The loss should be an Variable." - parameter_list = parameter_list if parameter_list \ - else self._parameter_list + parameter_list = ( + parameter_list if parameter_list else self._parameter_list + ) - params_grads = self.backward(loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) + params_grads = self.backward( + loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) - optimize_ops = self.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) + optimize_ops = self.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) return optimize_ops, params_grads @@ -1305,19 +1466,23 @@ class SGDOptimizer(Optimizer): """ - def __init__(self, - learning_rate, - parameter_list=None, - regularization=None, - grad_clip=None, - multi_precision=False, - name=None): + def __init__( + self, + learning_rate, + parameter_list=None, + regularization=None, + grad_clip=None, + multi_precision=False, + name=None, + ): assert learning_rate is not None - super(SGDOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(SGDOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "sgd" self._use_mkldnn = False self._multi_precision = multi_precision @@ -1331,19 +1496,23 @@ def _create_master_weight(self, param): var_name = param.name + "_fp32_master" var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -1357,7 +1526,10 @@ def _create_accumulators(self, block, parameters): if self._multi_precision and p.dtype == core.VarDesc.VarType.FP16: master_p = self._create_master_weight(p) continue - if p.dtype == core.VarDesc.VarType.FP16 and not self._multi_precision: + if ( + p.dtype == core.VarDesc.VarType.FP16 + and not self._multi_precision + ): warnings.warn( "Accumulating with FP16 in optimizer can lead to poor accuracy or slow convergence." "Consider using multi_precision=True option of the Adam optimizer." @@ -1366,19 +1538,35 @@ def _create_accumulators(self, block, parameters): @no_grad def _append_optimize_op(self, block, param_and_grad): - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 - master_weight = (self._master_weights[param_and_grad[0].name] - if find_master else None) + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) + master_weight = ( + self._master_weights[param_and_grad[0].name] + if find_master + else None + ) lr = self._create_param_lr(param_and_grad) if in_dygraph_mode(): - _C_ops.sgd_(param_and_grad[0], lr, param_and_grad[1], master_weight, - find_master) + _C_ops.sgd_( + param_and_grad[0], + lr, + param_and_grad[1], + master_weight, + find_master, + ) return None if _in_legacy_dygraph(): - _legacy_C_ops.sgd(param_and_grad[0], lr, param_and_grad[1], - master_weight, param_and_grad[0], master_weight) + _legacy_C_ops.sgd( + param_and_grad[0], + lr, + param_and_grad[1], + master_weight, + param_and_grad[0], + master_weight, + ) return None assert isinstance(block, framework.Block) @@ -1386,7 +1574,7 @@ def _append_optimize_op(self, block, param_and_grad): inputs = { "Param": param_and_grad[0], "Grad": param_and_grad[1], - "LearningRate": lr + "LearningRate": lr, } outputs = {"ParamOut": param_and_grad[0]} @@ -1397,11 +1585,13 @@ def _append_optimize_op(self, block, param_and_grad): inputs["MasterParam"] = master_weight outputs["MasterParamOut"] = master_weight - sgd_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + sgd_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return sgd_op @@ -1478,21 +1668,25 @@ class MomentumOptimizer(Optimizer): """ _velocity_acc_str = "velocity" - def __init__(self, - learning_rate, - momentum, - parameter_list=None, - use_nesterov=False, - regularization=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate, + momentum, + parameter_list=None, + use_nesterov=False, + regularization=None, + grad_clip=None, + name=None, + ): assert learning_rate is not None assert momentum is not None - super(MomentumOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(MomentumOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "momentum" self._momentum = momentum self._use_nesterov = bool(use_nesterov) @@ -1506,15 +1700,26 @@ def _create_accumulators(self, block, parameters): def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) - velocity_acc = self._get_accumulator(self._velocity_acc_str, - param_and_grad[0]) + velocity_acc = self._get_accumulator( + self._velocity_acc_str, param_and_grad[0] + ) lr = self._create_param_lr(param_and_grad) master_weight = None if framework._non_static_mode(): _, _, _ = _legacy_C_ops.momentum( - param_and_grad[0], param_and_grad[1], velocity_acc, lr, - master_weight, param_and_grad[0], velocity_acc, master_weight, - 'mu', self._momentum, 'use_nesterov', self._use_nesterov) + param_and_grad[0], + param_and_grad[1], + velocity_acc, + lr, + master_weight, + param_and_grad[0], + velocity_acc, + master_weight, + 'mu', + self._momentum, + 'use_nesterov', + self._use_nesterov, + ) return None attrs = {"mu": self._momentum, "use_nesterov": self._use_nesterov} @@ -1522,19 +1727,21 @@ def _append_optimize_op(self, block, param_and_grad): "Param": [param_and_grad[0]], "Grad": [param_and_grad[1]], "Velocity": [velocity_acc], - "LearningRate": [lr] + "LearningRate": [lr], } outputs = { "ParamOut": [param_and_grad[0]], - "VelocityOut": [velocity_acc] + "VelocityOut": [velocity_acc], } # create the momentum optimize op - momentum_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + momentum_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return momentum_op @@ -1607,32 +1814,36 @@ class DGCMomentumOptimizer(Optimizer): _u_velocity_acc_str = "_dgc_u_" _v_velocity_acc_str = "_dgc_v_" - def __init__(self, - learning_rate, - momentum, - rampup_begin_step, - rampup_step=1, - sparsity=[0.999], - parameter_list=None, - use_nesterov=False, - num_trainers=None, - regularization=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate, + momentum, + rampup_begin_step, + rampup_step=1, + sparsity=[0.999], + parameter_list=None, + use_nesterov=False, + num_trainers=None, + regularization=None, + grad_clip=None, + name=None, + ): if framework._non_static_mode(): raise Exception("In dygraph, don't support DGCMomentumOptimizer.") - assert core.is_compiled_with_cuda(), \ - "Paddle is not compiled with CUDA. DGC is only support GPU for now." + assert ( + core.is_compiled_with_cuda() + ), "Paddle is not compiled with CUDA. DGC is only support GPU for now." assert learning_rate is not None assert momentum is not None - super(DGCMomentumOptimizer, - self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(DGCMomentumOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "dgc_momentum" self._momentum = momentum self._use_nesterov = bool(use_nesterov) @@ -1651,17 +1862,20 @@ def __init__(self, raise TypeError( "The type of grad_clip should be 'GradientClipByNorm', because DGCMomentumOptimizer only support GradientClipByNorm" ) - assert isinstance( - num_trainers, int - ), "The type of num_trainers should be 'int', but received %s" % type( - num_trainers) - assert num_trainers > 0, "The value of num_trainers should be greater than 0!" + assert isinstance(num_trainers, int), ( + "The type of num_trainers should be 'int', but received %s" + % type(num_trainers) + ) + assert ( + num_trainers > 0 + ), "The value of num_trainers should be greater than 0!" self._num_trainers = num_trainers self._dgc_clip_norm = grad_clip.clip_norm * (num_trainers**-0.5) self.regular_type, self.regular_coeff = self._get_regularization_param( - self.regularization) + self.regularization + ) def _get_regularization_param(self, regularization): regular_type = 0 @@ -1670,6 +1884,7 @@ def _get_regularization_param(self, regularization): if regularization is not None: regular_coeff = regularization._regularization_coeff from .regularizer import L1Decay, L2Decay + if isinstance(regularization, L1Decay): regular_type = 1 elif isinstance(regularization, L2Decay): @@ -1680,17 +1895,20 @@ def _get_regularization_param(self, regularization): def _is_use_dgc(self, param_var, grad_var): var_numel = abs(reduce(lambda x, y: x * y, param_var.shape)) - if var_numel < 16384 or \ - param_var.type == core.VarDesc.VarType.SELECTED_ROWS or \ - grad_var.type == core.VarDesc.VarType.SELECTED_ROWS or \ - param_var.dtype != core.VarDesc.VarType.FP32 : + if ( + var_numel < 16384 + or param_var.type == core.VarDesc.VarType.SELECTED_ROWS + or grad_var.type == core.VarDesc.VarType.SELECTED_ROWS + or param_var.dtype != core.VarDesc.VarType.FP32 + ): return False return True def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) - velocity_acc = self._get_accumulator(self._u_velocity_acc_str, - param_and_grad[0]) + velocity_acc = self._get_accumulator( + self._u_velocity_acc_str, param_and_grad[0] + ) assert velocity_acc is not None inputs = { @@ -1709,36 +1927,42 @@ def _append_optimize_op(self, block, param_and_grad): type = "momentum" else: type = "dgc_momentum" - inputs.update({ - "current_step": self._global_step_var, - "nranks": self._nranks_var - }) + inputs.update( + { + "current_step": self._global_step_var, + "nranks": self._nranks_var, + } + ) outputs.update({'Grad_out': param_and_grad[1]}) attrs.update({"rampup_begin_step": float(self._rampup_begin_step)}) # create the dgc momentum optimize op - dgc_momentum_op = block.append_op(type=type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + dgc_momentum_op = block.append_op( + type=type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return dgc_momentum_op def _add_auto_increment_var(self, counter_name, begin, step=1): helper = LayerHelper('global_step_counter') counter, is_new_var = helper.create_or_get_global_variable( - name=counter_name, dtype='float32', shape=[1], persistable=True) + name=counter_name, dtype='float32', shape=[1], persistable=True + ) if is_new_var: - helper.set_variable_initializer(counter, - initializer=Constant( - value=float(begin - 1), - force_cpu=True)) + helper.set_variable_initializer( + counter, + initializer=Constant(value=float(begin - 1), force_cpu=True), + ) helper.main_program.global_block()._prepend_op( type='increment', inputs={'X': [counter]}, outputs={'Out': [counter]}, attrs={'step': float(step)}, - stop_gradient=True) + stop_gradient=True, + ) counter.stop_gradient = True return counter @@ -1746,12 +1970,13 @@ def _add_auto_increment_var(self, counter_name, begin, step=1): def _add_nranks_var(self, name, value=-1): helper = LayerHelper('global_step_counter') counter, is_new_var = helper.create_or_get_global_variable( - name=name, dtype='float32', shape=[1], persistable=True) + name=name, dtype='float32', shape=[1], persistable=True + ) if is_new_var: - helper.set_variable_initializer(counter, - initializer=Constant( - value=float(value), - force_cpu=True)) + helper.set_variable_initializer( + counter, + initializer=Constant(value=float(value), force_cpu=True), + ) counter.stop_gradient = True return counter @@ -1762,10 +1987,12 @@ def _append_dgc_ops(self, param_and_grads): # step counter self._global_step_var = self._add_auto_increment_var( - counter_name=core.dgc.kDGCCounterName(), begin=0) + counter_name=core.dgc.kDGCCounterName(), begin=0 + ) - self._nranks_var = self._add_nranks_var(name=core.dgc.kDGCNRanksName(), - value=-1) + self._nranks_var = self._add_nranks_var( + name=core.dgc.kDGCNRanksName(), value=-1 + ) # rampup begin step var for all_reduce_op_handle self._rampup_begin_step_var = tensor.create_global_var( @@ -1774,7 +2001,8 @@ def _append_dgc_ops(self, param_and_grads): persistable=True, name=core.dgc.kDGCRampUpBeginStepName(), value=self._rampup_begin_step * 1.0, - force_cpu=True) + force_cpu=True, + ) self.helper = LayerHelper(self.__class__.__name__) @@ -1787,29 +2015,32 @@ def _append_dgc_ops(self, param_and_grads): v_var = self._add_accumulator(self._v_velocity_acc_str, param_var) - k_var = tensor.create_global_var(shape=[1], - dtype=param_var.dtype, - persistable=True, - name=param_var.name + - core.dgc.kDGCKName(), - value=0.0, - force_cpu=True) - - encoded_var = tensor.create_global_var(shape=[1], - dtype=param_var.dtype, - persistable=True, - name=param_var.name + - core.dgc.kDGCEncodedName(), - value=0.0, - force_cpu=False) - - gather_var = tensor.create_global_var(shape=[1], - dtype=param_var.dtype, - persistable=True, - name=param_var.name + - core.dgc.kDGCGatherName(), - value=0.0, - force_cpu=False) + k_var = tensor.create_global_var( + shape=[1], + dtype=param_var.dtype, + persistable=True, + name=param_var.name + core.dgc.kDGCKName(), + value=0.0, + force_cpu=True, + ) + + encoded_var = tensor.create_global_var( + shape=[1], + dtype=param_var.dtype, + persistable=True, + name=param_var.name + core.dgc.kDGCEncodedName(), + value=0.0, + force_cpu=False, + ) + + gather_var = tensor.create_global_var( + shape=[1], + dtype=param_var.dtype, + persistable=True, + name=param_var.name + core.dgc.kDGCGatherName(), + value=0.0, + force_cpu=False, + ) # del back oprolevarname op_maker = core.op_proto_and_checker_maker @@ -1832,14 +2063,23 @@ def _append_dgc_ops(self, param_and_grads): clip_var = grad_var if self._dgc_clip_norm is not None: clip_var = self._append_clip_norm(grad_var, self._dgc_clip_norm) - self._dgc_op(param_var, clip_var, grad_var, u_var, v_var, k_var, - encoded_var, gather_var) + self._dgc_op( + param_var, + clip_var, + grad_var, + u_var, + v_var, + k_var, + encoded_var, + gather_var, + ) def _is_the_backward_op(self, op): op_maker = core.op_proto_and_checker_maker backward = core.op_proto_and_checker_maker.OpRole.Backward - if op_maker.kOpRoleVarAttrName() in op.attr_names and \ - int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(backward): + if op_maker.kOpRoleVarAttrName() in op.attr_names and int( + op.all_attrs()[op_maker.kOpRoleAttrName()] + ) == int(backward): return True return False @@ -1849,34 +2089,42 @@ def _clip_by_norm(self, x, max_norm, name=None): helper = LayerHelper("dgc_clip_by_norm_op", **args) if name is None: - name = unique_name.generate_with_ignorable_key(".".join( - [helper.name, 'tmp'])) - - out = helper.create_variable(type=x.type, - name=name, - dtype=x.dtype, - persistable=False) - - helper.append_op(type="dgc_clip_by_norm", - inputs={ - "X": x, - "current_step": self._global_step_var - }, - attrs={ - "max_norm": max_norm, - "rampup_begin_step": float(self._rampup_begin_step) - }, - outputs={"Out": out}) + name = unique_name.generate_with_ignorable_key( + ".".join([helper.name, 'tmp']) + ) + + out = helper.create_variable( + type=x.type, name=name, dtype=x.dtype, persistable=False + ) + + helper.append_op( + type="dgc_clip_by_norm", + inputs={"X": x, "current_step": self._global_step_var}, + attrs={ + "max_norm": max_norm, + "rampup_begin_step": float(self._rampup_begin_step), + }, + outputs={"Out": out}, + ) return out def _append_clip_norm(self, grad_var, clip_norm): with grad_var.block.program._backward_role_guard(): - return self._clip_by_norm(x=grad_var, - max_norm=clip_norm, - name=grad_var.name) + return self._clip_by_norm( + x=grad_var, max_norm=clip_norm, name=grad_var.name + ) - def _dgc_op(self, param_var, clip_var, grad_var, u_var, v_var, k_var, - encoded_var, gather_var): + def _dgc_op( + self, + param_var, + clip_var, + grad_var, + u_var, + v_var, + k_var, + encoded_var, + gather_var, + ): block = framework.default_main_program().global_block() op_maker = core.op_proto_and_checker_maker @@ -1885,47 +2133,44 @@ def _dgc_op(self, param_var, clip_var, grad_var, u_var, v_var, k_var, # The regularizer of the Parameters have higher priority if param_var.regularizer is not None: regular_type, regular_coeff = self._get_regularization_param( - param_var.regularizer) - - dgc_op = block.append_op(type="dgc", - inputs={ - "U": u_var, - "V": v_var, - "Grad": clip_var, - "Param": param_var, - "current_step": self._global_step_var, - "nranks": self._nranks_var, - }, - outputs={ - "U_out": u_var, - "V_out": v_var, - "EncodeGrad": encoded_var, - "k": k_var, - "Grad_out": grad_var, - "GatherBuff": gather_var, - }, - attrs={ - "m": - self._momentum, - "sparsity": - self._sparsity, - "use_nesterov": - self._use_nesterov, - "rampup_begin_step": - float(self._rampup_begin_step), - "rampup_step": - float(self._rampup_step), - "regular_coeff": - float(regular_coeff), - "regular_type": - int(regular_type), - }, - stop_gradient=True) + param_var.regularizer + ) + + dgc_op = block.append_op( + type="dgc", + inputs={ + "U": u_var, + "V": v_var, + "Grad": clip_var, + "Param": param_var, + "current_step": self._global_step_var, + "nranks": self._nranks_var, + }, + outputs={ + "U_out": u_var, + "V_out": v_var, + "EncodeGrad": encoded_var, + "k": k_var, + "Grad_out": grad_var, + "GatherBuff": gather_var, + }, + attrs={ + "m": self._momentum, + "sparsity": self._sparsity, + "use_nesterov": self._use_nesterov, + "rampup_begin_step": float(self._rampup_begin_step), + "rampup_step": float(self._rampup_step), + "regular_coeff": float(regular_coeff), + "regular_type": int(regular_type), + }, + stop_gradient=True, + ) backward = op_maker.OpRole.Backward dgc_op._set_attr(op_maker.kOpRoleAttrName(), backward) - dgc_op._set_attr(op_maker.kOpRoleVarAttrName(), - [param_var.name, grad_var.name]) + dgc_op._set_attr( + op_maker.kOpRoleVarAttrName(), [param_var.name, grad_var.name] + ) @imperative_base.no_grad def apply_gradients(self, params_grads): @@ -1935,8 +2180,11 @@ def apply_gradients(self, params_grads): self._append_dgc_ops(params_grads) params_grads = sorted(params_grads, key=lambda x: x[0].name) - params_grads, table_param_and_grad, table_optimize_op = \ - self._process_distribute_lookuptable(params_grads) + ( + params_grads, + table_param_and_grad, + table_optimize_op, + ) = self._process_distribute_lookuptable(params_grads) not_dgc_params_grads = [] dgc_params_grads = [] @@ -1952,10 +2200,12 @@ def apply_gradients(self, params_grads): not_dgc_params_grads = self._grad_clip(not_dgc_params_grads) else: not_dgc_params_grads = append_gradient_clip_ops( - not_dgc_params_grads) + not_dgc_params_grads + ) not_dgc_params_grads = self.append_regularization_ops( - not_dgc_params_grads, self.regularization) + not_dgc_params_grads, self.regularization + ) params_grads = not_dgc_params_grads + dgc_params_grads params_grads = sorted(params_grads, key=lambda x: x[0].name) @@ -2031,27 +2281,30 @@ class LarsMomentumOptimizer(Optimizer): """ _velocity_acc_str = "velocity" - def __init__(self, - learning_rate, - momentum, - lars_coeff=0.001, - lars_weight_decay=0.0005, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None, - exclude_from_weight_decay=None, - epsilon=0, - multi_precision=False, - rescale_grad=1.0): + def __init__( + self, + learning_rate, + momentum, + lars_coeff=0.001, + lars_weight_decay=0.0005, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + exclude_from_weight_decay=None, + epsilon=0, + multi_precision=False, + rescale_grad=1.0, + ): assert learning_rate is not None assert momentum is not None - super(LarsMomentumOptimizer, - self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(LarsMomentumOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "lars_momentum" self._momentum = momentum self._lars_coeff = float(lars_coeff) @@ -2073,19 +2326,23 @@ def _create_master_weight(self, param): var_name = param.name + '_fp32_master' var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -2099,15 +2356,22 @@ def _get_accumulator(self, name, param): """ if self._name is not None: name = self._name + "_" + name - find_master = self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 - target_param = self._master_weights[ - param.name] if find_master else param + find_master = ( + self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 + ) + target_param = ( + self._master_weights[param.name] if find_master else param + ) target_name = target_param.name - if (name not in self._accumulators - or target_name not in self._accumulators[name]): + if ( + name not in self._accumulators + or target_name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, target_name)) + name, target_name + ) + ) return self._accumulators[name][target_name] def _create_accumulators(self, block, parameters): @@ -2118,7 +2382,10 @@ def _create_accumulators(self, block, parameters): master_p = self._create_master_weight(p) self._add_accumulator(self._velocity_acc_str, master_p) continue - if p.dtype == core.VarDesc.VarType.FP16 and not self._multi_precision: + if ( + p.dtype == core.VarDesc.VarType.FP16 + and not self._multi_precision + ): warnings.warn( "Accumulating with FP16 in optimizer can lead to poor accuracy or slow convergence." "Consider using multi_precision=True option of the Lars optimizer." @@ -2135,14 +2402,20 @@ def _append_optimize_op(self, block, param_and_grad): _lars_weight_decay = 0.0 break - velocity_acc = self._get_accumulator(self._velocity_acc_str, - param_and_grad[0]) + velocity_acc = self._get_accumulator( + self._velocity_acc_str, param_and_grad[0] + ) lr = self._create_param_lr(param_and_grad) - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 - master_weight = (self._master_weights[param_and_grad[0].name] - if find_master else None) + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) + master_weight = ( + self._master_weights[param_and_grad[0].name] + if find_master + else None + ) attrs = { "mu": self._momentum, @@ -2150,14 +2423,14 @@ def _append_optimize_op(self, block, param_and_grad): "lars_weight_decay": [_lars_weight_decay], "multi_precision": find_master, "epsilon": self._epsilon, - "rescale_grad": self._rescale_grad + "rescale_grad": self._rescale_grad, } inputs = { "Param": param_and_grad[0], "Grad": param_and_grad[1], "Velocity": velocity_acc, - "LearningRate": lr + "LearningRate": lr, } outputs = {"ParamOut": param_and_grad[0], "VelocityOut": velocity_acc} @@ -2168,18 +2441,34 @@ def _append_optimize_op(self, block, param_and_grad): if framework._non_static_mode(): tmp, tmp2 = _legacy_C_ops.lars_momentum( - [param_and_grad[0]], [param_and_grad[1]], [velocity_acc], [lr], - [param_and_grad[0]], [velocity_acc], "mu", self._momentum, - "lars_coeff", self._lars_coeff, "lars_weight_decay", - [_lars_weight_decay], "multi_precision", find_master, "epsilon", - self._epsilon, "rescale_grad", self._rescale_grad) + [param_and_grad[0]], + [param_and_grad[1]], + [velocity_acc], + [lr], + [param_and_grad[0]], + [velocity_acc], + "mu", + self._momentum, + "lars_coeff", + self._lars_coeff, + "lars_weight_decay", + [_lars_weight_decay], + "multi_precision", + find_master, + "epsilon", + self._epsilon, + "rescale_grad", + self._rescale_grad, + ) else: # create the momentum optimize op - momentum_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + momentum_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return momentum_op @@ -2249,21 +2538,25 @@ class AdagradOptimizer(Optimizer): """ _moment_acc_str = "moment" - def __init__(self, - learning_rate, - epsilon=1.0e-6, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None, - initial_accumulator_value=0.0): + def __init__( + self, + learning_rate, + epsilon=1.0e-6, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + initial_accumulator_value=0.0, + ): assert learning_rate is not None assert epsilon is not None - super(AdagradOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(AdagradOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "adagrad" self._epsilon = epsilon self.initial_accumulator_value = initial_accumulator_value @@ -2272,26 +2565,38 @@ def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) for p in parameters: - self._add_accumulator(self._moment_acc_str, - p, - fill_value=self.initial_accumulator_value) + self._add_accumulator( + self._moment_acc_str, + p, + fill_value=self.initial_accumulator_value, + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) - moment_acc = self._get_accumulator(self._moment_acc_str, - param_and_grad[0]) + moment_acc = self._get_accumulator( + self._moment_acc_str, param_and_grad[0] + ) if in_dygraph_mode(): - _C_ops.adagrad_(param_and_grad[0], param_and_grad[1], moment_acc, - self._create_param_lr(param_and_grad), - self._epsilon) + _C_ops.adagrad_( + param_and_grad[0], + param_and_grad[1], + moment_acc, + self._create_param_lr(param_and_grad), + self._epsilon, + ) return None elif _in_legacy_dygraph(): - _legacy_C_ops.adagrad(param_and_grad[0], param_and_grad[1], - moment_acc, - self._create_param_lr(param_and_grad), - param_and_grad[0], moment_acc, "epsilon", - self._epsilon) + _legacy_C_ops.adagrad( + param_and_grad[0], + param_and_grad[1], + moment_acc, + self._create_param_lr(param_and_grad), + param_and_grad[0], + moment_acc, + "epsilon", + self._epsilon, + ) return None else: # Create the adagrad optimizer op @@ -2301,14 +2606,15 @@ def _append_optimize_op(self, block, param_and_grad): "Param": param_and_grad[0], "Grad": param_and_grad[1], "Moment": moment_acc, - "LearningRate": self._create_param_lr(param_and_grad) + "LearningRate": self._create_param_lr(param_and_grad), }, outputs={ "ParamOut": param_and_grad[0], - "MomentOut": moment_acc + "MomentOut": moment_acc, }, attrs={"epsilon": self._epsilon}, - stop_gradient=True) + stop_gradient=True, + ) return adagrad_op @@ -2476,31 +2782,34 @@ def get_decayed_betas(beta1_init, beta2_init, decay_steps, decay_rate, epsilon_i _beta1_pow_acc_str = "beta1_pow_acc" _beta2_pow_acc_str = "beta2_pow_acc" - def __init__(self, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None, - lazy_mode=False, - use_global_beta_pow=False, - flatten_param_grads=False, - align_size=-1): + def __init__( + self, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + lazy_mode=False, + use_global_beta_pow=False, + flatten_param_grads=False, + align_size=-1, + ): assert learning_rate is not None assert beta1 is not None assert beta2 is not None assert epsilon is not None - super(AdamOptimizer, - self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - flatten_param_grads=flatten_param_grads, - align_size=align_size, - name=name) + super(AdamOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + flatten_param_grads=flatten_param_grads, + align_size=align_size, + name=name, + ) self.type = "adam" self._beta1 = beta1 self._beta2 = beta2 @@ -2519,65 +2828,109 @@ def _create_accumulators(self, block, parameters): self._add_accumulator( name=self._beta1_pow_acc_str, param=p, - fill_value=0.9 if isinstance(self._beta1, Variable) \ - else self._beta1, + fill_value=0.9 + if isinstance(self._beta1, Variable) + else self._beta1, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) self._add_accumulator( name=self._beta2_pow_acc_str, param=p, - fill_value=0.999 if isinstance(self._beta2, Variable) \ - else self._beta2, + fill_value=0.999 + if isinstance(self._beta2, Variable) + else self._beta2, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) if self._use_global_beta_pow: self._add_global_accumulator( name=self._beta1_pow_acc_str, - fill_value=0.9 if isinstance(self._beta1, Variable) \ - else self._beta1, + fill_value=0.9 + if isinstance(self._beta1, Variable) + else self._beta1, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) self._add_global_accumulator( name=self._beta2_pow_acc_str, - fill_value=0.999 if isinstance(self._beta2, Variable) \ - else self._beta2, + fill_value=0.999 + if isinstance(self._beta2, Variable) + else self._beta2, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) - moment1 = self._get_accumulator(self._moment1_acc_str, - param_and_grad[0]) - moment2 = self._get_accumulator(self._moment2_acc_str, - param_and_grad[0]) + moment1 = self._get_accumulator( + self._moment1_acc_str, param_and_grad[0] + ) + moment2 = self._get_accumulator( + self._moment2_acc_str, param_and_grad[0] + ) if self._use_global_beta_pow: beta1_pow_acc = self._get_global_accumulator( - self._beta1_pow_acc_str) + self._beta1_pow_acc_str + ) beta2_pow_acc = self._get_global_accumulator( - self._beta2_pow_acc_str) + self._beta2_pow_acc_str + ) else: - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) - beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str, - param_and_grad[0]) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) + beta2_pow_acc = self._get_accumulator( + self._beta2_pow_acc_str, param_and_grad[0] + ) lr = self._create_param_lr(param_and_grad) # create the adam optimize op if framework._non_static_mode(): - _beta1 = self._beta1 if not isinstance( - self._beta1, Variable) else self._beta1.numpy().item(0) - _beta2 = self._beta2 if not isinstance( - self._beta2, Variable) else self._beta2.numpy().item(0) + _beta1 = ( + self._beta1 + if not isinstance(self._beta1, Variable) + else self._beta1.numpy().item(0) + ) + _beta2 = ( + self._beta2 + if not isinstance(self._beta2, Variable) + else self._beta2.numpy().item(0) + ) master_weight = None _, _, _, _, _, _ = _legacy_C_ops.adam( - param_and_grad[0], param_and_grad[1], lr, moment1, moment2, - beta1_pow_acc, beta2_pow_acc, master_weight, param_and_grad[0], - moment1, moment2, beta1_pow_acc, beta2_pow_acc, master_weight, - 'epsilon', self._epsilon, 'lazy_mode', self._lazy_mode, - 'min_row_size_to_use_multithread', 1000, 'beta1', _beta1, - 'beta2', _beta2, 'use_global_beta_pow', - self._use_global_beta_pow) + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + param_and_grad[0], + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + 'epsilon', + self._epsilon, + 'lazy_mode', + self._lazy_mode, + 'min_row_size_to_use_multithread', + 1000, + 'beta1', + _beta1, + 'beta2', + _beta2, + 'use_global_beta_pow', + self._use_global_beta_pow, + ) return None @@ -2588,7 +2941,7 @@ def _append_optimize_op(self, block, param_and_grad): "Moment1": [moment1], "Moment2": [moment2], "Beta1Pow": [beta1_pow_acc], - "Beta2Pow": [beta2_pow_acc] + "Beta2Pow": [beta2_pow_acc], } # Pass found_inf to adam, to skip update for not only param, but also momentum and beta_pow @@ -2607,7 +2960,7 @@ def _append_optimize_op(self, block, param_and_grad): attrs = { "lazy_mode": self._lazy_mode, "min_row_size_to_use_multithread": 1000, - 'use_global_beta_pow': self._use_global_beta_pow + 'use_global_beta_pow': self._use_global_beta_pow, } if isinstance(self._beta1, Variable): @@ -2623,23 +2976,26 @@ def _append_optimize_op(self, block, param_and_grad): else: attrs['epsilon'] = self._epsilon - adam_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + adam_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return adam_op def _finish_update(self, block, parameters_and_grads): - r"""Update beta1_pow and beta2_pow accumulator - """ + r"""Update beta1_pow and beta2_pow accumulator""" assert isinstance(block, framework.Block) if self._use_global_beta_pow: beta1_pow_acc = self._get_global_accumulator( - self._beta1_pow_acc_str) + self._beta1_pow_acc_str + ) beta2_pow_acc = self._get_global_accumulator( - self._beta2_pow_acc_str) + self._beta2_pow_acc_str + ) with block.program._optimized_guard([]): inputs = {"X": beta1_pow_acc} @@ -2648,18 +3004,22 @@ def _finish_update(self, block, parameters_and_grads): if isinstance(self._beta1, Variable): inputs["Y"] = self._beta1 # use elementwise_mul for better performance - block.append_op(type="elementwise_mul", - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + block.append_op( + type="elementwise_mul", + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) else: attrs['scale'] = self._beta1 - block.append_op(type="scale", - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + block.append_op( + type="scale", + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) inputs = {"X": beta2_pow_acc} outputs = {"Out": beta2_pow_acc} @@ -2667,18 +3027,22 @@ def _finish_update(self, block, parameters_and_grads): if isinstance(self._beta2, Variable): inputs["Y"] = self._beta2 # use elementwise_mul for better performance - block.append_op(type="elementwise_mul", - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + block.append_op( + type="elementwise_mul", + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) else: attrs['scale'] = self._beta2 - block.append_op(type="scale", - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + block.append_op( + type="scale", + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) class AdamaxOptimizer(Optimizer): @@ -2766,24 +3130,28 @@ class AdamaxOptimizer(Optimizer): _inf_norm_acc_str = "inf_norm" _beta1_pow_acc_str = "beta1_pow_acc" - def __init__(self, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + ): assert learning_rate is not None assert beta1 is not None assert beta2 is not None assert epsilon is not None - super(AdamaxOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(AdamaxOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "adamax" self._beta1 = beta1 self._beta2 = beta2 @@ -2794,31 +3162,54 @@ def _create_accumulators(self, block, parameters): for p in parameters: self._add_accumulator(self._moment_acc_str, p) self._add_accumulator(self._inf_norm_acc_str, p) - self._add_accumulator(name=self._beta1_pow_acc_str, - param=p, - fill_value=self._beta1, - shape=[1]) + self._add_accumulator( + name=self._beta1_pow_acc_str, + param=p, + fill_value=self._beta1, + shape=[1], + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) moment = self._get_accumulator(self._moment_acc_str, param_and_grad[0]) - inf_norm = self._get_accumulator(self._inf_norm_acc_str, - param_and_grad[0]) - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) + inf_norm = self._get_accumulator( + self._inf_norm_acc_str, param_and_grad[0] + ) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) if framework.in_dygraph_mode(): - _C_ops.adamax_(param_and_grad[0], param_and_grad[1], - self._create_param_lr(param_and_grad), moment, - inf_norm, beta1_pow_acc, self._beta1, self._beta2, - self._epsilon) + _C_ops.adamax_( + param_and_grad[0], + param_and_grad[1], + self._create_param_lr(param_and_grad), + moment, + inf_norm, + beta1_pow_acc, + self._beta1, + self._beta2, + self._epsilon, + ) elif framework._in_legacy_dygraph(): - _legacy_C_ops.adamax(param_and_grad[0], param_and_grad[1], - self._create_param_lr(param_and_grad), moment, - inf_norm, beta1_pow_acc, param_and_grad[0], - moment, inf_norm, "beta1", self._beta1, - "beta2", self._beta2, "epsilon", self._epsilon) + _legacy_C_ops.adamax( + param_and_grad[0], + param_and_grad[1], + self._create_param_lr(param_and_grad), + moment, + inf_norm, + beta1_pow_acc, + param_and_grad[0], + moment, + inf_norm, + "beta1", + self._beta1, + "beta2", + self._beta2, + "epsilon", + self._epsilon, + ) else: # create the adamax optimize op adamax_op = block.append_op( @@ -2829,47 +3220,53 @@ def _append_optimize_op(self, block, param_and_grad): "LearningRate": self._create_param_lr(param_and_grad), "Moment": moment, "InfNorm": inf_norm, - "Beta1Pow": beta1_pow_acc + "Beta1Pow": beta1_pow_acc, }, outputs={ "ParamOut": param_and_grad[0], "MomentOut": moment, - "InfNormOut": inf_norm + "InfNormOut": inf_norm, }, attrs={ "beta1": self._beta1, "beta2": self._beta2, - "epsilon": self._epsilon + "epsilon": self._epsilon, }, - stop_gradient=True) + stop_gradient=True, + ) return adamax_op def _finish_update(self, block, parameters_and_grads): - """Update Beta1 Power accumulator - """ + """Update Beta1 Power accumulator""" assert isinstance(block, framework.Block) for param, grad in parameters_and_grads: if grad is None or param.trainable is False: continue - with param.block.program._optimized_guard([param, grad - ]), name_scope('adamx'): - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param) + with param.block.program._optimized_guard( + [param, grad] + ), name_scope('adamx'): + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param + ) if framework._non_static_mode(): if framework.in_dygraph_mode(): - tmp = _C_ops.scale(beta1_pow_acc, self._beta1, 0.0, - True) + tmp = _C_ops.scale( + beta1_pow_acc, self._beta1, 0.0, True + ) else: - tmp = _legacy_C_ops.scale(beta1_pow_acc, "scale", - self._beta1) + tmp = _legacy_C_ops.scale( + beta1_pow_acc, "scale", self._beta1 + ) beta1_pow_acc.copy_(tmp, False) else: - block.append_op(type="scale", - inputs={"X": beta1_pow_acc}, - outputs={"Out": beta1_pow_acc}, - attrs={"scale": self._beta1}, - stop_gradient=True) + block.append_op( + type="scale", + inputs={"X": beta1_pow_acc}, + outputs={"Out": beta1_pow_acc}, + attrs={"scale": self._beta1}, + stop_gradient=True, + ) class DpsgdOptimizer(Optimizer): @@ -2917,18 +3314,21 @@ class DpsgdOptimizer(Optimizer): Currently, DpsgdOptimizer doesn't support sparse parameter optimization. """ - def __init__(self, - learning_rate=0.001, - clip=0.9, - batch_size=0.999, - sigma=1e-8, - parameter_list=None): + def __init__( + self, + learning_rate=0.001, + clip=0.9, + batch_size=0.999, + sigma=1e-8, + parameter_list=None, + ): assert learning_rate is not None assert clip is not None assert batch_size is not None assert sigma is not None - super(DpsgdOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list) + super(DpsgdOptimizer, self).__init__( + learning_rate=learning_rate, parameter_list=parameter_list + ) self.type = "dpsgd" self._clip = clip self._batch_size = batch_size @@ -2949,29 +3349,37 @@ def _append_optimize_op(self, block, param_and_grad): self._seed = 0 if framework._non_static_mode(): - _legacy_C_ops.dpsgd(param_and_grad[0], param_and_grad[1], - self._create_param_lr(param_and_grad), - param_and_grad[0], "clip", self._clip, - "batch_size", self._batch_size, "sigma", - self._sigma, "seed", self._seed) + _legacy_C_ops.dpsgd( + param_and_grad[0], + param_and_grad[1], + self._create_param_lr(param_and_grad), + param_and_grad[0], + "clip", + self._clip, + "batch_size", + self._batch_size, + "sigma", + self._sigma, + "seed", + self._seed, + ) else: - dpsgd_op = block.append_op(type=self.type, - inputs={ - "Param": - param_and_grad[0], - "Grad": - param_and_grad[1], - "LearningRate": - self._create_param_lr(param_and_grad) - }, - outputs={"ParamOut": param_and_grad[0]}, - attrs={ - "clip": self._clip, - "batch_size": self._batch_size, - "sigma": self._sigma, - "seed": self._seed - }, - stop_gradient=True) + dpsgd_op = block.append_op( + type=self.type, + inputs={ + "Param": param_and_grad[0], + "Grad": param_and_grad[1], + "LearningRate": self._create_param_lr(param_and_grad), + }, + outputs={"ParamOut": param_and_grad[0]}, + attrs={ + "clip": self._clip, + "batch_size": self._batch_size, + "sigma": self._sigma, + "seed": self._seed, + }, + stop_gradient=True, + ) return dpsgd_op @@ -3034,24 +3442,27 @@ class DecayedAdagradOptimizer(Optimizer): """ _moment_acc_str = "moment" - def __init__(self, - learning_rate, - decay=0.95, - epsilon=1.0e-6, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate, + decay=0.95, + epsilon=1.0e-6, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + ): assert learning_rate is not None assert decay is not None assert epsilon is not None - super(DecayedAdagradOptimizer, - self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(DecayedAdagradOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "decayed_adagrad" self._decay = decay self._epsilon = epsilon @@ -3065,16 +3476,23 @@ def _create_accumulators(self, block, parameters): def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) - moment_acc = self._get_accumulator(self._moment_acc_str, - param_and_grad[0]) + moment_acc = self._get_accumulator( + self._moment_acc_str, param_and_grad[0] + ) if framework._non_static_mode(): - _legacy_C_ops.decayed_adagrad(param_and_grad[0], param_and_grad[1], - moment_acc, - self._create_param_lr(param_and_grad), - param_and_grad[0], moment_acc, - "epsilon", self._epsilon, "decay", - self._decay) + _legacy_C_ops.decayed_adagrad( + param_and_grad[0], + param_and_grad[1], + moment_acc, + self._create_param_lr(param_and_grad), + param_and_grad[0], + moment_acc, + "epsilon", + self._epsilon, + "decay", + self._decay, + ) else: # Create the decayed adagrad optimizer op decayed_adagrad_op = block.append_op( @@ -3083,17 +3501,15 @@ def _append_optimize_op(self, block, param_and_grad): "Param": param_and_grad[0], "Grad": param_and_grad[1], "Moment": moment_acc, - "LearningRate": self._create_param_lr(param_and_grad) + "LearningRate": self._create_param_lr(param_and_grad), }, outputs={ "ParamOut": param_and_grad[0], - "MomentOut": moment_acc + "MomentOut": moment_acc, }, - attrs={ - "epsilon": self._epsilon, - "decay": self._decay - }, - stop_gradient=True) + attrs={"epsilon": self._epsilon, "decay": self._decay}, + stop_gradient=True, + ) return decayed_adagrad_op @@ -3155,25 +3571,29 @@ class AdadeltaOptimizer(Optimizer): _avg_squared_grad_acc_str = "_avg_squared_grad" _avg_squared_update_acc_str = "_avg_squared_update" - def __init__(self, - learning_rate, - epsilon=1.0e-6, - rho=0.95, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate, + epsilon=1.0e-6, + rho=0.95, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + ): if learning_rate is None: raise ValueError("learning_rate is not set.") if epsilon is None: raise ValueError("epsilon is not set.") if rho is None: raise ValueError("rho is not set.") - super(AdadeltaOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + super(AdadeltaOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) self.type = "adadelta" self._epsilon = epsilon self._rho = rho @@ -3191,46 +3611,53 @@ def _append_optimize_op(self, block, param_and_grad): raise TypeError("block is not instance of framework.Block.") avg_squared_grad_acc = self._get_accumulator( - self._avg_squared_grad_acc_str, param_and_grad[0]) + self._avg_squared_grad_acc_str, param_and_grad[0] + ) avg_squared_update_acc = self._get_accumulator( - self._avg_squared_update_acc_str, param_and_grad[0]) + self._avg_squared_update_acc_str, param_and_grad[0] + ) if framework.in_dygraph_mode(): - _C_ops.adadelta_(param_and_grad[0], param_and_grad[1], - avg_squared_grad_acc, avg_squared_update_acc, - self._rho, self._epsilon) + _C_ops.adadelta_( + param_and_grad[0], + param_and_grad[1], + avg_squared_grad_acc, + avg_squared_update_acc, + self._rho, + self._epsilon, + ) elif framework._in_legacy_dygraph(): - _legacy_C_ops.adadelta(param_and_grad[0], param_and_grad[1], - avg_squared_grad_acc, avg_squared_update_acc, - param_and_grad[0], avg_squared_grad_acc, - avg_squared_update_acc, "epsilon", - self._epsilon, "rho", self._rho) + _legacy_C_ops.adadelta( + param_and_grad[0], + param_and_grad[1], + avg_squared_grad_acc, + avg_squared_update_acc, + param_and_grad[0], + avg_squared_grad_acc, + avg_squared_update_acc, + "epsilon", + self._epsilon, + "rho", + self._rho, + ) else: # Create the adadelta optimizer op - adadelta_op = block.append_op(type=self.type, - inputs={ - "Param": - param_and_grad[0], - "Grad": - param_and_grad[1], - "AvgSquaredGrad": - avg_squared_grad_acc, - "AvgSquaredUpdate": - avg_squared_update_acc - }, - outputs={ - "ParamOut": - param_and_grad[0], - "AvgSquaredGradOut": - avg_squared_grad_acc, - "AvgSquaredUpdateOut": - avg_squared_update_acc - }, - attrs={ - "epsilon": self._epsilon, - "rho": self._rho - }, - stop_gradient=True) + adadelta_op = block.append_op( + type=self.type, + inputs={ + "Param": param_and_grad[0], + "Grad": param_and_grad[1], + "AvgSquaredGrad": avg_squared_grad_acc, + "AvgSquaredUpdate": avg_squared_update_acc, + }, + outputs={ + "ParamOut": param_and_grad[0], + "AvgSquaredGradOut": avg_squared_grad_acc, + "AvgSquaredUpdateOut": avg_squared_update_acc, + }, + attrs={"epsilon": self._epsilon, "rho": self._rho}, + stop_gradient=True, + ) return adadelta_op @@ -3346,21 +3773,25 @@ class RMSPropOptimizer(Optimizer): _mean_square_acc_str = "mean_square" _mean_grad_acc_str = "mean_grad" - def __init__(self, - learning_rate, - rho=0.95, - epsilon=1.0e-6, - momentum=0.0, - centered=False, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None): - super(RMSPropOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + def __init__( + self, + learning_rate, + rho=0.95, + epsilon=1.0e-6, + momentum=0.0, + centered=False, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + ): + super(RMSPropOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) if learning_rate is None: raise ValueError("learning_rate is not set.") if rho is None: @@ -3389,27 +3820,49 @@ def _append_optimize_op(self, block, param_and_grad): if not isinstance(block, framework.Block): raise TypeError("block is not instance of framework.Block.") - momentum_acc = self._get_accumulator(self._momentum_acc_str, - param_and_grad[0]) - mean_square_acc = self._get_accumulator(self._mean_square_acc_str, - param_and_grad[0]) - mean_grad_acc = self._get_accumulator(self._mean_grad_acc_str, - param_and_grad[0]) + momentum_acc = self._get_accumulator( + self._momentum_acc_str, param_and_grad[0] + ) + mean_square_acc = self._get_accumulator( + self._mean_square_acc_str, param_and_grad[0] + ) + mean_grad_acc = self._get_accumulator( + self._mean_grad_acc_str, param_and_grad[0] + ) if in_dygraph_mode(): - _C_ops.rmsprop_(param_and_grad[0], mean_square_acc, - param_and_grad[1], momentum_acc, - self._create_param_lr(param_and_grad), - mean_grad_acc, self._epsilon, self._rho, - self._momentum, self._centered) + _C_ops.rmsprop_( + param_and_grad[0], + mean_square_acc, + param_and_grad[1], + momentum_acc, + self._create_param_lr(param_and_grad), + mean_grad_acc, + self._epsilon, + self._rho, + self._momentum, + self._centered, + ) return None elif _in_legacy_dygraph(): - _legacy_C_ops.rmsprop(param_and_grad[0], mean_square_acc, - self._create_param_lr(param_and_grad), - param_and_grad[1], momentum_acc, - param_and_grad[0], momentum_acc, - mean_square_acc, mean_grad_acc, "epsilon", - self._epsilon, "decay", self._rho, "momentum", - self._momentum, "centered", self._centered) + _legacy_C_ops.rmsprop( + param_and_grad[0], + mean_square_acc, + self._create_param_lr(param_and_grad), + param_and_grad[1], + momentum_acc, + param_and_grad[0], + momentum_acc, + mean_square_acc, + mean_grad_acc, + "epsilon", + self._epsilon, + "decay", + self._rho, + "momentum", + self._momentum, + "centered", + self._centered, + ) return None else: rmsprop_op = block.append_op( @@ -3426,15 +3879,16 @@ def _append_optimize_op(self, block, param_and_grad): "ParamOut": param_and_grad[0], "MomentOut": momentum_acc, "MeanSquareOut": mean_square_acc, - "MeanGradOut": mean_grad_acc + "MeanGradOut": mean_grad_acc, }, attrs={ "epsilon": self._epsilon, "decay": self._rho, "momentum": self._momentum, - "centered": self._centered + "centered": self._centered, }, - stop_gradient=True) + stop_gradient=True, + ) return rmsprop_op @@ -3537,20 +3991,24 @@ class FtrlOptimizer(Optimizer): _squared_acc_str = "squared" _linear_acc_str = "linear" - def __init__(self, - learning_rate, - l1=0.0, - l2=0.0, - lr_power=-0.5, - parameter_list=None, - regularization=None, - grad_clip=None, - name=None): - super(FtrlOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - name=name) + def __init__( + self, + learning_rate, + l1=0.0, + l2=0.0, + lr_power=-0.5, + parameter_list=None, + regularization=None, + grad_clip=None, + name=None, + ): + super(FtrlOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + name=name, + ) if learning_rate is None: raise ValueError("learning_rate is not set.") @@ -3571,43 +4029,52 @@ def _append_optimize_op(self, block, param_and_grad): if not isinstance(block, framework.Block): raise TypeError("block is not instance of framework.Block.") - squared_acc = self._get_accumulator(self._squared_acc_str, - param_and_grad[0]) - linear_acc = self._get_accumulator(self._linear_acc_str, - param_and_grad[0]) + squared_acc = self._get_accumulator( + self._squared_acc_str, param_and_grad[0] + ) + linear_acc = self._get_accumulator( + self._linear_acc_str, param_and_grad[0] + ) if framework._non_static_mode(): - _legacy_C_ops.ftrl(param_and_grad[0], squared_acc, linear_acc, - param_and_grad[1], - self._create_param_lr(param_and_grad), - param_and_grad[0], squared_acc, linear_acc, "l1", - self._l1, "l2", self._l2, "lr_power", - self._lr_power) + _legacy_C_ops.ftrl( + param_and_grad[0], + squared_acc, + linear_acc, + param_and_grad[1], + self._create_param_lr(param_and_grad), + param_and_grad[0], + squared_acc, + linear_acc, + "l1", + self._l1, + "l2", + self._l2, + "lr_power", + self._lr_power, + ) else: - ftrl_op = block.append_op(type=self.type, - inputs={ - "Param": - param_and_grad[0], - "Grad": - param_and_grad[1], - "SquaredAccumulator": - squared_acc, - "LinearAccumulator": - linear_acc, - "LearningRate": - self._create_param_lr(param_and_grad), - }, - outputs={ - "ParamOut": param_and_grad[0], - "SquaredAccumOut": squared_acc, - "LinearAccumOut": linear_acc - }, - attrs={ - "l1": self._l1, - "l2": self._l2, - "lr_power": self._lr_power - }, - stop_gradient=True) + ftrl_op = block.append_op( + type=self.type, + inputs={ + "Param": param_and_grad[0], + "Grad": param_and_grad[1], + "SquaredAccumulator": squared_acc, + "LinearAccumulator": linear_acc, + "LearningRate": self._create_param_lr(param_and_grad), + }, + outputs={ + "ParamOut": param_and_grad[0], + "SquaredAccumOut": squared_acc, + "LinearAccumOut": linear_acc, + }, + attrs={ + "l1": self._l1, + "l2": self._l2, + "lr_power": self._lr_power, + }, + stop_gradient=True, + ) return ftrl_op @@ -3690,30 +4157,34 @@ def exclude_fn(param): _beta1_pow_acc_str = "beta1_pow_acc" _beta2_pow_acc_str = "beta2_pow_acc" - def __init__(self, - learning_rate=0.001, - lamb_weight_decay=0.01, - beta1=0.9, - beta2=0.999, - epsilon=1e-6, - parameter_list=None, - regularization=None, - grad_clip=None, - exclude_from_weight_decay_fn=None, - name=None): + def __init__( + self, + learning_rate=0.001, + lamb_weight_decay=0.01, + beta1=0.9, + beta2=0.999, + epsilon=1e-6, + parameter_list=None, + regularization=None, + grad_clip=None, + exclude_from_weight_decay_fn=None, + name=None, + ): assert learning_rate is not None assert lamb_weight_decay is not None assert beta1 is not None assert beta2 is not None assert epsilon is not None - super(LambOptimizer, self).__init__(learning_rate=learning_rate, - parameter_list=parameter_list, - regularization=regularization, - grad_clip=grad_clip, - beta1=beta1, - beta2=beta2, - epsilon=epsilon, - name=name) + super(LambOptimizer, self).__init__( + learning_rate=learning_rate, + parameter_list=parameter_list, + regularization=regularization, + grad_clip=grad_clip, + beta1=beta1, + beta2=beta2, + epsilon=epsilon, + name=name, + ) self.type = "lamb" self._weight_decay = lamb_weight_decay self._exclude_from_weight_decay_fn = exclude_from_weight_decay_fn @@ -3722,57 +4193,82 @@ def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) block.program._use_lamb = True - moment1 = self._get_accumulator(self._moment1_acc_str, - param_and_grad[0]) - moment2 = self._get_accumulator(self._moment2_acc_str, - param_and_grad[0]) - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) - beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str, - param_and_grad[0]) - - if self._exclude_from_weight_decay_fn is not None \ - and self._exclude_from_weight_decay_fn(param_and_grad[0]): + moment1 = self._get_accumulator( + self._moment1_acc_str, param_and_grad[0] + ) + moment2 = self._get_accumulator( + self._moment2_acc_str, param_and_grad[0] + ) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) + beta2_pow_acc = self._get_accumulator( + self._beta2_pow_acc_str, param_and_grad[0] + ) + + if ( + self._exclude_from_weight_decay_fn is not None + and self._exclude_from_weight_decay_fn(param_and_grad[0]) + ): weight_decay = 0.0 else: weight_decay = self._weight_decay lr = self._create_param_lr(param_and_grad) master_weight = None if framework._non_static_mode(): - _legacy_C_ops.lamb(param_and_grad[0], param_and_grad[1], lr, - moment1, moment2, beta1_pow_acc, beta2_pow_acc, - master_weight, param_and_grad[0], moment1, - moment2, beta1_pow_acc, beta2_pow_acc, - master_weight, 'beta1', self._beta1, 'beta2', - self._beta2, 'epsilon', self._epsilon, - 'weight_decay', weight_decay) + _legacy_C_ops.lamb( + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + param_and_grad[0], + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + 'beta1', + self._beta1, + 'beta2', + self._beta2, + 'epsilon', + self._epsilon, + 'weight_decay', + weight_decay, + ) return None # create the lamb optimize op - lamb_op = block.append_op(type=self.type, - inputs={ - "Param": param_and_grad[0], - "Grad": param_and_grad[1], - "LearningRate": lr, - "Moment1": moment1, - "Moment2": moment2, - "Beta1Pow": beta1_pow_acc, - "Beta2Pow": beta2_pow_acc - }, - outputs={ - "ParamOut": param_and_grad[0], - "Moment1Out": moment1, - "Moment2Out": moment2, - "Beta1PowOut": beta1_pow_acc, - "Beta2PowOut": beta2_pow_acc - }, - attrs={ - "beta1": self._beta1, - "beta2": self._beta2, - "epsilon": self._epsilon, - "weight_decay": weight_decay - }, - stop_gradient=True) + lamb_op = block.append_op( + type=self.type, + inputs={ + "Param": param_and_grad[0], + "Grad": param_and_grad[1], + "LearningRate": lr, + "Moment1": moment1, + "Moment2": moment2, + "Beta1Pow": beta1_pow_acc, + "Beta2Pow": beta2_pow_acc, + }, + outputs={ + "ParamOut": param_and_grad[0], + "Moment1Out": moment1, + "Moment2Out": moment2, + "Beta1PowOut": beta1_pow_acc, + "Beta2PowOut": beta2_pow_acc, + }, + attrs={ + "beta1": self._beta1, + "beta2": self._beta2, + "epsilon": self._epsilon, + "weight_decay": weight_decay, + }, + stop_gradient=True, + ) return lamb_op @@ -3886,38 +4382,44 @@ class ModelAverage(Optimizer): fetch_list=[loss.name]) """ - def __init__(self, - average_window_rate, - min_average_window=10000, - max_average_window=10000, - regularization=None, - name=None): + def __init__( + self, + average_window_rate, + min_average_window=10000, + max_average_window=10000, + regularization=None, + name=None, + ): if framework._non_static_mode(): raise Exception("In dygraph, don't support ModelAverage.") - super(ModelAverage, self).__init__(0.0, - regularization=regularization, - name=name) + super(ModelAverage, self).__init__( + 0.0, regularization=regularization, name=name + ) self.average_window = average_window_rate self.min_average_window = min_average_window self.max_average_window = max_average_window self.params_grads = [] - for param in framework.default_main_program().global_block( - ).all_parameters(): + for param in ( + framework.default_main_program().global_block().all_parameters() + ): if param.do_model_average != False: grad = param.block.create_var( - name=unique_name.generate_with_ignorable_key(".".join( - [param.name, 'tmp'])), + name=unique_name.generate_with_ignorable_key( + ".".join([param.name, 'tmp']) + ), dtype=param.dtype, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) self.params_grads.append((param, grad)) for param, grad in self.params_grads: if grad is None: continue with param.block.program._optimized_guard( - [param, grad]), name_scope('move_average'): + [param, grad] + ), name_scope('move_average'): self._append_average_accumulate_op(param) self.apply_program = Program() @@ -3939,20 +4441,25 @@ def _add_average_apply_op(self, block, param_grad): sum_2 = block._clone_variable(self._get_accumulator('sum_2', param)) sum_3 = block._clone_variable(self._get_accumulator('sum_3', param)) num_accumulates = block._clone_variable( - self._get_accumulator('num_accumulates', param)) + self._get_accumulator('num_accumulates', param) + ) old_num_accumulates = block._clone_variable( - self._get_accumulator('old_num_accumulates', param)) + self._get_accumulator('old_num_accumulates', param) + ) num_updates = block._clone_variable( - self._get_accumulator('num_updates', param)) + self._get_accumulator('num_updates', param) + ) # backup param value to grad layers.assign(input=param, output=grad) # param = (sum_1 + sum_2 + sum_3) / (num_accumulates + old_num_accumulates) tmp = layers.sum(x=[num_accumulates, old_num_accumulates]) sum = layers.sum(x=[sum_1, sum_2, sum_3]) tmp = layers.cast( - x=tmp, dtype='float32' if self._dtype == None else self._dtype) + x=tmp, dtype='float32' if self._dtype == None else self._dtype + ) sum = layers.cast( - x=sum, dtype='float32' if self._dtype == None else self._dtype) + x=sum, dtype='float32' if self._dtype == None else self._dtype + ) ops._elementwise_div(x=sum, y=tmp, out=param) def _add_average_restore_op(self, block, param_grad): @@ -3965,44 +4472,42 @@ def _append_average_accumulate_op(self, param): sum_1 = self._add_accumulator('sum_1', param) sum_2 = self._add_accumulator('sum_2', param) sum_3 = self._add_accumulator('sum_3', param) - num_accumulates = self._add_accumulator('num_accumulates', - param, - dtype='int64', - shape=[1]) - old_num_accumulates = self._add_accumulator('old_num_accumulates', - param, - dtype='int64', - shape=[1]) - num_updates = self._add_accumulator('num_updates', - param, - dtype='int64', - shape=[1]) - - self.helper.append_op(type='average_accumulates', - inputs={ - "param": param, - "in_sum_1": sum_1, - "in_sum_2": sum_2, - "in_sum_3": sum_3, - "in_num_accumulates": num_accumulates, - "in_old_num_accumulates": old_num_accumulates, - "in_num_updates": num_updates - }, - outputs={ - "out_sum_1": sum_1, - "out_sum_2": sum_2, - "out_sum_3": sum_3, - "out_num_accumulates": num_accumulates, - "out_old_num_accumulates": - old_num_accumulates, - "out_num_updates": num_updates, - }, - attrs={ - "average_window": self.average_window, - "min_average_window": self.min_average_window, - "max_average_window": self.max_average_window, - }, - stop_gradient=True) + num_accumulates = self._add_accumulator( + 'num_accumulates', param, dtype='int64', shape=[1] + ) + old_num_accumulates = self._add_accumulator( + 'old_num_accumulates', param, dtype='int64', shape=[1] + ) + num_updates = self._add_accumulator( + 'num_updates', param, dtype='int64', shape=[1] + ) + + self.helper.append_op( + type='average_accumulates', + inputs={ + "param": param, + "in_sum_1": sum_1, + "in_sum_2": sum_2, + "in_sum_3": sum_3, + "in_num_accumulates": num_accumulates, + "in_old_num_accumulates": old_num_accumulates, + "in_num_updates": num_updates, + }, + outputs={ + "out_sum_1": sum_1, + "out_sum_2": sum_2, + "out_sum_3": sum_3, + "out_num_accumulates": num_accumulates, + "out_old_num_accumulates": old_num_accumulates, + "out_num_updates": num_updates, + }, + attrs={ + "average_window": self.average_window, + "min_average_window": self.min_average_window, + "max_average_window": self.max_average_window, + }, + stop_gradient=True, + ) @signature_safe_contextmanager def apply(self, executor, need_restore=True): @@ -4117,7 +4622,7 @@ def restore(self, executor): class ExponentialMovingAverage(object): r""" - :api_attr: Static Graph + :api_attr: Static Graph Compute the moving average of parameters with exponential decay. Given a parameter :math:`\\theta`, its exponential moving average (EMA) @@ -4127,7 +4632,7 @@ class ExponentialMovingAverage(object): \\text{EMA}_0 & = 0 - \\text{EMA}_t & = \\text{decay} * \\text{EMA}_{t-1} + (1 - \\text{decay}) * \\theta_t + \\text{EMA}_t & = \\text{decay} * \\text{EMA}_{t-1} + (1 - \\text{decay}) * \\theta_t The average results calculated by **update()** method will be saved in temporary variables which are created and maintained by the object, and can @@ -4215,7 +4720,8 @@ class ExponentialMovingAverage(object): def __init__(self, decay=0.999, thres_steps=None, name=None): if framework._non_static_mode(): raise Exception( - "In dygraph, don't support ExponentialMovingAverage.") + "In dygraph, don't support ExponentialMovingAverage." + ) self._decay = decay self._thres_steps = thres_steps self._name = name if name is not None else '' @@ -4225,17 +4731,21 @@ def __init__(self, decay=0.999, thres_steps=None, name=None): self._params_tmps = [] for param in default_main_program().global_block().all_parameters(): if param.do_model_average != False: - tmp = param.block.create_var(name=unique_name.generate(".".join( - [self._name + param.name, 'ema_tmp'])), - dtype=param.dtype, - persistable=False, - stop_gradient=True) + tmp = param.block.create_var( + name=unique_name.generate( + ".".join([self._name + param.name, 'ema_tmp']) + ), + dtype=param.dtype, + persistable=False, + stop_gradient=True, + ) self._params_tmps.append((param, tmp)) self._ema_vars = {} for param, tmp in self._params_tmps: - with param.block.program._optimized_guard( - [param, tmp]), name_scope('moving_average'): + with param.block.program._optimized_guard([param, tmp]), name_scope( + 'moving_average' + ): self._ema_vars[param.name] = self._create_ema_vars(param) self.apply_program = Program() @@ -4250,8 +4760,9 @@ def __init__(self, decay=0.999, thres_steps=None, name=None): # bias correction with layers.control_flow.Switch() as switch: with switch.case(global_step > 0): - layers.assign(output=param, - input=ema / (1.0 - decay_pow)) + layers.assign( + output=param, input=ema / (1.0 - decay_pow) + ) with switch.default(): layers.assign(output=param, input=ema) @@ -4270,7 +4781,8 @@ def _get_ema_decay(self): value=self._decay, dtype='float32', persistable=True, - name="scheduled_ema_decay_rate") + name="scheduled_ema_decay_rate", + ) if self._thres_steps is not None: decay_t = (self._thres_steps + 1.0) / (self._thres_steps + 10.0) @@ -4279,16 +4791,18 @@ def _get_ema_decay(self): layers.tensor.assign(decay_t, decay_var) with switch.default(): layers.tensor.assign( - np.array([self._decay], dtype=np.float32), - decay_var) + np.array([self._decay], dtype=np.float32), decay_var + ) return decay_var def _get_decay_pow(self, block): - global_step = layers.create_global_var(name=self._step_counter_name, - shape=[1], - value=0, - dtype='int64', - persistable=True) + global_step = layers.create_global_var( + name=self._step_counter_name, + shape=[1], + value=0, + dtype='int64', + persistable=True, + ) global_step = layers.cast(global_step, "float32") decay_var = block._clone_variable(self._decay_var) decay_pow_acc = layers.elementwise_pow(decay_var, global_step) @@ -4300,7 +4814,8 @@ def _create_ema_vars(self, param): shape=param.shape, value=0.0, dtype=param.dtype, - persistable=True) + persistable=True, + ) return param_ema @@ -4310,18 +4825,21 @@ def update(self): train program. """ global_step = layers.autoincreased_step_counter( - counter_name=self._step_counter_name) + counter_name=self._step_counter_name + ) param_master_emas = [] for param, tmp in self._params_tmps: - with param.block.program._optimized_guard( - [param, tmp]), name_scope('moving_average'): + with param.block.program._optimized_guard([param, tmp]), name_scope( + 'moving_average' + ): param_ema = self._ema_vars[param.name] if param.name + '.master' in self._ema_vars: master_ema = self._ema_vars[param.name + '.master'] param_master_emas.append([param_ema, master_ema]) else: ema_t = param_ema * self._decay_var + param * ( - 1 - self._decay_var) + 1 - self._decay_var + ) layers.assign(input=ema_t, output=param_ema) # for fp16 params @@ -4332,8 +4850,9 @@ def update(self): outputs={"Out": param_ema}, attrs={ "in_dtype": master_ema.dtype, - "out_dtype": param_ema.dtype - }) + "out_dtype": param_ema.dtype, + }, + ) @signature_safe_contextmanager def apply(self, executor, need_restore=True): @@ -4363,7 +4882,7 @@ def restore(self, executor): class PipelineOptimizer(object): """ - :api_attr: Static Graph + :api_attr: Static Graph Pipeline Optimizer: Make a program to run as pipeline, that is splitting a program into multiple sections (sub-programs) and each section run on a @@ -4426,14 +4945,19 @@ def __init__(self, optimizer, num_microbatches=1, start_cpu_core_id=0): self._device = "gpu" if framework._non_static_mode(): raise Exception("In dygraph, don't support PipelineOptimizer.") - valid_optimizers = (Optimizer, paddle.optimizer.Optimizer, - paddle.fluid.contrib.mixed_precision.decorator. - OptimizerWithMixedPrecision) + valid_optimizers = ( + Optimizer, + paddle.optimizer.Optimizer, + paddle.fluid.contrib.mixed_precision.decorator.OptimizerWithMixedPrecision, + ) if not isinstance(optimizer, valid_optimizers): - raise ValueError("The 'optimizer' parameter for " - "PipelineOptimizer must be an instance of " - "{}, but the given type is {}.".format( - valid_optimizers, type(optimizer))) + raise ValueError( + "The 'optimizer' parameter for " + "PipelineOptimizer must be an instance of " + "{}, but the given type is {}.".format( + valid_optimizers, type(optimizer) + ) + ) self._optimizer = optimizer # Get the original optimizer defined by users, such as SGD @@ -4441,11 +4965,13 @@ def __init__(self, optimizer, num_microbatches=1, start_cpu_core_id=0): while hasattr(self._origin_optimizer, "inner_opt"): self._origin_optimizer = self._origin_optimizer.inner_opt - assert num_microbatches >= 1, ( - "num_microbatches must be a positive value.") + assert ( + num_microbatches >= 1 + ), "num_microbatches must be a positive value." self._num_microbatches = num_microbatches - assert start_cpu_core_id >= 0, ( - "start_cpu_core_id must be a non-negative integer.") + assert ( + start_cpu_core_id >= 0 + ), "start_cpu_core_id must be a non-negative integer." self._start_cpu_core_id = start_cpu_core_id self._place_list = None op_maker = core.op_proto_and_checker_maker @@ -4473,41 +4999,47 @@ def _insert_allreduce_op(self, op_idx, block): if op.type == "reduce_any": # cast the bool var to int32 to use allreduce_max op temp_var_name = unique_name.generate(out_name + "_cast_int32") - temp_var = block.create_var(name=temp_var_name, - shape=[1], - dtype="int32") - block._insert_op(op_idx + 1 + offset, - type='cast', - inputs={'X': out_var}, - outputs={'Out': temp_var}, - attrs={ - 'in_dtype': out_var.dtype, - 'out_dtype': temp_var.dtype, - self._op_role_key: self._op_role.Optimize - }) + temp_var = block.create_var( + name=temp_var_name, shape=[1], dtype="int32" + ) + block._insert_op( + op_idx + 1 + offset, + type='cast', + inputs={'X': out_var}, + outputs={'Out': temp_var}, + attrs={ + 'in_dtype': out_var.dtype, + 'out_dtype': temp_var.dtype, + self._op_role_key: self._op_role.Optimize, + }, + ) offset += 1 block._insert_op( op_idx + 1 + offset, type='c_allreduce_max' - if op.type == "reduce_any" else 'c_allreduce_sum', + if op.type == "reduce_any" + else 'c_allreduce_sum', inputs={'X': temp_var if op.type == "reduce_any" else out_var}, outputs={'Out': temp_var if op.type == "reduce_any" else out_var}, attrs={ 'ring_id': self.global_ring_id, self._op_role_key: self._op_role.Optimize, - 'use_calc_stream': True - }) + 'use_calc_stream': True, + }, + ) offset += 1 if op.type == "reduce_any": - block._insert_op(op_idx + 1 + offset, - type='cast', - inputs={'X': temp_var}, - outputs={'Out': out_var}, - attrs={ - 'in_dtype': temp_var.dtype, - 'out_dtype': out_var.dtype, - self._op_role_key: self._op_role.Optimize - }) + block._insert_op( + op_idx + 1 + offset, + type='cast', + inputs={'X': temp_var}, + outputs={'Out': out_var}, + attrs={ + 'in_dtype': temp_var.dtype, + 'out_dtype': out_var.dtype, + self._op_role_key: self._op_role.Optimize, + }, + ) offset += 1 return offset @@ -4563,13 +5095,15 @@ def _create_vars(self, block, ori_block): if var in used_var_set or "_blocking_queue" in var: continue used_var_set.add(var) - if block._find_var_recursive(str(var)): continue + if block._find_var_recursive(str(var)): + continue source_var = ori_block._var_recursive(str(var)) if source_var.type == core.VarDesc.VarType.READER: dest_var = block.create_var( name=var, type=core.VarDesc.VarType.READER, - persistable=source_var.persistable) + persistable=source_var.persistable, + ) elif isinstance(source_var, Parameter): dest_var = block.create_parameter( name=source_var.name, @@ -4581,14 +5115,16 @@ def _create_vars(self, block, ori_block): trainable=source_var.trainable, optimize_attr=source_var.optimize_attr, regularizer=source_var.regularizer, - error_clip=source_var.error_clip) + error_clip=source_var.error_clip, + ) else: dest_var = block._clone_variable(source_var, False) self._clone_var_attr(dest_var, source_var) # When use with sharding, allreduce_sum and allreduce_max # used for global gradient clip and amp will be added by sharding. op_idx += 1 - if self.use_sharding or not should_insert: continue + if self.use_sharding or not should_insert: + continue inserted_ops = self._insert_allreduce_op(op_idx - 1, block) added_op_num += inserted_ops op_idx += inserted_ops @@ -4598,15 +5134,18 @@ def _is_loss_grad_op(self, op): assert self._op_role_key in op.attr_names op_role = int(op.attr(self._op_role_key)) return op_role & int(self._op_role.Backward) and op_role & int( - self._op_role.Loss) + self._op_role.Loss + ) def _is_forward_op(self, op): - return self._op_role_key in op.attr_names and (int( - op.attr(self._op_role_key)) == int(self._op_role.Forward)) + return self._op_role_key in op.attr_names and ( + int(op.attr(self._op_role_key)) == int(self._op_role.Forward) + ) def _is_backward_op(self, op): return self._op_role_key in op.attr_names and ( - int(op.attr(self._op_role_key)) & int(self._op_role.Backward)) + int(op.attr(self._op_role_key)) & int(self._op_role.Backward) + ) def _is_loss_op(self, op): assert self._op_role_key in op.attr_names @@ -4614,11 +5153,15 @@ def _is_loss_op(self, op): def _is_optimize_op(self, op): return self._op_role_key in op.attr_names and ( - int(op.attr(self._op_role_key)) & int(self._op_role.Optimize)) + int(op.attr(self._op_role_key)) & int(self._op_role.Optimize) + ) def _is_update_op(self, op): - return 'Param' in op.input_names and 'Grad' in op.input_names and ( - "LearningRate" in op.input_names) + return ( + 'Param' in op.input_names + and 'Grad' in op.input_names + and ("LearningRate" in op.input_names) + ) def _split_program(self, main_program, devices): """ @@ -4665,10 +5208,11 @@ def _get_op_device_for_startup_program(self, var_name): get the real op_device attribute of the fill_constant as the device where the corresponding parameters on. """ - assert "beta1_pow_acc" in var_name or "beta2_pow_acc" in var_name, \ - 'For accumulators for Adam, the name must contain beta1_pow_acc ' \ + assert "beta1_pow_acc" in var_name or "beta2_pow_acc" in var_name, ( + 'For accumulators for Adam, the name must contain beta1_pow_acc ' 'or beta2_pow_acc.' - param_name = var_name[0:var_name.index('_beta')] + ) + param_name = var_name[0 : var_name.index('_beta')] device = self._param_device_map[param_name] return device @@ -4680,7 +5224,8 @@ def _split_startup_program(self, startup_program, device_id): if device == "cpu": assert op.type == "fill_constant", ( "For ops in startup program with the op_device attribute " - "of cpu, they must be of type fill_constant.") + "of cpu, they must be of type fill_constant." + ) output_var = op.output_arg_names[0] device = self._get_op_device_for_startup_program(output_var) @@ -4689,7 +5234,8 @@ def _split_startup_program(self, startup_program, device_id): else: # LR related ops device = None - if device and device_index != device_id: continue + if device and device_index != device_id: + continue op_desc = op.desc ap_op = new_startup_program.global_block().desc.append_op() ap_op.copy_from(op_desc) @@ -4709,7 +5255,8 @@ def _find_post_op(self, index, var_name): var_name = var_name.replace('.cast_fp16', '') post_ops = self.input_var_to_op[var_name] - if post_ops == None: return None + if post_ops == None: + return None result_op = None for post_op, post_idx in reversed(post_ops): if post_idx > index: @@ -4723,7 +5270,8 @@ def _find_prev_op(self, index, var_name): variable named var_name. """ prev_ops = self.output_var_to_op[var_name] - if prev_ops == None: return None + if prev_ops == None: + return None result_op = None for prev_op, prev_idx in reversed(prev_ops): if prev_idx < index: @@ -4749,7 +5297,8 @@ def _create_var(self, block, ref_var, name, dtype=None): lod_level=ref_var.lod_level, persistable=ref_var.persistable, is_data=ref_var.is_data, - need_check_feed=ref_var.desc.need_check_feed()) + need_check_feed=ref_var.desc.need_check_feed(), + ) self._clone_var_attr(new_var, ref_var) return new_var @@ -4775,11 +5324,16 @@ def _get_op_device_attr(self, op): """ Get the op_device attribute of a op. """ - device = op.attr(self._op_device_key) \ - if op.has_attr(self._op_device_key) else None + device = ( + op.attr(self._op_device_key) + if op.has_attr(self._op_device_key) + else None + ) if device: - assert device[0:3] == 'gpu' or device[0:3] == 'npu', "Now, only gpu and npu devices are " \ + assert device[0:3] == 'gpu' or device[0:3] == 'npu', ( + "Now, only gpu and npu devices are " "supported in pipeline parallemism." + ) return device def _add_op_device_attr_for_op(self, op, idx, block): @@ -4798,40 +5352,48 @@ def _add_op_device_attr_for_op(self, op, idx, block): elif op.type == "sum" and self._is_backward_op(op): # For sum ops that compute the sum of @RENAMED@ vars for name in op.desc.input_arg_names(): - assert '@RENAME@' in name, \ - "The op must be sum used to accumulate renamed vars." + assert ( + '@RENAME@' in name + ), "The op must be sum used to accumulate renamed vars." assert len(op.desc.output_arg_names()) == 1 out_name = op.desc.output_arg_names()[0] post_op = self._find_post_op(idx, out_name) assert post_op.has_attr( - 'op_device'), "{} has no op_device attr for var {}".format( - post_op.type, out_name) + 'op_device' + ), "{} has no op_device attr for var {}".format( + post_op.type, out_name + ) device = post_op.attr(self._op_device_key) assert device, "The post op must have op_device set." op._set_attr(self._op_device_key, device) - elif (op.type == "cast" - or op.type == "scale") and self._is_backward_op(op): + elif (op.type == "cast" or op.type == "scale") and self._is_backward_op( + op + ): prev_op = self._find_prev_op(idx, op.desc.input("X")[0]) op._set_attr(self._op_device_key, prev_op.attr(self._op_device_key)) elif op.type == "memcpy" and not self._is_optimize_op(op): # for checkpoint offloading - assert len(op.input_arg_names) == 1 and len( - op.output_arg_names) == 1 + assert ( + len(op.input_arg_names) == 1 and len(op.output_arg_names) == 1 + ) input_name = op.input_arg_names[0] output_name = op.output_arg_names[0] if '@Fetch' in output_name: post_op = self._find_post_op(idx, output_name) - op._set_attr(self._op_device_key, - post_op.attr(self._op_device_key)) + op._set_attr( + self._op_device_key, post_op.attr(self._op_device_key) + ) else: prev_op = self._find_prev_op(idx, op.desc.input("X")[0]) - op._set_attr(self._op_device_key, - prev_op.attr(self._op_device_key)) + op._set_attr( + self._op_device_key, prev_op.attr(self._op_device_key) + ) elif self._is_loss_op(op): # For loss * loss_scaling op added by AMP offset = 1 - while (not block.ops[idx + offset].has_attr(self._op_device_key) - or not block.ops[idx + offset].attr(self._op_device_key)): + while not block.ops[idx + offset].has_attr( + self._op_device_key + ) or not block.ops[idx + offset].attr(self._op_device_key): offset += 1 device = block.ops[idx + offset].attr(self._op_device_key) assert device, "Please put you program within device_guard scope." @@ -4847,19 +5409,26 @@ def _add_op_device_attr_for_op(self, op, idx, block): elif self._is_gradient_clip_op(op) or self._is_regularization_op(op): # For gradient clip and regularization ops, we set their op_device # attribute to the device where their corresponding parameters on. - assert self._op_role_var_key in op.attr_names, "gradient_clip " \ + assert self._op_role_var_key in op.attr_names, ( + "gradient_clip " "and regularization ops must have op_role_var attribute." + ) op_role_var = op.attr(self._op_role_var_key) - assert len(op_role_var) == 2, "op_role_var for gradient_clip " \ + assert len(op_role_var) == 2, ( + "op_role_var for gradient_clip " "regularization ops must have two elements." + ) param_name = op_role_var[0] device = self._param_device_map[param_name] # For sum op added by global gradient clip, it must be # put on all devices - if (op.type == 'sum' or op.type == 'sqrt' - or op.type == 'fill_constant' - or op.type == 'elementwise_max' - or op.type == 'elementwise_div'): + if ( + op.type == 'sum' + or op.type == 'sqrt' + or op.type == 'fill_constant' + or op.type == 'elementwise_max' + or op.type == 'elementwise_div' + ): device = f"{self._device}:all" op._set_attr(self._op_device_key, device) elif op.type == "alloc_float_status" or op.type == "clear_float_status": @@ -4876,12 +5445,18 @@ def _add_op_device_attr_for_op(self, op, idx, block): float_status_var.persistable = True else: other_known_ops = [ - 'update_loss_scaling', 'reduce_any', 'concat', 'sum', - 'check_finite_and_unscale', 'memcpy' + 'update_loss_scaling', + 'reduce_any', + 'concat', + 'sum', + 'check_finite_and_unscale', + 'memcpy', ] - assert op.type in other_known_ops, "For other ops without " \ - "op_device set, they must be one of {}, but it " \ + assert op.type in other_known_ops, ( + "For other ops without " + "op_device set, they must be one of {}, but it " "is {}".format(other_known_ops, op.type) + ) assert self._is_optimize_op(op) op._set_attr(self._op_device_key, f"{self._device}:all") @@ -4891,8 +5466,11 @@ def _add_op_device_attr(self, block): not that attribute set. """ for idx, op in enumerate(list(block.ops)): - if (op.type == "create_py_reader" or op.type == "read" - or op.type == "create_double_buffer_reader"): + if ( + op.type == "create_py_reader" + or op.type == "read" + or op.type == "create_double_buffer_reader" + ): # Copy read related ops to all section to make them exit # after each epoch. # We use "gpu:all" to represent the op should be put on all @@ -4901,7 +5479,8 @@ def _add_op_device_attr(self, block): op._set_attr(self._op_device_key, f"{self._device}:all") continue # op_device attribute has been set - if self._get_op_device_attr(op): continue + if self._get_op_device_attr(op): + continue self._add_op_device_attr_for_op(op, idx, block) def _check_validation(self, block): @@ -4922,33 +5501,42 @@ def _check_validation(self, block): ] for op in block.ops: if not op._has_kernel(op.type): - assert op.type == "conditional_block" and (op.attr( - self._op_role_key) == int(self._op_role.LRSched)), ( - "Now, the only supported op without kernel is " - "conditional_block, and its op role must be LRSched.") + assert op.type == "conditional_block" and ( + op.attr(self._op_role_key) == int(self._op_role.LRSched) + ), ( + "Now, the only supported op without kernel is " + "conditional_block, and its op role must be LRSched." + ) assert op.has_attr( - self._op_role_key), ("op ({}) has no {} attribute.".format( - op.type, self._op_role_key)) + self._op_role_key + ), "op ({}) has no {} attribute.".format(op.type, self._op_role_key) op_role = op.attr(self._op_role_key) - assert int(op_role) in valid_op_role_value, \ - "op_role {} for op {} must be one of {}".format( - op_role, - op.type, - valid_op_role_value) + assert ( + int(op_role) in valid_op_role_value + ), "op_role {} for op {} must be one of {}".format( + op_role, op.type, valid_op_role_value + ) assert op.has_attr( - self._op_device_key), ("op ({}) has no {} attribute.".format( - op.type, self._op_device_key)) + self._op_device_key + ), "op ({}) has no {} attribute.".format( + op.type, self._op_device_key + ) device = op.attr(self._op_device_key) - assert device, ("op_device attribute for op " - "{} has not been set.".format(op.type)) - if device == f"{self._device}:all": continue + assert ( + device + ), "op_device attribute for op " "{} has not been set.".format( + op.type + ) + if device == f"{self._device}:all": + continue dev_type = device.split(':')[0] assert dev_type == "gpu" or dev_type == 'npu', ( "Now only gpu and npu devices are supported " - "for pipeline parallelism.") + "for pipeline parallelism." + ) if device not in device_list: device_list.append(device) @@ -4971,16 +5559,18 @@ def _insert_sendrecv_ops_for_boundaries(self, block): break extra_index_info = { 'index': 0, - 'first_optimize_index': first_optimize_index + 'first_optimize_index': first_optimize_index, } for index, op in enumerate(list(block.ops)): cur_device = op.attr(self._op_device_key) - if cur_device == f"{self._device}:all": continue + if cur_device == f"{self._device}:all": + continue for var_name in op.input_arg_names: var = block.var(var_name) # skip data var - if var.is_data: continue + if var.is_data: + continue prev_device = None prev_op = self._find_prev_op(index, var_name) @@ -4990,13 +5580,15 @@ def _insert_sendrecv_ops_for_boundaries(self, block): prev_device = self._param_device_map[var_name] if not prev_device: - prev_device = prev_op.attr(self._op_device_key) \ - if prev_op else None + prev_device = ( + prev_op.attr(self._op_device_key) if prev_op else None + ) if prev_device is None or prev_device == f"{self._device}:all": continue - if prev_device == cur_device: continue + if prev_device == cur_device: + continue if var_name not in input_var_to_device: input_var_to_device[var_name] = [] @@ -5009,20 +5601,25 @@ def _check_stage(cur_id, prev_id): # check send/recv stage valid is_forward = self._is_forward_op(op) is_backward = self._is_backward_op(op) - assert is_forward or is_backward, \ - 'send/recv in pipeline should only be inserted in forward or backward,' \ + assert is_forward or is_backward, ( + 'send/recv in pipeline should only be inserted in forward or backward,' 'please check the op_role of op={}'.format(op) + ) if is_forward: - assert prev_id < cur_id, \ - "In forward, send/recv can only be passed forward, but now " \ + assert prev_id < cur_id, ( + "In forward, send/recv can only be passed forward, but now " "prev_stage={} great than cur_stage={}, please check op_device of op={}".format( - prev_id, cur_id, op) + prev_id, cur_id, op + ) + ) elif is_backward: - assert prev_id > cur_id, \ - "In backward, send/recv can only be passed backward, but now " \ + assert prev_id > cur_id, ( + "In backward, send/recv can only be passed backward, but now " "prev_stage={} less than cur_stage={}, please check op_device of op={}".format( - prev_id, cur_id, op) + prev_id, cur_id, op + ) + ) def _insert_send_recv(cur_id, prev_id): cur_dev = device_type + str(cur_id) @@ -5034,13 +5631,15 @@ def _insert_send_recv(cur_id, prev_id): _insert_send_recv(cur_id - 1, prev_id) _insert_send_recv(cur_id, cur_id - 1) input_var_to_device[var_name].append( - (cur_dev, prev_dev)) + (cur_dev, prev_dev) + ) return elif cur_id - prev_id < -1: _insert_send_recv(cur_id + 1, prev_id) _insert_send_recv(cur_id, cur_id + 1) input_var_to_device[var_name].append( - (cur_dev, prev_dev)) + (cur_dev, prev_dev) + ) return assert abs(cur_id - prev_id) == 1 @@ -5069,12 +5668,16 @@ def _insert_send_recv(cur_id, prev_id): self._op_role_key: op_role, 'use_calc_stream': True, 'peer': 1, - 'ring_id': ring_id - }) + 'ring_id': ring_id, + }, + ) extra_index_info['index'] += 1 var_shape = list(var.shape) - var_shape[0] = self.micro_batch_size if var_shape[ - 0] < 0 else var_shape[0] + var_shape[0] = ( + self.micro_batch_size + if var_shape[0] < 0 + else var_shape[0] + ) block._insert_op_without_sync( index=index + extra_index_info['index'], type='recv_v2', @@ -5086,17 +5689,22 @@ def _insert_send_recv(cur_id, prev_id): self._op_role_key: op_role, 'use_calc_stream': True, 'peer': 0, - 'ring_id': ring_id - }) + 'ring_id': ring_id, + }, + ) extra_index_info['index'] += 1 elif self.schedule_mode == '1F1B': # 1F1B var_shape = list(var.shape) - var_shape[0] = self.micro_batch_size if var_shape[ - 0] < 0 else var_shape[0] + var_shape[0] = ( + self.micro_batch_size + if var_shape[0] < 0 + else var_shape[0] + ) numel = np.prod(var_shape) - use_mp = (self.mp_degree > 1) and (numel % - self.mp_degree == 0) + use_mp = (self.mp_degree > 1) and ( + numel % self.mp_degree == 0 + ) if 'subprog' in var.name: # For recompute, if the checkpoints var is layer_norm_6.tmp_2 @@ -5119,32 +5727,34 @@ def _insert_send_recv(cur_id, prev_id): self._op_device_key: cur_dev, self._op_role_key: op_role, 'use_calc_stream': True, - }) + }, + ) extra_index_info['index'] += 1 return _check_stage(cur_id, prev_id) - block._insert_op_without_sync(index=index + - extra_index_info['index'], - type='c_sync_calc_stream', - inputs={'X': [var]}, - outputs={'Out': [var]}, - attrs={ - self._op_device_key: - prev_dev, - self._op_role_key: - op_role, - }) + block._insert_op_without_sync( + index=index + extra_index_info['index'], + type='c_sync_calc_stream', + inputs={'X': [var]}, + outputs={'Out': [var]}, + attrs={ + self._op_device_key: prev_dev, + self._op_role_key: op_role, + }, + ) extra_index_info['index'] += 1 prefix_name = var.name.split('@')[0] prefix_var = block.var(prefix_name) - is_param = True if isinstance(prefix_var, - Parameter) else False + is_param = ( + True if isinstance(prefix_var, Parameter) else False + ) block._insert_op_without_sync( index=index + extra_index_info['index'], type='send_v2' - if not use_mp or is_param else 'partial_send', + if not use_mp or is_param + else 'partial_send', inputs={'X': var}, attrs={ self._op_device_key: prev_dev, @@ -5155,12 +5765,14 @@ def _insert_send_recv(cur_id, prev_id): # if send_v2, num&id attr is not in op_attrs, will not insert 'num': self.mp_degree, 'id': self.mp_rank, - }) + }, + ) extra_index_info['index'] += 1 insert_index = None if int(op_role) == int(self._op_role.Backward): insert_index = extra_index_info[ - 'first_optimize_index'] + 'first_optimize_index' + ] new_op_role = self._op_role.Optimize else: insert_index = index @@ -5174,14 +5786,16 @@ def _insert_send_recv(cur_id, prev_id): self._op_device_key: prev_dev, self._op_role_key: new_op_role, 'ring_id': ring_id, - }) + }, + ) if int(op_role) == int(self._op_role.Forward): sync_comm_op._set_attr('pipeline_flag', '') extra_index_info['index'] += 1 block._insert_op_without_sync( index=index + extra_index_info['index'], type='recv_v2' - if not use_mp or is_param else 'partial_recv', + if not use_mp or is_param + else 'partial_recv', outputs={'Out': [var]}, attrs={ 'out_shape': var_shape, @@ -5194,7 +5808,8 @@ def _insert_send_recv(cur_id, prev_id): # if recv_v2, num&id attr is not in op_attrs, will not insert 'num': self.mp_degree, 'id': self.mp_rank, - }) + }, + ) extra_index_info['index'] += 1 if use_mp and not is_param: block._insert_op_without_sync( @@ -5210,27 +5825,33 @@ def _insert_send_recv(cur_id, prev_id): # if recv_v2, num&id attr is not in op_attrs, will not insert 'nranks': self.mp_degree, 'rank': self.mp_rank, - }) + }, + ) extra_index_info['index'] += 1 else: raise ValueError( "Now only 'F-then-B' and '1F1B' are supported." - "The given value is {}.".format(self.schedule_mode)) + "The given value is {}.".format(self.schedule_mode) + ) - _insert_send_recv(int(cur_device.split(':')[1]), - int(prev_device.split(':')[1])) + _insert_send_recv( + int(cur_device.split(':')[1]), + int(prev_device.split(':')[1]), + ) block._sync_with_cpp() def _insert_loss_scale(self, block): """ Scale the loss corresponding to number of micro-batches. """ - if self._num_microbatches == 1: return + if self._num_microbatches == 1: + return for index, op in reversed(tuple(enumerate(list(block.ops)))): if self._is_loss_grad_op(op): - assert op.type == 'fill_constant', \ - "loss_grad_op must be fill_constant op, " \ + assert op.type == 'fill_constant', ( + "loss_grad_op must be fill_constant op, " "but this op is {}".format(op.type) + ) assert op.has_attr('value') loss_scale = float(op.attr('value')) loss_scale = loss_scale / self._num_microbatches @@ -5239,24 +5860,25 @@ def _insert_loss_scale(self, block): def _rename_gradient_var_name(self, block): for index, op in enumerate(block.ops): - if not self._is_optimize_op(op): continue + if not self._is_optimize_op(op): + continue input_names = op.input_arg_names output_names = op.output_arg_names in_out_names = input_names + output_names - if op.type == 'cast' or op.type == "c_sync_comm_stream": continue + if op.type == 'cast' or op.type == "c_sync_comm_stream": + continue # append "MERGED" to the names of parameter gradients, # and mofify the op_role_var attribute (by rename_arg func). for name in in_out_names: - if not core.grad_var_suffix() in name: continue + if not core.grad_var_suffix() in name: + continue param_name = name.strip(core.grad_var_suffix()) new_grad_name = name + "@MERGED" self._rename_arg(op, name, new_grad_name) - def _accumulate_gradients(self, - block, - pp_allreduce_in_optimize=False, - strategy=None, - shard=None): + def _accumulate_gradients( + self, block, pp_allreduce_in_optimize=False, strategy=None, shard=None + ): """ Create a new merged gradient for each parameter and accumulate the corresponding gradient to it. @@ -5264,7 +5886,8 @@ def _accumulate_gradients(self, fp16_allreduce = strategy.fp16_allreduce if strategy else False if strategy and strategy.fuse_grad_merge: fused_gradient_names = self._accumulate_gradients_with_fuse( - block, fp16_allreduce, strategy.fuse_grad_size_in_MB, shard) + block, fp16_allreduce, strategy.fuse_grad_size_in_MB, shard + ) return fused_gradient_names merged_gradient_names = [] @@ -5288,22 +5911,30 @@ def _accumulate_gradients(self, # maybe have no optimize # if first_opt_op_idx == len(block.ops): return - if self._is_backward_op(op) and (self._op_role_var_key - in op.attr_names): + if self._is_backward_op(op) and ( + self._op_role_var_key in op.attr_names + ): op_role_var = op.attr(self._op_role_var_key) - if len(op_role_var) == 0: continue + if len(op_role_var) == 0: + continue assert len(op_role_var) % 2 == 0 for i in range(0, len(op_role_var), 2): offset = 0 param_name = op_role_var[i] - if not block.has_var(param_name): continue - if '@BroadCast' in param_name: continue + if not block.has_var(param_name): + continue + if '@BroadCast' in param_name: + continue param_grad_name = param_name + core.grad_var_suffix() merged_param_grad_name = param_grad_name + merged_suffix if not block.has_var(merged_param_grad_name): - self._create_var(block, block.vars[param_name], - merged_param_grad_name, dtype) + self._create_var( + block, + block.vars[param_name], + merged_param_grad_name, + dtype, + ) assert block.has_var(merged_param_grad_name) param_grad_var = block.var(param_grad_name) @@ -5315,22 +5946,19 @@ def _accumulate_gradients(self, inputs={}, outputs={'Out': [merged_param_grad_var]}, attrs={ - 'shape': - merged_param_grad_var.shape, - 'dtype': - merged_param_grad_var.dtype, - 'value': - float(0), + 'shape': merged_param_grad_var.shape, + 'dtype': merged_param_grad_var.dtype, + 'value': float(0), # a trick to run this op once per mini-batch - self._op_role_key: - self._op_role.Optimize.LRSched, - }) + self._op_role_key: self._op_role.Optimize.LRSched, + }, + ) offset += 1 grad_name = op_role_var[i + 1] grad_var = block.vars[grad_name] is_fp16_grad = 'cast_fp16' in grad_name - need_cast = (is_fp16_grad is not fp16_allreduce) + need_cast = is_fp16_grad is not fp16_allreduce if need_cast: # if fp16_allreduce: @@ -5339,20 +5967,20 @@ def _accumulate_gradients(self, # cast grad to fp32 to accumulate to merged gradient cast_grad_var_name = param_grad_name + '@TMP' cast_grad_var = self._create_var( - block, param_grad_var, cast_grad_var_name, dtype) + block, param_grad_var, cast_grad_var_name, dtype + ) cast_grad_var.persistable = False - block._insert_op(index=first_opt_op_idx + offset, - type='cast', - inputs={'X': grad_var}, - outputs={'Out': cast_grad_var}, - attrs={ - 'in_dtype': - grad_var.dtype, - 'out_dtype': - cast_grad_var.dtype, - self._op_role_key: - self._op_role.Backward, - }) + block._insert_op( + index=first_opt_op_idx + offset, + type='cast', + inputs={'X': grad_var}, + outputs={'Out': cast_grad_var}, + attrs={ + 'in_dtype': grad_var.dtype, + 'out_dtype': cast_grad_var.dtype, + self._op_role_key: self._op_role.Backward, + }, + ) offset += 1 grad_var = cast_grad_var @@ -5363,11 +5991,13 @@ def _accumulate_gradients(self, outputs={'Out': merged_param_grad_var}, attrs={ self._op_role_key: self._op_role.Backward, - }) + }, + ) offset += 1 merged_gradient_names.append(merged_param_grad_name) - if not fp16_allreduce: return merged_gradient_names + if not fp16_allreduce: + return merged_gradient_names first_opt_op_idx = None for index, op in reversed(tuple(enumerate(list(block.ops)))): @@ -5391,28 +6021,31 @@ def _accumulate_gradients(self, grad_var = block.var(grad_name) grad_var.persistable = False - block._insert_op(index=first_opt_op_idx, - type='cast', - inputs={'X': fp16_grad_var}, - outputs={'Out': grad_var}, - attrs={ - 'in_dtype': fp16_grad_var.dtype, - 'out_dtype': grad_var.dtype, - self._op_role_key: self._op_role.Optimize, - }) + block._insert_op( + index=first_opt_op_idx, + type='cast', + inputs={'X': fp16_grad_var}, + outputs={'Out': grad_var}, + attrs={ + 'in_dtype': fp16_grad_var.dtype, + 'out_dtype': grad_var.dtype, + self._op_role_key: self._op_role.Optimize, + }, + ) return merged_gradient_names - def _insert_accumulate_gradients_with_fuse(self, main_block, fp16, - fused_size, grad_param_pairs, - first_opt_op_idx): + def _insert_accumulate_gradients_with_fuse( + self, main_block, fp16, fused_size, grad_param_pairs, first_opt_op_idx + ): grad_param_pairs = self._sort_grad_param_by_dtype( - main_block, grad_param_pairs) + main_block, grad_param_pairs + ) grad_param_segments = [] merged_suffix = '@MERGED@FP16' if fp16 else '@MERGED' dtype = paddle.float16 if fp16 else paddle.float32 - cur_size = 0. + cur_size = 0.0 last_dtype = None # split the grad based on dtype and fused size for grad, param in grad_param_pairs: @@ -5423,7 +6056,8 @@ def _insert_accumulate_gradients_with_fuse(self, main_block, fp16, dtype=dtype, shape=real_grad.shape, persistable=True, - stop_gradient=False) + stop_gradient=False, + ) real_param = main_block.var(param) if hasattr(real_param, 'is_distributed'): merged_grad_var.is_distributed = real_param.is_distributed @@ -5431,13 +6065,16 @@ def _insert_accumulate_gradients_with_fuse(self, main_block, fp16, # two strategies for splitting the grad # 1. the current segment's size reach the user defined grad_size_in_MB # 2. the upcoming grad holds different dtype compared with grads in current segment - if len(grad_param_segments) == 0 \ - or cur_size + tmp_size > fused_size \ - or real_grad.dtype != last_dtype: + if ( + len(grad_param_segments) == 0 + or cur_size + tmp_size > fused_size + or real_grad.dtype != last_dtype + ): grad_param_segments.append( - ([real_grad], [real_param], [merged_grad_var])) + ([real_grad], [real_param], [merged_grad_var]) + ) last_dtype = real_grad.dtype - cur_size = 0. + cur_size = 0.0 else: grad_param_segments[-1][0].append(real_grad) grad_param_segments[-1][1].append(real_param) @@ -5450,21 +6087,28 @@ def _insert_accumulate_gradients_with_fuse(self, main_block, fp16, for grad_param_segment in grad_param_segments: grad_segment = grad_param_segment[0] merged_grad_segment = grad_param_segment[2] - fused_grad = main_block.create_var(name='FusedGrad_{}'.format( - grad_segment[0].name), - dtype=grad_segment[0].dtype, - persistable=False, - stop_gradient=False) + fused_grad = main_block.create_var( + name='FusedGrad_{}'.format(grad_segment[0].name), + dtype=grad_segment[0].dtype, + persistable=False, + stop_gradient=False, + ) # keep the '.cast_fp16' info in the fuse var name - fused_merged_grad_name_prefix = 'FusedMergedGrad.cast_fp16.' if \ - merged_grad_segment[0].dtype == paddle.float16 else 'FusedMergedGrad' - fused_merged_grad_name = fused_merged_grad_name_prefix + '_{}'.format( - merged_grad_segment[0].name) + fused_merged_grad_name_prefix = ( + 'FusedMergedGrad.cast_fp16.' + if merged_grad_segment[0].dtype == paddle.float16 + else 'FusedMergedGrad' + ) + fused_merged_grad_name = ( + fused_merged_grad_name_prefix + + '_{}'.format(merged_grad_segment[0].name) + ) fused_merged_grad = main_block.create_var( name=fused_merged_grad_name, dtype=merged_grad_segment[0].dtype, persistable=True, - stop_gradient=False) + stop_gradient=False, + ) fused_gradients.append(fused_grad) fused_merged_gradients.append(fused_merged_grad) @@ -5490,10 +6134,7 @@ def _insert_accumulate_gradients_with_fuse(self, main_block, fp16, first_back_op_idx + offset, type="coalesce_tensor", inputs={"Input": params}, - outputs={ - "Output": grads, - "FusedOutput": fused_grad - }, + outputs={"Output": grads, "FusedOutput": fused_grad}, attrs={ # Explanation of user_defined_size_of_dtype: # In coalesce op, the align size is 256 bytes @@ -5517,7 +6158,8 @@ def _insert_accumulate_gradients_with_fuse(self, main_block, fp16, # To avoid these problematic triggers, set constant is needed for npu "set_constant": core.is_compiled_with_npu(), "constant": float(0.0), - }) + }, + ) offset += 1 # For the gradient_merged_fused_var, given a init value during the coalesce op # this will remove a problematic fill_constant op. This op role of this coalesce @@ -5528,7 +6170,7 @@ def _insert_accumulate_gradients_with_fuse(self, main_block, fp16, inputs={"Input": params}, outputs={ "Output": merged_grads, - "FusedOutput": fused_merged_grad + "FusedOutput": fused_merged_grad, }, attrs={ "user_defined_size_of_dtype": 2, @@ -5537,8 +6179,9 @@ def _insert_accumulate_gradients_with_fuse(self, main_block, fp16, "copy_data": False, "use_align": True, "dtype": merged_grads[0].dtype, - self._op_role_key: self._op_role.Optimize.LRSched - }) + self._op_role_key: self._op_role.Optimize.LRSched, + }, + ) offset += 1 # insert gradient merge relating ops @@ -5548,25 +6191,28 @@ def _insert_accumulate_gradients_with_fuse(self, main_block, fp16, fused_grad = fused_gradients[i] fused_merged_grad = fused_merged_gradients[i] is_fp16_grad = 'cast_fp16' in fused_grad.name - need_cast = (is_fp16_grad is not fp16) + need_cast = is_fp16_grad is not fp16 if need_cast: # for fp16 allreduce, cast fp32 grad to fp16 # for fp32 allreduce, cast fp16 grad to fp32 cast_grad_var_name = fused_grad.name + '@TMP' - cast_grad_var = main_block.create_var(name=cast_grad_var_name, - dtype=dtype, - persistable=False, - stop_gradient=False) - main_block._insert_op(index=first_opt_op_idx + offset, - type='cast', - inputs={'X': fused_grad}, - outputs={'Out': cast_grad_var}, - attrs={ - 'in_dtype': fused_grad.dtype, - 'out_dtype': cast_grad_var.dtype, - self._op_role_key: - self._op_role.Backward, - }) + cast_grad_var = main_block.create_var( + name=cast_grad_var_name, + dtype=dtype, + persistable=False, + stop_gradient=False, + ) + main_block._insert_op( + index=first_opt_op_idx + offset, + type='cast', + inputs={'X': fused_grad}, + outputs={'Out': cast_grad_var}, + attrs={ + 'in_dtype': fused_grad.dtype, + 'out_dtype': cast_grad_var.dtype, + self._op_role_key: self._op_role.Backward, + }, + ) offset += 1 fused_grad = cast_grad_var main_block._insert_op( @@ -5574,7 +6220,8 @@ def _insert_accumulate_gradients_with_fuse(self, main_block, fp16, type='sum', inputs={'X': [fused_merged_grad, fused_grad]}, outputs={'Out': fused_merged_grad}, - attrs={self._op_role_key: self._op_role.Backward}) + attrs={self._op_role_key: self._op_role.Backward}, + ) offset += 1 if fp16: @@ -5585,21 +6232,24 @@ def _insert_accumulate_gradients_with_fuse(self, main_block, fp16, assert main_block.has_var(fp16_grad_name) fp16_grad = main_block.var(fp16_grad_name) fp32_grad_name = param + core.grad_var_suffix() + '@MERGED' - fp32_grad = main_block.create_var(name=fp32_grad_name, - dtype=paddle.float32, - shape=real_grad.shape, - persistable=False, - stop_gradient=False) - main_block._insert_op(index=first_opt_op_idx + offset, - type='cast', - inputs={'X': fp16_grad}, - outputs={'Out': fp32_grad}, - attrs={ - 'in_dtype': paddle.float16, - 'out_dtype': paddle.float32, - self._op_role_key: - self._op_role.Optimize, - }) + fp32_grad = main_block.create_var( + name=fp32_grad_name, + dtype=paddle.float32, + shape=real_grad.shape, + persistable=False, + stop_gradient=False, + ) + main_block._insert_op( + index=first_opt_op_idx + offset, + type='cast', + inputs={'X': fp16_grad}, + outputs={'Out': fp32_grad}, + attrs={ + 'in_dtype': paddle.float16, + 'out_dtype': paddle.float32, + self._op_role_key: self._op_role.Optimize, + }, + ) offset += 1 # replace the var with it's name, which will be used for inserting allreduce @@ -5608,11 +6258,9 @@ def _insert_accumulate_gradients_with_fuse(self, main_block, fp16, return fused_merged_gradients, first_opt_op_idx - def _accumulate_gradients_with_fuse(self, - main_block, - fp16, - fused_size, - shard=None): + def _accumulate_gradients_with_fuse( + self, main_block, fp16, fused_size, shard=None + ): first_opt_op_idx = None grad_param_pairs = [] # obtain all param/grad pairs that needed to be fused @@ -5632,8 +6280,9 @@ def _accumulate_gradients_with_fuse(self, if first_opt_op_idx == len(main_block.ops): return - if self._is_backward_op(op) and (self._op_role_var_key - in op.attr_names): + if self._is_backward_op(op) and ( + self._op_role_var_key in op.attr_names + ): op_role_var = op.attr(self._op_role_var_key) if len(op_role_var) == 0: continue @@ -5645,7 +6294,8 @@ def _accumulate_gradients_with_fuse(self, if '@BroadCast' in param_name: continue grad_param_pairs.append( - (op_role_var[i + 1], op_role_var[i])) + (op_role_var[i + 1], op_role_var[i]) + ) if len(grad_param_pairs) == 0: return @@ -5659,9 +6309,12 @@ def _accumulate_gradients_with_fuse(self, all_fused_merged_gradients = [] for pairs in device_to_pairs: - fused_merged_gradients, first_opt_op_idx = \ - self._insert_accumulate_gradients_with_fuse( - main_block, fp16, fused_size, pairs, first_opt_op_idx) + ( + fused_merged_gradients, + first_opt_op_idx, + ) = self._insert_accumulate_gradients_with_fuse( + main_block, fp16, fused_size, pairs, first_opt_op_idx + ) all_fused_merged_gradients += fused_merged_gradients main_block._sync_with_cpp() @@ -5697,8 +6350,12 @@ def _get_var_size(self, var): core.VarDesc.VarType.UINT8: 1, } assert -1 not in var.shape - return reduce(lambda x, y: x * y, - var.shape) * dtype_to_size[var.dtype] / 1024.0 / 1024.0 + return ( + reduce(lambda x, y: x * y, var.shape) + * dtype_to_size[var.dtype] + / 1024.0 + / 1024.0 + ) def _add_sub_blocks(self, main_block, program_list): main_program = main_block.program @@ -5719,12 +6376,14 @@ def _add_sub_blocks(self, main_block, program_list): def _get_device_info(self, block): for op in block.ops: - if not op._has_kernel(op.type): continue + if not op._has_kernel(op.type): + continue op_device = op.attr(self._op_device_key) return op_device - def _process_persistable_vars_in_multi_sections(self, main_program, - startup_prog, program_list): + def _process_persistable_vars_in_multi_sections( + self, main_program, startup_prog, program_list + ): """ Special Case: process persistable vars that exist in multiple sections, e.g., shared weight @@ -5735,9 +6394,11 @@ def _process_persistable_vars_in_multi_sections(self, main_program, for prog in program_list: block = prog.block(0) for var_name in block.vars: - if var_name == "double_buffer_0": continue + if var_name == "double_buffer_0": + continue var = block.var(var_name) - if not var.persistable: continue + if not var.persistable: + continue if not var_name in var_info: var_info[var_name] = [] if not prog in var_info[var_name]: @@ -5753,23 +6414,30 @@ def _process_persistable_vars_in_multi_sections(self, main_program, for prog in var_info[var_name]: block = prog.block(0) for op in block.ops: - if op.type == "recv_v2" or op.type == "create_py_reader" or \ - op.type == "read" or op.type == "update_loss_scaling": + if ( + op.type == "recv_v2" + or op.type == "create_py_reader" + or op.type == "read" + or op.type == "update_loss_scaling" + ): continue # We have processed lr related vars if op.attr(self._op_role_key) == int( - self._op_role.Optimize.LRSched): + self._op_role.Optimize.LRSched + ): continue if var_name in op.desc.output_arg_names(): assert var_name not in write_info, ( "two sections write the same var({}): second " - "op {}.".format(var_name, op)) + "op {}.".format(var_name, op) + ) write_info[var_name] = prog break for var_name in var_info.keys(): # Case 1: read only variables, no special process - if not var_name in write_info: continue + if not var_name in write_info: + continue # Case 2: one write multiple reads write_prog = write_info[var_name] @@ -5778,7 +6446,8 @@ def _process_persistable_vars_in_multi_sections(self, main_program, write_dev_index = int(write_device.split(':')[1]) all_progs = var_info[var_name] for prog in all_progs: - if prog == write_prog: continue + if prog == write_prog: + continue read_block = prog.block(0) read_device = self._get_device_info(read_block) read_dev_index = int(read_device.split(':')[1]) @@ -5799,69 +6468,60 @@ def _process_persistable_vars_in_multi_sections(self, main_program, 'X': write_block.var(var_name), }, attrs={ - self._op_device_key: - write_device, - 'use_calc_stream': - False, + self._op_device_key: write_device, + 'use_calc_stream': False, # A trick to make the role LRSched to avoid copy every # microbatch - self._op_role_key: - self._op_role.LRSched, - 'peer': - read_dev_index, - 'ring_id': - ring_id - }) + self._op_role_key: self._op_role.LRSched, + 'peer': read_dev_index, + 'ring_id': ring_id, + }, + ) read_block._insert_op( index=0, type='recv_v2', outputs={'Out': [read_block.var(var_name)]}, attrs={ - 'out_shape': - read_block.var(var_name).shape, - 'dtype': - read_block.var(var_name).dtype, - self._op_device_key: - read_device, - 'use_calc_stream': - False, + 'out_shape': read_block.var(var_name).shape, + 'dtype': read_block.var(var_name).dtype, + self._op_device_key: read_device, + 'use_calc_stream': False, # A trick to make the role LRSched to avoid copy every # microbatch - self._op_role_key: - self._op_role.LRSched, - 'peer': - write_dev_index, - 'ring_id': - ring_id - }) + self._op_role_key: self._op_role.LRSched, + 'peer': write_dev_index, + 'ring_id': ring_id, + }, + ) read_block._insert_op( index=1, type='c_sync_comm_stream', inputs={'X': [read_block.var(var_name)]}, outputs={'Out': [read_block.var(var_name)]}, attrs={ - self._op_device_key: - read_device, + self._op_device_key: read_device, # A trick to make the role LRSched to avoid copy every # microbatch - self._op_role_key: - self._op_role.LRSched, - 'ring_id': - ring_id - }) + self._op_role_key: self._op_role.LRSched, + 'ring_id': ring_id, + }, + ) def _is_gradient_clip_op(self, op): - return op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/gradient_clip") + return op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/gradient_clip") def _is_regularization_op(self, op): - return op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/regularization") + return op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/regularization") def _is_weight_decay_op(self, op): # in AdamW namescope is /optimizer_*/weight decay/ - return op.desc.has_attr("op_namescope") \ - and 'weight decay' in op.desc.attr("op_namescope") + return op.desc.has_attr( + "op_namescope" + ) and 'weight decay' in op.desc.attr("op_namescope") def _get_input_output_info(self, block): ''' @@ -5884,7 +6544,8 @@ def _optimize_forward_send_sync(self, program): """ optimize forward send's sync_comm_stream schedule """ - if self.schedule_mode != '1F1B': return + if self.schedule_mode != '1F1B': + return block = program.block(0) @@ -5896,11 +6557,13 @@ def _optimize_forward_send_sync(self, program): break # last pipeline stage - if backward_recv_index is None: return + if backward_recv_index is None: + return offset = 0 for index, op in enumerate(list(block.ops)): - if index >= backward_recv_index: break + if index >= backward_recv_index: + break if op.type == 'c_sync_comm_stream' and op.has_attr('pipeline_flag'): var_name = op.input_arg_names[0] var = block.var(var_name) @@ -5917,7 +6580,8 @@ def _optimize_forward_send_sync(self, program): type='nop', inputs={'X': [var]}, outputs={'Out': [var]}, - attrs={self._op_role_key: self._op_role.Backward}) + attrs={self._op_role_key: self._op_role.Backward}, + ) block._sync_with_cpp() def _mv_head_recv(self, program): @@ -5933,10 +6597,17 @@ def _mv_head_recv(self, program): insert_index = None op = program.global_block().ops[i] op_role = int(op.attr(self._op_role_key)) - if op_role == int( - self._op_role.Backward) and backward_insert_index is None: + if ( + op_role == int(self._op_role.Backward) + and backward_insert_index is None + ): backward_insert_index = i - if op.type != "partial_recv" and op.type != "partial_allgather" and op.type != "nop" and op.type != "recv_v2": + if ( + op.type != "partial_recv" + and op.type != "partial_allgather" + and op.type != "nop" + and op.type != "recv_v2" + ): continue if op_role == int(self._op_role.Forward): if i == forward_insert_index: @@ -5956,11 +6627,13 @@ def _mv_head_recv(self, program): op_outputs = dict() for name in op.output_names: op_outputs[name] = op.output(name) - block._insert_op_without_sync(index=insert_index, - type=op.type, - inputs=op_inputs, - outputs=op_outputs, - attrs=op.all_attrs()) + block._insert_op_without_sync( + index=insert_index, + type=op.type, + inputs=op_inputs, + outputs=op_outputs, + attrs=op.all_attrs(), + ) block._remove_op(i + 1) if op_role == int(self._op_role.Forward): forward_insert_index += 1 @@ -5995,13 +6668,12 @@ def _check_pipeline_persist_var(self, program): "However, some backward op don't need this var(NoNeedBufferVars), " "there will be no error at this time.\n" "So please check these persistable vars which changed in " - "forward and used in backward:\n{}".format(used_in_backward)) + "forward and used in backward:\n{}".format(used_in_backward) + ) - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): main_block = loss.block self.origin_main_block = main_block main_program = main_block.program @@ -6021,8 +6693,9 @@ def minimize(self, 'mp_rank', ] for key in required_keys: - assert key in pipeline_opt, \ - 'Please use pipeline with fleet to use {}.'.format(key) + assert ( + key in pipeline_opt + ), 'Please use pipeline with fleet to use {}.'.format(key) self.local_rank = pipeline_opt['local_rank'] self.schedule_mode = pipeline_opt['schedule_mode'] self.micro_batch_size = pipeline_opt['micro_batch_size'] @@ -6036,11 +6709,14 @@ def minimize(self, assert 0 <= self.mp_rank < self.mp_degree optimize_ops, params_grads = self._optimizer.minimize( - loss, startup_program, parameter_list, no_grad_set) + loss, startup_program, parameter_list, no_grad_set + ) self._param_device_map = self._origin_optimizer._param_device_map - self.output_var_to_op, self.input_var_to_op = \ - self._get_input_output_info(main_block) + ( + self.output_var_to_op, + self.input_var_to_op, + ) = self._get_input_output_info(main_block) # Step1: add default op_device attribute for ops. self._add_op_device_attr(main_block) device_list = self._check_validation(main_block) @@ -6058,7 +6734,8 @@ def device_cmp(device1, device2): sorted_device_list = sorted(device_list, key=cmp_to_key(device_cmp)) assert sorted_device_list == device_list, ( "With pipeline parallelism, you must use gpu devices one after " - "another in the order of their ids.") + "another in the order of their ids." + ) # Step2: add send and recv ops between section boundaries self._insert_sendrecv_ops_for_boundaries(main_block) @@ -6074,7 +6751,8 @@ def device_cmp(device1, device2): assert self.local_rank < len(device_list), ( "Manually specified " "pipeline stage must be less than total number of pipeline " - "stages.") + "stages." + ) else: self.local_rank %= len(device_list) # Step3.5: optimize forward send sync_comm to overlap send and recv @@ -6099,7 +6777,8 @@ def device_cmp(device1, device2): # Step6: Split startup program new_startup_program = self._split_startup_program( - startup_program, self.local_rank) + startup_program, self.local_rank + ) startup_program._pipeline_opt = { "startup_program": new_startup_program, @@ -6141,12 +6820,18 @@ def device_cmp(device1, device2): "num_microbatches": self._num_microbatches, "start_cpu_core_id": self._start_cpu_core_id, } - return optimize_ops, params_grads, program_list, self._pipeline_pair, self._pp_ring_map + return ( + optimize_ops, + params_grads, + program_list, + self._pipeline_pair, + self._pp_ring_map, + ) class RecomputeOptimizer(Optimizer): """ - :api_attr: Static Graph + :api_attr: Static Graph Recompute Optimizer Wrapper @@ -6225,8 +6910,8 @@ def _set_checkpoints(self, checkpoints): checkpoints, list ), "_checkpoints should be a list of Variable or a list of String" for ckpt in checkpoints: - assert ( - isinstance(ckpt, str) or isinstance(ckpt, Variable) + assert isinstance(ckpt, str) or isinstance( + ckpt, Variable ), "_checkpoints should be a list of Variable or a list of String" self._checkpoints = checkpoints @@ -6237,7 +6922,7 @@ def _enable_offload(self): @framework.deprecate_stat_dict def load(self, state_dict): """ - :api_attr: Static Graph + :api_attr: Static Graph load function is not supported by Recompute Optimizer for now. :return: None @@ -6273,7 +6958,8 @@ def mlp(input_x, input_y, hid_dim=128, label_dim=2): print(cpt.get_exception_message(e)) """ raise NotImplementedError( - "load function is not supported by Recompute Optimizer for now") + "load function is not supported by Recompute Optimizer for now" + ) def apply_gradients(self, params_grads): """ @@ -6331,14 +7017,16 @@ def _creat_vars(self, varname): shape=self.checkpoint_shape, dtype=self._main_program.global_block().var(varname).dtype, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) fetch_var = self._main_program.global_block().create_var( name=fetched_var_name, shape=self.checkpoint_shape, dtype=self._main_program.global_block().var(varname).dtype, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) return pinned_var_name, fetched_var_name @@ -6363,21 +7051,25 @@ def _append_fill_constant_ops(self, startup_program): shape=self.checkpoint_shape, dtype=self._main_program.global_block().var(var.name).dtype, persistable=False, - stop_gradient=True) - block.append_op(type='fill_constant', - outputs={'Out': varname}, - attrs={ - "shape": var.shape, - "dtype": var.dtype, - "value": 0.0, - "place_type": 2, - OP_ROLE_KEY: op_role, - }) + stop_gradient=True, + ) + block.append_op( + type='fill_constant', + outputs={'Out': varname}, + attrs={ + "shape": var.shape, + "dtype": var.dtype, + "value": 0.0, + "place_type": 2, + OP_ROLE_KEY: op_role, + }, + ) return - def _insert_async_memcpy_op(self, insert_idx, src_varname, dst_varname, - op_role, dst_place_type): + def _insert_async_memcpy_op( + self, insert_idx, src_varname, dst_varname, op_role, dst_place_type + ): OP_ROLE_KEY = core.op_proto_and_checker_maker.kOpRoleAttrName() self.block._insert_op_without_sync( insert_idx, @@ -6386,22 +7078,26 @@ def _insert_async_memcpy_op(self, insert_idx, src_varname, dst_varname, outputs={ 'Out': [self._main_program.global_block().var(dst_varname)] }, - attrs={ - "dst_place_type": int(dst_place_type), - OP_ROLE_KEY: op_role - }) + attrs={"dst_place_type": int(dst_place_type), OP_ROLE_KEY: op_role}, + ) def _insert_fetch_op(self, idx, varname): - assert varname in self.checkpoint_name2pinned_name, "Try to fetch {} from Pinned Memory, but it is NOT a checkpoint".format( - varname) + assert ( + varname in self.checkpoint_name2pinned_name + ), "Try to fetch {} from Pinned Memory, but it is NOT a checkpoint".format( + varname + ) pinned_varname = self.checkpoint_name2pinned_name[varname] fetch_varname = self.checkpoint_name2fetch_name[varname] self._insert_async_memcpy_op(idx, pinned_varname, fetch_varname, 1, 1) def _insert_offload_op(self, idx, varname): - assert varname in self.checkpoint_name2pinned_name, "Try to offload {} to Pinned Memory, but it is NOT a checkpoint".format( - varname) + assert ( + varname in self.checkpoint_name2pinned_name + ), "Try to offload {} to Pinned Memory, but it is NOT a checkpoint".format( + varname + ) pinned_varname = self.checkpoint_name2pinned_name[varname] self._insert_async_memcpy_op(idx, varname, pinned_varname, 0, 2) @@ -6410,8 +7106,9 @@ def _insert_sync_op(self, op_idx, checkpoint_name): pass def _record_fetch_op(self, idx): - assert len(self.un_fetch_checkpoint_names - ) > 0, "Could NOT found checkpoint to fetch" + assert ( + len(self.un_fetch_checkpoint_names) > 0 + ), "Could NOT found checkpoint to fetch" checkpoint_name = self.un_fetch_checkpoint_names.pop(-1) logging.debug("Record fetch [{}]".format(checkpoint_name)) self.idx2insertions[idx] = ("fetch", checkpoint_name) @@ -6420,14 +7117,18 @@ def _record_fetch_op(self, idx): def _record_offload_op(self, idx, checkpoint_name): expected_checkpoint_name = self.un_offload_checkpoint_names.pop(0) - assert checkpoint_name == expected_checkpoint_name, "expected to offload [{}] but got [{}]".format( - expected_checkpoint_name, checkpoint_name) + assert ( + checkpoint_name == expected_checkpoint_name + ), "expected to offload [{}] but got [{}]".format( + expected_checkpoint_name, checkpoint_name + ) logging.debug("Record offload [{}]".format(checkpoint_name)) self.idx2insertions[idx] = ("offload", checkpoint_name) def _record_sync_op(self, idx, checkpoint_name): - assert checkpoint_name not in self.synced_checkpoints, "Try to sync the checkpoint [{}] twice".format( - checkpoint_name) + assert ( + checkpoint_name not in self.synced_checkpoints + ), "Try to sync the checkpoint [{}] twice".format(checkpoint_name) self.synced_checkpoints.add(checkpoint_name) logging.debug("Record offload sync [{}]".format(checkpoint_name)) self.idx2insertions[idx] = ("sync", checkpoint_name) @@ -6450,14 +7151,16 @@ def _parse_backward(self): break assert self.bw_strart_op_idx < len( - self.block.ops), "Could NOT found backword op in prog" + self.block.ops + ), "Could NOT found backword op in prog" # fetch second to last checkpoint at the beginning of BW fetched_checkpoint_varname = self._record_fetch_op( - self.bw_strart_op_idx) + self.bw_strart_op_idx + ) last_last_fetch_checkpoint = None - for i, op in enumerate(self.block.ops[self.bw_strart_op_idx:]): + for i, op in enumerate(self.block.ops[self.bw_strart_op_idx :]): idx = self.bw_strart_op_idx + i input_vars = op.desc.input_arg_names() @@ -6467,28 +7170,39 @@ def _parse_backward(self): # fetch the offloade checkpoint when the first usage of its previous one if self.checkpoint_usage_count[input_var] == 0: # TODO (JZ-LIANG) sync memcpy_stream if extra stream for memcpy - second_to_last_fetch_checkpoint = fetched_checkpoint_varname + second_to_last_fetch_checkpoint = ( + fetched_checkpoint_varname + ) # there is NO fetch ahead the first checkpoint if input_var != self.sorted_checkpoint_names[0]: - fetched_checkpoint_varname = self._record_fetch_op( - idx) + fetched_checkpoint_varname = ( + self._record_fetch_op(idx) + ) # should check the current used checkpoint is ths last fetch one - assert second_to_last_fetch_checkpoint == input_var, "Current recompute segment should use [{}] BUT got [{}]".format( - second_to_last_fetch_checkpoint, input_var) + assert ( + second_to_last_fetch_checkpoint == input_var + ), "Current recompute segment should use [{}] BUT got [{}]".format( + second_to_last_fetch_checkpoint, input_var + ) # rename self.block.ops[idx]._rename_input( input_var, - self.checkpoint_name2fetch_name[input_var]) + self.checkpoint_name2fetch_name[input_var], + ) self.checkpoint_usage_count[input_var] += 1 else: raise ValueError( "use checkpoint [{}] before fetch in BW".format( - input_var)) + input_var + ) + ) - assert len(self.un_fetch_checkpoint_names - ) == 0, "{} checkpoints have NOT been Recorded".format( - self.un_fetch_checkpoint_names) + assert ( + len(self.un_fetch_checkpoint_names) == 0 + ), "{} checkpoints have NOT been Recorded".format( + self.un_fetch_checkpoint_names + ) def _update_backward(self): if len(self.idx2insertions) == 0: @@ -6500,15 +7214,18 @@ def _update_backward(self): if operation == "fetch": self._insert_fetch_op(op_idx, checkpoint_name) logging.debug( - "Insert [{}] fetch op.".format(checkpoint_name)) + "Insert [{}] fetch op.".format(checkpoint_name) + ) del self.idx2insertions[op_idx] elif operation == "sync": self._insert_sync_op(op_idx, checkpoint_name) logging.debug("Sync [{}] fetch op.".format(checkpoint_name)) self.block._sync_with_cpp() - assert len( - self.idx2insertions) == 0, "{} checkpoints left un-Fecthed".format( - [ele[1] for ele in self.idx2insertions.values()]) + assert ( + len(self.idx2insertions) == 0 + ), "{} checkpoints left un-Fecthed".format( + [ele[1] for ele in self.idx2insertions.values()] + ) def _parse_forward(self): @@ -6521,7 +7238,7 @@ def _parse_forward(self): for checkpoint_name in self.un_offload_checkpoint_names: self.checkpoint_usage_count_and_idx[checkpoint_name] = { 'count': 0, - 'idx': -1 + 'idx': -1, } self.synced_checkpoints = set() self.fw_strart_op_idx = len(self.block.ops) @@ -6531,11 +7248,13 @@ def _parse_forward(self): break assert self.fw_strart_op_idx < len( - self.block.ops), "Could NOT found Forward op in prog" + self.block.ops + ), "Could NOT found Forward op in prog" last_offload_checkpoint = None for i, op in enumerate( - self.block.ops[self.fw_strart_op_idx:self.bw_strart_op_idx]): + self.block.ops[self.fw_strart_op_idx : self.bw_strart_op_idx] + ): idx = self.fw_strart_op_idx + i output_vars = op.desc.output_arg_names() @@ -6543,91 +7262,129 @@ def _parse_forward(self): for output_var in output_vars: if output_var in need_offload_checkpoint_names: - assert len( - output_vars - ) == 1, "chekpoint should be the only Output of a certain op, but [{}] is from [{}]".format( - output_var, op) + assert ( + len(output_vars) == 1 + ), "chekpoint should be the only Output of a certain op, but [{}] is from [{}]".format( + output_var, op + ) if output_var in self.un_offload_checkpoint_names: # insert sync op if last checkpoint has not been sync if last_offload_checkpoint != None: - if self.checkpoint_usage_count_and_idx[ - last_offload_checkpoint]['count'] == 0: - self._record_sync_op(idx, - last_offload_checkpoint) + if ( + self.checkpoint_usage_count_and_idx[ + last_offload_checkpoint + ]['count'] + == 0 + ): + self._record_sync_op( + idx, last_offload_checkpoint + ) else: - last_usage_idx = self.checkpoint_usage_count_and_idx[ - last_offload_checkpoint]['idx'] - assert last_usage_idx > 0, "last_usage_idx of checkpoint [{}] should large than 0".format( - last_offload_checkpoint) - self._record_sync_op(last_usage_idx + 1, - last_offload_checkpoint) + last_usage_idx = ( + self.checkpoint_usage_count_and_idx[ + last_offload_checkpoint + ]['idx'] + ) + assert ( + last_usage_idx > 0 + ), "last_usage_idx of checkpoint [{}] should large than 0".format( + last_offload_checkpoint + ) + self._record_sync_op( + last_usage_idx + 1, last_offload_checkpoint + ) # insert offload op after the checkpoint's generation op self._record_offload_op(idx + 1, output_var) last_offload_checkpoint = output_var else: raise ValueError( - "There should be just ONE op that output checkpoint [{}]" - .format(output_var)) + "There should be just ONE op that output checkpoint [{}]".format( + output_var + ) + ) # need to sync the last need to offload checkpoint before the last checkpoint as output op if output_var == last_checkpoint: - assert len( - output_vars - ) == 1, "chekpoint should be the only Output of a certain op, but [{}] is from [{}]".format( - output_var, op) - assert last_offload_checkpoint == self.sorted_checkpoint_names[ - -2], "the last offload chekpoint before [{}] is suppose to be [{}], but got [{}]".format( - last_checkpoint, self.sorted_checkpoint_names[-2], - last_offload_checkpoint) + assert ( + len(output_vars) == 1 + ), "chekpoint should be the only Output of a certain op, but [{}] is from [{}]".format( + output_var, op + ) + assert ( + last_offload_checkpoint + == self.sorted_checkpoint_names[-2] + ), "the last offload chekpoint before [{}] is suppose to be [{}], but got [{}]".format( + last_checkpoint, + self.sorted_checkpoint_names[-2], + last_offload_checkpoint, + ) # sync if last checkpoint has not been sync - if self.checkpoint_usage_count_and_idx[ - last_offload_checkpoint]['idx'] == 0: + if ( + self.checkpoint_usage_count_and_idx[ + last_offload_checkpoint + ]['idx'] + == 0 + ): self._record_sync_op(idx, last_offload_checkpoint) else: last_usage_idx = self.checkpoint_usage_count_and_idx[ - last_offload_checkpoint]['idx'] - assert last_usage_idx > 0, "last_usage_idx of checkpoint [{}] should large than 0".format( - last_offload_checkpoint) - self._record_sync_op(last_usage_idx + 1, - last_offload_checkpoint) + last_offload_checkpoint + ]['idx'] + assert ( + last_usage_idx > 0 + ), "last_usage_idx of checkpoint [{}] should large than 0".format( + last_offload_checkpoint + ) + self._record_sync_op( + last_usage_idx + 1, last_offload_checkpoint + ) # record checkpoint usage for input_var in input_vars: if input_var in need_offload_checkpoint_names: - assert input_var not in self.synced_checkpoints, "checkpoint [{}] used after sync".format( - input_var) + assert ( + input_var not in self.synced_checkpoints + ), "checkpoint [{}] used after sync".format(input_var) self.checkpoint_usage_count_and_idx[input_var]['count'] += 1 self.checkpoint_usage_count_and_idx[input_var]['idx'] = idx - assert len(self.un_offload_checkpoint_names - ) == 0, "{} checkpoints have NOT been Recorded".format( - self.un_fetch_checkpoint_names) + assert ( + len(self.un_offload_checkpoint_names) == 0 + ), "{} checkpoints have NOT been Recorded".format( + self.un_fetch_checkpoint_names + ) assert len(self.synced_checkpoints) == len( need_offload_checkpoint_names ), "{} checkpoints have NOT been Recorded".format( - set(need_offload_checkpoint_names) - set(self.synced_checkpoints)) + set(need_offload_checkpoint_names) - set(self.synced_checkpoints) + ) def _update_forward(self): if len(self.idx2insertions) == 0: return for op_idx in reversed( - range(self.fw_strart_op_idx, self.bw_strart_op_idx)): + range(self.fw_strart_op_idx, self.bw_strart_op_idx) + ): if op_idx in self.idx2insertions: operation, checkpoint_name = self.idx2insertions[op_idx] if operation == "offload": self._insert_offload_op(op_idx, checkpoint_name) logging.debug( - "Insert [{}] offload op.".format(checkpoint_name)) + "Insert [{}] offload op.".format(checkpoint_name) + ) del self.idx2insertions[op_idx] elif operation == "sync": self._insert_sync_op(op_idx, checkpoint_name) logging.debug( - "Insert [{}] offload_sync op.".format(checkpoint_name)) + "Insert [{}] offload_sync op.".format(checkpoint_name) + ) del self.idx2insertions[op_idx] self.block._sync_with_cpp() - assert len(self.idx2insertions - ) == 0, "{} checkpoints left un-Offloaded".format( - [ele[1] for ele in self.idx2insertions.values()]) + assert ( + len(self.idx2insertions) == 0 + ), "{} checkpoints left un-Offloaded".format( + [ele[1] for ele in self.idx2insertions.values()] + ) def _check_offload_fetch(self): # TODO(JZ-LIANG) the single stream offload need no sync @@ -6647,21 +7404,28 @@ def _offload(self, loss, startup_program=None): startup_program = paddle.static.default_startup_program() with program_guard(self._main_program, startup_program): - assert len(self.checkpoint_shape) > 0, ( - "checkpoints shape {} should be an non empty list like: [12, 512, 1024]" - .format(self.checkpoint_shape)) - assert all([ele > 0 for ele in self.checkpoint_shape]), ( - "all ele in checkpoints shape {} should be a determined integer larger than 0" - .format(self.checkpoint_shape)) + assert ( + len(self.checkpoint_shape) > 0 + ), "checkpoints shape {} should be an non empty list like: [12, 512, 1024]".format( + self.checkpoint_shape + ) + assert all( + [ele > 0 for ele in self.checkpoint_shape] + ), "all ele in checkpoints shape {} should be a determined integer larger than 0".format( + self.checkpoint_shape + ) self.checkpoint_name2pinned_name = dict() self.checkpoint_name2fetch_name = dict() for checkpoint_varname in self.sorted_checkpoint_names: pinned_var_name, fetch_var_name = self._creat_vars( - checkpoint_varname) + checkpoint_varname + ) self.checkpoint_name2pinned_name[ - checkpoint_varname] = pinned_var_name + checkpoint_varname + ] = pinned_var_name self.checkpoint_name2fetch_name[ - checkpoint_varname] = fetch_var_name + checkpoint_varname + ] = fetch_var_name self._append_fill_constant_ops(startup_program) # TODO (JZ-LIANG) to provide two offload stragtegy in future # step 2. parse & update FW: rename, offload, sync @@ -6675,12 +7439,14 @@ def _offload(self, loss, startup_program=None): return - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): """ call append_backward with checkpoints. @@ -6722,12 +7488,14 @@ def mlp(input_x, input_y, hid_dim=128, label_dim=2): no_grad_set=None) print("Finished backward") """ - assert (self._checkpoints - is not None), "You should call _set_checkpoints first" + assert ( + self._checkpoints is not None + ), "You should call _set_checkpoints first" if framework._non_static_mode(): raise NotImplementedError( - "DyGraph current does not support recompute") + "DyGraph current does not support recompute" + ) self._dtype = loss.dtype program = loss.block.program @@ -6745,12 +7513,15 @@ def mlp(input_x, input_y, hid_dim=128, label_dim=2): loss, parameter_list, no_grad_set, - checkpoints=checkpoint_vars) + checkpoints=checkpoint_vars, + ) else: - params_grads = append_backward(loss, - parameter_list, - no_grad_set, - checkpoints=checkpoint_vars) + params_grads = append_backward( + loss, + parameter_list, + no_grad_set, + checkpoints=checkpoint_vars, + ) if self.enable_offload: self.sorted_checkpoint_names = sorted_checkpoint_names @@ -6797,39 +7568,43 @@ def mlp(input_x, input_y, hid_dim=128, label_dim=2): print("Finished apply_optimize") """ - func = self._optimizer.apply_optimize if hasattr( - self._optimizer, - 'apply_optimize') else self._optimizer._apply_optimize - return func(loss, - startup_program=startup_program, - params_grads=params_grads) - - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + func = ( + self._optimizer.apply_optimize + if hasattr(self._optimizer, 'apply_optimize') + else self._optimizer._apply_optimize + ) + return func( + loss, startup_program=startup_program, params_grads=params_grads + ) + + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): assert isinstance(loss, Variable), "The loss should be an Variable." - assert (self._checkpoints - is not None), "You should call _set_checkpoints first" + assert ( + self._checkpoints is not None + ), "You should call _set_checkpoints first" if framework._non_static_mode(): raise NotImplementedError( - "DyGraph current does not support recompute") - params_grads = self.backward(loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) + "DyGraph current does not support recompute" + ) + params_grads = self.backward( + loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) - optimize_ops = self.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) + optimize_ops = self.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) return optimize_ops, params_grads class LookaheadOptimizer(object): r""" - :api_attr: Static Graph + :api_attr: Static Graph This implements the Lookahead optimizer of the paper : https://arxiv.org/abs/1907.08610. @@ -6843,7 +7618,7 @@ class LookaheadOptimizer(object): slow\_param_t &= slow\_param_{t-1} + \\alpha * (fast\_param_{t-1} - slow\_param_{t-1}) - fast\_param_t &= slow\_param_t + fast\_param_t &= slow\_param_t Args: inner_optimizer (Optimizer): The optimizer that update fast params step by step. @@ -6892,11 +7667,11 @@ def __init__(self, inner_optimizer, alpha=0.5, k=5): if framework._non_static_mode(): raise Exception("In dygraph, don't support LookaheadOptimizer.") - assert (inner_optimizer is not None), "inner optimizer can not be None" + assert inner_optimizer is not None, "inner optimizer can not be None" assert ( 0.0 <= alpha <= 1.0 ), "alpha should be larger or equal to 0.0, and less or equal than 1.0" - assert (isinstance(k, int) and k > 0), "k should be a positive integer" + assert isinstance(k, int) and k > 0, "k should be a positive integer" self.inner_optimizer = inner_optimizer self.alpha = alpha @@ -6907,7 +7682,8 @@ def minimize(self, loss, startup_program=None): # Apply inner optimizer to the main_program mini_out = self.inner_optimizer.minimize( - loss, startup_program=startup_program) + loss, startup_program=startup_program + ) # Get startup_program and main_program if startup_program is None: @@ -6919,58 +7695,68 @@ def minimize(self, loss, startup_program=None): param_to_slow = {} for param in params: fast_var = main_block.var(param) - assert (fast_var is not None) - slow_var = main_block.create_var(name=param + "@SLOW", - shape=fast_var.shape, - dtype=fast_var.dtype, - persistable=True) + assert fast_var is not None + slow_var = main_block.create_var( + name=param + "@SLOW", + shape=fast_var.shape, + dtype=fast_var.dtype, + persistable=True, + ) param_to_slow[param] = slow_var # add some vars to the startup_program startup_block = startup_program.global_block() for param in params: fast_var = startup_block.var(param) - assert (fast_var is not None) - slow_var = startup_block.create_var(name=param + "@SLOW", - shape=fast_var.shape, - dtype=fast_var.dtype, - persistable=True) + assert fast_var is not None + slow_var = startup_block.create_var( + name=param + "@SLOW", + shape=fast_var.shape, + dtype=fast_var.dtype, + persistable=True, + ) - startup_block.append_op(type="assign", - inputs={"X": fast_var}, - outputs={"Out": slow_var}) + startup_block.append_op( + type="assign", inputs={"X": fast_var}, outputs={"Out": slow_var} + ) with framework.program_guard(main_block.program, startup_program): # Add Var k to main prog and startup prog - k = layers.create_global_var(name="lookahead_k", - shape=[1], - value=int(self.k), - dtype='int32', - persistable=True) + k = layers.create_global_var( + name="lookahead_k", + shape=[1], + value=int(self.k), + dtype='int32', + persistable=True, + ) # Add Var alpha to main prog and startup prog - alpha = layers.create_global_var(name="lookahead_alpha", - shape=[1], - value=float(self.alpha), - dtype='float32', - persistable=True) + alpha = layers.create_global_var( + name="lookahead_alpha", + shape=[1], + value=float(self.alpha), + dtype='float32', + persistable=True, + ) # Add Var step - step = layers.create_global_var(name="lookahead_step", - shape=[1], - value=int(0), - dtype='int32', - persistable=True) + step = layers.create_global_var( + name="lookahead_step", + shape=[1], + value=int(0), + dtype='int32', + persistable=True, + ) layers.increment(x=step, value=1.0, in_place=True) # lookahead - zero_var = layers.fill_constant(shape=[1], - dtype='float32', - value=0.0) + zero_var = layers.fill_constant( + shape=[1], dtype='float32', value=0.0 + ) - one_var = layers.fill_constant(shape=[1], - dtype='float32', - value=1.0) + one_var = layers.fill_constant( + shape=[1], dtype='float32', value=1.0 + ) mod = layers.elementwise_mod(step, k) with layers.control_flow.Switch() as switch: @@ -6986,8 +7772,9 @@ def minimize(self, loss, startup_program=None): tmp_var = layers.elementwise_add( layers.elementwise_mul(fast_var, alpha), layers.elementwise_mul( - slow_var, - layers.elementwise_sub(one_var, alpha))) + slow_var, layers.elementwise_sub(one_var, alpha) + ), + ) layers.assign(input=tmp_var, output=slow_var) layers.assign(input=tmp_var, output=fast_var) with switch.default(): @@ -7057,11 +7844,13 @@ def __init__(self, inner_optimizer, k_steps=1, avg=True): raise Exception( "In dygraph, we don't support GradientMergeOptimizer." "You can do Gradient merge by yourself with k-times forward + backward, " - "and one-time optimizer.minimize()") + "and one-time optimizer.minimize()" + ) - assert (inner_optimizer is not None), "inner optimizer can not be None" - assert (isinstance(k_steps, int) - and k_steps > 0), "k_steps should be a positive integer" + assert inner_optimizer is not None, "inner optimizer can not be None" + assert ( + isinstance(k_steps, int) and k_steps > 0 + ), "k_steps should be a positive integer" self.inner_optimizer = inner_optimizer self.k_steps = k_steps @@ -7075,12 +7864,14 @@ def _set_k_steps(self, k_steps): def _set_avg(self, avg): self.avg = avg - def backward(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ): assert isinstance(loss, Variable), "The loss should be an Variable." assert ( parameter_list is None @@ -7090,7 +7881,8 @@ def backward(self, ), "The no_grad_set should be None when using GradientMergeOptimizer" params_grads = self.inner_optimizer.backward( - loss, startup_program=startup_program) + loss, startup_program=startup_program + ) return params_grads def apply_optimize(self, loss, startup_program, params_grads): @@ -7102,26 +7894,33 @@ def apply_optimize(self, loss, startup_program, params_grads): def _is_the_backward_op(self, op): op_maker = core.op_proto_and_checker_maker backward = core.op_proto_and_checker_maker.OpRole.Backward - if op_maker.kOpRoleVarAttrName() in op.attr_names and \ - int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(backward): + if op_maker.kOpRoleVarAttrName() in op.attr_names and int( + op.all_attrs()[op_maker.kOpRoleAttrName()] + ) == int(backward): return True return False def _remove_op_role_var(self, param, grad): op_maker = core.op_proto_and_checker_maker op = grad.op - assert self._is_the_backward_op(op), \ - 'grad.op={} is not the backward op which produces the grad={}' \ - .format(op, grad.name) + assert self._is_the_backward_op( + op + ), 'grad.op={} is not the backward op which produces the grad={}'.format( + op, grad.name + ) block = grad.block var_attr = op.all_attrs()[op_maker.kOpRoleVarAttrName()] - assert param.name in var_attr, \ - 'when using GradientMergeOptimizer, param={} must be in var_attr={}' \ - .format(param.name, var_attr) - assert grad.name in var_attr, \ - 'when using GradientMergeOptimizer, grad={} must be in var_attr={}' \ - .format(param.name, var_attr) + assert ( + param.name in var_attr + ), 'when using GradientMergeOptimizer, param={} must be in var_attr={}'.format( + param.name, var_attr + ) + assert ( + grad.name in var_attr + ), 'when using GradientMergeOptimizer, grad={} must be in var_attr={}'.format( + param.name, var_attr + ) # remove (param, grad) from op_role_var var_attr.remove(param.name) @@ -7148,53 +7947,54 @@ def _add_gm_op_role_var(self, op, param, grad, cond): def _get_gm_cond_var(self, main_block): # Add const var - k_step_var = layers.create_global_var(name="gradient_merge_k", - shape=[1], - value=int(self.k_steps), - dtype='int32', - persistable=True, - force_cpu=True) - - zero_var = layers.create_global_var(name="gradient_merge_zero", - shape=[1], - value=int(0), - dtype='int32', - persistable=True, - force_cpu=True) + k_step_var = layers.create_global_var( + name="gradient_merge_k", + shape=[1], + value=int(self.k_steps), + dtype='int32', + persistable=True, + force_cpu=True, + ) + + zero_var = layers.create_global_var( + name="gradient_merge_zero", + shape=[1], + value=int(0), + dtype='int32', + persistable=True, + force_cpu=True, + ) # Add step var & cond var - step_var = layers.create_global_var(name="gradient_merge_step", - shape=[1], - value=int(0), - dtype='int32', - persistable=True, - force_cpu=True) + step_var = layers.create_global_var( + name="gradient_merge_step", + shape=[1], + value=int(0), + dtype='int32', + persistable=True, + force_cpu=True, + ) - cond_var = main_block.create_var(name="gradient_merge_cond", - shape=[1], - dtype='bool') + cond_var = main_block.create_var( + name="gradient_merge_cond", shape=[1], dtype='bool' + ) with device_guard("cpu"): # step_var = (step_var + 1) % k_step layers.increment(x=step_var, value=1.0, in_place=True) - main_block.append_op(type='elementwise_mod', - inputs={ - 'X': step_var, - 'Y': k_step_var - }, - outputs={'Out': step_var}, - attrs={ - 'axis': -1, - 'use_mkldnn': False - }) + main_block.append_op( + type='elementwise_mod', + inputs={'X': step_var, 'Y': k_step_var}, + outputs={'Out': step_var}, + attrs={'axis': -1, 'use_mkldnn': False}, + ) # cond_var = (step_var == 0) - main_block.append_op(type='equal', - inputs={ - 'X': step_var, - 'Y': zero_var - }, - outputs={'Out': cond_var}) + main_block.append_op( + type='equal', + inputs={'X': step_var, 'Y': zero_var}, + outputs={'Out': cond_var}, + ) return cond_var @@ -7206,7 +8006,7 @@ def apply_gradients(self, params_grads): cond = self._get_gm_cond_var(main_block) - #TODO(mapingshuo) support sparse embedding + # TODO(mapingshuo) support sparse embedding # step1: remove grad.op's op_role_var for param, grad in params_grads: assert ( @@ -7225,41 +8025,41 @@ def apply_gradients(self, params_grads): for param, grad in params_grads: param_name = param.name param_var = main_block.var(param_name) - assert (param_var is not None) - gradient_merge_var = main_block.create_var(name=param_name + - "@GRAD@GradientMerge", - shape=param_var.shape, - dtype=param_var.dtype, - persistable=True) + assert param_var is not None + gradient_merge_var = main_block.create_var( + name=param_name + "@GRAD@GradientMerge", + shape=param_var.shape, + dtype=param_var.dtype, + persistable=True, + ) param_to_gradient_merge[param_name] = gradient_merge_var startup_gradient_merge_var = startup_block.create_var( name=param_name + "@GRAD@GradientMerge", shape=param_var.shape, dtype=param_var.dtype, - persistable=True) - startup_block.append_op(type="fill_constant", - outputs={"Out": startup_gradient_merge_var}, - attrs={ - "shape": param_var.shape, - "dtype": param_var.dtype, - "value": float(0), - }) + persistable=True, + ) + startup_block.append_op( + type="fill_constant", + outputs={"Out": startup_gradient_merge_var}, + attrs={ + "shape": param_var.shape, + "dtype": param_var.dtype, + "value": float(0), + }, + ) # grad_merge += grad new_grad_op = main_block.append_op( type="elementwise_add", - inputs={ - 'X': grad, - 'Y': gradient_merge_var - }, + inputs={'X': grad, 'Y': gradient_merge_var}, outputs={'Out': gradient_merge_var}, - attrs={ - 'axis': -1, - 'use_mkldnn': False - }) - self._add_gm_op_role_var(new_grad_op, param, gradient_merge_var, - cond) + attrs={'axis': -1, 'use_mkldnn': False}, + ) + self._add_gm_op_role_var( + new_grad_op, param, gradient_merge_var, cond + ) new_params_grads.append([param, gradient_merge_var]) def true_apply_gradient(): @@ -7273,16 +8073,19 @@ def true_apply_gradient(): if self.avg: for param, new_grad in new_params_grads: # grad /= k_steps - cur_block.append_op(type='scale', - inputs={'X': new_grad}, - outputs={'Out': new_grad}, - attrs={ - 'scale': 1.0 / self.k_steps, - 'bias': 0.0, - 'bias_after_scale': False - }) - new_grad.op._set_attr(op_maker.kOpRoleAttrName(), - op_maker.OpRole.Backward) + cur_block.append_op( + type='scale', + inputs={'X': new_grad}, + outputs={'Out': new_grad}, + attrs={ + 'scale': 1.0 / self.k_steps, + 'bias': 0.0, + 'bias_after_scale': False, + }, + ) + new_grad.op._set_attr( + op_maker.kOpRoleAttrName(), op_maker.OpRole.Backward + ) for param, new_grad in new_params_grads: # NOTE. regularization will append ops to grad.block, @@ -7292,36 +8095,40 @@ def true_apply_gradient(): new_grad.block = cur_block self._optimize_ops = self.inner_optimizer.apply_gradients( - new_params_grads) + new_params_grads + ) # clear gradient_merge_vars for param, new_grad in new_params_grads: - layers.fill_constant(shape=new_grad.shape, - dtype=new_grad.dtype, - value=0.0, - out=new_grad) - new_grad.op._set_attr(op_maker.kOpRoleAttrName(), - op_maker.OpRole.Optimize) + layers.fill_constant( + shape=new_grad.shape, + dtype=new_grad.dtype, + value=0.0, + out=new_grad, + ) + new_grad.op._set_attr( + op_maker.kOpRoleAttrName(), op_maker.OpRole.Optimize + ) # step3. apply gradient layers.cond(cond, true_fn=true_apply_gradient, false_fn=None) return self._optimize_ops - def minimize(self, - loss, - startup_program=None, - parameter_list=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameter_list=None, no_grad_set=None + ): assert isinstance(loss, Variable), "The loss should be an Variable." - params_grads = self.backward(loss, - startup_program=startup_program, - parameter_list=parameter_list, - no_grad_set=no_grad_set) + params_grads = self.backward( + loss, + startup_program=startup_program, + parameter_list=parameter_list, + no_grad_set=no_grad_set, + ) - optimize_ops = self.apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) + optimize_ops = self.apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) return optimize_ops, params_grads diff --git a/python/paddle/fluid/parallel_executor.py b/python/paddle/fluid/parallel_executor.py index fe8a2b2610bb2..c0a45b041e6a5 100644 --- a/python/paddle/fluid/parallel_executor.py +++ b/python/paddle/fluid/parallel_executor.py @@ -27,7 +27,7 @@ class ParallelExecutor(object): """ - :api_attr: Static Graph + :api_attr: Static Graph The ParallelExecutor is an upgraded version of :code:`paddle.static.Executor` that supports multi-node model training and testing based on the data-parallel mode. In data-parallel mode, @@ -147,16 +147,18 @@ class ParallelExecutor(object): """ - def __init__(self, - use_cuda, - loss_name=None, - main_program=None, - share_vars_from=None, - exec_strategy=None, - build_strategy=None, - num_trainers=1, - trainer_id=0, - scope=None): + def __init__( + self, + use_cuda, + loss_name=None, + main_program=None, + share_vars_from=None, + exec_strategy=None, + build_strategy=None, + num_trainers=1, + trainer_id=0, + scope=None, + ): if build_strategy is None: build_strategy = BuildStrategy() @@ -164,22 +166,28 @@ def __init__(self, if num_trainers != 1 and build_strategy.num_trainers != num_trainers: sys.stderr.write( 'The value of build_strategy.num_trainers[%d] is overwritten ' - 'by the passed num_trainers[%d].\n' % - (build_strategy.num_trainers, num_trainers)) + 'by the passed num_trainers[%d].\n' + % (build_strategy.num_trainers, num_trainers) + ) build_strategy.num_trainers = num_trainers if trainer_id != 0 and build_strategy.trainer_id != trainer_id: sys.stderr.write( 'The value of build_strategy.trainer_id[%d] is overwritten ' - 'by the passed trainer_id[%d].\n' % - (build_strategy.trainer_id, trainer_id)) + 'by the passed trainer_id[%d].\n' + % (build_strategy.trainer_id, trainer_id) + ) build_strategy.trainer_id = trainer_id - self._places = framework.cuda_places( - ) if use_cuda else framework.cpu_places() + self._places = ( + framework.cuda_places() if use_cuda else framework.cpu_places() + ) self._scope = scope if scope is not None else executor.global_scope() - main_program = main_program if main_program is not None \ + main_program = ( + main_program + if main_program is not None else framework.default_main_program() + ) self._compiled_program = compiler.CompiledProgram(main_program) if share_vars_from: @@ -192,7 +200,9 @@ def __init__(self, build_strategy=build_strategy, exec_strategy=exec_strategy, share_vars_from=share_vars_from._compiled_program - if share_vars_from else None) + if share_vars_from + else None, + ) self._place = core.CUDAPlace(0) if use_cuda else core.CPUPlace() self._exe = executor.Executor(self._place) @@ -299,11 +309,13 @@ def run(self, fetch_list, feed=None, feed_dict=None, return_numpy=True): fetch_list=[loss.name]) """ - return self._exe.run(program=self._compiled_program, - scope=self._scope, - feed=feed, - fetch_list=fetch_list, - return_numpy=return_numpy) + return self._exe.run( + program=self._compiled_program, + scope=self._scope, + feed=feed, + fetch_list=fetch_list, + return_numpy=return_numpy, + ) @property def device_count(self): @@ -363,14 +375,20 @@ def drop_local_exe_scopes(self): parallel_exe.drop_local_exe_scopes() """ - check_type(self._compiled_program._executor, - "the Executor of compiled program", core.ParallelExecutor, - "ParallelExecutor.drop_local_exe_scopes") + check_type( + self._compiled_program._executor, + "the Executor of compiled program", + core.ParallelExecutor, + "ParallelExecutor.drop_local_exe_scopes", + ) self._compiled_program._executor.drop_local_exe_scopes() # This API is used to check whether DropLocalExeScopes can work. def _need_create_local_exe_scopes(self): - check_type(self._compiled_program._executor, - "the Executor of compiled program", core.ParallelExecutor, - "ParallelExecutor._need_create_local_exe_scopes") + check_type( + self._compiled_program._executor, + "the Executor of compiled program", + core.ParallelExecutor, + "ParallelExecutor._need_create_local_exe_scopes", + ) return self._compiled_program._executor._need_create_local_exe_scopes() diff --git a/python/paddle/fluid/param_attr.py b/python/paddle/fluid/param_attr.py index b43625759e37a..ba7bead304b95 100644 --- a/python/paddle/fluid/param_attr.py +++ b/python/paddle/fluid/param_attr.py @@ -71,24 +71,31 @@ class ParamAttr(object): paddle.nn.Linear(3, 4, weight_attr=weight_attr) """ - def __init__(self, - name=None, - initializer=None, - learning_rate=1.0, - regularizer=None, - trainable=True, - do_model_average=True, - need_clip=True): + def __init__( + self, + name=None, + initializer=None, + learning_rate=1.0, + regularizer=None, + trainable=True, + do_model_average=True, + need_clip=True, + ): check_type(name, "name", (str, type(None)), "ParamAttr") check_type(learning_rate, "learning_rate", (float, int), "ParamAttr") check_type(trainable, "trainable", (bool), "ParamAttr") check_type(do_model_average, "do_model_average", (bool), "ParamAttr") check_type(need_clip, "need_clip", (bool), "ParamAttr") - check_type(initializer, "initializer", (Initializer, type(None)), - "ParamAttr") - check_type(regularizer, "regularizer", - (WeightDecayRegularizer, type(None)), "ParamAttr") + check_type( + initializer, "initializer", (Initializer, type(None)), "ParamAttr" + ) + check_type( + regularizer, + "regularizer", + (WeightDecayRegularizer, type(None)), + "ParamAttr", + ) self.name = name if self.name == "": @@ -191,13 +198,11 @@ def _to_kwargs(self, with_initializer=False): """ kwargs = { 'name': self.name, - 'optimize_attr': { - 'learning_rate': self.learning_rate - }, + 'optimize_attr': {'learning_rate': self.learning_rate}, 'regularizer': self.regularizer, 'trainable': self.trainable, 'do_model_average': self.do_model_average, - 'need_clip': self.need_clip + 'need_clip': self.need_clip, } if with_initializer: kwargs['initializer'] = self.initializer @@ -277,21 +282,24 @@ class WeightNormParamAttr(ParamAttr): # these paramters for inference. params_with_weight_norm = [] - def __init__(self, - dim=None, - name=None, - initializer=None, - learning_rate=1.0, - regularizer=None, - trainable=True, - do_model_average=False, - need_clip=True): - super(WeightNormParamAttr, - self).__init__(name=name, - initializer=initializer, - learning_rate=learning_rate, - regularizer=regularizer, - trainable=trainable, - do_model_average=do_model_average, - need_clip=need_clip) + def __init__( + self, + dim=None, + name=None, + initializer=None, + learning_rate=1.0, + regularizer=None, + trainable=True, + do_model_average=False, + need_clip=True, + ): + super(WeightNormParamAttr, self).__init__( + name=name, + initializer=initializer, + learning_rate=learning_rate, + regularizer=regularizer, + trainable=trainable, + do_model_average=do_model_average, + need_clip=need_clip, + ) self.dim = dim diff --git a/python/paddle/fluid/profiler.py b/python/paddle/fluid/profiler.py index 827c8b42a5b52..6ebb2f9a08863 100644 --- a/python/paddle/fluid/profiler.py +++ b/python/paddle/fluid/profiler.py @@ -20,8 +20,11 @@ from paddle.utils.deprecated import deprecated __all__ = [ - 'cuda_profiler', 'reset_profiler', 'profiler', 'start_profiler', - 'stop_profiler' + 'cuda_profiler', + 'reset_profiler', + 'profiler', + 'start_profiler', + 'stop_profiler', ] NVPROF_CONFIG = [ @@ -39,8 +42,8 @@ since="2.3.0", update_to="paddle.profiler.Profiler", level=1, - reason= - "Please use new profiler tool, this profiler tool is no longer maintained.") + reason="Please use new profiler tool, this profiler tool is no longer maintained.", +) @signature_safe_contextmanager def cuda_profiler(output_file, output_mode=None, config=None): """ @@ -118,8 +121,8 @@ def npu_profiler(output_file, config=None): since="2.3.0", update_to="paddle.profiler.Profiler", level=1, - reason= - "Please use new profiler tool, this profiler tool is no longer maintained.") + reason="Please use new profiler tool, this profiler tool is no longer maintained.", +) def reset_profiler(): """ Clear the previous time record. It works for @@ -146,8 +149,8 @@ def reset_profiler(): since="2.3.0", update_to="paddle.profiler.Profiler", level=1, - reason= - "Please use new profiler tool, this profiler tool is no longer maintained.") + reason="Please use new profiler tool, this profiler tool is no longer maintained.", +) def start_profiler(state, tracer_option='Default'): """ Enable the profiler. Uers can use `fluid.profiler.start_profiler` and @@ -204,7 +207,8 @@ def start_profiler(state, tracer_option='Default'): if tracer_option not in ['Default', 'OpDetail', 'AllOpDetail']: raise ValueError( - "tracer option must be 'Default', 'OpDetail', 'AllOpDetail'.") + "tracer option must be 'Default', 'OpDetail', 'AllOpDetail'." + ) if tracer_option == "Default": prof_tracer_option = core.TracerOption.kDefault elif tracer_option == "OpDetail": @@ -220,8 +224,8 @@ def start_profiler(state, tracer_option='Default'): since="2.3.0", update_to="paddle.profiler.Profiler", level=1, - reason= - "Please use new profiler tool, this profiler tool is no longer maintained.") + reason="Please use new profiler tool, this profiler tool is no longer maintained.", +) def stop_profiler(sorted_key=None, profile_path='/tmp/profile'): """ Stop the profiler. Uers can use `fluid.profiler.start_profiler` and @@ -264,8 +268,10 @@ def stop_profiler(sorted_key=None, profile_path='/tmp/profile'): return sorted_key = 'default' if sorted_key is None else sorted_key if sorted_key not in ['default', 'calls', 'total', 'max', 'min', 'ave']: - raise ValueError("The sorted_key must be None or in 'calls', 'total', " - "'max', 'min' and 'ave'") + raise ValueError( + "The sorted_key must be None or in 'calls', 'total', " + "'max', 'min' and 'ave'" + ) key_map = { 'default': core.EventSortingKey.kDefault, 'calls': core.EventSortingKey.kCalls, @@ -283,13 +289,12 @@ def stop_profiler(sorted_key=None, profile_path='/tmp/profile'): since="2.3.0", update_to="paddle.profiler.Profiler", level=1, - reason= - "Please use new profiler tool, this profiler tool is no longer maintained.") + reason="Please use new profiler tool, this profiler tool is no longer maintained.", +) @signature_safe_contextmanager -def profiler(state, - sorted_key=None, - profile_path='/tmp/profile', - tracer_option='Default'): +def profiler( + state, sorted_key=None, profile_path='/tmp/profile', tracer_option='Default' +): """ The profiler interface. This profiler can be used to profile both CPU and GPU program. diff --git a/python/paddle/fluid/reader.py b/python/paddle/fluid/reader.py index 07dd03a2bc69e..5c2e92ae458e7 100644 --- a/python/paddle/fluid/reader.py +++ b/python/paddle/fluid/reader.py @@ -21,14 +21,39 @@ import time import copy -from .framework import Program, Variable, program_guard, default_main_program, default_startup_program, _non_static_mode, cpu_places, _current_expected_place, _in_eager_without_dygraph_check +from .framework import ( + Program, + Variable, + program_guard, + default_main_program, + default_startup_program, + _non_static_mode, + cpu_places, + _current_expected_place, + _in_eager_without_dygraph_check, +) from .executor import global_scope from .data_feeder import DataFeeder, BatchedTensorProvider -from .multiprocess_utils import multiprocess_queue_set, CleanupFuncRegistrar, _cleanup_mmap, _cleanup, _set_SIGCHLD_handler +from .multiprocess_utils import ( + multiprocess_queue_set, + CleanupFuncRegistrar, + _cleanup_mmap, + _cleanup, + _set_SIGCHLD_handler, +) from .dataloader import BatchSampler, Dataset, IterableDataset, Subset -from .dataloader.dataloader_iter import _DataLoaderIterSingleProcess, _DataLoaderIterMultiProcess, _DatasetKind, default_collate_fn +from .dataloader.dataloader_iter import ( + _DataLoaderIterSingleProcess, + _DataLoaderIterMultiProcess, + _DatasetKind, + default_collate_fn, +) from .dataloader.batch_sampler import _InfiniteIterableSampler -from .layers.io import monkey_patch_reader_methods, _copy_reader_var_, double_buffer +from .layers.io import ( + monkey_patch_reader_methods, + _copy_reader_var_, + double_buffer, +) from .unique_name import UniqueNameGenerator from .framework import _get_paddle_place, _get_paddle_place_list from paddle.fluid.framework import _set_expected_place, _current_expected_place @@ -39,8 +64,10 @@ import os import multiprocessing import signal + # NOTE: queue has a different name in python2 and python3 import queue + # NOTE: [ avoid hanging & failed quickly ] These value is used in getting data from another process QUEUE_GET_TIMEOUT = 60 @@ -120,7 +147,6 @@ def _reader_process_loop(batch_reader, data_queue): class DataLoaderBase(object): - def __init__(self): self._places = None @@ -151,12 +177,12 @@ def _check_input_array(cls, item): "this means the input data contains nested lists with different lengths. " "\n\t* Check the reader function passed to 'decorate_batch_generator'" " to locate the data causes this issue.\n\t* Please consider using " - "'fluid.create_lod_tensor' to convert it to a LoD-Tensor.") + "'fluid.create_lod_tensor' to convert it to a LoD-Tensor." + ) return arr class AuToTune(object): - def __init__(self, loader): self.loader = loader self.max_num_worker = multiprocessing.cpu_count() / 2 @@ -174,12 +200,14 @@ def __call__(self): # pick the best num_workers auto_tune_start = time.time() logging.debug("========= DataLoader Auto Tune =========") - logging.debug("User config for DataLoader: " + - str(self.loader.num_workers)) + logging.debug( + "User config for DataLoader: " + str(self.loader.num_workers) + ) best_num_workers = 0 min_cost = float("inf") - logging.debug("Tuning Range for num_workers: 0 ~ " + - str(self.max_num_worker)) + logging.debug( + "Tuning Range for num_workers: 0 ~ " + str(self.max_num_worker) + ) num_workers = 0 while num_workers < self.max_num_worker: auto_tune_loader.num_workers = num_workers @@ -188,25 +216,37 @@ def __call__(self): min_cost = avg_cost best_num_workers = num_workers else: - update_num = self.is_best(auto_tune_loader, best_num_workers, - min_cost, self.max_num_worker) + update_num = self.is_best( + auto_tune_loader, + best_num_workers, + min_cost, + self.max_num_worker, + ) if update_num == best_num_workers: break else: best_num_workers = update_num - logging.debug("num_workers: " + str(num_workers) + " avg_cost: " + - str(avg_cost)) + logging.debug( + "num_workers: " + + str(num_workers) + + " avg_cost: " + + str(avg_cost) + ) num_workers += 2 - logging.info("auto_tune dataLoader best_num_workers: " + - str(best_num_workers)) - logging.debug("AutoTuning Cost for DataLoader: " + - str(time.time() - auto_tune_start) + ' seconds') + logging.info( + "auto_tune dataLoader best_num_workers: " + str(best_num_workers) + ) + logging.debug( + "AutoTuning Cost for DataLoader: " + + str(time.time() - auto_tune_start) + + ' seconds' + ) # tune the default loader's num_workers return best_num_workers def need_autotune(self): - if (sys.platform == 'darwin' or sys.platform == 'win32'): + if sys.platform == 'darwin' or sys.platform == 'win32': return False else: return True @@ -219,8 +259,9 @@ def get_sub_dataset(self, dataset, batch_size): def get_autotune_loader(self): loader = copy.copy(self.loader) batch_size = self.loader.batch_sampler.batch_size - if isinstance(self.loader.batch_sampler, - paddle.io.DistributedBatchSampler): + if isinstance( + self.loader.batch_sampler, paddle.io.DistributedBatchSampler + ): dataset = self.loader.batch_sampler.dataset sub_dataset = self.get_sub_dataset(dataset, batch_size) loader.batch_sampler = paddle.io.DistributedBatchSampler( @@ -229,14 +270,16 @@ def get_autotune_loader(self): num_replicas=self.loader.batch_sampler.nranks, rank=self.loader.batch_sampler.local_rank, shuffle=self.loader.batch_sampler.shuffle, - drop_last=self.loader.batch_sampler.drop_last) + drop_last=self.loader.batch_sampler.drop_last, + ) elif isinstance(self.loader.batch_sampler, paddle.io.BatchSampler): dataset = self.loader.batch_sampler.sampler.data_source sub_dataset = self.get_sub_dataset(dataset, batch_size) loader.batch_sampler = paddle.io.BatchSampler( dataset=sub_dataset, batch_size=batch_size, - drop_last=self.loader.batch_sampler.drop_last) + drop_last=self.loader.batch_sampler.drop_last, + ) else: loader = None return loader @@ -261,10 +304,14 @@ def is_best(self, reader, best_workers, best_time, num_work_boundary): while num_workers < num_work_boundary and step < 5: self.loader.num_workers = num_workers time = self.evaluate_reader_cost(reader) - logging.debug("for back num_workers: " + str(num_workers) + - " avg_cost: " + str(time)) + logging.debug( + "for back num_workers: " + + str(num_workers) + + " avg_cost: " + + str(time) + ) step += 1 - if (time < best_time * 0.70 * boundary): + if time < best_time * 0.70 * boundary: return num_workers else: num_workers += 1 @@ -442,23 +489,25 @@ def forward(self, image, label=None): """ - def __init__(self, - dataset, - feed_list=None, - places=None, - return_list=True, - batch_sampler=None, - batch_size=1, - shuffle=False, - drop_last=False, - collate_fn=None, - num_workers=0, - use_buffer_reader=True, - prefetch_factor=2, - use_shared_memory=True, - timeout=0, - worker_init_fn=None, - persistent_workers=False): + def __init__( + self, + dataset, + feed_list=None, + places=None, + return_list=True, + batch_sampler=None, + batch_size=1, + shuffle=False, + drop_last=False, + collate_fn=None, + num_workers=0, + use_buffer_reader=True, + prefetch_factor=2, + use_shared_memory=True, + timeout=0, + worker_init_fn=None, + persistent_workers=False, + ): self.return_list = return_list self.collate_fn = collate_fn self.use_buffer_reader = use_buffer_reader @@ -468,8 +517,9 @@ def __init__(self, self.dataset = dataset if not return_list and not _non_static_mode(): - assert feed_list is not None, \ - "feed_list should be set when return_list=False" + assert ( + feed_list is not None + ), "feed_list should be set when return_list=False" self.feed_list = feed_list if places is None: @@ -481,11 +531,13 @@ def __init__(self, self.places = _convert_places(places) assert num_workers >= 0, "num_workers should be a non-negative value" - if num_workers > 0 and (sys.platform == 'darwin' - or sys.platform == 'win32'): + if num_workers > 0 and ( + sys.platform == 'darwin' or sys.platform == 'win32' + ): warnings.warn( - "DataLoader with multi-process mode is not supported on MacOs and Windows currently." \ - " Please use signle-process mode with num_workers = 0 instead") + "DataLoader with multi-process mode is not supported on MacOs and Windows currently." + " Please use signle-process mode with num_workers = 0 instead" + ) num_workers = 0 self.num_workers = num_workers @@ -502,44 +554,53 @@ def __init__(self, self.dataset_kind = _DatasetKind.ITER if shuffle: raise ValueError( - "IterableDataset not support shuffle, but got shuffle={}". - format(shuffle)) + "IterableDataset not support shuffle, but got shuffle={}".format( + shuffle + ) + ) if batch_sampler is not None: raise ValueError( - "IterableDataset expect unspecified batch_sampler") + "IterableDataset expect unspecified batch_sampler" + ) else: self.dataset_kind = _DatasetKind.MAP if batch_sampler is not None: - assert batch_size == 1 and not shuffle and not drop_last, \ - "batch_size/shuffle/drop_last should not be set when " \ + assert batch_size == 1 and not shuffle and not drop_last, ( + "batch_size/shuffle/drop_last should not be set when " "batch_sampler is given" + ) self.batch_sampler = batch_sampler self.batch_size = None elif batch_size is None: self.batch_sampler = None self.batch_size = None else: - assert batch_size > 0, \ - "batch_size should be None or a positive value when " \ + assert batch_size > 0, ( + "batch_size should be None or a positive value when " "batch_sampler is not given" + ) self.batch_size = batch_size if isinstance(dataset, IterableDataset): self.batch_sampler = _InfiniteIterableSampler( - dataset, batch_size) + dataset, batch_size + ) else: - self.batch_sampler = BatchSampler(dataset=dataset, - batch_size=batch_size, - shuffle=shuffle, - drop_last=drop_last) + self.batch_sampler = BatchSampler( + dataset=dataset, + batch_size=batch_size, + shuffle=shuffle, + drop_last=drop_last, + ) self.drop_last = drop_last self.auto_collate_batch = self.batch_sampler is not None self.pin_memory = False if _non_static_mode(): - self.pin_memory = True if use_pinned_memory( - ) is None else use_pinned_memory() + self.pin_memory = ( + True if use_pinned_memory() is None else use_pinned_memory() + ) self._persistent_workers = persistent_workers self._iterator = None @@ -570,13 +631,15 @@ def __call__(self): return self.__iter__() @staticmethod - def from_generator(feed_list=None, - capacity=None, - use_double_buffer=True, - iterable=True, - return_list=False, - use_multiprocess=False, - drop_last=True): + def from_generator( + feed_list=None, + capacity=None, + use_double_buffer=True, + iterable=True, + return_list=False, + use_multiprocess=False, + drop_last=True, + ): """ .. warning:: This API will be deprecated in the future, it is recommended to use @@ -888,12 +951,23 @@ def run_inference(drop_last): print(run_inference(drop_last=False)) # [1.0, 4.0, 9.0] """ if _non_static_mode(): - return DygraphGeneratorLoader(feed_list, capacity, - use_double_buffer, iterable, - return_list, use_multiprocess) + return DygraphGeneratorLoader( + feed_list, + capacity, + use_double_buffer, + iterable, + return_list, + use_multiprocess, + ) else: - return GeneratorLoader(feed_list, capacity, use_double_buffer, - iterable, return_list, drop_last) + return GeneratorLoader( + feed_list, + capacity, + use_double_buffer, + iterable, + return_list, + drop_last, + ) @staticmethod def from_dataset(dataset, places, drop_last=True): @@ -950,13 +1024,15 @@ class DygraphGeneratorLoader(DataLoaderBase): static graph GeneratorLoader, Separate implementation to keep code readable. """ - def __init__(self, - feed_list=None, - capacity=None, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=False): + def __init__( + self, + feed_list=None, + capacity=None, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=False, + ): self._batch_reader = None self._places = None self._feed_list = feed_list @@ -979,8 +1055,9 @@ def __init__(self, # NOTE: the multiprocessing in different platform is incompatible, we will solve it later self._use_multiprocess = use_multiprocess - if self._use_multiprocess and (sys.platform == 'darwin' - or sys.platform == 'win32'): + if self._use_multiprocess and ( + sys.platform == 'darwin' or sys.platform == 'win32' + ): warnings.warn( "NOTE: DygraphGeneratorLoader with multiprocess mode is not currently supported on MacOs and Windows." ) @@ -999,8 +1076,9 @@ def __init__(self, # mode, this thread is used to get next batch data from self._batch_reader, then # push it into self._blocking_queue self._thread = None - self._pin_memory = True if use_pinned_memory( - ) is None else use_pinned_memory() + self._pin_memory = ( + True if use_pinned_memory() is None else use_pinned_memory() + ) @property def queue(self): @@ -1042,14 +1120,20 @@ def _init_iterable(self): self._dtypes = [] self._need_check_feed = [] self._blocking_queue = core.init_lod_tensor_blocking_queue( - core.Variable(), self._capacity, False) + core.Variable(), self._capacity, False + ) self._reader = None - self._reader = core.create_py_reader(self.queue, self._var_names, - self._shapes, self._dtypes, - self._need_check_feed, - self._places, - self._use_double_buffer, True, - self._pin_memory) + self._reader = core.create_py_reader( + self.queue, + self._var_names, + self._shapes, + self._dtypes, + self._need_check_feed, + self._places, + self._use_double_buffer, + True, + self._pin_memory, + ) def _start(self): if self._use_multiprocess: @@ -1060,9 +1144,10 @@ def _start(self): # add _data_queue into global queue set global multiprocess_queue_set multiprocess_queue_set.add(self._data_queue) - self._process = multiprocessing.Process(target=_reader_process_loop, - args=(self._batch_reader, - self._data_queue)) + self._process = multiprocessing.Process( + target=_reader_process_loop, + args=(self._batch_reader, self._data_queue), + ) self._process.daemon = True self._process.start() @@ -1079,13 +1164,15 @@ def _start(self): self._thread_done_event = threading.Event() self._thread = threading.Thread( target=self._reader_thread_loop_for_multiprocess, - args=(_current_expected_place(), )) + args=(_current_expected_place(),), + ) self._thread.daemon = True self._thread.start() else: self._thread = threading.Thread( target=self._reader_thread_loop_for_singleprocess, - args=(_current_expected_place(), )) + args=(_current_expected_place(),), + ) self._thread.daemon = True self._thread.start() @@ -1097,8 +1184,9 @@ def _reset(self): def __iter__(self): assert self.iterable, "DataLoader is not iterable" - assert self._batch_reader is not None, \ - "Data source of DataLoader has not set yet" + assert ( + self._batch_reader is not None + ), "Data source of DataLoader has not set yet" self._init_iterable() self._start() @@ -1108,7 +1196,8 @@ def __next__(self): try: if _in_eager_without_dygraph_check(): return core.eager.read_next_tensor_list( - self._reader.read_next_list()[0]) + self._reader.read_next_list()[0] + ) else: return self._reader.read_next_var_list() except StopIteration: @@ -1193,23 +1282,22 @@ def _reader_thread_loop_for_singleprocess(self, legacy_expected_place): self._blocking_queue.kill() self._thread = None logging.warning( - "DygraphDataLoader reader thread raised an exception.") + "DygraphDataLoader reader thread raised an exception." + ) six.reraise(*sys.exc_info()) - def set_sample_generator(self, - reader, - batch_size, - drop_last=True, - places=None): + def set_sample_generator( + self, reader, batch_size, drop_last=True, places=None + ): assert batch_size > 0, "batch_size must be larger than 0" if isinstance(places, (list, tuple)): places = _get_paddle_place_list(places) else: places = _get_paddle_place(places) - self.set_sample_list_generator(paddle.batch(reader, - batch_size=batch_size, - drop_last=drop_last), - places=places) + self.set_sample_list_generator( + paddle.batch(reader, batch_size=batch_size, drop_last=drop_last), + places=places, + ) return self def set_sample_list_generator(self, reader, places=None): @@ -1241,20 +1329,22 @@ def set_batch_generator(self, reader, places=None): if places is None: places = _current_expected_place() self._places = _convert_places(places) - assert len(self._places) == 1, \ - "Number of places must be 1 in imperative mode" + assert ( + len(self._places) == 1 + ), "Number of places must be 1 in imperative mode" return self class GeneratorLoader(DataLoaderBase): - - def __init__(self, - feed_list=None, - capacity=None, - use_double_buffer=True, - iterable=True, - return_list=False, - drop_last=True): + def __init__( + self, + feed_list=None, + capacity=None, + use_double_buffer=True, + iterable=True, + return_list=False, + drop_last=True, + ): self._tensor_reader = None self._places = None self._thread = None @@ -1291,14 +1381,20 @@ def _init_iterable(self): v.desc.need_check_feed() for v in self._feed_list ] self._queue = core.init_lod_tensor_blocking_queue( - core.Variable(), self._capacity, self._keep_order) + core.Variable(), self._capacity, self._keep_order + ) self._reader = None - self._reader = core.create_py_reader(self.queue, self._var_names, - self._shapes, self._dtypes, - self._need_check_feed, - self._places, - self._use_double_buffer, - self._drop_last, False) + self._reader = core.create_py_reader( + self.queue, + self._var_names, + self._shapes, + self._dtypes, + self._need_check_feed, + self._places, + self._use_double_buffer, + self._drop_last, + False, + ) def _init_non_iterable(self): lod_levels = [] @@ -1317,13 +1413,15 @@ def _init_non_iterable(self): need_check_feed.append(int(feed_data.desc.need_check_feed())) queue_name = data_loader_unique_name_generator( - 'lod_tensor_blocking_queue') + 'lod_tensor_blocking_queue' + ) reader_name = data_loader_unique_name_generator('create_py_reader') double_buffer_name = data_loader_unique_name_generator('double_buffer') var = global_scope().var(queue_name) self._queue = core.init_lod_tensor_blocking_queue( - var, self._capacity, self._keep_order) + var, self._capacity, self._keep_order + ) if self._keep_order: block = default_main_program().current_block() @@ -1333,16 +1431,18 @@ def _init_non_iterable(self): reader_var = block.create_var(name=reader_name) dtype_int = [int(t) for t in dtypes] - block.append_op(type='create_py_reader', - inputs={'blocking_queue': [queue_name]}, - outputs={'Out': [reader_var]}, - attrs={ - 'shape_concat': shape_concat, - 'lod_levels': lod_levels, - 'dtypes': dtype_int, - 'need_check_feed': need_check_feed, - 'ranks': ranks - }) + block.append_op( + type='create_py_reader', + inputs={'blocking_queue': [queue_name]}, + outputs={'Out': [reader_var]}, + attrs={ + 'shape_concat': shape_concat, + 'lod_levels': lod_levels, + 'dtypes': dtype_int, + 'need_check_feed': need_check_feed, + 'ranks': ranks, + }, + ) reader_var.desc.set_dtypes(dtypes) reader_var.persistable = True @@ -1354,7 +1454,8 @@ def _init_non_iterable(self): reader.reset = self._queue.reset else: main_prog_var = _copy_reader_var_( - default_main_program().current_block(), reader_var) + default_main_program().current_block(), reader_var + ) main_prog_var.stop_gradient = True main_prog_var.persistable = True @@ -1362,8 +1463,9 @@ def _init_non_iterable(self): reader = monkey_patch_reader_methods(main_prog_var) if self._use_double_buffer: - double_buffer_reader = double_buffer(reader, - name=double_buffer_name) + double_buffer_reader = double_buffer( + reader, name=double_buffer_name + ) # we return a double buffer reader. However, the reset method comes from # py_reader. double_buffer_reader.reset = reader.reset @@ -1375,7 +1477,8 @@ def _init_non_iterable(self): type='read', inputs={'Reader': [self._reader]}, outputs={'Out': self._feed_list}, - attrs={'drop_last': self._drop_last}) + attrs={'drop_last': self._drop_last}, + ) @property def queue(self): @@ -1387,8 +1490,9 @@ def iterable(self): def __iter__(self): assert self.iterable, "DataLoader is not iterable" - assert self._tensor_reader is not None, \ - "Data source of DataLoader has not set yet" + assert ( + self._tensor_reader is not None + ), "Data source of DataLoader has not set yet" self._init_iterable() self._start() @@ -1409,15 +1513,18 @@ def __next__(self): six.reraise(*sys.exc_info()) def start(self): - assert not self._iterable, "start() cannot be called when DataLoader is iterable" + assert ( + not self._iterable + ), "start() cannot be called when DataLoader is iterable" self._start() def reset(self): - assert not self._iterable, "reset() cannot be called when DataLoader is iterable" + assert ( + not self._iterable + ), "reset() cannot be called when DataLoader is iterable" self._reset() def _start(self): - def __thread_main__(legacy_expected_place): try: # See _DataLoaderIterSingleProcess._thread_loop() for why set expected place here. @@ -1450,8 +1557,9 @@ def __thread_main__(legacy_expected_place): logging.warning('Your reader has raised an exception!') six.reraise(*sys.exc_info()) - self._thread = threading.Thread(target=__thread_main__, - args=(_current_expected_place(), )) + self._thread = threading.Thread( + target=__thread_main__, args=(_current_expected_place(),) + ) self._thread.daemon = True self._thread.start() @@ -1465,11 +1573,9 @@ def _reset(self): self._exited = False self._reader.reset() - def set_sample_generator(self, - reader, - batch_size, - drop_last=True, - places=None): + def set_sample_generator( + self, reader, batch_size, drop_last=True, places=None + ): assert batch_size > 0, "batch_size must be larger than 0" if isinstance(places, (list, tuple)): places = _get_paddle_place_list(places) @@ -1482,16 +1588,20 @@ def set_sample_generator(self, break if has_lod: - self.set_sample_list_generator(paddle.batch(reader, - batch_size=batch_size, - drop_last=drop_last), - places=places) + self.set_sample_list_generator( + paddle.batch( + reader, batch_size=batch_size, drop_last=drop_last + ), + places=places, + ) else: - reader = BatchedTensorProvider(feed_list=self._feed_list, - place=core.CPUPlace(), - batch_size=batch_size, - generator=reader, - drop_last=drop_last) + reader = BatchedTensorProvider( + feed_list=self._feed_list, + place=core.CPUPlace(), + batch_size=batch_size, + generator=reader, + drop_last=drop_last, + ) self.set_batch_generator(reader, places=places) return self @@ -1501,8 +1611,9 @@ def set_sample_list_generator(self, reader, places=None): else: places = _get_paddle_place(places) with program_guard(Program(), Program()): - feeder = DataFeeder(feed_list=self._feed_list, - place=core.CPUPlace()) + feeder = DataFeeder( + feed_list=self._feed_list, place=core.CPUPlace() + ) paddle_reader = feeder.decorate_reader(reader, multi_devices=False) def __tensor_reader_impl__(): @@ -1519,12 +1630,15 @@ def set_batch_generator(self, reader, places=None): places = _get_paddle_place(places) self._tensor_reader = reader if self._iterable: - assert places is not None, "Places cannot be None when DataLoader is iterable" + assert ( + places is not None + ), "Places cannot be None when DataLoader is iterable" self._places = _convert_places(places) else: if places is not None: logging.info( - 'places would be ommited when DataLoader is not iterable') + 'places would be ommited when DataLoader is not iterable' + ) return self @@ -1696,15 +1810,17 @@ def reader(): relu = fluid.layers.relu(image) """ - def __init__(self, - feed_list=None, - capacity=None, - use_double_buffer=True, - iterable=True, - return_list=False): - self._loader = DataLoader.from_generator(feed_list, capacity, - use_double_buffer, iterable, - return_list) + def __init__( + self, + feed_list=None, + capacity=None, + use_double_buffer=True, + iterable=True, + return_list=False, + ): + self._loader = DataLoader.from_generator( + feed_list, capacity, use_double_buffer, iterable, return_list + ) @property def queue(self): @@ -1725,8 +1841,8 @@ def start(self): Start the data feeding thread. Can only call when the reader object is not iterable. - Example: - .. code-block:: python + Example: + .. code-block:: python import paddle import paddle.fluid as fluid @@ -1754,7 +1870,7 @@ def generator(): reader.reset() break - ''' + ''' self._loader.start() def reset(self): @@ -1794,11 +1910,9 @@ def generator(): ''' self._loader.reset() - def decorate_sample_generator(self, - sample_generator, - batch_size, - drop_last=True, - places=None): + def decorate_sample_generator( + self, sample_generator, batch_size, drop_last=True, places=None + ): ''' Set the data source of the PyReader object. @@ -1861,8 +1975,9 @@ def generator(): executor.run(feed=data, fetch_list=[loss]) ''' - self._loader.set_sample_generator(sample_generator, batch_size, - drop_last, places) + self._loader.set_sample_generator( + sample_generator, batch_size, drop_last, places + ) def decorate_sample_list_generator(self, reader, places=None): ''' @@ -1987,11 +2102,12 @@ def generator(): class DatasetLoader(DataLoaderBase): - def __init__(self, dataset, places, drop_last): - assert isinstance(dataset, paddle.distributed.fleet.dataset.DatasetBase - ), "dataset must be type of DatasetBase" - assert not _non_static_mode( + assert isinstance( + dataset, paddle.distributed.fleet.dataset.DatasetBase + ), "dataset must be type of DatasetBase" + assert ( + not _non_static_mode() ), "DatasetLoader is not supported in dygraph mode yet" if isinstance(places, (list, tuple)): places = _get_paddle_place_list(places) @@ -2000,32 +2116,48 @@ def __init__(self, dataset, places, drop_last): thread_num = len(places) - assert len(dataset.filelist) >= thread_num, \ - "Filelist number of dataset {} must be not less than place number {}".format(len(dataset.filelist), thread_num) + assert ( + len(dataset.filelist) >= thread_num + ), "Filelist number of dataset {} must be not less than place number {}".format( + len(dataset.filelist), thread_num + ) if dataset.thread_num != 0 and dataset.thread_num != thread_num: logging.warn( 'thread_num {} which is set in Dataset is ignored'.format( - dataset.thread_num)) + dataset.thread_num + ) + ) dataset._set_thread(thread_num) - if isinstance(dataset, paddle.distributed.fleet.dataset.InMemoryDataset - ) and dataset.queue_num > thread_num: + if ( + isinstance( + dataset, paddle.distributed.fleet.dataset.InMemoryDataset + ) + and dataset.queue_num > thread_num + ): logging.warn( "queue_num {} which is set in Dataset is ignored".format( - dataset.queue_num)) + dataset.queue_num + ) + ) dataset._set_queue_num(thread_num) self._dataset = dataset use_slots = [ - slot.name for slot in dataset.proto_desc.multi_slot_desc.slots + slot.name + for slot in dataset.proto_desc.multi_slot_desc.slots if slot.is_used ] self._iterable_dataset = core.IterableDatasetWrapper( - dataset.dataset, use_slots, _convert_places(places), - dataset.proto_desc.batch_size, drop_last) + dataset.dataset, + use_slots, + _convert_places(places), + dataset.proto_desc.batch_size, + drop_last, + ) def __iter__(self): self._dataset._finish_to_run() diff --git a/python/paddle/fluid/regularizer.py b/python/paddle/fluid/regularizer.py index ebf3230984eb7..00240a1d3f4dd 100644 --- a/python/paddle/fluid/regularizer.py +++ b/python/paddle/fluid/regularizer.py @@ -37,13 +37,11 @@ def __init__(self): pass def __call__(self, param, grad, block): - """Add corresponding weight decay operations to the network - """ + """Add corresponding weight decay operations to the network""" raise NotImplementedError() def __str__(self): - """Debug string - """ + """Debug string""" raise NotImplementedError() @@ -134,21 +132,25 @@ def __call__(self, param, grad, block): if framework._non_static_mode(): if framework.in_dygraph_mode(): - return _C_ops.scale(param, self._regularization_coeff, 0.0, - True) + return _C_ops.scale( + param, self._regularization_coeff, 0.0, True + ) else: - return _legacy_C_ops.scale(param, "scale", - self._regularization_coeff) + return _legacy_C_ops.scale( + param, "scale", self._regularization_coeff + ) else: - decay = block.create_var(dtype=param.dtype, - shape=param.shape, - lod_level=param.lod_level) + decay = block.create_var( + dtype=param.dtype, shape=param.shape, lod_level=param.lod_level + ) # Append Op to calculate decay - block.append_op(type='scale', - inputs={"X": param}, - outputs={"Out": decay}, - attrs={"scale": self._regularization_coeff}) + block.append_op( + type='scale', + inputs={"X": param}, + outputs={"Out": decay}, + attrs={"scale": self._regularization_coeff}, + ) return decay @@ -245,12 +247,12 @@ def __call__(self, param, grad, block): sign = block.create_var(dtype=param.dtype, shape=param.shape) decay = block.create_var(dtype=param.dtype, shape=param.shape) else: - sign = block.create_var(dtype=param.dtype, - shape=param.shape, - lod_level=param.lod_level) - decay = block.create_var(dtype=param.dtype, - shape=param.shape, - lod_level=param.lod_level) + sign = block.create_var( + dtype=param.dtype, shape=param.shape, lod_level=param.lod_level + ) + decay = block.create_var( + dtype=param.dtype, shape=param.shape, lod_level=param.lod_level + ) if in_dygraph_mode(): sign = _C_ops.sign(param) return _C_ops.scale(sign, self._regularization_coeff, 0.0, True) @@ -259,10 +261,12 @@ def __call__(self, param, grad, block): block.append_op(type='sign', inputs={"X": param}, outputs={"Out": sign}) # Append scale op to the output of sign op - block.append_op(type='scale', - inputs={"X": sign}, - outputs={"Out": decay}, - attrs={"scale": self._regularization_coeff}) + block.append_op( + type='scale', + inputs={"X": sign}, + outputs={"Out": decay}, + attrs={"scale": self._regularization_coeff}, + ) return decay diff --git a/python/paddle/fluid/tests/book/notest_understand_sentiment.py b/python/paddle/fluid/tests/book/notest_understand_sentiment.py index f8e49fafac4d9..02a17b9f92c99 100644 --- a/python/paddle/fluid/tests/book/notest_understand_sentiment.py +++ b/python/paddle/fluid/tests/book/notest_understand_sentiment.py @@ -22,43 +22,41 @@ import os -def convolution_net(data, - label, - input_dim, - class_dim=2, - emb_dim=32, - hid_dim=32): - emb = fluid.layers.embedding(input=data, - size=[input_dim, emb_dim], - is_sparse=True) - conv_3 = fluid.nets.sequence_conv_pool(input=emb, - num_filters=hid_dim, - filter_size=3, - act="tanh", - pool_type="sqrt") - conv_4 = fluid.nets.sequence_conv_pool(input=emb, - num_filters=hid_dim, - filter_size=4, - act="tanh", - pool_type="sqrt") - prediction = fluid.layers.fc(input=[conv_3, conv_4], - size=class_dim, - act="softmax") +def convolution_net( + data, label, input_dim, class_dim=2, emb_dim=32, hid_dim=32 +): + emb = fluid.layers.embedding( + input=data, size=[input_dim, emb_dim], is_sparse=True + ) + conv_3 = fluid.nets.sequence_conv_pool( + input=emb, + num_filters=hid_dim, + filter_size=3, + act="tanh", + pool_type="sqrt", + ) + conv_4 = fluid.nets.sequence_conv_pool( + input=emb, + num_filters=hid_dim, + filter_size=4, + act="tanh", + pool_type="sqrt", + ) + prediction = fluid.layers.fc( + input=[conv_3, conv_4], size=class_dim, act="softmax" + ) cost = fluid.layers.cross_entropy(input=prediction, label=label) avg_cost = paddle.mean(cost) accuracy = fluid.layers.accuracy(input=prediction, label=label) return avg_cost, accuracy, prediction -def dyn_rnn_lstm(data, - label, - input_dim, - class_dim=2, - emb_dim=32, - lstm_size=128): - emb = fluid.layers.embedding(input=data, - size=[input_dim, emb_dim], - is_sparse=True) +def dyn_rnn_lstm( + data, label, input_dim, class_dim=2, emb_dim=32, lstm_size=128 +): + emb = fluid.layers.embedding( + input=data, size=[input_dim, emb_dim], is_sparse=True + ) sentence = fluid.layers.fc(input=emb, size=lstm_size, act='tanh') rnn = fluid.layers.DynamicRNN() @@ -73,13 +71,17 @@ def gate_common(ipt, hidden, size): return gate0 + gate1 forget_gate = fluid.layers.sigmoid( - x=gate_common(word, prev_hidden, lstm_size)) + x=gate_common(word, prev_hidden, lstm_size) + ) input_gate = fluid.layers.sigmoid( - x=gate_common(word, prev_hidden, lstm_size)) + x=gate_common(word, prev_hidden, lstm_size) + ) output_gate = fluid.layers.sigmoid( - x=gate_common(word, prev_hidden, lstm_size)) + x=gate_common(word, prev_hidden, lstm_size) + ) cell_gate = fluid.layers.sigmoid( - x=gate_common(word, prev_hidden, lstm_size)) + x=gate_common(word, prev_hidden, lstm_size) + ) cell = forget_gate * prev_cell + input_gate * cell_gate hidden = output_gate * fluid.layers.tanh(x=cell) @@ -95,18 +97,14 @@ def gate_common(ipt, hidden, size): return avg_cost, accuracy, prediction -def stacked_lstm_net(data, - label, - input_dim, - class_dim=2, - emb_dim=128, - hid_dim=512, - stacked_num=3): +def stacked_lstm_net( + data, label, input_dim, class_dim=2, emb_dim=128, hid_dim=512, stacked_num=3 +): assert stacked_num % 2 == 1 - emb = fluid.layers.embedding(input=data, - size=[input_dim, emb_dim], - is_sparse=True) + emb = fluid.layers.embedding( + input=data, size=[input_dim, emb_dim], is_sparse=True + ) # add bias attr # TODO(qijun) linear act @@ -117,54 +115,57 @@ def stacked_lstm_net(data, for i in range(2, stacked_num + 1): fc = fluid.layers.fc(input=inputs, size=hid_dim) - lstm, cell = fluid.layers.dynamic_lstm(input=fc, - size=hid_dim, - is_reverse=(i % 2) == 0) + lstm, cell = fluid.layers.dynamic_lstm( + input=fc, size=hid_dim, is_reverse=(i % 2) == 0 + ) inputs = [fc, lstm] fc_last = fluid.layers.sequence_pool(input=inputs[0], pool_type='max') lstm_last = fluid.layers.sequence_pool(input=inputs[1], pool_type='max') - prediction = fluid.layers.fc(input=[fc_last, lstm_last], - size=class_dim, - act='softmax') + prediction = fluid.layers.fc( + input=[fc_last, lstm_last], size=class_dim, act='softmax' + ) cost = fluid.layers.cross_entropy(input=prediction, label=label) avg_cost = paddle.mean(cost) accuracy = fluid.layers.accuracy(input=prediction, label=label) return avg_cost, accuracy, prediction -def train(word_dict, - net_method, - use_cuda, - parallel=False, - save_dirname=None, - is_local=True): +def train( + word_dict, + net_method, + use_cuda, + parallel=False, + save_dirname=None, + is_local=True, +): BATCH_SIZE = 128 PASS_NUM = 5 dict_dim = len(word_dict) class_dim = 2 - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") if not parallel: - cost, acc_out, prediction = net_method(data, - label, - input_dim=dict_dim, - class_dim=class_dim) + cost, acc_out, prediction = net_method( + data, label, input_dim=dict_dim, class_dim=class_dim + ) else: raise NotImplementedError() adagrad = fluid.optimizer.Adagrad(learning_rate=0.002) adagrad.minimize(cost) - train_data = paddle.batch(paddle.reader.shuffle( - paddle.dataset.imdb.train(word_dict), buf_size=1000), - batch_size=BATCH_SIZE) + train_data = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.imdb.train(word_dict), buf_size=1000 + ), + batch_size=BATCH_SIZE, + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[data, label], place=place) @@ -174,19 +175,23 @@ def train_loop(main_program): for pass_id in range(PASS_NUM): for data in train_data(): - cost_val, acc_val = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[cost, acc_out]) + cost_val, acc_val = exe.run( + main_program, + feed=feeder.feed(data), + fetch_list=[cost, acc_out], + ) print("cost=" + str(cost_val) + " acc=" + str(acc_val)) if cost_val < 0.4 and acc_val > 0.8: if save_dirname is not None: - fluid.io.save_inference_model(save_dirname, ["words"], - prediction, exe) + fluid.io.save_inference_model( + save_dirname, ["words"], prediction, exe + ) return if math.isnan(float(cost_val)): sys.exit("got NaN loss, training failed.") - raise AssertionError("Cost is too large for {0}".format( - net_method.__name__)) + raise AssertionError( + "Cost is too large for {0}".format(net_method.__name__) + ) if is_local: train_loop(fluid.default_main_program()) @@ -205,8 +210,9 @@ def train_loop(main_program): t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -226,8 +232,11 @@ def infer(word_dict, use_cuda, save_dirname=None): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(save_dirname, exe) word_dict_len = len(word_dict) @@ -244,19 +253,19 @@ def infer(word_dict, use_cuda, save_dirname=None): recursive_seq_lens = [[3, 4, 2]] base_shape = [1] # The range of random integers is [low, high] - tensor_words = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=word_dict_len - 1) + tensor_words = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=word_dict_len - 1 + ) # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. assert feed_target_names[0] == "words" - results = exe.run(inference_program, - feed={feed_target_names[0]: tensor_words}, - fetch_list=fetch_targets, - return_numpy=False) + results = exe.run( + inference_program, + feed={feed_target_names[0]: tensor_words}, + fetch_list=fetch_targets, + return_numpy=False, + ) print(results[0].recursive_sequence_lengths()) np_data = np.array(results[0]) print("Inference Shape: ", np_data.shape) @@ -267,16 +276,17 @@ def main(word_dict, net_method, use_cuda, parallel=False, save_dirname=None): if use_cuda and not fluid.core.is_compiled_with_cuda(): return - train(word_dict, - net_method, - use_cuda, - parallel=parallel, - save_dirname=save_dirname) + train( + word_dict, + net_method, + use_cuda, + parallel=parallel, + save_dirname=save_dirname, + ) infer(word_dict, use_cuda, save_dirname) class TestUnderstandSentiment(unittest.TestCase): - @classmethod def setUpClass(cls): cls.word_dict = paddle.dataset.imdb.word_dict() @@ -292,17 +302,21 @@ def new_program_scope(self): def test_conv_cpu(self): with self.new_program_scope(): - main(self.word_dict, - net_method=convolution_net, - use_cuda=False, - save_dirname="understand_sentiment_conv.inference.model") + main( + self.word_dict, + net_method=convolution_net, + use_cuda=False, + save_dirname="understand_sentiment_conv.inference.model", + ) def test_conv_cpu_parallel(self): with self.new_program_scope(): - main(self.word_dict, - net_method=convolution_net, - use_cuda=False, - parallel=True) + main( + self.word_dict, + net_method=convolution_net, + use_cuda=False, + parallel=True, + ) @unittest.skip(reason="make CI faster") def test_stacked_lstm_cpu(self): @@ -311,29 +325,35 @@ def test_stacked_lstm_cpu(self): self.word_dict, net_method=stacked_lstm_net, use_cuda=False, - save_dirname="understand_sentiment_stacked_lstm.inference.model" + save_dirname="understand_sentiment_stacked_lstm.inference.model", ) def test_stacked_lstm_cpu_parallel(self): with self.new_program_scope(): - main(self.word_dict, - net_method=stacked_lstm_net, - use_cuda=False, - parallel=True) + main( + self.word_dict, + net_method=stacked_lstm_net, + use_cuda=False, + parallel=True, + ) def test_conv_gpu(self): with self.new_program_scope(): - main(self.word_dict, - net_method=convolution_net, - use_cuda=True, - save_dirname="understand_sentiment_conv.inference.model") + main( + self.word_dict, + net_method=convolution_net, + use_cuda=True, + save_dirname="understand_sentiment_conv.inference.model", + ) def test_conv_gpu_parallel(self): with self.new_program_scope(): - main(self.word_dict, - net_method=convolution_net, - use_cuda=True, - parallel=True) + main( + self.word_dict, + net_method=convolution_net, + use_cuda=True, + parallel=True, + ) @unittest.skip(reason="make CI faster") def test_stacked_lstm_gpu(self): @@ -342,30 +362,36 @@ def test_stacked_lstm_gpu(self): self.word_dict, net_method=stacked_lstm_net, use_cuda=True, - save_dirname="understand_sentiment_stacked_lstm.inference.model" + save_dirname="understand_sentiment_stacked_lstm.inference.model", ) def test_stacked_lstm_gpu_parallel(self): with self.new_program_scope(): - main(self.word_dict, - net_method=stacked_lstm_net, - use_cuda=True, - parallel=True) + main( + self.word_dict, + net_method=stacked_lstm_net, + use_cuda=True, + parallel=True, + ) @unittest.skip(reason='make CI faster') def test_dynrnn_lstm_gpu(self): with self.new_program_scope(): - main(self.word_dict, - net_method=dyn_rnn_lstm, - use_cuda=True, - parallel=False) + main( + self.word_dict, + net_method=dyn_rnn_lstm, + use_cuda=True, + parallel=False, + ) def test_dynrnn_lstm_gpu_parallel(self): with self.new_program_scope(): - main(self.word_dict, - net_method=dyn_rnn_lstm, - use_cuda=True, - parallel=True) + main( + self.word_dict, + net_method=dyn_rnn_lstm, + use_cuda=True, + parallel=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/book/test_fit_a_line.py b/python/paddle/fluid/tests/book/test_fit_a_line.py index 788e8d12d836a..338008c48d576 100644 --- a/python/paddle/fluid/tests/book/test_fit_a_line.py +++ b/python/paddle/fluid/tests/book/test_fit_a_line.py @@ -32,7 +32,8 @@ def convert_uint16_to_float(in_list): in_list = numpy.asarray(in_list) out = numpy.vectorize( lambda x: struct.unpack('> 16)) + numpy.uint16(struct.unpack('> 16) + ) out = numpy.reshape(out, in_list.shape).view(numpy.uint16) return out @@ -73,15 +75,18 @@ def train(use_cuda, save_dirname, is_local, use_bf16, pure_bf16): sgd_optimizer, amp_lists=amp.bf16.AutoMixedPrecisionListsBF16(), use_bf16_guard=False, - use_pure_bf16=pure_bf16) - sgd_optimizer.minimize(avg_cost, - startup_program=fluid.default_startup_program()) + use_pure_bf16=pure_bf16, + ) + sgd_optimizer.minimize( + avg_cost, startup_program=fluid.default_startup_program() + ) BATCH_SIZE = 20 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.uci_housing.train(), buf_size=500), - batch_size=BATCH_SIZE) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.uci_housing.train(), buf_size=500), + batch_size=BATCH_SIZE, + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) @@ -91,29 +96,33 @@ def train_loop(main_program): exe.run(fluid.default_startup_program()) test_prog = main_program.clone(for_test=True) if pure_bf16: - sgd_optimizer.amp_init(exe.place, - test_program=test_prog, - use_bf16_test=True) + sgd_optimizer.amp_init( + exe.place, test_program=test_prog, use_bf16_test=True + ) PASS_NUM = 100 for pass_id in range(PASS_NUM): for data in train_reader(): - avg_loss_value, = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[avg_cost]) + (avg_loss_value,) = exe.run( + main_program, feed=feeder.feed(data), fetch_list=[avg_cost] + ) if avg_loss_value.dtype == numpy.uint16: avg_loss_value = convert_uint16_to_float(avg_loss_value) if avg_loss_value[0] < 10.0: if save_dirname is not None: - paddle.static.save_inference_model(save_dirname, [x], - [y_predict], - exe, - clip_extra=False) + paddle.static.save_inference_model( + save_dirname, + [x], + [y_predict], + exe, + clip_extra=False, + ) return if math.isnan(float(avg_loss_value)): sys.exit("got NaN loss, training failed.") - raise AssertionError("Fit a line cost is too large, {0:2.2}".format( - avg_loss_value[0])) + raise AssertionError( + "Fit a line cost is too large, {0:2.2}".format(avg_loss_value[0]) + ) if is_local: train_loop(fluid.default_main_program()) @@ -132,8 +141,9 @@ def train_loop(main_program): t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -153,30 +163,38 @@ def infer(use_cuda, save_dirname=None, use_bf16=False): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = paddle.static.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(save_dirname, exe) # The input's dimension should be 2-D and the second dim is 13 # The input data should be >= 0 batch_size = 10 - test_reader = paddle.batch(paddle.dataset.uci_housing.test(), - batch_size=batch_size) + test_reader = paddle.batch( + paddle.dataset.uci_housing.test(), batch_size=batch_size + ) test_data = next(test_reader()) - test_feat = numpy.array([data[0] - for data in test_data]).astype("float32") + test_feat = numpy.array([data[0] for data in test_data]).astype( + "float32" + ) if use_bf16: test_feat = convert_float_to_uint16(test_feat) - test_label = numpy.array([data[1] - for data in test_data]).astype("float32") + test_label = numpy.array([data[1] for data in test_data]).astype( + "float32" + ) assert feed_target_names[0] == 'x' - results = exe.run(inference_program, - feed={feed_target_names[0]: numpy.array(test_feat)}, - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed={feed_target_names[0]: numpy.array(test_feat)}, + fetch_list=fetch_targets, + ) if results[0].dtype == numpy.uint16: results[0] = convert_uint16_to_float(results[0]) print("infer shape: ", results[0].shape) @@ -201,7 +219,6 @@ def main(use_cuda, is_local=True, use_bf16=False, pure_bf16=False): class TestFitALineBase(unittest.TestCase): - @contextlib.contextmanager def program_scope_guard(self): prog = fluid.Program() @@ -213,7 +230,6 @@ def program_scope_guard(self): class TestFitALine(TestFitALineBase): - def test_cpu(self): with self.program_scope_guard(): main(use_cuda=False) @@ -223,10 +239,10 @@ def test_cuda(self): main(use_cuda=True) -@unittest.skipIf(not fluid.core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not fluid.core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestFitALineBF16(TestFitALineBase): - def test_bf16(self): with self.program_scope_guard(): main(use_cuda=False, use_bf16=True) diff --git a/python/paddle/fluid/tests/book/test_image_classification.py b/python/paddle/fluid/tests/book/test_image_classification.py index c8a4861370a65..ac4dbaec22682 100644 --- a/python/paddle/fluid/tests/book/test_image_classification.py +++ b/python/paddle/fluid/tests/book/test_image_classification.py @@ -26,21 +26,18 @@ def resnet_cifar10(input, depth=32): - - def conv_bn_layer(input, - ch_out, - filter_size, - stride, - padding, - act='relu', - bias_attr=False): - tmp = fluid.layers.conv2d(input=input, - filter_size=filter_size, - num_filters=ch_out, - stride=stride, - padding=padding, - act=None, - bias_attr=bias_attr) + def conv_bn_layer( + input, ch_out, filter_size, stride, padding, act='relu', bias_attr=False + ): + tmp = fluid.layers.conv2d( + input=input, + filter_size=filter_size, + num_filters=ch_out, + stride=stride, + padding=padding, + act=None, + bias_attr=bias_attr, + ) return fluid.layers.batch_norm(input=tmp, act=act) def shortcut(input, ch_in, ch_out, stride): @@ -63,33 +60,31 @@ def layer_warp(block_func, input, ch_in, ch_out, count, stride): assert (depth - 2) % 6 == 0 n = (depth - 2) // 6 - conv1 = conv_bn_layer(input=input, - ch_out=16, - filter_size=3, - stride=1, - padding=1) + conv1 = conv_bn_layer( + input=input, ch_out=16, filter_size=3, stride=1, padding=1 + ) res1 = layer_warp(basicblock, conv1, 16, 16, n, 1) res2 = layer_warp(basicblock, res1, 16, 32, n, 2) res3 = layer_warp(basicblock, res2, 32, 64, n, 2) - pool = fluid.layers.pool2d(input=res3, - pool_size=8, - pool_type='avg', - pool_stride=1) + pool = fluid.layers.pool2d( + input=res3, pool_size=8, pool_type='avg', pool_stride=1 + ) return pool def vgg16_bn_drop(input): - def conv_block(input, num_filter, groups, dropouts): - return fluid.nets.img_conv_group(input=input, - pool_size=2, - pool_stride=2, - conv_num_filter=[num_filter] * groups, - conv_filter_size=3, - conv_act='relu', - conv_with_batchnorm=True, - conv_batchnorm_drop_rate=dropouts, - pool_type='max') + return fluid.nets.img_conv_group( + input=input, + pool_size=2, + pool_stride=2, + conv_num_filter=[num_filter] * groups, + conv_filter_size=3, + conv_act='relu', + conv_with_batchnorm=True, + conv_batchnorm_drop_rate=dropouts, + pool_type='max', + ) conv1 = conv_block(input, 64, 2, [0.3, 0]) conv2 = conv_block(conv1, 128, 2, [0.4, 0]) @@ -135,12 +130,16 @@ def train(net_type, use_cuda, save_dirname, is_local): BATCH_SIZE = 128 PASS_NUM = 1 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.cifar.train10(), buf_size=128 * 10), - batch_size=BATCH_SIZE) + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.cifar.train10(), buf_size=128 * 10 + ), + batch_size=BATCH_SIZE, + ) - test_reader = paddle.batch(paddle.dataset.cifar.test10(), - batch_size=BATCH_SIZE) + test_reader = paddle.batch( + paddle.dataset.cifar.test10(), batch_size=BATCH_SIZE + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) @@ -157,9 +156,11 @@ def train_loop(main_program): acc_list = [] avg_loss_list = [] for tid, test_data in enumerate(test_reader()): - loss_t, acc_t = exe.run(program=test_program, - feed=feeder.feed(test_data), - fetch_list=[avg_cost, acc]) + loss_t, acc_t = exe.run( + program=test_program, + feed=feeder.feed(test_data), + fetch_list=[avg_cost, acc], + ) if math.isnan(float(loss_t)): sys.exit("got NaN loss, training failed.") acc_list.append(float(acc_t)) @@ -170,13 +171,18 @@ def train_loop(main_program): avg_loss_value = numpy.array(avg_loss_list).mean() print( - 'PassID {0:1}, BatchID {1:04}, Test Loss {2:2.2}, Acc {3:2.2}' - .format(pass_id, batch_id + 1, float(avg_loss_value), - float(acc_value))) + 'PassID {0:1}, BatchID {1:04}, Test Loss {2:2.2}, Acc {3:2.2}'.format( + pass_id, + batch_id + 1, + float(avg_loss_value), + float(acc_value), + ) + ) if acc_value > 0.01: # Low threshold for speeding up CI - fluid.io.save_inference_model(save_dirname, ["pixel"], - [predict], exe) + fluid.io.save_inference_model( + save_dirname, ["pixel"], [predict], exe + ) return if is_local: @@ -196,8 +202,9 @@ def train_loop(main_program): t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -217,8 +224,11 @@ def infer(use_cuda, save_dirname=None): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(save_dirname, exe) # The input's dimension of conv should be 4-D or 5-D. # Use normilized image pixels as input data, which should be in the range [0, 1.0]. @@ -227,14 +237,21 @@ def infer(use_cuda, save_dirname=None): # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. - results = exe.run(inference_program, - feed={feed_target_names[0]: tensor_img}, - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed={feed_target_names[0]: tensor_img}, + fetch_list=fetch_targets, + ) print("infer results: ", results[0]) - fluid.io.save_inference_model(save_dirname, feed_target_names, - fetch_targets, exe, inference_program) + fluid.io.save_inference_model( + save_dirname, + feed_target_names, + fetch_targets, + exe, + inference_program, + ) def main(net_type, use_cuda, is_local=True): @@ -244,7 +261,8 @@ def main(net_type, use_cuda, is_local=True): # Directory for saving the trained model temp_dir = tempfile.TemporaryDirectory() save_dirname = os.path.join( - temp_dir.name, "image_classification_" + net_type + ".inference.model") + temp_dir.name, "image_classification_" + net_type + ".inference.model" + ) train(net_type, use_cuda, save_dirname, is_local) infer(use_cuda, save_dirname) @@ -252,7 +270,6 @@ def main(net_type, use_cuda, is_local=True): class TestImageClassification(unittest.TestCase): - def test_vgg_cuda(self): with self.scope_prog_guard(): main('vgg', use_cuda=True) diff --git a/python/paddle/fluid/tests/book/test_label_semantic_roles.py b/python/paddle/fluid/tests/book/test_label_semantic_roles.py index 16e4a2827779d..69585b64c8951 100644 --- a/python/paddle/fluid/tests/book/test_label_semantic_roles.py +++ b/python/paddle/fluid/tests/book/test_label_semantic_roles.py @@ -50,26 +50,32 @@ def load_parameter(file_name, h, w): return np.fromfile(f, dtype=np.float32).reshape(h, w) -def db_lstm(word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, - **ignored): +def db_lstm( + word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, **ignored +): # 8 features - predicate_embedding = fluid.layers.embedding(input=predicate, - size=[pred_dict_len, word_dim], - dtype='float32', - is_sparse=IS_SPARSE, - param_attr='vemb') - - mark_embedding = fluid.layers.embedding(input=mark, - size=[mark_dict_len, mark_dim], - dtype='float32', - is_sparse=IS_SPARSE) + predicate_embedding = fluid.layers.embedding( + input=predicate, + size=[pred_dict_len, word_dim], + dtype='float32', + is_sparse=IS_SPARSE, + param_attr='vemb', + ) + + mark_embedding = fluid.layers.embedding( + input=mark, + size=[mark_dict_len, mark_dim], + dtype='float32', + is_sparse=IS_SPARSE, + ) word_input = [word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2] emb_layers = [ - fluid.layers.embedding(size=[word_dict_len, word_dim], - input=x, - param_attr=fluid.ParamAttr(name=embedding_name, - trainable=False)) + fluid.layers.embedding( + size=[word_dict_len, word_dim], + input=x, + param_attr=fluid.ParamAttr(name=embedding_name, trainable=False), + ) for x in word_input ] emb_layers.append(predicate_embedding) @@ -81,141 +87,172 @@ def db_lstm(word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, hidden_0 = fluid.layers.sums(input=hidden_0_layers) - lstm_0 = fluid.layers.dynamic_lstm(input=hidden_0, - size=hidden_dim, - candidate_activation='relu', - gate_activation='sigmoid', - cell_activation='sigmoid') + lstm_0 = fluid.layers.dynamic_lstm( + input=hidden_0, + size=hidden_dim, + candidate_activation='relu', + gate_activation='sigmoid', + cell_activation='sigmoid', + ) # stack L-LSTM and R-LSTM with direct edges input_tmp = [hidden_0, lstm_0] for i in range(1, depth): - mix_hidden = fluid.layers.sums(input=[ - fluid.layers.fc(input=input_tmp[0], size=hidden_dim), - fluid.layers.fc(input=input_tmp[1], size=hidden_dim) - ]) - - lstm = fluid.layers.dynamic_lstm(input=mix_hidden, - size=hidden_dim, - candidate_activation='relu', - gate_activation='sigmoid', - cell_activation='sigmoid', - is_reverse=((i % 2) == 1)) + mix_hidden = fluid.layers.sums( + input=[ + fluid.layers.fc(input=input_tmp[0], size=hidden_dim), + fluid.layers.fc(input=input_tmp[1], size=hidden_dim), + ] + ) + + lstm = fluid.layers.dynamic_lstm( + input=mix_hidden, + size=hidden_dim, + candidate_activation='relu', + gate_activation='sigmoid', + cell_activation='sigmoid', + is_reverse=((i % 2) == 1), + ) input_tmp = [mix_hidden, lstm] - feature_out = fluid.layers.sums(input=[ - fluid.layers.fc(input=input_tmp[0], size=label_dict_len, act='tanh'), - fluid.layers.fc(input=input_tmp[1], size=label_dict_len, act='tanh') - ]) + feature_out = fluid.layers.sums( + input=[ + fluid.layers.fc( + input=input_tmp[0], size=label_dict_len, act='tanh' + ), + fluid.layers.fc( + input=input_tmp[1], size=label_dict_len, act='tanh' + ), + ] + ) return feature_out def train(use_cuda, save_dirname=None, is_local=True): # define network topology - word = fluid.layers.data(name='word_data', - shape=[1], - dtype='int64', - lod_level=1) - predicate = fluid.layers.data(name='verb_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_n2 = fluid.layers.data(name='ctx_n2_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_n1 = fluid.layers.data(name='ctx_n1_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_0 = fluid.layers.data(name='ctx_0_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_p1 = fluid.layers.data(name='ctx_p1_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_p2 = fluid.layers.data(name='ctx_p2_data', - shape=[1], - dtype='int64', - lod_level=1) - mark = fluid.layers.data(name='mark_data', - shape=[1], - dtype='int64', - lod_level=1) + word = fluid.layers.data( + name='word_data', shape=[1], dtype='int64', lod_level=1 + ) + predicate = fluid.layers.data( + name='verb_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_n2 = fluid.layers.data( + name='ctx_n2_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_n1 = fluid.layers.data( + name='ctx_n1_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_0 = fluid.layers.data( + name='ctx_0_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_p1 = fluid.layers.data( + name='ctx_p1_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_p2 = fluid.layers.data( + name='ctx_p2_data', shape=[1], dtype='int64', lod_level=1 + ) + mark = fluid.layers.data( + name='mark_data', shape=[1], dtype='int64', lod_level=1 + ) feature_out = db_lstm(**locals()) - target = fluid.layers.data(name='target', - shape=[1], - dtype='int64', - lod_level=1) - crf_cost = fluid.layers.linear_chain_crf(input=feature_out, - label=target, - param_attr=fluid.ParamAttr( - name='crfw', - learning_rate=mix_hidden_lr)) + target = fluid.layers.data( + name='target', shape=[1], dtype='int64', lod_level=1 + ) + crf_cost = fluid.layers.linear_chain_crf( + input=feature_out, + label=target, + param_attr=fluid.ParamAttr(name='crfw', learning_rate=mix_hidden_lr), + ) avg_cost = paddle.mean(crf_cost) # TODO(qiao) # check other optimizers and check why out will be NAN sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.exponential_decay(learning_rate=0.01, - decay_steps=100000, - decay_rate=0.5, - staircase=True)) + learning_rate=fluid.layers.exponential_decay( + learning_rate=0.01, + decay_steps=100000, + decay_rate=0.5, + staircase=True, + ) + ) sgd_optimizer.minimize(avg_cost) # TODO(qiao) # add dependency track and move this config before optimizer crf_decode = fluid.layers.crf_decoding( - input=feature_out, param_attr=fluid.ParamAttr(name='crfw')) + input=feature_out, param_attr=fluid.ParamAttr(name='crfw') + ) - train_data = paddle.batch(paddle.reader.shuffle( - paddle.dataset.conll05.test(), buf_size=8192), - batch_size=BATCH_SIZE) + train_data = paddle.batch( + paddle.reader.shuffle(paddle.dataset.conll05.test(), buf_size=8192), + batch_size=BATCH_SIZE, + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - feeder = fluid.DataFeeder(feed_list=[ - word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, predicate, mark, target - ], - place=place) + feeder = fluid.DataFeeder( + feed_list=[ + word, + ctx_n2, + ctx_n1, + ctx_0, + ctx_p1, + ctx_p2, + predicate, + mark, + target, + ], + place=place, + ) exe = fluid.Executor(place) def train_loop(main_program): exe.run(fluid.default_startup_program()) - embedding_param = fluid.global_scope().find_var( - embedding_name).get_tensor() + embedding_param = ( + fluid.global_scope().find_var(embedding_name).get_tensor() + ) embedding_param.set( load_parameter(conll05.get_embedding(), word_dict_len, word_dim), - place) + place, + ) start_time = time.time() batch_id = 0 for pass_id in range(PASS_NUM): for data in train_data(): - cost = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[avg_cost]) + cost = exe.run( + main_program, feed=feeder.feed(data), fetch_list=[avg_cost] + ) cost = cost[0] if batch_id % 10 == 0: print("avg_cost:" + str(cost)) if batch_id != 0: - print("second per batch: " + - str((time.time() - start_time) / batch_id)) + print( + "second per batch: " + + str((time.time() - start_time) / batch_id) + ) # Set the threshold low to speed up the CI test if float(cost) < 80.0: if save_dirname is not None: # TODO(liuyiqun): Change the target to crf_decode fluid.io.save_inference_model( - save_dirname, [ - 'word_data', 'verb_data', 'ctx_n2_data', - 'ctx_n1_data', 'ctx_0_data', 'ctx_p1_data', - 'ctx_p2_data', 'mark_data' - ], [feature_out], exe) + save_dirname, + [ + 'word_data', + 'verb_data', + 'ctx_n2_data', + 'ctx_n1_data', + 'ctx_0_data', + 'ctx_p1_data', + 'ctx_p2_data', + 'mark_data', + ], + [feature_out], + exe, + ) return batch_id = batch_id + 1 @@ -241,8 +278,9 @@ def train_loop(main_program): t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -262,8 +300,11 @@ def infer(use_cuda, save_dirname=None): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(save_dirname, exe) # Setup input by creating LoDTensor to represent sequence of words. # Here each word is the basic element of the LoDTensor and the shape of @@ -278,46 +319,30 @@ def infer(use_cuda, save_dirname=None): recursive_seq_lens = [[3, 4, 2]] base_shape = [1] # The range of random integers is [low, high] - word = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=word_dict_len - 1) - pred = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=pred_dict_len - 1) - ctx_n2 = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=word_dict_len - 1) - ctx_n1 = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=word_dict_len - 1) - ctx_0 = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=word_dict_len - 1) - ctx_p1 = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=word_dict_len - 1) - ctx_p2 = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=word_dict_len - 1) - mark = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=mark_dict_len - 1) + word = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=word_dict_len - 1 + ) + pred = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=pred_dict_len - 1 + ) + ctx_n2 = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=word_dict_len - 1 + ) + ctx_n1 = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=word_dict_len - 1 + ) + ctx_0 = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=word_dict_len - 1 + ) + ctx_p1 = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=word_dict_len - 1 + ) + ctx_p2 = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=word_dict_len - 1 + ) + mark = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=mark_dict_len - 1 + ) # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. @@ -330,19 +355,21 @@ def infer(use_cuda, save_dirname=None): assert feed_target_names[6] == 'ctx_p2_data' assert feed_target_names[7] == 'mark_data' - results = exe.run(inference_program, - feed={ - feed_target_names[0]: word, - feed_target_names[1]: pred, - feed_target_names[2]: ctx_n2, - feed_target_names[3]: ctx_n1, - feed_target_names[4]: ctx_0, - feed_target_names[5]: ctx_p1, - feed_target_names[6]: ctx_p2, - feed_target_names[7]: mark - }, - fetch_list=fetch_targets, - return_numpy=False) + results = exe.run( + inference_program, + feed={ + feed_target_names[0]: word, + feed_target_names[1]: pred, + feed_target_names[2]: ctx_n2, + feed_target_names[3]: ctx_n1, + feed_target_names[4]: ctx_0, + feed_target_names[5]: ctx_p1, + feed_target_names[6]: ctx_p2, + feed_target_names[7]: mark, + }, + fetch_list=fetch_targets, + return_numpy=False, + ) print(results[0].recursive_sequence_lengths()) np_data = np.array(results[0]) print("Inference Shape: ", np_data.shape) @@ -354,8 +381,9 @@ def main(use_cuda, is_local=True): temp_dir = tempfile.TemporaryDirectory() # Directory for saving the trained model - save_dirname = os.path.join(temp_dir.name, - "label_semantic_roles.inference.model") + save_dirname = os.path.join( + temp_dir.name, "label_semantic_roles.inference.model" + ) train(use_cuda, save_dirname, is_local) infer(use_cuda, save_dirname) @@ -364,7 +392,6 @@ def main(use_cuda, is_local=True): class TestLabelSemanticRoles(unittest.TestCase): - def test_cuda(self): with self.scope_prog_guard(): main(use_cuda=True) diff --git a/python/paddle/fluid/tests/book/test_machine_translation.py b/python/paddle/fluid/tests/book/test_machine_translation.py index d97b474d6abc8..4ae6462f02f10 100644 --- a/python/paddle/fluid/tests/book/test_machine_translation.py +++ b/python/paddle/fluid/tests/book/test_machine_translation.py @@ -40,15 +40,16 @@ def encoder(is_sparse): # encoder - src_word_id = pd.data(name="src_word_id", - shape=[1], - dtype='int64', - lod_level=1) - src_embedding = pd.embedding(input=src_word_id, - size=[dict_size, word_dim], - dtype='float32', - is_sparse=is_sparse, - param_attr=fluid.ParamAttr(name='vemb')) + src_word_id = pd.data( + name="src_word_id", shape=[1], dtype='int64', lod_level=1 + ) + src_embedding = pd.embedding( + input=src_word_id, + size=[dict_size, word_dim], + dtype='float32', + is_sparse=is_sparse, + param_attr=fluid.ParamAttr(name='vemb'), + ) fc1 = pd.fc(input=src_embedding, size=hidden_dim * 4, act='tanh') lstm_hidden0, lstm_0 = pd.dynamic_lstm(input=fc1, size=hidden_dim * 4) @@ -58,27 +59,28 @@ def encoder(is_sparse): def decoder_train(context, is_sparse): # decoder - trg_language_word = pd.data(name="target_language_word", - shape=[1], - dtype='int64', - lod_level=1) - trg_embedding = pd.embedding(input=trg_language_word, - size=[dict_size, word_dim], - dtype='float32', - is_sparse=is_sparse, - param_attr=fluid.ParamAttr(name='vemb')) + trg_language_word = pd.data( + name="target_language_word", shape=[1], dtype='int64', lod_level=1 + ) + trg_embedding = pd.embedding( + input=trg_language_word, + size=[dict_size, word_dim], + dtype='float32', + is_sparse=is_sparse, + param_attr=fluid.ParamAttr(name='vemb'), + ) rnn = pd.DynamicRNN() with rnn.block(): current_word = rnn.step_input(trg_embedding) pre_state = rnn.memory(init=context) - current_state = pd.fc(input=[current_word, pre_state], - size=decoder_size, - act='tanh') + current_state = pd.fc( + input=[current_word, pre_state], size=decoder_size, act='tanh' + ) - current_score = pd.fc(input=current_state, - size=target_dict_dim, - act='softmax') + current_score = pd.fc( + input=current_state, size=target_dict_dim, act='softmax' + ) rnn.update_memory(pre_state, current_state) rnn.output(current_score) @@ -99,10 +101,9 @@ def decoder_decode(context, is_sparse): scores_array = pd.create_array('float32') init_ids = pd.data(name="init_ids", shape=[1], dtype="int64", lod_level=2) - init_scores = pd.data(name="init_scores", - shape=[1], - dtype="float32", - lod_level=2) + init_scores = pd.data( + name="init_scores", shape=[1], dtype="float32", lod_level=2 + ) pd.array_write(init_ids, array=ids_array, i=counter) pd.array_write(init_scores, array=scores_array, i=counter) @@ -118,32 +119,38 @@ def decoder_decode(context, is_sparse): # expand the recursive_sequence_lengths of pre_state to be the same with pre_score pre_state_expanded = pd.sequence_expand(pre_state, pre_score) - pre_ids_emb = pd.embedding(input=pre_ids, - size=[dict_size, word_dim], - dtype='float32', - is_sparse=is_sparse) + pre_ids_emb = pd.embedding( + input=pre_ids, + size=[dict_size, word_dim], + dtype='float32', + is_sparse=is_sparse, + ) # use rnn unit to update rnn - current_state = pd.fc(input=[pre_state_expanded, pre_ids_emb], - size=decoder_size, - act='tanh') + current_state = pd.fc( + input=[pre_state_expanded, pre_ids_emb], + size=decoder_size, + act='tanh', + ) current_state_with_lod = pd.lod_reset(x=current_state, y=pre_score) # use score to do beam search - current_score = pd.fc(input=current_state_with_lod, - size=target_dict_dim, - act='softmax') + current_score = pd.fc( + input=current_state_with_lod, size=target_dict_dim, act='softmax' + ) topk_scores, topk_indices = pd.topk(current_score, k=beam_size) # calculate accumulated scores after topk to reduce computation cost - accu_scores = pd.elementwise_add(x=pd.log(topk_scores), - y=pd.reshape(pre_score, shape=[-1]), - axis=0) - selected_ids, selected_scores = pd.beam_search(pre_ids, - pre_score, - topk_indices, - accu_scores, - beam_size, - end_id=10, - level=0) + accu_scores = pd.elementwise_add( + x=pd.log(topk_scores), y=pd.reshape(pre_score, shape=[-1]), axis=0 + ) + selected_ids, selected_scores = pd.beam_search( + pre_ids, + pre_score, + topk_indices, + accu_scores, + beam_size, + end_id=10, + level=0, + ) pd.increment(x=counter, value=1, in_place=True) @@ -159,7 +166,8 @@ def decoder_decode(context, is_sparse): pd.logical_and(x=length_cond, y=finish_cond, out=cond) translation_ids, translation_scores = pd.beam_search_decode( - ids=ids_array, scores=scores_array, beam_size=beam_size, end_id=10) + ids=ids_array, scores=scores_array, beam_size=beam_size, end_id=10 + ) # return init_ids, init_scores @@ -173,25 +181,31 @@ def train_main(use_cuda, is_sparse, is_local=True): context = encoder(is_sparse) rnn_out = decoder_train(context, is_sparse) - label = pd.data(name="target_language_next_word", - shape=[1], - dtype='int64', - lod_level=1) + label = pd.data( + name="target_language_next_word", shape=[1], dtype='int64', lod_level=1 + ) cost = pd.cross_entropy(input=rnn_out, label=label) avg_cost = pd.mean(cost) optimizer = fluid.optimizer.Adagrad( learning_rate=1e-4, regularization=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ), + ) optimizer.minimize(avg_cost) - train_data = paddle.batch(paddle.reader.shuffle( - paddle.dataset.wmt14.train(dict_size), buf_size=1000), - batch_size=batch_size) + train_data = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.wmt14.train(dict_size), buf_size=1000 + ), + batch_size=batch_size, + ) feed_order = [ - 'src_word_id', 'target_language_word', 'target_language_next_word' + 'src_word_id', + 'target_language_word', + 'target_language_next_word', ] exe = Executor(place) @@ -207,12 +221,18 @@ def train_loop(main_program): batch_id = 0 for pass_id in range(1): for data in train_data(): - outs = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[avg_cost]) + outs = exe.run( + main_program, feed=feeder.feed(data), fetch_list=[avg_cost] + ) avg_cost_val = np.array(outs[0]) - print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) + - " avg_cost=" + str(avg_cost_val)) + print( + 'pass_id=' + + str(pass_id) + + ' batch=' + + str(batch_id) + + " avg_cost=" + + str(avg_cost_val) + ) if batch_id > 3: break batch_id += 1 @@ -234,8 +254,9 @@ def train_loop(main_program): t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -254,21 +275,27 @@ def decode_main(use_cuda, is_sparse): exe.run(framework.default_startup_program()) init_ids_data = np.array([1 for _ in range(batch_size)], dtype='int64') - init_scores_data = np.array([1. for _ in range(batch_size)], - dtype='float32') + init_scores_data = np.array( + [1.0 for _ in range(batch_size)], dtype='float32' + ) init_ids_data = init_ids_data.reshape((batch_size, 1)) init_scores_data = init_scores_data.reshape((batch_size, 1)) init_recursive_seq_lens = [1] * batch_size init_recursive_seq_lens = [init_recursive_seq_lens, init_recursive_seq_lens] - init_ids = fluid.create_lod_tensor(init_ids_data, init_recursive_seq_lens, - place) - init_scores = fluid.create_lod_tensor(init_scores_data, - init_recursive_seq_lens, place) + init_ids = fluid.create_lod_tensor( + init_ids_data, init_recursive_seq_lens, place + ) + init_scores = fluid.create_lod_tensor( + init_scores_data, init_recursive_seq_lens, place + ) - train_data = paddle.batch(paddle.reader.shuffle( - paddle.dataset.wmt14.train(dict_size), buf_size=1000), - batch_size=batch_size) + train_data = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.wmt14.train(dict_size), buf_size=1000 + ), + batch_size=batch_size, + ) feed_order = ['src_word_id'] feed_list = [ @@ -286,7 +313,8 @@ def decode_main(use_cuda, is_sparse): framework.default_main_program(), feed=feed_dict, fetch_list=[translation_ids, translation_scores], - return_numpy=False) + return_numpy=False, + ) print(result_ids.recursive_sequence_lengths()) break @@ -306,8 +334,9 @@ def scope_prog_guard(): def inject_test_train(use_cuda, is_sparse): - f_name = 'test_{0}_{1}_train'.format('cuda' if use_cuda else 'cpu', - 'sparse' if is_sparse else 'dense') + f_name = 'test_{0}_{1}_train'.format( + 'cuda' if use_cuda else 'cpu', 'sparse' if is_sparse else 'dense' + ) def f(*args): with scope_prog_guard(): @@ -317,8 +346,9 @@ def f(*args): def inject_test_decode(use_cuda, is_sparse, decorator=None): - f_name = 'test_{0}_{1}_decode'.format('cuda' if use_cuda else 'cpu', - 'sparse' if is_sparse else 'dense') + f_name = 'test_{0}_{1}_decode'.format( + 'cuda' if use_cuda else 'cpu', 'sparse' if is_sparse else 'dense' + ) def f(*args): with scope_prog_guard(): @@ -340,11 +370,12 @@ def f(*args): _decorator_ = None if _use_cuda_: _decorator_ = unittest.skip( - reason='Beam Search does not support CUDA!') + reason='Beam Search does not support CUDA!' + ) - inject_test_decode(is_sparse=_is_sparse_, - use_cuda=_use_cuda_, - decorator=_decorator_) + inject_test_decode( + is_sparse=_is_sparse_, use_cuda=_use_cuda_, decorator=_decorator_ + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/book/test_recognize_digits.py b/python/paddle/fluid/tests/book/test_recognize_digits.py index f2afcdefa0626..fd99eb04d99fc 100644 --- a/python/paddle/fluid/tests/book/test_recognize_digits.py +++ b/python/paddle/fluid/tests/book/test_recognize_digits.py @@ -43,30 +43,36 @@ def mlp(img, label): def conv_net(img, label): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) return loss_net(conv_pool_2, label) -def train(nn_type, - use_cuda, - parallel, - save_dirname=None, - save_full_dirname=None, - model_filename=None, - params_filename=None, - is_local=True): +def train( + nn_type, + use_cuda, + parallel, + save_dirname=None, + save_full_dirname=None, + model_filename=None, + params_filename=None, + is_local=True, +): if use_cuda and not fluid.core.is_compiled_with_cuda(): return img = fluid.layers.data(name='img', shape=[1, 28, 28], dtype='float32') @@ -91,11 +97,13 @@ def train(nn_type, exe = fluid.Executor(place) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=BATCH_SIZE) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=BATCH_SIZE) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=BATCH_SIZE, + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=BATCH_SIZE + ) feeder = fluid.DataFeeder(feed_list=[img, label], place=place) def train_loop(main_program): @@ -113,7 +121,8 @@ def train_loop(main_program): acc_np, avg_loss_np = exe.run( program=test_program, feed=feeder.feed(test_data), - fetch_list=[acc, avg_loss]) + fetch_list=[acc, avg_loss], + ) acc_set.append(float(acc_np)) avg_loss_set.append(float(avg_loss_np)) # get test acc and loss @@ -123,23 +132,33 @@ def train_loop(main_program): # Smaller value to increase CI speed if save_dirname is not None: fluid.io.save_inference_model( - save_dirname, ["img"], [prediction], + save_dirname, + ["img"], + [prediction], exe, model_filename=model_filename, - params_filename=params_filename) + params_filename=params_filename, + ) if save_full_dirname is not None: fluid.io.save_inference_model( - save_full_dirname, [], [], + save_full_dirname, + [], + [], exe, model_filename=model_filename, params_filename=params_filename, - export_for_deployment=False) + export_for_deployment=False, + ) return else: print( - 'PassID {0:1}, BatchID {1:04}, Test Loss {2:2.2}, Acc {3:2.2}' - .format(pass_id, batch_id + 1, float(avg_loss_val), - float(acc_val))) + 'PassID {0:1}, BatchID {1:04}, Test Loss {2:2.2}, Acc {3:2.2}'.format( + pass_id, + batch_id + 1, + float(avg_loss_val), + float(acc_val), + ) + ) if math.isnan(float(avg_loss_val)): sys.exit("got NaN loss, training failed.") raise AssertionError("Loss of recognize digits is too large") @@ -161,18 +180,18 @@ def train_loop(main_program): t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": train_loop(t.get_trainer_program()) -def infer(use_cuda, - save_dirname=None, - model_filename=None, - params_filename=None): +def infer( + use_cuda, save_dirname=None, model_filename=None, params_filename=None +): if save_dirname is None: return @@ -185,22 +204,28 @@ def infer(use_cuda, # the feed_target_names (the names of variables that will be feeded # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe, - model_filename, - params_filename) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + save_dirname, exe, model_filename, params_filename + ) # The input's dimension of conv should be 4-D or 5-D. # Use normilized image pixels as input data, which should be in the range [-1.0, 1.0]. batch_size = 1 tensor_img = numpy.random.uniform( - -1.0, 1.0, [batch_size, 1, 28, 28]).astype("float32") + -1.0, 1.0, [batch_size, 1, 28, 28] + ).astype("float32") # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. - results = exe.run(inference_program, - feed={feed_target_names[0]: tensor_img}, - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed={feed_target_names[0]: tensor_img}, + fetch_list=fetch_targets, + ) print("infer results: ", results[0]) @@ -217,17 +242,21 @@ def main(use_cuda, parallel, nn_type, combine): params_filename = "__params_combined__" # call train() with is_local argument to run distributed train - train(nn_type=nn_type, - use_cuda=use_cuda, - parallel=parallel, - save_dirname=save_dirname, - save_full_dirname=save_full_dirname, - model_filename=model_filename, - params_filename=params_filename) - infer(use_cuda=use_cuda, - save_dirname=save_dirname, - model_filename=model_filename, - params_filename=params_filename) + train( + nn_type=nn_type, + use_cuda=use_cuda, + parallel=parallel, + save_dirname=save_dirname, + save_full_dirname=save_full_dirname, + model_filename=model_filename, + params_filename=params_filename, + ) + infer( + use_cuda=use_cuda, + save_dirname=save_dirname, + model_filename=model_filename, + params_filename=params_filename, + ) class TestRecognizeDigits(unittest.TestCase): @@ -235,7 +264,6 @@ class TestRecognizeDigits(unittest.TestCase): def inject_test_method(use_cuda, parallel, nn_type, combine): - def __impl__(self): prog = fluid.Program() startup_prog = fluid.Program() @@ -244,9 +272,12 @@ def __impl__(self): with fluid.program_guard(prog, startup_prog): main(use_cuda, parallel, nn_type, combine) - fn = 'test_{0}_{1}_{2}_{3}'.format(nn_type, 'cuda' if use_cuda else 'cpu', - 'parallel' if parallel else 'normal', - 'combine' if combine else 'separate') + fn = 'test_{0}_{1}_{2}_{3}'.format( + nn_type, + 'cuda' if use_cuda else 'cpu', + 'parallel' if parallel else 'normal', + 'combine' if combine else 'separate', + ) setattr(TestRecognizeDigits, fn, __impl__) @@ -255,7 +286,7 @@ def inject_all_tests(): for use_cuda in (False, True): if use_cuda and not core.is_compiled_with_cuda(): continue - for parallel in (False, ): + for parallel in (False,): for nn_type in ('mlp', 'conv'): inject_test_method(use_cuda, parallel, nn_type, True) diff --git a/python/paddle/fluid/tests/book/test_recommender_system.py b/python/paddle/fluid/tests/book/test_recommender_system.py index 3d9e14a7a1869..132ee1b5214e2 100644 --- a/python/paddle/fluid/tests/book/test_recommender_system.py +++ b/python/paddle/fluid/tests/book/test_recommender_system.py @@ -40,11 +40,13 @@ def get_usr_combined_features(): uid = layers.data(name='user_id', shape=[1], dtype='int64') - usr_emb = layers.embedding(input=uid, - dtype='float32', - size=[USR_DICT_SIZE, 32], - param_attr='user_table', - is_sparse=IS_SPARSE) + usr_emb = layers.embedding( + input=uid, + dtype='float32', + size=[USR_DICT_SIZE, 32], + param_attr='user_table', + is_sparse=IS_SPARSE, + ) usr_fc = layers.fc(input=usr_emb, size=32) @@ -52,35 +54,42 @@ def get_usr_combined_features(): usr_gender_id = layers.data(name='gender_id', shape=[1], dtype='int64') - usr_gender_emb = layers.embedding(input=usr_gender_id, - size=[USR_GENDER_DICT_SIZE, 16], - param_attr='gender_table', - is_sparse=IS_SPARSE) + usr_gender_emb = layers.embedding( + input=usr_gender_id, + size=[USR_GENDER_DICT_SIZE, 16], + param_attr='gender_table', + is_sparse=IS_SPARSE, + ) usr_gender_fc = layers.fc(input=usr_gender_emb, size=16) USR_AGE_DICT_SIZE = len(paddle.dataset.movielens.age_table) usr_age_id = layers.data(name='age_id', shape=[1], dtype="int64") - usr_age_emb = layers.embedding(input=usr_age_id, - size=[USR_AGE_DICT_SIZE, 16], - is_sparse=IS_SPARSE, - param_attr='age_table') + usr_age_emb = layers.embedding( + input=usr_age_id, + size=[USR_AGE_DICT_SIZE, 16], + is_sparse=IS_SPARSE, + param_attr='age_table', + ) usr_age_fc = layers.fc(input=usr_age_emb, size=16) USR_JOB_DICT_SIZE = paddle.dataset.movielens.max_job_id() + 1 usr_job_id = layers.data(name='job_id', shape=[1], dtype="int64") - usr_job_emb = layers.embedding(input=usr_job_id, - size=[USR_JOB_DICT_SIZE, 16], - param_attr='job_table', - is_sparse=IS_SPARSE) + usr_job_emb = layers.embedding( + input=usr_job_id, + size=[USR_JOB_DICT_SIZE, 16], + param_attr='job_table', + is_sparse=IS_SPARSE, + ) usr_job_fc = layers.fc(input=usr_job_emb, size=16) concat_embed = layers.concat( - input=[usr_fc, usr_gender_fc, usr_age_fc, usr_job_fc], axis=1) + input=[usr_fc, usr_gender_fc, usr_age_fc, usr_job_fc], axis=1 + ) usr_combined_features = layers.fc(input=concat_embed, size=200, act="tanh") @@ -93,47 +102,51 @@ def get_mov_combined_features(): mov_id = layers.data(name='movie_id', shape=[1], dtype='int64') - mov_emb = layers.embedding(input=mov_id, - dtype='float32', - size=[MOV_DICT_SIZE, 32], - param_attr='movie_table', - is_sparse=IS_SPARSE) + mov_emb = layers.embedding( + input=mov_id, + dtype='float32', + size=[MOV_DICT_SIZE, 32], + param_attr='movie_table', + is_sparse=IS_SPARSE, + ) mov_fc = layers.fc(input=mov_emb, size=32) CATEGORY_DICT_SIZE = len(paddle.dataset.movielens.movie_categories()) - category_id = layers.data(name='category_id', - shape=[1], - dtype='int64', - lod_level=1) + category_id = layers.data( + name='category_id', shape=[1], dtype='int64', lod_level=1 + ) - mov_categories_emb = layers.embedding(input=category_id, - size=[CATEGORY_DICT_SIZE, 32], - is_sparse=IS_SPARSE) + mov_categories_emb = layers.embedding( + input=category_id, size=[CATEGORY_DICT_SIZE, 32], is_sparse=IS_SPARSE + ) - mov_categories_hidden = layers.sequence_pool(input=mov_categories_emb, - pool_type="sum") + mov_categories_hidden = layers.sequence_pool( + input=mov_categories_emb, pool_type="sum" + ) MOV_TITLE_DICT_SIZE = len(paddle.dataset.movielens.get_movie_title_dict()) - mov_title_id = layers.data(name='movie_title', - shape=[1], - dtype='int64', - lod_level=1) + mov_title_id = layers.data( + name='movie_title', shape=[1], dtype='int64', lod_level=1 + ) - mov_title_emb = layers.embedding(input=mov_title_id, - size=[MOV_TITLE_DICT_SIZE, 32], - is_sparse=IS_SPARSE) + mov_title_emb = layers.embedding( + input=mov_title_id, size=[MOV_TITLE_DICT_SIZE, 32], is_sparse=IS_SPARSE + ) - mov_title_conv = nets.sequence_conv_pool(input=mov_title_emb, - num_filters=32, - filter_size=3, - act="tanh", - pool_type="sum") + mov_title_conv = nets.sequence_conv_pool( + input=mov_title_emb, + num_filters=32, + filter_size=3, + act="tanh", + pool_type="sum", + ) concat_embed = layers.concat( - input=[mov_fc, mov_categories_hidden, mov_title_conv], axis=1) + input=[mov_fc, mov_categories_hidden, mov_title_conv], axis=1 + ) # FIXME(dzh) : need tanh operator mov_combined_features = layers.fc(input=concat_embed, size=200, act="tanh") @@ -169,15 +182,23 @@ def train(use_cuda, save_dirname, is_local=True): exe = Executor(place) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.movielens.train(), buf_size=8192), - batch_size=BATCH_SIZE) - test_reader = paddle.batch(paddle.dataset.movielens.test(), - batch_size=BATCH_SIZE) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.movielens.train(), buf_size=8192), + batch_size=BATCH_SIZE, + ) + test_reader = paddle.batch( + paddle.dataset.movielens.test(), batch_size=BATCH_SIZE + ) feed_order = [ - 'user_id', 'gender_id', 'age_id', 'job_id', 'movie_id', 'category_id', - 'movie_title', 'score' + 'user_id', + 'gender_id', + 'age_id', + 'job_id', + 'movie_id', + 'category_id', + 'movie_title', + 'score', ] def train_loop(main_program): @@ -192,16 +213,20 @@ def train_loop(main_program): for pass_id in range(PASS_NUM): for batch_id, data in enumerate(train_reader()): # train a mini-batch - outs = exe.run(program=main_program, - feed=feeder.feed(data), - fetch_list=[avg_cost]) + outs = exe.run( + program=main_program, + feed=feeder.feed(data), + fetch_list=[avg_cost], + ) out = np.array(outs[0]) if (batch_id + 1) % 10 == 0: avg_cost_set = [] for test_data in test_reader(): - avg_cost_np = exe.run(program=test_program, - feed=feeder.feed(test_data), - fetch_list=[avg_cost]) + avg_cost_np = exe.run( + program=test_program, + feed=feeder.feed(test_data), + fetch_list=[avg_cost], + ) avg_cost_set.append(avg_cost_np[0]) break # test only 1 segment for speeding up CI @@ -211,10 +236,19 @@ def train_loop(main_program): # if avg_cost less than 6.0, we think our code is good. if save_dirname is not None: fluid.io.save_inference_model( - save_dirname, [ - "user_id", "gender_id", "age_id", "job_id", - "movie_id", "category_id", "movie_title" - ], [scale_infer], exe) + save_dirname, + [ + "user_id", + "gender_id", + "age_id", + "job_id", + "movie_id", + "category_id", + "movie_title", + ], + [scale_infer], + exe, + ) return if math.isnan(float(out[0])): @@ -237,8 +271,9 @@ def train_loop(main_program): t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -258,8 +293,11 @@ def infer(use_cuda, save_dirname=None): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(save_dirname, exe) # Use the first data from paddle.dataset.movielens.test() as input assert feed_target_names[0] == "user_id" @@ -288,27 +326,32 @@ def infer(use_cuda, save_dirname=None): assert feed_target_names[5] == "category_id" category_id = fluid.create_lod_tensor( - [np.array([10, 8, 9], dtype='int64')], [[3]], place) + [np.array([10, 8, 9], dtype='int64')], [[3]], place + ) assert feed_target_names[6] == "movie_title" movie_title = fluid.create_lod_tensor( - [np.array([1069, 4140, 2923, 710, 988], dtype='int64')], [[5]], - place) + [np.array([1069, 4140, 2923, 710, 988], dtype='int64')], + [[5]], + place, + ) # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. - results = exe.run(inference_program, - feed={ - feed_target_names[0]: user_id, - feed_target_names[1]: gender_id, - feed_target_names[2]: age_id, - feed_target_names[3]: job_id, - feed_target_names[4]: movie_id, - feed_target_names[5]: category_id, - feed_target_names[6]: movie_title - }, - fetch_list=fetch_targets, - return_numpy=False) + results = exe.run( + inference_program, + feed={ + feed_target_names[0]: user_id, + feed_target_names[1]: gender_id, + feed_target_names[2]: age_id, + feed_target_names[3]: job_id, + feed_target_names[4]: movie_id, + feed_target_names[5]: category_id, + feed_target_names[6]: movie_title, + }, + fetch_list=fetch_targets, + return_numpy=False, + ) print("inferred score: ", np.array(results[0])) @@ -318,8 +361,9 @@ def main(use_cuda): # Directory for saving the inference model temp_dir = tempfile.TemporaryDirectory() - save_dirname = os.path.join(temp_dir.name, - "recommender_system.inference.model") + save_dirname = os.path.join( + temp_dir.name, "recommender_system.inference.model" + ) train(use_cuda, save_dirname) infer(use_cuda, save_dirname) diff --git a/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py b/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py index b7de6f5da90cf..f896e24497599 100644 --- a/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py +++ b/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py @@ -41,19 +41,23 @@ def bi_lstm_encoder(input_seq, hidden_size): - input_forward_proj = fluid.layers.fc(input=input_seq, - size=hidden_size * 4, - bias_attr=True) - forward, _ = fluid.layers.dynamic_lstm(input=input_forward_proj, - size=hidden_size * 4, - use_peepholes=USE_PEEPHOLES) - input_backward_proj = fluid.layers.fc(input=input_seq, - size=hidden_size * 4, - bias_attr=True) - backward, _ = fluid.layers.dynamic_lstm(input=input_backward_proj, - size=hidden_size * 4, - is_reverse=True, - use_peepholes=USE_PEEPHOLES) + input_forward_proj = fluid.layers.fc( + input=input_seq, size=hidden_size * 4, bias_attr=True + ) + forward, _ = fluid.layers.dynamic_lstm( + input=input_forward_proj, + size=hidden_size * 4, + use_peepholes=USE_PEEPHOLES, + ) + input_backward_proj = fluid.layers.fc( + input=input_seq, size=hidden_size * 4, bias_attr=True + ) + backward, _ = fluid.layers.dynamic_lstm( + input=input_backward_proj, + size=hidden_size * 4, + is_reverse=True, + use_peepholes=USE_PEEPHOLES, + ) forward_last = fluid.layers.sequence_last_step(input=forward) backward_first = fluid.layers.sequence_first_step(input=backward) @@ -63,7 +67,6 @@ def bi_lstm_encoder(input_seq, hidden_size): # FIXME(peterzhang2029): Replace this function with the lstm_unit_op. def lstm_step(x_t, hidden_t_prev, cell_t_prev, size): - def linear(inputs): return fluid.layers.fc(input=inputs, size=size, bias_attr=True) @@ -72,26 +75,28 @@ def linear(inputs): output_gate = fluid.layers.sigmoid(x=linear([hidden_t_prev, x_t])) cell_tilde = fluid.layers.tanh(x=linear([hidden_t_prev, x_t])) - cell_t = fluid.layers.sums(input=[ - fluid.layers.elementwise_mul(x=forget_gate, y=cell_t_prev), - fluid.layers.elementwise_mul(x=input_gate, y=cell_tilde) - ]) + cell_t = fluid.layers.sums( + input=[ + fluid.layers.elementwise_mul(x=forget_gate, y=cell_t_prev), + fluid.layers.elementwise_mul(x=input_gate, y=cell_tilde), + ] + ) - hidden_t = fluid.layers.elementwise_mul(x=output_gate, - y=fluid.layers.tanh(x=cell_t)) + hidden_t = fluid.layers.elementwise_mul( + x=output_gate, y=fluid.layers.tanh(x=cell_t) + ) return hidden_t, cell_t -def lstm_decoder_without_attention(target_embedding, decoder_boot, context, - decoder_size): +def lstm_decoder_without_attention( + target_embedding, decoder_boot, context, decoder_size +): rnn = fluid.layers.DynamicRNN() cell_init = fluid.layers.fill_constant_batch_size_like( - input=decoder_boot, - value=0.0, - shape=[-1, decoder_size], - dtype='float32') + input=decoder_boot, value=0.0, shape=[-1, decoder_size], dtype='float32' + ) cell_init.stop_gradient = False with rnn.block(): @@ -100,15 +105,15 @@ def lstm_decoder_without_attention(target_embedding, decoder_boot, context, hidden_mem = rnn.memory(init=decoder_boot, need_reorder=True) cell_mem = rnn.memory(init=cell_init) - decoder_inputs = fluid.layers.concat(input=[context, current_word], - axis=1) + decoder_inputs = fluid.layers.concat( + input=[context, current_word], axis=1 + ) h, c = lstm_step(decoder_inputs, hidden_mem, cell_mem, decoder_size) rnn.update_memory(hidden_mem, h) rnn.update_memory(cell_mem, c) - out = fluid.layers.fc(input=h, - size=target_dict_dim, - bias_attr=True, - act='softmax') + out = fluid.layers.fc( + input=h, size=target_dict_dim, bias_attr=True, act='softmax' + ) rnn.output(out) return rnn() @@ -116,43 +121,44 @@ def lstm_decoder_without_attention(target_embedding, decoder_boot, context, def seq_to_seq_net(): """Construct a seq2seq network.""" - src_word_idx = fluid.layers.data(name='source_sequence', - shape=[1], - dtype='int64', - lod_level=1) + src_word_idx = fluid.layers.data( + name='source_sequence', shape=[1], dtype='int64', lod_level=1 + ) src_embedding = fluid.layers.embedding( input=src_word_idx, size=[source_dict_dim, embedding_dim], - dtype='float32') + dtype='float32', + ) src_forward_last, src_backward_first = bi_lstm_encoder( - input_seq=src_embedding, hidden_size=encoder_size) + input_seq=src_embedding, hidden_size=encoder_size + ) encoded_vector = fluid.layers.concat( - input=[src_forward_last, src_backward_first], axis=1) + input=[src_forward_last, src_backward_first], axis=1 + ) - decoder_boot = fluid.layers.fc(input=src_backward_first, - size=decoder_size, - bias_attr=False, - act='tanh') + decoder_boot = fluid.layers.fc( + input=src_backward_first, size=decoder_size, bias_attr=False, act='tanh' + ) - trg_word_idx = fluid.layers.data(name='target_sequence', - shape=[1], - dtype='int64', - lod_level=1) + trg_word_idx = fluid.layers.data( + name='target_sequence', shape=[1], dtype='int64', lod_level=1 + ) trg_embedding = fluid.layers.embedding( input=trg_word_idx, size=[target_dict_dim, embedding_dim], - dtype='float32') - - prediction = lstm_decoder_without_attention(trg_embedding, decoder_boot, - encoded_vector, decoder_size) - label = fluid.layers.data(name='label_sequence', - shape=[1], - dtype='int64', - lod_level=1) + dtype='float32', + ) + + prediction = lstm_decoder_without_attention( + trg_embedding, decoder_boot, encoded_vector, decoder_size + ) + label = fluid.layers.data( + name='label_sequence', shape=[1], dtype='int64', lod_level=1 + ) cost = fluid.layers.cross_entropy(input=prediction, label=label) avg_cost = paddle.mean(cost) @@ -165,9 +171,12 @@ def train(use_cuda, save_dirname=None): optimizer = fluid.optimizer.Adagrad(learning_rate=1e-4) optimizer.minimize(avg_cost) - train_data = paddle.batch(paddle.reader.shuffle( - paddle.dataset.wmt14.train(dict_size), buf_size=1000), - batch_size=batch_size) + train_data = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.wmt14.train(dict_size), buf_size=1000 + ), + batch_size=batch_size, + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = Executor(place) @@ -183,20 +192,31 @@ def train(use_cuda, save_dirname=None): batch_id = 0 for pass_id in range(2): for data in train_data(): - outs = exe.run(framework.default_main_program(), - feed=feeder.feed(data), - fetch_list=[avg_cost]) + outs = exe.run( + framework.default_main_program(), + feed=feeder.feed(data), + fetch_list=[avg_cost], + ) avg_cost_val = np.array(outs[0]) - print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) + - " avg_cost=" + str(avg_cost_val)) + print( + 'pass_id=' + + str(pass_id) + + ' batch=' + + str(batch_id) + + " avg_cost=" + + str(avg_cost_val) + ) if math.isnan(float(avg_cost_val[0])): sys.exit("got NaN loss, training failed.") if batch_id > 3: if save_dirname is not None: fluid.io.save_inference_model( - save_dirname, ['source_sequence', 'target_sequence'], - [prediction], exe) + save_dirname, + ['source_sequence', 'target_sequence'], + [prediction], + exe, + ) return batch_id += 1 @@ -215,8 +235,11 @@ def infer(use_cuda, save_dirname=None): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(save_dirname, exe) # Setup input by creating LoDTensor to represent sequence of words. # Here each word is the basic element of the LoDTensor and the shape of @@ -231,28 +254,26 @@ def infer(use_cuda, save_dirname=None): recursive_seq_lens = [[4, 6]] base_shape = [1] # The range of random integers is [low, high] - word_data = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=1) - trg_word = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=1) + word_data = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=1 + ) + trg_word = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=1 + ) # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. assert feed_target_names[0] == 'source_sequence' assert feed_target_names[1] == 'target_sequence' - results = exe.run(inference_program, - feed={ - feed_target_names[0]: word_data, - feed_target_names[1]: trg_word, - }, - fetch_list=fetch_targets, - return_numpy=False) + results = exe.run( + inference_program, + feed={ + feed_target_names[0]: word_data, + feed_target_names[1]: trg_word, + }, + fetch_list=fetch_targets, + return_numpy=False, + ) print(results[0].recursive_sequence_lengths()) np_data = np.array(results[0]) print("Inference shape: ", np_data.shape) @@ -265,8 +286,9 @@ def main(use_cuda): # Directory for saving the trained model temp_dir = tempfile.TemporaryDirectory() - save_dirname = os.path.join(temp_dir.name, - "rnn_encoder_decoder.inference.model") + save_dirname = os.path.join( + temp_dir.name, "rnn_encoder_decoder.inference.model" + ) train(use_cuda, save_dirname) infer(use_cuda, save_dirname) @@ -274,7 +296,6 @@ def main(use_cuda): class TestRnnEncoderDecoder(unittest.TestCase): - def test_cuda(self): with self.scope_prog_guard(): main(use_cuda=True) diff --git a/python/paddle/fluid/tests/book/test_word2vec_book.py b/python/paddle/fluid/tests/book/test_word2vec_book.py index ec7ab9cc83709..69cef4021c906 100644 --- a/python/paddle/fluid/tests/book/test_word2vec_book.py +++ b/python/paddle/fluid/tests/book/test_word2vec_book.py @@ -33,17 +33,21 @@ def get_place(target): return fluid.CPUPlace() else: raise ValueError( - "Target `{0}` is not on the support list: `cuda`, `xpu` and `cpu`.". - format(target)) - - -def train(target, - is_sparse, - is_parallel, - save_dirname, - is_local=True, - use_bf16=False, - pure_bf16=False): + "Target `{0}` is not on the support list: `cuda`, `xpu` and `cpu`.".format( + target + ) + ) + + +def train( + target, + is_sparse, + is_parallel, + save_dirname, + is_local=True, + use_bf16=False, + pure_bf16=False, +): PASS_NUM = 100 EMBED_SIZE = 32 HIDDEN_SIZE = 256 @@ -52,35 +56,44 @@ def train(target, IS_SPARSE = is_sparse def __network__(words): - embed_first = fluid.layers.embedding(input=words[0], - size=[dict_size, EMBED_SIZE], - dtype='float32', - is_sparse=IS_SPARSE, - param_attr='shared_w') - embed_second = fluid.layers.embedding(input=words[1], - size=[dict_size, EMBED_SIZE], - dtype='float32', - is_sparse=IS_SPARSE, - param_attr='shared_w') - embed_third = fluid.layers.embedding(input=words[2], - size=[dict_size, EMBED_SIZE], - dtype='float32', - is_sparse=IS_SPARSE, - param_attr='shared_w') - embed_forth = fluid.layers.embedding(input=words[3], - size=[dict_size, EMBED_SIZE], - dtype='float32', - is_sparse=IS_SPARSE, - param_attr='shared_w') + embed_first = fluid.layers.embedding( + input=words[0], + size=[dict_size, EMBED_SIZE], + dtype='float32', + is_sparse=IS_SPARSE, + param_attr='shared_w', + ) + embed_second = fluid.layers.embedding( + input=words[1], + size=[dict_size, EMBED_SIZE], + dtype='float32', + is_sparse=IS_SPARSE, + param_attr='shared_w', + ) + embed_third = fluid.layers.embedding( + input=words[2], + size=[dict_size, EMBED_SIZE], + dtype='float32', + is_sparse=IS_SPARSE, + param_attr='shared_w', + ) + embed_forth = fluid.layers.embedding( + input=words[3], + size=[dict_size, EMBED_SIZE], + dtype='float32', + is_sparse=IS_SPARSE, + param_attr='shared_w', + ) concat_embed = fluid.layers.concat( - input=[embed_first, embed_second, embed_third, embed_forth], axis=1) - hidden1 = fluid.layers.fc(input=concat_embed, - size=HIDDEN_SIZE, - act='sigmoid') - predict_word = fluid.layers.fc(input=hidden1, - size=dict_size, - act='softmax') + input=[embed_first, embed_second, embed_third, embed_forth], axis=1 + ) + hidden1 = fluid.layers.fc( + input=concat_embed, size=HIDDEN_SIZE, act='sigmoid' + ) + predict_word = fluid.layers.fc( + input=hidden1, size=dict_size, act='softmax' + ) cost = fluid.layers.cross_entropy(input=predict_word, label=words[4]) avg_cost = paddle.mean(cost) return avg_cost, predict_word @@ -96,7 +109,8 @@ def __network__(words): if not is_parallel: avg_cost, predict_word = __network__( - [first_word, second_word, third_word, forth_word, next_word]) + [first_word, second_word, third_word, forth_word, next_word] + ) else: raise NotImplementedError() @@ -105,20 +119,24 @@ def __network__(words): sgd_optimizer = paddle.static.amp.bf16.decorate_bf16( sgd_optimizer, amp_lists=paddle.static.amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_list={'softmax', 'concat'}, ), + custom_fp32_list={'softmax', 'concat'}, + ), use_bf16_guard=False, - use_pure_bf16=pure_bf16) + use_pure_bf16=pure_bf16, + ) sgd_optimizer.minimize(avg_cost, fluid.default_startup_program()) - train_reader = paddle.batch(paddle.dataset.imikolov.train(word_dict, N), - BATCH_SIZE) + train_reader = paddle.batch( + paddle.dataset.imikolov.train(word_dict, N), BATCH_SIZE + ) place = get_place(target) exe = fluid.Executor(place) feeder = fluid.DataFeeder( feed_list=[first_word, second_word, third_word, forth_word, next_word], - place=place) + place=place, + ) def train_loop(main_program): exe.run(fluid.default_startup_program()) @@ -127,15 +145,17 @@ def train_loop(main_program): for pass_id in range(PASS_NUM): for data in train_reader(): - avg_cost_np = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[avg_cost]) + avg_cost_np = exe.run( + main_program, feed=feeder.feed(data), fetch_list=[avg_cost] + ) if avg_cost_np[0] < 5.0: if save_dirname is not None and not pure_bf16: fluid.io.save_inference_model( save_dirname, ['firstw', 'secondw', 'thirdw', 'forthw'], - [predict_word], exe) + [predict_word], + exe, + ) return if math.isnan(float(avg_cost_np[0])): sys.exit("got NaN loss, training failed.") @@ -159,8 +179,9 @@ def train_loop(main_program): t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers) if training_role == "PSERVER": pserver_prog = t.get_pserver_program(current_endpoint) - pserver_startup = t.get_startup_program(current_endpoint, - pserver_prog) + pserver_startup = t.get_startup_program( + current_endpoint, pserver_prog + ) exe.run(pserver_startup) exe.run(pserver_prog) elif training_role == "TRAINER": @@ -179,8 +200,11 @@ def infer(target, save_dirname=None): # the feed_target_names (the names of variables that will be fed # data using feed operators), and the fetch_targets (variables that # we want to obtain data from using fetch operators). - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(save_dirname, exe) word_dict = paddle.dataset.imikolov.build_dict() dict_size = len(word_dict) @@ -195,26 +219,18 @@ def infer(target, save_dirname=None): recursive_seq_lens = [[1]] base_shape = [1] # The range of random integers is [low, high] - first_word = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=dict_size - 1) - second_word = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=dict_size - 1) - third_word = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=dict_size - 1) - fourth_word = fluid.create_random_int_lodtensor(recursive_seq_lens, - base_shape, - place, - low=0, - high=dict_size - 1) + first_word = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=dict_size - 1 + ) + second_word = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=dict_size - 1 + ) + third_word = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=dict_size - 1 + ) + fourth_word = fluid.create_random_int_lodtensor( + recursive_seq_lens, base_shape, place, low=0, high=dict_size - 1 + ) assert feed_target_names[0] == 'firstw' assert feed_target_names[1] == 'secondw' @@ -223,15 +239,17 @@ def infer(target, save_dirname=None): # Construct feed as a dictionary of {feed_target_name: feed_target_data} # and results will contain a list of data corresponding to fetch_targets. - results = exe.run(inference_program, - feed={ - feed_target_names[0]: first_word, - feed_target_names[1]: second_word, - feed_target_names[2]: third_word, - feed_target_names[3]: fourth_word - }, - fetch_list=fetch_targets, - return_numpy=False) + results = exe.run( + inference_program, + feed={ + feed_target_names[0]: first_word, + feed_target_names[1]: second_word, + feed_target_names[2]: third_word, + feed_target_names[3]: fourth_word, + }, + fetch_list=fetch_targets, + return_numpy=False, + ) def to_infer_tensor(lod_tensor): infer_tensor = fluid.core.PaddleTensor() @@ -282,18 +300,26 @@ def main(target, is_sparse, is_parallel, use_bf16, pure_bf16): # so only inference is turned on. train("cpu", is_sparse, is_parallel, save_dirname) else: - train(target, - is_sparse, - is_parallel, - save_dirname, - use_bf16=use_bf16, - pure_bf16=pure_bf16) + train( + target, + is_sparse, + is_parallel, + save_dirname, + use_bf16=use_bf16, + pure_bf16=pure_bf16, + ) infer(target, save_dirname) temp_dir.cleanup() -FULL_TEST = os.getenv('FULL_TEST', - '0').lower() in ['true', '1', 't', 'y', 'yes', 'on'] +FULL_TEST = os.getenv('FULL_TEST', '0').lower() in [ + 'true', + '1', + 't', + 'y', + 'yes', + 'on', +] SKIP_REASON = "Only run minimum number of tests in CI server, to make CI faster" @@ -301,15 +327,15 @@ class W2VTest(unittest.TestCase): pass -def inject_test_method(target, - is_sparse, - is_parallel, - use_bf16=False, - pure_bf16=False): +def inject_test_method( + target, is_sparse, is_parallel, use_bf16=False, pure_bf16=False +): fn_name = "test_{0}_{1}_{2}{3}".format( - target, "sparse" if is_sparse else "dense", + target, + "sparse" if is_sparse else "dense", "parallel" if is_parallel else "normal", - "_purebf16" if pure_bf16 else "_bf16" if use_bf16 else "") + "_purebf16" if pure_bf16 else "_bf16" if use_bf16 else "", + ) def __impl__(*args, **kwargs): prog = fluid.Program() @@ -319,20 +345,22 @@ def __impl__(*args, **kwargs): with fluid.program_guard(prog, startup_prog): main(target, is_sparse, is_parallel, use_bf16, pure_bf16) - if (not fluid.core.is_compiled_with_cuda() - or target == "cuda") and is_sparse: + if ( + not fluid.core.is_compiled_with_cuda() or target == "cuda" + ) and is_sparse: fn = __impl__ else: # skip the other test when on CI server - fn = unittest.skipUnless(condition=FULL_TEST, - reason=SKIP_REASON)(__impl__) + fn = unittest.skipUnless(condition=FULL_TEST, reason=SKIP_REASON)( + __impl__ + ) setattr(W2VTest, fn_name, fn) for target in ("cuda", "cpu", "xpu"): for is_sparse in (False, True): - for is_parallel in (False, ): + for is_parallel in (False,): inject_test_method(target, is_sparse, is_parallel) inject_test_method("cpu", False, False, True) inject_test_method("cpu", False, False, True, True) diff --git a/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_c_setup.py b/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_c_setup.py index 39d47d6f4483b..34110b6395c8d 100644 --- a/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_c_setup.py +++ b/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_c_setup.py @@ -24,7 +24,6 @@ # cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid # for C/ObjC but not for C++ class BuildExt(build_ext): - def build_extensions(self): if '-Wstrict-prototypes' in self.compiler.compiler_so: self.compiler.compiler_so.remove('-Wstrict-prototypes') @@ -48,8 +47,9 @@ def build_extensions(self): os.path.join(site_packages_path, 'paddle', 'include'), ] # include path third_party -compile_third_party_path = os.path.join(os.environ['PADDLE_BINARY_DIR'], - 'third_party') +compile_third_party_path = os.path.join( + os.environ['PADDLE_BINARY_DIR'], 'third_party' +) paddle_custom_kernel_include += [ os.path.join(compile_third_party_path, 'install/gflags/include'), # gflags os.path.join(compile_third_party_path, 'install/glog/include'), # glog @@ -69,10 +69,13 @@ def build_extensions(self): include_dirs=paddle_custom_kernel_include, library_dirs=paddle_custom_kernel_library_dir, libraries=libs, - extra_compile_args=paddle_extra_compile_args) + extra_compile_args=paddle_extra_compile_args, +) -setup(name='custom_kernel_dot_c', - version='1.0', - description='custom kernel fot compiling', - cmdclass={'build_ext': BuildExt}, - ext_modules=[custom_kernel_dot_module]) +setup( + name='custom_kernel_dot_c', + version='1.0', + description='custom kernel fot compiling', + cmdclass={'build_ext': BuildExt}, + ext_modules=[custom_kernel_dot_module], +) diff --git a/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_setup.py b/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_setup.py index 8b953ca87251a..0ed5610cda4c2 100644 --- a/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_setup.py +++ b/python/paddle/fluid/tests/custom_kernel/custom_kernel_dot_setup.py @@ -24,7 +24,6 @@ # cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid # for C/ObjC but not for C++ class BuildExt(build_ext): - def build_extensions(self): if '-Wstrict-prototypes' in self.compiler.compiler_so: self.compiler.compiler_so.remove('-Wstrict-prototypes') @@ -45,12 +44,15 @@ def build_extensions(self): # include path site_packages_path = site.getsitepackages() paddle_custom_kernel_include = list( - map(lambda path: os.path.join(path, 'paddle', 'include'), - site_packages_path)) + map( + lambda path: os.path.join(path, 'paddle', 'include'), site_packages_path + ) +) # include path third_party -compile_third_party_path = os.path.join(os.environ['PADDLE_BINARY_DIR'], - 'third_party') +compile_third_party_path = os.path.join( + os.environ['PADDLE_BINARY_DIR'], 'third_party' +) paddle_custom_kernel_include += [ os.path.join(compile_third_party_path, 'install/gflags/include'), # gflags os.path.join(compile_third_party_path, 'install/glog/include'), # glog @@ -58,7 +60,8 @@ def build_extensions(self): # libs path paddle_custom_kernel_library_dir = list( - map(lambda path: os.path.join(path, 'paddle', 'fluid'), site_packages_path)) + map(lambda path: os.path.join(path, 'paddle', 'fluid'), site_packages_path) +) # libs libs = [':libpaddle.so'] @@ -69,10 +72,13 @@ def build_extensions(self): include_dirs=paddle_custom_kernel_include, library_dirs=paddle_custom_kernel_library_dir, libraries=libs, - extra_compile_args=paddle_extra_compile_args) + extra_compile_args=paddle_extra_compile_args, +) -setup(name='custom_kernel_dot', - version='1.0', - description='custom kernel fot compiling', - cmdclass={'build_ext': BuildExt}, - ext_modules=[custom_kernel_dot_module]) +setup( + name='custom_kernel_dot', + version='1.0', + description='custom kernel fot compiling', + cmdclass={'build_ext': BuildExt}, + ext_modules=[custom_kernel_dot_module], +) diff --git a/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_dot.py b/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_dot.py index 8badd5dfbde81..6d941699dd9b3 100644 --- a/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_dot.py +++ b/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_dot.py @@ -20,14 +20,16 @@ # use dot as test case. class TestCustomKernelDot(unittest.TestCase): - def setUp(self): # compile so and set to current path cur_dir = os.path.dirname(os.path.abspath(__file__)) # --inplace to place output so file to current dir - cmd = 'cd {} && {} custom_kernel_dot_setup.py build_ext --inplace'.format( - cur_dir, sys.executable) + cmd = ( + 'cd {} && {} custom_kernel_dot_setup.py build_ext --inplace'.format( + cur_dir, sys.executable + ) + ) os.system(cmd) def test_custom_kernel_dot_run(self): @@ -37,6 +39,7 @@ def test_custom_kernel_dot_run(self): result = np.sum(x_data * y_data, axis=1).reshape([2, 1]) import paddle + paddle.set_device('cpu') x = paddle.to_tensor(x_data) y = paddle.to_tensor(y_data) @@ -46,18 +49,20 @@ def test_custom_kernel_dot_run(self): out.numpy(), result, err_msg='custom kernel dot out: {},\n numpy dot out: {}'.format( - out.numpy(), result)) + out.numpy(), result + ), + ) class TestCustomKernelDotC(unittest.TestCase): - def setUp(self): # compile so and set to current path cur_dir = os.path.dirname(os.path.abspath(__file__)) # --inplace to place output so file to current dir cmd = 'cd {} && {} custom_kernel_dot_c_setup.py build_ext --inplace'.format( - cur_dir, sys.executable) + cur_dir, sys.executable + ) os.system(cmd) def test_custom_kernel_dot_run(self): @@ -67,6 +72,7 @@ def test_custom_kernel_dot_run(self): result = np.sum(x_data * y_data, axis=1).reshape([2, 1]) import paddle + paddle.set_device('cpu') x = paddle.to_tensor(x_data) y = paddle.to_tensor(y_data) @@ -76,7 +82,9 @@ def test_custom_kernel_dot_run(self): out.numpy(), result, err_msg='custom kernel dot out: {},\n numpy dot out: {}'.format( - out.numpy(), result)) + out.numpy(), result + ), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_load.py b/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_load.py index ff7ff3e04a88e..1ebeb6a4585f1 100644 --- a/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_load.py +++ b/python/paddle/fluid/tests/custom_kernel/test_custom_kernel_load.py @@ -20,22 +20,25 @@ class TestCustomKernelLoad(unittest.TestCase): - def setUp(self): # compile so and set to current path cur_dir = os.path.dirname(os.path.abspath(__file__)) # --inplace to place output so file to current dir - cmd = 'cd {} && {} custom_kernel_dot_setup.py build_ext --inplace'.format( - cur_dir, sys.executable) + cmd = ( + 'cd {} && {} custom_kernel_dot_setup.py build_ext --inplace'.format( + cur_dir, sys.executable + ) + ) os.system(cmd) # get paddle lib path and place so paddle_lib_path = '' - site_dirs = site.getsitepackages() if hasattr( - site, 'getsitepackages') else [ - x for x in sys.path if 'site-packages' in x - ] + site_dirs = ( + site.getsitepackages() + if hasattr(site, 'getsitepackages') + else [x for x in sys.path if 'site-packages' in x] + ) for site_dir in site_dirs: lib_dir = os.path.sep.join([site_dir, 'paddle', 'libs']) if os.path.exists(lib_dir): @@ -47,10 +50,12 @@ def setUp(self): if os.path.exists(lib_dir): paddle_lib_path = lib_dir self.default_path = os.path.sep.join( - [paddle_lib_path, '..', '..', 'paddle-plugins']) + [paddle_lib_path, '..', '..', 'paddle-plugins'] + ) # copy so to default path - cmd = 'mkdir -p {} && cp ./*.so {}'.format(self.default_path, - self.default_path) + cmd = 'mkdir -p {} && cp ./*.so {}'.format( + self.default_path, self.default_path + ) os.system(cmd) # wait def test_custom_kernel_dot_load(self): @@ -60,6 +65,7 @@ def test_custom_kernel_dot_load(self): result = np.sum(x_data * y_data, axis=1).reshape([2, 1]) import paddle + paddle.set_device('cpu') x = paddle.to_tensor(x_data) y = paddle.to_tensor(y_data) @@ -69,7 +75,9 @@ def test_custom_kernel_dot_load(self): out.numpy(), result, err_msg='custom kernel dot out: {},\n numpy dot out: {}'.format( - out.numpy(), result)) + out.numpy(), result + ), + ) def tearDown(self): cmd = 'rm -rf {}'.format(self.default_path) diff --git a/python/paddle/fluid/tests/custom_op/custom_raw_op_kernel_op_setup.py b/python/paddle/fluid/tests/custom_op/custom_raw_op_kernel_op_setup.py index a1ae927997200..c29bf7014f847 100644 --- a/python/paddle/fluid/tests/custom_op/custom_raw_op_kernel_op_setup.py +++ b/python/paddle/fluid/tests/custom_op/custom_raw_op_kernel_op_setup.py @@ -41,9 +41,13 @@ macros.append(("THRUST_IGNORE_CUB_VERSION_CHECK", None)) include_dirs = list(paddle_includes) + [cwd] -setup(name=os.getenv("MODULE_NAME", "custom_raw_op_kernel_op_setup"), - ext_modules=extension(sources=sources, - include_dirs=include_dirs, - extra_compile_args=extra_compile_args, - _compile_dir=compile_dir, - define_macros=macros)) +setup( + name=os.getenv("MODULE_NAME", "custom_raw_op_kernel_op_setup"), + ext_modules=extension( + sources=sources, + include_dirs=include_dirs, + extra_compile_args=extra_compile_args, + _compile_dir=compile_dir, + define_macros=macros, + ), +) diff --git a/python/paddle/fluid/tests/custom_op/custom_relu_setup.py b/python/paddle/fluid/tests/custom_op/custom_relu_setup.py index 7dd3dd7aa7e38..e71c5584e6729 100644 --- a/python/paddle/fluid/tests/custom_op/custom_relu_setup.py +++ b/python/paddle/fluid/tests/custom_op/custom_relu_setup.py @@ -30,4 +30,6 @@ sources=sources, # test for multi ops include_dirs=paddle_includes, extra_compile_args=extra_compile_args, - verbose=True)) + verbose=True, + ), +) diff --git a/python/paddle/fluid/tests/custom_op/test_check_abi.py b/python/paddle/fluid/tests/custom_op/test_check_abi.py index 39bcf1ae628c6..ab37412362550 100644 --- a/python/paddle/fluid/tests/custom_op/test_check_abi.py +++ b/python/paddle/fluid/tests/custom_op/test_check_abi.py @@ -20,7 +20,6 @@ class TestABIBase(unittest.TestCase): - def test_environ(self): compiler_list = ['gcc', 'cl'] for compiler in compiler_list: @@ -35,7 +34,6 @@ def del_environ(self): class TestCheckCompiler(TestABIBase): - def test_expected_compiler(self): if utils.OS_NAME.startswith('linux'): gt = ['gcc', 'g++', 'gnu-c++', 'gnu-cc'] @@ -73,7 +71,8 @@ def test_wrong_compiler_warning(self): # check Compiler Compatibility WARNING self.assertTrue(len(error) == 1) self.assertTrue( - "Compiler Compatibility WARNING" in str(error[0].message)) + "Compiler Compatibility WARNING" in str(error[0].message) + ) def test_exception_windows(self): # clear environ @@ -86,8 +85,10 @@ def test_exception_windows(self): self.assertFalse(flag) # check ABI Compatibility WARNING self.assertTrue(len(error) == 1) - self.assertTrue("Failed to check compiler version for" in str( - error[0].message)) + self.assertTrue( + "Failed to check compiler version for" + in str(error[0].message) + ) def test_exception_linux(self): # clear environ @@ -107,8 +108,10 @@ def fake(): self.assertFalse(flag) # check ABI Compatibility WARNING self.assertTrue(len(error) == 1) - self.assertTrue("Failed to check compiler version for" in str( - error[0].message)) + self.assertTrue( + "Failed to check compiler version for" + in str(error[0].message) + ) # restore utils._expected_compiler_current_platform = raw_func @@ -137,7 +140,6 @@ def fake(): class TestRunCMDException(unittest.TestCase): - def test_exception(self): for verbose in [True, False]: with self.assertRaisesRegexp(RuntimeError, "Failed to run command"): diff --git a/python/paddle/fluid/tests/custom_op/test_context_pool.py b/python/paddle/fluid/tests/custom_op/test_context_pool.py index 69b8b18559ef9..0c4829fb90fdb 100644 --- a/python/paddle/fluid/tests/custom_op/test_context_pool.py +++ b/python/paddle/fluid/tests/custom_op/test_context_pool.py @@ -25,7 +25,8 @@ # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. file = '{}\\context_pool_jit\\context_pool_jit.pyd'.format( - get_build_directory()) + get_build_directory() +) if os.name == 'nt' and os.path.isfile(file): cmd = 'del {}'.format(file) run_cmd(cmd, True) @@ -37,11 +38,11 @@ extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cflags extra_cuda_cflags=extra_nvcc_args, # test for cflags - verbose=True) + verbose=True, +) class TestContextPool(unittest.TestCase): - def setUp(self): self.devices = ['cpu'] if paddle.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/custom_op/test_custom_attrs_jit.py b/python/paddle/fluid/tests/custom_op/test_custom_attrs_jit.py index a0be75c0a41eb..7109d957384c5 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_attrs_jit.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_attrs_jit.py @@ -25,7 +25,8 @@ # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. file = '{}\\custom_attrs_jit\\custom_attrs_jit.pyd'.format( - get_build_directory()) + get_build_directory() +) if os.name == 'nt' and os.path.isfile(file): cmd = 'del {}'.format(file) run_cmd(cmd, True) @@ -37,11 +38,11 @@ extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cflags extra_cuda_cflags=extra_nvcc_args, # test for cflags - verbose=True) + verbose=True, +) class TestJitCustomAttrs(unittest.TestCase): - def setUp(self): paddle.set_device('cpu') # prepare test value @@ -58,11 +59,18 @@ def setUp(self): def func_attr_value(self): x = paddle.ones([2, 2], dtype='float32') x.stop_gradient = False - out = custom_attrs.attr_test(x, self.bool_attr, self.int_attr, - self.float_attr, self.int64_attr, - self.str_attr, self.int_vec_attr, - self.float_vec_attr, self.int64_vec_attr, - self.str_vec_attr) + out = custom_attrs.attr_test( + x, + self.bool_attr, + self.int_attr, + self.float_attr, + self.int64_attr, + self.str_attr, + self.int_vec_attr, + self.float_vec_attr, + self.int64_vec_attr, + self.str_vec_attr, + ) out.stop_gradient = False out.backward() @@ -76,12 +84,18 @@ def test_attr_value(self): def func_const_attr_value(self): x = paddle.ones([2, 2], dtype='float32') x.stop_gradient = False - out = custom_attrs.const_attr_test(x, self.bool_attr, self.int_attr, - self.float_attr, self.int64_attr, - self.str_attr, self.int_vec_attr, - self.float_vec_attr, - self.int64_vec_attr, - self.str_vec_attr) + out = custom_attrs.const_attr_test( + x, + self.bool_attr, + self.int_attr, + self.float_attr, + self.int64_attr, + self.str_attr, + self.int_vec_attr, + self.float_vec_attr, + self.int64_vec_attr, + self.str_vec_attr, + ) out.stop_gradient = False out.backward() diff --git a/python/paddle/fluid/tests/custom_op/test_custom_concat.py b/python/paddle/fluid/tests/custom_op/test_custom_concat.py index ae3022411b18e..a0744ce0870df 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_concat.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_concat.py @@ -42,7 +42,8 @@ extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, +) def concat_dynamic(func, dtype, np_inputs, axis_v, with_attr=False): @@ -85,29 +86,29 @@ def concat_static(func, dtype, np_inputs, axis_v, with_attr=False): if with_attr: feed_dict = { "x1": np_inputs[0].astype(dtype), - "x2": np_inputs[1].astype(dtype) + "x2": np_inputs[1].astype(dtype), } else: feed_dict = { "x1": np_inputs[0].astype(dtype), "x2": np_inputs[1].astype(dtype), - "axis": axis + "axis": axis, } out_v, x1_grad_v, x2_grad_v = exe.run( static.default_main_program(), feed=feed_dict, - fetch_list=[out.name, x1.name + "@GRAD", x2.name + "@GRAD"]) + fetch_list=[out.name, x1.name + "@GRAD", x2.name + "@GRAD"], + ) paddle.disable_static() return out_v, x1_grad_v, x2_grad_v class TestCustomConcatDynamicAxisJit(unittest.TestCase): - def setUp(self): self.dtypes = ['float32', 'float64', 'int32', 'int64'] self.np_inputs = [ np.array([[1, 2, 3], [4, 5, 6]]), - np.array([[11, 12, 13], [14, 15, 16]]) + np.array([[11, 12, 13], [14, 15, 16]]), ] self.axises = [0, 1] @@ -116,15 +117,19 @@ def check_output(self, out, pd_out, name): out, pd_out, err_msg='custom op {}: {},\n paddle api {}: {}'.format( - name, out, name, pd_out)) + name, out, name, pd_out + ), + ) def func_dynamic(self): for dtype in self.dtypes: for axis in self.axises: - out, grad_inputs = concat_dynamic(custom_ops.custom_concat, - dtype, self.np_inputs, axis) - pd_out, pd_grad_inputs = concat_dynamic(paddle.concat, dtype, - self.np_inputs, axis) + out, grad_inputs = concat_dynamic( + custom_ops.custom_concat, dtype, self.np_inputs, axis + ) + pd_out, pd_grad_inputs = concat_dynamic( + paddle.concat, dtype, self.np_inputs, axis + ) self.check_output(out, pd_out, "out") for x_grad, pd_x_grad in zip(grad_inputs, pd_grad_inputs): @@ -138,11 +143,12 @@ def test_dynamic(self): def test_static(self): for dtype in self.dtypes: for axis in self.axises: - out, x1_grad, x2_grad = concat_static(custom_ops.custom_concat, - dtype, self.np_inputs, - axis) + out, x1_grad, x2_grad = concat_static( + custom_ops.custom_concat, dtype, self.np_inputs, axis + ) pd_out, pd_x1_grad, pd_x2_grad = concat_static( - paddle.concat, dtype, self.np_inputs, axis) + paddle.concat, dtype, self.np_inputs, axis + ) self.check_output(out, pd_out, "out") self.check_output(x1_grad, pd_x1_grad, "x1_grad") @@ -152,11 +158,15 @@ def func_dynamic_with_attr(self): for dtype in self.dtypes: for axis in self.axises: out, grad_inputs = concat_dynamic( - custom_ops.custom_concat_with_attr, dtype, self.np_inputs, - axis, True) - pd_out, pd_grad_inputs = concat_dynamic(paddle.concat, dtype, - self.np_inputs, axis, - True) + custom_ops.custom_concat_with_attr, + dtype, + self.np_inputs, + axis, + True, + ) + pd_out, pd_grad_inputs = concat_dynamic( + paddle.concat, dtype, self.np_inputs, axis, True + ) self.check_output(out, pd_out, "out") for x_grad, pd_x_grad in zip(grad_inputs, pd_grad_inputs): @@ -171,10 +181,15 @@ def test_static_with_attr(self): for dtype in self.dtypes: for axis in self.axises: out, x1_grad, x2_grad = concat_static( - custom_ops.custom_concat_with_attr, dtype, self.np_inputs, - axis, True) + custom_ops.custom_concat_with_attr, + dtype, + self.np_inputs, + axis, + True, + ) pd_out, pd_x1_grad, pd_x2_grad = concat_static( - paddle.concat, dtype, self.np_inputs, axis, True) + paddle.concat, dtype, self.np_inputs, axis, True + ) self.check_output(out, pd_out, "out") self.check_output(x1_grad, pd_x1_grad, "x1_grad") diff --git a/python/paddle/fluid/tests/custom_op/test_custom_conj.py b/python/paddle/fluid/tests/custom_op/test_custom_conj.py index a389a72df7350..a9990facd79f6 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_conj.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_conj.py @@ -36,12 +36,15 @@ extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, +) def is_complex(dtype): - return dtype == paddle.fluid.core.VarDesc.VarType.COMPLEX64 or \ - dtype == paddle.fluid.core.VarDesc.VarType.COMPLEX128 + return ( + dtype == paddle.fluid.core.VarDesc.VarType.COMPLEX64 + or dtype == paddle.fluid.core.VarDesc.VarType.COMPLEX128 + ) def to_complex(dtype): @@ -83,15 +86,16 @@ def conj_static(func, shape, dtype, np_input): exe = static.Executor() exe.run(static.default_startup_program()) - out_v, x_grad_v = exe.run(static.default_main_program(), - feed={"x": np_input}, - fetch_list=[out.name, x.name + "@GRAD"]) + out_v, x_grad_v = exe.run( + static.default_main_program(), + feed={"x": np_input}, + fetch_list=[out.name, x.name + "@GRAD"], + ) paddle.disable_static() return out_v, x_grad_v class TestCustomConjJit(unittest.TestCase): - def setUp(self): self.dtypes = ['float32', 'float64'] self.shape = [2, 20, 2, 3] @@ -101,7 +105,9 @@ def check_output(self, out, pd_out, name): out, pd_out, err_msg='custom op {}: {},\n paddle api {}: {}'.format( - name, out, name, pd_out)) + name, out, name, pd_out + ), + ) def run_dynamic(self, dtype, np_input): out, x_grad = conj_dynamic(custom_ops.custom_conj, dtype, np_input) @@ -111,10 +117,12 @@ def run_dynamic(self, dtype, np_input): self.check_output(x_grad, pd_x_grad, "x's grad") def run_static(self, dtype, np_input): - out, x_grad = conj_static(custom_ops.custom_conj, self.shape, dtype, - np_input) - pd_out, pd_x_grad = conj_static(paddle.conj, self.shape, dtype, - np_input) + out, x_grad = conj_static( + custom_ops.custom_conj, self.shape, dtype, np_input + ) + pd_out, pd_x_grad = conj_static( + paddle.conj, self.shape, dtype, np_input + ) self.check_output(out, pd_out, "out") self.check_output(x_grad, pd_x_grad, "x's grad") @@ -138,7 +146,8 @@ def test_static(self): def test_complex_dynamic(self): for dtype in self.dtypes: np_input = np.random.random(self.shape).astype( - dtype) + 1j * np.random.random(self.shape).astype(dtype) + dtype + ) + 1j * np.random.random(self.shape).astype(dtype) self.run_dynamic(to_complex(dtype), np_input) diff --git a/python/paddle/fluid/tests/custom_op/test_custom_linear.py b/python/paddle/fluid/tests/custom_op/test_custom_linear.py index 3ae650ee9494d..bb7bc0358f4e8 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_linear.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_linear.py @@ -37,7 +37,8 @@ extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, +) def linear_dynamic(func, device, dtype, np_x, np_weight, np_bias): @@ -56,9 +57,9 @@ def linear_static(func, device, dtype, np_x, np_weight, np_bias): with static.scope_guard(static.Scope()): with static.program_guard(static.Program()): x = static.data(name="x", shape=[None, np_x.shape[1]], dtype=dtype) - weight = static.data(name="weight", - shape=np_weight.shape, - dtype=dtype) + weight = static.data( + name="weight", shape=np_weight.shape, dtype=dtype + ) bias = static.data(name="bias", shape=np_bias.shape, dtype=dtype) x.stop_gradient = False weight.stop_gradient = False @@ -75,18 +76,20 @@ def linear_static(func, device, dtype, np_x, np_weight, np_bias): feed={ "x": np_x.astype(dtype), "weight": np_weight.astype(dtype), - "bias": np_bias.astype(dtype) + "bias": np_bias.astype(dtype), }, fetch_list=[ - out.name, x.name + "@GRAD", weight.name + "@GRAD", - bias.name + "@GRAD" - ]) + out.name, + x.name + "@GRAD", + weight.name + "@GRAD", + bias.name + "@GRAD", + ], + ) paddle.disable_static() return out_v, x_grad_v, weight_grad_v, bias_grad_v class TestCustomLinearJit(unittest.TestCase): - def setUp(self): self.dtypes = ['float32', 'float64'] self.devices = ['cpu'] @@ -101,36 +104,75 @@ def check_output(self, out, pd_out, name): out, pd_out, err_msg='custom op {}: {},\n paddle api {}: {}'.format( - name, out, name, pd_out)) + name, out, name, pd_out + ), + ) def test_static(self): for device in self.devices: for dtype in self.dtypes: - phi_out, phi_x_grad, phi_weight_grad, phi_bias_grad = linear_static( - custom_ops.phi_linear, device, dtype, self.np_x, - self.np_weight, self.np_bias) + ( + phi_out, + phi_x_grad, + phi_weight_grad, + phi_bias_grad, + ) = linear_static( + custom_ops.phi_linear, + device, + dtype, + self.np_x, + self.np_weight, + self.np_bias, + ) pd_out, pd_x_grad, pd_weight_grad, pd_bias_grad = linear_static( - F.linear, device, dtype, self.np_x, self.np_weight, - self.np_bias) + F.linear, + device, + dtype, + self.np_x, + self.np_weight, + self.np_bias, + ) self.check_output(phi_out, pd_out, "out") self.check_output(phi_x_grad, pd_x_grad, "x_grad") - self.check_output(phi_weight_grad, pd_weight_grad, - "weight_grad") + self.check_output( + phi_weight_grad, pd_weight_grad, "weight_grad" + ) self.check_output(phi_bias_grad, pd_bias_grad, "bias_grad") def func_dynamic(self): for device in self.devices: for dtype in self.dtypes: - phi_out, phi_x_grad, phi_weight_grad, phi_bias_grad = linear_dynamic( - custom_ops.phi_linear, device, dtype, self.np_x, - self.np_weight, self.np_bias) - pd_out, pd_x_grad, pd_weight_grad, pd_bias_grad = linear_dynamic( - F.linear, device, dtype, self.np_x, self.np_weight, - self.np_bias) + ( + phi_out, + phi_x_grad, + phi_weight_grad, + phi_bias_grad, + ) = linear_dynamic( + custom_ops.phi_linear, + device, + dtype, + self.np_x, + self.np_weight, + self.np_bias, + ) + ( + pd_out, + pd_x_grad, + pd_weight_grad, + pd_bias_grad, + ) = linear_dynamic( + F.linear, + device, + dtype, + self.np_x, + self.np_weight, + self.np_bias, + ) self.check_output(phi_out, pd_out, "phi_out") self.check_output(phi_x_grad, pd_x_grad, "x_grad") - self.check_output(phi_weight_grad, pd_weight_grad, - "weight_grad") + self.check_output( + phi_weight_grad, pd_weight_grad, "weight_grad" + ) self.check_output(phi_bias_grad, pd_bias_grad, "bias_grad") def test_dynamic(self): diff --git a/python/paddle/fluid/tests/custom_op/test_custom_raw_op_kernel_op.py b/python/paddle/fluid/tests/custom_op/test_custom_raw_op_kernel_op.py index 3cd550c95f00e..33f98c219d84a 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_raw_op_kernel_op.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_raw_op_kernel_op.py @@ -36,8 +36,9 @@ def prepare_module_path(): else: site_dir = site.getsitepackages()[0] custom_egg_path = [x for x in os.listdir(site_dir) if MODULE_NAME in x] - assert len(custom_egg_path - ) == 1, "Matched egg number is %d." % len(custom_egg_path) + assert len(custom_egg_path) == 1, "Matched egg number is %d." % len( + custom_egg_path + ) sys.path.append(os.path.join(site_dir, custom_egg_path[0])) @@ -46,7 +47,6 @@ def prepare_module_path(): # temporarily. @unittest.skipIf(os.name == "nt", "Windows does not support yet.") class TestCustomRawReluOp(unittest.TestCase): - @classmethod def setUpClass(cls): path = os.path.dirname(os.path.abspath(__file__)) @@ -78,11 +78,14 @@ def test_static(self): exe = paddle.static.Executor() exe.run(paddle.static.default_startup_program()) - x_np = np.random.uniform(low=-1.0, high=1.0, size=[2, - 3]).astype('float32') - y1_value, y2_value = exe.run(paddle.static.default_main_program(), - feed={x.name: x_np}, - fetch_list=[y1, y2]) + x_np = np.random.uniform(low=-1.0, high=1.0, size=[2, 3]).astype( + 'float32' + ) + y1_value, y2_value = exe.run( + paddle.static.default_main_program(), + feed={x.name: x_np}, + fetch_list=[y1, y2], + ) np.testing.assert_array_equal(y1_value, y2_value) paddle.disable_static() diff --git a/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py b/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py index 762a16b61ff82..f445410e906df 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_relu_model.py @@ -28,7 +28,8 @@ # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. file = '{}\\custom_relu_for_model_jit\\custom_relu_for_model_jit.pyd'.format( - get_build_directory()) + get_build_directory() +) if os.name == 'nt' and os.path.isfile(file): cmd = 'del {}'.format(file) run_cmd(cmd, True) @@ -47,7 +48,8 @@ extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, +) class Net(nn.Layer): @@ -59,7 +61,9 @@ def __init__(self, in_dim, out_dim, use_custom_op=False): super(Net, self).__init__() self.fc1 = nn.Linear(in_dim, in_dim) self.fc2 = nn.Linear(in_dim, out_dim) - self.relu_act = custom_module.custom_relu if use_custom_op else nn.functional.relu + self.relu_act = ( + custom_module.custom_relu if use_custom_op else nn.functional.relu + ) def forward(self, x): out = self.fc1(x) @@ -73,7 +77,6 @@ def forward(self, x): class TestDygraphModel(unittest.TestCase): - def tearDown(self): self.temp_dir.cleanup() @@ -85,8 +88,9 @@ def setUp(self): self.batch_num = 10 self.batch_size = 4 self.datas = [ - np.random.uniform( - size=[self.batch_size, self.in_dim]).astype('float32') + np.random.uniform(size=[self.batch_size, self.in_dim]).astype( + 'float32' + ) for i in range(self.batch_num) ] self.labels = [ @@ -100,14 +104,16 @@ def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.model_save_dir = os.path.join(self.temp_dir.name, 'infer_model') self.model_path_template = os.path.join( - self.model_save_dir, 'custom_relu_dygaph_model_{}.pdparams') + self.model_save_dir, 'custom_relu_dygaph_model_{}.pdparams' + ) self.model_dy2stat_path = os.path.join( - self.model_save_dir, 'infer_model/custom_relu_model_dy2sta') + self.model_save_dir, 'infer_model/custom_relu_model_dy2sta' + ) # for dy2stat - self.x_spec = paddle.static.InputSpec(shape=[None, self.in_dim], - dtype='float32', - name='x') + self.x_spec = paddle.static.InputSpec( + shape=[None, self.in_dim], dtype='float32', name='x' + ) def func_train_eval(self): for device in self.devices: @@ -120,24 +126,30 @@ def func_train_eval(self): # open this when dy2stat is ready for eager if _in_legacy_dygraph(): custom_relu_dy2stat_train_out = self.train_model( - use_custom_op=True, dy2stat=True) # for to_static - np.testing.assert_array_equal(origin_relu_train_out, - custom_relu_dy2stat_train_out) + use_custom_op=True, dy2stat=True + ) # for to_static + np.testing.assert_array_equal( + origin_relu_train_out, custom_relu_dy2stat_train_out + ) - np.testing.assert_array_equal(origin_relu_train_out, - custom_relu_train_out) + np.testing.assert_array_equal( + origin_relu_train_out, custom_relu_train_out + ) # for eval origin_relu_eval_out = self.eval_model(use_custom_op=False) custom_relu_eval_out = self.eval_model(use_custom_op=True) if _in_legacy_dygraph(): custom_relu_dy2stat_eval_out = self.eval_model( - use_custom_op=True, dy2stat=True) # for to_static - np.testing.assert_array_equal(origin_relu_eval_out, - custom_relu_dy2stat_eval_out) + use_custom_op=True, dy2stat=True + ) # for to_static + np.testing.assert_array_equal( + origin_relu_eval_out, custom_relu_dy2stat_eval_out + ) - np.testing.assert_array_equal(origin_relu_eval_out, - custom_relu_eval_out) + np.testing.assert_array_equal( + origin_relu_eval_out, custom_relu_eval_out + ) def test_train_eval(self): with _test_eager_guard(): @@ -154,8 +166,9 @@ def train_model(self, use_custom_op=False, dy2stat=False): if dy2stat: net = paddle.jit.to_static(net, input_spec=[self.x_spec]) mse_loss = paddle.nn.MSELoss() - sgd = paddle.optimizer.SGD(learning_rate=0.1, - parameters=net.parameters()) + sgd = paddle.optimizer.SGD( + learning_rate=0.1, parameters=net.parameters() + ) for batch_id in range(self.batch_num): x = paddle.to_tensor(self.datas[batch_id]) @@ -173,8 +186,9 @@ def train_model(self, use_custom_op=False, dy2stat=False): if dy2stat: paddle.jit.save(net, self.model_dy2stat_path) else: - paddle.save(net.state_dict(), - self.model_path_template.format(use_custom_op)) + paddle.save( + net.state_dict(), self.model_path_template.format(use_custom_op) + ) return out.numpy() @@ -185,7 +199,8 @@ def eval_model(self, use_custom_op=False, dy2stat=False): net = paddle.jit.load(self.model_dy2stat_path) else: state_dict = paddle.load( - self.model_path_template.format(use_custom_op)) + self.model_path_template.format(use_custom_op) + ) net.set_state_dict(state_dict) sample_x = paddle.to_tensor(self.datas[0]) @@ -196,7 +211,6 @@ def eval_model(self, use_custom_op=False, dy2stat=False): class TestStaticModel(unittest.TestCase): - def setUp(self): self.seed = 2021 self.in_dim = 10 @@ -204,8 +218,9 @@ def setUp(self): self.batch_num = 10 self.batch_size = 8 self.datas = [ - np.random.uniform( - size=[self.batch_size, self.in_dim]).astype('float32') + np.random.uniform(size=[self.batch_size, self.in_dim]).astype( + 'float32' + ) for i in range(self.batch_num) ] self.labels = [ @@ -219,7 +234,8 @@ def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.model_save_dir = os.path.join(self.temp_dir.name, 'infer_model') self.model_path_template = os.path.join( - self.model_save_dir, 'custom_relu_static_model_{}_{}') + self.model_save_dir, 'custom_relu_static_model_{}_{}' + ) paddle.enable_static() @@ -230,38 +246,44 @@ def tearDown(self): def test_train_eval(self): for device in self.devices: # for train - original_relu_train_out = self.train_model(device, - use_custom_op=False) + original_relu_train_out = self.train_model( + device, use_custom_op=False + ) custom_relu_train_out = self.train_model(device, use_custom_op=True) # using PE - original_relu_train_pe_out = self.train_model(device, - use_custom_op=False, - use_pe=True) - custom_relu_train_pe_out = self.train_model(device, - use_custom_op=True, - use_pe=True) - - np.testing.assert_array_equal(original_relu_train_out, - custom_relu_train_out) - np.testing.assert_array_equal(original_relu_train_pe_out, - custom_relu_train_pe_out) + original_relu_train_pe_out = self.train_model( + device, use_custom_op=False, use_pe=True + ) + custom_relu_train_pe_out = self.train_model( + device, use_custom_op=True, use_pe=True + ) + + np.testing.assert_array_equal( + original_relu_train_out, custom_relu_train_out + ) + np.testing.assert_array_equal( + original_relu_train_pe_out, custom_relu_train_pe_out + ) # for eval - original_relu_eval_out = self.eval_model(device, - use_custom_op=False) + original_relu_eval_out = self.eval_model( + device, use_custom_op=False + ) custom_relu_eval_out = self.eval_model(device, use_custom_op=True) # using PE - original_relu_eval_pe_out = self.eval_model(device, - use_custom_op=False, - use_pe=True) - custom_relu_eval_pe_out = self.eval_model(device, - use_custom_op=True, - use_pe=True) - - np.testing.assert_array_equal(original_relu_eval_out, - custom_relu_eval_out) - np.testing.assert_array_equal(original_relu_eval_pe_out, - custom_relu_eval_pe_out) + original_relu_eval_pe_out = self.eval_model( + device, use_custom_op=False, use_pe=True + ) + custom_relu_eval_pe_out = self.eval_model( + device, use_custom_op=True, use_pe=True + ) + + np.testing.assert_array_equal( + original_relu_eval_out, custom_relu_eval_out + ) + np.testing.assert_array_equal( + original_relu_eval_pe_out, custom_relu_eval_pe_out + ) def train_model(self, device, use_custom_op=False, use_pe=False): # reset random seed @@ -271,14 +293,15 @@ def train_model(self, device, use_custom_op=False, use_pe=False): paddle.set_device(device) with paddle.static.scope_guard(paddle.static.Scope()): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - x = paddle.static.data(shape=[None, self.in_dim], - name='x', - dtype='float32') - y = paddle.static.data(shape=[None, 1], - name='y', - dtype='float32') + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + x = paddle.static.data( + shape=[None, self.in_dim], name='x', dtype='float32' + ) + y = paddle.static.data( + shape=[None, 1], name='y', dtype='float32' + ) net = Net(self.in_dim, self.out_dim, use_custom_op) out = net(x) @@ -292,12 +315,14 @@ def train_model(self, device, use_custom_op=False, use_pe=False): # For PE if use_pe: - places = paddle.static.cpu_places( - ) if device == 'cpu' else paddle.static.cuda_places() + places = ( + paddle.static.cpu_places() + if device == 'cpu' + else paddle.static.cuda_places() + ) main_program = paddle.static.CompiledProgram( - paddle.static.default_main_program( - )).with_data_parallel(loss_name=loss.name, - places=places) + paddle.static.default_main_program() + ).with_data_parallel(loss_name=loss.name, places=places) else: main_program = paddle.static.default_main_program() @@ -305,17 +330,19 @@ def train_model(self, device, use_custom_op=False, use_pe=False): x_data = self.datas[batch_id] y_data = self.labels[batch_id] - res = exe.run(main_program, - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[out]) + res = exe.run( + main_program, + feed={'x': x_data, 'y': y_data}, + fetch_list=[out], + ) # save model paddle.static.save_inference_model( - self.model_path_template.format(use_custom_op, use_pe), [x], - [out], exe) + self.model_path_template.format(use_custom_op, use_pe), + [x], + [out], + exe, + ) return res[0] @@ -326,15 +353,20 @@ def eval_model(self, device, use_custom_op=False, use_pe=False): with paddle.static.program_guard(paddle.static.Program()): exe = paddle.static.Executor() - [inference_program, feed_target_names, - fetch_targets] = paddle.static.load_inference_model( - self.model_path_template.format(use_custom_op, use_pe), - exe) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model( + self.model_path_template.format(use_custom_op, use_pe), exe + ) x_data = self.datas[0] - results = exe.run(inference_program, - feed={feed_target_names[0]: x_data}, - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed={feed_target_names[0]: x_data}, + fetch_list=fetch_targets, + ) return results[0] diff --git a/python/paddle/fluid/tests/custom_op/test_custom_relu_op_jit.py b/python/paddle/fluid/tests/custom_op/test_custom_relu_op_jit.py index b1255bc3879b0..00b2860987b4d 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_relu_op_jit.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_relu_op_jit.py @@ -21,10 +21,12 @@ from utils import IS_MAC, extra_cc_args, extra_nvcc_args, paddle_includes from test_custom_relu_op_setup import custom_relu_dynamic, custom_relu_static from paddle.fluid.framework import _test_eager_guard + # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. file = '{}\\custom_relu_module_jit\\custom_relu_module_jit.pyd'.format( - get_build_directory()) + get_build_directory() +) if os.name == 'nt' and os.path.isfile(file): cmd = 'del {}'.format(file) run_cmd(cmd, True) @@ -43,16 +45,17 @@ extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, +) class TestJITLoad(unittest.TestCase): - def setUp(self): self.custom_ops = [ - custom_module.custom_relu, custom_module.custom_relu_dup, + custom_module.custom_relu, + custom_module.custom_relu_dup, custom_module.custom_relu_no_x_in_backward, - custom_module.custom_relu_out + custom_module.custom_relu_out, ] self.dtypes = ['float32', 'float64'] if paddle.is_compiled_with_cuda(): @@ -69,13 +72,16 @@ def test_static(self): x = np.random.uniform(-1, 1, [4, 8]).astype(dtype) for custom_op in self.custom_ops: out = custom_relu_static(custom_op, device, dtype, x) - pd_out = custom_relu_static(custom_op, device, dtype, x, - False) + pd_out = custom_relu_static( + custom_op, device, dtype, x, False + ) np.testing.assert_array_equal( out, pd_out, - err_msg='custom op out: {},\n paddle api out: {}'. - format(out, pd_out)) + err_msg='custom op out: {},\n paddle api out: {}'.format( + out, pd_out + ), + ) def func_dynamic(self): for device in self.devices: @@ -84,20 +90,26 @@ def func_dynamic(self): continue x = np.random.uniform(-1, 1, [4, 8]).astype(dtype) for custom_op in self.custom_ops: - out, x_grad = custom_relu_dynamic(custom_op, device, dtype, - x) + out, x_grad = custom_relu_dynamic( + custom_op, device, dtype, x + ) pd_out, pd_x_grad = custom_relu_dynamic( - custom_op, device, dtype, x, False) + custom_op, device, dtype, x, False + ) np.testing.assert_array_equal( out, pd_out, - err_msg='custom op out: {},\n paddle api out: {}'. - format(out, pd_out)) + err_msg='custom op out: {},\n paddle api out: {}'.format( + out, pd_out + ), + ) np.testing.assert_array_equal( x_grad, pd_x_grad, - err_msg='custom op x grad: {},\n paddle api x grad: {}'. - format(x_grad, pd_x_grad)) + err_msg='custom op x grad: {},\n paddle api x grad: {}'.format( + x_grad, pd_x_grad + ), + ) def test_dynamic(self): with _test_eager_guard(): @@ -141,7 +153,8 @@ def test_load_multiple_module(self): extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, + ) custom_conj = custom_module.custom_conj self.assertIsNotNone(custom_conj) diff --git a/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py b/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py index 32e34091d38a6..0f73b4a212413 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_relu_op_setup.py @@ -41,12 +41,9 @@ def custom_relu_dynamic(func, device, dtype, np_x, use_func=True): return out.numpy(), t.grad.numpy() -def custom_relu_static(func, - device, - dtype, - np_x, - use_func=True, - test_infer=False): +def custom_relu_static( + func, device, dtype, np_x, use_func=True, test_infer=False +): paddle.enable_static() paddle.set_device(device) @@ -60,9 +57,11 @@ def custom_relu_static(func, exe = static.Executor() exe.run(static.default_startup_program()) # in static mode, x data has been covered by out - out_v = exe.run(static.default_main_program(), - feed={'X': np_x}, - fetch_list=[out.name]) + out_v = exe.run( + static.default_main_program(), + feed={'X': np_x}, + fetch_list=[out.name], + ) paddle.disable_static() return out_v @@ -85,11 +84,11 @@ def custom_relu_static_pe(func, device, dtype, np_x, use_func=True): # in static mode, x data has been covered by out compiled_prog = static.CompiledProgram( - static.default_main_program()).with_data_parallel( - loss_name=out.name, places=places) - out_v = exe.run(compiled_prog, - feed={'X': np_x}, - fetch_list=[out.name]) + static.default_main_program() + ).with_data_parallel(loss_name=out.name, places=places) + out_v = exe.run( + compiled_prog, feed={'X': np_x}, fetch_list=[out.name] + ) paddle.disable_static() return out_v @@ -101,9 +100,9 @@ def custom_relu_static_inference(func, device, np_data, np_label, path_prefix): with static.scope_guard(static.Scope()): with static.program_guard(static.Program()): # simple module - data = static.data(name='data', - shape=[None, 1, 28, 28], - dtype='float32') + data = static.data( + name='data', shape=[None, 1, 28, 28], dtype='float32' + ) label = static.data(name='label', shape=[None, 1], dtype='int64') hidden = static.nn.fc(data, size=128) @@ -122,23 +121,21 @@ def custom_relu_static_inference(func, device, np_data, np_label, path_prefix): # train for i in range(4): - avg_loss_v = exe.run(static.default_main_program(), - feed={ - 'data': np_data, - 'label': np_label - }, - fetch_list=[avg_loss]) + avg_loss_v = exe.run( + static.default_main_program(), + feed={'data': np_data, 'label': np_label}, + fetch_list=[avg_loss], + ) # save inference model static.save_inference_model(path_prefix, [data], [predict], exe) # get train predict value - predict_v = exe.run(static.default_main_program(), - feed={ - 'data': np_data, - 'label': np_label - }, - fetch_list=[predict]) + predict_v = exe.run( + static.default_main_program(), + feed={'data': np_data, 'label': np_label}, + fetch_list=[predict], + ) return predict_v @@ -151,10 +148,9 @@ def custom_relu_double_grad_dynamic(func, device, dtype, np_x, use_func=True): out = func(t) if use_func else paddle.nn.functional.relu(t) out.stop_gradient = False - dx = paddle.grad(outputs=[out], - inputs=[t], - create_graph=True, - retain_graph=True) + dx = paddle.grad( + outputs=[out], inputs=[t], create_graph=True, retain_graph=True + ) dx[0].backward() @@ -163,16 +159,17 @@ def custom_relu_double_grad_dynamic(func, device, dtype, np_x, use_func=True): class TestNewCustomOpSetUpInstall(unittest.TestCase): - def setUp(self): cur_dir = os.path.dirname(os.path.abspath(__file__)) # compile, install the custom op egg into site-packages under background if os.name == 'nt': cmd = 'cd /d {} && python custom_relu_setup.py install'.format( - cur_dir) + cur_dir + ) else: cmd = 'cd {} && {} custom_relu_setup.py install'.format( - cur_dir, sys.executable) + cur_dir, sys.executable + ) run_cmd(cmd) # NOTE(Aurelius84): Normally, it's no need to add following codes for users. @@ -188,16 +185,18 @@ def setUp(self): custom_egg_path = [ x for x in os.listdir(site_dir) if 'custom_relu_module_setup' in x ] - assert len(custom_egg_path - ) == 1, "Matched egg number is %d." % len(custom_egg_path) + assert len(custom_egg_path) == 1, "Matched egg number is %d." % len( + custom_egg_path + ) sys.path.append(os.path.join(site_dir, custom_egg_path[0])) # usage: import the package directly import custom_relu_module_setup + # `custom_relu_dup` is same as `custom_relu_dup` self.custom_ops = [ custom_relu_module_setup.custom_relu, - custom_relu_module_setup.custom_relu_dup + custom_relu_module_setup.custom_relu_dup, ] self.dtypes = ['float32', 'float64'] @@ -220,13 +219,16 @@ def test_static(self): x = np.random.uniform(-1, 1, [4, 8]).astype(dtype) for custom_op in self.custom_ops: out = custom_relu_static(custom_op, device, dtype, x) - pd_out = custom_relu_static(custom_op, device, dtype, x, - False) + pd_out = custom_relu_static( + custom_op, device, dtype, x, False + ) np.testing.assert_array_equal( out, pd_out, - err_msg='custom op out: {},\n paddle api out: {}'. - format(out, pd_out)) + err_msg='custom op out: {},\n paddle api out: {}'.format( + out, pd_out + ), + ) def test_static_pe(self): for device in self.devices: @@ -236,13 +238,16 @@ def test_static_pe(self): x = np.random.uniform(-1, 1, [4, 8]).astype(dtype) for custom_op in self.custom_ops: out = custom_relu_static_pe(custom_op, device, dtype, x) - pd_out = custom_relu_static_pe(custom_op, device, dtype, x, - False) + pd_out = custom_relu_static_pe( + custom_op, device, dtype, x, False + ) np.testing.assert_array_equal( out, pd_out, - err_msg='custom op out: {},\n paddle api out: {}'. - format(out, pd_out)) + err_msg='custom op out: {},\n paddle api out: {}'.format( + out, pd_out + ), + ) def func_dynamic(self): for device in self.devices: @@ -251,20 +256,26 @@ def func_dynamic(self): continue x = np.random.uniform(-1, 1, [4, 8]).astype(dtype) for custom_op in self.custom_ops: - out, x_grad = custom_relu_dynamic(custom_op, device, dtype, - x) + out, x_grad = custom_relu_dynamic( + custom_op, device, dtype, x + ) pd_out, pd_x_grad = custom_relu_dynamic( - custom_op, device, dtype, x, False) + custom_op, device, dtype, x, False + ) np.testing.assert_array_equal( out, pd_out, - err_msg='custom op out: {},\n paddle api out: {}'. - format(out, pd_out)) + err_msg='custom op out: {},\n paddle api out: {}'.format( + out, pd_out + ), + ) np.testing.assert_array_equal( x_grad, pd_x_grad, - err_msg='custom op x grad: {},\n paddle api x grad: {}'. - format(x_grad, pd_x_grad)) + err_msg='custom op x grad: {},\n paddle api x grad: {}'.format( + x_grad, pd_x_grad + ), + ) def test_dynamic(self): with _test_eager_guard(): @@ -277,22 +288,29 @@ def test_static_save_and_load_inference_model(self): np_label = np.random.random((1, 1)).astype("int64") path_prefix = "custom_op_inference/custom_relu" for device in self.devices: - predict = custom_relu_static_inference(self.custom_ops[0], device, - np_data, np_label, - path_prefix) + predict = custom_relu_static_inference( + self.custom_ops[0], device, np_data, np_label, path_prefix + ) # load inference model with static.scope_guard(static.Scope()): exe = static.Executor() - [inference_program, feed_target_names, - fetch_targets] = static.load_inference_model(path_prefix, exe) - predict_infer = exe.run(inference_program, - feed={feed_target_names[0]: np_data}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = static.load_inference_model(path_prefix, exe) + predict_infer = exe.run( + inference_program, + feed={feed_target_names[0]: np_data}, + fetch_list=fetch_targets, + ) np.testing.assert_array_equal( predict, predict_infer, - err_msg='custom op predict: {},\n custom op infer predict: {}' - .format(predict, predict_infer)) + err_msg='custom op predict: {},\n custom op infer predict: {}'.format( + predict, predict_infer + ), + ) paddle.disable_static() def test_static_save_and_run_inference_predictor(self): @@ -302,26 +320,32 @@ def test_static_save_and_run_inference_predictor(self): path_prefix = "custom_op_inference/custom_relu" from paddle.inference import Config from paddle.inference import create_predictor + for device in self.devices: - predict = custom_relu_static_inference(self.custom_ops[0], device, - np_data, np_label, - path_prefix) + predict = custom_relu_static_inference( + self.custom_ops[0], device, np_data, np_label, path_prefix + ) # load inference model - config = Config(path_prefix + ".pdmodel", - path_prefix + ".pdiparams") + config = Config( + path_prefix + ".pdmodel", path_prefix + ".pdiparams" + ) predictor = create_predictor(config) input_tensor = predictor.get_input_handle( - predictor.get_input_names()[0]) + predictor.get_input_names()[0] + ) input_tensor.reshape(np_data.shape) input_tensor.copy_from_cpu(np_data.copy()) predictor.run() output_tensor = predictor.get_output_handle( - predictor.get_output_names()[0]) + predictor.get_output_names()[0] + ) predict_infer = output_tensor.copy_to_cpu() self.assertTrue( np.isclose(predict, predict_infer, rtol=5e-5).any(), "custom op predict: {},\n custom op infer predict: {}".format( - predict, predict_infer)) + predict, predict_infer + ), + ) paddle.disable_static() def test_func_double_grad_dynamic(self): @@ -331,33 +355,43 @@ def test_func_double_grad_dynamic(self): continue x = np.random.uniform(-1, 1, [4, 8]).astype(dtype) out, dx_grad = custom_relu_double_grad_dynamic( - self.custom_ops[0], device, dtype, x) + self.custom_ops[0], device, dtype, x + ) pd_out, pd_dx_grad = custom_relu_double_grad_dynamic( - self.custom_ops[0], device, dtype, x, False) + self.custom_ops[0], device, dtype, x, False + ) np.testing.assert_array_equal( out, pd_out, err_msg='custom op out: {},\n paddle api out: {}'.format( - out, pd_out)) + out, pd_out + ), + ) np.testing.assert_array_equal( dx_grad, pd_dx_grad, - err_msg='custom op dx grad: {},\n paddle api dx grad: {}'. - format(dx_grad, pd_dx_grad)) + err_msg='custom op dx grad: {},\n paddle api dx grad: {}'.format( + dx_grad, pd_dx_grad + ), + ) def test_with_dataloader(self): for device in self.devices: paddle.set_device(device) # data loader transform = Compose( - [Normalize(mean=[127.5], std=[127.5], data_format='CHW')]) - train_dataset = paddle.vision.datasets.MNIST(mode='train', - transform=transform) - train_loader = paddle.io.DataLoader(train_dataset, - batch_size=64, - shuffle=True, - drop_last=True, - num_workers=0) + [Normalize(mean=[127.5], std=[127.5], data_format='CHW')] + ) + train_dataset = paddle.vision.datasets.MNIST( + mode='train', transform=transform + ) + train_loader = paddle.io.DataLoader( + train_dataset, + batch_size=64, + shuffle=True, + drop_last=True, + num_workers=0, + ) for batch_id, (image, _) in enumerate(train_loader()): out = self.custom_ops[0](image) @@ -366,7 +400,9 @@ def test_with_dataloader(self): out, pd_out, err_msg='custom op out: {},\n paddle api out: {}'.format( - out, pd_out)) + out, pd_out + ), + ) if batch_id == 5: break diff --git a/python/paddle/fluid/tests/custom_op/test_custom_simple_slice.py b/python/paddle/fluid/tests/custom_op/test_custom_simple_slice.py index e7d60fd4296eb..b67b09131fa0c 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_simple_slice.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_simple_slice.py @@ -25,7 +25,8 @@ # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. file = '{}\\custom_simple_slice\\custom_simple_slice.pyd'.format( - get_build_directory()) + get_build_directory() +) if os.name == 'nt' and os.path.isfile(file): cmd = 'del {}'.format(file) run_cmd(cmd, True) @@ -36,11 +37,11 @@ extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, +) class TestCustomSimpleSliceJit(unittest.TestCase): - def func_slice_output(self): np_x = np.random.random((5, 2)).astype("float32") x = paddle.to_tensor(np_x) @@ -49,8 +50,10 @@ def func_slice_output(self): np.testing.assert_array_equal( custom_op_out, np_out, - err_msg='custom op: {},\n numpy: {}'.format(np_out, - custom_op_out.numpy())) + err_msg='custom op: {},\n numpy: {}'.format( + np_out, custom_op_out.numpy() + ), + ) def test_slice_output(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py b/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py index fb65422758f38..d3fec61f4bdda 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py @@ -36,7 +36,8 @@ extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cc flags extra_cuda_cflags=extra_nvcc_args, # test for nvcc flags - verbose=True) + verbose=True, +) def custom_tanh_double_grad_dynamic(func, device, dtype, np_x): @@ -48,10 +49,9 @@ def custom_tanh_double_grad_dynamic(func, device, dtype, np_x): out = func(t) out.stop_gradient = False - dx = paddle.grad(outputs=[out], - inputs=[t], - create_graph=True, - retain_graph=True) + dx = paddle.grad( + outputs=[out], inputs=[t], create_graph=True, retain_graph=True + ) dx[0].backward() @@ -62,7 +62,6 @@ def custom_tanh_double_grad_dynamic(func, device, dtype, np_x): class TestCustomTanhDoubleGradJit(unittest.TestCase): - def setUp(self): paddle.set_device('cpu') self.dtypes = ['float32', 'float64'] @@ -73,27 +72,35 @@ def func_double_grad_dynamic(self): for dtype in self.dtypes: x = np.random.uniform(-1, 1, [4, 8]).astype(dtype) out, dx_grad, dout = custom_tanh_double_grad_dynamic( - custom_ops.custom_tanh, device, dtype, x) + custom_ops.custom_tanh, device, dtype, x + ) pd_out, pd_dx_grad, pd_dout = custom_tanh_double_grad_dynamic( - paddle.tanh, device, dtype, x) + paddle.tanh, device, dtype, x + ) np.testing.assert_allclose( out, pd_out, rtol=1e-05, err_msg='custom op out: {},\n paddle api out: {}'.format( - out, pd_out)) + out, pd_out + ), + ) np.testing.assert_allclose( dx_grad, pd_dx_grad, rtol=1e-05, - err_msg='custom op dx grad: {},\n paddle api dx grad: {}'. - format(dx_grad, pd_dx_grad)) + err_msg='custom op dx grad: {},\n paddle api dx grad: {}'.format( + dx_grad, pd_dx_grad + ), + ) np.testing.assert_allclose( dout, pd_dout, rtol=1e-05, - err_msg='custom op out grad: {},\n paddle api out grad: {}'. - format(dout, pd_dout)) + err_msg='custom op out grad: {},\n paddle api out grad: {}'.format( + dout, pd_dout + ), + ) def test_func_double_grad_dynamic(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) diff --git a/python/paddle/fluid/tests/custom_op/test_dispatch_jit.py b/python/paddle/fluid/tests/custom_op/test_dispatch_jit.py index ff5192db7aafc..a5bbb405fa409 100644 --- a/python/paddle/fluid/tests/custom_op/test_dispatch_jit.py +++ b/python/paddle/fluid/tests/custom_op/test_dispatch_jit.py @@ -20,6 +20,7 @@ from utils import paddle_includes, extra_cc_args from paddle.utils.cpp_extension.extension_utils import run_cmd from paddle.fluid.framework import _test_eager_guard + # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. file = '{}\\dispatch_op\\dispatch_op.pyd'.format(get_build_directory()) @@ -32,11 +33,11 @@ sources=['dispatch_test_op.cc'], extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, - verbose=True) + verbose=True, +) class TestJitDispatch(unittest.TestCase): - def setUp(self): paddle.set_device('cpu') @@ -50,7 +51,8 @@ def run_dispatch_test_impl(self, func, dtype): np.testing.assert_array_equal( np_x, np_out, - err_msg='custom op x: {},\n custom op out: {}'.format(np_x, np_out)) + err_msg='custom op x: {},\n custom op out: {}'.format(np_x, np_out), + ) def run_dispatch_test(self, func, dtype): with _test_eager_guard(): @@ -69,32 +71,49 @@ def test_dispatch_complex(self): def test_dispatch_float_and_integer(self): dtypes = [ - "float32", "float64", "int32", "int64", "int8", "uint8", "int16" + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "int16", ] for dtype in dtypes: - self.run_dispatch_test(dispatch_op.dispatch_test_float_and_integer, - dtype) + self.run_dispatch_test( + dispatch_op.dispatch_test_float_and_integer, dtype + ) def test_dispatch_float_and_complex(self): dtypes = ["float32", "float64", "complex64", "complex128"] for dtype in dtypes: - self.run_dispatch_test(dispatch_op.dispatch_test_float_and_complex, - dtype) + self.run_dispatch_test( + dispatch_op.dispatch_test_float_and_complex, dtype + ) def test_dispatch_float_and_integer_and_complex(self): dtypes = [ - "float32", "float64", "int32", "int64", "int8", "uint8", "int16", - "complex64", "complex128" + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "int16", + "complex64", + "complex128", ] for dtype in dtypes: self.run_dispatch_test( - dispatch_op.dispatch_test_float_and_integer_and_complex, dtype) + dispatch_op.dispatch_test_float_and_integer_and_complex, dtype + ) def test_dispatch_float_and_half(self): dtypes = ["float32", "float64", "float16"] for dtype in dtypes: - self.run_dispatch_test(dispatch_op.dispatch_test_float_and_half, - dtype) + self.run_dispatch_test( + dispatch_op.dispatch_test_float_and_half, dtype + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/custom_op/test_multi_out_jit.py b/python/paddle/fluid/tests/custom_op/test_multi_out_jit.py index eb9b9e4677fd2..7b6c30cabac36 100644 --- a/python/paddle/fluid/tests/custom_op/test_multi_out_jit.py +++ b/python/paddle/fluid/tests/custom_op/test_multi_out_jit.py @@ -22,6 +22,7 @@ from paddle.utils.cpp_extension.extension_utils import run_cmd from utils import paddle_includes, extra_cc_args from paddle.fluid.framework import _test_eager_guard + # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. file = '{}\\multi_out_jit\\multi_out_jit.pyd'.format(get_build_directory()) @@ -35,11 +36,11 @@ sources=['multi_out_test_op.cc'], extra_include_paths=paddle_includes, # add for Coverage CI extra_cxx_cflags=extra_cc_args, # test for cflags - verbose=True) + verbose=True, +) class TestMultiOutputDtypes(unittest.TestCase): - def setUp(self): self.custom_op = multi_out_module.multi_out self.dtypes = ['float32', 'float64'] @@ -56,9 +57,11 @@ def run_static(self, device, dtype): exe = paddle.static.Executor() exe.run(paddle.static.default_startup_program()) - res = exe.run(paddle.static.default_main_program(), - feed={'X': x_data}, - fetch_list=outs) + res = exe.run( + paddle.static.default_main_program(), + feed={'X': x_data}, + fetch_list=outs, + ) return res @@ -69,12 +72,14 @@ def check_multi_outputs(self, outs, is_dynamic=False): one_int32 = one_int32.numpy() # Fake_float64 self.assertTrue('float64' in str(zero_float64.dtype)) - np.testing.assert_array_equal(zero_float64, - np.zeros([4, 8]).astype('float64')) + np.testing.assert_array_equal( + zero_float64, np.zeros([4, 8]).astype('float64') + ) # ZFake_int32 self.assertTrue('int32' in str(one_int32.dtype)) - np.testing.assert_array_equal(one_int32, - np.ones([4, 8]).astype('int32')) + np.testing.assert_array_equal( + one_int32, np.ones([4, 8]).astype('int32') + ) def test_static(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/custom_op/test_sysconfig.py b/python/paddle/fluid/tests/custom_op/test_sysconfig.py index 89ef36931f3ff..78c0cdf0316ea 100644 --- a/python/paddle/fluid/tests/custom_op/test_sysconfig.py +++ b/python/paddle/fluid/tests/custom_op/test_sysconfig.py @@ -18,7 +18,6 @@ class SysConfigTest(unittest.TestCase): - def test_include(self): inc_dir = paddle.sysconfig.get_include() inc_dirs = inc_dir.split(os.sep) diff --git a/python/paddle/fluid/tests/custom_op/utils.py b/python/paddle/fluid/tests/custom_op/utils.py index adce68cd0f969..c21a1bb1624f4 100644 --- a/python/paddle/fluid/tests/custom_op/utils.py +++ b/python/paddle/fluid/tests/custom_op/utils.py @@ -26,7 +26,7 @@ # PaddlePaddle whl. So here we specific `include_dirs` to avoid errors in CI. paddle_includes = [ os.path.join(site_packages_path, 'paddle', 'include'), - os.path.join(site_packages_path, 'paddle', 'include', 'third_party') + os.path.join(site_packages_path, 'paddle', 'include', 'third_party'), ] # Test for extra compile args diff --git a/python/paddle/fluid/tests/custom_runtime/custom_device_multi_process_collective.py b/python/paddle/fluid/tests/custom_runtime/custom_device_multi_process_collective.py index 21e9871042691..b658b885176f3 100644 --- a/python/paddle/fluid/tests/custom_runtime/custom_device_multi_process_collective.py +++ b/python/paddle/fluid/tests/custom_runtime/custom_device_multi_process_collective.py @@ -27,12 +27,21 @@ def train(prefix): device_ids = os.getenv("PADDLE_WORLD_DEVICE_IDS") current_device_id = os.getenv("PADDLE_LOCAL_DEVICE_IDS") - details = "selected_accelerators:{} selected_custom_devices:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{} device_ids:{} device_id:{}"\ - .format(selected_accelerators, selected_custom_devices, worker_endpoints, trainers_num, current_endpoint,trainer_id,device_ids, current_device_id) + details = "selected_accelerators:{} selected_custom_devices:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{} device_ids:{} device_id:{}".format( + selected_accelerators, + selected_custom_devices, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + device_ids, + current_device_id, + ) print(details) - with open("multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + with open( + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(details) diff --git a/python/paddle/fluid/tests/custom_runtime/process_group_xccl.py b/python/paddle/fluid/tests/custom_runtime/process_group_xccl.py index b830d0eccf526..201b2d3df8657 100644 --- a/python/paddle/fluid/tests/custom_runtime/process_group_xccl.py +++ b/python/paddle/fluid/tests/custom_runtime/process_group_xccl.py @@ -29,15 +29,16 @@ def init_process_group(strategy=None): is_master = True if rank == 0 else False store = paddle.fluid.core.TCPStore("127.0.0.1", 6173, is_master, nranks) pg_group = core.ProcessGroupCustom( - store, rank, nranks, - paddle.CustomPlace(ParallelEnv().device_type, - ParallelEnv().device_id)) + store, + rank, + nranks, + paddle.CustomPlace(ParallelEnv().device_type, ParallelEnv().device_id), + ) return pg_group class TestProcessGroupFp32(unittest.TestCase): - def setUp(self): paddle.seed(2022) random.seed(2022) @@ -50,8 +51,9 @@ def config(self): def test_create_process_group_xccl(self): with _test_eager_guard(): - paddle.set_device('custom_cpu:%d' % - paddle.distributed.ParallelEnv().dev_id) + paddle.set_device( + 'custom_cpu:%d' % paddle.distributed.ParallelEnv().dev_id + ) pg = init_process_group() @@ -147,8 +149,9 @@ def test_create_process_group_xccl(self): task.wait() # paddle.fluid.core._custom_device_synchronize("custom_cpu", -1) out_1 = paddle.slice(tensor_out, [0], [0], [out_shape[0] // 2]) - out_2 = paddle.slice(tensor_out, [0], [out_shape[0] // 2], - [out_shape[0]]) + out_2 = paddle.slice( + tensor_out, [0], [out_shape[0] // 2], [out_shape[0]] + ) # assert np.array_equal(tensor_x, out_1) # assert np.array_equal(tensor_y, out_2) print("test allgather api ok\n") @@ -163,10 +166,12 @@ def test_create_process_group_xccl(self): tensor_y = paddle.to_tensor(y) tensor_out1 = paddle.to_tensor(out1) tensor_out2 = paddle.to_tensor(out2) - raw_tensor_x_2 = paddle.slice(tensor_x, [0], [self.shape[0] // 2], - [self.shape[0]]) - raw_tensor_y_1 = paddle.slice(tensor_y, [0], [0], - [self.shape[0] // 2]) + raw_tensor_x_2 = paddle.slice( + tensor_x, [0], [self.shape[0] // 2], [self.shape[0]] + ) + raw_tensor_y_1 = paddle.slice( + tensor_y, [0], [0], [self.shape[0] // 2] + ) if pg.rank() == 0: task = pg.alltoall(tensor_x, tensor_out1) task.wait() @@ -176,8 +181,9 @@ def test_create_process_group_xccl(self): task = pg.alltoall(tensor_y, tensor_out2) task.wait() # paddle.fluid.core._custom_device_synchronize("custom_cpu", -1) - out1_2 = paddle.slice(tensor_out1, [0], [self.shape[0] // 2], - [self.shape[0]]) + out1_2 = paddle.slice( + tensor_out1, [0], [self.shape[0] // 2], [self.shape[0]] + ) out2_1 = paddle.slice(tensor_out2, [0], [0], [self.shape[0] // 2]) # if pg.rank() == 0: # assert np.array_equal(out1_2.numpy(), raw_tensor_y_1.numpy()) @@ -223,8 +229,9 @@ def test_create_process_group_xccl(self): task.wait() # paddle.fluid.core._custom_device_synchronize("custom_cpu", -1) out1 = paddle.slice(tensor_x, [0], [0], [self.shape[0]]) - out2 = paddle.slice(tensor_x, [0], [self.shape[0]], - [self.shape[0] * 2]) + out2 = paddle.slice( + tensor_x, [0], [self.shape[0]], [self.shape[0] * 2] + ) # if pg.rank() == 0: # assert np.array_equal(tensor_y, out1) # else: diff --git a/python/paddle/fluid/tests/custom_runtime/test_collective_process_group_xccl.py b/python/paddle/fluid/tests/custom_runtime/test_collective_process_group_xccl.py index 1127352d85d99..586e0e322ab38 100644 --- a/python/paddle/fluid/tests/custom_runtime/test_collective_process_group_xccl.py +++ b/python/paddle/fluid/tests/custom_runtime/test_collective_process_group_xccl.py @@ -20,19 +20,26 @@ import tempfile -def start_local_trainers(cluster, - pod, - training_script, - training_script_args, - eager_mode=True, - log_dir=None): - from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc # noqa: F401 +def start_local_trainers( + cluster, + pod, + training_script, + training_script_args, + eager_mode=True, + log_dir=None, +): + from paddle.distributed.utils.launch_utils import ( # noqa: F401 + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, + ) current_env = copy.copy(os.environ.copy()) - #paddle broadcast ncclUniqueId use socket, and - #proxy maybe make trainers unreachable, so delete them. - #if we set them to "", grpc will log error message "bad uri" - #so just delete them. + # paddle broadcast ncclUniqueId use socket, and + # proxy maybe make trainers unreachable, so delete them. + # if we set them to "", grpc will log error message "bad uri" + # so just delete them. current_env.pop("http_proxy", None) current_env.pop("https_proxy", None) @@ -41,8 +48,8 @@ def start_local_trainers(cluster, os.system("rm -rf log && mkdir -p log") for idx, t in enumerate(pod.trainers): proc_env = { - "FLAGS_selected_custom_cpus": - "%s" % ",".join([str(g) for g in t.gpus]), + "FLAGS_selected_custom_cpus": "%s" + % ",".join([str(g) for g in t.gpus]), "PADDLE_TRAINER_ID": "%d" % t.rank, "PADDLE_CURRENT_ENDPOINT": "%s" % t.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), @@ -65,10 +72,9 @@ def start_local_trainers(cluster, print("start trainer proc:{} env:{}".format(cmd, proc_env)) fn = open("workerlog.%d" % idx, "a") - proc = subprocess.Popen(cmd.split(" "), - env=current_env, - stdout=fn, - stderr=fn) + proc = subprocess.Popen( + cmd.split(" "), env=current_env, stdout=fn, stderr=fn + ) tp = TrainerProc() tp.proc = proc @@ -82,7 +88,12 @@ def start_local_trainers(cluster, def get_cluster_from_args(selected_gpus): - from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc # noqa: F401 + from paddle.distributed.utils.launch_utils import ( # noqa: F401 + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, + ) cluster_node_ips = '127.0.0.1' node_ip = '127.0.0.1' @@ -104,9 +115,13 @@ def get_cluster_from_args(selected_gpus): class TestMultipleCustomCPU(unittest.TestCase): - def run_mnist_2custom_cpu(self, target_file_name, eager_mode=True): - from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc # noqa: F401 + from paddle.distributed.utils.launch_utils import ( # noqa: F401 + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, + ) selected_devices = [0, 1] cluster = None @@ -114,11 +129,13 @@ def run_mnist_2custom_cpu(self, target_file_name, eager_mode=True): cluster, pod = get_cluster_from_args(selected_devices) - procs = start_local_trainers(cluster, - pod, - eager_mode=eager_mode, - training_script=target_file_name, - training_script_args=[]) + procs = start_local_trainers( + cluster, + pod, + eager_mode=eager_mode, + training_script=target_file_name, + training_script_args=[], + ) while True: alive = watch_local_trainers(procs, cluster.trainers_endpoints()) @@ -130,7 +147,6 @@ def run_mnist_2custom_cpu(self, target_file_name, eager_mode=True): class TestProcessGroup(TestMultipleCustomCPU): - def setUp(self): # compile so and set to current path cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -142,15 +158,18 @@ def setUp(self): && git checkout {} -b dev \ && cd backends/custom_cpu \ && mkdir build && cd build && cmake .. && make -j8'.format( - self.temp_dir.name, os.getenv('PLUGIN_URL'), - os.getenv('PLUGIN_TAG')) + self.temp_dir.name, os.getenv('PLUGIN_URL'), os.getenv('PLUGIN_TAG') + ) os.system(cmd) # set environment for loading and registering compiled custom kernels # only valid in current process os.environ['CUSTOM_DEVICE_ROOT'] = os.path.join( - cur_dir, '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( - self.temp_dir.name)) + cur_dir, + '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( + self.temp_dir.name + ), + ) os.environ['FLAGS_selected_custom_cpus'] = '0,1' os.environ['CUSTOM_CPU_VISIBLE_DEVICES'] = '0,1' os.environ['PADDLE_XCCL_BACKEND'] = 'custom_cpu' @@ -159,7 +178,12 @@ def tearDown(self): self.temp_dir.cleanup() def test_process_group_xccl(self): - from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc # noqa: F401 + from paddle.distributed.utils.launch_utils import ( # noqa: F401 + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, + ) self.run_mnist_2custom_cpu('process_group_xccl.py') diff --git a/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_plugin.py b/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_plugin.py index f1cfa8ef7392a..5d0e5ccc475bd 100644 --- a/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_plugin.py +++ b/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_plugin.py @@ -20,7 +20,6 @@ class TestCustomCPUPlugin(unittest.TestCase): - def setUp(self): # compile so and set to current path cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -32,15 +31,18 @@ def setUp(self): && git checkout {} -b dev \ && cd backends/custom_cpu \ && mkdir build && cd build && cmake .. && make -j8'.format( - self.temp_dir.name, os.getenv('PLUGIN_URL'), - os.getenv('PLUGIN_TAG')) + self.temp_dir.name, os.getenv('PLUGIN_URL'), os.getenv('PLUGIN_TAG') + ) os.system(cmd) # set environment for loading and registering compiled custom kernels # only valid in current process os.environ['CUSTOM_DEVICE_ROOT'] = os.path.join( - cur_dir, '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( - self.temp_dir.name)) + cur_dir, + '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( + self.temp_dir.name + ), + ) def tearDown(self): self.temp_dir.cleanup() @@ -65,16 +67,18 @@ def _test_custom_device_dataloader(self): paddle.set_device('custom_cpu') dataset = paddle.vision.datasets.MNIST( mode='test', - transform=paddle.vision.transforms.Compose([ - paddle.vision.transforms.CenterCrop(20), - paddle.vision.transforms.RandomResizedCrop(14), - paddle.vision.transforms.Normalize(), - paddle.vision.transforms.ToTensor() - ])) - loader = paddle.io.DataLoader(dataset, - batch_size=32, - num_workers=1, - shuffle=True) + transform=paddle.vision.transforms.Compose( + [ + paddle.vision.transforms.CenterCrop(20), + paddle.vision.transforms.RandomResizedCrop(14), + paddle.vision.transforms.Normalize(), + paddle.vision.transforms.ToTensor(), + ] + ), + ) + loader = paddle.io.DataLoader( + dataset, batch_size=32, num_workers=1, shuffle=True + ) for image, label in loader: self.assertTrue(image.place.is_custom_place()) self.assertTrue(label.place.is_custom_place()) @@ -84,13 +88,13 @@ def _test_custom_device_mnist(self): import paddle class MNIST(paddle.nn.Layer): - def __init__(self): super(MNIST, self).__init__() self.shape = 1 * 28 * 28 self.size = 10 self.output_weight = self.create_parameter( - [self.shape, self.size]) + [self.shape, self.size] + ) self.accuracy = paddle.metric.Accuracy() def forward(self, inputs, label=None): @@ -110,15 +114,17 @@ def forward(self, inputs, label=None): dataset = paddle.vision.datasets.MNIST( mode='train', transform=paddle.vision.transforms.Compose( - [paddle.vision.transforms.ToTensor()])) - loader = paddle.io.DataLoader(dataset, - batch_size=64, - num_workers=1, - shuffle=True) + [paddle.vision.transforms.ToTensor()] + ), + ) + loader = paddle.io.DataLoader( + dataset, batch_size=64, num_workers=1, shuffle=True + ) mnist = MNIST() - sgd = paddle.optimizer.SGD(learning_rate=0.01, - parameters=mnist.parameters()) + sgd = paddle.optimizer.SGD( + learning_rate=0.01, parameters=mnist.parameters() + ) data = next(loader()) img = data[0] @@ -139,6 +145,7 @@ def _test_eager_backward_api(self): grad = np.ones([2, 2]).astype("float32") import paddle + paddle.set_device('custom_cpu') paddle.device.get_available_device() x_tensor = paddle.to_tensor(x, stop_gradient=False) @@ -153,18 +160,21 @@ def _test_eager_backward_api(self): def _test_eager_copy_to(self): import paddle + x = np.random.random([2, 2]).astype("float32") # cpu -> custom - cpu_tensor = paddle.to_tensor(x, - dtype='float32', - place=paddle.CPUPlace()) + cpu_tensor = paddle.to_tensor( + x, dtype='float32', place=paddle.CPUPlace() + ) custom_cpu_tensor = cpu_tensor._copy_to( - paddle.CustomPlace('custom_cpu', 0), True) + paddle.CustomPlace('custom_cpu', 0), True + ) np.testing.assert_array_equal(custom_cpu_tensor, x) self.assertTrue(custom_cpu_tensor.place.is_custom_place()) # custom -> custom another_custom_cpu_tensor = custom_cpu_tensor._copy_to( - paddle.CustomPlace('custom_cpu', 0), True) + paddle.CustomPlace('custom_cpu', 0), True + ) np.testing.assert_array_equal(another_custom_cpu_tensor, x) self.assertTrue(another_custom_cpu_tensor.place.is_custom_place()) # custom -> cpu @@ -173,13 +183,15 @@ def _test_eager_copy_to(self): self.assertTrue(another_cpu_tensor.place.is_cpu_place()) # custom -> custom self another_custom_cpu_tensor = another_custom_cpu_tensor._copy_to( - paddle.CustomPlace('custom_cpu', 0), True) + paddle.CustomPlace('custom_cpu', 0), True + ) np.testing.assert_array_equal(another_custom_cpu_tensor, x) self.assertTrue(another_custom_cpu_tensor.place.is_custom_place()) def _test_fallback_kernel(self): # using (custom_cpu, add, int16) which is not registered import paddle + r = np.array([6, 6, 6], 'int16') x = paddle.to_tensor([5, 4, 3], 'int16') y = paddle.to_tensor([1, 2, 3], 'int16') @@ -188,8 +200,10 @@ def _test_fallback_kernel(self): def _test_scalar(self): import paddle - data_1 = paddle.to_tensor([[[[1.0, 4.0, 5.0, 7.0], [3.0, 4.0, 5.0, - 6.0]]]]) + + data_1 = paddle.to_tensor( + [[[[1.0, 4.0, 5.0, 7.0], [3.0, 4.0, 5.0, 6.0]]]] + ) k_t = paddle.to_tensor([3], dtype="int32") value_1, indices_1 = paddle.topk(data_1, k=k_t) diff --git a/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_profiler_plugin.py b/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_profiler_plugin.py index 2e307fbb826b5..82c2ddbbe73e7 100644 --- a/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_profiler_plugin.py +++ b/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_profiler_plugin.py @@ -19,7 +19,6 @@ class TestCustomCPUProfilerPlugin(unittest.TestCase): - def setUp(self): # compile so and set to current path cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -31,15 +30,18 @@ def setUp(self): && git checkout {} -b dev \ && cd backends/custom_cpu \ && mkdir build && cd build && cmake .. && make -j8'.format( - self.temp_dir.name, os.getenv('PLUGIN_URL'), - os.getenv('PLUGIN_TAG')) + self.temp_dir.name, os.getenv('PLUGIN_URL'), os.getenv('PLUGIN_TAG') + ) os.system(cmd) # set environment for loading and registering compiled custom kernels # only valid in current process os.environ['CUSTOM_DEVICE_ROOT'] = os.path.join( - cur_dir, '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( - self.temp_dir.name)) + cur_dir, + '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( + self.temp_dir.name + ), + ) def tearDown(self): self.temp_dir.cleanup() @@ -47,6 +49,7 @@ def tearDown(self): def test_custom_device(self): import paddle + with paddle.fluid.framework._test_eager_guard(): self._test_custom_profiler() @@ -57,9 +60,12 @@ def _test_custom_profiler(self): paddle.set_device('custom_cpu') x = paddle.to_tensor([1, 2, 3]) - p = profiler.Profiler(targets=[ - profiler.ProfilerTarget.CPU, profiler.ProfilerTarget.CUSTOM_DEVICE - ]) + p = profiler.Profiler( + targets=[ + profiler.ProfilerTarget.CPU, + profiler.ProfilerTarget.CUSTOM_DEVICE, + ] + ) p.start() for iter in range(10): x = x + 1 diff --git a/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_to_static.py b/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_to_static.py index e5e9638a0b531..2d8796152b588 100644 --- a/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_to_static.py +++ b/python/paddle/fluid/tests/custom_runtime/test_custom_cpu_to_static.py @@ -35,8 +35,11 @@ def train_func_base(epoch_id, train_loader, model, cost, optimizer): loss.backward() optimizer.step() optimizer.clear_grad() - print("Epoch [{}/{}], Step [{}/{}], Loss: {}".format( - epoch_id + 1, EPOCH_NUM, batch_id + 1, total_step, loss.numpy())) + print( + "Epoch [{}/{}], Step [{}/{}], Loss: {}".format( + epoch_id + 1, EPOCH_NUM, batch_id + 1, total_step, loss.numpy() + ) + ) epoch_end = time.time() print( f"Epoch ID: {epoch_id+1}, FP32 train epoch time: {(epoch_end - epoch_start) * 1000} ms" @@ -51,8 +54,9 @@ def train_func_ampo1(epoch_id, train_loader, model, cost, optimizer, scaler): for batch_id, (images, labels) in enumerate(train_loader()): # forward with paddle.amp.auto_cast( - custom_black_list={"flatten_contiguous_range", "greater_than"}, - level='O1'): + custom_black_list={"flatten_contiguous_range", "greater_than"}, + level='O1', + ): outputs = model(images) loss = cost(outputs, labels) # backward and optimize @@ -60,8 +64,11 @@ def train_func_ampo1(epoch_id, train_loader, model, cost, optimizer, scaler): scaled.backward() scaler.minimize(optimizer, scaled) optimizer.clear_grad() - print("Epoch [{}/{}], Step [{}/{}], Loss: {}".format( - epoch_id + 1, EPOCH_NUM, batch_id + 1, total_step, loss.numpy())) + print( + "Epoch [{}/{}], Step [{}/{}], Loss: {}".format( + epoch_id + 1, EPOCH_NUM, batch_id + 1, total_step, loss.numpy() + ) + ) epoch_end = time.time() print( f"Epoch ID: {epoch_id+1}, AMPO1 train epoch time: {(epoch_end - epoch_start) * 1000} ms" @@ -90,7 +97,6 @@ def test_func(epoch_id, test_loader, model, cost): class TestCustomCPUPlugin(unittest.TestCase): - def setUp(self): # compile so and set to current path cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -102,15 +108,18 @@ def setUp(self): && git checkout {} -b dev \ && cd backends/custom_cpu \ && mkdir build && cd build && cmake .. && make -j8'.format( - self.temp_dir.name, os.getenv('PLUGIN_URL'), - os.getenv('PLUGIN_TAG')) + self.temp_dir.name, os.getenv('PLUGIN_URL'), os.getenv('PLUGIN_TAG') + ) os.system(cmd) # set environment for loading and registering compiled custom kernels # only valid in current process os.environ['CUSTOM_DEVICE_ROOT'] = os.path.join( - cur_dir, '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( - self.temp_dir.name)) + cur_dir, + '{}/PaddleCustomDevice/backends/custom_cpu/build'.format( + self.temp_dir.name + ), + ) def tearDown(self): self.temp_dir.cleanup() @@ -123,7 +132,6 @@ def _test_to_static(self): import paddle class LeNet5(paddle.nn.Layer): - def __init__(self): super(LeNet5, self).__init__() self.fc = paddle.nn.Linear(in_features=1024, out_features=10) @@ -145,35 +153,44 @@ def forward(self, x): # cost and optimizer cost = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.001, - parameters=model.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=0.001, parameters=model.parameters() + ) # convert to static model build_strategy = paddle.static.BuildStrategy() mnist = paddle.jit.to_static(model, build_strategy=build_strategy) # data loader - transform = paddle.vision.transforms.Compose([ - paddle.vision.transforms.Resize((32, 32)), - paddle.vision.transforms.ToTensor(), - paddle.vision.transforms.Normalize(mean=(0.1307, ), std=(0.3081, )) - ]) - train_dataset = paddle.vision.datasets.MNIST(mode='train', - transform=transform, - download=True) - test_dataset = paddle.vision.datasets.MNIST(mode='test', - transform=transform, - download=True) - train_loader = paddle.io.DataLoader(train_dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) - test_loader = paddle.io.DataLoader(test_dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) + transform = paddle.vision.transforms.Compose( + [ + paddle.vision.transforms.Resize((32, 32)), + paddle.vision.transforms.ToTensor(), + paddle.vision.transforms.Normalize( + mean=(0.1307,), std=(0.3081,) + ), + ] + ) + train_dataset = paddle.vision.datasets.MNIST( + mode='train', transform=transform, download=True + ) + test_dataset = paddle.vision.datasets.MNIST( + mode='test', transform=transform, download=True + ) + train_loader = paddle.io.DataLoader( + train_dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) + test_loader = paddle.io.DataLoader( + test_dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) # train and eval for epoch_id in range(EPOCH_NUM): @@ -184,7 +201,6 @@ def _test_amp_o1(self): import paddle class LeNet5(paddle.nn.Layer): - def __init__(self): super(LeNet5, self).__init__() self.fc = paddle.nn.Linear(in_features=1024, out_features=10) @@ -206,42 +222,52 @@ def forward(self, x): # cost and optimizer cost = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.001, - parameters=model.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=0.001, parameters=model.parameters() + ) # convert to static model scaler = paddle.amp.GradScaler(init_loss_scaling=1024) - model, optimizer = paddle.amp.decorate(models=model, - optimizers=optimizer, - level='O1') + model, optimizer = paddle.amp.decorate( + models=model, optimizers=optimizer, level='O1' + ) # data loader - transform = paddle.vision.transforms.Compose([ - paddle.vision.transforms.Resize((32, 32)), - paddle.vision.transforms.ToTensor(), - paddle.vision.transforms.Normalize(mean=(0.1307, ), std=(0.3081, )) - ]) - train_dataset = paddle.vision.datasets.MNIST(mode='train', - transform=transform, - download=True) - test_dataset = paddle.vision.datasets.MNIST(mode='test', - transform=transform, - download=True) - train_loader = paddle.io.DataLoader(train_dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) - test_loader = paddle.io.DataLoader(test_dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) + transform = paddle.vision.transforms.Compose( + [ + paddle.vision.transforms.Resize((32, 32)), + paddle.vision.transforms.ToTensor(), + paddle.vision.transforms.Normalize( + mean=(0.1307,), std=(0.3081,) + ), + ] + ) + train_dataset = paddle.vision.datasets.MNIST( + mode='train', transform=transform, download=True + ) + test_dataset = paddle.vision.datasets.MNIST( + mode='test', transform=transform, download=True + ) + train_loader = paddle.io.DataLoader( + train_dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) + test_loader = paddle.io.DataLoader( + test_dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) # train and eval for epoch_id in range(EPOCH_NUM): - train_func_ampo1(epoch_id, train_loader, model, cost, optimizer, - scaler) + train_func_ampo1( + epoch_id, train_loader, model, cost, optimizer, scaler + ) test_func(epoch_id, test_loader, model, cost) diff --git a/python/paddle/fluid/tests/test_beam_search_decoder.py b/python/paddle/fluid/tests/test_beam_search_decoder.py index 66cd8e89d3237..3a5c8604648bc 100644 --- a/python/paddle/fluid/tests/test_beam_search_decoder.py +++ b/python/paddle/fluid/tests/test_beam_search_decoder.py @@ -22,7 +22,12 @@ import paddle.fluid.framework as framework import paddle.fluid.layers as layers from paddle.fluid.executor import Executor -from paddle.fluid.contrib.decoder.beam_search_decoder import BeamSearchDecoder, InitState, StateCell, TrainingDecoder +from paddle.fluid.contrib.decoder.beam_search_decoder import ( + BeamSearchDecoder, + InitState, + StateCell, + TrainingDecoder, +) import unittest paddle.enable_static() @@ -43,14 +48,15 @@ def encoder(): # encoder - src_word = layers.data(name="src_word", - shape=[1], - dtype='int64', - lod_level=1) - src_embedding = layers.embedding(input=src_word, - size=[dict_size, word_dim], - dtype='float32', - is_sparse=IS_SPARSE) + src_word = layers.data( + name="src_word", shape=[1], dtype='int64', lod_level=1 + ) + src_embedding = layers.embedding( + input=src_word, + size=[dict_size, word_dim], + dtype='float32', + is_sparse=IS_SPARSE, + ) fc1 = layers.fc(input=src_embedding, size=hidden_dim * 4, act='tanh') lstm_hidden0, lstm_0 = layers.dynamic_lstm(input=fc1, size=hidden_dim * 4) @@ -67,9 +73,9 @@ def updater(state_cell): current_word = state_cell.get_input('x') prev_h = state_cell.get_state('h') # make sure lod of h heritted from prev_h - h = layers.fc(input=[prev_h, current_word], - size=decoder_size, - act='tanh') + h = layers.fc( + input=[prev_h, current_word], size=decoder_size, act='tanh' + ) state_cell.set_state('h', h) return state_cell @@ -77,23 +83,26 @@ def updater(state_cell): def decoder_train(state_cell): # decoder - trg_language_word = layers.data(name="target_word", - shape=[1], - dtype='int64', - lod_level=1) - trg_embedding = layers.embedding(input=trg_language_word, - size=[dict_size, word_dim], - dtype='float32', - is_sparse=IS_SPARSE) + trg_language_word = layers.data( + name="target_word", shape=[1], dtype='int64', lod_level=1 + ) + trg_embedding = layers.embedding( + input=trg_language_word, + size=[dict_size, word_dim], + dtype='float32', + is_sparse=IS_SPARSE, + ) decoder = TrainingDecoder(state_cell) with decoder.block(): current_word = decoder.step_input(trg_embedding) decoder.state_cell.compute_state(inputs={'x': current_word}) - current_score = layers.fc(input=decoder.state_cell.get_state('h'), - size=target_dict_dim, - act='softmax') + current_score = layers.fc( + input=decoder.state_cell.get_state('h'), + size=target_dict_dim, + act='softmax', + ) decoder.state_cell.update_states() decoder.output(current_score) @@ -101,27 +110,27 @@ def decoder_train(state_cell): def decoder_decode(state_cell): - init_ids = layers.data(name="init_ids", - shape=[1], - dtype="int64", - lod_level=2) - init_scores = layers.data(name="init_scores", - shape=[1], - dtype="float32", - lod_level=2) - - decoder = BeamSearchDecoder(state_cell=state_cell, - init_ids=init_ids, - init_scores=init_scores, - target_dict_dim=target_dict_dim, - word_dim=word_dim, - input_var_dict={}, - topk_size=topk_size, - sparse_emb=IS_SPARSE, - max_len=max_length, - beam_size=beam_size, - end_id=1, - name=None) + init_ids = layers.data( + name="init_ids", shape=[1], dtype="int64", lod_level=2 + ) + init_scores = layers.data( + name="init_scores", shape=[1], dtype="float32", lod_level=2 + ) + + decoder = BeamSearchDecoder( + state_cell=state_cell, + init_ids=init_ids, + init_scores=init_scores, + target_dict_dim=target_dict_dim, + word_dim=word_dim, + input_var_dict={}, + topk_size=topk_size, + sparse_emb=IS_SPARSE, + max_len=max_length, + beam_size=beam_size, + end_id=1, + name=None, + ) decoder.decode() translation_ids, translation_scores = decoder() @@ -136,19 +145,21 @@ def train_main(use_cuda): context = encoder() state_cell = decoder_state_cell(context) rnn_out = decoder_train(state_cell) - label = layers.data(name="target_next_word", - shape=[1], - dtype='int64', - lod_level=1) + label = layers.data( + name="target_next_word", shape=[1], dtype='int64', lod_level=1 + ) cost = layers.cross_entropy(input=rnn_out, label=label) avg_cost = paddle.mean(x=cost) optimizer = fluid.optimizer.Adagrad(learning_rate=1e-3) optimizer.minimize(avg_cost) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.wmt14.train(dict_size), buf_size=1000), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.wmt14.train(dict_size), buf_size=1000 + ), + batch_size=batch_size, + ) feed_order = ['src_word', 'target_word', 'target_next_word'] exe = Executor(place) @@ -163,12 +174,18 @@ def train_loop(main_program): for pass_id in range(1): for batch_id, data in enumerate(train_reader()): - outs = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[avg_cost]) + outs = exe.run( + main_program, feed=feeder.feed(data), fetch_list=[avg_cost] + ) avg_cost_val = np.array(outs[0]) - print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) + - " avg_cost=" + str(avg_cost_val)) + print( + 'pass_id=' + + str(pass_id) + + ' batch=' + + str(batch_id) + + " avg_cost=" + + str(avg_cost_val) + ) if batch_id > 3: break @@ -188,8 +205,9 @@ def decode_main(use_cuda): exe.run(framework.default_startup_program()) init_ids_data = np.array([0 for _ in range(batch_size)], dtype='int64') - init_scores_data = np.array([1. for _ in range(batch_size)], - dtype='float32') + init_scores_data = np.array( + [1.0 for _ in range(batch_size)], dtype='float32' + ) init_ids_data = init_ids_data.reshape((batch_size, 1)) init_scores_data = init_scores_data.reshape((batch_size, 1)) init_lod = [1] * batch_size @@ -198,9 +216,12 @@ def decode_main(use_cuda): init_ids = fluid.create_lod_tensor(init_ids_data, init_lod, place) init_scores = fluid.create_lod_tensor(init_scores_data, init_lod, place) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.wmt14.train(dict_size), buf_size=1000), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.wmt14.train(dict_size), buf_size=1000 + ), + batch_size=batch_size, + ) feed_order = ['src_word'] feed_list = [ @@ -218,7 +239,8 @@ def decode_main(use_cuda): framework.default_main_program(), feed=feed_dict, fetch_list=[translation_ids, translation_scores], - return_numpy=False) + return_numpy=False, + ) print(result_ids.lod()) diff --git a/python/paddle/fluid/tests/test_data_feeder.py b/python/paddle/fluid/tests/test_data_feeder.py index 5f97a250d7537..515bf44c75518 100644 --- a/python/paddle/fluid/tests/test_data_feeder.py +++ b/python/paddle/fluid/tests/test_data_feeder.py @@ -20,7 +20,6 @@ class TestDataFeeder(unittest.TestCase): - def test_lod_level_0_converter(self): img = fluid.layers.data(name='image', shape=[1, 28, 28]) label = fluid.layers.data(name='label', shape=[1], dtype='int64') @@ -41,45 +40,48 @@ def test_lod_level_0_converter(self): def test_lod_level_1_converter(self): # lod_level = 1 # each sentence has a different number of words - sentences = fluid.layers.data(name='sentences', - shape=[1], - dtype='int64', - lod_level=1) + sentences = fluid.layers.data( + name='sentences', shape=[1], dtype='int64', lod_level=1 + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') feeder = fluid.DataFeeder([sentences, label], fluid.CPUPlace()) # lod = [[0, 3, 5, 9]] # data = [[1, 2, 3], [4, 5], [6, 7, 8, 9]] # label = [1] * len(data) - result = feeder.feed([([1, 2, 3], [1]), ([4, 5], [1]), - ([6, 7, 8, 9], [1])]) + result = feeder.feed( + [([1, 2, 3], [1]), ([4, 5], [1]), ([6, 7, 8, 9], [1])] + ) self.assertEqual(result['sentences'].shape(), [9, 1]) self.assertEqual(result['label'].shape(), [3, 1]) - self.assertEqual(result['sentences'].recursive_sequence_lengths(), - [[3, 2, 4]]) + self.assertEqual( + result['sentences'].recursive_sequence_lengths(), [[3, 2, 4]] + ) self.assertEqual(result['label'].recursive_sequence_lengths(), []) def test_lod_level_2_converter(self): # lod_level = 2 # paragraphs -> sentences -> words - paragraphs = fluid.layers.data(name='paragraphs', - shape=[1], - dtype='int64', - lod_level=2) + paragraphs = fluid.layers.data( + name='paragraphs', shape=[1], dtype='int64', lod_level=2 + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') feeder = fluid.DataFeeder([paragraphs, label], fluid.CPUPlace()) # lod = [[0, 2, 3], [0, 3, 5, 9]] # data = [[[1, 2, 3], [4, 5]], [[6, 7, 8, 9]]] # label = [1] * len(data) - result = feeder.feed([([[1, 2, 3], [4, 5]], [1]), ([[6, 7, 8, - 9]], [1])]) + result = feeder.feed( + [([[1, 2, 3], [4, 5]], [1]), ([[6, 7, 8, 9]], [1])] + ) self.assertEqual(result['paragraphs'].shape(), [9, 1]) self.assertEqual(result['label'].shape(), [2, 1]) - self.assertEqual(result['paragraphs'].recursive_sequence_lengths(), - [[2, 1], [3, 2, 4]]) + self.assertEqual( + result['paragraphs'].recursive_sequence_lengths(), + [[2, 1], [3, 2, 4]], + ) self.assertEqual(result['label'].recursive_sequence_lengths(), []) diff --git a/python/paddle/fluid/tests/test_detection.py b/python/paddle/fluid/tests/test_detection.py index bd3b8df738598..baa80a73aa245 100644 --- a/python/paddle/fluid/tests/test_detection.py +++ b/python/paddle/fluid/tests/test_detection.py @@ -28,7 +28,6 @@ class LayerTest(unittest.TestCase): - @classmethod def setUpClass(cls): cls.seed = 111 @@ -53,57 +52,66 @@ def static_graph(self): fluid.default_main_program().random_seed = self.seed yield - def get_static_graph_result(self, - feed, - fetch_list, - with_lod=False, - force_to_use_cpu=False): + def get_static_graph_result( + self, feed, fetch_list, with_lod=False, force_to_use_cpu=False + ): exe = fluid.Executor(self._get_place(force_to_use_cpu)) exe.run(fluid.default_startup_program()) - return exe.run(fluid.default_main_program(), - feed=feed, - fetch_list=fetch_list, - return_numpy=(not with_lod)) + return exe.run( + fluid.default_main_program(), + feed=feed, + fetch_list=fetch_list, + return_numpy=(not with_lod), + ) @contextlib.contextmanager def dynamic_graph(self, force_to_use_cpu=False): with fluid.dygraph.guard( - self._get_place(force_to_use_cpu=force_to_use_cpu)): + self._get_place(force_to_use_cpu=force_to_use_cpu) + ): fluid.default_startup_program().random_seed = self.seed fluid.default_main_program().random_seed = self.seed yield class TestDetection(unittest.TestCase): - def test_detection_output(self): program = Program() with program_guard(program): - pb = layers.data(name='prior_box', - shape=[10, 4], - append_batch_size=False, - dtype='float32') - pbv = layers.data(name='prior_box_var', - shape=[10, 4], - append_batch_size=False, - dtype='float32') - loc = layers.data(name='target_box', - shape=[2, 10, 4], - append_batch_size=False, - dtype='float32') - scores = layers.data(name='scores', - shape=[2, 10, 20], - append_batch_size=False, - dtype='float32') - out = layers.detection_output(scores=scores, - loc=loc, - prior_box=pb, - prior_box_var=pbv) - out2, index = layers.detection_output(scores=scores, - loc=loc, - prior_box=pb, - prior_box_var=pbv, - return_index=True) + pb = layers.data( + name='prior_box', + shape=[10, 4], + append_batch_size=False, + dtype='float32', + ) + pbv = layers.data( + name='prior_box_var', + shape=[10, 4], + append_batch_size=False, + dtype='float32', + ) + loc = layers.data( + name='target_box', + shape=[2, 10, 4], + append_batch_size=False, + dtype='float32', + ) + scores = layers.data( + name='scores', + shape=[2, 10, 20], + append_batch_size=False, + dtype='float32', + ) + out = layers.detection_output( + scores=scores, loc=loc, prior_box=pb, prior_box_var=pbv + ) + out2, index = layers.detection_output( + scores=scores, + loc=loc, + prior_box=pb, + prior_box_var=pbv, + return_index=True, + ) self.assertIsNotNone(out) self.assertIsNotNone(out2) self.assertIsNotNone(index) @@ -115,10 +123,12 @@ def test_box_coder_api(self): with program_guard(program): x = layers.data(name='x', shape=[4], dtype='float32') y = layers.data(name='z', shape=[4], dtype='float32', lod_level=1) - bcoder = layers.box_coder(prior_box=x, - prior_box_var=[0.1, 0.2, 0.1, 0.2], - target_box=y, - code_type='encode_center_size') + bcoder = layers.box_coder( + prior_box=x, + prior_box_var=[0.1, 0.2, 0.1, 0.2], + target_box=y, + code_type='encode_center_size', + ) self.assertIsNotNone(bcoder) print(str(program)) @@ -126,28 +136,30 @@ def test_box_coder_error(self): program = Program() with program_guard(program): x1 = fluid.data(name='x1', shape=[10, 4], dtype='int32') - y1 = fluid.data(name='y1', - shape=[10, 4], - dtype='float32', - lod_level=1) + y1 = fluid.data( + name='y1', shape=[10, 4], dtype='float32', lod_level=1 + ) x2 = fluid.data(name='x2', shape=[10, 4], dtype='float32') - y2 = fluid.data(name='y2', - shape=[10, 4], - dtype='int32', - lod_level=1) - - self.assertRaises(TypeError, - layers.box_coder, - prior_box=x1, - prior_box_var=[0.1, 0.2, 0.1, 0.2], - target_box=y1, - code_type='encode_center_size') - self.assertRaises(TypeError, - layers.box_coder, - prior_box=x2, - prior_box_var=[0.1, 0.2, 0.1, 0.2], - target_box=y2, - code_type='encode_center_size') + y2 = fluid.data( + name='y2', shape=[10, 4], dtype='int32', lod_level=1 + ) + + self.assertRaises( + TypeError, + layers.box_coder, + prior_box=x1, + prior_box_var=[0.1, 0.2, 0.1, 0.2], + target_box=y1, + code_type='encode_center_size', + ) + self.assertRaises( + TypeError, + layers.box_coder, + prior_box=x2, + prior_box_var=[0.1, 0.2, 0.1, 0.2], + target_box=y2, + code_type='encode_center_size', + ) def test_detection_api(self): program = Program() @@ -156,10 +168,12 @@ def test_detection_api(self): y = layers.data(name='y', shape=[4], dtype='float32') z = layers.data(name='z', shape=[4], dtype='float32', lod_level=1) iou = layers.iou_similarity(x=x, y=y) - bcoder = layers.box_coder(prior_box=x, - prior_box_var=y, - target_box=z, - code_type='encode_center_size') + bcoder = layers.box_coder( + prior_box=x, + prior_box_var=y, + target_box=z, + code_type='encode_center_size', + ) self.assertIsNotNone(iou) self.assertIsNotNone(bcoder) @@ -167,23 +181,21 @@ def test_detection_api(self): self.assertIsNotNone(matched_indices) self.assertIsNotNone(matched_dist) - gt = layers.data(name='gt', - shape=[1, 1], - dtype='int32', - lod_level=1) - trg, trg_weight = layers.target_assign(gt, - matched_indices, - mismatch_value=0) + gt = layers.data( + name='gt', shape=[1, 1], dtype='int32', lod_level=1 + ) + trg, trg_weight = layers.target_assign( + gt, matched_indices, mismatch_value=0 + ) self.assertIsNotNone(trg) self.assertIsNotNone(trg_weight) - gt2 = layers.data(name='gt2', - shape=[10, 4], - dtype='float32', - lod_level=1) - trg, trg_weight = layers.target_assign(gt2, - matched_indices, - mismatch_value=0) + gt2 = layers.data( + name='gt2', shape=[10, 4], dtype='float32', lod_level=1 + ) + trg, trg_weight = layers.target_assign( + gt2, matched_indices, mismatch_value=0 + ) self.assertIsNotNone(trg) self.assertIsNotNone(trg_weight) @@ -192,24 +204,26 @@ def test_detection_api(self): def test_ssd_loss(self): program = Program() with program_guard(program): - pb = layers.data(name='prior_box', - shape=[10, 4], - append_batch_size=False, - dtype='float32') - pbv = layers.data(name='prior_box_var', - shape=[10, 4], - append_batch_size=False, - dtype='float32') + pb = layers.data( + name='prior_box', + shape=[10, 4], + append_batch_size=False, + dtype='float32', + ) + pbv = layers.data( + name='prior_box_var', + shape=[10, 4], + append_batch_size=False, + dtype='float32', + ) loc = layers.data(name='target_box', shape=[10, 4], dtype='float32') scores = layers.data(name='scores', shape=[10, 21], dtype='float32') - gt_box = layers.data(name='gt_box', - shape=[4], - lod_level=1, - dtype='float32') - gt_label = layers.data(name='gt_label', - shape=[1], - lod_level=1, - dtype='int32') + gt_box = layers.data( + name='gt_box', shape=[4], lod_level=1, dtype='float32' + ) + gt_label = layers.data( + name='gt_label', shape=[1], lod_level=1, dtype='int32' + ) loss = layers.ssd_loss(loc, scores, gt_box, gt_label, pb, pbv) self.assertIsNotNone(loss) self.assertEqual(loss.shape[-1], 1) @@ -217,73 +231,75 @@ def test_ssd_loss(self): class TestPriorBox(unittest.TestCase): - def test_prior_box(self): program = Program() with program_guard(program): data_shape = [3, 224, 224] - images = fluid.layers.data(name='pixel', - shape=data_shape, - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=data_shape, dtype='float32' + ) conv1 = fluid.layers.conv2d(images, 3, 3, 2) - box, var = layers.prior_box(input=conv1, - image=images, - min_sizes=[100.0], - aspect_ratios=[1.], - flip=True, - clip=True) + box, var = layers.prior_box( + input=conv1, + image=images, + min_sizes=[100.0], + aspect_ratios=[1.0], + flip=True, + clip=True, + ) assert len(box.shape) == 4 assert box.shape == var.shape assert box.shape[3] == 4 class TestPriorBox2(unittest.TestCase): - def test_prior_box(self): program = Program() with program_guard(program): data_shape = [None, 3, None, None] images = fluid.data(name='pixel', shape=data_shape, dtype='float32') conv1 = fluid.layers.conv2d(images, 3, 3, 2) - box, var = layers.prior_box(input=conv1, - image=images, - min_sizes=[100.0], - aspect_ratios=[1.], - flip=True, - clip=True) + box, var = layers.prior_box( + input=conv1, + image=images, + min_sizes=[100.0], + aspect_ratios=[1.0], + flip=True, + clip=True, + ) assert len(box.shape) == 4 assert box.shape == var.shape assert box.shape[3] == 4 class TestDensityPriorBox(unittest.TestCase): - def test_density_prior_box(self): program = Program() with program_guard(program): data_shape = [3, 224, 224] - images = fluid.layers.data(name='pixel', - shape=data_shape, - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=data_shape, dtype='float32' + ) conv1 = fluid.layers.conv2d(images, 3, 3, 2) - box, var = layers.density_prior_box(input=conv1, - image=images, - densities=[3, 4], - fixed_sizes=[50., 60.], - fixed_ratios=[1.0], - clip=True) + box, var = layers.density_prior_box( + input=conv1, + image=images, + densities=[3, 4], + fixed_sizes=[50.0, 60.0], + fixed_ratios=[1.0], + clip=True, + ) assert len(box.shape) == 4 assert box.shape == var.shape assert box.shape[-1] == 4 class TestAnchorGenerator(unittest.TestCase): - def test_anchor_generator(self): data_shape = [3, 224, 224] - images = fluid.layers.data(name='pixel', - shape=data_shape, - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=data_shape, dtype='float32' + ) conv1 = fluid.layers.conv2d(images, 3, 3, 2) anchor, var = fluid.layers.anchor_generator( input=conv1, @@ -291,14 +307,14 @@ def test_anchor_generator(self): aspect_ratios=[0.5, 1.0, 2.0], variance=[0.1, 0.1, 0.2, 0.2], stride=[16.0, 16.0], - offset=0.5) + offset=0.5, + ) assert len(anchor.shape) == 4 assert anchor.shape == var.shape assert anchor.shape[3] == 4 class TestGenerateProposalLabels(unittest.TestCase): - def check_out(self, outs): rois = outs[0] labels_int32 = outs[1] @@ -320,27 +336,22 @@ def check_out(self, outs): def test_generate_proposal_labels(self): program = Program() with program_guard(program): - rpn_rois = fluid.data(name='rpn_rois', - shape=[4, 4], - dtype='float32', - lod_level=1) - gt_classes = fluid.data(name='gt_classes', - shape=[6], - dtype='int32', - lod_level=1) - is_crowd = fluid.data(name='is_crowd', - shape=[6], - dtype='int32', - lod_level=1) - gt_boxes = fluid.data(name='gt_boxes', - shape=[6, 4], - dtype='float32', - lod_level=1) + rpn_rois = fluid.data( + name='rpn_rois', shape=[4, 4], dtype='float32', lod_level=1 + ) + gt_classes = fluid.data( + name='gt_classes', shape=[6], dtype='int32', lod_level=1 + ) + is_crowd = fluid.data( + name='is_crowd', shape=[6], dtype='int32', lod_level=1 + ) + gt_boxes = fluid.data( + name='gt_boxes', shape=[6, 4], dtype='float32', lod_level=1 + ) im_info = fluid.data(name='im_info', shape=[1, 3], dtype='float32') - max_overlap = fluid.data(name='max_overlap', - shape=[4], - dtype='float32', - lod_level=1) + max_overlap = fluid.data( + name='max_overlap', shape=[4], dtype='float32', lod_level=1 + ) self.class_nums = 5 outs = fluid.layers.generate_proposal_labels( rpn_rois=rpn_rois, @@ -354,7 +365,8 @@ def test_generate_proposal_labels(self): bg_thresh_hi=0.5, bg_thresh_lo=0.0, bbox_reg_weights=[0.1, 0.1, 0.2, 0.2], - class_nums=self.class_nums) + class_nums=self.class_nums, + ) outs_1 = fluid.layers.generate_proposal_labels( rpn_rois=rpn_rois, gt_classes=gt_classes, @@ -370,7 +382,8 @@ def test_generate_proposal_labels(self): class_nums=self.class_nums, is_cascade_rcnn=True, max_overlap=max_overlap, - return_max_overlap=True) + return_max_overlap=True, + ) self.check_out(outs) self.check_out(outs_1) @@ -378,57 +391,69 @@ def test_generate_proposal_labels(self): class TestGenerateMaskLabels(unittest.TestCase): - def test_generate_mask_labels(self): program = Program() with program_guard(program): - im_info = layers.data(name='im_info', - shape=[1, 3], - dtype='float32', - lod_level=1, - append_batch_size=False) - gt_classes = layers.data(name='gt_classes', - shape=[2, 1], - dtype='int32', - lod_level=1, - append_batch_size=False) - is_crowd = layers.data(name='is_crowd', - shape=[2, 1], - dtype='int32', - lod_level=1, - append_batch_size=False) - gt_segms = layers.data(name='gt_segms', - shape=[20, 2], - dtype='float32', - lod_level=3, - append_batch_size=False) - rois = layers.data(name='rois', - shape=[4, 4], - dtype='float32', - lod_level=1, - append_batch_size=False) - labels_int32 = layers.data(name='labels_int32', - shape=[4, 1], - dtype='int32', - lod_level=1, - append_batch_size=False) + im_info = layers.data( + name='im_info', + shape=[1, 3], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) + gt_classes = layers.data( + name='gt_classes', + shape=[2, 1], + dtype='int32', + lod_level=1, + append_batch_size=False, + ) + is_crowd = layers.data( + name='is_crowd', + shape=[2, 1], + dtype='int32', + lod_level=1, + append_batch_size=False, + ) + gt_segms = layers.data( + name='gt_segms', + shape=[20, 2], + dtype='float32', + lod_level=3, + append_batch_size=False, + ) + rois = layers.data( + name='rois', + shape=[4, 4], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) + labels_int32 = layers.data( + name='labels_int32', + shape=[4, 1], + dtype='int32', + lod_level=1, + append_batch_size=False, + ) num_classes = 5 resolution = 14 - outs = fluid.layers.generate_mask_labels(im_info=im_info, - gt_classes=gt_classes, - is_crowd=is_crowd, - gt_segms=gt_segms, - rois=rois, - labels_int32=labels_int32, - num_classes=num_classes, - resolution=resolution) + outs = fluid.layers.generate_mask_labels( + im_info=im_info, + gt_classes=gt_classes, + is_crowd=is_crowd, + gt_segms=gt_segms, + rois=rois, + labels_int32=labels_int32, + num_classes=num_classes, + resolution=resolution, + ) mask_rois, roi_has_mask_int32, mask_int32 = outs assert mask_rois.shape[1] == 4 assert mask_int32.shape[1] == num_classes * resolution * resolution class TestMultiBoxHead(unittest.TestCase): - def test_multi_box_head(self): data_shape = [3, 224, 224] mbox_locs, mbox_confs, box, var = self.multi_box_head_output(data_shape) @@ -439,9 +464,9 @@ def test_multi_box_head(self): assert mbox_locs.shape[1] == mbox_confs.shape[1] def multi_box_head_output(self, data_shape): - images = fluid.layers.data(name='pixel', - shape=data_shape, - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=data_shape, dtype='float32' + ) conv1 = fluid.layers.conv2d(images, 3, 3, 2) conv2 = fluid.layers.conv2d(conv1, 3, 3, 2) conv3 = fluid.layers.conv2d(conv2, 3, 3, 2) @@ -454,37 +479,47 @@ def multi_box_head_output(self, data_shape): num_classes=21, min_ratio=20, max_ratio=90, - aspect_ratios=[[2.], [2., 3.], [2., 3.], [2., 3.], [2.], [2.]], + aspect_ratios=[ + [2.0], + [2.0, 3.0], + [2.0, 3.0], + [2.0, 3.0], + [2.0], + [2.0], + ], base_size=300, offset=0.5, flip=True, - clip=True) + clip=True, + ) return mbox_locs, mbox_confs, box, var class TestDetectionMAP(unittest.TestCase): - def test_detection_map(self): program = Program() with program_guard(program): - detect_res = layers.data(name='detect_res', - shape=[10, 6], - append_batch_size=False, - dtype='float32') - label = layers.data(name='label', - shape=[10, 6], - append_batch_size=False, - dtype='float32') + detect_res = layers.data( + name='detect_res', + shape=[10, 6], + append_batch_size=False, + dtype='float32', + ) + label = layers.data( + name='label', + shape=[10, 6], + append_batch_size=False, + dtype='float32', + ) map_out = detection.detection_map(detect_res, label, 21) self.assertIsNotNone(map_out) - self.assertEqual(map_out.shape, (1, )) + self.assertEqual(map_out.shape, (1,)) print(str(program)) class TestRpnTargetAssign(unittest.TestCase): - def test_rpn_target_assign(self): program = Program() with program_guard(program): @@ -492,49 +527,62 @@ def test_rpn_target_assign(self): cls_logits_shape = [10, 50, 2] anchor_shape = [50, 4] - bbox_pred = layers.data(name='bbox_pred', - shape=bbox_pred_shape, - append_batch_size=False, - dtype='float32') - cls_logits = layers.data(name='cls_logits', - shape=cls_logits_shape, - append_batch_size=False, - dtype='float32') - anchor_box = layers.data(name='anchor_box', - shape=anchor_shape, - append_batch_size=False, - dtype='float32') - anchor_var = layers.data(name='anchor_var', - shape=anchor_shape, - append_batch_size=False, - dtype='float32') - gt_boxes = layers.data(name='gt_boxes', - shape=[4], - lod_level=1, - dtype='float32') - is_crowd = layers.data(name='is_crowd', - shape=[1, 10], - dtype='int32', - lod_level=1, - append_batch_size=False) - im_info = layers.data(name='im_info', - shape=[1, 3], - dtype='float32', - lod_level=1, - append_batch_size=False) - outs = layers.rpn_target_assign(bbox_pred=bbox_pred, - cls_logits=cls_logits, - anchor_box=anchor_box, - anchor_var=anchor_var, - gt_boxes=gt_boxes, - is_crowd=is_crowd, - im_info=im_info, - rpn_batch_size_per_im=256, - rpn_straddle_thresh=0.0, - rpn_fg_fraction=0.5, - rpn_positive_overlap=0.7, - rpn_negative_overlap=0.3, - use_random=False) + bbox_pred = layers.data( + name='bbox_pred', + shape=bbox_pred_shape, + append_batch_size=False, + dtype='float32', + ) + cls_logits = layers.data( + name='cls_logits', + shape=cls_logits_shape, + append_batch_size=False, + dtype='float32', + ) + anchor_box = layers.data( + name='anchor_box', + shape=anchor_shape, + append_batch_size=False, + dtype='float32', + ) + anchor_var = layers.data( + name='anchor_var', + shape=anchor_shape, + append_batch_size=False, + dtype='float32', + ) + gt_boxes = layers.data( + name='gt_boxes', shape=[4], lod_level=1, dtype='float32' + ) + is_crowd = layers.data( + name='is_crowd', + shape=[1, 10], + dtype='int32', + lod_level=1, + append_batch_size=False, + ) + im_info = layers.data( + name='im_info', + shape=[1, 3], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) + outs = layers.rpn_target_assign( + bbox_pred=bbox_pred, + cls_logits=cls_logits, + anchor_box=anchor_box, + anchor_var=anchor_var, + gt_boxes=gt_boxes, + is_crowd=is_crowd, + im_info=im_info, + rpn_batch_size_per_im=256, + rpn_straddle_thresh=0.0, + rpn_fg_fraction=0.5, + rpn_positive_overlap=0.7, + rpn_negative_overlap=0.3, + use_random=False, + ) pred_scores = outs[0] pred_loc = outs[1] tgt_lbl = outs[2] @@ -553,29 +601,29 @@ def test_rpn_target_assign(self): class TestGenerateProposals(LayerTest): - def test_generate_proposals(self): scores_np = np.random.rand(2, 3, 4, 4).astype('float32') bbox_deltas_np = np.random.rand(2, 12, 4, 4).astype('float32') im_info_np = np.array([[8, 8, 0.5], [6, 6, 0.5]]).astype('float32') - anchors_np = np.reshape(np.arange(4 * 4 * 3 * 4), - [4, 4, 3, 4]).astype('float32') + anchors_np = np.reshape(np.arange(4 * 4 * 3 * 4), [4, 4, 3, 4]).astype( + 'float32' + ) variances_np = np.ones((4, 4, 3, 4)).astype('float32') with self.static_graph(): - scores = fluid.data(name='scores', - shape=[2, 3, 4, 4], - dtype='float32') - bbox_deltas = fluid.data(name='bbox_deltas', - shape=[2, 12, 4, 4], - dtype='float32') + scores = fluid.data( + name='scores', shape=[2, 3, 4, 4], dtype='float32' + ) + bbox_deltas = fluid.data( + name='bbox_deltas', shape=[2, 12, 4, 4], dtype='float32' + ) im_info = fluid.data(name='im_info', shape=[2, 3], dtype='float32') - anchors = fluid.data(name='anchors', - shape=[4, 4, 3, 4], - dtype='float32') - variances = fluid.data(name='var', - shape=[4, 4, 3, 4], - dtype='float32') + anchors = fluid.data( + name='anchors', shape=[4, 4, 3, 4], dtype='float32' + ) + variances = fluid.data( + name='var', shape=[4, 4, 3, 4], dtype='float32' + ) rois, roi_probs, rois_num = fluid.layers.generate_proposals( scores, bbox_deltas, @@ -584,17 +632,23 @@ def test_generate_proposals(self): variances, pre_nms_top_n=10, post_nms_top_n=5, - return_rois_num=True) - rois_stat, roi_probs_stat, rois_num_stat = self.get_static_graph_result( + return_rois_num=True, + ) + ( + rois_stat, + roi_probs_stat, + rois_num_stat, + ) = self.get_static_graph_result( feed={ 'scores': scores_np, 'bbox_deltas': bbox_deltas_np, 'im_info': im_info_np, 'anchors': anchors_np, - 'var': variances_np + 'var': variances_np, }, fetch_list=[rois, roi_probs, rois_num], - with_lod=False) + with_lod=False, + ) with self.dynamic_graph(): scores_dy = base.to_variable(scores_np) @@ -610,7 +664,8 @@ def test_generate_proposals(self): variances_dy, pre_nms_top_n=10, post_nms_top_n=5, - return_rois_num=True) + return_rois_num=True, + ) rois_dy = rois.numpy() roi_probs_dy = roi_probs.numpy() rois_num_dy = rois_num.numpy() @@ -621,7 +676,6 @@ def test_generate_proposals(self): class TestYoloDetection(unittest.TestCase): - def test_yolov3_loss(self): program = Program() with program_guard(program): @@ -629,14 +683,18 @@ def test_yolov3_loss(self): gt_box = layers.data(name='gt_box', shape=[10, 4], dtype='float32') gt_label = layers.data(name='gt_label', shape=[10], dtype='int32') gt_score = layers.data(name='gt_score', shape=[10], dtype='float32') - loss = layers.yolov3_loss(x, - gt_box, - gt_label, [10, 13, 30, 13], [0, 1], - 10, - 0.7, - 32, - gt_score=gt_score, - use_label_smooth=False) + loss = layers.yolov3_loss( + x, + gt_box, + gt_label, + [10, 13, 30, 13], + [0, 1], + 10, + 0.7, + 32, + gt_score=gt_score, + use_label_smooth=False, + ) self.assertIsNotNone(loss) @@ -645,8 +703,9 @@ def test_yolo_box(self): with program_guard(program): x = layers.data(name='x', shape=[30, 7, 7], dtype='float32') img_size = layers.data(name='img_size', shape=[2], dtype='int32') - boxes, scores = layers.yolo_box(x, img_size, [10, 13, 30, 13], 10, - 0.01, 32) + boxes, scores = layers.yolo_box( + x, img_size, [10, 13, 30, 13], 10, 0.01, 32 + ) self.assertIsNotNone(boxes) self.assertIsNotNone(scores) @@ -657,15 +716,19 @@ def test_yolov3_loss_with_scale(self): gt_box = layers.data(name='gt_box', shape=[10, 4], dtype='float32') gt_label = layers.data(name='gt_label', shape=[10], dtype='int32') gt_score = layers.data(name='gt_score', shape=[10], dtype='float32') - loss = layers.yolov3_loss(x, - gt_box, - gt_label, [10, 13, 30, 13], [0, 1], - 10, - 0.7, - 32, - gt_score=gt_score, - use_label_smooth=False, - scale_x_y=1.2) + loss = layers.yolov3_loss( + x, + gt_box, + gt_label, + [10, 13, 30, 13], + [0, 1], + 10, + 0.7, + 32, + gt_score=gt_score, + use_label_smooth=False, + scale_x_y=1.2, + ) self.assertIsNotNone(loss) @@ -674,38 +737,32 @@ def test_yolo_box_with_scale(self): with program_guard(program): x = layers.data(name='x', shape=[30, 7, 7], dtype='float32') img_size = layers.data(name='img_size', shape=[2], dtype='int32') - boxes, scores = layers.yolo_box(x, - img_size, [10, 13, 30, 13], - 10, - 0.01, - 32, - scale_x_y=1.2) + boxes, scores = layers.yolo_box( + x, img_size, [10, 13, 30, 13], 10, 0.01, 32, scale_x_y=1.2 + ) self.assertIsNotNone(boxes) self.assertIsNotNone(scores) class TestBoxClip(unittest.TestCase): - def test_box_clip(self): program = Program() with program_guard(program): - input_box = layers.data(name='input_box', - shape=[7, 4], - dtype='float32', - lod_level=1) + input_box = layers.data( + name='input_box', shape=[7, 4], dtype='float32', lod_level=1 + ) im_info = layers.data(name='im_info', shape=[3], dtype='float32') out = layers.box_clip(input_box, im_info) self.assertIsNotNone(out) class TestMulticlassNMS(unittest.TestCase): - def test_multiclass_nms(self): program = Program() with program_guard(program): - bboxes = layers.data(name='bboxes', - shape=[-1, 10, 4], - dtype='float32') + bboxes = layers.data( + name='bboxes', shape=[-1, 10, 4], dtype='float32' + ) scores = layers.data(name='scores', shape=[-1, 10], dtype='float32') output = layers.multiclass_nms(bboxes, scores, 0.3, 400, 200, 0.7) self.assertIsNotNone(output) @@ -713,57 +770,56 @@ def test_multiclass_nms(self): def test_multiclass_nms_error(self): program = Program() with program_guard(program): - bboxes1 = fluid.data(name='bboxes1', - shape=[10, 10, 4], - dtype='int32') - scores1 = fluid.data(name='scores1', - shape=[10, 10], - dtype='float32') - bboxes2 = fluid.data(name='bboxes2', - shape=[10, 10, 4], - dtype='float32') + bboxes1 = fluid.data( + name='bboxes1', shape=[10, 10, 4], dtype='int32' + ) + scores1 = fluid.data( + name='scores1', shape=[10, 10], dtype='float32' + ) + bboxes2 = fluid.data( + name='bboxes2', shape=[10, 10, 4], dtype='float32' + ) scores2 = fluid.data(name='scores2', shape=[10, 10], dtype='int32') - self.assertRaises(TypeError, - layers.multiclass_nms, - bboxes=bboxes1, - scores=scores1, - score_threshold=0.5, - nms_top_k=400, - keep_top_k=200) - self.assertRaises(TypeError, - layers.multiclass_nms, - bboxes=bboxes2, - scores=scores2, - score_threshold=0.5, - nms_top_k=400, - keep_top_k=200) + self.assertRaises( + TypeError, + layers.multiclass_nms, + bboxes=bboxes1, + scores=scores1, + score_threshold=0.5, + nms_top_k=400, + keep_top_k=200, + ) + self.assertRaises( + TypeError, + layers.multiclass_nms, + bboxes=bboxes2, + scores=scores2, + score_threshold=0.5, + nms_top_k=400, + keep_top_k=200, + ) class TestMulticlassNMS2(unittest.TestCase): - def test_multiclass_nms2(self): program = Program() with program_guard(program): - bboxes = layers.data(name='bboxes', - shape=[-1, 10, 4], - dtype='float32') + bboxes = layers.data( + name='bboxes', shape=[-1, 10, 4], dtype='float32' + ) scores = layers.data(name='scores', shape=[-1, 10], dtype='float32') - output = fluid.contrib.multiclass_nms2(bboxes, scores, 0.3, 400, - 200, 0.7) - output2, index = fluid.contrib.multiclass_nms2(bboxes, - scores, - 0.3, - 400, - 200, - 0.7, - return_index=True) + output = fluid.contrib.multiclass_nms2( + bboxes, scores, 0.3, 400, 200, 0.7 + ) + output2, index = fluid.contrib.multiclass_nms2( + bboxes, scores, 0.3, 400, 200, 0.7, return_index=True + ) self.assertIsNotNone(output) self.assertIsNotNone(output2) self.assertIsNotNone(index) class TestCollectFpnPropsals(LayerTest): - def test_collect_fpn_proposals(self): multi_bboxes_np = [] multi_scores_np = [] @@ -781,17 +837,21 @@ def test_collect_fpn_proposals(self): multi_scores = [] rois_num_per_level = [] for i in range(4): - bboxes = fluid.data(name='rois' + str(i), - shape=[5, 4], - dtype='float32', - lod_level=1) - scores = fluid.data(name='scores' + str(i), - shape=[5, 1], - dtype='float32', - lod_level=1) - rois_num = fluid.data(name='rois_num' + str(i), - shape=[None], - dtype='int32') + bboxes = fluid.data( + name='rois' + str(i), + shape=[5, 4], + dtype='float32', + lod_level=1, + ) + scores = fluid.data( + name='scores' + str(i), + shape=[5, 1], + dtype='float32', + lod_level=1, + ) + rois_num = fluid.data( + name='rois_num' + str(i), shape=[None], dtype='int32' + ) multi_bboxes.append(bboxes) multi_scores.append(scores) @@ -803,14 +863,16 @@ def test_collect_fpn_proposals(self): 2, 5, 10, - rois_num_per_level=rois_num_per_level) + rois_num_per_level=rois_num_per_level, + ) feed = {} for i in range(4): feed['rois' + str(i)] = multi_bboxes_np[i] feed['scores' + str(i)] = multi_scores_np[i] feed['rois_num' + str(i)] = rois_num_per_level_np[i] fpn_rois_stat, rois_num_stat = self.get_static_graph_result( - feed=feed, fetch_list=[fpn_rois, rois_num], with_lod=True) + feed=feed, fetch_list=[fpn_rois, rois_num], with_lod=True + ) fpn_rois_stat = np.array(fpn_rois_stat) rois_num_stat = np.array(rois_num_stat) @@ -831,7 +893,8 @@ def test_collect_fpn_proposals(self): 2, 5, 10, - rois_num_per_level=rois_num_per_level_dy) + rois_num_per_level=rois_num_per_level_dy, + ) fpn_rois_dy = fpn_rois_dy.numpy() rois_num_dy = rois_num_dy.numpy() @@ -839,72 +902,80 @@ def test_collect_fpn_proposals(self): np.testing.assert_array_equal(rois_num_stat, rois_num_dy) def test_collect_fpn_proposals_error(self): - def generate_input(bbox_type, score_type, name): multi_bboxes = [] multi_scores = [] for i in range(4): - bboxes = fluid.data(name='rois' + name + str(i), - shape=[10, 4], - dtype=bbox_type, - lod_level=1) - scores = fluid.data(name='scores' + name + str(i), - shape=[10, 1], - dtype=score_type, - lod_level=1) + bboxes = fluid.data( + name='rois' + name + str(i), + shape=[10, 4], + dtype=bbox_type, + lod_level=1, + ) + scores = fluid.data( + name='scores' + name + str(i), + shape=[10, 1], + dtype=score_type, + lod_level=1, + ) multi_bboxes.append(bboxes) multi_scores.append(scores) return multi_bboxes, multi_scores program = Program() with program_guard(program): - bbox1 = fluid.data(name='rois', - shape=[5, 10, 4], - dtype='float32', - lod_level=1) - score1 = fluid.data(name='scores', - shape=[5, 10, 1], - dtype='float32', - lod_level=1) + bbox1 = fluid.data( + name='rois', shape=[5, 10, 4], dtype='float32', lod_level=1 + ) + score1 = fluid.data( + name='scores', shape=[5, 10, 1], dtype='float32', lod_level=1 + ) bbox2, score2 = generate_input('int32', 'float32', '2') - self.assertRaises(TypeError, - layers.collect_fpn_proposals, - multi_rois=bbox1, - multi_scores=score1, - min_level=2, - max_level=5, - post_nms_top_n=2000) - self.assertRaises(TypeError, - layers.collect_fpn_proposals, - multi_rois=bbox2, - multi_scores=score2, - min_level=2, - max_level=5, - post_nms_top_n=2000) + self.assertRaises( + TypeError, + layers.collect_fpn_proposals, + multi_rois=bbox1, + multi_scores=score1, + min_level=2, + max_level=5, + post_nms_top_n=2000, + ) + self.assertRaises( + TypeError, + layers.collect_fpn_proposals, + multi_rois=bbox2, + multi_scores=score2, + min_level=2, + max_level=5, + post_nms_top_n=2000, + ) class TestDistributeFpnProposals(LayerTest): - def test_distribute_fpn_proposals(self): rois_np = np.random.rand(10, 4).astype('float32') rois_num_np = np.array([4, 6]).astype('int32') with self.static_graph(): rois = fluid.data(name='rois', shape=[10, 4], dtype='float32') rois_num = fluid.data(name='rois_num', shape=[None], dtype='int32') - multi_rois, restore_ind, rois_num_per_level = layers.distribute_fpn_proposals( + ( + multi_rois, + restore_ind, + rois_num_per_level, + ) = layers.distribute_fpn_proposals( fpn_rois=rois, min_level=2, max_level=5, refer_level=4, refer_scale=224, - rois_num=rois_num) + rois_num=rois_num, + ) fetch_list = multi_rois + [restore_ind] + rois_num_per_level - output_stat = self.get_static_graph_result(feed={ - 'rois': rois_np, - 'rois_num': rois_num_np - }, - fetch_list=fetch_list, - with_lod=True) + output_stat = self.get_static_graph_result( + feed={'rois': rois_np, 'rois_num': rois_num_np}, + fetch_list=fetch_list, + with_lod=True, + ) output_stat_np = [] for output in output_stat: output_np = np.array(output) @@ -914,13 +985,18 @@ def test_distribute_fpn_proposals(self): with self.dynamic_graph(): rois_dy = base.to_variable(rois_np) rois_num_dy = base.to_variable(rois_num_np) - multi_rois_dy, restore_ind_dy, rois_num_per_level_dy = layers.distribute_fpn_proposals( + ( + multi_rois_dy, + restore_ind_dy, + rois_num_per_level_dy, + ) = layers.distribute_fpn_proposals( fpn_rois=rois_dy, min_level=2, max_level=5, refer_level=4, refer_scale=224, - rois_num=rois_num_dy) + rois_num=rois_num_dy, + ) print(type(multi_rois_dy)) output_dy = multi_rois_dy + [restore_ind_dy] + rois_num_per_level_dy output_dy_np = [] @@ -935,83 +1011,93 @@ def test_distribute_fpn_proposals(self): def test_distribute_fpn_proposals_error(self): program = Program() with program_guard(program): - fpn_rois = fluid.data(name='data_error', - shape=[10, 4], - dtype='int32', - lod_level=1) - self.assertRaises(TypeError, - layers.distribute_fpn_proposals, - fpn_rois=fpn_rois, - min_level=2, - max_level=5, - refer_level=4, - refer_scale=224) + fpn_rois = fluid.data( + name='data_error', shape=[10, 4], dtype='int32', lod_level=1 + ) + self.assertRaises( + TypeError, + layers.distribute_fpn_proposals, + fpn_rois=fpn_rois, + min_level=2, + max_level=5, + refer_level=4, + refer_scale=224, + ) class TestBoxDecoderAndAssign(unittest.TestCase): - def test_box_decoder_and_assign(self): program = Program() with program_guard(program): pb = fluid.data(name='prior_box', shape=[None, 4], dtype='float32') pbv = fluid.data(name='prior_box_var', shape=[4], dtype='float32') - loc = fluid.data(name='target_box', - shape=[None, 4 * 81], - dtype='float32') - scores = fluid.data(name='scores', - shape=[None, 81], - dtype='float32') - decoded_box, output_assign_box = fluid.layers.box_decoder_and_assign( - pb, pbv, loc, scores, 4.135) + loc = fluid.data( + name='target_box', shape=[None, 4 * 81], dtype='float32' + ) + scores = fluid.data( + name='scores', shape=[None, 81], dtype='float32' + ) + ( + decoded_box, + output_assign_box, + ) = fluid.layers.box_decoder_and_assign(pb, pbv, loc, scores, 4.135) self.assertIsNotNone(decoded_box) self.assertIsNotNone(output_assign_box) def test_box_decoder_and_assign_error(self): - def generate_input(pb_type, pbv_type, loc_type, score_type, name): - pb = fluid.data(name='prior_box' + name, - shape=[None, 4], - dtype=pb_type) - pbv = fluid.data(name='prior_box_var' + name, - shape=[4], - dtype=pbv_type) - loc = fluid.data(name='target_box' + name, - shape=[None, 4 * 81], - dtype=loc_type) - scores = fluid.data(name='scores' + name, - shape=[None, 81], - dtype=score_type) + pb = fluid.data( + name='prior_box' + name, shape=[None, 4], dtype=pb_type + ) + pbv = fluid.data( + name='prior_box_var' + name, shape=[4], dtype=pbv_type + ) + loc = fluid.data( + name='target_box' + name, shape=[None, 4 * 81], dtype=loc_type + ) + scores = fluid.data( + name='scores' + name, shape=[None, 81], dtype=score_type + ) return pb, pbv, loc, scores program = Program() with program_guard(program): - pb1, pbv1, loc1, scores1 = generate_input('int32', 'float32', - 'float32', 'float32', '1') - pb2, pbv2, loc2, scores2 = generate_input('float32', 'float32', - 'int32', 'float32', '2') - pb3, pbv3, loc3, scores3 = generate_input('float32', 'float32', - 'float32', 'int32', '3') - self.assertRaises(TypeError, - layers.box_decoder_and_assign, - prior_box=pb1, - prior_box_var=pbv1, - target_box=loc1, - box_score=scores1, - box_clip=4.0) - self.assertRaises(TypeError, - layers.box_decoder_and_assign, - prior_box=pb2, - prior_box_var=pbv2, - target_box=loc2, - box_score=scores2, - box_clip=4.0) - self.assertRaises(TypeError, - layers.box_decoder_and_assign, - prior_box=pb3, - prior_box_var=pbv3, - target_box=loc3, - box_score=scores3, - box_clip=4.0) + pb1, pbv1, loc1, scores1 = generate_input( + 'int32', 'float32', 'float32', 'float32', '1' + ) + pb2, pbv2, loc2, scores2 = generate_input( + 'float32', 'float32', 'int32', 'float32', '2' + ) + pb3, pbv3, loc3, scores3 = generate_input( + 'float32', 'float32', 'float32', 'int32', '3' + ) + self.assertRaises( + TypeError, + layers.box_decoder_and_assign, + prior_box=pb1, + prior_box_var=pbv1, + target_box=loc1, + box_score=scores1, + box_clip=4.0, + ) + self.assertRaises( + TypeError, + layers.box_decoder_and_assign, + prior_box=pb2, + prior_box_var=pbv2, + target_box=loc2, + box_score=scores2, + box_clip=4.0, + ) + self.assertRaises( + TypeError, + layers.box_decoder_and_assign, + prior_box=pb3, + prior_box_var=pbv3, + target_box=loc3, + box_score=scores3, + box_clip=4.0, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/test_error_clip.py b/python/paddle/fluid/tests/test_error_clip.py index eac41b818c262..f6e1122e04562 100644 --- a/python/paddle/fluid/tests/test_error_clip.py +++ b/python/paddle/fluid/tests/test_error_clip.py @@ -36,12 +36,14 @@ prog_clip = prog.clone() prog_clip.block(0).var(hidden1.name)._set_error_clip( - fluid.clip.ErrorClipByValue(max=CLIP_MAX, min=CLIP_MIN)) + fluid.clip.ErrorClipByValue(max=CLIP_MAX, min=CLIP_MIN) +) avg_cost_clip = prog_clip.block(0).var(avg_cost.name) fluid.backward.append_backward(loss=avg_cost) -fluid.backward.append_backward(loss=avg_cost_clip, - callbacks=[fluid.clip.error_clip_callback]) +fluid.backward.append_backward( + loss=avg_cost_clip, callbacks=[fluid.clip.error_clip_callback] +) hidden1_grad = prog.block(0).var(hidden1.name + "@GRAD") hidden1_grad_clip = prog_clip.block(0).var(hidden1.name + "@GRAD") @@ -49,9 +51,10 @@ hidden2_grad = prog.block(0).var(hidden2.name + "@GRAD") hidden2_grad_clip = prog_clip.block(0).var(hidden2.name + "@GRAD") -train_reader = paddle.batch(paddle.reader.shuffle(paddle.dataset.mnist.train(), - buf_size=8192), - batch_size=BATCH_SIZE) +train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=8192), + batch_size=BATCH_SIZE, +) place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -63,15 +66,18 @@ count += 1 if count > 5: break - out1, out2 = exe.run(prog, - feed=feeder.feed(data), - fetch_list=[hidden1_grad, hidden2_grad]) + out1, out2 = exe.run( + prog, feed=feeder.feed(data), fetch_list=[hidden1_grad, hidden2_grad] + ) out1_clip, out2_clip = exe.run( prog_clip, feed=feeder.feed(data), - fetch_list=[hidden1_grad_clip, hidden2_grad_clip]) - if not ((out1.clip(min=CLIP_MIN, max=CLIP_MAX) == out1_clip).all() and - (out2 == out2_clip).all()): + fetch_list=[hidden1_grad_clip, hidden2_grad_clip], + ) + if not ( + (out1.clip(min=CLIP_MIN, max=CLIP_MAX) == out1_clip).all() + and (out2 == out2_clip).all() + ): exit(1) exit(0) diff --git a/python/paddle/fluid/tests/test_if_else_op.py b/python/paddle/fluid/tests/test_if_else_op.py index cc9f499641069..aa4aca0724c70 100644 --- a/python/paddle/fluid/tests/test_if_else_op.py +++ b/python/paddle/fluid/tests/test_if_else_op.py @@ -59,19 +59,19 @@ def not_test_raw_api(self): prob = layers.fc(input=hidden, size=10, act='softmax') layers.assign(input=prob, output=false_out) - prob = merge_lod_tensor(in_true=true_out, - in_false=false_out, - mask=cond, - x=image) + prob = merge_lod_tensor( + in_true=true_out, in_false=false_out, mask=cond, x=image + ) loss = layers.cross_entropy(input=prob, label=label) avg_loss = paddle.mean(loss) optimizer = MomentumOptimizer(learning_rate=0.001, momentum=0.9) optimizer.minimize(avg_loss, startup_prog) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=8192), - batch_size=10) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=8192), + batch_size=10, + ) place = core.CPUPlace() exe = Executor(place) @@ -84,12 +84,9 @@ def not_test_raw_api(self): y_data = np.array([x[1] for x in data]).astype("int64") y_data = np.expand_dims(y_data, axis=1) - outs = exe.run(prog, - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[avg_loss]) + outs = exe.run( + prog, feed={'x': x_data, 'y': y_data}, fetch_list=[avg_loss] + ) print(outs[0]) if outs[0] < 1.0: return @@ -126,9 +123,10 @@ def not_test_ifelse(self): optimizer = MomentumOptimizer(learning_rate=0.001, momentum=0.9) optimizer.minimize(avg_loss, startup_prog) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=8192), - batch_size=200) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=8192), + batch_size=200, + ) place = core.CPUPlace() exe = Executor(place) @@ -141,12 +139,9 @@ def not_test_ifelse(self): y_data = np.array([x[1] for x in data]).astype("int64") y_data = y_data.reshape((y_data.shape[0], 1)) - outs = exe.run(prog, - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[avg_loss]) + outs = exe.run( + prog, feed={'x': x_data, 'y': y_data}, fetch_list=[avg_loss] + ) print(outs[0]) if outs[0] < 1.0: return @@ -154,7 +149,6 @@ def not_test_ifelse(self): class TestIfElse(unittest.TestCase): - def set_test_case(self): # condiction is: self.data < self.cond_value self.cond_value = 0.5 @@ -174,9 +168,9 @@ def compare_ifelse_op_and_numpy(self, place): startup_prog = Program() with program_guard(prog, startup_prog): src = layers.data(name='data', shape=[1], dtype='float32') - cond = layers.fill_constant([1], - dtype='float32', - value=self.cond_value) + cond = layers.fill_constant( + [1], dtype='float32', value=self.cond_value + ) ifcond = layers.less_than(x=src, y=cond) ie = layers.IfElse(ifcond) with ie.true_block(): @@ -194,9 +188,11 @@ def compare_ifelse_op_and_numpy(self, place): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) fetch_list = [out] - o1, = exe.run(fluid.default_main_program(), - feed={'data': self.data}, - fetch_list=[out]) + (o1,) = exe.run( + fluid.default_main_program(), + feed={'data': self.data}, + fetch_list=[out], + ) o2 = self.numpy_cal() np.testing.assert_allclose( @@ -216,31 +212,28 @@ def test_cuda(self): class TestIfElseTrueBranch(TestIfElse): - def set_test_case(self): # condiction is: self.data < self.cond_value - self.cond_value = 10. + self.cond_value = 10.0 self.data = np.random.rand(25, 1).astype(np.float32) class TestIfElseFalseBranch(TestIfElse): - def set_test_case(self): # condiction is: self.data < self.cond_value - self.cond_value = -10. + self.cond_value = -10.0 self.data = np.random.rand(25, 1).astype(np.float32) class TestIfElseError(unittest.TestCase): - def test_input_type_error(self): main_program = Program() startup_program = Program() with program_guard(main_program, startup_program): src = layers.data(name='data', shape=[1], dtype='float32') - const_value = layers.fill_constant([1], - dtype='float32', - value=123.0) + const_value = layers.fill_constant( + [1], dtype='float32', value=123.0 + ) ifcond = layers.less_than(x=src, y=const_value) with self.assertRaises(TypeError): ie = layers.IfElse(set()) diff --git a/python/paddle/fluid/tests/test_lod_tensor.py b/python/paddle/fluid/tests/test_lod_tensor.py index 4650c3ec2dfac..b84a11dbdc674 100644 --- a/python/paddle/fluid/tests/test_lod_tensor.py +++ b/python/paddle/fluid/tests/test_lod_tensor.py @@ -14,28 +14,33 @@ import paddle.fluid as fluid import paddle.fluid.core as core -from paddle.fluid.lod_tensor import create_lod_tensor, create_random_int_lodtensor +from paddle.fluid.lod_tensor import ( + create_lod_tensor, + create_random_int_lodtensor, +) import numpy as np import unittest class TestLoDTensor(unittest.TestCase): - def test_pybind_recursive_seq_lens(self): tensor = fluid.LoDTensor() recursive_seq_lens = [] tensor.set_recursive_sequence_lengths(recursive_seq_lens) recursive_seq_lens = [[], [1], [3]] - self.assertRaises(Exception, tensor.set_recursive_sequence_lengths, - recursive_seq_lens) + self.assertRaises( + Exception, tensor.set_recursive_sequence_lengths, recursive_seq_lens + ) recursive_seq_lens = [[0], [2], [3]] - self.assertRaises(Exception, tensor.set_recursive_sequence_lengths, - recursive_seq_lens) + self.assertRaises( + Exception, tensor.set_recursive_sequence_lengths, recursive_seq_lens + ) recursive_seq_lens = [[1, 2, 3]] tensor.set_recursive_sequence_lengths(recursive_seq_lens) - self.assertEqual(tensor.recursive_sequence_lengths(), - recursive_seq_lens) + self.assertEqual( + tensor.recursive_sequence_lengths(), recursive_seq_lens + ) tensor.set(np.random.random([6, 1]), fluid.CPUPlace()) self.assertTrue(tensor.has_valid_recursive_sequence_lengths()) tensor.set(np.random.random([9, 1]), fluid.CPUPlace()) @@ -45,8 +50,9 @@ def test_pybind_recursive_seq_lens(self): # Moreover, last level's sum should be equal to the tensor height recursive_seq_lens = [[2, 3], [1, 3, 1, 2, 2]] tensor.set_recursive_sequence_lengths(recursive_seq_lens) - self.assertEqual(tensor.recursive_sequence_lengths(), - recursive_seq_lens) + self.assertEqual( + tensor.recursive_sequence_lengths(), recursive_seq_lens + ) tensor.set(np.random.random([8, 1]), fluid.CPUPlace()) self.assertFalse(tensor.has_valid_recursive_sequence_lengths()) recursive_seq_lens = [[2, 3], [1, 3, 1, 2, 1]] @@ -57,40 +63,54 @@ def test_pybind_recursive_seq_lens(self): def test_create_lod_tensor(self): # Create LoDTensor from a list - data = [[np.int64(1), np.int64(2), - np.int64(3)], [np.int64(3), np.int64(4)]] + data = [ + [np.int64(1), np.int64(2), np.int64(3)], + [np.int64(3), np.int64(4)], + ] wrong_recursive_seq_lens = [[2, 2]] correct_recursive_seq_lens = [[3, 2]] - self.assertRaises(AssertionError, create_lod_tensor, data, - wrong_recursive_seq_lens, fluid.CPUPlace()) - tensor = create_lod_tensor(data, correct_recursive_seq_lens, - fluid.CPUPlace()) - self.assertEqual(tensor.recursive_sequence_lengths(), - correct_recursive_seq_lens) + self.assertRaises( + AssertionError, + create_lod_tensor, + data, + wrong_recursive_seq_lens, + fluid.CPUPlace(), + ) + tensor = create_lod_tensor( + data, correct_recursive_seq_lens, fluid.CPUPlace() + ) + self.assertEqual( + tensor.recursive_sequence_lengths(), correct_recursive_seq_lens + ) self.assertEqual(tensor._dtype(), core.VarDesc.VarType.INT64) self.assertEqual(tensor.shape(), [5, 1]) np.testing.assert_array_equal( np.array(tensor), - np.array([1, 2, 3, 3, 4]).reshape(tensor.shape()).astype('int64')) + np.array([1, 2, 3, 3, 4]).reshape(tensor.shape()).astype('int64'), + ) # Create LoDTensor from numpy array data = np.random.random([10, 1]).astype('float64') recursive_seq_lens = [[2, 1], [3, 3, 4]] tensor = create_lod_tensor(data, recursive_seq_lens, fluid.CPUPlace()) - self.assertEqual(tensor.recursive_sequence_lengths(), - recursive_seq_lens) + self.assertEqual( + tensor.recursive_sequence_lengths(), recursive_seq_lens + ) self.assertEqual(tensor._dtype(), core.VarDesc.VarType.FP64) self.assertEqual(tensor.shape(), [10, 1]) np.testing.assert_array_equal(np.array(tensor), data) # Create LoDTensor from another LoDTensor, they are differnt instances new_recursive_seq_lens = [[2, 2, 1], [1, 2, 2, 3, 2]] - new_tensor = create_lod_tensor(tensor, new_recursive_seq_lens, - fluid.CPUPlace()) - self.assertEqual(tensor.recursive_sequence_lengths(), - recursive_seq_lens) - self.assertEqual(new_tensor.recursive_sequence_lengths(), - new_recursive_seq_lens) + new_tensor = create_lod_tensor( + tensor, new_recursive_seq_lens, fluid.CPUPlace() + ) + self.assertEqual( + tensor.recursive_sequence_lengths(), recursive_seq_lens + ) + self.assertEqual( + new_tensor.recursive_sequence_lengths(), new_recursive_seq_lens + ) def test_create_random_int_lodtensor(self): # The shape of a word, commonly used in speech and NLP problem, is [1] @@ -99,10 +119,12 @@ def test_create_random_int_lodtensor(self): dict_size = 10000 low = 0 high = dict_size - 1 - tensor = create_random_int_lodtensor(recursive_seq_lens, shape, - fluid.CPUPlace(), low, high) - self.assertEqual(tensor.recursive_sequence_lengths(), - recursive_seq_lens) + tensor = create_random_int_lodtensor( + recursive_seq_lens, shape, fluid.CPUPlace(), low, high + ) + self.assertEqual( + tensor.recursive_sequence_lengths(), recursive_seq_lens + ) self.assertEqual(tensor.shape(), [10, 1]) def test_print_lodtensor(self): @@ -111,43 +133,53 @@ def test_print_lodtensor(self): dict_size = 100 low = 0 high = dict_size - 1 - tensor = create_random_int_lodtensor(recursive_seq_lens, shape, - fluid.CPUPlace(), low, high) + tensor = create_random_int_lodtensor( + recursive_seq_lens, shape, fluid.CPUPlace(), low, high + ) print(tensor) self.assertTrue(isinstance(str(tensor), str)) if core.is_compiled_with_cuda(): - gtensor = create_random_int_lodtensor(recursive_seq_lens, shape, - fluid.CUDAPlace(0), low, high) + gtensor = create_random_int_lodtensor( + recursive_seq_lens, shape, fluid.CUDAPlace(0), low, high + ) print(gtensor) self.assertTrue(isinstance(str(gtensor), str)) def test_dlpack_support(self): tensor = fluid.create_lod_tensor( - np.array([[1], [2], [3], [4]]).astype('int'), [[1, 3]], - fluid.CPUPlace()) + np.array([[1], [2], [3], [4]]).astype('int'), + [[1, 3]], + fluid.CPUPlace(), + ) dltensor = tensor._to_dlpack() tensor_from_dlpack = fluid.core.from_dlpack(dltensor) self.assertTrue(isinstance(tensor_from_dlpack, fluid.core.Tensor)) np.testing.assert_array_equal( np.array(tensor_from_dlpack), - np.array([[1], [2], [3], [4]]).astype('int')) + np.array([[1], [2], [3], [4]]).astype('int'), + ) # when build with cuda if core.is_compiled_with_cuda(): gtensor = fluid.create_lod_tensor( - np.array([[1], [2], [3], [4]]).astype('int'), [[1, 3]], - fluid.CUDAPlace(0)) + np.array([[1], [2], [3], [4]]).astype('int'), + [[1, 3]], + fluid.CUDAPlace(0), + ) gdltensor = gtensor._to_dlpack() gtensor_from_dlpack = fluid.core.from_dlpack(gdltensor) self.assertTrue(isinstance(gtensor_from_dlpack, fluid.core.Tensor)) np.testing.assert_array_equal( np.array(gtensor_from_dlpack), - np.array([[1], [2], [3], [4]]).astype('int')) + np.array([[1], [2], [3], [4]]).astype('int'), + ) def test_as_type(self): tensor = fluid.create_lod_tensor( - np.array([[1], [2], [3], [4]]).astype('int'), [[1, 3]], - fluid.CPUPlace()) + np.array([[1], [2], [3], [4]]).astype('int'), + [[1, 3]], + fluid.CPUPlace(), + ) fp32_tensor = tensor._as_type(core.VarDesc.VarType.FP32) print(fp32_tensor) diff --git a/python/paddle/fluid/tests/test_python_operator_overriding.py b/python/paddle/fluid/tests/test_python_operator_overriding.py index 7fded67c9bac3..562b8cef5ff6a 100644 --- a/python/paddle/fluid/tests/test_python_operator_overriding.py +++ b/python/paddle/fluid/tests/test_python_operator_overriding.py @@ -25,7 +25,6 @@ class TestPythonOperatorOverride(unittest.TestCase): - def check_result(self, fn, place, dtype): shape = [9, 10] @@ -33,27 +32,22 @@ def check_result(self, fn, place, dtype): y_data = np.random.random(size=shape).astype(dtype) python_out = fn(x_data, y_data) - x_var = layers.create_global_var(name='x', - shape=shape, - value=0.0, - dtype=dtype, - persistable=True) - y_var = layers.create_global_var(name='y', - shape=shape, - value=0.0, - dtype=dtype, - persistable=True) + x_var = layers.create_global_var( + name='x', shape=shape, value=0.0, dtype=dtype, persistable=True + ) + y_var = layers.create_global_var( + name='y', shape=shape, value=0.0, dtype=dtype, persistable=True + ) out = fn(x_var, y_var) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - fluid_out = exe.run(fluid.default_main_program(), - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[out]) + fluid_out = exe.run( + fluid.default_main_program(), + feed={'x': x_data, 'y': y_data}, + fetch_list=[out], + ) np.testing.assert_array_equal(python_out, fluid_out[0]) @@ -79,8 +73,9 @@ def test_override(self): for place in places: for dtype in dtypes: for compare_fn in compare_fns: - with framework.program_guard(framework.Program(), - framework.Program()): + with framework.program_guard( + framework.Program(), framework.Program() + ): self.check_result(compare_fn, place, dtype) diff --git a/python/paddle/fluid/tests/test_sequential.py b/python/paddle/fluid/tests/test_sequential.py index 09cfbcdd7e378..7446bb83841aa 100644 --- a/python/paddle/fluid/tests/test_sequential.py +++ b/python/paddle/fluid/tests/test_sequential.py @@ -17,7 +17,6 @@ class TestDataFeeder(unittest.TestCase): - def test_lod_level_1_converter(self): sequential = paddle.nn.Sequential() diff --git a/python/paddle/fluid/tests/unittests/__init__.py b/python/paddle/fluid/tests/unittests/__init__.py index e427eb512474f..126cbeb493887 100644 --- a/python/paddle/fluid/tests/unittests/__init__.py +++ b/python/paddle/fluid/tests/unittests/__init__.py @@ -17,8 +17,10 @@ # please refer to https://stackoverflow.com/questions/8953844/import-module-from-subfolder import os + if os.name == 'nt': import sys + dirname, filename = os.path.split(os.path.abspath(__file__)) sys.path.insert(0, dirname) print(sys.path) diff --git a/python/paddle/fluid/tests/unittests/ascend_group.py b/python/paddle/fluid/tests/unittests/ascend_group.py index 32b42a00fc657..b0928f409319e 100644 --- a/python/paddle/fluid/tests/unittests/ascend_group.py +++ b/python/paddle/fluid/tests/unittests/ascend_group.py @@ -35,8 +35,9 @@ fleet.init(role) -def init_communicator(startup_program, main_program, current_endpoint, - endpoints, ring_id): +def init_communicator( + startup_program, main_program, current_endpoint, endpoints, ring_id +): nranks = len(endpoints) other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) @@ -44,55 +45,64 @@ def init_communicator(startup_program, main_program, current_endpoint, assert group_rank >= 0 block = startup_program.global_block() - nccl_id_var = block.create_var(name=unique_name.generate('nccl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_nccl_id', - inputs={}, - outputs={'Out': nccl_id_var}, - attrs={ - 'rank': group_rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints, - OP_ROLE_KEY: OpRole.Forward, - }) - block.append_op(type='c_comm_init', - inputs={'X': nccl_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': group_rank, - 'ring_id': ring_id, - OP_ROLE_KEY: OpRole.Forward, - }) + nccl_id_var = block.create_var( + name=unique_name.generate('nccl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_nccl_id', + inputs={}, + outputs={'Out': nccl_id_var}, + attrs={ + 'rank': group_rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + OP_ROLE_KEY: OpRole.Forward, + }, + ) + block.append_op( + type='c_comm_init', + inputs={'X': nccl_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': group_rank, + 'ring_id': ring_id, + OP_ROLE_KEY: OpRole.Forward, + }, + ) # add input op for test fill_var_name = "tensor@Filled" - fill_var = block.create_var(name=fill_var_name, - shape=[10, 10], - dtype='float32', - persistable=False, - stop_gradient=True) - block.append_op(type="fill_constant", - outputs={"Out": fill_var_name}, - attrs={ - "shape": [10, 10], - "dtype": fill_var.dtype, - "value": 1.0, - "place_type": 1 - }) + fill_var = block.create_var( + name=fill_var_name, + shape=[10, 10], + dtype='float32', + persistable=False, + stop_gradient=True, + ) + block.append_op( + type="fill_constant", + outputs={"Out": fill_var_name}, + attrs={ + "shape": [10, 10], + "dtype": fill_var.dtype, + "value": 1.0, + "place_type": 1, + }, + ) with fluid.program_guard(main_program): op_type = "c_allreduce_sum" data = fluid.layers.fill_constant(shape=[1], dtype='float32', value=2.5) helper = LayerHelper(op_type, **locals()) - helper.append_op(type=op_type, - inputs={'X': [data]}, - outputs={'Out': [data]}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True - }) + helper.append_op( + type=op_type, + inputs={'X': [data]}, + outputs={'Out': [data]}, + attrs={'ring_id': ring_id, 'use_calc_stream': True}, + ) print("startup program:", startup_program) print("main program:", main_program) @@ -102,7 +112,7 @@ def train(world_endpoints, world_device_ids, local_device_ids, local_rank): startup_programs = [] main_programs = [] - #trainer_endpoints=["127.0.0.1:6071","127.0.0.1:6072","127.0.0.1:6073","127.0.0.1:6074"] + # trainer_endpoints=["127.0.0.1:6071","127.0.0.1:6072","127.0.0.1:6073","127.0.0.1:6074"] trainer_endpoints = world_endpoints groups = [[], [], []] groups[0] = [trainer_endpoints[0], trainer_endpoints[1]] @@ -132,10 +142,12 @@ def train(world_endpoints, world_device_ids, local_device_ids, local_rank): main_program = main_programs[local_rank] loss = Loss(Block(main_program)) optimizer = ascend_optimizer.AscendOptimizer(None, fetch_list=[]) - optimizer.minimize(loss, - startup_program, - auto_dp=True, - rank_table_file=os.getenv("RANK_TABLE_FILE", None)) + optimizer.minimize( + loss, + startup_program, + auto_dp=True, + rank_table_file=os.getenv("RANK_TABLE_FILE", None), + ) exe = paddle.static.Executor(paddle.CPUPlace()) exe.run(startup_program) diff --git a/python/paddle/fluid/tests/unittests/ascend_multi_process_collective.py b/python/paddle/fluid/tests/unittests/ascend_multi_process_collective.py index 1853e529d51dc..113c4286f352e 100644 --- a/python/paddle/fluid/tests/unittests/ascend_multi_process_collective.py +++ b/python/paddle/fluid/tests/unittests/ascend_multi_process_collective.py @@ -27,12 +27,21 @@ def train(prefix): device_ids = os.getenv("PADDLE_WORLD_DEVICE_IDS") current_device_id = os.getenv("PADDLE_LOCAL_DEVICE_IDS") - details = "selected_accelerators:{} selected_npus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{} device_ids:{} device_id:{}"\ - .format(selected_accelerators, selected_npus, worker_endpoints, trainers_num, current_endpoint,trainer_id,device_ids, current_device_id) + details = "selected_accelerators:{} selected_npus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{} device_ids:{} device_id:{}".format( + selected_accelerators, + selected_npus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + device_ids, + current_device_id, + ) print(details) - with open("multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + with open( + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(details) diff --git a/python/paddle/fluid/tests/unittests/asp/asp_pruning_base.py b/python/paddle/fluid/tests/unittests/asp/asp_pruning_base.py index 6e9603e676179..e21d5f598ebf5 100644 --- a/python/paddle/fluid/tests/unittests/asp/asp_pruning_base.py +++ b/python/paddle/fluid/tests/unittests/asp/asp_pruning_base.py @@ -24,21 +24,18 @@ class TestASPHelperPruningBase(unittest.TestCase): - def setUp(self): self.main_program = fluid.Program() self.startup_program = fluid.Program() def build_model(): - img = fluid.data(name='img', - shape=[None, 3, 32, 32], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 3, 32, 32], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') - hidden = fluid.layers.conv2d(input=img, - num_filters=4, - filter_size=3, - padding=2, - act="relu") + hidden = fluid.layers.conv2d( + input=img, num_filters=4, filter_size=3, padding=2, act="relu" + ) hidden = fluid.layers.fc(input=hidden, size=32, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') return img, label, prediction @@ -46,23 +43,26 @@ def build_model(): with fluid.program_guard(self.main_program, self.startup_program): self.img, self.label, self.predict = build_model() - def run_inference_pruning_test(self, get_mask_gen_func, - get_mask_check_func): + def run_inference_pruning_test( + self, get_mask_gen_func, get_mask_check_func + ): place = paddle.CPUPlace() if core.is_compiled_with_cuda(): place = paddle.CUDAPlace(0) exe = fluid.Executor(place) - self.__pruning_and_checking(exe, place, get_mask_gen_func, - get_mask_check_func, False) + self.__pruning_and_checking( + exe, place, get_mask_gen_func, get_mask_check_func, False + ) def run_training_pruning_test(self, get_mask_gen_func, get_mask_check_func): with fluid.program_guard(self.main_program, self.startup_program): loss = paddle.mean( - fluid.layers.cross_entropy(input=self.predict, - label=self.label)) + fluid.layers.cross_entropy(input=self.predict, label=self.label) + ) optimizer = paddle.incubate.asp.decorate( - fluid.optimizer.SGD(learning_rate=0.01)) + fluid.optimizer.SGD(learning_rate=0.01) + ) optimizer.minimize(loss, self.startup_program) place = paddle.CPUPlace() @@ -70,19 +70,24 @@ def run_training_pruning_test(self, get_mask_gen_func, get_mask_check_func): place = paddle.CUDAPlace(0) exe = fluid.Executor(place) - self.__pruning_and_checking(exe, place, get_mask_gen_func, - get_mask_check_func, True) + self.__pruning_and_checking( + exe, place, get_mask_gen_func, get_mask_check_func, True + ) - def __pruning_and_checking(self, exe, place, mask_func_name, - check_func_name, with_mask): + def __pruning_and_checking( + self, exe, place, mask_func_name, check_func_name, with_mask + ): exe.run(self.startup_program) - paddle.incubate.asp.prune_model(self.main_program, - mask_algo=mask_func_name, - with_mask=with_mask) + paddle.incubate.asp.prune_model( + self.main_program, mask_algo=mask_func_name, with_mask=with_mask + ) for param in self.main_program.global_block().all_parameters(): if ASPHelper._is_supported_layer(self.main_program, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) self.assertTrue( paddle.fluid.contrib.sparsity.check_sparsity( - mat.T, func_name=check_func_name, n=2, m=4)) + mat.T, func_name=check_func_name, n=2, m=4 + ) + ) diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_customized_pruning.py b/python/paddle/fluid/tests/unittests/asp/test_asp_customized_pruning.py index fde68b39e7b09..cdccd1071421d 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_customized_pruning.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_customized_pruning.py @@ -19,12 +19,13 @@ import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.contrib import sparsity -from paddle.fluid.contrib.sparsity.supported_layer_list import supported_layers_and_prune_func_map +from paddle.fluid.contrib.sparsity.supported_layer_list import ( + supported_layers_and_prune_func_map, +) from paddle.fluid.dygraph.layers import Layer, _convert_camel_to_snake class MyOwnLayer(Layer): - def __init__(self): super(MyOwnLayer, self).__init__() @@ -47,7 +48,6 @@ def my_own_pruning(tensor, m, n, mask_algo, param_name): class TestASPAddSupportedLayer(unittest.TestCase): - def test_add_supported_layer_via_name(self): sparsity.add_supported_layer("test_supported_1") sparsity.add_supported_layer("test_supported_2", my_own_pruning) @@ -55,37 +55,41 @@ def test_add_supported_layer_via_name(self): my_own_layer_name = _convert_camel_to_snake(MyOwnLayer.__name__) self.assertTrue( - "test_supported_1" in supported_layers_and_prune_func_map) + "test_supported_1" in supported_layers_and_prune_func_map + ) + self.assertTrue( + "test_supported_2" in supported_layers_and_prune_func_map + ) self.assertTrue( - "test_supported_2" in supported_layers_and_prune_func_map) + "test_supported_2" in supported_layers_and_prune_func_map + ) self.assertTrue( - "test_supported_2" in supported_layers_and_prune_func_map) - self.assertTrue(supported_layers_and_prune_func_map["test_supported_2"] - == my_own_pruning) + supported_layers_and_prune_func_map["test_supported_2"] + == my_own_pruning + ) self.assertTrue( - my_own_layer_name in supported_layers_and_prune_func_map) + my_own_layer_name in supported_layers_and_prune_func_map + ) class TestASPDynamicCustomerizedPruneFunc(unittest.TestCase): - def setUp(self): paddle.disable_static() class CustomerLayer(paddle.nn.Layer): - def __init__(self): super(CustomerLayer, self).__init__() - self.weight = self.create_parameter(shape=[32, 32], - attr=None, - dtype='float32', - is_bias=False) + self.weight = self.create_parameter( + shape=[32, 32], attr=None, dtype='float32', is_bias=False + ) self.linear1 = paddle.nn.Linear(32, 32) self.linear2 = paddle.nn.Linear(32, 10) def forward(self, input_): - hidden = paddle.nn.functional.linear(x=input_, - weight=self.weight) + hidden = paddle.nn.functional.linear( + x=input_, weight=self.weight + ) hidden = self.linear1(hidden) out = self.linear2(hidden) return out @@ -93,8 +97,11 @@ def forward(self, input_): sparsity.add_supported_layer(CustomerLayer, my_own_pruning) self.layer = CustomerLayer() - self.customer_prefix = paddle.fluid.dygraph.layers._convert_camel_to_snake( - CustomerLayer.__name__) + self.customer_prefix = ( + paddle.fluid.dygraph.layers._convert_camel_to_snake( + CustomerLayer.__name__ + ) + ) self.supported_layer_count_ref = 3 def test_inference_pruning(self): @@ -106,23 +113,28 @@ def test_inference_pruning(self): mat = param.numpy() if sparsity.asp.ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): supported_layer_count += 1 - if (self.customer_prefix in param.name): + if self.customer_prefix in param.name: self.assertLessEqual( - np.sum(mat.flatten() - static_tensor.flatten()), 1e-4) + np.sum(mat.flatten() - static_tensor.flatten()), 1e-4 + ) else: self.assertTrue( sparsity.check_sparsity( mat.T, func_name=sparsity.CheckMethod.CHECK_1D, n=2, - m=4)) + m=4, + ) + ) self.assertEqual(supported_layer_count, self.supported_layer_count_ref) def test_training_pruning(self): - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=self.layer.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=self.layer.parameters() + ) optimizer = sparsity.decorate(optimizer) sparsity.prune_model(self.layer, mask_algo="mask_1d", with_mask=True) @@ -132,37 +144,49 @@ def test_training_pruning(self): mat = param.numpy() if sparsity.asp.ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): - mat_mask = sparsity.asp.ASPHelper._get_program_asp_info( - paddle.static.default_main_program()).mask_vars[ - param.name].numpy() + mat_mask = ( + sparsity.asp.ASPHelper._get_program_asp_info( + paddle.static.default_main_program() + ) + .mask_vars[param.name] + .numpy() + ) supported_layer_count += 1 - if (self.customer_prefix in param.name): + if self.customer_prefix in param.name: self.assertLessEqual( - np.sum(mat.flatten() - static_tensor.flatten()), 1e-4) + np.sum(mat.flatten() - static_tensor.flatten()), 1e-4 + ) self.assertLessEqual( - np.sum(mat_mask.flatten() - - static_tensor_mask.flatten()), 1e-4) + np.sum( + mat_mask.flatten() - static_tensor_mask.flatten() + ), + 1e-4, + ) else: self.assertTrue( sparsity.check_sparsity( mat.T, func_name=sparsity.CheckMethod.CHECK_1D, n=2, - m=4)) + m=4, + ) + ) self.assertTrue( sparsity.check_sparsity( mat_mask.T, func_name=sparsity.CheckMethod.CHECK_1D, n=2, - m=4)) + m=4, + ) + ) self.assertEqual(supported_layer_count, self.supported_layer_count_ref) class TestASPStaticCustomerizedPruneFunc(unittest.TestCase): - def setUp(self): paddle.enable_static() @@ -172,23 +196,19 @@ def setUp(self): self.customer_prefix = "customer_layer" def build_model(): - img = fluid.data(name='img', - shape=[None, 3, 32, 32], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 3, 32, 32], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') - hidden = fluid.layers.conv2d(input=img, - num_filters=4, - filter_size=3, - padding=2, - act="relu") - hidden = fluid.layers.fc(input=hidden, - size=32, - act='relu', - name=self.customer_prefix) - hidden = fluid.layers.fc(input=hidden, - size=32, - act='relu', - name=self.customer_prefix) + hidden = fluid.layers.conv2d( + input=img, num_filters=4, filter_size=3, padding=2, act="relu" + ) + hidden = fluid.layers.fc( + input=hidden, size=32, act='relu', name=self.customer_prefix + ) + hidden = fluid.layers.fc( + input=hidden, size=32, act='relu', name=self.customer_prefix + ) hidden = fluid.layers.fc(input=hidden, size=32, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') return img, label, prediction @@ -207,89 +227,110 @@ def build_model(): def test_inference_pruning(self): self.exe.run(self.startup_program) - sparsity.prune_model(self.main_program, - mask_algo="mask_1d", - with_mask=False) + sparsity.prune_model( + self.main_program, mask_algo="mask_1d", with_mask=False + ) supported_layer_count = 0 for param in self.main_program.global_block().all_parameters(): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) if sparsity.asp.ASPHelper._is_supported_layer( - self.main_program, param.name): + self.main_program, param.name + ): supported_layer_count += 1 - if (self.customer_prefix in param.name): + if self.customer_prefix in param.name: self.assertLessEqual( - np.sum(mat.flatten() - static_tensor.flatten()), 1e-4) + np.sum(mat.flatten() - static_tensor.flatten()), 1e-4 + ) else: - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( sparsity.check_sparsity( mat.T, func_name=sparsity.CheckMethod.CHECK_1D, n=2, - m=4)) + m=4, + ) + ) self.assertEqual(supported_layer_count, self.supported_layer_count_ref) def test_training_pruning(self): with fluid.program_guard(self.main_program, self.startup_program): loss = paddle.mean( - fluid.layers.cross_entropy(input=self.predict, - label=self.label)) + fluid.layers.cross_entropy(input=self.predict, label=self.label) + ) optimizer = sparsity.decorate( - fluid.optimizer.SGD(learning_rate=0.01)) + fluid.optimizer.SGD(learning_rate=0.01) + ) optimizer.minimize(loss, self.startup_program) self.exe.run(self.startup_program) - sparsity.prune_model(self.main_program, - mask_algo="mask_1d", - with_mask=True) + sparsity.prune_model( + self.main_program, mask_algo="mask_1d", with_mask=True + ) supported_layer_count = 0 for param in self.main_program.global_block().all_parameters(): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) if sparsity.asp.ASPHelper._is_supported_layer( - self.main_program, param.name): - mat_mask = np.array(fluid.global_scope().find_var( - sparsity.asp.ASPHelper._get_mask_name( - param.name)).get_tensor()) + self.main_program, param.name + ): + mat_mask = np.array( + fluid.global_scope() + .find_var(sparsity.asp.ASPHelper._get_mask_name(param.name)) + .get_tensor() + ) supported_layer_count += 1 - if (self.customer_prefix in param.name): + if self.customer_prefix in param.name: self.assertLessEqual( - np.sum(mat.flatten() - static_tensor.flatten()), 1e-4) + np.sum(mat.flatten() - static_tensor.flatten()), 1e-4 + ) self.assertLessEqual( - np.sum(mat_mask.flatten() - - static_tensor_mask.flatten()), 1e-4) + np.sum( + mat_mask.flatten() - static_tensor_mask.flatten() + ), + 1e-4, + ) else: - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - sparsity.check_sparsity(mat.T, n=2, m=4)) + sparsity.check_sparsity(mat.T, n=2, m=4) + ) self.assertFalse( - sparsity.check_sparsity(mat_mask.T, n=2, m=4)) + sparsity.check_sparsity(mat_mask.T, n=2, m=4) + ) else: self.assertTrue( sparsity.check_sparsity( mat.T, func_name=sparsity.CheckMethod.CHECK_1D, n=2, - m=4)) + m=4, + ) + ) self.assertTrue( sparsity.check_sparsity( mat_mask.T, func_name=sparsity.CheckMethod.CHECK_1D, n=2, - m=4)) + m=4, + ) + ) self.assertEqual(supported_layer_count, self.supported_layer_count_ref) diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_dynamic.py b/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_dynamic.py index a501ed4279474..389645139c578 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_dynamic.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_dynamic.py @@ -21,13 +21,11 @@ class MyLayer(paddle.nn.Layer): - def __init__(self): super(MyLayer, self).__init__() - self.conv1 = paddle.nn.Conv2D(in_channels=3, - out_channels=2, - kernel_size=3, - padding=2) + self.conv1 = paddle.nn.Conv2D( + in_channels=3, out_channels=2, kernel_size=3, padding=2 + ) self.linear1 = paddle.nn.Linear(1352, 32) self.linear2 = paddle.nn.Linear(32, 32) self.linear3 = paddle.nn.Linear(32, 10) @@ -42,7 +40,6 @@ def forward(self, img): class TestASPDynamicOptimize(unittest.TestCase): - def setUp(self): self.layer = MyLayer() @@ -52,41 +49,84 @@ def setUp(self): self.place = paddle.CUDAPlace(0) self.optimizer = paddle.optimizer.SGD( - learning_rate=0.01, parameters=self.layer.parameters()) + learning_rate=0.01, parameters=self.layer.parameters() + ) def test_is_supported_layers(self): program = paddle.static.default_main_program() names = [ - 'embedding_0.w_0', 'fack_layer_0.w_0', 'conv2d_0.w_0', - 'conv2d_0.b_0', 'conv2d_1.w_0', 'conv2d_1.b_0', 'fc_0.w_0', - 'fc_0.b_0', 'fc_1.w_0', 'fc_1.b_0', 'linear_2.w_0', 'linear_2.b_0' + 'embedding_0.w_0', + 'fack_layer_0.w_0', + 'conv2d_0.w_0', + 'conv2d_0.b_0', + 'conv2d_1.w_0', + 'conv2d_1.b_0', + 'fc_0.w_0', + 'fc_0.b_0', + 'fc_1.w_0', + 'fc_1.b_0', + 'linear_2.w_0', + 'linear_2.b_0', ] ref = [ - False, False, True, False, True, False, True, False, True, False, - True, False + False, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, ] for i, name in enumerate(names): self.assertTrue( - ref[i] == ASPHelper._is_supported_layer(program, name)) + ref[i] == ASPHelper._is_supported_layer(program, name) + ) paddle.incubate.asp.set_excluded_layers(['fc_1', 'conv2d_0']) ref = [ - False, False, False, False, True, False, True, False, False, False, - True, False + False, + False, + False, + False, + True, + False, + True, + False, + False, + False, + True, + False, ] for i, name in enumerate(names): self.assertTrue( - ref[i] == ASPHelper._is_supported_layer(program, name)) + ref[i] == ASPHelper._is_supported_layer(program, name) + ) paddle.incubate.asp.reset_excluded_layers() ref = [ - False, False, True, False, True, False, True, False, True, False, - True, False + False, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, ] for i, name in enumerate(names): self.assertTrue( - ref[i] == ASPHelper._is_supported_layer(program, name)) + ref[i] == ASPHelper._is_supported_layer(program, name) + ) def test_decorate(self): param_names = [param.name for param in self.layer.parameters()] @@ -96,7 +136,8 @@ def test_decorate(self): for name in param_names: mask_var = ASPHelper._get_program_asp_info(program).mask_vars.get( - name, None) + name, None + ) if ASPHelper._is_supported_layer(program, name): self.assertTrue(mask_var is not None) else: @@ -107,14 +148,18 @@ def test_asp_training(self): paddle.incubate.asp.prune_model(self.layer) - imgs = paddle.to_tensor(np.random.randn(32, 3, 24, 24), - dtype='float32', - place=self.place, - stop_gradient=False) - labels = paddle.to_tensor(np.random.randint(10, size=(32, 1)), - dtype='float32', - place=self.place, - stop_gradient=False) + imgs = paddle.to_tensor( + np.random.randn(32, 3, 24, 24), + dtype='float32', + place=self.place, + stop_gradient=False, + ) + labels = paddle.to_tensor( + np.random.randint(10, size=(32, 1)), + dtype='float32', + place=self.place, + stop_gradient=False, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') @@ -126,34 +171,41 @@ def test_asp_training(self): for param in self.layer.parameters(): if ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): mat = param.numpy() - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) def test_asp_training_with_amp(self): self.optimizer = paddle.incubate.asp.decorate(self.optimizer) paddle.incubate.asp.prune_model(self.layer) - imgs = paddle.to_tensor(np.random.randn(32, 3, 24, 24), - dtype='float32', - place=self.place, - stop_gradient=False) - labels = paddle.to_tensor(np.random.randint(10, size=(32, 1)), - dtype='float32', - place=self.place, - stop_gradient=False) + imgs = paddle.to_tensor( + np.random.randn(32, 3, 24, 24), + dtype='float32', + place=self.place, + stop_gradient=False, + ) + labels = paddle.to_tensor( + np.random.randint(10, size=(32, 1)), + dtype='float32', + place=self.place, + stop_gradient=False, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') scaler = paddle.amp.GradScaler(init_loss_scaling=1024) @@ -168,21 +220,24 @@ def test_asp_training_with_amp(self): for param in self.layer.parameters(): if ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): mat = param.numpy() - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_static.py b/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_static.py index b689a40f88fa7..b64763ebe7c93 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_static.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_optimize_static.py @@ -24,21 +24,18 @@ class TestASPStaticOptimize(unittest.TestCase): - def setUp(self): self.main_program = fluid.Program() self.startup_program = fluid.Program() def build_model(): - img = fluid.data(name='img', - shape=[None, 3, 24, 24], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 3, 24, 24], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') - hidden = fluid.layers.conv2d(input=img, - num_filters=4, - filter_size=3, - padding=2, - act="relu") + hidden = fluid.layers.conv2d( + input=img, num_filters=4, filter_size=3, padding=2, act="relu" + ) hidden = fluid.layers.fc(input=hidden, size=32, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') return img, label, prediction @@ -46,11 +43,11 @@ def build_model(): with fluid.program_guard(self.main_program, self.startup_program): self.img, self.label, predict = build_model() self.loss = paddle.mean( - fluid.layers.cross_entropy(input=predict, label=self.label)) + fluid.layers.cross_entropy(input=predict, label=self.label) + ) self.optimizer = fluid.optimizer.SGD(learning_rate=0.01) def test_get_not_ASP_relevant_vars(self): - def check_params(params, params_from_asp): if len(params_from_asp) != len(params): return False @@ -62,61 +59,112 @@ def check_params(params, params_from_asp): params = self.main_program.global_block().all_parameters() params_from_asp = ASPHelper._get_not_ASP_relevant_vars( - self.main_program) + self.main_program + ) self.assertTrue(check_params(params, params_from_asp)) with fluid.program_guard(self.main_program, self.startup_program): - ASPHelper._minimize(self.optimizer, self.loss, self.main_program, - self.startup_program) + ASPHelper._minimize( + self.optimizer, + self.loss, + self.main_program, + self.startup_program, + ) params_from_asp_after_opt = ASPHelper._get_not_ASP_relevant_vars( - self.main_program) + self.main_program + ) self.assertTrue(check_params(params, params_from_asp_after_opt)) def test_is_supported_layers(self): program = paddle.static.default_main_program() names = [ - 'embedding_0.w_0', 'fack_layer_0.w_0', 'conv2d_0.w_0', - 'conv2d_0.b_0', 'conv2d_1.w_0', 'conv2d_1.b_0', 'fc_0.w_0', - 'fc_0.b_0', 'fc_1.w_0', 'fc_1.b_0', 'linear_2.w_0', 'linear_2.b_0' + 'embedding_0.w_0', + 'fack_layer_0.w_0', + 'conv2d_0.w_0', + 'conv2d_0.b_0', + 'conv2d_1.w_0', + 'conv2d_1.b_0', + 'fc_0.w_0', + 'fc_0.b_0', + 'fc_1.w_0', + 'fc_1.b_0', + 'linear_2.w_0', + 'linear_2.b_0', ] ref = [ - False, False, True, False, True, False, True, False, True, False, - True, False + False, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, ] for i, name in enumerate(names): self.assertTrue( - ref[i] == ASPHelper._is_supported_layer(program, name)) + ref[i] == ASPHelper._is_supported_layer(program, name) + ) paddle.incubate.asp.set_excluded_layers(['fc_1', 'conv2d_0'], program) ref = [ - False, False, False, False, True, False, True, False, False, False, - True, False + False, + False, + False, + False, + True, + False, + True, + False, + False, + False, + True, + False, ] for i, name in enumerate(names): self.assertTrue( - ref[i] == ASPHelper._is_supported_layer(program, name)) + ref[i] == ASPHelper._is_supported_layer(program, name) + ) paddle.incubate.asp.reset_excluded_layers(program) ref = [ - False, False, True, False, True, False, True, False, True, False, - True, False + False, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, ] for i, name in enumerate(names): self.assertTrue( - ref[i] == ASPHelper._is_supported_layer(program, name)) + ref[i] == ASPHelper._is_supported_layer(program, name) + ) def test_decorate(self): param_names = self.__get_param_names( - self.main_program.global_block().all_parameters()) + self.main_program.global_block().all_parameters() + ) with fluid.program_guard(self.main_program, self.startup_program): self.optimizer = paddle.incubate.asp.decorate(self.optimizer) self.optimizer.minimize(self.loss, self.startup_program) param_names_after_minimize = self.__get_param_names( - self.main_program.global_block().all_parameters()) + self.main_program.global_block().all_parameters() + ) - self.__check_mask_variables_and_ops(param_names, - param_names_after_minimize) + self.__check_mask_variables_and_ops( + param_names, param_names_after_minimize + ) def test_asp_training(self): with fluid.program_guard(self.main_program, self.startup_program): @@ -132,63 +180,77 @@ def test_asp_training(self): exe.run(self.startup_program) paddle.incubate.asp.prune_model(self.main_program) - data = (np.random.randn(32, 3, 24, - 24), np.random.randint(10, size=(32, 1))) + data = ( + np.random.randn(32, 3, 24, 24), + np.random.randint(10, size=(32, 1)), + ) exe.run(self.main_program, feed=feeder.feed([data])) for param in self.main_program.global_block().all_parameters(): if ASPHelper._is_supported_layer(self.main_program, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) def test_asp_training_with_amp(self): if core.is_compiled_with_cuda(): place = paddle.CUDAPlace(0) with fluid.program_guard(self.main_program, self.startup_program): - self.optimizer = fluid.contrib.mixed_precision.decorator.decorate( - self.optimizer) + self.optimizer = ( + fluid.contrib.mixed_precision.decorator.decorate( + self.optimizer + ) + ) self.optimizer = paddle.incubate.asp.decorate(self.optimizer) self.optimizer.minimize(self.loss, self.startup_program) exe = fluid.Executor(place) - feeder = fluid.DataFeeder(feed_list=[self.img, self.label], - place=place) + feeder = fluid.DataFeeder( + feed_list=[self.img, self.label], place=place + ) exe.run(self.startup_program) paddle.incubate.asp.prune_model(self.main_program) - data = (np.random.randn(32, 3, 24, - 24), np.random.randint(10, size=(32, 1))) + data = ( + np.random.randn(32, 3, 24, 24), + np.random.randint(10, size=(32, 1)), + ) exe.run(self.main_program, feed=feeder.feed([data])) for param in self.main_program.global_block().all_parameters(): if ASPHelper._is_supported_layer(self.main_program, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) def __get_param_names(self, params): param_names = [] @@ -196,11 +258,15 @@ def __get_param_names(self, params): param_names.append(p.name) return param_names - def __check_mask_variables_and_ops(self, param_names, - param_names_after_minimize): + def __check_mask_variables_and_ops( + self, param_names, param_names_after_minimize + ): for n in param_names: - self.assertFalse(ASPHelper._is_supported_layer(self.main_program, n) and \ - ASPHelper._get_mask_name(n) not in param_names_after_minimize) + self.assertFalse( + ASPHelper._is_supported_layer(self.main_program, n) + and ASPHelper._get_mask_name(n) + not in param_names_after_minimize + ) mask_names = [] for n in param_names: @@ -209,8 +275,7 @@ def __check_mask_variables_and_ops(self, param_names, masking_ops = [] for op in self.main_program.global_block().ops: - if op.type == 'elementwise_mul' and \ - op.input('Y')[0] in mask_names: + if op.type == 'elementwise_mul' and op.input('Y')[0] in mask_names: masking_ops.append(op.input('Y')[0]) self.assertTrue(len(masking_ops) == len(mask_names)) diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_dynamic.py b/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_dynamic.py index 74f27d74b210c..ff86ba9e78222 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_dynamic.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_dynamic.py @@ -22,13 +22,11 @@ class MyLayer(paddle.nn.Layer): - def __init__(self): super(MyLayer, self).__init__() - self.conv1 = paddle.nn.Conv2D(in_channels=3, - out_channels=2, - kernel_size=3, - padding=2) + self.conv1 = paddle.nn.Conv2D( + in_channels=3, out_channels=2, kernel_size=3, padding=2 + ) self.linear1 = paddle.nn.Linear(1352, 32) self.linear2 = paddle.nn.Linear(32, 10) @@ -41,7 +39,6 @@ def forward(self, img): class TestASPDynamicPruningBase(unittest.TestCase): - def setUp(self): self.layer = MyLayer() @@ -49,72 +46,82 @@ def setUp(self): if core.is_compiled_with_cuda(): place = paddle.CUDAPlace(0) - self.img = paddle.to_tensor(np.random.uniform(low=-0.5, - high=0.5, - size=(32, 3, 24, 24)), - dtype=np.float32, - place=place, - stop_gradient=False) + self.img = paddle.to_tensor( + np.random.uniform(low=-0.5, high=0.5, size=(32, 3, 24, 24)), + dtype=np.float32, + place=place, + stop_gradient=False, + ) self.set_config() def set_config(self): self.mask_gen_func = 'mask_1d' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + ) def test_inference_pruning(self): self.__pruning_and_checking(False) def test_training_pruning(self): - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=self.layer.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=self.layer.parameters() + ) optimizer = paddle.incubate.asp.decorate(optimizer) self.__pruning_and_checking(True) def __pruning_and_checking(self, with_mask): - paddle.incubate.asp.prune_model(self.layer, - mask_algo=self.mask_gen_func, - with_mask=with_mask) + paddle.incubate.asp.prune_model( + self.layer, mask_algo=self.mask_gen_func, with_mask=with_mask + ) for param in self.layer.parameters(): if ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): mat = param.numpy() - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( paddle.fluid.contrib.sparsity.check_sparsity( - mat.T, func_name=self.mask_check_func, n=2, m=4)) + mat.T, func_name=self.mask_check_func, n=2, m=4 + ) + ) class TestASPDynamicPruning1D(TestASPDynamicPruningBase): - def set_config(self): self.mask_gen_func = 'mask_1d' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + ) class TestASPDynamicPruning2DBest(TestASPDynamicPruningBase): - def set_config(self): self.mask_gen_func = 'mask_2d_best' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + ) class TestASPDynamicPruning2DGreedy(TestASPDynamicPruningBase): - def set_config(self): self.mask_gen_func = 'mask_2d_greedy' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_static.py b/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_static.py index 084fbeef2d3dd..f312797f43a87 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_static.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_pruning_static.py @@ -24,21 +24,18 @@ class TestASPStaticPruningBase(unittest.TestCase): - def setUp(self): self.main_program = fluid.Program() self.startup_program = fluid.Program() def build_model(): - img = fluid.data(name='img', - shape=[None, 3, 24, 24], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 3, 24, 24], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') - hidden = fluid.layers.conv2d(input=img, - num_filters=2, - filter_size=3, - padding=2, - act="relu") + hidden = fluid.layers.conv2d( + input=img, num_filters=2, filter_size=3, padding=2, act="relu" + ) hidden = fluid.layers.fc(input=hidden, size=32, act='softmax') hidden = fluid.layers.fc(input=hidden, size=3, act='softmax') prediction = fluid.layers.fc(input=hidden, size=3, act='softmax') @@ -51,7 +48,9 @@ def build_model(): def set_config(self): self.mask_gen_func = 'mask_1d' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + ) def test_inference_pruning(self): place = paddle.CPUPlace() @@ -64,10 +63,11 @@ def test_inference_pruning(self): def test_training_pruning(self): with fluid.program_guard(self.main_program, self.startup_program): loss = paddle.mean( - fluid.layers.cross_entropy(input=self.predict, - label=self.label)) + fluid.layers.cross_entropy(input=self.predict, label=self.label) + ) optimizer = paddle.incubate.asp.decorate( - fluid.optimizer.SGD(learning_rate=0.01)) + fluid.optimizer.SGD(learning_rate=0.01) + ) optimizer.minimize(loss, self.startup_program) place = paddle.CPUPlace() @@ -79,45 +79,52 @@ def test_training_pruning(self): def __pruning_and_checking(self, exe, place, with_mask): exe.run(self.startup_program) - paddle.incubate.asp.prune_model(self.main_program, - mask_algo=self.mask_gen_func, - with_mask=with_mask) + paddle.incubate.asp.prune_model( + self.main_program, mask_algo=self.mask_gen_func, with_mask=with_mask + ) for param in self.main_program.global_block().all_parameters(): if ASPHelper._is_supported_layer(self.main_program, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( paddle.fluid.contrib.sparsity.check_sparsity( - mat.T, func_name=self.mask_check_func, n=2, m=4)) + mat.T, func_name=self.mask_check_func, n=2, m=4 + ) + ) class TestASPStaticPruning1D(TestASPStaticPruningBase): - def set_config(self): self.mask_gen_func = 'mask_1d' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D + ) class TestASPStaticPruning2DBest(TestASPStaticPruningBase): - def set_config(self): self.mask_gen_func = 'mask_2d_best' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + ) class TestASPStaticPruning2DGreedy(TestASPStaticPruningBase): - def set_config(self): self.mask_gen_func = 'mask_2d_greedy' - self.mask_check_func = paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + self.mask_check_func = ( + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_save_load.py b/python/paddle/fluid/tests/unittests/asp/test_asp_save_load.py index c059703f0cc03..4347371c311d2 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_save_load.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_save_load.py @@ -22,13 +22,11 @@ class MyLayer(paddle.nn.Layer): - def __init__(self): super(MyLayer, self).__init__() - self.conv1 = paddle.nn.Conv2D(in_channels=3, - out_channels=4, - kernel_size=3, - padding=2) + self.conv1 = paddle.nn.Conv2D( + in_channels=3, out_channels=4, kernel_size=3, padding=2 + ) self.linear1 = paddle.nn.Linear(4624, 32) self.linear2 = paddle.nn.Linear(32, 32) self.linear3 = paddle.nn.Linear(32, 10) @@ -43,7 +41,6 @@ def forward(self, img): class TestASPDynamicOptimize(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -54,7 +51,8 @@ def setUp(self): self.place = paddle.CUDAPlace(0) self.optimizer = paddle.optimizer.SGD( - learning_rate=0.01, parameters=self.layer.parameters()) + learning_rate=0.01, parameters=self.layer.parameters() + ) self.optimizer = paddle.incubate.asp.decorate(self.optimizer) paddle.incubate.asp.prune_model(self.layer) @@ -67,11 +65,13 @@ def test_save_and_load(self): paddle.save(self.optimizer.state_dict(), opt_path) asp_info = ASPHelper._get_program_asp_info( - paddle.static.default_main_program()) + paddle.static.default_main_program() + ) for param_name in asp_info.mask_vars: mask = asp_info.mask_vars[param_name] asp_info.update_mask_vars( - param_name, paddle.ones(shape=mask.shape, dtype=mask.dtype)) + param_name, paddle.ones(shape=mask.shape, dtype=mask.dtype) + ) asp_info.update_masks(param_name, np.ones(shape=mask.shape)) net_state_dict = paddle.load(net_path) @@ -80,14 +80,18 @@ def test_save_and_load(self): self.layer.set_state_dict(net_state_dict) self.optimizer.set_state_dict(opt_state_dict) - imgs = paddle.to_tensor(np.random.randn(64, 3, 32, 32), - dtype='float32', - place=self.place, - stop_gradient=False) - labels = paddle.to_tensor(np.random.randint(10, size=(64, 1)), - dtype='float32', - place=self.place, - stop_gradient=False) + imgs = paddle.to_tensor( + np.random.randn(64, 3, 32, 32), + dtype='float32', + place=self.place, + stop_gradient=False, + ) + labels = paddle.to_tensor( + np.random.randint(10, size=(64, 1)), + dtype='float32', + place=self.place, + stop_gradient=False, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') @@ -99,24 +103,26 @@ def test_save_and_load(self): for param in self.layer.parameters(): if ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): mat = param.numpy() - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) class TestASPStaticOptimize(unittest.TestCase): - def setUp(self): paddle.enable_static() @@ -124,15 +130,13 @@ def setUp(self): self.startup_program = fluid.Program() def build_model(): - img = fluid.data(name='img', - shape=[None, 3, 32, 32], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 3, 32, 32], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') - hidden = fluid.layers.conv2d(input=img, - num_filters=4, - filter_size=3, - padding=2, - act="relu") + hidden = fluid.layers.conv2d( + input=img, num_filters=4, filter_size=3, padding=2, act="relu" + ) hidden = fluid.layers.fc(input=hidden, size=32, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') return img, label, prediction @@ -140,7 +144,8 @@ def build_model(): with fluid.program_guard(self.main_program, self.startup_program): self.img, self.label, predict = build_model() self.loss = paddle.mean( - fluid.layers.cross_entropy(input=predict, label=self.label)) + fluid.layers.cross_entropy(input=predict, label=self.label) + ) self.optimizer = fluid.optimizer.SGD(learning_rate=0.01) self.optimizer = paddle.incubate.asp.decorate(self.optimizer) self.optimizer.minimize(self.loss, self.startup_program) @@ -166,29 +171,35 @@ def test_save_and_load(self): state_dict = paddle.load(param_path) prog.set_state_dict(state_dict) - feeder = fluid.DataFeeder(feed_list=[self.img, self.label], - place=self.place) + feeder = fluid.DataFeeder( + feed_list=[self.img, self.label], place=self.place + ) - data = (np.random.randn(64, 3, 32, - 32), np.random.randint(10, size=(64, 1))) + data = ( + np.random.randn(64, 3, 32, 32), + np.random.randint(10, size=(64, 1)), + ) self.exe.run(prog, feed=feeder.feed([data])) for param in prog.global_block().all_parameters(): if ASPHelper._is_supported_layer(prog, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/asp/test_asp_utils.py b/python/paddle/fluid/tests/unittests/asp/test_asp_utils.py index a94520662ef05..a55ba05a75d0e 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_asp_utils.py +++ b/python/paddle/fluid/tests/unittests/asp/test_asp_utils.py @@ -20,33 +20,50 @@ class TestASPUtils(unittest.TestCase): - def test_get_check_method(self): self.assertEqual( paddle.fluid.contrib.sparsity.CheckMethod.get_checking_method( - paddle.fluid.contrib.sparsity.MaskAlgo.MASK_1D), - paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D) + paddle.fluid.contrib.sparsity.MaskAlgo.MASK_1D + ), + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D, + ) self.assertEqual( paddle.fluid.contrib.sparsity.CheckMethod.get_checking_method( - paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_GREEDY), - paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D) + paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_GREEDY + ), + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D, + ) self.assertEqual( paddle.fluid.contrib.sparsity.CheckMethod.get_checking_method( - paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_BEST), - paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D) + paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_BEST + ), + paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D, + ) def test_density(self): - x = np.array([[1.0, 1.0, 1.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0], - [1.0, 0.0, 0.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0], - [0.0, 1.0, 0.0, 0.0, 1.0]]) + x = np.array( + [ + [1.0, 1.0, 1.0, 0.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0], + [1.0, 0.0, 0.0, 0.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0], + [0.0, 1.0, 0.0, 0.0, 1.0], + ] + ) self.assertEqual(paddle.incubate.asp.calculate_density(x), 0.56) x[:, 0] = 0.0 self.assertEqual(paddle.incubate.asp.calculate_density(x), 0.4) def test_check_mask_1d(self): - x = np.array([[1.0, 0.0, 0.0, 1.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0], - [1.0, 1.0, 0.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 1.0], - [0.0, 1.0, 0.0, 0.0, 1.0]]) + x = np.array( + [ + [1.0, 0.0, 0.0, 1.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0], + [0.0, 1.0, 0.0, 0.0, 1.0], + ] + ) self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_1d(x, 2, 4)) self.assertFalse(paddle.fluid.contrib.sparsity.check_mask_1d(x, 3, 4)) self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_1d(x, 2, 5)) @@ -58,18 +75,26 @@ def test_get_mask_1d(self): for _ in range(10): x = np.random.randint(10, size=(5, 5)) x = paddle.fluid.contrib.sparsity.get_mask_1d(x, 2, 4) - self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_1d( - x, 2, 4)) + self.assertTrue( + paddle.fluid.contrib.sparsity.check_mask_1d(x, 2, 4) + ) x = np.random.randn(5, 4) x = paddle.fluid.contrib.sparsity.get_mask_1d(x, 2, 4) - self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_1d( - x, 2, 4)) + self.assertTrue( + paddle.fluid.contrib.sparsity.check_mask_1d(x, 2, 4) + ) def test_check_mask_2d(self): - x = np.array([[1.0, 0.0, 0.0, 1.0, 1.0], [0.0, 1.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 1.0, 0.0, 1.0], [1.0, 1.0, 0.0, 0.0, 0.0], - [0.0, 1.0, 0.0, 0.0, 1.0]]) + x = np.array( + [ + [1.0, 0.0, 0.0, 1.0, 1.0], + [0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0, 1.0], + ] + ) self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 4)) self.assertFalse(paddle.fluid.contrib.sparsity.check_mask_2d(x, 3, 4)) self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 5)) @@ -81,28 +106,31 @@ def test_get_mask_2d_greedy(self): for _ in range(10): x = np.random.randint(10, size=(5, 5)) x = paddle.fluid.contrib.sparsity.get_mask_2d_greedy(x, 2, 4) - self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_2d( - x, 2, 4)) + self.assertTrue( + paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 4) + ) x = np.random.randn(5, 4) x = paddle.fluid.contrib.sparsity.get_mask_2d_greedy(x, 2, 4) - self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_2d( - x, 2, 4)) + self.assertTrue( + paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 4) + ) def test_get_mask_2d_best(self): for _ in range(10): x = np.random.randint(10, size=(5, 5)) x = paddle.fluid.contrib.sparsity.get_mask_2d_best(x, 2, 4) - self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_2d( - x, 2, 4)) + self.assertTrue( + paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 4) + ) x = np.random.randn(5, 4) x = paddle.fluid.contrib.sparsity.get_mask_2d_best(x, 2, 4) - self.assertTrue(paddle.fluid.contrib.sparsity.check_mask_2d( - x, 2, 4)) + self.assertTrue( + paddle.fluid.contrib.sparsity.check_mask_2d(x, 2, 4) + ) def test_threadsafe_valid_2d_patterns(self): - def get_reference(m=4, n=2): from itertools import permutations @@ -112,17 +140,20 @@ def get_reference(m=4, n=2): patterns = patterns + patterns patterns = np.asarray(list(set(permutations(patterns, m)))) - valid = ((patterns.sum(axis=1) <= n).sum( - axis=1) == m).nonzero()[0].reshape(-1) + valid = ( + ((patterns.sum(axis=1) <= n).sum(axis=1) == m) + .nonzero()[0] + .reshape(-1) + ) valid_patterns = np.empty((valid.shape[0], m, m)) valid_patterns[:] = patterns[valid[:]] return valid_patterns for _ in range(4): computing_thread = threading.Thread( - target=paddle.fluid.contrib.sparsity.utils. - _compute_valid_2d_patterns, - args=(2, 4)) + target=paddle.fluid.contrib.sparsity.utils._compute_valid_2d_patterns, + args=(2, 4), + ) computing_thread.start() time.sleep(3) patterns_map = paddle.fluid.contrib.sparsity.utils._valid_2d_patterns @@ -132,7 +163,8 @@ def get_reference(m=4, n=2): self.assertTrue(reference_key in patterns_map) self.assertTrue(len(patterns_map) == 1) self.assertTrue( - (reference_patterns == patterns_map[reference_key]).all()) + (reference_patterns == patterns_map[reference_key]).all() + ) def test_check_sparsity(self): for _ in range(10): @@ -173,49 +205,64 @@ def __test_1D_2D_sparsity_checking_methods(self, x_2d): mask, func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D, n=2, - m=4), paddle.fluid.contrib.sparsity.check_mask_1d(mask, 2, 4)) + m=4, + ), + paddle.fluid.contrib.sparsity.check_mask_1d(mask, 2, 4), + ) mask = paddle.fluid.contrib.sparsity.get_mask_2d_best(x_2d, 2, 4) self.assertEqual( paddle.fluid.contrib.sparsity.check_sparsity( mask, func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D, n=2, - m=4), paddle.fluid.contrib.sparsity.check_mask_2d(mask, 2, 4)) + m=4, + ), + paddle.fluid.contrib.sparsity.check_mask_2d(mask, 2, 4), + ) def __test_1D_2D_sparse_mask_generation_methods(self, x): mask = paddle.fluid.contrib.sparsity.create_mask( x, func_name=paddle.fluid.contrib.sparsity.MaskAlgo.MASK_1D, n=2, - m=4) + m=4, + ) self.assertTrue( paddle.fluid.contrib.sparsity.check_sparsity( mask, func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_1D, n=2, - m=4)) + m=4, + ) + ) mask = paddle.fluid.contrib.sparsity.create_mask( x, func_name=paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_GREEDY, n=2, - m=4) + m=4, + ) self.assertTrue( paddle.fluid.contrib.sparsity.check_sparsity( mask, func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D, n=2, - m=4)) + m=4, + ) + ) mask = paddle.fluid.contrib.sparsity.create_mask( x, func_name=paddle.fluid.contrib.sparsity.MaskAlgo.MASK_2D_BEST, n=2, - m=4) + m=4, + ) self.assertTrue( paddle.fluid.contrib.sparsity.check_sparsity( mask, func_name=paddle.fluid.contrib.sparsity.CheckMethod.CHECK_2D, n=2, - m=4)) + m=4, + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_dynamic.py b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_dynamic.py index d132b319f569a..a2116283a333d 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_dynamic.py +++ b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_dynamic.py @@ -29,7 +29,6 @@ class MyLayer(paddle.nn.Layer): - def __init__(self): super(MyLayer, self).__init__() self.linear1 = paddle.nn.Linear(32, 32) @@ -42,7 +41,6 @@ def forward(self, x): class TestFleetWithASPDynamic(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213" os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213" @@ -56,7 +54,8 @@ def setUp(self): self.place = paddle.CUDAPlace(0) self.optimizer = paddle.optimizer.SGD( - learning_rate=0.01, parameters=self.layer.parameters()) + learning_rate=0.01, parameters=self.layer.parameters() + ) def test_with_asp(self): fleet.init(is_collective=True) @@ -67,14 +66,18 @@ def test_with_asp(self): self.optimizer = fleet.distributed_optimizer(self.optimizer) self.layer = fleet.distributed_model(self.layer) - imgs = paddle.to_tensor(np.random.randn(64, 32), - dtype='float32', - place=self.place, - stop_gradient=False) - labels = paddle.to_tensor(np.random.randint(10, size=(64, 1)), - dtype='float32', - place=self.place, - stop_gradient=False) + imgs = paddle.to_tensor( + np.random.randn(64, 32), + dtype='float32', + place=self.place, + stop_gradient=False, + ) + labels = paddle.to_tensor( + np.random.randint(10, size=(64, 1)), + dtype='float32', + place=self.place, + stop_gradient=False, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') @@ -86,24 +89,26 @@ def test_with_asp(self): for param in self.layer.parameters(): if ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): mat = param.numpy() - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) class TestFleetWithASPAMPDynamic(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213" os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213" @@ -117,7 +122,8 @@ def setUp(self): self.place = paddle.CUDAPlace(0) self.optimizer = paddle.optimizer.SGD( - learning_rate=0.01, parameters=self.layer.parameters()) + learning_rate=0.01, parameters=self.layer.parameters() + ) def test_with_asp(self): fleet.init(is_collective=True) @@ -128,14 +134,18 @@ def test_with_asp(self): self.optimizer = fleet.distributed_optimizer(self.optimizer) self.layer = fleet.distributed_model(self.layer) - imgs = paddle.to_tensor(np.random.randn(64, 32), - dtype='float32', - place=self.place, - stop_gradient=False) - labels = paddle.to_tensor(np.random.randint(10, size=(64, 1)), - dtype='float32', - place=self.place, - stop_gradient=False) + imgs = paddle.to_tensor( + np.random.randn(64, 32), + dtype='float32', + place=self.place, + stop_gradient=False, + ) + labels = paddle.to_tensor( + np.random.randint(10, size=(64, 1)), + dtype='float32', + place=self.place, + stop_gradient=False, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') scaler = paddle.amp.GradScaler(init_loss_scaling=1024) @@ -150,20 +160,23 @@ def test_with_asp(self): for param in self.layer.parameters(): if ASPHelper._is_supported_layer( - paddle.static.default_main_program(), param.name): + paddle.static.default_main_program(), param.name + ): mat = param.numpy() - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_sharding.py b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_sharding.py index e5ff5f839e7be..9e74e10fbc505 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_sharding.py +++ b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_sharding.py @@ -32,7 +32,6 @@ class TestFleetWithASPSharding(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213" os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213" @@ -49,9 +48,9 @@ def setUp(self): def net(self, main_prog, startup_prog): with fluid.program_guard(main_prog, startup_prog): - input_x = paddle.static.data(name="x", - shape=[-1, 32], - dtype='float32') + input_x = paddle.static.data( + name="x", shape=[-1, 32], dtype='float32' + ) input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64') fc_1 = fluid.layers.fc(input=input_x, size=64, act='tanh') @@ -71,7 +70,7 @@ def net(self, main_prog, startup_prog): "sharding_degree": 8, "mp_degree": 1, "hybrid_dp": False, - "gradient_merge_acc_step": 1 + "gradient_merge_acc_step": 1, } dist_strategy.nccl_comm_num = 1 dist_strategy.asp = True @@ -80,18 +79,21 @@ def net(self, main_prog, startup_prog): def test_with_asp_sharding(self): fleet.init(is_collective=True) train_prog, startup_prog = fluid.Program(), fluid.Program() - avg_cost, strategy, input_x, input_y = self.net(train_prog, - startup_prog) + avg_cost, strategy, input_x, input_y = self.net( + train_prog, startup_prog + ) with fluid.program_guard(train_prog, startup_prog): optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) if paddle.fluid.is_compiled_with_cuda(): place = fluid.CUDAPlace( - int(os.environ.get('FLAGS_selected_gpus', 0))) + int(os.environ.get('FLAGS_selected_gpus', 0)) + ) else: place = fluid.CPUPlace() @@ -106,20 +108,23 @@ def test_with_asp_sharding(self): for param in train_prog.global_block().all_parameters(): if ASPHelper._is_supported_layer(train_prog, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_static.py b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_static.py index f205610179108..aa4b0208847aa 100644 --- a/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_static.py +++ b/python/paddle/fluid/tests/unittests/asp/test_fleet_with_asp_static.py @@ -32,7 +32,6 @@ class TestFleetWithASPStatic(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213" os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213" @@ -41,9 +40,9 @@ def setUp(self): def net(self, main_prog, startup_prog): with fluid.program_guard(main_prog, startup_prog): - input_x = paddle.static.data(name="x", - shape=[-1, 32], - dtype='float32') + input_x = paddle.static.data( + name="x", shape=[-1, 32], dtype='float32' + ) input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64') fc_1 = fluid.layers.fc(input=input_x, size=64, act='tanh') @@ -58,17 +57,22 @@ def net(self, main_prog, startup_prog): def test_with_asp(self): fleet.init(is_collective=True) train_prog, startup_prog = fluid.Program(), fluid.Program() - avg_cost, strategy, input_x, input_y = self.net(train_prog, - startup_prog) + avg_cost, strategy, input_x, input_y = self.net( + train_prog, startup_prog + ) with fluid.program_guard(train_prog, startup_prog): optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) - place = fluid.CUDAPlace( - 0) if paddle.fluid.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if paddle.fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[input_x, input_y], place=place) @@ -81,24 +85,26 @@ def test_with_asp(self): for param in train_prog.global_block().all_parameters(): if ASPHelper._is_supported_layer(train_prog, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) class TestFleetWithASPAMPStatic(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213" os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213" @@ -107,9 +113,9 @@ def setUp(self): def net(self, main_prog, startup_prog): with fluid.program_guard(main_prog, startup_prog): - input_x = paddle.static.data(name="x", - shape=[-1, 32], - dtype='float32') + input_x = paddle.static.data( + name="x", shape=[-1, 32], dtype='float32' + ) input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64') fc_1 = fluid.layers.fc(input=input_x, size=64, act='tanh') @@ -124,18 +130,23 @@ def net(self, main_prog, startup_prog): def test_with_asp_and_amp(self): fleet.init(is_collective=True) train_prog, startup_prog = fluid.Program(), fluid.Program() - avg_cost, strategy, input_x, input_y = self.net(train_prog, - startup_prog) + avg_cost, strategy, input_x, input_y = self.net( + train_prog, startup_prog + ) strategy.amp = True with fluid.program_guard(train_prog, startup_prog): optimizer = paddle.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) - place = fluid.CUDAPlace( - 0) if paddle.fluid.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if paddle.fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[input_x, input_y], place=place) @@ -150,41 +161,49 @@ def test_with_asp_and_amp(self): for param in train_prog.global_block().all_parameters(): if ASPHelper._is_supported_layer(train_prog, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) def test_with_asp_and_pure_fp16(self): fleet.init(is_collective=True) train_prog, startup_prog = fluid.Program(), fluid.Program() with paddle.static.amp.fp16_guard(): - avg_cost, strategy, \ - input_x, input_y = self.net(train_prog, - startup_prog) + avg_cost, strategy, input_x, input_y = self.net( + train_prog, startup_prog + ) strategy.amp = True strategy.amp_configs = {'use_pure_fp16': True} with fluid.program_guard(train_prog, startup_prog): with paddle.static.amp.fp16_guard(): optimizer = optimizer = paddle.optimizer.Momentum( - learning_rate=0.01, multi_precision=True) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + learning_rate=0.01, multi_precision=True + ) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) - place = fluid.CUDAPlace( - 0) if paddle.fluid.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if paddle.fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[input_x, input_y], place=place) @@ -199,20 +218,23 @@ def test_with_asp_and_pure_fp16(self): for param in train_prog.global_block().all_parameters(): if ASPHelper._is_supported_layer(train_prog, param.name): - mat = np.array(fluid.global_scope().find_var( - param.name).get_tensor()) - if (len(param.shape) == 4 - and param.shape[1] < 4) or (len(param.shape) == 2 - and param.shape[0] < 4): + mat = np.array( + fluid.global_scope().find_var(param.name).get_tensor() + ) + if (len(param.shape) == 4 and param.shape[1] < 4) or ( + len(param.shape) == 2 and param.shape[0] < 4 + ): self.assertFalse( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) else: self.assertTrue( - paddle.fluid.contrib.sparsity.check_sparsity(mat.T, - n=2, - m=4)) + paddle.fluid.contrib.sparsity.check_sparsity( + mat.T, n=2, m=4 + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/auto_checkpoint_utils.py b/python/paddle/fluid/tests/unittests/auto_checkpoint_utils.py index 4303c56675b21..a9eba03ae5687 100644 --- a/python/paddle/fluid/tests/unittests/auto_checkpoint_utils.py +++ b/python/paddle/fluid/tests/unittests/auto_checkpoint_utils.py @@ -25,7 +25,7 @@ BATCH_NUM = 4 BATCH_SIZE = 1 -#IMAGE_SIZE = 128 +# IMAGE_SIZE = 128 CLASS_NUM = 2 USE_GPU = False # whether use GPU to run model @@ -47,7 +47,6 @@ def get_random_images_and_labels(image_shape, label_shape): def sample_list_generator_creator(): - def __reader__(): for _ in range(BATCH_NUM): sample_list = [] @@ -61,21 +60,17 @@ def __reader__(): class AutoCheckpointBase(unittest.TestCase): - - def _init_env(self, - exe, - main_prog, - startup_prog, - minimize=True, - iterable=True): - + def _init_env( + self, exe, main_prog, startup_prog, minimize=True, iterable=True + ): def simple_net(): image = fluid.data(name='image', shape=[-1, 4, 4], dtype='float32') label = fluid.data(name='label', shape=[-1, 1], dtype='int64') fc_tmp = fluid.layers.fc(image, size=CLASS_NUM) cross_entropy = fluid.layers.softmax_with_cross_entropy( - fc_tmp, label) + fc_tmp, label + ) loss = fluid.layers.reduce_mean(cross_entropy) sgd = fluid.optimizer.SGD(learning_rate=1e-3) if minimize: @@ -87,17 +82,20 @@ def simple_net(): if minimize: compiled = fluid.CompiledProgram(main_prog).with_data_parallel( - loss_name=loss.name) + loss_name=loss.name + ) else: compiled = None loader = fluid.io.DataLoader.from_generator( feed_list=[image, label], capacity=64, use_double_buffer=True, - iterable=iterable) + iterable=iterable, + ) - loader.set_sample_list_generator(sample_list_generator_creator(), - places[0]) + loader.set_sample_list_generator( + sample_list_generator_creator(), places[0] + ) if minimize: exe.run(startup_prog) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/amp_pass_unittest.py b/python/paddle/fluid/tests/unittests/auto_parallel/amp_pass_unittest.py index c00a3367a986d..5d41e1cc484c5 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/amp_pass_unittest.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/amp_pass_unittest.py @@ -31,7 +31,9 @@ def apply_pass(use_amp=False, level=None): amp.enable = True amp.custom_white_list = ['softmax', 'layer_norm', 'gelu'] amp.custom_black_list = [ - 'c_softmax_with_cross_entropy', 'elementwise_div', 'reduce_sum' + 'c_softmax_with_cross_entropy', + 'elementwise_div', + 'reduce_sum', ] amp.init_loss_scaling = 32768 amp.use_fp16_guard = False @@ -47,7 +49,6 @@ def reset_prog(): class TestAMPPass(unittest.TestCase): - def setUp(self): self.rtol = 1e-5 self.atol = 1e-8 @@ -82,7 +83,9 @@ def check_results(self, ref_losses, check_losses, rtol=None, atol=None): rtol=rtol or self.rtol, atol=atol or self.atol, err_msg='pass {} has wrong results!, \nu={}\nv={}\ndiff={}'.format( - __class__, ref_losses, check_losses, ref_losses - check_losses)) + __class__, ref_losses, check_losses, ref_losses - check_losses + ), + ) def test_amp_pass(self): # mp2 training diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_model.py b/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_model.py index bccf0963ce103..2c03dab50ea51 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_model.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_model.py @@ -36,39 +36,39 @@ def get_random_inputs_and_labels(input_shape, label_shape): def batch_generator_creator(): - def __reader__(): for _ in range(batch_size): batch_input, batch_label = get_random_inputs_and_labels( [batch_size, sequence_len, hidden_size], - [batch_size, sequence_len, 1]) + [batch_size, sequence_len, 1], + ) yield batch_input, batch_label return __reader__ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.linear2 = nn.Linear(d_model, 1, weight_attr, bias_attr=bias_attr) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") @@ -85,29 +85,34 @@ def forward(self, input): def mlp_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, sequence_len, 1], - dtype='float32') + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + label = static.data( + name="label", shape=[batch_size, sequence_len, 1], dtype='float32' + ) auto.shard_tensor(input, _global_process_mesh, [None, None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) loss = paddle.mean(error_cost) - loader = paddle.io.DataLoader.from_generator(feed_list=[input, label], - capacity=4 * batch_size, - iterable=True) + loader = paddle.io.DataLoader.from_generator( + feed_list=[input, label], capacity=4 * batch_size, iterable=True + ) return loss, train_program, start_program, loader @@ -125,17 +130,24 @@ def train(): train_program = static.Program() start_program = static.Program() loss, train_program, start_program, loader = mlp_pretrain_forward( - train_program, start_program) + train_program, start_program + ) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) optimizer = fleet.distributed_optimizer(optimizer) - _, _, distributed_startup_program, distributed_main_program = optimizer.minimize( - loss, start_program) + ( + _, + _, + distributed_startup_program, + distributed_main_program, + ) = optimizer.minimize(loss, start_program) places = static.cuda_places() loader.set_batch_generator(batch_generator_creator(), places=places) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_gpt_planner.py b/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_gpt_planner.py index 3634f4258a085..06b5a808cf41d 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_gpt_planner.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_gpt_planner.py @@ -21,51 +21,58 @@ sys.path.append("..") import auto_parallel_gpt_model as modeling -from auto_parallel_gpt_model import GPTModel, GPTForPretraining, GPTPretrainingCriterion +from auto_parallel_gpt_model import ( + GPTModel, + GPTForPretraining, + GPTPretrainingCriterion, +) -def get_gpt_model(train_program, start_program, place, batch_size, sequence_len, - vocab_size): +def get_gpt_model( + train_program, start_program, place, batch_size, sequence_len, vocab_size +): modeling.init_global() with static.program_guard(train_program, start_program): - tokens = paddle.static.data(name="tokens", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = paddle.static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + tokens = paddle.static.data( + name="tokens", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = paddle.static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = paddle.static.data( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float32') - labels = paddle.static.data(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = paddle.static.data(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float32') + dtype='float32', + ) + labels = paddle.static.data( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = paddle.static.data( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float32' + ) data_holder = [tokens, position_ids, attention_mask, labels, loss_mask] - gpt = GPTModel(vocab_size=1000, - hidden_size=64, - num_hidden_layers=2, - num_attention_heads=8, - intermediate_size=256, - hidden_act="gelu", - hidden_dropout_prob=0.0, - attention_probs_dropout_prob=0.0, - max_position_embeddings=1024, - type_vocab_size=1, - initializer_range=0.02, - pad_token_id=0, - eos_token_id=7, - bos_token_id=0, - eol_token_id=3) - - model = GPTForPretraining(gpt, - vocab_size=1000, - hidden_size=64, - initializer_range=0.02) + gpt = GPTModel( + vocab_size=1000, + hidden_size=64, + num_hidden_layers=2, + num_attention_heads=8, + intermediate_size=256, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=1024, + type_vocab_size=1, + initializer_range=0.02, + pad_token_id=0, + eos_token_id=7, + bos_token_id=0, + eol_token_id=3, + ) + + model = GPTForPretraining( + gpt, vocab_size=1000, hidden_size=64, initializer_range=0.02 + ) preds = model(tokens, position_ids, attention_mask) criterion = GPTPretrainingCriterion() loss = criterion(preds, labels, loss_mask) @@ -102,17 +109,28 @@ def train(): sequence_len = 512 vocab_size = 1000 train_program, start_program, loss, gen_data = get_gpt_model( - train_program, start_program, place, batch_size, sequence_len, - vocab_size) - - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + train_program, + start_program, + place, + batch_size, + sequence_len, + vocab_size, + ) + + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) optimizer = fleet.distributed_optimizer(optimizer) - _, _, distributed_startup_program, distributed_main_program = optimizer.minimize( - loss, start_program) + ( + _, + _, + distributed_startup_program, + distributed_main_program, + ) = optimizer.minimize(loss, start_program) places = static.cuda_places() exe = paddle.static.Executor(places[0]) @@ -121,25 +139,29 @@ def train(): for step in range(10): tokens, position_ids, attention_mask, labels, loss_mask = gen_data() if loss.name in distributed_main_program.global_block().vars: - loss_print, = exe.run(distributed_main_program, - feed={ - "tokens": tokens, - "position_ids": position_ids, - "attention_mask": attention_mask, - "labels": labels, - "loss_mask": loss_mask - }, - fetch_list=[loss]) + (loss_print,) = exe.run( + distributed_main_program, + feed={ + "tokens": tokens, + "position_ids": position_ids, + "attention_mask": attention_mask, + "labels": labels, + "loss_mask": loss_mask, + }, + fetch_list=[loss], + ) print("step: %s, loss: %f" % (step, loss_print[0])) else: - exe.run(distributed_main_program, - feed={ - "tokens": tokens, - "position_ids": position_ids, - "attention_mask": attention_mask, - "labels": labels, - "loss_mask": loss_mask - }) + exe.run( + distributed_main_program, + feed={ + "tokens": tokens, + "position_ids": position_ids, + "attention_mask": attention_mask, + "labels": labels, + "loss_mask": loss_mask, + }, + ) print("step: %s, loss: %s" % (step, "None")) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_planner.py b/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_planner.py index b40a61ed34cb7..3111611895a90 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_planner.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/auto_parallel_relaunch_with_planner.py @@ -17,12 +17,15 @@ from paddle.distributed import fleet from paddle.distributed.auto_parallel.cost import CostEstimator from paddle.distributed.auto_parallel.cluster import Cluster -from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, +) def train(): from auto_parallel_relaunch_model import mlp_pretrain_forward from auto_parallel_relaunch_model import batch_generator_creator + dist_strategy = fleet.DistributedStrategy() # init parallel optimizer dist_strategy.auto_search = True @@ -30,17 +33,24 @@ def train(): train_program = static.Program() start_program = static.Program() loss, train_program, start_program, loader = mlp_pretrain_forward( - train_program, start_program) + train_program, start_program + ) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) optimizer = fleet.distributed_optimizer(optimizer) - _, _, distributed_startup_program, distributed_main_program = optimizer.minimize( - loss, start_program) + ( + _, + _, + distributed_startup_program, + distributed_main_program, + ) = optimizer.minimize(loss, start_program) # add cost estimator dist_context = get_default_distributed_context() @@ -51,11 +61,18 @@ def train(): dims_mapping = dist_op.dist_attr.get_input_dims_mapping(var_name) if dims_mapping is None: dist_op.dist_attr.set_input_dims_mapping( - var_name, [ - -1 for i in range( - len(train_program.global_block().vars[var_name]. - shape)) - ]) + var_name, + [ + -1 + for i in range( + len( + train_program.global_block() + .vars[var_name] + .shape + ) + ) + ], + ) cluster.gen_default_config_cluster(device_count=2) cost_estimator = CostEstimator(train_program, cluster) global_cost = cost_estimator.estimate(dist_context) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/clip_grad_by_global_norm.py b/python/paddle/fluid/tests/unittests/auto_parallel/clip_grad_by_global_norm.py index ba255fba27912..d25a8bcd4c5f8 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/clip_grad_by_global_norm.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/clip_grad_by_global_norm.py @@ -61,7 +61,6 @@ def reset_prog(): class TestGradientClipByGlobalNorm(unittest.TestCase): - def setUp(self): self.batch_size = 2 self.batch_num = 1 @@ -94,9 +93,10 @@ def check_result(self, dp_params, sharding_params): sharding_p, rtol=1e-05, atol=1e-08, - err_msg= - 'gradient clip by global norm has wrong results!, \nu={}\nv={}\ndiff={}' - .format(dp_p, sharding_p, dp_p - sharding_p)) + err_msg='gradient clip by global norm has wrong results!, \nu={}\nv={}\ndiff={}'.format( + dp_p, sharding_p, dp_p - sharding_p + ), + ) def test_grad_clip(self): # dp2 training @@ -108,7 +108,8 @@ def test_grad_clip(self): sharding_engine = self.get_engine(True) sharding_engine.fit(self.dataset, 3, batch_size=self.batch_size) sharding_param_values = get_parameter_value( - sharding_engine.main_program) + sharding_engine.main_program + ) self.check_result(dp_param_values, sharding_param_values) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/converter.py b/python/paddle/fluid/tests/unittests/auto_parallel/converter.py index 291dae6612c76..5e0506c3785db 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/converter.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/converter.py @@ -28,21 +28,21 @@ def test_convert(): tensor_name: { "process_shape": [2], "process_group": [0, 1], - "dims_mapping": [-1, -1] + "dims_mapping": [-1, -1], } } row_strategy = { tensor_name: { "process_shape": [2], "process_group": [0, 1], - "dims_mapping": [0, -1] + "dims_mapping": [0, -1], } } col_strategy = { tensor_name: { "process_shape": [2], "process_group": [0, 1], - "dims_mapping": [-1, 0] + "dims_mapping": [-1, 0], } } @@ -70,7 +70,7 @@ def test_convert(): new_name: { "process_shape": [2], "process_group": [0, 1], - "dims_mapping": [0, -1] + "dims_mapping": [0, -1], } } converter = Converter(tensor_dict, col_strategy, row_strategy) @@ -84,14 +84,14 @@ def test_convert(): "tensor_2": { "process_shape": [2], "process_group": [0, 1], - "dims_mapping": [-1, -1] + "dims_mapping": [-1, -1], } } row_strategy = { "tensor_2": { "process_shape": [2], "process_group": [0, 1], - "dims_mapping": [0, -1] + "dims_mapping": [0, -1], } } tensor_dict = {"tensor_2": [complete_tensor]} diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/engine_api.py b/python/paddle/fluid/tests/unittests/auto_parallel/engine_api.py index 38287e98a219a..c09edf0442566 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/engine_api.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/engine_api.py @@ -45,7 +45,6 @@ class MyDataset(Dataset): - def __init__(self, num_samples): super(MyDataset, self).__init__() self.num_samples = num_samples @@ -66,38 +65,38 @@ def get_random_inputs_and_labels(image_shape, label_shape): def batch_generator_creator(): - def __reader__(): for _ in range(batch_num): batch_input, batch_label = get_random_inputs_and_labels( - [batch_size, image_size], [batch_size, 1]) + [batch_size, image_size], [batch_size, 1] + ) yield batch_input, batch_label return __reader__ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.linear2 = nn.Linear(d_model, 1, weight_attr, bias_attr=bias_attr) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") @@ -121,16 +120,20 @@ def forward(self, input): def train_high_level(fetch): global is_fetch is_fetch = fetch - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) metric = paddle.metric.Accuracy() strategy = auto.Strategy() @@ -142,11 +145,13 @@ def train_high_level(fetch): train_dataset = MyDataset(batch_num * batch_size) eval_dataset1 = MyDataset(5 * batch_size) - history = engine.fit(train_data=train_dataset, - epochs=2, - batch_size=batch_size, - valid_data=eval_dataset1, - log_freq=1) + history = engine.fit( + train_data=train_dataset, + epochs=2, + batch_size=batch_size, + valid_data=eval_dataset1, + log_freq=1, + ) # eval eval_dataset2 = MyDataset(batch_size) @@ -165,16 +170,20 @@ def train_high_level(fetch): def train_low_level(): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) metric = paddle.metric.Accuracy() strategy = auto.Strategy() @@ -189,18 +198,18 @@ def train_low_level(): # Build normal normal dataloader # train train_dataset = MyDataset(batch_num * batch_size) - train_dataloader = engine.dataloader(train_dataset, - batch_size=batch_size, - mode="train") + train_dataloader = engine.dataloader( + train_dataset, batch_size=batch_size, mode="train" + ) engine.prepare(mode="train") for data in train_dataloader: outs = engine.run(data, feed=feed_dict, mode="train") # eval eval_dataset2 = MyDataset(batch_size) - eval_dataloader = engine.dataloader(eval_dataset2, - batch_size=batch_size, - mode="eval") + eval_dataloader = engine.dataloader( + eval_dataset2, batch_size=batch_size, mode="eval" + ) engine.prepare(mode="eval") for data in eval_dataloader: outs = engine.run(data, feed=feed_dict, mode="eval") @@ -223,9 +232,9 @@ def train_low_level(): # Build dataloader from generator # train train_dataset = MyDataset(batch_num * batch_size) - train_dataloader = engine.dataloader_from_generator(train_dataset, - batch_size=batch_size, - mode="train") + train_dataloader = engine.dataloader_from_generator( + train_dataset, batch_size=batch_size, mode="train" + ) engine.prepare(mode="train") for data in train_dataloader: outs = engine.run(data, feed=feed_dict, mode="train") @@ -233,17 +242,18 @@ def train_low_level(): # eval engine.to_mode("eval") eval_dataset2 = MyDataset(batch_size) - eval_dataloader = engine.dataloader_from_generator(eval_dataset2, - batch_size=batch_size) + eval_dataloader = engine.dataloader_from_generator( + eval_dataset2, batch_size=batch_size + ) engine.prepare() for data in eval_dataloader: outs = engine.run(data, feed=feed_dict) # predict test_dataset = MyDataset(batch_size) - predict_dataloader = engine.dataloader_from_generator(test_dataset, - batch_size=batch_size, - mode="predict") + predict_dataloader = engine.dataloader_from_generator( + test_dataset, batch_size=batch_size, mode="predict" + ) engine.prepare(mode="predict") for data in predict_dataloader: outs = engine.run(data, feed=feed_dict, mode="predict") @@ -257,16 +267,20 @@ def train_low_level(): def train_builtin_data_vars(): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) metric = paddle.metric.Accuracy() strategy = auto.Strategy() @@ -284,9 +298,9 @@ def train_builtin_data_vars(): with static.program_guard(engine.main_program, engine.startup_program): feed_list = engine.inputs + engine.labels print(feed_list) - loader = paddle.io.DataLoader.from_generator(feed_list=feed_list, - capacity=4 * batch_size, - iterable=False) + loader = paddle.io.DataLoader.from_generator( + feed_list=feed_list, capacity=4 * batch_size, iterable=False + ) places = static.cuda_places() loader.set_batch_generator(batch_generator_creator(), places=places) @@ -297,36 +311,40 @@ def train_builtin_data_vars(): while True: engine.run() except paddle.fluid.core.EOFException: - loader.reset( - ) # call DataLoader.reset() after catching EOFException + loader.reset() # call DataLoader.reset() after catching EOFException def train_non_builtin_data_vars(): main_program = static.Program() startup_program = static.Program() - with static.program_guard(main_program, - startup_program), utils.unique_name.guard(): - input = static.data(name="input", - shape=[batch_size, image_size], - dtype='float32') + with static.program_guard( + main_program, startup_program + ), utils.unique_name.guard(): + input = static.data( + name="input", shape=[batch_size, image_size], dtype='float32' + ) label = static.data(name="label", shape=[batch_size, 1], dtype='int64') - loader = paddle.io.DataLoader.from_generator(feed_list=[input, label], - capacity=4 * batch_size, - iterable=False) + loader = paddle.io.DataLoader.from_generator( + feed_list=[input, label], capacity=4 * batch_size, iterable=False + ) places = static.cuda_places() loader.set_batch_generator(batch_generator_creator(), places=places) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) metric = paddle.metric.Accuracy() predict = mlp(input) loss_var = loss(predict, label) @@ -334,53 +352,58 @@ def train_non_builtin_data_vars(): strategy = auto.Strategy() strategy.auto_mode = "semi" - engine = auto.Engine(loss=loss_var, - optimizer=optimizer, - metrics=metric, - strategy=strategy) + engine = auto.Engine( + loss=loss_var, optimizer=optimizer, metrics=metric, strategy=strategy + ) # train engine.to_mode("train") - engine.prepare(inputs=[input], - labels=[label], - main_program=main_program, - startup_program=startup_program) + engine.prepare( + inputs=[input], + labels=[label], + main_program=main_program, + startup_program=startup_program, + ) for _ in range(epoch_num): loader.start() # call DataLoader.start() before each epoch starts try: while True: engine.run() except paddle.fluid.core.EOFException: - loader.reset( - ) # call DataLoader.reset() after catching EOFException + loader.reset() # call DataLoader.reset() after catching EOFException def get_cost(): main_program = static.default_main_program() startup_program = static.default_startup_program() - with static.program_guard(main_program, - startup_program), utils.unique_name.guard(): - input = static.data(name="input", - shape=[batch_size, image_size], - dtype='float32') + with static.program_guard( + main_program, startup_program + ), utils.unique_name.guard(): + input = static.data( + name="input", shape=[batch_size, image_size], dtype='float32' + ) label = static.data(name="label", shape=[batch_size, 1], dtype='int64') - loader = paddle.io.DataLoader.from_generator(feed_list=[input, label], - capacity=4 * batch_size, - iterable=False) + loader = paddle.io.DataLoader.from_generator( + feed_list=[input, label], capacity=4 * batch_size, iterable=False + ) places = static.cuda_places() loader.set_batch_generator(batch_generator_creator(), places=places) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) metric = paddle.metric.Accuracy() predict = mlp(input) loss_var = loss(predict, label) @@ -388,24 +411,27 @@ def get_cost(): strategy = auto.Strategy() strategy.auto_mode = "semi" - engine = auto.Engine(loss=loss_var, - optimizer=optimizer, - metrics=metric, - strategy=strategy) + engine = auto.Engine( + loss=loss_var, optimizer=optimizer, metrics=metric, strategy=strategy + ) engine.cost() def get_cost_by_spec(): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) metric = paddle.metric.Accuracy() strategy = auto.Strategy() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/engine_api_dp.py b/python/paddle/fluid/tests/unittests/auto_parallel/engine_api_dp.py index deb084a79fe59..4a04f9c5a6a25 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/engine_api_dp.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/engine_api_dp.py @@ -34,7 +34,6 @@ class MyDataset(Dataset): - def __init__(self, num_samples): super(MyDataset, self).__init__() self.num_samples = num_samples @@ -49,27 +48,27 @@ def __len__(self): class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.linear2 = nn.Linear(d_model, 1, weight_attr, bias_attr=bias_attr) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") @@ -87,26 +86,28 @@ def forward(self, input): def train(fetch): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) dist_strategy = auto.Strategy() dist_strategy.auto_mode = "semi" # init engine - engine = auto.Engine(mlp, - loss, - optimizer, - paddle.metric.Accuracy(), - strategy=dist_strategy) + engine = auto.Engine( + mlp, loss, optimizer, paddle.metric.Accuracy(), strategy=dist_strategy + ) # train train_dataset = MyDataset(batch_num * batch_size) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/get_gpt_model.py b/python/paddle/fluid/tests/unittests/auto_parallel/get_gpt_model.py index 318773c71e09e..178ba60f81423 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/get_gpt_model.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/get_gpt_model.py @@ -21,14 +21,17 @@ sys.path.append("..") import auto_parallel_gpt_model as modeling -from auto_parallel_gpt_model import GPTModel, GPTForPretraining, GPTPretrainingCriterion +from auto_parallel_gpt_model import ( + GPTModel, + GPTForPretraining, + GPTPretrainingCriterion, +) sequence_len = 512 vocab_size = 1000 class FakeDataset(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples self.sequence_len = sequence_len @@ -40,8 +43,11 @@ def __getitem__(self, idx): random.seed(2021) tokens = np.random.randint(self.vocab_size, size=self.sequence_len) position_ids = np.arange(self.sequence_len) - attention_mask = np.tril(np.ones(self.sequence_len)).reshape( - (1, self.sequence_len, self.sequence_len)).astype(np.float32) + attention_mask = ( + np.tril(np.ones(self.sequence_len)) + .reshape((1, self.sequence_len, self.sequence_len)) + .astype(np.float32) + ) labels = np.random.randint(self.vocab_size, size=self.sequence_len) loss_mask = np.ones(self.sequence_len).astype(np.float32) return tokens, position_ids, attention_mask, labels, loss_mask @@ -51,30 +57,32 @@ def __len__(self): def create_data_holder(batch_size): - tokens = paddle.static.InputSpec(name="tokens", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = paddle.static.InputSpec(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + tokens = paddle.static.InputSpec( + name="tokens", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = paddle.static.InputSpec( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = paddle.static.InputSpec( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float32') - labels = paddle.static.InputSpec(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = paddle.static.InputSpec(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float32') + dtype='float32', + ) + labels = paddle.static.InputSpec( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = paddle.static.InputSpec( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float32' + ) return [tokens, position_ids, attention_mask], [labels, loss_mask] def generate_model(strategy): modeling.init_global() ranks = list(range(paddle.distributed.get_world_size())) - modeling._global_process_mesh = auto.ProcessMesh(mesh=ranks, - dim_names=["x"]) + modeling._global_process_mesh = auto.ProcessMesh( + mesh=ranks, dim_names=["x"] + ) if strategy == "serial": modeling._global_parallel_strategy = "serial" elif strategy == "mp": @@ -84,24 +92,25 @@ def generate_model(strategy): else: raise ValueError("Only support serial, mp2 and dp2.") - gpt = GPTModel(vocab_size=1000, - hidden_size=64, - num_hidden_layers=2, - num_attention_heads=8, - intermediate_size=256, - hidden_act="gelu", - hidden_dropout_prob=0.0, - attention_probs_dropout_prob=0.0, - max_position_embeddings=1024, - type_vocab_size=1, - initializer_range=0.02, - pad_token_id=0, - eos_token_id=7, - bos_token_id=0, - eol_token_id=3) - model = GPTForPretraining(gpt, - vocab_size=1000, - hidden_size=64, - initializer_range=0.02) + gpt = GPTModel( + vocab_size=1000, + hidden_size=64, + num_hidden_layers=2, + num_attention_heads=8, + intermediate_size=256, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=1024, + type_vocab_size=1, + initializer_range=0.02, + pad_token_id=0, + eos_token_id=7, + bos_token_id=0, + eol_token_id=3, + ) + model = GPTForPretraining( + gpt, vocab_size=1000, hidden_size=64, initializer_range=0.02 + ) criterion = GPTPretrainingCriterion() return model, criterion diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/gradient_merge_pass_unittest.py b/python/paddle/fluid/tests/unittests/auto_parallel/gradient_merge_pass_unittest.py index bdce36be605c1..8ed91061fd4e3 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/gradient_merge_pass_unittest.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/gradient_merge_pass_unittest.py @@ -43,7 +43,6 @@ def reset_prog(): class TestGradientMergePass(unittest.TestCase): - def setUp(self): self.rtol = 1e-5 self.atol = 1e-8 @@ -78,23 +77,23 @@ def check_results(self, ref_losses, check_losses): rtol=self.rtol, atol=self.atol, err_msg='pass {} has wrong results!, \nu={}\nv={}\ndiff={}'.format( - __class__, ref_losses, check_losses, ref_losses - check_losses)) + __class__, ref_losses, check_losses, ref_losses - check_losses + ), + ) def test_gradient_merge_pass(self): # dp2 training dp_engine = self.get_engine() - history = dp_engine.fit(self.dataset, - 3, - batch_size=self.batch_size, - log_freq=1) + history = dp_engine.fit( + self.dataset, 3, batch_size=self.batch_size, log_freq=1 + ) dp_losses = np.array(history.history["loss"]) # dp2 gradient merge training gm_engine = self.get_engine(True) - history = gm_engine.fit(self.dataset, - 3, - batch_size=self.batch_size, - log_freq=1) + history = gm_engine.fit( + self.dataset, 3, batch_size=self.batch_size, log_freq=1 + ) gm_losses = np.array(history.history["loss"]) # avg_loss = 0 diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/high_order_grad.py b/python/paddle/fluid/tests/unittests/auto_parallel/high_order_grad.py index 8e148719a5691..1ba31ad2e1d0e 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/high_order_grad.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/high_order_grad.py @@ -22,7 +22,6 @@ class FCNet: - def __init__(self, num_ins, num_outs, num_layers, hidden_size): self.num_ins = num_ins self.num_outs = num_outs @@ -43,12 +42,12 @@ def __init__(self, num_ins, num_outs, num_layers, hidden_size): lsize = self.hidden_size rsize = self.hidden_size - w = paddle.static.create_parameter(shape=[lsize, rsize], - dtype="float32", - is_bias=False) - b = paddle.static.create_parameter(shape=[rsize], - dtype="float32", - is_bias=True) + w = paddle.static.create_parameter( + shape=[lsize, rsize], dtype="float32", is_bias=False + ) + b = paddle.static.create_parameter( + shape=[rsize], dtype="float32", is_bias=True + ) self.weights.append(w) self.biases.append(b) @@ -62,13 +61,14 @@ def nn_func(self, ins): class LaplaceModel(paddle.nn.Layer): - def __init__(self, num_ins=2, num_outs=1, num_layers=5, hidden_size=20): super(LaplaceModel, self).__init__() - self.net = FCNet(num_ins=num_ins, - num_outs=num_outs, - num_layers=num_layers, - hidden_size=hidden_size) + self.net = FCNet( + num_ins=num_ins, + num_outs=num_outs, + num_layers=num_layers, + hidden_size=hidden_size, + ) def forward(self, inputs, bc_index): inputs.stop_gradient = False @@ -82,7 +82,6 @@ def forward(self, inputs, bc_index): class LaplaceDataset(paddle.io.Dataset): - def __init__(self, num_sample): self.num_sample = num_sample @@ -126,10 +125,9 @@ def main(): dist_strategy = auto.Strategy() dist_strategy.auto_mode = "semi" - engine = auto.Engine(laplace, - loss=loss_func, - optimizer=optimizer, - strategy=dist_strategy) + engine = auto.Engine( + laplace, loss=loss_func, optimizer=optimizer, strategy=dist_strategy + ) engine.fit(train_dataset, train_sample_split=2, batch_size=None) dist_context = engine.dist_context diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/iterable_dataset.py b/python/paddle/fluid/tests/unittests/auto_parallel/iterable_dataset.py index d5eb88bc6b55a..b953471eef813 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/iterable_dataset.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/iterable_dataset.py @@ -36,7 +36,6 @@ class MyDataset(paddle.io.IterableDataset): - def __init__(self, num_samples): self.num_samples = num_samples @@ -48,17 +47,18 @@ def __iter__(self): class MyDataset1(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples self.data = [] for i in range(self.num_samples): input1 = np.random.uniform(size=image_size).astype("float32") - label1 = np.array(np.random.randint(0, class_num - 1, - dtype="int64")) + label1 = np.array( + np.random.randint(0, class_num - 1, dtype="int64") + ) input2 = np.random.uniform(size=image_size).astype("float32") - label2 = np.array(np.random.randint(0, class_num - 1, - dtype="int64")) + label2 = np.array( + np.random.randint(0, class_num - 1, dtype="int64") + ) input = np.stack((input1, input2)) label = np.stack((label1, label2)) self.data.append((input, label)) @@ -71,27 +71,27 @@ def __len__(self): class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.linear2 = nn.Linear(d_model, 1, weight_attr, bias_attr=bias_attr) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") @@ -108,27 +108,29 @@ def forward(self, input): def train(fetch): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) dist_strategy = auto.Strategy() dist_strategy.auto_mode = "semi" dist_strategy.split_data = True # init engine - engine = auto.Engine(mlp, - loss, - optimizer, - paddle.metric.Accuracy(), - strategy=dist_strategy) + engine = auto.Engine( + mlp, loss, optimizer, paddle.metric.Accuracy(), strategy=dist_strategy + ) # train train_dataset = MyDataset(batch_num * batch_size) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/optimization_tuner_api.py b/python/paddle/fluid/tests/unittests/auto_parallel/optimization_tuner_api.py index 3cc24a538fe08..b05e27d4c5433 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/optimization_tuner_api.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/optimization_tuner_api.py @@ -31,27 +31,27 @@ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.linear2 = nn.Linear(d_model, 1, weight_attr, bias_attr=bias_attr) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") @@ -68,16 +68,20 @@ def forward(self, input): def train(fetch): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) dist_strategy = auto.Strategy() dist_strategy.auto_mode = "semi" @@ -97,15 +101,13 @@ def train(fetch): tuning.verbose = True dataset = MyDataset(batch_num * batch_size) - engine = auto.Engine(mlp, - loss, - optimizer, - paddle.metric.Accuracy(), - strategy=dist_strategy) + engine = auto.Engine( + mlp, loss, optimizer, paddle.metric.Accuracy(), strategy=dist_strategy + ) engine._tune(dataset, batch_size=batch_size) # check tuned - assert (engine._dist_contexts['train'].strategy.sharding.stage != 3) + assert engine._dist_contexts['train'].strategy.sharding.stage != 3 if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/recompute_pass_unittest.py b/python/paddle/fluid/tests/unittests/auto_parallel/recompute_pass_unittest.py index 9c17af12d73de..1aa83f1a8c978 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/recompute_pass_unittest.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/recompute_pass_unittest.py @@ -38,7 +38,6 @@ def reset_prog(): class TestRecomputePass(unittest.TestCase): - def setUp(self): self.rtol = 1e-6 self.atol = 1e-8 @@ -73,7 +72,9 @@ def check_results(self, ref_losses, check_losses): rtol=self.rtol, atol=self.atol, err_msg='pass {} has wrong results!, \nu={}\nv={}\ndiff={}'.format( - __class__, ref_losses, check_losses, ref_losses - check_losses)) + __class__, ref_losses, check_losses, ref_losses - check_losses + ), + ) def test_recompute_pass(self): # mp2 training diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/sharding_pass_unittest.py b/python/paddle/fluid/tests/unittests/auto_parallel/sharding_pass_unittest.py index a549cd8e6d8eb..80c2b74ade226 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/sharding_pass_unittest.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/sharding_pass_unittest.py @@ -43,7 +43,6 @@ def reset_prog(): class TestShardingPass(unittest.TestCase): - def setUp(self): self.rtol = 1e-6 self.atol = 1e-8 @@ -78,7 +77,9 @@ def check_results(self, ref_losses, check_losses): rtol=self.rtol, atol=self.atol, err_msg='pass {} has wrong results!, \nu={}\nv={}\ndiff={}'.format( - __class__, ref_losses, check_losses, ref_losses - check_losses)) + __class__, ref_losses, check_losses, ref_losses - check_losses + ), + ) def test_sharding_pass(self): # dp2 training @@ -88,25 +89,25 @@ def test_sharding_pass(self): # sharding2 stage1 training sharding1_engine = self.get_engine(True, 1) - history = sharding1_engine.fit(self.dataset, - 3, - batch_size=self.batch_size) + history = sharding1_engine.fit( + self.dataset, 3, batch_size=self.batch_size + ) sharding1_losses = np.array(history.history["loss"]) self.check_results(dp_losses, sharding1_losses) # sharding2 stage2 training sharding2_engine = self.get_engine(True, 2) - history = sharding2_engine.fit(self.dataset, - 3, - batch_size=self.batch_size) + history = sharding2_engine.fit( + self.dataset, 3, batch_size=self.batch_size + ) sharding2_losses = np.array(history.history["loss"]) self.check_results(dp_losses, sharding2_losses) # sharding2 stage3 training sharding3_engine = self.get_engine(True, 3) - history = sharding3_engine.fit(self.dataset, - 3, - batch_size=self.batch_size) + history = sharding3_engine.fit( + self.dataset, 3, batch_size=self.batch_size + ) sharding3_losses = np.array(history.history["loss"]) self.check_results(dp_losses, sharding3_losses) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_auto_parallel_relaunch.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_auto_parallel_relaunch.py index 53734e21d9a35..42d309866c396 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_auto_parallel_relaunch.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_auto_parallel_relaunch.py @@ -93,7 +93,6 @@ class TestAutoParallelReLaunch(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -101,10 +100,12 @@ def tearDown(self): self.temp_dir.cleanup() def test_relaunch(self): - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") - mapping_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_rank_mapping.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) + mapping_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_rank_mapping.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: @@ -115,20 +116,32 @@ def test_relaunch(self): json.dump(mapping_josn_object, mapping_josn_file) file_dir = os.path.dirname(os.path.abspath(__file__)) - launch_model_path = os.path.join(file_dir, - "auto_parallel_relaunch_model.py") + launch_model_path = os.path.join( + file_dir, "auto_parallel_relaunch_model.py" + ) if os.environ.get("WITH_COVERAGE", "OFF") == "ON": coverage_args = ["-m", "coverage", "run", "--branch", "-p"] else: coverage_args = [] - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--log_dir", self.temp_dir.name, - "--cluster_topo_path", cluster_json_path, "--rank_mapping_path", - mapping_json_path, "--enable_auto_mapping", "True", - launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--log_dir", + self.temp_dir.name, + "--cluster_topo_path", + cluster_json_path, + "--rank_mapping_path", + mapping_json_path, + "--enable_auto_mapping", + "True", + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() self.assertEqual(process.returncode, 0) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_base_cost.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_base_cost.py index 745351f3d7e47..b03f3408d1f12 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_base_cost.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_base_cost.py @@ -28,10 +28,18 @@ from paddle.distributed import fleet from paddle.distributed.auto_parallel.parallelizer import AutoParallelizer from paddle.distributed.auto_parallel.cluster import Cluster -from paddle.distributed.auto_parallel.cost.base_cost import build_comp_desc_from_dist_op -from paddle.distributed.auto_parallel.cost.base_cost import build_comm_desc_from_dist_op -from paddle.distributed.auto_parallel.cost.base_cost import build_comm_costs_from_descs -from paddle.distributed.auto_parallel.cost.base_cost import build_comp_costs_from_descs +from paddle.distributed.auto_parallel.cost.base_cost import ( + build_comp_desc_from_dist_op, +) +from paddle.distributed.auto_parallel.cost.base_cost import ( + build_comm_desc_from_dist_op, +) +from paddle.distributed.auto_parallel.cost.base_cost import ( + build_comm_costs_from_descs, +) +from paddle.distributed.auto_parallel.cost.base_cost import ( + build_comp_costs_from_descs, +) from paddle.distributed.auto_parallel.cost.base_cost import build_dp_costs from paddle.distributed.auto_parallel.cost import AllreduceSumOpCost from paddle.distributed.auto_parallel.cost import _g_op_cost_factory @@ -39,33 +47,34 @@ paddle.enable_static() _global_parallel_strategy = "dp_mp_pp" -_global_process_mesh = auto.ProcessMesh([[[0, 1], [4, 5]], [[2, 3], [6, 7]]], - dim_names=["x", "y", "z"]) +_global_process_mesh = auto.ProcessMesh( + [[[0, 1], [4, 5]], [[2, 3], [6, 7]]], dim_names=["x", "y", "z"] +) PP_MESH_0 = auto.ProcessMesh([[0, 1], [4, 5]], dim_names=["x", "y"]) PP_MESH_1 = auto.ProcessMesh([[2, 3], [6, 7]], dim_names=["x", "y"]) class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -81,29 +90,33 @@ def forward(self, input): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) fill_constant_out = paddle.fluid.layers.fill_constant_batch_size_like( - input=input, shape=[batch_size], value=1, dtype="int32") + input=input, shape=[batch_size], value=1, dtype="int32" + ) embedding = paddle.nn.Embedding(10, hidden_size, sparse=True) embedding_out = embedding(fill_constant_out) auto.shard_tensor(input, PP_MESH_0, ["x", None]) auto.shard_tensor(label, PP_MESH_1, ["x", None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(embedding_out) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -115,8 +128,9 @@ def mlp_forward(train_program, start_program): def get_prog(train_program, startup_program, dist_context, rank_id): global _global_process_mesh dist_context.process_mesh = _global_process_mesh - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) fleet._user_defined_strategy = fleet.DistributedStrategy() fleet.user_defined_optimizer = paddle.fluid.optimizer.AdamOptimizer() @@ -126,19 +140,21 @@ def get_prog(train_program, startup_program, dist_context, rank_id): # serial forward & backward completion completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) return train_program, startup_program, params_grads class TestBaseCost(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -147,8 +163,9 @@ def tearDown(self): def test_base_cost(self): # Build cluster - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) @@ -160,7 +177,8 @@ def test_base_cost(self): dist_context = DistributedContext() rank_id = 2 train_program, startup_program, params_grads = get_prog( - train_program, startup_program, dist_context, rank_id) + train_program, startup_program, dist_context, rank_id + ) for op in train_program.global_block().ops: dist_op = dist_context.get_dist_op_for_program(op) @@ -171,13 +189,15 @@ def test_base_cost(self): var_names = None if op.input_arg_names: var_names = op.input_arg_names[0] - comm_descs = build_comm_desc_from_dist_op("c_allreduce_sum", - dist_op, - dist_context, - var_names, - attrs=None, - parallel_axis=0, - group_ranks=None) + comm_descs = build_comm_desc_from_dist_op( + "c_allreduce_sum", + dist_op, + dist_context, + var_names, + attrs=None, + parallel_axis=0, + group_ranks=None, + ) self.assertTrue(isinstance(comm_descs, dict) and comm_descs) comm_descs = build_comm_desc_from_dist_op( "c_allreduce_sum", @@ -186,22 +206,38 @@ def test_base_cost(self): var_names, attrs=None, parallel_axis=None, - group_ranks=processes) + group_ranks=processes, + ) self.assertTrue(isinstance(comm_descs, dict) and comm_descs) comm_costs = build_comm_costs_from_descs( - AllreduceSumOpCost, dist_context, processes, comm_descs, - cluster) + AllreduceSumOpCost, + dist_context, + processes, + comm_descs, + cluster, + ) self.assertTrue(comm_costs) comp_costs = build_comp_costs_from_descs( - _g_op_cost_factory[op.type], dist_context, processes, - comp_descs, cluster) + _g_op_cost_factory[op.type], + dist_context, + processes, + comp_descs, + cluster, + ) self.assertTrue(comp_costs) result = [] - build_dp_costs(result, dist_op, dist_context, var_names[0], - None, 0, cluster) + build_dp_costs( + result, + dist_op, + dist_context, + var_names[0], + None, + 0, + cluster, + ) self.assertTrue(result) # Remove unnecessary files diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster.py index 1aeed0c72ef7b..f5c4d53c06016 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster.py @@ -1968,7 +1968,6 @@ class TestCluster(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -1977,8 +1976,9 @@ def tearDown(self): def test_single_machine(self): # Build cluster - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster_single.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster_single.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: @@ -2003,8 +2003,9 @@ def test_single_machine(self): def test_multi_machine(self): # Build cluster - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster_multi.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster_multi.json" + ) cluster_json_object = json.loads(multi_cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster_v2.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster_v2.py index acadc4842a213..832cdf75f9224 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster_v2.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_cluster_v2.py @@ -19,7 +19,6 @@ class TestDeviceMesh(unittest.TestCase): - def test_device_mesh(self): name = "my_device_mesh" mesh = [[0, 1, 2], [3, 4, 5]] @@ -77,11 +76,11 @@ def test_device_mesh(self): self.assertEqual(device_mesh.machine(0).device(2), dev2) self.assertEqual(device_mesh.machine(1).link(3, 4), link2) self.assertEqual( - device_mesh.machine(0).devices, - device_mesh.machine(0).devices) + device_mesh.machine(0).devices, device_mesh.machine(0).devices + ) self.assertEqual( - device_mesh.machine(0).links, - device_mesh.machine(0).links) + device_mesh.machine(0).links, device_mesh.machine(0).links + ) self.assertEqual(device_mesh.device_type, "GPU") self.assertEqual(device_mesh.devices, device_mesh.devices) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_comm_cost.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_comm_cost.py index 5744cf6d39206..32b180ffdc16a 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_comm_cost.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_comm_cost.py @@ -32,7 +32,6 @@ class TestCommOpCost(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -41,8 +40,9 @@ def tearDown(self): def test_comm_cost(self): # Build cluster - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster0.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster0.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) @@ -55,54 +55,72 @@ def test_comm_cost(self): comm_context = CommContext(cluster) # Check AllreduceSumCost 128MB ring cost - allreduce_sum_op_desc = build_comm_desc("c_allreduce_sum", - [0, 1, 2, 3, 4, 5, 6, 7], - paddle.float32, - [1, 32 * (10**6)]) + allreduce_sum_op_desc = build_comm_desc( + "c_allreduce_sum", + [0, 1, 2, 3, 4, 5, 6, 7], + paddle.float32, + [1, 32 * (10**6)], + ) allreduce_sum_op_cost = AllreduceSumOpCost( - op_desc=allreduce_sum_op_desc, comm_context=comm_context) + op_desc=allreduce_sum_op_desc, comm_context=comm_context + ) # Check AllgatherOpCost cost - allgather_op_desc = build_comm_desc("c_allgather", - [0, 1, 2, 3, 4, 5, 6, 7], - paddle.float32, [1, 32 * (10**6)]) - allgather_op_cost = AllgatherOpCost(op_desc=allgather_op_desc, - comm_context=comm_context) + allgather_op_desc = build_comm_desc( + "c_allgather", + [0, 1, 2, 3, 4, 5, 6, 7], + paddle.float32, + [1, 32 * (10**6)], + ) + allgather_op_cost = AllgatherOpCost( + op_desc=allgather_op_desc, comm_context=comm_context + ) self.assertTrue(allgather_op_cost.time > 0) # Check BroadcastOpCost cost - broadcast_op_desc = build_comm_desc("c_broadcast", - [0, 1, 2, 3, 4, 5, 6, 7], - paddle.float32, [1, 32 * (10**6)]) - broadcast_op_cost = BroadcastOpCost(op_desc=broadcast_op_desc, - comm_context=comm_context) + broadcast_op_desc = build_comm_desc( + "c_broadcast", + [0, 1, 2, 3, 4, 5, 6, 7], + paddle.float32, + [1, 32 * (10**6)], + ) + broadcast_op_cost = BroadcastOpCost( + op_desc=broadcast_op_desc, comm_context=comm_context + ) self.assertTrue(broadcast_op_cost.time > 0) # Check SendOpCost cost - send_op_desc = build_comm_desc("send_v2", [0, 1], paddle.float32, - [1, 32 * (10**6)]) - send_op_cost = SendOpCost(op_desc=send_op_desc, - comm_context=comm_context) + send_op_desc = build_comm_desc( + "send_v2", [0, 1], paddle.float32, [1, 32 * (10**6)] + ) + send_op_cost = SendOpCost( + op_desc=send_op_desc, comm_context=comm_context + ) self.assertTrue(send_op_cost.time > 0) # Check RecvOpCost cost - recv_op_desc = build_comm_desc("recv_v2", [0, 1], paddle.float32, - [1, 32 * (10**6)]) - recv_op_cost = RecvOpCost(op_desc=recv_op_desc, - comm_context=comm_context) + recv_op_desc = build_comm_desc( + "recv_v2", [0, 1], paddle.float32, [1, 32 * (10**6)] + ) + recv_op_cost = RecvOpCost( + op_desc=recv_op_desc, comm_context=comm_context + ) self.assertTrue(recv_op_cost.time > 0) # Check IdentityOpCost cost - identity_op_desc = build_comm_desc("c_identity", [0, 1], paddle.float32, - [1, 32 * (10**6)]) - identity_op_cost = IdentityOpCost(op_desc=identity_op_desc, - comm_context=comm_context) + identity_op_desc = build_comm_desc( + "c_identity", [0, 1], paddle.float32, [1, 32 * (10**6)] + ) + identity_op_cost = IdentityOpCost( + op_desc=identity_op_desc, comm_context=comm_context + ) self.assertTrue(identity_op_cost.time >= 0) def test_cross_machine_comm_cost(self): # Build cluster - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster1.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster1.json" + ) cluster_json_object = json.loads(multi_cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) @@ -118,40 +136,53 @@ def test_cross_machine_comm_cost(self): allreduce_sum_op_desc = build_comm_desc( "c_allreduce_sum", [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - paddle.float32, [1, 32 * (10**6)]) + paddle.float32, + [1, 32 * (10**6)], + ) allreduce_sum_op_cost = AllreduceSumOpCost( - op_desc=allreduce_sum_op_desc, comm_context=comm_context) + op_desc=allreduce_sum_op_desc, comm_context=comm_context + ) # Check AllgatherOpCost cost allgather_op_desc = build_comm_desc( "c_allgather", [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - paddle.float32, [1, 32 * (10**6)]) - allgather_op_cost = AllgatherOpCost(op_desc=allgather_op_desc, - comm_context=comm_context) + paddle.float32, + [1, 32 * (10**6)], + ) + allgather_op_cost = AllgatherOpCost( + op_desc=allgather_op_desc, comm_context=comm_context + ) self.assertTrue(allgather_op_cost.time > 0) # Check BroadcastOpCost cost broadcast_op_desc = build_comm_desc( "c_broadcast", [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - paddle.float32, [1, 32 * (10**6)]) - broadcast_op_cost = BroadcastOpCost(op_desc=broadcast_op_desc, - comm_context=comm_context) + paddle.float32, + [1, 32 * (10**6)], + ) + broadcast_op_cost = BroadcastOpCost( + op_desc=broadcast_op_desc, comm_context=comm_context + ) self.assertTrue(broadcast_op_cost.time > 0) # Check SendOpCost cost - send_op_desc = build_comm_desc("send_v2", [0, 1], paddle.float32, - [1, 32 * (10**6)]) - send_op_cost = SendOpCost(op_desc=send_op_desc, - comm_context=comm_context) + send_op_desc = build_comm_desc( + "send_v2", [0, 1], paddle.float32, [1, 32 * (10**6)] + ) + send_op_cost = SendOpCost( + op_desc=send_op_desc, comm_context=comm_context + ) self.assertTrue(send_op_cost.time > 0) # Check RecvOpCost cost - recv_op_desc = build_comm_desc("recv_v2", [0, 1], paddle.float32, - [1, 32 * (10**6)]) - recv_op_cost = RecvOpCost(op_desc=recv_op_desc, - comm_context=comm_context) + recv_op_desc = build_comm_desc( + "recv_v2", [0, 1], paddle.float32, [1, 32 * (10**6)] + ) + recv_op_cost = RecvOpCost( + op_desc=recv_op_desc, comm_context=comm_context + ) self.assertTrue(recv_op_cost.time > 0) # Remove unnecessary files diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_comp_cost.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_comp_cost.py index f57b7d8631115..8520654f37e93 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_comp_cost.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_comp_cost.py @@ -20,56 +20,100 @@ from paddle.distributed.auto_parallel.cost.comp_op_cost import AssignOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import AssignValueOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import BeamSearchOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import BeamSearchDecodeOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + BeamSearchDecodeOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import CastOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import ConcatOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ElementwiseAddOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ElementwiseAddGradOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ElementwiseDivOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ElementwiseDivGradOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ElementwiseMulOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ElementwiseMulGradOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ElementwiseSubOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ElementwiseAddOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ElementwiseAddGradOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ElementwiseDivOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ElementwiseDivGradOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ElementwiseMulOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ElementwiseMulGradOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ElementwiseSubOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import EmbeddingOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import EmbeddingGradOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import FillConstantOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import FillConstantBatchSizeLikeOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + EmbeddingGradOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + FillConstantOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + FillConstantBatchSizeLikeOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import GatherOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import GeluOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import GeluGradOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import GreaterEqualOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + GreaterEqualOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import IncrementOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import IsEmptyOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import LayerNormOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import LayerNormGradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + LayerNormGradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import LessThanOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import LogicalNotOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import LogicalAndOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import LodResetOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import LogOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import LookupTableV2OpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import LookupTableV2GradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + LookupTableV2OpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + LookupTableV2GradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import MatmulOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import MatmulV2OpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import MatmulV2GradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + MatmulV2GradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import MemcpyOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import MulOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import MulGradOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import OneHotOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ReadFromArrayOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ReadFromArrayOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import ReduceSumOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ReduceSumGradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ReduceSumGradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import Reshape2OpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import Reshape2GradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + Reshape2GradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import ReduceMeanOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import ReduceMeanGradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + ReduceMeanGradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import SamplingIdOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import ScaleOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import SliceOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import SoftmaxOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import SoftmaxGradOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import SoftmaxWithCrossEntropyOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import SoftmaxWithCrossEntropyGradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + SoftmaxWithCrossEntropyOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + SoftmaxWithCrossEntropyGradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import SplitOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import Squeeze2OpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import SquareOpCost @@ -77,18 +121,25 @@ from paddle.distributed.auto_parallel.cost.comp_op_cost import SumOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import TopKOpCost from paddle.distributed.auto_parallel.cost.comp_op_cost import Transpose2OpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import Transpose2GradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + Transpose2GradOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import Unsqueeze2OpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import WriteToArrayOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + WriteToArrayOpCost, +) from paddle.distributed.auto_parallel.cost.comp_op_cost import DropoutGradOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import FusedSoftmaxMaskUpperTriangleOpCost -from paddle.distributed.auto_parallel.cost.comp_op_cost import FusedSoftmaxMaskUpperTriangleGradOpCost +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + FusedSoftmaxMaskUpperTriangleOpCost, +) +from paddle.distributed.auto_parallel.cost.comp_op_cost import ( + FusedSoftmaxMaskUpperTriangleGradOpCost, +) from test_cluster import cluster_json class TestCompOpCost(unittest.TestCase): - def test_comp_cost(self): # Build cluster file_dir = os.path.dirname(os.path.abspath(__file__)) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_conditional_block_reshard.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_conditional_block_reshard.py index 86371cbae6436..e7ecdece944e1 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_conditional_block_reshard.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_conditional_block_reshard.py @@ -23,36 +23,44 @@ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=64, - intermediate_size=4 * 64, - initializer_range=0.02): + def __init__( + self, hidden_size=64, intermediate_size=4 * 64, initializer_range=0.02 + ): super(MLPLayer, self).__init__() self.norm = nn.LayerNorm(hidden_size, epsilon=1e-5) self.linear0 = nn.Linear( hidden_size, intermediate_size, - paddle.ParamAttr(initializer=nn.initializer.Normal( - mean=0.0, std=initializer_range)), - bias_attr=None) + paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ) + ), + bias_attr=None, + ) self.linear1 = nn.Linear( intermediate_size, hidden_size, - paddle.ParamAttr(initializer=nn.initializer.Normal( - mean=0.0, std=initializer_range)), - bias_attr=None) + paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ) + ), + bias_attr=None, + ) def forward(self, input): out = self.norm(input) - auto.shard_tensor(self.linear0.weight, auto.ProcessMesh([0, 1], "x"), - [None, "x"]) + auto.shard_tensor( + self.linear0.weight, auto.ProcessMesh([0, 1], "x"), [None, "x"] + ) out = self.linear0(out) out = F.gelu(out, approximate=True) - auto.shard_tensor(self.linear1.weight, auto.ProcessMesh([0, 1], "x"), - ["x", None]) + auto.shard_tensor( + self.linear1.weight, auto.ProcessMesh([0, 1], "x"), ["x", None] + ) out = self.linear1(out) if paddle.mean(out) < 2: @@ -75,7 +83,6 @@ def loss_fn(predict, label): class TestSubblock(unittest.TestCase): - def test_subblock(self): mlp = MLPLayer() @@ -87,9 +94,9 @@ def test_subblock(self): input_sepc = InputSpec([4, 64], 'float32', 'input') label_spec = InputSpec([4, 1], 'float32', 'label') - engine.prepare(inputs_spec=[input_sepc], - labels_spec=[label_spec], - mode="predict") + engine.prepare( + inputs_spec=[input_sepc], labels_spec=[label_spec], mode="predict" + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_converter.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_converter.py index 7c838513d7987..4b8b964ed4336 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_converter.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_converter.py @@ -21,7 +21,6 @@ class TestConverter(unittest.TestCase): - def test_converter(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "converter.py") @@ -32,10 +31,19 @@ def test_converter(self): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() @@ -57,7 +65,7 @@ def test_input_invalid(self): 'tmp_0': { "process_shape": [1], "process_group": [0], - "dims_mapping": [-1] + "dims_mapping": [-1], } } with self.assertRaises(TypeError): diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_assign.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_assign.py index b21dd606d8cb7..39d146c2e7f88 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_assign.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_assign.py @@ -26,8 +26,9 @@ def make_program(): x = paddle.static.data(name='x', shape=[4, 4, 8], dtype='float32') y = paddle.static.data(name='y', shape=[4, 4, 8], dtype='float32') - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["d"]), - [None, "d", None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["d"]), [None, "d", None] + ) z = paddle.add(x, y) paddle.assign(x, output=z) @@ -48,14 +49,14 @@ def parallelizer(program_func, rank): dist_context.block_state.parse_forward_blocks(main_program) partitioner = Partitioner(dist_context, rank) - dist_main_prog, _, _ = partitioner.partition(main_program, start_program, - []) + dist_main_prog, _, _ = partitioner.partition( + main_program, start_program, [] + ) return dist_main_prog, dist_context class TestDistAssign(unittest.TestCase): - def test_dist_assign(self): dist_main_prog, dist_context = parallelizer(make_program, 0) @@ -72,9 +73,11 @@ def test_dist_assign(self): dist_out = dist_context.get_dist_tensor_for_program(out_var) x_dims_mapping = dist_op.dist_attr.get_input_dims_mapping( - x_name) + x_name + ) out_dims_mapping = dist_op.dist_attr.get_output_dims_mapping( - out_name) + out_name + ) assert x_dims_mapping == out_dims_mapping assert out_dims_mapping == dist_out.dist_attr.dims_mapping diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_attr_v2.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_attr_v2.py index 9977fb4654aa8..acff242fd8c34 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_attr_v2.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_attr_v2.py @@ -24,7 +24,6 @@ class TestDistAttr(unittest.TestCase): - def test_tensor_dist_attr_ctor(self): train_program = static.Program() start_program = static.Program() @@ -40,8 +39,9 @@ def test_tensor_dist_attr_ctor(self): dist_attr.dims_mapping = [0, -1] dist_attr.batch_dim = 1 dist_attr.dynamic_dims = [1, 1] - self.assertEqual(dist_attr.process_mesh, - ProcessMesh([[0, 1, 2], [3, 4, 5]])) + self.assertEqual( + dist_attr.process_mesh, ProcessMesh([[0, 1, 2], [3, 4, 5]]) + ) self.assertEqual(dist_attr.dims_mapping, [0, -1]) self.assertEqual(dist_attr.batch_dim, 1) self.assertEqual(dist_attr.dynamic_dims, [1, 1]) @@ -59,16 +59,18 @@ def test_tensor_dist_attr(self): dist_attr.dims_mapping = [0, -1] dist_attr.batch_dim = 1 dist_attr.dynamic_dims = [1, 1] - self.assertEqual(input.dist_attr.process_mesh, - ProcessMesh([[0, 1, 2], [3, 4, 5]])) + self.assertEqual( + input.dist_attr.process_mesh, ProcessMesh([[0, 1, 2], [3, 4, 5]]) + ) self.assertEqual(input.dist_attr.dims_mapping, [0, -1]) self.assertEqual(input.dist_attr.batch_dim, 1) self.assertEqual(input.dist_attr.dynamic_dims, [1, 1]) self.assertTrue(input.dist_attr.verify()) input1.dist_attr = dist_attr - self.assertEqual(input1.dist_attr.process_mesh, - ProcessMesh([[0, 1, 2], [3, 4, 5]])) + self.assertEqual( + input1.dist_attr.process_mesh, ProcessMesh([[0, 1, 2], [3, 4, 5]]) + ) self.assertEqual(input1.dist_attr.dims_mapping, [0, -1]) self.assertEqual(input1.dist_attr.batch_dim, 1) self.assertEqual(input1.dist_attr.dynamic_dims, [1, 1]) @@ -100,19 +102,24 @@ def test_operator_dist_attr_ctor(self): op_dist_attr.set_output_dist_attr(output.name, output_dist_attr) self.assertEqual(op_dist_attr.process_mesh, process_mesh) self.assertEqual( - op_dist_attr.input_dist_attr(input.name).process_mesh, process_mesh) + op_dist_attr.input_dist_attr(input.name).process_mesh, process_mesh + ) self.assertEqual( - op_dist_attr.input_dist_attr(input1.name).process_mesh, - process_mesh) + op_dist_attr.input_dist_attr(input1.name).process_mesh, process_mesh + ) self.assertEqual( op_dist_attr.output_dist_attr(output.name).process_mesh, - process_mesh) + process_mesh, + ) self.assertEqual( - op_dist_attr.input_dist_attr(input.name).dims_mapping, [0, -1]) + op_dist_attr.input_dist_attr(input.name).dims_mapping, [0, -1] + ) self.assertEqual( - op_dist_attr.input_dist_attr(input1.name).dims_mapping, [-1, 1]) + op_dist_attr.input_dist_attr(input1.name).dims_mapping, [-1, 1] + ) self.assertEqual( - op_dist_attr.output_dist_attr(output.name).dims_mapping, [0, 1]) + op_dist_attr.output_dist_attr(output.name).dims_mapping, [0, 1] + ) self.assertTrue(op_dist_attr.verify()) self.assertTrue(str(op_dist_attr), str(op_dist_attr)) @@ -164,19 +171,23 @@ def test_operator_dist_attr(self): self.assertEqual(op.desc.dist_attr.process_mesh, process_mesh) self.assertEqual( - op.dist_attr.input_dist_attr(input.name).process_mesh, process_mesh) + op.dist_attr.input_dist_attr(input.name).process_mesh, process_mesh + ) self.assertEqual( - op.dist_attr.input_dist_attr(input1.name).process_mesh, - process_mesh) + op.dist_attr.input_dist_attr(input1.name).process_mesh, process_mesh + ) self.assertEqual( - op.dist_attr.input_dist_attr(input.name).dims_mapping, [0, -1]) + op.dist_attr.input_dist_attr(input.name).dims_mapping, [0, -1] + ) self.assertEqual( - op.dist_attr.input_dist_attr(input.name).dims_mapping, [0, -1]) + op.dist_attr.input_dist_attr(input.name).dims_mapping, [0, -1] + ) self.assertEqual( - op.desc.dist_attr.input_dist_attr(input1.name).dims_mapping, - [-1, 1]) + op.desc.dist_attr.input_dist_attr(input1.name).dims_mapping, [-1, 1] + ) self.assertEqual( - op.dist_attr.output_dist_attr(output.name).dims_mapping, [0, 1]) + op.dist_attr.output_dist_attr(output.name).dims_mapping, [0, 1] + ) self.assertTrue(op.desc.dist_attr.verify()) self.assertTrue(str(op_dist_attr), str(op_dist_attr)) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_context.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_context.py index ea24f65479561..c872f9e9d0c4f 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_context.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_context.py @@ -32,7 +32,7 @@ sequence_len = 512 _g_process_mesh = [ auto.ProcessMesh([0, 1], dim_names=["x"]), - auto.ProcessMesh([2, 3], dim_names=["x"]) + auto.ProcessMesh([2, 3], dim_names=["x"]), ] @@ -43,41 +43,45 @@ def get_random_inputs_and_labels(input_shape, label_shape): def batch_generator_creator(): - def __reader__(): for _ in range(batch_size): batch_input, batch_label = get_random_inputs_and_labels( [batch_size, sequence_len, hidden_size], - [batch_size, sequence_len, 1]) + [batch_size, sequence_len, 1], + ) yield batch_input, batch_label return __reader__ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size - param_initializer = nn.initializer.Normal(mean=0.0, - std=initializer_range) + param_initializer = nn.initializer.Normal( + mean=0.0, std=initializer_range + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.linear0 = nn.Linear( d_model, dim_feedforward, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) self.linear1 = nn.Linear( dim_feedforward, d_model, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) def forward(self, input): out = self.norm(input) @@ -99,78 +103,106 @@ def get_program(): start_program = static.Program() with static.program_guard(train_program, start_program): # input - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, sequence_len, 1], - dtype='float32') + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + label = static.data( + name="label", shape=[batch_size, sequence_len, 1], dtype='float32' + ) data_holder = [input, label] # dataloader - dataloader = paddle.io.DataLoader.from_generator(feed_list=data_holder, - capacity=4 * - batch_size, - iterable=False) - dataloader.set_batch_generator(batch_generator_creator(), - places=paddle.static.cuda_places()) + dataloader = paddle.io.DataLoader.from_generator( + feed_list=data_holder, capacity=4 * batch_size, iterable=False + ) + dataloader.set_batch_generator( + batch_generator_creator(), places=paddle.static.cuda_places() + ) # data dist_attr auto.shard_tensor(input, _g_process_mesh[0], ["x", None, None]) auto.shard_tensor(label, _g_process_mesh[0], ["x", None, None]) - mlp_start = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_start = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) pred = mlp_start(input) - mlp_mid = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_mid = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) pred = mlp_mid(pred) - mlp_end = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_end = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) pred = mlp_end(pred) error_cost = paddle.nn.functional.square_error_cost(pred, label) loss = paddle.mean(error_cost) - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) feed_vars = {"inputs": [input], "labels": [label]} fetch_vars = {"loss": [loss]} - return train_program, start_program, dataloader, loss, optimizer, feed_vars, fetch_vars + return ( + train_program, + start_program, + dataloader, + loss, + optimizer, + feed_vars, + fetch_vars, + ) class TestDistributedContext(unittest.TestCase): - def test_backup_restore(self): - train_program, start_program, dataloader, loss, optimizer, feed_vars, fetch_vars = get_program( + ( + train_program, + start_program, + dataloader, + loss, + optimizer, + feed_vars, + fetch_vars, + ) = get_program() + dist_context = DistributedContext( + train_program, start_program, optimizer, loss, feed_vars, fetch_vars ) - dist_context = DistributedContext(train_program, start_program, - optimizer, loss, feed_vars, - fetch_vars) dist_context.initialize() dist_context._backup(serial=True, dist=True) - dist_context._restore(serial=True, - serial_mode="to_backup", - dist=True, - dist_mode="to_backup") + dist_context._restore( + serial=True, + serial_mode="to_backup", + dist=True, + dist_mode="to_backup", + ) dist_context._backup(serial=True, dist=True) - dist_context._restore(serial=True, - serial_mode="to_original", - dist=True, - dist_mode="to_original") + dist_context._restore( + serial=True, + serial_mode="to_original", + dist=True, + dist_mode="to_original", + ) dist_context._backup(serial=True, dist=True) dist_context._restore(serial=True, dist=True, dist_mode="to_default") @@ -179,25 +211,44 @@ def test_backup_restore(self): dist_context._restore(serial=True, dist=True, dist_mode="to_nothing") def test_deepcopy(self): - train_program, start_program, dataloader, loss, optimizer, feed_vars, fetch_vars = get_program( + ( + train_program, + start_program, + dataloader, + loss, + optimizer, + feed_vars, + fetch_vars, + ) = get_program() + dist_context = DistributedContext( + train_program, start_program, optimizer, loss, feed_vars, fetch_vars ) - dist_context = DistributedContext(train_program, start_program, - optimizer, loss, feed_vars, - fetch_vars) dist_context.initialize() copy_dist_context = copy.deepcopy(dist_context) copy_list = [ - "_original_serial_main_program", "_original_serial_startup_program", \ - "_serial_main_program", "_serial_startup_program", "_serial_graph", \ - "_dist_main_programs", "_dist_startup_programs", \ - "_serial_ordered_nodes", "_serial_ordered_tensor_nodes", \ - "_serial_ordered_op_nodes", "_original_serial_loss", \ - "_original_serial_feed_vars", "_original_serial_fetch_vars", \ - "_serial_loss", "_serial_feed_vars", "_serial_fetch_vars", "_serial_optimizer", \ - "_backup_serial_main_program_stack", "_backup_serial_startup_program_stack", \ - "_pass_context"] + "_original_serial_main_program", + "_original_serial_startup_program", + "_serial_main_program", + "_serial_startup_program", + "_serial_graph", + "_dist_main_programs", + "_dist_startup_programs", + "_serial_ordered_nodes", + "_serial_ordered_tensor_nodes", + "_serial_ordered_op_nodes", + "_original_serial_loss", + "_original_serial_feed_vars", + "_original_serial_fetch_vars", + "_serial_loss", + "_serial_feed_vars", + "_serial_fetch_vars", + "_serial_optimizer", + "_backup_serial_main_program_stack", + "_backup_serial_startup_program_stack", + "_pass_context", + ] for i in range(len(copy_list)): copy_obj = "copy_dist_context." + copy_list[i] diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_embedding.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_embedding.py index 034a2cf189235..c302cb7e8179d 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_embedding.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_embedding.py @@ -27,36 +27,41 @@ def make_program_lookup_table_v1_mp_dp(): block = main_program.global_block() with paddle.static.program_guard(main_program, start_program): - src_ids = paddle.static.data(name='src_ids', - shape=[12, 512, 1], - dtype='int64') + src_ids = paddle.static.data( + name='src_ids', shape=[12, 512, 1], dtype='int64' + ) src_ids.stop_gradient = True emb_out = paddle.fluid.layers.embedding( input=src_ids, size=[64, 128], param_attr=paddle.fluid.ParamAttr(name="emb_weight"), dtype="float32", - is_sparse=False) + is_sparse=False, + ) loss = paddle.fluid.layers.reduce_mean(emb_out) auto.shard_tensor( - src_ids, auto.ProcessMesh([[0, 1], [2, 3]], dim_names=["x", "y"]), - ["x", None, None]) + src_ids, + auto.ProcessMesh([[0, 1], [2, 3]], dim_names=["x", "y"]), + ["x", None, None], + ) emb_weight = block.vars["emb_weight"] auto.shard_tensor( - emb_weight, auto.ProcessMesh([[0, 1], [2, 3]], - dim_names=["x", "y"]), ["y", None]) + emb_weight, + auto.ProcessMesh([[0, 1], [2, 3]], dim_names=["x", "y"]), + ["y", None], + ) return main_program, start_program, loss class TestDistPNorm(unittest.TestCase): - def test_lookup_table_v1_mp_dp(self): for rank in range(4): dist_main_prog, dist_context = parallelizer( - make_program_lookup_table_v1_mp_dp, rank) + make_program_lookup_table_v1_mp_dp, rank + ) ops = dist_main_prog.global_block().ops op_types = [] @@ -64,9 +69,16 @@ def test_lookup_table_v1_mp_dp(self): op_types.append(op.type) assert op_types == [ - 'reshape2', 'c_embedding', 'c_allreduce_sum', 'reduce_mean', - 'fill_constant', 'reduce_mean_grad', 'c_identity', - 'c_embedding_grad', 'c_allreduce_sum', 'scale' + 'reshape2', + 'c_embedding', + 'c_allreduce_sum', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'c_identity', + 'c_embedding_grad', + 'c_allreduce_sum', + 'scale', ] diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_matmul.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_matmul.py index 6ca0619800e93..c53113ae76cf8 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_matmul.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_matmul.py @@ -84,10 +84,9 @@ def matmul_dp2mp2(init_x, init_y, trans_x, trans_y): y = init_y(trans_y) x.stop_gradient = False y.stop_gradient = False - out = paddle.fluid.layers.matmul(x, - y, - transpose_x=trans_x, - transpose_y=trans_y) + out = paddle.fluid.layers.matmul( + x, y, transpose_x=trans_x, transpose_y=trans_y + ) loss = paddle.mean(out) return main_program, start_program, loss @@ -123,19 +122,23 @@ def parallelizer(program_func, *args, **kwargs): dist_context.block_state.parse_backward_blocks(main_program) partitioner = Partitioner(dist_context, 0) - dist_main_prog, _, _ = partitioner.partition(main_program, start_program, - []) + dist_main_prog, _, _ = partitioner.partition( + main_program, start_program, [] + ) return dist_main_prog, dist_context class TestDistMatmul(unittest.TestCase): - def check_col_program(self, main_program, dist_ctx): # [0, -1] * [-1, 1] --> [0, 1] ref_ops = [ - "c_identity", "matmul", "reduce_mean", "fill_constant", - "reduce_mean_grad", "matmul_grad" + "c_identity", + "matmul", + "reduce_mean", + "fill_constant", + "reduce_mean_grad", + "matmul_grad", ] ops = [] block = main_program.global_block() @@ -148,10 +151,12 @@ def check_col_program(self, main_program, dist_ctx): assert op_dist_attr.impl_idx == 0 assert op_dist_attr.impl_type == "matmul" out_dims_mapping = op_dist_attr.get_output_dims_mapping( - out_name) + out_name + ) assert out_dims_mapping == [0, 1] tensor_dist_attr = dist_ctx.get_tensor_dist_attr_for_program( - out_var) + out_var + ) assert tensor_dist_attr.dims_mapping == [0, 1] if op.type == "matmul_grad": op_dist_attr = dist_ctx.get_op_dist_attr_for_program(op) @@ -163,8 +168,12 @@ def check_col_program(self, main_program, dist_ctx): def check_row_program(self, main_program, dist_ctx): # [0, -1, 1] * [1, -1] --> [0, -1, -1] ref_ops = [ - "matmul", "c_allreduce_sum", "reduce_mean", "fill_constant", - "reduce_mean_grad", "matmul_grad" + "matmul", + "c_allreduce_sum", + "reduce_mean", + "fill_constant", + "reduce_mean_grad", + "matmul_grad", ] ops = [] block = main_program.global_block() @@ -177,10 +186,12 @@ def check_row_program(self, main_program, dist_ctx): assert op_dist_attr.impl_idx == 1 assert op_dist_attr.impl_type == "matmul" out_dims_mapping = op_dist_attr.get_output_dims_mapping( - out_name) + out_name + ) assert out_dims_mapping == [0, -1, -1] tensor_dist_attr = dist_ctx.get_tensor_dist_attr_for_program( - out_var) + out_var + ) assert tensor_dist_attr.dims_mapping == [0, -1, -1] if op.type == "matmul_grad": op_dist_attr = dist_ctx.get_op_dist_attr_for_program(op) @@ -190,10 +201,10 @@ def check_row_program(self, main_program, dist_ctx): class TestDistMatmulCol(TestDistMatmul): - def init(self, trans_x, trans_y): - dist_main_prog, dist_ctx = parallelizer(matmul_dp2mp2, init_x_col, - init_y_col, trans_x, trans_y) + dist_main_prog, dist_ctx = parallelizer( + matmul_dp2mp2, init_x_col, init_y_col, trans_x, trans_y + ) return dist_main_prog, dist_ctx def test_matmul_col(self): @@ -214,10 +225,10 @@ def test_trans_x_trans_y(self): class TestDistMatmulRow(TestDistMatmul): - def init(self, trans_x, trans_y): - dist_main_prog, dist_ctx = parallelizer(matmul_dp2mp2, init_x_row, - init_y_row, trans_x, trans_y) + dist_main_prog, dist_ctx = parallelizer( + matmul_dp2mp2, init_x_row, init_y_row, trans_x, trans_y + ) return dist_main_prog, dist_ctx def test_matmul_row(self): @@ -238,12 +249,15 @@ def test_trans_x_trans_y(self): class TestDistMatmulV2(unittest.TestCase): - def check_col_program(self, main_program, dist_ctx): # [0, -1] * [-1, 1] --> [0, 1] ref_ops = [ - "c_identity", "matmul_v2", "reduce_mean", "fill_constant", - "reduce_mean_grad", "matmul_v2_grad" + "c_identity", + "matmul_v2", + "reduce_mean", + "fill_constant", + "reduce_mean_grad", + "matmul_v2_grad", ] ops = [] block = main_program.global_block() @@ -256,10 +270,12 @@ def check_col_program(self, main_program, dist_ctx): assert op_dist_attr.impl_idx == 0 assert op_dist_attr.impl_type == "matmul_v2" out_dims_mapping = op_dist_attr.get_output_dims_mapping( - out_name) + out_name + ) assert out_dims_mapping == [0, 1] tensor_dist_attr = dist_ctx.get_tensor_dist_attr_for_program( - out_var) + out_var + ) assert tensor_dist_attr.dims_mapping == [0, 1] if op.type == "matmul_v2_grad": op_dist_attr = dist_ctx.get_op_dist_attr_for_program(op) @@ -271,8 +287,12 @@ def check_col_program(self, main_program, dist_ctx): def check_row_program(self, main_program, dist_ctx): # [0, -1, 1] * [1, -1] --> [0, -1, -1] ref_ops = [ - "matmul_v2", "c_allreduce_sum", "reduce_mean", "fill_constant", - "reduce_mean_grad", "matmul_v2_grad" + "matmul_v2", + "c_allreduce_sum", + "reduce_mean", + "fill_constant", + "reduce_mean_grad", + "matmul_v2_grad", ] ops = [] block = main_program.global_block() @@ -285,10 +305,12 @@ def check_row_program(self, main_program, dist_ctx): assert op_dist_attr.impl_idx == 1 assert op_dist_attr.impl_type == "matmul_v2" out_dims_mapping = op_dist_attr.get_output_dims_mapping( - out_name) + out_name + ) assert out_dims_mapping == [0, -1, -1] tensor_dist_attr = dist_ctx.get_tensor_dist_attr_for_program( - out_var) + out_var + ) assert tensor_dist_attr.dims_mapping == [0, -1, -1] if op.type == "matmul_v2_grad": op_dist_attr = dist_ctx.get_op_dist_attr_for_program(op) @@ -298,10 +320,10 @@ def check_row_program(self, main_program, dist_ctx): class TestDistMatmulV2Col(TestDistMatmulV2): - def init(self, trans_x, trans_y): - dist_main_prog, dist_ctx = parallelizer(matmulv2_dp2mp2, init_x_col, - init_y_col, trans_x, trans_y) + dist_main_prog, dist_ctx = parallelizer( + matmulv2_dp2mp2, init_x_col, init_y_col, trans_x, trans_y + ) return dist_main_prog, dist_ctx def test_matmul_col(self): @@ -322,10 +344,10 @@ def test_trans_x_trans_y(self): class TestDistMatmulV2Row(TestDistMatmulV2): - def init(self, trans_x, trans_y): - dist_main_prog, dist_ctx = parallelizer(matmulv2_dp2mp2, init_x_row, - init_y_row, trans_x, trans_y) + dist_main_prog, dist_ctx = parallelizer( + matmulv2_dp2mp2, init_x_row, init_y_row, trans_x, trans_y + ) return dist_main_prog, dist_ctx def test_matmul_row(self): diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_op_cost.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_op_cost.py index d7bdbfd777473..517debaa58842 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_op_cost.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_op_cost.py @@ -18,7 +18,10 @@ import paddle from paddle.distributed.fleet import auto from paddle.distributed.auto_parallel.cluster import Cluster -from paddle.distributed.auto_parallel.operators.common import get_distributed_operator_impl_container, is_elementwise_op +from paddle.distributed.auto_parallel.operators.common import ( + get_distributed_operator_impl_container, + is_elementwise_op, +) from paddle.fluid import program_guard from paddle.fluid.backward import append_backward @@ -42,7 +45,8 @@ def parallelizer(program_func, rank): # generate backward and complete backward with paddle.static.program_guard(main_program, startup_program): params_grads = append_backward( - loss, None, None, None, distop_context=dist_context.dist_op_context) + loss, None, None, None, distop_context=dist_context.dist_op_context + ) completer.complete_backward_annotation(main_program) dist_context.block_state.parse_backward_blocks(main_program) @@ -57,23 +61,23 @@ def parallelizer(program_func, rank): class TestDistOpCost(unittest.TestCase): - def test_dist_op_cost_part1(self): - def make_program(): main_program = paddle.static.Program() start_program = paddle.static.Program() with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 8], dtype='float32') x.stop_gradient = True - label = paddle.static.data(name="label", - shape=[4, 1], - dtype='float32') + label = paddle.static.data( + name="label", shape=[4, 1], dtype='float32' + ) label.stop_gradient = True - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x", None] + ) tmp = paddle.fluid.layers.fill_constant_batch_size_like( - input=x, shape=[2, 8], value=1, dtype='float32') + input=x, shape=[2, 8], value=1, dtype='float32' + ) weight_attr = paddle.ParamAttr() linear = paddle.nn.Linear(8, 1, weight_attr=weight_attr) linear_out = linear(x) @@ -89,43 +93,53 @@ def make_program(): cluster = Cluster() cluster.gen_default_config_cluster(device_count=2) for idx, op in enumerate(ops): - if op.type != "matmul_v2" and op.type != "matmul_v2_grad" and op.type != "sgd": + if ( + op.type != "matmul_v2" + and op.type != "matmul_v2_grad" + and op.type != "sgd" + ): dist_op = dist_context.get_dist_op_for_program(op) op_dist_attr = dist_op.dist_attr processes = op_dist_attr.process_mesh.processes if is_elementwise_op(op.type): container = get_distributed_operator_impl_container( - "elementwise") + "elementwise" + ) else: container = get_distributed_operator_impl_container( - op_dist_attr.impl_type) + op_dist_attr.impl_type + ) dist_impl = container.impls[op_dist_attr.impl_idx] - dist_op_cost = dist_impl.calc_cost(op.attr('op_role'), dist_op, - dist_context, cluster) + dist_op_cost = dist_impl.calc_cost( + op.attr('op_role'), dist_op, dist_context, cluster + ) self.assertTrue(dist_op_cost) def test_dist_op_cost_part2(self): - def make_program(): main_program = paddle.static.Program() start_program = paddle.static.Program() with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4], dtype='float32') x.stop_gradient = True - label = paddle.static.data(name="label", - shape=[8, 1], - dtype='float32') + label = paddle.static.data( + name="label", shape=[8, 1], dtype='float32' + ) label.stop_gradient = True - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x"]) - - auto.shard_tensor(label, - auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x"] + ) + + auto.shard_tensor( + label, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) # embedding tmp = paddle.fluid.layers.fill_constant_batch_size_like( - input=x, shape=[4], value=1, dtype='int32') + input=x, shape=[4], value=1, dtype='int32' + ) embedding = paddle.nn.Embedding(10, 8) out = embedding(tmp) # row parallel embedding @@ -133,47 +147,64 @@ def make_program(): if op.type == "lookup_table_v2": W = main_program.global_block().vars[op.input("W")[0]] auto.shard_tensor( - W, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) - out = paddle.fluid.layers.transpose(out, - [1, 0]) # [8, 2] [-1, 0] + W, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) + out = paddle.fluid.layers.transpose( + out, [1, 0] + ) # [8, 2] [-1, 0] # matmul param1 = paddle.fluid.layers.create_parameter( - [4, 8], paddle.float32) # [2, 8] [0, -1] - auto.shard_tensor(param1, - auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) + [4, 8], paddle.float32 + ) # [2, 8] [0, -1] + auto.shard_tensor( + param1, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) param2 = paddle.fluid.layers.create_parameter( - [8, 8], paddle.float32) # [8, 4] [-1, 0] - auto.shard_tensor(param2, - auto.ProcessMesh([0, 1], dim_names=["x"]), - [None, "x"]) - out1 = paddle.fluid.layers.matmul(out, - param1) # [8, 8] [-1, -1] + [8, 8], paddle.float32 + ) # [8, 4] [-1, 0] + auto.shard_tensor( + param2, + auto.ProcessMesh([0, 1], dim_names=["x"]), + [None, "x"], + ) + out1 = paddle.fluid.layers.matmul( + out, param1 + ) # [8, 8] [-1, -1] tmp_param = paddle.fluid.layers.create_parameter( - [8, 8], paddle.float32) # [8, 8] [-1, -1] - auto.shard_tensor(param2, - auto.ProcessMesh([0, 1], dim_names=["x"]), - [None, None]) + [8, 8], paddle.float32 + ) # [8, 8] [-1, -1] + auto.shard_tensor( + param2, + auto.ProcessMesh([0, 1], dim_names=["x"]), + [None, None], + ) tmp_out = paddle.fluid.layers.matmul(out1, tmp_param) - out2 = paddle.fluid.layers.matmul(tmp_out, - param2) # [8, 4] [-1, 0] + out2 = paddle.fluid.layers.matmul( + tmp_out, param2 + ) # [8, 4] [-1, 0] - out8 = paddle.fluid.layers.transpose(out2, - [1, 0]) # [4, 8] [0, -1] + out8 = paddle.fluid.layers.transpose( + out2, [1, 0] + ) # [4, 8] [0, -1] # reshape out9 = paddle.reshape(out8, [8, 2, 4]) # [4, 2, 4] [0, -1, -1] tmp_reshape_out = paddle.reshape(out9, [8, 4, 2]) - out10 = paddle.reshape(tmp_reshape_out, - [8, 8]) # [4, 8] [0, -1] + out10 = paddle.reshape( + tmp_reshape_out, [8, 8] + ) # [4, 8] [0, -1] # softmax softmax = paddle.nn.Softmax() out11 = softmax(out10) error_cost = paddle.nn.functional.square_error_cost( - out11, label) + out11, label + ) loss = paddle.mean(error_cost) return main_program, start_program, loss @@ -187,37 +218,43 @@ def make_program(): processes = op_dist_attr.process_mesh.processes if is_elementwise_op(op.type): container = get_distributed_operator_impl_container( - "elementwise") + "elementwise" + ) else: container = get_distributed_operator_impl_container( - op_dist_attr.impl_type) + op_dist_attr.impl_type + ) dist_impl = container.impls[op_dist_attr.impl_idx] - dist_op_cost = dist_impl.calc_cost(op.attr('op_role'), dist_op, - dist_context, cluster) + dist_op_cost = dist_impl.calc_cost( + op.attr('op_role'), dist_op, dist_context, cluster + ) self.assertTrue(dist_op_cost) def test_dist_op_cost_part3(self): - def make_program(): main_program = paddle.static.Program() start_program = paddle.static.Program() with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4], dtype='float32') x.stop_gradient = True - label = paddle.static.data(name="label", - shape=[8, 1], - dtype='float32') + label = paddle.static.data( + name="label", shape=[8, 1], dtype='float32' + ) label.stop_gradient = True - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x"]) - - auto.shard_tensor(label, - auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x"] + ) + + auto.shard_tensor( + label, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) # embedding tmp = paddle.fluid.layers.fill_constant_batch_size_like( - input=x, shape=[4], value=1, dtype='int32') + input=x, shape=[4], value=1, dtype='int32' + ) embedding = paddle.nn.Embedding(10, 8) out = embedding(tmp) # row parallel embedding @@ -225,46 +262,61 @@ def make_program(): if op.type == "lookup_table_v2": W = main_program.global_block().vars[op.input("W")[0]] auto.shard_tensor( - W, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) - out = paddle.fluid.layers.transpose(out, - [1, 0]) # [8, 2] [-1, 0] + W, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) + out = paddle.fluid.layers.transpose( + out, [1, 0] + ) # [8, 2] [-1, 0] # matmul_v2 param1 = paddle.fluid.layers.create_parameter( - [4, 8], paddle.float32) # [2, 8] [0, -1] - auto.shard_tensor(param1, - auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) + [4, 8], paddle.float32 + ) # [2, 8] [0, -1] + auto.shard_tensor( + param1, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) param2 = paddle.fluid.layers.create_parameter( - [8, 8], paddle.float32) # [8, 4] [-1, 0] - auto.shard_tensor(param2, - auto.ProcessMesh([0, 1], dim_names=["x"]), - [None, "x"]) + [8, 8], paddle.float32 + ) # [8, 4] [-1, 0] + auto.shard_tensor( + param2, + auto.ProcessMesh([0, 1], dim_names=["x"]), + [None, "x"], + ) out1 = paddle.matmul(out, param1) # [8, 8] [-1, -1] tmp_param = paddle.fluid.layers.create_parameter( - [8, 8], paddle.float32) # [8, 8] [-1, -1] - auto.shard_tensor(param2, - auto.ProcessMesh([0, 1], dim_names=["x"]), - [None, None]) + [8, 8], paddle.float32 + ) # [8, 8] [-1, -1] + auto.shard_tensor( + param2, + auto.ProcessMesh([0, 1], dim_names=["x"]), + [None, None], + ) tmp_out = paddle.matmul(out1, tmp_param) out2 = paddle.matmul(tmp_out, param2) # [8, 4] [-1, 0] - out8 = paddle.fluid.layers.transpose(out2, - [1, 0]) # [4, 8] [0, -1] + out8 = paddle.fluid.layers.transpose( + out2, [1, 0] + ) # [4, 8] [0, -1] # reshape out9 = paddle.reshape(out8, [8, 2, 4]) # [4, 2, 4] [0, -1, -1] tmp_reshape_out = paddle.reshape(out9, [8, 4, 2]) - out10 = paddle.reshape(tmp_reshape_out, - [8, 8]) # [4, 8] [0, -1] + out10 = paddle.reshape( + tmp_reshape_out, [8, 8] + ) # [4, 8] [0, -1] # softmax softmax = paddle.nn.Softmax() out11 = softmax(out10) error_cost = paddle.nn.functional.square_error_cost( - out11, label) + out11, label + ) loss = paddle.mean(error_cost) return main_program, start_program, loss @@ -278,36 +330,42 @@ def make_program(): processes = op_dist_attr.process_mesh.processes if is_elementwise_op(op.type): container = get_distributed_operator_impl_container( - "elementwise") + "elementwise" + ) else: container = get_distributed_operator_impl_container( - op_dist_attr.impl_type) + op_dist_attr.impl_type + ) dist_impl = container.impls[op_dist_attr.impl_idx] - dist_op_cost = dist_impl.calc_cost(op.attr('op_role'), dist_op, - dist_context, cluster) + dist_op_cost = dist_impl.calc_cost( + op.attr('op_role'), dist_op, dist_context, cluster + ) self.assertTrue(dist_op_cost) def test_dist_op_cost_part4(self): - def make_program(): main_program = paddle.static.Program() start_program = paddle.static.Program() with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4], dtype='float32') x.stop_gradient = True - label = paddle.static.data(name="label", - shape=[8, 1], - dtype='float32') + label = paddle.static.data( + name="label", shape=[8, 1], dtype='float32' + ) label.stop_gradient = True - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x"]) - auto.shard_tensor(label, - auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x"] + ) + auto.shard_tensor( + label, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) # embedding tmp = paddle.fluid.layers.fill_constant_batch_size_like( - input=x, shape=[4], value=1, dtype='int32') + input=x, shape=[4], value=1, dtype='int32' + ) embedding = paddle.nn.Embedding(10, 8) out = embedding(tmp) # row parallel embedding @@ -315,48 +373,64 @@ def make_program(): if op.type == "lookup_table_v2": W = main_program.global_block().vars[op.input("W")[0]] auto.shard_tensor( - W, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) - out = paddle.fluid.layers.transpose(out, - [1, 0]) # [8, 2] [-1, 0] + W, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) + out = paddle.fluid.layers.transpose( + out, [1, 0] + ) # [8, 2] [-1, 0] # mul param1 = paddle.fluid.layers.create_parameter( - [4, 8], paddle.float32) # [2, 8] [0, -1] - auto.shard_tensor(param1, - auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None]) + [4, 8], paddle.float32 + ) # [2, 8] [0, -1] + auto.shard_tensor( + param1, + auto.ProcessMesh([0, 1], dim_names=["x"]), + ["x", None], + ) param2 = paddle.fluid.layers.create_parameter( - [8, 8], paddle.float32) # [8, 4] [-1, 0] - auto.shard_tensor(param2, - auto.ProcessMesh([0, 1], dim_names=["x"]), - [None, "x"]) + [8, 8], paddle.float32 + ) # [8, 4] [-1, 0] + auto.shard_tensor( + param2, + auto.ProcessMesh([0, 1], dim_names=["x"]), + [None, "x"], + ) out1 = paddle.fluid.layers.mul(out, param1) # [8, 8] [-1, -1] tmp_param = paddle.fluid.layers.create_parameter( - [8, 8], paddle.float32) # [8, 8] [-1, -1] - auto.shard_tensor(param2, - auto.ProcessMesh([0, 1], dim_names=["x"]), - [None, None]) + [8, 8], paddle.float32 + ) # [8, 8] [-1, -1] + auto.shard_tensor( + param2, + auto.ProcessMesh([0, 1], dim_names=["x"]), + [None, None], + ) tmp_out = paddle.fluid.layers.mul(out1, tmp_param) - out2 = paddle.fluid.layers.mul(tmp_out, - param2) # [8, 4] [-1, 0] + out2 = paddle.fluid.layers.mul( + tmp_out, param2 + ) # [8, 4] [-1, 0] - out8 = paddle.fluid.layers.transpose(out2, - [1, 0]) # [4, 8] [0, -1] + out8 = paddle.fluid.layers.transpose( + out2, [1, 0] + ) # [4, 8] [0, -1] # reshape out9 = paddle.reshape(out8, [8, 2, 4]) # [4, 2, 4] [0, -1, -1] tmp_reshape_out = paddle.reshape(out9, [8, 4, 2]) - out10 = paddle.reshape(tmp_reshape_out, - [8, 8]) # [4, 8] [0, -1] + out10 = paddle.reshape( + tmp_reshape_out, [8, 8] + ) # [4, 8] [0, -1] # softmax softmax = paddle.nn.Softmax() out11 = softmax(out10) error_cost = paddle.nn.functional.square_error_cost( - out11, label) + out11, label + ) loss = paddle.mean(error_cost) return main_program, start_program, loss @@ -370,14 +444,17 @@ def make_program(): processes = op_dist_attr.process_mesh.processes if is_elementwise_op(op.type): container = get_distributed_operator_impl_container( - "elementwise") + "elementwise" + ) else: container = get_distributed_operator_impl_container( - op_dist_attr.impl_type) + op_dist_attr.impl_type + ) dist_impl = container.impls[op_dist_attr.impl_idx] - dist_op_cost = dist_impl.calc_cost(op.attr('op_role'), dist_op, - dist_context, cluster) + dist_op_cost = dist_impl.calc_cost( + op.attr('op_role'), dist_op, dist_context, cluster + ) self.assertTrue(dist_op_cost) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_pnorm.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_pnorm.py index 9f54f2a774afa..66ce40e4e33cb 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_pnorm.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_pnorm.py @@ -28,8 +28,9 @@ def make_program_dp2(): with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 5, 6], dtype='float32') x.stop_gradient = False - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None, None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x", None, None] + ) tmp_0 = paddle.norm(x, p=2) return main_program, start_program, tmp_0 @@ -40,8 +41,9 @@ def make_program_serial(): with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 5, 6], dtype='float32') x.stop_gradient = False - auto.shard_tensor(x, auto.ProcessMesh([0], dim_names=["x"]), - [None, None, None]) + auto.shard_tensor( + x, auto.ProcessMesh([0], dim_names=["x"]), [None, None, None] + ) tmp_0 = paddle.norm(x, p=2) return main_program, start_program, tmp_0 @@ -60,19 +62,20 @@ def parallelizer(program_func, rank): with program_guard(main_program, start_program): params_grads = append_backward( - loss, distop_context=dist_context.dist_op_context) + loss, distop_context=dist_context.dist_op_context + ) completer.complete_backward_annotation(main_program) dist_context.block_state.parse_backward_blocks(main_program) partitioner = Partitioner(dist_context, rank) - dist_main_prog, _, _ = partitioner.partition(main_program, start_program, - []) + dist_main_prog, _, _ = partitioner.partition( + main_program, start_program, [] + ) return dist_main_prog, dist_context class TestDistPNorm(unittest.TestCase): - def test_dist_pnorm_dp2(self): for rank in range(2): @@ -102,7 +105,11 @@ def test_dist_pnorm_dp2(self): assert output_attr.dims_mapping[0] == 0 assert set(output_attr.dims_mapping[1:]) == set([-1]) assert op_types == [ - "c_allgather", "p_norm", "fill_constant", "p_norm_grad", "slice" + "c_allgather", + "p_norm", + "fill_constant", + "p_norm_grad", + "slice", ] def test_dist_pnorm_serial(self): diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_reshape.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_reshape.py index 02797d690f23c..035fd23562440 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_reshape.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_reshape.py @@ -25,8 +25,9 @@ def make_program_dp2(): with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 4, 8], dtype='float32') x.stop_gradient = False - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None, None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x", None, None] + ) tmp_0 = paddle.reshape(x, shape=[0, 0, 4, 2]) tmp_1 = paddle.reshape(tmp_0, shape=[0, 0, 8]) @@ -47,14 +48,14 @@ def parallelizer(program_func, rank): dist_context.block_state.parse_forward_blocks(main_program) partitioner = Partitioner(dist_context, rank) - dist_main_prog, _, _ = partitioner.partition(main_program, start_program, - []) + dist_main_prog, _, _ = partitioner.partition( + main_program, start_program, [] + ) return dist_main_prog, dist_context class TestDistReshape(unittest.TestCase): - def test_dist_reshape_mp2(self): for rank in range(2): diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_shape.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_shape.py index 5e18b7d90c519..767a4cbccb22f 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_shape.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_shape.py @@ -25,8 +25,9 @@ def make_program(): with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 4, 8], dtype='float32') x.stop_gradient = False - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None, None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x", None, None] + ) shape = paddle.shape(x) return main_program, start_program @@ -44,14 +45,14 @@ def parallelizer(program_func, rank): dist_context.block_state.parse_forward_blocks(main_program) partitioner = Partitioner(dist_context, rank) - dist_main_prog, _, _ = partitioner.partition(main_program, start_program, - []) + dist_main_prog, _, _ = partitioner.partition( + main_program, start_program, [] + ) return dist_main_prog, dist_context class TestDistShape(unittest.TestCase): - def test_dist_shape(self): dist_main_prog, dist_context = parallelizer(make_program, 0) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_slice.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_slice.py index 89261e6c9bf82..7e4d5eaee90dd 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_slice.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_slice.py @@ -24,8 +24,9 @@ def make_program_dp2(): start_program = paddle.fluid.Program() with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 5, 6], dtype='float32') - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None, None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x", None, None] + ) tmp_0 = x[0] tmp_1 = x[:, 0, :] @@ -39,8 +40,9 @@ def make_program_serial(): start_program = paddle.fluid.Program() with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 5, 6], dtype='float32') - auto.shard_tensor(x, auto.ProcessMesh([0], dim_names=["x"]), - [None, None, None]) + auto.shard_tensor( + x, auto.ProcessMesh([0], dim_names=["x"]), [None, None, None] + ) tmp_0 = x[0] tmp_1 = x[:, 0, :] @@ -64,14 +66,14 @@ def parallelizer(program_func, rank): dist_context.block_state.parse_forward_blocks(main_program) partitioner = Partitioner(dist_context, rank) - dist_main_prog, _, _ = partitioner.partition(main_program, start_program, - []) + dist_main_prog, _, _ = partitioner.partition( + main_program, start_program, [] + ) return dist_main_prog, dist_context class TestDistSlice(unittest.TestCase): - def test_dist_slice_dp2(self): for rank in range(2): dist_main_prog, dist_context = parallelizer(make_program_dp2, rank) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_split.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_split.py index 7d7c60f4bdd48..18e79b8cf95d0 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_split.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_dist_split.py @@ -25,8 +25,9 @@ def make_program_dp2(): with paddle.static.program_guard(main_program, start_program): x = paddle.static.data(name='x', shape=[4, 12, 16], dtype='float32') x.stop_gradient = False - auto.shard_tensor(x, auto.ProcessMesh([0, 1], dim_names=["x"]), - ["x", None, None]) + auto.shard_tensor( + x, auto.ProcessMesh([0, 1], dim_names=["x"]), ["x", None, None] + ) out0, out1, out2 = paddle.split(x, num_or_sections=3, axis=1) return main_program, start_program @@ -44,14 +45,14 @@ def parallelizer(program_func, rank): dist_context.block_state.parse_forward_blocks(main_program) partitioner = Partitioner(dist_context, rank) - dist_main_prog, _, _ = partitioner.partition(main_program, start_program, - []) + dist_main_prog, _, _ = partitioner.partition( + main_program, start_program, [] + ) return dist_main_prog, dist_context class TestDistSplit(unittest.TestCase): - def test_dist_split_dp2(self): for rank in range(2): diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api.py index c8aba02719cbe..880f0e582bb9f 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api.py @@ -20,7 +20,6 @@ class TestEngineAPI(unittest.TestCase): - def test_engine_api(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "engine_api.py") @@ -31,10 +30,19 @@ def test_engine_api(self): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api_dp.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api_dp.py index fab8db0182ea7..b3dad0551c273 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api_dp.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_api_dp.py @@ -20,7 +20,6 @@ class TestEngineAPI(unittest.TestCase): - def test_engine_api(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "engine_api_dp.py") @@ -31,10 +30,19 @@ def test_engine_api(self): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_callbacks.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_callbacks.py index 9baaee353f715..5c7b2f1d1777c 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_callbacks.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_engine_callbacks.py @@ -31,7 +31,6 @@ class TestCallbacks(unittest.TestCase): - def setUp(self): self.save_dir = tempfile.mkdtemp() @@ -51,14 +50,16 @@ def run_callback(self): engine = auto.Engine(LeNet(), strategy=strategy) engine.prepare(inputs_spec, mode="predict") - cbks = config_callbacks(engine=engine, - batch_size=128, - epochs=epochs, - steps=steps, - log_freq=freq, - verbose=self.verbose, - metrics=['loss', 'acc'], - save_dir=self.save_dir) + cbks = config_callbacks( + engine=engine, + batch_size=128, + epochs=epochs, + steps=steps, + log_freq=freq, + verbose=self.verbose, + metrics=['loss', 'acc'], + save_dir=self.save_dir, + ) cbks.on_begin('train') logs = {'loss': 50.341673, 'acc': 0.00256} @@ -119,7 +120,6 @@ def test_callback_verbose_3(self): class TestCallbacksEngine(unittest.TestCase): - def setUp(self): self.save_dir = tempfile.mkdtemp() transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])]) @@ -136,37 +136,39 @@ def prepare_engine(self): base_lr = 1e-3 boundaries = [5, 8] values = [base_lr * (0.1**i) for i in range(len(boundaries) + 1)] - lr = paddle.optimizer.lr.PiecewiseDecay(boundaries=boundaries, - values=values, - verbose=False) - optimizer = paddle.optimizer.Adam(learning_rate=lr, - parameters=model.parameters()) + lr = paddle.optimizer.lr.PiecewiseDecay( + boundaries=boundaries, values=values, verbose=False + ) + optimizer = paddle.optimizer.Adam( + learning_rate=lr, parameters=model.parameters() + ) auto.fetch(model.parameters()[0], "param0", logging=True) metrics = paddle.metric.Accuracy(topk=(1, 2)) self.engine = auto.Engine(model, loss, optimizer, metrics) def test_fit_eval(self): - history = self.engine.fit(train_data=self.train_dataset, - valid_data=self.test_dataset, - batch_size=128, - steps_per_epoch=60, - valid_steps=40, - log_freq=20, - save_dir=self.save_dir, - save_freq=1) + history = self.engine.fit( + train_data=self.train_dataset, + valid_data=self.test_dataset, + batch_size=128, + steps_per_epoch=60, + valid_steps=40, + log_freq=20, + save_dir=self.save_dir, + save_freq=1, + ) print(history.history) def test_eval(self): - self.engine.evaluate(valid_data=self.test_dataset, - batch_size=128, - steps=40, - log_freq=10) + self.engine.evaluate( + valid_data=self.test_dataset, batch_size=128, steps=40, log_freq=10 + ) def test_predict(self): logger_cbks = paddle.callbacks.ProgBarLogger() - self.engine.predict(test_data=self.test_dataset, - batch_size=128, - callbacks=[logger_cbks]) + self.engine.predict( + test_data=self.test_dataset, batch_size=128, callbacks=[logger_cbks] + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_high_order_grad.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_high_order_grad.py index 7c3629182faae..7e2728e8e97b6 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_high_order_grad.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_high_order_grad.py @@ -20,7 +20,6 @@ class TestHighOrderGrad(unittest.TestCase): - def test_dp2(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "high_order_grad.py") @@ -31,10 +30,19 @@ def test_dp2(self): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_interface.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_interface.py index a36c142dc863a..332cb134a542c 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_interface.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_interface.py @@ -18,7 +18,9 @@ import paddle.nn.functional as F import paddle.static as static from paddle.distributed.fleet import auto -from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, +) from paddle.distributed.auto_parallel.process_mesh import ProcessMesh paddle.enable_static() @@ -27,71 +29,83 @@ epoch_num = 10 hidden_size = 1024 sequence_len = 512 -process_mesh1 = ProcessMesh(mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], - dim_names=["x", "y"]) +process_mesh1 = ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["x", "y"] +) process_mesh2 = ProcessMesh(mesh=[0, 1, 2, 3], dim_names=["x"]) class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size - param_initializer = nn.initializer.Normal(mean=0.0, - std=initializer_range) + param_initializer = nn.initializer.Normal( + mean=0.0, std=initializer_range + ) self.linear0 = nn.Linear( d_model, dim_feedforward, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) self.linear1 = nn.Linear( dim_feedforward, d_model, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) def forward(self, input): auto.shard_tensor(self.linear0.weight, process_mesh1[0], [None, "y"]) - linear0 = auto.shard_op(self.linear0, process_mesh1, - [["y", None, None]], [[None, "x", None]]) + linear0 = auto.shard_op( + self.linear0, + process_mesh1, + [["y", None, None]], + [[None, "x", None]], + ) linear0_out = linear0(input) gelu = auto.shard_op(F.gelu, process_mesh1, [["y", "x", None], None]) gelu_out = gelu(linear0_out, approximate=True) auto.shard_tensor(self.linear1.weight, shard_spec=["y", None]) - linear1 = auto.shard_op(self.linear1, - process_mesh1[1], - out_shard_specs=[["y", None, None]]) + linear1 = auto.shard_op( + self.linear1, process_mesh1[1], out_shard_specs=[["y", None, None]] + ) linear1_out = linear1(gelu_out) return self.linear0, self.linear1, linear0_out, gelu_out, linear1_out class TestAutoParallelAPI(unittest.TestCase): - def test_api(self): # input - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, sequence_len, 1], - dtype='float32') + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + label = static.data( + name="label", shape=[batch_size, sequence_len, 1], dtype='float32' + ) auto.shard_tensor(input, process_mesh1, ["x", None, None]) auto.shard_tensor(label, process_mesh1, ["y", None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) with ProcessMesh(process_mesh1.mesh, process_mesh1.dim_names): linear0, linear1, linear0_out, gelu_out, linear1_out = mlp(input) @@ -124,32 +138,42 @@ def test_api(self): self.assertTrue(dist_input.dist_attr.is_annotated("dims_mapping")) dist_linear0_weight = default_dist_context.get_dist_tensor_for_program( - linear0.weight) - self.assertEqual(dist_linear0_weight.dist_attr.process_mesh, - process_mesh1[0]) + linear0.weight + ) + self.assertEqual( + dist_linear0_weight.dist_attr.process_mesh, process_mesh1[0] + ) self.assertEqual(dist_linear0_weight.dist_attr.dims_mapping, [-1, 0]) self.assertTrue( - dist_linear0_weight.dist_attr.is_annotated("process_mesh")) + dist_linear0_weight.dist_attr.is_annotated("process_mesh") + ) self.assertTrue( - dist_linear0_weight.dist_attr.is_annotated("dims_mapping")) + dist_linear0_weight.dist_attr.is_annotated("dims_mapping") + ) dist_linear1_weight = default_dist_context.get_dist_tensor_for_program( - linear1.weight) - self.assertEqual(dist_linear1_weight.dist_attr.process_mesh, - process_mesh1) + linear1.weight + ) + self.assertEqual( + dist_linear1_weight.dist_attr.process_mesh, process_mesh1 + ) self.assertEqual(dist_linear1_weight.dist_attr.dims_mapping, [1, -1]) self.assertTrue( - dist_linear1_weight.dist_attr.is_annotated("process_mesh")) + dist_linear1_weight.dist_attr.is_annotated("process_mesh") + ) self.assertTrue( - dist_linear1_weight.dist_attr.is_annotated("dims_mapping")) + dist_linear1_weight.dist_attr.is_annotated("dims_mapping") + ) dist_linear1_out = default_dist_context.get_dist_tensor_for_program( - linear1_out) + linear1_out + ) self.assertEqual(dist_linear1_out.dist_attr.process_mesh, process_mesh1) self.assertEqual(dist_linear1_out.dist_attr.dims_mapping, [-1, -1, -1]) self.assertTrue(dist_linear1_out.dist_attr.is_annotated("process_mesh")) self.assertFalse( - dist_linear1_out.dist_attr.is_annotated("dims_mapping")) + dist_linear1_out.dist_attr.is_annotated("dims_mapping") + ) dist_op = default_dist_context.get_dist_op_for_program(matmul0) self.assertEqual(dist_op.dist_attr.process_mesh, process_mesh1) @@ -167,7 +191,8 @@ def test_api(self): self.assertEqual(dist_op.dist_attr.impl_type, "default") self.assertEqual(dist_op.dist_attr.impl_idx, 0) tensor_dist_attr = dist_op.dist_attr.get_output_dist_attr( - linear0_out.name) + linear0_out.name + ) self.assertEqual(tensor_dist_attr.process_mesh, process_mesh1) self.assertEqual(tensor_dist_attr.dims_mapping, [-1, 0, -1]) self.assertTrue(tensor_dist_attr.is_annotated("process_mesh")) @@ -180,7 +205,8 @@ def test_api(self): self.assertEqual(dist_op.dist_attr.impl_idx, 0) self.assertTrue(dist_op.dist_attr.is_annotated("process_mesh")) tensor_dist_attr = dist_op.dist_attr.get_input_dist_attr( - linear0_out.name) + linear0_out.name + ) self.assertEqual(tensor_dist_attr.process_mesh, process_mesh1) self.assertEqual(tensor_dist_attr.dims_mapping, [1, 0, -1]) self.assertTrue(tensor_dist_attr.is_annotated("process_mesh")) @@ -208,7 +234,8 @@ def test_api(self): self.assertEqual(dist_op.dist_attr.impl_idx, 0) self.assertTrue(dist_op.dist_attr.is_annotated("process_mesh")) tensor_dist_attr = dist_op.dist_attr.get_output_dist_attr( - linear1_out.name) + linear1_out.name + ) self.assertEqual(tensor_dist_attr.process_mesh, process_mesh1[1]) self.assertEqual(tensor_dist_attr.dims_mapping, [0, -1, -1]) self.assertTrue(tensor_dist_attr.is_annotated("process_mesh")) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_iterable_dataset.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_iterable_dataset.py index 8a7d9d576024d..f431878ae20a9 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_iterable_dataset.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_iterable_dataset.py @@ -20,7 +20,6 @@ class TestEngineAPI(unittest.TestCase): - def test_engine_api(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "iterable_dataset.py") @@ -31,10 +30,19 @@ def test_engine_api(self): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_lr_grad_clip.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_lr_grad_clip.py index fc5240a369da1..49ff195a50ff4 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_lr_grad_clip.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_lr_grad_clip.py @@ -23,7 +23,6 @@ class TestEngineBase(unittest.TestCase): - def setUp(self): self.batch_size = 4 self.batch_num = 5 @@ -35,15 +34,18 @@ def setUp(self): self.init_engine() def init_model(self): - self.mlp = MLPLayer(hidden_size=self.hidden_size, - intermediate_size=4 * self.hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + self.mlp = MLPLayer( + hidden_size=self.hidden_size, + intermediate_size=4 * self.hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) self.loss = paddle.nn.CrossEntropyLoss() def init_optimizer(self): - self.optimizer = paddle.optimizer.SGD(learning_rate=0.00001, - parameters=self.mlp.parameters()) + self.optimizer = paddle.optimizer.SGD( + learning_rate=0.00001, parameters=self.mlp.parameters() + ) def init_dataset(self): self.dataset = MyDataset(self.batch_num * self.batch_size) @@ -52,17 +54,19 @@ def init_engine(self): # inputs = InputSpec([self.batch_size, self.hidden_size], 'float32', 'x') # labels = InputSpec([self.batch_size], 'int64', 'label') - self.engine = auto.Engine(model=self.mlp, - loss=self.loss, - optimizer=self.optimizer, - metrics=paddle.metric.Accuracy()) + self.engine = auto.Engine( + model=self.mlp, + loss=self.loss, + optimizer=self.optimizer, + metrics=paddle.metric.Accuracy(), + ) class TestLRScheduler(TestEngineBase): - def init_optimizer(self): scheduler = paddle.optimizer.lr.CosineAnnealingDecay( - learning_rate=0.00001, T_max=10) + learning_rate=0.00001, T_max=10 + ) self.optimizer = paddle.optimizer.SGD(learning_rate=scheduler) def test_lr_scheduler(self): @@ -73,11 +77,11 @@ def test_lr_scheduler(self): class TestGradClipByGlobalNorm(TestEngineBase): - def init_optimizer(self): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) - self.optimizer = paddle.optimizer.SGD(learning_rate=0.00001, - grad_clip=clip) + self.optimizer = paddle.optimizer.SGD( + learning_rate=0.00001, grad_clip=clip + ) def test_grad_clip(self): @@ -89,19 +93,20 @@ def check_program(self): ops = self.engine.main_program.global_block().ops has_grad_clip = False for op in ops: - if op.desc.has_attr("op_namescope") \ - and op.desc.attr("op_namescope").startswith("/gradient_clip"): + if op.desc.has_attr("op_namescope") and op.desc.attr( + "op_namescope" + ).startswith("/gradient_clip"): has_grad_clip = True break assert has_grad_clip is True class TestGradClipByNorm(TestGradClipByGlobalNorm): - def init_optimizer(self): clip = paddle.nn.ClipGradByNorm(clip_norm=1.0) - self.optimizer = paddle.optimizer.SGD(learning_rate=0.00001, - grad_clip=clip) + self.optimizer = paddle.optimizer.SGD( + learning_rate=0.00001, grad_clip=clip + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_new_cost_model.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_new_cost_model.py index 59db55596af09..963d16d239e0f 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_new_cost_model.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_new_cost_model.py @@ -20,9 +20,15 @@ import paddle import paddle.distributed.auto_parallel.cost as cost_model -from paddle.distributed.auto_parallel.cost.base_cost import build_comp_desc_from_op -from paddle.distributed.auto_parallel.cost.base_cost import build_comp_desc_str_for_predict -from paddle.distributed.auto_parallel.cost.base_cost import calc_time_by_modeling +from paddle.distributed.auto_parallel.cost.base_cost import ( + build_comp_desc_from_op, +) +from paddle.distributed.auto_parallel.cost.base_cost import ( + build_comp_desc_str_for_predict, +) +from paddle.distributed.auto_parallel.cost.base_cost import ( + calc_time_by_modeling, +) from paddle.distributed.auto_parallel.cluster import Cluster from paddle.distributed.auto_parallel.cost import CommContext from test_cluster import cluster_json @@ -37,7 +43,6 @@ def check_cost(cost): class TestCost(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -60,7 +65,8 @@ def test_comp_cost(self): matmul_v2_op = op break matmul_v2_cost = cost_model._g_op_cost_factory["matmul_v2"]( - op=matmul_v2_op) + op=matmul_v2_op + ) desc = build_comp_desc_from_op(op=matmul_v2_op) desc_str = build_comp_desc_str_for_predict(desc) self.assertIsNotNone(desc_str) @@ -73,8 +79,9 @@ def test_comp_cost(self): def test_comm_cost(self): # Build cluster - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) @@ -90,7 +97,8 @@ def test_comm_cost(self): desc["inputs"] = {"X": [(paddle.float32, [100, 200])]} desc["group_ranks"] = [0, 1] allreduce_cost = cost_model._g_op_cost_factory["c_allreduce_sum"]( - op_desc=desc, comm_context=CommContext(cluster)) + op_desc=desc, comm_context=CommContext(cluster) + ) self.assertTrue(check_cost(allreduce_cost.cost)) # Remove unnecessary files @@ -99,8 +107,9 @@ def test_comm_cost(self): def test_cost_estimator(self): # Build cluster - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) @@ -108,8 +117,9 @@ def test_cost_estimator(self): cluster.build_from_file(cluster_json_path) train_program = paddle.static.Program() - cost_estimator = cost_model.CostEstimator(train_program, - cluster=cluster) + cost_estimator = cost_model.CostEstimator( + train_program, cluster=cluster + ) self.assertIsNotNone(cost_estimator) # Remove unnecessary files diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_optimization_tuner_api.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_optimization_tuner_api.py index 68f2376a83400..a8be44683338e 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_optimization_tuner_api.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_optimization_tuner_api.py @@ -21,7 +21,6 @@ class TestOptimizationTunerAPI(unittest.TestCase): - def test_engine_api(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "optimization_tuner_api.py") @@ -32,10 +31,19 @@ def test_engine_api(self): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "launch", "--gpus", "0,1", "--log_dir", tmp_dir.name, - launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "launch", + "--gpus", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner.py index ab48e2838f9b9..ea3bd45c03507 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner.py @@ -19,14 +19,21 @@ from paddle.distributed import fleet from paddle.distributed.auto_parallel.cluster import Cluster -from paddle.distributed.auto_parallel.dist_context import DistributedContext, set_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + DistributedContext, + set_default_distributed_context, +) from paddle.distributed.auto_parallel.tuner.parallel_tuner import ParallelTuner from paddle.distributed.auto_parallel.process_mesh import ProcessMesh import sys sys.path.append("..") import auto_parallel_gpt_model as modeling -from auto_parallel_gpt_model import GPTModel, GPTForPretraining, GPTPretrainingCriterion +from auto_parallel_gpt_model import ( + GPTModel, + GPTForPretraining, + GPTPretrainingCriterion, +) paddle.enable_static() @@ -36,7 +43,7 @@ sequence_len = 512 _g_process_mesh = [ ProcessMesh([0, 1], dim_names=["x"]), - ProcessMesh([2, 3], dim_names=["x"]) + ProcessMesh([2, 3], dim_names=["x"]), ] @@ -59,76 +66,100 @@ def get_program_v3(): # ProcessMesh([[4, 5], [6, 7]], dim_names=["x", "y"]) # ] with static.program_guard(train_program, start_program): - tokens = paddle.static.data(name="tokens", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = paddle.static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + tokens = paddle.static.data( + name="tokens", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = paddle.static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = paddle.static.data( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float32') - labels = paddle.static.data(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = paddle.static.data(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float32') + dtype='float32', + ) + labels = paddle.static.data( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = paddle.static.data( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float32' + ) data_holder = [tokens, position_ids, attention_mask, labels, loss_mask] - gpt = GPTModel(vocab_size=1000, - hidden_size=1024, - num_hidden_layers=2, - num_attention_heads=16, - intermediate_size=4 * 1024, - hidden_act="gelu", - hidden_dropout_prob=0.0, - attention_probs_dropout_prob=0.0, - max_position_embeddings=1024, - type_vocab_size=1, - initializer_range=0.02, - pad_token_id=0, - eos_token_id=7, - bos_token_id=0, - eol_token_id=3, - pp_degree=1) - - model = GPTForPretraining(gpt, - vocab_size=1000, - hidden_size=64, - initializer_range=0.02) + gpt = GPTModel( + vocab_size=1000, + hidden_size=1024, + num_hidden_layers=2, + num_attention_heads=16, + intermediate_size=4 * 1024, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=1024, + type_vocab_size=1, + initializer_range=0.02, + pad_token_id=0, + eos_token_id=7, + bos_token_id=0, + eol_token_id=3, + pp_degree=1, + ) + + model = GPTForPretraining( + gpt, vocab_size=1000, hidden_size=64, initializer_range=0.02 + ) preds = model(tokens, position_ids, attention_mask) criterion = GPTPretrainingCriterion() loss = criterion(preds, labels, loss_mask) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) feed_vars = { "inputs": [tokens, position_ids, attention_mask, loss_mask], - "labels": [labels] + "labels": [labels], } fetch_vars = {"loss": [loss]} - return train_program, start_program, None, loss, optimizer, feed_vars, fetch_vars + return ( + train_program, + start_program, + None, + loss, + optimizer, + feed_vars, + fetch_vars, + ) class TestParallelTunerTrain(unittest.TestCase): - def test_tune_with_train(self): flag = False set_default_distributed_context(DistributedContext()) - train_program, start_program, dataloader, loss, optimizer, feed_vars, fetch_vars = get_program_v3( - ) + ( + train_program, + start_program, + dataloader, + loss, + optimizer, + feed_vars, + fetch_vars, + ) = get_program_v3() cluster = Cluster() cluster.gen_default_config_cluster(node_count=1, device_count=8) - dist_context = DistributedContext(train_program, start_program, - optimizer, loss, feed_vars, - fetch_vars, cluster) + dist_context = DistributedContext( + train_program, + start_program, + optimizer, + loss, + feed_vars, + fetch_vars, + cluster, + ) dist_context.initialize() parallel_tuner = ParallelTuner(dist_context, max_trials=3, mode="train") parallel_tuner.tune() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_full.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_full.py index 27833a6a18500..4a7aa6fd208c7 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_full.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_full.py @@ -19,7 +19,10 @@ from paddle.distributed import fleet from paddle.distributed.auto_parallel.cluster import Cluster -from paddle.distributed.auto_parallel.dist_context import DistributedContext, set_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + DistributedContext, + set_default_distributed_context, +) from paddle.distributed.auto_parallel.tuner.parallel_tuner import ParallelTuner from paddle.distributed.auto_parallel.process_mesh import ProcessMesh from paddle.distributed.auto_parallel.planner_v2 import Planner @@ -28,7 +31,11 @@ sys.path.append("..") import auto_parallel_gpt_model as modeling -from auto_parallel_gpt_model import GPTModel, GPTForPretraining, GPTPretrainingCriterion +from auto_parallel_gpt_model import ( + GPTModel, + GPTForPretraining, + GPTPretrainingCriterion, +) paddle.enable_static() @@ -38,7 +45,7 @@ sequence_len = 512 _g_process_mesh = [ ProcessMesh([0, 1], dim_names=["x"]), - ProcessMesh([2, 3], dim_names=["x"]) + ProcessMesh([2, 3], dim_names=["x"]), ] @@ -58,86 +65,111 @@ def get_program_v3(): modeling._global_parallel_strategy = "dp_mp_pp" modeling.DPMPPP_MESH_LIST = [ ProcessMesh([[0, 1], [2, 3]], dim_names=["x", "y"]), - ProcessMesh([[4, 5], [6, 7]], dim_names=["x", "y"]) + ProcessMesh([[4, 5], [6, 7]], dim_names=["x", "y"]), ] with static.program_guard(train_program, start_program): - tokens = paddle.static.data(name="tokens", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = paddle.static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + tokens = paddle.static.data( + name="tokens", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = paddle.static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = paddle.static.data( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float32') - labels = paddle.static.data(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = paddle.static.data(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float32') + dtype='float32', + ) + labels = paddle.static.data( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = paddle.static.data( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float32' + ) data_holder = [tokens, position_ids, attention_mask, labels, loss_mask] - gpt = GPTModel(vocab_size=1000, - hidden_size=1024, - num_hidden_layers=2, - num_attention_heads=16, - intermediate_size=4 * 1024, - hidden_act="gelu", - hidden_dropout_prob=0.0, - attention_probs_dropout_prob=0.0, - max_position_embeddings=1024, - type_vocab_size=1, - initializer_range=0.02, - pad_token_id=0, - eos_token_id=7, - bos_token_id=0, - eol_token_id=3, - pp_degree=len(modeling.DPMPPP_MESH_LIST)) - - model = GPTForPretraining(gpt, - vocab_size=1000, - hidden_size=64, - initializer_range=0.02) + gpt = GPTModel( + vocab_size=1000, + hidden_size=1024, + num_hidden_layers=2, + num_attention_heads=16, + intermediate_size=4 * 1024, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=1024, + type_vocab_size=1, + initializer_range=0.02, + pad_token_id=0, + eos_token_id=7, + bos_token_id=0, + eol_token_id=3, + pp_degree=len(modeling.DPMPPP_MESH_LIST), + ) + + model = GPTForPretraining( + gpt, vocab_size=1000, hidden_size=64, initializer_range=0.02 + ) preds = model(tokens, position_ids, attention_mask) criterion = GPTPretrainingCriterion() loss = criterion(preds, labels, loss_mask) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) feed_vars = { "inputs": [tokens, position_ids, attention_mask, loss_mask], - "labels": [labels] + "labels": [labels], } fetch_vars = {"loss": [loss]} - return train_program, start_program, None, loss, optimizer, feed_vars, fetch_vars + return ( + train_program, + start_program, + None, + loss, + optimizer, + feed_vars, + fetch_vars, + ) class TestParallelTunerFull(unittest.TestCase): - def test_tune_with_planner(self): flag = False set_default_distributed_context(DistributedContext()) - train_program, start_program, dataloader, loss, optimizer, feed_vars, fetch_vars = get_program_v3( - ) + ( + train_program, + start_program, + dataloader, + loss, + optimizer, + feed_vars, + fetch_vars, + ) = get_program_v3() cluster = Cluster() cluster.gen_default_config_cluster(node_count=1, device_count=8) strategy = Strategy() strategy.auto_mode = "full" - dist_context = DistributedContext(train_program, start_program, - optimizer, loss, feed_vars, - fetch_vars, cluster, strategy) + dist_context = DistributedContext( + train_program, + start_program, + optimizer, + loss, + feed_vars, + fetch_vars, + cluster, + strategy, + ) dist_context.initialize() planner = Planner("train", dist_context) - planner._parallel_tuner = ParallelTuner(planner._dist_context, - mode=planner._mode, - max_trials=3) + planner._parallel_tuner = ParallelTuner( + planner._dist_context, mode=planner._mode, max_trials=3 + ) planner.plan() flag = True self.assertTrue(flag) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_predict.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_predict.py index 2d7a2c10579a7..30e7eabcc1156 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_predict.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_parallel_tuner_predict.py @@ -19,14 +19,21 @@ from paddle.distributed import fleet from paddle.distributed.auto_parallel.cluster import Cluster -from paddle.distributed.auto_parallel.dist_context import DistributedContext, set_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + DistributedContext, + set_default_distributed_context, +) from paddle.distributed.auto_parallel.tuner.parallel_tuner import ParallelTuner from paddle.distributed.auto_parallel.process_mesh import ProcessMesh import sys sys.path.append("..") import auto_parallel_gpt_model as modeling -from auto_parallel_gpt_model import GPTModel, GPTForPretraining, GPTPretrainingCriterion +from auto_parallel_gpt_model import ( + GPTModel, + GPTForPretraining, + GPTPretrainingCriterion, +) paddle.enable_static() @@ -36,7 +43,7 @@ sequence_len = 512 _g_process_mesh = [ ProcessMesh([0, 1], dim_names=["x"]), - ProcessMesh([2, 3], dim_names=["x"]) + ProcessMesh([2, 3], dim_names=["x"]), ] @@ -56,84 +63,108 @@ def get_program_v3(): modeling._global_parallel_strategy = "dp_mp_pp" modeling.DPMPPP_MESH_LIST = [ ProcessMesh([[0, 1], [2, 3]], dim_names=["x", "y"]), - ProcessMesh([[4, 5], [6, 7]], dim_names=["x", "y"]) + ProcessMesh([[4, 5], [6, 7]], dim_names=["x", "y"]), ] with static.program_guard(train_program, start_program): - tokens = paddle.static.data(name="tokens", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = paddle.static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + tokens = paddle.static.data( + name="tokens", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = paddle.static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = paddle.static.data( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float32') - labels = paddle.static.data(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = paddle.static.data(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float32') + dtype='float32', + ) + labels = paddle.static.data( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = paddle.static.data( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float32' + ) data_holder = [tokens, position_ids, attention_mask, labels, loss_mask] - gpt = GPTModel(vocab_size=1000, - hidden_size=1024, - num_hidden_layers=2, - num_attention_heads=16, - intermediate_size=4 * 1024, - hidden_act="gelu", - hidden_dropout_prob=0.0, - attention_probs_dropout_prob=0.0, - max_position_embeddings=1024, - type_vocab_size=1, - initializer_range=0.02, - pad_token_id=0, - eos_token_id=7, - bos_token_id=0, - eol_token_id=3, - pp_degree=len(modeling.DPMPPP_MESH_LIST)) - - model = GPTForPretraining(gpt, - vocab_size=1000, - hidden_size=64, - initializer_range=0.02) + gpt = GPTModel( + vocab_size=1000, + hidden_size=1024, + num_hidden_layers=2, + num_attention_heads=16, + intermediate_size=4 * 1024, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=1024, + type_vocab_size=1, + initializer_range=0.02, + pad_token_id=0, + eos_token_id=7, + bos_token_id=0, + eol_token_id=3, + pp_degree=len(modeling.DPMPPP_MESH_LIST), + ) + + model = GPTForPretraining( + gpt, vocab_size=1000, hidden_size=64, initializer_range=0.02 + ) preds = model(tokens, position_ids, attention_mask) criterion = GPTPretrainingCriterion() loss = criterion(preds, labels, loss_mask) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) feed_vars = { "inputs": [tokens, position_ids, attention_mask, loss_mask], - "labels": [labels] + "labels": [labels], } fetch_vars = {"loss": [loss]} - return train_program, start_program, None, loss, optimizer, feed_vars, fetch_vars + return ( + train_program, + start_program, + None, + loss, + optimizer, + feed_vars, + fetch_vars, + ) class TestParallelTunerPredict(unittest.TestCase): - def test_tune_predict(self): flag = False set_default_distributed_context(DistributedContext()) - train_program, start_program, dataloader, loss, optimizer, feed_vars, fetch_vars = get_program_v3( - ) + ( + train_program, + start_program, + dataloader, + loss, + optimizer, + feed_vars, + fetch_vars, + ) = get_program_v3() cluster = Cluster() cluster.gen_default_config_cluster(node_count=1, device_count=8) - dist_context = DistributedContext(train_program, start_program, - optimizer, loss, feed_vars, - fetch_vars, cluster) + dist_context = DistributedContext( + train_program, + start_program, + optimizer, + loss, + feed_vars, + fetch_vars, + cluster, + ) dist_context.initialize() - parallel_tuner = ParallelTuner(dist_context, - max_trials=3, - mode="predict") + parallel_tuner = ParallelTuner( + dist_context, max_trials=3, mode="predict" + ) parallel_tuner.tune() flag = True diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_amp.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_amp.py index 09de1d18c1ac5..2d8f3c3a8e3ca 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_amp.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_amp.py @@ -20,7 +20,6 @@ class TestAMPPass(unittest.TestCase): - def test_mp2(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "amp_pass_unittest.py") @@ -31,10 +30,19 @@ def test_mp2(self): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_grad_clip.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_grad_clip.py index bfc5eb18a065d..6febed424282e 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_grad_clip.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_grad_clip.py @@ -20,11 +20,11 @@ class TestGradientClip(unittest.TestCase): - def test_dp2(self): file_dir = os.path.dirname(os.path.abspath(__file__)) - launch_model_path = os.path.join(file_dir, - "clip_grad_by_global_norm.py") + launch_model_path = os.path.join( + file_dir, "clip_grad_by_global_norm.py" + ) if os.environ.get("WITH_COVERAGE", "OFF") == "ON": coverage_args = ["-m", "coverage", "run", "--branch", "-p"] @@ -32,10 +32,19 @@ def test_dp2(self): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_gradient_merge.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_gradient_merge.py index 7ce62fa1389f4..0ca2cae2e1bd3 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_gradient_merge.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_gradient_merge.py @@ -20,11 +20,11 @@ class TestGradientMergePass(unittest.TestCase): - def test_dp2(self): file_dir = os.path.dirname(os.path.abspath(__file__)) - launch_model_path = os.path.join(file_dir, - "gradient_merge_pass_unittest.py") + launch_model_path = os.path.join( + file_dir, "gradient_merge_pass_unittest.py" + ) if os.environ.get("WITH_COVERAGE", "OFF") == "ON": coverage_args = ["-m", "coverage", "run", "--branch", "-p"] @@ -32,10 +32,19 @@ def test_dp2(self): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_quantization.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_quantization.py index e8df938d62282..1181f090cba80 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_quantization.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_quantization.py @@ -34,7 +34,6 @@ def apply_pass(): class TestQuantizationPass(unittest.TestCase): - def test_qat_pass(self): batch_size = 8 diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_recompute.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_recompute.py index c6062faa26078..ff873bec4f5b6 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_recompute.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_recompute.py @@ -20,7 +20,6 @@ class TestRecomputePass(unittest.TestCase): - def test_mp2(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "recompute_pass_unittest.py") @@ -31,10 +30,19 @@ def test_mp2(self): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_sharding.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_sharding.py index 31467b6d505ed..eab74b451305f 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_sharding.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_pass_sharding.py @@ -20,7 +20,6 @@ class TestShardingPass(unittest.TestCase): - def test_dp2sharding2(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join(file_dir, "sharding_pass_unittest.py") @@ -31,10 +30,19 @@ def test_dp2sharding2(self): coverage_args = [] tmp_dir = tempfile.TemporaryDirectory() - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--devices", "0,1", "--log_dir", - tmp_dir.name, launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--devices", + "0,1", + "--log_dir", + tmp_dir.name, + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_prim_dist_op.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_prim_dist_op.py index 028d2b820a52b..2c22f3fb1c134 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_prim_dist_op.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_prim_dist_op.py @@ -22,7 +22,10 @@ from paddle.distributed.auto_parallel.completion import Completer from paddle.distributed.auto_parallel.partitioner import Partitioner from paddle.distributed.auto_parallel.utils import set_var_dist_attr -from paddle.distributed.auto_parallel.dist_context import DistributedContext, get_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + DistributedContext, + get_default_distributed_context, +) paddle.enable_static() enable_prim() @@ -31,60 +34,68 @@ class TestPrimDistOp(unittest.TestCase): - def setUp(self): self.main_program = paddle.static.Program() self.startup_program = paddle.static.Program() self.layer_help = LayerHelper('TestPrimDistOp') - with paddle.static.program_guard(self.main_program, - self.startup_program): + with paddle.static.program_guard( + self.main_program, self.startup_program + ): self.init_prog() def init_prog(self): # block = self.main_program.global_block() # block = self.main_program.global_block() - self.w = self.layer_help.create_parameter(dtype="float", - shape=[20], - attr=None) - self.w_grad = paddle.static.data(name='w_grad', - shape=[20], - dtype='float') + self.w = self.layer_help.create_parameter( + dtype="float", shape=[20], attr=None + ) + self.w_grad = paddle.static.data( + name='w_grad', shape=[20], dtype='float' + ) self.tmp1 = paddle.static.data(name='tmp1', shape=[20], dtype='float') self.tmp2 = paddle.static.data(name='tmp2', shape=[20], dtype='float') - self.batch_reduced = paddle.static.data(name='batch_reduced', - shape=[1], - dtype='float') + self.batch_reduced = paddle.static.data( + name='batch_reduced', shape=[1], dtype='float' + ) self.attrs = {} default_dist_context = get_default_distributed_context() _global_process_mesh = auto.ProcessMesh(list(range(nranks))) - tensor_dist_attr = set_var_dist_attr(default_dist_context, - self.tmp1, [-1], - _global_process_mesh, - mark_annotated=True) - tensor_dist_attr = set_var_dist_attr(default_dist_context, - self.tmp1, [-1], - _global_process_mesh, - mark_annotated=True) - - op = self.layer_help.append_op(type="add_p", - inputs={ - 'X': self.tmp1, - 'Y': self.w - }, - outputs={'Z': self.w_grad}, - attrs=self.attrs) - - op = self.layer_help.append_op(type="reduce_sum_p", - inputs={'X': self.tmp2}, - outputs={'Y': self.batch_reduced}, - attrs={"axis": [0]}) + tensor_dist_attr = set_var_dist_attr( + default_dist_context, + self.tmp1, + [-1], + _global_process_mesh, + mark_annotated=True, + ) + tensor_dist_attr = set_var_dist_attr( + default_dist_context, + self.tmp1, + [-1], + _global_process_mesh, + mark_annotated=True, + ) + + op = self.layer_help.append_op( + type="add_p", + inputs={'X': self.tmp1, 'Y': self.w}, + outputs={'Z': self.w_grad}, + attrs=self.attrs, + ) + + op = self.layer_help.append_op( + type="reduce_sum_p", + inputs={'X': self.tmp2}, + outputs={'Y': self.batch_reduced}, + attrs={"axis": [0]}, + ) def test_loss_and_grad_allreduce(self): - dist_context = DistributedContext(self.main_program, - self.startup_program) + dist_context = DistributedContext( + self.main_program, self.startup_program + ) completer = Completer(dist_context) completer.complete_prim_annotation(self.main_program) dist_context.block_state.parse_forward_blocks(self.main_program) @@ -95,7 +106,8 @@ def test_loss_and_grad_allreduce(self): dist_context.data_parallel_group = list(range(nranks)) partitioner = Partitioner(dist_context, rank) dist_main_prog, dist_startup_prog, _ = partitioner.partition( - self.main_program, self.startup_program, [(self.w, self.w_grad)]) + self.main_program, self.startup_program, [(self.w, self.w_grad)] + ) ops = dist_main_prog.global_block().ops self.assertTrue(ops[1].type == "c_allreduce_sum") diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh.py index 6b83c4da24596..2e698efc416fd 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh.py @@ -19,7 +19,9 @@ import paddle.nn.functional as F import paddle.static as static from paddle.distributed.auto_parallel.process_mesh import ProcessMesh -from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, +) paddle.enable_static() @@ -30,29 +32,33 @@ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size - param_initializer = nn.initializer.Normal(mean=0.0, - std=initializer_range) + param_initializer = nn.initializer.Normal( + mean=0.0, std=initializer_range + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.linear0 = nn.Linear( d_model, dim_feedforward, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) self.linear1 = nn.Linear( dim_feedforward, d_model, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) def forward(self, input): out = self.norm(input) @@ -63,7 +69,6 @@ def forward(self, input): class TestProcessMesh(unittest.TestCase): - def test_construction(self): mesh = [[0, 1, 2], [3, 4, 5]] process_mesh = ProcessMesh(mesh, dim_names=["x", "y"]) @@ -106,17 +111,21 @@ def test_construction(self): def test_context_manager(self): mesh = np.array([1, 2, 3, 4]) - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, sequence_len, 1], - dtype='float32') - - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + label = static.data( + name="label", shape=[batch_size, sequence_len, 1], dtype='float32' + ) + + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) with ProcessMesh(mesh, "d"): out = mlp(input) @@ -127,15 +136,18 @@ def test_context_manager(self): for block in default_program.blocks: for tensor in block.vars.values(): dist_tensor = default_dist_context.get_dist_tensor_for_program( - tensor) + tensor + ) if dist_tensor is not None: - self.assertEqual(dist_tensor.dist_attr.process_mesh, - ProcessMesh(mesh)) + self.assertEqual( + dist_tensor.dist_attr.process_mesh, ProcessMesh(mesh) + ) for op in block.ops: dist_op = default_dist_context.get_dist_op_for_program(op) if dist_op is not None: - self.assertEqual(dist_op.dist_attr.process_mesh, - ProcessMesh(mesh)) + self.assertEqual( + dist_op.dist_attr.process_mesh, ProcessMesh(mesh) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh_v2.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh_v2.py index 3c58f9e8cd393..7a813e703056a 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh_v2.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_process_mesh_v2.py @@ -14,11 +14,13 @@ import unittest from paddle.distributed.auto_parallel.process_mesh_v2 import ( - ProcessMesh, compute_compatible_process_mesh, merge_process_mesh) + ProcessMesh, + compute_compatible_process_mesh, + merge_process_mesh, +) class TestProcessMesh(unittest.TestCase): - def test_process_mesh(self): mesh = [[0, 1, 2], [3, 4, 5]] mesh2 = [[0, 1], [2, 3]] @@ -41,34 +43,41 @@ def test_process_mesh(self): self.assertEqual(str(process_mesh), str(process_mesh)) def test_compute_compatible_process_mesh(self): - process_mesh1 = ProcessMesh([[0, 1, 2], [3, 4, 5]], - dim_names=["x", "y"]) + process_mesh1 = ProcessMesh( + [[0, 1, 2], [3, 4, 5]], dim_names=["x", "y"] + ) compatible_process_mesh = compute_compatible_process_mesh( - [process_mesh1, None]) + [process_mesh1, None] + ) self.assertEqual(compatible_process_mesh, process_mesh1) compatible_process_mesh = compute_compatible_process_mesh( - [None, process_mesh1]) + [None, process_mesh1] + ) self.assertEqual(compatible_process_mesh, process_mesh1) process_mesh2 = ProcessMesh([[0, 1, 2], [3, 4, 5]]) compatible_process_mesh = compute_compatible_process_mesh( - [process_mesh1, process_mesh2]) + [process_mesh1, process_mesh2] + ) self.assertEqual(compatible_process_mesh, process_mesh1) self.assertEqual(compatible_process_mesh, process_mesh2) process_mesh2 = ProcessMesh([[0, 1, 2, 3, 4, 5]]) compatible_process_mesh = compute_compatible_process_mesh( - [process_mesh1, process_mesh2]) + [process_mesh1, process_mesh2] + ) self.assertEqual(compatible_process_mesh, process_mesh1) process_mesh2 = ProcessMesh([[0, 1, 2]]) compatible_process_mesh = compute_compatible_process_mesh( - [process_mesh1, process_mesh2]) + [process_mesh1, process_mesh2] + ) self.assertEqual(compatible_process_mesh, process_mesh1) def test_merge_process_mesh(self): - process_mesh1 = ProcessMesh([[0, 1, 2], [3, 4, 5]], - dim_names=["x", "y"]) + process_mesh1 = ProcessMesh( + [[0, 1, 2], [3, 4, 5]], dim_names=["x", "y"] + ) merged_process_mesh = merge_process_mesh([process_mesh1, None]) print(merged_process_mesh) self.assertEqual(merged_process_mesh, ProcessMesh([0, 1, 2, 3, 4, 5])) @@ -85,8 +94,9 @@ def test_merge_process_mesh(self): process_mesh2 = ProcessMesh([[6, 7]]) merged_process_mesh = merge_process_mesh([process_mesh1, process_mesh2]) - self.assertEqual(merged_process_mesh, - ProcessMesh([0, 1, 2, 3, 4, 5, 6, 7])) + self.assertEqual( + merged_process_mesh, ProcessMesh([0, 1, 2, 3, 4, 5, 6, 7]) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_recorder.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_recorder.py index d9594b951983f..5f07a71176896 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_recorder.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_recorder.py @@ -19,7 +19,6 @@ class TestRecorder(unittest.TestCase): - def test_register(self): recorder = rd.MetricsRecorder() recorder.register("metric") @@ -35,8 +34,9 @@ def test_update(self): recorder = rd.MetricsRecorder() recorder.update("metric", 4, 1000) self.assertEqual(recorder.records["metric"].direction, "min") - self.assertEqual(recorder.get_records("metric"), - [rd.MetricRecord(4, 1000)]) + self.assertEqual( + recorder.get_records("metric"), [rd.MetricRecord(4, 1000)] + ) def test_get_records(self): recorder = rd.MetricsRecorder() @@ -44,12 +44,15 @@ def test_get_records(self): recorder.update("metric", 2, step=1) recorder.update("metric", 3, step=2) recorder.update("metric", 4, step=3) - self.assertEqual(recorder.get_records("metric"), [ - rd.MetricRecord(1, 0), - rd.MetricRecord(2, 1), - rd.MetricRecord(3, 2), - rd.MetricRecord(4, 3), - ]) + self.assertEqual( + recorder.get_records("metric"), + [ + rd.MetricRecord(1, 0), + rd.MetricRecord(2, 1), + rd.MetricRecord(3, 2), + rd.MetricRecord(4, 3), + ], + ) def test_set_records(self): recorder = rd.MetricsRecorder() @@ -62,12 +65,15 @@ def test_set_records(self): rd.MetricRecord(4, 3), ], ) - self.assertEqual(recorder.get_records("metric"), [ - rd.MetricRecord(1, 0), - rd.MetricRecord(2, 1), - rd.MetricRecord(3, 2), - rd.MetricRecord(4, 3), - ]) + self.assertEqual( + recorder.get_records("metric"), + [ + rd.MetricRecord(1, 0), + rd.MetricRecord(2, 1), + rd.MetricRecord(3, 2), + rd.MetricRecord(4, 3), + ], + ) def test_get_best_value(self): recorder = rd.MetricsRecorder() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_gpt_planner.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_gpt_planner.py index bde0615a41dcd..0ffe09bfd1ee4 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_gpt_planner.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_gpt_planner.py @@ -21,7 +21,6 @@ class TestPlannerReLaunch(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -31,10 +30,12 @@ def tearDown(self): def test_relaunch_with_planner(self): from test_auto_parallel_relaunch import cluster_json, mapping_josn - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") - mapping_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_rank_mapping.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) + mapping_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_rank_mapping.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: @@ -46,19 +47,31 @@ def test_relaunch_with_planner(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join( - file_dir, "auto_parallel_relaunch_with_gpt_planner.py") + file_dir, "auto_parallel_relaunch_with_gpt_planner.py" + ) if os.environ.get("WITH_COVERAGE", "OFF") == "ON": coverage_args = ["-m", "coverage", "run", "--branch", "-p"] else: coverage_args = [] - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--log_dir", self.temp_dir.name, - "--cluster_topo_path", cluster_json_path, "--rank_mapping_path", - mapping_json_path, "--enable_auto_mapping", "True", - launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--log_dir", + self.temp_dir.name, + "--cluster_topo_path", + cluster_json_path, + "--rank_mapping_path", + mapping_json_path, + "--enable_auto_mapping", + "True", + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() self.assertEqual(process.returncode, 0) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_planner.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_planner.py index 3c17b6d126e58..b8469c1a40782 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_planner.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_relaunch_with_planner.py @@ -21,7 +21,6 @@ class TestPlannerReLaunch(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -31,10 +30,12 @@ def tearDown(self): def test_relaunch_with_planner(self): from test_auto_parallel_relaunch import cluster_json, mapping_josn - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") - mapping_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_rank_mapping.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) + mapping_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_rank_mapping.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: @@ -46,19 +47,31 @@ def test_relaunch_with_planner(self): file_dir = os.path.dirname(os.path.abspath(__file__)) launch_model_path = os.path.join( - file_dir, "auto_parallel_relaunch_with_planner.py") + file_dir, "auto_parallel_relaunch_with_planner.py" + ) if os.environ.get("WITH_COVERAGE", "OFF") == "ON": coverage_args = ["-m", "coverage", "run", "--branch", "-p"] else: coverage_args = [] - cmd = [sys.executable, "-u"] + coverage_args + [ - "-m", "paddle.distributed.launch", "--log_dir", self.temp_dir.name, - "--cluster_topo_path", cluster_json_path, "--rank_mapping_path", - mapping_json_path, "--enable_auto_mapping", "True", - launch_model_path - ] + cmd = ( + [sys.executable, "-u"] + + coverage_args + + [ + "-m", + "paddle.distributed.launch", + "--log_dir", + self.temp_dir.name, + "--cluster_topo_path", + cluster_json_path, + "--rank_mapping_path", + mapping_json_path, + "--enable_auto_mapping", + "True", + launch_model_path, + ] + ) process = subprocess.Popen(cmd) process.wait() self.assertEqual(process.returncode, 0) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_strategy.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_strategy.py index 08cd1dee0e681..54b11c4934cd0 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_strategy.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_strategy.py @@ -18,7 +18,6 @@ class TestStrategy(unittest.TestCase): - def test_default_config(self): strategy = auto.Strategy() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_to_static.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_to_static.py index e751ef64be2d9..3c8b71e71394e 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_to_static.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_to_static.py @@ -34,7 +34,6 @@ class MyDataset(Dataset): - def __init__(self, num_samples): super(MyDataset, self).__init__() self.num_samples = num_samples @@ -49,26 +48,26 @@ def __len__(self): class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) - - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=None) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=None) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) + + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=None + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=None + ) self.linear2 = nn.Linear(d_model, 1, weight_attr, bias_attr=None) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") @@ -85,17 +84,19 @@ def forward(self, input): class TestWholeProgram(unittest.TestCase): - def test_apply_optimzier(self): paddle.disable_static() - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) metrics = paddle.metric.Accuracy() loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.SGD(learning_rate=0.00001, - parameters=mlp.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.00001, parameters=mlp.parameters() + ) inputs = InputSpec([batch_size, hidden_size], 'float32', 'x') labels = InputSpec([batch_size], 'int64', 'label') @@ -114,7 +115,8 @@ def test_apply_optimzier(self): optimize_ops, _ = program_helper.apply_optimizer(optimizer) all_ops = program_helper.main_program.block(0).ops sgd_ops = [ - op for op in program_helper.main_program.block(0).ops + op + for op in program_helper.main_program.block(0).ops if op.type == 'sgd' ] self.assertEqual(len(all_ops), 37) @@ -124,16 +126,18 @@ def test_apply_optimzier(self): class TestToStatic(unittest.TestCase): - def test_to_static(self): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() - optimizer = paddle.optimizer.SGD(learning_rate=0.00001, - parameters=mlp.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.00001, parameters=mlp.parameters() + ) dataset = MyDataset(batch_num * batch_size) @@ -141,11 +145,13 @@ def test_to_static(self): # labels = InputSpec([batch_size], 'int64', 'label') assert _non_static_mode() == True - engine = auto.Engine(model=mlp, - loss=loss, - optimizer=optimizer, - metrics=paddle.metric.Accuracy(), - strategy=None) + engine = auto.Engine( + model=mlp, + loss=loss, + optimizer=optimizer, + metrics=paddle.metric.Accuracy(), + strategy=None, + ) engine.fit(dataset, batch_size=batch_size) engine.evaluate(dataset, batch_size=batch_size) engine.predict(dataset, batch_size=batch_size) @@ -153,14 +159,15 @@ def test_to_static(self): class TestLazyInit(unittest.TestCase): - def test_lazy_init(self): with LazyGuard(): - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) loss = paddle.nn.CrossEntropyLoss() metrics = paddle.metric.Accuracy() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_trial.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_trial.py index e39991fcaa514..fc52d1c394eff 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_trial.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_trial.py @@ -19,7 +19,6 @@ class TestTiral(unittest.TestCase): - def test_trial(self): space = ts.TunableSpace() space.choice("choice", [0, 1, 2, 3], default=2) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_space.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_space.py index 58ff36aba09db..badc90275fd38 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_space.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_space.py @@ -18,7 +18,6 @@ class TestTunableSpace(unittest.TestCase): - def test_fixed(self): space = ts.TunableSpace() fixed = space.fixed("fixed", default=4) @@ -73,10 +72,9 @@ def test_int_range(self): def test_float_range(self): space = ts.TunableSpace() - float_range = space.float_range("float_range", - start=0.4, - stop=4.4, - default=2.0) + float_range = space.float_range( + "float_range", start=0.4, stop=4.4, default=2.0 + ) self.assertEqual(space.values["float_range"], 2.0) self.assertEqual(len(space.variables), 1) self.assertEqual(space.variables["float_range"].name, "float_range") diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_variable.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_variable.py index ce0a076c83e7e..641f7b4347e36 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_variable.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_tunable_variable.py @@ -18,7 +18,6 @@ class TestTunableVariable(unittest.TestCase): - def test_fixed(self): fixed = tv.Fixed("fixed", True) fixed = tv.Fixed.from_state(fixed.get_state()) @@ -64,12 +63,9 @@ def test_int_range(self): self.assertIn(int_range.random(1234), [1, 2, 3, 4]) self.assertNotEqual(int_range.default, 4) - int_range = tv.IntRange("int_range", - start=1, - stop=8, - step=2, - default=3, - endpoint=True) + int_range = tv.IntRange( + "int_range", start=1, stop=8, step=2, default=3, endpoint=True + ) int_range = tv.IntRange.from_state(int_range.get_state()) self.assertEqual(int_range.default, 3) self.assertIn(int_range.random(), [1, 3, 5, 7]) @@ -77,10 +73,9 @@ def test_int_range(self): self.assertNotEqual(int_range.default, 2) def test_float_range(self): - float_range = tv.FloatRange("float_range", - start=0.4, - stop=4.4, - default=2.0) + float_range = tv.FloatRange( + "float_range", start=0.4, stop=4.4, default=2.0 + ) float_range = tv.FloatRange.from_state(float_range.get_state()) self.assertEqual(float_range.default, 2.0) self.assertGreaterEqual(float_range.random(), 0.4) @@ -88,12 +83,14 @@ def test_float_range(self): self.assertNotAlmostEqual(float_range.random(), 1) self.assertNotAlmostEqual(float_range.random(), 4.4) - float_range = tv.FloatRange("float_range", - start=0.4, - stop=8.4, - step=2.0, - default=3.0, - endpoint=True) + float_range = tv.FloatRange( + "float_range", + start=0.4, + stop=8.4, + step=2.0, + default=3.0, + endpoint=True, + ) float_range = tv.FloatRange.from_state(float_range.get_state()) self.assertEqual(float_range.default, 3.0) self.assertGreaterEqual(float_range.random(), 0.4) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_completion.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_completion.py index bba1e2ba6f19f..9a8b9e1a8f36e 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_completion.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_completion.py @@ -40,50 +40,56 @@ def get_random_inputs_and_labels(input_shape, label_shape): def batch_generator_creator(): - def __reader__(): for _ in range(batch_size): batch_input, batch_label = get_random_inputs_and_labels( [batch_size, sequence_len, hidden_size], - [batch_size, sequence_len, 1]) + [batch_size, sequence_len, 1], + ) yield batch_input, batch_label return __reader__ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size - param_initializer = nn.initializer.Normal(mean=0.0, - std=initializer_range) + param_initializer = nn.initializer.Normal( + mean=0.0, std=initializer_range + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.linear0 = nn.Linear( d_model, dim_feedforward, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) self.linear1 = nn.Linear( dim_feedforward, d_model, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) def forward(self, input): out = self.norm(input) - auto.shard_tensor(self.linear0.weight, _g_process_mesh[:, 0], - [None, 'x']) + auto.shard_tensor( + self.linear0.weight, _g_process_mesh[:, 0], [None, 'x'] + ) out = self.linear0(out) out = F.gelu(out, approximate=True) - auto.shard_tensor(self.linear1.weight, _g_process_mesh[:, 1], - ['x', None]) + auto.shard_tensor( + self.linear1.weight, _g_process_mesh[:, 1], ['x', None] + ) out = self.linear1(out) return out @@ -95,15 +101,19 @@ def loop_cond(i, loop_len, input_array): def loop_body(i, loop_len, input_array): pre_input = paddle.tensor.array_read(array=input_array, i=i) - mlp_while0 = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) - - mlp_while1 = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_while0 = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) + + mlp_while1 = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) output = mlp_while0(pre_input) cur_pred = mlp_while1(output) @@ -128,41 +138,46 @@ def get_program(): loop_len = paddle.full(shape=[1], fill_value=epoch_num, dtype='int64') # input - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, sequence_len, 1], - dtype='float32') + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + label = static.data( + name="label", shape=[batch_size, sequence_len, 1], dtype='float32' + ) data_holder = [input, label] # dataloader - dataloader = paddle.io.DataLoader.from_generator(feed_list=data_holder, - capacity=4 * - batch_size, - iterable=False) - dataloader.set_batch_generator(batch_generator_creator(), - places=paddle.static.cuda_places()) + dataloader = paddle.io.DataLoader.from_generator( + feed_list=data_holder, capacity=4 * batch_size, iterable=False + ) + dataloader.set_batch_generator( + batch_generator_creator(), places=paddle.static.cuda_places() + ) # data dist_attr auto.shard_tensor(input, _g_process_mesh[:, 0], [None, None, None]) auto.shard_tensor(label, _g_process_mesh[:, 0], [None, None, None]) - mlp_start = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_start = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) pred = mlp_start(input) input_array = paddle.tensor.array_write(pred, i) i, loop_len, input_array = static.nn.while_loop( - cond=loop_cond, - body=loop_body, - loop_vars=[i, loop_len, input_array]) + cond=loop_cond, body=loop_body, loop_vars=[i, loop_len, input_array] + ) end_pred = paddle.tensor.array_read(array=input_array, i=i) - mlp_end = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_end = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) pred = mlp_end(end_pred) error_cost = paddle.nn.functional.square_error_cost(pred, label) @@ -172,13 +187,13 @@ def get_program(): class TestMLP(unittest.TestCase): - def test_completer(self): train_program, start_program, dataloader, i, loss = get_program() dist_context = DistributedContext() completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) # print_program_with_dist_attr(complete_train_program, dist_context) def test_completer_by_dist_op(self): @@ -186,7 +201,8 @@ def test_completer_by_dist_op(self): dist_context = DistributedContext() completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) complete_train_program = completer._complete_tensor_dist_attr_by_op() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_partition.py b/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_partition.py index 2e3f1d936338a..4bb9272748bf2 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_partition.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel/test_while_op_partition.py @@ -25,7 +25,9 @@ from paddle.distributed.auto_parallel.completion import Completer from paddle.distributed.auto_parallel.partitioner import Partitioner from paddle.distributed.auto_parallel.utils import make_data_unshard -from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, +) paddle.enable_static() @@ -43,41 +45,45 @@ def get_random_inputs_and_labels(input_shape, label_shape): def batch_generator_creator(): - def __reader__(): for _ in range(batch_size): batch_input, batch_label = get_random_inputs_and_labels( [batch_size, sequence_len, hidden_size], - [batch_size, sequence_len, 1]) + [batch_size, sequence_len, 1], + ) yield batch_input, batch_label return __reader__ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size - param_initializer = nn.initializer.Normal(mean=0.0, - std=initializer_range) + param_initializer = nn.initializer.Normal( + mean=0.0, std=initializer_range + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.linear0 = nn.Linear( d_model, dim_feedforward, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) self.linear1 = nn.Linear( dim_feedforward, d_model, weight_attr=paddle.ParamAttr(initializer=param_initializer), - bias_attr=None) + bias_attr=None, + ) def forward(self, input): @@ -114,41 +120,46 @@ def get_program(): auto.shard_tensor(i, _g_process_mesh, [None]) # 循环次数 - loop_len = fluid.layers.fill_constant(shape=[1], - dtype='int64', - value=epoch_num) + loop_len = fluid.layers.fill_constant( + shape=[1], dtype='int64', value=epoch_num + ) auto.shard_tensor(loop_len, _g_process_mesh, [None]) # input - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, sequence_len, 1], - dtype='float32') + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + label = static.data( + name="label", shape=[batch_size, sequence_len, 1], dtype='float32' + ) data_holder = [input, label] # dataloader - dataloader = paddle.io.DataLoader.from_generator(feed_list=data_holder, - capacity=4 * - batch_size, - iterable=False) - dataloader.set_batch_generator(batch_generator_creator(), - places=paddle.static.cuda_places()) + dataloader = paddle.io.DataLoader.from_generator( + feed_list=data_holder, capacity=4 * batch_size, iterable=False + ) + dataloader.set_batch_generator( + batch_generator_creator(), places=paddle.static.cuda_places() + ) # data dist_attr auto.shard_tensor(input, _g_process_mesh, [None, None, None]) auto.shard_tensor(label, _g_process_mesh, [None, None, None]) # fill constant bsz like tmp = paddle.fluid.layers.fill_constant_batch_size_like( - input=input, shape=[-1, 16, 0, 48], dtype='float32', value=0) + input=input, shape=[-1, 16, 0, 48], dtype='float32', value=0 + ) auto.shard_tensor(tmp, _g_process_mesh, [None, 'x', None, None]) # model - mlp_start = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_start = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) pred = mlp_start(input) input_array = fluid.layers.array_write(pred, i) @@ -168,10 +179,12 @@ def get_program(): pre_input = fluid.layers.array_read(array=input_array, i=i) auto.shard_tensor(pre_input, _g_process_mesh, [None, None, None]) - mlp_while = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_while = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) cur_pred = mlp_while(pre_input) # 更新循环条件 @@ -182,10 +195,12 @@ def get_program(): end_pred = fluid.layers.array_read(array=input_array, i=i) auto.shard_tensor(end_pred, _g_process_mesh, [None, None, None]) - mlp_end = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp_end = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) pred = mlp_end(end_pred) error_cost = paddle.nn.functional.square_error_cost(pred, label) @@ -345,24 +360,25 @@ def partition(train_program, start_program, dist_context): rank = paddle.distributed.get_rank() partitioner = Partitioner(dist_context, rank) dist_main_prog, dist_startup_prog, _ = partitioner.partition( - train_program, start_program, []) + train_program, start_program, [] + ) return dist_main_prog, dist_startup_prog class TestMLP(unittest.TestCase): - def test_partitioner(self): train_program, start_program, dataloader, i, loss = get_program() dist_context = get_default_distributed_context() - train_program, start_program = completion(train_program, start_program, - dist_context) + train_program, start_program = completion( + train_program, start_program, dist_context + ) dist_context.block_state.parse_forward_blocks(train_program) - dist_main_prog, dist_startup_prog = partition(train_program, - start_program, - dist_context) + dist_main_prog, dist_startup_prog = partition( + train_program, start_program, dist_context + ) global_block_ops = dist_main_prog.blocks[0].ops fill_op = None diff --git a/python/paddle/fluid/tests/unittests/auto_parallel_autoconvert.py b/python/paddle/fluid/tests/unittests/auto_parallel_autoconvert.py index b8c64c0af5454..f2ced04fe56af 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel_autoconvert.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel_autoconvert.py @@ -26,9 +26,19 @@ from paddle.distributed import fleet from paddle.fluid.initializer import NumpyArrayInitializer -from paddle.distributed.auto_parallel.utils import save_distributed_checkpoint, load_distributed_checkpoint, load_checkpoint_into_program -from paddle.distributed.auto_parallel.utils import get_dist_attr, merge_and_slice_parameter, load_parameter_into_program -from paddle.distributed.auto_parallel.dist_context import set_default_distributed_context +from paddle.distributed.auto_parallel.utils import ( + save_distributed_checkpoint, + load_distributed_checkpoint, + load_checkpoint_into_program, +) +from paddle.distributed.auto_parallel.utils import ( + get_dist_attr, + merge_and_slice_parameter, + load_parameter_into_program, +) +from paddle.distributed.auto_parallel.dist_context import ( + set_default_distributed_context, +) paddle.enable_static() _global_parallel_strategy = None @@ -38,11 +48,9 @@ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=64, - intermediate_size=4 * 64, - initializer_range=0.02): + def __init__( + self, hidden_size=64, intermediate_size=4 * 64, initializer_range=0.02 + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size @@ -52,14 +60,12 @@ def __init__(self, weight_attr0 = paddle.ParamAttr(initializer=NumpyArrayInitializer(arr0)) weight_attr1 = paddle.ParamAttr(initializer=NumpyArrayInitializer(arr1)) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr0, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr1, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr0, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr1, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -67,15 +73,19 @@ def forward(self, input): auto.shard_tensor(self.linear0.weight, PP_MESH_0, [None, None]) auto.shard_tensor(self.linear1.weight, PP_MESH_1, [None, None]) elif _global_parallel_strategy == "mp": - auto.shard_tensor(self.linear0.weight, _global_process_mesh, - [None, "x"]) - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - ["x", None]) + auto.shard_tensor( + self.linear0.weight, _global_process_mesh, [None, "x"] + ) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, ["x", None] + ) elif _global_parallel_strategy == "dp": - auto.shard_tensor(self.linear0.weight, _global_process_mesh, - [None, None]) - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - [None, None]) + auto.shard_tensor( + self.linear0.weight, _global_process_mesh, [None, None] + ) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, [None, None] + ) out = self.norm(input) out = self.linear0(out) @@ -85,16 +95,17 @@ def forward(self, input): def mlp_forward(train_program, start_program): - with static.program_guard(train_program,start_program), \ - utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 64 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) if _global_parallel_strategy == "pp": auto.shard_tensor(input, PP_MESH_0, [None, None]) @@ -104,9 +115,11 @@ def mlp_forward(train_program, start_program): elif _global_parallel_strategy == "mp": auto.shard_tensor(input, _global_process_mesh, [None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) loss = paddle.mean(error_cost) @@ -119,28 +132,35 @@ def get_distributed_program(): dist_strategy = fleet.DistributedStrategy() dist_strategy.semi_auto = True fleet.init(is_collective=True, strategy=dist_strategy) - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) optimizer = paddle.fluid.optimizer.SGDOptimizer(learning_rate=0.01) optimizer = fleet.distributed_optimizer(optimizer) _, _, dist_startup_prog, dist_main_prog = optimizer.minimize( - loss, startup_program) + loss, startup_program + ) return dist_main_prog, dist_startup_prog, loss class TestMLPAutoConvert(unittest.TestCase): - def setUp(self): paddle.seed(2021) random.seed(2021) np.random.seed(2021) def tearDown(self): - os.remove("./model_state_rank{}.pdmodel".format( - str(paddle.distributed.get_rank()))) - os.remove("./dist_attr_rank{}.pdattr".format( - str(paddle.distributed.get_rank()))) + os.remove( + "./model_state_rank{}.pdmodel".format( + str(paddle.distributed.get_rank()) + ) + ) + os.remove( + "./dist_attr_rank{}.pdattr".format( + str(paddle.distributed.get_rank()) + ) + ) def test_mlp_mp2pp(self): set_default_distributed_context(None) @@ -159,16 +179,18 @@ def test_mlp_mp2pp(self): for step in range(20): if step == 10: - save_distributed_checkpoint(dist_main_prog, - ".", - dist_attr_path=".") - - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + save_distributed_checkpoint( + dist_main_prog, ".", dist_attr_path="." + ) + + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) last_res = res[0] set_default_distributed_context(None) @@ -179,50 +201,65 @@ def test_mlp_mp2pp(self): global PP_MESH_1 PP_MESH_1 = auto.ProcessMesh(mesh=[1], dim_names=["pp1"]) - dist_main_prog_load, dist_start_prog_load, loss_load = get_distributed_program( - ) + ( + dist_main_prog_load, + dist_start_prog_load, + loss_load, + ) = get_distributed_program() place = paddle.set_device("gpu") exe = paddle.static.Executor(place) exe.run(dist_start_prog_load) ckpt_path = [ - "./model_state_rank0.pdmodel", "./model_state_rank1.pdmodel" + "./model_state_rank0.pdmodel", + "./model_state_rank1.pdmodel", ] dist_attr_path = [ - "./dist_attr_rank0.pdattr", "./dist_attr_rank1.pdattr" + "./dist_attr_rank0.pdattr", + "./dist_attr_rank1.pdattr", ] - load_checkpoint_into_program(ckpt_path, dist_attr_path, - dist_main_prog_load) + load_checkpoint_into_program( + ckpt_path, dist_attr_path, dist_main_prog_load + ) for step in range(10, 20): if paddle.distributed.get_rank() in [0]: - res = exe.run(dist_main_prog_load, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }) + res = exe.run( + dist_main_prog_load, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + ) else: - res = exe.run(dist_main_prog_load, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss_load]) + res = exe.run( + dist_main_prog_load, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss_load], + ) if paddle.distributed.get_rank() in [1]: self.assertEqual(last_res, res[0]) class TestMLPAutoConvert2(unittest.TestCase): - def setUp(self): paddle.seed(2021) random.seed(2021) np.random.seed(2021) def tearDown(self): - os.remove("./model_state_rank{}.pdmodel".format( - str(paddle.distributed.get_rank()))) - os.remove("./dist_attr_rank{}.pdattr".format( - str(paddle.distributed.get_rank()))) + os.remove( + "./model_state_rank{}.pdmodel".format( + str(paddle.distributed.get_rank()) + ) + ) + os.remove( + "./dist_attr_rank{}.pdattr".format( + str(paddle.distributed.get_rank()) + ) + ) def test_mlp_pp2mp(self): set_default_distributed_context(None) @@ -247,18 +284,22 @@ def test_mlp_pp2mp(self): save_distributed_checkpoint(dist_main_prog, ".", ".", add_info) if paddle.distributed.get_rank() in [0]: - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + ) else: - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) if paddle.distributed.get_rank() in [1]: last_res = res[0] @@ -266,41 +307,49 @@ def test_mlp_pp2mp(self): _global_parallel_strategy = "mp" _global_process_mesh = auto.ProcessMesh([0, 1], dim_names=["x"]) - dist_main_prog_load, dist_start_prog_load, loss_load = get_distributed_program( - ) + ( + dist_main_prog_load, + dist_start_prog_load, + loss_load, + ) = get_distributed_program() place = paddle.set_device("gpu") exe = paddle.static.Executor(place) exe.run(dist_start_prog_load) ckpt_path = [ - "./model_state_rank0.pdmodel", "./model_state_rank1.pdmodel" + "./model_state_rank0.pdmodel", + "./model_state_rank1.pdmodel", ] dist_attr_path = [ - "./dist_attr_rank0.pdattr", "./dist_attr_rank1.pdattr" + "./dist_attr_rank0.pdattr", + "./dist_attr_rank1.pdattr", ] param_dict, pre_dist_attr, add_info = load_distributed_checkpoint( - ckpt_path, dist_attr_path) + ckpt_path, dist_attr_path + ) batch = add_info["batch"] batch_size = add_info["batch_size"] start_index = batch * batch_size input = input[start_index:, :] label = label[start_index:, :] cur_dist_attr = get_dist_attr(dist_main_prog_load) - sliced_param_dict = merge_and_slice_parameter(param_dict, pre_dist_attr, - cur_dist_attr) + sliced_param_dict = merge_and_slice_parameter( + param_dict, pre_dist_attr, cur_dist_attr + ) load_parameter_into_program(sliced_param_dict, dist_main_prog_load) for step in range(10): - res = exe.run(dist_main_prog_load, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss_load]) + res = exe.run( + dist_main_prog_load, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss_load], + ) if paddle.distributed.get_rank() in [1]: self.assertEqual(last_res, res[0]) class TestMLPAutoConvertInvalid(unittest.TestCase): - def setUp(self): paddle.seed(2021) random.seed(2021) @@ -314,24 +363,32 @@ def test_input_invalid(self): _global_process_mesh = auto.ProcessMesh([0, 1], dim_names=["x"]) dist_main_prog, _, _ = get_distributed_program() with self.assertRaises(TypeError): - save_distributed_checkpoint(dist_main_prog, [""], [""], - addition_info=[0]) + save_distributed_checkpoint( + dist_main_prog, [""], [""], addition_info=[0] + ) with self.assertRaises(ValueError): - save_distributed_checkpoint(dist_main_prog, [""], [""], - addition_info={"step": 0}) + save_distributed_checkpoint( + dist_main_prog, [""], [""], addition_info={"step": 0} + ) with self.assertRaises(ValueError): - save_distributed_checkpoint(dist_main_prog, [""], [""], - addition_info={"batch": 0.0}) + save_distributed_checkpoint( + dist_main_prog, [""], [""], addition_info={"batch": 0.0} + ) with self.assertRaises(ValueError): - load_checkpoint_into_program(["./model_state_rank.pdmodel"], - ["./dist_attr_rank.pdattr"], - dist_main_prog) + load_checkpoint_into_program( + ["./model_state_rank.pdmodel"], + ["./dist_attr_rank.pdattr"], + dist_main_prog, + ) with self.assertRaises(ValueError): - load_distributed_checkpoint(["./model_state_rank.pdmodel"], - ["./dist_attr_rank.pdattr"]) + load_distributed_checkpoint( + ["./model_state_rank.pdmodel"], ["./dist_attr_rank.pdattr"] + ) with self.assertRaises(TypeError): - load_distributed_checkpoint({"0": "./model_state_rank.pdmodel"}, - {"1": "./dist_attr_rank.pdattr"}) + load_distributed_checkpoint( + {"0": "./model_state_rank.pdmodel"}, + {"1": "./dist_attr_rank.pdattr"}, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/auto_parallel_data_unshard.py b/python/paddle/fluid/tests/unittests/auto_parallel_data_unshard.py index e08739039c8ac..53a1250f1912e 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel_data_unshard.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel_data_unshard.py @@ -28,9 +28,7 @@ class TestDataUnshard(unittest.TestCase): - def test_dp2pp1mp1(self): - def create_model(train_program, start_program): with paddle.static.program_guard(train_program, start_program): @@ -39,7 +37,8 @@ def create_model(train_program, start_program): label = paddle.static.data(name='label', shape=[2, 8]) weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=0.02)) + initializer=nn.initializer.Normal(mean=0.0, std=0.02) + ) linear0 = nn.Linear(8, 8, weight_attr) linear1 = nn.Linear(8, 8, weight_attr) @@ -52,7 +51,8 @@ def create_model(train_program, start_program): gelu_out = F.gelu(linear0_out) linear1_out = linear1(gelu_out) error_cost = paddle.nn.functional.square_error_cost( - linear1_out, label) + linear1_out, label + ) loss = paddle.mean(error_cost) return train_program, start_program, loss, input, label @@ -60,20 +60,27 @@ def create_model(train_program, start_program): start_program = paddle.static.Program() # serial program train_program, start_program, loss, input, label = create_model( - train_program, start_program) + train_program, start_program + ) dist_strategy = fleet.DistributedStrategy() dist_strategy.semi_auto = True fleet.init(is_collective=True, strategy=dist_strategy) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) optimizer = fleet.distributed_optimizer(optimizer) - _, _, distributed_startup_program, distributed_main_program = optimizer.minimize( - loss, start_program) + ( + _, + _, + distributed_startup_program, + distributed_main_program, + ) = optimizer.minimize(loss, start_program) worker_index = paddle.distributed.get_rank() paddle.seed(worker_index + 2021) @@ -87,20 +94,20 @@ def create_model(train_program, start_program): input_data = np.array(range(2 * 8)).reshape([2, 8]).astype("float32") label_data = np.random.randint(0, 10, [2, 8]).astype("float32") - fetchs = [loss.name, 'split@RESHARD.tmp_0'] if worker_index == 0 else [ - loss.name, 'split@RESHARD.tmp_1' - ] - loss_np, shard_data_np = exe.run(distributed_main_program, - feed={ - "input": input_data, - "label": label_data - }, - fetch_list=fetchs) + fetchs = ( + [loss.name, 'split@RESHARD.tmp_0'] + if worker_index == 0 + else [loss.name, 'split@RESHARD.tmp_1'] + ) + loss_np, shard_data_np = exe.run( + distributed_main_program, + feed={"input": input_data, "label": label_data}, + fetch_list=fetchs, + ) desired = input_data[worker_index].reshape(shard_data_np.shape) np.testing.assert_allclose(shard_data_np, desired) def dp1pp1mp2(self): - def create_model(train_program, start_program): with paddle.static.program_guard(train_program, start_program): @@ -109,7 +116,8 @@ def create_model(train_program, start_program): label = paddle.static.data(name='label', shape=[8, 8]) weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=0.02)) + initializer=nn.initializer.Normal(mean=0.0, std=0.02) + ) linear0 = nn.Linear(8, 8, weight_attr) linear1 = nn.Linear(8, 8, weight_attr) @@ -124,7 +132,8 @@ def create_model(train_program, start_program): linear1_out = linear1(gelu_out) error_cost = paddle.nn.functional.square_error_cost( - linear1_out, label) + linear1_out, label + ) loss = paddle.mean(error_cost) return train_program, start_program, loss, input, label @@ -132,20 +141,27 @@ def create_model(train_program, start_program): start_program = paddle.static.Program() # serial program train_program, start_program, loss, input, label = create_model( - train_program, start_program) + train_program, start_program + ) dist_strategy = fleet.DistributedStrategy() dist_strategy.semi_auto = True fleet.init(is_collective=True, strategy=dist_strategy) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) optimizer = fleet.distributed_optimizer(optimizer) - _, _, distributed_startup_program, distributed_main_program = optimizer.minimize( - loss, start_program) + ( + _, + _, + distributed_startup_program, + distributed_main_program, + ) = optimizer.minimize(loss, start_program) worker_index = paddle.distributed.get_rank() paddle.seed(worker_index + 2021) @@ -160,12 +176,11 @@ def create_model(train_program, start_program): label_data = np.random.randint(0, 10, [8, 8]).astype("float32") fetchs = [loss.name, 'input'] - loss_np, shard_data_np = exe.run(distributed_main_program, - feed={ - "input": input_data, - "label": label_data - }, - fetch_list=fetchs) + loss_np, shard_data_np = exe.run( + distributed_main_program, + feed={"input": input_data, "label": label_data}, + fetch_list=fetchs, + ) desired = input_data.reshape(shard_data_np.shape) np.testing.assert_allclose(shard_data_np, desired) diff --git a/python/paddle/fluid/tests/unittests/auto_parallel_gpt_model.py b/python/paddle/fluid/tests/unittests/auto_parallel_gpt_model.py index 3940aa0170ba7..c4dc52ebfd2f9 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel_gpt_model.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel_gpt_model.py @@ -41,20 +41,23 @@ class MultiHeadAttention(nn.Layer): Multi-Head Attention performs multiple parallel attention to jointly attending to information from different representation subspaces. """ + Cache = collections.namedtuple("Cache", ["k", "v"]) StaticCache = collections.namedtuple("StaticCache", ["k", "v"]) - def __init__(self, - embed_dim, - num_heads, - dropout=0., - kdim=None, - vdim=None, - need_weights=False, - weight_attr=None, - bias_attr=None, - fuse=False, - mesh_idx=None): + def __init__( + self, + embed_dim, + num_heads, + dropout=0.0, + kdim=None, + vdim=None, + need_weights=False, + weight_attr=None, + bias_attr=None, + fuse=False, + mesh_idx=None, + ): super(MultiHeadAttention, self).__init__() self.embed_dim = embed_dim self.kdim = kdim if kdim is not None else embed_dim @@ -65,36 +68,43 @@ def __init__(self, self.fuse = fuse self.mesh_idx = mesh_idx self.head_dim = embed_dim // num_heads - assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" if self.fuse: assert self.kdim == embed_dim assert self.vdim == embed_dim - self.qkv_proj = nn.Linear(embed_dim, - 3 * embed_dim, - weight_attr, - bias_attr=bias_attr) + self.qkv_proj = nn.Linear( + embed_dim, 3 * embed_dim, weight_attr, bias_attr=bias_attr + ) else: - self.q_proj = nn.Linear(embed_dim, - embed_dim, - weight_attr=weight_attr, - bias_attr=bias_attr) - self.k_proj = nn.Linear(self.kdim, - embed_dim, - weight_attr=weight_attr, - bias_attr=bias_attr) - self.v_proj = nn.Linear(self.vdim, - embed_dim, - weight_attr=weight_attr, - bias_attr=bias_attr) - self.out_proj = nn.Linear(embed_dim, - embed_dim, - weight_attr=weight_attr, - bias_attr=bias_attr) + self.q_proj = nn.Linear( + embed_dim, + embed_dim, + weight_attr=weight_attr, + bias_attr=bias_attr, + ) + self.k_proj = nn.Linear( + self.kdim, + embed_dim, + weight_attr=weight_attr, + bias_attr=bias_attr, + ) + self.v_proj = nn.Linear( + self.vdim, + embed_dim, + weight_attr=weight_attr, + bias_attr=bias_attr, + ) + self.out_proj = nn.Linear( + embed_dim, embed_dim, weight_attr=weight_attr, bias_attr=bias_attr + ) def _fuse_prepare_qkv(self, query): mix_layer = self.qkv_proj(query) - mix_layer = paddle.reshape_(mix_layer, - [0, 0, self.num_heads, 3 * self.head_dim]) + mix_layer = paddle.reshape_( + mix_layer, [0, 0, self.num_heads, 3 * self.head_dim] + ) mix_layer = paddle.transpose(mix_layer, [0, 2, 1, 3]) q, k, v = paddle.split(mix_layer, num_or_sections=3, axis=-1) return q, k, v @@ -107,17 +117,21 @@ def _prepare_qkv(self, query, key, value, use_cache=False, cache=None): """ q = self.q_proj(query) if _global_parallel_strategy == "mp": - auto.shard_tensor(self.q_proj.weight, _global_process_mesh, - [None, "x"]) + auto.shard_tensor( + self.q_proj.weight, _global_process_mesh, [None, "x"] + ) elif _global_parallel_strategy == "dp_mp": - auto.shard_tensor(self.q_proj.weight, _global_process_mesh, - [None, "y"]) + auto.shard_tensor( + self.q_proj.weight, _global_process_mesh, [None, "y"] + ) elif _global_parallel_strategy == "mp_pp": - auto.shard_tensor(self.q_proj.weight, MPPP_MESH_LIST[self.mesh_idx], - [None, "x"]) + auto.shard_tensor( + self.q_proj.weight, MPPP_MESH_LIST[self.mesh_idx], [None, "x"] + ) elif _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.q_proj.weight, - DPMPPP_MESH_LIST[self.mesh_idx], [None, "y"]) + auto.shard_tensor( + self.q_proj.weight, DPMPPP_MESH_LIST[self.mesh_idx], [None, "y"] + ) q = tensor.reshape(x=q, shape=[0, 0, self.num_heads, self.head_dim]) q = tensor.transpose(x=q, perm=[0, 2, 1, 3]) @@ -146,30 +160,38 @@ def compute_kv(self, key, value): """ k = self.k_proj(key) if _global_parallel_strategy == "mp": - auto.shard_tensor(self.k_proj.weight, _global_process_mesh, - [None, "x"]) + auto.shard_tensor( + self.k_proj.weight, _global_process_mesh, [None, "x"] + ) elif _global_parallel_strategy == "dp_mp": - auto.shard_tensor(self.k_proj.weight, _global_process_mesh, - [None, "y"]) + auto.shard_tensor( + self.k_proj.weight, _global_process_mesh, [None, "y"] + ) elif _global_parallel_strategy == "mp_pp": - auto.shard_tensor(self.k_proj.weight, MPPP_MESH_LIST[self.mesh_idx], - [None, "x"]) + auto.shard_tensor( + self.k_proj.weight, MPPP_MESH_LIST[self.mesh_idx], [None, "x"] + ) elif _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.k_proj.weight, - DPMPPP_MESH_LIST[self.mesh_idx], [None, "y"]) + auto.shard_tensor( + self.k_proj.weight, DPMPPP_MESH_LIST[self.mesh_idx], [None, "y"] + ) v = self.v_proj(value) if _global_parallel_strategy == "mp": - auto.shard_tensor(self.v_proj.weight, _global_process_mesh, - [None, "x"]) + auto.shard_tensor( + self.v_proj.weight, _global_process_mesh, [None, "x"] + ) elif _global_parallel_strategy == "dp_mp": - auto.shard_tensor(self.v_proj.weight, _global_process_mesh, - [None, "y"]) + auto.shard_tensor( + self.v_proj.weight, _global_process_mesh, [None, "y"] + ) elif _global_parallel_strategy == "mp_pp": - auto.shard_tensor(self.v_proj.weight, MPPP_MESH_LIST[self.mesh_idx], - [None, "x"]) + auto.shard_tensor( + self.v_proj.weight, MPPP_MESH_LIST[self.mesh_idx], [None, "x"] + ) elif _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.v_proj.weight, - DPMPPP_MESH_LIST[self.mesh_idx], [None, "y"]) + auto.shard_tensor( + self.v_proj.weight, DPMPPP_MESH_LIST[self.mesh_idx], [None, "y"] + ) k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) k = tensor.transpose(x=k, perm=[0, 2, 1, 3]) v = tensor.reshape(x=v, shape=[0, 0, self.num_heads, self.head_dim]) @@ -190,24 +212,22 @@ def gen_cache(self, key, value=None, type=Cache): input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) v = layers.fill_constant_batch_size_like( input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) return self.Cache(k, v) else: # incremental_state with initial value, mainly for usage like UniLM return self.Cache(key, value) - def forward(self, - query, - key, - value, - attn_mask=None, - use_cache=False, - cache=None): + def forward( + self, query, key, value, attn_mask=None, use_cache=False, cache=None + ): """ Applies multi-head attention to map queries and a set of key-value pairs to outputs. @@ -221,20 +241,22 @@ def forward(self, else: q, k, v = self._prepare_qkv(query, key, value, use_cache, cache) else: - q, k, v, cache = self._prepare_qkv(query, key, value, use_cache, - cache) - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.head_dim**-0.5) + q, k, v, cache = self._prepare_qkv( + query, key, value, use_cache, cache + ) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.head_dim**-0.5 + ) if attn_mask is not None: product = product + attn_mask weights = F.softmax(product) if self.dropout: - weights = F.dropout(weights, - self.dropout, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) # combine heads out = tensor.transpose(out, perm=[0, 2, 1, 3]) @@ -242,17 +264,23 @@ def forward(self, # project to output out = self.out_proj(out) if _global_parallel_strategy == "mp": - auto.shard_tensor(self.out_proj.weight, _global_process_mesh, - ["x", None]) + auto.shard_tensor( + self.out_proj.weight, _global_process_mesh, ["x", None] + ) elif _global_parallel_strategy == "dp_mp": - auto.shard_tensor(self.out_proj.weight, _global_process_mesh, - ["y", None]) + auto.shard_tensor( + self.out_proj.weight, _global_process_mesh, ["y", None] + ) elif _global_parallel_strategy == "mp_pp": - auto.shard_tensor(self.out_proj.weight, - MPPP_MESH_LIST[self.mesh_idx], ["x", None]) + auto.shard_tensor( + self.out_proj.weight, MPPP_MESH_LIST[self.mesh_idx], ["x", None] + ) elif _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.out_proj.weight, - DPMPPP_MESH_LIST[self.mesh_idx], ["y", None]) + auto.shard_tensor( + self.out_proj.weight, + DPMPPP_MESH_LIST[self.mesh_idx], + ["y", None], + ) outs = [out] if self.need_weights: @@ -279,13 +307,15 @@ def __init__(self, decoder_layers, num_layers, norm=None, hidden_size=None): raise ValueError("Only support LayerNorm") self.checkpoints = [] - def forward(self, - tgt, - memory, - tgt_mask=None, - memory_mask=None, - use_cache=False, - cache=None): + def forward( + self, + tgt, + memory, + tgt_mask=None, + memory_mask=None, + use_cache=False, + cache=None, + ): """ Applies a stack of N Transformer decoder layers on inputs. If `norm` is provided, also applies layer normalization on the output of last decoder @@ -295,134 +325,170 @@ def forward(self, new_caches = [] self.checkpoints = [] if _global_parallel_strategy == "pp": - auto.shard_tensor(output, PP_MESH_LIST[0], - [None for i in range(len(output.shape))]) + auto.shard_tensor( + output, + PP_MESH_LIST[0], + [None for i in range(len(output.shape))], + ) if _global_parallel_strategy == "dp_pp": - auto.shard_tensor(output, DPPP_MESH_LIST[0], ["x"] + - [None for i in range(len(output.shape) - 1)]) + auto.shard_tensor( + output, + DPPP_MESH_LIST[0], + ["x"] + [None for i in range(len(output.shape) - 1)], + ) if _global_parallel_strategy == "mp_pp": - auto.shard_tensor(output, MPPP_MESH_LIST[0], - [None for i in range(len(output.shape))]) + auto.shard_tensor( + output, + MPPP_MESH_LIST[0], + [None for i in range(len(output.shape))], + ) if _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(output, DPMPPP_MESH_LIST[0], ["x"] + - [None for i in range(len(output.shape) - 1)]) + auto.shard_tensor( + output, + DPMPPP_MESH_LIST[0], + ["x"] + [None for i in range(len(output.shape) - 1)], + ) for i, mod in enumerate(self.layers): if cache is None: if use_cache: if _global_parallel_strategy == "pp": output, new_cache = auto.shard_op( - mod, PP_MESH_LIST[mod.mesh_idx])(output, memory, - tgt_mask, - use_cache, cache) + mod, PP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, PP_MESH_LIST[mod.mesh_idx], - [None for i in range(len(output.shape))]) + output, + PP_MESH_LIST[mod.mesh_idx], + [None for i in range(len(output.shape))], + ) elif _global_parallel_strategy == "dp_pp": output, new_cache = auto.shard_op( - mod, DPPP_MESH_LIST[mod.mesh_idx])(output, memory, - tgt_mask, - use_cache, cache) + mod, DPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, DPPP_MESH_LIST[mod.mesh_idx], ["x"] + - [None for i in range(len(output.shape) - 1)]) + output, + DPPP_MESH_LIST[mod.mesh_idx], + ["x"] + + [None for i in range(len(output.shape) - 1)], + ) elif _global_parallel_strategy == "mp_pp": output, new_cache = auto.shard_op( - mod, MPPP_MESH_LIST[mod.mesh_idx])(output, memory, - tgt_mask, - use_cache, cache) + mod, MPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, MPPP_MESH_LIST[mod.mesh_idx], - [None for i in range(len(output.shape))]) + output, + MPPP_MESH_LIST[mod.mesh_idx], + [None for i in range(len(output.shape))], + ) elif _global_parallel_strategy == "dp_mp_pp": output, new_cache = auto.shard_op( - mod, - DPMPPP_MESH_LIST[mod.mesh_idx])(output, memory, - tgt_mask, use_cache, - cache) + mod, DPMPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, DPMPPP_MESH_LIST[mod.mesh_idx], - [None for i in range(len(output.shape))]) + output, + DPMPPP_MESH_LIST[mod.mesh_idx], + [None for i in range(len(output.shape))], + ) else: - output, new_cache = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache) + output, new_cache = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache, + ) new_caches.append(new_cache) else: if _global_parallel_strategy == "pp": output = auto.shard_op(mod, PP_MESH_LIST[mod.mesh_idx])( - output, memory, tgt_mask, use_cache, cache) + output, memory, tgt_mask, use_cache, cache + ) auto.shard_tensor( - output, PP_MESH_LIST[mod.mesh_idx], - [None for i in range(len(output.shape))]) + output, + PP_MESH_LIST[mod.mesh_idx], + [None for i in range(len(output.shape))], + ) elif _global_parallel_strategy == "dp_pp": output = auto.shard_op( - mod, DPPP_MESH_LIST[mod.mesh_idx])(output, memory, - tgt_mask, - use_cache, cache) + mod, DPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, DPPP_MESH_LIST[mod.mesh_idx], ["x"] + - [None for i in range(len(output.shape) - 1)]) + output, + DPPP_MESH_LIST[mod.mesh_idx], + ["x"] + + [None for i in range(len(output.shape) - 1)], + ) elif _global_parallel_strategy == "mp_pp": output = auto.shard_op( - mod, MPPP_MESH_LIST[mod.mesh_idx])(output, memory, - tgt_mask, - use_cache, cache) + mod, MPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, MPPP_MESH_LIST[mod.mesh_idx], - [None for i in range(len(output.shape))]) + output, + MPPP_MESH_LIST[mod.mesh_idx], + [None for i in range(len(output.shape))], + ) elif _global_parallel_strategy == "dp_mp_pp": - output = auto.shard_op(mod, - DPMPPP_MESH_LIST[mod.mesh_idx])( - output, memory, tgt_mask, - use_cache, cache) + output = auto.shard_op( + mod, DPMPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, DPMPPP_MESH_LIST[mod.mesh_idx], ["x"] + - [None for i in range(len(output.shape) - 1)]) + output, + DPMPPP_MESH_LIST[mod.mesh_idx], + ["x"] + + [None for i in range(len(output.shape) - 1)], + ) else: - output = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache) + output = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache, + ) else: if _global_parallel_strategy == "pp": output, new_cache = auto.shard_op( - mod, - PP_MESH_LIST[mod.mesh_idx])(output, memory, tgt_mask, - use_cache, cache) - auto.shard_tensor(output, PP_MESH_LIST[mod.mesh_idx], - [None for i in range(len(output.shape))]) + mod, PP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) + auto.shard_tensor( + output, + PP_MESH_LIST[mod.mesh_idx], + [None for i in range(len(output.shape))], + ) elif _global_parallel_strategy == "dp_pp": output, new_cache = auto.shard_op( - mod, - DPPP_MESH_LIST[mod.mesh_idx])(output, memory, tgt_mask, - use_cache, cache) + mod, DPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, DPPP_MESH_LIST[mod.mesh_idx], - ["x"] + [None for i in range(len(output.shape) - 1)]) + output, + DPPP_MESH_LIST[mod.mesh_idx], + ["x"] + [None for i in range(len(output.shape) - 1)], + ) elif _global_parallel_strategy == "mp_pp": output, new_cache = auto.shard_op( - mod, - MPPP_MESH_LIST[mod.mesh_idx])(output, memory, tgt_mask, - use_cache, cache) - auto.shard_tensor(output, MPPP_MESH_LIST[mod.mesh_idx], - [None for i in range(len(output.shape))]) + mod, MPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) + auto.shard_tensor( + output, + MPPP_MESH_LIST[mod.mesh_idx], + [None for i in range(len(output.shape))], + ) elif _global_parallel_strategy == "dp_mp_pp": output, new_cache = auto.shard_op( - mod, DPMPPP_MESH_LIST[mod.mesh_idx])(output, memory, - tgt_mask, - use_cache, cache) + mod, DPMPPP_MESH_LIST[mod.mesh_idx] + )(output, memory, tgt_mask, use_cache, cache) auto.shard_tensor( - output, DPMPPP_MESH_LIST[mod.mesh_idx], - ["x"] + [None for i in range(len(output.shape) - 1)]) + output, + DPMPPP_MESH_LIST[mod.mesh_idx], + ["x"] + [None for i in range(len(output.shape) - 1)], + ) else: - output, new_cache = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache[i]) + output, new_cache = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache[i], + ) new_caches.append(new_cache) self.checkpoints.append(output.name) if self.norm is not None: @@ -436,7 +502,7 @@ def gen_cache(self, memory, do_zip=False): produced by `TransformerDecoderLayer.gen_cache`. See `TransformerDecoderLayer.gen_cache` for more details. If `do_zip` is True, apply `zip` on these tuples to get a list with two elements. - """ + """ cache = [layer.gen_cache(memory) for layer in self.layers] if do_zip: cache = list(zip(*cache)) @@ -449,18 +515,20 @@ class TransformerDecoderLayer(nn.Layer): It contains multiheadattention and some linear layers. """ - def __init__(self, - d_model, - nhead, - dim_feedforward, - dropout=0.1, - activation="gelu", - attn_dropout=None, - act_dropout=None, - normalize_before=True, - weight_attr=None, - bias_attr=None, - mesh_idx=None): + def __init__( + self, + d_model, + nhead, + dim_feedforward, + dropout=0.1, + activation="gelu", + attn_dropout=None, + act_dropout=None, + normalize_before=True, + weight_attr=None, + bias_attr=None, + mesh_idx=None, + ): self._config = locals() self._config.pop("self") self._config.pop("__class__", None) # py3 @@ -471,20 +539,20 @@ def __init__(self, self.normalize_before = normalize_before weight_attrs = _convert_param_attr_to_list(weight_attr, 3) bias_attrs = _convert_param_attr_to_list(bias_attr, 3) - self.self_attn = MultiHeadAttention(d_model, - nhead, - dropout=attn_dropout, - weight_attr=weight_attrs[0], - bias_attr=bias_attrs[0], - mesh_idx=self.mesh_idx) - self.linear1 = nn.Linear(d_model, - dim_feedforward, - weight_attrs[2], - bias_attr=bias_attrs[2]) - self.linear2 = nn.Linear(dim_feedforward, - d_model, - weight_attrs[2], - bias_attr=bias_attrs[2]) + self.self_attn = MultiHeadAttention( + d_model, + nhead, + dropout=attn_dropout, + weight_attr=weight_attrs[0], + bias_attr=bias_attrs[0], + mesh_idx=self.mesh_idx, + ) + self.linear1 = nn.Linear( + d_model, dim_feedforward, weight_attrs[2], bias_attr=bias_attrs[2] + ) + self.linear2 = nn.Linear( + dim_feedforward, d_model, weight_attrs[2], bias_attr=bias_attrs[2] + ) self.norm1 = nn.LayerNorm(d_model, epsilon=1e-5) self.norm2 = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout1 = nn.Dropout(dropout, mode="upscale_in_train") @@ -498,8 +566,9 @@ def forward(self, tgt, memory, tgt_mask=None, use_cache=False, cache=None): if use_cache is False: tgt = self.self_attn(tgt, tgt, tgt, tgt_mask, use_cache, cache) else: - tgt, incremental_cache = self.self_attn(tgt, tgt, tgt, tgt_mask, - use_cache, cache) + tgt, incremental_cache = self.self_attn( + tgt, tgt, tgt, tgt_mask, use_cache, cache + ) tgt = residual + self.dropout1(tgt) if not self.normalize_before: tgt = self.norm1(tgt) @@ -507,40 +576,54 @@ def forward(self, tgt, memory, tgt_mask=None, use_cache=False, cache=None): if self.normalize_before: tgt = self.norm2(tgt) if _global_parallel_strategy == "mp": - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - [None, "x"]) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, [None, "x"] + ) elif _global_parallel_strategy == "dp_mp": - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - [None, "y"]) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, [None, "y"] + ) elif _global_parallel_strategy == "mp_pp": - auto.shard_tensor(self.linear1.weight, - MPPP_MESH_LIST[self.mesh_idx], [None, "x"]) + auto.shard_tensor( + self.linear1.weight, MPPP_MESH_LIST[self.mesh_idx], [None, "x"] + ) if _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.linear1.weight, - DPMPPP_MESH_LIST[self.mesh_idx], [None, "y"]) + auto.shard_tensor( + self.linear1.weight, + DPMPPP_MESH_LIST[self.mesh_idx], + [None, "y"], + ) if _global_parallel_strategy == "mp": - auto.shard_tensor(self.linear2.weight, _global_process_mesh, - ["x", None]) + auto.shard_tensor( + self.linear2.weight, _global_process_mesh, ["x", None] + ) elif _global_parallel_strategy == "dp_mp": - auto.shard_tensor(self.linear2.weight, _global_process_mesh, - ["y", None]) + auto.shard_tensor( + self.linear2.weight, _global_process_mesh, ["y", None] + ) elif _global_parallel_strategy == "mp_pp": - auto.shard_tensor(self.linear2.weight, - MPPP_MESH_LIST[self.mesh_idx], ["x", None]) + auto.shard_tensor( + self.linear2.weight, MPPP_MESH_LIST[self.mesh_idx], ["x", None] + ) elif _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.linear2.weight, - DPMPPP_MESH_LIST[self.mesh_idx], ["y", None]) + auto.shard_tensor( + self.linear2.weight, + DPMPPP_MESH_LIST[self.mesh_idx], + ["y", None], + ) tgt = self.dropout2( - self.linear2(F.gelu(self.linear1(tgt), approximate=True))) + self.linear2(F.gelu(self.linear1(tgt), approximate=True)) + ) tgt = residual + tgt if not self.normalize_before: tgt = self.norm2(tgt) return tgt if use_cache is False else (tgt, incremental_cache) def gen_cache(self, memory): - incremental_cache = self.self_attn.gen_cache(memory, - type=self.self_attn.Cache) + incremental_cache = self.self_attn.gen_cache( + memory, type=self.self_attn.Cache + ) return incremental_cache @@ -549,26 +632,36 @@ class GPTEmbeddings(nn.Layer): Include embeddings from word, position and token_type embeddings """ - def __init__(self, - vocab_size, - hidden_size=768, - hidden_dropout_prob=0.1, - max_position_embeddings=512, - type_vocab_size=16, - initializer_range=0.02): + def __init__( + self, + vocab_size, + hidden_size=768, + hidden_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + ): super(GPTEmbeddings, self).__init__() self.word_embeddings = nn.Embedding( vocab_size, hidden_size, - weight_attr=paddle.ParamAttr(name="word_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, std=initializer_range))) + weight_attr=paddle.ParamAttr( + name="word_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ), + ), + ) self.position_embeddings = nn.Embedding( max_position_embeddings, hidden_size, - weight_attr=paddle.ParamAttr(name="pos_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, std=initializer_range))) + weight_attr=paddle.ParamAttr( + name="pos_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ), + ), + ) self.dropout = nn.Dropout(hidden_dropout_prob) def forward(self, input_ids, position_ids=None): @@ -578,17 +671,21 @@ def forward(self, input_ids, position_ids=None): position_ids = seq_length - ones input_embedings = self.word_embeddings(input_ids) if _global_parallel_strategy == "mp": - auto.shard_tensor(self.word_embeddings.weight, _global_process_mesh, - ["x", None]) + auto.shard_tensor( + self.word_embeddings.weight, _global_process_mesh, ["x", None] + ) elif _global_parallel_strategy == "dp_mp": - auto.shard_tensor(self.word_embeddings.weight, _global_process_mesh, - ["y", None]) + auto.shard_tensor( + self.word_embeddings.weight, _global_process_mesh, ["y", None] + ) elif _global_parallel_strategy == "mp_pp": - auto.shard_tensor(self.word_embeddings.weight, MPPP_MESH_LIST[0], - ["x", None]) + auto.shard_tensor( + self.word_embeddings.weight, MPPP_MESH_LIST[0], ["x", None] + ) elif _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.word_embeddings.weight, DPMPPP_MESH_LIST[0], - ["y", None]) + auto.shard_tensor( + self.word_embeddings.weight, DPMPPP_MESH_LIST[0], ["y", None] + ) position_embeddings = self.position_embeddings(position_ids) embeddings = input_embedings + position_embeddings @@ -601,36 +698,42 @@ class GPTModel(nn.Layer): The base model of gpt. """ - def __init__(self, - vocab_size=50304, - hidden_size=1024, - num_hidden_layers=24, - num_attention_heads=16, - intermediate_size=4096, - hidden_act="gelu", - hidden_dropout_prob=0., - attention_probs_dropout_prob=0., - max_position_embeddings=512, - type_vocab_size=16, - initializer_range=0.02, - pad_token_id=0, - eos_token_id=7, - bos_token_id=0, - eol_token_id=3, - pp_degree=None): + def __init__( + self, + vocab_size=50304, + hidden_size=1024, + num_hidden_layers=24, + num_attention_heads=16, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + pad_token_id=0, + eos_token_id=7, + bos_token_id=0, + eol_token_id=3, + pp_degree=None, + ): super(GPTModel, self).__init__() self.pad_token_id = pad_token_id self.initializer_range = initializer_range self.hidden_size = hidden_size self.vocab_size = vocab_size self.layer_per_stage = None - self.pipline_mode = (pp_degree is not None and pp_degree > 1) + self.pipline_mode = pp_degree is not None and pp_degree > 1 if self.pipline_mode: self.layer_per_stage = num_hidden_layers // pp_degree - self.embeddings = GPTEmbeddings(vocab_size, hidden_size, - hidden_dropout_prob, - max_position_embeddings, - type_vocab_size, self.initializer_range) + self.embeddings = GPTEmbeddings( + vocab_size, + hidden_size, + hidden_dropout_prob, + max_position_embeddings, + type_vocab_size, + self.initializer_range, + ) decoder_layers = nn.LayerList() for i in range(num_hidden_layers): mesh_index = None @@ -638,59 +741,82 @@ def __init__(self, if self.layer_per_stage is not None: mesh_index = i // self.layer_per_stage decoder_layers.append( - DecoderLayer(d_model=hidden_size, - nhead=num_attention_heads, - dim_feedforward=intermediate_size, - dropout=hidden_dropout_prob, - activation=hidden_act, - attn_dropout=attention_probs_dropout_prob, - act_dropout=hidden_dropout_prob, - weight_attr=paddle.ParamAttr( - initializer=nn.initializer.Normal( - mean=0.0, std=self.initializer_range)), - bias_attr=None, - mesh_idx=mesh_index)) + DecoderLayer( + d_model=hidden_size, + nhead=num_attention_heads, + dim_feedforward=intermediate_size, + dropout=hidden_dropout_prob, + activation=hidden_act, + attn_dropout=attention_probs_dropout_prob, + act_dropout=hidden_dropout_prob, + weight_attr=paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ) + ), + bias_attr=None, + mesh_idx=mesh_index, + ) + ) Decoder = TransformerDecoder - self.decoder = Decoder(decoder_layers, - num_hidden_layers, - norm="LayerNorm", - hidden_size=hidden_size) + self.decoder = Decoder( + decoder_layers, + num_hidden_layers, + norm="LayerNorm", + hidden_size=hidden_size, + ) self.checkpoints = [] - def forward(self, - input_ids, - position_ids=None, - attention_mask=None, - use_cache=False, - cache=None): + def forward( + self, + input_ids, + position_ids=None, + attention_mask=None, + use_cache=False, + cache=None, + ): self.checkpoints = [] if position_ids is None: past_length = 0 if cache is not None: past_length = paddle.shape(cache[0].k)[-2] - position_ids = paddle.arange(past_length, - paddle.shape(input_ids)[-1] + - past_length, - dtype='int64') + position_ids = paddle.arange( + past_length, + paddle.shape(input_ids)[-1] + past_length, + dtype='int64', + ) position_ids = position_ids.unsqueeze(0) position_ids = paddle.fluid.layers.expand_as( - position_ids, input_ids) - embedding_output = self.embeddings(input_ids=input_ids, - position_ids=position_ids) + position_ids, input_ids + ) + embedding_output = self.embeddings( + input_ids=input_ids, position_ids=position_ids + ) if _global_parallel_strategy == "pp": - auto.shard_tensor(input_ids, PP_MESH_LIST[0], - [None for i in range(len(input_ids.shape))]) + auto.shard_tensor( + input_ids, + PP_MESH_LIST[0], + [None for i in range(len(input_ids.shape))], + ) if _global_parallel_strategy == "dp_pp": - auto.shard_tensor(input_ids, DPPP_MESH_LIST[0], ["x"] + - [None for i in range(len(input_ids.shape) - 1)]) + auto.shard_tensor( + input_ids, + DPPP_MESH_LIST[0], + ["x"] + [None for i in range(len(input_ids.shape) - 1)], + ) if _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(input_ids, DPMPPP_MESH_LIST[0], ["x"] + - [None for i in range(len(input_ids.shape) - 1)]) - encoder_outputs = self.decoder(embedding_output, - memory=None, - tgt_mask=attention_mask, - use_cache=use_cache, - cache=cache) + auto.shard_tensor( + input_ids, + DPMPPP_MESH_LIST[0], + ["x"] + [None for i in range(len(input_ids.shape) - 1)], + ) + encoder_outputs = self.decoder( + embedding_output, + memory=None, + tgt_mask=attention_mask, + use_cache=use_cache, + cache=cache, + ) self.checkpoints.extend(self.decoder.checkpoints) return encoder_outputs @@ -711,22 +837,26 @@ def __init__( super(GPTForPretraining, self).__init__() self.gpt = gpt - def forward(self, - input_ids, - position_ids=None, - attention_mask=None, - masked_positions=None, - use_cache=False, - cache=None): + def forward( + self, + input_ids, + position_ids=None, + attention_mask=None, + masked_positions=None, + use_cache=False, + cache=None, + ): input_ids.stop_gradient = True position_ids.stop_gradient = True attention_mask.stop_gradient = True - outputs = self.gpt(input_ids, - position_ids=position_ids, - attention_mask=attention_mask, - use_cache=use_cache, - cache=cache) + outputs = self.gpt( + input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + use_cache=use_cache, + cache=cache, + ) if use_cache: encoder_outputs, cached_kvs = outputs[:2] else: @@ -766,8 +896,9 @@ def forward(self, w_dims_mapping = ["y"] + [None for i in range(len(w.shape) - 1)] if mesh: - matmul = auto.shard_op(paddle.matmul, mesh, - [x_dims_mapping, w_dims_mapping, None]) + matmul = auto.shard_op( + paddle.matmul, mesh, [x_dims_mapping, w_dims_mapping, None] + ) logits = matmul(x, w, transpose_y=True) else: logits = paddle.matmul(x, w, transpose_y=True) @@ -795,26 +926,31 @@ def forward(self, prediction_scores, masked_lm_labels, loss_mask): mesh = None if _global_parallel_strategy == "dp": mesh = _global_process_mesh - dims_mapping = ["x" - ] + [None for i in range(len(loss_mask.shape) - 1)] + dims_mapping = ["x"] + [ + None for i in range(len(loss_mask.shape) - 1) + ] elif _global_parallel_strategy == "dp_mp": mesh = _global_process_mesh - dims_mapping = ["x" - ] + [None for i in range(len(loss_mask.shape) - 1)] + dims_mapping = ["x"] + [ + None for i in range(len(loss_mask.shape) - 1) + ] elif _global_parallel_strategy == "dp_pp": mesh = DPPP_MESH_LIST[-1] - dims_mapping = ["x" - ] + [None for i in range(len(loss_mask.shape) - 1)] + dims_mapping = ["x"] + [ + None for i in range(len(loss_mask.shape) - 1) + ] elif _global_parallel_strategy == "dp_mp_pp": mesh = DPMPPP_MESH_LIST[-1] - dims_mapping = ["x" - ] + [None for i in range(len(loss_mask.shape) - 1)] + dims_mapping = ["x"] + [ + None for i in range(len(loss_mask.shape) - 1) + ] if mesh: auto.shard_tensor(loss_mask, mesh, dims_mapping) - masked_lm_loss = self.loss_func(prediction_scores, - masked_lm_labels.unsqueeze(2)) + masked_lm_loss = self.loss_func( + prediction_scores, masked_lm_labels.unsqueeze(2) + ) loss_mask = loss_mask.reshape([-1]) masked_lm_loss = paddle.sum(masked_lm_loss.reshape([-1]) * loss_mask) total_loss = masked_lm_loss / loss_mask.sum() diff --git a/python/paddle/fluid/tests/unittests/auto_parallel_save_load.py b/python/paddle/fluid/tests/unittests/auto_parallel_save_load.py index 0fc773cc9bc00..ce8bc690a95b8 100644 --- a/python/paddle/fluid/tests/unittests/auto_parallel_save_load.py +++ b/python/paddle/fluid/tests/unittests/auto_parallel_save_load.py @@ -27,7 +27,10 @@ from paddle.distributed import fleet from paddle.fluid.initializer import NumpyArrayInitializer -from paddle.distributed.auto_parallel.utils import save_distributed_checkpoint, load_checkpoint_into_program +from paddle.distributed.auto_parallel.utils import ( + save_distributed_checkpoint, + load_checkpoint_into_program, +) paddle.enable_static() _global_parallel_strategy = None @@ -37,11 +40,9 @@ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=64, - intermediate_size=4 * 64, - initializer_range=0.02): + def __init__( + self, hidden_size=64, intermediate_size=4 * 64, initializer_range=0.02 + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size @@ -50,14 +51,12 @@ def __init__(self, weight_attr = paddle.ParamAttr(initializer=NumpyArrayInitializer(arr)) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -65,15 +64,19 @@ def forward(self, input): auto.shard_tensor(self.linear0.weight, PP_MESH_0, [None, None]) auto.shard_tensor(self.linear1.weight, PP_MESH_1, [None, None]) elif _global_parallel_strategy == "mp": - auto.shard_tensor(self.linear0.weight, _global_process_mesh, - [None, "x"]) - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - ["x", None]) + auto.shard_tensor( + self.linear0.weight, _global_process_mesh, [None, "x"] + ) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, ["x", None] + ) elif _global_parallel_strategy == "dp": - auto.shard_tensor(self.linear0.weight, _global_process_mesh, - [None, None]) - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - [None, None]) + auto.shard_tensor( + self.linear0.weight, _global_process_mesh, [None, None] + ) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, [None, None] + ) out = self.norm(input) out = self.linear0(out) @@ -84,17 +87,18 @@ def forward(self, input): def mlp_forward(train_program, start_program): - with static.program_guard(train_program,start_program), \ - utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 64 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) if _global_parallel_strategy == "pp": auto.shard_tensor(input, PP_MESH_0, [None, None]) @@ -104,9 +108,11 @@ def mlp_forward(train_program, start_program): elif _global_parallel_strategy == "mp": auto.shard_tensor(input, _global_process_mesh, [None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -123,19 +129,20 @@ def get_distributed_program(): dist_strategy.semi_auto = True fleet.init(is_collective=True, strategy=dist_strategy) - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) optimizer = paddle.fluid.optimizer.SGDOptimizer(learning_rate=0.01) optimizer = fleet.distributed_optimizer(optimizer) _, _, dist_startup_prog, dist_main_prog = optimizer.minimize( - loss, startup_program) + loss, startup_program + ) return dist_main_prog, dist_startup_prog, loss class TestMLPSaveLoad(unittest.TestCase): - def setUp(self): paddle.seed(2021) random.seed(2021) @@ -160,30 +167,34 @@ def test_mlp_dp(self): os.makedirs(path, exist_ok=True) save_distributed_checkpoint(dist_main_prog, path, path) - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) last_res = res[0] ckpt_path = [ "./output_dp0/model_state_rank0.pdmodel", - "./output_dp1/model_state_rank1.pdmodel" + "./output_dp1/model_state_rank1.pdmodel", ] dist_attr_path = [ "./output_dp0/dist_attr_rank0.pdattr", - "./output_dp1/dist_attr_rank1.pdattr" + "./output_dp1/dist_attr_rank1.pdattr", ] load_checkpoint_into_program(ckpt_path, dist_attr_path, dist_main_prog) for step in range(10, 20): - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) self.assertEqual(last_res, res[0]) shutil.rmtree("./output_dp{}".format(paddle.distributed.get_rank())) @@ -208,30 +219,34 @@ def test_mlp_mp(self): os.makedirs(path, exist_ok=True) save_distributed_checkpoint(dist_main_prog, path, path) - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) last_res = res[0] ckpt_path = [ "./output_mp0/model_state_rank0.pdmodel", - "./output_mp1/model_state_rank1.pdmodel" + "./output_mp1/model_state_rank1.pdmodel", ] dist_attr_path = [ "./output_mp0/dist_attr_rank0.pdattr", - "./output_mp1/dist_attr_rank1.pdattr" + "./output_mp1/dist_attr_rank1.pdattr", ] load_checkpoint_into_program(ckpt_path, dist_attr_path, dist_main_prog) for step in range(10, 20): - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) self.assertEqual(last_res, res[0]) shutil.rmtree("./output_mp{}".format(paddle.distributed.get_rank())) @@ -261,45 +276,53 @@ def test_mlp_pp(self): save_distributed_checkpoint(dist_main_prog, path, path) if paddle.distributed.get_rank() in [0]: - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + ) else: - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) if paddle.distributed.get_rank() in [1]: last_res = res[0] ckpt_path = [ "./output_pp0/model_state_rank0.pdmodel", - "./output_pp1/model_state_rank1.pdmodel" + "./output_pp1/model_state_rank1.pdmodel", ] dist_attr_path = [ "./output_pp0/dist_attr_rank0.pdattr", - "./output_pp1/dist_attr_rank1.pdattr" + "./output_pp1/dist_attr_rank1.pdattr", ] load_checkpoint_into_program(ckpt_path, dist_attr_path, dist_main_prog) for step in range(10, 20): if paddle.distributed.get_rank() in [0]: - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + ) else: - res = exe.run(dist_main_prog, - feed={ - "input": input[step * 4:(step + 1) * 4, :], - "label": label[step * 4:(step + 1) * 4, :] - }, - fetch_list=[loss]) + res = exe.run( + dist_main_prog, + feed={ + "input": input[step * 4 : (step + 1) * 4, :], + "label": label[step * 4 : (step + 1) * 4, :], + }, + fetch_list=[loss], + ) if paddle.distributed.get_rank() in [1]: self.assertEqual(last_res, res[0]) diff --git a/python/paddle/fluid/tests/unittests/autograd/config.py b/python/paddle/fluid/tests/unittests/autograd/config.py index 311ca49d39555..ff2d64a43bbc9 100644 --- a/python/paddle/fluid/tests/unittests/autograd/config.py +++ b/python/paddle/fluid/tests/unittests/autograd/config.py @@ -23,27 +23,11 @@ # derivative. It's a empirical value provided by Paddle Science team. TOLERANCE = { "float32": { - "first_order_grad": { - "rtol": 1e-3, - "atol": 1e-3, - "eps": 1e-4 - }, - "second_order_grad": { - "rtol": 1e-2, - "atol": 1e-2, - "eps": 1e-2 - } + "first_order_grad": {"rtol": 1e-3, "atol": 1e-3, "eps": 1e-4}, + "second_order_grad": {"rtol": 1e-2, "atol": 1e-2, "eps": 1e-2}, }, "float64": { - "first_order_grad": { - "rtol": 1e-7, - "atol": 1e-7, - "eps": 1e-7 - }, - "second_order_grad": { - "rtol": 1e-5, - "atol": 1e-5, - "eps": 1e-5 - } - } + "first_order_grad": {"rtol": 1e-7, "atol": 1e-7, "eps": 1e-7}, + "second_order_grad": {"rtol": 1e-5, "atol": 1e-5, "eps": 1e-5}, + }, } diff --git a/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_dynamic.py b/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_dynamic.py index 054edf2ea28c1..81f1bdede6f41 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_dynamic.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_dynamic.py @@ -34,7 +34,6 @@ def make_v(f, inputs): class TestAutogradFunctional(unittest.TestCase): - @classmethod def setUpClass(cls): cls.RAW_INPUTS = { @@ -52,8 +51,9 @@ def setUp(self): def gen_input(self, inp, stop_gradient=False): if isinstance(inp, paddle.Tensor): return inp - return paddle.to_tensor(self.RAW_INPUTS[inp], - stop_gradient=stop_gradient) + return paddle.to_tensor( + self.RAW_INPUTS[inp], stop_gradient=stop_gradient + ) def gen_inputs(self, inputs): if isinstance(inputs, list): @@ -62,13 +62,9 @@ def gen_inputs(self, inputs): inputs = [self.gen_input(inputs)] return inputs - def gen_test_pairs(self, - func, - inputs, - v=None, - create_graph=False, - allow_unused=False): - + def gen_test_pairs( + self, func, inputs, v=None, create_graph=False, allow_unused=False + ): def vjp_test(): nonlocal v xs = self.gen_inputs(inputs) @@ -86,27 +82,27 @@ def grad_test(): v = self.gen_inputs(v) outputs = func(*xs) if v is not None: - inputs_grad = paddle.grad(outputs, - xs, - v, - create_graph=create_graph, - allow_unused=allow_unused) + inputs_grad = paddle.grad( + outputs, + xs, + v, + create_graph=create_graph, + allow_unused=allow_unused, + ) else: - inputs_grad = paddle.grad(outputs, - xs, - create_graph=create_graph, - allow_unused=allow_unused) + inputs_grad = paddle.grad( + outputs, + xs, + create_graph=create_graph, + allow_unused=allow_unused, + ) return outputs, inputs_grad return vjp_test, grad_test - def gen_jvp_tests(self, - func, - inputs, - v=None, - create_graph=False, - allow_unused=False): - + def gen_jvp_tests( + self, func, inputs, v=None, create_graph=False, allow_unused=False + ): def jvp_test(): nonlocal v xs = self.gen_inputs(inputs) @@ -117,13 +113,15 @@ def jvp_test(): xs, v, create_graph=create_graph, - allow_unused=allow_unused) + allow_unused=allow_unused, + ) else: outputs, outputs_grad = paddle.incubate.autograd.jvp( func, xs, create_graph=create_graph, - allow_unused=allow_unused) + allow_unused=allow_unused, + ) return outputs, outputs_grad return jvp_test @@ -144,7 +142,6 @@ def check_results(self, ref, res): class TestVJP(TestAutogradFunctional): - def func_vjp_i1o1(self): test_cases = [ [reduce, 'A'], # noqa @@ -222,20 +219,29 @@ def test_all_cases(self): def test_input_single_tensor(self): self.assertIsInstance( paddle.incubate.autograd.vjp(paddle.tanh, paddle.rand((3, 4)))[1], - paddle.fluid.framework.Variable) + paddle.fluid.framework.Variable, + ) @utils.place(config.DEVICES) @utils.parameterize( (utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'expected_exception'), - (('v_shape_not_equal_ys', utils.square, np.random.rand(3), - np.random.rand(1), RuntimeError), )) + ( + ( + 'v_shape_not_equal_ys', + utils.square, + np.random.rand(3), + np.random.rand(1), + RuntimeError, + ), + ), +) class TestVJPException(unittest.TestCase): - def func_vjp(self): with self.assertRaises(self.expected_exception): - paddle.incubate.autograd.vjp(self.fun, paddle.to_tensor(self.xs), - paddle.to_tensor(self.v)) + paddle.incubate.autograd.vjp( + self.fun, paddle.to_tensor(self.xs), paddle.to_tensor(self.v) + ) def test_all_cases(self): with _test_eager_guard(): @@ -245,7 +251,8 @@ def test_all_cases(self): def jac(grad_fn, f, inputs): assert grad_fn in [ - paddle.incubate.autograd.vjp, paddle.incubate.autograd.jvp + paddle.incubate.autograd.vjp, + paddle.incubate.autograd.jvp, ] if grad_fn is paddle.incubate.autograd.jvp: vs = [paddle.zeros_like(x) for x in inputs] @@ -274,7 +281,6 @@ def jac(grad_fn, f, inputs): class TestJVP(TestAutogradFunctional): - def func_jvp_i1o1(self): test_cases = [ [reduce, 'A'], # noqa @@ -330,36 +336,58 @@ def test_all_cases(self): @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'func', 'xs'), ( - ('1d_in_1d_out', utils.square, np.array([2., 3.])), - ('3d_in_3d_out', utils.square, np.random.rand(2, 3, 4)), - ('single_in_single_out', utils.square, np.random.rand(2, 3)), - ('multi_in_single_out', paddle.matmul, - (np.random.rand(2, 2), np.random.rand(2, 2))), -)) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'func', 'xs'), + ( + ('1d_in_1d_out', utils.square, np.array([2.0, 3.0])), + ('3d_in_3d_out', utils.square, np.random.rand(2, 3, 4)), + ('single_in_single_out', utils.square, np.random.rand(2, 3)), + ( + 'multi_in_single_out', + paddle.matmul, + (np.random.rand(2, 2), np.random.rand(2, 2)), + ), + ), +) class TestJacobianNoBatch(unittest.TestCase): - def setUp(self): - self._dtype = self.xs[0].dtype if isinstance( - self.xs, typing.Sequence) else self.xs.dtype - self._eps = config.TOLERANCE.get(str( - self._dtype)).get("first_order_grad").get("eps") - self._rtol = config.TOLERANCE.get(str( - self._dtype)).get("first_order_grad").get("rtol") - self._atol = config.TOLERANCE.get(str( - self._dtype)).get("first_order_grad").get("atol") + self._dtype = ( + self.xs[0].dtype + if isinstance(self.xs, typing.Sequence) + else self.xs.dtype + ) + self._eps = ( + config.TOLERANCE.get(str(self._dtype)) + .get("first_order_grad") + .get("eps") + ) + self._rtol = ( + config.TOLERANCE.get(str(self._dtype)) + .get("first_order_grad") + .get("rtol") + ) + self._atol = ( + config.TOLERANCE.get(str(self._dtype)) + .get("first_order_grad") + .get("atol") + ) def func_jacobian(self): - xs = [paddle.to_tensor(x) for x in self.xs] if isinstance( - self.xs, typing.Sequence) else paddle.to_tensor(self.xs) + xs = ( + [paddle.to_tensor(x) for x in self.xs] + if isinstance(self.xs, typing.Sequence) + else paddle.to_tensor(self.xs) + ) self._actual = paddle.incubate.autograd.Jacobian(self.func, xs, False) self._expected = self._get_expected() Index = collections.namedtuple('Index', ('type', 'value')) - indexes = (Index('all', (slice(0, None, None), slice(0, None, None))), - Index('row', (0, slice(0, None, None))), - Index('col', (slice(0, None, None), 0)), - Index('multi-row', (slice(0, 2, 1), slice(0, None, None)))) + indexes = ( + Index('all', (slice(0, None, None), slice(0, None, None))), + Index('row', (0, slice(0, None, None))), + Index('col', (slice(0, None, None), 0)), + Index('multi-row', (slice(0, 2, 1), slice(0, None, None))), + ) self.assertEqual(self._actual[:].numpy().dtype, self._expected.dtype) for index in indexes: np.testing.assert_allclose( @@ -367,15 +395,18 @@ def func_jacobian(self): self._expected.__getitem__(index.value), rtol=self._rtol, atol=self._atol, - err_msg= - f'Testcase {index.type} index not passed, value is {index.value}' + err_msg=f'Testcase {index.type} index not passed, value is {index.value}', ) def _get_expected(self): - xs = [paddle.to_tensor(x) for x in self.xs] if isinstance( - self.xs, typing.Sequence) else paddle.to_tensor(self.xs) - jac = utils._compute_numerical_jacobian(self.func, xs, self._eps, - self._dtype) + xs = ( + [paddle.to_tensor(x) for x in self.xs] + if isinstance(self.xs, typing.Sequence) + else paddle.to_tensor(self.xs) + ) + jac = utils._compute_numerical_jacobian( + self.func, xs, self._eps, self._dtype + ) return utils._np_concat_matrix_sequence(jac, utils.MatrixFormat.NM) def test_all_cases(self): @@ -385,42 +416,71 @@ def test_all_cases(self): @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'func', 'xs'), ( - ('1d_in_1d_out', utils.square, np.array([[1., 2., 3.], [3., 4., 3.]])), - ('3d_in_3d_out', utils.square, np.random.rand(2, 3, 4)), - ('multi_in_single_out', utils.square, np.random.rand(2, 3)), -)) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'func', 'xs'), + ( + ( + '1d_in_1d_out', + utils.square, + np.array([[1.0, 2.0, 3.0], [3.0, 4.0, 3.0]]), + ), + ('3d_in_3d_out', utils.square, np.random.rand(2, 3, 4)), + ('multi_in_single_out', utils.square, np.random.rand(2, 3)), + ), +) class TestJacobianBatchFirst(unittest.TestCase): - def setUp(self): - self._dtype = self.xs[0].dtype if isinstance( - self.xs, typing.Sequence) else self.xs.dtype - self._eps = config.TOLERANCE.get(str( - self._dtype)).get("first_order_grad").get("eps") - self._rtol = config.TOLERANCE.get(str( - self._dtype)).get("first_order_grad").get("rtol") - self._atol = config.TOLERANCE.get(str( - self._dtype)).get("first_order_grad").get("atol") + self._dtype = ( + self.xs[0].dtype + if isinstance(self.xs, typing.Sequence) + else self.xs.dtype + ) + self._eps = ( + config.TOLERANCE.get(str(self._dtype)) + .get("first_order_grad") + .get("eps") + ) + self._rtol = ( + config.TOLERANCE.get(str(self._dtype)) + .get("first_order_grad") + .get("rtol") + ) + self._atol = ( + config.TOLERANCE.get(str(self._dtype)) + .get("first_order_grad") + .get("atol") + ) def func_jacobian(self): - xs = [paddle.to_tensor(x) for x in self.xs] if isinstance( - self.xs, typing.Sequence) else paddle.to_tensor(self.xs) + xs = ( + [paddle.to_tensor(x) for x in self.xs] + if isinstance(self.xs, typing.Sequence) + else paddle.to_tensor(self.xs) + ) self._actual = paddle.incubate.autograd.Jacobian(self.func, xs, True) self._expected = self._get_expected() Index = collections.namedtuple('Index', ('type', 'value')) - indexes = (Index( - 'all', - (slice(0, None, None), slice(0, None, None), slice(0, None, None))), - Index('row', - (slice(0, None, None), 0, slice(0, None, None))), - Index('col', - (slice(0, None, None), slice(0, None, None), 0)), - Index('batch', (slice(0, 2, None), slice( - 0, None, None), slice(0, None, None))), - Index('multi_row', - (slice(0, 1, None), slice(0, 2, 1), slice( - 0, None, None)))) + indexes = ( + Index( + 'all', + ( + slice(0, None, None), + slice(0, None, None), + slice(0, None, None), + ), + ), + Index('row', (slice(0, None, None), 0, slice(0, None, None))), + Index('col', (slice(0, None, None), slice(0, None, None), 0)), + Index( + 'batch', + (slice(0, 2, None), slice(0, None, None), slice(0, None, None)), + ), + Index( + 'multi_row', + (slice(0, 1, None), slice(0, 2, 1), slice(0, None, None)), + ), + ) self.assertEqual(self._actual[:].numpy().dtype, self._expected.dtype) for index in indexes: np.testing.assert_allclose( @@ -428,18 +488,22 @@ def func_jacobian(self): self._expected.__getitem__(index.value), rtol=self._rtol, atol=self._atol, - err_msg= - f'Testcase {index.type} index not passed, value is {index.value}' + err_msg=f'Testcase {index.type} index not passed, value is {index.value}', ) def _get_expected(self): - xs = [paddle.to_tensor(x) for x in self.xs] if isinstance( - self.xs, typing.Sequence) else paddle.to_tensor(self.xs) - jac = utils._compute_numerical_batch_jacobian(self.func, xs, self._eps, - self._dtype, False) + xs = ( + [paddle.to_tensor(x) for x in self.xs] + if isinstance(self.xs, typing.Sequence) + else paddle.to_tensor(self.xs) + ) + jac = utils._compute_numerical_batch_jacobian( + self.func, xs, self._eps, self._dtype, False + ) jac = utils._np_concat_matrix_sequence(jac, utils.MatrixFormat.NBM) - return utils._np_transpose_matrix_format(jac, utils.MatrixFormat.NBM, - utils.MatrixFormat.BNM) + return utils._np_transpose_matrix_format( + jac, utils.MatrixFormat.NBM, utils.MatrixFormat.BNM + ) def test_all_cases(self): with _test_eager_guard(): @@ -448,64 +512,74 @@ def test_all_cases(self): class TestHessianNoBatch(unittest.TestCase): - @classmethod def setUpClass(self): self.shape = (2, 2) self.dtype = 'float32' self.np_dtype = np.float32 - self.numerical_delta = config.TOLERANCE.get( - self.dtype).get("second_order_grad").get("eps") - self.rtol = config.TOLERANCE.get( - self.dtype).get("second_order_grad").get("rtol") - self.atol = config.TOLERANCE.get( - self.dtype).get("second_order_grad").get("atol") + self.numerical_delta = ( + config.TOLERANCE.get(self.dtype).get("second_order_grad").get("eps") + ) + self.rtol = ( + config.TOLERANCE.get(self.dtype) + .get("second_order_grad") + .get("rtol") + ) + self.atol = ( + config.TOLERANCE.get(self.dtype) + .get("second_order_grad") + .get("atol") + ) self.x = paddle.rand(shape=self.shape, dtype=self.dtype) self.y = paddle.rand(shape=self.shape, dtype=self.dtype) def func_single_input(self): - def func(x): return paddle.sum(paddle.matmul(x, x)) numerical_hessian = utils._compute_numerical_hessian( - func, self.x, self.numerical_delta, self.np_dtype) + func, self.x, self.numerical_delta, self.np_dtype + ) numerical_hessian = utils._np_concat_matrix_sequence(numerical_hessian) self.x.stop_gradient = False hessian = paddle.incubate.autograd.Hessian(func, self.x) - np.testing.assert_allclose(hessian[:].numpy(), numerical_hessian, - self.rtol, self.atol) + np.testing.assert_allclose( + hessian[:].numpy(), numerical_hessian, self.rtol, self.atol + ) def func_multi_input(self): - def func(x, y): return paddle.sum(paddle.matmul(x, y)) numerical_hessian = utils._compute_numerical_hessian( - func, [self.x, self.y], self.numerical_delta, self.np_dtype) + func, [self.x, self.y], self.numerical_delta, self.np_dtype + ) numerical_hessian = utils._np_concat_matrix_sequence(numerical_hessian) self.x.stop_gradient = False self.y.stop_gradient = False hessian = paddle.incubate.autograd.Hessian(func, [self.x, self.y]) - np.testing.assert_allclose(hessian[:].numpy(), - numerical_hessian, - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + hessian[:].numpy(), + numerical_hessian, + rtol=self.rtol, + atol=self.atol, + ) def func_allow_unused_true(self): - def func(x, y): return paddle.sum(paddle.matmul(x, x)) numerical_hessian = utils._compute_numerical_hessian( - func, [self.x, self.y], self.numerical_delta, self.np_dtype) + func, [self.x, self.y], self.numerical_delta, self.np_dtype + ) numerical_hessian = utils._np_concat_matrix_sequence(numerical_hessian) self.x.stop_gradient = False self.y.stop_gradient = False hessian = paddle.incubate.autograd.Hessian(func, [self.x, self.y]) - np.testing.assert_allclose(hessian[:].numpy(), numerical_hessian, - self.rtol, self.atol) + np.testing.assert_allclose( + hessian[:].numpy(), numerical_hessian, self.rtol, self.atol + ) def func_create_graph_true(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -514,17 +588,18 @@ def func(x): return paddle.sum(F.sigmoid(x)) numerical_hessian = utils._compute_numerical_hessian( - func, self.x, self.numerical_delta, self.np_dtype) + func, self.x, self.numerical_delta, self.np_dtype + ) numerical_hessian = utils._np_concat_matrix_sequence(numerical_hessian) self.x.stop_gradient = False hessian = paddle.incubate.autograd.Hessian(func, self.x) assert hessian[:].stop_gradient == False - np.testing.assert_allclose(hessian[:].numpy(), numerical_hessian, - self.rtol, self.atol) + np.testing.assert_allclose( + hessian[:].numpy(), numerical_hessian, self.rtol, self.atol + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_out_not_single(self): - def func(x): return x * x @@ -548,7 +623,6 @@ def test_all_cases(self): class TestHessianBatchFirst(unittest.TestCase): - @classmethod def setUpClass(self): self.x_shape = (5, 2) @@ -557,108 +631,117 @@ def setUpClass(self): self.nbatch, self.nrow = 5, 2 self.dtype = 'float32' self.np_dtype = np.float32 - self.numerical_delta = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('eps') - self.rtol = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('rtol') - self.atol = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('atol') + self.numerical_delta = ( + config.TOLERANCE.get(self.dtype).get('second_order_grad').get('eps') + ) + self.rtol = ( + config.TOLERANCE.get(self.dtype) + .get('second_order_grad') + .get('rtol') + ) + self.atol = ( + config.TOLERANCE.get(self.dtype) + .get('second_order_grad') + .get('atol') + ) self.x = paddle.rand(shape=self.x_shape, dtype=self.dtype) self.weight = paddle.rand(shape=self.weight_shape, dtype=self.dtype) self.y = paddle.rand(shape=self.y_shape, dtype=self.dtype) def func_single_input(self): - def func(x): return paddle.matmul(x * x, self.weight)[:, 0:1] expected = utils._compute_numerical_batch_hessian( - func, self.x, self.numerical_delta, self.np_dtype) + func, self.x, self.numerical_delta, self.np_dtype + ) H = paddle.incubate.autograd.Hessian(func, self.x, is_batched=True) - actual = utils._np_transpose_matrix_format(H[:].numpy(), - utils.MatrixFormat.BNM, - utils.MatrixFormat.NBM) + actual = utils._np_transpose_matrix_format( + H[:].numpy(), utils.MatrixFormat.BNM, utils.MatrixFormat.NBM + ) actual = actual.reshape((H.shape[1], -1)) np.testing.assert_allclose(actual, expected, self.rtol, self.atol) def func_multi_input(self): - def func(x, y): return paddle.matmul(x * x * y * y, self.weight)[:, 0:1] xs_len = 2 expected = utils._compute_numerical_batch_hessian( - func, [self.x, self.y], self.numerical_delta, self.np_dtype) + func, [self.x, self.y], self.numerical_delta, self.np_dtype + ) expected = np.reshape( np.array(expected), - (xs_len, xs_len, self.nrow, self.nbatch, self.nrow)) + (xs_len, xs_len, self.nrow, self.nbatch, self.nrow), + ) expected = [[n for n in row] for row in expected] expected = utils._np_concat_matrix_sequence(expected) self.x.stop_gradient = False self.y.stop_gradient = False - H = paddle.incubate.autograd.Hessian(func, [self.x, self.y], - is_batched=True) - actual = utils._np_transpose_matrix_format(H[:].numpy(), - utils.MatrixFormat.BNM, - utils.MatrixFormat.NBM) + H = paddle.incubate.autograd.Hessian( + func, [self.x, self.y], is_batched=True + ) + actual = utils._np_transpose_matrix_format( + H[:].numpy(), utils.MatrixFormat.BNM, utils.MatrixFormat.NBM + ) np.testing.assert_allclose(actual, expected, self.rtol, self.atol) def func_allow_unused(self): - def func(x, y): return paddle.matmul(x * x, self.weight)[:, 0:1] xs_len = 2 expected = utils._compute_numerical_batch_hessian( - func, [self.x, self.y], self.numerical_delta, self.np_dtype) + func, [self.x, self.y], self.numerical_delta, self.np_dtype + ) expected = np.reshape( np.array(expected), - (xs_len, xs_len, self.nrow, self.nbatch, self.nrow)) + (xs_len, xs_len, self.nrow, self.nbatch, self.nrow), + ) expected = [[n for n in row] for row in expected] expected = utils._np_concat_matrix_sequence(expected) - expected = utils._np_transpose_matrix_format(expected, - utils.MatrixFormat.NBM, - utils.MatrixFormat.BNM) + expected = utils._np_transpose_matrix_format( + expected, utils.MatrixFormat.NBM, utils.MatrixFormat.BNM + ) - actual = paddle.incubate.autograd.Hessian(func, [self.x, self.y], - is_batched=True)[:] + actual = paddle.incubate.autograd.Hessian( + func, [self.x, self.y], is_batched=True + )[:] - np.testing.assert_allclose(actual, - expected, - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + actual, expected, rtol=self.rtol, atol=self.atol + ) def func_stop_gradient(self): - def func(x): return paddle.matmul(x * x, self.weight)[:, 0:1] expected = utils._compute_numerical_batch_hessian( - func, self.x, self.numerical_delta, self.np_dtype) + func, self.x, self.numerical_delta, self.np_dtype + ) x = self.x.clone() x.stop_gradient = True H = paddle.incubate.autograd.Hessian(func, self.x, is_batched=True)[:] - actual = utils._np_transpose_matrix_format(H[:].numpy(), - utils.MatrixFormat.BNM, - utils.MatrixFormat.NBM) + actual = utils._np_transpose_matrix_format( + H[:].numpy(), utils.MatrixFormat.BNM, utils.MatrixFormat.NBM + ) actual = actual.reshape((H.shape[1], -1)) np.testing.assert_allclose(actual, expected, self.rtol, self.atol) def func_out_not_single(self): - def func(x): - return (x * x) + return x * x with self.assertRaises(RuntimeError): - paddle.incubate.autograd.Hessian(func, - paddle.ones((3, 3)), - is_batched=True) + paddle.incubate.autograd.Hessian( + func, paddle.ones((3, 3)), is_batched=True + ) def test_all_cases(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_prim.py b/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_prim.py index de12fd1ef901a..70fb6b1bbfae6 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_prim.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_prim.py @@ -22,23 +22,35 @@ @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), ( - ('unary_float32', paddle.tanh, (np.random.rand(2, 3), ), 'float32'), - ('binary_float32', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), 'float32'), - ('unary_float64', paddle.tanh, (np.random.rand(2, 3), ), 'float64'), - ('binary_float64', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), 'float64'), -)) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), + ( + ('unary_float32', paddle.tanh, (np.random.rand(2, 3),), 'float32'), + ( + 'binary_float32', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + 'float32', + ), + ('unary_float64', paddle.tanh, (np.random.rand(2, 3),), 'float64'), + ( + 'binary_float64', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + 'float64', + ), + ), +) class TestJacobianPrim(unittest.TestCase): - @classmethod def setUpClass(cls): cls.args = [arg.astype(cls.dtype) for arg in cls.args] - cls._rtol = config.TOLERANCE.get( - cls.dtype).get('first_order_grad').get('rtol') - cls._atol = config.TOLERANCE.get( - cls.dtype).get('first_order_grad').get('atol') + cls._rtol = ( + config.TOLERANCE.get(cls.dtype).get('first_order_grad').get('rtol') + ) + cls._atol = ( + config.TOLERANCE.get(cls.dtype).get('first_order_grad').get('atol') + ) def setUp(self): paddle.enable_static() @@ -49,7 +61,6 @@ def tearDown(self): paddle.disable_static() def test_jacobian_prim(self): - def wrapper(fun, args): mp = paddle.static.Program() sp = paddle.static.Program() @@ -65,10 +76,11 @@ def wrapper(fun, args): paddle.incubate.autograd.prim2orig() exe = paddle.static.Executor() exe.run(sp) - [jac] = exe.run(mp, - feed={f'arg{i}': arg - for i, arg in enumerate(args)}, - fetch_list=[jac]) + [jac] = exe.run( + mp, + feed={f'arg{i}': arg for i, arg in enumerate(args)}, + fetch_list=[jac], + ) return jac paddle.incubate.autograd.enable_prim() @@ -76,30 +88,41 @@ def wrapper(fun, args): paddle.incubate.autograd.disable_prim() orig_jac = wrapper(self.fun, self.args) - np.testing.assert_allclose(orig_jac, - prim_jac, - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + orig_jac, prim_jac, rtol=self._rtol, atol=self._atol + ) @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), ( - ('unary_float32', paddle.tanh, (np.random.rand(1), ), 'float32'), - ('binary_float32', paddle.multiply, - (np.random.rand(1), np.random.rand(1)), 'float32'), - ('unary_float64', paddle.tanh, (np.random.rand(1), ), 'float64'), - ('binary_float64', paddle.multiply, - (np.random.rand(1), np.random.rand(1)), 'float64'), -)) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), + ( + ('unary_float32', paddle.tanh, (np.random.rand(1),), 'float32'), + ( + 'binary_float32', + paddle.multiply, + (np.random.rand(1), np.random.rand(1)), + 'float32', + ), + ('unary_float64', paddle.tanh, (np.random.rand(1),), 'float64'), + ( + 'binary_float64', + paddle.multiply, + (np.random.rand(1), np.random.rand(1)), + 'float64', + ), + ), +) class TestHessianPrim(unittest.TestCase): - @classmethod def setUpClass(cls): cls.args = [arg.astype(cls.dtype) for arg in cls.args] - cls._rtol = config.TOLERANCE.get( - cls.dtype).get('second_order_grad').get('rtol') - cls._atol = config.TOLERANCE.get( - cls.dtype).get('second_order_grad').get('atol') + cls._rtol = ( + config.TOLERANCE.get(cls.dtype).get('second_order_grad').get('rtol') + ) + cls._atol = ( + config.TOLERANCE.get(cls.dtype).get('second_order_grad').get('atol') + ) def setUp(self): paddle.enable_static() @@ -110,7 +133,6 @@ def tearDown(self): paddle.disable_static() def test_jacobian_prim(self): - def wrapper(fun, args): mp = paddle.static.Program() sp = paddle.static.Program() @@ -126,11 +148,11 @@ def wrapper(fun, args): paddle.incubate.autograd.prim2orig() exe = paddle.static.Executor() exe.run(sp) - [hessian - ] = exe.run(mp, - feed={f'arg{i}': arg - for i, arg in enumerate(args)}, - fetch_list=[hessian]) + [hessian] = exe.run( + mp, + feed={f'arg{i}': arg for i, arg in enumerate(args)}, + fetch_list=[hessian], + ) return hessian paddle.incubate.autograd.enable_prim() @@ -138,30 +160,41 @@ def wrapper(fun, args): paddle.incubate.autograd.disable_prim() orig_jac = wrapper(self.fun, self.args) - np.testing.assert_allclose(orig_jac, - prim_jac, - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + orig_jac, prim_jac, rtol=self._rtol, atol=self._atol + ) @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), ( - ('unary_float32', paddle.tanh, (np.random.rand(2, 3), ), 'float32'), - ('binary_float32', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), 'float32'), - ('unary_float64', paddle.tanh, (np.random.rand(2, 3), ), 'float64'), - ('binary_float64', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), 'float64'), -)) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), + ( + ('unary_float32', paddle.tanh, (np.random.rand(2, 3),), 'float32'), + ( + 'binary_float32', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + 'float32', + ), + ('unary_float64', paddle.tanh, (np.random.rand(2, 3),), 'float64'), + ( + 'binary_float64', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + 'float64', + ), + ), +) class TestJvpPrim(unittest.TestCase): - @classmethod def setUpClass(cls): cls.args = [arg.astype(cls.dtype) for arg in cls.args] - cls._rtol = config.TOLERANCE.get( - cls.dtype).get('first_order_grad').get('rtol') - cls._atol = config.TOLERANCE.get( - cls.dtype).get('first_order_grad').get('atol') + cls._rtol = ( + config.TOLERANCE.get(cls.dtype).get('first_order_grad').get('rtol') + ) + cls._atol = ( + config.TOLERANCE.get(cls.dtype).get('first_order_grad').get('atol') + ) def setUp(self): paddle.enable_static() @@ -172,7 +205,6 @@ def tearDown(self): paddle.disable_static() def test_jacobian_prim(self): - def wrapper(fun, args): mp = paddle.static.Program() sp = paddle.static.Program() @@ -190,9 +222,9 @@ def wrapper(fun, args): exe.run(sp) jvp_res = exe.run( mp, - feed={f'arg{i}': arg - for i, arg in enumerate(args)}, - fetch_list=[jvp_res]) + feed={f'arg{i}': arg for i, arg in enumerate(args)}, + fetch_list=[jvp_res], + ) return jvp_res paddle.incubate.autograd.enable_prim() @@ -200,30 +232,41 @@ def wrapper(fun, args): paddle.incubate.autograd.disable_prim() orig_jvp = wrapper(self.fun, self.args) - np.testing.assert_allclose(orig_jvp, - prim_jvp, - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + orig_jvp, prim_jvp, rtol=self._rtol, atol=self._atol + ) @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), ( - ('unary_float32', paddle.tanh, (np.random.rand(2, 3), ), 'float32'), - ('binary_float32', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), 'float32'), - ('unary_float64', paddle.tanh, (np.random.rand(2, 3), ), 'float64'), - ('binary_float64', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), 'float64'), -)) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'args', 'dtype'), + ( + ('unary_float32', paddle.tanh, (np.random.rand(2, 3),), 'float32'), + ( + 'binary_float32', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + 'float32', + ), + ('unary_float64', paddle.tanh, (np.random.rand(2, 3),), 'float64'), + ( + 'binary_float64', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + 'float64', + ), + ), +) class TestVjpPrim(unittest.TestCase): - @classmethod def setUpClass(cls): cls.args = [arg.astype(cls.dtype) for arg in cls.args] - cls._rtol = config.TOLERANCE.get( - cls.dtype).get('first_order_grad').get('rtol') - cls._atol = config.TOLERANCE.get( - cls.dtype).get('first_order_grad').get('atol') + cls._rtol = ( + config.TOLERANCE.get(cls.dtype).get('first_order_grad').get('rtol') + ) + cls._atol = ( + config.TOLERANCE.get(cls.dtype).get('first_order_grad').get('atol') + ) def setUp(self): paddle.enable_static() @@ -234,7 +277,6 @@ def tearDown(self): paddle.disable_static() def test_jacobian_prim(self): - def wrapper(fun, args): mp = paddle.static.Program() sp = paddle.static.Program() @@ -252,9 +294,9 @@ def wrapper(fun, args): exe.run(sp) vjp_res = exe.run( mp, - feed={f'arg{i}': arg - for i, arg in enumerate(args)}, - fetch_list=[vjp_res]) + feed={f'arg{i}': arg for i, arg in enumerate(args)}, + fetch_list=[vjp_res], + ) return vjp_res paddle.incubate.autograd.enable_prim() @@ -263,10 +305,9 @@ def wrapper(fun, args): orig_vjp = wrapper(self.fun, self.args) for orig, prim in zip(orig_vjp, prim_vjp): - np.testing.assert_allclose(orig, - prim, - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + orig, prim, rtol=self._rtol, atol=self._atol + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_static.py b/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_static.py index 2102cd81ca720..fd3d7d9458ae1 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_static.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_autograd_functional_static.py @@ -26,29 +26,71 @@ @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'stop_gradient'), ( - ('tensor_input', utils.reduce, np.random.rand(2, 3), None, False), - ('tensor_sequence_input', utils.reduce, np.random.rand(2, 3), None, False), - ('v_not_none', utils.reduce, np.random.rand(2, - 3), np.random.rand(1), False), - ('xs_stop_gradient', utils.reduce, np.random.rand( - 2, 3), np.random.rand(1), True), - ('func_mutmul', utils.matmul, - (np.random.rand(3, 2), np.random.rand(2, 3)), None, False), - ('func_mul', utils.mul, - (np.random.rand(3, 3), np.random.rand(3, 3)), None, False), - ('func_out_two', utils.o2, - (np.random.rand(10), np.random.rand(10)), None, False), -)) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'stop_gradient'), + ( + ('tensor_input', utils.reduce, np.random.rand(2, 3), None, False), + ( + 'tensor_sequence_input', + utils.reduce, + np.random.rand(2, 3), + None, + False, + ), + ( + 'v_not_none', + utils.reduce, + np.random.rand(2, 3), + np.random.rand(1), + False, + ), + ( + 'xs_stop_gradient', + utils.reduce, + np.random.rand(2, 3), + np.random.rand(1), + True, + ), + ( + 'func_mutmul', + utils.matmul, + (np.random.rand(3, 2), np.random.rand(2, 3)), + None, + False, + ), + ( + 'func_mul', + utils.mul, + (np.random.rand(3, 3), np.random.rand(3, 3)), + None, + False, + ), + ( + 'func_out_two', + utils.o2, + (np.random.rand(10), np.random.rand(10)), + None, + False, + ), + ), +) class TestVJP(unittest.TestCase): - def setUp(self): - self.dtype = str(self.xs[0].dtype) if isinstance( - self.xs, typing.Sequence) else str(self.xs.dtype) - self._rtol = config.TOLERANCE.get(str( - self.dtype)).get("first_order_grad").get("rtol") - self._atol = config.TOLERANCE.get(str( - self.dtype)).get("first_order_grad").get("atol") + self.dtype = ( + str(self.xs[0].dtype) + if isinstance(self.xs, typing.Sequence) + else str(self.xs.dtype) + ) + self._rtol = ( + config.TOLERANCE.get(str(self.dtype)) + .get("first_order_grad") + .get("rtol") + ) + self._atol = ( + config.TOLERANCE.get(str(self.dtype)) + .get("first_order_grad") + .get("atol") + ) def _vjp(self): exe = paddle.static.Executor() @@ -56,9 +98,11 @@ def _vjp(self): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=self.stop_gradient) - ys, xs_grads = paddle.incubate.autograd.vjp(self.fun, static_xs, - static_v) + self.xs, self.v, stop_gradient=self.stop_gradient + ) + ys, xs_grads = paddle.incubate.autograd.vjp( + self.fun, static_xs, static_v + ) exe.run(sp) return exe.run(mp, feed=feed, fetch_list=[ys, xs_grads]) @@ -68,9 +112,13 @@ def _expected_vjp(self): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) xs_grads = paddle.static.gradients(ys, static_xs, static_v) exe.run(sp) return exe.run(mp, feed=feed, fetch_list=[ys, xs_grads]) @@ -80,19 +128,25 @@ def test_vjp(self): expected = self._expected_vjp() self.assertEqual(len(actual), len(expected)) for i in range(len(actual)): - np.testing.assert_allclose(actual[i], - expected[i], - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + actual[i], expected[i], rtol=self._rtol, atol=self._atol + ) @utils.place(config.DEVICES) @utils.parameterize( (utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'expected_exception'), - (('v_shape_not_equal_ys', utils.square, np.random.rand(3), - np.random.rand(1), RuntimeError), )) + ( + ( + 'v_shape_not_equal_ys', + utils.square, + np.random.rand(3), + np.random.rand(1), + RuntimeError, + ), + ), +) class TestVJPException(unittest.TestCase): - def setUp(self): self.exe = paddle.static.Executor() @@ -101,9 +155,11 @@ def _vjp(self): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v) - ys, xs_grads = paddle.incubate.autograd.vjp(self.fun, static_xs, - static_v) + self.xs, self.v + ) + ys, xs_grads = paddle.incubate.autograd.vjp( + self.fun, static_xs, static_v + ) self.exe.run(sp) return self.exe.run(mp, feed, fetch_list=[ys, xs_grads]) @@ -176,11 +232,11 @@ def make_tensors(inps): all_data_shapes = { - 'A': [[1., 2.]], - 'B': [[1., 2.], [2., 1.]], - 'C': [[2., 2.], [2., 1.]], - 'D': [[[2., 2.], [2., 1.]], [[1., 2.], [2., 1.]]], - 'E': [[[3., 4.], [2., 3.]], [[2., 1.], [1., 3.]]], + 'A': [[1.0, 2.0]], + 'B': [[1.0, 2.0], [2.0, 1.0]], + 'C': [[2.0, 2.0], [2.0, 1.0]], + 'D': [[[2.0, 2.0], [2.0, 1.0]], [[1.0, 2.0], [2.0, 1.0]]], + 'E': [[[3.0, 4.0], [2.0, 3.0]], [[2.0, 1.0], [1.0, 3.0]]], } @@ -190,7 +246,6 @@ def prepare_data(test, input_shapes, dtype): class TestJacobianFloat32(unittest.TestCase): - @classmethod def setUpClass(self): paddle.enable_static() @@ -201,8 +256,9 @@ def setUpClass(self): self.dtype = 'float32' self.np_dtype = np.float32 prepare_data(self, all_data_shapes, self.dtype) - self.eps = config.TOLERANCE.get( - self.dtype).get('first_order_grad').get('eps') + self.eps = ( + config.TOLERANCE.get(self.dtype).get('first_order_grad').get('eps') + ) # self.rtol = config.TOLERANCE.get(self.dtype).get('first_order_grad').get('rtol') # self.atol = config.TOLERANCE.get(self.dtype).get('first_order_grad').get('atol') # Do't use tolerance in config, which will cause this test case failed. @@ -227,17 +283,17 @@ def run_test_by_fullmatrix(self, pd_f, np_f, inps, batch=False): else: feeds = {'x': inps} pd_jacobians = exe.run(main, feed=feeds, fetch_list=[full_jacobian])[0] - np_jacobians = approx_jacobian(np_f, - inps, - self.dtype, - self.eps, - batch=batch) + np_jacobians = approx_jacobian( + np_f, inps, self.dtype, self.eps, batch=batch + ) if batch: np_jacobians = utils._np_transpose_matrix_format( - np_jacobians, utils.MatrixFormat.NBM, utils.MatrixFormat.BNM) + np_jacobians, utils.MatrixFormat.NBM, utils.MatrixFormat.BNM + ) - np.testing.assert_allclose(pd_jacobians, np_jacobians, self.rtol, - self.atol) + np.testing.assert_allclose( + pd_jacobians, np_jacobians, self.rtol, self.atol + ) def run_test_by_rows(self, pd_f, np_f, inps, batch=False): main = fluid.Program() @@ -261,8 +317,9 @@ def run_test_by_rows(self, pd_f, np_f, inps, batch=False): pd_jac = exe.run(main, feed=feeds, fetch_list=[rows]) np_jac = approx_jacobian(np_f, inps, self.dtype, self.eps, batch=batch) for i in range(nrow): - np.testing.assert_allclose(pd_jac[i], np_jac[i], self.rtol, - self.atol) + np.testing.assert_allclose( + pd_jac[i], np_jac[i], self.rtol, self.atol + ) def run_test_by_entries(self, pd_f, np_f, inps, batch=False): main = fluid.Program() @@ -293,7 +350,6 @@ def run_test_by_entries(self, pd_f, np_f, inps, batch=False): np.testing.assert_allclose(pd_entry, np_entry, self.rtol, self.atol) def test_square(self): - def pd_f(x): return paddle.multiply(x, x) @@ -305,7 +361,6 @@ def np_f(x): self.run_test_by_entries(pd_f, np_f, self.A) def test_mul(self): - def pd_f(x, y): return paddle.multiply(x, y) @@ -322,7 +377,6 @@ def np_f(xs): self.run_test_by_entries(pd_f, np_f, [self.B, self.C]) def test_matmul(self): - def pd_f(x, y): return paddle.matmul(x, y) @@ -335,7 +389,6 @@ def np_f(xs): self.run_test_by_entries(pd_f, np_f, [self.B, self.C]) def test_batch_matmul(self): - def pd_f(x, y): return paddle.matmul(x, y) @@ -349,7 +402,6 @@ def np_f(xs): class TestJacobianFloat64(TestJacobianFloat32): - @classmethod def setUpClass(self): paddle.enable_static() @@ -359,16 +411,18 @@ def setUpClass(self): self.place = fluid.CPUPlace() self.dtype = 'float64' prepare_data(self, all_data_shapes, self.dtype) - self.eps = config.TOLERANCE.get( - self.dtype).get('first_order_grad').get('eps') - self.rtol = config.TOLERANCE.get( - self.dtype).get('first_order_grad').get('rtol') - self.atol = config.TOLERANCE.get( - self.dtype).get('first_order_grad').get('atol') + self.eps = ( + config.TOLERANCE.get(self.dtype).get('first_order_grad').get('eps') + ) + self.rtol = ( + config.TOLERANCE.get(self.dtype).get('first_order_grad').get('rtol') + ) + self.atol = ( + config.TOLERANCE.get(self.dtype).get('first_order_grad').get('atol') + ) class TestHessianFloat32(unittest.TestCase): - @classmethod def setUpClass(self): paddle.enable_static() @@ -378,12 +432,19 @@ def setUpClass(self): self.place = fluid.CPUPlace() self.dtype = 'float32' prepare_data(self, all_data_shapes, self.dtype) - self.eps = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('eps') - self.rtol = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('rtol') - self.atol = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('atol') + self.eps = ( + config.TOLERANCE.get(self.dtype).get('second_order_grad').get('eps') + ) + self.rtol = ( + config.TOLERANCE.get(self.dtype) + .get('second_order_grad') + .get('rtol') + ) + self.atol = ( + config.TOLERANCE.get(self.dtype) + .get('second_order_grad') + .get('atol') + ) def run_test_by_fullmatrix(self, pd_f, inps, np_hess, batch=False): main = fluid.Program() @@ -403,7 +464,6 @@ def run_test_by_fullmatrix(self, pd_f, inps, np_hess, batch=False): np.testing.assert_allclose(pd_hess, np_hess, self.rtol, self.atol) def test_square(self): - def pd_f(x): """Input is a square matrix.""" return paddle.matmul(x, x.T).flatten().sum() @@ -418,7 +478,6 @@ def np_hess(x): class TestHessianFloat64(TestHessianFloat32): - @classmethod def setUpClass(self): paddle.enable_static() @@ -428,12 +487,19 @@ def setUpClass(self): self.place = fluid.CPUPlace() self.dtype = 'float64' prepare_data(self, all_data_shapes, self.dtype) - self.eps = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('eps') - self.rtol = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('rtol') - self.atol = config.TOLERANCE.get( - self.dtype).get('second_order_grad').get('atol') + self.eps = ( + config.TOLERANCE.get(self.dtype).get('second_order_grad').get('eps') + ) + self.rtol = ( + config.TOLERANCE.get(self.dtype) + .get('second_order_grad') + .get('rtol') + ) + self.atol = ( + config.TOLERANCE.get(self.dtype) + .get('second_order_grad') + .get('atol') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py index f345f58394575..e90d6871c300e 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py @@ -24,14 +24,14 @@ ############################ Test linearize rules ############################ class TestAddPJVPAndTranspose(unittest.TestCase): - def setUp(self): self.main_program = paddle.static.Program() self.startup_program = paddle.static.Program() self.layer_help = LayerHelper('TestPrim2Orig') - with paddle.static.program_guard(self.main_program, - self.startup_program): + with paddle.static.program_guard( + self.main_program, self.startup_program + ): self.init_data() def init_data(self): @@ -41,8 +41,9 @@ def init_data(self): Y = paddle.static.data(name='Y', shape=[2, 2], dtype='float') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -67,12 +68,15 @@ def init_data(self): ] def test_op(self): - with paddle.static.program_guard(self.main_program, - self.startup_program): - op = self.layer_help.append_op(type=self.op_type, - inputs=self.prim_input, - outputs=self.prim_output, - attrs=self.prim_attrs) + with paddle.static.program_guard( + self.main_program, self.startup_program + ): + op = self.layer_help.append_op( + type=self.op_type, + inputs=self.prim_input, + outputs=self.prim_output, + attrs=self.prim_attrs, + ) jvp_out = _jvp(op, *self.jvp_args) jvp_out = flatten(jvp_out) @@ -91,7 +95,6 @@ def test_op(self): class TestSubPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'sub_p' @@ -99,8 +102,9 @@ def init_data(self): Y = paddle.static.data(name='Y', shape=[5, 6], dtype='int64') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -123,12 +127,11 @@ def init_data(self): 'sub_p', # transpose op: 'fill_constant_p', - 'sub_p' + 'sub_p', ] class TestMulPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'mul_p' @@ -136,8 +139,9 @@ def init_data(self): Y = paddle.static.data(name='Y', shape=[5, 6], dtype='int64') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -163,12 +167,11 @@ def init_data(self): 'mul_p', 'add_p', # transpose op: - 'mul_p' + 'mul_p', ] class TestDivPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'div_p' @@ -176,8 +179,9 @@ def init_data(self): Y = paddle.static.data(name='Y', shape=[5, 6], dtype='int64') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -205,12 +209,11 @@ def init_data(self): 'mul_p', 'sub_p', # transpose op: - 'div_p' + 'div_p', ] class TestSqrtPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'sqrt_p' @@ -219,14 +222,15 @@ def init_data(self): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -242,7 +246,6 @@ def init_data(self): class TestRSqrtPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'rsqrt_p' @@ -251,14 +254,15 @@ def init_data(self): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -275,7 +279,6 @@ def init_data(self): class TestTanhPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'tanh_p' @@ -284,14 +287,15 @@ def init_data(self): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -307,7 +311,6 @@ def init_data(self): class TestSinPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'sin_p' @@ -316,14 +319,15 @@ def init_data(self): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -337,7 +341,6 @@ def init_data(self): class TestCosPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'cos_p' @@ -346,14 +349,15 @@ def init_data(self): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -369,7 +373,6 @@ def init_data(self): class TestExpPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'exp_p' @@ -378,14 +381,15 @@ def init_data(self): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -398,7 +402,6 @@ def init_data(self): class TestErfPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'erf_p' @@ -407,14 +410,15 @@ def init_data(self): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -434,7 +438,6 @@ def init_data(self): class TestAbsPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'abs_p' @@ -443,14 +446,15 @@ def init_data(self): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -467,7 +471,6 @@ def init_data(self): class TestCastPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'cast_p' @@ -476,14 +479,15 @@ def init_data(self): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'dtype': paddle.float64} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} # Set transpose @@ -498,12 +502,11 @@ def init_data(self): # jvp op: 'cast_p', # transpose op: - 'cast_p' + 'cast_p', ] class TestLogPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'log_p' @@ -512,14 +515,15 @@ def init_data(self): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} self.all_ops = [ @@ -532,7 +536,6 @@ def init_data(self): class TestReshapePJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'reshape_p' @@ -541,14 +544,15 @@ def init_data(self): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'shape': [2, 32]} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[8, 8], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} # Set transpose @@ -570,7 +574,6 @@ def init_data(self): class TestBroadcastPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'broadcast_p' @@ -579,21 +582,22 @@ def init_data(self): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'shape': [2, 10, 7]} # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[10, 7], dtype='int64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} # Set transpose check_dot = lambda v: v is X - Y_BAR = paddle.static.data(name='Y_BAR', - shape=[2, 10, 7], - dtype='int64') + Y_BAR = paddle.static.data( + name='Y_BAR', shape=[2, 10, 7], dtype='int64' + ) self.transpose_args = (check_dot, Y_BAR) self.transpose_out_shape_map = { 0: X, @@ -606,12 +610,11 @@ def init_data(self): 'broadcast_p', # transpose op: 'reduce_sum_p', - 'reshape_p' + 'reshape_p', ] class TestTransposePJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'transpose_p' @@ -620,23 +623,24 @@ def init_data(self): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'axis': [0, 2, 3, 1]} # Set JVP - X_DOT = paddle.static.data(name='X_DOT', - shape=[2, 3, 4, 5], - dtype='int64') - self.jvp_args = (X_DOT, ) + X_DOT = paddle.static.data( + name='X_DOT', shape=[2, 3, 4, 5], dtype='int64' + ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} # Set transpose check_dot = lambda v: v is X - Y_BAR = paddle.static.data(name='Y_BAR', - shape=[2, 4, 5, 3], - dtype='int64') + Y_BAR = paddle.static.data( + name='Y_BAR', shape=[2, 4, 5, 3], dtype='int64' + ) self.transpose_args = (check_dot, Y_BAR) self.transpose_out_shape_map = { 0: X, @@ -653,7 +657,6 @@ def init_data(self): class TestSplitPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'split_p' @@ -664,16 +667,18 @@ def init_data(self): self.prim_output = { 'YS': [ self.layer_help.create_variable_for_type_inference( - dtype=X.dtype) for i in range(4) + dtype=X.dtype + ) + for i in range(4) ] } self.prim_attrs = {'num_or_sections': [2, 3, 4, 1], 'axis': 2} # Set JVP - X_DOT = paddle.static.data(name='X_DOT', - shape=[2, 7, 10], - dtype='int64') - self.jvp_args = (X_DOT, ) + X_DOT = paddle.static.data( + name='X_DOT', shape=[2, 7, 10], dtype='int64' + ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = { 0: self.prim_output['YS'][0], 1: self.prim_output['YS'][1], @@ -705,7 +710,6 @@ def init_data(self): class TestConcatPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'concat_p' @@ -716,8 +720,9 @@ def init_data(self): 'XS': [X, Y, Z], } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'axis': 1} @@ -727,14 +732,14 @@ def init_data(self): paddle.static.data(name='X_DOT2', shape=[3, 2, 5], dtype='float64'), paddle.static.data(name='X_DOT3', shape=[3, 3, 5], dtype='float64'), ] - self.jvp_args = (XS_DOT, ) + self.jvp_args = (XS_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} # Set transpose check_dot = lambda v: v is X or v is Y or v is Z - Y_BAR = paddle.static.data(name='Y_BAR', - shape=[3, 14, 5], - dtype='float64') + Y_BAR = paddle.static.data( + name='Y_BAR', shape=[3, 14, 5], dtype='float64' + ) self.transpose_args = (check_dot, Y_BAR) self.transpose_out_shape_map = { 0: X, @@ -753,30 +758,30 @@ def init_data(self): class TestReduceSumPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'reduce_sum_p' X = paddle.static.data(name='X', shape=[2, 3, 4, 5], dtype='float64') self.prim_input = {'X': X} self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'axis': [2], 'keepdim': False} # Set JVP - X_DOT = paddle.static.data(name='X_DOT1', - shape=[2, 3, 4, 5], - dtype='float64') - self.jvp_args = (X_DOT, ) + X_DOT = paddle.static.data( + name='X_DOT1', shape=[2, 3, 4, 5], dtype='float64' + ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} # Set transpose check_dot = lambda v: v is X - Y_BAR = paddle.static.data(name='Y_BAR', - shape=[2, 3, 5], - dtype='float64') + Y_BAR = paddle.static.data( + name='Y_BAR', shape=[2, 3, 5], dtype='float64' + ) self.transpose_args = (check_dot, Y_BAR) self.transpose_out_shape_map = { 0: X, @@ -794,7 +799,6 @@ def init_data(self): class TestMatmulPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'matmul_p' @@ -802,8 +806,9 @@ def init_data(self): Y = paddle.static.data(name='Y', shape=[3, 4], dtype='float64') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -835,7 +840,6 @@ def init_data(self): class TestSliceSelectPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'slice_select_p' @@ -844,19 +848,20 @@ def init_data(self): 'X': X, } self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = { 'axis': [1], 'starts': [0], 'ends': [20], - 'strides': [2] + 'strides': [2], } # Set JVP X_DOT = paddle.static.data(name='X_DOT', shape=[3, 20], dtype='float64') - self.jvp_args = (X_DOT, ) + self.jvp_args = (X_DOT,) self.jvp_out_shape_map = {0: self.prim_output['Y']} # Set transpose @@ -879,7 +884,6 @@ def init_data(self): class TestSliceAssignPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'slice_assign_p' @@ -887,14 +891,15 @@ def init_data(self): Y = paddle.static.data(name='Y', shape=[3, 5], dtype='float64') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = { 'axis': [1], 'starts': [0], 'ends': [10], - 'strides': [2] + 'strides': [2], } # Set JVP @@ -917,23 +922,23 @@ def init_data(self): # transpose op: 'slice_assign_p', 'slice_select_p', - 'fill_constant_p' + 'fill_constant_p', ] class TestGatherPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'gather_p' X = paddle.static.data(name='X', shape=[9, 5], dtype='float64') - IndexTensor = paddle.static.data(name='IndexTensor', - shape=[3], - dtype='int32') + IndexTensor = paddle.static.data( + name='IndexTensor', shape=[3], dtype='int32' + ) self.prim_input = {'X': X, 'IndexTensor': IndexTensor} self.prim_output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'axis': 1} @@ -965,19 +970,19 @@ def init_data(self): class TestScatterAddPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'scatter_add_p' X = paddle.static.data(name='X', shape=[9, 5], dtype='float64') Y = paddle.static.data(name='Y', shape=[9, 3], dtype='float64') - IndexTensor = paddle.static.data(name='IndexTensor', - shape=[3], - dtype='int32') + IndexTensor = paddle.static.data( + name='IndexTensor', shape=[3], dtype='int32' + ) self.prim_input = {'X': X, 'Y': Y, 'IndexTensor': IndexTensor} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {'axis': 1} @@ -1001,12 +1006,11 @@ def init_data(self): # transpose op: 'scatter_add_p', 'fill_constant_p', - 'gather_p' + 'gather_p', ] class TestSelectPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'select_p' @@ -1016,15 +1020,16 @@ def init_data(self): self.prim_input = {'Condition': Cond, 'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} # Set JVP - Cond_DOT = paddle.static.data(name='Cond_DOT', - shape=[9, 5], - dtype='float64') + Cond_DOT = paddle.static.data( + name='Cond_DOT', shape=[9, 5], dtype='float64' + ) X_DOT = paddle.static.data(name='X_DOT', shape=[9, 5], dtype='float64') Y_DOT = paddle.static.data(name='Y_DOT', shape=[9, 5], dtype='float64') self.jvp_args = (Cond_DOT, X_DOT, Y_DOT) @@ -1051,7 +1056,6 @@ def init_data(self): class TestEqPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'eq_p' @@ -1060,8 +1064,9 @@ def init_data(self): self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -1081,7 +1086,6 @@ def init_data(self): class TestGtPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'gt_p' @@ -1090,8 +1094,9 @@ def init_data(self): self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -1111,7 +1116,6 @@ def init_data(self): class TestGePJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'ge_p' @@ -1120,8 +1124,9 @@ def init_data(self): self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -1141,7 +1146,6 @@ def init_data(self): class TestNePJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'ne_p' @@ -1150,8 +1154,9 @@ def init_data(self): self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -1171,7 +1176,6 @@ def init_data(self): class TestPowPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'pow_p' @@ -1179,8 +1183,9 @@ def init_data(self): Y = paddle.static.data(name='Y', shape=[5, 6], dtype='float32') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} @@ -1211,7 +1216,6 @@ def init_data(self): class TestMaxPJVPAndTranspose(TestAddPJVPAndTranspose): - def init_data(self): # Set prim op self.op_type = 'max_p' @@ -1219,8 +1223,9 @@ def init_data(self): Y = paddle.static.data(name='Y', shape=[5, 6], dtype='float32') self.prim_input = {'X': X, 'Y': Y} self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.prim_attrs = {} diff --git a/python/paddle/fluid/tests/unittests/autograd/test_minimize.py b/python/paddle/fluid/tests/unittests/autograd/test_minimize.py index 10259802c6933..e3de0f24e0d52 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_minimize.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_minimize.py @@ -17,14 +17,16 @@ import numpy as np import paddle from paddle.incubate.autograd.primx import prim2orig -from paddle.incubate.autograd.utils import (disable_prim, enable_prim, - prim_enabled) +from paddle.incubate.autograd.utils import ( + disable_prim, + enable_prim, + prim_enabled, +) paddle.enable_static() class TestMinimize(unittest.TestCase): - def model(self, x, w, bias, opt): paddle.seed(0) place = paddle.CPUPlace() @@ -36,12 +38,12 @@ def model(self, x, w, bias, opt): with paddle.static.program_guard(main, startup): input_x = paddle.static.data('x', x.shape, dtype=x.dtype) input_x.stop_gradient = False - params_w = paddle.static.create_parameter(shape=w.shape, - dtype=w.dtype, - is_bias=False) - params_bias = paddle.static.create_parameter(shape=bias.shape, - dtype=bias.dtype, - is_bias=True) + params_w = paddle.static.create_parameter( + shape=w.shape, dtype=w.dtype, is_bias=False + ) + params_bias = paddle.static.create_parameter( + shape=bias.shape, dtype=bias.dtype, is_bias=True + ) y = paddle.tanh(paddle.matmul(input_x, params_w) + params_bias) loss = paddle.norm(y, p=2) opt = opt @@ -49,13 +51,9 @@ def model(self, x, w, bias, opt): if prim_enabled(): prim2orig(main.block(0)) exe.run(startup) - grads = exe.run(main, - feed={ - 'x': x, - 'w': w, - 'bias': bias - }, - fetch_list=grads) + grads = exe.run( + main, feed={'x': x, 'w': w, 'bias': bias}, fetch_list=grads + ) return grads def test_adam(self): diff --git a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py index 914ea38fa9ad7..69161d89a1162 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py @@ -24,14 +24,14 @@ ############################ Test orig2prim rules ############################ class TestElementWiseAddOrig2Prim(unittest.TestCase): - def setUp(self): self.main_program = paddle.static.Program() self.startup_program = paddle.static.Program() self.layer_help = LayerHelper('TestOrig2Prim') - with paddle.static.program_guard(self.main_program, - self.startup_program): + with paddle.static.program_guard( + self.main_program, self.startup_program + ): self.init_data() def init_data(self): @@ -41,8 +41,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -52,12 +53,15 @@ def init_data(self): self.out_map = {0: self.output['Out']} def test_op(self): - with paddle.static.program_guard(self.main_program, - self.startup_program): - op = self.layer_help.append_op(type=self.op_type, - inputs=self.input, - outputs=self.output, - attrs=self.attrs) + with paddle.static.program_guard( + self.main_program, self.startup_program + ): + op = self.layer_help.append_op( + type=self.op_type, + inputs=self.input, + outputs=self.output, + attrs=self.attrs, + ) prim_out = _orig2prim(op, *self.orig2prim_args) all_ops = [op.type for op in self.main_program.block(0).ops] @@ -69,7 +73,6 @@ def test_op(self): class TestSqrtOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'sqrt' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -78,19 +81,19 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['sqrt', 'sqrt_p'] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestElementWiseMulOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'elementwise_mul' X = paddle.static.data(name='X', shape=[8, 8], dtype='float') @@ -98,8 +101,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -110,7 +114,6 @@ def init_data(self): class TestElementWiseDivOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'elementwise_div' X = paddle.static.data(name='X', shape=[8, 8], dtype='float') @@ -118,8 +121,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -130,7 +134,6 @@ def init_data(self): class TestMatmulV2Orig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'matmul_v2' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -138,8 +141,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'trans_x': True, 'trans_y': True} @@ -149,7 +153,6 @@ def init_data(self): class TestTanhOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'tanh' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -158,18 +161,18 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['tanh', 'tanh_p'] self.out_map = {0: self.output['Out']} class TestSinOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'sin' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -178,18 +181,18 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['sin', 'sin_p'] self.out_map = {0: self.output['Out']} class TestCosOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'cos' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -198,18 +201,18 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['cos', 'cos_p'] self.out_map = {0: self.output['Out']} class TestExpOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'exp' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -218,18 +221,18 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['exp', 'exp_p'] self.out_map = {0: self.output['Out']} class TestErfOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'erf' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -238,18 +241,18 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['erf', 'erf_p'] self.out_map = {0: self.output['Out']} class TestAbsOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'abs' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -258,18 +261,18 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['abs', 'abs_p'] self.out_map = {0: self.output['Out']} class TestLogOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'log' X = paddle.static.data(name='X', shape=[3, 4], dtype='float') @@ -278,18 +281,18 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['log', 'log_p'] self.out_map = {0: self.output['Out']} class TestReshape2Orig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'reshape2' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -298,10 +301,10 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - X, - 'XShape': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': X, + 'XShape': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ), } self.attrs = {'shape': [6, 5]} @@ -316,7 +319,6 @@ def init_data(self): class TestConcatOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'concat' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -326,8 +328,9 @@ def init_data(self): 'X': [X, Y], } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': 0} @@ -340,7 +343,6 @@ def init_data(self): class TestSliceOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'slice' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -349,8 +351,9 @@ def init_data(self): 'Input': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = { 'axes': [0], @@ -364,7 +367,6 @@ def init_data(self): class TestFillZerosLikeOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'fill_zeros_like' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -373,18 +375,18 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['fill_zeros_like', 'fill_constant_p'] self.out_map = {0: self.output['Out']} class TestFillAnyLikeOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'fill_any_like' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -393,18 +395,18 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['fill_any_like', 'fill_constant_p'] self.out_map = {0: self.output['Out']} class TestFillAnyLikeOrig2Prim2(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'fill_any_like' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -413,18 +415,18 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'dtype': paddle.float32, 'value': 5} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['fill_any_like', 'fill_constant_p'] self.out_map = {0: self.output['Out']} class TestSumOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'sum' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -432,18 +434,18 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = ((X, Y), ) + self.orig2prim_args = ((X, Y),) self.all_ops = ['sum', 'add_p'] self.out_map = {0: self.output['Out']} class TestPNormOrig2Prim1(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'p_norm' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -452,21 +454,21 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = { 'porder': 1, 'asvector': True, } - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['p_norm', 'reshape_p', 'abs_p', 'reduce_sum_p'] self.out_map = {0: self.output['Out']} class TestPNormOrig2Prim2(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'p_norm' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -475,23 +477,27 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = { 'porder': 2, 'asvector': True, } - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = [ - 'p_norm', 'reshape_p', 'sqrt_p', 'reduce_sum_p', 'mul_p' + 'p_norm', + 'reshape_p', + 'sqrt_p', + 'reduce_sum_p', + 'mul_p', ] self.out_map = {0: self.output['Out']} class TestIndexSelectOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'index_select' X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') @@ -499,8 +505,9 @@ def init_data(self): self.input = {'X': X, 'Index': Index} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = { 'dim': 0, @@ -515,7 +522,6 @@ def init_data(self): class TestElementwiseSubOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'elementwise_sub' X = paddle.static.data(name='X', shape=[5, 6], dtype='int32') @@ -523,8 +529,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = { 'dim': 0, @@ -539,7 +546,6 @@ def init_data(self): class TestScaleOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'scale' X = paddle.static.data(name='X', shape=[10, 7], dtype='int32') @@ -548,8 +554,9 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'scale': 2.0, 'bias': 1.0, 'bias_after_scale': True} @@ -558,13 +565,16 @@ def init_data(self): X, ) self.all_ops = [ - 'scale', 'fill_constant_p', 'fill_constant_p', 'mul_p', 'add_p' + 'scale', + 'fill_constant_p', + 'fill_constant_p', + 'mul_p', + 'add_p', ] self.out_map = {0: self.output['Out']} class TestAssignOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'assign' X = paddle.static.data(name='X', shape=[10, 7], dtype='int32') @@ -573,18 +583,18 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['assign', 'fill_constant_p', 'add_p'] self.out_map = {0: self.output['Out']} class TestWhereOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'where' Cond = paddle.static.data(name='Condition', shape=[5, 6], dtype='bool') @@ -593,8 +603,9 @@ def init_data(self): self.input = {'Condition': Cond, 'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} self.orig2prim_args = (Cond, X, Y) @@ -603,7 +614,6 @@ def init_data(self): class TestEqualOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'equal' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') @@ -611,8 +621,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Out': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} self.orig2prim_args = (X, Y) @@ -622,7 +633,6 @@ def init_data(self): class TestNeOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'not_equal' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') @@ -630,8 +640,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Out': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} self.orig2prim_args = (X, Y) @@ -641,7 +652,6 @@ def init_data(self): class TestGtOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'greater_than' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') @@ -649,8 +659,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Out': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} self.orig2prim_args = (X, Y) @@ -660,7 +671,6 @@ def init_data(self): class TestGeOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'greater_equal' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') @@ -668,8 +678,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Out': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} self.orig2prim_args = (X, Y) @@ -679,7 +690,6 @@ def init_data(self): class TestPowOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'elementwise_pow' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') @@ -687,8 +697,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -699,7 +710,6 @@ def init_data(self): class TestMaxOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'elementwise_max' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') @@ -707,8 +717,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -719,97 +730,119 @@ def init_data(self): class TestGeluOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'gelu' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'approximate': False} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = [ - 'gelu', 'add_p', 'erf_p', 'fill_constant_p', 'fill_constant_p', - 'fill_constant_p', 'mul_p', 'mul_p', 'mul_p' + 'gelu', + 'add_p', + 'erf_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'mul_p', + 'mul_p', + 'mul_p', ] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestGeluApproximateOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'gelu' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'approximate': True} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = [ - 'add_p', 'add_p', 'fill_constant_p', 'fill_constant_p', - 'fill_constant_p', 'fill_constant_p', 'fill_constant_p', 'gelu', - 'mul_p', 'mul_p', 'mul_p', 'mul_p', 'pow_p', 'tanh_p' + 'add_p', + 'add_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'gelu', + 'mul_p', + 'mul_p', + 'mul_p', + 'mul_p', + 'pow_p', + 'tanh_p', ] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestDropoutOrig2PrimCase1(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'dropout' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Mask': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.uint8), - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype), + 'Mask': self.layer_help.create_variable_for_type_inference( + dtype=paddle.uint8 + ), + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ), } self.attrs = { 'dropout_prob': 0.5, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.orig2prim_args = (None, X) self.all_ops = [ - 'bernoulli_p', 'mul_p', 'fill_constant_p', 'div_p', 'cast_p', - 'dropout' + 'bernoulli_p', + 'mul_p', + 'fill_constant_p', + 'div_p', + 'cast_p', + 'dropout', ] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Mask'], 1: self.output['Out']} class TestDropoutOrig2PrimCase2(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'dropout' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Mask': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.uint8), - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype), + 'Mask': self.layer_help.create_variable_for_type_inference( + dtype=paddle.uint8 + ), + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ), } self.attrs = { 'dropout_prob': 0.5, 'is_test': False, - 'dropout_implementation': 'downgrade_in_infer' + 'dropout_implementation': 'downgrade_in_infer', } self.orig2prim_args = (None, X) @@ -819,23 +852,23 @@ def init_data(self): class TestDropoutOrig2PrimCase3(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'dropout' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Mask': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.uint8), - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype), + 'Mask': self.layer_help.create_variable_for_type_inference( + dtype=paddle.uint8 + ), + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ), } self.attrs = { 'dropout_prob': 0.5, 'is_test': True, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.orig2prim_args = (None, X) @@ -845,35 +878,38 @@ def init_data(self): class TestDropoutOrig2PrimCase4(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'dropout' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Mask': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.uint8), - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype), + 'Mask': self.layer_help.create_variable_for_type_inference( + dtype=paddle.uint8 + ), + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ), } self.attrs = { 'dropout_prob': 0.5, 'is_test': True, - 'dropout_implementation': 'downgrade_in_infer' + 'dropout_implementation': 'downgrade_in_infer', } self.orig2prim_args = (None, X) self.all_ops = [ - 'bernoulli_p', 'fill_constant_p', 'mul_p', 'cast_p', 'dropout' + 'bernoulli_p', + 'fill_constant_p', + 'mul_p', + 'cast_p', + 'dropout', ] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Mask'], 1: self.output['Out']} class TestReduceSumOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'reduce_sum' @@ -881,87 +917,90 @@ def init_data(self): self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': [0, 1], 'keep_dim': False} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['reduce_sum', 'reduce_sum_p'] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestReduceMeanOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'reduce_mean' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': [0, 1], 'keep_dim': False} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = [ - 'reduce_mean', 'reduce_sum_p', 'fill_constant_p', 'div_p' + 'reduce_mean', + 'reduce_sum_p', + 'fill_constant_p', + 'div_p', ] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestSizeOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'size' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'Input': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.int64) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=paddle.int64 + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['size', 'fill_constant_p'] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestCastOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'cast' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'in_dtype': X.dtype, 'out_dtype': paddle.float64} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['cast', 'cast_p'] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestPowScalarOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'pow' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } - self.attrs = {'factor': 2.} + self.attrs = {'factor': 2.0} self.orig2prim_args = (None, X) self.all_ops = ['pow', 'pow_p', 'fill_constant_p'] # { prim_op_output_index: orig_op_output_var } @@ -969,25 +1008,24 @@ def init_data(self): class TestSquareOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'square' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['square', 'pow_p', 'fill_constant_p'] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestRSqrtOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'rsqrt' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -996,19 +1034,19 @@ def init_data(self): 'X': X, } self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = ['rsqrt', 'rsqrt_p'] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} class TestBatchnormOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'batch_norm' x = paddle.static.data(name='X', shape=[5, 8], dtype='float') @@ -1022,20 +1060,23 @@ def init_data(self): "Scale": [w], "Bias": [b], "Mean": [m], - "Variance": [v] + "Variance": [v], } saved_variance = self.layer_help.create_variable_for_type_inference( - dtype=x.dtype, stop_gradient=True) + dtype=x.dtype, stop_gradient=True + ) batch_norm_out = self.layer_help.create_variable_for_type_inference( - x.dtype) + x.dtype + ) saved_mean = self.layer_help.create_variable_for_type_inference( - dtype=x.dtype, stop_gradient=True) + dtype=x.dtype, stop_gradient=True + ) self.output = { "Y": [batch_norm_out], "MeanOut": [m], "VarianceOut": [v], "SavedMean": [saved_mean], - "SavedVariance": [saved_variance] + "SavedVariance": [saved_variance], } self.attrs = { @@ -1050,31 +1091,60 @@ def init_data(self): } self.orig2prim_args = (b, m, None, w, v, x) self.all_ops = [ - 'add_p', 'add_p', 'add_p', 'add_p', 'batch_norm', 'broadcast_p', - 'broadcast_p', 'broadcast_p', 'broadcast_p', 'broadcast_p', 'div_p', - 'div_p', 'div_p', 'fill_constant_p', 'fill_constant_p', - 'fill_constant_p', 'fill_constant_p', 'fill_constant_p', - 'fill_constant_p', 'fill_constant_p', 'fill_constant_p', - 'fill_constant_p', 'mul_p', 'mul_p', 'mul_p', 'mul_p', 'mul_p', - 'pow_p', 'reduce_sum_p', 'reduce_sum_p', 'reshape_p', 'reshape_p', - 'reshape_p', 'reshape_p', 'sqrt_p', 'sub_p', 'sub_p', 'sub_p', - 'sub_p' + 'add_p', + 'add_p', + 'add_p', + 'add_p', + 'batch_norm', + 'broadcast_p', + 'broadcast_p', + 'broadcast_p', + 'broadcast_p', + 'broadcast_p', + 'div_p', + 'div_p', + 'div_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'mul_p', + 'mul_p', + 'mul_p', + 'mul_p', + 'mul_p', + 'pow_p', + 'reduce_sum_p', + 'reduce_sum_p', + 'reshape_p', + 'reshape_p', + 'reshape_p', + 'reshape_p', + 'sqrt_p', + 'sub_p', + 'sub_p', + 'sub_p', + 'sub_p', ] # { prim_op_output_index: orig_op_output_var } self.out_map = {} class TestFillConstantOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'fill_constant' - self.attrs = {'value': 1., 'shape': (2, 3), 'dtype': paddle.float32} + self.attrs = {'value': 1.0, 'shape': (2, 3), 'dtype': paddle.float32} self.input = {} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.float32) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=paddle.float32 + ) } self.orig2prim_args = (None, None, None) @@ -1084,14 +1154,13 @@ def init_data(self): class TestUniformRandomOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'uniform_random' self.input = {} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.float32) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=paddle.float32 + ) } self.attrs = {'shape': [1, 2]} @@ -1101,7 +1170,6 @@ def init_data(self): class TestSigmoidOrig2Prim(TestElementWiseAddOrig2Prim): - def init_data(self): self.op_type = 'sigmoid' X = paddle.static.data(name='X', shape=[3], dtype='float32') @@ -1109,15 +1177,21 @@ def init_data(self): self.attrs = {} self.input = {'X': X} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.float32) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=paddle.float32 + ) } - self.orig2prim_args = (X, ) + self.orig2prim_args = (X,) self.all_ops = [ - 'sigmoid', 'div_p', 'fill_constant_p', 'add_p', 'fill_constant_p', - 'exp_p', 'fill_constant_p', 'sub_p' + 'sigmoid', + 'div_p', + 'fill_constant_p', + 'add_p', + 'fill_constant_p', + 'exp_p', + 'fill_constant_p', + 'sub_p', ] self.out_map = {0: self.output['Out']} diff --git a/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py b/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py index 13423690ed869..b864bdaa45f03 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py @@ -24,14 +24,14 @@ ############################ Test prim2orig rules ############################ class TestAddPPrim2Orig(unittest.TestCase): - def setUp(self): self.main_program = paddle.static.Program() self.startup_program = paddle.static.Program() self.layer_help = LayerHelper('TestPrim2Orig') - with paddle.static.program_guard(self.main_program, - self.startup_program): + with paddle.static.program_guard( + self.main_program, self.startup_program + ): self.init_data() def init_data(self): @@ -41,8 +41,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -52,12 +53,15 @@ def init_data(self): self.out_map = {self.output['Z']: 0} def test_op(self): - with paddle.static.program_guard(self.main_program, - self.startup_program): - op = self.layer_help.append_op(type=self.op_type, - inputs=self.input, - outputs=self.output, - attrs=self.attrs) + with paddle.static.program_guard( + self.main_program, self.startup_program + ): + op = self.layer_help.append_op( + type=self.op_type, + inputs=self.input, + outputs=self.output, + attrs=self.attrs, + ) orig_out = _prim2orig(op, *self.prim2orig_args) all_ops = [op.type for op in self.main_program.block(0).ops] @@ -68,7 +72,6 @@ def test_op(self): class TestSubPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'sub_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -76,8 +79,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -87,7 +91,6 @@ def init_data(self): class TestMulPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'mul_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -95,8 +98,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -106,7 +110,6 @@ def init_data(self): class TestDivPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'div_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -114,8 +117,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -125,7 +129,6 @@ def init_data(self): class TestSqrtPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'sqrt_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -134,18 +137,18 @@ def init_data(self): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['sqrt_p', 'sqrt'] self.out_map = {self.output['Y']: 0} class TestTanhPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'tanh_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -154,18 +157,18 @@ def init_data(self): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['tanh_p', 'tanh'] self.out_map = {self.output['Y']: 0} class TestSinPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'sin_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -174,18 +177,18 @@ def init_data(self): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['sin_p', 'sin'] self.out_map = {self.output['Y']: 0} class TestCosPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'cos_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -194,18 +197,18 @@ def init_data(self): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['cos_p', 'cos'] self.out_map = {self.output['Y']: 0} class TestExpPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'exp_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -214,18 +217,18 @@ def init_data(self): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['exp_p', 'exp'] self.out_map = {self.output['Y']: 0} class TestErfPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'erf_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -234,18 +237,18 @@ def init_data(self): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['erf_p', 'erf'] self.out_map = {self.output['Y']: 0} class TestAbsPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'abs_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -254,18 +257,18 @@ def init_data(self): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['abs_p', 'abs'] self.out_map = {self.output['Y']: 0} class TestLogPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'log_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -274,18 +277,18 @@ def init_data(self): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['log_p', 'log'] self.out_map = {self.output['Y']: 0} class TestReshapePPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'reshape_p' X = paddle.static.data(name='X', shape=[2, 8], dtype='float64') @@ -294,18 +297,18 @@ def init_data(self): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'shape': [4, 4]} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['reshape_p', 'reshape2'] self.out_map = {self.output['Y']: 0} class TestBroadcastPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'broadcast_p' X = paddle.static.data(name='X', shape=[2, 8], dtype='float64') @@ -314,18 +317,18 @@ def init_data(self): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'shape': [10, 2, 8]} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['broadcast_p', 'expand_v2'] self.out_map = {self.output['Y']: 0} class TestTransposePPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'transpose_p' X = paddle.static.data(name='X', shape=[7, 8, 9, 10], dtype='float64') @@ -334,18 +337,18 @@ def init_data(self): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': [1, 2, 0, 3]} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['transpose_p', 'transpose2'] self.out_map = {self.output['Y']: 0} class TestSplitPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'split_p' X = paddle.static.data(name='X', shape=[3, 9, 5], dtype='float64') @@ -356,12 +359,14 @@ def init_data(self): self.output = { 'YS': [ self.layer_help.create_variable_for_type_inference( - dtype=X.dtype) for i in range(3) + dtype=X.dtype + ) + for i in range(3) ] } self.attrs = {'num_or_sections': [2, 3, 4], 'axis': 1} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['split_p', 'split'] self.out_map = { self.output['YS'][0]: 0, @@ -371,7 +376,6 @@ def init_data(self): class TestConcatPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'concat_p' X = paddle.static.data(name='X', shape=[3, 9, 5], dtype='float64') @@ -382,36 +386,36 @@ def init_data(self): 'XS': [X, Y, Z], } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': 0} - self.prim2orig_args = ((X, Y, Z), ) + self.prim2orig_args = ((X, Y, Z),) self.all_ops = ['concat_p', 'concat'] self.out_map = {self.output['Y']: 0} class TestReducePPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'reduce_sum_p' X = paddle.static.data(name='X', shape=[3, 9, 5], dtype='float64') self.input = {'X': X} self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': [1], 'keepdim': True} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['reduce_sum_p', 'reduce_sum'] self.out_map = {self.output['Y']: 0} class TestMatmulPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'matmul_p' X = paddle.static.data(name='X', shape=[9, 5], dtype='float64') @@ -419,8 +423,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -430,7 +435,6 @@ def init_data(self): class TestSliceSelectPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'slice_select_p' X = paddle.static.data(name='X', shape=[9, 5], dtype='float64') @@ -439,18 +443,18 @@ def init_data(self): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': [0], 'starts': [1], 'ends': [8], 'strides': [2]} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['slice_select_p', 'strided_slice'] self.out_map = {self.output['Y']: 0} class TestSliceAssignPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'slice_assign_p' X = paddle.static.data(name='X', shape=[9, 5], dtype='float64') @@ -458,8 +462,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'axis': [1], 'starts': [0], 'ends': [3], 'strides': [1]} @@ -469,18 +474,18 @@ def init_data(self): class TestGatherPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'gather_p' X = paddle.static.data(name='X', shape=[9, 5], dtype='float64') - IndexTensor = paddle.static.data(name='IndexTensor', - shape=[3], - dtype='int32') + IndexTensor = paddle.static.data( + name='IndexTensor', shape=[3], dtype='int32' + ) self.input = {'X': X, 'IndexTensor': IndexTensor} self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = { 'axis': 0, @@ -495,19 +500,19 @@ def init_data(self): class TestScatterAddPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'scatter_add_p' X = paddle.static.data(name='X', shape=[9, 5], dtype='float64') Y = paddle.static.data(name='Y', shape=[3, 5], dtype='float64') - IndexTensor = paddle.static.data(name='IndexTensor', - shape=[3], - dtype='int32') + IndexTensor = paddle.static.data( + name='IndexTensor', shape=[3], dtype='int32' + ) self.input = {'X': X, 'Y': Y, 'IndexTensor': IndexTensor} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = { 'axis': 0, @@ -515,20 +520,23 @@ def init_data(self): self.prim2orig_args = (IndexTensor, X, Y) self.all_ops = [ - 'scatter_add_p', 'fill_any_like', 'scatter', 'elementwise_add' + 'scatter_add_p', + 'fill_any_like', + 'scatter', + 'elementwise_add', ] self.out_map = {self.output['Z']: 0} class TestFillConstantPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'fill_constant_p' self.input = {} self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(paddle.int32) + 'Y': self.layer_help.create_variable_for_type_inference( + paddle.int32 + ) } self.attrs = {'value': 10, 'shape': [5, 5], 'dtype': paddle.int32} @@ -538,7 +546,6 @@ def init_data(self): class TestSelectPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'select_p' Cond = paddle.static.data(name='Condition', shape=[5, 6], dtype='bool') @@ -547,8 +554,9 @@ def init_data(self): self.input = {'Condition': Cond, 'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} self.prim2orig_args = (Cond, X, Y) @@ -557,7 +565,6 @@ def init_data(self): class TestEqPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'eq_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -565,8 +572,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Z': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} @@ -576,7 +584,6 @@ def init_data(self): class TestNePPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'ne_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -584,8 +591,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Z': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} @@ -595,7 +603,6 @@ def init_data(self): class TestGtPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'gt_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -603,8 +610,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Z': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} @@ -614,7 +622,6 @@ def init_data(self): class TestGePPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'ge_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -622,8 +629,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype='bool') + 'Z': self.layer_help.create_variable_for_type_inference( + dtype='bool' + ) } self.attrs = {} @@ -633,7 +641,6 @@ def init_data(self): class TestPowPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'pow_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -641,8 +648,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -652,7 +660,6 @@ def init_data(self): class TestMaxPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'max_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -660,8 +667,9 @@ def init_data(self): self.input = {'X': X, 'Y': Y} self.output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Z': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} @@ -671,15 +679,14 @@ def init_data(self): class TestBernoulliPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'bernoulli_p' self.input = {} self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.float64) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=paddle.float64 + ) } self.attrs = {'shape': [7, 8], 'dtype': paddle.float64, 'p': 0.5} @@ -689,7 +696,6 @@ def init_data(self): class TestCastPPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'cast_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -698,18 +704,18 @@ def init_data(self): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {'dtype': paddle.int64} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['cast_p', 'cast'] self.out_map = {self.output['Y']: 0} class TestRsqrtPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'rsqrt_p' X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') @@ -718,33 +724,33 @@ def init_data(self): 'X': X, } self.output = { - 'Y': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + 'Y': self.layer_help.create_variable_for_type_inference( + dtype=X.dtype + ) } self.attrs = {} - self.prim2orig_args = (X, ) + self.prim2orig_args = (X,) self.all_ops = ['rsqrt_p', 'rsqrt'] self.out_map = {self.output['Y']: 0} class TestUniformRandomPrim2Orig(TestAddPPrim2Orig): - def init_data(self): self.op_type = 'uniform_random_p' self.input = {} self.output = { - 'Out': - self.layer_help.create_variable_for_type_inference( - dtype=paddle.float64) + 'Out': self.layer_help.create_variable_for_type_inference( + dtype=paddle.float64 + ) } self.attrs = { 'shape': [1, 2, 3], 'min': -1.0, 'max': 1.0, 'seed': 0, - 'dtype': paddle.float64 + 'dtype': paddle.float64, } self.prim2orig_args = () diff --git a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py index 2451ed4190c9e..7f1a06c9240ba 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py @@ -27,13 +27,30 @@ @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'xs', 'dtype'), ( - ('uniform_random', - lambda: paddle.uniform([1, 2, 3], dtype='float32', min=0, max=1.0, seed=1), - (), 'int32'), ('sigmoid', paddle.nn.functional.sigmoid, - (np.random.rand(5, ), ), 'float32'))) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'xs', 'dtype'), + ( + ( + 'uniform_random', + lambda: paddle.uniform( + [1, 2, 3], dtype='float32', min=0, max=1.0, seed=1 + ), + (), + 'int32', + ), + ( + 'sigmoid', + paddle.nn.functional.sigmoid, + ( + np.random.rand( + 5, + ), + ), + 'float32', + ), + ), +) class TestFowardApi(unittest.TestCase): - @classmethod def setUpClass(cls): cls.xs = tuple(x.astype(cls.dtype) for x in cls.xs) @@ -47,14 +64,14 @@ def tearDown(self): paddle.disable_static() def test_grad(self): - def expected(): paddle.incubate.autograd.disable_prim() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs = utils.gen_static_inputs_and_feed( - self.xs, stop_gradient=False) + self.xs, stop_gradient=False + ) out = self.fun(*static_xs) exe = paddle.static.Executor() exe.run(sp) @@ -68,7 +85,8 @@ def actual(): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs = utils.gen_static_inputs_and_feed( - self.xs, stop_gradient=False) + self.xs, stop_gradient=False + ) out = self.fun(*static_xs) primx.orig2prim(mp.block(0)) primx.prim2orig(mp.block(0)) @@ -86,18 +104,32 @@ def actual(): @utils.place(config.DEVICES) -@utils.parameterize((utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'dtype'), - (('dropout', paddle.nn.functional.dropout, - (np.random.rand(5000, 5000), ), None, 'float32'), )) +@utils.parameterize( + (utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'dtype'), + ( + ( + 'dropout', + paddle.nn.functional.dropout, + (np.random.rand(5000, 5000),), + None, + 'float32', + ), + ), +) class TestDropoutGrad(unittest.TestCase): - @classmethod def setUpClass(cls): cls.xs = tuple(x.astype(cls.dtype) for x in cls.xs) - cls._rtol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("rtol") - cls._atol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("atol") + cls._rtol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("rtol") + ) + cls._atol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("atol") + ) def setUp(self): paddle.enable_static() @@ -108,16 +140,17 @@ def tearDown(self): paddle.disable_static() def test_grad(self): - def expected(): paddle.incubate.autograd.disable_prim() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) + self.xs, self.v, stop_gradient=False + ) _, ys_grad = paddle.incubate.autograd.vjp( - self.fun, static_xs, static_v) + self.fun, static_xs, static_v + ) exe = paddle.static.Executor() exe.run(sp) out = exe.run(mp, feed=feed, fetch_list=ys_grad) @@ -130,9 +163,13 @@ def actual(): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.incubate.autograd.grad(ys, static_xs, static_v) paddle.incubate.autograd.prim2orig(mp.block(0)) exe = paddle.static.Executor() @@ -151,17 +188,30 @@ def actual(): @utils.place(config.DEVICES) @utils.parameterize( (utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'dtype'), - (('matmul', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), None, 'float32'), )) + ( + ( + 'matmul', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + None, + 'float32', + ), + ), +) class TestWithoutProgramGuard(unittest.TestCase): - @classmethod def setUpClass(cls): cls.xs = tuple(x.astype(cls.dtype) for x in cls.xs) - cls._rtol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("rtol") - cls._atol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("atol") + cls._rtol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("rtol") + ) + cls._atol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("atol") + ) def setUp(self): paddle.enable_static() @@ -172,18 +222,22 @@ def tearDown(self): paddle.disable_static() def test_forward_grad_without_program_guard(self): - def with_program_guard(): paddle.incubate.autograd.enable_prim() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.incubate.autograd.forward_grad( - ys, static_xs, static_v) + ys, static_xs, static_v + ) paddle.incubate.autograd.prim2orig(mp.block(0)) exe = paddle.static.Executor() exe.run(sp) @@ -194,11 +248,16 @@ def with_program_guard(): def without_program_guard(): paddle.incubate.autograd.enable_prim() feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.incubate.autograd.forward_grad( - ys, static_xs, static_v) + ys, static_xs, static_v + ) sp = paddle.fluid.framework.default_startup_program() mp = paddle.fluid.framework.default_main_program() exe = paddle.static.Executor() @@ -210,22 +269,27 @@ def without_program_guard(): expected = with_program_guard() actual = without_program_guard() self.assertEqual(type(actual), type(expected)) - np.testing.assert_allclose(np.concatenate(actual), - np.concatenate(expected), - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + np.concatenate(actual), + np.concatenate(expected), + rtol=self._rtol, + atol=self._atol, + ) def test_grad_without_program_guard(self): - def with_program_guard(): paddle.incubate.autograd.enable_prim() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) xs_grad = paddle.incubate.autograd.grad(ys, static_xs, static_v) paddle.incubate.autograd.prim2orig(mp.block(0)) exe = paddle.static.Executor() @@ -237,9 +301,13 @@ def with_program_guard(): def without_program_guard(): paddle.incubate.autograd.enable_prim() feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) xs_grad = paddle.incubate.autograd.grad(ys, static_xs, static_v) sp = paddle.fluid.framework.default_startup_program() mp = paddle.fluid.framework.default_main_program() @@ -253,43 +321,91 @@ def without_program_guard(): actual = without_program_guard() for i, j in zip(actual, expected): self.assertEqual(type(i), type(j)) - np.testing.assert_allclose(np.concatenate(i), - np.concatenate(j), - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + np.concatenate(i), + np.concatenate(j), + rtol=self._rtol, + atol=self._atol, + ) @utils.place(config.DEVICES) @utils.parameterize( (utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'dtype'), - (('matmul', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), None, 'float32'), - ('multiply', paddle.multiply, - (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float64'), - ('add', paddle.add, - (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float32'), - ('input_not_sequence', paddle.tanh, - (np.random.rand(5, 5), ), None, 'float64'), - ('input_gradients_not_none', paddle.matmul, - (np.random.rand(3, 3), np.random.rand(3, 3)), - (np.random.rand(3, 3), np.random.rand(3, 3)), 'float64'), - ('log', paddle.log, (np.random.rand(3, 4), ), None, 'float32'), - ('abs', paddle.abs, (np.random.uniform(-10, 10, - (10, 10)), ), None, 'float32'), - ('rsqrt', paddle.rsqrt, (np.random.rand(100, 200), ), None, 'float32'), - ('sigmoid', paddle.nn.functional.sigmoid, - (np.random.rand(5, ), ), None, 'float32'))) + ( + ( + 'matmul', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + None, + 'float32', + ), + ( + 'multiply', + paddle.multiply, + (np.random.rand(2, 3), np.random.rand(2, 3)), + None, + 'float64', + ), + ( + 'add', + paddle.add, + (np.random.rand(2, 3), np.random.rand(2, 3)), + None, + 'float32', + ), + ( + 'input_not_sequence', + paddle.tanh, + (np.random.rand(5, 5),), + None, + 'float64', + ), + ( + 'input_gradients_not_none', + paddle.matmul, + (np.random.rand(3, 3), np.random.rand(3, 3)), + (np.random.rand(3, 3), np.random.rand(3, 3)), + 'float64', + ), + ('log', paddle.log, (np.random.rand(3, 4),), None, 'float32'), + ( + 'abs', + paddle.abs, + (np.random.uniform(-10, 10, (10, 10)),), + None, + 'float32', + ), + ('rsqrt', paddle.rsqrt, (np.random.rand(100, 200),), None, 'float32'), + ( + 'sigmoid', + paddle.nn.functional.sigmoid, + ( + np.random.rand( + 5, + ), + ), + None, + 'float32', + ), + ), +) # paddle.where, paddle.pow, paddle.maximum has no double grad definition, # can not compute forward grad use double trick class TestForwardGrad(unittest.TestCase): - @classmethod def setUpClass(cls): cls.xs = tuple(x.astype(cls.dtype) for x in cls.xs) - cls._rtol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("rtol") - cls._atol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("atol") + cls._rtol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("rtol") + ) + cls._atol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("atol") + ) def setUp(self): paddle.enable_static() @@ -300,16 +416,17 @@ def tearDown(self): paddle.disable_static() def test_forward_grad(self): - def expected(): paddle.incubate.autograd.disable_prim() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) + self.xs, self.v, stop_gradient=False + ) _, ys_grad = paddle.incubate.autograd.jvp( - self.fun, static_xs, static_v) + self.fun, static_xs, static_v + ) exe = paddle.static.Executor() exe.run(sp) out = exe.run(mp, feed=feed, fetch_list=ys_grad) @@ -322,11 +439,16 @@ def actual(): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.incubate.autograd.forward_grad( - ys, static_xs, static_v) + ys, static_xs, static_v + ) paddle.incubate.autograd.prim2orig(mp.block(0)) exe = paddle.static.Executor() exe.run(sp) @@ -337,10 +459,12 @@ def actual(): actual = actual() expected = expected() self.assertEqual(type(actual), type(expected)) - np.testing.assert_allclose(np.concatenate(actual), - np.concatenate(expected), - rtol=self._rtol, - atol=self._atol) + np.testing.assert_allclose( + np.concatenate(actual), + np.concatenate(expected), + rtol=self._rtol, + atol=self._atol, + ) def test_prim_disabled(self): paddle.incubate.autograd.disable_prim() @@ -349,11 +473,16 @@ def test_prim_disabled(self): with self.assertRaises(RuntimeError): with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.incubate.autograd.forward_grad( - ys, static_xs, static_v) + ys, static_xs, static_v + ) paddle.incubate.autograd.prim2orig(mp.block(0)) exe = paddle.static.Executor() exe.run(sp) @@ -364,11 +493,13 @@ def test_illegal_param(self): paddle.incubate.autograd.enable_prim() with self.assertRaises(TypeError): paddle.incubate.autograd.forward_grad( - 1, paddle.static.data('inputs', shape=[1])) + 1, paddle.static.data('inputs', shape=[1]) + ) with self.assertRaises(TypeError): paddle.incubate.autograd.forward_grad( - paddle.static.data('targets', shape=[1]), 1) + paddle.static.data('targets', shape=[1]), 1 + ) paddle.incubate.autograd.disable_prim() @@ -379,107 +510,247 @@ def test_illegal_param(self): @utils.parameterize( (utils.TEST_CASE_NAME, 'fun', 'xs', 'v', 'dtype'), ( - ('matmul', paddle.matmul, - (np.random.rand(2, 3), np.random.rand(3, 2)), None, 'float32'), - ('multiply', paddle.multiply, - (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float64'), - ('div', paddle.divide, - (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float64'), - ('add', paddle.add, - (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float32'), - ('input_not_sequence', paddle.tanh, - (np.random.rand(5, 5), ), None, 'float64'), - ('input_gradients_not_none', paddle.matmul, - (np.random.rand(3, 3), np.random.rand(3, 3)), - (np.random.rand(3, 3), ), 'float64'), - ('sin', paddle.sin, (np.random.rand(100, 200), ), None, 'float32'), - ('rsqrt', paddle.rsqrt, (np.random.rand(100, 200), ), None, 'float32'), - ('cos', paddle.cos, (np.random.rand(200, 90), ), None, 'float32'), - ('exp', paddle.exp, (np.random.rand(299, 320), ), None, 'float32'), + ( + 'matmul', + paddle.matmul, + (np.random.rand(2, 3), np.random.rand(3, 2)), + None, + 'float32', + ), + ( + 'multiply', + paddle.multiply, + (np.random.rand(2, 3), np.random.rand(2, 3)), + None, + 'float64', + ), + ( + 'div', + paddle.divide, + (np.random.rand(2, 3), np.random.rand(2, 3)), + None, + 'float64', + ), + ( + 'add', + paddle.add, + (np.random.rand(2, 3), np.random.rand(2, 3)), + None, + 'float32', + ), + ( + 'input_not_sequence', + paddle.tanh, + (np.random.rand(5, 5),), + None, + 'float64', + ), + ( + 'input_gradients_not_none', + paddle.matmul, + (np.random.rand(3, 3), np.random.rand(3, 3)), + (np.random.rand(3, 3),), + 'float64', + ), + ('sin', paddle.sin, (np.random.rand(100, 200),), None, 'float32'), + ('rsqrt', paddle.rsqrt, (np.random.rand(100, 200),), None, 'float32'), + ('cos', paddle.cos, (np.random.rand(200, 90),), None, 'float32'), + ('exp', paddle.exp, (np.random.rand(299, 320),), None, 'float32'), # In where op, grad of condition computed by paddle.static.gradients is None, # and paddle.incubate.autograd.grad will replace None with zeros while transpose # will just return None because cond_dot is unused, that is a diff. - ('select', where_wrap, - (np.random.rand(3, 4), np.random.rand(3, 4)), None, 'float32'), + ( + 'select', + where_wrap, + (np.random.rand(3, 4), np.random.rand(3, 4)), + None, + 'float32', + ), # pow_p and pow has diff when compute z_dot of 0^0 - ('pow', paddle.pow, - (np.array([1, 2, 3]), np.array([0, 2, 7])), None, 'float32'), + ( + 'pow', + paddle.pow, + (np.array([1, 2, 3]), np.array([0, 2, 7])), + None, + 'float32', + ), # To make max_p consistent with paddle.maximum, be sure x.grad = 0 and y.grad = 1 when x==y. - ('max', paddle.maximum, ( - np.array([1, 2, 3]), - np.array([2, 2, 2]), - ), None, 'float32'), - ('erf', paddle.erf, (np.random.rand(300, 288), ), None, 'float32'), - ('gelu', paddle.nn.functional.gelu, - (np.random.rand(200, 189), ), None, 'float32'), - ('gelu_approximate', lambda x: paddle.nn.functional.gelu(x, True), - (np.random.rand(200, 189), ), None, 'float32'), - ('sum', paddle.sum, (np.random.rand(200, 345), ), None, 'float32'), - ('sigmoid', paddle.nn.functional.sigmoid, - (np.random.rand(5, ), ), None, 'float32'), - ('sum_with_axis', lambda x: paddle.sum(x, axis=1), - (np.random.rand(200, 345), ), None, 'float32'), - ('sum_with_keepdim', lambda x: paddle.sum(x, keepdim=True), - (np.random.rand(200, 345), ), None, 'float32'), - ('mean', paddle.mean, (np.random.rand(200, 345), ), None, 'float32'), - ('mean_with_axis', lambda x: paddle.mean(x, axis=1), - (np.random.rand(200, 345), ), None, 'float32'), - ('mean_with_keepdim', lambda x: paddle.mean(x, keepdim=True), - (np.random.rand(200, 345), ), None, 'float32'), - ('mean_with_axis_keepdim', - lambda x: paddle.mean(x, axis=0, keepdim=True), - (np.random.rand(200, 345), ), None, 'float32'), - ('abs', paddle.abs, (np.random.uniform(-10, 10, - (200, 345)), ), None, 'float32'), - ('cast_float', lambda x: paddle.cast(x, paddle.float64), - (np.random.rand(10, 20), ), None, 'float32'), - ('cast_int', lambda x: paddle.cast(x, paddle.int32), - (np.random.rand(10, 20), ), None, 'float32'), - ('square', paddle.square, (np.random.rand(100), ), None, 'float32'), - ('pow_scalar', lambda x: paddle.pow(x, 2), - (np.random.rand(20, 30), ), None, 'float32'), - ('var', paddle.var, (np.random.rand(200, 324), ), None, 'float32'), - ('var_with_axis', lambda x: paddle.var(x, axis=1), - (np.random.rand(10, 20, 30), ), None, 'float32'), - ('var_without_unbiased', - lambda x: paddle.var(x, axis=1, unbiased=False), - (np.random.rand(10, 20, 30), ), None, 'float32'), - ('var_with_keepdim', lambda x: paddle.var(x, axis=1, keepdim=True), - (np.random.rand(10, 20, 30), ), None, 'float32'), - ('bn', lambda x, w, b: paddle.nn.functional.batch_norm( - x, paddle.ones((10, )), paddle.ones( - (10, )), w, b), (np.random.rand(10, 10), np.random.rand(10), - np.random.rand(10)), None, 'float32'), - ('bn_train', lambda x, w, b: paddle.nn.functional.batch_norm( - x, paddle.ones((10, )), paddle.ones((10, )), w, b, training=True), - (np.random.rand( - 10, 10), np.random.rand(10), np.random.rand(10)), None, 'float32'), - ('bn_nhwc', lambda x, w, b: paddle.nn.functional.batch_norm( - x, - paddle.ones((10, )) + 1, - paddle.ones((10, )), - w, - b, - training=True, - data_format='NHWC', - ), (np.random.rand( - 10, 10), np.random.rand(10), np.random.rand(10)), None, 'float32'), - ('bn_global_stat', - lambda x, w, b: paddle.nn.functional.batch_norm(x, - paddle.ones( - (10, )) + 3.2, - paddle.ones( - (10, )) + 6.7, - w, - b, - training=True, - data_format='NHWC', - use_global_stats=True), - (np.random.rand( - 10, 10), np.random.rand(10), np.random.rand(10)), None, 'float32'), - )) + ( + 'max', + paddle.maximum, + ( + np.array([1, 2, 3]), + np.array([2, 2, 2]), + ), + None, + 'float32', + ), + ('erf', paddle.erf, (np.random.rand(300, 288),), None, 'float32'), + ( + 'gelu', + paddle.nn.functional.gelu, + (np.random.rand(200, 189),), + None, + 'float32', + ), + ( + 'gelu_approximate', + lambda x: paddle.nn.functional.gelu(x, True), + (np.random.rand(200, 189),), + None, + 'float32', + ), + ('sum', paddle.sum, (np.random.rand(200, 345),), None, 'float32'), + ( + 'sigmoid', + paddle.nn.functional.sigmoid, + ( + np.random.rand( + 5, + ), + ), + None, + 'float32', + ), + ( + 'sum_with_axis', + lambda x: paddle.sum(x, axis=1), + (np.random.rand(200, 345),), + None, + 'float32', + ), + ( + 'sum_with_keepdim', + lambda x: paddle.sum(x, keepdim=True), + (np.random.rand(200, 345),), + None, + 'float32', + ), + ('mean', paddle.mean, (np.random.rand(200, 345),), None, 'float32'), + ( + 'mean_with_axis', + lambda x: paddle.mean(x, axis=1), + (np.random.rand(200, 345),), + None, + 'float32', + ), + ( + 'mean_with_keepdim', + lambda x: paddle.mean(x, keepdim=True), + (np.random.rand(200, 345),), + None, + 'float32', + ), + ( + 'mean_with_axis_keepdim', + lambda x: paddle.mean(x, axis=0, keepdim=True), + (np.random.rand(200, 345),), + None, + 'float32', + ), + ( + 'abs', + paddle.abs, + (np.random.uniform(-10, 10, (200, 345)),), + None, + 'float32', + ), + ( + 'cast_float', + lambda x: paddle.cast(x, paddle.float64), + (np.random.rand(10, 20),), + None, + 'float32', + ), + ( + 'cast_int', + lambda x: paddle.cast(x, paddle.int32), + (np.random.rand(10, 20),), + None, + 'float32', + ), + ('square', paddle.square, (np.random.rand(100),), None, 'float32'), + ( + 'pow_scalar', + lambda x: paddle.pow(x, 2), + (np.random.rand(20, 30),), + None, + 'float32', + ), + ('var', paddle.var, (np.random.rand(200, 324),), None, 'float32'), + ( + 'var_with_axis', + lambda x: paddle.var(x, axis=1), + (np.random.rand(10, 20, 30),), + None, + 'float32', + ), + ( + 'var_without_unbiased', + lambda x: paddle.var(x, axis=1, unbiased=False), + (np.random.rand(10, 20, 30),), + None, + 'float32', + ), + ( + 'var_with_keepdim', + lambda x: paddle.var(x, axis=1, keepdim=True), + (np.random.rand(10, 20, 30),), + None, + 'float32', + ), + ( + 'bn', + lambda x, w, b: paddle.nn.functional.batch_norm( + x, paddle.ones((10,)), paddle.ones((10,)), w, b + ), + (np.random.rand(10, 10), np.random.rand(10), np.random.rand(10)), + None, + 'float32', + ), + ( + 'bn_train', + lambda x, w, b: paddle.nn.functional.batch_norm( + x, paddle.ones((10,)), paddle.ones((10,)), w, b, training=True + ), + (np.random.rand(10, 10), np.random.rand(10), np.random.rand(10)), + None, + 'float32', + ), + ( + 'bn_nhwc', + lambda x, w, b: paddle.nn.functional.batch_norm( + x, + paddle.ones((10,)) + 1, + paddle.ones((10,)), + w, + b, + training=True, + data_format='NHWC', + ), + (np.random.rand(10, 10), np.random.rand(10), np.random.rand(10)), + None, + 'float32', + ), + ( + 'bn_global_stat', + lambda x, w, b: paddle.nn.functional.batch_norm( + x, + paddle.ones((10,)) + 3.2, + paddle.ones((10,)) + 6.7, + w, + b, + training=True, + data_format='NHWC', + use_global_stats=True, + ), + (np.random.rand(10, 10), np.random.rand(10), np.random.rand(10)), + None, + 'float32', + ), + ), +) class TestGrad(unittest.TestCase): - def setUp(self): paddle.enable_static() paddle.incubate.autograd.enable_prim() @@ -491,22 +762,29 @@ def tearDown(self): @classmethod def setUpClass(cls): cls.xs = tuple(x.astype(cls.dtype) for x in cls.xs) - cls._rtol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("rtol") - cls._atol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("atol") + cls._rtol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("rtol") + ) + cls._atol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("atol") + ) def test_grad(self): - def expected(): paddle.incubate.autograd.disable_prim() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) + self.xs, self.v, stop_gradient=False + ) _, ys_grad = paddle.incubate.autograd.vjp( - self.fun, static_xs, static_v) + self.fun, static_xs, static_v + ) exe = paddle.static.Executor() exe.run(sp) out = exe.run(mp, feed=feed, fetch_list=ys_grad) @@ -519,9 +797,13 @@ def actual(): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.incubate.autograd.grad(ys, static_xs, static_v) paddle.incubate.autograd.prim2orig(mp.block(0)) exe = paddle.static.Executor() @@ -540,24 +822,29 @@ def test_illegal_param(self): paddle.incubate.autograd.enable_prim() with self.assertRaises(TypeError): paddle.incubate.autograd.grad( - 1, paddle.static.data('inputs', shape=[1])) + 1, paddle.static.data('inputs', shape=[1]) + ) with self.assertRaises(TypeError): paddle.incubate.autograd.grad( - paddle.static.data('targets', shape=[1]), 1) + paddle.static.data('targets', shape=[1]), 1 + ) paddle.incubate.autograd.disable_prim() def test_disable_prim(self): - def expected(): paddle.incubate.autograd.disable_prim() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.incubate.autograd.grad(ys, static_xs, static_v) exe = paddle.static.Executor() exe.run(sp) @@ -571,9 +858,13 @@ def actual(): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun(static_xs) + ) ys_grad = paddle.static.gradients(ys, static_xs, static_v) exe = paddle.static.Executor() exe.run(sp) @@ -599,7 +890,7 @@ def multiply_pd(x): sin_ag = lambda xs: anp.sin(xs[0]) cos_ag = lambda xs: anp.cos(xs[0]) exp_ag = lambda xs: anp.exp(xs[0]) -pow_ag = lambda xs: xs[0]**xs[1] +pow_ag = lambda xs: xs[0] ** xs[1] log_ag = lambda xs: anp.log(xs[0]) erf_ag = lambda xs: ascipy.special.erf(xs[0]) sigmoid_ag = lambda xs: 1.0 / (1 + anp.exp(-xs[0])) @@ -616,28 +907,63 @@ def gelu_ag(x, approximate=False): @utils.place(config.DEVICES) @utils.parameterize( - (utils.TEST_CASE_NAME, 'fun_pd', 'fun_ag', 'xs', 'v', 'dtype'), ( - ('multiply', multiply_pd, multiply_ag, - (np.random.rand(3, 5), ), None, 'float32'), - ('sin', paddle.sin, sin_ag, (np.random.rand(2, 3), ), None, 'float32'), - ('cos', paddle.cos, cos_ag, (np.random.rand(3, 4), ), None, 'float32'), - ('exp', paddle.exp, exp_ag, (np.random.rand(2, 3), ), None, 'float32'), - ('pow', paddle.pow, pow_ag, - (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float32'), - ('log', paddle.log, log_ag, (np.random.rand(3, 8), ), None, 'float32'), - ('erf', paddle.erf, erf_ag, - (np.random.rand(100, 200), ), None, 'float32'), - ('gelu', paddle.nn.functional.gelu, lambda xs: gelu_ag(xs[0]), - (np.random.rand(10, 20, 30), ), None, 'float32'), - ('gelu_approximate', - lambda x: paddle.nn.functional.gelu(x, approximate=True), - lambda xs: gelu_ag(xs[0], approximate=True), - (np.random.rand(10, 20, 30), ), None, 'float32'), - ('sigmoid', paddle.nn.functional.sigmoid, sigmoid_ag, - (np.random.rand(10, 20), ), None, 'float32'), - )) + (utils.TEST_CASE_NAME, 'fun_pd', 'fun_ag', 'xs', 'v', 'dtype'), + ( + ( + 'multiply', + multiply_pd, + multiply_ag, + (np.random.rand(3, 5),), + None, + 'float32', + ), + ('sin', paddle.sin, sin_ag, (np.random.rand(2, 3),), None, 'float32'), + ('cos', paddle.cos, cos_ag, (np.random.rand(3, 4),), None, 'float32'), + ('exp', paddle.exp, exp_ag, (np.random.rand(2, 3),), None, 'float32'), + ( + 'pow', + paddle.pow, + pow_ag, + (np.random.rand(2, 3), np.random.rand(2, 3)), + None, + 'float32', + ), + ('log', paddle.log, log_ag, (np.random.rand(3, 8),), None, 'float32'), + ( + 'erf', + paddle.erf, + erf_ag, + (np.random.rand(100, 200),), + None, + 'float32', + ), + ( + 'gelu', + paddle.nn.functional.gelu, + lambda xs: gelu_ag(xs[0]), + (np.random.rand(10, 20, 30),), + None, + 'float32', + ), + ( + 'gelu_approximate', + lambda x: paddle.nn.functional.gelu(x, approximate=True), + lambda xs: gelu_ag(xs[0], approximate=True), + (np.random.rand(10, 20, 30),), + None, + 'float32', + ), + ( + 'sigmoid', + paddle.nn.functional.sigmoid, + sigmoid_ag, + (np.random.rand(10, 20),), + None, + 'float32', + ), + ), +) class TestGradWithHigherOrder(unittest.TestCase): - def setUp(self): paddle.enable_static() paddle.incubate.autograd.enable_prim() @@ -649,13 +975,18 @@ def tearDown(self): @classmethod def setUpClass(cls): cls.xs = tuple(x.astype(cls.dtype) for x in cls.xs) - cls._rtol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("rtol") - cls._atol = config.TOLERANCE.get(str( - cls.dtype)).get("first_order_grad").get("atol") + cls._rtol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("rtol") + ) + cls._atol = ( + config.TOLERANCE.get(str(cls.dtype)) + .get("first_order_grad") + .get("atol") + ) def test_grad(self): - def expected(): egrad = autograd.elementwise_grad grad_3 = egrad(egrad(egrad(self.fun_ag)))(self.xs) @@ -671,9 +1002,13 @@ def actual(): startup = paddle.static.Program() with paddle.static.program_guard(main, startup): feed, static_xs, static_v = utils.gen_static_data_and_feed( - self.xs, self.v, stop_gradient=False) - ys = self.fun_pd(*static_xs) if isinstance( - static_xs, typing.Sequence) else self.fun_pd(static_xs) + self.xs, self.v, stop_gradient=False + ) + ys = ( + self.fun_pd(*static_xs) + if isinstance(static_xs, typing.Sequence) + else self.fun_pd(static_xs) + ) grad1 = paddle_grad(ys, static_xs, static_v) grad2 = paddle_grad(grad1, static_xs, static_v) diff --git a/python/paddle/fluid/tests/unittests/autograd/test_primops.py b/python/paddle/fluid/tests/unittests/autograd/test_primops.py index 677c064bb786c..592229aa3dc22 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_primops.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_primops.py @@ -27,8 +27,14 @@ @utils.place(config.DEVICES) @utils.parameterize( - (utils.TEST_CASE_NAME, 'op', 'args', 'kwargs', 'expected_shape', - 'expected_dtype'), + ( + utils.TEST_CASE_NAME, + 'op', + 'args', + 'kwargs', + 'expected_shape', + 'expected_dtype', + ), ( ('add', primops.add, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'float64'), ('sub', primops.sub, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'float64'), @@ -43,73 +49,144 @@ ('erf', primops.erf, randn(2, 3), {}, (2, 3), 'float64'), ('abs', primops.abs, randn(2, 3), {}, (2, 3), 'float64'), ('log', primops.log, randn(2, 3), {}, (2, 3), 'float64'), - ('cast', primops.cast, randn(2, 3), { - 'dtype': paddle.int64 - }, (2, 3), 'int64'), - ('reshape', primops.reshape, randn(2, 3), { - 'shape': (3, 2) - }, (3, 2), 'float64'), - ('broadcast', primops.broadcast, randn(2), { - 'shape': (3, 2) - }, (3, 2), 'float64'), - ('transpose', primops.transpose, randn(2, 3), { - 'axis': (1, 0) - }, (3, 2), 'float64'), - ('concat_axis0', primops.concat, ((randn(2, 3), randn(2, 3)), ), { - 'axis': 0 - }, (4, 3), 'float64'), - ('concat_axis1', primops.concat, ((randn(2, 3), randn(2, 3)), ), { - 'axis': 1 - }, (2, 6), 'float64'), - ('reduce_axis1', primops.reduce_sum, randn(2, 3), { - 'axis': (1, ) - }, (2, ), 'float64'), - ('reduce_axis01', primops.reduce_sum, randn(2, 3), { - 'axis': (0, 1) - }, (1, ), 'float64'), - ('split', primops.split, randn(2, 3), { - 'num_or_sections': [1, 2], - 'axis': 1 - }, ((2, 1), (2, 2)), ('float64', 'float64')), - ('matmul', primops.matmul, (randn(2, 3), randn(3, 2)), {}, - (2, 2), 'float64'), - ('slice_select', primops.slice_select, randn(3, 2), { - 'axis': [0], - 'starts': [0], - 'ends': [2], - 'strides': [1] - }, (2, 2), 'float64'), - ('slice_assign', primops.slice_assign, (randn(2, 3), randn(2, 2)), { - 'axis': [1], - 'starts': [1], - 'ends': [3], - 'strides': [1] - }, (2, 3), 'float64'), - ('gather', primops.gather, (randn(3, 2), randint(0, 2, - (5, ), np.int32)), { - 'axis': 0 - }, (5, 2), 'float64'), - ('scatter_add', primops.scatter_add, - (randn(3, 2), randn(5, 2), randint(0, 2, (5, ), np.int32)), { - 'axis': 0 - }, (3, 2), 'float64'), - ('fill_const', primops.fill_const, (), { - 'value': 10, - 'shape': (3, 2), - 'dtype': paddle.float32 - }, (3, 2), 'float32'), + ( + 'cast', + primops.cast, + randn(2, 3), + {'dtype': paddle.int64}, + (2, 3), + 'int64', + ), + ( + 'reshape', + primops.reshape, + randn(2, 3), + {'shape': (3, 2)}, + (3, 2), + 'float64', + ), + ( + 'broadcast', + primops.broadcast, + randn(2), + {'shape': (3, 2)}, + (3, 2), + 'float64', + ), + ( + 'transpose', + primops.transpose, + randn(2, 3), + {'axis': (1, 0)}, + (3, 2), + 'float64', + ), + ( + 'concat_axis0', + primops.concat, + ((randn(2, 3), randn(2, 3)),), + {'axis': 0}, + (4, 3), + 'float64', + ), + ( + 'concat_axis1', + primops.concat, + ((randn(2, 3), randn(2, 3)),), + {'axis': 1}, + (2, 6), + 'float64', + ), + ( + 'reduce_axis1', + primops.reduce_sum, + randn(2, 3), + {'axis': (1,)}, + (2,), + 'float64', + ), + ( + 'reduce_axis01', + primops.reduce_sum, + randn(2, 3), + {'axis': (0, 1)}, + (1,), + 'float64', + ), + ( + 'split', + primops.split, + randn(2, 3), + {'num_or_sections': [1, 2], 'axis': 1}, + ((2, 1), (2, 2)), + ('float64', 'float64'), + ), + ( + 'matmul', + primops.matmul, + (randn(2, 3), randn(3, 2)), + {}, + (2, 2), + 'float64', + ), + ( + 'slice_select', + primops.slice_select, + randn(3, 2), + {'axis': [0], 'starts': [0], 'ends': [2], 'strides': [1]}, + (2, 2), + 'float64', + ), + ( + 'slice_assign', + primops.slice_assign, + (randn(2, 3), randn(2, 2)), + {'axis': [1], 'starts': [1], 'ends': [3], 'strides': [1]}, + (2, 3), + 'float64', + ), + ( + 'gather', + primops.gather, + (randn(3, 2), randint(0, 2, (5,), np.int32)), + {'axis': 0}, + (5, 2), + 'float64', + ), + ( + 'scatter_add', + primops.scatter_add, + (randn(3, 2), randn(5, 2), randint(0, 2, (5,), np.int32)), + {'axis': 0}, + (3, 2), + 'float64', + ), + ( + 'fill_const', + primops.fill_const, + (), + {'value': 10, 'shape': (3, 2), 'dtype': paddle.float32}, + (3, 2), + 'float32', + ), ('neg', primops.neg, randn(2, 3), {}, (2, 3), 'float64'), - ('select', primops.select, - (randn(2, 3) > 0, randn(2, 3), randn(2, 3)), {}, (2, 3), 'float64'), + ( + 'select', + primops.select, + (randn(2, 3) > 0, randn(2, 3), randn(2, 3)), + {}, + (2, 3), + 'float64', + ), ('eq', primops.eq, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'bool'), ('ne', primops.ne, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'bool'), ('gt', primops.gt, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'bool'), ('ge', primops.ge, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'bool'), ('pow', primops.pow, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'float64'), ('max', primops.max, (randn(2, 3), randn(2, 3)), {}, (2, 3), 'float64'), - )) + ), +) class TestPrimops(unittest.TestCase): - @classmethod def setUpClass(cls): paddle.enable_static() @@ -136,15 +213,18 @@ def arr2var(self, arr): """convert numpy ndarray to paddle Variable recursively.""" return [ paddle.static.data(f'x{uuid.uuid4()}', v.shape, v.dtype) - if isinstance(v, np.ndarray) else self.arr2var(v) for v in arr + if isinstance(v, np.ndarray) + else self.arr2var(v) + for v in arr ] def _as_tuple(self, input): if isinstance(input, (tuple, list)) and len(input) == 0: return input if not isinstance(input, (tuple, list)) or all( - isinstance(i, int) for i in input): - return (input, ) + isinstance(i, int) for i in input + ): + return (input,) return input diff --git a/python/paddle/fluid/tests/unittests/autograd/test_transform.py b/python/paddle/fluid/tests/unittests/autograd/test_transform.py index f29b0d9b7164c..3498f217bc667 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_transform.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_transform.py @@ -22,13 +22,13 @@ class TestAutoGradTransformForAdd(unittest.TestCase): - def setUp(self): self.main_program = paddle.static.Program() self.startup_program = paddle.static.Program() - with paddle.static.program_guard(self.main_program, - self.startup_program): + with paddle.static.program_guard( + self.main_program, self.startup_program + ): self.init_data() def init_data(self): @@ -36,13 +36,13 @@ def init_data(self): self.xs_shape_map = {0: (20, 40), 1: (20, 40)} # { output_index: output_shape } self.ys_shape_map = {0: (20, 40)} - X0 = paddle.static.data(name='X0', - shape=self.xs_shape_map[0], - dtype='float32') + X0 = paddle.static.data( + name='X0', shape=self.xs_shape_map[0], dtype='float32' + ) X0.stop_gradient = False - X1 = paddle.static.data(name='X1', - shape=self.xs_shape_map[1], - dtype='float32') + X1 = paddle.static.data( + name='X1', shape=self.xs_shape_map[1], dtype='float32' + ) X1.stop_gradient = False A = paddle.tanh(X0) @@ -94,28 +94,56 @@ def init_data(self): 'fill_constant_p', # transposed op 'mul_p', - 'mul_p' + 'mul_p', ] self.prim2orig_ops_with_blacklist = [ - 'tanh', 'tanh', 'add_p', 'fill_constant', 'fill_constant', - 'fill_constant', 'elementwise_mul', 'sub_p', 'fill_constant', - 'elementwise_mul', 'sub_p', 'fill_constant', 'elementwise_mul', - 'elementwise_mul', 'rsqrt', 'fill_constant', 'elementwise_div', - 'elementwise_div', 'elementwise_mul' + 'tanh', + 'tanh', + 'add_p', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'elementwise_mul', + 'sub_p', + 'fill_constant', + 'elementwise_mul', + 'sub_p', + 'fill_constant', + 'elementwise_mul', + 'elementwise_mul', + 'rsqrt', + 'fill_constant', + 'elementwise_div', + 'elementwise_div', + 'elementwise_mul', ] self.prim2orig_ops = [ - 'tanh', 'tanh', 'elementwise_add', 'fill_constant', 'fill_constant', - 'fill_constant', 'elementwise_mul', 'elementwise_sub', - 'fill_constant', 'elementwise_mul', 'elementwise_sub', - 'fill_constant', 'elementwise_mul', 'elementwise_mul', 'rsqrt', - 'fill_constant', 'elementwise_div', 'elementwise_div', - 'elementwise_mul' + 'tanh', + 'tanh', + 'elementwise_add', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'elementwise_mul', + 'elementwise_sub', + 'fill_constant', + 'elementwise_mul', + 'elementwise_sub', + 'fill_constant', + 'elementwise_mul', + 'elementwise_mul', + 'rsqrt', + 'fill_constant', + 'elementwise_div', + 'elementwise_div', + 'elementwise_mul', ] def test_run(self): # Must using with program_guard(), otherwise prim ops will append other block - with paddle.static.program_guard(self.main_program, - self.startup_program): + with paddle.static.program_guard( + self.main_program, self.startup_program + ): ad = Transform(self.main_program.block(0)) orig_ops = [op.type for op in self.main_program.block(0).ops] self.assertEqual(sorted(orig_ops), sorted(self.orig_ops)) @@ -150,11 +178,13 @@ def test_run(self): self.assertEqual(flatten_ys_bar[k].shape, v) # Test prim2orig with blacklist - prim2orig(block=self.main_program.block(0), - blacklist=['add_p', 'sub_p']) + prim2orig( + block=self.main_program.block(0), blacklist=['add_p', 'sub_p'] + ) prim2orig_ops = [op.type for op in self.main_program.block(0).ops] - self.assertEqual(sorted(prim2orig_ops), - sorted(self.prim2orig_ops_with_blacklist)) + self.assertEqual( + sorted(prim2orig_ops), sorted(self.prim2orig_ops_with_blacklist) + ) # Test prim2orig prim2orig(block=self.main_program.block(0)) @@ -163,19 +193,18 @@ def test_run(self): class TestAutoGradTransformForMatmul(TestAutoGradTransformForAdd): - def init_data(self): # { input_index: input_shape } self.xs_shape_map = {0: (100, 2), 1: (5, 2)} # { output_index: output_shape } self.ys_shape_map = {0: (100, 5)} - X0 = paddle.static.data('X0', - shape=self.xs_shape_map[0], - dtype='float32') + X0 = paddle.static.data( + 'X0', shape=self.xs_shape_map[0], dtype='float32' + ) X0.stop_gradient = False - X1 = paddle.static.data('X1', - shape=self.xs_shape_map[1], - dtype='float32') + X1 = paddle.static.data( + 'X1', shape=self.xs_shape_map[1], dtype='float32' + ) X1.stop_gradient = False A = paddle.reshape(X1, [2, 5]) @@ -189,8 +218,13 @@ def init_data(self): self.orig_ops = ['reshape2', 'scale', 'matmul_v2'] self.orig2prim_ops = [ - 'reshape_p', 'fill_constant_p', 'fill_constant_p', - 'fill_constant_p', 'mul_p', 'add_p', 'matmul_p' + 'reshape_p', + 'fill_constant_p', + 'fill_constant_p', + 'fill_constant_p', + 'mul_p', + 'add_p', + 'matmul_p', ] self.linearize_ops = self.orig2prim_ops + [ # call fill_const() in linearize() function @@ -204,7 +238,7 @@ def init_data(self): # 'add_p', 'matmul_p', 'matmul_p', - 'add_p' + 'add_p', ] self.transpose_ops = self.orig2prim_ops + [ # call fill_const() in transpose() function @@ -264,24 +298,23 @@ def init_data(self): class TestAutoGradTransformForIndexSelect(TestAutoGradTransformForAdd): - def init_data(self): # { input_index: input_shape } - self.xs_shape_map = {0: (7, 8, 9), 1: (8, 1), 2: (7, 8, 9), 3: (3, )} + self.xs_shape_map = {0: (7, 8, 9), 1: (8, 1), 2: (7, 8, 9), 3: (3,)} # { output_index: output_shape } self.ys_shape_map = {0: (3, 16, 9)} - X0 = paddle.static.data('X0', - shape=self.xs_shape_map[0], - dtype='float32') + X0 = paddle.static.data( + 'X0', shape=self.xs_shape_map[0], dtype='float32' + ) X0.stop_gradient = False - X1 = paddle.static.data('X1', - shape=self.xs_shape_map[1], - dtype='float32') + X1 = paddle.static.data( + 'X1', shape=self.xs_shape_map[1], dtype='float32' + ) X1.stop_gradient = False - X2 = paddle.static.data('X2', - shape=self.xs_shape_map[2], - dtype='float32') + X2 = paddle.static.data( + 'X2', shape=self.xs_shape_map[2], dtype='float32' + ) X2.stop_gradient = False X3 = paddle.static.data('X3', shape=self.xs_shape_map[3], dtype='int32') X3.stop_gradient = False @@ -297,12 +330,23 @@ def init_data(self): Y, ] self.orig_ops = [ - 'elementwise_add', 'p_norm', 'elementwise_sub', 'concat', - 'index_select' + 'elementwise_add', + 'p_norm', + 'elementwise_sub', + 'concat', + 'index_select', ] self.orig2prim_ops = [ - 'broadcast_p', 'add_p', 'reshape_p', 'mul_p', 'reduce_sum_p', - 'sqrt_p', 'broadcast_p', 'sub_p', 'concat_p', 'gather_p' + 'broadcast_p', + 'add_p', + 'reshape_p', + 'mul_p', + 'reduce_sum_p', + 'sqrt_p', + 'broadcast_p', + 'sub_p', + 'concat_p', + 'gather_p', ] self.linearize_ops = self.orig2prim_ops + [ # call fill_const() in linearize() function @@ -324,7 +368,7 @@ def init_data(self): 'broadcast_p', 'sub_p', 'concat_p', - 'gather_p' + 'gather_p', ] self.transpose_ops = self.orig2prim_ops + [ # call fill_const() in transpose() function @@ -357,26 +401,81 @@ def init_data(self): ] self.prim2orig_ops_with_blacklist = [ - 'expand_v2', 'add_p', 'reshape2', 'elementwise_mul', 'reduce_sum', - 'sqrt', 'expand_v2', 'sub_p', 'concat', 'gather', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'elementwise_mul', 'reduce_sum', 'reshape2', - 'reshape2', 'elementwise_mul', 'elementwise_mul', 'reshape2', - 'expand_v2', 'elementwise_div', 'reduce_sum', 'reshape2', - 'fill_constant', 'sub_p', 'split', 'fill_constant', 'fill_any_like', - 'add_p', 'scatter', 'elementwise_add', 'add_p' + 'expand_v2', + 'add_p', + 'reshape2', + 'elementwise_mul', + 'reduce_sum', + 'sqrt', + 'expand_v2', + 'sub_p', + 'concat', + 'gather', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'elementwise_mul', + 'reduce_sum', + 'reshape2', + 'reshape2', + 'elementwise_mul', + 'elementwise_mul', + 'reshape2', + 'expand_v2', + 'elementwise_div', + 'reduce_sum', + 'reshape2', + 'fill_constant', + 'sub_p', + 'split', + 'fill_constant', + 'fill_any_like', + 'add_p', + 'scatter', + 'elementwise_add', + 'add_p', ] self.prim2orig_ops = [ - 'expand_v2', 'elementwise_add', 'reshape2', 'elementwise_mul', - 'reduce_sum', 'sqrt', 'expand_v2', 'elementwise_sub', 'concat', - 'gather', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', - 'elementwise_mul', 'reduce_sum', 'reshape2', 'reshape2', - 'elementwise_mul', 'elementwise_mul', 'reshape2', 'expand_v2', - 'elementwise_div', 'reduce_sum', 'reshape2', 'fill_constant', - 'elementwise_sub', 'split', 'fill_constant', 'fill_any_like', - 'elementwise_add', 'scatter', 'elementwise_add', 'elementwise_add' + 'expand_v2', + 'elementwise_add', + 'reshape2', + 'elementwise_mul', + 'reduce_sum', + 'sqrt', + 'expand_v2', + 'elementwise_sub', + 'concat', + 'gather', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'elementwise_mul', + 'reduce_sum', + 'reshape2', + 'reshape2', + 'elementwise_mul', + 'elementwise_mul', + 'reshape2', + 'expand_v2', + 'elementwise_div', + 'reduce_sum', + 'reshape2', + 'fill_constant', + 'elementwise_sub', + 'split', + 'fill_constant', + 'fill_any_like', + 'elementwise_add', + 'scatter', + 'elementwise_add', + 'elementwise_add', ] diff --git a/python/paddle/fluid/tests/unittests/autograd/utils.py b/python/paddle/fluid/tests/unittests/autograd/utils.py index bbbce4eb22f5b..abcc71dd43f1f 100644 --- a/python/paddle/fluid/tests/unittests/autograd/utils.py +++ b/python/paddle/fluid/tests/unittests/autograd/utils.py @@ -32,20 +32,22 @@ def _product(t): def _get_item(t, idx): assert isinstance( - t, - paddle.fluid.framework.Variable), "The first argument t must be Tensor." - assert isinstance(idx, - int), "The second argument idx must be an int number." + t, paddle.fluid.framework.Variable + ), "The first argument t must be Tensor." + assert isinstance( + idx, int + ), "The second argument idx must be an int number." flat_t = paddle.reshape(t, [-1]) return flat_t.__getitem__(idx) def _set_item(t, idx, value): assert isinstance( - t, - paddle.fluid.framework.Variable), "The first argument t must be Tensor." - assert isinstance(idx, - int), "The second argument idx must be an int number." + t, paddle.fluid.framework.Variable + ), "The first argument t must be Tensor." + assert isinstance( + idx, int + ), "The second argument idx must be an int number." flat_t = paddle.reshape(t, [-1]) flat_t.__setitem__(idx, value) return paddle.reshape(flat_t, t.shape) @@ -60,8 +62,9 @@ def _compute_numerical_jacobian(func, xs, delta, np_dtype): for i in range(fout_size): jac_i = list([] for _ in range(fin_size)) for j in range(fin_size): - jac_i[j] = np.zeros((_product(ys[i].shape), _product(xs[j].shape)), - dtype=np_dtype) + jac_i[j] = np.zeros( + (_product(ys[i].shape), _product(xs[j].shape)), dtype=np_dtype + ) jacobian[i] = jac_i for j in range(fin_size): @@ -81,7 +84,7 @@ def _compute_numerical_jacobian(func, xs, delta, np_dtype): for p in range(_product(ys[i].shape)): y_pos = _get_item(ys_pos[i], p) y_neg = _get_item(ys_neg[i], p) - jacobian[i][j][p][q] = (y_pos - y_neg) / delta / 2. + jacobian[i][j][p][q] = (y_pos - y_neg) / delta / 2.0 return jacobian @@ -94,7 +97,8 @@ def _compute_numerical_hessian(func, xs, delta, np_dtype): hessian_i = list([] for _ in range(fin_size)) for j in range(fin_size): hessian_i[j] = np.zeros( - (_product(xs[i].shape), _product(xs[j].shape)), dtype=np_dtype) + (_product(xs[i].shape), _product(xs[j].shape)), dtype=np_dtype + ) hessian[i] = hessian_i for i in range(fin_size): @@ -105,15 +109,19 @@ def _compute_numerical_hessian(func, xs, delta, np_dtype): x_pos = orig + delta xs[j] = _set_item(xs[j], q, x_pos) jacobian_pos = _compute_numerical_jacobian( - func, xs, delta, np_dtype) + func, xs, delta, np_dtype + ) x_neg = orig - delta xs[j] = _set_item(xs[j], q, x_neg) jacobian_neg = _compute_numerical_jacobian( - func, xs, delta, np_dtype) + func, xs, delta, np_dtype + ) xs[j] = _set_item(xs[j], q, orig) hessian[i][j][p][q] = ( - jacobian_pos[0][i][0][p] - - jacobian_neg[0][i][0][p]) / delta / 2. + (jacobian_pos[0][i][0][p] - jacobian_neg[0][i][0][p]) + / delta + / 2.0 + ) return hessian @@ -125,11 +133,9 @@ def concat_to_matrix(xs, is_batched=False): return np.concatenate(rows, 1) if is_batched else np.concatenate(rows, 0) -def _compute_numerical_batch_jacobian(func, - xs, - delta, - np_dtype, - merge_batch=True): +def _compute_numerical_batch_jacobian( + func, xs, delta, np_dtype, merge_batch=True +): no_batch_jacobian = _compute_numerical_jacobian(func, xs, delta, np_dtype) xs = list(as_tensors(xs)) ys = list(as_tensors(func(*xs))) @@ -192,7 +198,8 @@ def _compute_numerical_batch_hessian(func, xs, delta, np_dtype): mid = len(hessian_res) // 2 for i in range(mid): hessian_result.append( - np.stack((hessian_res[i], hessian_res[mid + i]), axis=0)) + np.stack((hessian_res[i], hessian_res[mid + i]), axis=0) + ) return hessian_result @@ -205,8 +212,9 @@ def _compute_numerical_vjp(func, xs, v, delta, np_dtype): vjp = [np.zeros((_product(x.shape)), dtype=np_dtype) for x in xs] for j in range(len(xs)): for q in range(_product(xs[j].shape)): - vjp[j][q] = np.sum(jacobian[:, j, :, q].reshape(flat_v.shape) * - flat_v) + vjp[j][q] = np.sum( + jacobian[:, j, :, q].reshape(flat_v.shape) * flat_v + ) vjp = [vjp[j].reshape(xs[j].shape) for j in range(len(xs))] return vjp @@ -218,8 +226,9 @@ def _compute_numerical_vhp(func, xs, v, delta, np_dtype): vhp = [np.zeros((_product(x.shape)), dtype=np_dtype) for x in xs] for j in range(len(xs)): for q in range(_product(xs[j].shape)): - vhp[j][q] = np.sum(hessian[:, j, :, q].reshape(flat_v.shape) * - flat_v) + vhp[j][q] = np.sum( + hessian[:, j, :, q].reshape(flat_v.shape) * flat_v + ) vhp = [vhp[j].reshape(xs[j].shape) for j in range(len(xs))] return vhp @@ -256,7 +265,6 @@ def unuse(x, y): def nested(x): - def inner(y): return x * y @@ -286,8 +294,7 @@ def place(devices, key='place'): def decorate(cls): module = sys.modules[cls.__module__].__dict__ raw_classes = { - k: v - for k, v in module.items() if k.startswith(cls.__name__) + k: v for k, v in module.items() if k.startswith(cls.__name__) } for raw_name, raw_cls in raw_classes.items(): @@ -295,7 +302,7 @@ def decorate(cls): test_cls = dict(raw_cls.__dict__) test_cls.update({key: d}) new_name = raw_name + '.' + d.__class__.__name__ - module[new_name] = type(new_name, (raw_cls, ), test_cls) + module[new_name] = type(new_name, (raw_cls,), test_cls) del module[raw_name] return cls @@ -324,10 +331,13 @@ def decorate(cls): } test_cls.update(values) name = cls.__name__ + str(i) - name = name + '.' + \ - values.get('suffix') if values.get('suffix') else name + name = ( + name + '.' + values.get('suffix') + if values.get('suffix') + else name + ) - test_cls_module[name] = type(name, (cls, ), test_cls) + test_cls_module[name] = type(name, (cls,), test_cls) for m in list(cls.__dict__): if m.startswith("test"): @@ -375,8 +385,12 @@ def concat_row(xs): else: return np.concatenate(xs, axis=1) - supported_format = (MatrixFormat.NBM, MatrixFormat.BNM, MatrixFormat.NMB, - MatrixFormat.NM) + supported_format = ( + MatrixFormat.NBM, + MatrixFormat.BNM, + MatrixFormat.NMB, + MatrixFormat.NM, + ) if src_format not in supported_format: raise ValueError( f"Supported Jacobian format is {supported_format}, but got {src_format}" diff --git a/python/paddle/fluid/tests/unittests/benchmark.py b/python/paddle/fluid/tests/unittests/benchmark.py index 3a27fca61efee..d6fa01e398cbd 100644 --- a/python/paddle/fluid/tests/unittests/benchmark.py +++ b/python/paddle/fluid/tests/unittests/benchmark.py @@ -20,7 +20,6 @@ class BenchmarkSuite(OpTest): - def timeit_function(self, callback, iters, *args, **kwargs): assert iters != 0, "Iters should >= 1" start = time.time() @@ -30,20 +29,23 @@ def timeit_function(self, callback, iters, *args, **kwargs): return elapse / iters def _assert_cpu_gpu_same(self, cpu_outs, gpu_outs, fetch_list, atol): - for item_cpu_out, item_gpu_out, variable in zip(cpu_outs, gpu_outs, - fetch_list): + for item_cpu_out, item_gpu_out, variable in zip( + cpu_outs, gpu_outs, fetch_list + ): # the cpu version is baseline, expect gpu version keep same with cpu version. expect = item_cpu_out expect_t = np.array(item_cpu_out) actual = item_gpu_out actual_t = np.array(item_gpu_out) var_name = variable if isinstance(variable, str) else variable.name - np.testing.assert_allclose(actual_t, - expect_t, - rtol=1e-05, - atol=atol) - self.assertListEqual(actual.lod(), expect.lod(), - "Output (" + var_name + ") has different lod") + np.testing.assert_allclose( + actual_t, expect_t, rtol=1e-05, atol=atol + ) + self.assertListEqual( + actual.lod(), + expect.lod(), + "Output (" + var_name + ") has different lod", + ) def _get_input_names(self): inputs = [] @@ -83,18 +85,23 @@ def timeit_output(self, iters=100): for place in places: elapses.append(self.timeit_output_with_place(place, iters)) for place, elapse in zip(places, elapses): - print("One pass of ({2}_op) at {0} cost {1}".format( - str(place), elapse, self.op_type)) + print( + "One pass of ({2}_op) at {0} cost {1}".format( + str(place), elapse, self.op_type + ) + ) def timeit_grad_with_place(self, place, iters=100): inputs_to_check = self._get_input_names() output_names = self._get_output_names() - return self.timeit_function(self._get_gradient, - iters, - inputs_to_check, - place, - output_names, - no_grad_set=None) + return self.timeit_function( + self._get_gradient, + iters, + inputs_to_check, + place, + output_names, + no_grad_set=None, + ) def timeit_grad(self, iters=100): places = self._get_places() @@ -102,5 +109,8 @@ def timeit_grad(self, iters=100): for place in places: elapses.append(self.timeit_grad_with_place(place, iters)) for place, elapse in zip(places, elapses): - print("One pass of ({2}_grad_op) at {0} cost {1}".format( - str(place), elapse, self.op_type)) + print( + "One pass of ({2}_grad_op) at {0} cost {1}".format( + str(place), elapse, self.op_type + ) + ) diff --git a/python/paddle/fluid/tests/unittests/benchmark_sum_op.py b/python/paddle/fluid/tests/unittests/benchmark_sum_op.py index d3d909e17e49d..e8cb02f16cb1a 100644 --- a/python/paddle/fluid/tests/unittests/benchmark_sum_op.py +++ b/python/paddle/fluid/tests/unittests/benchmark_sum_op.py @@ -21,7 +21,6 @@ class TestSumOp(BenchmarkSuite): - def setUp(self): self.op_type = "sum" self.customize_testcase() diff --git a/python/paddle/fluid/tests/unittests/c_embedding_op_base.py b/python/paddle/fluid/tests/unittests/c_embedding_op_base.py index ef3c942d2b096..e9b1ca4d8aa80 100644 --- a/python/paddle/fluid/tests/unittests/c_embedding_op_base.py +++ b/python/paddle/fluid/tests/unittests/c_embedding_op_base.py @@ -32,7 +32,6 @@ def get_c_embedding(start, end, table, ids): class TestCEmbeddingCPU(OpTest): - def setUp(self): self.init_dtype() self.initcase() @@ -46,8 +45,9 @@ def setUp(self): def initcase(self): self.op_type = "c_embedding" table = np.random.random((17, 64)).astype(self.dtype) - ids = np.random.randint(low=0, high=17 * 2, - size=(2, 4)).astype(self.ids_dtype) + ids = np.random.randint(low=0, high=17 * 2, size=(2, 4)).astype( + self.ids_dtype + ) self.start_index = 10 self.end_index = self.start_index + 17 @@ -72,7 +72,6 @@ def init_dtype(self): class TestCEmbeddingOpBase(TestCEmbeddingCPU): - def setUp(self): self.init_dtype() self.initcase() @@ -106,7 +105,6 @@ def init_dtype(self): class TestCEmbeddingOpFP32(TestCEmbeddingOpBase): - def setUp(self): self.init_dtype() self.initcase() @@ -114,8 +112,9 @@ def setUp(self): def initcase(self): self.op_type = "c_embedding" table = np.random.random((17, 64)).astype(self.dtype) - ids = np.random.randint(low=0, high=17 * 2, - size=(2, 4)).astype(self.ids_dtype) + ids = np.random.randint(low=0, high=17 * 2, size=(2, 4)).astype( + self.ids_dtype + ) self.start_index = 10 ids[0][1] = 12 ids[0][2] = 12 diff --git a/python/paddle/fluid/tests/unittests/check_nan_inf_base.py b/python/paddle/fluid/tests/unittests/check_nan_inf_base.py index 1c4bfd387c42c..e1319378728ad 100644 --- a/python/paddle/fluid/tests/unittests/check_nan_inf_base.py +++ b/python/paddle/fluid/tests/unittests/check_nan_inf_base.py @@ -30,8 +30,9 @@ def generator(): batch_size = 5 for i in range(5): - curr_train_x = np.random.randint(batch_size, - size=(batch_size, 3)).astype("float32") + curr_train_x = np.random.randint( + batch_size, size=(batch_size, 3) + ).astype("float32") if i >= 2: curr_train_x[0, :] = np.nan curr_train_x[-1, :] = np.inf @@ -62,7 +63,8 @@ def net(): hidden = fluid.layers.fc(input=hidden, size=3, act=None) cost, y_predict = fluid.layers.softmax_with_cross_entropy( - hidden, y, return_softmax=True) + hidden, y, return_softmax=True + ) acc_top1 = fluid.layers.accuracy(input=y_predict, label=y, k=1) avg_cost = paddle.mean(cost) @@ -88,14 +90,15 @@ def check(use_cuda): for train_data, y_label in generator(): outs = exe.run( main, - feed={ - 'x': train_data, - 'y': y_label - }, - fetch_list=[y_predict.name, avg_cost.name, acc_top1.name]) + feed={'x': train_data, 'y': y_label}, + fetch_list=[y_predict.name, avg_cost.name, acc_top1.name], + ) step += 1 - print('iter={:.0f},cost={},acc1={}'.format( - step, outs[1][0], outs[2][0])) + print( + 'iter={:.0f},cost={},acc1={}'.format( + step, outs[1][0], outs[2][0] + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py b/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py index 9622ee4e9a92e..233cfee3087c5 100644 --- a/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py +++ b/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py @@ -28,8 +28,9 @@ def generator(): batch_size = 5 for i in range(5): - curr_train_x = np.random.randint(batch_size, - size=(batch_size, 3)).astype("float32") + curr_train_x = np.random.randint( + batch_size, size=(batch_size, 3) + ).astype("float32") if i >= 2: curr_train_x[0, :] = np.nan curr_train_x[-1, :] = np.inf @@ -42,7 +43,6 @@ def generator(): class TestLayer(nn.Layer): - def __init__(self): super(TestLayer, self).__init__() self.linear1 = nn.Linear(3, 400) @@ -82,8 +82,11 @@ def check(use_cuda): acc_top1 = paddle.metric.accuracy(input=y_pred, label=y, k=1) - print('iter={:.0f}, cost={}, acc1={}'.format(step, avg_cost.numpy(), - acc_top1.numpy())) + print( + 'iter={:.0f}, cost={}, acc1={}'.format( + step, avg_cost.numpy(), acc_top1.numpy() + ) + ) sgd.step() sgd.clear_grad() diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allgather_api.py b/python/paddle/fluid/tests/unittests/collective/collective_allgather_api.py index a69e6c59e0edc..94be00887a2e9 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allgather_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allgather_api.py @@ -24,7 +24,6 @@ class TestCollectiveAllgatherAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -47,29 +46,30 @@ def run_trainer(self, args): if args['backend'] == 'nccl': device_id = int(os.getenv("FLAGS_selected_gpus", "0")) place = fluid.CUDAPlace( - device_id) #if args.use_gpu else fluid.CPUPlace() + device_id + ) # if args.use_gpu else fluid.CPUPlace() elif args['backend'] == 'bkcl': device_id = int(os.getenv("FLAGS_selected_xpus", "0")) place = fluid.XPUPlace(device_id) else: place = fluid.CPUPlace() - indata = test_base.create_test_data(shape=(10, 1000), - dtype=args["dtype"], - seed=os.getpid()) - assert args[ - 'static_mode'] == 1, "collective_allgather_api only support static mode" - result = self.get_model(train_prog, - startup_prog, - rank, - dtype=args["dtype"]) + indata = test_base.create_test_data( + shape=(10, 1000), dtype=args["dtype"], seed=os.getpid() + ) + assert ( + args['static_mode'] == 1 + ), "collective_allgather_api only support static mode" + result = self.get_model( + train_prog, startup_prog, rank, dtype=args["dtype"] + ) exe = fluid.Executor(place) exe.run(startup_prog) fetch_list = [] for elem in result: fetch_list.append(elem.name) - out = exe.run(train_prog, - feed={'tindata': indata}, - fetch_list=fetch_list) + out = exe.run( + train_prog, feed={'tindata': indata}, fetch_list=fetch_list + ) sys.stdout.buffer.write(pickle.dumps(out)) diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allgather_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_allgather_api_dygraph.py index 4d5f82e288220..4477b663a5832 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allgather_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allgather_api_dygraph.py @@ -19,7 +19,6 @@ class TestCollectiveAllgatherAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allgather_object_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_allgather_object_api_dygraph.py index 76610a8404135..03771e30d7e67 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allgather_object_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allgather_object_api_dygraph.py @@ -18,7 +18,6 @@ class TestCollectiveAllgatherObjectAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api.py b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api.py index 9c86afc5f4b0a..a28f8b175da21 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api.py @@ -21,15 +21,14 @@ class TestCollectiveAllreduceAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) paddle.distributed.all_reduce(tindata) return [tindata] diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api_dygraph.py index 9bdbaa18177e1..30cb1e5a377fa 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_api_dygraph.py @@ -19,7 +19,6 @@ class TestCollectiveAllreduceAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_new_group_api.py b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_new_group_api.py index 5eb983c890583..b3258e858a20b 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_new_group_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_new_group_api.py @@ -21,15 +21,14 @@ class TestCollectiveAllreduceNewGroupAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) gp = paddle.distributed.new_group([0, 1]) paddle.distributed.all_reduce(tindata, group=gp, sync_op=True) return [tindata] diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op.py b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op.py index a88b73ce04aac..69072d40aa892 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op.py @@ -22,30 +22,34 @@ class TestCollectiveAllreduce(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program): ring_id = 0 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofallreduce", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_allreduce_sum", - inputs={'X': tindata}, - attrs={'ring_id': ring_id}, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_allreduce_sum", + inputs={'X': tindata}, + attrs={'ring_id': ring_id}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op_wait.py b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op_wait.py index 10b867df69ac9..c700d49415bf3 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op_wait.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_allreduce_op_wait.py @@ -22,22 +22,22 @@ class TestCollectiveAllreduce(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program): ring_id = 0 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofallreduce", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) # tout = tin + tin - tin = tin if True: @@ -58,21 +58,27 @@ def get_model(self, main_prog, startup_program): outputs={'Out': toutdata}, ) - main_prog.global_block().append_op(type='c_wait_compute', - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + main_prog.global_block().append_op( + type='c_wait_compute', + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) - main_prog.global_block().append_op(type="c_allreduce_sum", - inputs={'X': toutdata}, - attrs={'ring_id': ring_id}, - outputs={'Out': toutdata}, - attr={'use_calc_stream': False}) + main_prog.global_block().append_op( + type="c_allreduce_sum", + inputs={'X': toutdata}, + attrs={'ring_id': ring_id}, + outputs={'Out': toutdata}, + attr={'use_calc_stream': False}, + ) - main_prog.global_block().append_op(type="c_wait_comm", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + main_prog.global_block().append_op( + type="c_wait_comm", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) # tout = tin + tout - tin = tout if True: diff --git a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api.py b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api.py index 57fbc69429a9a..e3f23f0cd0118 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api.py @@ -21,15 +21,14 @@ class TestCollectiveAllToAllAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) tindata = paddle.split(tindata, 2, axis=0) tout_data = [] paddle.distributed.alltoall(tindata, tout_data) diff --git a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api_dygraph.py index eb19cadb11426..560fa8162c511 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_api_dygraph.py @@ -19,7 +19,6 @@ class TestCollectiveAllToAllAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single.py b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single.py index e0e0fc74d3767..0c9a1253393df 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single.py @@ -20,13 +20,14 @@ class TestCollectiveAllToAllSingle(unittest.TestCase): - def setUp(self): - assert not paddle.distributed.is_initialized(), \ - "The distributed environment has not been initialized." + assert ( + not paddle.distributed.is_initialized() + ), "The distributed environment has not been initialized." dist.init_parallel_env() - assert paddle.distributed.is_initialized(), \ - "The distributed environment has been initialized." + assert ( + paddle.distributed.is_initialized() + ), "The distributed environment has been initialized." paddle.fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -38,7 +39,8 @@ def test_collective_alltoall_single(self): input = paddle.ones([size, size], dtype='int64') * rank output = paddle.empty([size, size], dtype='int64') expected_output = paddle.concat( - [paddle.ones([1, size], dtype='int64') * i for i in range(size)]) + [paddle.ones([1, size], dtype='int64') * i for i in range(size)] + ) group = dist.new_group([0, 1]) dist.alltoall_single(input, output, group=group) @@ -52,18 +54,22 @@ def test_collective_alltoall_single(self): input = paddle.ones([sum(in_split_sizes), size], dtype='float32') * rank output = paddle.empty([(rank + 1) * size, size], dtype='float32') - expected_output = paddle.concat([ - paddle.ones([rank + 1, size], dtype='float32') * i - for i in range(size) - ]) + expected_output = paddle.concat( + [ + paddle.ones([rank + 1, size], dtype='float32') * i + for i in range(size) + ] + ) group = dist.new_group([0, 1]) - task = dist.alltoall_single(input, - output, - in_split_sizes, - out_split_sizes, - sync_op=False, - group=group) + task = dist.alltoall_single( + input, + output, + in_split_sizes, + out_split_sizes, + sync_op=False, + group=group, + ) task.wait() np.testing.assert_allclose(output.numpy(), expected_output.numpy()) @@ -71,8 +77,9 @@ def test_collective_alltoall_single(self): def tearDown(self): dist.destroy_process_group() - assert not paddle.distributed.is_initialized(), \ - "The distributed environment has been deinitialized." + assert ( + not paddle.distributed.is_initialized() + ), "The distributed environment has been deinitialized." if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single_api_dygraph.py index f66b3a74bfd21..0b5bf5ddde59d 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_alltoall_single_api_dygraph.py @@ -19,7 +19,6 @@ class TestCollectiveAllToAllSingleAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_barrier_api.py b/python/paddle/fluid/tests/unittests/collective/collective_barrier_api.py index 43fcca0155b3f..95054f678a398 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_barrier_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_barrier_api.py @@ -20,7 +20,6 @@ class TestCollectiveBarrierAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_batch_isend_irecv.py b/python/paddle/fluid/tests/unittests/collective/collective_batch_isend_irecv.py index 2226d302291ca..6757e61fd0712 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_batch_isend_irecv.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_batch_isend_irecv.py @@ -20,7 +20,6 @@ class TestCollectiveBatchIsendIrecv(unittest.TestCase): - def setUp(self): dist.init_parallel_env() paddle.fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -33,8 +32,9 @@ def test_collective_batch_isend_irecv(self): # paddle.tensor([1, 2]) # Rank-1 recv_t = paddle.empty(shape=[2], dtype=send_t.dtype) send_op = dist.P2POp(dist.isend, send_t, (rank + 1) % world_size) - recv_op = dist.P2POp(dist.irecv, recv_t, - (rank - 1 + world_size) % world_size) + recv_op = dist.P2POp( + dist.irecv, recv_t, (rank - 1 + world_size) % world_size + ) tasks = dist.batch_isend_irecv([send_op, recv_op]) for task in tasks: diff --git a/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api.py b/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api.py index 0a14a53660955..d7f07e440d8f2 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api.py @@ -21,15 +21,14 @@ class TestCollectiveBroadcastAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) paddle.distributed.broadcast(tindata, src=1) return [tindata] diff --git a/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api_dygraph.py index 9004d27d56183..82bc998a24967 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_broadcast_api_dygraph.py @@ -19,7 +19,6 @@ class TestCollectiveBroadcastAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_broadcast_op.py b/python/paddle/fluid/tests/unittests/collective/collective_broadcast_op.py index ff99792c9428a..902e49b2648fd 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_broadcast_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_broadcast_op.py @@ -22,7 +22,6 @@ class TestCollectiveBroadcast(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,26 +29,28 @@ def get_model(self, main_prog, startup_program): ring_id = 0 rootid = 1 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofbroadcast", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_broadcast", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'root': rootid - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_broadcast", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'root': rootid}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_concat_op.py b/python/paddle/fluid/tests/unittests/collective/collective_concat_op.py index 34d49aeb284d3..533357fc44d7f 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_concat_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_concat_op.py @@ -22,7 +22,6 @@ class TestCollectiveConcat(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,23 +29,22 @@ def get_model(self, main_prog, startup_program): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofconcat", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_concat", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'rank': self.rank, - 'nranks': nranks - }, - outputs={'Out': toutdata}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_concat", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'rank': self.rank, 'nranks': nranks}, + outputs={'Out': toutdata}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_global_gather.py b/python/paddle/fluid/tests/unittests/collective/collective_global_gather.py index 4403e06681b64..6aa268b50a350 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_global_gather.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_global_gather.py @@ -26,7 +26,6 @@ class TestCollectiveGlobalGatherAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -38,19 +37,19 @@ def get_model(self, main_prog, startup_program, rank, indata=None): n_expert = 2 world_size = 2 tot_expert = n_expert * world_size - local_input_buf = paddle.static.data(name="local_input_buf", - shape=[-1, in_feat], - dtype="float32") - local_expert_count = paddle.static.data(name="local_expert_count", - shape=[tot_expert], - dtype="int64") - global_expert_count = paddle.static.data(name="global_expert_count", - shape=[tot_expert], - dtype="int64") + local_input_buf = paddle.static.data( + name="local_input_buf", shape=[-1, in_feat], dtype="float32" + ) + local_expert_count = paddle.static.data( + name="local_expert_count", shape=[tot_expert], dtype="int64" + ) + global_expert_count = paddle.static.data( + name="global_expert_count", shape=[tot_expert], dtype="int64" + ) - output = moe_utils.global_gather(local_input_buf, - local_expert_count, - global_expert_count) + output = moe_utils.global_gather( + local_input_buf, local_expert_count, global_expert_count + ) return [output] @@ -65,7 +64,8 @@ def run_trainer(self, args): if args['backend'] == 'nccl': device_id = int(os.getenv("FLAGS_selected_gpus", "0")) place = fluid.CUDAPlace( - device_id) #if args.use_gpu else fluid.CPUPlace() + device_id + ) # if args.use_gpu else fluid.CPUPlace() elif args['backend'] == 'bkcl': device_id = int(os.getenv("FLAGS_selected_xpus", "0")) place = fluid.XPUPlace(device_id) @@ -81,19 +81,22 @@ def run_trainer(self, args): # Call paddle.distributed.alltoall() under legacy dygraph _enable_legacy_dygraph() np.random.seed(os.getpid()) - local_expert_count = np.random.randint(1, 4, - size=tot_expert).astype("int64") + local_expert_count = np.random.randint(1, 4, size=tot_expert).astype( + "int64" + ) local_expert_count = paddle.to_tensor(local_expert_count) global_expert_count = [] - paddle.distributed.alltoall(paddle.split(local_expert_count, 2, axis=0), - global_expert_count) + paddle.distributed.alltoall( + paddle.split(local_expert_count, 2, axis=0), global_expert_count + ) global_expert_count = paddle.concat(global_expert_count, axis=0) global_expert_count = global_expert_count.numpy() local_expert_count = local_expert_count.numpy() fwd_expert_count = sum(global_expert_count) np.random.seed(os.getpid()) - local_input_buf = np.random.rand(fwd_expert_count, - in_feat).astype("float32") + local_input_buf = np.random.rand(fwd_expert_count, in_feat).astype( + "float32" + ) paddle.enable_static() if args['static_mode']: @@ -103,13 +106,15 @@ def run_trainer(self, args): fetch_list = [] for elem in result: fetch_list.append(elem.name) - out = exe.run(train_prog, - feed={ - 'local_expert_count': local_expert_count, - 'global_expert_count': global_expert_count, - 'local_input_buf': local_input_buf - }, - fetch_list=fetch_list) + out = exe.run( + train_prog, + feed={ + 'local_expert_count': local_expert_count, + 'global_expert_count': global_expert_count, + 'local_input_buf': local_input_buf, + }, + fetch_list=fetch_list, + ) sys.stdout.buffer.write(pickle.dumps(out)) diff --git a/python/paddle/fluid/tests/unittests/collective/collective_global_gather_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_global_gather_dygraph.py index acba43268f2d4..fb8d831f0450d 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_global_gather_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_global_gather_dygraph.py @@ -21,7 +21,6 @@ class TestCollectiveGlobalGatherAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -34,22 +33,24 @@ def get_model(self, main_prog, startup_program, rank, indata=None): world_size = 2 tot_expert = n_expert * world_size local_expert_count = np.random.randint( - 1, 4, size=tot_expert).astype("int") + 1, 4, size=tot_expert + ).astype("int") local_expert_count = paddle.to_tensor(local_expert_count) global_expert_count = [] paddle.distributed.alltoall( - paddle.split(local_expert_count, 2, axis=0), - global_expert_count) + paddle.split(local_expert_count, 2, axis=0), global_expert_count + ) global_expert_count = paddle.concat(global_expert_count, axis=0) fwd_expert_count = sum(global_expert_count) np.random.seed(seed) - local_input_buf = np.random.rand(fwd_expert_count, - in_feat).astype("float32") + local_input_buf = np.random.rand(fwd_expert_count, in_feat).astype( + "float32" + ) local_input_buf = paddle.to_tensor(local_input_buf) local_input_buf.stop_gradient = False - output = moe_utils.global_gather(local_input_buf, - local_expert_count, - global_expert_count) + output = moe_utils.global_gather( + local_input_buf, local_expert_count, global_expert_count + ) output.stop_gradient = False c = output * output c.stop_gradient = False diff --git a/python/paddle/fluid/tests/unittests/collective/collective_global_scatter.py b/python/paddle/fluid/tests/unittests/collective/collective_global_scatter.py index a7103d3c64186..7fb0dc6ff717a 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_global_scatter.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_global_scatter.py @@ -25,7 +25,6 @@ class TestCollectiveGlobalScatterAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -37,20 +36,20 @@ def get_model(self, main_prog, startup_program, rank, indata=None): n_expert = 2 world_size = 2 tot_expert = n_expert * world_size - local_input_buf = paddle.static.data(name="local_input_buf", - shape=[-1, in_feat], - dtype="float32") - local_expert_count = paddle.static.data(name="local_expert_count", - shape=[tot_expert], - dtype="int64") + local_input_buf = paddle.static.data( + name="local_input_buf", shape=[-1, in_feat], dtype="float32" + ) + local_expert_count = paddle.static.data( + name="local_expert_count", shape=[tot_expert], dtype="int64" + ) global_expert_count = [] paddle.distributed.alltoall( - paddle.split(local_expert_count, 2, axis=0), - global_expert_count) + paddle.split(local_expert_count, 2, axis=0), global_expert_count + ) global_expert_count = paddle.concat(global_expert_count, axis=0) - output = moe_utils.global_scatter(local_input_buf, - local_expert_count, - global_expert_count) + output = moe_utils.global_scatter( + local_input_buf, local_expert_count, global_expert_count + ) return [output] def run_trainer(self, args): @@ -64,7 +63,8 @@ def run_trainer(self, args): if args['backend'] == 'nccl': device_id = int(os.getenv("FLAGS_selected_gpus", "0")) place = fluid.CUDAPlace( - device_id) #if args.use_gpu else fluid.CPUPlace() + device_id + ) # if args.use_gpu else fluid.CPUPlace() elif args['backend'] == 'bkcl': device_id = int(os.getenv("FLAGS_selected_xpus", "0")) place = fluid.XPUPlace(device_id) @@ -75,11 +75,13 @@ def run_trainer(self, args): n_expert = 2 world_size = 2 tot_expert = n_expert * world_size - local_expert_count = np.random.randint(1, 4, - size=tot_expert).astype("int64") + local_expert_count = np.random.randint(1, 4, size=tot_expert).astype( + "int64" + ) fwd_expert_count = sum(local_expert_count) - local_input_buf = np.random.rand(fwd_expert_count, - in_feat).astype("float32") + local_input_buf = np.random.rand(fwd_expert_count, in_feat).astype( + "float32" + ) if args['static_mode']: result = self.get_model(train_prog, startup_prog, rank) exe = fluid.Executor(place) @@ -87,12 +89,14 @@ def run_trainer(self, args): fetch_list = [] for elem in result: fetch_list.append(elem.name) - out = exe.run(train_prog, - feed={ - 'local_expert_count': local_expert_count, - 'local_input_buf': local_input_buf - }, - fetch_list=fetch_list) + out = exe.run( + train_prog, + feed={ + 'local_expert_count': local_expert_count, + 'local_input_buf': local_input_buf, + }, + fetch_list=fetch_list, + ) sys.stdout.buffer.write(pickle.dumps(out)) diff --git a/python/paddle/fluid/tests/unittests/collective/collective_global_scatter_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_global_scatter_dygraph.py index b90905d5d397e..324f079ee8ccc 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_global_scatter_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_global_scatter_dygraph.py @@ -21,7 +21,6 @@ class TestCollectiveGlobalScatterAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -34,21 +33,23 @@ def get_model(self, main_prog, startup_program, rank, indata=None): world_size = 2 tot_expert = n_expert * world_size local_expert_count = np.random.randint( - 1, 4, size=tot_expert).astype("int") + 1, 4, size=tot_expert + ).astype("int") fwd_expert_count = sum(local_expert_count) - local_input_buf = np.random.rand(fwd_expert_count, - in_feat).astype("float32") + local_input_buf = np.random.rand(fwd_expert_count, in_feat).astype( + "float32" + ) local_expert_count = paddle.to_tensor(local_expert_count) local_input_buf = paddle.to_tensor(local_input_buf) global_expert_count = [] paddle.distributed.alltoall( - paddle.split(local_expert_count, 2, axis=0), - global_expert_count) + paddle.split(local_expert_count, 2, axis=0), global_expert_count + ) global_expert_count = paddle.concat(global_expert_count, axis=0) local_input_buf.stop_gradient = False - output = moe_utils.global_scatter(local_input_buf, - local_expert_count, - global_expert_count) + output = moe_utils.global_scatter( + local_input_buf, local_expert_count, global_expert_count + ) output.stop_gradient = False c = output * output c.backward() diff --git a/python/paddle/fluid/tests/unittests/collective/collective_identity_op.py b/python/paddle/fluid/tests/unittests/collective/collective_identity_op.py index 0486ea616421a..8fc2449f4f232 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_identity_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_identity_op.py @@ -22,7 +22,6 @@ class TestCollectiveIdentity(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,22 +29,22 @@ def get_model(self, main_prog, startup_program): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofgather", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_identity", - inputs={'X': tindata}, - outputs={'Out': toutdata}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_identity", + inputs={'X': tindata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id, 'nranks': nranks}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_isend_irecv_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_isend_irecv_api_dygraph.py index 37a38b218c5dc..d76e48f5ba5bc 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_isend_irecv_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_isend_irecv_api_dygraph.py @@ -19,7 +19,6 @@ class TestCollectiveIsendIrecvAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_reduce_api.py b/python/paddle/fluid/tests/unittests/collective/collective_reduce_api.py index 83f4e3458624a..81e6d7daa2770 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_reduce_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_reduce_api.py @@ -21,15 +21,14 @@ class TestCollectiveReduceAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) paddle.distributed.reduce(tindata, dst=0) return [tindata] diff --git a/python/paddle/fluid/tests/unittests/collective/collective_reduce_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_reduce_api_dygraph.py index 5e9dfc8265ea1..5486e317d897a 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_reduce_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_reduce_api_dygraph.py @@ -19,7 +19,6 @@ class TestCollectiveReduceAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_reduce_op.py b/python/paddle/fluid/tests/unittests/collective/collective_reduce_op.py index d30b845098ce5..a0b34247a8de4 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_reduce_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_reduce_op.py @@ -22,7 +22,6 @@ class TestCollectiveReduce(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,26 +29,28 @@ def get_model(self, main_prog, startup_program): ring_id = 0 rootid = 1 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofreduce", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_reduce_sum", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'root_id': rootid - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_reduce_sum", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'root_id': rootid}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_reduce_op_calc_stream.py b/python/paddle/fluid/tests/unittests/collective/collective_reduce_op_calc_stream.py index ab3fe725df849..c7fe247d45585 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_reduce_op_calc_stream.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_reduce_op_calc_stream.py @@ -22,7 +22,6 @@ class TestCollectiveReduce(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,27 +29,32 @@ def get_model(self, main_prog, startup_program): ring_id = 0 rootid = 1 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofreduce", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_reduce_sum", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True, - 'root_id': rootid - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_reduce_sum", + inputs={'X': tindata}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': True, + 'root_id': rootid, + }, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter.py b/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter.py index d7c67e60a46ab..a8a7c67d96ee8 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter.py @@ -20,7 +20,6 @@ class TestCollectiveReduceScatter(unittest.TestCase): - def setUp(self): dist.init_parallel_env() paddle.fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -76,9 +75,9 @@ def test_collective_reduce_scatter_base(self): # [1, 2, 3, 4] # Rank-1 output = paddle.empty(shape=[2], dtype=input.dtype) - task = paddle.distributed.collective._reduce_scatter_base(output, - input, - sync_op=False) + task = paddle.distributed.collective._reduce_scatter_base( + output, input, sync_op=False + ) task.wait() diff --git a/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter_api_dygraph.py index c9df2459a78e0..617f9889873f4 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_reduce_scatter_api_dygraph.py @@ -19,7 +19,6 @@ class TestCollectiveReduceScatterAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_scatter_api.py b/python/paddle/fluid/tests/unittests/collective/collective_scatter_api.py index f198ae467d560..ec2ca037bbfdf 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_scatter_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_scatter_api.py @@ -21,19 +21,20 @@ class TestCollectiveScatterAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32', - append_batch_size=False) - toutdata = layers.fill_constant(shape=[5, 1000], - dtype='float32', - value=1.0) + tindata = layers.data( + name="tindata", + shape=[10, 1000], + dtype='float32', + append_batch_size=False, + ) + toutdata = layers.fill_constant( + shape=[5, 1000], dtype='float32', value=1.0 + ) tensor_list = None if rank == 1: tensor_list = paddle.split(tindata, 2, axis=0) diff --git a/python/paddle/fluid/tests/unittests/collective/collective_scatter_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_scatter_api_dygraph.py index 8f27f84a32d52..9061a1b5a4f06 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_scatter_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_scatter_api_dygraph.py @@ -19,7 +19,6 @@ class TestCollectiveScatterAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -32,9 +31,9 @@ def get_model(self, main_prog, startup_program, rank, indata=None): if rank == 0: dist.scatter(subdata1, src=1) else: - dist.scatter(subdata1, - tensor_list=[subdata1, subdata2], - src=1) + dist.scatter( + subdata1, tensor_list=[subdata1, subdata2], src=1 + ) return [subdata1.cast("float32").numpy()] else: tindata = paddle.to_tensor(indata) @@ -42,9 +41,9 @@ def get_model(self, main_prog, startup_program, rank, indata=None): if rank == 0: dist.scatter(subdata1, src=1) else: - dist.scatter(subdata1, - tensor_list=[subdata1, subdata2], - src=1) + dist.scatter( + subdata1, tensor_list=[subdata1, subdata2], src=1 + ) return [subdata1.numpy()] diff --git a/python/paddle/fluid/tests/unittests/collective/collective_scatter_op.py b/python/paddle/fluid/tests/unittests/collective/collective_scatter_op.py index 4a8a1acfe5f8c..29cfe40117bc7 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_scatter_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_scatter_op.py @@ -22,7 +22,6 @@ class TestCollectiveScatter(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,27 +29,28 @@ def get_model(self, main_prog, startup_program): ring_id = 0 rootid = 1 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofreduce", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_scatter", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'root': rootid, - 'nranks': 2 - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_scatter", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'root': rootid, 'nranks': 2}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api.py b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api.py index d22080e8cdf53..b8c233051ee6b 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api.py @@ -21,16 +21,17 @@ class TestCollectiveSendRecvAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32', - append_batch_size=False) + tindata = layers.data( + name="tindata", + shape=[10, 1000], + dtype='float32', + append_batch_size=False, + ) if rank == 0: paddle.distributed.send(tindata, dst=1) else: diff --git a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api_dygraph.py index b4bf24ffbfaa9..701c8c8158293 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_api_dygraph.py @@ -19,7 +19,6 @@ class TestCollectiveSendRecvAPI(test_base.TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 diff --git a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op.py b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op.py index 6324e619f9ea5..708326a3f4968 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op.py @@ -21,36 +21,40 @@ class TestCollectiveSendRecv(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program): ring_id = self.global_ring_id with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float64', - append_batch_size=False) + tindata = layers.data( + name="tindata", + shape=[10, 1000], + dtype='float64', + append_batch_size=False, + ) if self.rank == 0: - main_prog.global_block().append_op(type="send_v2", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'peer': 1, - 'use_calc_stream': True - }) + main_prog.global_block().append_op( + type="send_v2", + inputs={'X': tindata}, + attrs={ + 'ring_id': ring_id, + 'peer': 1, + 'use_calc_stream': True, + }, + ) else: - main_prog.global_block().append_op(type="recv_v2", - outputs={'Out': tindata}, - attrs={ - 'peer': 0, - 'ring_id': ring_id, - 'dtype': tindata.dtype, - 'out_shape': - tindata.shape, - 'use_calc_stream': True, - }) + main_prog.global_block().append_op( + type="recv_v2", + outputs={'Out': tindata}, + attrs={ + 'peer': 0, + 'ring_id': ring_id, + 'dtype': tindata.dtype, + 'out_shape': tindata.shape, + 'use_calc_stream': True, + }, + ) return tindata diff --git a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_array.py b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_array.py index ec12db6f5be04..eea4ef4615f85 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_array.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_array.py @@ -22,39 +22,46 @@ class TestCollectiveSendRecv(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program): ring_id = self.global_ring_id with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float64', - append_batch_size=False) + tindata = layers.data( + name="tindata", + shape=[10, 1000], + dtype='float64', + append_batch_size=False, + ) if self.rank == 0: data1 = fluid.layers.assign( - np.array([[0, 1, 2]], dtype='float32')) + np.array([[0, 1, 2]], dtype='float32') + ) data2 = fluid.layers.assign( - np.array([[3, 4, 5]], dtype='float32')) + np.array([[3, 4, 5]], dtype='float32') + ) elif self.rank == 1: data1 = fluid.layers.assign( - np.array([[3, 4, 5]], dtype='float32')) + np.array([[3, 4, 5]], dtype='float32') + ) data2 = fluid.layers.assign( - np.array([[0, 1, 2]], dtype='float32')) + np.array([[0, 1, 2]], dtype='float32') + ) tensor_array = fluid.layers.create_array(dtype='float32') i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=0) fluid.layers.array_write(data1, i, tensor_array) fluid.layers.array_write(data2, i + 1, tensor_array) if self.rank == 0: - main_prog.global_block().append_op(type="send_v2", - inputs={'X': tensor_array}, - attrs={ - 'ring_id': ring_id, - 'peer': 1, - 'use_calc_stream': True - }) + main_prog.global_block().append_op( + type="send_v2", + inputs={'X': tensor_array}, + attrs={ + 'ring_id': ring_id, + 'peer': 1, + 'use_calc_stream': True, + }, + ) else: main_prog.global_block().append_op( type="recv_v2", @@ -65,7 +72,8 @@ def get_model(self, main_prog, startup_program): 'dtype': data1.dtype, 'out_shape': [1, 3], 'use_calc_stream': True, - }) + }, + ) return tensor_array diff --git a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_dynamic_shape.py b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_dynamic_shape.py index da6ef74c11460..28804d1570fbb 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_dynamic_shape.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_sendrecv_op_dynamic_shape.py @@ -21,41 +21,46 @@ class TestCollectiveSendRecvDynamicShape(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program): ring_id = self.global_ring_id with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float64', - append_batch_size=False) + tindata = layers.data( + name="tindata", + shape=[10, 1000], + dtype='float64', + append_batch_size=False, + ) if self.rank == 0: - main_prog.global_block().append_op(type="send_v2", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'peer': 1, - 'use_calc_stream': True, - 'dynamic_shape': True - }) + main_prog.global_block().append_op( + type="send_v2", + inputs={'X': tindata}, + attrs={ + 'ring_id': ring_id, + 'peer': 1, + 'use_calc_stream': True, + 'dynamic_shape': True, + }, + ) else: - main_prog.global_block().append_op(type="recv_v2", - outputs={'Out': tindata}, - attrs={ - 'peer': 0, - 'ring_id': ring_id, - 'dtype': tindata.dtype, - 'out_shape': - tindata.shape, - 'use_calc_stream': True, - 'dynamic_shape': True - }) + main_prog.global_block().append_op( + type="recv_v2", + outputs={'Out': tindata}, + attrs={ + 'peer': 0, + 'ring_id': ring_id, + 'dtype': tindata.dtype, + 'out_shape': tindata.shape, + 'use_calc_stream': True, + 'dynamic_shape': True, + }, + ) return tindata if __name__ == "__main__": - runtime_main(TestCollectiveSendRecvDynamicShape, "sendrecv_dynamic_shape", - 0) + runtime_main( + TestCollectiveSendRecvDynamicShape, "sendrecv_dynamic_shape", 0 + ) diff --git a/python/paddle/fluid/tests/unittests/collective/collective_split_op.py b/python/paddle/fluid/tests/unittests/collective/collective_split_op.py index 2e71ccf304939..57a8aa4fcc5d1 100644 --- a/python/paddle/fluid/tests/unittests/collective/collective_split_op.py +++ b/python/paddle/fluid/tests/unittests/collective/collective_split_op.py @@ -22,7 +22,6 @@ class TestCollectiveAllGather(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,23 +29,22 @@ def get_model(self, main_prog, startup_program): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofsplit", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_split", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'rank': self.rank, - 'nranks': nranks - }, - outputs={'Out': toutdata}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_split", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'rank': self.rank, 'nranks': nranks}, + outputs={'Out': toutdata}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective/column_parallel_linear_api.py b/python/paddle/fluid/tests/unittests/collective/column_parallel_linear_api.py index 5f6178dbfd652..accecd08c1047 100644 --- a/python/paddle/fluid/tests/unittests/collective/column_parallel_linear_api.py +++ b/python/paddle/fluid/tests/unittests/collective/column_parallel_linear_api.py @@ -22,7 +22,6 @@ class TestColumnParallelLinearAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -32,18 +31,22 @@ def get_model(self, main_prog, startup_program, rank): np.random.seed(2020) np_array = np.random.rand(1000, 16) - data = paddle.static.data(name='tindata', - shape=[10, 1000], - dtype="float32") + data = paddle.static.data( + name='tindata', shape=[10, 1000], dtype="float32" + ) paddle.distributed.broadcast(data, src=0) if rank == 0: param_attr = paddle.fluid.ParamAttr( initializer=paddle.fluid.initializer.NumpyArrayInitializer( - np_array[:, 0:8]), ) + np_array[:, 0:8] + ), + ) else: param_attr = paddle.fluid.ParamAttr( initializer=paddle.fluid.initializer.NumpyArrayInitializer( - np_array[:, 8:16]), ) + np_array[:, 8:16] + ), + ) linear_out = paddle.distributed.split( data, diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_allgather_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_allgather_api_dygraph.py index f4ff1e5b7b37c..a04e30dfae140 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_allgather_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_allgather_api_dygraph.py @@ -19,8 +19,7 @@ import test_collective_api_base as test_collective_base -class StreamAllgatherTestCase(): - +class StreamAllgatherTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ def __init__(self): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -39,47 +39,53 @@ def run_test_case(self): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) rank = dist.get_rank() tensor = paddle.to_tensor(test_data_list[rank]) # case 1: pass an empty tensor list empty_tensor_list = [] - task = dist.stream.all_gather(empty_tensor_list, - tensor, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.all_gather( + empty_tensor_list, + tensor, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() - assert np.allclose(empty_tensor_list, - test_data_list, - rtol=1e-05, - atol=1e-05) + assert np.allclose( + empty_tensor_list, test_data_list, rtol=1e-05, atol=1e-05 + ) # case 2: pass a pre-sized tensor list full_tensor_list = [paddle.empty_like(tensor) for _ in test_data_list] - task = dist.stream.all_gather(full_tensor_list, - tensor, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.all_gather( + full_tensor_list, + tensor, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() - assert np.allclose(full_tensor_list, - test_data_list, - rtol=1e-05, - atol=1e-05) + assert np.allclose( + full_tensor_list, test_data_list, rtol=1e-05, atol=1e-05 + ) # case 3: pass a pre-sized tensor result_tensor = paddle.concat( - [paddle.to_tensor(data) for data in test_data_list]) + [paddle.to_tensor(data) for data in test_data_list] + ) out_tensor = paddle.empty_like(result_tensor) - task = dist.stream.all_gather(out_tensor, - tensor, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.all_gather( + out_tensor, + tensor, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() assert np.allclose(out_tensor, result_tensor, rtol=1e-05, atol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_allreduce_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_allreduce_api_dygraph.py index abf455203517d..be8841345e2f5 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_allreduce_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_allreduce_api_dygraph.py @@ -19,8 +19,7 @@ import test_collective_api_base as test_collective_base -class StreamAllReduceTestCase(): - +class StreamAllReduceTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ def __init__(self): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -39,15 +39,16 @@ def run_test_case(self): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) rank = dist.get_rank() tensor = paddle.to_tensor(test_data_list[rank]) - task = dist.stream.all_reduce(tensor, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.all_reduce( + tensor, sync_op=self._sync_op, use_calc_stream=self._use_calc_stream + ) if not self._sync_op: task.wait() diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_api_dygraph.py index 8842bcd08b640..07693342c6b4a 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_api_dygraph.py @@ -19,8 +19,7 @@ import test_collective_api_base as test_collective_base -class StreamAllToAllTestCase(): - +class StreamAllToAllTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ def __init__(self): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -39,17 +39,23 @@ def run_test_case(self): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) nranks = len(test_data_list) data1 = test_data_list[0] data2 = test_data_list[1] result1 = np.vstack( - [data1[0:data1.shape[0] // 2, :], data2[0:data2.shape[0] // 2, :]]) + [ + data1[0 : data1.shape[0] // 2, :], + data2[0 : data2.shape[0] // 2, :], + ] + ) result2 = np.vstack( - [data1[data1.shape[0] // 2:, :], data2[data2.shape[0] // 2:, :]]) + [data1[data1.shape[0] // 2 :, :], data2[data2.shape[0] // 2 :, :]] + ) rank = dist.get_rank() tensor = paddle.to_tensor(test_data_list[rank]) @@ -57,48 +63,52 @@ def run_test_case(self): # case 1: pass an empty tensor list empty_tensor_list = [] - task = dist.stream.alltoall(empty_tensor_list, [t1, t2], - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.alltoall( + empty_tensor_list, + [t1, t2], + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() result_tensor_list = np.vstack(empty_tensor_list) if rank == 0: - assert np.allclose(result_tensor_list, - result1, - rtol=1e-05, - atol=1e-05) + assert np.allclose( + result_tensor_list, result1, rtol=1e-05, atol=1e-05 + ) else: - assert np.allclose(result_tensor_list, - result2, - rtol=1e-05, - atol=1e-05) + assert np.allclose( + result_tensor_list, result2, rtol=1e-05, atol=1e-05 + ) # case 2: pass a pre-sized tensor list full_tensor_list = [paddle.empty_like(t1) for _ in test_data_list] - task = dist.stream.alltoall(full_tensor_list, [t1, t2], - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.alltoall( + full_tensor_list, + [t1, t2], + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() result_tensor_list = np.vstack(full_tensor_list) if rank == 0: - assert np.allclose(result_tensor_list, - result1, - rtol=1e-05, - atol=1e-05) + assert np.allclose( + result_tensor_list, result1, rtol=1e-05, atol=1e-05 + ) else: - assert np.allclose(result_tensor_list, - result2, - rtol=1e-05, - atol=1e-05) + assert np.allclose( + result_tensor_list, result2, rtol=1e-05, atol=1e-05 + ) # case 3: pass a pre-sized tensor out_tensor = paddle.empty_like(tensor) - task = dist.stream.alltoall(out_tensor, - tensor, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.alltoall( + out_tensor, + tensor, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() if rank == 0: diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_single_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_single_api_dygraph.py index bb36693b0fbc9..cb235dd9d1172 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_single_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_alltoall_single_api_dygraph.py @@ -19,8 +19,7 @@ import test_collective_api_base as test_collective_base -class StreamAllToAllSingleTestCase(): - +class StreamAllToAllSingleTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ def __init__(self): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -39,17 +39,23 @@ def run_test_case(self): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) nranks = len(test_data_list) data1 = paddle.to_tensor(test_data_list[0]) data2 = paddle.to_tensor(test_data_list[1]) result1 = np.vstack( - (data1[0:data1.shape[0] // 2, :], data2[0:data2.shape[0] // 2, :])) + ( + data1[0 : data1.shape[0] // 2, :], + data2[0 : data2.shape[0] // 2, :], + ) + ) result2 = np.vstack( - (data1[data1.shape[0] // 2:, :], data2[data2.shape[0] // 2:, :])) + (data1[data1.shape[0] // 2 :, :], data2[data2.shape[0] // 2 :, :]) + ) rank = dist.get_rank() tensor = paddle.to_tensor(test_data_list[rank]) @@ -59,7 +65,8 @@ def run_test_case(self): out_tensor, tensor, sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() if rank == 0: diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_broadcast_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_broadcast_api_dygraph.py index 487dfd6ae6894..68d496a11e48d 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_broadcast_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_broadcast_api_dygraph.py @@ -19,8 +19,7 @@ import test_collective_api_base as test_collective_base -class StreamBroadcastTestCase(): - +class StreamBroadcastTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ def __init__(self): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -38,12 +38,15 @@ def run_test_case(self): src_rank = 1 result = test_collective_base.create_test_data( - shape=self._shape, dtype=self._dtype, seed=self._seeds[src_rank]) + shape=self._shape, dtype=self._dtype, seed=self._seeds[src_rank] + ) tensor = paddle.to_tensor(result) - task = dist.stream.broadcast(tensor, - src=src_rank, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.broadcast( + tensor, + src=src_rank, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_api_dygraph.py index a487eac566ab5..444390d20c949 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_api_dygraph.py @@ -19,8 +19,7 @@ import test_collective_api_base as test_collective_base -class StreamReduceTestCase(): - +class StreamReduceTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ def __init__(self): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -39,16 +39,19 @@ def run_test_case(self): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) rank = dist.get_rank() tensor = paddle.to_tensor(test_data_list[rank]) - task = dist.stream.reduce(tensor, - dst=1, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.reduce( + tensor, + dst=1, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() @@ -56,10 +59,9 @@ def run_test_case(self): if rank == 1: assert np.allclose(tensor, result, rtol=1e-05, atol=1e-05) else: - assert np.allclose(tensor, - test_data_list[rank], - rtol=1e-05, - atol=1e-05) + assert np.allclose( + tensor, test_data_list[rank], rtol=1e-05, atol=1e-05 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_scatter_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_scatter_api_dygraph.py index effaf1cb6c99a..d65d99ba8e55b 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_scatter_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_reduce_scatter_api_dygraph.py @@ -17,11 +17,12 @@ import paddle import paddle.distributed as dist import test_collective_api_base as test_collective_base -from paddle.distributed.communication.stream.reduce_scatter import _reduce_scatter_base +from paddle.distributed.communication.stream.reduce_scatter import ( + _reduce_scatter_base, +) -class StreamReduceScatterTestCase(): - +class StreamReduceScatterTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -31,7 +32,8 @@ def __init__(self): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -40,12 +42,13 @@ def run_test_case(self): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) reduce_result = sum(test_data_list) - result1 = reduce_result[0:reduce_result.shape[0] // 2] - result2 = reduce_result[reduce_result.shape[0] // 2:] + result1 = reduce_result[0 : reduce_result.shape[0] // 2] + result2 = reduce_result[reduce_result.shape[0] // 2 :] rank = dist.get_rank() tensor = paddle.to_tensor(test_data_list[rank]) @@ -53,9 +56,12 @@ def run_test_case(self): # case 1: pass a pre-sized tensor list t1, t2 = paddle.split(tensor, 2, axis=0) result_tensor = paddle.empty_like(t1) - task = dist.stream.reduce_scatter(result_tensor, [t1, t2], - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.reduce_scatter( + result_tensor, + [t1, t2], + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() if rank == 0: @@ -65,10 +71,12 @@ def run_test_case(self): # case 2: pass a pre-sized tensor result_tensor = paddle.empty_like(t1) - task = dist.stream.reduce_scatter(result_tensor, - tensor, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.reduce_scatter( + result_tensor, + tensor, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() if rank == 0: @@ -78,10 +86,12 @@ def run_test_case(self): # case 3: test the legacy API result_tensor = paddle.empty_like(t1) - task = _reduce_scatter_base(result_tensor, - tensor, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = _reduce_scatter_base( + result_tensor, + tensor, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() if rank == 0: diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_scatter_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_scatter_api_dygraph.py index 6060e5050ca09..b4f01798a6677 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_scatter_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_scatter_api_dygraph.py @@ -19,8 +19,7 @@ import test_collective_api_base as test_collective_base -class StreamScatterTestCase(): - +class StreamScatterTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ def __init__(self): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -39,24 +39,28 @@ def run_test_case(self): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) src_rank = 1 src_data = test_data_list[src_rank] - result1 = src_data[0:src_data.shape[0] // 2] - result2 = src_data[src_data.shape[0] // 2:] + result1 = src_data[0 : src_data.shape[0] // 2] + result2 = src_data[src_data.shape[0] // 2 :] rank = dist.get_rank() # case 1: pass a pre-sized tensor list tensor = paddle.to_tensor(test_data_list[rank]) t1, t2 = paddle.split(tensor, 2, axis=0) - task = dist.stream.scatter(t1, [t1, t2], - src=src_rank, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.scatter( + t1, + [t1, t2], + src=src_rank, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() if rank == src_rank: @@ -67,11 +71,13 @@ def run_test_case(self): # case 2: pass a pre-sized tensor tensor = paddle.to_tensor(src_data) t1 = paddle.empty_like(t1) - task = dist.stream.scatter(t1, - tensor, - src=src_rank, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.scatter( + t1, + tensor, + src=src_rank, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() if rank == src_rank: diff --git a/python/paddle/fluid/tests/unittests/collective/communication_stream_sendrecv_api_dygraph.py b/python/paddle/fluid/tests/unittests/collective/communication_stream_sendrecv_api_dygraph.py index f9b2806fc1b84..aa7891a7ab752 100644 --- a/python/paddle/fluid/tests/unittests/collective/communication_stream_sendrecv_api_dygraph.py +++ b/python/paddle/fluid/tests/unittests/collective/communication_stream_sendrecv_api_dygraph.py @@ -19,8 +19,7 @@ import test_collective_api_base as test_collective_base -class StreamSendRecvTestCase(): - +class StreamSendRecvTestCase: def __init__(self): self._sync_op = eval(os.getenv("sync_op")) self._use_calc_stream = eval(os.getenv("use_calc_stream")) @@ -30,7 +29,8 @@ def __init__(self): self._seeds = eval(os.getenv("seeds")) if self._backend not in ["nccl", "gloo"]: raise NotImplementedError( - "Only support nccl and gloo as the backend for now.") + "Only support nccl and gloo as the backend for now." + ) os.environ["PADDLE_DISTRI_BACKEND"] = self._backend def run_test_case(self): @@ -39,9 +39,10 @@ def run_test_case(self): test_data_list = [] for seed in self._seeds: test_data_list.append( - test_collective_base.create_test_data(shape=self._shape, - dtype=self._dtype, - seed=seed)) + test_collective_base.create_test_data( + shape=self._shape, dtype=self._dtype, seed=seed + ) + ) src_rank = 0 dst_rank = 1 @@ -49,15 +50,19 @@ def run_test_case(self): rank = dist.get_rank() tensor = paddle.to_tensor(test_data_list[rank]) if rank == 0: - task = dist.stream.send(tensor, - dst=dst_rank, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.send( + tensor, + dst=dst_rank, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) else: - task = dist.stream.recv(tensor, - src=src_rank, - sync_op=self._sync_op, - use_calc_stream=self._use_calc_stream) + task = dist.stream.recv( + tensor, + src=src_rank, + sync_op=self._sync_op, + use_calc_stream=self._use_calc_stream, + ) if not self._sync_op: task.wait() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/auto_parallel_parallelizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/auto_parallel_parallelizer.py index 185e2c23fed95..d87cacb21389f 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/auto_parallel_parallelizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/auto_parallel_parallelizer.py @@ -30,27 +30,27 @@ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.linear2 = nn.Linear(d_model, 1, weight_attr, bias_attr=bias_attr) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") @@ -67,24 +67,29 @@ def forward(self, input): def mlp_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, sequence_len, 1], - dtype='float32') + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + label = static.data( + name="label", shape=[batch_size, sequence_len, 1], dtype='float32' + ) auto.shard_tensor(input, _global_process_mesh, [None, None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) predict = mlp(input) @@ -95,7 +100,6 @@ def mlp_pretrain_forward(train_program, start_program): class TestMLPAutoParallelizer(unittest.TestCase): - def test_mlp_serial(self): global _global_process_mesh @@ -114,17 +118,24 @@ def test_mlp_serial(self): train_program = static.Program() start_program = static.Program() loss, train_program, start_program = mlp_pretrain_forward( - train_program, start_program) + train_program, start_program + ) - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) optimizer = fleet.distributed_optimizer(optimizer) - _, _, distributed_startup_program, distributed_main_program = optimizer.minimize( - loss, start_program) + ( + _, + _, + distributed_startup_program, + distributed_main_program, + ) = optimizer.minimize(loss, start_program) suffix = core.kAutoParallelSuffix() for block in distributed_main_program.blocks: for op in block.ops: diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/c_comm_init_op.py b/python/paddle/fluid/tests/unittests/collective/fleet/c_comm_init_op.py index 4c981156ebb74..a90c7f1baae68 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/c_comm_init_op.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/c_comm_init_op.py @@ -15,14 +15,15 @@ import unittest import os import paddle.fluid as fluid -from paddle.distributed.fleet.base.private_helper_function import wait_server_ready +from paddle.distributed.fleet.base.private_helper_function import ( + wait_server_ready, +) import paddle paddle.enable_static() class TestCCommInitOp(unittest.TestCase): - def setUp(self): self.endpoints = os.getenv("PADDLE_TRAINER_ENDPOINTS").split(',') self.current_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT") @@ -42,24 +43,29 @@ def test_specifying_devices(self): nccl_id_var = block.create_var( name=fluid.unique_name.generate('nccl_id'), persistable=True, - type=fluid.core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_nccl_id', - inputs={}, - outputs={'Out': nccl_id_var}, - attrs={ - 'rank': self.rank, - 'endpoint': self.current_endpoint, - 'other_endpoints': self.other_endpoints - }) - block.append_op(type='c_comm_init', - inputs={'X': nccl_id_var}, - outputs={}, - attrs={ - 'nranks': self.nranks, - 'rank': self.rank, - 'ring_id': 0, - 'device_id': self.gpu_id - }) + type=fluid.core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_nccl_id', + inputs={}, + outputs={'Out': nccl_id_var}, + attrs={ + 'rank': self.rank, + 'endpoint': self.current_endpoint, + 'other_endpoints': self.other_endpoints, + }, + ) + block.append_op( + type='c_comm_init', + inputs={'X': nccl_id_var}, + outputs={}, + attrs={ + 'nranks': self.nranks, + 'rank': self.rank, + 'ring_id': 0, + 'device_id': self.gpu_id, + }, + ) self.exe.run(program) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge.py b/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge.py index dc144f997d464..9e2e2da3876bc 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge.py @@ -26,7 +26,6 @@ class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -39,23 +38,33 @@ def get_model(self, batch_size=2): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() # Optimization - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = fluid.optimizer.GradientMergeOptimizer(opt, 2) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) opt.minimize(avg_cost) - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge_raw_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge_raw_optimizer.py index ff31a7016a673..f4de5280d0f7d 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge_raw_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dist_mnist_gradient_merge_raw_optimizer.py @@ -23,7 +23,6 @@ class TestDistMnistGradientMergeRawOptimizer(TestDistRunnerBase): - def get_model(self, batch_size=2, single_device=False): paddle.enable_static() paddle.seed(1) @@ -54,9 +53,9 @@ def get_model(self, batch_size=2, single_device=False): strategy.without_graph_optimization = True fleet.init(is_collective=True, strategy=strategy) - image = paddle.static.data(name='image', - shape=[None, 1, 28, 28], - dtype="float32") + image = paddle.static.data( + name='image', shape=[None, 1, 28, 28], dtype="float32" + ) label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') predict = cnn_model(image) acc = paddle.metric.accuracy(predict, label) @@ -68,7 +67,8 @@ def get_model(self, batch_size=2, single_device=False): optimizer = fluid.optimizer.GradientMergeOptimizer( optimizer, k_steps=strategy.gradient_merge_configs["k_steps"], - avg=strategy.gradient_merge_configs["avg"]) + avg=strategy.gradient_merge_configs["avg"], + ) world_size = 1 else: optimizer = fleet.distributed_optimizer(optimizer) @@ -88,10 +88,12 @@ def get_model(self, batch_size=2, single_device=False): else: assert start_allreduce_idx == 1 - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) return test_program, cost, train_reader, test_reader, acc, predict diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api.py index bac023a8e7d95..1345032493409 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api.py @@ -21,7 +21,10 @@ from paddle.fluid.dygraph.nn import Linear from paddle.distributed import fleet from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.sharding import group_sharded_parallel, save_group_sharded_model +from paddle.distributed.sharding import ( + group_sharded_parallel, + save_group_sharded_model, +) epoch = 10 paddle.seed(2022) @@ -33,7 +36,6 @@ class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -49,7 +51,6 @@ def forward(self, inputs): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -62,45 +63,46 @@ def __reader__(): def optimizer_setting(model, use_multi_precision, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) optimizer = paddle.optimizer.Momentum( - parameters=[{ - "params": list(model.parameters()) - }] if opt_group else list(model.parameters()), + parameters=[{"params": list(model.parameters())}] + if opt_group + else list(model.parameters()), learning_rate=0.001, weight_decay=0.00001, grad_clip=clip, - multi_precision=use_multi_precision) + multi_precision=use_multi_precision, + ) return optimizer -def train_mlp(model, - shard_level, - use_multi_precision, - output_dir, - amp_level='O1'): +def train_mlp( + model, shard_level, use_multi_precision, output_dir, amp_level='O1' +): group = paddle.distributed.new_group([0, 1]) - optimizer = optimizer_setting(model=model, - use_multi_precision=use_multi_precision) - model = paddle.amp.decorate(models=model, - level=amp_level, - save_dtype='float32') + optimizer = optimizer_setting( + model=model, use_multi_precision=use_multi_precision + ) + model = paddle.amp.decorate( + models=model, level=amp_level, save_dtype='float32' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=32768) - model, optimizer, scaler = group_sharded_parallel(model=model, - optimizer=optimizer, - level=shard_level, - scaler=scaler) - - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + model, optimizer, scaler = group_sharded_parallel( + model=model, optimizer=optimizer, level=shard_level, scaler=scaler + ) + + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -111,8 +113,9 @@ def train_mlp(model, img.stop_gradient = True with paddle.amp.auto_cast(True, level=amp_level): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) if not use_multi_precision: @@ -138,44 +141,56 @@ def test_sharding_api(): output_dir = tempfile.mkdtemp() # fp16 - stage2_params = train_mlp(mlp1, - shard_level="os_g", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O2') - stage3_params = train_mlp(mlp2, - shard_level="p_g_os", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O2') + stage2_params = train_mlp( + mlp1, + shard_level="os_g", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O2', + ) + stage3_params = train_mlp( + mlp2, + shard_level="p_g_os", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O2', + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-4, - atol=1e-3) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-4, + atol=1e-3, + ) # AMP mlp3, mlp4 = MLP(), MLP() mlp3.set_state_dict(state_dict) mlp4.set_state_dict(state_dict) - stage2_params = train_mlp(mlp3, - shard_level="os_g", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O1') - stage3_params = train_mlp(mlp4, - shard_level="p_g_os", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O1') + stage2_params = train_mlp( + mlp3, + shard_level="os_g", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O1', + ) + stage3_params = train_mlp( + mlp4, + shard_level="p_g_os", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O1', + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-4, - atol=1e-3) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-4, + atol=1e-3, + ) shutil.rmtree(output_dir) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api_eager.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api_eager.py index f57dfd7c98d7a..2f07007c66a0b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api_eager.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_api_eager.py @@ -19,7 +19,10 @@ import paddle.fluid as fluid from paddle.fluid.dygraph.nn import Linear from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.sharding import group_sharded_parallel, save_group_sharded_model +from paddle.distributed.sharding import ( + group_sharded_parallel, + save_group_sharded_model, +) epoch = 10 paddle.seed(2022) @@ -31,7 +34,6 @@ class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -47,7 +49,6 @@ def forward(self, inputs): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -60,47 +61,55 @@ def __reader__(): def optimizer_setting(model, use_multi_precision, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) optimizer = paddle.optimizer.Momentum( - parameters=[{ - "params": list(model.parameters()) - }] if opt_group else list(model.parameters()), + parameters=[{"params": list(model.parameters())}] + if opt_group + else list(model.parameters()), learning_rate=0.001, weight_decay=0.00001, grad_clip=clip, - multi_precision=use_multi_precision) + multi_precision=use_multi_precision, + ) return optimizer -def train_mlp(model, - shard_level, - use_multi_precision, - output_dir, - amp_level='O1', - sync_buffers=False, - dp_group=None): - optimizer = optimizer_setting(model=model, - use_multi_precision=use_multi_precision) - model = paddle.amp.decorate(models=model, - level=amp_level, - save_dtype='float32') +def train_mlp( + model, + shard_level, + use_multi_precision, + output_dir, + amp_level='O1', + sync_buffers=False, + dp_group=None, +): + optimizer = optimizer_setting( + model=model, use_multi_precision=use_multi_precision + ) + model = paddle.amp.decorate( + models=model, level=amp_level, save_dtype='float32' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=32768) - model, optimizer, scaler = group_sharded_parallel(model=model, - optimizer=optimizer, - level=shard_level, - scaler=scaler, - sync_buffers=sync_buffers, - dp_group=dp_group) - - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + model, optimizer, scaler = group_sharded_parallel( + model=model, + optimizer=optimizer, + level=shard_level, + scaler=scaler, + sync_buffers=sync_buffers, + dp_group=dp_group, + ) + + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -111,8 +120,9 @@ def train_mlp(model, img.stop_gradient = True with paddle.amp.auto_cast(True, level=amp_level): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) if not use_multi_precision: @@ -138,51 +148,64 @@ def test_sharding_api(): output_dir = tempfile.mkdtemp() - #test sharding + dp, just for test + # test sharding + dp, just for test dp_group = paddle.distributed.new_group( - list(range(paddle.distributed.get_world_size()))) - - stage2_dp_params = train_mlp(mlp1, - shard_level="os_g", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O2', - sync_buffers=True, - dp_group=dp_group) + list(range(paddle.distributed.get_world_size())) + ) + + stage2_dp_params = train_mlp( + mlp1, + shard_level="os_g", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O2', + sync_buffers=True, + dp_group=dp_group, + ) # fp16 - stage2_params = train_mlp(mlp1, - shard_level="os_g", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O2') - stage3_params = train_mlp(mlp2, - shard_level="p_g_os", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O2') + stage2_params = train_mlp( + mlp1, + shard_level="os_g", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O2', + ) + stage3_params = train_mlp( + mlp2, + shard_level="p_g_os", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O2', + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-4, - atol=1e-3) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-4, + atol=1e-3, + ) # AMP mlp3, mlp4 = MLP(), MLP() mlp3.set_state_dict(state_dict) mlp4.set_state_dict(state_dict) - stage2_params = train_mlp(mlp3, - shard_level="os_g", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O1') - stage3_params = train_mlp(mlp4, - shard_level="p_g_os", - use_multi_precision=True, - output_dir=output_dir, - amp_level='O1') + stage2_params = train_mlp( + mlp3, + shard_level="os_g", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O1', + ) + stage3_params = train_mlp( + mlp4, + shard_level="p_g_os", + use_multi_precision=True, + output_dir=output_dir, + amp_level='O1', + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2.py index 40767d6fa86fc..970f7a5b03774 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2.py @@ -23,8 +23,12 @@ from paddle.fluid.dygraph.nn import Linear from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import GroupShardedOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import GroupShardedStage2 +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import ( + GroupShardedOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import ( + GroupShardedStage2, +) seed = 2022 epoch = 2 @@ -35,7 +39,6 @@ class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -51,7 +54,6 @@ def forward(self, inputs): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -63,42 +65,50 @@ def __reader__(): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) - optimizer = paddle.optimizer.AdamW(parameters=[{ - "params": model.parameters(), - }] if opt_group else model.parameters(), - learning_rate=0.001, - weight_decay=0.00001, - grad_clip=clip, - multi_precision=use_pure_fp16) + optimizer = paddle.optimizer.AdamW( + parameters=[ + { + "params": model.parameters(), + } + ] + if opt_group + else model.parameters(), + learning_rate=0.001, + weight_decay=0.00001, + grad_clip=clip, + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - sharding_stage, - batch_size=100, - use_pure_fp16=False, - accumulate_grad=False, - opt_group=False, - save_model=False, - test_minimize=False): +def train_mlp( + model, + sharding_stage, + batch_size=100, + use_pure_fp16=False, + accumulate_grad=False, + opt_group=False, + save_model=False, + test_minimize=False, +): if sharding_stage != "dp": group = paddle.distributed.new_group([0, 1], backend="nccl") if opt_group: - optimizer = optimizer_setting(model=model, - use_pure_fp16=use_pure_fp16, - opt_group=opt_group) + optimizer = optimizer_setting( + model=model, use_pure_fp16=use_pure_fp16, opt_group=opt_group + ) else: optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if sharding_stage == 2: optimizer = GroupShardedOptimizerStage2( - params=optimizer._parameter_list, optim=optimizer, group=group) + params=optimizer._parameter_list, optim=optimizer, group=group + ) - model = GroupShardedStage2(model, - optimizer, - group=group, - buffer_max_size=2**21) + model = GroupShardedStage2( + model, optimizer, group=group, buffer_max_size=2**21 + ) else: model = paddle.DataParallel(model) @@ -108,18 +118,21 @@ def train_mlp(model, optimizer.minimize() except: print( - "====== Find sharding_stage2_optimizer.minimize() error ======") + "====== Find sharding_stage2_optimizer.minimize() error ======" + ) return - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) if sharding_stage == 2: @@ -174,50 +187,50 @@ def test_dp_stage2(): mlp7.set_state_dict(state_dict) # DP VS stage2 - dp_params = train_mlp(mlp1, - sharding_stage="dp", - use_pure_fp16=False, - opt_group=False) - stage2_params = train_mlp(mlp2, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False) + dp_params = train_mlp( + mlp1, sharding_stage="dp", use_pure_fp16=False, opt_group=False + ) + stage2_params = train_mlp( + mlp2, sharding_stage=2, use_pure_fp16=False, opt_group=False + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6 + ) # stage2 accumulate grad stage2_params = train_mlp(mlp3, sharding_stage=2, accumulate_grad=True) - stage2_accumulate_grad = train_mlp(mlp4, - sharding_stage=2, - batch_size=20, - accumulate_grad=True) + stage2_accumulate_grad = train_mlp( + mlp4, sharding_stage=2, batch_size=20, accumulate_grad=True + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage2_accumulate_grad[i].numpy(), - rtol=1e-5, - atol=1e-5) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage2_accumulate_grad[i].numpy(), + rtol=1e-5, + atol=1e-5, + ) # stage2 param list VS param group - stage2_params = train_mlp(mlp5, - sharding_stage=2, - use_pure_fp16=False, - opt_group=True) + stage2_params = train_mlp( + mlp5, sharding_stage=2, use_pure_fp16=False, opt_group=True + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6 + ) # save/load model output_dir = tempfile.mkdtemp() model_file = os.path.join(output_dir, "model.pdmodel") optimizer_file = os.path.join(output_dir, "model.pdopt") - model_stage2, optimizer_stage2 = train_mlp(mlp6, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False, - save_model=True) + model_stage2, optimizer_stage2 = train_mlp( + mlp6, + sharding_stage=2, + use_pure_fp16=False, + opt_group=False, + save_model=True, + ) paddle.save(model_stage2.state_dict(), model_file) paddle.save(optimizer_stage2.state_dict(), optimizer_file) m_state_dict = paddle.load(model_file) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_comm_overlap.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_comm_overlap.py index 7f16d926f5375..56c90009da673 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_comm_overlap.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_comm_overlap.py @@ -23,8 +23,12 @@ from paddle.fluid.dygraph.nn import Linear from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import GroupShardedOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import GroupShardedStage2 +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import ( + GroupShardedOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import ( + GroupShardedStage2, +) seed = 2022 epoch = 2 @@ -35,7 +39,6 @@ class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -51,7 +54,6 @@ def forward(self, inputs): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -63,42 +65,50 @@ def __reader__(): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) - optimizer = paddle.optimizer.AdamW(parameters=[{ - "params": model.parameters(), - }] if opt_group else model.parameters(), - learning_rate=0.001, - weight_decay=0.00001, - grad_clip=clip, - multi_precision=use_pure_fp16) + optimizer = paddle.optimizer.AdamW( + parameters=[ + { + "params": model.parameters(), + } + ] + if opt_group + else model.parameters(), + learning_rate=0.001, + weight_decay=0.00001, + grad_clip=clip, + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - sharding_stage, - batch_size=100, - use_pure_fp16=False, - accumulate_grad=False, - opt_group=False, - save_model=False, - test_minimize=False): +def train_mlp( + model, + sharding_stage, + batch_size=100, + use_pure_fp16=False, + accumulate_grad=False, + opt_group=False, + save_model=False, + test_minimize=False, +): if sharding_stage != "dp": group = paddle.distributed.new_group([0, 1], backend="nccl") if opt_group: - optimizer = optimizer_setting(model=model, - use_pure_fp16=use_pure_fp16, - opt_group=opt_group) + optimizer = optimizer_setting( + model=model, use_pure_fp16=use_pure_fp16, opt_group=opt_group + ) else: optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if sharding_stage == 2: origin_model = model optimizer = GroupShardedOptimizerStage2( - params=optimizer._parameter_list, optim=optimizer, group=group) - model = GroupShardedStage2(model, - optimizer, - group=group, - buffer_max_size=2**21) + params=optimizer._parameter_list, optim=optimizer, group=group + ) + model = GroupShardedStage2( + model, optimizer, group=group, buffer_max_size=2**21 + ) model._set_reduce_overlap(True) optimizer._set_broadcast_overlap(True, model) else: @@ -110,18 +120,21 @@ def train_mlp(model, optimizer.minimize() except: print( - "====== Find sharding_stage2_optimizer.minimize() error ======") + "====== Find sharding_stage2_optimizer.minimize() error ======" + ) return - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) if sharding_stage == 2: @@ -178,50 +191,50 @@ def test_dp_stage2(): mlp7.set_state_dict(state_dict) # DP VS stage2 - dp_params = train_mlp(mlp1, - sharding_stage="dp", - use_pure_fp16=False, - opt_group=False) - stage2_params = train_mlp(mlp2, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False) + dp_params = train_mlp( + mlp1, sharding_stage="dp", use_pure_fp16=False, opt_group=False + ) + stage2_params = train_mlp( + mlp2, sharding_stage=2, use_pure_fp16=False, opt_group=False + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6 + ) # stage2 accumulate grad stage2_params = train_mlp(mlp3, sharding_stage=2, accumulate_grad=True) - stage2_accumulate_grad = train_mlp(mlp4, - sharding_stage=2, - batch_size=20, - accumulate_grad=True) + stage2_accumulate_grad = train_mlp( + mlp4, sharding_stage=2, batch_size=20, accumulate_grad=True + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage2_accumulate_grad[i].numpy(), - rtol=1e-5, - atol=1e-5) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage2_accumulate_grad[i].numpy(), + rtol=1e-5, + atol=1e-5, + ) # stage2 param list VS param group - stage2_params = train_mlp(mlp5, - sharding_stage=2, - use_pure_fp16=False, - opt_group=True) + stage2_params = train_mlp( + mlp5, sharding_stage=2, use_pure_fp16=False, opt_group=True + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6 + ) # save/load model output_dir = tempfile.mkdtemp() model_file = os.path.join(output_dir, "model.pdmodel") optimizer_file = os.path.join(output_dir, "model.pdopt") - model_stage2, optimizer_stage2 = train_mlp(mlp6, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False, - save_model=True) + model_stage2, optimizer_stage2 = train_mlp( + mlp6, + sharding_stage=2, + use_pure_fp16=False, + opt_group=False, + save_model=True, + ) paddle.save(model_stage2.state_dict(), model_file) paddle.save(optimizer_stage2.state_dict(), optimizer_file) m_state_dict = paddle.load(model_file) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_offload.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_offload.py index 3420114d3f9be..710dc90fcf53b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_offload.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage2_offload.py @@ -18,11 +18,21 @@ import paddle from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import GroupShardedOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import GroupShardedStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import GroupShardedScaler - -from dygraph_group_sharded_stage2 import MLP, reader_decorator, optimizer_setting +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import ( + GroupShardedOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import ( + GroupShardedStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import ( + GroupShardedScaler, +) + +from dygraph_group_sharded_stage2 import ( + MLP, + reader_decorator, + optimizer_setting, +) seed = 2021 epoch = 2 @@ -40,20 +50,22 @@ def train_mlp(model, offload=False): scaler = paddle.amp.GradScaler(init_loss_scaling=1024) scaler = GroupShardedScaler(scaler) - optimizer = GroupShardedOptimizerStage2(params=optimizer._parameter_list, - optim=optimizer, - offload=offload) + optimizer = GroupShardedOptimizerStage2( + params=optimizer._parameter_list, optim=optimizer, offload=offload + ) model = GroupShardedStage2(model, optimizer, buffer_max_size=2**21) - train_reader = paddle.batch(reader_decorator(linear_size), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(linear_size), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -66,8 +78,9 @@ def train_mlp(model, offload=False): with paddle.amp.auto_cast(True, level='O2'): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) scaler.scale(avg_loss).backward() @@ -93,10 +106,12 @@ def test_sharding_stage2_offload(): mlp_offload_params = train_mlp(mlp_offload, offload=True) for i in range(len(mlp_params)): - np.testing.assert_allclose(mlp_params[i].numpy(), - mlp_offload_params[i].numpy(), - rtol=5e-3, - atol=5e-3) + np.testing.assert_allclose( + mlp_params[i].numpy(), + mlp_offload_params[i].numpy(), + rtol=5e-3, + atol=5e-3, + ) return diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3.py index fe629541376d2..d6f8ef5efb772 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3.py @@ -23,10 +23,18 @@ from paddle.fluid.dygraph.nn import Linear from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import GroupShardedOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import GroupShardedStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import GroupShardedStage3 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import GroupShardedScaler +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import ( + GroupShardedOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import ( + GroupShardedStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import ( + GroupShardedStage3, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import ( + GroupShardedScaler, +) epoch = 10 paddle.seed(2022) @@ -37,7 +45,6 @@ class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -53,7 +60,6 @@ def forward(self, inputs): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -66,53 +72,58 @@ def __reader__(): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) optimizer = paddle.optimizer.Momentum( - parameters=[{ - "params": list(model.parameters()) - }] if opt_group else list(model.parameters()), + parameters=[{"params": list(model.parameters())}] + if opt_group + else list(model.parameters()), learning_rate=0.001, weight_decay=0.00001, grad_clip=clip, - multi_precision=use_pure_fp16) + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - sharding_stage, - use_pure_fp16=False, - accumulate_grad=False, - batch_size=100, - opt_group=False, - sync_comm=False, - test_minimize=False, - save_model=False): +def train_mlp( + model, + sharding_stage, + use_pure_fp16=False, + accumulate_grad=False, + batch_size=100, + opt_group=False, + sync_comm=False, + test_minimize=False, + save_model=False, +): group = paddle.distributed.new_group([0, 1]) if opt_group: - optimizer = optimizer_setting(model=model, - use_pure_fp16=use_pure_fp16, - opt_group=opt_group) + optimizer = optimizer_setting( + model=model, use_pure_fp16=use_pure_fp16, opt_group=opt_group + ) else: optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if use_pure_fp16: - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=32768) scaler = GroupShardedScaler(scaler) if sharding_stage == 2: optimizer = GroupShardedOptimizerStage2( - params=optimizer._parameter_list, optim=optimizer, group=group) - model = GroupShardedStage2(model, - optimizer, - group=group, - buffer_max_size=2**21) + params=optimizer._parameter_list, optim=optimizer, group=group + ) + model = GroupShardedStage2( + model, optimizer, group=group, buffer_max_size=2**21 + ) elif sharding_stage == 3: - model = GroupShardedStage3(model, - optimizer=optimizer, - group=group, - sync_comm=sync_comm, - segment_size=2**15) + model = GroupShardedStage3( + model, + optimizer=optimizer, + group=group, + sync_comm=sync_comm, + segment_size=2**15, + ) # check optimizer.minimize() error if test_minimize: @@ -120,18 +131,21 @@ def train_mlp(model, optimizer.minimize() except: print( - "====== Find sharding_stage3_optimizer.minimize() error ======") + "====== Find sharding_stage3_optimizer.minimize() error ======" + ) return - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -142,8 +156,9 @@ def train_mlp(model, img.stop_gradient = True with paddle.amp.auto_cast(True, level='O2'): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) if batch_size == 20: @@ -178,8 +193,19 @@ def train_mlp(model, def test_stage2_stage3(): paddle.distributed.init_parallel_env() - mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6, mlp7, mlp8, mlp9, mlp10 = MLP( - ), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP() + mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6, mlp7, mlp8, mlp9, mlp10 = ( + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + ) state_dict = mlp.state_dict() mlp1.set_state_dict(state_dict) mlp2.set_state_dict(state_dict) @@ -193,78 +219,87 @@ def test_stage2_stage3(): mlp10.set_state_dict(state_dict) # fp32 - stage2_params = train_mlp(mlp1, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False) - stage3_params = train_mlp(mlp2, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False) + stage2_params = train_mlp( + mlp1, sharding_stage=2, use_pure_fp16=False, opt_group=False + ) + stage3_params = train_mlp( + mlp2, sharding_stage=3, use_pure_fp16=False, opt_group=False + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-6, - atol=1e-6) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-6, + atol=1e-6, + ) # fp32 accumulate grad - stage3_params = train_mlp(mlp3, - sharding_stage=3, - use_pure_fp16=False, - accumulate_grad=True, - opt_group=True) - stage3_params_add = train_mlp(mlp4, - sharding_stage=3, - use_pure_fp16=False, - accumulate_grad=True, - batch_size=20, - opt_group=True) + stage3_params = train_mlp( + mlp3, + sharding_stage=3, + use_pure_fp16=False, + accumulate_grad=True, + opt_group=True, + ) + stage3_params_add = train_mlp( + mlp4, + sharding_stage=3, + use_pure_fp16=False, + accumulate_grad=True, + batch_size=20, + opt_group=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_add[i].numpy(), - rtol=1e-6, - atol=1e-4) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_add[i].numpy(), + rtol=1e-6, + atol=1e-4, + ) # fp16 - stage2_params = train_mlp(mlp5, - sharding_stage=2, - use_pure_fp16=True, - opt_group=False) - stage3_params = train_mlp(mlp6, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False) + stage2_params = train_mlp( + mlp5, sharding_stage=2, use_pure_fp16=True, opt_group=False + ) + stage3_params = train_mlp( + mlp6, sharding_stage=3, use_pure_fp16=True, opt_group=False + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-4, - atol=1e-3) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-4, + atol=1e-3, + ) # fp16 sync_comm - stage3_params = train_mlp(mlp7, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False) - stage3_params_re = train_mlp(mlp8, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False, - sync_comm=True) + stage3_params = train_mlp( + mlp7, sharding_stage=3, use_pure_fp16=True, opt_group=False + ) + stage3_params_re = train_mlp( + mlp8, + sharding_stage=3, + use_pure_fp16=True, + opt_group=False, + sync_comm=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_re[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + stage3_params[i].numpy(), stage3_params_re[i].numpy(), rtol=1e-6 + ) # save/load model output_dir = tempfile.mkdtemp() model_file = os.path.join(output_dir, "model.pdmodel") optimizer_file = os.path.join(output_dir, "model.pdopt") - model_stage3, optimizer_stage3 = train_mlp(mlp9, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False, - save_model=True) + model_stage3, optimizer_stage3 = train_mlp( + mlp9, + sharding_stage=3, + use_pure_fp16=False, + opt_group=False, + save_model=True, + ) paddle.save(model_stage3.state_dict(), model_file) paddle.save(optimizer_stage3.state_dict(), optimizer_file) m_state_dict = paddle.load(model_file) @@ -274,11 +309,13 @@ def test_stage2_stage3(): shutil.rmtree(output_dir) # check optimizer.minimize() error - train_mlp(mlp10, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False, - test_minimize=True) + train_mlp( + mlp10, + sharding_stage=3, + use_pure_fp16=False, + opt_group=False, + test_minimize=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3_offload.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3_offload.py index 2c586b56946f2..0c63d2e1d8a30 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3_offload.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_group_sharded_stage3_offload.py @@ -20,8 +20,12 @@ from paddle.fluid.dygraph.nn import Linear from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import GroupShardedStage3 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import GroupShardedScaler +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import ( + GroupShardedStage3, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import ( + GroupShardedScaler, +) epoch = 10 paddle.seed(2022) @@ -32,7 +36,6 @@ class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -48,7 +51,6 @@ def forward(self, inputs): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -60,48 +62,56 @@ def __reader__(): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) - optimizer = paddle.optimizer.AdamW(parameters=[{ - "params": model.parameters() - }] if opt_group else model.parameters(), - learning_rate=0.001, - weight_decay=0.00001, - grad_clip=clip, - multi_precision=use_pure_fp16) + optimizer = paddle.optimizer.AdamW( + parameters=[{"params": model.parameters()}] + if opt_group + else model.parameters(), + learning_rate=0.001, + weight_decay=0.00001, + grad_clip=clip, + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - use_pure_fp16=False, - accumulate_grad=False, - offload=False, - batch_size=100, - convert2cpu=False): +def train_mlp( + model, + use_pure_fp16=False, + accumulate_grad=False, + offload=False, + batch_size=100, + convert2cpu=False, +): group = paddle.distributed.new_group([0, 1]) optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if use_pure_fp16: - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=32768) scaler = GroupShardedScaler(scaler) - model = GroupShardedStage3(model, - optimizer=optimizer, - group=group, - offload=offload, - segment_size=2**15) - - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + model = GroupShardedStage3( + model, + optimizer=optimizer, + group=group, + offload=offload, + segment_size=2**15, + ) + + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -112,8 +122,9 @@ def train_mlp(model, img.stop_gradient = True with paddle.amp.auto_cast(True, level='O2'): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) if accumulate_grad: @@ -147,8 +158,15 @@ def train_mlp(model, def test_stage3_offload(): paddle.distributed.init_parallel_env() - mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6 = MLP(), MLP(), MLP(), MLP(), MLP( - ), MLP(), MLP() + mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6 = ( + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + ) state_dict = mlp.state_dict() mlp1.set_state_dict(state_dict) mlp2.set_state_dict(state_dict) @@ -161,36 +179,43 @@ def test_stage3_offload(): stage3_params = train_mlp(mlp1, use_pure_fp16=False) stage3_params_offload = train_mlp(mlp2, use_pure_fp16=False, offload=True) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_offload[i].numpy(), - rtol=1e-6, - atol=1e-8) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_offload[i].numpy(), + rtol=1e-6, + atol=1e-8, + ) # fp16 offload stage3_params = train_mlp(mlp3, use_pure_fp16=True) stage3_params_offload = train_mlp(mlp4, use_pure_fp16=True, offload=True) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_offload[i].numpy(), - rtol=1e-2, - atol=1e-2) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_offload[i].numpy(), + rtol=1e-2, + atol=1e-2, + ) # fp32 accumulate grad offload - stage3_params = train_mlp(mlp5, - use_pure_fp16=False, - batch_size=20, - accumulate_grad=True) - stage3_params_offload = train_mlp(mlp6, - use_pure_fp16=False, - accumulate_grad=True, - offload=True, - batch_size=20, - convert2cpu=True) + stage3_params = train_mlp( + mlp5, use_pure_fp16=False, batch_size=20, accumulate_grad=True + ) + stage3_params_offload = train_mlp( + mlp6, + use_pure_fp16=False, + accumulate_grad=True, + offload=True, + batch_size=20, + convert2cpu=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_offload[i].numpy(), - rtol=1e-6, - atol=1e-8) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_offload[i].numpy(), + rtol=1e-6, + atol=1e-8, + ) return diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_optimizer_stage2.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_optimizer_stage2.py index 2f5a73830d706..78dbd9b2ccdd7 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_optimizer_stage2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_optimizer_stage2.py @@ -22,7 +22,9 @@ from paddle.fluid.framework import _test_eager_guard from paddle.distributed.fleet.utils.internal_storage import GradStorage -from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ShardingOptimizerStage2 +from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ( + ShardingOptimizerStage2, +) base_lr = 0.1 momentum_rate = 0.9 @@ -34,7 +36,6 @@ class MLP(fluid.Layer): - def __init__(self, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -48,7 +49,6 @@ def forward(self, inputs): def reader_decorator(): - def __reader__(): for _ in range(100): img = np.random.rand(10).astype('float32') @@ -63,7 +63,8 @@ def optimizer_setting(parameter_list=None): learning_rate=base_lr, momentum=momentum_rate, weight_decay=paddle.regularizer.L2Decay(l2_decay), - parameters=parameter_list) + parameters=parameter_list, + ) return optimizer @@ -74,18 +75,20 @@ def train_mlp(): mlp = MLP() optimizer = optimizer_setting(parameter_list=mlp.parameters()) - oss_optimizer = ShardingOptimizerStage2(params=mlp.parameters(), - optim=optimizer, - group=group) + oss_optimizer = ShardingOptimizerStage2( + params=mlp.parameters(), optim=optimizer, group=group + ) # cover grad_storage code trainable_param2align = dict() for p in mlp.parameters(): trainable_param2align[p.name] = 0 - grad_storage = GradStorage(10000, - dtype=paddle.float32, - device="gpu", - destination=0, - parm2align=trainable_param2align) + grad_storage = GradStorage( + 10000, + dtype=paddle.float32, + device="gpu", + destination=0, + parm2align=trainable_param2align, + ) for p in mlp.parameters(): grad_storage.can_add_grad_view(p, trainable_param2align[p.name]) grad_storage.add_grad(p, trainable_param2align[p.name]) @@ -93,15 +96,17 @@ def train_mlp(): grad_storage.rebuild() grad_storage.reset_checked_in() - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2.py index a025030af2d6c..2303efccd02e1 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2.py @@ -24,8 +24,12 @@ from paddle.distributed import fleet from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ShardingOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ShardingStage2 +from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ( + ShardingOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ( + ShardingStage2, +) seed = 2022 epoch = 2 @@ -36,7 +40,7 @@ "dp_degree": 2, "mp_degree": 1, "pp_degree": 1, - "sharding_degree": 1 + "sharding_degree": 1, } np.random.seed(seed) @@ -44,7 +48,6 @@ class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -60,7 +63,6 @@ def forward(self, inputs): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -72,58 +74,63 @@ def __reader__(): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) - optimizer = paddle.optimizer.AdamW(parameters=[{ - "params": model.parameters() - }] if opt_group else model.parameters(), - learning_rate=0.001, - weight_decay=0.00001, - grad_clip=clip, - multi_precision=use_pure_fp16) + optimizer = paddle.optimizer.AdamW( + parameters=[{"params": model.parameters()}] + if opt_group + else model.parameters(), + learning_rate=0.001, + weight_decay=0.00001, + grad_clip=clip, + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - sharding_stage, - batch_size=100, - use_pure_fp16=False, - accumulate_grad=False, - opt_group=False, - save_model=False): +def train_mlp( + model, + sharding_stage, + batch_size=100, + use_pure_fp16=False, + accumulate_grad=False, + opt_group=False, + save_model=False, +): if sharding_stage == "dp": hcg = fleet.get_hybrid_communicate_group() group = hcg.get_check_parallel_group() else: group = paddle.distributed.new_group([0, 1]) if opt_group: - optimizer = optimizer_setting(model=model, - use_pure_fp16=use_pure_fp16, - opt_group=opt_group) + optimizer = optimizer_setting( + model=model, use_pure_fp16=use_pure_fp16, opt_group=opt_group + ) else: optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if sharding_stage == 2: - optimizer = ShardingOptimizerStage2(params=model.parameters(), - optim=optimizer, - group=group) - - model = ShardingStage2(model, - optimizer, - group=group, - buffer_max_size=2**21) + optimizer = ShardingOptimizerStage2( + params=model.parameters(), optim=optimizer, group=group + ) + + model = ShardingStage2( + model, optimizer, group=group, buffer_max_size=2**21 + ) else: optimizer = fleet.distributed_optimizer(optimizer) model = fleet.distributed_model(model) - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) if sharding_stage == 2: @@ -175,50 +182,50 @@ def test_dp_stage2(): mlp6.set_state_dict(state_dict) # DP VS stage2 - dp_params = train_mlp(mlp1, - sharding_stage="dp", - use_pure_fp16=False, - opt_group=False) - stage2_params = train_mlp(mlp2, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False) + dp_params = train_mlp( + mlp1, sharding_stage="dp", use_pure_fp16=False, opt_group=False + ) + stage2_params = train_mlp( + mlp2, sharding_stage=2, use_pure_fp16=False, opt_group=False + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6 + ) # stage2 accumulate grad stage2_params = train_mlp(mlp3, sharding_stage=2, accumulate_grad=True) - stage2_accumulate_grad = train_mlp(mlp4, - sharding_stage=2, - batch_size=20, - accumulate_grad=True) + stage2_accumulate_grad = train_mlp( + mlp4, sharding_stage=2, batch_size=20, accumulate_grad=True + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage2_accumulate_grad[i].numpy(), - rtol=1e-5, - atol=1e-5) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage2_accumulate_grad[i].numpy(), + rtol=1e-5, + atol=1e-5, + ) # stage2 param list VS param group - stage2_params = train_mlp(mlp5, - sharding_stage=2, - use_pure_fp16=False, - opt_group=True) + stage2_params = train_mlp( + mlp5, sharding_stage=2, use_pure_fp16=False, opt_group=True + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6 + ) # save/load model output_dir = tempfile.mkdtemp() model_file = os.path.join(output_dir, "model.pdmodel") optimizer_file = os.path.join(output_dir, "model.pdopt") - model_stage2, optimizer_stage2 = train_mlp(mlp6, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False, - save_model=True) + model_stage2, optimizer_stage2 = train_mlp( + mlp6, + sharding_stage=2, + use_pure_fp16=False, + opt_group=False, + save_model=True, + ) paddle.save(model_stage2.state_dict(), model_file) paddle.save(optimizer_stage2.state_dict(), optimizer_file) m_state_dict = paddle.load(model_file) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2_offload.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2_offload.py index 7469591c9ed5b..ce795de881c7b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2_offload.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage2_offload.py @@ -19,9 +19,15 @@ from paddle.distributed import fleet from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ShardingOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ShardingStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ShardingScaler +from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ( + ShardingOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ( + ShardingStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ( + ShardingScaler, +) from dygraph_sharding_stage2 import MLP, reader_decorator, optimizer_setting @@ -35,7 +41,7 @@ "dp_degree": 2, "mp_degree": 1, "pp_degree": 1, - "sharding_degree": 1 + "sharding_degree": 1, } np.random.seed(seed) @@ -49,20 +55,22 @@ def train_mlp(model, offload=False): scaler = paddle.amp.GradScaler(init_loss_scaling=1024) scaler = ShardingScaler(scaler) - optimizer = ShardingOptimizerStage2(params=model.parameters(), - optim=optimizer, - offload=offload) + optimizer = ShardingOptimizerStage2( + params=model.parameters(), optim=optimizer, offload=offload + ) model = ShardingStage2(model, optimizer, buffer_max_size=2**21) - train_reader = paddle.batch(reader_decorator(linear_size), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(linear_size), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -75,8 +83,9 @@ def train_mlp(model, offload=False): with paddle.amp.auto_cast(True, level='O2'): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) scaler.scale(avg_loss).backward() @@ -101,10 +110,12 @@ def test_sharding_stage2_offload(): mlp_offload_params = train_mlp(mlp_offload, offload=True) for i in range(len(mlp_params)): - np.testing.assert_allclose(mlp_params[i].numpy(), - mlp_offload_params[i].numpy(), - rtol=5e-3, - atol=5e-3) + np.testing.assert_allclose( + mlp_params[i].numpy(), + mlp_offload_params[i].numpy(), + rtol=5e-3, + atol=5e-3, + ) return diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3.py index 4e9bc7a1cace6..c5d58934060f6 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3.py @@ -24,10 +24,18 @@ from paddle.distributed import fleet from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ShardingOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ShardingStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage3 import ShardingStage3 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ShardingScaler +from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ( + ShardingOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ( + ShardingStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage3 import ( + ShardingStage3, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ( + ShardingScaler, +) epoch = 10 paddle.seed(2021) @@ -38,7 +46,6 @@ class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -54,7 +61,6 @@ def forward(self, inputs): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -67,53 +73,54 @@ def __reader__(): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) optimizer = paddle.optimizer.Momentum( - parameters=[{ - "params": list(model.parameters()) - }] if opt_group else list(model.parameters()), + parameters=[{"params": list(model.parameters())}] + if opt_group + else list(model.parameters()), learning_rate=0.001, weight_decay=0.00001, grad_clip=clip, - multi_precision=use_pure_fp16) + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - sharding_stage, - use_pure_fp16=False, - accumulate_grad=False, - batch_size=100, - opt_group=False, - sync_comm=False, - test_minimize=False, - save_model=False): +def train_mlp( + model, + sharding_stage, + use_pure_fp16=False, + accumulate_grad=False, + batch_size=100, + opt_group=False, + sync_comm=False, + test_minimize=False, + save_model=False, +): group = paddle.distributed.new_group([0, 1]) if opt_group: - optimizer = optimizer_setting(model=model, - use_pure_fp16=use_pure_fp16, - opt_group=opt_group) + optimizer = optimizer_setting( + model=model, use_pure_fp16=use_pure_fp16, opt_group=opt_group + ) else: optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if use_pure_fp16: - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=32768) scaler = ShardingScaler(scaler) if sharding_stage == 2: - optimizer = ShardingOptimizerStage2(params=model.parameters(), - optim=optimizer, - group=group) - model = ShardingStage2(model, - optimizer, - group=group, - buffer_max_size=2**21) + optimizer = ShardingOptimizerStage2( + params=model.parameters(), optim=optimizer, group=group + ) + model = ShardingStage2( + model, optimizer, group=group, buffer_max_size=2**21 + ) elif sharding_stage == 3: - model = ShardingStage3(model, - optimizer=optimizer, - group=group, - sync_comm=sync_comm) + model = ShardingStage3( + model, optimizer=optimizer, group=group, sync_comm=sync_comm + ) # check optimizer.minimize() error if test_minimize: @@ -121,18 +128,21 @@ def train_mlp(model, optimizer.minimize() except: print( - "====== Find sharding_stage3_optimizer.minimize() error ======") + "====== Find sharding_stage3_optimizer.minimize() error ======" + ) return - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -143,8 +153,9 @@ def train_mlp(model, img.stop_gradient = True with paddle.amp.auto_cast(True, level='O2'): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) if batch_size == 20: @@ -178,8 +189,19 @@ def train_mlp(model, def test_stage2_stage3(): - mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6, mlp7, mlp8, mlp9, mlp10 = MLP( - ), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP() + mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6, mlp7, mlp8, mlp9, mlp10 = ( + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + ) state_dict = mlp.state_dict() mlp1.set_state_dict(state_dict) mlp2.set_state_dict(state_dict) @@ -193,78 +215,87 @@ def test_stage2_stage3(): mlp10.set_state_dict(state_dict) # fp32 - stage2_params = train_mlp(mlp1, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False) - stage3_params = train_mlp(mlp2, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False) + stage2_params = train_mlp( + mlp1, sharding_stage=2, use_pure_fp16=False, opt_group=False + ) + stage3_params = train_mlp( + mlp2, sharding_stage=3, use_pure_fp16=False, opt_group=False + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-6, - atol=1e-6) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-6, + atol=1e-6, + ) # fp32 accumulate grad - stage3_params = train_mlp(mlp3, - sharding_stage=3, - use_pure_fp16=False, - accumulate_grad=True, - opt_group=True) - stage3_params_add = train_mlp(mlp4, - sharding_stage=3, - use_pure_fp16=False, - accumulate_grad=True, - batch_size=20, - opt_group=True) + stage3_params = train_mlp( + mlp3, + sharding_stage=3, + use_pure_fp16=False, + accumulate_grad=True, + opt_group=True, + ) + stage3_params_add = train_mlp( + mlp4, + sharding_stage=3, + use_pure_fp16=False, + accumulate_grad=True, + batch_size=20, + opt_group=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_add[i].numpy(), - rtol=1e-6, - atol=1e-4) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_add[i].numpy(), + rtol=1e-6, + atol=1e-4, + ) # fp16 - stage2_params = train_mlp(mlp5, - sharding_stage=2, - use_pure_fp16=True, - opt_group=False) - stage3_params = train_mlp(mlp6, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False) + stage2_params = train_mlp( + mlp5, sharding_stage=2, use_pure_fp16=True, opt_group=False + ) + stage3_params = train_mlp( + mlp6, sharding_stage=3, use_pure_fp16=True, opt_group=False + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-4, - atol=1e-3) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-4, + atol=1e-3, + ) # fp16 sync_comm - stage3_params = train_mlp(mlp7, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False) - stage3_params_re = train_mlp(mlp8, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False, - sync_comm=True) + stage3_params = train_mlp( + mlp7, sharding_stage=3, use_pure_fp16=True, opt_group=False + ) + stage3_params_re = train_mlp( + mlp8, + sharding_stage=3, + use_pure_fp16=True, + opt_group=False, + sync_comm=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_re[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + stage3_params[i].numpy(), stage3_params_re[i].numpy(), rtol=1e-6 + ) # save/load model output_dir = tempfile.mkdtemp() model_file = os.path.join(output_dir, "model.pdmodel") optimizer_file = os.path.join(output_dir, "model.pdopt") - model_stage3, optimizer_stage3 = train_mlp(mlp9, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False, - save_model=True) + model_stage3, optimizer_stage3 = train_mlp( + mlp9, + sharding_stage=3, + use_pure_fp16=False, + opt_group=False, + save_model=True, + ) paddle.save(model_stage3.state_dict(), model_file) paddle.save(optimizer_stage3.state_dict(), optimizer_file) m_state_dict = paddle.load(model_file) @@ -274,11 +305,13 @@ def test_stage2_stage3(): shutil.rmtree(output_dir) # check optimizer.minimize() error - train_mlp(mlp10, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False, - test_minimize=True) + train_mlp( + mlp10, + sharding_stage=3, + use_pure_fp16=False, + opt_group=False, + test_minimize=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3_offload.py b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3_offload.py index d56ec33694b07..f7eee14046d7b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3_offload.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/dygraph_sharding_stage3_offload.py @@ -21,8 +21,12 @@ from paddle.distributed import fleet from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage3 import ShardingStage3 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ShardingScaler +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage3 import ( + ShardingStage3, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_utils import ( + ShardingScaler, +) epoch = 10 paddle.seed(2022) @@ -33,7 +37,6 @@ class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -49,7 +52,6 @@ def forward(self, inputs): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -61,47 +63,52 @@ def __reader__(): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) - optimizer = paddle.optimizer.AdamW(parameters=[{ - "params": model.parameters() - }] if opt_group else model.parameters(), - learning_rate=0.001, - weight_decay=0.00001, - grad_clip=clip, - multi_precision=use_pure_fp16) + optimizer = paddle.optimizer.AdamW( + parameters=[{"params": model.parameters()}] + if opt_group + else model.parameters(), + learning_rate=0.001, + weight_decay=0.00001, + grad_clip=clip, + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - use_pure_fp16=False, - accumulate_grad=False, - offload=False, - batch_size=100, - convert2cpu=False): +def train_mlp( + model, + use_pure_fp16=False, + accumulate_grad=False, + offload=False, + batch_size=100, + convert2cpu=False, +): group = paddle.distributed.new_group([0, 1]) optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if use_pure_fp16: - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=32768) scaler = ShardingScaler(scaler) - model = ShardingStage3(model, - optimizer=optimizer, - group=group, - offload=offload) - - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + model = ShardingStage3( + model, optimizer=optimizer, group=group, offload=offload + ) + + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -112,8 +119,9 @@ def train_mlp(model, img.stop_gradient = True with paddle.amp.auto_cast(True, level='O2'): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) if accumulate_grad: @@ -146,8 +154,15 @@ def train_mlp(model, def test_stage3_offload(): - mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6 = MLP(), MLP(), MLP(), MLP(), MLP( - ), MLP(), MLP() + mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6 = ( + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + ) state_dict = mlp.state_dict() mlp1.set_state_dict(state_dict) mlp2.set_state_dict(state_dict) @@ -160,36 +175,43 @@ def test_stage3_offload(): stage3_params = train_mlp(mlp1, use_pure_fp16=False) stage3_params_offload = train_mlp(mlp2, use_pure_fp16=False, offload=True) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_offload[i].numpy(), - rtol=1e-6, - atol=1e-8) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_offload[i].numpy(), + rtol=1e-6, + atol=1e-8, + ) # fp16 offload stage3_params = train_mlp(mlp3, use_pure_fp16=True) stage3_params_offload = train_mlp(mlp4, use_pure_fp16=True, offload=True) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_offload[i].numpy(), - rtol=1e-2, - atol=1e-2) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_offload[i].numpy(), + rtol=1e-2, + atol=1e-2, + ) # fp32 accumulate grad offload - stage3_params = train_mlp(mlp5, - use_pure_fp16=False, - batch_size=20, - accumulate_grad=True) - stage3_params_offload = train_mlp(mlp6, - use_pure_fp16=False, - accumulate_grad=True, - offload=True, - batch_size=20, - convert2cpu=True) + stage3_params = train_mlp( + mlp5, use_pure_fp16=False, batch_size=20, accumulate_grad=True + ) + stage3_params_offload = train_mlp( + mlp6, + use_pure_fp16=False, + accumulate_grad=True, + offload=True, + batch_size=20, + convert2cpu=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_offload[i].numpy(), - rtol=1e-6, - atol=1e-8) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_offload[i].numpy(), + rtol=1e-6, + atol=1e-8, + ) return diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_communicate_group.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_communicate_group.py index 0e81b08a7ebd7..0c6bf6e9decce 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_communicate_group.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_communicate_group.py @@ -18,11 +18,11 @@ class TestNewGroupAPI(object): - def __init__(self): paddle.distributed.init_parallel_env() - topo = fleet.CommunicateTopology(["data", "model", "sharding", "pipe"], - [2, 1, 1, 1]) + topo = fleet.CommunicateTopology( + ["data", "model", "sharding", "pipe"], [2, 1, 1, 1] + ) self.hcg = fleet.HybridCommunicateGroup(topo) d1 = np.array([1, 2, 3]) @@ -49,10 +49,13 @@ def test_all(self): tmp = np.array([0, 0, 0]) result = paddle.to_tensor(tmp) - paddle.distributed.scatter(result, [self.tensor2, self.tensor1], - src=dp_src_rank, - group=dp_gp, - sync_op=True) + paddle.distributed.scatter( + result, + [self.tensor2, self.tensor1], + src=dp_src_rank, + group=dp_gp, + sync_op=True, + ) if dp_rank == 0: assert np.array_equal(result, self.tensor2) elif dp_rank == 1: @@ -63,13 +66,13 @@ def test_all(self): assert np.array_equal(result, self.tensor1) print("test broadcast api ok") - paddle.distributed.reduce(result, - dst=dp_src_rank, - group=dp_gp, - sync_op=True) + paddle.distributed.reduce( + result, dst=dp_src_rank, group=dp_gp, sync_op=True + ) if dp_rank == 0: - assert np.array_equal(result, paddle.add(self.tensor1, - self.tensor1)) + assert np.array_equal( + result, paddle.add(self.tensor1, self.tensor1) + ) elif dp_rank == 1: assert np.array_equal(result, self.tensor1) print("test reduce api ok") @@ -77,7 +80,8 @@ def test_all(self): paddle.distributed.all_reduce(result, sync_op=True) assert np.array_equal( result, - paddle.add(paddle.add(self.tensor1, self.tensor1), self.tensor1)) + paddle.add(paddle.add(self.tensor1, self.tensor1), self.tensor1), + ) print("test all_reduce api ok") paddle.distributed.wait(result, dp_gp, use_calc_stream=True) @@ -85,10 +89,9 @@ def test_all(self): print("test wait api ok") result = [] - paddle.distributed.all_gather(result, - self.tensor1, - group=dp_gp, - sync_op=True) + paddle.distributed.all_gather( + result, self.tensor1, group=dp_gp, sync_op=True + ) assert np.array_equal(result[0], self.tensor1) assert np.array_equal(result[1], self.tensor1) print("test all_gather api ok") diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_inference_helper.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_inference_helper.py index 83b0aaf76c888..e746f30fa57a2 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_inference_helper.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_inference_helper.py @@ -19,7 +19,9 @@ import numpy as np import paddle.fluid.layers as layers import paddle.distributed.fleet as fleet -from paddle.distributed.fleet.utils.hybrid_parallel_inference import HybridParallelInferenceHelper +from paddle.distributed.fleet.utils.hybrid_parallel_inference import ( + HybridParallelInferenceHelper, +) paddle.enable_static() @@ -40,7 +42,6 @@ def numpy_while(x, w1=1.0, w2=2.0, max_len=2): class TestHybridParallelInferenceHelperClass(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() fleet.init(is_collective=True, strategy=strategy) @@ -59,29 +60,27 @@ def test_hybrid_parallel_inference_helper_mp1pp2(self): with paddle.static.program_guard(main_program, startup_program): with paddle.fluid.device_guard(f'{device}:0'): - X = paddle.static.data(name='X', - shape=[None, 2], - dtype='float32') + X = paddle.static.data( + name='X', shape=[None, 2], dtype='float32' + ) with paddle.fluid.device_guard(f'{device}:all'): - max_len = layers.fill_constant(shape=[1], - dtype="int64", - value=2, - force_cpu=False, - name="n") - step_idx = layers.fill_constant(shape=[1], - dtype="int64", - value=0, - force_cpu=False, - name="i") + max_len = layers.fill_constant( + shape=[1], dtype="int64", value=2, force_cpu=False, name="n" + ) + step_idx = layers.fill_constant( + shape=[1], dtype="int64", value=0, force_cpu=False, name="i" + ) data = layers.array_write(X, step_idx) - cond_int = layers.fill_constant(shape=[1], - dtype="int64", - value=0, - force_cpu=False, - name="cond_int") + cond_int = layers.fill_constant( + shape=[1], + dtype="int64", + value=0, + force_cpu=False, + name="cond_int", + ) print(cond_int.shape) cond = layers.less_than(x=step_idx, y=max_len) while_op = layers.While(cond, is_test=True) @@ -94,20 +93,26 @@ def test_hybrid_parallel_inference_helper_mp1pp2(self): with paddle.fluid.device_guard(f'{device}:0'): param_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Constant(1.0)) - weight1 = paddle.static.create_parameter(shape=[2, 5], - dtype='float32', - attr=param_attr, - is_bias=False) + initializer=paddle.nn.initializer.Constant(1.0) + ) + weight1 = paddle.static.create_parameter( + shape=[2, 5], + dtype='float32', + attr=param_attr, + is_bias=False, + ) hidden1 = paddle.matmul(input, weight1) with paddle.fluid.device_guard(f'{device}:1'): param_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Constant(2.0)) - weight2 = paddle.static.create_parameter(shape=[5, 2], - dtype='float32', - attr=param_attr, - is_bias=False) + initializer=paddle.nn.initializer.Constant(2.0) + ) + weight2 = paddle.static.create_parameter( + shape=[5, 2], + dtype='float32', + attr=param_attr, + is_bias=False, + ) hidden2 = paddle.matmul(hidden1, weight2) layers.array_write(hidden2, i=step_idx, array=data) @@ -136,18 +141,20 @@ def test_hybrid_parallel_inference_helper_mp1pp2(self): num_pp=2, init_comm=nranks > 1, ) - helper.gen_infer_program(['array_write_0.out'], ['cond_int.tmp_0'], - debug=True) + helper.gen_infer_program( + ['array_write_0.out'], ['cond_int.tmp_0'], debug=True + ) exe = paddle.static.Executor(paddle.CUDAPlace(dev_id)) exe.run(startup_program) for step in range(2): - init_data = np.random.uniform(low=0.0, high=1.0, - size=[2, 2]).astype('float32') - [res] = exe.run(main_program, - feed={"X": init_data}, - fetch_list=[out]) + init_data = np.random.uniform( + low=0.0, high=1.0, size=[2, 2] + ).astype('float32') + [res] = exe.run( + main_program, feed={"X": init_data}, fetch_list=[out] + ) res_np = numpy_while(init_data) assert len(res) == len(res_np) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_amp.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_amp.py index 4906a1c86cac5..172753b2fbc22 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_amp.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_amp.py @@ -19,22 +19,22 @@ class TestMPClipGrad(TestDistMPTraning): - def build_optimizer(self, model): grad_clip = paddle.nn.ClipGradByGlobalNorm(2.0) - scheduler = paddle.optimizer.lr.ExponentialDecay(learning_rate=0.001, - gamma=0.999, - verbose=True) - optimizer = paddle.optimizer.SGD(scheduler, - grad_clip=grad_clip, - parameters=[{ - 'params': - model.parameters(), - 'weight_decay': - 0.001, - 'learning_rate': - 0.1 - }]) + scheduler = paddle.optimizer.lr.ExponentialDecay( + learning_rate=0.001, gamma=0.999, verbose=True + ) + optimizer = paddle.optimizer.SGD( + scheduler, + grad_clip=grad_clip, + parameters=[ + { + 'params': model.parameters(), + 'weight_decay': 0.001, + 'learning_rate': 0.1, + } + ], + ) return optimizer def train_batch(self, batch, model, optimizer, is_mp): diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_clip_grad.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_clip_grad.py index c9bff4a261d20..91d9789bfa06b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_clip_grad.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_clip_grad.py @@ -16,20 +16,19 @@ from hybrid_parallel_mp_model import TestDistMPTraning import unittest -#log = logging.getLogger("HybridParallel") -#log.setLevel(logging.WARNING) +# log = logging.getLogger("HybridParallel") +# log.setLevel(logging.WARNING) class TestMPClipGrad(TestDistMPTraning): - def build_optimizer(self, model): grad_clip = paddle.nn.ClipGradByGlobalNorm(2.0) - scheduler = paddle.optimizer.lr.ExponentialDecay(learning_rate=0.001, - gamma=0.999, - verbose=True) - optimizer = paddle.optimizer.SGD(scheduler, - grad_clip=grad_clip, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.ExponentialDecay( + learning_rate=0.001, gamma=0.999, verbose=True + ) + optimizer = paddle.optimizer.SGD( + scheduler, grad_clip=grad_clip, parameters=model.parameters() + ) return optimizer diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_fp16.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_fp16.py index 611e85ed4a8f5..6a28316c1f485 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_fp16.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_fp16.py @@ -19,20 +19,18 @@ class TestMPFP16(TestDistMPTraning): - def build_optimizer(self, model): grad_clip = paddle.nn.ClipGradByGlobalNorm(1.0) - scheduler = paddle.optimizer.lr.ExponentialDecay(learning_rate=0.001, - gamma=0.999, - verbose=True) - optimizer = paddle.optimizer.SGD(scheduler, - grad_clip=grad_clip, - parameters=model.parameters()) - - model, optimizer = paddle.amp.decorate(models=model, - optimizers=optimizer, - level='O2', - save_dtype='float32') + scheduler = paddle.optimizer.lr.ExponentialDecay( + learning_rate=0.001, gamma=0.999, verbose=True + ) + optimizer = paddle.optimizer.SGD( + scheduler, grad_clip=grad_clip, parameters=model.parameters() + ) + + model, optimizer = paddle.amp.decorate( + models=model, optimizers=optimizer, level='O2', save_dtype='float32' + ) return optimizer diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_layers.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_layers.py index 5f20209da4af8..9a39177127173 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_layers.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_layers.py @@ -31,7 +31,6 @@ def set_random_seed(seed): class ColumnLinearNet(fluid.dygraph.Layer): - def __init__(self, input_size, output_size, global_dtype): super(ColumnLinearNet, self).__init__() self.parallel_linear = fleet.meta_parallel.ColumnParallelLinear( @@ -40,7 +39,8 @@ def __init__(self, input_size, output_size, global_dtype): weight_attr=None, has_bias=True, gather_output=True, - name="test_column_linear") + name="test_column_linear", + ) def forward(self, x): output = self.parallel_linear(x) @@ -48,7 +48,6 @@ def forward(self, x): class RowLinearNet(fluid.dygraph.Layer): - def __init__(self, input_size, output_size): super(RowLinearNet, self).__init__() self.parallel_linear = fleet.meta_parallel.RowParallelLinear( @@ -56,7 +55,8 @@ def __init__(self, input_size, output_size): out_features=output_size, has_bias=True, input_is_parallel=False, - name="test_row_linear") + name="test_row_linear", + ) def forward(self, x): output = self.parallel_linear(x) @@ -64,11 +64,11 @@ def forward(self, x): class EmbeddingNet(fluid.dygraph.Layer): - def __init__(self, vocab_size, hidden_size): super(EmbeddingNet, self).__init__() self.embedding = fleet.meta_parallel.VocabParallelEmbedding( - vocab_size, hidden_size) + vocab_size, hidden_size + ) def forward(self, x): output = self.embedding(x) @@ -76,19 +76,22 @@ def forward(self, x): class SimpleMatmul(fluid.dygraph.Layer): - def __init__(self, weight, output_size, global_dtype): super(SimpleMatmul, self).__init__() self.weight = paddle.create_parameter( shape=weight.shape, dtype=global_dtype, attr=paddle.ParamAttr( - initializer=paddle.nn.initializer.Assign(weight))) + initializer=paddle.nn.initializer.Assign(weight) + ), + ) self.bias = self.create_parameter( shape=[output_size], dtype=global_dtype, attr=paddle.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) def forward(self, x): output = paddle.matmul(x, self.weight) + self.bias @@ -96,7 +99,6 @@ def forward(self, x): class SimpleEmbedding(fluid.dygraph.Layer): - def __init__(self, vocab_size, hidden_size, weight): super(SimpleEmbedding, self).__init__() self.embedding = paddle.nn.Embedding( @@ -104,7 +106,9 @@ def __init__(self, vocab_size, hidden_size, weight): hidden_size, weight_attr=paddle.framework.ParamAttr( name="origin_embedding", - initializer=paddle.nn.initializer.Assign(weight))) + initializer=paddle.nn.initializer.Assign(weight), + ), + ) def forward(self, x): output = self.embedding(x) @@ -112,14 +116,13 @@ def forward(self, x): class TestDistTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 2 strategy.hybrid_configs = { "dp_degree": 1, "mp_degree": self.model_parallel_size, - "pp_degree": 1 + "pp_degree": 1, } fleet.init(is_collective=True, strategy=strategy) @@ -144,10 +147,12 @@ def test_column_parallel_layer(self): model_b = SimpleMatmul(integral_w, output_size, global_dtype) - optimizer_a = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model_a.parameters()) - optimizer_b = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model_b.parameters()) + optimizer_a = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model_a.parameters() + ) + optimizer_b = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model_b.parameters() + ) for idx in range(5): input = paddle.randn([batch_size, input_size], global_dtype) input.stop_gradient = True @@ -192,11 +197,13 @@ def test_row_parallel_layer(self): model_b = SimpleMatmul(integral_w, output_size, global_dtype) - optimizer_a = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model_a.parameters()) + optimizer_a = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model_a.parameters() + ) - optimizer_b = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model_b.parameters()) + optimizer_b = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model_b.parameters() + ) for idx in range(5): input = paddle.randn([batch_size, input_size], global_dtype) @@ -213,9 +220,9 @@ def test_row_parallel_layer(self): optimizer_a.step() optimizer_b.step() - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=5e-6) + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=5e-6 + ) def test_parallel_embedding(self): batch_size = 17 @@ -240,21 +247,25 @@ def test_parallel_embedding(self): for idx in range(len(integral_w)): tmp = paddle.gather( integral_w[idx], - paddle.to_tensor(list(range(vocab_size_per_card)))) + paddle.to_tensor(list(range(vocab_size_per_card))), + ) result_w.append(tmp) integral_w = paddle.concat(result_w, axis=0) model_b = SimpleEmbedding(vocab_size, hidden_size, integral_w) - optimizer_a = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model_a.parameters()) + optimizer_a = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model_a.parameters() + ) - optimizer_b = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model_b.parameters()) + optimizer_b = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model_b.parameters() + ) for _ in range(5): - np_input_data = np.random.randint(0, vocab_size, - (batch_size, seq_length)) + np_input_data = np.random.randint( + 0, vocab_size, (batch_size, seq_length) + ) input_data = paddle.to_tensor(np_input_data, dtype="int32") output_a = model_a(input_data) @@ -292,21 +303,23 @@ def test_parallel_cross_entropy(self): np.random.seed(seed) for _ in range(5): - np_label = np.random.randint(0, vocab_size, - (batch_size, seq_length)) + np_label = np.random.randint( + 0, vocab_size, (batch_size, seq_length) + ) label = paddle.to_tensor(np_label, dtype="int64") data = paddle.randn( shape=[batch_size, seq_length, class_size_per_card], - dtype='float32') + dtype='float32', + ) data.stop_gradient = False check_group = dist.new_group(list(range(self.model_parallel_size))) integral_data = [] partial_data = data.clone().detach() - paddle.distributed.all_gather(integral_data, - partial_data, - group=check_group) + paddle.distributed.all_gather( + integral_data, partial_data, group=check_group + ) integral_data = paddle.concat(integral_data, axis=-1) integral_data = integral_data.detach().clone() integral_data.stop_gradient = False @@ -315,23 +328,23 @@ def test_parallel_cross_entropy(self): loss_b = model_b(integral_data, label).sum() / batch_size print("loss_a: ", loss_a.numpy(), "loss_b: ", loss_b.numpy()) - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=1e-6) + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=1e-6 + ) loss_a.backward() loss_b.backward() integral_grad = [] partial_grad = data.grad.clone().detach() - paddle.distributed.all_gather(integral_grad, - partial_grad, - group=check_group) + paddle.distributed.all_gather( + integral_grad, partial_grad, group=check_group + ) integral_grad = paddle.concat(integral_grad, axis=-1) - np.testing.assert_allclose(integral_data.grad.numpy(), - integral_grad.numpy(), - rtol=1e-6) + np.testing.assert_allclose( + integral_data.grad.numpy(), integral_grad.numpy(), rtol=1e-6 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_model.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_model.py index 707bd95a4902e..b43d0663ac7ea 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_model.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_model.py @@ -44,7 +44,8 @@ def parallel_matmul(lm_output, logit_weights, parallel_output): if world_size > 1: input_parallel = paddle.distributed.collective._c_identity( - lm_output, group=model_parallel_group) + lm_output, group=model_parallel_group + ) logits = paddle.matmul(input_parallel, logit_weights, transpose_y=True) @@ -52,53 +53,69 @@ def parallel_matmul(lm_output, logit_weights, parallel_output): return logits return paddle.distributed.collective._c_concat( - logits, group=model_parallel_group) + logits, group=model_parallel_group + ) else: logits = paddle.matmul(lm_output, logit_weights, transpose_y=True) return logits class SimpleMPNet(fluid.dygraph.Layer): - - def __init__(self, vocab_size, hidden_size, inner_size, output_size, np_fc1, - np_fc2, mp_id): + def __init__( + self, + vocab_size, + hidden_size, + inner_size, + output_size, + np_fc1, + np_fc2, + mp_id, + ): super(SimpleMPNet, self).__init__() if mp_id == 0: - init_fc1_data = np_fc1[:, :(inner_size // 2)] - init_fc2_data = np_fc2[:(inner_size // 2), :] + init_fc1_data = np_fc1[:, : (inner_size // 2)] + init_fc2_data = np_fc2[: (inner_size // 2), :] else: - init_fc1_data = np_fc1[:, (inner_size // 2):] - init_fc2_data = np_fc2[(inner_size // 2):, :] + init_fc1_data = np_fc1[:, (inner_size // 2) :] + init_fc2_data = np_fc2[(inner_size // 2) :, :] self.linear1 = fleet.meta_parallel.ColumnParallelLinear( hidden_size, inner_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(init_fc1_data)), + initializer=paddle.nn.initializer.Assign(init_fc1_data) + ), gather_output=False, - has_bias=True) + has_bias=True, + ) self.linear2 = fleet.meta_parallel.RowParallelLinear( inner_size, hidden_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(init_fc2_data)), + initializer=paddle.nn.initializer.Assign(init_fc2_data) + ), input_is_parallel=True, - has_bias=True) + has_bias=True, + ) self.linear3 = paddle.nn.Linear( hidden_size, output_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0)), + initializer=paddle.nn.initializer.Constant(0.0) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.embedding = fleet.meta_parallel.VocabParallelEmbedding( vocab_size, hidden_size, - weight_attr=paddle.nn.initializer.Constant(value=0.5)) + weight_attr=paddle.nn.initializer.Constant(value=0.5), + ) def forward(self, x): x = self.embedding(x) @@ -110,39 +127,49 @@ def forward(self, x): class SimpleDPNet(fluid.dygraph.Layer): - - def __init__(self, vocab_size, hidden_size, inner_size, output_size, np_fc1, - np_fc2): + def __init__( + self, vocab_size, hidden_size, inner_size, output_size, np_fc1, np_fc2 + ): super(SimpleDPNet, self).__init__() self.linear1 = paddle.nn.Linear( hidden_size, inner_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(np_fc1)), + initializer=paddle.nn.initializer.Assign(np_fc1) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.linear2 = paddle.nn.Linear( inner_size, hidden_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(np_fc2)), + initializer=paddle.nn.initializer.Assign(np_fc2) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.linear3 = paddle.nn.Linear( hidden_size, output_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0)), + initializer=paddle.nn.initializer.Constant(0.0) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.embedding = paddle.nn.Embedding( vocab_size, hidden_size, - weight_attr=paddle.nn.initializer.Constant(value=0.5)) + weight_attr=paddle.nn.initializer.Constant(value=0.5), + ) def forward(self, x): x = self.embedding(x) @@ -154,7 +181,6 @@ def forward(self, x): class TestDistMPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 2 @@ -162,7 +188,7 @@ def setUp(self): strategy.hybrid_configs = { "dp_degree": self.data_parallel_size, "mp_degree": self.model_parallel_size, - "pp_degree": 1 + "pp_degree": 1, } fleet.init(is_collective=True, strategy=strategy) @@ -175,8 +201,9 @@ def train_batch(self, batch, model, optimizer, is_mp): return loss def build_optimizer(self, model): - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return optimizer def build_model_optimizer(self): @@ -190,34 +217,50 @@ def build_model_optimizer(self): np_fc1 = np.random.random_sample((hidden_size, inner_size)) np_fc2 = np.random.random_sample((inner_size, hidden_size)) - model_a = SimpleMPNet(vocab_size, hidden_size, inner_size, output_size, - np_fc1, np_fc2, mp_id) + model_a = SimpleMPNet( + vocab_size, + hidden_size, + inner_size, + output_size, + np_fc1, + np_fc2, + mp_id, + ) optimizer_a = self.build_optimizer(model_a) model_a = fleet.distributed_model(model_a) optimizer_a = fleet.distributed_optimizer(optimizer_a) - model_b = SimpleDPNet(vocab_size, hidden_size, inner_size, output_size, - np_fc1, np_fc2) + model_b = SimpleDPNet( + vocab_size, hidden_size, inner_size, output_size, np_fc1, np_fc2 + ) optimizer_b = self.build_optimizer(model_b) return model_a, optimizer_a, model_b, optimizer_b def test_mp_model(self): - model_a, optimizer_a, model_b, optimizer_b = self.build_model_optimizer( - ) + ( + model_a, + optimizer_a, + model_b, + optimizer_b, + ) = self.build_model_optimizer() for _ in range(5): - np_data = np.random.randint(0, vocab_size, ( - batch_size, - seq_length, - )) + np_data = np.random.randint( + 0, + vocab_size, + ( + batch_size, + seq_length, + ), + ) batch = paddle.to_tensor(np_data) loss_a = self.train_batch(batch, model_a, optimizer_a, True) loss_b = self.train_batch(batch, model_b, optimizer_b, False) - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=1e-6) + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=1e-6 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_random.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_random.py index f779b3298b8dc..96e76b37c7aac 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_random.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_mp_random.py @@ -20,14 +20,13 @@ class TestDistTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 2 strategy.hybrid_configs = { "dp_degree": 1, "mp_degree": self.model_parallel_size, - "pp_degree": 1 + "pp_degree": 1, } fleet.init(is_collective=True, strategy=strategy) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_amp.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_amp.py index 41bd79b58b542..af4e68be8f67b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_amp.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_amp.py @@ -34,7 +34,6 @@ def set_random_seed(seed, dp_id, rank_id): class TestDistPPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -47,7 +46,7 @@ def setUp(self): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -61,14 +60,16 @@ def test_pp_model(self): grad_clip = paddle.nn.ClipGradByGlobalNorm(1.0) - #construct model a + # construct model a model_a = AlexNet(10) - scheduler_a = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer_a = paddle.optimizer.SGD(learning_rate=scheduler_a, - grad_clip=grad_clip, - parameters=model_a.parameters()) + scheduler_a = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer_a = paddle.optimizer.SGD( + learning_rate=scheduler_a, + grad_clip=grad_clip, + parameters=model_a.parameters(), + ) scaler_a = paddle.amp.GradScaler(init_loss_scaling=2**5) @@ -79,12 +80,14 @@ def test_pp_model(self): # construct model b model_b = AlexNetPipeDesc(num_stages=self.pipeline_parallel_size) - scheduler_b = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer_b = paddle.optimizer.SGD(learning_rate=scheduler_b, - grad_clip=grad_clip, - parameters=model_b.parameters()) + scheduler_b = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer_b = paddle.optimizer.SGD( + learning_rate=scheduler_b, + grad_clip=grad_clip, + parameters=model_b.parameters(), + ) model_b = fleet.distributed_model(model_b) optimizer_b = fleet.distributed_optimizer(optimizer_b) scaler_b = paddle.amp.GradScaler(init_loss_scaling=2**5) @@ -94,15 +97,21 @@ def test_pp_model(self): param.set_value(parameters[idx + pp_id * (param_len // 2)]) # construct reader - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size, drop_last=True + ) for step_id, data in enumerate(train_reader()): - x_data = np.array([x[0] for x in data]).astype('float32').reshape( - batch_size, 1, 28, 28) - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(batch_size, 1) + x_data = ( + np.array([x[0] for x in data]) + .astype('float32') + .reshape(batch_size, 1, 28, 28) + ) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(batch_size, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) img.stop_gradient = True @@ -120,15 +129,14 @@ def test_pp_model(self): scheduler_a.step() with paddle.amp.auto_cast(): - loss_b = model_b.train_batch([img, label], - optimizer_b, - scheduler_b, - scaler=scaler_b) + loss_b = model_b.train_batch( + [img, label], optimizer_b, scheduler_b, scaler=scaler_b + ) print("loss: ", loss_a.numpy(), loss_b.numpy()) - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=5e-5) + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=5e-5 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_clip_grad.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_clip_grad.py index 8f1d8e930e2b1..379a5f6417a25 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_clip_grad.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_clip_grad.py @@ -18,31 +18,30 @@ class TestPPClipGrad(TestDistPPTraning): - def build_optimizer(self, model): grad_clip = paddle.nn.ClipGradByGlobalNorm(0.5) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.SGD(learning_rate=scheduler, - grad_clip=grad_clip, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=scheduler, + grad_clip=grad_clip, + parameters=model.parameters(), + ) return scheduler, optimizer class TestPPClipGradParamGroup(TestDistPPTraning): - def build_optimizer(self, model): grad_clip = paddle.nn.ClipGradByGlobalNorm(0.5) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.Momentum(learning_rate=scheduler, - grad_clip=grad_clip, - parameters=[{ - "params": - model.parameters() - }]) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.Momentum( + learning_rate=scheduler, + grad_clip=grad_clip, + parameters=[{"params": model.parameters()}], + ) return scheduler, optimizer diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_embedding.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_embedding.py index d55f989054cec..1c14eb7f67c6a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_embedding.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_embedding.py @@ -40,29 +40,29 @@ def set_random_seed(seed, dp_id, rank_id): class SimpleNet(Layer): - def __init__(self): super(SimpleNet, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, hidden_size) self.softmax_weight = self.create_parameter( - shape=[hidden_size, vocab_size]) - self.softmax_bias = self.create_parameter(shape=[vocab_size], - is_bias=False) + shape=[hidden_size, vocab_size] + ) + self.softmax_bias = self.create_parameter( + shape=[vocab_size], is_bias=False + ) def forward(self, x1, x2, y1): x_emb = self.word_embeddings(x1) fc = fluid.layers.matmul(x_emb, self.softmax_weight) fc = fluid.layers.elementwise_add(fc, self.softmax_bias) projection = fluid.layers.reshape(fc, shape=[-1, vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=y1, - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=y1, soft_label=False + ) return loss.mean() class EmbeddingNet(Layer): - def __init__(self): super(EmbeddingNet, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, hidden_size) @@ -74,11 +74,11 @@ def forward(self, args): class MatmulNet(Layer): - def __init__(self): super(MatmulNet, self).__init__() self.softmax_weight = self.create_parameter( - shape=[hidden_size, vocab_size]) + shape=[hidden_size, vocab_size] + ) def forward(self, args): x1, x2 = args @@ -88,7 +88,6 @@ def forward(self, args): class BiasNet(Layer): - def __init__(self): super(BiasNet, self).__init__() self.softmax_bias = self.create_parameter(shape=[vocab_size]) @@ -101,20 +100,18 @@ def forward(self, args): class LossNet(Layer): - def __init__(self): super(LossNet, self).__init__() def forward(self, args, y1): projection, x2 = args - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=y1[0], - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=y1[0], soft_label=False + ) return loss.mean() class SimpleNetPipe(Layer): - def __init__(self): super(SimpleNetPipe, self).__init__() self.features = Sequential(EmbeddingNet(), MatmulNet(), BiasNet()) @@ -125,7 +122,6 @@ def to_layers(self): class TestDistEmbeddingTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -138,7 +134,7 @@ def setUp(self): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -150,22 +146,28 @@ def test_pp_model(self): rank_id = dist.get_rank() set_random_seed(1024, dp_id, rank_id) - #construct model a + # construct model a model_a = SimpleNet() scheduler_a = paddle.optimizer.lr.PiecewiseDecay( - boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True) - optimizer_a = paddle.optimizer.SGD(learning_rate=scheduler_a, - parameters=model_a.parameters()) + boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True + ) + optimizer_a = paddle.optimizer.SGD( + learning_rate=scheduler_a, parameters=model_a.parameters() + ) init_net = SimpleNetPipe() - model_b = PipelineLayer(layers=init_net.to_layers(), - num_stages=self.pipeline_parallel_size, - loss_fn=LossNet()) + model_b = PipelineLayer( + layers=init_net.to_layers(), + num_stages=self.pipeline_parallel_size, + loss_fn=LossNet(), + ) scheduler_b = paddle.optimizer.lr.PiecewiseDecay( - boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True) - optimizer_b = paddle.optimizer.SGD(learning_rate=scheduler_b, - parameters=model_b.parameters()) + boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True + ) + optimizer_b = paddle.optimizer.SGD( + learning_rate=scheduler_b, parameters=model_b.parameters() + ) model_b = fleet.distributed_model(model_b) optimizer_b = fleet.distributed_optimizer(optimizer_b) @@ -202,8 +204,9 @@ def test_pp_model(self): optimizer_a.clear_grad() scheduler_a.step() - loss_b = model_b.train_batch([(x1, x2), (y1, )], optimizer_b, - scheduler_b) + loss_b = model_b.train_batch( + [(x1, x2), (y1,)], optimizer_b, scheduler_b + ) print("loss", loss_a.numpy(), loss_b.numpy()) np.testing.assert_allclose(loss_a.numpy(), loss_b.numpy()) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_fp16.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_fp16.py index 06813a340ebd9..b0429167036e5 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_fp16.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_fp16.py @@ -34,7 +34,6 @@ def set_random_seed(seed, dp_id, rank_id): class TestDistPPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -47,7 +46,7 @@ def setUp(self): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -61,25 +60,29 @@ def test_pp_model(self): grad_clip = paddle.nn.ClipGradByGlobalNorm(1.0) - #construct model a + # construct model a model_a = AlexNet(10) - scheduler_a = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer_a = paddle.optimizer.SGD(learning_rate=scheduler_a, - grad_clip=grad_clip, - parameters=model_a.parameters()) + scheduler_a = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer_a = paddle.optimizer.SGD( + learning_rate=scheduler_a, + grad_clip=grad_clip, + parameters=model_a.parameters(), + ) scaler_a = paddle.amp.GradScaler(init_loss_scaling=2**5) # construct model b model_b = AlexNetPipeDesc(num_stages=self.pipeline_parallel_size) - scheduler_b = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer_b = paddle.optimizer.SGD(learning_rate=scheduler_b, - grad_clip=grad_clip, - parameters=model_b.parameters()) + scheduler_b = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer_b = paddle.optimizer.SGD( + learning_rate=scheduler_b, + grad_clip=grad_clip, + parameters=model_b.parameters(), + ) param_len = len(model_a.parameters()) parameters = [] @@ -89,14 +92,18 @@ def test_pp_model(self): for idx, param in enumerate(model_b.parameters()): param.set_value(parameters[idx + pp_id * (param_len // 2)]) - model_a, optimizer_a = paddle.amp.decorate(models=model_a, - optimizers=optimizer_a, - level='O2', - save_dtype='float32') - model_b, optimizer_b = paddle.amp.decorate(models=model_b, - optimizers=optimizer_b, - level='O2', - save_dtype='float32') + model_a, optimizer_a = paddle.amp.decorate( + models=model_a, + optimizers=optimizer_a, + level='O2', + save_dtype='float32', + ) + model_b, optimizer_b = paddle.amp.decorate( + models=model_b, + optimizers=optimizer_b, + level='O2', + save_dtype='float32', + ) model_b = fleet.distributed_model(model_b) optimizer_b = fleet.distributed_optimizer(optimizer_b) @@ -104,15 +111,21 @@ def test_pp_model(self): scaler_b = fleet.distributed_scaler(scaler_b) # construct reader - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size, drop_last=True + ) for step_id, data in enumerate(train_reader()): - x_data = np.array([x[0] for x in data]).astype('float32').reshape( - batch_size, 1, 28, 28) - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(batch_size, 1) + x_data = ( + np.array([x[0] for x in data]) + .astype('float32') + .reshape(batch_size, 1, 28, 28) + ) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(batch_size, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) img.stop_gradient = True @@ -129,15 +142,14 @@ def test_pp_model(self): scheduler_a.step() with paddle.amp.auto_cast(enable=True, level='O2'): - loss_b = model_b.train_batch([img, label], - optimizer_b, - scheduler_b, - scaler=scaler_b) + loss_b = model_b.train_batch( + [img, label], optimizer_b, scheduler_b, scaler=scaler_b + ) print("loss: ", loss_a.numpy(), loss_b.numpy()) - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=5e-3) + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=5e-3 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_layer_with_virtual_stage.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_layer_with_virtual_stage.py index 73807abdc18c8..f23df47ae4b56 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_layer_with_virtual_stage.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_layer_with_virtual_stage.py @@ -17,12 +17,15 @@ from paddle.distributed import fleet import paddle.nn as nn from paddle.fluid.dygraph.layers import Layer -from paddle.distributed.fleet.meta_parallel import LayerDesc, PipelineLayer, PipelineParallelWithInterleave +from paddle.distributed.fleet.meta_parallel import ( + LayerDesc, + PipelineLayer, + PipelineParallelWithInterleave, +) import paddle.nn.functional as F class ReshapeHelp(Layer): - def __init__(self, shape): super(ReshapeHelp, self).__init__() self.shape = shape @@ -32,7 +35,6 @@ def forward(self, x): class MLPForVirtualStageLayerTest(PipelineLayer): - def __init__(self, num_classes=10, **kwargs): self.num_classes = num_classes decs = [ @@ -45,21 +47,19 @@ def __init__(self, num_classes=10, **kwargs): LayerDesc(nn.Linear, 2, self.num_classes), LayerDesc(nn.Linear, self.num_classes, 2), ] - super(MLPForVirtualStageLayerTest, - self).__init__(layers=decs, - loss_fn=nn.CrossEntropyLoss(), - **kwargs) + super(MLPForVirtualStageLayerTest, self).__init__( + layers=decs, loss_fn=nn.CrossEntropyLoss(), **kwargs + ) class TestPipeLayerAPI(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.pipeline_parallel_size = 2 strategy.hybrid_configs = { "dp_degree": 1, "mp_degree": 1, - "pp_degree": self.pipeline_parallel_size + "pp_degree": self.pipeline_parallel_size, } fleet.init(is_collective=True, strategy=strategy) self.rank = fleet.worker_index() @@ -74,8 +74,9 @@ def test_pipelayer_desc(self): recompute_ctx={ "mp_group": self.hcg.get_model_parallel_group(), "offload": False, - "partition": False - }) + "partition": False, + }, + ) assert len(pipe_model.parameters()) > 0 model_chunks = pipe_model.get_model_chunks() assert model_chunks is not None @@ -84,14 +85,14 @@ def test_pipelayer_desc(self): optimizer = paddle.optimizer.SGD(parameters=pipe_model.parameters()) try: - model_chunks[0](paddle.to_tensor([1., 2.])) + model_chunks[0](paddle.to_tensor([1.0, 2.0])) raise NotImplementedError except PermissionError: pass # fake call for the forward function of virtual pipeline layer for i in range(len(model_chunks)): - out = pipe_model(paddle.to_tensor([1., 2.]), chunk_id=i) + out = pipe_model(paddle.to_tensor([1.0, 2.0]), chunk_id=i) assert list(out.shape) == [2] out = F.relu(out) loss = paddle.mean(out) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_recompute.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_recompute.py index fde0572d3d8eb..e9443b511e05c 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_recompute.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_recompute.py @@ -42,7 +42,6 @@ def set_random_seed(seed, dp_id, rank_id): class EmbeddingNet(Layer): - def __init__(self): super(EmbeddingNet, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, hidden_size) @@ -56,7 +55,6 @@ def forward(self, x): class TransformerNet(Layer): - def __init__(self): super(TransformerNet, self).__init__() self.linear1 = nn.Linear(d_model, dim_feedforward) @@ -72,7 +70,9 @@ def forward(self, x): q = self.q_proj(x) k = self.k_proj(x) v = self.v_proj(x) - product = layers.matmul(x=q, y=k, transpose_y=True, alpha=d_model**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=d_model**-0.5 + ) weights = F.softmax(product) weights = F.dropout(weights, 0.2) @@ -86,20 +86,17 @@ def forward(self, x): class EmbeddingPipe(EmbeddingNet): - def forward(self, x): return super().forward(x) class TransformerNetPipe(TransformerNet): - def forward(self, x): output = super().forward(x) return output class CriterionPipe(Layer): - def __init__(self): super(CriterionPipe, self).__init__() @@ -109,7 +106,6 @@ def forward(self, out, label): class ModelPipe(PipelineLayer): - def __init__(self, hcg): self.descs = [] self.descs.append(LayerDesc(EmbeddingPipe)) @@ -118,20 +114,21 @@ def __init__(self, hcg): for x in range(2): self.descs.append(LayerDesc(TransformerNetPipe)) - super().__init__(layers=self.descs, - loss_fn=CriterionPipe(), - topology=self.hcg.topology(), - seg_method="layer:TransformerNetPipe", - recompute_interval=1, - recompute_ctx={ - "mp_group": self.hcg.get_model_parallel_group(), - "offload": False, - "partition": False - }) + super().__init__( + layers=self.descs, + loss_fn=CriterionPipe(), + topology=self.hcg.topology(), + seg_method="layer:TransformerNetPipe", + recompute_interval=1, + recompute_ctx={ + "mp_group": self.hcg.get_model_parallel_group(), + "offload": False, + "partition": False, + }, + ) class TestDistPPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -144,7 +141,7 @@ def setUp(self): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -158,11 +155,12 @@ def test_pp_model(self): set_random_seed(1024, dp_id, rank_id) model = ModelPipe(hcg) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.SGD(learning_rate=scheduler, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=scheduler, parameters=model.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load.py index 5d52093a86f20..b7990ff023780 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load.py @@ -29,7 +29,6 @@ class TestDistPPSaveLoadTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -42,7 +41,7 @@ def setUp(self): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -56,11 +55,12 @@ def test_pp_model(self): set_random_seed(1024, dp_id, rank_id) model = ModelPipe(topology) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.SGD(learning_rate=scheduler, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=scheduler, parameters=model.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) @@ -74,14 +74,16 @@ def test_pp_model(self): loss = model.train_batch([x, x], optimizer, scheduler) model._layers.save_state_dict(output_dir) - paddle.save(optimizer.state_dict(), - os.path.join(output_dir, "model_state.pdopt")) + paddle.save( + optimizer.state_dict(), + os.path.join(output_dir, "model_state.pdopt"), + ) # construct data test_steps = 5 - np_data = np.random.randint(0, - vocab_size, - size=[test_steps, batch_size, length]) + np_data = np.random.randint( + 0, vocab_size, size=[test_steps, batch_size, length] + ) origin_loss = [] for step_id in range(5): @@ -101,8 +103,12 @@ def test_pp_model(self): x = paddle.to_tensor(x_data) x.stop_gradient = True loss = model.train_batch([x, x], optimizer, scheduler) - print("origin loss: ", origin_loss[step_id], "current loss: ", - loss.numpy()) + print( + "origin loss: ", + origin_loss[step_id], + "current loss: ", + loss.numpy(), + ) np.testing.assert_allclose(loss.numpy(), origin_loss[step_id]) # finally, remove the model/optimizer path diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load_with_virtual_stage.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load_with_virtual_stage.py index f71e2511a2023..dfe4ecae40392 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load_with_virtual_stage.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_save_load_with_virtual_stage.py @@ -20,7 +20,10 @@ import tempfile import paddle.distributed as dist import paddle.distributed.fleet as fleet -from hybrid_parallel_pp_transformer_with_virtual_stage import ModelPipe, set_random_seed +from hybrid_parallel_pp_transformer_with_virtual_stage import ( + ModelPipe, + set_random_seed, +) batch_size = 8 length = 8 @@ -29,7 +32,6 @@ class TestDistPPSaveLoadTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -42,7 +44,7 @@ def setUp(self): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -56,11 +58,12 @@ def test_pp_model(self): set_random_seed(1024, dp_id, rank_id) model = ModelPipe(topology) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.SGD(learning_rate=scheduler, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=scheduler, parameters=model.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) @@ -74,14 +77,16 @@ def test_pp_model(self): loss = model.train_batch([x, x], optimizer, scheduler) model._layers.save_state_dict(output_dir) - paddle.save(optimizer.state_dict(), - os.path.join(output_dir, "model_state.pdopt")) + paddle.save( + optimizer.state_dict(), + os.path.join(output_dir, "model_state.pdopt"), + ) # construct data test_steps = 5 - np_data = np.random.randint(0, - vocab_size, - size=[test_steps, batch_size, length]) + np_data = np.random.randint( + 0, vocab_size, size=[test_steps, batch_size, length] + ) origin_loss = [] for step_id in range(5): @@ -101,8 +106,12 @@ def test_pp_model(self): x = paddle.to_tensor(x_data) x.stop_gradient = True loss = model.train_batch([x, x], optimizer, scheduler) - print("origin loss: ", origin_loss[step_id], "current loss: ", - loss.numpy()) + print( + "origin loss: ", + origin_loss[step_id], + "current loss: ", + loss.numpy(), + ) np.testing.assert_allclose(loss.numpy(), origin_loss[step_id]) # finally, remove the model/optimizer path diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer.py index de0475018d24f..e1f4aa1762e12 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer.py @@ -42,15 +42,15 @@ def set_random_seed(seed, dp_id, rank_id): class EmbeddingNet(Layer): - def __init__(self): super(EmbeddingNet, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, hidden_size) self.position_embeddings = nn.Embedding(vocab_size, hidden_size) def forward(self, x): - attention_mask = paddle.tensor.triu((paddle.ones( - (length, length), dtype="float32") * -1e9), 1) + attention_mask = paddle.tensor.triu( + (paddle.ones((length, length), dtype="float32") * -1e9), 1 + ) no_used = paddle.ones((3, 3), dtype="int32") @@ -65,7 +65,6 @@ def forward(self, x): class TransformerNet(Layer): - def __init__(self): super(TransformerNet, self).__init__() self.linear1 = nn.Linear(d_model, dim_feedforward) @@ -81,7 +80,9 @@ def forward(self, x, mask): q = self.q_proj(x) k = self.k_proj(x) v = self.v_proj(x) - product = layers.matmul(x=q, y=k, transpose_y=True, alpha=d_model**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=d_model**-0.5 + ) weights = F.softmax(product + mask) # TODO(shenliang03) For save/load in PipeLineParallel, can’t support dropout temporarily. @@ -96,13 +97,11 @@ def forward(self, x, mask): class EmbeddingPipe(EmbeddingNet): - def forward(self, x): return super().forward(x) class TransformerNetPipe(TransformerNet): - def forward(self, args): x, mask, no_used, p_emb = args[0], args[1], args[2], args[3] @@ -113,7 +112,6 @@ def forward(self, args): class CriterionPipe(Layer): - def __init__(self): super(CriterionPipe, self).__init__() @@ -123,7 +121,6 @@ def forward(self, out, label): class ModelPipe(PipelineLayer): - def __init__(self, topology): self.descs = [] self.descs.append(LayerDesc(EmbeddingPipe)) @@ -133,14 +130,15 @@ def __init__(self, topology): self.descs.append(lambda x: x[0]) - super().__init__(layers=self.descs, - loss_fn=CriterionPipe(), - topology=topology, - seg_method="layer:TransformerNetPipe") + super().__init__( + layers=self.descs, + loss_fn=CriterionPipe(), + topology=topology, + seg_method="layer:TransformerNetPipe", + ) class TestDistPPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -153,7 +151,7 @@ def setUp(self): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -167,11 +165,12 @@ def test_pp_model(self): set_random_seed(1024, dp_id, rank_id) model = ModelPipe(topology) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.SGD(learning_rate=scheduler, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=scheduler, parameters=model.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py index edad5e5bc99bd..89e0410e2ca7f 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_transformer_with_virtual_stage.py @@ -43,15 +43,15 @@ def set_random_seed(seed, dp_id, rank_id): class EmbeddingNet(Layer): - def __init__(self): super(EmbeddingNet, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, hidden_size) self.position_embeddings = nn.Embedding(vocab_size, hidden_size) def forward(self, x): - attention_mask = paddle.tensor.triu((paddle.ones( - (length, length), dtype="float32") * -1e9), 1) + attention_mask = paddle.tensor.triu( + (paddle.ones((length, length), dtype="float32") * -1e9), 1 + ) no_used = paddle.ones((3, 3), dtype="int32") @@ -66,7 +66,6 @@ def forward(self, x): class TransformerNet(Layer): - def __init__(self): super(TransformerNet, self).__init__() self.linear1 = nn.Linear(d_model, dim_feedforward) @@ -82,7 +81,9 @@ def forward(self, x, mask): q = self.q_proj(x) k = self.k_proj(x) v = self.v_proj(x) - product = layers.matmul(x=q, y=k, transpose_y=True, alpha=d_model**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=d_model**-0.5 + ) weights = F.softmax(product + mask) tgt = layers.matmul(weights, v) @@ -95,13 +96,11 @@ def forward(self, x, mask): class EmbeddingPipe(EmbeddingNet): - def forward(self, x): return super().forward(x) class TransformerNetPipe(TransformerNet): - def forward(self, args): x, mask, no_used, p_emb = args[0], args[1], args[2], args[3] @@ -112,7 +111,6 @@ def forward(self, args): class CriterionPipe(Layer): - def __init__(self): super(CriterionPipe, self).__init__() @@ -122,7 +120,6 @@ def forward(self, out, label): class ModelPipe(PipelineLayer): - def __init__(self, topology): self.descs = [] self.descs.append(LayerDesc(EmbeddingPipe)) @@ -137,11 +134,11 @@ def __init__(self, topology): loss_fn=CriterionPipe(), topology=topology, num_virtual_pipeline_stages=num_virtual_pipeline_stages, - seg_method="layer:TransformerNetPipe") + seg_method="layer:TransformerNetPipe", + ) class TestDistPPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -154,7 +151,7 @@ def setUp(self): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -168,11 +165,12 @@ def test_pp_model(self): set_random_seed(1024, dp_id, rank_id) model = ModelPipe(topology) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.SGD(learning_rate=scheduler, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=scheduler, parameters=model.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_qat.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_qat.py index 03e357993493f..80e7b8b4e695c 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_qat.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_qat.py @@ -79,7 +79,8 @@ def parallel_matmul(lm_output, logit_weights, parallel_output): if world_size > 1: input_parallel = paddle.distributed.collective._c_identity( - lm_output, group=model_parallel_group) + lm_output, group=model_parallel_group + ) logits = paddle.matmul(input_parallel, logit_weights, transpose_y=True) @@ -87,22 +88,23 @@ def parallel_matmul(lm_output, logit_weights, parallel_output): return logits return paddle.distributed.collective._c_concat( - logits, group=model_parallel_group) + logits, group=model_parallel_group + ) else: logits = paddle.matmul(lm_output, logit_weights, transpose_y=True) return logits class PACT(nn.Layer): - def __init__(self, init_value=20): super(PACT, self).__init__() alpha_attr = paddle.ParamAttr( name=self.full_name() + ".pact", - initializer=paddle.nn.initializer.Constant(value=init_value)) - self.alpha = self.create_parameter(shape=[1], - attr=alpha_attr, - dtype='float32') + initializer=paddle.nn.initializer.Constant(value=init_value), + ) + self.alpha = self.create_parameter( + shape=[1], attr=alpha_attr, dtype='float32' + ) def forward(self, x): out_left = paddle.nn.functional.relu(x - self.alpha) @@ -112,46 +114,61 @@ def forward(self, x): class SimpleMPNet(nn.Layer): - - def __init__(self, vocab_size, hidden_size, inner_size, output_size, np_fc1, - np_fc2, mp_id): + def __init__( + self, + vocab_size, + hidden_size, + inner_size, + output_size, + np_fc1, + np_fc2, + mp_id, + ): super(SimpleMPNet, self).__init__() if mp_id == 0: - init_fc1_data = np_fc1[:, :(inner_size // 2)] - init_fc2_data = np_fc2[:(inner_size // 2), :] + init_fc1_data = np_fc1[:, : (inner_size // 2)] + init_fc2_data = np_fc2[: (inner_size // 2), :] else: - init_fc1_data = np_fc1[:, (inner_size // 2):] - init_fc2_data = np_fc2[(inner_size // 2):, :] + init_fc1_data = np_fc1[:, (inner_size // 2) :] + init_fc2_data = np_fc2[(inner_size // 2) :, :] self.linear1 = fleet.meta_parallel.ColumnParallelLinear( hidden_size, inner_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(init_fc1_data)), + initializer=paddle.nn.initializer.Assign(init_fc1_data) + ), gather_output=False, - has_bias=True) + has_bias=True, + ) self.linear2 = fleet.meta_parallel.RowParallelLinear( inner_size, hidden_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(init_fc2_data)), + initializer=paddle.nn.initializer.Assign(init_fc2_data) + ), input_is_parallel=True, - has_bias=True) + has_bias=True, + ) self.linear3 = paddle.nn.Linear( hidden_size, output_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0)), + initializer=paddle.nn.initializer.Constant(0.0) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.embedding = fleet.meta_parallel.VocabParallelEmbedding( vocab_size, hidden_size, - weight_attr=paddle.nn.initializer.Constant(value=1.)) + weight_attr=paddle.nn.initializer.Constant(value=1.0), + ) def forward(self, x): x = self.embedding(x) @@ -163,53 +180,62 @@ def forward(self, x): class SimpleDPNet(nn.Layer): - - def __init__(self, vocab_size, hidden_size, inner_size, output_size, np_fc1, - np_fc2): + def __init__( + self, vocab_size, hidden_size, inner_size, output_size, np_fc1, np_fc2 + ): super(SimpleDPNet, self).__init__() self.linear1 = paddle.nn.Linear( hidden_size, inner_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(np_fc1)), + initializer=paddle.nn.initializer.Assign(np_fc1) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.linear2 = paddle.nn.Linear( inner_size, hidden_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(np_fc2)), + initializer=paddle.nn.initializer.Assign(np_fc2) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.linear3 = paddle.nn.Linear( hidden_size, output_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0)), + initializer=paddle.nn.initializer.Constant(0.0) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.embedding = paddle.nn.Embedding( vocab_size, hidden_size, - weight_attr=paddle.nn.initializer.Constant(value=1.)) + weight_attr=paddle.nn.initializer.Constant(value=1.0), + ) def forward(self, x): x = self.embedding(x) x = self.linear1(x) x = self.linear2(x) x = self.linear3(x) - x = paddle.matmul(x, - get_attr(self.embedding, "weight"), - transpose_y=True) + x = paddle.matmul( + x, get_attr(self.embedding, "weight"), transpose_y=True + ) return x class TestDistMPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 2 @@ -217,7 +243,7 @@ def setUp(self): strategy.hybrid_configs = { "dp_degree": self.data_parallel_size, "mp_degree": self.model_parallel_size, - "pp_degree": 1 + "pp_degree": 1, } fleet.init(is_collective=True, strategy=strategy) self.onnx_format = False @@ -234,14 +260,14 @@ def train_batch(self, batch, model, optimizer, is_mp): return loss def build_optimizer(self, model): - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return optimizer - def build_model_optimizer(self, - weight_quantize_type, - activation_quantize_type, - use_pact=False): + def build_model_optimizer( + self, weight_quantize_type, activation_quantize_type, use_pact=False + ): hcg = fleet.get_hybrid_communicate_group() word_size = hcg.get_model_parallel_world_size() mp_id = hcg.get_model_parallel_rank() @@ -251,23 +277,32 @@ def build_model_optimizer(self, weight_quantize_type=weight_quantize_type, activation_quantize_type=activation_quantize_type, fuse_conv_bn=self.fuse_conv_bn, - act_preprocess_layer=PACT if use_pact else None) + act_preprocess_layer=PACT if use_pact else None, + ) set_random_seed(1024, dp_id, rank_id) np_fc1 = np.ones((hidden_size, inner_size)) np_fc2 = np.ones( - (inner_size, - hidden_size)) #np.random.random_sample((inner_size, hidden_size)) + (inner_size, hidden_size) + ) # np.random.random_sample((inner_size, hidden_size)) - model_a = SimpleMPNet(vocab_size, hidden_size, inner_size, output_size, - np_fc1, np_fc2, mp_id) + model_a = SimpleMPNet( + vocab_size, + hidden_size, + inner_size, + output_size, + np_fc1, + np_fc2, + mp_id, + ) model_a = imperative_qat.quantize(model_a) optimizer_a = self.build_optimizer(model_a) model_a = fleet.distributed_model(model_a) optimizer_a = fleet.distributed_optimizer(optimizer_a) - model_b = SimpleDPNet(vocab_size, hidden_size, inner_size, output_size, - np_fc1, np_fc2) + model_b = SimpleDPNet( + vocab_size, hidden_size, inner_size, output_size, np_fc1, np_fc2 + ) model_b = imperative_qat.quantize(model_b) optimizer_b = self.build_optimizer(model_b) @@ -277,21 +312,27 @@ def train(self, model_a, optimizer_a, model_b, optimizer_b): for epoch in range(5): - np_data = np.random.randint(0, vocab_size, ( - batch_size, - seq_length, - )) + np_data = np.random.randint( + 0, + vocab_size, + ( + batch_size, + seq_length, + ), + ) batch = paddle.to_tensor(np_data) loss_a = self.train_batch(batch, model_a, optimizer_a, True) loss_b = self.train_batch(batch, model_b, optimizer_b, False) - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=1e-6) + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=1e-6 + ) def test_mp_model_1(self): - if not fluid.core.is_compiled_with_cuda( - ) or fluid.core.get_cuda_device_count() == 0: + if ( + not fluid.core.is_compiled_with_cuda() + or fluid.core.get_cuda_device_count() == 0 + ): return selected_gpus = get_gpus('0,1') cluster = None @@ -299,12 +340,15 @@ def test_mp_model_1(self): model_a, optimizer_a, model_b, optimizer_b = self.build_model_optimizer( weight_quantize_type='abs_max', - activation_quantize_type='moving_average_abs_max') + activation_quantize_type='moving_average_abs_max', + ) self.train(model_a, optimizer_a, model_b, optimizer_b) def test_mp_model_2(self): - if not fluid.core.is_compiled_with_cuda( - ) or fluid.core.get_cuda_device_count() == 0: + if ( + not fluid.core.is_compiled_with_cuda() + or fluid.core.get_cuda_device_count() == 0 + ): return selected_gpus = get_gpus('0,1') cluster = None @@ -313,7 +357,8 @@ def test_mp_model_2(self): model_a, optimizer_a, model_b, optimizer_b = self.build_model_optimizer( weight_quantize_type='channel_wise_abs_max', activation_quantize_type='moving_average_abs_max', - use_pact=True) + use_pact=True, + ) self.train(model_a, optimizer_a, model_b, optimizer_b) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_sharding_model.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_sharding_model.py index 96a7825dc801c..147ba1e494c6a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_sharding_model.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_sharding_model.py @@ -18,7 +18,9 @@ import paddle.distributed as dist import paddle.fluid as fluid import paddle.distributed.fleet as fleet -from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.dygraph_sharding_optimizer import DygraphShardingOptimizer +from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.dygraph_sharding_optimizer import ( + DygraphShardingOptimizer, +) import unittest vocab_size = 20 @@ -38,7 +40,8 @@ def parallel_matmul(lm_output, logit_weights, parallel_output): if world_size > 1: input_parallel = paddle.distributed.collective._c_identity( - lm_output, group=model_parallel_group) + lm_output, group=model_parallel_group + ) logits = paddle.matmul(input_parallel, logit_weights, transpose_y=True) @@ -46,53 +49,69 @@ def parallel_matmul(lm_output, logit_weights, parallel_output): return logits return paddle.distributed.collective._c_concat( - logits, group=model_parallel_group) + logits, group=model_parallel_group + ) else: logits = paddle.matmul(lm_output, logit_weights, transpose_y=True) return logits class SimpleMPNet(fluid.dygraph.Layer): - - def __init__(self, vocab_size, hidden_size, inner_size, output_size, np_fc1, - np_fc2, mp_id): + def __init__( + self, + vocab_size, + hidden_size, + inner_size, + output_size, + np_fc1, + np_fc2, + mp_id, + ): super(SimpleMPNet, self).__init__() if mp_id == 0: - init_fc1_data = np_fc1[:, :(inner_size // 2)] - init_fc2_data = np_fc2[:(inner_size // 2), :] + init_fc1_data = np_fc1[:, : (inner_size // 2)] + init_fc2_data = np_fc2[: (inner_size // 2), :] else: - init_fc1_data = np_fc1[:, (inner_size // 2):] - init_fc2_data = np_fc2[(inner_size // 2):, :] + init_fc1_data = np_fc1[:, (inner_size // 2) :] + init_fc2_data = np_fc2[(inner_size // 2) :, :] self.linear1 = fleet.meta_parallel.ColumnParallelLinear( hidden_size, inner_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(init_fc1_data)), + initializer=paddle.nn.initializer.Assign(init_fc1_data) + ), gather_output=False, - has_bias=True) + has_bias=True, + ) self.linear2 = fleet.meta_parallel.RowParallelLinear( inner_size, hidden_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(init_fc2_data)), + initializer=paddle.nn.initializer.Assign(init_fc2_data) + ), input_is_parallel=True, - has_bias=True) + has_bias=True, + ) self.linear3 = paddle.nn.Linear( hidden_size, output_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0)), + initializer=paddle.nn.initializer.Constant(0.0) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.embedding = fleet.meta_parallel.VocabParallelEmbedding( vocab_size, hidden_size, - weight_attr=paddle.nn.initializer.Constant(value=0.5)) + weight_attr=paddle.nn.initializer.Constant(value=0.5), + ) def forward(self, x): x = self.embedding(x) @@ -104,39 +123,49 @@ def forward(self, x): class SimpleDPNet(fluid.dygraph.Layer): - - def __init__(self, vocab_size, hidden_size, inner_size, output_size, np_fc1, - np_fc2): + def __init__( + self, vocab_size, hidden_size, inner_size, output_size, np_fc1, np_fc2 + ): super(SimpleDPNet, self).__init__() self.linear1 = paddle.nn.Linear( hidden_size, inner_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(np_fc1)), + initializer=paddle.nn.initializer.Assign(np_fc1) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.linear2 = paddle.nn.Linear( inner_size, hidden_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Assign(np_fc2)), + initializer=paddle.nn.initializer.Assign(np_fc2) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.linear3 = paddle.nn.Linear( hidden_size, output_size, weight_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0)), + initializer=paddle.nn.initializer.Constant(0.0) + ), bias_attr=paddle.framework.ParamAttr( - initializer=paddle.nn.initializer.Constant(0.0))) + initializer=paddle.nn.initializer.Constant(0.0) + ), + ) self.embedding = paddle.nn.Embedding( vocab_size, hidden_size, - weight_attr=paddle.nn.initializer.Constant(value=0.5)) + weight_attr=paddle.nn.initializer.Constant(value=0.5), + ) def forward(self, x): x = self.embedding(x) @@ -148,7 +177,6 @@ def forward(self, x): class TestDistMPTraning(unittest.TestCase): - def setUp(self): random.seed(2021) np.random.seed(2021) @@ -163,10 +191,15 @@ def setUp(self): } fleet.init(is_collective=True, strategy=self.strategy) self.data = [ - np.random.randint(0, vocab_size, ( - batch_size, - seq_length, - )) for _ in range(STEPS) + np.random.randint( + 0, + vocab_size, + ( + batch_size, + seq_length, + ), + ) + for _ in range(STEPS) ] def train_batch(self, batch, model, optimizer): @@ -178,11 +211,9 @@ def train_batch(self, batch, model, optimizer): optimizer.clear_grad() return loss - def build_optimizer(self, - model, - strategy=None, - is_sharding=True, - Optimizer="adam"): + def build_optimizer( + self, model, strategy=None, is_sharding=True, Optimizer="adam" + ): clip = paddle.nn.ClipGradByGlobalNorm(0.5) if Optimizer == "adam": if is_sharding: @@ -193,13 +224,15 @@ def build_optimizer(self, inner_optimizer_class=paddle.optimizer.AdamW, learning_rate=0.001, weight_decay=0.00001, - grad_clip=clip) + grad_clip=clip, + ) else: optimizer = paddle.optimizer.AdamW( parameters=model.parameters(), learning_rate=0.001, weight_decay=0.00001, - grad_clip=clip) + grad_clip=clip, + ) else: if is_sharding: optimizer = DygraphShardingOptimizer( @@ -208,12 +241,14 @@ def build_optimizer(self, params=model.parameters(), inner_optimizer_class=paddle.optimizer.Momentum, learning_rate=0.001, - grad_clip=clip) + grad_clip=clip, + ) else: optimizer = paddle.optimizer.Momentum( learning_rate=0.001, parameters=model.parameters(), - grad_clip=clip) + grad_clip=clip, + ) return optimizer def build_model_optimizer(self, Optimizer="adam"): @@ -226,37 +261,48 @@ def build_model_optimizer(self, Optimizer="adam"): np_fc1 = np.random.random_sample((hidden_size, inner_size)) np_fc2 = np.random.random_sample((inner_size, hidden_size)) - model_a = SimpleDPNet(vocab_size, hidden_size, inner_size, output_size, - np_fc1, np_fc2) - optimizer_a = self.build_optimizer(model_a, - strategy=self.strategy, - is_sharding=True, - Optimizer=Optimizer) + model_a = SimpleDPNet( + vocab_size, hidden_size, inner_size, output_size, np_fc1, np_fc2 + ) + optimizer_a = self.build_optimizer( + model_a, + strategy=self.strategy, + is_sharding=True, + Optimizer=Optimizer, + ) model_a = fleet.distributed_model(model_a) optimizer_a = fleet.distributed_optimizer(optimizer_a) - model_b = SimpleDPNet(vocab_size, hidden_size, inner_size, output_size, - np_fc1, np_fc2) - optimizer_b = self.build_optimizer(model_b, - strategy=self.strategy, - is_sharding=False, - Optimizer=Optimizer) + model_b = SimpleDPNet( + vocab_size, hidden_size, inner_size, output_size, np_fc1, np_fc2 + ) + optimizer_b = self.build_optimizer( + model_b, + strategy=self.strategy, + is_sharding=False, + Optimizer=Optimizer, + ) return model_a, optimizer_a, model_b, optimizer_b def sharding_model(self, Optimizer, sharded_accumulators): model_a, optimizer_a, model_b, optimizer_b = self.build_model_optimizer( - Optimizer=Optimizer) + Optimizer=Optimizer + ) self.assertTrue( - isinstance(optimizer_a._inner_opt, DygraphShardingOptimizer)) + isinstance(optimizer_a._inner_opt, DygraphShardingOptimizer) + ) for idx in range(STEPS): if idx == 2 and paddle.distributed.get_rank() == 0: self.assertTrue( - set(optimizer_a._inner_opt._inner_optimizer.state_dict(). - keys()) == sharded_accumulators) + set( + optimizer_a._inner_opt._inner_optimizer.state_dict().keys() + ) + == sharded_accumulators + ) if paddle.distributed.get_rank() == 0: batch_sharding = paddle.to_tensor(self.data[idx][:2]) @@ -268,31 +314,49 @@ def sharding_model(self, Optimizer, sharded_accumulators): loss_b = self.train_batch(batch_single, model_b, optimizer_b) for j in range(len(model_a.parameters())): - np.testing.assert_allclose(model_a.parameters()[j].numpy(), - model_b.parameters()[j].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + model_a.parameters()[j].numpy(), + model_b.parameters()[j].numpy(), + rtol=1e-6, + ) def test_sharding_adam(self): - sharded_accumulators = set([ - 'linear_0.w_0_moment1_0', 'linear_1.b_0_moment1_0', - 'linear_2.b_0_moment1_0', 'embedding_0.w_0_moment1_0', - 'linear_0.w_0_moment2_0', 'linear_1.b_0_moment2_0', - 'linear_2.b_0_moment2_0', 'embedding_0.w_0_moment2_0', - 'linear_0.w_0_beta1_pow_acc_0', 'linear_1.b_0_beta1_pow_acc_0', - 'linear_2.b_0_beta1_pow_acc_0', 'embedding_0.w_0_beta1_pow_acc_0', - 'linear_0.w_0_beta2_pow_acc_0', 'linear_1.b_0_beta2_pow_acc_0', - 'linear_2.b_0_beta2_pow_acc_0', 'embedding_0.w_0_beta2_pow_acc_0' - ]) - self.sharding_model(Optimizer="adam", - sharded_accumulators=sharded_accumulators) + sharded_accumulators = set( + [ + 'linear_0.w_0_moment1_0', + 'linear_1.b_0_moment1_0', + 'linear_2.b_0_moment1_0', + 'embedding_0.w_0_moment1_0', + 'linear_0.w_0_moment2_0', + 'linear_1.b_0_moment2_0', + 'linear_2.b_0_moment2_0', + 'embedding_0.w_0_moment2_0', + 'linear_0.w_0_beta1_pow_acc_0', + 'linear_1.b_0_beta1_pow_acc_0', + 'linear_2.b_0_beta1_pow_acc_0', + 'embedding_0.w_0_beta1_pow_acc_0', + 'linear_0.w_0_beta2_pow_acc_0', + 'linear_1.b_0_beta2_pow_acc_0', + 'linear_2.b_0_beta2_pow_acc_0', + 'embedding_0.w_0_beta2_pow_acc_0', + ] + ) + self.sharding_model( + Optimizer="adam", sharded_accumulators=sharded_accumulators + ) def test_sharding_momentum(self): - sharded_accumulators = set([ - 'linear_6.w_0_velocity_0', 'linear_7.b_0_velocity_0', - 'linear_8.b_0_velocity_0', 'embedding_2.w_0_velocity_0' - ]) - self.sharding_model(Optimizer="Momentum", - sharded_accumulators=sharded_accumulators) + sharded_accumulators = set( + [ + 'linear_6.w_0_velocity_0', + 'linear_7.b_0_velocity_0', + 'linear_8.b_0_velocity_0', + 'embedding_2.w_0_velocity_0', + ] + ) + self.sharding_model( + Optimizer="Momentum", sharded_accumulators=sharded_accumulators + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_shared_weight.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_shared_weight.py index 1696b966d7c52..1176d70504b7c 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_shared_weight.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_shared_weight.py @@ -44,15 +44,16 @@ def set_random_seed(seed, dp_id, rank_id): class SimpleNet(Layer): - def __init__(self): super(SimpleNet, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, hidden_size) self.softmax_weight = self.create_parameter( - shape=[hidden_size, vocab_size]) - self.softmax_bias = self.create_parameter(shape=[vocab_size], - is_bias=False) + shape=[hidden_size, vocab_size] + ) + self.softmax_bias = self.create_parameter( + shape=[vocab_size], is_bias=False + ) def forward(self, x1, x2, y1): x_emb = self.word_embeddings(x1) @@ -62,14 +63,13 @@ def forward(self, x1, x2, y1): projection = paddle.matmul(projection, self.word_embeddings.weight) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=y1, - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=y1, soft_label=False + ) return loss.mean() class EmbeddingPipe(Layer): - def __init__(self): super(EmbeddingPipe, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, hidden_size) @@ -85,11 +85,11 @@ def forward(self, args): class MatmulNet(Layer): - def __init__(self): super(MatmulNet, self).__init__() self.softmax_weight = self.create_parameter( - shape=[hidden_size, vocab_size]) + shape=[hidden_size, vocab_size] + ) def forward(self, args): x1, x2 = args @@ -99,7 +99,6 @@ def forward(self, args): class BiasNet(Layer): - def __init__(self): super(BiasNet, self).__init__() self.softmax_bias = self.create_parameter(shape=[vocab_size]) @@ -112,26 +111,25 @@ def forward(self, args): class LossNet(Layer): - def __init__(self): super(LossNet, self).__init__() def forward(self, args, y1): projection = args - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=y1[0], - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=y1[0], soft_label=False + ) return loss.mean() class SimpleNetPipe(PipelineLayer): - def __init__(self, **kwargs): self.descs = [] self.descs.append( - SharedLayerDesc('embed', - EmbeddingPipe, - shared_weight_attr='embedding_weight')) + SharedLayerDesc( + 'embed', EmbeddingPipe, shared_weight_attr='embedding_weight' + ) + ) self.descs.append(LayerDesc(MatmulNet)) self.descs.append(LayerDesc(BiasNet)) @@ -140,18 +138,20 @@ def _logits_helper(embedding, output): return paddle.matmul(output[0], embedding.embedding_weight) self.descs.append( - SharedLayerDesc('embed', - EmbeddingPipe, - forward_func=_logits_helper, - shared_weight_attr='embedding_weight')) + SharedLayerDesc( + 'embed', + EmbeddingPipe, + forward_func=_logits_helper, + shared_weight_attr='embedding_weight', + ) + ) - super(SimpleNetPipe, self).__init__(layers=self.descs, - loss_fn=LossNet(), - **kwargs) + super(SimpleNetPipe, self).__init__( + layers=self.descs, loss_fn=LossNet(), **kwargs + ) class TestDistEmbeddingTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -164,7 +164,7 @@ def setUp(self): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) @@ -176,19 +176,23 @@ def test_pp_model(self): rank_id = dist.get_rank() set_random_seed(1024, dp_id, rank_id) - #construct model a + # construct model a model_a = SimpleNet() scheduler_a = paddle.optimizer.lr.PiecewiseDecay( - boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True) - optimizer_a = paddle.optimizer.SGD(learning_rate=scheduler_a, - parameters=model_a.parameters()) + boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True + ) + optimizer_a = paddle.optimizer.SGD( + learning_rate=scheduler_a, parameters=model_a.parameters() + ) model_b = SimpleNetPipe(topology=hcg.topology()) scheduler_b = paddle.optimizer.lr.PiecewiseDecay( - boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True) - optimizer_b = paddle.optimizer.SGD(learning_rate=scheduler_b, - parameters=model_b.parameters()) + boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04], verbose=True + ) + optimizer_b = paddle.optimizer.SGD( + learning_rate=scheduler_b, parameters=model_b.parameters() + ) model_b = fleet.distributed_model(model_b) optimizer_b = fleet.distributed_optimizer(optimizer_b) @@ -228,8 +232,9 @@ def test_pp_model(self): optimizer_a.clear_grad() scheduler_a.step() - loss_b = model_b.train_batch([(x1, x2), (y1, )], optimizer_b, - scheduler_b) + loss_b = model_b.train_batch( + [(x1, x2), (y1,)], optimizer_b, scheduler_b + ) print("loss", loss_a.numpy(), loss_b.numpy()) np.testing.assert_allclose(loss_a.numpy(), loss_b.numpy()) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/new_group.py b/python/paddle/fluid/tests/unittests/collective/fleet/new_group.py index b6e537681b213..9fa469ea5e234 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/new_group.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/new_group.py @@ -17,7 +17,6 @@ class TestNewGroupAPI(object): - def __init__(self): paddle.distributed.init_parallel_env() d1 = np.array([1, 2, 3]) @@ -32,10 +31,9 @@ def test_all(self): tmp = np.array([0, 0, 0]) result = paddle.to_tensor(tmp) - paddle.distributed.scatter(result, [self.tensor2, self.tensor1], - src=0, - group=gp, - sync_op=True) + paddle.distributed.scatter( + result, [self.tensor2, self.tensor1], src=0, group=gp, sync_op=True + ) if gp.rank == 0: assert np.array_equal(result, self.tensor2) elif gp.rank == 1: @@ -48,8 +46,9 @@ def test_all(self): paddle.distributed.reduce(result, dst=0, group=gp, sync_op=True) if gp.rank == 0: - assert np.array_equal(result, paddle.add(self.tensor1, - self.tensor1)) + assert np.array_equal( + result, paddle.add(self.tensor1, self.tensor1) + ) elif gp.rank == 1: assert np.array_equal(result, self.tensor1) print("test reduce api ok") @@ -57,7 +56,8 @@ def test_all(self): paddle.distributed.all_reduce(result, sync_op=True) assert np.array_equal( result, - paddle.add(paddle.add(self.tensor1, self.tensor1), self.tensor1)) + paddle.add(paddle.add(self.tensor1, self.tensor1), self.tensor1), + ) print("test all_reduce api ok") paddle.distributed.wait(result, gp, use_calc_stream=True) @@ -65,10 +65,9 @@ def test_all(self): print("test wait api ok") result = [] - paddle.distributed.all_gather(result, - self.tensor1, - group=gp, - sync_op=True) + paddle.distributed.all_gather( + result, self.tensor1, group=gp, sync_op=True + ) assert np.array_equal(result[0], self.tensor1) assert np.array_equal(result[1], self.tensor1) print("test all_gather api ok") diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_class_center_sample.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_class_center_sample.py index ef7f54409872a..bc75ede1d7e88 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_class_center_sample.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_class_center_sample.py @@ -37,10 +37,13 @@ def class_center_sample_numpy(label, classes_list, num_samples): unique_label_per_device = [] for i in range(nranks): - index = np.logical_and(unique_label >= class_interval[i], - unique_label < class_interval[i + 1]) - pos_class_center_per_device.append(unique_label[index] - - class_interval[i]) + index = np.logical_and( + unique_label >= class_interval[i], + unique_label < class_interval[i + 1], + ) + pos_class_center_per_device.append( + unique_label[index] - class_interval[i] + ) unique_label_per_device.append(unique_label[index]) num_samples_per_device = [] @@ -50,8 +53,9 @@ def class_center_sample_numpy(label, classes_list, num_samples): remapped_dict = {} for i in range(nranks): - for idx, v in enumerate(unique_label_per_device[i], - sampled_class_interval[i]): + for idx, v in enumerate( + unique_label_per_device[i], sampled_class_interval[i] + ): remapped_dict[v] = idx remapped_label = [] @@ -62,7 +66,6 @@ def class_center_sample_numpy(label, classes_list, num_samples): class TestParallelClassCenterSampleOp(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() fleet.init(is_collective=True, strategy=strategy) @@ -83,24 +86,35 @@ def test_class_center_sample(self): for dtype in ('int32', 'int64'): for _ in range(5): - classes_list = np.random.randint(10, 15, (nranks, )) + classes_list = np.random.randint(10, 15, (nranks,)) num_class = np.sum(classes_list) - np_label = np.random.randint(0, - num_class, (batch_size, ), - dtype=dtype) + np_label = np.random.randint( + 0, num_class, (batch_size,), dtype=dtype + ) label = paddle.to_tensor(np_label, dtype=dtype) - np_remapped_label, np_sampled_class_center_per_device = class_center_sample_numpy( - np_label, classes_list, num_samples) - remapped_label, sampled_class_index = paddle.nn.functional.class_center_sample( - label, classes_list[rank_id], num_samples) - np.testing.assert_allclose(remapped_label.numpy(), - np_remapped_label) + ( + np_remapped_label, + np_sampled_class_center_per_device, + ) = class_center_sample_numpy( + np_label, classes_list, num_samples + ) + ( + remapped_label, + sampled_class_index, + ) = paddle.nn.functional.class_center_sample( + label, classes_list[rank_id], num_samples + ) + np.testing.assert_allclose( + remapped_label.numpy(), np_remapped_label + ) np_sampled_class_index = np_sampled_class_center_per_device[ - rank_id] + rank_id + ] np.testing.assert_allclose( - sampled_class_index.numpy()[:len(np_sampled_class_index)], - np_sampled_class_index) + sampled_class_index.numpy()[: len(np_sampled_class_index)], + np_sampled_class_index, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_different.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_different.py index 24a29f2a71149..203396b0fd008 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_different.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_different.py @@ -25,14 +25,15 @@ class SimpleNet(fluid.Layer): - def __init__(self, hidden_size, vocab_size, is_sparse=False): super(SimpleNet, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size - self.embedding = Embedding(size=[self.vocab_size, self.hidden_size], - dtype='float32', - is_sparse=is_sparse) + self.embedding = Embedding( + size=[self.vocab_size, self.hidden_size], + dtype='float32', + is_sparse=is_sparse, + ) self.lin_a = paddle.nn.Linear(self.hidden_size, self.vocab_size) self.lin_b = paddle.nn.Linear(self.vocab_size, 1) @@ -57,10 +58,9 @@ def forward(self, input, label, conf): projection = paddle.reshape(projection, shape=[-1, 1]) output = paddle.gather(projection, emb_mask_inds) target = paddle.gather(label, emb_mask_inds) - loss_box = F.smooth_l1_loss(output, - target, - reduction='sum', - delta=1.0) + loss_box = F.smooth_l1_loss( + output, target, reduction='sum', delta=1.0 + ) loss_box = loss_box / len(conf) return loss_box @@ -72,36 +72,60 @@ def forward(self, input, label, conf): hidden_size = 5 vocab_size = 100 -conf_dataset = [[0], [0], [0], [0], [1], [0], [1], [0], [0], [1], [0], [1], [1], - [1], [1], [1], [1], [1], [1], [1], [1], [0], [0], [1]] +conf_dataset = [ + [0], + [0], + [0], + [0], + [1], + [0], + [1], + [0], + [0], + [1], + [0], + [1], + [1], + [1], + [1], + [1], + [1], + [1], + [1], + [1], + [1], + [0], + [0], + [1], +] def fake_sample_reader(): - def __reader__(): for i in range(batch_num): x_data = np.random.randint(0, vocab_size) - y_data = np.random.random_sample((1, )).astype('float32') - conf_data = np.array( - conf_dataset[i % len(conf_dataset)]).astype('int64') + y_data = np.random.random_sample((1,)).astype('float32') + conf_data = np.array(conf_dataset[i % len(conf_dataset)]).astype( + 'int64' + ) yield x_data, y_data, conf_data return __reader__ class TestSimpleNet(TestParallelDyGraphRunnerBase): - def get_model(self): - model = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - is_sparse=False) + model = SimpleNet( + hidden_size=hidden_size, vocab_size=vocab_size, is_sparse=False + ) - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_same.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_same.py index 8b9aaf5294369..f9097fc8461fc 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_same.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_control_flow_same.py @@ -29,15 +29,18 @@ class SimpleNet(fluid.Layer): - def __init__(self): super(SimpleNet, self).__init__() - self.net_a = paddle.nn.Sequential(paddle.nn.Linear(10, 20), - paddle.nn.Linear(20, 20), - paddle.nn.Linear(20, 5)) - self.net_b = paddle.nn.Sequential(paddle.nn.Linear(10, 20), - paddle.nn.Linear(20, 20), - paddle.nn.Linear(20, 5)) + self.net_a = paddle.nn.Sequential( + paddle.nn.Linear(10, 20), + paddle.nn.Linear(20, 20), + paddle.nn.Linear(20, 5), + ) + self.net_b = paddle.nn.Sequential( + paddle.nn.Linear(10, 20), + paddle.nn.Linear(20, 20), + paddle.nn.Linear(20, 5), + ) self.net_unused = Linear(10, 20) self.step = 0 @@ -51,24 +54,23 @@ def forward(self, x): def fake_sample_reader(): - def __reader__(): for i in range(batch_num): - x_data = np.random.random_sample((10, )).astype('float32') + x_data = np.random.random_sample((10,)).astype('float32') yield x_data return __reader__ class TestSimpleNet(TestParallelDyGraphRunnerBase): - def get_model(self): model = SimpleNet() - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer def run_one_loop(self, model, optimizer, batch): diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync.py index b00639202a684..633cdbcb0af3b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync.py @@ -20,7 +20,12 @@ import paddle.fluid as fluid import paddle.distributed as dist from paddle.fluid.dygraph.nn import Linear -from test_dist_base import print_to_err, print_to_out, runtime_main, TestParallelDyGraphRunnerBase +from test_dist_base import ( + print_to_err, + print_to_out, + runtime_main, + TestParallelDyGraphRunnerBase, +) seed = 90 RUN_STEP = 20 @@ -29,7 +34,6 @@ class SimpleNet(fluid.Layer): - def __init__(self): super(SimpleNet, self).__init__() self.net_a = Linear(input_dim=10, output_dim=20) @@ -44,14 +48,14 @@ def forward(self, x): class TestNoSync(TestParallelDyGraphRunnerBase): - def get_model(self): model = SimpleNet() - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer def run_one_loop(self, model, optimizer, batch): @@ -67,7 +71,7 @@ def run_trainer_func(self, args): device_id = int(os.getenv("FLAGS_selected_gpus", "0")) place = fluid.CUDAPlace(device_id) else: - assert ("Only support CUDAPlace for now.") + assert "Only support CUDAPlace for now." with fluid.dygraph.guard(place): fluid.default_startup_program().random_seed = seed @@ -80,9 +84,11 @@ def run_trainer_func(self, args): dist.init_parallel_env() print_to_err( type(self).__name__, - "begin to prepare context in dygraph with nccl2") + "begin to prepare context in dygraph with nccl2", + ) model = paddle.DataParallel( - model, find_unused_parameters=args.find_unused_parameters) + model, find_unused_parameters=args.find_unused_parameters + ) print_to_err(type(self).__name__, "model built in dygraph") out_losses = self.model_train(args, model, opt, train_reader) print_to_out(out_losses) @@ -109,7 +115,8 @@ def run_trainer_with_spawn_func(self, args): model, train_reader, opt = self.get_model() if args.update_method in ["nccl2", "gloo"]: model = paddle.DataParallel( - model, find_unused_parameters=args.find_unused_parameters) + model, find_unused_parameters=args.find_unused_parameters + ) out_losses = self.model_train(args, model, opt, train_reader) print_to_out(out_losses) @@ -139,10 +146,9 @@ def model_train(self, args, model, opt, train_reader): def fake_sample_reader(): - def __reader__(): for i in range(batch_num): - x_data = np.random.random_sample((10, )).astype('float32') + x_data = np.random.random_sample((10,)).astype('float32') yield x_data return __reader__ diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_control_flow.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_control_flow.py index afbb4cf01229b..efe27d461ad6d 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_control_flow.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_control_flow.py @@ -27,7 +27,6 @@ class SimpleNetControlFlow(fluid.Layer): - def __init__(self): super(SimpleNetControlFlow, self).__init__() self.net_a = Linear(input_dim=10, output_dim=20) @@ -46,14 +45,14 @@ def forward(self, x): class TestNoSyncControlFlow(TestNoSync): - def get_model(self): model = SimpleNetControlFlow() - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer def run_one_loop(self, model, optimizer, batch): @@ -66,10 +65,9 @@ def run_one_loop(self, model, optimizer, batch): def fake_sample_reader(): - def __reader__(): for i in range(batch_num): - x_data = np.random.random_sample((10, )).astype('float32') + x_data = np.random.random_sample((10,)).astype('float32') yield x_data return __reader__ diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_gradient_check.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_gradient_check.py index 51f5a3650ce50..43c36ab491b01 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_gradient_check.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_gradient_check.py @@ -29,17 +29,19 @@ class SimpleNet(fluid.Layer): - def __init__(self, train_id): super(SimpleNet, self).__init__() - self.w1 = self.create_parameter(shape=[in_dim, out_dim], - dtype="float32") - self.w2 = self.create_parameter(shape=[in_dim, out_dim], - dtype="float32") + self.w1 = self.create_parameter( + shape=[in_dim, out_dim], dtype="float32" + ) + self.w2 = self.create_parameter( + shape=[in_dim, out_dim], dtype="float32" + ) self.share_net = Linear(out_dim, 1) - self.unused_param = self.create_parameter(shape=[out_dim, in_dim], - dtype="float32") + self.unused_param = self.create_parameter( + shape=[out_dim, in_dim], dtype="float32" + ) # just for test sync_params_buffers self.register_buffer("queue", paddle.randn([10, 5])) @@ -49,9 +51,10 @@ def __init__(self, train_id): self.trainer_id = train_id def forward(self, x): - is_use = (paddle.equal_all( - x, paddle.ones(shape=(batch, in_dim))).numpy()[0] - and self.trainer_id == 1) + is_use = ( + paddle.equal_all(x, paddle.ones(shape=(batch, in_dim))).numpy()[0] + and self.trainer_id == 1 + ) if is_use: tmp = paddle.matmul(x, self.w1) @@ -62,7 +65,6 @@ def forward(self, x): class TestDistTraning(unittest.TestCase): - def test_multiple_gpus(self): self.trainer_id = dist.get_rank() dist.init_parallel_env() @@ -85,11 +87,13 @@ def test_multiple_gpus(self): if step_id % 5 != 0: with model_a.no_sync(): - self.dp_layer(step_id, model_a, model_b, random_input, - ones_input) + self.dp_layer( + step_id, model_a, model_b, random_input, ones_input + ) else: - self.dp_layer(step_id, model_a, model_b, random_input, - ones_input) + self.dp_layer( + step_id, model_a, model_b, random_input, ones_input + ) self.check_gradient(model_a.parameters()) self.check_gradient(model_b.parameters()) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_unused_params.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_unused_params.py index bd8e598baf165..38804f11b2a87 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_unused_params.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_no_sync_unused_params.py @@ -27,7 +27,6 @@ class SimpleNetUnusedParam(fluid.Layer): - def __init__(self): super(SimpleNetUnusedParam, self).__init__() self.net_a = Linear(input_dim=10, output_dim=20) @@ -45,14 +44,14 @@ def forward(self, x): class TestNoSyncUnusedParam(TestNoSync): - def get_model(self): model = SimpleNetUnusedParam() - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer def run_one_loop(self, model, optimizer, batch): @@ -65,10 +64,9 @@ def run_one_loop(self, model, optimizer, batch): def fake_sample_reader(): - def __reader__(): for i in range(batch_num): - x_data = np.random.random_sample((10, )).astype('float32') + x_data = np.random.random_sample((10,)).astype('float32') yield x_data return __reader__ diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_se_resnext.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_se_resnext.py index 09878cec22974..0a0759f4380a1 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_se_resnext.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_se_resnext.py @@ -33,12 +33,12 @@ "name": "cosine_decay", "batch_size": batch_size, "epochs": [40, 80, 100], - "steps": [0.1, 0.01, 0.001, 0.0001] + "steps": [0.1, 0.01, 0.001, 0.0001], }, "batch_size": batch_size, "lr": 0.0125, "total_images": 6149, - "num_epochs": 200 + "num_epochs": 200, } @@ -56,42 +56,47 @@ def optimizer_setting(params, parameter_list=None): num_epochs = params["num_epochs"] if fluid._non_static_mode(): optimizer = fluid.optimizer.Momentum( - learning_rate=fluid.layers.cosine_decay(learning_rate=lr, - step_each_epoch=step, - epochs=num_epochs), + learning_rate=fluid.layers.cosine_decay( + learning_rate=lr, step_each_epoch=step, epochs=num_epochs + ), momentum=momentum_rate, regularization=fluid.regularizer.L2Decay(l2_decay), - parameter_list=parameter_list) + parameter_list=parameter_list, + ) else: optimizer = fluid.optimizer.Momentum( - learning_rate=fluid.layers.cosine_decay(learning_rate=lr, - step_each_epoch=step, - epochs=num_epochs), + learning_rate=fluid.layers.cosine_decay( + learning_rate=lr, step_each_epoch=step, epochs=num_epochs + ), momentum=momentum_rate, - regularization=fluid.regularizer.L2Decay(l2_decay)) + regularization=fluid.regularizer.L2Decay(l2_decay), + ) return optimizer class ConvBNLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - bias_attr=False) + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + bias_attr=False, + ) # disable BatchNorm in multi-card. disable LayerNorm because of complex input_shape # self._batch_norm = BatchNorm(num_filters, act=act) @@ -104,7 +109,6 @@ def forward(self, inputs): class SqueezeExcitation(fluid.dygraph.Layer): - def __init__(self, num_channels, reduction_ratio): super(SqueezeExcitation, self).__init__() @@ -115,15 +119,19 @@ def __init__(self, num_channels, reduction_ratio): num_channels, num_channels // reduction_ratio, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv)), - act='relu') + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + act='relu', + ) stdv = 1.0 / math.sqrt(num_channels / 16.0 * 1.0) self._excitation = Linear( num_channels // reduction_ratio, num_channels, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv)), - act='sigmoid') + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + act='sigmoid', + ) def forward(self, input): y = self._pool(input) @@ -135,39 +143,49 @@ def forward(self, input): class BottleneckBlock(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - stride, - cardinality, - reduction_ratio, - shortcut=True): + def __init__( + self, + num_channels, + num_filters, + stride, + cardinality, + reduction_ratio, + shortcut=True, + ): super(BottleneckBlock, self).__init__() - self.conv0 = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=1, - act="relu") - self.conv1 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=stride, - groups=cardinality, - act="relu") - self.conv2 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters * 2, - filter_size=1, - act=None) - - self.scale = SqueezeExcitation(num_channels=num_filters * 2, - reduction_ratio=reduction_ratio) + self.conv0 = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=1, + act="relu", + ) + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + groups=cardinality, + act="relu", + ) + self.conv2 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters * 2, + filter_size=1, + act=None, + ) + + self.scale = SqueezeExcitation( + num_channels=num_filters * 2, reduction_ratio=reduction_ratio + ) if not shortcut: - self.short = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters * 2, - filter_size=1, - stride=stride) + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters * 2, + filter_size=1, + stride=stride, + ) self.shortcut = shortcut @@ -189,67 +207,76 @@ def forward(self, inputs): class SeResNeXt(fluid.dygraph.Layer): - def __init__(self, layers=50, class_dim=102): super(SeResNeXt, self).__init__() self.layers = layers supported_layers = [50, 101, 152] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format(supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) if layers == 50: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 6, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) elif layers == 101: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 23, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) elif layers == 152: cardinality = 64 reduction_ratio = 16 depth = [3, 8, 36, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=3, - stride=2, - act='relu') - self.conv1 = ConvBNLayer(num_channels=64, - num_filters=64, - filter_size=3, - stride=1, - act='relu') - self.conv2 = ConvBNLayer(num_channels=64, - num_filters=128, - filter_size=3, - stride=1, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=3, + stride=2, + act='relu', + ) + self.conv1 = ConvBNLayer( + num_channels=64, + num_filters=64, + filter_size=3, + stride=1, + act='relu', + ) + self.conv2 = ConvBNLayer( + num_channels=64, + num_filters=128, + filter_size=3, + stride=1, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) self.bottleneck_block_list = [] num_channels = 64 @@ -258,19 +285,22 @@ def __init__(self, layers=50, class_dim=102): for i in range(depth[block]): bottleneck_block = self.add_sublayer( 'bb_%d_%d' % (block, i), - BottleneckBlock(num_channels=num_channels, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - cardinality=cardinality, - reduction_ratio=reduction_ratio, - shortcut=shortcut)) + BottleneckBlock( + num_channels=num_channels, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + cardinality=cardinality, + reduction_ratio=reduction_ratio, + shortcut=shortcut, + ), + ) num_channels = bottleneck_block._num_channels_out self.bottleneck_block_list.append(bottleneck_block) shortcut = True - self.pool2d_avg = Pool2D(pool_size=7, - pool_type='avg', - global_pooling=True) + self.pool2d_avg = Pool2D( + pool_size=7, pool_type='avg', global_pooling=True + ) stdv = 1.0 / math.sqrt(2048 * 1.0) self.pool2d_avg_output = num_filters[len(num_filters) - 1] * 2 * 1 * 1 @@ -279,7 +309,9 @@ def __init__(self, layers=50, class_dim=102): self.pool2d_avg_output, class_dim, param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv))) + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + ) def forward(self, inputs): if self.layers == 50 or self.layers == 101: @@ -300,20 +332,23 @@ def forward(self, inputs): class TestSeResNeXt(TestParallelDyGraphRunnerBase): - def get_model(self): model = SeResNeXt() - train_reader = paddle.batch(paddle.dataset.flowers.test(use_xmap=False), - batch_size=train_parameters["batch_size"], - drop_last=True) - optimizer = optimizer_setting(train_parameters, - parameter_list=model.parameters()) + train_reader = paddle.batch( + paddle.dataset.flowers.test(use_xmap=False), + batch_size=train_parameters["batch_size"], + drop_last=True, + ) + optimizer = optimizer_setting( + train_parameters, parameter_list=model.parameters() + ) return model, train_reader, optimizer def run_one_loop(self, model, opt, data): bs = len(data) - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') + dy_x_data = np.array([x[0].reshape(3, 224, 224) for x in data]).astype( + 'float32' + ) dy_x_data = dy_x_data / 255.0 y_data = np.array([x[1] for x in data]).astype('int64').reshape(bs, 1) img = to_variable(dy_x_data) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_sync_batch_norm.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_sync_batch_norm.py index 6d1f100252ca4..475eb1fa6d1d3 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_sync_batch_norm.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_sync_batch_norm.py @@ -23,37 +23,42 @@ class TestLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(TestLayer, self).__init__() - self._conv = Conv2D(in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=False) + self._conv = Conv2D( + in_channels=num_channels, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + bias_attr=False, + ) self._sync_batch_norm = SyncBatchNorm(num_filters) - self._conv2 = Conv2D(in_channels=num_filters, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=False) + self._conv2 = Conv2D( + in_channels=num_filters, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + bias_attr=False, + ) - self._sync_batch_norm2 = SyncBatchNorm(num_filters, - weight_attr=False, - bias_attr=False) + self._sync_batch_norm2 = SyncBatchNorm( + num_filters, weight_attr=False, bias_attr=False + ) def forward(self, inputs): y = self._conv(inputs) @@ -65,20 +70,23 @@ def forward(self, inputs): class TestSyncBatchNorm(TestParallelDyGraphRunnerBase): - def get_model(self): model = TestLayer(3, 64, 7) - train_reader = paddle.batch(paddle.dataset.flowers.test(use_xmap=False), - batch_size=32, - drop_last=True) - opt = fluid.optimizer.Adam(learning_rate=1e-3, - parameter_list=model.parameters()) + train_reader = paddle.batch( + paddle.dataset.flowers.test(use_xmap=False), + batch_size=32, + drop_last=True, + ) + opt = fluid.optimizer.Adam( + learning_rate=1e-3, parameter_list=model.parameters() + ) return model, train_reader, opt def run_one_loop(self, model, opt, data): batch_size = len(data) - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') + dy_x_data = np.array([x[0].reshape(3, 224, 224) for x in data]).astype( + 'float32' + ) img = to_variable(dy_x_data) img.stop_gradient = False diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_transformer.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_transformer.py index afe7799632d3a..97805222f8011 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_transformer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_dygraph_transformer.py @@ -16,10 +16,17 @@ import paddle import paddle.fluid as fluid -from paddle.fluid.dygraph import Embedding, LayerNorm, Linear, to_variable, Layer +from paddle.fluid.dygraph import ( + Embedding, + LayerNorm, + Linear, + to_variable, + Layer, +) from paddle.optimizer.lr import NoamDecay from test_dist_base import runtime_main, TestParallelDyGraphRunnerBase + """ Note(chenweihang): To compare loss of single-card and multi-card in our dist test framework, two parameters need to be adjusted: @@ -33,6 +40,7 @@ class TrainTaskConfig(object): """ TrainTaskConfig """ + # the epoch number to train. pass_num = 20 # the number of sequences contained in a mini-batch. @@ -117,12 +125,17 @@ class ModelHyperParams(object): # encoder. # The actual data shape of src_slf_attn_bias is: # [batch_size, n_head, max_src_len_in_batch, max_src_len_in_batch] - "src_slf_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "src_slf_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # The actual data shape of trg_word is: # [batch_size, max_trg_len_in_batch, 1] - "trg_word": [(batch_size, seq_len, 1), "int64", - 2], # lod_level is only used in fast decoder. + "trg_word": [ + (batch_size, seq_len, 1), + "int64", + 2, + ], # lod_level is only used in fast decoder. # The actual data shape of trg_pos is: # [batch_size, max_trg_len_in_batch, 1] "trg_pos": [(batch_size, seq_len, 1), "int64"], @@ -130,14 +143,18 @@ class ModelHyperParams(object): # subsequent words in the decoder. # The actual data shape of trg_slf_attn_bias is: # [batch_size, n_head, max_trg_len_in_batch, max_trg_len_in_batch] - "trg_slf_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "trg_slf_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # This input is used to remove attention weights on paddings of the source # input in the encoder-decoder attention. # The actual data shape of trg_src_attn_bias is: # [batch_size, n_head, max_trg_len_in_batch, max_src_len_in_batch] - "trg_src_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "trg_src_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # This input is used in independent decoder program for inference. # The actual data shape of enc_output is: # [batch_size, max_src_len_in_batch, d_model] @@ -153,7 +170,7 @@ class ModelHyperParams(object): "init_score": [(batch_size, 1), "float32", 2], # This input is used in beam-search decoder for the first gather # (cell states updation) - "init_idx": [(batch_size, ), "int32"], + "init_idx": [(batch_size,), "int32"], } # Names of word embedding table which might be reused for weight sharing. @@ -200,26 +217,30 @@ def position_encoding_init(n_position, d_pos_vec): channels = d_pos_vec position = np.arange(n_position) num_timescales = channels // 2 - log_timescale_increment = (np.log(float(1e4) / float(1)) / - (num_timescales - 1)) - inv_timescales = np.exp( - np.arange(num_timescales)) * -log_timescale_increment + log_timescale_increment = np.log(float(1e4) / float(1)) / ( + num_timescales - 1 + ) + inv_timescales = ( + np.exp(np.arange(num_timescales)) * -log_timescale_increment + ) scaled_time = np.expand_dims(position, 1) * np.expand_dims( - inv_timescales, 0) + inv_timescales, 0 + ) signal = np.concatenate([np.sin(scaled_time), np.cos(scaled_time)], axis=1) signal = np.pad(signal, [[0, 0], [0, np.mod(channels, 2)]], 'constant') position_enc = signal return position_enc.astype("float32") -pos_inp1 = position_encoding_init(ModelHyperParams.max_length, - ModelHyperParams.d_model) -pos_inp2 = position_encoding_init(ModelHyperParams.max_length, - ModelHyperParams.d_model) +pos_inp1 = position_encoding_init( + ModelHyperParams.max_length, ModelHyperParams.d_model +) +pos_inp2 = position_encoding_init( + ModelHyperParams.max_length, ModelHyperParams.d_model +) class PrePostProcessLayer(Layer): - def __init__(self, d_model, process_cmd, shape_len=None): super(PrePostProcessLayer, self).__init__() for cmd in process_cmd: @@ -227,11 +248,14 @@ def __init__(self, d_model, process_cmd, shape_len=None): self._layer_norm = LayerNorm( normalized_shape=d_model, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(1.)), + initializer=fluid.initializer.Constant(1.0) + ), bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.))) + initializer=fluid.initializer.Constant(0.0) + ), + ) - def forward(self, prev_out, out, process_cmd, dropout_rate=0.): + def forward(self, prev_out, out, process_cmd, dropout_rate=0.0): for cmd in process_cmd: if cmd == "a": # add residual connection out = out + prev_out if prev_out is not None else out @@ -243,12 +267,12 @@ def forward(self, prev_out, out, process_cmd, dropout_rate=0.): out, dropout_prob=dropout_rate, seed=ModelHyperParams.dropout_seed, - is_test=False) + is_test=False, + ) return out class PositionwiseFeedForwardLayer(Layer): - def __init__(self, d_inner_hid, d_hid, dropout_rate): super(PositionwiseFeedForwardLayer, self).__init__() self._i2h = Linear(d_hid, d_inner_hid, act="relu") @@ -258,25 +282,28 @@ def __init__(self, d_inner_hid, d_hid, dropout_rate): def forward(self, x): hidden = self._i2h(x) if self._dropout_rate: - hidden = fluid.layers.dropout(hidden, - dropout_prob=self._dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) + hidden = fluid.layers.dropout( + hidden, + dropout_prob=self._dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False, + ) out = self._h2o(hidden) return out class MultiHeadAttentionLayer(Layer): - - def __init__(self, - d_key, - d_value, - d_model, - n_head=1, - dropout_rate=0., - cache=None, - gather_idx=None, - static_kv=False): + def __init__( + self, + d_key, + d_value, + d_model, + n_head=1, + dropout_rate=0.0, + cache=None, + gather_idx=None, + static_kv=False, + ): super(MultiHeadAttentionLayer, self).__init__() self._n_head = n_head self._d_key = d_key @@ -299,20 +326,25 @@ def forward(self, queries, keys, values, attn_bias): # split head reshaped_q = fluid.layers.reshape( - x=q, shape=[0, 0, self._n_head, self._d_key], inplace=False) + x=q, shape=[0, 0, self._n_head, self._d_key], inplace=False + ) transpose_q = fluid.layers.transpose(x=reshaped_q, perm=[0, 2, 1, 3]) reshaped_k = fluid.layers.reshape( - x=k, shape=[0, 0, self._n_head, self._d_key], inplace=False) + x=k, shape=[0, 0, self._n_head, self._d_key], inplace=False + ) transpose_k = fluid.layers.transpose(x=reshaped_k, perm=[0, 2, 1, 3]) reshaped_v = fluid.layers.reshape( - x=v, shape=[0, 0, self._n_head, self._d_value], inplace=False) + x=v, shape=[0, 0, self._n_head, self._d_value], inplace=False + ) transpose_v = fluid.layers.transpose(x=reshaped_v, perm=[0, 2, 1, 3]) # scale dot product attention - product = fluid.layers.matmul(x=transpose_q, - y=transpose_k, - transpose_y=True, - alpha=self._d_model**-0.5) + product = fluid.layers.matmul( + x=transpose_q, + y=transpose_k, + transpose_y=True, + alpha=self._d_model**-0.5, + ) if attn_bias is not None: product += attn_bias weights = fluid.layers.softmax(product) @@ -321,7 +353,8 @@ def forward(self, queries, keys, values, attn_bias): weights, dropout_prob=self._dropout_rate, seed=ModelHyperParams.dropout_seed, - is_test=False) + is_test=False, + ) out = fluid.layers.matmul(weights_droped, transpose_v) else: out = fluid.layers.matmul(weights, transpose_v) @@ -333,7 +366,8 @@ def forward(self, queries, keys, values, attn_bias): final_out = fluid.layers.reshape( x=trans_x, shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]], - inplace=False) + inplace=False, + ) # fc to output proj_out = self._proj_fc(final_out) @@ -341,118 +375,150 @@ def forward(self, queries, keys, values, attn_bias): class EncoderSubLayer(Layer): - - def __init__(self, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): + def __init__( + self, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + ): super(EncoderSubLayer, self).__init__() self._preprocess_cmd = preprocess_cmd self._postprocess_cmd = postprocess_cmd self._prepostprocess_dropout = prepostprocess_dropout - self._preprocess_layer = PrePostProcessLayer(d_model, - self._preprocess_cmd, 3) + self._preprocess_layer = PrePostProcessLayer( + d_model, self._preprocess_cmd, 3 + ) self._multihead_attention_layer = MultiHeadAttentionLayer( - d_key, d_value, d_model, n_head, attention_dropout) - self._postprocess_layer = PrePostProcessLayer(d_model, - self._postprocess_cmd, - None) - self._preprocess_layer2 = PrePostProcessLayer(d_model, - self._preprocess_cmd, 3) + d_key, d_value, d_model, n_head, attention_dropout + ) + self._postprocess_layer = PrePostProcessLayer( + d_model, self._postprocess_cmd, None + ) + self._preprocess_layer2 = PrePostProcessLayer( + d_model, self._preprocess_cmd, 3 + ) self._positionwise_feed_forward = PositionwiseFeedForwardLayer( - d_inner_hid, d_model, relu_dropout) - self._postprocess_layer2 = PrePostProcessLayer(d_model, - self._postprocess_cmd, - None) + d_inner_hid, d_model, relu_dropout + ) + self._postprocess_layer2 = PrePostProcessLayer( + d_model, self._postprocess_cmd, None + ) def forward(self, enc_input, attn_bias): pre_process_multihead = self._preprocess_layer( - None, enc_input, self._preprocess_cmd, self._prepostprocess_dropout) - attn_output = self._multihead_attention_layer(pre_process_multihead, - None, None, attn_bias) - attn_output = self._postprocess_layer(enc_input, attn_output, - self._postprocess_cmd, - self._prepostprocess_dropout) + None, enc_input, self._preprocess_cmd, self._prepostprocess_dropout + ) + attn_output = self._multihead_attention_layer( + pre_process_multihead, None, None, attn_bias + ) + attn_output = self._postprocess_layer( + enc_input, + attn_output, + self._postprocess_cmd, + self._prepostprocess_dropout, + ) pre_process2_output = self._preprocess_layer2( - None, attn_output, self._preprocess_cmd, - self._prepostprocess_dropout) + None, + attn_output, + self._preprocess_cmd, + self._prepostprocess_dropout, + ) ffd_output = self._positionwise_feed_forward(pre_process2_output) - return self._postprocess_layer2(attn_output, ffd_output, - self._postprocess_cmd, - self._prepostprocess_dropout) + return self._postprocess_layer2( + attn_output, + ffd_output, + self._postprocess_cmd, + self._prepostprocess_dropout, + ) class EncoderLayer(Layer): - - def __init__(self, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): + def __init__( + self, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + ): super(EncoderLayer, self).__init__() self._preprocess_cmd = preprocess_cmd self._encoder_sublayers = list() self._prepostprocess_dropout = prepostprocess_dropout self._n_layer = n_layer - self._preprocess_layer = PrePostProcessLayer(d_model, - self._preprocess_cmd, 3) + self._preprocess_layer = PrePostProcessLayer( + d_model, self._preprocess_cmd, 3 + ) for i in range(n_layer): self._encoder_sublayers.append( self.add_sublayer( 'esl_%d' % i, - EncoderSubLayer(n_head, d_key, d_value, d_model, - d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, - preprocess_cmd, postprocess_cmd))) + EncoderSubLayer( + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ), + ) + ) def forward(self, enc_input, attn_bias): for i in range(self._n_layer): enc_output = self._encoder_sublayers[i](enc_input, attn_bias) enc_input = enc_output - return self._preprocess_layer(None, enc_output, self._preprocess_cmd, - self._prepostprocess_dropout) + return self._preprocess_layer( + None, enc_output, self._preprocess_cmd, self._prepostprocess_dropout + ) class PrepareEncoderDecoderLayer(Layer): - - def __init__(self, - src_vocab_size, - src_emb_dim, - src_max_len, - dropout_rate, - is_sparse=False, - word_emb_param_name=None, - pos_enc_param_name=None): + def __init__( + self, + src_vocab_size, + src_emb_dim, + src_max_len, + dropout_rate, + is_sparse=False, + word_emb_param_name=None, + pos_enc_param_name=None, + ): super(PrepareEncoderDecoderLayer, self).__init__() self._src_max_len = src_max_len self._src_emb_dim = src_emb_dim self._src_vocab_size = src_vocab_size self._dropout_rate = dropout_rate - self._input_emb = Embedding(size=[src_vocab_size, src_emb_dim], - is_sparse=is_sparse, - padding_idx=0, - param_attr=fluid.ParamAttr( - name=word_emb_param_name, - initializer=fluid.initializer.Normal( - 0., src_emb_dim**-0.5))) + self._input_emb = Embedding( + size=[src_vocab_size, src_emb_dim], + is_sparse=is_sparse, + padding_idx=0, + param_attr=fluid.ParamAttr( + name=word_emb_param_name, + initializer=fluid.initializer.Normal(0.0, src_emb_dim**-0.5), + ), + ) if pos_enc_param_name is pos_enc_param_names[0]: pos_inp = pos_inp1 @@ -464,41 +530,50 @@ def __init__(self, param_attr=fluid.ParamAttr( name=pos_enc_param_name, initializer=fluid.initializer.NumpyArrayInitializer(pos_inp), - trainable=False)) + trainable=False, + ), + ) def forward(self, src_word, src_pos): src_word_emb = self._input_emb(src_word) - src_word_emb = fluid.layers.scale(x=src_word_emb, - scale=self._src_emb_dim**0.5) + src_word_emb = fluid.layers.scale( + x=src_word_emb, scale=self._src_emb_dim**0.5 + ) # # TODO change this to fit dynamic length input src_pos_emb = self._pos_emb(src_pos) src_pos_emb.stop_gradient = True enc_input = src_word_emb + src_pos_emb - return fluid.layers.dropout( - enc_input, - dropout_prob=self._dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) if self._dropout_rate else enc_input + return ( + fluid.layers.dropout( + enc_input, + dropout_prob=self._dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False, + ) + if self._dropout_rate + else enc_input + ) class WrapEncoderLayer(Layer): - - def __init__(self, - src_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - is_sparse=False): + def __init__( + self, + src_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + is_sparse=False, + ): """ The wrapper assembles together all needed layers for the encoder. """ @@ -511,11 +586,21 @@ def __init__(self, prepostprocess_dropout, is_sparse=is_sparse, word_emb_param_name=word_emb_param_names[0], - pos_enc_param_name=pos_enc_param_names[0]) - self._encoder = EncoderLayer(n_layer, n_head, d_key, d_value, d_model, - d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, - preprocess_cmd, postprocess_cmd) + pos_enc_param_name=pos_enc_param_names[0], + ) + self._encoder = EncoderLayer( + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ) def forward(self, enc_inputs): src_word, src_pos, src_slf_attn_bias = enc_inputs @@ -525,26 +610,28 @@ def forward(self, enc_inputs): class DecoderSubLayer(Layer): - - def __init__(self, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - cache=None, - gather_idx=None): + def __init__( + self, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + cache=None, + gather_idx=None, + ): super(DecoderSubLayer, self).__init__() self._postprocess_cmd = postprocess_cmd self._preprocess_cmd = preprocess_cmd self._prepostprcess_dropout = prepostprocess_dropout - self._pre_process_layer = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + self._pre_process_layer = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._multihead_attention_layer = MultiHeadAttentionLayer( d_key, d_value, @@ -552,11 +639,14 @@ def __init__(self, n_head, attention_dropout, cache=cache, - gather_idx=gather_idx) - self._post_process_layer = PrePostProcessLayer(d_model, postprocess_cmd, - None) - self._pre_process_layer2 = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + gather_idx=gather_idx, + ) + self._post_process_layer = PrePostProcessLayer( + d_model, postprocess_cmd, None + ) + self._pre_process_layer2 = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._multihead_attention_layer2 = MultiHeadAttentionLayer( d_key, d_value, @@ -565,63 +655,86 @@ def __init__(self, attention_dropout, cache=cache, gather_idx=gather_idx, - static_kv=True) - self._post_process_layer2 = PrePostProcessLayer(d_model, - postprocess_cmd, None) - self._pre_process_layer3 = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + static_kv=True, + ) + self._post_process_layer2 = PrePostProcessLayer( + d_model, postprocess_cmd, None + ) + self._pre_process_layer3 = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._positionwise_feed_forward_layer = PositionwiseFeedForwardLayer( - d_inner_hid, d_model, relu_dropout) - self._post_process_layer3 = PrePostProcessLayer(d_model, - postprocess_cmd, None) + d_inner_hid, d_model, relu_dropout + ) + self._post_process_layer3 = PrePostProcessLayer( + d_model, postprocess_cmd, None + ) def forward(self, dec_input, enc_output, slf_attn_bias, dec_enc_attn_bias): - pre_process_rlt = self._pre_process_layer(None, dec_input, - self._preprocess_cmd, - self._prepostprcess_dropout) + pre_process_rlt = self._pre_process_layer( + None, dec_input, self._preprocess_cmd, self._prepostprcess_dropout + ) slf_attn_output = self._multihead_attention_layer( - pre_process_rlt, None, None, slf_attn_bias) + pre_process_rlt, None, None, slf_attn_bias + ) slf_attn_output_pp = self._post_process_layer( - dec_input, slf_attn_output, self._postprocess_cmd, - self._prepostprcess_dropout) - pre_process_rlt2 = self._pre_process_layer2(None, slf_attn_output_pp, - self._preprocess_cmd, - self._prepostprcess_dropout) + dec_input, + slf_attn_output, + self._postprocess_cmd, + self._prepostprcess_dropout, + ) + pre_process_rlt2 = self._pre_process_layer2( + None, + slf_attn_output_pp, + self._preprocess_cmd, + self._prepostprcess_dropout, + ) enc_attn_output_pp = self._multihead_attention_layer2( - pre_process_rlt2, enc_output, enc_output, dec_enc_attn_bias) - enc_attn_output = self._post_process_layer2(slf_attn_output_pp, - enc_attn_output_pp, - self._postprocess_cmd, - self._prepostprcess_dropout) - pre_process_rlt3 = self._pre_process_layer3(None, enc_attn_output, - self._preprocess_cmd, - self._prepostprcess_dropout) + pre_process_rlt2, enc_output, enc_output, dec_enc_attn_bias + ) + enc_attn_output = self._post_process_layer2( + slf_attn_output_pp, + enc_attn_output_pp, + self._postprocess_cmd, + self._prepostprcess_dropout, + ) + pre_process_rlt3 = self._pre_process_layer3( + None, + enc_attn_output, + self._preprocess_cmd, + self._prepostprcess_dropout, + ) ffd_output = self._positionwise_feed_forward_layer(pre_process_rlt3) - dec_output = self._post_process_layer3(enc_attn_output, ffd_output, - self._postprocess_cmd, - self._prepostprcess_dropout) + dec_output = self._post_process_layer3( + enc_attn_output, + ffd_output, + self._postprocess_cmd, + self._prepostprcess_dropout, + ) return dec_output class DecoderLayer(Layer): - - def __init__(self, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - caches=None, - gather_idx=None): + def __init__( + self, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + caches=None, + gather_idx=None, + ): super(DecoderLayer, self).__init__() - self._pre_process_layer = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + self._pre_process_layer = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._decoder_sub_layers = list() self._n_layer = n_layer self._preprocess_cmd = preprocess_cmd @@ -630,53 +743,62 @@ def __init__(self, self._decoder_sub_layers.append( self.add_sublayer( 'dsl_%d' % i, - DecoderSubLayer(n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - cache=None if caches is None else caches[i], - gather_idx=gather_idx))) - - def forward(self, dec_input, enc_output, dec_slf_attn_bias, - dec_enc_attn_bias): + DecoderSubLayer( + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + cache=None if caches is None else caches[i], + gather_idx=gather_idx, + ), + ) + ) + + def forward( + self, dec_input, enc_output, dec_slf_attn_bias, dec_enc_attn_bias + ): for i in range(self._n_layer): - tmp_dec_output = self._decoder_sub_layers[i](dec_input, enc_output, - dec_slf_attn_bias, - dec_enc_attn_bias) + tmp_dec_output = self._decoder_sub_layers[i]( + dec_input, enc_output, dec_slf_attn_bias, dec_enc_attn_bias + ) dec_input = tmp_dec_output - dec_output = self._pre_process_layer(None, tmp_dec_output, - self._preprocess_cmd, - self._prepostprocess_dropout) + dec_output = self._pre_process_layer( + None, + tmp_dec_output, + self._preprocess_cmd, + self._prepostprocess_dropout, + ) return dec_output class WrapDecoderLayer(Layer): - - def __init__(self, - trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - caches=None, - gather_idx=None, - is_sparse=False): + def __init__( + self, + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + caches=None, + gather_idx=None, + is_sparse=False, + ): """ The wrapper assembles together all needed layers for the encoder. """ @@ -689,20 +811,23 @@ def __init__(self, prepostprocess_dropout, is_sparse=is_sparse, word_emb_param_name=word_emb_param_names[1], - pos_enc_param_name=pos_enc_param_names[1]) - self._decoder_layer = DecoderLayer(n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - caches=caches, - gather_idx=gather_idx) + pos_enc_param_name=pos_enc_param_names[1], + ) + self._decoder_layer = DecoderLayer( + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + caches=caches, + gather_idx=gather_idx, + ) self._weight_sharing = weight_sharing if not weight_sharing: self._fc = Linear(d_model, trg_vocab_size, bias_attr=False) @@ -710,17 +835,20 @@ def __init__(self, def forward(self, dec_inputs=None, enc_output=None): trg_word, trg_pos, trg_slf_attn_bias, trg_src_attn_bias = dec_inputs dec_input = self._prepare_decoder_layer(trg_word, trg_pos) - dec_output = self._decoder_layer(dec_input, enc_output, - trg_slf_attn_bias, trg_src_attn_bias) + dec_output = self._decoder_layer( + dec_input, enc_output, trg_slf_attn_bias, trg_src_attn_bias + ) dec_output_reshape = fluid.layers.reshape( - dec_output, shape=[-1, dec_output.shape[-1]], inplace=False) + dec_output, shape=[-1, dec_output.shape[-1]], inplace=False + ) if self._weight_sharing: predict = fluid.layers.matmul( x=dec_output_reshape, y=self._prepare_decoder_layer._input_emb.weight, - transpose_y=True) + transpose_y=True, + ) else: predict = self._fc(dec_output_reshape) @@ -732,81 +860,91 @@ def forward(self, dec_inputs=None, enc_output=None): class TransFormer(Layer): - - def __init__(self, - src_vocab_size, - trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - label_smooth_eps, - use_py_reader=False, - is_test=False, - is_sparse=False): + def __init__( + self, + src_vocab_size, + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + label_smooth_eps, + use_py_reader=False, + is_test=False, + is_sparse=False, + ): super(TransFormer, self).__init__() self._label_smooth_eps = label_smooth_eps self._trg_vocab_size = trg_vocab_size if weight_sharing: - assert src_vocab_size == trg_vocab_size, ( - "Vocabularies in source and target should be same for weight sharing." - ) - self._wrap_encoder_layer = WrapEncoderLayer(src_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - is_sparse=is_sparse) - self._wrap_decoder_layer = WrapDecoderLayer(trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - is_sparse=is_sparse) + assert ( + src_vocab_size == trg_vocab_size + ), "Vocabularies in source and target should be same for weight sharing." + self._wrap_encoder_layer = WrapEncoderLayer( + src_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + is_sparse=is_sparse, + ) + self._wrap_decoder_layer = WrapDecoderLayer( + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + is_sparse=is_sparse, + ) if weight_sharing: - self._wrap_decoder_layer._prepare_decoder_layer._input_emb.weight = self._wrap_encoder_layer._prepare_encoder_layer._input_emb.weight + self._wrap_decoder_layer._prepare_decoder_layer._input_emb.weight = ( + self._wrap_encoder_layer._prepare_encoder_layer._input_emb.weight + ) def forward(self, enc_inputs, dec_inputs, label, weights): enc_output = self._wrap_encoder_layer(enc_inputs) predict = self._wrap_decoder_layer(dec_inputs, enc_output) if self._label_smooth_eps: label_out = fluid.layers.label_smooth( - label=fluid.layers.one_hot(input=label, - depth=self._trg_vocab_size), - epsilon=self._label_smooth_eps) + label=fluid.layers.one_hot( + input=label, depth=self._trg_vocab_size + ), + epsilon=self._label_smooth_eps, + ) cost = fluid.layers.softmax_with_cross_entropy( logits=predict, label=label_out, - soft_label=True if self._label_smooth_eps else False) + soft_label=True if self._label_smooth_eps else False, + ) weighted_cost = cost * weights sum_cost = fluid.layers.reduce_sum(weighted_cost) token_num = fluid.layers.reduce_sum(weights) @@ -820,47 +958,54 @@ def forward(self, enc_inputs, dec_inputs, label, weights): def fake_data_reader(): - def __reader__(): iteration = TrainTaskConfig.batch_size * batch_num for _ in range(iteration): # random data np.random.seed = 90 - src_word_np = np.arange(1, seq_len + 1).reshape([seq_len - ]).astype('int64') - src_pos_np = np.random.randint(1, - seq_len, - size=(seq_len), - dtype='int64') - src_slf_attn_bias_np = np.random.randn(ModelHyperParams.n_head, - seq_len, - seq_len).astype('float32') - - trg_word_np = np.arange(1, seq_len + 1).reshape([seq_len - ]).astype('int64') - trg_pos_np = np.random.randint(1, - seq_len, - size=(seq_len), - dtype='int64') - trg_slf_attn_bias_np = np.random.randn(ModelHyperParams.n_head, - seq_len, - seq_len).astype('float32') - trg_src_attn_bias_np = np.random.randn(ModelHyperParams.n_head, - seq_len, - seq_len).astype('float32') - - lbl_word_np = np.random.randint(1, - ModelHyperParams.src_vocab_size - 1, - size=(seq_len, 1), - dtype='int64') + src_word_np = ( + np.arange(1, seq_len + 1).reshape([seq_len]).astype('int64') + ) + src_pos_np = np.random.randint( + 1, seq_len, size=(seq_len), dtype='int64' + ) + src_slf_attn_bias_np = np.random.randn( + ModelHyperParams.n_head, seq_len, seq_len + ).astype('float32') + + trg_word_np = ( + np.arange(1, seq_len + 1).reshape([seq_len]).astype('int64') + ) + trg_pos_np = np.random.randint( + 1, seq_len, size=(seq_len), dtype='int64' + ) + trg_slf_attn_bias_np = np.random.randn( + ModelHyperParams.n_head, seq_len, seq_len + ).astype('float32') + trg_src_attn_bias_np = np.random.randn( + ModelHyperParams.n_head, seq_len, seq_len + ).astype('float32') + + lbl_word_np = np.random.randint( + 1, + ModelHyperParams.src_vocab_size - 1, + size=(seq_len, 1), + dtype='int64', + ) # Note(chenweihang): weight will introduce diff, so use constant here lbl_weight_np = np.ones((seq_len, 1)).astype('int64') data_inputs = [ - src_word_np, src_pos_np, src_slf_attn_bias_np, trg_word_np, - trg_pos_np, trg_slf_attn_bias_np, trg_src_attn_bias_np, - lbl_word_np, lbl_weight_np + src_word_np, + src_pos_np, + src_slf_attn_bias_np, + trg_word_np, + trg_pos_np, + trg_slf_attn_bias_np, + trg_src_attn_bias_np, + lbl_word_np, + lbl_weight_np, ] yield data_inputs @@ -884,20 +1029,30 @@ def np_to_variable(data): lbl_weight_np = lbl_weight_np.reshape(batch_size * seq_len, 1) data_inputs = [ - src_word_np, src_pos_np, src_slf_attn_bias_np, trg_word_np, trg_pos_np, - trg_slf_attn_bias_np, trg_src_attn_bias_np, lbl_word_np, lbl_weight_np + src_word_np, + src_pos_np, + src_slf_attn_bias_np, + trg_word_np, + trg_pos_np, + trg_slf_attn_bias_np, + trg_src_attn_bias_np, + lbl_word_np, + lbl_weight_np, ] var_inputs = [] - for i, field in enumerate(encoder_data_input_fields + - decoder_data_input_fields[:-1] + - label_data_input_fields): + for i, field in enumerate( + encoder_data_input_fields + + decoder_data_input_fields[:-1] + + label_data_input_fields + ): var_inputs.append(to_variable(data_inputs[i], name=field)) - enc_inputs = var_inputs[0:len(encoder_data_input_fields)] - dec_inputs = var_inputs[len(encoder_data_input_fields - ):len(encoder_data_input_fields) + - len(decoder_data_input_fields[:-1])] + enc_inputs = var_inputs[0 : len(encoder_data_input_fields)] + dec_inputs = var_inputs[ + len(encoder_data_input_fields) : len(encoder_data_input_fields) + + len(decoder_data_input_fields[:-1]) + ] label = var_inputs[-2] weights = var_inputs[-1] @@ -908,38 +1063,45 @@ def np_to_variable(data): class TestTransformer(TestParallelDyGraphRunnerBase): - def get_model(self): - model = TransFormer(ModelHyperParams.src_vocab_size, - ModelHyperParams.trg_vocab_size, - ModelHyperParams.max_length + 1, - ModelHyperParams.n_layer, - ModelHyperParams.n_head, - ModelHyperParams.d_key, - ModelHyperParams.d_value, - ModelHyperParams.d_model, - ModelHyperParams.d_inner_hid, - ModelHyperParams.prepostprocess_dropout, - ModelHyperParams.attention_dropout, - ModelHyperParams.relu_dropout, - ModelHyperParams.preprocess_cmd, - ModelHyperParams.postprocess_cmd, - ModelHyperParams.weight_sharing, - TrainTaskConfig.label_smooth_eps, - is_sparse=True) - train_reader = paddle.batch(fake_data_reader(), - TrainTaskConfig.batch_size) + model = TransFormer( + ModelHyperParams.src_vocab_size, + ModelHyperParams.trg_vocab_size, + ModelHyperParams.max_length + 1, + ModelHyperParams.n_layer, + ModelHyperParams.n_head, + ModelHyperParams.d_key, + ModelHyperParams.d_value, + ModelHyperParams.d_model, + ModelHyperParams.d_inner_hid, + ModelHyperParams.prepostprocess_dropout, + ModelHyperParams.attention_dropout, + ModelHyperParams.relu_dropout, + ModelHyperParams.preprocess_cmd, + ModelHyperParams.postprocess_cmd, + ModelHyperParams.weight_sharing, + TrainTaskConfig.label_smooth_eps, + is_sparse=True, + ) + train_reader = paddle.batch( + fake_data_reader(), TrainTaskConfig.batch_size + ) if naive_optimize: - optimizer = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=model.parameters()) + optimizer = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=model.parameters() + ) else: - optimizer = fluid.optimizer.Adam(learning_rate=NoamDecay( - ModelHyperParams.d_model, TrainTaskConfig.warmup_steps, - TrainTaskConfig.learning_rate), - beta1=TrainTaskConfig.beta1, - beta2=TrainTaskConfig.beta2, - epsilon=TrainTaskConfig.eps, - parameter_list=model.parameters()) + optimizer = fluid.optimizer.Adam( + learning_rate=NoamDecay( + ModelHyperParams.d_model, + TrainTaskConfig.warmup_steps, + TrainTaskConfig.learning_rate, + ), + beta1=TrainTaskConfig.beta1, + beta2=TrainTaskConfig.beta2, + epsilon=TrainTaskConfig.eps, + parameter_list=model.parameters(), + ) return model, train_reader, optimizer @@ -947,7 +1109,8 @@ def run_one_loop(self, model, optimizer, batch): enc_inputs, dec_inputs, label, weights = np_to_variable(batch) dy_sum_cost, dy_avg_cost, dy_predict, dy_token_num = model( - enc_inputs, dec_inputs, label, weights) + enc_inputs, dec_inputs, label, weights + ) return dy_avg_cost diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_margin_cross_entropy.py b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_margin_cross_entropy.py index 9d852785f6db0..343cce6f4bb3b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/parallel_margin_cross_entropy.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/parallel_margin_cross_entropy.py @@ -30,7 +30,6 @@ def set_random_seed(seed): class TestParallelMarginSoftmaxCrossEntropyOp(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() fleet.init(is_collective=True, strategy=strategy) @@ -60,57 +59,71 @@ def test_parallel_margin_softmax_cross_entropy(self): num_class = np.sum(num_class_per_card) for margin1, margin2, margin3, scale in zip( - margin1s, margin2s, margin3s, scales): + margin1s, margin2s, margin3s, scales + ): for _ in range(5): - np_label = np.random.randint(0, num_class, - (batch_size, )) + np_label = np.random.randint( + 0, num_class, (batch_size,) + ) label = paddle.to_tensor(np_label, dtype="int64") - input = paddle.randn(shape=[batch_size, feature_length], - dtype=dtype) + input = paddle.randn( + shape=[batch_size, feature_length], dtype=dtype + ) input.stop_gradient = False input_l2 = paddle.sqrt( - paddle.sum(paddle.square(input), - axis=1, - keepdim=True)) + paddle.sum( + paddle.square(input), axis=1, keepdim=True + ) + ) norm_input = paddle.divide(input, input_l2) weight = paddle.randn( shape=[feature_length, num_class_per_card[rank_id]], - dtype=dtype) + dtype=dtype, + ) weight.stop_gradient = False weight_l2 = paddle.sqrt( - paddle.sum(paddle.square(weight), - axis=0, - keepdim=True)) + paddle.sum( + paddle.square(weight), axis=0, keepdim=True + ) + ) norm_weight = paddle.divide(weight, weight_l2) data = paddle.matmul(norm_input, norm_weight) data.stop_gradient = False - sta = np.sum( - num_class_per_card[:rank_id]) if rank_id > 0 else 0 - end = np.sum(num_class_per_card[:rank_id + 1]) - - integral_data = np.zeros((batch_size, num_class), - dtype=dtype) - integral_data[:, - sta:end] = data.clone().detach().numpy() - integral_data = paddle.to_tensor(integral_data, - dtype=dtype) + sta = ( + np.sum(num_class_per_card[:rank_id]) + if rank_id > 0 + else 0 + ) + end = np.sum(num_class_per_card[: rank_id + 1]) + + integral_data = np.zeros( + (batch_size, num_class), dtype=dtype + ) + integral_data[:, sta:end] = ( + data.clone().detach().numpy() + ) + integral_data = paddle.to_tensor( + integral_data, dtype=dtype + ) paddle.distributed.all_reduce( integral_data, op=paddle.distributed.ReduceOp.SUM, - group=check_group) + group=check_group, + ) integral_data = integral_data.detach().clone() integral_data.stop_gradient = False # add arcface margin to logit theta = paddle.acos(integral_data) one_hot_label = paddle.nn.functional.one_hot( - label, num_classes=num_class) + label, num_classes=num_class + ) one_hot_label.stop_gradient = False if margin1 != 1.0: @@ -123,7 +136,10 @@ def test_parallel_margin_softmax_cross_entropy(self): diff = one_hot_label * (margin_cos - integral_data) arc_data = (integral_data + diff) * scale - loss_a, softmax_a = paddle.nn.functional.margin_cross_entropy( + ( + loss_a, + softmax_a, + ) = paddle.nn.functional.margin_cross_entropy( data, label, margin1=margin1, @@ -132,54 +148,69 @@ def test_parallel_margin_softmax_cross_entropy(self): scale=scale, group=check_group, return_softmax=True, - reduction=None) - loss_b, softmax_b = paddle.nn.functional.softmax_with_cross_entropy( + reduction=None, + ) + ( + loss_b, + softmax_b, + ) = paddle.nn.functional.softmax_with_cross_entropy( logits=arc_data, label=paddle.reshape(label, (-1, 1)), - return_softmax=True) - - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=1e-5, - atol=1e-7) - - integral_prob = np.zeros((batch_size, num_class), - dtype=dtype) - integral_prob[:, sta:end] = softmax_a.clone().detach( - ).numpy() - integral_prob = paddle.to_tensor(integral_prob, - dtype=dtype) + return_softmax=True, + ) + + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=1e-5, atol=1e-7 + ) + + integral_prob = np.zeros( + (batch_size, num_class), dtype=dtype + ) + integral_prob[:, sta:end] = ( + softmax_a.clone().detach().numpy() + ) + integral_prob = paddle.to_tensor( + integral_prob, dtype=dtype + ) paddle.distributed.all_reduce( integral_prob, op=paddle.distributed.ReduceOp.SUM, - group=check_group) + group=check_group, + ) integral_prob = integral_prob.detach().clone() integral_prob.stop_gradient = False - np.testing.assert_allclose(integral_prob.numpy(), - softmax_b.numpy(), - rtol=1e-5, - atol=1e-6) + np.testing.assert_allclose( + integral_prob.numpy(), + softmax_b.numpy(), + rtol=1e-5, + atol=1e-6, + ) loss_a = loss_a.sum() / batch_size loss_b = loss_b.sum() / batch_size loss_a.backward() loss_b.backward() - integral_grad = np.zeros((batch_size, num_class), - dtype=dtype) + integral_grad = np.zeros( + (batch_size, num_class), dtype=dtype + ) integral_grad[:, sta:end] = data.grad.clone().detach() - integral_grad = paddle.to_tensor(integral_grad, - dtype=dtype) + integral_grad = paddle.to_tensor( + integral_grad, dtype=dtype + ) paddle.distributed.all_reduce( integral_grad, op=paddle.distributed.ReduceOp.SUM, - group=check_group) - - np.testing.assert_allclose(integral_data.grad.numpy(), - integral_grad.numpy(), - rtol=1e-5, - atol=1e-7) + group=check_group, + ) + + np.testing.assert_allclose( + integral_data.grad.numpy(), + integral_grad.numpy(), + rtol=1e-5, + atol=1e-7, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist.py b/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist.py index bd4e761a7a1d4..3e173c790ea3f 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist.py @@ -36,8 +36,10 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_pool_2 = fluid.nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -45,13 +47,15 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) SIZE = 10 input_shape = conv_pool_2.shape param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 with fluid.device_guard("gpu:1"): predict = fluid.layers.fc( @@ -59,26 +63,29 @@ def cnn_model(data): size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) # To cover @RENAMED@GRADIENT predict2 = fluid.layers.fc( input=conv_pool_1, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) predict += predict2 return predict class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data with fluid.device_guard("gpu:0"): - images = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype=DTYPE) + images = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype=DTYPE + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') if dist_strategy: @@ -86,7 +93,8 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): feed_list=[images, label], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) # Train program predict = cnn_model(images) with fluid.device_guard("gpu:1"): @@ -96,9 +104,9 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Evaluator with fluid.device_guard("gpu:1"): batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() base_lr = self.lr @@ -110,15 +118,18 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): opt = paddle.optimizer.AdamW( learning_rate=lr_val, - grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0)) + grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0), + ) acc_steps = 2 # accumulated steps for pipeline if dist_strategy: # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) fleet.init(is_collective=True) strategy = fleet.DistributedStrategy() strategy.pipeline = True @@ -126,23 +137,41 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): strategy.pipeline_configs = { 'micro_batch_size': batch_size, 'schedule_mode': '1F1B', - 'accumulate_steps': acc_steps + 'accumulate_steps': acc_steps, } - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size * acc_steps) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size * acc_steps) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size * acc_steps + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size * acc_steps + ) if dist_strategy: - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict, data_loader + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + data_loader, + ) else: - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_multi_device.py b/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_multi_device.py index 1933b55f023a8..e7f8a82f166b7 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_multi_device.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_multi_device.py @@ -36,8 +36,10 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_pool_2 = fluid.nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -45,13 +47,15 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) SIZE = 10 input_shape = conv_pool_2.shape param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 with fluid.device_guard("gpu:1"): predict = fluid.layers.fc( @@ -59,26 +63,29 @@ def cnn_model(data): size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) # To cover @RENAMED@GRADIENT predict2 = fluid.layers.fc( input=conv_pool_1, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) predict += predict2 return predict class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data with fluid.device_guard("gpu:0"): - images = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype=DTYPE) + images = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype=DTYPE + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') if dist_strategy: @@ -86,7 +93,8 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): feed_list=[images, label], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) # Train program predict = cnn_model(images) with fluid.device_guard("gpu:1"): @@ -96,9 +104,9 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Evaluator with fluid.device_guard("gpu:1"): batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() base_lr = self.lr @@ -110,15 +118,18 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): opt = fluid.optimizer.Momentum( learning_rate=lr_val, momentum=0.9, - grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0)) + grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0), + ) acc_steps = 2 # accumulated steps for pipeline if dist_strategy: # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) fleet.init(is_collective=True) strategy = fleet.DistributedStrategy() strategy.pipeline = True @@ -126,23 +137,41 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): strategy.pipeline_configs = { 'micro_batch_size': batch_size, 'schedule_mode': 'F-then-B', - 'accumulate_steps': acc_steps + 'accumulate_steps': acc_steps, } - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size * acc_steps) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size * acc_steps) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size * acc_steps + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size * acc_steps + ) if dist_strategy: - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict, data_loader + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + data_loader, + ) else: - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_one_device.py b/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_one_device.py index 33964a6400e24..899cc24fdbad6 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_one_device.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/pipeline_mnist_one_device.py @@ -36,8 +36,10 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_pool_2 = fluid.nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -45,34 +47,37 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) SIZE = 10 input_shape = conv_pool_2.shape param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 predict = fluid.layers.fc( input=conv_pool_2, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) return predict class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data device_id = 0 if dist_strategy: fleet.init(is_collective=True) with fluid.device_guard("gpu:0"): - images = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype=DTYPE) + images = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype=DTYPE + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') if dist_strategy: @@ -80,7 +85,8 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): feed_list=[images, label], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) # Train program predict = cnn_model(images) with fluid.device_guard("gpu:0"): @@ -90,9 +96,9 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Evaluator with fluid.device_guard("gpu:0"): batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() base_lr = self.lr @@ -104,28 +110,46 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): opt = fluid.optimizer.Momentum(learning_rate=lr_val, momentum=0.9) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) if dist_strategy: strategy = fleet.DistributedStrategy() strategy.pipeline = True strategy.pipeline_configs = { 'schedule_mode': 'F-then-B', - 'micro_batch_size': batch_size + 'micro_batch_size': batch_size, } - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) if dist_strategy: - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict, data_loader + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + data_loader, + ) else: - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_col.py b/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_col.py index 40ec79b9844f1..cd874ae42b09d 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_col.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_col.py @@ -27,60 +27,63 @@ OUT_SIZE = 2 * MODEL_PARALLEL_SIZE # Fix seed for test -#fluid.default_startup_program().random_seed = 1 -#fluid.default_main_program().random_seed = 1 +# fluid.default_startup_program().random_seed = 1 +# fluid.default_main_program().random_seed = 1 def get_param_attr(weight, bias): weight_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(weight)) + initializer=fluid.initializer.NumpyArrayInitializer(weight) + ) bias_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(bias)) + initializer=fluid.initializer.NumpyArrayInitializer(bias) + ) return weight_attr, bias_attr def create_model(data, rank): np.random.seed(2021) np_weight = np.random.uniform(-1, 1, size=(IN_SIZE, OUT_SIZE)).astype(DTYPE) - np_bias = np.random.uniform(-1, 1, size=(OUT_SIZE, )).astype(DTYPE) + np_bias = np.random.uniform(-1, 1, size=(OUT_SIZE,)).astype(DTYPE) if rank is not None: start_col = 0 if rank == 0 else OUT_SIZE // 2 - np_weight_part = np_weight[:, start_col:start_col + OUT_SIZE // 2] - np_bias_part = np_bias[start_col:start_col + OUT_SIZE // 2] + np_weight_part = np_weight[:, start_col : start_col + OUT_SIZE // 2] + np_bias_part = np_bias[start_col : start_col + OUT_SIZE // 2] weight_attr, bias_attr = get_param_attr(np_weight_part, np_bias_part) - result = paddle.distributed.split(data, - size=(IN_SIZE, OUT_SIZE), - operation='linear', - axis=1, - num_partitions=MODEL_PARALLEL_SIZE, - weight_attr=weight_attr, - bias_attr=bias_attr) + result = paddle.distributed.split( + data, + size=(IN_SIZE, OUT_SIZE), + operation='linear', + axis=1, + num_partitions=MODEL_PARALLEL_SIZE, + weight_attr=weight_attr, + bias_attr=bias_attr, + ) else: weight_attr, bias_attr = get_param_attr(np_weight, np_bias) - result = fluid.layers.fc(data, - size=OUT_SIZE, - param_attr=weight_attr, - bias_attr=bias_attr) + result = fluid.layers.fc( + data, size=OUT_SIZE, param_attr=weight_attr, bias_attr=bias_attr + ) predict = paddle.sum(result) return predict class TestModelParallel(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data - data_in = fluid.data(name='data_in', - shape=[batch_size, IN_SIZE], - dtype=DTYPE) + data_in = fluid.data( + name='data_in', shape=[batch_size, IN_SIZE], dtype=DTYPE + ) if dist_strategy: data_loader = fluid.io.DataLoader.from_generator( feed_list=[data_in], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) if dist_strategy: fleet.init(is_collective=True) @@ -93,8 +96,9 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): opt = fluid.optimizer.SGD(0.1) if dist_strategy: - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_row.py b/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_row.py index c14dd021729c3..44b0858712ae1 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_row.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_by_row.py @@ -27,61 +27,67 @@ OUT_SIZE = 2 * MODEL_PARALLEL_SIZE # Fix seed for test -#fluid.default_startup_program().random_seed = 1 -#fluid.default_main_program().random_seed = 1 +# fluid.default_startup_program().random_seed = 1 +# fluid.default_main_program().random_seed = 1 def get_param_attr(weight, bias): weight_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(weight)) + initializer=fluid.initializer.NumpyArrayInitializer(weight) + ) bias_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(bias)) + initializer=fluid.initializer.NumpyArrayInitializer(bias) + ) return weight_attr, bias_attr def create_model(data, rank): np.random.seed(2021) np_weight = np.random.uniform(-1, 1, size=(IN_SIZE, OUT_SIZE)).astype(DTYPE) - np_bias = np.random.uniform(-1, 1, size=(OUT_SIZE, )).astype(DTYPE) + np_bias = np.random.uniform(-1, 1, size=(OUT_SIZE,)).astype(DTYPE) if rank is not None: start_row = 0 if rank == 0 else IN_SIZE // 2 - np_weight_part = np_weight[start_row:start_row + IN_SIZE // 2, :] + np_weight_part = np_weight[start_row : start_row + IN_SIZE // 2, :] weight_attr, bias_attr = get_param_attr(np_weight_part, np_bias) - result = paddle.distributed.split(data, - size=(IN_SIZE, OUT_SIZE), - operation='linear', - axis=0, - num_partitions=MODEL_PARALLEL_SIZE, - weight_attr=weight_attr, - bias_attr=bias_attr) + result = paddle.distributed.split( + data, + size=(IN_SIZE, OUT_SIZE), + operation='linear', + axis=0, + num_partitions=MODEL_PARALLEL_SIZE, + weight_attr=weight_attr, + bias_attr=bias_attr, + ) else: weight_attr, bias_attr = get_param_attr(np_weight, np_bias) result = fluid.layers.fc( data, size=OUT_SIZE, param_attr=paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(np_weight)), - bias_attr=bias_attr) + initializer=fluid.initializer.NumpyArrayInitializer(np_weight) + ), + bias_attr=bias_attr, + ) predict = paddle.sum(result) return predict class TestModelParallel(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data - data_in = fluid.data(name='data_in', - shape=[batch_size, IN_SIZE], - dtype=DTYPE) + data_in = fluid.data( + name='data_in', shape=[batch_size, IN_SIZE], dtype=DTYPE + ) if dist_strategy: data_loader = fluid.io.DataLoader.from_generator( feed_list=[data_in], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) if dist_strategy: fleet.init(is_collective=True) @@ -94,8 +100,9 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): opt = fluid.optimizer.SGD(0.1) if dist_strategy: - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_embedding.py b/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_embedding.py index f147190a5d22c..832cad58eb8b6 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_embedding.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/static_model_parallel_embedding.py @@ -27,8 +27,8 @@ OUT_SIZE = 2 * MODEL_PARALLEL_SIZE # Fix seed for test -#fluid.default_startup_program().random_seed = 1 -#fluid.default_main_program().random_seed = 1 +# fluid.default_startup_program().random_seed = 1 +# fluid.default_main_program().random_seed = 1 def create_model(data, rank): @@ -36,15 +36,18 @@ def create_model(data, rank): np_weight = np.random.uniform(-1, 1, size=(IN_SIZE, OUT_SIZE)).astype(DTYPE) if rank is not None: start_row = 0 if rank == 0 else IN_SIZE // 2 - np_weight_part = np_weight[start_row:start_row + IN_SIZE // 2, :] + np_weight_part = np_weight[start_row : start_row + IN_SIZE // 2, :] result = paddle.distributed.split( data, size=(IN_SIZE, OUT_SIZE), operation='linear', axis=0, num_partitions=MODEL_PARALLEL_SIZE, - weight_attr=paddle.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(np_weight_part)), + weight_attr=paddle.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + np_weight_part + ) + ), bias_attr=False, ) else: @@ -52,7 +55,8 @@ def create_model(data, rank): data, size=OUT_SIZE, param_attr=paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(np_weight)), + initializer=fluid.initializer.NumpyArrayInitializer(np_weight) + ), bias_attr=False, ) @@ -61,19 +65,19 @@ def create_model(data, rank): class TestModelParallel(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data - data_in = fluid.data(name='data_in', - shape=[batch_size, IN_SIZE], - dtype=DTYPE) + data_in = fluid.data( + name='data_in', shape=[batch_size, IN_SIZE], dtype=DTYPE + ) if dist_strategy: data_loader = fluid.io.DataLoader.from_generator( feed_list=[data_in], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) if dist_strategy: fleet.init(is_collective=True) @@ -86,8 +90,9 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): opt = fluid.optimizer.SGD(0.1) if dist_strategy: - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint.py index 3a06bc29d0f11..cb61e2c9a8ab2 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint.py @@ -20,14 +20,16 @@ import paddle.fluid.incubate.checkpoint.auto_checkpoint as acp from paddle.fluid.incubate.checkpoint.checkpoint_saver import PaddleModel -from paddle.fluid.tests.unittests.auto_checkpoint_utils import AutoCheckpointBase, get_logger +from paddle.fluid.tests.unittests.auto_checkpoint_utils import ( + AutoCheckpointBase, + get_logger, +) paddle.enable_static() logger = get_logger() class AutoCheckPointACLBase(AutoCheckpointBase): - def setUp(self): get_logger() logger.info("enter tests") @@ -44,7 +46,7 @@ def setUp(self): "PADDLE_EDL_HDFS_CHECKPOINT_PATH": "auto_checkpoint", "PADDLE_EDL_ONLY_FOR_CE_TEST": "1", "PADDLE_EDL_FS_CACHE": ".auto_checkpoint_test", - "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0" + "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0", } os.environ.update(proc_env) @@ -66,7 +68,8 @@ def _run_normal(self): logger.info("begin _run_normal") compiled, data_loader, optimizer, loss, image, label = self._init_env( - exe, main_prog, startup_prog) + exe, main_prog, startup_prog + ) for i in range(3): self.assertEqual(acp._get_train_epoch_range(), None) self.assertEqual(acp.g_acp_type, None) @@ -91,8 +94,9 @@ def _not_use_train(self): logger.info("begin _not_use_train") exe, main_prog, startup_prog = self._generate() - compiled, data_loader, optimizer, loss, image, label = \ - self._init_env(exe, main_prog, startup_prog) + compiled, data_loader, optimizer, loss, image, label = self._init_env( + exe, main_prog, startup_prog + ) epochs = [] for i in acp.train_epoch_range(3, 0): @@ -111,8 +115,9 @@ def _run_save_0(self, break_epoch_no=None): exe, main_prog, startup_prog = self._generate() - compiled, data_loader, optimizer, loss, image, label = \ - self._init_env(exe, main_prog, startup_prog) + compiled, data_loader, optimizer, loss, image, label = self._init_env( + exe, main_prog, startup_prog + ) o = None i = 0 @@ -149,7 +154,8 @@ def _run_load_0(self, break_epoch_no=None): fs.delete(save_dir) compiled, data_loader, optimizer, loss, image, label = self._init_env( - exe, main_prog, startup_prog) + exe, main_prog, startup_prog + ) o = None i = 0 @@ -195,7 +201,6 @@ def _test_corner_epoch_no(self, break_epoch_no): class AutoCheckpointTest(AutoCheckPointACLBase): - def setUp(self): get_logger() logger.info("enter tests") @@ -212,7 +217,7 @@ def setUp(self): "PADDLE_EDL_HDFS_CHECKPOINT_PATH": "auto_checkpoint_0", "PADDLE_EDL_ONLY_FOR_CE_TEST": "1", "PADDLE_EDL_FS_CACHE": ".auto_checkpoint_test_0", - "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0" + "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0", } os.environ.update(proc_env) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint1.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint1.py index 2db7b1e8f8068..583f84430624d 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint1.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint1.py @@ -24,7 +24,6 @@ class AutoCheckpointTest1(AutoCheckPointACLBase): - def setUp(self): get_logger() logger.info("enter tests") @@ -41,7 +40,7 @@ def setUp(self): "PADDLE_EDL_HDFS_CHECKPOINT_PATH": "auto_checkpoint_1", "PADDLE_EDL_ONLY_FOR_CE_TEST": "1", "PADDLE_EDL_FS_CACHE": ".auto_checkpoint_test_1", - "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0" + "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0", } os.environ.update(proc_env) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint2.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint2.py index 96b538ee9b846..3c0cc24c6efb1 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint2.py @@ -24,7 +24,6 @@ class AutoCheckpointTest2(AutoCheckPointACLBase): - def setUp(self): get_logger() logger.info("enter tests") @@ -41,7 +40,7 @@ def setUp(self): "PADDLE_EDL_HDFS_CHECKPOINT_PATH": "auto_checkpoint_2", "PADDLE_EDL_ONLY_FOR_CE_TEST": "1", "PADDLE_EDL_FS_CACHE": ".auto_checkpoint_test_2", - "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0" + "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0", } os.environ.update(proc_env) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint3.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint3.py index 0e75f8e0c4773..beb19fa0b1eb4 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint3.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint3.py @@ -24,7 +24,6 @@ class AutoCheckpointTest3(AutoCheckPointACLBase): - def setUp(self): get_logger() logger.info("enter tests") @@ -41,7 +40,7 @@ def setUp(self): "PADDLE_EDL_HDFS_CHECKPOINT_PATH": "auto_checkpoint_3", "PADDLE_EDL_ONLY_FOR_CE_TEST": "1", "PADDLE_EDL_FS_CACHE": ".auto_checkpoint_test_3", - "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0" + "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0", } os.environ.update(proc_env) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_dist_basic.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_dist_basic.py index 331f6b104509e..702c3eb24a3fc 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_dist_basic.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_dist_basic.py @@ -30,7 +30,6 @@ class AutoCheckpointTestDist(AutoCheckPointACLBase): - def setUp(self): get_logger() logger.info("enter tests") @@ -47,7 +46,7 @@ def setUp(self): "PADDLE_EDL_HDFS_CHECKPOINT_PATH": "auto_checkpoint_dist_basic", "PADDLE_EDL_ONLY_FOR_CE_TEST": "1", "PADDLE_EDL_FS_CACHE": ".auto_checkpoint_test_dist_basic", - "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0" + "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0", } os.environ.update(proc_env) @@ -65,8 +64,9 @@ def test_distributed_basic(self): # basic exe, main_prog, startup_prog = self._generate() - compiled, data_loader, optimizer, loss, image, label = \ - self._init_env(exe, main_prog, startup_prog, minimize=False) + compiled, data_loader, optimizer, loss, image, label = self._init_env( + exe, main_prog, startup_prog, minimize=False + ) # fleet os.environ["TRAINING_ROLE"] = "TRAINER" @@ -91,9 +91,9 @@ def test_distributed_basic(self): logger.info("_run_save_0 name:{} epoch_no:{}".format(o.name, i)) for data in data_loader(): - fetch = exe.run(fleet.main_program, - feed=data, - fetch_list=[loss]) + fetch = exe.run( + fleet.main_program, feed=data, fetch_list=[loss] + ) self.assertEqual(len(o._exe_status), 1) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_multiple.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_multiple.py index 61e0733aa84a1..6bb59c5d2aa0c 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_multiple.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_auto_checkpoint_multiple.py @@ -27,7 +27,6 @@ class AutoCheckpointTestMul(AutoCheckPointACLBase): - def setUp(self): get_logger() logger.info("enter tests") @@ -44,7 +43,7 @@ def setUp(self): "PADDLE_EDL_HDFS_CHECKPOINT_PATH": "auto_checkpoint_dist_multiple", "PADDLE_EDL_ONLY_FOR_CE_TEST": "1", "PADDLE_EDL_FS_CACHE": ".auto_checkpoint_test_dist_multiple", - "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0" + "PADDLE_EDL_SAVE_CHECKPOINT_INTER": "0", } os.environ.update(proc_env) @@ -62,11 +61,23 @@ def test_multiple(self): exe, main_prog1, startup_prog1 = self._generate() _, main_prog2, startup_prog2 = self._generate() - compiled1, data_loader1, optimizer1, loss1, image1, label1 = \ - self._init_env(exe, main_prog1, startup_prog1) - - compiled2, data_loader2, optimizer2, loss2, image2, label2 = \ - self._init_env(exe, main_prog2, startup_prog2) + ( + compiled1, + data_loader1, + optimizer1, + loss1, + image1, + label1, + ) = self._init_env(exe, main_prog1, startup_prog1) + + ( + compiled2, + data_loader2, + optimizer2, + loss2, + image2, + label2, + ) = self._init_env(exe, main_prog2, startup_prog2) o = None epochs = [] diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_half_async.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_half_async.py index ab5bd396d2c2c..fb27d8608801a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_half_async.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_half_async.py @@ -27,7 +27,6 @@ class TestCommunicatorHalfAsyncEnd2End(unittest.TestCase): - def net(self): x = fluid.layers.data(name='x', shape=[13], dtype='float32') y_predict = fluid.layers.fc(input=x, size=1, act=None) @@ -38,7 +37,6 @@ def net(self): return avg_cost, x, y def fake_reader(self): - def reader(): for i in range(10000): x = numpy.random.random((1, 13)).astype('float32') @@ -74,9 +72,11 @@ def run_trainer(self, role, strategy): feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) for batch_id, data in enumerate(train_reader()): - exe.run(paddle.static.default_main_program(), - feed=feeder.feed(data), - fetch_list=[]) + exe.run( + paddle.static.default_main_program(), + feed=feeder.feed(data), + fetch_list=[], + ) fleet.stop_worker() @@ -89,9 +89,11 @@ def run_ut(self): role = role_maker.UserDefinedRoleMaker( current_id=0, role=role_maker.Role.WORKER - if training_role == "TRAINER" else role_maker.Role.SERVER, + if training_role == "TRAINER" + else role_maker.Role.SERVER, worker_num=1, - server_endpoints=["127.0.0.1:6002"]) + server_endpoints=["127.0.0.1:6002"], + ) if training_role == "TRAINER": self.run_trainer(role, strategy) @@ -137,9 +139,11 @@ def runTest(self): _python = sys.executable ps_cmd = "{} {}".format(_python, server_file) - ps_proc = subprocess.Popen(ps_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ps_proc = subprocess.Popen( + ps_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) os.environ["http_proxy"] = "" os.environ["https_proxy"] = "" diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_sync.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_sync.py index 16cba174192b2..f938079beb235 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_sync.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_communicator_sync.py @@ -27,7 +27,6 @@ class TestCommunicator(unittest.TestCase): - def net(self): x = fluid.layers.data(name='x', shape=[1], dtype='float32') y = fluid.layers.data(name='y', shape=[1], dtype='float32') @@ -43,8 +42,9 @@ def test_communicator_sync(self): os.environ["PADDLE_PORT"] = "36001" os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" fleet.init(role_maker.PaddleCloudRoleMaker()) avg_cost = self.net() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_momentum_op.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_momentum_op.py index 0bc6f6daeca0a..763df5bf73d7a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_momentum_op.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_momentum_op.py @@ -20,7 +20,6 @@ class TestDGCMomentumOp1(unittest.TestCase): - def get_tensor(self, name, value, place=None): tensor = self.scope.var(name).get_tensor() tensor.set(value, self.place if place is None else place) @@ -49,13 +48,17 @@ def setup(self, place, step=0.0): self.param_name, self.param_tensor = self.get_tensor('Param', param) self.grad_name, self.grad_tensor = self.get_tensor('Grad', grad) self.velocity_name, self.velocity_tensor = self.get_tensor( - 'Velocity', velocity) + 'Velocity', velocity + ) self.learning_rate_name, self.learning_rate_tensor = self.get_tensor( - 'LearningRate', learning_rate) + 'LearningRate', learning_rate + ) self.current_step_name, self.current_step_tensor = self.get_tensor( - 'current_step', current_step, core.CPUPlace()) + 'current_step', current_step, core.CPUPlace() + ) self.nranks_name, self.nranks_tensor = self.get_tensor( - 'nranks', nranks, core.CPUPlace()) + 'nranks', nranks, core.CPUPlace() + ) self.kwargs = { # inputs @@ -65,12 +68,10 @@ def setup(self, place, step=0.0): 'LearningRate': self.learning_rate_name, 'current_step': self.current_step_name, 'nranks': self.nranks_name, - # attrs 'mu': mu, 'use_nesterov': use_nesterov, 'rampup_begin_step': rampup_begin_step, - # outputs 'ParamOut': self.param_name, 'VelocityOut': self.velocity_name, @@ -79,8 +80,9 @@ def setup(self, place, step=0.0): velocity_out = mu * velocity + grad / nranks if use_nesterov: - param_out = param - grad * learning_rate - \ - velocity_out * mu * learning_rate + param_out = ( + param - grad * learning_rate - velocity_out * mu * learning_rate + ) else: param_out = param - learning_rate * velocity_out @@ -89,7 +91,7 @@ def setup(self, place, step=0.0): self.outputs = { 'ParamOut': param_out, 'VelocityOut': velocity_out, - 'SGDOut': sgd_out + 'SGDOut': sgd_out, } def check(self, actual_t, expect_t, place, out_name, atol=1e-5): @@ -98,8 +100,16 @@ def check(self, actual_t, expect_t, place, out_name, atol=1e-5): expect_t, rtol=1e-05, atol=atol, - err_msg='Output (' + out_name + ') has diff at ' + str(place) + - '\nExpect ' + str(expect_t) + '\n' + 'But Got' + str(actual_t)) + err_msg='Output (' + + out_name + + ') has diff at ' + + str(place) + + '\nExpect ' + + str(expect_t) + + '\n' + + 'But Got' + + str(actual_t), + ) def check_momentum_step(self, place): self.setup(place=place) @@ -107,11 +117,19 @@ def check_momentum_step(self, place): dgc_momentum_op = Operator(self.op_type, **self.kwargs) dgc_momentum_op.run(self.scope, self.place) - self.check(np.array(self.param_tensor), self.outputs['ParamOut'], - self.place, self.param_name) + self.check( + np.array(self.param_tensor), + self.outputs['ParamOut'], + self.place, + self.param_name, + ) - self.check(np.array(self.velocity_tensor), self.outputs['VelocityOut'], - self.place, self.velocity_name) + self.check( + np.array(self.velocity_tensor), + self.outputs['VelocityOut'], + self.place, + self.velocity_name, + ) def check_sgd_step(self, place): self.setup(place=place, step=15.0) @@ -119,8 +137,12 @@ def check_sgd_step(self, place): dgc_momentum_op = Operator(self.op_type, **self.kwargs) dgc_momentum_op.run(self.scope, self.place) - self.check(np.array(self.param_tensor), self.outputs['SGDOut'], - self.place, self.param_name) + self.check( + np.array(self.param_tensor), + self.outputs['SGDOut'], + self.place, + self.param_name, + ) def test_cuda_place(self): if not core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_op.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_op.py index 78d22325cf8e6..6df74578aacad 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_op.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_op.py @@ -23,7 +23,6 @@ class TestDGCOp(unittest.TestCase): - def setup(self, place, array_size=g_array_size): size = array_size np.random.seed(5) # fix seed @@ -72,20 +71,23 @@ def setup(self, place, array_size=g_array_size): self.param_tensor.set(self.param, place) self.current_step_tensor = self.scope.var( - self.current_step_name).get_tensor() + self.current_step_name + ).get_tensor() self.current_step_tensor.set(self.current_step, core.CPUPlace()) self.nranks_tensor = self.scope.var(self.nranks_name).get_tensor() self.nranks_tensor.set(self.nranks, core.CPUPlace()) self.encode_grad_tensor = self.scope.var( - self.encode_grad_name).get_tensor() + self.encode_grad_name + ).get_tensor() self.k_tensor = self.scope.var(self.k_name).get_tensor() self.k_tensor.set(self.k, core.CPUPlace()) self.gather_buff_tensor = self.scope.var( - self.gather_buff_name).get_tensor() + self.gather_buff_name + ).get_tensor() def check(self, actual_t, expect_t, place, out_name, atol=1e-5): np.testing.assert_allclose( @@ -93,8 +95,16 @@ def check(self, actual_t, expect_t, place, out_name, atol=1e-5): expect_t, rtol=1e-05, atol=atol, - err_msg='Output (' + out_name + ') has diff at ' + str(place) + - '\nExpect ' + str(expect_t) + '\n' + 'But Got' + str(actual_t)) + err_msg='Output (' + + out_name + + ') has diff at ' + + str(place) + + '\nExpect ' + + str(expect_t) + + '\n' + + 'But Got' + + str(actual_t), + ) def test_run_and_check(self): self.setup(place=core.CUDAPlace(0)) @@ -106,7 +116,6 @@ def test_run_and_check(self): 'Param': self.param_name, 'current_step': self.current_step_name, 'nranks': self.nranks_name, - # outputs 'U_out': self.u_name, 'V_out': self.v_name, @@ -114,7 +123,6 @@ def test_run_and_check(self): 'Grad_out': self.grad_name, 'k': self.k_name, 'GatherBuff': self.gather_buff_name, - # attrs 'm': 0.9, 'sparsity': [0.75, 0.9375, 0.984375, 0.996, 0.999], @@ -127,7 +135,7 @@ def test_run_and_check(self): dgc_op = Operator('dgc', **kwargs) - #atol = 1e-6 + # atol = 1e-6 dgc_op.run(self.scope, self.place) u_out = np.array(self.u_tensor) @@ -144,7 +152,7 @@ def test_run_and_check(self): self.assertEqual(k, int(g_array_size * 0.25)) index = encode_grad_out[0:k].view(dtype=np.int32) - value = encode_grad_out[k:2 * k] + value = encode_grad_out[k : 2 * k] acl = 1e-7 diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_optimizer.py index 50f1b389a0a47..0da05a377b36d 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dgc_optimizer.py @@ -24,20 +24,20 @@ class TestDGCMomentumOptimizer(unittest.TestCase): - class MockDGCMomentum(optimizer.DGCMomentumOptimizer): - def get_accumulators(self): return self._accumulators def get_velocity_str(self): return self._u_velocity_acc_str - def check_dgc_momentum_optimizer(self, - dims=[5, 10, 8], - name="momentum", - regularization=None, - use_recompute=False): + def check_dgc_momentum_optimizer( + self, + dims=[5, 10, 8], + name="momentum", + regularization=None, + use_recompute=False, + ): init_program = framework.Program() program = framework.Program() block = program.global_block() @@ -47,23 +47,25 @@ def check_dgc_momentum_optimizer(self, lod_level=0, name="mul.x", optimize_attr={'learning_rate': 1.1}, - regularizer=None if regularization is not None else - regularizer.L2DecayRegularizer(2e-4)) - mul_y = block.create_var(dtype="float32", - shape=[dims[1], dims[2]], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[dims[0], dims[2]], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) + regularizer=None + if regularization is not None + else regularizer.L2DecayRegularizer(2e-4), + ) + mul_y = block.create_var( + dtype="float32", shape=[dims[1], dims[2]], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", + shape=[dims[0], dims[2]], + lod_level=0, + name="mul.out", + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) learning_rate = 0.01 dgc_momentum_optimizer = self.MockDGCMomentum( @@ -72,33 +74,40 @@ def check_dgc_momentum_optimizer(self, rampup_begin_step=0, num_trainers=2, regularization=regularization, - grad_clip=clip.GradientClipByNorm(1.0)) + grad_clip=clip.GradientClipByNorm(1.0), + ) if use_recompute: dgc_momentum_optimizer = optimizer.RecomputeOptimizer( - dgc_momentum_optimizer) + dgc_momentum_optimizer + ) dgc_momentum_optimizer._set_checkpoints([]) - dgc_momentum_optimizer.get_accumulators = dgc_momentum_optimizer._optimizer.get_accumulators - dgc_momentum_optimizer.get_velocity_str = dgc_momentum_optimizer._optimizer.get_velocity_str - - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + dgc_momentum_optimizer.get_accumulators = ( + dgc_momentum_optimizer._optimizer.get_accumulators + ) + dgc_momentum_optimizer.get_velocity_str = ( + dgc_momentum_optimizer._optimizer.get_velocity_str + ) + + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) # params_grads = append_backward(mean_out) params_grads = dgc_momentum_optimizer.backward( - mean_out, startup_program=init_program) + mean_out, startup_program=init_program + ) with framework.program_guard(program, init_program): opts = dgc_momentum_optimizer.apply_gradients(params_grads) accumulator_count = 1 if name == "momentum" else 2 self.assertEqual(len(params_grads), 1) - self.assertEqual(len(dgc_momentum_optimizer.get_accumulators()), - accumulator_count) + self.assertEqual( + len(dgc_momentum_optimizer.get_accumulators()), accumulator_count + ) self.assertEqual(len(opts), 2) sgd_op = opts[-1] @@ -109,7 +118,8 @@ def check_dgc_momentum_optimizer(self, accumulators = dgc_momentum_optimizer.get_accumulators() self.assertEqual(len(accumulators), accumulator_count) self.assertTrue( - dgc_momentum_optimizer.get_velocity_str() in accumulators) + dgc_momentum_optimizer.get_velocity_str() in accumulators + ) velocity_acc = accumulators[dgc_momentum_optimizer.get_velocity_str()] self.assertEqual(len(velocity_acc), 1) self.assertTrue(mul_x.name in velocity_acc) @@ -139,22 +149,26 @@ def test_tpyeError(self): momentum=0.2, rampup_begin_step=0, num_trainers=2, - grad_clip=clip.GradientClipByGlobalNorm(1.0)) + grad_clip=clip.GradientClipByGlobalNorm(1.0), + ) def test_momentum_without_dgc(self): self.check_dgc_momentum_optimizer( - regularization=regularizer.L1Decay(1e-4)) + regularization=regularizer.L1Decay(1e-4) + ) def test_momentum_with_dgc(self): # 16 * 1024 = 16384, use dgc momentum self.check_dgc_momentum_optimizer( dims=[16, 1024, 8], name="dgc_momentum", - regularization=regularizer.L2Decay(1e-4)) + regularization=regularizer.L2Decay(1e-4), + ) # check param.regularizer in dgc - self.check_dgc_momentum_optimizer(dims=[16, 1024, 8], - name="dgc_momentum") + self.check_dgc_momentum_optimizer( + dims=[16, 1024, 8], name="dgc_momentum" + ) def test_momentum_with_dgc_recompute(self): # 16 * 1024 = 16384, use dgc momentum @@ -162,7 +176,8 @@ def test_momentum_with_dgc_recompute(self): dims=[16, 1024, 8], name="dgc_momentum", regularization=regularizer.L2Decay(1e-4), - use_recompute=True) + use_recompute=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_dgc_nccl.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_dgc_nccl.py index 8a4fdfcd07bb8..7a2b2425f2489 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_dgc_nccl.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_dgc_nccl.py @@ -27,7 +27,11 @@ def count_of_sparse_all_reduce_calls(file_name): # NOTE(Aurelius84): The log file contains some binary contents that causes error # while `grep`. So we add `-a` to fix it. # -a, --text equivalent to --binary-files=text, make binaries equivalent to text. - cmd = 'grep -a sparse_all_reduce_op_handle ' + file_name + ' | grep in_numel | wc -l' + cmd = ( + 'grep -a sparse_all_reduce_op_handle ' + + file_name + + ' | grep in_numel | wc -l' + ) child = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) result = child.communicate()[0] print('test_info: result = ' + str(result)) @@ -37,7 +41,6 @@ def count_of_sparse_all_reduce_calls(file_name): class TestDistMnistNCCL2DGC(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -47,17 +50,22 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place(os.path.abspath("../../dist_mnist.py"), - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + os.path.abspath("../../dist_mnist.py"), + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) def tearDown(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - log_file = os.path.join(self.temp_dir.name, - 'test_dist_mnist_dgc_nccl_tr0_err.log') + log_file = os.path.join( + self.temp_dir.name, 'test_dist_mnist_dgc_nccl_tr0_err.log' + ) result = count_of_sparse_all_reduce_calls(log_file) # only 1 layer use dgc now, run_step=5, rampup_begin_step=2, so 1 * (5 - 2) = 3 @@ -68,7 +76,6 @@ def tearDown(self): class TestDistMnistNCCL2DGCMultiCards(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -78,19 +85,23 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place_multi_cards( os.path.abspath("../../dist_mnist.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) def tearDown(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): log_file = os.path.join( self.temp_dir.name, - 'test_dist_mnist_dgc_nccl_dgc_2cards_local.log') + 'test_dist_mnist_dgc_nccl_dgc_2cards_local.log', + ) result = count_of_sparse_all_reduce_calls(log_file) # same as above, but use two cards self.assertEqual(result, 6) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_gradient_merge.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_gradient_merge.py index 54c9d6b26c370..c0b978560e18c 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_gradient_merge.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_mnist_gradient_merge.py @@ -21,7 +21,6 @@ class TestDistMnistGradMerge(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -29,14 +28,15 @@ def _setup_config(self): def test_dist_train(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_mnist_gradient_merge.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_mnist_gradient_merge.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestDistMnistGradMergeNoFuse(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -45,14 +45,15 @@ def _setup_config(self): def test_dist_train(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_mnist_gradient_merge.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name + "_no_fuse") + self.check_with_place( + "dist_mnist_gradient_merge.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name + "_no_fuse", + ) class TestDistMnistGradMergeRawOptimizerBase(TestDistBase): - def _setup_config(self): self._use_reader_alloc = False self._nccl2_mode = True @@ -66,19 +67,21 @@ def test_dist_train(self): if fluid.core.is_compiled_with_cuda(): avg = str(self.enable_avg()) log_name = flag_name + "_raw_optimizer_gm_avg_" + avg - self.check_with_place("dist_mnist_gradient_merge_raw_optimizer.py", - delta=1e-5, - check_error_log=True, - log_name=log_name, - need_envs={ - 'FLAGS_apply_pass_to_program': '1', - 'enable_gm_avg': avg, - }) + self.check_with_place( + "dist_mnist_gradient_merge_raw_optimizer.py", + delta=1e-5, + check_error_log=True, + log_name=log_name, + need_envs={ + 'FLAGS_apply_pass_to_program': '1', + 'enable_gm_avg': avg, + }, + ) class TestDistMnistGradMergeRawOptimizerAvg( - TestDistMnistGradMergeRawOptimizerBase): - + TestDistMnistGradMergeRawOptimizerBase +): def enable_avg(self): return True diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_se_resnext_dgc.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_se_resnext_dgc.py index 28936e3684514..7020e63248e86 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_se_resnext_dgc.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dist_se_resnext_dgc.py @@ -20,7 +20,6 @@ class TestDistSeResnetNCCL2DGC(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -31,11 +30,14 @@ def _setup_config(self): @unittest.skip(reason="Skip unstable ci") def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place(os.path.abspath("../../dist_se_resnext.py"), - delta=30, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + os.path.abspath("../../dist_se_resnext.py"), + delta=30, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_distributed_strategy.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_distributed_strategy.py index 92a6715a6424e..140a910c336ee 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_distributed_strategy.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_distributed_strategy.py @@ -15,15 +15,21 @@ import unittest import paddle import paddle.fluid as fluid -from paddle.fluid.transpiler.distribute_transpiler import DistributeTranspilerConfig, ServerRuntimeConfig -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet +from paddle.fluid.transpiler.distribute_transpiler import ( + DistributeTranspilerConfig, + ServerRuntimeConfig, +) +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + StrategyFactory, +) +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, +) import paddle.fluid.incubate.fleet.base.role_maker as role_maker import os class TestStrategyFactor(unittest.TestCase): - def test_sync_strategy(self): os.environ['CPU_NUM'] = "2" strategy = StrategyFactory.create_sync_strategy() @@ -48,19 +54,23 @@ def test_sync_strategy(self): # test set_program_config exception program_config_dict['unknown'] = None - self.assertRaises(Exception, strategy.set_program_config, - program_config_dict) + self.assertRaises( + Exception, strategy.set_program_config, program_config_dict + ) program_config_illegal = None - self.assertRaises(Exception, strategy.set_program_config, - program_config_illegal) + self.assertRaises( + Exception, strategy.set_program_config, program_config_illegal + ) trainer_runtime_config = strategy.get_trainer_runtime_config() trainer_runtime_config.runtime_configs[ - 'communicator_send_queue_size'] = '50' + 'communicator_send_queue_size' + ] = '50' runtime_configs = trainer_runtime_config.get_communicator_flags() self.assertIn('communicator_send_queue_size', runtime_configs) - self.assertNotIn('communicator_independent_recv_thread', - runtime_configs) + self.assertNotIn( + 'communicator_independent_recv_thread', runtime_configs + ) self.assertEqual(runtime_configs['communicator_send_queue_size'], '2') def test_geo_strategy(self): @@ -87,19 +97,22 @@ def test_geo_strategy(self): # test set_build_strategy exception build_strategy_dict['unknown'] = None - self.assertRaises(Exception, strategy.set_build_strategy, - build_strategy_dict) + self.assertRaises( + Exception, strategy.set_build_strategy, build_strategy_dict + ) build_strategy_illegal = None - self.assertRaises(Exception, strategy.set_build_strategy, - build_strategy_illegal) + self.assertRaises( + Exception, strategy.set_build_strategy, build_strategy_illegal + ) os.environ["CPU_NUM"] = '100' trainer_runtime_config = strategy.get_trainer_runtime_config() runtime_configs = trainer_runtime_config.get_communicator_flags() self.assertIn('communicator_thread_pool_size', runtime_configs) self.assertIn('communicator_send_wait_times', runtime_configs) - self.assertNotIn('communicator_independent_recv_thread', - runtime_configs) + self.assertNotIn( + 'communicator_independent_recv_thread', runtime_configs + ) def test_async_strategy(self): os.environ["CPU_NUM"] = '100' @@ -111,28 +124,40 @@ def test_async_strategy(self): trainer_runtime_config = strategy.get_trainer_runtime_config() self.assertEqual( - trainer_runtime_config. - runtime_configs['communicator_send_queue_size'], '100') + trainer_runtime_config.runtime_configs[ + 'communicator_send_queue_size' + ], + '100', + ) # test set_trainer_runtime_config using dict trainer_runtime_config_dict = dict() trainer_runtime_config_dict['communicator_send_queue_size'] = '20' strategy.set_trainer_runtime_config(trainer_runtime_config_dict) trainer_runtime_config = strategy.get_trainer_runtime_config() - trainer_communicator_flags = trainer_runtime_config.get_communicator_flags( + trainer_communicator_flags = ( + trainer_runtime_config.get_communicator_flags() + ) + self.assertIn( + 'communicator_send_queue_size', trainer_communicator_flags ) - self.assertIn('communicator_send_queue_size', - trainer_communicator_flags) self.assertEqual( - trainer_communicator_flags['communicator_send_queue_size'], '20') + trainer_communicator_flags['communicator_send_queue_size'], '20' + ) # test set_trainer_runtime_config exception trainer_runtime_config_dict['unknown'] = None - self.assertRaises(Exception, strategy.set_trainer_runtime_config, - trainer_runtime_config_dict) + self.assertRaises( + Exception, + strategy.set_trainer_runtime_config, + trainer_runtime_config_dict, + ) trainer_runtime_config_illegal = None - self.assertRaises(Exception, strategy.set_trainer_runtime_config, - trainer_runtime_config_illegal) + self.assertRaises( + Exception, + strategy.set_trainer_runtime_config, + trainer_runtime_config_illegal, + ) # test set_execute_strategy using fluid.ExecutionStrategy exec_strategy_class = fluid.ExecutionStrategy() @@ -150,11 +175,13 @@ def test_async_strategy(self): # test set_execute_strategy exception exec_strategy_dict['unknown'] = None - self.assertRaises(Exception, strategy.set_execute_strategy, - exec_strategy_dict) + self.assertRaises( + Exception, strategy.set_execute_strategy, exec_strategy_dict + ) exec_strategy_illegal = None - self.assertRaises(Exception, strategy.set_execute_strategy, - exec_strategy_illegal) + self.assertRaises( + Exception, strategy.set_execute_strategy, exec_strategy_illegal + ) def test_half_async_strategy(self): strategy = StrategyFactory.create_half_async_strategy() @@ -178,31 +205,39 @@ def test_half_async_strategy(self): # test set_server_runtime_config exception server_runtime_config_dict['unknown'] = None - self.assertRaises(Exception, strategy.set_server_runtime_config, - server_runtime_config_dict) + self.assertRaises( + Exception, + strategy.set_server_runtime_config, + server_runtime_config_dict, + ) server_runtime_config_illegal = None - self.assertRaises(Exception, strategy.set_server_runtime_config, - server_runtime_config_illegal) + self.assertRaises( + Exception, + strategy.set_server_runtime_config, + server_runtime_config_illegal, + ) os.environ["CPU_NUM"] = '100' trainer_runtime_config = strategy.get_trainer_runtime_config() trainer_runtime_config.runtime_configs[ - 'communicator_send_queue_size'] = '50' + 'communicator_send_queue_size' + ] = '50' runtime_configs = trainer_runtime_config.get_communicator_flags() self.assertIn('communicator_send_queue_size', runtime_configs) - self.assertNotIn('communicator_independent_recv_thread', - runtime_configs) + self.assertNotIn( + 'communicator_independent_recv_thread', runtime_configs + ) self.assertEqual(runtime_configs['communicator_send_queue_size'], '100') class TestCreateDefaultStrategy(unittest.TestCase): - def test_default_strategy(self): role = role_maker.UserDefinedRoleMaker( current_id=0, role=role_maker.Role.WORKER, worker_num=2, - server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"]) + server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"], + ) fleet.init(role) def type_error_optimizer(): @@ -213,13 +248,13 @@ def type_error_optimizer(): class TestHalfAsyncStrategy(unittest.TestCase): - def test_half_async_strategy(self): role = role_maker.UserDefinedRoleMaker( current_id=0, role=role_maker.Role.WORKER, worker_num=2, - server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"]) + server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"], + ) fleet.init(role) half_async_config = DistributeTranspilerConfig() @@ -233,7 +268,6 @@ def test_half_async_strategy(self): class TestDebugInfo(unittest.TestCase): - def test_debug_info(self): x = fluid.layers.data(name='x', shape=[1], dtype='float32') y = fluid.layers.data(name='y', shape=[1], dtype='float32') @@ -245,16 +279,19 @@ def test_debug_info(self): current_id=0, role=role_maker.Role.WORKER, worker_num=2, - server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"]) + server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"], + ) fleet.init(role) optimizer = fluid.optimizer.SGD(0.0001) strategy = StrategyFactory.create_sync_strategy() - strategy.set_debug_opt({ - "dump_param": ["fc_0.tmp_0"], - "dump_fields": ["fc_0.tmp_0", "fc_0.tmp_0@GRAD"], - "dump_fields_path": "dump_text/" - }) + strategy.set_debug_opt( + { + "dump_param": ["fc_0.tmp_0"], + "dump_fields": ["fc_0.tmp_0", "fc_0.tmp_0@GRAD"], + "dump_fields_path": "dump_text/", + } + ) optimizer = fleet.distributed_optimizer(optimizer, strategy) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute.py index afd619a390b7b..9cadbf4d7b88b 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute.py @@ -24,35 +24,41 @@ def get_fc_block(block_idx, input_size, is_last=False): block_name = "block_" + str(block_idx) block = paddle.nn.Sequential( - (block_name + "_fc_0", - paddle.nn.Linear(input_size, input_size, bias_attr=False)), + ( + block_name + "_fc_0", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ), (block_name + "_dropout", paddle.nn.Dropout(p=0.5)), (block_name + "_relu_1", paddle.nn.ReLU()), - (block_name + "_fc_1", - paddle.nn.Linear(input_size, input_size, bias_attr=False)), + ( + block_name + "_fc_1", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ), (block_name + "_relu_2", paddle.nn.ReLU()), ) if is_last: - block.add_sublayer(block_name + "_fc_2", - paddle.nn.Linear(input_size, 1, - bias_attr=False)) # add sublayer + block.add_sublayer( + block_name + "_fc_2", + paddle.nn.Linear(input_size, 1, bias_attr=False), + ) # add sublayer else: - block.add_sublayer(block_name + "_fc_2", - paddle.nn.Linear(input_size, - input_size, - bias_attr=False)) # add sublayer + block.add_sublayer( + block_name + "_fc_2", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ) # add sublayer return block class Naive_fc_net(paddle.nn.Layer): - - def __init__(self, - input_size=10, - recompute_blocks=[1, 3], - use_fleet_sq=False, - segments=1, - use_raw_recompute=False, - recompute_kwargs={}): + def __init__( + self, + input_size=10, + recompute_blocks=[1, 3], + use_fleet_sq=False, + segments=1, + use_raw_recompute=False, + recompute_kwargs={}, + ): super(Naive_fc_net, self).__init__() self.recompute_blocks = recompute_blocks self.recompute_kwargs = recompute_kwargs @@ -67,28 +73,37 @@ def __init__(self, self.runfunc4 = get_fc_block(4, input_size, is_last=True) if self.use_fleet_sq and not use_raw_recompute: - self.runfuncs = paddle.nn.Sequential(self.runfunc0, self.runfunc1, - self.runfunc2, self.runfunc3, - self.runfunc4) + self.runfuncs = paddle.nn.Sequential( + self.runfunc0, + self.runfunc1, + self.runfunc2, + self.runfunc3, + self.runfunc4, + ) self.layers = [ - self.runfunc0, self.runfunc1, self.runfunc2, self.runfunc3, - self.runfunc4 + self.runfunc0, + self.runfunc1, + self.runfunc2, + self.runfunc3, + self.runfunc4, ] # default segments = 2 if use_raw_recompute: self.layers = [ paddle.nn.Sequential(self.runfunc0, self.runfunc1), - paddle.nn.Sequential(self.runfunc2, self.runfunc3, - self.runfunc4) + paddle.nn.Sequential( + self.runfunc2, self.runfunc3, self.runfunc4 + ), ] def forward(self, inputs): if self.use_fleet_sq and not self.use_raw_recompute: - return recompute_sequential({"segments": self.segments}, - self.runfuncs, inputs) + return recompute_sequential( + {"segments": self.segments}, self.runfuncs, inputs + ) if self.use_raw_recompute: inputs = recompute(self.layers[0], inputs) @@ -96,36 +111,42 @@ def forward(self, inputs): for i in range(len(self.layers)): if i in self.recompute_blocks: - inputs = recompute(self.layers[i], inputs, - **self.recompute_kwargs) + inputs = recompute( + self.layers[i], inputs, **self.recompute_kwargs + ) else: inputs = self.layers[i](inputs) return inputs -def run_model(recompute_block=[], - recompute_kwargs={}, - use_fleet_sq=False, - use_raw_recompute=False, - segments=1, - enable_autocast=False, - pure_fp16=False): +def run_model( + recompute_block=[], + recompute_kwargs={}, + use_fleet_sq=False, + use_raw_recompute=False, + segments=1, + enable_autocast=False, + pure_fp16=False, +): gen = paddle.seed(10) gen.manual_seed(10) np.random.seed(10) random.seed(10) batch_size, input_size = 1, 10 - model = Naive_fc_net(input_size, - recompute_blocks=recompute_block, - use_fleet_sq=use_fleet_sq, - use_raw_recompute=use_raw_recompute, - segments=segments, - recompute_kwargs=recompute_kwargs) + model = Naive_fc_net( + input_size, + recompute_blocks=recompute_block, + use_fleet_sq=use_fleet_sq, + use_raw_recompute=use_raw_recompute, + segments=segments, + recompute_kwargs=recompute_kwargs, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) if enable_autocast: scaler = paddle.amp.GradScaler() @@ -158,9 +179,7 @@ def run_model(recompute_block=[], class TestPyLayer(unittest.TestCase): - def test_base_case(self, enable_autocast=False, pure_fp16=False): - def check_identical(loss_ref, param_ref, grad_ref, loss, param, grad): self.assertEqual(loss_ref, loss) self.assertEqual(param_ref, param) @@ -170,43 +189,56 @@ def check_identical(loss_ref, param_ref, grad_ref, loss, param, grad): loss_ref, param_ref, grad_ref = run_model( recompute_block=[], enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + pure_fp16=pure_fp16, + ) # recompute second block - loss, param, grad = run_model(recompute_block=[1], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute fourth block - loss, param, grad = run_model(recompute_block=[3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute second to fourth block - loss, param, grad = run_model(recompute_block=[1, 2, 3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 2, 3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute second & fourth block - loss, param, grad = run_model(recompute_block=[1, 3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute second & fourth block using fleet - loss, param, grad = run_model(recompute_block=[1, 3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute using recompute_sequential, segments=1 - loss, param, grad = run_model(recompute_block=[], - use_fleet_sq=True, - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[], + use_fleet_sq=True, + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # with base recompute, and segments=2 @@ -214,14 +246,17 @@ def check_identical(loss_ref, param_ref, grad_ref, loss, param, grad): recompute_block=[], enable_autocast=enable_autocast, use_raw_recompute=True, - pure_fp16=pure_fp16) + pure_fp16=pure_fp16, + ) # recompute using recompute_sequential, segments=2 - loss, param, grad = run_model(recompute_block=[], - use_fleet_sq=True, - segments=2, - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[], + use_fleet_sq=True, + segments=2, + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) def test_fc_net_with_dropout(self): @@ -237,8 +272,9 @@ def test_recompute_kwargs(self): paddle.set_device("gpu") kwargs = {"is_test": False} with self.assertRaises(TypeError): - loss_ref, param_ref, grad_ref = run_model(recompute_block=[2], - recompute_kwargs=kwargs) + loss_ref, param_ref, grad_ref = run_model( + recompute_block=[2], recompute_kwargs=kwargs + ) def test_recompute_cpu_rng(self): paddle.set_device("cpu") diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute_for_eager.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute_for_eager.py index d69c52144f5f9..9683f681451b6 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute_for_eager.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_recompute_for_eager.py @@ -27,35 +27,41 @@ def get_fc_block(block_idx, input_size, is_last=False): block_name = "block_" + str(block_idx) block = paddle.nn.Sequential( - (block_name + "_fc_0", - paddle.nn.Linear(input_size, input_size, bias_attr=False)), + ( + block_name + "_fc_0", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ), (block_name + "_dropout", paddle.nn.Dropout(p=0.5)), (block_name + "_relu_1", paddle.nn.ReLU()), - (block_name + "_fc_1", - paddle.nn.Linear(input_size, input_size, bias_attr=False)), + ( + block_name + "_fc_1", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ), (block_name + "_relu_2", paddle.nn.ReLU()), ) if is_last: - block.add_sublayer(block_name + "_fc_2", - paddle.nn.Linear(input_size, 1, - bias_attr=False)) # add sublayer + block.add_sublayer( + block_name + "_fc_2", + paddle.nn.Linear(input_size, 1, bias_attr=False), + ) # add sublayer else: - block.add_sublayer(block_name + "_fc_2", - paddle.nn.Linear(input_size, - input_size, - bias_attr=False)) # add sublayer + block.add_sublayer( + block_name + "_fc_2", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ) # add sublayer return block class Naive_fc_net(paddle.nn.Layer): - - def __init__(self, - input_size=10, - recompute_blocks=[1, 3], - use_fleet_sq=False, - segments=1, - use_raw_recompute=False, - recompute_kwargs={}): + def __init__( + self, + input_size=10, + recompute_blocks=[1, 3], + use_fleet_sq=False, + segments=1, + use_raw_recompute=False, + recompute_kwargs={}, + ): super(Naive_fc_net, self).__init__() self.recompute_blocks = recompute_blocks self.recompute_kwargs = recompute_kwargs @@ -70,28 +76,37 @@ def __init__(self, self.runfunc4 = get_fc_block(4, input_size, is_last=True) if self.use_fleet_sq and not use_raw_recompute: - self.runfuncs = paddle.nn.Sequential(self.runfunc0, self.runfunc1, - self.runfunc2, self.runfunc3, - self.runfunc4) + self.runfuncs = paddle.nn.Sequential( + self.runfunc0, + self.runfunc1, + self.runfunc2, + self.runfunc3, + self.runfunc4, + ) self.layers = [ - self.runfunc0, self.runfunc1, self.runfunc2, self.runfunc3, - self.runfunc4 + self.runfunc0, + self.runfunc1, + self.runfunc2, + self.runfunc3, + self.runfunc4, ] # default segments = 2 if use_raw_recompute: self.layers = [ paddle.nn.Sequential(self.runfunc0, self.runfunc1), - paddle.nn.Sequential(self.runfunc2, self.runfunc3, - self.runfunc4) + paddle.nn.Sequential( + self.runfunc2, self.runfunc3, self.runfunc4 + ), ] def forward(self, inputs): if self.use_fleet_sq and not self.use_raw_recompute: return paddle.incubate.distributed.fleet.recompute_sequential( - {"segments": self.segments}, self.runfuncs, inputs) + {"segments": self.segments}, self.runfuncs, inputs + ) if self.use_raw_recompute: inputs = recompute(self.layers[0], inputs) @@ -99,36 +114,42 @@ def forward(self, inputs): for i in range(len(self.layers)): if i in self.recompute_blocks: - inputs = recompute(self.layers[i], inputs, - **self.recompute_kwargs) + inputs = recompute( + self.layers[i], inputs, **self.recompute_kwargs + ) else: inputs = self.layers[i](inputs) return inputs -def run_model(recompute_block=[], - recompute_kwargs={}, - use_fleet_sq=False, - use_raw_recompute=False, - segments=1, - enable_autocast=False, - pure_fp16=False): +def run_model( + recompute_block=[], + recompute_kwargs={}, + use_fleet_sq=False, + use_raw_recompute=False, + segments=1, + enable_autocast=False, + pure_fp16=False, +): gen = paddle.seed(10) gen.manual_seed(10) np.random.seed(10) random.seed(10) batch_size, input_size = 1, 10 - model = Naive_fc_net(input_size, - recompute_blocks=recompute_block, - use_fleet_sq=use_fleet_sq, - use_raw_recompute=use_raw_recompute, - segments=segments, - recompute_kwargs=recompute_kwargs) + model = Naive_fc_net( + input_size, + recompute_blocks=recompute_block, + use_fleet_sq=use_fleet_sq, + use_raw_recompute=use_raw_recompute, + segments=segments, + recompute_kwargs=recompute_kwargs, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) if enable_autocast: scaler = paddle.amp.GradScaler() @@ -161,9 +182,7 @@ def run_model(recompute_block=[], class TestPyLayer(unittest.TestCase): - def test_base_case(self, enable_autocast=False, pure_fp16=False): - def check_identical(loss_ref, param_ref, grad_ref, loss, param, grad): self.assertEqual(loss_ref, loss) self.assertEqual(param_ref, param) @@ -173,37 +192,48 @@ def check_identical(loss_ref, param_ref, grad_ref, loss, param, grad): loss_ref, param_ref, grad_ref = run_model( recompute_block=[], enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + pure_fp16=pure_fp16, + ) # recompute second block - loss, param, grad = run_model(recompute_block=[1], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute fourth block - loss, param, grad = run_model(recompute_block=[3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute second to fourth block - loss, param, grad = run_model(recompute_block=[1, 2, 3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 2, 3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute second & fourth block - loss, param, grad = run_model(recompute_block=[1, 3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # recompute_sequential with segments=1 using fleet - loss, param, grad = run_model(recompute_block=[], - use_fleet_sq=True, - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[], + use_fleet_sq=True, + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # with base recompute, and segments=2 @@ -211,14 +241,17 @@ def check_identical(loss_ref, param_ref, grad_ref, loss, param, grad): recompute_block=[], enable_autocast=enable_autocast, use_raw_recompute=True, - pure_fp16=pure_fp16) + pure_fp16=pure_fp16, + ) # recompute using paddle.incubate.distributed.fleet.recompute_sequential, segments=2 - loss, param, grad = run_model(recompute_block=[], - use_fleet_sq=True, - segments=2, - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[], + use_fleet_sq=True, + segments=2, + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) def test_fc_net_with_dropout(self): @@ -228,7 +261,8 @@ def test_fc_net_without_restore_rng(self): loss_ref, param_ref, grad_ref = run_model( recompute_block=[2], recompute_kwargs={"preserve_rng_state": False}, - enable_autocast=True) + enable_autocast=True, + ) def test_fc_net_with_amp(self): self.test_base_case(enable_autocast=True) @@ -240,8 +274,9 @@ def test_recompute_kwargs(self): paddle.set_device("gpu") kwargs = {"is_test": False} with self.assertRaises(TypeError): - loss_ref, param_ref, grad_ref = run_model(recompute_block=[2], - recompute_kwargs=kwargs) + loss_ref, param_ref, grad_ref = run_model( + recompute_block=[2], recompute_kwargs=kwargs + ) def test_recompute_cpu_rng(self): paddle.set_device("cpu") diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_optimizer_stage2.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_optimizer_stage2.py index f4d3f9728ef6d..8087fc98b6a2b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_optimizer_stage2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_optimizer_stage2.py @@ -21,8 +21,9 @@ class TestDygraphShardingOptimizerStage2(TestMultipleGpus): # check sharding logic as well as the accuracy with single mode def test_dygraph_sharding_optimizer_stage2(self): - self.run_mnist_2gpu('dygraph_sharding_optimizer_stage2.py', - eager_mode=False) + self.run_mnist_2gpu( + 'dygraph_sharding_optimizer_stage2.py', eager_mode=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage2.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage2.py index 07a9b1296e43c..0dd1d4961b379 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage2.py @@ -27,8 +27,9 @@ def test_dygraph_sharding_stage2(self): def test_dygraph_sharding_stage2_offload(self): self.run_mnist_2gpu('dygraph_group_sharded_stage2_offload.py') - self.run_mnist_2gpu('dygraph_sharding_stage2_offload.py', - eager_mode=False) + self.run_mnist_2gpu( + 'dygraph_sharding_stage2_offload.py', eager_mode=False + ) def test_dygraph_sharding_stage2_with_comm_overlap(self): self.run_mnist_2gpu('dygraph_group_sharded_stage2_comm_overlap.py') diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3.py index 82e7b8e12bfc9..ea4edbd34934f 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_dygraph_sharding_stage3.py @@ -28,8 +28,9 @@ def test_dygraph_sharding_stage3(self): self.run_mnist_2gpu('dygraph_sharding_stage3.py', eager_mode=False) def test_dygraph_sharding_stage3_offload(self): - self.run_mnist_2gpu('dygraph_sharding_stage3_offload.py', - eager_mode=False) + self.run_mnist_2gpu( + 'dygraph_sharding_stage3_offload.py', eager_mode=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_init.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_init.py index a7df64c1d9211..83e09ba98e02a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_init.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_init.py @@ -26,23 +26,22 @@ def gen_data(): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } def mlp(input_x, input_y, hid_dim=128, label_dim=2): fc_1 = paddle.static.nn.fc(x=input_x, size=hid_dim, activation='tanh') fc_2 = paddle.static.nn.fc(x=fc_1, size=hid_dim, activation='tanh') - prediction = paddle.static.nn.fc(x=[fc_2], - size=label_dim, - activation='softmax') + prediction = paddle.static.nn.fc( + x=[fc_2], size=label_dim, activation='softmax' + ) cost = F.cross_entropy(input=prediction, label=input_y) avg_cost = paddle.mean(x=cost) return avg_cost class TestFleetAMPInit(unittest.TestCase): - def test_fleet_amp_init(self): if not fluid.core.is_compiled_with_cuda(): return @@ -54,19 +53,20 @@ def test_fleet_amp_init(self): fleet.init(role) with paddle.static.program_guard(main_program, startup_program): - input_x = paddle.static.data(name="x", - shape=[None, 32], - dtype='float32') - input_y = paddle.static.data(name="y", - shape=[None, 1], - dtype='int64') + input_x = paddle.static.data( + name="x", shape=[None, 32], dtype='float32' + ) + input_y = paddle.static.data( + name="y", shape=[None, 1], dtype='int64' + ) cost = mlp(input_x, input_y) optimizer = paddle.optimizer.Momentum( learning_rate=0.001, momentum=0.9, weight_decay=fluid.regularizer.L2Decay(1e-4), - multi_precision=True) + multi_precision=True, + ) optimizer = paddle.static.amp.decorate(optimizer) optimizer = fleet.distributed_optimizer(optimizer) @@ -82,9 +82,9 @@ def test_fleet_amp_init(self): step = 1 for i in range(step): - cost_val = exe.run(program=main_program, - feed=gen_data(), - fetch_list=[cost.name]) + cost_val = exe.run( + program=main_program, feed=gen_data(), fetch_list=[cost.name] + ) def test_fleet_amp_meta_optimizer_init(self): if not fluid.core.is_compiled_with_cuda(): @@ -97,19 +97,20 @@ def test_fleet_amp_meta_optimizer_init(self): fleet.init(role) with paddle.static.program_guard(main_program, startup_program): - input_x = paddle.static.data(name="x", - shape=[None, 32], - dtype='float32') - input_y = paddle.static.data(name="y", - shape=[None, 1], - dtype='int64') + input_x = paddle.static.data( + name="x", shape=[None, 32], dtype='float32' + ) + input_y = paddle.static.data( + name="y", shape=[None, 1], dtype='int64' + ) cost = mlp(input_x, input_y) optimizer = paddle.optimizer.Momentum( learning_rate=0.001, momentum=0.9, weight_decay=fluid.regularizer.L2Decay(1e-4), - multi_precision=True) + multi_precision=True, + ) strategy = paddle.distributed.fleet.DistributedStrategy() strategy.amp = True @@ -131,9 +132,9 @@ def test_fleet_amp_meta_optimizer_init(self): step = 3 for i in range(step): - cost_val = exe.run(program=main_program, - feed=gen_data(), - fetch_list=[cost.name]) + cost_val = exe.run( + program=main_program, feed=gen_data(), fetch_list=[cost.name] + ) print(cost_val) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_meta_optimizer.py index 7eb235a549933..b2afb900c97c4 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_amp_meta_optimizer.py @@ -24,14 +24,14 @@ class TestFleetAMPOptimizer(TestFleetMetaOptimizer): - def test_amp_optimizer_backward(self): - """ test amp optimizer backward """ + """test amp optimizer backward""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = AMPOptimizer(opt) self.set_strategy(strategy, 'amp') @@ -44,12 +44,13 @@ def test_amp_optimizer_backward(self): self.assertNotIn('check_finite_and_unscale', ops) def test_amp_optimizer_backward_gradients(self): - """ test amp optimizer backward + gradients""" + """test amp optimizer backward + gradients""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = AMPOptimizer(opt) self.set_strategy(strategy, 'amp') @@ -64,12 +65,13 @@ def test_amp_optimizer_backward_gradients(self): self.assertIn('check_finite_and_unscale', ops) def test_amp_optimizer_backward_optimize(self): - """ test amp optimizer backward + optimizer """ + """test amp optimizer backward + optimizer""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = AMPOptimizer(opt) self.set_strategy(strategy, 'amp') @@ -83,7 +85,7 @@ def test_amp_optimizer_backward_optimize(self): self.assertIn('check_finite_and_unscale', ops) def test_amp_optimizer(self): - """ test amp """ + """test amp""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'amp') @@ -94,7 +96,7 @@ def test_amp_optimizer(self): self.assertIn('check_finite_and_unscale', ops) def test_pure_fp16_optimizer(self): - """ test pure fp16 """ + """test pure fp16""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'pure_fp16') @@ -109,7 +111,7 @@ def test_pure_fp16_optimizer(self): self.assertIn('check_finite_and_unscale', ops) def test_amp_distributed_optimizer(self): - """ test amp when distributed """ + """test amp when distributed""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'amp') @@ -125,7 +127,7 @@ def test_amp_distributed_optimizer(self): self.assertEqual(check_count, len(train_prog.all_parameters())) def test_amp_recompute_optimizer(self): - """ test amp + recompute """ + """test amp + recompute""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'amp') @@ -145,7 +147,7 @@ def test_amp_recompute_optimizer(self): self.assertIn('subprog', ''.join(outs)) def test_amp_recompute_lars_optimizer(self): - """ test amp + recompute """ + """test amp + recompute""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'amp') diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_checkpoint.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_checkpoint.py index ef091343204f3..a9c92849b0127 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_checkpoint.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_checkpoint.py @@ -26,7 +26,6 @@ class FleetTest(unittest.TestCase): - def _test_checkpoint(self, fs, dir_path): file_name = "persistables" @@ -39,8 +38,9 @@ def _test_checkpoint(self, fs, dir_path): image = fluid.data(name='img', shape=[None, 28, 28], dtype='float32') label = fluid.data(name='label', shape=[None, 1], dtype='int64') - feeder = fluid.DataFeeder(feed_list=[image, label], - place=fluid.CPUPlace()) + feeder = fluid.DataFeeder( + feed_list=[image, label], place=fluid.CPUPlace() + ) predict = fluid.layers.fc(input=image, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=predict, label=label) avg_loss = paddle.mean(loss) @@ -54,26 +54,24 @@ def _test_checkpoint(self, fs, dir_path): status = ExeTrainStatus() status.epoch_no = 2 - _, n1 = fleet.save_checkpoint(exe, - dir_path, - trainer_id=0, - train_status=status, - fs=fs) + _, n1 = fleet.save_checkpoint( + exe, dir_path, trainer_id=0, train_status=status, fs=fs + ) status2 = ExeTrainStatus() - fleet.load_checkpoint(exe, - dir_path, - trainer_id=0, - fs=fs, - train_status=status2) + fleet.load_checkpoint( + exe, dir_path, trainer_id=0, fs=fs, train_status=status2 + ) self.assertEqual(status2, status) - _, n2 = fleet.save_checkpoint(exe, - dir_path, - trainer_id=0, - train_status=status, - fs=fs, - remain_all_checkpoint=False) + _, n2 = fleet.save_checkpoint( + exe, + dir_path, + trainer_id=0, + train_status=status, + fs=fs, + remain_all_checkpoint=False, + ) self.assertEqual(n2, n1 + 1) c = CheckpointSaver(fs) @@ -82,36 +80,42 @@ def _test_checkpoint(self, fs, dir_path): # unnormal # test remain_all_checkpoint - fleet.save_checkpoint(exe, - dir_path, - trainer_id=0, - train_status=status, - fs=fs, - remain_all_checkpoint=False) + fleet.save_checkpoint( + exe, + dir_path, + trainer_id=0, + train_status=status, + fs=fs, + remain_all_checkpoint=False, + ) # can't save under a file fs = LocalFS() cache_path = "./.load_cache" fs.touch(cache_path) try: - fleet.save_checkpoint(exe, - dir_path, - trainer_id=0, - train_status=status, - fs=fs, - cache_path=cache_path) + fleet.save_checkpoint( + exe, + dir_path, + trainer_id=0, + train_status=status, + fs=fs, + cache_path=cache_path, + ) self.assertFalse(True) except: pass # can't load under a file try: - fleet.load_checkpoint(exe, - dir_path, - trainer_id=0, - train_status=status2, - fs=fs, - cache_path=cache_path) + fleet.load_checkpoint( + exe, + dir_path, + trainer_id=0, + train_status=status2, + fs=fs, + cache_path=cache_path, + ) self.assertFalse(True) except: pass diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_dgc_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_dgc_meta_optimizer.py index 522b563bc5683..e75771e5c0097 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_dgc_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_dgc_meta_optimizer.py @@ -25,15 +25,15 @@ class TestFleetDGCOptimizer(TestFleetMetaOptimizer): - def test_dgc_optimizer_backward(self): - """ test dgc optimizer backward """ + """test dgc optimizer backward""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'dgc') - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) dgc_opt = DGCOptimizer(opt) role = role_maker.PaddleCloudRoleMaker(is_collective=True) dgc_opt._set_basic_info(avg_cost, role, opt, strategy) @@ -43,13 +43,14 @@ def test_dgc_optimizer_backward(self): self.assertNotIn('dgc', ops) def test_dgc_optimizer_gradients(self): - """ test dgc optimizer backward + gradients """ + """test dgc optimizer backward + gradients""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'dgc') - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) dgc_opt = DGCOptimizer(opt) role = role_maker.PaddleCloudRoleMaker(is_collective=True) dgc_opt._set_basic_info(avg_cost, role, opt, strategy) @@ -62,13 +63,14 @@ def test_dgc_optimizer_gradients(self): self.assertIn('dgc_momentum', ops) def test_dgc_optimizer_optimize(self): - """ test dgc optimizer backward + optimize """ + """test dgc optimizer backward + optimize""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'dgc') - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) dgc_opt = DGCOptimizer(opt) role = role_maker.PaddleCloudRoleMaker(is_collective=True) dgc_opt._set_basic_info(avg_cost, role, opt, strategy) @@ -130,8 +132,8 @@ def test_dgc_recompute_optimizer(self): self.assertIn('subprog', ''.join(outs)) def test_amp_recompute_lars_dgc_not_apply_optimizer(self): - """ test amp + recompute + lars + dgc, - amp -/-> dgc, max_path is amp-->recompute-->lars + """test amp + recompute + lars + dgc, + amp -/-> dgc, max_path is amp-->recompute-->lars """ train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_distributed_strategy.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_distributed_strategy.py index d2b57d9a8d0bc..51105cbcb1028 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_distributed_strategy.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_distributed_strategy.py @@ -17,7 +17,6 @@ class TestStrategyConfig(unittest.TestCase): - def test_amp(self): strategy = paddle.distributed.fleet.DistributedStrategy() strategy.amp = True @@ -35,7 +34,7 @@ def test_amp_configs(self): "incr_every_n_steps": 1000, "incr_ratio": 2.0, "use_dynamic_loss_scaling": True, - "decr_ratio": 0.5 + "decr_ratio": 0.5, } strategy.amp_configs = configs self.assertEqual(strategy.amp_configs["init_loss_scaling"], 32768) @@ -78,7 +77,7 @@ def test_hybrid_parallel_configs(self): strategy.hybrid_configs = { "dp_degree": 1, "mp_degree": 2, - "pp_degree": 4 + "pp_degree": 4, } self.assertEqual(strategy.hybrid_configs["dp_degree"], 1) self.assertEqual(strategy.hybrid_configs["mp_degree"], 2) @@ -259,19 +258,26 @@ def test_sparse_table_configs(self): strategy = paddle.distributed.fleet.DistributedStrategy() configs = {} configs['emb'] = { - "table_parameters.emb.accessor.embed_sgd_param.adagrad.learning_rate": - 0.05, + "table_parameters.emb.accessor.embed_sgd_param.adagrad.learning_rate": 0.05, "table_parameters.emb.accessor.table_accessor_save_param.num": 2, - "table_parameters.emb.accessor.table_accessor_save_param.param": - [1, 2] + "table_parameters.emb.accessor.table_accessor_save_param.param": [ + 1, + 2, + ], } strategy.sparse_table_configs = configs self.assertEqual( - strategy.sparse_table_configs[0].accessor.embed_sgd_param.adagrad. - learning_rate, 0.05) + strategy.sparse_table_configs[ + 0 + ].accessor.embed_sgd_param.adagrad.learning_rate, + 0.05, + ) self.assertEqual( - strategy.sparse_table_configs[0].accessor. - table_accessor_save_param[0].param, 1) + strategy.sparse_table_configs[0] + .accessor.table_accessor_save_param[0] + .param, + 1, + ) strategy.adam_d2sum = True self.assertEqual(strategy.adam_d2sum, True) @@ -279,7 +285,7 @@ def test_sparse_table_configs(self): "uri": "123", "user": "456", "passwd": "789", - "hadoop_bin": "hadoop" + "hadoop_bin": "hadoop", } self.assertEqual(strategy.fs_client_param.user, "456") @@ -289,68 +295,90 @@ def test_fleet_desc_configs(self): configs['emb'] = {"sparse_optimizer": "adagrad"} strategy.fleet_desc_configs = configs self.assertEqual( - strategy.sparse_table_configs[0].accessor.embed_sgd_param.adagrad. - learning_rate, 0.05) + strategy.sparse_table_configs[ + 0 + ].accessor.embed_sgd_param.adagrad.learning_rate, + 0.05, + ) strategy = paddle.distributed.fleet.DistributedStrategy() configs = {} configs['emb'] = {"sparse_optimizer": "naive"} strategy.fleet_desc_configs = configs self.assertEqual( - strategy.sparse_table_configs[0].accessor.embed_sgd_param.naive. - learning_rate, 0.05) + strategy.sparse_table_configs[ + 0 + ].accessor.embed_sgd_param.naive.learning_rate, + 0.05, + ) strategy = paddle.distributed.fleet.DistributedStrategy() configs = {} configs['emb'] = {"sparse_optimizer": "adam"} strategy.fleet_desc_configs = configs self.assertEqual( - strategy.sparse_table_configs[0].accessor.embed_sgd_param.adam. - beta1_decay_rate, 0.9) + strategy.sparse_table_configs[ + 0 + ].accessor.embed_sgd_param.adam.beta1_decay_rate, + 0.9, + ) strategy = paddle.distributed.fleet.DistributedStrategy() configs = {} configs['emb'] = { "sparse_accessor_class": "DownpourUnitAccessor", - "embed_sparse_optimizer": "std_adagrad" + "embed_sparse_optimizer": "std_adagrad", } strategy.fleet_desc_configs = configs self.assertEqual( - strategy.sparse_table_configs[0].accessor.ctr_accessor_param. - show_scale, False) + strategy.sparse_table_configs[ + 0 + ].accessor.ctr_accessor_param.show_scale, + False, + ) self.assertEqual( - strategy.sparse_table_configs[0].accessor.embed_sgd_param.adagrad. - initial_range, 0) + strategy.sparse_table_configs[ + 0 + ].accessor.embed_sgd_param.adagrad.initial_range, + 0, + ) strategy = paddle.distributed.fleet.DistributedStrategy() configs = {} configs['emb'] = { "sparse_accessor_class": "DownpourCtrDoubleAccessor", - "embed_sparse_optimizer": "std_adagrad" + "embed_sparse_optimizer": "std_adagrad", } strategy.fleet_desc_configs = configs self.assertEqual( - strategy.sparse_table_configs[0].accessor.embed_sgd_param.adagrad. - initial_range, 0.0001) + strategy.sparse_table_configs[ + 0 + ].accessor.embed_sgd_param.adagrad.initial_range, + 0.0001, + ) strategy = paddle.distributed.fleet.DistributedStrategy() configs = {} configs['emb'] = {"sparse_optimizer": "shared_adam"} strategy.fleet_desc_configs = configs self.assertEqual( - strategy.sparse_table_configs[0].accessor.embed_sgd_param.adam. - beta1_decay_rate, 0.9) + strategy.sparse_table_configs[ + 0 + ].accessor.embed_sgd_param.adam.beta1_decay_rate, + 0.9, + ) def test_trainer_desc_configs(self): strategy = paddle.distributed.fleet.DistributedStrategy() configs = { "dump_fields_path": "dump_data", "dump_fields": ["xxx", "yyy"], - "dump_param": ['zzz'] + "dump_param": ['zzz'], } strategy.trainer_desc_configs = configs - self.assertEqual(strategy.trainer_desc_configs["dump_fields_path"], - "dump_data") + self.assertEqual( + strategy.trainer_desc_configs["dump_fields_path"], "dump_data" + ) self.assertEqual(len(strategy.trainer_desc_configs["dump_fields"]), 2) self.assertEqual(len(strategy.trainer_desc_configs["dump_param"]), 1) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_fp16_allreduce_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_fp16_allreduce_meta_optimizer.py index 363843dd5e839..e170c9567ef28 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_fp16_allreduce_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_fp16_allreduce_meta_optimizer.py @@ -23,27 +23,27 @@ class TestFleetFP16CompressOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" def net(self, main_prog, startup_prog, dtype='float32'): with fluid.program_guard(main_prog, startup_prog): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype=dtype) - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype=dtype + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -62,7 +62,8 @@ def test_fp16_allreduce_optimizer(self): ops = [op.type for op in avg_cost.block.ops] cast_out = [ - op.output('Out')[0] for op in avg_cost.block.ops + op.output('Out')[0] + for op in avg_cost.block.ops if op.type == 'cast' ] diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_gradient_merge_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_gradient_merge_meta_optimizer.py index 99bc614b14ee1..459a3233148d9 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_gradient_merge_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_gradient_merge_meta_optimizer.py @@ -21,9 +21,10 @@ class TestFleetGradientMergeMetaOptimizer(TestFleetMetaOptimizer): - def test_gradient_merge_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'gradient_merge') @@ -33,7 +34,9 @@ def test_gradient_merge_optimizer(self): self.assertIn('@GradientMerge', ''.join(vars)) def test_recom_gm_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'gradient_merge') @@ -45,7 +48,9 @@ def test_recom_gm_optimizer(self): self.assertIn('subprog', ''.join(vars)) def test_gm_amp_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'gradient_merge') @@ -57,7 +62,9 @@ def test_gm_amp_optimizer(self): self.assertIn('cast', ''.join(vars)) def test_gm_pure_fp16_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'gradient_merge') @@ -67,8 +74,9 @@ def test_gm_pure_fp16_optimizer(self): params = train_prog.all_parameters() for param in train_prog.all_parameters(): - self.assertEqual(param.dtype, - paddle.fluid.core.VarDesc.VarType.FP16) + self.assertEqual( + param.dtype, paddle.fluid.core.VarDesc.VarType.FP16 + ) vars = [x.name for x in train_prog.list_vars()] self.assertIn('@GradientMerge', ''.join(vars)) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_execution_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_execution_meta_optimizer.py index bc6a554f84d8f..a146800d0cfaa 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_execution_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_execution_meta_optimizer.py @@ -21,7 +21,6 @@ class TestFleetGraphExecutionMetaOptimizer(unittest.TestCase): - def setUp(self): try: self._dist_ut_port_0 = int(os.environ["PADDLE_DIST_UT_PORT"]) @@ -34,58 +33,53 @@ def test_graph_execution_optimizer_not_apply(self): port_a = self._dist_ut_port_0 port_b = self._dist_ut_port_1 node_a = { - "PADDLE_TRAINER_ID": - "0", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_a), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "0", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_a), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } node_b = { - "PADDLE_TRAINER_ID": - "1", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_b), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "1", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_b), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } def node_func(): import paddle.distributed.fleet as fleet + fleet.init(is_collective=True) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) exe = paddle.fluid.Executor(place=paddle.fluid.CPUPlace()) @@ -102,60 +96,55 @@ def test_graph_execution_optimizer(self): port_b = self._dist_ut_port_1 + 2 node_a = { - "PADDLE_TRAINER_ID": - "0", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_a), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "0", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_a), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } node_b = { - "PADDLE_TRAINER_ID": - "1", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_b), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "1", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_b), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } def node_func(): import paddle.distributed.fleet as fleet + fleet.init(is_collective=True) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() strategy.nccl_comm_num = 2 strategy.sync_nccl_allreduce = True optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) exe = paddle.fluid.Executor(place=paddle.fluid.CPUPlace()) exe.run(paddle.fluid.default_startup_program()) @@ -165,7 +154,7 @@ def node_func(): def gen_data(): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } for i in range(10): @@ -182,58 +171,53 @@ def test_graph_execution_optimizer_not_apply_v2(self): port_a = self._dist_ut_port_0 + 4 port_b = self._dist_ut_port_1 + 4 node_a = { - "PADDLE_TRAINER_ID": - "0", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_a), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "0", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_a), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } node_b = { - "PADDLE_TRAINER_ID": - "1", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_b), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "1", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_b), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } def node_func(): import paddle.distributed.fleet as fleet + fleet.init(is_collective=True) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) exe = paddle.fluid.Executor(place=paddle.fluid.CPUPlace()) @@ -249,60 +233,55 @@ def test_graph_execution_optimizer_v2(self): port_a = self._dist_ut_port_0 + 6 port_b = self._dist_ut_port_1 + 6 node_a = { - "PADDLE_TRAINER_ID": - "0", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_a), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "0", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_a), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } node_b = { - "PADDLE_TRAINER_ID": - "1", - "PADDLE_CURRENT_ENDPOINT": - "127.0.0.1:{}".format(port_b), - "PADDLE_TRAINERS_NUM": - "2", - "PADDLE_TRAINER_ENDPOINTS": - "127.0.0.1:{},127.0.0.1:{}".format(port_a, port_b), - "http_proxy": - "", - "https_proxy": - "" + "PADDLE_TRAINER_ID": "1", + "PADDLE_CURRENT_ENDPOINT": "127.0.0.1:{}".format(port_b), + "PADDLE_TRAINERS_NUM": "2", + "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:{},127.0.0.1:{}".format( + port_a, port_b + ), + "http_proxy": "", + "https_proxy": "", } def node_func(): import paddle.distributed.fleet as fleet + fleet.init(is_collective=True) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() strategy.nccl_comm_num = 2 strategy.sync_nccl_allreduce = True optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) exe = paddle.fluid.Executor(place=paddle.fluid.CPUPlace()) exe.run(paddle.fluid.default_startup_program()) @@ -312,7 +291,7 @@ def node_func(): def gen_data(): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } for i in range(10): diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_executor.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_executor.py index 3d30baba9a3f6..0a75eeac8cb4c 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_executor.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_graph_executor.py @@ -21,7 +21,6 @@ class TestFleetGraphExecutionMetaOptimizer(unittest.TestCase): - def test_graph_execution_optimizer(self): node_a = { "PADDLE_TRAINER_ID": "0", @@ -30,7 +29,7 @@ def test_graph_execution_optimizer(self): "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:36001,127.0.0.1:36002", "http_proxy": "", "https_proxy": "", - "FLAGS_CONVERT_GRAPH_TO_PROGRAM": "1" + "FLAGS_CONVERT_GRAPH_TO_PROGRAM": "1", } node_b = { @@ -40,34 +39,36 @@ def test_graph_execution_optimizer(self): "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:36001,127.0.0.1:36002", "http_proxy": "", "https_proxy": "", - "FLAGS_CONVERT_GRAPH_TO_PROGRAM": "1" + "FLAGS_CONVERT_GRAPH_TO_PROGRAM": "1", } def node_func(): role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() strategy.nccl_comm_num = 2 strategy.sync_nccl_allreduce = True optimizer = paddle.optimizer.SGD(learning_rate=0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) exe = paddle.fluid.Executor(place=paddle.fluid.CPUPlace()) exe.run(paddle.fluid.default_startup_program()) @@ -77,7 +78,7 @@ def node_func(): def gen_data(): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } for i in range(5): diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_hybrid_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_hybrid_meta_optimizer.py index 3062812223d64..5933b3a96a4c8 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_hybrid_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_hybrid_meta_optimizer.py @@ -24,11 +24,11 @@ class TestFleetHybridOptimizer(TestFleetMetaOptimizer): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "3" - os.environ["PADDLE_TRAINER_ENDPOINTS"] = \ - "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" + os.environ[ + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" # pre-assigned ring id self.mp_ring_id = 0 self.sharding_ring_id = 1 @@ -65,40 +65,122 @@ def test_opt_sharding_with_pp(self): main_prog_op_types = [op.type for op in main_prog_ops] # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'softmax', 'cross_entropy2', 'reduce_mean', - 'fill_constant', 'reduce_mean_grad', 'cross_entropy_grad2', - 'softmax_grad', 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'fill_constant', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_sync_comm_stream', 'momentum', 'momentum', - 'momentum', 'momentum', 'momentum', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.dp_ring_id, created_ring_ids) @@ -107,16 +189,20 @@ def test_opt_sharding_with_pp(self): # check correctness of pp group pp_group_waiting_prots = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and \ - op.desc.output_arg_names()[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): pp_group_waiting_prots = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_prots, ['127.0.0.1:36003']) # check correctness of sharding group dp_group_waiting_ports = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" \ - and op.desc.output_arg_names()[0] == "comm_id_3": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_3" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) @@ -147,34 +233,109 @@ def test_opt_sharding_with_pp_with_allreduce_fuse(self): main_prog_op_types = [op.type for op in main_prog_ops] # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'softmax', 'cross_entropy2', 'reduce_mean', - 'fill_constant', 'reduce_mean_grad', 'cross_entropy_grad2', - 'softmax_grad', 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'fill_constant', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'coalesce_tensor', 'c_reduce_sum', 'coalesce_tensor', - 'c_reduce_sum', 'c_sync_comm_stream', 'momentum', 'momentum', - 'momentum', 'momentum', 'momentum', 'coalesce_tensor', - 'c_broadcast', 'coalesce_tensor', 'c_broadcast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'coalesce_tensor', + 'c_reduce_sum', + 'coalesce_tensor', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'coalesce_tensor', + 'c_broadcast', + 'coalesce_tensor', + 'c_broadcast', + ], + ) def test_opt_sharding_with_pp_amp_gclip(self): train_prog, startup_prog = static.Program(), static.Program() @@ -194,11 +355,9 @@ def test_opt_sharding_with_pp_amp_gclip(self): strategy.fuse_grad_size_in_MB = 32 clip = paddle.fluid.clip.GradientClipByGlobalNorm(1.0) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - grad_clip=clip) + self.optimizer( + avg_cost, strategy, train_prog, startup_prog, grad_clip=clip + ) train_prog = train_prog._pipeline_opt['section_program'] startup_prog = startup_prog._pipeline_opt['startup_program'] self.debug_program(train_prog, startup_prog) @@ -211,46 +370,169 @@ def test_opt_sharding_with_pp_amp_gclip(self): main_prog_op_types = [op.type for op in main_prog_ops] # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'cast', - 'tanh', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'cast', - 'tanh', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'cast', - 'tanh', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'softmax', - 'cast', 'cross_entropy2', 'reduce_mean', 'elementwise_mul', - 'fill_constant', 'elementwise_mul_grad', 'reduce_mean_grad', - 'cross_entropy_grad2', 'cast', 'softmax_grad', - 'elementwise_add_grad', 'mul_grad', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'cast', 'c_sync_calc_stream', - 'send_v2', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'coalesce_tensor', 'c_reduce_sum', 'coalesce_tensor', - 'c_reduce_sum', 'c_sync_comm_stream', 'check_finite_and_unscale', - 'cast', 'c_allreduce_max', 'c_allreduce_max', 'cast', - 'update_loss_scaling', 'squared_l2_norm', 'squared_l2_norm', - 'squared_l2_norm', 'squared_l2_norm', 'squared_l2_norm', 'sum', - 'c_allreduce_sum', 'c_allreduce_sum', 'sqrt', 'fill_constant', - 'elementwise_max', 'elementwise_div', 'elementwise_mul', - 'elementwise_mul', 'elementwise_mul', 'elementwise_mul', - 'elementwise_mul', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'coalesce_tensor', 'c_broadcast', 'coalesce_tensor', - 'c_broadcast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'coalesce_tensor', + 'c_reduce_sum', + 'coalesce_tensor', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'squared_l2_norm', + 'squared_l2_norm', + 'squared_l2_norm', + 'squared_l2_norm', + 'squared_l2_norm', + 'sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'coalesce_tensor', + 'c_broadcast', + 'coalesce_tensor', + 'c_broadcast', + ], + ) def test_opt_sharding_with_pp_amp_gclip_fuse_gm(self): train_prog, startup_prog = static.Program(), static.Program() @@ -271,11 +553,9 @@ def test_opt_sharding_with_pp_amp_gclip_fuse_gm(self): strategy.fuse_grad_merge = True clip = paddle.fluid.clip.GradientClipByGlobalNorm(1.0) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - grad_clip=clip) + self.optimizer( + avg_cost, strategy, train_prog, startup_prog, grad_clip=clip + ) train_prog = train_prog._pipeline_opt['section_program'] startup_prog = startup_prog._pipeline_opt['startup_program'] self.debug_program(train_prog, startup_prog) @@ -288,43 +568,151 @@ def test_opt_sharding_with_pp_amp_gclip_fuse_gm(self): main_prog_op_types = [op.type for op in main_prog_ops] # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'cast', - 'tanh', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'cast', - 'tanh', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'cast', - 'tanh', 'cast', 'cast', 'mul', 'cast', 'elementwise_add', 'softmax', - 'cast', 'cross_entropy2', 'reduce_mean', 'elementwise_mul', - 'coalesce_tensor', 'coalesce_tensor', 'coalesce_tensor', - 'coalesce_tensor', 'fill_constant', 'elementwise_mul_grad', - 'reduce_mean_grad', 'cross_entropy_grad2', 'cast', 'softmax_grad', - 'elementwise_add_grad', 'mul_grad', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'cast', 'c_sync_calc_stream', - 'send_v2', 'cast', 'sum', 'cast', 'sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_sync_comm_stream', 'check_finite_and_unscale', - 'cast', 'c_allreduce_max', 'c_allreduce_max', 'cast', - 'update_loss_scaling', 'squared_l2_norm', 'squared_l2_norm', - 'squared_l2_norm', 'squared_l2_norm', 'squared_l2_norm', 'sum', - 'c_allreduce_sum', 'c_allreduce_sum', 'sqrt', 'fill_constant', - 'elementwise_max', 'elementwise_div', 'elementwise_mul', - 'elementwise_mul', 'elementwise_mul', 'elementwise_mul', - 'elementwise_mul', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'coalesce_tensor', 'c_broadcast', 'coalesce_tensor', - 'c_broadcast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'c_sync_calc_stream', + 'send_v2', + 'cast', + 'sum', + 'cast', + 'sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'squared_l2_norm', + 'squared_l2_norm', + 'squared_l2_norm', + 'squared_l2_norm', + 'squared_l2_norm', + 'sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'coalesce_tensor', + 'c_broadcast', + 'coalesce_tensor', + 'c_broadcast', + ], + ) def test_opt_sharding_with_pp_amp_ckp_fuse_gm_optcast(self): train_prog, startup_prog = static.Program(), static.Program() @@ -337,8 +725,12 @@ def test_opt_sharding_with_pp_amp_ckp_fuse_gm_optcast(self): } strategy.recompute = True strategy.recompute_configs = { - "checkpoints": - ["fc_0.tmp_2", "fc_1.tmp_2", "fc_2.tmp_2", "fc_3.tmp_2"] + "checkpoints": [ + "fc_0.tmp_2", + "fc_1.tmp_2", + "fc_2.tmp_2", + "fc_3.tmp_2", + ] } strategy.sharding = True @@ -368,49 +760,158 @@ def test_opt_sharding_with_pp_amp_ckp_fuse_gm_optcast(self): main_prog_op_types = [op.type for op in main_prog_ops] # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', - 'cast', 'c_broadcast', 'cast', 'c_broadcast', 'cast', 'c_broadcast', - 'cast', 'c_broadcast', 'cast', 'c_broadcast', 'cast', 'c_broadcast', - 'cast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'mul', 'elementwise_add', 'cast', 'tanh', 'cast', - 'mul', 'elementwise_add', 'cast', 'tanh', 'cast', 'mul', - 'elementwise_add', 'cast', 'tanh', 'cast', 'mul', 'cast', - 'elementwise_add', 'cast', 'softmax', 'cast', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'coalesce_tensor', - 'coalesce_tensor', 'coalesce_tensor', 'coalesce_tensor', - 'coalesce_tensor', 'coalesce_tensor', 'fill_constant', - 'elementwise_mul_grad', 'reduce_mean_grad', 'cross_entropy_grad2', - 'cast', 'softmax_grad', 'cast', 'elementwise_add_grad', 'cast', - 'mul_grad', 'cast', 'tanh_grad', 'cast', 'elementwise_add_grad', - 'mul_grad', 'cast', 'tanh_grad', 'cast', 'elementwise_add_grad', - 'mul_grad', 'cast', 'cast', 'mul', 'elementwise_add', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'c_sync_calc_stream', 'send_v2', 'cast', 'sum', 'sum', 'cast', - 'sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'c_allreduce_max', 'cast', 'update_loss_scaling', - 'momentum', 'cast', 'momentum', 'cast', 'momentum', 'cast', - 'momentum', 'momentum', 'cast', 'coalesce_tensor', 'c_broadcast', - 'c_broadcast', 'coalesce_tensor', 'c_broadcast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'cast', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'c_sync_calc_stream', + 'send_v2', + 'cast', + 'sum', + 'sum', + 'cast', + 'sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'momentum', + 'cast', + 'coalesce_tensor', + 'c_broadcast', + 'c_broadcast', + 'coalesce_tensor', + 'c_broadcast', + ], + ) class TestFleetHybridOptimizerBoundary(TestFleetMetaOptimizer): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "3" - os.environ["PADDLE_TRAINER_ENDPOINTS"] = \ - "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" + os.environ[ + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" # pre-assigned ring id self.mp_ring_id = 0 self.sharding_ring_id = 1 @@ -440,11 +941,9 @@ def test_opt_sharding_with_pp_amp_gclip_boundary(self): strategy.fuse_grad_size_in_MB = 32 clip = paddle.fluid.clip.GradientClipByGlobalNorm(1.0) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - grad_clip=clip) + self.optimizer( + avg_cost, strategy, train_prog, startup_prog, grad_clip=clip + ) train_prog = train_prog._pipeline_opt['section_program'] startup_prog = startup_prog._pipeline_opt['startup_program'] self.debug_program(train_prog, startup_prog) @@ -461,32 +960,74 @@ def test_opt_sharding_with_pp_amp_gclip_boundary(self): if is_loss_grad_op(op): self.assertEqual(op.type, 'fill_constant') self.assertTrue(op.has_attr('value')) - scale = strategy.pipeline_configs[ - 'accumulate_steps'] * strategy.sharding_configs['dp_degree'] + scale = ( + strategy.pipeline_configs['accumulate_steps'] + * strategy.sharding_configs['dp_degree'] + ) loss_scale = 1.0 / scale self.assertAlmostEqual(float(op.attr('value')), loss_scale) # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', - 'c_comm_init', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'matmul', 'cast', 'reduce_mean', - 'elementwise_mul', 'fill_constant', 'elementwise_mul_grad', - 'reduce_mean_grad', 'cast', 'matmul_grad', 'c_sync_calc_stream', - 'send_v2', 'fill_constant', 'cast', 'sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'c_allreduce_max', 'cast', 'update_loss_scaling', - 'fill_constant', 'c_allreduce_sum', 'c_allreduce_sum', 'sqrt', - 'fill_constant', 'elementwise_max', 'elementwise_div', 'c_broadcast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'matmul', + 'cast', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cast', + 'matmul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'cast', + 'sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'fill_constant', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'c_broadcast', + ], + ) def test_opt_sharding_with_pp_amp_gclip_boundary_card1(self): - """ test optimizer sharding without parameter in card0 """ + """test optimizer sharding without parameter in card0""" os.environ["PADDLE_TRAINER_ID"] = "1" train_prog, startup_prog = static.Program(), static.Program() avg_cost, strategy = self.boundary_net(train_prog, startup_prog) @@ -504,11 +1045,9 @@ def test_opt_sharding_with_pp_amp_gclip_boundary_card1(self): strategy.fuse_grad_size_in_MB = 32 clip = paddle.fluid.clip.GradientClipByGlobalNorm(1.0) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - grad_clip=clip) + self.optimizer( + avg_cost, strategy, train_prog, startup_prog, grad_clip=clip + ) train_prog = train_prog._pipeline_opt['section_program'] startup_prog = startup_prog._pipeline_opt['startup_program'] self.debug_program(train_prog, startup_prog) @@ -521,24 +1060,67 @@ def test_opt_sharding_with_pp_amp_gclip_boundary_card1(self): main_prog_op_types = [op.type for op in main_prog_ops] # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'matmul', 'cast', 'reduce_mean', - 'elementwise_mul', 'fill_constant', 'elementwise_mul_grad', - 'reduce_mean_grad', 'cast', 'matmul_grad', 'c_sync_calc_stream', - 'send_v2', 'fill_constant', 'cast', 'sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'c_allreduce_max', 'cast', 'update_loss_scaling', - 'squared_l2_norm', 'sum', 'c_allreduce_sum', 'c_allreduce_sum', - 'sqrt', 'fill_constant', 'elementwise_max', 'elementwise_div', - 'elementwise_mul', 'momentum', 'c_broadcast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'matmul', + 'cast', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cast', + 'matmul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'cast', + 'sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'squared_l2_norm', + 'sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'elementwise_mul', + 'momentum', + 'c_broadcast', + ], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lamb_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lamb_meta_optimizer.py index 1c20d2e45be03..a8fa8c7107b4a 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lamb_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lamb_meta_optimizer.py @@ -23,31 +23,32 @@ class TestFleetLambMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "1" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002" def net(self, main_prog, startup_prog): with fluid.program_guard(main_prog, startup_prog): with fluid.unique_name.guard(): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') - - fc_1 = paddle.fluid.layers.fc(input=input_x, - size=64, - act='tanh') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) + + fc_1 = paddle.fluid.layers.fc( + input=input_x, size=64, act='tanh' + ) fc_2 = paddle.fluid.layers.fc(input=fc_1, size=256, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -78,8 +79,9 @@ def test_lamb_not_apply_with_momentum(self): startup_prog = fluid.Program() train_prog = fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) - optimizer = paddle.fluid.optimizer.Momentum(learning_rate=0.1, - momentum=0.9) + optimizer = paddle.fluid.optimizer.Momentum( + learning_rate=0.1, momentum=0.9 + ) optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) optimizer.minimize(avg_cost) @@ -101,7 +103,8 @@ def test_lamb_exclude_fn(self): optimizer.minimize(avg_cost) ops_without_wd = [ - op for op in avg_cost.block.ops + op + for op in avg_cost.block.ops if op.type == 'lamb' and op.attr('op_role_var')[0].endswith('.b_0') ] for op in ops_without_wd: @@ -110,16 +113,17 @@ def test_lamb_exclude_fn(self): def test_lamb_apply_with_amp(self): role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lars_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lars_meta_optimizer.py index b560cdaa66ef4..0a7298631cf9d 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lars_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_lars_meta_optimizer.py @@ -23,31 +23,32 @@ class TestFleetLarsMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "1" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002" def net(self, main_prog, startup_prog): with fluid.program_guard(main_prog, startup_prog): with fluid.unique_name.guard(): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') - - fc_1 = paddle.fluid.layers.fc(input=input_x, - size=64, - act='tanh') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) + + fc_1 = paddle.fluid.layers.fc( + input=input_x, size=64, act='tanh' + ) fc_2 = paddle.fluid.layers.fc(input=fc_1, size=256, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -67,8 +68,9 @@ def test_lars_optimizer(self): startup_prog = fluid.Program() train_prog = fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) - optimizer = paddle.fluid.optimizer.Momentum(learning_rate=0.01, - momentum=0.9) + optimizer = paddle.fluid.optimizer.Momentum( + learning_rate=0.01, momentum=0.9 + ) optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) optimizer.minimize(avg_cost) @@ -94,16 +96,21 @@ def test_lars_exclude_fn(self): startup_prog = fluid.Program() train_prog = fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) - optimizer = paddle.fluid.optimizer.Momentum(learning_rate=0.01, - momentum=0.9) + optimizer = paddle.fluid.optimizer.Momentum( + learning_rate=0.01, momentum=0.9 + ) optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) optimizer.minimize(avg_cost) ops_without_wd = [ - op for op in avg_cost.block.ops - if op.type == 'lars_momentum' and ("batch_norm" in op.attr( - 'op_role_var')[0] or ".b" in op.attr('op_role_var')[0]) + op + for op in avg_cost.block.ops + if op.type == 'lars_momentum' + and ( + "batch_norm" in op.attr('op_role_var')[0] + or ".b" in op.attr('op_role_var')[0] + ) ] for op in ops_without_wd: self.assertEqual(op.attr('lars_weight_decay')[0], 0) @@ -111,16 +118,17 @@ def test_lars_exclude_fn(self): def test_lars_apply_with_amp(self): role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -143,8 +151,9 @@ def test_lars_apply_with_amp(self): "exclude_from_weight_decay": ["batch_norm", ".b"], } - optimizer = paddle.fluid.optimizer.Momentum(learning_rate=0.01, - momentum=0.9) + optimizer = paddle.fluid.optimizer.Momentum( + learning_rate=0.01, momentum=0.9 + ) optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) optimizer.minimize(avg_cost) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_localsgd_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_localsgd_meta_optimizer.py index 2765b8c3eaea0..a3711c6f84180 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_localsgd_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_localsgd_meta_optimizer.py @@ -23,7 +23,6 @@ class TestFleetLocalSGDMetaOptimizer(TestFleetMetaOptimizer): - def test_localsgd_optimizer(self): train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) @@ -32,7 +31,8 @@ def test_localsgd_optimizer(self): ops = [op.type for op in avg_cost.block.ops] outs = [ - ''.join(op.output('Out')) for op in avg_cost.block.ops + ''.join(op.output('Out')) + for op in avg_cost.block.ops if op.type == 'conditional_block' ] @@ -48,7 +48,8 @@ def test_localsgd_amp_optimizer(self): ops = [op.type for op in avg_cost.block.ops] outs = [ - ''.join(op.output('Out')) for op in avg_cost.block.ops + ''.join(op.output('Out')) + for op in avg_cost.block.ops if op.type == 'conditional_block' ] @@ -61,7 +62,6 @@ def test_localsgd_amp_optimizer(self): class TestFleetAdaptiveLocalSGDMetaOptimizer(TestFleetMetaOptimizer): - def test_adaptive_localsgd_optimizer(self): train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) @@ -70,7 +70,8 @@ def test_adaptive_localsgd_optimizer(self): ops = [op.type for op in avg_cost.block.ops] outs = [ - ''.join(op.output('Out')) for op in avg_cost.block.ops + ''.join(op.output('Out')) + for op in avg_cost.block.ops if op.type == 'conditional_block' ] @@ -86,7 +87,8 @@ def test_localsgd_amp_optimizer(self): ops = [op.type for op in avg_cost.block.ops] outs = [ - ''.join(op.output('Out')) for op in avg_cost.block.ops + ''.join(op.output('Out')) + for op in avg_cost.block.ops if op.type == 'conditional_block' ] diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_log.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_log.py index 985905bdc815f..93027dfb8b7dd 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_log.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_log.py @@ -19,7 +19,6 @@ class TestFleetLog(unittest.TestCase): - def setUp(self): fleet.init(log_level="DEBUG") diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_meta_optimizer_base.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_meta_optimizer_base.py index 0c8db67dcb5f4..f127e072dd11c 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_meta_optimizer_base.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_meta_optimizer_base.py @@ -17,40 +17,44 @@ from paddle import fluid import paddle.distributed.fleet as fleet import paddle.distributed.fleet.base.role_maker as role_maker -from paddle.distributed.fleet.meta_optimizers.meta_optimizer_base import MetaOptimizerBase +from paddle.distributed.fleet.meta_optimizers.meta_optimizer_base import ( + MetaOptimizerBase, +) class TestFleetMetaOptimizerBase(unittest.TestCase): - def net(main_prog, startup_prog): with fluid.program_guard(main_prog, startup_prog): with fluid.unique_name.guard(): role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') - - fc_1 = paddle.fluid.layers.fc(input=input_x, - size=64, - act='tanh') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) + + fc_1 = paddle.fluid.layers.fc( + input=input_x, size=64, act='tanh' + ) fc_2 = paddle.fluid.layers.fc(input=fc_1, size=256, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) opt = MetaOptimizerBase(optimizer) opt_ops, params_grads = opt.minimize(avg_cost) - opt.apply_optimize(avg_cost, - paddle.static.default_startup_program(), - params_grads) + opt.apply_optimize( + avg_cost, + paddle.static.default_startup_program(), + params_grads, + ) return None net(fluid.default_startup_program(), fluid.default_main_program()) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer.py index 279a2e21f70ef..bee864e1f24f8 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer.py @@ -24,23 +24,23 @@ class TestFleetMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "1" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002" def net(self): with static.device_guard("gpu:0"): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') - input_z = paddle.fluid.layers.data(name="z", - shape=[1], - dtype="float32") + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) + input_z = paddle.fluid.layers.data( + name="z", shape=[1], dtype="float32" + ) with static.device_guard("gpu:all"): input_z = input_z * 1.0 input_z.stop_gradient = True @@ -52,11 +52,12 @@ def net(self): # for pipeline check_pipeline_persist_var coverage fc_2.persistable = True fc_2 = fc_2 * input_z - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) return avg_cost @@ -68,7 +69,7 @@ def test_pipeline_optimizer(self): strategy.pipeline = True strategy.pipeline_configs = { 'micro_batch_size': 1, - 'accumulate_steps': 2 + 'accumulate_steps': 2, } train_prog, startup_prog = static.Program(), static.Program() @@ -77,12 +78,13 @@ def test_pipeline_optimizer(self): avg_cost = self.net() optimizer = paddle.fluid.optimizer.Adam(0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) def test_pipeline_amp_optimizer(self): - """ test pipeline& with device:all """ + """test pipeline& with device:all""" role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) @@ -91,7 +93,7 @@ def test_pipeline_amp_optimizer(self): strategy.pipeline = True strategy.pipeline_configs = { 'micro_batch_size': 1, - 'accumulate_steps': 2 + 'accumulate_steps': 2, } train_prog, startup_prog = static.Program(), static.Program() @@ -100,8 +102,9 @@ def test_pipeline_amp_optimizer(self): avg_cost = self.net() optimizer = paddle.fluid.optimizer.Adam(0.01) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) ops = train_prog._pipeline_opt['section_program'].global_block().ops diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer_with_recompute.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer_with_recompute.py index c45c81c35b42b..8ce94b61a6334 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer_with_recompute.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_pipeline_meta_optimizer_with_recompute.py @@ -20,24 +20,25 @@ class TestFleetMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "1" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002" def test_pipeline_optimizer(self): import paddle.distributed.fleet as fleet import paddle.distributed.fleet.base.role_maker as role_maker + role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) with paddle.fluid.device_guard("gpu:0"): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') fc_3 = paddle.fluid.layers.fc(input=fc_2, size=64, act='tanh') @@ -47,11 +48,12 @@ def test_pipeline_optimizer(self): with paddle.fluid.device_guard("gpu:1"): fc_7 = paddle.fluid.layers.fc(input=fc_6, size=64, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_7], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_7], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -59,7 +61,7 @@ def test_pipeline_optimizer(self): strategy.pipeline_configs = { 'micro_batch_size': 1, 'accumulate_steps': 2, - 'schedule_mode': '1F1B' + 'schedule_mode': '1F1B', } checkpoints = ['fc_5.tmp_0', 'fc_7.tmp_0'] @@ -67,7 +69,7 @@ def test_pipeline_optimizer(self): strategy.recompute_configs = { "checkpoints": checkpoints, "enable_offload": False, - "checkpoint_shape": [] + "checkpoint_shape": [], } optimizer = paddle.fluid.optimizer.Adam(0.01) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_private_function.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_private_function.py index a4854c695f8cc..94c8437167e81 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_private_function.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_private_function.py @@ -18,11 +18,10 @@ class TestFleetPrivateFunction(unittest.TestCase): - def test_wait_port(self): - def init_server(port): import time + time.sleep(5) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(("127.0.0.1", port)) @@ -33,10 +32,11 @@ def init_server(port): c.close() break - thr = threading.Thread(target=init_server, args=(9292, )) + thr = threading.Thread(target=init_server, args=(9292,)) thr.start() import paddle.distributed.fleet as fleet + ep = ["127.0.0.1:9292"] fleet.base.private_helper_function.wait_server_ready(ep) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_raw_program_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_raw_program_meta_optimizer.py index 3fde52958d353..e985415d943cc 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_raw_program_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_raw_program_meta_optimizer.py @@ -20,27 +20,29 @@ class TestFleetMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "1" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002" def test_pipeline_optimizer(self): import paddle.distributed.fleet as fleet import paddle.distributed.fleet.base.role_maker as role_maker + role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_recompute_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_recompute_meta_optimizer.py index 0045f7311deee..9de35f8ca35a1 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_recompute_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_recompute_meta_optimizer.py @@ -22,15 +22,15 @@ class TestFleetRecomputeMetaOptimizer(TestFleetMetaOptimizer): - def test_recompute_optimizer_backward(self): - """ test recompute optimizer backward """ + """test recompute optimizer backward""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'recompute') - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = RecomputeOptimizer(opt) opt.user_defined_strategy = strategy params_grads = opt.backward(avg_cost, startup_prog) @@ -41,13 +41,14 @@ def test_recompute_optimizer_backward(self): self.assertIn('subprog', ''.join(outs)) def test_recompute_optimizer_backward_gradients(self): - """ test recompute optimizer backward + gradients """ + """test recompute optimizer backward + gradients""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'recompute') - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = RecomputeOptimizer(opt) opt.user_defined_strategy = strategy params_grads = opt.backward(avg_cost, startup_prog) @@ -60,13 +61,14 @@ def test_recompute_optimizer_backward_gradients(self): self.assertIn('subprog', ''.join(outs)) def test_recompute_optimizer_backward_optimize(self): - """ test recompute optimizer backward + optimize """ + """test recompute optimizer backward + optimize""" train_prog, startup_prog = fluid.Program(), fluid.Program() avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'recompute') - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = RecomputeOptimizer(opt) opt.user_defined_strategy = strategy params_grads = opt.backward(avg_cost, startup_prog) @@ -126,7 +128,8 @@ def test_recompute_offload(self): self.optimizer(avg_cost, strategy, train_prog, startup_prog) ops = [op.type for op in avg_cost.block.ops] outs = [ - op.output('Out')[0] for op in avg_cost.block.ops + op.output('Out')[0] + for op in avg_cost.block.ops if op.type == 'memcpy' ] self.assertIn('memcpy', ops) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_rolemaker_new.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_rolemaker_new.py index 18ef1f786332e..18c2487b66a95 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_rolemaker_new.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_rolemaker_new.py @@ -59,9 +59,11 @@ def setUp(self): """Set up, set envs.""" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.2:36001" os.environ["POD_IP"] = "127.0.0.1" def test_tr_rolemaker(self): @@ -103,8 +105,9 @@ def test_ps_rolemaker(self): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" - ro = role_maker.PaddleCloudRoleMaker(is_collective=False, - init_gloo=False) + ro = role_maker.PaddleCloudRoleMaker( + is_collective=False, init_gloo=False + ) self.assertEqual(ro._server_index(), 0) self.assertFalse(ro._is_worker()) self.assertTrue(ro._is_server()) @@ -139,7 +142,8 @@ def test_ps_rolemaker(self): server_endpoints=["127.0.0.1:36001", "127.0.0.1:36001"], role=role_maker.Role.SERVER, current_id=0, - worker_num=2) + worker_num=2, + ) self.assertEqual(ro._server_num(), 2) ro._generate_role() self.assertTrue(ro._is_server()) @@ -152,7 +156,8 @@ def test_tr_rolemaker(self): server_endpoints=["127.0.0.1:36001", "127.0.0.1:36001"], role=role_maker.Role.WORKER, current_id=0, - worker_num=2) + worker_num=2, + ) self.assertIn("127.0.0.1:36001", ro._get_pserver_endpoints()) self.assertTrue(ro._is_worker()) @@ -160,7 +165,6 @@ def test_tr_rolemaker(self): class TestGlooWithCloudRoleMaker(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINERS_NUM"] = "1" os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001" @@ -287,6 +291,7 @@ def test_fs_gloo4(self): role = role_maker.PaddleCloudRoleMaker(is_collective=True) role._generate_role() import time + time.sleep(3) def test_fs_gloo5(self): @@ -443,8 +448,9 @@ def net(): x = paddle.fluid.layers.data(name='x', shape=[13], dtype='float32') y_predict = paddle.fluid.layers.fc(input=x, size=1, act=None) y = paddle.fluid.layers.data(name='y', shape=[1], dtype='float32') - cost = paddle.fluid.layers.square_error_cost(input=y_predict, - label=y) + cost = paddle.fluid.layers.square_error_cost( + input=y_predict, label=y + ) avg_cost = paddle.mean(cost) return avg_cost diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_sharding_meta_optimizer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_sharding_meta_optimizer.py index 6c14b49278c5a..57a199c133395 100755 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_sharding_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_sharding_meta_optimizer.py @@ -25,9 +25,10 @@ class TestFleetShardingMetaOptimizer(TestFleetMetaOptimizer): - def test_sharding_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') @@ -40,28 +41,74 @@ def test_sharding_optimizer(self): self.assertIn('@BroadCast', ''.join(vars)) self.assertEqual( set(parameters), - set([ - "fc_1.b_0", "fc_2.b_0", "fc_2.w_0", "fc_1.b_0_velocity_0", - "fc_2.b_0_velocity_0", "fc_2.w_0_velocity_0", "learning_rate_0" - ])) - - self.assertEqual(ops, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'mul', 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', - 'mul', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'fill_constant', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'mul_grad', 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'momentum', 'momentum', 'momentum' - ]) + set( + [ + "fc_1.b_0", + "fc_2.b_0", + "fc_2.w_0", + "fc_1.b_0_velocity_0", + "fc_2.b_0_velocity_0", + "fc_2.w_0_velocity_0", + "learning_rate_0", + ] + ), + ) + + self.assertEqual( + ops, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_sharding_amp_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') @@ -77,33 +124,101 @@ def test_sharding_amp_optimizer(self): self.assertIn('check_finite_and_unscale', ops) self.assertEqual( set(parameters), - set([ - "fc_1.b_0", "fc_2.b_0", "fc_2.w_0", "fc_1.b_0_velocity_0", - "fc_2.b_0_velocity_0", "fc_2.w_0_velocity_0", "learning_rate_0", - "loss_scaling_0", "num_bad_steps_0", "num_good_steps_0" - ])) - - self.assertEqual(ops, [ - 'cast', 'cast', 'cast', 'fill_constant', 'fill_constant', - 'fill_constant', 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_sync_comm_stream', 'cast', 'mul', 'elementwise_add', 'cast', - 'tanh', 'cast', 'mul', 'elementwise_add', 'cast', 'tanh', 'cast', - 'mul', 'elementwise_add', 'softmax', 'cast', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'fill_constant', - 'elementwise_mul_grad', 'reduce_mean_grad', 'cross_entropy_grad2', - 'cast', 'softmax_grad', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'cast', 'cast', 'cast', - 'check_finite_and_unscale', 'cast', 'c_allreduce_max', 'cast', - 'update_loss_scaling', 'momentum', 'momentum', 'momentum' - ]) + set( + [ + "fc_1.b_0", + "fc_2.b_0", + "fc_2.w_0", + "fc_1.b_0_velocity_0", + "fc_2.b_0_velocity_0", + "fc_2.w_0_velocity_0", + "learning_rate_0", + "loss_scaling_0", + "num_bad_steps_0", + "num_good_steps_0", + ] + ), + ) + + self.assertEqual( + ops, + [ + 'cast', + 'cast', + 'cast', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'cast', + 'cast', + 'cast', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_sharding_recompute_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') @@ -120,29 +235,78 @@ def test_sharding_recompute_optimizer(self): self.assertIn('subprog', ''.join(vars)) self.assertEqual( set(parameters), - set([ - "fc_1.b_0", "fc_2.b_0", "fc_2.w_0", "fc_1.b_0_velocity_0", - "fc_2.b_0_velocity_0", "fc_2.w_0_velocity_0", "learning_rate_0" - ])) - - self.assertEqual(ops, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'mul', 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', - 'mul', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'fill_constant', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'mul_grad', 'mul', 'elementwise_add', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'mul', 'elementwise_add', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'momentum', 'momentum', 'momentum' - ]) + set( + [ + "fc_1.b_0", + "fc_2.b_0", + "fc_2.w_0", + "fc_1.b_0_velocity_0", + "fc_2.b_0_velocity_0", + "fc_2.w_0_velocity_0", + "learning_rate_0", + ] + ), + ) + + self.assertEqual( + ops, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'mul', + 'elementwise_add', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'mul', + 'elementwise_add', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_sharding_amp_recompute_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') @@ -163,36 +327,115 @@ def test_sharding_amp_recompute_optimizer(self): self.assertEqual( set(parameters), - set([ - "fc_1.b_0", "fc_2.b_0", "fc_2.w_0", "fc_1.b_0_velocity_0", - "fc_2.b_0_velocity_0", "fc_2.w_0_velocity_0", "learning_rate_0", - "loss_scaling_0", "num_bad_steps_0", "num_good_steps_0" - ])) - self.assertEqual(ops, [ - 'cast', 'cast', 'cast', 'cast', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'cast', 'mul', 'elementwise_add', 'cast', 'tanh', 'cast', 'mul', - 'elementwise_add', 'cast', 'tanh', 'cast', 'mul', 'elementwise_add', - 'softmax', 'cast', 'cross_entropy2', 'reduce_mean', - 'elementwise_mul', 'fill_constant', 'elementwise_mul_grad', - 'reduce_mean_grad', 'cross_entropy_grad2', 'cast', 'softmax_grad', - 'elementwise_add_grad', 'mul_grad', 'cast', 'cast', 'mul', - 'elementwise_add', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'cast', 'mul', - 'elementwise_add', 'cast', 'tanh_grad', 'cast', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_sync_comm_stream', 'cast', - 'cast', 'cast', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'momentum', - 'momentum', 'momentum' - ]) + set( + [ + "fc_1.b_0", + "fc_2.b_0", + "fc_2.w_0", + "fc_1.b_0_velocity_0", + "fc_2.b_0_velocity_0", + "fc_2.w_0_velocity_0", + "learning_rate_0", + "loss_scaling_0", + "num_bad_steps_0", + "num_good_steps_0", + ] + ), + ) + self.assertEqual( + ops, + [ + 'cast', + 'cast', + 'cast', + 'cast', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'cast', + 'cast', + 'cast', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_sharding_amp_asp_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') @@ -212,44 +455,115 @@ def test_sharding_amp_asp_optimizer(self): self.assertEqual( set(parameters), - set([ - 'fc_2.b_0', 'num_good_steps_0', 'fc_2.w_0', 'loss_scaling_0', - 'num_bad_steps_0', 'fc_2.w_0_velocity_0', 'fc_2.w_0.asp_mask', - 'learning_rate_0', 'fc_1.b_0', 'fc_1.w_0.asp_mask', - 'fc_0.w_0.asp_mask', 'fc_1.b_0_velocity_0', - 'fc_2.b_0_velocity_0' - ])) - self.assertEqual(ops, [ - 'cast', 'cast', 'cast', 'fill_constant', 'fill_constant', - 'fill_constant', 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_sync_comm_stream', 'cast', 'mul', 'elementwise_add', 'cast', - 'tanh', 'cast', 'mul', 'elementwise_add', 'cast', 'tanh', 'cast', - 'mul', 'elementwise_add', 'softmax', 'cast', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'fill_constant', - 'elementwise_mul_grad', 'reduce_mean_grad', 'cross_entropy_grad2', - 'cast', 'softmax_grad', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'cast', 'cast', 'cast', - 'check_finite_and_unscale', 'cast', 'c_allreduce_max', 'cast', - 'update_loss_scaling', 'momentum', 'momentum', 'momentum', - 'elementwise_mul' - ]) + set( + [ + 'fc_2.b_0', + 'num_good_steps_0', + 'fc_2.w_0', + 'loss_scaling_0', + 'num_bad_steps_0', + 'fc_2.w_0_velocity_0', + 'fc_2.w_0.asp_mask', + 'learning_rate_0', + 'fc_1.b_0', + 'fc_1.w_0.asp_mask', + 'fc_0.w_0.asp_mask', + 'fc_1.b_0_velocity_0', + 'fc_2.b_0_velocity_0', + ] + ), + ) + self.assertEqual( + ops, + [ + 'cast', + 'cast', + 'cast', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'mul', + 'elementwise_add', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'cast', + 'cast', + 'cast', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'momentum', + 'momentum', + 'elementwise_mul', + ], + ) def test_sharding_weight_decay(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') regularization = paddle.fluid.regularizer.L2Decay(0.0001) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - regularization=regularization) + self.optimizer( + avg_cost, + strategy, + train_prog, + startup_prog, + regularization=regularization, + ) parameters = [ x.name for x in train_prog.list_vars() if x.persistable == True ] @@ -258,38 +572,87 @@ def test_sharding_weight_decay(self): self.assertIn('@BroadCast', ''.join(vars)) self.assertEqual( set(parameters), - set([ - "fc_1.b_0", "fc_2.b_0", "fc_2.w_0", "fc_1.b_0_velocity_0", - "fc_2.b_0_velocity_0", "fc_2.w_0_velocity_0", "learning_rate_0" - ])) - - self.assertEqual(ops, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'mul', 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', - 'mul', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'fill_constant', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'mul_grad', 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'scale', 'sum', 'scale', 'sum', 'scale', - 'sum', 'momentum', 'momentum', 'momentum' - ]) + set( + [ + "fc_1.b_0", + "fc_2.b_0", + "fc_2.w_0", + "fc_1.b_0_velocity_0", + "fc_2.b_0_velocity_0", + "fc_2.w_0_velocity_0", + "learning_rate_0", + ] + ), + ) + + self.assertEqual( + ops, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'scale', + 'sum', + 'scale', + 'sum', + 'scale', + 'sum', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_sharding_gradient_clip(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') clip = paddle.fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - grad_clip=clip) + self.optimizer( + avg_cost, strategy, train_prog, startup_prog, grad_clip=clip + ) parameters = [ x.name for x in train_prog.list_vars() if x.persistable == True ] @@ -298,32 +661,86 @@ def test_sharding_gradient_clip(self): self.assertIn('@BroadCast', ''.join(vars)) self.assertEqual( set(parameters), - set([ - "fc_1.b_0", "fc_2.b_0", "fc_2.w_0", "fc_1.b_0_velocity_0", - "fc_2.b_0_velocity_0", "fc_2.w_0_velocity_0", "learning_rate_0" - ])) - - self.assertEqual(ops, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'mul', 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', - 'mul', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'fill_constant', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'mul_grad', 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'squared_l2_norm', 'squared_l2_norm', - 'squared_l2_norm', 'sum', 'c_allreduce_sum', 'sqrt', - 'fill_constant', 'elementwise_max', 'elementwise_div', - 'elementwise_mul', 'elementwise_mul', 'elementwise_mul', 'momentum', - 'momentum', 'momentum' - ]) + set( + [ + "fc_1.b_0", + "fc_2.b_0", + "fc_2.w_0", + "fc_1.b_0_velocity_0", + "fc_2.b_0_velocity_0", + "fc_2.w_0_velocity_0", + "learning_rate_0", + ] + ), + ) + + self.assertEqual( + ops, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'squared_l2_norm', + 'squared_l2_norm', + 'squared_l2_norm', + 'sum', + 'c_allreduce_sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_sharding_clone_for_test(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.net(train_prog, startup_prog) self.set_strategy(strategy, 'sharding') @@ -334,21 +751,41 @@ def test_sharding_clone_for_test(self): sharding.utils.add_sync_comm(test_prog, 1) ops = [op.type for op in test_prog.global_block().ops] - self.assertEqual(ops, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'mul', 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', - 'mul', 'elementwise_add', 'softmax', 'cross_entropy2', 'reduce_mean' - ]) + self.assertEqual( + ops, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + ], + ) class TestFleetShardingHybridOptimizer(TestFleetMetaOptimizer): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "3" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" # pre-assigned ring id self.mp_ring_id = 0 @@ -359,7 +796,9 @@ def setUp(self): def test_sharding_with_mp(self): # NOTE(JZ-LIANG) MP parallelism need user to build model with MP API - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, _ = self.net(train_prog, startup_prog) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -371,7 +810,7 @@ def test_sharding_with_mp(self): "sharding_degree": 2, "hybrid_dp": False, "gradient_merge_acc_step": 1, - "mp_degree": 2 + "mp_degree": 2, } self.optimizer(avg_cost, strategy, train_prog, startup_prog) startup_prog_ops = startup_prog.global_block().ops @@ -379,7 +818,8 @@ def test_sharding_with_mp(self): # should has ring id for MP created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.mp_ring_id, created_ring_ids) @@ -387,8 +827,10 @@ def test_sharding_with_mp(self): # check correctness of MP group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): sharding_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(sharding_group_waiting_ports, ['127.0.0.1:36003']) @@ -396,14 +838,18 @@ def test_sharding_with_mp(self): # check correctness of sharding group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_1": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_1" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) def test_sharding_hybrid_dp(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, _ = self.net(train_prog, startup_prog) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -416,7 +862,7 @@ def test_sharding_hybrid_dp(self): "dp_degree": 2, "hybrid_dp": True, "gradient_merge_acc_step": 1, - "mp_degree": 1 + "mp_degree": 1, } strategy.fuse_all_reduce_ops = False @@ -426,7 +872,8 @@ def test_sharding_hybrid_dp(self): # check ring id for outter dp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.dp_ring_id, created_ring_ids) @@ -434,8 +881,10 @@ def test_sharding_hybrid_dp(self): # check correctness of sharding group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): sharding_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(sharding_group_waiting_ports, ['127.0.0.1:36003']) @@ -443,8 +892,10 @@ def test_sharding_hybrid_dp(self): # check correctness of dp group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_1": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_1" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) @@ -453,32 +904,74 @@ def test_sharding_hybrid_dp(self): if is_loss_grad_op(op): self.assertEqual(op.type, 'fill_constant') self.assertTrue(op.has_attr('value')) - scale = strategy.sharding_configs[ - 'sharding_degree'] * strategy.sharding_configs['dp_degree'] + scale = ( + strategy.sharding_configs['sharding_degree'] + * strategy.sharding_configs['dp_degree'] + ) loss_scale = 1.0 / scale self.assertAlmostEqual(float(op.attr('value')), loss_scale) # check program (allreudce) ops = [op.type for op in main_prog_ops] - self.assertEqual(ops, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'mul', 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', - 'mul', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'fill_constant', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'mul_grad', 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'c_allreduce_sum', 'c_allreduce_sum', - 'c_allreduce_sum', 'c_sync_comm_stream', 'momentum', 'momentum', - 'momentum' - ]) + self.assertEqual( + ops, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'c_sync_comm_stream', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_sharding_hybrid_dp_gm(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, _ = self.net(train_prog, startup_prog) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -491,7 +984,7 @@ def test_sharding_hybrid_dp_gm(self): "dp_degree": 2, "hybrid_dp": True, "gradient_merge_acc_step": 4, - "mp_degree": 1 + "mp_degree": 1, } self.optimizer(avg_cost, strategy, train_prog, startup_prog) startup_prog_ops = startup_prog.global_block().ops @@ -499,7 +992,8 @@ def test_sharding_hybrid_dp_gm(self): # check ring id for outter dp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.dp_ring_id, created_ring_ids) @@ -507,8 +1001,10 @@ def test_sharding_hybrid_dp_gm(self): # check correctness of sharding group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): sharding_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(sharding_group_waiting_ports, ['127.0.0.1:36003']) @@ -516,35 +1012,87 @@ def test_sharding_hybrid_dp_gm(self): # check correctness of dp group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_1": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_1" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) # check program fw_bw_ops = [op.type for op in train_prog.blocks[0].ops] opt_ops = [op.type for op in train_prog.blocks[2].ops] - self.assertEqual(fw_bw_ops, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_sync_comm_stream', - 'mul', 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', - 'mul', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'fill_constant', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'mul_grad', 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_sync_comm_stream', 'elementwise_add', 'elementwise_add', - 'elementwise_add', 'increment', 'elementwise_mod', 'equal', - 'conditional_block' - ]) - self.assertEqual(opt_ops, [ - 'c_allreduce_sum', 'c_allreduce_sum', 'c_allreduce_sum', 'scale', - 'scale', 'scale', 'momentum', 'momentum', 'momentum', - 'fill_constant', 'fill_constant', 'fill_constant' - ]) + self.assertEqual( + fw_bw_ops, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'elementwise_add', + 'elementwise_add', + 'elementwise_add', + 'increment', + 'elementwise_mod', + 'equal', + 'conditional_block', + ], + ) + self.assertEqual( + opt_ops, + [ + 'c_allreduce_sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'scale', + 'scale', + 'scale', + 'momentum', + 'momentum', + 'momentum', + 'fill_constant', + 'fill_constant', + 'fill_constant', + ], + ) # # check loss scale for gradient merge scale_ = -1 @@ -554,7 +1102,9 @@ def test_sharding_hybrid_dp_gm(self): self.assertEqual(scale_, 0.25) def test_sharding_with_pp(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.sharding = True @@ -565,7 +1115,7 @@ def test_sharding_with_pp(self): "hybrid_dp": False, "gradient_merge_acc_step": 4, "mp_degree": 1, - "pp_degree": 2 + "pp_degree": 2, } strategy.pipeline = True strategy.pipeline_configs = { @@ -585,39 +1135,112 @@ def test_sharding_with_pp(self): main_prog_op_types = [op.type for op in main_prog_ops] print(startup_prog_op_types) # global, sharding, pp_send, pp_recv - self.assertEqual(startup_prog_op_types, [ - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init' - ]) - - self.assertEqual(main_prog_op_types, [ - 'fill_constant', 'fill_constant', 'fill_constant', - 'c_sync_calc_stream', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_sync_comm_stream', 'recv_v2', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'softmax', - 'cross_entropy2', 'reduce_mean', 'fill_constant', - 'reduce_mean_grad', 'cross_entropy_grad2', 'softmax_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'c_sync_calc_stream', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', 'c_reduce_sum', - 'c_reduce_sum', 'c_reduce_sum', 'c_sync_comm_stream', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'fill_constant', 'sum', 'fill_constant', 'sum', - 'c_sync_comm_stream', 'momentum', 'momentum', 'momentum', - 'momentum', 'momentum' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_sync_calc_stream', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_sync_comm_stream', + 'recv_v2', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'c_sync_calc_stream', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_reduce_sum', + 'c_sync_comm_stream', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'c_sync_comm_stream', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + ], + ) # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.sharding_ring_id, created_ring_ids) @@ -626,8 +1249,10 @@ def test_sharding_with_pp(self): # check correctness of pp group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): sharding_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(sharding_group_waiting_ports, ['127.0.0.1:36003']) @@ -635,14 +1260,18 @@ def test_sharding_with_pp(self): # check correctness of sharding group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_1": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_1" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) def test_sharding_dp_with_allreduce_fuse(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, _ = self.net(train_prog, startup_prog) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -655,7 +1284,7 @@ def test_sharding_dp_with_allreduce_fuse(self): "dp_degree": 2, "hybrid_dp": True, "gradient_merge_acc_step": 1, - "mp_degree": 1 + "mp_degree": 1, } strategy.fuse_all_reduce_ops = True strategy.fuse_grad_size_in_MB = 2 @@ -672,7 +1301,9 @@ def test_sharding_dp_with_allreduce_fuse(self): assert 'FusedGrad' in op.input_arg_names[0] def test_hybrid_with_mp_pp_amp_gclip(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) self.set_strategy(strategy, 'amp') @@ -690,11 +1321,9 @@ def test_hybrid_with_mp_pp_amp_gclip(self): "accumulate_steps": 4, } clip = paddle.fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - grad_clip=clip) + self.optimizer( + avg_cost, strategy, train_prog, startup_prog, grad_clip=clip + ) train_prog = train_prog._pipeline_opt['section_program'] startup_prog = startup_prog._pipeline_opt['startup_program'] @@ -706,45 +1335,166 @@ def test_hybrid_with_mp_pp_amp_gclip(self): main_prog_op_types = [op.type for op in main_prog_ops] # ring: mp, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'partial_recv', 'partial_allgather', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'cast', 'tanh', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'cast', 'tanh', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'cast', 'tanh', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'softmax', 'cast', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'fill_constant', - 'elementwise_mul_grad', 'reduce_mean_grad', 'cross_entropy_grad2', - 'cast', 'softmax_grad', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'c_sync_calc_stream', 'partial_send', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'c_sync_comm_stream', - 'check_finite_and_unscale', 'cast', 'c_allreduce_max', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'fill_constant', - 'c_allreduce_sum', 'c_allreduce_sum', 'sqrt', 'fill_constant', - 'elementwise_max', 'elementwise_div', 'elementwise_mul', - 'elementwise_mul', 'elementwise_mul', 'elementwise_mul', - 'elementwise_mul', 'elementwise_mul', 'elementwise_mul', - 'elementwise_mul', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'momentum', 'momentum', 'momentum' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'partial_recv', + 'partial_allgather', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'c_sync_calc_stream', + 'partial_send', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'fill_constant', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + ], + ) # pp + mp, partial send recv self.assertIn('partial_recv', main_prog_op_types) @@ -759,7 +1509,8 @@ def test_hybrid_with_mp_pp_amp_gclip(self): # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.mp_ring_id, created_ring_ids) @@ -768,8 +1519,10 @@ def test_hybrid_with_mp_pp_amp_gclip(self): # check correctness of pp group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): mp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(mp_group_waiting_ports, ['127.0.0.1:36003']) @@ -777,14 +1530,18 @@ def test_hybrid_with_mp_pp_amp_gclip(self): # check correctness of sharding group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_1": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_1" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_mp_pp_amp_gclip_for_optimizer(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) self.set_strategy(strategy, 'amp') @@ -802,12 +1559,14 @@ def test_hybrid_with_mp_pp_amp_gclip_for_optimizer(self): "accumulate_steps": 4, } clip = paddle.fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0) - self.optimizer(avg_cost, - strategy, - train_prog, - startup_prog, - grad_clip=clip, - name="adamw") + self.optimizer( + avg_cost, + strategy, + train_prog, + startup_prog, + grad_clip=clip, + name="adamw", + ) train_prog = train_prog._pipeline_opt['section_program'] startup_prog = startup_prog._pipeline_opt['startup_program'] @@ -819,51 +1578,191 @@ def test_hybrid_with_mp_pp_amp_gclip_for_optimizer(self): main_prog_op_types = [op.type for op in main_prog_ops] # ring: mp, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'partial_recv', 'partial_allgather', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'cast', 'tanh', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'cast', 'tanh', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'cast', 'tanh', 'cast', 'cast', 'mul', 'cast', - 'elementwise_add', 'softmax', 'cast', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'fill_constant', - 'elementwise_mul_grad', 'reduce_mean_grad', 'cross_entropy_grad2', - 'cast', 'softmax_grad', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'tanh_grad', 'cast', 'elementwise_add_grad', 'mul_grad', 'cast', - 'c_sync_calc_stream', 'partial_send', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'fill_constant', 'cast', - 'sum', 'fill_constant', 'cast', 'sum', 'c_sync_comm_stream', - 'check_finite_and_unscale', 'cast', 'c_allreduce_max', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'memcpy', - 'fill_constant', 'c_allreduce_sum', 'c_allreduce_sum', 'sqrt', - 'fill_constant', 'elementwise_max', 'elementwise_div', - 'elementwise_mul', 'elementwise_mul', 'elementwise_mul', - 'elementwise_mul', 'elementwise_mul', 'elementwise_mul', - 'elementwise_mul', 'elementwise_mul', 'adamw', 'adamw', 'adamw', - 'adamw', 'adamw', 'adamw', 'adamw', 'adamw' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'partial_recv', + 'partial_allgather', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'cast', + 'tanh', + 'cast', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cast', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'cast', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'tanh_grad', + 'cast', + 'elementwise_add_grad', + 'mul_grad', + 'cast', + 'c_sync_calc_stream', + 'partial_send', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'cast', + 'sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'memcpy', + 'fill_constant', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'elementwise_mul', + 'adamw', + 'adamw', + 'adamw', + 'adamw', + 'adamw', + 'adamw', + 'adamw', + 'adamw', + ], + ) # pp + mp, partial send recv self.assertIn('partial_recv', main_prog_op_types) @@ -878,7 +1777,8 @@ def test_hybrid_with_mp_pp_amp_gclip_for_optimizer(self): # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.mp_ring_id, created_ring_ids) @@ -887,8 +1787,10 @@ def test_hybrid_with_mp_pp_amp_gclip_for_optimizer(self): # check correctness of pp group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): mp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(mp_group_waiting_ports, ['127.0.0.1:36003']) @@ -896,14 +1798,18 @@ def test_hybrid_with_mp_pp_amp_gclip_for_optimizer(self): # check correctness of sharding group sharding_group_waiting_port = None for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_1": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_1" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_pp_dp_amp_fp16allreduce(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.amp = True @@ -936,46 +1842,145 @@ def test_hybrid_with_pp_dp_amp_fp16allreduce(self): main_prog_op_types = [op.type for op in main_prog_ops] # ring: mp, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', - 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', - 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', 'elementwise_add', - 'softmax', 'cross_entropy2', 'reduce_mean', 'elementwise_mul', - 'fill_constant', 'elementwise_mul_grad', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'cast', 'mul_grad', 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'fill_constant', 'cast', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'fill_constant', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'coalesce_tensor', 'c_allreduce_sum', - 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'momentum', - 'momentum', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'momentum' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'coalesce_tensor', + 'c_allreduce_sum', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + ], + ) # amp check_finite_and_unscale, allreduce(pp) self.assertEqual(main_prog_op_types.count('c_allreduce_max'), 1) # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.pp_pair_ring_id, created_ring_ids) @@ -983,22 +1988,28 @@ def test_hybrid_with_pp_dp_amp_fp16allreduce(self): # check correctness of pp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36003']) # check correctness of dp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_3": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_3" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_sharding_pp_amp_fp16allreduce_in_optimize(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.amp = True @@ -1033,14 +2044,33 @@ def test_hybrid_with_sharding_pp_amp_fp16allreduce_in_optimize(self): main_prog_op_types = [op.type for op in main_prog_ops] # ring: sharding, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + ], + ) # FIXME(wangxi): some bug in sharding+pp with pp_allreduce_in_optimize # self.assertEqual(main_prog_op_types, []) @@ -1050,7 +2080,8 @@ def test_hybrid_with_sharding_pp_amp_fp16allreduce_in_optimize(self): # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.sharding_ring_id, created_ring_ids) @@ -1058,22 +2089,28 @@ def test_hybrid_with_sharding_pp_amp_fp16allreduce_in_optimize(self): # check correctness of sharding group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): sharding_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(sharding_group_waiting_ports, ['127.0.0.1:36003']) # check correctness of pp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_1": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_1" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_pp_dp_amp_fp16allreduce_optimize_cast(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.amp = True @@ -1107,48 +2144,152 @@ def test_hybrid_with_pp_dp_amp_fp16allreduce_optimize_cast(self): main_prog_op_types = [op.type for op in main_prog_ops] # ring: mp, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'cast', - 'c_broadcast', 'cast', 'c_broadcast', 'cast', 'c_broadcast', 'cast', - 'c_broadcast', 'cast', 'c_broadcast', 'cast', 'c_broadcast', 'cast', - 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', 'mul', - 'cast', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'fill_constant', - 'elementwise_mul_grad', 'reduce_mean_grad', 'cross_entropy_grad2', - 'softmax_grad', 'elementwise_add_grad', 'cast', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'fill_constant', 'cast', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'fill_constant', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'coalesce_tensor', 'c_allreduce_sum', - 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'momentum', - 'cast', 'momentum', 'cast', 'momentum', 'cast', 'momentum', 'cast', - 'momentum', 'cast', 'momentum', 'cast', 'momentum', 'momentum', - 'cast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'coalesce_tensor', + 'c_allreduce_sum', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'momentum', + 'cast', + ], + ) # amp check_finite_and_unscale, allreduce(pp) self.assertEqual(main_prog_op_types.count('c_allreduce_max'), 1) # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.pp_pair_ring_id, created_ring_ids) @@ -1156,22 +2297,28 @@ def test_hybrid_with_pp_dp_amp_fp16allreduce_optimize_cast(self): # check correctness of pp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36003']) # check correctness of dp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_3": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_3" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_pp_dp_amp_fp16allreduce_optimize_offload(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.amp = True @@ -1205,51 +2352,173 @@ def test_hybrid_with_pp_dp_amp_fp16allreduce_optimize_offload(self): main_prog_op_types = [op.type for op in main_prog_ops] # ring: mp, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'cast', 'memcpy', - 'c_broadcast', 'cast', 'memcpy', 'c_broadcast', 'cast', 'memcpy', - 'c_broadcast', 'cast', 'memcpy', 'c_broadcast', 'cast', 'memcpy', - 'c_broadcast', 'cast', 'memcpy', 'c_broadcast', 'cast', 'memcpy', - 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', 'mul', - 'cast', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'fill_constant', - 'elementwise_mul_grad', 'reduce_mean_grad', 'cross_entropy_grad2', - 'softmax_grad', 'elementwise_add_grad', 'cast', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'fill_constant', 'cast', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'fill_constant', 'sum', 'fill_constant', - 'sum', 'fill_constant', 'sum', 'fill_constant', 'sum', - 'fill_constant', 'sum', 'coalesce_tensor', 'c_allreduce_sum', - 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'memcpy', - 'momentum', 'cast', 'memcpy', 'memcpy', 'momentum', 'cast', - 'memcpy', 'memcpy', 'momentum', 'cast', 'memcpy', 'memcpy', - 'momentum', 'cast', 'memcpy', 'memcpy', 'momentum', 'cast', - 'memcpy', 'memcpy', 'momentum', 'cast', 'memcpy', 'momentum', - 'memcpy', 'momentum', 'cast', 'memcpy' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'cast', + 'memcpy', + 'c_broadcast', + 'cast', + 'memcpy', + 'c_broadcast', + 'cast', + 'memcpy', + 'c_broadcast', + 'cast', + 'memcpy', + 'c_broadcast', + 'cast', + 'memcpy', + 'c_broadcast', + 'cast', + 'memcpy', + 'c_broadcast', + 'cast', + 'memcpy', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'fill_constant', + 'cast', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'fill_constant', + 'sum', + 'coalesce_tensor', + 'c_allreduce_sum', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'memcpy', + 'momentum', + 'cast', + 'memcpy', + 'memcpy', + 'momentum', + 'cast', + 'memcpy', + 'memcpy', + 'momentum', + 'cast', + 'memcpy', + 'memcpy', + 'momentum', + 'cast', + 'memcpy', + 'memcpy', + 'momentum', + 'cast', + 'memcpy', + 'memcpy', + 'momentum', + 'cast', + 'memcpy', + 'momentum', + 'memcpy', + 'momentum', + 'cast', + 'memcpy', + ], + ) # amp check_finite_and_unscale, allreduce(pp) self.assertEqual(main_prog_op_types.count('c_allreduce_max'), 1) # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.pp_pair_ring_id, created_ring_ids) @@ -1257,23 +2526,30 @@ def test_hybrid_with_pp_dp_amp_fp16allreduce_optimize_offload(self): # check correctness of pp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36003']) # check correctness of dp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_3": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_3" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_pp_dp_amp_fp16allreduce_optimize_cast_with_gradient_fuse( - self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + self, + ): + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.amp = True @@ -1308,46 +2584,142 @@ def test_hybrid_with_pp_dp_amp_fp16allreduce_optimize_cast_with_gradient_fuse( main_prog_op_types = [op.type for op in main_prog_ops] # ring: mp, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'cast', - 'c_broadcast', 'cast', 'c_broadcast', 'cast', 'c_broadcast', 'cast', - 'c_broadcast', 'cast', 'c_broadcast', 'cast', 'c_broadcast', 'cast', - 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', 'mul', - 'cast', 'elementwise_add', 'softmax', 'cross_entropy2', - 'reduce_mean', 'elementwise_mul', 'coalesce_tensor', - 'coalesce_tensor', 'coalesce_tensor', 'coalesce_tensor', - 'fill_constant', 'elementwise_mul_grad', 'reduce_mean_grad', - 'cross_entropy_grad2', 'softmax_grad', 'elementwise_add_grad', - 'cast', 'mul_grad', 'tanh_grad', 'elementwise_add_grad', 'mul_grad', - 'tanh_grad', 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'sum', 'cast', 'sum', 'c_allreduce_sum', 'c_allreduce_sum', 'cast', - 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', 'cast', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'momentum', - 'cast', 'momentum', 'cast', 'momentum', 'cast', 'momentum', 'cast', - 'momentum', 'cast', 'momentum', 'cast', 'momentum', 'momentum', - 'cast' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + 'cast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'sum', + 'cast', + 'sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'cast', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'cast', + 'momentum', + 'momentum', + 'cast', + ], + ) # amp check_finite_and_unscale, allreduce(pp) self.assertEqual(main_prog_op_types.count('c_allreduce_max'), 1) # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.pp_pair_ring_id, created_ring_ids) @@ -1355,22 +2727,28 @@ def test_hybrid_with_pp_dp_amp_fp16allreduce_optimize_cast_with_gradient_fuse( # check correctness of pp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36003']) # check correctness of dp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_3": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_3" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_pp_dp_amp_with_gradient_fuse(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.amp = True @@ -1403,44 +2781,127 @@ def test_hybrid_with_pp_dp_amp_with_gradient_fuse(self): main_prog_op_types = [op.type for op in main_prog_ops] # ring: mp, pp_group, pp_pair, pp_pair - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', - 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', - 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', 'elementwise_add', - 'softmax', 'cross_entropy2', 'reduce_mean', 'elementwise_mul', - 'coalesce_tensor', 'coalesce_tensor', 'coalesce_tensor', - 'coalesce_tensor', 'fill_constant', 'elementwise_mul_grad', - 'reduce_mean_grad', 'cross_entropy_grad2', 'softmax_grad', - 'elementwise_add_grad', 'cast', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'cast', 'sum', 'sum', 'c_allreduce_sum', 'c_allreduce_sum', - 'c_sync_comm_stream', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'momentum', - 'momentum', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'momentum' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'cast', + 'sum', + 'sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'c_sync_comm_stream', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + ], + ) # amp check_finite_and_unscale, allreduce(pp) self.assertEqual(main_prog_op_types.count('c_allreduce_max'), 1) # should has ring id for pp created_ring_ids = [ - op.desc.attr("ring_id") for op in startup_prog_ops + op.desc.attr("ring_id") + for op in startup_prog_ops if op.type == "c_comm_init" ] self.assertIn(self.pp_pair_ring_id, created_ring_ids) @@ -1448,22 +2909,28 @@ def test_hybrid_with_pp_dp_amp_with_gradient_fuse(self): # check correctness of pp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_0": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_0" + ): pp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(pp_group_waiting_ports, ['127.0.0.1:36003']) # check correctness of dp group for op in startup_prog_ops: - if op.type == "c_gen_nccl_id" and op.desc.output_arg_names( - )[0] == "comm_id_3": + if ( + op.type == "c_gen_nccl_id" + and op.desc.output_arg_names()[0] == "comm_id_3" + ): dp_group_waiting_ports = op.desc.attr("other_endpoints") self.assertEqual(dp_group_waiting_ports, ['127.0.0.1:36002']) def test_hybrid_with_pp_dp_amp_with_gradient_fuse_and_avg_after_sum(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.amp = True @@ -1481,11 +2948,11 @@ def test_hybrid_with_pp_dp_amp_with_gradient_fuse_and_avg_after_sum(self): strategy.pipeline_configs = { "schedule_mode": "1F1B", "micro_batch_size": 2, - "accumulate_steps": 4 + "accumulate_steps": 4, } strategy.gradient_scale_configs = { 'scale_strategy': 'avg', - 'scale_gradient': True + 'scale_gradient': True, } strategy.fuse_grad_merge = True self.optimizer(avg_cost, strategy, train_prog, startup_prog) @@ -1499,40 +2966,125 @@ def test_hybrid_with_pp_dp_amp_with_gradient_fuse_and_avg_after_sum(self): startup_prog_op_types = [op.type for op in startup_prog_ops] main_prog_op_types = [op.type for op in main_prog_ops] - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', - 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', - 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', 'elementwise_add', - 'softmax', 'cross_entropy2', 'reduce_mean', 'elementwise_mul', - 'coalesce_tensor', 'coalesce_tensor', 'coalesce_tensor', - 'coalesce_tensor', 'fill_constant', 'elementwise_mul_grad', - 'reduce_mean_grad', 'cross_entropy_grad2', 'softmax_grad', - 'elementwise_add_grad', 'cast', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'cast', 'sum', 'sum', 'c_allreduce_sum', 'c_allreduce_sum', - 'c_sync_comm_stream', 'scale', 'check_finite_and_unscale', 'cast', - 'c_allreduce_max', 'cast', 'update_loss_scaling', 'momentum', - 'momentum', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'momentum' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'cast', + 'sum', + 'sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'c_sync_comm_stream', + 'scale', + 'check_finite_and_unscale', + 'cast', + 'c_allreduce_max', + 'cast', + 'update_loss_scaling', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_hybrid_with_pp_dp_with_gradient_fuse_and_avg_after_sum(self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.sharding = True @@ -1546,11 +3098,11 @@ def test_hybrid_with_pp_dp_with_gradient_fuse_and_avg_after_sum(self): strategy.pipeline_configs = { "schedule_mode": "1F1B", "micro_batch_size": 2, - "accumulate_steps": 4 + "accumulate_steps": 4, } strategy.gradient_scale_configs = { 'scale_strategy': 'avg', - 'scale_gradient': True + 'scale_gradient': True, } strategy.fuse_grad_merge = True self.optimizer(avg_cost, strategy, train_prog, startup_prog) @@ -1564,36 +3116,103 @@ def test_hybrid_with_pp_dp_with_gradient_fuse_and_avg_after_sum(self): startup_prog_op_types = [op.type for op in startup_prog_ops] main_prog_op_types = [op.type for op in main_prog_ops] - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', - 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'tanh', 'mul', 'elementwise_add', 'tanh', 'mul', - 'elementwise_add', 'softmax', 'cross_entropy2', 'reduce_mean', - 'coalesce_tensor', 'coalesce_tensor', 'fill_constant', - 'reduce_mean_grad', 'cross_entropy_grad2', 'softmax_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'sum', 'c_allreduce_sum', 'c_sync_comm_stream', 'scale', 'momentum', - 'momentum', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'momentum' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'tanh', + 'mul', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'coalesce_tensor', + 'coalesce_tensor', + 'fill_constant', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'sum', + 'c_allreduce_sum', + 'c_sync_comm_stream', + 'scale', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + ], + ) def test_hybrid_with_pp_dp_with_amp_no_dynamic_gradient_fuse_and_avg_after_sum( - self): - train_prog, startup_prog = paddle.fluid.Program(), paddle.fluid.Program( + self, + ): + train_prog, startup_prog = ( + paddle.fluid.Program(), + paddle.fluid.Program(), ) avg_cost, strategy = self.pp_net(train_prog, startup_prog) strategy.sharding = True @@ -1606,17 +3225,17 @@ def test_hybrid_with_pp_dp_with_amp_no_dynamic_gradient_fuse_and_avg_after_sum( strategy.amp = True strategy.amp_configs = { 'custom_black_varnames': ['fc_6.b_0'], - 'use_dynamic_loss_scaling': False + 'use_dynamic_loss_scaling': False, } strategy.pipeline = True strategy.pipeline_configs = { "schedule_mode": "1F1B", "micro_batch_size": 2, - "accumulate_steps": 4 + "accumulate_steps": 4, } strategy.gradient_scale_configs = { 'scale_strategy': 'avg', - 'scale_gradient': True + 'scale_gradient': True, } strategy.fuse_grad_merge = True self.optimizer(avg_cost, strategy, train_prog, startup_prog) @@ -1630,35 +3249,115 @@ def test_hybrid_with_pp_dp_with_amp_no_dynamic_gradient_fuse_and_avg_after_sum( startup_prog_op_types = [op.type for op in startup_prog_ops] main_prog_op_types = [op.type for op in main_prog_ops] - self.assertEqual(startup_prog_op_types, [ - 'uniform_random', 'fill_constant', 'uniform_random', - 'fill_constant', 'uniform_random', 'fill_constant', - 'uniform_random', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_gen_nccl_id', 'c_comm_init', 'c_gen_nccl_id', 'c_comm_init', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast' - ]) - - self.assertEqual(main_prog_op_types, [ - 'recv_v2', 'cast', 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', - 'mul', 'cast', 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', - 'elementwise_add', 'tanh', 'cast', 'mul', 'cast', 'elementwise_add', - 'softmax', 'cross_entropy2', 'reduce_mean', 'elementwise_mul', - 'coalesce_tensor', 'coalesce_tensor', 'coalesce_tensor', - 'coalesce_tensor', 'fill_constant', 'elementwise_mul_grad', - 'reduce_mean_grad', 'cross_entropy_grad2', 'softmax_grad', - 'elementwise_add_grad', 'cast', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'tanh_grad', - 'elementwise_add_grad', 'mul_grad', 'c_sync_calc_stream', 'send_v2', - 'cast', 'sum', 'sum', 'c_allreduce_sum', 'c_allreduce_sum', - 'c_sync_comm_stream', 'scale', 'scale', 'check_finite_and_unscale', - 'momentum', 'momentum', 'momentum', 'momentum', 'momentum', - 'momentum', 'momentum', 'momentum' - ]) + self.assertEqual( + startup_prog_op_types, + [ + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'uniform_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + ], + ) + + self.assertEqual( + main_prog_op_types, + [ + 'recv_v2', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'tanh', + 'cast', + 'mul', + 'cast', + 'elementwise_add', + 'softmax', + 'cross_entropy2', + 'reduce_mean', + 'elementwise_mul', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'coalesce_tensor', + 'fill_constant', + 'elementwise_mul_grad', + 'reduce_mean_grad', + 'cross_entropy_grad2', + 'softmax_grad', + 'elementwise_add_grad', + 'cast', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'tanh_grad', + 'elementwise_add_grad', + 'mul_grad', + 'c_sync_calc_stream', + 'send_v2', + 'cast', + 'sum', + 'sum', + 'c_allreduce_sum', + 'c_allreduce_sum', + 'c_sync_comm_stream', + 'scale', + 'scale', + 'check_finite_and_unscale', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + 'momentum', + ], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_static_mp_layers.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_static_mp_layers.py index 19f926748cf1d..fe583a4cbc6fb 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_static_mp_layers.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_static_mp_layers.py @@ -23,7 +23,6 @@ class ColumnLinearNet(fluid.dygraph.Layer): - def __init__(self, input_size, output_size): super(ColumnLinearNet, self).__init__() self.parallel_linear = fleet.meta_parallel.ColumnParallelLinear( @@ -32,7 +31,8 @@ def __init__(self, input_size, output_size): weight_attr=None, has_bias=True, gather_output=True, - name="test_column_linear") + name="test_column_linear", + ) def forward(self, x): output = self.parallel_linear(x) @@ -40,7 +40,6 @@ def forward(self, x): class RowLinearNet(fluid.dygraph.Layer): - def __init__(self, input_size, output_size): super(RowLinearNet, self).__init__() self.parallel_linear = fleet.meta_parallel.RowParallelLinear( @@ -48,7 +47,8 @@ def __init__(self, input_size, output_size): out_features=output_size, has_bias=True, input_is_parallel=False, - name="test_row_linear") + name="test_row_linear", + ) def forward(self, x): output = self.parallel_linear(x) @@ -56,11 +56,11 @@ def forward(self, x): class EmbeddingNet(fluid.dygraph.Layer): - def __init__(self, vocab_size, hidden_size): super(EmbeddingNet, self).__init__() self.embedding = fleet.meta_parallel.VocabParallelEmbedding( - vocab_size, hidden_size) + vocab_size, hidden_size + ) def forward(self, x): output = self.embedding(x) @@ -68,11 +68,11 @@ def forward(self, x): class TestDistTraning(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "2" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002,127.0.0.1:36003,127.0.0.1:36004" strategy = fleet.DistributedStrategy() self.model_parallel_size = 2 @@ -97,19 +97,22 @@ def test_column_parallel_layer(self): x = paddle.static.data(name='x', shape=[None, input_size]) y = model_a(x) - #print(main_program) + # print(main_program) ops = main_program.global_block().ops ops = [op.type for op in ops] self.assertEqual( - ops, ['c_identity', 'matmul_v2', 'elementwise_add', 'c_concat']) + ops, ['c_identity', 'matmul_v2', 'elementwise_add', 'c_concat'] + ) weight = model_a.parallel_linear.weight bias = model_a.parallel_linear.bias self.assertEqual( weight.shape, - (input_size, output_size // self.model_parallel_size)) - self.assertEqual(bias.shape, - (output_size // self.model_parallel_size, )) + (input_size, output_size // self.model_parallel_size), + ) + self.assertEqual( + bias.shape, (output_size // self.model_parallel_size,) + ) def test_row_parallel_layer(self): main_program, startup_program = self.get_program() @@ -120,19 +123,21 @@ def test_row_parallel_layer(self): x = paddle.static.data(name='x', shape=[None, input_size]) y = model_a(x) - #print(main_program) + # print(main_program) ops = main_program.global_block().ops ops = [op.type for op in ops] self.assertEqual( ops, - ['c_split', 'matmul_v2', 'c_allreduce_sum', 'elementwise_add']) + ['c_split', 'matmul_v2', 'c_allreduce_sum', 'elementwise_add'], + ) weight = model_a.parallel_linear.weight bias = model_a.parallel_linear.bias self.assertEqual( weight.shape, - (input_size // self.model_parallel_size, output_size)) - self.assertEqual(bias.shape, (output_size, )) + (input_size // self.model_parallel_size, output_size), + ) + self.assertEqual(bias.shape, (output_size,)) def test_parallel_embedding(self): main_program, startup_program = self.get_program() @@ -143,12 +148,12 @@ def test_parallel_embedding(self): # model_a model_a = EmbeddingNet(vocab_size, hidden_size) - x = paddle.static.data(name='x', - shape=[None, seq_len], - dtype='int64') + x = paddle.static.data( + name='x', shape=[None, seq_len], dtype='int64' + ) y = model_a(x) - #print(main_program) + # print(main_program) ops = main_program.global_block().ops ops = [op.type for op in ops] self.assertEqual(ops, ['c_embedding', 'c_allreduce_sum']) @@ -156,7 +161,8 @@ def test_parallel_embedding(self): weight = model_a.embedding.weight self.assertEqual( weight.shape, - (vocab_size // self.model_parallel_size, hidden_size)) + (vocab_size // self.model_parallel_size, hidden_size), + ) def test_parallel_cross_entropy(self): main_program, startup_program = self.get_program() @@ -170,17 +176,19 @@ def test_parallel_cross_entropy(self): model_a = fleet.meta_parallel.ParallelCrossEntropy() x = paddle.static.data( - name='x', shape=[batch_size, seq_length, class_size_per_card]) - label = paddle.static.data(name='label', - shape=[batch_size, seq_length], - dtype='int64') + name='x', shape=[batch_size, seq_length, class_size_per_card] + ) + label = paddle.static.data( + name='label', shape=[batch_size, seq_length], dtype='int64' + ) loss_a = model_a(x, label) - #print(main_program) + # print(main_program) ops = main_program.global_block().ops ops = [op.type for op in ops] - self.assertEqual(ops, - ['unsqueeze2', 'c_softmax_with_cross_entropy']) + self.assertEqual( + ops, ['unsqueeze2', 'c_softmax_with_cross_entropy'] + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_utils.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_utils.py index d7266cf29ab78..6fc6a6d9a2f21 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_utils.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_fleet_utils.py @@ -31,8 +31,9 @@ class TestFleetUtils(unittest.TestCase): train_dir = os.path.join("fleet_util_data", "train_program") def download_files(self): - path = download(self.proto_data_url, self.module_name, - self.proto_data_md5) + path = download( + self.proto_data_url, self.module_name, self.proto_data_md5 + ) print('data is downloaded at ' + path) tar = tarfile.open(path) unzip_folder = tempfile.mkdtemp() @@ -50,29 +51,35 @@ def test_program_type_trans(self): text_program = "pruned_main_program.pbtxt" binary_program = "pruned_main_program.bin" fleet_util = FleetUtil() - text_to_binary = fleet_util.program_type_trans(program_dir, - text_program, True) - binary_to_text = fleet_util.program_type_trans(program_dir, - binary_program, False) + text_to_binary = fleet_util.program_type_trans( + program_dir, text_program, True + ) + binary_to_text = fleet_util.program_type_trans( + program_dir, binary_program, False + ) self.assertTrue( - os.path.exists(os.path.join(program_dir, text_to_binary))) + os.path.exists(os.path.join(program_dir, text_to_binary)) + ) self.assertTrue( - os.path.exists(os.path.join(program_dir, binary_to_text))) + os.path.exists(os.path.join(program_dir, binary_to_text)) + ) def test_parse_program_proto(self): data_dir = self.download_files() parse_program_file_path = os.path.join( - data_dir, os.path.join(self.pruned_dir, - "pruned_main_program.pbtxt")) + data_dir, os.path.join(self.pruned_dir, "pruned_main_program.pbtxt") + ) is_text_parse_program = True parse_output_dir = os.path.join(data_dir, self.pruned_dir) fleet_util = FleetUtil() - fleet_util.parse_program_proto(parse_program_file_path, - is_text_parse_program, parse_output_dir) + fleet_util.parse_program_proto( + parse_program_file_path, is_text_parse_program, parse_output_dir + ) ops_log = os.path.join(parse_output_dir, "ops.log") vars_log = os.path.join(parse_output_dir, "vars_all.log") - vars_persistable = os.path.join(parse_output_dir, - "vars_persistable.log") + vars_persistable = os.path.join( + parse_output_dir, "vars_persistable.log" + ) self.assertTrue(os.path.exists(ops_log)) self.assertTrue(os.path.exists(vars_log)) self.assertTrue(os.path.exists(vars_persistable)) @@ -89,7 +96,7 @@ class config: feed_config.feeded_vars_types = [np.float32, np.float32] feed_config.feeded_vars_filelist = [ os.path.join(data_dir, os.path.join(self.pruned_dir, "concat_1")), - os.path.join(data_dir, os.path.join(self.pruned_dir, "concat_2")) + os.path.join(data_dir, os.path.join(self.pruned_dir, "concat_2")), ] fetch_config = config() @@ -106,7 +113,9 @@ class config: fleet_util = FleetUtil() # test saved var's shape - conf.dump_program_filename = "pruned_main_program.save_var_shape_not_match" + conf.dump_program_filename = ( + "pruned_main_program.save_var_shape_not_match" + ) self.assertRaises(Exception, fleet_util.check_vars_and_dump, conf) # test program.proto without feed_op and fetch_op @@ -114,10 +123,13 @@ class config: results = fleet_util.check_vars_and_dump(conf) self.assertTrue(len(results) == 1) np.testing.assert_array_almost_equal( - results[0], np.array([[3.0590223e-07]], dtype=np.float32)) + results[0], np.array([[3.0590223e-07]], dtype=np.float32) + ) # test feed_var's shape - conf.dump_program_filename = "pruned_main_program.feed_var_shape_not_match" + conf.dump_program_filename = ( + "pruned_main_program.feed_var_shape_not_match" + ) self.assertRaises(Exception, fleet_util.check_vars_and_dump, conf) # test correct case with feed_vars_filelist @@ -125,7 +137,8 @@ class config: results = fleet_util.check_vars_and_dump(conf) self.assertTrue(len(results) == 1) np.testing.assert_array_almost_equal( - results[0], np.array([[3.0590223e-07]], dtype=np.float32)) + results[0], np.array([[3.0590223e-07]], dtype=np.float32) + ) # test correct case without feed_vars_filelist conf.feed_config.feeded_vars_filelist = None @@ -145,14 +158,17 @@ class config: conf = config() conf.train_prog_path = os.path.join( - data_dir, os.path.join(self.train_dir, "join_main_program.pbtxt")) + data_dir, os.path.join(self.train_dir, "join_main_program.pbtxt") + ) conf.is_text_train_program = True # test not match conf.pruned_prog_path = os.path.join( data_dir, - os.path.join(self.pruned_dir, - "pruned_main_program.save_var_shape_not_match")) + os.path.join( + self.pruned_dir, "pruned_main_program.save_var_shape_not_match" + ), + ) conf.is_text_pruned_program = True conf.draw = False fleet_util = FleetUtil() @@ -161,8 +177,8 @@ class config: # test match conf.pruned_prog_path = os.path.join( - data_dir, os.path.join(self.pruned_dir, - "pruned_main_program.pbtxt")) + data_dir, os.path.join(self.pruned_dir, "pruned_main_program.pbtxt") + ) if sys.platform == 'win32' or sys.platform == 'sys.platform': conf.draw = False else: @@ -177,29 +193,39 @@ def test_draw_program(self): else: data_dir = self.download_files() program_path = os.path.join( - data_dir, os.path.join(self.train_dir, - "join_main_program.pbtxt")) + data_dir, + os.path.join(self.train_dir, "join_main_program.pbtxt"), + ) is_text = True program = utils.load_program(program_path, is_text) output_dir = os.path.join(data_dir, self.train_dir) output_filename_1 = "draw_prog_1" output_filename_2 = "draw_prog_2" fleet_util = FleetUtil() - fleet_util.draw_from_program_file(program_path, is_text, output_dir, - output_filename_1) + fleet_util.draw_from_program_file( + program_path, is_text, output_dir, output_filename_1 + ) fleet_util.draw_from_program(program, output_dir, output_filename_2) self.assertTrue( os.path.exists( - os.path.join(output_dir, output_filename_1 + ".dot"))) + os.path.join(output_dir, output_filename_1 + ".dot") + ) + ) self.assertTrue( os.path.exists( - os.path.join(output_dir, output_filename_1 + ".pdf"))) + os.path.join(output_dir, output_filename_1 + ".pdf") + ) + ) self.assertTrue( os.path.exists( - os.path.join(output_dir, output_filename_2 + ".dot"))) + os.path.join(output_dir, output_filename_2 + ".dot") + ) + ) self.assertTrue( os.path.exists( - os.path.join(output_dir, output_filename_2 + ".pdf"))) + os.path.join(output_dir, output_filename_2 + ".pdf") + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs1.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs1.py index 694cfe0cacc1f..b141b1ed65bc6 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs1.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs1.py @@ -23,12 +23,10 @@ class FSTest1(FSTestBase): - def test_timeout(self): - fs = HDFSClient("/usr/local/hadoop-2.7.7/", - None, - time_out=6 * 1000, - sleep_inter=100) + fs = HDFSClient( + "/usr/local/hadoop-2.7.7/", None, time_out=6 * 1000, sleep_inter=100 + ) src = "hdfs_test_timeout" dst = "new_hdfs_test_timeout" fs.delete(dst) @@ -40,7 +38,8 @@ def test_timeout(self): try: fs.mv(src, dst, test_exists=False) self.assertFalse( - 1, "can't execute cmd:{} output:{}".format(cmd, output)) + 1, "can't execute cmd:{} output:{}".format(cmd, output) + ) except FSTimeOut as e: print("execute mv {} to {} timeout".format(src, dst)) @@ -49,10 +48,9 @@ def test_timeout(self): print("second mv ret:{} output:{}".format(ret, output)) def test_is_dir(self): - fs = HDFSClient("/usr/local/hadoop-2.7.7/", - None, - time_out=6 * 1000, - sleep_inter=100) + fs = HDFSClient( + "/usr/local/hadoop-2.7.7/", None, time_out=6 * 1000, sleep_inter=100 + ) self.assertFalse(fs.is_dir("./test_hdfs.py")) s = """ java.io.IOException: Input/output error @@ -67,23 +65,24 @@ def test_is_dir(self): at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:65) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:79) at org.apache.hadoop.fs.FsShell.main(FsShell.java:2353) - """ + """ # fmt: off, avoid remove tabs in string print("split lines:", s.splitlines()) self.assertTrue(fs._test_match(s.splitlines()) != None) def test_config(self): config = {"fs.default.name": "hdfs://xxx", "hadoop.job.ugi": "ugi"} - fs = HDFSClient("/usr/local/hadoop-2.7.7/", - config, - time_out=6 * 1000, - sleep_inter=100) + fs = HDFSClient( + "/usr/local/hadoop-2.7.7/", + config, + time_out=6 * 1000, + sleep_inter=100, + ) def test_exists(self): - fs = HDFSClient("/usr/local/hadoop-2.7.7/", - None, - time_out=6 * 1000, - sleep_inter=100) + fs = HDFSClient( + "/usr/local/hadoop-2.7.7/", None, time_out=6 * 1000, sleep_inter=100 + ) self.assertFalse(fs.is_exist(os.path.abspath("./xxxx"))) self.assertFalse(fs.is_dir(os.path.abspath("./xxxx"))) self.assertTrue(fs.is_dir(os.path.abspath("./xxx/.."))) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs2.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs2.py index 682036a681a22..787a7c933449c 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs2.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs2.py @@ -22,12 +22,10 @@ class FSTest2(FSTestBase): - def test_hdfs(self): - fs = HDFSClient("/usr/local/hadoop-2.7.7/", - None, - time_out=5 * 1000, - sleep_inter=100) + fs = HDFSClient( + "/usr/local/hadoop-2.7.7/", None, time_out=5 * 1000, sleep_inter=100 + ) self._test_rm(fs) self._test_touch(fs) self._test_dirs(fs) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs3.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs3.py index 6bf304dc1e796..9aa0f30e99cfc 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs3.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_hdfs3.py @@ -22,12 +22,10 @@ class FSTest3(FSTestBase): - def test_hdfs(self): - fs = HDFSClient("/usr/local/hadoop-2.7.7/", - None, - time_out=5 * 1000, - sleep_inter=100) + fs = HDFSClient( + "/usr/local/hadoop-2.7.7/", None, time_out=5 * 1000, sleep_inter=100 + ) self._test_mkdirs(fs) self._test_list_dir(fs) self._test_try_upload(fs) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_hybrid_parallel_inference_helper.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_hybrid_parallel_inference_helper.py index 06b3ec02de0c7..b0d07f8e92b87 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_hybrid_parallel_inference_helper.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_hybrid_parallel_inference_helper.py @@ -18,7 +18,6 @@ class TestHybridParallelInferenceHelper(TestMultipleGpus): - def test_hybrid_parallel_inference_helper(self): self.run_mnist_2gpu('hybrid_parallel_inference_helper.py') diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision.py index 026fa8fb9d44a..916a21359a4f8 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision.py @@ -32,31 +32,33 @@ class SimpleConv(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(SimpleConv, self).__init__() - self._conv = fluid.dygraph.Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - bias_attr=None, - use_cudnn=True) + self._conv = fluid.dygraph.Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + bias_attr=None, + use_cudnn=True, + ) def forward(self, inputs): return self._conv(inputs) class TestAutoCast(unittest.TestCase): - def amp_guard_white_op(self): data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32') with fluid.dygraph.guard(): @@ -93,30 +95,37 @@ def custom_op_list(self): tracer = fluid.framework._dygraph_tracer() base_white_list = fluid.dygraph.amp.auto_cast.WHITE_LIST base_black_list = fluid.dygraph.amp.auto_cast.BLACK_LIST - with fluid.dygraph.amp_guard(custom_white_list=["log"], - custom_black_list=["conv2d"]): + with fluid.dygraph.amp_guard( + custom_white_list=["log"], custom_black_list=["conv2d"] + ): white_list, black_list = tracer._get_amp_op_list() self.assertTrue( - set(white_list) == (set(base_white_list) | {"log"}) - - {"conv2d"}) + set(white_list) + == (set(base_white_list) | {"log"}) - {"conv2d"} + ) self.assertTrue( - set(black_list) == (set(base_black_list) - {"log"}) - | {"conv2d"}) + set(black_list) + == (set(base_black_list) - {"log"}) | {"conv2d"} + ) base_white_list = fluid.dygraph.amp.auto_cast.PURE_FP16_WHITE_LIST base_black_list = fluid.dygraph.amp.auto_cast.PURE_FP16_BLACK_LIST - with fluid.dygraph.amp_guard(custom_white_list=["log"], - custom_black_list=["conv2d"], - level='O2'): + with fluid.dygraph.amp_guard( + custom_white_list=["log"], + custom_black_list=["conv2d"], + level='O2', + ): white_list, black_list = tracer._get_amp_op_list() self.assertTrue( - set(white_list) == (set(base_white_list) | {"log"}) - - {"conv2d"}) + set(white_list) + == (set(base_white_list) | {"log"}) - {"conv2d"} + ) self.assertTrue( - set(black_list) == (set(base_black_list) - {"log"}) - | {"conv2d"}) + set(black_list) + == (set(base_black_list) - {"log"}) | {"conv2d"} + ) def test_custom_op_list(self): self.custom_op_list() @@ -126,13 +135,16 @@ def custom_op_list_exception(self): def func(): with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - with fluid.dygraph.amp_guard(custom_white_list=["conv2d"], - custom_black_list=["conv2d"]): + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + with fluid.dygraph.amp_guard( + custom_white_list=["conv2d"], custom_black_list=["conv2d"] + ): inp = fluid.dygraph.to_variable(inp_np) out = model(inp) @@ -149,35 +161,34 @@ def amp_guard_upsupported_fp16_op(self): with fluid.dygraph.amp_guard(True): out_amp_fp16 = conv2d(data) out_amp_fp32 = paddle.expand_as( - out_amp_fp16, - out_amp_fp16) # expand_as_v2 has no fp16 kernel + out_amp_fp16, out_amp_fp16 + ) # expand_as_v2 has no fp16 kernel with fluid.dygraph.amp_guard(True, level='O2'): out_purefp16_fp16 = conv2d(data) out_purefp16_fp32 = paddle.expand_as( - out_purefp16_fp16, - out_purefp16_fp16) # expand_as_v2 has no fp16 kernel + out_purefp16_fp16, out_purefp16_fp16 + ) # expand_as_v2 has no fp16 kernel self.assertTrue(data.dtype == fluid.core.VarDesc.VarType.FP32) self.assertTrue(out_amp_fp16.dtype == fluid.core.VarDesc.VarType.FP16) self.assertTrue(out_amp_fp32.dtype == fluid.core.VarDesc.VarType.FP32) self.assertTrue( - out_purefp16_fp16.dtype == fluid.core.VarDesc.VarType.FP16) + out_purefp16_fp16.dtype == fluid.core.VarDesc.VarType.FP16 + ) self.assertTrue( - out_purefp16_fp32.dtype == fluid.core.VarDesc.VarType.FP32) + out_purefp16_fp32.dtype == fluid.core.VarDesc.VarType.FP32 + ) def test_amp_guard_upsupported_fp16_op(self): self.amp_guard_upsupported_fp16_op() def mode_exception(self): - def func(): data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32') with fluid.dygraph.guard(): - conv2d = fluid.dygraph.Conv2D(3, - 2, - 3, - bias_attr=False, - act=None) + conv2d = fluid.dygraph.Conv2D( + 3, 2, 3, bias_attr=False, act=None + ) data = fluid.dygraph.to_variable(data) with fluid.dygraph.amp_guard(level='O'): out = conv2d(data) @@ -189,15 +200,14 @@ def test_mode_exception(self): class TestAmpScaler(unittest.TestCase): - def scale(self): with fluid.dygraph.guard(): data = paddle.rand([10, 1024]) scaler = paddle.fluid.dygraph.AmpScaler(init_loss_scaling=1024) scaled_data = scaler.scale(data) self.assertEqual( - np.array_equal(scaled_data.numpy(), - data.numpy() * 1024), True) + np.array_equal(scaled_data.numpy(), data.numpy() * 1024), True + ) def test_scale(self): self.scale() @@ -209,13 +219,16 @@ def run_simple_conv(inp_np, use_scaler=True): paddle.seed(10) paddle.framework.random._manual_program_seed(10) with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) optimizer = fluid.optimizer.SGDOptimizer( - learning_rate=0.01, parameter_list=model.parameters()) + learning_rate=0.01, parameter_list=model.parameters() + ) scaler = fluid.dygraph.AmpScaler(init_loss_scaling=1024) data = fluid.dygraph.to_variable(inp_np) @@ -226,7 +239,8 @@ def run_simple_conv(inp_np, use_scaler=True): scaled_loss = scaler.scale(loss) scaled_loss.backward() optimize_ops, params_grads = scaler.minimize( - optimizer, scaled_loss) + optimizer, scaled_loss + ) else: print('use no scaler') loss.backward() @@ -236,19 +250,25 @@ def run_simple_conv(inp_np, use_scaler=True): outs_with_scaler = run_simple_conv(inp_np, use_scaler=True) outs_no_scaler = run_simple_conv(inp_np, use_scaler=False) - self.assertEqual(outs_with_scaler[0], - []) # optimize_ops is [] in dygraph mode - self.assertEqual(outs_no_scaler[0], - []) # optimize_ops is [] in dygraph mode + self.assertEqual( + outs_with_scaler[0], [] + ) # optimize_ops is [] in dygraph mode + self.assertEqual( + outs_no_scaler[0], [] + ) # optimize_ops is [] in dygraph mode for i in range(len(outs_with_scaler[1])): # check each grad - np.testing.assert_allclose(outs_with_scaler[1][i][1].numpy(), - outs_no_scaler[1][i][1].numpy(), - rtol=1e-05) + np.testing.assert_allclose( + outs_with_scaler[1][i][1].numpy(), + outs_no_scaler[1][i][1].numpy(), + rtol=1e-05, + ) # check each parameter - np.testing.assert_allclose(outs_with_scaler[1][i][0].numpy(), - outs_no_scaler[1][i][0].numpy(), - rtol=1e-05) + np.testing.assert_allclose( + outs_with_scaler[1][i][0].numpy(), + outs_no_scaler[1][i][0].numpy(), + rtol=1e-05, + ) def test_minimize(self): self.minimize() @@ -260,13 +280,16 @@ def run_simple_conv(inp_np, use_scaler=True): paddle.seed(10) paddle.framework.random._manual_program_seed(10) with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = fluid.dygraph.to_variable(inp_np) @@ -289,9 +312,11 @@ def run_simple_conv(inp_np, use_scaler=True): for i in range(len(outs_with_scaler)): # check each parameter - np.testing.assert_allclose(outs_with_scaler[i].numpy(), - outs_no_scaler[i].numpy(), - rtol=1e-05) + np.testing.assert_allclose( + outs_with_scaler[i].numpy(), + outs_no_scaler[i].numpy(), + rtol=1e-05, + ) def test_step(self): self.step() @@ -300,16 +325,19 @@ def nan_inf(self): inp_np = np.random.random(size=[1, 3, 128, 128]).astype(np.float32) inp_np[0][1][2][3] = np.nan with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) params_init = {} for param in model.parameters(): params_init[param.name] = param.numpy() optimizer = fluid.optimizer.SGDOptimizer( - learning_rate=0.01, parameter_list=model.parameters()) + learning_rate=0.01, parameter_list=model.parameters() + ) scaler = fluid.dygraph.AmpScaler(init_loss_scaling=1024) data = fluid.dygraph.to_variable(inp_np) @@ -322,18 +350,19 @@ def nan_inf(self): for param in model.parameters(): # param not update when tensor contains nan or inf - np.testing.assert_array_equal(param.numpy(), - params_init[param.name]) + np.testing.assert_array_equal( + param.numpy(), params_init[param.name] + ) def test_nan_inf(self): self.nan_inf() def step_update_exception(self): - def func1(): model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = paddle.rand([10, 3, 32, 32]) conv = model(data) @@ -347,8 +376,9 @@ def func1(): def func2(): model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = paddle.rand([10, 3, 32, 32]) conv = model(data) @@ -362,8 +392,9 @@ def func2(): def func3(): model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = paddle.rand([10, 3, 32, 32]) conv = model(data) @@ -380,13 +411,15 @@ def test_step_update_exception(self): def test_get_and_set(self): with fluid.dygraph.guard(): - scaler = paddle.amp.GradScaler(enable=True, - init_loss_scaling=1024, - incr_ratio=2.0, - decr_ratio=0.5, - incr_every_n_steps=1000, - decr_every_n_nan_or_inf=2, - use_dynamic_loss_scaling=True) + scaler = paddle.amp.GradScaler( + enable=True, + init_loss_scaling=1024, + incr_ratio=2.0, + decr_ratio=0.5, + incr_every_n_steps=1000, + decr_every_n_nan_or_inf=2, + use_dynamic_loss_scaling=True, + ) self.assertEqual(scaler.is_enable() == True, True) self.assertEqual(scaler.get_init_loss_scaling() == 1024, True) self.assertEqual(scaler.get_incr_ratio() == 2.0, True) @@ -407,13 +440,15 @@ def test_get_and_set(self): def test_state_dict_and_load_state_dict(self): with fluid.dygraph.guard(): - scaler1 = paddle.amp.GradScaler(enable=True, - init_loss_scaling=14, - incr_ratio=233.0, - decr_ratio=0.523, - incr_every_n_steps=1090, - decr_every_n_nan_or_inf=20, - use_dynamic_loss_scaling=True) + scaler1 = paddle.amp.GradScaler( + enable=True, + init_loss_scaling=14, + incr_ratio=233.0, + decr_ratio=0.523, + incr_every_n_steps=1090, + decr_every_n_nan_or_inf=20, + use_dynamic_loss_scaling=True, + ) scaler_state = scaler1.state_dict() scaler2 = paddle.amp.GradScaler(enable=True) scaler2.load_state_dict(scaler_state) @@ -428,7 +463,6 @@ def test_state_dict_and_load_state_dict(self): self.assertEqual(scaler3.is_enable() == False, True) def test_state_dict_and_load_state_dict_error(self): - def test_error(): state_empty = {} scaler = paddle.amp.GradScaler(enable=True) @@ -438,7 +472,6 @@ def test_error(): def reader_decorator(reader): - def __reader__(): for item in reader(): img = np.array(item[0]).astype('float32').reshape(3, 224, 224) @@ -449,11 +482,9 @@ def __reader__(): class TestGradScalerStateDict(unittest.TestCase): - - def train_resnet(self, - enable_amp=True, - use_data_loader=True, - use_save_load=True): + def train_resnet( + self, enable_amp=True, use_data_loader=True, use_save_load=True + ): seed = 90 batch_size = train_parameters["batch_size"] @@ -463,30 +494,35 @@ def train_resnet(self, paddle.framework.random._manual_program_seed(seed) resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) np.random.seed(seed) train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size) + paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.amp.GradScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.amp.GradScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if use_data_loader: - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator( capacity=4, use_double_buffer=True, iterable=True, - return_list=True) + return_list=True, + ) train_loader.set_sample_list_generator(train_reader) train_reader = train_loader @@ -496,13 +532,19 @@ def train_resnet(self, if use_data_loader: img, label = data else: - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) @@ -525,8 +567,9 @@ def train_resnet(self, for param in resnet.parameters(): if param.trainable: np_array = np.array(param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -543,43 +586,37 @@ def train_resnet(self, return dy_out, dy_param_value, dy_grad_value def test_with_state_dict(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_use_state_dict = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=True) - out_no_state_dict = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=False) + out_use_state_dict = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=True + ) + out_no_state_dict = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=False + ) print('save_load:', out_use_state_dict[0], out_no_state_dict[0]) - np.testing.assert_allclose(out_use_state_dict[0], - out_no_state_dict[0], - rtol=1e-05) + np.testing.assert_allclose( + out_use_state_dict[0], out_no_state_dict[0], rtol=1e-05 + ) func_isinstance() class TestAmpDecorator(unittest.TestCase): - def test_mode_exception(self): - def func(): with fluid.dygraph.guard(): model = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) opt = paddle.optimizer.SGD(parameters=model.parameters()) - model, opt = paddle.amp.decorate(models=model, - optimizers=opt, - level='O') + model, opt = paddle.amp.decorate( + models=model, optimizers=opt, level='O' + ) self.assertRaises(ValueError, func) def test_input_type_exception(self): - def test_error_model(): - class MyModel(object): - def __init__(self): print("A fake Model") @@ -598,9 +635,7 @@ def test_error_distributed_model(): self.assertRaises(RuntimeError, test_error_distributed_model) def test_error_optimizer(): - class MyOptimizer(object): - def __init__(self): print("A fake Optimizer") @@ -613,48 +648,59 @@ def __init__(self): def test_set_master_weight(self): model1 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt1 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model1.parameters(), - multi_precision=True) + opt1 = paddle.optimizer.Adam( + learning_rate=0.0001, + parameters=model1.parameters(), + multi_precision=True, + ) model2 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt2 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model2.parameters(), - multi_precision=False) - - model1, opt1 = paddle.amp.decorate(models=model1, - optimizers=opt1, - level='O2', - master_weight=None) + opt2 = paddle.optimizer.Adam( + learning_rate=0.0001, + parameters=model2.parameters(), + multi_precision=False, + ) + + model1, opt1 = paddle.amp.decorate( + models=model1, optimizers=opt1, level='O2', master_weight=None + ) self.assertEqual(opt1._multi_precision, True) - models, opt2 = paddle.amp.decorate(models=[model1, model2], - optimizers=opt2, - level='O2', - master_weight=None) + models, opt2 = paddle.amp.decorate( + models=[model1, model2], + optimizers=opt2, + level='O2', + master_weight=None, + ) self.assertEqual(opt2._multi_precision, True) model3 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt3 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model3.parameters()) + opt3 = paddle.optimizer.Adam( + learning_rate=0.0001, parameters=model3.parameters() + ) model4 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt4 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model4.parameters()) - - model3, opts = paddle.amp.decorate(models=model3, - optimizers=[opt3, opt4], - level='O2', - master_weight=True) + opt4 = paddle.optimizer.Adam( + learning_rate=0.0001, parameters=model4.parameters() + ) + + model3, opts = paddle.amp.decorate( + models=model3, + optimizers=[opt3, opt4], + level='O2', + master_weight=True, + ) self.assertEqual(opts[0]._multi_precision, True) self.assertEqual(opts[1]._multi_precision, True) models = [model3, model4] optimizers = [opt3, opt4] - models, optimizers = paddle.amp.decorate(models=models, - optimizers=optimizers, - level='O2', - master_weight=False) + models, optimizers = paddle.amp.decorate( + models=models, + optimizers=optimizers, + level='O2', + master_weight=False, + ) self.assertEqual(optimizers[0]._multi_precision, False) self.assertEqual(optimizers[1]._multi_precision, False) @@ -689,20 +735,19 @@ def test_floating_only(self): buffer = paddle.to_tensor(np.array([5]).astype("int32")) model.register_buffer("buffer_name", buffer, persistable=True) model = paddle.amp.decorate(models=model, level='O2') - self.assertEqual((model._buffers["buffer_name"].dtype == paddle.int32), - True) + self.assertEqual( + (model._buffers["buffer_name"].dtype == paddle.int32), True + ) class TestStateDictHookForAMP(unittest.TestCase): - def test_state_dict_hook(self): - def func_isinstance(): paddle.seed(100) model = paddle.nn.Linear(2, 4) - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) param_value_ori = {} for param in model.parameters(): param_value_ori[param.name] = param.numpy() @@ -723,7 +768,6 @@ def func_isinstance(): class TestPureFp16SaveLoad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -731,22 +775,19 @@ def tearDown(self): self.temp_dir.cleanup() def test_save_dtype_exception(self): - def func(): paddle.disable_static() model = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) opt = paddle.optimizer.SGD(parameters=model.parameters()) - paddle.amp.decorate(models=model, - optimizers=opt, - level='O2', - save_dtype='int') + paddle.amp.decorate( + models=model, optimizers=opt, level='O2', save_dtype='int' + ) self.assertRaises(ValueError, func) - def train_resnet(self, - enable_amp=True, - use_data_loader=True, - use_save_load=True): + def train_resnet( + self, enable_amp=True, use_data_loader=True, use_save_load=True + ): seed = 90 batch_size = train_parameters["batch_size"] @@ -756,38 +797,45 @@ def train_resnet(self, paddle.framework.random._manual_program_seed(seed) resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) np.random.seed(seed) train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size) + paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.amp.GradScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.amp.GradScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if use_data_loader: - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator( capacity=4, use_double_buffer=True, iterable=True, - return_list=True) + return_list=True, + ) train_loader.set_sample_list_generator(train_reader) train_reader = train_loader if enable_amp: - resnet, optimizer = paddle.amp.decorate(models=resnet, - optimizers=optimizer, - level='O2', - save_dtype='float32') + resnet, optimizer = paddle.amp.decorate( + models=resnet, + optimizers=optimizer, + level='O2', + save_dtype='float32', + ) for batch_id, data in enumerate(train_reader()): if batch_id >= batch_num: @@ -795,13 +843,19 @@ def train_resnet(self, if use_data_loader: img, label = data else: - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) @@ -825,8 +879,9 @@ def train_resnet(self, for param in resnet.parameters(): if param.trainable: np_array = np.array(param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -839,7 +894,7 @@ def train_resnet(self, obj = { 'model': resnet.state_dict(), 'opt': optimizer.state_dict(), - 'scaler': scaler.state_dict() + 'scaler': scaler.state_dict(), } path = os.path.join(self.temp_dir.name, 'model.pdparams') paddle.save(obj, path) @@ -847,39 +902,40 @@ def train_resnet(self, obj_load = paddle.load(path) resnet = ResNet(use_cudnn=True) optimizer = optimizer_setting( - train_parameters, parameter_list=resnet.parameters()) + train_parameters, parameter_list=resnet.parameters() + ) resnet.set_state_dict(obj_load['model']) optimizer.set_state_dict(obj_load['opt']) scaler.load_state_dict(obj_load['scaler']) - resnet, optimizer = paddle.amp.decorate(models=resnet, - optimizers=optimizer, - level='O2', - save_dtype='float32') + resnet, optimizer = paddle.amp.decorate( + models=resnet, + optimizers=optimizer, + level='O2', + save_dtype='float32', + ) if use_data_loader: train_reader._reset() return dy_out, dy_param_value, dy_grad_value def test_with_save_load(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_use_save_load = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=True) - out_no_save_load = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=False) + out_use_save_load = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=True + ) + out_no_save_load = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=False + ) print('save_load:', out_use_save_load[0], out_no_save_load[0]) - np.testing.assert_allclose(out_use_save_load[0], - out_no_save_load[0], - rtol=1e-05) + np.testing.assert_allclose( + out_use_save_load[0], out_no_save_load[0], rtol=1e-05 + ) func_isinstance() class TestPureFp16InferenceSaveLoad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -895,21 +951,20 @@ def inference_save_load(self): # define a random dataset class RandomDataset(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, - (1, )).astype('int64') + label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype( + 'int64' + ) return image, label def __len__(self): return self.num_samples class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) @@ -920,10 +975,12 @@ def forward(self, x): def train(layer, loader, loss_fn, opt): for epoch_id in range(EPOCH_NUM): for batch_id, (image, label) in enumerate(loader()): - with paddle.amp.auto_cast(enable=True, - custom_white_list=None, - custom_black_list=None, - level='O2'): + with paddle.amp.auto_cast( + enable=True, + custom_white_list=None, + custom_black_list=None, + level='O2', + ): out = layer(image) loss = loss_fn(out, label) loss.backward() @@ -932,27 +989,31 @@ def train(layer, loader, loss_fn, opt): # train layer = LinearNet() - adam = paddle.optimizer.Adam(learning_rate=0.001, - parameters=layer.parameters(), - multi_precision=True) + adam = paddle.optimizer.Adam( + learning_rate=0.001, + parameters=layer.parameters(), + multi_precision=True, + ) loss_fn = nn.CrossEntropyLoss() - layer, adam = paddle.amp.decorate(models=layer, - optimizers=adam, - save_dtype='float32') + layer, adam = paddle.amp.decorate( + models=layer, optimizers=adam, save_dtype='float32' + ) dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) - loader = paddle.io.DataLoader(dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) + loader = paddle.io.DataLoader( + dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) train(layer, loader, loss_fn, adam) # save path = os.path.join(self.temp_dir.name, 'example_model/linear') - paddle.jit.save(layer, - path, - input_spec=[InputSpec(shape=[IMAGE_SIZE], name='x')]) + paddle.jit.save( + layer, path, input_spec=[InputSpec(shape=[IMAGE_SIZE], name='x')] + ) # jit.load loaded_layer = paddle.jit.load(path) @@ -966,12 +1027,17 @@ def train(layer, loader, loss_fn, opt): # load_inference_model paddle.enable_static() exe = paddle.static.Executor() - [inference_program, feed_target_names, - fetch_targets] = (paddle.static.load_inference_model(path, exe)) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(path, exe) tensor_img = x - results = exe.run(inference_program, - feed={feed_target_names[0]: tensor_img}, - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed={feed_target_names[0]: tensor_img}, + fetch_list=fetch_targets, + ) print("pred.numpy()", pred.numpy()) print("result", results[0]) np.testing.assert_array_equal(pred.numpy(), results[0]) @@ -986,11 +1052,13 @@ class TestResnet2(unittest.TestCase): Use paddle-2.0 API """ - def train_resnet(self, - enable_amp=True, - level='O1', - use_data_loader=False, - use_param_group=False): + def train_resnet( + self, + enable_amp=True, + level='O1', + use_data_loader=False, + use_param_group=False, + ): seed = 90 batch_size = train_parameters["batch_size"] @@ -1014,43 +1082,42 @@ def train_resnet(self, # NOTE(zhiqiu): The Membership test operations(in / not in) calls "is" and "equal", # see details: https://docs.python.org/3/reference/expressions.html#membership-test-operations. # So do not use other_params = [p for p in resnet.parameters() if p not in conv_params] - optimizer = paddle.optimizer.Momentum(parameters=[{ - 'params': - conv_params, - 'learning_rate': - 0.01 - }, { - 'params': - other_params, - 'learning_rate': - 0.001 - }], - multi_precision=True) + optimizer = paddle.optimizer.Momentum( + parameters=[ + {'params': conv_params, 'learning_rate': 0.01}, + {'params': other_params, 'learning_rate': 0.001}, + ], + multi_precision=True, + ) else: optimizer = paddle.optimizer.SGD(parameters=resnet.parameters()) np.random.seed(seed) train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size) + paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.amp.GradScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.amp.GradScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if use_data_loader: - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator( capacity=4, use_double_buffer=True, iterable=True, - return_list=True) + return_list=True, + ) train_loader.set_sample_list_generator(train_reader) train_reader = train_loader @@ -1063,13 +1130,19 @@ def train_resnet(self, if use_data_loader: img, label = data else: - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) @@ -1094,8 +1167,9 @@ def train_resnet(self, for param in resnet.parameters(): if param.trainable: np_array = np.array(param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -1107,70 +1181,65 @@ def train_resnet(self, return dy_out, dy_param_value, dy_grad_value def test_resnet(self): - def func_isinstance(): with fluid.dygraph.guard(): out_fp32 = self.train_resnet(enable_amp=False) out_amp = self.train_resnet(enable_amp=True) out_pure_fp16 = self.train_resnet(enable_amp=True, level='O2') print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.01) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01 + ) func_isinstance() def test_with_data_loader(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_fp32 = self.train_resnet(enable_amp=False, - use_data_loader=True) - out_amp = self.train_resnet(enable_amp=True, - use_data_loader=True) - out_pure_fp16 = self.train_resnet(enable_amp=True, - use_data_loader=True, - level='O2') + out_fp32 = self.train_resnet( + enable_amp=False, use_data_loader=True + ) + out_amp = self.train_resnet( + enable_amp=True, use_data_loader=True + ) + out_pure_fp16 = self.train_resnet( + enable_amp=True, use_data_loader=True, level='O2' + ) print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.01) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01 + ) func_isinstance() def test_param_group(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_fp32 = self.train_resnet(enable_amp=False, - use_data_loader=True, - use_param_group=True) - out_amp = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_param_group=True) - out_pure_fp16 = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_param_group=True, - level='O2') + out_fp32 = self.train_resnet( + enable_amp=False, use_data_loader=True, use_param_group=True + ) + out_amp = self.train_resnet( + enable_amp=True, use_data_loader=True, use_param_group=True + ) + out_pure_fp16 = self.train_resnet( + enable_amp=True, + use_data_loader=True, + use_param_group=True, + level='O2', + ) print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.01) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01 + ) func_isinstance() @@ -1191,42 +1260,54 @@ def train_resnet(self, enable_amp=True, level='O1'): paddle.framework.random._manual_program_seed(seed) resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) optimizer = paddle.optimizer.Momentum( - parameters=resnet.parameters(), multi_precision=True) + parameters=resnet.parameters(), multi_precision=True + ) np.random.seed(seed) train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), - batch_size=batch_size) + batch_size=batch_size, + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.fluid.dygraph.AmpScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.fluid.dygraph.AmpScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if enable_amp and (level == 'O2'): resnet, optimizer = paddle.fluid.dygraph.amp_decorate( - models=resnet, optimizers=optimizer, level='O2') + models=resnet, optimizers=optimizer, level='O2' + ) for batch_id, data in enumerate(train_reader()): if batch_id >= batch_num: break - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = fluid.dygraph.to_variable(dy_x_data) label = fluid.dygraph.to_variable(y_data) label.stop_gradient = True - with paddle.fluid.dygraph.amp_guard(enable=enable_amp, - level=level): + with paddle.fluid.dygraph.amp_guard( + enable=enable_amp, level=level + ): out = resnet(img) loss = fluid.layers.cross_entropy(input=out, label=label) @@ -1243,9 +1324,11 @@ def train_resnet(self, enable_amp=True, level='O1'): for param in resnet.parameters(): if param.trainable: np_array = np.array( - param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + param._grad_ivar().value().get_tensor() + ) + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -1256,31 +1339,27 @@ def train_resnet(self, enable_amp=True, level='O1'): return dy_out, dy_param_value, dy_grad_value def test_resnet(self): - def func_isinstance(): out_fp32 = self.train_resnet(enable_amp=False) out_amp = self.train_resnet(enable_amp=True) out_pure_fp16 = self.train_resnet(enable_amp=True, level='O2') print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=0.01) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.1) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=0.01 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.1 + ) func_isinstance() class TestLayerNormFp16(unittest.TestCase): - r''' layer_norm and batch_norm support mixed inputs, i.e., only input x is fp16 + r'''layer_norm and batch_norm support mixed inputs, i.e., only input x is fp16 and other params are fp32. ''' def test_layer_norm_fp16(self): - def func_isinstance(): if fluid.is_compiled_with_cuda(): with fluid.dygraph.guard(fluid.CUDAPlace(0)): @@ -1290,7 +1369,8 @@ def func_isinstance(): out = layer_norm(x) self.assertTrue( - out.dtype == fluid.core.VarDesc.VarType.FP16) + out.dtype == fluid.core.VarDesc.VarType.FP16 + ) func_isinstance() @@ -1298,7 +1378,8 @@ def func_isinstance(): @unittest.skipIf( paddle.is_compiled_with_cuda() and not core.is_bfloat16_supported(core.CUDAPlace(0)), - "skip bf16 test if cuda is in use but bf16 is not supported by gpu arch.") + "skip bf16 test if cuda is in use but bf16 is not supported by gpu arch.", +) class TestBf16(unittest.TestCase): ''' test amp for BF16 @@ -1306,43 +1387,37 @@ class TestBf16(unittest.TestCase): def train(self, enable_amp=True, amp_level='O1'): paddle.seed(100) - input = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.) + input = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0) conv = paddle.nn.Conv2D(4, 6, (3, 3)) if amp_level == 'O2': - conv = paddle.amp.decorate(models=conv, - level=amp_level, - dtype='bfloat16') - with paddle.amp.auto_cast(enable=enable_amp, - level=amp_level, - dtype='bfloat16'): + conv = paddle.amp.decorate( + models=conv, level=amp_level, dtype='bfloat16' + ) + with paddle.amp.auto_cast( + enable=enable_amp, level=amp_level, dtype='bfloat16' + ): output = conv(input) output = output.cast('float32') return output.numpy() def test_bf16(self): - def func_isinstance(): out_fp32 = self.train(enable_amp=False) out_bf16_O1 = self.train(enable_amp=True, amp_level='O1') out_bf16_O2 = self.train(enable_amp=True, amp_level='O2') - np.testing.assert_allclose(out_fp32, - out_bf16_O1, - rtol=0.001, - atol=0.1) - np.testing.assert_allclose(out_fp32, - out_bf16_O2, - rtol=0.001, - atol=0.1) + np.testing.assert_allclose( + out_fp32, out_bf16_O1, rtol=0.001, atol=0.1 + ) + np.testing.assert_allclose( + out_fp32, out_bf16_O2, rtol=0.001, atol=0.1 + ) func_isinstance() class TestAmpWithPyLyer(unittest.TestCase): - def test_pylayer(self): - class MyMM(PyLayer): - @staticmethod def forward(ctx, a, b): ctx.save_for_backward(a, b) @@ -1367,9 +1442,7 @@ def backward(ctx, grad): class TestAmpWithHook(unittest.TestCase): - def test_hook_change_dtype(self): - def func_isinstance(): with paddle.fluid.dygraph.guard(): v = paddle.rand([3, 3]) @@ -1390,7 +1463,6 @@ def foo(grad): func_isinstance() def test_hook_change_place(self): - def func_isinstance(): with paddle.fluid.dygraph.guard(): v = paddle.rand([3, 3]) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py index 7ff434987a9c1..83c9462a89e9d 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_imperative_auto_mixed_precision_for_eager.py @@ -31,31 +31,33 @@ class SimpleConv(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(SimpleConv, self).__init__() - self._conv = fluid.dygraph.Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - bias_attr=None, - use_cudnn=True) + self._conv = fluid.dygraph.Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + bias_attr=None, + use_cudnn=True, + ) def forward(self, inputs): return self._conv(inputs) class TestAutoCast(unittest.TestCase): - def amp_guard_white_op(self): data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32') with fluid.dygraph.guard(): @@ -92,30 +94,37 @@ def custom_op_list(self): tracer = fluid.framework._dygraph_tracer() base_white_list = fluid.dygraph.amp.auto_cast.WHITE_LIST base_black_list = fluid.dygraph.amp.auto_cast.BLACK_LIST - with fluid.dygraph.amp_guard(custom_white_list=["log"], - custom_black_list=["conv2d"]): + with fluid.dygraph.amp_guard( + custom_white_list=["log"], custom_black_list=["conv2d"] + ): white_list, black_list = tracer._get_amp_op_list() self.assertTrue( - set(white_list) == (set(base_white_list) | {"log"}) - - {"conv2d"}) + set(white_list) + == (set(base_white_list) | {"log"}) - {"conv2d"} + ) self.assertTrue( - set(black_list) == (set(base_black_list) - {"log"}) - | {"conv2d"}) + set(black_list) + == (set(base_black_list) - {"log"}) | {"conv2d"} + ) base_white_list = fluid.dygraph.amp.auto_cast.PURE_FP16_WHITE_LIST base_black_list = fluid.dygraph.amp.auto_cast.PURE_FP16_BLACK_LIST - with fluid.dygraph.amp_guard(custom_white_list=["log"], - custom_black_list=["conv2d"], - level='O2'): + with fluid.dygraph.amp_guard( + custom_white_list=["log"], + custom_black_list=["conv2d"], + level='O2', + ): white_list, black_list = tracer._get_amp_op_list() self.assertTrue( - set(white_list) == (set(base_white_list) | {"log"}) - - {"conv2d"}) + set(white_list) + == (set(base_white_list) | {"log"}) - {"conv2d"} + ) self.assertTrue( - set(black_list) == (set(base_black_list) - {"log"}) - | {"conv2d"}) + set(black_list) + == (set(base_black_list) - {"log"}) | {"conv2d"} + ) def test_custom_op_list(self): self.custom_op_list() @@ -125,13 +134,16 @@ def custom_op_list_exception(self): def func(): with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - with fluid.dygraph.amp_guard(custom_white_list=["conv2d"], - custom_black_list=["conv2d"]): + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + with fluid.dygraph.amp_guard( + custom_white_list=["conv2d"], custom_black_list=["conv2d"] + ): inp = fluid.dygraph.to_variable(inp_np) out = model(inp) @@ -148,35 +160,34 @@ def amp_guard_upsupported_fp16_op(self): with fluid.dygraph.amp_guard(True): out_amp_fp16 = conv2d(data) out_amp_fp32 = paddle.expand_as( - out_amp_fp16, - out_amp_fp16) # expand_as_v2 has no fp16 kernel + out_amp_fp16, out_amp_fp16 + ) # expand_as_v2 has no fp16 kernel with fluid.dygraph.amp_guard(True, level='O2'): out_purefp16_fp16 = conv2d(data) out_purefp16_fp32 = paddle.expand_as( - out_purefp16_fp16, - out_purefp16_fp16) # expand_as_v2 has no fp16 kernel + out_purefp16_fp16, out_purefp16_fp16 + ) # expand_as_v2 has no fp16 kernel self.assertTrue(data.dtype == fluid.core.VarDesc.VarType.FP32) self.assertTrue(out_amp_fp16.dtype == fluid.core.VarDesc.VarType.FP16) self.assertTrue(out_amp_fp32.dtype == fluid.core.VarDesc.VarType.FP32) self.assertTrue( - out_purefp16_fp16.dtype == fluid.core.VarDesc.VarType.FP16) + out_purefp16_fp16.dtype == fluid.core.VarDesc.VarType.FP16 + ) self.assertTrue( - out_purefp16_fp32.dtype == fluid.core.VarDesc.VarType.FP32) + out_purefp16_fp32.dtype == fluid.core.VarDesc.VarType.FP32 + ) def test_amp_guard_upsupported_fp16_op(self): self.amp_guard_upsupported_fp16_op() def mode_exception(self): - def func(): data = np.random.uniform(-1, 1, [10, 3, 32, 32]).astype('float32') with fluid.dygraph.guard(): - conv2d = fluid.dygraph.Conv2D(3, - 2, - 3, - bias_attr=False, - act=None) + conv2d = fluid.dygraph.Conv2D( + 3, 2, 3, bias_attr=False, act=None + ) data = fluid.dygraph.to_variable(data) with fluid.dygraph.amp_guard(level='O'): out = conv2d(data) @@ -188,15 +199,14 @@ def test_mode_exception(self): class TestAmpScaler(unittest.TestCase): - def scale(self): with fluid.dygraph.guard(): data = paddle.rand([10, 1024]) scaler = paddle.fluid.dygraph.AmpScaler(init_loss_scaling=1024) scaled_data = scaler.scale(data) self.assertEqual( - np.array_equal(scaled_data.numpy(), - data.numpy() * 1024), True) + np.array_equal(scaled_data.numpy(), data.numpy() * 1024), True + ) def test_scale(self): self.scale() @@ -208,13 +218,16 @@ def run_simple_conv(inp_np, use_scaler=True): paddle.seed(10) paddle.framework.random._manual_program_seed(10) with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) optimizer = fluid.optimizer.SGDOptimizer( - learning_rate=0.01, parameter_list=model.parameters()) + learning_rate=0.01, parameter_list=model.parameters() + ) scaler = fluid.dygraph.AmpScaler(init_loss_scaling=1024) data = fluid.dygraph.to_variable(inp_np) @@ -225,7 +238,8 @@ def run_simple_conv(inp_np, use_scaler=True): scaled_loss = scaler.scale(loss) scaled_loss.backward() optimize_ops, params_grads = scaler.minimize( - optimizer, scaled_loss) + optimizer, scaled_loss + ) else: print('use no scaler') loss.backward() @@ -235,19 +249,25 @@ def run_simple_conv(inp_np, use_scaler=True): outs_with_scaler = run_simple_conv(inp_np, use_scaler=True) outs_no_scaler = run_simple_conv(inp_np, use_scaler=False) - self.assertEqual(outs_with_scaler[0], - []) # optimize_ops is [] in dygraph mode - self.assertEqual(outs_no_scaler[0], - []) # optimize_ops is [] in dygraph mode + self.assertEqual( + outs_with_scaler[0], [] + ) # optimize_ops is [] in dygraph mode + self.assertEqual( + outs_no_scaler[0], [] + ) # optimize_ops is [] in dygraph mode for i in range(len(outs_with_scaler[1])): # check each grad - np.testing.assert_allclose(outs_with_scaler[1][i][1].numpy(), - outs_no_scaler[1][i][1].numpy(), - rtol=1e-05) + np.testing.assert_allclose( + outs_with_scaler[1][i][1].numpy(), + outs_no_scaler[1][i][1].numpy(), + rtol=1e-05, + ) # check each parameter - np.testing.assert_allclose(outs_with_scaler[1][i][0].numpy(), - outs_no_scaler[1][i][0].numpy(), - rtol=1e-05) + np.testing.assert_allclose( + outs_with_scaler[1][i][0].numpy(), + outs_no_scaler[1][i][0].numpy(), + rtol=1e-05, + ) def test_minimize(self): self.minimize() @@ -259,13 +279,16 @@ def run_simple_conv(inp_np, use_scaler=True): paddle.seed(10) paddle.framework.random._manual_program_seed(10) with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = fluid.dygraph.to_variable(inp_np) @@ -288,9 +311,11 @@ def run_simple_conv(inp_np, use_scaler=True): for i in range(len(outs_with_scaler)): # check each parameter - np.testing.assert_allclose(outs_with_scaler[i].numpy(), - outs_no_scaler[i].numpy(), - rtol=1e-05) + np.testing.assert_allclose( + outs_with_scaler[i].numpy(), + outs_no_scaler[i].numpy(), + rtol=1e-05, + ) def test_step(self): self.step() @@ -299,16 +324,19 @@ def nan_inf(self): inp_np = np.random.random(size=[1, 3, 128, 128]).astype(np.float32) inp_np[0][1][2][3] = np.nan with fluid.dygraph.guard(): - model = SimpleConv(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') + model = SimpleConv( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) params_init = {} for param in model.parameters(): params_init[param.name] = param.numpy() optimizer = fluid.optimizer.SGDOptimizer( - learning_rate=0.01, parameter_list=model.parameters()) + learning_rate=0.01, parameter_list=model.parameters() + ) scaler = fluid.dygraph.AmpScaler(init_loss_scaling=1024) data = fluid.dygraph.to_variable(inp_np) @@ -321,18 +349,19 @@ def nan_inf(self): for param in model.parameters(): # param not update when tensor contains nan or inf - np.testing.assert_array_equal(param.numpy(), - params_init[param.name]) + np.testing.assert_array_equal( + param.numpy(), params_init[param.name] + ) def test_nan_inf(self): self.nan_inf() def step_update_exception(self): - def func1(): model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = paddle.rand([10, 3, 32, 32]) conv = model(data) @@ -346,8 +375,9 @@ def func1(): def func2(): model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = paddle.rand([10, 3, 32, 32]) conv = model(data) @@ -361,8 +391,9 @@ def func2(): def func3(): model = paddle.nn.Conv2D(3, 2, 3, bias_attr=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) data = paddle.rand([10, 3, 32, 32]) conv = model(data) @@ -379,13 +410,15 @@ def test_step_update_exception(self): def test_get_and_set(self): with fluid.dygraph.guard(): - scaler = paddle.amp.GradScaler(enable=True, - init_loss_scaling=1024, - incr_ratio=2.0, - decr_ratio=0.5, - incr_every_n_steps=1000, - decr_every_n_nan_or_inf=2, - use_dynamic_loss_scaling=True) + scaler = paddle.amp.GradScaler( + enable=True, + init_loss_scaling=1024, + incr_ratio=2.0, + decr_ratio=0.5, + incr_every_n_steps=1000, + decr_every_n_nan_or_inf=2, + use_dynamic_loss_scaling=True, + ) self.assertEqual(scaler.is_enable() == True, True) self.assertEqual(scaler.get_init_loss_scaling() == 1024, True) self.assertEqual(scaler.get_incr_ratio() == 2.0, True) @@ -406,13 +439,15 @@ def test_get_and_set(self): def test_state_dict_and_load_state_dict(self): with fluid.dygraph.guard(): - scaler1 = paddle.amp.GradScaler(enable=True, - init_loss_scaling=14, - incr_ratio=233.0, - decr_ratio=0.523, - incr_every_n_steps=1090, - decr_every_n_nan_or_inf=20, - use_dynamic_loss_scaling=True) + scaler1 = paddle.amp.GradScaler( + enable=True, + init_loss_scaling=14, + incr_ratio=233.0, + decr_ratio=0.523, + incr_every_n_steps=1090, + decr_every_n_nan_or_inf=20, + use_dynamic_loss_scaling=True, + ) scaler_state = scaler1.state_dict() scaler2 = paddle.amp.GradScaler(enable=True) scaler2.load_state_dict(scaler_state) @@ -427,7 +462,6 @@ def test_state_dict_and_load_state_dict(self): self.assertEqual(scaler3.is_enable() == False, True) def test_state_dict_and_load_state_dict_error(self): - def test_error(): state_empty = {} scaler = paddle.amp.GradScaler(enable=True) @@ -437,7 +471,6 @@ def test_error(): def reader_decorator(reader): - def __reader__(): for item in reader(): img = np.array(item[0]).astype('float32').reshape(3, 224, 224) @@ -448,11 +481,9 @@ def __reader__(): class TestGradScalerStateDict(unittest.TestCase): - - def train_resnet(self, - enable_amp=True, - use_data_loader=True, - use_save_load=True): + def train_resnet( + self, enable_amp=True, use_data_loader=True, use_save_load=True + ): seed = 90 batch_size = train_parameters["batch_size"] @@ -462,30 +493,35 @@ def train_resnet(self, paddle.framework.random._manual_program_seed(seed) resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) np.random.seed(seed) train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size) + paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.amp.GradScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.amp.GradScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if use_data_loader: - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator( capacity=4, use_double_buffer=True, iterable=True, - return_list=True) + return_list=True, + ) train_loader.set_sample_list_generator(train_reader) train_reader = train_loader @@ -495,13 +531,19 @@ def train_resnet(self, if use_data_loader: img, label = data else: - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) @@ -524,8 +566,9 @@ def train_resnet(self, for param in resnet.parameters(): if param.trainable: np_array = np.array(param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -542,43 +585,37 @@ def train_resnet(self, return dy_out, dy_param_value, dy_grad_value def test_with_state_dict(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_use_state_dict = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=True) - out_no_state_dict = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=False) + out_use_state_dict = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=True + ) + out_no_state_dict = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=False + ) print('save_load:', out_use_state_dict[0], out_no_state_dict[0]) - np.testing.assert_allclose(out_use_state_dict[0], - out_no_state_dict[0], - rtol=1e-05) + np.testing.assert_allclose( + out_use_state_dict[0], out_no_state_dict[0], rtol=1e-05 + ) func_isinstance() class TestAmpDecorator(unittest.TestCase): - def test_mode_exception(self): - def func(): with fluid.dygraph.guard(): model = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) opt = paddle.optimizer.SGD(parameters=model.parameters()) - model, opt = paddle.amp.decorate(models=model, - optimizers=opt, - level='O') + model, opt = paddle.amp.decorate( + models=model, optimizers=opt, level='O' + ) self.assertRaises(ValueError, func) def test_input_type_exception(self): - def test_error_model(): - class MyModel(object): - def __init__(self): print("A fake Model") @@ -597,9 +634,7 @@ def test_error_distributed_model(): self.assertRaises(RuntimeError, test_error_distributed_model) def test_error_optimizer(): - class MyOptimizer(object): - def __init__(self): print("A fake Optimizer") @@ -612,48 +647,59 @@ def __init__(self): def test_set_master_weight(self): model1 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt1 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model1.parameters(), - multi_precision=True) + opt1 = paddle.optimizer.Adam( + learning_rate=0.0001, + parameters=model1.parameters(), + multi_precision=True, + ) model2 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt2 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model2.parameters(), - multi_precision=False) - - model1, opt1 = paddle.amp.decorate(models=model1, - optimizers=opt1, - level='O2', - master_weight=None) + opt2 = paddle.optimizer.Adam( + learning_rate=0.0001, + parameters=model2.parameters(), + multi_precision=False, + ) + + model1, opt1 = paddle.amp.decorate( + models=model1, optimizers=opt1, level='O2', master_weight=None + ) self.assertEqual(opt1._multi_precision, True) - models, opt2 = paddle.amp.decorate(models=[model1, model2], - optimizers=opt2, - level='O2', - master_weight=None) + models, opt2 = paddle.amp.decorate( + models=[model1, model2], + optimizers=opt2, + level='O2', + master_weight=None, + ) self.assertEqual(opt2._multi_precision, True) model3 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt3 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model3.parameters()) + opt3 = paddle.optimizer.Adam( + learning_rate=0.0001, parameters=model3.parameters() + ) model4 = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) - opt4 = paddle.optimizer.Adam(learning_rate=0.0001, - parameters=model4.parameters()) - - model3, opts = paddle.amp.decorate(models=model3, - optimizers=[opt3, opt4], - level='O2', - master_weight=True) + opt4 = paddle.optimizer.Adam( + learning_rate=0.0001, parameters=model4.parameters() + ) + + model3, opts = paddle.amp.decorate( + models=model3, + optimizers=[opt3, opt4], + level='O2', + master_weight=True, + ) self.assertEqual(opts[0]._multi_precision, True) self.assertEqual(opts[1]._multi_precision, True) models = [model3, model4] optimizers = [opt3, opt4] - models, optimizers = paddle.amp.decorate(models=models, - optimizers=optimizers, - level='O2', - master_weight=False) + models, optimizers = paddle.amp.decorate( + models=models, + optimizers=optimizers, + level='O2', + master_weight=False, + ) self.assertEqual(optimizers[0]._multi_precision, False) self.assertEqual(optimizers[1]._multi_precision, False) @@ -685,15 +731,13 @@ def test_skip_BatchNorm_Layer_norm(self): class TestStateDictHookForAMP(unittest.TestCase): - def test_state_dict_hook(self): - def func_isinstance(): paddle.seed(100) model = paddle.nn.Linear(2, 4) - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) param_value_ori = {} for param in model.parameters(): param_value_ori[param.name] = param.numpy() @@ -714,7 +758,6 @@ def func_isinstance(): class TestPureFp16SaveLoad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -722,22 +765,19 @@ def tearDown(self): self.temp_dir.cleanup() def test_save_dtype_exception(self): - def func(): paddle.disable_static() model = fluid.dygraph.Conv2D(3, 2, 3, bias_attr=False, act=None) opt = paddle.optimizer.SGD(parameters=model.parameters()) - paddle.amp.decorate(models=model, - optimizers=opt, - level='O2', - save_dtype='int') + paddle.amp.decorate( + models=model, optimizers=opt, level='O2', save_dtype='int' + ) self.assertRaises(ValueError, func) - def train_resnet(self, - enable_amp=True, - use_data_loader=True, - use_save_load=True): + def train_resnet( + self, enable_amp=True, use_data_loader=True, use_save_load=True + ): seed = 90 batch_size = train_parameters["batch_size"] @@ -747,38 +787,45 @@ def train_resnet(self, paddle.framework.random._manual_program_seed(seed) resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) np.random.seed(seed) train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size) + paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.amp.GradScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.amp.GradScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if use_data_loader: - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator( capacity=4, use_double_buffer=True, iterable=True, - return_list=True) + return_list=True, + ) train_loader.set_sample_list_generator(train_reader) train_reader = train_loader if enable_amp: - resnet, optimizer = paddle.amp.decorate(models=resnet, - optimizers=optimizer, - level='O2', - save_dtype='float32') + resnet, optimizer = paddle.amp.decorate( + models=resnet, + optimizers=optimizer, + level='O2', + save_dtype='float32', + ) for batch_id, data in enumerate(train_reader()): if batch_id >= batch_num: @@ -786,13 +833,19 @@ def train_resnet(self, if use_data_loader: img, label = data else: - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) @@ -816,8 +869,9 @@ def train_resnet(self, for param in resnet.parameters(): if param.trainable: np_array = np.array(param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -830,7 +884,7 @@ def train_resnet(self, obj = { 'model': resnet.state_dict(), 'opt': optimizer.state_dict(), - 'scaler': scaler.state_dict() + 'scaler': scaler.state_dict(), } path = os.path.join(self.temp_dir.name, 'model.pdparams') paddle.save(obj, path) @@ -838,39 +892,40 @@ def train_resnet(self, obj_load = paddle.load(path) resnet = ResNet(use_cudnn=True) optimizer = optimizer_setting( - train_parameters, parameter_list=resnet.parameters()) + train_parameters, parameter_list=resnet.parameters() + ) resnet.set_state_dict(obj_load['model']) optimizer.set_state_dict(obj_load['opt']) scaler.load_state_dict(obj_load['scaler']) - resnet, optimizer = paddle.amp.decorate(models=resnet, - optimizers=optimizer, - level='O2', - save_dtype='float32') + resnet, optimizer = paddle.amp.decorate( + models=resnet, + optimizers=optimizer, + level='O2', + save_dtype='float32', + ) if use_data_loader: train_reader._reset() return dy_out, dy_param_value, dy_grad_value def test_with_save_load(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_use_save_load = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=True) - out_no_save_load = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_save_load=False) + out_use_save_load = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=True + ) + out_no_save_load = self.train_resnet( + enable_amp=True, use_data_loader=True, use_save_load=False + ) print('save_load:', out_use_save_load[0], out_no_save_load[0]) - np.testing.assert_allclose(out_use_save_load[0], - out_no_save_load[0], - rtol=1e-05) + np.testing.assert_allclose( + out_use_save_load[0], out_no_save_load[0], rtol=1e-05 + ) func_isinstance() class TestPureFp16InferenceSaveLoad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -886,21 +941,20 @@ def inference_save_load(self): # define a random dataset class RandomDataset(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, - (1, )).astype('int64') + label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype( + 'int64' + ) return image, label def __len__(self): return self.num_samples class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) @@ -911,10 +965,12 @@ def forward(self, x): def train(layer, loader, loss_fn, opt): for epoch_id in range(EPOCH_NUM): for batch_id, (image, label) in enumerate(loader()): - with paddle.amp.auto_cast(enable=True, - custom_white_list=None, - custom_black_list=None, - level='O2'): + with paddle.amp.auto_cast( + enable=True, + custom_white_list=None, + custom_black_list=None, + level='O2', + ): out = layer(image) loss = loss_fn(out, label) loss.backward() @@ -923,27 +979,31 @@ def train(layer, loader, loss_fn, opt): # train layer = LinearNet() - adam = paddle.optimizer.Adam(learning_rate=0.001, - parameters=layer.parameters(), - multi_precision=True) + adam = paddle.optimizer.Adam( + learning_rate=0.001, + parameters=layer.parameters(), + multi_precision=True, + ) loss_fn = nn.CrossEntropyLoss() - layer, adam = paddle.amp.decorate(models=layer, - optimizers=adam, - save_dtype='float32') + layer, adam = paddle.amp.decorate( + models=layer, optimizers=adam, save_dtype='float32' + ) dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) - loader = paddle.io.DataLoader(dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) + loader = paddle.io.DataLoader( + dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) train(layer, loader, loss_fn, adam) # save path = os.path.join(self.temp_dir.name, 'example_model/linear') - paddle.jit.save(layer, - path, - input_spec=[InputSpec(shape=[IMAGE_SIZE], name='x')]) + paddle.jit.save( + layer, path, input_spec=[InputSpec(shape=[IMAGE_SIZE], name='x')] + ) # jit.load loaded_layer = paddle.jit.load(path) @@ -957,12 +1017,17 @@ def train(layer, loader, loss_fn, opt): # load_inference_model paddle.enable_static() exe = paddle.static.Executor() - [inference_program, feed_target_names, - fetch_targets] = (paddle.static.load_inference_model(path, exe)) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(path, exe) tensor_img = x - results = exe.run(inference_program, - feed={feed_target_names[0]: tensor_img}, - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed={feed_target_names[0]: tensor_img}, + fetch_list=fetch_targets, + ) print("pred.numpy()", pred.numpy()) print("result", results[0]) np.testing.assert_array_equal(pred.numpy(), results[0]) @@ -977,11 +1042,13 @@ class TestResnet2(unittest.TestCase): Use paddle-2.0 API """ - def train_resnet(self, - enable_amp=True, - level='O1', - use_data_loader=False, - use_param_group=False): + def train_resnet( + self, + enable_amp=True, + level='O1', + use_data_loader=False, + use_param_group=False, + ): seed = 90 batch_size = train_parameters["batch_size"] @@ -1005,43 +1072,42 @@ def train_resnet(self, # NOTE(zhiqiu): The Membership test operations(in / not in) calls "is" and "equal", # see details: https://docs.python.org/3/reference/expressions.html#membership-test-operations. # So do not use other_params = [p for p in resnet.parameters() if p not in conv_params] - optimizer = paddle.optimizer.Momentum(parameters=[{ - 'params': - conv_params, - 'learning_rate': - 0.01 - }, { - 'params': - other_params, - 'learning_rate': - 0.001 - }], - multi_precision=True) + optimizer = paddle.optimizer.Momentum( + parameters=[ + {'params': conv_params, 'learning_rate': 0.01}, + {'params': other_params, 'learning_rate': 0.001}, + ], + multi_precision=True, + ) else: optimizer = paddle.optimizer.SGD(parameters=resnet.parameters()) np.random.seed(seed) train_reader = paddle.batch( - paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size) + paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.amp.GradScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.amp.GradScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if use_data_loader: - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator( capacity=4, use_double_buffer=True, iterable=True, - return_list=True) + return_list=True, + ) train_loader.set_sample_list_generator(train_reader) train_reader = train_loader @@ -1054,13 +1120,19 @@ def train_resnet(self, if use_data_loader: img, label = data else: - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) @@ -1085,8 +1157,9 @@ def train_resnet(self, for param in resnet.parameters(): if param.trainable: np_array = np.array(param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -1098,70 +1171,65 @@ def train_resnet(self, return dy_out, dy_param_value, dy_grad_value def test_resnet(self): - def func_isinstance(): with fluid.dygraph.guard(): out_fp32 = self.train_resnet(enable_amp=False) out_amp = self.train_resnet(enable_amp=True) out_pure_fp16 = self.train_resnet(enable_amp=True, level='O2') print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.01) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01 + ) func_isinstance() def test_with_data_loader(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_fp32 = self.train_resnet(enable_amp=False, - use_data_loader=True) - out_amp = self.train_resnet(enable_amp=True, - use_data_loader=True) - out_pure_fp16 = self.train_resnet(enable_amp=True, - use_data_loader=True, - level='O2') + out_fp32 = self.train_resnet( + enable_amp=False, use_data_loader=True + ) + out_amp = self.train_resnet( + enable_amp=True, use_data_loader=True + ) + out_pure_fp16 = self.train_resnet( + enable_amp=True, use_data_loader=True, level='O2' + ) print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.01) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01 + ) func_isinstance() def test_param_group(self): - def func_isinstance(): with fluid.dygraph.guard(): - out_fp32 = self.train_resnet(enable_amp=False, - use_data_loader=True, - use_param_group=True) - out_amp = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_param_group=True) - out_pure_fp16 = self.train_resnet(enable_amp=True, - use_data_loader=True, - use_param_group=True, - level='O2') + out_fp32 = self.train_resnet( + enable_amp=False, use_data_loader=True, use_param_group=True + ) + out_amp = self.train_resnet( + enable_amp=True, use_data_loader=True, use_param_group=True + ) + out_pure_fp16 = self.train_resnet( + enable_amp=True, + use_data_loader=True, + use_param_group=True, + level='O2', + ) print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.01) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.01 + ) func_isinstance() @@ -1182,42 +1250,54 @@ def train_resnet(self, enable_amp=True, level='O1'): paddle.framework.random._manual_program_seed(seed) resnet = ResNet(use_cudnn=True) - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) optimizer = paddle.optimizer.Momentum( - parameters=resnet.parameters(), multi_precision=True) + parameters=resnet.parameters(), multi_precision=True + ) np.random.seed(seed) train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), - batch_size=batch_size) + batch_size=batch_size, + ) dy_param_init_value = {} for param in resnet.parameters(): dy_param_init_value[param.name] = param.numpy() program = None - scaler = paddle.fluid.dygraph.AmpScaler(enable=enable_amp, - init_loss_scaling=2.**10) + scaler = paddle.fluid.dygraph.AmpScaler( + enable=enable_amp, init_loss_scaling=2.0**10 + ) if enable_amp and (level == 'O2'): resnet, optimizer = paddle.fluid.dygraph.amp_decorate( - models=resnet, optimizers=optimizer, level='O2') + models=resnet, optimizers=optimizer, level='O2' + ) for batch_id, data in enumerate(train_reader()): if batch_id >= batch_num: break - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - if len(np.array([x[1] - for x in data]).astype('int64')) != batch_size: + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + if ( + len(np.array([x[1] for x in data]).astype('int64')) + != batch_size + ): continue - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = fluid.dygraph.to_variable(dy_x_data) label = fluid.dygraph.to_variable(y_data) label.stop_gradient = True - with paddle.fluid.dygraph.amp_guard(enable=enable_amp, - level=level): + with paddle.fluid.dygraph.amp_guard( + enable=enable_amp, level=level + ): out = resnet(img) loss = fluid.layers.cross_entropy(input=out, label=label) @@ -1234,9 +1314,11 @@ def train_resnet(self, enable_amp=True, level='O1'): for param in resnet.parameters(): if param.trainable: np_array = np.array( - param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - fluid.core.grad_var_suffix()] = np_array + param._grad_ivar().value().get_tensor() + ) + dy_grad_value[ + param.name + fluid.core.grad_var_suffix() + ] = np_array resnet.clear_gradients() @@ -1247,31 +1329,27 @@ def train_resnet(self, enable_amp=True, level='O1'): return dy_out, dy_param_value, dy_grad_value def test_resnet(self): - def func_isinstance(): out_fp32 = self.train_resnet(enable_amp=False) out_amp = self.train_resnet(enable_amp=True) out_pure_fp16 = self.train_resnet(enable_amp=True, level='O2') print(out_fp32[0], out_amp[0], out_pure_fp16[0]) - np.testing.assert_allclose(out_fp32[0], - out_amp[0], - rtol=1e-05, - atol=0.01) - np.testing.assert_allclose(out_fp32[0], - out_pure_fp16[0], - rtol=1e-05, - atol=0.1) + np.testing.assert_allclose( + out_fp32[0], out_amp[0], rtol=1e-05, atol=0.01 + ) + np.testing.assert_allclose( + out_fp32[0], out_pure_fp16[0], rtol=1e-05, atol=0.1 + ) func_isinstance() class TestLayerNormFp16(unittest.TestCase): - r''' layer_norm and batch_norm support mixed inputs, i.e., only input x is fp16 + r'''layer_norm and batch_norm support mixed inputs, i.e., only input x is fp16 and other params are fp32. ''' def test_layer_norm_fp16(self): - def func_isinstance(): if fluid.is_compiled_with_cuda(): with fluid.dygraph.guard(fluid.CUDAPlace(0)): @@ -1281,7 +1359,8 @@ def func_isinstance(): out = layer_norm(x) self.assertTrue( - out.dtype == fluid.core.VarDesc.VarType.FP16) + out.dtype == fluid.core.VarDesc.VarType.FP16 + ) func_isinstance() @@ -1293,45 +1372,41 @@ class TestBf16(unittest.TestCase): def train(self, enable_amp=True, amp_level='O1'): paddle.seed(100) - input = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.) + input = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0) conv = paddle.nn.Conv2D(4, 6, (3, 3)) if amp_level == 'O2': - conv = paddle.amp.decorate(models=conv, - level=amp_level, - dtype='bfloat16') - with paddle.amp.auto_cast(enable=enable_amp, - level=amp_level, - dtype='bfloat16'): + conv = paddle.amp.decorate( + models=conv, level=amp_level, dtype='bfloat16' + ) + with paddle.amp.auto_cast( + enable=enable_amp, level=amp_level, dtype='bfloat16' + ): output = conv(input) output = output.cast('float32') return output.numpy() def test_bf16(self): - def func_isinstance(): - if fluid.core.is_compiled_with_cuda( - ) and fluid.core.is_bfloat16_supported(paddle.CUDAPlace(0)): + if ( + fluid.core.is_compiled_with_cuda() + and fluid.core.is_bfloat16_supported(paddle.CUDAPlace(0)) + ): out_fp32 = self.train(enable_amp=False) out_bf16_O1 = self.train(enable_amp=True, amp_level='O1') out_bf16_O2 = self.train(enable_amp=True, amp_level='O2') - np.testing.assert_allclose(out_fp32, - out_bf16_O1, - rtol=0.001, - atol=0.1) - np.testing.assert_allclose(out_fp32, - out_bf16_O2, - rtol=0.001, - atol=0.1) + np.testing.assert_allclose( + out_fp32, out_bf16_O1, rtol=0.001, atol=0.1 + ) + np.testing.assert_allclose( + out_fp32, out_bf16_O2, rtol=0.001, atol=0.1 + ) func_isinstance() class TestAmpWithPyLyer(unittest.TestCase): - def test_pylayer(self): - class MyMM(PyLayer): - @staticmethod def forward(ctx, a, b): ctx.save_for_backward(a, b) @@ -1356,9 +1431,7 @@ def backward(ctx, grad): class TestAmpWithHook(unittest.TestCase): - def test_hook_change_dtype(self): - def func_isinstance(): with paddle.fluid.dygraph.guard(): v = paddle.rand([3, 3]) @@ -1379,7 +1452,6 @@ def foo(grad): func_isinstance() def test_hook_change_place(self): - def func_isinstance(): with paddle.fluid.dygraph.guard(): v = paddle.rand([3, 3]) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_ir_pass_pipeline.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_ir_pass_pipeline.py index 1b445f8f9873f..7d11c03a1f177 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_ir_pass_pipeline.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_ir_pass_pipeline.py @@ -17,7 +17,6 @@ class TestPipelineWithIRPass(test_pipeline.TestPipeline): - def need_envs(self): return {'FLAGS_apply_pass_to_program': '1'} diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_mixed_precision.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_mixed_precision.py index 37fcf7c6509ba..46f12447b1d85 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_mixed_precision.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_mixed_precision.py @@ -22,7 +22,6 @@ class SimpleNet(nn.Layer): - def __init__(self, input_size, output_size): super(SimpleNet, self).__init__() self.linear1 = nn.Linear(input_size, output_size) @@ -36,16 +35,15 @@ def forward(self, x): x = self.linear1(x) # currently, paddle's relu may hide nan/inf, relu(nan) = 0, relu(inf)= inf # so, do not use it here. - #x = self.relu1(x) + # x = self.relu1(x) x = self.linear2(x) - #x = self.relu2(x) + # x = self.relu2(x) x = self.linear3(x) return x class AMPTest(unittest.TestCase): - def setUp(self): self.place = paddle.CUDAPlace(0) @@ -61,10 +59,11 @@ def net(self): loss = mse(out, label) opt = paddle.fluid.optimizer.Adam( - learning_rate=0.0001, parameter_list=model.parameters()) # 定义优化器 - opt = paddle.static.amp.decorate(opt, - init_loss_scaling=128.0, - use_dynamic_loss_scaling=True) + learning_rate=0.0001, parameter_list=model.parameters() + ) # 定义优化器 + opt = paddle.static.amp.decorate( + opt, init_loss_scaling=128.0, use_dynamic_loss_scaling=True + ) opt.minimize(loss) return model, loss, opt @@ -79,11 +78,17 @@ def test_skip_update(self): model, loss, opt = self.net() weight = model.linear1.weight moment1 = opt._optimizer._get_accumulator( - opt._optimizer._moment1_acc_str, weight) + opt._optimizer._moment1_acc_str, weight + ) beta_pow1 = opt._optimizer._get_accumulator( - opt._optimizer._beta1_pow_acc_str, weight) + opt._optimizer._beta1_pow_acc_str, weight + ) fetch_list = [ - loss, weight, moment1, beta_pow1, 'find_infinite_scale.tmp_0' + loss, + weight, + moment1, + beta_pow1, + 'find_infinite_scale.tmp_0', ] exe = paddle.static.Executor(self.place) @@ -98,20 +103,24 @@ def test_skip_update(self): ] weight_, moment1_, beta_pow1_ = exe.run( - startup_prog, fetch_list=[weight, moment1, beta_pow1]) - pre_weight_, pre_moment1_, pre_beta_pow1_ = weight_, moment1_, beta_pow1_ + startup_prog, fetch_list=[weight, moment1, beta_pow1] + ) + pre_weight_, pre_moment1_, pre_beta_pow1_ = ( + weight_, + moment1_, + beta_pow1_, + ) for i in range(nums_batch): if i % 2: train_data[i][10] = np.inf loss_, weight_, moment1_, beta_pow1_, found_inf = exe.run( main_prog, - feed={ - "X": train_data[i], - "Y": labels[i] - }, - fetch_list=fetch_list) - print(loss_, weight_[0][0], moment1_[0][0], beta_pow1_, - found_inf) + feed={"X": train_data[i], "Y": labels[i]}, + fetch_list=fetch_list, + ) + print( + loss_, weight_[0][0], moment1_[0][0], beta_pow1_, found_inf + ) if i % 2: self.assertTrue(found_inf) np.testing.assert_array_equal(weight_, pre_weight_) @@ -122,7 +131,11 @@ def test_skip_update(self): self.assertFalse(np.array_equal(weight_, pre_weight_)) self.assertFalse(np.array_equal(moment1_, pre_moment1_)) self.assertFalse(np.array_equal(beta_pow1_, pre_beta_pow1_)) - pre_weight_, pre_moment1_, pre_beta_pow1_ = weight_, moment1_, beta_pow1_ + pre_weight_, pre_moment1_, pre_beta_pow1_ = ( + weight_, + moment1_, + beta_pow1_, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_class_center_sample.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_class_center_sample.py index 0d22628c8aae5..a583afda701d7 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_class_center_sample.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_class_center_sample.py @@ -19,7 +19,6 @@ class TestParallelClassCenterSample(TestMultipleGpus): - def test_parallel_class_center_sample(self): self.run_mnist_2gpu('parallel_class_center_sample.py') self.run_mnist_2gpu('parallel_class_center_sample.py', eager_mode=False) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_control_flow.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_control_flow.py index c991818a5fbbd..9bbccc25e7343 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_control_flow.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_control_flow.py @@ -22,7 +22,6 @@ class TestDygraphControlFlowSame(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -31,14 +30,15 @@ def _setup_config(self): def test_net(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_control_flow_same.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_control_flow_same.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestFleetDygraphControlFlowSame(TestDygraphControlFlowSame): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -48,7 +48,6 @@ def _setup_config(self): class TestFleetDygraphControlFlowSameAccGrad(TestDygraphControlFlowSame): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -58,7 +57,6 @@ def _setup_config(self): class TestDygraphControlFlowDiff(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -67,14 +65,15 @@ def _setup_config(self): def test_net(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_control_flow_different.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_control_flow_different.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestFleetDygraphControlFlowDiff(TestDygraphControlFlowDiff): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -84,7 +83,6 @@ def _setup_config(self): class TestFleetDygraphControlFlowDiffAccGrad(TestDygraphControlFlowDiff): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mnist.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mnist.py index 9d5fc50872712..a0933061bce82 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mnist.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mnist.py @@ -25,7 +25,6 @@ class TestParallelDygraphMnist(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -38,14 +37,14 @@ def test_mnist(self): os.path.abspath("../../parallel_dygraph_mnist.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) -#TODO(liuyuhui): Multi-Card Baidu Kunlun XPU training exist accuracy problems -#it is difficult to find out immediately where the problem is, -#and we will work with frameworkers' help to fix it. +# TODO(liuyuhui): Multi-Card Baidu Kunlun XPU training exist accuracy problems +# it is difficult to find out immediately where the problem is, +# and we will work with frameworkers' help to fix it. class TestParallelDygraphMnistXPU(TestDistBase): - def _setup_config(self): self._sync_mode = False self._bkcl_mode = True @@ -58,18 +57,17 @@ def test_mnist_xpu(self): os.path.abspath("../../parallel_dygraph_mnist.py"), delta=1e-4, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) class TestParallelDygraphMnistSpawn(TestDistSpawnRunner): - def test_mnist_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): self.check_dist_result_with_spawn(test_class=TestMnist, delta=1e-5) class TestParallelDygraphMnistAccGrad(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -84,11 +82,11 @@ def test_mnist(self): os.path.abspath("../../parallel_dygraph_mnist.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) class TestFleetDygraphMnistXPU(TestDistBase): - def _setup_config(self): self._sync_mode = False self._bkcl_mode = True @@ -102,7 +100,8 @@ def test_mnist(self): os.path.abspath("../../parallel_dygraph_mnist.py"), delta=1e-4, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mp_layers.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mp_layers.py index 21aa82db3dc99..aeee6a4667678 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mp_layers.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_mp_layers.py @@ -19,7 +19,6 @@ class TestModelParallelLayer(TestMultipleGpus): - def test_hybrid_parallel_mp_layer(self): self.run_mnist_2gpu('hybrid_parallel_mp_layers.py') self.run_mnist_2gpu('hybrid_parallel_mp_layers.py', eager_mode=False) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync.py index 8a1c1e7a0c58d..c44af13dc4ea9 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync.py @@ -27,7 +27,6 @@ class TestParallelDygraphNoSync(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -36,14 +35,15 @@ def _setup_config(self): def test_no_sync(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_no_sync.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_no_sync.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphNoSyncUnusedParam(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -52,14 +52,15 @@ def _setup_config(self): def test_no_sync_ununsed_param(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_no_sync_unused_params.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_no_sync_unused_params.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphNoSyncControlFlow(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -68,39 +69,40 @@ def _setup_config(self): def test_no_sync_control_flow(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_no_sync_control_flow.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_no_sync_control_flow.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphNoSyncSpawn(TestDistSpawnRunner): - def test_no_sync_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): self.check_dist_result_with_spawn(test_class=TestNoSync, delta=1e-5) class TestParallelDygraphNoSyncUnusedParamSpawn(TestDistSpawnRunner): - def _args_config(self, args): args.find_unused_parameters = True def test_no_sync_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): - self.check_dist_result_with_spawn(test_class=TestNoSyncUnusedParam, - delta=1e-5) + self.check_dist_result_with_spawn( + test_class=TestNoSyncUnusedParam, delta=1e-5 + ) class TestParallelDygraphNoSyncControlFlowSpawn(TestDistSpawnRunner): - def _args_config(self, args): args.find_unused_parameters = True def test_no_sync_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): - self.check_dist_result_with_spawn(test_class=TestNoSyncControlFlow, - delta=1e-5) + self.check_dist_result_with_spawn( + test_class=TestNoSyncControlFlow, delta=1e-5 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync_gradient_check.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync_gradient_check.py index d170cd0ed54df..e6467e07815a1 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync_gradient_check.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_no_sync_gradient_check.py @@ -19,11 +19,11 @@ class TestDataParallelLayer(TestMultipleGpus): - def test_parallel_dygraph_dataparallel_no_sync(self): self.run_mnist_2gpu('parallel_dygraph_no_sync_gradient_check.py') - self.run_mnist_2gpu('parallel_dygraph_no_sync_gradient_check.py', - eager_mode=False) + self.run_mnist_2gpu( + 'parallel_dygraph_no_sync_gradient_check.py', eager_mode=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel.py index ebd1096fdc296..f45104de32c8a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel.py @@ -19,13 +19,14 @@ class TestHybridPipeParallel(TestMultipleGpus): - def test_hybrid_parallel_pp_layer(self): self.run_mnist_2gpu( - os.path.abspath('../../hybrid_parallel_pp_layer.py')) + os.path.abspath('../../hybrid_parallel_pp_layer.py') + ) self.run_mnist_2gpu( os.path.abspath('../../hybrid_parallel_pp_layer.py'), - eager_mode=False) + eager_mode=False, + ) def test_hybrid_parallel_pp_tuple_inputs(self): self.run_mnist_2gpu('hybrid_parallel_pp_embedding.py') @@ -33,8 +34,9 @@ def test_hybrid_parallel_pp_tuple_inputs(self): def test_hybrid_parallel_shared_weight(self): self.run_mnist_2gpu('hybrid_parallel_shared_weight.py') - self.run_mnist_2gpu('hybrid_parallel_shared_weight.py', - eager_mode=False) + self.run_mnist_2gpu( + 'hybrid_parallel_shared_weight.py', eager_mode=False + ) def test_pipeline_parallel_amp(self): self.run_mnist_2gpu('hybrid_parallel_pp_amp.py') @@ -46,8 +48,9 @@ def test_pipeline_parallel_fp16(self): def test_hybrid_parallel_transformer(self): self.run_mnist_2gpu('hybrid_parallel_pp_transformer.py') - self.run_mnist_2gpu('hybrid_parallel_pp_transformer.py', - eager_mode=False) + self.run_mnist_2gpu( + 'hybrid_parallel_pp_transformer.py', eager_mode=False + ) def test_hybrid_parallel_save_load(self): self.run_mnist_2gpu('hybrid_parallel_pp_save_load.py') diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_with_virtual_stage.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_with_virtual_stage.py index 7af7d6dba1a07..d09daa0764cb0 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_with_virtual_stage.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_pipeline_parallel_with_virtual_stage.py @@ -19,17 +19,18 @@ class TestHybridPipeParallelWithVirtualStage(TestMultipleGpus): - def test_hybrid_parallel_pp_layer_with_virtual_stage(self): self.run_mnist_2gpu('hybrid_parallel_pp_layer_with_virtual_stage.py') def test_hybrid_parallel_pp_transformer_with_virtual_stage(self): self.run_mnist_2gpu( - 'hybrid_parallel_pp_transformer_with_virtual_stage.py') + 'hybrid_parallel_pp_transformer_with_virtual_stage.py' + ) def test_hybrid_parallel_save_load_with_virtual_stage(self): self.run_mnist_2gpu( - 'hybrid_parallel_pp_save_load_with_virtual_stage.py') + 'hybrid_parallel_pp_save_load_with_virtual_stage.py' + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_qat.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_qat.py index 842f716611834..4ea7ebc9162b5 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_qat.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_qat.py @@ -19,7 +19,12 @@ import os import subprocess -from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc +from paddle.distributed.utils.launch_utils import ( + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, +) def get_cluster_from_args(selected_gpus): @@ -47,17 +52,19 @@ def get_gpus(selected_gpus): return selected_gpus -def start_local_trainers(cluster, - pod, - training_script, - training_script_args, - eager_mode=True, - log_dir=None): +def start_local_trainers( + cluster, + pod, + training_script, + training_script_args, + eager_mode=True, + log_dir=None, +): current_env = copy.copy(os.environ.copy()) - #paddle broadcast ncclUniqueId use socket, and - #proxy maybe make trainers unreachable, so delete them. - #if we set them to "", grpc will log error message "bad uri" - #so just delete them. + # paddle broadcast ncclUniqueId use socket, and + # proxy maybe make trainers unreachable, so delete them. + # if we set them to "", grpc will log error message "bad uri" + # so just delete them. current_env.pop("http_proxy", None) current_env.pop("https_proxy", None) @@ -68,7 +75,7 @@ def start_local_trainers(cluster, "PADDLE_TRAINER_ID": "%d" % t.rank, "PADDLE_CURRENT_ENDPOINT": "%s" % t.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), - "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()) + "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } if not eager_mode: @@ -101,10 +108,11 @@ def start_local_trainers(cluster, class TestMultipleGpus(unittest.TestCase): - def run_2gpu(self, target_file_name, eager_mode=True): - if not fluid.core.is_compiled_with_cuda( - ) or fluid.core.get_cuda_device_count() == 0: + if ( + not fluid.core.is_compiled_with_cuda() + or fluid.core.get_cuda_device_count() == 0 + ): return selected_gpus = get_gpus('0,1') @@ -113,11 +121,13 @@ def run_2gpu(self, target_file_name, eager_mode=True): cluster, pod = get_cluster_from_args(selected_gpus) - procs = start_local_trainers(cluster, - pod, - eager_mode=eager_mode, - training_script=target_file_name, - training_script_args=[]) + procs = start_local_trainers( + cluster, + pod, + eager_mode=eager_mode, + training_script=target_file_name, + training_script_args=[], + ) while True: alive = watch_local_trainers(procs, cluster.trainers_endpoints()) @@ -129,7 +139,6 @@ def run_2gpu(self, target_file_name, eager_mode=True): class TestDataParallelQAT(TestMultipleGpus): - def test_multiple_gpus_qat(self): self.run_2gpu('hybrid_parallel_qat.py') diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_se_resnext.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_se_resnext.py index 0bdbb20787e2d..68fb9468f4baa 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_se_resnext.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_se_resnext.py @@ -25,7 +25,6 @@ class TestParallelDygraphSeResNeXt(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -33,18 +32,20 @@ def _setup_config(self): def test_se_resnext(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_se_resnext.py", - delta=0.01, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_se_resnext.py", + delta=0.01, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphSeResNeXtSpawn(TestDistSpawnRunner): - def test_se_resnext_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): - self.check_dist_result_with_spawn(test_class=TestSeResNeXt, - delta=0.01) + self.check_dist_result_with_spawn( + test_class=TestSeResNeXt, delta=0.01 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sharding_parallel.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sharding_parallel.py index f6c6e3378e43c..d0b17e8625770 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sharding_parallel.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sharding_parallel.py @@ -23,8 +23,9 @@ class TestHybridParallel(TestMultipleGpus): # check sharding logic as well as the accuracy with single mode def test_hybrid_parallel_sharding_logic(self): self.run_mnist_2gpu('hybrid_parallel_sharding_model.py') - self.run_mnist_2gpu('hybrid_parallel_sharding_model.py', - eager_mode=False) + self.run_mnist_2gpu( + 'hybrid_parallel_sharding_model.py', eager_mode=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding.py index 2f8ea8feddd2e..b1d304350cf64 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding.py @@ -25,7 +25,6 @@ class TestParallelDygraphSparseEmdedding(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -37,11 +36,11 @@ def test_sparse_embedding(self): os.path.abspath("../../parallel_dygraph_sparse_embedding.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) class TestParallelDygraphSparseEmdeddingFP64(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -49,19 +48,22 @@ def _setup_config(self): def test_sparse_embedding_fp64(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place(os.path.abspath( - "../../parallel_dygraph_sparse_embedding_fp64.py"), - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + os.path.abspath( + "../../parallel_dygraph_sparse_embedding_fp64.py" + ), + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphSparseEmdeddingSpawn(TestDistSpawnRunner): - def test_sparse_embedding_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): - self.check_dist_result_with_spawn(test_class=TestSparseEmbedding, - delta=1e-5) + self.check_dist_result_with_spawn( + test_class=TestSparseEmbedding, delta=1e-5 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding_over_height.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding_over_height.py index b082da1b910d9..b82bce4e1e19a 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding_over_height.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sparse_embedding_over_height.py @@ -19,13 +19,14 @@ import paddle.fluid as fluid from test_dist_base import TestDistBase from spawn_runner_base import TestDistSpawnRunner -from parallel_dygraph_sparse_embedding_over_height import TestSparseEmbeddingOverHeight +from parallel_dygraph_sparse_embedding_over_height import ( + TestSparseEmbeddingOverHeight, +) flag_name = os.path.splitext(__file__)[0] class TestParallelDygraphSparseEmdeddingOverHeight(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -33,19 +34,22 @@ def _setup_config(self): def test_sparse_embedding(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place(os.path.abspath( - "../../parallel_dygraph_sparse_embedding_over_height.py"), - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + os.path.abspath( + "../../parallel_dygraph_sparse_embedding_over_height.py" + ), + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphSparseEmdeddingOverHeightSpawn(TestDistSpawnRunner): - def test_sparse_embedding_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): self.check_dist_result_with_spawn( - test_class=TestSparseEmbeddingOverHeight, delta=1e-5) + test_class=TestSparseEmbeddingOverHeight, delta=1e-5 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sync_batch_norm.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sync_batch_norm.py index 26e62bc75ec94..1dd0ae278495b 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sync_batch_norm.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_sync_batch_norm.py @@ -22,7 +22,6 @@ class TestParallelDygraphMnist(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -30,10 +29,12 @@ def _setup_config(self): def test_mnist(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_sync_batch_norm.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_sync_batch_norm.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_tensor_parallel.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_tensor_parallel.py index c2b1b17fc52cf..c01d6ed6f080d 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_tensor_parallel.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_tensor_parallel.py @@ -19,7 +19,6 @@ class TestHybridParallel(TestMultipleGpus): - def test_hybrid_parallel_mp_random(self): self.run_mnist_2gpu('hybrid_parallel_mp_random.py') self.run_mnist_2gpu('hybrid_parallel_mp_random.py', eager_mode=False) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_transformer.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_transformer.py index 8d12dfdce5891..58ad40894fefc 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_transformer.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_transformer.py @@ -22,7 +22,6 @@ class TestParallelDygraphTransformer(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -30,14 +29,15 @@ def _setup_config(self): def test_transformer(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_transformer.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_transformer.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphTransformerAccGrad(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -47,10 +47,12 @@ def _setup_config(self): def test_transformer(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_transformer.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_transformer.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_unused_variables.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_unused_variables.py index 0a740b13c4edd..a36eae2ccda01 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_unused_variables.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_dygraph_unused_variables.py @@ -25,7 +25,6 @@ class TestParallelDygraphUnusedVar(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -37,11 +36,11 @@ def test_net(self): os.path.abspath("../../parallel_dygraph_unused_variables.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) class TestFleetDygraphUnusedVar(TestParallelDygraphUnusedVar): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -50,15 +49,14 @@ def _setup_config(self): class TestSparseEmbeddingUnusedVarsSpawn(TestDistSpawnRunner): - def test_mnist_with_spawn(self): if fluid.core.is_compiled_with_cuda() and sys.version_info >= (3, 4): self.check_dist_result_with_spawn( - test_class=TestSparseEmbeddingUnusedVars, delta=1e-5) + test_class=TestSparseEmbeddingUnusedVars, delta=1e-5 + ) class TestParallelDygraphNoVar(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -70,11 +68,11 @@ def test_net(self): os.path.abspath("../../parallel_dygraph_none_var.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) class TestParallelDygraphSharedUnusedVariables(TestDistBase): - def _setup_config(self): self._sync_mode = False self._nccl2_mode = True @@ -86,7 +84,8 @@ def test_mnist(self): os.path.abspath("../../parallel_dygraph_shared_unused_var.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_margin_cross_entropy.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_margin_cross_entropy.py index b983be50f7309..b4f7cfb39d838 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_margin_cross_entropy.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_parallel_margin_cross_entropy.py @@ -19,11 +19,11 @@ class TestParallelMarginSoftmaxWithCrossEntropy(TestMultipleGpus): - def test_parallel_margin_cross_entropy(self): self.run_mnist_2gpu('parallel_margin_cross_entropy.py') - self.run_mnist_2gpu('parallel_margin_cross_entropy.py', - eager_mode=False) + self.run_mnist_2gpu( + 'parallel_margin_cross_entropy.py', eager_mode=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_pipeline.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_pipeline.py index e9fd934f2869d..a495ac13c77f0 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_pipeline.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_pipeline.py @@ -24,7 +24,6 @@ class TestPipeline(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -41,19 +40,23 @@ def test_dist_train(self): # Now pipeline only gets the loss value of the last # microbatch, so it is not consistable with the # non-pipeline one. - self.check_with_place("pipeline_mnist.py", - delta=1e0, - check_error_log=True, - log_name=flag_name, - need_envs=self.need_envs()) + self.check_with_place( + "pipeline_mnist.py", + delta=1e0, + check_error_log=True, + log_name=flag_name, + need_envs=self.need_envs(), + ) def test_dist_train_multi_device(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("pipeline_mnist_multi_device.py", - check_error_log=True, - delta=1e0, - log_name=flag_name, - need_envs=self.need_envs()) + self.check_with_place( + "pipeline_mnist_multi_device.py", + check_error_log=True, + delta=1e0, + log_name=flag_name, + need_envs=self.need_envs(), + ) def test_dist_train_one_device(self): if fluid.core.is_compiled_with_cuda(): @@ -61,7 +64,8 @@ def test_dist_train_one_device(self): "pipeline_mnist_one_device.py", check_error_log=True, log_name=flag_name, - need_envs={"PADDLE_MANUAL_PIPELINE_STAGE": "0"}) + need_envs={"PADDLE_MANUAL_PIPELINE_STAGE": "0"}, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_recv_save_op.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_recv_save_op.py index 927b02d74eb3e..c75aadd450a75 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_recv_save_op.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_recv_save_op.py @@ -46,20 +46,18 @@ def run_pserver(pserver_id): param.set(param_array, place) optimize_block = program._create_block(program.global_block().idx) - program.global_block().append_op(type="listen_and_serv", - inputs={'X': []}, - outputs={}, - attrs={ - "optimize_blocks": - [optimize_block], - "endpoint": - '127.0.0.1:0', - "Fanin": - 1, - "distributed_mode": - DistributedMode.SYNC, - "grad_to_block_id": [] - }) + program.global_block().append_op( + type="listen_and_serv", + inputs={'X': []}, + outputs={}, + attrs={ + "optimize_blocks": [optimize_block], + "endpoint": '127.0.0.1:0', + "Fanin": 1, + "distributed_mode": DistributedMode.SYNC, + "grad_to_block_id": [], + }, + ) exe = fluid.Executor(place) exe.run(program) @@ -67,12 +65,11 @@ def run_pserver(pserver_id): @unittest.skip("do not need currently") class TestListenAndServOp(unittest.TestCase): - def setUp(self): self.ps_timeout = 5 def _start_pserver(self, pserver_id, pserver_func): - p = Process(target=pserver_func, args=(pserver_id, )) + p = Process(target=pserver_func, args=(pserver_id,)) p.daemon = True p.start() return p @@ -104,15 +101,17 @@ def _run_nce_op_two_pserver(self, place, port0, port1, model_file): emaps = ['127.0.0.1:' + str(port0), '127.0.0.1:' + str(port1)] # create and run recv and save operator - remote_recv_op = Operator("recv_save", - trainer_id=0, - shape=[10, 8], - slice_shapes=["5,8", "5,8"], - slice_varnames=["table", "table"], - remote_varnames=['table', 'table'], - is_sparse=False, - endpoints=emaps, - file_path=model_file) + remote_recv_op = Operator( + "recv_save", + trainer_id=0, + shape=[10, 8], + slice_shapes=["5,8", "5,8"], + slice_varnames=["table", "table"], + remote_varnames=['table', 'table'], + is_sparse=False, + endpoints=emaps, + file_path=model_file, + ) remote_recv_op.run(scope, place) @@ -125,44 +124,53 @@ def _load_slice_var(self, model_file): type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[10, 8], dtype="float32", - persistable=True) + persistable=True, + ) slice0 = load_block.create_var( name="var.slice0", type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[3, 8], dtype="float32", - persistable=True) + persistable=True, + ) slice1 = load_block.create_var( name="var.slice1", type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[5, 8], dtype="float32", - persistable=True) - - load_block.append_op(type='load', - inputs={}, - outputs={'Out': [origin]}, - attrs={'file_path': model_file}) - - load_block.append_op(type='load', - inputs={}, - outputs={'Out': [slice0]}, - attrs={ - 'file_path': model_file, - 'seek': 2 * 8, - 'shape': slice0.shape - }) - - load_block.append_op(type='load', - inputs={}, - outputs={'Out': [slice1]}, - attrs={ - 'file_path': model_file, - 'seek': 5 * 8, - 'shape': slice1.shape - }) + persistable=True, + ) + + load_block.append_op( + type='load', + inputs={}, + outputs={'Out': [origin]}, + attrs={'file_path': model_file}, + ) + + load_block.append_op( + type='load', + inputs={}, + outputs={'Out': [slice0]}, + attrs={ + 'file_path': model_file, + 'seek': 2 * 8, + 'shape': slice0.shape, + }, + ) + + load_block.append_op( + type='load', + inputs={}, + outputs={'Out': [slice1]}, + attrs={ + 'file_path': model_file, + 'seek': 5 * 8, + 'shape': slice1.shape, + }, + ) exe = fluid.Executor(place=fluid.CPUPlace()) exe.run(load_prog) @@ -179,8 +187,9 @@ def _load_slice_var(self, model_file): np.testing.assert_equal(origin[5:10], slice1) def _save_by_io_persistables(self, place, port0, port1, dirname, var_name): - self._run_nce_op_two_pserver(place, port0, port1, - os.path.join(dirname, var_name)) + self._run_nce_op_two_pserver( + place, port0, port1, os.path.join(dirname, var_name) + ) def test_recv_save_op_remote(self): # run pserver on CPU in sync mode @@ -198,8 +207,9 @@ def test_recv_save_op_remote(self): param_name = "table" for place in places: - self._save_by_io_persistables(place, port0, port1, param_dir, - param_name) + self._save_by_io_persistables( + place, port0, port1, param_dir, param_name + ) # raise SIGTERM to pserver os.kill(p0.pid, signal.SIGINT) diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_rnn_dp.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_rnn_dp.py index 8976f7919d294..9d116d74b257c 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_rnn_dp.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_rnn_dp.py @@ -24,27 +24,30 @@ class RNNEncoder(nn.Layer): - - def __init__(self, - input_size, - hidden_size, - num_layers=1, - direction="forward", - dropout=0.0, - pooling_type=None, - **kwargs): + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + direction="forward", + dropout=0.0, + pooling_type=None, + **kwargs + ): super().__init__() self._input_size = input_size self._hidden_size = hidden_size self._direction = direction self._pooling_type = pooling_type - self.rnn_layer = nn.SimpleRNN(input_size=input_size, - hidden_size=hidden_size, - num_layers=num_layers, - direction=direction, - dropout=dropout, - **kwargs) + self.rnn_layer = nn.SimpleRNN( + input_size=input_size, + hidden_size=hidden_size, + num_layers=num_layers, + direction=direction, + dropout=dropout, + **kwargs + ) def get_input_dim(self): return self._input_size @@ -57,34 +60,40 @@ def get_output_dim(self): def forward(self, inputs, sequence_length): encoded_text, last_hidden = self.rnn_layer( - inputs, sequence_length=sequence_length) + inputs, sequence_length=sequence_length + ) output = paddle.max(encoded_text, axis=1) return output class RNNModel(nn.Layer): - - def __init__(self, - vocab_size, - num_classes, - emb_dim=128, - padding_idx=0, - rnn_hidden_size=198, - direction='forward', - rnn_layers=1, - dropout_rate=0.0, - pooling_type=None, - fc_hidden_size=96): + def __init__( + self, + vocab_size, + num_classes, + emb_dim=128, + padding_idx=0, + rnn_hidden_size=198, + direction='forward', + rnn_layers=1, + dropout_rate=0.0, + pooling_type=None, + fc_hidden_size=96, + ): super().__init__() - self.embedder = nn.Embedding(num_embeddings=vocab_size, - embedding_dim=emb_dim, - padding_idx=padding_idx) - self.rnn_encoder = RNNEncoder(emb_dim, - rnn_hidden_size, - num_layers=rnn_layers, - direction=direction, - dropout=dropout_rate, - pooling_type=pooling_type) + self.embedder = nn.Embedding( + num_embeddings=vocab_size, + embedding_dim=emb_dim, + padding_idx=padding_idx, + ) + self.rnn_encoder = RNNEncoder( + emb_dim, + rnn_hidden_size, + num_layers=rnn_layers, + direction=direction, + dropout=dropout_rate, + pooling_type=pooling_type, + ) self.fc = nn.Linear(self.rnn_encoder.get_output_dim(), fc_hidden_size) self.output_layer = nn.Linear(fc_hidden_size, num_classes) @@ -97,26 +106,30 @@ def forward(self, text, seq_len): def rnn_pretrain_forward(train_program, start_program, topo=None): - with static.program_guard(train_program, - start_program), paddle.utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), paddle.utils.unique_name.guard(): batch_size = 1 - tokens = static.data(name="tokens", - shape=[batch_size, -1], - dtype="int64") + tokens = static.data( + name="tokens", shape=[batch_size, -1], dtype="int64" + ) seq_len = static.data(name="ids", shape=[batch_size], dtype="int64") labels = static.data(name="labels", shape=[batch_size], dtype="int64") data_holders = [tokens, seq_len, labels] vocab_size = 10 num_classes = 2 pad_token_id = 0 - model = RNNModel(vocab_size, - num_classes, - direction='forward', - padding_idx=pad_token_id, - pooling_type='max') - - optimizer = paddle.optimizer.Adam(parameters=model.parameters(), - learning_rate=0.001) + model = RNNModel( + vocab_size, + num_classes, + direction='forward', + padding_idx=pad_token_id, + pooling_type='max', + ) + + optimizer = paddle.optimizer.Adam( + parameters=model.parameters(), learning_rate=0.001 + ) criterion = paddle.nn.CrossEntropyLoss() preds = model(tokens, seq_len) loss = criterion(preds, labels) @@ -125,23 +138,30 @@ def rnn_pretrain_forward(train_program, start_program, topo=None): class TestFleetMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "1" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002" def test_rnn_raw_optimizer(self): import paddle.distributed.fleet as fleet import paddle.distributed.fleet.base.role_maker as role_maker + role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) train_program = static.Program() start_program = static.Program() - train_program, start_program, loss, optimizer, data_holders = \ - rnn_pretrain_forward(train_program, start_program) + ( + train_program, + start_program, + loss, + optimizer, + data_holders, + ) = rnn_pretrain_forward(train_program, start_program) with paddle.static.program_guard( - train_program, start_program), paddle.utils.unique_name.guard(): + train_program, start_program + ), paddle.utils.unique_name.guard(): strategy = fleet.DistributedStrategy() strategy.without_graph_optimization = True strategy.fuse_all_reduce_ops = True diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_static_model_parallel.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_static_model_parallel.py index 06a9b05682270..6f51af2795281 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_static_model_parallel.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_static_model_parallel.py @@ -23,7 +23,6 @@ class TestStaticModelParallel(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -33,27 +32,36 @@ def _setup_config(self): def test_dist_static_model_parallel(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("static_model_parallel_by_row.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "static_model_parallel_by_row.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) def test_dist_static_model_parallel2(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("static_model_parallel_by_col.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "static_model_parallel_by_col.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) def test_dist_static_model_parallel3(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("static_model_parallel_embedding.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "static_model_parallel_embedding.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/test_tcp_store.py b/python/paddle/fluid/tests/unittests/collective/fleet/test_tcp_store.py index ff5cff2c9cb9f..01a39bc6241db 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/test_tcp_store.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/test_tcp_store.py @@ -18,7 +18,6 @@ class TestTCPStore(unittest.TestCase): - def test_tcp_store(self): dist_port = int(os.getenv("PADDLE_DIST_UT_PORT", 6170)) print("get dist_port:", dist_port) diff --git a/python/paddle/fluid/tests/unittests/collective/init_process_group.py b/python/paddle/fluid/tests/unittests/collective/init_process_group.py index dfb9dcf556c21..f45e4004831ef 100644 --- a/python/paddle/fluid/tests/unittests/collective/init_process_group.py +++ b/python/paddle/fluid/tests/unittests/collective/init_process_group.py @@ -19,7 +19,6 @@ class TestProcessGroupFp32(unittest.TestCase): - def setUp(self): self.config() diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/common.py b/python/paddle/fluid/tests/unittests/collective/multinode/common.py index cb2ff2ea09e6c..7849b6575c088 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/common.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/common.py @@ -17,8 +17,8 @@ def init_parallel_env(mode, global_batch_size, seed=1024): ''' - Args: - mode:(str) DP1-MP1-PP1-SH1-O1 + Args: + mode:(str) DP1-MP1-PP1-SH1-O1 ''' def parse_mode(mode): @@ -42,7 +42,7 @@ def parse_mode(mode): "dp_degree": DP, "mp_degree": MP, "pp_degree": PP, - "sharding_degree": SH + "sharding_degree": SH, } accumulate_steps = 1 @@ -50,7 +50,7 @@ def parse_mode(mode): if PP > 1: strategy.pipeline_configs = { "accumulate_steps": accumulate_steps, - "micro_batch_size": global_batch_size // DP // accumulate_steps + "micro_batch_size": global_batch_size // DP // accumulate_steps, } # set control in tensor parallel diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dp.py b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dp.py index 5583041db8b67..bb904b3d6bd8e 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dp.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dp.py @@ -12,31 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -from test_collective_multi_nodes import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_multi_nodes import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) class TestDygrapgHybridDP(TestCollectiveAPIRunnerBase): - def __init__(self): pass def check_pass(self, *args, **kwargs): from common import init_parallel_env import paddle + hcg = init_parallel_env("DP16-MP1-PP1-SH1-O1", 2) import numpy as np + dp_group = hcg.get_data_parallel_group() np.random.seed(1024) data = np.random.random((10 * dp_group.nranks, 100)).reshape( - (dp_group.nranks, -1, 100)) + (dp_group.nranks, -1, 100) + ) data_part = paddle.to_tensor(data[dp_group.rank]) paddle.distributed.collective.all_reduce(data_part) data_reduced = data_part data_sumed = np.sum(data, axis=0) - assert np.allclose(data_sumed, - data_reduced.numpy(), - rtol=1e-8, - atol=1e-8) + assert np.allclose( + data_sumed, data_reduced.numpy(), rtol=1e-8, atol=1e-8 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dpppmp.py b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dpppmp.py index 6de0954b0e7e4..055e3e24fbd00 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dpppmp.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_dpppmp.py @@ -15,7 +15,10 @@ import numpy as np import paddle import paddle.distributed.fleet as fleet -from test_collective_multi_nodes import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_multi_nodes import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) from paddle import nn import numpy as np @@ -28,15 +31,14 @@ def weight_init(mp, shape, col=True, seed=1024): else: if col: step = shape[1] // mp.nranks - _w = w[:, mp.rank * step:mp.rank * step + step] + _w = w[:, mp.rank * step : mp.rank * step + step] else: step = shape[0] // mp.nranks - _w = w[mp.rank * step:mp.rank * step + step, :] + _w = w[mp.rank * step : mp.rank * step + step, :] return paddle.fluid.initializer.NumpyArrayInitializer(_w) class Criterion(nn.Layer): - def __init__(self): super(Criterion, self).__init__() self.loss_func = nn.MSELoss(reduction="mean") @@ -47,7 +49,6 @@ def forward(self, pred, label): class ModelPipeline(fleet.meta_parallel.PipelineLayer): - def __init__(self, hcg): paddle.seed(1024) dp_linear = nn.Linear(32, 128) @@ -62,35 +63,38 @@ def __init__(self, hcg): 512, weight_attr=weight_init(mp, (128, 512), True, 1204 + i), has_bias=True, - gather_output=False) + gather_output=False, + ) mp_linear_2 = fleet.meta_parallel.RowParallelLinear( 512, 128, weight_attr=weight_init(mp, (512, 128), False, 2012 + i), has_bias=True, - input_is_parallel=True) + input_is_parallel=True, + ) else: - mp_linear_1 = nn.Linear(128, - 512, - weight_attr=weight_init( - None, (128, 512), True, 1204 + i)) - mp_linear_2 = nn.Linear(512, - 128, - weight_attr=weight_init( - None, (512, 128), True, 2012 + i)) + mp_linear_1 = nn.Linear( + 128, + 512, + weight_attr=weight_init(None, (128, 512), True, 1204 + i), + ) + mp_linear_2 = nn.Linear( + 512, + 128, + weight_attr=weight_init(None, (512, 128), True, 2012 + i), + ) act = nn.ReLU6() layer_seq = nn.Sequential(mp_linear_1, mp_linear_2, act) self.layers_pp.append(layer_seq) out = nn.Linear(128, 32) self.layers_pp.append(out) - super(ModelPipeline, self).__init__(layers=self.layers_pp, - loss_fn=Criterion(), - topology=self.topology) + super(ModelPipeline, self).__init__( + layers=self.layers_pp, loss_fn=Criterion(), topology=self.topology + ) class Model(nn.Layer): - def __init__(self, hcg): super(Model, self).__init__() paddle.seed(1024) @@ -105,22 +109,26 @@ def __init__(self, hcg): 512, weight_attr=weight_init(mp, (128, 512), True, 1204 + i), has_bias=True, - gather_output=False) + gather_output=False, + ) mp_linear_2 = fleet.meta_parallel.RowParallelLinear( 512, 128, weight_attr=weight_init(mp, (512, 128), False, 2012 + i), has_bias=True, - input_is_parallel=True) + input_is_parallel=True, + ) else: - mp_linear_1 = nn.Linear(128, - 512, - weight_attr=weight_init( - None, (128, 512), True, 1204 + i)) - mp_linear_2 = nn.Linear(512, - 128, - weight_attr=weight_init( - None, (512, 128), True, 2012 + i)) + mp_linear_1 = nn.Linear( + 128, + 512, + weight_attr=weight_init(None, (128, 512), True, 1204 + i), + ) + mp_linear_2 = nn.Linear( + 512, + 128, + weight_attr=weight_init(None, (512, 128), True, 2012 + i), + ) act = nn.ReLU6() layer_seq = nn.Sequential(mp_linear_1, mp_linear_2, act) self.layers_pp.append(layer_seq) @@ -134,7 +142,6 @@ def forward(self, x): class TestDygrapgHybridDPPPMP(TestCollectiveAPIRunnerBase): - def __init__(self): pass @@ -143,9 +150,11 @@ def check_pass(self, *args, **kwargs): from common import init_parallel_env import paddle from paddle.distributed import fleet + hcg = init_parallel_env("DP4-MP2-PP2-SH1-O1", 64) pp_degree = hcg.get_pipe_parallel_world_size() import numpy as np + crit = Criterion() if pp_degree <= 1: model = Model(hcg) @@ -154,10 +163,12 @@ def check_pass(self, *args, **kwargs): model_base = Model(None) - optimizer = paddle.optimizer.Adam(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=0.01, parameters=model.parameters() + ) optimizer_base = paddle.optimizer.Adam( - learning_rate=0.01, parameters=model_base.parameters()) + learning_rate=0.01, parameters=model_base.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_fp16.py b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_fp16.py index 0c4eddffeecec..04eacbc7fc445 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_fp16.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_fp16.py @@ -15,7 +15,10 @@ import numpy as np import paddle import paddle.distributed.fleet as fleet -from test_collective_multi_nodes import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_multi_nodes import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) from paddle import nn import numpy as np @@ -28,15 +31,14 @@ def weight_init(mp, shape, col=True, seed=1024): else: if col: step = shape[1] // mp.nranks - _w = w[:, mp.rank * step:mp.rank * step + step] + _w = w[:, mp.rank * step : mp.rank * step + step] else: step = shape[0] // mp.nranks - _w = w[mp.rank * step:mp.rank * step + step, :] + _w = w[mp.rank * step : mp.rank * step + step, :] return paddle.fluid.initializer.NumpyArrayInitializer(_w) class Criterion(nn.Layer): - def __init__(self): super(Criterion, self).__init__() self.loss_func = nn.MSELoss(reduction="mean") @@ -47,7 +49,6 @@ def forward(self, pred, label): class ModelPipeline(fleet.meta_parallel.PipelineLayer): - def __init__(self, hcg): paddle.seed(1024) dp_linear = nn.Linear(32, 128) @@ -62,35 +63,38 @@ def __init__(self, hcg): 512, weight_attr=weight_init(mp, (128, 512), True, 1204 + i), has_bias=True, - gather_output=False) + gather_output=False, + ) mp_linear_2 = fleet.meta_parallel.RowParallelLinear( 512, 128, weight_attr=weight_init(mp, (512, 128), False, 2012 + i), has_bias=True, - input_is_parallel=True) + input_is_parallel=True, + ) else: - mp_linear_1 = nn.Linear(128, - 512, - weight_attr=weight_init( - None, (128, 512), True, 1204 + i)) - mp_linear_2 = nn.Linear(512, - 128, - weight_attr=weight_init( - None, (512, 128), True, 2012 + i)) + mp_linear_1 = nn.Linear( + 128, + 512, + weight_attr=weight_init(None, (128, 512), True, 1204 + i), + ) + mp_linear_2 = nn.Linear( + 512, + 128, + weight_attr=weight_init(None, (512, 128), True, 2012 + i), + ) act = nn.ReLU6() layer_seq = nn.Sequential(mp_linear_1, mp_linear_2, act) self.layers_pp.append(layer_seq) out = nn.Linear(128, 32) self.layers_pp.append(out) - super(ModelPipeline, self).__init__(layers=self.layers_pp, - loss_fn=Criterion(), - topology=self.topology) + super(ModelPipeline, self).__init__( + layers=self.layers_pp, loss_fn=Criterion(), topology=self.topology + ) class Model(nn.Layer): - def __init__(self, hcg): super(Model, self).__init__() paddle.seed(1024) @@ -105,22 +109,26 @@ def __init__(self, hcg): 512, weight_attr=weight_init(mp, (128, 512), True, 1204 + i), has_bias=True, - gather_output=False) + gather_output=False, + ) mp_linear_2 = fleet.meta_parallel.RowParallelLinear( 512, 128, weight_attr=weight_init(mp, (512, 128), False, 2012 + i), has_bias=True, - input_is_parallel=True) + input_is_parallel=True, + ) else: - mp_linear_1 = nn.Linear(128, - 512, - weight_attr=weight_init( - None, (128, 512), True, 1204 + i)) - mp_linear_2 = nn.Linear(512, - 128, - weight_attr=weight_init( - None, (512, 128), True, 2012 + i)) + mp_linear_1 = nn.Linear( + 128, + 512, + weight_attr=weight_init(None, (128, 512), True, 1204 + i), + ) + mp_linear_2 = nn.Linear( + 512, + 128, + weight_attr=weight_init(None, (512, 128), True, 2012 + i), + ) act = nn.ReLU6() layer_seq = nn.Sequential(mp_linear_1, mp_linear_2, act) self.layers_pp.append(layer_seq) @@ -134,7 +142,6 @@ def forward(self, x): class TestDygraphHybridFp16(TestCollectiveAPIRunnerBase): - def __init__(self): pass @@ -143,9 +150,11 @@ def check_pass(self, *args, **kwargs): from common import init_parallel_env import paddle from paddle.distributed import fleet + hcg = init_parallel_env("DP4-MP2-PP2-SH1-O1", 64) pp_degree = hcg.get_pipe_parallel_world_size() import numpy as np + crit = Criterion() if pp_degree <= 1: model = Model(hcg) @@ -154,17 +163,20 @@ def check_pass(self, *args, **kwargs): model_base = Model(None) - optimizer = paddle.optimizer.Adam(learning_rate=0.01, - parameters=model.parameters(), - multi_precision=True) + optimizer = paddle.optimizer.Adam( + learning_rate=0.01, + parameters=model.parameters(), + multi_precision=True, + ) optimizer_base = paddle.optimizer.Adam( - learning_rate=0.01, parameters=model_base.parameters()) + learning_rate=0.01, parameters=model_base.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=4096) scaler = fleet.distributed_scaler(scaler) - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) @@ -177,9 +189,9 @@ def check_pass(self, *args, **kwargs): for _ in range(2): if pp_degree > 1: with paddle.amp.auto_cast(True, level='O2'): - loss = model.train_batch([x, y], - optimizer=optimizer, - scaler=scaler) + loss = model.train_batch( + [x, y], optimizer=optimizer, scaler=scaler + ) else: with paddle.amp.auto_cast(True, level='O2'): output = model(x) diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_recompute.py b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_recompute.py index 16e6d1602f39e..f0441f4eab838 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_recompute.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/dygraph_hybrid_recompute.py @@ -15,7 +15,10 @@ import numpy as np import paddle import paddle.distributed.fleet as fleet -from test_collective_multi_nodes import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_multi_nodes import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) from paddle import nn import numpy as np @@ -30,15 +33,14 @@ def weight_init(mp, shape, col=True, seed=1024): else: if col: step = shape[1] // mp.nranks - _w = w[:, mp.rank * step:mp.rank * step + step] + _w = w[:, mp.rank * step : mp.rank * step + step] else: step = shape[0] // mp.nranks - _w = w[mp.rank * step:mp.rank * step + step, :] + _w = w[mp.rank * step : mp.rank * step + step, :] return paddle.fluid.initializer.NumpyArrayInitializer(_w) class Criterion(nn.Layer): - def __init__(self): super(Criterion, self).__init__() self.loss_func = nn.MSELoss(reduction="mean") @@ -49,7 +51,6 @@ def forward(self, pred, label): class RecomputeMatmulBlock(nn.Layer): - def __init__(self, mp, seed, m, n, k): super(RecomputeMatmulBlock, self).__init__() self.mp = mp @@ -59,22 +60,22 @@ def __init__(self, mp, seed, m, n, k): n, weight_attr=weight_init(mp, (m, n), True, seed), has_bias=True, - gather_output=False) + gather_output=False, + ) mp_linear_2 = fleet.meta_parallel.RowParallelLinear( n, k, weight_attr=weight_init(mp, (n, k), False, seed + 1), has_bias=True, - input_is_parallel=True) + input_is_parallel=True, + ) else: - mp_linear_1 = nn.Linear(m, - n, - weight_attr=weight_init( - None, (m, n), True, seed)) - mp_linear_2 = nn.Linear(n, - k, - weight_attr=weight_init( - None, (n, k), True, seed + 1)) + mp_linear_1 = nn.Linear( + m, n, weight_attr=weight_init(None, (m, n), True, seed) + ) + mp_linear_2 = nn.Linear( + n, k, weight_attr=weight_init(None, (n, k), True, seed + 1) + ) self.layers = nn.Sequential(mp_linear_1, mp_linear_2) def forward(self, x): @@ -88,7 +89,6 @@ def forward(self, x): class ModelPipeline(fleet.meta_parallel.PipelineLayer): - def __init__(self, hcg): paddle.seed(1024) dp_linear = nn.Linear(32, 64) @@ -104,13 +104,12 @@ def __init__(self, hcg): out = nn.Linear(64, 32) self.layers_pp.append(out) - super(ModelPipeline, self).__init__(layers=self.layers_pp, - loss_fn=Criterion(), - topology=self.topology) + super(ModelPipeline, self).__init__( + layers=self.layers_pp, loss_fn=Criterion(), topology=self.topology + ) class Model(nn.Layer): - def __init__(self, hcg): super(Model, self).__init__() paddle.seed(1024) @@ -133,7 +132,6 @@ def forward(self, x): class TestDygrapgHybridRecompute(TestCollectiveAPIRunnerBase): - def __init__(self): pass @@ -142,9 +140,11 @@ def check_pass(self, *args, **kwargs): from common import init_parallel_env import paddle from paddle.distributed import fleet + hcg = init_parallel_env("DP4-MP2-PP2-SH1-O1", 64) pp_degree = hcg.get_pipe_parallel_world_size() import numpy as np + crit = Criterion() if pp_degree <= 1: model = Model(hcg) @@ -153,10 +153,12 @@ def check_pass(self, *args, **kwargs): model_base = Model(None) - optimizer = paddle.optimizer.Adam(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=0.01, parameters=model.parameters() + ) optimizer_base = paddle.optimizer.Adam( - learning_rate=0.01, parameters=model_base.parameters()) + learning_rate=0.01, parameters=model_base.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_group_sharded_stage3.py b/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_group_sharded_stage3.py index 9c432eefa8a29..081cd04afb389 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_group_sharded_stage3.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_group_sharded_stage3.py @@ -23,10 +23,18 @@ from paddle.fluid.dygraph.nn import Linear from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import GroupShardedOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import GroupShardedStage2 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import GroupShardedStage3 -from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import GroupShardedScaler +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_optimizer_stage2 import ( + GroupShardedOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage2 import ( + GroupShardedStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_stage3 import ( + GroupShardedStage3, +) +from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_utils import ( + GroupShardedScaler, +) epoch = 10 paddle.seed(2022) @@ -37,7 +45,6 @@ class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -57,7 +64,6 @@ def forward(self, inputs): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -70,54 +76,60 @@ def __reader__(): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) optimizer = paddle.optimizer.Momentum( - parameters=[{ - "params": list(model.parameters()) - }] if opt_group else list(model.parameters()), + parameters=[{"params": list(model.parameters())}] + if opt_group + else list(model.parameters()), learning_rate=0.001, weight_decay=0.00001, grad_clip=clip, - multi_precision=use_pure_fp16) + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - sharding_stage, - use_pure_fp16=False, - accumulate_grad=False, - batch_size=100, - opt_group=False, - sync_comm=False, - test_minimize=False, - save_model=False): +def train_mlp( + model, + sharding_stage, + use_pure_fp16=False, + accumulate_grad=False, + batch_size=100, + opt_group=False, + sync_comm=False, + test_minimize=False, + save_model=False, +): group = paddle.distributed.new_group( - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + ) if opt_group: - optimizer = optimizer_setting(model=model, - use_pure_fp16=use_pure_fp16, - opt_group=opt_group) + optimizer = optimizer_setting( + model=model, use_pure_fp16=use_pure_fp16, opt_group=opt_group + ) else: optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if use_pure_fp16: - model = paddle.amp.decorate(models=model, - level='O2', - save_dtype='float32') + model = paddle.amp.decorate( + models=model, level='O2', save_dtype='float32' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=32768) scaler = GroupShardedScaler(scaler) if sharding_stage == 2: optimizer = GroupShardedOptimizerStage2( - params=optimizer._parameter_list, optim=optimizer, group=group) - model = GroupShardedStage2(model, - optimizer, - group=group, - buffer_max_size=2**21) + params=optimizer._parameter_list, optim=optimizer, group=group + ) + model = GroupShardedStage2( + model, optimizer, group=group, buffer_max_size=2**21 + ) elif sharding_stage == 3: - model = GroupShardedStage3(model, - optimizer=optimizer, - group=group, - sync_comm=sync_comm, - segment_size=2**15) + model = GroupShardedStage3( + model, + optimizer=optimizer, + group=group, + sync_comm=sync_comm, + segment_size=2**15, + ) # check optimizer.minimize() error if test_minimize: @@ -125,18 +137,21 @@ def train_mlp(model, optimizer.minimize() except: print( - "====== Find sharding_stage3_optimizer.minimize() error ======") + "====== Find sharding_stage3_optimizer.minimize() error ======" + ) return - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) for eop in range(epoch): @@ -147,8 +162,9 @@ def train_mlp(model, img.stop_gradient = True with paddle.amp.auto_cast(True, level='O2'): out = model(img) - loss = paddle.nn.functional.cross_entropy(input=out, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=out, label=label + ) avg_loss = paddle.mean(x=loss.cast(dtype=paddle.float32)) if batch_size == 20: @@ -183,8 +199,19 @@ def train_mlp(model, def test_stage2_stage3(): paddle.distributed.init_parallel_env() - mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6, mlp7, mlp8, mlp9, mlp10 = MLP( - ), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP(), MLP() + mlp, mlp1, mlp2, mlp3, mlp4, mlp5, mlp6, mlp7, mlp8, mlp9, mlp10 = ( + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + MLP(), + ) state_dict = mlp.state_dict() mlp1.set_state_dict(state_dict) mlp2.set_state_dict(state_dict) @@ -198,79 +225,88 @@ def test_stage2_stage3(): mlp10.set_state_dict(state_dict) # fp32 - stage2_params = train_mlp(mlp1, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False) - stage3_params = train_mlp(mlp2, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False) + stage2_params = train_mlp( + mlp1, sharding_stage=2, use_pure_fp16=False, opt_group=False + ) + stage3_params = train_mlp( + mlp2, sharding_stage=3, use_pure_fp16=False, opt_group=False + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-6, - atol=1e-6) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-6, + atol=1e-6, + ) # fp32 accumulate grad - stage3_params = train_mlp(mlp3, - sharding_stage=3, - use_pure_fp16=False, - accumulate_grad=True, - opt_group=True) - stage3_params_add = train_mlp(mlp4, - sharding_stage=3, - use_pure_fp16=False, - accumulate_grad=True, - batch_size=20, - opt_group=True) + stage3_params = train_mlp( + mlp3, + sharding_stage=3, + use_pure_fp16=False, + accumulate_grad=True, + opt_group=True, + ) + stage3_params_add = train_mlp( + mlp4, + sharding_stage=3, + use_pure_fp16=False, + accumulate_grad=True, + batch_size=20, + opt_group=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_add[i].numpy(), - rtol=1e-6, - atol=1e-4) + np.testing.assert_allclose( + stage3_params[i].numpy(), + stage3_params_add[i].numpy(), + rtol=1e-6, + atol=1e-4, + ) # fp16 - stage2_params = train_mlp(mlp5, - sharding_stage=2, - use_pure_fp16=True, - opt_group=False) - stage3_params = train_mlp(mlp6, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False) + stage2_params = train_mlp( + mlp5, sharding_stage=2, use_pure_fp16=True, opt_group=False + ) + stage3_params = train_mlp( + mlp6, sharding_stage=3, use_pure_fp16=True, opt_group=False + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage3_params[i].numpy(), - rtol=1e-4, - atol=1e-3) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage3_params[i].numpy(), + rtol=1e-4, + atol=1e-3, + ) # fp16 sync_comm - stage3_params = train_mlp(mlp7, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False) - stage3_params_re = train_mlp(mlp8, - sharding_stage=3, - use_pure_fp16=True, - opt_group=False, - sync_comm=True) + stage3_params = train_mlp( + mlp7, sharding_stage=3, use_pure_fp16=True, opt_group=False + ) + stage3_params_re = train_mlp( + mlp8, + sharding_stage=3, + use_pure_fp16=True, + opt_group=False, + sync_comm=True, + ) for i in range(len(stage3_params)): - np.testing.assert_allclose(stage3_params[i].numpy(), - stage3_params_re[i].numpy(), - rtol=1e-6) + np.testing.assert_allclose( + stage3_params[i].numpy(), stage3_params_re[i].numpy(), rtol=1e-6 + ) # save/load model output_dir = tempfile.mkdtemp() try: model_file = os.path.join(output_dir, "model.pdmodel") optimizer_file = os.path.join(output_dir, "model.pdopt") - model_stage3, optimizer_stage3 = train_mlp(mlp9, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False, - save_model=True) + model_stage3, optimizer_stage3 = train_mlp( + mlp9, + sharding_stage=3, + use_pure_fp16=False, + opt_group=False, + save_model=True, + ) paddle.save(model_stage3.state_dict(), model_file) paddle.save(optimizer_stage3.state_dict(), optimizer_file) m_state_dict = paddle.load(model_file) @@ -284,11 +320,13 @@ def test_stage2_stage3(): shutil.rmtree(output_dir) # check optimizer.minimize() error - train_mlp(mlp10, - sharding_stage=3, - use_pure_fp16=False, - opt_group=False, - test_minimize=True) + train_mlp( + mlp10, + sharding_stage=3, + use_pure_fp16=False, + opt_group=False, + test_minimize=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_sharding_stage2.py b/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_sharding_stage2.py index affb6d31f2ae5..eeff08c3557ce 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_sharding_stage2.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/mn_dygraph_sharding_stage2.py @@ -24,8 +24,12 @@ from paddle.distributed import fleet from paddle.fluid.framework import _test_eager_guard -from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ShardingOptimizerStage2 -from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ShardingStage2 +from paddle.distributed.fleet.meta_optimizers.dygraph_optimizer.sharding_optimizer_stage2 import ( + ShardingOptimizerStage2, +) +from paddle.distributed.fleet.meta_parallel.sharding.sharding_stage2 import ( + ShardingStage2, +) seed = 2022 epoch = 2 @@ -36,7 +40,7 @@ "dp_degree": 16, "mp_degree": 1, "pp_degree": 1, - "sharding_degree": 1 + "sharding_degree": 1, } np.random.seed(seed) @@ -44,7 +48,6 @@ class MLP(fluid.Layer): - def __init__(self, linear_size=1000, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -64,7 +67,6 @@ def forward(self, inputs): def reader_decorator(linear_size=1000): - def __reader__(): for _ in range(100): img = np.random.rand(linear_size).astype('float32') @@ -76,59 +78,65 @@ def __reader__(): def optimizer_setting(model, use_pure_fp16, opt_group=False): clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) - optimizer = paddle.optimizer.AdamW(parameters=[{ - "params": model.parameters() - }] if opt_group else model.parameters(), - learning_rate=0.001, - weight_decay=0.00001, - grad_clip=clip, - multi_precision=use_pure_fp16) + optimizer = paddle.optimizer.AdamW( + parameters=[{"params": model.parameters()}] + if opt_group + else model.parameters(), + learning_rate=0.001, + weight_decay=0.00001, + grad_clip=clip, + multi_precision=use_pure_fp16, + ) return optimizer -def train_mlp(model, - sharding_stage, - batch_size=100, - use_pure_fp16=False, - accumulate_grad=False, - opt_group=False, - save_model=False): +def train_mlp( + model, + sharding_stage, + batch_size=100, + use_pure_fp16=False, + accumulate_grad=False, + opt_group=False, + save_model=False, +): if sharding_stage == "dp": hcg = fleet.get_hybrid_communicate_group() group = hcg.get_check_parallel_group() else: group = paddle.distributed.new_group( - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + ) if opt_group: - optimizer = optimizer_setting(model=model, - use_pure_fp16=use_pure_fp16, - opt_group=opt_group) + optimizer = optimizer_setting( + model=model, use_pure_fp16=use_pure_fp16, opt_group=opt_group + ) else: optimizer = optimizer_setting(model=model, use_pure_fp16=use_pure_fp16) if sharding_stage == 2: - optimizer = ShardingOptimizerStage2(params=model.parameters(), - optim=optimizer, - group=group) - - model = ShardingStage2(model, - optimizer, - group=group, - buffer_max_size=2**21) + optimizer = ShardingOptimizerStage2( + params=model.parameters(), optim=optimizer, group=group + ) + + model = ShardingStage2( + model, optimizer, group=group, buffer_max_size=2**21 + ) else: optimizer = fleet.distributed_optimizer(optimizer) model = fleet.distributed_model(model) - train_reader = paddle.batch(reader_decorator(), - batch_size=batch_size, - drop_last=True) - - train_loader = paddle.io.DataLoader.from_generator(capacity=32, - use_double_buffer=True, - iterable=True, - return_list=True, - use_multiprocess=True) + train_reader = paddle.batch( + reader_decorator(), batch_size=batch_size, drop_last=True + ) + + train_loader = paddle.io.DataLoader.from_generator( + capacity=32, + use_double_buffer=True, + iterable=True, + return_list=True, + use_multiprocess=True, + ) train_loader.set_sample_list_generator(train_reader) if sharding_stage == 2: @@ -180,53 +188,51 @@ def test_dp_stage2(): mlp6.set_state_dict(state_dict) # DP VS stage2 - dp_params = train_mlp(mlp1, - sharding_stage="dp", - use_pure_fp16=False, - opt_group=False) - stage2_params = train_mlp(mlp2, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False) + dp_params = train_mlp( + mlp1, sharding_stage="dp", use_pure_fp16=False, opt_group=False + ) + stage2_params = train_mlp( + mlp2, sharding_stage=2, use_pure_fp16=False, opt_group=False + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6, - atol=5e-4) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6, atol=5e-4 + ) # stage2 accumulate grad stage2_params = train_mlp(mlp3, sharding_stage=2, accumulate_grad=True) - stage2_accumulate_grad = train_mlp(mlp4, - sharding_stage=2, - batch_size=20, - accumulate_grad=True) + stage2_accumulate_grad = train_mlp( + mlp4, sharding_stage=2, batch_size=20, accumulate_grad=True + ) for i in range(len(stage2_params)): - np.testing.assert_allclose(stage2_params[i].numpy(), - stage2_accumulate_grad[i].numpy(), - rtol=1e-5, - atol=1e-5) + np.testing.assert_allclose( + stage2_params[i].numpy(), + stage2_accumulate_grad[i].numpy(), + rtol=1e-5, + atol=1e-5, + ) # stage2 param list VS param group - stage2_params = train_mlp(mlp5, - sharding_stage=2, - use_pure_fp16=False, - opt_group=True) + stage2_params = train_mlp( + mlp5, sharding_stage=2, use_pure_fp16=False, opt_group=True + ) for i in range(len(dp_params)): - np.testing.assert_allclose(dp_params[i].numpy(), - stage2_params[i].numpy(), - rtol=1e-6, - atol=5e-4) + np.testing.assert_allclose( + dp_params[i].numpy(), stage2_params[i].numpy(), rtol=1e-6, atol=5e-4 + ) # save/load model output_dir = tempfile.mkdtemp() try: model_file = os.path.join(output_dir, "model.pdmodel") optimizer_file = os.path.join(output_dir, "model.pdopt") - model_stage2, optimizer_stage2 = train_mlp(mlp6, - sharding_stage=2, - use_pure_fp16=False, - opt_group=False, - save_model=True) + model_stage2, optimizer_stage2 = train_mlp( + mlp6, + sharding_stage=2, + use_pure_fp16=False, + opt_group=False, + save_model=True, + ) paddle.save(model_stage2.state_dict(), model_file) paddle.save(optimizer_stage2.state_dict(), optimizer_file) m_state_dict = paddle.load(model_file) diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/test_collective_multi_nodes.py b/python/paddle/fluid/tests/unittests/collective/multinode/test_collective_multi_nodes.py index c7ad96c061ef7..868585f81a0be 100644 --- a/python/paddle/fluid/tests/unittests/collective/multinode/test_collective_multi_nodes.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/test_collective_multi_nodes.py @@ -20,10 +20,10 @@ class TestCollectiveAPIRunnerBase(object): - def check_pass(self, *args, **kwargs): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def run_trainer(self, *args, **kwargs): self.check_pass(*args, **kwargs) @@ -37,7 +37,6 @@ def runtime_main(test_class, col_type=None): class TestDistBase(unittest.TestCase): - def setUp(self): self._trainers = 4 self._init_env() @@ -46,15 +45,17 @@ def _init_env(self): self._python_interp = sys.executable self.temp_dir = tempfile.TemporaryDirectory() - def check_with_place(self, - model_file, - backend="nccl", - static_mode=False, - check_error_log=False, - need_envs={}, - eager_mode=True, - args=[], - kwargs={}): + def check_with_place( + self, + model_file, + backend="nccl", + static_mode=False, + check_error_log=False, + need_envs={}, + eager_mode=True, + args=[], + kwargs={}, + ): required_envs = { "FLAGS_fraction_of_gpu_memory_to_use": "0.15", "FLAGS_eager_delete_tensor_gb": "0.0", @@ -68,7 +69,7 @@ def check_with_place(self, "PADDLE_WITH_GLOO": "0", "BACKEND": backend, "PADDLE_DISTRI_BACKEND": backend, - "PADDLE_USE_GPU": "1" + "PADDLE_USE_GPU": "1", } required_envs.update(need_envs) if check_error_log: @@ -87,15 +88,18 @@ def _run_cluster(self, model_file, envs): filted_envs = dict() for k in envs.keys(): if "PADDLE_" == k[:7] and k not in [ - "PADDLE_NNODES", "PADDLE_MASTER" + "PADDLE_NNODES", + "PADDLE_MASTER", ]: continue filted_envs[k] = envs[k] - launcher = subprocess.Popen(run_cluster_process.strip().split(), - stdout=sys.stderr, - stderr=sys.stdout, - env=filted_envs) + launcher = subprocess.Popen( + run_cluster_process.strip().split(), + stdout=sys.stderr, + stderr=sys.stdout, + env=filted_envs, + ) launcher.communicate(timeout=240) if launcher.poll() is None: diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dp.py b/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dp.py index 05124e0ddd5dd..80973d735555c 100755 --- a/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dp.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dp.py @@ -22,15 +22,14 @@ class TestDYgraphDPMode(TestDistBase): - def setUp(self): self._trainers = 16 self._init_env() def test_col_parallel_linear(self): - self.check_with_place("dygraph_hybrid_dp.py", - backend="nccl", - need_envs=os.environ) + self.check_with_place( + "dygraph_hybrid_dp.py", backend="nccl", need_envs=os.environ + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dpppmp.py b/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dpppmp.py index aa2a01454ae9b..1435dd8ef0e1d 100755 --- a/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dpppmp.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_hybrid_dpppmp.py @@ -22,25 +22,24 @@ class TestDYgraphHybrid(TestDistBase): - def setUp(self): self._trainers = 16 self._init_env() def test_hybrid_dpppmp(self): - self.check_with_place("dygraph_hybrid_dpppmp.py", - backend="nccl", - need_envs=os.environ) + self.check_with_place( + "dygraph_hybrid_dpppmp.py", backend="nccl", need_envs=os.environ + ) def test_hybrid_recompute(self): - self.check_with_place("dygraph_hybrid_recompute.py", - backend="nccl", - need_envs=os.environ) + self.check_with_place( + "dygraph_hybrid_recompute.py", backend="nccl", need_envs=os.environ + ) def test_hybrid_fp16(self): - self.check_with_place("dygraph_hybrid_fp16.py", - backend="nccl", - need_envs=os.environ) + self.check_with_place( + "dygraph_hybrid_fp16.py", backend="nccl", need_envs=os.environ + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_sharding.py b/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_sharding.py index f0da628a4c8a3..d1689330d4aa9 100755 --- a/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_sharding.py +++ b/python/paddle/fluid/tests/unittests/collective/multinode/test_multinode_dygraph_sharding.py @@ -22,20 +22,23 @@ class TestDYgrapShardingDP(TestDistBase): - def setUp(self): self._trainers = 16 self._init_env() def test_hybrid_sharding_stage2(self): - self.check_with_place("mn_dygraph_sharding_stage2.py", - backend="nccl", - need_envs=os.environ) + self.check_with_place( + "mn_dygraph_sharding_stage2.py", + backend="nccl", + need_envs=os.environ, + ) def test_hybrid_sharding_stage3(self): - self.check_with_place("mn_dygraph_group_sharded_stage3.py", - backend="nccl", - need_envs=os.environ) + self.check_with_place( + "mn_dygraph_group_sharded_stage3.py", + backend="nccl", + need_envs=os.environ, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/orthogonal_strategy.py b/python/paddle/fluid/tests/unittests/collective/orthogonal_strategy.py index 5dbc624b79bb0..7d39b6772336c 100644 --- a/python/paddle/fluid/tests/unittests/collective/orthogonal_strategy.py +++ b/python/paddle/fluid/tests/unittests/collective/orthogonal_strategy.py @@ -14,27 +14,37 @@ import unittest import paddle.distributed as dist -from paddle.distributed.fleet.base.strategy_group import DPGroup, ShardingGroup, MPGroup, PPGroup +from paddle.distributed.fleet.base.strategy_group import ( + DPGroup, + ShardingGroup, + MPGroup, + PPGroup, +) from paddle.distributed.fleet.base.orthogonal_strategy import OrthogonalStrategy class TestOrthogonalStrategyAPI(unittest.TestCase): - def setUp(self): self._num_of_ranks = 2 dist.init_parallel_env() self._global_rank = dist.get_rank() self._strategy = OrthogonalStrategy( - [("dp", 2, DPGroup), ("mp", 1, MPGroup), - ("sharding", 1, ShardingGroup), ("pp", 1, PPGroup)], - fused_strategy_dict={"checkness": ["mp", "sharding", "pp"]}) + [ + ("dp", 2, DPGroup), + ("mp", 1, MPGroup), + ("sharding", 1, ShardingGroup), + ("pp", 1, PPGroup), + ], + fused_strategy_dict={"checkness": ["mp", "sharding", "pp"]}, + ) def test_orthogonal_strategy(self): dp_group = self._strategy.strategy_group("dp") self.assertEqual(dp_group.world_size, self._num_of_ranks) self.assertEqual(dp_group.group.nranks, self._num_of_ranks) - self.assertEqual(self._strategy.rank_in_strategy("dp"), - self._global_rank) + self.assertEqual( + self._strategy.rank_in_strategy("dp"), self._global_rank + ) fused_group = self._strategy.fused_strategy_group("checkness") self.assertEqual(fused_group.world_size, 1) diff --git a/python/paddle/fluid/tests/unittests/collective/parallel_embedding_api.py b/python/paddle/fluid/tests/unittests/collective/parallel_embedding_api.py index 8b6ab746ce858..0c118441b6594 100644 --- a/python/paddle/fluid/tests/unittests/collective/parallel_embedding_api.py +++ b/python/paddle/fluid/tests/unittests/collective/parallel_embedding_api.py @@ -22,7 +22,6 @@ class TestParallelEmbeddingAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -36,24 +35,30 @@ def get_model(self, main_prog, startup_program, rank): paddle.seed(2020) data_in = paddle.randint(0, size[0], shape=(10, 4)) - data = paddle.static.data(name='tindata', - shape=[10, 1000], - dtype="float32") + data = paddle.static.data( + name='tindata', shape=[10, 1000], dtype="float32" + ) per_part_size = size[0] // 2 if rank == 0: param_attr = paddle.fluid.ParamAttr( initializer=paddle.fluid.initializer.NumpyArrayInitializer( - np_array[0:per_part_size, :]), ) + np_array[0:per_part_size, :] + ), + ) else: param_attr = paddle.fluid.ParamAttr( initializer=paddle.fluid.initializer.NumpyArrayInitializer( - np_array[per_part_size:size[0], :]), ) + np_array[per_part_size : size[0], :] + ), + ) - emb_out = paddle.distributed.split(data_in, - size, - operation="embedding", - num_partitions=2, - weight_attr=param_attr) + emb_out = paddle.distributed.split( + data_in, + size, + operation="embedding", + num_partitions=2, + weight_attr=param_attr, + ) return [data_in, emb_out] diff --git a/python/paddle/fluid/tests/unittests/collective/process_group_gloo.py b/python/paddle/fluid/tests/unittests/collective/process_group_gloo.py index 4e678ef955e59..3ace517991322 100644 --- a/python/paddle/fluid/tests/unittests/collective/process_group_gloo.py +++ b/python/paddle/fluid/tests/unittests/collective/process_group_gloo.py @@ -24,7 +24,6 @@ class TestProcessGroupFp32(unittest.TestCase): - def setUp(self): paddle.seed(2022) random.seed(2022) @@ -40,8 +39,9 @@ def test_create_process_group_gloo(self): nranks = ParallelEnv().nranks rank = ParallelEnv().local_rank is_master = True if rank == 0 else False - store = paddle.fluid.core.TCPStore("127.0.0.1", 6272, is_master, - nranks, 30) + store = paddle.fluid.core.TCPStore( + "127.0.0.1", 6272, is_master, nranks, 30 + ) place = paddle.fluid.core.CPUPlace() pg = paddle.fluid.core.ProcessGroupGloo(store, rank, nranks, place) @@ -135,8 +135,9 @@ def test_create_process_group_gloo(self): task = pg.all_gather(tensor_y, tensor_out) task.wait() out_1 = paddle.slice(tensor_out, [0], [0], [out_shape[0] // 2]) - out_2 = paddle.slice(tensor_out, [0], [out_shape[0] // 2], - [out_shape[0]]) + out_2 = paddle.slice( + tensor_out, [0], [out_shape[0] // 2], [out_shape[0]] + ) assert np.array_equal(tensor_x, out_1) assert np.array_equal(tensor_y, out_2) print("test allgather api ok\n") @@ -175,8 +176,9 @@ def test_create_process_group_gloo(self): task = pg.scatter(tensor_x, tensor_y, 0) task.wait() out1 = paddle.slice(tensor_x, [0], [0], [self.shape[0]]) - out2 = paddle.slice(tensor_x, [0], [self.shape[0]], - [self.shape[0] * 2]) + out2 = paddle.slice( + tensor_x, [0], [self.shape[0]], [self.shape[0] * 2] + ) if pg.rank() == 0: assert np.array_equal(tensor_y, out1) else: diff --git a/python/paddle/fluid/tests/unittests/collective/process_group_mpi.py b/python/paddle/fluid/tests/unittests/collective/process_group_mpi.py index f3ca19ebe7eae..998e323bfb116 100644 --- a/python/paddle/fluid/tests/unittests/collective/process_group_mpi.py +++ b/python/paddle/fluid/tests/unittests/collective/process_group_mpi.py @@ -42,12 +42,14 @@ def init_process_group(strategy=None): place = core.CPUPlace() _set_expected_place(place) - group = Group(rank, - world_size, - id=0, - ranks=list(range(world_size)), - pg=pg, - name=_default_group_name) + group = Group( + rank, + world_size, + id=0, + ranks=list(range(world_size)), + pg=pg, + name=_default_group_name, + ) _set_group_map_by_name(_default_group_name, group) _set_group_map(gid, group) _set_group_map_backend(group, "mpi") @@ -84,15 +86,15 @@ def test_allreduce_max(pg, shape, dtype): max_result = paddle.maximum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.all_reduce(tensor_x, - dist.ReduceOp.MAX, - use_calc_stream=False) + task = dist.all_reduce( + tensor_x, dist.ReduceOp.MAX, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_x, max_result) else: - task = dist.all_reduce(tensor_y, - dist.ReduceOp.MAX, - use_calc_stream=False) + task = dist.all_reduce( + tensor_y, dist.ReduceOp.MAX, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_y, max_result) print("test allreduce max api ok") @@ -109,15 +111,15 @@ def test_allreduce_min(pg, shape, dtype): min_result = paddle.minimum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.all_reduce(tensor_x, - dist.ReduceOp.MIN, - use_calc_stream=False) + task = dist.all_reduce( + tensor_x, dist.ReduceOp.MIN, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_x, min_result) else: - task = dist.all_reduce(tensor_y, - dist.ReduceOp.MIN, - use_calc_stream=False) + task = dist.all_reduce( + tensor_y, dist.ReduceOp.MIN, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_y, min_result) print("test allreduce min api ok") @@ -134,15 +136,15 @@ def test_allreduce_prod(pg, shape, dtype): prod_result = np.multiply(x, y) if pg.rank() == 0: - task = dist.all_reduce(tensor_x, - dist.ReduceOp.PROD, - use_calc_stream=False) + task = dist.all_reduce( + tensor_x, dist.ReduceOp.PROD, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_x, prod_result) else: - task = dist.all_reduce(tensor_y, - dist.ReduceOp.PROD, - use_calc_stream=False) + task = dist.all_reduce( + tensor_y, dist.ReduceOp.PROD, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_y, prod_result) print("test allreduce prod api ok") @@ -196,7 +198,7 @@ def test_allgather(pg, shape, dtype): else: tensor_out_list = [ paddle.empty_like(tensor_x), - paddle.empty_like(tensor_x) + paddle.empty_like(tensor_x), ] task = dist.all_gather(tensor_out_list, tensor_y, use_calc_stream=False) tensor_out = paddle.concat(tensor_out_list) @@ -309,17 +311,15 @@ def test_reduce_max(pg, shape, dtype): max_result = paddle.maximum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.reduce(tensor_x, - 0, - dist.ReduceOp.MAX, - use_calc_stream=False) + task = dist.reduce( + tensor_x, 0, dist.ReduceOp.MAX, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_x, max_result) else: - task = dist.reduce(tensor_y, - 0, - dist.ReduceOp.MAX, - use_calc_stream=False) + task = dist.reduce( + tensor_y, 0, dist.ReduceOp.MAX, use_calc_stream=False + ) task.wait() print("test reduce max api ok") @@ -335,17 +335,15 @@ def test_reduce_min(pg, shape, dtype): min_result = paddle.minimum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.reduce(tensor_x, - 0, - dist.ReduceOp.MIN, - use_calc_stream=False) + task = dist.reduce( + tensor_x, 0, dist.ReduceOp.MIN, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_x, min_result) else: - task = dist.reduce(tensor_y, - 0, - dist.ReduceOp.MIN, - use_calc_stream=False) + task = dist.reduce( + tensor_y, 0, dist.ReduceOp.MIN, use_calc_stream=False + ) task.wait() print("test reduce min api ok") @@ -361,17 +359,15 @@ def test_reduce_prod(pg, shape, dtype): prod_result = np.multiply(x, y) if pg.rank() == 0: - task = dist.reduce(tensor_x, - 0, - dist.ReduceOp.PROD, - use_calc_stream=False) + task = dist.reduce( + tensor_x, 0, dist.ReduceOp.PROD, use_calc_stream=False + ) task.wait() assert np.array_equal(tensor_x, prod_result) else: - task = dist.reduce(tensor_y, - 0, - dist.ReduceOp.PROD, - use_calc_stream=False) + task = dist.reduce( + tensor_y, 0, dist.ReduceOp.PROD, use_calc_stream=False + ) task.wait() print("test reduce prod api ok") @@ -436,7 +432,6 @@ def test_send_recv(pg, sub_group, shape, dtype): class TestProcessGroup(unittest.TestCase): - def setUp(self): paddle.seed(2022) random.seed(2022) diff --git a/python/paddle/fluid/tests/unittests/collective/process_group_nccl.py b/python/paddle/fluid/tests/unittests/collective/process_group_nccl.py index aa3a9575f42ff..ff949d8f14cf4 100644 --- a/python/paddle/fluid/tests/unittests/collective/process_group_nccl.py +++ b/python/paddle/fluid/tests/unittests/collective/process_group_nccl.py @@ -32,7 +32,6 @@ def init_process_group(strategy=None): class TestProcessGroupFp32(unittest.TestCase): - def setUp(self): paddle.seed(2022) random.seed(2022) @@ -45,8 +44,9 @@ def config(self): def test_create_process_group_nccl(self): with _test_eager_guard(): - paddle.set_device('gpu:%d' % - paddle.distributed.ParallelEnv().dev_id) + paddle.set_device( + 'gpu:%d' % paddle.distributed.ParallelEnv().dev_id + ) pg = init_process_group() print("rank:", pg.rank(), "size:", pg.size(), "name:", pg.name()) @@ -81,15 +81,15 @@ def test_create_process_group_nccl(self): max_result = paddle.maximum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.all_reduce(tensor_x, - dist.ReduceOp.MAX, - sync_op=False) + task = dist.all_reduce( + tensor_x, dist.ReduceOp.MAX, sync_op=False + ) task.wait() assert np.array_equal(tensor_x, max_result) else: - task = dist.all_reduce(tensor_y, - dist.ReduceOp.MAX, - sync_op=False) + task = dist.all_reduce( + tensor_y, dist.ReduceOp.MAX, sync_op=False + ) task.wait() assert np.array_equal(tensor_y, max_result) @@ -106,15 +106,15 @@ def test_create_process_group_nccl(self): min_result = paddle.minimum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.all_reduce(tensor_x, - dist.ReduceOp.MIN, - sync_op=False) + task = dist.all_reduce( + tensor_x, dist.ReduceOp.MIN, sync_op=False + ) task.wait() assert np.array_equal(tensor_x, min_result) else: - task = dist.all_reduce(tensor_y, - dist.ReduceOp.MIN, - sync_op=False) + task = dist.all_reduce( + tensor_y, dist.ReduceOp.MIN, sync_op=False + ) task.wait() assert np.array_equal(tensor_y, min_result) @@ -131,15 +131,15 @@ def test_create_process_group_nccl(self): prod_result = np.multiply(x, y) if pg.rank() == 0: - task = dist.all_reduce(tensor_x, - dist.ReduceOp.PROD, - sync_op=False) + task = dist.all_reduce( + tensor_x, dist.ReduceOp.PROD, sync_op=False + ) task.wait() assert np.array_equal(tensor_x, prod_result) else: - task = dist.all_reduce(tensor_y, - dist.ReduceOp.PROD, - sync_op=False) + task = dist.all_reduce( + tensor_y, dist.ReduceOp.PROD, sync_op=False + ) task.wait() assert np.array_equal(tensor_y, prod_result) @@ -196,14 +196,15 @@ def test_create_process_group_nccl(self): else: tensor_out_list = [ paddle.empty_like(tensor_x), - paddle.empty_like(tensor_x) + paddle.empty_like(tensor_x), ] task = dist.all_gather(tensor_out_list, tensor_y, sync_op=False) paddle.device.cuda.synchronize() tensor_out = paddle.concat(tensor_out_list) out_1 = paddle.slice(tensor_out, [0], [0], [out_shape[0] // 2]) - out_2 = paddle.slice(tensor_out, [0], [out_shape[0] // 2], - [out_shape[0]]) + out_2 = paddle.slice( + tensor_out, [0], [out_shape[0] // 2], [out_shape[0]] + ) assert np.array_equal(tensor_x, out_1) assert np.array_equal(tensor_y, out_2) print("test allgather api ok\n") @@ -219,8 +220,9 @@ def test_create_process_group_nccl(self): paddle.device.cuda.synchronize() tensor_out = paddle.concat(tensor_out_list) out_1 = paddle.slice(tensor_out, [0], [0], [out_shape[0] // 2]) - out_2 = paddle.slice(tensor_out, [0], [out_shape[0] // 2], - [out_shape[0]]) + out_2 = paddle.slice( + tensor_out, [0], [out_shape[0] // 2], [out_shape[0]] + ) assert np.array_equal(tensor_x, out_1) assert np.array_equal(tensor_y, out_2) print("test allgather api2 ok\n") @@ -235,10 +237,12 @@ def test_create_process_group_nccl(self): tensor_y = paddle.to_tensor(y) tensor_out1 = paddle.to_tensor(out1) tensor_out2 = paddle.to_tensor(out2) - raw_tensor_x_2 = paddle.slice(tensor_x, [0], [self.shape[0] // 2], - [self.shape[0]]) - raw_tensor_y_1 = paddle.slice(tensor_y, [0], [0], - [self.shape[0] // 2]) + raw_tensor_x_2 = paddle.slice( + tensor_x, [0], [self.shape[0] // 2], [self.shape[0]] + ) + raw_tensor_y_1 = paddle.slice( + tensor_y, [0], [0], [self.shape[0] // 2] + ) if pg.rank() == 0: task = pg.alltoall(tensor_x, tensor_out1) task.wait() @@ -250,8 +254,9 @@ def test_create_process_group_nccl(self): task = dist.alltoall([in_1, in_2], out_tensor_list) paddle.device.cuda.synchronize() tensor_out2 = paddle.concat(out_tensor_list) - out1_2 = paddle.slice(tensor_out1, [0], [self.shape[0] // 2], - [self.shape[0]]) + out1_2 = paddle.slice( + tensor_out1, [0], [self.shape[0] // 2], [self.shape[0]] + ) out2_1 = paddle.slice(tensor_out2, [0], [0], [self.shape[0] // 2]) if pg.rank() == 0: assert np.array_equal(out1_2.numpy(), raw_tensor_y_1.numpy()) @@ -267,10 +272,12 @@ def test_create_process_group_nccl(self): tensor_y = paddle.to_tensor(y) tensor_out1 = paddle.to_tensor(out1) tensor_out2 = paddle.to_tensor(out2) - raw_tensor_x_2 = paddle.slice(tensor_x, [0], [self.shape[0] // 2], - [self.shape[0]]) - raw_tensor_y_1 = paddle.slice(tensor_y, [0], [0], - [self.shape[0] // 2]) + raw_tensor_x_2 = paddle.slice( + tensor_x, [0], [self.shape[0] // 2], [self.shape[0]] + ) + raw_tensor_y_1 = paddle.slice( + tensor_y, [0], [0], [self.shape[0] // 2] + ) if pg.rank() == 0: task = pg.alltoall(tensor_x, tensor_out1) task.wait() @@ -282,8 +289,9 @@ def test_create_process_group_nccl(self): task = dist.alltoall([in_1, in_2], out_tensor_list) paddle.device.cuda.synchronize() tensor_out2 = paddle.concat(out_tensor_list) - out1_2 = paddle.slice(tensor_out1, [0], [self.shape[0] // 2], - [self.shape[0]]) + out1_2 = paddle.slice( + tensor_out1, [0], [self.shape[0] // 2], [self.shape[0]] + ) out2_1 = paddle.slice(tensor_out2, [0], [0], [self.shape[0] // 2]) if pg.rank() == 0: assert np.array_equal(out1_2.numpy(), raw_tensor_y_1.numpy()) @@ -321,17 +329,15 @@ def test_create_process_group_nccl(self): max_result = paddle.maximum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.reduce(tensor_x, - 0, - dist.ReduceOp.MAX, - sync_op=False) + task = dist.reduce( + tensor_x, 0, dist.ReduceOp.MAX, sync_op=False + ) task.wait() assert np.array_equal(tensor_x, max_result) else: - task = dist.reduce(tensor_y, - 0, - dist.ReduceOp.MAX, - sync_op=False) + task = dist.reduce( + tensor_y, 0, dist.ReduceOp.MAX, sync_op=False + ) task.wait() print("test reduce max api ok") @@ -347,17 +353,15 @@ def test_create_process_group_nccl(self): min_result = paddle.minimum(tensor_x, tensor_y) if pg.rank() == 0: - task = dist.reduce(tensor_x, - 0, - dist.ReduceOp.MIN, - sync_op=False) + task = dist.reduce( + tensor_x, 0, dist.ReduceOp.MIN, sync_op=False + ) task.wait() assert np.array_equal(tensor_x, min_result) else: - task = dist.reduce(tensor_y, - 0, - dist.ReduceOp.MIN, - sync_op=False) + task = dist.reduce( + tensor_y, 0, dist.ReduceOp.MIN, sync_op=False + ) task.wait() print("test reduce min api ok") @@ -373,17 +377,15 @@ def test_create_process_group_nccl(self): prod_result = np.multiply(x, y) if pg.rank() == 0: - task = dist.reduce(tensor_x, - 0, - dist.ReduceOp.PROD, - sync_op=False) + task = dist.reduce( + tensor_x, 0, dist.ReduceOp.PROD, sync_op=False + ) task.wait() assert np.array_equal(tensor_x, prod_result) else: - task = dist.reduce(tensor_y, - 0, - dist.ReduceOp.PROD, - sync_op=False) + task = dist.reduce( + tensor_y, 0, dist.ReduceOp.PROD, sync_op=False + ) task.wait() print("test reduce prod api ok") @@ -398,7 +400,7 @@ def test_create_process_group_nccl(self): if pg.rank() == 0: in_1, in_2 = paddle.split(tensor_x, 2) task = dist.scatter(tensor_y, [in_1, in_2], 0, sync_op=True) - #task.wait() + # task.wait() paddle.device.cuda.synchronize() # rank 1 else: @@ -406,8 +408,9 @@ def test_create_process_group_nccl(self): task.wait() paddle.device.cuda.synchronize() out1 = paddle.slice(tensor_x, [0], [0], [self.shape[0]]) - out2 = paddle.slice(tensor_x, [0], [self.shape[0]], - [self.shape[0] * 2]) + out2 = paddle.slice( + tensor_x, [0], [self.shape[0]], [self.shape[0] * 2] + ) if pg.rank() == 0: assert np.array_equal(tensor_y, out1) else: @@ -450,7 +453,6 @@ def test_create_process_group_nccl(self): class TestProcessGroupFp16(TestProcessGroupFp32): - def setUp(self): paddle.seed(2022) random.seed(2022) diff --git a/python/paddle/fluid/tests/unittests/collective/row_parallel_linear_api.py b/python/paddle/fluid/tests/unittests/collective/row_parallel_linear_api.py index 854ef9cb69e0d..e20f57f39ae12 100644 --- a/python/paddle/fluid/tests/unittests/collective/row_parallel_linear_api.py +++ b/python/paddle/fluid/tests/unittests/collective/row_parallel_linear_api.py @@ -22,7 +22,6 @@ class TestRowParallelLinearAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -32,19 +31,23 @@ def get_model(self, main_prog, startup_program, rank): np.random.seed(2020) np_array = np.random.rand(1000, 16) - data = paddle.static.data(name='tindata', - shape=[10, 1000], - dtype="float32") + data = paddle.static.data( + name='tindata', shape=[10, 1000], dtype="float32" + ) paddle.distributed.broadcast(data, src=0) data = paddle.split(data, 2, axis=1)[rank] if rank == 0: param_attr = paddle.fluid.ParamAttr( initializer=paddle.fluid.initializer.NumpyArrayInitializer( - np_array[0:500, :]), ) + np_array[0:500, :] + ), + ) else: param_attr = paddle.fluid.ParamAttr( initializer=paddle.fluid.initializer.NumpyArrayInitializer( - np_array[500:1000, :]), ) + np_array[500:1000, :] + ), + ) linear_out = paddle.distributed.split( data, diff --git a/python/paddle/fluid/tests/unittests/collective/strategy_group.py b/python/paddle/fluid/tests/unittests/collective/strategy_group.py index 247a232aecc43..75d2995d62a99 100644 --- a/python/paddle/fluid/tests/unittests/collective/strategy_group.py +++ b/python/paddle/fluid/tests/unittests/collective/strategy_group.py @@ -16,7 +16,13 @@ import numpy as np import paddle import paddle.distributed as dist -from paddle.distributed.fleet.base.strategy_group import StrategyGroupBase, DPGroup, MPGroup, PPGroup, ShardingGroup +from paddle.distributed.fleet.base.strategy_group import ( + StrategyGroupBase, + DPGroup, + MPGroup, + PPGroup, + ShardingGroup, +) def _check_using_all_reduce(group): @@ -39,7 +45,6 @@ def _check_using_recv(group, src): class TestStrategyGroupAPI(unittest.TestCase): - def setUp(self): self._num_of_ranks = 2 self._list_of_rank = [[0, 1]] @@ -74,7 +79,12 @@ def test_sharding_parallel_group(self): def test_pipeline_parallel_group(self): pp_group = PPGroup(self._list_of_rank) - send_next_group, send_prev_group, recv_next_group, recv_prev_group = pp_group.p2p_groups + ( + send_next_group, + send_prev_group, + recv_next_group, + recv_prev_group, + ) = pp_group.p2p_groups if self._global_rank == 0: self.assertEqual(pp_group.rank_of_next_stage, 1) self.assertEqual(pp_group.rank_of_prev_stage, 1) diff --git a/python/paddle/fluid/tests/unittests/collective/test_allreduce.py b/python/paddle/fluid/tests/unittests/collective/test_allreduce.py index d137f0f3ec452..19763579b1a42 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_allreduce.py +++ b/python/paddle/fluid/tests/unittests/collective/test_allreduce.py @@ -21,7 +21,6 @@ class TestAllReduceOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/collective/test_broadcast.py b/python/paddle/fluid/tests/unittests/collective/test_broadcast.py index ff171d084d3f6..8aa7be89d15e7 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_broadcast.py +++ b/python/paddle/fluid/tests/unittests/collective/test_broadcast.py @@ -21,7 +21,6 @@ class TestCBroadcastOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/collective/test_c_concat.py b/python/paddle/fluid/tests/unittests/collective/test_c_concat.py index bc241c1e556f5..c28e313744aaf 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_c_concat.py +++ b/python/paddle/fluid/tests/unittests/collective/test_c_concat.py @@ -21,7 +21,6 @@ class TestConcatOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/collective/test_c_identity.py b/python/paddle/fluid/tests/unittests/collective/test_c_identity.py index 936e2961c57c2..585c0aff0e4d4 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_c_identity.py +++ b/python/paddle/fluid/tests/unittests/collective/test_c_identity.py @@ -21,7 +21,6 @@ class TestIdentityOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/collective/test_c_split.py b/python/paddle/fluid/tests/unittests/collective/test_c_split.py index 43dc766ad24b9..b75398ca6bd76 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_c_split.py +++ b/python/paddle/fluid/tests/unittests/collective/test_c_split.py @@ -21,7 +21,6 @@ class TestSplitOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_api.py index 9040564ce1206..f0e9b3bba2f60 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_api.py @@ -21,59 +21,96 @@ class TestCollectiveAllgatherAPI(TestDistBase): - def _setup_config(self): pass def test_allgather_nccl(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "complex64", "complex128" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "complex64", + "complex128", ] for dtype in dtypes_to_test: - self.check_with_place("collective_allgather_api.py", - "allgather", - "nccl", - dtype=dtype) + self.check_with_place( + "collective_allgather_api.py", "allgather", "nccl", dtype=dtype + ) def test_allgather_gloo(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "complex64", "complex128" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "complex64", + "complex128", ] for dtype in dtypes_to_test: - self.check_with_place("collective_allgather_api.py", - "allgather", - "gloo", - "3", - dtype=dtype) + self.check_with_place( + "collective_allgather_api.py", + "allgather", + "gloo", + "3", + dtype=dtype, + ) def test_allgatther_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "complex64", "complex128" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "complex64", + "complex128", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_allgather_api_dygraph.py", - "allgather", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_allgather_api_dygraph.py", + "allgather", + "nccl", + static_mode="0", + dtype=dtype, + ) def test_allgather_gloo_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "bfloat16", "complex64", "complex128" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "bfloat16", + "complex64", + "complex128", ] for dtype in dtypes_to_test: - self.check_with_place("collective_allgather_api_dygraph.py", - "allgather", - "gloo", - "3", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_allgather_api_dygraph.py", + "allgather", + "gloo", + "3", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_object_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_object_api.py index 90539e89c0b0e..4a174010c5102 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_object_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_allgather_object_api.py @@ -17,35 +17,42 @@ class TestCollectiveAllgatherObjectAPI(test_base.TestDistBase): - def _setup_config(self): pass def test_allgather_nccl(self): - self.check_with_place("collective_allgather_object_api_dygraph.py", - "allgather_object", - "nccl", - static_mode="0", - dtype="pylist") - self.check_with_place("collective_allgather_object_api_dygraph.py", - "allgather_object", - "nccl", - static_mode="0", - dtype="pydict") + self.check_with_place( + "collective_allgather_object_api_dygraph.py", + "allgather_object", + "nccl", + static_mode="0", + dtype="pylist", + ) + self.check_with_place( + "collective_allgather_object_api_dygraph.py", + "allgather_object", + "nccl", + static_mode="0", + dtype="pydict", + ) def test_allgather_gloo_dygraph(self): - self.check_with_place("collective_allgather_object_api_dygraph.py", - "allgather_object", - "gloo", - "3", - static_mode="0", - dtype="pylist") - self.check_with_place("collective_allgather_object_api_dygraph.py", - "allgather_object", - "gloo", - "3", - static_mode="0", - dtype="pydict") + self.check_with_place( + "collective_allgather_object_api_dygraph.py", + "allgather_object", + "gloo", + "3", + static_mode="0", + dtype="pylist", + ) + self.check_with_place( + "collective_allgather_object_api_dygraph.py", + "allgather_object", + "gloo", + "3", + static_mode="0", + dtype="pydict", + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_allreduce_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_allreduce_api.py index a5080f78bcee2..6b2b9c0ade8f4 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_allreduce_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_allreduce_api.py @@ -21,50 +21,69 @@ class TestCollectiveAllreduceAPI(TestDistBase): - def _setup_config(self): pass def test_allreduce_nccl(self): if paddle.fluid.core.is_compiled_with_cuda(): - self.check_with_place("collective_allreduce_api.py", "allreduce", - "nccl") + self.check_with_place( + "collective_allreduce_api.py", "allreduce", "nccl" + ) def test_allreduce_bkcl(self): if paddle.fluid.core.is_compiled_with_xpu(): - self.check_with_place("collective_allreduce_api.py", "allreduce", - "bkcl") + self.check_with_place( + "collective_allreduce_api.py", "allreduce", "bkcl" + ) def test_allreduce_gloo(self): - self.check_with_place("collective_allreduce_api.py", "allreduce", - "gloo", "2") + self.check_with_place( + "collective_allreduce_api.py", "allreduce", "gloo", "2" + ) def test_allreduce_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_allreduce_api_dygraph.py", - "allreduce", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_allreduce_api_dygraph.py", + "allreduce", + "nccl", + static_mode="0", + dtype=dtype, + ) def test_allreduce_gloo_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "bfloat16" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "bfloat16", ] for dtype in dtypes_to_test: - self.check_with_place("collective_allreduce_api_dygraph.py", - "allreduce", - "gloo", - "2", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_allreduce_api_dygraph.py", + "allreduce", + "gloo", + "2", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_api.py index 1edb06ae512d6..acbbdec2ca89f 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_api.py @@ -21,7 +21,6 @@ class TestCollectiveAllToAllAPI(TestDistBase): - def _setup_config(self): pass @@ -30,17 +29,25 @@ def test_alltoall_nccl(self): def test_alltoall_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_alltoall_api_dygraph.py", - "alltoall", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_alltoall_api_dygraph.py", + "alltoall", + "nccl", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single.py b/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single.py index 9ef0a7639e18b..6915f07e8fa3a 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single.py @@ -19,7 +19,6 @@ class TestCollectiveAllToAllSingle(TestMultipleGpus): - def test_collective_alltoall_single(self): self.run_mnist_2gpu('collective_alltoall_single.py', eager_mode=True) diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single_api.py index e3ef3f302f33e..26917d8aba5db 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_alltoall_single_api.py @@ -17,23 +17,30 @@ class TestCollectiveAllToAllSingleAPI(test_base.TestDistBase): - def _setup_config(self): pass def test_alltooall_single_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_alltoall_single_api_dygraph.py", - "alltoall", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_alltoall_single_api_dygraph.py", + "alltoall", + "nccl", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_barrier_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_barrier_api.py index 8a883ce989c18..daa23ff7a57dc 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_barrier_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_barrier_api.py @@ -21,7 +21,6 @@ class TestCollectiveBarrierAPI(TestDistBase): - def _setup_config(self): pass @@ -29,8 +28,9 @@ def test_barrier_nccl(self): self.check_with_place("collective_barrier_api.py", "barrier", "nccl") def test_barrier_gloo(self): - self.check_with_place("collective_barrier_api.py", "barrier", "gloo", - "5") + self.check_with_place( + "collective_barrier_api.py", "barrier", "gloo", "5" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_batch_isend_irecv.py b/python/paddle/fluid/tests/unittests/collective/test_collective_batch_isend_irecv.py index e9fad9aa1c1f6..892e813660e28 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_batch_isend_irecv.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_batch_isend_irecv.py @@ -19,7 +19,6 @@ class TestCollectiveBatchIsendIrecv(TestMultipleGpus): - def test_collective_batch_isend_irecv(self): self.run_mnist_2gpu('collective_batch_isend_irecv.py', eager_mode=True) diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_broadcast_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_broadcast_api.py index 8f4e747b622eb..045d1eb317628 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_broadcast_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_broadcast_api.py @@ -21,44 +21,62 @@ class TestCollectiveBroadcastAPI(TestDistBase): - def _setup_config(self): pass def test_broadcast_nccl(self): - self.check_with_place("collective_broadcast_api.py", "broadcast", - "nccl") + self.check_with_place( + "collective_broadcast_api.py", "broadcast", "nccl" + ) def test_broadcast_gloo(self): - self.check_with_place("collective_broadcast_api.py", "broadcast", - "gloo", "0") + self.check_with_place( + "collective_broadcast_api.py", "broadcast", "gloo", "0" + ) def test_broadcast_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_broadcast_api_dygraph.py", - "broadcast", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_broadcast_api_dygraph.py", + "broadcast", + "nccl", + static_mode="0", + dtype=dtype, + ) def test_broadcast_gloo_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "bfloat16" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "bfloat16", ] for dtype in dtypes_to_test: - self.check_with_place("collective_broadcast_api_dygraph.py", - "broadcast", - "gloo", - "0", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_broadcast_api_dygraph.py", + "broadcast", + "gloo", + "0", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_cpu_barrier_with_gloo.py b/python/paddle/fluid/tests/unittests/collective/test_collective_cpu_barrier_with_gloo.py index 157e7106308ab..6049ccbd5d1cf 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_cpu_barrier_with_gloo.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_cpu_barrier_with_gloo.py @@ -26,12 +26,11 @@ class CollectiveCPUBarrierWithGlooTest(unittest.TestCase): - def find_free_port(self): - def _free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -43,8 +42,9 @@ def _free_port(): def barrier_func(self, id, rank_num, server_endpoint, out_dict, sleep_time): try: - paddle.distributed.gloo_init_parallel_env(id, rank_num, - server_endpoint) + paddle.distributed.gloo_init_parallel_env( + id, rank_num, server_endpoint + ) # 1st barrier # Run barrier to synchronize processes after starting paddle.distributed.gloo_barrier() @@ -52,7 +52,7 @@ def barrier_func(self, id, rank_num, server_endpoint, out_dict, sleep_time): # Let rank 0 sleep for one second and check that all processes # saw that artificial delay through the barrier start = time.time() - if (id == 0): + if id == 0: time.sleep(sleep_time) paddle.distributed.gloo_barrier() end = time.time() @@ -66,8 +66,9 @@ def barrier_op(self, id, rank_num, server_endpoint, out_dict, sleep_time): try: main_prog = fluid.Program() startup_prog = fluid.Program() - paddle.distributed.gloo_init_parallel_env(id, rank_num, - server_endpoint) + paddle.distributed.gloo_init_parallel_env( + id, rank_num, server_endpoint + ) place = fluid.CPUPlace() with fluid.program_guard(main_prog, startup_prog): paddle.distributed.barrier() @@ -77,7 +78,7 @@ def barrier_op(self, id, rank_num, server_endpoint, out_dict, sleep_time): # Let rank 0 sleep for one second and check that all processes # saw that artificial delay through the barrier start = time.time() - if (id == 0): + if id == 0: time.sleep(sleep_time) exe.run(main_prog) end = time.time() @@ -97,9 +98,10 @@ def test_barrier_func_with_multiprocess(self): procs_out_dict = manager.dict() jobs = [] for id in range(num_of_ranks): - p = multiprocessing.Process(target=self.barrier_func, - args=(id, num_of_ranks, ep_str, - procs_out_dict, sleep_time)) + p = multiprocessing.Process( + target=self.barrier_func, + args=(id, num_of_ranks, ep_str, procs_out_dict, sleep_time), + ) jobs.append(p) p.start() for proc in jobs: @@ -117,9 +119,10 @@ def test_barrier_op_with_multiprocess(self): procs_out_dict = manager.dict() jobs = [] for id in range(num_of_ranks): - p = multiprocessing.Process(target=self.barrier_op, - args=(id, num_of_ranks, ep_str, - procs_out_dict, sleep_time)) + p = multiprocessing.Process( + target=self.barrier_op, + args=(id, num_of_ranks, ep_str, procs_out_dict, sleep_time), + ) jobs.append(p) p.start() for proc in jobs: diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_global_gather.py b/python/paddle/fluid/tests/unittests/collective/test_collective_global_gather.py index 85406dd00bb5a..cc9dbfc10e38b 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_global_gather.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_global_gather.py @@ -19,28 +19,32 @@ class TestCollectiveGlobalGatherAPI(TestDistBase): - def _setup_config(self): pass def test_global_gather_nccl(self): paddle.enable_static() - self.check_with_place("collective_global_gather.py", "global_gather", - "nccl") + self.check_with_place( + "collective_global_gather.py", "global_gather", "nccl" + ) def test_global_gather_nccl_dygraph(self): - self.check_with_place("collective_global_gather_dygraph.py", - "global_gather", - "nccl", - static_mode="0", - eager_mode=False) + self.check_with_place( + "collective_global_gather_dygraph.py", + "global_gather", + "nccl", + static_mode="0", + eager_mode=False, + ) def test_global_gather_nccl_dygraph_eager(self): - self.check_with_place("collective_global_gather_dygraph.py", - "global_gather", - "nccl", - static_mode="0", - eager_mode=True) + self.check_with_place( + "collective_global_gather_dygraph.py", + "global_gather", + "nccl", + static_mode="0", + eager_mode=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_global_scatter.py b/python/paddle/fluid/tests/unittests/collective/test_collective_global_scatter.py index c2668c62047de..2b83f136c1e94 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_global_scatter.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_global_scatter.py @@ -19,28 +19,32 @@ class TestCollectiveSelectScatterAPI(TestDistBase): - def _setup_config(self): pass def test_global_scatter_nccl(self): paddle.enable_static() - self.check_with_place("collective_global_scatter.py", "global_scatter", - "nccl") + self.check_with_place( + "collective_global_scatter.py", "global_scatter", "nccl" + ) def test_global_scatter_nccl_dygraph(self): - self.check_with_place("collective_global_scatter_dygraph.py", - "global_scatter", - "nccl", - static_mode="0", - eager_mode=False) + self.check_with_place( + "collective_global_scatter_dygraph.py", + "global_scatter", + "nccl", + static_mode="0", + eager_mode=False, + ) def test_global_scatter_nccl_dygraph_eager(self): - self.check_with_place("collective_global_scatter_dygraph.py", - "global_scatter", - "nccl", - static_mode="0", - eager_mode=True) + self.check_with_place( + "collective_global_scatter_dygraph.py", + "global_scatter", + "nccl", + static_mode="0", + eager_mode=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_isend_irecv_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_isend_irecv_api.py index 2b0727cae0c8e..d313fc5a35eb6 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_isend_irecv_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_isend_irecv_api.py @@ -17,23 +17,30 @@ class TestCollectiveIsendIrecvAPI(test_base.TestDistBase): - def _setup_config(self): pass def test_isend_irecv_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_isend_irecv_api_dygraph.py", - "sendrecv", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_isend_irecv_api_dygraph.py", + "sendrecv", + "nccl", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_optimizer.py b/python/paddle/fluid/tests/unittests/collective/test_collective_optimizer.py index 3e4920f8f4f9e..98c4b10466dcd 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_optimizer.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_optimizer.py @@ -26,11 +26,13 @@ import unittest import paddle.fluid as fluid -from paddle.fluid.incubate.fleet.collective import CollectiveOptimizer, DistributedStrategy +from paddle.fluid.incubate.fleet.collective import ( + CollectiveOptimizer, + DistributedStrategy, +) class CollectiveOptimizerTest(unittest.TestCase): - def test_ds_as_None(self): optimizer = fluid.optimizer.AdamOptimizer() dist_optimizer = CollectiveOptimizer(optimizer, strategy=None) @@ -40,8 +42,9 @@ def test_recompute_checkpoints(self): dist_strategy = DistributedStrategy() dist_strategy.forward_recompute = True dist_strategy.recompute_checkpoints = "NoneListTest" - self.assertRaises(ValueError, CollectiveOptimizer, optimizer, - dist_strategy) + self.assertRaises( + ValueError, CollectiveOptimizer, optimizer, dist_strategy + ) dist_strategy.recompute_checkpoints = [] dist_optimizer = CollectiveOptimizer(optimizer, dist_strategy) self.assertRaises(ValueError, dist_optimizer.minimize, None) @@ -58,7 +61,8 @@ def test_recompute_strategy(self): def test_amp_strategy(self): optimizer = fluid.optimizer.AdamOptimizer() optimizer = fluid.contrib.mixed_precision.decorate( - optimizer, init_loss_scaling=1.0, use_dynamic_loss_scaling=True) + optimizer, init_loss_scaling=1.0, use_dynamic_loss_scaling=True + ) dist_strategy = DistributedStrategy() dist_strategy.use_amp = True dist_optimizer = CollectiveOptimizer(optimizer, strategy=dist_strategy) diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_process_group.py b/python/paddle/fluid/tests/unittests/collective/test_collective_process_group.py index 8fe1090e1fff7..fdefe6bda4c8e 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_process_group.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_process_group.py @@ -17,7 +17,6 @@ class TestProcessGroup(TestMultipleGpus): - def test_process_group_nccl(self): self.run_mnist_2gpu('process_group_nccl.py') diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce.py b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce.py index e5cc544109750..96033e2e30763 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce.py @@ -21,7 +21,6 @@ class TestCReduceOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_api.py index 35bff97f91619..ae946d9975016 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_api.py @@ -21,7 +21,6 @@ class TestCollectiveReduceAPI(TestDistBase): - def _setup_config(self): pass @@ -38,30 +37,47 @@ def test_reduce_gloo(self): def test_reduce_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_reduce_api_dygraph.py", - "reduce", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_reduce_api_dygraph.py", + "reduce", + "nccl", + static_mode="0", + dtype=dtype, + ) def test_reduce_gloo_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "bfloat16" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "bfloat16", ] for dtype in dtypes_to_test: - self.check_with_place("collective_reduce_api_dygraph.py", - "reduce", - "gloo", - "1", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_reduce_api_dygraph.py", + "reduce", + "gloo", + "1", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter.py b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter.py index a9151e1466539..634a7ecee0c3a 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter.py @@ -19,7 +19,6 @@ class TestCollectiveReduceScatter(TestMultipleGpus): - def test_collective_reduce_scatter(self): self.run_mnist_2gpu('collective_reduce_scatter.py', eager_mode=True) diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter_api.py index 669478f58a37d..26dfbd2fe13bb 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_reduce_scatter_api.py @@ -17,23 +17,30 @@ class TestCollectiveReduceScatterAPI(test_base.TestDistBase): - def _setup_config(self): pass def test_reduce_scatter_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_reduce_scatter_api_dygraph.py", - "reduce_scatter", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_reduce_scatter_api_dygraph.py", + "reduce_scatter", + "nccl", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_scatter.py b/python/paddle/fluid/tests/unittests/collective/test_collective_scatter.py index e4d26459967dc..1f36e5928ed8c 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_scatter.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_scatter.py @@ -21,7 +21,6 @@ class TestCScatterOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_scatter_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_scatter_api.py index ab7de7975feed..f421e2ccccf3b 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_scatter_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_scatter_api.py @@ -21,43 +21,60 @@ class TestCollectiveScatterAPI(TestDistBase): - def _setup_config(self): pass def test_scatter_gloo(self): - self.check_with_place("collective_scatter_api.py", "scatter", "gloo", - "4") + self.check_with_place( + "collective_scatter_api.py", "scatter", "gloo", "4" + ) def test_scatter_nccl(self): self.check_with_place("collective_scatter_api.py", "scatter", "nccl") def test_scatter_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_scatter_api_dygraph.py", - "scatter", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_scatter_api_dygraph.py", + "scatter", + "nccl", + static_mode="0", + dtype=dtype, + ) def test_scatter_gloo_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool", "bfloat16" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", + "bfloat16", ] for dtype in dtypes_to_test: - self.check_with_place("collective_scatter_api_dygraph.py", - "scatter", - "gloo", - "4", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_scatter_api_dygraph.py", + "scatter", + "gloo", + "4", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv.py b/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv.py index 8f1467cf65203..aa558a73a201e 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv.py @@ -21,7 +21,6 @@ class TestSendRecvOp(TestDistBase): - def _setup_config(self): pass @@ -29,12 +28,14 @@ def test_sendrecv(self): self.check_with_place("collective_sendrecv_op.py", "sendrecv") def test_sendrecv_dynamic_shape(self): - self.check_with_place("collective_sendrecv_op_dynamic_shape.py", - "sendrecv_dynamic_shape") + self.check_with_place( + "collective_sendrecv_op_dynamic_shape.py", "sendrecv_dynamic_shape" + ) def test_sendrecv_array(self): - self.check_with_place("collective_sendrecv_op_array.py", - "sendrecv_array") + self.check_with_place( + "collective_sendrecv_op_array.py", "sendrecv_array" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv_api.py b/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv_api.py index 3db6df5d46e19..72bd44666ff08 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_sendrecv_api.py @@ -21,28 +21,35 @@ class TestCollectiveSendRecvAPI(TestDistBase): - def _setup_config(self): pass - #def test_sendrecv_nccl(self): + # def test_sendrecv_nccl(self): # if paddle.fluid.core.is_compiled_with_cuda(): # self.check_with_place("collective_sendrecv_api.py", "sendrecv", # "nccl") def test_sendrecv_nccl_dygraph(self): dtypes_to_test = [ - "float16", "float32", "float64", "int32", "int64", "int8", "uint8", - "bool" + "float16", + "float32", + "float64", + "int32", + "int64", + "int8", + "uint8", + "bool", ] if self._nccl_version >= 2100: dtypes_to_test.append("bfloat16") for dtype in dtypes_to_test: - self.check_with_place("collective_sendrecv_api_dygraph.py", - "sendrecv", - "nccl", - static_mode="0", - dtype=dtype) + self.check_with_place( + "collective_sendrecv_api_dygraph.py", + "sendrecv", + "nccl", + static_mode="0", + dtype=dtype, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_split_col_linear.py b/python/paddle/fluid/tests/unittests/collective/test_collective_split_col_linear.py index c2983d3ea7993..ee5b334bf2f79 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_split_col_linear.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_split_col_linear.py @@ -21,13 +21,13 @@ class TestColParallelLinearAPI(TestDistBase): - def _setup_config(self): pass def test_col_parallel_linear(self): - self.check_with_place("column_parallel_linear_api.py", - "column_parallel_linear", "nccl") + self.check_with_place( + "column_parallel_linear_api.py", "column_parallel_linear", "nccl" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding.py b/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding.py index fbab3e68cea4e..5b7e6dc7d7360 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding.py @@ -21,13 +21,13 @@ class TestParallelEmbeddingAPI(TestDistBase): - def _setup_config(self): pass def test_parallel_embedding(self): - self.check_with_place("parallel_embedding_api.py", "parallel_embedding", - "nccl") + self.check_with_place( + "parallel_embedding_api.py", "parallel_embedding", "nccl" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding_none_divisible.py b/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding_none_divisible.py index b5888390bcae3..4cf5a59f15335 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding_none_divisible.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_split_embedding_none_divisible.py @@ -20,15 +20,14 @@ class TestCollectiveSplitAssert(unittest.TestCase): - def network(self): fleet.init() - data = paddle.static.data(name='tindata', - shape=[10, 1000], - dtype="float32") - emb_out = paddle.distributed.split(data, (7, 8), - operation="embedding", - num_partitions=2) + data = paddle.static.data( + name='tindata', shape=[10, 1000], dtype="float32" + ) + emb_out = paddle.distributed.split( + data, (7, 8), operation="embedding", num_partitions=2 + ) def test_assert(self): with self.assertRaises(AssertionError): diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_split_row_linear.py b/python/paddle/fluid/tests/unittests/collective/test_collective_split_row_linear.py index c89b1cc7a065d..8253208e8e57b 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_split_row_linear.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_split_row_linear.py @@ -21,13 +21,13 @@ class TestRowParallelLinearAPI(TestDistBase): - def _setup_config(self): pass def test_row_parallel_linear(self): - self.check_with_place("row_parallel_linear_api.py", - "row_parallel_linear", "nccl") + self.check_with_place( + "row_parallel_linear_api.py", "row_parallel_linear", "nccl" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_collective_wait.py b/python/paddle/fluid/tests/unittests/collective/test_collective_wait.py index cf4bcd98fa0bc..934279b7622b4 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_collective_wait.py +++ b/python/paddle/fluid/tests/unittests/collective/test_collective_wait.py @@ -21,14 +21,13 @@ class TestCWaitOp(TestDistBase): - def _setup_config(self): pass def test_allreduce_wait(self): - self.check_with_place("collective_allreduce_op_wait.py", - "allreduce", - check_error_log=True) + self.check_with_place( + "collective_allreduce_op_wait.py", "allreduce", check_error_log=True + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_api_base.py b/python/paddle/fluid/tests/unittests/collective/test_communication_api_base.py index 22edac0015535..ee4601972f0dd 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_api_base.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_api_base.py @@ -22,7 +22,6 @@ class CommunicationTestDistBase(unittest.TestCase): - def setUp(self, save_log_dir=None, num_of_devices=2, timeout=120): self._python_interp = sys.executable self._save_log_dir = save_log_dir @@ -41,18 +40,24 @@ def run_test_case(self, script_file, user_defined_envs=None): start_command_list = start_command.strip().split() try: - self._launcher = subprocess.run(start_command_list, - env=runtime_envs, - timeout=self._timeout, - check=True) + self._launcher = subprocess.run( + start_command_list, + env=runtime_envs, + timeout=self._timeout, + check=True, + ) except subprocess.TimeoutExpired as err: raise TimeoutError( - "Timeout while running command {}, try to set a longer period, {} is not enough." - .format(err.cmd, err.timeout)) + "Timeout while running command {}, try to set a longer period, {} is not enough.".format( + err.cmd, err.timeout + ) + ) except subprocess.CalledProcessError as err: raise RuntimeError( - "Error occurs when running this test case. The return code of command {} is {}" - .format(err.cmd, err.returncode)) + "Error occurs when running this test case. The return code of command {} is {}".format( + err.cmd, err.returncode + ) + ) def tearDown(self): if self._save_log_dir: @@ -63,7 +68,8 @@ def tearDown(self): shutil.copytree(self._log_dir.name, dir_name) else: raise RuntimeError( - "Directory {} exists, failed to save log.".format(dir_name)) + "Directory {} exists, failed to save log.".format(dir_name) + ) def gen_product_envs_list(default_envs, changeable_envs): diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allgather_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allgather_api.py index 306248a033542..d9d22dca3d161 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allgather_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allgather_api.py @@ -17,29 +17,32 @@ class TestCommunicationStreamAllgatherAPI(test_base.CommunicationTestDistBase): - def setUp(self): - super(TestCommunicationStreamAllgatherAPI, self).setUp(num_of_devices=2, - timeout=120) + super(TestCommunicationStreamAllgatherAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_allgather_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue - self.run_test_case("communication_stream_allgather_api_dygraph.py", - user_defined_envs=envs) + self.run_test_case( + "communication_stream_allgather_api_dygraph.py", + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamAllgatherAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allreduce_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allreduce_api.py index 7fe30a5d50fb3..8823466b282ea 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allreduce_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_allreduce_api.py @@ -17,29 +17,32 @@ class TestCommunicationStreamAllreduceAPI(test_base.CommunicationTestDistBase): - def setUp(self): - super(TestCommunicationStreamAllreduceAPI, self).setUp(num_of_devices=2, - timeout=120) + super(TestCommunicationStreamAllreduceAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_allreduce_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue - self.run_test_case("communication_stream_allreduce_api_dygraph.py", - user_defined_envs=envs) + self.run_test_case( + "communication_stream_allreduce_api_dygraph.py", + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamAllreduceAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_api.py index b6094d6932176..0169825e11451 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_api.py @@ -17,29 +17,32 @@ class TestCommunicationStreamAllToAllAPI(test_base.CommunicationTestDistBase): - def setUp(self): - super(TestCommunicationStreamAllToAllAPI, self).setUp(num_of_devices=2, - timeout=120) + super(TestCommunicationStreamAllToAllAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_alltoall_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue - self.run_test_case("communication_stream_alltoall_api_dygraph.py", - user_defined_envs=envs) + self.run_test_case( + "communication_stream_alltoall_api_dygraph.py", + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamAllToAllAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_single_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_single_api.py index 81836dbc13138..6a322b511375a 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_single_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_alltoall_single_api.py @@ -17,31 +17,34 @@ class TestCommunicationStreamAllToAllSingleAPI( - test_base.CommunicationTestDistBase): - + test_base.CommunicationTestDistBase +): def setUp(self): - super(TestCommunicationStreamAllToAllSingleAPI, - self).setUp(num_of_devices=2, timeout=120) + super(TestCommunicationStreamAllToAllSingleAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_alltoall_single_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue self.run_test_case( "communication_stream_alltoall_single_api_dygraph.py", - user_defined_envs=envs) + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamAllToAllSingleAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_broadcast_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_broadcast_api.py index 693ef95a204e7..0bdf7e6b9044e 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_broadcast_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_broadcast_api.py @@ -17,29 +17,32 @@ class TestCommunicationStreamBroadcastAPI(test_base.CommunicationTestDistBase): - def setUp(self): - super(TestCommunicationStreamBroadcastAPI, self).setUp(num_of_devices=2, - timeout=120) + super(TestCommunicationStreamBroadcastAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_broadcast_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue - self.run_test_case("communication_stream_broadcast_api_dygraph.py", - user_defined_envs=envs) + self.run_test_case( + "communication_stream_broadcast_api_dygraph.py", + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamBroadcastAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_api.py index e0b50f6056fef..0fc539ba1a770 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_api.py @@ -17,29 +17,32 @@ class TestCommunicationStreamReduceAPI(test_base.CommunicationTestDistBase): - def setUp(self): - super(TestCommunicationStreamReduceAPI, self).setUp(num_of_devices=2, - timeout=120) + super(TestCommunicationStreamReduceAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_reduce_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue - self.run_test_case("communication_stream_reduce_api_dygraph.py", - user_defined_envs=envs) + self.run_test_case( + "communication_stream_reduce_api_dygraph.py", + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamReduceAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_scatter_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_scatter_api.py index 1f6522d436225..bb8b923762259 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_scatter_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_reduce_scatter_api.py @@ -17,31 +17,34 @@ class TestCommunicationStreamReduceScatterAPI( - test_base.CommunicationTestDistBase): - + test_base.CommunicationTestDistBase +): def setUp(self): - super(TestCommunicationStreamReduceScatterAPI, - self).setUp(num_of_devices=2, timeout=120) + super(TestCommunicationStreamReduceScatterAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_reduce_scatter_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue self.run_test_case( "communication_stream_reduce_scatter_api_dygraph.py", - user_defined_envs=envs) + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamReduceScatterAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_scatter_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_scatter_api.py index 2de99b40e73f1..5219085bf3b0d 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_scatter_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_scatter_api.py @@ -17,29 +17,32 @@ class TestCommunicationStreamScatterAPI(test_base.CommunicationTestDistBase): - def setUp(self): - super(TestCommunicationStreamScatterAPI, self).setUp(num_of_devices=2, - timeout=120) + super(TestCommunicationStreamScatterAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_reduce_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue - self.run_test_case("communication_stream_scatter_api_dygraph.py", - user_defined_envs=envs) + self.run_test_case( + "communication_stream_scatter_api_dygraph.py", + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamScatterAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_sendrecv_api.py b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_sendrecv_api.py index b32ceb2a95229..5bd668aa4b893 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_communication_stream_sendrecv_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_communication_stream_sendrecv_api.py @@ -17,29 +17,32 @@ class TestCommunicationStreamSendRecvAPI(test_base.CommunicationTestDistBase): - def setUp(self): - super(TestCommunicationStreamSendRecvAPI, self).setUp(num_of_devices=2, - timeout=120) + super(TestCommunicationStreamSendRecvAPI, self).setUp( + num_of_devices=2, timeout=120 + ) self._default_envs = { "backend": "nccl", "shape": "(100, 200)", "dtype": "float32", - "seeds": str(self._seeds) + "seeds": str(self._seeds), } self._changeable_envs = { "sync_op": ["True", "False"], - "use_calc_stream": ["True", "False"] + "use_calc_stream": ["True", "False"], } def test_sendrecv_stream(self): - envs_list = test_base.gen_product_envs_list(self._default_envs, - self._changeable_envs) + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) for envs in envs_list: if eval(envs["use_calc_stream"]) and not eval(envs["sync_op"]): continue - self.run_test_case("communication_stream_sendrecv_api_dygraph.py", - user_defined_envs=envs) + self.run_test_case( + "communication_stream_sendrecv_api_dygraph.py", + user_defined_envs=envs, + ) def tearDown(self): super(TestCommunicationStreamSendRecvAPI, self).tearDown() diff --git a/python/paddle/fluid/tests/unittests/collective/test_eager_dist_api.py b/python/paddle/fluid/tests/unittests/collective/test_eager_dist_api.py index 8fe1090e1fff7..fdefe6bda4c8e 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_eager_dist_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_eager_dist_api.py @@ -17,7 +17,6 @@ class TestProcessGroup(TestMultipleGpus): - def test_process_group_nccl(self): self.run_mnist_2gpu('process_group_nccl.py') diff --git a/python/paddle/fluid/tests/unittests/collective/test_gen_nccl_id_op.py b/python/paddle/fluid/tests/unittests/collective/test_gen_nccl_id_op.py index 80afdd393f0ec..4656c97e67023 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_gen_nccl_id_op.py +++ b/python/paddle/fluid/tests/unittests/collective/test_gen_nccl_id_op.py @@ -34,30 +34,35 @@ def run_gen_ncc_id(attr): with paddle.static.program_guard(main_program, startup_program): nccl_id_var = startup_program.global_block().create_var( - name="NCCLID", persistable=True, type=core.VarDesc.VarType.RAW) + name="NCCLID", persistable=True, type=core.VarDesc.VarType.RAW + ) for i in range(1, nccl_comm_num): startup_program.global_block().create_var( name="NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) if use_hallreduce: for i in range(0, nccl_comm_num): startup_program.global_block().create_var( name="Hierarchical_inter_NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().create_var( name="Hierarchical_exter_NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().append_op( type="gen_nccl_id", inputs={}, outputs={"NCCLID": nccl_id_var}, - attrs=attr) + attrs=attr, + ) place = paddle.CPUPlace() exe = paddle.static.Executor(place) @@ -65,7 +70,6 @@ def run_gen_ncc_id(attr): class TestGenNcclIdOp(unittest.TestCase): - def setUp(self): try: self._dist_ut_port_0 = int(os.environ["PADDLE_DIST_UT_PORT"]) @@ -98,7 +102,7 @@ def gen_nccl_id(self, nranks=2): for i in range(nranks): attr['trainer_id'] = i # NOTE: multiprocessing cannot be covered by coverage - p = Process(target=run_gen_ncc_id, args=(attr, )) + p = Process(target=run_gen_ncc_id, args=(attr,)) p.start() procs.append(p) diff --git a/python/paddle/fluid/tests/unittests/collective/test_new_group_api.py b/python/paddle/fluid/tests/unittests/collective/test_new_group_api.py index dfa0fb2402fd9..e70f7073c09d2 100644 --- a/python/paddle/fluid/tests/unittests/collective/test_new_group_api.py +++ b/python/paddle/fluid/tests/unittests/collective/test_new_group_api.py @@ -21,13 +21,13 @@ class TestCollectiveAllreduceAPI(TestDistBase): - def _setup_config(self): pass def test_allreduce_nccl(self): - self.check_with_place("collective_allreduce_new_group_api.py", - "allreduce", "nccl") + self.check_with_place( + "collective_allreduce_new_group_api.py", "allreduce", "nccl" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/collective/world_size_and_rank.py b/python/paddle/fluid/tests/unittests/collective/world_size_and_rank.py index 5916370f7533c..f6ae8444ef4f8 100644 --- a/python/paddle/fluid/tests/unittests/collective/world_size_and_rank.py +++ b/python/paddle/fluid/tests/unittests/collective/world_size_and_rank.py @@ -17,7 +17,6 @@ class TestWorldSizeAndRankAPI(unittest.TestCase): - def setUp(self): self._num_of_ranks = 2 self._subgroup_ranks = [0, 1] @@ -33,9 +32,11 @@ def test_given_group_world_size(self): self.assertEqual(dist.get_world_size(self._subgroup), world_size) def test_given_group_rank(self): - rank = self._subgroup_ranks.index( - self._global_rank - ) if self._global_rank in self._subgroup_ranks else -1 + rank = ( + self._subgroup_ranks.index(self._global_rank) + if self._global_rank in self._subgroup_ranks + else -1 + ) self.assertEqual(dist.get_rank(self._subgroup), rank) diff --git a/python/paddle/fluid/tests/unittests/collective_allgather_op.py b/python/paddle/fluid/tests/unittests/collective_allgather_op.py index e897e1638852f..ab39b834fdbc7 100644 --- a/python/paddle/fluid/tests/unittests/collective_allgather_op.py +++ b/python/paddle/fluid/tests/unittests/collective_allgather_op.py @@ -22,7 +22,6 @@ class TestCollectiveAllGather(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,26 +29,28 @@ def get_model(self, main_prog, startup_program): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofgather", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_allgather", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_allgather", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'nranks': nranks}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective_reducescatter.py b/python/paddle/fluid/tests/unittests/collective_reducescatter.py index 480f9aeb3805f..53f53046321d5 100644 --- a/python/paddle/fluid/tests/unittests/collective_reducescatter.py +++ b/python/paddle/fluid/tests/unittests/collective_reducescatter.py @@ -21,7 +21,6 @@ class TestCollectiveReduceScatter(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -29,9 +28,9 @@ def get_model(self, main_prog, startup_program): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = fluid.layers.collective._c_reducescatter(tindata, nranks) toutdata = fluid.layers.collective._c_sync_comm_stream(toutdata, 0) return toutdata diff --git a/python/paddle/fluid/tests/unittests/collective_reducescatter_op.py b/python/paddle/fluid/tests/unittests/collective_reducescatter_op.py index 042817f8c3f2c..1605c16183c73 100644 --- a/python/paddle/fluid/tests/unittests/collective_reducescatter_op.py +++ b/python/paddle/fluid/tests/unittests/collective_reducescatter_op.py @@ -22,7 +22,6 @@ class TestCollectiveReduceScatter(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,26 +29,28 @@ def get_model(self, main_prog, startup_program): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofrs", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_reducescatter", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_reducescatter", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'nranks': nranks}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/ctr_dataset_reader.py b/python/paddle/fluid/tests/unittests/ctr_dataset_reader.py index 981fe0f0ef53d..5cc96cf1c4e58 100644 --- a/python/paddle/fluid/tests/unittests/ctr_dataset_reader.py +++ b/python/paddle/fluid/tests/unittests/ctr_dataset_reader.py @@ -61,12 +61,10 @@ def load_lr_input_record(sent): class CtrReader(object): - def __init__(self): pass def _reader_creator(self, filelist): - def get_rand(low=0.0, high=1.0): return random.random() @@ -85,9 +83,7 @@ def reader(): class DatasetCtrReader(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def get_rand(low=0.0, high=1.0): return random.random() @@ -97,9 +93,10 @@ def iter(): dnn_input = load_dnn_input_record(fs[0]) lr_input = load_lr_input_record(fs[1]) click = [int(fs[2])] - yield ("dnn_data", dnn_input), \ - ("lr_data", lr_input), \ - ("click", click) + yield ("dnn_data", dnn_input), ("lr_data", lr_input), ( + "click", + click, + ) return iter @@ -115,7 +112,9 @@ def prepare_data(): lines = f.readlines() err_info = "wrong meta format" assert len(lines) == 2, err_info - assert 'dnn_input_dim:' in lines[0] and 'lr_input_dim:' in lines[1], err_info + assert ( + 'dnn_input_dim:' in lines[0] and 'lr_input_dim:' in lines[1] + ), err_info res = map(int, [_.split(':')[1] for _ in lines]) res = list(res) dnn_input_dim = res[0] @@ -126,10 +125,9 @@ def prepare_data(): return dnn_input_dim, lr_input_dim, train_file_path -def gen_fake_line(dnn_data_num=7, - dnn_data_range=1e5, - lr_data_num=5, - lr_data_range=1e5): +def gen_fake_line( + dnn_data_num=7, dnn_data_range=1e5, lr_data_num=5, lr_data_range=1e5 +): line = "" # for deep data @@ -188,16 +186,17 @@ def prepare_fake_data(file_nums=4, file_lines=500): warnings.warn("Fake data write in {}".format(file_dir)) for file_index in range(file_nums): with open( - os.path.join(file_dir, - "ctr_train_data_part_{}".format(file_index)), - 'w+') as fin: + os.path.join(file_dir, "ctr_train_data_part_{}".format(file_index)), + 'w+', + ) as fin: file_str = "" file_str += gen_zero_line() for line_index in range(file_lines - 1): file_str += gen_fake_line() fin.write(file_str) warnings.warn( - "Write done ctr_train_data_part_{}".format(file_index)) + "Write done ctr_train_data_part_{}".format(file_index) + ) file_list = [os.path.join(file_dir, x) for x in os.listdir(file_dir)] assert len(file_list) == file_nums diff --git a/python/paddle/fluid/tests/unittests/decorator_helper.py b/python/paddle/fluid/tests/unittests/decorator_helper.py index 0d7887d9189d1..d1165e2a91994 100644 --- a/python/paddle/fluid/tests/unittests/decorator_helper.py +++ b/python/paddle/fluid/tests/unittests/decorator_helper.py @@ -18,9 +18,7 @@ def many_times(times): - def __impl__(fn): - def __fn__(*args, **kwargs): for _ in range(times): fn(*args, **kwargs) @@ -31,9 +29,7 @@ def __fn__(*args, **kwargs): def prog_scope(): - def __impl__(fn): - def __fn__(*args, **kwargs): prog = fluid.Program() startup_prog = fluid.Program() diff --git a/python/paddle/fluid/tests/unittests/detected_gpu.py b/python/paddle/fluid/tests/unittests/detected_gpu.py index 47287278ed96d..e77c6f5c8ffe2 100644 --- a/python/paddle/fluid/tests/unittests/detected_gpu.py +++ b/python/paddle/fluid/tests/unittests/detected_gpu.py @@ -18,8 +18,10 @@ print("compile with cuda:", fluid.core.is_compiled_with_cuda()) print("get_cuda_device_count:", fluid.core.get_cuda_device_count()) -if fluid.core.is_compiled_with_cuda( -) and fluid.core.get_cuda_device_count() > 0: +if ( + fluid.core.is_compiled_with_cuda() + and fluid.core.get_cuda_device_count() > 0 +): sys.exit(0) else: sys.exit(1) diff --git a/python/paddle/fluid/tests/unittests/dist_allreduce_op.py b/python/paddle/fluid/tests/unittests/dist_allreduce_op.py index 49ef5d64bc73b..af769963158c2 100644 --- a/python/paddle/fluid/tests/unittests/dist_allreduce_op.py +++ b/python/paddle/fluid/tests/unittests/dist_allreduce_op.py @@ -35,8 +35,10 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_pool_2 = fluid.nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -44,25 +46,28 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) SIZE = 10 input_shape = conv_pool_2.shape param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 predict = fluid.layers.fc( input=conv_pool_2, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) return predict class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2, single_device=False): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -75,17 +80,19 @@ def get_model(self, batch_size=2, single_device=False): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) # Optimization # TODO(typhoonzero): fix distributed adam optimizer @@ -104,7 +111,14 @@ def get_model(self, batch_size=2, single_device=False): data_parallel_param_grads.append([p, grad_reduce]) opt.apply_gradients(data_parallel_param_grads) - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_ctr.py b/python/paddle/fluid/tests/unittests/dist_ctr.py index b5578eca6ad08..ab10ca2d63997 100644 --- a/python/paddle/fluid/tests/unittests/dist_ctr.py +++ b/python/paddle/fluid/tests/unittests/dist_ctr.py @@ -28,26 +28,31 @@ class TestDistCTR2x2(TestDistRunnerBase): - def get_model(self, batch_size=2): dnn_input_dim, lr_input_dim = dist_ctr_reader.load_data_meta() """ network definition """ - dnn_data = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - lr_data = fluid.layers.data(name="lr_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="int64", - lod_level=0, - append_batch_size=False) + dnn_data = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + lr_data = fluid.layers.data( + name="lr_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=0, + append_batch_size=False, + ) # build dnn model dnn_layer_dims = [128, 64, 32, 1] @@ -57,10 +62,13 @@ def get_model(self, batch_size=2): size=[dnn_input_dim, dnn_layer_dims[0]], param_attr=fluid.ParamAttr( name="deep_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=IS_SPARSE) - dnn_pool = fluid.layers.sequence_pool(input=dnn_embedding, - pool_type="sum") + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=IS_SPARSE, + ) + dnn_pool = fluid.layers.sequence_pool( + input=dnn_embedding, pool_type="sum" + ) dnn_out = dnn_pool for i, dim in enumerate(dnn_layer_dims[1:]): fc = fluid.layers.fc( @@ -68,8 +76,10 @@ def get_model(self, batch_size=2): size=dim, act="relu", param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01)), - name='dnn-fc-%d' % i) + initializer=fluid.initializer.Constant(value=0.01) + ), + name='dnn-fc-%d' % i, + ) dnn_out = fc # build lr model @@ -79,16 +89,19 @@ def get_model(self, batch_size=2): size=[lr_input_dim, 1], param_attr=fluid.ParamAttr( name="wide_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=IS_SPARSE) + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=IS_SPARSE, + ) lr_pool = fluid.layers.sequence_pool(input=lr_embbding, pool_type="sum") merge_layer = fluid.layers.concat(input=[dnn_out, lr_pool], axis=1) predict = fluid.layers.fc(input=merge_layer, size=2, act='softmax') acc = fluid.layers.accuracy(input=predict, label=label) - auc_var, batch_auc_var, auc_states = fluid.layers.auc(input=predict, - label=label) + auc_var, batch_auc_var, auc_states = fluid.layers.auc( + input=predict, label=label + ) cost = fluid.layers.cross_entropy(input=predict, label=label) avg_cost = paddle.mean(x=cost) @@ -98,24 +111,35 @@ def get_model(self, batch_size=2): use_l2_decay = bool(os.getenv('USE_L2_DECAY', 0)) if use_l2_decay: regularization = fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-1) + regularization_coeff=1e-1 + ) use_lr_decay = bool(os.getenv('LR_DECAY', 0)) lr = 0.0001 if use_lr_decay: - lr = fluid.layers.exponential_decay(learning_rate=0.0001, - decay_steps=10000, - decay_rate=0.999, - staircase=True) - - sgd_optimizer = fluid.optimizer.SGD(learning_rate=lr, - regularization=regularization) + lr = fluid.layers.exponential_decay( + learning_rate=0.0001, + decay_steps=10000, + decay_rate=0.999, + staircase=True, + ) + + sgd_optimizer = fluid.optimizer.SGD( + learning_rate=lr, regularization=regularization + ) sgd_optimizer.minimize(avg_cost) dataset = dist_ctr_reader.Dataset() train_reader = paddle.batch(dataset.train(), batch_size=batch_size) test_reader = paddle.batch(dataset.test(), batch_size=batch_size) - return inference_program, avg_cost, train_reader, test_reader, None, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + None, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_ctr_reader.py b/python/paddle/fluid/tests/unittests/dist_ctr_reader.py index 4bc231e4eafe6..fafb5f19c5f7d 100644 --- a/python/paddle/fluid/tests/unittests/dist_ctr_reader.py +++ b/python/paddle/fluid/tests/unittests/dist_ctr_reader.py @@ -109,7 +109,6 @@ def load_lr_input_record(sent): class Dataset(object): - def train(self): ''' Load trainset. @@ -164,7 +163,9 @@ def load_data_meta(): lines = read_data('data.meta.txt') err_info = "wrong meta format" assert len(lines) == 2, err_info - assert 'dnn_input_dim:' in lines[0] and 'lr_input_dim:' in lines[1], err_info + assert ( + 'dnn_input_dim:' in lines[0] and 'lr_input_dim:' in lines[1] + ), err_info res = map(int, [_.split(':')[1] for _ in lines]) res = list(res) logger.info('dnn input dim: %d' % res[0]) diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py b/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py index 53ef00064bfca..dd01938556f14 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py @@ -35,7 +35,6 @@ def fake_ctr_reader(): - def reader(): for _ in range(1000): deep = np.random.random_integers(0, 1e5 - 1, size=16).tolist() @@ -63,38 +62,47 @@ def net(self, args, is_train=True, batch_size=4, lr=0.01): """ dnn_input_dim, lr_input_dim = int(1e5), int(1e5) - dnn_data = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - lr_data = fluid.layers.data(name="lr_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="int64", - lod_level=0, - append_batch_size=False) + dnn_data = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + lr_data = fluid.layers.data( + name="lr_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=0, + append_batch_size=False, + ) datas = [dnn_data, lr_data, label] if args.reader == "pyreader": if is_train: - self.reader = fluid.io.PyReader(feed_list=datas, - capacity=64, - iterable=False, - use_double_buffer=False) + self.reader = fluid.io.PyReader( + feed_list=datas, + capacity=64, + iterable=False, + use_double_buffer=False, + ) else: - self.test_reader = fluid.io.PyReader(feed_list=datas, - capacity=64, - iterable=False, - use_double_buffer=False) - - -# build dnn model + self.test_reader = fluid.io.PyReader( + feed_list=datas, + capacity=64, + iterable=False, + use_double_buffer=False, + ) + + # build dnn model dnn_layer_dims = [128, 128, 64, 32, 1] dnn_embedding = fluid.layers.embedding( is_distributed=False, @@ -102,11 +110,14 @@ def net(self, args, is_train=True, batch_size=4, lr=0.01): size=[dnn_input_dim, dnn_layer_dims[0]], param_attr=fluid.ParamAttr( name="deep_embedding", - initializer=fluid.initializer.Constant(value=0.01)), + initializer=fluid.initializer.Constant(value=0.01), + ), is_sparse=True, - padding_idx=0) - dnn_pool = fluid.layers.sequence_pool(input=dnn_embedding, - pool_type="sum") + padding_idx=0, + ) + dnn_pool = fluid.layers.sequence_pool( + input=dnn_embedding, pool_type="sum" + ) dnn_out = dnn_pool for i, dim in enumerate(dnn_layer_dims[1:]): fc = fluid.layers.fc( @@ -114,8 +125,10 @@ def net(self, args, is_train=True, batch_size=4, lr=0.01): size=dim, act="relu", param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01)), - name='dnn-fc-%d' % i) + initializer=fluid.initializer.Constant(value=0.01) + ), + name='dnn-fc-%d' % i, + ) dnn_out = fc # build lr model @@ -125,9 +138,11 @@ def net(self, args, is_train=True, batch_size=4, lr=0.01): size=[lr_input_dim, 1], param_attr=fluid.ParamAttr( name="wide_embedding", - initializer=fluid.initializer.Constant(value=0.01)), + initializer=fluid.initializer.Constant(value=0.01), + ), is_sparse=True, - padding_idx=0) + padding_idx=0, + ) lr_pool = fluid.layers.sequence_pool(input=lr_embbding, pool_type="sum") merge_layer = fluid.layers.concat(input=[dnn_out, lr_pool], axis=1) @@ -135,8 +150,9 @@ def net(self, args, is_train=True, batch_size=4, lr=0.01): predict = fluid.layers.fc(input=merge_layer, size=2, act='softmax') acc = fluid.layers.accuracy(input=predict, label=label) - auc_var, batch_auc_var, auc_states = fluid.layers.auc(input=predict, - label=label) + auc_var, batch_auc_var, auc_states = fluid.layers.auc( + input=predict, label=label + ) cost = fluid.layers.cross_entropy(input=predict, label=label) avg_cost = paddle.mean(x=cost) @@ -176,11 +192,14 @@ def do_distributed_testing(self, fleet): try: while True: batch_idx += 1 - loss_val = exe.run(program=paddle.static.default_main_program(), - fetch_list=[self.avg_cost.name]) + loss_val = exe.run( + program=paddle.static.default_main_program(), + fetch_list=[self.avg_cost.name], + ) loss_val = np.mean(loss_val) message = "TEST ---> batch_idx: {} loss: {}\n".format( - batch_idx, loss_val) + batch_idx, loss_val + ) fleet.util.print_on_rank(message, 0) except fluid.core.EOFException: self.test_reader.reset() @@ -208,8 +227,10 @@ def do_pyreader_training(self, fleet): try: pass_start = time.time() while True: - loss_val = exe.run(program=fluid.default_main_program(), - fetch_list=[self.avg_cost.name]) + loss_val = exe.run( + program=fluid.default_main_program(), + fetch_list=[self.avg_cost.name], + ) loss_val = np.mean(loss_val) # TODO(randomly fail) # reduce_output = fleet.util.all_reduce( @@ -217,7 +238,8 @@ def do_pyreader_training(self, fleet): # loss_all_trainer = fleet.util.all_gather(float(loss_val)) # loss_val = float(reduce_output) / len(loss_all_trainer) message = "TRAIN ---> pass: {} loss: {}\n".format( - epoch_id, loss_val) + epoch_id, loss_val + ) fleet.util.print_on_rank(message, 0) pass_time = time.time() - pass_start @@ -229,9 +251,9 @@ def do_pyreader_training(self, fleet): fleet.save_persistables(exe, dirname=dirname) model_dir = tempfile.mkdtemp() - fleet.save_inference_model(exe, model_dir, - [feed.name for feed in self.feeds], - self.avg_cost) + fleet.save_inference_model( + exe, model_dir, [feed.name for feed in self.feeds], self.avg_cost + ) if fleet.is_first_worker(): self.check_model_right(model_dir) shutil.rmtree(model_dir) @@ -251,29 +273,36 @@ def do_dataset_training_queuedataset(self, fleet): dataset = paddle.distributed.QueueDataset() pipe_command = 'python ctr_dataset_reader.py' - dataset.init(batch_size=batch_size, - use_var=self.feeds, - pipe_command=pipe_command, - thread_num=thread_num) + dataset.init( + batch_size=batch_size, + use_var=self.feeds, + pipe_command=pipe_command, + thread_num=thread_num, + ) dataset.set_filelist(filelist) for epoch_id in range(1): pass_start = time.time() dataset.set_filelist(filelist) - exe.train_from_dataset(program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[self.avg_cost], - fetch_info=["cost"], - print_period=2, - debug=int(os.getenv("Debug", "0"))) + exe.train_from_dataset( + program=fluid.default_main_program(), + dataset=dataset, + fetch_list=[self.avg_cost], + fetch_info=["cost"], + print_period=2, + debug=int(os.getenv("Debug", "0")), + ) pass_time = time.time() - pass_start if os.getenv("SAVE_MODEL") == "1": model_dir = tempfile.mkdtemp() - fleet.save_inference_model(exe, model_dir, - [feed.name for feed in self.feeds], - self.avg_cost) + fleet.save_inference_model( + exe, + model_dir, + [feed.name for feed in self.feeds], + self.avg_cost, + ) if fleet.is_first_worker(): self.check_model_right(model_dir) shutil.rmtree(model_dir) @@ -311,20 +340,25 @@ def do_dataset_training(self, fleet): for epoch_id in range(1): pass_start = time.time() - exe.train_from_dataset(program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[self.avg_cost], - fetch_info=["cost"], - print_period=2, - debug=int(os.getenv("Debug", "0"))) + exe.train_from_dataset( + program=fluid.default_main_program(), + dataset=dataset, + fetch_list=[self.avg_cost], + fetch_info=["cost"], + print_period=2, + debug=int(os.getenv("Debug", "0")), + ) pass_time = time.time() - pass_start dataset.release_memory() if os.getenv("SAVE_MODEL") == "1": model_dir = tempfile.mkdtemp() - fleet.save_inference_model(exe, model_dir, - [feed.name for feed in self.feeds], - self.avg_cost) + fleet.save_inference_model( + exe, + model_dir, + [feed.name for feed in self.feeds], + self.avg_cost, + ) fleet.load_inference_model(model_dir, mode=0) if fleet.is_first_worker(): self.check_model_right(model_dir) @@ -341,9 +375,12 @@ def do_dataset_training(self, fleet): dense_param_dirname = os.getenv("SAVE_DENSE_PARAM_DIRNAME", None) if dense_param_dirname: - fleet.save_dense_params(exe, dense_param_dirname, - fluid.global_scope(), - fluid.default_main_program()) + fleet.save_dense_params( + exe, + dense_param_dirname, + fluid.global_scope(), + fluid.default_main_program(), + ) save_one_table_dirname = os.getenv("SAVE_ONE_TABLE_DIRNAME", None) if save_one_table_dirname: @@ -355,5 +392,6 @@ def do_dataset_training(self, fleet): fleet.save_persistables(exe, patch_dirname, None, 5) fleet.check_save_pre_patch_done() + if __name__ == "__main__": runtime_main(TestDistCTR2x2) diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_ctr_ps_gpu.py b/python/paddle/fluid/tests/unittests/dist_fleet_ctr_ps_gpu.py index 13ec262697e5d..62b41a3f45548 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_ctr_ps_gpu.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_ctr_ps_gpu.py @@ -70,15 +70,19 @@ def do_pyreader_training(self, fleet): try: pass_start = time.time() while True: - loss_val = exe.run(program=fleet.main_program, - fetch_list=[self.avg_cost.name]) + loss_val = exe.run( + program=fleet.main_program, + fetch_list=[self.avg_cost.name], + ) loss_val = np.mean(loss_val) - reduce_output = fleet.util.all_reduce(np.array(loss_val), - mode="sum") + reduce_output = fleet.util.all_reduce( + np.array(loss_val), mode="sum" + ) loss_all_trainer = fleet.util.all_gather(float(loss_val)) loss_val = float(reduce_output) / len(loss_all_trainer) message = "TRAIN ---> pass: {} loss: {}\n".format( - epoch_id, loss_val) + epoch_id, loss_val + ) fleet.util.print_on_rank(message, 0) pass_time = time.time() - pass_start @@ -86,9 +90,9 @@ def do_pyreader_training(self, fleet): self.reader.reset() model_dir = tempfile.mkdtemp() - fleet.save_inference_model(exe, model_dir, - [feed.name for feed in self.feeds], - self.avg_cost) + fleet.save_inference_model( + exe, model_dir, [feed.name for feed in self.feeds], self.avg_cost + ) if fleet.is_first_worker(): self.check_model_right(model_dir) if fleet.is_first_worker(): @@ -96,8 +100,11 @@ def do_pyreader_training(self, fleet): shutil.rmtree(model_dir) def do_dataset_training(self, fleet): - dnn_input_dim, lr_input_dim, train_file_path = ctr_dataset_reader.prepare_data( - ) + ( + dnn_input_dim, + lr_input_dim, + train_file_path, + ) = ctr_dataset_reader.prepare_data() device_id = int(os.getenv("FLAGS_selected_gpus", "0")) place = fluid.CUDAPlace(device_id) @@ -125,19 +132,24 @@ def do_dataset_training(self, fleet): for epoch_id in range(1): pass_start = time.time() dataset.set_filelist(filelist) - exe.train_from_dataset(program=fleet.main_program, - dataset=dataset, - fetch_list=[self.avg_cost], - fetch_info=["cost"], - print_period=2, - debug=int(os.getenv("Debug", "0"))) + exe.train_from_dataset( + program=fleet.main_program, + dataset=dataset, + fetch_list=[self.avg_cost], + fetch_info=["cost"], + print_period=2, + debug=int(os.getenv("Debug", "0")), + ) pass_time = time.time() - pass_start if os.getenv("SAVE_MODEL") == "1": model_dir = tempfile.mkdtemp() - fleet.save_inference_model(exe, model_dir, - [feed.name for feed in self.feeds], - self.avg_cost) + fleet.save_inference_model( + exe, + model_dir, + [feed.name for feed in self.feeds], + self.avg_cost, + ) if fleet.is_first_worker(): self.check_model_right(model_dir) if fleet.is_first_worker(): diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_debug_gloo.py b/python/paddle/fluid/tests/unittests/dist_fleet_debug_gloo.py index 82f7a2bc672ed..5b0c66538bbd7 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_debug_gloo.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_debug_gloo.py @@ -13,21 +13,24 @@ # limitations under the License. import logging -#import paddle.fluid.incubate.fleet.base.role_maker as role_maker + +# import paddle.fluid.incubate.fleet.base.role_maker as role_maker import paddle.distributed.fleet.base.role_maker as role_maker -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, +) logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") logger = logging.getLogger("fluid") logger.setLevel(logging.INFO) -#role = role_maker.GeneralRoleMaker( -#init_timeout_seconds=100, -#run_timeout_seconds=100, -#http_ip_port="127.0.0.1:26001") +# role = role_maker.GeneralRoleMaker( +# init_timeout_seconds=100, +# run_timeout_seconds=100, +# http_ip_port="127.0.0.1:26001") -#role = role_maker.PaddleCloudRoleMaker(http_ip_port="127.0.0.1:26001") +# role = role_maker.PaddleCloudRoleMaker(http_ip_port="127.0.0.1:26001") -#role = role_maker.GeneralRoleMaker(path="./tmp4") +# role = role_maker.GeneralRoleMaker(path="./tmp4") logger.info("Begin") res = [0, 0] @@ -37,7 +40,7 @@ fleet.init(role) print("init wancheng") # -#if fleet.is_worker(): +# if fleet.is_worker(): # import time # time.sleep(3) @@ -49,7 +52,7 @@ elif fleet.worker_index() == 1: role._all_reduce(role._node_type_comm, b) -#logger.info(res) -#print("res ", res) +# logger.info(res) +# print("res ", res) -#role._barrier_all() +# role._barrier_all() diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_heter_pipeline_ctr.py b/python/paddle/fluid/tests/unittests/dist_fleet_heter_pipeline_ctr.py index c4e78ba16e05e..5f3aaf4efdc8e 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_heter_pipeline_ctr.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_heter_pipeline_ctr.py @@ -49,21 +49,27 @@ def net(self, args, batch_size=4, lr=0.01): dnn_input_dim, lr_input_dim = int(1e5), int(1e5) with fluid.device_guard("cpu"): - dnn_data = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - lr_data = fluid.layers.data(name="lr_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="float32", - lod_level=0, - append_batch_size=False) + dnn_data = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + lr_data = fluid.layers.data( + name="lr_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="float32", + lod_level=0, + append_batch_size=False, + ) datas = [dnn_data, lr_data, label] @@ -75,10 +81,13 @@ def net(self, args, batch_size=4, lr=0.01): size=[dnn_input_dim, dnn_layer_dims[0]], param_attr=fluid.ParamAttr( name="deep_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=True) - dnn_pool = fluid.layers.sequence_pool(input=dnn_embedding, - pool_type="sum") + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) + dnn_pool = fluid.layers.sequence_pool( + input=dnn_embedding, pool_type="sum" + ) dnn_out = dnn_pool # build lr model @@ -88,10 +97,13 @@ def net(self, args, batch_size=4, lr=0.01): size=[lr_input_dim, 1], param_attr=fluid.ParamAttr( name="wide_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=True) - lr_pool = fluid.layers.sequence_pool(input=lr_embbding, - pool_type="sum") + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) + lr_pool = fluid.layers.sequence_pool( + input=lr_embbding, pool_type="sum" + ) with fluid.device_guard("gpu"): for i, dim in enumerate(dnn_layer_dims[1:]): @@ -100,8 +112,10 @@ def net(self, args, batch_size=4, lr=0.01): size=dim, act="relu", param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01)), - name='dnn-fc-%d' % i) + initializer=fluid.initializer.Constant(value=0.01) + ), + name='dnn-fc-%d' % i, + ) dnn_out = fc with fluid.device_guard("cpu"): @@ -135,8 +149,9 @@ def do_dataset_training(self, fleet): train_file_list = ctr_dataset_reader.prepare_fake_data() exe = fluid.Executor(fluid.CPUPlace()) - real_program = fluid.default_main_program( - )._heter_pipeline_opt["section_program"] + real_program = fluid.default_main_program()._heter_pipeline_opt[ + "section_program" + ] print(real_program) exe.run(fluid.default_startup_program()) @@ -161,12 +176,14 @@ def do_dataset_training(self, fleet): for epoch_id in range(1): pass_start = time.time() dataset.set_filelist(filelist) - exe.train_from_dataset(program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[self.avg_cost], - fetch_info=["cost"], - print_period=2, - debug=int(os.getenv("Debug", "0"))) + exe.train_from_dataset( + program=fluid.default_main_program(), + dataset=dataset, + fetch_list=[self.avg_cost], + fetch_info=["cost"], + print_period=2, + debug=int(os.getenv("Debug", "0")), + ) pass_time = time.time() - pass_start print("do_dataset_training done. using time {}".format(pass_time)) exe.close() @@ -176,24 +193,27 @@ def do_dataset_heter_training(self, fleet): exe = fluid.Executor() exe.run(fluid.default_startup_program()) fleet.init_worker() - real_program = fluid.default_main_program( - )._heter_pipeline_opt["section_program"] + real_program = fluid.default_main_program()._heter_pipeline_opt[ + "section_program" + ] print(real_program) thread_num = int(os.getenv("CPU_NUM", 2)) batch_size = 128 pass_start = time.time() - exe.train_from_dataset(program=fluid.default_main_program(), - fetch_list=[self.avg_cost], - fetch_info=["cost"], - print_period=2, - debug=int(os.getenv("Debug", "0"))) + exe.train_from_dataset( + program=fluid.default_main_program(), + fetch_list=[self.avg_cost], + fetch_info=["cost"], + print_period=2, + debug=int(os.getenv("Debug", "0")), + ) exe.close() pass_time = time.time() - pass_start print("do_dataset_heter_training done. using time {}".format(pass_time)) - #for epoch_id in range(1): + # for epoch_id in range(1): # pass_start = time.time() # dataset.set_filelist(filelist) # exe.train_from_dataset( diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer.py b/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer.py index 54c3782225777..fabf762f63b1d 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer.py @@ -37,8 +37,10 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_pool_2 = fluid.nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -46,25 +48,28 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) SIZE = 10 input_shape = conv_pool_2.shape param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 predict = fluid.layers.fc( input=conv_pool_2, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) return predict class TestFleetMetaOptimizerPrecision(TestDistRunnerBase): - def get_model(self, batch_size=2, single_device=False): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -77,17 +82,19 @@ def get_model(self, batch_size=2, single_device=False): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) test_program = fluid.default_main_program().clone(for_test=True) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) optimizer = paddle.fluid.optimizer.Adam(0.01) if single_device: @@ -97,11 +104,19 @@ def get_model(self, batch_size=2, single_device=False): fleet.init(role) strategy = paddle.distributed.fleet.DistributedStrategy() strategy.without_graph_optimization = True - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) - return test_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer_fuse_allreduce.py b/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer_fuse_allreduce.py index 5db0377b36a0e..cf6a1415bcb76 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer_fuse_allreduce.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_raw_program_optimizer_fuse_allreduce.py @@ -37,8 +37,10 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_pool_2 = fluid.nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -46,25 +48,28 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) SIZE = 10 input_shape = conv_pool_2.shape param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 predict = fluid.layers.fc( input=conv_pool_2, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) return predict class TestFleetMetaOptimizerFuseAllReducePrecision(TestDistRunnerBase): - def get_model(self, batch_size=2, single_device=False): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -77,17 +82,19 @@ def get_model(self, batch_size=2, single_device=False): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) test_program = fluid.default_main_program().clone(for_test=True) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) optimizer = paddle.fluid.optimizer.Adam(0.01) if single_device: @@ -100,11 +107,19 @@ def get_model(self, batch_size=2, single_device=False): strategy.fuse_all_reduce_ops = True strategy._calc_comm_same_stream = False strategy.fuse_grad_size_in_num = 8 - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) - return test_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_simnet_bow.py b/python/paddle/fluid/tests/unittests/dist_fleet_simnet_bow.py index 84687999d9c50..f9d926ad1c733 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_simnet_bow.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_simnet_bow.py @@ -41,7 +41,6 @@ def fake_simnet_reader(): - def reader(): for _ in range(1000): q = np.random.random_integers(0, 1500 - 1, size=1).tolist() @@ -57,62 +56,66 @@ def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost -def train_network(batch_size, - is_distributed=False, - is_sparse=False, - is_self_contained_lr=False, - is_pyreader=False): +def train_network( + batch_size, + is_distributed=False, + is_sparse=False, + is_self_contained_lr=False, + is_pyreader=False, +): # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) datas = [q, label, pt, nt] reader = None if is_pyreader: - reader = fluid.io.PyReader(feed_list=datas, - capacity=64, - iterable=False, - use_double_buffer=False) + reader = fluid.io.PyReader( + feed_list=datas, + capacity=64, + iterable=False, + use_double_buffer=False, + ) # embedding q_emb = fluid.embedding( @@ -120,8 +123,10 @@ def train_network(batch_size, is_distributed=is_distributed, size=[dict_dim, emb_dim], param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01), name="__emb__"), - is_sparse=is_sparse) + initializer=fluid.initializer.Constant(value=0.01), name="__emb__" + ), + is_sparse=is_sparse, + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -133,7 +138,8 @@ def train_network(batch_size, param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr), + learning_rate=base_lr, + ), ) # embedding @@ -144,8 +150,10 @@ def train_network(batch_size, param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -155,8 +163,10 @@ def train_network(batch_size, input=pt_ss, size=hid_dim, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01), name="__fc__"), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + initializer=fluid.initializer.Constant(value=0.01), name="__fc__" + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # embedding nt_emb = fluid.embedding( @@ -164,8 +174,10 @@ def train_network(batch_size, is_distributed=is_distributed, size=[dict_dim, emb_dim], param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01), name="__emb__"), - is_sparse=is_sparse) + initializer=fluid.initializer.Constant(value=0.01), name="__emb__" + ), + is_sparse=is_sparse, + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -175,8 +187,10 @@ def train_network(batch_size, input=nt_ss, size=hid_dim, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01), name="__fc__"), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + initializer=fluid.initializer.Constant(value=0.01), name="__fc__" + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -192,9 +206,13 @@ class TestDistSimnetBow2x2(FleetDistRunnerBase): """ def net(self, args, batch_size=4, lr=0.01): - avg_cost, _, predict, self.reader = \ - train_network(batch_size=batch_size, is_distributed=False, - is_sparse=True, is_self_contained_lr=False, is_pyreader=(args.reader == "pyreader")) + avg_cost, _, predict, self.reader = train_network( + batch_size=batch_size, + is_distributed=False, + is_sparse=True, + is_self_contained_lr=False, + is_pyreader=(args.reader == "pyreader"), + ) self.avg_cost = avg_cost self.predict = predict @@ -229,11 +247,14 @@ def do_pyreader_training(self, fleet): try: pass_start = time.time() while True: - loss_val = exe.run(program=fluid.default_main_program(), - fetch_list=[self.avg_cost.name]) + loss_val = exe.run( + program=fluid.default_main_program(), + fetch_list=[self.avg_cost.name], + ) loss_val = np.mean(loss_val) message = "TRAIN ---> pass: {} loss: {}\n".format( - epoch_id, loss_val) + epoch_id, loss_val + ) fleet.util.print_on_rank(message, 0) pass_time = time.time() - pass_start diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_sparse_embedding_ctr.py b/python/paddle/fluid/tests/unittests/dist_fleet_sparse_embedding_ctr.py index c6a976b089b4a..c1410348e0fc4 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_sparse_embedding_ctr.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_sparse_embedding_ctr.py @@ -26,7 +26,6 @@ def fake_ctr_reader(): - def reader(): for _ in range(1000): deep = np.random.random_integers(0, 1e10, size=16).tolist() @@ -54,29 +53,37 @@ def net(self, args, batch_size=4, lr=0.01): """ dnn_input_dim, lr_input_dim = 10, 10 - dnn_data = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - lr_data = fluid.layers.data(name="lr_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="int64", - lod_level=0, - append_batch_size=False) + dnn_data = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + lr_data = fluid.layers.data( + name="lr_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=0, + append_batch_size=False, + ) datas = [dnn_data, lr_data, label] if args.reader == "pyreader": - self.reader = fluid.io.PyReader(feed_list=datas, - capacity=64, - iterable=False, - use_double_buffer=False) + self.reader = fluid.io.PyReader( + feed_list=datas, + capacity=64, + iterable=False, + use_double_buffer=False, + ) # build dnn model initializer = int(os.getenv("INITIALIZER", "0")) @@ -98,9 +105,11 @@ def net(self, args, batch_size=4, lr=0.01): size=[dnn_input_dim, dnn_layer_dims[0]], is_test=inference, entry=entry, - param_attr=fluid.ParamAttr(name="deep_embedding", initializer=init)) - dnn_pool = fluid.layers.sequence_pool(input=dnn_embedding, - pool_type="sum") + param_attr=fluid.ParamAttr(name="deep_embedding", initializer=init), + ) + dnn_pool = fluid.layers.sequence_pool( + input=dnn_embedding, pool_type="sum" + ) dnn_out = dnn_pool for i, dim in enumerate(dnn_layer_dims[1:]): fc = fluid.layers.fc( @@ -108,8 +117,10 @@ def net(self, args, batch_size=4, lr=0.01): size=dim, act="relu", param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01)), - name='dnn-fc-%d' % i) + initializer=fluid.initializer.Constant(value=0.01) + ), + name='dnn-fc-%d' % i, + ) dnn_out = fc # build lr model @@ -120,7 +131,9 @@ def net(self, args, batch_size=4, lr=0.01): entry=entry, param_attr=fluid.ParamAttr( name="wide_embedding", - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01), + ), + ) lr_pool = fluid.layers.sequence_pool(input=lr_embbding, pool_type="sum") merge_layer = fluid.layers.concat(input=[dnn_out, lr_pool], axis=1) @@ -159,19 +172,27 @@ def do_pyreader_training(self, fleet): self.reader.start() try: while True: - loss_val = exe.run(program=fluid.default_main_program(), - fetch_list=[self.avg_cost.name]) + loss_val = exe.run( + program=fluid.default_main_program(), + fetch_list=[self.avg_cost.name], + ) loss_val = np.mean(loss_val) - print("TRAIN ---> pass: {} loss: {}\n".format( - epoch_id, loss_val)) + print( + "TRAIN ---> pass: {} loss: {}\n".format( + epoch_id, loss_val + ) + ) except fluid.core.EOFException: self.reader.reset() model_dir = os.getenv("MODEL_DIR", None) if model_dir: - fleet.save_inference_model(exe, model_dir, - [feed.name for feed in self.feeds], - self.avg_cost) + fleet.save_inference_model( + exe, + model_dir, + [feed.name for feed in self.feeds], + self.avg_cost, + ) fleet.load_model(model_dir, mode=1) diff --git a/python/paddle/fluid/tests/unittests/dist_mnist.py b/python/paddle/fluid/tests/unittests/dist_mnist.py index 1a0f61411a3ac..856ac1b930bbf 100644 --- a/python/paddle/fluid/tests/unittests/dist_mnist.py +++ b/python/paddle/fluid/tests/unittests/dist_mnist.py @@ -36,8 +36,10 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_pool_2 = fluid.nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -45,25 +47,28 @@ def cnn_model(data): pool_size=2, pool_stride=2, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) SIZE = 10 input_shape = conv_pool_2.shape param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 predict = fluid.layers.fc( input=conv_pool_2, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01) + ), + ) return predict class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -76,9 +81,9 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() # Optimization @@ -88,24 +93,34 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): if not use_dgc: opt = fluid.optimizer.Momentum(learning_rate=self.lr, momentum=0.9) else: - opt = fluid.optimizer.DGCMomentumOptimizer(learning_rate=self.lr, - momentum=0.9, - rampup_begin_step=2) + opt = fluid.optimizer.DGCMomentumOptimizer( + learning_rate=self.lr, momentum=0.9, rampup_begin_step=2 + ) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) if dist_strategy: - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=dist_strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=dist_strategy + ) _, param_grads = dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_mnist_batch_merge.py b/python/paddle/fluid/tests/unittests/dist_mnist_batch_merge.py index 67981130fc265..1e022a1c44518 100644 --- a/python/paddle/fluid/tests/unittests/dist_mnist_batch_merge.py +++ b/python/paddle/fluid/tests/unittests/dist_mnist_batch_merge.py @@ -35,7 +35,6 @@ def test_merge_reader(repeat_batch_size=8): class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -48,9 +47,9 @@ def get_model(self, batch_size=2): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() # Optimization @@ -58,10 +57,18 @@ def get_model(self, batch_size=2): # Reader train_reader = paddle.batch(test_merge_reader, batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) opt.minimize(avg_cost) - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_mnist_fp16_allreduce.py b/python/paddle/fluid/tests/unittests/dist_mnist_fp16_allreduce.py index c9d37247b7767..c0d92d01e6c6a 100644 --- a/python/paddle/fluid/tests/unittests/dist_mnist_fp16_allreduce.py +++ b/python/paddle/fluid/tests/unittests/dist_mnist_fp16_allreduce.py @@ -14,7 +14,9 @@ import paddle import paddle.fluid as fluid -from paddle.distributed.fleet.meta_optimizers import FP16AllReduceOptimizer as FP16AllReduce +from paddle.distributed.fleet.meta_optimizers import ( + FP16AllReduceOptimizer as FP16AllReduce, +) from test_dist_base import TestDistRunnerBase, runtime_main from dist_mnist import cnn_model @@ -27,7 +29,6 @@ class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -40,23 +41,33 @@ def get_model(self, batch_size=2): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() # Optimization - opt = fluid.optimizer.MomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.MomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) opt = FP16AllReduce(opt) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) opt.minimize(avg_cost) - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_mnist_lars.py b/python/paddle/fluid/tests/unittests/dist_mnist_lars.py index b93dfb0594eda..c117efb84f830 100644 --- a/python/paddle/fluid/tests/unittests/dist_mnist_lars.py +++ b/python/paddle/fluid/tests/unittests/dist_mnist_lars.py @@ -26,7 +26,6 @@ class TestDistMnist2x2(TestDistRunnerBase): - def get_model(self, batch_size=2): # Input data images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) @@ -39,22 +38,32 @@ def get_model(self, batch_size=2): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() # Optimization - opt = fluid.optimizer.LarsMomentumOptimizer(learning_rate=0.001, - momentum=0.9) + opt = fluid.optimizer.LarsMomentumOptimizer( + learning_rate=0.001, momentum=0.9 + ) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) opt.minimize(avg_cost) - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_save_load.py b/python/paddle/fluid/tests/unittests/dist_save_load.py index 8d0083aaffea9..eb36010ea6fac 100644 --- a/python/paddle/fluid/tests/unittests/dist_save_load.py +++ b/python/paddle/fluid/tests/unittests/dist_save_load.py @@ -28,9 +28,7 @@ class TestDistSaveLoad2x2(TestDistSimnetBow2x2): - def _load_persistable_vars(self, executor, dirname, program): - def _is_checkpoint_var(var): """ the checkpoint will not save or load all the variables. @@ -38,9 +36,11 @@ def _is_checkpoint_var(var): : param var(Variable) """ - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.RAW: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.RAW + ): return False # @GRAD are named for gradient variables, checkpoint will not save it. if "@GRAD" in var.name: @@ -58,21 +58,30 @@ def _is_checkpoint_var(var): return var.persistable - io.load_vars(executor, - dirname=dirname, - main_program=program, - predicate=_is_checkpoint_var, - filename=None) + io.load_vars( + executor, + dirname=dirname, + main_program=program, + predicate=_is_checkpoint_var, + filename=None, + ) def run_pserver(self, args): self.get_model(batch_size=2) # NOTE: pserver should not call memory optimize - t = self.get_transpiler(args.trainer_id, fluid.default_main_program(), - args.endpoints, args.trainers, args.sync_mode, - False, args.current_endpoint) + t = self.get_transpiler( + args.trainer_id, + fluid.default_main_program(), + args.endpoints, + args.trainers, + args.sync_mode, + False, + args.current_endpoint, + ) pserver_prog = t.get_pserver_program(args.current_endpoint) - startup_prog = t.get_startup_program(args.current_endpoint, - pserver_prog) + startup_prog = t.get_startup_program( + args.current_endpoint, pserver_prog + ) need_load = bool(int(os.getenv("LOAD", "0"))) model_dir = os.getenv("MODEL_DIR", "") @@ -87,14 +96,23 @@ def run_pserver(self, args): exe.run(pserver_prog) def run_trainer(self, args): - test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \ - self.get_model(batch_size=2) + ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) = self.get_model(batch_size=2) if args.update_method == "pserver": - t = self.get_transpiler(args.trainer_id, - fluid.default_main_program(), - args.endpoints, args.trainers, - args.sync_mode) + t = self.get_transpiler( + args.trainer_id, + fluid.default_main_program(), + args.endpoints, + args.trainers, + args.sync_mode, + ) trainer_prog = t.get_trainer_program() else: @@ -114,17 +132,24 @@ def run_trainer(self, args): build_stra = fluid.BuildStrategy() if args.use_reduce: - build_stra.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce + build_stra.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + ) else: - build_stra.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.AllReduce + build_stra.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.AllReduce + ) - exe = fluid.ParallelExecutor(args.use_cuda, - loss_name=avg_cost.name, - exec_strategy=strategy, - build_strategy=build_stra) + exe = fluid.ParallelExecutor( + args.use_cuda, + loss_name=avg_cost.name, + exec_strategy=strategy, + build_strategy=build_stra, + ) feed_var_list = [ - var for var in trainer_prog.global_block().vars.values() + var + for var in trainer_prog.global_block().vars.values() if var.is_data ] @@ -149,13 +174,15 @@ def get_data(): if save_mode == "LOCAL": if need_save: for _ in range(RUN_STEP): - loss, = exe.run(fetch_list=[avg_cost.name], - feed=feeder.feed(get_data())) + (loss,) = exe.run( + fetch_list=[avg_cost.name], feed=feeder.feed(get_data()) + ) if need_save and model_dir: io.save_persistables(startup_exe, model_dir, trainer_prog) var = np.array( - fluid.global_scope().find_var('__fc_b__').get_tensor()) + fluid.global_scope().find_var('__fc_b__').get_tensor() + ) sys.stdout.buffer.write(pickle.dumps(np.ravel(var).tolist())) elif save_mode == "DIST": @@ -163,18 +190,26 @@ def get_data(): loss = None if need_save: for idx in range(8): - loss, = exe.run(fetch_list=[avg_cost.name], - feed=feeder.feed(get_data())) - if need_save and model_dir and idx == skip_steps and args.trainer_id == 0: - io.save_persistables(startup_exe, model_dir, - trainer_prog) + (loss,) = exe.run( + fetch_list=[avg_cost.name], feed=feeder.feed(get_data()) + ) + if ( + need_save + and model_dir + and idx == skip_steps + and args.trainer_id == 0 + ): + io.save_persistables( + startup_exe, model_dir, trainer_prog + ) else: for idx in range(8): data = get_data() if idx <= skip_steps: continue - loss, = exe.run(fetch_list=[avg_cost.name], - feed=feeder.feed(data)) + (loss,) = exe.run( + fetch_list=[avg_cost.name], feed=feeder.feed(data) + ) sys.stdout.buffer.write(pickle.dumps(loss.tolist())) else: raise Exception("save_mode must be LOCAL or DIST") diff --git a/python/paddle/fluid/tests/unittests/dist_se_resnext.py b/python/paddle/fluid/tests/unittests/dist_se_resnext.py index 5061bb8595615..0d8ed873f0398 100644 --- a/python/paddle/fluid/tests/unittests/dist_se_resnext.py +++ b/python/paddle/fluid/tests/unittests/dist_se_resnext.py @@ -31,13 +31,12 @@ "learning_strategy": { "name": "piecewise_decay", "epochs": [30, 60, 90], - "steps": [0.1, 0.01, 0.001, 0.0001] - } + "steps": [0.1, 0.01, 0.001, 0.0001], + }, } -class SE_ResNeXt(): - +class SE_ResNeXt: def __init__(self, layers=50): self.params = train_parameters self.layers = layers @@ -45,64 +44,65 @@ def __init__(self, layers=50): def net(self, input, class_dim=1000): layers = self.layers supported_layers = [50, 101, 152] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format(supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) if layers == 50: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 6, 3] num_filters = [128, 256, 512, 1024] - conv = self.conv_bn_layer(input=input, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - conv = fluid.layers.pool2d(input=conv, - pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + conv = self.conv_bn_layer( + input=input, num_filters=64, filter_size=7, stride=2, act='relu' + ) + conv = fluid.layers.pool2d( + input=conv, + pool_size=3, + pool_stride=2, + pool_padding=1, + pool_type='max', + ) elif layers == 101: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 23, 3] num_filters = [128, 256, 512, 1024] - conv = self.conv_bn_layer(input=input, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - conv = fluid.layers.pool2d(input=conv, - pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + conv = self.conv_bn_layer( + input=input, num_filters=64, filter_size=7, stride=2, act='relu' + ) + conv = fluid.layers.pool2d( + input=conv, + pool_size=3, + pool_stride=2, + pool_padding=1, + pool_type='max', + ) elif layers == 152: cardinality = 64 reduction_ratio = 16 depth = [3, 8, 36, 3] num_filters = [128, 256, 512, 1024] - conv = self.conv_bn_layer(input=input, - num_filters=64, - filter_size=3, - stride=2, - act='relu') - conv = self.conv_bn_layer(input=conv, - num_filters=64, - filter_size=3, - stride=1, - act='relu') - conv = self.conv_bn_layer(input=conv, - num_filters=128, - filter_size=3, - stride=1, - act='relu') + conv = self.conv_bn_layer( + input=input, num_filters=64, filter_size=3, stride=2, act='relu' + ) + conv = self.conv_bn_layer( + input=conv, num_filters=64, filter_size=3, stride=1, act='relu' + ) + conv = self.conv_bn_layer( + input=conv, num_filters=128, filter_size=3, stride=1, act='relu' + ) conv = fluid.layers.pool2d( - input=conv, pool_size=3, pool_stride=2, pool_padding=1, \ - pool_type='max') + input=conv, + pool_size=3, + pool_stride=2, + pool_padding=1, + pool_type='max', + ) for block in range(len(depth)): for i in range(depth[block]): @@ -111,20 +111,22 @@ def net(self, input, class_dim=1000): num_filters=num_filters[block], stride=2 if i == 0 and block != 0 else 1, cardinality=cardinality, - reduction_ratio=reduction_ratio) + reduction_ratio=reduction_ratio, + ) - pool = fluid.layers.pool2d(input=conv, - pool_size=7, - pool_type='avg', - global_pooling=True) + pool = fluid.layers.pool2d( + input=conv, pool_size=7, pool_type='avg', global_pooling=True + ) drop = fluid.layers.dropout(x=pool, dropout_prob=0.2) stdv = 1.0 / math.sqrt(drop.shape[1] * 1.0) out = fluid.layers.fc( input=drop, size=class_dim, act='softmax', - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.05))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.05) + ), + ) return out def shortcut(self, input, ch_out, stride): @@ -135,37 +137,36 @@ def shortcut(self, input, ch_out, stride): else: return input - def bottleneck_block(self, input, num_filters, stride, cardinality, - reduction_ratio): - conv0 = self.conv_bn_layer(input=input, - num_filters=num_filters, - filter_size=1, - act='relu') - conv1 = self.conv_bn_layer(input=conv0, - num_filters=num_filters, - filter_size=3, - stride=stride, - groups=cardinality, - act='relu') - conv2 = self.conv_bn_layer(input=conv1, - num_filters=num_filters * 2, - filter_size=1, - act=None) - scale = self.squeeze_excitation(input=conv2, - num_channels=num_filters * 2, - reduction_ratio=reduction_ratio) + def bottleneck_block( + self, input, num_filters, stride, cardinality, reduction_ratio + ): + conv0 = self.conv_bn_layer( + input=input, num_filters=num_filters, filter_size=1, act='relu' + ) + conv1 = self.conv_bn_layer( + input=conv0, + num_filters=num_filters, + filter_size=3, + stride=stride, + groups=cardinality, + act='relu', + ) + conv2 = self.conv_bn_layer( + input=conv1, num_filters=num_filters * 2, filter_size=1, act=None + ) + scale = self.squeeze_excitation( + input=conv2, + num_channels=num_filters * 2, + reduction_ratio=reduction_ratio, + ) short = self.shortcut(input, num_filters * 2, stride) return fluid.layers.elementwise_add(x=short, y=scale, act='relu') - def conv_bn_layer(self, - input, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def conv_bn_layer( + self, input, num_filters, filter_size, stride=1, groups=1, act=None + ): conv = fluid.layers.conv2d( input=input, num_filters=num_filters, @@ -175,41 +176,45 @@ def conv_bn_layer(self, groups=groups, act=None, # avoid pserver CPU init differs from GPU - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.05)), - bias_attr=False) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.05) + ), + bias_attr=False, + ) return fluid.layers.batch_norm(input=conv, act=act) def squeeze_excitation(self, input, num_channels, reduction_ratio): - pool = fluid.layers.pool2d(input=input, - pool_size=0, - pool_type='avg', - global_pooling=True) + pool = fluid.layers.pool2d( + input=input, pool_size=0, pool_type='avg', global_pooling=True + ) stdv = 1.0 / math.sqrt(pool.shape[1] * 1.0) squeeze = fluid.layers.fc( input=pool, size=num_channels // reduction_ratio, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.05)), - act='relu') + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.05) + ), + act='relu', + ) stdv = 1.0 / math.sqrt(squeeze.shape[1] * 1.0) excitation = fluid.layers.fc( input=squeeze, size=num_channels, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.05)), - act='sigmoid') + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.05) + ), + act='sigmoid', + ) scale = fluid.layers.elementwise_mul(x=input, y=excitation, axis=0) return scale class DistSeResneXt2x2(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False): # Input data - image = fluid.layers.data(name="data", - shape=[3, 224, 224], - dtype='float32') + image = fluid.layers.data( + name="data", shape=[3, 224, 224], dtype='float32' + ) label = fluid.layers.data(name="int64", shape=[1], dtype='int64') # Train program @@ -235,24 +240,30 @@ def get_model(self, batch_size=2, use_dgc=False): if not use_dgc: optimizer = fluid.optimizer.Momentum( - learning_rate=fluid.layers.piecewise_decay(boundaries=bd, - values=lr), + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr + ), momentum=0.9, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) else: optimizer = fluid.optimizer.DGCMomentumOptimizer( - learning_rate=fluid.layers.piecewise_decay(boundaries=bd, - values=lr), + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr + ), momentum=0.9, rampup_begin_step=0, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) optimizer.minimize(avg_cost) # Reader - train_reader = paddle.batch(paddle.dataset.flowers.test(use_xmap=False), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.flowers.test(use_xmap=False), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.flowers.test(use_xmap=False), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.flowers.test(use_xmap=False), batch_size=batch_size + ) return test_program, avg_cost, train_reader, test_reader, acc_top1, out diff --git a/python/paddle/fluid/tests/unittests/dist_sharding_save.py b/python/paddle/fluid/tests/unittests/dist_sharding_save.py index b4733b61af85a..d816d1b93abb2 100755 --- a/python/paddle/fluid/tests/unittests/dist_sharding_save.py +++ b/python/paddle/fluid/tests/unittests/dist_sharding_save.py @@ -37,20 +37,21 @@ def runtime_main(): fleet.init(role) with fluid.program_guard(train_prog, startup_prog): with fluid.unique_name.guard(): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=256, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -61,10 +62,12 @@ def runtime_main(): "sharding_degree": 2, } - optimizer = paddle.fluid.optimizer.Momentum(learning_rate=0.01, - momentum=0.9) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = paddle.fluid.optimizer.Momentum( + learning_rate=0.01, momentum=0.9 + ) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(avg_cost) # execution @@ -73,17 +76,16 @@ def runtime_main(): exe = fluid.Executor(place) exe.run(startup_prog) dirname = "./ut_sharding_save_model" - sharding.utils.save_persistables(exe, - dirname, - main_program=train_prog, - filename=None) + sharding.utils.save_persistables( + exe, dirname, main_program=train_prog, filename=None + ) out_losses = [] sys.stdout.buffer.write(pickle.dumps(out_losses)) if __name__ == "__main__": - #NOTE(liangjianzhong): dist unittest should be imlpement using runtime_main in test_dist_base.py + # NOTE(liangjianzhong): dist unittest should be imlpement using runtime_main in test_dist_base.py # but the runtime_main in test_dist_base.py use the fleet, DistributedStrategy from # paddle.fluid.incubate.fleet.collective which is not support by sharding (paddle.distributed.fleet). # this should be update in future. diff --git a/python/paddle/fluid/tests/unittests/dist_text_classification.py b/python/paddle/fluid/tests/unittests/dist_text_classification.py index c8315161f0fd9..417ff66e0cb7e 100644 --- a/python/paddle/fluid/tests/unittests/dist_text_classification.py +++ b/python/paddle/fluid/tests/unittests/dist_text_classification.py @@ -43,19 +43,23 @@ def get_worddict(dict_path): return word_dict, dict_dim -def conv_net(input, - dict_dim, - emb_dim=128, - window_size=3, - num_filters=128, - fc0_dim=96, - class_dim=2): +def conv_net( + input, + dict_dim, + emb_dim=128, + window_size=3, + num_filters=128, + fc0_dim=96, + class_dim=2, +): emb = fluid.layers.embedding( input=input, size=[dict_dim, emb_dim], is_sparse=False, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) conv_3 = fluid.nets.sequence_conv_pool( input=emb, @@ -63,30 +67,35 @@ def conv_net(input, filter_size=window_size, act="tanh", pool_type="max", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) fc_0 = fluid.layers.fc( input=[conv_3], size=fc0_dim, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) prediction = fluid.layers.fc( input=[fc_0], size=class_dim, act="softmax", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.01))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.01) + ), + ) return prediction def inference_network(dict_dim): - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) out = conv_net(data, dict_dim) return out @@ -107,17 +116,16 @@ def get_optimizer(learning_rate): class TestDistTextClassification2x2(TestDistRunnerBase): - def get_model(self, batch_size=2): - vocab = os.path.join(paddle.dataset.common.DATA_HOME, - "text_classification", "imdb.vocab") + vocab = os.path.join( + paddle.dataset.common.DATA_HOME, "text_classification", "imdb.vocab" + ) word_dict, dict_dim = get_worddict(vocab) # Input data - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') # Train program @@ -134,7 +142,14 @@ def get_model(self, batch_size=2): # Reader train_reader, test_reader = get_reader(word_dict, batch_size) - return inference_program, avg_cost, train_reader, test_reader, acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + acc, + predict, + ) def tokenize(pattern): @@ -143,8 +158,10 @@ def tokenize(pattern): """ with tarfile.open( - paddle.dataset.common.download(DATA_URL, 'text_classification', - DATA_MD5)) as tarf: + paddle.dataset.common.download( + DATA_URL, 'text_classification', DATA_MD5 + ) + ) as tarf: # Note that we should use tarfile.next(), which does # sequential access of member files, other than # tarfile.extractfile, which does random access and might @@ -154,7 +171,8 @@ def tokenize(pattern): if bool(pattern.match(tf.name)): # newline and punctuations removal and ad-hoc tokenization. yield tarf.extractfile(tf).read().rstrip(b'\n\r').translate( - None, string.punctuation.encode('latin-1')).lower().split() + None, string.punctuation.encode('latin-1') + ).lower().split() tf = tarf.next() @@ -188,8 +206,11 @@ def train(word_idx): :return: Training reader creator :rtype: callable """ - return reader_creator(re.compile(r"train/pos/.*\.txt$"), - re.compile(r"train/neg/.*\.txt$"), word_idx) + return reader_creator( + re.compile(r"train/pos/.*\.txt$"), + re.compile(r"train/neg/.*\.txt$"), + word_idx, + ) def test(word_idx): @@ -204,8 +225,11 @@ def test(word_idx): :return: Test reader creator :rtype: callable """ - return reader_creator(re.compile(r"test/pos/.*\.txt$"), - re.compile(r"test/neg/.*\.txt$"), word_idx) + return reader_creator( + re.compile(r"test/pos/.*\.txt$"), + re.compile(r"test/neg/.*\.txt$"), + word_idx, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_transformer.py b/python/paddle/fluid/tests/unittests/dist_transformer.py index 23bcd1957e0f4..4765ee7d82bf5 100644 --- a/python/paddle/fluid/tests/unittests/dist_transformer.py +++ b/python/paddle/fluid/tests/unittests/dist_transformer.py @@ -35,7 +35,7 @@ fluid.default_main_program().random_seed = 1 -#from transformer_config import ModelHyperParams, TrainTaskConfig, merge_cfg_from_list +# from transformer_config import ModelHyperParams, TrainTaskConfig, merge_cfg_from_list class TrainTaskConfig(object): # only support GPU currently use_gpu = True @@ -72,7 +72,8 @@ class TrainTaskConfig(object): check_acc = True data_path = expanduser("~") + ( - "/.cache/paddle/dataset/test_dist_transformer/") + "/.cache/paddle/dataset/test_dist_transformer/" + ) src_vocab_fpath = data_path + "vocab.bpe.32000" trg_vocab_fpath = data_path + "vocab.bpe.32000" train_file_pattern = data_path + "train.tok.clean.bpe.32000.en-de" @@ -179,12 +180,17 @@ def merge_cfg_from_list(cfg_list, g_cfgs): # encoder. # The actual data shape of src_slf_attn_bias is: # [batch_size, n_head, max_src_len_in_batch, max_src_len_in_batch] - "src_slf_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "src_slf_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # The actual data shape of trg_word is: # [batch_size * max_trg_len_in_batch, 1] - "trg_word": [(batch_size, seq_len, 1), "int64", - 2], # lod_level is only used in fast decoder. + "trg_word": [ + (batch_size, seq_len, 1), + "int64", + 2, + ], # lod_level is only used in fast decoder. # The actual data shape of trg_pos is: # [batch_size * max_trg_len_in_batch, 1] "trg_pos": [(batch_size, seq_len, 1), "int64"], @@ -192,14 +198,18 @@ def merge_cfg_from_list(cfg_list, g_cfgs): # subsequent words in the decoder. # The actual data shape of trg_slf_attn_bias is: # [batch_size, n_head, max_trg_len_in_batch, max_trg_len_in_batch] - "trg_slf_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "trg_slf_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # This input is used to remove attention weights on paddings of the source # input in the encoder-decoder attention. # The actual data shape of trg_src_attn_bias is: # [batch_size, n_head, max_trg_len_in_batch, max_src_len_in_batch] - "trg_src_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "trg_src_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # This input is used in independent decoder program for inference. # The actual data shape of enc_output is: # [batch_size, max_src_len_in_batch, d_model] @@ -212,8 +222,8 @@ def merge_cfg_from_list(cfg_list, g_cfgs): # [batch_size * max_trg_len_in_batch, 1] "lbl_weight": [(batch_size * seq_len, 1), "float32"], # These inputs are used to change the shape tensor in beam-search decoder. - "trg_slf_attn_pre_softmax_shape_delta": [(2, ), "int32"], - "trg_slf_attn_post_softmax_shape_delta": [(4, ), "int32"], + "trg_slf_attn_pre_softmax_shape_delta": [(2,), "int32"], + "trg_slf_attn_post_softmax_shape_delta": [(4,), "int32"], "init_score": [(batch_size, 1), "float32"], } @@ -257,7 +267,7 @@ def merge_cfg_from_list(cfg_list, g_cfgs): # "trg_slf_attn_post_softmax_shape_delta", ) -#from optim import LearningRateScheduler +# from optim import LearningRateScheduler class LearningRateScheduler(object): """ Wrapper for learning rate scheduling as described in the Transformer paper. @@ -265,12 +275,14 @@ class LearningRateScheduler(object): learning rate will be fed into the main_program as input data. """ - def __init__(self, - d_model, - warmup_steps, - learning_rate=0.001, - current_steps=0, - name="learning_rate"): + def __init__( + self, + d_model, + warmup_steps, + learning_rate=0.001, + current_steps=0, + name="learning_rate", + ): self.current_steps = current_steps self.warmup_steps = warmup_steps self.d_model = d_model @@ -280,68 +292,91 @@ def __init__(self, shape=[1], value=float(learning_rate), dtype="float32", - persistable=True) + persistable=True, + ) def update_learning_rate(self): self.current_steps += 1 - lr_value = np.power(self.d_model, -0.5) * np.min([ - np.power(self.current_steps, -0.5), - np.power(self.warmup_steps, -1.5) * self.current_steps - ]) * self.static_lr + lr_value = ( + np.power(self.d_model, -0.5) + * np.min( + [ + np.power(self.current_steps, -0.5), + np.power(self.warmup_steps, -1.5) * self.current_steps, + ] + ) + * self.static_lr + ) return np.array([lr_value], dtype="float32") -#from transformer_train import train_loop -def pad_batch_data(insts, - pad_idx, - n_head, - is_target=False, - is_label=False, - return_attn_bias=True, - return_max_len=True, - return_num_token=False): +# from transformer_train import train_loop +def pad_batch_data( + insts, + pad_idx, + n_head, + is_target=False, + is_label=False, + return_attn_bias=True, + return_max_len=True, + return_num_token=False, +): """ Pad the instances to the max sequence length in batch, and generate the corresponding position data and attention bias. """ return_list = [] max_len = max(len(inst) for inst in insts) - num_token = functools.reduce(lambda x, y: x + y, - [len(inst) - for inst in insts]) if return_num_token else 0 + num_token = ( + functools.reduce(lambda x, y: x + y, [len(inst) for inst in insts]) + if return_num_token + else 0 + ) # Any token included in dict can be used to pad, since the paddings' loss # will be masked out by weights and make no effect on parameter gradients. inst_data = np.array( - [inst + [pad_idx] * (max_len - len(inst)) for inst in insts]) + [inst + [pad_idx] * (max_len - len(inst)) for inst in insts] + ) return_list += [inst_data.astype("int64").reshape([-1, 1])] if is_label: # label weight - inst_weight = np.array([[1.] * len(inst) + [0.] * (max_len - len(inst)) - for inst in insts]) + inst_weight = np.array( + [ + [1.0] * len(inst) + [0.0] * (max_len - len(inst)) + for inst in insts + ] + ) return_list += [inst_weight.astype("float32").reshape([-1, 1])] else: # position data - inst_pos = np.array([ - list(range(1, - len(inst) + 1)) + [0] * (max_len - len(inst)) - for inst in insts - ]) + inst_pos = np.array( + [ + list(range(1, len(inst) + 1)) + [0] * (max_len - len(inst)) + for inst in insts + ] + ) return_list += [inst_pos.astype("int64").reshape([-1, 1])] if return_attn_bias: if is_target: # This is used to avoid attention on paddings and subsequent # words. slf_attn_bias_data = np.ones((inst_data.shape[0], max_len, max_len)) - slf_attn_bias_data = np.triu(slf_attn_bias_data, - 1).reshape([-1, 1, max_len, max_len]) - slf_attn_bias_data = np.tile(slf_attn_bias_data, - [1, n_head, 1, 1]) * [-1e9] + slf_attn_bias_data = np.triu(slf_attn_bias_data, 1).reshape( + [-1, 1, max_len, max_len] + ) + slf_attn_bias_data = np.tile( + slf_attn_bias_data, [1, n_head, 1, 1] + ) * [-1e9] else: # This is used to avoid attention on paddings. - slf_attn_bias_data = np.array([[0] * len(inst) + [-1e9] * - (max_len - len(inst)) - for inst in insts]) + slf_attn_bias_data = np.array( + [ + [0] * len(inst) + [-1e9] * (max_len - len(inst)) + for inst in insts + ] + ) slf_attn_bias_data = np.tile( slf_attn_bias_data.reshape([-1, 1, 1, max_len]), - [1, n_head, max_len, 1]) + [1, n_head, max_len, 1], + ) return_list += [slf_attn_bias_data.astype("float32")] if return_max_len: return_list += [max_len] @@ -350,22 +385,26 @@ def pad_batch_data(insts, return return_list if len(return_list) > 1 else return_list[0] -def prepare_batch_input(insts, data_input_names, src_pad_idx, trg_pad_idx, - n_head, d_model): +def prepare_batch_input( + insts, data_input_names, src_pad_idx, trg_pad_idx, n_head, d_model +): """ Put all padded data needed by training into a dict. """ src_word, src_pos, src_slf_attn_bias, src_max_len = pad_batch_data( - [inst[0] for inst in insts], src_pad_idx, n_head, is_target=False) + [inst[0] for inst in insts], src_pad_idx, n_head, is_target=False + ) src_word = src_word.reshape(-1, src_max_len, 1) src_pos = src_pos.reshape(-1, src_max_len, 1) trg_word, trg_pos, trg_slf_attn_bias, trg_max_len = pad_batch_data( - [inst[1] for inst in insts], trg_pad_idx, n_head, is_target=True) + [inst[1] for inst in insts], trg_pad_idx, n_head, is_target=True + ) trg_word = trg_word.reshape(-1, trg_max_len, 1) trg_pos = trg_pos.reshape(-1, trg_max_len, 1) - trg_src_attn_bias = np.tile(src_slf_attn_bias[:, :, ::src_max_len, :], - [1, 1, trg_max_len, 1]).astype("float32") + trg_src_attn_bias = np.tile( + src_slf_attn_bias[:, :, ::src_max_len, :], [1, 1, trg_max_len, 1] + ).astype("float32") lbl_word, lbl_weight, num_token = pad_batch_data( [inst[2] for inst in insts], @@ -375,14 +414,27 @@ def prepare_batch_input(insts, data_input_names, src_pad_idx, trg_pad_idx, is_label=True, return_attn_bias=False, return_max_len=False, - return_num_token=True) + return_num_token=True, + ) data_input_dict = dict( list( - zip(data_input_names, [ - src_word, src_pos, src_slf_attn_bias, trg_word, trg_pos, - trg_slf_attn_bias, trg_src_attn_bias, lbl_word, lbl_weight - ]))) + zip( + data_input_names, + [ + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_pos, + trg_slf_attn_bias, + trg_src_attn_bias, + lbl_word, + lbl_weight, + ], + ) + ) + ) return data_input_dict, np.asarray([num_token], dtype="float32") @@ -407,7 +459,7 @@ def __impl__(): if len(data) > count: inst_num_per_part = len(data) // count yield [ - data[inst_num_per_part * i:inst_num_per_part * (i + 1)] + data[inst_num_per_part * i : inst_num_per_part * (i + 1)] for i in range(count) ] @@ -423,21 +475,28 @@ def split_data(data, num_part): data = data[0] inst_num_per_part = len(data) // num_part return [ - data[inst_num_per_part * i:inst_num_per_part * (i + 1)] + data[inst_num_per_part * i : inst_num_per_part * (i + 1)] for i in range(num_part) ] -def test_context(test_program, avg_cost, train_exe, dev_count, data_input_names, - sum_cost, token_num): +def test_context( + test_program, + avg_cost, + train_exe, + dev_count, + data_input_names, + sum_cost, + token_num, +): val_data = DataReader( src_vocab_fpath=TrainTaskConfig.src_vocab_fpath, trg_vocab_fpath=TrainTaskConfig.trg_vocab_fpath, fpattern=TrainTaskConfig.val_file_pattern, token_delimiter=TrainTaskConfig.token_delimiter, use_token_batch=TrainTaskConfig.use_token_batch, - batch_size=TrainTaskConfig.batch_size * - (1 if TrainTaskConfig.use_token_batch else dev_count), + batch_size=TrainTaskConfig.batch_size + * (1 if TrainTaskConfig.use_token_batch else dev_count), pool_size=TrainTaskConfig.pool_size, sort_type=TrainTaskConfig.sort_type, start_mark=TrainTaskConfig.special_token[0], @@ -447,37 +506,47 @@ def test_context(test_program, avg_cost, train_exe, dev_count, data_input_names, max_length=ModelHyperParams.max_length - 2, clip_last_batch=False, shuffle=False, - shuffle_batch=False) + shuffle_batch=False, + ) build_strategy = fluid.BuildStrategy() strategy = fluid.ExecutionStrategy() strategy.num_threads = 1 - test_exe = fluid.ParallelExecutor(use_cuda=TrainTaskConfig.use_gpu, - main_program=test_program, - share_vars_from=train_exe, - build_strategy=build_strategy, - exec_strategy=strategy) + test_exe = fluid.ParallelExecutor( + use_cuda=TrainTaskConfig.use_gpu, + main_program=test_program, + share_vars_from=train_exe, + build_strategy=build_strategy, + exec_strategy=strategy, + ) def test(exe=test_exe): test_total_cost = 0 test_total_token = 0 test_data = read_multiple( reader=val_data.batch_generator, - count=dev_count if TrainTaskConfig.use_token_batch else 1) + count=dev_count if TrainTaskConfig.use_token_batch else 1, + ) for batch_id, data in enumerate(test_data()): feed_list = [] for place_id, data_buffer in enumerate( - split_data(data, num_part=dev_count)): + split_data(data, num_part=dev_count) + ): data_input_dict, _ = prepare_batch_input( - data_buffer, data_input_names, ModelHyperParams.eos_idx, - ModelHyperParams.eos_idx, ModelHyperParams.n_head, - ModelHyperParams.d_model) + data_buffer, + data_input_names, + ModelHyperParams.eos_idx, + ModelHyperParams.eos_idx, + ModelHyperParams.n_head, + ModelHyperParams.d_model, + ) feed_list.append(data_input_dict) - outs = exe.run(feed=feed_list, - fetch_list=[sum_cost.name, token_num.name]) + outs = exe.run( + feed=feed_list, fetch_list=[sum_cost.name, token_num.name] + ) sum_cost_val, token_num_val = np.array(outs[0]), np.array(outs[1]) test_total_cost += sum_cost_val.sum() test_total_token += token_num_val.sum() @@ -488,8 +557,17 @@ def test(exe=test_exe): return test -def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler, - token_num, predict, test_program): +def train_loop( + exe, + train_progm, + dev_count, + sum_cost, + avg_cost, + lr_scheduler, + token_num, + predict, + test_program, +): # Initialize the parameters. if TrainTaskConfig.ckpt_path: lr_scheduler.current_steps = TrainTaskConfig.start_step @@ -502,8 +580,8 @@ def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler, fpattern=TrainTaskConfig.train_file_pattern, token_delimiter=TrainTaskConfig.token_delimiter, use_token_batch=TrainTaskConfig.use_token_batch, - batch_size=TrainTaskConfig.batch_size * - (1 if TrainTaskConfig.use_token_batch else dev_count), + batch_size=TrainTaskConfig.batch_size + * (1 if TrainTaskConfig.use_token_batch else dev_count), pool_size=TrainTaskConfig.pool_size, sort_type=TrainTaskConfig.sort_type, shuffle=TrainTaskConfig.shuffle, @@ -513,39 +591,60 @@ def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler, unk_mark=TrainTaskConfig.special_token[2], # count start and end tokens out max_length=ModelHyperParams.max_length - 2, - clip_last_batch=False) + clip_last_batch=False, + ) train_data = read_multiple( reader=train_data.batch_generator, - count=dev_count if TrainTaskConfig.use_token_batch else 1) + count=dev_count if TrainTaskConfig.use_token_batch else 1, + ) build_strategy = fluid.BuildStrategy() # Since the token number differs among devices, customize gradient scale to # use token average cost among multi-devices. and the gradient scale is # `1 / token_number` for average cost. - build_strategy.gradient_scale_strategy = fluid.BuildStrategy.GradientScaleStrategy.Customized + build_strategy.gradient_scale_strategy = ( + fluid.BuildStrategy.GradientScaleStrategy.Customized + ) strategy = fluid.ExecutionStrategy() strategy.num_threads = 1 - train_exe = fluid.ParallelExecutor(use_cuda=TrainTaskConfig.use_gpu, - loss_name=sum_cost.name, - main_program=train_progm, - build_strategy=build_strategy, - exec_strategy=strategy) + train_exe = fluid.ParallelExecutor( + use_cuda=TrainTaskConfig.use_gpu, + loss_name=sum_cost.name, + main_program=train_progm, + build_strategy=build_strategy, + exec_strategy=strategy, + ) - data_input_names = encoder_data_input_fields + decoder_data_input_fields[: - -1] + label_data_input_fields + data_input_names = ( + encoder_data_input_fields + + decoder_data_input_fields[:-1] + + label_data_input_fields + ) if TrainTaskConfig.val_file_pattern is not None: - test = test_context(test_program, avg_cost, train_exe, dev_count, - data_input_names, sum_cost, token_num) + test = test_context( + test_program, + avg_cost, + train_exe, + dev_count, + data_input_names, + sum_cost, + token_num, + ) # the best cross-entropy value with label smoothing - loss_normalizer = -((1. - TrainTaskConfig.label_smooth_eps) * np.log( - (1. - TrainTaskConfig.label_smooth_eps)) + - TrainTaskConfig.label_smooth_eps * - np.log(TrainTaskConfig.label_smooth_eps / - (ModelHyperParams.trg_vocab_size - 1) + 1e-20)) + loss_normalizer = -( + (1.0 - TrainTaskConfig.label_smooth_eps) + * np.log((1.0 - TrainTaskConfig.label_smooth_eps)) + + TrainTaskConfig.label_smooth_eps + * np.log( + TrainTaskConfig.label_smooth_eps + / (ModelHyperParams.trg_vocab_size - 1) + + 1e-20 + ) + ) init = False for pass_id in range(TrainTaskConfig.pass_num): pass_start_time = time.time() @@ -560,36 +659,44 @@ def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler, lr_rate = lr_scheduler.update_learning_rate() for place_id, data_buffer in enumerate( - split_data(data, num_part=dev_count)): + split_data(data, num_part=dev_count) + ): data_input_dict, num_token = prepare_batch_input( - data_buffer, data_input_names, ModelHyperParams.eos_idx, - ModelHyperParams.eos_idx, ModelHyperParams.n_head, - ModelHyperParams.d_model) + data_buffer, + data_input_names, + ModelHyperParams.eos_idx, + ModelHyperParams.eos_idx, + ModelHyperParams.n_head, + ModelHyperParams.d_model, + ) total_num_token += num_token feed_kv_pairs = list(data_input_dict.items()) if TrainTaskConfig.local: feed_kv_pairs += list( - {lr_scheduler.learning_rate.name: lr_rate}.items()) + {lr_scheduler.learning_rate.name: lr_rate}.items() + ) feed_list.append(dict(feed_kv_pairs)) if not init: for pos_enc_param_name in pos_enc_param_names: pos_enc = position_encoding_init( ModelHyperParams.max_length + 1, - ModelHyperParams.d_model) + ModelHyperParams.d_model, + ) feed_list[place_id][pos_enc_param_name] = pos_enc if not TrainTaskConfig.check_acc: for feed_dict in feed_list: - feed_dict[sum_cost.name + "@GRAD"] = 1. / total_num_token + feed_dict[sum_cost.name + "@GRAD"] = 1.0 / total_num_token else: b = 100 * TrainTaskConfig.batch_size a = np.asarray([b], dtype="float32") for feed_dict in feed_list: - feed_dict[sum_cost.name + "@GRAD"] = 1. / a + feed_dict[sum_cost.name + "@GRAD"] = 1.0 / a - outs = train_exe.run(fetch_list=[sum_cost.name, token_num.name], - feed=feed_list) + outs = train_exe.run( + fetch_list=[sum_cost.name, token_num.name], feed=feed_list + ) sum_cost_val, token_num_val = np.array(outs[0]), np.array(outs[1]) total_sum_cost = sum_cost_val.sum() @@ -603,10 +710,10 @@ def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler, val_avg_cost, val_ppl = test() print("[%f]" % val_avg_cost) else: - assert (False) + assert False -#import transformer_reader as reader +# import transformer_reader as reader class SortType(object): GLOBAL = 'global' POOL = 'pool' @@ -614,7 +721,6 @@ class SortType(object): class Converter(object): - def __init__(self, vocab, beg, end, unk, delimiter): self._vocab = vocab self._beg = beg @@ -623,14 +729,17 @@ def __init__(self, vocab, beg, end, unk, delimiter): self._delimiter = delimiter def __call__(self, sentence): - return [self._beg] + [ - self._vocab.get(w, self._unk) - for w in sentence.split(self._delimiter) - ] + [self._end] + return ( + [self._beg] + + [ + self._vocab.get(w, self._unk) + for w in sentence.split(self._delimiter) + ] + + [self._end] + ) class ComposedConverter(object): - def __init__(self, converters): self._converters = converters @@ -642,7 +751,6 @@ def __call__(self, parallel_sentence): class SentenceBatchCreator(object): - def __init__(self, batch_size): self.batch = [] self._batch_size = batch_size @@ -656,7 +764,6 @@ def append(self, info): class TokenBatchCreator(object): - def __init__(self, batch_size): self.batch = [] self.max_len = -1 @@ -676,7 +783,6 @@ def append(self, info): class SampleInfo(object): - def __init__(self, i, max_len, min_len): self.i = i self.min_len = min_len @@ -684,7 +790,6 @@ def __init__(self, i, max_len, min_len): class MinMaxFilter(object): - def __init__(self, max_len, min_len, underlying_creator): self._min_len = min_len self._max_len = max_len @@ -774,26 +879,28 @@ class DataReader(object): :type seed: int """ - def __init__(self, - src_vocab_fpath, - trg_vocab_fpath, - fpattern, - batch_size, - pool_size, - sort_type=SortType.GLOBAL, - clip_last_batch=True, - tar_fname=None, - min_length=0, - max_length=100, - shuffle=True, - shuffle_batch=False, - use_token_batch=False, - field_delimiter="\t", - token_delimiter=" ", - start_mark="", - end_mark="", - unk_mark="", - seed=0): + def __init__( + self, + src_vocab_fpath, + trg_vocab_fpath, + fpattern, + batch_size, + pool_size, + sort_type=SortType.GLOBAL, + clip_last_batch=True, + tar_fname=None, + min_length=0, + max_length=100, + shuffle=True, + shuffle_batch=False, + use_token_batch=False, + field_delimiter="\t", + token_delimiter=" ", + start_mark="", + end_mark="", + unk_mark="", + seed=0, + ): self._src_vocab = self.load_dict(src_vocab_fpath) self._only_src = True if trg_vocab_fpath is not None: @@ -810,26 +917,33 @@ def __init__(self, self._max_length = max_length self._field_delimiter = field_delimiter self._token_delimiter = token_delimiter - self.load_src_trg_ids(end_mark, fpattern, start_mark, tar_fname, - unk_mark) + self.load_src_trg_ids( + end_mark, fpattern, start_mark, tar_fname, unk_mark + ) self._random = random.Random(x=seed) - def load_src_trg_ids(self, end_mark, fpattern, start_mark, tar_fname, - unk_mark): + def load_src_trg_ids( + self, end_mark, fpattern, start_mark, tar_fname, unk_mark + ): converters = [ - Converter(vocab=self._src_vocab, - beg=self._src_vocab[start_mark], - end=self._src_vocab[end_mark], - unk=self._src_vocab[unk_mark], - delimiter=self._token_delimiter) + Converter( + vocab=self._src_vocab, + beg=self._src_vocab[start_mark], + end=self._src_vocab[end_mark], + unk=self._src_vocab[unk_mark], + delimiter=self._token_delimiter, + ) ] if not self._only_src: converters.append( - Converter(vocab=self._trg_vocab, - beg=self._trg_vocab[start_mark], - end=self._trg_vocab[end_mark], - unk=self._trg_vocab[unk_mark], - delimiter=self._token_delimiter)) + Converter( + vocab=self._trg_vocab, + beg=self._trg_vocab[start_mark], + end=self._trg_vocab[end_mark], + unk=self._trg_vocab[unk_mark], + delimiter=self._token_delimiter, + ) + ) converters = ComposedConverter(converters) @@ -857,9 +971,9 @@ def _load_lines(self, fpattern, tar_fname): for line in f.extractfile(tar_fname): line = line.decode() fields = line.strip("\n").split(self._field_delimiter) - if (not self._only_src - and len(fields) == 2) or (self._only_src - and len(fields) == 1): + if (not self._only_src and len(fields) == 2) or ( + self._only_src and len(fields) == 1 + ): yield fields else: for fpath in fpaths: @@ -870,9 +984,9 @@ def _load_lines(self, fpattern, tar_fname): for line in f: line = line.decode() fields = line.strip("\n").split(self._field_delimiter) - if (not self._only_src - and len(fields) == 2) or (self._only_src - and len(fields) == 1): + if (not self._only_src and len(fields) == 2) or ( + self._only_src and len(fields) == 1 + ): yield fields @staticmethod @@ -890,9 +1004,9 @@ def load_dict(dict_path, reverse=False): def batch_generator(self): # global sort or global shuffle if self._sort_type == SortType.GLOBAL: - infos = sorted(self._sample_infos, - key=lambda x: x.max_len, - reverse=True) + infos = sorted( + self._sample_infos, key=lambda x: x.max_len, reverse=True + ) else: if self._shuffle: infos = self._sample_infos @@ -902,16 +1016,20 @@ def batch_generator(self): if self._sort_type == SortType.POOL: for i in range(0, len(infos), self._pool_size): - infos[i:i + self._pool_size] = sorted( - infos[i:i + self._pool_size], key=lambda x: x.max_len) + infos[i : i + self._pool_size] = sorted( + infos[i : i + self._pool_size], key=lambda x: x.max_len + ) # concat batch batches = [] - batch_creator = TokenBatchCreator( - self._batch_size - ) if self._use_token_batch else SentenceBatchCreator(self._batch_size) - batch_creator = MinMaxFilter(self._max_length, self._min_length, - batch_creator) + batch_creator = ( + TokenBatchCreator(self._batch_size) + if self._use_token_batch + else SentenceBatchCreator(self._batch_size) + ) + batch_creator = MinMaxFilter( + self._max_length, self._min_length, batch_creator + ) for info in infos: batch = batch_creator.append(info) @@ -930,34 +1048,49 @@ def batch_generator(self): if self._only_src: yield [[self._src_seq_ids[idx]] for idx in batch_ids] else: - yield [(self._src_seq_ids[idx], self._trg_seq_ids[idx][:-1], - self._trg_seq_ids[idx][1:]) for idx in batch_ids] + yield [ + ( + self._src_seq_ids[idx], + self._trg_seq_ids[idx][:-1], + self._trg_seq_ids[idx][1:], + ) + for idx in batch_ids + ] -#from transformer_model import transformer +# from transformer_model import transformer def position_encoding_init(n_position, d_pos_vec): """ Generate the initial values for the sinusoid position encoding table. """ - position_enc = np.array([[ - pos / np.power(10000, 2 * (j // 2) / d_pos_vec) - for j in range(d_pos_vec) - ] if pos != 0 else np.zeros(d_pos_vec) for pos in range(n_position)]) + position_enc = np.array( + [ + [ + pos / np.power(10000, 2 * (j // 2) / d_pos_vec) + for j in range(d_pos_vec) + ] + if pos != 0 + else np.zeros(d_pos_vec) + for pos in range(n_position) + ] + ) position_enc[1:, 0::2] = np.sin(position_enc[1:, 0::2]) # dim 2i position_enc[1:, 1::2] = np.cos(position_enc[1:, 1::2]) # dim 2i+1 return position_enc.astype("float32") -def multi_head_attention(queries, - keys, - values, - attn_bias, - d_key, - d_value, - d_model, - n_head=1, - dropout_rate=0., - cache=None): +def multi_head_attention( + queries, + keys, + values, + attn_bias, + d_key, + d_value, + d_model, + n_head=1, + dropout_rate=0.0, + cache=None, +): """ Multi-Head Attention. Note that attn_bias is added to the logit before computing softmax activiation to mask certain selected positions so that @@ -965,27 +1098,34 @@ def multi_head_attention(queries, """ if not (len(queries.shape) == len(keys.shape) == len(values.shape) == 3): raise ValueError( - "Inputs: queries, keys and values should all be 3-D tensors.") + "Inputs: queries, keys and values should all be 3-D tensors." + ) def __compute_qkv(queries, keys, values, n_head, d_key, d_value): """ Add linear projection to queries, keys, and values. """ - q = layers.fc(input=queries, - size=d_key * n_head, - num_flatten_dims=2, - param_attr=const_para_attr, - bias_attr=const_bias_attr) - k = layers.fc(input=keys, - size=d_key * n_head, - num_flatten_dims=2, - param_attr=const_para_attr, - bias_attr=const_bias_attr) - v = layers.fc(input=values, - size=d_value * n_head, - num_flatten_dims=2, - param_attr=const_para_attr, - bias_attr=const_bias_attr) + q = layers.fc( + input=queries, + size=d_key * n_head, + num_flatten_dims=2, + param_attr=const_para_attr, + bias_attr=const_bias_attr, + ) + k = layers.fc( + input=keys, + size=d_key * n_head, + num_flatten_dims=2, + param_attr=const_para_attr, + bias_attr=const_bias_attr, + ) + v = layers.fc( + input=values, + size=d_value * n_head, + num_flatten_dims=2, + param_attr=const_para_attr, + bias_attr=const_bias_attr, + ) return q, k, v def __split_heads(x, n_head): @@ -1001,8 +1141,9 @@ def __split_heads(x, n_head): hidden_size = x.shape[-1] # The value 0 in shape attr means copying the corresponding dimension # size of the input as the output dimension size. - reshaped = layers.reshape(x=x, - shape=[0, 0, n_head, hidden_size // n_head]) + reshaped = layers.reshape( + x=x, shape=[0, 0, n_head, hidden_size // n_head] + ) # permute the dimensions into: # [batch_size, n_head, max_sequence_len, hidden_size_per_head] @@ -1013,7 +1154,8 @@ def __combine_heads(x): Transpose and then reshape the last two dimensions of input tensor x so that it becomes one dimension, which is reverse to __split_heads. """ - if len(x.shape) == 3: return x + if len(x.shape) == 3: + return x if len(x.shape) != 4: raise ValueError("Input(x) should be a 4-D Tensor.") @@ -1022,7 +1164,8 @@ def __combine_heads(x): # size of the input as the output dimension size. return layers.reshape( x=trans_x, - shape=list(map(int, [0, 0, trans_x.shape[2] * trans_x.shape[3]]))) + shape=list(map(int, [0, 0, trans_x.shape[2] * trans_x.shape[3]])), + ) def scaled_dot_product_attention(q, k, v, attn_bias, d_model, dropout_rate): """ @@ -1034,10 +1177,12 @@ def scaled_dot_product_attention(q, k, v, attn_bias, d_model, dropout_rate): product += attn_bias weights = layers.softmax(product) if dropout_rate: - weights = layers.dropout(weights, - dropout_prob=dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) + weights = layers.dropout( + weights, + dropout_prob=dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False, + ) out = layers.matmul(weights, v) return out @@ -1051,17 +1196,20 @@ def scaled_dot_product_attention(q, k, v, attn_bias, d_model, dropout_rate): k = __split_heads(k, n_head) v = __split_heads(v, n_head) - ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_model, - dropout_rate) + ctx_multiheads = scaled_dot_product_attention( + q, k, v, attn_bias, d_model, dropout_rate + ) out = __combine_heads(ctx_multiheads) # Project back to the model size. - proj_out = layers.fc(input=out, - size=d_model, - num_flatten_dims=2, - param_attr=const_para_attr, - bias_attr=const_bias_attr) + proj_out = layers.fc( + input=out, + size=d_model, + num_flatten_dims=2, + param_attr=const_para_attr, + bias_attr=const_bias_attr, + ) return proj_out @@ -1071,21 +1219,25 @@ def positionwise_feed_forward(x, d_inner_hid, d_hid): This module consists of two linear transformations with a ReLU activation in between, which is applied to each position separately and identically. """ - hidden = layers.fc(input=x, - size=d_inner_hid, - num_flatten_dims=2, - act="relu", - param_attr=const_para_attr, - bias_attr=const_bias_attr) - out = layers.fc(input=hidden, - size=d_hid, - num_flatten_dims=2, - param_attr=const_para_attr, - bias_attr=const_bias_attr) + hidden = layers.fc( + input=x, + size=d_inner_hid, + num_flatten_dims=2, + act="relu", + param_attr=const_para_attr, + bias_attr=const_bias_attr, + ) + out = layers.fc( + input=hidden, + size=d_hid, + num_flatten_dims=2, + param_attr=const_para_attr, + bias_attr=const_bias_attr, + ) return out -def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0.): +def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0.0): """ Add residual connection, layer normalization and droput to the out tensor optionally according to the value of process_cmd. @@ -1096,16 +1248,20 @@ def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0.): if cmd == "a": # add residual connection out = out + prev_out if prev_out else out elif cmd == "n": # add layer normalization - out = layers.layer_norm(out, - begin_norm_axis=len(out.shape) - 1, - param_attr=fluid.initializer.Constant(1.), - bias_attr=fluid.initializer.Constant(0.)) + out = layers.layer_norm( + out, + begin_norm_axis=len(out.shape) - 1, + param_attr=fluid.initializer.Constant(1.0), + bias_attr=fluid.initializer.Constant(0.0), + ) elif cmd == "d": # add dropout if dropout_rate: - out = layers.dropout(out, - dropout_prob=dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) + out = layers.dropout( + out, + dropout_prob=dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False, + ) return out @@ -1113,14 +1269,16 @@ def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0.): post_process_layer = pre_post_process_layer -def prepare_encoder(src_word, - src_pos, - src_vocab_size, - src_emb_dim, - src_max_len, - dropout_rate=0., - word_emb_param_name=None, - pos_enc_param_name=None): +def prepare_encoder( + src_word, + src_pos, + src_vocab_size, + src_emb_dim, + src_max_len, + dropout_rate=0.0, + word_emb_param_name=None, + pos_enc_param_name=None, +): """Add word embeddings and position encodings. The output tensor has a shape of: [batch_size, max_src_length_in_batch, d_model]. @@ -1132,14 +1290,18 @@ def prepare_encoder(src_word, size=[src_vocab_size, src_emb_dim], param_attr=fluid.ParamAttr( name=word_emb_param_name, - initializer=fluid.initializer.ConstantInitializer(0.001))) + initializer=fluid.initializer.ConstantInitializer(0.001), + ), + ) else: src_word_emb = layers.embedding( src_word, size=[src_vocab_size, src_emb_dim], - param_attr=fluid.ParamAttr(name=word_emb_param_name, - initializer=fluid.initializer.Normal( - 0., src_emb_dim**-0.5))) + param_attr=fluid.ParamAttr( + name=word_emb_param_name, + initializer=fluid.initializer.Normal(0.0, src_emb_dim**-0.5), + ), + ) src_word_emb = layers.scale(x=src_word_emb, scale=src_emb_dim**0.5) src_pos_enc = layers.embedding( @@ -1148,76 +1310,109 @@ def prepare_encoder(src_word, param_attr=fluid.ParamAttr( name=pos_enc_param_name, trainable=False, - initializer=fluid.initializer.ConstantInitializer(0.001))) + initializer=fluid.initializer.ConstantInitializer(0.001), + ), + ) src_pos_enc.stop_gradient = True enc_input = src_word_emb + src_pos_enc - return layers.dropout(enc_input, - dropout_prob=dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) if dropout_rate else enc_input - - -prepare_encoder = partial(prepare_encoder, - pos_enc_param_name=pos_enc_param_names[0]) -prepare_decoder = partial(prepare_encoder, - pos_enc_param_name=pos_enc_param_names[1]) - - -def encoder_layer(enc_input, - attn_bias, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate=0.): + return ( + layers.dropout( + enc_input, + dropout_prob=dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False, + ) + if dropout_rate + else enc_input + ) + + +prepare_encoder = partial( + prepare_encoder, pos_enc_param_name=pos_enc_param_names[0] +) +prepare_decoder = partial( + prepare_encoder, pos_enc_param_name=pos_enc_param_names[1] +) + + +def encoder_layer( + enc_input, + attn_bias, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, +): """The encoder layers that can be stacked to form a deep encoder. This module consits of a multi-head (self) attention followed by position-wise feed-forward networks and both the two components companied with the post_process_layer to add residual connection, layer normalization and droput. """ - attn_output = multi_head_attention(enc_input, enc_input, enc_input, - attn_bias, d_key, d_value, d_model, - n_head, dropout_rate) - attn_output = post_process_layer(enc_input, attn_output, "dan", - dropout_rate) + attn_output = multi_head_attention( + enc_input, + enc_input, + enc_input, + attn_bias, + d_key, + d_value, + d_model, + n_head, + dropout_rate, + ) + attn_output = post_process_layer( + enc_input, attn_output, "dan", dropout_rate + ) ffd_output = positionwise_feed_forward(attn_output, d_inner_hid, d_model) return post_process_layer(attn_output, ffd_output, "dan", dropout_rate) -def encoder(enc_input, +def encoder( + enc_input, + attn_bias, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, +): + """ + The encoder is composed of a stack of identical layers returned by calling + encoder_layer. + """ + for i in range(n_layer): + enc_output = encoder_layer( + enc_input, attn_bias, - n_layer, n_head, d_key, d_value, d_model, d_inner_hid, - dropout_rate=0.): - """ - The encoder is composed of a stack of identical layers returned by calling - encoder_layer. - """ - for i in range(n_layer): - enc_output = encoder_layer(enc_input, attn_bias, n_head, d_key, d_value, - d_model, d_inner_hid, dropout_rate) + dropout_rate, + ) enc_input = enc_output return enc_output -def decoder_layer(dec_input, - enc_output, - slf_attn_bias, - dec_enc_attn_bias, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate=0., - cache=None): - """ The layer to be stacked in decoder part. +def decoder_layer( + dec_input, + enc_output, + slf_attn_bias, + dec_enc_attn_bias, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, + cache=None, +): + """The layer to be stacked in decoder part. The structure of this module is similar to that in the encoder part except a multi-head attention is added to implement encoder-decoder attention. """ @@ -1270,18 +1465,20 @@ def decoder_layer(dec_input, return dec_output -def decoder(dec_input, - enc_output, - dec_slf_attn_bias, - dec_enc_attn_bias, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate=0., - caches=None): +def decoder( + dec_input, + enc_output, + dec_slf_attn_bias, + dec_enc_attn_bias, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, + caches=None, +): """ The decoder is composed of a stack of identical decoder_layer layers. """ @@ -1290,17 +1487,19 @@ def decoder(dec_input, if caches is not None: cache = caches[i] - dec_output = decoder_layer(dec_input, - enc_output, - dec_slf_attn_bias, - dec_enc_attn_bias, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate, - cache=cache) + dec_output = decoder_layer( + dec_input, + enc_output, + dec_slf_attn_bias, + dec_enc_attn_bias, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate, + cache=cache, + ) dec_input = dec_output return dec_output @@ -1311,12 +1510,15 @@ def make_all_inputs(input_fields): """ inputs = [] for input_field in input_fields: - input_var = layers.data(name=input_field, - shape=input_descs[input_field][0], - dtype=input_descs[input_field][1], - lod_level=input_descs[input_field][2] - if len(input_descs[input_field]) == 3 else 0, - append_batch_size=False) + input_var = layers.data( + name=input_field, + shape=input_descs[input_field][0], + dtype=input_descs[input_field][1], + lod_level=input_descs[input_field][2] + if len(input_descs[input_field]) == 3 + else 0, + append_batch_size=False, + ) inputs.append(input_var) return inputs @@ -1336,9 +1538,9 @@ def transformer( label_smooth_eps, ): if weight_sharing: - assert src_vocab_size == src_vocab_size, ( - "Vocabularies in source and target should be same for weight sharing." - ) + assert ( + src_vocab_size == src_vocab_size + ), "Vocabularies in source and target should be same for weight sharing." enc_inputs = make_all_inputs(encoder_data_input_fields) enc_output = wrap_encoder( @@ -1376,14 +1578,16 @@ def transformer( # cancel padding index in calculating the loss. label, weights = make_all_inputs(label_data_input_fields) if label_smooth_eps: - label = layers.label_smooth(label=layers.one_hot(input=label, - depth=trg_vocab_size), - epsilon=label_smooth_eps) + label = layers.label_smooth( + label=layers.one_hot(input=label, depth=trg_vocab_size), + epsilon=label_smooth_eps, + ) cost = layers.softmax_with_cross_entropy( logits=layers.reshape(predict, shape=[-1, trg_vocab_size]), label=label, - soft_label=True if label_smooth_eps else False) + soft_label=True if label_smooth_eps else False, + ) weighted_cost = cost * weights sum_cost = layers.reduce_sum(weighted_cost) token_num = layers.reduce_sum(weights) @@ -1392,95 +1596,122 @@ def transformer( return sum_cost, avg_cost, predict, token_num -def wrap_encoder(src_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate, - weight_sharing, - enc_inputs=None): +def wrap_encoder( + src_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate, + weight_sharing, + enc_inputs=None, +): """ The wrapper assembles together all needed layers for the encoder. """ if enc_inputs is None: # This is used to implement independent encoder program in inference. - src_word, src_pos, src_slf_attn_bias = \ - make_all_inputs(encoder_data_input_fields) + src_word, src_pos, src_slf_attn_bias = make_all_inputs( + encoder_data_input_fields + ) else: - src_word, src_pos, src_slf_attn_bias = \ - enc_inputs - enc_input = prepare_encoder(src_word, - src_pos, - src_vocab_size, - d_model, - max_length, - dropout_rate, - word_emb_param_name=word_emb_param_names[0]) - enc_output = encoder(enc_input, src_slf_attn_bias, n_layer, n_head, d_key, - d_value, d_model, d_inner_hid, dropout_rate) + src_word, src_pos, src_slf_attn_bias = enc_inputs + enc_input = prepare_encoder( + src_word, + src_pos, + src_vocab_size, + d_model, + max_length, + dropout_rate, + word_emb_param_name=word_emb_param_names[0], + ) + enc_output = encoder( + enc_input, + src_slf_attn_bias, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate, + ) return enc_output -def wrap_decoder(trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate, - weight_sharing, - dec_inputs=None, - enc_output=None, - caches=None): +def wrap_decoder( + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate, + weight_sharing, + dec_inputs=None, + enc_output=None, + caches=None, +): """ The wrapper assembles together all needed layers for the decoder. """ if dec_inputs is None: # This is used to implement independent decoder program in inference. - trg_word, trg_pos, trg_slf_attn_bias, trg_src_attn_bias, \ - enc_output = make_all_inputs( - decoder_data_input_fields) + ( + trg_word, + trg_pos, + trg_slf_attn_bias, + trg_src_attn_bias, + enc_output, + ) = make_all_inputs(decoder_data_input_fields) else: trg_word, trg_pos, trg_slf_attn_bias, trg_src_attn_bias = dec_inputs - dec_input = prepare_decoder(trg_word, - trg_pos, - trg_vocab_size, - d_model, - max_length, - dropout_rate, - word_emb_param_name=word_emb_param_names[0] - if weight_sharing else word_emb_param_names[1]) - dec_output = decoder(dec_input, - enc_output, - trg_slf_attn_bias, - trg_src_attn_bias, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate, - caches=caches) + dec_input = prepare_decoder( + trg_word, + trg_pos, + trg_vocab_size, + d_model, + max_length, + dropout_rate, + word_emb_param_name=word_emb_param_names[0] + if weight_sharing + else word_emb_param_names[1], + ) + dec_output = decoder( + dec_input, + enc_output, + trg_slf_attn_bias, + trg_src_attn_bias, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate, + caches=caches, + ) # Return logits for training and probs for inference. if weight_sharing: - predict = layers.matmul(x=dec_output, - y=fluid.framework._get_var( - word_emb_param_names[0]), - transpose_y=True) + predict = layers.matmul( + x=dec_output, + y=fluid.framework._get_var(word_emb_param_names[0]), + transpose_y=True, + ) else: - predict = layers.fc(input=dec_output, - size=trg_vocab_size, - num_flatten_dims=2, - param_attr=const_para_attr, - bias_attr=const_bias_attr) + predict = layers.fc( + input=dec_output, + size=trg_vocab_size, + num_flatten_dims=2, + param_attr=const_para_attr, + bias_attr=const_bias_attr, + ) if dec_inputs is None: predict = layers.softmax(predict) return predict @@ -1506,86 +1737,108 @@ def fast_decode( Use beam search to decode. Caches will be used to store states of history steps which can make the decoding faster. """ - enc_output = wrap_encoder(src_vocab_size, max_in_len, n_layer, n_head, - d_key, d_value, d_model, d_inner_hid, - dropout_rate, weight_sharing) - start_tokens, init_scores, trg_src_attn_bias = \ - make_all_inputs(fast_decoder_data_input_fields ) + enc_output = wrap_encoder( + src_vocab_size, + max_in_len, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate, + weight_sharing, + ) + start_tokens, init_scores, trg_src_attn_bias = make_all_inputs( + fast_decoder_data_input_fields + ) def beam_search(): - max_len = layers.fill_constant(shape=[1], - dtype=start_tokens.dtype, - value=max_out_len) - step_idx = layers.fill_constant(shape=[1], - dtype=start_tokens.dtype, - value=0) + max_len = layers.fill_constant( + shape=[1], dtype=start_tokens.dtype, value=max_out_len + ) + step_idx = layers.fill_constant( + shape=[1], dtype=start_tokens.dtype, value=0 + ) cond = layers.less_than(x=step_idx, y=max_len) while_op = layers.While(cond) # array states will be stored for each step. - ids = layers.array_write(layers.reshape(start_tokens, (-1, 1)), - step_idx) + ids = layers.array_write( + layers.reshape(start_tokens, (-1, 1)), step_idx + ) scores = layers.array_write(init_scores, step_idx) # cell states will be overwrited at each step. # caches contains states of history steps to reduce redundant # computation in decoder. - caches = [{ - "k": - layers.fill_constant_batch_size_like(input=start_tokens, - shape=[-1, 0, d_model], - dtype=enc_output.dtype, - value=0), - "v": - layers.fill_constant_batch_size_like(input=start_tokens, - shape=[-1, 0, d_model], - dtype=enc_output.dtype, - value=0) - } for i in range(n_layer)] + caches = [ + { + "k": layers.fill_constant_batch_size_like( + input=start_tokens, + shape=[-1, 0, d_model], + dtype=enc_output.dtype, + value=0, + ), + "v": layers.fill_constant_batch_size_like( + input=start_tokens, + shape=[-1, 0, d_model], + dtype=enc_output.dtype, + value=0, + ), + } + for i in range(n_layer) + ] with while_op.block(): pre_ids = layers.array_read(array=ids, i=step_idx) pre_ids = layers.reshape(pre_ids, (-1, 1, 1)) pre_scores = layers.array_read(array=scores, i=step_idx) # sequence_expand can gather sequences according to lod thus can be # used in beam search to sift states corresponding to selected ids. - pre_src_attn_bias = layers.sequence_expand(x=trg_src_attn_bias, - y=pre_scores) + pre_src_attn_bias = layers.sequence_expand( + x=trg_src_attn_bias, y=pre_scores + ) pre_enc_output = layers.sequence_expand(x=enc_output, y=pre_scores) - pre_caches = [{ - "k": - layers.sequence_expand(x=cache["k"], y=pre_scores), - "v": - layers.sequence_expand(x=cache["v"], y=pre_scores), - } for cache in caches] + pre_caches = [ + { + "k": layers.sequence_expand(x=cache["k"], y=pre_scores), + "v": layers.sequence_expand(x=cache["v"], y=pre_scores), + } + for cache in caches + ] pre_pos = layers.elementwise_mul( x=layers.fill_constant_batch_size_like( - input= - pre_enc_output, # can't use pre_ids here since it has lod + input=pre_enc_output, # can't use pre_ids here since it has lod value=1, shape=[-1, 1, 1], - dtype=pre_ids.dtype), + dtype=pre_ids.dtype, + ), y=layers.increment(x=step_idx, value=1.0, in_place=False), - axis=0) - logits = wrap_decoder(trg_vocab_size, - max_in_len, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate, - weight_sharing, - dec_inputs=(pre_ids, pre_pos, None, - pre_src_attn_bias), - enc_output=pre_enc_output, - caches=pre_caches) + axis=0, + ) + logits = wrap_decoder( + trg_vocab_size, + max_in_len, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate, + weight_sharing, + dec_inputs=(pre_ids, pre_pos, None, pre_src_attn_bias), + enc_output=pre_enc_output, + caches=pre_caches, + ) logits = layers.reshape(logits, (-1, trg_vocab_size)) topk_scores, topk_indices = layers.topk( - input=layers.softmax(logits), k=beam_size) - accu_scores = layers.elementwise_add(x=layers.log(topk_scores), - y=layers.reshape(pre_scores, - shape=[-1]), - axis=0) + input=layers.softmax(logits), k=beam_size + ) + accu_scores = layers.elementwise_add( + x=layers.log(topk_scores), + y=layers.reshape(pre_scores, shape=[-1]), + axis=0, + ) # beam_search op uses lod to distinguish branches. topk_indices = layers.lod_reset(topk_indices, pre_ids) selected_ids, selected_scores = layers.beam_search( @@ -1594,7 +1847,8 @@ def beam_search(): ids=topk_indices, scores=accu_scores, beam_size=beam_size, - end_id=eos_idx) + end_id=eos_idx, + ) layers.increment(x=step_idx, value=1.0, in_place=True) # update states @@ -1610,7 +1864,8 @@ def beam_search(): layers.logical_and(x=length_cond, y=finish_cond, out=cond) finished_ids, finished_scores = layers.beam_search_decode( - ids, scores, beam_size=beam_size, end_id=eos_idx) + ids, scores, beam_size=beam_size, end_id=eos_idx + ) return finished_ids, finished_scores finished_ids, finished_scores = beam_search() @@ -1619,16 +1874,25 @@ def beam_search(): def get_model(is_dist, is_async): sum_cost, avg_cost, predict, token_num = transformer( - ModelHyperParams.src_vocab_size, ModelHyperParams.trg_vocab_size, - ModelHyperParams.max_length + 1, ModelHyperParams.n_layer, - ModelHyperParams.n_head, ModelHyperParams.d_key, - ModelHyperParams.d_value, ModelHyperParams.d_model, - ModelHyperParams.d_inner_hid, ModelHyperParams.dropout, - ModelHyperParams.weight_sharing, TrainTaskConfig.label_smooth_eps) - - local_lr_scheduler = LearningRateScheduler(ModelHyperParams.d_model, - TrainTaskConfig.warmup_steps, - TrainTaskConfig.learning_rate) + ModelHyperParams.src_vocab_size, + ModelHyperParams.trg_vocab_size, + ModelHyperParams.max_length + 1, + ModelHyperParams.n_layer, + ModelHyperParams.n_head, + ModelHyperParams.d_key, + ModelHyperParams.d_value, + ModelHyperParams.d_model, + ModelHyperParams.d_inner_hid, + ModelHyperParams.dropout, + ModelHyperParams.weight_sharing, + TrainTaskConfig.label_smooth_eps, + ) + + local_lr_scheduler = LearningRateScheduler( + ModelHyperParams.d_model, + TrainTaskConfig.warmup_steps, + TrainTaskConfig.learning_rate, + ) # Context to do validation. test_program = fluid.default_main_program().clone(for_test=True) @@ -1637,24 +1901,33 @@ def get_model(is_dist, is_async): learning_rate=local_lr_scheduler.learning_rate, beta1=TrainTaskConfig.beta1, beta2=TrainTaskConfig.beta2, - epsilon=TrainTaskConfig.eps) + epsilon=TrainTaskConfig.eps, + ) optimizer.minimize(sum_cost) elif is_async: optimizer = fluid.optimizer.SGD(0.003) optimizer.minimize(sum_cost) else: - lr_decay = fluid.layers\ - .learning_rate_scheduler\ - .noam_decay(ModelHyperParams.d_model, - TrainTaskConfig.warmup_steps) - - optimizer = fluid.optimizer.Adam(learning_rate=lr_decay, - beta1=TrainTaskConfig.beta1, - beta2=TrainTaskConfig.beta2, - epsilon=TrainTaskConfig.eps) + lr_decay = fluid.layers.learning_rate_scheduler.noam_decay( + ModelHyperParams.d_model, TrainTaskConfig.warmup_steps + ) + + optimizer = fluid.optimizer.Adam( + learning_rate=lr_decay, + beta1=TrainTaskConfig.beta1, + beta2=TrainTaskConfig.beta2, + epsilon=TrainTaskConfig.eps, + ) optimizer.minimize(sum_cost) - return sum_cost, avg_cost, predict, token_num, local_lr_scheduler, test_program + return ( + sum_cost, + avg_cost, + predict, + token_num, + local_lr_scheduler, + test_program, + ) def update_args(): @@ -1662,24 +1935,33 @@ def update_args(): trg_dict = DataReader.load_dict(TrainTaskConfig.trg_vocab_fpath) dict_args = [ "src_vocab_size", - str(len(src_dict)), "trg_vocab_size", - str(len(trg_dict)), "bos_idx", - str(src_dict[TrainTaskConfig.special_token[0]]), "eos_idx", - str(src_dict[TrainTaskConfig.special_token[1]]), "unk_idx", - str(src_dict[TrainTaskConfig.special_token[2]]) + str(len(src_dict)), + "trg_vocab_size", + str(len(trg_dict)), + "bos_idx", + str(src_dict[TrainTaskConfig.special_token[0]]), + "eos_idx", + str(src_dict[TrainTaskConfig.special_token[1]]), + "unk_idx", + str(src_dict[TrainTaskConfig.special_token[2]]), ] merge_cfg_from_list(dict_args, [TrainTaskConfig, ModelHyperParams]) class DistTransformer2x2(TestDistRunnerBase): - def run_pserver(self, args): get_model(True, not args.sync_mode) - t = self.get_transpiler(args.trainer_id, fluid.default_main_program(), - args.endpoints, args.trainers, args.sync_mode) + t = self.get_transpiler( + args.trainer_id, + fluid.default_main_program(), + args.endpoints, + args.trainers, + args.sync_mode, + ) pserver_prog = t.get_pserver_program(args.current_endpoint) - startup_prog = t.get_startup_program(args.current_endpoint, - pserver_prog) + startup_prog = t.get_startup_program( + args.current_endpoint, pserver_prog + ) place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -1688,18 +1970,31 @@ def run_pserver(self, args): def run_trainer(self, args): TrainTaskConfig.use_gpu = args.use_cuda - sum_cost, avg_cost, predict, token_num, local_lr_scheduler, test_program = get_model( - args.is_dist, not args.sync_mode) + ( + sum_cost, + avg_cost, + predict, + token_num, + local_lr_scheduler, + test_program, + ) = get_model(args.is_dist, not args.sync_mode) if args.is_dist: - t = self.get_transpiler(args.trainer_id, - fluid.default_main_program(), - args.endpoints, args.trainers, - args.sync_mode) + t = self.get_transpiler( + args.trainer_id, + fluid.default_main_program(), + args.endpoints, + args.trainers, + args.sync_mode, + ) trainer_prog = t.get_trainer_program() TrainTaskConfig.batch_size = 10 - TrainTaskConfig.train_file_pattern = TrainTaskConfig.data_path + "train.tok.clean.bpe.32000.en-de.train_{}".format( - args.trainer_id) + TrainTaskConfig.train_file_pattern = ( + TrainTaskConfig.data_path + + "train.tok.clean.bpe.32000.en-de.train_{}".format( + args.trainer_id + ) + ) else: TrainTaskConfig.batch_size = 20 trainer_prog = fluid.default_main_program() @@ -1713,8 +2008,17 @@ def run_trainer(self, args): TrainTaskConfig.local = not args.is_dist - train_loop(startup_exe, trainer_prog, 1, sum_cost, avg_cost, - local_lr_scheduler, token_num, predict, test_program) + train_loop( + startup_exe, + trainer_prog, + 1, + sum_cost, + avg_cost, + local_lr_scheduler, + token_num, + predict, + test_program, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dist_word2vec.py b/python/paddle/fluid/tests/unittests/dist_word2vec.py index ca322d30cdc62..0941dabcc5e5b 100644 --- a/python/paddle/fluid/tests/unittests/dist_word2vec.py +++ b/python/paddle/fluid/tests/unittests/dist_word2vec.py @@ -28,7 +28,6 @@ class TestDistWord2vec2x2(TestDistRunnerBase): - def get_model(self, batch_size=2): BATCH_SIZE = batch_size @@ -40,7 +39,9 @@ def __network__(words): is_sparse=IS_SPARSE, param_attr=fluid.ParamAttr( name='shared_w', - initializer=fluid.initializer.Constant(value=0.1))) + initializer=fluid.initializer.Constant(value=0.1), + ), + ) embed_second = fluid.layers.embedding( input=words[1], size=[dict_size, EMBED_SIZE], @@ -48,7 +49,9 @@ def __network__(words): is_sparse=IS_SPARSE, param_attr=fluid.ParamAttr( name='shared_w', - initializer=fluid.initializer.Constant(value=0.1))) + initializer=fluid.initializer.Constant(value=0.1), + ), + ) embed_third = fluid.layers.embedding( input=words[2], size=[dict_size, EMBED_SIZE], @@ -56,7 +59,9 @@ def __network__(words): is_sparse=IS_SPARSE, param_attr=fluid.ParamAttr( name='shared_w', - initializer=fluid.initializer.Constant(value=0.1))) + initializer=fluid.initializer.Constant(value=0.1), + ), + ) embed_forth = fluid.layers.embedding( input=words[3], size=[dict_size, EMBED_SIZE], @@ -64,25 +69,33 @@ def __network__(words): is_sparse=IS_SPARSE, param_attr=fluid.ParamAttr( name='shared_w', - initializer=fluid.initializer.Constant(value=0.1))) + initializer=fluid.initializer.Constant(value=0.1), + ), + ) concat_embed = fluid.layers.concat( input=[embed_first, embed_second, embed_third, embed_forth], - axis=1) + axis=1, + ) hidden1 = fluid.layers.fc( input=concat_embed, size=HIDDEN_SIZE, act='sigmoid', param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.1))) + initializer=fluid.initializer.Constant(value=0.1) + ), + ) predict_word = fluid.layers.fc( input=hidden1, size=dict_size, act='softmax', param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.1))) - cost = fluid.layers.cross_entropy(input=predict_word, - label=words[4]) + initializer=fluid.initializer.Constant(value=0.1) + ), + ) + cost = fluid.layers.cross_entropy( + input=predict_word, label=words[4] + ) avg_cost = paddle.mean(cost) return avg_cost, predict_word @@ -90,30 +103,41 @@ def __network__(words): dict_size = len(word_dict) first_word = fluid.layers.data(name='firstw', shape=[1], dtype='int64') - second_word = fluid.layers.data(name='secondw', - shape=[1], - dtype='int64') + second_word = fluid.layers.data( + name='secondw', shape=[1], dtype='int64' + ) third_word = fluid.layers.data(name='thirdw', shape=[1], dtype='int64') forth_word = fluid.layers.data(name='forthw', shape=[1], dtype='int64') next_word = fluid.layers.data(name='nextw', shape=[1], dtype='int64') avg_cost, predict_word = __network__( - [first_word, second_word, third_word, forth_word, next_word]) + [first_word, second_word, third_word, forth_word, next_word] + ) inference_program = paddle.fluid.default_main_program().clone() sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001) sgd_optimizer.minimize(avg_cost) - train_reader = paddle.batch(paddle.dataset.imikolov.train(word_dict, N), - BATCH_SIZE) - test_reader = paddle.batch(paddle.dataset.imikolov.test(word_dict, N), - BATCH_SIZE) + train_reader = paddle.batch( + paddle.dataset.imikolov.train(word_dict, N), BATCH_SIZE + ) + test_reader = paddle.batch( + paddle.dataset.imikolov.test(word_dict, N), BATCH_SIZE + ) - return inference_program, avg_cost, train_reader, test_reader, None, predict_word + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + None, + predict_word, + ) if __name__ == "__main__": import os + os.environ['CPU_NUM'] = '1' os.environ['USE_CUDA'] = "FALSE" runtime_main(TestDistWord2vec2x2) diff --git a/python/paddle/fluid/tests/unittests/distributed_fused_lamb_test_base.py b/python/paddle/fluid/tests/unittests/distributed_fused_lamb_test_base.py index 230272a1cff77..b2ef0976d62e7 100644 --- a/python/paddle/fluid/tests/unittests/distributed_fused_lamb_test_base.py +++ b/python/paddle/fluid/tests/unittests/distributed_fused_lamb_test_base.py @@ -73,7 +73,6 @@ def prune_fwd_bwd_ops(program, start_idx): class GradClipDecorator(ClipGradBase): - def __init__(self, clip, clip_after_allreduce): self.clip = clip self.clip_after_allreduce = clip_after_allreduce @@ -89,17 +88,18 @@ def _insert_allreduce_ops(self, params_grads): scale = 1.0 / world_size # scale = 1.0 for p, g in params_grads: - block.append_op(type='c_allreduce_sum', - inputs={'X': [g]}, - outputs={'Out': [g]}, - attrs={ - 'ring_id': 0, - 'use_calc_stream': True - }) - block.append_op(type='scale', - inputs={'X': [g]}, - outputs={'Out': [g]}, - attrs={'scale': scale}) + block.append_op( + type='c_allreduce_sum', + inputs={'X': [g]}, + outputs={'Out': [g]}, + attrs={'ring_id': 0, 'use_calc_stream': True}, + ) + block.append_op( + type='scale', + inputs={'X': [g]}, + outputs={'Out': [g]}, + attrs={'scale': scale}, + ) def _static_clip(self, params_grads): if self.clip_after_allreduce: @@ -112,7 +112,6 @@ def _static_clip(self, params_grads): class IdentityGradClip(ClipGradBase): - def _dygraph_clip(self, params_grads): return params_grads @@ -129,12 +128,14 @@ def run_model(use_distributed_lamb, use_fp16, use_master_param_norm, **kwargs): with paddle.static.program_guard(main, startup): with paddle.fluid.unique_name.guard(): with paddle.static.amp.fp16_guard(): - image = paddle.static.data(name='image', - shape=[None, 3, 224, 224], - dtype=paddle.float32) - label = paddle.static.data(name='label', - shape=[None, 1], - dtype=paddle.int64) + image = paddle.static.data( + name='image', + shape=[None, 3, 224, 224], + dtype=paddle.float32, + ) + label = paddle.static.data( + name='label', shape=[None, 1], dtype=paddle.int64 + ) model = resnet() pred = model(image) loss_fn = paddle.nn.loss.CrossEntropyLoss() @@ -160,18 +161,24 @@ def run_model(use_distributed_lamb, use_fp16, use_master_param_norm, **kwargs): kwargs.pop('clip_after_allreduce', None) kwargs.pop('alignment', None) kwargs.pop('use_master_acc_grad', None) - base_clip = grad_clip if grad_clip is not None else IdentityGradClip( + base_clip = ( + grad_clip if grad_clip is not None else IdentityGradClip() + ) + kwargs['grad_clip'] = GradClipDecorator( + base_clip, clip_after_allreduce ) - kwargs['grad_clip'] = GradClipDecorator(base_clip, - clip_after_allreduce) kwargs.pop('gradient_accumulation_steps', None) optimizer = optimizer_class(**kwargs) get_parameter = optimizer._get_parameter amp_list = paddle.static.amp.AutoMixedPrecisionLists( custom_white_list=[ - 'batch_norm', 'batch_norm_grad', 'conv2d', 'conv2d_grad' - ]) + 'batch_norm', + 'batch_norm_grad', + 'conv2d', + 'conv2d_grad', + ] + ) if use_fp16: if not use_distributed_lamb: optimizer._multi_precision = True @@ -182,14 +189,16 @@ def run_model(use_distributed_lamb, use_fp16, use_master_param_norm, **kwargs): init_loss_scaling=1.0, use_dynamic_loss_scaling=False, use_pure_fp16=use_fp16, - use_fp16_guard=use_fp16) + use_fp16_guard=use_fp16, + ) amp_init = optimizer.amp_init else: amp_init = None if gm_steps > 1 and not use_distributed_lamb: optimizer = paddle.fluid.optimizer.GradientMergeOptimizer( - optimizer, k_steps=gm_steps, avg=False) + optimizer, k_steps=gm_steps, avg=False + ) params_grads = optimizer.backward(loss, startup) op_num = len(main.global_block().ops) @@ -222,8 +231,9 @@ def gen_random_grad_tensor(grad): def reader(): for _ in range(6): - yield dict([(grad.name, gen_random_grad_tensor(grad)) - for grad in grads]) + yield dict( + [(grad.name, gen_random_grad_tensor(grad)) for grad in grads] + ) scope = paddle.static.Scope() fetch_list = params @@ -253,7 +263,6 @@ def reader(): class TestDistributedFusedLamb(unittest.TestCase): - @classmethod def setUpClass(cls): if not paddle.is_compiled_with_cuda(): @@ -266,28 +275,28 @@ def setUpClass(cls): def config(self): clip_after_allreduce = bool( - distutils.util.strtobool(os.getenv('CLIP_AFTER_ALLREDUCE', 'True'))) + distutils.util.strtobool(os.getenv('CLIP_AFTER_ALLREDUCE', 'True')) + ) max_global_norm = float(os.getenv('MAX_GLOBAL_NORM', -1.0)) gm_steps = int(os.getenv('GRADIENT_MERGE_STEPS', 1)) use_master_acc_grad = bool(int(os.getenv('USE_MASTER_ACC_GRAD', '1'))) - print('clip_after_allreduce = {}, max_global_norm = {}'.format( - clip_after_allreduce, max_global_norm)) + print( + 'clip_after_allreduce = {}, max_global_norm = {}'.format( + clip_after_allreduce, max_global_norm + ) + ) return { - 'clip_after_allreduce': - clip_after_allreduce, - 'gradient_accumulation_steps': - gm_steps, - 'grad_clip': - paddle.nn.ClipGradByGlobalNorm(max_global_norm) - if max_global_norm > 0 else None, - 'use_master_acc_grad': - use_master_acc_grad, + 'clip_after_allreduce': clip_after_allreduce, + 'gradient_accumulation_steps': gm_steps, + 'grad_clip': paddle.nn.ClipGradByGlobalNorm(max_global_norm) + if max_global_norm > 0 + else None, + 'use_master_acc_grad': use_master_acc_grad, } - def run_main(self, - use_fp16, - use_master_param_norm=True, - use_master_acc_grad=True): + def run_main( + self, use_fp16, use_master_param_norm=True, use_master_acc_grad=True + ): if not paddle.is_compiled_with_cuda(): return @@ -316,7 +325,8 @@ def run_main(self, for ret1, ret2 in zip(result1, result2): max_diff = np.max(np.abs(ret1 - ret2)) msg = 'max_diff = {} atol = {} when use_fp16 = {} , use_master_param_norm = {}'.format( - max_diff, atol, use_fp16, use_master_param_norm) + max_diff, atol, use_fp16, use_master_param_norm + ) self.assertTrue(max_diff < atol, msg) print(msg) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/auto_parallel_pass_test_base.py b/python/paddle/fluid/tests/unittests/distributed_passes/auto_parallel_pass_test_base.py index 4028bae08a45a..5b9bd4faf3845 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/auto_parallel_pass_test_base.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/auto_parallel_pass_test_base.py @@ -25,11 +25,14 @@ sys.path.append("..") import auto_parallel_gpt_model as modeling -from auto_parallel_gpt_model import GPTModel, GPTForPretraining, GPTPretrainingCriterion +from auto_parallel_gpt_model import ( + GPTModel, + GPTForPretraining, + GPTPretrainingCriterion, +) class AutoPallelPassTestBase(DistPassTestBase): - def setUp(self): paddle.enable_static() seed = int(os.environ.get('SEED', -1)) @@ -60,14 +63,12 @@ def apply_no_passes(self): fleet.init(is_collective=True, strategy=dist_strategy) def check_main(self, gpus=None, **kwargs): - no_pass_rets = self._distributed_launch(model=None, - apply_pass=False, - gpus=gpus, - **kwargs) - pass_rets = self._distributed_launch(model=None, - apply_pass=True, - gpus=gpus, - **kwargs) + no_pass_rets = self._distributed_launch( + model=None, apply_pass=False, gpus=gpus, **kwargs + ) + pass_rets = self._distributed_launch( + model=None, apply_pass=True, gpus=gpus, **kwargs + ) self.check_results(no_pass_rets, pass_rets) def _run_gpu_main(self, model, apply_pass, dump_file, **kwargs): @@ -78,12 +79,18 @@ def _run_gpu_main(self, model, apply_pass, dump_file, **kwargs): self.apply_passes() else: self.apply_no_passes() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): with paddle.static.scope_guard(scope): with paddle.fluid.unique_name.guard(): - main_prog, startup_prog, inputs, outputs, data_loader = self.get_model( - place, **kwargs) + ( + main_prog, + startup_prog, + inputs, + outputs, + data_loader, + ) = self.get_model(place, **kwargs) inputs = self._to_var_names(inputs) outputs = self._to_var_names(outputs) @@ -98,8 +105,9 @@ def _run_gpu_main(self, model, apply_pass, dump_file, **kwargs): fetch_values = exe.run(main_prog, fetch_list=outputs) if paddle.distributed.get_rank() == 0: output_dict = OrderedDict(zip(outputs, fetch_values)) - print('batch {}, outputs {}'.format( - batch_id, output_dict)) + print( + 'batch {}, outputs {}'.format(batch_id, output_dict) + ) all_fetch_values.append(fetch_values) batch_id += 1 except paddle.fluid.core.EOFException: @@ -108,9 +116,9 @@ def _run_gpu_main(self, model, apply_pass, dump_file, **kwargs): with open(dump_file, "wb") as f: pickle.dump(all_fetch_values, f) - def get_gpt_model(self, strategy, place, batch_size, sequence_len, - vocab_size, **kwargs): - + def get_gpt_model( + self, strategy, place, batch_size, sequence_len, vocab_size, **kwargs + ): def gen_data(): np.random.seed(2021) for _ in range(10): @@ -121,14 +129,19 @@ def gen_data(): loss_mask = [] for _ in range(batch_size): tokens.append( - np.random.randint(vocab_size, - size=sequence_len).astype("int64")) + np.random.randint(vocab_size, size=sequence_len).astype( + "int64" + ) + ) position_ids.append(np.arange(sequence_len).astype("int64")) attention_mask.append( - [np.tril(np.ones(sequence_len)).astype("float32")]) + [np.tril(np.ones(sequence_len)).astype("float32")] + ) labels.append( - np.random.randint(vocab_size, - size=sequence_len).astype("int64")) + np.random.randint(vocab_size, size=sequence_len).astype( + "int64" + ) + ) loss_mask.append(np.ones(sequence_len).astype("float32")) yield tokens, position_ids, attention_mask, labels, loss_mask @@ -136,65 +149,74 @@ def gen_data(): modeling.init_global() if strategy == "dp": modeling._global_parallel_strategy = "dp" - modeling._global_process_mesh = auto.ProcessMesh(mesh=[0, 1], - dim_names=["x"]) + modeling._global_process_mesh = auto.ProcessMesh( + mesh=[0, 1], dim_names=["x"] + ) elif strategy == "mp": modeling._global_parallel_strategy = "mp" - modeling._global_process_mesh = auto.ProcessMesh(mesh=[0, 1], - dim_names=["x"]) + modeling._global_process_mesh = auto.ProcessMesh( + mesh=[0, 1], dim_names=["x"] + ) else: raise ValueError("'get_gpt_model' only support dp and mp.") - tokens = paddle.static.data(name="tokens", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = paddle.static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + tokens = paddle.static.data( + name="tokens", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = paddle.static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = paddle.static.data( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float32') - labels = paddle.static.data(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = paddle.static.data(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float32') + dtype='float32', + ) + labels = paddle.static.data( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = paddle.static.data( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float32' + ) data_holder = [tokens, position_ids, attention_mask, labels, loss_mask] data_loader = paddle.fluid.io.DataLoader.from_generator( - feed_list=data_holder, capacity=70, iterable=False) + feed_list=data_holder, capacity=70, iterable=False + ) data_loader.set_batch_generator(gen_data, paddle.static.cuda_places()) if modeling._global_parallel_strategy == "dp": - auto.shard_tensor(tokens, modeling._global_process_mesh, - ["x", None]) + auto.shard_tensor( + tokens, modeling._global_process_mesh, ["x", None] + ) elif modeling._global_parallel_strategy == "pp": auto.shard_tensor(tokens, modeling.PP_MESH_LIST[0], [None, None]) - auto.shard_tensor(attention_mask, modeling.PP_MESH_LIST[0], - [None, None, None, None]) - - gpt = GPTModel(vocab_size=1000, - hidden_size=64, - num_hidden_layers=2, - num_attention_heads=8, - intermediate_size=256, - hidden_act="gelu", - hidden_dropout_prob=0.0, - attention_probs_dropout_prob=0.0, - max_position_embeddings=1024, - type_vocab_size=1, - initializer_range=0.02, - pad_token_id=0, - eos_token_id=7, - bos_token_id=0, - eol_token_id=3) - - model = GPTForPretraining(gpt, - vocab_size=1000, - hidden_size=64, - initializer_range=0.02) + auto.shard_tensor( + attention_mask, + modeling.PP_MESH_LIST[0], + [None, None, None, None], + ) + + gpt = GPTModel( + vocab_size=1000, + hidden_size=64, + num_hidden_layers=2, + num_attention_heads=8, + intermediate_size=256, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=1024, + type_vocab_size=1, + initializer_range=0.02, + pad_token_id=0, + eos_token_id=7, + bos_token_id=0, + eol_token_id=3, + ) + + model = GPTForPretraining( + gpt, vocab_size=1000, hidden_size=64, initializer_range=0.02 + ) preds = model(tokens, position_ids, attention_mask) criterion = GPTPretrainingCriterion() loss = criterion(preds, labels, loss_mask) @@ -202,17 +224,26 @@ def gen_data(): clip = paddle.nn.ClipGradByNorm(clip_norm=1.0) if kwargs.get('optimizer', None) == "LarsMomentum": optimizer = paddle.fluid.optimizer.LarsMomentumOptimizer( - learning_rate=0.001, momentum=0.9) + learning_rate=0.001, momentum=0.9 + ) else: - optimizer = paddle.optimizer.Adam(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=clip) + optimizer = paddle.optimizer.Adam( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=clip, + ) optimizer = fleet.distributed_optimizer(optimizer) startup_program = paddle.static.default_startup_program() _, _, dist_startup_prog, dist_main_prog = optimizer.minimize( - loss, startup_program) - - return dist_main_prog, dist_startup_prog, data_holder, [loss - ], data_loader + loss, startup_program + ) + + return ( + dist_main_prog, + dist_startup_prog, + data_holder, + [loss], + data_loader, + ) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/check_pass_conflict_example.py b/python/paddle/fluid/tests/unittests/distributed_passes/check_pass_conflict_example.py index ffb8ea8e381c3..1223fbc0c1936 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/check_pass_conflict_example.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/check_pass_conflict_example.py @@ -19,7 +19,6 @@ class CheckPassConflictTest1(PassConflictChecker): - def pass_config(self): return [ new_pass("fuse_all_reduce", {"max_memory_size": 1024 * 1024}), @@ -31,7 +30,6 @@ def test_resnet(self): class CheckPassConflictTest2(PassConflictChecker): - def pass_config(self): return [ new_pass("fuse_elewise_add_act"), diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/dist_pass_test_base.py b/python/paddle/fluid/tests/unittests/distributed_passes/dist_pass_test_base.py index 6b0d53d2d00b6..5f62faeb4228f 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/dist_pass_test_base.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/dist_pass_test_base.py @@ -42,7 +42,7 @@ def prepare_python_path_and_return_module(path): python_path = dirname os.environ[env_name] = python_path print('GLOG_v=', os.environ.get('GLOG_v', None), flush=1) - return filename[:-len(py_suffix)] + return filename[: -len(py_suffix)] def remove_path_if_exists(path): @@ -57,7 +57,6 @@ def remove_path_if_exists(path): # NOTE: only support GPU now class DistPassTestBase(unittest.TestCase): - def setUp(self): paddle.enable_static() if paddle.is_compiled_with_cuda(): @@ -86,30 +85,31 @@ def apply_passes(self, main_prog, startup_prog): raise NotImplementedError() def check_main(self, model=None, gpus=None, **kwargs): - pass_rets = self._distributed_launch(model=model, - apply_pass=True, - gpus=gpus, - **kwargs) - no_pass_rets = self._distributed_launch(model=model, - apply_pass=False, - gpus=gpus, - **kwargs) + pass_rets = self._distributed_launch( + model=model, apply_pass=True, gpus=gpus, **kwargs + ) + no_pass_rets = self._distributed_launch( + model=model, apply_pass=False, gpus=gpus, **kwargs + ) self.check_results(no_pass_rets, pass_rets) def check_results(self, no_pass_rets, pass_rets): self.assertEqual(len(no_pass_rets), len(pass_rets)) for no_pass_ret, pass_ret in zip(no_pass_rets, pass_rets): self.assertEqual(len(no_pass_ret), len(pass_ret)) - for i, (out_var_no_pass, - out_var_pass) in enumerate(zip(no_pass_ret, pass_ret)): + for i, (out_var_no_pass, out_var_pass) in enumerate( + zip(no_pass_ret, pass_ret) + ): if out_var_no_pass is None: self.assertTrue(out_var_pass is None) else: - np.testing.assert_allclose(out_var_no_pass, - out_var_pass, - rtol=self.rtol, - atol=self.atol, - equal_nan=self.equal_nan) + np.testing.assert_allclose( + out_var_no_pass, + out_var_pass, + rtol=self.rtol, + atol=self.atol, + equal_nan=self.equal_nan, + ) @classmethod def _to_var_names(cls, names_or_vars): @@ -129,12 +129,14 @@ def _run_gpu_main(self, model, apply_pass, dump_file, **kwargs): scope = paddle.static.Scope() if model is None: model = self.get_model - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): with paddle.static.scope_guard(scope): with paddle.fluid.unique_name.guard(): main_prog, startup_prog, inputs, outputs, reader = model( - place, **kwargs) + place, **kwargs + ) inputs = self._to_var_names(inputs) outputs = self._to_var_names(outputs) if apply_pass: @@ -146,7 +148,8 @@ def _run_gpu_main(self, model, apply_pass, dump_file, **kwargs): exe.run(startup_prog) for batch_id, input_data in enumerate(reader()): assert len(input_data) == len(inputs), "{} vs {}".format( - len(input_data), len(inputs)) + len(input_data), len(inputs) + ) feed = dict(zip(inputs, input_data)) fetch_values = exe.run(main_prog, feed=feed, fetch_list=outputs) if paddle.distributed.get_rank() == 0: @@ -204,26 +207,30 @@ def _distributed_launch(self, model, apply_pass, gpus=None, **kwargs): with open(model_dump_file, 'wb') as f: pickle.dump(model, f) - cmd = [ - sys.executable, - "-u", - ] + coverage_args + [ - "-m", - "launch", - "--log_dir", - output_dir, - "--gpus", - gpus, - os.path.join(file_dir, "pass_run_main.py"), - "--file_path", - inspect.getfile(type(self)), - "--class_name", - type(self).__name__, - "--input_file", - input_dump_file, - "--output_dir", - output_dir, - ] + cmd = ( + [ + sys.executable, + "-u", + ] + + coverage_args + + [ + "-m", + "launch", + "--log_dir", + output_dir, + "--gpus", + gpus, + os.path.join(file_dir, "pass_run_main.py"), + "--file_path", + inspect.getfile(type(self)), + "--class_name", + type(self).__name__, + "--input_file", + input_dump_file, + "--output_dir", + output_dir, + ] + ) if apply_pass: cmd += ["--apply_pass"] if model is not None: @@ -232,17 +239,22 @@ def _distributed_launch(self, model, apply_pass, gpus=None, **kwargs): prepare_python_path_and_return_module(__file__) exitcode = os.system(' '.join(cmd)) self.assertEqual( - exitcode, 0, - "Pass test failed with apply_pass = {}, please view log in {}". - format(apply_pass, output_dir)) + exitcode, + 0, + "Pass test failed with apply_pass = {}, please view log in {}".format( + apply_pass, output_dir + ), + ) results = [] for i in range(num_gpus): dump_file = '{0}/{1}.bin'.format(output_dir, i) self.assertTrue( os.path.exists(dump_file), - "Pass test failed with apply_pass = {}, please view log in {}" - .format(apply_pass, output_dir)) + "Pass test failed with apply_pass = {}, please view log in {}".format( + apply_pass, output_dir + ), + ) with open(dump_file, "rb") as f: results.append(pickle.load(f)) return results @@ -252,7 +264,6 @@ def _distributed_launch(self, model, apply_pass, gpus=None, **kwargs): class PassConflictChecker(DistPassTestBase): - def setUp(self): os.environ['DEBUG'] = '1' # to save the debug directory super(PassConflictChecker, self).setUp() @@ -270,14 +281,20 @@ def apply_passes(self, main_prog, startup_prog): auto_pass_manager = PassManager(passes, auto_solve_conflict=True) new_passes = auto_pass_manager.passes self.assertEqual( - len(passes), len(new_passes), + len(passes), + len(new_passes), "After solving conflicts, the left passes are: {}".format( - auto_pass_manager.names)) + auto_pass_manager.names + ), + ) for i, (p1, p2) in enumerate(zip(passes, new_passes)): self.assertEqual( - id(p1), id(p2), - "After solving conflicts, the {}-th pass is different: {} vs {}" - .format(i, p1.name, p2.name)) + id(p1), + id(p2), + "After solving conflicts, the {}-th pass is different: {} vs {}".format( + i, p1.name, p2.name + ), + ) auto_pass_manager.apply([main_prog], [startup_prog]) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/model_zoo.py b/python/paddle/fluid/tests/unittests/distributed_passes/model_zoo.py index 9a48d117bb128..a0bff06e29cd9 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/model_zoo.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/model_zoo.py @@ -28,16 +28,15 @@ def get_seed_from_env(): return int(os.environ.get("SEED", 0)) -def resnet_model(place, - batch_size, - image_shape=[3, 224, 224], - num_classes=1000): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') - label = paddle.static.data(shape=[batch_size, 1], - dtype='int64', - name='label') +def resnet_model( + place, batch_size, image_shape=[3, 224, 224], num_classes=1000 +): + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) + label = paddle.static.data( + shape=[batch_size, 1], dtype='int64', name='label' + ) model = resnet(pretrained=False) loss_fn = nn.loss.CrossEntropyLoss() pred_out = model(image) @@ -58,9 +57,9 @@ def reader(): np.random.seed(seed + rank) for _ in range(10): image_np = np.random.random(size=image.shape).astype('float32') - label_np = np.random.randint(low=0, - high=num_classes, - size=label.shape).astype('int64') + label_np = np.random.randint( + low=0, high=num_classes, size=label.shape + ).astype('int64') yield image_np, label_np main_program = paddle.static.default_main_program() @@ -69,12 +68,12 @@ def reader(): def simple_net(place, batch_size, image_shape=[784], num_classes=10): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') - label = paddle.static.data(shape=[batch_size, 1], - dtype='int64', - name='label') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) + label = paddle.static.data( + shape=[batch_size, 1], dtype='int64', name='label' + ) linears = [nn.Linear(784, 784) for _ in range(3)] hidden = image for linear in linears: @@ -98,9 +97,9 @@ def reader(): np.random.seed(seed + rank) for _ in range(10): image_np = np.random.random(size=image.shape).astype('float32') - label_np = np.random.randint(low=0, - high=num_classes, - size=label.shape).astype('int64') + label_np = np.random.randint( + low=0, high=num_classes, size=label.shape + ).astype('int64') yield image_np, label_np main_program = paddle.static.default_main_program() diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/pass_run_main.py b/python/paddle/fluid/tests/unittests/distributed_passes/pass_run_main.py index cd98d39a09ead..cfea9a05c4e3e 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/pass_run_main.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/pass_run_main.py @@ -18,35 +18,43 @@ import importlib import os from paddle.distributed.fleet.launch_utils import run_with_coverage -from dist_pass_test_base import prepare_python_path_and_return_module, DistPassTestBase +from dist_pass_test_base import ( + prepare_python_path_and_return_module, + DistPassTestBase, +) def parse_args(): parser = argparse.ArgumentParser( - description='arguments for distributed pass tests') + description='arguments for distributed pass tests' + ) parser.add_argument('--file_path', type=str, help='The test file path.') parser.add_argument( '--class_name', type=str, - help= - 'The test class name. It is the class name that inherits the DistPassTestBase class.' + help='The test class name. It is the class name that inherits the DistPassTestBase class.', + ) + parser.add_argument( + '--apply_pass', + default=False, + action="store_true", + help='Whether to apply distributed passes.', ) - parser.add_argument('--apply_pass', - default=False, - action="store_true", - help='Whether to apply distributed passes.') parser.add_argument( '--input_file', type=str, - help='The input file which contains the dumped input arguments.') + help='The input file which contains the dumped input arguments.', + ) parser.add_argument( '--output_dir', type=str, - help='The output directory to save the logs and output results.') + help='The output directory to save the logs and output results.', + ) parser.add_argument( '--model_file', type=str, - help='The input model file which contains the dumped model function.') + help='The input model file which contains the dumped model function.', + ) return parser.parse_args() diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/ps_pass_test_base.py b/python/paddle/fluid/tests/unittests/distributed_passes/ps_pass_test_base.py index bd2fa6b019848..87f275c21c6f3 100755 --- a/python/paddle/fluid/tests/unittests/distributed_passes/ps_pass_test_base.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/ps_pass_test_base.py @@ -17,11 +17,12 @@ import shlex import unittest from paddle.fluid.tests.unittests.distributed_passes.dist_pass_test_base import ( # noqa: F401 - prepare_python_path_and_return_module, remove_path_if_exists) + prepare_python_path_and_return_module, + remove_path_if_exists, +) class PsPassTestBase(unittest.TestCase): - def init(self): self.config = {} self.config['ps_mode_config'] = "" @@ -46,30 +47,43 @@ def ps_launch(self, ps_mode="cpu-ps"): if ps_mode == "cpu-ps" or ps_mode == 'heter-ps': os.environ['WITH_DISTRIBUTE'] = 'ON' - cmd = [ - sys.executable, - "-u", - ] + [ - "-m", "launch", "--log_dir", self.config['log_dir'], - "--worker_num", self.config['worker_num'], "--server_num", - self.config['server_num'] + cmd = [sys.executable, "-u",] + [ + "-m", + "launch", + "--log_dir", + self.config['log_dir'], + "--worker_num", + self.config['worker_num'], + "--server_num", + self.config['server_num'], ] if ps_mode == 'heter-ps': os.environ['FLAGS_START_PORT'] = '12004' cmd += [ - '--heter_worker_num', self.config['heter_worker_num'], - '--heter_devices', self.config['heter_devices'] + '--heter_worker_num', + self.config['heter_worker_num'], + '--heter_devices', + self.config['heter_devices'], ] cmd += [ - "../ps/ps_dnn_trainer.py", "-m", self.config['ps_mode_config'], - "--run_minimize", self.config['run_minimize'], - "--run_single_pass", self.config['run_single_pass'], - "--run_the_one_ps", self.config['run_the_one_ps'], - "--debug_new_pass", self.config['debug_new_pass'], - "--debug_new_minimize", self.config['debug_new_minimize'], - "--applied_pass_name", self.config['applied_pass_name'], - "--debug_the_one_ps", self.config['debug_the_one_ps'] + "../ps/ps_dnn_trainer.py", + "-m", + self.config['ps_mode_config'], + "--run_minimize", + self.config['run_minimize'], + "--run_single_pass", + self.config['run_single_pass'], + "--run_the_one_ps", + self.config['run_the_one_ps'], + "--debug_new_pass", + self.config['debug_new_pass'], + "--debug_new_minimize", + self.config['debug_new_minimize'], + "--applied_pass_name", + self.config['applied_pass_name'], + "--debug_the_one_ps", + self.config['debug_the_one_ps'], ] elif ps_mode == "gpu-ps": os.environ['FLAGS_LAUNCH_BARRIER'] = '0' @@ -87,15 +101,25 @@ def ps_launch(self, ps_mode="cpu-ps"): os.environ['PADDLE_TRAINER_ID'] = '0' cmd = [ - sys.executable, "-u", "../ps/ps_dnn_trainer.py", "-m", - self.config['ps_mode_config'], "--run_minimize", - self.config['run_minimize'], "--run_single_pass", - self.config['run_single_pass'], "--run_the_one_ps", - self.config['run_the_one_ps'], "--debug_new_pass", - self.config['debug_new_pass'], "--debug_new_minimize", - self.config['debug_new_minimize'], "--applied_pass_name", - self.config['applied_pass_name'], "--debug_the_one_ps", - self.config['debug_the_one_ps'] + sys.executable, + "-u", + "../ps/ps_dnn_trainer.py", + "-m", + self.config['ps_mode_config'], + "--run_minimize", + self.config['run_minimize'], + "--run_single_pass", + self.config['run_single_pass'], + "--run_the_one_ps", + self.config['run_the_one_ps'], + "--debug_new_pass", + self.config['debug_new_pass'], + "--debug_new_minimize", + self.config['debug_new_minimize'], + "--applied_pass_name", + self.config['applied_pass_name'], + "--debug_the_one_ps", + self.config['debug_the_one_ps'], ] cmd = [shlex.quote(c) for c in cmd] diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_amp_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_amp_pass.py index 0d404d1732d42..e4173e11dbcf7 100755 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_amp_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_amp_pass.py @@ -22,7 +22,6 @@ class TestAMPPass(AutoPallelPassTestBase): - def init(self): if paddle.is_compiled_with_cuda(): paddle.set_flags({'FLAGS_cudnn_deterministic': 1}) @@ -51,14 +50,14 @@ def apply_passes(self): fleet.init(is_collective=True, strategy=dist_strategy) def test_bs_8(self): - self.check_main(gpus=[0, 1], - batch_size=8, - sequence_len=512, - vocab_size=1000) + self.check_main( + gpus=[0, 1], batch_size=8, sequence_len=512, vocab_size=1000 + ) def get_model(self, place, batch_size, sequence_len, vocab_size): - return self.get_gpt_model("mp", place, batch_size, sequence_len, - vocab_size) + return self.get_gpt_model( + "mp", place, batch_size, sequence_len, vocab_size + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_data_parallel_optimization_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_data_parallel_optimization_pass.py index 7da95ddf8d21d..530d4b2661e63 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_data_parallel_optimization_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_data_parallel_optimization_pass.py @@ -19,7 +19,9 @@ import unittest import paddle import paddle.distributed.fleet as fleet -from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, +) from paddle.distributed.passes import PassContext, new_pass from auto_parallel_pass_test_base import AutoPallelPassTestBase @@ -27,7 +29,6 @@ class TestDataParallelPassWithScale1(AutoPallelPassTestBase): - def init(self): if paddle.is_compiled_with_cuda(): paddle.set_flags({'FLAGS_cudnn_deterministic': 1}) @@ -55,24 +56,29 @@ def apply_no_passes(self): self._apply_pass = False def test_bs_8(self): - self.check_main(gpus=[0, 1], - batch_size=8, - sequence_len=512, - vocab_size=1000) + self.check_main( + gpus=[0, 1], batch_size=8, sequence_len=512, vocab_size=1000 + ) # test scaling with fillconstant def get_model(self, place, batch_size, sequence_len, vocab_size): - dist_main_prog, dist_startup_prog, data_holder, [ - loss - ], gen_data = self.get_gpt_model('dp', place, batch_size, sequence_len, - vocab_size) + ( + dist_main_prog, + dist_startup_prog, + data_holder, + [loss], + gen_data, + ) = self.get_gpt_model( + 'dp', place, batch_size, sequence_len, vocab_size + ) if self._apply_pass: config = {} config["dist_context"] = get_default_distributed_context() config["global_rank"] = paddle.distributed.get_rank() - dp_pass = new_pass("auto_parallel_data_parallel_optimization", - config) + dp_pass = new_pass( + "auto_parallel_data_parallel_optimization", config + ) dp_pass.apply([dist_main_prog], [dist_startup_prog], PassContext()) return dist_main_prog, dist_startup_prog, data_holder, [loss], gen_data @@ -83,20 +89,27 @@ class TestDataParallelPassWithScale2(TestDataParallelPassWithScale1): # test scaling with optimizer rescale_grad def get_model(self, place, batch_size, sequence_len, vocab_size): - dist_main_prog, dist_startup_prog, data_holder, [ - loss - ], gen_data = self.get_gpt_model('dp', - place, - batch_size, - sequence_len, - vocab_size, - optimizer='LarsMomentum') + ( + dist_main_prog, + dist_startup_prog, + data_holder, + [loss], + gen_data, + ) = self.get_gpt_model( + 'dp', + place, + batch_size, + sequence_len, + vocab_size, + optimizer='LarsMomentum', + ) if self._apply_pass: config = {} config["dist_context"] = get_default_distributed_context() config["global_rank"] = paddle.distributed.get_rank() - dp_pass = new_pass("auto_parallel_data_parallel_optimization", - config) + dp_pass = new_pass( + "auto_parallel_data_parallel_optimization", config + ) dp_pass.apply([dist_main_prog], [dist_startup_prog], PassContext()) return dist_main_prog, dist_startup_prog, data_holder, [loss], gen_data diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_fp16_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_fp16_pass.py index 2dfe965fc01b8..b3b2857e2920b 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_fp16_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_fp16_pass.py @@ -22,7 +22,6 @@ class TestPF16Pass(AutoPallelPassTestBase): - def init(self): if paddle.is_compiled_with_cuda(): paddle.set_flags({'FLAGS_cudnn_deterministic': 1}) @@ -42,29 +41,28 @@ def apply_passes(self): 'layer_norm', 'gelu', ], - "custom_black_list": - ['c_softmax_with_cross_entropy', 'elementwise_div', 'reduce_sum'], - "init_loss_scaling": - 32768, - "use_dynamic_loss_scaling": - True, - "use_pure_fp16": - True, - "use_fp16_guard": - False + "custom_black_list": [ + 'c_softmax_with_cross_entropy', + 'elementwise_div', + 'reduce_sum', + ], + "init_loss_scaling": 32768, + "use_dynamic_loss_scaling": True, + "use_pure_fp16": True, + "use_fp16_guard": False, } dist_strategy.semi_auto = True fleet.init(is_collective=True, strategy=dist_strategy) def test_bs_8(self): - self.check_main(gpus=[0, 1], - batch_size=8, - sequence_len=512, - vocab_size=1000) + self.check_main( + gpus=[0, 1], batch_size=8, sequence_len=512, vocab_size=1000 + ) def get_model(self, place, batch_size, sequence_len, vocab_size): - return self.get_gpt_model("mp", place, batch_size, sequence_len, - vocab_size) + return self.get_gpt_model( + "mp", place, batch_size, sequence_len, vocab_size + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_gradient_merge_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_gradient_merge_pass.py index 3b6eb35937eb6..ddb4c3cea056f 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_gradient_merge_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_gradient_merge_pass.py @@ -33,11 +33,9 @@ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=128, - intermediate_size=4 * 128, - initializer_range=0.02): + def __init__( + self, hidden_size=128, intermediate_size=4 * 128, initializer_range=0.02 + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size @@ -47,30 +45,24 @@ def __init__(self, weight_attr0 = paddle.ParamAttr(initializer=NumpyArrayInitializer(arr0)) weight_attr1 = paddle.ParamAttr(initializer=NumpyArrayInitializer(arr1)) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr0, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr1, - bias_attr=bias_attr) - self.linear2 = nn.Linear(d_model, - dim_feedforward, - weight_attr0, - bias_attr=bias_attr) - self.linear3 = nn.Linear(dim_feedforward, - d_model, - weight_attr1, - bias_attr=bias_attr) - self.linear4 = nn.Linear(d_model, - dim_feedforward, - weight_attr0, - bias_attr=bias_attr) - self.linear5 = nn.Linear(dim_feedforward, - d_model, - weight_attr1, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr0, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr1, bias_attr=bias_attr + ) + self.linear2 = nn.Linear( + d_model, dim_feedforward, weight_attr0, bias_attr=bias_attr + ) + self.linear3 = nn.Linear( + dim_feedforward, d_model, weight_attr1, bias_attr=bias_attr + ) + self.linear4 = nn.Linear( + d_model, dim_feedforward, weight_attr0, bias_attr=bias_attr + ) + self.linear5 = nn.Linear( + dim_feedforward, d_model, weight_attr1, bias_attr=bias_attr + ) self.norm0 = nn.LayerNorm(d_model, epsilon=1e-5) self.norm1 = nn.LayerNorm(d_model, epsilon=1e-5) self.norm2 = nn.LayerNorm(d_model, epsilon=1e-5) @@ -94,11 +86,14 @@ def forward(self, input): def mlp_forward(input, label, hidden_size): - auto.shard_tensor(input, auto.ProcessMesh([0], dim_names=["x"]), - [None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + auto.shard_tensor( + input, auto.ProcessMesh([0], dim_names=["x"]), [None, None] + ) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) loss = paddle.mean(error_cost) @@ -106,7 +101,6 @@ def mlp_forward(input, label, hidden_size): class TestGradientMergePass(AutoPallelPassTestBase): - def init(self): paddle.seed(2022) random.seed(2022) @@ -120,18 +114,22 @@ def apply_passes(self): fleet.init(is_collective=True, strategy=dist_strategy) def test_result(self): - no_pass_rets = self._distributed_launch(model=None, - apply_pass=False, - gpus=[0], - batch_size=32, - hidden_size=128, - max_step=2) - pass_rets = self._distributed_launch(model=None, - apply_pass=True, - gpus=[0], - batch_size=8, - hidden_size=128, - max_step=8) + no_pass_rets = self._distributed_launch( + model=None, + apply_pass=False, + gpus=[0], + batch_size=32, + hidden_size=128, + max_step=2, + ) + pass_rets = self._distributed_launch( + model=None, + apply_pass=True, + gpus=[0], + batch_size=8, + hidden_size=128, + max_step=8, + ) # avg loss for gradient_merge pass avg_loss = 0 pass_avg_ret_list = [] @@ -146,49 +144,63 @@ def test_result(self): for no_pass_ret, pass_ret in zip(no_pass_rets[0], pass_avg_ret_list): print(f"no_pass_ret={no_pass_ret}, pass_ret={pass_ret}") self.assertTrue( - np.isclose(no_pass_ret, - pass_ret, - rtol=self.rtol, - atol=self.atol, - equal_nan=self.equal_nan)) + np.isclose( + no_pass_ret, + pass_ret, + rtol=self.rtol, + atol=self.atol, + equal_nan=self.equal_nan, + ) + ) def get_model(self, place, batch_size, hidden_size, max_step): - def gen_data(): for i in range(max_step): - x_data = input_data[i * batch_size:(i + 1) * batch_size, :] - y_data = label_data[i * batch_size:(i + 1) * batch_size, :] + x_data = input_data[i * batch_size : (i + 1) * batch_size, :] + y_data = label_data[i * batch_size : (i + 1) * batch_size, :] yield x_data, y_data train_program = static.Program() startup_program = static.Program() - with static.program_guard(train_program, startup_program), \ - utils.unique_name.guard(): - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + with static.program_guard( + train_program, startup_program + ), utils.unique_name.guard(): + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) input.stop_gradient = False data_holder = [input, label] data_loader = paddle.fluid.io.DataLoader.from_generator( - feed_list=data_holder, capacity=70, iterable=False) - data_loader.set_batch_generator(gen_data, - paddle.static.cuda_places()) + feed_list=data_holder, capacity=70, iterable=False + ) + data_loader.set_batch_generator( + gen_data, paddle.static.cuda_places() + ) loss = mlp_forward(input, label, hidden_size) optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.01) optimizer = fleet.distributed_optimizer(optimizer) - _, self._params_grads, dist_startup_prog, dist_main_prog = optimizer.minimize( - loss, startup_program) + ( + _, + self._params_grads, + dist_startup_prog, + dist_main_prog, + ) = optimizer.minimize(loss, startup_program) input_data = np.random.random(size=(128, hidden_size)).astype('float32') label_data = np.random.random(size=(128, 1)).astype('float32') - return dist_main_prog, dist_startup_prog, [input, - label], [loss], data_loader + return ( + dist_main_prog, + dist_startup_prog, + [input, label], + [loss], + data_loader, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_recompute_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_recompute_pass.py index fa5fd193aba0e..ddea56b2a1464 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_recompute_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_recompute_pass.py @@ -22,7 +22,6 @@ class TestRecomputePass(AutoPallelPassTestBase): - def init(self): if paddle.is_compiled_with_cuda(): paddle.set_flags({'FLAGS_cudnn_deterministic': 1}) @@ -42,21 +41,21 @@ def apply_passes(self): fleet.init(is_collective=True, strategy=dist_strategy) def test_bs_8(self): - self.check_main(gpus=[0, 1], - batch_size=8, - sequence_len=512, - vocab_size=1000) + self.check_main( + gpus=[0, 1], batch_size=8, sequence_len=512, vocab_size=1000 + ) def get_model(self, place, batch_size, sequence_len, vocab_size): - return self.get_gpt_model("mp", place, batch_size, sequence_len, - vocab_size) + return self.get_gpt_model( + "mp", place, batch_size, sequence_len, vocab_size + ) class TestRecomputePassDP(TestRecomputePass): - def get_model(self, place, batch_size, sequence_len, vocab_size): - return self.get_gpt_model("dp", place, batch_size, sequence_len, - vocab_size) + return self.get_gpt_model( + "dp", place, batch_size, sequence_len, vocab_size + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_sharding_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_sharding_pass.py index 21a9a81aa0882..83e4667a1a9c2 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_sharding_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_auto_parallel_sharding_pass.py @@ -25,7 +25,6 @@ class TestShardingPass(AutoPallelPassTestBase): - def init(self): if paddle.is_compiled_with_cuda(): paddle.set_flags({'FLAGS_cudnn_deterministic': 1}) @@ -55,14 +54,14 @@ def apply_no_passes(self): fleet.init(is_collective=True, strategy=dist_strategy) def test_bs_8(self): - self.check_main(gpus=[0, 1], - batch_size=8, - sequence_len=512, - vocab_size=1000) + self.check_main( + gpus=[0, 1], batch_size=8, sequence_len=512, vocab_size=1000 + ) def get_model(self, place, batch_size, sequence_len, vocab_size): - return self.get_gpt_model('dp', place, batch_size, sequence_len, - vocab_size) + return self.get_gpt_model( + 'dp', place, batch_size, sequence_len, vocab_size + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_adam_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_adam_pass.py index c1cdd0b7336e3..b570786f20c3a 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_adam_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_adam_pass.py @@ -25,7 +25,6 @@ class DemoNet(nn.Layer): - def __init__(self): super(DemoNet, self).__init__() @@ -42,15 +41,14 @@ def forward(self, x): class TestFuseAdamPass(DistPassTestBase): - def init(self): self.atol = 1e-4 self.rtol = 1e-4 def get_model(self, place, batch_size=32, image_shape=[224, 224, 3]): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) model = DemoNet() pred_out = model(image) @@ -86,10 +84,14 @@ def apply_passes(self, main_prog, startup_prog): for op in main_prog.global_block().ops: op_type.append(op.type) if op.type == "adam": - self.assertTrue("@FUSEDVAR@_adam_Param_batch_norm2d_0.b_0" in - op.input("Param")) - self.assertTrue("@FUSEDVAR@_adam_Grad_batch_norm2d_0.b_0@GRAD" - in op.input("Grad")) + self.assertTrue( + "@FUSEDVAR@_adam_Param_batch_norm2d_0.b_0" + in op.input("Param") + ) + self.assertTrue( + "@FUSEDVAR@_adam_Grad_batch_norm2d_0.b_0@GRAD" + in op.input("Grad") + ) self.assertTrue("coalesce_tensor" in op_type) def test_fuse_adam(self): diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_all_reduce_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_all_reduce_pass.py index 06cd2ac6da49e..46754a1193c10 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_all_reduce_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_all_reduce_pass.py @@ -19,16 +19,17 @@ class TestFuseAllReducePass(DistPassTestBase): - def init(self): self.atol = 0.0 self.rtol = 0.0 def apply_passes(self, main_prog, startup_prog): - pass_manager = PassManager([ - new_pass("fuse_elewise_add_act"), - new_pass("fuse_all_reduce", {"max_memory_size": 1024 * 1024}) - ]) + pass_manager = PassManager( + [ + new_pass("fuse_elewise_add_act"), + new_pass("fuse_all_reduce", {"max_memory_size": 1024 * 1024}), + ] + ) pass_manager.apply([main_prog], [startup_prog]) print(pass_manager.names) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_act_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_act_pass.py index 12aeae57b09aa..cf00549e52f73 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_act_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_act_pass.py @@ -25,7 +25,6 @@ class BatchNormActNet(nn.Layer): - def __init__(self): super(BatchNormActNet, self).__init__() @@ -42,15 +41,14 @@ def forward(self, x): class TestFuseBatchNormActPass(DistPassTestBase): - def init(self): self.atol = 1e-4 self.rtol = 1e-4 def get_model(self, place, batch_size=32, image_shape=[224, 224, 3]): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) model = BatchNormActNet() pred_out = model(image) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_add_act_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_add_act_pass.py index 72dfd60d4a0bd..715dad4dc5bea 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_add_act_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_bn_add_act_pass.py @@ -25,7 +25,6 @@ class BatchNormAddActNet(nn.Layer): - def __init__(self): super(BatchNormAddActNet, self).__init__() @@ -46,15 +45,14 @@ def forward(self, x): class TestFuseBatchNormAddActPass(DistPassTestBase): - def init(self): self.atol = 1e-4 self.rtol = 1e-4 def get_model(self, place, batch_size=32, image_shape=[224, 224, 3]): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) model = BatchNormAddActNet() pred_out = model(image) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_momentum_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_momentum_pass.py index c725a9b91569c..4642be5839e89 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_momentum_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_momentum_pass.py @@ -25,7 +25,6 @@ class DemoNet(nn.Layer): - def __init__(self): super(DemoNet, self).__init__() @@ -42,15 +41,14 @@ def forward(self, x): class TestFuseAdamPass(DistPassTestBase): - def init(self): self.atol = 1e-4 self.rtol = 1e-4 def get_model(self, place, batch_size=32, image_shape=[224, 224, 3]): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) model = DemoNet() pred_out = model(image) @@ -86,11 +84,14 @@ def apply_passes(self, main_prog, startup_prog): for op in main_prog.global_block().ops: op_type.append(op.type) if op.type == "momentum": - self.assertTrue("@FUSEDVAR@_momentum_Param_batch_norm2d_0.b_0" - in op.input("Param")) self.assertTrue( - "@FUSEDVAR@_momentum_Grad_batch_norm2d_0.b_0@GRAD" in - op.input("Grad")) + "@FUSEDVAR@_momentum_Param_batch_norm2d_0.b_0" + in op.input("Param") + ) + self.assertTrue( + "@FUSEDVAR@_momentum_Grad_batch_norm2d_0.b_0@GRAD" + in op.input("Grad") + ) self.assertTrue("coalesce_tensor" in op_type) def test_fuse_adam(self): diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_relu_depthwise_conv_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_relu_depthwise_conv_pass.py index 61897b37ea7c5..c2c1404a967f1 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_relu_depthwise_conv_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_relu_depthwise_conv_pass.py @@ -25,7 +25,6 @@ class ReluDepthwiseConvNet(nn.Layer): - def __init__(self): super(ReluDepthwiseConvNet, self).__init__() @@ -42,15 +41,14 @@ def forward(self, x): class TestFuseReluDepthwiseConvPass(DistPassTestBase): - def init(self): self.atol = 1e-4 self.rtol = 1e-4 def get_model(self, place, batch_size=32, image_shape=[3, 224, 224]): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) model = ReluDepthwiseConvNet() pred_out = model(image) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_sgd_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_sgd_pass.py index ab752c60ed342..68ca472f6d436 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_sgd_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_fuse_sgd_pass.py @@ -25,7 +25,6 @@ class DemoNet(nn.Layer): - def __init__(self): super(DemoNet, self).__init__() @@ -42,15 +41,14 @@ def forward(self, x): class TestFuseAdamPass(DistPassTestBase): - def init(self): self.atol = 1e-4 self.rtol = 1e-4 def get_model(self, place, batch_size=32, image_shape=[224, 224, 3]): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) model = DemoNet() pred_out = model(image) @@ -86,10 +84,14 @@ def apply_passes(self, main_prog, startup_prog): for op in main_prog.global_block().ops: op_type.append(op.type) if op.type == "sgd": - self.assertTrue("@FUSEDVAR@_sgd_Param_batch_norm2d_0.b_0" in - op.input("Param")) - self.assertTrue("@FUSEDVAR@_sgd_Grad_batch_norm2d_0.b_0@GRAD" in - op.input("Grad")) + self.assertTrue( + "@FUSEDVAR@_sgd_Param_batch_norm2d_0.b_0" + in op.input("Param") + ) + self.assertTrue( + "@FUSEDVAR@_sgd_Grad_batch_norm2d_0.b_0@GRAD" + in op.input("Grad") + ) self.assertTrue("coalesce_tensor" in op_type) def test_fuse_adam(self): diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_inplace_addto_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_inplace_addto_pass.py index 0431c53c11c27..e6f2adfee538d 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_inplace_addto_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_dist_inplace_addto_pass.py @@ -25,7 +25,6 @@ class DemoNet(nn.Layer): - def __init__(self): super(DemoNet, self).__init__() @@ -43,16 +42,15 @@ def forward(self, x): class TestInplaceAddtoPass(DistPassTestBase): - def init(self): self.atol = 0.0 self.rtol = 0.0 paddle.fluid.set_flags({"FLAGS_max_inplace_grad_add": 8}) def get_model(self, place, batch_size=32, image_shape=[224, 224, 3]): - image = paddle.static.data(shape=[batch_size] + image_shape, - dtype='float32', - name='image') + image = paddle.static.data( + shape=[batch_size] + image_shape, dtype='float32', name='image' + ) model = DemoNet() pred_out = model(image) @@ -83,7 +81,8 @@ def reader(): def apply_passes(self, main_prog, startup_prog): pass_manager = PassManager( - [new_pass("inplace_addto_op", {"use_cuda": True})]) + [new_pass("inplace_addto_op", {"use_cuda": True})] + ) pass_manager.apply([main_prog], [startup_prog]) print(pass_manager.names) diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_server_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_server_pass.py index e4844b664c2ed..b0725c2bf305e 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_server_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_server_pass.py @@ -17,7 +17,6 @@ class TestPsServerPass(PsPassTestBase): - def init(self): pass diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_trainer_pass.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_trainer_pass.py index 9b7f7820c8f2c..d71a5b07b3079 100755 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_trainer_pass.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_ps_trainer_pass.py @@ -19,7 +19,6 @@ class TestPsTrainerPass(PsPassTestBase): - def setUp(self): pass @@ -142,7 +141,7 @@ def test_ps_optimizer_minimize_gpu(self): self.config['debug_new_minimize'] = '0' self.config['log_dir'] = ps_log_root_dir + "gpubox_log_old_minimize" remove_path_if_exists(self.config['log_dir']) - #self.ps_launch("gpu-ps") + # self.ps_launch("gpu-ps") self.config['debug_new_minimize'] = '1' self.config['log_dir'] = ps_log_root_dir + "gpubox_log_new_minimize" @@ -163,19 +162,25 @@ def test_append_send_ops_pass(self): self.config['applied_pass_name'] = "append_send_ops_pass" self.config['debug_new_pass'] = '0' - self.config['log_dir'] = ps_log_root_dir + "log_old_" + self.config[ - 'applied_pass_name'] + self.config['log_dir'] = ( + ps_log_root_dir + "log_old_" + self.config['applied_pass_name'] + ) remove_path_if_exists(self.config['log_dir']) self.ps_launch("cpu-ps") self.config['debug_new_pass'] = '1' - self.config['log_dir'] = ps_log_root_dir + "log_new_" + self.config[ - 'applied_pass_name'] + self.config['log_dir'] = ( + ps_log_root_dir + "log_new_" + self.config['applied_pass_name'] + ) remove_path_if_exists(self.config['log_dir']) self.ps_launch("cpu-ps") - file1 = './ps_log/async_append_send_ops_pass_debug:_0_worker_main.prototxt' - file2 = './ps_log/async_append_send_ops_pass_debug:_1_worker_main.prototxt' + file1 = ( + './ps_log/async_append_send_ops_pass_debug:_0_worker_main.prototxt' + ) + file2 = ( + './ps_log/async_append_send_ops_pass_debug:_1_worker_main.prototxt' + ) if self.check(file1, file2): logger.info('test_append_send_ops_pass passed!') else: diff --git a/python/paddle/fluid/tests/unittests/distributed_passes/test_white_lists.py b/python/paddle/fluid/tests/unittests/distributed_passes/test_white_lists.py index 645fa38099df1..307b41d027ca4 100644 --- a/python/paddle/fluid/tests/unittests/distributed_passes/test_white_lists.py +++ b/python/paddle/fluid/tests/unittests/distributed_passes/test_white_lists.py @@ -13,12 +13,17 @@ # limitations under the License. import unittest -from paddle.distributed.passes.pass_base import register_pass, PassBase, new_pass -from paddle.distributed.passes.pass_base import _make_rule_from_white_lists_dict as make_white_lists_rule +from paddle.distributed.passes.pass_base import ( + register_pass, + PassBase, + new_pass, +) +from paddle.distributed.passes.pass_base import ( + _make_rule_from_white_lists_dict as make_white_lists_rule, +) class TestConcretePass(PassBase): - def __init__(self): super(TestConcretePass, self).__init__() @@ -34,41 +39,35 @@ def _apply_single_impl(self, main_program, startup_program, context): @register_pass("A") class A(TestConcretePass): - def __init__(self): super(A, self).__init__() @register_pass("B") class B(TestConcretePass): - def __init__(self): super(B, self).__init__() @register_pass("C") class C(TestConcretePass): - def __init__(self): super(C, self).__init__() @register_pass("D") class D(TestConcretePass): - def __init__(self): super(D, self).__init__() @register_pass("E") class E(TestConcretePass): - def __init__(self): super(E, self).__init__() class TestMakeWhiteListsRule(unittest.TestCase): - def test_main(self): before_white_lists = {"A": ["B", "C"]} after_white_lists = {"D": ["C"]} diff --git a/python/paddle/fluid/tests/unittests/distribution/config.py b/python/paddle/fluid/tests/unittests/distribution/config.py index aee76250e5d14..29a27890bad2c 100644 --- a/python/paddle/fluid/tests/unittests/distribution/config.py +++ b/python/paddle/fluid/tests/unittests/distribution/config.py @@ -26,6 +26,6 @@ 'float32': 1e-03, 'complex64': 1e-3, 'float64': 1e-5, - 'complex128': 1e-5 + 'complex128': 1e-5, } ATOL = {'float32': 0.0, 'complex64': 0, 'float64': 0.0, 'complex128': 0} diff --git a/python/paddle/fluid/tests/unittests/distribution/mock_data.py b/python/paddle/fluid/tests/unittests/distribution/mock_data.py index 60299505f218a..db08fe1c88688 100644 --- a/python/paddle/fluid/tests/unittests/distribution/mock_data.py +++ b/python/paddle/fluid/tests/unittests/distribution/mock_data.py @@ -17,8 +17,9 @@ class Exponential(paddle.distribution.ExponentialFamily): """mock exponential distribution, which support computing entropy and - kl use bregman divergence + kl use bregman divergence """ + _mean_carrier_measure = 0 def __init__(self, rate): @@ -34,15 +35,14 @@ def entropy(self): @property def _natural_parameters(self): - return (-self._rate, ) + return (-self._rate,) def _log_normalizer(self, x): return -paddle.log(-x) class DummyExpFamily(paddle.distribution.ExponentialFamily): - """dummy class extend from exponential family - """ + """dummy class extend from exponential family""" def __init__(self, *args): pass @@ -52,7 +52,7 @@ def entropy(self): @property def _natural_parameters(self): - return (1.0, ) + return (1.0,) def _log_normalizer(self, x): return -paddle.log(-x) diff --git a/python/paddle/fluid/tests/unittests/distribution/parameterize.py b/python/paddle/fluid/tests/unittests/distribution/parameterize.py index 72c9ac03325a5..9c3341e34ce72 100644 --- a/python/paddle/fluid/tests/unittests/distribution/parameterize.py +++ b/python/paddle/fluid/tests/unittests/distribution/parameterize.py @@ -25,16 +25,14 @@ def xrand(shape=(10, 10, 10), dtype=config.DEFAULT_DTYPE, min=1.0, max=10.0): - return ((np.random.rand(*shape).astype(dtype)) * (max - min) + min) + return (np.random.rand(*shape).astype(dtype)) * (max - min) + min def place(devices, key='place'): - def decorate(cls): module = sys.modules[cls.__module__].__dict__ raw_classes = { - k: v - for k, v in module.items() if k.startswith(cls.__name__) + k: v for k, v in module.items() if k.startswith(cls.__name__) } for raw_name, raw_cls in raw_classes.items(): @@ -42,7 +40,7 @@ def decorate(cls): test_cls = dict(raw_cls.__dict__) test_cls.update({key: d}) new_name = raw_name + '.' + d.__class__.__name__ - module[new_name] = type(new_name, (raw_cls, ), test_cls) + module[new_name] = type(new_name, (raw_cls,), test_cls) del module[raw_name] return cls @@ -61,7 +59,7 @@ def decorate(cls): name = cls.__name__ + str(k) name = name + '.' + v.get('suffix') if v.get('suffix') else name - test_cls_module[name] = type(name, (cls, ), test_cls) + test_cls_module[name] = type(name, (cls,), test_cls) for m in list(cls.__dict__): if m.startswith("test"): @@ -71,10 +69,9 @@ def decorate(cls): return decorate -def parameterize_func(input, - name_func=None, - doc_func=None, - skip_on_empty=False): +def parameterize_func( + input, name_func=None, doc_func=None, skip_on_empty=False +): doc_func = doc_func or default_doc_func name_func = name_func or default_name_func @@ -88,13 +85,15 @@ def wrapper(f, instance=None): raise ValueError( "Parameters iterable is empty (hint: use " "`parameterized.expand([], skip_on_empty=True)` to skip " - "this test when the input is empty)") + "this test when the input is empty)" + ) return wraps(f)(skip_on_empty_helper) digits = len(str(len(parameters) - 1)) for num, p in enumerate(parameters): - name = name_func(f, "{num:0>{digits}}".format(digits=digits, - num=num), p) + name = name_func( + f, "{num:0>{digits}}".format(digits=digits, num=num), p + ) # If the original function has patches applied by 'mock.patch', # re-construct all patches on the just former decoration layer # of param_as_standalone_func so as not to share @@ -113,9 +112,7 @@ def wrapper(f, instance=None): def reapply_patches_if_need(func): - def dummy_wrapper(orgfunc): - @wraps(orgfunc) def dummy_func(*args, **kwargs): return orgfunc(*args, **kwargs) @@ -138,7 +135,7 @@ def delete_patches_if_need(func): def default_name_func(func, num, p): base_name = func.__name__ - name_suffix = "_%s" % (num, ) + name_suffix = "_%s" % (num,) if len(p.args) > 0 and isinstance(p.args[0], str): name_suffix += "_" + to_safe_name(p.args[0]) @@ -167,7 +164,6 @@ def default_doc_func(func, num, p): def param_as_standalone_func(p, func, name): - @functools.wraps(func) def standalone_func(*a): return func(*(a + p.args), **p.kwargs) @@ -209,36 +205,35 @@ def skip_on_empty_helper(*a, **kw): class param(_param): - def __new__(cls, *args, **kwargs): return _param.__new__(cls, args, kwargs) @classmethod def explicit(cls, args=None, kwargs=None): - """ Creates a ``param`` by explicitly specifying ``args`` and - ``kwargs``:: - >>> param.explicit([1,2,3]) - param(*(1, 2, 3)) - >>> param.explicit(kwargs={"foo": 42}) - param(*(), **{"foo": "42"}) - """ + """Creates a ``param`` by explicitly specifying ``args`` and + ``kwargs``:: + >>> param.explicit([1,2,3]) + param(*(1, 2, 3)) + >>> param.explicit(kwargs={"foo": 42}) + param(*(), **{"foo": "42"}) + """ args = args or () kwargs = kwargs or {} return cls(*args, **kwargs) @classmethod def from_decorator(cls, args): - """ Returns an instance of ``param()`` for ``@parameterized`` argument - ``args``:: - >>> param.from_decorator((42, )) - param(args=(42, ), kwargs={}) - >>> param.from_decorator("foo") - param(args=("foo", ), kwargs={}) - """ + """Returns an instance of ``param()`` for ``@parameterized`` argument + ``args``:: + >>> param.from_decorator((42, )) + param(args=(42, ), kwargs={}) + >>> param.from_decorator("foo") + param(args=("foo", ), kwargs={}) + """ if isinstance(args, param): return args elif isinstance(args, str): - args = (args, ) + args = (args,) try: return cls(*args) except TypeError as e: @@ -246,7 +241,8 @@ def from_decorator(cls, args): raise raise TypeError( "Parameters must be tuples, but %r is not (hint: use '(%r, )')" - % (args, args), ) + % (args, args), + ) def __repr__(self): return "param(*%r, **%r)" % self diff --git a/python/paddle/fluid/tests/unittests/distribution/test_dirichlet_op.py b/python/paddle/fluid/tests/unittests/distribution/test_dirichlet_op.py index 90209d9529b23..9b1183a96ebd8 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_dirichlet_op.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_dirichlet_op.py @@ -30,7 +30,7 @@ class TestDirichletOp(OpTest): def setUp(self): self.op_type = "dirichlet" - self.alpha = np.array((1., 2.)) + self.alpha = np.array((1.0, 2.0)) self.sample_shape = (100000, 2) self.inputs = {'Alpha': np.broadcast_to(self.alpha, self.sample_shape)} @@ -47,5 +47,7 @@ def _hypothesis_testing(self, outs): scipy.stats.kstest( outs[0][:, 0], # scipy dirichlet have not cdf, use beta to replace it. - scipy.stats.beta(a=self.alpha[0], b=self.alpha[1]).cdf)[0], - 0.01) + scipy.stats.beta(a=self.alpha[0], b=self.alpha[1]).cdf, + )[0], + 0.01, + ) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution.py index 30024c10fee38..8d592683aba61 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution.py @@ -25,8 +25,7 @@ paddle.enable_static() -class DistributionNumpy(): - +class DistributionNumpy: def sample(self): raise NotImplementedError @@ -44,9 +43,8 @@ def probs(self, value): class DistributionTestName(unittest.TestCase): - def get_prefix(self, string): - return (string.split('.')[0]) + return string.split('.')[0] def test_normal_name(self): name = 'test_normal' @@ -136,15 +134,18 @@ def test_categorical_name(self): @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( (parameterize.TEST_CASE_NAME, 'batch_shape', 'event_shape'), - [('test-tuple', (10, 20), (10, 20)), - ('test-list', [100, 100], [100, 200, 300]), - ('test-null-eventshape', (100, 100), ())]) + [ + ('test-tuple', (10, 20), (10, 20)), + ('test-list', [100, 100], [100, 200, 300]), + ('test-null-eventshape', (100, 100), ()), + ], +) class TestDistributionShape(unittest.TestCase): - def setUp(self): paddle.disable_static() self.dist = paddle.distribution.Distribution( - batch_shape=self.batch_shape, event_shape=self.event_shape) + batch_shape=self.batch_shape, event_shape=self.event_shape + ) def tearDown(self): paddle.enable_static() @@ -162,15 +163,15 @@ def test_prob(self): self.dist.prob(paddle.to_tensor(parameterize.xrand())) def test_extend_shape(self): - shapes = [(34, 20), (56, ), ()] + shapes = [(34, 20), (56,), ()] for shape in shapes: self.assertTrue( self.dist._extend_shape(shape), - shape + self.dist.batch_shape + self.dist.event_shape) + shape + self.dist.batch_shape + self.dist.event_shape, + ) class TestDistributionException(unittest.TestCase): - def setUp(self): self._d = paddle.distribution.Distribution() diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta.py index 18315440a21dc..d6febc7ac4d46 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta.py @@ -25,11 +25,15 @@ @place(DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'alpha', 'beta'), - [('test-scale', 1.0, 2.0), ('test-tensor', xrand(), xrand()), - ('test-broadcast', xrand((2, 1)), xrand((2, 5)))]) +@parameterize_cls( + (TEST_CASE_NAME, 'alpha', 'beta'), + [ + ('test-scale', 1.0, 2.0), + ('test-tensor', xrand(), xrand()), + ('test-broadcast', xrand((2, 1)), xrand((2, 5))), + ], +) class TestBeta(unittest.TestCase): - def setUp(self): # scale no need convert to tensor for scale input unittest alpha, beta = self.alpha, self.beta @@ -46,7 +50,8 @@ def test_mean(self): self._paddle_beta.mean, scipy.stats.beta.mean(self.alpha, self.beta), rtol=RTOL.get(str(self._paddle_beta.alpha.numpy().dtype)), - atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype))) + atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype)), + ) def test_variance(self): with paddle.fluid.dygraph.guard(self.place): @@ -54,7 +59,8 @@ def test_variance(self): self._paddle_beta.variance, scipy.stats.beta.var(self.alpha, self.beta), rtol=RTOL.get(str(self._paddle_beta.alpha.numpy().dtype)), - atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype))) + atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype)), + ) def test_prob(self): value = [np.random.rand(*self._paddle_beta.alpha.shape)] @@ -65,7 +71,8 @@ def test_prob(self): self._paddle_beta.prob(paddle.to_tensor(v)), scipy.stats.beta.pdf(v, self.alpha, self.beta), rtol=RTOL.get(str(self._paddle_beta.alpha.numpy().dtype)), - atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype))) + atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype)), + ) def test_log_prob(self): value = [np.random.rand(*self._paddle_beta.alpha.shape)] @@ -76,7 +83,8 @@ def test_log_prob(self): self._paddle_beta.log_prob(paddle.to_tensor(v)), scipy.stats.beta.logpdf(v, self.alpha, self.beta), rtol=RTOL.get(str(self._paddle_beta.alpha.numpy().dtype)), - atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype))) + atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype)), + ) def test_entropy(self): with paddle.fluid.dygraph.guard(self.place): @@ -84,23 +92,26 @@ def test_entropy(self): self._paddle_beta.entropy(), scipy.stats.beta.entropy(self.alpha, self.beta), rtol=RTOL.get(str(self._paddle_beta.alpha.numpy().dtype)), - atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype))) + atol=ATOL.get(str(self._paddle_beta.alpha.numpy().dtype)), + ) def test_sample_shape(self): cases = [ { 'input': [], - 'expect': [] + paddle.squeeze(self._paddle_beta.alpha).shape + 'expect': [] + paddle.squeeze(self._paddle_beta.alpha).shape, }, { 'input': [2, 3], - 'expect': [2, 3] + paddle.squeeze(self._paddle_beta.alpha).shape + 'expect': [2, 3] + + paddle.squeeze(self._paddle_beta.alpha).shape, }, ] for case in cases: self.assertTrue( - self._paddle_beta.sample(case.get('input')).shape == case.get( - 'expect')) + self._paddle_beta.sample(case.get('input')).shape + == case.get('expect') + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta_static.py index 8976293e36c09..d6976f8b685ba 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_beta_static.py @@ -27,100 +27,119 @@ @param.place(config.DEVICES) -@param.parameterize_cls((param.TEST_CASE_NAME, 'alpha', 'beta'), - [('test-tensor', xrand((10, 10)), xrand((10, 10))), - ('test-broadcast', xrand((2, 1)), xrand((2, 5))), - ('test-larger-data', xrand((10, 20)), xrand( - (10, 20)))]) +@param.parameterize_cls( + (param.TEST_CASE_NAME, 'alpha', 'beta'), + [ + ('test-tensor', xrand((10, 10)), xrand((10, 10))), + ('test-broadcast', xrand((2, 1)), xrand((2, 5))), + ('test-larger-data', xrand((10, 20)), xrand((10, 20))), + ], +) class TestBeta(unittest.TestCase): - def setUp(self): self.program = paddle.static.Program() self.executor = paddle.static.Executor(self.place) with paddle.static.program_guard(self.program): # scale no need convert to tensor for scale input unittest - alpha = paddle.static.data('alpha', self.alpha.shape, - self.alpha.dtype) + alpha = paddle.static.data( + 'alpha', self.alpha.shape, self.alpha.dtype + ) beta = paddle.static.data('beta', self.beta.shape, self.beta.dtype) self._paddle_beta = paddle.distribution.Beta(alpha, beta) self.feeds = {'alpha': self.alpha, 'beta': self.beta} def test_mean(self): with paddle.static.program_guard(self.program): - [mean] = self.executor.run(self.program, - feed=self.feeds, - fetch_list=[self._paddle_beta.mean]) - np.testing.assert_allclose(mean, - scipy.stats.beta.mean( - self.alpha, self.beta), - rtol=RTOL.get(str(self.alpha.dtype)), - atol=ATOL.get(str(self.alpha.dtype))) + [mean] = self.executor.run( + self.program, + feed=self.feeds, + fetch_list=[self._paddle_beta.mean], + ) + np.testing.assert_allclose( + mean, + scipy.stats.beta.mean(self.alpha, self.beta), + rtol=RTOL.get(str(self.alpha.dtype)), + atol=ATOL.get(str(self.alpha.dtype)), + ) def test_variance(self): with paddle.static.program_guard(self.program): - [variance - ] = self.executor.run(self.program, - feed=self.feeds, - fetch_list=[self._paddle_beta.variance]) - np.testing.assert_allclose(variance, - scipy.stats.beta.var( - self.alpha, self.beta), - rtol=RTOL.get(str(self.alpha.dtype)), - atol=ATOL.get(str(self.alpha.dtype))) + [variance] = self.executor.run( + self.program, + feed=self.feeds, + fetch_list=[self._paddle_beta.variance], + ) + np.testing.assert_allclose( + variance, + scipy.stats.beta.var(self.alpha, self.beta), + rtol=RTOL.get(str(self.alpha.dtype)), + atol=ATOL.get(str(self.alpha.dtype)), + ) def test_prob(self): with paddle.static.program_guard(self.program): - value = paddle.static.data('value', self._paddle_beta.alpha.shape, - self._paddle_beta.alpha.dtype) + value = paddle.static.data( + 'value', + self._paddle_beta.alpha.shape, + self._paddle_beta.alpha.dtype, + ) prob = self._paddle_beta.prob(value) random_number = np.random.rand(*self._paddle_beta.alpha.shape) feeds = dict(self.feeds, value=random_number) - [prob] = self.executor.run(self.program, - feed=feeds, - fetch_list=[prob]) - np.testing.assert_allclose(prob, - scipy.stats.beta.pdf( - random_number, self.alpha, - self.beta), - rtol=RTOL.get(str(self.alpha.dtype)), - atol=ATOL.get(str(self.alpha.dtype))) + [prob] = self.executor.run( + self.program, feed=feeds, fetch_list=[prob] + ) + np.testing.assert_allclose( + prob, + scipy.stats.beta.pdf(random_number, self.alpha, self.beta), + rtol=RTOL.get(str(self.alpha.dtype)), + atol=ATOL.get(str(self.alpha.dtype)), + ) def test_log_prob(self): with paddle.static.program_guard(self.program): - value = paddle.static.data('value', self._paddle_beta.alpha.shape, - self._paddle_beta.alpha.dtype) + value = paddle.static.data( + 'value', + self._paddle_beta.alpha.shape, + self._paddle_beta.alpha.dtype, + ) prob = self._paddle_beta.log_prob(value) random_number = np.random.rand(*self._paddle_beta.alpha.shape) feeds = dict(self.feeds, value=random_number) - [prob] = self.executor.run(self.program, - feed=feeds, - fetch_list=[prob]) - np.testing.assert_allclose(prob, - scipy.stats.beta.logpdf( - random_number, self.alpha, - self.beta), - rtol=RTOL.get(str(self.alpha.dtype)), - atol=ATOL.get(str(self.alpha.dtype))) + [prob] = self.executor.run( + self.program, feed=feeds, fetch_list=[prob] + ) + np.testing.assert_allclose( + prob, + scipy.stats.beta.logpdf(random_number, self.alpha, self.beta), + rtol=RTOL.get(str(self.alpha.dtype)), + atol=ATOL.get(str(self.alpha.dtype)), + ) def test_entropy(self): with paddle.static.program_guard(self.program): - [entropy - ] = self.executor.run(self.program, - feed=self.feeds, - fetch_list=[self._paddle_beta.entropy()]) - np.testing.assert_allclose(entropy, - scipy.stats.beta.entropy( - self.alpha, self.beta), - rtol=RTOL.get(str(self.alpha.dtype)), - atol=ATOL.get(str(self.alpha.dtype))) + [entropy] = self.executor.run( + self.program, + feed=self.feeds, + fetch_list=[self._paddle_beta.entropy()], + ) + np.testing.assert_allclose( + entropy, + scipy.stats.beta.entropy(self.alpha, self.beta), + rtol=RTOL.get(str(self.alpha.dtype)), + atol=ATOL.get(str(self.alpha.dtype)), + ) def test_sample(self): with paddle.static.program_guard(self.program): - [data] = self.executor.run(self.program, - feed=self.feeds, - fetch_list=self._paddle_beta.sample()) - self.assertTrue(data.shape, - np.broadcast_arrays(self.alpha, self.beta)[0].shape) + [data] = self.executor.run( + self.program, + feed=self.feeds, + fetch_list=self._paddle_beta.sample(), + ) + self.assertTrue( + data.shape, np.broadcast_arrays(self.alpha, self.beta)[0].shape + ) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_categorical.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_categorical.py index 4be500b6084db..0ccfb1e0769fd 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_categorical.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_categorical.py @@ -26,7 +26,6 @@ class CategoricalNumpy(DistributionNumpy): - def __init__(self, logits): self.logits = np.array(logits).astype('float32') @@ -35,25 +34,26 @@ def entropy(self): e_logits = np.exp(logits) z = np.sum(e_logits, axis=-1, keepdims=True) prob = e_logits / z - return -1. * np.sum(prob * (logits - np.log(z)), axis=-1) + return -1.0 * np.sum(prob * (logits - np.log(z)), axis=-1) def kl_divergence(self, other): logits = self.logits - np.max(self.logits, axis=-1, keepdims=True) other_logits = other.logits - np.max( - other.logits, axis=-1, keepdims=True) + other.logits, axis=-1, keepdims=True + ) e_logits = np.exp(logits) other_e_logits = np.exp(other_logits) z = np.sum(e_logits, axis=-1, keepdims=True) other_z = np.sum(other_e_logits, axis=-1, keepdims=True) prob = e_logits / z - return np.sum(prob * - (logits - np.log(z) - other_logits + np.log(other_z)), - axis=-1, - keepdims=True) + return np.sum( + prob * (logits - np.log(z) - other_logits + np.log(other_z)), + axis=-1, + keepdims=True, + ) class CategoricalTest(unittest.TestCase): - def setUp(self, use_gpu=False, batch_size=3, dims=5): self.use_gpu = use_gpu if not use_gpu: @@ -79,8 +79,9 @@ def init_numpy_data(self, batch_size, dims): # input logtis is 2-D Tensor # value used in probs and log_prob method is 1-D Tensor self.logits_np = np.random.rand(batch_size, dims).astype('float32') - self.other_logits_np = np.random.rand(batch_size, - dims).astype('float32') + self.other_logits_np = np.random.rand(batch_size, dims).astype( + 'float32' + ) self.value_np = np.array([2, 1, 3]).astype('int64') self.logits_shape = [batch_size, dims] @@ -103,15 +104,15 @@ def init_dynamic_data(self, batch_size, dims): def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.logits_static = fluid.data(name='logits', - shape=self.logits_shape, - dtype='float32') - self.other_logits_static = fluid.data(name='other_logits', - shape=self.logits_shape, - dtype='float32') - self.value_static = fluid.data(name='value', - shape=self.value_shape, - dtype='int64') + self.logits_static = fluid.data( + name='logits', shape=self.logits_shape, dtype='float32' + ) + self.other_logits_static = fluid.data( + name='other_logits', shape=self.logits_shape, dtype='float32' + ) + self.value_static = fluid.data( + name='value', shape=self.value_shape, dtype='int64' + ) def get_numpy_selected_probs(self, probability): np_probs = np.zeros(self.dist_shape + self.value_shape) @@ -124,36 +125,33 @@ def compare_with_numpy(self, fetch_list, tolerance=1e-6): sample, entropy, kl, probs, log_prob = fetch_list log_tolerance = 1e-4 - np.testing.assert_equal(sample.shape, - self.sample_shape + self.dist_shape) + np.testing.assert_equal( + sample.shape, self.sample_shape + self.dist_shape + ) np_categorical = CategoricalNumpy(self.logits_np) np_other_categorical = CategoricalNumpy(self.other_logits_np) np_entropy = np_categorical.entropy() np_kl = np_categorical.kl_divergence(np_other_categorical) - np.testing.assert_allclose(entropy, - np_entropy, - rtol=log_tolerance, - atol=log_tolerance) - np.testing.assert_allclose(kl, - np_kl, - rtol=log_tolerance, - atol=log_tolerance) + np.testing.assert_allclose( + entropy, np_entropy, rtol=log_tolerance, atol=log_tolerance + ) + np.testing.assert_allclose( + kl, np_kl, rtol=log_tolerance, atol=log_tolerance + ) sum_dist = np.sum(self.logits_np, axis=-1, keepdims=True) probability = self.logits_np / sum_dist np_probs = self.get_numpy_selected_probs(probability) np_log_prob = np.log(np_probs) - np.testing.assert_allclose(probs, - np_probs, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(log_prob, - np_log_prob, - rtol=tolerance, - atol=tolerance) + np.testing.assert_allclose( + probs, np_probs, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + log_prob, np_log_prob, rtol=tolerance, atol=tolerance + ) def test_categorical_distribution_dygraph(self, tolerance=1e-6): paddle.disable_static(self.place) @@ -186,25 +184,25 @@ def test_categorical_distribution_static(self, tolerance=1e-6): feed_vars = { 'logits': self.logits_np, 'other_logits': self.other_logits_np, - 'value': self.value_np + 'value': self.value_np, } self.executor.run(fluid.default_startup_program()) - fetch_list = self.executor.run(program=self.test_program, - feed=feed_vars, - fetch_list=fetch_list) + fetch_list = self.executor.run( + program=self.test_program, feed=feed_vars, fetch_list=fetch_list + ) self.compare_with_numpy(fetch_list) class CategoricalTest2(CategoricalTest): - def init_numpy_data(self, batch_size, dims): # input logtis is 2-D Tensor with dtype Float64 # value used in probs and log_prob method is 1-D Tensor self.logits_np = np.random.rand(batch_size, dims).astype('float64') - self.other_logits_np = np.random.rand(batch_size, - dims).astype('float64') + self.other_logits_np = np.random.rand(batch_size, dims).astype( + 'float64' + ) self.value_np = np.array([2, 1, 3]).astype('int64') self.logits_shape = [batch_size, dims] @@ -214,19 +212,18 @@ def init_numpy_data(self, batch_size, dims): def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.logits_static = fluid.data(name='logits', - shape=self.logits_shape, - dtype='float64') - self.other_logits_static = fluid.data(name='other_logits', - shape=self.logits_shape, - dtype='float64') - self.value_static = fluid.data(name='value', - shape=self.value_shape, - dtype='int64') + self.logits_static = fluid.data( + name='logits', shape=self.logits_shape, dtype='float64' + ) + self.other_logits_static = fluid.data( + name='other_logits', shape=self.logits_shape, dtype='float64' + ) + self.value_static = fluid.data( + name='value', shape=self.value_shape, dtype='int64' + ) class CategoricalTest3(CategoricalTest): - def init_dynamic_data(self, batch_size, dims): # input logtis is 2-D numpy.ndarray with dtype Float32 # value used in probs and log_prob method is 1-D Tensor @@ -238,19 +235,19 @@ def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): self.logits_static = self.logits_np self.other_logits_static = self.other_logits_np - self.value_static = fluid.data(name='value', - shape=self.value_shape, - dtype='int64') + self.value_static = fluid.data( + name='value', shape=self.value_shape, dtype='int64' + ) class CategoricalTest4(CategoricalTest): - def init_numpy_data(self, batch_size, dims): # input logtis is 2-D numpy.ndarray with dtype Float64 # value used in probs and log_prob method is 1-D Tensor self.logits_np = np.random.rand(batch_size, dims).astype('float64') - self.other_logits_np = np.random.rand(batch_size, - dims).astype('float64') + self.other_logits_np = np.random.rand(batch_size, dims).astype( + 'float64' + ) self.value_np = np.array([2, 1, 3]).astype('int64') self.logits_shape = [batch_size, dims] @@ -267,14 +264,13 @@ def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): self.logits_static = self.logits_np self.other_logits_static = self.other_logits_np - self.value_static = fluid.data(name='value', - shape=self.value_shape, - dtype='int64') + self.value_static = fluid.data( + name='value', shape=self.value_shape, dtype='int64' + ) # test shape of logits and value used in probs and log_prob method class CategoricalTest5(CategoricalTest): - def init_numpy_data(self, batch_size, dims): # input logtis is 1-D Tensor # value used in probs and log_prob method is 1-D Tensor @@ -295,7 +291,6 @@ def get_numpy_selected_probs(self, probability): class CategoricalTest6(CategoricalTest): - def init_numpy_data(self, batch_size, dims): # input logtis is 2-D Tensor # value used in probs and log_prob method has the same number of batches with input @@ -317,7 +312,6 @@ def get_numpy_selected_probs(self, probability): class CategoricalTest7(CategoricalTest): - def init_numpy_data(self, batch_size, dims): # input logtis is 3-D Tensor # value used in probs and log_prob method has the same number of distribuions with input @@ -340,7 +334,6 @@ def get_numpy_selected_probs(self, probability): class CategoricalTest8(CategoricalTest): - def init_dynamic_data(self, batch_size, dims): # input logtis is 2-D list # value used in probs and log_prob method is 1-D Tensor @@ -352,13 +345,12 @@ def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): self.logits_static = self.logits_np.tolist() self.other_logits_static = self.other_logits_np.tolist() - self.value_static = fluid.data(name='value', - shape=self.value_shape, - dtype='int64') + self.value_static = fluid.data( + name='value', shape=self.value_shape, dtype='int64' + ) class CategoricalTest9(CategoricalTest): - def init_dynamic_data(self, batch_size, dims): # input logtis is 2-D tuple # value used in probs and log_prob method is 1-D Tensor @@ -370,13 +362,12 @@ def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): self.logits_static = tuple(self.logits_np.tolist()) self.other_logits_static = tuple(self.other_logits_np.tolist()) - self.value_static = fluid.data(name='value', - shape=self.value_shape, - dtype='int64') + self.value_static = fluid.data( + name='value', shape=self.value_shape, dtype='int64' + ) class DistributionTestError(unittest.TestCase): - def test_distribution_error(self): distribution = Distribution() @@ -384,13 +375,15 @@ def test_distribution_error(self): self.assertRaises(NotImplementedError, distribution.entropy) normal = Normal(0.0, 1.0) - self.assertRaises(NotImplementedError, distribution.kl_divergence, - normal) + self.assertRaises( + NotImplementedError, distribution.kl_divergence, normal + ) value_npdata = np.array([0.8], dtype="float32") value_tensor = layers.create_tensor(dtype="float32") - self.assertRaises(NotImplementedError, distribution.log_prob, - value_tensor) + self.assertRaises( + NotImplementedError, distribution.log_prob, value_tensor + ) self.assertRaises(NotImplementedError, distribution.probs, value_tensor) def test_normal_error(self): @@ -456,8 +449,9 @@ def test_categorical_error(self): categorical_other = Uniform(1.0, 2.0) # type of other must be an instance of Categorical - self.assertRaises(TypeError, categorical.kl_divergence, - categorical_other) + self.assertRaises( + TypeError, categorical.kl_divergence, categorical_other + ) def test_shape_not_match_error(): # shape of value must match shape of logits diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_constraint.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_constraint.py index cbf0431d9f34a..3ef352d54bf37 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_constraint.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_constraint.py @@ -23,10 +23,10 @@ np.random.seed(2022) -@param.param_cls((param.TEST_CASE_NAME, 'value'), - [('NotImplement', np.random.rand(2, 3))]) +@param.param_cls( + (param.TEST_CASE_NAME, 'value'), [('NotImplement', np.random.rand(2, 3))] +) class TestConstraint(unittest.TestCase): - def setUp(self): self._constraint = constraint.Constraint() @@ -35,10 +35,10 @@ def test_costraint(self): self._constraint(self.value) -@param.param_cls((param.TEST_CASE_NAME, 'value', 'expect'), - [('real', 1., True)]) +@param.param_cls( + (param.TEST_CASE_NAME, 'value', 'expect'), [('real', 1.0, True)] +) class TestReal(unittest.TestCase): - def setUp(self): self._constraint = constraint.Real() @@ -46,10 +46,11 @@ def test_costraint(self): self.assertEqual(self._constraint(self.value), self.expect) -@param.param_cls((param.TEST_CASE_NAME, 'lower', 'upper', 'value', 'expect'), - [('in_range', 0, 1, 0.5, True), ('out_range', 0, 1, 2, False)]) +@param.param_cls( + (param.TEST_CASE_NAME, 'lower', 'upper', 'value', 'expect'), + [('in_range', 0, 1, 0.5, True), ('out_range', 0, 1, 2, False)], +) class TestRange(unittest.TestCase): - def setUp(self): self._constraint = constraint.Range(self.lower, self.upper) @@ -57,10 +58,11 @@ def test_costraint(self): self.assertEqual(self._constraint(self.value), self.expect) -@param.param_cls((param.TEST_CASE_NAME, 'value', 'expect'), - [('positive', 1, True), ('negative', -1, False)]) +@param.param_cls( + (param.TEST_CASE_NAME, 'value', 'expect'), + [('positive', 1, True), ('negative', -1, False)], +) class TestPositive(unittest.TestCase): - def setUp(self): self._constraint = constraint.Positive() @@ -68,11 +70,14 @@ def test_costraint(self): self.assertEqual(self._constraint(self.value), self.expect) -@param.param_cls((param.TEST_CASE_NAME, 'value', 'expect'), - [('simplex', paddle.to_tensor([0.5, 0.5]), True), - ('non_simplex', paddle.to_tensor([-0.5, 0.5]), False)]) +@param.param_cls( + (param.TEST_CASE_NAME, 'value', 'expect'), + [ + ('simplex', paddle.to_tensor([0.5, 0.5]), True), + ('non_simplex', paddle.to_tensor([-0.5, 0.5]), False), + ], +) class TestSimplex(unittest.TestCase): - def setUp(self): self._constraint = constraint.Simplex() diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet.py index cb5534edcd541..399fe66b0a067 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet.py @@ -28,14 +28,15 @@ @param.param_cls( (param.TEST_CASE_NAME, 'concentration'), [ - ('test-one-dim', param.xrand((89, ))), + ('test-one-dim', param.xrand((89,))), # ('test-multi-dim', config.xrand((10, 20, 30))) - ]) + ], +) class TestDirichlet(unittest.TestCase): - def setUp(self): self._paddle_diric = paddle.distribution.Dirichlet( - paddle.to_tensor(self.concentration)) + paddle.to_tensor(self.concentration) + ) def test_mean(self): with paddle.fluid.dygraph.guard(self.place): @@ -43,7 +44,8 @@ def test_mean(self): self._paddle_diric.mean, scipy.stats.dirichlet.mean(self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_variance(self): with paddle.fluid.dygraph.guard(self.place): @@ -51,7 +53,8 @@ def test_variance(self): self._paddle_diric.variance, scipy.stats.dirichlet.var(self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_prob(self): value = [np.random.rand(*self.concentration.shape)] @@ -63,7 +66,8 @@ def test_prob(self): self._paddle_diric.prob(paddle.to_tensor(v)), scipy.stats.dirichlet.pdf(v, self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_log_prob(self): value = [np.random.rand(*self.concentration.shape)] @@ -75,7 +79,8 @@ def test_log_prob(self): self._paddle_diric.log_prob(paddle.to_tensor(v)), scipy.stats.dirichlet.logpdf(v, self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_entropy(self): with paddle.fluid.dygraph.guard(self.place): @@ -83,28 +88,35 @@ def test_entropy(self): self._paddle_diric.entropy(), scipy.stats.dirichlet.entropy(self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_natural_parameters(self): self.assertTrue( - isinstance(self._paddle_diric._natural_parameters, tuple)) + isinstance(self._paddle_diric._natural_parameters, tuple) + ) def test_log_normalizer(self): self.assertTrue( np.all( self._paddle_diric._log_normalizer( - paddle.to_tensor(param.xrand((100, 100, - 100)))).numpy() < 0.0)) + paddle.to_tensor(param.xrand((100, 100, 100))) + ).numpy() + < 0.0 + ) + ) @param.place(DEVICES) - @param.param_cls((param.TEST_CASE_NAME, 'concentration'), - [('test-zero-dim', np.array(1.0))]) + @param.param_cls( + (param.TEST_CASE_NAME, 'concentration'), + [('test-zero-dim', np.array(1.0))], + ) class TestDirichletException(unittest.TestCase): - def TestInit(self): with self.assertRaises(ValueError): - paddle.distribution.Dirichlet(paddle.squeeze( - self.concentration)) + paddle.distribution.Dirichlet( + paddle.squeeze(self.concentration) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet_static.py index 8b0d847ad16e7..edf66a26e0219 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_dirichlet_static.py @@ -26,82 +26,97 @@ @place(DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'concentration'), - [('test-one-dim', np.random.rand(89) + 5.0)]) +@parameterize_cls( + (TEST_CASE_NAME, 'concentration'), + [('test-one-dim', np.random.rand(89) + 5.0)], +) class TestDirichlet(unittest.TestCase): - def setUp(self): self.program = paddle.static.Program() self.executor = paddle.static.Executor() with paddle.static.program_guard(self.program): - conc = paddle.static.data('conc', self.concentration.shape, - self.concentration.dtype) + conc = paddle.static.data( + 'conc', self.concentration.shape, self.concentration.dtype + ) self._paddle_diric = paddle.distribution.Dirichlet(conc) self.feeds = {'conc': self.concentration} def test_mean(self): with paddle.static.program_guard(self.program): - [out] = self.executor.run(self.program, - feed=self.feeds, - fetch_list=[self._paddle_diric.mean]) + [out] = self.executor.run( + self.program, + feed=self.feeds, + fetch_list=[self._paddle_diric.mean], + ) np.testing.assert_allclose( out, scipy.stats.dirichlet.mean(self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_variance(self): with paddle.static.program_guard(self.program): - [out] = self.executor.run(self.program, - feed=self.feeds, - fetch_list=[self._paddle_diric.variance]) + [out] = self.executor.run( + self.program, + feed=self.feeds, + fetch_list=[self._paddle_diric.variance], + ) np.testing.assert_allclose( out, scipy.stats.dirichlet.var(self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_prob(self): with paddle.static.program_guard(self.program): random_number = np.random.rand(*self.concentration.shape) random_number = random_number / random_number.sum() feeds = dict(self.feeds, value=random_number) - value = paddle.static.data('value', random_number.shape, - random_number.dtype) + value = paddle.static.data( + 'value', random_number.shape, random_number.dtype + ) out = self._paddle_diric.prob(value) - [out] = self.executor.run(self.program, - feed=feeds, - fetch_list=[out]) + [out] = self.executor.run( + self.program, feed=feeds, fetch_list=[out] + ) np.testing.assert_allclose( out, scipy.stats.dirichlet.pdf(random_number, self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_log_prob(self): with paddle.static.program_guard(self.program): random_number = np.random.rand(*self.concentration.shape) random_number = random_number / random_number.sum() feeds = dict(self.feeds, value=random_number) - value = paddle.static.data('value', random_number.shape, - random_number.dtype) + value = paddle.static.data( + 'value', random_number.shape, random_number.dtype + ) out = self._paddle_diric.log_prob(value) - [out] = self.executor.run(self.program, - feed=feeds, - fetch_list=[out]) + [out] = self.executor.run( + self.program, feed=feeds, fetch_list=[out] + ) np.testing.assert_allclose( out, scipy.stats.dirichlet.logpdf(random_number, self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) def test_entropy(self): with paddle.static.program_guard(self.program): - [out] = self.executor.run(self.program, - feed=self.feeds, - fetch_list=[self._paddle_diric.entropy()]) + [out] = self.executor.run( + self.program, + feed=self.feeds, + fetch_list=[self._paddle_diric.entropy()], + ) np.testing.assert_allclose( out, scipy.stats.dirichlet.entropy(self.concentration), rtol=RTOL.get(str(self.concentration.dtype)), - atol=ATOL.get(str(self.concentration.dtype))) + atol=ATOL.get(str(self.concentration.dtype)), + ) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily.py index ca54c978ca4ee..001696385e0bf 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily.py @@ -27,30 +27,46 @@ @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( (parameterize.TEST_CASE_NAME, 'dist'), - [('test-mock-exp', - mock.Exponential( - rate=paddle.rand([100, 200, 99], dtype=config.DEFAULT_DTYPE)))]) + [ + ( + 'test-mock-exp', + mock.Exponential( + rate=paddle.rand([100, 200, 99], dtype=config.DEFAULT_DTYPE) + ), + ) + ], +) class TestExponentialFamily(unittest.TestCase): - def test_entropy(self): np.testing.assert_allclose( self.dist.entropy(), paddle.distribution.ExponentialFamily.entropy(self.dist), rtol=config.RTOL.get(config.DEFAULT_DTYPE), - atol=config.ATOL.get(config.DEFAULT_DTYPE)) + atol=config.ATOL.get(config.DEFAULT_DTYPE), + ) @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( (config.TEST_CASE_NAME, 'dist'), - [('test-dummy', mock.DummyExpFamily(0.5, 0.5)), - ('test-dirichlet', - paddle.distribution.Dirichlet(paddle.to_tensor(parameterize.xrand()))), - ('test-beta', - paddle.distribution.Beta(paddle.to_tensor(parameterize.xrand()), - paddle.to_tensor(parameterize.xrand())))]) + [ + ('test-dummy', mock.DummyExpFamily(0.5, 0.5)), + ( + 'test-dirichlet', + paddle.distribution.Dirichlet( + paddle.to_tensor(parameterize.xrand()) + ), + ), + ( + 'test-beta', + paddle.distribution.Beta( + paddle.to_tensor(parameterize.xrand()), + paddle.to_tensor(parameterize.xrand()), + ), + ), + ], +) class TestExponentialFamilyException(unittest.TestCase): - def test_entropy_exception(self): with self.assertRaises(NotImplementedError): paddle.distribution.ExponentialFamily.entropy(self.dist) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily_static.py index 2d1493a4e52f1..7c2fe5b5a4df0 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_expfamily_static.py @@ -27,7 +27,6 @@ @parameterize.place(config.DEVICES) class TestExponentialFamily(unittest.TestCase): - def setUp(self): self.program = paddle.static.Program() self.executor = paddle.static.Executor() @@ -45,17 +44,21 @@ def test_entropy(self): fetch_list=[ self.mock_dist.entropy(), paddle.distribution.ExponentialFamily.entropy( - self.mock_dist) - ]) + self.mock_dist + ), + ], + ) np.testing.assert_allclose( out1, out2, rtol=config.RTOL.get(config.DEFAULT_DTYPE), - atol=config.ATOL.get(config.DEFAULT_DTYPE)) + atol=config.ATOL.get(config.DEFAULT_DTYPE), + ) def test_entropy_exception(self): with paddle.static.program_guard(self.program): with self.assertRaises(NotImplementedError): paddle.distribution.ExponentialFamily.entropy( - mock.DummyExpFamily(0.5, 0.5)) + mock.DummyExpFamily(0.5, 0.5) + ) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel.py index 3f3cfd8e8bb1a..3aa7c5e6436fc 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel.py @@ -24,47 +24,58 @@ @parameterize.place(config.DEVICES) -@parameterize.parameterize_cls((parameterize.TEST_CASE_NAME, 'loc', 'scale'), [ - ('one-dim', parameterize.xrand((4, )), parameterize.xrand((4, ))), - ('multi-dim', parameterize.xrand((5, 3)), parameterize.xrand((5, 3))), -]) +@parameterize.parameterize_cls( + (parameterize.TEST_CASE_NAME, 'loc', 'scale'), + [ + ('one-dim', parameterize.xrand((4,)), parameterize.xrand((4,))), + ('multi-dim', parameterize.xrand((5, 3)), parameterize.xrand((5, 3))), + ], +) class TestGumbel(unittest.TestCase): - def setUp(self): - self._dist = Gumbel(loc=paddle.to_tensor(self.loc), - scale=paddle.to_tensor(self.scale)) + self._dist = Gumbel( + loc=paddle.to_tensor(self.loc), scale=paddle.to_tensor(self.scale) + ) def test_mean(self): mean = self._dist.mean self.assertEqual(mean.numpy().dtype, self._np_mean().dtype) - np.testing.assert_allclose(mean, - self._np_mean(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + mean, + self._np_mean(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_variance(self): var = self._dist.variance self.assertEqual(var.numpy().dtype, self._np_variance().dtype) - np.testing.assert_allclose(var, - self._np_variance(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + var, + self._np_variance(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_stddev(self): stddev = self._dist.stddev self.assertEqual(stddev.numpy().dtype, self._np_stddev().dtype) - np.testing.assert_allclose(stddev, - self._np_stddev(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + stddev, + self._np_stddev(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_entropy(self): entropy = self._dist.entropy() self.assertEqual(entropy.numpy().dtype, self._np_entropy().dtype) - np.testing.assert_allclose(entropy, - self._np_entropy(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + entropy, + self._np_entropy(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_sample(self): @@ -73,16 +84,18 @@ def test_sample(self): sample_values = samples.numpy() self.assertEqual(sample_values.dtype, self.scale.dtype) - np.testing.assert_allclose(sample_values.mean(axis=0), - scipy.stats.gumbel_r.mean(self.loc, - scale=self.scale), - rtol=0.1, - atol=config.ATOL.get(str(self.loc.dtype))) - np.testing.assert_allclose(sample_values.var(axis=0), - scipy.stats.gumbel_r.var(self.loc, - scale=self.scale), - rtol=0.1, - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + sample_values.mean(axis=0), + scipy.stats.gumbel_r.mean(self.loc, scale=self.scale), + rtol=0.1, + atol=config.ATOL.get(str(self.loc.dtype)), + ) + np.testing.assert_allclose( + sample_values.var(axis=0), + scipy.stats.gumbel_r.var(self.loc, scale=self.scale), + rtol=0.1, + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_rsample(self): @@ -91,16 +104,18 @@ def test_rsample(self): sample_values = samples.numpy() self.assertEqual(sample_values.dtype, self.scale.dtype) - np.testing.assert_allclose(sample_values.mean(axis=0), - scipy.stats.gumbel_r.mean(self.loc, - scale=self.scale), - rtol=0.1, - atol=config.ATOL.get(str(self.loc.dtype))) - np.testing.assert_allclose(sample_values.var(axis=0), - scipy.stats.gumbel_r.var(self.loc, - scale=self.scale), - rtol=0.1, - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + sample_values.mean(axis=0), + scipy.stats.gumbel_r.mean(self.loc, scale=self.scale), + rtol=0.1, + atol=config.ATOL.get(str(self.loc.dtype)), + ) + np.testing.assert_allclose( + sample_values.var(axis=0), + scipy.stats.gumbel_r.var(self.loc, scale=self.scale), + rtol=0.1, + atol=config.ATOL.get(str(self.loc.dtype)), + ) def _np_mean(self): return self.loc + self.scale * np.euler_gamma @@ -110,7 +125,8 @@ def _np_stddev(self): def _np_variance(self): return np.divide( - np.multiply(np.power(self.scale, 2), np.power(np.pi, 2)), 6) + np.multiply(np.power(self.scale, 2), np.power(np.pi, 2)), 6 + ) def _np_entropy(self): return np.log(self.scale) + 1 + np.euler_gamma @@ -118,39 +134,52 @@ def _np_entropy(self): @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'loc', 'scale', 'value'), [ - ('value-float', np.array([0.1, 0.4]), np.array([1., 4. - ]), np.array([3., 7.])), + (parameterize.TEST_CASE_NAME, 'loc', 'scale', 'value'), + [ + ( + 'value-float', + np.array([0.1, 0.4]), + np.array([1.0, 4.0]), + np.array([3.0, 7.0]), + ), ('value-int', np.array([0.1, 0.4]), np.array([1, 4]), np.array([3, 7])), - ('value-multi-dim', np.array([0.1, 0.4]), np.array( - [1, 4]), np.array([[5., 4], [6, 2]])), - ]) + ( + 'value-multi-dim', + np.array([0.1, 0.4]), + np.array([1, 4]), + np.array([[5.0, 4], [6, 2]]), + ), + ], +) class TestGumbelPDF(unittest.TestCase): - def setUp(self): - self._dist = Gumbel(loc=paddle.to_tensor(self.loc), - scale=paddle.to_tensor(self.scale)) + self._dist = Gumbel( + loc=paddle.to_tensor(self.loc), scale=paddle.to_tensor(self.scale) + ) def test_prob(self): np.testing.assert_allclose( self._dist.prob(paddle.to_tensor(self.value)), scipy.stats.gumbel_r.pdf(self.value, self.loc, self.scale), rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_log_prob(self): np.testing.assert_allclose( self._dist.log_prob(paddle.to_tensor(self.value)), scipy.stats.gumbel_r.logpdf(self.value, self.loc, self.scale), rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_cdf(self): - np.testing.assert_allclose(self._dist.cdf(paddle.to_tensor(self.value)), - scipy.stats.gumbel_r.cdf( - self.value, self.loc, self.scale), - rtol=0.02, - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self._dist.cdf(paddle.to_tensor(self.value)), + scipy.stats.gumbel_r.cdf(self.value, self.loc, self.scale), + rtol=0.02, + atol=config.ATOL.get(str(self.loc.dtype)), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel_static.py index 1c39da5fccf37..7e0176dee561c 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_gumbel_static.py @@ -27,20 +27,23 @@ @parameterize.place(config.DEVICES) -@parameterize.parameterize_cls((parameterize.TEST_CASE_NAME, 'loc', 'scale'), [ - ('one-dim', parameterize.xrand((4, )), parameterize.xrand((4, ))), - ('multi-dim', parameterize.xrand((5, 3)), parameterize.xrand((5, 3))), -]) +@parameterize.parameterize_cls( + (parameterize.TEST_CASE_NAME, 'loc', 'scale'), + [ + ('one-dim', parameterize.xrand((4,)), parameterize.xrand((4,))), + ('multi-dim', parameterize.xrand((5, 3)), parameterize.xrand((5, 3))), + ], +) class TestGumbel(unittest.TestCase): - def setUp(self): startup_program = paddle.static.Program() main_program = paddle.static.Program() executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) self._dist = Gumbel(loc=loc, scale=scale) self.sample_shape = [50000] mean = self._dist.mean @@ -52,50 +55,63 @@ def setUp(self): self.feeds = {'loc': self.loc, 'scale': self.scale} executor.run(startup_program) - [self.mean, self.var, self.stddev, self.entropy, - self.samples] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [ + self.mean, + self.var, + self.stddev, + self.entropy, + self.samples, + ] = executor.run(main_program, feed=self.feeds, fetch_list=fetch_list) def test_mean(self): self.assertEqual(str(self.mean.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.mean, - self._np_mean(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.mean, + self._np_mean(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_variance(self): self.assertEqual(str(self.var.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.var, - self._np_variance(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.var, + self._np_variance(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_stddev(self): self.assertEqual( - str(self.stddev.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.stddev, - self._np_stddev(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + str(self.stddev.dtype).split('.')[-1], self.scale.dtype + ) + np.testing.assert_allclose( + self.stddev, + self._np_stddev(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_entropy(self): self.assertEqual( - str(self.entropy.dtype).split('.')[-1], self.scale.dtype) + str(self.entropy.dtype).split('.')[-1], self.scale.dtype + ) def test_sample(self): self.assertEqual(self.samples.dtype, self.scale.dtype) - np.testing.assert_allclose(self.samples.mean(axis=0), - scipy.stats.gumbel_r.mean(self.loc, - scale=self.scale), - rtol=0.1, - atol=config.ATOL.get(str(self.scale.dtype))) - np.testing.assert_allclose(self.samples.var(axis=0), - scipy.stats.gumbel_r.var(self.loc, - scale=self.scale), - rtol=0.1, - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.samples.mean(axis=0), + scipy.stats.gumbel_r.mean(self.loc, scale=self.scale), + rtol=0.1, + atol=config.ATOL.get(str(self.scale.dtype)), + ) + np.testing.assert_allclose( + self.samples.var(axis=0), + scipy.stats.gumbel_r.var(self.loc, scale=self.scale), + rtol=0.1, + atol=config.ATOL.get(str(self.scale.dtype)), + ) def _np_mean(self): return self.loc + self.scale * np.euler_gamma @@ -105,7 +121,8 @@ def _np_stddev(self): def _np_variance(self): return np.divide( - np.multiply(np.power(self.scale, 2), np.power(np.pi, 2)), 6) + np.multiply(np.power(self.scale, 2), np.power(np.pi, 2)), 6 + ) def _np_entropy(self): return np.log(self.scale) + 1 + np.euler_gamma @@ -113,15 +130,24 @@ def _np_entropy(self): @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'loc', 'scale', 'value'), [ - ('value-float', np.array([0.1, 0.4]), np.array([1., 4. - ]), np.array([3., 7.])), + (parameterize.TEST_CASE_NAME, 'loc', 'scale', 'value'), + [ + ( + 'value-float', + np.array([0.1, 0.4]), + np.array([1.0, 4.0]), + np.array([3.0, 7.0]), + ), ('value-int', np.array([0.1, 0.4]), np.array([1, 4]), np.array([3, 7])), - ('value-multi-dim', np.array([0.1, 0.4]), np.array( - [1, 4]), np.array([[5., 4], [6, 2]])), - ]) + ( + 'value-multi-dim', + np.array([0.1, 0.4]), + np.array([1, 4]), + np.array([[5.0, 4], [6, 2]]), + ), + ], +) class TestGumbelPDF(unittest.TestCase): - def setUp(self): startup_program = paddle.static.Program() main_program = paddle.static.Program() @@ -129,10 +155,12 @@ def setUp(self): with paddle.static.program_guard(main_program, startup_program): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) - value = paddle.static.data('value', self.value.shape, - self.value.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) + value = paddle.static.data( + 'value', self.value.shape, self.value.dtype + ) self._dist = Gumbel(loc=loc, scale=scale) prob = self._dist.prob(value) log_prob = self._dist.log_prob(value) @@ -141,31 +169,33 @@ def setUp(self): self.feeds = {'loc': self.loc, 'scale': self.scale, 'value': self.value} executor.run(startup_program) - [self.prob, self.log_prob, - self.cdf] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [self.prob, self.log_prob, self.cdf] = executor.run( + main_program, feed=self.feeds, fetch_list=fetch_list + ) def test_prob(self): - np.testing.assert_allclose(self.prob, - scipy.stats.gumbel_r.pdf( - self.value, self.loc, self.scale), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self.prob, + scipy.stats.gumbel_r.pdf(self.value, self.loc, self.scale), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_log_prob(self): - np.testing.assert_allclose(self.log_prob, - scipy.stats.gumbel_r.logpdf( - self.value, self.loc, self.scale), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self.log_prob, + scipy.stats.gumbel_r.logpdf(self.value, self.loc, self.scale), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_cdf(self): - np.testing.assert_allclose(self.cdf, - scipy.stats.gumbel_r.cdf( - self.value, self.loc, self.scale), - rtol=0.3, - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self.cdf, + scipy.stats.gumbel_r.cdf(self.value, self.loc, self.scale), + rtol=0.3, + atol=config.ATOL.get(str(self.loc.dtype)), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent.py index eef5b5a6e442c..c4c83ecb12b3c 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent.py @@ -25,35 +25,45 @@ @param.place(config.DEVICES) @param.param_cls( (param.TEST_CASE_NAME, 'base', 'reinterpreted_batch_rank'), - [('base_beta', - paddle.distribution.Beta(paddle.rand([1, 2]), paddle.rand([1, 2])), 1)]) + [ + ( + 'base_beta', + paddle.distribution.Beta(paddle.rand([1, 2]), paddle.rand([1, 2])), + 1, + ) + ], +) class TestIndependent(unittest.TestCase): - def setUp(self): - self._t = paddle.distribution.Independent(self.base, - self.reinterpreted_batch_rank) + self._t = paddle.distribution.Independent( + self.base, self.reinterpreted_batch_rank + ) def test_mean(self): np.testing.assert_allclose( self.base.mean, self._t.mean, rtol=config.RTOL.get(str(self.base.alpha.numpy().dtype)), - atol=config.ATOL.get(str(self.base.alpha.numpy().dtype))) + atol=config.ATOL.get(str(self.base.alpha.numpy().dtype)), + ) def test_variance(self): np.testing.assert_allclose( self.base.variance, self._t.variance, rtol=config.RTOL.get(str(self.base.alpha.numpy().dtype)), - atol=config.ATOL.get(str(self.base.alpha.numpy().dtype))) + atol=config.ATOL.get(str(self.base.alpha.numpy().dtype)), + ) def test_entropy(self): np.testing.assert_allclose( - self._np_sum_rightmost(self.base.entropy().numpy(), - self.reinterpreted_batch_rank), + self._np_sum_rightmost( + self.base.entropy().numpy(), self.reinterpreted_batch_rank + ), self._t.entropy(), rtol=config.RTOL.get(str(self.base.alpha.numpy().dtype)), - atol=config.ATOL.get(str(self.base.alpha.numpy().dtype))) + atol=config.ATOL.get(str(self.base.alpha.numpy().dtype)), + ) def _np_sum_rightmost(self, value, n): return np.sum(value, tuple(range(-n, 0))) if n > 0 else value @@ -63,10 +73,12 @@ def test_log_prob(self): np.testing.assert_allclose( self._np_sum_rightmost( self.base.log_prob(paddle.to_tensor(value)).numpy(), - self.reinterpreted_batch_rank), + self.reinterpreted_batch_rank, + ), self._t.log_prob(paddle.to_tensor(value)).numpy(), rtol=config.RTOL.get(str(self.base.alpha.numpy().dtype)), - atol=config.ATOL.get(str(self.base.alpha.numpy().dtype))) + atol=config.ATOL.get(str(self.base.alpha.numpy().dtype)), + ) # TODO(cxxly): Add Kolmogorov-Smirnov test for sample result. def test_sample(self): @@ -79,16 +91,28 @@ def test_sample(self): @param.place(config.DEVICES) @param.param_cls( - (param.TEST_CASE_NAME, 'base', 'reinterpreted_batch_rank', - 'expected_exception'), - [('base_not_transform', '', 1, TypeError), - ('rank_less_than_zero', paddle.distribution.Transform(), -1, ValueError)]) + ( + param.TEST_CASE_NAME, + 'base', + 'reinterpreted_batch_rank', + 'expected_exception', + ), + [ + ('base_not_transform', '', 1, TypeError), + ( + 'rank_less_than_zero', + paddle.distribution.Transform(), + -1, + ValueError, + ), + ], +) class TestIndependentException(unittest.TestCase): - def test_init(self): with self.assertRaises(self.expected_exception): paddle.distribution.IndependentTransform( - self.base, self.reinterpreted_batch_rank) + self.base, self.reinterpreted_batch_rank + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent_static.py index 47bf1021e1a3a..06881bdcdc244 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_independent_static.py @@ -26,10 +26,17 @@ @param.place(config.DEVICES) @param.param_cls( (param.TEST_CASE_NAME, 'base', 'reinterpreted_batch_rank', 'alpha', 'beta'), - [('base_beta', paddle.distribution.Beta, 1, np.random.rand( - 1, 2), np.random.rand(1, 2))]) + [ + ( + 'base_beta', + paddle.distribution.Beta, + 1, + np.random.rand(1, 2), + np.random.rand(1, 2), + ) + ], +) class TestIndependent(unittest.TestCase): - def setUp(self): value = np.random.rand(1) self.dtype = value.dtype @@ -37,12 +44,14 @@ def setUp(self): sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): - alpha = paddle.static.data('alpha', self.alpha.shape, - self.alpha.dtype) + alpha = paddle.static.data( + 'alpha', self.alpha.shape, self.alpha.dtype + ) beta = paddle.static.data('beta', self.beta.shape, self.beta.dtype) self.base = self.base(alpha, beta) - t = paddle.distribution.Independent(self.base, - self.reinterpreted_batch_rank) + t = paddle.distribution.Independent( + self.base, self.reinterpreted_batch_rank + ) mean = t.mean variance = t.variance entropy = t.entropy() @@ -55,50 +64,69 @@ def setUp(self): base_log_prob = self.base.log_prob(static_value) fetch_list = [ - mean, variance, entropy, log_prob, base_mean, base_variance, - base_entropy, base_log_prob + mean, + variance, + entropy, + log_prob, + base_mean, + base_variance, + base_entropy, + base_log_prob, ] exe.run(sp) [ - self.mean, self.variance, self.entropy, self.log_prob, - self.base_mean, self.base_variance, self.base_entropy, - self.base_log_prob - ] = exe.run(mp, - feed={ - 'value': value, - 'alpha': self.alpha, - 'beta': self.beta - }, - fetch_list=fetch_list) + self.mean, + self.variance, + self.entropy, + self.log_prob, + self.base_mean, + self.base_variance, + self.base_entropy, + self.base_log_prob, + ] = exe.run( + mp, + feed={'value': value, 'alpha': self.alpha, 'beta': self.beta}, + fetch_list=fetch_list, + ) def test_mean(self): - np.testing.assert_allclose(self.mean, - self.base_mean, - rtol=config.RTOL.get(str(self.dtype)), - atol=config.ATOL.get(str(self.dtype))) + np.testing.assert_allclose( + self.mean, + self.base_mean, + rtol=config.RTOL.get(str(self.dtype)), + atol=config.ATOL.get(str(self.dtype)), + ) def test_variance(self): - np.testing.assert_allclose(self.variance, - self.base_variance, - rtol=config.RTOL.get(str(self.dtype)), - atol=config.ATOL.get(str(self.dtype))) + np.testing.assert_allclose( + self.variance, + self.base_variance, + rtol=config.RTOL.get(str(self.dtype)), + atol=config.ATOL.get(str(self.dtype)), + ) def test_entropy(self): - np.testing.assert_allclose(self._np_sum_rightmost( - self.base_entropy, self.reinterpreted_batch_rank), - self.entropy, - rtol=config.RTOL.get(str(self.dtype)), - atol=config.ATOL.get(str(self.dtype))) + np.testing.assert_allclose( + self._np_sum_rightmost( + self.base_entropy, self.reinterpreted_batch_rank + ), + self.entropy, + rtol=config.RTOL.get(str(self.dtype)), + atol=config.ATOL.get(str(self.dtype)), + ) def _np_sum_rightmost(self, value, n): return np.sum(value, tuple(range(-n, 0))) if n > 0 else value def test_log_prob(self): - np.testing.assert_allclose(self._np_sum_rightmost( - self.base_log_prob, self.reinterpreted_batch_rank), - self.log_prob, - rtol=config.RTOL.get(str(self.dtype)), - atol=config.ATOL.get(str(self.dtype))) + np.testing.assert_allclose( + self._np_sum_rightmost( + self.base_log_prob, self.reinterpreted_batch_rank + ), + self.log_prob, + rtol=config.RTOL.get(str(self.dtype)), + atol=config.ATOL.get(str(self.dtype)), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace.py index 867fc1846a34d..f93e3fac71a36 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace.py @@ -23,42 +23,47 @@ @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'loc', 'scale'), [ - ('one-dim', parameterize.xrand((2, )),\ - parameterize.xrand((2, ))), - ('multi-dim', parameterize.xrand((5, 5)),\ - parameterize.xrand((5, 5))), - ]) + (parameterize.TEST_CASE_NAME, 'loc', 'scale'), + [ + ('one-dim', parameterize.xrand((2,)), parameterize.xrand((2,))), + ('multi-dim', parameterize.xrand((5, 5)), parameterize.xrand((5, 5))), + ], +) class TestLaplace(unittest.TestCase): - def setUp(self): - self._dist = paddle.distribution.Laplace(loc=paddle.to_tensor(self.loc), - scale=paddle.to_tensor(\ - self.scale)) + self._dist = paddle.distribution.Laplace( + loc=paddle.to_tensor(self.loc), scale=paddle.to_tensor(self.scale) + ) def test_mean(self): mean = self._dist.mean self.assertEqual(mean.numpy().dtype, self.scale.dtype) - np.testing.assert_allclose(mean, - self._np_mean(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + mean, + self._np_mean(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_variance(self): var = self._dist.variance self.assertEqual(var.numpy().dtype, self.scale.dtype) - np.testing.assert_allclose(var, - self._np_variance(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + var, + self._np_variance(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_stddev(self): stddev = self._dist.stddev self.assertEqual(stddev.numpy().dtype, self.scale.dtype) - np.testing.assert_allclose(stddev, - self._np_stddev(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + stddev, + self._np_stddev(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_entropy(self): entropy = self._dist.entropy() @@ -66,27 +71,30 @@ def test_entropy(self): def test_sample(self): - sample_shape = (50000, ) + sample_shape = (50000,) samples = self._dist.sample(sample_shape) sample_values = samples.numpy() self.assertEqual(samples.numpy().dtype, self.scale.dtype) - self.assertEqual(tuple(samples.shape), - tuple(self._dist._extend_shape(sample_shape))) + self.assertEqual( + tuple(samples.shape), tuple(self._dist._extend_shape(sample_shape)) + ) self.assertEqual(samples.shape, list(sample_shape + self.loc.shape)) self.assertEqual(sample_values.shape, sample_shape + self.loc.shape) - np.testing.assert_allclose(sample_values.mean(axis=0), - scipy.stats.laplace.mean(self.loc, - scale=self.scale), - rtol=0.2, - atol=0.) - np.testing.assert_allclose(sample_values.var(axis=0), - scipy.stats.laplace.var(self.loc, - scale=self.scale), - rtol=0.1, - atol=0.) + np.testing.assert_allclose( + sample_values.mean(axis=0), + scipy.stats.laplace.mean(self.loc, scale=self.scale), + rtol=0.2, + atol=0.0, + ) + np.testing.assert_allclose( + sample_values.var(axis=0), + scipy.stats.laplace.var(self.loc, scale=self.scale), + rtol=0.1, + atol=0.0, + ) def _np_mean(self): return self.loc @@ -103,18 +111,20 @@ def _np_entropy(self): @parameterize.place(config.DEVICES) -@parameterize.parameterize_cls((parameterize.TEST_CASE_NAME, 'loc', 'scale'), [ - ('float', 1., 2.), - ('int', 3, 4), -]) +@parameterize.parameterize_cls( + (parameterize.TEST_CASE_NAME, 'loc', 'scale'), + [ + ('float', 1.0, 2.0), + ('int', 3, 4), + ], +) class TestLaplaceKS(unittest.TestCase): - def setUp(self): self._dist = paddle.distribution.Laplace(loc=self.loc, scale=self.scale) def test_sample(self): - sample_shape = (20000, ) + sample_shape = (20000,) samples = self._dist.sample(sample_shape) sample_values = samples.numpy() self.assertTrue(self._kstest(self.loc, self.scale, sample_values)) @@ -122,8 +132,8 @@ def test_sample(self): def _kstest(self, loc, scale, samples): # Uses the Kolmogorov-Smirnov test for goodness of fit. ks, p_value = scipy.stats.kstest( - samples, - scipy.stats.laplace(loc, scale=scale).cdf) + samples, scipy.stats.laplace(loc, scale=scale).cdf + ) return ks < 0.02 @@ -131,78 +141,101 @@ def _kstest(self, loc, scale, samples): @parameterize.parameterize_cls( (parameterize.TEST_CASE_NAME, 'loc', 'scale', 'value'), [ - ('value-float', np.array([0.2, 0.3]),\ - np.array([2., 3.]), np.array([2., 5.])), - ('value-int', np.array([0.2, 0.3]),\ - np.array([2., 3.]), np.array([2, 5])), - ('value-multi-dim', np.array([0.2, 0.3]), np.array([2., 3.]),\ - np.array([[4., 6], [8, 2]])), - ]) + ( + 'value-float', + np.array([0.2, 0.3]), + np.array([2.0, 3.0]), + np.array([2.0, 5.0]), + ), + ( + 'value-int', + np.array([0.2, 0.3]), + np.array([2.0, 3.0]), + np.array([2, 5]), + ), + ( + 'value-multi-dim', + np.array([0.2, 0.3]), + np.array([2.0, 3.0]), + np.array([[4.0, 6], [8, 2]]), + ), + ], +) class TestLaplacePDF(unittest.TestCase): - def setUp(self): - self._dist = paddle.distribution.Laplace(loc=paddle.to_tensor(self.loc), - scale=paddle.to_tensor(\ - self.scale)) + self._dist = paddle.distribution.Laplace( + loc=paddle.to_tensor(self.loc), scale=paddle.to_tensor(self.scale) + ) def test_prob(self): np.testing.assert_allclose( self._dist.prob(paddle.to_tensor(self.value)), scipy.stats.laplace.pdf(self.value, self.loc, self.scale), rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_log_prob(self): np.testing.assert_allclose( self._dist.log_prob(paddle.to_tensor(self.value)), scipy.stats.laplace.logpdf(self.value, self.loc, self.scale), rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_cdf(self): - np.testing.assert_allclose(self._dist.cdf(paddle.to_tensor(self.value)), - scipy.stats.laplace.cdf( - self.value, self.loc, self.scale), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self._dist.cdf(paddle.to_tensor(self.value)), + scipy.stats.laplace.cdf(self.value, self.loc, self.scale), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_icdf(self): np.testing.assert_allclose( self._dist.icdf(paddle.to_tensor(self.value)), scipy.stats.laplace.ppf(self.value, self.loc, self.scale), rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + atol=config.ATOL.get(str(self.loc.dtype)), + ) @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'loc1', 'scale1',\ - 'loc2', 'scale2'), [ - ('kl', np.array([0.0]), np.array([1.0]), \ - np.array([1.0]), np.array([0.5])) - ]) + (parameterize.TEST_CASE_NAME, 'loc1', 'scale1', 'loc2', 'scale2'), + [ + ( + 'kl', + np.array([0.0]), + np.array([1.0]), + np.array([1.0]), + np.array([0.5]), + ) + ], +) class TestLaplaceAndLaplaceKL(unittest.TestCase): - def setUp(self): - self._dist_1 = paddle.distribution.Laplace(loc=paddle.to_tensor(self.loc1), - scale=paddle.to_tensor(\ - self.scale1)) - self._dist_2 = paddle.distribution.Laplace(loc=paddle.to_tensor(self.loc2), - scale=paddle.to_tensor(\ - self.scale2)) + self._dist_1 = paddle.distribution.Laplace( + loc=paddle.to_tensor(self.loc1), scale=paddle.to_tensor(self.scale1) + ) + self._dist_2 = paddle.distribution.Laplace( + loc=paddle.to_tensor(self.loc2), scale=paddle.to_tensor(self.scale2) + ) def test_kl_divergence(self): - np.testing.assert_allclose(paddle.distribution.kl_divergence( - self._dist_1, self._dist_2), - self._np_kl(), - atol=0, - rtol=0.50) + np.testing.assert_allclose( + paddle.distribution.kl_divergence(self._dist_1, self._dist_2), + self._np_kl(), + atol=0, + rtol=0.50, + ) def _np_kl(self): - x = np.linspace(scipy.stats.laplace.ppf(0.01),\ - scipy.stats.laplace.ppf(0.99), 1000) - d1 = scipy.stats.laplace.pdf(x, loc=0., scale=1.) - d2 = scipy.stats.laplace.pdf(x, loc=1., scale=0.5) + x = np.linspace( + scipy.stats.laplace.ppf(0.01), scipy.stats.laplace.ppf(0.99), 1000 + ) + d1 = scipy.stats.laplace.pdf(x, loc=0.0, scale=1.0) + d2 = scipy.stats.laplace.pdf(x, loc=1.0, scale=0.5) return scipy.stats.entropy(d1, d2) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace_static.py index 62fe225849c43..7c8ecf1609b50 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_laplace_static.py @@ -26,24 +26,24 @@ @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'loc', 'scale'), [ - ('one-dim', parameterize.xrand((2, )),\ - parameterize.xrand((2, ))), - ('multi-dim', parameterize.xrand((5, 5)),\ - parameterize.xrand((5, 5))), - ]) + (parameterize.TEST_CASE_NAME, 'loc', 'scale'), + [ + ('one-dim', parameterize.xrand((2,)), parameterize.xrand((2,))), + ('multi-dim', parameterize.xrand((5, 5)), parameterize.xrand((5, 5))), + ], +) class TestLaplace(unittest.TestCase): - def setUp(self): startup_program = paddle.static.Program() main_program = paddle.static.Program() executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) self._dist = paddle.distribution.Laplace(loc=loc, scale=scale) - self.sample_shape = (50000, ) + self.sample_shape = (50000,) mean = self._dist.mean var = self._dist.variance stddev = self._dist.stddev @@ -53,56 +53,71 @@ def setUp(self): self.feeds = {'loc': self.loc, 'scale': self.scale} executor.run(startup_program) - [self.mean, self.var, self.stddev, self.entropy, - self.samples] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [ + self.mean, + self.var, + self.stddev, + self.entropy, + self.samples, + ] = executor.run(main_program, feed=self.feeds, fetch_list=fetch_list) def test_mean(self): self.assertEqual(str(self.mean.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.mean, - self._np_mean(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.mean, + self._np_mean(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_variance(self): self.assertEqual(str(self.var.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.var, - self._np_variance(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.var, + self._np_variance(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_stddev(self): self.assertEqual( - str(self.stddev.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.stddev, - self._np_stddev(), - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + str(self.stddev.dtype).split('.')[-1], self.scale.dtype + ) + np.testing.assert_allclose( + self.stddev, + self._np_stddev(), + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_entropy(self): self.assertEqual( - str(self.entropy.dtype).split('.')[-1], self.scale.dtype) + str(self.entropy.dtype).split('.')[-1], self.scale.dtype + ) def test_sample(self): self.assertEqual(self.samples.dtype, self.scale.dtype) - self.assertEqual(tuple(self.samples.shape), - tuple(self._dist._extend_shape(self.sample_shape))) + self.assertEqual( + tuple(self.samples.shape), + tuple(self._dist._extend_shape(self.sample_shape)), + ) self.assertEqual(self.samples.shape, self.sample_shape + self.loc.shape) self.assertEqual(self.samples.shape, self.sample_shape + self.loc.shape) - np.testing.assert_allclose(self.samples.mean(axis=0), - scipy.stats.laplace.mean(self.loc, - scale=self.scale), - rtol=0.2, - atol=0.) - np.testing.assert_allclose(self.samples.var(axis=0), - scipy.stats.laplace.var(self.loc, - scale=self.scale), - rtol=0.1, - atol=0.) + np.testing.assert_allclose( + self.samples.mean(axis=0), + scipy.stats.laplace.mean(self.loc, scale=self.scale), + rtol=0.2, + atol=0.0, + ) + np.testing.assert_allclose( + self.samples.var(axis=0), + scipy.stats.laplace.var(self.loc, scale=self.scale), + rtol=0.1, + atol=0.0, + ) def _np_mean(self): return self.loc @@ -122,15 +137,27 @@ def _np_entropy(self): @parameterize.parameterize_cls( (parameterize.TEST_CASE_NAME, 'loc', 'scale', 'value'), [ - ('value-float', np.array([0.2, 0.3]),\ - np.array([2., 3.]), np.array([2., 5.])), - ('value-int', np.array([0.2, 0.3]),\ - np.array([2., 3.]), np.array([2, 5])), - ('value-multi-dim', np.array([0.2, 0.3]), np.array([2., 3.]),\ - np.array([[4., 6], [8, 2]])), - ]) + ( + 'value-float', + np.array([0.2, 0.3]), + np.array([2.0, 3.0]), + np.array([2.0, 5.0]), + ), + ( + 'value-int', + np.array([0.2, 0.3]), + np.array([2.0, 3.0]), + np.array([2, 5]), + ), + ( + 'value-multi-dim', + np.array([0.2, 0.3]), + np.array([2.0, 3.0]), + np.array([[4.0, 6], [8, 2]]), + ), + ], +) class TestLaplacePDF(unittest.TestCase): - def setUp(self): startup_program = paddle.static.Program() main_program = paddle.static.Program() @@ -138,10 +165,12 @@ def setUp(self): with paddle.static.program_guard(main_program, startup_program): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) - value = paddle.static.data('value', self.value.shape, - self.value.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) + value = paddle.static.data( + 'value', self.value.shape, self.value.dtype + ) self._dist = paddle.distribution.Laplace(loc=loc, scale=scale) prob = self._dist.prob(value) log_prob = self._dist.log_prob(value) @@ -151,49 +180,57 @@ def setUp(self): self.feeds = {'loc': self.loc, 'scale': self.scale, 'value': self.value} executor.run(startup_program) - [self.prob, self.log_prob, self.cdf, - self.icdf] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [self.prob, self.log_prob, self.cdf, self.icdf] = executor.run( + main_program, feed=self.feeds, fetch_list=fetch_list + ) def test_prob(self): - np.testing.assert_allclose(self.prob, - scipy.stats.laplace.pdf( - self.value, self.loc, self.scale), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self.prob, + scipy.stats.laplace.pdf(self.value, self.loc, self.scale), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_log_prob(self): - np.testing.assert_allclose(self.log_prob, - scipy.stats.laplace.logpdf( - self.value, self.loc, self.scale), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self.log_prob, + scipy.stats.laplace.logpdf(self.value, self.loc, self.scale), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_cdf(self): - np.testing.assert_allclose(self.cdf, - scipy.stats.laplace.cdf( - self.value, self.loc, self.scale), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self.cdf, + scipy.stats.laplace.cdf(self.value, self.loc, self.scale), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_icdf(self): - np.testing.assert_allclose(self.icdf, - scipy.stats.laplace.ppf( - self.value, self.loc, self.scale), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + np.testing.assert_allclose( + self.icdf, + scipy.stats.laplace.ppf(self.value, self.loc, self.scale), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'loc1', 'scale1',\ - 'loc2', 'scale2'), [ - ('kl', np.array([0.0]), np.array([1.0]), \ - np.array([1.0]), np.array([0.5])) - ]) + (parameterize.TEST_CASE_NAME, 'loc1', 'scale1', 'loc2', 'scale2'), + [ + ( + 'kl', + np.array([0.0]), + np.array([1.0]), + np.array([1.0]), + np.array([0.5]), + ) + ], +) class TestLaplaceAndLaplaceKL(unittest.TestCase): - def setUp(self): self.mp = paddle.static.Program() self.sp = paddle.static.Program() @@ -201,34 +238,37 @@ def setUp(self): with paddle.static.program_guard(self.mp, self.sp): loc1 = paddle.static.data('loc1', self.loc1.shape, self.loc1.dtype) - scale1 = paddle.static.data('scale1', self.scale1.shape, - self.scale1.dtype) + scale1 = paddle.static.data( + 'scale1', self.scale1.shape, self.scale1.dtype + ) loc2 = paddle.static.data('loc2', self.loc2.shape, self.loc2.dtype) - scale2 = paddle.static.data('scale2', self.scale2.shape, - self.scale2.dtype) + scale2 = paddle.static.data( + 'scale2', self.scale2.shape, self.scale2.dtype + ) self._dist_1 = paddle.distribution.Laplace(loc=loc1, scale=scale1) self._dist_2 = paddle.distribution.Laplace(loc=loc2, scale=scale2) self.feeds = { 'loc1': self.loc1, 'scale1': self.scale1, 'loc2': self.loc2, - 'scale2': self.scale2 + 'scale2': self.scale2, } def test_kl_divergence(self): with paddle.static.program_guard(self.mp, self.sp): out = paddle.distribution.kl_divergence(self._dist_1, self._dist_2) self.executor.run(self.sp) - [out] = self.executor.run(self.mp, - feed=self.feeds, - fetch_list=[out]) + [out] = self.executor.run( + self.mp, feed=self.feeds, fetch_list=[out] + ) np.testing.assert_allclose(out, self._np_kl(), atol=0, rtol=0.50) def _np_kl(self): - x = np.linspace(scipy.stats.laplace.ppf(0.01),\ - scipy.stats.laplace.ppf(0.99), 1000) - d1 = scipy.stats.laplace.pdf(x, loc=0., scale=1.) - d2 = scipy.stats.laplace.pdf(x, loc=1., scale=0.5) + x = np.linspace( + scipy.stats.laplace.ppf(0.01), scipy.stats.laplace.ppf(0.99), 1000 + ) + d1 = scipy.stats.laplace.pdf(x, loc=0.0, scale=1.0) + d2 = scipy.stats.laplace.pdf(x, loc=1.0, scale=0.5) return scipy.stats.entropy(d1, d2) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal.py index a7c97047505c3..be51e1ecf0aa4 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal.py @@ -28,7 +28,6 @@ class LogNormalNumpy(DistributionNumpy): - def __init__(self, loc, scale): self.loc = np.array(loc) self.scale = np.array(scale) @@ -49,67 +48,86 @@ def variance(self): def log_prob(self, value): var = self.scale * self.scale log_scale = np.log(self.scale) - return -( - (np.log(value) - self.loc) * - (np.log(value) - self.loc)) / (2. * var) - log_scale - math.log( - math.sqrt(2. * math.pi)) - np.log(value) + return ( + -((np.log(value) - self.loc) * (np.log(value) - self.loc)) + / (2.0 * var) + - log_scale + - math.log(math.sqrt(2.0 * math.pi)) + - np.log(value) + ) def probs(self, value): var = self.scale * self.scale return np.exp( - -1. * ((np.log(value) - self.loc) * (np.log(value) - self.loc)) / - (2. * var)) / (math.sqrt(2 * math.pi) * self.scale * value) + -1.0 + * ((np.log(value) - self.loc) * (np.log(value) - self.loc)) + / (2.0 * var) + ) / (math.sqrt(2 * math.pi) * self.scale * value) def entropy(self): - return 0.5 + self.loc + 0.5 * np.log( - np.array(2. * math.pi).astype(self.loc.dtype)) + np.log(self.scale) + return ( + 0.5 + + self.loc + + 0.5 * np.log(np.array(2.0 * math.pi).astype(self.loc.dtype)) + + np.log(self.scale) + ) def kl_divergence(self, other): - var_ratio = (self.scale / other.scale) + var_ratio = self.scale / other.scale var_ratio = var_ratio * var_ratio - t1 = ((self.loc - other.loc) / other.scale) - t1 = (t1 * t1) + t1 = (self.loc - other.loc) / other.scale + t1 = t1 * t1 return 0.5 * (var_ratio + t1 - 1 - np.log(var_ratio)) @place(config.DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'loc', 'scale', 'value'), - [('one-dim', xrand((2, )), xrand((2, )), xrand((2, ))), - ('multi-dim', xrand((3, 3)), xrand((3, 3)), xrand((3, 3)))]) +@parameterize_cls( + (TEST_CASE_NAME, 'loc', 'scale', 'value'), + [ + ('one-dim', xrand((2,)), xrand((2,)), xrand((2,))), + ('multi-dim', xrand((3, 3)), xrand((3, 3)), xrand((3, 3))), + ], +) class LogNormalTest(unittest.TestCase): - def setUp(self): paddle.disable_static() - self.paddle_lognormal = LogNormal(loc=paddle.to_tensor(self.loc), - scale=paddle.to_tensor(self.scale)) + self.paddle_lognormal = LogNormal( + loc=paddle.to_tensor(self.loc), scale=paddle.to_tensor(self.scale) + ) self.np_lognormal = LogNormalNumpy(self.loc, self.scale) def test_mean(self): mean = self.paddle_lognormal.mean np_mean = self.np_lognormal.mean self.assertEqual(mean.numpy().dtype, np_mean.dtype) - np.testing.assert_allclose(mean, - np_mean, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + mean, + np_mean, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_variance(self): var = self.paddle_lognormal.variance np_var = self.np_lognormal.variance self.assertEqual(var.numpy().dtype, np_var.dtype) - np.testing.assert_allclose(var, - np_var, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + var, + np_var, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_entropy(self): entropy = self.paddle_lognormal.entropy() np_entropy = self.np_lognormal.entropy() self.assertEqual(entropy.numpy().dtype, np_entropy.dtype) - np.testing.assert_allclose(entropy, - np_entropy, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + entropy, + np_entropy, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_probs(self): with paddle.fluid.dygraph.guard(self.place): @@ -119,94 +137,110 @@ def test_probs(self): probs, np_probs, rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_log_prob(self): with paddle.fluid.dygraph.guard(self.place): log_prob = self.paddle_lognormal.log_prob( - paddle.to_tensor(self.value)) + paddle.to_tensor(self.value) + ) np_log_prob = self.np_lognormal.log_prob(self.value) np.testing.assert_allclose( log_prob, np_log_prob, rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + atol=config.ATOL.get(str(self.scale.dtype)), + ) @place(config.DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'loc', 'scale'), [('sample', xrand( - (4, )), xrand((4, ), min=0, max=1))]) +@parameterize_cls( + (TEST_CASE_NAME, 'loc', 'scale'), + [('sample', xrand((4,)), xrand((4,), min=0, max=1))], +) class TestLogNormalSample(unittest.TestCase): - def setUp(self): paddle.disable_static() self.paddle_lognormal = LogNormal(loc=self.loc, scale=self.scale) n = 100000 - self.sample_shape = (n, ) - self.rsample_shape = (n, ) + self.sample_shape = (n,) + self.rsample_shape = (n,) self.samples = self.paddle_lognormal.sample(self.sample_shape) self.rsamples = self.paddle_lognormal.rsample(self.rsample_shape) def test_sample(self): samples_mean = self.samples.mean(axis=0) samples_var = self.samples.var(axis=0) - np.testing.assert_allclose(samples_mean, - self.paddle_lognormal.mean, - rtol=0.1, - atol=0) - np.testing.assert_allclose(samples_var, - self.paddle_lognormal.variance, - rtol=0.1, - atol=0) + np.testing.assert_allclose( + samples_mean, self.paddle_lognormal.mean, rtol=0.1, atol=0 + ) + np.testing.assert_allclose( + samples_var, self.paddle_lognormal.variance, rtol=0.1, atol=0 + ) rsamples_mean = self.rsamples.mean(axis=0) rsamples_var = self.rsamples.var(axis=0) - np.testing.assert_allclose(rsamples_mean, - self.paddle_lognormal.mean, - rtol=0.1, - atol=0) - np.testing.assert_allclose(rsamples_var, - self.paddle_lognormal.variance, - rtol=0.1, - atol=0) + np.testing.assert_allclose( + rsamples_mean, self.paddle_lognormal.mean, rtol=0.1, atol=0 + ) + np.testing.assert_allclose( + rsamples_var, self.paddle_lognormal.variance, rtol=0.1, atol=0 + ) batch_shape = (self.loc + self.scale).shape - self.assertEqual(self.samples.shape, - list(self.sample_shape + batch_shape)) - self.assertEqual(self.rsamples.shape, - list(self.rsample_shape + batch_shape)) + self.assertEqual( + self.samples.shape, list(self.sample_shape + batch_shape) + ) + self.assertEqual( + self.rsamples.shape, list(self.rsample_shape + batch_shape) + ) for i in range(len(self.scale)): self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.samples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.samples[:, i]) + ) self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i]) + ) def _kstest(self, loc, scale, samples): # Uses the Kolmogorov-Smirnov test for goodness of fit. ks, _ = scipy.stats.kstest( - samples, - scipy.stats.lognorm(s=scale, scale=np.exp(loc)).cdf) + samples, scipy.stats.lognorm(s=scale, scale=np.exp(loc)).cdf + ) return ks < 0.02 @place(config.DEVICES) @parameterize_cls( (TEST_CASE_NAME, 'loc1', 'scale1', 'loc2', 'scale2'), - [('one-dim', xrand((2, )), xrand((2, )), xrand((2, )), xrand((2, ))), - ('multi-dim', xrand((2, 2)), xrand((2, 2)), xrand((2, 2)), xrand((2, 2)))]) + [ + ('one-dim', xrand((2,)), xrand((2,)), xrand((2,)), xrand((2,))), + ( + 'multi-dim', + xrand((2, 2)), + xrand((2, 2)), + xrand((2, 2)), + xrand((2, 2)), + ), + ], +) class TestLogNormalKL(unittest.TestCase): - def setUp(self): paddle.disable_static() - self.ln_a = LogNormal(loc=paddle.to_tensor(self.loc1), - scale=paddle.to_tensor(self.scale1)) - self.ln_b = LogNormal(loc=paddle.to_tensor(self.loc2), - scale=paddle.to_tensor(self.scale2)) - self.normal_a = Normal(loc=paddle.to_tensor(self.loc1), - scale=paddle.to_tensor(self.scale1)) - self.normal_b = Normal(loc=paddle.to_tensor(self.loc2), - scale=paddle.to_tensor(self.scale2)) + self.ln_a = LogNormal( + loc=paddle.to_tensor(self.loc1), scale=paddle.to_tensor(self.scale1) + ) + self.ln_b = LogNormal( + loc=paddle.to_tensor(self.loc2), scale=paddle.to_tensor(self.scale2) + ) + self.normal_a = Normal( + loc=paddle.to_tensor(self.loc1), scale=paddle.to_tensor(self.scale1) + ) + self.normal_b = Normal( + loc=paddle.to_tensor(self.loc2), scale=paddle.to_tensor(self.scale2) + ) def test_kl_divergence(self): kl0 = self.ln_a.kl_divergence(self.ln_b) @@ -216,28 +250,34 @@ def test_kl_divergence(self): self.assertEqual(tuple(kl0.shape), self.scale1.shape) self.assertEqual(tuple(kl1.shape), self.scale1.shape) - np.testing.assert_allclose(kl0, - kl_formula, - rtol=config.RTOL.get(str(self.scale1.dtype)), - atol=config.ATOL.get(str(self.scale1.dtype))) - np.testing.assert_allclose(kl1, - kl_formula, - rtol=config.RTOL.get(str(self.scale1.dtype)), - atol=config.ATOL.get(str(self.scale1.dtype))) - np.testing.assert_allclose(kl_normal, - kl_formula, - rtol=config.RTOL.get(str(self.scale1.dtype)), - atol=config.ATOL.get(str(self.scale1.dtype))) + np.testing.assert_allclose( + kl0, + kl_formula, + rtol=config.RTOL.get(str(self.scale1.dtype)), + atol=config.ATOL.get(str(self.scale1.dtype)), + ) + np.testing.assert_allclose( + kl1, + kl_formula, + rtol=config.RTOL.get(str(self.scale1.dtype)), + atol=config.ATOL.get(str(self.scale1.dtype)), + ) + np.testing.assert_allclose( + kl_normal, + kl_formula, + rtol=config.RTOL.get(str(self.scale1.dtype)), + atol=config.ATOL.get(str(self.scale1.dtype)), + ) def _kl(self, dist1, dist2): loc1 = np.array(dist1.loc) loc2 = np.array(dist2.loc) scale1 = np.array(dist1.scale) scale2 = np.array(dist2.scale) - var_ratio = (scale1 / scale2) + var_ratio = scale1 / scale2 var_ratio = var_ratio * var_ratio - t1 = ((loc1 - loc2) / scale2) - t1 = (t1 * t1) + t1 = (loc1 - loc2) / scale2 + t1 = t1 * t1 return 0.5 * (var_ratio + t1 - 1 - np.log(var_ratio)) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal_static.py index 75a9e497f34b7..9e5cab2da3751 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_lognormal_static.py @@ -27,11 +27,14 @@ @place(config.DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'loc', 'scale', 'value'), - [('one-dim', xrand((2, )), xrand((2, )), xrand((2, ))), - ('multi-dim', xrand((3, 3)), xrand((3, 3)), xrand((3, 3)))]) +@parameterize_cls( + (TEST_CASE_NAME, 'loc', 'scale', 'value'), + [ + ('one-dim', xrand((2,)), xrand((2,)), xrand((2,))), + ('multi-dim', xrand((3, 3)), xrand((3, 3)), xrand((3, 3))), + ], +) class TestLogNormal(unittest.TestCase): - def setUp(self): paddle.enable_static() startup_program = paddle.static.Program() @@ -39,10 +42,12 @@ def setUp(self): executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) - value = paddle.static.data('value', self.value.shape, - self.value.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) + value = paddle.static.data( + 'value', self.value.shape, self.value.dtype + ) self.paddle_lognormal = LogNormal(loc=loc, scale=scale) self.np_lognormal = LogNormalNumpy(loc=self.loc, scale=self.scale) mean = self.paddle_lognormal.mean @@ -54,56 +59,71 @@ def setUp(self): self.feeds = {'loc': self.loc, 'scale': self.scale, 'value': self.value} executor.run(startup_program) - [self.mean, self.var, self.entropy, self.probs, - self.log_prob] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [ + self.mean, + self.var, + self.entropy, + self.probs, + self.log_prob, + ] = executor.run(main_program, feed=self.feeds, fetch_list=fetch_list) def test_mean(self): np_mean = self.np_lognormal.mean self.assertEqual(str(self.mean.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.mean, - np_mean, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.mean, + np_mean, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_var(self): np_var = self.np_lognormal.variance self.assertEqual(str(self.var.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.var, - np_var, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.var, + np_var, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_entropy(self): np_entropy = self.np_lognormal.entropy() self.assertEqual( - str(self.entropy.dtype).split('.')[-1], self.scale.dtype) - np.testing.assert_allclose(self.entropy, - np_entropy, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + str(self.entropy.dtype).split('.')[-1], self.scale.dtype + ) + np.testing.assert_allclose( + self.entropy, + np_entropy, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_probs(self): np_probs = self.np_lognormal.probs(self.value) - np.testing.assert_allclose(self.probs, - np_probs, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.probs, + np_probs, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) def test_log_prob(self): np_log_prob = self.np_lognormal.log_prob(self.value) - np.testing.assert_allclose(self.log_prob, - np_log_prob, - rtol=config.RTOL.get(str(self.scale.dtype)), - atol=config.ATOL.get(str(self.scale.dtype))) + np.testing.assert_allclose( + self.log_prob, + np_log_prob, + rtol=config.RTOL.get(str(self.scale.dtype)), + atol=config.ATOL.get(str(self.scale.dtype)), + ) @place(config.DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'loc', 'scale'), [('sample', xrand( - (4, )), xrand((4, ), min=0, max=1))]) +@parameterize_cls( + (TEST_CASE_NAME, 'loc', 'scale'), + [('sample', xrand((4,)), xrand((4,), min=0, max=1))], +) class TestLogNormalSample(unittest.TestCase): - def setUp(self): paddle.enable_static() startup_program = paddle.static.Program() @@ -111,11 +131,12 @@ def setUp(self): executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) n = 100000 - self.sample_shape = (n, ) - self.rsample_shape = (n, ) + self.sample_shape = (n,) + self.rsample_shape = (n,) self.paddle_lognormal = LogNormal(loc=loc, scale=scale) mean = self.paddle_lognormal.mean variance = self.paddle_lognormal.variance @@ -125,10 +146,9 @@ def setUp(self): self.feeds = {'loc': self.loc, 'scale': self.scale} executor.run(startup_program) - [self.mean, self.variance, self.samples, - self.rsamples] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [self.mean, self.variance, self.samples, self.rsamples] = executor.run( + main_program, feed=self.feeds, fetch_list=fetch_list + ) def test_sample(self): samples_mean = self.samples.mean(axis=0) @@ -139,10 +159,9 @@ def test_sample(self): rsamples_mean = self.rsamples.mean(axis=0) rsamples_var = self.rsamples.var(axis=0) np.testing.assert_allclose(rsamples_mean, self.mean, rtol=0.1, atol=0) - np.testing.assert_allclose(rsamples_var, - self.variance, - rtol=0.1, - atol=0) + np.testing.assert_allclose( + rsamples_var, self.variance, rtol=0.1, atol=0 + ) batch_shape = (self.loc + self.scale).shape self.assertEqual(self.samples.shape, self.sample_shape + batch_shape) @@ -150,25 +169,35 @@ def test_sample(self): for i in range(len(self.scale)): self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.samples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.samples[:, i]) + ) self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i]) + ) def _kstest(self, loc, scale, samples): # Uses the Kolmogorov-Smirnov test for goodness of fit. ks, _ = scipy.stats.kstest( - samples, - scipy.stats.lognorm(s=scale, scale=np.exp(loc)).cdf) + samples, scipy.stats.lognorm(s=scale, scale=np.exp(loc)).cdf + ) return ks < 0.02 @place(config.DEVICES) @parameterize_cls( (TEST_CASE_NAME, 'loc1', 'scale1', 'loc2', 'scale2'), - [('one-dim', xrand((2, )), xrand((2, )), xrand((2, )), xrand((2, ))), - ('multi-dim', xrand((2, 2)), xrand((2, 2)), xrand((2, 2)), xrand((2, 2)))]) + [ + ('one-dim', xrand((2,)), xrand((2,)), xrand((2,)), xrand((2,))), + ( + 'multi-dim', + xrand((2, 2)), + xrand((2, 2)), + xrand((2, 2)), + xrand((2, 2)), + ), + ], +) class TestLogNormalKL(unittest.TestCase): - def setUp(self): paddle.enable_static() startup_program = paddle.static.Program() @@ -176,11 +205,13 @@ def setUp(self): executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): loc1 = paddle.static.data('loc1', self.loc1.shape, self.loc1.dtype) - scale1 = paddle.static.data('scale1', self.scale1.shape, - self.scale1.dtype) + scale1 = paddle.static.data( + 'scale1', self.scale1.shape, self.scale1.dtype + ) loc2 = paddle.static.data('loc2', self.loc2.shape, self.loc2.dtype) - scale2 = paddle.static.data('scale2', self.scale2.shape, - self.scale2.dtype) + scale2 = paddle.static.data( + 'scale2', self.scale2.shape, self.scale2.dtype + ) self.ln_a = LogNormal(loc=loc1, scale=scale1) self.ln_b = LogNormal(loc=loc2, scale=scale2) @@ -197,40 +228,45 @@ def setUp(self): 'loc1': self.loc1, 'scale1': self.scale1, 'loc2': self.loc2, - 'scale2': self.scale2 + 'scale2': self.scale2, } executor.run(startup_program) - [self.kl0, self.kl1, self.kl_normal, - self.kl_formula] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [self.kl0, self.kl1, self.kl_normal, self.kl_formula] = executor.run( + main_program, feed=self.feeds, fetch_list=fetch_list + ) def test_kl_divergence(self): - np.testing.assert_allclose(self.kl0, - self.kl_formula, - rtol=config.RTOL.get(str(self.scale1.dtype)), - atol=config.ATOL.get(str(self.scale1.dtype))) - - np.testing.assert_allclose(self.kl1, - self.kl_formula, - rtol=config.RTOL.get(str(self.scale1.dtype)), - atol=config.ATOL.get(str(self.scale1.dtype))) - - np.testing.assert_allclose(self.kl_normal, - self.kl_formula, - rtol=config.RTOL.get(str(self.scale1.dtype)), - atol=config.ATOL.get(str(self.scale1.dtype))) + np.testing.assert_allclose( + self.kl0, + self.kl_formula, + rtol=config.RTOL.get(str(self.scale1.dtype)), + atol=config.ATOL.get(str(self.scale1.dtype)), + ) + + np.testing.assert_allclose( + self.kl1, + self.kl_formula, + rtol=config.RTOL.get(str(self.scale1.dtype)), + atol=config.ATOL.get(str(self.scale1.dtype)), + ) + + np.testing.assert_allclose( + self.kl_normal, + self.kl_formula, + rtol=config.RTOL.get(str(self.scale1.dtype)), + atol=config.ATOL.get(str(self.scale1.dtype)), + ) def _kl(self, dist1, dist2): loc1 = dist1.loc loc2 = dist2.loc - scale1 = (dist1.scale) - scale2 = (dist2.scale) - var_ratio = (scale1 / scale2) + scale1 = dist1.scale + scale2 = dist2.scale + var_ratio = scale1 / scale2 var_ratio = var_ratio * var_ratio - t1 = ((loc1 - loc2) / scale2) - t1 = (t1 * t1) + t1 = (loc1 - loc2) / scale2 + t1 = t1 * t1 return 0.5 * (var_ratio + t1 - 1 - np.log(var_ratio)) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial.py index 0bec1c5a58cd9..2c18469936cb5 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial.py @@ -24,41 +24,49 @@ @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'total_count', 'probs'), [ - ('one-dim', 10, parameterize.xrand((3, ))), + (parameterize.TEST_CASE_NAME, 'total_count', 'probs'), + [ + ('one-dim', 10, parameterize.xrand((3,))), ('multi-dim', 9, parameterize.xrand((10, 20))), ('prob-sum-one', 10, np.array([0.5, 0.2, 0.3])), - ('prob-sum-non-one', 10, np.array([2., 3., 5.])), - ]) + ('prob-sum-non-one', 10, np.array([2.0, 3.0, 5.0])), + ], +) class TestMultinomial(unittest.TestCase): - def setUp(self): self._dist = paddle.distribution.Multinomial( - total_count=self.total_count, probs=paddle.to_tensor(self.probs)) + total_count=self.total_count, probs=paddle.to_tensor(self.probs) + ) def test_mean(self): mean = self._dist.mean self.assertEqual(mean.numpy().dtype, self.probs.dtype) - np.testing.assert_allclose(mean, - self._np_mean(), - rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + np.testing.assert_allclose( + mean, + self._np_mean(), + rtol=config.RTOL.get(str(self.probs.dtype)), + atol=config.ATOL.get(str(self.probs.dtype)), + ) def test_variance(self): var = self._dist.variance self.assertEqual(var.numpy().dtype, self.probs.dtype) - np.testing.assert_allclose(var, - self._np_variance(), - rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + np.testing.assert_allclose( + var, + self._np_variance(), + rtol=config.RTOL.get(str(self.probs.dtype)), + atol=config.ATOL.get(str(self.probs.dtype)), + ) def test_entropy(self): entropy = self._dist.entropy() self.assertEqual(entropy.numpy().dtype, self.probs.dtype) - np.testing.assert_allclose(entropy, - self._np_entropy(), - rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + np.testing.assert_allclose( + entropy, + self._np_entropy(), + rtol=config.RTOL.get(str(self.probs.dtype)), + atol=config.ATOL.get(str(self.probs.dtype)), + ) def test_sample(self): sample_shape = () @@ -66,26 +74,28 @@ def test_sample(self): self.assertEqual(samples.numpy().dtype, self.probs.dtype) self.assertEqual( tuple(samples.shape), - sample_shape + self._dist.batch_shape + self._dist.event_shape) + sample_shape + self._dist.batch_shape + self._dist.event_shape, + ) - sample_shape = (6, ) + sample_shape = (6,) samples = self._dist.sample(sample_shape) self.assertEqual(samples.numpy().dtype, self.probs.dtype) self.assertEqual( tuple(samples.shape), - sample_shape + self._dist.batch_shape + self._dist.event_shape) + sample_shape + self._dist.batch_shape + self._dist.event_shape, + ) self.assertTrue( - np.all(samples.sum(-1).numpy() == self._dist.total_count)) + np.all(samples.sum(-1).numpy() == self._dist.total_count) + ) - sample_shape = (5000, ) + sample_shape = (5000,) samples = self._dist.sample(sample_shape) sample_mean = samples.mean(axis=0) # Tolerance value 0.2 is empirical value which is consistent with # TensorFlow - np.testing.assert_allclose(sample_mean, - self._dist.mean, - atol=0, - rtol=0.20) + np.testing.assert_allclose( + sample_mean, self._dist.mean, atol=0, rtol=0.20 + ) def _np_variance(self): probs = self.probs / self.probs.sum(-1, keepdims=True) @@ -104,40 +114,54 @@ def _np_entropy(self): @parameterize.parameterize_cls( (parameterize.TEST_CASE_NAME, 'total_count', 'probs', 'value'), [ - ('value-float', 10, np.array([0.2, 0.3, 0.5]), np.array([2., 3., 5.])), + ( + 'value-float', + 10, + np.array([0.2, 0.3, 0.5]), + np.array([2.0, 3.0, 5.0]), + ), ('value-int', 10, np.array([0.2, 0.3, 0.5]), np.array([2, 3, 5])), - ('value-multi-dim', 10, np.array([[0.3, 0.7], [0.5, 0.5] - ]), np.array([[4., 6], [8, 2]])), + ( + 'value-multi-dim', + 10, + np.array([[0.3, 0.7], [0.5, 0.5]]), + np.array([[4.0, 6], [8, 2]]), + ), # ('value-sum-non-n', 10, np.array([0.5, 0.2, 0.3]), np.array([4,5,2])), - ]) + ], +) class TestMultinomialPmf(unittest.TestCase): - def setUp(self): self._dist = paddle.distribution.Multinomial( - total_count=self.total_count, probs=paddle.to_tensor(self.probs)) + total_count=self.total_count, probs=paddle.to_tensor(self.probs) + ) def test_prob(self): np.testing.assert_allclose( self._dist.prob(paddle.to_tensor(self.value)), - scipy.stats.multinomial.pmf(self.value, self.total_count, - self.probs), + scipy.stats.multinomial.pmf( + self.value, self.total_count, self.probs + ), rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + atol=config.ATOL.get(str(self.probs.dtype)), + ) @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (config.TEST_CASE_NAME, 'total_count', 'probs'), [ + (config.TEST_CASE_NAME, 'total_count', 'probs'), + [ ('total_count_le_one', 0, np.array([0.3, 0.7])), ('total_count_float', np.array([0.3, 0.7])), ('probs_zero_dim', np.array(0)), - ]) + ], +) class TestMultinomialException(unittest.TestCase): - def TestInit(self): with self.assertRaises(ValueError): - paddle.distribution.Multinomial(self.total_count, - paddle.to_tensor(self.probs)) + paddle.distribution.Multinomial( + self.total_count, paddle.to_tensor(self.probs) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial_static.py index 56341d7fc0ef8..90efb815dea59 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_multinomial_static.py @@ -26,61 +26,74 @@ @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'total_count', 'probs'), [ - ('one-dim', 5, parameterize.xrand((3, ))), + (parameterize.TEST_CASE_NAME, 'total_count', 'probs'), + [ + ('one-dim', 5, parameterize.xrand((3,))), ('multi-dim', 9, parameterize.xrand((2, 3))), ('prob-sum-one', 5, np.array([0.5, 0.2, 0.3])), - ('prob-sum-non-one', 5, np.array([2., 3., 5.])), - ]) + ('prob-sum-non-one', 5, np.array([2.0, 3.0, 5.0])), + ], +) class TestMultinomial(unittest.TestCase): - def setUp(self): startup_program = paddle.static.Program() main_program = paddle.static.Program() executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): - probs = paddle.static.data('probs', self.probs.shape, - self.probs.dtype) + probs = paddle.static.data( + 'probs', self.probs.shape, self.probs.dtype + ) dist = paddle.distribution.Multinomial(self.total_count, probs) mean = dist.mean var = dist.variance entropy = dist.entropy() - mini_samples = dist.sample(shape=(6, )) - large_samples = dist.sample(shape=(5000, )) + mini_samples = dist.sample(shape=(6,)) + large_samples = dist.sample(shape=(5000,)) fetch_list = [mean, var, entropy, mini_samples, large_samples] feed = {'probs': self.probs} executor.run(startup_program) [ - self.mean, self.var, self.entropy, self.mini_samples, - self.large_samples + self.mean, + self.var, + self.entropy, + self.mini_samples, + self.large_samples, ] = executor.run(main_program, feed=feed, fetch_list=fetch_list) def test_mean(self): self.assertEqual(str(self.mean.dtype).split('.')[-1], self.probs.dtype) - np.testing.assert_allclose(self.mean, - self._np_mean(), - rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + np.testing.assert_allclose( + self.mean, + self._np_mean(), + rtol=config.RTOL.get(str(self.probs.dtype)), + atol=config.ATOL.get(str(self.probs.dtype)), + ) def test_variance(self): self.assertEqual(str(self.var.dtype).split('.')[-1], self.probs.dtype) - np.testing.assert_allclose(self.var, - self._np_variance(), - rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + np.testing.assert_allclose( + self.var, + self._np_variance(), + rtol=config.RTOL.get(str(self.probs.dtype)), + atol=config.ATOL.get(str(self.probs.dtype)), + ) def test_entropy(self): self.assertEqual( - str(self.entropy.dtype).split('.')[-1], self.probs.dtype) - np.testing.assert_allclose(self.entropy, - self._np_entropy(), - rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + str(self.entropy.dtype).split('.')[-1], self.probs.dtype + ) + np.testing.assert_allclose( + self.entropy, + self._np_entropy(), + rtol=config.RTOL.get(str(self.probs.dtype)), + atol=config.ATOL.get(str(self.probs.dtype)), + ) def test_sample(self): self.assertEqual( - str(self.mini_samples.dtype).split('.')[-1], self.probs.dtype) + str(self.mini_samples.dtype).split('.')[-1], self.probs.dtype + ) self.assertTrue(np.all(self.mini_samples.sum(-1) == self.total_count)) sample_mean = self.large_samples.mean(axis=0) @@ -103,60 +116,75 @@ def _np_entropy(self): @parameterize.parameterize_cls( (parameterize.TEST_CASE_NAME, 'total_count', 'probs', 'value'), [ - ('value-float', 5, np.array([0.2, 0.3, 0.5]), np.array([1., 1., 3.])), + ( + 'value-float', + 5, + np.array([0.2, 0.3, 0.5]), + np.array([1.0, 1.0, 3.0]), + ), ('value-int', 5, np.array([0.2, 0.3, 0.5]), np.array([2, 2, 1])), - ('value-multi-dim', 5, np.array([[0.3, 0.7], [0.5, 0.5] - ]), np.array([[1., 4.], [2., 3.]])), + ( + 'value-multi-dim', + 5, + np.array([[0.3, 0.7], [0.5, 0.5]]), + np.array([[1.0, 4.0], [2.0, 3.0]]), + ), # ('value-sum-non-n', 10, np.array([0.5, 0.2, 0.3]), np.array([4,5,2])), - ]) + ], +) class TestMultinomialPmf(unittest.TestCase): - def setUp(self): startup_program = paddle.static.Program() main_program = paddle.static.Program() executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): - probs = paddle.static.data('probs', self.probs.shape, - self.probs.dtype) - value = paddle.static.data('value', self.value.shape, - self.value.dtype) + probs = paddle.static.data( + 'probs', self.probs.shape, self.probs.dtype + ) + value = paddle.static.data( + 'value', self.value.shape, self.value.dtype + ) dist = paddle.distribution.Multinomial(self.total_count, probs) pmf = dist.prob(value) feed = {'probs': self.probs, 'value': self.value} fetch_list = [pmf] executor.run(startup_program) - [self.pmf] = executor.run(main_program, - feed=feed, - fetch_list=fetch_list) + [self.pmf] = executor.run( + main_program, feed=feed, fetch_list=fetch_list + ) def test_prob(self): - np.testing.assert_allclose(self.pmf, - scipy.stats.multinomial.pmf( - self.value, self.total_count, - self.probs), - rtol=config.RTOL.get(str(self.probs.dtype)), - atol=config.ATOL.get(str(self.probs.dtype))) + np.testing.assert_allclose( + self.pmf, + scipy.stats.multinomial.pmf( + self.value, self.total_count, self.probs + ), + rtol=config.RTOL.get(str(self.probs.dtype)), + atol=config.ATOL.get(str(self.probs.dtype)), + ) @parameterize.place(config.DEVICES) @parameterize.parameterize_cls( - (parameterize.TEST_CASE_NAME, 'total_count', 'probs'), [ + (parameterize.TEST_CASE_NAME, 'total_count', 'probs'), + [ ('total_count_le_one', 0, np.array([0.3, 0.7])), ('total_count_float', np.array([0.3, 0.7])), ('probs_zero_dim', np.array(0)), - ]) + ], +) class TestMultinomialException(unittest.TestCase): - def setUp(self): startup_program = paddle.static.Program() self.main_program = paddle.static.Program() self.executor = paddle.static.Executor(self.place) with paddle.static.program_guard(self.main_program, startup_program): - probs = paddle.static.data('probs', self.probs.shape, - self.probs.dtype) + probs = paddle.static.data( + 'probs', self.probs.shape, self.probs.dtype + ) dist = paddle.distribution.Multinomial(self.total_count, probs) self.feed = {'probs': self.probs} diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_normal.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_normal.py index d90405599554d..3f75a8c78c2ce 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_normal.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_normal.py @@ -29,7 +29,6 @@ class NormalNumpy(DistributionNumpy): - def __init__(self, loc, scale): self.loc = np.array(loc) self.scale = np.array(scale) @@ -44,29 +43,34 @@ def sample(self, shape): def log_prob(self, value): var = self.scale * self.scale log_scale = np.log(self.scale) - return -((value - self.loc) * - (value - self.loc)) / (2. * var) - log_scale - math.log( - math.sqrt(2. * math.pi)) + return ( + -((value - self.loc) * (value - self.loc)) / (2.0 * var) + - log_scale + - math.log(math.sqrt(2.0 * math.pi)) + ) def probs(self, value): var = self.scale * self.scale - return np.exp(-1. * ((value - self.loc) * (value - self.loc)) / - (2. * var)) / (math.sqrt(2 * math.pi) * self.scale) + return np.exp( + -1.0 * ((value - self.loc) * (value - self.loc)) / (2.0 * var) + ) / (math.sqrt(2 * math.pi) * self.scale) def entropy(self): - return 0.5 + 0.5 * np.log( - np.array(2. * math.pi).astype(self.loc.dtype)) + np.log(self.scale) + return ( + 0.5 + + 0.5 * np.log(np.array(2.0 * math.pi).astype(self.loc.dtype)) + + np.log(self.scale) + ) def kl_divergence(self, other): - var_ratio = (self.scale / other.scale) + var_ratio = self.scale / other.scale var_ratio = var_ratio * var_ratio - t1 = ((self.loc - other.loc) / other.scale) - t1 = (t1 * t1) + t1 = (self.loc - other.loc) / other.scale + t1 = t1 * t1 return 0.5 * (var_ratio + t1 - 1 - np.log(var_ratio)) class NormalTest(unittest.TestCase): - def setUp(self, use_gpu=False, batch_size=2, dims=3): self.use_gpu = use_gpu if not use_gpu: @@ -112,9 +116,9 @@ def init_static_data(self, batch_size, dims): self.static_other_loc = self.other_loc_np self.static_other_scale = self.other_scale_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[], dtype='float32' + ) def compare_with_numpy(self, fetch_list, sample_shape=7, tolerance=1e-6): sample, entropy, log_prob, probs, kl = fetch_list @@ -134,22 +138,18 @@ def compare_with_numpy(self, fetch_list, sample_shape=7, tolerance=1e-6): # So set the tolerance from 1e-6 to 1e-4. log_tolerance = 1e-4 np.testing.assert_equal(sample.shape, np_sample.shape) - np.testing.assert_allclose(entropy, - np_entropy, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(log_prob, - np_lp, - rtol=log_tolerance, - atol=log_tolerance) - np.testing.assert_allclose(probs, - np_p, - rtol=log_tolerance, - atol=log_tolerance) - np.testing.assert_allclose(kl, - np_kl, - rtol=log_tolerance, - atol=log_tolerance) + np.testing.assert_allclose( + entropy, np_entropy, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + log_prob, np_lp, rtol=log_tolerance, atol=log_tolerance + ) + np.testing.assert_allclose( + probs, np_p, rtol=log_tolerance, atol=log_tolerance + ) + np.testing.assert_allclose( + kl, np_kl, rtol=log_tolerance, atol=log_tolerance + ) def test_normal_distribution_dygraph(self, sample_shape=7, tolerance=1e-6): paddle.disable_static(self.place) @@ -174,8 +174,9 @@ def test_normal_distribution_static(self, sample_shape=7, tolerance=1e-6): entropy = normal.entropy() log_prob = normal.log_prob(self.static_values) probs = normal.probs(self.static_values) - other_normal = Normal(self.static_other_loc, - self.static_other_scale) + other_normal = Normal( + self.static_other_loc, self.static_other_scale + ) kl = normal.kl_divergence(other_normal) fetch_list = [sample, entropy, log_prob, probs, kl] @@ -185,19 +186,18 @@ def test_normal_distribution_static(self, sample_shape=7, tolerance=1e-6): 'scale': self.scale_np, 'values': self.values_np, 'other_loc': self.other_loc_np, - 'other_scale': self.other_scale_np + 'other_scale': self.other_scale_np, } self.executor.run(fluid.default_startup_program()) - fetch_list = self.executor.run(program=self.test_program, - feed=feed_vars, - fetch_list=fetch_list) + fetch_list = self.executor.run( + program=self.test_program, feed=feed_vars, fetch_list=fetch_list + ) self.compare_with_numpy(fetch_list) class NormalTest2(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc ans scale are 'int' self.loc_np = int((np.random.ranf() - 0.5) * 8) @@ -213,7 +213,6 @@ def init_numpy_data(self, batch_size, dims): class NormalTest3(NormalTest): - def init_numpy_data(self, batch_size, dims): # test broadcast: loc is float, scale is numpy.ndarray with dtype 'float32'. self.loc_np = (np.random.ranf() - 0.5) * 4 @@ -223,11 +222,13 @@ def init_numpy_data(self, batch_size, dims): self.values_np = np.random.randn(batch_size, dims).astype('float32') # used to construct another Normal object to calculate kl_divergence self.other_loc_np = (np.random.ranf() - 0.5) * 4 - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) while not np.all(self.scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) def init_static_data(self, batch_size, dims): self.static_loc = self.loc_np @@ -235,13 +236,12 @@ def init_static_data(self, batch_size, dims): self.static_other_loc = self.other_loc_np self.static_other_scale = self.other_scale_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class NormalTest4(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc and scale are numpy.ndarray with dtype 'float32'. self.loc_np = np.random.randn(batch_size, dims).astype('float32') @@ -251,11 +251,13 @@ def init_numpy_data(self, batch_size, dims): self.values_np = np.random.randn(batch_size, dims).astype('float32') # used to construct another Normal object to calculate kl_divergence self.other_loc_np = np.random.randn(batch_size, dims).astype('float32') - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) while not np.all(self.scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) def init_static_data(self, batch_size, dims): self.static_loc = self.loc_np @@ -263,13 +265,12 @@ def init_static_data(self, batch_size, dims): self.static_other_loc = self.other_loc_np self.static_other_scale = self.other_scale_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class NormalTest5(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc and scale are numpy.ndarray with dtype 'float64'. self.loc_np = np.random.randn(batch_size, dims).astype('float64') @@ -279,11 +280,13 @@ def init_numpy_data(self, batch_size, dims): self.values_np = np.random.randn(batch_size, dims).astype('float64') # used to construct another Normal object to calculate kl_divergence self.other_loc_np = np.random.randn(batch_size, dims).astype('float64') - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float64') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float64' + ) while not np.all(self.scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float64') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float64' + ) def init_dynamic_data(self, batch_size, dims): self.dynamic_loc = self.loc_np @@ -298,13 +301,12 @@ def init_static_data(self, batch_size, dims): self.static_other_loc = self.other_loc_np self.static_other_scale = self.other_scale_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float64') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float64' + ) class NormalTest6(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc and scale are Tensor with dtype 'VarType.FP32'. self.loc_np = np.random.randn(batch_size, dims).astype('float32') @@ -314,11 +316,13 @@ def init_numpy_data(self, batch_size, dims): self.values_np = np.random.randn(batch_size, dims).astype('float32') # used to construct another Normal object to calculate kl_divergence self.other_loc_np = np.random.randn(batch_size, dims).astype('float32') - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) while not np.all(self.scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) def init_dynamic_data(self, batch_size, dims): self.dynamic_loc = paddle.to_tensor(self.loc_np) @@ -329,25 +333,24 @@ def init_dynamic_data(self, batch_size, dims): def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.static_loc = layers.data(name='loc', - shape=[dims], - dtype='float32') - self.static_scale = layers.data(name='scale', - shape=[dims], - dtype='float32') - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') - self.static_other_loc = layers.data(name='other_loc', - shape=[dims], - dtype='float32') - self.static_other_scale = layers.data(name='other_scale', - shape=[dims], - dtype='float32') + self.static_loc = layers.data( + name='loc', shape=[dims], dtype='float32' + ) + self.static_scale = layers.data( + name='scale', shape=[dims], dtype='float32' + ) + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) + self.static_other_loc = layers.data( + name='other_loc', shape=[dims], dtype='float32' + ) + self.static_other_scale = layers.data( + name='other_scale', shape=[dims], dtype='float32' + ) class NormalTest7(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc and scale are Tensor with dtype 'VarType.FP64'. self.loc_np = np.random.randn(batch_size, dims).astype('float64') @@ -357,42 +360,45 @@ def init_numpy_data(self, batch_size, dims): self.values_np = np.random.randn(batch_size, dims).astype('float64') # used to construct another Normal object to calculate kl_divergence self.other_loc_np = np.random.randn(batch_size, dims).astype('float64') - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float64') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float64' + ) while not np.all(self.scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float64') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float64' + ) def init_dynamic_data(self, batch_size, dims): self.dynamic_loc = paddle.to_tensor(self.loc_np, dtype='float64') self.dynamic_scale = paddle.to_tensor(self.scale_np, dtype='float64') self.dynamic_values = paddle.to_tensor(self.values_np, dtype='float64') - self.dynamic_other_loc = paddle.to_tensor(self.other_loc_np, - dtype='float64') - self.dynamic_other_scale = paddle.to_tensor(self.other_scale_np, - dtype='float64') + self.dynamic_other_loc = paddle.to_tensor( + self.other_loc_np, dtype='float64' + ) + self.dynamic_other_scale = paddle.to_tensor( + self.other_scale_np, dtype='float64' + ) def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.static_loc = layers.data(name='loc', - shape=[dims], - dtype='float64') - self.static_scale = layers.data(name='scale', - shape=[dims], - dtype='float64') - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float64') - self.static_other_loc = layers.data(name='other_loc', - shape=[dims], - dtype='float64') - self.static_other_scale = layers.data(name='other_scale', - shape=[dims], - dtype='float64') + self.static_loc = layers.data( + name='loc', shape=[dims], dtype='float64' + ) + self.static_scale = layers.data( + name='scale', shape=[dims], dtype='float64' + ) + self.static_values = layers.data( + name='values', shape=[dims], dtype='float64' + ) + self.static_other_loc = layers.data( + name='other_loc', shape=[dims], dtype='float64' + ) + self.static_other_scale = layers.data( + name='other_scale', shape=[dims], dtype='float64' + ) class NormalTest8(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc and scale are Tensor with dtype 'VarType.FP64'. value's dtype is 'VarType.FP32'. self.loc_np = np.random.randn(batch_size, dims).astype('float64') @@ -402,59 +408,66 @@ def init_numpy_data(self, batch_size, dims): self.values_np = np.random.randn(batch_size, dims).astype('float32') # used to construct another Normal object to calculate kl_divergence self.other_loc_np = np.random.randn(batch_size, dims).astype('float64') - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float64') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float64' + ) while not np.all(self.scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float64') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float64' + ) def init_dynamic_data(self, batch_size, dims): self.dynamic_loc = paddle.to_tensor(self.loc_np, dtype='float64') self.dynamic_scale = paddle.to_tensor(self.scale_np, dtype='float64') self.dynamic_values = paddle.to_tensor(self.values_np) - self.dynamic_other_loc = paddle.to_tensor(self.other_loc_np, - dtype='float64') - self.dynamic_other_scale = paddle.to_tensor(self.other_scale_np, - dtype='float64') + self.dynamic_other_loc = paddle.to_tensor( + self.other_loc_np, dtype='float64' + ) + self.dynamic_other_scale = paddle.to_tensor( + self.other_scale_np, dtype='float64' + ) def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.static_loc = layers.data(name='loc', - shape=[dims], - dtype='float64') - self.static_scale = layers.data(name='scale', - shape=[dims], - dtype='float64') - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') - self.static_other_loc = layers.data(name='other_loc', - shape=[dims], - dtype='float64') - self.static_other_scale = layers.data(name='other_scale', - shape=[dims], - dtype='float64') + self.static_loc = layers.data( + name='loc', shape=[dims], dtype='float64' + ) + self.static_scale = layers.data( + name='scale', shape=[dims], dtype='float64' + ) + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) + self.static_other_loc = layers.data( + name='other_loc', shape=[dims], dtype='float64' + ) + self.static_other_scale = layers.data( + name='other_scale', shape=[dims], dtype='float64' + ) class NormalTest9(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc and scale are list. - self.loc_np = np.random.randn(batch_size, - dims).astype('float32').tolist() + self.loc_np = ( + np.random.randn(batch_size, dims).astype('float32').tolist() + ) self.scale_np = np.random.randn(batch_size, dims).astype('float32') while not np.all(self.scale_np > 0): self.scale_np = np.random.randn(batch_size, dims).astype('float32') self.scale_np = self.scale_np.tolist() self.values_np = np.random.randn(batch_size, dims).astype('float32') # used to construct another Normal object to calculate kl_divergence - self.other_loc_np = np.random.randn(batch_size, - dims).astype('float32').tolist() - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_loc_np = ( + np.random.randn(batch_size, dims).astype('float32').tolist() + ) + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) while not np.all(self.other_scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) self.other_scale_np = self.other_scale_np.tolist() def init_static_data(self, batch_size, dims): @@ -463,17 +476,17 @@ def init_static_data(self, batch_size, dims): self.static_other_loc = self.other_loc_np self.static_other_scale = self.other_scale_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class NormalTest10(NormalTest): - def init_numpy_data(self, batch_size, dims): # loc and scale are tuple. self.loc_np = tuple( - np.random.randn(batch_size, dims).astype('float32').tolist()) + np.random.randn(batch_size, dims).astype('float32').tolist() + ) self.scale_np = np.random.randn(batch_size, dims).astype('float32') while not np.all(self.scale_np > 0): self.scale_np = np.random.randn(batch_size, dims).astype('float32') @@ -481,12 +494,15 @@ def init_numpy_data(self, batch_size, dims): self.values_np = np.random.randn(batch_size, dims).astype('float32') # used to construct another Normal object to calculate kl_divergence self.other_loc_np = tuple( - np.random.randn(batch_size, dims).astype('float32').tolist()) - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + np.random.randn(batch_size, dims).astype('float32').tolist() + ) + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) while not np.all(self.other_scale_np > 0): - self.other_scale_np = np.random.randn(batch_size, - dims).astype('float32') + self.other_scale_np = np.random.randn(batch_size, dims).astype( + 'float32' + ) self.other_scale_np = tuple(self.other_scale_np.tolist()) def init_static_data(self, batch_size, dims): @@ -495,72 +511,73 @@ def init_static_data(self, batch_size, dims): self.static_other_loc = self.other_loc_np self.static_other_scale = self.other_scale_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) @place(config.DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'loc', 'scale'), [('sample', xrand( - (4, )), xrand((4, )))]) +@parameterize_cls( + (TEST_CASE_NAME, 'loc', 'scale'), [('sample', xrand((4,)), xrand((4,)))] +) class TestNormalSampleDygraph(unittest.TestCase): - def setUp(self): paddle.disable_static() self.paddle_normal = Normal(loc=self.loc, scale=self.scale) n = 100000 - self.sample_shape = (n, ) - self.rsample_shape = (n, ) + self.sample_shape = (n,) + self.rsample_shape = (n,) self.samples = self.paddle_normal.sample(self.sample_shape) self.rsamples = self.paddle_normal.rsample(self.rsample_shape) def test_sample(self): samples_mean = self.samples.mean(axis=0) samples_var = self.samples.var(axis=0) - np.testing.assert_allclose(samples_mean, - self.paddle_normal.mean, - rtol=0.1, - atol=0) - np.testing.assert_allclose(samples_var, - self.paddle_normal.variance, - rtol=0.1, - atol=0) + np.testing.assert_allclose( + samples_mean, self.paddle_normal.mean, rtol=0.1, atol=0 + ) + np.testing.assert_allclose( + samples_var, self.paddle_normal.variance, rtol=0.1, atol=0 + ) rsamples_mean = self.rsamples.mean(axis=0) rsamples_var = self.rsamples.var(axis=0) - np.testing.assert_allclose(rsamples_mean, - self.paddle_normal.mean, - rtol=0.1, - atol=0) - np.testing.assert_allclose(rsamples_var, - self.paddle_normal.variance, - rtol=0.1, - atol=0) + np.testing.assert_allclose( + rsamples_mean, self.paddle_normal.mean, rtol=0.1, atol=0 + ) + np.testing.assert_allclose( + rsamples_var, self.paddle_normal.variance, rtol=0.1, atol=0 + ) batch_shape = (self.loc + self.scale).shape - self.assertEqual(self.samples.shape, - list(self.sample_shape + batch_shape)) - self.assertEqual(self.rsamples.shape, - list(self.rsample_shape + batch_shape)) + self.assertEqual( + self.samples.shape, list(self.sample_shape + batch_shape) + ) + self.assertEqual( + self.rsamples.shape, list(self.rsample_shape + batch_shape) + ) for i in range(len(self.scale)): self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.samples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.samples[:, i]) + ) self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i]) + ) def _kstest(self, loc, scale, samples): # Uses the Kolmogorov-Smirnov test for goodness of fit. - ks, _ = scipy.stats.kstest(samples, - scipy.stats.norm(loc=loc, scale=scale).cdf) + ks, _ = scipy.stats.kstest( + samples, scipy.stats.norm(loc=loc, scale=scale).cdf + ) return ks < 0.02 @place(config.DEVICES) -@parameterize_cls((TEST_CASE_NAME, 'loc', 'scale'), [('sample', xrand( - (4, )), xrand((4, )))]) +@parameterize_cls( + (TEST_CASE_NAME, 'loc', 'scale'), [('sample', xrand((4,)), xrand((4,)))] +) class TestNormalSampleStaic(unittest.TestCase): - def setUp(self): paddle.enable_static() startup_program = paddle.static.Program() @@ -568,11 +585,12 @@ def setUp(self): executor = paddle.static.Executor(self.place) with paddle.static.program_guard(main_program, startup_program): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) n = 100000 - self.sample_shape = (n, ) - self.rsample_shape = (n, ) + self.sample_shape = (n,) + self.rsample_shape = (n,) self.paddle_normal = Normal(loc=loc, scale=scale) mean = self.paddle_normal.mean variance = self.paddle_normal.variance @@ -582,10 +600,9 @@ def setUp(self): self.feeds = {'loc': self.loc, 'scale': self.scale} executor.run(startup_program) - [self.mean, self.variance, self.samples, - self.rsamples] = executor.run(main_program, - feed=self.feeds, - fetch_list=fetch_list) + [self.mean, self.variance, self.samples, self.rsamples] = executor.run( + main_program, feed=self.feeds, fetch_list=fetch_list + ) def test_sample(self): samples_mean = self.samples.mean(axis=0) @@ -596,10 +613,9 @@ def test_sample(self): rsamples_mean = self.rsamples.mean(axis=0) rsamples_var = self.rsamples.var(axis=0) np.testing.assert_allclose(rsamples_mean, self.mean, rtol=0.1, atol=0) - np.testing.assert_allclose(rsamples_var, - self.variance, - rtol=0.1, - atol=0) + np.testing.assert_allclose( + rsamples_var, self.variance, rtol=0.1, atol=0 + ) batch_shape = (self.loc + self.scale).shape self.assertEqual(self.samples.shape, self.sample_shape + batch_shape) @@ -607,14 +623,17 @@ def test_sample(self): for i in range(len(self.scale)): self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.samples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.samples[:, i]) + ) self.assertTrue( - self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i])) + self._kstest(self.loc[i], self.scale[i], self.rsamples[:, i]) + ) def _kstest(self, loc, scale, samples): # Uses the Kolmogorov-Smirnov test for goodness of fit. - ks, _ = scipy.stats.kstest(samples, - scipy.stats.norm(loc=loc, scale=scale).cdf) + ks, _ = scipy.stats.kstest( + samples, scipy.stats.norm(loc=loc, scale=scale).cdf + ) return ks < 0.02 diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform.py index 4f997d40bb094..f3f89068b3510 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform.py @@ -28,22 +28,33 @@ @param.place(config.DEVICES) class TestTransform(unittest.TestCase): - def setUp(self): self._t = transform.Transform() - @param.param_func([(paddle.distribution.Distribution(), - paddle.distribution.TransformedDistribution), - (paddle.distribution.ExpTransform(), - paddle.distribution.ChainTransform)]) + @param.param_func( + [ + ( + paddle.distribution.Distribution(), + paddle.distribution.TransformedDistribution, + ), + ( + paddle.distribution.ExpTransform(), + paddle.distribution.ChainTransform, + ), + ] + ) def test_call(self, input, expected_type): t = transform.Transform() self.assertIsInstance(t(input), expected_type) - @param.param_func([(transform.Type.BIJECTION, True), - (transform.Type.INJECTION, True), - (transform.Type.SURJECTION, False), - (transform.Type.OTHER, False)]) + @param.param_func( + [ + (transform.Type.BIJECTION, True), + (transform.Type.INJECTION, True), + (transform.Type.SURJECTION, False), + (transform.Type.OTHER, False), + ] + ) def test_is_injective(self, type, expected): transform.Transform._type = type self.assertEqual(self._t._is_injective(), expected) @@ -54,26 +65,30 @@ def test_domain(self): def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Real)) - @param.param_func([(0, TypeError), (paddle.rand( - (2, 3)), NotImplementedError)]) + @param.param_func( + [(0, TypeError), (paddle.rand((2, 3)), NotImplementedError)] + ) def test_forward(self, input, expected): with self.assertRaises(expected): self._t.forward(input) - @param.param_func([(0, TypeError), (paddle.rand( - (2, 3)), NotImplementedError)]) + @param.param_func( + [(0, TypeError), (paddle.rand((2, 3)), NotImplementedError)] + ) def test_inverse(self, input, expected): with self.assertRaises(expected): self._t.inverse(input) - @param.param_func([(0, TypeError), (paddle.rand( - (2, 3)), NotImplementedError)]) + @param.param_func( + [(0, TypeError), (paddle.rand((2, 3)), NotImplementedError)] + ) def test_forward_log_det_jacobian(self, input, expected): with self.assertRaises(expected): self._t.forward_log_det_jacobian(input) - @param.param_func([(0, TypeError), (paddle.rand( - (2, 3)), NotImplementedError)]) + @param.param_func( + [(0, TypeError), (paddle.rand((2, 3)), NotImplementedError)] + ) def test_inverse_log_det_jacobian(self, input, expected): with self.assertRaises(expected): self._t.inverse_log_det_jacobian(input) @@ -91,7 +106,6 @@ def test_inverse_shape(self, shape, expected): @param.place(config.DEVICES) class TestAbsTransform(unittest.TestCase): - def setUp(self): self._t = transform.AbsTransform() @@ -108,48 +122,66 @@ def test_codomain(self): self.assertEqual(self._t._codomain.event_rank, 0) self.assertEqual(self._t._codomain.is_discrete, False) - @param.param_func([(np.array([-1., 1., 0.]), np.array([1., 1., 0.])), - (np.array([[1., -1., -0.1], [-3., -0.1, 0]]), - np.array([[1., 1., 0.1], [3., 0.1, 0]]))]) + @param.param_func( + [ + (np.array([-1.0, 1.0, 0.0]), np.array([1.0, 1.0, 0.0])), + ( + np.array([[1.0, -1.0, -0.1], [-3.0, -0.1, 0]]), + np.array([[1.0, 1.0, 0.1], [3.0, 0.1, 0]]), + ), + ] + ) def test_forward(self, input, expected): - np.testing.assert_allclose(self._t.forward( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.forward(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) - @param.param_func([(np.array(1.), (-np.array(1.), np.array(1.)))]) + @param.param_func([(np.array(1.0), (-np.array(1.0), np.array(1.0)))]) def test_inverse(self, input, expected): actual0, actual1 = self._t.inverse(paddle.to_tensor(input)) expected0, expected1 = expected - np.testing.assert_allclose(actual0.numpy(), - expected0, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - np.testing.assert_allclose(actual1.numpy(), - expected1, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + actual0.numpy(), + expected0, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + np.testing.assert_allclose( + actual1.numpy(), + expected1, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def test_forward_log_det_jacobian(self): with self.assertRaises(NotImplementedError): - self._t.forward_log_det_jacobian(paddle.rand((10, ))) + self._t.forward_log_det_jacobian(paddle.rand((10,))) - @param.param_func([ - (np.array(1.), (np.array(0.), np.array(0.))), - ]) + @param.param_func( + [ + (np.array(1.0), (np.array(0.0), np.array(0.0))), + ] + ) def test_inverse_log_det_jacobian(self, input, expected): actual0, actual1 = self._t.inverse_log_det_jacobian( - paddle.to_tensor(input)) + paddle.to_tensor(input) + ) expected0, expected1 = expected - np.testing.assert_allclose(actual0.numpy(), - expected0, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - np.testing.assert_allclose(actual1.numpy(), - expected1, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + actual0.numpy(), + expected0, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + np.testing.assert_allclose( + actual1.numpy(), + expected1, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -161,20 +193,25 @@ def test_inverse_shape(self, shape, expected_shape): @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'loc', 'scale'), [ - ('normal', np.random.rand(8, 10), np.random.rand(8, 10)), - ('broadcast', np.random.rand(2, 10), np.random.rand(10)), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'loc', 'scale'), + [ + ('normal', np.random.rand(8, 10), np.random.rand(8, 10)), + ('broadcast', np.random.rand(2, 10), np.random.rand(10)), + ], +) class TestAffineTransform(unittest.TestCase): - def setUp(self): - self._t = transform.AffineTransform(paddle.to_tensor(self.loc), - paddle.to_tensor(self.scale)) + self._t = transform.AffineTransform( + paddle.to_tensor(self.loc), paddle.to_tensor(self.scale) + ) - @param.param_func([ - (paddle.rand([1]), 0, TypeError), - (0, paddle.rand([1]), TypeError), - ]) + @param.param_func( + [ + (paddle.rand([1]), 0, TypeError), + (0, paddle.rand([1]), TypeError), + ] + ) def test_init_exception(self, loc, scale, exc): with self.assertRaises(exc): paddle.distribution.AffineTransform(loc, scale) @@ -204,7 +241,8 @@ def test_forward(self): self._t.forward(paddle.to_tensor(x)).numpy(), self._np_forward(x), rtol=config.RTOL.get(str(self._t.loc.numpy().dtype)), - atol=config.ATOL.get(str(self._t.loc.numpy().dtype))) + atol=config.ATOL.get(str(self._t.loc.numpy().dtype)), + ) def test_inverse(self): y = np.random.random(self.loc.shape) @@ -212,7 +250,8 @@ def test_inverse(self): self._t.inverse(paddle.to_tensor(y)).numpy(), self._np_inverse(y), rtol=config.RTOL.get(str(self._t.loc.numpy().dtype)), - atol=config.ATOL.get(str(self._t.loc.numpy().dtype))) + atol=config.ATOL.get(str(self._t.loc.numpy().dtype)), + ) def _np_forward(self, x): return self.loc + self.scale * x @@ -232,7 +271,8 @@ def test_inverse_log_det_jacobian(self): self._t.inverse_log_det_jacobian(paddle.to_tensor(y)).numpy(), self._np_inverse_jacobian(y), rtol=config.RTOL.get(str(self._t.loc.numpy().dtype)), - atol=config.ATOL.get(str(self._t.loc.numpy().dtype))) + atol=config.ATOL.get(str(self._t.loc.numpy().dtype)), + ) def test_forward_log_det_jacobian(self): x = np.random.random(self.scale.shape) @@ -240,24 +280,26 @@ def test_forward_log_det_jacobian(self): self._t.forward_log_det_jacobian(paddle.to_tensor(x)).numpy(), self._np_forward_jacobian(x), rtol=config.RTOL.get(str(self._t.loc.numpy().dtype)), - atol=config.ATOL.get(str(self._t.loc.numpy().dtype))) + atol=config.ATOL.get(str(self._t.loc.numpy().dtype)), + ) def test_forward_shape(self): shape = self.loc.shape self.assertEqual( tuple(self._t.forward_shape(shape)), - np.broadcast(np.random.random(shape), self.loc, self.scale).shape) + np.broadcast(np.random.random(shape), self.loc, self.scale).shape, + ) def test_inverse_shape(self): shape = self.scale.shape self.assertEqual( tuple(self._t.forward_shape(shape)), - np.broadcast(np.random.random(shape), self.loc, self.scale).shape) + np.broadcast(np.random.random(shape), self.loc, self.scale).shape, + ) @param.place(config.DEVICES) class TestExpTransform(unittest.TestCase): - def setUp(self): self._t = transform.ExpTransform() @@ -274,48 +316,73 @@ def test_codomain(self): self.assertEqual(self._t._codomain.event_rank, 0) self.assertEqual(self._t._codomain.is_discrete, False) - @param.param_func([(np.array([0., 1., 2., - 3.]), np.exp(np.array([0., 1., 2., 3.]))), - (np.array([[0., 1., 2., 3.], [-5., 6., 7., 8.]]), - np.exp(np.array([[0., 1., 2., 3.], [-5., 6., 7., - 8.]])))]) + @param.param_func( + [ + ( + np.array([0.0, 1.0, 2.0, 3.0]), + np.exp(np.array([0.0, 1.0, 2.0, 3.0])), + ), + ( + np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]]), + np.exp(np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]])), + ), + ] + ) def test_forward(self, input, expected): - np.testing.assert_allclose(self._t.forward( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., 3.]), np.log(np.array([1., 2., 3.]))), - (np.array([[1., 2., 3.], [6., 7., 8.]]), - np.log(np.array([[1., 2., 3.], [6., 7., 8.]])))]) + np.testing.assert_allclose( + self._t.forward(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]), np.log(np.array([1.0, 2.0, 3.0]))), + ( + np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]), + np.log(np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]])), + ), + ] + ) def test_inverse(self, input, expected): - np.testing.assert_allclose(self._t.inverse( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + np.testing.assert_allclose( + self._t.inverse(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_forward_log_det_jacobian(self, input): - np.testing.assert_allclose(self._t.forward_log_det_jacobian( - paddle.to_tensor(input)).numpy(), - self._np_forward_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.forward_log_det_jacobian(paddle.to_tensor(input)).numpy(), + self._np_forward_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_forward_jacobian(self, x): return x - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_inverse_log_det_jacobian(self, input): - np.testing.assert_allclose(self._t.inverse_log_det_jacobian( - paddle.to_tensor(input)).numpy(), - self._np_inverse_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.inverse_log_det_jacobian(paddle.to_tensor(input)).numpy(), + self._np_inverse_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_inverse_jacobian(self, y): return -self._np_forward_jacobian(np.log(y)) @@ -331,127 +398,248 @@ def test_inverse_shape(self, shape, expected_shape): @param.place(config.DEVICES) class TestChainTransform(unittest.TestCase): - - @param.param_func([(paddle.distribution.Transform, TypeError), - ([0], TypeError)]) + @param.param_func( + [(paddle.distribution.Transform, TypeError), ([0], TypeError)] + ) def test_init_exception(self, transforms, exception): with self.assertRaises(exception): paddle.distribution.ChainTransform(transforms) - @param.param_func(((transform.ChainTransform( - (transform.AbsTransform(), - transform.AffineTransform(paddle.rand([1]), paddle.rand([1])))), - False), (transform.ChainTransform(( - transform.AffineTransform(paddle.rand([1]), - paddle.rand([1])), - transform.ExpTransform(), - )), True))) + @param.param_func( + ( + ( + transform.ChainTransform( + ( + transform.AbsTransform(), + transform.AffineTransform( + paddle.rand([1]), paddle.rand([1]) + ), + ) + ), + False, + ), + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.rand([1]), paddle.rand([1]) + ), + transform.ExpTransform(), + ) + ), + True, + ), + ) + ) def test_is_injective(self, chain, expected): self.assertEqual(chain._is_injective(), expected) - @param.param_func(((transform.ChainTransform( - (transform.IndependentTransform(transform.ExpTransform(), 1), - transform.IndependentTransform(transform.ExpTransform(), 10), - transform.IndependentTransform(transform.ExpTransform(), 8))), - variable.Independent(variable.real, 10)), )) + @param.param_func( + ( + ( + transform.ChainTransform( + ( + transform.IndependentTransform( + transform.ExpTransform(), 1 + ), + transform.IndependentTransform( + transform.ExpTransform(), 10 + ), + transform.IndependentTransform( + transform.ExpTransform(), 8 + ), + ) + ), + variable.Independent(variable.real, 10), + ), + ) + ) def test_domain(self, input, expected): self.assertIsInstance(input._domain, type(expected)) self.assertEqual(input._domain.event_rank, expected.event_rank) self.assertEqual(input._domain.is_discrete, expected.is_discrete) - @param.param_func(((transform.ChainTransform( - (transform.IndependentTransform(transform.ExpTransform(), 9), - transform.IndependentTransform(transform.ExpTransform(), 4), - transform.IndependentTransform(transform.ExpTransform(), 5))), - variable.Independent(variable.real, 9)), )) + @param.param_func( + ( + ( + transform.ChainTransform( + ( + transform.IndependentTransform( + transform.ExpTransform(), 9 + ), + transform.IndependentTransform( + transform.ExpTransform(), 4 + ), + transform.IndependentTransform( + transform.ExpTransform(), 5 + ), + ) + ), + variable.Independent(variable.real, 9), + ), + ) + ) def test_codomain(self, input, expected): self.assertIsInstance(input._codomain, variable.Independent) self.assertEqual(input._codomain.event_rank, expected.event_rank) self.assertEqual(input._codomain.is_discrete, expected.is_discrete) - @param.param_func([ - (transform.ChainTransform( - (transform.AffineTransform(paddle.to_tensor(0.0), - paddle.to_tensor(1.0)), - transform.ExpTransform())), np.array([0., 1., 2., 3.]), - np.exp(np.array([0., 1., 2., 3.]) * 1.0)), - (transform.ChainTransform( - (transform.ExpTransform(), transform.TanhTransform())), - np.array([[0., -1., 2., -3.], [-5., 6., 7., -8.]]), - np.tanh(np.exp(np.array([[0., -1., 2., -3.], [-5., 6., 7., -8.]])))) - ]) + @param.param_func( + [ + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.to_tensor(0.0), paddle.to_tensor(1.0) + ), + transform.ExpTransform(), + ) + ), + np.array([0.0, 1.0, 2.0, 3.0]), + np.exp(np.array([0.0, 1.0, 2.0, 3.0]) * 1.0), + ), + ( + transform.ChainTransform( + (transform.ExpTransform(), transform.TanhTransform()) + ), + np.array([[0.0, -1.0, 2.0, -3.0], [-5.0, 6.0, 7.0, -8.0]]), + np.tanh( + np.exp( + np.array( + [[0.0, -1.0, 2.0, -3.0], [-5.0, 6.0, 7.0, -8.0]] + ) + ) + ), + ), + ] + ) def test_forward(self, chain, input, expected): - np.testing.assert_allclose(chain.forward( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([ - (transform.ChainTransform( - (transform.AffineTransform(paddle.to_tensor(0.0), - paddle.to_tensor(-1.0)), - transform.ExpTransform())), np.array([0., 1., 2., 3.]), - np.log(np.array([0., 1., 2., 3.])) / (-1.0)), - (transform.ChainTransform( - (transform.ExpTransform(), transform.TanhTransform())), - np.array([[0., 1., 2., 3.], [5., 6., 7., 8.]]), - np.log(np.arctanh(np.array([[0., 1., 2., 3.], [5., 6., 7., 8.]])))) - ]) + np.testing.assert_allclose( + chain.forward(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.to_tensor(0.0), paddle.to_tensor(-1.0) + ), + transform.ExpTransform(), + ) + ), + np.array([0.0, 1.0, 2.0, 3.0]), + np.log(np.array([0.0, 1.0, 2.0, 3.0])) / (-1.0), + ), + ( + transform.ChainTransform( + (transform.ExpTransform(), transform.TanhTransform()) + ), + np.array([[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0]]), + np.log( + np.arctanh( + np.array([[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0]]) + ) + ), + ), + ] + ) def test_inverse(self, chain, input, expected): - np.testing.assert_allclose(chain.inverse( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([ - (transform.ChainTransform( - (transform.AffineTransform(paddle.to_tensor(0.0), - paddle.to_tensor(-1.0)), - transform.PowerTransform(paddle.to_tensor(2.0)))), - np.array([1., 2., 3.]), np.log(2. * np.array([1., 2., 3.]))), - ]) + np.testing.assert_allclose( + chain.inverse(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.to_tensor(0.0), paddle.to_tensor(-1.0) + ), + transform.PowerTransform(paddle.to_tensor(2.0)), + ) + ), + np.array([1.0, 2.0, 3.0]), + np.log(2.0 * np.array([1.0, 2.0, 3.0])), + ), + ] + ) def test_forward_log_det_jacobian(self, chain, input, expected): - np.testing.assert_allclose(chain.forward_log_det_jacobian( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([ - (transform.ChainTransform( - (transform.AffineTransform(paddle.to_tensor(0.0), - paddle.to_tensor(-1.0)), - transform.ExpTransform())), (2, 3, 5), (2, 3, 5)), - ]) + np.testing.assert_allclose( + chain.forward_log_det_jacobian(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.to_tensor(0.0), paddle.to_tensor(-1.0) + ), + transform.ExpTransform(), + ) + ), + (2, 3, 5), + (2, 3, 5), + ), + ] + ) def test_forward_shape(self, chain, shape, expected_shape): self.assertEqual(chain.forward_shape(shape), expected_shape) - @param.param_func([ - (transform.ChainTransform( - (transform.AffineTransform(paddle.to_tensor(0.0), - paddle.to_tensor(-1.0)), - transform.ExpTransform())), (2, 3, 5), (2, 3, 5)), - ]) + @param.param_func( + [ + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.to_tensor(0.0), paddle.to_tensor(-1.0) + ), + transform.ExpTransform(), + ) + ), + (2, 3, 5), + (2, 3, 5), + ), + ] + ) def test_inverse_shape(self, chain, shape, expected_shape): self.assertEqual(chain.inverse_shape(shape), expected_shape) @param.place(config.DEVICES) @param.param_cls( - (param.TEST_CASE_NAME, 'base', 'reinterpreted_batch_rank', 'x'), [ - ('rank-over-zero', transform.ExpTransform(), 2, np.random.rand(2, 3, - 3)), - ]) + (param.TEST_CASE_NAME, 'base', 'reinterpreted_batch_rank', 'x'), + [ + ( + 'rank-over-zero', + transform.ExpTransform(), + 2, + np.random.rand(2, 3, 3), + ), + ], +) class TestIndependentTransform(unittest.TestCase): - def setUp(self): - self._t = transform.IndependentTransform(self.base, - self.reinterpreted_batch_rank) + self._t = transform.IndependentTransform( + self.base, self.reinterpreted_batch_rank + ) - @param.param_func([(0, 0, TypeError), - (paddle.distribution.Transform(), -1, ValueError)]) + @param.param_func( + [(0, 0, TypeError), (paddle.distribution.Transform(), -1, ValueError)] + ) def test_init_exception(self, base, rank, exc): with self.assertRaises(exc): paddle.distribution.IndependentTransform(base, rank) @@ -463,42 +651,52 @@ def test_domain(self): self.assertTrue(isinstance(self._t._domain, variable.Independent)) self.assertEqual( self._t._domain.event_rank, - self.base._domain.event_rank + self.reinterpreted_batch_rank) - self.assertEqual(self._t._domain.is_discrete, - self.base._domain.is_discrete) + self.base._domain.event_rank + self.reinterpreted_batch_rank, + ) + self.assertEqual( + self._t._domain.is_discrete, self.base._domain.is_discrete + ) def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Independent)) self.assertEqual( self._t._codomain.event_rank, - self.base._codomain.event_rank + self.reinterpreted_batch_rank) - self.assertEqual(self._t._codomain.is_discrete, - self.base._codomain.is_discrete) + self.base._codomain.event_rank + self.reinterpreted_batch_rank, + ) + self.assertEqual( + self._t._codomain.is_discrete, self.base._codomain.is_discrete + ) def test_forward(self): np.testing.assert_allclose( self._t.forward(paddle.to_tensor(self.x)).numpy(), self.base.forward(paddle.to_tensor(self.x)).numpy(), rtol=config.RTOL.get(str(self.x.dtype)), - atol=config.ATOL.get(str(self.x.dtype))) + atol=config.ATOL.get(str(self.x.dtype)), + ) def test_inverse(self): np.testing.assert_allclose( self._t.inverse(paddle.to_tensor(self.x)).numpy(), self.base.inverse(paddle.to_tensor(self.x)).numpy(), rtol=config.RTOL.get(str(self.x.dtype)), - atol=config.ATOL.get(str(self.x.dtype))) + atol=config.ATOL.get(str(self.x.dtype)), + ) def test_forward_log_det_jacobian(self): actual = self._t.forward_log_det_jacobian(paddle.to_tensor(self.x)) - self.assertEqual(tuple(actual.shape), - self.x.shape[:-self.reinterpreted_batch_rank]) - expected = self.base.forward_log_det_jacobian(paddle.to_tensor( - self.x)).sum(list(range(-self.reinterpreted_batch_rank, 0))) - np.testing.assert_allclose(actual.numpy(), - expected.numpy(), - rtol=config.RTOL.get(str(self.x.dtype)), - atol=config.ATOL.get(str(self.x.dtype))) + self.assertEqual( + tuple(actual.shape), self.x.shape[: -self.reinterpreted_batch_rank] + ) + expected = self.base.forward_log_det_jacobian( + paddle.to_tensor(self.x) + ).sum(list(range(-self.reinterpreted_batch_rank, 0))) + np.testing.assert_allclose( + actual.numpy(), + expected.numpy(), + rtol=config.RTOL.get(str(self.x.dtype)), + atol=config.ATOL.get(str(self.x.dtype)), + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -511,13 +709,12 @@ def test_inverse_shape(self, shape, expected_shape): @param.place(config.DEVICES) class TestPowerTransform(unittest.TestCase): - def setUp(self): - self._t = transform.PowerTransform(paddle.to_tensor(2.)) + self._t = transform.PowerTransform(paddle.to_tensor(2.0)) def test_init(self): with self.assertRaises(TypeError): - transform.PowerTransform(1.) + transform.PowerTransform(1.0) def test_is_injective(self): self.assertTrue(self._t._is_injective()) @@ -532,34 +729,50 @@ def test_codomain(self): self.assertEqual(self._t._codomain.event_rank, 0) self.assertEqual(self._t._codomain.is_discrete, False) - @param.param_func([(np.array([2.]), np.array([0., -1., 2.]), - np.power(np.array([0., -1., 2.]), 2.)), - (np.array([[0.], [3.]]), np.array([[1., 0.], [5., 6.]]), - np.power(np.array([[1., 0.], [5., 6.]]), - np.array([[0.], [3.]])))]) + @param.param_func( + [ + ( + np.array([2.0]), + np.array([0.0, -1.0, 2.0]), + np.power(np.array([0.0, -1.0, 2.0]), 2.0), + ), + ( + np.array([[0.0], [3.0]]), + np.array([[1.0, 0.0], [5.0, 6.0]]), + np.power( + np.array([[1.0, 0.0], [5.0, 6.0]]), np.array([[0.0], [3.0]]) + ), + ), + ] + ) def test_forward(self, power, x, y): t = transform.PowerTransform(paddle.to_tensor(power)) - np.testing.assert_allclose(t.forward(paddle.to_tensor(x)).numpy(), - y, - rtol=config.RTOL.get(str(x.dtype)), - atol=config.ATOL.get(str(x.dtype))) + np.testing.assert_allclose( + t.forward(paddle.to_tensor(x)).numpy(), + y, + rtol=config.RTOL.get(str(x.dtype)), + atol=config.ATOL.get(str(x.dtype)), + ) - @param.param_func([(np.array([2.]), np.array([4.]), np.array([2.]))]) + @param.param_func([(np.array([2.0]), np.array([4.0]), np.array([2.0]))]) def test_inverse(self, power, y, x): t = transform.PowerTransform(paddle.to_tensor(power)) - np.testing.assert_allclose(t.inverse(paddle.to_tensor(y)).numpy(), - x, - rtol=config.RTOL.get(str(x.dtype)), - atol=config.ATOL.get(str(x.dtype))) + np.testing.assert_allclose( + t.inverse(paddle.to_tensor(y)).numpy(), + x, + rtol=config.RTOL.get(str(x.dtype)), + atol=config.ATOL.get(str(x.dtype)), + ) - @param.param_func(((np.array([2.]), np.array([3., 1.4, 0.8])), )) + @param.param_func(((np.array([2.0]), np.array([3.0, 1.4, 0.8])),)) def test_forward_log_det_jacobian(self, power, x): t = transform.PowerTransform(paddle.to_tensor(power)) - np.testing.assert_allclose(t.forward_log_det_jacobian( - paddle.to_tensor(x)).numpy(), - self._np_forward_jacobian(power, x), - rtol=config.RTOL.get(str(x.dtype)), - atol=config.ATOL.get(str(x.dtype))) + np.testing.assert_allclose( + t.forward_log_det_jacobian(paddle.to_tensor(x)).numpy(), + self._np_forward_jacobian(power, x), + rtol=config.RTOL.get(str(x.dtype)), + atol=config.ATOL.get(str(x.dtype)), + ) def _np_forward_jacobian(self, alpha, x): return np.abs(np.log(alpha * np.power(x, alpha - 1))) @@ -575,7 +788,6 @@ def test_inverse_shape(self, shape, expected_shape): @param.place(config.DEVICES) class TestTanhTransform(unittest.TestCase): - def setUp(self): self._t = transform.TanhTransform() @@ -594,57 +806,83 @@ def test_codomain(self): self.assertEqual(self._t._codomain._constraint._lower, -1) self.assertEqual(self._t._codomain._constraint._upper, 1) - @param.param_func([(np.array([0., 1., 2., - 3.]), np.tanh(np.array([0., 1., 2., 3.]))), - (np.array([[0., 1., 2., 3.], [-5., 6., 7., 8.]]), - np.tanh(np.array([[0., 1., 2., 3.], [-5., 6., 7., - 8.]])))]) + @param.param_func( + [ + ( + np.array([0.0, 1.0, 2.0, 3.0]), + np.tanh(np.array([0.0, 1.0, 2.0, 3.0])), + ), + ( + np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]]), + np.tanh( + np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]]) + ), + ), + ] + ) def test_forward(self, input, expected): - np.testing.assert_allclose(self._t.forward( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., - 3.]), np.arctanh(np.array([1., 2., 3.]))), - (np.array([[1., 2., 3.], [6., 7., 8.]]), - np.arctanh(np.array([[1., 2., 3.], [6., 7., 8.]])))]) + np.testing.assert_allclose( + self._t.forward(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]), np.arctanh(np.array([1.0, 2.0, 3.0]))), + ( + np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]), + np.arctanh(np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]])), + ), + ] + ) def test_inverse(self, input, expected): - np.testing.assert_allclose(self._t.inverse( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + np.testing.assert_allclose( + self._t.inverse(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_forward_log_det_jacobian(self, input): - np.testing.assert_allclose(self._t.forward_log_det_jacobian( - paddle.to_tensor(input)).numpy(), - self._np_forward_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.forward_log_det_jacobian(paddle.to_tensor(input)).numpy(), + self._np_forward_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_forward_jacobian(self, x): - return 2. * (np.log(2.) - x - self._np_softplus(-2. * x)) + return 2.0 * (np.log(2.0) - x - self._np_softplus(-2.0 * x)) - def _np_softplus(self, x, beta=1., threshold=20.): + def _np_softplus(self, x, beta=1.0, threshold=20.0): if np.any(beta * x > threshold): return x - return 1. / beta * np.log1p(np.exp(beta * x)) + return 1.0 / beta * np.log1p(np.exp(beta * x)) def _np_inverse_jacobian(self, y): return -self._np_forward_jacobian(np.arctanh(y)) - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_inverse_log_det_jacobian(self, input): - np.testing.assert_allclose(self._t.inverse_log_det_jacobian( - paddle.to_tensor(input)).numpy(), - self._np_inverse_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.inverse_log_det_jacobian(paddle.to_tensor(input)).numpy(), + self._np_inverse_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -656,20 +894,24 @@ def test_inverse_shape(self, shape, expected_shape): @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'in_event_shape', 'out_event_shape'), [ - ('regular_shape', (2, 3), (3, 2)), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'in_event_shape', 'out_event_shape'), + [ + ('regular_shape', (2, 3), (3, 2)), + ], +) class TestReshapeTransform(unittest.TestCase): - def setUp(self): - self._t = transform.ReshapeTransform(self.in_event_shape, - self.out_event_shape) + self._t = transform.ReshapeTransform( + self.in_event_shape, self.out_event_shape + ) @param.param_func([(0, 0, TypeError), ((1, 2), (1, 3), ValueError)]) def test_init_exception(self, in_event_shape, out_event_shape, exc): with self.assertRaises(exc): - paddle.distribution.ReshapeTransform(in_event_shape, - out_event_shape) + paddle.distribution.ReshapeTransform( + in_event_shape, out_event_shape + ) def test_is_injective(self): self.assertTrue(self._t._is_injective()) @@ -682,24 +924,30 @@ def test_codomain(self): def test_forward(self): x = paddle.ones(self.in_event_shape) - np.testing.assert_allclose(self._t.forward(x), - paddle.ones(self.out_event_shape), - rtol=config.RTOL.get(str(x.numpy().dtype)), - atol=config.ATOL.get(str(x.numpy().dtype))) + np.testing.assert_allclose( + self._t.forward(x), + paddle.ones(self.out_event_shape), + rtol=config.RTOL.get(str(x.numpy().dtype)), + atol=config.ATOL.get(str(x.numpy().dtype)), + ) def test_inverse(self): x = paddle.ones(self.out_event_shape) - np.testing.assert_allclose(self._t.inverse(x).numpy(), - paddle.ones(self.in_event_shape).numpy(), - rtol=config.RTOL.get(str(x.numpy().dtype)), - atol=config.ATOL.get(str(x.numpy().dtype))) + np.testing.assert_allclose( + self._t.inverse(x).numpy(), + paddle.ones(self.in_event_shape).numpy(), + rtol=config.RTOL.get(str(x.numpy().dtype)), + atol=config.ATOL.get(str(x.numpy().dtype)), + ) def test_forward_log_det_jacobian(self): x = paddle.ones(self.in_event_shape) - np.testing.assert_allclose(self._t.forward_log_det_jacobian(x).numpy(), - paddle.zeros([1]).numpy(), - rtol=config.RTOL.get(str(x.numpy().dtype)), - atol=config.ATOL.get(str(x.numpy().dtype))) + np.testing.assert_allclose( + self._t.forward_log_det_jacobian(x).numpy(), + paddle.zeros([1]).numpy(), + rtol=config.RTOL.get(str(x.numpy().dtype)), + atol=config.ATOL.get(str(x.numpy().dtype)), + ) def test_in_event_shape(self): self.assertEqual(self._t.in_event_shape, self.in_event_shape) @@ -718,14 +966,13 @@ def test_inverse_shape_exception(self, shape, exc): self._t.inverse_shape(shape) -def _np_softplus(x, beta=1., threshold=20.): +def _np_softplus(x, beta=1.0, threshold=20.0): if np.any(beta * x > threshold): return x - return 1. / beta * np.log1p(np.exp(beta * x)) + return 1.0 / beta * np.log1p(np.exp(beta * x)) class TestSigmoidTransform(unittest.TestCase): - def setUp(self): self._t = transform.SigmoidTransform() @@ -738,32 +985,43 @@ def test_domain(self): def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Variable)) - @param.param_func(((np.ones( - (5, 10)), 1 / (1 + np.exp(-np.ones((5, 10))))), )) + @param.param_func( + ((np.ones((5, 10)), 1 / (1 + np.exp(-np.ones((5, 10))))),) + ) def test_forward(self, input, expected): - np.testing.assert_allclose(self._t.forward(paddle.to_tensor(input)), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.forward(paddle.to_tensor(input)), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func( - ((np.ones(10), np.log(np.ones(10)) - np.log1p(-np.ones(10))), )) + ((np.ones(10), np.log(np.ones(10)) - np.log1p(-np.ones(10))),) + ) def test_inverse(self, input, expected): - np.testing.assert_allclose(self._t.inverse( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.inverse(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func( - ((np.ones(10), - -_np_softplus(-np.ones(10)) - _np_softplus(np.ones(10))), )) + ( + ( + np.ones(10), + -_np_softplus(-np.ones(10)) - _np_softplus(np.ones(10)), + ), + ) + ) def test_forward_log_det_jacobian(self, input, expected): - np.testing.assert_allclose(self._t.forward_log_det_jacobian( - paddle.to_tensor(input)).numpy(), - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.forward_log_det_jacobian(paddle.to_tensor(input)).numpy(), + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -775,7 +1033,6 @@ def test_inverse_shape(self, shape, expected_shape): class TestSoftmaxTransform(unittest.TestCase): - def setUp(self): self._t = transform.SoftmaxTransform() @@ -788,19 +1045,23 @@ def test_domain(self): def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Variable)) - @param.param_func(((np.random.random((5, 10)), ), )) + @param.param_func(((np.random.random((5, 10)),),)) def test_forward(self, input): - np.testing.assert_allclose(self._t.forward(paddle.to_tensor(input)), - self._np_forward(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.forward(paddle.to_tensor(input)), + self._np_forward(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) - @param.param_func(((np.random.random(10), ), )) + @param.param_func(((np.random.random(10),),)) def test_inverse(self, input): - np.testing.assert_allclose(self._t.inverse(paddle.to_tensor(input)), - self._np_inverse(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.inverse(paddle.to_tensor(input)), + self._np_inverse(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_forward(self, x): x = np.exp(x - np.max(x, -1, keepdims=True)[0]) @@ -833,7 +1094,6 @@ def test_inverse_shape(self, shape, expected_shape): class TestStickBreakingTransform(unittest.TestCase): - def setUp(self): self._t = transform.StickBreakingTransform() @@ -846,13 +1106,14 @@ def test_domain(self): def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Variable)) - @param.param_func(((np.random.random((10)), ), )) + @param.param_func(((np.random.random((10)),),)) def test_forward(self, input): - np.testing.assert_allclose(self._t.inverse( - self._t.forward(paddle.to_tensor(input))), - input, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + self._t.inverse(self._t.forward(paddle.to_tensor(input))), + input, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func([((2, 3, 5), (2, 3, 6))]) def test_forward_shape(self, shape, expected_shape): @@ -862,19 +1123,22 @@ def test_forward_shape(self, shape, expected_shape): def test_inverse_shape(self, shape, expected_shape): self.assertEqual(self._t.inverse_shape(shape), expected_shape) - @param.param_func(((np.random.random((10)), ), )) + @param.param_func(((np.random.random((10)),),)) def test_forward_log_det_jacobian(self, x): self.assertEqual( - self._t.forward_log_det_jacobian(paddle.to_tensor(x)).shape, [1]) + self._t.forward_log_det_jacobian(paddle.to_tensor(x)).shape, [1] + ) # Todo @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'transforms', 'axis'), [ - ('simple_one_transform', [transform.ExpTransform()], 0), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'transforms', 'axis'), + [ + ('simple_one_transform', [transform.ExpTransform()], 0), + ], +) class TestStackTransform(unittest.TestCase): - def setUp(self): self._t = transform.StackTransform(self.transforms, self.axis) @@ -887,25 +1151,42 @@ def test_domain(self): def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Stack)) - @param.param_func([(np.array([[0., 1., 2., 3.]]), ), - (np.array([[-5., 6., 7., 8.]]), )]) + @param.param_func( + [ + (np.array([[0.0, 1.0, 2.0, 3.0]]),), + (np.array([[-5.0, 6.0, 7.0, 8.0]]),), + ] + ) def test_forward(self, input): - self.assertEqual(tuple(self._t.forward(paddle.to_tensor(input)).shape), - input.shape) + self.assertEqual( + tuple(self._t.forward(paddle.to_tensor(input)).shape), input.shape + ) - @param.param_func([(np.array([[1., 2., 3.]]), ), - (np.array([[6., 7., 8.]], ), )]) + @param.param_func( + [ + (np.array([[1.0, 2.0, 3.0]]),), + ( + np.array( + [[6.0, 7.0, 8.0]], + ), + ), + ] + ) def test_inverse(self, input): - self.assertEqual(tuple(self._t.inverse(paddle.to_tensor(input)).shape), - input.shape) + self.assertEqual( + tuple(self._t.inverse(paddle.to_tensor(input)).shape), input.shape + ) - @param.param_func([(np.array([[1., 2., 3.]]), ), (np.array([[6., 7., - 8.]]), )]) + @param.param_func( + [(np.array([[1.0, 2.0, 3.0]]),), (np.array([[6.0, 7.0, 8.0]]),)] + ) def test_forward_log_det_jacobian(self, input): self.assertEqual( tuple( - self._t.forward_log_det_jacobian( - paddle.to_tensor(input)).shape), input.shape) + self._t.forward_log_det_jacobian(paddle.to_tensor(input)).shape + ), + input.shape, + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -918,9 +1199,13 @@ def test_inverse_shape(self, shape, expected_shape): def test_axis(self): self.assertEqual(self._t.axis, self.axis) - @param.param_func([(0, 0, TypeError), ([0], 0, TypeError), - ([paddle.distribution.ExpTransform()], 'axis', TypeError) - ]) + @param.param_func( + [ + (0, 0, TypeError), + ([0], 0, TypeError), + ([paddle.distribution.ExpTransform()], 'axis', TypeError), + ] + ) def test_init_exception(self, transforms, axis, exc): with self.assertRaises(exc): paddle.distribution.StackTransform(transforms, axis) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform_static.py index 4af17de382e25..c5131468faa99 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transform_static.py @@ -28,14 +28,17 @@ @param.place(config.DEVICES) class TestTransform(unittest.TestCase): - def setUp(self): self._t = transform.Transform() - @param.param_func([(transform.Type.BIJECTION, True), - (transform.Type.INJECTION, True), - (transform.Type.SURJECTION, False), - (transform.Type.OTHER, False)]) + @param.param_func( + [ + (transform.Type.BIJECTION, True), + (transform.Type.INJECTION, True), + (transform.Type.SURJECTION, False), + (transform.Type.OTHER, False), + ] + ) def test_is_injective(self, type, expected): transform.Transform._type = type self.assertEqual(self._t._is_injective(), expected) @@ -46,8 +49,12 @@ def test_domain(self): def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Real)) - @param.param_func([(np.array(0), NotImplementedError), - (np.random.random((2, 3)), NotImplementedError)]) + @param.param_func( + [ + (np.array(0), NotImplementedError), + (np.random.random((2, 3)), NotImplementedError), + ] + ) def test_forward(self, input, expected): with self.assertRaises(expected): exe = paddle.static.Executor() @@ -55,14 +62,19 @@ def test_forward(self, input, expected): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): t = transform.Transform() - static_input = paddle.static.data('input', input.shape, - input.dtype) + static_input = paddle.static.data( + 'input', input.shape, input.dtype + ) output = t.forward(static_input) exe.run(sp) exe.run(mp, feed={'input': input}, fetch_list=[output]) - @param.param_func([(np.array(0), NotImplementedError), - (np.random.random((2, 3)), NotImplementedError)]) + @param.param_func( + [ + (np.array(0), NotImplementedError), + (np.random.random((2, 3)), NotImplementedError), + ] + ) def test_inverse(self, input, expected): with self.assertRaises(expected): exe = paddle.static.Executor() @@ -70,14 +82,19 @@ def test_inverse(self, input, expected): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): t = transform.Transform() - static_input = paddle.static.data('input', input.shape, - input.dtype) + static_input = paddle.static.data( + 'input', input.shape, input.dtype + ) output = t.inverse(static_input) exe.run(sp) exe.run(mp, feed={'input': input}, fetch_list=[output]) - @param.param_func([(np.array(0), NotImplementedError), - (paddle.rand((2, 3)), NotImplementedError)]) + @param.param_func( + [ + (np.array(0), NotImplementedError), + (paddle.rand((2, 3)), NotImplementedError), + ] + ) def test_forward_log_det_jacobian(self, input, expected): with self.assertRaises(expected): exe = paddle.static.Executor() @@ -85,14 +102,19 @@ def test_forward_log_det_jacobian(self, input, expected): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): t = transform.Transform() - static_input = paddle.static.data('input', input.shape, - input.dtype) + static_input = paddle.static.data( + 'input', input.shape, input.dtype + ) output = t.forward_log_det_jacobian(static_input) exe.run(sp) exe.run(mp, feed={'input': input}, fetch_list=[output]) - @param.param_func([(np.array(0), NotImplementedError), - (paddle.rand((2, 3)), NotImplementedError)]) + @param.param_func( + [ + (np.array(0), NotImplementedError), + (paddle.rand((2, 3)), NotImplementedError), + ] + ) def test_inverse_log_det_jacobian(self, input, expected): with self.assertRaises(expected): exe = paddle.static.Executor() @@ -100,8 +122,9 @@ def test_inverse_log_det_jacobian(self, input, expected): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): t = transform.Transform() - static_input = paddle.static.data('input', input.shape, - input.dtype) + static_input = paddle.static.data( + 'input', input.shape, input.dtype + ) output = t.inverse_log_det_jacobian(static_input) exe.run(sp) exe.run(mp, feed={'input': input}, fetch_list=[output]) @@ -119,7 +142,6 @@ def test_inverse_shape(self, shape, expected): @param.place(config.DEVICES) class TestAbsTransform(unittest.TestCase): - def setUp(self): self._t = transform.AbsTransform() @@ -136,9 +158,15 @@ def test_codomain(self): self.assertEqual(self._t._codomain.event_rank, 0) self.assertEqual(self._t._codomain.is_discrete, False) - @param.param_func([(np.array([-1., 1., 0.]), np.array([1., 1., 0.])), - (np.array([[1., -1., -0.1], [-3., -0.1, 0]]), - np.array([[1., 1., 0.1], [3., 0.1, 0]]))]) + @param.param_func( + [ + (np.array([-1.0, 1.0, 0.0]), np.array([1.0, 1.0, 0.0])), + ( + np.array([[1.0, -1.0, -0.1], [-3.0, -0.1, 0]]), + np.array([[1.0, 1.0, 0.1], [3.0, 0.1, 0]]), + ), + ] + ) def test_forward(self, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -149,12 +177,14 @@ def test_forward(self, input, expected): output = t.forward(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1.]), (-np.array([1.]), np.array([1.])))]) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func([(np.array([1.0]), (-np.array([1.0]), np.array([1.0])))]) def test_inverse(self, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -164,36 +194,43 @@ def test_inverse(self, input, expected): static_input = paddle.static.data('input', input.shape, input.dtype) actual0, actual1 = t.inverse(static_input) exe.run(sp) - [actual0, actual1] = exe.run(mp, - feed={'input': input}, - fetch_list=[actual0, actual1]) + [actual0, actual1] = exe.run( + mp, feed={'input': input}, fetch_list=[actual0, actual1] + ) expected0, expected1 = expected - np.testing.assert_allclose(actual0, - expected0, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - np.testing.assert_allclose(actual1, - expected1, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + actual0, + expected0, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + np.testing.assert_allclose( + actual1, + expected1, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def test_forward_log_det_jacobian(self): - input = np.random.random((10, )) + input = np.random.random((10,)) with self.assertRaises(NotImplementedError): exe = paddle.static.Executor() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): t = transform.AbsTransform() - static_input = paddle.static.data('input', input.shape, - input.dtype) + static_input = paddle.static.data( + 'input', input.shape, input.dtype + ) output = t.forward_log_det_jacobian(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - @param.param_func([ - (np.array([1.]), (np.array([0.]), np.array([0.]))), - ]) + @param.param_func( + [ + (np.array([1.0]), (np.array([0.0]), np.array([0.0]))), + ] + ) def test_inverse_log_det_jacobian(self, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -203,18 +240,22 @@ def test_inverse_log_det_jacobian(self, input, expected): static_input = paddle.static.data('input', input.shape, input.dtype) actual0, actual1 = t.inverse_log_det_jacobian(static_input) exe.run(sp) - [actual0, actual1] = exe.run(mp, - feed={'input': input}, - fetch_list=[actual0, actual1]) + [actual0, actual1] = exe.run( + mp, feed={'input': input}, fetch_list=[actual0, actual1] + ) expected0, expected1 = expected - np.testing.assert_allclose(actual0, - expected0, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - np.testing.assert_allclose(actual1, - expected1, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + actual0, + expected0, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + np.testing.assert_allclose( + actual1, + expected1, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -226,19 +267,22 @@ def test_inverse_shape(self, shape, expected_shape): @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'loc', 'scale'), [ - ('normal', np.random.rand(8, 10), np.random.rand(8, 10)), - ('broadcast', np.random.rand(2, 10), np.random.rand(10)), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'loc', 'scale'), + [ + ('normal', np.random.rand(8, 10), np.random.rand(8, 10)), + ('broadcast', np.random.rand(2, 10), np.random.rand(10)), + ], +) class TestAffineTransform(unittest.TestCase): - def setUp(self): sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) self._t = transform.AffineTransform(loc, scale) def test_is_injective(self): @@ -261,24 +305,26 @@ def test_forward(self): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) t = transform.AffineTransform(loc, scale) - static_input = paddle.static.data('input', self.loc.shape, - self.loc.dtype) + static_input = paddle.static.data( + 'input', self.loc.shape, self.loc.dtype + ) output = t.forward(static_input) exe.run(sp) - [output] = exe.run(mp, - feed={ - 'input': input, - 'loc': self.loc, - 'scale': self.scale - }, - fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_forward(input), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + [output] = exe.run( + mp, + feed={'input': input, 'loc': self.loc, 'scale': self.scale}, + fetch_list=[output], + ) + np.testing.assert_allclose( + output, + self._np_forward(input), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_inverse(self): input = np.random.random(self.loc.shape) @@ -287,24 +333,26 @@ def test_inverse(self): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) t = transform.AffineTransform(loc, scale) - static_input = paddle.static.data('input', self.loc.shape, - self.loc.dtype) + static_input = paddle.static.data( + 'input', self.loc.shape, self.loc.dtype + ) output = t.inverse(static_input) exe.run(sp) - [output] = exe.run(mp, - feed={ - 'input': input, - 'loc': self.loc, - 'scale': self.scale - }, - fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_inverse(input), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + [output] = exe.run( + mp, + feed={'input': input, 'loc': self.loc, 'scale': self.scale}, + fetch_list=[output], + ) + np.testing.assert_allclose( + output, + self._np_inverse(input), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def _np_forward(self, x): return self.loc + self.scale * x @@ -325,23 +373,24 @@ def test_inverse_log_det_jacobian(self): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) t = transform.AffineTransform(loc, scale) static_input = paddle.static.data('input', input.shape, input.dtype) output = t.inverse_log_det_jacobian(static_input) exe.run(sp) - [output] = exe.run(mp, - feed={ - 'input': input, - 'loc': self.loc, - 'scale': self.scale - }, - fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_inverse_jacobian(input), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + [output] = exe.run( + mp, + feed={'input': input, 'loc': self.loc, 'scale': self.scale}, + fetch_list=[output], + ) + np.testing.assert_allclose( + output, + self._np_inverse_jacobian(input), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_forward_log_det_jacobian(self): input = np.random.random(self.scale.shape) @@ -350,40 +399,42 @@ def test_forward_log_det_jacobian(self): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): loc = paddle.static.data('loc', self.loc.shape, self.loc.dtype) - scale = paddle.static.data('scale', self.scale.shape, - self.scale.dtype) + scale = paddle.static.data( + 'scale', self.scale.shape, self.scale.dtype + ) t = transform.AffineTransform(loc, scale) static_input = paddle.static.data('input', input.shape, input.dtype) output = t.forward_log_det_jacobian(static_input) exe.run(sp) - [output] = exe.run(mp, - feed={ - 'input': input, - 'loc': self.loc, - 'scale': self.scale - }, - fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_forward_jacobian(input), - rtol=config.RTOL.get(str(self.loc.dtype)), - atol=config.ATOL.get(str(self.loc.dtype))) + [output] = exe.run( + mp, + feed={'input': input, 'loc': self.loc, 'scale': self.scale}, + fetch_list=[output], + ) + np.testing.assert_allclose( + output, + self._np_forward_jacobian(input), + rtol=config.RTOL.get(str(self.loc.dtype)), + atol=config.ATOL.get(str(self.loc.dtype)), + ) def test_forward_shape(self): shape = self.loc.shape self.assertEqual( tuple(self._t.forward_shape(shape)), - np.broadcast(np.random.random(shape), self.loc, self.scale).shape) + np.broadcast(np.random.random(shape), self.loc, self.scale).shape, + ) def test_inverse_shape(self): shape = self.scale.shape self.assertEqual( tuple(self._t.forward_shape(shape)), - np.broadcast(np.random.random(shape), self.loc, self.scale).shape) + np.broadcast(np.random.random(shape), self.loc, self.scale).shape, + ) @param.place(config.DEVICES) class TestExpTransform(unittest.TestCase): - def setUp(self): self._t = transform.ExpTransform() @@ -400,11 +451,18 @@ def test_codomain(self): self.assertEqual(self._t._codomain.event_rank, 0) self.assertEqual(self._t._codomain.is_discrete, False) - @param.param_func([(np.array([0., 1., 2., - 3.]), np.exp(np.array([0., 1., 2., 3.]))), - (np.array([[0., 1., 2., 3.], [-5., 6., 7., 8.]]), - np.exp(np.array([[0., 1., 2., 3.], [-5., 6., 7., - 8.]])))]) + @param.param_func( + [ + ( + np.array([0.0, 1.0, 2.0, 3.0]), + np.exp(np.array([0.0, 1.0, 2.0, 3.0])), + ), + ( + np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]]), + np.exp(np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]])), + ), + ] + ) def test_forward(self, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -415,14 +473,22 @@ def test_forward(self, input, expected): output = t.forward(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., 3.]), np.log(np.array([1., 2., 3.]))), - (np.array([[1., 2., 3.], [6., 7., 8.]]), - np.log(np.array([[1., 2., 3.], [6., 7., 8.]])))]) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]), np.log(np.array([1.0, 2.0, 3.0]))), + ( + np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]), + np.log(np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]])), + ), + ] + ) def test_inverse(self, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -433,13 +499,19 @@ def test_inverse(self, input, expected): output = t.inverse(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_forward_log_det_jacobian(self, input): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -450,16 +522,22 @@ def test_forward_log_det_jacobian(self, input): output = t.forward_log_det_jacobian(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_forward_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + output, + self._np_forward_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_forward_jacobian(self, x): return x - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_inverse_log_det_jacobian(self, input): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -470,10 +548,12 @@ def test_inverse_log_det_jacobian(self, input): output = t.inverse_log_det_jacobian(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_inverse_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + output, + self._np_inverse_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_inverse_jacobian(self, y): return -self._np_forward_jacobian(np.log(y)) @@ -489,44 +569,102 @@ def test_inverse_shape(self, shape, expected_shape): @param.place(config.DEVICES) class TestChainTransform(unittest.TestCase): - - @param.param_func(((transform.ChainTransform( - (transform.AbsTransform(), - transform.AffineTransform(paddle.rand([1]), paddle.rand([1])))), - False), (transform.ChainTransform(( - transform.AffineTransform(paddle.rand([1]), - paddle.rand([1])), - transform.ExpTransform(), - )), True))) + @param.param_func( + ( + ( + transform.ChainTransform( + ( + transform.AbsTransform(), + transform.AffineTransform( + paddle.rand([1]), paddle.rand([1]) + ), + ) + ), + False, + ), + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.rand([1]), paddle.rand([1]) + ), + transform.ExpTransform(), + ) + ), + True, + ), + ) + ) def test_is_injective(self, chain, expected): self.assertEqual(chain._is_injective(), expected) - @param.param_func(((transform.ChainTransform( - (transform.IndependentTransform(transform.ExpTransform(), 1), - transform.IndependentTransform(transform.ExpTransform(), 10), - transform.IndependentTransform(transform.ExpTransform(), 8))), - variable.Independent(variable.real, 10)), )) + @param.param_func( + ( + ( + transform.ChainTransform( + ( + transform.IndependentTransform( + transform.ExpTransform(), 1 + ), + transform.IndependentTransform( + transform.ExpTransform(), 10 + ), + transform.IndependentTransform( + transform.ExpTransform(), 8 + ), + ) + ), + variable.Independent(variable.real, 10), + ), + ) + ) def test_domain(self, input, expected): self.assertIsInstance(input._domain, type(expected)) self.assertEqual(input._domain.event_rank, expected.event_rank) self.assertEqual(input._domain.is_discrete, expected.is_discrete) - @param.param_func(((transform.ChainTransform( - (transform.IndependentTransform(transform.ExpTransform(), 9), - transform.IndependentTransform(transform.ExpTransform(), 4), - transform.IndependentTransform(transform.ExpTransform(), 5))), - variable.Independent(variable.real, 9)), )) + @param.param_func( + ( + ( + transform.ChainTransform( + ( + transform.IndependentTransform( + transform.ExpTransform(), 9 + ), + transform.IndependentTransform( + transform.ExpTransform(), 4 + ), + transform.IndependentTransform( + transform.ExpTransform(), 5 + ), + ) + ), + variable.Independent(variable.real, 9), + ), + ) + ) def test_codomain(self, input, expected): self.assertIsInstance(input._codomain, variable.Independent) self.assertEqual(input._codomain.event_rank, expected.event_rank) self.assertEqual(input._codomain.is_discrete, expected.is_discrete) - @param.param_func([ - (transform.ChainTransform( - (transform.ExpTransform(), transform.TanhTransform())), - np.array([[0., -1., 2., -3.], [-5., 6., 7., -8.]]), - np.tanh(np.exp(np.array([[0., -1., 2., -3.], [-5., 6., 7., -8.]])))) - ]) + @param.param_func( + [ + ( + transform.ChainTransform( + (transform.ExpTransform(), transform.TanhTransform()) + ), + np.array([[0.0, -1.0, 2.0, -3.0], [-5.0, 6.0, 7.0, -8.0]]), + np.tanh( + np.exp( + np.array( + [[0.0, -1.0, 2.0, -3.0], [-5.0, 6.0, 7.0, -8.0]] + ) + ) + ), + ) + ] + ) def test_forward(self, chain, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -537,17 +675,28 @@ def test_forward(self, chain, input, expected): output = t.forward(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([ - (transform.ChainTransform( - (transform.ExpTransform(), transform.TanhTransform())), - np.array([[0., 1., 2., 3.], [5., 6., 7., 8.]]), - np.log(np.arctanh(np.array([[0., 1., 2., 3.], [5., 6., 7., 8.]])))) - ]) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + ( + transform.ChainTransform( + (transform.ExpTransform(), transform.TanhTransform()) + ), + np.array([[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0]]), + np.log( + np.arctanh( + np.array([[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0]]) + ) + ), + ) + ] + ) def test_inverse(self, chain, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -558,41 +707,69 @@ def test_inverse(self, chain, input, expected): output = t.inverse(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([ - (transform.ChainTransform( - (transform.AffineTransform(paddle.full([1], 0.0), - paddle.full([1], -1.0)), - transform.ExpTransform())), (2, 3, 5), (2, 3, 5)), - ]) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.full([1], 0.0), paddle.full([1], -1.0) + ), + transform.ExpTransform(), + ) + ), + (2, 3, 5), + (2, 3, 5), + ), + ] + ) def test_forward_shape(self, chain, shape, expected_shape): self.assertEqual(chain.forward_shape(shape), expected_shape) - @param.param_func([ - (transform.ChainTransform( - (transform.AffineTransform(paddle.full([1], 0.0), - paddle.full([1], -1.0)), - transform.ExpTransform())), (2, 3, 5), (2, 3, 5)), - ]) + @param.param_func( + [ + ( + transform.ChainTransform( + ( + transform.AffineTransform( + paddle.full([1], 0.0), paddle.full([1], -1.0) + ), + transform.ExpTransform(), + ) + ), + (2, 3, 5), + (2, 3, 5), + ), + ] + ) def test_inverse_shape(self, chain, shape, expected_shape): self.assertEqual(chain.forward_shape(shape), expected_shape) @param.place(config.DEVICES) @param.param_cls( - (param.TEST_CASE_NAME, 'base', 'reinterpreted_batch_rank', 'x'), [ - ('rank-over-zero', transform.ExpTransform(), 2, np.random.rand(2, 3, - 3)), - ]) + (param.TEST_CASE_NAME, 'base', 'reinterpreted_batch_rank', 'x'), + [ + ( + 'rank-over-zero', + transform.ExpTransform(), + 2, + np.random.rand(2, 3, 3), + ), + ], +) class TestIndependentTransform(unittest.TestCase): - def setUp(self): - self._t = transform.IndependentTransform(self.base, - self.reinterpreted_batch_rank) + self._t = transform.IndependentTransform( + self.base, self.reinterpreted_batch_rank + ) def test_is_injective(self): self.assertEqual(self._t._is_injective(), self.base._is_injective()) @@ -601,81 +778,98 @@ def test_domain(self): self.assertTrue(isinstance(self._t._domain, variable.Independent)) self.assertEqual( self._t._domain.event_rank, - self.base._domain.event_rank + self.reinterpreted_batch_rank) - self.assertEqual(self._t._domain.is_discrete, - self.base._domain.is_discrete) + self.base._domain.event_rank + self.reinterpreted_batch_rank, + ) + self.assertEqual( + self._t._domain.is_discrete, self.base._domain.is_discrete + ) def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Independent)) self.assertEqual( self._t._codomain.event_rank, - self.base._codomain.event_rank + self.reinterpreted_batch_rank) - self.assertEqual(self._t._codomain.is_discrete, - self.base._codomain.is_discrete) + self.base._codomain.event_rank + self.reinterpreted_batch_rank, + ) + self.assertEqual( + self._t._codomain.is_discrete, self.base._codomain.is_discrete + ) def test_forward(self): exe = paddle.static.Executor() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): - t = transform.IndependentTransform(self.base, - self.reinterpreted_batch_rank) - static_input = paddle.static.data('input', self.x.shape, - self.x.dtype) + t = transform.IndependentTransform( + self.base, self.reinterpreted_batch_rank + ) + static_input = paddle.static.data( + 'input', self.x.shape, self.x.dtype + ) output = t.forward(static_input) expected = self.base.forward(static_input) exe.run(sp) - [output, expected] = exe.run(mp, - feed={'input': self.x}, - fetch_list=[output, expected]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(self.x.dtype)), - atol=config.ATOL.get(str(self.x.dtype))) + [output, expected] = exe.run( + mp, feed={'input': self.x}, fetch_list=[output, expected] + ) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(self.x.dtype)), + atol=config.ATOL.get(str(self.x.dtype)), + ) def test_inverse(self): exe = paddle.static.Executor() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): - t = transform.IndependentTransform(self.base, - self.reinterpreted_batch_rank) - static_input = paddle.static.data('input', self.x.shape, - self.x.dtype) + t = transform.IndependentTransform( + self.base, self.reinterpreted_batch_rank + ) + static_input = paddle.static.data( + 'input', self.x.shape, self.x.dtype + ) output = t.inverse(static_input) expected = self.base.inverse(static_input) exe.run(sp) - [output, expected] = exe.run(mp, - feed={'input': self.x}, - fetch_list=[output, expected]) - np.testing.assert_allclose(expected, - output, - rtol=config.RTOL.get(str(self.x.dtype)), - atol=config.ATOL.get(str(self.x.dtype))) + [output, expected] = exe.run( + mp, feed={'input': self.x}, fetch_list=[output, expected] + ) + np.testing.assert_allclose( + expected, + output, + rtol=config.RTOL.get(str(self.x.dtype)), + atol=config.ATOL.get(str(self.x.dtype)), + ) def test_forward_log_det_jacobian(self): exe = paddle.static.Executor() sp = paddle.static.Program() mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): - t = transform.IndependentTransform(self.base, - self.reinterpreted_batch_rank) - static_input = paddle.static.data('input', self.x.shape, - self.x.dtype) + t = transform.IndependentTransform( + self.base, self.reinterpreted_batch_rank + ) + static_input = paddle.static.data( + 'input', self.x.shape, self.x.dtype + ) output = t.forward_log_det_jacobian(static_input) expected = self.base.forward_log_det_jacobian( - static_input.sum(list(range(-self.reinterpreted_batch_rank, - 0)))) + static_input.sum(list(range(-self.reinterpreted_batch_rank, 0))) + ) exe.run(sp) - [actual, expected] = exe.run(mp, - feed={'input': self.x}, - fetch_list=[output, expected]) - self.assertEqual(tuple(actual.shape), - self.x.shape[:-self.reinterpreted_batch_rank]) - np.testing.assert_allclose(actual, - expected, - rtol=config.RTOL.get(str(self.x.dtype)), - atol=config.ATOL.get(str(self.x.dtype))) + [actual, expected] = exe.run( + mp, feed={'input': self.x}, fetch_list=[output, expected] + ) + self.assertEqual( + tuple(actual.shape), self.x.shape[: -self.reinterpreted_batch_rank] + ) + np.testing.assert_allclose( + actual, + expected, + rtol=config.RTOL.get(str(self.x.dtype)), + atol=config.ATOL.get(str(self.x.dtype)), + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -688,13 +882,12 @@ def test_inverse_shape(self, shape, expected_shape): @param.place(config.DEVICES) class TestPowerTransform(unittest.TestCase): - def setUp(self): - self._t = transform.PowerTransform(paddle.full([1], 2.)) + self._t = transform.PowerTransform(paddle.full([1], 2.0)) def test_init(self): with self.assertRaises(TypeError): - transform.PowerTransform(1.) + transform.PowerTransform(1.0) def test_is_injective(self): self.assertTrue(self._t._is_injective()) @@ -709,11 +902,22 @@ def test_codomain(self): self.assertEqual(self._t._codomain.event_rank, 0) self.assertEqual(self._t._codomain.is_discrete, False) - @param.param_func([(np.array([2.]), np.array([0., -1., 2.]), - np.power(np.array([0., -1., 2.]), 2.)), - (np.array([[0.], [3.]]), np.array([[1., 0.], [5., 6.]]), - np.power(np.array([[1., 0.], [5., 6.]]), - np.array([[0.], [3.]])))]) + @param.param_func( + [ + ( + np.array([2.0]), + np.array([0.0, -1.0, 2.0]), + np.power(np.array([0.0, -1.0, 2.0]), 2.0), + ), + ( + np.array([[0.0], [3.0]]), + np.array([[1.0, 0.0], [5.0, 6.0]]), + np.power( + np.array([[1.0, 0.0], [5.0, 6.0]]), np.array([[0.0], [3.0]]) + ), + ), + ] + ) def test_forward(self, power, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -724,18 +928,17 @@ def test_forward(self, power, input, expected): static_input = paddle.static.data('input', input.shape, input.dtype) output = t.forward(static_input) exe.run(sp) - [output] = exe.run(mp, - feed={ - 'input': input, - 'power': power - }, - fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([2.]), np.array([4.]), np.array([2.]))]) + [output] = exe.run( + mp, feed={'input': input, 'power': power}, fetch_list=[output] + ) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func([(np.array([2.0]), np.array([4.0]), np.array([2.0]))]) def test_inverse(self, power, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -746,18 +949,17 @@ def test_inverse(self, power, input, expected): static_input = paddle.static.data('input', input.shape, input.dtype) output = t.inverse(static_input) exe.run(sp) - [output] = exe.run(mp, - feed={ - 'input': input, - 'power': power - }, - fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func(((np.array([2.]), np.array([3., 1.4, 0.8])), )) + [output] = exe.run( + mp, feed={'input': input, 'power': power}, fetch_list=[output] + ) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func(((np.array([2.0]), np.array([3.0, 1.4, 0.8])),)) def test_forward_log_det_jacobian(self, power, input): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -768,16 +970,15 @@ def test_forward_log_det_jacobian(self, power, input): static_input = paddle.static.data('input', input.shape, input.dtype) output = t.forward_log_det_jacobian(static_input) exe.run(sp) - [output] = exe.run(mp, - feed={ - 'input': input, - 'power': power - }, - fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_forward_jacobian(power, input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + [output] = exe.run( + mp, feed={'input': input, 'power': power}, fetch_list=[output] + ) + np.testing.assert_allclose( + output, + self._np_forward_jacobian(power, input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_forward_jacobian(self, alpha, x): return np.abs(np.log(alpha * np.power(x, alpha - 1))) @@ -793,7 +994,6 @@ def test_inverse_shape(self, shape, expected_shape): @param.place(config.DEVICES) class TestTanhTransform(unittest.TestCase): - def setUp(self): self._t = transform.TanhTransform() @@ -812,11 +1012,20 @@ def test_codomain(self): self.assertEqual(self._t._codomain._constraint._lower, -1) self.assertEqual(self._t._codomain._constraint._upper, 1) - @param.param_func([(np.array([0., 1., 2., - 3.]), np.tanh(np.array([0., 1., 2., 3.]))), - (np.array([[0., 1., 2., 3.], [-5., 6., 7., 8.]]), - np.tanh(np.array([[0., 1., 2., 3.], [-5., 6., 7., - 8.]])))]) + @param.param_func( + [ + ( + np.array([0.0, 1.0, 2.0, 3.0]), + np.tanh(np.array([0.0, 1.0, 2.0, 3.0])), + ), + ( + np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]]), + np.tanh( + np.array([[0.0, 1.0, 2.0, 3.0], [-5.0, 6.0, 7.0, 8.0]]) + ), + ), + ] + ) def test_forward(self, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -827,15 +1036,22 @@ def test_forward(self, input, expected): output = t.forward(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., - 3.]), np.arctanh(np.array([1., 2., 3.]))), - (np.array([[1., 2., 3.], [6., 7., 8.]]), - np.arctanh(np.array([[1., 2., 3.], [6., 7., 8.]])))]) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]), np.arctanh(np.array([1.0, 2.0, 3.0]))), + ( + np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]), + np.arctanh(np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]])), + ), + ] + ) def test_inverse(self, input, expected): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -846,13 +1062,19 @@ def test_inverse(self, input, expected): output = t.inverse(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) - - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_forward_log_det_jacobian(self, input): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -863,24 +1085,30 @@ def test_forward_log_det_jacobian(self, input): output = t.forward_log_det_jacobian(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_forward_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + output, + self._np_forward_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) def _np_forward_jacobian(self, x): - return 2. * (np.log(2.) - x - self._np_softplus(-2. * x)) + return 2.0 * (np.log(2.0) - x - self._np_softplus(-2.0 * x)) - def _np_softplus(self, x, beta=1., threshold=20.): + def _np_softplus(self, x, beta=1.0, threshold=20.0): if np.any(beta * x > threshold): return x - return 1. / beta * np.log1p(np.exp(beta * x)) + return 1.0 / beta * np.log1p(np.exp(beta * x)) def _np_inverse_jacobian(self, y): return -self._np_forward_jacobian(np.arctanh(y)) - @param.param_func([(np.array([1., 2., 3.]), ), - (np.array([[1., 2., 3.], [6., 7., 8.]]), )]) + @param.param_func( + [ + (np.array([1.0, 2.0, 3.0]),), + (np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0]]),), + ] + ) def test_inverse_log_det_jacobian(self, input): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -891,10 +1119,12 @@ def test_inverse_log_det_jacobian(self, input): output = t.inverse_log_det_jacobian(static_input) exe.run(sp) [output] = exe.run(mp, feed={'input': input}, fetch_list=[output]) - np.testing.assert_allclose(output, - self._np_inverse_jacobian(input), - rtol=config.RTOL.get(str(input.dtype)), - atol=config.ATOL.get(str(input.dtype))) + np.testing.assert_allclose( + output, + self._np_inverse_jacobian(input), + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) def test_forward_shape(self, shape, expected_shape): @@ -906,14 +1136,17 @@ def test_inverse_shape(self, shape, expected_shape): @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'in_event_shape', 'out_event_shape'), [ - ('regular_shape', (2, 3), (3, 2)), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'in_event_shape', 'out_event_shape'), + [ + ('regular_shape', (2, 3), (3, 2)), + ], +) class TestReshapeTransform(unittest.TestCase): - def setUp(self): - self._t = transform.ReshapeTransform(self.in_event_shape, - self.out_event_shape) + self._t = transform.ReshapeTransform( + self.in_event_shape, self.out_event_shape + ) def test_is_injective(self): self.assertTrue(self._t._is_injective()) @@ -930,16 +1163,19 @@ def test_forward(self): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): x = paddle.ones(self.in_event_shape) - t = transform.ReshapeTransform(self.in_event_shape, - self.out_event_shape) + t = transform.ReshapeTransform( + self.in_event_shape, self.out_event_shape + ) output = self._t.forward(x) exe.run(sp) [output] = exe.run(mp, feed={}, fetch_list=[output]) expected = np.ones(self.out_event_shape) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(expected.dtype)), - atol=config.ATOL.get(str(expected.dtype))) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(expected.dtype)), + atol=config.ATOL.get(str(expected.dtype)), + ) def test_inverse(self): exe = paddle.static.Executor() @@ -947,17 +1183,20 @@ def test_inverse(self): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): x = paddle.ones(self.out_event_shape) - t = transform.ReshapeTransform(self.in_event_shape, - self.out_event_shape) + t = transform.ReshapeTransform( + self.in_event_shape, self.out_event_shape + ) output = self._t.inverse(x) exe.run(sp) [output] = exe.run(mp, feed={}, fetch_list=[output]) expected = np.ones(self.in_event_shape) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(expected.dtype)), - atol=config.ATOL.get(str(expected.dtype))) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(expected.dtype)), + atol=config.ATOL.get(str(expected.dtype)), + ) def test_forward_log_det_jacobian(self): exe = paddle.static.Executor() @@ -965,16 +1204,19 @@ def test_forward_log_det_jacobian(self): mp = paddle.static.Program() with paddle.static.program_guard(mp, sp): x = paddle.ones(self.in_event_shape) - t = transform.ReshapeTransform(self.in_event_shape, - self.out_event_shape) + t = transform.ReshapeTransform( + self.in_event_shape, self.out_event_shape + ) output = self._t.forward_log_det_jacobian(x) exe.run(sp) [output] = exe.run(mp, feed={}, fetch_list=[output]) expected = np.zeros([1]) - np.testing.assert_allclose(output, - expected, - rtol=config.RTOL.get(str(expected.dtype)), - atol=config.ATOL.get(str(expected.dtype))) + np.testing.assert_allclose( + output, + expected, + rtol=config.RTOL.get(str(expected.dtype)), + atol=config.ATOL.get(str(expected.dtype)), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution.py index c1aebf7e0d66f..3c38c8cb4dfa3 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution.py @@ -21,14 +21,21 @@ @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'base', 'transforms'), - [('base_normal', paddle.distribution.Normal( - 0., 1.), [paddle.distribution.ExpTransform()])]) +@param.param_cls( + (param.TEST_CASE_NAME, 'base', 'transforms'), + [ + ( + 'base_normal', + paddle.distribution.Normal(0.0, 1.0), + [paddle.distribution.ExpTransform()], + ) + ], +) class TestIndependent(unittest.TestCase): - def setUp(self): self._t = paddle.distribution.TransformedDistribution( - self.base, self.transforms) + self.base, self.transforms + ) def _np_sum_rightmost(self, value, n): return np.sum(value, tuple(range(-n, 0))) if n > 0 else value @@ -39,7 +46,8 @@ def test_log_prob(self): self.simple_log_prob(value, self.base, self.transforms), self._t.log_prob(value), rtol=config.RTOL.get(str(value.numpy().dtype)), - atol=config.ATOL.get(str(value.numpy().dtype))) + atol=config.ATOL.get(str(value.numpy().dtype)), + ) def simple_log_prob(self, value, base, transforms): log_prob = 0.0 diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution_static.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution_static.py index b9f54498f64d8..f9c7ec6ef749f 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_transformed_distribution_static.py @@ -23,15 +23,21 @@ @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'base', 'transforms'), - [('base_normal', paddle.distribution.Normal, - [paddle.distribution.ExpTransform()])]) +@param.param_cls( + (param.TEST_CASE_NAME, 'base', 'transforms'), + [ + ( + 'base_normal', + paddle.distribution.Normal, + [paddle.distribution.ExpTransform()], + ) + ], +) class TestIndependent(unittest.TestCase): - def setUp(self): value = np.array([0.5]) - loc = np.array([0.]) - scale = np.array([1.]) + loc = np.array([0.0]) + scale = np.array([1.0]) shape = [5, 10, 8] self.dtype = value.dtype exe = paddle.static.Executor() @@ -43,28 +49,32 @@ def setUp(self): static_scale = paddle.static.data('scale', scale.shape, scale.dtype) self.base = self.base(static_loc, static_scale) self._t = paddle.distribution.TransformedDistribution( - self.base, self.transforms) + self.base, self.transforms + ) actual_log_prob = self._t.log_prob(static_value) expected_log_prob = self.transformed_log_prob( - static_value, self.base, self.transforms) + static_value, self.base, self.transforms + ) sample_data = self._t.sample(shape) exe.run(sp) - [self.actual_log_prob, self.expected_log_prob, - self.sample_data] = exe.run( - mp, - feed={ - 'value': value, - 'loc': loc, - 'scale': scale - }, - fetch_list=[actual_log_prob, expected_log_prob, sample_data]) + [ + self.actual_log_prob, + self.expected_log_prob, + self.sample_data, + ] = exe.run( + mp, + feed={'value': value, 'loc': loc, 'scale': scale}, + fetch_list=[actual_log_prob, expected_log_prob, sample_data], + ) def test_log_prob(self): - np.testing.assert_allclose(self.actual_log_prob, - self.expected_log_prob, - rtol=config.RTOL.get(str(self.dtype)), - atol=config.ATOL.get(str(self.dtype))) + np.testing.assert_allclose( + self.actual_log_prob, + self.expected_log_prob, + rtol=config.RTOL.get(str(self.dtype)), + atol=config.ATOL.get(str(self.dtype)), + ) def transformed_log_prob(self, value, base, transforms): log_prob = 0.0 diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_uniform.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_uniform.py index 6aafa9cb356ce..522e9bebce417 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_uniform.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_uniform.py @@ -27,7 +27,6 @@ class UniformNumpy(DistributionNumpy): - def __init__(self, low, high): self.low = np.array(low) self.high = np.array(high) @@ -37,8 +36,9 @@ def __init__(self, low, high): def sample(self, shape): shape = tuple(shape) + (self.low + self.high).shape - return self.low + (np.random.uniform(size=shape) * - (self.high - self.low)) + return self.low + ( + np.random.uniform(size=shape) * (self.high - self.low) + ) def log_prob(self, value): lb = np.less(self.low, value).astype(self.low.dtype) @@ -55,7 +55,6 @@ def entropy(self): class UniformTest(unittest.TestCase): - def setUp(self, use_gpu=False, batch_size=5, dims=6): self.use_gpu = use_gpu if not use_gpu: @@ -90,9 +89,9 @@ def init_static_data(self, batch_size, dims): self.static_low = self.low_np self.static_high = self.high_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[], dtype='float32' + ) def compare_with_numpy(self, fetch_list, sample_shape=7, tolerance=1e-6): sample, entropy, log_prob, probs = fetch_list @@ -104,14 +103,12 @@ def compare_with_numpy(self, fetch_list, sample_shape=7, tolerance=1e-6): np_p = np_uniform.probs(self.values_np) np.testing.assert_equal(sample.shape, np_sample.shape) - np.testing.assert_allclose(entropy, - np_entropy, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(log_prob, - np_lp, - rtol=tolerance, - atol=tolerance) + np.testing.assert_allclose( + entropy, np_entropy, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + log_prob, np_lp, rtol=tolerance, atol=tolerance + ) np.testing.assert_allclose(probs, np_p, rtol=tolerance, atol=tolerance) def test_uniform_distribution_static(self, sample_shape=7, tolerance=1e-6): @@ -127,13 +124,13 @@ def test_uniform_distribution_static(self, sample_shape=7, tolerance=1e-6): feed_vars = { 'low': self.low_np, 'high': self.high_np, - 'values': self.values_np + 'values': self.values_np, } self.executor.run(fluid.default_startup_program()) - fetch_list = self.executor.run(program=self.test_program, - feed=feed_vars, - fetch_list=fetch_list) + fetch_list = self.executor.run( + program=self.test_program, feed=feed_vars, fetch_list=fetch_list + ) self.compare_with_numpy(fetch_list) @@ -157,7 +154,6 @@ def test_uniform_distribution_dygraph(self): class UniformTest2(UniformTest): - def init_numpy_data(self, batch_size, dims): # low ans high are 'int' self.low_np = int(np.random.uniform(-2, 1)) @@ -166,48 +162,48 @@ def init_numpy_data(self, batch_size, dims): class UniformTest3(UniformTest): - def init_numpy_data(self, batch_size, dims): # test broadcast: low is float, high is numpy.ndarray with dtype 'float32'. self.low_np = np.random.uniform(-2, 1) - self.high_np = np.random.uniform(5.0, 15.0, - (batch_size, dims)).astype('float32') + self.high_np = np.random.uniform(5.0, 15.0, (batch_size, dims)).astype( + 'float32' + ) self.values_np = np.random.randn(batch_size, dims).astype('float32') def init_static_data(self, batch_size, dims): self.static_low = self.low_np self.static_high = self.high_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class UniformTest4(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are numpy.ndarray with dtype 'float32'. self.low_np = np.random.randn(batch_size, dims).astype('float32') - self.high_np = np.random.uniform(5.0, 15.0, - (batch_size, dims)).astype('float32') + self.high_np = np.random.uniform(5.0, 15.0, (batch_size, dims)).astype( + 'float32' + ) self.values_np = np.random.randn(batch_size, dims).astype('float32') def init_static_data(self, batch_size, dims): self.static_low = self.low_np self.static_high = self.high_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class UniformTest5(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are numpy.ndarray with dtype 'float64'. self.low_np = np.random.randn(batch_size, dims).astype('float64') - self.high_np = np.random.uniform(5.0, 15.0, - (batch_size, dims)).astype('float64') + self.high_np = np.random.uniform(5.0, 15.0, (batch_size, dims)).astype( + 'float64' + ) self.values_np = np.random.randn(batch_size, dims).astype('float64') def init_dynamic_data(self, batch_size, dims): @@ -219,18 +215,18 @@ def init_static_data(self, batch_size, dims): self.static_low = self.low_np self.static_high = self.high_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float64') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float64' + ) class UniformTest6(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are Tensor with dtype 'VarType.FP32'. self.low_np = np.random.randn(batch_size, dims).astype('float32') - self.high_np = np.random.uniform(5.0, 15.0, - (batch_size, dims)).astype('float32') + self.high_np = np.random.uniform(5.0, 15.0, (batch_size, dims)).astype( + 'float32' + ) self.values_np = np.random.randn(batch_size, dims).astype('float32') def init_dynamic_data(self, batch_size, dims): @@ -240,24 +236,24 @@ def init_dynamic_data(self, batch_size, dims): def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.static_low = layers.data(name='low', - shape=[dims], - dtype='float32') - self.static_high = layers.data(name='high', - shape=[dims], - dtype='float32') - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_low = layers.data( + name='low', shape=[dims], dtype='float32' + ) + self.static_high = layers.data( + name='high', shape=[dims], dtype='float32' + ) + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class UniformTest7(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are Tensor with dtype 'VarType.FP64'. self.low_np = np.random.randn(batch_size, dims).astype('float64') - self.high_np = np.random.uniform(5.0, 15.0, - (batch_size, dims)).astype('float64') + self.high_np = np.random.uniform(5.0, 15.0, (batch_size, dims)).astype( + 'float64' + ) self.values_np = np.random.randn(batch_size, dims).astype('float64') def init_dynamic_data(self, batch_size, dims): @@ -267,24 +263,24 @@ def init_dynamic_data(self, batch_size, dims): def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.static_low = layers.data(name='low', - shape=[dims], - dtype='float64') - self.static_high = layers.data(name='high', - shape=[dims], - dtype='float64') - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float64') + self.static_low = layers.data( + name='low', shape=[dims], dtype='float64' + ) + self.static_high = layers.data( + name='high', shape=[dims], dtype='float64' + ) + self.static_values = layers.data( + name='values', shape=[dims], dtype='float64' + ) class UniformTest8(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are Tensor with dtype 'VarType.FP64'. value's dtype is 'VarType.FP32'. self.low_np = np.random.randn(batch_size, dims).astype('float64') - self.high_np = np.random.uniform(5.0, 15.0, - (batch_size, dims)).astype('float64') + self.high_np = np.random.uniform(5.0, 15.0, (batch_size, dims)).astype( + 'float64' + ) self.values_np = np.random.randn(batch_size, dims).astype('float32') def init_dynamic_data(self, batch_size, dims): @@ -294,77 +290,81 @@ def init_dynamic_data(self, batch_size, dims): def init_static_data(self, batch_size, dims): with fluid.program_guard(self.test_program): - self.static_low = layers.data(name='low', - shape=[dims], - dtype='float64') - self.static_high = layers.data(name='high', - shape=[dims], - dtype='float64') - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_low = layers.data( + name='low', shape=[dims], dtype='float64' + ) + self.static_high = layers.data( + name='high', shape=[dims], dtype='float64' + ) + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class UniformTest9(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are numpy.ndarray with dtype 'float32'. # high < low. self.low_np = np.random.randn(batch_size, dims).astype('float32') - self.high_np = np.random.uniform(-10.0, -5.0, - (batch_size, dims)).astype('float32') + self.high_np = np.random.uniform( + -10.0, -5.0, (batch_size, dims) + ).astype('float32') self.values_np = np.random.randn(batch_size, dims).astype('float32') def init_static_data(self, batch_size, dims): self.static_low = self.low_np self.static_high = self.high_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class UniformTest10(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are list. - self.low_np = np.random.randn(batch_size, - dims).astype('float32').tolist() - self.high_np = np.random.uniform( - 5.0, 15.0, (batch_size, dims)).astype('float32').tolist() + self.low_np = ( + np.random.randn(batch_size, dims).astype('float32').tolist() + ) + self.high_np = ( + np.random.uniform(5.0, 15.0, (batch_size, dims)) + .astype('float32') + .tolist() + ) self.values_np = np.random.randn(batch_size, dims).astype('float32') def init_static_data(self, batch_size, dims): self.static_low = self.low_np self.static_high = self.high_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class UniformTest11(UniformTest): - def init_numpy_data(self, batch_size, dims): # low and high are tuple. self.low_np = tuple( - np.random.randn(batch_size, dims).astype('float32').tolist()) + np.random.randn(batch_size, dims).astype('float32').tolist() + ) self.high_np = tuple( - np.random.uniform(5.0, 15.0, - (batch_size, dims)).astype('float32').tolist()) + np.random.uniform(5.0, 15.0, (batch_size, dims)) + .astype('float32') + .tolist() + ) self.values_np = np.random.randn(batch_size, dims).astype('float32') def init_static_data(self, batch_size, dims): self.static_low = self.low_np self.static_high = self.high_np with fluid.program_guard(self.test_program): - self.static_values = layers.data(name='values', - shape=[dims], - dtype='float32') + self.static_values = layers.data( + name='values', shape=[dims], dtype='float32' + ) class UniformTestSample(unittest.TestCase): - def setUp(self): self.init_param() @@ -382,7 +382,6 @@ def test_uniform_sample(self): class UniformTestSample2(UniformTestSample): - def init_param(self): self.low = -5.0 self.high = 2.0 diff --git a/python/paddle/fluid/tests/unittests/distribution/test_distribution_variable.py b/python/paddle/fluid/tests/unittests/distribution/test_distribution_variable.py index 982a47f5e1806..ead60fa9740b8 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_distribution_variable.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_distribution_variable.py @@ -25,47 +25,52 @@ @param.param_cls( (param.TEST_CASE_NAME, 'is_discrete', 'event_rank', 'constraint'), - [('NotImplement', False, 0, constraint.Constraint())]) + [('NotImplement', False, 0, constraint.Constraint())], +) class TestVariable(unittest.TestCase): - def setUp(self): - self._var = variable.Variable(self.is_discrete, self.event_rank, - self.constraint) + self._var = variable.Variable( + self.is_discrete, self.event_rank, self.constraint + ) - @param.param_func([(1, )]) + @param.param_func([(1,)]) def test_costraint(self, value): with self.assertRaises(NotImplementedError): self._var.constraint(value) -@param.param_cls((param.TEST_CASE_NAME, 'base', 'rank'), - [('real_base', variable.real, 10)]) +@param.param_cls( + (param.TEST_CASE_NAME, 'base', 'rank'), [('real_base', variable.real, 10)] +) class TestIndependent(unittest.TestCase): - def setUp(self): self._var = variable.Independent(self.base, self.rank) - @param.param_func([ - (paddle.rand([2, 3, 4]), ValueError), - ]) + @param.param_func( + [ + (paddle.rand([2, 3, 4]), ValueError), + ] + ) def test_costraint(self, value, expect): with self.assertRaises(expect): self._var.constraint(value) -@param.param_cls((param.TEST_CASE_NAME, 'vars', 'axis'), - [('real_base', [variable.real], 10)]) +@param.param_cls( + (param.TEST_CASE_NAME, 'vars', 'axis'), [('real_base', [variable.real], 10)] +) class TestStack(unittest.TestCase): - def setUp(self): self._var = variable.Stack(self.vars, self.axis) def test_is_discrete(self): self.assertEqual(self._var.is_discrete, False) - @param.param_func([ - (paddle.rand([2, 3, 4]), ValueError), - ]) + @param.param_func( + [ + (paddle.rand([2, 3, 4]), ValueError), + ] + ) def test_costraint(self, value, expect): with self.assertRaises(expect): self._var.constraint(value) diff --git a/python/paddle/fluid/tests/unittests/distribution/test_kl.py b/python/paddle/fluid/tests/unittests/distribution/test_kl.py index 582a9f5942f90..5ce6d02542210 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_kl.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_kl.py @@ -30,19 +30,26 @@ @param.place(config.DEVICES) -@param.parameterize_cls((param.TEST_CASE_NAME, 'a1', 'b1', 'a2', 'b2'), [ - ('test_regular_input', 6.0 * np.random.random( - (4, 5)) + 1e-4, 6.0 * np.random.random( - (4, 5)) + 1e-4, 6.0 * np.random.random( - (4, 5)) + 1e-4, 6.0 * np.random.random((4, 5)) + 1e-4), -]) +@param.parameterize_cls( + (param.TEST_CASE_NAME, 'a1', 'b1', 'a2', 'b2'), + [ + ( + 'test_regular_input', + 6.0 * np.random.random((4, 5)) + 1e-4, + 6.0 * np.random.random((4, 5)) + 1e-4, + 6.0 * np.random.random((4, 5)) + 1e-4, + 6.0 * np.random.random((4, 5)) + 1e-4, + ), + ], +) class TestKLBetaBeta(unittest.TestCase): - def setUp(self): - self.p = paddle.distribution.Beta(paddle.to_tensor(self.a1), - paddle.to_tensor(self.b1)) - self.q = paddle.distribution.Beta(paddle.to_tensor(self.a2), - paddle.to_tensor(self.b2)) + self.p = paddle.distribution.Beta( + paddle.to_tensor(self.a1), paddle.to_tensor(self.b1) + ) + self.q = paddle.distribution.Beta( + paddle.to_tensor(self.a2), paddle.to_tensor(self.b2) + ) def test_kl_divergence(self): with paddle.fluid.dygraph.guard(self.place): @@ -50,22 +57,31 @@ def test_kl_divergence(self): paddle.distribution.kl_divergence(self.p, self.q), self.scipy_kl_beta_beta(self.a1, self.b1, self.a2, self.b2), rtol=config.RTOL.get(str(self.a1.dtype)), - atol=config.ATOL.get(str(self.a1.dtype))) + atol=config.ATOL.get(str(self.a1.dtype)), + ) def scipy_kl_beta_beta(self, a1, b1, a2, b2): - return (scipy.special.betaln(a2, b2) - scipy.special.betaln(a1, b1) + - (a1 - a2) * scipy.special.digamma(a1) + - (b1 - b2) * scipy.special.digamma(b1) + - (a2 - a1 + b2 - b1) * scipy.special.digamma(a1 + b1)) + return ( + scipy.special.betaln(a2, b2) + - scipy.special.betaln(a1, b1) + + (a1 - a2) * scipy.special.digamma(a1) + + (b1 - b2) * scipy.special.digamma(b1) + + (a2 - a1 + b2 - b1) * scipy.special.digamma(a1 + b1) + ) @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'conc1', 'conc2'), [ - ('test-regular-input', np.random.random( - (5, 7, 8, 10)), np.random.random((5, 7, 8, 10))), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'conc1', 'conc2'), + [ + ( + 'test-regular-input', + np.random.random((5, 7, 8, 10)), + np.random.random((5, 7, 8, 10)), + ), + ], +) class TestKLDirichletDirichlet(unittest.TestCase): - def setUp(self): self.p = paddle.distribution.Dirichlet(paddle.to_tensor(self.conc1)) self.q = paddle.distribution.Dirichlet(paddle.to_tensor(self.conc2)) @@ -76,17 +92,25 @@ def test_kl_divergence(self): paddle.distribution.kl_divergence(self.p, self.q), self.scipy_kl_diric_diric(self.conc1, self.conc2), rtol=config.RTOL.get(str(self.conc1.dtype)), - atol=config.ATOL.get(str(self.conc1.dtype))) + atol=config.ATOL.get(str(self.conc1.dtype)), + ) def scipy_kl_diric_diric(self, conc1, conc2): return ( - scipy.special.gammaln(np.sum(conc1, -1)) - - scipy.special.gammaln(np.sum(conc2, -1)) - np.sum( - scipy.special.gammaln(conc1) - scipy.special.gammaln(conc2), -1) + scipy.special.gammaln(np.sum(conc1, -1)) + - scipy.special.gammaln(np.sum(conc2, -1)) + - np.sum( + scipy.special.gammaln(conc1) - scipy.special.gammaln(conc2), -1 + ) + np.sum( - (conc1 - conc2) * - (scipy.special.digamma(conc1) - - scipy.special.digamma(np.sum(conc1, -1, keepdims=True))), -1)) + (conc1 - conc2) + * ( + scipy.special.digamma(conc1) + - scipy.special.digamma(np.sum(conc1, -1, keepdims=True)) + ), + -1, + ) + ) class DummyDistribution(paddle.distribution.Distribution): @@ -94,10 +118,11 @@ class DummyDistribution(paddle.distribution.Distribution): @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'p', 'q'), - [('test-unregister', DummyDistribution(), DummyDistribution)]) +@param.param_cls( + (param.TEST_CASE_NAME, 'p', 'q'), + [('test-unregister', DummyDistribution(), DummyDistribution)], +) class TestDispatch(unittest.TestCase): - def test_dispatch_with_unregister(self): with self.assertRaises(NotImplementedError): paddle.distribution.kl_divergence(self.p, self.q) @@ -106,18 +131,27 @@ def test_dispatch_with_unregister(self): @param.place(config.DEVICES) @param.param_cls( (param.TEST_CASE_NAME, 'p', 'q'), - [('test-diff-dist', mock.Exponential(paddle.rand((100, 200, 100)) + 1.0), - mock.Exponential(paddle.rand((100, 200, 100)) + 2.0)), - ('test-same-dist', mock.Exponential( - paddle.to_tensor(1.0)), mock.Exponential(paddle.to_tensor(1.0)))]) + [ + ( + 'test-diff-dist', + mock.Exponential(paddle.rand((100, 200, 100)) + 1.0), + mock.Exponential(paddle.rand((100, 200, 100)) + 2.0), + ), + ( + 'test-same-dist', + mock.Exponential(paddle.to_tensor(1.0)), + mock.Exponential(paddle.to_tensor(1.0)), + ), + ], +) class TestKLExpfamilyExpFamily(unittest.TestCase): - def test_kl_expfamily_expfamily(self): - np.testing.assert_allclose(paddle.distribution.kl_divergence( - self.p, self.q), - kl._kl_expfamily_expfamily(self.p, self.q), - rtol=config.RTOL.get(config.DEFAULT_DTYPE), - atol=config.ATOL.get(config.DEFAULT_DTYPE)) + np.testing.assert_allclose( + paddle.distribution.kl_divergence(self.p, self.q), + kl._kl_expfamily_expfamily(self.p, self.q), + rtol=config.RTOL.get(config.DEFAULT_DTYPE), + atol=config.ATOL.get(config.DEFAULT_DTYPE), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/distribution/test_kl_static.py b/python/paddle/fluid/tests/unittests/distribution/test_kl_static.py index f5e354e85de58..ea1f8a4aab239 100644 --- a/python/paddle/fluid/tests/unittests/distribution/test_kl_static.py +++ b/python/paddle/fluid/tests/unittests/distribution/test_kl_static.py @@ -30,14 +30,19 @@ @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'a1', 'b1', 'a2', 'b2'), [ - ('test_regular_input', 6.0 * np.random.random( - (4, 5)) + 1e-4, 6.0 * np.random.random( - (4, 5)) + 1e-4, 6.0 * np.random.random( - (4, 5)) + 1e-4, 6.0 * np.random.random((4, 5)) + 1e-4), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'a1', 'b1', 'a2', 'b2'), + [ + ( + 'test_regular_input', + 6.0 * np.random.random((4, 5)) + 1e-4, + 6.0 * np.random.random((4, 5)) + 1e-4, + 6.0 * np.random.random((4, 5)) + 1e-4, + 6.0 * np.random.random((4, 5)) + 1e-4, + ), + ], +) class TestKLBetaBeta(unittest.TestCase): - def setUp(self): self.mp = paddle.static.Program() self.sp = paddle.static.Program() @@ -55,46 +60,57 @@ def setUp(self): 'a1': self.a1, 'b1': self.b1, 'a2': self.a2, - 'b2': self.b2 + 'b2': self.b2, } def test_kl_divergence(self): with paddle.static.program_guard(self.mp, self.sp): out = paddle.distribution.kl_divergence(self.p, self.q) self.executor.run(self.sp) - [out] = self.executor.run(self.mp, - feed=self.feeds, - fetch_list=[out]) + [out] = self.executor.run( + self.mp, feed=self.feeds, fetch_list=[out] + ) - np.testing.assert_allclose(out, - self.scipy_kl_beta_beta( - self.a1, self.b1, self.a2, self.b2), - rtol=config.RTOL.get(str(self.a1.dtype)), - atol=config.ATOL.get(str(self.a1.dtype))) + np.testing.assert_allclose( + out, + self.scipy_kl_beta_beta(self.a1, self.b1, self.a2, self.b2), + rtol=config.RTOL.get(str(self.a1.dtype)), + atol=config.ATOL.get(str(self.a1.dtype)), + ) def scipy_kl_beta_beta(self, a1, b1, a2, b2): - return (scipy.special.betaln(a2, b2) - scipy.special.betaln(a1, b1) + - (a1 - a2) * scipy.special.digamma(a1) + - (b1 - b2) * scipy.special.digamma(b1) + - (a2 - a1 + b2 - b1) * scipy.special.digamma(a1 + b1)) + return ( + scipy.special.betaln(a2, b2) + - scipy.special.betaln(a1, b1) + + (a1 - a2) * scipy.special.digamma(a1) + + (b1 - b2) * scipy.special.digamma(b1) + + (a2 - a1 + b2 - b1) * scipy.special.digamma(a1 + b1) + ) @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'conc1', 'conc2'), [ - ('test-regular-input', np.random.random( - (5, 7, 8, 10)), np.random.random((5, 7, 8, 10))), -]) +@param.param_cls( + (param.TEST_CASE_NAME, 'conc1', 'conc2'), + [ + ( + 'test-regular-input', + np.random.random((5, 7, 8, 10)), + np.random.random((5, 7, 8, 10)), + ), + ], +) class TestKLDirichletDirichlet(unittest.TestCase): - def setUp(self): self.mp = paddle.static.Program() self.sp = paddle.static.Program() self.executor = paddle.static.Executor(self.place) with paddle.static.program_guard(self.mp, self.sp): - conc1 = paddle.static.data('conc1', self.conc1.shape, - self.conc1.dtype) - conc2 = paddle.static.data('conc2', self.conc2.shape, - self.conc2.dtype) + conc1 = paddle.static.data( + 'conc1', self.conc1.shape, self.conc1.dtype + ) + conc2 = paddle.static.data( + 'conc2', self.conc2.shape, self.conc2.dtype + ) self.p = paddle.distribution.Dirichlet(conc1) self.q = paddle.distribution.Dirichlet(conc2) self.feeds = {'conc1': self.conc1, 'conc2': self.conc2} @@ -104,24 +120,32 @@ def test_kl_divergence(self): with paddle.static.program_guard(self.mp, self.sp): out = paddle.distribution.kl_divergence(self.p, self.q) self.executor.run(self.sp) - [out] = self.executor.run(self.mp, - feed=self.feeds, - fetch_list=[out]) + [out] = self.executor.run( + self.mp, feed=self.feeds, fetch_list=[out] + ) np.testing.assert_allclose( out, self.scipy_kl_diric_diric(self.conc1, self.conc2), rtol=config.RTOL.get(str(self.conc1.dtype)), - atol=config.ATOL.get(str(self.conc1.dtype))) + atol=config.ATOL.get(str(self.conc1.dtype)), + ) def scipy_kl_diric_diric(self, conc1, conc2): return ( - scipy.special.gammaln(np.sum(conc1, -1)) - - scipy.special.gammaln(np.sum(conc2, -1)) - np.sum( - scipy.special.gammaln(conc1) - scipy.special.gammaln(conc2), -1) + scipy.special.gammaln(np.sum(conc1, -1)) + - scipy.special.gammaln(np.sum(conc2, -1)) + - np.sum( + scipy.special.gammaln(conc1) - scipy.special.gammaln(conc2), -1 + ) + np.sum( - (conc1 - conc2) * - (scipy.special.digamma(conc1) - - scipy.special.digamma(np.sum(conc1, -1, keepdims=True))), -1)) + (conc1 - conc2) + * ( + scipy.special.digamma(conc1) + - scipy.special.digamma(np.sum(conc1, -1, keepdims=True)) + ), + -1, + ) + ) class DummyDistribution(paddle.distribution.Distribution): @@ -129,10 +153,10 @@ class DummyDistribution(paddle.distribution.Distribution): @param.place(config.DEVICES) -@param.param_cls((param.TEST_CASE_NAME, 'p', 'q'), - [('test-dispatch-exception')]) +@param.param_cls( + (param.TEST_CASE_NAME, 'p', 'q'), [('test-dispatch-exception')] +) class TestDispatch(unittest.TestCase): - def setUp(self): self.mp = paddle.static.Program() self.sp = paddle.static.Program() @@ -150,23 +174,29 @@ def test_dispatch_with_unregister(self): @param.place(config.DEVICES) -@param.param_cls((config.TEST_CASE_NAME, 'rate1', 'rate2'), - [('test-diff-dist', np.random.rand(100, 200, 100) + 1.0, - np.random.rand(100, 200, 100) + 2.0), - ('test-same-dist', np.array([1.0]), np.array([1.0]))]) +@param.param_cls( + (config.TEST_CASE_NAME, 'rate1', 'rate2'), + [ + ( + 'test-diff-dist', + np.random.rand(100, 200, 100) + 1.0, + np.random.rand(100, 200, 100) + 2.0, + ), + ('test-same-dist', np.array([1.0]), np.array([1.0])), + ], +) class TestKLExpfamilyExpFamily(unittest.TestCase): - def setUp(self): self.mp = paddle.static.Program() self.sp = paddle.static.Program() self.executor = paddle.static.Executor(self.place) with paddle.static.program_guard(self.mp, self.sp): - rate1 = paddle.static.data('rate1', - shape=self.rate1.shape, - dtype=self.rate1.dtype) - rate2 = paddle.static.data('rate2', - shape=self.rate2.shape, - dtype=self.rate2.dtype) + rate1 = paddle.static.data( + 'rate1', shape=self.rate1.shape, dtype=self.rate1.dtype + ) + rate2 = paddle.static.data( + 'rate2', shape=self.rate2.shape, dtype=self.rate2.dtype + ) self.p = mock.Exponential(rate1) self.q = mock.Exponential(rate2) self.feeds = {'rate1': self.rate1, 'rate2': self.rate2} @@ -176,15 +206,16 @@ def test_kl_expfamily_expfamily(self): out1 = paddle.distribution.kl_divergence(self.p, self.q) out2 = kl._kl_expfamily_expfamily(self.p, self.q) self.executor.run(self.sp) - [out1, out2] = self.executor.run(self.mp, - feed=self.feeds, - fetch_list=[out1, out2]) + [out1, out2] = self.executor.run( + self.mp, feed=self.feeds, fetch_list=[out1, out2] + ) np.testing.assert_allclose( out1, out2, rtol=config.RTOL.get(config.DEFAULT_DTYPE), - atol=config.ATOL.get(config.DEFAULT_DTYPE)) + atol=config.ATOL.get(config.DEFAULT_DTYPE), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_fleet_api.py b/python/paddle/fluid/tests/unittests/dygraph_fleet_api.py index bf6139ffae680..a2834516d7166 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_fleet_api.py +++ b/python/paddle/fluid/tests/unittests/dygraph_fleet_api.py @@ -21,7 +21,6 @@ class TestDygraphFleetAPI(unittest.TestCase): - def setUp(self): paddle.seed(2022) random.seed(2022) @@ -35,12 +34,14 @@ def config(self): def test_dygraph_fleet_api(self): import paddle.distributed.fleet as fleet import paddle.distributed as dist + strategy = fleet.DistributedStrategy() strategy.amp = True strategy.recompute = True fleet.init(is_collective=True, strategy=strategy) - net = paddle.nn.Sequential(paddle.nn.Linear(10, 1), - paddle.nn.Linear(1, 2)) + net = paddle.nn.Sequential( + paddle.nn.Linear(10, 1), paddle.nn.Linear(1, 2) + ) net = dist.fleet.distributed_model(net) data = np.random.uniform(-1, 1, [30, 10]).astype('float32') data = paddle.to_tensor(data) diff --git a/python/paddle/fluid/tests/unittests/dygraph_recompute_hybrid.py b/python/paddle/fluid/tests/unittests/dygraph_recompute_hybrid.py index 105621498fed6..7dbd6b6d7ef35 100755 --- a/python/paddle/fluid/tests/unittests/dygraph_recompute_hybrid.py +++ b/python/paddle/fluid/tests/unittests/dygraph_recompute_hybrid.py @@ -24,34 +24,40 @@ def get_fc_block(block_idx, input_size, is_last=False): block_name = "block_" + str(block_idx) block = paddle.nn.Sequential( - (block_name + "_fc_0", - paddle.nn.Linear(input_size, input_size, bias_attr=False)), + ( + block_name + "_fc_0", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ), (block_name + "_dropout", paddle.nn.Dropout(p=0.5)), (block_name + "_relu_1", paddle.nn.ReLU()), - (block_name + "_fc_1", - paddle.nn.Linear(input_size, input_size, bias_attr=False)), + ( + block_name + "_fc_1", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ), (block_name + "_relu_2", paddle.nn.ReLU()), ) if is_last: - block.add_sublayer(block_name + "_fc_2", - paddle.nn.Linear(input_size, 1, - bias_attr=False)) # add sublayer + block.add_sublayer( + block_name + "_fc_2", + paddle.nn.Linear(input_size, 1, bias_attr=False), + ) # add sublayer else: - block.add_sublayer(block_name + "_fc_2", - paddle.nn.Linear(input_size, - input_size, - bias_attr=False)) # add sublayer + block.add_sublayer( + block_name + "_fc_2", + paddle.nn.Linear(input_size, input_size, bias_attr=False), + ) # add sublayer return block class Naive_fc_net(paddle.nn.Layer): - - def __init__(self, - input_size=10, - recompute_blocks=[1, 3], - offload=False, - partition=False, - recompute_kwargs={}): + def __init__( + self, + input_size=10, + recompute_blocks=[1, 3], + offload=False, + partition=False, + recompute_kwargs={}, + ): super(Naive_fc_net, self).__init__() self.recompute_blocks = recompute_blocks self.recompute_kwargs = recompute_kwargs @@ -65,8 +71,11 @@ def __init__(self, self.runfunc4 = get_fc_block(4, input_size, is_last=True) self.layers = [ - self.runfunc0, self.runfunc1, self.runfunc2, self.runfunc3, - self.runfunc4 + self.runfunc0, + self.runfunc1, + self.runfunc2, + self.runfunc3, + self.runfunc4, ] def forward(self, inputs): @@ -76,34 +85,43 @@ def forward(self, inputs): { "mp_group": fleet.fleet._hcg.get_model_parallel_group(), "offload": self.offload, - "partition": self.partition - }, self.layers[i], inputs, **self.recompute_kwargs) + "partition": self.partition, + }, + self.layers[i], + inputs, + **self.recompute_kwargs + ) else: inputs = self.layers[i](inputs) return inputs -def run_model(recompute_block=[], - recompute_kwargs={}, - offload=False, - partition=False, - enable_autocast=False, - pure_fp16=False): +def run_model( + recompute_block=[], + recompute_kwargs={}, + offload=False, + partition=False, + enable_autocast=False, + pure_fp16=False, +): gen = paddle.seed(10) gen.manual_seed(10) np.random.seed(10) random.seed(10) batch_size, input_size = 1, 10 - model = Naive_fc_net(input_size, - recompute_blocks=recompute_block, - offload=offload, - partition=partition, - recompute_kwargs=recompute_kwargs) + model = Naive_fc_net( + input_size, + recompute_blocks=recompute_block, + offload=offload, + partition=partition, + recompute_kwargs=recompute_kwargs, + ) loss_fn = paddle.nn.MSELoss(reduction='mean') - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, parameters=model.parameters() + ) model = fleet.distributed_model(model) optimizer = fleet.distributed_optimizer(optimizer) @@ -140,7 +158,6 @@ def run_model(recompute_block=[], class TestPyLayer(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 2 @@ -154,7 +171,6 @@ def setUp(self): fleet.init(is_collective=True, strategy=strategy) def test_base_case(self, enable_autocast=False, pure_fp16=False): - def check_identical(loss_ref, param_ref, grad_ref, loss, param, grad): self.assertEqual(loss_ref, loss) self.assertEqual(param_ref, param) @@ -164,34 +180,43 @@ def check_identical(loss_ref, param_ref, grad_ref, loss, param, grad): loss_ref, param_ref, grad_ref = run_model( recompute_block=[], enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + pure_fp16=pure_fp16, + ) # with recompute, offload=False, partition=False - loss, param, grad = run_model(recompute_block=[1, 3], - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 3], + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # with recompute, offload=True, partition=False - loss, param, grad = run_model(recompute_block=[1, 2, 3], - offload=True, - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 2, 3], + offload=True, + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # with recompute, offload=False, partition=True - loss, param, grad = run_model(recompute_block=[1], - partition=True, - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1], + partition=True, + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) # with recompute, offload=True, partition=True - loss, param, grad = run_model(recompute_block=[1, 3, 4], - offload=True, - partition=True, - enable_autocast=enable_autocast, - pure_fp16=pure_fp16) + loss, param, grad = run_model( + recompute_block=[1, 3, 4], + offload=True, + partition=True, + enable_autocast=enable_autocast, + pure_fp16=pure_fp16, + ) check_identical(loss_ref, param_ref, grad_ref, loss, param, grad) def test_fc_net_with_dropout(self): @@ -207,8 +232,9 @@ def test_recompute_kwargs(self): paddle.set_device("gpu") kwargs = {"is_test": False} with self.assertRaises(TypeError): - loss_ref, param_ref, grad_ref = run_model(recompute_block=[2], - recompute_kwargs=kwargs) + loss_ref, param_ref, grad_ref = run_model( + recompute_block=[2], recompute_kwargs=kwargs + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_dygraph_model.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_dygraph_model.py index 60c2a4ef0670c..d9ccc587f7c05 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_dygraph_model.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_dygraph_model.py @@ -21,89 +21,105 @@ class PositionwiseFeedForwardLayer(Layer): - - def __init__(self, - hidden_act, - d_inner_hid, - d_model, - dropout_rate, - param_initializer=None, - name=""): + def __init__( + self, + hidden_act, + d_inner_hid, + d_model, + dropout_rate, + param_initializer=None, + name="", + ): super(PositionwiseFeedForwardLayer, self).__init__() - self._i2h = Linear(input_dim=d_model, - output_dim=d_inner_hid, - param_attr=fluid.ParamAttr( - name=name + '_fc_0.w_0', - initializer=param_initializer), - bias_attr=name + '_fc_0.b_0', - act=hidden_act) - - self._h2o = Linear(input_dim=d_inner_hid, - output_dim=d_model, - param_attr=fluid.ParamAttr( - name=name + '_fc_1.w_0', - initializer=param_initializer), - bias_attr=name + '_fc_1.b_0') + self._i2h = Linear( + input_dim=d_model, + output_dim=d_inner_hid, + param_attr=fluid.ParamAttr( + name=name + '_fc_0.w_0', initializer=param_initializer + ), + bias_attr=name + '_fc_0.b_0', + act=hidden_act, + ) + + self._h2o = Linear( + input_dim=d_inner_hid, + output_dim=d_model, + param_attr=fluid.ParamAttr( + name=name + '_fc_1.w_0', initializer=param_initializer + ), + bias_attr=name + '_fc_1.b_0', + ) self._dropout_rate = dropout_rate def forward(self, x): hidden = self._i2h(x) if self._dropout_rate: - hidden = fluid.layers.dropout(hidden, - dropout_prob=self._dropout_rate, - is_test=False) + hidden = fluid.layers.dropout( + hidden, dropout_prob=self._dropout_rate, is_test=False + ) out = self._h2o(hidden) return out class EncoderSubLayer(Layer): - - def __init__(self, - hidden_act, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da", - param_initializer=None, - name=""): + def __init__( + self, + hidden_act, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + param_initializer=None, + name="", + ): super(EncoderSubLayer, self).__init__() self.name = name self._preprocess_cmd = preprocess_cmd self._postprocess_cmd = postprocess_cmd self._prepostprocess_dropout = prepostprocess_dropout - self._preprocess_layer = PrePostProcessLayer(self._preprocess_cmd, - d_model, - prepostprocess_dropout) + self._preprocess_layer = PrePostProcessLayer( + self._preprocess_cmd, d_model, prepostprocess_dropout + ) self._multihead_attention_layer = MultiHeadAttention( - d_key, d_value, d_model, n_head, attention_dropout, - param_initializer) + d_key, + d_value, + d_model, + n_head, + attention_dropout, + param_initializer, + ) self._postprocess_layer = PrePostProcessLayer( - self._postprocess_cmd, d_model, self._prepostprocess_dropout) + self._postprocess_cmd, d_model, self._prepostprocess_dropout + ) self._preprocess_layer2 = PrePostProcessLayer( - self._preprocess_cmd, d_model, self._prepostprocess_dropout) + self._preprocess_cmd, d_model, self._prepostprocess_dropout + ) self._positionwise_feed_forward = PositionwiseFeedForwardLayer( hidden_act, d_inner_hid, d_model, relu_dropout, param_initializer, - name=name + "_ffn") + name=name + "_ffn", + ) self._postprocess_layer2 = PrePostProcessLayer( - self._postprocess_cmd, d_model, self._prepostprocess_dropout) + self._postprocess_cmd, d_model, self._prepostprocess_dropout + ) def forward(self, enc_input, attn_bias): pre_process_multihead = self._preprocess_layer(enc_input) - attn_output = self._multihead_attention_layer(pre_process_multihead, - None, None, attn_bias) + attn_output = self._multihead_attention_layer( + pre_process_multihead, None, None, attn_bias + ) attn_output = self._postprocess_layer(attn_output, enc_input) pre_process2_output = self._preprocess_layer2(attn_output) ffd_output = self._positionwise_feed_forward(pre_process2_output) @@ -111,22 +127,23 @@ def forward(self, enc_input, attn_bias): class EncoderLayer(Layer): - - def __init__(self, - hidden_act, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da", - param_initializer=None, - name=""): + def __init__( + self, + hidden_act, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + param_initializer=None, + name="", + ): super(EncoderLayer, self).__init__() self._preprocess_cmd = preprocess_cmd @@ -135,25 +152,30 @@ def __init__(self, self._n_layer = n_layer self._hidden_act = hidden_act self._preprocess_layer = PrePostProcessLayer( - self._preprocess_cmd, 3, self._prepostprocess_dropout) + self._preprocess_cmd, 3, self._prepostprocess_dropout + ) for i in range(n_layer): self._encoder_sublayers.append( self.add_sublayer( 'esl_%d' % i, - EncoderSubLayer(hidden_act, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - param_initializer, - name=name + '_layer_' + str(i)))) + EncoderSubLayer( + hidden_act, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + param_initializer, + name=name + '_layer_' + str(i), + ), + ) + ) def forward(self, enc_input, attn_bias): for i in range(self._n_layer): @@ -164,7 +186,6 @@ def forward(self, enc_input, attn_bias): class BertModelLayer(Layer): - def __init__(self, config, return_pooled_out=True, use_fp16=False): super(BertModelLayer, self).__init__() @@ -185,36 +206,46 @@ def __init__(self, config, return_pooled_out=True, use_fp16=False): self._dtype = "float16" if use_fp16 else "float32" self._param_initializer = fluid.initializer.TruncatedNormal( - scale=config['initializer_range']) + scale=config['initializer_range'] + ) - self._src_emb = Embedding(size=[self._voc_size, self._emb_size], - param_attr=fluid.ParamAttr( - name=self._word_emb_name, - initializer=self._param_initializer), - dtype=self._dtype) + self._src_emb = Embedding( + size=[self._voc_size, self._emb_size], + param_attr=fluid.ParamAttr( + name=self._word_emb_name, initializer=self._param_initializer + ), + dtype=self._dtype, + ) self._pos_emb = Embedding( size=[self._max_position_seq_len, self._emb_size], - param_attr=fluid.ParamAttr(name=self._pos_emb_name, - initializer=self._param_initializer), - dtype=self._dtype) - - self._sent_emb = Embedding(size=[self._sent_types, self._emb_size], - param_attr=fluid.ParamAttr( - name=self._sent_emb_name, - initializer=self._param_initializer), - dtype=self._dtype) - - self.pooled_fc = Linear(input_dim=self._emb_size, - output_dim=self._emb_size, - param_attr=fluid.ParamAttr( - name="pooled_fc.w_0", - initializer=self._param_initializer), - bias_attr="pooled_fc.b_0", - act="tanh") + param_attr=fluid.ParamAttr( + name=self._pos_emb_name, initializer=self._param_initializer + ), + dtype=self._dtype, + ) + + self._sent_emb = Embedding( + size=[self._sent_types, self._emb_size], + param_attr=fluid.ParamAttr( + name=self._sent_emb_name, initializer=self._param_initializer + ), + dtype=self._dtype, + ) + + self.pooled_fc = Linear( + input_dim=self._emb_size, + output_dim=self._emb_size, + param_attr=fluid.ParamAttr( + name="pooled_fc.w_0", initializer=self._param_initializer + ), + bias_attr="pooled_fc.b_0", + act="tanh", + ) self.pre_process_layer = PrePostProcessLayer( - "nd", self._emb_size, self._prepostprocess_dropout) + "nd", self._emb_size, self._prepostprocess_dropout + ) self._encoder = EncoderLayer( hidden_act=self._hidden_act, @@ -229,7 +260,8 @@ def __init__(self, config, return_pooled_out=True, use_fp16=False): relu_dropout=0, preprocess_cmd="", postprocess_cmd="dan", - param_initializer=self._param_initializer) + param_initializer=self._param_initializer, + ) def forward(self, src_ids, position_ids, sentence_ids, input_mask): src_emb = self._src_emb(src_ids) @@ -241,16 +273,15 @@ def forward(self, src_ids, position_ids, sentence_ids, input_mask): emb_out = self.pre_process_layer(emb_out) - self_attn_mask = fluid.layers.matmul(x=input_mask, - y=input_mask, - transpose_y=True) - self_attn_mask = fluid.layers.scale(x=self_attn_mask, - scale=10000.0, - bias=-1.0, - bias_after_scale=False) - n_head_self_attn_mask = fluid.layers.stack(x=[self_attn_mask] * - self._n_head, - axis=1) + self_attn_mask = fluid.layers.matmul( + x=input_mask, y=input_mask, transpose_y=True + ) + self_attn_mask = fluid.layers.scale( + x=self_attn_mask, scale=10000.0, bias=-1.0, bias_after_scale=False + ) + n_head_self_attn_mask = fluid.layers.stack( + x=[self_attn_mask] * self._n_head, axis=1 + ) n_head_self_attn_mask.stop_gradient = True enc_output = self._encoder(emb_out, n_head_self_attn_mask) @@ -258,26 +289,27 @@ def forward(self, src_ids, position_ids, sentence_ids, input_mask): # TODO(zhhsplendid): uncomment this in next PR which we support various # length of early return # - #if not self.return_pooled_out: + # if not self.return_pooled_out: # return enc_output - next_sent_feat = fluid.layers.slice(input=enc_output, - axes=[1], - starts=[0], - ends=[1]) + next_sent_feat = fluid.layers.slice( + input=enc_output, axes=[1], starts=[0], ends=[1] + ) next_sent_feat = self.pooled_fc(next_sent_feat) - next_sent_feat = fluid.layers.reshape(next_sent_feat, - shape=[-1, self._emb_size]) + next_sent_feat = fluid.layers.reshape( + next_sent_feat, shape=[-1, self._emb_size] + ) return enc_output, next_sent_feat class PretrainModelLayer(Layer): - - def __init__(self, - config, - return_pooled_out=True, - weight_sharing=False, - use_fp16=False): + def __init__( + self, + config, + return_pooled_out=True, + weight_sharing=False, + use_fp16=False, + ): super(PretrainModelLayer, self).__init__() self.config = config self._voc_size = config['vocab_size'] @@ -287,84 +319,113 @@ def __init__(self, self._word_emb_name = "word_embedding" self._param_initializer = fluid.initializer.TruncatedNormal( - scale=config['initializer_range']) + scale=config['initializer_range'] + ) self._weight_sharing = weight_sharing self.use_fp16 = use_fp16 self._dtype = "float16" if use_fp16 else "float32" - self.bert_layer = BertModelLayer(config=self.config, - return_pooled_out=True, - use_fp16=self.use_fp16) + self.bert_layer = BertModelLayer( + config=self.config, return_pooled_out=True, use_fp16=self.use_fp16 + ) self.pre_process_layer = PrePostProcessLayer( - "n", self._emb_size, self._prepostprocess_dropout) - - self.pooled_fc = Linear(input_dim=self._emb_size, - output_dim=self._emb_size, - param_attr=fluid.ParamAttr( - name="mask_lm_trans_fc.w_0", - initializer=self._param_initializer), - bias_attr="mask_lm_trans_fc.b_0", - act="tanh") + "n", self._emb_size, self._prepostprocess_dropout + ) + + self.pooled_fc = Linear( + input_dim=self._emb_size, + output_dim=self._emb_size, + param_attr=fluid.ParamAttr( + name="mask_lm_trans_fc.w_0", initializer=self._param_initializer + ), + bias_attr="mask_lm_trans_fc.b_0", + act="tanh", + ) self.mask_lm_out_bias_attr = fluid.ParamAttr( name="mask_lm_out_fc.b_0", - initializer=fluid.initializer.Constant(value=0.0)) + initializer=fluid.initializer.Constant(value=0.0), + ) if not self._weight_sharing: - self.out_fc = Linear(input_dim=self._emb_size, - output_dim=self._voc_size, - param_attr=fluid.ParamAttr( - name="mask_lm_out_fc.w_0", - initializer=self._param_initializer), - bias_attr=self.mask_lm_out_bias_attr) + self.out_fc = Linear( + input_dim=self._emb_size, + output_dim=self._voc_size, + param_attr=fluid.ParamAttr( + name="mask_lm_out_fc.w_0", + initializer=self._param_initializer, + ), + bias_attr=self.mask_lm_out_bias_attr, + ) else: self.fc_create_params = self.create_parameter( shape=[self._voc_size], dtype=self._dtype, attr=self.mask_lm_out_bias_attr, - is_bias=True) - - self.next_sent_fc = Linear(input_dim=self._emb_size, - output_dim=2, - param_attr=fluid.ParamAttr( - name="next_sent_fc.w_0", - initializer=self._param_initializer), - bias_attr="next_sent_fc.b_0") + is_bias=True, + ) + + self.next_sent_fc = Linear( + input_dim=self._emb_size, + output_dim=2, + param_attr=fluid.ParamAttr( + name="next_sent_fc.w_0", initializer=self._param_initializer + ), + bias_attr="next_sent_fc.b_0", + ) @declarative - def forward(self, src_ids, position_ids, sentence_ids, input_mask, - mask_label, mask_pos, labels): + def forward( + self, + src_ids, + position_ids, + sentence_ids, + input_mask, + mask_label, + mask_pos, + labels, + ): mask_pos = fluid.layers.cast(x=mask_pos, dtype='int32') - enc_output, next_sent_feat = self.bert_layer(src_ids, position_ids, - sentence_ids, input_mask) - reshaped_emb_out = fluid.layers.reshape(x=enc_output, - shape=[-1, self._emb_size]) + enc_output, next_sent_feat = self.bert_layer( + src_ids, position_ids, sentence_ids, input_mask + ) + reshaped_emb_out = fluid.layers.reshape( + x=enc_output, shape=[-1, self._emb_size] + ) mask_feat = fluid.layers.gather(input=reshaped_emb_out, index=mask_pos) mask_trans_feat = self.pooled_fc(mask_feat) mask_trans_feat = self.pre_process_layer(mask_trans_feat) if self._weight_sharing: - fc_out = fluid.layers.matmul(x=mask_trans_feat, - y=self.bert_layer._src_emb._w, - transpose_y=True) + fc_out = fluid.layers.matmul( + x=mask_trans_feat, + y=self.bert_layer._src_emb._w, + transpose_y=True, + ) fc_out += self.fc_create_params else: fc_out = self.out_fc(mask_trans_feat) - mask_lm_loss = fluid.layers.softmax_with_cross_entropy(logits=fc_out, - label=mask_label) + mask_lm_loss = fluid.layers.softmax_with_cross_entropy( + logits=fc_out, label=mask_label + ) mean_mask_lm_loss = paddle.mean(mask_lm_loss) next_sent_fc_out = self.next_sent_fc(next_sent_feat) - next_sent_loss, next_sent_softmax = fluid.layers.softmax_with_cross_entropy( - logits=next_sent_fc_out, label=labels, return_softmax=True) + ( + next_sent_loss, + next_sent_softmax, + ) = fluid.layers.softmax_with_cross_entropy( + logits=next_sent_fc_out, label=labels, return_softmax=True + ) - next_sent_acc = fluid.layers.accuracy(input=next_sent_softmax, - label=labels) + next_sent_acc = fluid.layers.accuracy( + input=next_sent_softmax, label=labels + ) mean_next_sent_loss = paddle.mean(next_sent_loss) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_utils.py index 3fb23049f1b81..0cc22162d202c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/bert_utils.py @@ -35,7 +35,7 @@ def get_bert_config(): "pooler_size_per_head": 8, "pooler_type": "first_token_transform", "type_vocab_size": 2, - "vocab_size": 21128 + "vocab_size": 21128, } return bert_config @@ -87,8 +87,9 @@ def mask(batch_tokens, total_token_num, vocab_size, CLS=1, SEP=2, MASK=3): # ensure at least mask one word in a sentence while not mask_flag: - token_index = int(self_random.randint(1, high=len(sent) - 1, - size=1)) + token_index = int( + self_random.randint(1, high=len(sent) - 1, size=1) + ) if sent[token_index] != SEP and sent[token_index] != CLS: mask_label.append(sent[token_index]) sent[token_index] = MASK @@ -99,12 +100,14 @@ def mask(batch_tokens, total_token_num, vocab_size, CLS=1, SEP=2, MASK=3): return batch_tokens, mask_label, mask_pos -def pad_batch_data(insts, - pad_idx=0, - return_pos=False, - return_input_mask=False, - return_max_len=False, - return_num_token=False): +def pad_batch_data( + insts, + pad_idx=0, + return_pos=False, + return_input_mask=False, + return_max_len=False, + return_num_token=False, +): """ Pad the instances to the max sequence length in batch, and generate the corresponding position data and input mask. @@ -114,24 +117,27 @@ def pad_batch_data(insts, # Any token included in dict can be used to pad, since the paddings' loss # will be masked out by weights and make no effect on parameter gradients. - inst_data = np.array([ - list(inst) + list([pad_idx] * (max_len - len(inst))) for inst in insts - ]) + inst_data = np.array( + [list(inst) + list([pad_idx] * (max_len - len(inst))) for inst in insts] + ) return_list += [inst_data.astype("int64").reshape([-1, max_len])] # position data if return_pos: - inst_pos = np.array([ - list(range(0, len(inst))) + [pad_idx] * (max_len - len(inst)) - for inst in insts - ]) + inst_pos = np.array( + [ + list(range(0, len(inst))) + [pad_idx] * (max_len - len(inst)) + for inst in insts + ] + ) return_list += [inst_pos.astype("int64").reshape([-1, max_len])] if return_input_mask: # This is used to avoid attention on paddings. input_mask_data = np.array( - [[1] * len(inst) + [0] * (max_len - len(inst)) for inst in insts]) + [[1] * len(inst) + [0] * (max_len - len(inst)) for inst in insts] + ) input_mask_data = np.expand_dims(input_mask_data, axis=-1) return_list += [input_mask_data.astype("float32")] @@ -147,16 +153,18 @@ def pad_batch_data(insts, return return_list if len(return_list) > 1 else return_list[0] -def prepare_batch_data(insts, - total_token_num, - voc_size=0, - pad_id=None, - cls_id=None, - sep_id=None, - mask_id=None, - return_input_mask=True, - return_max_len=True, - return_num_token=False): +def prepare_batch_data( + insts, + total_token_num, + voc_size=0, + pad_id=None, + cls_id=None, + sep_id=None, + mask_id=None, + return_input_mask=True, + return_max_len=True, + return_num_token=False, +): """ 1. generate Tensor of data 2. generate Tensor of position @@ -175,30 +183,38 @@ def prepare_batch_data(insts, # First step: do mask without padding if mask_id >= 0: - out, mask_label, mask_pos = mask(batch_src_ids, - total_token_num, - vocab_size=voc_size, - CLS=cls_id, - SEP=sep_id, - MASK=mask_id) + out, mask_label, mask_pos = mask( + batch_src_ids, + total_token_num, + vocab_size=voc_size, + CLS=cls_id, + SEP=sep_id, + MASK=mask_id, + ) else: out = batch_src_ids # Second step: padding - src_id, self_input_mask = pad_batch_data(out, - pad_idx=pad_id, - return_input_mask=True) - pos_id = pad_batch_data(batch_pos_ids, - pad_idx=pad_id, - return_pos=False, - return_input_mask=False) - sent_id = pad_batch_data(batch_sent_ids, - pad_idx=pad_id, - return_pos=False, - return_input_mask=False) + src_id, self_input_mask = pad_batch_data( + out, pad_idx=pad_id, return_input_mask=True + ) + pos_id = pad_batch_data( + batch_pos_ids, pad_idx=pad_id, return_pos=False, return_input_mask=False + ) + sent_id = pad_batch_data( + batch_sent_ids, + pad_idx=pad_id, + return_pos=False, + return_input_mask=False, + ) if mask_id >= 0: return_list = [ - src_id, pos_id, sent_id, self_input_mask, mask_label, mask_pos + src_id, + pos_id, + sent_id, + self_input_mask, + mask_label, + mask_pos, ] + labels_list else: return_list = [src_id, pos_id, sent_id, self_input_mask] + labels_list @@ -208,16 +224,17 @@ def prepare_batch_data(insts, class DataReader(object): - - def __init__(self, - batch_size=4096, - in_tokens=True, - max_seq_len=512, - shuffle_files=False, - epoch=100, - voc_size=0, - is_test=False, - generate_neg_sample=False): + def __init__( + self, + batch_size=4096, + in_tokens=True, + max_seq_len=512, + shuffle_files=False, + epoch=100, + voc_size=0, + is_test=False, + generate_neg_sample=False, + ): self.batch_size = batch_size self.in_tokens = in_tokens @@ -237,8 +254,10 @@ def __init__(self, self.is_test = is_test self.generate_neg_sample = generate_neg_sample if self.in_tokens: - assert self.batch_size >= self.max_seq_len, "The number of " \ - "tokens in batch should not be smaller than max seq length." + assert self.batch_size >= self.max_seq_len, ( + "The number of " + "tokens in batch should not be smaller than max seq length." + ) if self.is_test: self.epoch = 1 @@ -253,21 +272,22 @@ def build_fake_data(self): sent0_len = self_random.randint(50, 100) sent1_len = self_random.randint(50, 100) - token_ids = [1] \ - + [self_random.randint(0, 10000) for i in range(sent0_len-1)] \ - + [self_random.randint(0, 10000) for i in range(sent1_len-1)] \ - + [2] + token_ids = ( + [1] + + [self_random.randint(0, 10000) for i in range(sent0_len - 1)] + + [self_random.randint(0, 10000) for i in range(sent1_len - 1)] + + [2] + ) - sent_ids = [0 for i in range(sent0_len) - ] + [1 for i in range(sent1_len)] + sent_ids = [0 for i in range(sent0_len)] + [ + 1 for i in range(sent1_len) + ] pos_ids = [i for i in range(sent0_len + sent1_len)] label = 1 yield token_ids, sent_ids, pos_ids, label def data_generator(self): - def wrapper(): - def reader(): for epoch in range(self.epoch): self.current_epoch = epoch + 1 @@ -291,25 +311,30 @@ def batch_reader(reader, batch_size, in_tokens): total_token_num += len(token_ids) else: yield batch, total_token_num - batch, total_token_num, max_len = [ - parsed_line - ], len(token_ids), len(token_ids) + batch, total_token_num, max_len = ( + [parsed_line], + len(token_ids), + len(token_ids), + ) if len(batch) > 0: yield batch, total_token_num for batch_data, total_token_num in batch_reader( - reader, self.batch_size, self.in_tokens): - yield prepare_batch_data(batch_data, - total_token_num, - voc_size=self.voc_size, - pad_id=self.pad_id, - cls_id=self.cls_id, - sep_id=self.sep_id, - mask_id=self.mask_id, - return_input_mask=True, - return_max_len=False, - return_num_token=False) + reader, self.batch_size, self.in_tokens + ): + yield prepare_batch_data( + batch_data, + total_token_num, + voc_size=self.voc_size, + pad_id=self.pad_id, + cls_id=self.cls_id, + sep_id=self.sep_id, + mask_id=self.mask_id, + return_input_mask=True, + return_max_len=False, + return_num_token=False, + ) return wrapper @@ -324,11 +349,13 @@ class ModelHyperParams(object): def get_feed_data_reader(bert_config): args = ModelHyperParams() - data_reader = DataReader(batch_size=args.batch_size, - in_tokens=args.in_tokens, - voc_size=bert_config['vocab_size'], - epoch=args.epoch, - max_seq_len=args.max_seq_len, - generate_neg_sample=args.generate_neg_sample) + data_reader = DataReader( + batch_size=args.batch_size, + in_tokens=args.in_tokens, + voc_size=bert_config['vocab_size'], + epoch=args.epoch, + max_seq_len=args.max_seq_len, + generate_neg_sample=args.generate_neg_sample, + ) return data_reader diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/darknet.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/darknet.py index d188c693baa0d..ea72368fedb62 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/darknet.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/darknet.py @@ -1,16 +1,16 @@ # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import paddle.fluid as fluid from paddle.fluid.param_attr import ParamAttr @@ -20,35 +20,44 @@ class ConvBNLayer(fluid.dygraph.Layer): - - def __init__(self, - ch_in, - ch_out, - filter_size=3, - stride=1, - groups=1, - padding=0, - act="leaky", - is_test=True): + def __init__( + self, + ch_in, + ch_out, + filter_size=3, + stride=1, + groups=1, + padding=0, + act="leaky", + is_test=True, + ): super(ConvBNLayer, self).__init__() - self.conv = Conv2D(num_channels=ch_in, - num_filters=ch_out, - filter_size=filter_size, - stride=stride, - padding=padding, - groups=groups, - param_attr=ParamAttr( - initializer=fluid.initializer.Normal(0., 0.02)), - bias_attr=False, - act=None) + self.conv = Conv2D( + num_channels=ch_in, + num_filters=ch_out, + filter_size=filter_size, + stride=stride, + padding=padding, + groups=groups, + param_attr=ParamAttr( + initializer=fluid.initializer.Normal(0.0, 0.02) + ), + bias_attr=False, + act=None, + ) self.batch_norm = BatchNorm( num_channels=ch_out, is_test=is_test, - param_attr=ParamAttr(initializer=fluid.initializer.Normal(0., 0.02), - regularizer=L2Decay(0.)), - bias_attr=ParamAttr(initializer=fluid.initializer.Constant(0.0), - regularizer=L2Decay(0.))) + param_attr=ParamAttr( + initializer=fluid.initializer.Normal(0.0, 0.02), + regularizer=L2Decay(0.0), + ), + bias_attr=ParamAttr( + initializer=fluid.initializer.Constant(0.0), + regularizer=L2Decay(0.0), + ), + ) self.act = act @@ -61,23 +70,20 @@ def forward(self, inputs): class DownSample(fluid.dygraph.Layer): - - def __init__(self, - ch_in, - ch_out, - filter_size=3, - stride=2, - padding=1, - is_test=True): + def __init__( + self, ch_in, ch_out, filter_size=3, stride=2, padding=1, is_test=True + ): super(DownSample, self).__init__() - self.conv_bn_layer = ConvBNLayer(ch_in=ch_in, - ch_out=ch_out, - filter_size=filter_size, - stride=stride, - padding=padding, - is_test=is_test) + self.conv_bn_layer = ConvBNLayer( + ch_in=ch_in, + ch_out=ch_out, + filter_size=filter_size, + stride=stride, + padding=padding, + is_test=is_test, + ) self.ch_out = ch_out def forward(self, inputs): @@ -86,22 +92,25 @@ def forward(self, inputs): class BasicBlock(fluid.dygraph.Layer): - def __init__(self, ch_in, ch_out, is_test=True): super(BasicBlock, self).__init__() - self.conv1 = ConvBNLayer(ch_in=ch_in, - ch_out=ch_out, - filter_size=1, - stride=1, - padding=0, - is_test=is_test) - self.conv2 = ConvBNLayer(ch_in=ch_out, - ch_out=ch_out * 2, - filter_size=3, - stride=1, - padding=1, - is_test=is_test) + self.conv1 = ConvBNLayer( + ch_in=ch_in, + ch_out=ch_out, + filter_size=1, + stride=1, + padding=0, + is_test=is_test, + ) + self.conv2 = ConvBNLayer( + ch_in=ch_out, + ch_out=ch_out * 2, + filter_size=3, + stride=1, + padding=1, + is_test=is_test, + ) def forward(self, inputs): conv1 = self.conv1(inputs) @@ -111,7 +120,6 @@ def forward(self, inputs): class LayerWarp(fluid.dygraph.Layer): - def __init__(self, ch_in, ch_out, count, is_test=True): super(LayerWarp, self).__init__() @@ -120,7 +128,8 @@ def __init__(self, ch_in, ch_out, count, is_test=True): for i in range(1, count): res_out = self.add_sublayer( "basic_block_%d" % (i), - BasicBlock(ch_out * 2, ch_out, is_test=is_test)) + BasicBlock(ch_out * 2, ch_out, is_test=is_test), + ) self.res_out_list.append(res_out) self.ch_out = ch_out @@ -135,18 +144,19 @@ def forward(self, inputs): class DarkNet53_conv_body(fluid.dygraph.Layer): - def __init__(self, ch_in=3, is_test=True): super(DarkNet53_conv_body, self).__init__() self.stages = DarkNet_cfg[53] self.stages = self.stages[0:5] - self.conv0 = ConvBNLayer(ch_in=ch_in, - ch_out=32, - filter_size=3, - stride=1, - padding=1, - is_test=is_test) + self.conv0 = ConvBNLayer( + ch_in=ch_in, + ch_out=32, + filter_size=3, + stride=1, + padding=1, + is_test=is_test, + ) self.downsample0 = DownSample(ch_in=32, ch_out=32 * 2, is_test=is_test) self.darknet53_conv_block_list = [] @@ -155,14 +165,18 @@ def __init__(self, ch_in=3, is_test=True): for i, stage in enumerate(self.stages): conv_block = self.add_sublayer( "stage_%d" % (i), - LayerWarp(int(ch_in[i]), 32 * (2**i), stage, is_test=is_test)) + LayerWarp(int(ch_in[i]), 32 * (2**i), stage, is_test=is_test), + ) self.darknet53_conv_block_list.append(conv_block) for i in range(len(self.stages) - 1): downsample = self.add_sublayer( "stage_%d_downsample" % i, - DownSample(ch_in=32 * (2**(i + 1)), - ch_out=32 * (2**(i + 2)), - is_test=is_test)) + DownSample( + ch_in=32 * (2 ** (i + 1)), + ch_out=32 * (2 ** (i + 2)), + is_test=is_test, + ), + ) self.downsample_list.append(downsample) def forward(self, inputs): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/decos.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/decos.py index ee566fbcb3348..73bbfffcabf8c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/decos.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/decos.py @@ -18,7 +18,6 @@ def deco1(fun): - @wraps(fun) def inner(*args, **kwargs): print('in decos.deco1, added 1') @@ -30,9 +29,7 @@ def inner(*args, **kwargs): def deco2(x=0): - def inner_deco(func): - @wraps(func) def inner(*args, **kwargs): print('in decos.deco2, added {}'.format(x)) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/ifelse_simple_func.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/ifelse_simple_func.py index d8f2a379d617d..bbd0f25c05f12 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/ifelse_simple_func.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/ifelse_simple_func.py @@ -135,7 +135,8 @@ def dyfunc_with_if_else_early_return2(): def dyfunc_with_if_else_with_list_geneator(x): if 10 > 5: y = paddle.add_n( - [paddle.full(shape=[2], fill_value=v) for v in range(5)]) + [paddle.full(shape=[2], fill_value=v) for v in range(5)] + ) else: y = x return y @@ -162,9 +163,9 @@ def nested_if_else(x_v): if paddle.mean(y).numpy()[0] < batch_size: y = fluid.layers.abs(y) else: - tmp = fluid.layers.fill_constant(y.shape, - dtype='float32', - value=-1) + tmp = fluid.layers.fill_constant( + y.shape, dtype='float32', value=-1 + ) y = y - tmp else: y = x_v - bias @@ -180,15 +181,15 @@ def nested_if_else_2(x): x_shape_0 = x.shape[0] if x_shape_0 < 1: if fluid.layers.shape(y).numpy()[0] < 1: - res = fluid.layers.fill_constant(value=2, - shape=x.shape, - dtype="int32") + res = fluid.layers.fill_constant( + value=2, shape=x.shape, dtype="int32" + ) # `z` is a new var here. z = y + 1 else: - res = fluid.layers.fill_constant(value=3, - shape=x.shape, - dtype="int32") + res = fluid.layers.fill_constant( + value=3, shape=x.shape, dtype="int32" + ) else: res = x return res @@ -213,33 +214,35 @@ def nested_if_else_3(x): else: y_shape = fluid.layers.shape(y) if y_shape.numpy()[0] < 1: - res = fluid.layers.fill_constant(value=2, - shape=x.shape, - dtype="int32") + res = fluid.layers.fill_constant( + value=2, shape=x.shape, dtype="int32" + ) # `z` is created in above code block. z = y + 1 else: - res = fluid.layers.fill_constant(value=3, - shape=x.shape, - dtype="int32") + res = fluid.layers.fill_constant( + value=3, shape=x.shape, dtype="int32" + ) # `out` is a new var. out = x + 1 return res class NetWithControlFlowIf(fluid.dygraph.Layer): - def __init__(self, hidden_dim=16): super(NetWithControlFlowIf, self).__init__() self.hidden_dim = hidden_dim self.fc = fluid.dygraph.Linear( input_dim=hidden_dim, output_dim=5, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5))) - self.alpha = 10. + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), + ) + self.alpha = 10.0 self.constant_vars = {} @paddle.jit.to_static @@ -247,18 +250,21 @@ def forward(self, input): hidden_dim = input.shape[-1] if hidden_dim != self.hidden_dim: raise ValueError( - "hidden_dim {} of input is not equal to FC.weight[0]: {}". - format(hidden_dim, self.hidden_dim)) - - self.constant_vars['bias'] = fluid.layers.fill_constant([5], - dtype='float32', - value=1) + "hidden_dim {} of input is not equal to FC.weight[0]: {}".format( + hidden_dim, self.hidden_dim + ) + ) + + self.constant_vars['bias'] = fluid.layers.fill_constant( + [5], dtype='float32', value=1 + ) # Control flow `if` statement fc_out = self.fc(input) if paddle.mean(fc_out).numpy()[0] < 0: y = fc_out + self.constant_vars['bias'] self.constant_vars['w'] = fluid.layers.fill_constant( - [5], dtype='float32', value=10) + [5], dtype='float32', value=10 + ) if y.numpy()[0] < self.alpha: # Create new var, but is not used. x = 10 @@ -268,12 +274,13 @@ def forward(self, input): if y.numpy()[-1] < self.alpha: # Modify variable of class self.constant_vars['w'] = fluid.layers.fill_constant( - [hidden_dim], dtype='float32', value=9) + [hidden_dim], dtype='float32', value=9 + ) y = fluid.layers.abs(y) else: - tmp = fluid.layers.fill_constant(y.shape, - dtype='float32', - value=-1) + tmp = fluid.layers.fill_constant( + y.shape, dtype='float32', value=-1 + ) y = y - tmp else: y = fc_out - self.constant_vars['bias'] @@ -284,8 +291,12 @@ def forward(self, input): def if_with_and_or(x_v, label=None): batch_size = fluid.layers.shape(x_v) - if x_v is not None and (paddle.mean(x_v).numpy()[0] > 0 or label - is not None) and batch_size[0] > 1 and True: + if ( + x_v is not None + and (paddle.mean(x_v).numpy()[0] > 0 or label is not None) + and batch_size[0] > 1 + and True + ): x_v = x_v - 1 else: x_v = x_v + 1 @@ -317,8 +328,12 @@ def if_with_and_or_2(x, y=None): def if_with_and_or_3(x, y=None): batch_size = fluid.layers.shape(x) mean_res = paddle.mean(x) - if x is not None and batch_size[0] > 1 and y is not None and mean_res.numpy( - )[0] > 0: + if ( + x is not None + and batch_size[0] > 1 + and y is not None + and mean_res.numpy()[0] > 0 + ): x = x + 1 if mean_res.numpy()[0] > 0 and (x is not None and batch_size[0] > 1) and y: x = x - 1 @@ -328,19 +343,19 @@ def if_with_and_or_3(x, y=None): def if_with_and_or_4(x, y=None): batch_size = fluid.layers.shape(x) mean_res = paddle.mean(x) - if (x is not None and batch_size[0] > 1) or (y is not None - and mean_res.numpy()[0] > 0): + if (x is not None and batch_size[0] > 1) or ( + y is not None and mean_res.numpy()[0] > 0 + ): x = x + 1 - if (x is not None or batch_size[0] > 1) and (y is not None - or mean_res.numpy()[0] > 0): + if (x is not None or batch_size[0] > 1) and ( + y is not None or mean_res.numpy()[0] > 0 + ): x = x - 1 return x def if_with_class_var(x, y=None): - class Foo(object): - def __init__(self): self.a = 1 self.b = 2 diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/predictor_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/predictor_utils.py index 2759177e9780b..554cf95e82483 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/predictor_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/predictor_utils.py @@ -43,7 +43,8 @@ def _load_model_and_set_config(self): if os.path.exists(os.path.join(self.model_path, self.params_file)): config = AnalysisConfig( os.path.join(self.model_path, self.model_file), - os.path.join(self.model_path, self.params_file)) + os.path.join(self.model_path, self.params_file), + ) else: config = AnalysisConfig(os.path.join(self.model_path)) @@ -73,7 +74,7 @@ def _get_analysis_outputs(self, config): tensor_shapes = predictor.get_input_tensor_shape() names = predictor.get_input_names() for i, name in enumerate(names): - #assert name in self.feeds_var, '{} not in feeded dict'.format(name) + # assert name in self.feeds_var, '{} not in feeded dict'.format(name) shape = tensor_shapes[name] tensor = predictor.get_input_tensor(name) feed_data = self.feeds_var[i] diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_dygraph_model.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_dygraph_model.py index 5af9b2ab94ba5..266bcf4e7b786 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_dygraph_model.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_dygraph_model.py @@ -24,24 +24,26 @@ from paddle.fluid.dygraph.nn import Embedding from seq2seq_utils import Seq2SeqModelHyperParams as args -INF = 1. * 1e5 +INF = 1.0 * 1e5 alpha = 0.6 -uniform_initializer = lambda x: fluid.initializer.UniformInitializer(low=-x, - high=x) +uniform_initializer = lambda x: fluid.initializer.UniformInitializer( + low=-x, high=x +) zero_constant = fluid.initializer.Constant(0.0) class BasicLSTMUnit(Layer): - - def __init__(self, - hidden_size, - input_size, - param_attr=None, - bias_attr=None, - gate_activation=None, - activation=None, - forget_bias=1.0, - dtype='float32'): + def __init__( + self, + hidden_size, + input_size, + param_attr=None, + bias_attr=None, + gate_activation=None, + activation=None, + forget_bias=1.0, + dtype='float32', + ): super(BasicLSTMUnit, self).__init__(dtype) self._hiden_size = hidden_size @@ -56,12 +58,15 @@ def __init__(self, self._weight = self.create_parameter( attr=self._param_attr, shape=[self._input_size + self._hiden_size, 4 * self._hiden_size], - dtype=self._dtype) + dtype=self._dtype, + ) - self._bias = self.create_parameter(attr=self._bias_attr, - shape=[4 * self._hiden_size], - dtype=self._dtype, - is_bias=True) + self._bias = self.create_parameter( + attr=self._bias_attr, + shape=[4 * self._hiden_size], + dtype=self._dtype, + is_bias=True, + ) def forward(self, input, pre_hidden, pre_cell): concat_input_hidden = layers.concat([input, pre_hidden], 1) @@ -70,9 +75,11 @@ def forward(self, input, pre_hidden, pre_cell): gate_input = layers.elementwise_add(gate_input, self._bias) i, j, f, o = layers.split(gate_input, num_or_sections=4, dim=-1) new_cell = layers.elementwise_add( - layers.elementwise_mul(pre_cell, - layers.sigmoid(f + self._forget_bias)), - layers.elementwise_mul(layers.sigmoid(i), layers.tanh(j))) + layers.elementwise_mul( + pre_cell, layers.sigmoid(f + self._forget_bias) + ), + layers.elementwise_mul(layers.sigmoid(i), layers.tanh(j)), + ) new_hidden = layers.tanh(new_cell) * layers.sigmoid(o) @@ -80,20 +87,21 @@ def forward(self, input, pre_hidden, pre_cell): class BaseModel(fluid.dygraph.Layer): - - def __init__(self, - hidden_size, - src_vocab_size, - tar_vocab_size, - batch_size, - num_layers=1, - init_scale=0.1, - dropout=None, - beam_size=1, - beam_start_token=1, - beam_end_token=2, - beam_max_step_num=2, - mode='train'): + def __init__( + self, + hidden_size, + src_vocab_size, + tar_vocab_size, + batch_size, + num_layers=1, + init_scale=0.1, + dropout=None, + beam_size=1, + beam_start_token=1, + beam_end_token=2, + beam_max_step_num=2, + mode='train', + ): super(BaseModel, self).__init__() self.hidden_size = hidden_size self.src_vocab_size = src_vocab_size @@ -116,40 +124,54 @@ def __init__(self, self.src_embeder = Embedding( size=[self.src_vocab_size, self.hidden_size], param_attr=fluid.ParamAttr( - initializer=uniform_initializer(init_scale))) + initializer=uniform_initializer(init_scale) + ), + ) self.tar_embeder = Embedding( size=[self.tar_vocab_size, self.hidden_size], is_sparse=False, param_attr=fluid.ParamAttr( - initializer=uniform_initializer(init_scale))) + initializer=uniform_initializer(init_scale) + ), + ) self.enc_units = [] for i in range(num_layers): self.enc_units.append( self.add_sublayer( "enc_units_%d" % i, - BasicLSTMUnit(hidden_size=self.hidden_size, - input_size=self.hidden_size, - param_attr=param_attr, - bias_attr=bias_attr, - forget_bias=forget_bias))) + BasicLSTMUnit( + hidden_size=self.hidden_size, + input_size=self.hidden_size, + param_attr=param_attr, + bias_attr=bias_attr, + forget_bias=forget_bias, + ), + ) + ) self.dec_units = [] for i in range(num_layers): self.dec_units.append( self.add_sublayer( "dec_units_%d" % i, - BasicLSTMUnit(hidden_size=self.hidden_size, - input_size=self.hidden_size, - param_attr=param_attr, - bias_attr=bias_attr, - forget_bias=forget_bias))) - - self.fc = fluid.dygraph.nn.Linear(self.hidden_size, - self.tar_vocab_size, - param_attr=param_attr, - bias_attr=False) + BasicLSTMUnit( + hidden_size=self.hidden_size, + input_size=self.hidden_size, + param_attr=param_attr, + bias_attr=bias_attr, + forget_bias=forget_bias, + ), + ) + ) + + self.fc = fluid.dygraph.nn.Linear( + self.hidden_size, + self.tar_vocab_size, + param_attr=param_attr, + bias_attr=False, + ) def _transpose_batch_time(self, x): return fluid.layers.transpose(x, [1, 0] + list(range(2, len(x.shape)))) @@ -168,8 +190,9 @@ def _expand_to_beam_size(self, x): return x def _real_state(self, state, new_state, step_mask): - new_state = fluid.layers.elementwise_mul(new_state, step_mask, axis=0) - \ - fluid.layers.elementwise_mul(state, (step_mask - 1), axis=0) + new_state = fluid.layers.elementwise_mul( + new_state, step_mask, axis=0 + ) - fluid.layers.elementwise_mul(state, (step_mask - 1), axis=0) return new_state def _gather(self, x, indices, batch_pos): @@ -187,26 +210,28 @@ def forward(self, inputs): # NOTE: modify model code about `enc_hidden` and `enc_cell` to transforme dygraph code successfully. # Because nested list can't be transformed now. enc_hidden_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32')) + np.zeros((self.batch_size, self.hidden_size), dtype='float32') + ) enc_cell_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32')) + np.zeros((self.batch_size, self.hidden_size), dtype='float32') + ) zero = fluid.layers.zeros(shape=[1], dtype="int64") enc_hidden = fluid.layers.create_array(dtype="float32") enc_cell = fluid.layers.create_array(dtype="float32") for i in range(self.num_layers): index = zero + i - enc_hidden = fluid.layers.array_write(enc_hidden_0, - index, - array=enc_hidden) - enc_cell = fluid.layers.array_write(enc_cell_0, - index, - array=enc_cell) + enc_hidden = fluid.layers.array_write( + enc_hidden_0, index, array=enc_hidden + ) + enc_cell = fluid.layers.array_write( + enc_cell_0, index, array=enc_cell + ) max_seq_len = src_emb.shape[0] - enc_len_mask = fluid.layers.sequence_mask(src_sequence_length, - maxlen=max_seq_len, - dtype="float32") + enc_len_mask = fluid.layers.sequence_mask( + src_sequence_length, maxlen=max_seq_len, dtype="float32" + ) enc_len_mask = fluid.layers.transpose(enc_len_mask, [1, 0]) # TODO: Because diff exits if call while_loop in static graph. @@ -218,21 +243,24 @@ def forward(self, inputs): step_mask = enc_len_mask[k] new_enc_hidden, new_enc_cell = [], [] for i in range(self.num_layers): - enc_new_hidden, enc_new_cell = self.enc_units[i](enc_step_input, - enc_hidden[i], - enc_cell[i]) + enc_new_hidden, enc_new_cell = self.enc_units[i]( + enc_step_input, enc_hidden[i], enc_cell[i] + ) if self.dropout != None and self.dropout > 0.0: enc_step_input = fluid.layers.dropout( enc_new_hidden, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) else: enc_step_input = enc_new_hidden new_enc_hidden.append( - self._real_state(enc_hidden[i], enc_new_hidden, step_mask)) + self._real_state(enc_hidden[i], enc_new_hidden, step_mask) + ) new_enc_cell.append( - self._real_state(enc_cell[i], enc_new_cell, step_mask)) + self._real_state(enc_cell[i], enc_new_cell, step_mask) + ) enc_hidden, enc_cell = new_enc_hidden, new_enc_cell @@ -245,30 +273,31 @@ def forward(self, inputs): step_input = tar_emb[j] new_dec_hidden, new_dec_cell = [], [] for i in range(self.num_layers): - new_hidden, new_cell = self.dec_units[i](step_input, - dec_hidden[i], - dec_cell[i]) + new_hidden, new_cell = self.dec_units[i]( + step_input, dec_hidden[i], dec_cell[i] + ) new_dec_hidden.append(new_hidden) new_dec_cell.append(new_cell) if self.dropout != None and self.dropout > 0.0: step_input = fluid.layers.dropout( new_hidden, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) else: step_input = new_hidden dec_output.append(step_input) dec_output = fluid.layers.stack(dec_output) dec_output = self.fc(self._transpose_batch_time(dec_output)) - loss = fluid.layers.softmax_with_cross_entropy(logits=dec_output, - label=label, - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=dec_output, label=label, soft_label=False + ) loss = fluid.layers.squeeze(loss, axes=[2]) max_tar_seq_len = fluid.layers.shape(tar)[1] - tar_mask = fluid.layers.sequence_mask(tar_sequence_length, - maxlen=max_tar_seq_len, - dtype='float32') + tar_mask = fluid.layers.sequence_mask( + tar_sequence_length, maxlen=max_tar_seq_len, dtype='float32' + ) loss = loss * tar_mask loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -283,26 +312,28 @@ def beam_search(self, inputs): src_emb = self.src_embeder(self._transpose_batch_time(src)) enc_hidden_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32')) + np.zeros((self.batch_size, self.hidden_size), dtype='float32') + ) enc_cell_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32')) + np.zeros((self.batch_size, self.hidden_size), dtype='float32') + ) zero = fluid.layers.zeros(shape=[1], dtype="int64") enc_hidden = fluid.layers.create_array(dtype="float32") enc_cell = fluid.layers.create_array(dtype="float32") for j in range(self.num_layers): index = zero + j - enc_hidden = fluid.layers.array_write(enc_hidden_0, - index, - array=enc_hidden) - enc_cell = fluid.layers.array_write(enc_cell_0, - index, - array=enc_cell) + enc_hidden = fluid.layers.array_write( + enc_hidden_0, index, array=enc_hidden + ) + enc_cell = fluid.layers.array_write( + enc_cell_0, index, array=enc_cell + ) max_seq_len = src_emb.shape[0] - enc_len_mask = fluid.layers.sequence_mask(src_sequence_length, - maxlen=max_seq_len, - dtype="float32") + enc_len_mask = fluid.layers.sequence_mask( + src_sequence_length, maxlen=max_seq_len, dtype="float32" + ) enc_len_mask = fluid.layers.transpose(enc_len_mask, [1, 0]) for k in range(args.max_seq_len): @@ -312,40 +343,50 @@ def beam_search(self, inputs): new_enc_hidden, new_enc_cell = [], [] for i in range(self.num_layers): - enc_new_hidden, enc_new_cell = self.enc_units[i](enc_step_input, - enc_hidden[i], - enc_cell[i]) + enc_new_hidden, enc_new_cell = self.enc_units[i]( + enc_step_input, enc_hidden[i], enc_cell[i] + ) if self.dropout != None and self.dropout > 0.0: enc_step_input = fluid.layers.dropout( enc_new_hidden, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) else: enc_step_input = enc_new_hidden new_enc_hidden.append( - self._real_state(enc_hidden[i], enc_new_hidden, step_mask)) + self._real_state(enc_hidden[i], enc_new_hidden, step_mask) + ) new_enc_cell.append( - self._real_state(enc_cell[i], enc_new_cell, step_mask)) + self._real_state(enc_cell[i], enc_new_cell, step_mask) + ) enc_hidden, enc_cell = new_enc_hidden, new_enc_cell # beam search batch_beam_shape = (self.batch_size, self.beam_size) - vocab_size_tensor = to_variable(np.full( - (1), self.tar_vocab_size)).astype("int64") + vocab_size_tensor = to_variable( + np.full((1), self.tar_vocab_size) + ).astype("int64") start_token_tensor = to_variable( - np.full(batch_beam_shape, self.beam_start_token, dtype='int64')) + np.full(batch_beam_shape, self.beam_start_token, dtype='int64') + ) end_token_tensor = to_variable( - np.full(batch_beam_shape, self.beam_end_token, dtype='int64')) + np.full(batch_beam_shape, self.beam_end_token, dtype='int64') + ) step_input = self.tar_embeder(start_token_tensor) beam_finished = to_variable( - np.full(batch_beam_shape, 0, dtype='float32')) + np.full(batch_beam_shape, 0, dtype='float32') + ) beam_state_log_probs = to_variable( - np.array([[0.] + [-self.kinf] * (self.beam_size - 1)], - dtype="float32")) - beam_state_log_probs = fluid.layers.expand(beam_state_log_probs, - [self.batch_size, 1]) + np.array( + [[0.0] + [-self.kinf] * (self.beam_size - 1)], dtype="float32" + ) + ) + beam_state_log_probs = fluid.layers.expand( + beam_state_log_probs, [self.batch_size, 1] + ) dec_hidden, dec_cell = enc_hidden, enc_cell dec_hidden = [self._expand_to_beam_size(ele) for ele in dec_hidden] dec_cell = [self._expand_to_beam_size(ele) for ele in dec_cell] @@ -353,7 +394,10 @@ def beam_search(self, inputs): batch_pos = fluid.layers.expand( fluid.layers.unsqueeze( to_variable(np.arange(0, self.batch_size, 1, dtype="int64")), - [1]), [1, self.beam_size]) + [1], + ), + [1, self.beam_size], + ) predicted_ids = [] parent_ids = [] @@ -369,16 +413,17 @@ def beam_search(self, inputs): dec_cell = [self._merge_batch_beams(state) for state in dec_cell] for i in range(self.num_layers): - new_hidden, new_cell = self.dec_units[i](step_input, - dec_hidden[i], - dec_cell[i]) + new_hidden, new_cell = self.dec_units[i]( + step_input, dec_hidden[i], dec_cell[i] + ) new_dec_hidden.append(new_hidden) new_dec_cell.append(new_cell) if self.dropout != None and self.dropout > 0.0: step_input = fluid.layers.dropout( new_hidden, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) else: step_input = new_hidden @@ -386,28 +431,40 @@ def beam_search(self, inputs): cell_outputs = self.fc(cell_outputs) step_log_probs = fluid.layers.log( - fluid.layers.softmax(cell_outputs)) + fluid.layers.softmax(cell_outputs) + ) noend_array = [-self.kinf] * self.tar_vocab_size noend_array[self.beam_end_token] = 0 noend_mask_tensor = to_variable( - np.array(noend_array, dtype='float32')) + np.array(noend_array, dtype='float32') + ) step_log_probs = fluid.layers.elementwise_mul( - fluid.layers.expand(fluid.layers.unsqueeze(beam_finished, [2]), [1, 1, self.tar_vocab_size]), - noend_mask_tensor, axis=-1) - \ - fluid.layers.elementwise_mul(step_log_probs, (beam_finished - 1), axis=0) - log_probs = fluid.layers.elementwise_add(x=step_log_probs, - y=beam_state_log_probs, - axis=0) + fluid.layers.expand( + fluid.layers.unsqueeze(beam_finished, [2]), + [1, 1, self.tar_vocab_size], + ), + noend_mask_tensor, + axis=-1, + ) - fluid.layers.elementwise_mul( + step_log_probs, (beam_finished - 1), axis=0 + ) + log_probs = fluid.layers.elementwise_add( + x=step_log_probs, y=beam_state_log_probs, axis=0 + ) scores = fluid.layers.reshape( - log_probs, [-1, self.beam_size * self.tar_vocab_size]) - topk_scores, topk_indices = fluid.layers.topk(input=scores, - k=self.beam_size) + log_probs, [-1, self.beam_size * self.tar_vocab_size] + ) + topk_scores, topk_indices = fluid.layers.topk( + input=scores, k=self.beam_size + ) beam_indices = fluid.layers.elementwise_floordiv( - topk_indices, vocab_size_tensor) + topk_indices, vocab_size_tensor + ) token_indices = fluid.layers.elementwise_mod( - topk_indices, vocab_size_tensor) + topk_indices, vocab_size_tensor + ) next_log_probs = self._gather(scores, topk_indices, batch_pos) x = 0 @@ -434,7 +491,8 @@ def beam_search(self, inputs): next_finished = fluid.layers.cast(next_finished, "bool") next_finished = fluid.layers.logical_or( next_finished, - fluid.layers.equal(token_indices, end_token_tensor)) + fluid.layers.equal(token_indices, end_token_tensor), + ) next_finished = fluid.layers.cast(next_finished, "float32") dec_hidden, dec_cell = new_dec_hidden, new_dec_cell @@ -452,20 +510,21 @@ def beam_search(self, inputs): class AttentionModel(fluid.dygraph.Layer): - - def __init__(self, - hidden_size, - src_vocab_size, - tar_vocab_size, - batch_size, - num_layers=1, - init_scale=0.1, - dropout=None, - beam_size=1, - beam_start_token=1, - beam_end_token=2, - beam_max_step_num=2, - mode='train'): + def __init__( + self, + hidden_size, + src_vocab_size, + tar_vocab_size, + batch_size, + num_layers=1, + init_scale=0.1, + dropout=None, + beam_size=1, + beam_start_token=1, + beam_end_token=2, + beam_max_step_num=2, + mode='train', + ): super(AttentionModel, self).__init__() self.hidden_size = hidden_size self.src_vocab_size = src_vocab_size @@ -489,25 +548,33 @@ def __init__(self, size=[self.src_vocab_size, self.hidden_size], param_attr=fluid.ParamAttr( name='source_embedding', - initializer=uniform_initializer(init_scale))) + initializer=uniform_initializer(init_scale), + ), + ) self.tar_embeder = Embedding( size=[self.tar_vocab_size, self.hidden_size], is_sparse=False, param_attr=fluid.ParamAttr( name='target_embedding', - initializer=uniform_initializer(init_scale))) + initializer=uniform_initializer(init_scale), + ), + ) self.enc_units = [] for i in range(num_layers): self.enc_units.append( self.add_sublayer( "enc_units_%d" % i, - BasicLSTMUnit(hidden_size=self.hidden_size, - input_size=self.hidden_size, - param_attr=param_attr, - bias_attr=bias_attr, - forget_bias=forget_bias))) + BasicLSTMUnit( + hidden_size=self.hidden_size, + input_size=self.hidden_size, + param_attr=param_attr, + bias_attr=bias_attr, + forget_bias=forget_bias, + ), + ) + ) self.dec_units = [] for i in range(num_layers): @@ -515,50 +582,67 @@ def __init__(self, self.dec_units.append( self.add_sublayer( "dec_units_%d" % i, - BasicLSTMUnit(hidden_size=self.hidden_size, - input_size=self.hidden_size * 2, - param_attr=ParamAttr( - name="dec_units_%d" % i, - initializer=uniform_initializer( - self.init_scale)), - bias_attr=bias_attr, - forget_bias=forget_bias))) + BasicLSTMUnit( + hidden_size=self.hidden_size, + input_size=self.hidden_size * 2, + param_attr=ParamAttr( + name="dec_units_%d" % i, + initializer=uniform_initializer( + self.init_scale + ), + ), + bias_attr=bias_attr, + forget_bias=forget_bias, + ), + ) + ) else: self.dec_units.append( self.add_sublayer( "dec_units_%d" % i, - BasicLSTMUnit(hidden_size=self.hidden_size, - input_size=self.hidden_size, - param_attr=ParamAttr( - name="dec_units_%d" % i, - initializer=uniform_initializer( - self.init_scale)), - bias_attr=bias_attr, - forget_bias=forget_bias))) + BasicLSTMUnit( + hidden_size=self.hidden_size, + input_size=self.hidden_size, + param_attr=ParamAttr( + name="dec_units_%d" % i, + initializer=uniform_initializer( + self.init_scale + ), + ), + bias_attr=bias_attr, + forget_bias=forget_bias, + ), + ) + ) self.attn_fc = fluid.dygraph.nn.Linear( self.hidden_size, self.hidden_size, - param_attr=ParamAttr(name="self_attn_fc", - initializer=uniform_initializer( - self.init_scale)), - bias_attr=False) + param_attr=ParamAttr( + name="self_attn_fc", + initializer=uniform_initializer(self.init_scale), + ), + bias_attr=False, + ) self.concat_fc = fluid.dygraph.nn.Linear( 2 * self.hidden_size, self.hidden_size, - param_attr=ParamAttr(name="self_concat_fc", - initializer=uniform_initializer( - self.init_scale)), - bias_attr=False) - - self.fc = fluid.dygraph.nn.Linear(self.hidden_size, - self.tar_vocab_size, - param_attr=ParamAttr( - name="self_fc", - initializer=uniform_initializer( - self.init_scale)), - bias_attr=False) + param_attr=ParamAttr( + name="self_concat_fc", + initializer=uniform_initializer(self.init_scale), + ), + bias_attr=False, + ) + + self.fc = fluid.dygraph.nn.Linear( + self.hidden_size, + self.tar_vocab_size, + param_attr=ParamAttr( + name="self_fc", initializer=uniform_initializer(self.init_scale) + ), + bias_attr=False, + ) def _transpose_batch_time(self, x): return fluid.layers.transpose(x, [1, 0] + list(range(2, len(x.shape)))) @@ -571,16 +655,16 @@ def tile_beam_merge_with_batch(self, x): expand_times = [1] * len(x.shape) expand_times[1] = self.beam_size x = fluid.layers.expand(x, expand_times) # [batch_size, beam_size, ...] - x = fluid.layers.transpose(x, - list(range(2, len(x.shape))) + - [0, 1]) # [..., batch_size, beam_size] + x = fluid.layers.transpose( + x, list(range(2, len(x.shape))) + [0, 1] + ) # [..., batch_size, beam_size] # use 0 to copy to avoid wrong shape - x = fluid.layers.reshape(x, shape=[0] * (len(x.shape) - 2) + - [-1]) # [..., batch_size * beam_size] + x = fluid.layers.reshape( + x, shape=[0] * (len(x.shape) - 2) + [-1] + ) # [..., batch_size * beam_size] x = fluid.layers.transpose( - x, [len(x.shape) - 1] + - list(range(0, - len(x.shape) - 1))) # [batch_size * beam_size, ...] + x, [len(x.shape) - 1] + list(range(0, len(x.shape) - 1)) + ) # [batch_size * beam_size, ...] return x def _split_batch_beams(self, x): @@ -594,8 +678,9 @@ def _expand_to_beam_size(self, x): return x def _real_state(self, state, new_state, step_mask): - new_state = fluid.layers.elementwise_mul(new_state, step_mask, axis=0) - \ - fluid.layers.elementwise_mul(state, (step_mask - 1), axis=0) + new_state = fluid.layers.elementwise_mul( + new_state, step_mask, axis=0 + ) - fluid.layers.elementwise_mul(state, (step_mask - 1), axis=0) return new_state def _gather(self, x, indices, batch_pos): @@ -635,29 +720,31 @@ def forward(self, inputs): # NOTE: modify model code about `enc_hidden` and `enc_cell` to transforme dygraph code successfully. # Because nested list can't be transformed now. enc_hidden_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32')) + np.zeros((self.batch_size, self.hidden_size), dtype='float32') + ) enc_hidden_0.stop_gradient = True enc_cell_0 = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32')) + np.zeros((self.batch_size, self.hidden_size), dtype='float32') + ) enc_hidden_0.stop_gradient = True zero = fluid.layers.zeros(shape=[1], dtype="int64") enc_hidden = fluid.layers.create_array(dtype="float32") enc_cell = fluid.layers.create_array(dtype="float32") for i in range(self.num_layers): index = zero + i - enc_hidden = fluid.layers.array_write(enc_hidden_0, - index, - array=enc_hidden) - enc_cell = fluid.layers.array_write(enc_cell_0, - index, - array=enc_cell) + enc_hidden = fluid.layers.array_write( + enc_hidden_0, index, array=enc_hidden + ) + enc_cell = fluid.layers.array_write( + enc_cell_0, index, array=enc_cell + ) max_seq_len = src_emb.shape[0] - enc_len_mask = fluid.layers.sequence_mask(src_sequence_length, - maxlen=max_seq_len, - dtype="float32") - enc_padding_mask = (enc_len_mask - 1.0) + enc_len_mask = fluid.layers.sequence_mask( + src_sequence_length, maxlen=max_seq_len, dtype="float32" + ) + enc_padding_mask = enc_len_mask - 1.0 enc_len_mask = fluid.layers.transpose(enc_len_mask, [1, 0]) enc_outputs = [] @@ -670,21 +757,24 @@ def forward(self, inputs): step_mask = enc_len_mask[k] new_enc_hidden, new_enc_cell = [], [] for i in range(self.num_layers): - enc_new_hidden, enc_new_cell = self.enc_units[i](enc_step_input, - enc_hidden[i], - enc_cell[i]) + enc_new_hidden, enc_new_cell = self.enc_units[i]( + enc_step_input, enc_hidden[i], enc_cell[i] + ) if self.dropout != None and self.dropout > 0.0: enc_step_input = fluid.layers.dropout( enc_new_hidden, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) else: enc_step_input = enc_new_hidden new_enc_hidden.append( - self._real_state(enc_hidden[i], enc_new_hidden, step_mask)) + self._real_state(enc_hidden[i], enc_new_hidden, step_mask) + ) new_enc_cell.append( - self._real_state(enc_cell[i], enc_new_cell, step_mask)) + self._real_state(enc_cell[i], enc_new_cell, step_mask) + ) enc_outputs.append(enc_step_input) enc_hidden, enc_cell = new_enc_hidden, new_enc_cell @@ -693,7 +783,8 @@ def forward(self, inputs): # train input_feed = to_variable( - np.zeros((self.batch_size, self.hidden_size), dtype='float32')) + np.zeros((self.batch_size, self.hidden_size), dtype='float32') + ) # NOTE: set stop_gradient here, otherwise grad var is null input_feed.stop_gradient = True dec_hidden, dec_cell = enc_hidden, enc_cell @@ -707,16 +798,17 @@ def forward(self, inputs): step_input = fluid.layers.concat([step_input, input_feed], 1) new_dec_hidden, new_dec_cell = [], [] for i in range(self.num_layers): - new_hidden, new_cell = self.dec_units[i](step_input, - dec_hidden[i], - dec_cell[i]) + new_hidden, new_cell = self.dec_units[i]( + step_input, dec_hidden[i], dec_cell[i] + ) new_dec_hidden.append(new_hidden) new_dec_cell.append(new_cell) if self.dropout != None and self.dropout > 0.0: step_input = fluid.layers.dropout( new_hidden, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) else: step_input = new_hidden dec_att = self.attention(step_input, enc_outputs, enc_padding_mask) @@ -729,14 +821,14 @@ def forward(self, inputs): dec_output = fluid.layers.stack(dec_output) dec_output = self.fc(self._transpose_batch_time(dec_output)) - loss = fluid.layers.softmax_with_cross_entropy(logits=dec_output, - label=label, - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=dec_output, label=label, soft_label=False + ) loss = fluid.layers.squeeze(loss, axes=[2]) max_tar_seq_len = fluid.layers.shape(tar)[1] - tar_mask = fluid.layers.sequence_mask(tar_sequence_length, - maxlen=max_tar_seq_len, - dtype='float32') + tar_mask = fluid.layers.sequence_mask( + tar_sequence_length, maxlen=max_tar_seq_len, dtype='float32' + ) loss = loss * tar_mask loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_utils.py index 23e25a974eb7c..44a55c79c92e3 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/seq2seq_utils.py @@ -40,7 +40,7 @@ def to_pad_np(data, source=False): mask = np.zeros((bs), dtype='int32') for i, ele in enumerate(data): - ids[i, :len(ele)] = ele + ids[i, : len(ele)] = ele if not source: mask[i] = len(ele) - 1 else: @@ -61,7 +61,7 @@ def to_pad_np(data, source=False): new_cache = sorted(b_src, key=lambda k: len(k[0])) for i in range(cache_num): - batch_data = new_cache[i * batch_size:(i + 1) * batch_size] + batch_data = new_cache[i * batch_size : (i + 1) * batch_size] src_cache = [w[0] for w in batch_data] tar_cache = [w[1] for w in batch_data] src_ids, src_mask = to_pad_np(src_cache, source=True) @@ -83,7 +83,7 @@ def to_pad_np(data, source=False): for i in range(cache_num): batch_end = min(len(new_cache), (i + 1) * batch_size) - batch_data = new_cache[i * batch_size:batch_end] + batch_data = new_cache[i * batch_size : batch_end] src_cache = [w[0] for w in batch_data] tar_cache = [w[1] for w in batch_data] src_ids, src_mask = to_pad_np(src_cache, source=True) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model.py index a18da2173feab..b9a32c6e52a0d 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model.py @@ -41,12 +41,14 @@ def ops(self): """ # TODO(huihuangzheng): The original code set the is_sparse=True, but it # causes crush in dy2stat. Set it to True after fixing it. - emb = Embedding(size=[self.dict_size, self.emb_dim], - is_sparse=True, - padding_idx=self.padding_idx, - param_attr=attr.ParamAttr( - name=self.name, - initializer=fluid.initializer.Xavier())) + emb = Embedding( + size=[self.dict_size, self.emb_dim], + is_sparse=True, + padding_idx=self.padding_idx, + param_attr=attr.ParamAttr( + name=self.name, initializer=fluid.initializer.Xavier() + ), + ) return emb @@ -68,10 +70,12 @@ def ops(self): """ operation """ - fc = FC(size=self.fc_dim, - param_attr=attr.ParamAttr(name="%s.w" % self.name), - bias_attr=attr.ParamAttr(name="%s.b" % self.name), - act=self.act) + fc = FC( + size=self.fc_dim, + param_attr=attr.ParamAttr(name="%s.w" % self.name), + bias_attr=attr.ParamAttr(name="%s.b" % self.name), + act=self.act, + ) return fc @@ -307,14 +311,16 @@ class FC(Layer): conv = fc(data) """ - def __init__(self, - size, - num_flatten_dims=1, - param_attr=None, - bias_attr=None, - act=None, - is_test=False, - dtype="float32"): + def __init__( + self, + size, + num_flatten_dims=1, + param_attr=None, + bias_attr=None, + act=None, + is_test=False, + dtype="float32", + ): super(FC, self).__init__(dtype) self._size = size @@ -328,27 +334,32 @@ def __init__(self, def _build_once(self, input): i = 0 for inp, param in self._helper.iter_inputs_and_params( - input, self._param_attr): + input, self._param_attr + ): input_shape = inp.shape param_shape = [ - reduce(lambda a, b: a * b, input_shape[self._num_flatten_dims:], - 1) + reduce( + lambda a, b: a * b, input_shape[self._num_flatten_dims :], 1 + ) ] + [self._size] self.__w.append( self.add_parameter( '_w%d' % i, - self.create_parameter(attr=param, - shape=param_shape, - dtype=self._dtype, - is_bias=False))) + self.create_parameter( + attr=param, + shape=param_shape, + dtype=self._dtype, + is_bias=False, + ), + ) + ) i += 1 size = list([self._size]) - self._b = self.create_parameter(attr=self._bias_attr, - shape=size, - dtype=self._dtype, - is_bias=True) + self._b = self.create_parameter( + attr=self._bias_attr, shape=size, dtype=self._dtype, is_bias=True + ) # TODO(songyouwei): We should remove _w property @property @@ -384,18 +395,18 @@ def forward(self, input): mul_results = list() i = 0 for inp, param in self._helper.iter_inputs_and_params( - input, self._param_attr): + input, self._param_attr + ): tmp = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type="mul", - inputs={ - "X": inp, - "Y": self.__w[i] - }, - outputs={"Out": tmp}, - attrs={ - "x_num_col_dims": self._num_flatten_dims, - "y_num_col_dims": 1 - }) + self._helper.append_op( + type="mul", + inputs={"X": inp, "Y": self.__w[i]}, + outputs={"Out": tmp}, + attrs={ + "x_num_col_dims": self._num_flatten_dims, + "y_num_col_dims": 1, + }, + ) i += 1 mul_results.append(tmp) @@ -403,22 +414,25 @@ def forward(self, input): pre_bias = mul_results[0] else: pre_bias = self._helper.create_variable_for_type_inference( - self._dtype) - self._helper.append_op(type="sum", - inputs={"X": mul_results}, - outputs={"Out": pre_bias}, - attrs={"use_mkldnn": False}) + self._dtype + ) + self._helper.append_op( + type="sum", + inputs={"X": mul_results}, + outputs={"Out": pre_bias}, + attrs={"use_mkldnn": False}, + ) if self._b is not None: pre_activation = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [self._b] - }, - outputs={'Out': [pre_activation]}, - attrs={'axis': self._num_flatten_dims}) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [self._b]}, + outputs={'Out': [pre_activation]}, + attrs={'axis': self._num_flatten_dims}, + ) else: pre_activation = pre_bias # Currently, we don't support inplace in dygraph mode @@ -450,7 +464,10 @@ def compute(self, pos, neg): constant.ops(neg, neg.shape, "float32", 0.0), elementwise_add.ops( elementwise_sub.ops(neg, pos), - constant.ops(neg, neg.shape, "float32", self.margin)))) + constant.ops(neg, neg.shape, "float32", self.margin), + ), + ) + ) return loss @@ -469,8 +486,9 @@ def __init__(self, conf_dict): self.emb_dim = conf_dict["net"]["emb_dim"] self.bow_dim = conf_dict["net"]["bow_dim"] self.seq_len = conf_dict["seq_len"] - self.emb_layer = EmbeddingLayer(self.dict_size, self.emb_dim, - "emb").ops() + self.emb_layer = EmbeddingLayer( + self.dict_size, self.emb_dim, "emb" + ).ops() self.bow_layer = Linear(self.bow_dim, self.bow_dim) self.bow_layer_po = FCLayer(self.bow_dim, None, "fc").ops() self.softmax_layer = FCLayer(2, "softmax", "cos_sim").ops() @@ -484,10 +502,12 @@ def forward(self, left, right): # embedding layer left_emb = self.emb_layer(left) right_emb = self.emb_layer(right) - left_emb = fluid.layers.reshape(left_emb, - shape=[-1, self.seq_len, self.bow_dim]) - right_emb = fluid.layers.reshape(right_emb, - shape=[-1, self.seq_len, self.bow_dim]) + left_emb = fluid.layers.reshape( + left_emb, shape=[-1, self.seq_len, self.bow_dim] + ) + right_emb = fluid.layers.reshape( + right_emb, shape=[-1, self.seq_len, self.bow_dim] + ) bow_left = fluid.layers.reduce_sum(left_emb, dim=1) bow_right = fluid.layers.reduce_sum(right_emb, dim=1) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model_v2.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model_v2.py index b82a7b663bd51..cc8f90594e437 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model_v2.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model_v2.py @@ -43,7 +43,9 @@ def ops(self): padding_idx=self.padding_idx, param_attr=paddle.ParamAttr( name=self.name, - initializer=paddle.nn.initializer.XavierUniform())) + initializer=paddle.nn.initializer.XavierUniform(), + ), + ) return emb @@ -65,10 +67,12 @@ def ops(self): """ operation """ - fc = FC(size=self.fc_dim, - param_attr=paddle.ParamAttr(name="%s.w" % self.name), - bias_attr=paddle.ParamAttr(name="%s.b" % self.name), - act=self.act) + fc = FC( + size=self.fc_dim, + param_attr=paddle.ParamAttr(name="%s.w" % self.name), + bias_attr=paddle.ParamAttr(name="%s.b" % self.name), + act=self.act, + ) return fc @@ -293,14 +297,16 @@ class FC(paddle.nn.Layer): """ - def __init__(self, - size, - num_flatten_dims=1, - param_attr=None, - bias_attr=None, - act=None, - is_test=False, - dtype="float32"): + def __init__( + self, + size, + num_flatten_dims=1, + param_attr=None, + bias_attr=None, + act=None, + is_test=False, + dtype="float32", + ): super(FC, self).__init__(dtype) self._size = size @@ -314,27 +320,32 @@ def __init__(self, def _build_once(self, input): i = 0 for inp, param in self._helper.iter_inputs_and_params( - input, self._param_attr): + input, self._param_attr + ): input_shape = inp.shape param_shape = [ - reduce(lambda a, b: a * b, input_shape[self._num_flatten_dims:], - 1) + reduce( + lambda a, b: a * b, input_shape[self._num_flatten_dims :], 1 + ) ] + [self._size] self.__w.append( self.add_parameter( '_w%d' % i, - self.create_parameter(attr=param, - shape=param_shape, - dtype=self._dtype, - is_bias=False))) + self.create_parameter( + attr=param, + shape=param_shape, + dtype=self._dtype, + is_bias=False, + ), + ) + ) i += 1 size = list([self._size]) - self._b = self.create_parameter(attr=self._bias_attr, - shape=size, - dtype=self._dtype, - is_bias=True) + self._b = self.create_parameter( + attr=self._bias_attr, shape=size, dtype=self._dtype, is_bias=True + ) # TODO(songyouwei): We should remove _w property @property @@ -370,18 +381,18 @@ def forward(self, input): mul_results = list() i = 0 for inp, param in self._helper.iter_inputs_and_params( - input, self._param_attr): + input, self._param_attr + ): tmp = self._helper.create_variable_for_type_inference(self._dtype) - self._helper.append_op(type="mul", - inputs={ - "X": inp, - "Y": self.__w[i] - }, - outputs={"Out": tmp}, - attrs={ - "x_num_col_dims": self._num_flatten_dims, - "y_num_col_dims": 1 - }) + self._helper.append_op( + type="mul", + inputs={"X": inp, "Y": self.__w[i]}, + outputs={"Out": tmp}, + attrs={ + "x_num_col_dims": self._num_flatten_dims, + "y_num_col_dims": 1, + }, + ) i += 1 mul_results.append(tmp) @@ -389,22 +400,25 @@ def forward(self, input): pre_bias = mul_results[0] else: pre_bias = self._helper.create_variable_for_type_inference( - self._dtype) - self._helper.append_op(type="sum", - inputs={"X": mul_results}, - outputs={"Out": pre_bias}, - attrs={"use_mkldnn": False}) + self._dtype + ) + self._helper.append_op( + type="sum", + inputs={"X": mul_results}, + outputs={"Out": pre_bias}, + attrs={"use_mkldnn": False}, + ) if self._b is not None: pre_activation = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - self._helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [self._b] - }, - outputs={'Out': [pre_activation]}, - attrs={'axis': self._num_flatten_dims}) + dtype=self._dtype + ) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [self._b]}, + outputs={'Out': [pre_activation]}, + attrs={'axis': self._num_flatten_dims}, + ) else: pre_activation = pre_bias # Currently, we don't support inplace in dygraph mode @@ -436,7 +450,10 @@ def compute(self, pos, neg): constant.ops(neg, neg.shape, "float32", 0.0), elementwise_add.ops( elementwise_sub.ops(neg, pos), - constant.ops(neg, neg.shape, "float32", self.margin)))) + constant.ops(neg, neg.shape, "float32", self.margin), + ), + ) + ) return loss @@ -455,10 +472,12 @@ def __init__(self, conf_dict): self.emb_dim = conf_dict["net"]["emb_dim"] self.bow_dim = conf_dict["net"]["bow_dim"] self.seq_len = conf_dict["seq_len"] - self.emb_layer = EmbeddingLayer(self.dict_size, self.emb_dim, - "emb").ops() - self.bow_layer = paddle.nn.Linear(in_features=self.bow_dim, - out_features=self.bow_dim) + self.emb_layer = EmbeddingLayer( + self.dict_size, self.emb_dim, "emb" + ).ops() + self.bow_layer = paddle.nn.Linear( + in_features=self.bow_dim, out_features=self.bow_dim + ) self.bow_layer_po = FCLayer(self.bow_dim, None, "fc").ops() self.softmax_layer = FCLayer(2, "softmax", "cos_sim").ops() @@ -471,10 +490,12 @@ def forward(self, left, right): # embedding layer left_emb = self.emb_layer(left) right_emb = self.emb_layer(right) - left_emb = paddle.reshape(left_emb, - shape=[-1, self.seq_len, self.bow_dim]) - right_emb = paddle.reshape(right_emb, - shape=[-1, self.seq_len, self.bow_dim]) + left_emb = paddle.reshape( + left_emb, shape=[-1, self.seq_len, self.bow_dim] + ) + right_emb = paddle.reshape( + right_emb, shape=[-1, self.seq_len, self.bow_dim] + ) bow_left = paddle.fluid.layers.reduce_sum(left_emb, dim=1) bow_right = paddle.fluid.layers.reduce_sum(right_emb, dim=1) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_assert.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_assert.py index 05f11c090de02..c975a50bffd65 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_assert.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_assert.py @@ -33,7 +33,6 @@ def dyfunc_assert_non_variable(x=True): class TestAssertVariable(unittest.TestCase): - def _run(self, func, x, with_exception, to_static): ProgramTranslator().enable(to_static) if with_exception: @@ -49,28 +48,28 @@ def _run_dy_static(self, func, x, with_exception): self._run(func, x, with_exception, False) def test_non_variable(self): - self._run_dy_static(dyfunc_assert_non_variable, - x=False, - with_exception=True) - self._run_dy_static(dyfunc_assert_non_variable, - x=True, - with_exception=False) + self._run_dy_static( + dyfunc_assert_non_variable, x=False, with_exception=True + ) + self._run_dy_static( + dyfunc_assert_non_variable, x=True, with_exception=False + ) def test_bool_variable(self): - self._run_dy_static(dyfunc_assert_variable, - x=numpy.array([False]), - with_exception=True) - self._run_dy_static(dyfunc_assert_variable, - x=numpy.array([True]), - with_exception=False) + self._run_dy_static( + dyfunc_assert_variable, x=numpy.array([False]), with_exception=True + ) + self._run_dy_static( + dyfunc_assert_variable, x=numpy.array([True]), with_exception=False + ) def test_int_variable(self): - self._run_dy_static(dyfunc_assert_variable, - x=numpy.array([0]), - with_exception=True) - self._run_dy_static(dyfunc_assert_variable, - x=numpy.array([1]), - with_exception=False) + self._run_dy_static( + dyfunc_assert_variable, x=numpy.array([0]), with_exception=True + ) + self._run_dy_static( + dyfunc_assert_variable, x=numpy.array([1]), with_exception=False + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py index dbbbae2732e97..8315caf3e32c5 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ast_util.py @@ -21,7 +21,11 @@ import paddle.fluid as fluid from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_func -from ifelse_simple_func import dyfunc_with_if_else, dyfunc_with_if_else2, nested_if_else +from ifelse_simple_func import ( + dyfunc_with_if_else, + dyfunc_with_if_else2, + nested_if_else, +) class TestAST2Func(unittest.TestCase): @@ -37,7 +41,6 @@ def _ast2func(self, func): return transformed_func def test_ast2func(self): - def func(x, y): return x + y @@ -55,7 +58,6 @@ def test_ast2func_dygraph(self): self.assertTrue((true_ret == test_ret).all()) def test_ast2func_static(self): - def func(x): y = fluid.layers.relu(x) loss = paddle.mean(y) @@ -74,8 +76,9 @@ def func(x): def test_ast2func_error(self): with self.assertRaises(Exception) as e: self.assertRaises(TypeError, ast_to_func("x = a + b", 'foo')) - self.assertTrue("Type of ast_root should be gast.AST or ast.AST" in str( - e.exception)) + self.assertTrue( + "Type of ast_root should be gast.AST or ast.AST" in str(e.exception) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py index 9074c84e086e9..656d2768dcac2 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_basic_api_transformation.py @@ -70,16 +70,22 @@ def dyfunc_bool_to_tensor(x): class TestDygraphBasicApi_ToVariable(unittest.TestCase): - def setUp(self): self.input = np.ones(5).astype("int32") self.test_funcs = [ - dyfunc_to_tensor, dyfunc_bool_to_tensor, dyfunc_int_to_tensor, - dyfunc_float_to_tensor, dyfunc_to_variable, dyfunc_to_variable_2, - dyfunc_to_variable_3 + dyfunc_to_tensor, + dyfunc_bool_to_tensor, + dyfunc_int_to_tensor, + dyfunc_float_to_tensor, + dyfunc_to_variable, + dyfunc_to_variable_2, + dyfunc_to_variable_3, ] - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def get_dygraph_output(self): with fluid.dygraph.guard(): @@ -111,13 +117,18 @@ def dyfunc_BilinearTensorProduct(layer1, layer2): input1_dim=5, input2_dim=4, output_dim=1000, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), + ) - res = bilinearTensorProduct(fluid.dygraph.base.to_variable(layer1), - fluid.dygraph.base.to_variable(layer2)) + res = bilinearTensorProduct( + fluid.dygraph.base.to_variable(layer1), + fluid.dygraph.base.to_variable(layer2), + ) return res @@ -126,10 +137,12 @@ def dyfunc_Conv2D(input): num_channels=3, num_filters=2, filter_size=3, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)), + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), ) res = conv2d(input) return res @@ -140,10 +153,12 @@ def dyfunc_Conv3D(input): num_channels=3, num_filters=2, filter_size=3, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)), + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), ) res = conv3d(input) return res @@ -155,10 +170,12 @@ def dyfunc_Conv2DTranspose(input): num_filters=12, filter_size=12, use_cudnn=False, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)), + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), ) ret = conv2dTranspose(input) return ret @@ -170,10 +187,12 @@ def dyfunc_Conv3DTranspose(input): num_filters=12, filter_size=12, use_cudnn=False, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)), + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), ) ret = conv3dTranspose(input) return ret @@ -184,24 +203,24 @@ def dyfunc_Linear(input): input_dim=10, output_dim=5, act='relu', - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)), + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), ) res = fc(input) return res def dyfunc_Pool2D(input): - fluid.dygraph.Pool2D(pool_size=2, - pool_type='avg', - pool_stride=1, - global_pooling=False) - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='avg', - pool_stride=1, - global_pooling=False) + fluid.dygraph.Pool2D( + pool_size=2, pool_type='avg', pool_stride=1, global_pooling=False + ) + pool2d = fluid.dygraph.Pool2D( + pool_size=2, pool_type='avg', pool_stride=1, global_pooling=False + ) res = pool2d(input) return res @@ -209,7 +228,8 @@ def dyfunc_Pool2D(input): def dyfunc_Prelu(input): prelu0 = fluid.PRelu( mode='all', - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant(1.0))) + param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant(1.0)), + ) res = prelu0(input=input) return res @@ -252,7 +272,6 @@ def test_transformed_static_result(self): class TestDygraphBasicApi_BilinearTensorProduct(TestDygraphBasicApi): - def setUp(self): self.input1 = np.random.random((5, 5)).astype('float32') self.input2 = np.random.random((5, 4)).astype('float32') @@ -271,8 +290,9 @@ def get_static_output(self): main_program = fluid.Program() main_program.random_seed = SEED with fluid.program_guard(main_program, startup_program): - static_out = dygraph_to_static_func(self.dygraph_func)(self.input1, - self.input2) + static_out = dygraph_to_static_func(self.dygraph_func)( + self.input1, self.input2 + ) exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) @@ -281,42 +301,36 @@ def get_static_output(self): class TestDygraphBasicApi_Conv2D(TestDygraphBasicApi): - def setUp(self): self.input = np.random.random((1, 3, 3, 5)).astype('float32') self.dygraph_func = dyfunc_Conv2D class TestDygraphBasicApi_Conv3D(TestDygraphBasicApi): - def setUp(self): self.input = np.random.random((1, 3, 3, 3, 5)).astype('float32') self.dygraph_func = dyfunc_Conv3D class TestDygraphBasicApi_Conv2DTranspose(TestDygraphBasicApi): - def setUp(self): self.input = np.random.random((5, 3, 32, 32)).astype('float32') self.dygraph_func = dyfunc_Conv2DTranspose class TestDygraphBasicApi_Conv3DTranspose(TestDygraphBasicApi): - def setUp(self): self.input = np.random.random((5, 3, 12, 32, 32)).astype('float32') self.dygraph_func = dyfunc_Conv3DTranspose class TestDygraphBasicApi_Linear(TestDygraphBasicApi): - def setUp(self): self.input = np.random.random((4, 3, 10)).astype('float32') self.dygraph_func = dyfunc_Linear class TestDygraphBasicApi_Prelu(TestDygraphBasicApi): - def setUp(self): self.input = np.ones([5, 20, 10, 10]).astype('float32') self.dygraph_func = dyfunc_Prelu @@ -325,39 +339,36 @@ def setUp(self): # 2. test Apis that inherit from LearningRateDecay def dyfunc_CosineDecay(): base_lr = 0.1 - CosineDecay = fluid.dygraph.CosineDecay(learning_rate=base_lr, - step_each_epoch=10000, - epochs=120) + CosineDecay = fluid.dygraph.CosineDecay( + learning_rate=base_lr, step_each_epoch=10000, epochs=120 + ) lr = CosineDecay() return lr def dyfunc_ExponentialDecay(): base_lr = 0.1 - exponential_decay = fluid.dygraph.ExponentialDecay(learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True) + exponential_decay = fluid.dygraph.ExponentialDecay( + learning_rate=base_lr, decay_steps=10000, decay_rate=0.5, staircase=True + ) lr = exponential_decay() return lr def dyfunc_InverseTimeDecay(): base_lr = 0.1 - inverse_time_decay = fluid.dygraph.InverseTimeDecay(learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True) + inverse_time_decay = fluid.dygraph.InverseTimeDecay( + learning_rate=base_lr, decay_steps=10000, decay_rate=0.5, staircase=True + ) lr = inverse_time_decay() return lr def dyfunc_NaturalExpDecay(): base_lr = 0.1 - natural_exp_decay = fluid.dygraph.NaturalExpDecay(learning_rate=base_lr, - decay_steps=10000, - decay_rate=0.5, - staircase=True) + natural_exp_decay = fluid.dygraph.NaturalExpDecay( + learning_rate=base_lr, decay_steps=10000, decay_rate=0.5, staircase=True + ) lr = natural_exp_decay() return lr @@ -386,7 +397,6 @@ def dyfunc_PolynomialDecay(): class TestDygraphBasicApi_CosineDecay(unittest.TestCase): - def setUp(self): self.dygraph_func = dyfunc_CosineDecay @@ -417,43 +427,38 @@ def test_transformed_static_result(self): class TestDygraphBasicApi_ExponentialDecay(TestDygraphBasicApi_CosineDecay): - def setUp(self): self.dygraph_func = dyfunc_ExponentialDecay class TestDygraphBasicApi_InverseTimeDecay(TestDygraphBasicApi_CosineDecay): - def setUp(self): self.dygraph_func = dyfunc_InverseTimeDecay class TestDygraphBasicApi_NaturalExpDecay(TestDygraphBasicApi_CosineDecay): - def setUp(self): self.dygraph_func = dyfunc_NaturalExpDecay class TestDygraphBasicApi_NoamDecay(TestDygraphBasicApi_CosineDecay): - def setUp(self): self.dygraph_func = dyfunc_NoamDecay class TestDygraphBasicApi_PiecewiseDecay(TestDygraphBasicApi_CosineDecay): - def setUp(self): self.dygraph_func = dyfunc_PiecewiseDecay class TestDygraphBasicApi_PolynomialDecay(TestDygraphBasicApi_CosineDecay): - def setUp(self): self.dygraph_func = dyfunc_PolynomialDecay def _dygraph_fn(): import paddle.fluid as fluid + x = np.random.random((1, 3)).astype('float32') with fluid.dygraph.guard(): fluid.dygraph.to_variable(x) @@ -461,7 +466,6 @@ def _dygraph_fn(): class TestDygraphApiRecognition(unittest.TestCase): - def setUp(self): self.src = inspect.getsource(_dygraph_fn) self.root = gast.parse(self.src) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bert.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bert.py index 1929df30fe307..7fb9eca260636 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bert.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bert.py @@ -29,15 +29,15 @@ from predictor_utils import PredictorTools program_translator = ProgramTranslator() -place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) SEED = 2020 STEP_NUM = 10 PRINT_STEP = 2 class TestBert(unittest.TestCase): - def setUp(self): self.bert_config = get_bert_config() self.data_reader = get_feed_data_reader(self.bert_config) @@ -46,8 +46,9 @@ def setUp(self): self.model_save_prefix = os.path.join(self.model_save_dir, 'bert') self.model_filename = 'bert' + INFER_MODEL_SUFFIX self.params_filename = 'bert' + INFER_PARAMS_SUFFIX - self.dy_state_dict_save_path = os.path.join(self.temp_dir.name, - 'bert.dygraph') + self.dy_state_dict_save_path = os.path.join( + self.temp_dir.name, 'bert.dygraph' + ) def tearDown(self): self.temp_dir.cleanup() @@ -57,20 +58,30 @@ def train(self, bert_config, data_reader, to_static): fluid.default_main_program().random_seed = SEED fluid.default_startup_program().random_seed = SEED - data_loader = fluid.io.DataLoader.from_generator(capacity=50, - iterable=True) - data_loader.set_batch_generator(data_reader.data_generator(), - places=place) + data_loader = fluid.io.DataLoader.from_generator( + capacity=50, iterable=True + ) + data_loader.set_batch_generator( + data_reader.data_generator(), places=place + ) - bert = PretrainModelLayer(config=bert_config, - weight_sharing=False, - use_fp16=False) + bert = PretrainModelLayer( + config=bert_config, weight_sharing=False, use_fp16=False + ) optimizer = fluid.optimizer.Adam(parameter_list=bert.parameters()) step_idx = 0 speed_list = [] for input_data in data_loader(): - src_ids, pos_ids, sent_ids, input_mask, mask_label, mask_pos, labels = input_data + ( + src_ids, + pos_ids, + sent_ids, + input_mask, + mask_label, + mask_pos, + labels, + ) = input_data next_sent_acc, mask_lm_loss, total_loss = bert( src_ids=src_ids, position_ids=pos_ids, @@ -78,7 +89,8 @@ def train(self, bert_config, data_reader, to_static): input_mask=input_mask, mask_label=mask_label, mask_pos=mask_pos, - labels=labels) + labels=labels, + ) total_loss.backward() optimizer.minimize(total_loss) bert.clear_gradients() @@ -89,15 +101,18 @@ def train(self, bert_config, data_reader, to_static): if step_idx % PRINT_STEP == 0: if step_idx == 0: - print("Step: %d, loss: %f, ppl: %f, next_sent_acc: %f" % - (step_idx, loss, ppl, acc)) + print( + "Step: %d, loss: %f, ppl: %f, next_sent_acc: %f" + % (step_idx, loss, ppl, acc) + ) avg_batch_time = time.time() else: speed = PRINT_STEP / (time.time() - avg_batch_time) speed_list.append(speed) print( "Step: %d, loss: %f, ppl: %f, next_sent_acc: %f, speed: %.3f steps/s" - % (step_idx, loss, ppl, acc, speed)) + % (step_idx, loss, ppl, acc, speed) + ) avg_batch_time = time.time() step_idx += 1 @@ -105,8 +120,9 @@ def train(self, bert_config, data_reader, to_static): if to_static: fluid.dygraph.jit.save(bert, self.model_save_prefix) else: - fluid.dygraph.save_dygraph(bert.state_dict(), - self.dy_state_dict_save_path) + fluid.dygraph.save_dygraph( + bert.state_dict(), self.dy_state_dict_save_path + ) break return loss, ppl @@ -122,38 +138,56 @@ def predict_static(self, data): paddle.enable_static() exe = fluid.Executor(place) # load inference model - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(self.model_save_dir, - executor=exe, - model_filename=self.model_filename, - params_filename=self.params_filename) - pred_res = exe.run(inference_program, - feed=dict(zip(feed_target_names, data)), - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + self.model_save_dir, + executor=exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + ) + pred_res = exe.run( + inference_program, + feed=dict(zip(feed_target_names, data)), + fetch_list=fetch_targets, + ) return pred_res def predict_dygraph(self, bert_config, data): program_translator.enable(False) with fluid.dygraph.guard(place): - bert = PretrainModelLayer(config=bert_config, - weight_sharing=False, - use_fp16=False) + bert = PretrainModelLayer( + config=bert_config, weight_sharing=False, use_fp16=False + ) model_dict, _ = fluid.dygraph.load_dygraph( - self.dy_state_dict_save_path) + self.dy_state_dict_save_path + ) bert.set_dict(model_dict) bert.eval() input_vars = [fluid.dygraph.to_variable(x) for x in data] - src_ids, pos_ids, sent_ids, input_mask, mask_label, mask_pos, labels = input_vars - pred_res = bert(src_ids=src_ids, - position_ids=pos_ids, - sentence_ids=sent_ids, - input_mask=input_mask, - mask_label=mask_label, - mask_pos=mask_pos, - labels=labels) + ( + src_ids, + pos_ids, + sent_ids, + input_mask, + mask_label, + mask_pos, + labels, + ) = input_vars + pred_res = bert( + src_ids=src_ids, + position_ids=pos_ids, + sentence_ids=sent_ids, + input_mask=input_mask, + mask_label=mask_label, + mask_pos=mask_pos, + labels=labels, + ) pred_res = [var.numpy() for var in pred_res] return pred_res @@ -163,24 +197,42 @@ def predict_dygraph_jit(self, data): bert = fluid.dygraph.jit.load(self.model_save_prefix) bert.eval() - src_ids, pos_ids, sent_ids, input_mask, mask_label, mask_pos, labels = data - pred_res = bert(src_ids, pos_ids, sent_ids, input_mask, mask_label, - mask_pos, labels) + ( + src_ids, + pos_ids, + sent_ids, + input_mask, + mask_label, + mask_pos, + labels, + ) = data + pred_res = bert( + src_ids, + pos_ids, + sent_ids, + input_mask, + mask_label, + mask_pos, + labels, + ) pred_res = [var.numpy() for var in pred_res] return pred_res def predict_analysis_inference(self, data): - output = PredictorTools(self.model_save_dir, self.model_filename, - self.params_filename, data) + output = PredictorTools( + self.model_save_dir, self.model_filename, self.params_filename, data + ) out = output() return out def test_train(self): - static_loss, static_ppl = self.train_static(self.bert_config, - self.data_reader) - dygraph_loss, dygraph_ppl = self.train_dygraph(self.bert_config, - self.data_reader) + static_loss, static_ppl = self.train_static( + self.bert_config, self.data_reader + ) + dygraph_loss, dygraph_ppl = self.train_dygraph( + self.bert_config, self.data_reader + ) np.testing.assert_allclose(static_loss, dygraph_loss, rtol=1e-05) np.testing.assert_allclose(static_ppl, dygraph_ppl, rtol=1e-05) @@ -194,29 +246,38 @@ def verify_predict(self): predictor_pred_res = self.predict_analysis_inference(data) for dy_res, st_res, dy_jit_res, predictor_res in zip( - dygraph_pred_res, static_pred_res, dygraph_jit_pred_res, - predictor_pred_res): + dygraph_pred_res, + static_pred_res, + dygraph_jit_pred_res, + predictor_pred_res, + ): np.testing.assert_allclose( st_res, dy_res, rtol=1e-05, err_msg='dygraph_res: {},\n static_res: {}'.format( dy_res[~np.isclose(st_res, dy_res)], - st_res[~np.isclose(st_res, dy_res)])) + st_res[~np.isclose(st_res, dy_res)], + ), + ) np.testing.assert_allclose( st_res, dy_jit_res, rtol=1e-05, err_msg='dygraph_jit_res: {},\n static_res: {}'.format( dy_jit_res[~np.isclose(st_res, dy_jit_res)], - st_res[~np.isclose(st_res, dy_jit_res)])) + st_res[~np.isclose(st_res, dy_jit_res)], + ), + ) np.testing.assert_allclose( st_res, predictor_res, rtol=1e-05, err_msg='dygraph_jit_res: {},\n static_res: {}'.format( predictor_res[~np.isclose(st_res, predictor_res)], - st_res[~np.isclose(st_res, predictor_res)])) + st_res[~np.isclose(st_res, predictor_res)], + ), + ) break diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bmn.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bmn.py index e54c6274502ba..0f25b6ec71a7f 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bmn.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bmn.py @@ -38,9 +38,10 @@ fluid.set_flags({'FLAGS_cudnn_deterministic': True}) -def get_interp1d_mask(tscale, dscale, prop_boundary_ratio, num_sample, - num_sample_perbin): - """ generate sample mask for each point in Boundary-Matching Map """ +def get_interp1d_mask( + tscale, dscale, prop_boundary_ratio, num_sample, num_sample_perbin +): + """generate sample mask for each point in Boundary-Matching Map""" mask_mat = [] for start_index in range(tscale): mask_mat_vector = [] @@ -51,9 +52,13 @@ def get_interp1d_mask(tscale, dscale, prop_boundary_ratio, num_sample, center_len = float(p_xmax - p_xmin) + 1 sample_xmin = p_xmin - center_len * prop_boundary_ratio sample_xmax = p_xmax + center_len * prop_boundary_ratio - p_mask = _get_interp1d_bin_mask(sample_xmin, sample_xmax, - tscale, num_sample, - num_sample_perbin) + p_mask = _get_interp1d_bin_mask( + sample_xmin, + sample_xmax, + tscale, + num_sample, + num_sample_perbin, + ) else: p_mask = np.zeros([tscale, num_sample]) mask_mat_vector.append(p_mask) @@ -66,9 +71,10 @@ def get_interp1d_mask(tscale, dscale, prop_boundary_ratio, num_sample, return sample_mask -def _get_interp1d_bin_mask(seg_xmin, seg_xmax, tscale, num_sample, - num_sample_perbin): - """ generate sample mask for a boundary-matching pair """ +def _get_interp1d_bin_mask( + seg_xmin, seg_xmax, tscale, num_sample, num_sample_perbin +): + """generate sample mask for a boundary-matching pair""" plen = float(seg_xmax - seg_xmin) plen_sample = plen / (num_sample * num_sample_perbin - 1.0) total_samples = [ @@ -77,8 +83,9 @@ def _get_interp1d_bin_mask(seg_xmin, seg_xmax, tscale, num_sample, ] p_mask = [] for idx in range(num_sample): - bin_samples = total_samples[idx * num_sample_perbin:(idx + 1) * - num_sample_perbin] + bin_samples = total_samples[ + idx * num_sample_perbin : (idx + 1) * num_sample_perbin + ] bin_vector = np.zeros([tscale]) for sample in bin_samples: sample_upper = math.ceil(sample) @@ -94,34 +101,39 @@ def _get_interp1d_bin_mask(seg_xmin, seg_xmax, tscale, num_sample, class Conv1D(fluid.dygraph.Layer): - - def __init__(self, - prefix, - num_channels=256, - num_filters=256, - size_k=3, - padding=1, - groups=1, - act="relu"): + def __init__( + self, + prefix, + num_channels=256, + num_filters=256, + size_k=3, + padding=1, + groups=1, + act="relu", + ): super(Conv1D, self).__init__() fan_in = num_channels * size_k * 1 - k = 1. / math.sqrt(fan_in) - param_attr = ParamAttr(name=prefix + "_w", - initializer=fluid.initializer.Uniform(low=-k, - high=k)) - bias_attr = ParamAttr(name=prefix + "_b", - initializer=fluid.initializer.Uniform(low=-k, - high=k)) - - self._conv2d = fluid.dygraph.Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=(1, size_k), - stride=1, - padding=(0, padding), - groups=groups, - act=act, - param_attr=param_attr, - bias_attr=bias_attr) + k = 1.0 / math.sqrt(fan_in) + param_attr = ParamAttr( + name=prefix + "_w", + initializer=fluid.initializer.Uniform(low=-k, high=k), + ) + bias_attr = ParamAttr( + name=prefix + "_b", + initializer=fluid.initializer.Uniform(low=-k, high=k), + ) + + self._conv2d = fluid.dygraph.Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=(1, size_k), + stride=1, + padding=(0, padding), + groups=groups, + act=act, + param_attr=param_attr, + bias_attr=bias_attr, + ) def forward(self, x): x = fluid.layers.unsqueeze(input=x, axes=[2]) @@ -131,7 +143,6 @@ def forward(self, x): class BMN(fluid.dygraph.Layer): - def __init__(self, cfg): super(BMN, self).__init__() @@ -146,55 +157,65 @@ def __init__(self, cfg): self.hidden_dim_3d = 512 # Base Module - self.b_conv1 = Conv1D(prefix="Base_1", - num_channels=cfg.feat_dim, - num_filters=self.hidden_dim_1d, - size_k=3, - padding=1, - groups=4, - act="relu") - self.b_conv2 = Conv1D(prefix="Base_2", - num_filters=self.hidden_dim_1d, - size_k=3, - padding=1, - groups=4, - act="relu") + self.b_conv1 = Conv1D( + prefix="Base_1", + num_channels=cfg.feat_dim, + num_filters=self.hidden_dim_1d, + size_k=3, + padding=1, + groups=4, + act="relu", + ) + self.b_conv2 = Conv1D( + prefix="Base_2", + num_filters=self.hidden_dim_1d, + size_k=3, + padding=1, + groups=4, + act="relu", + ) # Temporal Evaluation Module - self.ts_conv1 = Conv1D(prefix="TEM_s1", - num_filters=self.hidden_dim_1d, - size_k=3, - padding=1, - groups=4, - act="relu") - self.ts_conv2 = Conv1D(prefix="TEM_s2", - num_filters=1, - size_k=1, - padding=0, - act="sigmoid") - self.te_conv1 = Conv1D(prefix="TEM_e1", - num_filters=self.hidden_dim_1d, - size_k=3, - padding=1, - groups=4, - act="relu") - self.te_conv2 = Conv1D(prefix="TEM_e2", - num_filters=1, - size_k=1, - padding=0, - act="sigmoid") - - #Proposal Evaluation Module - self.p_conv1 = Conv1D(prefix="PEM_1d", - num_filters=self.hidden_dim_2d, - size_k=3, - padding=1, - act="relu") + self.ts_conv1 = Conv1D( + prefix="TEM_s1", + num_filters=self.hidden_dim_1d, + size_k=3, + padding=1, + groups=4, + act="relu", + ) + self.ts_conv2 = Conv1D( + prefix="TEM_s2", num_filters=1, size_k=1, padding=0, act="sigmoid" + ) + self.te_conv1 = Conv1D( + prefix="TEM_e1", + num_filters=self.hidden_dim_1d, + size_k=3, + padding=1, + groups=4, + act="relu", + ) + self.te_conv2 = Conv1D( + prefix="TEM_e2", num_filters=1, size_k=1, padding=0, act="sigmoid" + ) + + # Proposal Evaluation Module + self.p_conv1 = Conv1D( + prefix="PEM_1d", + num_filters=self.hidden_dim_2d, + size_k=3, + padding=1, + act="relu", + ) # init to speed up - sample_mask = get_interp1d_mask(self.tscale, self.dscale, - self.prop_boundary_ratio, - self.num_sample, self.num_sample_perbin) + sample_mask = get_interp1d_mask( + self.tscale, + self.dscale, + self.prop_boundary_ratio, + self.num_sample, + self.num_sample_perbin, + ) self.sample_mask = fluid.dygraph.base.to_variable(sample_mask) self.sample_mask.stop_gradient = True @@ -206,7 +227,8 @@ def __init__(self, cfg): padding=0, act="relu", param_attr=ParamAttr(name="PEM_3d1_w"), - bias_attr=ParamAttr(name="PEM_3d1_b")) + bias_attr=ParamAttr(name="PEM_3d1_b"), + ) self.p_conv2d1 = fluid.dygraph.Conv2D( num_channels=512, @@ -216,7 +238,8 @@ def __init__(self, cfg): padding=0, act="relu", param_attr=ParamAttr(name="PEM_2d1_w"), - bias_attr=ParamAttr(name="PEM_2d1_b")) + bias_attr=ParamAttr(name="PEM_2d1_b"), + ) self.p_conv2d2 = fluid.dygraph.Conv2D( num_channels=128, num_filters=self.hidden_dim_2d, @@ -225,7 +248,8 @@ def __init__(self, cfg): padding=1, act="relu", param_attr=ParamAttr(name="PEM_2d2_w"), - bias_attr=ParamAttr(name="PEM_2d2_b")) + bias_attr=ParamAttr(name="PEM_2d2_b"), + ) self.p_conv2d3 = fluid.dygraph.Conv2D( num_channels=128, num_filters=self.hidden_dim_2d, @@ -234,7 +258,8 @@ def __init__(self, cfg): padding=1, act="relu", param_attr=ParamAttr(name="PEM_2d3_w"), - bias_attr=ParamAttr(name="PEM_2d3_b")) + bias_attr=ParamAttr(name="PEM_2d3_b"), + ) self.p_conv2d4 = fluid.dygraph.Conv2D( num_channels=128, num_filters=2, @@ -243,7 +268,8 @@ def __init__(self, cfg): padding=0, act="sigmoid", param_attr=ParamAttr(name="PEM_2d4_w"), - bias_attr=ParamAttr(name="PEM_2d4_b")) + bias_attr=ParamAttr(name="PEM_2d4_b"), + ) @to_static def forward(self, x): @@ -263,8 +289,9 @@ def forward(self, x): xp = self.p_conv1(x) # BM layer xp = fluid.layers.matmul(xp, self.sample_mask) - xp = fluid.layers.reshape(xp, - shape=[0, 0, -1, self.dscale, self.tscale]) + xp = fluid.layers.reshape( + xp, shape=[0, 0, -1, self.dscale, self.tscale] + ) xp = self.p_conv3d1(xp) xp = fluid.layers.squeeze(xp, axes=[2]) @@ -275,51 +302,54 @@ def forward(self, x): return xp, xs, xe -def bmn_loss_func(pred_bm, pred_start, pred_end, gt_iou_map, gt_start, gt_end, - cfg): - +def bmn_loss_func( + pred_bm, pred_start, pred_end, gt_iou_map, gt_start, gt_end, cfg +): def _get_mask(cfg): dscale = cfg.dscale tscale = cfg.tscale bm_mask = [] for idx in range(dscale): - mask_vector = [1 for i in range(tscale - idx) - ] + [0 for i in range(idx)] + mask_vector = [1 for i in range(tscale - idx)] + [ + 0 for i in range(idx) + ] bm_mask.append(mask_vector) bm_mask = np.array(bm_mask, dtype=np.float32) - self_bm_mask = fluid.layers.create_global_var(shape=[dscale, tscale], - value=0, - dtype=DATATYPE, - persistable=True) + self_bm_mask = fluid.layers.create_global_var( + shape=[dscale, tscale], value=0, dtype=DATATYPE, persistable=True + ) fluid.layers.assign(bm_mask, self_bm_mask) self_bm_mask.stop_gradient = True return self_bm_mask def tem_loss_func(pred_start, pred_end, gt_start, gt_end): - def bi_loss(pred_score, gt_label): - pred_score = fluid.layers.reshape(x=pred_score, - shape=[-1], - inplace=False) - gt_label = fluid.layers.reshape(x=gt_label, - shape=[-1], - inplace=False) + pred_score = fluid.layers.reshape( + x=pred_score, shape=[-1], inplace=False + ) + gt_label = fluid.layers.reshape( + x=gt_label, shape=[-1], inplace=False + ) gt_label.stop_gradient = True pmask = fluid.layers.cast(x=(gt_label > 0.5), dtype=DATATYPE) - num_entries = fluid.layers.cast(fluid.layers.shape(pmask), - dtype=DATATYPE) - num_positive = fluid.layers.cast(fluid.layers.reduce_sum(pmask), - dtype=DATATYPE) + num_entries = fluid.layers.cast( + fluid.layers.shape(pmask), dtype=DATATYPE + ) + num_positive = fluid.layers.cast( + fluid.layers.reduce_sum(pmask), dtype=DATATYPE + ) ratio = num_entries / num_positive coef_0 = 0.5 * ratio / (ratio - 1) coef_1 = 0.5 * ratio epsilon = 0.000001 # temp = fluid.layers.log(pred_score + epsilon) loss_pos = fluid.layers.elementwise_mul( - fluid.layers.log(pred_score + epsilon), pmask) + fluid.layers.log(pred_score + epsilon), pmask + ) loss_pos = coef_1 * fluid.layers.reduce_mean(loss_pos) loss_neg = fluid.layers.elementwise_mul( - fluid.layers.log(1.0 - pred_score + epsilon), (1.0 - pmask)) + fluid.layers.log(1.0 - pred_score + epsilon), (1.0 - pmask) + ) loss_neg = coef_0 * fluid.layers.reduce_mean(loss_neg) loss = -1 * (loss_pos + loss_neg) return loss @@ -336,39 +366,47 @@ def pem_reg_loss_func(pred_score, gt_iou_map, mask): u_hmask = fluid.layers.cast(x=gt_iou_map > 0.7, dtype=DATATYPE) u_mmask = fluid.layers.logical_and(gt_iou_map <= 0.7, gt_iou_map > 0.3) u_mmask = fluid.layers.cast(x=u_mmask, dtype=DATATYPE) - u_lmask = fluid.layers.logical_and(gt_iou_map <= 0.3, gt_iou_map >= 0.) + u_lmask = fluid.layers.logical_and(gt_iou_map <= 0.3, gt_iou_map >= 0.0) u_lmask = fluid.layers.cast(x=u_lmask, dtype=DATATYPE) u_lmask = fluid.layers.elementwise_mul(u_lmask, mask) - num_h = fluid.layers.cast(fluid.layers.reduce_sum(u_hmask), - dtype=DATATYPE) - num_m = fluid.layers.cast(fluid.layers.reduce_sum(u_mmask), - dtype=DATATYPE) - num_l = fluid.layers.cast(fluid.layers.reduce_sum(u_lmask), - dtype=DATATYPE) + num_h = fluid.layers.cast( + fluid.layers.reduce_sum(u_hmask), dtype=DATATYPE + ) + num_m = fluid.layers.cast( + fluid.layers.reduce_sum(u_mmask), dtype=DATATYPE + ) + num_l = fluid.layers.cast( + fluid.layers.reduce_sum(u_lmask), dtype=DATATYPE + ) r_m = num_h / num_m u_smmask = fluid.layers.assign( local_random.uniform( - 0., 1., - [gt_iou_map.shape[1], gt_iou_map.shape[2]]).astype(DATATYPE)) + 0.0, 1.0, [gt_iou_map.shape[1], gt_iou_map.shape[2]] + ).astype(DATATYPE) + ) u_smmask = fluid.layers.elementwise_mul(u_mmask, u_smmask) - u_smmask = fluid.layers.cast(x=(u_smmask > (1. - r_m)), dtype=DATATYPE) + u_smmask = fluid.layers.cast(x=(u_smmask > (1.0 - r_m)), dtype=DATATYPE) r_l = num_h / num_l u_slmask = fluid.layers.assign( local_random.uniform( - 0., 1., - [gt_iou_map.shape[1], gt_iou_map.shape[2]]).astype(DATATYPE)) + 0.0, 1.0, [gt_iou_map.shape[1], gt_iou_map.shape[2]] + ).astype(DATATYPE) + ) u_slmask = fluid.layers.elementwise_mul(u_lmask, u_slmask) - u_slmask = fluid.layers.cast(x=(u_slmask > (1. - r_l)), dtype=DATATYPE) + u_slmask = fluid.layers.cast(x=(u_slmask > (1.0 - r_l)), dtype=DATATYPE) weights = u_hmask + u_smmask + u_slmask weights.stop_gradient = True loss = fluid.layers.square_error_cost(pred_score, gt_iou_map) loss = fluid.layers.elementwise_mul(loss, weights) - loss = 0.5 * fluid.layers.reduce_sum(loss) / fluid.layers.reduce_sum( - weights) + loss = ( + 0.5 + * fluid.layers.reduce_sum(loss) + / fluid.layers.reduce_sum(weights) + ) return loss @@ -386,24 +424,22 @@ def pem_cls_loss_func(pred_score, gt_iou_map, mask): coef_1 = 0.5 * ratio epsilon = 0.000001 loss_pos = fluid.layers.elementwise_mul( - fluid.layers.log(pred_score + epsilon), pmask) + fluid.layers.log(pred_score + epsilon), pmask + ) loss_pos = coef_1 * fluid.layers.reduce_sum(loss_pos) loss_neg = fluid.layers.elementwise_mul( - fluid.layers.log(1.0 - pred_score + epsilon), nmask) + fluid.layers.log(1.0 - pred_score + epsilon), nmask + ) loss_neg = coef_0 * fluid.layers.reduce_sum(loss_neg) loss = -1 * (loss_pos + loss_neg) / num_entries return loss - pred_bm_reg = fluid.layers.squeeze(fluid.layers.slice(pred_bm, - axes=[1], - starts=[0], - ends=[1]), - axes=[1]) - pred_bm_cls = fluid.layers.squeeze(fluid.layers.slice(pred_bm, - axes=[1], - starts=[1], - ends=[2]), - axes=[1]) + pred_bm_reg = fluid.layers.squeeze( + fluid.layers.slice(pred_bm, axes=[1], starts=[0], ends=[1]), axes=[1] + ) + pred_bm_cls = fluid.layers.squeeze( + fluid.layers.slice(pred_bm, axes=[1], starts=[1], ends=[2]), axes=[1] + ) bm_mask = _get_mask(cfg) @@ -446,36 +482,35 @@ def optimizer(cfg, parameter_list): fluid.layers.piecewise_decay(boundaries=bd, values=lr), parameter_list=parameter_list, regularization=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=l2_weight_decay)) + regularization_coeff=l2_weight_decay + ), + ) return optimizer def fake_data_reader(args, mode='train'): - def iou_with_anchors(anchors_min, anchors_max, box_min, box_max): - """Compute jaccard score between a box and the anchors. - """ + """Compute jaccard score between a box and the anchors.""" len_anchors = anchors_max - anchors_min int_xmin = np.maximum(anchors_min, box_min) int_xmax = np.minimum(anchors_max, box_max) - inter_len = np.maximum(int_xmax - int_xmin, 0.) + inter_len = np.maximum(int_xmax - int_xmin, 0.0) union_len = len_anchors - inter_len + box_max - box_min jaccard = np.divide(inter_len, union_len) return jaccard def ioa_with_anchors(anchors_min, anchors_max, box_min, box_max): - """Compute intersection between score a box and the anchors. - """ + """Compute intersection between score a box and the anchors.""" len_anchors = anchors_max - anchors_min int_xmin = np.maximum(anchors_min, box_min) int_xmax = np.minimum(anchors_max, box_max) - inter_len = np.maximum(int_xmax - int_xmin, 0.) + inter_len = np.maximum(int_xmax - int_xmin, 0.0) scores = np.divide(inter_len, len_anchors) return scores def get_match_map(tscale): match_map = [] - tgap = 1. / tscale + tgap = 1.0 / tscale for idx in range(tscale): tmp_match_window = [] xmin = tgap * idx @@ -499,17 +534,21 @@ def get_video_label(match_map, anchor_xmin, anchor_xmax): gt_bbox = [] gt_iou_map = [] for idx in range(label_num): - duration = local_random.uniform(video_second * 0.4, - video_second * 0.8) - start_t = local_random.uniform(0.1 * video_second, - video_second - duration) + duration = local_random.uniform( + video_second * 0.4, video_second * 0.8 + ) + start_t = local_random.uniform( + 0.1 * video_second, video_second - duration + ) tmp_start = max(min(1, start_t / video_second), 0) tmp_end = max(min(1, (start_t + duration) / video_second), 0) gt_bbox.append([tmp_start, tmp_end]) - tmp_gt_iou_map = iou_with_anchors(match_map[:, 0], match_map[:, 1], - tmp_start, tmp_end) - tmp_gt_iou_map = np.reshape(tmp_gt_iou_map, - [args.dscale, args.tscale]) + tmp_gt_iou_map = iou_with_anchors( + match_map[:, 0], match_map[:, 1], tmp_start, tmp_end + ) + tmp_gt_iou_map = np.reshape( + tmp_gt_iou_map, [args.dscale, args.tscale] + ) gt_iou_map.append(tmp_gt_iou_map) gt_iou_map = np.array(gt_iou_map) gt_iou_map = np.max(gt_iou_map, axis=0) @@ -517,25 +556,38 @@ def get_video_label(match_map, anchor_xmin, anchor_xmax): gt_bbox = np.array(gt_bbox) gt_xmins = gt_bbox[:, 0] gt_xmaxs = gt_bbox[:, 1] - gt_len_small = 3. / args.tscale + gt_len_small = 3.0 / args.tscale gt_start_bboxs = np.stack( - (gt_xmins - gt_len_small / 2, gt_xmins + gt_len_small / 2), axis=1) + (gt_xmins - gt_len_small / 2, gt_xmins + gt_len_small / 2), axis=1 + ) gt_end_bboxs = np.stack( - (gt_xmaxs - gt_len_small / 2, gt_xmaxs + gt_len_small / 2), axis=1) + (gt_xmaxs - gt_len_small / 2, gt_xmaxs + gt_len_small / 2), axis=1 + ) match_score_start = [] for jdx in range(len(anchor_xmin)): match_score_start.append( np.max( - ioa_with_anchors(anchor_xmin[jdx], anchor_xmax[jdx], - gt_start_bboxs[:, 0], gt_start_bboxs[:, - 1]))) + ioa_with_anchors( + anchor_xmin[jdx], + anchor_xmax[jdx], + gt_start_bboxs[:, 0], + gt_start_bboxs[:, 1], + ) + ) + ) match_score_end = [] for jdx in range(len(anchor_xmin)): match_score_end.append( np.max( - ioa_with_anchors(anchor_xmin[jdx], anchor_xmax[jdx], - gt_end_bboxs[:, 0], gt_end_bboxs[:, 1]))) + ioa_with_anchors( + anchor_xmin[jdx], + anchor_xmax[jdx], + gt_end_bboxs[:, 0], + gt_end_bboxs[:, 1], + ) + ) + ) gt_start = np.array(match_score_start) gt_end = np.array(match_score_end) @@ -548,18 +600,22 @@ def reader(): for video_idx in range(iter_num): video_feat = local_random.random_sample( - [args.feat_dim, args.tscale]).astype('float32') + [args.feat_dim, args.tscale] + ).astype('float32') gt_iou_map, gt_start, gt_end = get_video_label( - match_map, anchor_xmin, anchor_xmax) + match_map, anchor_xmin, anchor_xmax + ) if mode == 'train' or mode == 'valid': batch_out.append((video_feat, gt_iou_map, gt_start, gt_end)) elif mode == 'test': batch_out.append( - (video_feat, gt_iou_map, gt_start, gt_end, video_idx)) + (video_feat, gt_iou_map, gt_start, gt_end, video_idx) + ) else: raise NotImplementedError( - 'mode {} not implemented'.format(mode)) + 'mode {} not implemented'.format(mode) + ) if len(batch_out) == args.batch_size: yield batch_out batch_out = [] @@ -589,20 +645,26 @@ def val_bmn(model, args): pred_bm, pred_start, pred_end = model(x_data) loss, tem_loss, pem_reg_loss, pem_cls_loss = bmn_loss_func( - pred_bm, pred_start, pred_end, gt_iou_map, gt_start, gt_end, args) + pred_bm, pred_start, pred_end, gt_iou_map, gt_start, gt_end, args + ) avg_loss = paddle.mean(loss) loss_data += [ avg_loss.numpy()[0], tem_loss.numpy()[0], pem_reg_loss.numpy()[0], - pem_cls_loss.numpy()[0] + pem_cls_loss.numpy()[0], ] - print('[VALID] iter {} '.format(batch_id) - + '\tLoss = {}, \ttem_loss = {}, \tpem_reg_loss = {}, \tpem_cls_loss = {}'.format( - '%f' % avg_loss.numpy()[0], '%f' % tem_loss.numpy()[0], \ - '%f' % pem_reg_loss.numpy()[0], '%f' % pem_cls_loss.numpy()[0])) + print( + '[VALID] iter {} '.format(batch_id) + + '\tLoss = {}, \ttem_loss = {}, \tpem_reg_loss = {}, \tpem_cls_loss = {}'.format( + '%f' % avg_loss.numpy()[0], + '%f' % tem_loss.numpy()[0], + '%f' % pem_reg_loss.numpy()[0], + '%f' % pem_cls_loss.numpy()[0], + ) + ) if batch_id == args.valid_batch_num: break @@ -610,11 +672,13 @@ def val_bmn(model, args): class TestTrain(unittest.TestCase): - def setUp(self): self.args = Args() - self.place = fluid.CPUPlace() if not fluid.is_compiled_with_cuda() \ + self.place = ( + fluid.CPUPlace() + if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0) + ) self.temp_dir = tempfile.TemporaryDirectory() self.model_save_dir = os.path.join(self.temp_dir.name, 'inference') @@ -643,14 +707,18 @@ def train_bmn(self, args, place, to_static): for epoch in range(args.epoch): for batch_id, data in enumerate(train_reader()): - video_feat = np.array([item[0] - for item in data]).astype(DATATYPE) - gt_iou_map = np.array([item[1] - for item in data]).astype(DATATYPE) - gt_start = np.array([item[2] - for item in data]).astype(DATATYPE) - gt_end = np.array([item[3] - for item in data]).astype(DATATYPE) + video_feat = np.array([item[0] for item in data]).astype( + DATATYPE + ) + gt_iou_map = np.array([item[1] for item in data]).astype( + DATATYPE + ) + gt_start = np.array([item[2] for item in data]).astype( + DATATYPE + ) + gt_end = np.array([item[3] for item in data]).astype( + DATATYPE + ) x_data = to_variable(video_feat) gt_iou_map = to_variable(gt_iou_map) @@ -663,8 +731,14 @@ def train_bmn(self, args, place, to_static): pred_bm, pred_start, pred_end = bmn(x_data) loss, tem_loss, pem_reg_loss, pem_cls_loss = bmn_loss_func( - pred_bm, pred_start, pred_end, gt_iou_map, gt_start, - gt_end, args) + pred_bm, + pred_start, + pred_end, + gt_iou_map, + gt_start, + gt_end, + args, + ) avg_loss = paddle.mean(loss) avg_loss.backward() @@ -675,15 +749,21 @@ def train_bmn(self, args, place, to_static): avg_loss.numpy()[0], tem_loss.numpy()[0], pem_reg_loss.numpy()[0], - pem_cls_loss.numpy()[0] + pem_cls_loss.numpy()[0], ] - if args.log_interval > 0 and (batch_id % args.log_interval - == 0): - print('[TRAIN] Epoch {}, iter {} '.format(epoch, batch_id) - + '\tLoss = {}, \ttem_loss = {}, \tpem_reg_loss = {}, \tpem_cls_loss = {}'.format( - '%f' % avg_loss.numpy()[0], '%f' % tem_loss.numpy()[0], \ - '%f' % pem_reg_loss.numpy()[0], '%f' % pem_cls_loss.numpy()[0])) + if args.log_interval > 0 and ( + batch_id % args.log_interval == 0 + ): + print( + '[TRAIN] Epoch {}, iter {} '.format(epoch, batch_id) + + '\tLoss = {}, \ttem_loss = {}, \tpem_reg_loss = {}, \tpem_cls_loss = {}'.format( + '%f' % avg_loss.numpy()[0], + '%f' % tem_loss.numpy()[0], + '%f' % pem_reg_loss.numpy()[0], + '%f' % pem_cls_loss.numpy()[0], + ) + ) # validation if batch_id % args.valid_interval == 0 and batch_id > 0: @@ -696,8 +776,9 @@ def train_bmn(self, args, place, to_static): if to_static: fluid.dygraph.jit.save(bmn, self.model_save_prefix) else: - fluid.dygraph.save_dygraph(bmn.state_dict(), - self.dy_param_path) + fluid.dygraph.save_dygraph( + bmn.state_dict(), self.dy_param_path + ) break return np.array(loss_data) @@ -711,8 +792,10 @@ def test_train(self): rtol=1e-05, err_msg='dygraph_res: {},\n static_res: {}'.format( dygraph_res[~np.isclose(dygraph_res, static_res)], - static_res[~np.isclose(dygraph_res, static_res)]), - atol=1e-8) + static_res[~np.isclose(dygraph_res, static_res)], + ), + atol=1e-8, + ) # Prediction needs trained models, so put `test_predict` at last of `test_train` self.verify_predict() @@ -729,32 +812,41 @@ def verify_predict(self): predictor_pred_res = self.predict_analysis_inference(video_data) for dy_res, st_res, dy_jit_res, predictor_res in zip( - dygraph_pred_res, static_pred_res, dygraph_jit_pred_res, - predictor_pred_res): + dygraph_pred_res, + static_pred_res, + dygraph_jit_pred_res, + predictor_pred_res, + ): np.testing.assert_allclose( st_res, dy_res, rtol=1e-05, err_msg='dygraph_res: {},\n static_res: {}'.format( dy_res[~np.isclose(st_res, dy_res)], - st_res[~np.isclose(st_res, dy_res)]), - atol=1e-8) + st_res[~np.isclose(st_res, dy_res)], + ), + atol=1e-8, + ) np.testing.assert_allclose( st_res, dy_jit_res, rtol=1e-05, err_msg='dygraph_jit_res: {},\n static_res: {}'.format( dy_jit_res[~np.isclose(st_res, dy_jit_res)], - st_res[~np.isclose(st_res, dy_jit_res)]), - atol=1e-8) + st_res[~np.isclose(st_res, dy_jit_res)], + ), + atol=1e-8, + ) np.testing.assert_allclose( st_res, predictor_res, rtol=1e-05, err_msg='dygraph_jit_res: {},\n static_res: {}'.format( predictor_res[~np.isclose(st_res, predictor_res)], - st_res[~np.isclose(st_res, predictor_res)]), - atol=1e-8) + st_res[~np.isclose(st_res, predictor_res)], + ), + atol=1e-8, + ) break def predict_dygraph(self, data): @@ -776,14 +868,21 @@ def predict_static(self, data): paddle.enable_static() exe = fluid.Executor(self.place) # load inference model - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(self.model_save_dir, - executor=exe, - model_filename=self.model_filename, - params_filename=self.params_filename) - pred_res = exe.run(inference_program, - feed={feed_target_names[0]: data}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + self.model_save_dir, + executor=exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + ) + pred_res = exe.run( + inference_program, + feed={feed_target_names[0]: data}, + fetch_list=fetch_targets, + ) return pred_res @@ -799,8 +898,12 @@ def predict_dygraph_jit(self, data): return pred_res def predict_analysis_inference(self, data): - output = PredictorTools(self.model_save_dir, self.model_filename, - self.params_filename, [data]) + output = PredictorTools( + self.model_save_dir, + self.model_filename, + self.params_filename, + [data], + ) out = output() return out diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py index b5840bc45e3e6..45b149617d2bf 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_break_continue.py @@ -17,7 +17,9 @@ import paddle import paddle.fluid as fluid from paddle.fluid.dygraph.jit import declarative -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramTranslator, +) from paddle.fluid.dygraph.dygraph_to_static.utils import Dygraph2StaticException SEED = 2020 @@ -25,7 +27,6 @@ class TestDy2staticException(unittest.TestCase): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = None @@ -159,9 +160,7 @@ def test_for_in_else(x): def while_loop_class_var(x): - class Foo(object): - def __init__(self): self.a = 3 self.b = 4 @@ -205,11 +204,13 @@ def test_optim_break_in_while(x): class TestContinueInFor(unittest.TestCase): - def setUp(self): self.input = np.zeros((1)).astype('int64') - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.init_dygraph_func() def init_dygraph_func(self): @@ -233,59 +234,52 @@ def test_transformed_static_result(self): static_res, rtol=1e-05, err_msg='dygraph res is {}\nstatic_res is {}'.format( - dygraph_res, static_res)) + dygraph_res, static_res + ), + ) class TestContinueInForAtEnd(TestContinueInFor): - def init_dygraph_func(self): self.dygraph_func = test_continue_in_for_at_end class TestBreakInFor(TestContinueInFor): - def init_dygraph_func(self): self.dygraph_func = test_break_in_for class TestBreakInForAtEnd(TestContinueInFor): - def init_dygraph_func(self): self.dygraph_func = test_break_in_for_at_end class TestBreakContinueInFor(TestContinueInFor): - def init_dygraph_func(self): self.dygraph_func = test_break_continue_in_for class TestForInElse(TestContinueInFor): - def init_dygraph_func(self): self.dygraph_func = test_for_in_else class TestContinueInWhile(TestContinueInFor): - def init_dygraph_func(self): self.dygraph_func = test_continue_in_while class TestBreakInWhile(TestContinueInWhile): - def init_dygraph_func(self): self.dygraph_func = test_break_in_while class TestWhileLoopClassVar(TestContinueInWhile): - def init_dygraph_func(self): self.dygraph_func = while_loop_class_var class TestOptimBreakInFor(TestDy2staticException): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = test_optim_break_in_for @@ -293,7 +287,6 @@ def setUp(self): class TestOptimBreakInWhile(TestContinueInWhile): - def init_dygraph_func(self): self.dygraph_func = test_optim_break_in_while diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cache_program.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cache_program.py index f23d9ba783c30..7df7c63331235 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cache_program.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cache_program.py @@ -26,7 +26,6 @@ class TestCacheProgram(unittest.TestCase): - def setUp(self): self.batch_num = 5 self.dygraph_class = Pool2D @@ -44,26 +43,35 @@ def test_cache(self): cur_out = out # Check forward ops prev_ops = cur_ops - cur_ops = Counter([ - op.type for op in fluid.default_main_program().block(0).ops - ]) + cur_ops = Counter( + [ + op.type + for op in fluid.default_main_program().block(0).ops + ] + ) if batch_id > 0: - prev_out_numpy = prev_out[0].numpy() if isinstance( - prev_out, (tuple, list)) else prev_out.numpy() - cur_out_numpy = cur_out[0].numpy() if isinstance( - cur_out, (tuple, list)) else cur_out.numpy() + prev_out_numpy = ( + prev_out[0].numpy() + if isinstance(prev_out, (tuple, list)) + else prev_out.numpy() + ) + cur_out_numpy = ( + cur_out[0].numpy() + if isinstance(cur_out, (tuple, list)) + else cur_out.numpy() + ) np.testing.assert_allclose( prev_out_numpy, cur_out_numpy, rtol=1e-05, - err_msg= - 'Output in previous batch is {}\n Output in current batch is \n{}' - .format(prev_out_numpy, cur_out_numpy)) + err_msg='Output in previous batch is {}\n Output in current batch is \n{}'.format( + prev_out_numpy, cur_out_numpy + ), + ) self.assertEqual(prev_ops, cur_ops) class TestCacheProgram2(TestCacheProgram): - def setUp(self): self.batch_num = 5 self.dygraph_class = Linear @@ -71,7 +79,6 @@ def setUp(self): class TestCacheProgramWithOptimizer(unittest.TestCase): - def setUp(self): self.dygraph_class = Linear self.data = np.random.random((4, 10)).astype('float32') @@ -90,7 +97,8 @@ def train(self, to_static=False): with fluid.dygraph.guard(fluid.CPUPlace()): dygraph_net = self.dygraph_class() adam = fluid.optimizer.AdamOptimizer( - learning_rate=0.001, parameter_list=dygraph_net.parameters()) + learning_rate=0.001, parameter_list=dygraph_net.parameters() + ) loss_data = [] for batch_id in range(self.batch_num): input = fluid.dygraph.to_variable(self.data) @@ -111,7 +119,9 @@ def test_with_optimizer(self): static_loss, rtol=1e-05, err_msg='dygraph is {}\n static_res is \n{}'.format( - dygraph_loss, static_loss)) + dygraph_loss, static_loss + ), + ) def simple_func(x): @@ -121,7 +131,6 @@ def simple_func(x): class TestConvertWithCache(unittest.TestCase): - def test_cache(self): static_func = convert_to_static(simple_func) # Get transformed function from cache. @@ -152,7 +161,6 @@ def sum_under_while(limit): class TestToOutputWithCache(unittest.TestCase): - def test_output(self): with fluid.dygraph.guard(): ret = sum_even_until_limit(80, 10) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cast.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cast.py index fc233ffbc02fe..17f598f0f17e8 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cast.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cast.py @@ -59,19 +59,23 @@ def test_mix_cast(x): class TestCastBase(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.prepare() self.set_func() def prepare(self): self.input_shape = (16, 32) self.input_dtype = 'float32' - self.input = np.random.binomial( - 4, 0.3, size=np.product(self.input_shape)).reshape( - self.input_shape).astype(self.input_dtype) + self.input = ( + np.random.binomial(4, 0.3, size=np.product(self.input_shape)) + .reshape(self.input_shape) + .astype(self.input_dtype) + ) self.cast_dtype = 'bool' def set_func(self): @@ -87,24 +91,29 @@ def test_cast_result(self): self.assertTrue( res.dtype == self.cast_dtype, msg='The target dtype is {}, but the casted dtype is {}.'.format( - self.cast_dtype, res.dtype)) + self.cast_dtype, res.dtype + ), + ) ref_val = self.input.astype(self.cast_dtype) np.testing.assert_allclose( res, ref_val, rtol=1e-05, err_msg='The casted value is {}.\nThe correct value is {}.'.format( - res, ref_val)) + res, ref_val + ), + ) class TestIntCast(TestCastBase): - def prepare(self): - self.input_shape = (1, ) + self.input_shape = (1,) self.input_dtype = 'float32' - self.input = np.random.normal( - loc=6, scale=10, size=np.product(self.input_shape)).reshape( - self.input_shape).astype(self.input_dtype) + self.input = ( + np.random.normal(loc=6, scale=10, size=np.product(self.input_shape)) + .reshape(self.input_shape) + .astype(self.input_dtype) + ) self.cast_dtype = 'int32' def set_func(self): @@ -112,13 +121,14 @@ def set_func(self): class TestFloatCast(TestCastBase): - def prepare(self): self.input_shape = (8, 16) self.input_dtype = 'bool' - self.input = np.random.binomial( - 2, 0.5, size=np.product(self.input_shape)).reshape( - self.input_shape).astype(self.input_dtype) + self.input = ( + np.random.binomial(2, 0.5, size=np.product(self.input_shape)) + .reshape(self.input_shape) + .astype(self.input_dtype) + ) self.cast_dtype = 'float32' def set_func(self): @@ -126,13 +136,14 @@ def set_func(self): class TestMixCast(TestCastBase): - def prepare(self): self.input_shape = (8, 32) self.input_dtype = 'float32' - self.input = np.random.normal( - loc=6, scale=10, size=np.product(self.input_shape)).reshape( - self.input_shape).astype(self.input_dtype) + self.input = ( + np.random.normal(loc=6, scale=10, size=np.product(self.input_shape)) + .reshape(self.input_shape) + .astype(self.input_dtype) + ) self.cast_int = 'int' self.cast_float = 'float32' self.cast_bool = 'bool' @@ -146,19 +157,26 @@ def test_cast_result(self): self.assertTrue( res.dtype == self.cast_dtype, msg='The target dtype is {}, but the casted dtype is {}.'.format( - self.cast_dtype, res.dtype)) - ref_val = self.input.astype(self.cast_int).astype( - self.cast_float).astype(self.cast_bool).astype(self.cast_dtype) + self.cast_dtype, res.dtype + ), + ) + ref_val = ( + self.input.astype(self.cast_int) + .astype(self.cast_float) + .astype(self.cast_bool) + .astype(self.cast_dtype) + ) np.testing.assert_allclose( res, ref_val, rtol=1e-05, err_msg='The casted value is {}.\nThe correct value is {}.'.format( - res, ref_val)) + res, ref_val + ), + ) class TestNotVarCast(TestCastBase): - def prepare(self): self.input = 3.14 self.cast_dtype = 'int' @@ -173,7 +191,9 @@ def test_cast_result(self): self.assertTrue( res == ref_val, msg='The casted value is {}.\nThe correct value is {}.'.format( - res, ref_val)) + res, ref_val + ), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_closure_analysis.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_closure_analysis.py index 846576400d7f5..8712950e01b05 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_closure_analysis.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_closure_analysis.py @@ -15,7 +15,9 @@ import unittest import paddle -from paddle.fluid.dygraph.dygraph_to_static.utils import FunctionNameLivenessAnalysis +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + FunctionNameLivenessAnalysis, +) from paddle.utils import gast import inspect from numpy import append @@ -24,7 +26,6 @@ class JudgeVisitor(gast.NodeVisitor): - def __init__(self, ans, mod): self.ans = ans self.mod = mod @@ -34,27 +35,30 @@ def visit_FunctionDef(self, node): expected = self.ans.get(node.name, set()) exp_mod = self.mod.get(node.name, set()) assert scope.existed_vars() == expected, "Not Equals." - assert scope.modified_vars( - ) == exp_mod, "Not Equals in function:{} . expect {} , but get {}".format( - node.name, exp_mod, scope.modified_vars()) + assert ( + scope.modified_vars() == exp_mod + ), "Not Equals in function:{} . expect {} , but get {}".format( + node.name, exp_mod, scope.modified_vars() + ) self.generic_visit(node) class JudgePushPopVisitor(gast.NodeVisitor): - def __init__(self, push_pop_vars): self.pp_var = push_pop_vars def visit_FunctionDef(self, node): scope = node.pd_scope expected = self.pp_var.get(node.name, set()) - assert scope.push_pop_vars == expected, "Not Equals in function:{} . expect {} , but get {}".format( - node.name, expected, scope.push_pop_vars) + assert ( + scope.push_pop_vars == expected + ), "Not Equals in function:{} . expect {} , but get {}".format( + node.name, expected, scope.push_pop_vars + ) self.generic_visit(node) def test_normal_0(x): - def func(): if True: i = 1 @@ -103,8 +107,7 @@ def func(*args, **kargs): def test_push_pop_1(x, *args, **kargs): - """ push_pop_vars in main_function is : `l`, `k` - """ + """push_pop_vars in main_function is : `l`, `k`""" l = [] k = [] for i in range(10): @@ -114,8 +117,7 @@ def test_push_pop_1(x, *args, **kargs): def test_push_pop_2(x, *args, **kargs): - """ push_pop_vars in main_function is : `k` - """ + """push_pop_vars in main_function is : `k`""" l = [] k = [] @@ -128,10 +130,10 @@ def func(): def test_push_pop_3(x, *args, **kargs): - """ push_pop_vars in main_function is : `k` - NOTE: One may expect `k` and `l` because l - is nonlocal. Name bind analysis is - not implemented yet. + """push_pop_vars in main_function is : `k` + NOTE: One may expect `k` and `l` because l + is nonlocal. Name bind analysis is + not implemented yet. """ l = [] k = [] @@ -146,8 +148,7 @@ def func(): def test_push_pop_4(x, *args, **kargs): - """ push_pop_vars in main_function is : `k` - """ + """push_pop_vars in main_function is : `k`""" l = [] k = [] for i in range(10): @@ -160,20 +161,19 @@ def test_push_pop_4(x, *args, **kargs): class TestClosureAnalysis(unittest.TestCase): - def setUp(self): self.judge_type = "var and w_vars" self.init_dygraph_func() def init_dygraph_func(self): self.all_dygraph_funcs = [ - test_nonlocal, test_global, test_normal_0, test_normal_argument + test_nonlocal, + test_global, + test_normal_0, + test_normal_argument, ] self.answer = [ - { - 'func': set('k'), - 'test_nonlocal': set('i') - }, + {'func': set('k'), 'test_nonlocal': set('i')}, { 'func': set({'i'}), }, @@ -186,34 +186,27 @@ def init_dygraph_func(self): ] self.modified_var = [ - { - 'func': set('ki'), - 'test_nonlocal': set('i') - }, - { - 'func': set({'i'}), - 'test_global': set({"t"}) - }, + {'func': set('ki'), 'test_nonlocal': set('i')}, + {'func': set({'i'}), 'test_global': set({"t"})}, { 'func': set('i'), }, - { - 'func': set('i'), - 'test_normal_argument': set('x') - }, + {'func': set('i'), 'test_normal_argument': set('x')}, ] def test_main(self): if self.judge_type == 'push_pop_vars': - for push_pop_vars, func in zip(self.push_pop_vars, - self.all_dygraph_funcs): + for push_pop_vars, func in zip( + self.push_pop_vars, self.all_dygraph_funcs + ): test_func = inspect.getsource(func) gast_root = gast.parse(test_func) name_visitor = FunctionNameLivenessAnalysis(gast_root) JudgePushPopVisitor(push_pop_vars).visit(gast_root) else: - for mod, ans, func in zip(self.modified_var, self.answer, - self.all_dygraph_funcs): + for mod, ans, func in zip( + self.modified_var, self.answer, self.all_dygraph_funcs + ): test_func = inspect.getsource(func) gast_root = gast.parse(test_func) name_visitor = FunctionNameLivenessAnalysis(gast_root) @@ -227,41 +220,48 @@ def TestClosureAnalysis_Attribute_func(): class TestClosureAnalysis_Attribute(TestClosureAnalysis): - def init_dygraph_func(self): self.all_dygraph_funcs = [TestClosureAnalysis_Attribute_func] self.answer = [{"TestClosureAnalysis_Attribute_func": set({'i'})}] - self.modified_var = [{ - "TestClosureAnalysis_Attribute_func": - set({'i', 'self.current.function'}) - }] + self.modified_var = [ + { + "TestClosureAnalysis_Attribute_func": set( + {'i', 'self.current.function'} + ) + } + ] class TestClosureAnalysis_PushPop(TestClosureAnalysis): - def init_dygraph_func(self): self.judge_type = "push_pop_vars" self.all_dygraph_funcs = [ - test_push_pop_1, test_push_pop_2, test_push_pop_3, test_push_pop_4 + test_push_pop_1, + test_push_pop_2, + test_push_pop_3, + test_push_pop_4, + ] + self.push_pop_vars = [ + { + "test_push_pop_1": set({'l', 'k'}), + }, + { + "test_push_pop_2": set({'k'}), + "func": set("l"), + }, + { + "test_push_pop_3": set({'k'}), + "func": set("l"), + }, + { + "test_push_pop_4": set({'k', 'l'}), + }, ] - self.push_pop_vars = [{ - "test_push_pop_1": set({'l', 'k'}), - }, { - "test_push_pop_2": set({'k'}), - "func": set("l"), - }, { - "test_push_pop_3": set({'k'}), - "func": set("l"), - }, { - "test_push_pop_4": set({'k', 'l'}), - }] class TestPushPopTrans(unittest.TestCase): - def test(self): - def vlist_of_dict(x): ma = {'a': []} for i in range(3): @@ -299,7 +299,6 @@ def vlist_of_dict(x): print(paddle.jit.to_static(vlist_of_dict)(x)) def test4(self): - def vlist_of_dict(x): a = np.array([1, 2, 3]) for i in range(3): @@ -311,7 +310,6 @@ def vlist_of_dict(x): print(paddle.jit.to_static(vlist_of_dict)(x)) def test5(self): - def vlist_of_dict(x): a = np.array([1, 2, 3]) for i in range(3): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py index 73f0262daa1b9..f22ca8776165a 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_container.py @@ -20,14 +20,13 @@ class BufferLayers(paddle.nn.Layer): - def __init__(self, out_channel): super(BufferLayers, self).__init__() self.out_channel = out_channel def forward(self, x): mean = paddle.mean(x) - if mean < 0.: + if mean < 0.0: x = x * self._mask() out = x - mean @@ -38,13 +37,13 @@ def _mask(self): class SequentialNet(paddle.nn.Layer): - def __init__(self, sub_layer, in_channel, out_channel): super(SequentialNet, self).__init__() self.layer = paddle.nn.Sequential( ('l1', paddle.nn.Linear(in_channel, in_channel)), ('l2', paddle.nn.Linear(in_channel, out_channel)), - ('l3', sub_layer(out_channel))) + ('l3', sub_layer(out_channel)), + ) def forward(self, x): out = self.layer(x) @@ -52,7 +51,6 @@ def forward(self, x): class NestSequentialNet(paddle.nn.Layer): - def __init__(self): super().__init__() group1 = paddle.nn.Sequential( @@ -70,7 +68,6 @@ def forward(self, x): class TestSequential(unittest.TestCase): - def setUp(self): paddle.set_device('cpu') self.seed = 2021 @@ -100,7 +97,8 @@ def _run(self, to_static): load_out, out, rtol=1e-05, - err_msg='load_out is {}\\st_out is {}'.format(load_out, out)) + err_msg='load_out is {}\\st_out is {}'.format(load_out, out), + ) return out @@ -113,7 +111,9 @@ def test_train(self): st_out, rtol=1e-05, err_msg='dygraph_res is {}\nstatic_res is {}'.format( - dy_out, st_out)) + dy_out, st_out + ), + ) def _test_load(self, net, x): paddle.jit.save(net, self.model_path) @@ -123,11 +123,11 @@ def _test_load(self, net, x): class TestNestSequential(TestSequential): - def _init_config(self): self.net = NestSequentialNet() - self.model_path = os.path.join(self.temp_dir.name, - 'nested_sequential_net') + self.model_path = os.path.join( + self.temp_dir.name, 'nested_sequential_net' + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call.py index 2e0983ad5ee1f..241df1517e2db 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call.py @@ -20,7 +20,9 @@ import paddle import paddle.fluid as fluid from paddle.fluid.dygraph import ProgramTranslator -from paddle.fluid.dygraph.dygraph_to_static.convert_call_func import CONVERSION_OPTIONS +from paddle.fluid.dygraph.dygraph_to_static.convert_call_func import ( + CONVERSION_OPTIONS, +) from test_program_translator import get_source_code import paddle.jit.dy2static as _jst @@ -64,7 +66,6 @@ def dyfunc_with_third_library_logging(x_v): class A: - @staticmethod def add(a, b): """ @@ -81,11 +82,13 @@ def dyfunc_with_staticmethod(x_v): class TestRecursiveCall1(unittest.TestCase): - def setUp(self): self.input = np.random.random([10, 16]).astype('float32') - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.init_test_func() def init_test_func(self): @@ -111,24 +114,28 @@ def test_transformed_static_result(self): static_res, rtol=1e-05, err_msg='dygraph res is {}\nstatic_res is {}'.format( - dygraph_res, static_res)) + dygraph_res, static_res + ), + ) lambda_fun = lambda x: x class MyConvLayer(fluid.dygraph.Layer): - def __init__(self): super(MyConvLayer, self).__init__() self._conv = fluid.dygraph.Conv2D( num_channels=3, num_filters=2, filter_size=3, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), + ) @paddle.jit.to_static def forward(self, inputs): @@ -144,7 +151,6 @@ def dymethod(self, x_v): class MyLayer(fluid.dygraph.Layer): - def __init__(self): super(MyLayer, self).__init__() @@ -153,10 +159,13 @@ def __init__(self): input_dim=5, output_dim=1, act='relu', - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), + ) @paddle.jit.to_static def forward(self, inputs): @@ -166,11 +175,13 @@ def forward(self, inputs): class TestRecursiveCall2(unittest.TestCase): - def setUp(self): self.input = np.random.random((1, 3, 3, 5)).astype('float32') - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.set_func() def set_func(self): @@ -198,13 +209,11 @@ def test_transformed_static_result(self): class TestThirdPartyLibrary(TestRecursiveCall2): - def set_func(self): self.dygraph_func = dyfunc_with_third_library_logging class TestStaticMethod(TestRecursiveCall2): - def set_func(self): self.dygraph_func = dyfunc_with_staticmethod @@ -230,7 +239,6 @@ def func_convert_then_not_to_static(x): class TestClass(paddle.nn.Layer): - @paddle.jit.not_to_static def called_member(self, x): return paddle.sum(x) @@ -242,7 +250,6 @@ def forward(self, x): class TestNotToConvert(TestRecursiveCall2): - def set_func(self): self.dygraph_func = func_not_to_static @@ -253,19 +260,16 @@ def test_conversion_options(self): class TestNotToConvert2(TestRecursiveCall2): - def set_func(self): self.dygraph_func = func_convert_then_not_to_static class TestNotToConvert3(TestRecursiveCall2): - def set_func(self): self.dygraph_func = TestClass() class TestDynamicToStaticCode(unittest.TestCase): - def setUp(self): self.set_func() self.set_answer_func() @@ -274,9 +278,7 @@ def set_func(self): self.func = func_not_to_static def set_answer_func(self): - - class StaticCode(): - + class StaticCode: @paddle.jit.not_to_static def func_not_to_static(x): res = func_sum(x) @@ -298,18 +300,17 @@ def test_code(self): answer_code, transformed_code, msg="\ntransformed_code : \n{}\nanswer_code : \n{}".format( - transformed_code, answer_code)) + transformed_code, answer_code + ), + ) class TestDynamicToStaticCode2(TestDynamicToStaticCode): - def set_func(self): self.func = func_convert_then_not_to_static def set_answer_func(self): - - class StaticCode(): - + class StaticCode: def func_convert_then_not_to_static(x): __return_value_0 = None y = _jst.Call(func_not_to_static)(x) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call_generator.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call_generator.py index 9a417cfe5853b..a938d67915854 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call_generator.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_call_generator.py @@ -24,14 +24,12 @@ def dyfunc_generator(): def main_func(): - """ Error will raise, but we only report a warning not intercept - """ + """Error will raise, but we only report a warning not intercept""" for i in dyfunc_generator(): print(i) class TestConvertGenerator(unittest.TestCase): - def test_raise_error(self): with self.assertRaises(Exception): to_static(main_func)() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_operators.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_operators.py index 65aec5ca8dd0e..284febe07cf4d 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_operators.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_convert_operators.py @@ -18,14 +18,12 @@ class CallNotExist(paddle.nn.Layer): - def __call__(self): # call a non-exist API to trigger exception return paddle.nn.not_exist_api class ForwardNotExist(paddle.nn.Layer): - def forward(self): return 0 @@ -35,9 +33,7 @@ def forward(self): class TestConvertCall(unittest.TestCase): - def test_class_exception(self): - @paddle.jit.to_static def call_not_exist(): net = CallNotExist() @@ -55,16 +51,17 @@ def forward_not_exist(): class TestConvertShapeCompare(unittest.TestCase): - def test_non_variable(self): - self.assertEqual(paddle.jit.dy2static.convert_shape_compare(1, "<", 2), - True) self.assertEqual( - paddle.jit.dy2static.convert_shape_compare(1, "<", 2, "<=", 3), - True) + paddle.jit.dy2static.convert_shape_compare(1, "<", 2), True + ) + self.assertEqual( + paddle.jit.dy2static.convert_shape_compare(1, "<", 2, "<=", 3), True + ) self.assertEqual( paddle.jit.dy2static.convert_shape_compare(1, ">", 2, "<=", 3), - False) + False, + ) def error_func(): """ @@ -73,77 +70,108 @@ def error_func(): raise ValueError("Used for test") self.assertEqual( - paddle.jit.dy2static.convert_shape_compare(1, ">", 2, "<=", - lambda: error_func()), - False) + paddle.jit.dy2static.convert_shape_compare( + 1, ">", 2, "<=", lambda: error_func() + ), + False, + ) self.assertEqual( - paddle.jit.dy2static.convert_shape_compare(1, "<", 2, "in", - [1, 2, 3]), True) + paddle.jit.dy2static.convert_shape_compare( + 1, "<", 2, "in", [1, 2, 3] + ), + True, + ) self.assertEqual( - paddle.jit.dy2static.convert_shape_compare(1, "<", 2, "not in", - [1, 2, 3]), False) + paddle.jit.dy2static.convert_shape_compare( + 1, "<", 2, "not in", [1, 2, 3] + ), + False, + ) self.assertEqual( paddle.jit.dy2static.convert_shape_compare(1, "<", 2, "is", 3), - False) + False, + ) self.assertEqual( - paddle.jit.dy2static.convert_shape_compare(1, "<", 2, "is not", - [1, 2, 3]), True) + paddle.jit.dy2static.convert_shape_compare( + 1, "<", 2, "is not", [1, 2, 3] + ), + True, + ) self.assertEqual( - paddle.jit.dy2static.convert_shape_compare([1, 2], "==", [1, 2], - "!=", [1, 2, 3]), True) + paddle.jit.dy2static.convert_shape_compare( + [1, 2], "==", [1, 2], "!=", [1, 2, 3] + ), + True, + ) self.assertEqual( - paddle.jit.dy2static.convert_shape_compare([1, 2], "!=", [1, 2, 3], - "==", [1, 2]), False) + paddle.jit.dy2static.convert_shape_compare( + [1, 2], "!=", [1, 2, 3], "==", [1, 2] + ), + False, + ) def test_variable(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.static.data(name='x', shape=[3, 2], dtype='float32') y = paddle.static.data(name='y', shape=[3, 2], dtype='float32') self.assertEqual( paddle.jit.dy2static.convert_shape_compare( - x, "is", x, "is not", y), True) + x, "is", x, "is not", y + ), + True, + ) self.assertEqual( paddle.jit.dy2static.convert_shape_compare( - x, "is not", x, "is not", y), False) + x, "is not", x, "is not", y + ), + False, + ) self.assertEqual( paddle.jit.dy2static.convert_shape_compare(x, "is", x, "is", y), - False) + False, + ) eq_out = paddle.jit.dy2static.convert_shape_compare(x, "==", y) not_eq_out = paddle.jit.dy2static.convert_shape_compare(x, "!=", y) long_eq_out = paddle.jit.dy2static.convert_shape_compare( - x, "==", x, "!=", y) - - place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + x, "==", x, "!=", y + ) + + place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) - x_y_eq_out = exe.run(feed={ - "x": np.ones([3, 2]).astype(np.float32), - "y": np.ones([3, 2]).astype(np.float32) - }, - fetch_list=[eq_out, not_eq_out, long_eq_out]) - np.testing.assert_array_equal(np.array(x_y_eq_out), - np.array([[True], [False], [False]])) + x_y_eq_out = exe.run( + feed={ + "x": np.ones([3, 2]).astype(np.float32), + "y": np.ones([3, 2]).astype(np.float32), + }, + fetch_list=[eq_out, not_eq_out, long_eq_out], + ) + np.testing.assert_array_equal( + np.array(x_y_eq_out), np.array([[True], [False], [False]]) + ) set_a_zero = np.ones([3, 2]).astype(np.float32) set_a_zero[0][0] = 0.0 x_y_not_eq_out = exe.run( - feed={ - "x": np.ones([3, 2]).astype(np.float32), - "y": set_a_zero - }, - fetch_list=[eq_out, not_eq_out, long_eq_out]) - np.testing.assert_array_equal(np.array(x_y_not_eq_out), - np.array([[False], [True], [True]])) + feed={"x": np.ones([3, 2]).astype(np.float32), "y": set_a_zero}, + fetch_list=[eq_out, not_eq_out, long_eq_out], + ) + np.testing.assert_array_equal( + np.array(x_y_not_eq_out), np.array([[False], [True], [True]]) + ) paddle.disable_static() class ShapeLayer(paddle.nn.Layer): - def __init__(self): super(ShapeLayer, self).__init__() @@ -158,7 +186,6 @@ def forward(self, x): class TestChooseShapeAttrOrApiWithLayer(unittest.TestCase): - def test_tensor_shape(self): x = paddle.zeros(shape=[4, 1], dtype='float32') net = ShapeLayer() @@ -168,7 +195,6 @@ def test_tensor_shape(self): class TestIfElseNoValue(unittest.TestCase): - def test_else_ret_none(self): input_x = paddle.to_tensor([[1, 2, 3], [4, 5, 6]]) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cpu_cuda_to_tensor.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cpu_cuda_to_tensor.py index 53149f758baff..60f7f70ad4bd7 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cpu_cuda_to_tensor.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cpu_cuda_to_tensor.py @@ -18,9 +18,7 @@ class TestCpuCuda(unittest.TestCase): - def test_cpu_cuda(self): - def func(x): x = paddle.to_tensor([1, 2, 3, 4]) x = x.cuda() @@ -33,9 +31,7 @@ def func(x): class TestToTensor(unittest.TestCase): - def test_to_tensor_with_variable_list(self): - def func(x): ones = paddle.to_tensor([1]) twos = paddle.to_tensor([2]) @@ -44,15 +40,15 @@ def func(x): x = paddle.to_tensor([3]) print(paddle.jit.to_static(func).code) - np.testing.assert_allclose(paddle.jit.to_static(func)(x).numpy(), - np.array([1, 2, 3, 4]), - rtol=1e-05) + np.testing.assert_allclose( + paddle.jit.to_static(func)(x).numpy(), + np.array([1, 2, 3, 4]), + rtol=1e-05, + ) class TestToTensor1(unittest.TestCase): - def test_to_tensor_with_variable_list(self): - def func(x): ones = paddle.to_tensor([1]) twos = paddle.to_tensor([2]) @@ -64,24 +60,26 @@ def func(x): x = paddle.to_tensor([3]) print(paddle.jit.to_static(func).code) - np.testing.assert_allclose(paddle.jit.to_static(func)(x).numpy(), - np.array([1, 2, 3, 4]), - rtol=1e-05) + np.testing.assert_allclose( + paddle.jit.to_static(func)(x).numpy(), + np.array([1, 2, 3, 4]), + rtol=1e-05, + ) class TestToTensor2(unittest.TestCase): - def test_to_tensor_with_variable_list(self): - def func(x): x = paddle.to_tensor([[1], [2], [3], [4]]) return x x = paddle.to_tensor([3]) print(paddle.jit.to_static(func).code) - np.testing.assert_allclose(paddle.jit.to_static(func)(x).numpy(), - np.array([[1], [2], [3], [4]]), - rtol=1e-05) + np.testing.assert_allclose( + paddle.jit.to_static(func)(x).numpy(), + np.array([[1], [2], [3], [4]]), + rtol=1e-05, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cycle_gan.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cycle_gan.py index c5dd0ac064b0e..03db89350795c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cycle_gan.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_cycle_gan.py @@ -30,6 +30,7 @@ from PIL import Image, ImageOps import os + # Use GPU:0 to elimate the influence of other tasks. os.environ["CUDA_VISIBLE_DEVICES"] = "1" @@ -61,19 +62,22 @@ class Cycle_Gan(fluid.dygraph.Layer): - def __init__(self, input_channel, istrain=True): super(Cycle_Gan, self).__init__() self.build_generator_resnet_9blocks_a = build_generator_resnet_9blocks( - input_channel) + input_channel + ) self.build_generator_resnet_9blocks_b = build_generator_resnet_9blocks( - input_channel) + input_channel + ) if istrain: self.build_gen_discriminator_a = build_gen_discriminator( - input_channel) + input_channel + ) self.build_gen_discriminator_b = build_gen_discriminator( - input_channel) + input_channel + ) @declarative def forward(self, input_A, input_B): @@ -86,9 +90,11 @@ def forward(self, input_A, input_B): cyc_B = self.build_generator_resnet_9blocks_a(fake_A) diff_A = fluid.layers.abs( - fluid.layers.elementwise_sub(x=input_A, y=cyc_A)) + fluid.layers.elementwise_sub(x=input_A, y=cyc_A) + ) diff_B = fluid.layers.abs( - fluid.layers.elementwise_sub(x=input_B, y=cyc_B)) + fluid.layers.elementwise_sub(x=input_B, y=cyc_B) + ) cyc_A_loss = fluid.layers.reduce_mean(diff_A) * lambda_A cyc_B_loss = fluid.layers.reduce_mean(diff_B) * lambda_B cyc_loss = cyc_A_loss + cyc_B_loss @@ -100,17 +106,41 @@ def forward(self, input_A, input_B): g_B_loss = fluid.layers.reduce_mean(fluid.layers.square(fake_rec_B - 1)) G = g_A_loss + g_B_loss idt_A = self.build_generator_resnet_9blocks_a(input_B) - idt_loss_A = fluid.layers.reduce_mean( - fluid.layers.abs(fluid.layers.elementwise_sub( - x=input_B, y=idt_A))) * lambda_B * lambda_identity + idt_loss_A = ( + fluid.layers.reduce_mean( + fluid.layers.abs( + fluid.layers.elementwise_sub(x=input_B, y=idt_A) + ) + ) + * lambda_B + * lambda_identity + ) idt_B = self.build_generator_resnet_9blocks_b(input_A) - idt_loss_B = fluid.layers.reduce_mean( - fluid.layers.abs(fluid.layers.elementwise_sub( - x=input_A, y=idt_B))) * lambda_A * lambda_identity + idt_loss_B = ( + fluid.layers.reduce_mean( + fluid.layers.abs( + fluid.layers.elementwise_sub(x=input_A, y=idt_B) + ) + ) + * lambda_A + * lambda_identity + ) idt_loss = fluid.layers.elementwise_add(idt_loss_A, idt_loss_B) g_loss = cyc_loss + G + idt_loss - return fake_A, fake_B, cyc_A, cyc_B, g_A_loss, g_B_loss, idt_loss_A, idt_loss_B, cyc_A_loss, cyc_B_loss, g_loss + return ( + fake_A, + fake_B, + cyc_A, + cyc_B, + g_A_loss, + g_B_loss, + idt_loss_A, + idt_loss_B, + cyc_A_loss, + cyc_B_loss, + g_loss, + ) @declarative def discriminatorA(self, input_A, input_B): @@ -134,23 +164,26 @@ def discriminatorB(self, input_A, input_B): class build_resnet_block(fluid.dygraph.Layer): - def __init__(self, dim, use_bias=False): super(build_resnet_block, self).__init__() - self.conv0 = conv2d(num_channels=dim, - num_filters=dim, - filter_size=3, - stride=1, - stddev=0.02, - use_bias=False) - self.conv1 = conv2d(num_channels=dim, - num_filters=dim, - filter_size=3, - stride=1, - stddev=0.02, - relu=False, - use_bias=False) + self.conv0 = conv2d( + num_channels=dim, + num_filters=dim, + filter_size=3, + stride=1, + stddev=0.02, + use_bias=False, + ) + self.conv1 = conv2d( + num_channels=dim, + num_filters=dim, + filter_size=3, + stride=1, + stddev=0.02, + relu=False, + use_bias=False, + ) self.dim = dim def forward(self, inputs): @@ -163,33 +196,39 @@ def forward(self, inputs): class build_generator_resnet_9blocks(fluid.dygraph.Layer): - def __init__(self, input_channel): super(build_generator_resnet_9blocks, self).__init__() - self.conv0 = conv2d(num_channels=input_channel, - num_filters=32, - filter_size=7, - stride=1, - padding=0, - stddev=0.02) - self.conv1 = conv2d(num_channels=32, - num_filters=64, - filter_size=3, - stride=2, - padding=1, - stddev=0.02) - self.conv2 = conv2d(num_channels=64, - num_filters=128, - filter_size=3, - stride=2, - padding=1, - stddev=0.02) + self.conv0 = conv2d( + num_channels=input_channel, + num_filters=32, + filter_size=7, + stride=1, + padding=0, + stddev=0.02, + ) + self.conv1 = conv2d( + num_channels=32, + num_filters=64, + filter_size=3, + stride=2, + padding=1, + stddev=0.02, + ) + self.conv2 = conv2d( + num_channels=64, + num_filters=128, + filter_size=3, + stride=2, + padding=1, + stddev=0.02, + ) self.build_resnet_block_list = [] dim = 128 for i in range(9): - Build_Resnet_Block = self.add_sublayer("generator_%d" % (i + 1), - build_resnet_block(dim)) + Build_Resnet_Block = self.add_sublayer( + "generator_%d" % (i + 1), build_resnet_block(dim) + ) self.build_resnet_block_list.append(Build_Resnet_Block) self.deconv0 = DeConv2D( num_channels=dim, @@ -200,22 +239,26 @@ def __init__(self, input_channel): padding=[1, 1], outpadding=[0, 1, 0, 1], ) - self.deconv1 = DeConv2D(num_channels=32 * 2, - num_filters=32, - filter_size=3, - stride=2, - stddev=0.02, - padding=[1, 1], - outpadding=[0, 1, 0, 1]) - self.conv3 = conv2d(num_channels=32, - num_filters=input_channel, - filter_size=7, - stride=1, - stddev=0.02, - padding=0, - relu=False, - norm=False, - use_bias=True) + self.deconv1 = DeConv2D( + num_channels=32 * 2, + num_filters=32, + filter_size=3, + stride=2, + stddev=0.02, + padding=[1, 1], + outpadding=[0, 1, 0, 1], + ) + self.conv3 = conv2d( + num_channels=32, + num_filters=input_channel, + filter_size=7, + stride=1, + stddev=0.02, + padding=0, + relu=False, + norm=False, + use_bias=True, + ) def forward(self, inputs): pad_input = fluid.layers.pad2d(inputs, [3, 3, 3, 3], mode="reflect") @@ -233,49 +276,58 @@ def forward(self, inputs): class build_gen_discriminator(fluid.dygraph.Layer): - def __init__(self, input_channel): super(build_gen_discriminator, self).__init__() - self.conv0 = conv2d(num_channels=input_channel, - num_filters=64, - filter_size=4, - stride=2, - stddev=0.02, - padding=1, - norm=False, - use_bias=True, - relufactor=0.2) - self.conv1 = conv2d(num_channels=64, - num_filters=128, - filter_size=4, - stride=2, - stddev=0.02, - padding=1, - relufactor=0.2) - self.conv2 = conv2d(num_channels=128, - num_filters=IMAGE_SIZE, - filter_size=4, - stride=2, - stddev=0.02, - padding=1, - relufactor=0.2) - self.conv3 = conv2d(num_channels=IMAGE_SIZE, - num_filters=512, - filter_size=4, - stride=1, - stddev=0.02, - padding=1, - relufactor=0.2) - self.conv4 = conv2d(num_channels=512, - num_filters=1, - filter_size=4, - stride=1, - stddev=0.02, - padding=1, - norm=False, - relu=False, - use_bias=True) + self.conv0 = conv2d( + num_channels=input_channel, + num_filters=64, + filter_size=4, + stride=2, + stddev=0.02, + padding=1, + norm=False, + use_bias=True, + relufactor=0.2, + ) + self.conv1 = conv2d( + num_channels=64, + num_filters=128, + filter_size=4, + stride=2, + stddev=0.02, + padding=1, + relufactor=0.2, + ) + self.conv2 = conv2d( + num_channels=128, + num_filters=IMAGE_SIZE, + filter_size=4, + stride=2, + stddev=0.02, + padding=1, + relufactor=0.2, + ) + self.conv3 = conv2d( + num_channels=IMAGE_SIZE, + num_filters=512, + filter_size=4, + stride=1, + stddev=0.02, + padding=1, + relufactor=0.2, + ) + self.conv4 = conv2d( + num_channels=512, + num_filters=1, + filter_size=4, + stride=1, + stddev=0.02, + padding=1, + norm=False, + relu=False, + use_bias=True, + ) def forward(self, inputs): y = self.conv0(inputs) @@ -289,35 +341,42 @@ def forward(self, inputs): class conv2d(fluid.dygraph.Layer): """docstring for Conv2D""" - def __init__(self, - num_channels, - num_filters=64, - filter_size=7, - stride=1, - stddev=0.02, - padding=0, - norm=True, - relu=True, - relufactor=0.0, - use_bias=False): + def __init__( + self, + num_channels, + num_filters=64, + filter_size=7, + stride=1, + stddev=0.02, + padding=0, + norm=True, + relu=True, + relufactor=0.0, + use_bias=False, + ): super(conv2d, self).__init__() if use_bias == False: con_bias_attr = False else: con_bias_attr = fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.0)) - - self.conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=padding, - use_cudnn=use_cudnn, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.NormalInitializer( - loc=0.0, scale=stddev)), - bias_attr=con_bias_attr) + initializer=fluid.initializer.Constant(0.0) + ) + + self.conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=padding, + use_cudnn=use_cudnn, + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=stddev + ) + ), + bias_attr=con_bias_attr, + ) # Note(Aurelius84): The calculation of GPU kernel in BN is non-deterministic, # failure rate is 1/100 in Dev but seems incremental in CE platform. # If on GPU, we disable BN temporarily. @@ -328,10 +387,13 @@ def __init__(self, use_global_stats=True, # set True to use deterministic algorithm num_channels=num_filters, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.NormalInitializer(1.0, 0.02)), + initializer=fluid.initializer.NormalInitializer(1.0, 0.02) + ), bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.0)), - trainable_statistics=True) + initializer=fluid.initializer.Constant(0.0) + ), + trainable_statistics=True, + ) self.relufactor = relufactor self.use_bias = use_bias @@ -348,26 +410,28 @@ def forward(self, inputs): class DeConv2D(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters=64, - filter_size=7, - stride=1, - stddev=0.02, - padding=[0, 0], - outpadding=[0, 0, 0, 0], - relu=True, - norm=True, - relufactor=0.0, - use_bias=False): + def __init__( + self, + num_channels, + num_filters=64, + filter_size=7, + stride=1, + stddev=0.02, + padding=[0, 0], + outpadding=[0, 0, 0, 0], + relu=True, + norm=True, + relufactor=0.0, + use_bias=False, + ): super(DeConv2D, self).__init__() if use_bias == False: de_bias_attr = False else: de_bias_attr = fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.0)) + initializer=fluid.initializer.Constant(0.0) + ) self._deconv = Conv2DTranspose( num_channels, @@ -377,9 +441,12 @@ def __init__(self, padding=padding, use_cudnn=use_cudnn, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.NormalInitializer(loc=0.0, - scale=stddev)), - bias_attr=de_bias_attr) + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=stddev + ) + ), + bias_attr=de_bias_attr, + ) if fluid.is_compiled_with_cuda(): norm = False if norm: @@ -387,10 +454,13 @@ def __init__(self, use_global_stats=True, # set True to use deterministic algorithm num_channels=num_filters, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.NormalInitializer(1.0, 0.02)), + initializer=fluid.initializer.NormalInitializer(1.0, 0.02) + ), bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.0)), - trainable_statistics=True) + initializer=fluid.initializer.Constant(0.0) + ), + trainable_statistics=True, + ) self.outpadding = outpadding self.relufactor = relufactor @@ -400,10 +470,9 @@ def __init__(self, def forward(self, inputs): conv = self._deconv(inputs) - conv = fluid.layers.pad2d(conv, - paddings=self.outpadding, - mode='constant', - pad_value=0.0) + conv = fluid.layers.pad2d( + conv, paddings=self.outpadding, mode='constant', pad_value=0.0 + ) if self.norm: conv = self.bn(conv) @@ -413,7 +482,6 @@ def forward(self, inputs): class ImagePool(object): - def __init__(self, pool_size=50): self.pool = [] self.count = 0 @@ -436,11 +504,11 @@ def pool_image(self, image): def reader_creater(): - def reader(): while True: fake_image = np.uint8( - np.random.random((IMAGE_SIZE + 30, IMAGE_SIZE + 30, 3)) * 255) + np.random.random((IMAGE_SIZE + 30, IMAGE_SIZE + 30, 3)) * 255 + ) image = Image.fromarray(fake_image) # Resize image = image.resize((286, 286), Image.BICUBIC) @@ -474,20 +542,29 @@ class Args(object): def optimizer_setting(parameters): lr = 0.0002 - optimizer = fluid.optimizer.Adam(learning_rate=fluid.layers.piecewise_decay( - boundaries=[ - 100 * step_per_epoch, 120 * step_per_epoch, 140 * step_per_epoch, - 160 * step_per_epoch, 180 * step_per_epoch - ], - values=[lr, lr * 0.8, lr * 0.6, lr * 0.4, lr * 0.2, lr * 0.1]), - parameter_list=parameters, - beta1=0.5) + optimizer = fluid.optimizer.Adam( + learning_rate=fluid.layers.piecewise_decay( + boundaries=[ + 100 * step_per_epoch, + 120 * step_per_epoch, + 140 * step_per_epoch, + 160 * step_per_epoch, + 180 * step_per_epoch, + ], + values=[lr, lr * 0.8, lr * 0.6, lr * 0.4, lr * 0.2, lr * 0.1], + ), + parameter_list=parameters, + beta1=0.5, + ) return optimizer def train(args, to_static): - place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ + place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + ) program_translator.enable(to_static) @@ -507,8 +584,10 @@ def train(args, to_static): cycle_gan = Cycle_Gan(input_channel=data_shape[1], istrain=True) t_time = 0 - vars_G = cycle_gan.build_generator_resnet_9blocks_a.parameters( - ) + cycle_gan.build_generator_resnet_9blocks_b.parameters() + vars_G = ( + cycle_gan.build_generator_resnet_9blocks_a.parameters() + + cycle_gan.build_generator_resnet_9blocks_b.parameters() + ) vars_da = cycle_gan.build_gen_discriminator_a.parameters() vars_db = cycle_gan.build_gen_discriminator_b.parameters() @@ -525,17 +604,28 @@ def train(args, to_static): s_time = time.time() data_A = np.array( - [data_A[0].reshape(3, IMAGE_SIZE, - IMAGE_SIZE)]).astype("float32") + [data_A[0].reshape(3, IMAGE_SIZE, IMAGE_SIZE)] + ).astype("float32") data_B = np.array( - [data_B[0].reshape(3, IMAGE_SIZE, - IMAGE_SIZE)]).astype("float32") + [data_B[0].reshape(3, IMAGE_SIZE, IMAGE_SIZE)] + ).astype("float32") data_A = to_variable(data_A) data_B = to_variable(data_B) # optimize the g_A network - fake_A, fake_B, cyc_A, cyc_B, g_A_loss, g_B_loss, idt_loss_A, idt_loss_B, cyc_A_loss, cyc_B_loss, g_loss = cycle_gan( - data_A, data_B) + ( + fake_A, + fake_B, + cyc_A, + cyc_B, + g_A_loss, + g_B_loss, + idt_loss_A, + idt_loss_B, + cyc_A_loss, + cyc_B_loss, + g_loss, + ) = cycle_gan(data_A, data_B) g_loss.backward() optimizer1.minimize(g_loss) @@ -543,21 +633,24 @@ def train(args, to_static): fake_pool_B = B_pool.pool_image(fake_B).numpy() fake_pool_B = np.array( - [fake_pool_B[0].reshape(3, IMAGE_SIZE, - IMAGE_SIZE)]).astype("float32") + [fake_pool_B[0].reshape(3, IMAGE_SIZE, IMAGE_SIZE)] + ).astype("float32") fake_pool_B = to_variable(fake_pool_B) fake_pool_A = A_pool.pool_image(fake_A).numpy() fake_pool_A = np.array( - [fake_pool_A[0].reshape(3, IMAGE_SIZE, - IMAGE_SIZE)]).astype("float32") + [fake_pool_A[0].reshape(3, IMAGE_SIZE, IMAGE_SIZE)] + ).astype("float32") fake_pool_A = to_variable(fake_pool_A) # optimize the d_A network rec_B, fake_pool_rec_B = cycle_gan.discriminatorA( - data_B, fake_pool_B) - d_loss_A = (fluid.layers.square(fake_pool_rec_B) + - fluid.layers.square(rec_B - 1)) / 2.0 + data_B, fake_pool_B + ) + d_loss_A = ( + fluid.layers.square(fake_pool_rec_B) + + fluid.layers.square(rec_B - 1) + ) / 2.0 d_loss_A = fluid.layers.reduce_mean(d_loss_A) d_loss_A.backward() @@ -566,9 +659,12 @@ def train(args, to_static): # optimize the d_B network rec_A, fake_pool_rec_A = cycle_gan.discriminatorB( - data_A, fake_pool_A) - d_loss_B = (fluid.layers.square(fake_pool_rec_A) + - fluid.layers.square(rec_A - 1)) / 2.0 + data_A, fake_pool_A + ) + d_loss_B = ( + fluid.layers.square(fake_pool_rec_A) + + fluid.layers.square(rec_A - 1) + ) / 2.0 d_loss_B = fluid.layers.reduce_mean(d_loss_B) d_loss_B.backward() @@ -578,8 +674,15 @@ def train(args, to_static): # Log generator loss and discriminator loss cur_batch_loss = [ - g_loss, d_loss_A, d_loss_B, g_A_loss, cyc_A_loss, - idt_loss_A, g_B_loss, cyc_B_loss, idt_loss_B + g_loss, + d_loss_A, + d_loss_B, + g_A_loss, + cyc_A_loss, + idt_loss_A, + g_B_loss, + cyc_B_loss, + idt_loss_B, ] cur_batch_loss = [x.numpy()[0] for x in cur_batch_loss] @@ -587,8 +690,10 @@ def train(args, to_static): t_time += batch_time if batch_id % args.log_step == 0: print( - "batch: {}\t Batch_time_cost: {}\n g_loss: {}\t d_A_loss: {}\t d_B_loss:{}\n g_A_loss: {}\t g_A_cyc_loss: {}\t g_A_idt_loss: {}\n g_B_loss: {}\t g_B_cyc_loss: {}\t g_B_idt_loss: {}" - .format(batch_id, batch_time, *cur_batch_loss)) + "batch: {}\t Batch_time_cost: {}\n g_loss: {}\t d_A_loss: {}\t d_B_loss:{}\n g_A_loss: {}\t g_A_cyc_loss: {}\t g_A_idt_loss: {}\n g_B_loss: {}\t g_B_cyc_loss: {}\t g_B_idt_loss: {}".format( + batch_id, batch_time, *cur_batch_loss + ) + ) if batch_id > args.train_step: break @@ -598,7 +703,6 @@ def train(args, to_static): class TestCycleGANModel(unittest.TestCase): - def setUp(self): self.args = Args() @@ -617,8 +721,10 @@ def test_train(self): if not fluid.is_compiled_with_cuda(): assert_func = np.array_equal - self.assertTrue(assert_func(dy_out, st_out), - msg="dy_out:\n {}\n st_out:\n{}".format(dy_out, st_out)) + self.assertTrue( + assert_func(dy_out, st_out), + msg="dy_out:\n {}\n st_out:\n{}".format(dy_out, st_out), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_declarative.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_declarative.py index 947cfa35d434e..6ea76d7592588 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_declarative.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_declarative.py @@ -19,8 +19,17 @@ import paddle import paddle.fluid as fluid from paddle.static import InputSpec -from paddle.fluid.dygraph import to_variable, declarative, ProgramTranslator, Layer, jit -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ConcreteProgram, StaticFunction +from paddle.fluid.dygraph import ( + to_variable, + declarative, + ProgramTranslator, + Layer, + jit, +) +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ConcreteProgram, + StaticFunction, +) from test_basic_api_transformation import dyfunc_to_variable @@ -28,7 +37,6 @@ class SimpleNet(Layer): - def __init__(self): super(SimpleNet, self).__init__() self.linear = fluid.dygraph.Linear(10, 3) @@ -55,10 +63,9 @@ def func_with_list(self, l, int_val=1): z = z + int_val return z - @declarative(input_spec=[{ - 'x': InputSpec([None, 10]), - 'y': InputSpec([None, 10]) - }]) + @declarative( + input_spec=[{'x': InputSpec([None, 10]), 'y': InputSpec([None, 10])}] + ) def func_with_dict(self, d): x = d['x'] y = d['y'] @@ -66,12 +73,14 @@ def func_with_dict(self, d): return z - @declarative(input_spec=[[ - InputSpec([None]), { - 'x': InputSpec([None, 10]), - 'y': InputSpec([None, 10]) - } - ]]) + @declarative( + input_spec=[ + [ + InputSpec([None]), + {'x': InputSpec([None, 10]), 'y': InputSpec([None, 10])}, + ] + ] + ) def func_with_list_dict(self, dl): bias = dl[0] x = dl[1]['x'] @@ -84,7 +93,6 @@ def func_with_list_dict(self, dl): class TestStaticFunctionInstance(unittest.TestCase): - def test_instance_same_class(self): with fluid.dygraph.guard(fluid.CPUPlace()): net_1 = SimpleNet() @@ -102,7 +110,6 @@ def test_instance_same_class(self): class TestInputSpec(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.model_path = os.path.join(self.temp_dir.name, 'simple_net') @@ -114,7 +121,7 @@ def test_with_input_spec(self): with fluid.dygraph.guard(fluid.CPUPlace()): x = to_variable(np.ones([4, 10]).astype('float32')) y = to_variable(np.ones([4, 10]).astype('float32') * 2) - int_val = 4. + int_val = 4.0 net = SimpleNet() @@ -150,7 +157,7 @@ def test_with_error(self): with fluid.dygraph.guard(fluid.CPUPlace()): x = to_variable(np.ones([4, 10]).astype('float32')) y = to_variable(np.ones([4, 10]).astype('float32') * 2) - int_val = 4. + int_val = 4.0 net = SimpleNet() @@ -160,26 +167,28 @@ def test_with_error(self): # 2. requires len(input_spec) <= len(args) with self.assertRaises(ValueError): - net.add_func = declarative(net.add_func, - input_spec=[ - InputSpec([-1, 10]), - InputSpec([-1, 10]), - InputSpec([10]) - ]) + net.add_func = declarative( + net.add_func, + input_spec=[ + InputSpec([-1, 10]), + InputSpec([-1, 10]), + InputSpec([10]), + ], + ) net.add_func(x, y) def test_concrete_program(self): with fluid.dygraph.guard(fluid.CPUPlace()): x = to_variable(np.ones([4, 10]).astype('float32')) y = to_variable(np.ones([4, 10]).astype('float32') * 2) - int_val = 4. + int_val = 4.0 net = SimpleNet() # We can get concrete_program by specificing InputSpec information. Faking input is no need. net.add_func = declarative( net.add_func, - input_spec=[InputSpec([-1, 10]), - InputSpec([-1, 10], name='y')]) + input_spec=[InputSpec([-1, 10]), InputSpec([-1, 10], name='y')], + ) cp1 = net.add_func.concrete_program self.assertTrue(cp1.inputs[-1].shape == (-1, 10)) self.assertTrue(cp1.inputs[-1].name == 'y') @@ -187,10 +196,10 @@ def test_concrete_program(self): # generate another program net.add_func = declarative( net.add_func, - input_spec=[InputSpec([10]), - InputSpec([10], name='label')]) + input_spec=[InputSpec([10]), InputSpec([10], name='label')], + ) cp2 = net.add_func.concrete_program - self.assertTrue(cp2.inputs[-1].shape == (10, )) + self.assertTrue(cp2.inputs[-1].shape == (10,)) self.assertTrue(cp2.inputs[-1].name == 'label') # Note(Aurelius84): New instance will be returned if we use `declarative(foo)` every time. # So number of cache program is 1. @@ -204,7 +213,6 @@ def foo_func(a, b, c=1, d=2): class TestDifferentInputSpecCacheProgram(unittest.TestCase): - def setUp(self): program_trans.enable(True) @@ -218,33 +226,33 @@ def test_with_different_input(self): # [16, 10] + [10] (varbase) out_1 = foo(to_variable(x_data), to_variable(y_data)) - np.testing.assert_allclose(x_data + y_data, - out_1.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_data + y_data, out_1.numpy(), rtol=1e-05 + ) self.assertTrue(len(foo.program_cache) == 1) self.assertTrue(len(foo.program_cache.concrete_programs()) == 1) first_program = foo.program_cache.last() # [16, 10] + [10] (numpy) out_2 = foo(to_variable(x_data), y_data) - np.testing.assert_allclose(x_data + y_data, - out_2.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_data + y_data, out_2.numpy(), rtol=1e-05 + ) self.assertTrue(len(foo.program_cache) == 1) # [16, 10] + [10] (numpy) out_3 = foo(to_variable(x_data), z_data) - np.testing.assert_allclose(x_data + z_data, - out_3.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_data + z_data, out_3.numpy(), rtol=1e-05 + ) # hit cache program self.assertTrue(len(foo.program_cache) == 1) # [16, 10] + [10] (numpy) with other different arguments (c=3) out_4 = foo(to_variable(x_data), z_data, 3) - np.testing.assert_allclose(x_data + z_data, - out_4.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_data + z_data, out_4.numpy(), rtol=1e-05 + ) # create a new program self.assertTrue(len(foo.program_cache) == 2) @@ -258,26 +266,29 @@ def test_get_concrete_program(self): foo = declarative(foo_func) # 1. specific InputSpec for `x`/`y` - concrete_program_1 = foo.get_concrete_program(InputSpec([None, 10]), - InputSpec([10])) + concrete_program_1 = foo.get_concrete_program( + InputSpec([None, 10]), InputSpec([10]) + ) self.assertTrue(len(foo.program_cache) == 1) # 2. specific `c`/`d` explicitly with same default value - concrete_program_2 = foo.get_concrete_program(InputSpec([None, 10]), - InputSpec([10]), 1, 2) + concrete_program_2 = foo.get_concrete_program( + InputSpec([None, 10]), InputSpec([10]), 1, 2 + ) self.assertTrue(concrete_program_2 == concrete_program_1) self.assertTrue(len(foo.program_cache) == 1) # 3. specific `c` = 2 - concrete_program_3 = foo.get_concrete_program(InputSpec([None, 10]), - InputSpec([10]), - c=2) + concrete_program_3 = foo.get_concrete_program( + InputSpec([None, 10]), InputSpec([10]), c=2 + ) self.assertTrue(concrete_program_3 != concrete_program_1) self.assertTrue(len(foo.program_cache) == 2) # 4. specific x.shape = [10] - concrete_program_4 = foo.get_concrete_program(InputSpec([10]), - InputSpec([10])) + concrete_program_4 = foo.get_concrete_program( + InputSpec([10]), InputSpec([10]) + ) self.assertTrue(concrete_program_4 != concrete_program_1) self.assertTrue(len(foo.program_cache) == 3) @@ -287,19 +298,21 @@ def test_get_concrete_program(self): # 6. specific unknown kwargs `e`=4 with self.assertRaises(TypeError): - concrete_program_5 = foo.get_concrete_program(InputSpec([10]), - InputSpec([10]), - e=4) + concrete_program_5 = foo.get_concrete_program( + InputSpec([10]), InputSpec([10]), e=4 + ) def test_concrete_program(self): with fluid.dygraph.guard(fluid.CPUPlace()): # usage 1 - foo_1 = paddle.jit.to_static(foo_func, - input_spec=[ - InputSpec([10], name='x'), - InputSpec([10], name='y') - ]) + foo_1 = paddle.jit.to_static( + foo_func, + input_spec=[ + InputSpec([10], name='x'), + InputSpec([10], name='y'), + ], + ) self.assertTrue(isinstance(foo_1.concrete_program, ConcreteProgram)) # usage 2 @@ -314,7 +327,6 @@ def test_concrete_program(self): class TestInputDefaultName(unittest.TestCase): - def setUp(self): paddle.disable_static() self.net = SimpleNet() @@ -339,7 +351,6 @@ def test_nest_input(self): class TestDeclarativeAPI(unittest.TestCase): - def test_error(self): func = declarative(dyfunc_to_variable) @@ -358,7 +369,6 @@ def test_error(self): class TestDecorateModelDirectly(unittest.TestCase): - def setUp(self): paddle.disable_static() program_trans.enable(True) @@ -385,27 +395,28 @@ def test_input_spec(self): class TestErrorWithInitFromStaticMode(unittest.TestCase): - def test_raise_error(self): # disable imperative paddle.enable_static() net = SimpleNet() - with self.assertRaisesRegexp(RuntimeError, - "only available in dynamic mode"): + with self.assertRaisesRegexp( + RuntimeError, "only available in dynamic mode" + ): net.forward.concrete_program - with self.assertRaisesRegexp(RuntimeError, - "only available in dynamic mode"): + with self.assertRaisesRegexp( + RuntimeError, "only available in dynamic mode" + ): net.forward.inputs - with self.assertRaisesRegexp(RuntimeError, - "only available in dynamic mode"): + with self.assertRaisesRegexp( + RuntimeError, "only available in dynamic mode" + ): net.forward.outputs class CallNonForwardFuncNet(paddle.nn.Layer): - def __init__(self): super(CallNonForwardFuncNet, self).__init__() self.sub = CallNonForwardFuncSubNet() @@ -416,7 +427,6 @@ def forward(self): class CallNonForwardFuncSubNet(paddle.nn.Layer): - def __init__(self): super(CallNonForwardFuncSubNet, self).__init__() self.a = paddle.to_tensor([1, 2]) @@ -427,7 +437,6 @@ def func(self): class TestCallNonForwardFunc(unittest.TestCase): - def test_call_non_forward(self): paddle.disable_static() net = CallNonForwardFuncNet() @@ -437,7 +446,6 @@ def test_call_non_forward(self): class SetBuffersNet1(paddle.nn.Layer): - def __init__(self): super(SetBuffersNet1, self).__init__() self.a = paddle.to_tensor([1]) @@ -449,7 +457,6 @@ def forward(self): class SetBuffersNet2(paddle.nn.Layer): - def __init__(self): super(SetBuffersNet2, self).__init__() self.b = paddle.to_tensor([2]) @@ -462,7 +469,6 @@ def forward(self): class TestSetBuffers(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.model_path = os.path.join(self.temp_dir.name, 'SetBuffersNet1') @@ -487,13 +493,11 @@ def test_set_buffers2(self): class ClassNoInheritLayer: - def func(self, x): return x + 1 class TestClassNoInheritLayer(unittest.TestCase): - def test_to_static(self): paddle.disable_static() net = ClassNoInheritLayer() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_decorator_transform.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_decorator_transform.py index bda090fc4c9ed..2b73a1075e5a3 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_decorator_transform.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_decorator_transform.py @@ -22,12 +22,11 @@ def deco1(func): - @wraps(func) def inner(*args, **kwargs): print('in deco1, added 1') _x = 2 - if (_x < 1): + if _x < 1: _x += 1 else: _x -= 1 @@ -39,7 +38,6 @@ def inner(*args, **kwargs): def deco2(fun): - @wraps(fun) def inner(*args, **kwargs): print('in deco2, added 2') @@ -51,9 +49,7 @@ def inner(*args, **kwargs): def deco3(x=3): - def inner_deco(func): - @wraps(func) def inner(*args, **kwargs): print('in deco3, added {}'.format(x)) @@ -67,9 +63,7 @@ def inner(*args, **kwargs): def deco4(func=None, x=0): - def decorated(pyfunc): - @wraps(pyfunc) def inner_deco(*args, **kwargs): print('in deco4, added {}'.format(x)) @@ -187,7 +181,6 @@ def deco_with_paddle_api(): class TestDecoratorTransform(unittest.TestCase): - def test_deco_transform(self): outs = forward() np.testing.assert_allclose(outs[0], np.array(3), rtol=1e-05) @@ -206,9 +199,11 @@ def test_contextmanager_warning(self): warn1() flag = False for warn in w: - if (issubclass(warn.category, UserWarning) - ) and "A context manager decorator is used" in str( - warn.message): + if ( + issubclass(warn.category, UserWarning) + ) and "A context manager decorator is used" in str( + warn.message + ): flag = True break self.assertTrue(flag) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_deepcopy.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_deepcopy.py index ecb2d97fa4482..a4823be643029 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_deepcopy.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_deepcopy.py @@ -15,14 +15,15 @@ import unittest import paddle import numpy as np -from paddle.fluid.dygraph.dygraph_to_static.program_translator import StaticFunction +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + StaticFunction, +) from test_rollback import Net, foo from copy import deepcopy class TestDeepCopy(unittest.TestCase): - def test_net(self): net = Net() net = paddle.jit.to_static(net) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_dict.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_dict.py index f6b71eb726542..fcb986045b876 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_dict.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_dict.py @@ -18,32 +18,41 @@ import paddle import paddle.fluid as fluid from paddle.jit import to_static -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramTranslator, +) -PLACE = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +PLACE = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) class SubNetWithDict(fluid.dygraph.Layer): - def __init__(self, hidden_size=16, output_size=16): super(SubNetWithDict, self).__init__() - init_weight = lambda x: fluid.ParamAttr(initializer=fluid.initializer. - Constant(x)) - - self.q_fc = fluid.dygraph.Linear(input_dim=hidden_size, - output_dim=output_size, - bias_attr=False, - param_attr=init_weight(0.6)) - self.k_fc = fluid.dygraph.Linear(input_dim=hidden_size, - output_dim=output_size, - bias_attr=False, - param_attr=init_weight(0.5)) - self.v_fc = fluid.dygraph.Linear(input_dim=hidden_size, - output_dim=output_size, - bias_attr=False, - param_attr=init_weight(0.2)) + init_weight = lambda x: fluid.ParamAttr( + initializer=fluid.initializer.Constant(x) + ) + + self.q_fc = fluid.dygraph.Linear( + input_dim=hidden_size, + output_dim=output_size, + bias_attr=False, + param_attr=init_weight(0.6), + ) + self.k_fc = fluid.dygraph.Linear( + input_dim=hidden_size, + output_dim=output_size, + bias_attr=False, + param_attr=init_weight(0.5), + ) + self.v_fc = fluid.dygraph.Linear( + input_dim=hidden_size, + output_dim=output_size, + bias_attr=False, + param_attr=init_weight(0.2), + ) def forward(self, input, cache=None): input = fluid.dygraph.to_variable(input) @@ -66,7 +75,6 @@ def forward(self, input, cache=None): class MainNetWithDict(fluid.dygraph.Layer): - def __init__(self, batch_size=64, hidden_size=16, output_size=16): super(MainNetWithDict, self).__init__() self.batch_size = batch_size @@ -78,16 +86,16 @@ def __init__(self, batch_size=64, hidden_size=16, output_size=16): def forward(self, input, max_len=4): input = fluid.dygraph.to_variable(input) cache = { - "k": - fluid.layers.fill_constant( + "k": fluid.layers.fill_constant( shape=[self.batch_size, self.output_size], dtype='float32', - value=0), - "v": - fluid.layers.fill_constant( + value=0, + ), + "v": fluid.layers.fill_constant( shape=[self.batch_size, self.output_size], dtype='float32', - value=0), + value=0, + ), } # TODO(Aurelius84): The following code will be converted into: # max_len = layers.cond(layers.shape(input)[0] != max_len, @@ -165,11 +173,13 @@ def test_dic_pop_2(x): class TestDictPop(unittest.TestCase): - def setUp(self): self.input = np.random.random((3)).astype('int32') - self.place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self._set_test_func() def _set_test_func(self): @@ -197,17 +207,17 @@ def test_transformed_result(self): static_res, rtol=1e-05, err_msg='dygraph result is {}\nstatic result is {}'.format( - dygraph_res, static_res)) + dygraph_res, static_res + ), + ) class TestDictPop2(TestDictPop): - def _set_test_func(self): self.dygraph_func = test_dic_pop_2 class NetWithDictPop(paddle.nn.Layer): - def __init__(self): super(NetWithDictPop, self).__init__() @@ -224,7 +234,6 @@ def forward(self, x, **kwargs): class TestDictPop3(TestNetWithDict): - def setUp(self): self.x = np.array([2, 2]).astype('float32') @@ -240,15 +249,16 @@ def test_ast_to_func(self): dygraph_result = self._run_dygraph() static_result = self._run_static() - self.assertTrue((dygraph_result == static_result).all(), - msg="dygraph result: {}\nstatic result: {}".format( - dygraph_result, static_result)) + self.assertTrue( + (dygraph_result == static_result).all(), + msg="dygraph result: {}\nstatic result: {}".format( + dygraph_result, static_result + ), + ) class TestDictCmpInFor(unittest.TestCase): - def test_with_for(self): - def func(): pos = [1, 3] neg = [-1, -3] @@ -257,15 +267,14 @@ def func(): for (x, y) in zip(pos, neg): val = x - y dict_val.update( - {k: val + dict_val[k] - for k, v in dict_val.items()}) + {k: val + dict_val[k] for k, v in dict_val.items()} + ) return dict_val self.assertEqual(paddle.jit.to_static(func)()['minus'], 8) def test_with_for_enumerate(self): - def func(): pos = [1, 3] neg = [-1, -3] @@ -274,8 +283,8 @@ def func(): for i, (x, y) in enumerate(zip(pos, neg)): val = x - y dict_val.update( - {k: val + dict_val[k] - for k, v in dict_val.items()}) + {k: val + dict_val[k] for k, v in dict_val.items()} + ) return dict_val diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_drop_path.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_drop_path.py index 201da70b01770..c5eb3d7567efe 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_drop_path.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_drop_path.py @@ -26,7 +26,6 @@ def drop_path(x, training=False): class DropPath(paddle.nn.Layer): - def __init__(self): super(DropPath, self).__init__() @@ -36,7 +35,6 @@ def forward(self, x): class TestTrainEval(unittest.TestCase): - def setUp(self): self.model = DropPath() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_duplicate_output.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_duplicate_output.py index 9534d1dc615c8..625051d9364e5 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_duplicate_output.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_duplicate_output.py @@ -26,14 +26,12 @@ class SimpleNet(paddle.nn.Layer): - def __init__(self): super().__init__() self._linear = paddle.nn.Linear(1, 1) def forward(self, x): - """ forward with duplicate outputs. - """ + """forward with duplicate outputs.""" x = self._linear(x) return x, x diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_fetch_feed.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_fetch_feed.py index 4eada80aec87b..6e7389cc132d9 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_fetch_feed.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_fetch_feed.py @@ -23,13 +23,11 @@ class Pool2D(fluid.dygraph.Layer): - def __init__(self): super(Pool2D, self).__init__() - self.pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='avg', - pool_stride=1, - global_pooling=False) + self.pool2d = fluid.dygraph.Pool2D( + pool_size=2, pool_type='avg', pool_stride=1, global_pooling=False + ) @declarative def forward(self, x): @@ -42,17 +40,19 @@ def get_result(x): class Linear(fluid.dygraph.Layer): - def __init__(self, input_dim=10, output_dim=5): super(Linear, self).__init__() self.fc = fluid.dygraph.Linear( input_dim, output_dim, act='relu', - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), + ) @declarative def forward(self, x): @@ -62,7 +62,6 @@ def forward(self, x): class TestPool2D(unittest.TestCase): - def setUp(self): self.dygraph_class = Pool2D self.data = np.random.random((1, 2, 4, 4)).astype('float32') @@ -95,11 +94,12 @@ def test_declarative(self): static_res, rtol=1e-05, err_msg='dygraph_res is {}\n static_res is \n{}'.format( - dygraph_res, static_res)) + dygraph_res, static_res + ), + ) class TestLinear(TestPool2D): - def setUp(self): self.dygraph_class = Linear self.data = np.random.random((4, 10)).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_for_enumerate.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_for_enumerate.py index 6b399b5e9be7d..ff4e3e8433046 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_for_enumerate.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_for_enumerate.py @@ -290,7 +290,6 @@ def for_tuple_as_enumerate_value(x_array): # 20. test for function in a class class ForwardContainsForLayer(paddle.nn.Layer): - def __init__(self): super(ForwardContainsForLayer, self).__init__() self.high = 5 @@ -326,8 +325,8 @@ def for_original_tuple(): # 23. for zip error @paddle.jit.to_static( - input_spec=[InputSpec(shape=[None, 10]), - InputSpec(shape=[None, 10])]) + input_spec=[InputSpec(shape=[None, 10]), InputSpec(shape=[None, 10])] +) def for_zip_error(x, y): for i, j in zip(x, y): a = i + j @@ -336,8 +335,8 @@ def for_zip_error(x, y): # 24. for zip @paddle.jit.to_static( - input_spec=[InputSpec(shape=[2, 10]), - InputSpec(shape=[2, 10])]) + input_spec=[InputSpec(shape=[2, 10]), InputSpec(shape=[2, 10])] +) def for_zip(x, y): for i, j in zip(x, y): a = i + j @@ -345,10 +344,12 @@ def for_zip(x, y): class TestTransformBase(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.set_input() self.set_test_func() @@ -357,7 +358,8 @@ def set_input(self): def set_test_func(self): raise NotImplementedError( - "For Enumerate test should implement set_test_func") + "For Enumerate test should implement set_test_func" + ) def _run(self, to_static): program_translator.enable(to_static) @@ -372,22 +374,20 @@ def get_static_output(self): class TestTransform(TestTransformBase): - def transformed_result_compare(self): dy_outs = self.get_dygraph_output() if not isinstance(dy_outs, (tuple, list)): - dy_outs = (dy_outs, ) + dy_outs = (dy_outs,) st_outs = self.get_static_output() if not isinstance(st_outs, (tuple, list)): - st_outs = (st_outs, ) + st_outs = (st_outs,) for x, y in zip(dy_outs, st_outs): np.testing.assert_allclose(x.numpy(), y.numpy(), rtol=1e-05) class TestTransformForOriginalList(TestTransform): - def _run(self, to_static): program_translator.enable(to_static) with fluid.dygraph.guard(): @@ -395,7 +395,6 @@ def _run(self, to_static): class TestTransformError(TestTransformBase): - def transformed_error(self, etype): with self.assertRaises(etype): dy_out = self.get_dygraph_output() @@ -403,7 +402,6 @@ def transformed_error(self, etype): class TestForInRange(TestTransform): - def set_input(self): self.input = np.array([5]) @@ -415,7 +413,6 @@ def test_transformed_result_compare(self): class TestForIterList(TestTransform): - def set_test_func(self): self.dygraph_func = for_iter_list @@ -424,19 +421,16 @@ def test_transformed_result_compare(self): class TestForEnumerateSimple(TestForIterList): - def set_test_func(self): self.dygraph_func = for_enumerate_list class TestForInRangeWithBreak(TestForInRange): - def set_test_func(self): self.dygraph_func = for_in_range_with_break class TestForIterVarNumpy(TestTransform): - def set_input(self): self.input = np.array([1, 2, 3, 4, 5]) @@ -448,103 +442,86 @@ def test_transformed_result_compare(self): class TestForEnumerateVarNumpy(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var_numpy class TestForEnumerateVarNumpyWithStart(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var_numpy_with_start class TestForEnumerateVarNumpyWithBreak(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var_numpy_with_break class TestForEnumerateVarNumpyWithContinue(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var_numpy_with_continue class TestForEnumerateVarNumpyWithStartAndBreak(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var_numpy_with_start_break class TestForEnumerateVarNumpyWithStartAndContinue(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var_numpy_with_start_continue class TestForIterVar(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_iter_var class TestForIterVarIdx(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_iter_var_idx class TestForEnumerateVar(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var class TestForEnumerateVarWithNestedRange(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_enumerate_var_with_nested_range class TestForIterVarList(TestForInRange): - def set_test_func(self): self.dygraph_func = for_iter_var_list class TestForEnumerateVarList(TestForInRange): - def set_test_func(self): self.dygraph_func = for_enumerate_var_list class TestForTupleAsIterVar(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_tuple_as_iter_var class TestForTupleAsEnumerateIter(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_tuple_as_enumerate_iter class TestForTupleAsEnumerateValue(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = for_tuple_as_enumerate_value class TestForwardContainsForLayer(TestForIterVarNumpy): - def set_test_func(self): self.dygraph_func = ForwardContainsForLayer() class TestForOriginalList(TestTransformForOriginalList): - def set_test_func(self): self.dygraph_func = for_original_list @@ -553,7 +530,6 @@ def test_transformed_result_compare(self): class TestForOriginalTuple(TestTransformForOriginalList): - def set_test_func(self): self.dygraph_func = for_original_tuple @@ -562,7 +538,6 @@ def test_transformed_result_compare(self): class TestForZip(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_full_name_usage.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_full_name_usage.py index 7a79eaf28db5b..a245b607f881d 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_full_name_usage.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_full_name_usage.py @@ -45,7 +45,6 @@ def decorated_call_decorated(x): class DoubleDecorated(object): - @classmethod @declarative def double_decorated_func1(self, x): @@ -58,20 +57,19 @@ def double_decorated_func2(self, x): class TestFullNameDecorator(unittest.TestCase): - def test_run_success(self): x = np.ones([1, 2]).astype("float32") answer = np.zeros([1, 2]).astype("float32") with fluid.dygraph.guard(): - np.testing.assert_allclose(dygraph_decorated_func(x).numpy(), - answer, - rtol=1e-05) - np.testing.assert_allclose(jit_decorated_func(x).numpy(), - answer, - rtol=1e-05) - np.testing.assert_allclose(decorated_call_decorated(x).numpy(), - answer, - rtol=1e-05) + np.testing.assert_allclose( + dygraph_decorated_func(x).numpy(), answer, rtol=1e-05 + ) + np.testing.assert_allclose( + jit_decorated_func(x).numpy(), answer, rtol=1e-05 + ) + np.testing.assert_allclose( + decorated_call_decorated(x).numpy(), answer, rtol=1e-05 + ) with self.assertRaises(NotImplementedError): DoubleDecorated().double_decorated_func1(x) with self.assertRaises(NotImplementedError): @@ -79,12 +77,13 @@ def test_run_success(self): class TestImportProgramTranslator(unittest.TestCase): - def test_diff_pkg_same_cls(self): dygraph_prog_trans = fluid.dygraph.ProgramTranslator() - dy_to_stat_prog_trans = fluid.dygraph.dygraph_to_static.ProgramTranslator( + dy_to_stat_prog_trans = ( + fluid.dygraph.dygraph_to_static.ProgramTranslator() ) - full_pkg_prog_trans = fluid.dygraph.dygraph_to_static.program_translator.ProgramTranslator( + full_pkg_prog_trans = ( + fluid.dygraph.dygraph_to_static.program_translator.ProgramTranslator() ) self.assertEqual(dygraph_prog_trans, dy_to_stat_prog_trans) self.assertEqual(dygraph_prog_trans, full_pkg_prog_trans) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_function_spec.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_function_spec.py index 9fdb6e7c6d36d..37bc3f5dc12c7 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_function_spec.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_function_spec.py @@ -24,7 +24,6 @@ class TestFunctionSpec(unittest.TestCase): - def test_constructor(self): foo_spec = FunctionSpec(foo_func) args_name = foo_spec.args_name @@ -51,11 +50,9 @@ def test_unified_args_and_kwargs(self): self.assertTrue(len(kwargs) == 0) # case 2: foo(a=10, b=20, d=4) - args, kwargs = foo_spec.unified_args_and_kwargs([], { - 'a': 10, - 'b': 20, - 'd': 4 - }) + args, kwargs = foo_spec.unified_args_and_kwargs( + [], {'a': 10, 'b': 20, 'd': 4} + ) self.assertTupleEqual(args, (10, 20, 1, 4)) self.assertTrue(len(kwargs) == 0) @@ -83,7 +80,8 @@ def test_args_to_input_spec(self): # case 1 foo_spec = FunctionSpec(foo_func, input_spec=[a_spec, b_spec]) input_with_spec, _ = foo_spec.args_to_input_spec( - (a_tensor, b_tensor, 1, 2), {}) + (a_tensor, b_tensor, 1, 2), {} + ) self.assertTrue(len(input_with_spec) == 4) self.assertTrue(input_with_spec[0] == a_spec) # a @@ -93,8 +91,9 @@ def test_args_to_input_spec(self): # case 2 foo_spec = FunctionSpec(foo_func, input_spec=[a_spec]) - input_with_spec, _ = foo_spec.args_to_input_spec((a_tensor, b_tensor), - {}) + input_with_spec, _ = foo_spec.args_to_input_spec( + (a_tensor, b_tensor), {} + ) self.assertTrue(len(input_with_spec) == 2) self.assertTrue(input_with_spec[0] == a_spec) # a self.assertTupleEqual(input_with_spec[1].shape, (4, 10)) # b.shape @@ -104,14 +103,15 @@ def test_args_to_input_spec(self): # assert kwargs is None if set `input_spec` foo_spec = FunctionSpec(foo_func, input_spec=[a_spec]) with self.assertRaises(ValueError): - input_with_spec = foo_spec.args_to_input_spec((a_tensor, b_tensor), - {'c': 4}) + input_with_spec = foo_spec.args_to_input_spec( + (a_tensor, b_tensor), {'c': 4} + ) # case 4 # assert len(args) >= len(self._input_spec) foo_spec = FunctionSpec(foo_func, input_spec=[a_spec, b_spec]) with self.assertRaises(ValueError): - input_with_spec = foo_spec.args_to_input_spec((a_tensor, ), {}) + input_with_spec = foo_spec.args_to_input_spec((a_tensor,), {}) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grad.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grad.py index ae484bd72c68c..38d93e3fba357 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grad.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grad.py @@ -20,7 +20,6 @@ class GradLayer(paddle.nn.Layer): - def __init__(self): super(GradLayer, self).__init__() @@ -33,7 +32,6 @@ def forward(self, x): class GradLinearLayer(paddle.nn.Layer): - def __init__(self): super(GradLinearLayer, self).__init__() self.linear = paddle.nn.Linear(5, 5, bias_attr=False) @@ -45,15 +43,13 @@ def forward(self, x): for i in range(10): tmp = self.linear(tmp) out = tmp - dx = paddle.grad([out], [x], - None, - create_graph=True, - allow_unused=False)[0] + dx = paddle.grad( + [out], [x], None, create_graph=True, allow_unused=False + )[0] return dx class NoGradLinearLayer(paddle.nn.Layer): - def __init__(self): super(NoGradLinearLayer, self).__init__() self.linear = paddle.nn.Linear(5, 5, bias_attr=False) @@ -70,7 +66,6 @@ def forward(self, x): class TestGrad(unittest.TestCase): - def setUp(self): self.func = GradLayer() self.x = paddle.ones(shape=[10, 2, 5], dtype='float32') @@ -90,17 +85,18 @@ def test_forward(self): class TestGradLinear(TestGrad): - def setUp(self): self.func = GradLinearLayer() self.x = paddle.ones(shape=[10, 2, 5], dtype='float32') self.x.stop_gradient = False self.temp_dir = tempfile.TemporaryDirectory() - self.infer_model_path = os.path.join(self.temp_dir.name, - 'double_grad_infer_model') - self.train_model_path = os.path.join(self.temp_dir.name, - 'double_grad_train_model') + self.infer_model_path = os.path.join( + self.temp_dir.name, 'double_grad_infer_model' + ) + self.train_model_path = os.path.join( + self.temp_dir.name, 'double_grad_train_model' + ) def tearDown(self): self.temp_dir.cleanup() @@ -118,9 +114,11 @@ def test_save_infer_program(self): def test_save_train_program(self): grad_clip = paddle.nn.ClipGradByGlobalNorm(2.0) - optimizer = paddle.optimizer.SGD(learning_rate=0.01, - grad_clip=grad_clip, - parameters=self.func.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.01, + grad_clip=grad_clip, + parameters=self.func.parameters(), + ) for i in range(10): out = self.func(self.x) avg_loss = paddle.mean(paddle.abs(out - 1)) @@ -138,17 +136,18 @@ def test_save_train_program(self): class TestNoGradLinear(TestGradLinear): - def setUp(self): self.func = NoGradLinearLayer() self.x = paddle.ones(shape=[10, 2, 5], dtype='float32') self.x.stop_gradient = False self.temp_dir = tempfile.TemporaryDirectory() - self.infer_model_path = os.path.join(self.temp_dir.name, - 'no_grad_infer_model') - self.train_model_path = os.path.join(self.temp_dir.name, - 'no_grad_train_model') + self.infer_model_path = os.path.join( + self.temp_dir.name, 'no_grad_infer_model' + ) + self.train_model_path = os.path.join( + self.temp_dir.name, 'no_grad_train_model' + ) def tearDown(self): self.temp_dir.cleanup() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_gradient_aggregation.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_gradient_aggregation.py index 67232aa9a83bd..b30338ce0e406 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_gradient_aggregation.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_gradient_aggregation.py @@ -22,7 +22,6 @@ class SimpleNet(paddle.nn.Layer): - def __init__(self): super(SimpleNet, self).__init__() self.linear1 = paddle.nn.Linear(10, 3) @@ -32,25 +31,30 @@ def forward(self, x): out1 = self.linear1(x) out2 = self.linear2(out1) return [out1, out2] # 梯度为0 - #return [out1] # 梯度正常 - #return [out2, out1] # 梯度正常 + # return [out1] # 梯度正常 + # return [out2, out1] # 梯度正常 class TestGradientAggregationInDy2Static(unittest.TestCase): - def test_to_static(self): - def simplenet_grad(inp, to_static=False): net = SimpleNet() - if to_static: net = paddle.jit.to_static(net) + if to_static: + net = paddle.jit.to_static(net) loss = net(inp) loss[0].backward() return net.linear1.weight.grad - inp = paddle.to_tensor(np.random.randn(10, )).astype("float32") - np.testing.assert_allclose(simplenet_grad(inp, True).numpy(), - simplenet_grad(inp, False).numpy(), - rtol=1e-05) + inp = paddle.to_tensor( + np.random.randn( + 10, + ) + ).astype("float32") + np.testing.assert_allclose( + simplenet_grad(inp, True).numpy(), + simplenet_grad(inp, False).numpy(), + rtol=1e-05, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grid_generator.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grid_generator.py index caf349ecf8ebe..3f80130737f42 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grid_generator.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_grid_generator.py @@ -23,7 +23,6 @@ class GridGenerator(nn.Layer): - def __init__(self, in_channels, num_fiducial): super(GridGenerator, self).__init__() self.eps = 1e-6 @@ -32,15 +31,16 @@ def __init__(self, in_channels, num_fiducial): initializer = nn.initializer.Constant(value=0.0) param_attr = ParamAttr(learning_rate=0.0, initializer=initializer) bias_attr = ParamAttr(learning_rate=0.0, initializer=initializer) - self.fc = nn.Linear(in_channels, - 6, - weight_attr=param_attr, - bias_attr=bias_attr) - - @paddle.jit.to_static(input_spec=[ - paddle.static.InputSpec(shape=[None, 3, 32, 100], dtype='float32'), - paddle.static.InputSpec(shape=[32, 100], dtype='float32') - ]) + self.fc = nn.Linear( + in_channels, 6, weight_attr=param_attr, bias_attr=bias_attr + ) + + @paddle.jit.to_static( + input_spec=[ + paddle.static.InputSpec(shape=[None, 3, 32, 100], dtype='float32'), + paddle.static.InputSpec(shape=[32, 100], dtype='float32'), + ] + ) def forward(self, batch_C_prime, I_r_size): """ Generate the grid for the grid_sampler. @@ -54,7 +54,7 @@ def forward(self, batch_C_prime, I_r_size): return C def build_C_paddle(self): - """ Return coordinates of fiducial points in I_r; C """ + """Return coordinates of fiducial points in I_r; C""" F = self.F ctrl_pts_x = paddle.linspace(-1.0, 1.0, int(F / 2)) ctrl_pts_y_top = -1 * paddle.ones([int(F / 2)]) @@ -68,16 +68,18 @@ def build_P_paddle(self, I_r_size): I_r_width, I_r_height = I_r_size I_r_grid_x = paddle.divide( (paddle.arange(-I_r_width, I_r_width, 2).astype('float32') + 1.0), - paddle.to_tensor(I_r_width).astype('float32')) + paddle.to_tensor(I_r_width).astype('float32'), + ) I_r_grid_y = paddle.divide( (paddle.arange(-I_r_height, I_r_height, 2).astype('float32') + 1.0), - paddle.to_tensor(I_r_height).astype('float32')) + paddle.to_tensor(I_r_height).astype('float32'), + ) P = paddle.stack(paddle.meshgrid(I_r_grid_x, I_r_grid_y), axis=2) P = paddle.transpose(P, perm=[1, 0, 2]) return P.reshape([-1, 2]) def build_inv_delta_C_paddle(self, C): - """ Return inv_delta_C which is needed to calculate T """ + """Return inv_delta_C which is needed to calculate T""" F = self.F hat_C = paddle.zeros((F, F), dtype='float32') for i in range(0, F): @@ -89,16 +91,19 @@ def build_inv_delta_C_paddle(self, C): hat_C[i, j] = r hat_C[j, i] = r hat_C = (hat_C**2) * paddle.log(hat_C) - delta_C = paddle.concat([ - paddle.concat([paddle.ones((F, 1)), C, hat_C], axis=1), - paddle.concat( - [paddle.zeros((2, 3)), - paddle.transpose(C, perm=[1, 0])], - axis=1), - paddle.concat([paddle.zeros( - (1, 3)), paddle.ones((1, F))], axis=1) - ], - axis=0) + delta_C = paddle.concat( + [ + paddle.concat([paddle.ones((F, 1)), C, hat_C], axis=1), + paddle.concat( + [paddle.zeros((2, 3)), paddle.transpose(C, perm=[1, 0])], + axis=1, + ), + paddle.concat( + [paddle.zeros((1, 3)), paddle.ones((1, F))], axis=1 + ), + ], + axis=0, + ) inv_delta_C = paddle.inverse(delta_C) return inv_delta_C @@ -111,8 +116,9 @@ def build_P_hat_paddle(self, C, P): P_diff = P_tile - C_tile rbf_norm = paddle.norm(P_diff, p=2, axis=2, keepdim=False) - rbf = paddle.multiply(paddle.square(rbf_norm), - paddle.log(rbf_norm + eps)) + rbf = paddle.multiply( + paddle.square(rbf_norm), paddle.log(rbf_norm + eps) + ) P_hat = paddle.concat([paddle.ones((n, 1)), P, rbf], axis=1) return P_hat @@ -125,7 +131,6 @@ def get_expand_tensor(self, batch_C_prime): class TestGridGenerator(unittest.TestCase): - def setUp(self): self.x = paddle.uniform(shape=[1, 20, 2], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py index d40c37f3fd666..da69243cbe91a 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ifelse.py @@ -17,11 +17,37 @@ import paddle from paddle.fluid.dygraph.jit import declarative -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramTranslator, +) from paddle.fluid.dygraph.dygraph_to_static.utils import Dygraph2StaticException import paddle.fluid.core as core -from ifelse_simple_func import NetWithControlFlowIf, add_fn, dyfunc_empty_nonlocal, dyfunc_ifelse_ret_int1, dyfunc_ifelse_ret_int2, dyfunc_ifelse_ret_int3, dyfunc_ifelse_ret_int4, dyfunc_with_if_else, dyfunc_with_if_else2, dyfunc_with_if_else3, dyfunc_with_if_else_with_list_geneator, fluid, if_tensor_case, if_with_and_or, if_with_and_or_1, if_with_and_or_2, if_with_and_or_3, if_with_and_or_4, if_with_class_var, loss_fn, nested_if_else, nested_if_else_2, nested_if_else_3 +from ifelse_simple_func import ( + NetWithControlFlowIf, + add_fn, + dyfunc_empty_nonlocal, + dyfunc_ifelse_ret_int1, + dyfunc_ifelse_ret_int2, + dyfunc_ifelse_ret_int3, + dyfunc_ifelse_ret_int4, + dyfunc_with_if_else, + dyfunc_with_if_else2, + dyfunc_with_if_else3, + dyfunc_with_if_else_with_list_geneator, + fluid, + if_tensor_case, + if_with_and_or, + if_with_and_or_1, + if_with_and_or_2, + if_with_and_or_3, + if_with_and_or_4, + if_with_class_var, + loss_fn, + nested_if_else, + nested_if_else_2, + nested_if_else_3, +) np.random.seed(1) @@ -32,7 +58,6 @@ class TestDy2staticException(unittest.TestCase): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = None @@ -75,49 +100,42 @@ def test_ast_to_func(self): class TestDygraphIfElse2(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = dyfunc_with_if_else2 class TestDygraphIfElse3(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = dyfunc_with_if_else3 class TestDygraphIfElse4(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = dyfunc_empty_nonlocal class TestDygraphIfElseWithListGenerator(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = dyfunc_with_if_else_with_list_geneator class TestDygraphNestedIfElse(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = nested_if_else class TestDygraphNestedIfElse2(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = nested_if_else_2 class TestDygraphNestedIfElse3(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = nested_if_else_3 @@ -150,7 +168,6 @@ def body(i, ten, y): class TestDygraphIfElse6(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = dyfunc_ifExp_with_while @@ -175,56 +192,48 @@ def map_func(func, tensor_list): class TestDygraphIfElse7(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = dyfunc_ifExp class TestDygraphIfElseWithAndOr(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_with_and_or class TestDygraphIfElseWithAndOr1(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_with_and_or_1 class TestDygraphIfElseWithAndOr2(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_with_and_or_2 class TestDygraphIfElseWithAndOr3(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_with_and_or_3 class TestDygraphIfElseWithAndOr4(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_with_and_or_4 class TestDygraphIfElseWithClassVar(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_with_class_var class TestDygraphIfTensor(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_tensor_case @@ -279,14 +288,12 @@ def call_external_func(x, label=None): class TestAst2FuncWithExternalFunc(TestDygraphIfElse): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = call_external_func class NetWithExternalFunc(fluid.dygraph.Layer): - @declarative def forward(self, x, label=None): if paddle.mean(x) < 0: @@ -307,14 +314,12 @@ def softmax(x): class TestNetWithExternalFunc(TestDygraphIfElseNet): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.Net = NetWithExternalFunc class DiffModeNet1(paddle.nn.Layer): - def __init__(self, mode): super(DiffModeNet1, self).__init__() self.mode = mode @@ -331,7 +336,6 @@ def forward(self, x, y): class DiffModeNet2(paddle.nn.Layer): - def __init__(self, mode): super(DiffModeNet2, self).__init__() self.mode = mode @@ -371,27 +375,28 @@ def _run(self, mode, to_static): def test_train_mode(self): self.assertTrue( - (self._run(mode='train', - to_static=True) == self._run(mode='train', - to_static=False)).all()) + ( + self._run(mode='train', to_static=True) + == self._run(mode='train', to_static=False) + ).all() + ) def test_infer_mode(self): self.assertTrue( - (self._run(mode='infer', - to_static=True) == self._run(mode='infer', - to_static=False)).all()) + ( + self._run(mode='infer', to_static=True) + == self._run(mode='infer', to_static=False) + ).all() + ) class TestDiffModeNet2(TestDiffModeNet): - def init_net(self): self.Net = DiffModeNet2 class TestNewVarCreateInOneBranch(unittest.TestCase): - def test_var_used_in_another_for(self): - def case_func(training): # targets and targets_list is dynamically defined by training if training: @@ -414,7 +419,6 @@ def case_func(training): class TestDy2StIfElseRetInt1(unittest.TestCase): - def setUp(self): self.x = np.random.random([5]).astype('float32') self.dyfunc = dyfunc_ifelse_ret_int1 @@ -433,7 +437,6 @@ def test_ast_to_func(self): class TestDy2StIfElseRetInt2(TestDy2staticException): - def setUp(self): self.x = np.random.random([5]).astype('float32') self.error = "Your if/else have different number of return value." @@ -441,7 +444,6 @@ def setUp(self): class TestDy2StIfElseRetInt3(TestDy2StIfElseRetInt1): - def setUp(self): self.x = np.random.random([5]).astype('float32') self.dyfunc = dyfunc_ifelse_ret_int3 @@ -452,7 +454,6 @@ def test_ast_to_func(self): class TestDy2StIfElseRetInt4(TestDy2StIfElseRetInt1): - def setUp(self): self.x = np.random.random([5]).astype('float32') self.dyfunc = dyfunc_ifelse_ret_int4 @@ -472,12 +473,11 @@ def test_ast_to_func(self): class IfElseNet(paddle.nn.Layer): - def __init__(self): super(IfElseNet, self).__init__() - self.param = self.create_parameter(shape=[3, 2], - dtype='float32', - is_bias=False) + self.param = self.create_parameter( + shape=[3, 2], dtype='float32', is_bias=False + ) @paddle.jit.to_static def forward(self, a, b, c): @@ -493,7 +493,6 @@ def forward(self, a, b, c): class TestDy2StIfElseBackward(unittest.TestCase): - def test_run_backward(self): a = paddle.randn((4, 3), dtype='float32') a.stop_gradient = False @@ -506,9 +505,9 @@ def test_run_backward(self): net.train() out = net(a, b, c) out.backward() - np.testing.assert_allclose((b + net.param).numpy(), - out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + (b + net.param).numpy(), out.numpy(), rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_isinstance.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_isinstance.py index dc3b4dd74a60a..3b46602cac2ab 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_isinstance.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_isinstance.py @@ -31,13 +31,11 @@ class SimpleReturnLayer(nn.Layer): - def forward(self, x): return x class AddAttrLayer(nn.Layer): - def __init__(self): super(AddAttrLayer, self).__init__() self.attr = None @@ -48,21 +46,19 @@ def forward(self, x): class IsInstanceLayer(nn.Layer): - def __init__(self, layer): super(IsInstanceLayer, self).__init__() self.layer = layer @paddle.jit.to_static def forward(self, x): - if isinstance(self.layer, (AddAttrLayer, )): + if isinstance(self.layer, (AddAttrLayer,)): self.layer.attr = x res = self.layer(x) return res class SequentialLayer(nn.Layer): - def __init__(self, layers): super(SequentialLayer, self).__init__() self.layers = nn.LayerList(layers) @@ -88,7 +84,6 @@ def train(model, to_static): class TestIsinstance(unittest.TestCase): - def test_isinstance_simple_return_layer(self): model = IsInstanceLayer(SimpleReturnLayer()) self._test_model(model) @@ -108,11 +103,12 @@ def test_sequential_layer(self): def _test_model(self, model): st_out = train(model, to_static=True) dy_out = train(model, to_static=False) - np.testing.assert_allclose(dy_out, - st_out, - rtol=1e-05, - err_msg='dy_out:\n {}\n st_out:\n{}'.format( - dy_out, st_out)) + np.testing.assert_allclose( + dy_out, + st_out, + rtol=1e-05, + err_msg='dy_out:\n {}\n st_out:\n{}'.format(dy_out, st_out), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_jit_property_save.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_jit_property_save.py index 481d30b8ba200..0d26ab51e1a51 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_jit_property_save.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_jit_property_save.py @@ -17,8 +17,7 @@ class TestPropertySave(unittest.TestCase): - """test jit property save - """ + """test jit property save""" def setUp(self): a = paddle.framework.core.Property() @@ -44,8 +43,7 @@ def test_load_float(self): self.a.get_float(1) def test_set(self): - """test propety set. - """ + """test propety set.""" try: a = paddle.framework.core.Property() a.set_float('float', 10.0) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lac.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lac.py index 7ffbfbb6533fd..15d18fe838a05 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lac.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lac.py @@ -38,30 +38,33 @@ input_specs = [ paddle.static.InputSpec([None, None], 'int64'), paddle.static.InputSpec([None, None], 'int64'), - paddle.static.InputSpec([None], 'int64') + paddle.static.InputSpec([None], 'int64'), ] class DynamicGRU(fluid.dygraph.Layer): - - def __init__(self, - size, - h_0=None, - param_attr=None, - bias_attr=None, - is_reverse=False, - gate_activation='sigmoid', - candidate_activation='tanh', - origin_mode=False, - init_size=None): + def __init__( + self, + size, + h_0=None, + param_attr=None, + bias_attr=None, + is_reverse=False, + gate_activation='sigmoid', + candidate_activation='tanh', + origin_mode=False, + init_size=None, + ): super(DynamicGRU, self).__init__() - self.gru_unit = GRUUnit(size * 3, - param_attr=param_attr, - bias_attr=bias_attr, - activation=candidate_activation, - gate_activation=gate_activation, - origin_mode=origin_mode) + self.gru_unit = GRUUnit( + size * 3, + param_attr=param_attr, + bias_attr=bias_attr, + activation=candidate_activation, + gate_activation=gate_activation, + origin_mode=origin_mode, + ) self.size = size self.h_0 = h_0 @@ -81,15 +84,16 @@ def forward(self, inputs): j = i # input_ = inputs[:, j:j+1, :] # original code - input_ = fluid.layers.slice(inputs, - axes=[1], - starts=[j], - ends=[j + 1]) - input_ = fluid.layers.reshape(input_, [-1, input_.shape[2]], - inplace=False) + input_ = fluid.layers.slice( + inputs, axes=[1], starts=[j], ends=[j + 1] + ) + input_ = fluid.layers.reshape( + input_, [-1, input_.shape[2]], inplace=False + ) hidden, reset, gate = self.gru_unit(input_, hidden) - hidden_ = fluid.layers.reshape(hidden, [-1, 1, hidden.shape[1]], - inplace=False) + hidden_ = fluid.layers.reshape( + hidden, [-1, 1, hidden.shape[1]], inplace=False + ) res.append(hidden_) if self.is_reverse: @@ -99,7 +103,6 @@ def forward(self, inputs): class BiGRU(fluid.dygraph.Layer): - def __init__(self, input_dim, grnn_hidden_dim, init_bound, h_0=None): super(BiGRU, self).__init__() @@ -107,38 +110,54 @@ def __init__(self, input_dim, grnn_hidden_dim, init_bound, h_0=None): input_dim=input_dim, output_dim=grnn_hidden_dim * 3, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(low=-init_bound, - high=init_bound), + initializer=fluid.initializer.Uniform( + low=-init_bound, high=init_bound + ), regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4))) + regularization_coeff=1e-4 + ), + ), + ) self.gru = DynamicGRU( size=grnn_hidden_dim, h_0=h_0, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(low=-init_bound, - high=init_bound), + initializer=fluid.initializer.Uniform( + low=-init_bound, high=init_bound + ), regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4))) + regularization_coeff=1e-4 + ), + ), + ) self.pre_gru_r = Linear( input_dim=input_dim, output_dim=grnn_hidden_dim * 3, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(low=-init_bound, - high=init_bound), + initializer=fluid.initializer.Uniform( + low=-init_bound, high=init_bound + ), regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4))) + regularization_coeff=1e-4 + ), + ), + ) self.gru_r = DynamicGRU( size=grnn_hidden_dim, is_reverse=True, h_0=h_0, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(low=-init_bound, - high=init_bound), + initializer=fluid.initializer.Uniform( + low=-init_bound, high=init_bound + ), regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4))) + regularization_coeff=1e-4 + ), + ), + ) def forward(self, input_feature): res_pre_gru = self.pre_gru(input_feature) @@ -152,7 +171,6 @@ def forward(self, input_feature): class LinearChainCRF(fluid.dygraph.Layer): - def __init__(self, param_attr, size=None, is_test=False, dtype='float32'): super(LinearChainCRF, self).__init__() @@ -163,7 +181,8 @@ def __init__(self, param_attr, size=None, is_test=False, dtype='float32'): self._transition = self.create_parameter( attr=self._param_attr, shape=[self._size + 2, self._size], - dtype=self._dtype) + dtype=self._dtype, + ) @property def weight(self): @@ -176,41 +195,46 @@ def weight(self, value): def forward(self, input, label, length=None): if _non_static_mode(): _, _, _, log_likelihood = _legacy_C_ops.linear_chain_crf( - input, self._transition, label, length, "is_test", - self._is_test) + input, self._transition, label, length, "is_test", self._is_test + ) return log_likelihood alpha = self._helper.create_variable_for_type_inference( - dtype=self._dtype) + dtype=self._dtype + ) emission_exps = self._helper.create_variable_for_type_inference( - dtype=self._dtype) + dtype=self._dtype + ) transition_exps = self._helper.create_variable_for_type_inference( - dtype=self._dtype) + dtype=self._dtype + ) log_likelihood = self._helper.create_variable_for_type_inference( - dtype=self._dtype) + dtype=self._dtype + ) this_inputs = { "Emission": [input], "Transition": self._transition, - "Label": [label] + "Label": [label], } if length is not None: this_inputs['Length'] = [length] - self._helper.append_op(type='linear_chain_crf', - inputs=this_inputs, - outputs={ - "Alpha": [alpha], - "EmissionExps": [emission_exps], - "TransitionExps": transition_exps, - "LogLikelihood": log_likelihood - }, - attrs={ - "is_test": self._is_test, - }) + self._helper.append_op( + type='linear_chain_crf', + inputs=this_inputs, + outputs={ + "Alpha": [alpha], + "EmissionExps": [emission_exps], + "TransitionExps": transition_exps, + "LogLikelihood": log_likelihood, + }, + attrs={ + "is_test": self._is_test, + }, + ) return log_likelihood class CRFDecoding(fluid.dygraph.Layer): - def __init__(self, param_attr, size=None, is_test=False, dtype='float32'): super(CRFDecoding, self).__init__() @@ -221,7 +245,8 @@ def __init__(self, param_attr, size=None, is_test=False, dtype='float32'): self._transition = self.create_parameter( attr=self._param_attr, shape=[self._size + 2, self._size], - dtype=self._dtype) + dtype=self._dtype, + ) @property def weight(self): @@ -233,33 +258,35 @@ def weight(self, value): def forward(self, input, label=None, length=None): if _non_static_mode(): - return _legacy_C_ops.crf_decoding(input, self._transition, label, - length, "is_test", self._is_test) + return _legacy_C_ops.crf_decoding( + input, self._transition, label, length, "is_test", self._is_test + ) viterbi_path = self._helper.create_variable_for_type_inference( - dtype=self._dtype) + dtype=self._dtype + ) this_inputs = { "Emission": [input], "Transition": self._transition, - "Label": label + "Label": label, } if length is not None: this_inputs['Length'] = [length] - self._helper.append_op(type='crf_decoding', - inputs=this_inputs, - outputs={"ViterbiPath": [viterbi_path]}, - attrs={ - "is_test": self._is_test, - }) + self._helper.append_op( + type='crf_decoding', + inputs=this_inputs, + outputs={"ViterbiPath": [viterbi_path]}, + attrs={ + "is_test": self._is_test, + }, + ) return viterbi_path class ChunkEval(fluid.dygraph.Layer): - - def __init__(self, - num_chunk_types, - chunk_scheme, - excluded_chunk_types=None): + def __init__( + self, num_chunk_types, chunk_scheme, excluded_chunk_types=None + ): super(ChunkEval, self).__init__() self.num_chunk_types = num_chunk_types self.chunk_scheme = chunk_scheme @@ -267,54 +294,69 @@ def __init__(self, def forward(self, input, label, seq_length=None): if _non_static_mode(): - return _legacy_C_ops.chunk_eval(input, label, seq_length, - "num_chunk_types", - self.num_chunk_types, - "chunk_scheme", self.chunk_scheme, - "excluded_chunk_types", - self.excluded_chunk_types or []) + return _legacy_C_ops.chunk_eval( + input, + label, + seq_length, + "num_chunk_types", + self.num_chunk_types, + "chunk_scheme", + self.chunk_scheme, + "excluded_chunk_types", + self.excluded_chunk_types or [], + ) precision = self._helper.create_variable_for_type_inference( - dtype="float32") + dtype="float32" + ) recall = self._helper.create_variable_for_type_inference( - dtype="float32") + dtype="float32" + ) f1_score = self._helper.create_variable_for_type_inference( - dtype="float32") + dtype="float32" + ) num_infer_chunks = self._helper.create_variable_for_type_inference( - dtype="int64") + dtype="int64" + ) num_label_chunks = self._helper.create_variable_for_type_inference( - dtype="int64") + dtype="int64" + ) num_correct_chunks = self._helper.create_variable_for_type_inference( - dtype="int64") + dtype="int64" + ) this_input = {"Inference": [input], "Label": [label]} if seq_length is not None: this_input["SeqLength"] = [seq_length] - self._helper.append_op(type='chunk_eval', - inputs=this_input, - outputs={ - "Precision": [precision], - "Recall": [recall], - "F1-Score": [f1_score], - "NumInferChunks": [num_infer_chunks], - "NumLabelChunks": [num_label_chunks], - "NumCorrectChunks": [num_correct_chunks] - }, - attrs={ - "num_chunk_types": - self.num_chunk_types, - "chunk_scheme": - self.chunk_scheme, - "excluded_chunk_types": - self.excluded_chunk_types or [] - }) - return (precision, recall, f1_score, num_infer_chunks, num_label_chunks, - num_correct_chunks) + self._helper.append_op( + type='chunk_eval', + inputs=this_input, + outputs={ + "Precision": [precision], + "Recall": [recall], + "F1-Score": [f1_score], + "NumInferChunks": [num_infer_chunks], + "NumLabelChunks": [num_label_chunks], + "NumCorrectChunks": [num_correct_chunks], + }, + attrs={ + "num_chunk_types": self.num_chunk_types, + "chunk_scheme": self.chunk_scheme, + "excluded_chunk_types": self.excluded_chunk_types or [], + }, + ) + return ( + precision, + recall, + f1_score, + num_infer_chunks, + num_label_chunks, + num_correct_chunks, + ) class LexNet(fluid.dygraph.Layer): - def __init__(self, args, length=None): super(LexNet, self).__init__() """ @@ -330,21 +372,26 @@ def __init__(self, args, length=None): self.vocab_size = args.vocab_size self.num_labels = args.num_labels self.grnn_hidden_dim = args.grnn_hidden_dim - self.emb_lr = args.emb_learning_rate if 'emb_learning_rate' in dir( - args) else 1.0 - self.crf_lr = args.emb_learning_rate if 'crf_learning_rate' in dir( - args) else 1.0 + self.emb_lr = ( + args.emb_learning_rate if 'emb_learning_rate' in dir(args) else 1.0 + ) + self.crf_lr = ( + args.emb_learning_rate if 'crf_learning_rate' in dir(args) else 1.0 + ) self.bigru_num = args.bigru_num self.init_bound = 0.1 self.word_embedding = Embedding( size=[self.vocab_size, self.word_emb_dim], dtype='float32', - param_attr=fluid.ParamAttr(learning_rate=self.emb_lr, - name="word_emb", - initializer=fluid.initializer.Uniform( - low=-self.init_bound, - high=self.init_bound))) + param_attr=fluid.ParamAttr( + learning_rate=self.emb_lr, + name="word_emb", + initializer=fluid.initializer.Uniform( + low=-self.init_bound, high=self.init_bound + ), + ), + ) h_0 = np.zeros((args.batch_size, self.grnn_hidden_dim), dtype="float32") h_0 = to_variable(h_0) @@ -355,34 +402,51 @@ def __init__(self, args, length=None): self.bigru_units.append( self.add_sublayer( "bigru_units%d" % i, - BiGRU(self.grnn_hidden_dim, - self.grnn_hidden_dim, - self.init_bound, - h_0=h_0))) + BiGRU( + self.grnn_hidden_dim, + self.grnn_hidden_dim, + self.init_bound, + h_0=h_0, + ), + ) + ) else: self.bigru_units.append( self.add_sublayer( "bigru_units%d" % i, - BiGRU(self.grnn_hidden_dim * 2, - self.grnn_hidden_dim, - self.init_bound, - h_0=h_0))) - - self.fc = Linear(input_dim=self.grnn_hidden_dim * 2, - output_dim=self.num_labels, - param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform( - low=-self.init_bound, high=self.init_bound), - regularizer=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=1e-4))) - - self.linear_chain_crf = LinearChainCRF(param_attr=fluid.ParamAttr( - name='linear_chain_crfw', learning_rate=self.crf_lr), - size=self.num_labels) - - self.crf_decoding = CRFDecoding(param_attr=fluid.ParamAttr( - name='crfw', learning_rate=self.crf_lr), - size=self.num_labels) + BiGRU( + self.grnn_hidden_dim * 2, + self.grnn_hidden_dim, + self.init_bound, + h_0=h_0, + ), + ) + ) + + self.fc = Linear( + input_dim=self.grnn_hidden_dim * 2, + output_dim=self.num_labels, + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Uniform( + low=-self.init_bound, high=self.init_bound + ), + regularizer=fluid.regularizer.L2DecayRegularizer( + regularization_coeff=1e-4 + ), + ), + ) + + self.linear_chain_crf = LinearChainCRF( + param_attr=fluid.ParamAttr( + name='linear_chain_crfw', learning_rate=self.crf_lr + ), + size=self.num_labels, + ) + + self.crf_decoding = CRFDecoding( + param_attr=fluid.ParamAttr(name='crfw', learning_rate=self.crf_lr), + size=self.num_labels, + ) # share weight self.crf_decoding.weight = self.linear_chain_crf.weight @@ -400,9 +464,9 @@ def forward(self, word, target, length=None): emission = self.fc(bigru_output) - crf_cost = self.linear_chain_crf(input=emission, - label=target, - length=length) + crf_cost = self.linear_chain_crf( + input=emission, label=target, length=length + ) avg_cost = paddle.mean(x=crf_cost) crf_decode = self.crf_decoding(input=emission, length=length) return avg_cost, crf_decode @@ -429,10 +493,16 @@ def __reader__(): batch, init_lens = [], [] for i in range(iter_num * batch_size): cur_len = local_random.randint(3, max_seq_len) - word_ids = local_random.randint(0, vocab_size, - [cur_len]).astype('int64').tolist() - label_ids = local_random.randint( - 0, num_labels, [cur_len]).astype('int64').tolist() + word_ids = ( + local_random.randint(0, vocab_size, [cur_len]) + .astype('int64') + .tolist() + ) + label_ids = ( + local_random.randint(0, num_labels, [cur_len]) + .astype('int64') + .tolist() + ) batch.append((word_ids, label_ids)) init_lens.append(cur_len) if len(batch) == batch_size: @@ -457,9 +527,9 @@ def __reader__(): def create_dataloader(reader, place): - data_loader = fluid.io.DataLoader.from_generator(capacity=16, - use_double_buffer=True, - iterable=True) + data_loader = fluid.io.DataLoader.from_generator( + capacity=16, use_double_buffer=True, iterable=True + ) data_loader.set_sample_list_generator(reader, places=place) @@ -467,11 +537,13 @@ def create_dataloader(reader, place): class TestLACModel(unittest.TestCase): - def setUp(self): self.args = Args() - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.temp_dir = tempfile.TemporaryDirectory() self.model_save_dir = os.path.join(self.temp_dir.name, 'inference') self.model_save_prefix = os.path.join(self.model_save_dir, 'lac') @@ -481,22 +553,28 @@ def setUp(self): def train(self, args, to_static): program_translator.enable(to_static) - place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.dygraph.guard(place): paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) - reader = get_random_input_data(args.batch_size, args.vocab_size, - args.num_labels) + reader = get_random_input_data( + args.batch_size, args.vocab_size, args.num_labels + ) train_loader = create_dataloader(reader, place) model = LexNet(args) optimizer = fluid.optimizer.AdamOptimizer( learning_rate=args.base_learning_rate, - parameter_list=model.parameters()) - chunk_eval = ChunkEval(int(math.ceil((args.num_labels - 1) / 2.0)), - "IOB") + parameter_list=model.parameters(), + ) + chunk_eval = ChunkEval( + int(math.ceil((args.num_labels - 1) / 2.0)), "IOB" + ) step = 0 chunk_evaluator = fluid.metrics.ChunkEvaluator() @@ -517,11 +595,16 @@ def train(self, args, to_static): end_time = time.time() if step % args.print_steps == 0: - (precision, recall, f1_score, num_infer_chunks, - num_label_chunks, - num_correct_chunks) = chunk_eval(input=crf_decode, - label=targets, - seq_length=length) + ( + precision, + recall, + f1_score, + num_infer_chunks, + num_label_chunks, + num_correct_chunks, + ) = chunk_eval( + input=crf_decode, label=targets, seq_length=length + ) outputs = [avg_cost, precision, recall, f1_score] avg_cost, precision, recall, f1_score = [ np.mean(x.numpy()) for x in outputs @@ -529,8 +612,15 @@ def train(self, args, to_static): print( "[train] step = %d, loss = %f, P: %f, R: %f, F1: %f, elapsed time %f" - % (step, avg_cost, precision, recall, f1_score, - end_time - start_time)) + % ( + step, + avg_cost, + precision, + recall, + f1_score, + end_time - start_time, + ) + ) step += 1 # save inference model @@ -539,10 +629,12 @@ def train(self, args, to_static): layer=model, path=self.model_save_prefix, input_spec=[input_specs[0], input_specs[-1]], - output_spec=[crf_decode]) + output_spec=[crf_decode], + ) else: - fluid.dygraph.save_dygraph(model.state_dict(), - self.dy_param_path) + fluid.dygraph.save_dygraph( + model.state_dict(), self.dy_param_path + ) return np.array(loss_data) @@ -554,14 +646,16 @@ def test_train(self): st_out, rtol=1e-05, err_msg='dygraph output:\n{},\nstatic output:\n {}.'.format( - dy_out, st_out)) + dy_out, st_out + ), + ) # Prediction needs trained models, so put `test_predict` at last of `test_train` # self.verify_predict() def verify_predict(self): - reader = get_random_input_data(self.args.batch_size, - self.args.vocab_size, - self.args.num_labels) + reader = get_random_input_data( + self.args.batch_size, self.args.vocab_size, self.args.num_labels + ) for batch in reader(): batch = [np.vstack(var) for var in zip(*batch)] dy_pre = self.predict_dygraph(batch) @@ -580,8 +674,9 @@ def predict_dygraph(self, batch): model.set_dict(model_dict) model.eval() - _, pred_res = model(to_variable(words), to_variable(targets), - to_variable(length)) + _, pred_res = model( + to_variable(words), to_variable(targets), to_variable(length) + ) return pred_res.numpy() @@ -593,19 +688,23 @@ def predict_static(self, batch): paddle.enable_static() exe = fluid.Executor(self.place) # load inference model - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(self.model_save_dir, - executor=exe, - model_filename=self.model_filename, - params_filename=self.params_filename) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + self.model_save_dir, + executor=exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + ) words, targets, length = batch - pred_res = exe.run(inference_program, - feed={ - feed_target_names[0]: words, - feed_target_names[1]: length - }, - fetch_list=fetch_targets) + pred_res = exe.run( + inference_program, + feed={feed_target_names[0]: words, feed_target_names[1]: length}, + fetch_list=fetch_targets, + ) return pred_res[0] def predict_dygraph_jit(self, batch): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lambda.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lambda.py index 121954edf9865..d0b45a050bc16 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lambda.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lambda.py @@ -79,18 +79,23 @@ def call_lambda_with_ifExpr2(x): class TestLambda(unittest.TestCase): - def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.x = np.array([1, 3]).astype('float32') - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.init_func() def init_func(self): self.dyfuncs = [ - call_lambda_as_func, call_lambda_directly, call_lambda_in_func, - call_lambda_with_ifExpr, call_lambda_with_ifExpr2 + call_lambda_as_func, + call_lambda_directly, + call_lambda_in_func, + call_lambda_with_ifExpr, + call_lambda_with_ifExpr2, ] def run_static(self, func): @@ -109,7 +114,8 @@ def run_dygraph(self, func, to_static=False): def test_ast_to_func(self): for func in self.dyfuncs: self.assertTrue( - (self.run_dygraph(func) == self.run_static(func)).all()) + (self.run_dygraph(func) == self.run_static(func)).all() + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_layer_hook.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_layer_hook.py index b2a23b7d86faa..ad4aa8e173f61 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_layer_hook.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_layer_hook.py @@ -24,13 +24,14 @@ def forward_post_hook1(layer, input, output): def forward_pre_hook1(layer, input): - input_return = (input[0] * 2, ) + input_return = (input[0] * 2,) return input_return class SimpleNet(paddle.nn.Layer): - - def __init__(self, ): + def __init__( + self, + ): super(SimpleNet, self).__init__() self.fc1 = paddle.nn.Linear(10, 10) # sublayer1 register post hook @@ -53,7 +54,6 @@ def forward(self, x): class TestNestLayerHook(unittest.TestCase): - def setUp(self): paddle.seed(2022) self.x = paddle.randn([4, 10]) @@ -90,12 +90,15 @@ def test_hook(self): dy_out, rtol=1e-05, err_msg='dygraph_res is {}\nstatic_res is {}'.format( - dy_out, st_out)) + dy_out, st_out + ), + ) np.testing.assert_allclose( st_out, load_out, rtol=1e-05, - err_msg='load_out is {}\nstatic_res is {}'.format(load_out, st_out)) + err_msg='load_out is {}\nstatic_res is {}'.format(load_out, st_out), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_len.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_len.py index c2a7fb2d1b9df..01e5f1e1074cc 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_len.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_len.py @@ -42,10 +42,12 @@ def len_with_lod_tensor_array(x): class TestLen(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.x_data = np.random.random([10, 16]).astype('float32') self.init_func() @@ -70,7 +72,6 @@ def test_len(self): class TestLenWithTensorArray(TestLen): - def init_func(self): self.func = len_with_lod_tensor_array @@ -80,11 +81,13 @@ def init_func(self): def len_with_selected_rows(place): block = fluid.default_main_program().global_block() # create selected_rows variable - var = block.create_var(name="X", - dtype="float32", - shape=[-1], - persistable=True, - type=fluid.core.VarDesc.VarType.SELECTED_ROWS) + var = block.create_var( + name="X", + dtype="float32", + shape=[-1], + persistable=True, + type=fluid.core.VarDesc.VarType.SELECTED_ROWS, + ) # y is Variable(SelectedRows) y = fluid.layers.merge_selected_rows(var) y_len = convert_call(len)(y) @@ -110,14 +113,17 @@ def len_with_selected_rows(place): class TestLenWithSelectedRows(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def test_len(self): selected_rows_var_len, var_tensor_len = len_with_selected_rows( - self.place) + self.place + ) self.assertEqual(selected_rows_var_len, var_tensor_len) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py index a1f681193e515..58ef815b82c71 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_list.py @@ -42,7 +42,8 @@ def test_list_append_in_if(x): a.append(x) else: a.append( - fluid.layers.fill_constant(shape=[1, 2], value=9, dtype="int64")) + fluid.layers.fill_constant(shape=[1, 2], value=9, dtype="int64") + ) # TODO(Aurelius84): Currently, run_program_op doesn't support output LoDTensorArray. return a[0] @@ -98,9 +99,9 @@ def test_list_append_in_for_loop_with_concat(x, iter_num): def test_list_append_in_while_loop(x, iter_num): x = fluid.dygraph.to_variable(x) - iter_num = fluid.layers.fill_constant(shape=[1], - value=iter_num, - dtype="int32") + iter_num = fluid.layers.fill_constant( + shape=[1], value=iter_num, dtype="int32" + ) a = [] i = 0 while i < iter_num: @@ -111,9 +112,9 @@ def test_list_append_in_while_loop(x, iter_num): def test_list_append_in_while_loop_with_stack(x, iter_num): x = fluid.dygraph.to_variable(x) - iter_num = fluid.layers.fill_constant(shape=[1], - value=iter_num, - dtype="int32") + iter_num = fluid.layers.fill_constant( + shape=[1], value=iter_num, dtype="int32" + ) a = [] i = 0 while i < iter_num.numpy()[0]: @@ -180,9 +181,9 @@ def test_list_pop_in_for_loop(x, iter_num): def test_list_pop_in_while_loop(x, iter_num): x = fluid.dygraph.to_variable(x) - iter_num = fluid.layers.fill_constant(shape=[1], - value=iter_num, - dtype="int32") + iter_num = fluid.layers.fill_constant( + shape=[1], value=iter_num, dtype="int32" + ) a = [] b = [x] b.append(x) @@ -199,10 +200,12 @@ def test_list_pop_in_while_loop(x, iter_num): class TestListWithoutControlFlow(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.init_data() self.init_dygraph_func() @@ -252,24 +255,25 @@ def test_transformed_static_result(self): dy_res, rtol=1e-05, err_msg='dygraph_res is {}\nstatic_res is {}'.format( - dy_res, stat_res)) + dy_res, stat_res + ), + ) class TestListInIf(TestListWithoutControlFlow): - def init_dygraph_func(self): self.all_dygraph_funcs = [test_list_append_in_if] class TestListInWhileLoop(TestListWithoutControlFlow): - def init_data(self): self.input = np.random.random((3)).astype('int32') self.iter_num = 3 def init_dygraph_func(self): self.all_dygraph_funcs = [ - test_list_append_in_while_loop, test_list_pop_in_while_loop + test_list_append_in_while_loop, + test_list_pop_in_while_loop, ] def train(self, to_static=False): @@ -284,21 +288,19 @@ def train(self, to_static=False): class TestListInWhileLoopWithStack(TestListInWhileLoop): - def init_dygraph_func(self): self.all_dygraph_funcs = [test_list_append_in_while_loop_with_stack] class TestListInForLoop(TestListInWhileLoop): - def init_dygraph_func(self): self.all_dygraph_funcs = [ - test_list_append_in_for_loop, test_list_pop_in_for_loop + test_list_append_in_for_loop, + test_list_pop_in_for_loop, ] class TestListInForLoopWithConcat(TestListInWhileLoopWithStack): - def init_dygraph_func(self): self.all_dygraph_funcs = [ test_list_append_in_for_loop_with_concat, @@ -306,11 +308,10 @@ def init_dygraph_func(self): class TestListInForLoopWithSubscript(TestListWithoutControlFlow): - def init_dygraph_func(self): self.all_dygraph_funcs = [ test_list_append_in_for_subscript, - test_list_append_in_while_loop_subscript + test_list_append_in_while_loop_subscript, ] def init_data(self): @@ -318,7 +319,6 @@ def init_data(self): class ListWithCondNet(paddle.nn.Layer): - def __init__(self): super(ListWithCondNet, self).__init__() @@ -342,13 +342,12 @@ def forward(self, x, index): class TestListWithCondGradInferVarType(unittest.TestCase): - def test_to_static(self): net = ListWithCondNet() x = paddle.to_tensor([2, 3, 4], dtype='float32') index = paddle.to_tensor([1]) res = net(x, index) - self.assertEqual(res[0], 48.) + self.assertEqual(res[0], 48.0) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py index d563758cdd61e..cdc63ca353ee2 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logging_utils.py @@ -26,7 +26,6 @@ class TestLoggingUtils(unittest.TestCase): - def setUp(self): self.verbosity_level = 1 self.code_level = 3 @@ -50,19 +49,23 @@ def test_verbosity(self): def test_also_to_stdout(self): logging_utils._TRANSLATOR_LOGGER.need_to_echo_log_to_stdout = None self.assertEqual( - logging_utils._TRANSLATOR_LOGGER.need_to_echo_log_to_stdout, False) + logging_utils._TRANSLATOR_LOGGER.need_to_echo_log_to_stdout, False + ) paddle.jit.set_verbosity(also_to_stdout=False) self.assertEqual( - logging_utils._TRANSLATOR_LOGGER.need_to_echo_log_to_stdout, False) + logging_utils._TRANSLATOR_LOGGER.need_to_echo_log_to_stdout, False + ) logging_utils._TRANSLATOR_LOGGER.need_to_echo_node_to_stdout = None self.assertEqual( - logging_utils._TRANSLATOR_LOGGER.need_to_echo_code_to_stdout, False) + logging_utils._TRANSLATOR_LOGGER.need_to_echo_code_to_stdout, False + ) paddle.jit.set_code_level(also_to_stdout=True) self.assertEqual( - logging_utils._TRANSLATOR_LOGGER.need_to_echo_code_to_stdout, True) + logging_utils._TRANSLATOR_LOGGER.need_to_echo_code_to_stdout, True + ) with self.assertRaises(AssertionError): paddle.jit.set_verbosity(also_to_stdout=1) @@ -99,8 +102,9 @@ def test_log_api(self): logging_utils.set_code_level(1, True) logging_utils.log_transformed_code(1, ast_code, "TestTransformer") logging_utils.set_code_level(logging_utils.LOG_AllTransformer, True) - logging_utils.log_transformed_code(logging_utils.LOG_AllTransformer, - ast_code, "TestTransformer") + logging_utils.log_transformed_code( + logging_utils.LOG_AllTransformer, ast_code, "TestTransformer" + ) def test_log_message(self): stream = io.StringIO() @@ -121,7 +125,8 @@ def test_log_message(self): logging_utils.log(2, log_msg_2) result_msg = '\n'.join( - [warn_msg, error_msg, "(Level 1) " + log_msg_1, ""]) + [warn_msg, error_msg, "(Level 1) " + log_msg_1, ""] + ) self.assertEqual(result_msg, stream.getvalue()) def test_log_transformed_code(self): @@ -135,12 +140,14 @@ def test_log_transformed_code(self): with mock.patch.object(sys, 'stdout', stream): paddle.jit.set_code_level(1) - logging_utils.log_transformed_code(1, ast_code, - "BasicApiTransformer") + logging_utils.log_transformed_code( + 1, ast_code, "BasicApiTransformer" + ) paddle.jit.set_code_level() - logging_utils.log_transformed_code(logging_utils.LOG_AllTransformer, - ast_code, "All Transformers") + logging_utils.log_transformed_code( + logging_utils.LOG_AllTransformer, ast_code, "All Transformers" + ) self.assertIn(source_code, stream.getvalue()) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py index a88047839048e..4ab65c4766070 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_logical.py @@ -23,7 +23,9 @@ import paddle import paddle.fluid as fluid from paddle.fluid.dygraph import ProgramTranslator -from paddle.fluid.dygraph.dygraph_to_static.logical_transformer import cmpop_node_to_str +from paddle.fluid.dygraph.dygraph_to_static.logical_transformer import ( + cmpop_node_to_str, +) program_translator = ProgramTranslator() @@ -171,16 +173,19 @@ def test_shape_not_equal(x): class TestLogicalBase(unittest.TestCase): - def setUp(self): self.input = np.array([3]).astype('int32') - self.place = paddle.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self._set_test_func() def _set_test_func(self): raise NotImplementedError( - "Method 'set_test_func' should be implemented.") + "Method 'set_test_func' should be implemented." + ) def _run(self, to_static): program_translator.enable(to_static) @@ -196,7 +201,6 @@ def _run_static(self): class TestLogicalNot(TestLogicalBase): - def _set_test_func(self): self.dygraph_func = test_logical_not @@ -208,11 +212,12 @@ def test_transformed_result(self): static_res, rtol=1e-05, err_msg='dygraph result is {}\nstatic_result is {}'.format( - dygraph_res, static_res)) + dygraph_res, static_res + ), + ) class TestLogicalNot2(TestLogicalBase): - def _set_test_func(self): self.dygraph_func = test_logical_not_2 @@ -224,55 +229,49 @@ def test_transformed_result(self): static_res, rtol=1e-05, err_msg='dygraph result is {}\nstatic_result is {}'.format( - dygraph_res, static_res)) + dygraph_res, static_res + ), + ) class TestLogicalAnd(TestLogicalNot): - def _set_test_func(self): self.dygraph_func = test_logical_and class TestLogicalAnd2(TestLogicalNot): - def _set_test_func(self): self.dygraph_func = test_logical_and_2 class TestLogicalOr(TestLogicalNot): - def _set_test_func(self): self.dygraph_func = test_logical_or class TestLogicalOr2(TestLogicalNot): - def _set_test_func(self): self.dygraph_func = test_logical_or_2 class TestLogicalNotAndOr(TestLogicalNot): - def _set_test_func(self): self.dygraph_func = test_logical_not_and_or class TestShapeEqual(TestLogicalNot): - def _set_test_func(self): self.input = np.ones([1, 2, 3]).astype('float32') self.dygraph_func = test_shape_equal class TestShapeNotEqual(TestLogicalNot): - def _set_test_func(self): self.input = np.ones([1, 2, 3]).astype('float32') self.dygraph_func = test_shape_not_equal class TestCmpopNodeToStr(unittest.TestCase): - def test_exception(self): with self.assertRaises(KeyError): cmpop_node_to_str(gast.Or()) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py index 478d573558e40..6bc237f3781b6 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_loop.py @@ -63,12 +63,15 @@ def relu(y): def while_loop_dyfunc_with_none(x): - i = fluid.dygraph.to_variable(x)\ - if x is not None \ - else fluid.dygraph.to_variable(x+1) + i = ( + fluid.dygraph.to_variable(x) + if x is not None + else fluid.dygraph.to_variable(x + 1) + ) # Use `to_variable` so that static analysis can analyze the type of X is Tensor x = fluid.dygraph.to_variable( - x) # TODO(liym27): Delete it if the type of parameter x can be resolved + x + ) # TODO(liym27): Delete it if the type of parameter x can be resolved flag = 1 while x < 10: i = i + x if flag is not None else x + i @@ -150,9 +153,7 @@ def while_loop_bool_op2(x): def while_loop_class_var(x): - class Foo(object): - def __init__(self): self.a = 3 self.b = 4 @@ -178,9 +179,7 @@ def loop_var_contains_property(x): def for_loop_class_var(max_len): - class Foo(object): - def __init__(self): self.a = 3 self.b = 4 @@ -189,9 +188,9 @@ def __init__(self): foo = Foo() # Use `to_variable` so that static analysis can analyze the type of X is Tensor - max_len = fluid.layers.fill_constant(shape=[1], - value=max_len, - dtype="int32") + max_len = fluid.layers.fill_constant( + shape=[1], value=max_len, dtype="int32" + ) for i in range(max_len): foo.b = fluid.layers.zeros(shape=[1], dtype='float32') @@ -230,17 +229,18 @@ def for_loop_dufunc_with_listcomp(array): class TestNameVisitor(unittest.TestCase): - def setUp(self): self.loop_funcs = [ - while_loop_dyfunc, for_loop_dyfunc, while_loop_dyfunc_with_none, - for_loop_dufunc_with_listcomp + while_loop_dyfunc, + for_loop_dyfunc, + while_loop_dyfunc_with_none, + for_loop_dufunc_with_listcomp, ] self.loop_var_names = [ set(["i", "x"]), set(["i", "ret", "max_len"]), set(["i", "x"]), - set(["j", "array", "res", "x"]) + set(["j", "array", "res", "x"]), ] self.create_var_names = [set(), set(["ret"]), set(), set(["res", "x"])] @@ -254,8 +254,10 @@ def test_loop_vars(self): name_visitor = NameVisitor(gast_root) for node in gast.walk(gast_root): if isinstance(node, (gast.While, gast.For)): - loop_var_names, create_var_names = name_visitor.get_loop_var_names( - node) + ( + loop_var_names, + create_var_names, + ) = name_visitor.get_loop_var_names(node) self.assertEqual(loop_var_names, self.loop_var_names[i]) self.assertEqual(create_var_names, self.create_var_names[i]) @@ -268,34 +270,41 @@ def test_nested_loop_vars(self): self.loop_var_names = [ set(["j", "two"]), set(["i", "three", "b"]), - set(["i"]) + set(["i"]), ] self.create_var_names = [set(), set(["b"]), set()] i = 0 for node in gast.walk(gast_root): if isinstance(node, (gast.While, gast.For)): - loop_var_names, create_var_names = name_visitor.get_loop_var_names( - node) + ( + loop_var_names, + create_var_names, + ) = name_visitor.get_loop_var_names(node) self.assertEqual( loop_var_names, self.loop_var_names[i], - msg="loop_var_names : {}, \nexpected loop_var_names : {}". - format(loop_var_names, self.loop_var_names[i])) + msg="loop_var_names : {}, \nexpected loop_var_names : {}".format( + loop_var_names, self.loop_var_names[i] + ), + ) self.assertEqual( create_var_names, self.create_var_names[i], - msg= - "i = {}\ncreate_var_names : {}, \nexpected create_var_names : {}" - .format(i, create_var_names, self.create_var_names[i])) + msg="i = {}\ncreate_var_names : {}, \nexpected create_var_names : {}".format( + i, create_var_names, self.create_var_names[i] + ), + ) i += 1 class TestTransformWhileLoop(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.x = np.zeros(shape=(1), dtype=np.int32) self._init_dyfunc() @@ -329,58 +338,52 @@ def test_ast_to_func(self): class TestTransformWhileLoopWithoutTensor(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = while_loop_dyfunc_without_tensor class TestTransformWhileLoopWithConflicVar(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = while_loop_dyfun_with_conflict_var class TestTransformWhileLoopWithNone(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = while_loop_dyfunc_with_none class TestForBreakSingleReturn(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = for_break_single_return class TestWhileLoopBoolOp(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = while_loop_bool_op class TestWhileLoopBoolOp2(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = while_loop_bool_op2 class TestWhileLoopClassVar(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = while_loop_class_var class TestLoopVarContainsProperty(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = loop_var_contains_property class TestTransformForLoop(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.len = 100 self._init_dyfunc() @@ -402,43 +405,37 @@ def _run(self, to_static): return ret.numpy() def test_ast_to_func(self): - np.testing.assert_allclose(self._run_dygraph(), - self._run_static(), - rtol=1e-05) + np.testing.assert_allclose( + self._run_dygraph(), self._run_static(), rtol=1e-05 + ) class TestTransformForLoop2(TestTransformForLoop): - def _init_dyfunc(self): self.dyfunc = for_loop_dyfunc2 class TestTransformForLoop3(TestTransformForLoop): - def _init_dyfunc(self): self.dyfunc = for_loop_dyfunc3 class TestTransformForLoop4(TestTransformForLoop): - def _init_dyfunc(self): self.dyfunc = for_loop_dyfunc4 class TestClassVarInForLoop(TestTransformForLoop): - def _init_dyfunc(self): self.dyfunc = for_loop_class_var class TestVarCreateInForLoop(TestTransformForLoop): - def _init_dyfunc(self): self.dyfunc = var_create_in_for_loop class TestErrorInForLoop(TestTransformForLoop): - def _init_dyfunc(self): self.dyfunc = for_loop_dyfunc_not_support diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lstm.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lstm.py index d47445c05e98f..b4bbfd789c09f 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lstm.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_lstm.py @@ -21,13 +21,11 @@ class LSTMLayer(nn.Layer): - def __init__(self, in_channels, hidden_size): super(LSTMLayer, self).__init__() - self.cell = nn.LSTM(in_channels, - hidden_size, - direction='bidirectional', - num_layers=2) + self.cell = nn.LSTM( + in_channels, hidden_size, direction='bidirectional', num_layers=2 + ) def forward(self, x): x, _ = self.cell(x) @@ -35,7 +33,6 @@ def forward(self, x): class Net(nn.Layer): - def __init__(self, in_channels, hidden_size): super(Net, self).__init__() self.lstm = LSTMLayer(in_channels, hidden_size) @@ -46,7 +43,6 @@ def forward(self, x): class TestLstm(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -79,15 +75,17 @@ def test_save_in_eval(self, with_training=True): x.stop_gradient = False dygraph_out = net(x) loss = paddle.mean(dygraph_out) - sgd = paddle.optimizer.SGD(learning_rate=0.001, - parameters=net.parameters()) + sgd = paddle.optimizer.SGD( + learning_rate=0.001, parameters=net.parameters() + ) loss.backward() sgd.step() # switch eval mode firstly net.eval() x = paddle.randn((2, 10, 12)) net = paddle.jit.to_static( - net, input_spec=[paddle.static.InputSpec(shape=[-1, 10, 12])]) + net, input_spec=[paddle.static.InputSpec(shape=[-1, 10, 12])] + ) model_path = os.path.join(self.temp_dir.name, 'simple_lstm') paddle.jit.save(net, model_path) @@ -101,7 +99,9 @@ def test_save_in_eval(self, with_training=True): static_out.numpy(), rtol=1e-05, err_msg='dygraph_out is {}\n static_out is \n{}'.format( - dygraph_out, static_out)) + dygraph_out, static_out + ), + ) # switch back into train mode. net.train() train_out = net(x) @@ -110,14 +110,15 @@ def test_save_in_eval(self, with_training=True): train_out.numpy(), rtol=1e-05, err_msg='dygraph_out is {}\n static_out is \n{}'.format( - dygraph_out, train_out)) + dygraph_out, train_out + ), + ) def test_save_without_training(self): self.test_save_in_eval(with_training=False) class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self.fc = nn.Linear(10, 12) @@ -131,7 +132,6 @@ def forward(self, x): class TestSaveInEvalMode(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -145,15 +145,17 @@ def test_save_in_eval(self): x.stop_gradient = False dygraph_out = net(x) loss = paddle.mean(dygraph_out) - sgd = paddle.optimizer.SGD(learning_rate=0.001, - parameters=net.parameters()) + sgd = paddle.optimizer.SGD( + learning_rate=0.001, parameters=net.parameters() + ) loss.backward() sgd.step() # switch eval mode firstly net.eval() # save directly net = paddle.jit.to_static( - net, input_spec=[paddle.static.InputSpec(shape=[-1, 10])]) + net, input_spec=[paddle.static.InputSpec(shape=[-1, 10])] + ) model_path = os.path.join(self.temp_dir.name, 'linear_net') paddle.jit.save(net, model_path) @@ -169,11 +171,12 @@ def test_save_in_eval(self): infer_out.numpy(), rtol=1e-05, err_msg='eval_out is {}\n infer_out is \n{}'.format( - eval_out, infer_out)) + eval_out, infer_out + ), + ) class TestEvalAfterSave(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -186,8 +189,9 @@ def test_eval_after_save(self): x.stop_gradient = False dy_out = net(x) loss = paddle.mean(dy_out) - sgd = paddle.optimizer.SGD(learning_rate=0.001, - parameters=net.parameters()) + sgd = paddle.optimizer.SGD( + learning_rate=0.001, parameters=net.parameters() + ) loss.backward() sgd.step() x = paddle.randn((2, 10, 12)).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py index 7d4074954026c..bc4ce83114278 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist.py @@ -37,44 +37,49 @@ class SimpleImgConvPool(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - pool_size, - pool_stride, - pool_padding=0, - pool_type='max', - global_pooling=False, - conv_stride=1, - conv_padding=0, - conv_dilation=1, - conv_groups=1, - act=None, - use_cudnn=True, - param_attr=None, - bias_attr=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + pool_size, + pool_stride, + pool_padding=0, + pool_type='max', + global_pooling=False, + conv_stride=1, + conv_padding=0, + conv_dilation=1, + conv_groups=1, + act=None, + use_cudnn=True, + param_attr=None, + bias_attr=None, + ): super(SimpleImgConvPool, self).__init__() - self._conv2d = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=conv_stride, - padding=conv_padding, - dilation=conv_dilation, - groups=conv_groups, - param_attr=None, - bias_attr=None, - act=act, - use_cudnn=use_cudnn) - - self._pool2d = Pool2D(pool_size=pool_size, - pool_type=pool_type, - pool_stride=pool_stride, - pool_padding=pool_padding, - global_pooling=global_pooling, - use_cudnn=use_cudnn) + self._conv2d = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=conv_stride, + padding=conv_padding, + dilation=conv_dilation, + groups=conv_groups, + param_attr=None, + bias_attr=None, + act=act, + use_cudnn=use_cudnn, + ) + + self._pool2d = Pool2D( + pool_size=pool_size, + pool_type=pool_type, + pool_stride=pool_stride, + pool_padding=pool_padding, + global_pooling=global_pooling, + use_cudnn=use_cudnn, + ) def forward(self, inputs): x = self._conv2d(inputs) @@ -83,33 +88,30 @@ def forward(self, inputs): class MNIST(fluid.dygraph.Layer): - def __init__(self): super(MNIST, self).__init__() - self._simple_img_conv_pool_1 = SimpleImgConvPool(1, - 20, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_1 = SimpleImgConvPool( + 1, 20, 5, 2, 2, act="relu" + ) - self._simple_img_conv_pool_2 = SimpleImgConvPool(20, - 50, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_2 = SimpleImgConvPool( + 20, 50, 5, 2, 2, act="relu" + ) self.pool_2_shape = 50 * 4 * 4 SIZE = 10 - scale = (2.0 / (self.pool_2_shape**2 * SIZE))**0.5 - self._fc = Linear(self.pool_2_shape, - 10, - param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.NormalInitializer( - loc=0.0, scale=scale)), - act="softmax") + scale = (2.0 / (self.pool_2_shape**2 * SIZE)) ** 0.5 + self._fc = Linear( + self.pool_2_shape, + 10, + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=scale + ) + ), + act="softmax", + ) def forward(self, inputs, label=None): x = self.inference(inputs) @@ -131,15 +133,19 @@ def inference(self, inputs): class TestMNIST(unittest.TestCase): - def setUp(self): self.epoch_num = 1 self.batch_size = 64 - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() - self.train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=self.batch_size, - drop_last=True) + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + self.train_reader = paddle.batch( + paddle.dataset.mnist.train(), + batch_size=self.batch_size, + drop_last=True, + ) self.temp_dir = tempfile.TemporaryDirectory() def tearDown(self): @@ -167,7 +173,9 @@ def test_mnist_to_static(self): static_loss, rtol=1e-05, err_msg='dygraph is {}\n static_res is \n{}'.format( - dygraph_loss, static_loss)) + dygraph_loss, static_loss + ), + ) with _test_eager_guard(): dygraph_loss = self.train_dygraph() static_loss = self.train_static() @@ -176,7 +184,9 @@ def test_mnist_to_static(self): static_loss, rtol=1e-05, err_msg='dygraph is {}\n static_res is \n{}'.format( - dygraph_loss, static_loss)) + dygraph_loss, static_loss + ), + ) def test_mnist_declarative_cpu_vs_mkldnn(self): dygraph_loss_cpu = self.train_dygraph() @@ -190,7 +200,9 @@ def test_mnist_declarative_cpu_vs_mkldnn(self): dygraph_loss_mkldnn, rtol=1e-05, err_msg='cpu dygraph is {}\n mkldnn dygraph is \n{}'.format( - dygraph_loss_cpu, dygraph_loss_mkldnn)) + dygraph_loss_cpu, dygraph_loss_mkldnn + ), + ) def train(self, to_static=False): @@ -201,17 +213,21 @@ def train(self, to_static=False): mnist = MNIST() if to_static: mnist = paddle.jit.to_static(mnist) - adam = AdamOptimizer(learning_rate=0.001, - parameter_list=mnist.parameters()) + adam = AdamOptimizer( + learning_rate=0.001, parameter_list=mnist.parameters() + ) for epoch in range(self.epoch_num): start = time() for batch_id, data in enumerate(self.train_reader()): - dy_x_data = np.array([ - x[0].reshape(1, 28, 28) for x in data - ]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + dy_x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = to_variable(dy_x_data) label = to_variable(y_data) @@ -226,66 +242,87 @@ def train(self, to_static=False): mnist.clear_gradients() if batch_id % 10 == 0: print( - "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}" - .format(epoch, batch_id, avg_loss.numpy(), - acc.numpy(), - time() - start)) + "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}".format( + epoch, + batch_id, + avg_loss.numpy(), + acc.numpy(), + time() - start, + ) + ) start = time() if batch_id == 50: mnist.eval() prediction, acc, avg_loss = mnist(img, label) loss_data.append(avg_loss.numpy()[0]) # new save load check - self.check_jit_save_load(mnist, [dy_x_data], [img], - to_static, prediction) + self.check_jit_save_load( + mnist, [dy_x_data], [img], to_static, prediction + ) break return loss_data def check_jit_save_load(self, model, inputs, input_spec, to_static, gt_out): if to_static: infer_model_path = os.path.join( - self.temp_dir.name, 'test_mnist_inference_model_by_jit_save') + self.temp_dir.name, 'test_mnist_inference_model_by_jit_save' + ) model_save_dir = os.path.join(self.temp_dir.name, 'inference') model_save_prefix = os.path.join(model_save_dir, 'mnist') model_filename = "mnist" + INFER_MODEL_SUFFIX params_filename = "mnist" + INFER_PARAMS_SUFFIX - fluid.dygraph.jit.save(layer=model, - path=model_save_prefix, - input_spec=input_spec, - output_spec=[gt_out]) + fluid.dygraph.jit.save( + layer=model, + path=model_save_prefix, + input_spec=input_spec, + output_spec=[gt_out], + ) # load in static mode static_infer_out = self.jit_load_and_run_inference_static( - model_save_dir, model_filename, params_filename, inputs) - np.testing.assert_allclose(gt_out.numpy(), - static_infer_out, - rtol=1e-05) + model_save_dir, model_filename, params_filename, inputs + ) + np.testing.assert_allclose( + gt_out.numpy(), static_infer_out, rtol=1e-05 + ) # load in dygraph mode dygraph_infer_out = self.jit_load_and_run_inference_dygraph( - model_save_prefix, inputs) - np.testing.assert_allclose(gt_out.numpy(), - dygraph_infer_out, - rtol=1e-05) + model_save_prefix, inputs + ) + np.testing.assert_allclose( + gt_out.numpy(), dygraph_infer_out, rtol=1e-05 + ) # load in Paddle-Inference - predictor_infer_out = self.predictor_load_and_run_inference_analysis( - model_save_dir, model_filename, params_filename, inputs) - np.testing.assert_allclose(gt_out.numpy(), - predictor_infer_out, - rtol=1e-05) + predictor_infer_out = ( + self.predictor_load_and_run_inference_analysis( + model_save_dir, model_filename, params_filename, inputs + ) + ) + np.testing.assert_allclose( + gt_out.numpy(), predictor_infer_out, rtol=1e-05 + ) @switch_to_static_graph - def jit_load_and_run_inference_static(self, model_path, model_filename, - params_filename, inputs): + def jit_load_and_run_inference_static( + self, model_path, model_filename, params_filename, inputs + ): paddle.enable_static() exe = fluid.Executor(self.place) - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(dirname=model_path, - executor=exe, - model_filename=model_filename, - params_filename=params_filename) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + dirname=model_path, + executor=exe, + model_filename=model_filename, + params_filename=params_filename, + ) assert len(inputs) == len(feed_target_names) - results = exe.run(inference_program, - feed=dict(zip(feed_target_names, inputs)), - fetch_list=fetch_targets) + results = exe.run( + inference_program, + feed=dict(zip(feed_target_names, inputs)), + fetch_list=fetch_targets, + ) return np.array(results[0]) @@ -294,12 +331,13 @@ def jit_load_and_run_inference_dygraph(self, model_path, inputs): pred = infer_net(inputs[0]) return pred.numpy() - def predictor_load_and_run_inference_analysis(self, model_path, - model_filename, - params_filename, inputs): - output = PredictorTools(model_path, model_filename, params_filename, - inputs) - out, = output() + def predictor_load_and_run_inference_analysis( + self, model_path, model_filename, params_filename, inputs + ): + output = PredictorTools( + model_path, model_filename, params_filename, inputs + ) + (out,) = output() return out diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_amp.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_amp.py index 56b55766af64b..3610e72b9539b 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_amp.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_amp.py @@ -24,7 +24,6 @@ class TestAMP(TestMNIST): - def train_static(self): return self.train(to_static=True) @@ -43,7 +42,9 @@ def test_mnist_to_static(self): rtol=1e-05, atol=0.001, err_msg='dygraph is {}\n static_res is \n{}'.format( - dygraph_loss, static_loss)) + dygraph_loss, static_loss + ), + ) def train(self, to_static=False): paddle.seed(SEED) @@ -53,8 +54,9 @@ def train(self, to_static=False): print("Successfully to apply @to_static.") mnist = paddle.jit.to_static(mnist) - adam = AdamOptimizer(learning_rate=0.001, - parameter_list=mnist.parameters()) + adam = AdamOptimizer( + learning_rate=0.001, parameter_list=mnist.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) @@ -62,10 +64,14 @@ def train(self, to_static=False): for epoch in range(self.epoch_num): start = time() for batch_id, data in enumerate(self.train_reader()): - dy_x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + dy_x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) @@ -83,9 +89,14 @@ def train(self, to_static=False): mnist.clear_gradients() if batch_id % 10 == 0: print( - "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}" - .format(epoch, batch_id, avg_loss.numpy(), acc.numpy(), - time() - start)) + "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}".format( + epoch, + batch_id, + avg_loss.numpy(), + acc.numpy(), + time() - start, + ) + ) start = time() if batch_id == 50: break diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_pure_fp16.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_pure_fp16.py index beca9d3f93357..8bdf81537429b 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_pure_fp16.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mnist_pure_fp16.py @@ -23,7 +23,6 @@ class TestPureFP16(TestMNIST): - def train_static(self): return self.train(to_static=True) @@ -41,7 +40,9 @@ def test_mnist_to_static(self): rtol=1e-05, atol=0.001, err_msg='dygraph is {}\n static_res is \n{}'.format( - dygraph_loss, static_loss)) + dygraph_loss, static_loss + ), + ) def train(self, to_static=False): np.random.seed(SEED) @@ -58,33 +59,39 @@ def train(self, to_static=False): build_strategy.enable_inplace = False mnist = paddle.jit.to_static(mnist, build_strategy=build_strategy) - optimizer = paddle.optimizer.Adam(learning_rate=0.001, - parameters=mnist.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=0.001, parameters=mnist.parameters() + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) - mnist, optimizer = paddle.amp.decorate(models=mnist, - optimizers=optimizer, - level='O2', - save_dtype='float32') + mnist, optimizer = paddle.amp.decorate( + models=mnist, optimizers=optimizer, level='O2', save_dtype='float32' + ) loss_data = [] for epoch in range(self.epoch_num): start = time() for batch_id, data in enumerate(self.train_reader()): - dy_x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(-1, 1) + dy_x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(-1, 1) + ) img = paddle.to_tensor(dy_x_data) label = paddle.to_tensor(y_data) label.stop_gradient = True - with paddle.amp.auto_cast(enable=True, - custom_white_list=None, - custom_black_list=None, - level='O2'): + with paddle.amp.auto_cast( + enable=True, + custom_white_list=None, + custom_black_list=None, + level='O2', + ): prediction, acc, avg_loss = mnist(img, label=label) scaled = scaler.scale(avg_loss) @@ -96,9 +103,14 @@ def train(self, to_static=False): mnist.clear_gradients() if batch_id % 2 == 0: print( - "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}" - .format(epoch, batch_id, avg_loss.numpy(), acc.numpy(), - time() - start)) + "Loss at epoch {} step {}: loss: {:}, acc: {}, cost: {}".format( + epoch, + batch_id, + avg_loss.numpy(), + acc.numpy(), + time() - start, + ) + ) start = time() if batch_id == 10: break diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mobile_net.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mobile_net.py index ca9a78f5f308d..e13a41db89a22 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mobile_net.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_mobile_net.py @@ -39,32 +39,35 @@ class ConvBNLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - filter_size, - num_filters, - stride, - padding, - channels=None, - num_groups=1, - act='relu', - use_cudnn=True, - name=None): + def __init__( + self, + num_channels, + filter_size, + num_filters, + stride, + padding, + channels=None, + num_groups=1, + act='relu', + use_cudnn=True, + name=None, + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=padding, - groups=num_groups, - act=None, - use_cudnn=use_cudnn, - param_attr=ParamAttr(initializer=MSRA(), - name=self.full_name() + - "_weights"), - bias_attr=False) + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=padding, + groups=num_groups, + act=None, + use_cudnn=use_cudnn, + param_attr=ParamAttr( + initializer=MSRA(), name=self.full_name() + "_weights" + ), + bias_attr=False, + ) self._batch_norm = BatchNorm( num_filters, @@ -72,7 +75,8 @@ def __init__(self, param_attr=ParamAttr(name=self.full_name() + "_bn" + "_scale"), bias_attr=ParamAttr(name=self.full_name() + "_bn" + "_offset"), moving_mean_name=self.full_name() + "_bn" + '_mean', - moving_variance_name=self.full_name() + "_bn" + '_variance') + moving_variance_name=self.full_name() + "_bn" + '_variance', + ) def forward(self, inputs, if_act=False): y = self._conv(inputs) @@ -83,32 +87,35 @@ def forward(self, inputs, if_act=False): class DepthwiseSeparable(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters1, - num_filters2, - num_groups, - stride, - scale, - name=None): + def __init__( + self, + num_channels, + num_filters1, + num_filters2, + num_groups, + stride, + scale, + name=None, + ): super(DepthwiseSeparable, self).__init__() - self._depthwise_conv = ConvBNLayer(num_channels=num_channels, - num_filters=int(num_filters1 * - scale), - filter_size=3, - stride=stride, - padding=1, - num_groups=int(num_groups * scale), - use_cudnn=True) + self._depthwise_conv = ConvBNLayer( + num_channels=num_channels, + num_filters=int(num_filters1 * scale), + filter_size=3, + stride=stride, + padding=1, + num_groups=int(num_groups * scale), + use_cudnn=True, + ) self._pointwise_conv = ConvBNLayer( num_channels=int(num_filters1 * scale), filter_size=1, num_filters=int(num_filters2 * scale), stride=1, - padding=0) + padding=0, + ) def forward(self, inputs): y = self._depthwise_conv(inputs) @@ -117,118 +124,148 @@ def forward(self, inputs): class MobileNetV1(fluid.dygraph.Layer): - def __init__(self, scale=1.0, class_dim=1000): super(MobileNetV1, self).__init__() self.scale = scale self.dwsl = [] - self.conv1 = ConvBNLayer(num_channels=3, - filter_size=3, - channels=3, - num_filters=int(32 * scale), - stride=2, - padding=1) - - dws21 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 32 * scale), - num_filters1=32, - num_filters2=64, - num_groups=32, - stride=1, - scale=scale), - name="conv2_1") + self.conv1 = ConvBNLayer( + num_channels=3, + filter_size=3, + channels=3, + num_filters=int(32 * scale), + stride=2, + padding=1, + ) + + dws21 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(32 * scale), + num_filters1=32, + num_filters2=64, + num_groups=32, + stride=1, + scale=scale, + ), + name="conv2_1", + ) self.dwsl.append(dws21) - dws22 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 64 * scale), - num_filters1=64, - num_filters2=128, - num_groups=64, - stride=2, - scale=scale), - name="conv2_2") + dws22 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(64 * scale), + num_filters1=64, + num_filters2=128, + num_groups=64, + stride=2, + scale=scale, + ), + name="conv2_2", + ) self.dwsl.append(dws22) - dws31 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 128 * scale), - num_filters1=128, - num_filters2=128, - num_groups=128, - stride=1, - scale=scale), - name="conv3_1") + dws31 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(128 * scale), + num_filters1=128, + num_filters2=128, + num_groups=128, + stride=1, + scale=scale, + ), + name="conv3_1", + ) self.dwsl.append(dws31) - dws32 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 128 * scale), - num_filters1=128, - num_filters2=256, - num_groups=128, - stride=2, - scale=scale), - name="conv3_2") + dws32 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(128 * scale), + num_filters1=128, + num_filters2=256, + num_groups=128, + stride=2, + scale=scale, + ), + name="conv3_2", + ) self.dwsl.append(dws32) - dws41 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 256 * scale), - num_filters1=256, - num_filters2=256, - num_groups=256, - stride=1, - scale=scale), - name="conv4_1") + dws41 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(256 * scale), + num_filters1=256, + num_filters2=256, + num_groups=256, + stride=1, + scale=scale, + ), + name="conv4_1", + ) self.dwsl.append(dws41) - dws42 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 256 * scale), - num_filters1=256, - num_filters2=512, - num_groups=256, - stride=2, - scale=scale), - name="conv4_2") + dws42 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(256 * scale), + num_filters1=256, + num_filters2=512, + num_groups=256, + stride=2, + scale=scale, + ), + name="conv4_2", + ) self.dwsl.append(dws42) for i in range(5): - tmp = self.add_sublayer(sublayer=DepthwiseSeparable( + tmp = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(512 * scale), + num_filters1=512, + num_filters2=512, + num_groups=512, + stride=1, + scale=scale, + ), + name="conv5_" + str(i + 1), + ) + self.dwsl.append(tmp) + + dws56 = self.add_sublayer( + sublayer=DepthwiseSeparable( num_channels=int(512 * scale), num_filters1=512, - num_filters2=512, + num_filters2=1024, num_groups=512, - stride=1, - scale=scale), - name="conv5_" + str(i + 1)) - self.dwsl.append(tmp) - - dws56 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 512 * scale), - num_filters1=512, - num_filters2=1024, - num_groups=512, - stride=2, - scale=scale), - name="conv5_6") + stride=2, + scale=scale, + ), + name="conv5_6", + ) self.dwsl.append(dws56) - dws6 = self.add_sublayer(sublayer=DepthwiseSeparable(num_channels=int( - 1024 * scale), - num_filters1=1024, - num_filters2=1024, - num_groups=1024, - stride=1, - scale=scale), - name="conv6") + dws6 = self.add_sublayer( + sublayer=DepthwiseSeparable( + num_channels=int(1024 * scale), + num_filters1=1024, + num_filters2=1024, + num_groups=1024, + stride=1, + scale=scale, + ), + name="conv6", + ) self.dwsl.append(dws6) self.pool2d_avg = Pool2D(pool_type='avg', global_pooling=True) - self.out = Linear(int(1024 * scale), - class_dim, - param_attr=ParamAttr(initializer=MSRA(), - name=self.full_name() + - "fc7_weights"), - bias_attr=ParamAttr(name="fc7_offset")) + self.out = Linear( + int(1024 * scale), + class_dim, + param_attr=ParamAttr( + initializer=MSRA(), name=self.full_name() + "fc7_weights" + ), + bias_attr=ParamAttr(name="fc7_offset"), + ) @declarative def forward(self, inputs): @@ -242,7 +279,6 @@ def forward(self, inputs): class InvertedResidualUnit(fluid.dygraph.Layer): - def __init__( self, num_channels, @@ -255,30 +291,36 @@ def __init__( ): super(InvertedResidualUnit, self).__init__() num_expfilter = int(round(num_in_filter * expansion_factor)) - self._expand_conv = ConvBNLayer(num_channels=num_channels, - num_filters=num_expfilter, - filter_size=1, - stride=1, - padding=0, - act=None, - num_groups=1) - - self._bottleneck_conv = ConvBNLayer(num_channels=num_expfilter, - num_filters=num_expfilter, - filter_size=filter_size, - stride=stride, - padding=padding, - num_groups=num_expfilter, - act=None, - use_cudnn=True) - - self._linear_conv = ConvBNLayer(num_channels=num_expfilter, - num_filters=num_filters, - filter_size=1, - stride=1, - padding=0, - act=None, - num_groups=1) + self._expand_conv = ConvBNLayer( + num_channels=num_channels, + num_filters=num_expfilter, + filter_size=1, + stride=1, + padding=0, + act=None, + num_groups=1, + ) + + self._bottleneck_conv = ConvBNLayer( + num_channels=num_expfilter, + num_filters=num_expfilter, + filter_size=filter_size, + stride=stride, + padding=padding, + num_groups=num_expfilter, + act=None, + use_cudnn=True, + ) + + self._linear_conv = ConvBNLayer( + num_channels=num_expfilter, + num_filters=num_filters, + filter_size=1, + stride=1, + padding=0, + act=None, + num_groups=1, + ) def forward(self, inputs, ifshortcut): y = self._expand_conv(inputs, if_act=True) @@ -290,29 +332,33 @@ def forward(self, inputs, ifshortcut): class InvresiBlocks(fluid.dygraph.Layer): - def __init__(self, in_c, t, c, n, s): super(InvresiBlocks, self).__init__() - self._first_block = InvertedResidualUnit(num_channels=in_c, - num_in_filter=in_c, - num_filters=c, - stride=s, - filter_size=3, - padding=1, - expansion_factor=t) + self._first_block = InvertedResidualUnit( + num_channels=in_c, + num_in_filter=in_c, + num_filters=c, + stride=s, + filter_size=3, + padding=1, + expansion_factor=t, + ) self._inv_blocks = [] for i in range(1, n): - tmp = self.add_sublayer(sublayer=InvertedResidualUnit( - num_channels=c, - num_in_filter=c, - num_filters=c, - stride=1, - filter_size=3, - padding=1, - expansion_factor=t), - name=self.full_name() + "_" + str(i + 1)) + tmp = self.add_sublayer( + sublayer=InvertedResidualUnit( + num_channels=c, + num_in_filter=c, + num_filters=c, + stride=1, + filter_size=3, + padding=1, + expansion_factor=t, + ), + name=self.full_name() + "_" + str(i + 1), + ) self._inv_blocks.append(tmp) def forward(self, inputs): @@ -323,7 +369,6 @@ def forward(self, inputs): class MobileNetV2(fluid.dygraph.Layer): - def __init__(self, class_dim=1000, scale=1.0): super(MobileNetV2, self).__init__() self.scale = scale @@ -339,48 +384,54 @@ def __init__(self, class_dim=1000, scale=1.0): (6, 320, 1, 1), ] - #1. conv1 - self._conv1 = ConvBNLayer(num_channels=3, - num_filters=int(32 * scale), - filter_size=3, - stride=2, - act=None, - padding=1) - - #2. bottleneck sequences + # 1. conv1 + self._conv1 = ConvBNLayer( + num_channels=3, + num_filters=int(32 * scale), + filter_size=3, + stride=2, + act=None, + padding=1, + ) + + # 2. bottleneck sequences self._invl = [] i = 1 in_c = int(32 * scale) for layer_setting in bottleneck_params_list: t, c, n, s = layer_setting i += 1 - tmp = self.add_sublayer(sublayer=InvresiBlocks(in_c=in_c, - t=t, - c=int(c * scale), - n=n, - s=s), - name='conv' + str(i)) + tmp = self.add_sublayer( + sublayer=InvresiBlocks( + in_c=in_c, t=t, c=int(c * scale), n=n, s=s + ), + name='conv' + str(i), + ) self._invl.append(tmp) in_c = int(c * scale) - #3. last_conv + # 3. last_conv self._out_c = int(1280 * scale) if scale > 1.0 else 1280 - self._conv9 = ConvBNLayer(num_channels=in_c, - num_filters=self._out_c, - filter_size=1, - stride=1, - act=None, - padding=0) - - #4. pool + self._conv9 = ConvBNLayer( + num_channels=in_c, + num_filters=self._out_c, + filter_size=1, + stride=1, + act=None, + padding=0, + ) + + # 4. pool self._pool2d_avg = Pool2D(pool_type='avg', global_pooling=True) - #5. fc + # 5. fc tmp_param = ParamAttr(name=self.full_name() + "fc10_weights") - self._fc = Linear(self._out_c, - class_dim, - param_attr=tmp_param, - bias_attr=ParamAttr(name="fc10_offset")) + self._fc = Linear( + self._out_c, + class_dim, + param_attr=tmp_param, + bias_attr=ParamAttr(name="fc10_offset"), + ) @declarative def forward(self, inputs): @@ -399,7 +450,8 @@ def create_optimizer(args, parameter_list): learning_rate=args.lr, momentum=args.momentum_rate, regularization=fluid.regularizer.L2Decay(args.l2_decay), - parameter_list=parameter_list) + parameter_list=parameter_list, + ) return optimizer @@ -430,8 +482,11 @@ class Args(object): class_dim = 50 print_step = 1 train_step = 10 - place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) model_save_dir = None model_save_prefix = None model_filename = None @@ -477,8 +532,9 @@ def train_mobilenet(args, to_static): t_end = time.time() softmax_out = fluid.layers.softmax(out, use_cudnn=False) - loss = fluid.layers.cross_entropy(input=softmax_out, - label=label) + loss = fluid.layers.cross_entropy( + input=softmax_out, label=label + ) avg_loss = paddle.mean(x=loss) acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) @@ -493,17 +549,29 @@ def train_mobilenet(args, to_static): t2 = time.time() train_batch_elapse = t2 - t1 if batch_id % args.print_step == 0: - print("epoch id: %d, batch step: %d, avg_loss %0.5f acc_top1 %0.5f acc_top5 %0.5f %2.4f sec net_t:%2.4f back_t:%2.4f read_t:%2.4f" % \ - (eop, batch_id, avg_loss.numpy(), acc_top1.numpy(), acc_top5.numpy(), train_batch_elapse, - t_end - t_start, t_end_back - t_start_back, t1 - t_last)) + print( + "epoch id: %d, batch step: %d, avg_loss %0.5f acc_top1 %0.5f acc_top5 %0.5f %2.4f sec net_t:%2.4f back_t:%2.4f read_t:%2.4f" + % ( + eop, + batch_id, + avg_loss.numpy(), + acc_top1.numpy(), + acc_top5.numpy(), + train_batch_elapse, + t_end - t_start, + t_end_back - t_start_back, + t1 - t_last, + ) + ) batch_id += 1 t_last = time.time() if batch_id > args.train_step: if to_static: fluid.dygraph.jit.save(net, args.model_save_prefix) else: - fluid.dygraph.save_dygraph(net.state_dict(), - args.dy_state_dict_save_path) + fluid.dygraph.save_dygraph( + net.state_dict(), args.dy_state_dict_save_path + ) break return np.array(loss_data) @@ -514,15 +582,22 @@ def predict_static(args, data): exe = fluid.Executor(args.place) # load inference model - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(args.model_save_dir, - executor=exe, - model_filename=args.model_filename, - params_filename=args.params_filename) - - pred_res = exe.run(inference_program, - feed={feed_target_names[0]: data}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + args.model_save_dir, + executor=exe, + model_filename=args.model_filename, + params_filename=args.params_filename, + ) + + pred_res = exe.run( + inference_program, + feed={feed_target_names[0]: data}, + fetch_list=fetch_targets, + ) return pred_res[0] @@ -554,51 +629,57 @@ def predict_dygraph_jit(args, data): def predict_analysis_inference(args, data): - output = PredictorTools(args.model_save_dir, args.model_filename, - args.params_filename, [data]) - out, = output() + output = PredictorTools( + args.model_save_dir, args.model_filename, args.params_filename, [data] + ) + (out,) = output() return out class TestMobileNet(unittest.TestCase): - def setUp(self): self.args = Args() self.temp_dir = tempfile.TemporaryDirectory() - self.args.model_save_dir = os.path.join(self.temp_dir.name, - "./inference") + self.args.model_save_dir = os.path.join( + self.temp_dir.name, "./inference" + ) def tearDown(self): self.temp_dir.cleanup() def train(self, model_name, to_static): self.args.model = model_name - self.args.model_save_prefix = os.path.join(self.temp_dir.name, - "./inference/" + model_name) + self.args.model_save_prefix = os.path.join( + self.temp_dir.name, "./inference/" + model_name + ) self.args.model_filename = model_name + INFER_MODEL_SUFFIX self.args.params_filename = model_name + INFER_PARAMS_SUFFIX self.args.dy_state_dict_save_path = os.path.join( - self.temp_dir.name, model_name + ".dygraph") + self.temp_dir.name, model_name + ".dygraph" + ) out = train_mobilenet(self.args, to_static) return out def assert_same_loss(self, model_name): dy_out = self.train(model_name, to_static=False) st_out = self.train(model_name, to_static=True) - np.testing.assert_allclose(dy_out, - st_out, - rtol=1e-05, - err_msg='dy_out: {}, st_out: {}'.format( - dy_out, st_out)) + np.testing.assert_allclose( + dy_out, + st_out, + rtol=1e-05, + err_msg='dy_out: {}, st_out: {}'.format(dy_out, st_out), + ) def assert_same_predict(self, model_name): self.args.model = model_name - self.args.model_save_prefix = os.path.join(self.temp_dir.name, - "./inference/" + model_name) + self.args.model_save_prefix = os.path.join( + self.temp_dir.name, "./inference/" + model_name + ) self.args.model_filename = model_name + INFER_MODEL_SUFFIX self.args.params_filename = model_name + INFER_PARAMS_SUFFIX self.args.dy_state_dict_save_path = os.path.join( - self.temp_dir.name, model_name + ".dygraph") + self.temp_dir.name, model_name + ".dygraph" + ) local_random = np.random.RandomState(SEED) image = local_random.random_sample([1, 3, 224, 224]).astype('float32') dy_pre = predict_dygraph(self.args, image) @@ -609,20 +690,25 @@ def assert_same_predict(self, model_name): dy_pre, st_pre, rtol=1e-05, - err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre)) + err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre), + ) np.testing.assert_allclose( dy_jit_pre, st_pre, rtol=1e-05, err_msg='dy_jit_pre:\n {}\n, st_pre: \n{}.'.format( - dy_jit_pre, st_pre)) + dy_jit_pre, st_pre + ), + ) np.testing.assert_allclose( predictor_pre, st_pre, rtol=1e-05, atol=1e-05, err_msg='inference_pred_res:\n {}\n, st_pre: \n{}.'.format( - predictor_pre, st_pre)) + predictor_pre, st_pre + ), + ) def test_mobile_net(self): # MobileNet-V1 diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_multi_forward.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_multi_forward.py index 3b81cd0749fc6..f320e9f010cfc 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_multi_forward.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_multi_forward.py @@ -18,20 +18,20 @@ class MyLayer(paddle.nn.Layer): - def __init__(self): super().__init__() self.linear = paddle.nn.Linear(1, 1) - @paddle.jit.to_static(input_spec=[ - paddle.static.InputSpec(shape=[None, None], dtype=paddle.float32) - ]) + @paddle.jit.to_static( + input_spec=[ + paddle.static.InputSpec(shape=[None, None], dtype=paddle.float32) + ] + ) def forward(self, x): return self.linear(x) class TestBackward(unittest.TestCase): - def test_order_0(self): """ loss = 1 * w * 1 + 2 * w * 2 diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_op_attr.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_op_attr.py index 9bdc6cfcb2c85..8bfc7a83dfec8 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_op_attr.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_op_attr.py @@ -19,7 +19,6 @@ class MySub(paddle.nn.Layer): - def __init__(self): super(MySub, self).__init__() @@ -28,7 +27,6 @@ def forward(self, x, y, name=None): class NetWithOpAttr(paddle.nn.Layer): - def __init__(self, in_num, out_num): super(NetWithOpAttr, self).__init__() @@ -44,7 +42,7 @@ def forward(self, x): @paddle.jit.to_static(input_spec=[InputSpec([10, 16])]) def with_cond(self, x): - if paddle.mean(x) > 0.: + if paddle.mean(x) > 0.0: out = self.linear(x) else: out = self.sub(x, x) @@ -53,7 +51,6 @@ def with_cond(self, x): class CheckOpAttr(unittest.TestCase): - def setUp(self): self.in_num = 16 self.out_num = 16 @@ -65,7 +62,7 @@ def expected_results(self): "int_val": 10, "int_vals": [10, 20], "float_val": 3.8, - "float_vals": [3.8, -0.2] + "float_vals": [3.8, -0.2], } self.bn_attrs = {"bool_val": True, "bool_vals": [True, False]} self.sub_attrs = {"int_vals": [10, 20], "bool_vals": [True, False]} @@ -75,7 +72,7 @@ def expected_results(self): 'elementwise_add': self.fc_attrs, 'batch_norm': self.bn_attrs, 'tanh': self.bn_attrs, - 'elementwise_sub': self.sub_attrs + 'elementwise_sub': self.sub_attrs, } def test_set_op_attrs(self): @@ -89,8 +86,9 @@ def test_set_op_attrs(self): self.assertEqual(len(net.linear._forward_pre_hooks), 1) self.assertEqual(len(net.linear._forward_post_hooks), 1) # to_static - net = paddle.jit.to_static(net, - input_spec=[InputSpec.from_tensor(self.x)]) + net = paddle.jit.to_static( + net, input_spec=[InputSpec.from_tensor(self.x)] + ) # assert attrs have be set. self.check_op_attrs(net.forward.concrete_program.main_program) @@ -103,7 +101,8 @@ def check_op_attrs(self, main_program): for cur_block in main_program.blocks: ops = cur_block.ops for op in ops: - if op.type not in self.infos: continue + if op.type not in self.infos: + continue for attr_name, expect_vals in self.infos[op.type].items(): op_vals = op.desc.attr(attr_name) if not isinstance(expect_vals, list): @@ -120,8 +119,9 @@ def check_op_attrs(self, main_program): def test_set_op_attrs_with_sub_block(self): net = NetWithOpAttr(self.in_num, self.out_num) # set attrs - net.linear._set_op_attrs({"int_vals": [0, - 0]}) # test overwrite behavior + net.linear._set_op_attrs( + {"int_vals": [0, 0]} + ) # test overwrite behavior net.linear._set_op_attrs(self.fc_attrs) net.bn._set_op_attrs(self.bn_attrs) net.sub._set_op_attrs(self.sub_attrs) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_param_guard.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_param_guard.py index 8247787b51572..90444ede53c54 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_param_guard.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_param_guard.py @@ -20,7 +20,6 @@ class NetWithParameterList(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(NetWithParameterList, self).__init__() weight = self.create_parameter([in_size, out_size]) @@ -36,7 +35,6 @@ def forward(self, x): class NetWithParameterListIter(NetWithParameterList): - def __init__(self, in_size, out_size): super(NetWithParameterListIter, self).__init__(in_size, out_size) @@ -51,7 +49,6 @@ def forward(self, x): class TestParameterList(unittest.TestCase): - def setUp(self): self.seed = 2021 self.iter_num = 5 @@ -89,13 +86,14 @@ def test_parameter_list_iter(self): class NetWithRawParamList(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(NetWithRawParamList, self).__init__() - weight = self.add_parameter('w', - self.create_parameter([in_size, out_size])) + weight = self.add_parameter( + 'w', self.create_parameter([in_size, out_size]) + ) bias = self.add_parameter( - 'b', self.create_parameter([out_size], is_bias=True)) + 'b', self.create_parameter([out_size], is_bias=True) + ) self.params = [weight] self.bias_dict = {'b': bias} @@ -108,7 +106,6 @@ def forward(self, x): class TestRawParameterList(unittest.TestCase): - def setUp(self): self.seed = 2021 self.iter_num = 5 @@ -142,7 +139,6 @@ def test_parameter_list(self): class NetWithSubLayerParamList(paddle.nn.Layer): - def __init__(self, sub_layer): super(NetWithSubLayerParamList, self).__init__() self.sub_layer = sub_layer @@ -158,7 +154,6 @@ def forward(self, x): class TestSubLayerParameterList(TestRawParameterList): - def init_net(self): fc = paddle.nn.Linear(10, 3) self.net = NetWithSubLayerParamList(fc) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_params_no_grad.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_params_no_grad.py index f44faa642a00e..24ff71b066d1e 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_params_no_grad.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_params_no_grad.py @@ -20,7 +20,6 @@ class Net(nn.Layer): - def __init__(self): super(Net, self).__init__() self.emb1 = nn.Embedding(100, 16) @@ -54,10 +53,11 @@ def train(): class TestParamsNoGrad(unittest.TestCase): - def test_two_card(self): - if paddle.is_compiled_with_cuda() and len( - paddle.static.cuda_places()) > 1: + if ( + paddle.is_compiled_with_cuda() + and len(paddle.static.cuda_places()) > 1 + ): dist.spawn(train, nprocs=2, gpus='0,1') diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py index 72dca36d2101a..d11f4e9f15c6f 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py @@ -54,7 +54,6 @@ def fake_data(shape): class TestWithNestedInput(unittest.TestCase): - def setUp(self): self.x = None self.y = None @@ -62,15 +61,14 @@ def setUp(self): def fake_input(self): self.x = fake_data([10, 16]) self.y = [ - fake_data([10, 16]), "preprocess_cmd", 64, { - 'z': [fake_data([10, 12]), - fake_data([10, 12])], + fake_data([10, 16]), + "preprocess_cmd", + 64, + { + 'z': [fake_data([10, 12]), fake_data([10, 12])], 'c': fake_data([10, 10]), - 'd': { - 'da': 12, - 'dc': fake_data([10, 10]) - } - } + 'd': {'da': 12, 'dc': fake_data([10, 10])}, + }, ] def _run(self, to_static): @@ -92,7 +90,6 @@ def test_nest(self): class TestWithNestedOutput(unittest.TestCase): - def setUp(self): self.x = None self.y = None @@ -120,17 +117,17 @@ def test_nest(self): self.assertTrue(len(dygraph_res) == len(static_res)) for dy_var, st_var in zip(dygraph_res, static_res): - if isinstance(dy_var, - (fluid.core.VarBase, fluid.core.eager.Tensor)): - np.testing.assert_allclose(dy_var.numpy(), - st_var.numpy(), - rtol=1e-05) + if isinstance( + dy_var, (fluid.core.VarBase, fluid.core.eager.Tensor) + ): + np.testing.assert_allclose( + dy_var.numpy(), st_var.numpy(), rtol=1e-05 + ) else: self.assertTrue(dy_var, st_var) class TestWithTrainAndEval(unittest.TestCase): - def test_switch_eval_and_train(self): program_translator = ProgramTranslator() @@ -142,25 +139,27 @@ def test_switch_eval_and_train(self): _, train_partial_layer = linear_net.forward.program_cache.last()[-1] # check default mode is for training - self.assertEqual(train_partial_layer.program, - train_partial_layer._train_program) + self.assertEqual( + train_partial_layer.program, train_partial_layer._train_program + ) # switch to run test program after `eval()` linear_net.eval() linear_net(x) _, eval_partial_layer = linear_net.forward.program_cache.last()[-1] - self.assertEqual(eval_partial_layer.program, - eval_partial_layer._infer_program) + self.assertEqual( + eval_partial_layer.program, eval_partial_layer._infer_program + ) # switch back into training linear_net.train() linear_net(x) - self.assertEqual(train_partial_layer.program, - train_partial_layer._train_program) + self.assertEqual( + train_partial_layer.program, train_partial_layer._train_program + ) class TestWithNoGrad(unittest.TestCase): - def test_with_no_grad(self): with fluid.dygraph.guard(): linear_net = Linear() @@ -171,18 +170,19 @@ def test_with_no_grad(self): linear_net.train() linear_net(x) _, partial_layer = linear_net.forward.program_cache.last()[-1] - self.assertEqual(partial_layer.program, - partial_layer._train_program) + self.assertEqual( + partial_layer.program, partial_layer._train_program + ) class GPT2LMHeadModel(fluid.dygraph.Layer): - def __init__(self): super(GPT2LMHeadModel, self).__init__() self.embedding0 = paddle.nn.Embedding(20, 16) self.embedding1 = paddle.nn.Embedding(20, 32) self.lm_head_weight = paddle.to_tensor( - np.random.rand(2, 3).astype('float32')) + np.random.rand(2, 3).astype('float32') + ) @declarative def forward(self, x): @@ -192,7 +192,6 @@ def forward(self, x): class TestPruneUnusedParamInProgram(unittest.TestCase): - def test_prune(self): input_ids = np.array([[15, 11, 6, 3, 18, 13]]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_place.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_place.py index 96cffc32a19f6..58e8e7b6728f5 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_place.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_place.py @@ -17,7 +17,6 @@ class TestPlace(unittest.TestCase): - def test_place(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_print.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_print.py index 90737695aaff3..3603cac199ed5 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_print.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_print.py @@ -152,11 +152,13 @@ def dyfunc_print_continue_vars(x): class TestPrintBase(unittest.TestCase): - def setUp(self): self.input = numpy.ones(5).astype("int32") - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.set_test_func() def set_test_func(self): @@ -176,7 +178,6 @@ def get_static_output(self): class TestPrintVariable(TestPrintBase): - def set_test_func(self): self.dygraph_func = dyfunc_print_variable @@ -186,37 +187,31 @@ def test_transformed_static_result(self): class TestPrintNdArray(TestPrintVariable): - def set_test_func(self): self.dygraph_func = dyfunc_print_ndarray class TestPrintWithFormat(TestPrintVariable): - def set_test_func(self): self.dygraph_func = dyfunc_print_with_format class TestPrintWithFormat2(TestPrintVariable): - def set_test_func(self): self.dygraph_func = dyfunc_print_with_format2 class TestPrintWithIfElse(TestPrintVariable): - def set_test_func(self): self.dygraph_func = dyfunc_print_with_ifelse class TestPrintMultipleVar(TestPrintVariable): - def set_test_func(self): self.dygraph_func = dyfunc_print_multi_vars class TestPrintContinueVar(TestPrintVariable): - def set_test_func(self): self.dygraph_func = dyfunc_print_continue_vars diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py index 7bd415fdc8a7b..762836a937be7 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_program_translator.py @@ -26,7 +26,11 @@ from paddle.fluid.dygraph.dygraph_to_static.utils import func_to_source_code import paddle.jit.dy2static as _jst -from ifelse_simple_func import dyfunc_with_if_else, dyfunc_with_if_else_early_return1, dyfunc_with_if_else_early_return2 +from ifelse_simple_func import ( + dyfunc_with_if_else, + dyfunc_with_if_else_early_return1, + dyfunc_with_if_else_early_return2, +) np.random.seed(0) @@ -60,8 +64,7 @@ def get_source_code(func): return source_code -class StaticCode1(): - +class StaticCode1: def dyfunc_with_if_else(x_v, label=None): loss = _jst.UndefinedVar('loss') __return_1 = _jst.UndefinedVar('__return_1') @@ -70,11 +73,11 @@ def dyfunc_with_if_else(x_v, label=None): def get_args_0(): nonlocal x_v - return x_v, + return (x_v,) def set_args_0(__args): nonlocal x_v - x_v, = __args + (x_v,) = __args def true_fn_0(): nonlocal x_v @@ -86,12 +89,15 @@ def false_fn_0(): x_v = x_v + 1 return - _jst.IfElse(paddle.mean(x_v)[0] > 5, - true_fn_0, - false_fn_0, - get_args_0, - set_args_0, ('x_v', ), - push_pop_names=None) + _jst.IfElse( + paddle.mean(x_v)[0] > 5, + true_fn_0, + false_fn_0, + get_args_0, + set_args_0, + ('x_v',), + push_pop_names=None, + ) def get_args_1(): nonlocal __return_0, __return_1, __return_value_0, loss @@ -114,17 +120,19 @@ def false_fn_1(): __return_value_0 = x_v return - _jst.IfElse(label is not None, - true_fn_1, - false_fn_1, - get_args_1, - set_args_1, - ('__return_0', '__return_1', '__return_value_0', 'loss'), - push_pop_names=None) + _jst.IfElse( + label is not None, + true_fn_1, + false_fn_1, + get_args_1, + set_args_1, + ('__return_0', '__return_1', '__return_value_0', 'loss'), + push_pop_names=None, + ) return __return_value_0 -class StaticCode2(): +class StaticCode2: # TODO: Transform return statement def dyfunc_with_if_else(x_v, label=None): loss = _jst.UndefinedVar('loss') @@ -134,11 +142,11 @@ def dyfunc_with_if_else(x_v, label=None): def get_args_2(): nonlocal x_v - return x_v, + return (x_v,) def set_args_2(__args): nonlocal x_v - x_v, = __args + (x_v,) = __args def true_fn_2(): nonlocal x_v @@ -150,12 +158,15 @@ def false_fn_2(): x_v = x_v + 1 return - _jst.IfElse(paddle.mean(x_v)[0] > 5, - true_fn_2, - false_fn_2, - get_args_2, - set_args_2, ('x_v', ), - push_pop_names=None) + _jst.IfElse( + paddle.mean(x_v)[0] > 5, + true_fn_2, + false_fn_2, + get_args_2, + set_args_2, + ('x_v',), + push_pop_names=None, + ) def get_args_3(): nonlocal __return_2, __return_3, __return_value_1, loss @@ -178,18 +189,19 @@ def false_fn_3(): __return_value_1 = x_v return - _jst.IfElse(label is not None, - true_fn_3, - false_fn_3, - get_args_3, - set_args_3, - ('__return_2', '__return_3', '__return_value_1', 'loss'), - push_pop_names=None) + _jst.IfElse( + label is not None, + true_fn_3, + false_fn_3, + get_args_3, + set_args_3, + ('__return_2', '__return_3', '__return_value_1', 'loss'), + push_pop_names=None, + ) return __return_value_1 class NetWithError(fluid.dygraph.layers.Layer): - @declarative def forward(self, x): linear = fluid.dygraph.Linear(32, 64) @@ -198,7 +210,6 @@ def forward(self, x): class TestDygraphToStaticCode(unittest.TestCase): - def setUp(self): # set to print all string diff when assertEqual fails self.maxDiff = None @@ -210,7 +221,8 @@ def test_decorator(self): answer = get_source_code(StaticCode1.dyfunc_with_if_else) self.assertEqual( answer.replace('\n', '').replace(' ', ''), - code.replace('\n', '').replace(' ', '')) + code.replace('\n', '').replace(' ', ''), + ) def test_program_translator(self): answer = get_source_code(StaticCode2.dyfunc_with_if_else) @@ -219,11 +231,11 @@ def test_program_translator(self): print(code) self.assertEqual( answer.replace('\n', '').replace(' ', ''), - code.replace('\n', '').replace(' ', '')) + code.replace('\n', '').replace(' ', ''), + ) class TestEnableDeclarative(unittest.TestCase): - def setUp(self): self.x = np.random.randn(30, 10, 32).astype('float32') self.weight = np.random.randn(32, 64).astype('float32') @@ -240,16 +252,20 @@ def test_enable_disable_get_output(self): self.program_translator.enable(True) with fluid.dygraph.guard(): static_output = self.program_translator.get_output( - simple_func, self.x, self.weight) + simple_func, self.x, self.weight + ) self.program_translator.enable(False) with fluid.dygraph.guard(): dygraph_output = self.program_translator.get_output( - simple_func, self.x, self.weight) - np.testing.assert_allclose(static_output.numpy(), - dygraph_output.numpy(), - rtol=1e-05, - atol=1e-4) + simple_func, self.x, self.weight + ) + np.testing.assert_allclose( + static_output.numpy(), + dygraph_output.numpy(), + rtol=1e-05, + atol=1e-4, + ) def test_enable_disable_get_func(self): @@ -266,14 +282,18 @@ def test_enable_disable_get_func(self): self.assertTrue(callable(dygraph_func)) dygraph_output = dygraph_func(self.x, self.weight) self.assertTrue( - isinstance(dygraph_output, - (fluid.core.VarBase, fluid.core.eager.Tensor))) + isinstance( + dygraph_output, + (fluid.core.VarBase, fluid.core.eager.Tensor), + ) + ) def test_enable_disable_get_program(self): self.program_translator.enable(True) static_output = self.program_translator.get_program( - simple_func, self.x, self.weight) + simple_func, self.x, self.weight + ) self.assertTrue(isinstance(static_output, tuple)) self.assertEqual(len(static_output), 4) self.assertTrue(isinstance(static_output[0], fluid.Program)) @@ -288,10 +308,14 @@ def test_enable_disable_get_program(self): self.program_translator.enable(False) with fluid.dygraph.guard(): dygraph_output = self.program_translator.get_program( - simple_func, self.x, self.weight) + simple_func, self.x, self.weight + ) self.assertTrue( - isinstance(dygraph_output, - (fluid.core.VarBase, fluid.core.eager.Tensor))) + isinstance( + dygraph_output, + (fluid.core.VarBase, fluid.core.eager.Tensor), + ) + ) def test_enable_disable_declarative(self): @@ -302,14 +326,15 @@ def test_enable_disable_declarative(self): self.program_translator.enable(False) with fluid.dygraph.guard(): dygraph_output = decorated_simple_func(self.x, self.weight) - np.testing.assert_allclose(static_output.numpy(), - dygraph_output.numpy(), - rtol=1e-05, - atol=1e-4) + np.testing.assert_allclose( + static_output.numpy(), + dygraph_output.numpy(), + rtol=1e-05, + atol=1e-4, + ) class Net(fluid.dygraph.layers.Layer): - def __init__(self): super(Net, self).__init__() @@ -318,7 +343,6 @@ def forward(self, x): class TestErrorWithInitFromStaticMode(unittest.TestCase): - def setUp(self): self.program_translator = ProgramTranslator() self.x = np.random.randn(10, 32).astype('float32') @@ -329,17 +353,18 @@ def test_raise_error(self): net = Net() self.program_translator.enable(True) - with self.assertRaisesRegexp(RuntimeError, - "only available in dynamic mode"): + with self.assertRaisesRegexp( + RuntimeError, "only available in dynamic mode" + ): self.program_translator.get_output(net.forward, self.x) - with self.assertRaisesRegexp(RuntimeError, - "only available in dynamic mode"): + with self.assertRaisesRegexp( + RuntimeError, "only available in dynamic mode" + ): self.program_translator.get_program(net.forward, self.x) class SwitchModeNet(paddle.nn.Layer): - def __init__(self): super(SwitchModeNet, self).__init__() @@ -358,7 +383,6 @@ def switch_mode_funciton(): class TestFunctionTrainEvalMode(unittest.TestCase): - def test_switch_mode(self): paddle.disable_static() switch_mode_funciton.eval() @@ -388,7 +412,6 @@ def test_raise_error(self): class TestIfElseEarlyReturn(unittest.TestCase): - def test_ifelse_early_return1(self): answer = np.zeros([2, 2]) + 1 static_func = paddle.jit.to_static(dyfunc_with_if_else_early_return1) @@ -403,7 +426,6 @@ def test_ifelse_early_return2(self): class TestRemoveCommentInDy2St(unittest.TestCase): - def func_with_comment(self): # Comment1 x = paddle.to_tensor([1, 2, 3]) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm.py index b37d4af6b39e9..12150a3d5a038 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm.py @@ -32,13 +32,9 @@ class SimpleLSTMRNN(fluid.Layer): - - def __init__(self, - hidden_size, - num_steps, - num_layers=2, - init_scale=0.1, - dropout=None): + def __init__( + self, hidden_size, num_steps, num_layers=2, init_scale=0.1, dropout=None + ): super(SimpleLSTMRNN, self).__init__() self._hidden_size = hidden_size self._num_layers = num_layers @@ -57,19 +53,26 @@ def __init__(self, weight_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 2, self._hidden_size * 4], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)) + low=-self._init_scale, high=self._init_scale + ), + ) self.weight_1_arr.append(self.add_parameter('w_%d' % i, weight_1)) bias_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 4], dtype="float32", - default_initializer=fluid.initializer.Constant(0.0)) + default_initializer=fluid.initializer.Constant(0.0), + ) self.bias_arr.append(self.add_parameter('b_%d' % i, bias_1)) def forward(self, input_embedding, init_hidden=None, init_cell=None): @@ -93,11 +96,12 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): gate_input = fluid.layers.matmul(x=nn, y=weight_1) gate_input = fluid.layers.elementwise_add(gate_input, bias) - i, j, f, o = fluid.layers.split(gate_input, - num_or_sections=4, - dim=-1) + i, j, f, o = fluid.layers.split( + gate_input, num_or_sections=4, dim=-1 + ) c = pre_cell * fluid.layers.sigmoid(f) + fluid.layers.sigmoid( - i) * fluid.layers.tanh(j) + i + ) * fluid.layers.tanh(j) m = fluid.layers.tanh(c) * fluid.layers.sigmoid(o) hidden_array[k] = m cell_array[k] = c @@ -107,31 +111,36 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): step_input = fluid.layers.dropout( step_input, dropout_prob=self._dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) res.append(step_input) real_res = fluid.layers.concat(res, 1) real_res = fluid.layers.reshape( - real_res, [-1, self._num_steps, self._hidden_size]) + real_res, [-1, self._num_steps, self._hidden_size] + ) last_hidden = fluid.layers.concat(hidden_array, 1) last_hidden = fluid.layers.reshape( - last_hidden, shape=[-1, self._num_layers, self._hidden_size]) + last_hidden, shape=[-1, self._num_layers, self._hidden_size] + ) last_hidden = fluid.layers.transpose(x=last_hidden, perm=[1, 0, 2]) last_cell = fluid.layers.concat(cell_array, 1) last_cell = fluid.layers.reshape( - last_cell, shape=[-1, self._num_layers, self._hidden_size]) + last_cell, shape=[-1, self._num_layers, self._hidden_size] + ) last_cell = fluid.layers.transpose(x=last_cell, perm=[1, 0, 2]) return real_res, last_hidden, last_cell class PtbModel(fluid.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_layers=2, - num_steps=20, - init_scale=0.1, - dropout=None): + def __init__( + self, + hidden_size, + vocab_size, + num_layers=2, + num_steps=20, + init_scale=0.1, + dropout=None, + ): super(PtbModel, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -139,11 +148,13 @@ def __init__(self, self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout - self.simple_lstm_rnn = SimpleLSTMRNN(hidden_size, - num_steps, - num_layers=num_layers, - init_scale=init_scale, - dropout=dropout) + self.simple_lstm_rnn = SimpleLSTMRNN( + hidden_size, + num_steps, + num_layers=num_layers, + init_scale=init_scale, + dropout=dropout, + ) self.embedding = Embedding( size=[vocab_size, hidden_size], dtype='float32', @@ -151,19 +162,26 @@ def __init__(self, param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def build_once(self, input, label, init_hidden, init_cell): pass @@ -172,29 +190,34 @@ def build_once(self, input, label, init_hidden, init_cell): def forward(self, input, label, init_hidden, init_cell): init_h = fluid.layers.reshape( - init_hidden, shape=[self.num_layers, -1, self.hidden_size]) + init_hidden, shape=[self.num_layers, -1, self.hidden_size] + ) init_c = fluid.layers.reshape( - init_cell, shape=[self.num_layers, -1, self.hidden_size]) + init_cell, shape=[self.num_layers, -1, self.hidden_size] + ) x_emb = self.embedding(input) x_emb = fluid.layers.reshape( - x_emb, shape=[-1, self.num_steps, self.hidden_size]) + x_emb, shape=[-1, self.num_steps, self.hidden_size] + ) if self.dropout is not None and self.dropout > 0.0: x_emb = fluid.layers.dropout( x_emb, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn_out, last_hidden, last_cell = self.simple_lstm_rnn( - x_emb, init_h, init_c) + x_emb, init_h, init_c + ) projection = fluid.layers.matmul(rnn_out, self.softmax_weight) projection = fluid.layers.elementwise_add(projection, self.softmax_bias) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -221,15 +244,18 @@ def train(place): with fluid.dygraph.guard(place): paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale, - dropout=dropout) - - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=ptb_model.parameters()) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + dropout=dropout, + ) + + sgd = SGDOptimizer( + learning_rate=1e-3, parameter_list=ptb_model.parameters() + ) for epoch_id in range(max_epoch): @@ -237,10 +263,12 @@ def train(place): iters = 0.0 total_sample = 0 - init_hidden_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + init_hidden_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) @@ -256,7 +284,8 @@ def train(place): y = to_variable(y_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) out_loss = dy_loss.numpy() dy_loss.backward() @@ -269,15 +298,21 @@ def train(place): if step_id % PRINT_STEP == 0: if step_id == 0: logging.info( - "epoch %d | step %d, loss %0.3f" % - (epoch_id, step_id, total_loss / total_sample)) + "epoch %d | step %d, loss %0.3f" + % (epoch_id, step_id, total_loss / total_sample) + ) avg_batch_time = time.time() else: speed = PRINT_STEP / (time.time() - avg_batch_time) logging.info( "epoch %d | step %d, loss %0.3f, speed %.3f steps/s" - % (epoch_id, step_id, total_loss / total_sample, - speed)) + % ( + epoch_id, + step_id, + total_loss / total_sample, + speed, + ) + ) avg_batch_time = time.time() return out_loss, last_hidden.numpy(), last_cell.numpy() @@ -294,10 +329,12 @@ def train_static(place): class TestPtb(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + ) def test_check_result(self): loss_1, hidden_1, cell_1 = train_static(self.place) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm_v2.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm_v2.py index ac3c40b599afa..56b0a44bdb538 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm_v2.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_ptb_lm_v2.py @@ -26,13 +26,9 @@ class SimpleLSTMRNN(paddle.nn.Layer): - - def __init__(self, - hidden_size, - num_steps, - num_layers=2, - init_scale=0.1, - dropout=None): + def __init__( + self, hidden_size, num_steps, num_layers=2, init_scale=0.1, dropout=None + ): super(SimpleLSTMRNN, self).__init__() self._hidden_size = hidden_size self._num_layers = num_layers @@ -49,19 +45,28 @@ def __init__(self, for i in range(self._num_layers): weight_1 = self.create_parameter( - attr=paddle.ParamAttr(initializer=paddle.nn.initializer.Uniform( - low=-self._init_scale, high=self._init_scale)), + attr=paddle.ParamAttr( + initializer=paddle.nn.initializer.Uniform( + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 2, self._hidden_size * 4], dtype="float32", default_initializer=paddle.nn.initializer.Uniform( - low=-self._init_scale, high=self._init_scale)) + low=-self._init_scale, high=self._init_scale + ), + ) self.weight_1_arr.append(self.add_parameter('w_%d' % i, weight_1)) bias_1 = self.create_parameter( - attr=paddle.ParamAttr(initializer=paddle.nn.initializer.Uniform( - low=-self._init_scale, high=self._init_scale)), + attr=paddle.ParamAttr( + initializer=paddle.nn.initializer.Uniform( + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 4], dtype="float32", - default_initializer=paddle.nn.initializer.Constant(0.0)) + default_initializer=paddle.nn.initializer.Constant(0.0), + ) self.bias_arr.append(self.add_parameter('b_%d' % i, bias_1)) def forward(self, input_embedding, init_hidden=None, init_cell=None): @@ -85,11 +90,12 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): gate_input = paddle.matmul(x=nn, y=weight_1) gate_input = paddle.add(x=gate_input, y=bias) - i, j, f, o = paddle.split(x=gate_input, - num_or_sections=4, - axis=-1) + i, j, f, o = paddle.split( + x=gate_input, num_or_sections=4, axis=-1 + ) c = pre_cell * paddle.nn.functional.sigmoid( - f) + paddle.nn.functional.sigmoid(i) * paddle.tanh(j) + f + ) + paddle.nn.functional.sigmoid(i) * paddle.tanh(j) m = paddle.tanh(c) * paddle.nn.functional.sigmoid(o) hidden_array[k] = m cell_array[k] = c @@ -99,31 +105,36 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): step_input = paddle.nn.functional.dropout( step_input, dropout_prob=self._dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) res.append(step_input) real_res = paddle.concat(x=res, axis=1) - real_res = paddle.reshape(real_res, - [-1, self._num_steps, self._hidden_size]) + real_res = paddle.reshape( + real_res, [-1, self._num_steps, self._hidden_size] + ) last_hidden = paddle.concat(x=hidden_array, axis=1) last_hidden = paddle.reshape( - last_hidden, shape=[-1, self._num_layers, self._hidden_size]) + last_hidden, shape=[-1, self._num_layers, self._hidden_size] + ) last_hidden = paddle.transpose(x=last_hidden, perm=[1, 0, 2]) last_cell = paddle.concat(x=cell_array, axis=1) last_cell = paddle.reshape( - last_cell, shape=[-1, self._num_layers, self._hidden_size]) + last_cell, shape=[-1, self._num_layers, self._hidden_size] + ) last_cell = paddle.transpose(x=last_cell, perm=[1, 0, 2]) return real_res, last_hidden, last_cell class PtbModel(paddle.nn.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_layers=2, - num_steps=20, - init_scale=0.1, - dropout=None): + def __init__( + self, + hidden_size, + vocab_size, + num_layers=2, + num_steps=20, + init_scale=0.1, + dropout=None, + ): super(PtbModel, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -131,31 +142,40 @@ def __init__(self, self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout - self.simple_lstm_rnn = SimpleLSTMRNN(hidden_size, - num_steps, - num_layers=num_layers, - init_scale=init_scale, - dropout=dropout) + self.simple_lstm_rnn = SimpleLSTMRNN( + hidden_size, + num_steps, + num_layers=num_layers, + init_scale=init_scale, + dropout=dropout, + ) self.embedding = paddle.fluid.dygraph.nn.Embedding( size=[vocab_size, hidden_size], dtype='float32', is_sparse=False, param_attr=paddle.ParamAttr( name='embedding_para', - initializer=paddle.nn.initializer.Uniform(low=-init_scale, - high=init_scale))) + initializer=paddle.nn.initializer.Uniform( + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_weight = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype="float32", default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.vocab_size], dtype="float32", default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def build_once(self, input, label, init_hidden, init_cell): pass @@ -163,29 +183,35 @@ def build_once(self, input, label, init_hidden, init_cell): @paddle.jit.to_static def forward(self, input, label, init_hidden, init_cell): - init_h = paddle.reshape(init_hidden, - shape=[self.num_layers, -1, self.hidden_size]) + init_h = paddle.reshape( + init_hidden, shape=[self.num_layers, -1, self.hidden_size] + ) - init_c = paddle.reshape(init_cell, - shape=[self.num_layers, -1, self.hidden_size]) + init_c = paddle.reshape( + init_cell, shape=[self.num_layers, -1, self.hidden_size] + ) x_emb = self.embedding(input) - x_emb = paddle.reshape(x_emb, - shape=[-1, self.num_steps, self.hidden_size]) + x_emb = paddle.reshape( + x_emb, shape=[-1, self.num_steps, self.hidden_size] + ) if self.dropout is not None and self.dropout > 0.0: x_emb = paddle.nn.functional.dropout( x_emb, dropout_prob=self.dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn_out, last_hidden, last_cell = self.simple_lstm_rnn( - x_emb, init_h, init_c) + x_emb, init_h, init_c + ) projection = paddle.matmul(x=rnn_out, y=self.softmax_weight) projection = paddle.add(x=projection, y=self.softmax_bias) loss = paddle.nn.functional.softmax_with_cross_entropy( - logits=projection, label=label, soft_label=False) + logits=projection, label=label, soft_label=False + ) loss = paddle.reshape(loss, shape=[-1, self.num_steps]) loss = paddle.mean(loss, axis=[0]) loss = paddle.fluid.layers.reduce_sum(loss) @@ -212,15 +238,18 @@ def train(place): paddle.disable_static(place) paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale, - dropout=dropout) - - sgd = paddle.optimizer.SGD(learning_rate=1e-3, - parameters=ptb_model.parameters()) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + dropout=dropout, + ) + + sgd = paddle.optimizer.SGD( + learning_rate=1e-3, parameters=ptb_model.parameters() + ) for epoch_id in range(max_epoch): @@ -228,19 +257,19 @@ def train(place): iters = 0.0 total_sample = 0 - init_hidden_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') - - init_hidden = paddle.to_tensor(data=init_hidden_data, - dtype=None, - place=None, - stop_gradient=True) - init_cell = paddle.to_tensor(data=init_cell_data, - dtype=None, - place=None, - stop_gradient=True) + init_hidden_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) + + init_hidden = paddle.to_tensor( + data=init_hidden_data, dtype=None, place=None, stop_gradient=True + ) + init_cell = paddle.to_tensor( + data=init_cell_data, dtype=None, place=None, stop_gradient=True + ) for step_id in range(batch_num): x_data = np.arange(12).reshape(4, 3).astype('int64') y_data = np.arange(1, 13).reshape(4, 3).astype('int64') @@ -249,17 +278,16 @@ def train(place): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, num_steps, 1)) - x = paddle.to_tensor(data=x_data, - dtype=None, - place=None, - stop_gradient=True) - y = paddle.to_tensor(data=y_data, - dtype=None, - place=None, - stop_gradient=True) - - dy_loss, last_hidden, last_cell = ptb_model(x, y, init_hidden, - init_cell) + x = paddle.to_tensor( + data=x_data, dtype=None, place=None, stop_gradient=True + ) + y = paddle.to_tensor( + data=y_data, dtype=None, place=None, stop_gradient=True + ) + + dy_loss, last_hidden, last_cell = ptb_model( + x, y, init_hidden, init_cell + ) out_loss = dy_loss.numpy() dy_loss.backward() @@ -271,14 +299,17 @@ def train(place): total_sample += 1 if step_id % PRINT_STEP == 0: if step_id == 0: - logging.info("epoch %d | step %d, loss %0.3f" % - (epoch_id, step_id, total_loss / total_sample)) + logging.info( + "epoch %d | step %d, loss %0.3f" + % (epoch_id, step_id, total_loss / total_sample) + ) avg_batch_time = time.time() else: speed = PRINT_STEP / (time.time() - avg_batch_time) logging.info( - "epoch %d | step %d, loss %0.3f, speed %.3f steps/s" % - (epoch_id, step_id, total_loss / total_sample, speed)) + "epoch %d | step %d, loss %0.3f, speed %.3f steps/s" + % (epoch_id, step_id, total_loss / total_sample, speed) + ) avg_batch_time = time.time() ret = out_loss, last_hidden.numpy(), last_cell.numpy() @@ -297,10 +328,12 @@ def train_static(place): class TestPtb(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_check_result(self): loss_1, hidden_1, cell_1 = train_static(self.place) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_reinforcement_learning.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_reinforcement_learning.py index 99361fa3949b3..d4bcd98e6449f 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_reinforcement_learning.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_reinforcement_learning.py @@ -29,7 +29,6 @@ class Policy(Layer): - def __init__(self): super(Policy, self).__init__() @@ -74,17 +73,18 @@ def train(args, place, to_static): eps = np.finfo(np.float32).eps.item() optimizer = fluid.optimizer.AdamaxOptimizer( - learning_rate=1e-2, parameter_list=policy.parameters()) + learning_rate=1e-2, parameter_list=policy.parameters() + ) def get_mean_and_std(values=[]): - n = 0. - s = 0. + n = 0.0 + s = 0.0 for val in values: s += val n += 1 mean = s / n - std = 0. + std = 0.0 for val in values: std += (val - mean) * (val - mean) std /= n @@ -99,14 +99,14 @@ def sample_action(probs): while idx < len(probs) and sample > probs[idx]: sample -= probs[idx] idx += 1 - mask = [0.] * len(probs) - mask[idx] = 1. + mask = [0.0] * len(probs) + mask[idx] = 1.0 return idx, np.array([mask]).astype("float32") def choose_best_action(probs): idx = 0 if probs[0] > probs[1] else 1 - mask = [1., 0.] if idx == 0 else [0., 1.] + mask = [1.0, 0.0] if idx == 0 else [0.0, 1.0] return idx, np.array([mask]).astype("float32") @@ -189,9 +189,10 @@ def finish_episode(): running_reward = 0.05 * ep_reward + (1 - 0.05) * running_reward if i_episode % args.log_interval == 0: print( - 'Episode {}\tLast reward: {:.2f}\tAverage reward: {:.2f}\t loss_probs: {}' - .format(i_episode, ep_reward, running_reward, - loss.numpy()[0])) + 'Episode {}\tLast reward: {:.2f}\tAverage reward: {:.2f}\t loss_probs: {}'.format( + i_episode, ep_reward, running_reward, loss.numpy()[0] + ) + ) if i_episode > args.train_step: break @@ -200,10 +201,12 @@ def finish_episode(): class TestDeclarative(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + ) self.args = Args() def test_train(self): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet.py index 86d215faadd32..a1f85bd3da3f1 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet.py @@ -36,8 +36,9 @@ # NOTE: Reduce batch_size from 8 to 2 to avoid unittest timeout. batch_size = 2 epoch_num = 1 -place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ - else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) program_translator = ProgramTranslator() @@ -50,30 +51,34 @@ def optimizer_setting(parameter_list=None): learning_rate=base_lr, momentum=momentum_rate, regularization=fluid.regularizer.L2Decay(l2_decay), - parameter_list=parameter_list) + parameter_list=parameter_list, + ) return optimizer class ConvBNLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - bias_attr=False) + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + bias_attr=False, + ) self._batch_norm = BatchNorm(num_filters, act=act) @@ -85,29 +90,36 @@ def forward(self, inputs): class BottleneckBlock(fluid.dygraph.Layer): - def __init__(self, num_channels, num_filters, stride, shortcut=True): super(BottleneckBlock, self).__init__() - self.conv0 = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=1, - act='relu') - self.conv1 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=stride, - act='relu') - self.conv2 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters * 4, - filter_size=1, - act=None) + self.conv0 = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=1, + act='relu', + ) + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + act='relu', + ) + self.conv2 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters * 4, + filter_size=1, + act=None, + ) if not shortcut: - self.short = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters * 4, - filter_size=1, - stride=stride) + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters * 4, + filter_size=1, + stride=stride, + ) self.shortcut = shortcut @@ -125,20 +137,23 @@ def forward(self, inputs): y = fluid.layers.elementwise_add(x=short, y=conv2) - layer_helper = fluid.layer_helper.LayerHelper(self.full_name(), - act='relu') + layer_helper = fluid.layer_helper.LayerHelper( + self.full_name(), act='relu' + ) return layer_helper.append_activation(y) class ResNet(fluid.dygraph.Layer): - def __init__(self, layers=50, class_dim=102): super(ResNet, self).__init__() self.layers = layers supported_layers = [50, 101, 152] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format(supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) if layers == 50: depth = [3, 4, 6, 3] @@ -149,15 +164,12 @@ def __init__(self, layers=50, class_dim=102): num_channels = [64, 256, 512, 1024] num_filters = [64, 128, 256, 512] - self.conv = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool2d_max = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv = ConvBNLayer( + num_channels=3, num_filters=64, filter_size=7, stride=2, act='relu' + ) + self.pool2d_max = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) self.bottleneck_block_list = [] for block in range(len(depth)): @@ -165,17 +177,21 @@ def __init__(self, layers=50, class_dim=102): for i in range(depth[block]): bottleneck_block = self.add_sublayer( 'bb_%d_%d' % (block, i), - BottleneckBlock(num_channels=num_channels[block] - if i == 0 else num_filters[block] * 4, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - shortcut=shortcut)) + BottleneckBlock( + num_channels=num_channels[block] + if i == 0 + else num_filters[block] * 4, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + shortcut=shortcut, + ), + ) self.bottleneck_block_list.append(bottleneck_block) shortcut = True - self.pool2d_avg = Pool2D(pool_size=7, - pool_type='avg', - global_pooling=True) + self.pool2d_avg = Pool2D( + pool_size=7, pool_type='avg', global_pooling=True + ) self.pool2d_avg_output = num_filters[len(num_filters) - 1] * 4 * 1 * 1 @@ -186,7 +202,9 @@ def __init__(self, layers=50, class_dim=102): class_dim, act='softmax', param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv))) + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + ) def forward(self, inputs): y = self.conv(inputs) @@ -201,7 +219,6 @@ def forward(self, inputs): def reader_decorator(reader): - def __reader__(): for item in reader(): img = np.array(item[0]).astype('float32').reshape(3, 224, 224) @@ -212,15 +229,15 @@ def __reader__(): class ResNetHelper: - def __init__(self): self.temp_dir = tempfile.TemporaryDirectory() self.model_save_dir = os.path.join(self.temp_dir.name, 'inference') self.model_save_prefix = os.path.join(self.model_save_dir, 'resnet') self.model_filename = 'resnet' + INFER_MODEL_SUFFIX self.params_filename = 'resnet' + INFER_PARAMS_SUFFIX - self.dy_state_dict_save_path = os.path.join(self.temp_dir.name, - 'resnet.dygraph') + self.dy_state_dict_save_path = os.path.join( + self.temp_dir.name, 'resnet.dygraph' + ) def __del__(self): self.temp_dir.cleanup() @@ -234,18 +251,21 @@ def train(self, to_static, build_strategy=None): paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) - data_loader = fluid.io.DataLoader.from_generator(capacity=5, - iterable=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) + data_loader = fluid.io.DataLoader.from_generator( + capacity=5, iterable=True + ) data_loader.set_sample_list_generator(train_reader) resnet = ResNet() if to_static: - resnet = paddle.jit.to_static(resnet, - build_strategy=build_strategy) + resnet = paddle.jit.to_static( + resnet, build_strategy=build_strategy + ) optimizer = optimizer_setting(parameter_list=resnet.parameters()) for epoch in range(epoch_num): @@ -261,12 +281,12 @@ def train(self, to_static, build_strategy=None): pred = resnet(img) loss = fluid.layers.cross_entropy(input=pred, label=label) avg_loss = paddle.mean(x=loss) - acc_top1 = fluid.layers.accuracy(input=pred, - label=label, - k=1) - acc_top5 = fluid.layers.accuracy(input=pred, - label=label, - k=5) + acc_top1 = fluid.layers.accuracy( + input=pred, label=label, k=1 + ) + acc_top5 = fluid.layers.accuracy( + input=pred, label=label, k=5 + ) avg_loss.backward() optimizer.minimize(avg_loss) @@ -279,17 +299,27 @@ def train(self, to_static, build_strategy=None): end_time = time.time() if batch_id % 2 == 0: - print( "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" % \ - ( epoch, batch_id, total_loss.numpy() / total_sample, \ - total_acc1.numpy() / total_sample, total_acc5.numpy() / total_sample, end_time-start_time)) + print( + "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" + % ( + epoch, + batch_id, + total_loss.numpy() / total_sample, + total_acc1.numpy() / total_sample, + total_acc5.numpy() / total_sample, + end_time - start_time, + ) + ) if batch_id == 10: if to_static: - fluid.dygraph.jit.save(resnet, - self.model_save_prefix) + fluid.dygraph.jit.save( + resnet, self.model_save_prefix + ) else: fluid.dygraph.save_dygraph( resnet.state_dict(), - self.dy_state_dict_save_path) + self.dy_state_dict_save_path, + ) # avoid dataloader throw abort signaal data_loader._reset() break @@ -302,7 +332,8 @@ def predict_dygraph(self, data): resnet = ResNet() model_dict, _ = fluid.dygraph.load_dygraph( - self.dy_state_dict_save_path) + self.dy_state_dict_save_path + ) resnet.set_dict(model_dict) resnet.eval() @@ -313,15 +344,22 @@ def predict_dygraph(self, data): def predict_static(self, data): paddle.enable_static() exe = fluid.Executor(place) - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(self.model_save_dir, - executor=exe, - model_filename=self.model_filename, - params_filename=self.params_filename) - - pred_res = exe.run(inference_program, - feed={feed_target_names[0]: data}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + self.model_save_dir, + executor=exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + ) + + pred_res = exe.run( + inference_program, + feed={feed_target_names[0]: data}, + fetch_list=fetch_targets, + ) return pred_res[0] @@ -335,14 +373,17 @@ def predict_dygraph_jit(self, data): return pred_res.numpy() def predict_analysis_inference(self, data): - output = PredictorTools(self.model_save_dir, self.model_filename, - self.params_filename, [data]) - out, = output() + output = PredictorTools( + self.model_save_dir, + self.model_filename, + self.params_filename, + [data], + ) + (out,) = output() return out class TestResnet(unittest.TestCase): - def setUp(self): self.resnet_helper = ResNetHelper() @@ -360,19 +401,24 @@ def verify_predict(self): dy_pre, st_pre, rtol=1e-05, - err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre)) + err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre), + ) np.testing.assert_allclose( dy_jit_pre, st_pre, rtol=1e-05, err_msg='dy_jit_pre:\n {}\n, st_pre: \n{}.'.format( - dy_jit_pre, st_pre)) + dy_jit_pre, st_pre + ), + ) np.testing.assert_allclose( predictor_pre, st_pre, rtol=1e-05, err_msg='predictor_pre:\n {}\n, st_pre: \n{}.'.format( - predictor_pre, st_pre)) + predictor_pre, st_pre + ), + ) def test_resnet(self): static_loss = self.train(to_static=True) @@ -382,7 +428,9 @@ def test_resnet(self): dygraph_loss, rtol=1e-05, err_msg='static_loss: {} \n dygraph_loss: {}'.format( - static_loss, dygraph_loss)) + static_loss, dygraph_loss + ), + ) self.verify_predict() def test_in_static_mode_mkldnn(self): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_amp.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_amp.py index c535b7223db47..aa53f663deb1f 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_amp.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_amp.py @@ -25,8 +25,9 @@ # NOTE: Reduce batch_size from 8 to 2 to avoid unittest timeout. batch_size = 2 epoch_num = 1 -place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ - else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) program_translator = ProgramTranslator() @@ -58,10 +59,13 @@ def train(to_static, build_strategy=None): for batch_id in range(100): start_time = time.time() img = paddle.to_tensor( - np.random.random([batch_size, 3, 224, - 224]).astype('float32')) + np.random.random([batch_size, 3, 224, 224]).astype( + 'float32' + ) + ) label = paddle.to_tensor( - np.random.randint(0, 100, [batch_size, 1], dtype='int64')) + np.random.randint(0, 100, [batch_size, 1], dtype='int64') + ) img.stop_gradient = True label.stop_gradient = True @@ -87,9 +91,17 @@ def train(to_static, build_strategy=None): end_time = time.time() if batch_id % 2 == 0: - print( "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" % \ - ( epoch, batch_id, total_loss.numpy() / total_sample, \ - total_acc1.numpy() / total_sample, total_acc5.numpy() / total_sample, end_time-start_time)) + print( + "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" + % ( + epoch, + batch_id, + total_loss.numpy() / total_sample, + total_acc1.numpy() / total_sample, + total_acc5.numpy() / total_sample, + end_time - start_time, + ) + ) if batch_id == 10: break @@ -97,7 +109,6 @@ def train(to_static, build_strategy=None): class TestResnet(unittest.TestCase): - def train(self, to_static): program_translator.enable(to_static) return train(to_static) @@ -110,7 +121,9 @@ def test_resnet(self): dygraph_loss, rtol=1e-05, err_msg='static_loss: {} \n dygraph_loss: {}'.format( - static_loss, dygraph_loss)) + static_loss, dygraph_loss + ), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_pure_fp16.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_pure_fp16.py index f4d724826ab7d..9951d67c2b1a7 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_pure_fp16.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_pure_fp16.py @@ -46,10 +46,9 @@ def train(to_static, build_strategy=None): optimizer = optimizer_setting(parameter_list=resnet.parameters()) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) - resnet, optimizer = paddle.amp.decorate(models=resnet, - optimizers=optimizer, - level='O2', - save_dtype='float32') + resnet, optimizer = paddle.amp.decorate( + models=resnet, optimizers=optimizer, level='O2', save_dtype='float32' + ) for epoch in range(epoch_num): loss_data = [] @@ -61,16 +60,20 @@ def train(to_static, build_strategy=None): for batch_id in range(100): start_time = time.time() img = paddle.to_tensor( - np.random.random([batch_size, 3, 224, 224]).astype('float32')) + np.random.random([batch_size, 3, 224, 224]).astype('float32') + ) label = paddle.to_tensor( - np.random.randint(0, 100, [batch_size, 1], dtype='int64')) + np.random.randint(0, 100, [batch_size, 1], dtype='int64') + ) img.stop_gradient = True label.stop_gradient = True - with paddle.amp.auto_cast(enable=True, - custom_white_list=None, - custom_black_list=None, - level='O2'): + with paddle.amp.auto_cast( + enable=True, + custom_white_list=None, + custom_black_list=None, + level='O2', + ): pred = resnet(img) loss = fluid.layers.cross_entropy(input=pred, label=label) avg_loss = paddle.mean(x=pred) @@ -90,9 +93,17 @@ def train(to_static, build_strategy=None): end_time = time.time() if batch_id % 2 == 0: - print( "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" % \ - ( epoch, batch_id, total_loss.numpy() / total_sample, \ - total_acc1.numpy() / total_sample, total_acc5.numpy() / total_sample, end_time-start_time)) + print( + "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" + % ( + epoch, + batch_id, + total_loss.numpy() / total_sample, + total_acc1.numpy() / total_sample, + total_acc5.numpy() / total_sample, + end_time - start_time, + ) + ) if batch_id == 10: break @@ -100,7 +111,6 @@ def train(to_static, build_strategy=None): class TestResnet(unittest.TestCase): - def train(self, to_static): program_translator.enable(to_static) build_strategy = paddle.static.BuildStrategy() @@ -120,7 +130,9 @@ def test_resnet(self): rtol=1e-05, atol=0.001, err_msg='static_loss: {} \n dygraph_loss: {}'.format( - static_loss, dygraph_loss)) + static_loss, dygraph_loss + ), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_v2.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_v2.py index 301da29d6d9e6..f4f2e8154494c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_v2.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet_v2.py @@ -34,8 +34,9 @@ # NOTE: Reduce batch_size from 8 to 2 to avoid unittest timeout. batch_size = 2 epoch_num = 1 -place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ - else paddle.CPUPlace() +place = ( + paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() +) program_translator = paddle.jit.ProgramTranslator() @@ -48,29 +49,33 @@ def optimizer_setting(parameter_list=None): learning_rate=base_lr, momentum=momentum_rate, weight_decay=paddle.regularizer.L2Decay(l2_decay), - parameters=parameter_list) + parameters=parameter_list, + ) return optimizer class ConvBNLayer(paddle.nn.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(ConvBNLayer, self).__init__() - self._conv = paddle.nn.Conv2D(in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=False) + self._conv = paddle.nn.Conv2D( + in_channels=num_channels, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + bias_attr=False, + ) self._batch_norm = paddle.nn.BatchNorm(num_filters, act=act) @@ -82,29 +87,36 @@ def forward(self, inputs): class BottleneckBlock(paddle.nn.Layer): - def __init__(self, num_channels, num_filters, stride, shortcut=True): super(BottleneckBlock, self).__init__() - self.conv0 = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=1, - act='relu') - self.conv1 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=stride, - act='relu') - self.conv2 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters * 4, - filter_size=1, - act=None) + self.conv0 = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=1, + act='relu', + ) + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + act='relu', + ) + self.conv2 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters * 4, + filter_size=1, + act=None, + ) if not shortcut: - self.short = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters * 4, - filter_size=1, - stride=stride) + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters * 4, + filter_size=1, + stride=stride, + ) self.shortcut = shortcut @@ -122,20 +134,23 @@ def forward(self, inputs): y = paddle.add(x=short, y=conv2) - layer_helper = paddle.fluid.layer_helper.LayerHelper(self.full_name(), - act='relu') + layer_helper = paddle.fluid.layer_helper.LayerHelper( + self.full_name(), act='relu' + ) return layer_helper.append_activation(y) class ResNet(paddle.nn.Layer): - def __init__(self, layers=50, class_dim=102): super(ResNet, self).__init__() self.layers = layers supported_layers = [50, 101, 152] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format(supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) if layers == 50: depth = [3, 4, 6, 3] @@ -146,15 +161,12 @@ def __init__(self, layers=50, class_dim=102): num_channels = [64, 256, 512, 1024] num_filters = [64, 128, 256, 512] - self.conv = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool2d_max = paddle.fluid.dygraph.Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv = ConvBNLayer( + num_channels=3, num_filters=64, filter_size=7, stride=2, act='relu' + ) + self.pool2d_max = paddle.fluid.dygraph.Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) self.bottleneck_block_list = [] for block in range(len(depth)): @@ -162,17 +174,21 @@ def __init__(self, layers=50, class_dim=102): for i in range(depth[block]): bottleneck_block = self.add_sublayer( 'bb_%d_%d' % (block, i), - BottleneckBlock(num_channels=num_channels[block] - if i == 0 else num_filters[block] * 4, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - shortcut=shortcut)) + BottleneckBlock( + num_channels=num_channels[block] + if i == 0 + else num_filters[block] * 4, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + shortcut=shortcut, + ), + ) self.bottleneck_block_list.append(bottleneck_block) shortcut = True - self.pool2d_avg = paddle.fluid.dygraph.Pool2D(pool_size=7, - pool_type='avg', - global_pooling=True) + self.pool2d_avg = paddle.fluid.dygraph.Pool2D( + pool_size=7, pool_type='avg', global_pooling=True + ) self.pool2d_avg_output = num_filters[len(num_filters) - 1] * 4 * 1 * 1 @@ -182,7 +198,9 @@ def __init__(self, layers=50, class_dim=102): in_features=self.pool2d_avg_output, out_features=class_dim, weight_attr=paddle.ParamAttr( - initializer=paddle.nn.initializer.Uniform(-stdv, stdv))) + initializer=paddle.nn.initializer.Uniform(-stdv, stdv) + ), + ) @paddle.jit.to_static def forward(self, inputs): @@ -199,7 +217,6 @@ def forward(self, inputs): def reader_decorator(reader): - def __reader__(): for item in reader(): img = np.array(item[0]).astype('float32').reshape(3, 224, 224) @@ -210,17 +227,22 @@ def __reader__(): class TestResnet(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.model_save_dir = os.path.join(self.temp_dir.name, "./inference") - self.model_save_prefix = os.path.join(self.temp_dir.name, - "./inference/resnet_v2") - self.model_filename = "resnet_v2" + paddle.fluid.dygraph.io.INFER_MODEL_SUFFIX - self.params_filename = "resnet_v2" + paddle.fluid.dygraph.io.INFER_PARAMS_SUFFIX - self.dy_state_dict_save_path = os.path.join(self.temp_dir.name, - "./resnet_v2.dygraph") + self.model_save_prefix = os.path.join( + self.temp_dir.name, "./inference/resnet_v2" + ) + self.model_filename = ( + "resnet_v2" + paddle.fluid.dygraph.io.INFER_MODEL_SUFFIX + ) + self.params_filename = ( + "resnet_v2" + paddle.fluid.dygraph.io.INFER_PARAMS_SUFFIX + ) + self.dy_state_dict_save_path = os.path.join( + self.temp_dir.name, "./resnet_v2.dygraph" + ) def tearDown(self): self.temp_dir.cleanup() @@ -234,12 +256,14 @@ def do_train(self, to_static): paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) - train_reader = paddle.batch(reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) - data_loader = paddle.io.DataLoader.from_generator(capacity=5, - iterable=True) + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True, + ) + data_loader = paddle.io.DataLoader.from_generator( + capacity=5, iterable=True + ) data_loader.set_sample_list_generator(train_reader) resnet = ResNet() @@ -256,8 +280,9 @@ def do_train(self, to_static): img, label = data pred = resnet(img) - loss = paddle.nn.functional.cross_entropy(input=pred, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=pred, label=label + ) avg_loss = paddle.mean(x=loss) acc_top1 = paddle.metric.accuracy(input=pred, label=label, k=1) acc_top5 = paddle.metric.accuracy(input=pred, label=label, k=5) @@ -273,15 +298,24 @@ def do_train(self, to_static): end_time = time.time() if batch_id % 2 == 0: - print( "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" % \ - ( epoch, batch_id, total_loss.numpy() / total_sample, \ - total_acc1.numpy() / total_sample, total_acc5.numpy() / total_sample, end_time-start_time)) + print( + "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" + % ( + epoch, + batch_id, + total_loss.numpy() / total_sample, + total_acc1.numpy() / total_sample, + total_acc5.numpy() / total_sample, + end_time - start_time, + ) + ) if batch_id == 10: if to_static: paddle.jit.save(resnet, self.model_save_prefix) else: paddle.fluid.dygraph.save_dygraph( - resnet.state_dict(), self.dy_state_dict_save_path) + resnet.state_dict(), self.dy_state_dict_save_path + ) # avoid dataloader throw abort signaal data_loader._reset() break @@ -295,15 +329,16 @@ def predict_dygraph(self, data): resnet = ResNet() model_dict, _ = paddle.fluid.dygraph.load_dygraph( - self.dy_state_dict_save_path) + self.dy_state_dict_save_path + ) resnet.set_dict(model_dict) resnet.eval() pred_res = resnet( - paddle.to_tensor(data=data, - dtype=None, - place=None, - stop_gradient=True)) + paddle.to_tensor( + data=data, dtype=None, place=None, stop_gradient=True + ) + ) ret = pred_res.numpy() paddle.enable_static() @@ -311,16 +346,22 @@ def predict_dygraph(self, data): def predict_static(self, data): exe = paddle.static.Executor(place) - [inference_program, feed_target_names, - fetch_targets] = paddle.static.load_inference_model( - self.model_save_dir, - executor=exe, - model_filename=self.model_filename, - params_filename=self.params_filename) - - pred_res = exe.run(inference_program, - feed={feed_target_names[0]: data}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model( + self.model_save_dir, + executor=exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + ) + + pred_res = exe.run( + inference_program, + feed={feed_target_names[0]: data}, + fetch_list=fetch_targets, + ) return pred_res[0] @@ -336,9 +377,13 @@ def predict_dygraph_jit(self, data): return ret def predict_analysis_inference(self, data): - output = PredictorTools(self.model_save_dir, self.model_filename, - self.params_filename, [data]) - out, = output() + output = PredictorTools( + self.model_save_dir, + self.model_filename, + self.params_filename, + [data], + ) + (out,) = output() return out def train(self, to_static): @@ -355,19 +400,24 @@ def verify_predict(self): dy_pre, st_pre, rtol=1e-05, - err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre)) + err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre), + ) np.testing.assert_allclose( dy_jit_pre, st_pre, rtol=1e-05, err_msg='dy_jit_pre:\n {}\n, st_pre: \n{}.'.format( - dy_jit_pre, st_pre)) + dy_jit_pre, st_pre + ), + ) np.testing.assert_allclose( predictor_pre, st_pre, rtol=1e-05, err_msg='predictor_pre:\n {}\n, st_pre: \n{}.'.format( - predictor_pre, st_pre)) + predictor_pre, st_pre + ), + ) def test_resnet(self): static_loss = self.train(to_static=True) @@ -377,7 +427,9 @@ def test_resnet(self): dygraph_loss, rtol=1e-05, err_msg='static_loss: {} \n dygraph_loss: {}'.format( - static_loss, dygraph_loss)) + static_loss, dygraph_loss + ), + ) self.verify_predict() def test_in_static_mode_mkldnn(self): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_return.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_return.py index e3323dd084a6a..a194cede9c0df 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_return.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_return.py @@ -168,7 +168,7 @@ def test_return_list_many_values(x): def test_return_tuple_one_value(x): x = fluid.dygraph.to_variable(x) x += 1 - return (x, ) + return (x,) @to_static @@ -250,7 +250,6 @@ def test_return_in_for_2(x): @to_static def test_return_nested(x): - def func(): rr = 0 if True: @@ -266,11 +265,13 @@ def func(): class TestReturnBase(unittest.TestCase): - def setUp(self): self.input = np.ones((1)).astype('int32') - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.init_dygraph_func() self.program_translator = ProgramTranslator() @@ -294,9 +295,9 @@ def _test_value_impl(self): self.assertTrue(isinstance(static_res, tuple)) self.assertEqual(len(dygraph_res), len(static_res)) for i in range(len(dygraph_res)): - np.testing.assert_allclose(dygraph_res[i], - static_res[i], - rtol=1e-05) + np.testing.assert_allclose( + dygraph_res[i], static_res[i], rtol=1e-05 + ) elif isinstance(dygraph_res, np.ndarray): np.testing.assert_allclose(dygraph_res, static_res, rtol=1e-05) else: @@ -311,134 +312,113 @@ def test_transformed_static_result(self): class TestInsideFuncBase(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_inside_func_base class TestReturnIf(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_if class TestReturnOnlyIf(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_if_else_2 class TestReturnInFor(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_in_for class TestReturnInWhile(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_in_while class TestReturnIfDiff(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_diff_return class TestReturnIfElse(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_if_else class TestReturnInWhile2(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_in_while_2 self.error = "Found return statement in While or For body and loop" class TestReturnInFor2(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_in_for_2 self.error = "Found return statement in While or For body and loop" class TestRecursiveReturn(TestReturnBase): - def init_dygraph_func(self): self.input = self.input.astype(np.float32) self.dygraph_func = test_recursive_return class TestReturnDifferentLengthIfBody(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_different_length_if_body self.error = "Your if/else have different number of return value." class TestReturnDifferentLengthElse(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_different_length_else self.error = "Your if/else have different number of return value." class TestNoReturn(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_no_return class TestReturnNone(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_none self.error = "Your if/else have different number of return value." class TestReturnNoVariable(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_no_variable self.error = "Your if/else have different number of return value." class TestReturnListOneValue(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_list_one_value class TestReturnListManyValue(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_list_many_values class TestReturnTupleOneValue(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_tuple_one_value class TestReturnTupleManyValue(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_tuple_many_values class TestReturnNested(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_nested class TestReturnSpecial(TestReturnBase): - def init_dygraph_func(self): self.dygraph_func = test_return_without_paddle_cond diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_rollback.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_rollback.py index f949e9c0da78d..90c017ee9e769 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_rollback.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_rollback.py @@ -16,11 +16,12 @@ import paddle import numpy as np from paddle.fluid.dygraph.dygraph_to_static.utils import func_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.program_translator import StaticFunction +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + StaticFunction, +) class Net(paddle.nn.Layer): - def __init__(self): super(Net, self).__init__() self.sub = SubNet() @@ -38,7 +39,6 @@ def infer(self, x): class SubNet(paddle.nn.Layer): - def __init__(self): super(SubNet, self).__init__() @@ -59,15 +59,14 @@ def bar(self, x, flag=True): def foo(x, flag=False): if flag: - out = x * 2. + out = x * 2.0 else: - out = x / 2. + out = x / 2.0 return out class TestRollBackPlainFunction(unittest.TestCase): - def setUp(self): paddle.set_device("cpu") @@ -86,7 +85,6 @@ def test_plain_func(self): class TestRollBackNet(unittest.TestCase): - def setUp(self): paddle.set_device("cpu") @@ -118,8 +116,9 @@ def test_net(self): self.assertFalse(isinstance(net.infer, StaticFunction)) self.assertFalse("true_fn" in func_to_source_code(net.sub.forward)) dy_infer_out = net.infer(x) - np.testing.assert_array_equal(st_infer_out.numpy(), - dy_infer_out.numpy()) + np.testing.assert_array_equal( + st_infer_out.numpy(), dy_infer_out.numpy() + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py index 60aa64abe66f4..f1c3bd4c1b617 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_inference_model.py @@ -21,20 +21,22 @@ import paddle.fluid as fluid from paddle.fluid.dygraph.dygraph_to_static import ProgramTranslator from paddle.fluid.dygraph.jit import declarative -from paddle.fluid.dygraph.dygraph_to_static.partial_program import partial_program_from +from paddle.fluid.dygraph.dygraph_to_static.partial_program import ( + partial_program_from, +) from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX SEED = 2020 np.random.seed(SEED) -place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) program_translator = ProgramTranslator() class SimpleFcLayer(fluid.dygraph.Layer): - def __init__(self, fc_size): super(SimpleFcLayer, self).__init__() self._linear = fluid.dygraph.Linear(fc_size, fc_size) @@ -48,7 +50,6 @@ def forward(self, x): class TestDyToStaticSaveInferenceModel(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -64,8 +65,9 @@ def test_save_inference_model(self): x = fluid.dygraph.to_variable(x_data) layer = SimpleFcLayer(fc_size) - adam = fluid.optimizer.SGD(learning_rate=0.1, - parameter_list=layer.parameters()) + adam = fluid.optimizer.SGD( + learning_rate=0.1, parameter_list=layer.parameters() + ) for i in range(5): loss, pred = layer(x) @@ -74,65 +76,78 @@ def test_save_inference_model(self): layer.clear_gradients() # test for saving model in dygraph.guard infer_model_prefix = os.path.join( - self.temp_dir.name, "test_dy2stat_inference_in_guard/model") - infer_model_dir = os.path.join(self.temp_dir.name, - "test_dy2stat_inference_in_guard") - fluid.dygraph.jit.save(layer=layer, - path=infer_model_prefix, - input_spec=[x], - output_spec=[pred]) + self.temp_dir.name, "test_dy2stat_inference_in_guard/model" + ) + infer_model_dir = os.path.join( + self.temp_dir.name, "test_dy2stat_inference_in_guard" + ) + fluid.dygraph.jit.save( + layer=layer, + path=infer_model_prefix, + input_spec=[x], + output_spec=[pred], + ) # Check the correctness of the inference dygraph_out, _ = layer(x) self.check_save_inference_model(layer, [x_data], dygraph_out.numpy()) - self.check_save_inference_model(layer, [x_data], - dygraph_out.numpy(), - fetch=[loss]) - self.check_save_inference_model(layer, [x_data], - dygraph_out.numpy(), - feed=[x]) - - def check_save_inference_model(self, - model, - inputs, - gt_out, - feed=None, - fetch=None): + self.check_save_inference_model( + layer, [x_data], dygraph_out.numpy(), fetch=[loss] + ) + self.check_save_inference_model( + layer, [x_data], dygraph_out.numpy(), feed=[x] + ) + + def check_save_inference_model( + self, model, inputs, gt_out, feed=None, fetch=None + ): expected_persistable_vars = set([p.name for p in model.parameters()]) - infer_model_prefix = os.path.join(self.temp_dir.name, - "test_dy2stat_inference/model") - infer_model_dir = os.path.join(self.temp_dir.name, - "test_dy2stat_inference") + infer_model_prefix = os.path.join( + self.temp_dir.name, "test_dy2stat_inference/model" + ) + infer_model_dir = os.path.join( + self.temp_dir.name, "test_dy2stat_inference" + ) model_filename = "model" + INFER_MODEL_SUFFIX params_filename = "model" + INFER_PARAMS_SUFFIX - fluid.dygraph.jit.save(layer=model, - path=infer_model_prefix, - input_spec=feed if feed else None, - output_spec=fetch if fetch else None) + fluid.dygraph.jit.save( + layer=model, + path=infer_model_prefix, + input_spec=feed if feed else None, + output_spec=fetch if fetch else None, + ) # Check the correctness of the inference - infer_out = self.load_and_run_inference(infer_model_dir, model_filename, - params_filename, inputs) + infer_out = self.load_and_run_inference( + infer_model_dir, model_filename, params_filename, inputs + ) np.testing.assert_allclose(gt_out, infer_out, rtol=1e-05) - def load_and_run_inference(self, model_path, model_filename, - params_filename, inputs): + def load_and_run_inference( + self, model_path, model_filename, params_filename, inputs + ): paddle.enable_static() exe = fluid.Executor(place) - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(dirname=model_path, - executor=exe, - model_filename=model_filename, - params_filename=params_filename) - results = exe.run(inference_program, - feed=dict(zip(feed_target_names, inputs)), - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + dirname=model_path, + executor=exe, + model_filename=model_filename, + params_filename=params_filename, + ) + results = exe.run( + inference_program, + feed=dict(zip(feed_target_names, inputs)), + fetch_list=fetch_targets, + ) return np.array(results[0]) class TestPartialProgramRaiseError(unittest.TestCase): - def test_param_type(self): program_translator = ProgramTranslator() program_translator.enable(True) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_load.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_load.py index 42be11182c082..0898b72730441 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_load.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_save_load.py @@ -25,16 +25,17 @@ np.random.seed(2020) -place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) class TestDyToStaticSaveLoad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() - self.model_path = os.path.join(self.temp_dir.name, - "test_dy2stat_save_load") + self.model_path = os.path.join( + self.temp_dir.name, "test_dy2stat_save_load" + ) def tearDown(self): self.temp_dir.cleanup() @@ -49,8 +50,9 @@ def test_save_load_same_result(self): program_translator.enable(True) x = fluid.dygraph.to_variable(x_data) net = Linear(32, 64) - adam = AdamOptimizer(learning_rate=0.1, - parameter_list=net.parameters()) + adam = AdamOptimizer( + learning_rate=0.1, parameter_list=net.parameters() + ) for i in range(batch_num): static_out, static_loss = net(x) @@ -81,12 +83,12 @@ def test_save_load_same_result(self): program_translator.enable(False) dygraph_out, dygraph_loss = dygraph_net(x) - np.testing.assert_allclose(dygraph_out.numpy(), - static_out.numpy(), - rtol=1e-05) - np.testing.assert_allclose(dygraph_loss.numpy(), - static_loss.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + dygraph_out.numpy(), static_out.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + dygraph_loss.numpy(), static_loss.numpy(), rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py index 2a86c3f77a1f6..74f6bdb2d6101 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_se_resnet.py @@ -38,8 +38,9 @@ PRINT_STEP = 2 STEP_NUM = 10 -place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ - else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) # Note: Set True to eliminate randomness. # 1. For one operation, cuDNN has several algorithms, @@ -52,7 +53,7 @@ "name": "cosine_decay", "batch_size": BATCH_SIZE, "epochs": [40, 80, 100], - "steps": [0.1, 0.01, 0.001, 0.0001] + "steps": [0.1, 0.01, 0.001, 0.0001], }, "lr": 0.0125, "total_images": 6149, @@ -78,35 +79,39 @@ def optimizer_setting(params, parameter_list): lr = params["lr"] num_epochs = params["num_epochs"] optimizer = fluid.optimizer.Momentum( - learning_rate=fluid.layers.cosine_decay(learning_rate=lr, - step_each_epoch=step, - epochs=num_epochs), + learning_rate=fluid.layers.cosine_decay( + learning_rate=lr, step_each_epoch=step, epochs=num_epochs + ), momentum=momentum_rate, regularization=fluid.regularizer.L2Decay(l2_decay), - parameter_list=parameter_list) + parameter_list=parameter_list, + ) return optimizer class ConvBNLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - bias_attr=False) + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + bias_attr=False, + ) self._batch_norm = BatchNorm(num_filters, act=act) @@ -118,7 +123,6 @@ def forward(self, inputs): class SqueezeExcitation(fluid.dygraph.Layer): - def __init__(self, num_channels, reduction_ratio): super(SqueezeExcitation, self).__init__() @@ -129,15 +133,19 @@ def __init__(self, num_channels, reduction_ratio): num_channels, num_channels // reduction_ratio, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv)), - act='relu') + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + act='relu', + ) stdv = 1.0 / math.sqrt(num_channels / 16.0 * 1.0) self._excitation = Linear( num_channels // reduction_ratio, num_channels, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv)), - act='sigmoid') + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + act='sigmoid', + ) def forward(self, input): y = self._pool(input) @@ -149,39 +157,49 @@ def forward(self, input): class BottleneckBlock(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - stride, - cardinality, - reduction_ratio, - shortcut=True): + def __init__( + self, + num_channels, + num_filters, + stride, + cardinality, + reduction_ratio, + shortcut=True, + ): super(BottleneckBlock, self).__init__() - self.conv0 = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=1, - act="relu") - self.conv1 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=stride, - groups=cardinality, - act="relu") - self.conv2 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters * 2, - filter_size=1, - act=None) - - self.scale = SqueezeExcitation(num_channels=num_filters * 2, - reduction_ratio=reduction_ratio) + self.conv0 = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=1, + act="relu", + ) + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + groups=cardinality, + act="relu", + ) + self.conv2 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters * 2, + filter_size=1, + act=None, + ) + + self.scale = SqueezeExcitation( + num_channels=num_filters * 2, reduction_ratio=reduction_ratio + ) if not shortcut: - self.short = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters * 2, - filter_size=1, - stride=stride) + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters * 2, + filter_size=1, + stride=stride, + ) self.shortcut = shortcut @@ -203,67 +221,76 @@ def forward(self, inputs): class SeResNeXt(fluid.dygraph.Layer): - def __init__(self, layers=50, class_dim=102): super(SeResNeXt, self).__init__() self.layers = layers supported_layers = [50, 101, 152] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format(supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) if layers == 50: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 6, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) elif layers == 101: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 23, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) elif layers == 152: cardinality = 64 reduction_ratio = 16 depth = [3, 8, 36, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=3, - stride=2, - act='relu') - self.conv1 = ConvBNLayer(num_channels=64, - num_filters=64, - filter_size=3, - stride=1, - act='relu') - self.conv2 = ConvBNLayer(num_channels=64, - num_filters=128, - filter_size=3, - stride=1, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=3, + stride=2, + act='relu', + ) + self.conv1 = ConvBNLayer( + num_channels=64, + num_filters=64, + filter_size=3, + stride=1, + act='relu', + ) + self.conv2 = ConvBNLayer( + num_channels=64, + num_filters=128, + filter_size=3, + stride=1, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) self.bottleneck_block_list = [] num_channels = 64 @@ -274,19 +301,22 @@ def __init__(self, layers=50, class_dim=102): for i in range(depth[block]): bottleneck_block = self.add_sublayer( 'bb_%d_%d' % (block, i), - BottleneckBlock(num_channels=num_channels, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - cardinality=cardinality, - reduction_ratio=reduction_ratio, - shortcut=shortcut)) + BottleneckBlock( + num_channels=num_channels, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + cardinality=cardinality, + reduction_ratio=reduction_ratio, + shortcut=shortcut, + ), + ) num_channels = bottleneck_block._num_channels_out self.bottleneck_block_list.append(bottleneck_block) shortcut = True - self.pool2d_avg = Pool2D(pool_size=7, - pool_type='avg', - global_pooling=True) + self.pool2d_avg = Pool2D( + pool_size=7, pool_type='avg', global_pooling=True + ) stdv = 1.0 / math.sqrt(2048 * 1.0) self.pool2d_avg_output = num_filters[len(num_filters) - 1] * 2 * 1 * 1 @@ -295,7 +325,9 @@ def __init__(self, layers=50, class_dim=102): self.pool2d_avg_output, class_dim, param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv))) + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + ) @declarative def forward(self, inputs, label): @@ -326,21 +358,23 @@ def forward(self, inputs, label): class TestSeResnet(unittest.TestCase): - def setUp(self): - self.train_reader = paddle.batch(paddle.dataset.flowers.train( - use_xmap=False, cycle=True), - batch_size=BATCH_SIZE, - drop_last=True) + self.train_reader = paddle.batch( + paddle.dataset.flowers.train(use_xmap=False, cycle=True), + batch_size=BATCH_SIZE, + drop_last=True, + ) self.temp_dir = tempfile.TemporaryDirectory() self.model_save_dir = os.path.join(self.temp_dir.name, "inference") - self.model_save_prefix = os.path.join(self.temp_dir.name, - "inference/se_resnet") + self.model_save_prefix = os.path.join( + self.temp_dir.name, "inference/se_resnet" + ) self.model_filename = "se_resnet" + INFER_MODEL_SUFFIX self.params_filename = "se_resnet" + INFER_PARAMS_SUFFIX - self.dy_state_dict_save_path = os.path.join(self.temp_dir.name, - "se_resnet.dygraph") + self.dy_state_dict_save_path = os.path.join( + self.temp_dir.name, "se_resnet.dygraph" + ) def tearDown(self): self.temp_dir.cleanup() @@ -355,8 +389,9 @@ def train(self, train_reader, to_static): paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) se_resnext = SeResNeXt() - optimizer = optimizer_setting(train_parameters, - se_resnext.parameters()) + optimizer = optimizer_setting( + train_parameters, se_resnext.parameters() + ) for epoch_id in range(EPOCH_NUM): total_loss = 0.0 @@ -366,12 +401,14 @@ def train(self, train_reader, to_static): step_idx = 0 speed_list = [] for step_id, data in enumerate(train_reader()): - dy_x_data = np.array([ - x[0].reshape(3, 224, 224) for x in data - ]).astype('float32') - y_data = np.array([x[1] - for x in data]).astype('int64').reshape( - BATCH_SIZE, 1) + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(BATCH_SIZE, 1) + ) img = to_variable(dy_x_data) label = to_variable(y_data) @@ -392,32 +429,54 @@ def train(self, train_reader, to_static): total_sample += 1 if step_id % PRINT_STEP == 0: if step_id == 0: - logging.info( "epoch %d | step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f" % \ - ( epoch_id, step_id, total_loss / total_sample, \ - total_acc1 / total_sample, total_acc5 / total_sample)) + logging.info( + "epoch %d | step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f" + % ( + epoch_id, + step_id, + total_loss / total_sample, + total_acc1 / total_sample, + total_acc5 / total_sample, + ) + ) avg_batch_time = time.time() else: speed = PRINT_STEP / (time.time() - avg_batch_time) speed_list.append(speed) - logging.info( "epoch %d | step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, speed %.3f steps/s" % \ - ( epoch_id, step_id, total_loss / total_sample, \ - total_acc1 / total_sample, total_acc5 / total_sample, speed)) + logging.info( + "epoch %d | step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, speed %.3f steps/s" + % ( + epoch_id, + step_id, + total_loss / total_sample, + total_acc1 / total_sample, + total_acc5 / total_sample, + speed, + ) + ) avg_batch_time = time.time() step_idx += 1 if step_idx == STEP_NUM: if to_static: - fluid.dygraph.jit.save(se_resnext, - self.model_save_prefix, - [img], - output_spec=[pred]) + fluid.dygraph.jit.save( + se_resnext, + self.model_save_prefix, + [img], + output_spec=[pred], + ) else: fluid.dygraph.save_dygraph( se_resnext.state_dict(), - self.dy_state_dict_save_path) + self.dy_state_dict_save_path, + ) break - return pred.numpy(), avg_loss.numpy(), acc_top1.numpy( - ), acc_top5.numpy() + return ( + pred.numpy(), + avg_loss.numpy(), + acc_top1.numpy(), + acc_top5.numpy(), + ) def predict_dygraph(self, data): program_translator = ProgramTranslator() @@ -426,7 +485,8 @@ def predict_dygraph(self, data): se_resnext = SeResNeXt() model_dict, _ = fluid.dygraph.load_dygraph( - self.dy_state_dict_save_path) + self.dy_state_dict_save_path + ) se_resnext.set_dict(model_dict) se_resnext.eval() @@ -440,15 +500,22 @@ def predict_dygraph(self, data): def predict_static(self, data): paddle.enable_static() exe = fluid.Executor(place) - [inference_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(self.model_save_dir, - executor=exe, - model_filename=self.model_filename, - params_filename=self.params_filename) - - pred_res = exe.run(inference_program, - feed={feed_target_names[0]: data}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + self.model_save_dir, + executor=exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + ) + + pred_res = exe.run( + inference_program, + feed={feed_target_names[0]: data}, + fetch_list=fetch_targets, + ) return pred_res[0] @@ -462,8 +529,12 @@ def predict_dygraph_jit(self, data): return pred_res.numpy() def predict_analysis_inference(self, data): - output = PredictorTools(self.model_save_dir, self.model_filename, - self.params_filename, [data]) + output = PredictorTools( + self.model_save_dir, + self.model_filename, + self.params_filename, + [data], + ) out = output() return out @@ -477,13 +548,16 @@ def verify_predict(self): dy_pre, st_pre, rtol=1e-05, - err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre)) + err_msg='dy_pre:\n {}\n, st_pre: \n{}.'.format(dy_pre, st_pre), + ) np.testing.assert_allclose( dy_jit_pre, st_pre, rtol=1e-05, err_msg='dy_jit_pre:\n {}\n, st_pre: \n{}.'.format( - dy_jit_pre, st_pre)) + dy_jit_pre, st_pre + ), + ) flat_st_pre = st_pre.flatten() flat_predictor_pre = np.array(predictor_pre).flatten() @@ -494,34 +568,42 @@ def verify_predict(self): flat_st_pre[i], delta=1e-6, msg="predictor_pre:\n {}\n, st_pre: \n{}.".format( - flat_predictor_pre[i], flat_st_pre[i])) + flat_predictor_pre[i], flat_st_pre[i] + ), + ) def test_check_result(self): - pred_1, loss_1, acc1_1, acc5_1 = self.train(self.train_reader, - to_static=False) - pred_2, loss_2, acc1_2, acc5_2 = self.train(self.train_reader, - to_static=True) + pred_1, loss_1, acc1_1, acc5_1 = self.train( + self.train_reader, to_static=False + ) + pred_2, loss_2, acc1_2, acc5_2 = self.train( + self.train_reader, to_static=True + ) np.testing.assert_allclose( pred_1, pred_2, rtol=1e-05, - err_msg='static pred: {} \ndygraph pred: {}'.format(pred_1, pred_2)) + err_msg='static pred: {} \ndygraph pred: {}'.format(pred_1, pred_2), + ) np.testing.assert_allclose( loss_1, loss_2, rtol=1e-05, - err_msg='static loss: {} \ndygraph loss: {}'.format(loss_1, loss_2)) + err_msg='static loss: {} \ndygraph loss: {}'.format(loss_1, loss_2), + ) np.testing.assert_allclose( acc1_1, acc1_2, rtol=1e-05, - err_msg='static acc1: {} \ndygraph acc1: {}'.format(acc1_1, acc1_2)) + err_msg='static acc1: {} \ndygraph acc1: {}'.format(acc1_1, acc1_2), + ) np.testing.assert_allclose( acc5_1, acc5_2, rtol=1e-05, - err_msg='static acc5: {} \ndygraph acc5: {}'.format(acc5_1, acc5_2)) + err_msg='static acc5: {} \ndygraph acc5: {}'.format(acc5_1, acc5_2), + ) self.verify_predict() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_sentiment.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_sentiment.py index f581f45ca59ab..aab18c98ab477 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_sentiment.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_sentiment.py @@ -33,21 +33,24 @@ class SimpleConvPool(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - use_cudnn=True, - batch_size=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + use_cudnn=True, + batch_size=None, + ): super(SimpleConvPool, self).__init__() self.batch_size = batch_size - self._conv2d = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - padding=[1, 1], - use_cudnn=use_cudnn, - act='tanh') + self._conv2d = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + padding=[1, 1], + use_cudnn=use_cudnn, + act='tanh', + ) def forward(self, inputs): x = self._conv2d(inputs) @@ -57,7 +60,6 @@ def forward(self, inputs): class CNN(fluid.dygraph.Layer): - def __init__(self, dict_dim, batch_size, seq_len): super(CNN, self).__init__() self.dict_dim = dict_dim @@ -69,29 +71,37 @@ def __init__(self, dict_dim, batch_size, seq_len): self.win_size = [3, self.hid_dim] self.batch_size = batch_size self.seq_len = seq_len - self.embedding = Embedding(size=[self.dict_dim + 1, self.emb_dim], - dtype='float32', - is_sparse=False) - self._simple_conv_pool_1 = SimpleConvPool(self.channels, - self.hid_dim, - self.win_size, - batch_size=self.batch_size) - self._fc1 = Linear(input_dim=self.hid_dim * self.seq_len, - output_dim=self.fc_hid_dim, - act="softmax") - self._fc_prediction = Linear(input_dim=self.fc_hid_dim, - output_dim=self.class_dim, - act="softmax") + self.embedding = Embedding( + size=[self.dict_dim + 1, self.emb_dim], + dtype='float32', + is_sparse=False, + ) + self._simple_conv_pool_1 = SimpleConvPool( + self.channels, + self.hid_dim, + self.win_size, + batch_size=self.batch_size, + ) + self._fc1 = Linear( + input_dim=self.hid_dim * self.seq_len, + output_dim=self.fc_hid_dim, + act="softmax", + ) + self._fc_prediction = Linear( + input_dim=self.fc_hid_dim, output_dim=self.class_dim, act="softmax" + ) @declarative def forward(self, inputs, label=None): emb = self.embedding(inputs) - o_np_mask = (fluid.layers.reshape(inputs, [-1, 1]) != - self.dict_dim).astype(dtype='float32') + o_np_mask = ( + fluid.layers.reshape(inputs, [-1, 1]) != self.dict_dim + ).astype(dtype='float32') mask_emb = fluid.layers.expand(o_np_mask, [1, self.hid_dim]) emb = emb * mask_emb emb = fluid.layers.reshape( - emb, shape=[-1, self.channels, self.seq_len, self.hid_dim]) + emb, shape=[-1, self.channels, self.seq_len, self.hid_dim] + ) conv_3 = self._simple_conv_pool_1(emb) fc_1 = self._fc1(conv_3) prediction = self._fc_prediction(fc_1) @@ -103,7 +113,6 @@ def forward(self, inputs, label=None): class BOW(fluid.dygraph.Layer): - def __init__(self, dict_dim, batch_size, seq_len): super(BOW, self).__init__() self.dict_dim = dict_dim @@ -113,24 +122,27 @@ def __init__(self, dict_dim, batch_size, seq_len): self.class_dim = 2 self.batch_size = batch_size self.seq_len = seq_len - self.embedding = Embedding(size=[self.dict_dim + 1, self.emb_dim], - dtype='float32', - is_sparse=False) - self._fc1 = Linear(input_dim=self.hid_dim, - output_dim=self.hid_dim, - act="tanh") - self._fc2 = Linear(input_dim=self.hid_dim, - output_dim=self.fc_hid_dim, - act="tanh") - self._fc_prediction = Linear(input_dim=self.fc_hid_dim, - output_dim=self.class_dim, - act="softmax") + self.embedding = Embedding( + size=[self.dict_dim + 1, self.emb_dim], + dtype='float32', + is_sparse=False, + ) + self._fc1 = Linear( + input_dim=self.hid_dim, output_dim=self.hid_dim, act="tanh" + ) + self._fc2 = Linear( + input_dim=self.hid_dim, output_dim=self.fc_hid_dim, act="tanh" + ) + self._fc_prediction = Linear( + input_dim=self.fc_hid_dim, output_dim=self.class_dim, act="softmax" + ) @declarative def forward(self, inputs, label=None): emb = self.embedding(inputs) - o_np_mask = (fluid.layers.reshape(inputs, [-1, 1]) != - self.dict_dim).astype(dtype='float32') + o_np_mask = ( + fluid.layers.reshape(inputs, [-1, 1]) != self.dict_dim + ).astype(dtype='float32') mask_emb = fluid.layers.expand(o_np_mask, [1, self.hid_dim]) emb = emb * mask_emb emb = fluid.layers.reshape(emb, shape=[-1, self.seq_len, self.hid_dim]) @@ -147,7 +159,6 @@ def forward(self, inputs, label=None): class GRU(fluid.dygraph.Layer): - def __init__(self, dict_dim, batch_size, seq_len): super(GRU, self).__init__() self.dict_dim = dict_dim @@ -157,30 +168,34 @@ def __init__(self, dict_dim, batch_size, seq_len): self.class_dim = 2 self.batch_size = batch_size self.seq_len = seq_len - self.embedding = Embedding(size=[self.dict_dim + 1, self.emb_dim], - dtype='float32', - param_attr=fluid.ParamAttr(learning_rate=30), - is_sparse=False) + self.embedding = Embedding( + size=[self.dict_dim + 1, self.emb_dim], + dtype='float32', + param_attr=fluid.ParamAttr(learning_rate=30), + is_sparse=False, + ) h_0 = np.zeros((self.batch_size, self.hid_dim), dtype="float32") h_0 = to_variable(h_0) self._fc1 = Linear(input_dim=self.hid_dim, output_dim=self.hid_dim * 3) - self._fc2 = Linear(input_dim=self.hid_dim, - output_dim=self.fc_hid_dim, - act="tanh") - self._fc_prediction = Linear(input_dim=self.fc_hid_dim, - output_dim=self.class_dim, - act="softmax") + self._fc2 = Linear( + input_dim=self.hid_dim, output_dim=self.fc_hid_dim, act="tanh" + ) + self._fc_prediction = Linear( + input_dim=self.fc_hid_dim, output_dim=self.class_dim, act="softmax" + ) self._gru = DynamicGRU(size=self.hid_dim, h_0=h_0) @declarative def forward(self, inputs, label=None): emb = self.embedding(inputs) - o_np_mask = (fluid.layers.reshape(inputs, [-1, 1]) != - self.dict_dim).astype('float32') + o_np_mask = ( + fluid.layers.reshape(inputs, [-1, 1]) != self.dict_dim + ).astype('float32') mask_emb = fluid.layers.expand(o_np_mask, [1, self.hid_dim]) emb = emb * mask_emb - emb = fluid.layers.reshape(emb, - shape=[self.batch_size, -1, self.hid_dim]) + emb = fluid.layers.reshape( + emb, shape=[self.batch_size, -1, self.hid_dim] + ) fc_1 = self._fc1(emb) gru_hidden = self._gru(fc_1) gru_hidden = fluid.layers.reduce_max(gru_hidden, dim=1) @@ -195,7 +210,6 @@ def forward(self, inputs, label=None): class BiGRU(fluid.dygraph.Layer): - def __init__(self, dict_dim, batch_size, seq_len): super(BiGRU, self).__init__() self.dict_dim = dict_dim @@ -205,42 +219,47 @@ def __init__(self, dict_dim, batch_size, seq_len): self.class_dim = 2 self.batch_size = batch_size self.seq_len = seq_len - self.embedding = Embedding(size=[self.dict_dim + 1, self.emb_dim], - dtype='float32', - param_attr=fluid.ParamAttr(learning_rate=30), - is_sparse=False) + self.embedding = Embedding( + size=[self.dict_dim + 1, self.emb_dim], + dtype='float32', + param_attr=fluid.ParamAttr(learning_rate=30), + is_sparse=False, + ) h_0 = np.zeros((self.batch_size, self.hid_dim), dtype="float32") h_0 = to_variable(h_0) self._fc1 = Linear(input_dim=self.hid_dim, output_dim=self.hid_dim * 3) - self._fc2 = Linear(input_dim=self.hid_dim * 2, - output_dim=self.fc_hid_dim, - act="tanh") - self._fc_prediction = Linear(input_dim=self.fc_hid_dim, - output_dim=self.class_dim, - act="softmax") - self._gru_forward = DynamicGRU(size=self.hid_dim, - h_0=h_0, - is_reverse=False) - self._gru_backward = DynamicGRU(size=self.hid_dim, - h_0=h_0, - is_reverse=True) + self._fc2 = Linear( + input_dim=self.hid_dim * 2, output_dim=self.fc_hid_dim, act="tanh" + ) + self._fc_prediction = Linear( + input_dim=self.fc_hid_dim, output_dim=self.class_dim, act="softmax" + ) + self._gru_forward = DynamicGRU( + size=self.hid_dim, h_0=h_0, is_reverse=False + ) + self._gru_backward = DynamicGRU( + size=self.hid_dim, h_0=h_0, is_reverse=True + ) @declarative def forward(self, inputs, label=None): emb = self.embedding(inputs) - o_np_mask = (fluid.layers.reshape(inputs, [-1, 1]) != - self.dict_dim).astype('float32') + o_np_mask = ( + fluid.layers.reshape(inputs, [-1, 1]) != self.dict_dim + ).astype('float32') mask_emb = fluid.layers.expand(o_np_mask, [1, self.hid_dim]) emb = emb * mask_emb - emb = fluid.layers.reshape(emb, - shape=[self.batch_size, -1, self.hid_dim]) + emb = fluid.layers.reshape( + emb, shape=[self.batch_size, -1, self.hid_dim] + ) fc_1 = self._fc1(emb) gru_forward = self._gru_forward(fc_1) gru_backward = self._gru_backward(fc_1) gru_forward_tanh = fluid.layers.tanh(gru_forward) gru_backward_tanh = fluid.layers.tanh(gru_backward) encoded_vector = fluid.layers.concat( - input=[gru_forward_tanh, gru_backward_tanh], axis=2) + input=[gru_forward_tanh, gru_backward_tanh], axis=2 + ) encoded_vector = fluid.layers.reduce_max(encoded_vector, dim=1) fc_2 = self._fc2(encoded_vector) prediction = self._fc_prediction(fc_2) @@ -261,11 +280,13 @@ def reader(): batch_data = [] while True: label = local_random.randint(0, class_num) - seq_len = local_random.randint(padding_size // 2, - int(padding_size * 1.2)) + seq_len = local_random.randint( + padding_size // 2, int(padding_size * 1.2) + ) word_ids = local_random.randint(0, vocab_size, [seq_len]).tolist() - word_ids = word_ids[:padding_size] + [vocab_size - ] * (padding_size - seq_len) + word_ids = word_ids[:padding_size] + [vocab_size] * ( + padding_size - seq_len + ) batch_data.append((word_ids, [label], seq_len)) if len(batch_data) == batch_size: yield batch_data @@ -287,16 +308,20 @@ class Args(object): def train(args, to_static): program_translator.enable(to_static) - place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ + place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + ) with fluid.dygraph.guard(place): np.random.seed(SEED) paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) - train_reader = fake_data_reader(args.class_num, args.vocab_size, - args.batch_size, args.padding_size) + train_reader = fake_data_reader( + args.class_num, args.vocab_size, args.batch_size, args.padding_size + ) train_loader = fluid.io.DataLoader.from_generator(capacity=24) train_loader.set_sample_list_generator(train_reader) @@ -309,7 +334,8 @@ def train(args, to_static): elif args.model_type == 'bigru_net': model = BiGRU(args.vocab_size, args.batch_size, args.padding_size) sgd_optimizer = fluid.optimizer.Adagrad( - learning_rate=args.lr, parameter_list=model.parameters()) + learning_rate=args.lr, parameter_list=model.parameters() + ) loss_data = [] for eop in range(args.epoch): @@ -333,9 +359,14 @@ def train(args, to_static): # used_time may be 0.0, cause zero division error if used_time < 1e-5: used_time = 1e-5 - print("step: %d, ave loss: %f, speed: %f steps/s" % - (batch_id, avg_cost.numpy()[0], - args.log_step / used_time)) + print( + "step: %d, ave loss: %f, speed: %f steps/s" + % ( + batch_id, + avg_cost.numpy()[0], + args.log_step / used_time, + ) + ) time_begin = time.time() if batch_id == args.train_step: @@ -345,7 +376,6 @@ def train(args, to_static): class TestSentiment(unittest.TestCase): - def setUp(self): self.args = Args() @@ -353,11 +383,12 @@ def train_model(self, model_type='cnn_net'): self.args.model_type = model_type st_out = train(self.args, True) dy_out = train(self.args, False) - np.testing.assert_allclose(dy_out, - st_out, - rtol=1e-05, - err_msg='dy_out:\n {}\n st_out:\n {}'.format( - dy_out, st_out)) + np.testing.assert_allclose( + dy_out, + st_out, + rtol=1e-05, + err_msg='dy_out:\n {}\n st_out:\n {}'.format(dy_out, st_out), + ) def test_train(self): model_types = ['cnn_net', 'bow_net', 'gru_net', 'bigru_net'] diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_seq2seq.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_seq2seq.py index 7ed2d12f5a810..d4932b710c391 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_seq2seq.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_seq2seq.py @@ -26,8 +26,9 @@ from seq2seq_utils import Seq2SeqModelHyperParams from seq2seq_utils import get_data_iter -place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) program_translator = ProgramTranslator() STEP_NUM = 10 PRINT_STEP = 2 @@ -51,26 +52,32 @@ def train(args, attn_model=False): fluid.default_main_program().random_seed = 2020 if attn_model: - model = AttentionModel(args.hidden_size, - args.src_vocab_size, - args.tar_vocab_size, - args.batch_size, - num_layers=args.num_layers, - init_scale=args.init_scale, - dropout=args.dropout) + model = AttentionModel( + args.hidden_size, + args.src_vocab_size, + args.tar_vocab_size, + args.batch_size, + num_layers=args.num_layers, + init_scale=args.init_scale, + dropout=args.dropout, + ) else: - model = BaseModel(args.hidden_size, - args.src_vocab_size, - args.tar_vocab_size, - args.batch_size, - num_layers=args.num_layers, - init_scale=args.init_scale, - dropout=args.dropout) + model = BaseModel( + args.hidden_size, + args.src_vocab_size, + args.tar_vocab_size, + args.batch_size, + num_layers=args.num_layers, + init_scale=args.init_scale, + dropout=args.dropout, + ) gloabl_norm_clip = GradientClipByGlobalNorm(args.max_grad_norm) - optimizer = fluid.optimizer.SGD(args.learning_rate, - parameter_list=model.parameters(), - grad_clip=gloabl_norm_clip) + optimizer = fluid.optimizer.SGD( + args.learning_rate, + parameter_list=model.parameters(), + grad_clip=gloabl_norm_clip, + ) model.train() train_data_iter = get_data_iter(args.batch_size) @@ -96,8 +103,15 @@ def train(args, attn_model=False): if batch_id % PRINT_STEP == 0: print( "Batch:[%d]; Time: %.5f s; loss: %.5f; total_loss: %.5f; word num: %.5f; ppl: %.5f" - % (batch_id, batch_time, loss.numpy(), total_loss.numpy(), - word_count, np.exp(total_loss.numpy() / word_count))) + % ( + batch_id, + batch_time, + loss.numpy(), + total_loss.numpy(), + word_count, + np.exp(total_loss.numpy() / word_count), + ) + ) if attn_model: # NOTE: Please see code of AttentionModel. @@ -108,7 +122,9 @@ def train(args, attn_model=False): if batch_id + 1 >= STEP_NUM: break - model_path = args.attn_model_path if attn_model else args.base_model_path + model_path = ( + args.attn_model_path if attn_model else args.base_model_path + ) model_dir = os.path.join(model_path) if not os.path.exists(model_dir): @@ -121,27 +137,33 @@ def infer(args, attn_model=False): with fluid.dygraph.guard(place): if attn_model: - model = AttentionModel(args.hidden_size, - args.src_vocab_size, - args.tar_vocab_size, - args.batch_size, - beam_size=args.beam_size, - num_layers=args.num_layers, - init_scale=args.init_scale, - dropout=0.0, - mode='beam_search') + model = AttentionModel( + args.hidden_size, + args.src_vocab_size, + args.tar_vocab_size, + args.batch_size, + beam_size=args.beam_size, + num_layers=args.num_layers, + init_scale=args.init_scale, + dropout=0.0, + mode='beam_search', + ) else: - model = BaseModel(args.hidden_size, - args.src_vocab_size, - args.tar_vocab_size, - args.batch_size, - beam_size=args.beam_size, - num_layers=args.num_layers, - init_scale=args.init_scale, - dropout=0.0, - mode='beam_search') - - model_path = args.attn_model_path if attn_model else args.base_model_path + model = BaseModel( + args.hidden_size, + args.src_vocab_size, + args.tar_vocab_size, + args.batch_size, + beam_size=args.beam_size, + num_layers=args.num_layers, + init_scale=args.init_scale, + dropout=0.0, + mode='beam_search', + ) + + model_path = ( + args.attn_model_path if attn_model else args.base_model_path + ) state_dict, _ = fluid.dygraph.load_dygraph(model_path) model.set_dict(state_dict) model.eval() @@ -158,16 +180,18 @@ def infer(args, attn_model=False): class TestSeq2seq(unittest.TestCase): - def setUp(self): self.args = Seq2SeqModelHyperParams self.temp_dir = tempfile.TemporaryDirectory() - self.args.base_model_path = os.path.join(self.temp_dir.name, - self.args.base_model_path) - self.args.attn_model_path = os.path.join(self.temp_dir.name, - self.args.attn_model_path) - self.args.reload_model = os.path.join(self.temp_dir.name, - self.args.reload_model) + self.args.base_model_path = os.path.join( + self.temp_dir.name, self.args.base_model_path + ) + self.args.attn_model_path = os.path.join( + self.temp_dir.name, self.args.attn_model_path + ) + self.args.reload_model = os.path.join( + self.temp_dir.name, self.args.reload_model + ) def tearDown(self): self.temp_dir.cleanup() @@ -190,17 +214,23 @@ def _test_train(self, attn_model=False): dygraph_loss = self.run_dygraph(mode="train", attn_model=attn_model) static_loss = self.run_static(mode="train", attn_model=attn_model) result = np.allclose(dygraph_loss, static_loss) - self.assertTrue(result, - msg="\ndygraph_loss = {} \nstatic_loss = {}".format( - dygraph_loss, static_loss)) + self.assertTrue( + result, + msg="\ndygraph_loss = {} \nstatic_loss = {}".format( + dygraph_loss, static_loss + ), + ) def _test_predict(self, attn_model=False): pred_dygraph = self.run_dygraph(mode="test", attn_model=attn_model) pred_static = self.run_static(mode="test", attn_model=attn_model) result = np.allclose(pred_static, pred_dygraph) - self.assertTrue(result, - msg="\npred_dygraph = {} \npred_static = {}".format( - pred_dygraph, pred_static)) + self.assertTrue( + result, + msg="\npred_dygraph = {} \npred_static = {}".format( + pred_dygraph, pred_static + ), + ) def test_base_model(self): self._test_train(attn_model=False) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_setter_helper.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_setter_helper.py index 0a5d567996d92..405eb089d0c51 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_setter_helper.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_setter_helper.py @@ -28,10 +28,10 @@ def setter(values): class TestGetterSetterHelper(unittest.TestCase): - def test_1(self): - helper = GetterSetterHelper(getter, setter, ['a', 'b', 'e'], - ['d', 'f', 'e']) + helper = GetterSetterHelper( + getter, setter, ['a', 'b', 'e'], ['d', 'f', 'e'] + ) print(helper.union()) expect_union = ['a', 'b', 'd', 'e', 'f'] assert helper.union() == expect_union diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet.py index 2bc344ae95a6b..466c6affcb846 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet.py @@ -36,22 +36,24 @@ def create_conf_dict(): def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument("--batch_size", - type=int, - default=32, - help="Total examples' number in batch for training.") - parser.add_argument("--seq_len", - type=int, - default=32, - help="The length of each sentence.") - parser.add_argument("--epoch", - type=int, - default=1, - help="The number of training epoch.") - parser.add_argument("--fake_sample_size", - type=int, - default=128, - help="The number of samples of fake data.") + parser.add_argument( + "--batch_size", + type=int, + default=32, + help="Total examples' number in batch for training.", + ) + parser.add_argument( + "--seq_len", type=int, default=32, help="The length of each sentence." + ) + parser.add_argument( + "--epoch", type=int, default=1, help="The number of training epoch." + ) + parser.add_argument( + "--fake_sample_size", + type=int, + default=128, + help="The number of samples of fake data.", + ) args = parser.parse_args([]) return args @@ -72,7 +74,6 @@ def fake_vocabulary(): class FakeReaderProcessor(object): - def __init__(self, args, vocab): self.vocab = vocab self.seq_len = args.seq_len @@ -83,10 +84,10 @@ def __init__(self, args, vocab): pos_title = query[:] neg_title = [26 - q for q in query] self.data_samples.append( - np.array([query, pos_title, neg_title]).astype(np.int64)) + np.array([query, pos_title, neg_title]).astype(np.int64) + ) def get_reader(self, mode, epoch=0): - def reader_with_pairwise(): if mode == "train": for i in range(self.sample_size): @@ -125,7 +126,8 @@ def train(conf_dict, to_static): beta1=0.9, beta2=0.999, epsilon=1e-08, - parameter_list=net.parameters()) + parameter_list=net.parameters(), + ) metric = fluid.metrics.Auc(name="auc") @@ -133,14 +135,14 @@ def train(conf_dict, to_static): losses = [] train_loader = fluid.io.DataLoader.from_generator( - capacity=16, - return_list=True, - iterable=True, - use_double_buffer=True) - get_train_examples = simnet_process.get_reader("train", - epoch=args.epoch) + capacity=16, return_list=True, iterable=True, use_double_buffer=True + ) + get_train_examples = simnet_process.get_reader( + "train", epoch=args.epoch + ) train_loader.set_sample_list_generator( - paddle.batch(get_train_examples, batch_size=args.batch_size), place) + paddle.batch(get_train_examples, batch_size=args.batch_size), place + ) for left, pos_right, neg_right in train_loader(): left = fluid.layers.reshape(left, shape=[-1, 1]) @@ -160,7 +162,6 @@ def train(conf_dict, to_static): class TestSimnet(unittest.TestCase): - def test_dygraph_static_same_loss(self): if fluid.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py index f2c72e9932ea7..bbb408e48c328 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py @@ -34,22 +34,24 @@ def create_conf_dict(): def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument("--batch_size", - type=int, - default=32, - help="Total examples' number in batch for training.") - parser.add_argument("--seq_len", - type=int, - default=32, - help="The length of each sentence.") - parser.add_argument("--epoch", - type=int, - default=1, - help="The number of training epoch.") - parser.add_argument("--fake_sample_size", - type=int, - default=128, - help="The number of samples of fake data.") + parser.add_argument( + "--batch_size", + type=int, + default=32, + help="Total examples' number in batch for training.", + ) + parser.add_argument( + "--seq_len", type=int, default=32, help="The length of each sentence." + ) + parser.add_argument( + "--epoch", type=int, default=1, help="The number of training epoch." + ) + parser.add_argument( + "--fake_sample_size", + type=int, + default=128, + help="The number of samples of fake data.", + ) args = parser.parse_args([]) return args @@ -70,7 +72,6 @@ def fake_vocabulary(): class FakeReaderProcessor(object): - def __init__(self, args, vocab): self.vocab = vocab self.seq_len = args.seq_len @@ -81,10 +82,10 @@ def __init__(self, args, vocab): pos_title = query[:] neg_title = [26 - q for q in query] self.data_samples.append( - np.array([query, pos_title, neg_title]).astype(np.int64)) + np.array([query, pos_title, neg_title]).astype(np.int64) + ) def get_reader(self, mode, epoch=0): - def reader_with_pairwise(): if mode == "train": for i in range(self.sample_size): @@ -118,24 +119,26 @@ def train(conf_dict, to_static): net = BOW(conf_dict) loss = HingeLoss(conf_dict) - optimizer = paddle.optimizer.Adam(learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - parameters=net.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + parameters=net.parameters(), + ) metric = paddle.metric.Auc(name="auc") global_step = 0 losses = [] - train_loader = paddle.io.DataLoader.from_generator(capacity=16, - return_list=True, - iterable=True, - use_double_buffer=True) + train_loader = paddle.io.DataLoader.from_generator( + capacity=16, return_list=True, iterable=True, use_double_buffer=True + ) get_train_examples = simnet_process.get_reader("train", epoch=args.epoch) train_loader.set_sample_list_generator( - paddle.batch(get_train_examples, batch_size=args.batch_size), place) + paddle.batch(get_train_examples, batch_size=args.batch_size), place + ) for left, pos_right, neg_right in train_loader(): left = paddle.reshape(left, shape=[-1, 1]) @@ -156,7 +159,6 @@ def train(conf_dict, to_static): class TestSimnet(unittest.TestCase): - def test_dygraph_static_same_loss(self): if paddle.is_compiled_with_cuda(): paddle.fluid.set_flags({"FLAGS_cudnn_deterministic": True}) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_slice.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_slice.py index 96bcc6d016e0c..ffd3e8879334d 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_slice.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_slice.py @@ -97,7 +97,6 @@ def test_set_value(x): class LayerWithSetValue(paddle.nn.Layer): - def __init__(self, input_dim, hidden): super(LayerWithSetValue, self).__init__() self.linear = paddle.nn.Linear(input_dim, hidden) @@ -110,11 +109,13 @@ def forward(self, x): class TestSliceWithoutControlFlow(unittest.TestCase): - def setUp(self): self.init_input() - self.place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.init_dygraph_func() paddle.disable_static() @@ -142,25 +143,21 @@ def test_transformed_static_result(self): class TestSliceInIf(TestSliceWithoutControlFlow): - def init_dygraph_func(self): self.dygraph_func = test_slice_in_if class TestSliceInWhileLoop(TestSliceWithoutControlFlow): - def init_dygraph_func(self): self.dygraph_func = test_slice_in_while_loop class TestSliceInForLoop(TestSliceWithoutControlFlow): - def init_dygraph_func(self): self.dygraph_func = test_slice_in_for_loop class TestSetValue(TestSliceWithoutControlFlow): - def init_input(self): self.input = np.full([3, 4, 5], 5).astype('float32') @@ -169,11 +166,11 @@ def init_dygraph_func(self): class TestSetValueWithLayerAndSave(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() - self.model_path = os.path.join(self.temp_dir.name, - "layer_use_set_value") + self.model_path = os.path.join( + self.temp_dir.name, "layer_use_set_value" + ) def tearDown(self): self.temp_dir.cleanup() @@ -182,10 +179,9 @@ def test_set_value_with_save(self): prog_trans.enable(True) model = LayerWithSetValue(input_dim=10, hidden=1) x = paddle.full(shape=[5, 10], fill_value=5.0, dtype="float32") - paddle.jit.save(layer=model, - path=self.model_path, - input_spec=[x], - output_spec=None) + paddle.jit.save( + layer=model, path=self.model_path, input_spec=[x], output_spec=None + ) class TestSliceSupplementSpecialCase(unittest.TestCase): @@ -218,18 +214,20 @@ def func(inps): return inps[::2], inps[::-2] origin_result = func(inps) - sfunc = paddle.jit.to_static(func, - input_spec=[InputSpec(shape=[None, 4, 4])]) + sfunc = paddle.jit.to_static( + func, input_spec=[InputSpec(shape=[None, 4, 4])] + ) static_result = sfunc(inps) - np.testing.assert_array_equal(origin_result[0].numpy(), - static_result[0].numpy()) - np.testing.assert_array_equal(origin_result[1].numpy(), - static_result[1].numpy()) + np.testing.assert_array_equal( + origin_result[0].numpy(), static_result[0].numpy() + ) + np.testing.assert_array_equal( + origin_result[1].numpy(), static_result[1].numpy() + ) class TestPaddleStridedSlice(unittest.TestCase): - def test_compare_paddle_strided_slice_with_numpy(self): paddle.disable_static() array = np.arange(5) @@ -238,19 +236,21 @@ def test_compare_paddle_strided_slice_with_numpy(self): s1 = 3 e1 = 1 stride1 = -2 - sl = paddle.strided_slice(pt, - axes=[ - 0, - ], - starts=[ - s1, - ], - ends=[ - e1, - ], - strides=[ - stride1, - ]) + sl = paddle.strided_slice( + pt, + axes=[ + 0, + ], + starts=[ + s1, + ], + ends=[ + e1, + ], + strides=[ + stride1, + ], + ) self.assertTrue(array[s1:e1:stride1], sl) @@ -259,27 +259,27 @@ def test_compare_paddle_strided_slice_with_numpy(self): s2 = [8, -1] e2 = [1, -5] stride2 = [-2, -3] - sl = paddle.strided_slice(pt, - axes=[0, 1], - starts=s2, - ends=e2, - strides=stride2) + sl = paddle.strided_slice( + pt, axes=[0, 1], starts=s2, ends=e2, strides=stride2 + ) np.testing.assert_array_equal( - sl.numpy(), array[s2[0]:e2[0]:stride2[0], s2[1]:e2[1]:stride2[1]]) + sl.numpy(), + array[s2[0] : e2[0] : stride2[0], s2[1] : e2[1] : stride2[1]], + ) array = np.arange(6 * 7 * 8).reshape((6, 7, 8)) pt = paddle.to_tensor(array) s2 = [7, -1] e2 = [2, -5] stride2 = [-2, -3] - sl = paddle.strided_slice(pt, - axes=[0, 2], - starts=s2, - ends=e2, - strides=stride2) + sl = paddle.strided_slice( + pt, axes=[0, 2], starts=s2, ends=e2, strides=stride2 + ) - array_slice = array[s2[0]:e2[0]:stride2[0], ::, s2[1]:e2[1]:stride2[1]] + array_slice = array[ + s2[0] : e2[0] : stride2[0], ::, s2[1] : e2[1] : stride2[1] + ] np.testing.assert_array_equal(sl.numpy(), array_slice) @@ -289,7 +289,6 @@ def slice_zero_shape_tensor(x): class TestSliceZeroShapeTensor(unittest.TestCase): - def test_slice(self): paddle.disable_static() x = paddle.ones([0, 0, 0, 0]) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py index cfbf47a318ffa..8ee2fcd0d593e 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py @@ -18,7 +18,6 @@ class Net(Layer): - def __init__(self): super(Net, self).__init__() self.fc = paddle.nn.Linear(16, 3) @@ -36,7 +35,6 @@ def forward(self, x, y, m, n): class TestArgsSpecName(unittest.TestCase): - def read_from_dataset(self): self.x = paddle.randn([4, 2, 8]) self.y = paddle.randn([4, 2, 8]) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py index 8f06b96122d25..7a2e0e09b2145 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_static_analysis.py @@ -19,7 +19,10 @@ import paddle.fluid as fluid import unittest -from paddle.fluid.dygraph.dygraph_to_static import NodeVarType, StaticAnalysisVisitor +from paddle.fluid.dygraph.dygraph_to_static import ( + NodeVarType, + StaticAnalysisVisitor, +) def func_to_test1(a, b): @@ -71,7 +74,7 @@ def func_to_test3(): 'i': {NodeVarType.BOOLEAN}, 'j': {NodeVarType.UNKNOWN}, 'k': {NodeVarType.FLOAT}, - 'l': {NodeVarType.PADDLE_RETURN_TYPES} + 'l': {NodeVarType.PADDLE_RETURN_TYPES}, } @@ -87,12 +90,11 @@ def func_to_test4(): 'a': {NodeVarType.NUMPY_NDARRAY}, 'b': {NodeVarType.NUMPY_NDARRAY}, 'c': {NodeVarType.TENSOR}, - 'd': {NodeVarType.TENSOR} + 'd': {NodeVarType.TENSOR}, } def func_to_test5(): - def inner_int_func(): return 1 @@ -139,7 +141,7 @@ def add(x, y): 'i': {NodeVarType.INT}, 'x': {NodeVarType.INT}, 'y': {NodeVarType.INT}, - 'add': {NodeVarType.INT} + 'add': {NodeVarType.INT}, } @@ -156,21 +158,30 @@ def func_to_test7(a: int, b: float, c: paddle.Tensor, d: float = 'diff'): 'd': {NodeVarType.STRING}, 'e': {NodeVarType.PADDLE_RETURN_TYPES}, 'f': {NodeVarType.PADDLE_RETURN_TYPES}, - 'g': {NodeVarType.TENSOR} + 'g': {NodeVarType.TENSOR}, } test_funcs = [ - func_to_test1, func_to_test2, func_to_test3, func_to_test4, func_to_test5, - func_to_test6, func_to_test7 + func_to_test1, + func_to_test2, + func_to_test3, + func_to_test4, + func_to_test5, + func_to_test6, + func_to_test7, ] result_var_type = [ - result_var_type1, result_var_type2, result_var_type3, result_var_type4, - result_var_type5, result_var_type6, result_var_type7 + result_var_type1, + result_var_type2, + result_var_type3, + result_var_type4, + result_var_type5, + result_var_type6, + result_var_type7, ] class TestStaticAnalysis(unittest.TestCase): - def _check_wrapper(self, wrapper, node_to_wrapper_map): self.assertEqual(node_to_wrapper_map[wrapper.node], wrapper) if wrapper.parent is not None: diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_methods.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_methods.py index 3d88a03247ce9..ecda3427e7ea2 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_methods.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_methods.py @@ -25,7 +25,6 @@ def tensor_clone(x): class TestTensorClone(unittest.TestCase): - def _run(self, to_static): prog_trans = paddle.jit.ProgramTranslator() prog_trans.enable(to_static) @@ -46,7 +45,6 @@ def tensor_numpy(x): class TestTensorDygraphOnlyMethodError(unittest.TestCase): - def _run(self, to_static): prog_trans = paddle.jit.ProgramTranslator() prog_trans.enable(to_static) @@ -68,7 +66,6 @@ def tensor_item(x): class TestTensorItem(unittest.TestCase): - def _run(self, to_static): prog_trans = paddle.jit.ProgramTranslator() prog_trans.enable(to_static) @@ -92,7 +89,6 @@ def tensor_size(x): class TestTensorSize(unittest.TestCase): - def _run(self, to_static): prog_trans = paddle.jit.ProgramTranslator() prog_trans.enable(to_static) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_shape.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_shape.py index 55412fb0c49a5..5d06d1c694ec6 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_shape.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tensor_shape.py @@ -109,13 +109,13 @@ def dyfunc_with_if_1(x): # `res.shape[0]` is transformed into # `paddle.jit.dy2static.convert_var_shape(res)[0]` if res.shape[0] > 1: - res = fluid.layers.fill_constant(value=2, - shape=x.shape, - dtype="int32") + res = fluid.layers.fill_constant( + value=2, shape=x.shape, dtype="int32" + ) else: - res = fluid.layers.fill_constant(value=3, - shape=x.shape, - dtype="int32") + res = fluid.layers.fill_constant( + value=3, shape=x.shape, dtype="int32" + ) return res @@ -231,11 +231,13 @@ def dyfunc_dict_assign_shape(): # 1. Basic tests without control flow class TestTensorShapeBasic(unittest.TestCase): - def setUp(self): self.input = np.ones(5).astype("int32") - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self._set_input_spec() self._set_expected_op_num() self.init_test_func() @@ -277,9 +279,11 @@ def _compute_op_num(self, program): for block in program.blocks: self.shape_op_num += len( - [op for op in block.ops if op.type == "shape"]) + [op for op in block.ops if op.type == "shape"] + ) self.slice_op_num += len( - [op for op in block.ops if op.type == "slice"]) + [op for op in block.ops if op.type == "slice"] + ) def test_op_num(self): static_layer = paddle.jit.to_static(self.dygraph_func, self.input_spec) @@ -291,7 +295,6 @@ def test_op_num(self): class TestTensorShapeBasic2(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_tensor_shape_2 @@ -302,19 +305,16 @@ def _set_expected_op_num(self): class TestTensorShapeBasic3(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_tensor_shape_3 class TestTensorShapeBasic4(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_tensor_shape_4 class TestTensorShapeBasic5(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_tensor_shape_5 @@ -325,7 +325,6 @@ def _set_expected_op_num(self): class TestTensorShapeBasic6(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_tensor_shape_6 @@ -336,7 +335,6 @@ def _set_expected_op_num(self): class TestTupleShape1(TestTensorShapeBasic): - def init_test_func(self): self.input = np.ones((5, 7)).astype("int32") self.input_spec = [ @@ -351,7 +349,6 @@ def _set_expected_op_num(self): class TestTupleShape2(TestTensorShapeBasic): - def init_test_func(self): self.input = np.ones((5, 7)).astype("int32") self.input_spec = [ @@ -366,7 +363,6 @@ def _set_expected_op_num(self): class TestTupleShape3(TestTensorShapeBasic): - def init_test_func(self): self.input = np.ones((5, 7)).astype("int32") self.input_spec = [paddle.static.InputSpec(shape=[5, 7], dtype="int32")] @@ -379,7 +375,6 @@ def _set_expected_op_num(self): class TestPaddleShapeApi(TestTensorShapeBasic): - def init_test_func(self): self.input = np.ones((5, 7)).astype("int32") self.input_spec = [paddle.static.InputSpec(shape=[5, 7], dtype="int32")] @@ -393,7 +388,6 @@ def _set_expected_op_num(self): # 2. Tests with control flow if class TestTensorShapeInIf1(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_with_if_1 @@ -404,7 +398,6 @@ def _set_expected_op_num(self): class TestTensorShapeInIf2(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_with_if_2 @@ -416,7 +409,6 @@ def _set_expected_op_num(self): # 3. Tests with control flow for loop class TestTensorShapeInFor1(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_with_for_1 @@ -427,7 +419,6 @@ def _set_expected_op_num(self): class TestTensorShapeInFor2(TestTensorShapeInFor1): - def init_test_func(self): self.dygraph_func = dyfunc_with_for_2 @@ -438,7 +429,6 @@ def _set_expected_op_num(self): class TestTensorShapeInFor3(TestTensorShapeInFor1): - def init_test_func(self): self.dygraph_func = dyfunc_with_for_3 @@ -450,7 +440,6 @@ def _set_expected_op_num(self): # 4. Tests with control flow while loop class TestTensorShapeInWhile1(TestTensorShapeInFor1): - def init_test_func(self): self.dygraph_func = dyfunc_with_while_1 @@ -461,7 +450,6 @@ def _set_expected_op_num(self): class TestTensorShapeInWhile2(TestTensorShapeInFor1): - def init_test_func(self): self.dygraph_func = dyfunc_with_while_2 @@ -472,7 +460,6 @@ def _set_expected_op_num(self): class TestTensorShapeInWhile3(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_with_while_3 @@ -483,7 +470,6 @@ def _set_expected_op_num(self): class TestTensorShapeInWhile4(TestTensorShapeBasic): - def init_test_func(self): self.dygraph_func = dyfunc_with_while_4 @@ -495,7 +481,6 @@ def _set_expected_op_num(self): # 5. Test op num for negetive dim class TestOpNumBasicWithTensorShape(unittest.TestCase): - def setUp(self): self._set_input_spec() self._set_test_func() @@ -521,9 +506,11 @@ def _compute_op_num(self, program): for block in program.blocks: self.shape_op_num += len( - [op for op in block.ops if op.type == "shape"]) + [op for op in block.ops if op.type == "shape"] + ) self.slice_op_num += len( - [op for op in block.ops if op.type == "slice"]) + [op for op in block.ops if op.type == "slice"] + ) def test_op_num(self): static_layer = paddle.jit.to_static(self.dygraph_func, self.input_spec) @@ -536,7 +523,6 @@ def test_op_num(self): class TestOpNumBasicWithTensorShape4(TestOpNumBasicWithTensorShape): - def _set_test_func(self): self.dygraph_func = dyfunc_tensor_shape_4 @@ -547,7 +533,6 @@ def _set_expected_op_num(self): class TestOpNumWithTensorShapeTuple1(TestOpNumBasicWithTensorShape): - def _set_test_func(self): self.dygraph_func = dyfunc_tuple_shape_1 @@ -558,7 +543,6 @@ def _set_expected_op_num(self): class TestOpNumWithTensorShapeInIf1(TestOpNumBasicWithTensorShape): - def _set_test_func(self): self.dygraph_func = dyfunc_with_if_1 @@ -569,7 +553,6 @@ def _set_expected_op_num(self): class TestOpNumWithTensorShapeInFor1(TestOpNumBasicWithTensorShape): - def _set_test_func(self): self.dygraph_func = dyfunc_with_for_1 @@ -580,7 +563,6 @@ def _set_expected_op_num(self): class TestOpNumWithTensorShapeInWhile1(TestOpNumBasicWithTensorShape): - def _set_test_func(self): self.dygraph_func = dyfunc_with_while_1 @@ -591,7 +573,6 @@ def _set_expected_op_num(self): class TestChangeShapeAfterAssign(TestTensorShapeBasic): - def init_test_func(self): self.input = np.ones((2, 3)).astype("int32") self.input_spec = [ @@ -613,15 +594,14 @@ def dyfunc_with_static_convert_var_shape(x): else: # Test for correctly to find `batch_size__static_convert_var_shape_suffix_0` in # deeply nested scope. - res = fluid.layers.fill_constant(value=8, - shape=[batch_size], - dtype="int32") + res = fluid.layers.fill_constant( + value=8, shape=[batch_size], dtype="int32" + ) return res class TestFindStatiConvertVarShapeSuffixVar(unittest.TestCase): - def test(self): x_spec = paddle.static.InputSpec(shape=[None, 10]) func = paddle.jit.to_static(dyfunc_with_if_2, input_spec=[x_spec]) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_to_tensor.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_to_tensor.py index af543a5d97bb1..2e8e9f06a1b05 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_to_tensor.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_to_tensor.py @@ -37,10 +37,9 @@ def case2(x): place = paddle.CUDAPlace(0) else: place = paddle.CPUPlace() - a = paddle.to_tensor([1.0, 2.0, 3.0], - place=place, - dtype="int64", - stop_gradient=False) + a = paddle.to_tensor( + [1.0, 2.0, 3.0], place=place, dtype="int64", stop_gradient=False + ) return a @@ -84,7 +83,6 @@ def case6(x): class TestToTensorReturnVal(unittest.TestCase): - def test_to_tensor_badreturn(self): paddle.disable_static() x = paddle.to_tensor([3]) @@ -133,7 +131,6 @@ def test_to_tensor_badreturn(self): class TestStatic(unittest.TestCase): - def test_static(self): paddle.enable_static() main_prog = Program() @@ -144,10 +141,12 @@ def test_static(self): else: place = paddle.CPUPlace() - x = paddle.to_tensor(paddle.randn([5, 2]), - dtype='float64', - stop_gradient=False, - place=place) + x = paddle.to_tensor( + paddle.randn([5, 2]), + dtype='float64', + stop_gradient=False, + place=place, + ) out = paddle.static.nn.fc(x, 1) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py index f7b16c06cb8a4..d9eb890394f80 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py @@ -23,11 +23,16 @@ import paddle.fluid as fluid import transformer_util as util -from transformer_dygraph_model import CrossEntropyCriterion, Transformer, position_encoding_init +from transformer_dygraph_model import ( + CrossEntropyCriterion, + Transformer, + position_encoding_init, +) trainer_count = 1 -place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +place = ( + fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() +) SEED = 10 STEP_NUM = 10 @@ -42,46 +47,73 @@ def train_static(args, batch_generator): with fluid.program_guard(train_prog, startup_prog): with fluid.unique_name.guard(): # define input and reader - input_field_names = util.encoder_data_input_fields + \ - util.decoder_data_input_fields[:-1] + util.label_data_input_fields + input_field_names = ( + util.encoder_data_input_fields + + util.decoder_data_input_fields[:-1] + + util.label_data_input_fields + ) input_descs = util.get_input_descs(args) - input_slots = [{ - "name": name, - "shape": input_descs[name][0], - "dtype": input_descs[name][1] - } for name in input_field_names] + input_slots = [ + { + "name": name, + "shape": input_descs[name][0], + "dtype": input_descs[name][1], + } + for name in input_field_names + ] input_field = util.InputField(input_slots) # Define DataLoader data_loader = fluid.io.DataLoader.from_generator( - input_field.feed_list, capacity=60) + input_field.feed_list, capacity=60 + ) data_loader.set_batch_generator(batch_generator, places=place) # define model transformer = Transformer( - args.src_vocab_size, args.trg_vocab_size, args.max_length + 1, - args.n_layer, args.n_head, args.d_key, args.d_value, - args.d_model, args.d_inner_hid, args.prepostprocess_dropout, - args.attention_dropout, args.relu_dropout, args.preprocess_cmd, - args.postprocess_cmd, args.weight_sharing, args.bos_idx, - args.eos_idx) + args.src_vocab_size, + args.trg_vocab_size, + args.max_length + 1, + args.n_layer, + args.n_head, + args.d_key, + args.d_value, + args.d_model, + args.d_inner_hid, + args.prepostprocess_dropout, + args.attention_dropout, + args.relu_dropout, + args.preprocess_cmd, + args.postprocess_cmd, + args.weight_sharing, + args.bos_idx, + args.eos_idx, + ) logits = transformer(*input_field.feed_list[:7]) # define loss criterion = CrossEntropyCriterion(args.label_smooth_eps) lbl_word, lbl_weight = input_field.feed_list[7:] - sum_cost, avg_cost, token_num = criterion(logits, lbl_word, - lbl_weight) + sum_cost, avg_cost, token_num = criterion( + logits, lbl_word, lbl_weight + ) # define optimizer learning_rate = fluid.layers.learning_rate_scheduler.noam_decay( - args.d_model, args.warmup_steps, args.learning_rate) - optimizer = fluid.optimizer.Adam(learning_rate=learning_rate, - beta1=args.beta1, - beta2=args.beta2, - epsilon=float(args.eps)) + args.d_model, args.warmup_steps, args.learning_rate + ) + optimizer = fluid.optimizer.Adam( + learning_rate=learning_rate, + beta1=args.beta1, + beta2=args.beta2, + epsilon=float(args.eps), + ) optimizer.minimize(avg_cost) # the best cross-entropy value with label smoothing - loss_normalizer = -((1. - args.label_smooth_eps) * np.log( - (1. - args.label_smooth_eps)) + args.label_smooth_eps * - np.log(args.label_smooth_eps / - (args.trg_vocab_size - 1) + 1e-20)) + loss_normalizer = -( + (1.0 - args.label_smooth_eps) + * np.log((1.0 - args.label_smooth_eps)) + + args.label_smooth_eps + * np.log( + args.label_smooth_eps / (args.trg_vocab_size - 1) + 1e-20 + ) + ) step_idx = 0 total_batch_num = 0 avg_loss = [] @@ -90,12 +122,15 @@ def train_static(args, batch_generator): for pass_id in range(args.epoch): batch_id = 0 for feed_dict in data_loader: - outs = exe.run(program=train_prog, - feed=feed_dict, - fetch_list=[sum_cost.name, token_num.name]) + outs = exe.run( + program=train_prog, + feed=feed_dict, + fetch_list=[sum_cost.name, token_num.name], + ) if step_idx % args.print_step == 0: sum_cost_val, token_num_val = np.array(outs[0]), np.array( - outs[1]) + outs[1] + ) total_sum_cost = sum_cost_val.sum() total_token_num = token_num_val.sum() total_avg_cost = total_sum_cost / total_token_num @@ -103,27 +138,40 @@ def train_static(args, batch_generator): if step_idx == 0: logging.info( "step_idx: %d, epoch: %d, batch: %d, avg loss: %f, " - "normalized loss: %f, ppl: %f" % - (step_idx, pass_id, batch_id, total_avg_cost, - total_avg_cost - loss_normalizer, - np.exp([min(total_avg_cost, 100)]))) + "normalized loss: %f, ppl: %f" + % ( + step_idx, + pass_id, + batch_id, + total_avg_cost, + total_avg_cost - loss_normalizer, + np.exp([min(total_avg_cost, 100)]), + ) + ) avg_batch_time = time.time() else: logging.info( "step_idx: %d, epoch: %d, batch: %d, avg loss: %f, " - "normalized loss: %f, ppl: %f, speed: %.2f steps/s" % - (step_idx, pass_id, batch_id, total_avg_cost, - total_avg_cost - loss_normalizer, - np.exp([min(total_avg_cost, 100)]), args.print_step / - (time.time() - avg_batch_time))) + "normalized loss: %f, ppl: %f, speed: %.2f steps/s" + % ( + step_idx, + pass_id, + batch_id, + total_avg_cost, + total_avg_cost - loss_normalizer, + np.exp([min(total_avg_cost, 100)]), + args.print_step / (time.time() - avg_batch_time), + ) + ) avg_batch_time = time.time() batch_id += 1 step_idx += 1 total_batch_num = total_batch_num + 1 if step_idx == STEP_NUM: if args.save_dygraph_model_path: - model_path = os.path.join(args.save_static_model_path, - "transformer") + model_path = os.path.join( + args.save_static_model_path, "transformer" + ) fluid.save(train_prog, model_path) break return np.array(avg_loss) @@ -139,30 +187,45 @@ def train_dygraph(args, batch_generator): train_loader.set_batch_generator(batch_generator, places=place) # define model transformer = Transformer( - args.src_vocab_size, args.trg_vocab_size, args.max_length + 1, - args.n_layer, args.n_head, args.d_key, args.d_value, args.d_model, - args.d_inner_hid, args.prepostprocess_dropout, - args.attention_dropout, args.relu_dropout, args.preprocess_cmd, - args.postprocess_cmd, args.weight_sharing, args.bos_idx, - args.eos_idx) + args.src_vocab_size, + args.trg_vocab_size, + args.max_length + 1, + args.n_layer, + args.n_head, + args.d_key, + args.d_value, + args.d_model, + args.d_inner_hid, + args.prepostprocess_dropout, + args.attention_dropout, + args.relu_dropout, + args.preprocess_cmd, + args.postprocess_cmd, + args.weight_sharing, + args.bos_idx, + args.eos_idx, + ) # define loss criterion = CrossEntropyCriterion(args.label_smooth_eps) # define optimizer learning_rate = fluid.layers.learning_rate_scheduler.noam_decay( - args.d_model, args.warmup_steps, args.learning_rate) + args.d_model, args.warmup_steps, args.learning_rate + ) # define optimizer optimizer = fluid.optimizer.Adam( learning_rate=learning_rate, beta1=args.beta1, beta2=args.beta2, epsilon=float(args.eps), - parameter_list=transformer.parameters()) + parameter_list=transformer.parameters(), + ) # the best cross-entropy value with label smoothing loss_normalizer = -( - (1. - args.label_smooth_eps) * np.log( - (1. - args.label_smooth_eps)) + - args.label_smooth_eps * np.log(args.label_smooth_eps / - (args.trg_vocab_size - 1) + 1e-20)) + (1.0 - args.label_smooth_eps) + * np.log((1.0 - args.label_smooth_eps)) + + args.label_smooth_eps + * np.log(args.label_smooth_eps / (args.trg_vocab_size - 1) + 1e-20) + ) ce_time = [] ce_ppl = [] avg_loss = [] @@ -171,14 +234,29 @@ def train_dygraph(args, batch_generator): pass_start_time = time.time() batch_id = 0 for input_data in train_loader(): - (src_word, src_pos, src_slf_attn_bias, trg_word, trg_pos, - trg_slf_attn_bias, trg_src_attn_bias, lbl_word, - lbl_weight) = input_data - logits = transformer(src_word, src_pos, src_slf_attn_bias, - trg_word, trg_pos, trg_slf_attn_bias, - trg_src_attn_bias) + ( + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_pos, + trg_slf_attn_bias, + trg_src_attn_bias, + lbl_word, + lbl_weight, + ) = input_data + logits = transformer( + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_pos, + trg_slf_attn_bias, + trg_src_attn_bias, + ) sum_cost, avg_cost, token_num = criterion( - logits, lbl_word, lbl_weight) + logits, lbl_word, lbl_weight + ) avg_cost.backward() optimizer.minimize(avg_cost) transformer.clear_gradients() @@ -188,20 +266,32 @@ def train_dygraph(args, batch_generator): if step_idx == 0: logging.info( "step_idx: %d, epoch: %d, batch: %d, avg loss: %f, " - "normalized loss: %f, ppl: %f" % - (step_idx, pass_id, batch_id, total_avg_cost, - total_avg_cost - loss_normalizer, - np.exp([min(total_avg_cost, 100)]))) + "normalized loss: %f, ppl: %f" + % ( + step_idx, + pass_id, + batch_id, + total_avg_cost, + total_avg_cost - loss_normalizer, + np.exp([min(total_avg_cost, 100)]), + ) + ) avg_batch_time = time.time() else: logging.info( "step_idx: %d, epoch: %d, batch: %d, avg loss: %f, " "normalized loss: %f, ppl: %f, speed: %.2f steps/s" - % (step_idx, pass_id, batch_id, total_avg_cost, - total_avg_cost - loss_normalizer, - np.exp([min(total_avg_cost, 100) - ]), args.print_step / - (time.time() - avg_batch_time))) + % ( + step_idx, + pass_id, + batch_id, + total_avg_cost, + total_avg_cost - loss_normalizer, + np.exp([min(total_avg_cost, 100)]), + args.print_step + / (time.time() - avg_batch_time), + ) + ) ce_ppl.append(np.exp([min(total_avg_cost, 100)])) avg_batch_time = time.time() batch_id += 1 @@ -213,10 +303,12 @@ def train_dygraph(args, batch_generator): os.makedirs(model_dir) fluid.save_dygraph( transformer.state_dict(), - os.path.join(model_dir, "transformer")) + os.path.join(model_dir, "transformer"), + ) fluid.save_dygraph( optimizer.state_dict(), - os.path.join(model_dir, "transformer")) + os.path.join(model_dir, "transformer"), + ) break time_consumed = time.time() - pass_start_time ce_time.append(time_consumed) @@ -234,22 +326,37 @@ def predict_dygraph(args, batch_generator): # define model transformer = Transformer( - args.src_vocab_size, args.trg_vocab_size, args.max_length + 1, - args.n_layer, args.n_head, args.d_key, args.d_value, args.d_model, - args.d_inner_hid, args.prepostprocess_dropout, - args.attention_dropout, args.relu_dropout, args.preprocess_cmd, - args.postprocess_cmd, args.weight_sharing, args.bos_idx, - args.eos_idx) + args.src_vocab_size, + args.trg_vocab_size, + args.max_length + 1, + args.n_layer, + args.n_head, + args.d_key, + args.d_value, + args.d_model, + args.d_inner_hid, + args.prepostprocess_dropout, + args.attention_dropout, + args.relu_dropout, + args.preprocess_cmd, + args.postprocess_cmd, + args.weight_sharing, + args.bos_idx, + args.eos_idx, + ) # load the trained model model_dict, _ = util.load_dygraph( - os.path.join(args.save_dygraph_model_path, "transformer")) + os.path.join(args.save_dygraph_model_path, "transformer") + ) # to avoid a longer length than training, reset the size of position # encoding to max_length model_dict["encoder.pos_encoder.weight"] = position_encoding_init( - args.max_length + 1, args.d_model) + args.max_length + 1, args.d_model + ) model_dict["decoder.pos_encoder.weight"] = position_encoding_init( - args.max_length + 1, args.d_model) + args.max_length + 1, args.d_model + ) transformer.load_dict(model_dict) # set evaluate mode @@ -258,8 +365,13 @@ def predict_dygraph(args, batch_generator): step_idx = 0 speed_list = [] for input_data in test_loader(): - (src_word, src_pos, src_slf_attn_bias, trg_word, - trg_src_attn_bias) = input_data + ( + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_src_attn_bias, + ) = input_data seq_ids, seq_scores = transformer.beam_search( src_word, src_pos, @@ -269,28 +381,32 @@ def predict_dygraph(args, batch_generator): bos_id=args.bos_idx, eos_id=args.eos_idx, beam_size=args.beam_size, - max_len=args.max_out_len) + max_len=args.max_out_len, + ) seq_ids = seq_ids.numpy() seq_scores = seq_scores.numpy() if step_idx % args.print_step == 0: if step_idx == 0: logging.info( "Dygraph Predict: step_idx: %d, 1st seq_id: %d, 1st seq_score: %.2f" - % (step_idx, seq_ids[0][0][0], seq_scores[0][0])) + % (step_idx, seq_ids[0][0][0], seq_scores[0][0]) + ) avg_batch_time = time.time() else: speed = args.print_step / (time.time() - avg_batch_time) speed_list.append(speed) logging.info( "Dygraph Predict: step_idx: %d, 1st seq_id: %d, 1st seq_score: %.2f, speed: %.3f steps/s" - % (step_idx, seq_ids[0][0][0], seq_scores[0][0], speed)) + % (step_idx, seq_ids[0][0][0], seq_scores[0][0], speed) + ) avg_batch_time = time.time() step_idx += 1 if step_idx == STEP_NUM: break - logging.info("Dygraph Predict: avg_speed: %.4f steps/s" % - (np.mean(speed_list))) + logging.info( + "Dygraph Predict: avg_speed: %.4f steps/s" % (np.mean(speed_list)) + ) return seq_ids, seq_scores @@ -301,33 +417,53 @@ def predict_static(args, batch_generator): paddle.framework.random._manual_program_seed(SEED) # define input and reader - input_field_names = util.encoder_data_input_fields + util.fast_decoder_data_input_fields + input_field_names = ( + util.encoder_data_input_fields + util.fast_decoder_data_input_fields + ) input_descs = util.get_input_descs(args, 'test') - input_slots = [{ - "name": name, - "shape": input_descs[name][0], - "dtype": input_descs[name][1] - } for name in input_field_names] + input_slots = [ + { + "name": name, + "shape": input_descs[name][0], + "dtype": input_descs[name][1], + } + for name in input_field_names + ] input_field = util.InputField(input_slots) feed_list = input_field.feed_list - loader = fluid.io.DataLoader.from_generator(feed_list=feed_list, - capacity=10) + loader = fluid.io.DataLoader.from_generator( + feed_list=feed_list, capacity=10 + ) # define model transformer = Transformer( - args.src_vocab_size, args.trg_vocab_size, args.max_length + 1, - args.n_layer, args.n_head, args.d_key, args.d_value, args.d_model, - args.d_inner_hid, args.prepostprocess_dropout, - args.attention_dropout, args.relu_dropout, args.preprocess_cmd, - args.postprocess_cmd, args.weight_sharing, args.bos_idx, - args.eos_idx) - - out_ids, out_scores = transformer.beam_search(*feed_list, - bos_id=args.bos_idx, - eos_id=args.eos_idx, - beam_size=args.beam_size, - max_len=args.max_out_len) + args.src_vocab_size, + args.trg_vocab_size, + args.max_length + 1, + args.n_layer, + args.n_head, + args.d_key, + args.d_value, + args.d_model, + args.d_inner_hid, + args.prepostprocess_dropout, + args.attention_dropout, + args.relu_dropout, + args.preprocess_cmd, + args.postprocess_cmd, + args.weight_sharing, + args.bos_idx, + args.eos_idx, + ) + + out_ids, out_scores = transformer.beam_search( + *feed_list, + bos_id=args.bos_idx, + eos_id=args.eos_idx, + beam_size=args.beam_size, + max_len=args.max_out_len + ) # This is used here to set dropout to the test mode. test_prog = test_prog.clone(for_test=True) @@ -335,8 +471,9 @@ def predict_static(args, batch_generator): # define the executor and program for training exe = fluid.Executor(place) - util.load(test_prog, os.path.join(args.save_static_model_path, - "transformer"), exe) + util.load( + test_prog, os.path.join(args.save_static_model_path, "transformer"), exe + ) loader.set_batch_generator(batch_generator, places=place) @@ -347,32 +484,35 @@ def predict_static(args, batch_generator): test_prog, feed=feed_dict, fetch_list=[out_ids.name, out_scores.name], - return_numpy=True) + return_numpy=True, + ) if step_idx % args.print_step == 0: if step_idx == 0: logging.info( "Static Predict: step_idx: %d, 1st seq_id: %d, 1st seq_score: %.2f," - % (step_idx, seq_ids[0][0][0], seq_scores[0][0])) + % (step_idx, seq_ids[0][0][0], seq_scores[0][0]) + ) avg_batch_time = time.time() else: speed = args.print_step / (time.time() - avg_batch_time) speed_list.append(speed) logging.info( "Static Predict: step_idx: %d, 1st seq_id: %d, 1st seq_score: %.2f, speed: %.3f steps/s" - % (step_idx, seq_ids[0][0][0], seq_scores[0][0], speed)) + % (step_idx, seq_ids[0][0][0], seq_scores[0][0], speed) + ) avg_batch_time = time.time() step_idx += 1 if step_idx == STEP_NUM: break - logging.info("Static Predict: avg_speed: %.4f steps/s" % - (np.mean(speed_list))) + logging.info( + "Static Predict: avg_speed: %.4f steps/s" % (np.mean(speed_list)) + ) return seq_ids, seq_scores class TestTransformer(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -382,11 +522,14 @@ def tearDwon(self): def prepare(self, mode='train'): args = util.ModelHyperParams() args.save_dygraph_model_path = os.path.join( - self.temp_dir.name, args.save_dygraph_model_path) - args.save_static_model_path = os.path.join(self.temp_dir.name, - args.save_static_model_path) - args.inference_model_dir = os.path.join(self.temp_dir.name, - args.inference_model_dir) + self.temp_dir.name, args.save_dygraph_model_path + ) + args.save_static_model_path = os.path.join( + self.temp_dir.name, args.save_static_model_path + ) + args.inference_model_dir = os.path.join( + self.temp_dir.name, args.inference_model_dir + ) args.output_file = os.path.join(self.temp_dir.name, args.output_file) batch_generator = util.get_feed_data_reader(args, mode) return args, batch_generator @@ -395,9 +538,9 @@ def _test_train(self): args, batch_generator = self.prepare(mode='train') static_avg_loss = train_static(args, batch_generator) dygraph_avg_loss = train_dygraph(args, batch_generator) - np.testing.assert_allclose(static_avg_loss, - dygraph_avg_loss, - rtol=1e-05) + np.testing.assert_allclose( + static_avg_loss, dygraph_avg_loss, rtol=1e-05 + ) def _test_predict(self): args, batch_generator = self.prepare(mode='test') diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tsm.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tsm.py index 7561e7aa13798..36232b627fdfa 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tsm.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_tsm.py @@ -1,16 +1,16 @@ # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import argparse import numpy as np @@ -30,43 +30,52 @@ def parse_args(): parser = argparse.ArgumentParser("Paddle Video train script") - parser.add_argument('--config', - type=str, - default='tsm.yaml', - help='path to config file of model') - parser.add_argument('--use_gpu', - type=bool, - default=fluid.is_compiled_with_cuda(), - help='default use gpu.') + parser.add_argument( + '--config', + type=str, + default='tsm.yaml', + help='path to config file of model', + ) + parser.add_argument( + '--use_gpu', + type=bool, + default=fluid.is_compiled_with_cuda(), + help='default use gpu.', + ) args = parser.parse_args(['--config', 'tsm.yaml']) return args class ConvBNLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=None, - act=None, - param_attr=fluid.param_attr.ParamAttr(), - bias_attr=False) - - self._batch_norm = BatchNorm(num_filters, - act=act, - param_attr=fluid.param_attr.ParamAttr(), - bias_attr=fluid.param_attr.ParamAttr()) + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=None, + act=None, + param_attr=fluid.param_attr.ParamAttr(), + bias_attr=False, + ) + + self._batch_norm = BatchNorm( + num_filters, + act=act, + param_attr=fluid.param_attr.ParamAttr(), + bias_attr=fluid.param_attr.ParamAttr(), + ) def forward(self, inputs): y = self._conv(inputs) @@ -76,34 +85,38 @@ def forward(self, inputs): class BottleneckBlock(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - stride, - shortcut=True, - seg_num=8): + def __init__( + self, num_channels, num_filters, stride, shortcut=True, seg_num=8 + ): super(BottleneckBlock, self).__init__() - self.conv0 = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=1, - act='relu') - self.conv1 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=stride, - act='relu') - self.conv2 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters * 4, - filter_size=1, - act=None) + self.conv0 = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=1, + act='relu', + ) + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + act='relu', + ) + self.conv2 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters * 4, + filter_size=1, + act=None, + ) if not shortcut: - self.short = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters * 4, - filter_size=1, - stride=stride) + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters * 4, + filter_size=1, + stride=stride, + ) self.shortcut = shortcut self.seg_num = seg_num self._num_channels_out = int(num_filters * 4) @@ -122,7 +135,6 @@ def forward(self, inputs): class TSM_ResNet(fluid.dygraph.Layer): - def __init__(self, name_scope, config, mode): super(TSM_ResNet, self).__init__(name_scope) @@ -130,8 +142,9 @@ def __init__(self, name_scope, config, mode): self.seg_num = config.MODEL.seg_num self.class_dim = config.MODEL.num_classes self.reshape_list = [ - config.MODEL.seglen * 3, config[mode.upper()]['target_size'], - config[mode.upper()]['target_size'] + config.MODEL.seglen * 3, + config[mode.upper()]['target_size'], + config[mode.upper()]['target_size'], ] if self.layers == 50: @@ -140,15 +153,12 @@ def __init__(self, name_scope, config, mode): raise NotImplementedError num_filters = [64, 128, 256, 512] - self.conv = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool2d_max = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv = ConvBNLayer( + num_channels=3, num_filters=64, filter_size=7, stride=2, act='relu' + ) + self.pool2d_max = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) self.bottleneck_block_list = [] num_channels = 64 @@ -158,19 +168,23 @@ def __init__(self, name_scope, config, mode): for i in range(depth[block]): bottleneck_block = self.add_sublayer( 'bb_%d_%d' % (block, i), - BottleneckBlock(num_channels=num_channels, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - shortcut=shortcut, - seg_num=self.seg_num)) + BottleneckBlock( + num_channels=num_channels, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + shortcut=shortcut, + seg_num=self.seg_num, + ), + ) num_channels = int(bottleneck_block._num_channels_out) self.bottleneck_block_list.append(bottleneck_block) shortcut = True - self.pool2d_avg = Pool2D(pool_size=7, - pool_type='avg', - global_pooling=True) + self.pool2d_avg = Pool2D( + pool_size=7, pool_type='avg', global_pooling=True + ) import math + stdv = 1.0 / math.sqrt(2048 * 1.0) self.out = Linear( @@ -178,9 +192,12 @@ def __init__(self, name_scope, config, mode): self.class_dim, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv)), + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), bias_attr=fluid.param_attr.ParamAttr( - learning_rate=2.0, regularizer=fluid.regularizer.L2Decay(0.))) + learning_rate=2.0, regularizer=fluid.regularizer.L2Decay(0.0) + ), + ) @declarative def forward(self, inputs): @@ -199,7 +216,6 @@ def forward(self, inputs): class FakeDataReader(object): - def __init__(self, mode, cfg): self.format = cfg.MODEL.format self.num_classes = cfg.MODEL.num_classes @@ -207,13 +223,18 @@ def __init__(self, mode, cfg): self.seglen = cfg.MODEL.seglen self.target_size = cfg[mode.upper()]['target_size'] - self.img_mean = np.array(cfg.MODEL.image_mean).reshape( - [3, 1, 1]).astype(np.float32) - self.img_std = np.array(cfg.MODEL.image_std).reshape([3, 1, 1]).astype( - np.float32) - - self.batch_size = 1 if sys.platform == 'darwin' or os.name == 'nt' else cfg[ - mode.upper()]['batch_size'] + self.img_mean = ( + np.array(cfg.MODEL.image_mean).reshape([3, 1, 1]).astype(np.float32) + ) + self.img_std = ( + np.array(cfg.MODEL.image_std).reshape([3, 1, 1]).astype(np.float32) + ) + + self.batch_size = ( + 1 + if sys.platform == 'darwin' or os.name == 'nt' + else cfg[mode.upper()]['batch_size'] + ) self.generator_out = [] self.total_iter = 3 for i in range(self.total_iter): @@ -222,15 +243,20 @@ def __init__(self, mode, cfg): label = np.int64(random.randint(0, self.num_classes - 1)) random_mean = self.img_mean[0][0][0] random_std = self.img_std[0][0][0] - imgs = np.random.normal(random_mean, random_std, [ - self.seg_num, self.seglen * 3, self.target_size, - self.target_size - ]).astype(np.float32) + imgs = np.random.normal( + random_mean, + random_std, + [ + self.seg_num, + self.seglen * 3, + self.target_size, + self.target_size, + ], + ).astype(np.float32) batch_out.append((imgs, label)) self.generator_out.append(batch_out) def create_reader(self): - def batch_reader(): for i in range(self.total_iter): yield self.generator_out[i] @@ -240,7 +266,9 @@ def batch_reader(): def create_optimizer(cfg, params): total_videos = cfg.total_videos - batch_size = 1 if sys.platform == 'darwin' or os.name == 'nt' else cfg.batch_size + batch_size = ( + 1 if sys.platform == 'darwin' or os.name == 'nt' else cfg.batch_size + ) step = int(total_videos / batch_size + 1) bd = [e * step for e in cfg.decay_epochs] base_lr = cfg.learning_rate @@ -253,7 +281,8 @@ def create_optimizer(cfg, params): learning_rate=fluid.layers.piecewise_decay(boundaries=bd, values=lr), momentum=momentum, regularization=fluid.regularizer.L2Decay(l2_weight_decay), - parameter_list=params) + parameter_list=params, + ) return optimizer @@ -277,8 +306,9 @@ def train(args, fake_data_reader, to_static): video_model = TSM_ResNet("TSM", train_config, 'Train') - optimizer = create_optimizer(train_config.TRAIN, - video_model.parameters()) + optimizer = create_optimizer( + train_config.TRAIN, video_model.parameters() + ) train_reader = fake_data_reader.create_reader() @@ -297,16 +327,16 @@ def train(args, fake_data_reader, to_static): labels = to_variable(y_data) labels.stop_gradient = True outputs = video_model(imgs) - loss = fluid.layers.cross_entropy(input=outputs, - label=labels, - ignore_index=-1) + loss = fluid.layers.cross_entropy( + input=outputs, label=labels, ignore_index=-1 + ) avg_loss = paddle.mean(loss) - acc_top1 = fluid.layers.accuracy(input=outputs, - label=labels, - k=1) - acc_top5 = fluid.layers.accuracy(input=outputs, - label=labels, - k=5) + acc_top1 = fluid.layers.accuracy( + input=outputs, label=labels, k=1 + ) + acc_top5 = fluid.layers.accuracy( + input=outputs, label=labels, k=5 + ) avg_loss.backward() optimizer.minimize(avg_loss) @@ -317,26 +347,35 @@ def train(args, fake_data_reader, to_static): total_acc5 += acc_top5.numpy()[0] total_sample += 1 - print('TRAIN Epoch {}, iter {}, loss = {}, acc1 {}, acc5 {}'. - format(epoch, batch_id, - avg_loss.numpy()[0], - acc_top1.numpy()[0], - acc_top5.numpy()[0])) - ret.extend([ - avg_loss.numpy()[0], - acc_top1.numpy()[0], - acc_top5.numpy()[0] - ]) + print( + 'TRAIN Epoch {}, iter {}, loss = {}, acc1 {}, acc5 {}'.format( + epoch, + batch_id, + avg_loss.numpy()[0], + acc_top1.numpy()[0], + acc_top5.numpy()[0], + ) + ) + ret.extend( + [ + avg_loss.numpy()[0], + acc_top1.numpy()[0], + acc_top5.numpy()[0], + ] + ) print( - 'TRAIN End, Epoch {}, avg_loss= {}, avg_acc1= {}, avg_acc5= {}'. - format(epoch, total_loss / total_sample, - total_acc1 / total_sample, total_acc5 / total_sample)) + 'TRAIN End, Epoch {}, avg_loss= {}, avg_acc1= {}, avg_acc5= {}'.format( + epoch, + total_loss / total_sample, + total_acc1 / total_sample, + total_acc5 / total_sample, + ) + ) return ret class TestTsm(unittest.TestCase): - def test_dygraph_static_same_loss(self): if fluid.is_compiled_with_cuda(): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typehint.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typehint.py index b8addd53d5b18..206ab40d0bc9b 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typehint.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typehint.py @@ -32,10 +32,12 @@ def function(x: A) -> A: class TestTransformWhileLoop(unittest.TestCase): - def setUp(self): - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.x = np.zeros(shape=(1), dtype=np.int32) self._init_dyfunc() @@ -69,7 +71,6 @@ def test_ast_to_func(self): class TestTypeHint(TestTransformWhileLoop): - def _init_dyfunc(self): self.dyfunc = function diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typing.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typing.py index b55fb242199f4..e4dea9d6abf84 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typing.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_typing.py @@ -20,7 +20,6 @@ class BaseLayer(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(BaseLayer, self).__init__() self._linear = paddle.nn.Linear(in_size, out_size) @@ -32,7 +31,6 @@ def build(self, x): class LinearNetWithTuple(BaseLayer): - def __init__(self, in_size, out_size): super(LinearNetWithTuple, self).__init__(in_size, out_size) @@ -42,7 +40,6 @@ def forward(self, x) -> Tuple[paddle.Tensor, str]: class LinearNetWithTuple2(BaseLayer): - def __init__(self, in_size, out_size): super(LinearNetWithTuple2, self).__init__(in_size, out_size) @@ -52,7 +49,6 @@ def forward(self, x) -> Tuple[paddle.Tensor, np.array]: class LinearNetWithList(BaseLayer): - def __init__(self, in_size, out_size): super(LinearNetWithList, self).__init__(in_size, out_size) @@ -62,7 +58,6 @@ def forward(self, x) -> List[paddle.Tensor]: class LinearNetWithDict(BaseLayer): - def __init__(self, in_size, out_size): super(LinearNetWithDict, self).__init__(in_size, out_size) @@ -72,7 +67,6 @@ def forward(self, x) -> Dict[str, paddle.Tensor]: class TestTyping(unittest.TestCase): - def setUp(self): self.in_num = 16 self.out_num = 16 @@ -105,7 +99,6 @@ def test_type(self): class TestTypingTuple(TestTyping): - def build_net(self): return LinearNetWithTuple2(self.in_num, self.out_num) @@ -116,7 +109,6 @@ def run_dy(self): class TestTypingList(TestTyping): - def build_net(self): return LinearNetWithList(self.in_num, self.out_num) @@ -126,7 +118,6 @@ def run_dy(self): class TestTypingDict(TestTyping): - def build_net(self): return LinearNetWithDict(self.in_num, self.out_num) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_utils.py index 30300ddf2e4b1..4edb81504458c 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_utils.py @@ -20,7 +20,6 @@ class TestIndexInList(unittest.TestCase): - def test_index_in_list(self): list_to_test = [1, 2, 3, 4, 5] self.assertEqual(index_in_list(list_to_test, 4), 3) @@ -37,8 +36,7 @@ def dyfunc_assign(input): [x, y] = m, n = z -class StaticCode(): - +class StaticCode: def dyfunc_assign(input): b = 1 a = b @@ -53,7 +51,6 @@ def dyfunc_assign(input): class TestIsPaddle(unittest.TestCase): - def fake_module(self): return types.ModuleType('paddlenlp') diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py index 901778916934a..353a5e8b79738 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_variable_trans_func.py @@ -15,29 +15,33 @@ import unittest from paddle.fluid.dygraph.dygraph_to_static.utils import ast_to_source_code -from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import create_fill_constant_node +from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + create_fill_constant_node, +) class TestVariableTransFunc(unittest.TestCase): - def test_create_fill_constant_node(self): node = create_fill_constant_node("a", 1.0) source = "a = paddle.full(shape=[1], dtype='float64', fill_value=1.0, name='a')" self.assertEqual( ast_to_source_code(node).replace('\n', '').replace(' ', ''), - source.replace(' ', '')) + source.replace(' ', ''), + ) node = create_fill_constant_node("b", True) source = "b = paddle.full(shape=[1], dtype='bool', fill_value=True, name='b')" self.assertEqual( ast_to_source_code(node).replace('\n', '').replace(' ', ''), - source.replace(' ', '')) + source.replace(' ', ''), + ) node = create_fill_constant_node("c", 4293) source = "c = paddle.full(shape=[1], dtype='int64', fill_value=4293, name='c')" self.assertEqual( ast_to_source_code(node).replace('\n', '').replace(' ', ''), - source.replace(' ', '')) + source.replace(' ', ''), + ) self.assertIsNone(create_fill_constant_node("e", None)) self.assertIsNone(create_fill_constant_node("e", [])) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_warning.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_warning.py index 846cb9e7aa15e..a232c70fa7172 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_warning.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_warning.py @@ -37,7 +37,6 @@ def false_fn(): class TestReturnNoneInIfelse(unittest.TestCase): - def test_dy2static_warning(self): paddle.disable_static() with warnings.catch_warnings(record=True) as w: @@ -46,9 +45,10 @@ def test_dy2static_warning(self): flag = False for warn in w: if ( - issubclass(warn.category, UserWarning) + issubclass(warn.category, UserWarning) ) and "Set var to 'None' in ifelse block might lead to error." in str( - warn.message): + warn.message + ): flag = True break self.assertTrue(flag) @@ -63,9 +63,10 @@ def test_cond_warning(self): flag = False for warn in w: if ( - issubclass(warn.category, UserWarning) + issubclass(warn.category, UserWarning) ) and "Set var to 'None' in ifelse block might lead to error." in str( - warn.message): + warn.message + ): flag = True break self.assertTrue(flag) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_word2vec.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_word2vec.py index fbabe1bc36bbb..a597e33518787 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_word2vec.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_word2vec.py @@ -55,15 +55,15 @@ def build_dict(corpus, min_freq=3): word_freq_dict[word] = 0 word_freq_dict[word] += 1 - word_freq_dict = sorted(word_freq_dict.items(), - key=lambda x: x[1], - reverse=True) + word_freq_dict = sorted( + word_freq_dict.items(), key=lambda x: x[1], reverse=True + ) word2id_dict = dict() word2id_freq = dict() id2word_dict = dict() - word2id_freq[0] = 1. + word2id_freq[0] = 1.0 word2id_dict['[oov]'] = 0 id2word_dict[0] = '[oov]' @@ -85,8 +85,10 @@ def build_dict(corpus, min_freq=3): vocab_size = len(word2id_freq) print("there are totoally %d different words in the corpus" % vocab_size) for _, (word, word_id) in zip(range(50), word2id_dict.items()): - print("word %s, its id %d, its word freq %d" % - (word, word_id, word2id_freq[word_id])) + print( + "word %s, its id %d, its word freq %d" + % (word, word_id, word2id_freq[word_id]) + ) def convert_corpus_to_id(corpus, word2id_dict): @@ -94,7 +96,9 @@ def convert_corpus_to_id(corpus, word2id_dict): for line in corpus: new_line = [ word2id_dict[word] - if word in word2id_dict else word2id_dict['[oov]'] for word in line + if word in word2id_dict + else word2id_dict['[oov]'] + for word in line ] new_corpus.append(new_line) return new_corpus @@ -104,10 +108,10 @@ def convert_corpus_to_id(corpus, word2id_dict): def subsampling(corpus, word2id_freq): - def keep(word_id): return random.uniform(0, 1) < math.sqrt( - 1e-4 / word2id_freq[word_id] * len(corpus)) + 1e-4 / word2id_freq[word_id] * len(corpus) + ) new_corpus = [] for line in corpus: @@ -119,11 +123,13 @@ def keep(word_id): corpus = subsampling(corpus, word2id_freq) -def build_data(corpus, - word2id_dict, - word2id_freq, - max_window_size=3, - negative_sample_num=10): +def build_data( + corpus, + word2id_dict, + word2id_freq, + max_window_size=3, + negative_sample_num=10, +): dataset = [] @@ -132,13 +138,15 @@ def build_data(corpus, window_size = random.randint(1, max_window_size) center_word = line[center_word_idx] - positive_word_range = (max(0, center_word_idx - window_size), - min( - len(line) - 1, - center_word_idx + window_size)) + positive_word_range = ( + max(0, center_word_idx - window_size), + min(len(line) - 1, center_word_idx + window_size), + ) positive_word_candidates = [ - line[idx] for idx in range(positive_word_range[0], - positive_word_range[1] + 1) + line[idx] + for idx in range( + positive_word_range[0], positive_word_range[1] + 1 + ) if idx != center_word_idx and line[idx] != line[center_word_idx] ] @@ -161,8 +169,10 @@ def build_data(corpus, dataset = build_data(corpus, word2id_dict, word2id_freq) for _, (center_word, target_word, label) in zip(range(50), dataset): - print("center_word %s, target %s, label %d" % - (id2word_dict[center_word], id2word_dict[target_word], label)) + print( + "center_word %s, target %s, label %d" + % (id2word_dict[center_word], id2word_dict[target_word], label) + ) def build_batch(dataset, batch_size, epoch_num): @@ -185,9 +195,14 @@ def build_batch(dataset, batch_size, epoch_num): if len(center_word_batch) == batch_size: yield np.array(center_word_batch).astype("int64"), np.array( - target_word_batch).astype("int64"), np.array( - label_batch).astype("float32"), np.array( - eval_word_batch).astype("int64") + target_word_batch + ).astype("int64"), np.array(label_batch).astype( + "float32" + ), np.array( + eval_word_batch + ).astype( + "int64" + ) center_word_batch = [] target_word_batch = [] label_batch = [] @@ -195,12 +210,15 @@ def build_batch(dataset, batch_size, epoch_num): if len(center_word_batch) > 0: yield np.array(center_word_batch).astype("int64"), np.array( - target_word_batch).astype("int64"), np.array(label_batch).astype( - "float32"), np.array(eval_word_batch).astype("int64") + target_word_batch + ).astype("int64"), np.array(label_batch).astype("float32"), np.array( + eval_word_batch + ).astype( + "int64" + ) class SkipGram(fluid.dygraph.Layer): - def __init__(self, name_scope, vocab_size, embedding_size, init_scale=0.1): super(SkipGram, self).__init__(name_scope) self.vocab_size = vocab_size @@ -213,7 +231,10 @@ def __init__(self, name_scope, vocab_size, embedding_size, init_scale=0.1): name='embedding_para', initializer=fluid.initializer.UniformInitializer( low=-0.5 / self.embedding_size, - high=0.5 / self.embedding_size))) + high=0.5 / self.embedding_size, + ), + ), + ) self.embedding_out = Embedding( size=[self.vocab_size, self.embedding_size], @@ -222,7 +243,10 @@ def __init__(self, name_scope, vocab_size, embedding_size, init_scale=0.1): name='embedding_out_para', initializer=fluid.initializer.UniformInitializer( low=-0.5 / self.embedding_size, - high=0.5 / self.embedding_size))) + high=0.5 / self.embedding_size, + ), + ), + ) @declarative def forward(self, center_words, target_words, label): @@ -231,8 +255,9 @@ def forward(self, center_words, target_words, label): # center_words_emb = [batch_size, embedding_size] # target_words_emb = [batch_size, embedding_size] - word_sim = fluid.layers.elementwise_mul(center_words_emb, - target_words_emb) + word_sim = fluid.layers.elementwise_mul( + center_words_emb, target_words_emb + ) word_sim = fluid.layers.reduce_sum(word_sim, dim=-1) pred = fluid.layers.sigmoid(word_sim) @@ -257,27 +282,34 @@ def train(to_static): random.seed(0) np.random.seed(0) - place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.dygraph.guard(place): fluid.default_startup_program().random_seed = 1000 fluid.default_main_program().random_seed = 1000 - skip_gram_model = SkipGram("skip_gram_model", vocab_size, - embedding_size) + skip_gram_model = SkipGram( + "skip_gram_model", vocab_size, embedding_size + ) adam = fluid.optimizer.AdamOptimizer( learning_rate=learning_rate, - parameter_list=skip_gram_model.parameters()) + parameter_list=skip_gram_model.parameters(), + ) step = 0 ret = [] for center_words, target_words, label, eval_words in build_batch( - dataset, batch_size, epoch_num): + dataset, batch_size, epoch_num + ): center_words_var = fluid.dygraph.to_variable(center_words) target_words_var = fluid.dygraph.to_variable(target_words) label_var = fluid.dygraph.to_variable(label) - pred, loss = skip_gram_model(center_words_var, target_words_var, - label_var) + pred, loss = skip_gram_model( + center_words_var, target_words_var, label_var + ) loss.backward() adam.minimize(loss) @@ -291,7 +323,6 @@ def train(to_static): class TestWord2Vec(unittest.TestCase): - def test_dygraph_static_same_loss(self): dygraph_loss = train(to_static=False) static_loss = train(to_static=True) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_yolov3.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_yolov3.py index 0fb791a3a28ca..eecd6806fa540 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_yolov3.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_yolov3.py @@ -47,27 +47,26 @@ def get_mean_value(self): class FakeDataReader(object): - def __init__(self): self.generator_out = [] self.total_iter = cfg.max_iter for i in range(self.total_iter): batch_out = [] for j in range(cfg.batch_size): - img = np.random.normal(0.485, 0.229, - [3, cfg.input_size, cfg.input_size]) + img = np.random.normal( + 0.485, 0.229, [3, cfg.input_size, cfg.input_size] + ) point1 = cfg.input_size / 4 point2 = cfg.input_size / 2 gt_boxes = np.array([[point1, point1, point2, point2]]) - gt_labels = np.random.randint(low=0, - high=cfg.class_num, - size=[1]) + gt_labels = np.random.randint( + low=0, high=cfg.class_num, size=[1] + ) gt_scores = np.zeros([1]) batch_out.append([img, gt_boxes, gt_labels, gt_scores]) self.generator_out.append(batch_out) def reader(self): - def generator(): for i in range(self.total_iter): yield self.generator_out[i] @@ -97,9 +96,9 @@ def train(to_static): learning_rate = cfg.learning_rate values = [learning_rate * (gamma**i) for i in range(step_num + 1)] - lr = fluid.dygraph.PiecewiseDecay(boundaries=boundaries, - values=values, - begin=0) + lr = fluid.dygraph.PiecewiseDecay( + boundaries=boundaries, values=values, begin=0 + ) lr = fluid.layers.linear_lr_warmup( learning_rate=lr, @@ -112,7 +111,8 @@ def train(to_static): learning_rate=lr, regularization=fluid.regularizer.L2Decay(cfg.weight_decay), momentum=cfg.momentum, - parameter_list=model.parameters()) + parameter_list=model.parameters(), + ) start_time = time.time() snapshot_loss = 0 @@ -150,9 +150,13 @@ def train(to_static): snapshot_time += start_time - prev_start_time total_sample += 1 - print("Iter {:d}, loss {:.6f}, time {:.5f}".format( - iter_id, smoothed_loss.get_mean_value(), - start_time - prev_start_time)) + print( + "Iter {:d}, loss {:.6f}, time {:.5f}".format( + iter_id, + smoothed_loss.get_mean_value(), + start_time - prev_start_time, + ) + ) ret.append(smoothed_loss.get_mean_value()) loss.backward() @@ -164,14 +168,12 @@ def train(to_static): class TestYolov3(unittest.TestCase): - def test_dygraph_static_same_loss(self): dygraph_loss = train(to_static=False) static_loss = train(to_static=True) - np.testing.assert_allclose(dygraph_loss, - static_loss, - rtol=0.001, - atol=1e-05) + np.testing.assert_allclose( + dygraph_loss, static_loss, rtol=0.001, atol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_dygraph_model.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_dygraph_model.py index 4c33f88469ceb..27139811c0abf 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_dygraph_model.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_dygraph_model.py @@ -16,7 +16,13 @@ import paddle.fluid as fluid import paddle.fluid.layers as layers -from paddle.fluid.dygraph import Embedding, Layer, LayerNorm, Linear, to_variable +from paddle.fluid.dygraph import ( + Embedding, + Layer, + LayerNorm, + Linear, + to_variable, +) from paddle.fluid.dygraph.jit import dygraph_to_static_func from paddle.fluid.layers.utils import map_structure import paddle @@ -29,12 +35,15 @@ def position_encoding_init(n_position, d_pos_vec): channels = d_pos_vec position = np.arange(n_position) num_timescales = channels // 2 - log_timescale_increment = (np.log(float(1e4) / float(1)) / - (num_timescales - 1)) - inv_timescales = np.exp( - np.arange(num_timescales)) * -log_timescale_increment + log_timescale_increment = np.log(float(1e4) / float(1)) / ( + num_timescales - 1 + ) + inv_timescales = ( + np.exp(np.arange(num_timescales)) * -log_timescale_increment + ) scaled_time = np.expand_dims(position, 1) * np.expand_dims( - inv_timescales, 0) + inv_timescales, 0 + ) signal = np.concatenate([np.sin(scaled_time), np.cos(scaled_time)], axis=1) signal = np.pad(signal, [[0, 0], [0, np.mod(channels, 2)]], 'constant') position_enc = signal @@ -42,7 +51,6 @@ def position_encoding_init(n_position, d_pos_vec): class PrePostProcessLayer(Layer): - def __init__(self, process_cmd, d_model, dropout_rate): super(PrePostProcessLayer, self).__init__() self.process_cmd = process_cmd @@ -53,18 +61,24 @@ def __init__(self, process_cmd, d_model, dropout_rate): elif cmd == "n": # add layer normalization self.functors.append( self.add_sublayer( - "layer_norm_%d" % - len([layer for layer in self.children()]), + "layer_norm_%d" + % len([layer for layer in self.children()]), LayerNorm( normalized_shape=d_model, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(1.)), + initializer=fluid.initializer.Constant(1.0) + ), bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.))))) + initializer=fluid.initializer.Constant(0.0) + ), + ), + ) + ) elif cmd == "d": # add dropout if dropout_rate: self.functors.append( - lambda x: layers.dropout(x, dropout_prob=dropout_rate)) + lambda x: layers.dropout(x, dropout_prob=dropout_rate) + ) def forward(self, x, residual=None): for i, cmd in enumerate(self.process_cmd): @@ -76,14 +90,15 @@ def forward(self, x, residual=None): class MultiHeadAttention(Layer): - - def __init__(self, - d_key, - d_value, - d_model, - n_head=1, - dropout_rate=0., - param_initializer=None): + def __init__( + self, + d_key, + d_value, + d_model, + n_head=1, + dropout_rate=0.0, + param_initializer=None, + ): super(MultiHeadAttention, self).__init__() self.n_head = n_head self.d_key = d_key @@ -94,22 +109,26 @@ def __init__(self, input_dim=d_model, output_dim=d_key * n_head, bias_attr=False, - param_attr=fluid.ParamAttr(initializer=param_initializer)) + param_attr=fluid.ParamAttr(initializer=param_initializer), + ) self.k_fc = Linear( input_dim=d_model, output_dim=d_key * n_head, bias_attr=False, - param_attr=fluid.ParamAttr(initializer=param_initializer)) + param_attr=fluid.ParamAttr(initializer=param_initializer), + ) self.v_fc = Linear( input_dim=d_model, output_dim=d_value * n_head, bias_attr=False, - param_attr=fluid.ParamAttr(initializer=param_initializer)) + param_attr=fluid.ParamAttr(initializer=param_initializer), + ) self.proj_fc = Linear( input_dim=d_value * n_head, output_dim=d_model, bias_attr=False, - param_attr=fluid.ParamAttr(initializer=param_initializer)) + param_attr=fluid.ParamAttr(initializer=param_initializer), + ) def forward(self, queries, keys, values, attn_bias, cache=None): # compute q ,k ,v @@ -132,10 +151,9 @@ def forward(self, queries, keys, values, attn_bias, cache=None): v = layers.concat([cache_v, v], axis=2) cache["k"], cache["v"] = k, v # scale dot product attention - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.d_model**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.d_model**-0.5 + ) if attn_bias is not None: product += attn_bias weights = layers.softmax(product) @@ -149,7 +167,6 @@ def forward(self, queries, keys, values, attn_bias, cache=None): class FFN(Layer): - def __init__(self, d_inner_hid, d_model, dropout_rate): super(FFN, self).__init__() self.dropout_rate = dropout_rate @@ -165,37 +182,44 @@ def forward(self, x): class EncoderLayer(Layer): - - def __init__(self, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): + def __init__( + self, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + ): super(EncoderLayer, self).__init__() - self.preprocesser1 = PrePostProcessLayer(preprocess_cmd, d_model, - prepostprocess_dropout) - self.self_attn = MultiHeadAttention(d_key, d_value, d_model, n_head, - attention_dropout) - self.postprocesser1 = PrePostProcessLayer(postprocess_cmd, d_model, - prepostprocess_dropout) + self.preprocesser1 = PrePostProcessLayer( + preprocess_cmd, d_model, prepostprocess_dropout + ) + self.self_attn = MultiHeadAttention( + d_key, d_value, d_model, n_head, attention_dropout + ) + self.postprocesser1 = PrePostProcessLayer( + postprocess_cmd, d_model, prepostprocess_dropout + ) - self.preprocesser2 = PrePostProcessLayer(preprocess_cmd, d_model, - prepostprocess_dropout) + self.preprocesser2 = PrePostProcessLayer( + preprocess_cmd, d_model, prepostprocess_dropout + ) self.ffn = FFN(d_inner_hid, d_model, relu_dropout) - self.postprocesser2 = PrePostProcessLayer(postprocess_cmd, d_model, - prepostprocess_dropout) + self.postprocesser2 = PrePostProcessLayer( + postprocess_cmd, d_model, prepostprocess_dropout + ) def forward(self, enc_input, attn_bias): - attn_output = self.self_attn(self.preprocesser1(enc_input), None, None, - attn_bias) + attn_output = self.self_attn( + self.preprocesser1(enc_input), None, None, attn_bias + ) attn_output = self.postprocesser1(attn_output, enc_input) ffn_output = self.ffn(self.preprocesser2(attn_output)) ffn_output = self.postprocesser2(ffn_output, attn_output) @@ -203,19 +227,20 @@ def forward(self, enc_input, attn_bias): class Encoder(Layer): - - def __init__(self, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): + def __init__( + self, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + ): super(Encoder, self).__init__() @@ -224,12 +249,23 @@ def __init__(self, self.encoder_layers.append( self.add_sublayer( "layer_%d" % i, - EncoderLayer(n_head, d_key, d_value, d_model, d_inner_hid, - prepostprocess_dropout, attention_dropout, - relu_dropout, preprocess_cmd, - postprocess_cmd))) - self.processer = PrePostProcessLayer(preprocess_cmd, d_model, - prepostprocess_dropout) + EncoderLayer( + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ), + ) + ) + self.processer = PrePostProcessLayer( + preprocess_cmd, d_model, prepostprocess_dropout + ) def forward(self, enc_input, attn_bias): for encoder_layer in self.encoder_layers: @@ -240,14 +276,15 @@ def forward(self, enc_input, attn_bias): class Embedder(Layer): - def __init__(self, vocab_size, emb_dim, bos_idx=0): super(Embedder, self).__init__() self.word_embedder = Embedding( size=[vocab_size, emb_dim], padding_idx=bos_idx, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Normal(0., emb_dim**-0.5))) + initializer=fluid.initializer.Normal(0.0, emb_dim**-0.5) + ), + ) def forward(self, word): word_emb = self.word_embedder(word) @@ -255,11 +292,23 @@ def forward(self, word): class WrapEncoder(Layer): - - def __init__(self, src_vocab_size, max_length, n_layer, n_head, d_key, - d_value, d_model, d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, preprocess_cmd, - postprocess_cmd, word_embedder): + def __init__( + self, + src_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + word_embedder, + ): super(WrapEncoder, self).__init__() self.emb_dropout = prepostprocess_dropout self.emb_dim = d_model @@ -268,12 +317,24 @@ def __init__(self, src_vocab_size, max_length, n_layer, n_head, d_key, size=[max_length, self.emb_dim], param_attr=fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - position_encoding_init(max_length, self.emb_dim)), - trainable=False)) - self.encoder = Encoder(n_layer, n_head, d_key, d_value, d_model, - d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, preprocess_cmd, - postprocess_cmd) + position_encoding_init(max_length, self.emb_dim) + ), + trainable=False, + ), + ) + self.encoder = Encoder( + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ) def forward(self, src_word, src_pos, src_slf_attn_bias): word_emb = self.word_embedder(src_word) @@ -281,71 +342,96 @@ def forward(self, src_word, src_pos, src_slf_attn_bias): pos_enc = self.pos_encoder(src_pos) pos_enc.stop_gradient = True emb = word_emb + pos_enc - enc_input = layers.dropout( - emb, - dropout_prob=self.emb_dropout, - ) if self.emb_dropout else emb + enc_input = ( + layers.dropout( + emb, + dropout_prob=self.emb_dropout, + ) + if self.emb_dropout + else emb + ) enc_output = self.encoder(enc_input, src_slf_attn_bias) return enc_output class DecoderLayer(Layer): - - def __init__(self, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): + def __init__( + self, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + ): super(DecoderLayer, self).__init__() - self.preprocesser1 = PrePostProcessLayer(preprocess_cmd, d_model, - prepostprocess_dropout) - self.self_attn = MultiHeadAttention(d_key, d_value, d_model, n_head, - attention_dropout) - self.postprocesser1 = PrePostProcessLayer(postprocess_cmd, d_model, - prepostprocess_dropout) - self.preprocesser2 = PrePostProcessLayer(preprocess_cmd, d_model, - prepostprocess_dropout) - self.cross_attn = MultiHeadAttention(d_key, d_value, d_model, n_head, - attention_dropout) - self.postprocesser2 = PrePostProcessLayer(postprocess_cmd, d_model, - prepostprocess_dropout) - self.preprocesser3 = PrePostProcessLayer(preprocess_cmd, d_model, - prepostprocess_dropout) + self.preprocesser1 = PrePostProcessLayer( + preprocess_cmd, d_model, prepostprocess_dropout + ) + self.self_attn = MultiHeadAttention( + d_key, d_value, d_model, n_head, attention_dropout + ) + self.postprocesser1 = PrePostProcessLayer( + postprocess_cmd, d_model, prepostprocess_dropout + ) + self.preprocesser2 = PrePostProcessLayer( + preprocess_cmd, d_model, prepostprocess_dropout + ) + self.cross_attn = MultiHeadAttention( + d_key, d_value, d_model, n_head, attention_dropout + ) + self.postprocesser2 = PrePostProcessLayer( + postprocess_cmd, d_model, prepostprocess_dropout + ) + self.preprocesser3 = PrePostProcessLayer( + preprocess_cmd, d_model, prepostprocess_dropout + ) self.ffn = FFN(d_inner_hid, d_model, relu_dropout) - self.postprocesser3 = PrePostProcessLayer(postprocess_cmd, d_model, - prepostprocess_dropout) + self.postprocesser3 = PrePostProcessLayer( + postprocess_cmd, d_model, prepostprocess_dropout + ) - def forward(self, - dec_input, - enc_output, - self_attn_bias, - cross_attn_bias, - cache=None): - self_attn_output = self.self_attn(self.preprocesser1(dec_input), None, - None, self_attn_bias, cache) + def forward( + self, dec_input, enc_output, self_attn_bias, cross_attn_bias, cache=None + ): + self_attn_output = self.self_attn( + self.preprocesser1(dec_input), None, None, self_attn_bias, cache + ) self_attn_output = self.postprocesser1(self_attn_output, dec_input) cross_attn_output = self.cross_attn( - self.preprocesser2(self_attn_output), enc_output, enc_output, - cross_attn_bias) - cross_attn_output = self.postprocesser2(cross_attn_output, - self_attn_output) + self.preprocesser2(self_attn_output), + enc_output, + enc_output, + cross_attn_bias, + ) + cross_attn_output = self.postprocesser2( + cross_attn_output, self_attn_output + ) ffn_output = self.ffn(self.preprocesser3(cross_attn_output)) ffn_output = self.postprocesser3(ffn_output, cross_attn_output) return ffn_output class Decoder(Layer): - - def __init__(self, n_layer, n_head, d_key, d_value, d_model, d_inner_hid, - prepostprocess_dropout, attention_dropout, relu_dropout, - preprocess_cmd, postprocess_cmd): + def __init__( + self, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ): super(Decoder, self).__init__() self.decoder_layers = list() @@ -353,33 +439,63 @@ def __init__(self, n_layer, n_head, d_key, d_value, d_model, d_inner_hid, self.decoder_layers.append( self.add_sublayer( "layer_%d" % i, - DecoderLayer(n_head, d_key, d_value, d_model, d_inner_hid, - prepostprocess_dropout, attention_dropout, - relu_dropout, preprocess_cmd, - postprocess_cmd))) - self.processer = PrePostProcessLayer(preprocess_cmd, d_model, - prepostprocess_dropout) - - def forward(self, + DecoderLayer( + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ), + ) + ) + self.processer = PrePostProcessLayer( + preprocess_cmd, d_model, prepostprocess_dropout + ) + + def forward( + self, + dec_input, + enc_output, + self_attn_bias, + cross_attn_bias, + caches=None, + ): + for i, decoder_layer in enumerate(self.decoder_layers): + dec_output = decoder_layer( dec_input, enc_output, self_attn_bias, cross_attn_bias, - caches=None): - for i, decoder_layer in enumerate(self.decoder_layers): - dec_output = decoder_layer(dec_input, enc_output, self_attn_bias, - cross_attn_bias, - None if caches is None else caches[i]) + None if caches is None else caches[i], + ) dec_input = dec_output return self.processer(dec_output) class WrapDecoder(Layer): - - def __init__(self, trg_vocab_size, max_length, n_layer, n_head, d_key, - d_value, d_model, d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, preprocess_cmd, - postprocess_cmd, share_input_output_embed, word_embedder): + def __init__( + self, + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + share_input_output_embed, + word_embedder, + ): super(WrapDecoder, self).__init__() self.emb_dropout = prepostprocess_dropout @@ -389,40 +505,58 @@ def __init__(self, trg_vocab_size, max_length, n_layer, n_head, d_key, size=[max_length, self.emb_dim], param_attr=fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - position_encoding_init(max_length, self.emb_dim)), - trainable=False)) - self.decoder = Decoder(n_layer, n_head, d_key, d_value, d_model, - d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, preprocess_cmd, - postprocess_cmd) + position_encoding_init(max_length, self.emb_dim) + ), + trainable=False, + ), + ) + self.decoder = Decoder( + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ) if share_input_output_embed: - self.linear = lambda x: layers.matmul(x=x, - y=self.word_embedder. - word_embedder.weight, - transpose_y=True) + self.linear = lambda x: layers.matmul( + x=x, y=self.word_embedder.word_embedder.weight, transpose_y=True + ) else: - self.linear = Linear(input_dim=d_model, - output_dim=trg_vocab_size, - bias_attr=False) - - def forward(self, - trg_word, - trg_pos, - trg_slf_attn_bias, - trg_src_attn_bias, - enc_output, - caches=None): + self.linear = Linear( + input_dim=d_model, output_dim=trg_vocab_size, bias_attr=False + ) + + def forward( + self, + trg_word, + trg_pos, + trg_slf_attn_bias, + trg_src_attn_bias, + enc_output, + caches=None, + ): word_emb = self.word_embedder(trg_word) word_emb = layers.scale(x=word_emb, scale=self.emb_dim**0.5) pos_enc = self.pos_encoder(trg_pos) pos_enc.stop_gradient = True emb = word_emb + pos_enc - dec_input = layers.dropout( - emb, - dropout_prob=self.emb_dropout, - ) if self.emb_dropout else emb - dec_output = self.decoder(dec_input, enc_output, trg_slf_attn_bias, - trg_src_attn_bias, caches) + dec_input = ( + layers.dropout( + emb, + dropout_prob=self.emb_dropout, + ) + if self.emb_dropout + else emb + ) + dec_output = self.decoder( + dec_input, enc_output, trg_slf_attn_bias, trg_src_attn_bias, caches + ) dec_output = layers.reshape( dec_output, shape=[-1, dec_output.shape[-1]], @@ -432,20 +566,21 @@ def forward(self, class CrossEntropyCriterion(object): - def __init__(self, label_smooth_eps): self.label_smooth_eps = label_smooth_eps def __call__(self, predict, label, weights): if self.label_smooth_eps: - label_out = layers.label_smooth(label=layers.one_hot( - input=label, depth=predict.shape[-1]), - epsilon=self.label_smooth_eps) + label_out = layers.label_smooth( + label=layers.one_hot(input=label, depth=predict.shape[-1]), + epsilon=self.label_smooth_eps, + ) cost = layers.softmax_with_cross_entropy( logits=predict, label=label_out, - soft_label=True if self.label_smooth_eps else False) + soft_label=True if self.label_smooth_eps else False, + ) weighted_cost = cost * weights sum_cost = layers.reduce_sum(weighted_cost) token_num = layers.reduce_sum(weights) @@ -455,49 +590,72 @@ def __call__(self, predict, label, weights): class Transformer(Layer): - - def __init__(self, - src_vocab_size, - trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - bos_id=0, - eos_id=1): + def __init__( + self, + src_vocab_size, + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + bos_id=0, + eos_id=1, + ): super(Transformer, self).__init__() - src_word_embedder = Embedder(vocab_size=src_vocab_size, - emb_dim=d_model, - bos_idx=bos_id) - self.encoder = WrapEncoder(src_vocab_size, max_length, n_layer, n_head, - d_key, d_value, d_model, d_inner_hid, - prepostprocess_dropout, attention_dropout, - relu_dropout, preprocess_cmd, - postprocess_cmd, src_word_embedder) + src_word_embedder = Embedder( + vocab_size=src_vocab_size, emb_dim=d_model, bos_idx=bos_id + ) + self.encoder = WrapEncoder( + src_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + src_word_embedder, + ) if weight_sharing: - assert src_vocab_size == trg_vocab_size, ( - "Vocabularies in source and target should be same for weight sharing." - ) + assert ( + src_vocab_size == trg_vocab_size + ), "Vocabularies in source and target should be same for weight sharing." trg_word_embedder = src_word_embedder else: - trg_word_embedder = Embedder(vocab_size=trg_vocab_size, - emb_dim=d_model, - bos_idx=bos_id) - self.decoder = WrapDecoder(trg_vocab_size, max_length, n_layer, n_head, - d_key, d_value, d_model, d_inner_hid, - prepostprocess_dropout, attention_dropout, - relu_dropout, preprocess_cmd, - postprocess_cmd, weight_sharing, - trg_word_embedder) + trg_word_embedder = Embedder( + vocab_size=trg_vocab_size, emb_dim=d_model, bos_idx=bos_id + ) + self.decoder = WrapDecoder( + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + trg_word_embedder, + ) self.trg_vocab_size = trg_vocab_size self.n_layer = n_layer @@ -506,28 +664,39 @@ def __init__(self, self.d_value = d_value @dygraph_to_static_func - def forward(self, src_word, src_pos, src_slf_attn_bias, trg_word, trg_pos, - trg_slf_attn_bias, trg_src_attn_bias): + def forward( + self, + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_pos, + trg_slf_attn_bias, + trg_src_attn_bias, + ): enc_output = self.encoder(src_word, src_pos, src_slf_attn_bias) - predict = self.decoder(trg_word, trg_pos, trg_slf_attn_bias, - trg_src_attn_bias, enc_output) + predict = self.decoder( + trg_word, trg_pos, trg_slf_attn_bias, trg_src_attn_bias, enc_output + ) return predict @dygraph_to_static_func - def beam_search(self, - src_word, - src_pos, - src_slf_attn_bias, - trg_word, - trg_src_attn_bias, - bos_id=0, - eos_id=1, - beam_size=4, - max_len=256): - + def beam_search( + self, + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_src_attn_bias, + bos_id=0, + eos_id=1, + beam_size=4, + max_len=256, + ): def expand_to_beam_size(tensor, beam_size): - tensor = layers.reshape(tensor, [tensor.shape[0], 1] + - list(tensor.shape[1:])) + tensor = layers.reshape( + tensor, [tensor.shape[0], 1] + list(tensor.shape[1:]) + ) tile_dims = [1] * len(tensor.shape) tile_dims[1] = beam_size return layers.expand(tensor, tile_dims) @@ -536,44 +705,60 @@ def merge_batch_beams(tensor): var_dim_in_state = 2 # count in beam dim tensor = layers.transpose( tensor, - list(range(var_dim_in_state, len(tensor.shape))) + - list(range(0, var_dim_in_state))) + list(range(var_dim_in_state, len(tensor.shape))) + + list(range(0, var_dim_in_state)), + ) - tensor = layers.reshape(tensor, [0] * - (len(tensor.shape) - var_dim_in_state) + - [batch_size * beam_size]) + tensor = layers.reshape( + tensor, + [0] * (len(tensor.shape) - var_dim_in_state) + + [batch_size * beam_size], + ) res = layers.transpose( tensor, list( - range((len(tensor.shape) + 1 - var_dim_in_state), - len(tensor.shape))) + - list(range(0, (len(tensor.shape) + 1 - var_dim_in_state)))) + range( + (len(tensor.shape) + 1 - var_dim_in_state), + len(tensor.shape), + ) + ) + + list(range(0, (len(tensor.shape) + 1 - var_dim_in_state))), + ) return res def split_batch_beams(tensor): var_dim_in_state = 1 tensor = layers.transpose( tensor, - list(range(var_dim_in_state, len(tensor.shape))) + - list(range(0, var_dim_in_state))) - tensor = layers.reshape(tensor, [0] * - (len(tensor.shape) - var_dim_in_state) + - [batch_size, beam_size]) + list(range(var_dim_in_state, len(tensor.shape))) + + list(range(0, var_dim_in_state)), + ) + tensor = layers.reshape( + tensor, + [0] * (len(tensor.shape) - var_dim_in_state) + + [batch_size, beam_size], + ) res = layers.transpose( tensor, list( - range((len(tensor.shape) - 1 - var_dim_in_state), - len(tensor.shape))) + - list(range(0, (len(tensor.shape) - 1 - var_dim_in_state)))) + range( + (len(tensor.shape) - 1 - var_dim_in_state), + len(tensor.shape), + ) + ) + + list(range(0, (len(tensor.shape) - 1 - var_dim_in_state))), + ) return res def mask_probs(probs, finished, noend_mask_tensor): finished = layers.cast(finished, dtype=probs.dtype) - probs = layers.elementwise_mul(layers.expand( - layers.unsqueeze(finished, [2]), [1, 1, self.trg_vocab_size]), - noend_mask_tensor, - axis=-1) - layers.elementwise_mul( - probs, (finished - 1), axis=0) + probs = layers.elementwise_mul( + layers.expand( + layers.unsqueeze(finished, [2]), [1, 1, self.trg_vocab_size] + ), + noend_mask_tensor, + axis=-1, + ) - layers.elementwise_mul(probs, (finished - 1), axis=0) return probs def gather(input, indices, batch_pos): @@ -585,87 +770,108 @@ def gather(input, indices, batch_pos): batch_size = enc_output.shape[0] # constant number - inf = float(1. * 1e7) + inf = float(1.0 * 1e7) max_len = (enc_output.shape[1] + 20) if max_len is None else max_len - vocab_size_tensor = layers.fill_constant(shape=[1], - dtype="int64", - value=self.trg_vocab_size) + vocab_size_tensor = layers.fill_constant( + shape=[1], dtype="int64", value=self.trg_vocab_size + ) end_token_tensor = to_variable( - np.full([batch_size, beam_size], eos_id, dtype="int64")) + np.full([batch_size, beam_size], eos_id, dtype="int64") + ) noend_array = [-inf] * self.trg_vocab_size noend_array[eos_id] = 0 noend_mask_tensor = to_variable(np.array(noend_array, dtype="float32")) batch_pos = layers.expand( layers.unsqueeze( - to_variable(np.arange(0, batch_size, 1, dtype="int64")), [1]), - [1, beam_size]) + to_variable(np.arange(0, batch_size, 1, dtype="int64")), [1] + ), + [1, beam_size], + ) predict_ids = [] parent_ids = [] ### initialize states of beam search ### log_probs = to_variable( - np.array([[0.] + [-inf] * (beam_size - 1)] * batch_size, - dtype="float32")) + np.array( + [[0.0] + [-inf] * (beam_size - 1)] * batch_size, dtype="float32" + ) + ) - finished = to_variable(np.full([batch_size, beam_size], 0, - dtype="bool")) + finished = to_variable( + np.full([batch_size, beam_size], 0, dtype="bool") + ) - trg_word = layers.fill_constant(shape=[batch_size * beam_size, 1], - dtype="int64", - value=bos_id) + trg_word = layers.fill_constant( + shape=[batch_size * beam_size, 1], dtype="int64", value=bos_id + ) trg_src_attn_bias = merge_batch_beams( - expand_to_beam_size(trg_src_attn_bias, beam_size)) + expand_to_beam_size(trg_src_attn_bias, beam_size) + ) enc_output = merge_batch_beams( - expand_to_beam_size(enc_output, beam_size)) + expand_to_beam_size(enc_output, beam_size) + ) # init states (caches) for transformer, need to be updated according to selected beam - caches = [{ - "k": - layers.fill_constant( - shape=[batch_size, beam_size, self.n_head, 0, self.d_key], - dtype=enc_output.dtype, - value=0), - "v": - layers.fill_constant( - shape=[batch_size, beam_size, self.n_head, 0, self.d_value], - dtype=enc_output.dtype, - value=0), - } for i in range(self.n_layer)] + caches = [ + { + "k": layers.fill_constant( + shape=[batch_size, beam_size, self.n_head, 0, self.d_key], + dtype=enc_output.dtype, + value=0, + ), + "v": layers.fill_constant( + shape=[batch_size, beam_size, self.n_head, 0, self.d_value], + dtype=enc_output.dtype, + value=0, + ), + } + for i in range(self.n_layer) + ] for i in range(paddle.to_tensor(max_len)): - trg_pos = layers.fill_constant(shape=trg_word.shape, - dtype="int64", - value=i) - caches = map_structure(merge_batch_beams, - caches) # TODO: modified for dygraph2static - logits = self.decoder(trg_word, trg_pos, None, trg_src_attn_bias, - enc_output, caches) + trg_pos = layers.fill_constant( + shape=trg_word.shape, dtype="int64", value=i + ) + caches = map_structure( + merge_batch_beams, caches + ) # TODO: modified for dygraph2static + logits = self.decoder( + trg_word, trg_pos, None, trg_src_attn_bias, enc_output, caches + ) caches = map_structure(split_batch_beams, caches) step_log_probs = split_batch_beams( - fluid.layers.log(fluid.layers.softmax(logits))) - - step_log_probs = mask_probs(step_log_probs, finished, - noend_mask_tensor) - log_probs = layers.elementwise_add(x=step_log_probs, - y=log_probs, - axis=0) - log_probs = layers.reshape(log_probs, - [-1, beam_size * self.trg_vocab_size]) + fluid.layers.log(fluid.layers.softmax(logits)) + ) + + step_log_probs = mask_probs( + step_log_probs, finished, noend_mask_tensor + ) + log_probs = layers.elementwise_add( + x=step_log_probs, y=log_probs, axis=0 + ) + log_probs = layers.reshape( + log_probs, [-1, beam_size * self.trg_vocab_size] + ) scores = log_probs - topk_scores, topk_indices = fluid.layers.topk(input=scores, - k=beam_size) + topk_scores, topk_indices = fluid.layers.topk( + input=scores, k=beam_size + ) beam_indices = fluid.layers.elementwise_floordiv( - topk_indices, vocab_size_tensor) + topk_indices, vocab_size_tensor + ) token_indices = fluid.layers.elementwise_mod( - topk_indices, vocab_size_tensor) + topk_indices, vocab_size_tensor + ) # update states - caches = map_structure(lambda x: gather(x, beam_indices, batch_pos), - caches) + caches = map_structure( + lambda x: gather(x, beam_indices, batch_pos), caches + ) log_probs = gather(log_probs, topk_indices, batch_pos) finished = gather(finished, beam_indices, batch_pos) finished = layers.logical_or( - finished, layers.equal(token_indices, end_token_tensor)) + finished, layers.equal(token_indices, end_token_tensor) + ) trg_word = layers.reshape(token_indices, [-1, 1]) predict_ids.append(token_indices) @@ -677,7 +883,8 @@ def gather(input, indices, batch_pos): predict_ids = layers.stack(predict_ids, axis=0) parent_ids = layers.stack(parent_ids, axis=0) finished_seq = layers.transpose( - layers.gather_tree(predict_ids, parent_ids), [1, 2, 0]) + layers.gather_tree(predict_ids, parent_ids), [1, 2, 0] + ) finished_scores = topk_scores return finished_seq, finished_scores diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_util.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_util.py index 3b4d1438c5f00..7e77f1ad033db 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_util.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/transformer_util.py @@ -31,36 +31,45 @@ def get_input_descs(args, mode="train"): input_descs_train = { "src_word": [(batch_size, seq_len), "int64", 2], "src_pos": [(batch_size, seq_len), "int64"], - "src_slf_attn_bias": [(batch_size, n_head, seq_len, seq_len), - "float32"], + "src_slf_attn_bias": [ + (batch_size, n_head, seq_len, seq_len), + "float32", + ], "trg_word": [(batch_size, seq_len), "int64", 2], "trg_pos": [(batch_size, seq_len), "int64"], - "trg_slf_attn_bias": [(batch_size, n_head, seq_len, seq_len), - "float32"], - "trg_src_attn_bias": - [(batch_size, n_head, seq_len, seq_len), - "float32"], # TODO: 1 for predict, seq_len for train + "trg_slf_attn_bias": [ + (batch_size, n_head, seq_len, seq_len), + "float32", + ], + "trg_src_attn_bias": [ + (batch_size, n_head, seq_len, seq_len), + "float32", + ], # TODO: 1 for predict, seq_len for train "enc_output": [(batch_size, seq_len, d_model), "float32"], "lbl_word": [(None, 1), "int64"], "lbl_weight": [(None, 1), "float32"], "init_score": [(batch_size, 1), "float32", 2], - "init_idx": [(batch_size, ), "int32"], + "init_idx": [(batch_size,), "int32"], } input_descs_predict = { "src_word": [(batch_size, seq_len), "int64", 2], "src_pos": [(batch_size, seq_len), "int64"], - "src_slf_attn_bias": [(batch_size, n_head, seq_len, seq_len), - "float32"], + "src_slf_attn_bias": [ + (batch_size, n_head, seq_len, seq_len), + "float32", + ], "trg_word": [(batch_size, seq_len), "int64", 2], "trg_pos": [(batch_size, seq_len), "int64"], - "trg_slf_attn_bias": [(batch_size, n_head, seq_len, seq_len), - "float32"], + "trg_slf_attn_bias": [ + (batch_size, n_head, seq_len, seq_len), + "float32", + ], "trg_src_attn_bias": [(batch_size, n_head, 1, seq_len), "float32"], "enc_output": [(batch_size, seq_len, d_model), "float32"], "lbl_word": [(None, 1), "int64"], "lbl_weight": [(None, 1), "float32"], "init_score": [(batch_size, 1), "float32", 2], - "init_idx": [(batch_size, ), "int32"], + "init_idx": [(batch_size,), "int32"], } return input_descs_train if mode == "train" else input_descs_predict @@ -125,43 +134,58 @@ class ModelHyperParams(object): weight_sharing = True -def pad_batch_data(insts, - pad_idx, - n_head, - is_target=False, - is_label=False, - return_attn_bias=True, - return_max_len=True, - return_num_token=False): +def pad_batch_data( + insts, + pad_idx, + n_head, + is_target=False, + is_label=False, + return_attn_bias=True, + return_max_len=True, + return_num_token=False, +): return_list = [] max_len = max(len(inst) for inst in insts) inst_data = np.array( - [inst + [pad_idx] * (max_len - len(inst)) for inst in insts]) + [inst + [pad_idx] * (max_len - len(inst)) for inst in insts] + ) return_list += [inst_data.astype("int64").reshape([-1, 1])] if is_label: # label weight - inst_weight = np.array([[1.] * len(inst) + [0.] * (max_len - len(inst)) - for inst in insts]) + inst_weight = np.array( + [ + [1.0] * len(inst) + [0.0] * (max_len - len(inst)) + for inst in insts + ] + ) return_list += [inst_weight.astype("float32").reshape([-1, 1])] else: # position data - inst_pos = np.array([ - list(range(0, len(inst))) + [0] * (max_len - len(inst)) - for inst in insts - ]) + inst_pos = np.array( + [ + list(range(0, len(inst))) + [0] * (max_len - len(inst)) + for inst in insts + ] + ) return_list += [inst_pos.astype("int64").reshape([-1, 1])] if return_attn_bias: if is_target: slf_attn_bias_data = np.ones((inst_data.shape[0], max_len, max_len)) - slf_attn_bias_data = np.triu(slf_attn_bias_data, - 1).reshape([-1, 1, max_len, max_len]) - slf_attn_bias_data = np.tile(slf_attn_bias_data, - [1, n_head, 1, 1]) * [-1e9] + slf_attn_bias_data = np.triu(slf_attn_bias_data, 1).reshape( + [-1, 1, max_len, max_len] + ) + slf_attn_bias_data = np.tile( + slf_attn_bias_data, [1, n_head, 1, 1] + ) * [-1e9] else: - slf_attn_bias_data = np.array([[0] * len(inst) + [-1e9] * - (max_len - len(inst)) - for inst in insts]) + slf_attn_bias_data = np.array( + [ + [0] * len(inst) + [-1e9] * (max_len - len(inst)) + for inst in insts + ] + ) slf_attn_bias_data = np.tile( slf_attn_bias_data.reshape([-1, 1, 1, max_len]), - [1, n_head, max_len, 1]) + [1, n_head, max_len, 1], + ) return_list += [slf_attn_bias_data.astype("float32")] if return_max_len: return_list += [max_len] @@ -175,16 +199,19 @@ def pad_batch_data(insts, def prepare_train_input(insts, src_pad_idx, trg_pad_idx, n_head): src_word, src_pos, src_slf_attn_bias, src_max_len = pad_batch_data( - [inst[0] for inst in insts], src_pad_idx, n_head, is_target=False) + [inst[0] for inst in insts], src_pad_idx, n_head, is_target=False + ) src_word = src_word.reshape(-1, src_max_len) src_pos = src_pos.reshape(-1, src_max_len) trg_word, trg_pos, trg_slf_attn_bias, trg_max_len = pad_batch_data( - [inst[1] for inst in insts], trg_pad_idx, n_head, is_target=True) + [inst[1] for inst in insts], trg_pad_idx, n_head, is_target=True + ) trg_word = trg_word.reshape(-1, trg_max_len) trg_pos = trg_pos.reshape(-1, trg_max_len) - trg_src_attn_bias = np.tile(src_slf_attn_bias[:, :, ::src_max_len, :], - [1, 1, trg_max_len, 1]).astype("float32") + trg_src_attn_bias = np.tile( + src_slf_attn_bias[:, :, ::src_max_len, :], [1, 1, trg_max_len, 1] + ).astype("float32") lbl_word, lbl_weight, num_token = pad_batch_data( [inst[2] for inst in insts], @@ -194,13 +221,21 @@ def prepare_train_input(insts, src_pad_idx, trg_pad_idx, n_head): is_label=True, return_attn_bias=False, return_max_len=False, - return_num_token=True) + return_num_token=True, + ) lbl_word = lbl_word.reshape(-1, 1) lbl_weight = lbl_weight.reshape(-1, 1) data_inputs = [ - src_word, src_pos, src_slf_attn_bias, trg_word, trg_pos, - trg_slf_attn_bias, trg_src_attn_bias, lbl_word, lbl_weight + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_pos, + trg_slf_attn_bias, + trg_src_attn_bias, + lbl_word, + lbl_weight, ] return data_inputs @@ -208,55 +243,66 @@ def prepare_train_input(insts, src_pad_idx, trg_pad_idx, n_head): def prepare_infer_input(insts, src_pad_idx, bos_idx, n_head): src_word, src_pos, src_slf_attn_bias, src_max_len = pad_batch_data( - [inst[0] for inst in insts], src_pad_idx, n_head, is_target=False) + [inst[0] for inst in insts], src_pad_idx, n_head, is_target=False + ) # start tokens trg_word = np.asarray([[bos_idx]] * len(insts), dtype="int64") - trg_src_attn_bias = np.tile(src_slf_attn_bias[:, :, ::src_max_len, :], - [1, 1, 1, 1]).astype("float32") + trg_src_attn_bias = np.tile( + src_slf_attn_bias[:, :, ::src_max_len, :], [1, 1, 1, 1] + ).astype("float32") trg_word = trg_word.reshape(-1, 1) src_word = src_word.reshape(-1, src_max_len) src_pos = src_pos.reshape(-1, src_max_len) data_inputs = [ - src_word, src_pos, src_slf_attn_bias, trg_word, trg_src_attn_bias + src_word, + src_pos, + src_slf_attn_bias, + trg_word, + trg_src_attn_bias, ] return data_inputs def get_feed_data_reader(args, mode='train'): - def __for_train__(): - train_reader = paddle.batch(wmt16.train(args.src_vocab_size, - args.trg_vocab_size), - batch_size=args.batch_size) + train_reader = paddle.batch( + wmt16.train(args.src_vocab_size, args.trg_vocab_size), + batch_size=args.batch_size, + ) for batch in train_reader(): - tensors = prepare_train_input(batch, args.eos_idx, args.eos_idx, - args.n_head) + tensors = prepare_train_input( + batch, args.eos_idx, args.eos_idx, args.n_head + ) yield tensors def __for_test__(): - test_reader = paddle.batch(wmt16.test(args.src_vocab_size, - args.trg_vocab_size), - batch_size=args.batch_size) + test_reader = paddle.batch( + wmt16.test(args.src_vocab_size, args.trg_vocab_size), + batch_size=args.batch_size, + ) for batch in test_reader(): - tensors = prepare_infer_input(batch, args.eos_idx, args.eos_idx, - args.n_head) + tensors = prepare_infer_input( + batch, args.eos_idx, args.eos_idx, args.n_head + ) yield tensors return __for_train__ if mode == 'train' else __for_test__ class InputField(object): - def __init__(self, input_slots): self.feed_list = [] for slot in input_slots: self.feed_list.append( - fluid.layers.data(name=slot['name'], - shape=slot['shape'], - dtype=slot['dtype'], - lod_level=slot.get('lod_level', 0), - append_batch_size=False)) + fluid.layers.data( + name=slot['name'], + shape=slot['shape'], + dtype=slot['dtype'], + lod_level=slot.get('lod_level', 0), + append_batch_size=False, + ) + ) def load(program, model_path, executor=None, var_list=None): @@ -269,7 +315,8 @@ def load(program, model_path, executor=None, var_list=None): warnings.warn( "An UnicodeDecodeError is catched, which might be caused by loading " "a python2 saved model. Encoding of pickle.load would be set and " - "load again automatically.") + "load again automatically." + ) load_bak = pickle.load pickle.load = partial(load_bak, encoding="latin1") fluid.load(program, model_path, executor, var_list) @@ -282,16 +329,19 @@ def load_dygraph(model_path, keep_name_table=False): """ try: para_dict, opti_dict = fluid.load_dygraph( - model_path, keep_name_table=keep_name_table) + model_path, keep_name_table=keep_name_table + ) return para_dict, opti_dict except UnicodeDecodeError: warnings.warn( "An UnicodeDecodeError is catched, which might be caused by loading " "a python2 saved model. Encoding of pickle.load would be set and " - "load again automatically.") + "load again automatically." + ) load_bak = pickle.load pickle.load = partial(load_bak, encoding="latin1") para_dict, opti_dict = fluid.load_dygraph( - model_path, keep_name_table=keep_name_table) + model_path, keep_name_table=keep_name_table + ) pickle.load = load_bak return para_dict, opti_dict diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/tsm_config_utils.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/tsm_config_utils.py index 3169311710e4f..1332e0cb86bf6 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/tsm_config_utils.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/tsm_config_utils.py @@ -1,16 +1,16 @@ # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import logging @@ -25,7 +25,6 @@ class AttrDict(dict): - def __getattr__(self, key): return self[key] @@ -39,6 +38,7 @@ def __setattr__(self, key, value): def parse_config(cfg_file): """Load a config file into AttrDict""" import yaml + with open(cfg_file, 'r') as fopen: yaml_config = AttrDict(yaml.load(fopen, Loader=yaml.Loader)) create_attr_dict(yaml_config) @@ -47,6 +47,7 @@ def parse_config(cfg_file): def create_attr_dict(yaml_config): from ast import literal_eval + for key, value in yaml_config.items(): if type(value) is dict: yaml_config[key] = value = AttrDict(value) @@ -78,7 +79,8 @@ def merge_configs(cfg, sec, args_dict): def print_configs(cfg, mode): logger.info( - "---------------- {:>5} Arguments ----------------".format(mode)) + "---------------- {:>5} Arguments ----------------".format(mode) + ) for sec, sec_items in cfg.items(): logger.info("{}:".format(sec)) for k, v in sec_items.items(): diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/yolov3.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/yolov3.py index e502d786430c4..0a50751b1e781 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/yolov3.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/yolov3.py @@ -1,16 +1,16 @@ # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import os import sys @@ -26,7 +26,6 @@ class AttrDict(dict): - def __init__(self, *args, **kwargs): super(AttrDict, self).__init__(*args, **kwargs) @@ -78,12 +77,29 @@ def __setattr__(self, name, value): cfg.pixel_stds = [0.229, 0.224, 0.225] # anchors box weight and height cfg.anchors = [ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, 373, 326 + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, ] # anchor mask of each yolo layer cfg.anchor_masks = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] # IoU threshold to ignore objectness loss of pred box -cfg.ignore_thresh = .7 +cfg.ignore_thresh = 0.7 # # SOLVER options # @@ -97,7 +113,7 @@ def __setattr__(self, name, value): cfg.no_mixup_iter = 10 if fluid.is_compiled_with_cuda() else 1 # warm up to learning rate cfg.warm_up_iter = 10 if fluid.is_compiled_with_cuda() else 1 -cfg.warm_up_factor = 0. +cfg.warm_up_factor = 0.0 # lr steps_with_decay cfg.lr_steps = [400000, 450000] cfg.lr_gamma = 0.1 @@ -115,49 +131,61 @@ def __setattr__(self, name, value): class YoloDetectionBlock(fluid.dygraph.Layer): - def __init__(self, ch_in, channel, is_test=True): super(YoloDetectionBlock, self).__init__() - assert channel % 2 == 0, \ - "channel {} cannot be divided by 2".format(channel) - - self.conv0 = ConvBNLayer(ch_in=ch_in, - ch_out=channel, - filter_size=1, - stride=1, - padding=0, - is_test=is_test) - self.conv1 = ConvBNLayer(ch_in=channel, - ch_out=channel * 2, - filter_size=3, - stride=1, - padding=1, - is_test=is_test) - self.conv2 = ConvBNLayer(ch_in=channel * 2, - ch_out=channel, - filter_size=1, - stride=1, - padding=0, - is_test=is_test) - self.conv3 = ConvBNLayer(ch_in=channel, - ch_out=channel * 2, - filter_size=3, - stride=1, - padding=1, - is_test=is_test) - self.route = ConvBNLayer(ch_in=channel * 2, - ch_out=channel, - filter_size=1, - stride=1, - padding=0, - is_test=is_test) - self.tip = ConvBNLayer(ch_in=channel, - ch_out=channel * 2, - filter_size=3, - stride=1, - padding=1, - is_test=is_test) + assert channel % 2 == 0, "channel {} cannot be divided by 2".format( + channel + ) + + self.conv0 = ConvBNLayer( + ch_in=ch_in, + ch_out=channel, + filter_size=1, + stride=1, + padding=0, + is_test=is_test, + ) + self.conv1 = ConvBNLayer( + ch_in=channel, + ch_out=channel * 2, + filter_size=3, + stride=1, + padding=1, + is_test=is_test, + ) + self.conv2 = ConvBNLayer( + ch_in=channel * 2, + ch_out=channel, + filter_size=1, + stride=1, + padding=0, + is_test=is_test, + ) + self.conv3 = ConvBNLayer( + ch_in=channel, + ch_out=channel * 2, + filter_size=3, + stride=1, + padding=1, + is_test=is_test, + ) + self.route = ConvBNLayer( + ch_in=channel * 2, + ch_out=channel, + filter_size=1, + stride=1, + padding=0, + is_test=is_test, + ) + self.tip = ConvBNLayer( + ch_in=channel, + ch_out=channel * 2, + filter_size=3, + stride=1, + padding=1, + is_test=is_test, + ) def forward(self, inputs): out = self.conv0(inputs) @@ -170,7 +198,6 @@ def forward(self, inputs): class Upsample(fluid.dygraph.Layer): - def __init__(self, scale=2): super(Upsample, self).__init__() self.scale = scale @@ -178,24 +205,22 @@ def __init__(self, scale=2): def forward(self, inputs): # get dynamic upsample output shape shape_nchw = fluid.layers.shape(inputs) - shape_hw = fluid.layers.slice(shape_nchw, - axes=[0], - starts=[2], - ends=[4]) + shape_hw = fluid.layers.slice( + shape_nchw, axes=[0], starts=[2], ends=[4] + ) shape_hw.stop_gradient = True in_shape = fluid.layers.cast(shape_hw, dtype='int32') out_shape = in_shape * self.scale out_shape.stop_gradient = True # reisze by actual_shape - out = fluid.layers.resize_nearest(input=inputs, - scale=self.scale, - actual_shape=out_shape) + out = fluid.layers.resize_nearest( + input=inputs, scale=self.scale, actual_shape=out_shape + ) return out class YOLOv3(fluid.dygraph.Layer): - def __init__(self, ch_in, is_train=True, use_random=False): super(YOLOv3, self).__init__() @@ -210,47 +235,60 @@ def __init__(self, ch_in, is_train=True, use_random=False): for i in range(3): yolo_block = self.add_sublayer( "yolo_detecton_block_%d" % (i), - YoloDetectionBlock(ch_in_list[i], - channel=512 // (2**i), - is_test=not self.is_train)) + YoloDetectionBlock( + ch_in_list[i], + channel=512 // (2**i), + is_test=not self.is_train, + ), + ) self.yolo_blocks.append(yolo_block) num_filters = len(cfg.anchor_masks[i]) * (cfg.class_num + 5) block_out = self.add_sublayer( "block_out_%d" % (i), - Conv2D(num_channels=1024 // (2**i), - num_filters=num_filters, - filter_size=1, - stride=1, - padding=0, - act=None, - param_attr=ParamAttr( - initializer=fluid.initializer.Normal(0., 0.02)), - bias_attr=ParamAttr( - initializer=fluid.initializer.Constant(0.0), - regularizer=L2Decay(0.)))) + Conv2D( + num_channels=1024 // (2**i), + num_filters=num_filters, + filter_size=1, + stride=1, + padding=0, + act=None, + param_attr=ParamAttr( + initializer=fluid.initializer.Normal(0.0, 0.02) + ), + bias_attr=ParamAttr( + initializer=fluid.initializer.Constant(0.0), + regularizer=L2Decay(0.0), + ), + ), + ) self.block_outputs.append(block_out) if i < 2: route = self.add_sublayer( "route2_%d" % i, - ConvBNLayer(ch_in=512 // (2**i), - ch_out=256 // (2**i), - filter_size=1, - stride=1, - padding=0, - is_test=(not self.is_train))) + ConvBNLayer( + ch_in=512 // (2**i), + ch_out=256 // (2**i), + filter_size=1, + stride=1, + padding=0, + is_test=(not self.is_train), + ), + ) self.route_blocks_2.append(route) self.upsample = Upsample() @declarative - def forward(self, - inputs, - gtbox=None, - gtlabel=None, - gtscore=None, - im_id=None, - im_shape=None): + def forward( + self, + inputs, + gtbox=None, + gtlabel=None, + gtscore=None, + im_id=None, + im_shape=None, + ): self.outputs = [] self.boxes = [] self.scores = [] @@ -287,7 +325,8 @@ def forward(self, class_num=cfg.class_num, ignore_thresh=cfg.ignore_thresh, downsample_ratio=self.downsample, - use_label_smooth=cfg.label_smooth) + use_label_smooth=cfg.label_smooth, + ) self.losses.append(fluid.layers.reduce_mean(loss)) else: @@ -302,10 +341,12 @@ def forward(self, class_num=cfg.class_num, conf_thresh=cfg.valid_thresh, downsample_ratio=self.downsample, - name="yolo_box" + str(i)) + name="yolo_box" + str(i), + ) self.boxes.append(boxes) self.scores.append( - fluid.layers.transpose(scores, perm=[0, 2, 1])) + fluid.layers.transpose(scores, perm=[0, 2, 1]) + ) self.downsample //= 2 if not self.is_train: @@ -313,13 +354,15 @@ def forward(self, yolo_boxes = fluid.layers.concat(self.boxes, axis=1) yolo_scores = fluid.layers.concat(self.scores, axis=2) - pred = fluid.layers.multiclass_nms(bboxes=yolo_boxes, - scores=yolo_scores, - score_threshold=cfg.valid_thresh, - nms_top_k=cfg.nms_topk, - keep_top_k=cfg.nms_posk, - nms_threshold=cfg.nms_thresh, - background_label=-1) + pred = fluid.layers.multiclass_nms( + bboxes=yolo_boxes, + scores=yolo_scores, + score_threshold=cfg.valid_thresh, + nms_top_k=cfg.nms_topk, + keep_top_k=cfg.nms_posk, + nms_threshold=cfg.nms_thresh, + background_label=-1, + ) return pred else: return sum(self.losses) diff --git a/python/paddle/fluid/tests/unittests/elastic_demo.py b/python/paddle/fluid/tests/unittests/elastic_demo.py index af26abd0d8878..37a1c93bddd32 100644 --- a/python/paddle/fluid/tests/unittests/elastic_demo.py +++ b/python/paddle/fluid/tests/unittests/elastic_demo.py @@ -15,10 +15,16 @@ import os, sys import time -sys.stderr.write("{}-DISTRIBUTED_TRAINER_ENDPOINTS={}\n".format( - os.environ['PADDLE_TRAINER_ID'], - os.environ['DISTRIBUTED_TRAINER_ENDPOINTS'])) -sys.stderr.write("{}-PADDLE_TRAINERS={}\n".format( - os.environ['PADDLE_TRAINER_ID'], os.environ['PADDLE_TRAINERS'])) +sys.stderr.write( + "{}-DISTRIBUTED_TRAINER_ENDPOINTS={}\n".format( + os.environ['PADDLE_TRAINER_ID'], + os.environ['DISTRIBUTED_TRAINER_ENDPOINTS'], + ) +) +sys.stderr.write( + "{}-PADDLE_TRAINERS={}\n".format( + os.environ['PADDLE_TRAINER_ID'], os.environ['PADDLE_TRAINERS'] + ) +) time.sleep(600) diff --git a/python/paddle/fluid/tests/unittests/fake_reader.py b/python/paddle/fluid/tests/unittests/fake_reader.py index b1e90d8b1879e..997e9f0fc73a3 100644 --- a/python/paddle/fluid/tests/unittests/fake_reader.py +++ b/python/paddle/fluid/tests/unittests/fake_reader.py @@ -15,23 +15,24 @@ import numpy as np -def fake_imdb_reader(word_dict_size, - sample_num, - lower_seq_len=100, - upper_seq_len=200, - class_dim=2): - +def fake_imdb_reader( + word_dict_size, + sample_num, + lower_seq_len=100, + upper_seq_len=200, + class_dim=2, +): def __reader__(): for _ in range(sample_num): - length = np.random.random_integers(low=lower_seq_len, - high=upper_seq_len, - size=[1])[0] - ids = np.random.random_integers(low=0, - high=word_dict_size - 1, - size=[length]).astype('int64') - label = np.random.random_integers(low=0, - high=class_dim - 1, - size=[1]).astype('int64')[0] + length = np.random.random_integers( + low=lower_seq_len, high=upper_seq_len, size=[1] + )[0] + ids = np.random.random_integers( + low=0, high=word_dict_size - 1, size=[length] + ).astype('int64') + label = np.random.random_integers( + low=0, high=class_dim - 1, size=[1] + ).astype('int64')[0] yield ids, label return __reader__ diff --git a/python/paddle/fluid/tests/unittests/feed_data_reader.py b/python/paddle/fluid/tests/unittests/feed_data_reader.py index 6d7e73ac3c09a..9a1f15ddff5ec 100644 --- a/python/paddle/fluid/tests/unittests/feed_data_reader.py +++ b/python/paddle/fluid/tests/unittests/feed_data_reader.py @@ -17,7 +17,6 @@ def cyclic_reader(reader): - def __reader__(): while True: for data in reader(): @@ -27,7 +26,6 @@ def __reader__(): class FeedDataReader(object): - def __init__(self, feed_list, reader): self._feed_list = [] for var in feed_list: @@ -62,8 +60,11 @@ def get_next(self, exe, program): if program._is_data_parallel: use_executor = False if program._places is None: - device_num = len(fluid.cuda_places()) if use_cuda else len( - fluid.cpu_places()) + device_num = ( + len(fluid.cuda_places()) + if use_cuda + else len(fluid.cpu_places()) + ) else: device_num = len(program._places) else: diff --git a/python/paddle/fluid/tests/unittests/fft/spectral_op_np.py b/python/paddle/fluid/tests/unittests/fft/spectral_op_np.py index 0dc958f54fcf1..65918a0cd59ff 100644 --- a/python/paddle/fluid/tests/unittests/fft/spectral_op_np.py +++ b/python/paddle/fluid/tests/unittests/fft/spectral_op_np.py @@ -34,8 +34,9 @@ def _get_norm_mode(norm, forward): def _get_inv_norm(n, norm_mode): - assert isinstance(norm_mode, - NormMode), "invalid norm_type {}".format(norm_mode) + assert isinstance(norm_mode, NormMode), "invalid norm_type {}".format( + norm_mode + ) if norm_mode == NormMode.none: return 1.0 if norm_mode == NormMode.by_sqrt_n: @@ -69,8 +70,9 @@ def _fftc2r(a, n=None, axis=-1, norm=None, forward=None): if n is None: n = (a.shape[axis] - 1) * 2 inv_norm = _get_inv_norm(n, norm) - output = _raw_fft(a.conj() if forward else a, n, axis, True, False, - inv_norm) + output = _raw_fft( + a.conj() if forward else a, n, axis, True, False, inv_norm + ) return output @@ -145,8 +147,10 @@ def _fft_fill_conj_grad(x, axes, length_to_double): last_fft_axis = axes[-1] shape = x.shape for multi_index in np.ndindex(*shape): - if 0 < multi_index[last_fft_axis] and multi_index[ - last_fft_axis] <= length_to_double: + if ( + 0 < multi_index[last_fft_axis] + and multi_index[last_fft_axis] <= length_to_double + ): x[multi_index] *= 2 return x diff --git a/python/paddle/fluid/tests/unittests/fft/test_fft.py b/python/paddle/fluid/tests/unittests/fft/test_fft.py index e6ea9865ae3c3..332a61c07fa60 100644 --- a/python/paddle/fluid/tests/unittests/fft/test_fft.py +++ b/python/paddle/fluid/tests/unittests/fft/test_fft.py @@ -30,31 +30,28 @@ 'float32': 1e-03, 'complex64': 1e-3, 'float64': 1e-7, - 'complex128': 1e-7 + 'complex128': 1e-7, } ATOL = {'float32': 0.0, 'complex64': 0, 'float64': 0.0, 'complex128': 0} -def rand_x(dims=1, - dtype='float64', - min_dim_len=1, - max_dim_len=10, - complex=False): +def rand_x( + dims=1, dtype='float64', min_dim_len=1, max_dim_len=10, complex=False +): shape = [np.random.randint(min_dim_len, max_dim_len) for i in range(dims)] if complex: - return np.random.randn( - *shape).astype(dtype) + 1.j * np.random.randn(*shape).astype(dtype) + return np.random.randn(*shape).astype(dtype) + 1.0j * np.random.randn( + *shape + ).astype(dtype) else: return np.random.randn(*shape).astype(dtype) def place(devices, key='place'): - def decorate(cls): module = sys.modules[cls.__module__].__dict__ raw_classes = { - k: v - for k, v in module.items() if k.startswith(cls.__name__) + k: v for k, v in module.items() if k.startswith(cls.__name__) } for raw_name, raw_cls in raw_classes.items(): @@ -62,7 +59,7 @@ def decorate(cls): test_cls = dict(raw_cls.__dict__) test_cls.update({key: d}) new_name = raw_name + '.' + d.__class__.__name__ - module[new_name] = type(new_name, (raw_cls, ), test_cls) + module[new_name] = type(new_name, (raw_cls,), test_cls) del module[raw_name] return cls @@ -82,7 +79,7 @@ def decorate(cls): name = cls.__name__ + str(k) name = name + '.' + v.get('suffix') if v.get('suffix') else name - test_cls_module[name] = type(name, (cls, ), test_cls) + test_cls_module[name] = type(name, (cls,), test_cls) for m in list(cls.__dict__): if m.startswith("test"): @@ -95,68 +92,107 @@ def decorate(cls): @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), - ('test_x_complex', rand_x(5, complex=True), None, -1, 'backward'), - ('test_n_grater_input_length', rand_x(5, - max_dim_len=5), 11, -1, 'backward'), - ('test_n_smaller_than_input_length', rand_x( - 5, min_dim_len=5, complex=True), 3, -1, 'backward'), - ('test_axis_not_last', rand_x(5), None, 3, 'backward'), - ('test_norm_forward', rand_x(5), None, 3, 'forward'), - ('test_norm_ortho', rand_x(5), None, 3, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), + ('test_x_complex', rand_x(5, complex=True), None, -1, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + 11, + -1, + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5, complex=True), + 3, + -1, + 'backward', + ), + ('test_axis_not_last', rand_x(5), None, 3, 'backward'), + ('test_norm_forward', rand_x(5), None, 3, 'forward'), + ('test_norm_ortho', rand_x(5), None, 3, 'ortho'), + ], +) class TestFft(unittest.TestCase): - def test_fft(self): - """Test fft with norm condition - """ + """Test fft with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.fft(self.x, self.n, self.axis, - self.norm), - paddle.fft.fft(paddle.to_tensor(self.x), - self.n, self.axis, - self.norm), - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + np.testing.assert_allclose( + scipy.fft.fft(self.x, self.n, self.axis, self.norm), + paddle.fft.fft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), - ('test_x_complex', rand_x(5, complex=True), None, -1, 'backward'), - ('test_n_grater_input_length', rand_x(5, - max_dim_len=5), 11, -1, 'backward'), - ('test_n_smaller_than_input_length', rand_x( - 5, min_dim_len=5, complex=True), 3, -1, 'backward'), - ('test_axis_not_last', rand_x(5), None, 3, 'backward'), - ('test_norm_forward', rand_x(5), None, 3, 'forward'), - ('test_norm_ortho', rand_x(5), None, 3, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), + ('test_x_complex', rand_x(5, complex=True), None, -1, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + 11, + -1, + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5, complex=True), + 3, + -1, + 'backward', + ), + ('test_axis_not_last', rand_x(5), None, 3, 'backward'), + ('test_norm_forward', rand_x(5), None, 3, 'forward'), + ('test_norm_ortho', rand_x(5), None, 3, 'ortho'), + ], +) class TestIfft(unittest.TestCase): - def test_fft(self): - """Test ifft with norm condition - """ + """Test ifft with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.ifft(self.x, self.n, self.axis, - self.norm), - paddle.fft.ifft(paddle.to_tensor(self.x), - self.n, self.axis, - self.norm), - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + np.testing.assert_allclose( + scipy.fft.ifft(self.x, self.n, self.axis, self.norm), + paddle.fft.ifft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), - ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', rand_x(2), None, -1, 'random', ValueError)] + [ + ('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), + ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), + ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + rand_x(2), + None, + -1, + 'random', + ValueError, + ), + ], ) class TestFftException(unittest.TestCase): - def test_fft(self): """Test fft with buoudary condition Test case include: @@ -166,55 +202,108 @@ def test_fft(self): - norm out of range """ with self.assertRaises(self.expect_exception): - paddle.fft.fft(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.fft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), - ('test_x_complex128', rand_x(5, complex=True), None, (0, 1), 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (0, 1), 'backward'), - ('test_n_smaller_than_input_length', rand_x(5, min_dim_len=5, complex=True), - (4, 4), (0, 1), 'backward'), - ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), - ('test_axis_none', rand_x(5), None, None, 'backward'), - ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), - ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), + ( + 'test_x_complex128', + rand_x(5, complex=True), + None, + (0, 1), + 'backward', + ), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5, complex=True), + (4, 4), + (0, 1), + 'backward', + ), + ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), + ('test_axis_none', rand_x(5), None, None, 'backward'), + ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), + ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), + ], +) class TestFft2(unittest.TestCase): - def test_fft2(self): - """Test fft2 with norm condition - """ + """Test fft2 with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.fft2(self.x, self.n, self.axis, - self.norm), - paddle.fft.fft2(paddle.to_tensor(self.x), - self.n, self.axis, - self.norm), - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + np.testing.assert_allclose( + scipy.fft.fft2(self.x, self.n, self.axis, self.norm), + paddle.fft.fft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_x_complex_input', rand_x(2, complex=True), None, - (0, 1), None, ValueError), - ('test_x_1dim_tensor', rand_x(1), None, (0, 1), None, ValueError), - ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), - ('test_n_len_not_equal_axis', rand_x(5, max_dim_len=5), 11, - (0, 1), 'backward', ValueError), - ('test_n_zero', rand_x(2), (0, 0), (0, 1), 'backward', ValueError), - ('test_axis_out_of_range', rand_x(2), None, - (0, 1, 2), 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_axis_not_sequence', rand_x(5), None, -10, 'backward', ValueError), - ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError)]) + [ + ( + 'test_x_complex_input', + rand_x(2, complex=True), + None, + (0, 1), + None, + ValueError, + ), + ('test_x_1dim_tensor', rand_x(1), None, (0, 1), None, ValueError), + ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), + ( + 'test_n_len_not_equal_axis', + rand_x(5, max_dim_len=5), + 11, + (0, 1), + 'backward', + ValueError, + ), + ('test_n_zero', rand_x(2), (0, 0), (0, 1), 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(2), + None, + (0, 1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_axis_not_sequence', + rand_x(5), + None, + -10, + 'backward', + ValueError, + ), + ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestFft2Exception(unittest.TestCase): - def test_fft2(self): """Test fft2 with buoudary condition Test case include: @@ -227,267 +316,525 @@ def test_fft2(self): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.fft2(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.fft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), - ('test_x_complex128', rand_x(5, complex=True), None, None, 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (1, 2), 'backward'), - ('test_n_smaller_input_length', rand_x(5, min_dim_len=5, complex=True), - (3, 3), (1, 2), 'backward'), - ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), - ('test_norm_forward', rand_x(5), None, None, 'forward'), - ('test_norm_ortho', rand_x(5), None, None, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), + ('test_x_complex128', rand_x(5, complex=True), None, None, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (1, 2), + 'backward', + ), + ( + 'test_n_smaller_input_length', + rand_x(5, min_dim_len=5, complex=True), + (3, 3), + (1, 2), + 'backward', + ), + ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), + ('test_norm_forward', rand_x(5), None, None, 'forward'), + ('test_norm_ortho', rand_x(5), None, None, 'ortho'), + ], +) class TestFftn(unittest.TestCase): - def test_fftn(self): - """Test fftn with norm condition - """ + """Test fftn with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.fftn(self.x, self.n, self.axis, - self.norm), - paddle.fft.fftn(paddle.to_tensor(self.x), - self.n, self.axis, - self.norm), - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + np.testing.assert_allclose( + scipy.fft.fftn(self.x, self.n, self.axis, self.norm), + paddle.fft.fftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), - ('test_x_complex128', rand_x(5, complex=True), None, None, 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (1, 2), 'backward'), - ('test_n_smaller_input_length', rand_x(5, min_dim_len=5, complex=True), - (3, 3), (1, 2), 'backward'), - ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), - ('test_norm_forward', rand_x(5), None, None, 'forward'), - ('test_norm_ortho', rand_x(5), None, None, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), + ('test_x_complex128', rand_x(5, complex=True), None, None, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (1, 2), + 'backward', + ), + ( + 'test_n_smaller_input_length', + rand_x(5, min_dim_len=5, complex=True), + (3, 3), + (1, 2), + 'backward', + ), + ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), + ('test_norm_forward', rand_x(5), None, None, 'forward'), + ('test_norm_ortho', rand_x(5), None, None, 'ortho'), + ], +) class TestIFftn(unittest.TestCase): - def test_ifftn(self): - """Test ifftn with norm condition - """ + """Test ifftn with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.ifftn(self.x, self.n, self.axis, self.norm), - paddle.fft.ifftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.ifftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, -1, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 4, -1, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 2, -1, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, 1, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, 1, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "ortho"), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + -1, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 4, + -1, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 2, + -1, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + 1, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + 1, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "ortho", + ), + ], +) class TestHfft(unittest.TestCase): - def test_hfft(self): - """Test hfft with norm condition - """ + """Test hfft with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.hfft(self.x, self.n, self.axis, - self.norm), - paddle.fft.hfft(paddle.to_tensor(self.x), - self.n, self.axis, - self.norm), - rtol=1e-5, - atol=0) + np.testing.assert_allclose( + scipy.fft.hfft(self.x, self.n, self.axis, self.norm), + paddle.fft.hfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), + rtol=1e-5, + atol=0, + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, -1, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 4, -1, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 2, -1, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "ortho"), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + -1, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 4, + -1, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 2, + -1, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "ortho", + ), + ], +) class TestIrfft(unittest.TestCase): - def test_irfft(self): - """Test irfft with norm condition - """ + """Test irfft with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.irfft(self.x, self.n, self.axis, self.norm), - paddle.fft.irfft(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.irfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=1e-5, - atol=0) + atol=0, + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, None, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [4], None, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [2], None, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "ortho"), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + None, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [4], + None, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [2], + None, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "ortho", + ), + ], +) class TestIrfftn(unittest.TestCase): - def test_irfftn(self): - """Test irfftn with norm condition - """ + """Test irfftn with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.irfftn(self.x, self.n, self.axis, self.norm), - paddle.fft.irfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.irfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=1e-5, - atol=0) + atol=0, + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, None, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [4], None, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [2], None, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "ortho"), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + None, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [4], + None, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [2], + None, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "ortho", + ), + ], +) class TestHfftn(unittest.TestCase): - def test_hfftn(self): - """Test hfftn with norm condition - """ + """Test hfftn with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.hfftn(self.x, self.n, self.axis, self.norm), - paddle.fft.hfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.hfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=1e-5, - atol=0) + atol=0, + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, (-2, -1), "backward"), - ('test_with_s', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - [2, 2], (-2, -1), "backward", ValueError), - ('test_axis_not_last', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "backward"), - ('test_norm_forward', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "forward"), - ('test_norm_ortho', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "ortho"), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + (-2, -1), + "backward", + ), + ( + 'test_with_s', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [2, 2], + (-2, -1), + "backward", + ValueError, + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "ortho", + ), + ], +) class TestHfft2(unittest.TestCase): - def test_hfft2(self): - """Test hfft2 with norm condition - """ + """Test hfft2 with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.hfft2(self.x, self.s, self.axis, self.norm), - paddle.fft.hfft2(paddle.to_tensor(self.x), self.s, self.axis, - self.norm), + paddle.fft.hfft2( + paddle.to_tensor(self.x), self.s, self.axis, self.norm + ), rtol=1e-5, - atol=0) + atol=0, + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, (-2, -1), "backward"), - ('test_n_equal_input_length', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (4, 6), - (-2, -1), "backward"), - ('test_axis_not_last', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "backward"), - ('test_norm_forward', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "forward"), - ('test_norm_ortho', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "ortho"), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + (-2, -1), + "backward", + ), + ( + 'test_n_equal_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (4, 6), + (-2, -1), + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "ortho", + ), + ], +) class TestIrfft2(unittest.TestCase): - def test_irfft2(self): - """Test irfft2 with norm condition - """ + """Test irfft2 with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.irfft2(self.x, self.s, self.axis, self.norm), - paddle.fft.irfft2(paddle.to_tensor(self.x), self.s, self.axis, - self.norm), + paddle.fft.irfft2( + paddle.to_tensor(self.x), self.s, self.axis, self.norm + ), rtol=1e-5, - atol=0) + atol=0, + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, -1, 'backward', RuntimeError), - ('test_n_nagative', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), -1, -1, 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4) + 1j * np.random.randn(4, 4), 0, -1, - 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (1, 2, 3), -1, 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, 10, 'backward', ValueError), - ('test_axis_with_array', np.random.randn(4) + 1j * np.random.randn(4), None, - (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, -1, 'random', ValueError) -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + -1, + 'backward', + RuntimeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + -1, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + 0, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2, 3), + -1, + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 10, + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + -1, + 'random', + ValueError, + ), + ], +) class TestHfftException(unittest.TestCase): - def test_hfft(self): """Test hfft with buoudary condition Test case include: @@ -500,27 +847,66 @@ def test_hfft(self): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.hfft(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.hfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_n_nagative', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), -1, -1, 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4) + 1j * np.random.randn(4, 4), 0, -1, - 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (1, 2), -1, 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, 10, 'backward', ValueError), - ('test_axis_with_array', np.random.randn(4) + 1j * np.random.randn(4), None, - (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError) -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + -1, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + 0, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + -1, + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 10, + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestIrfftException(unittest.TestCase): - def test_irfft(self): """ Test irfft with buoudary condition @@ -533,34 +919,84 @@ def test_irfft(self): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.irfft(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.irfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, (-2, -1), 'backward', RuntimeError), - ('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (0, 0), (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - 3, None, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (1, 2), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, -1, - 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError)]) + [ + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + (-2, -1), + 'backward', + RuntimeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + None, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + -1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestHfft2Exception(unittest.TestCase): - def test_hfft2(self): """ Test hfft2 with buoudary condition @@ -574,34 +1010,82 @@ def test_hfft2(self): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.hfft2(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.hfft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_zero_point', - np.random.randn(4, 4, 1) + 1j * np.random.randn(4, 4, 1), None, - (-2, -1), "backward", ValueError), - ('test_n_zero', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (0, 0), (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - 3, -1, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-3, -2, -1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (1, 2), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, 1, - 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError)]) + [ + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_zero_point', + np.random.randn(4, 4, 1) + 1j * np.random.randn(4, 4, 1), + None, + (-2, -1), + "backward", + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-3, -2, -1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestIrfft2Exception(unittest.TestCase): - def test_irfft2(self): """ Test irfft2 with buoudary condition @@ -615,34 +1099,84 @@ def test_irfft2(self): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.irfft2(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.irfft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, (-2, -1), 'backward', RuntimeError), - ('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (0, 0), (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - 3, -1, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-3, -2, -1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (10, 20), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, 1, - 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError)]) + [ + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + (-2, -1), + 'backward', + RuntimeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-3, -2, -1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (10, 20), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestHfftnException(unittest.TestCase): - def test_hfftn(self): """Test hfftn with buoudary condition Test case include: @@ -655,31 +1189,74 @@ def test_hfftn(self): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.hfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.hfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (0, 0), (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - 3, -1, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-3, -2, -1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (10, 20), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, 1, - 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError)]) + [ + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-3, -2, -1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (10, 20), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestIrfftnException(unittest.TestCase): - def test_irfftn(self): """Test irfftn with buoudary condition Test case include: @@ -691,46 +1268,75 @@ def test_irfftn(self): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.irfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.irfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), - ('test_n_grater_than_input_length', rand_x( - 5, max_dim_len=5), 11, -1, 'backward'), - ('test_n_smaller_than_input_length', rand_x( - 5, min_dim_len=5), 3, -1, 'backward'), - ('test_axis_not_last', rand_x(5), None, 3, 'backward'), - ('test_norm_forward', rand_x(5), None, 3, 'forward'), - ('test_norm_ortho', rand_x(5), None, 3, 'ortho')]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), + ( + 'test_n_grater_than_input_length', + rand_x(5, max_dim_len=5), + 11, + -1, + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + 3, + -1, + 'backward', + ), + ('test_axis_not_last', rand_x(5), None, 3, 'backward'), + ('test_norm_forward', rand_x(5), None, 3, 'forward'), + ('test_norm_ortho', rand_x(5), None, 3, 'ortho'), + ], +) class TestRfft(unittest.TestCase): - def test_rfft(self): - """Test rfft with norm condition - """ + """Test rfft with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.rfft(self.x, self.n, self.axis, - self.norm), - paddle.fft.rfft(paddle.to_tensor(self.x), - self.n, self.axis, - self.norm), - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + np.testing.assert_allclose( + scipy.fft.rfft(self.x, self.n, self.axis, self.norm), + paddle.fft.rfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), - ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', rand_x(2), None, -1, 'random', ValueError)] + [ + ('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), + ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), + ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + rand_x(2), + None, + -1, + 'random', + ValueError, + ), + ], ) class TestRfftException(unittest.TestCase): - def test_rfft(self): """Test rfft with buoudary condition Test case include: @@ -741,51 +1347,93 @@ def test_rfft(self): - the dimensions of n and axis are different """ with self.assertRaises(self.expect_exception): - paddle.fft.rfft(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.rfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (0, 1), 'backward'), - ('test_n_smaller_than_input_length', rand_x(5, min_dim_len=5), (4, 4), - (0, 1), 'backward'), - ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), - ('test_axis_none', rand_x(5), None, None, 'backward'), - ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), - ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + (4, 4), + (0, 1), + 'backward', + ), + ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), + ('test_axis_none', rand_x(5), None, None, 'backward'), + ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), + ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), + ], +) class TestRfft2(unittest.TestCase): - def test_rfft2(self): - """Test rfft2 with norm condition - """ + """Test rfft2 with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.rfft2(self.x, self.n, self.axis, self.norm), - paddle.fft.rfft2(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.rfft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex_input', rand_x(2, complex=True), None, - (0, 1), 'backward', RuntimeError), - ('test_x_1dim_tensor', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), - ('test_n_zero', rand_x(2), 0, (0, 1), 'backward', ValueError), - ('test_axis_out_of_range', rand_x(2), None, - (0, 1, 2), 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_axis_not_sequence', rand_x(5), None, -10, 'backward', ValueError), - ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_x_complex_input', + rand_x(2, complex=True), + None, + (0, 1), + 'backward', + RuntimeError, + ), + ('test_x_1dim_tensor', rand_x(1), None, (0, 1), 'backward', ValueError), + ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), + ('test_n_zero', rand_x(2), 0, (0, 1), 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(2), + None, + (0, 1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_axis_not_sequence', + rand_x(5), + None, + -10, + 'backward', + ValueError, + ), + ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestRfft2Exception(unittest.TestCase): - def test_rfft2(self): """Test rfft2 with buoudary condition Test case include: @@ -798,47 +1446,83 @@ def test_rfft2(self): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.rfft2(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.rfft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (1, 2), 'backward'), - ('test_n_smaller_input_length', rand_x(5, min_dim_len=5), (3, 3), - (1, 2), 'backward'), - ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), - ('test_norm_forward', rand_x(5), None, None, 'forward'), - ('test_norm_ortho', rand_x(5), None, None, 'ortho'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (1, 2), + 'backward', + ), + ( + 'test_n_smaller_input_length', + rand_x(5, min_dim_len=5), + (3, 3), + (1, 2), + 'backward', + ), + ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), + ('test_norm_forward', rand_x(5), None, None, 'forward'), + ('test_norm_ortho', rand_x(5), None, None, 'ortho'), + ], +) class TestRfftn(unittest.TestCase): - def test_rfftn(self): - """Test rfftn with norm condition - """ + """Test rfftn with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.rfftn(self.x, self.n, self.axis, self.norm), - paddle.fft.rfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.rfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex', rand_x( - 4, complex=True), None, None, 'backward', RuntimeError), - ('test_n_nagative', rand_x(4), (-1, -1), (1, 2), 'backward', ValueError), - ('test_n_not_sequence', rand_x(4), -1, None, 'backward', ValueError), - ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, [0, 1], 'backward', ValueError), - ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError) -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_x_complex', + rand_x(4, complex=True), + None, + None, + 'backward', + RuntimeError, + ), + ( + 'test_n_nagative', + rand_x(4), + (-1, -1), + (1, 2), + 'backward', + ValueError, + ), + ('test_n_not_sequence', rand_x(4), -1, None, 'backward', ValueError), + ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(1), + None, + [0, 1], + 'backward', + ValueError, + ), + ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestRfftnException(unittest.TestCase): - def test_rfftn(self): """Test rfftn with buoudary condition Test case include: @@ -849,45 +1533,75 @@ def test_rfftn(self): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.rfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.rfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), - ('test_n_grater_than_input_length', rand_x( - 5, max_dim_len=5), 11, -1, 'backward'), - ('test_n_smaller_than_input_length', rand_x( - 5, min_dim_len=5), 3, -1, 'backward'), - ('test_axis_not_last', rand_x(5), None, 3, 'backward'), - ('test_norm_forward', rand_x(5), None, 3, 'forward'), - ('test_norm_ortho', rand_x(5), None, 3, 'ortho')]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), + ( + 'test_n_grater_than_input_length', + rand_x(5, max_dim_len=5), + 11, + -1, + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + 3, + -1, + 'backward', + ), + ('test_axis_not_last', rand_x(5), None, 3, 'backward'), + ('test_norm_forward', rand_x(5), None, 3, 'forward'), + ('test_norm_ortho', rand_x(5), None, 3, 'ortho'), + ], +) class TestIhfft(unittest.TestCase): - def test_ihfft(self): - """Test ihfft with norm condition - """ + """Test ihfft with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.ihfft(self.x, self.n, self.axis, self.norm), - paddle.fft.ihfft(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.ihfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), - ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', rand_x(2), None, -1, 'random', ValueError)] + [ + ('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), + ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), + ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + rand_x(2), + None, + -1, + 'random', + ValueError, + ), + ], ) class TestIhfftException(unittest.TestCase): - def test_ihfft(self): """Test ihfft with buoudary condition Test case include: @@ -897,53 +1611,101 @@ def test_ihfft(self): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.ihfft(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.ihfft( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (11, 11), - (0, 1), 'backward'), - ('test_n_smaller_than_input_length', rand_x(5, min_dim_len=5), (1, 1), - (0, 1), 'backward'), - ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), - ('test_axis_none', rand_x(5), None, None, 'backward'), - ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), - ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (11, 11), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + (1, 1), + (0, 1), + 'backward', + ), + ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), + ('test_axis_none', rand_x(5), None, None, 'backward'), + ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), + ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), + ], +) class TestIhfft2(unittest.TestCase): - def test_ihfft2(self): - """Test ihfft2 with norm condition - """ + """Test ihfft2 with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.ihfft2(self.x, self.n, self.axis, self.norm), - paddle.fft.ihfft2(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.ihfft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_x_complex_input', rand_x(2, complex=True), None, - (0, 1), None, ValueError), - ('test_x_1dim_tensor', rand_x(1), None, (0, 1), None, ValueError), - ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), - ('test_n_len_not_equal_axis', rand_x(5, max_dim_len=5), 11, - (0, 1), 'backward', ValueError), - ('test_n_zero', rand_x(2), (0, 0), (0, 1), 'backward', ValueError), - ('test_axis_out_of_range', rand_x(2), None, - (0, 1, 2), 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_axis_not_sequence', rand_x(5), None, -10, 'backward', ValueError), - ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError)]) + [ + ( + 'test_x_complex_input', + rand_x(2, complex=True), + None, + (0, 1), + None, + ValueError, + ), + ('test_x_1dim_tensor', rand_x(1), None, (0, 1), None, ValueError), + ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), + ( + 'test_n_len_not_equal_axis', + rand_x(5, max_dim_len=5), + 11, + (0, 1), + 'backward', + ValueError, + ), + ('test_n_zero', rand_x(2), (0, 0), (0, 1), 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(2), + None, + (0, 1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_axis_not_sequence', + rand_x(5), + None, + -10, + 'backward', + ValueError, + ), + ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestIhfft2Exception(unittest.TestCase): - def test_ihfft2(self): """Test ihfft2 with buoudary condition Test case include: @@ -956,46 +1718,75 @@ def test_ihfft2(self): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.ihfft2(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.ihfft2( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (11, 11), - (0, 1), 'backward'), - ('test_n_smaller_input_length', rand_x(5, min_dim_len=5), (1, 1), - (0, 1), 'backward'), - ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), - ('test_norm_forward', rand_x(5), None, None, 'forward'), - ('test_norm_ortho', rand_x(5), None, None, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (11, 11), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_input_length', + rand_x(5, min_dim_len=5), + (1, 1), + (0, 1), + 'backward', + ), + ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), + ('test_norm_forward', rand_x(5), None, None, 'forward'), + ('test_norm_ortho', rand_x(5), None, None, 'ortho'), + ], +) class TestIhfftn(unittest.TestCase): - def test_ihfftn(self): - """Test ihfftn with norm condition - """ + """Test ihfftn with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.ihfftn(self.x, self.n, self.axis, self.norm), - paddle.fft.ihfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm), + paddle.fft.ihfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ), rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex', rand_x( - 4, complex=True), None, None, 'backward', RuntimeError), - ('test_n_nagative', rand_x(4), -1, None, 'backward', ValueError), - ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, [0, 1], 'backward', ValueError), - ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError) -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_x_complex', + rand_x(4, complex=True), + None, + None, + 'backward', + RuntimeError, + ), + ('test_n_nagative', rand_x(4), -1, None, 'backward', ValueError), + ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(1), + None, + [0, 1], + 'backward', + ValueError, + ), + ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestIhfftnException(unittest.TestCase): - def test_ihfftn(self): """Test ihfftn with buoudary condition Test case include: @@ -1006,88 +1797,107 @@ def test_ihfftn(self): """ with paddle.fluid.dygraph.guard(self.place): with self.assertRaises(self.expect_exception): - paddle.fft.ihfftn(paddle.to_tensor(self.x), self.n, self.axis, - self.norm) + paddle.fft.ihfftn( + paddle.to_tensor(self.x), self.n, self.axis, self.norm + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'n', 'd', 'dtype'), [ - ('test_without_d', 20, 1, 'float32'), - ('test_with_d', 20, 0.5, 'float32'), -]) +@parameterize( + (TEST_CASE_NAME, 'n', 'd', 'dtype'), + [ + ('test_without_d', 20, 1, 'float32'), + ('test_with_d', 20, 0.5, 'float32'), + ], +) class TestFftFreq(unittest.TestCase): - def test_fftfreq(self): - """Test fftfreq with norm condition - """ + """Test fftfreq with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.fftfreq(self.n, self.d).astype(self.dtype), paddle.fft.fftfreq(self.n, self.d, self.dtype).numpy(), rtol=RTOL.get(str(self.dtype)), - atol=ATOL.get(str(self.dtype))) + atol=ATOL.get(str(self.dtype)), + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'n', 'd', 'dtype'), [ - ('test_without_d', 20, 1, 'float32'), - ('test_with_d', 20, 0.5, 'float32'), -]) +@parameterize( + (TEST_CASE_NAME, 'n', 'd', 'dtype'), + [ + ('test_without_d', 20, 1, 'float32'), + ('test_with_d', 20, 0.5, 'float32'), + ], +) class TestRfftFreq(unittest.TestCase): - def test_rfftfreq(self): - """Test rfftfreq with norm condition - """ + """Test rfftfreq with norm condition""" with paddle.fluid.dygraph.guard(self.place): np.testing.assert_allclose( scipy.fft.rfftfreq(self.n, self.d).astype(self.dtype), paddle.fft.rfftfreq(self.n, self.d, self.dtype).numpy(), rtol=RTOL.get(str(self.dtype)), - atol=ATOL.get(str(self.dtype))) + atol=ATOL.get(str(self.dtype)), + ) @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'axes', 'dtype'), [ - ('test_1d', np.random.randn(10), (0, ), 'float64'), - ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), - ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), - ('test_2d_odd_with_all_axes', - np.random.randn(5, 5) + 1j * np.random.randn(5, 5), None, 'complex128'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'axes', 'dtype'), + [ + ('test_1d', np.random.randn(10), (0,), 'float64'), + ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), + ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), + ( + 'test_2d_odd_with_all_axes', + np.random.randn(5, 5) + 1j * np.random.randn(5, 5), + None, + 'complex128', + ), + ], +) class TestFftShift(unittest.TestCase): - def test_fftshift(self): - """Test fftshift with norm condition - """ + """Test fftshift with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.fftshift(self.x, self.axes), - paddle.fft.fftshift( - paddle.to_tensor(self.x), - self.axes).numpy(), - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + np.testing.assert_allclose( + scipy.fft.fftshift(self.x, self.axes), + paddle.fft.fftshift( + paddle.to_tensor(self.x), self.axes + ).numpy(), + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'axes'), - [('test_1d', np.random.randn(10), (0, ), 'float64'), - ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), - ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), - ('test_2d_odd_with_all_axes', - np.random.randn(5, 5) + 1j * np.random.randn(5, 5), None, 'complex128')]) + [ + ('test_1d', np.random.randn(10), (0,), 'float64'), + ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), + ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), + ( + 'test_2d_odd_with_all_axes', + np.random.randn(5, 5) + 1j * np.random.randn(5, 5), + None, + 'complex128', + ), + ], +) class TestIfftShift(unittest.TestCase): - def test_ifftshift(self): - """Test ifftshift with norm condition - """ + """Test ifftshift with norm condition""" with paddle.fluid.dygraph.guard(self.place): - np.testing.assert_allclose(scipy.fft.ifftshift(self.x, self.axes), - paddle.fft.ifftshift( - paddle.to_tensor(self.x), - self.axes).numpy(), - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + np.testing.assert_allclose( + scipy.fft.ifftshift(self.x, self.axes), + paddle.fft.ifftshift( + paddle.to_tensor(self.x), self.axes + ).numpy(), + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/fft/test_fft_with_static_graph.py b/python/paddle/fluid/tests/unittests/fft/test_fft_with_static_graph.py index 7b286c5086c73..35828ed160664 100644 --- a/python/paddle/fluid/tests/unittests/fft/test_fft_with_static_graph.py +++ b/python/paddle/fluid/tests/unittests/fft/test_fft_with_static_graph.py @@ -19,8 +19,15 @@ import paddle import scipy.fft -from test_fft import (ATOL, DEVICES, RTOL, TEST_CASE_NAME, parameterize, place, - rand_x) +from test_fft import ( + ATOL, + DEVICES, + RTOL, + TEST_CASE_NAME, + parameterize, + place, + rand_x, +) @contextlib.contextmanager @@ -40,70 +47,126 @@ def stgraph(func, place, x, n, axes, norm): @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), - ('test_x_complex64', rand_x(5, np.float64, - complex=True), None, -1, 'backward'), - ('test_n_grater_than_input_length', rand_x( - 5, max_dim_len=5), 11, -1, 'backward'), - ('test_n_smaller_than_input_length', rand_x( - 5, min_dim_len=5), 3, -1, 'backward'), - ('test_axis_not_last', rand_x(5), None, 3, 'backward'), - ('test_norm_forward', rand_x(5), None, 3, 'forward'), - ('test_norm_ortho', rand_x(5), None, 3, 'ortho')]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), + ( + 'test_x_complex64', + rand_x(5, np.float64, complex=True), + None, + -1, + 'backward', + ), + ( + 'test_n_grater_than_input_length', + rand_x(5, max_dim_len=5), + 11, + -1, + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + 3, + -1, + 'backward', + ), + ('test_axis_not_last', rand_x(5), None, 3, 'backward'), + ('test_norm_forward', rand_x(5), None, 3, 'forward'), + ('test_norm_ortho', rand_x(5), None, 3, 'ortho'), + ], +) class TestFft(unittest.TestCase): - def test_static_rfft(self): - with stgraph(paddle.fft.fft, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.fft(self.x, self.n, self.axis, - self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + with stgraph( + paddle.fft.fft, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.fft(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), - ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', rand_x(2), None, -1, 'random', ValueError)] + [ + ('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), + ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), + ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + rand_x(2), + None, + -1, + 'random', + ValueError, + ), + ], ) class TestFftException(unittest.TestCase): - def test_fft(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.fft, self.place, self.x, self.n, self.axis, - self.norm) as y: + with stgraph( + paddle.fft.fft, self.place, self.x, self.n, self.axis, self.norm + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), - ('test_x_complex128', rand_x(5, complex=True), None, (0, 1), 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (0, 1), 'backward'), - ('test_n_smaller_than_input_length', rand_x(5, min_dim_len=5), (4, 4), - (0, 1), 'backward'), - ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), - ('test_axis_none', rand_x(5), None, None, 'backward'), - ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), - ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), + ( + 'test_x_complex128', + rand_x(5, complex=True), + None, + (0, 1), + 'backward', + ), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + (4, 4), + (0, 1), + 'backward', + ), + ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), + ('test_axis_none', rand_x(5), None, None, 'backward'), + ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), + ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), + ], +) class TestFft2(unittest.TestCase): - def test_static_fft2(self): - with stgraph(paddle.fft.fft2, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.fft2(self.x, self.n, self.axis, - self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + with stgraph( + paddle.fft.fft2, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.fft2(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @@ -114,274 +177,593 @@ def test_static_fft2(self): ('test_x_1dim_tensor', rand_x(1), None, (0, 1), 'backward', ValueError), ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), ('test_n_zero', rand_x(2), 0, (0, 1), 'backward', ValueError), - ('test_axis_out_of_range', rand_x(2), None, - (0, 1, 2), 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, - (0, 1), 'backward', ValueError), - ('test_axis_not_sequence', rand_x(5), None, -10, 'backward', - ValueError), - ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError) - ]) + ( + 'test_axis_out_of_range', + rand_x(2), + None, + (0, 1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_axis_not_sequence', + rand_x(5), + None, + -10, + 'backward', + ValueError, + ), + ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestFft2Exception(unittest.TestCase): - def test_static_fft2(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.fft2, self.place, self.x, self.n, self.axis, - self.norm) as y: + with stgraph( + paddle.fft.fft2, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), - ('test_x_complex128', rand_x(5, np.float64, - complex=True), None, None, 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (1, 2), 'backward'), - ('test_n_smaller_input_length', rand_x(5, min_dim_len=5), (3, 3), - (1, 2), 'backward'), - ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), - ('test_norm_forward', rand_x(5), None, None, 'forward'), - ('test_norm_ortho', rand_x(5), None, None, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), + ( + 'test_x_complex128', + rand_x(5, np.float64, complex=True), + None, + None, + 'backward', + ), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (1, 2), + 'backward', + ), + ( + 'test_n_smaller_input_length', + rand_x(5, min_dim_len=5), + (3, 3), + (1, 2), + 'backward', + ), + ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), + ('test_norm_forward', rand_x(5), None, None, 'forward'), + ('test_norm_ortho', rand_x(5), None, None, 'ortho'), + ], +) class TestFftn(unittest.TestCase): - def test_static_fftn(self): - with stgraph(paddle.fft.fftn, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.fftn(self.x, self.n, self.axis, - self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex', rand_x(4, - complex=True), None, None, 'backward', TypeError), - ('test_n_nagative', rand_x(4), (-1, -1), (1, 2), 'backward', ValueError), - ('test_n_not_sequence', rand_x(4), -1, None, 'backward', ValueError), - ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, [0, 1], 'backward', ValueError), - ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError) -]) -class TestRfftnException(unittest.TestCase): + with stgraph( + paddle.fft.fftn, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.fftn(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_x_complex', + rand_x(4, complex=True), + None, + None, + 'backward', + TypeError, + ), + ( + 'test_n_nagative', + rand_x(4), + (-1, -1), + (1, 2), + 'backward', + ValueError, + ), + ('test_n_not_sequence', rand_x(4), -1, None, 'backward', ValueError), + ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(1), + None, + [0, 1], + 'backward', + ValueError, + ), + ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) +class TestRfftnException(unittest.TestCase): def test_static_rfftn(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.rfftn, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.rfftn, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, -1, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 4, -1, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 2, -1, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, 1, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, 1, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "ortho"), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + -1, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 4, + -1, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 2, + -1, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + 1, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + 1, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "ortho", + ), + ], +) class TestHfft(unittest.TestCase): - """Test hfft with norm condition - """ + """Test hfft with norm condition""" def test_hfft(self): - with stgraph(paddle.fft.hfft, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.hfft(self.x, self.n, self.axis, - self.norm), - y, - rtol=1e-5, - atol=0) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, -1, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 4, -1, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 2, -1, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, -1, "ortho"), -]) + with stgraph( + paddle.fft.hfft, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.hfft(self.x, self.n, self.axis, self.norm), + y, + rtol=1e-5, + atol=0, + ) + + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + -1, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 4, + -1, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 2, + -1, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + -1, + "ortho", + ), + ], +) class TestIrfft(unittest.TestCase): - """Test irfft with norm condition - """ + """Test irfft with norm condition""" def test_irfft(self): - with stgraph(paddle.fft.irfft, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.irfft(self.x, self.n, - self.axis, self.norm), - y, - rtol=1e-5, - atol=0) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, None, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [4], None, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [2], None, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "ortho"), -]) + with stgraph( + paddle.fft.irfft, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.irfft(self.x, self.n, self.axis, self.norm), + y, + rtol=1e-5, + atol=0, + ) + + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + None, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [4], + None, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [2], + None, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "ortho", + ), + ], +) class Testirfftn(unittest.TestCase): - """Test irfftn with norm condition - """ + """Test irfftn with norm condition""" def test_static_irfftn(self): - with stgraph(paddle.fft.irfftn, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.irfftn(self.x, self.n, - self.axis, self.norm), - y, - rtol=1e-5, - atol=0) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, None, "backward"), - ('test_n_grater_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [4], None, "backward"), - ('test_n_smaller_than_input_length', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), [2], None, "backward"), - ('test_axis_not_last', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "backward"), - ('test_norm_forward', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "forward"), - ('test_norm_ortho', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), None, None, "ortho"), -]) + with stgraph( + paddle.fft.irfftn, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.irfftn(self.x, self.n, self.axis, self.norm), + y, + rtol=1e-5, + atol=0, + ) + + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + None, + "backward", + ), + ( + 'test_n_grater_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [4], + None, + "backward", + ), + ( + 'test_n_smaller_than_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [2], + None, + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + None, + "ortho", + ), + ], +) class Testhfftn(unittest.TestCase): - """Test hfftn with norm condition - """ + """Test hfftn with norm condition""" def test_static_hfftn(self): - with stgraph(paddle.fft.hfftn, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.hfftn(self.x, self.n, - self.axis, self.norm), - y, - rtol=1e-5, - atol=0) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, (-2, -1), "backward"), - ('test_n_grater_input_length', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), [4, 8], - (-2, -1), "backward"), - ('test_n_smaller_input_length', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), [2, 4], - (-2, -1), "backward"), - ('test_axis_not_last', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "backward"), - ('test_norm_forward', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "forward"), - ('test_norm_ortho', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "ortho"), -]) + with stgraph( + paddle.fft.hfftn, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.hfftn(self.x, self.n, self.axis, self.norm), + y, + rtol=1e-5, + atol=0, + ) + + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + (-2, -1), + "backward", + ), + ( + 'test_n_grater_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [4, 8], + (-2, -1), + "backward", + ), + ( + 'test_n_smaller_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + [2, 4], + (-2, -1), + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "ortho", + ), + ], +) class Testhfft2(unittest.TestCase): - """Test hfft2 with norm condition - """ + """Test hfft2 with norm condition""" def test_static_hfft2(self): - with stgraph(paddle.fft.hfft2, self.place, self.x, self.s, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.hfft2(self.x, self.s, - self.axis, self.norm), - y, - rtol=1e-5, - atol=0) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), [ - ('test_x_complex128', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.complex128), None, (-2, -1), "backward"), - ('test_n_equal_input_length', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (2, 4), - (-2, -1), "backward"), - ('test_axis_not_last', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "backward"), - ('test_norm_forward', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "forward"), - ('test_norm_ortho', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), None, - (-2, -1), "ortho"), -]) + with stgraph( + paddle.fft.hfft2, self.place, self.x, self.s, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.hfft2(self.x, self.s, self.axis, self.norm), + y, + rtol=1e-5, + atol=0, + ) + + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 's', 'axis', 'norm'), + [ + ( + 'test_x_complex128', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.complex128 + ), + None, + (-2, -1), + "backward", + ), + ( + 'test_n_equal_input_length', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (2, 4), + (-2, -1), + "backward", + ), + ( + 'test_axis_not_last', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "backward", + ), + ( + 'test_norm_forward', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "forward", + ), + ( + 'test_norm_ortho', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + None, + (-2, -1), + "ortho", + ), + ], +) class TestIrfft2(unittest.TestCase): - """Test irfft2 with norm condition - """ + """Test irfft2 with norm condition""" def test_static_irfft2(self): - with stgraph(paddle.fft.irfft2, self.place, self.x, self.s, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.irfft2(self.x, self.s, - self.axis, self.norm), - y, - rtol=1e-5, - atol=0) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_input_dtype', np.random.randn(4, 4, - 4), None, -1, 'backward', TypeError), - ('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, -1, 'backward', TypeError), - ('test_n_nagative', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), -1, -1, 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4) + 1j * np.random.randn(4, 4), 0, -1, - 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (1, 2, 3), -1, 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, 10, 'backward', ValueError), - ('test_axis_with_array', np.random.randn(4) + 1j * np.random.randn(4), None, - (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, -1, 'random', ValueError) -]) + with stgraph( + paddle.fft.irfft2, self.place, self.x, self.s, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.irfft2(self.x, self.s, self.axis, self.norm), + y, + rtol=1e-5, + atol=0, + ) + + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_input_dtype', + np.random.randn(4, 4, 4), + None, + -1, + 'backward', + TypeError, + ), + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + -1, + 'backward', + TypeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + -1, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + 0, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2, 3), + -1, + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 10, + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + -1, + 'random', + ValueError, + ), + ], +) class TestHfftException(unittest.TestCase): '''Test hfft with buoudary condition Test case include: @@ -393,31 +775,89 @@ class TestHfftException(unittest.TestCase): def test_static_hfft(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.hfft, self.place, self.x, self.n, self.axis, - self.norm) as y: + with stgraph( + paddle.fft.hfft, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_input_dtype', np.random.randn(4, 4, - 4), None, -1, 'backward', TypeError), - ('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, -1, 'backward', TypeError), - ('test_n_nagative', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), -1, -1, 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4) + 1j * np.random.randn(4, 4), 0, -1, - 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (1, 2), -1, 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, 10, 'backward', ValueError), - ('test_axis_with_array', np.random.randn(4) + 1j * np.random.randn(4), None, - (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError) -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_input_dtype', + np.random.randn(4, 4, 4), + None, + -1, + 'backward', + TypeError, + ), + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + -1, + 'backward', + TypeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + -1, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + 0, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + -1, + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 10, + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestIrfftException(unittest.TestCase): '''Test Irfft with buoudary condition Test case include: @@ -430,35 +870,97 @@ class TestIrfftException(unittest.TestCase): def test_static_irfft(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.irfft, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.irfft, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_input_dtype', np.random.randn( - 4, 4, 4), None, None, 'backward', TypeError), - ('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, (-2, -1), 'backward', TypeError), - ('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (0, 0), (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - 3, None, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (1, 2), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, -1, - 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError)]) + [ + ( + 'test_input_dtype', + np.random.randn(4, 4, 4), + None, + None, + 'backward', + TypeError, + ), + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + (-2, -1), + 'backward', + TypeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + None, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + -1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestHfft2Exception(unittest.TestCase): '''Test hfft2 with buoudary condition Test case include: @@ -471,35 +973,97 @@ class TestHfft2Exception(unittest.TestCase): def test_static_hfft2(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.hfft2, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.hfft2, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_input_dtype', np.random.randn( - 4, 4, 4), None, None, 'backward', TypeError), - ('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, (-2, -1), 'backward', TypeError), - ('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (0, 0), (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - 3, -1, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-3, -2, -1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (1, 2), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, 1, - 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError)]) + [ + ( + 'test_input_dtype', + np.random.randn(4, 4, 4), + None, + None, + 'backward', + TypeError, + ), + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + (-2, -1), + 'backward', + TypeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-3, -2, -1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestIrfft2Exception(unittest.TestCase): '''Test irfft2 with buoudary condition Test case include: @@ -512,35 +1076,97 @@ class TestIrfft2Exception(unittest.TestCase): def test_static_irfft2(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.irfft2, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.irfft2, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_input_dtype', np.random.randn( - 4, 4, 4), None, None, 'backward', TypeError), - ('test_bool_input', - (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( - np.bool_), None, (-2, -1), 'backward', TypeError), - ('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_n_zero', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - (0, 0), (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), - 3, -1, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-3, -2, -1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (10, 20), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, 1, - 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError)]) + [ + ( + 'test_input_dtype', + np.random.randn(4, 4, 4), + None, + None, + 'backward', + TypeError, + ), + ( + 'test_bool_input', + (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4)).astype( + np.bool_ + ), + None, + (-2, -1), + 'backward', + TypeError, + ), + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-3, -2, -1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (10, 20), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestHfftnException(unittest.TestCase): '''Test hfftn with buoudary condition Test case include: @@ -553,8 +1179,14 @@ class TestHfftnException(unittest.TestCase): def test_static_hfftn(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.hfftn, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.hfftn, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @@ -562,29 +1194,75 @@ def test_static_hfftn(self): @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_input_dtype', np.random.randn( - 4, 4, 4), None, None, 'backward', TypeError), + ( + 'test_input_dtype', + np.random.randn(4, 4, 4), + None, + None, + 'backward', + TypeError, + ), # ('test_bool_input', # (np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4) # ).astype(np.bool_), None, (-2, -1), 'backward', ValueError), - ('test_n_nagative', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (-1, -2), - (-2, -1), 'backward', ValueError), - ('test_n_zero', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (0, 0), - (-2, -1), 'backward', ValueError), - ('test_n_type', np.random.randn(4, 4, 4) + - 1j * np.random.randn(4, 4, 4), 3, -1, 'backward', ValueError), - ('test_n_axis_dim', - np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), (1, 2), - (-3, -2, -1), 'backward', ValueError), - ('test_axis_out_of_range', np.random.randn(4) + 1j * np.random.randn(4), - None, (10, 20), 'backward', ValueError), - ('test_axis_type', np.random.randn(4) + 1j * np.random.randn(4), None, - 1, 'backward', ValueError), - ('test_norm_not_in_enum_value', np.random.randn(4, 4) + - 1j * np.random.randn(4, 4), None, None, 'random', ValueError) - ]) + ( + 'test_n_nagative', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (-1, -2), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_zero', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (0, 0), + (-2, -1), + 'backward', + ValueError, + ), + ( + 'test_n_type', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + 3, + -1, + 'backward', + ValueError, + ), + ( + 'test_n_axis_dim', + np.random.randn(4, 4, 4) + 1j * np.random.randn(4, 4, 4), + (1, 2), + (-3, -2, -1), + 'backward', + ValueError, + ), + ( + 'test_axis_out_of_range', + np.random.randn(4) + 1j * np.random.randn(4), + None, + (10, 20), + 'backward', + ValueError, + ), + ( + 'test_axis_type', + np.random.randn(4) + 1j * np.random.randn(4), + None, + 1, + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + np.random.randn(4, 4) + 1j * np.random.randn(4, 4), + None, + None, + 'random', + ValueError, + ), + ], +) class TestIrfftnException(unittest.TestCase): '''Test irfftn with buoudary condition Test case include: @@ -597,291 +1275,545 @@ class TestIrfftnException(unittest.TestCase): def test_static_irfftn(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.irfftn, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.irfftn, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), - ('test_n_grater_than_input_length', rand_x( - 5, max_dim_len=5), 11, -1, 'backward'), - ('test_n_smaller_than_input_length', rand_x( - 5, min_dim_len=5), 3, -1, 'backward'), - ('test_axis_not_last', rand_x(5), None, 3, 'backward'), - ('test_norm_forward', rand_x(5), None, 3, 'forward'), - ('test_norm_ortho', rand_x(5), None, 3, 'ortho')]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), + ( + 'test_n_grater_than_input_length', + rand_x(5, max_dim_len=5), + 11, + -1, + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + 3, + -1, + 'backward', + ), + ('test_axis_not_last', rand_x(5), None, 3, 'backward'), + ('test_norm_forward', rand_x(5), None, 3, 'forward'), + ('test_norm_ortho', rand_x(5), None, 3, 'ortho'), + ], +) class TestRfft(unittest.TestCase): - def test_static_rfft(self): - with stgraph(paddle.fft.rfft, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.rfft(self.x, self.n, self.axis, - self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + with stgraph( + paddle.fft.rfft, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.rfft(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), - ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', rand_x(2), None, -1, 'random', ValueError)] + [ + ('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), + ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), + ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + rand_x(2), + None, + -1, + 'random', + ValueError, + ), + ], ) class TestRfftException(unittest.TestCase): - def test_rfft(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.rfft, self.place, self.x, self.n, self.axis, - self.norm) as y: + with stgraph( + paddle.fft.rfft, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (0, 1), 'backward'), - ('test_n_smaller_than_input_length', rand_x(5, min_dim_len=5), (4, 4), - (0, 1), 'backward'), - ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), - ('test_axis_none', rand_x(5), None, None, 'backward'), - ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), - ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + (4, 4), + (0, 1), + 'backward', + ), + ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), + ('test_axis_none', rand_x(5), None, None, 'backward'), + ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), + ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), + ], +) class TestRfft2(unittest.TestCase): - def test_static_rfft2(self): - with stgraph(paddle.fft.rfft2, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.rfft2(self.x, self.n, - self.axis, self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + with stgraph( + paddle.fft.rfft2, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.rfft2(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex_input', rand_x(2, complex=True), None, - (0, 1), 'backward', TypeError), + ( + 'test_x_complex_input', + rand_x(2, complex=True), + None, + (0, 1), + 'backward', + TypeError, + ), # ('test_x_not_tensor', [0, 1], None, (0, 1), 'backward', ValueError), ('test_x_1dim_tensor', rand_x(1), None, (0, 1), 'backward', ValueError), ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), ('test_n_zero', rand_x(2), 0, (0, 1), 'backward', ValueError), - ('test_axis_out_of_range', rand_x(2), None, - (0, 1, 2), 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, - (0, 1), 'backward', ValueError), - ('test_axis_not_sequence', rand_x(5), None, -10, 'backward', - ValueError), - ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError) - ]) + ( + 'test_axis_out_of_range', + rand_x(2), + None, + (0, 1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_axis_not_sequence', + rand_x(5), + None, + -10, + 'backward', + ValueError, + ), + ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestRfft2Exception(unittest.TestCase): - def test_static_rfft(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.rfft2, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.rfft2, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (6, 6), - (1, 2), 'backward'), - ('test_n_smaller_input_length', rand_x(5, min_dim_len=5), (3, 3), - (1, 2), 'backward'), - ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), - ('test_norm_forward', rand_x(5), None, None, 'forward'), - ('test_norm_ortho', rand_x(5), None, None, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (6, 6), + (1, 2), + 'backward', + ), + ( + 'test_n_smaller_input_length', + rand_x(5, min_dim_len=5), + (3, 3), + (1, 2), + 'backward', + ), + ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), + ('test_norm_forward', rand_x(5), None, None, 'forward'), + ('test_norm_ortho', rand_x(5), None, None, 'ortho'), + ], +) class TestRfftn(unittest.TestCase): - def test_static_rfft(self): - with stgraph(paddle.fft.rfftn, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.rfftn(self.x, self.n, - self.axis, self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex', rand_x(4, - complex=True), None, None, 'backward', TypeError), - ('test_n_nagative', rand_x(4), (-1, -1), (1, 2), 'backward', ValueError), - ('test_n_not_sequence', rand_x(4), -1, None, 'backward', ValueError), - ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, [0, 1], 'backward', ValueError), - ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError) -]) -class TestRfftnException(unittest.TestCase): + with stgraph( + paddle.fft.rfftn, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.rfftn(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_x_complex', + rand_x(4, complex=True), + None, + None, + 'backward', + TypeError, + ), + ( + 'test_n_nagative', + rand_x(4), + (-1, -1), + (1, 2), + 'backward', + ValueError, + ), + ('test_n_not_sequence', rand_x(4), -1, None, 'backward', ValueError), + ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(1), + None, + [0, 1], + 'backward', + ValueError, + ), + ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) +class TestRfftnException(unittest.TestCase): def test_static_rfftn(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.rfftn, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.rfftn, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), - ('test_n_grater_than_input_length', rand_x( - 5, max_dim_len=5), 11, -1, 'backward'), - ('test_n_smaller_than_input_length', rand_x( - 5, min_dim_len=5), 3, -1, 'backward'), - ('test_axis_not_last', rand_x(5), None, 3, 'backward'), - ('test_norm_forward', rand_x(5), None, 3, 'forward'), - ('test_norm_ortho', rand_x(5), None, 3, 'ortho')]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5, np.float64), None, -1, 'backward'), + ( + 'test_n_grater_than_input_length', + rand_x(5, max_dim_len=5), + 11, + -1, + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + 3, + -1, + 'backward', + ), + ('test_axis_not_last', rand_x(5), None, 3, 'backward'), + ('test_norm_forward', rand_x(5), None, 3, 'forward'), + ('test_norm_ortho', rand_x(5), None, 3, 'ortho'), + ], +) class TestIhfft(unittest.TestCase): - def test_static_ihfft(self): - with stgraph(paddle.fft.ihfft, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.ihfft(self.x, self.n, - self.axis, self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + with stgraph( + paddle.fft.ihfft, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.ihfft(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), - [('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), - ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, (0, 1), 'backward', ValueError), - ('test_norm_not_in_enum_value', rand_x(2), None, -1, 'random', ValueError)] + [ + ('test_n_nagative', rand_x(2), -1, -1, 'backward', ValueError), + ('test_n_zero', rand_x(2), 0, -1, 'backward', ValueError), + ('test_axis_out_of_range', rand_x(1), None, 10, 'backward', ValueError), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_norm_not_in_enum_value', + rand_x(2), + None, + -1, + 'random', + ValueError, + ), + ], ) class TestIhfftException(unittest.TestCase): - def test_static_ihfft(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.ihfft, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.ihfft, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), [ - ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (11, 11), - (0, 1), 'backward'), - ('test_n_smaller_than_input_length', rand_x(5, min_dim_len=5), (1, 1), - (0, 1), 'backward'), - ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), - ('test_axis_none', rand_x(5), None, None, 'backward'), - ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), - ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), + [ + ('test_x_float64', rand_x(5), None, (0, 1), 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (11, 11), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_than_input_length', + rand_x(5, min_dim_len=5), + (1, 1), + (0, 1), + 'backward', + ), + ('test_axis_random', rand_x(5), None, (1, 2), 'backward'), + ('test_axis_none', rand_x(5), None, None, 'backward'), + ('test_norm_forward', rand_x(5), None, (0, 1), 'forward'), + ('test_norm_ortho', rand_x(5), None, (0, 1), 'ortho'), + ], +) class TestIhfft2(unittest.TestCase): - def test_static_ihfft2(self): - with stgraph(paddle.fft.ihfft2, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.ihfft2(self.x, self.n, - self.axis, self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) + with stgraph( + paddle.fft.ihfft2, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.ihfft2(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex_input', rand_x(2, complex=True), None, - (0, 1), None, ValueError), + ( + 'test_x_complex_input', + rand_x(2, complex=True), + None, + (0, 1), + None, + ValueError, + ), # ('test_x_not_tensor', [0, 1], None, (0, 1), None, ValueError), ('test_x_1dim_tensor', rand_x(1), None, (0, 1), None, ValueError), ('test_n_nagative', rand_x(2), -1, (0, 1), 'backward', ValueError), - ('test_n_len_not_equal_axis', rand_x(5, max_dim_len=5), 11, - (0, 1), 'backward', ValueError), + ( + 'test_n_len_not_equal_axis', + rand_x(5, max_dim_len=5), + 11, + (0, 1), + 'backward', + ValueError, + ), ('test_n_zero', rand_x(2), (0, 0), (0, 1), 'backward', ValueError), - ('test_axis_out_of_range', rand_x(2), None, - (0, 1, 2), 'backward', ValueError), - ('test_axis_with_array', rand_x(1), None, - (0, 1), 'backward', ValueError), - ('test_axis_not_sequence', rand_x(5), None, -10, 'backward', - ValueError), - ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError) - ]) + ( + 'test_axis_out_of_range', + rand_x(2), + None, + (0, 1, 2), + 'backward', + ValueError, + ), + ( + 'test_axis_with_array', + rand_x(1), + None, + (0, 1), + 'backward', + ValueError, + ), + ( + 'test_axis_not_sequence', + rand_x(5), + None, + -10, + 'backward', + ValueError, + ), + ('test_norm_not_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) class TestIhfft2Exception(unittest.TestCase): - def test_static_ihfft2(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.ihfft2, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.ihfft2, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm'), - [('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), - ('test_n_grater_input_length', rand_x(5, max_dim_len=5), (11, 11), - (0, 1), 'backward'), - ('test_n_smaller_input_length', rand_x(5, min_dim_len=5), (1, 1), - (0, 1), 'backward'), - ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), - ('test_norm_forward', rand_x(5), None, None, 'forward'), - ('test_norm_ortho', rand_x(5), None, None, 'ortho')]) + [ + ('test_x_float64', rand_x(5, np.float64), None, None, 'backward'), + ( + 'test_n_grater_input_length', + rand_x(5, max_dim_len=5), + (11, 11), + (0, 1), + 'backward', + ), + ( + 'test_n_smaller_input_length', + rand_x(5, min_dim_len=5), + (1, 1), + (0, 1), + 'backward', + ), + ('test_axis_not_default', rand_x(5), None, (1, 2), 'backward'), + ('test_norm_forward', rand_x(5), None, None, 'forward'), + ('test_norm_ortho', rand_x(5), None, None, 'ortho'), + ], +) class TestIhfftn(unittest.TestCase): - def test_static_ihfftn(self): - with stgraph(paddle.fft.ihfftn, self.place, self.x, self.n, self.axis, - self.norm) as y: - np.testing.assert_allclose(scipy.fft.ihfftn(self.x, self.n, - self.axis, self.norm), - y, - rtol=RTOL.get(str(self.x.dtype)), - atol=ATOL.get(str(self.x.dtype))) - - -@place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), [ - ('test_x_complex', rand_x(4, - complex=True), None, None, 'backward', TypeError), - ('test_n_nagative', rand_x(4), -1, None, 'backward', ValueError), - ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), - ('test_axis_out_of_range', rand_x(1), None, [0, 1], 'backward', ValueError), - ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError) -]) -class TestIhfftnException(unittest.TestCase): + with stgraph( + paddle.fft.ihfftn, self.place, self.x, self.n, self.axis, self.norm + ) as y: + np.testing.assert_allclose( + scipy.fft.ihfftn(self.x, self.n, self.axis, self.norm), + y, + rtol=RTOL.get(str(self.x.dtype)), + atol=ATOL.get(str(self.x.dtype)), + ) + +@place(DEVICES) +@parameterize( + (TEST_CASE_NAME, 'x', 'n', 'axis', 'norm', 'expect_exception'), + [ + ( + 'test_x_complex', + rand_x(4, complex=True), + None, + None, + 'backward', + TypeError, + ), + ('test_n_nagative', rand_x(4), -1, None, 'backward', ValueError), + ('test_n_zero', rand_x(4), 0, None, 'backward', ValueError), + ( + 'test_axis_out_of_range', + rand_x(1), + None, + [0, 1], + 'backward', + ValueError, + ), + ('test_norm_not_in_enum', rand_x(2), None, -1, 'random', ValueError), + ], +) +class TestIhfftnException(unittest.TestCase): def test_static_ihfftn(self): with self.assertRaises(self.expect_exception): - with stgraph(paddle.fft.ihfftn, self.place, self.x, self.n, - self.axis, self.norm) as y: + with stgraph( + paddle.fft.ihfftn, + self.place, + self.x, + self.n, + self.axis, + self.norm, + ) as y: pass @place(DEVICES) -@parameterize((TEST_CASE_NAME, 'x', 'axes', 'dtype'), [ - ('test_1d', np.random.randn(10), (0, ), 'float64'), - ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), - ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), - ('test_2d_odd_with_all_axes', - np.random.randn(5, 5) + 1j * np.random.randn(5, 5), None, 'complex128'), -]) +@parameterize( + (TEST_CASE_NAME, 'x', 'axes', 'dtype'), + [ + ('test_1d', np.random.randn(10), (0,), 'float64'), + ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), + ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), + ( + 'test_2d_odd_with_all_axes', + np.random.randn(5, 5) + 1j * np.random.randn(5, 5), + None, + 'complex128', + ), + ], +) class TestFftShift(unittest.TestCase): - def test_fftshift(self): - """Test fftshift with norm condition - """ + """Test fftshift with norm condition""" paddle.enable_static() mp, sp = paddle.static.Program(), paddle.static.Program() with paddle.static.program_guard(mp, sp): @@ -898,16 +1830,21 @@ def test_fftshift(self): @place(DEVICES) @parameterize( (TEST_CASE_NAME, 'x', 'axes'), - [('test_1d', np.random.randn(10), (0, ), 'float64'), - ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), - ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), - ('test_2d_odd_with_all_axes', - np.random.randn(5, 5) + 1j * np.random.randn(5, 5), None, 'complex128')]) + [ + ('test_1d', np.random.randn(10), (0,), 'float64'), + ('test_2d', np.random.randn(10, 10), (0, 1), 'float64'), + ('test_2d_with_all_axes', np.random.randn(10, 10), None, 'float64'), + ( + 'test_2d_odd_with_all_axes', + np.random.randn(5, 5) + 1j * np.random.randn(5, 5), + None, + 'complex128', + ), + ], +) class TestIfftShift(unittest.TestCase): - def test_ifftshift(self): - """Test ifftshift with norm condition - """ + """Test ifftshift with norm condition""" paddle.enable_static() mp, sp = paddle.static.Program(), paddle.static.Program() with paddle.static.program_guard(mp, sp): diff --git a/python/paddle/fluid/tests/unittests/fft/test_spectral_op.py b/python/paddle/fluid/tests/unittests/fft/test_spectral_op.py index bc362d1b7a643..64b6cc111db25 100644 --- a/python/paddle/fluid/tests/unittests/fft/test_spectral_op.py +++ b/python/paddle/fluid/tests/unittests/fft/test_spectral_op.py @@ -17,7 +17,14 @@ import re import sys -from spectral_op_np import fft_c2c, fft_r2c, fft_c2r, fft_c2c_backward, fft_r2c_backward, fft_c2r_backward +from spectral_op_np import ( + fft_c2c, + fft_r2c, + fft_c2r, + fft_c2c_backward, + fft_r2c_backward, + fft_c2r_backward, +) from paddle import _C_ops sys.path.append("../") @@ -32,8 +39,11 @@ def parameterize(attrs, input_values=None): if isinstance(attrs, str): attrs = [attrs] - input_dicts = (attrs if input_values is None else - [dict(zip(attrs, vals)) for vals in input_values]) + input_dicts = ( + attrs + if input_values is None + else [dict(zip(attrs, vals)) for vals in input_values] + ) def decorator(base_class): test_class_module = sys.modules[base_class.__module__].__dict__ @@ -43,8 +53,7 @@ def decorator(base_class): name = class_name(base_class, idx, input_dict) - test_class_module[name] = type(name, (base_class, ), - test_class_dict) + test_class_module[name] = type(name, (base_class,), test_class_dict) for method_name in list(base_class.__dict__): if method_name.startswith("test"): @@ -60,7 +69,8 @@ def to_safe_name(s): def class_name(cls, num, params_dict): suffix = to_safe_name( - next((v for v in params_dict.values() if isinstance(v, str)), "")) + next((v for v in params_dict.values() if isinstance(v, str)), "") + ) if TEST_CASE_NAME in params_dict: suffix = to_safe_name(params_dict["test_case"]) return "{}_{}{}".format(cls.__name__, num, suffix and "_" + suffix) @@ -80,20 +90,55 @@ def fft_c2r_python_api(x, axes, norm, forward, last_dim_size=0): @parameterize( (TEST_CASE_NAME, 'x', 'axes', 'norm', 'forward'), - [('test_axes_is_sqe_type', (np.random.random( - (12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), [0, 1], 'forward', True), - ('test_axis_not_last', (np.random.random( - (4, 8, 4)) + 1j * np.random.random( - (4, 8, 4))).astype(np.complex128), (0, 1), "backward", False), - ('test_norm_forward', (np.random.random((12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), (0, ), "forward", False), - ('test_norm_backward', (np.random.random((12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), (0, ), "backward", True), - ('test_norm_ortho', (np.random.random((12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), (1, ), "ortho", True)]) + [ + ( + 'test_axes_is_sqe_type', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + [0, 1], + 'forward', + True, + ), + ( + 'test_axis_not_last', + ( + np.random.random((4, 8, 4)) + 1j * np.random.random((4, 8, 4)) + ).astype(np.complex128), + (0, 1), + "backward", + False, + ), + ( + 'test_norm_forward', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + (0,), + "forward", + False, + ), + ( + 'test_norm_backward', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + (0,), + "backward", + True, + ), + ( + 'test_norm_ortho', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + (1,), + "ortho", + True, + ), + ], +) class TestFFTC2COp(OpTest): - def setUp(self): self.op_type = "fft_c2c" self.dtype = self.x.dtype @@ -105,90 +150,173 @@ def setUp(self): self.attrs = { 'axes': self.axes, 'normalization': self.norm, - "forward": self.forward + "forward": self.forward, } self.outputs = {'Out': out} - self.out_grad = (np.random.random(self.x.shape) + - 1j * np.random.random(self.x.shape)).astype( - self.x.dtype) - self.x_grad = fft_c2c_backward(self.out_grad, self.axes, self.norm, - self.forward) + self.out_grad = ( + np.random.random(self.x.shape) + 1j * np.random.random(self.x.shape) + ).astype(self.x.dtype) + self.x_grad = fft_c2c_backward( + self.out_grad, self.axes, self.norm, self.forward + ) def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad("X", - "Out", - user_defined_grads=[self.x_grad], - user_defined_grad_outputs=[self.out_grad], - check_eager=True) + self.check_grad( + "X", + "Out", + user_defined_grads=[self.x_grad], + user_defined_grad_outputs=[self.out_grad], + check_eager=True, + ) @parameterize( (TEST_CASE_NAME, 'x', 'axes', 'norm', 'forward', 'last_dim_size'), - [('test_axes_is_sqe_type', (np.random.random( - (12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), [0, 1], 'forward', True, 26), - ('test_axis_not_last', (np.random.random( - (4, 7, 4)) + 1j * np.random.random((4, 7, 4))).astype(np.complex128), - (0, 1), "backward", False, None), - ('test_norm_forward', (np.random.random((12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), (0, ), "forward", False, 22), - ('test_norm_backward', (np.random.random((12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), (0, ), "backward", True, 22), - ('test_norm_ortho', (np.random.random((12, 14)) + 1j * np.random.random( - (12, 14))).astype(np.complex128), (1, ), "ortho", True, 26)]) + [ + ( + 'test_axes_is_sqe_type', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + [0, 1], + 'forward', + True, + 26, + ), + ( + 'test_axis_not_last', + ( + np.random.random((4, 7, 4)) + 1j * np.random.random((4, 7, 4)) + ).astype(np.complex128), + (0, 1), + "backward", + False, + None, + ), + ( + 'test_norm_forward', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + (0,), + "forward", + False, + 22, + ), + ( + 'test_norm_backward', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + (0,), + "backward", + True, + 22, + ), + ( + 'test_norm_ortho', + ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(np.complex128), + (1,), + "ortho", + True, + 26, + ), + ], +) class TestFFTC2ROp(OpTest): - def setUp(self): self.op_type = "fft_c2r" self.dtype = self.x.dtype self.python_api = fft_c2r_python_api - out = fft_c2r(self.x, self.axes, self.norm, self.forward, - self.last_dim_size) + out = fft_c2r( + self.x, self.axes, self.norm, self.forward, self.last_dim_size + ) self.inputs = {'X': self.x} self.attrs = { "axes": self.axes, "normalization": self.norm, "forward": self.forward, - "last_dim_size": self.last_dim_size + "last_dim_size": self.last_dim_size, } self.outputs = {'Out': out} self.out_grad = np.random.random(out.shape).astype(out.dtype) - self.x_grad = fft_c2r_backward(self.x, self.out_grad, self.axes, - self.norm, self.forward, - self.last_dim_size) + self.x_grad = fft_c2r_backward( + self.x, + self.out_grad, + self.axes, + self.norm, + self.forward, + self.last_dim_size, + ) def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(["X"], - "Out", - user_defined_grads=[self.x_grad], - user_defined_grad_outputs=[self.out_grad], - check_eager=True) + self.check_grad( + ["X"], + "Out", + user_defined_grads=[self.x_grad], + user_defined_grad_outputs=[self.out_grad], + check_eager=True, + ) @parameterize( (TEST_CASE_NAME, 'x', 'axes', 'norm', 'forward', 'onesided'), - [('test_axes_is_sqe_type', np.random.randn(12, 18).astype(np.float64), - (0, 1), 'forward', True, True), - ('test_axis_not_last', np.random.randn(4, 8, 4).astype(np.float64), - (0, 1), "backward", False, True), - ('test_norm_forward', np.random.randn(12, 18).astype(np.float64), - (0, 1), "forward", False, False), - ('test_norm_backward', np.random.randn(12, 18).astype(np.float64), - (0, ), "backward", True, False), - ('test_norm_ortho', np.random.randn(12, 18).astype(np.float64), - (1, ), "ortho", True, False)]) + [ + ( + 'test_axes_is_sqe_type', + np.random.randn(12, 18).astype(np.float64), + (0, 1), + 'forward', + True, + True, + ), + ( + 'test_axis_not_last', + np.random.randn(4, 8, 4).astype(np.float64), + (0, 1), + "backward", + False, + True, + ), + ( + 'test_norm_forward', + np.random.randn(12, 18).astype(np.float64), + (0, 1), + "forward", + False, + False, + ), + ( + 'test_norm_backward', + np.random.randn(12, 18).astype(np.float64), + (0,), + "backward", + True, + False, + ), + ( + 'test_norm_ortho', + np.random.randn(12, 18).astype(np.float64), + (1,), + "ortho", + True, + False, + ), + ], +) class TestFFTR2COp(OpTest): - def setUp(self): self.op_type = "fft_r2c" self.dtype = self.x.dtype @@ -201,20 +329,28 @@ def setUp(self): 'axes': self.axes, 'normalization': self.norm, "forward": self.forward, - 'onesided': self.onesided + 'onesided': self.onesided, } self.outputs = {'Out': out} self.out_grad = np.random.random(out.shape).astype(out.dtype) - self.x_grad = fft_r2c_backward(self.x, self.out_grad, self.axes, - self.norm, self.forward, self.onesided) + self.x_grad = fft_r2c_backward( + self.x, + self.out_grad, + self.axes, + self.norm, + self.forward, + self.onesided, + ) def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad("X", - "Out", - user_defined_grads=[self.x_grad], - user_defined_grad_outputs=[self.out_grad], - check_eager=True) + self.check_grad( + "X", + "Out", + user_defined_grads=[self.x_grad], + user_defined_grad_outputs=[self.out_grad], + check_eager=True, + ) diff --git a/python/paddle/fluid/tests/unittests/find_ports.py b/python/paddle/fluid/tests/unittests/find_ports.py index 5618d40344831..868efe0ee8e9d 100644 --- a/python/paddle/fluid/tests/unittests/find_ports.py +++ b/python/paddle/fluid/tests/unittests/find_ports.py @@ -24,8 +24,7 @@ def train(): worker_endpoints = worker_endpoints_env trainers_num = len(worker_endpoints.split(',')) - name = "worker_endpoints:{}" \ - .format(worker_endpoints) + name = "worker_endpoints:{}".format(worker_endpoints) print(name) file_name = os.getenv("PADDLE_LAUNCH_LOG") diff --git a/python/paddle/fluid/tests/unittests/fleet_heter_ps_training.py b/python/paddle/fluid/tests/unittests/fleet_heter_ps_training.py index e99021f282291..08e1e1d7b2450 100644 --- a/python/paddle/fluid/tests/unittests/fleet_heter_ps_training.py +++ b/python/paddle/fluid/tests/unittests/fleet_heter_ps_training.py @@ -30,32 +30,38 @@ def get_dataset(inputs): def net(batch_size=4, lr=0.01): """ - network definition - - Args: - batch_size(int): the size of mini-batch for training - lr(float): learning rate of training - Returns: - avg_cost: LoDTensor of cost. - """ + network definition + + Args: + batch_size(int): the size of mini-batch for training + lr(float): learning rate of training + Returns: + avg_cost: LoDTensor of cost. + """ dnn_input_dim, lr_input_dim = int(2), int(2) with fluid.device_guard("cpu"): - dnn_data = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - lr_data = fluid.layers.data(name="lr_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="float32", - lod_level=0, - append_batch_size=False) + dnn_data = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + lr_data = fluid.layers.data( + name="lr_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="float32", + lod_level=0, + append_batch_size=False, + ) datas = [dnn_data, lr_data, label] @@ -67,10 +73,13 @@ def net(batch_size=4, lr=0.01): size=[dnn_input_dim, dnn_layer_dims[0]], param_attr=fluid.ParamAttr( name="deep_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=True) - dnn_pool = fluid.layers.sequence_pool(input=dnn_embedding, - pool_type="sum") + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) + dnn_pool = fluid.layers.sequence_pool( + input=dnn_embedding, pool_type="sum" + ) dnn_out = dnn_pool # build lr model @@ -80,8 +89,10 @@ def net(batch_size=4, lr=0.01): size=[lr_input_dim, 1], param_attr=fluid.ParamAttr( name="wide_embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=True) + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) lr_pool = fluid.layers.sequence_pool(input=lr_embbding, pool_type="sum") with fluid.device_guard("gpu"): @@ -91,8 +102,10 @@ def net(batch_size=4, lr=0.01): size=dim, act="relu", param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01)), - name='dnn-fc-%d' % i) + initializer=fluid.initializer.Constant(value=0.01) + ), + name='dnn-fc-%d' % i, + ) dnn_out = fc merge_layer = fluid.layers.concat(input=[dnn_out, lr_pool], axis=1) @@ -124,22 +137,22 @@ def net(batch_size=4, lr=0.01): if fleet.is_server(): pass - #fleet.init_server() - #fleet.run_server() + # fleet.init_server() + # fleet.run_server() elif fleet.is_heter_worker(): pass - #fleet.init_heter_worker() - #fleet.run_heter_worker(dataset=dataset) + # fleet.init_heter_worker() + # fleet.run_heter_worker(dataset=dataset) fleet.stop_worker() elif fleet.is_worker(): pass - #place = fluid.CPUPlace() - #exe = fluid.Executor(place) - #exe.run(fluid.default_startup_program()) - #fleet.init_worker() - #step = 1 - #for i in range(step): + # place = fluid.CPUPlace() + # exe = fluid.Executor(place) + # exe.run(fluid.default_startup_program()) + # fleet.init_worker() + # step = 1 + # for i in range(step): # exe.train_from_dataset( # program=fluid.default_main_program(), dataset=dataset, debug=False) - #exe.close() - #fleet.stop_worker() + # exe.close() + # fleet.stop_worker() diff --git a/python/paddle/fluid/tests/unittests/fleet_meta_optimizer_base.py b/python/paddle/fluid/tests/unittests/fleet_meta_optimizer_base.py index ebeeb1e272f09..ee0f203881f65 100755 --- a/python/paddle/fluid/tests/unittests/fleet_meta_optimizer_base.py +++ b/python/paddle/fluid/tests/unittests/fleet_meta_optimizer_base.py @@ -22,15 +22,16 @@ class TestFleetMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "1" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.1:36002" self._debug = False def debug_program(self, main_prog, startup_prog): - if not self._debug: return + if not self._debug: + return main_prog_ops = main_prog.global_block().ops startup_prog_ops = startup_prog.global_block().ops @@ -38,8 +39,11 @@ def debug_program(self, main_prog, startup_prog): main_prog_op_types = [op.type for op in main_prog_ops] startup_prog_op_types = [op.type for op in startup_prog_ops] - print("=== debug program and ops in func [{}] ===".format( - inspect.stack()[1].function)) + print( + "=== debug program and ops in func [{}] ===".format( + inspect.stack()[1].function + ) + ) print(main_prog) print(main_prog_op_types) print(startup_prog) @@ -50,29 +54,29 @@ def net(self, main_prog, startup_prog): with fluid.unique_name.guard(): role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') - - fc_1 = paddle.fluid.layers.fc(input=input_x, - size=64, - act='tanh') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) + + fc_1 = paddle.fluid.layers.fc( + input=input_x, size=64, act='tanh' + ) fc_2 = paddle.fluid.layers.fc(input=fc_1, size=256, act='tanh') - prediction = paddle.fluid.layers.fc(input=[fc_2], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[fc_2], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() return avg_cost, strategy def pp_net(self, main_prog, startup_prog, pp_degree=2): - def fc_block(input_x): fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') @@ -84,23 +88,24 @@ def fc_block(input_x): role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) with fluid.device_guard("gpu:0"): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_y = paddle.fluid.layers.data(name="y", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_y = paddle.fluid.layers.data( + name="y", shape=[1], dtype='int64' + ) for stage_idx in range(pp_degree): with fluid.device_guard("gpu:" + str(stage_idx)): input_x = fc_block(input_x) with fluid.device_guard("gpu:" + str(pp_degree - 1)): - prediction = paddle.fluid.layers.fc(input=[input_x], - size=2, - act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.fluid.layers.fc( + input=[input_x], size=2, act='softmax' + ) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -120,14 +125,16 @@ def boundary_net(self, main_prog, startup_prog): strategy = fleet.DistributedStrategy() return avg_cost, strategy - def optimizer(self, - loss, - strategy, - train_prog, - startup_prog, - name='momentum', - regularization=None, - grad_clip=None): + def optimizer( + self, + loss, + strategy, + train_prog, + startup_prog, + name='momentum', + regularization=None, + grad_clip=None, + ): with fluid.program_guard(train_prog, startup_prog): with fluid.unique_name.guard(): if name == 'momentum': @@ -135,18 +142,23 @@ def optimizer(self, learning_rate=0.01, momentum=0.9, regularization=regularization, - grad_clip=grad_clip) + grad_clip=grad_clip, + ) elif name == 'adam': optimizer = paddle.fluid.optimizer.Adam( learning_rate=0.01, regularization=regularization, - grad_clip=grad_clip) + grad_clip=grad_clip, + ) elif name == 'adamw': - optimizer = paddle.optimizer.AdamW(learning_rate=0.01, - weight_decay=0.01, - grad_clip=grad_clip) - optimizer = fleet.distributed_optimizer(optimizer, - strategy=strategy) + optimizer = paddle.optimizer.AdamW( + learning_rate=0.01, + weight_decay=0.01, + grad_clip=grad_clip, + ) + optimizer = fleet.distributed_optimizer( + optimizer, strategy=strategy + ) optimizer.minimize(loss) def set_strategy(self, strategy, name): @@ -182,7 +194,7 @@ def set_strategy(self, strategy, name): strategy.dgc_configs = { "rampup_begin_step": 128, "rampup_step": 100, - "sparsity": [0.996, 0.999] + "sparsity": [0.996, 0.999], } elif name == 'recompute': strategy.recompute = True @@ -230,7 +242,7 @@ def set_strategy(self, strategy, name): strategy.recompute_configs = { "checkpoints": ["fc_0.tmp_2", "fc_1.tmp_2"], "enable_offload": True, - "checkpoint_shape": [256] + "checkpoint_shape": [256], } elif name == "pipeline": strategy.pipeline = True diff --git a/python/paddle/fluid/tests/unittests/fleet_ps_training.py b/python/paddle/fluid/tests/unittests/fleet_ps_training.py index 65fa1ef935ef1..d14605a6179f7 100644 --- a/python/paddle/fluid/tests/unittests/fleet_ps_training.py +++ b/python/paddle/fluid/tests/unittests/fleet_ps_training.py @@ -15,7 +15,9 @@ import paddle.fluid as fluid from utils import gen_data from nets import mlp -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, +) from paddle.fluid.incubate.fleet.base import role_maker input_x = fluid.layers.data(name="x", shape=[32], dtype='float32') @@ -43,8 +45,10 @@ exe.run(fleet.startup_program) step = 1001 for i in range(step): - cost_val = exe.run(program=fleet.main_program, - feed=gen_data(), - fetch_list=[cost.name]) - print("worker_index: %d, step%d cost = %f" % - (fleet.worker_index(), i, cost_val[0])) + cost_val = exe.run( + program=fleet.main_program, feed=gen_data(), fetch_list=[cost.name] + ) + print( + "worker_index: %d, step%d cost = %f" + % (fleet.worker_index(), i, cost_val[0]) + ) diff --git a/python/paddle/fluid/tests/unittests/gradient_checker.py b/python/paddle/fluid/tests/unittests/gradient_checker.py index d928abe2ca99c..6e20037185285 100644 --- a/python/paddle/fluid/tests/unittests/gradient_checker.py +++ b/python/paddle/fluid/tests/unittests/gradient_checker.py @@ -21,6 +21,7 @@ import paddle.fluid.core as core from paddle.fluid.backward import _append_grad_suffix_, _as_list from paddle.fluid.framework import _test_eager_guard + try: from collections.abc import Sequence except: @@ -91,8 +92,11 @@ def make_jacobian(x, y_size, np_dtype): return np.zeros((_product(x.shape), y_size), dtype=np_dtype) elif isinstance(x, Sequence): jacobians = list( - filter(lambda t: t is not None, - (make_jacobian(item, y_size, np_dtype) for item in x))) + filter( + lambda t: t is not None, + (make_jacobian(item, y_size, np_dtype) for item in x), + ) + ) return jacobians else: None @@ -151,7 +155,7 @@ def run(): _set_item(x_t, i, orig, np_type) for j in range(len(y)): - jacobian[j][i, :] = (y_pos[j] - y_neg[j]) / delta / 2. + jacobian[j][i, :] = (y_pos[j] - y_neg[j]) / delta / 2.0 return jacobian @@ -180,10 +184,9 @@ def _compute_analytical_jacobian(program, x, y, place, scope): np_type = dtype_to_np_dtype(y.dtype) # create dy Variable in Program - dy = program.global_block().create_var(name=dy_name, - shape=y.shape, - dtype=np_type, - persistable=True) + dy = program.global_block().create_var( + name=dy_name, shape=y.shape, dtype=np_type, persistable=True + ) # append backward dx = fluid.gradients(y, x, dy) @@ -213,23 +216,26 @@ def _compute_analytical_jacobian(program, x, y, place, scope): if dx_res[j] is not None: jacobian[dx_idx][:, i] = dx_res[j].flatten() else: - jacobian[dx_idx][:, i] = np.zeros(dx[dx_idx].shape, - dtype=np_type).flatten() + jacobian[dx_idx][:, i] = np.zeros( + dx[dx_idx].shape, dtype=np_type + ).flatten() _set_item(dy_t, i, 0, np_type) return jacobian -def grad_check(x, - y, - x_init=None, - place=None, - program=None, - eps=1e-6, - atol=1e-5, - rtol=1e-3, - raise_exception=True): +def grad_check( + x, + y, + x_init=None, + place=None, + program=None, + eps=1e-6, + atol=1e-5, + rtol=1e-3, + raise_exception=True, +): """ Check numerical and analytical gradients for dy/dx. Each Jacobian gradients is a 2-D array with shape [xi_size, yi_size]. @@ -279,8 +285,10 @@ def fail_test(msg): # init inputs if x_init is not None if x_init: if len(x_init) != len(x): - raise ValueError('len(x_init) (=%d) is not the same' - ' as len(x) (= %d)' % (len(x_init), len(x))) + raise ValueError( + 'len(x_init) (=%d) is not the same' + ' as len(x) (= %d)' % (len(x_init), len(x)) + ) # init variable in main program for var, arr in zip(x, x_init): assert var.shape == arr.shape @@ -310,31 +318,37 @@ def fail_test(msg): clone_x.append(b.var(xi.name)) break analytical.append( - _compute_analytical_jacobian(prog, clone_x, clone_y, place, scope)) + _compute_analytical_jacobian(prog, clone_x, clone_y, place, scope) + ) for i, (x_idx, y_idx) in enumerate( - product(*[range(len(x)), range(len(y))])): + product(*[range(len(x)), range(len(y))]) + ): a = analytical[y_idx][x_idx] n = numerical[x_idx][y_idx] if not np.allclose(a, n, rtol, atol): - msg = 'Jacobian mismatch for output %s ' \ - 'with respect to input %s on %s,\n' \ - 'numerical:%s\nanalytical:%s\n' \ - % (y[y_idx].name, x[x_idx].name, str(place), n, a) + msg = ( + 'Jacobian mismatch for output %s ' + 'with respect to input %s on %s,\n' + 'numerical:%s\nanalytical:%s\n' + % (y[y_idx].name, x[x_idx].name, str(place), n, a) + ) return fail_test(msg) return True -def double_grad_check(x, - y, - x_init=None, - y_grads=None, - place=None, - program=None, - eps=1e-6, - atol=1e-5, - rtol=1e-3, - raise_exception=True): +def double_grad_check( + x, + y, + x_init=None, + y_grads=None, + place=None, + program=None, + eps=1e-6, + atol=1e-5, + rtol=1e-3, + raise_exception=True, +): """ Check gradients of gradients. This function will append backward to the program before second order gradient check. @@ -375,10 +389,9 @@ def double_grad_check(x, for yi in y: dyi_name = _append_grad_suffix_(yi.name) np_type = dtype_to_np_dtype(yi.dtype) - dy = program.global_block().create_var(name=dyi_name, - shape=yi.shape, - dtype=np_type, - persistable=True) + dy = program.global_block().create_var( + name=dyi_name, shape=yi.shape, dtype=np_type, persistable=True + ) dy.stop_gradient = False v = np.random.random(size=yi.shape).astype(np_type) set_var_in_scope(scope, place, dyi_name, v) @@ -407,17 +420,19 @@ def double_grad_check(x, # check triple grad and two outputs of the triple Kernel -def triple_grad_check(x, - y, - x_init=None, - y_grads=None, - x_grads_grads=None, - place=None, - program=None, - eps=1e-6, - atol=1e-5, - rtol=1e-3, - raise_exception=True): +def triple_grad_check( + x, + y, + x_init=None, + y_grads=None, + x_grads_grads=None, + place=None, + program=None, + eps=1e-6, + atol=1e-5, + rtol=1e-3, + raise_exception=True, +): """ Check triple gradients. This function will append backward to the program before third order gradient check. @@ -459,10 +474,9 @@ def triple_grad_check(x, for yi in y: dyi_name = _append_grad_suffix_(yi.name) np_type = dtype_to_np_dtype(yi.dtype) - dy = program.global_block().create_var(name=dyi_name, - shape=yi.shape, - dtype=np_type, - persistable=True) + dy = program.global_block().create_var( + name=dyi_name, shape=yi.shape, dtype=np_type, persistable=True + ) dy.stop_gradient = False v = np.random.random(size=yi.shape).astype(np_type) set_var_in_scope(scope, place, dyi_name, v) @@ -484,10 +498,9 @@ def triple_grad_check(x, for dxi in target_grads: ddxi_name = _append_grad_suffix_(dxi.name) np_type = dtype_to_np_dtype(dxi.dtype) - ddx = program.global_block().create_var(name=ddxi_name, - shape=dxi.shape, - dtype=np_type, - persistable=True) + ddx = program.global_block().create_var( + name=ddxi_name, shape=dxi.shape, dtype=np_type, persistable=True + ) ddx.stop_gradient = False v = np.random.random(size=dxi.shape).astype(np_type) set_var_in_scope(scope, place, ddxi_name, v) @@ -507,30 +520,30 @@ def triple_grad_check(x, target_grads_grads = fluid.gradients(target_grads, x, x_grads_grads) # filter None in target_grads_grads for Dy/Dx may be None in kernel - filted = [(i, dyi) for i, dyi in enumerate(target_grads_grads) - if dyi is not None] + filted = [ + (i, dyi) for i, dyi in enumerate(target_grads_grads) if dyi is not None + ] filted_idx, filted_target_grads_grads = zip(*filted) x += x_grads_grads x_init += x_grads_grads_init # x <=> [x, dout, ddx] - grad_check(x=x, - y=filted_target_grads_grads, - x_init=x_init, - place=place, - program=program, - eps=eps, - atol=atol, - rtol=rtol) - - -def get_static_double_grad(x, - y, - x_init=None, - dy_init=None, - place=None, - program=None): + grad_check( + x=x, + y=filted_target_grads_grads, + x_init=x_init, + place=place, + program=program, + eps=eps, + atol=atol, + rtol=rtol, + ) + + +def get_static_double_grad( + x, y, x_init=None, dy_init=None, place=None, program=None +): """ Get Double Grad result of static graph. @@ -554,10 +567,9 @@ def get_static_double_grad(x, yi = y[i] dyi_name = _append_grad_suffix_(yi.name) np_type = dtype_to_np_dtype(yi.dtype) - dy = program.global_block().create_var(name=dyi_name, - shape=yi.shape, - dtype=np_type, - persistable=True) + dy = program.global_block().create_var( + name=dyi_name, shape=yi.shape, dtype=np_type, persistable=True + ) dy.stop_gradient = False set_var_in_scope(scope, place, dyi_name, dy_init[i]) y_grads.append(dy) @@ -598,8 +610,10 @@ def get_static_double_grad(x, # init inputs if x_init is not None if x_init: if len(x_init) != len(x): - raise ValueError('len(x_init) (=%d) is not the same' - ' as len(x) (= %d)' % (len(x_init), len(x))) + raise ValueError( + 'len(x_init) (=%d) is not the same' + ' as len(x) (= %d)' % (len(x_init), len(x)) + ) # init variable in main program for var, arr in zip(x, x_init): assert var.shape == arr.shape @@ -611,10 +625,9 @@ def get_static_double_grad(x, np_type = dtype_to_np_dtype(yi.dtype) dy_name = _append_grad_suffix_(yi.name) # create dy Variable in Program - dy = program.global_block().create_var(name=dy_name, - shape=yi.shape, - dtype=np_type, - persistable=True) + dy = program.global_block().create_var( + name=dy_name, shape=yi.shape, dtype=np_type, persistable=True + ) # init dy tensor in scope value = np.ones(yi.shape, dtype=np_type) dy_t = set_var_in_scope(scope, place, dy_name, value) @@ -633,11 +646,9 @@ def get_static_double_grad(x, return ddx_res -def get_eager_double_grad(func, - x_init=None, - dy_init=None, - place=None, - return_mid_result=False): +def get_eager_double_grad( + func, x_init=None, dy_init=None, place=None, return_mid_result=False +): """ Get Double Grad result of dygraph. @@ -670,11 +681,13 @@ def get_eager_double_grad(func, dys.append(dy_tensor) # calculate first derivative outputs = func(inputs) - d_inputs = paddle.grad(outputs=outputs, - inputs=inputs, - grad_outputs=dys, - create_graph=True, - allow_unused=True) + d_inputs = paddle.grad( + outputs=outputs, + inputs=inputs, + grad_outputs=dys, + create_graph=True, + allow_unused=True, + ) d_inputs = [d_input for d_input in d_inputs if d_input is not None] # calcluate second derivative @@ -691,29 +704,34 @@ def get_eager_double_grad(func, ddy.stop_gradient = False ddys.append(ddy) - dd_inputs = paddle.grad(outputs=d_inputs, - inputs=inputs, - grad_outputs=ddys, - create_graph=create_graph, - allow_unused=True) + dd_inputs = paddle.grad( + outputs=d_inputs, + inputs=inputs, + grad_outputs=ddys, + create_graph=create_graph, + allow_unused=True, + ) if return_mid_result: - return [dd_input for dd_input in dd_inputs - if dd_input is not None], inputs + ddys + return [ + dd_input for dd_input in dd_inputs if dd_input is not None + ], inputs + ddys else: return [ dd_input.numpy() for dd_input in dd_inputs if dd_input is not None ] -def double_grad_check_for_dygraph(func, - x, - y, - x_init=None, - place=None, - atol=1e-5, - rtol=1e-3, - raise_exception=True): +def double_grad_check_for_dygraph( + func, + x, + y, + x_init=None, + place=None, + atol=1e-5, + rtol=1e-3, + raise_exception=True, +): """ Check second order gradients of dygraph. This function will compare the second order gradients of dygraph and second order gradients of static graph @@ -755,34 +773,38 @@ def fail_test(msg): paddle.disable_static() with _test_eager_guard(): - eager_double_grad = get_eager_double_grad(func, x_init, y_grads_init, - place) + eager_double_grad = get_eager_double_grad( + func, x_init, y_grads_init, place + ) paddle.enable_static() - static_double_grad = get_static_double_grad(x, y, x_init, y_grads_init, - place) + static_double_grad = get_static_double_grad( + x, y, x_init, y_grads_init, place + ) if len(static_double_grad) != len(eager_double_grad): - msg = "The output grad tensor's number of static graph is different with dygraph, " \ + msg = ( + "The output grad tensor's number of static graph is different with dygraph, " "please check the python api unit test used." + ) raise RuntimeError(msg) for i in range(len(static_double_grad)): - if not np.allclose(static_double_grad[i], eager_double_grad[i], rtol, - atol): - msg = 'Check eager double result fail. Mismatch between static_graph double grad ' \ - 'and eager double grad on %s, the output double grad tensor\'s index is : %d \n' \ - 'static:%s\n eager:%s\n' \ + if not np.allclose( + static_double_grad[i], eager_double_grad[i], rtol, atol + ): + msg = ( + 'Check eager double result fail. Mismatch between static_graph double grad ' + 'and eager double grad on %s, the output double grad tensor\'s index is : %d \n' + 'static:%s\n eager:%s\n' % (str(place), i, static_double_grad[i], eager_double_grad[i]) + ) return fail_test(msg) -def get_static_triple_grad(x, - y, - x_init=None, - dy_init=None, - place=None, - program=None): +def get_static_triple_grad( + x, y, x_init=None, dy_init=None, place=None, program=None +): """ Get Triple Grad result of static graph. @@ -805,10 +827,9 @@ def get_static_triple_grad(x, yi = y[i] dyi_name = _append_grad_suffix_(yi.name) np_type = dtype_to_np_dtype(yi.dtype) - dy = program.global_block().create_var(name=dyi_name, - shape=yi.shape, - dtype=np_type, - persistable=True) + dy = program.global_block().create_var( + name=dyi_name, shape=yi.shape, dtype=np_type, persistable=True + ) dy.stop_gradient = False set_var_in_scope(scope, place, dyi_name, dy_init[i]) y_grads.append(dy) @@ -828,19 +849,14 @@ def get_static_triple_grad(x, value = np.ones(dxi.shape, dtype=np_type) x_grads_grads_init.append(value) - return get_static_double_grad(x, - y, - x_init, - dy_init=x_grads_grads_init, - place=place, - program=program) + return get_static_double_grad( + x, y, x_init, dy_init=x_grads_grads_init, place=place, program=program + ) -def get_eager_triple_grad(func, - x_init=None, - dy_init=None, - place=None, - return_mid_result=False): +def get_eager_triple_grad( + func, x_init=None, dy_init=None, place=None, return_mid_result=False +): """ Get triple Grad result of dygraph. @@ -853,11 +869,9 @@ def get_eager_triple_grad(func, Returns: A list of numpy array that stores second derivative result calulated by dygraph """ - dd_y, dd_x = get_eager_double_grad(func, - x_init, - dy_init, - place, - return_mid_result=True) + dd_y, dd_x = get_eager_double_grad( + func, x_init, dy_init, place, return_mid_result=True + ) # calcluate third derivative dddys = [] @@ -866,23 +880,24 @@ def get_eager_triple_grad(func, dddy = paddle.ones(shape=dd_yi.shape, dtype=dd_yi.dtype) dddy.stop_gradient = False dddys.append(dddy) - ddd_inputs = paddle.grad(outputs=dd_y, - inputs=dd_x, - grad_outputs=dddys, - allow_unused=True) + ddd_inputs = paddle.grad( + outputs=dd_y, inputs=dd_x, grad_outputs=dddys, allow_unused=True + ) return [ ddd_input.numpy() for ddd_input in ddd_inputs if ddd_input is not None ] -def triple_grad_check_for_dygraph(func, - x, - y, - x_init=None, - place=None, - atol=1e-5, - rtol=1e-3, - raise_exception=True): +def triple_grad_check_for_dygraph( + func, + x, + y, + x_init=None, + place=None, + atol=1e-5, + rtol=1e-3, + raise_exception=True, +): """ Check third order gradients of dygraph. This function will compare the third order gradients of dygraph and third order gradients of static graph @@ -924,23 +939,30 @@ def fail_test(msg): paddle.disable_static() with _test_eager_guard(): - eager_triple_grad = get_eager_triple_grad(func, x_init, y_grads_init, - place) + eager_triple_grad = get_eager_triple_grad( + func, x_init, y_grads_init, place + ) paddle.enable_static() - static_triple_grad = get_static_triple_grad(x, y, x_init, y_grads_init, - place) + static_triple_grad = get_static_triple_grad( + x, y, x_init, y_grads_init, place + ) if len(static_triple_grad) != len(eager_triple_grad): - msg = "The output grad tensor's number of static graph is different with dygraph, " \ + msg = ( + "The output grad tensor's number of static graph is different with dygraph, " "please check the python api unit test used." + ) raise RuntimeError(msg) for i in range(len(static_triple_grad)): - if not np.allclose(static_triple_grad[i], eager_triple_grad[i], rtol, - atol): - msg = 'Check eager double result fail. Mismatch between static_graph double grad ' \ - 'and eager double grad on %s, the output double grad tensor\'s index is : %d \n' \ - 'static:%s\n eager:%s\n' \ + if not np.allclose( + static_triple_grad[i], eager_triple_grad[i], rtol, atol + ): + msg = ( + 'Check eager double result fail. Mismatch between static_graph double grad ' + 'and eager double grad on %s, the output double grad tensor\'s index is : %d \n' + 'static:%s\n eager:%s\n' % (str(place), i, static_triple_grad[i], eager_triple_grad[i]) + ) return fail_test(msg) diff --git a/python/paddle/fluid/tests/unittests/hccl_tools.py b/python/paddle/fluid/tests/unittests/hccl_tools.py index f47fe9962fe6d..85011b0d237b1 100644 --- a/python/paddle/fluid/tests/unittests/hccl_tools.py +++ b/python/paddle/fluid/tests/unittests/hccl_tools.py @@ -48,24 +48,28 @@ def parse_args(): Examples: >>> parse_args() """ - parser = ArgumentParser(description="mindspore distributed training launch " - "helper utilty that will generate hccl" - " config file") + parser = ArgumentParser( + description="mindspore distributed training launch " + "helper utilty that will generate hccl" + " config file" + ) parser.add_argument( "--device_num", type=str, default="[0,8)", - help= - "The number of the Ascend accelerators used. please note that the Ascend accelerators" + help="The number of the Ascend accelerators used. please note that the Ascend accelerators" "used must be continuous, such [0,4) means to use four chips " "0,1,2,3; [0,1) means to use chip 0; The first four chips are" "a group, and the last four chips are a group. In addition to" "the [0,8) chips are allowed, other cross-group such as [3,6)" - "are prohibited.") - parser.add_argument("--visible_devices", - type=str, - default="0,1,2,3,4,5,6,7", - help="will use the visible devices sequentially") + "are prohibited.", + ) + parser.add_argument( + "--visible_devices", + type=str, + default="0,1,2,3,4,5,6,7", + help="will use the visible devices sequentially", + ) parser.add_argument("--server_ip", type=str, default="", help="server ip") args = parser.parse_args() return args @@ -108,20 +112,25 @@ def main(): first_num = int(args.device_num[1]) last_num = int(args.device_num[3]) if first_num < 0 or last_num > 8: - raise ValueError("device num {} must be in range [0,8] !".format( - args.device_num)) + raise ValueError( + "device num {} must be in range [0,8] !".format(args.device_num) + ) if first_num > last_num: raise ValueError( "First num {} of device num {} must less than last num {} !".format( - first_num, args.device_num, last_num)) + first_num, args.device_num, last_num + ) + ) if first_num < 4: if last_num > 4: if first_num == 0 and last_num == 8: pass else: raise ValueError( - "device num {} must be in the same group of [0,4] or [4,8] !" - .format(args.device_num)) + "device num {} must be in the same group of [0,4] or [4,8] !".format( + args.device_num + ) + ) device_num_list = list(range(first_num, last_num)) print("device_num_list:", device_num_list) @@ -146,26 +155,32 @@ def main(): device = { 'device_id': device_id, 'device_ip': device_ip, - 'rank_id': str(rank_id) + 'rank_id': str(rank_id), } - print('rank_id:{}, device_id:{}, device_ip:{}'.format( - rank_id, device_id, device_ip)) + print( + 'rank_id:{}, device_id:{}, device_ip:{}'.format( + rank_id, device_id, device_ip + ) + ) rank_id += 1 device_list.append(device) - hccn_table['server_list'].append({ - 'server_id': server_id, - 'device': device_list, - 'host_nic_ip': 'reserve' - }) + hccn_table['server_list'].append( + { + 'server_id': server_id, + 'device': device_list, + 'host_nic_ip': 'reserve', + } + ) hccn_table['status'] = 'completed' # save hccn_table to file table_path = os.getcwd() table_fn = os.path.join( table_path, - 'hccl_{}p_{}_{}.json'.format(len(device_num_list), - "".join(map(str, - device_num_list)), server_id)) + 'hccl_{}p_{}_{}.json'.format( + len(device_num_list), "".join(map(str, device_num_list)), server_id + ), + ) with open(table_fn, 'w') as table_fp: json.dump(hccn_table, table_fp, indent=4) sys.stdout.flush() diff --git a/python/paddle/fluid/tests/unittests/hdfs_test_utils.py b/python/paddle/fluid/tests/unittests/hdfs_test_utils.py index 0f87e56033601..30d1c6149379f 100644 --- a/python/paddle/fluid/tests/unittests/hdfs_test_utils.py +++ b/python/paddle/fluid/tests/unittests/hdfs_test_utils.py @@ -15,13 +15,17 @@ import unittest import os -from paddle.distributed.fleet.utils.fs import FSFileExistsError, FSFileNotExistsError, HDFSClient, LocalFS +from paddle.distributed.fleet.utils.fs import ( + FSFileExistsError, + FSFileNotExistsError, + HDFSClient, + LocalFS, +) java_home = os.environ["JAVA_HOME"] class FSTestBase(unittest.TestCase): - def _test_dirs(self, fs): dir_path = os.path.abspath("./test_dir") fs.delete(dir_path) @@ -217,10 +221,12 @@ def _test_rm(self, fs): pass def _test_list_dir(self, fs): - fs = HDFSClient("/usr/local/hadoop-2.7.7/", - None, - time_out=15 * 1000, - sleep_inter=100) + fs = HDFSClient( + "/usr/local/hadoop-2.7.7/", + None, + time_out=15 * 1000, + sleep_inter=100, + ) fs.ls_dir("test_not_exists") def _test_touch(self, fs): diff --git a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py index 2b85788ae56c6..2e02163061fa7 100644 --- a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py +++ b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_alexnet.py @@ -34,7 +34,6 @@ def set_random_seed(seed, dp_id, rank_id): class TestDistPPTraning(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.model_parallel_size = 1 @@ -47,16 +46,17 @@ def setUp(self): } strategy.pipeline_configs = { "accumulate_steps": batch_size // micro_batch_size, - "micro_batch_size": micro_batch_size + "micro_batch_size": micro_batch_size, } fleet.init(is_collective=True, strategy=strategy) def build_optimizer(self, model): - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=[2], - values=[0.001, 0.002], - verbose=True) - optimizer = paddle.optimizer.SGD(learning_rate=scheduler, - parameters=model.parameters()) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=[2], values=[0.001, 0.002], verbose=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=scheduler, parameters=model.parameters() + ) return scheduler, optimizer def test_pp_model(self): @@ -67,7 +67,7 @@ def test_pp_model(self): rank_id = dist.get_rank() set_random_seed(1024, dp_id, rank_id) - #construct model a + # construct model a model_a = AlexNet(10) scheduler_a, optimizer_a = self.build_optimizer(model_a) @@ -87,15 +87,21 @@ def test_pp_model(self): param.set_value(parameters[idx + pp_id * (param_len // 2)]) # construct reader - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size, drop_last=True + ) for step_id, data in enumerate(train_reader()): - x_data = np.array([x[0] for x in data]).astype('float32').reshape( - batch_size, 1, 28, 28) - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(batch_size, 1) + x_data = ( + np.array([x[0] for x in data]) + .astype('float32') + .reshape(batch_size, 1, 28, 28) + ) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(batch_size, 1) + ) img = paddle.to_tensor(x_data) label = paddle.to_tensor(y_data) img.stop_gradient = True @@ -113,9 +119,9 @@ def test_pp_model(self): loss_b = model_b.train_batch([img, label], optimizer_b, scheduler_b) print("loss: ", loss_a.numpy(), loss_b.numpy()) - np.testing.assert_allclose(loss_a.numpy(), - loss_b.numpy(), - rtol=5e-5) + np.testing.assert_allclose( + loss_a.numpy(), loss_b.numpy(), rtol=5e-5 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py index 7cdce186cf8bc..13badaf944002 100644 --- a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py +++ b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py @@ -23,7 +23,6 @@ class ReshapeHelp(Layer): - def __init__(self, shape): super(ReshapeHelp, self).__init__() self.shape = shape @@ -33,7 +32,6 @@ def forward(self, x): class AlexNet(Layer): - def __init__(self, num_classes=10): super(AlexNet, self).__init__() self.features = Sequential( @@ -64,7 +62,6 @@ def forward(self, x, y): class AlexNetPipe(AlexNet): - def to_layers(self): feat = [self.features[i] for i in range(len(self.features))] loss_fn = [self.reshape_layer, self.classifier] @@ -73,7 +70,6 @@ def to_layers(self): class AlexNetPipeDesc(PipelineLayer): - def __init__(self, num_classes=10, **kwargs): self.num_classes = num_classes decs = [ @@ -93,20 +89,19 @@ def __init__(self, num_classes=10, **kwargs): LayerDesc(ReshapeHelp, shape=[-1, 256]), LayerDesc(nn.Linear, 256, self.num_classes), # classifier ] - super(AlexNetPipeDesc, self).__init__(layers=decs, - loss_fn=nn.CrossEntropyLoss(), - **kwargs) + super(AlexNetPipeDesc, self).__init__( + layers=decs, loss_fn=nn.CrossEntropyLoss(), **kwargs + ) class TestPipeLayerAPI(unittest.TestCase): - def setUp(self): strategy = fleet.DistributedStrategy() self.pipeline_parallel_size = 2 strategy.hybrid_configs = { "dp_degree": 1, "mp_degree": 1, - "pp_degree": self.pipeline_parallel_size + "pp_degree": self.pipeline_parallel_size, } fleet.init(is_collective=True, strategy=strategy) self.hcg = fleet.get_hybrid_communicate_group() @@ -117,9 +112,11 @@ def test_pipelayer_desc(self): def test_pipelayer_sequential(self): init_net = AlexNetPipe() - pipe_model = PipelineLayer(layers=init_net.to_layers(), - num_stages=self.pipeline_parallel_size, - loss_fn=nn.CrossEntropyLoss()) + pipe_model = PipelineLayer( + layers=init_net.to_layers(), + num_stages=self.pipeline_parallel_size, + loss_fn=nn.CrossEntropyLoss(), + ) stage_id = self.hcg.get_stage_id() init_parameters = init_net.parameters() pipe_parameters = pipe_model.parameters() diff --git a/python/paddle/fluid/tests/unittests/interpreter/test_standalone_controlflow.py b/python/paddle/fluid/tests/unittests/interpreter/test_standalone_controlflow.py index 5482eda54467f..39dee27fe4e33 100644 --- a/python/paddle/fluid/tests/unittests/interpreter/test_standalone_controlflow.py +++ b/python/paddle/fluid/tests/unittests/interpreter/test_standalone_controlflow.py @@ -27,32 +27,28 @@ # and new executor twice and check the result. # please override the _get_feeds() and build_prgram() class TestCompatibility(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.iter_run = 4 def _get_feed(self): - """ return the feeds - """ + """return the feeds""" return None def build_program(self): - def true_func(): - return layers.fill_constant(shape=[1, 2], dtype='int32', - value=1), layers.fill_constant( - shape=[2, 3], - dtype='bool', - value=True) + return layers.fill_constant( + shape=[1, 2], dtype='int32', value=1 + ), layers.fill_constant(shape=[2, 3], dtype='bool', value=True) def false_func(): - return layers.fill_constant(shape=[3, 4], dtype='float32', - value=3), layers.fill_constant( - shape=[4, 5], - dtype='int64', - value=2) + return layers.fill_constant( + shape=[3, 4], dtype='float32', value=3 + ), layers.fill_constant(shape=[4, 5], dtype='int64', value=2) main_program = Program() startup_program = Program() @@ -101,14 +97,11 @@ def test_with_feed(self): class TestWhile(TestCompatibility): - def _get_feed(self): - """ return the feeds - """ + """return the feeds""" return None def build_program(self): - def cond(i, ten): return i < ten @@ -119,10 +112,12 @@ def body(i, ten): main_program = paddle.static.default_main_program() startup_program = paddle.static.default_startup_program() with paddle.static.program_guard(main_program, startup_program): - i = paddle.full(shape=[1], fill_value=0, - dtype='int64') # loop counter - ten = paddle.full(shape=[1], fill_value=10, - dtype='int64') # loop length + i = paddle.full( + shape=[1], fill_value=0, dtype='int64' + ) # loop counter + ten = paddle.full( + shape=[1], fill_value=10, dtype='int64' + ) # loop length i, ten = paddle.static.nn.while_loop(cond, body, [i, ten]) exe = paddle.static.Executor(paddle.CPUPlace()) diff --git a/python/paddle/fluid/tests/unittests/interpreter/test_standalone_executor.py b/python/paddle/fluid/tests/unittests/interpreter/test_standalone_executor.py index 9da058dfee6ae..f960ad6b04935 100644 --- a/python/paddle/fluid/tests/unittests/interpreter/test_standalone_executor.py +++ b/python/paddle/fluid/tests/unittests/interpreter/test_standalone_executor.py @@ -30,10 +30,12 @@ class TestDryRun(unittest.TestCase): - def setUp(self): - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.place = core.Place() self.place.set_place(place) @@ -57,7 +59,8 @@ def test_dry_run(self): standaloneexecutor = StandaloneExecutor(self.place, main_program.desc) # test for cost_info cost_info = standaloneexecutor.dry_run( - scope, {"a": np.ones([2, 2], dtype="float32")}) + scope, {"a": np.ones([2, 2], dtype="float32")} + ) self.check_cost_info(cost_info) def check_cost_info(self, cost_info): @@ -103,11 +106,13 @@ def build_program(): class ExecutorStatisticsTestCase(unittest.TestCase): - def setUp(self): self.iter_n = 3 - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.perf_path = './perfstat' def test_parallel_executor_statistics(self): @@ -140,7 +145,8 @@ def run_with_statistics(self, executor=None): with framework._enable_standalone_executor(enable): exe = paddle.static.Executor(self.place) helper_profiler = profiler.Profiler( - targets=[profiler.ProfilerTarget.CPU], scheduler=(1, 2)) + targets=[profiler.ProfilerTarget.CPU], scheduler=(1, 2) + ) helper_profiler.start() for i in range(self.iter_n): exe.run(main_program, fetch_list=fetch_list) @@ -157,11 +163,13 @@ def run_with_statistics(self, executor=None): class MultiStreamModelTestCase(unittest.TestCase): - def setUp(self): self.iter_n = 2 - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) def test_result(self): ground_truths = self.run_test(False) @@ -180,16 +188,19 @@ def run_test(self, use_new_executor=True): outs = [] for i in range(self.iter_n): outs.append( - exe.run(main_program, scope=scope, fetch_list=fetch_list)) + exe.run(main_program, scope=scope, fetch_list=fetch_list) + ) print(outs) return outs class SwitchExecutorInterfaceWithFeed(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.iter_run = 2 def build_program(self, is_double=False): @@ -205,24 +216,27 @@ def build_program(self, is_double=False): return main_program, startup_program, [c] - def _run(self, - feed, - use_str=False, - is_double=False, - add_wrong_fetch=False, - use_compiled=False): + def _run( + self, + feed, + use_str=False, + is_double=False, + add_wrong_fetch=False, + use_compiled=False, + ): paddle.seed(2020) main_program, startup_program, fetch_vars = self.build_program( - is_double) + is_double + ) exe = paddle.static.Executor(self.place) exe.run(startup_program) if use_compiled: main_program = paddle.static.CompiledProgram( - main_program).with_data_parallel(fetch_vars[0].name, - places=[self.place]) + main_program + ).with_data_parallel(fetch_vars[0].name, places=[self.place]) if use_str: # test for fetch name fetch_vars = [x.name for x in fetch_vars] @@ -239,15 +253,13 @@ def _run(self, def run_raw_executor(self, feed, use_compiled=False): with framework._enable_standalone_executor(False): # run construct program 1 - out1 = self._run(feed, - use_str=False, - is_double=False, - use_compiled=use_compiled) + out1 = self._run( + feed, use_str=False, is_double=False, use_compiled=use_compiled + ) # run construct program 2 with same executor - out2 = self._run(feed, - use_str=True, - is_double=True, - use_compiled=use_compiled) + out2 = self._run( + feed, use_str=True, is_double=True, use_compiled=use_compiled + ) return [out1, out2] @@ -303,7 +315,6 @@ def test_empty_program(self): class TestException(unittest.TestCase): - def setUp(self): self.place = paddle.CPUPlace() self.fetch_vars = None @@ -315,10 +326,9 @@ def build_program(self): w = paddle.rand([10, 3]) ids = paddle.static.data(name="id", shape=[5], dtype='int64') data = paddle.static.data(name="data", shape=[3], dtype='float32') - emb = paddle.nn.functional.embedding(x=ids, - weight=w, - sparse=False, - name="embedding") + emb = paddle.nn.functional.embedding( + x=ids, weight=w, sparse=False, name="embedding" + ) emb = emb + data return main_program, startup_program, emb @@ -342,43 +352,52 @@ def run_new_executor(self, feed): return out def test_exception(self): - feed = [{ - 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), - 'data': np.array([1, 2, 3]).astype(np.float32), - }, { - 'id': np.array([1, 2, 3, 4, 11]).astype(np.int64), - 'data': np.array([1, 2, 3]).astype(np.float32), - }] + feed = [ + { + 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), + 'data': np.array([1, 2, 3]).astype(np.float32), + }, + { + 'id': np.array([1, 2, 3, 4, 11]).astype(np.int64), + 'data': np.array([1, 2, 3]).astype(np.float32), + }, + ] self.assertRaises(ValueError, self.run_new_executor, feed) def test_nan(self): flags = {'FLAGS_check_nan_inf': True, 'FLAGS_benchmark': True} paddle.fluid.set_flags(flags) - feed = [{ - 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), - 'data': np.array([1, 2, 3]).astype(np.float32), - }, { - 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), - 'data': np.array([1, 2, 3]).astype(np.float32), - }] + feed = [ + { + 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), + 'data': np.array([1, 2, 3]).astype(np.float32), + }, + { + 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), + 'data': np.array([1, 2, 3]).astype(np.float32), + }, + ] feed[1]['data'][0] = np.nan self.assertRaises(RuntimeError, self.run_new_executor, feed) def test_scope_find_temp_var(self): - feed = [{ - 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), - 'data': np.array([1, 2, 3]).astype(np.float32), - }, { - 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), - 'data': np.array([2, 2, 2]).astype(np.float32), - }] + feed = [ + { + 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), + 'data': np.array([1, 2, 3]).astype(np.float32), + }, + { + 'id': np.array([1, 2, 3, 4, 5]).astype(np.int64), + 'data': np.array([2, 2, 2]).astype(np.float32), + }, + ] self.run_new_executor(feed) - self.assertIsNone(paddle.static.global_scope().find_var( - self.fetch_vars.name)) + self.assertIsNone( + paddle.static.global_scope().find_var(self.fetch_vars.name) + ) class TestInplaceApiWithDataTransform(unittest.TestCase): - def test_increment(self): if paddle.fluid.core.is_compiled_with_cuda(): with paddle.fluid.device_guard("gpu:0"): @@ -389,8 +408,9 @@ def test_increment(self): with framework._enable_standalone_executor(): for i in range(10): - a, = exe.run(paddle.static.default_main_program(), - fetch_list=[x]) + (a,) = exe.run( + paddle.static.default_main_program(), fetch_list=[x] + ) self.assertEqual(a[0], 1) diff --git a/python/paddle/fluid/tests/unittests/interpreter/test_standalone_multiply_write.py b/python/paddle/fluid/tests/unittests/interpreter/test_standalone_multiply_write.py index 0a707d1683fd1..e5e44b3a28b78 100644 --- a/python/paddle/fluid/tests/unittests/interpreter/test_standalone_multiply_write.py +++ b/python/paddle/fluid/tests/unittests/interpreter/test_standalone_multiply_write.py @@ -22,19 +22,17 @@ class TestMultiplyWrite(TestCompatibility): - def _get_feed(self): - """ return the feeds - """ + """return the feeds""" return None def build_program(self): main_program = Program() startup_program = Program() with paddle.static.program_guard(main_program, startup_program): - out = paddle.full((1, ), 1) - inp1 = paddle.full((1, ), 2) - inp2 = paddle.full((1, ), 3) + out = paddle.full((1,), 1) + inp1 = paddle.full((1,), 2) + inp2 = paddle.full((1,), 3) paddle.fluid.layers.assign(inp1, out) paddle.fluid.layers.assign(inp2, out) diff --git a/python/paddle/fluid/tests/unittests/ipu/custom_ops/deprecated/test_custom_nllloss_ipu.py b/python/paddle/fluid/tests/unittests/ipu/custom_ops/deprecated/test_custom_nllloss_ipu.py index 9ae7b307ca543..55a9b1338ab9b 100644 --- a/python/paddle/fluid/tests/unittests/ipu/custom_ops/deprecated/test_custom_nllloss_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/custom_ops/deprecated/test_custom_nllloss_ipu.py @@ -22,22 +22,23 @@ from paddle.utils.cpp_extension import load sys.path.append( - os.path.dirname(os.path.dirname(os.path.dirname( - os.path.abspath(__file__))))) + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) from op_test_ipu import IPUOpTest def load_custom_ops(): cur_dir = os.path.dirname(os.path.realpath(__file__)) - custom_ops = load(name="custom_nll_loss", - sources=[f"{cur_dir}/custom_nllloss.cc"], - extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], - extra_ldflags=['-lpopfloat']) + custom_ops = load( + name="custom_nll_loss", + sources=[f"{cur_dir}/custom_nllloss.cc"], + extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], + extra_ldflags=['-lpopfloat'], + ) return custom_ops class TestBase(IPUOpTest): - def setUp(self): self.load_custom_ops() self.set_atol() @@ -75,12 +76,12 @@ def set_feed_attr(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) out = self.op(x, label, **self.op_attrs) out = paddle.mean(out) self.fetch_list = [out.name] @@ -96,7 +97,6 @@ def test(self): class TestCase1(TestBase): - def set_test_op(self): self.op = self.custom_ops.custom_nll_loss self.op_attrs = { diff --git a/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_checkpointoutput_ipu.py b/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_checkpointoutput_ipu.py index 698cef211db66..866c69ffbdd07 100644 --- a/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_checkpointoutput_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_checkpointoutput_ipu.py @@ -27,16 +27,17 @@ def load_custom_ops(): cur_dir = os.path.dirname(os.path.realpath(__file__)) - custom_ops = load(name="checkpointoutput", - sources=[ - f"{cur_dir}/custom_checkpointoutput.cc", - ], - extra_cxx_cflags=['-DONNX_NAMESPACE=onnx']) + custom_ops = load( + name="checkpointoutput", + sources=[ + f"{cur_dir}/custom_checkpointoutput.cc", + ], + extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], + ) return custom_ops class TestCheckpointoutput(IPUOpTest): - def setUp(self): self.load_custom_ops() self.set_atol() @@ -66,9 +67,9 @@ def set_feed_attr(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.add(x, x) x = self.op(x, **self.op_attrs) x = paddle.mean(x) diff --git a/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_leaky_relu_ipu.py b/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_leaky_relu_ipu.py index 70d51940865cf..bb54c0e40da32 100644 --- a/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_leaky_relu_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_leaky_relu_ipu.py @@ -29,17 +29,18 @@ def load_custom_ops(): # load custom ops cur_dir = os.path.dirname(os.path.realpath(__file__)) - custom_ops = load(name="custom_jit_ops", - sources=[ - f"{cur_dir}/leaky_relu_cpu.cc", - f"{cur_dir}/leaky_relu_ipu.cc", - ], - extra_cxx_cflags=['-DONNX_NAMESPACE=onnx']) + custom_ops = load( + name="custom_jit_ops", + sources=[ + f"{cur_dir}/leaky_relu_cpu.cc", + f"{cur_dir}/leaky_relu_ipu.cc", + ], + extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], + ) return custom_ops class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -49,8 +50,9 @@ def setUp(self): def set_feed(self): self.feed = { - "x": np.random.uniform(low=-2, high=2, size=[3, - 5]).astype('float32'), + "x": np.random.uniform(low=-2, high=2, size=[3, 5]).astype( + 'float32' + ), } def set_feed_attr(self): @@ -74,9 +76,11 @@ def _test_base(self, run_ipu=True): with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) # custom op out = custom_ops.custom_leaky_relu(x, **self.attrs) fetch_list = [out.name] @@ -96,14 +100,16 @@ def _test_base(self, run_ipu=True): # add name mapping for paddle custom op and popart custom ops # `paddle_op` was defined in leaky_relu_cpu.cc # `popart_op`, `domain` and `version` was defined in leaky_relu_ipu.cc - ipu_strategy.add_custom_op(paddle_op="custom_leaky_relu", - popart_op="LeakyRelu", - domain='custom.ops', - version=1) + ipu_strategy.add_custom_op( + paddle_op="custom_leaky_relu", + popart_op="LeakyRelu", + domain='custom.ops', + version=1, + ) program = paddle.static.IpuCompiledProgram( - main_prog, scope=scope, - ipu_strategy=ipu_strategy).compile(feed_list, fetch_list) + main_prog, scope=scope, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) else: program = main_prog @@ -114,10 +120,9 @@ def test_base(self): res0 = self._test_base(False) res1 = self._test_base(True) - np.testing.assert_allclose(res0.flatten(), - res1.flatten(), - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + res0.flatten(), res1.flatten(), rtol=1e-05, atol=self.atol + ) self.assertTrue(res0.shape == res1.shape) diff --git a/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_ops_ipu.py b/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_ops_ipu.py index 744ea86663a7c..9dc405dca1467 100644 --- a/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_ops_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/custom_ops/test_custom_ops_ipu.py @@ -28,56 +28,67 @@ # just load one custom-op for the data race issue under parallel mode def load_custom_detach(): cur_dir = os.path.dirname(os.path.realpath(__file__)) - custom_ops = load(name="custom_detach", - sources=[ - f"{cur_dir}/custom_detach.cc", - ], - extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], - extra_ldflags=['-lpopfloat']) + custom_ops = load( + name="custom_detach", + sources=[ + f"{cur_dir}/custom_detach.cc", + ], + extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], + extra_ldflags=['-lpopfloat'], + ) return custom_ops def load_custom_identity(): cur_dir = os.path.dirname(os.path.realpath(__file__)) - custom_ops = load(name="custom_identity", - sources=[ - f"{cur_dir}/custom_identity.cc", - ], - extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], - extra_ldflags=['-lpopfloat']) + custom_ops = load( + name="custom_identity", + sources=[ + f"{cur_dir}/custom_identity.cc", + ], + extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], + extra_ldflags=['-lpopfloat'], + ) return custom_ops def load_custom_nll(): cur_dir = os.path.dirname(os.path.realpath(__file__)) - custom_ops = load(name="custom_nll", - sources=[ - f"{cur_dir}/custom_nll.cc", - ], - extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], - extra_ldflags=['-lpopfloat']) + custom_ops = load( + name="custom_nll", + sources=[ + f"{cur_dir}/custom_nll.cc", + ], + extra_cxx_cflags=['-DONNX_NAMESPACE=onnx'], + extra_ldflags=['-lpopfloat'], + ) return custom_ops def build_ipu_strategy(): ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.add_custom_op(paddle_op="custom_detach", - popart_op="Detach", - domain="ai.graphcore", - version=1) - ipu_strategy.add_custom_op(paddle_op="custom_identity", - popart_op="Identity", - domain="ai.onnx", - version=11) - ipu_strategy.add_custom_op(paddle_op="custom_nll", - popart_op="Nll", - domain="ai.graphcore", - version=1) + ipu_strategy.add_custom_op( + paddle_op="custom_detach", + popart_op="Detach", + domain="ai.graphcore", + version=1, + ) + ipu_strategy.add_custom_op( + paddle_op="custom_identity", + popart_op="Identity", + domain="ai.onnx", + version=11, + ) + ipu_strategy.add_custom_op( + paddle_op="custom_nll", + popart_op="Nll", + domain="ai.graphcore", + version=1, + ) return ipu_strategy class TestBase(IPUOpTest): - def setUp(self): self.load_custom_ops() self.set_atol() @@ -107,9 +118,9 @@ def set_feed_attr(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = self.op(x, **self.op_attrs) out = paddle.mean(out) self.fetch_list = [out.name] @@ -127,7 +138,6 @@ def test(self): class TestIdentity(TestBase): - def load_custom_ops(self): self.custom_ops = load_custom_identity() @@ -137,7 +147,6 @@ def set_test_op(self): class TestNll(TestBase): - def load_custom_ops(self): self.custom_ops = load_custom_nll() @@ -159,12 +168,12 @@ def set_test_op(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) out = self.op(x, label, **self.op_attrs) out = paddle.mean(out) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_data_parallel_ipu.py b/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_data_parallel_ipu.py index 891aa501c5079..dbeed8e4da5ab 100644 --- a/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_data_parallel_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_data_parallel_ipu.py @@ -27,7 +27,6 @@ @unittest.skip('Disable distributed tests on auto CI.') class TestBase(IPUOpTest): - def set_attrs(self, enable_ipu, optimizer, log, onchip=False, rts=False): self.ipu_options = { "enable_pipelining": True, @@ -38,8 +37,8 @@ def set_attrs(self, enable_ipu, optimizer, log, onchip=False, rts=False): "replicated_graph_count": 2, "location_optimizer": { "on_chip": onchip, - "use_replicated_tensor_sharding": rts - } + "use_replicated_tensor_sharding": rts, + }, } self.cpu_bs = 16 @@ -63,19 +62,17 @@ def test(self): with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - image = paddle.static.data(name='image', - shape=[bs, 3, 10, 10], - dtype='float32') + image = paddle.static.data( + name='image', shape=[bs, 3, 10, 10], dtype='float32' + ) with paddle.static.ipu_shard_guard(index=0, stage=0): - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) with paddle.static.ipu_shard_guard(index=1, stage=1): - conv2 = paddle.static.nn.conv2d(conv1, - num_filters=3, - filter_size=3, - bias_attr=False) + conv2 = paddle.static.nn.conv2d( + conv1, num_filters=3, filter_size=3, bias_attr=False + ) # should consider influence of bs loss = paddle.mean(conv2) @@ -104,28 +101,35 @@ def test(self): ipu_strategy.set_graph_config( num_ipus=2 * self.ipu_options['replicated_graph_count'], is_training=True, - enable_manual_shard=True) + enable_manual_shard=True, + ) ipu_strategy.set_options(self.ipu_options) - ipu_strategy.set_options({ - "enable_distribution": - True, - "enable_distributed_replicated_graphs": - True, - "global_replica_offset": - int(os.environ.get("PADDLE_TRAINER_ID")) * 2, - "global_replication_factor": - 4 - }) + ipu_strategy.set_options( + { + "enable_distribution": True, + "enable_distributed_replicated_graphs": True, + "global_replica_offset": int( + os.environ.get("PADDLE_TRAINER_ID") + ) + * 2, + "global_replication_factor": 4, + } + ) program = paddle.static.IpuCompiledProgram( - main_prog, ipu_strategy=ipu_strategy).compile( - feed_list, fetch_list) + main_prog, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) feed = { - "image": - np.tile(data, [ - self.ipu_options['replicated_graph_count'] * - self.ipu_options['batches_per_step'] * - self.ipu_options['accumulation_factor'], 1, 1, 1 - ]) + "image": np.tile( + data, + [ + self.ipu_options['replicated_graph_count'] + * self.ipu_options['batches_per_step'] + * self.ipu_options['accumulation_factor'], + 1, + 1, + 1, + ], + ) } else: @@ -176,11 +180,13 @@ def _get_comm(): onchip = True if sys.argv[3] == "True" else False rts = True if sys.argv[4] == "True" else False test = TestBase() - test.set_attrs(enable_ipu=True, - optimizer=optimizer, - log=log, - onchip=onchip, - rts=rts) + test.set_attrs( + enable_ipu=True, + optimizer=optimizer, + log=log, + onchip=onchip, + rts=rts, + ) test.test() # Run cpu tests for compare elif len(sys.argv) == 3: diff --git a/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_pod128_sample.py b/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_pod128_sample.py index f81ed48f04ffd..a966f4308228b 100644 --- a/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_pod128_sample.py +++ b/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_pod128_sample.py @@ -71,43 +71,42 @@ def TestDistTraining(): exe.run(startup_prog) ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(num_ipus=64, - is_training=True, - enable_manual_shard=True) + ipu_strategy.set_graph_config( + num_ipus=64, is_training=True, enable_manual_shard=True + ) ipu_strategy.set_pipelining_config( enable_pipelining=True, batches_per_step=1, enable_gradient_accumulation=True, - accumulation_factor=4) - ipu_strategy.set_options({ - "enable_distribution": - True, - "enable_replicated_graphs": - True, - "replicated_graph_count": - 32, - "enable_distributed_replicated_graphs": - True, - "global_replica_offset": - # Paddle : int(os.environ.get("PADDLE_TRAINER_ID")) * 32 - # PopRun : int(os.environ.get("POPDIST_REPLICA_INDEX_OFFSET")) - int(os.environ.get("PADDLE_TRAINER_ID")) * 32, - "global_replication_factor": - 64, - "location_optimizer": { - "on_chip": False, - "use_replicated_tensor_sharding": True + accumulation_factor=4, + ) + ipu_strategy.set_options( + { + "enable_distribution": True, + "enable_replicated_graphs": True, + "replicated_graph_count": 32, + "enable_distributed_replicated_graphs": True, + "global_replica_offset": + # Paddle : int(os.environ.get("PADDLE_TRAINER_ID")) * 32 + # PopRun : int(os.environ.get("POPDIST_REPLICA_INDEX_OFFSET")) + int(os.environ.get("PADDLE_TRAINER_ID")) * 32, + "global_replication_factor": 64, + "location_optimizer": { + "on_chip": False, + "use_replicated_tensor_sharding": True, + }, } - }) + ) ipu_program = paddle.static.IpuCompiledProgram( - main_prog, ipu_strategy=ipu_strategy) + main_prog, ipu_strategy=ipu_strategy + ) program = ipu_program.compile(feed_list, fetch_list) for i in range(10): - res = exe.run(program, - feed={"x": input_data}, - fetch_list=fetch_list) + res = exe.run( + program, feed={"x": input_data}, fetch_list=fetch_list + ) print("index: {}, result: {}".format(i, res)) diff --git a/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_sample.py b/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_sample.py index d42977b5962d3..a3ffc6694a93e 100644 --- a/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_sample.py +++ b/python/paddle/fluid/tests/unittests/ipu/distributed/test_dist_sample.py @@ -91,57 +91,73 @@ def Test(use_dist, file_name): if use_dist: ipu_strategy.set_graph_config(num_ipus=2, is_training=True) # Set distributed envs - ipu_strategy.set_options({ - "enable_distribution": - True, - "enable_replicated_graphs": - True, - "replicated_graph_count": - 2, - "enable_distributed_replicated_graphs": - True, - "global_replica_offset": - int(os.environ.get("PADDLE_TRAINER_ID")) * 2, - "global_replication_factor": - 4 - }) + ipu_strategy.set_options( + { + "enable_distribution": True, + "enable_replicated_graphs": True, + "replicated_graph_count": 2, + "enable_distributed_replicated_graphs": True, + "global_replica_offset": int( + os.environ.get("PADDLE_TRAINER_ID") + ) + * 2, + "global_replication_factor": 4, + } + ) else: ipu_strategy.set_graph_config(num_ipus=4, is_training=True) - ipu_strategy.set_options({ - "enable_replicated_graphs": True, - "replicated_graph_count": 4, - }) + ipu_strategy.set_options( + { + "enable_replicated_graphs": True, + "replicated_graph_count": 4, + } + ) ipu_program = paddle.static.IpuCompiledProgram( - main_prog, ipu_strategy=ipu_strategy) + main_prog, ipu_strategy=ipu_strategy + ) program = ipu_program.compile(feed_list, fetch_list) if use_dist: if os.environ.get("PADDLE_TRAINER_ID") == "0": - input_data = np.concatenate([ - np.array([[[1], [3]], [[2], [4]], - [[4], [127]]]).astype(np.int32), - np.array([[[1], [3]], [[2], [4]], - [[4], [127]]]).astype(np.int32) - ]) + input_data = np.concatenate( + [ + np.array( + [[[1], [3]], [[2], [4]], [[4], [127]]] + ).astype(np.int32), + np.array( + [[[1], [3]], [[2], [4]], [[4], [127]]] + ).astype(np.int32), + ] + ) else: - input_data = np.concatenate([ - np.array([[[8], [60]], [[50], [77]], - [[90], [13]]]).astype(np.int32), - np.array([[[8], [60]], [[50], [77]], - [[90], [13]]]).astype(np.int32) - ]) + input_data = np.concatenate( + [ + np.array( + [[[8], [60]], [[50], [77]], [[90], [13]]] + ).astype(np.int32), + np.array( + [[[8], [60]], [[50], [77]], [[90], [13]]] + ).astype(np.int32), + ] + ) else: - input_data = np.concatenate([ - np.array([[[1], [3]], [[2], [4]], - [[4], [127]]]).astype(np.int32), - np.array([[[1], [3]], [[2], [4]], - [[4], [127]]]).astype(np.int32), - np.array([[[8], [60]], [[50], [77]], - [[90], [13]]]).astype(np.int32), - np.array([[[8], [60]], [[50], [77]], - [[90], [13]]]).astype(np.int32) - ]) + input_data = np.concatenate( + [ + np.array([[[1], [3]], [[2], [4]], [[4], [127]]]).astype( + np.int32 + ), + np.array([[[1], [3]], [[2], [4]], [[4], [127]]]).astype( + np.int32 + ), + np.array( + [[[8], [60]], [[50], [77]], [[90], [13]]] + ).astype(np.int32), + np.array( + [[[8], [60]], [[50], [77]], [[90], [13]]] + ).astype(np.int32), + ] + ) feed_data = {"x": input_data} for step in range(10): diff --git a/python/paddle/fluid/tests/unittests/ipu/op_test_ipu.py b/python/paddle/fluid/tests/unittests/ipu/op_test_ipu.py index 9cb70d59d3e59..d518dec3eb7ab 100644 --- a/python/paddle/fluid/tests/unittests/ipu/op_test_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/op_test_ipu.py @@ -55,7 +55,6 @@ class ExecutionMode(IntEnum): class IPUTest(unittest.TestCase): - @classmethod def setUpClass(cls): # Get random seeds @@ -84,10 +83,10 @@ def use_ipumodel(cls): return True -@unittest.skipIf(not paddle.is_compiled_with_ipu(), - "core is not compiled with IPU") +@unittest.skipIf( + not paddle.is_compiled_with_ipu(), "core is not compiled with IPU" +) class IPUD2STest(IPUTest): - @classmethod def setUpClass(cls): super().setUpClass() @@ -101,11 +100,11 @@ def tearDown(self): paddle.framework.core.IpuBackend.get_instance().reset() -@unittest.skipIf(not paddle.is_compiled_with_ipu(), - "core is not compiled with IPU") +@unittest.skipIf( + not paddle.is_compiled_with_ipu(), "core is not compiled with IPU" +) class IPUOpTest(IPUTest): - """Base Class for single op unit tests using static graph on IPU. - """ + """Base Class for single op unit tests using static graph on IPU.""" @classmethod def setUpClass(cls): @@ -159,7 +158,6 @@ def set_training(self): # Decorator for static graph building def static_graph(builder): - def wrapper(self, *args, **kwargs): self.scope = paddle.static.Scope() self.main_prog = paddle.static.Program() @@ -168,9 +166,11 @@ def wrapper(self, *args, **kwargs): self.startup_prog.random_seed = self.SEED with paddle.static.scope_guard(self.scope): with paddle.utils.unique_name.guard( - paddle.utils.unique_name.generate('')): - with paddle.static.program_guard(self.main_prog, - self.startup_prog): + paddle.utils.unique_name.generate('') + ): + with paddle.static.program_guard( + self.main_prog, self.startup_prog + ): builder(self, *args, **kwargs) return wrapper @@ -181,11 +181,11 @@ def cast_model_to_fp16(cls, main_program): amp_list = paddle.static.amp.CustomOpLists() amp_list.unsupported_list = {'scale'} to_fp16_var_names = paddle.static.amp.cast_model_to_fp16( - main_program, amp_list, use_fp16_guard=False) + main_program, amp_list, use_fp16_guard=False + ) paddle.static.amp.cast_parameters_to_fp16( - paddle.CPUPlace(), - main_program, - to_fp16_var_names=to_fp16_var_names) + paddle.CPUPlace(), main_program, to_fp16_var_names=to_fp16_var_names + ) def run_op_test(self, exec_mode, ipu_strategy=None): # NOTE: some op has no inputs @@ -208,14 +208,12 @@ def run_op_test(self, exec_mode, ipu_strategy=None): # TODO(ipu) remove in the future version of popart # keep the log clean, no side effects for tests without profiling ipu_strategy.set_options( - {'engine_options': { - 'debug.retainDebugInformation': 'false' - }}) + {'engine_options': {'debug.retainDebugInformation': 'false'}} + ) program = paddle.static.IpuCompiledProgram( - self.main_prog, - ipu_strategy=ipu_strategy).compile(self.feed_list, - self.fetch_list) + self.main_prog, ipu_strategy=ipu_strategy + ).compile(self.feed_list, self.fetch_list) else: program = self.main_prog @@ -226,9 +224,9 @@ def run_op_test(self, exec_mode, ipu_strategy=None): if self.is_training: result = [] for _ in range(self.epoch): - loss_res = exe.run(program, - feed=feed, - fetch_list=self.fetch_list) + loss_res = exe.run( + program, feed=feed, fetch_list=self.fetch_list + ) result.append(loss_res) else: result = exe.run(program, feed=feed, fetch_list=self.fetch_list) @@ -253,16 +251,16 @@ def check(self, check_shape=False, output_dict=None): for cpu_fp32_res, ipu_fp32_res in zip(cpu_fp32, ipu_fp32): cpu_fp32_res = np.asarray(cpu_fp32_res).astype(np.float32).flatten() ipu_fp32_res = np.asarray(ipu_fp32_res).astype(np.float32).flatten() - pass_check = np.allclose(ipu_fp32_res, - cpu_fp32_res, - rtol=self.rtol, - atol=self.atol) + pass_check = np.allclose( + ipu_fp32_res, cpu_fp32_res, rtol=self.rtol, atol=self.atol + ) if not pass_check: max_atol = np.abs(ipu_fp32_res - cpu_fp32_res).max() cpu_fp32_abs = np.abs(cpu_fp32_res) cpu_fp32_abs[cpu_fp32_abs == 0.0] = 1e-20 - max_rtol = (np.abs(ipu_fp32_res - cpu_fp32_res) / - cpu_fp32_abs).max() + max_rtol = ( + np.abs(ipu_fp32_res - cpu_fp32_res) / cpu_fp32_abs + ).max() raise AssertionError( f"ipu_fp32 check failed. max_atol is {max_atol}, max_rtol is {max_rtol}" ) @@ -274,22 +272,28 @@ def check(self, check_shape=False, output_dict=None): ipu_fp16 = output_dict[ExecutionMode.IPU_FP16] if len(cpu_fp32) != len(ipu_fp16): raise ValueError( - "different outputs number between ipu and cpu.") + "different outputs number between ipu and cpu." + ) for cpu_fp32_res, ipu_fp16_res in zip(cpu_fp32, ipu_fp16): - cpu_fp32_res = np.asarray(cpu_fp32_res).astype( - np.float32).flatten() - ipu_fp16_res = np.asarray(ipu_fp16_res).astype( - np.float32).flatten() - pass_check = np.allclose(ipu_fp16_res, - cpu_fp32_res, - rtol=self.rtol_fp16, - atol=self.atol_fp16) + cpu_fp32_res = ( + np.asarray(cpu_fp32_res).astype(np.float32).flatten() + ) + ipu_fp16_res = ( + np.asarray(ipu_fp16_res).astype(np.float32).flatten() + ) + pass_check = np.allclose( + ipu_fp16_res, + cpu_fp32_res, + rtol=self.rtol_fp16, + atol=self.atol_fp16, + ) if not pass_check: max_atol = np.abs(ipu_fp16_res - cpu_fp32_res).max() cpu_fp32_abs = np.abs(cpu_fp32_res) cpu_fp32_abs[cpu_fp32_abs == 0.0] = 1e-20 - max_rtol = (np.abs(ipu_fp16_res - cpu_fp32_res) / - cpu_fp32_abs).max() + max_rtol = ( + np.abs(ipu_fp16_res - cpu_fp32_res) / cpu_fp32_abs + ).max() raise AssertionError( f"ipu_fp16 check failed. max_atol is {max_atol}, max_rtol is {max_rtol}" ) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_activation_ops_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_activation_ops_ipu.py index 97ee7a45e001c..672195469d2b2 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_activation_ops_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_activation_ops_ipu.py @@ -22,7 +22,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_test_op() @@ -47,9 +46,9 @@ def set_feed_attr(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = self.op(x, **self.op_attrs) self.fetch_list = [out.name] @@ -65,7 +64,6 @@ def test(self): class TestBReluCase0(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[1, 3, 10, 10]) * 30 self.feed_fp32 = {'in_0': data.astype(np.float32)} @@ -78,35 +76,30 @@ def set_test_op(self): class TestBReluCase1(TestBReluCase0): - def set_test_op(self): self.op = paddle.fluid.layers.brelu self.op_attrs = {"t_min": 0.1, 't_max': 10.0} class TestEluCase1(TestBase): - def set_test_op(self): self.op = F.elu self.op_attrs = {"alpha": 0.3} class TestHardShrinkCase0(TestBase): - def set_test_op(self): self.op = F.hardshrink self.op_attrs = {} class TestHardSigmoidCase0(TestBase): - def set_test_op(self): self.op = F.hardsigmoid self.op_attrs = {} class TestHardSigmoidCase1(TestBase): - def set_test_op(self): self.op = F.hardsigmoid self.op_attrs = { @@ -116,126 +109,108 @@ def set_test_op(self): class TestHardSwishCase0(TestBase): - def set_test_op(self): self.op = F.hardswish self.op_attrs = {} class TestLeakyReluCase0(TestBase): - def set_test_op(self): self.op = F.leaky_relu self.op_attrs = {} class TestLeakyReluCase1(TestBase): - def set_test_op(self): self.op = F.leaky_relu self.op_attrs = {'negative_slope': 0.2333} class TestLog10Case0(TestBase): - def set_test_op(self): self.op = paddle.log10 self.op_attrs = {} class TestLog1pCase0(TestBase): - def set_test_op(self): self.op = paddle.log1p self.op_attrs = {} class TestLog2Case0(TestBase): - def set_test_op(self): self.op = paddle.log2 self.op_attrs = {} class TestLogSigmoidCase0(TestBase): - def set_test_op(self): self.op = F.log_sigmoid self.op_attrs = {} class TestLogSoftmaxCase0(TestBase): - def set_test_op(self): self.op = F.log_softmax self.op_attrs = {} class TestMishCase0(TestBase): - def set_test_op(self): self.op = F.mish self.op_attrs = {} class TestRelu6Case0(TestBase): - def set_test_op(self): self.op = F.relu6 self.op_attrs = {} class TestRsqrtCase0(TestBase): - def set_test_op(self): self.op = paddle.rsqrt self.op_attrs = {} class TestSeluCase0(TestBase): - def set_test_op(self): self.op = F.selu self.op_attrs = {} class TestSiluCase0(TestBase): - def set_test_op(self): self.op = F.silu self.op_attrs = {} class TestSoftShrinkCase0(TestBase): - def set_test_op(self): self.op = F.softshrink self.op_attrs = {} class TestSoftShrinkCase1(TestBase): - def set_test_op(self): self.op = F.softshrink self.op_attrs = {'threshold': 0.2333} class TestSquareCase0(TestBase): - def set_test_op(self): self.op = paddle.square self.op_attrs = {} class TestSwishCase0(TestBase): - def set_test_op(self): self.op = F.swish self.op_attrs = {} class TestTanhShrinkCase0(TestBase): - def set_atol(self): super().set_atol() self.atol = 1e-7 @@ -246,14 +221,12 @@ def set_test_op(self): class TestThresholdedReluCase0(TestBase): - def set_test_op(self): self.op = F.thresholded_relu self.op_attrs = {} class TestThresholdedReluCase1(TestBase): - def set_test_op(self): self.op = F.thresholded_relu self.op_attrs = {'threshold': 0.2333} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_affine_channel_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_affine_channel_op_ipu.py index 836b99099ffe0..0e76225c5618d 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_affine_channel_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_affine_channel_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -48,16 +47,18 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - data = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + data = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) input_scale = paddle.fluid.layers.create_parameter( - shape=[self.feed_shape[0][1]], dtype="float32") + shape=[self.feed_shape[0][1]], dtype="float32" + ) input_bias = paddle.fluid.layers.create_parameter( - shape=[self.feed_shape[0][1]], dtype="float32") - out = paddle.fluid.layers.affine_channel(data, - scale=input_scale, - bias=input_bias) + shape=[self.feed_shape[0][1]], dtype="float32" + ) + out = paddle.fluid.layers.affine_channel( + data, scale=input_scale, bias=input_bias + ) self.fetch_list = [out.name] def run_model(self, exec_mode): @@ -72,7 +73,6 @@ def test(self): class TestCase1(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[2, 4, 64, 64]) self.feed_fp32 = {'data': data.astype(np.float32)} @@ -81,7 +81,6 @@ def set_data_feed(self): @unittest.skip("Only support NCHW") class TestNHWC(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['data_layout'] = 'NHWC' diff --git a/python/paddle/fluid/tests/unittests/ipu/test_arg_max_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_arg_max_op_ipu.py index 078e744ae507d..181ae2bfbc759 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_arg_max_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_arg_max_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.argmax(x, **self.attrs) self.fetch_list = [out.name] @@ -64,7 +63,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axis": 0} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_arg_min_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_arg_min_op_ipu.py index 30c604901e877..9c1de175dafd3 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_arg_min_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_arg_min_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.argmin(x, **self.attrs) self.fetch_list = [out.name] @@ -64,7 +63,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axis": 0} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_argsort_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_argsort_op_ipu.py index 3f19da43c71c3..0f6c28c82cdb4 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_argsort_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_argsort_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -47,9 +46,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out, _ = paddle.fluid.layers.argsort(x, **self.attrs) self.fetch_list = [out.name] @@ -67,7 +66,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'axis': 0, @@ -76,7 +74,6 @@ def set_op_attrs(self): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { 'axis': 1, diff --git a/python/paddle/fluid/tests/unittests/ipu/test_assign_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_assign_op_ipu.py index 93cdaf018b400..08b40fc23c803 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_assign_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_assign_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -39,9 +38,9 @@ def set_feed_attr(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.assign(x) out = paddle.fluid.layers.elementwise_add(x, x) self.fetch_list = [out.name] @@ -58,7 +57,6 @@ def test(self): class TestAssignFp32Value(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[2, 3, 1]) self.feed_fp32 = {'in_0': data.astype(np.float32)} @@ -69,16 +67,15 @@ def set_data_feed(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) assign = paddle.assign(self.assign_fp32) out = paddle.fluid.layers.elementwise_add(x, assign) self.fetch_list = [out.name] class TestAssignBoolValue(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[2, 3, 1]) self.feed_fp32 = {'in_0': data.astype(np.float32)} @@ -88,9 +85,9 @@ def set_data_feed(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.less_than(x, x) assign = paddle.assign(self.assign_bool) x = paddle.logical_and(x, assign) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_avg_shard_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_avg_shard_ipu.py index a3be5458ad83f..1834104606e4c 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_avg_shard_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_avg_shard_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,25 +44,21 @@ def set_feed_attr(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - x = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) - x = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) - x = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) - x = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + x = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) + x = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) + x = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) + x = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) self.fetch_list = [x.name] def run_model(self, exec_mode): diff --git a/python/paddle/fluid/tests/unittests/ipu/test_batch_norm_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_batch_norm_op_ipu.py index 08e5049a790eb..f8a45e0bd2ab3 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_batch_norm_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_batch_norm_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -52,13 +51,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - x = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + x = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) x = paddle.fluid.layers.batch_norm(x, **self.attrs) self.fetch_list = [x.name] @@ -74,7 +72,6 @@ def test(self): class TestCase1(TestBase): - def set_atol(self): self.atol = 1e-6 self.rtol = 1e-6 @@ -89,7 +86,6 @@ def set_op_attrs(self): class TestCase2(TestBase): - def set_atol(self): self.atol = 1e-6 self.rtol = 1e-6 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_binary_cross_entropy_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_binary_cross_entropy_op_ipu.py index 113412b834110..caf28126d2269 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_binary_cross_entropy_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_binary_cross_entropy_op_ipu.py @@ -22,7 +22,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -35,11 +34,11 @@ def set_data_feed(self): target = np.random.uniform(size=[3, 4, 2, 2]) self.feed_fp32 = { "x": x.astype(np.float32), - "target": target.astype(np.float32) + "target": target.astype(np.float32), } self.feed_fp16 = { "x": x.astype(np.float16), - "target": target.astype(np.float16) + "target": target.astype(np.float16), } def set_feed_attr(self): @@ -53,12 +52,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self, on_ipu): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - target = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + target = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = F.binary_cross_entropy(x, target, **self.attrs) self.fetch_list = [out.name] @@ -74,7 +73,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'reduction': 'sum', @@ -82,7 +80,6 @@ def set_op_attrs(self): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { 'reduction': 'none', diff --git a/python/paddle/fluid/tests/unittests/ipu/test_bmm_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_bmm_op_ipu.py index 8ea20cebf07a3..dd1d99a8748c2 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_bmm_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_bmm_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,12 +44,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.bmm(x, y, **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_cast_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_cast_op_ipu.py index 675489d1e4cd5..45b191afed1ec 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_cast_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_cast_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -48,9 +47,11 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) out = paddle.cast(x, **self.attrs) self.fetch_list = [out.name] @@ -66,7 +67,6 @@ def test(self): class TestEnableFp16(TestBase): - @property def fp16_enabled(self): return True @@ -85,7 +85,6 @@ def set_op_attrs(self): class TestCase2(TestBase): - def set_atol(self): super().set_atol() self.atol = 1e-3 @@ -102,7 +101,6 @@ def set_op_attrs(self): class TestCase3(TestBase): - def set_data_feed(self): self.feed_fp32 = { "x": np.random.uniform(size=[1, 3, 3, 3]).astype('float32'), @@ -114,7 +112,6 @@ def set_op_attrs(self): class TestCase4(TestBase): - def set_data_feed(self): self.feed_fp32 = { "x": np.random.uniform(size=[1, 3, 3, 3]).astype('int32'), @@ -126,7 +123,6 @@ def set_op_attrs(self): class TestCase5(TestBase): - def set_data_feed(self): self.feed_fp32 = { "x": np.random.uniform(size=[1, 3, 3, 3]).astype('float16'), @@ -138,7 +134,6 @@ def set_op_attrs(self): class TestCase6(TestBase): - def set_data_feed(self): self.feed_fp32 = { "x": np.random.uniform(size=[1, 3, 3, 3]).astype('int32'), @@ -151,7 +146,6 @@ def set_op_attrs(self): @unittest.skip('float64 is not supported') class TestCase7(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['dtype'] = 'float64' @@ -159,7 +153,6 @@ def set_op_attrs(self): @unittest.skip('skip float16 to float32') class TestCase8(TestBase): - def set_data_feed(self): self.feed_fp32 = { "x": np.random.uniform(size=[1, 3, 3, 3]).astype('float16'), @@ -172,16 +165,15 @@ def set_op_attrs(self): @unittest.skip('int32 to int8 is not supported') class TestCase9(TestBase): - def set_atol(self): super().set_atol() self.atol = 1 def set_data_feed(self): self.feed_fp32 = { - "x": - np.random.randint(low=1, high=100, size=[1, 3, 3, - 3]).astype('int32'), + "x": np.random.randint(low=1, high=100, size=[1, 3, 3, 3]).astype( + 'int32' + ), } def set_op_attrs(self): diff --git a/python/paddle/fluid/tests/unittests/ipu/test_clip_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_clip_op_ipu.py index a221ad617671d..a570024eb7024 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_clip_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_clip_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -48,9 +47,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.clip(x, **self.attrs) self.fetch_list = [x.name] @@ -66,111 +65,96 @@ def test(self): class TestNoMin(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['max'] = 3.4 class TestNoMax(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['min'] = 0.1 class TestNoMinNoMax(TestBase): - def set_op_attrs(self): self.attrs = {} class TestMinMaxTensor(TestBase): - @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - - min = paddle.fluid.layers.fill_constant(name="min", - shape=[1], - dtype='float32', - value=0.1) - max = paddle.fluid.layers.fill_constant(name="max", - shape=[1], - dtype='float32', - value=3.4) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + + min = paddle.fluid.layers.fill_constant( + name="min", shape=[1], dtype='float32', value=0.1 + ) + max = paddle.fluid.layers.fill_constant( + name="max", shape=[1], dtype='float32', value=3.4 + ) x = paddle.clip(x, min=min, max=max) self.fetch_list = [x.name] class TestMinTensor(TestBase): - @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - - min = paddle.fluid.layers.fill_constant(name="min", - shape=[1], - dtype='float32', - value=0.1) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + + min = paddle.fluid.layers.fill_constant( + name="min", shape=[1], dtype='float32', value=0.1 + ) x = paddle.clip(x, min=min) self.fetch_list = [x.name] class TestMaxTensor(TestBase): - @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - - max = paddle.fluid.layers.fill_constant(name="max", - shape=[1], - dtype='float32', - value=3.4) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + + max = paddle.fluid.layers.fill_constant( + name="max", shape=[1], dtype='float32', value=3.4 + ) x = paddle.clip(x, max=max) self.fetch_list = [x.name] class TestCombine1(TestBase): - @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - - min = paddle.fluid.layers.fill_constant(name="min", - shape=[1], - dtype='float32', - value=0.1) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + + min = paddle.fluid.layers.fill_constant( + name="min", shape=[1], dtype='float32', value=0.1 + ) x = paddle.clip(x, min=min, max=3.4) self.fetch_list = [x.name] class TestCombine2(TestBase): - @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - - max = paddle.fluid.layers.fill_constant(name="max", - shape=[1], - dtype='float32', - value=3.4) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + + max = paddle.fluid.layers.fill_constant( + name="max", shape=[1], dtype='float32', value=3.4 + ) x = paddle.clip(x, min=0.1, max=max) self.fetch_list = [x.name] class TestIntInput(TestBase): - def set_feed(self): data = np.random.uniform(size=[5, 5]) self.feed_fp32 = {'x': data.astype(np.int32)} @@ -180,16 +164,15 @@ def set_feed(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='int32' + ) x = paddle.clip(x, min=0.1, max=3.4) self.fetch_list = [x.name] class TestIntMinMax(TestBase): - def set_feed(self): data = np.random.uniform(size=[5, 5]) self.feed_fp32 = {'x': data.astype(np.int32)} @@ -199,17 +182,15 @@ def set_feed(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='int32') - min = paddle.fluid.layers.fill_constant(name="min", - shape=[1], - dtype='int32', - value=1) - max = paddle.fluid.layers.fill_constant(name="max", - shape=[1], - dtype='int32', - value=3) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='int32' + ) + min = paddle.fluid.layers.fill_constant( + name="min", shape=[1], dtype='int32', value=1 + ) + max = paddle.fluid.layers.fill_constant( + name="max", shape=[1], dtype='int32', value=3 + ) x = paddle.clip(x, min=min, max=max) self.fetch_list = [x.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_concat_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_concat_op_ipu.py index 733a5291cf50b..588cae77fec33 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_concat_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_concat_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -34,11 +33,11 @@ def set_data_feed(self): data2 = np.random.uniform(size=[1, 3, 10, 10]) self.feed_fp32 = { 'x': data1.astype(np.float32), - 'y': data2.astype(np.float32) + 'y': data2.astype(np.float32), } self.feed_fp16 = { 'x': data1.astype(np.float16), - 'y': data2.astype(np.float16) + 'y': data2.astype(np.float16), } def set_feed_attr(self): @@ -50,12 +49,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.concat([x, y], **self.attrs) self.fetch_list = [out.name] @@ -71,7 +70,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axis": 1} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_conv2d_transpose_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_conv2d_transpose_op_ipu.py index 824b6b628defa..af1e74bfd124c 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_conv2d_transpose_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_conv2d_transpose_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -52,9 +51,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.static.nn.conv2d_transpose(x, **self.attrs) self.fetch_list = [x.name] @@ -70,7 +69,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['stride'] = 2 @@ -78,7 +76,6 @@ def set_op_attrs(self): @unittest.skip("Only support dilation=1") class TestCase2(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['stride'] = 2 @@ -86,21 +83,18 @@ def set_op_attrs(self): class TestCase3(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['padding'] = 2 class TestCase4(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['padding'] = "SAME" class TestCase5(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['stride'] = 2 @@ -108,14 +102,12 @@ def set_op_attrs(self): class TestCase6(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['padding'] = "VALID" class TestCase7(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['padding'] = "VALID" @@ -123,7 +115,6 @@ def set_op_attrs(self): class TestCase8(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['filter_size'] = 4 @@ -149,7 +140,6 @@ def set_op_attrs(self): class TestCase11(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['groups'] = 3 @@ -157,17 +147,16 @@ def set_op_attrs(self): # depthwise_conv2d_transpose Op class TestCase12(TestBase): - def set_feed(self): data = np.random.uniform(size=[1, 3, 10, 10]) weight = np.random.uniform(size=[3, 1, 3, 3]) self.feed_fp32 = { 'in_0': data.astype(np.float32), - 'in_1': weight.astype(np.float32) + 'in_1': weight.astype(np.float32), } self.feed_fp16 = { 'in_0': data.astype(np.float16), - 'in_1': weight.astype(np.float16) + 'in_1': weight.astype(np.float16), } self.feed_shape = [x.shape for x in self.feed_fp32.values()] self.feed_list = list(self.feed_fp32.keys()) @@ -178,12 +167,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - weight = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + weight = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) x = paddle.nn.functional.conv2d_transpose(x, weight, **self.attrs) self.fetch_list = [x.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_conv_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_conv_op_ipu.py index 076d51a5e6713..89474001e781a 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_conv_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_conv_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -53,9 +52,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.fluid.layers.conv2d(x, **self.attrs) self.fetch_list = [x.name] @@ -71,35 +70,30 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['num_filters'] = 1 class TestCase2(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['filter_size'] = [3, 3] class TestCase2_1(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['filter_size'] = [3, 2] class TestCase3(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['stride'] = [2, 3] class TestCase4(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['dilation'] = [2, 2] @@ -113,21 +107,18 @@ def set_op_attrs(self): class TestCase6(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['padding'] = 2 class TestCase7(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['padding'] = [2, 3] class TestCase8(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['padding'] = [1, 2, 2, 3] @@ -135,17 +126,16 @@ def set_op_attrs(self): # depthwise_conv2d Op class TestCase9(TestBase): - def set_feed(self): data = np.random.uniform(size=[1, 3, 10, 10]) weight = np.random.uniform(size=[3, 1, 3, 3]) self.feed_fp32 = { 'in_0': data.astype(np.float32), - 'in_1': weight.astype(np.float32) + 'in_1': weight.astype(np.float32), } self.feed_fp16 = { 'in_0': data.astype(np.float16), - 'in_1': weight.astype(np.float16) + 'in_1': weight.astype(np.float16), } self.feed_shape = [x.shape for x in self.feed_fp32.values()] self.feed_list = list(self.feed_fp32.keys()) @@ -156,12 +146,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - weight = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + weight = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) x = paddle.nn.functional.conv2d(x, weight, **self.attrs) self.fetch_list = [x.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_cross_entropy2_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_cross_entropy2_op_ipu.py index 92cf442fe27cc..b7db271775dbd 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_cross_entropy2_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_cross_entropy2_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -34,11 +33,11 @@ def set_data_feed(self): label = np.arange(3).reshape([3, 1]) self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } def set_feed_attr(self): @@ -52,20 +51,20 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self, on_ipu): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) if on_ipu: - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) else: - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int64') - out = paddle.fluid.layers.cross_entropy(input=x, - label=label, - **self.attrs) + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int64' + ) + out = paddle.fluid.layers.cross_entropy( + input=x, label=label, **self.attrs + ) self.fetch_list = [out.name] def run_model(self, exec_mode): @@ -82,7 +81,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'soft_label': False, @@ -91,23 +89,21 @@ def set_op_attrs(self): class TestCase2(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[30, 70]) label = np.arange(30).reshape([30, 1]) self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } @unittest.skip("soft_label=True is not supported") class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = { 'soft_label': True, @@ -115,32 +111,30 @@ def set_op_attrs(self): class TestCase4(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3, 5, 7]) label = np.random.randint(0, 7, [3, 5, 1], dtype='int64') self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } class TestCase5(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3, 5, 6, 7]) label = np.random.randint(0, 7, [3, 5, 6], dtype='int64') self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_cumsum_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_cumsum_op_ipu.py index 5f859b064feac..d31a78e1230d9 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_cumsum_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_cumsum_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -49,9 +48,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) out = paddle.fluid.layers.cumsum(x, **self.attrs) self.fetch_list = [out.name] @@ -67,49 +66,44 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"exclusive": True, "reverse": False} class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"exclusive": False, "reverse": True} class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = {"exclusive": True, "reverse": True} class TestCase4(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[1, 128]) self.feed_fp32 = {"x": x.astype(np.int32)} @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="int32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="int32" + ) out = paddle.fluid.layers.cumsum(x, **self.attrs) self.fetch_list = [out.name] class TestCase5(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[1, 128]) self.feed_fp32 = {"x": x.astype(np.int64)} @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="int64") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="int64" + ) out = paddle.fluid.layers.cumsum(x, **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_data_norm_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_data_norm_op_ipu.py index de84b94bb7dde..9013a56ae1c8e 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_data_norm_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_data_norm_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -40,9 +39,9 @@ def set_feed(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.static.nn.data_norm(input=x, **self.attrs) self.fetch_list = [x.name] @@ -58,15 +57,14 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"in_place": True} @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.static.nn.data_norm(input=x, **self.attrs) x = x + 1 self.fetch_list = [x.name] @@ -74,25 +72,21 @@ def build_model(self): @unittest.skip("Do not support in_place=True when test single data_norm Op") class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"in_place": True} class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = {"data_layout": "NHWC"} class TestCase4(TestBase): - def set_op_attrs(self): self.attrs = {"epsilon": 0.001} class TestCase5(TestBase): - def set_op_attrs(self): self.attrs = {"do_model_average_for_mean_and_var": True} @@ -101,16 +95,12 @@ class TestCase6(TestBase): # If enable_scale_and_shift=True, it requires to set values of scale and bias in `param_attr` def set_op_attrs(self): self.attrs = { - "param_attr": { - "scale_w": 0.5, - "bias": 0.1 - }, - "enable_scale_and_shift": True + "param_attr": {"scale_w": 0.5, "bias": 0.1}, + "enable_scale_and_shift": True, } class TestCase7(TestBase): - def set_op_attrs(self): self.attrs = { "param_attr": { @@ -118,9 +108,9 @@ def set_op_attrs(self): "batch_sum": 0.1, "batch_square": 1e3, "scale_w": 0.5, - "bias": 0.1 + "bias": 0.1, }, - "enable_scale_and_shift": True + "enable_scale_and_shift": True, } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_dist_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_dist_op_ipu.py index 5f8db4faba744..a029f8a287bae 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_dist_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_dist_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -34,11 +33,11 @@ def set_data_feed(self): data_y = np.random.uniform(size=[7, 1, 5]) self.feed_fp32 = { "x": data_x.astype(np.float32), - "y": data_y.astype(np.float32) + "y": data_y.astype(np.float32), } self.feed_fp16 = { "x": data_x.astype(np.float16), - "y": data_y.astype(np.float16) + "y": data_y.astype(np.float16), } def set_feed_attr(self): @@ -51,12 +50,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.dist(x, y, **self.attrs) self.fetch_list = [out.name] @@ -72,19 +71,16 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"p": 0} class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"p": float("inf")} class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = {"p": float("-inf")} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_dot_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_dot_op_ipu.py index ed0c36f53eb0c..cf32eb09f5a38 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_dot_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_dot_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,12 +44,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.dot(x, y, **self.attrs) self.fetch_list = [out.name] @@ -66,7 +65,6 @@ def test(self): class TestCase1(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[6]) y = np.random.uniform(size=[6]) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_dropout_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_dropout_op_ipu.py index d104b39c29246..d6b3813a6e6c0 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_dropout_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_dropout_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -42,14 +41,14 @@ def set_op_attrs(self): self.attrs = { "dropout_prob": 0.5, "is_test": True, - "dropout_implementation": "downgrade_in_infer" + "dropout_implementation": "downgrade_in_infer", } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.fluid.layers.dropout(x, **self.attrs) out = paddle.fluid.layers.elementwise_add(x, x) self.fetch_list = [out.name] @@ -66,22 +65,20 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "dropout_prob": 0.5, "is_test": True, - "dropout_implementation": "upscale_in_train" + "dropout_implementation": "upscale_in_train", } class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { "dropout_prob": 0.0, "is_test": False, - "dropout_implementation": "upscale_in_train" + "dropout_implementation": "upscale_in_train", } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_dy2static_fp16_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_dy2static_fp16_ipu.py index 098f3d6b2ba33..3450978d42755 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_dy2static_fp16_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_dy2static_fp16_ipu.py @@ -21,14 +21,12 @@ class SimpleLayer(paddle.nn.Layer): - def __init__(self, use_ipu=False): super(SimpleLayer, self).__init__() self.use_ipu = use_ipu - self.conv = paddle.nn.Conv2D(in_channels=3, - out_channels=1, - kernel_size=2, - stride=1) + self.conv = paddle.nn.Conv2D( + in_channels=3, out_channels=1, kernel_size=2, stride=1 + ) def forward(self, x, target=None): x = self.conv(x) @@ -45,7 +43,6 @@ def forward(self, x, target=None): class TestBase(IPUD2STest): - def setUp(self): super().setUp() self.save_path = tempfile.TemporaryDirectory() @@ -59,28 +56,33 @@ def _test(self, use_ipu=False): np.random.seed(self.SEED) model = SimpleLayer(use_ipu) specs = [ - paddle.static.InputSpec(name="x", - shape=[32, 3, 10, 10], - dtype="float32"), + paddle.static.InputSpec( + name="x", shape=[32, 3, 10, 10], dtype="float32" + ), paddle.static.InputSpec(name="target", shape=[32], dtype="int64"), ] model = paddle.jit.to_static(model, input_spec=specs) - optim = paddle.optimizer.Adam(learning_rate=0.01, - parameters=model.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.01, parameters=model.parameters() + ) data = paddle.uniform((32, 3, 10, 10), dtype='float32') label = paddle.randint(0, 10, shape=[32], dtype='int64') model_path = '{}/model_state_dict_{}.pdparams'.format( - self.save_path, 'ipu' if use_ipu else 'cpu') + self.save_path, 'ipu' if use_ipu else 'cpu' + ) optim_path = '{}/optim_state_dict_{}.pdopt'.format( - self.save_path, 'ipu' if use_ipu else 'cpu') + self.save_path, 'ipu' if use_ipu else 'cpu' + ) if use_ipu: paddle.set_device('ipu') ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(num_ipus=1, - is_training=True, - micro_batch_size=1, - enable_manual_shard=False) + ipu_strategy.set_graph_config( + num_ipus=1, + is_training=True, + micro_batch_size=1, + enable_manual_shard=False, + ) ipu_strategy.set_precision_config(enable_fp16=True) ipu_strategy.set_optimizer(optim) data = data.astype(np.float16) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_dy2static_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_dy2static_ipu.py index 5a7d0836d6f1a..d8dd65646bfc3 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_dy2static_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_dy2static_ipu.py @@ -17,7 +17,9 @@ import numpy as np import paddle -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramCache +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramCache, +) from paddle.fluid.tests.unittests.ipu.op_test_ipu import IPUD2STest from paddle.jit import to_static from paddle.optimizer.lr import LRScheduler @@ -25,18 +27,18 @@ class SimpleLayer(paddle.nn.Layer): - - def __init__(self, - loss_op=None, - use_softmax=True, - use_reduction=True, - use_identity_loss=True): + def __init__( + self, + loss_op=None, + use_softmax=True, + use_reduction=True, + use_identity_loss=True, + ): super(SimpleLayer, self).__init__() self.loss_op = loss_op - self.conv = paddle.nn.Conv2D(in_channels=3, - out_channels=1, - kernel_size=2, - stride=1) + self.conv = paddle.nn.Conv2D( + in_channels=3, out_channels=1, kernel_size=2, stride=1 + ) self.use_softmax = use_softmax self.use_reduction = use_reduction self.use_identity_loss = use_identity_loss @@ -61,7 +63,6 @@ def forward(self, x, target=None): class TestBase(IPUD2STest): - def setUp(self): self.set_op_attrs() self.set_data_feed() @@ -74,25 +75,30 @@ def set_data_feed(self): self.label = paddle.randint(0, 10, shape=[8], dtype='int64') def create_model(self, use_ipu=False): - return SimpleLayer(loss_op=self.loss_op, - use_softmax=True, - use_reduction=not use_ipu, - use_identity_loss=use_ipu) + return SimpleLayer( + loss_op=self.loss_op, + use_softmax=True, + use_reduction=not use_ipu, + use_identity_loss=use_ipu, + ) def _test(self, use_ipu=False): paddle.seed(self.SEED) np.random.seed(self.SEED) model = self.create_model(use_ipu) - optim = paddle.optimizer.Adam(learning_rate=0.01, - parameters=model.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.01, parameters=model.parameters() + ) if use_ipu: paddle.set_device('ipu') ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(num_ipus=1, - is_training=True, - micro_batch_size=1, - enable_manual_shard=False) + ipu_strategy.set_graph_config( + num_ipus=1, + is_training=True, + micro_batch_size=1, + enable_manual_shard=False, + ) ipu_strategy.set_optimizer(optim) epochs = 100 @@ -118,7 +124,6 @@ def test_training(self): class TestSaveLoad(TestBase): - def setUp(self): super().setUp() self.save_path = tempfile.TemporaryDirectory() @@ -131,20 +136,25 @@ def _test(self, use_ipu=False): paddle.seed(self.SEED) np.random.seed(self.SEED) model = self.create_model(use_ipu) - optim = paddle.optimizer.Adam(learning_rate=0.01, - parameters=model.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.01, parameters=model.parameters() + ) model_path = '{}/model_state_dict_{}.pdparams'.format( - self.save_path, 'ipu' if use_ipu else 'cpu') + self.save_path, 'ipu' if use_ipu else 'cpu' + ) optim_path = '{}/optim_state_dict_{}.pdopt'.format( - self.save_path, 'ipu' if use_ipu else 'cpu') + self.save_path, 'ipu' if use_ipu else 'cpu' + ) if use_ipu: paddle.set_device('ipu') ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(num_ipus=1, - is_training=True, - micro_batch_size=1, - enable_manual_shard=False) + ipu_strategy.set_graph_config( + num_ipus=1, + is_training=True, + micro_batch_size=1, + enable_manual_shard=False, + ) ipu_strategy.set_optimizer(optim) epochs = 100 @@ -182,7 +192,6 @@ def _test(self, use_ipu=False): class TestPatch(IPUD2STest): - def setUp(cls): paddle.disable_static() @@ -201,16 +210,16 @@ def test(self, use_ipu=False): class TestWithoutIdentityLoss1(TestBase): - def create_model(self, use_ipu=False): - return SimpleLayer(loss_op=self.loss_op, - use_softmax=True, - use_reduction=True, - use_identity_loss=False) + return SimpleLayer( + loss_op=self.loss_op, + use_softmax=True, + use_reduction=True, + use_identity_loss=False, + ) class TestWithoutIdentityLoss2(TestBase): - def set_op_attrs(self): self.loss_op = paddle.fluid.layers.softmax_with_cross_entropy @@ -219,14 +228,15 @@ def set_data_feed(self): self.label = paddle.randint(0, 10, shape=[8, 1], dtype='int64') def create_model(self, use_ipu=False): - return SimpleLayer(loss_op=self.loss_op, - use_softmax=False, - use_reduction=True, - use_identity_loss=False) + return SimpleLayer( + loss_op=self.loss_op, + use_softmax=False, + use_reduction=True, + use_identity_loss=False, + ) class TestWithoutIdentityLoss3(TestBase): - def set_op_attrs(self): self.loss_op = partial(paddle.fluid.layers.kldiv_loss, reduction="none") @@ -235,14 +245,15 @@ def set_data_feed(self): self.label = paddle.rand(shape=[8, 81], dtype='float32') def create_model(self, use_ipu=False): - return SimpleLayer(loss_op=self.loss_op, - use_softmax=True, - use_reduction=True, - use_identity_loss=False) + return SimpleLayer( + loss_op=self.loss_op, + use_softmax=True, + use_reduction=True, + use_identity_loss=False, + ) class TestWithoutIdentityLoss4(TestBase): - def set_op_attrs(self): self.loss_op = paddle.nn.functional.binary_cross_entropy @@ -251,27 +262,31 @@ def set_data_feed(self): self.label = paddle.rand(shape=[8, 81], dtype='float32') def create_model(self, use_ipu=False): - return SimpleLayer(loss_op=self.loss_op, - use_softmax=True, - use_reduction=False, - use_identity_loss=False) + return SimpleLayer( + loss_op=self.loss_op, + use_softmax=True, + use_reduction=False, + use_identity_loss=False, + ) class TestWithoutIdentityLoss5(TestBase): - def set_op_attrs(self): self.loss_op = paddle.fluid.layers.sigmoid_cross_entropy_with_logits def set_data_feed(self): self.data = paddle.uniform((8, 3, 10, 10), dtype='float32') - self.label = paddle.randint(0, 10, shape=[8, 81], - dtype='int64').astype('float32') + self.label = paddle.randint(0, 10, shape=[8, 81], dtype='int64').astype( + 'float32' + ) def create_model(self, use_ipu=False): - return SimpleLayer(loss_op=self.loss_op, - use_softmax=True, - use_reduction=True, - use_identity_loss=False) + return SimpleLayer( + loss_op=self.loss_op, + use_softmax=True, + use_reduction=True, + use_identity_loss=False, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_elemetwise_x_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_elemetwise_x_op_ipu.py index 9c35e43970e74..8dcb3097c2f52 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_elemetwise_x_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_elemetwise_x_op_ipu.py @@ -21,7 +21,6 @@ class TestMul(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -43,12 +42,12 @@ def set_feed_attr(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = self.op(x, y, **self.attrs) self.fetch_list = [out.name] @@ -125,43 +124,36 @@ def test_case3(self): class TestAdd(TestMul): - def set_test_op(self): self.op = paddle.fluid.layers.elementwise_add class TestSub(TestMul): - def set_test_op(self): self.op = paddle.fluid.layers.elementwise_sub class TestDiv(TestMul): - def set_test_op(self): self.op = paddle.fluid.layers.elementwise_div class TestMin(TestMul): - def set_test_op(self): self.op = paddle.fluid.layers.elementwise_min class TestMax(TestMul): - def set_test_op(self): self.op = paddle.fluid.layers.elementwise_max class TestPow(TestMul): - def set_test_op(self): self.op = paddle.fluid.layers.elementwise_pow class TestMod(TestMul): - def set_atol(self): self.atol = 1e-7 self.rtol = 1e-5 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_equal_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_equal_op_ipu.py index 77a78a7cb78ca..e0ae7fa24676a 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_equal_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_equal_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -50,12 +49,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.equal(x, y, **self.attrs) self.fetch_list = [out.name] @@ -71,7 +70,6 @@ def test(self): class TestCase1(TestBase): - def set_data_feed(self): x = np.ones([1, 10]) y = np.ones([1, 10]) @@ -80,7 +78,6 @@ def set_data_feed(self): class TestCase2(TestBase): - def set_data_feed(self): x = np.ones([1, 10]) y = np.arange(0, 10).reshape([1, 10]) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_eval_model_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_eval_model_ipu.py index f3b1d227d95a1..335a78c5982f3 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_eval_model_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_eval_model_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_data_feed() @@ -57,25 +56,27 @@ def _test_optimizer(self, run_ipu=True): with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - image = paddle.static.data(name='image', - shape=[1, 3, 10, 10], - dtype='float32') - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + image = paddle.static.data( + name='image', shape=[1, 3, 10, 10], dtype='float32' + ) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) loss = paddle.mean(conv1) weight_decay = self.attrs['weight_decay'] - opt = paddle.optimizer.SGD(learning_rate=1e-1, - weight_decay=weight_decay) + opt = paddle.optimizer.SGD( + learning_rate=1e-1, weight_decay=weight_decay + ) if self.attrs['optimizer'] == 'adam': - opt = paddle.optimizer.Adam(learning_rate=1e-1, - weight_decay=weight_decay) + opt = paddle.optimizer.Adam( + learning_rate=1e-1, weight_decay=weight_decay + ) elif self.attrs['optimizer'] == 'lamb': - opt = paddle.optimizer.Lamb(learning_rate=1e-1, - lamb_weight_decay=weight_decay) + opt = paddle.optimizer.Lamb( + learning_rate=1e-1, lamb_weight_decay=weight_decay + ) opt.minimize(loss) if run_ipu: @@ -92,8 +93,8 @@ def _test_optimizer(self, run_ipu=True): ipu_strategy.set_graph_config(is_training=True) ipu_strategy.set_options({"runtime_options.enable_eval": True}) program = paddle.static.IpuCompiledProgram( - main_prog, - ipu_strategy=ipu_strategy).compile(feed_list, fetch_list) + main_prog, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) else: program = main_prog @@ -102,16 +103,17 @@ def _test_optimizer(self, run_ipu=True): for epoch in range(200): if epoch == 100: ipu_strategy.set_options( - {"runtime_options.enable_eval": False}) - loss_res = exe.run(program, - feed=self.feed, - fetch_list=[loss]) + {"runtime_options.enable_eval": False} + ) + loss_res = exe.run( + program, feed=self.feed, fetch_list=[loss] + ) result.append(loss_res) else: for epoch in range(100): - loss_res = exe.run(program, - feed=self.feed, - fetch_list=[loss]) + loss_res = exe.run( + program, feed=self.feed, fetch_list=[loss] + ) result.append(loss_res) return np.array(result) @@ -120,10 +122,9 @@ def test(self): ipu_loss = self._test_optimizer(True).flatten() cpu_loss = self._test_optimizer(False).flatten() self.assertTrue(ipu_loss[0] == ipu_loss[99]) - np.testing.assert_allclose(ipu_loss[100:], - cpu_loss, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + ipu_loss[100:], cpu_loss, rtol=1e-05, atol=self.atol + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_expand_as_v2_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_expand_as_v2_op_ipu.py index ee68eba5e543f..b66e1a6c5bd62 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_expand_as_v2_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_expand_as_v2_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -33,11 +32,11 @@ def set_data_feed(self): data_y = np.random.uniform(size=[2, 2, 3]) self.feed_fp32 = { 'x': data_x.astype(np.float32), - 'y': data_y.astype(np.float32) + 'y': data_y.astype(np.float32), } self.feed_fp16 = { 'x': data_x.astype(np.float16), - 'y': data_y.astype(np.float16) + 'y': data_y.astype(np.float16), } def set_feed_attr(self): @@ -47,12 +46,12 @@ def set_feed_attr(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype="float32" + ) out = paddle.expand_as(x, y) self.fetch_list = [out.name] @@ -68,33 +67,31 @@ def test(self): class TestCase1(TestBase): - def set_data_feed(self): data_x = np.random.uniform(size=[2, 3]) data_y = np.random.uniform(size=[2, 4, 2, 3]) self.feed_fp32 = { 'x': data_x.astype(np.float32), - 'y': data_y.astype(np.float32) + 'y': data_y.astype(np.float32), } self.feed_fp16 = { 'x': data_x.astype(np.float16), - 'y': data_y.astype(np.float16) + 'y': data_y.astype(np.float16), } @unittest.skip("corresponding dimensions must have the same value.") class TestCase2(TestBase): - def set_data_feed(self): data_x = np.random.uniform(size=[2, 3]) data_y = np.random.uniform(size=[2, 4, 3, 3]) self.feed_fp32 = { 'x': data_x.astype(np.float32), - 'y': data_y.astype(np.float32) + 'y': data_y.astype(np.float32), } self.feed_fp16 = { 'x': data_x.astype(np.float16), - 'y': data_y.astype(np.float16) + 'y': data_y.astype(np.float16), } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_expand_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_expand_op_ipu.py index 843ec0438d74d..784a6a41a41f5 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_expand_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_expand_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) out = paddle.fluid.layers.expand(x, **self.attrs) self.fetch_list = [out.name] @@ -62,7 +61,6 @@ def test(self): class TestCase1(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[2, 2]) self.feed_fp32 = {"x": x.astype(np.float32)} @@ -78,14 +76,15 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) expand_times = paddle.fluid.layers.fill_constant( - shape=[len(self.feed_shape[0])], dtype="int32", value=2) - out = paddle.fluid.layers.expand(x, - expand_times=expand_times, - **self.attrs) + shape=[len(self.feed_shape[0])], dtype="int32", value=2 + ) + out = paddle.fluid.layers.expand( + x, expand_times=expand_times, **self.attrs + ) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_expand_v2_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_expand_v2_op_ipu.py index 5cb949a1943e7..cb98ed3253f29 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_expand_v2_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_expand_v2_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ def set_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) out = paddle.expand(x, **self.attrs) self.fetch_list = [out.name] @@ -62,13 +61,11 @@ def test(self): class TestCase1(TestBase): - def set_attrs(self): self.attrs = {"shape": [5, 2, 2, 3]} class TestCase2(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[2, 1, 3]) self.feed_fp32 = {'x': data.astype(np.float32)} @@ -80,14 +77,12 @@ def set_attrs(self): @unittest.skip("corresponding dimensions must have the same value.") class TestCase3(TestBase): - def set_attrs(self): self.attrs = {"shape": [5, 2, 4, 3]} @unittest.skip("Do not support `shape` = Tensors.") class TestCase4(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[3, 3]) self.feed_fp32 = {'x': data.astype(np.float32)} @@ -95,9 +90,9 @@ def set_data_feed(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) self.attrs = { 'name': 'y', 'shape': [3], diff --git a/python/paddle/fluid/tests/unittests/ipu/test_fill_any_like_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_fill_any_like_op_ipu.py index 74ecba6f18c86..ae43689dfb499 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_fill_any_like_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_fill_any_like_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -43,9 +42,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x_fill = paddle.full_like(x, **self.attrs) out = paddle.fluid.layers.elementwise_add(x_fill, x_fill) self.fetch_list = [out.name] @@ -62,13 +61,11 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {'fill_value': 3, 'dtype': 'int32'} class TestError(TestBase): - @IPUOpTest.static_graph def build_model(self): x = paddle.fluid.data('x', [-1, 3, 13], 'float32') @@ -82,8 +79,9 @@ def test(self): def test_error(): self.run_op_test(IPUOpTest.ExecutionMode.IPU_FP32) - self.assertRaisesRegex(Exception, "Please check tensor shape setting", - test_error) + self.assertRaisesRegex( + Exception, "Please check tensor shape setting", test_error + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_fill_constant_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_fill_constant_op_ipu.py index 0e1e17cb65ec9..71dfb2d409747 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_fill_constant_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_fill_constant_op_ipu.py @@ -20,7 +20,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -63,7 +62,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'name': 'x', diff --git a/python/paddle/fluid/tests/unittests/ipu/test_flatten_contiguous_range_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_flatten_contiguous_range_op_ipu.py index 4723f753fb698..11bebbe36f70b 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_flatten_contiguous_range_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_flatten_contiguous_range_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,9 +44,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.flatten(x=x, **self.attrs) self.fetch_list = [out.name] @@ -63,7 +62,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['start_axis'] = 0 @@ -71,7 +69,6 @@ def set_op_attrs(self): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['start_axis'] = 1 @@ -79,7 +76,6 @@ def set_op_attrs(self): class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['start_axis'] = 1 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_flatten_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_flatten_op_ipu.py index d7c1da14e296f..f363e40c5fdce 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_flatten_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_flatten_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.flatten(x=x, **self.attrs) self.fetch_list = [out.name] @@ -62,14 +61,12 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['axis'] = 0 class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['axis'] = 2 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_flip_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_flip_op_ipu.py index 07e0acb60a123..abdb9d07b3a89 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_flip_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_flip_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -51,9 +50,11 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) x = paddle.flip(x, **self.attrs) self.fetch_list = [x.name] @@ -69,7 +70,6 @@ def test(self): class TestCase1(TestBase): - def set_feed(self): data = np.random.randint(0, 10, size=[3, 2, 2]) self.feed_fp32 = {'x': data.astype(np.int32)} @@ -77,7 +77,6 @@ def set_feed(self): class TestCase2(TestBase): - def set_feed(self): data = np.random.randint(0, 2, size=[4, 3, 2, 2]) self.feed_fp32 = {'x': data.astype(np.bool)} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_fp16_support_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_fp16_support_ipu.py index 708dd0f405424..8433e45f46ce1 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_fp16_support_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_fp16_support_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -50,22 +49,19 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - conv1 = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) - conv2 = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + conv1 = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) + conv2 = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) add1 = conv1 + conv2 - conv3 = paddle.static.nn.conv2d(add1, - num_filters=8, - filter_size=8, - bias_attr=False) + conv3 = paddle.static.nn.conv2d( + add1, num_filters=8, filter_size=8, bias_attr=False + ) out = paddle.fluid.layers.relu(conv3, **self.attrs) self.fetch_list = [out.name] @@ -81,7 +77,6 @@ def test(self): class TestIntInput(TestBase): - def set_data_feed(self): embedding = np.random.uniform(size=[10, 20]) indice = np.array([1, 3, 5]).astype(np.int32) @@ -96,12 +91,12 @@ def set_data_feed(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) out = paddle.fluid.layers.gather(x, index=y) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_gather_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_gather_op_ipu.py index 13a48e5a98f1b..5913db81f31cf 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_gather_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_gather_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,12 +43,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) out = paddle.fluid.layers.gather(x, index=y, **self.attrs) self.fetch_list = [out.name] @@ -65,7 +64,6 @@ def test(self): class TestCase1(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[100]) y = np.array([1, 3, 5]) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_gelu_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_gelu_op_ipu.py index 2d14621d5fc7e..031fd25777249 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_gelu_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_gelu_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -43,9 +42,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.gelu(x, **self.attrs) self.fetch_list = [out.name] @@ -61,7 +60,6 @@ def test(self): class TestCase1(TestBase): - def set_atol(self): self.atol = 1e-10 self.rtol = 1e-6 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_gradient_clip_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_gradient_clip_ipu.py index b63d176ff2791..d8b1aaf4c4a29 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_gradient_clip_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_gradient_clip_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_data_feed() @@ -60,13 +59,12 @@ def set_training(self): @IPUOpTest.static_graph def build_model(self): - image = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + image = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) loss = paddle.mean(conv1) self.fetch_list = [loss.name] @@ -74,20 +72,23 @@ def build_model(self): # Only support ClipGradByGlobalNorm clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) if self.attrs['optimizer'] == 'sgd': - opt = paddle.optimizer.SGD(learning_rate=1e-1, - weight_decay=weight_decay, - grad_clip=clip) + opt = paddle.optimizer.SGD( + learning_rate=1e-1, weight_decay=weight_decay, grad_clip=clip + ) elif self.attrs['optimizer'] == 'adam': - opt = paddle.optimizer.Adam(learning_rate=1e-1, - weight_decay=weight_decay, - grad_clip=clip) + opt = paddle.optimizer.Adam( + learning_rate=1e-1, weight_decay=weight_decay, grad_clip=clip + ) elif self.attrs['optimizer'] == 'lamb': - opt = paddle.optimizer.Lamb(learning_rate=1e-1, - lamb_weight_decay=weight_decay, - grad_clip=clip) + opt = paddle.optimizer.Lamb( + learning_rate=1e-1, + lamb_weight_decay=weight_decay, + grad_clip=clip, + ) else: raise ValueError( - f"Not supported optimizer {self.attrs['optimizer']} for test") + f"Not supported optimizer {self.attrs['optimizer']} for test" + ) opt.minimize(loss) def run_model(self, exec_mode): @@ -102,7 +103,6 @@ def test(self): class TestAdam(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'adam', @@ -111,7 +111,6 @@ def set_attrs(self): class TestLamb(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'lamb', diff --git a/python/paddle/fluid/tests/unittests/ipu/test_greater_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_greater_op_ipu.py index 4f2e9a1a94bfc..c48117dd40126 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_greater_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_greater_op_ipu.py @@ -21,7 +21,6 @@ class TestGreaterThan(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -35,12 +34,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = self.op(x, y, **self.attrs) self.fetch_list = [out.name] @@ -114,25 +113,21 @@ def test_case3(self): class TestLessThan(TestGreaterThan): - def set_test_op(self): self.op = paddle.fluid.layers.less_than class TestEqual(TestGreaterThan): - def set_test_op(self): self.op = paddle.fluid.layers.equal class TestGreaterEqual(TestGreaterThan): - def set_test_op(self): self.op = paddle.fluid.layers.greater_equal class TestLessEqual(TestGreaterThan): - def set_test_op(self): self.op = paddle.fluid.layers.less_equal diff --git a/python/paddle/fluid/tests/unittests/ipu/test_groupnorm_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_groupnorm_op_ipu.py index 70f1a6a28e97b..fbffe321b5575 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_groupnorm_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_groupnorm_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -53,30 +52,27 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) if self.is_training: ch = self.feed_shape[0][1] - conv1 = paddle.static.nn.conv2d(x, - num_filters=ch, - filter_size=3, - bias_attr=False) + conv1 = paddle.static.nn.conv2d( + x, num_filters=ch, filter_size=3, bias_attr=False + ) scale = paddle.ParamAttr(trainable=True) bias = paddle.ParamAttr(trainable=True) - out = paddle.fluid.layers.nn.group_norm(conv1, - param_attr=scale, - bias_attr=bias, - **self.attrs) + out = paddle.fluid.layers.nn.group_norm( + conv1, param_attr=scale, bias_attr=bias, **self.attrs + ) loss = paddle.mean(out) adam = paddle.optimizer.Adam(learning_rate=1e-2) adam.minimize(loss) self.fetch_list = [loss.name] else: - out = paddle.fluid.layers.nn.group_norm(x, - param_attr=True, - bias_attr=True, - **self.attrs) + out = paddle.fluid.layers.nn.group_norm( + x, param_attr=True, bias_attr=True, **self.attrs + ) self.fetch_list = [out.name] def run_model(self, exec_mode): @@ -91,7 +87,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "groups": 4, @@ -101,7 +96,6 @@ def set_op_attrs(self): class TestTrainCase1(TestBase): - def set_training(self): self.is_training = True self.epoch = 20 @@ -109,7 +103,6 @@ def set_training(self): @unittest.skipIf(IPUOpTest.use_ipumodel(), "skip for ipumodel") class TestTrainCase2(TestBase): - def set_atol(self): self.atol = 7e-4 self.rtol = 1e-6 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_huber_loss_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_huber_loss_op_ipu.py index 613fbd944c26a..5030e368083ee 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_huber_loss_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_huber_loss_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -34,11 +33,11 @@ def set_data_feed(self): target = np.random.uniform(size=[3, 4, 2, 2]) self.feed_fp32 = { "x": x.astype(np.float32), - "target": target.astype(np.float32) + "target": target.astype(np.float32), } self.feed_fp16 = { "x": x.astype(np.float16), - "target": target.astype(np.float16) + "target": target.astype(np.float16), } def set_feed_attr(self): @@ -52,12 +51,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self, on_ipu): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - target = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + target = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.huber_loss(x, target, **self.attrs) self.fetch_list = [out.name] @@ -73,7 +72,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'delta': 0.5, @@ -81,7 +79,6 @@ def set_op_attrs(self): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { 'delta': 0.0, diff --git a/python/paddle/fluid/tests/unittests/ipu/test_identity_loss_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_identity_loss_ipu.py index 53baf0de0acce..8e2d2110cf119 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_identity_loss_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_identity_loss_ipu.py @@ -20,14 +20,15 @@ import paddle.fluid.compiler as compiler import paddle.optimizer import paddle.static -from paddle.fluid.tests.unittests.ipu.op_test_ipu import (IPUOpTest, - np_dtype_to_fluid_str) +from paddle.fluid.tests.unittests.ipu.op_test_ipu import ( + IPUOpTest, + np_dtype_to_fluid_str, +) paddle.enable_static() class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -41,8 +42,9 @@ def set_op(self): def set_feed(self): self.feed = { - "x": np.random.uniform(low=-2, high=2, size=[3, - 5]).astype('float32'), + "x": np.random.uniform(low=-2, high=2, size=[3, 5]).astype( + 'float32' + ), } def set_feed_attr(self): @@ -62,9 +64,11 @@ def _test_base(self, reduction): with fluid.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) out = self.op(x, reduction) fetch_list = [out.name] @@ -77,7 +81,8 @@ def _test_base(self, reduction): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config(num_ipus=1, is_training=False) ipu_compiler = compiler.IpuCompiledProgram( - main_prog, ipu_strategy=ipu_strategy) + main_prog, ipu_strategy=ipu_strategy + ) program = ipu_compiler.compile(feed_list, fetch_list) ipu_res = exe.run(program, self.feed, fetch_list) @@ -92,10 +97,9 @@ def _test_base(self, reduction): # none cpu_res = self.feed['x'] - np.testing.assert_allclose(ipu_res[0], - cpu_res, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + ipu_res[0], cpu_res, rtol=1e-05, atol=self.atol + ) def test_base(self): # TODO: use string instead of int for reduction diff --git a/python/paddle/fluid/tests/unittests/ipu/test_inference_model_io_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_inference_model_io_ipu.py index 4a1aa8cba4418..c370debfc4b30 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_inference_model_io_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_inference_model_io_ipu.py @@ -22,7 +22,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_data_feed() @@ -60,19 +59,24 @@ def _test_save(self): startup_prog.random_seed = self.SEED generator = paddle.fluid.unique_name.UniqueNameGenerator() self.full_name = '/'.join( - [self.attrs['path'].name, self.attrs['model_name']]) + [self.attrs['path'].name, self.attrs['model_name']] + ) with paddle.fluid.unique_name.guard(generator): with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - conv1 = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False, - name='conv2d') + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype='float32', + ) + conv1 = paddle.static.nn.conv2d( + x, + num_filters=3, + filter_size=3, + bias_attr=False, + name='conv2d', + ) loss = paddle.mean(conv1) if self.attrs['is_training']: @@ -93,24 +97,22 @@ def _test_save(self): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config( - is_training=self.attrs['is_training']) + is_training=self.attrs['is_training'] + ) program = paddle.static.IpuCompiledProgram( - main_prog, - ipu_strategy=ipu_strategy).compile(self.feed_list, - fetch_list) + main_prog, ipu_strategy=ipu_strategy + ).compile(self.feed_list, fetch_list) result = [] for i in range(self.attrs['steps']): - tmp = exe.run(program, - feed=self.feed, - fetch_list=fetch_list) + tmp = exe.run( + program, feed=self.feed, fetch_list=fetch_list + ) result.append(tmp) - paddle.static.save_inference_model(self.full_name, - x, - loss, - exe, - program=program.org_program) + paddle.static.save_inference_model( + self.full_name, x, loss, exe, program=program.org_program + ) def _test_load(self, run_ipu): if run_ipu: @@ -119,8 +121,11 @@ def _test_load(self, run_ipu): place = paddle.CPUPlace() exe = paddle.static.Executor(place) - [inference_program, feed_target_names, fetch_targets - ] = (paddle.static.load_inference_model(self.full_name, exe)) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(self.full_name, exe) if run_ipu: feed_list = feed_target_names @@ -128,8 +133,8 @@ def _test_load(self, run_ipu): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config(is_training=False) program = paddle.static.IpuCompiledProgram( - inference_program, - ipu_strategy=ipu_strategy).compile(feed_list, fetch_list) + inference_program, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) else: program = inference_program @@ -147,7 +152,6 @@ def test_base(self): class TestAdam(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['steps'] = 100 @@ -158,7 +162,6 @@ def set_op_attrs(self): class TestLamb(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['steps'] = 100 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_instancenorm_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_instancenorm_op_ipu.py index b24e4be7ae738..3479899ebda8c 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_instancenorm_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_instancenorm_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -49,31 +48,28 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) if self.is_training: ch = self.feed_shape[0][1] - conv1 = paddle.static.nn.conv2d(x, - num_filters=ch, - filter_size=3, - bias_attr=False) + conv1 = paddle.static.nn.conv2d( + x, num_filters=ch, filter_size=3, bias_attr=False + ) scale = paddle.ParamAttr(trainable=True) bias = paddle.ParamAttr(trainable=True) - out = paddle.fluid.layers.nn.instance_norm(conv1, - param_attr=scale, - bias_attr=bias, - **self.attrs) + out = paddle.fluid.layers.nn.instance_norm( + conv1, param_attr=scale, bias_attr=bias, **self.attrs + ) loss = paddle.mean(out) adam = paddle.optimizer.Adam(learning_rate=1e-2) adam.minimize(loss) self.fetch_list = [loss.name] else: - out = paddle.fluid.layers.nn.instance_norm(x, - param_attr=True, - bias_attr=True, - **self.attrs) + out = paddle.fluid.layers.nn.instance_norm( + x, param_attr=True, bias_attr=True, **self.attrs + ) self.fetch_list = [out.name] def run_model(self, exec_mode): @@ -88,7 +84,6 @@ def test(self): class TestTrainCase1(TestBase): - def set_training(self): self.is_training = True self.epoch = 10 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_interpolate_ops_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_interpolate_ops_ipu.py index 108e953659dc5..e31320585db1c 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_interpolate_ops_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_interpolate_ops_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,9 +44,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) out = paddle.nn.functional.interpolate(x, **self.attrs) self.fetch_list = [out.name] @@ -63,14 +62,12 @@ def test(self): class TestCase0(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs["size"] = [3, 4] class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs["scale_factor"] = [2, 1] @@ -78,19 +75,16 @@ def set_op_attrs(self): @unittest.skip("Only one of size or scale_factor should be defined") class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"size": [12, 12], "scale_factor": [2, 1]} class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = {"scale_factor": 2.5} class TestBilinear(TestBase): - @property def fp16_enabled(self): return False @@ -107,7 +101,6 @@ def set_op_attrs(self): # Take long time class TestBicubic(TestBase): - @property def fp16_enabled(self): return False @@ -124,7 +117,6 @@ def set_op_attrs(self): # Trilinear requires 5-D input class TestTrilinear(TestBase): - @property def fp16_enabled(self): return False @@ -144,13 +136,12 @@ def set_op_attrs(self): self.attrs = { "size": [12, 12, 12], "mode": "trilinear", - "data_format": "NCDHW" + "data_format": "NCDHW", } # Linear requires 3-D input class TestLinear(TestBase): - @property def fp16_enabled(self): return False @@ -171,9 +162,9 @@ def set_op_attrs(self): @unittest.skip( - "Transfer to Pool Op with 2-D ksize, now we only support 1-D ksize.") + "Transfer to Pool Op with 2-D ksize, now we only support 1-D ksize." +) class TestArea(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[2, 3, 6, 6]) self.feed_fp32 = {"x": x.astype(np.float32)} @@ -185,7 +176,6 @@ def set_op_attrs(self): # align_corners option can only be set with the interpolating modes: linear | bilinear | bicubic | trilinear class TestAlignCorners(TestBase): - @property def fp16_enabled(self): return False @@ -194,13 +184,12 @@ def set_op_attrs(self): self.attrs = { "size": [12, 12], "align_corners": True, - "mode": "bilinear" + "mode": "bilinear", } # class TestAlignMode(TestBase): - def set_op_attrs(self): self.attrs = {"size": [12, 12], "align_mode": 1} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_ipu_shard_api_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_ipu_shard_api_ipu.py index f04a3bb55e5d9..bb430fe35a97b 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_ipu_shard_api_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_ipu_shard_api_ipu.py @@ -22,7 +22,6 @@ class TestIpuShard(unittest.TestCase): - def _test(self): # build graph main_prog = paddle.static.Program() @@ -59,14 +58,12 @@ def _test(self): def test_ipu_shard(self): ipu_index_list = self._test() expected_ipu_index_list = [1, 2, 3, 1, 2, 1, 2] - np.testing.assert_allclose(ipu_index_list, - expected_ipu_index_list, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + ipu_index_list, expected_ipu_index_list, rtol=1e-05, atol=0 + ) class TestIpuPipeline(unittest.TestCase): - def _test(self): # build graph main_prog = paddle.static.Program() @@ -104,10 +101,9 @@ def test_ipu_shard(self): ipu_index_list = self._test() expected_ipu_index_list = [1, 2, 3, 1, 2, 1, 2] - np.testing.assert_allclose(ipu_index_list, - expected_ipu_index_list, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + ipu_index_list, expected_ipu_index_list, rtol=1e-05, atol=0 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_ipu_strategy_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_ipu_strategy_ipu.py index ed4b258b75437..bd00f9c9a6534 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_ipu_strategy_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_ipu_strategy_ipu.py @@ -21,13 +21,13 @@ class TestIpuStrategy(unittest.TestCase): - def test_set_options(self): ipu_strategy = paddle.static.IpuStrategy() all_option_names = ipu_strategy._ipu_strategy.get_all_option_names() skip_options = [] skip_options.append( - 'mean_accumulation_and_replication_reduction_strategy') + 'mean_accumulation_and_replication_reduction_strategy' + ) skip_options.append('random_seed') for option_name in all_option_names: @@ -48,7 +48,9 @@ def test_set_options(self): try: ipu_strategy.set_options({option_name: set_value}) new_value = ipu_strategy.get_option(option_name) - assert new_value == set_value, f"set {option_name} to {set_value} failed" + assert ( + new_value == set_value + ), f"set {option_name} to {set_value} failed" except: raise Exception(f"set {option_name} to {set_value} failed") @@ -70,21 +72,24 @@ def test_set_other_options(self): options['engine_options'] = { 'debug.allowOutOfMemory': 'true', 'autoReport.directory': 'path', - 'autoReport.all': 'true' + 'autoReport.all': 'true', } options['random_seed'] = 1234 for k, v in options.items(): ipu_strategy.set_options({k: v}) - if (isinstance(v, list)): - assert v.sort() == ipu_strategy.get_option( - k).sort(), f"set {k} to {v} failed " + if isinstance(v, list): + assert ( + v.sort() == ipu_strategy.get_option(k).sort() + ), f"set {k} to {v} failed " else: assert v == ipu_strategy.get_option( - k), f"set {k} to {v} failed " + k + ), f"set {k} to {v} failed " # The custom logger need 2 int as inputs logger = lambda progress, total: print( - f"compile progrss: {progress}/{total}") + f"compile progrss: {progress}/{total}" + ) ipu_strategy.set_options({'compilation_progress_logger': logger}) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_kldiv_loss_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_kldiv_loss_op_ipu.py index ef3e173fdeda9..d08df5399d939 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_kldiv_loss_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_kldiv_loss_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -34,11 +33,11 @@ def set_data_feed(self): target = np.random.uniform(size=[3, 4, 2, 2]) self.feed_fp32 = { "x": x.astype(np.float32), - "target": target.astype(np.float32) + "target": target.astype(np.float32), } self.feed_fp16 = { "x": x.astype(np.float16), - "target": target.astype(np.float16) + "target": target.astype(np.float16), } def set_feed_attr(self): @@ -52,12 +51,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self, on_ipu): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - target = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + target = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.kldiv_loss(x, target, **self.attrs) self.fetch_list = [out.name] @@ -73,7 +72,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'reduction': 'sum', @@ -81,7 +79,6 @@ def set_op_attrs(self): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { 'reduction': 'none', diff --git a/python/paddle/fluid/tests/unittests/ipu/test_layernorm_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_layernorm_op_ipu.py index 9bf457d6f924f..b6ca7c34714ea 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_layernorm_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_layernorm_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -56,30 +55,27 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) if self.is_training: ch = self.feed_shape[0][1] - conv1 = paddle.static.nn.conv2d(x, - num_filters=ch, - filter_size=3, - bias_attr=False) + conv1 = paddle.static.nn.conv2d( + x, num_filters=ch, filter_size=3, bias_attr=False + ) scale = paddle.ParamAttr(trainable=True) bias = paddle.ParamAttr(trainable=True) - out = paddle.fluid.layers.nn.layer_norm(conv1, - param_attr=scale, - bias_attr=bias, - **self.attrs) + out = paddle.fluid.layers.nn.layer_norm( + conv1, param_attr=scale, bias_attr=bias, **self.attrs + ) loss = paddle.mean(out) self.fetch_list = [loss.name] else: scale = self.attrs['scale'] bias = self.attrs['shift'] - out = paddle.fluid.layers.nn.layer_norm(x, - param_attr=scale, - bias_attr=bias, - **self.attrs) + out = paddle.fluid.layers.nn.layer_norm( + x, param_attr=scale, bias_attr=bias, **self.attrs + ) self.fetch_list = [out.name] if self.is_training: @@ -89,8 +85,9 @@ def build_model(self): elif self.optimizer == 'adam': optimizer = paddle.optimizer.Adam(learning_rate=1e-2) elif self.optimizer == 'lamb': - optimizer = paddle.optimizer.Lamb(learning_rate=1e-2, - lamb_weight_decay=0.0) + optimizer = paddle.optimizer.Lamb( + learning_rate=1e-2, lamb_weight_decay=0.0 + ) if optimizer is not None: optimizer.minimize(loss) @@ -107,7 +104,6 @@ def test(self): @unittest.skip('raise error') class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "scale": False, @@ -119,7 +115,6 @@ def set_op_attrs(self): @unittest.skip('raise error') class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { "scale": True, @@ -130,7 +125,6 @@ def set_op_attrs(self): class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = { "scale": True, @@ -142,13 +136,12 @@ def set_op_attrs(self): class TestTrainCase1(TestBase): - def set_op_attrs(self): self.attrs = { "scale": True, "shift": True, "begin_norm_axis": 1, - "epsilon": 1e-05 + "epsilon": 1e-05, } self.optimizer = 'sgd' @@ -162,7 +155,6 @@ def set_training(self): class TestTrainCase3(TestBase): - def set_atol(self): super().set_atol() self.atol = 5e-3 @@ -172,7 +164,7 @@ def set_op_attrs(self): "scale": True, "shift": True, "begin_norm_axis": 2, - "epsilon": 1e-05 + "epsilon": 1e-05, } self.optimizer = 'lamb' diff --git a/python/paddle/fluid/tests/unittests/ipu/test_logical_not_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_logical_not_op_ipu.py index d8eaa2f81bceb..c75f6faa65b65 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_logical_not_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_logical_not_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -40,9 +39,9 @@ def set_feed_attr(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="bool") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="bool" + ) out = paddle.fluid.layers.logical_not(x) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_logical_x_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_logical_x_op_ipu.py index 79c22f47da5c9..a2d3817f7f54b 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_logical_x_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_logical_x_op_ipu.py @@ -21,7 +21,6 @@ class TestLogicalAnd(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -39,12 +38,16 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype=self.feed_dtype[1]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) + y = paddle.static.data( + name=self.feed_list[1], + shape=self.feed_shape[1], + dtype=self.feed_dtype[1], + ) out = self.op(x, y, **self.attrs) self.fetch_list = [out.name] @@ -79,7 +82,6 @@ def test_case0(self): class TestLogicalOr(TestLogicalAnd): - def set_test_op(self): self.op = paddle.fluid.layers.logical_or diff --git a/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_op_ipu.py index ffcf8a64f53f9..b2aa05fb73438 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,14 +44,14 @@ def set_op_attrs(self): "is_sparse": False, "is_distributed": False, "padding_idx": -1, - "dtype": 'float32' + "dtype": 'float32', } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='int64') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='int64' + ) out = paddle.fluid.layers.embedding(x, **self.attrs) if self.is_training: loss = paddle.mean(out) @@ -76,7 +75,6 @@ def test(self): class TestTrainCase1(TestBase): - def set_atol(self): self.atol = 1e-7 self.rtol = 1e-6 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_v2_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_v2_op_ipu.py index 2c8e7159cf217..4d8ca5bb03728 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_v2_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_lookuptable_v2_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,14 +44,14 @@ def set_op_attrs(self): "embedding_dim": 16, "sparse": False, "padding_idx": -1, - "weight_attr": None + "weight_attr": None, } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='int64') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='int64' + ) embedding = paddle.nn.Embedding(**self.attrs) out = embedding(x) if self.is_training: @@ -77,7 +76,6 @@ def test(self): class TestTrainCase1(TestBase): - def set_atol(self): self.atol = 1e-7 self.rtol = 1e-6 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_lr_sheduler_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_lr_sheduler_ipu.py index 137f3f61c5358..7fffe224458af 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_lr_sheduler_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_lr_sheduler_ipu.py @@ -21,7 +21,6 @@ class LR_New(LRScheduler): - def __init__(self, learning_rate=1e-5, last_epoch=-1, verbose=False): super(LR_New, self).__init__(learning_rate, last_epoch, verbose) @@ -32,16 +31,14 @@ def get_lr(self): class TestConvNet(IPUOpTest): - @IPUOpTest.static_graph def build_model(self): - image = paddle.static.data(name='image', - shape=[1, 3, 10, 10], - dtype='float32') - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + image = paddle.static.data( + name='image', shape=[1, 3, 10, 10], dtype='float32' + ) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) loss = paddle.mean(conv1) opt = paddle.optimizer.Lamb(learning_rate=LR_New()) @@ -61,9 +58,8 @@ def run_model(self, run_ipu=True): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config(is_training=True) program = paddle.static.IpuCompiledProgram( - self.main_prog, - ipu_strategy=ipu_strategy).compile(self.feed_list, - self.fetch_list) + self.main_prog, ipu_strategy=ipu_strategy + ).compile(self.feed_list, self.fetch_list) else: program = self.main_prog @@ -71,9 +67,9 @@ def run_model(self, run_ipu=True): for _ in range(100): if hasattr(program, "lr_sheduler"): program.lr_sheduler.step() - loss_res = exe.run(program, - feed=self.feed, - fetch_list=self.fetch_list) + loss_res = exe.run( + program, feed=self.feed, fetch_list=self.fetch_list + ) result.append(loss_res) return np.array(result) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_margin_rank_loss_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_margin_rank_loss_op_ipu.py index e9964156a128b..5861009fd8518 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_margin_rank_loss_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_margin_rank_loss_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -55,15 +54,15 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self, on_ipu): - label = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - left = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') - right = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype='float32') + label = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + left = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) + right = paddle.static.data( + name=self.feed_list[2], shape=self.feed_shape[2], dtype='float32' + ) out = paddle.fluid.layers.margin_rank_loss(label, left, right) self.fetch_list = [out.name] @@ -79,7 +78,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'margin': 0.5, diff --git a/python/paddle/fluid/tests/unittests/ipu/test_matmul_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_matmul_op_ipu.py index bf2af886959b5..ad53cfbae9c2c 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_matmul_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_matmul_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -49,12 +48,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.matmul(x, y, **self.attrs) self.fetch_list = [out.name] @@ -71,7 +70,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "transpose_x": True, @@ -81,7 +79,6 @@ def set_op_attrs(self): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { "transpose_x": True, @@ -97,7 +94,6 @@ def set_atol(self): class TestCase3(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[5, 4, 3, 2]) y = np.random.uniform(size=[5, 4, 2, 3]) @@ -107,7 +103,6 @@ def set_data_feed(self): class TestCase4(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[4, 3, 2]) y = np.random.uniform(size=[4, 2, 3]) @@ -117,7 +112,6 @@ def set_data_feed(self): class TestCase5(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[4, 2, 3]) y = np.random.uniform(size=[3, 2]) @@ -127,7 +121,6 @@ def set_data_feed(self): class TestCase6(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3]) @@ -137,7 +130,6 @@ def set_data_feed(self): @unittest.skip("not supported") class TestCase6_2(TestCase6): - def set_data_feed(self): x = np.random.uniform(size=[3]) @@ -153,7 +145,6 @@ def set_op_attrs(self): class TestCase7(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[1, 3, 4, 5]) y = np.random.uniform(size=[1, 3, 4, 5]) @@ -166,7 +157,6 @@ def set_op_attrs(self): class TestCase8(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3, 1]) y = np.random.uniform(size=[1, 2]) @@ -177,7 +167,6 @@ def set_data_feed(self): @unittest.skip("not supported") class TestCase8_2(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3]) y = np.random.uniform(size=[2]) @@ -195,7 +184,6 @@ def set_op_attrs(self): @unittest.skip("dim > 4 is not supported") class TestCase9(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[6, 5, 4, 2, 3]) @@ -204,7 +192,6 @@ def set_data_feed(self): class TestCase10(TestBase): - def set_op_attrs(self): self.attrs = { "transpose_y": True, @@ -219,7 +206,6 @@ def set_data_feed(self): class TestCase11(TestBase): - def set_op_attrs(self): self.attrs = { "transpose_x": True, diff --git a/python/paddle/fluid/tests/unittests/ipu/test_matmul_serilize_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_matmul_serilize_ipu.py index 21aa7c4b99236..6ee930f247530 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_matmul_serilize_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_matmul_serilize_ipu.py @@ -27,7 +27,6 @@ def set_serialize_factor(serialize_factor): class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -51,12 +50,16 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype=self.feed_dtype[1]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) + y = paddle.static.data( + name=self.feed_list[1], + shape=self.feed_shape[1], + dtype=self.feed_dtype[1], + ) # decrator maybe the best choice, but need to modify api out = paddle.matmul(x, y, **self.attrs) set_serialize_factor(4) @@ -75,8 +78,8 @@ def run_model(self, run_ipu): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config(is_training=self.is_training) program = paddle.static.IpuCompiledProgram( - self.main_prog, - ipu_strategy=ipu_strategy).compile(feed_list, self.fetch_list) + self.main_prog, ipu_strategy=ipu_strategy + ).compile(feed_list, self.fetch_list) else: program = self.main_prog result = exe.run(program, feed=self.feed, fetch_list=self.fetch_list) @@ -85,10 +88,9 @@ def run_model(self, run_ipu): def test_base(self): res0 = self.run_model(False) res1 = self.run_model(True) - np.testing.assert_allclose(res0.flatten(), - res1.flatten(), - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + res0.flatten(), res1.flatten(), rtol=1e-05, atol=self.atol + ) self.assertTrue(res0.shape == res1.shape) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_matmul_v2_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_matmul_v2_op_ipu.py index 37f575f64bd99..c71395ed272a0 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_matmul_v2_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_matmul_v2_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,12 +44,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.matmul(x, y, **self.attrs) self.fetch_list = [out.name] @@ -66,7 +65,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "transpose_x": True, @@ -75,7 +73,6 @@ def set_op_attrs(self): class TestCase3(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[5, 4, 2, 3]) y = np.random.uniform(size=[5, 4, 3, 2]) @@ -85,7 +82,6 @@ def set_data_feed(self): class TestCase4(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[4, 2, 3]) y = np.random.uniform(size=[4, 3, 2]) @@ -95,7 +91,6 @@ def set_data_feed(self): class TestCase5(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[4, 2, 3]) y = np.random.uniform(size=[3, 2]) @@ -105,7 +100,6 @@ def set_data_feed(self): class TestCase6(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3]) y = np.random.uniform(size=[3]) @@ -116,7 +110,6 @@ def set_data_feed(self): @unittest.skip("not supported") class TestCase6_2(TestCase6): - def set_data_feed(self): x = np.random.uniform(size=[3]) y = np.random.uniform(size=[3]) @@ -129,7 +122,6 @@ def set_op_attrs(self): class TestCase7(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3, 1]) y = np.random.uniform(size=[1, 2]) @@ -140,7 +132,6 @@ def set_data_feed(self): @unittest.skip("dim > 4 is not supported") class TestCase8(TestBase): - def set_data_feed(self): self.feed = { "x": np.random.uniform(size=[6, 5, 4, 2, 3]).astype('float32'), @@ -149,7 +140,6 @@ def set_data_feed(self): class TestCase9(TestBase): - def set_op_attrs(self): self.attrs = { "transpose_y": True, @@ -164,7 +154,6 @@ def set_data_feed(self): class TestCase10(TestBase): - def set_op_attrs(self): self.attrs = { "transpose_x": True, diff --git a/python/paddle/fluid/tests/unittests/ipu/test_mean_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_mean_op_ipu.py index 0f60ed2485e7e..7c6b1e2973843 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_mean_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_mean_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -43,9 +42,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.mean(x) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_meshgrid_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_meshgrid_op_ipu.py index 56242fea3672e..d23f0eb58ef71 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_meshgrid_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_meshgrid_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -40,11 +39,11 @@ def set_feed(self): data2 = np.random.uniform(size=[20]) self.feed_fp32 = { 'x': data1.astype(np.float32), - 'y': data2.astype(np.float32) + 'y': data2.astype(np.float32), } self.feed_fp16 = { 'x': data1.astype(np.float16), - 'y': data2.astype(np.float16) + 'y': data2.astype(np.float16), } def set_feed_attr(self): @@ -58,12 +57,16 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype=self.feed_dtype[1]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) + y = paddle.static.data( + name=self.feed_list[1], + shape=self.feed_shape[1], + dtype=self.feed_dtype[1], + ) r1, r2 = paddle.meshgrid(x, y) self.fetch_list = [r1.name, r2.name] @@ -81,7 +84,6 @@ def test(self): class TestCase1(TestBase): - def set_feed(self): data1 = np.random.uniform(size=[10]) data2 = np.random.uniform(size=[20]) @@ -89,41 +91,46 @@ def set_feed(self): self.feed_fp32 = { 'x': data1.astype(np.float32), 'y': data2.astype(np.float32), - 'z': data3.astype(np.float32) + 'z': data3.astype(np.float32), } self.feed_fp16 = { 'x': data1.astype(np.float16), 'y': data2.astype(np.float16), - 'z': data3.astype(np.float16) + 'z': data3.astype(np.float16), } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype=self.feed_dtype[1]) - z = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype=self.feed_dtype[2]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) + y = paddle.static.data( + name=self.feed_list[1], + shape=self.feed_shape[1], + dtype=self.feed_dtype[1], + ) + z = paddle.static.data( + name=self.feed_list[2], + shape=self.feed_shape[2], + dtype=self.feed_dtype[2], + ) r1, r2, r3 = paddle.meshgrid(x, y, z) self.fetch_list = [r1.name, r2.name, r3.name] class TestCase2(TestBase): - def set_feed(self): data1 = np.random.uniform(size=[100]) data2 = np.random.uniform(size=[200]) self.feed_fp32 = { 'x': data1.astype(np.int32), - 'y': data2.astype(np.int32) + 'y': data2.astype(np.int32), } self.feed_fp16 = { 'x': data1.astype(np.int32), - 'y': data2.astype(np.int32) + 'y': data2.astype(np.int32), } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_inference_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_inference_ipu.py index 7118466a52101..ab62ec4e2760a 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_inference_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_inference_ipu.py @@ -22,7 +22,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_data_feed() @@ -51,9 +50,9 @@ def set_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) # using fp32 x = paddle.static.nn.conv2d(input=x, num_filters=3, filter_size=3) @@ -77,7 +76,8 @@ def run_model(self, exec_mode): amp_list = paddle.static.amp.CustomOpLists() amp_list.unsupported_list = {} to_fp16_var_names = paddle.static.amp.cast_model_to_fp16( - self.main_prog, amp_list, use_fp16_guard=True) + self.main_prog, amp_list, use_fp16_guard=True + ) if self.is_ipu_mode(exec_mode): place = paddle.CPUPlace() @@ -91,27 +91,29 @@ def run_model(self, exec_mode): paddle.static.amp.cast_parameters_to_fp16( paddle.CPUPlace(), self.main_prog, - to_fp16_var_names=to_fp16_var_names) + to_fp16_var_names=to_fp16_var_names, + ) if self.is_ipu_mode(exec_mode): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config( is_training=False, num_ipus=self.num_ipus, - enable_manual_shard=self.enable_manual_shard) + enable_manual_shard=self.enable_manual_shard, + ) ipu_strategy.set_pipelining_config( enable_pipelining=self.enable_pipelining, - batches_per_step=self.batches_per_step) + batches_per_step=self.batches_per_step, + ) program = paddle.static.IpuCompiledProgram( - self.main_prog, - ipu_strategy=ipu_strategy).compile(self.feed_list, - self.fetch_list) + self.main_prog, ipu_strategy=ipu_strategy + ).compile(self.feed_list, self.fetch_list) else: program = self.main_prog - result = exe.run(program, - feed=self.feed_fp32, - fetch_list=self.fetch_list) + result = exe.run( + program, feed=self.feed_fp32, fetch_list=self.fetch_list + ) self.output_dict[exec_mode] = result[0] def test(self): @@ -122,15 +124,14 @@ def test(self): class TestPipline(TestBase): - @IPUOpTest.static_graph def build_model(self, exec_mode): feed_shape = list(self.feed_shape[0]) if self.is_ipu_mode(exec_mode): feed_shape[0] = 1 - x = paddle.static.data(name=self.feed_list[0], - shape=feed_shape, - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=feed_shape, dtype='float32' + ) with paddle.static.ipu_shard_guard(index=0, stage=0): # using fp32 x = paddle.static.nn.conv2d(input=x, num_filters=3, filter_size=3) @@ -140,9 +141,9 @@ def build_model(self, exec_mode): with paddle.static.ipu_shard_guard(index=1, stage=1): # using fp16 with paddle.static.amp.fp16_guard(): - x = paddle.static.nn.conv2d(input=x, - num_filters=6, - filter_size=3) + x = paddle.static.nn.conv2d( + input=x, num_filters=6, filter_size=3 + ) x = paddle.static.nn.batch_norm(x, act='relu') x = F.max_pool2d(x, kernel_size=2, stride=2) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_training_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_training_ipu.py index 51a0e91a29c3b..0ea8d5557f219 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_training_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_mixed_precision_training_ipu.py @@ -22,7 +22,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -57,9 +56,9 @@ def set_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) # using fp32 x = paddle.static.nn.conv2d(input=x, num_filters=3, filter_size=3) @@ -87,7 +86,8 @@ def run_model(self, exec_mode): amp_list = paddle.static.amp.CustomOpLists() amp_list.unsupported_list = {} to_fp16_var_names = paddle.static.amp.cast_model_to_fp16( - self.main_prog, amp_list) + self.main_prog, amp_list + ) if self.is_ipu_mode(exec_mode): place = paddle.CPUPlace() @@ -101,29 +101,31 @@ def run_model(self, exec_mode): paddle.static.amp.cast_parameters_to_fp16( paddle.CPUPlace(), self.main_prog, - to_fp16_var_names=to_fp16_var_names) + to_fp16_var_names=to_fp16_var_names, + ) if self.is_ipu_mode(exec_mode): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config( is_training=self.is_training, num_ipus=self.num_ipus, - enable_manual_shard=self.enable_manual_shard) + enable_manual_shard=self.enable_manual_shard, + ) ipu_strategy.set_pipelining_config( enable_pipelining=self.enable_pipelining, - batches_per_step=self.batches_per_step) + batches_per_step=self.batches_per_step, + ) program = paddle.static.IpuCompiledProgram( - self.main_prog, - ipu_strategy=ipu_strategy).compile(self.feed_list, - self.fetch_list) + self.main_prog, ipu_strategy=ipu_strategy + ).compile(self.feed_list, self.fetch_list) else: program = self.main_prog result = [] for _ in range(self.epoch): - out = exe.run(program, - feed=self.feed_fp32, - fetch_list=self.fetch_list) + out = exe.run( + program, feed=self.feed_fp32, fetch_list=self.fetch_list + ) result.append(out) self.output_dict[exec_mode] = result @@ -135,15 +137,14 @@ def test(self): class TestPipline(TestBase): - @IPUOpTest.static_graph def build_model(self, exec_mode): feed_shape = list(self.feed_shape[0]) if self.is_ipu_mode(exec_mode): feed_shape[0] = 1 - x = paddle.static.data(name=self.feed_list[0], - shape=feed_shape, - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=feed_shape, dtype='float32' + ) with paddle.static.ipu_shard_guard(index=0, stage=0): # using fp32 @@ -154,9 +155,9 @@ def build_model(self, exec_mode): with paddle.static.ipu_shard_guard(index=1, stage=1): # using fp16 with paddle.static.amp.fp16_guard(): - x = paddle.static.nn.conv2d(input=x, - num_filters=6, - filter_size=3) + x = paddle.static.nn.conv2d( + input=x, num_filters=6, filter_size=3 + ) x = paddle.static.nn.batch_norm(x, act='relu') x = F.max_pool2d(x, kernel_size=2, stride=2) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_model_parallel_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_model_parallel_ipu.py index 55a6569b7dd1c..2592d0ddee59c 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_model_parallel_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_model_parallel_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -59,19 +58,17 @@ def _test_base(self, run_ipu=True): bs = self.ipu_bs if run_ipu else self.cpu_bs with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - image = paddle.static.data(name='image', - shape=[bs, 3, 10, 10], - dtype='float32') + image = paddle.static.data( + name='image', shape=[bs, 3, 10, 10], dtype='float32' + ) with paddle.static.ipu_shard_guard(index=0): - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) with paddle.static.ipu_shard_guard(index=1): - conv2 = paddle.static.nn.conv2d(conv1, - num_filters=3, - filter_size=3, - bias_attr=False) + conv2 = paddle.static.nn.conv2d( + conv1, num_filters=3, filter_size=3, bias_attr=False + ) # should consider influence of bs loss = paddle.mean(conv2) @@ -101,11 +98,12 @@ def _test_base(self, run_ipu=True): ipu_strategy.set_graph_config( num_ipus=2 * self.ipu_options['replicated_graph_count'], is_training=self.is_training, - enable_manual_shard=True) + enable_manual_shard=True, + ) ipu_strategy.set_options(self.ipu_options) program = paddle.static.IpuCompiledProgram( - main_prog, - ipu_strategy=ipu_strategy).compile(feed_list, fetch_list) + main_prog, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) else: program = main_prog @@ -126,14 +124,12 @@ def test(self): cpu_outputs = self._test_base(False) ipu_outputs = self._test_base(True) - np.testing.assert_allclose(cpu_outputs, - ipu_outputs, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + cpu_outputs, ipu_outputs, rtol=1e-05, atol=self.atol + ) class TestReplicaInference(TestBase): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -150,14 +146,13 @@ def set_data_feed(self): np_image = np.random.rand(1, 3, 10, 10).astype(np.float32) self.feed_cpu = {"image": np_image} self.feed_ipu = { - "image": - np.tile(np_image, - [self.ipu_options['replicated_graph_count'], 1, 1, 1]) + "image": np.tile( + np_image, [self.ipu_options['replicated_graph_count'], 1, 1, 1] + ) } class TestReplicaCollectiveInference(TestBase): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -166,13 +161,11 @@ def set_attrs(self): "accumulation_factor": 1, "enable_replicated_graphs": True, "replicated_graph_count": 2, - "accumulate_outer_fragment": { - 0: [] - }, + "accumulate_outer_fragment": {0: []}, "replicated_collectives_settings": { "prepare_schedule_for_merging_collectives": True, - "merge_all_reduce_collectives": True - } + "merge_all_reduce_collectives": True, + }, } self.cpu_bs = 1 self.ipu_bs = 1 @@ -181,14 +174,13 @@ def set_data_feed(self): np_image = np.random.rand(1, 3, 10, 10).astype(np.float32) self.feed_cpu = {"image": np_image} self.feed_ipu = { - "image": - np.tile(np_image, - [self.ipu_options['replicated_graph_count'], 1, 1, 1]) + "image": np.tile( + np_image, [self.ipu_options['replicated_graph_count'], 1, 1, 1] + ) } class TestPipelineInference(TestBase): - def set_attrs(self): self.ipu_options = { "batches_per_step": 2, @@ -205,13 +197,13 @@ def set_data_feed(self): np_image = np.random.rand(1, 3, 10, 10).astype(np.float32) self.feed_cpu = {"image": np_image} self.feed_ipu = { - "image": - np.tile(np_image, [self.ipu_options['batches_per_step'], 1, 1, 1]) + "image": np.tile( + np_image, [self.ipu_options['batches_per_step'], 1, 1, 1] + ) } class TestTrainBase(TestBase): - def set_training(self): self.is_training = True self.epoch = 10 @@ -231,7 +223,6 @@ def set_attrs(self): class TestReplicaTrain(TestTrainBase): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -239,7 +230,7 @@ def set_attrs(self): "enable_gradient_accumulation": False, "accumulation_factor": 1, "enable_replicated_graphs": True, - "replicated_graph_count": 2 + "replicated_graph_count": 2, } self.cpu_bs = 2 self.ipu_bs = 1 @@ -249,23 +240,21 @@ def set_data_feed(self): np_image = np.random.rand(1, 3, 10, 10).astype(np.float32) self.feed_cpu = {"image": np.tile(np_image, [self.cpu_bs, 1, 1, 1])} self.feed_ipu = { - "image": - np.tile(np_image, - [self.ipu_options['replicated_graph_count'], 1, 1, 1]) + "image": np.tile( + np_image, [self.ipu_options['replicated_graph_count'], 1, 1, 1] + ) } def test(self): cpu_outputs = self._test_base(False) ipu_outputs = self._test_base(True)[::2] - np.testing.assert_allclose(cpu_outputs, - ipu_outputs, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + cpu_outputs, ipu_outputs, rtol=1e-05, atol=self.atol + ) class TestReplicaCollectiveTrain(TestTrainBase): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -274,13 +263,11 @@ def set_attrs(self): "accumulation_factor": 1, "enable_replicated_graphs": True, "replicated_graph_count": 2, - "accumulate_outer_fragment": { - 0: [] - }, + "accumulate_outer_fragment": {0: []}, "replicated_collectives_settings": { "prepare_schedule_for_merging_collectives": True, - "merge_all_reduce_collectives": True - } + "merge_all_reduce_collectives": True, + }, } self.cpu_bs = 2 self.ipu_bs = 1 @@ -290,23 +277,21 @@ def set_data_feed(self): np_image = np.random.rand(1, 3, 10, 10).astype(np.float32) self.feed_cpu = {"image": np.tile(np_image, [self.cpu_bs, 1, 1, 1])} self.feed_ipu = { - "image": - np.tile(np_image, - [self.ipu_options['replicated_graph_count'], 1, 1, 1]) + "image": np.tile( + np_image, [self.ipu_options['replicated_graph_count'], 1, 1, 1] + ) } def test(self): cpu_outputs = self._test_base(False) ipu_outputs = self._test_base(True)[::2] - np.testing.assert_allclose(cpu_outputs, - ipu_outputs, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + cpu_outputs, ipu_outputs, rtol=1e-05, atol=self.atol + ) class TestPipelineTrain(TestTrainBase): - def set_attrs(self): self.ipu_options = { "batches_per_step": 3, @@ -323,22 +308,22 @@ def set_attrs(self): def set_data_feed(self): np_image = np.random.rand(1, 3, 10, 10).astype(np.float32) self.feed_cpu = {"image": np.tile(np_image, [self.cpu_bs, 1, 1, 1])} - bps_acc = self.ipu_options['batches_per_step'] * self.ipu_options[ - 'accumulation_factor'] + bps_acc = ( + self.ipu_options['batches_per_step'] + * self.ipu_options['accumulation_factor'] + ) self.feed_ipu = {"image": np.tile(np_image, [bps_acc, 1, 1, 1])} def test(self): cpu_outputs = self._test_base(False) ipu_outputs = self._test_base(True)[::3] - np.testing.assert_allclose(cpu_outputs, - ipu_outputs, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + cpu_outputs, ipu_outputs, rtol=1e-05, atol=self.atol + ) class TestAdamTrain(TestTrainBase): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -354,7 +339,6 @@ def set_attrs(self): class TestAdamReplicaTrain(TestReplicaTrain): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -370,7 +354,6 @@ def set_attrs(self): class TestAdamPipelineTrain(TestPipelineTrain): - def set_attrs(self): self.ipu_options = { "batches_per_step": 3, @@ -386,7 +369,6 @@ def set_attrs(self): class TestAdamRecomputationTrain(TestPipelineTrain): - def set_attrs(self): self.ipu_options = { "batches_per_step": 3, @@ -403,7 +385,6 @@ def set_attrs(self): class TestLambTrain(TestAdamTrain): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -419,7 +400,6 @@ def set_attrs(self): class TestLambReplicaTrain(TestAdamReplicaTrain): - def set_attrs(self): self.ipu_options = { "batches_per_step": 1, @@ -435,7 +415,6 @@ def set_attrs(self): class TestLambPipelineTrain(TestAdamPipelineTrain): - def set_attrs(self): self.ipu_options = { "batches_per_step": 3, diff --git a/python/paddle/fluid/tests/unittests/ipu/test_model_pipeline_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_model_pipeline_ipu.py index fb5f25619bf96..6ee6b842332b6 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_model_pipeline_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_model_pipeline_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_training() self.set_data_feed() @@ -37,29 +36,28 @@ def set_feed_attr(self): @IPUOpTest.static_graph def build_model(self): - image = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + image = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) with paddle.static.ipu_shard_guard(index=0): - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) with paddle.static.ipu_shard_guard(index=1): - conv2 = paddle.static.nn.conv2d(conv1, - num_filters=3, - filter_size=3, - bias_attr=False) + conv2 = paddle.static.nn.conv2d( + conv1, num_filters=3, filter_size=3, bias_attr=False + ) loss = paddle.mean(conv2) self.fetch_list = [loss.name] def run_model(self, exec_mode): ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(num_ipus=2, - is_training=False, - enable_manual_shard=True) - ipu_strategy.set_pipelining_config(enable_pipelining=True, - batches_per_step=2) + ipu_strategy.set_graph_config( + num_ipus=2, is_training=False, enable_manual_shard=True + ) + ipu_strategy.set_pipelining_config( + enable_pipelining=True, batches_per_step=2 + ) self.run_op_test(exec_mode, ipu_strategy=ipu_strategy) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ipu/test_mul_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_mul_op_ipu.py index a5ace5f1bf1c9..c1c2116a41302 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_mul_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_mul_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -48,12 +47,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.mul(x, y, **self.attrs) self.fetch_list = [out.name] @@ -69,7 +68,6 @@ def test(self): class TestCase1(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[1, 2, 5]) y = np.random.uniform(size=[5, 3]) @@ -84,7 +82,6 @@ def set_op_attrs(self): class TestCase2(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3, 4, 2, 9]) y = np.random.uniform(size=[3, 6, 1, 2, 3]) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_not_equal_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_not_equal_op_ipu.py index c00a60775eb7f..465ed7d06b704 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_not_equal_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_not_equal_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -50,12 +49,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.not_equal(x, y, **self.attrs) self.fetch_list = [out.name] @@ -71,7 +70,6 @@ def test(self): class TestCase1(TestBase): - def set_data_feed(self): x = np.ones([1, 10]) y = np.ones([1, 10]) @@ -80,7 +78,6 @@ def set_data_feed(self): class TestCase2(TestBase): - def set_data_feed(self): x = np.ones([1, 10]) y = np.arange(0, 10).reshape([1, 10]) @@ -89,7 +86,6 @@ def set_data_feed(self): class TestScalar(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -116,10 +112,10 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - out = (x != 0.5) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + out = x != 0.5 self.fetch_list = [out.name] def run_model(self, exec_mode): diff --git a/python/paddle/fluid/tests/unittests/ipu/test_one_hot_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_one_hot_op_ipu.py index fe5b658426eee..53e8bab2adc1f 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_one_hot_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_one_hot_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -43,9 +42,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='int32' + ) out = paddle.fluid.layers.one_hot(x, **self.attrs) self.fetch_list = [out.name] @@ -62,7 +61,6 @@ def test(self): @unittest.skip('does not support allow_out_of_range=True') class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"depth": 4, "allow_out_of_range": True} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_one_hot_v2_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_one_hot_v2_op_ipu.py index e958cfd1f89ba..99294f9c5f109 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_one_hot_v2_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_one_hot_v2_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -43,9 +42,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='int32' + ) out = paddle.fluid.input.one_hot(x, **self.attrs) self.fetch_list = [out.name] @@ -62,7 +61,6 @@ def test(self): @unittest.skip('does not support allow_out_of_range=True') class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"depth": 4, "allow_out_of_range": True} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_optimizer_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_optimizer_ipu.py index f14b2174d398e..d8b8116078be4 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_optimizer_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_optimizer_ipu.py @@ -20,7 +20,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_data_feed() @@ -57,25 +56,27 @@ def _test_optimizer(self, run_ipu=True): with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - image = paddle.static.data(name='image', - shape=[1, 3, 10, 10], - dtype='float32') - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + image = paddle.static.data( + name='image', shape=[1, 3, 10, 10], dtype='float32' + ) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) loss = paddle.mean(conv1) weight_decay = self.attrs['weight_decay'] - opt = paddle.optimizer.SGD(learning_rate=1e-1, - weight_decay=weight_decay) + opt = paddle.optimizer.SGD( + learning_rate=1e-1, weight_decay=weight_decay + ) if self.attrs['optimizer'] == 'adam': - opt = paddle.optimizer.Adam(learning_rate=1e-1, - weight_decay=weight_decay) + opt = paddle.optimizer.Adam( + learning_rate=1e-1, weight_decay=weight_decay + ) elif self.attrs['optimizer'] == 'lamb': - opt = paddle.optimizer.Lamb(learning_rate=1e-1, - lamb_weight_decay=weight_decay) + opt = paddle.optimizer.Lamb( + learning_rate=1e-1, lamb_weight_decay=weight_decay + ) opt.minimize(loss) if run_ipu: @@ -91,18 +92,23 @@ def _test_optimizer(self, run_ipu=True): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config(is_training=True) ipu_strategy.set_options( - {'loss_scaling': self.attrs["loss_scaling"]}) + {'loss_scaling': self.attrs["loss_scaling"]} + ) if "use_no_bias_optimizer" in self.attrs.keys(): - ipu_strategy.set_options({ - "use_no_bias_optimizer": - self.attrs["use_no_bias_optimizer"] - }) + ipu_strategy.set_options( + { + "use_no_bias_optimizer": self.attrs[ + "use_no_bias_optimizer" + ] + } + ) if "accl1_type" in self.attrs.keys(): ipu_strategy.set_options( - {"accl1_type": self.attrs["accl1_type"]}) + {"accl1_type": self.attrs["accl1_type"]} + ) program = paddle.static.IpuCompiledProgram( - main_prog, - ipu_strategy=ipu_strategy).compile(feed_list, fetch_list) + main_prog, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) else: program = main_prog @@ -118,15 +124,13 @@ def test(self): ipu_loss = self._test_optimizer(True).flatten() cpu_loss = self._test_optimizer(False).flatten() - np.testing.assert_allclose(ipu_loss, - cpu_loss, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + ipu_loss, cpu_loss, rtol=1e-05, atol=self.atol + ) @unittest.skip('do not support L2 regularization') class TestSGD(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'sgd', @@ -137,7 +141,6 @@ def set_attrs(self): @unittest.skip('do not support L2 regularization') class TestAdamCase1(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'adam', @@ -147,7 +150,6 @@ def set_attrs(self): class TestAdamCase2(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'adam', @@ -158,7 +160,6 @@ def set_attrs(self): @unittest.skip('cpu do not support AdamNoBias') class TestAdamNoBias(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'adam', @@ -170,7 +171,6 @@ def set_attrs(self): @unittest.skip('cpu do not support FLOAT16') class TestAdamCase3(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'adam', @@ -182,7 +182,6 @@ def set_attrs(self): @unittest.skip('seems cpu output wrong') class TestLambCase1(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'lamb', @@ -193,7 +192,6 @@ def set_attrs(self): @unittest.skip('seems cpu output wrong') class TestLamb(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'lamb', @@ -204,25 +202,23 @@ def set_attrs(self): @unittest.skip('cpu do not support LambNoBias') class TestLambNoBias(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'lamb', "weight_decay": 0.1, "loss_scaling": 6.0, - "use_no_bias_optimizer": True + "use_no_bias_optimizer": True, } @unittest.skip('cpu do not support FLOAT16') class TestLambCase2(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'lamb', "weight_decay": 0.1, "loss_scaling": 6.0, - "accl1_type": "FLOAT16" + "accl1_type": "FLOAT16", } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_p_norm_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_p_norm_op_ipu.py index bd6ff58751d3f..3cdcc509e4a53 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_p_norm_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_p_norm_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -40,9 +39,9 @@ def set_feed(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) x = paddle.nn.functional.normalize(x, **self.attrs) self.fetch_list = [x.name] @@ -58,13 +57,11 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axis": 1} class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"p": 3.5, "axis": 1, "epsilon": 1e-3} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_pad_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_pad_op_ipu.py index 741fc2189f03e..325b95b689112 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_pad_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_pad_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -40,9 +39,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) pad = paddle.nn.functional.pad(x, **self.attrs) self.fetch_list = [pad.name] @@ -59,15 +58,14 @@ def test(self): @unittest.skip("Do not support `pad` as a tensor") class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {} @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) const_attrs = { 'name': 'y', 'shape': [4], @@ -80,7 +78,6 @@ def build_model(self): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [2, 5], "data_format": "NCL"} @@ -93,7 +90,6 @@ def set_feed(self): class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [2, 5, 2, 3, 6, 3], "data_format": "NCDHW"} @@ -106,42 +102,36 @@ def set_feed(self): class TestCase4(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [2, 2, 1, 1], "mode": "reflect"} @unittest.skip("replicate mode is not supported") class TestCase5(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [1, 2, 3, 4], "mode": "replicate"} @unittest.skip("circular mode is not supported") class TestCase6(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [1, 2, 3, 4], "mode": "circular"} @unittest.skip("Only support NCL, NCHW, NCDHW") class TestCase7(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [1, 2], "data_format": "NLC"} @unittest.skip("Only support NCL, NCHW, NCDHW") class TestCase7(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [1, 2, 3, 4], "data_format": "NHWC"} @unittest.skip("Only support NCL, NCHW, NCDHW") class TestCase7(TestBase): - def set_op_attrs(self): self.attrs = {"pad": [1, 2, 3, 4, 1, 3], "data_format": "NDHWC"} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_pool_avg_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_pool_avg_op_ipu.py index 8a2aa26f1c2d8..058e3b30a5315 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_pool_avg_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_pool_avg_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -53,9 +52,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.pool2d(x, **self.attrs) self.fetch_list = [out.name] @@ -71,42 +70,36 @@ def test(self): class TestCase1(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_size'] = 3 class TestCase1_2(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_size'] = [3, 1] class TestCase2(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_stride'] = 2 class TestCase2_2(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_stride'] = [2, 1] class TestCase3(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_padding'] = [1, 1] class TestCase3_2(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_padding'] = [1, 1, 2, 2] @@ -114,7 +107,6 @@ def set_attrs(self): @unittest.skip('the results has a positional offset') class TestCase3_3(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_padding'] = [1, 2, 1, 1] @@ -122,7 +114,6 @@ def set_attrs(self): @unittest.skip('paddle output has nan') class TestCase3_4(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['pool_size'] = 1 @@ -130,40 +121,36 @@ def set_attrs(self): class TestCase4(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['global_pooling'] = True class TestCase5(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['ceil_mode'] = True class TestCase6(TestBase): - def set_attrs(self): super().set_attrs() self.attrs['exclusive'] = False class TestAdaptive(TestBase): - def set_op_attrs(self): self.attrs = { "pool_size": 1, "pool_type": 'avg', - "require_index": False + "require_index": False, } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.adaptive_pool2d(x, **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_pool_max_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_pool_max_op_ipu.py index dca1103a0cd98..aff790a775a9f 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_pool_max_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_pool_max_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -53,9 +52,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.pool2d(x, **self.attrs) self.fetch_list = [out.name] @@ -71,42 +70,36 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_size'] = 3 class TestCase1_2(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_size'] = [3, 1] class TestCase2(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_stride'] = 2 class TestCase2_2(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_stride'] = [2, 1] class TestCase3(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_padding'] = [1, 1] class TestCase3_2(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_padding'] = [1, 1, 2, 2] @@ -114,7 +107,6 @@ def set_op_attrs(self): @unittest.skip('auto_pad is not currently supported') class TestCase3_3(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_padding'] = 'VALID' @@ -122,47 +114,42 @@ def set_op_attrs(self): @unittest.skip('auto_pad is not currently supported') class TestCase3_4(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['pool_padding'] = 'SAME' class TestCase4(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['global_pooling'] = True class TestCase5(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['ceil_mode'] = True class TestCase6(TestBase): - def set_op_attrs(self): super().set_op_attrs() self.attrs['exclusive'] = False class TestAdaptive(TestBase): - def set_op_attrs(self): self.attrs = { "pool_size": 1, "pool_type": 'max', - "require_index": False + "require_index": False, } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.adaptive_pool2d(x, **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_pow_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_pow_op_ipu.py index 8355f5eefde8c..f61a8b8a24e39 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_pow_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_pow_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.pow(x, **self.attrs) self.fetch_list = [out.name] @@ -62,18 +61,17 @@ def test(self): class TestCase1(TestBase): - def set_data_feed(self): data1 = np.random.uniform(size=[1, 3, 2, 2]) data2 = np.array([2.0]) self.feed_fp32 = { "x": data1.astype(np.float32), - "y": data2.astype(np.float32) + "y": data2.astype(np.float32), } self.feed_fp16 = { "x": data1.astype(np.float16), - "y": data2.astype(np.float16) + "y": data2.astype(np.float16), } def set_op_attrs(self): @@ -81,12 +79,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - factor = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + factor = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.pow(x, factor=factor, **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_prelu_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_prelu_op_ipu.py index b80560dccb3f4..a91b18d10ac50 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_prelu_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_prelu_op_ipu.py @@ -22,7 +22,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -46,9 +45,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) array = np.random.uniform(size=[1]).astype(np.float32) result1 = paddle.zeros(shape=[1], dtype='float32') @@ -70,12 +69,11 @@ def test(self): class TestCase1(TestBase): - @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) array = np.random.uniform(size=[3]).astype(np.float32) result1 = paddle.zeros(shape=[3], dtype='float32') weight = paddle.assign(array, result1) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_print_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_print_op_ipu.py index 57935c8a657f6..70a2eba3c797b 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_print_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_print_op_ipu.py @@ -22,7 +22,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -49,9 +48,11 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) out = paddle.fluid.layers.conv2d(x, num_filters=3, filter_size=3) out = paddle.fluid.layers.Print(out, **self.attrs) @@ -74,13 +75,11 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"message": "input_data"} class TestTrainCase1(TestBase): - def set_op_attrs(self): # "forward" : print forward # "backward" : print forward and backward @@ -94,7 +93,6 @@ def set_training(self): @unittest.skip("attrs are not supported") class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { "first_n": 10, @@ -103,18 +101,16 @@ def set_op_attrs(self): "print_tensor_type": True, "print_tensor_shape": True, "print_tensor_layout": True, - "print_tensor_lod": True + "print_tensor_lod": True, } class SimpleLayer(paddle.nn.Layer): - def __init__(self): super(SimpleLayer, self).__init__() - self.conv = paddle.nn.Conv2D(in_channels=3, - out_channels=1, - kernel_size=2, - stride=1) + self.conv = paddle.nn.Conv2D( + in_channels=3, out_channels=1, kernel_size=2, stride=1 + ) @to_static() def forward(self, x, target=None): @@ -130,7 +126,6 @@ def forward(self, x, target=None): class TestD2S(IPUD2STest): - def setUp(self): self.set_data_feed() @@ -142,16 +137,19 @@ def _test(self, use_ipu=False): paddle.seed(self.SEED) np.random.seed(self.SEED) model = SimpleLayer() - optim = paddle.optimizer.Adam(learning_rate=0.01, - parameters=model.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.01, parameters=model.parameters() + ) if use_ipu: paddle.set_device('ipu') ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(num_ipus=1, - is_training=True, - micro_batch_size=1, - enable_manual_shard=False) + ipu_strategy.set_graph_config( + num_ipus=1, + is_training=True, + micro_batch_size=1, + enable_manual_shard=False, + ) ipu_strategy.set_optimizer(optim) result = [] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_rank_loss_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_rank_loss_op_ipu.py index ad3bbde11923a..bebe0e2232770 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_rank_loss_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_rank_loss_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -49,15 +48,15 @@ def set_feed_attr(self): @IPUOpTest.static_graph def build_model(self, on_ipu): - label = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - left = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') - right = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype='float32') + label = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + left = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) + right = paddle.static.data( + name=self.feed_list[2], shape=self.feed_shape[2], dtype='float32' + ) out = paddle.fluid.layers.rank_loss(label, left, right) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_reduce_x_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_reduce_x_op_ipu.py index c78165f86e21a..9139b1db69af9 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_reduce_x_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_reduce_x_op_ipu.py @@ -21,7 +21,6 @@ class TestMean(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -37,9 +36,9 @@ def set_feed_attr(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = self.op(x, **self.attrs) self.fetch_list = [out.name] @@ -123,39 +122,34 @@ def test_case7(self): class TestMax(TestMean): - def set_test_op(self): self.op = paddle.fluid.layers.reduce_max class TestMin(TestMean): - def set_test_op(self): self.op = paddle.fluid.layers.reduce_min class TestProd(TestMean): - def set_test_op(self): self.op = paddle.fluid.layers.reduce_prod class TestSum(TestMean): - def set_test_op(self): self.op = paddle.fluid.layers.reduce_sum class TestLogsumexp(TestMean): - def set_test_op(self): self.op = paddle.logsumexp @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) if 'dim' in self.attrs: self.attrs['axis'] = self.attrs['dim'] del self.attrs['dim'] @@ -167,7 +161,6 @@ def build_model(self): class TestAll(TestMean): - @property def fp16_enabled(self): return False @@ -184,9 +177,9 @@ def set_data_feed1(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='bool') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='bool' + ) out = self.op(x, **self.attrs) self.fetch_list = [out.name] @@ -195,7 +188,6 @@ def set_test_op(self): class TestAny(TestAll): - def set_test_op(self): self.op = paddle.fluid.layers.reduce_any diff --git a/python/paddle/fluid/tests/unittests/ipu/test_reshape_inplace_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_reshape_inplace_op_ipu.py index 66358d83ee680..c6016d2ec13db 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_reshape_inplace_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_reshape_inplace_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -47,9 +46,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) add = paddle.fluid.layers.elementwise_add(x, x) out = paddle.fluid.layers.reshape(add, **self.attrs) self.fetch_list = [out.name] @@ -66,7 +65,6 @@ def test(self): class TestCase1(TestBase): - def set_attrs(self): self.attrs = { "shape": [-1, 0, 10], diff --git a/python/paddle/fluid/tests/unittests/ipu/test_reshape_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_reshape_op_ipu.py index 2da63cf733004..74bae31111ba1 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_reshape_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_reshape_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,9 +44,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.reshape(x=x, **self.attrs) self.fetch_list = [out.name] @@ -63,7 +62,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['shape'] = [2, 3, -1, 2] @@ -71,7 +69,6 @@ def set_op_attrs(self): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {} self.attrs['shape'] = [-1, 0, 3, 2] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_save_load_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_save_load_ipu.py index 1e8387ac2284f..feed2f739be20 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_save_load_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_save_load_ipu.py @@ -24,7 +24,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_data_feed() @@ -54,14 +53,14 @@ def set_optimizer(self): def build_model(self): generator = paddle.fluid.unique_name.UniqueNameGenerator() with paddle.fluid.unique_name.guard(generator): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - conv1 = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False, - name='conv2d') + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype='float32', + ) + conv1 = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False, name='conv2d' + ) loss = paddle.mean(conv1) # apply optimizer self.optimizer().minimize(loss) @@ -83,7 +82,8 @@ def run_model(self, exec_mode, save_otherwise_load): ipu_strategy.set_precision_config(enable_fp16=True) IPUOpTest.cast_model_to_fp16(self.main_prog) ipu_compiler = paddle.static.IpuCompiledProgram( - self.main_prog, ipu_strategy=ipu_strategy) + self.main_prog, ipu_strategy=ipu_strategy + ) program = ipu_compiler.compile(self.feed_list, self.fetch_list) feed = self.feed_fp32 @@ -91,16 +91,19 @@ def run_model(self, exec_mode, save_otherwise_load): feed = self.feed_fp16 result = [] - run_steps = self.attrs['steps'] if save_otherwise_load \ + run_steps = ( + self.attrs['steps'] + if save_otherwise_load else self.attrs['steps'] - self.attrs['save_at_step'] + ) for i in range(run_steps): tmp = exe.run(program, feed=feed, fetch_list=self.fetch_list) - if save_otherwise_load and \ - i == self.attrs['save_at_step'] - 1: + if save_otherwise_load and i == self.attrs['save_at_step'] - 1: ipu_compiler._backend.weights_to_host() - paddle.static.save(self.main_prog, - self.attrs['model_path'].name) + paddle.static.save( + self.main_prog, self.attrs['model_path'].name + ) if save_otherwise_load and i >= self.attrs['save_at_step']: result.append(tmp) @@ -112,72 +115,61 @@ def run_model(self, exec_mode, save_otherwise_load): def test_base(self): res0 = self.run_model(IPUOpTest.ExecutionMode.IPU_FP32, True) res1 = self.run_model(IPUOpTest.ExecutionMode.IPU_FP32, False) - np.testing.assert_allclose(res0.flatten(), - res1.flatten(), - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + res0.flatten(), res1.flatten(), rtol=1e-05, atol=self.atol + ) self.attrs['model_path'].cleanup() class TestMomentum(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Momentum, learning_rate=1e-1) class TestAdam(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adam, learning_rate=1e-1) class TestLamb(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Lamb, learning_rate=1e-1) class TestAdamW(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.AdamW, learning_rate=1e-1) class TestAdamax(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adamax, learning_rate=1e-1) class TestAdagrad(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adagrad, learning_rate=1e-1) class TestAdadelta(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adagrad, learning_rate=1e-1) class TestRMSProp(TestBase): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.RMSProp, learning_rate=1e-1) class TestCenteredRMSProp(TestBase): - def set_optimizer(self): - self.optimizer = partial(paddle.optimizer.RMSProp, - learning_rate=1e-1, - centered=True) + self.optimizer = partial( + paddle.optimizer.RMSProp, learning_rate=1e-1, centered=True + ) @unittest.skipIf(IPUOpTest.use_ipumodel(), "skip for ipumodel") class TestSGDFP16(TestBase): - def set_attrs(self): self.attrs = {} self.attrs['steps'] = 100 @@ -190,67 +182,57 @@ def set_optimizer(self): def test_base(self): res0 = self.run_model(IPUOpTest.ExecutionMode.IPU_FP16, True) res1 = self.run_model(IPUOpTest.ExecutionMode.IPU_FP16, False) - np.testing.assert_allclose(res0.flatten(), - res1.flatten(), - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + res0.flatten(), res1.flatten(), rtol=1e-05, atol=self.atol + ) self.attrs['model_path'].cleanup() class TestMomentumFp16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Momentum, learning_rate=1e-1) class TestAdamFP16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adam, learning_rate=1e-1) class TestLambFP16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Lamb, learning_rate=1e-1) class TestAdamWFP16FP16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.AdamW, learning_rate=1e-1) class TestAdamaxFP16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adamax, learning_rate=1e-1) class TestAdagradFP16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adagrad, learning_rate=1e-1) class TestAdadeltaFP16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.Adagrad, learning_rate=1e-1) class TestRMSPropFP16(TestSGDFP16): - def set_optimizer(self): self.optimizer = partial(paddle.optimizer.RMSProp, learning_rate=1e-1) class TestCenteredRMSPropFP16(TestSGDFP16): - def set_optimizer(self): - self.optimizer = partial(paddle.optimizer.RMSProp, - learning_rate=1e-1, - centered=True) + self.optimizer = partial( + paddle.optimizer.RMSProp, learning_rate=1e-1, centered=True + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_scale_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_scale_op_ipu.py index 296d365fea602..44ef0aa9fa4be 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_scale_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_scale_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -52,9 +51,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.scale(x, **self.attrs) self.fetch_list = [out.name] @@ -70,7 +69,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "scale": 5.0, @@ -80,7 +78,6 @@ def set_op_attrs(self): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { "scale": 1.0, @@ -90,7 +87,6 @@ def set_op_attrs(self): class TestCase3(TestBase): - def set_op_attrs(self): self.attrs = { "scale": 5.0, @@ -100,7 +96,6 @@ def set_op_attrs(self): class TestCase4(TestBase): - def set_op_attrs(self): self.attrs = { "scale": 1.0, @@ -110,7 +105,6 @@ def set_op_attrs(self): class TestCase5(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3, 3, 10, 10]) y = np.array([3.0]) @@ -125,12 +119,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.scale(x, scale=y, **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_scaled_optimizer_state_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_scaled_optimizer_state_ipu.py index 3960be248eca9..ca6d99ab2e50f 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_scaled_optimizer_state_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_scaled_optimizer_state_ipu.py @@ -20,7 +20,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -50,26 +49,27 @@ def set_attrs(self): self.attrs = { "optimizer": 'lamb', "weight_decay": 0.0, - "scaled_optimizer_state": True + "scaled_optimizer_state": True, } @IPUOpTest.static_graph def build_model(self): - image = paddle.static.data(name='image', - shape=[1, 3, 10, 10], - dtype='float32') - conv1 = paddle.static.nn.conv2d(image, - num_filters=3, - filter_size=3, - bias_attr=False) + image = paddle.static.data( + name='image', shape=[1, 3, 10, 10], dtype='float32' + ) + conv1 = paddle.static.nn.conv2d( + image, num_filters=3, filter_size=3, bias_attr=False + ) loss = paddle.mean(conv1) weight_decay = self.attrs['weight_decay'] - opt = paddle.optimizer.Adam(learning_rate=1e-1, - weight_decay=weight_decay) + opt = paddle.optimizer.Adam( + learning_rate=1e-1, weight_decay=weight_decay + ) if self.attrs['optimizer'] == 'lamb': - opt = paddle.optimizer.Lamb(learning_rate=1e-1, - lamb_weight_decay=weight_decay) + opt = paddle.optimizer.Lamb( + learning_rate=1e-1, lamb_weight_decay=weight_decay + ) opt.minimize(loss) self.feed_list = [image.name] self.fetch_list = [loss.name] @@ -79,15 +79,21 @@ def run_model(self, exec_mode): ipu_strategy.set_graph_config(is_training=self.is_training) if self.is_ipu_mode(exec_mode): if "use_no_bias_optimizer" in self.attrs.keys(): - ipu_strategy.set_options({ - "use_no_bias_optimizer": - self.attrs["use_no_bias_optimizer"] - }) + ipu_strategy.set_options( + { + "use_no_bias_optimizer": self.attrs[ + "use_no_bias_optimizer" + ] + } + ) if "scaled_optimizer_state" in self.attrs.keys(): - ipu_strategy.set_options({ - "scaled_optimizer_state": - self.attrs["scaled_optimizer_state"] - }) + ipu_strategy.set_options( + { + "scaled_optimizer_state": self.attrs[ + "scaled_optimizer_state" + ] + } + ) self.run_op_test(exec_mode, ipu_strategy=ipu_strategy) def test(self): @@ -99,12 +105,11 @@ def test(self): class TestScaledAdam(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'adam', "weight_decay": 0.0, - "scaled_optimizer_state": True + "scaled_optimizer_state": True, } def set_atol(self): @@ -115,25 +120,23 @@ def set_atol(self): @unittest.skip('cpu do not support AdamNoBias') class TestScaledAdamNoBias(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'adam', "weight_decay": 0.0, "use_no_bias_optimizer": True, - "scaled_optimizer_state": True + "scaled_optimizer_state": True, } @unittest.skip('cpu do not support LambNoBias') class TestScaledLambNoBias(TestBase): - def set_attrs(self): self.attrs = { "optimizer": 'lamb', "weight_decay": 0.0, "use_no_bias_optimizer": True, - "scaled_optimizer_state": True + "scaled_optimizer_state": True, } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_set_batch_size_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_set_batch_size_ipu.py index 9bce0b5df73df..6cd27f2472335 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_set_batch_size_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_set_batch_size_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -49,31 +48,28 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - conv1 = paddle.static.nn.conv2d(x, - num_filters=3, - filter_size=3, - bias_attr=False) - conv2 = paddle.static.nn.conv2d(conv1, - num_filters=3, - filter_size=3, - bias_attr=False) - conv3 = paddle.static.nn.conv2d(conv2, - num_filters=3, - filter_size=3, - bias_attr=False) - conv4 = paddle.static.nn.conv2d(conv3, - num_filters=3, - filter_size=3, - bias_attr=False) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + conv1 = paddle.static.nn.conv2d( + x, num_filters=3, filter_size=3, bias_attr=False + ) + conv2 = paddle.static.nn.conv2d( + conv1, num_filters=3, filter_size=3, bias_attr=False + ) + conv3 = paddle.static.nn.conv2d( + conv2, num_filters=3, filter_size=3, bias_attr=False + ) + conv4 = paddle.static.nn.conv2d( + conv3, num_filters=3, filter_size=3, bias_attr=False + ) self.fetch_list = [conv4.name] def run_model(self, exec_mode): ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(is_training=self.is_training, - micro_batch_size=2) + ipu_strategy.set_graph_config( + is_training=self.is_training, micro_batch_size=2 + ) self.run_op_test(exec_mode, ipu_strategy) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ipu/test_set_ipu_shard_api.py b/python/paddle/fluid/tests/unittests/ipu/test_set_ipu_shard_api.py index 37423fda1fa9b..ccd0b6b0d2eed 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_set_ipu_shard_api.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_set_ipu_shard_api.py @@ -23,7 +23,6 @@ class SimpleNet(paddle.nn.Layer): - def __init__(self, input_size, output_size): super(SimpleNet, self).__init__() self.linear1 = nn.Linear(input_size, output_size) @@ -46,15 +45,14 @@ def linear_relu2(self, x): class TestSetIpuShard(unittest.TestCase): - def _test(self): # build graph main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog): x = paddle.static.data(name='X', shape=[10, 46], dtype='float32') - label = paddle.static.data(name='Y', - shape=[10, 46], - dtype='float32') + label = paddle.static.data( + name='Y', shape=[10, 46], dtype='float32' + ) model = SimpleNet(46, 46) set_ipu_shard(model.linear1, index=1) @@ -74,22 +72,20 @@ def test_set_ipu_shard(self): ipu_index_list = self._test() expected_ipu_index_list = [1, 1, 2, 3, 3, 3, 4, 4] - np.testing.assert_allclose(ipu_index_list, - expected_ipu_index_list, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + ipu_index_list, expected_ipu_index_list, rtol=1e-05, atol=0 + ) class TestSetIpuPipeline(unittest.TestCase): - def _test(self): # build graph main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog): x = paddle.static.data(name='X', shape=[10, 46], dtype='float32') - label = paddle.static.data(name='Y', - shape=[10, 46], - dtype='float32') + label = paddle.static.data( + name='Y', shape=[10, 46], dtype='float32' + ) model = SimpleNet(46, 46) set_ipu_shard(model.linear1, stage=1) @@ -109,29 +105,27 @@ def test_set_ipu_shard(self): ipu_index_list = self._test() expected_ipu_index_list = [1, 1, 2, 3, 3, 3, 4, 4] - np.testing.assert_allclose(ipu_index_list, - expected_ipu_index_list, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + ipu_index_list, expected_ipu_index_list, rtol=1e-05, atol=0 + ) class TestSetIpuShardAndPipeline(unittest.TestCase): - def _test(self): # build graph main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog): x = paddle.static.data(name='X', shape=[10, 46], dtype='float32') - label = paddle.static.data(name='Y', - shape=[10, 46], - dtype='float32') + label = paddle.static.data( + name='Y', shape=[10, 46], dtype='float32' + ) model = SimpleNet(46, 46) set_ipu_shard(model.linear1, index=1, stage=2) set_ipu_shard(model.relu1, index=2, stage=3) - model.linear_relu2 = set_ipu_shard(model.linear_relu2, - index=3, - stage=4) + model.linear_relu2 = set_ipu_shard( + model.linear_relu2, index=3, stage=4 + ) model.linear3 = set_ipu_shard(model.linear3, index=4, stage=1) out = model(x) @@ -148,25 +142,38 @@ def _test(self): def test_set_ipu_shard(self): ipu_index_list = self._test() expected_ipu_index_list = [ - 1, 1, 2, 3, 3, 3, 4, 4, 2, 2, 3, 4, 4, 4, 1, 1 + 1, + 1, + 2, + 3, + 3, + 3, + 4, + 4, + 2, + 2, + 3, + 4, + 4, + 4, + 1, + 1, ] - np.testing.assert_allclose(ipu_index_list, - expected_ipu_index_list, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + ipu_index_list, expected_ipu_index_list, rtol=1e-05, atol=0 + ) class TestSetIpuForModel(unittest.TestCase): - def _test(self): # build graph main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog): x = paddle.static.data(name='X', shape=[10, 46], dtype='float32') - label = paddle.static.data(name='Y', - shape=[10, 46], - dtype='float32') + label = paddle.static.data( + name='Y', shape=[10, 46], dtype='float32' + ) model = SimpleNet(46, 46) set_ipu_shard(model, index=1, stage=2) @@ -185,19 +192,31 @@ def _test(self): def test_set_ipu_shard(self): ipu_index_list = self._test() expected_ipu_index_list = [ - 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, ] - np.testing.assert_allclose(ipu_index_list, - expected_ipu_index_list, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + ipu_index_list, expected_ipu_index_list, rtol=1e-05, atol=0 + ) class TestSetIpuMixedModel(unittest.TestCase): - def setUp(self): - def linear_relu2_mixed(self, x): with paddle.static.ipu_shard_guard(index=2, stage=3): x = self.linear2(x) @@ -216,9 +235,9 @@ def _test(self): main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog): x = paddle.static.data(name='X', shape=[10, 46], dtype='float32') - label = paddle.static.data(name='Y', - shape=[10, 46], - dtype='float32') + label = paddle.static.data( + name='Y', shape=[10, 46], dtype='float32' + ) model = SimpleNet(46, 46) set_ipu_shard(model.linear1, index=1, stage=2) @@ -239,13 +258,27 @@ def _test(self): def test_set_ipu_shard(self): ipu_index_list = self._test() expected_ipu_index_list = [ - 1, 1, 2, 2, 2, 3, 4, 4, 2, 2, 3, 3, 3, 4, 1, 1 + 1, + 1, + 2, + 2, + 2, + 3, + 4, + 4, + 2, + 2, + 3, + 3, + 3, + 4, + 1, + 1, ] - np.testing.assert_allclose(ipu_index_list, - expected_ipu_index_list, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + ipu_index_list, expected_ipu_index_list, rtol=1e-05, atol=0 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_share_data_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_share_data_op_ipu.py index 7d355858f1364..59acdf921983b 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_share_data_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_share_data_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -39,15 +38,15 @@ def set_feed_attr(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - self.main_prog.global_block().append_op(type="share_data", - inputs={"X": x}, - outputs={'Out': y}) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + self.main_prog.global_block().append_op( + type="share_data", inputs={"X": x}, outputs={'Out': y} + ) out = paddle.fluid.layers.elementwise_add(y, y) self.fetch_list = [out.name] @@ -63,7 +62,6 @@ def test(self): class TestCase1(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[2, 3, 1]) self.feed_fp32 = {'in_0': data.astype(np.float32)} @@ -74,15 +72,15 @@ def set_data_feed(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - self.main_prog.global_block().append_op(type="share_data", - inputs={"X": x}, - outputs={'Out': y}) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + self.main_prog.global_block().append_op( + type="share_data", inputs={"X": x}, outputs={'Out': y} + ) out = paddle.fluid.layers.elementwise_add(x, y) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_sigmoid_cross_entropy_with_logits_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_sigmoid_cross_entropy_with_logits_op_ipu.py index 7a6cf28d44389..c6b0f6a494d0b 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_sigmoid_cross_entropy_with_logits_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_sigmoid_cross_entropy_with_logits_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -34,11 +33,11 @@ def set_data_feed(self): label = np.arange(10).reshape([10]) self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.float32) + "label": label.astype(np.float32), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.float16) + "label": label.astype(np.float16), } def set_feed_attr(self): @@ -52,14 +51,15 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self, on_ipu): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.sigmoid_cross_entropy_with_logits( - x, label, **self.attrs) + x, label, **self.attrs + ) self.fetch_list = [out.name] def run_model(self, exec_mode): @@ -74,7 +74,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'ignore_index': 1, @@ -82,7 +81,6 @@ def set_op_attrs(self): class TestCase2(TestBase): - def set_atol(self): # epsilon is added when normalize is True, use larger atol. self.atol = 1e-6 diff --git a/python/paddle/fluid/tests/unittests/ipu/test_slice_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_slice_op_ipu.py index 3bcbe417b9861..6b5f6588bd356 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_slice_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_slice_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -48,9 +47,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.slice(x, **self.attrs) self.fetch_list = [out.name] @@ -66,7 +65,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "axes": [0, 1], @@ -77,7 +75,6 @@ def set_op_attrs(self): @unittest.skip('dynamic graph is not support on IPU') class TestCase2(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[4, 5, 6]) s = np.array([0, 0, 2]) @@ -85,12 +82,12 @@ def set_data_feed(self): self.feed_fp32 = { "x": x.astype(np.float32), "starts": s.astype(np.int32), - "ends": e.astype(np.int32) + "ends": e.astype(np.int32), } self.feed_fp16 = { "x": x.astype(np.float16), "starts": s.astype(np.int32), - "ends": e.astype(np.int32) + "ends": e.astype(np.int32), } def set_op_attrs(self): @@ -98,19 +95,18 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - starts = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') - ends = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype='int32') - out = paddle.fluid.layers.slice(x, - starts=starts, - ends=ends, - **self.attrs) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + starts = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) + ends = paddle.static.data( + name=self.feed_list[2], shape=self.feed_shape[2], dtype='int32' + ) + out = paddle.fluid.layers.slice( + x, starts=starts, ends=ends, **self.attrs + ) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_softmax_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_softmax_op_ipu.py index ebc05942b9358..8c1dfa0427516 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_softmax_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_softmax_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.softmax(x, **self.attrs) self.fetch_list = [out.name] @@ -62,7 +61,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axis": 2} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_softmax_with_cross_entropy_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_softmax_with_cross_entropy_op_ipu.py index d3084154a063e..c5a789942aa9c 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_softmax_with_cross_entropy_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_softmax_with_cross_entropy_op_ipu.py @@ -22,7 +22,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -35,11 +34,11 @@ def set_data_feed(self): label = np.arange(3).reshape([3, 1]) self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } def set_feed_attr(self): @@ -53,17 +52,17 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self, on_ipu): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) if on_ipu: - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) else: - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int64') + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int64' + ) out = F.softmax_with_cross_entropy(x, label, **self.attrs) self.fetch_list = [out.name] @@ -81,7 +80,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { 'soft_label': False, @@ -90,37 +88,34 @@ def set_op_attrs(self): class TestCase2(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[30, 70]) label = np.arange(30).reshape([30, 1]) self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } class TestCase3(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[3, 5, 7]) label = np.random.randint(0, 7, [3, 5, 1], dtype='int64') self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } class TestCase4(TestBase): - def set_op_attrs(self): self.attrs = { 'soft_label': False, @@ -130,17 +125,17 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self, on_ipu): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) if on_ipu: - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) else: - label = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int64') + label = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int64' + ) loss, softmax = F.softmax_with_cross_entropy(x, label, **self.attrs) self.fetch_list = [loss.name, softmax.name] @@ -158,7 +153,6 @@ def test(self): class TestCase5(TestCase4): - def set_op_attrs(self): self.attrs = { 'soft_label': False, @@ -172,16 +166,15 @@ def set_data_feed(self): label = np.random.randint(0, 5, [3, 1, 7, 11], dtype='int64') self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } class TestCase6(TestCase4): - def set_op_attrs(self): self.attrs = { 'soft_label': False, @@ -195,11 +188,11 @@ def set_data_feed(self): label = np.random.randint(0, 7, [3, 5, 1, 9, 11], dtype='int64') self.feed_fp32 = { "x": x.astype(np.float32), - "label": label.astype(np.int64) + "label": label.astype(np.int64), } self.feed_fp16 = { "x": x.astype(np.float16), - "label": label.astype(np.int32) + "label": label.astype(np.int32), } diff --git a/python/paddle/fluid/tests/unittests/ipu/test_split_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_split_op_ipu.py index 8d8c5190692dc..dada2225edf8a 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_split_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_split_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -43,9 +42,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.split(x, **self.attrs) self.fetch_list = [fetch.name for fetch in out] @@ -63,7 +62,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"num_or_sections": [2, 8], "axis": 2} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_squeeze_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_squeeze_op_ipu.py index 9039dfdb3f006..f3e312c863e51 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_squeeze_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_squeeze_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.squeeze(x, **self.attrs) self.fetch_list = [out.name] @@ -62,13 +61,11 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axes": []} class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"axes": [-2]} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_stack_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_stack_op_ipu.py index fa0a48081b4a4..1f1fbf6d789ae 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_stack_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_stack_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -36,12 +35,12 @@ def set_data_feed(self): self.feed_fp32 = { "x": x.astype(np.float32), "y": y.astype(np.float32), - "z": z.astype(np.float32) + "z": z.astype(np.float32), } self.feed_fp16 = { "x": x.astype(np.float16), "y": y.astype(np.float16), - "z": z.astype(np.float16) + "z": z.astype(np.float16), } def set_feed_attr(self): @@ -54,15 +53,15 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') - z = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) + z = paddle.static.data( + name=self.feed_list[2], shape=self.feed_shape[2], dtype='float32' + ) out = paddle.fluid.layers.stack([x, y, z], **self.attrs) self.fetch_list = [out.name] @@ -78,7 +77,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axis": -2} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_strided_slice_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_strided_slice_op_ipu.py index f41ad6074265a..5f1bed2938dbd 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_strided_slice_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_strided_slice_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,14 +43,14 @@ def set_op_attrs(self): "axes": [1, 2, 3], "starts": [-3, 0, 2], "ends": [3, 2, 4], - "strides": [1, 1, 1] + "strides": [1, 1, 1], } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.strided_slice(x, **self.attrs) self.fetch_list = [out.name] @@ -67,7 +66,6 @@ def test(self): class TestCase1(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[2, 4]) self.feed_fp32 = {"in_0": data.astype(np.float32)} @@ -78,13 +76,12 @@ def set_op_attrs(self): "axes": [0, 1], "starts": [1, 3], "ends": [2, 0], - "strides": [1, -1] + "strides": [1, -1], } @unittest.skip('Only strides of 1 or -1 are supported.') class TestCase2(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[2, 4]) self.feed_fp32 = {"in_0": data.astype(np.float32)} @@ -95,13 +92,12 @@ def set_op_attrs(self): "axes": [0, 1], "starts": [1, 3], "ends": [-1, 1000], - "strides": [1, 3] + "strides": [1, 3], } @unittest.skip('dynamic graph is not support on IPU') class TestCase3(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[4, 5, 6]) s = np.array([0, 0, 2]) @@ -109,12 +105,12 @@ def set_data_feed(self): self.feed_fp32 = { "x": x.astype(np.float32), "starts": s.astype(np.int32), - "ends": e.astype(np.int32) + "ends": e.astype(np.int32), } self.feed_fp16 = { "x": x.astype(np.float16), "starts": s.astype(np.int32), - "ends": e.astype(np.int32) + "ends": e.astype(np.int32), } def set_op_attrs(self): @@ -122,19 +118,18 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - starts = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') - ends = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype='int32') - out = paddle.fluid.layers.strided_slice(x, - starts=starts, - ends=ends, - **self.attrs) + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + starts = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) + ends = paddle.static.data( + name=self.feed_list[2], shape=self.feed_shape[2], dtype='int32' + ) + out = paddle.fluid.layers.strided_slice( + x, starts=starts, ends=ends, **self.attrs + ) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_sum_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_sum_op_ipu.py index 3c4f9ff80d557..9ed200626c199 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_sum_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_sum_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -45,12 +44,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.fluid.layers.sum([x, y], **self.attrs) self.fetch_list = [out.name] @@ -66,7 +65,6 @@ def test(self): class TestCase1(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[1, 3, 2, 2]) y = np.random.uniform(size=[1, 3, 2, 2]) @@ -74,25 +72,25 @@ def set_data_feed(self): self.feed_fp32 = { "x": x.astype(np.float32), "y": y.astype(np.float32), - "z": z.astype(np.float32) + "z": z.astype(np.float32), } self.feed_fp16 = { "x": x.astype(np.float16), "y": y.astype(np.float16), - "z": z.astype(np.float16) + "z": z.astype(np.float16), } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') - z = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) + z = paddle.static.data( + name=self.feed_list[2], shape=self.feed_shape[2], dtype='float32' + ) out = paddle.fluid.layers.sum([x, y, z], **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_tile_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_tile_op_ipu.py index ecdbe20f75f1a..becacf6b8be43 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_tile_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_tile_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -46,9 +45,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.tile(x, **self.attrs) self.fetch_list = [out.name] @@ -64,7 +63,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "repeat_times": [2, 3, 2], @@ -73,7 +71,6 @@ def set_op_attrs(self): @unittest.skip('dynamic graph is not support on IPU') class TestCase3(TestBase): - def set_data_feed(self): x = np.random.uniform(size=[4, 5, 6]) r = np.array([3, 2, 4]) @@ -91,12 +88,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - r = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + r = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) out = paddle.tile(x, repeat_times=r, **self.attrs) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_topk_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_topk_op_ipu.py index 4194887ab2f05..bfc31118b4efe 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_topk_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_topk_op_ipu.py @@ -21,7 +21,6 @@ class TestTopKOp(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -50,17 +49,16 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) if not self.use_k_as_const_variable: topk_values, topk_indices = self.op(x, **self.attrs) else: # !important, popart cannot accept non const tensor - K_t = paddle.fluid.layers.fill_constant(shape=[1], - dtype='int32', - value=self.k, - name="in_2") + K_t = paddle.fluid.layers.fill_constant( + shape=[1], dtype='int32', value=self.k, name="in_2" + ) topk_values, topk_indices = self.op(x, K_t, **self.attrs) self.fetch_list = [topk_values.name, topk_indices.name] @@ -83,14 +81,12 @@ def test(self): class TestCase2(TestTopKOp): - def set_test_op(self): self.op = paddle.topk @unittest.skip("Trying to get data as int64 but it is of type int32") class TestCase3(TestTopKOp): - def set_op_attrs(self): self.use_k_as_const_variable = True self.attrs = {} @@ -99,7 +95,6 @@ def set_op_attrs(self): @unittest.skip("Trying to get data as int64 but it is of type int32") class TestCase4(TestCase3): - def set_test_op(self): self.op = paddle.topk diff --git a/python/paddle/fluid/tests/unittests/ipu/test_transpose_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_transpose_op_ipu.py index e8f3cae36739f..34e0457c2ded0 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_transpose_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_transpose_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.transpose(x, **self.attrs) self.fetch_list = [out.name] @@ -62,13 +61,11 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"perm": [0, 1, 2, 3]} class TestCase2(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[1, 2, 3, 4, 5]) self.feed_fp32 = {"x": data.astype(np.float32)} @@ -79,7 +76,6 @@ def set_op_attrs(self): class TestCase_ZeroDim(TestBase): - def set_data_feed(self): data = np.random.uniform(size=[]) self.feed_fp32 = {"x": data.astype(np.float32)} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_unary_ops_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_unary_ops_ipu.py index bbf0f7b6996ed..9670e1da6334d 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_unary_ops_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_unary_ops_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_test_op() @@ -44,9 +43,9 @@ def set_feed_attr(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = self.op(x, **self.op_attrs) self.fetch_list = [out.name] @@ -62,7 +61,6 @@ def test(self): class TestAcos(TestBase): - @property def fp16_enabled(self): return False @@ -77,63 +75,54 @@ def set_test_op(self): class TestAsin(TestAcos): - def set_test_op(self): self.op = paddle.fluid.layers.asin self.op_attrs = {} class TestSinh(TestAcos): - def set_test_op(self): self.op = paddle.fluid.layers.sinh self.op_attrs = {} class TestAtan(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.atan self.op_attrs = {} class TestCeil(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.ceil self.op_attrs = {} class TestCos(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.cos self.op_attrs = {} class TestCosh(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.cosh self.op_attrs = {} class TestErf(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.erf self.op_attrs = {} class TestExp(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.exp self.op_attrs = {} class TestFloor(TestBase): - @property def fp16_enabled(self): return False @@ -144,84 +133,72 @@ def set_test_op(self): class TestLog(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.log self.op_attrs = {} class TestReciprocal(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.reciprocal self.op_attrs = {} class TestRelu(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.relu self.op_attrs = {} class TestRound(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.round self.op_attrs = {} class TestSigmoid(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.sigmoid self.op_attrs = {} class TestSign(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.sign self.op_attrs = {} class TestSin(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.sin self.op_attrs = {} class TestSoftplus(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.softplus self.op_attrs = {} class TestSoftsign(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.softsign self.op_attrs = {} class TestSqrt(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.sqrt self.op_attrs = {} class TestTan(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.tan self.op_attrs = {} class TestTanh(TestBase): - def set_test_op(self): self.op = paddle.fluid.layers.tanh self.op_attrs = {} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_unsqueeze_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_unsqueeze_op_ipu.py index 3f3b9f4f89062..d5f3c5913e4ef 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_unsqueeze_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_unsqueeze_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -44,9 +43,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.fluid.layers.unsqueeze(x, **self.attrs) self.fetch_list = [out.name] @@ -62,13 +61,11 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = {"axes": -1} class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = {"axes": [1, 2]} diff --git a/python/paddle/fluid/tests/unittests/ipu/test_unstack_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_unstack_op_ipu.py index 46900011752e7..382a70ba012d4 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_unstack_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_unstack_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -46,9 +45,9 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) out = paddle.unstack(x, **self.attrs) self.fetch_list = [x.name for x in out] @@ -64,7 +63,6 @@ def test(self): class TestCase1(TestBase): - def set_op_attrs(self): self.attrs = { "axis": 2, @@ -72,7 +70,6 @@ def set_op_attrs(self): class TestCase2(TestBase): - def set_op_attrs(self): self.attrs = { "axis": 0, diff --git a/python/paddle/fluid/tests/unittests/ipu/test_varname_inplace_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_varname_inplace_ipu.py index 56e7eef4cb27f..e42e5b4d54e0b 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_varname_inplace_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_varname_inplace_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -54,9 +53,11 @@ def _test_base(self, run_ipu=True): with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype=self.feed_dtype[0]) + x = paddle.static.data( + name=self.feed_list[0], + shape=self.feed_shape[0], + dtype=self.feed_dtype[0], + ) add1 = paddle.fluid.layers.elementwise_add(x, x) reshape = paddle.fluid.layers.reshape(add1, **self.attrs) add2 = paddle.fluid.layers.elementwise_add(reshape, reshape) @@ -75,7 +76,8 @@ def _test_base(self, run_ipu=True): exe.run(startup_prog) scale1_out = main_prog.global_block().ops[4].output("Out")[0] main_prog.global_block().ops[4]._rename_output( - scale1_out, add2.name) + scale1_out, add2.name + ) main_prog.global_block().ops[5]._rename_input(scale1_out, add2.name) if run_ipu: @@ -83,8 +85,8 @@ def _test_base(self, run_ipu=True): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config(is_training=self.is_training) program = paddle.static.IpuCompiledProgram( - main_prog, - ipu_strategy=ipu_strategy).compile(feed_list, fetch_list) + main_prog, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) else: program = main_prog @@ -95,10 +97,9 @@ def test_base(self): res0 = self._test_base(True) res1 = self._test_base(False) - np.testing.assert_allclose(res0.flatten(), - res1.flatten(), - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + res0.flatten(), res1.flatten(), rtol=1e-05, atol=self.atol + ) self.assertTrue(res0.shape == res1.shape) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_warpctc_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_warpctc_op_ipu.py index d173ae77e5cbc..1161699893ad4 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_warpctc_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_warpctc_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -39,20 +38,24 @@ def set_data_feed(self): self.max_seq_length = 5 self.max_label_length = 3 self.num_classes = 5 - self.logits_length = np.array([self.max_seq_length] * self.batch_size, - dtype=np.int64) - self.labels_length = np.array([self.max_label_length] * self.batch_size, - dtype=np.int64) + self.logits_length = np.array( + [self.max_seq_length] * self.batch_size, dtype=np.int64 + ) + self.labels_length = np.array( + [self.max_label_length] * self.batch_size, dtype=np.int64 + ) self.blank = self.num_classes - 1 self.norm_by_times = False logits = np.random.uniform( - 0.1, 1.0, [self.max_seq_length, self.batch_size, self.num_classes - ]).astype("float32") - labels = np.random.randint(0, - self.num_classes - 1, - [self.batch_size, self.max_label_length], - dtype="int32") + 0.1, 1.0, [self.max_seq_length, self.batch_size, self.num_classes] + ).astype("float32") + labels = np.random.randint( + 0, + self.num_classes - 1, + [self.batch_size, self.max_label_length], + dtype="int32", + ) self.feed_fp32 = { "Logits": logits, @@ -79,26 +82,28 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - data = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype="float32") - logits = paddle.nn.Linear(self.num_classes, - self.num_classes, - bias_attr=False)(data) - labels = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='int32') - input_length = paddle.static.data(name=self.feed_list[2], - shape=self.feed_shape[2], - dtype='int64') - label_length = paddle.static.data(name=self.feed_list[3], - shape=self.feed_shape[3], - dtype='int64') - out = paddle.fluid.layers.warpctc(logits, - labels, - input_length=input_length, - label_length=label_length, - **self.attrs) + data = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype="float32" + ) + logits = paddle.nn.Linear( + self.num_classes, self.num_classes, bias_attr=False + )(data) + labels = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='int32' + ) + input_length = paddle.static.data( + name=self.feed_list[2], shape=self.feed_shape[2], dtype='int64' + ) + label_length = paddle.static.data( + name=self.feed_list[3], shape=self.feed_shape[3], dtype='int64' + ) + out = paddle.fluid.layers.warpctc( + logits, + labels, + input_length=input_length, + label_length=label_length, + **self.attrs + ) loss = paddle.mean(out) adam = paddle.optimizer.Adam(learning_rate=1e-2) adam.minimize(loss) diff --git a/python/paddle/fluid/tests/unittests/ipu/test_weight_decay_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_weight_decay_ipu.py index 605942983212a..613032c6262bf 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_weight_decay_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_weight_decay_ipu.py @@ -24,7 +24,6 @@ @unittest.skipIf(IPUOpTest.use_ipumodel(), "skip for ipumodel") class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_data_feed() @@ -57,7 +56,6 @@ def set_attrs(self): } def _test_optimizer(self, run_ipu=True): - def exclude_fn(param): return param.name.endswith('.w_0') @@ -70,22 +68,23 @@ def exclude_fn(param): with paddle.static.scope_guard(scope): with paddle.static.program_guard(main_prog, startup_prog): - image = paddle.static.data(name='image', - shape=[1, 3, 10, 10], - dtype='float32') + image = paddle.static.data( + name='image', shape=[1, 3, 10, 10], dtype='float32' + ) bias = paddle.fluid.layers.create_parameter( - shape=[1, 3, 10, 10], is_bias=True, dtype='float32') + shape=[1, 3, 10, 10], is_bias=True, dtype='float32' + ) add1 = image + bias - conv1 = paddle.static.nn.conv2d(add1, - num_filters=3, - filter_size=3, - bias_attr=False) + conv1 = paddle.static.nn.conv2d( + add1, num_filters=3, filter_size=3, bias_attr=False + ) loss = paddle.mean(conv1) opt = paddle.optimizer.Lamb( learning_rate=1e-3, lamb_weight_decay=self.attrs['weight_decay'], - exclude_from_weight_decay_fn=exclude_fn) + exclude_from_weight_decay_fn=exclude_fn, + ) opt.minimize(loss) if run_ipu: @@ -102,10 +101,11 @@ def exclude_fn(param): ipu_strategy = paddle.static.IpuStrategy() ipu_strategy.set_graph_config(is_training=True) ipu_strategy.set_options( - {'loss_scaling': self.attrs["loss_scaling"]}) + {'loss_scaling': self.attrs["loss_scaling"]} + ) program = paddle.static.IpuCompiledProgram( - main_prog, - ipu_strategy=ipu_strategy).compile(feed_list, fetch_list) + main_prog, ipu_strategy=ipu_strategy + ).compile(feed_list, fetch_list) else: program = main_prog @@ -121,10 +121,9 @@ def test(self): ipu_loss = self._test_optimizer(True).flatten() cpu_loss = self._test_optimizer(False).flatten() - np.testing.assert_allclose(ipu_loss, - cpu_loss, - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + ipu_loss, cpu_loss, rtol=1e-05, atol=self.atol + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_weight_sharing_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_weight_sharing_ipu.py index 4fe9ef1273357..8eb0edf4cfb9a 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_weight_sharing_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_weight_sharing_ipu.py @@ -21,7 +21,6 @@ class TestWeightSharing(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -51,24 +50,27 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='int64') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='int64' + ) with paddle.static.ipu_shard_guard(index=0, stage=0): y = paddle.fluid.layers.embedding( input=x, size=[768, 768], dtype='float32', param_attr=paddle.fluid.ParamAttr(name='word_embedding'), - is_sparse=False) + is_sparse=False, + ) with paddle.static.ipu_shard_guard(index=1, stage=1): z = paddle.fluid.layers.fc( - input=y, size=768, param_attr=paddle.fluid.ParamAttr(name="fc")) + input=y, size=768, param_attr=paddle.fluid.ParamAttr(name="fc") + ) with paddle.static.ipu_shard_guard(index=0, stage=2): out = paddle.fluid.layers.matmul( x=z, y=self.main_prog.global_block().var('word_embedding'), - transpose_y=True) + transpose_y=True, + ) self.feed_list = [x.name] self.fetch_list = [out.name] @@ -82,15 +84,17 @@ def run_model(self, run_ipu): exe.run(self.startup_prog) if run_ipu: ipu_strategy = paddle.static.IpuStrategy() - ipu_strategy.set_graph_config(num_ipus=2, - is_training=self.is_training, - enable_manual_shard=True) - ipu_strategy.set_pipelining_config(enable_pipelining=True, - batches_per_step=3) + ipu_strategy.set_graph_config( + num_ipus=2, + is_training=self.is_training, + enable_manual_shard=True, + ) + ipu_strategy.set_pipelining_config( + enable_pipelining=True, batches_per_step=3 + ) program = paddle.static.IpuCompiledProgram( - self.main_prog, - ipu_strategy=ipu_strategy).compile(self.feed_list, - self.fetch_list) + self.main_prog, ipu_strategy=ipu_strategy + ).compile(self.feed_list, self.fetch_list) else: program = self.main_prog @@ -102,10 +106,9 @@ def test_base(self): res0 = self.run_model(False) res1 = self.run_model(True) - np.testing.assert_allclose(res0.flatten(), - res1[0].flatten(), - rtol=1e-05, - atol=self.atol) + np.testing.assert_allclose( + res0.flatten(), res1[0].flatten(), rtol=1e-05, atol=self.atol + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ipu/test_where_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_where_op_ipu.py index 091f5dc960df1..15a815bf5a716 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_where_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_where_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -34,11 +33,11 @@ def set_data_feed(self): data_y = np.random.uniform(size=[4, 5, 6]) self.feed_fp32 = { "in_0": data_x.astype(np.float32), - "in_1": data_y.astype(np.float32) + "in_1": data_y.astype(np.float32), } self.feed_fp16 = { "in_0": data_x.astype(np.float16), - "in_1": data_y.astype(np.float16) + "in_1": data_y.astype(np.float16), } def set_feed_attr(self): @@ -51,12 +50,12 @@ def set_op_attrs(self): @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') - y = paddle.static.data(name=self.feed_list[1], - shape=self.feed_shape[1], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) + y = paddle.static.data( + name=self.feed_list[1], shape=self.feed_shape[1], dtype='float32' + ) out = paddle.where(x > 1, x, y) self.fetch_list = [out.name] diff --git a/python/paddle/fluid/tests/unittests/ipu/test_yolo_box_op_ipu.py b/python/paddle/fluid/tests/unittests/ipu/test_yolo_box_op_ipu.py index 3e19ca2b0c9d2..328e46d430c87 100644 --- a/python/paddle/fluid/tests/unittests/ipu/test_yolo_box_op_ipu.py +++ b/python/paddle/fluid/tests/unittests/ipu/test_yolo_box_op_ipu.py @@ -21,7 +21,6 @@ class TestBase(IPUOpTest): - def setUp(self): self.set_atol() self.set_training() @@ -50,14 +49,14 @@ def set_op_attrs(self): "class_num": 80, "anchors": [10, 13, 16, 30, 33, 23], "conf_thresh": 0.01, - "downsample_ratio": 32 + "downsample_ratio": 32, } @IPUOpTest.static_graph def build_model(self): - x = paddle.static.data(name=self.feed_list[0], - shape=self.feed_shape[0], - dtype='float32') + x = paddle.static.data( + name=self.feed_list[0], shape=self.feed_shape[0], dtype='float32' + ) attrs = { 'name': 'img_size', 'shape': [1, 2], diff --git a/python/paddle/fluid/tests/unittests/ir/inference/auto_scan_test.py b/python/paddle/fluid/tests/unittests/ir/inference/auto_scan_test.py index ed191d5560378..d76a17101298f 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/auto_scan_test.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/auto_scan_test.py @@ -24,7 +24,12 @@ from paddle.fluid.core import PassVersionChecker import paddle.inference as paddle_infer from typing import Optional, List, Callable, Dict, Any -from program_config import OpConfig, ProgramConfig, create_fake_model, create_quant_model +from program_config import ( + OpConfig, + ProgramConfig, + create_fake_model, + create_quant_model, +) import hypothesis from hypothesis import given, settings @@ -32,22 +37,28 @@ logging.basicConfig(level=logging.INFO, format="%(message)s") -settings.register_profile("ci", - max_examples=100, - suppress_health_check=hypothesis.HealthCheck.all(), - deadline=None, - print_blob=True, - derandomize=True, - report_multiple_bugs=False) -settings.register_profile("dev", - max_examples=1000, - suppress_health_check=hypothesis.HealthCheck.all(), - deadline=None, - print_blob=True, - derandomize=True, - report_multiple_bugs=False) -if float(os.getenv('TEST_NUM_PERCENT_CASES', default='1.0')) < 1 or \ - os.getenv('HYPOTHESIS_TEST_PROFILE', 'dev') == 'ci': +settings.register_profile( + "ci", + max_examples=100, + suppress_health_check=hypothesis.HealthCheck.all(), + deadline=None, + print_blob=True, + derandomize=True, + report_multiple_bugs=False, +) +settings.register_profile( + "dev", + max_examples=1000, + suppress_health_check=hypothesis.HealthCheck.all(), + deadline=None, + print_blob=True, + derandomize=True, + report_multiple_bugs=False, +) +if ( + float(os.getenv('TEST_NUM_PERCENT_CASES', default='1.0')) < 1 + or os.getenv('HYPOTHESIS_TEST_PROFILE', 'dev') == 'ci' +): settings.load_profile("ci") else: settings.load_profile("dev") @@ -69,15 +80,15 @@ class IgnoreReasons(enum.Enum): class AutoScanTest(unittest.TestCase): - def __init__(self, *args, **kwargs): np.random.seed(1024) paddle.enable_static() super(AutoScanTest, self).__init__(*args, **kwargs) self.ignore_cases = [] abs_dir = os.path.abspath(os.path.dirname(__file__)) - self.cache_dir = os.path.join(abs_dir, - str(self.__module__) + '_cache_dir') + self.cache_dir = os.path.join( + abs_dir, str(self.__module__) + '_cache_dir' + ) self.available_passes_in_framework = set() self.num_ran_programs = 0 self.num_invalid_programs = 0 @@ -97,23 +108,29 @@ def sample_predictor_configs(self): raise NotImplementedError @abc.abstractmethod - def add_ignore_check_case(self, teller: [ - Callable[[ProgramConfig, paddle_infer.Config], bool] - ], reason: IgnoreReasons, note: str): + def add_ignore_check_case( + self, + teller: [Callable[[ProgramConfig, paddle_infer.Config], bool]], + reason: IgnoreReasons, + note: str, + ): self.ignore_cases.append((teller, reason, note)) def is_program_valid(self, program_config: ProgramConfig) -> bool: return True - def run_test_config(self, model, params, prog_config, pred_config, - feed_data) -> Dict[str, np.ndarray]: + def run_test_config( + self, model, params, prog_config, pred_config, feed_data + ) -> Dict[str, np.ndarray]: ''' Test a single case. ''' pred_config.set_model_buffer(model, len(model), params, len(params)) predictor = paddle_infer.create_predictor(pred_config) - self.available_passes_in_framework = self.available_passes_in_framework | set( - pred_config.pass_builder().all_passes()) + self.available_passes_in_framework = ( + self.available_passes_in_framework + | set(pred_config.pass_builder().all_passes()) + ) for name, _ in prog_config.inputs.items(): input_tensor = predictor.get_input_handle(name) input_tensor.copy_from_cpu(feed_data[name]['data']) @@ -121,20 +138,28 @@ def run_test_config(self, model, params, prog_config, pred_config, input_tensor.set_lod(feed_data[name]['lod']) predictor.run() result = {} - for out_name, o_name in zip(prog_config.outputs, - predictor.get_output_names()): + for out_name, o_name in zip( + prog_config.outputs, predictor.get_output_names() + ): result[out_name] = predictor.get_output_handle(o_name).copy_to_cpu() return result @abc.abstractmethod - def assert_tensors_near(self, atol: float, rtol: float, - tensor: Dict[str, np.array], - baseline: Dict[str, np.array]): + def assert_tensors_near( + self, + atol: float, + rtol: float, + tensor: Dict[str, np.array], + baseline: Dict[str, np.array], + ): for key, arr in tensor.items(): self.assertTrue( baseline[key].shape == arr.shape, - "The output shapes are not equal, the baseline shape is " + - str(baseline[key].shape) + ', but got ' + str(arr.shape)) + "The output shapes are not equal, the baseline shape is " + + str(baseline[key].shape) + + ', but got ' + + str(arr.shape), + ) diff = abs(baseline[key] - arr) np.testing.assert_allclose( baseline[key], @@ -142,22 +167,28 @@ def assert_tensors_near(self, atol: float, rtol: float, rtol=rtol, atol=atol, err_msg='Output has diff, Maximum absolute error: {}'.format( - np.amax(diff))) + np.amax(diff) + ), + ) @abc.abstractmethod def run_test(self, quant=False): raise NotImplementedError - def generate_op_config(self, ops_config: List[Dict[str, - Any]]) -> List[OpConfig]: + def generate_op_config( + self, ops_config: List[Dict[str, Any]] + ) -> List[OpConfig]: ops = [] for i in range(len(ops_config)): op_config = ops_config[i] ops.append( - OpConfig(type=op_config['op_type'], - inputs=op_config['op_inputs'], - outputs=op_config['op_outputs'], - attrs=op_config['op_attrs'])) + OpConfig( + type=op_config['op_type'], + inputs=op_config['op_inputs'], + outputs=op_config['op_outputs'], + attrs=op_config['op_attrs'], + ) + ) return ops @abc.abstractmethod @@ -173,11 +204,13 @@ def success_log(self, msg: str): logging.info("SUCCESS: " + msg) @abc.abstractmethod - def create_inference_config(self, - passes: Optional[List[str]] = None, - use_gpu: bool = False, - use_mkldnn: bool = False, - ir_optim: Optional[bool] = None): + def create_inference_config( + self, + passes: Optional[List[str]] = None, + use_gpu: bool = False, + use_mkldnn: bool = False, + ir_optim: Optional[bool] = None, + ): config = paddle_infer.Config() config.switch_ir_debug(True) config.set_optim_cache_dir(self.cache_dir) @@ -195,7 +228,6 @@ def create_inference_config(self, class MkldnnAutoScanTest(AutoScanTest): - def __init__(self, *args, **kwargs): super(MkldnnAutoScanTest, self).__init__(*args, **kwargs) @@ -215,7 +247,7 @@ def run_test(self, quant=False, *args, **kwargs): for name, tensor_config in prog_config.inputs.items(): feed_data[name] = { 'data': tensor_config.data, - 'lod': tensor_config.lod + 'lod': tensor_config.lod, } results: List[Dict[str, np.ndarray]] = [] @@ -223,23 +255,31 @@ def run_test(self, quant=False, *args, **kwargs): base_config = self.create_inference_config(ir_optim=False) logging.info('RUN program_config: ' + str(prog_config)) results.append( - self.run_test_config(model, params, prog_config, base_config, - feed_data)) + self.run_test_config( + model, params, prog_config, base_config, feed_data + ) + ) self.success_log('RUN_CPU_BASELINE done') - for pred_config, ( - atol, rtol) in self.sample_predictor_configs(prog_config): + for pred_config, (atol, rtol) in self.sample_predictor_configs( + prog_config + ): # skip info ignore_flag = False for ignore_info in self.ignore_cases: if ignore_info[0](prog_config, pred_config): ignore_flag = True - if ignore_info[ - 1] == IgnoreReasons.MKLDNN_ACCURACY_ERROR: + if ( + ignore_info[1] + == IgnoreReasons.MKLDNN_ACCURACY_ERROR + ): self.ignore_log( - "[MKLDNN_ACCURACY_ERROR] " + ignore_info[2] + - ' ' + ' vs ' + - self.inference_config_str(pred_config)) + "[MKLDNN_ACCURACY_ERROR] " + + ignore_info[2] + + ' ' + + ' vs ' + + self.inference_config_str(pred_config) + ) else: raise NotImplementedError break @@ -251,20 +291,26 @@ def run_test(self, quant=False, *args, **kwargs): try: results.append( - self.run_test_config(model, params, prog_config, - pred_config, feed_data)) - self.assert_tensors_near(atol, rtol, results[-1], - results[0]) + self.run_test_config( + model, params, prog_config, pred_config, feed_data + ) + ) + self.assert_tensors_near( + atol, rtol, results[-1], results[0] + ) except Exception as e: self.fail_log( - self.inference_config_str(pred_config) + - '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e))) + self.inference_config_str(pred_config) + + '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e)) + ) if not ignore_flag: status = False continue - self.success_log('RUN predictor_config ' + - self.inference_config_str(pred_config) + - ' done') + self.success_log( + 'RUN predictor_config ' + + self.inference_config_str(pred_config) + + ' done' + ) self.assertTrue(status) @@ -278,7 +324,6 @@ def inference_config_str(self, config) -> str: class PassAutoScanTest(AutoScanTest): - def __init__(self, *args, **kwargs): super(PassAutoScanTest, self).__init__(*args, **kwargs) self.passes = [] @@ -299,13 +344,17 @@ def add_ignore_pass_case(self): def assert_op_list(self, op_list_after_fusion): if not self.passes: raise ValueError( - "In PassAutoScan you should give a valid pass name.") - last_passed_program = os.path.join(self.cache_dir, - self.passes[-1] + ".pdmodel") + "In PassAutoScan you should give a valid pass name." + ) + last_passed_program = os.path.join( + self.cache_dir, self.passes[-1] + ".pdmodel" + ) if not os.path.exists(last_passed_program): raise ValueError( - "Cannot find file {}, please make sure that your pass name is correct" - .format(last_passed_program)) + "Cannot find file {}, please make sure that your pass name is correct".format( + last_passed_program + ) + ) model_bytes = paddle.static.load_from_file(last_passed_program) pg = paddle.static.deserialize_program(model_bytes) main_block = pg.desc.block(0) @@ -317,16 +366,19 @@ def assert_op_list(self, op_list_after_fusion): self.assertTrue( op_list_after_fusion == after_op_list, "Expected operator list after fusion is {}, but now it's {}".format( - op_list_after_fusion, after_op_list), + op_list_after_fusion, after_op_list + ), ) - def run_and_statis(self, - quant=False, - max_examples=100, - reproduce=None, - min_success_num=25, - max_duration=180, - passes=None): + def run_and_statis( + self, + quant=False, + max_examples=100, + reproduce=None, + min_success_num=25, + max_duration=180, + passes=None, + ): if os.getenv('HYPOTHESIS_TEST_PROFILE', 'ci') == "dev": max_examples *= 10 min_success_num *= 10 @@ -343,7 +395,9 @@ def run_and_statis(self, report_multiple_bugs=False, ) settings.load_profile("ci") - assert passes is not None, "Parameter of passes must be defined in function run_and_statis." + assert ( + passes is not None + ), "Parameter of passes must be defined in function run_and_statis." self.passes = passes self.add_ignore_pass_case() @@ -361,32 +415,44 @@ def run_test(prog_config): logging.info("Start to running test of {}".format(type(self))) loop_func() logging.info( - "===================Statistical Information===================") - logging.info("Number of Generated Programs: {}".format( - self.num_ran_programs + self.num_invalid_programs)) - logging.info("Number of Invalid Programs: {}".format( - self.num_invalid_programs)) + "===================Statistical Information===================" + ) + logging.info( + "Number of Generated Programs: {}".format( + self.num_ran_programs + self.num_invalid_programs + ) + ) + logging.info( + "Number of Invalid Programs: {}".format(self.num_invalid_programs) + ) logging.info("Number of Ran Programs: {}".format(self.num_ran_programs)) logging.info("Number of Ignore Tests: {}".format(self.num_ignore_tests)) - successful_ran_programs = int(self.num_ran_programs - - self.num_ignore_tests / - max(self.num_predictor_kinds, 1)) + successful_ran_programs = int( + self.num_ran_programs + - self.num_ignore_tests / max(self.num_predictor_kinds, 1) + ) logging.info( - "Number of successfully ran programs approximately equal to {}". - format(successful_ran_programs)) + "Number of successfully ran programs approximately equal to {}".format( + successful_ran_programs + ) + ) if successful_ran_programs < min_success_num: logging.warning( "satisfied_programs = ran_programs - num_ignore_tests / num_predictor_kinds" ) logging.error( - "At least {} programs need to ran successfully, but now only about {} programs satisfied." - .format(min_success_num, successful_ran_programs)) + "At least {} programs need to ran successfully, but now only about {} programs satisfied.".format( + min_success_num, successful_ran_programs + ) + ) assert False used_time = time.time() - start_time if max_duration > 0 and used_time > max_duration: logging.error( - "The duration exceeds {} seconds, if this is necessary, try to set a larger number for parameter `max_duration`." - .format(max_duration)) + "The duration exceeds {} seconds, if this is necessary, try to set a larger number for parameter `max_duration`.".format( + max_duration + ) + ) assert False def run_test(self, quant=False, prog_configs=None): @@ -406,13 +472,16 @@ def run_test(self, quant=False, prog_configs=None): for name, tensor_config in prog_config.inputs.items(): feed_data[name] = { 'data': tensor_config.data, - 'lod': tensor_config.lod + 'lod': tensor_config.lod, } logging.info('RUN program_config: ' + str(prog_config)) self.num_predictor_kinds = 0 - for pred_config, op_list, ( - atol, rtol) in self.sample_predictor_configs(prog_config): + for ( + pred_config, + op_list, + (atol, rtol), + ) in self.sample_predictor_configs(prog_config): self.num_predictor_kinds += 1 # skip info @@ -423,9 +492,12 @@ def run_test(self, quant=False, prog_configs=None): self.num_ignore_tests += 1 if ignore_info[1] == IgnoreReasons.PASS_ACCURACY_ERROR: self.ignore_log( - "[PASS_ACCURACY_ERROR] " + ignore_info[2] + - ' ' + ' vs ' + - self.inference_config_str(pred_config)) + "[PASS_ACCURACY_ERROR] " + + ignore_info[2] + + ' ' + + ' vs ' + + self.inference_config_str(pred_config) + ) else: raise NotImplementedError break @@ -437,37 +509,44 @@ def run_test(self, quant=False, prog_configs=None): # baseline: no ir_optim run base_config = self.create_inference_config( - ir_optim=False, use_gpu=pred_config.use_gpu()) + ir_optim=False, use_gpu=pred_config.use_gpu() + ) try: # baseline - base_result = self.run_test_config(model, params, - prog_config, base_config, - feed_data) - self.success_log('RUN_BASELINE ' + - self.inference_config_str(base_config) + - ' done') + base_result = self.run_test_config( + model, params, prog_config, base_config, feed_data + ) + self.success_log( + 'RUN_BASELINE ' + + self.inference_config_str(base_config) + + ' done' + ) if os.path.exists(self.cache_dir): shutil.rmtree(self.cache_dir) - pred_result = self.run_test_config(model, params, - prog_config, pred_config, - feed_data) - self.assert_tensors_near(atol, rtol, pred_result, - base_result) + pred_result = self.run_test_config( + model, params, prog_config, pred_config, feed_data + ) + self.assert_tensors_near( + atol, rtol, pred_result, base_result + ) if not ignore_flag: self.assert_op_list(op_list) except Exception as e: self.fail_log( - self.inference_config_str(pred_config) + - '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e))) + self.inference_config_str(pred_config) + + '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e)) + ) if not ignore_flag: status = False continue - self.success_log('RUN predictor_config ' + - self.inference_config_str(pred_config) + - ' done') + self.success_log( + 'RUN predictor_config ' + + self.inference_config_str(pred_config) + + ' done' + ) status = self.check_op_version() and status self.assertTrue(status) @@ -502,14 +581,20 @@ def create_trt_inference_config(self) -> paddle_infer.Config: class TrtLayerAutoScanTest(AutoScanTest): - class TensorRTParam: ''' TensorRT subgraph engine parameters. ''' - def __init__(self, workspace_size, max_batch_size, min_subgraph_size, - precision, use_static, use_calib_mode): + def __init__( + self, + workspace_size, + max_batch_size, + min_subgraph_size, + precision, + use_static, + use_calib_mode, + ): self.workspace_size = workspace_size self.max_batch_size = max_batch_size self.min_subgraph_size = min_subgraph_size @@ -519,11 +604,16 @@ def __init__(self, workspace_size, max_batch_size, min_subgraph_size, class DynamicShapeParam: ''' - Prepare TensorRT subgraph engine dynamic shape parameters. - ''' + Prepare TensorRT subgraph engine dynamic shape parameters. + ''' - def __init__(self, min_input_shape, max_input_shape, opt_input_shape, - disable_trt_plugin_fp16): + def __init__( + self, + min_input_shape, + max_input_shape, + opt_input_shape, + disable_trt_plugin_fp16, + ): self.min_input_shape = min_input_shape self.max_input_shape = max_input_shape self.opt_input_shape = opt_input_shape @@ -537,10 +627,12 @@ def __init__(self, *args, **kwargs): min_subgraph_size=0, precision=paddle_infer.PrecisionType.Float32, use_static=True, - use_calib_mode=False) + use_calib_mode=False, + ) self.dynamic_shape = self.DynamicShapeParam({}, {}, {}, False) self.num_percent_cases = float( - os.getenv('TEST_NUM_PERCENT_CASES', default='1.0')) + os.getenv('TEST_NUM_PERCENT_CASES', default='1.0') + ) # Use a seperate random generator for skipping tests self.skip_rng = np.random.default_rng(int(time.strftime("%W"))) @@ -558,31 +650,43 @@ def create_inference_config(self, use_trt=True) -> paddle_infer.Config: min_subgraph_size=self.trt_param.min_subgraph_size, precision_mode=self.trt_param.precision, use_static=self.trt_param.use_static, - use_calib_mode=self.trt_param.use_calib_mode) + use_calib_mode=self.trt_param.use_calib_mode, + ) if self.dynamic_shape.min_input_shape and ( - self.dynamic_shape.min_input_shape.keys() == - self.dynamic_shape.max_input_shape.keys() == - self.dynamic_shape.opt_input_shape.keys()): + self.dynamic_shape.min_input_shape.keys() + == self.dynamic_shape.max_input_shape.keys() + == self.dynamic_shape.opt_input_shape.keys() + ): config.set_trt_dynamic_shape_info( self.dynamic_shape.min_input_shape, self.dynamic_shape.max_input_shape, self.dynamic_shape.opt_input_shape, - self.dynamic_shape.disable_trt_plugin_fp16) + self.dynamic_shape.disable_trt_plugin_fp16, + ) return config - def assert_tensors_near(self, atol: float, rtol: float, - tensor: Dict[str, np.array], - baseline: Dict[str, np.array]): + def assert_tensors_near( + self, + atol: float, + rtol: float, + tensor: Dict[str, np.array], + baseline: Dict[str, np.array], + ): for key, arr in tensor.items(): self.assertEqual( - baseline[key].shape, arr.shape, - 'The output shapes are not equal, the baseline shape is ' + - str(baseline[key].shape) + ', but got ' + str(arr.shape)) + baseline[key].shape, + arr.shape, + 'The output shapes are not equal, the baseline shape is ' + + str(baseline[key].shape) + + ', but got ' + + str(arr.shape), + ) np.testing.assert_allclose(baseline[key], arr, rtol=rtol, atol=atol) def assert_op_size(self, trt_engine_num, paddle_op_num): last_passed_program = os.path.join( - self.cache_dir, 'transpose_flatten_concat_fuse_pass.pdmodel') + self.cache_dir, 'transpose_flatten_concat_fuse_pass.pdmodel' + ) model_bytes = paddle.static.load_from_file(last_passed_program) pg = paddle.static.deserialize_program(model_bytes) main_block = pg.desc.block(0) @@ -593,13 +697,19 @@ def assert_op_size(self, trt_engine_num, paddle_op_num): trt_engine_size = sum(op_types) paddle_op_size = op_size - trt_engine_size self.assertEqual( - trt_engine_num, trt_engine_size, + trt_engine_num, + trt_engine_size, 'Expected trt_engine_num is {}, but got {}!'.format( - trt_engine_num, trt_engine_size)) + trt_engine_num, trt_engine_size + ), + ) self.assertEqual( - paddle_op_num, paddle_op_size, + paddle_op_num, + paddle_op_size, 'Expected paddle_op_num is {}, but got {}!'.format( - paddle_op_num, paddle_op_size)) + paddle_op_num, paddle_op_size + ), + ) def inference_config_str(self, config: paddle_infer.Config) -> str: dic = {} @@ -639,21 +749,26 @@ def random_to_skip(): for name, tensor_config in prog_config.inputs.items(): feed_data[name] = { 'data': tensor_config.data, - 'lod': tensor_config.lod + 'lod': tensor_config.lod, } results: List[Dict[str, np.ndarray]] = [] if not skip_baseline: - #baseline: gpu run + # baseline: gpu run logging.info('RUN program_config: ' + str(prog_config)) gpu_config = self.create_inference_config(use_trt=False) results.append( - self.run_test_config(model, params, prog_config, gpu_config, - feed_data)) + self.run_test_config( + model, params, prog_config, gpu_config, feed_data + ) + ) self.success_log('RUN_GPU_BASELINE done') - for pred_config, nodes_num, threshold in self.sample_predictor_configs( - prog_config): + for ( + pred_config, + nodes_num, + threshold, + ) in self.sample_predictor_configs(prog_config): if os.path.exists(self.cache_dir): shutil.rmtree(self.cache_dir) @@ -662,17 +777,24 @@ def random_to_skip(): atol = threshold rtol = 1e-8 elif isinstance(threshold, list) or isinstance( - threshold, tuple): + threshold, tuple + ): atol = threshold[0] rtol = threshold[1] else: raise NotImplementedError - if pred_config.tensorrt_precision_mode( - ) != paddle_infer.PrecisionType.Int8 and quant: + if ( + pred_config.tensorrt_precision_mode() + != paddle_infer.PrecisionType.Int8 + and quant + ): continue - if pred_config.tensorrt_precision_mode( - ) == paddle_infer.PrecisionType.Int8 and not quant: + if ( + pred_config.tensorrt_precision_mode() + == paddle_infer.PrecisionType.Int8 + and not quant + ): continue ignore_flag = False @@ -682,11 +804,15 @@ def random_to_skip(): if reason == IgnoreReasons.TRT_NOT_IMPLEMENTED: self.ignore_log( '[TRT_NOT_IMPLEMENTED] {} vs {}'.format( - note, - self.inference_config_str(pred_config))) + note, self.inference_config_str(pred_config) + ) + ) elif reason == IgnoreReasons.TRT_NOT_SUPPORT: - self.ignore_log('[TRT_NOT_SUPPORT] {} vs {}'.format( - note, self.inference_config_str(pred_config))) + self.ignore_log( + '[TRT_NOT_SUPPORT] {} vs {}'.format( + note, self.inference_config_str(pred_config) + ) + ) else: raise NotImplementedError break @@ -697,30 +823,45 @@ def random_to_skip(): try: pred_config_deserialize = paddle_infer.Config(pred_config) results.append( - self.run_test_config(model, params, prog_config, - pred_config, feed_data)) - self.assert_tensors_near(atol, rtol, results[-1], - results[0]) + self.run_test_config( + model, params, prog_config, pred_config, feed_data + ) + ) + self.assert_tensors_near( + atol, rtol, results[-1], results[0] + ) trt_engine_num, paddle_op_num = nodes_num self.assert_op_size(trt_engine_num, paddle_op_num) # deserialize test if trt_engine_num > 0: - self.run_test_config(model, params, prog_config, - pred_config_deserialize, feed_data) - - self.success_log('RUN predictor_config {} done'.format( - self.inference_config_str(pred_config))) + self.run_test_config( + model, + params, + prog_config, + pred_config_deserialize, + feed_data, + ) + + self.success_log( + 'RUN predictor_config {} done'.format( + self.inference_config_str(pred_config) + ) + ) except Exception as e: self.fail_log( - self.inference_config_str(pred_config) + - '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e))) + self.inference_config_str(pred_config) + + '\033[1;31m \nERROR INFO: {}\033[0m'.format(str(e)) + ) all_passes = False self.assertTrue(all_passes) # TODO(wilber): just for backward compatible - def add_skip_case(self, teller: [ - Callable[[ProgramConfig, paddle_infer.Config], bool] - ], reason: IgnoreReasons, note: str): + def add_skip_case( + self, + teller: [Callable[[ProgramConfig, paddle_infer.Config], bool]], + reason: IgnoreReasons, + note: str, + ): self.ignore_cases.append((teller, reason, note)) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/inference_pass_test.py b/python/paddle/fluid/tests/unittests/ir/inference/inference_pass_test.py index 60980fbb732f8..cc493349037e4 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/inference_pass_test.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/inference_pass_test.py @@ -24,7 +24,6 @@ class InferencePassTest(unittest.TestCase): - def __init__(self, methodName='runTest'): paddle.enable_static() super(InferencePassTest, self).__init__(methodName) @@ -49,24 +48,28 @@ def __init__(self, methodName='runTest'): def _get_place(self): return set([False, core.is_compiled_with_cuda()]) - def _save_models(self, dirname, feeded_var_names, target_vars, executor, - program, scope): + def _save_models( + self, dirname, feeded_var_names, target_vars, executor, program, scope + ): with fluid.scope_guard(scope): # save models as combined to ensure that # there won't be too many useless files # after finishing a couple of tests. - fluid.io.save_inference_model(dirname, feeded_var_names, - target_vars, executor, program) + fluid.io.save_inference_model( + dirname, feeded_var_names, target_vars, executor, program + ) def _get_paddle_outs(self, executor, program, scope): ''' Return PaddlePaddle outputs. ''' with fluid.scope_guard(scope): - outs = executor.run(program=program, - feed=self.feeds, - fetch_list=self.fetch_list, - return_numpy=False) + outs = executor.run( + program=program, + feed=self.feeds, + fetch_list=self.fetch_list, + return_numpy=False, + ) return outs def _get_inference_outs(self, config): @@ -95,10 +98,9 @@ def _get_inference_outs(self, config): return outs - def _get_analysis_config(self, - use_gpu=False, - use_trt=False, - use_mkldnn=False): + def _get_analysis_config( + self, use_gpu=False, use_trt=False, use_mkldnn=False + ): ''' Return a new object of AnalysisConfig. ''' @@ -116,19 +118,22 @@ def _get_analysis_config(self, self.trt_parameters.min_subgraph_size, self.trt_parameters.precision, self.trt_parameters.use_static, - self.trt_parameters.use_calib_mode) + self.trt_parameters.use_calib_mode, + ) if self.trt_parameters.use_inspector: config.enable_tensorrt_inspector() self.assertTrue( config.tensorrt_inspector_enabled(), - "The inspector option is not set correctly.") + "The inspector option is not set correctly.", + ) if self.dynamic_shape_params: config.set_trt_dynamic_shape_info( self.dynamic_shape_params.min_input_shape, self.dynamic_shape_params.max_input_shape, self.dynamic_shape_params.optim_input_shape, - self.dynamic_shape_params.disable_trt_plugin_fp16) + self.dynamic_shape_params.disable_trt_plugin_fp16, + ) if self.enable_tensorrt_varseqlen: config.enable_tensorrt_varseqlen() @@ -145,18 +150,16 @@ def check_output(self, atol=1e-5): or disable TensorRT, enable MKLDNN or disable MKLDNN are all the same. ''' - self.assertFalse(self.feeds is None, - "The inputs of the model is None. ") + self.assertFalse( + self.feeds is None, "The inputs of the model is None. " + ) use_gpu = self._get_place() for place_ in use_gpu: self.check_output_with_option(place_, atol) - def check_output_with_option(self, - use_gpu, - atol=1e-5, - flatten=False, - quant=False, - rtol=1e-5): + def check_output_with_option( + self, use_gpu, atol=1e-5, flatten=False, quant=False, rtol=1e-5 + ): ''' Check whether calculating on CPU and GPU, enable TensorRT or disable TensorRT, enable MKLDNN or disable MKLDNN @@ -168,17 +171,26 @@ def check_output_with_option(self, device = "GPU" if use_gpu else "CPU" with fluid.scope_guard(scope): executor.run(self.startup_program) - self._save_models(self.path, list(self.feeds.keys()), self.fetch_list, - executor, self.main_program, scope) + self._save_models( + self.path, + list(self.feeds.keys()), + self.fetch_list, + executor, + self.main_program, + scope, + ) paddle_outs = self._get_paddle_outs(executor, self.main_program, scope) inference_outs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu)) + self._get_analysis_config(use_gpu=use_gpu) + ) # Check whether the results calculated on CPU and on GPU are the same. self.assertTrue( len(paddle_outs) == len(inference_outs), - "The number of outputs is different between inference and training forward at {}" - .format(device)) + "The number of outputs is different between inference and training forward at {}".format( + device + ), + ) for out, inference_out in zip(paddle_outs, inference_outs): paddle_out = np.array(out) @@ -191,28 +203,35 @@ def check_output_with_option(self, inference_out, rtol=1e-05, atol=atol, - err_msg= - 'Output has diff between inference and training forward at {} '. - format(device)) + err_msg='Output has diff between inference and training forward at {} '.format( + device + ), + ) # Check whether the trt results and the GPU results are the same. if use_gpu and self.enable_trt: tensorrt_outputs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu, - use_trt=self.enable_trt)) + self._get_analysis_config( + use_gpu=use_gpu, use_trt=self.enable_trt + ) + ) if self.trt_parameters.use_static: - #deserialize + # deserialize tensorrt_outputs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu, - use_trt=self.enable_trt)) + self._get_analysis_config( + use_gpu=use_gpu, use_trt=self.enable_trt + ) + ) self.assertTrue( len(tensorrt_outputs) == len(paddle_outs), - "The number of outputs is different between GPU and TensorRT. ") + "The number of outputs is different between GPU and TensorRT. ", + ) - for paddle_out, tensorrt_output in zip(paddle_outs, - tensorrt_outputs): + for paddle_out, tensorrt_output in zip( + paddle_outs, tensorrt_outputs + ): paddle_out = np.array(paddle_out) if flatten: paddle_out = paddle_out.flatten() @@ -223,17 +242,21 @@ def check_output_with_option(self, paddle_out, rtol=rtol, atol=atol, - err_msg='Output has diff between GPU and TensorRT. ') + err_msg='Output has diff between GPU and TensorRT. ', + ) # Check whether the mkldnn results and the CPU results are the same. if (not use_gpu) and self.enable_mkldnn: mkldnn_outputs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu, - use_mkldnn=self.enable_mkldnn)) + self._get_analysis_config( + use_gpu=use_gpu, use_mkldnn=self.enable_mkldnn + ) + ) self.assertTrue( len(paddle_outs) == len(mkldnn_outputs), - "The number of outputs is different between CPU and MKLDNN. ") + "The number of outputs is different between CPU and MKLDNN. ", + ) if self.enable_mkldnn_bfloat16: atol = 0.01 @@ -243,21 +266,24 @@ def check_output_with_option(self, mkldnn_output, rtol=1e-05, atol=atol, - err_msg='Output has diff between CPU and MKLDNN. ') + err_msg='Output has diff between CPU and MKLDNN. ', + ) class TensorRTParam: ''' Prepare TensorRT subgraph engine parameters. ''' - def __init__(self, - workspace_size, - max_batch_size, - min_subgraph_size, - precision, - use_static, - use_calib_mode, - use_inspector=False): + def __init__( + self, + workspace_size, + max_batch_size, + min_subgraph_size, + precision, + use_static, + use_calib_mode, + use_inspector=False, + ): self.workspace_size = workspace_size self.max_batch_size = max_batch_size self.min_subgraph_size = min_subgraph_size @@ -271,8 +297,13 @@ class DynamicShapeParam: Prepare TensorRT subgraph engine dynamic shape parameters. ''' - def __init__(self, min_input_shape, max_input_shape, optim_input_shape, - disable_trt_plugin_fp16): + def __init__( + self, + min_input_shape, + max_input_shape, + optim_input_shape, + disable_trt_plugin_fp16, + ): self.min_input_shape = min_input_shape self.max_input_shape = max_input_shape self.optim_input_shape = optim_input_shape diff --git a/python/paddle/fluid/tests/unittests/ir/inference/program_config.py b/python/paddle/fluid/tests/unittests/ir/inference/program_config.py index 5ad6f1c885cf9..74375d9367d4e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/program_config.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/program_config.py @@ -32,10 +32,12 @@ class TensorConfig: A config builder for a input or a weight. ''' - def __init__(self, - lod: Optional[List[List[int]]] = None, - data_gen: Optional[Callable[..., np.array]] = None, - shape: Optional[List[List[int]]] = None): + def __init__( + self, + lod: Optional[List[List[int]]] = None, + data_gen: Optional[Callable[..., np.array]] = None, + shape: Optional[List[List[int]]] = None, + ): ''' shape: The shape of the tensor. dtype: The data type of the tensor. @@ -48,7 +50,9 @@ def __init__(self, self.dtype = data_gen().dtype self.shape = data_gen().shape else: - assert shape is not None, "While data_gen is not defined, shape must not be None" + assert ( + shape is not None + ), "While data_gen is not defined, shape must not be None" self.data = np.random.normal(0.0, 1.0, shape).astype(np.float32) self.shape = shape self.dtype = self.data.dtype @@ -64,16 +68,18 @@ class VarType(enum.Enum): class OpConfig: - ''' A config builder for generating a Op. ''' - - def __init__(self, - type: str, - inputs: Dict[str, List[str]], - outputs: Dict[str, List[str]], - attrs: Dict[str, Any] = None, - outputs_var_type: Dict[str, VarType] = None, - outputs_dtype: Dict[str, np.dtype] = None, - **kwargs): + '''A config builder for generating a Op.''' + + def __init__( + self, + type: str, + inputs: Dict[str, List[str]], + outputs: Dict[str, List[str]], + attrs: Dict[str, Any] = None, + outputs_var_type: Dict[str, VarType] = None, + outputs_dtype: Dict[str, np.dtype] = None, + **kwargs, + ): self.type = type self.inputs = inputs self.outputs = outputs @@ -91,26 +97,50 @@ def __repr__(self): _OP_WITHOUT_KERNEL_SET = { - 'feed', 'fetch', 'recurrent', 'go', 'rnn_memory_helper_grad', - 'conditional_block', 'while', 'send', 'recv', 'listen_and_serv', - 'fl_listen_and_serv', 'ncclInit', 'select', 'checkpoint_notify', - 'gen_bkcl_id', 'c_gen_bkcl_id', 'gen_nccl_id', 'c_gen_nccl_id', - 'c_comm_init', 'c_sync_calc_stream', 'c_sync_comm_stream', - 'queue_generator', 'dequeue', 'enqueue', 'heter_listen_and_serv', - 'c_wait_comm', 'c_wait_compute', 'c_gen_hccl_id', 'c_comm_init_hccl', - 'copy_cross_scope' + 'feed', + 'fetch', + 'recurrent', + 'go', + 'rnn_memory_helper_grad', + 'conditional_block', + 'while', + 'send', + 'recv', + 'listen_and_serv', + 'fl_listen_and_serv', + 'ncclInit', + 'select', + 'checkpoint_notify', + 'gen_bkcl_id', + 'c_gen_bkcl_id', + 'gen_nccl_id', + 'c_gen_nccl_id', + 'c_comm_init', + 'c_sync_calc_stream', + 'c_sync_comm_stream', + 'queue_generator', + 'dequeue', + 'enqueue', + 'heter_listen_and_serv', + 'c_wait_comm', + 'c_wait_compute', + 'c_gen_hccl_id', + 'c_comm_init_hccl', + 'copy_cross_scope', } class BlockConfig: - ''' A config builder for generating a Block. ''' - - def __init__(self, - ops: List[OpConfig], - vars: List[str], - vars_dtype: Dict[str, np.dtype] = None, - vars_var_type: Dict[str, VarType] = None, - vars_lod_level: Dict[str, int] = None): + '''A config builder for generating a Block.''' + + def __init__( + self, + ops: List[OpConfig], + vars: List[str], + vars_dtype: Dict[str, np.dtype] = None, + vars_var_type: Dict[str, VarType] = None, + vars_lod_level: Dict[str, int] = None, + ): self.ops = ops self.vars = vars self.vars_dtype = vars_dtype @@ -121,10 +151,14 @@ def fill_block_desc(self, block_desc): for name in self.vars: var_desc = block_desc.var(name.encode()) var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR) - if self.vars_lod_level is not None and name in self.vars_lod_level.keys( + if ( + self.vars_lod_level is not None + and name in self.vars_lod_level.keys() ): var_desc.set_lod_level(self.vars_lod_level[name]) - if self.vars_var_type is not None and name in self.vars_var_type.keys( + if ( + self.vars_var_type is not None + and name in self.vars_var_type.keys() ): if self.vars_var_type[name] == VarType.LOD_TENSOR_ARRAY: var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR_ARRAY) @@ -134,7 +168,8 @@ def fill_block_desc(self, block_desc): var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32)) if self.vars_dtype is not None and name in self.vars_dtype.keys(): var_desc.set_dtype( - convert_np_dtype_to_dtype_(self.vars_dtype[name])) + convert_np_dtype_to_dtype_(self.vars_dtype[name]) + ) for op_config in self.ops: op_desc = block_desc.append_op() @@ -150,22 +185,32 @@ def fill_block_desc(self, block_desc): continue var_desc = block_desc.var(v.encode()) var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR) - if op_config.outputs_var_type is not None and v in op_config.outputs_var_type.keys( + if ( + op_config.outputs_var_type is not None + and v in op_config.outputs_var_type.keys() ): - if op_config.outputs_var_type[ - v] == VarType.LOD_TENSOR_ARRAY: + if ( + op_config.outputs_var_type[v] + == VarType.LOD_TENSOR_ARRAY + ): var_desc.set_type( - core.VarDesc.VarType.LOD_TENSOR_ARRAY) - elif op_config.outputs_var_type[ - v] == VarType.STEP_SCOPES: + core.VarDesc.VarType.LOD_TENSOR_ARRAY + ) + elif ( + op_config.outputs_var_type[v] == VarType.STEP_SCOPES + ): var_desc.set_type(core.VarDesc.VarType.STEP_SCOPES) continue var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32)) - if op_config.outputs_dtype is not None and v in op_config.outputs_dtype.keys( + if ( + op_config.outputs_dtype is not None + and v in op_config.outputs_dtype.keys() ): var_desc.set_dtype( convert_np_dtype_to_dtype_( - op_config.outputs_dtype[v])) + op_config.outputs_dtype[v] + ) + ) if op_config.type not in _OP_WITHOUT_KERNEL_SET: op_desc.infer_var_type(block_desc) op_desc.infer_shape(block_desc) @@ -173,10 +218,15 @@ def fill_block_desc(self, block_desc): class ProgramConfig: - ''' A config builder for generating a Program. ''' - - def __init__(self, ops: List[OpConfig], weights: Dict[str, TensorConfig], - inputs: Dict[str, TensorConfig], outputs: List[str]): + '''A config builder for generating a Program.''' + + def __init__( + self, + ops: List[OpConfig], + weights: Dict[str, TensorConfig], + inputs: Dict[str, TensorConfig], + outputs: List[str], + ): self.ops = ops # if no weight need to save, we create a place_holder to help seriazlie params. if not weights: @@ -209,7 +259,7 @@ def __repr__(self): def create_fake_model(program_config): - ''' Create a Paddle model(in memory) according to the given config. ''' + '''Create a Paddle model(in memory) according to the given config.''' paddle.enable_static() main_program_desc = core.ProgramDesc() util_program = fluid.Program() @@ -249,21 +299,22 @@ def create_fake_model(program_config): shape=tensor_config.shape, type=core.VarDesc.VarType.LOD_TENSOR, name=name, - initializer=NumpyArrayInitializer(tensor_config.data)) + initializer=NumpyArrayInitializer(tensor_config.data), + ) in_vars = [] for name in sorted(save_var_map.keys()): in_vars.append(save_var_map[name]) out_var = util_program.global_block().create_var( - type=core.VarDesc.VarType.RAW, name="out_var_0") + type=core.VarDesc.VarType.RAW, name="out_var_0" + ) out_var.desc.set_persistable(True) - util_program.global_block().append_op(type='save_combine', - inputs={'X': in_vars}, - outputs={'Y': out_var}, - attrs={ - 'file_path': '', - 'save_to_memory': True - }) + util_program.global_block().append_op( + type='save_combine', + inputs={'X': in_vars}, + outputs={'Y': out_var}, + attrs={'file_path': '', 'save_to_memory': True}, + ) for op_config in program_config.ops: op_desc = main_block_desc.append_op() op_desc.set_type(op_config.type) @@ -283,19 +334,26 @@ def create_fake_model(program_config): continue var_desc = main_block_desc.var(v.encode()) var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR) - if op_config.outputs_var_type is not None and v in op_config.outputs_var_type.keys( + if ( + op_config.outputs_var_type is not None + and v in op_config.outputs_var_type.keys() ): - if op_config.outputs_var_type[ - v] == VarType.LOD_TENSOR_ARRAY: + if ( + op_config.outputs_var_type[v] + == VarType.LOD_TENSOR_ARRAY + ): var_desc.set_type(core.VarDesc.VarType.LOD_TENSOR_ARRAY) elif op_config.outputs_var_type[v] == VarType.STEP_SCOPES: var_desc.set_type(core.VarDesc.VarType.STEP_SCOPES) continue var_desc.set_dtype(convert_np_dtype_to_dtype_(np.float32)) - if op_config.outputs_dtype is not None and v in op_config.outputs_dtype.keys( + if ( + op_config.outputs_dtype is not None + and v in op_config.outputs_dtype.keys() ): var_desc.set_dtype( - convert_np_dtype_to_dtype_(op_config.outputs_dtype[v])) + convert_np_dtype_to_dtype_(op_config.outputs_dtype[v]) + ) if op_config.type not in _OP_WITHOUT_KERNEL_SET: op_desc.infer_var_type(main_block_desc) op_desc.infer_shape(main_block_desc) @@ -327,19 +385,26 @@ def create_fake_model(program_config): return model, params -def create_quant_model(model, - params, - activation_quantize_type='moving_average_abs_max', - weight_quantize_type='channel_wise_abs_max', - save=False): +def create_quant_model( + model, + params, + activation_quantize_type='moving_average_abs_max', + weight_quantize_type='channel_wise_abs_max', + save=False, +): place = paddle.CUDAPlace(0) scope = global_scope() exe = paddle.static.Executor(place) - [inference_program, feed_target_names, - fetch_targets] = paddle.static.load_inference_model(path_prefix=None, - executor=exe, - model_filename=model, - params_filename=params) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model( + path_prefix=None, + executor=exe, + model_filename=model, + params_filename=params, + ) graph = IrGraph(core.Graph(inference_program.desc), for_test=True) out_scale_op_list = [ @@ -431,11 +496,11 @@ def create_quant_model(model, def _get_op_output_var_names(op): """ """ - assert isinstance(op, (IrNode, Operator)), \ - "The input op should be IrNode or Operator." + assert isinstance( + op, (IrNode, Operator) + ), "The input op should be IrNode or Operator." var_names = [] - op_name = op.name() if isinstance(op, IrNode) \ - else op.type + op_name = op.name() if isinstance(op, IrNode) else op.type if op_name not in op_real_in_out_name: return [] @@ -452,7 +517,8 @@ def _get_op_output_var_names(op): scope=scope, place=place, activation_quantize_type=activation_quantize_type, - weight_quantize_type=weight_quantize_type) + weight_quantize_type=weight_quantize_type, + ) transform_pass.apply(graph) op_nodes = graph.all_op_nodes() @@ -461,15 +527,18 @@ def _get_op_output_var_names(op): var_names = _get_op_output_var_names(op_node) for var_name in var_names: in_node = graph._find_node_by_name(op_node.outputs, var_name) - if in_node.dtype() not in \ - [core.VarDesc.VarType.FP64, core.VarDesc.VarType.FP32]: + if in_node.dtype() not in [ + core.VarDesc.VarType.FP64, + core.VarDesc.VarType.FP32, + ]: continue op_node.op()._set_attr("out_threshold", 3.0) # Freeze graph for inference, but the weight of fc/conv is still float type. freeze_pass = QuantizationFreezePass( - scope=scope, place=place, weight_quantize_type=weight_quantize_type) + scope=scope, place=place, weight_quantize_type=weight_quantize_type + ) freeze_pass.apply(graph) main_program = graph.to_program() @@ -487,18 +556,21 @@ def _get_op_output_var_names(op): tensor.set(np.ones(tensor.shape(), dtype=np.float32), place) if save: - fluid.io.save_inference_model('test_inference_model', - feed_target_names, - fetch_targets, - exe, - main_program=main_program) + fluid.io.save_inference_model( + 'test_inference_model', + feed_target_names, + fetch_targets, + exe, + main_program=main_program, + ) feed_vars = [ main_program.global_block().var(name) for name in feed_target_names ] - serialized_program = paddle.static.serialize_program(feed_vars, - fetch_targets, - program=main_program) + serialized_program = paddle.static.serialize_program( + feed_vars, fetch_targets, program=main_program + ) serialized_params = paddle.static.serialize_persistables( - feed_vars, fetch_targets, executor=exe, program=main_program) + feed_vars, fetch_targets, executor=exe, program=main_program + ) return serialized_program, serialized_params diff --git a/python/paddle/fluid/tests/unittests/ir/inference/quant_dequant_test.py b/python/paddle/fluid/tests/unittests/ir/inference/quant_dequant_test.py index c41e505332f85..bdbff1a385d7d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/quant_dequant_test.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/quant_dequant_test.py @@ -24,14 +24,13 @@ from paddle.fluid.contrib.slim.quantization import OutScaleForTrainingPass from paddle.fluid.contrib.slim.quantization import OutScaleForInferencePass from paddle.fluid.contrib.slim.quantization import AddQuantDequantPass -from paddle.fluid import (core, Program, Variable) +from paddle.fluid import core, Program, Variable from paddle.fluid.io import prepend_feed_ops, append_fetch_ops from paddle.fluid.core import create_paddle_predictor from paddle.fluid.core import AnalysisConfig class QuantDequantTest(unittest.TestCase): - def __init__(self, methodName='runTest'): super(QuantDequantTest, self).__init__(methodName) paddle.enable_static() @@ -60,28 +59,34 @@ def __init__(self, methodName='runTest'): def _normalize_program(self, program, feed_vars, fetch_vars): if not isinstance(program, Program): raise TypeError( - "program type must be `fluid.Program`, but received `%s`" % - type(program)) + "program type must be `fluid.Program`, but received `%s`" + % type(program) + ) if not isinstance(feed_vars, list): feed_vars = [feed_vars] if not all(isinstance(v, Variable) for v in feed_vars): raise TypeError( - "feed_vars type must be a Variable or a list of Variable.") + "feed_vars type must be a Variable or a list of Variable." + ) if not isinstance(fetch_vars, list): fetch_vars = [fetch_vars] if not all(isinstance(v, Variable) for v in fetch_vars): raise TypeError( - "fetch_vars type must be a Variable or a list of Variable.") + "fetch_vars type must be a Variable or a list of Variable." + ) # remind users to set auc_states to 0 if auc op were found. for op in program.global_block().ops: # clear device of Op - device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName( + device_attr_name = ( + core.op_proto_and_checker_maker.kOpDeviceAttrName() ) op._set_attr(device_attr_name, "") if op.type == 'auc': - warnings.warn("Be sure that you have set auc states to 0 " - "before saving inference model.") + warnings.warn( + "Be sure that you have set auc states to 0 " + "before saving inference model." + ) break # serialize program @@ -98,7 +103,8 @@ def _normalize_program(self, program, feed_vars, fetch_vars): feed_var_names = [var.name for var in feed_vars] copy_program = copy_program._prune_with_input( - feeded_var_names=feed_var_names, targets=fetch_vars) + feeded_var_names=feed_var_names, targets=fetch_vars + ) copy_program = copy_program._inference_optimize(prune_read_op=True) fetch_var_names = [var.name for var in fetch_vars] prepend_feed_ops(copy_program, feed_var_names) @@ -106,25 +112,30 @@ def _normalize_program(self, program, feed_vars, fetch_vars): copy_program.desc._set_version() return copy_program - def _save_models(self, dirname, feeded_var_names, target_vars, executor, - program, scope): + def _save_models( + self, dirname, feeded_var_names, target_vars, executor, program, scope + ): with fluid.scope_guard(scope): - fluid.io.save_inference_model(dirname, - feeded_var_names, - target_vars, - executor, - program, - clip_extra=True) + fluid.io.save_inference_model( + dirname, + feeded_var_names, + target_vars, + executor, + program, + clip_extra=True, + ) def _get_paddle_outs(self, feed, fetch_list, executor, program, scope): ''' Return PaddlePaddle outputs. ''' with fluid.scope_guard(scope): - outs = executor.run(program=program, - feed=feed, - fetch_list=fetch_list, - return_numpy=True) + outs = executor.run( + program=program, + feed=feed, + fetch_list=fetch_list, + return_numpy=True, + ) return outs def _get_inference_outs(self, config): @@ -152,10 +163,9 @@ def _get_inference_outs(self, config): ] return outs - def _get_analysis_config(self, - use_gpu=False, - use_trt=False, - use_mkldnn=False): + def _get_analysis_config( + self, use_gpu=False, use_trt=False, use_mkldnn=False + ): ''' Return a new object of AnalysisConfig. ''' @@ -173,14 +183,16 @@ def _get_analysis_config(self, self.trt_parameters.min_subgraph_size, self.trt_parameters.precision, self.trt_parameters.use_static, - self.trt_parameters.use_calib_mode) + self.trt_parameters.use_calib_mode, + ) if self.dynamic_shape_params: config.set_trt_dynamic_shape_info( self.dynamic_shape_params.min_input_shape, self.dynamic_shape_params.max_input_shape, self.dynamic_shape_params.optim_input_shape, - self.dynamic_shape_params.disable_trt_plugin_fp16) + self.dynamic_shape_params.disable_trt_plugin_fp16, + ) if self.enable_tensorrt_varseqlen: config.enable_tensorrt_varseqlen() @@ -191,12 +203,9 @@ def _get_analysis_config(self, print('config summary:', config.summary()) return config - def check_output_with_option(self, - use_gpu, - atol=1e-5, - flatten=False, - quant=False, - rtol=1e-5): + def check_output_with_option( + self, use_gpu, atol=1e-5, flatten=False, quant=False, rtol=1e-5 + ): ''' Check whether calculating on CPU and GPU, enable TensorRT or disable TensorRT, enable MKLDNN or disable MKLDNN @@ -211,14 +220,16 @@ def check_output_with_option(self, executor.run(self.startup_program) executor.run(self.test_startup_program) main_graph = IrGraph(core.Graph(self.main_program.desc), for_test=False) - test_graph = IrGraph(core.Graph(self.test_main_program.desc), - for_test=True) + test_graph = IrGraph( + core.Graph(self.test_main_program.desc), for_test=True + ) transform_pass = QuantizationTransformPass( scope=scope, place=place, activation_quantize_type=self.activation_quantize_type, - weight_quantize_type=self.weight_quantize_type) + weight_quantize_type=self.weight_quantize_type, + ) transform_pass.apply(main_graph) transform_pass.apply(test_graph) @@ -237,17 +248,19 @@ def check_output_with_option(self, iters = 10 batch_size = 1 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=batch_size) - feeder = fluid.DataFeeder(feed_list=[self.data, self.label], - place=place) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=batch_size, + ) + feeder = fluid.DataFeeder( + feed_list=[self.data, self.label], place=place + ) with fluid.scope_guard(scope): for _ in range(iters): data = next(train_reader()) - loss_v = executor.run(binary, - feed=feeder.feed(data), - fetch_list=[self.loss]) + loss_v = executor.run( + binary, feed=feeder.feed(data), fetch_list=[self.loss] + ) scale_inference_pass = OutScaleForInferencePass(scope=scope) scale_inference_pass.apply(test_graph) @@ -256,29 +269,40 @@ def check_output_with_option(self, freeze_pass = QuantizationFreezePass( scope=scope, place=place, - weight_quantize_type=self.weight_quantize_type) + weight_quantize_type=self.weight_quantize_type, + ) freeze_pass.apply(test_graph) self.main_program = test_graph.to_program() with fluid.scope_guard(scope): - self.main_program = self._normalize_program(self.main_program, - self.data, - self.fetch_list) - - self._save_models(self.path, list(self.feeds.keys()), self.fetch_list, - executor, self.main_program, scope) + self.main_program = self._normalize_program( + self.main_program, self.data, self.fetch_list + ) - paddle_outs = self._get_paddle_outs(self.feeds, self.fetch_list, - executor, self.main_program, scope) + self._save_models( + self.path, + list(self.feeds.keys()), + self.fetch_list, + executor, + self.main_program, + scope, + ) + + paddle_outs = self._get_paddle_outs( + self.feeds, self.fetch_list, executor, self.main_program, scope + ) inference_outs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu)) + self._get_analysis_config(use_gpu=use_gpu) + ) # Check whether the results calculated on CPU and on GPU are the same. self.assertTrue( len(paddle_outs) == len(inference_outs), - "The number of outputs is different between inference and training forward at {}" - .format(device)) + "The number of outputs is different between inference and training forward at {}".format( + device + ), + ) for out, inference_out in zip(paddle_outs, inference_outs): paddle_out = np.array(out) @@ -292,28 +316,35 @@ def check_output_with_option(self, inference_out, rtol=1e-05, atol=atol, - err_msg= - 'Output has diff between inference and training forward at {} '. - format(device)) + err_msg='Output has diff between inference and training forward at {} '.format( + device + ), + ) # Check whether the trt results and the GPU results are the same. if use_gpu and self.enable_trt: tensorrt_outputs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu, - use_trt=self.enable_trt)) + self._get_analysis_config( + use_gpu=use_gpu, use_trt=self.enable_trt + ) + ) if self.trt_parameters.use_static: - #deserialize + # deserialize tensorrt_outputs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu, - use_trt=self.enable_trt)) + self._get_analysis_config( + use_gpu=use_gpu, use_trt=self.enable_trt + ) + ) self.assertTrue( len(tensorrt_outputs) == len(paddle_outs), - "The number of outputs is different between GPU and TensorRT. ") + "The number of outputs is different between GPU and TensorRT. ", + ) - for paddle_out, tensorrt_output in zip(paddle_outs, - tensorrt_outputs): + for paddle_out, tensorrt_output in zip( + paddle_outs, tensorrt_outputs + ): paddle_out = np.array(paddle_out) if flatten: @@ -325,17 +356,21 @@ def check_output_with_option(self, tensorrt_output, rtol=rtol, atol=atol, - err_msg='Output has diff between GPU and TensorRT. ') + err_msg='Output has diff between GPU and TensorRT. ', + ) # Check whether the mkldnn results and the CPU results are the same. if (not use_gpu) and self.enable_mkldnn: mkldnn_outputs = self._get_inference_outs( - self._get_analysis_config(use_gpu=use_gpu, - use_mkldnn=self.enable_mkldnn)) + self._get_analysis_config( + use_gpu=use_gpu, use_mkldnn=self.enable_mkldnn + ) + ) self.assertTrue( len(paddle_outs) == len(mkldnn_outputs), - "The number of outputs is different between CPU and MKLDNN. ") + "The number of outputs is different between CPU and MKLDNN. ", + ) if self.enable_mkldnn_bfloat16: atol = 0.01 @@ -345,15 +380,23 @@ def check_output_with_option(self, mkldnn_output, rtol=1e-05, atol=atol, - err_msg='Output has diff between CPU and MKLDNN. ') + err_msg='Output has diff between CPU and MKLDNN. ', + ) class TensorRTParam: ''' Prepare TensorRT subgraph engine parameters. ''' - def __init__(self, workspace_size, max_batch_size, min_subgraph_size, - precision, use_static, use_calib_mode): + def __init__( + self, + workspace_size, + max_batch_size, + min_subgraph_size, + precision, + use_static, + use_calib_mode, + ): self.workspace_size = workspace_size self.max_batch_size = max_batch_size self.min_subgraph_size = min_subgraph_size @@ -366,8 +409,13 @@ class DynamicShapeParam: Prepare TensorRT subgraph engine dynamic shape parameters. ''' - def __init__(self, min_input_shape, max_input_shape, optim_input_shape, - disable_trt_plugin_fp16): + def __init__( + self, + min_input_shape, + max_input_shape, + optim_input_shape, + disable_trt_plugin_fp16, + ): self.min_input_shape = min_input_shape self.max_input_shape = max_input_shape self.optim_input_shape = optim_input_shape diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_adaptive_pool2d_convert_global_pass_autoscan.py b/python/paddle/fluid/tests/unittests/ir/inference/test_adaptive_pool2d_convert_global_pass_autoscan.py index aa2d29d613710..c0814dc920b9e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_adaptive_pool2d_convert_global_pass_autoscan.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_adaptive_pool2d_convert_global_pass_autoscan.py @@ -21,55 +21,60 @@ class TestAdaptivePool2dConvertGlobalPass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_config(self, draw): x_shape = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=4, max_size=4 + ) + ) pooling_type = draw(st.sampled_from(["max", "avg"])) - data_format = "NCHW" #trt support this format only + data_format = "NCHW" # trt support this format only strides = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) paddings = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) ceil_mode = draw(st.booleans()) exclusive = draw(st.booleans()) global_pooling = draw(st.booleans()) padding_algorithm = draw(st.sampled_from(["EXPLICIT", "SAME", "VAILD"])) - pool_op = OpConfig("pool2d", - inputs={"X": ["input_data"]}, - outputs={"Out": ["pool_output"]}, - ksize=[1, 1], - adaptive=True, - pooling_type=pooling_type, - data_format=data_format, - strides=strides, - paddings=paddings, - ceil_mode=ceil_mode, - global_pooling=global_pooling, - padding_algorithm=padding_algorithm, - exclusive=exclusive) + pool_op = OpConfig( + "pool2d", + inputs={"X": ["input_data"]}, + outputs={"Out": ["pool_output"]}, + ksize=[1, 1], + adaptive=True, + pooling_type=pooling_type, + data_format=data_format, + strides=strides, + paddings=paddings, + ceil_mode=ceil_mode, + global_pooling=global_pooling, + padding_algorithm=padding_algorithm, + exclusive=exclusive, + ) ops = [pool_op] - program_config = ProgramConfig(ops=ops, - weights={}, - inputs={ - "input_data": - TensorConfig(shape=x_shape), - }, - outputs=["pool_output"]) + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "input_data": TensorConfig(shape=x_shape), + }, + outputs=["pool_output"], + ) return program_config @@ -81,14 +86,17 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['pool2d'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_examples=300, - passes=["adaptive_pool2d_convert_global_pass"], - min_success_num=40) + self.run_and_statis( + quant=False, + max_examples=300, + passes=["adaptive_pool2d_convert_global_pass"], + min_success_num=40, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py index d619438de4ec0..fdc995fee7899 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_act_mkldnn_fuse_pass.py @@ -48,12 +48,36 @@ def is_program_valid(self, prog_config): filter_shape = prog_config.weights["filter"].shape input_shape = prog_config.inputs["input_x"].shape if padding_algorithm == "VALID": - if ((input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + (input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) + / strides[0] + + 1 + ) <= 1 or ( + (input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) + / strides[1] + + 1 + ) <= 1: return False if padding_algorithm == "EXPLICIT": - if ((input_shape[2] + paddings[0] + paddings[1] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] + paddings[2] + paddings[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + ( + input_shape[2] + + paddings[0] + + paddings[1] + - (dilations[0] * (filter_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) <= 1 or ( + ( + input_shape[3] + + paddings[2] + + paddings[3] + - (dilations[1] * (filter_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) <= 1: return False if data_format == "NCHW": if input_shape[1] != filter_shape[1] * groups: @@ -70,9 +94,10 @@ def is_program_valid(self, prog_config): def sample_program_config(self, draw): # 1. Generate shape of input:X of conv2d x_shape = draw( - st.lists(st.integers(min_value=5, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=5, max_value=100), min_size=4, max_size=4 + ) + ) x_shape[1] = draw(st.integers(min_value=5, max_value=10)) # 2. Generate legal attr:data_format of conv2d @@ -80,9 +105,10 @@ def sample_program_config(self, draw): # 3. Generate legal shape of input:Y of conv2d f_shape = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=4, max_size=4 + ) + ) if data_format == "NCHW": f_shape[1] = x_shape[1] else: @@ -90,35 +116,41 @@ def sample_program_config(self, draw): # 4. Generate legal attr:strides of conv2d strides = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=2, max_size=2 + ) + ) # 5. Generate legal attr:padding_algorithm of conv2d padding_algorithm = draw(st.sampled_from(["EXPLICIT", "SAME", "VALID"])) # 6. Generate legal attr:padding of conv2d padding = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=4, max_size=4 + ) + ) # 7. Generate legal attr:groups of conv2d groups = draw(st.integers(min_value=1, max_value=3)) # 8. Generate legal attr:dilations of conv2d dilations = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=2, max_size=2 + ) + ) # 9. Generate legal input:ResidualData of conv2d res_shape = [] if draw(st.booleans()): res_shape = draw( - st.lists(st.integers(min_value=1, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=100), + min_size=4, + max_size=4, + ) + ) # 10. Generate legal shape of input:bias of conv2d conv_bias_shape = [] @@ -135,58 +167,69 @@ def sample_program_config(self, draw): } weights = { "filter": TensorConfig(shape=f_shape), - "conv_bias": TensorConfig(shape=conv_bias_shape) + "conv_bias": TensorConfig(shape=conv_bias_shape), } use_mkldnn = True else: inputs = { "Input": ["input_x"], "Filter": ["filter"], - "ResidualData": ["residualdata"] + "ResidualData": ["residualdata"], } weights = {"filter": TensorConfig(shape=f_shape)} use_mkldnn = False # 11. Generate legal act type of conv2d act_type = draw( - st.sampled_from(["relu", "leaky_relu", "relu6", "swish"])) - - conv2d_op = OpConfig("conv2d", - inputs=inputs, - outputs={"Output": ["conv2d_out"]}, - strides=strides, - padding_algorithm=padding_algorithm, - paddings=padding, - groups=groups, - dilations=dilations, - data_format=data_format, - use_mkldnn=True) + st.sampled_from(["relu", "leaky_relu", "relu6", "swish"]) + ) + + conv2d_op = OpConfig( + "conv2d", + inputs=inputs, + outputs={"Output": ["conv2d_out"]}, + strides=strides, + padding_algorithm=padding_algorithm, + paddings=padding, + groups=groups, + dilations=dilations, + data_format=data_format, + use_mkldnn=True, + ) # 11. Generate legal attr of act act_op = None self.passes = ["conv_activation_mkldnn_fuse_pass"] if act_type == "relu6": threshold = draw(st.floats(min_value=1.0, max_value=10.0)) - act_op = OpConfig("relu6", - inputs={"X": ["conv2d_out"]}, - outputs={"Out": ["relu_out"]}, - threshold=threshold) + act_op = OpConfig( + "relu6", + inputs={"X": ["conv2d_out"]}, + outputs={"Out": ["relu_out"]}, + threshold=threshold, + ) elif act_type == "leaky_relu": alpha = draw(st.floats(min_value=0.1, max_value=1.0)) - act_op = OpConfig("leaky_relu", - inputs={"X": ["conv2d_out"]}, - outputs={"Out": ["relu_out"]}, - alpha=alpha) + act_op = OpConfig( + "leaky_relu", + inputs={"X": ["conv2d_out"]}, + outputs={"Out": ["relu_out"]}, + alpha=alpha, + ) elif act_type == "relu": - act_op = OpConfig("relu", - inputs={"X": ["conv2d_out"]}, - outputs={"Out": ["relu_out"]}) + act_op = OpConfig( + "relu", + inputs={"X": ["conv2d_out"]}, + outputs={"Out": ["relu_out"]}, + ) elif act_type == "swish": beta = draw(st.floats(min_value=0.1, max_value=1.0)) - act_op = OpConfig("swish", - inputs={"X": ["conv2d_out"]}, - outputs={"Out": ["swish_out"]}, - beta=beta) + act_op = OpConfig( + "swish", + inputs={"X": ["conv2d_out"]}, + outputs={"Out": ["swish_out"]}, + beta=beta, + ) ops = [conv2d_op, act_op] @@ -195,7 +238,7 @@ def sample_program_config(self, draw): weights=weights, inputs={ "input_x": TensorConfig(shape=x_shape), - "residualdata": TensorConfig(shape=res_shape) + "residualdata": TensorConfig(shape=res_shape), }, outputs=ops[-1].outputs["Out"], ) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bias_mkldnn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bias_mkldnn_fuse_pass.py index eb02c6e1c77ca..ec7c8db66d033 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bias_mkldnn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bias_mkldnn_fuse_pass.py @@ -48,12 +48,36 @@ def is_program_valid(self, prog_config): filter_shape = prog_config.weights["filter"].shape input_shape = prog_config.inputs["input_x"].shape if padding_algorithm == "VALID": - if ((input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + (input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) + / strides[0] + + 1 + ) <= 1 or ( + (input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) + / strides[1] + + 1 + ) <= 1: return False if padding_algorithm == "EXPLICIT": - if ((input_shape[2] + paddings[0] + paddings[1] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] + paddings[2] + paddings[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + ( + input_shape[2] + + paddings[0] + + paddings[1] + - (dilations[0] * (filter_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) <= 1 or ( + ( + input_shape[3] + + paddings[2] + + paddings[3] + - (dilations[1] * (filter_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) <= 1: return False if data_format == "NCHW": if input_shape[1] != filter_shape[1] * groups: @@ -70,9 +94,10 @@ def is_program_valid(self, prog_config): def sample_program_config(self, draw): # 1. Generate shape of input:X of conv2d x_shape = draw( - st.lists(st.integers(min_value=5, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=5, max_value=100), min_size=4, max_size=4 + ) + ) x_shape[1] = draw(st.integers(min_value=5, max_value=10)) # 2. Generate legal attr:data_format of conv2d @@ -80,9 +105,10 @@ def sample_program_config(self, draw): # 3. Generate legal shape of input:Y of conv2d f_shape = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=4, max_size=4 + ) + ) if data_format == "NCHW": f_shape[1] = x_shape[1] else: @@ -90,27 +116,30 @@ def sample_program_config(self, draw): # 4. Generate legal attr:strides of conv2d strides = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) # 5. Generate legal attr:padding_algorithm of conv2d padding_algorithm = draw(st.sampled_from(["EXPLICIT", "SAME", "VALID"])) # 6. Generate legal attr:padding of conv2d padding = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=4, max_size=4 + ) + ) # 7. Generate legal attr:groups of conv2d groups = draw(st.integers(min_value=1, max_value=3)) # 8. Generate legal attr:dilations of conv2d dilations = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) # 9. Generate legal shape of input:bias of elementwise_add bias_shape = [f_shape[0]] @@ -137,7 +166,7 @@ def sample_program_config(self, draw): weights = { "filter": TensorConfig(shape=f_shape), "bias": TensorConfig(shape=bias_shape), - "conv_bias": TensorConfig(shape=conv_bias_shape) + "conv_bias": TensorConfig(shape=conv_bias_shape), } use_mkldnn = True else: @@ -147,28 +176,29 @@ def sample_program_config(self, draw): } weights = { "filter": TensorConfig(shape=f_shape), - "bias": TensorConfig(shape=bias_shape) + "bias": TensorConfig(shape=bias_shape), } use_mkldnn = False - conv2d_op = OpConfig("conv2d", - inputs=inputs, - outputs={"Output": ["conv2d_out"]}, - strides=strides, - padding_algorithm=padding_algorithm, - paddings=padding, - groups=groups, - dilations=dilations, - data_format=data_format, - use_mkldnn=use_mkldnn) - - add_op = OpConfig("elementwise_add", - inputs={ - "X": ["conv2d_out"], - "Y": ["bias"] - }, - outputs={"Out": ["add_out"]}, - axis=axis) + conv2d_op = OpConfig( + "conv2d", + inputs=inputs, + outputs={"Output": ["conv2d_out"]}, + strides=strides, + padding_algorithm=padding_algorithm, + paddings=padding, + groups=groups, + dilations=dilations, + data_format=data_format, + use_mkldnn=use_mkldnn, + ) + + add_op = OpConfig( + "elementwise_add", + inputs={"X": ["conv2d_out"], "Y": ["bias"]}, + outputs={"Out": ["add_out"]}, + axis=axis, + ) ops = [conv2d_op, add_op] @@ -176,13 +206,14 @@ def sample_program_config(self, draw): ops=ops, weights=weights, inputs={"input_x": TensorConfig(shape=x_shape)}, - outputs=ops[-1].outputs["Out"]) + outputs=ops[-1].outputs["Out"], + ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=350, - passes=["conv_bias_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, max_examples=350, passes=["conv_bias_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bn_fuse_pass.py index 0657ba51cdc7c..e16fd8b10c2f8 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_bn_fuse_pass.py @@ -23,7 +23,6 @@ class TestConvBnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -46,24 +45,29 @@ def sample_program_config(self, draw): out_channel = groups * out_channel_factor batch_size = draw(st.integers(min_value=1, max_value=4)) dilations = draw( - st.lists(st.integers(min_value=1, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=2), min_size=2, max_size=2 + ) + ) paddings = draw( - st.lists(st.integers(min_value=0, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=0, max_value=2), min_size=2, max_size=2 + ) + ) strides = draw( - st.lists(st.integers(min_value=1, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=2), min_size=2, max_size=2 + ) + ) has_bias = draw(st.booleans()) use_mkldnn = draw(st.booleans()) epsilon = draw(st.floats(min_value=0.0, max_value=0.001)) - x_shape = [ - batch_size, in_channel, 64, 64 - ] if data_format == "NCHW" else [batch_size, 64, 64, in_channel] + x_shape = ( + [batch_size, in_channel, 64, 64] + if data_format == "NCHW" + else [batch_size, 64, 64, in_channel] + ) w_shape = [out_channel, filter_channel, filter_size, filter_size] scale_shape = [out_channel] bias_shape = [out_channel] @@ -91,41 +95,45 @@ def generate_bn_Mean(): def generate_bn_Var(): return np.random.random(var_shape).astype(np.float32) - conv2d_op = OpConfig("conv2d", - inputs={ - "Input": ["conv2d_input"], - "Filter": ["conv2d_weight"], - }, - outputs={"Output": ["conv2d_out"]}, - data_format=data_format, - dilations=dilations, - padding_algorithm=padding_algorithm, - groups=groups, - paddings=paddings, - strides=strides, - use_mkldnn=use_mkldnn, - has_bias=has_bias, - is_test=True) - bn_op = OpConfig("batch_norm", - inputs={ - "X": ["conv2d_out"], - "Scale": ["batch_norm_Scale"], - "Bias": ["batch_norm_Bias"], - "Mean": ["batch_norm_Mean"], - "Variance": ["batch_norm_Variance"], - }, - outputs={ - "Y": ["batch_norm_Y"], - "MeanOut": ["batch_norm_Mean"], - "VarianceOut": ["batch_norm_Variance"], - "SavedMean": ["batch_norm_SavedMean"], - "SavedVariance": ["batch_norm_SavedVariance"], - "ReserveSpace": ["batch_norm_ReserveSpace"], - }, - epsilon=epsilon, - trainable_statistics=False, - data_layout=data_format, - is_test=True) + conv2d_op = OpConfig( + "conv2d", + inputs={ + "Input": ["conv2d_input"], + "Filter": ["conv2d_weight"], + }, + outputs={"Output": ["conv2d_out"]}, + data_format=data_format, + dilations=dilations, + padding_algorithm=padding_algorithm, + groups=groups, + paddings=paddings, + strides=strides, + use_mkldnn=use_mkldnn, + has_bias=has_bias, + is_test=True, + ) + bn_op = OpConfig( + "batch_norm", + inputs={ + "X": ["conv2d_out"], + "Scale": ["batch_norm_Scale"], + "Bias": ["batch_norm_Bias"], + "Mean": ["batch_norm_Mean"], + "Variance": ["batch_norm_Variance"], + }, + outputs={ + "Y": ["batch_norm_Y"], + "MeanOut": ["batch_norm_Mean"], + "VarianceOut": ["batch_norm_Variance"], + "SavedMean": ["batch_norm_SavedMean"], + "SavedVariance": ["batch_norm_SavedVariance"], + "ReserveSpace": ["batch_norm_ReserveSpace"], + }, + epsilon=epsilon, + trainable_statistics=False, + data_layout=data_format, + is_test=True, + ) if has_bias == True: conv2d_op.inputs["Bias"] = ["conv2d_bias"] ops = [conv2d_op, bn_op] @@ -133,25 +141,25 @@ def generate_bn_Var(): program_config = ProgramConfig( ops=ops, inputs={ - "conv2d_input": - TensorConfig(data_gen=partial(generate_conv2d_Input)), + "conv2d_input": TensorConfig( + data_gen=partial(generate_conv2d_Input) + ), }, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial(generate_conv2d_Filter)), - "batch_norm_Scale": - TensorConfig(data_gen=generate_bn_Scale), - "batch_norm_Bias": - TensorConfig(data_gen=generate_bn_Bias), - "batch_norm_Mean": - TensorConfig(data_gen=generate_bn_Mean), - "batch_norm_Variance": - TensorConfig(data_gen=generate_bn_Var), + "conv2d_weight": TensorConfig( + data_gen=partial(generate_conv2d_Filter) + ), + "batch_norm_Scale": TensorConfig(data_gen=generate_bn_Scale), + "batch_norm_Bias": TensorConfig(data_gen=generate_bn_Bias), + "batch_norm_Mean": TensorConfig(data_gen=generate_bn_Mean), + "batch_norm_Variance": TensorConfig(data_gen=generate_bn_Var), }, - outputs=["batch_norm_Y"]) + outputs=["batch_norm_Y"], + ) if has_bias == True: program_config.weights["conv2d_bias"] = TensorConfig( - data_gen=partial(generate_conv2d_Bias)) + data_gen=partial(generate_conv2d_Bias) + ) return program_config def sample_predictor_configs(self, program_config): @@ -174,34 +182,40 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=1, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) if program_config.ops[0].attrs['has_bias']: yield config, ['conv2d', 'elementwise_add'], (1e-5, 1e-5) else: # it will enter conv_elementwise_add_fuse_pass yield config, ['conv2d_fusion'], (1e-5, 1e-5) def add_ignore_pass_case(self): - def teller1(program_config, predictor_config): - if program_config.ops[0].attrs[ - 'data_format'] == "NHWC" and not predictor_config.mkldnn_enabled( - ): + if ( + program_config.ops[0].attrs['data_format'] == "NHWC" + and not predictor_config.mkldnn_enabled() + ): return True return False # mkldnn Output has diff with bias! def teller2(program_config, predictor_config): - return predictor_config.mkldnn_enabled( - ) and program_config.ops[0].attrs['has_bias'] == True + return ( + predictor_config.mkldnn_enabled() + and program_config.ops[0].attrs['has_bias'] == True + ) self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The output format of conv2d is wrong when data_format attribute is NHWC" + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The output format of conv2d is wrong when data_format attribute is NHWC", ) self.add_ignore_check_case( - teller2, IgnoreReasons.PASS_ACCURACY_ERROR, - "Currently mkldnn Output has diff with bias!") + teller2, + IgnoreReasons.PASS_ACCURACY_ERROR, + "Currently mkldnn Output has diff with bias!", + ) def test(self): self.run_and_statis( diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add2_act_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add2_act_fuse_pass.py index d08da522e1a3e..75d0e445f7aff 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add2_act_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add2_act_fuse_pass.py @@ -24,21 +24,21 @@ class TestConvElementwiseAdd2ActPass(PassAutoScanTest): r""" - x_var f_var(persistable) - \ / - conv2d - | - conv2d_var y_var(persistable) - \ / - elementwise_add - | - x1_var elementwise_add_out_var - \ / - elementwise_add - | - act - | - act_var + x_var f_var(persistable) + \ / + conv2d + | + conv2d_var y_var(persistable) + \ / + elementwise_add + | + x1_var elementwise_add_out_var + \ / + elementwise_add + | + act + | + act_var """ def sample_predictor_configs(self, program_config): @@ -58,16 +58,66 @@ def is_program_valid(self, prog_config): if data_format != "NCHW": return False if padding_algorithm == "VALID": - if int(((input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1)) <= 0 or \ - int(((input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1)) <= 0: + if ( + int( + ( + ( + input_shape[2] + - (dilations[0] * (filter_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) + ) + <= 0 + or int( + ( + ( + input_shape[3] + - (dilations[1] * (filter_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) + ) + <= 0 + ): return False if padding_algorithm == "EXPLICIT": - if int(((input_shape[2] + paddings[0] + paddings[1] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1)) <= 0 or \ - int(((input_shape[3] + paddings[2] + paddings[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1)) <= 0: + if ( + int( + ( + ( + input_shape[2] + + paddings[0] + + paddings[1] + - (dilations[0] * (filter_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) + ) + <= 0 + or int( + ( + ( + input_shape[3] + + paddings[2] + + paddings[3] + - (dilations[1] * (filter_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) + ) + <= 0 + ): return False if padding_algorithm == "SAME": - if int((input_shape[2] + strides[0] - 1) / strides[0]) <= 0 or int( - (input_shape[3] + strides[1] - 1) / strides[1]) <= 0: + if ( + int((input_shape[2] + strides[0] - 1) / strides[0]) <= 0 + or int((input_shape[3] + strides[1] - 1) / strides[1]) <= 0 + ): return False if data_format == "NCHW": if input_shape[1] != filter_shape[1] * groups: @@ -89,9 +139,12 @@ def sample_program_config(self, draw): while is_not_valid: # 1. Generate shape of input:X of conv2d x_shape = draw( - st.lists(st.integers(min_value=1, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=100), + min_size=4, + max_size=4, + ) + ) x_shape[1] = draw(st.integers(min_value=1, max_value=10)) # 2. Generate legal attr:data_format of conv2d @@ -99,9 +152,12 @@ def sample_program_config(self, draw): # 3. Generate legal shape of input:Y of conv2d f_shape = draw( - st.lists(st.integers(min_value=1, max_value=7), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=7), + min_size=4, + max_size=4, + ) + ) if data_format == "NCHW": f_shape[1] = x_shape[1] else: @@ -109,51 +165,94 @@ def sample_program_config(self, draw): # 4. Generate legal attr:strides of conv2d strides = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), + min_size=2, + max_size=2, + ) + ) # 5. Generate legal attr:padding_algorithm of conv2d padding_algorithm = draw( - st.sampled_from(["EXPLICIT", "SAME", "VALID"])) + st.sampled_from(["EXPLICIT", "SAME", "VALID"]) + ) # 6. Generate legal attr:padding of conv2d padding = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=5), + min_size=4, + max_size=4, + ) + ) # 7. Generate legal attr:groups of conv2d groups = draw(st.integers(min_value=1, max_value=3)) # 8. Generate legal attr:dilations of conv2d dilations = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), + min_size=2, + max_size=2, + ) + ) # 9. Generate legal elemntwise_add: X of conv2d bias_2_dict = dict() bias_2_dict[1] = [ - x_shape[0], f_shape[0], - int(((x_shape[2] + padding[0] + padding[1] - - (dilations[0] * (f_shape[2] - 1) + 1)) / strides[0] + 1)), - int(((x_shape[3] + padding[2] + padding[3] - - (dilations[1] * (f_shape[3] - 1) + 1)) / strides[1] + 1)) + x_shape[0], + f_shape[0], + int( + ( + ( + x_shape[2] + + padding[0] + + padding[1] + - (dilations[0] * (f_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) + ), + int( + ( + ( + x_shape[3] + + padding[2] + + padding[3] + - (dilations[1] * (f_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) + ), ] bias_2_dict[2] = [ - x_shape[0], f_shape[0], + x_shape[0], + f_shape[0], int((x_shape[2] + strides[0] - 1) / strides[0]), - int((x_shape[3] + strides[1] - 1) / strides[1]) + int((x_shape[3] + strides[1] - 1) / strides[1]), ] bias_2_dict[3] = [ - x_shape[0], f_shape[0], - int(((x_shape[2] - (dilations[0] * - (f_shape[2] - 1) + 1)) / strides[0] + 1)), - int(((x_shape[3] - (dilations[1] * - (f_shape[3] - 1) + 1)) / strides[1] + 1)) + x_shape[0], + f_shape[0], + int( + ( + (x_shape[2] - (dilations[0] * (f_shape[2] - 1) + 1)) + / strides[0] + + 1 + ) + ), + int( + ( + (x_shape[3] - (dilations[1] * (f_shape[3] - 1) + 1)) + / strides[1] + + 1 + ) + ), ] bias_index = 1 if padding_algorithm == "SAME": @@ -176,37 +275,34 @@ def sample_program_config(self, draw): # 12. Generate legal attr:axis of elementwise_add_2 axis_2 = -1 - conv2d_op = OpConfig("conv2d", - inputs={ - "Input": ["input_x"], - "Filter": ["filter"] - }, - outputs={"Output": ["conv2d_out"]}, - strides=strides, - padding_algorithm=padding_algorithm, - paddings=padding, - groups=groups, - dilations=dilations, - data_format=data_format) - add_1_op = OpConfig("elementwise_add", - inputs={ - "X": ["conv2d_out"], - "Y": ["bias_1"] - }, - outputs={"Out": ["add_1_out"]}, - axis=axis_1) - - add_2_op = OpConfig("elementwise_add", - inputs={ - "X": ["bias_2"], - "Y": ["add_1_out"] - }, - outputs={"Out": ["add_out"]}, - axis=axis_2) - - relu_op = OpConfig("relu", - inputs={"X": ["add_out"]}, - outputs={"Out": ["relu_out"]}) + conv2d_op = OpConfig( + "conv2d", + inputs={"Input": ["input_x"], "Filter": ["filter"]}, + outputs={"Output": ["conv2d_out"]}, + strides=strides, + padding_algorithm=padding_algorithm, + paddings=padding, + groups=groups, + dilations=dilations, + data_format=data_format, + ) + add_1_op = OpConfig( + "elementwise_add", + inputs={"X": ["conv2d_out"], "Y": ["bias_1"]}, + outputs={"Out": ["add_1_out"]}, + axis=axis_1, + ) + + add_2_op = OpConfig( + "elementwise_add", + inputs={"X": ["bias_2"], "Y": ["add_1_out"]}, + outputs={"Out": ["add_out"]}, + axis=axis_2, + ) + + relu_op = OpConfig( + "relu", inputs={"X": ["add_out"]}, outputs={"Out": ["relu_out"]} + ) ops = [conv2d_op, add_1_op, add_2_op, relu_op] @@ -218,16 +314,18 @@ def sample_program_config(self, draw): }, inputs={ "input_x": TensorConfig(shape=x_shape), - "bias_2": TensorConfig(shape=bias_2_shape) + "bias_2": TensorConfig(shape=bias_2_shape), }, outputs=ops[-1].outputs["Out"], ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=300, - passes=["conv_elementwise_add2_act_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=300, + passes=["conv_elementwise_add2_act_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_act_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_act_fuse_pass.py index 7c1ae063ce849..ca34981951916 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_act_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_act_fuse_pass.py @@ -55,12 +55,36 @@ def is_program_valid(self, prog_config): if data_format != "NCHW": return False if padding_algorithm == "VALID": - if ((input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + (input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) + / strides[0] + + 1 + ) <= 1 or ( + (input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) + / strides[1] + + 1 + ) <= 1: return False if padding_algorithm == "EXPLICIT": - if ((input_shape[2] + paddings[0] + paddings[1] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] + paddings[2] + paddings[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + ( + input_shape[2] + + paddings[0] + + paddings[1] + - (dilations[0] * (filter_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) <= 1 or ( + ( + input_shape[3] + + paddings[2] + + paddings[3] + - (dilations[1] * (filter_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) <= 1: return False if data_format == "NCHW": if input_shape[1] != filter_shape[1] * groups: @@ -77,9 +101,10 @@ def is_program_valid(self, prog_config): def sample_program_config(self, draw): # 1. Generate shape of input:X of conv2d x_shape = draw( - st.lists(st.integers(min_value=1, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=100), min_size=4, max_size=4 + ) + ) x_shape[1] = draw(st.integers(min_value=1, max_value=10)) # 2. Generate legal attr:data_format of conv2d @@ -87,9 +112,10 @@ def sample_program_config(self, draw): # 3. Generate legal shape of input:Y of conv2d f_shape = draw( - st.lists(st.integers(min_value=1, max_value=7), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=7), min_size=4, max_size=4 + ) + ) if data_format == "NCHW": f_shape[1] = x_shape[1] else: @@ -97,35 +123,41 @@ def sample_program_config(self, draw): # 4. Generate legal attr:strides of conv2d strides = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=2, max_size=2 + ) + ) # 5. Generate legal attr:padding_algorithm of conv2d padding_algorithm = draw(st.sampled_from(["EXPLICIT", "SAME", "VALID"])) # 6. Generate legal attr:padding of conv2d padding = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=4, max_size=4 + ) + ) # 7. Generate legal attr:groups of conv2d groups = draw(st.integers(min_value=1, max_value=3)) # 8. Generate legal attr:dilations of conv2d dilations = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=2, max_size=2 + ) + ) # 9. Generate legal input:ResidualData of conv2d res_shape = [] if draw(st.booleans()): res_shape = draw( - st.lists(st.integers(min_value=1, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=100), + min_size=4, + max_size=4, + ) + ) # 10. Generate legal shape of input:bias of elementwise_add bias_shape = [f_shape[0]] @@ -133,30 +165,31 @@ def sample_program_config(self, draw): # 11. Generate legal attr:axis of elementwise_add axis = 1 - conv2d_op = OpConfig("conv2d", - inputs={ - "Input": ["input_x"], - "Filter": ["filter"], - "ResidualData": ["residualdata"] - }, - outputs={"Output": ["conv2d_out"]}, - strides=strides, - padding_algorithm=padding_algorithm, - paddings=padding, - groups=groups, - dilations=dilations, - data_format=data_format) - add_op = OpConfig("elementwise_add", - inputs={ - "X": ["conv2d_out"], - "Y": ["bias"] - }, - outputs={"Out": ["add_out"]}, - axis=axis) - - relu_op = OpConfig("relu", - inputs={"X": ["add_out"]}, - outputs={"Out": ["relu_out"]}) + conv2d_op = OpConfig( + "conv2d", + inputs={ + "Input": ["input_x"], + "Filter": ["filter"], + "ResidualData": ["residualdata"], + }, + outputs={"Output": ["conv2d_out"]}, + strides=strides, + padding_algorithm=padding_algorithm, + paddings=padding, + groups=groups, + dilations=dilations, + data_format=data_format, + ) + add_op = OpConfig( + "elementwise_add", + inputs={"X": ["conv2d_out"], "Y": ["bias"]}, + outputs={"Out": ["add_out"]}, + axis=axis, + ) + + relu_op = OpConfig( + "relu", inputs={"X": ["add_out"]}, outputs={"Out": ["relu_out"]} + ) ops = [conv2d_op, add_op, relu_op] @@ -168,16 +201,18 @@ def sample_program_config(self, draw): }, inputs={ "input_x": TensorConfig(shape=x_shape), - "residualdata": TensorConfig(shape=res_shape) + "residualdata": TensorConfig(shape=res_shape), }, outputs=ops[-1].outputs["Out"], ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=400, - passes=["conv_elementwise_add_act_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=400, + passes=["conv_elementwise_add_act_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_fuse_pass.py index 303c8017e95d2..6501d6c953b61 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_elementwise_add_fuse_pass.py @@ -23,7 +23,6 @@ class TestConvEltwiseAddFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -46,21 +45,26 @@ def sample_program_config(self, draw): out_channel = groups * out_channel_factor batch_size = draw(st.integers(min_value=1, max_value=4)) dilations = draw( - st.lists(st.integers(min_value=1, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=2), min_size=2, max_size=2 + ) + ) paddings = draw( - st.lists(st.integers(min_value=0, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=0, max_value=2), min_size=2, max_size=2 + ) + ) strides = draw( - st.lists(st.integers(min_value=1, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=2), min_size=2, max_size=2 + ) + ) - x_shape = [ - batch_size, in_channel, 64, 64 - ] if data_format == "NCHW" else [batch_size, 64, 64, in_channel] + x_shape = ( + [batch_size, in_channel, 64, 64] + if data_format == "NCHW" + else [batch_size, 64, 64, in_channel] + ) w_shape = [out_channel, filter_channel, filter_size, filter_size] scale_shape = [out_channel] bias_shape = [out_channel] @@ -77,26 +81,27 @@ def generate_bias(): def generate_scale_bias(): return np.random.random(bias_shape).astype(np.float32) - conv2d_op = OpConfig("conv2d", - inputs={ - "Input": ["input_data"], - "Filter": ["conv2d_weight"], - }, - outputs={"Output": ["conv_output"]}, - data_format=data_format, - dilations=dilations, - padding_algorithm=padding_algorithm, - groups=groups, - paddings=paddings, - strides=strides, - is_test=True) - eltwise_op = OpConfig("elementwise_add", - inputs={ - "X": ["conv_output"], - "Y": ["conv2d_bias"] - }, - outputs={"Out": ["elementwise_output"]}, - axis=axis) + conv2d_op = OpConfig( + "conv2d", + inputs={ + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + outputs={"Output": ["conv_output"]}, + data_format=data_format, + dilations=dilations, + padding_algorithm=padding_algorithm, + groups=groups, + paddings=paddings, + strides=strides, + is_test=True, + ) + eltwise_op = OpConfig( + "elementwise_add", + inputs={"X": ["conv_output"], "Y": ["conv2d_bias"]}, + outputs={"Out": ["elementwise_output"]}, + axis=axis, + ) ops = [conv2d_op, eltwise_op] program_config = ProgramConfig( @@ -105,12 +110,15 @@ def generate_scale_bias(): "input_data": TensorConfig(data_gen=partial(generate_input)), }, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial(generate_weight)), - "conv2d_bias": - TensorConfig(data_gen=partial(generate_scale_bias)), + "conv2d_weight": TensorConfig( + data_gen=partial(generate_weight) + ), + "conv2d_bias": TensorConfig( + data_gen=partial(generate_scale_bias) + ), }, - outputs=["elementwise_output"]) + outputs=["elementwise_output"], + ) return program_config def sample_predictor_configs(self, program_config): @@ -125,7 +133,8 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=1, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['conv2d_fusion'], (1e-4, 1e-4) def add_ignore_pass_case(self): @@ -137,10 +146,11 @@ def teller1(program_config, predictor_config): return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, "The output format of conv2d is wrong when data_format attribute is NHWC, \ it will trigger Broadcast dimension mismatch bug \ - when data_format attribute is NHWC and axis of eltwise op is 1 for this pass." + when data_format attribute is NHWC and axis of eltwise op is 1 for this pass.", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_eltwiseadd_bn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_eltwiseadd_bn_fuse_pass.py index a88150aecbf3b..9a25806c25f29 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_eltwiseadd_bn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_eltwiseadd_bn_fuse_pass.py @@ -25,19 +25,19 @@ class TestConvEltwiseaddBnFusePass(PassAutoScanTest): r""" - x_var f_var(persistable) - \ / - conv2d - | - conv2d_var bias_var(persistable) - \ / - elementwise_add - | - elementwise_add_var Scale(persistable) Bias(persistable) Mean(persistable) Variance(persistable) - | - batch_norm - | - Y MeanOut VarianceOut SavedMeanSavedVariance + x_var f_var(persistable) + \ / + conv2d + | + conv2d_var bias_var(persistable) + \ / + elementwise_add + | + elementwise_add_var Scale(persistable) Bias(persistable) Mean(persistable) Variance(persistable) + | + batch_norm + | + Y MeanOut VarianceOut SavedMeanSavedVariance """ def sample_predictor_configs(self, program_config): @@ -66,12 +66,36 @@ def is_program_valid(self, prog_config): if data_format != "NCHW": return False if padding_algorithm == "VALID": - if ((input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + (input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) + / strides[0] + + 1 + ) <= 1 or ( + (input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) + / strides[1] + + 1 + ) <= 1: return False if padding_algorithm == "EXPLICIT": - if ((input_shape[2] + paddings[0] + paddings[1] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] + paddings[2] + paddings[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + ( + input_shape[2] + + paddings[0] + + paddings[1] + - (dilations[0] * (filter_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) <= 1 or ( + ( + input_shape[3] + + paddings[2] + + paddings[3] + - (dilations[1] * (filter_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) <= 1: return False if data_format == "NCHW": @@ -105,9 +129,10 @@ def is_program_valid(self, prog_config): def sample_program_config(self, draw): # 1. Generate shape of input:X of conv2d x_shape = draw( - st.lists(st.integers(min_value=10, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=10, max_value=100), min_size=4, max_size=4 + ) + ) x_shape[1] = draw(st.integers(min_value=1, max_value=10)) # 2. Generate legal attr:data_format of conv2d @@ -115,9 +140,10 @@ def sample_program_config(self, draw): # 2. Generate legal shape of input:Y of conv2d f_shape = draw( - st.lists(st.integers(min_value=1, max_value=7), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=7), min_size=4, max_size=4 + ) + ) if data_format == "NCHW": f_shape[1] = x_shape[1] else: @@ -125,35 +151,41 @@ def sample_program_config(self, draw): # 3. Generate legal attr:strides of conv2d strides = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=2, max_size=2 + ) + ) # 4. Generate legal attr:padding_algorithm of conv2d padding_algorithm = draw(st.sampled_from(["EXPLICIT", "SAME", "VALID"])) # 5. Generate legal attr:padding of conv2d padding = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=4, max_size=4 + ) + ) # 6. Generate legal attr:groups of conv2d groups = draw(st.integers(min_value=1, max_value=3)) # 7. Generate legal attr:dilations of conv2d dilations = draw( - st.lists(st.integers(min_value=1, max_value=5), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=5), min_size=2, max_size=2 + ) + ) # 9. Generate legal input:ResidualData of conv2d res_shape = [] if draw(st.booleans()): res_shape = draw( - st.lists(st.integers(min_value=1, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=100), + min_size=4, + max_size=4, + ) + ) # 10. Generate legal shape of input:bias of elementwise_add bias_shape = [f_shape[0]] @@ -177,51 +209,54 @@ def sample_program_config(self, draw): epsilon = draw(st.floats(min_value=0.00001, max_value=0.001)) def generate_batch_variance(): - return (0.1 + - (1.0 - 0.1) * np.random.random(bn_variance_shape)).astype( - np.float32) - - conv2d_op = OpConfig("conv2d", - inputs={ - "Input": ["input_x"], - "Filter": ["filter"], - "ResidualData": ["residualdata"] - }, - outputs={"Output": ["conv2d_out"]}, - strides=strides, - padding_algorithm=padding_algorithm, - paddings=padding, - groups=groups, - dilations=dilations, - data_format=data_format) - add_op = OpConfig("elementwise_add", - inputs={ - "X": ["conv2d_out"], - "Y": ["bias"] - }, - outputs={"Out": ["add_out"]}, - axis=axis) - - bn_op = OpConfig("batch_norm", - inputs={ - "X": ["add_out"], - "Scale": ["scale_in"], - "Bias": ["bias_in"], - "Mean": ["mean_in"], - "Variance": ["variance_in"] - }, - outputs={ - "Y": ["y_out"], - "MeanOut": ["mean_in"], - "VarianceOut": ["variance_in"], - "SavedMean": ["SavedMean_out"], - "SavedVariance": ["SavedVariance_out"], - "ReserveSpace": ["ReserveSpace_out"] - }, - epsilon=epsilon, - is_test=True, - trainable_statistics=False, - data_layout=data_format) + return ( + 0.1 + (1.0 - 0.1) * np.random.random(bn_variance_shape) + ).astype(np.float32) + + conv2d_op = OpConfig( + "conv2d", + inputs={ + "Input": ["input_x"], + "Filter": ["filter"], + "ResidualData": ["residualdata"], + }, + outputs={"Output": ["conv2d_out"]}, + strides=strides, + padding_algorithm=padding_algorithm, + paddings=padding, + groups=groups, + dilations=dilations, + data_format=data_format, + ) + add_op = OpConfig( + "elementwise_add", + inputs={"X": ["conv2d_out"], "Y": ["bias"]}, + outputs={"Out": ["add_out"]}, + axis=axis, + ) + + bn_op = OpConfig( + "batch_norm", + inputs={ + "X": ["add_out"], + "Scale": ["scale_in"], + "Bias": ["bias_in"], + "Mean": ["mean_in"], + "Variance": ["variance_in"], + }, + outputs={ + "Y": ["y_out"], + "MeanOut": ["mean_in"], + "VarianceOut": ["variance_in"], + "SavedMean": ["SavedMean_out"], + "SavedVariance": ["SavedVariance_out"], + "ReserveSpace": ["ReserveSpace_out"], + }, + epsilon=epsilon, + is_test=True, + trainable_statistics=False, + data_layout=data_format, + ) ops = [conv2d_op, add_op, bn_op] @@ -243,15 +278,18 @@ def generate_batch_variance(): }, inputs={ "input_x": TensorConfig(shape=x_shape), - "residualdata": TensorConfig(shape=res_shape) + "residualdata": TensorConfig(shape=res_shape), }, - outputs=outputs) + outputs=outputs, + ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=300, - passes=["conv_eltwiseadd_bn_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=300, + passes=["conv_eltwiseadd_bn_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_bn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_bn_fuse_pass.py index 90d81b50471c0..3749479a6cc86 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_bn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_bn_fuse_pass.py @@ -34,10 +34,12 @@ class TestConvTransposeBnFusePass(PassAutoScanTest): ''' def test(self): - self.run_and_statis(quant=False, - max_examples=150, - max_duration=250, - passes=["conv_transpose_bn_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=150, + max_duration=250, + passes=["conv_transpose_bn_fuse_pass"], + ) def sample_program_config(self, draw): # generate random number @@ -47,26 +49,31 @@ def sample_program_config(self, draw): random_input_dim2 = draw(st.integers(min_value=20, max_value=50)) random_groups = draw(st.integers(min_value=1, max_value=2)) random_dilations = draw( - st.lists(st.integers(min_value=1, max_value=3), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=3), min_size=2, max_size=2 + ) + ) random_strides = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) random_paddings = draw( - st.lists(st.integers(min_value=0, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=0, max_value=4), min_size=2, max_size=2 + ) + ) random_padding_algorithm = draw( - st.sampled_from(["EXPLICIT", "SAME", "VALID"])) + st.sampled_from(["EXPLICIT", "SAME", "VALID"]) + ) random_data_layout = draw(st.sampled_from(["NCHW", "NHWC"])) random_use_mkldnn = draw(st.booleans()) random_output_size = [] random_filter = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) random_out_channel = draw(st.integers(min_value=10, max_value=25)) random_epsilon = draw(st.floats(min_value=0.0, max_value=0.001)) @@ -87,24 +94,24 @@ def generate_conv2d_Filter(): return np.random.random(shape).astype(np.float32) def generate_batch_norm_Scale(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) def generate_batch_norm_Bias(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) def generate_batch_norm_Mean(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) def generate_batch_norm_Variance(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) # define op conv2d_op = OpConfig( @@ -112,7 +119,7 @@ def generate_batch_norm_Variance(): inputs={ "Input": ["conv2d_Input"], "Filter": ["conv2d_Filter"], - #"Bias": ["conv2d_Bias"], + # "Bias": ["conv2d_Bias"], }, outputs={ "Output": ["conv2d_Out"], @@ -128,33 +135,34 @@ def generate_batch_norm_Variance(): 'output_padding': random_output_size, 'use_mkldnn': random_use_mkldnn, 'is_test': True, - }) - - batch_norm_op = OpConfig(type="batch_norm", - inputs={ - "X": ["conv2d_Out"], - "Scale": ["batch_norm_Scale"], - "Bias": ["batch_norm_Bias"], - "Mean": ["batch_norm_Mean"], - "Variance": ["batch_norm_Variance"], - }, - outputs={ - "Y": ["batch_norm_Y"], - "MeanOut": ["batch_norm_Mean"], - "VarianceOut": ["batch_norm_Variance"], - "SavedMean": ["batch_norm_SavedMean"], - "SavedVariance": - ["batch_norm_SavedVariance"], - "ReserveSpace": - ["batch_norm_ReserveSpace"], - }, - attrs={ - 'epsilon': random_epsilon, - 'is_test': True, - 'trainable_statistics': False, - 'data_layout': random_data_layout, - 'use_mkldnn': random_use_mkldnn, - }) + }, + ) + + batch_norm_op = OpConfig( + type="batch_norm", + inputs={ + "X": ["conv2d_Out"], + "Scale": ["batch_norm_Scale"], + "Bias": ["batch_norm_Bias"], + "Mean": ["batch_norm_Mean"], + "Variance": ["batch_norm_Variance"], + }, + outputs={ + "Y": ["batch_norm_Y"], + "MeanOut": ["batch_norm_Mean"], + "VarianceOut": ["batch_norm_Variance"], + "SavedMean": ["batch_norm_SavedMean"], + "SavedVariance": ["batch_norm_SavedVariance"], + "ReserveSpace": ["batch_norm_ReserveSpace"], + }, + attrs={ + 'epsilon': random_epsilon, + 'is_test': True, + 'trainable_statistics': False, + 'data_layout': random_data_layout, + 'use_mkldnn': random_use_mkldnn, + }, + ) # define model_net model_net = [conv2d_op, batch_norm_op] @@ -165,18 +173,22 @@ def generate_batch_norm_Variance(): "conv2d_Input": TensorConfig(data_gen=generate_conv2d_Input), }, weights={ - "conv2d_Filter": - TensorConfig(data_gen=generate_conv2d_Filter), - "batch_norm_Scale": - TensorConfig(data_gen=generate_batch_norm_Scale), - "batch_norm_Bias": - TensorConfig(data_gen=generate_batch_norm_Bias), - "batch_norm_Mean": - TensorConfig(data_gen=generate_batch_norm_Mean), - "batch_norm_Variance": - TensorConfig(data_gen=generate_batch_norm_Variance), + "conv2d_Filter": TensorConfig(data_gen=generate_conv2d_Filter), + "batch_norm_Scale": TensorConfig( + data_gen=generate_batch_norm_Scale + ), + "batch_norm_Bias": TensorConfig( + data_gen=generate_batch_norm_Bias + ), + "batch_norm_Mean": TensorConfig( + data_gen=generate_batch_norm_Mean + ), + "batch_norm_Variance": TensorConfig( + data_gen=generate_batch_norm_Variance + ), }, - outputs=["batch_norm_Y"]) + outputs=["batch_norm_Y"], + ) return program_config @@ -201,7 +213,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def add_ignore_pass_case(self): - def teller1(program_config, predictor_config): if program_config.ops[0].attrs['data_format'] == "NHWC": return True @@ -213,9 +224,13 @@ def teller2(program_config, predictor_config): return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The output format of conv2d_transpose is wrong when data_format attribute is NHWC" + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The output format of conv2d_transpose is wrong when data_format attribute is NHWC", ) - self.add_ignore_check_case(teller2, IgnoreReasons.PASS_ACCURACY_ERROR, - "there is diff when group >1 in this pass") + self.add_ignore_check_case( + teller2, + IgnoreReasons.PASS_ACCURACY_ERROR, + "there is diff when group >1 in this pass", + ) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_eltwiseadd_bn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_eltwiseadd_bn_fuse_pass.py index 91d9b863e2f9a..8b5bfc91548e6 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_eltwiseadd_bn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_conv_transpose_eltwiseadd_bn_fuse_pass.py @@ -38,10 +38,12 @@ class TestConvTransposeEltwiseaddBnFusePass(PassAutoScanTest): ''' def test(self): - self.run_and_statis(quant=False, - max_examples=150, - max_duration=250, - passes=["conv_transpose_eltwiseadd_bn_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=150, + max_duration=250, + passes=["conv_transpose_eltwiseadd_bn_fuse_pass"], + ) def sample_program_config(self, draw): # generate random number @@ -51,26 +53,31 @@ def sample_program_config(self, draw): random_input_dim2 = draw(st.integers(min_value=20, max_value=50)) random_groups = draw(st.integers(min_value=1, max_value=2)) random_dilations = draw( - st.lists(st.integers(min_value=1, max_value=3), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=3), min_size=2, max_size=2 + ) + ) random_strides = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) random_paddings = draw( - st.lists(st.integers(min_value=0, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=0, max_value=4), min_size=2, max_size=2 + ) + ) random_padding_algorithm = draw( - st.sampled_from(["EXPLICIT", "SAME", "VALID"])) + st.sampled_from(["EXPLICIT", "SAME", "VALID"]) + ) random_data_layout = draw(st.sampled_from(["NCHW", "NHWC"])) random_use_mkldnn = draw(st.booleans()) random_output_size = [] random_filter = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) random_out_channel = draw(st.integers(min_value=20, max_value=25)) random_epsilon = draw(st.floats(min_value=0.0, max_value=0.001)) @@ -91,89 +98,93 @@ def generate_conv2d_Filter(): return np.random.random(shape).astype(np.float32) def generate_elementwise_add_Y(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) def generate_batch_norm_Scale(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) def generate_batch_norm_Bias(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) def generate_batch_norm_Mean(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) def generate_batch_norm_Variance(): - return np.random.random([ - random_out_channel * random_groups * random_groups - ]).astype(np.float32) + return np.random.random( + [random_out_channel * random_groups * random_groups] + ).astype(np.float32) # define op - conv2d_op = OpConfig(type="conv2d_transpose", - inputs={ - "Input": ["conv2d_Input"], - "Filter": ["conv2d_Filter"], - }, - outputs={ - "Output": ["conv2d_Out"], - }, - attrs={ - 'groups': random_groups, - 'dilations': random_dilations, - 'strides': random_strides, - 'paddings': random_paddings, - 'padding_algorithm': random_padding_algorithm, - 'data_format': random_data_layout, - 'output_size': random_output_size, - 'output_padding': random_output_size, - 'use_mkldnn': random_use_mkldnn, - 'is_test': True, - }) - - elementwise_op = OpConfig(type="elementwise_add", - inputs={ - "X": ["conv2d_Out"], - "Y": ["elementwise_add_Y"], - }, - outputs={ - "Out": ["elementwise_add_Out"], - }, - attrs={ - 'axis': 1, - }) - - batch_norm_op = OpConfig(type="batch_norm", - inputs={ - "X": ["elementwise_add_Out"], - "Scale": ["batch_norm_Scale"], - "Bias": ["batch_norm_Bias"], - "Mean": ["batch_norm_Mean"], - "Variance": ["batch_norm_Variance"], - }, - outputs={ - "Y": ["batch_norm_Y"], - "MeanOut": ["batch_norm_Mean"], - "VarianceOut": ["batch_norm_Variance"], - "SavedMean": ["batch_norm_SavedMean"], - "SavedVariance": - ["batch_norm_SavedVariance"], - "ReserveSpace": - ["batch_norm_ReserveSpace"], - }, - attrs={ - 'epsilon': random_epsilon, - 'is_test': True, - 'trainable_statistics': False, - 'data_layout': random_data_layout, - 'use_mkldnn': random_use_mkldnn, - }) + conv2d_op = OpConfig( + type="conv2d_transpose", + inputs={ + "Input": ["conv2d_Input"], + "Filter": ["conv2d_Filter"], + }, + outputs={ + "Output": ["conv2d_Out"], + }, + attrs={ + 'groups': random_groups, + 'dilations': random_dilations, + 'strides': random_strides, + 'paddings': random_paddings, + 'padding_algorithm': random_padding_algorithm, + 'data_format': random_data_layout, + 'output_size': random_output_size, + 'output_padding': random_output_size, + 'use_mkldnn': random_use_mkldnn, + 'is_test': True, + }, + ) + + elementwise_op = OpConfig( + type="elementwise_add", + inputs={ + "X": ["conv2d_Out"], + "Y": ["elementwise_add_Y"], + }, + outputs={ + "Out": ["elementwise_add_Out"], + }, + attrs={ + 'axis': 1, + }, + ) + + batch_norm_op = OpConfig( + type="batch_norm", + inputs={ + "X": ["elementwise_add_Out"], + "Scale": ["batch_norm_Scale"], + "Bias": ["batch_norm_Bias"], + "Mean": ["batch_norm_Mean"], + "Variance": ["batch_norm_Variance"], + }, + outputs={ + "Y": ["batch_norm_Y"], + "MeanOut": ["batch_norm_Mean"], + "VarianceOut": ["batch_norm_Variance"], + "SavedMean": ["batch_norm_SavedMean"], + "SavedVariance": ["batch_norm_SavedVariance"], + "ReserveSpace": ["batch_norm_ReserveSpace"], + }, + attrs={ + 'epsilon': random_epsilon, + 'is_test': True, + 'trainable_statistics': False, + 'data_layout': random_data_layout, + 'use_mkldnn': random_use_mkldnn, + }, + ) # define model_net model_net = [conv2d_op, elementwise_op, batch_norm_op] @@ -185,20 +196,25 @@ def generate_batch_norm_Variance(): "conv2d_Input": TensorConfig(data_gen=generate_conv2d_Input), }, weights={ - "conv2d_Filter": - TensorConfig(data_gen=generate_conv2d_Filter), - "elementwise_add_Y": - TensorConfig(data_gen=generate_elementwise_add_Y), - "batch_norm_Scale": - TensorConfig(data_gen=generate_batch_norm_Scale), - "batch_norm_Bias": - TensorConfig(data_gen=generate_batch_norm_Bias), - "batch_norm_Mean": - TensorConfig(data_gen=generate_batch_norm_Mean), - "batch_norm_Variance": - TensorConfig(data_gen=generate_batch_norm_Variance), + "conv2d_Filter": TensorConfig(data_gen=generate_conv2d_Filter), + "elementwise_add_Y": TensorConfig( + data_gen=generate_elementwise_add_Y + ), + "batch_norm_Scale": TensorConfig( + data_gen=generate_batch_norm_Scale + ), + "batch_norm_Bias": TensorConfig( + data_gen=generate_batch_norm_Bias + ), + "batch_norm_Mean": TensorConfig( + data_gen=generate_batch_norm_Mean + ), + "batch_norm_Variance": TensorConfig( + data_gen=generate_batch_norm_Variance + ), }, - outputs=["batch_norm_Y"]) + outputs=["batch_norm_Y"], + ) return program_config @@ -223,7 +239,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def add_ignore_pass_case(self): - def teller1(program_config, predictor_config): if program_config.ops[0].attrs['data_format'] == "NHWC": return True @@ -235,8 +250,12 @@ def teller2(program_config, predictor_config): return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The output format of conv2d_transpose is wrong when data_format attribute is NHWC" + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The output format of conv2d_transpose is wrong when data_format attribute is NHWC", + ) + self.add_ignore_check_case( + teller2, + IgnoreReasons.PASS_ACCURACY_ERROR, + "there is diff when group >1 in this pass", ) - self.add_ignore_check_case(teller2, IgnoreReasons.PASS_ACCURACY_ERROR, - "there is diff when group >1 in this pass") diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_delete_c_identity_op_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_delete_c_identity_op_pass.py index 7efe5e97180b9..b008c134f8ece 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_delete_c_identity_op_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_delete_c_identity_op_pass.py @@ -20,7 +20,6 @@ class TestDeleteCIdentityPass(PassAutoScanTest): - def sample_predictor_configs(self, program_config): config = self.create_trt_inference_config() config.enable_tensorrt_engine( @@ -29,28 +28,34 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['relu'], (1e-5, 1e-5) def sample_program_config(self, draw): n = draw(st.integers(min_value=1, max_value=2)) - relu_op = OpConfig("relu", - inputs={"X": ["relu_x"]}, - outputs={"Out": ["relu_out"]}) - c_identity_op = OpConfig("c_identity", - inputs={"X": ["relu_out"]}, - outputs={"Out": ["id_out"]}) + relu_op = OpConfig( + "relu", inputs={"X": ["relu_x"]}, outputs={"Out": ["relu_out"]} + ) + c_identity_op = OpConfig( + "c_identity", + inputs={"X": ["relu_out"]}, + outputs={"Out": ["id_out"]}, + ) program_config = ProgramConfig( ops=[relu_op, c_identity_op], weights={}, inputs={"relu_x": TensorConfig(shape=[n])}, - outputs=["id_out"]) + outputs=["id_out"], + ) return program_config def test(self): - self.run_and_statis(max_examples=2, - min_success_num=2, - passes=["delete_c_identity_op_pass"]) + self.run_and_statis( + max_examples=2, + min_success_num=2, + passes=["delete_c_identity_op_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_emb_eltwise_layernorm_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_emb_eltwise_layernorm_fuse_pass.py index ce51cda6e45a8..3cbd48dea6d4e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_emb_eltwise_layernorm_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_emb_eltwise_layernorm_fuse_pass.py @@ -24,21 +24,21 @@ class TestEmbeddingEltwiseLayerNormFusePass(PassAutoScanTest): r''' - in_var1 emb_var in_var2 emb_var in_var3 emb_var in_var emb_var - | | | | | | | | - lookup_table lookup_table lookup_table ... lookup_table - | | | | - lkt_var lkt_var lkt_var lkt_var - \ / | ... | - elementwise_add | | - \ / | - elementwise_add | - | | - elt_var / - \ / - elementwise_add - | - layer_norm + in_var1 emb_var in_var2 emb_var in_var3 emb_var in_var emb_var + | | | | | | | | + lookup_table lookup_table lookup_table ... lookup_table + | | | | + lkt_var lkt_var lkt_var lkt_var + \ / | ... | + elementwise_add | | + \ / | + elementwise_add | + | | + elt_var / + \ / + elementwise_add + | + layer_norm ''' def is_program_valid(self, program_config: ProgramConfig) -> bool: @@ -54,16 +54,20 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: if program_config.ops[3].attrs['axis'] not in [-1, 2]: return False - if not (program_config.ops[5].attrs['epsilon'] >= 0 - and program_config.ops[5].attrs['epsilon'] <= 0.001): + if not ( + program_config.ops[5].attrs['epsilon'] >= 0 + and program_config.ops[5].attrs['epsilon'] <= 0.001 + ): return False if program_config.ops[5].attrs['begin_norm_axis'] != 2: return False # input check - if program_config.weights['embedding_weight1'].shape[ - 1] != program_config.weights['layer_norm_scale'].shape[0]: + if ( + program_config.weights['embedding_weight1'].shape[1] + != program_config.weights['layer_norm_scale'].shape[0] + ): return False return True @@ -83,17 +87,17 @@ def sample_program_config(self, draw): def generate_input(attrs): if attrs[0]['op_type'] == 'lookup_table': - return np.random.randint(0, - attrs[3]['weight_size'][0], - size=(attrs[3]['batch_size'], - attrs[3]['input_dim'], - 1)).astype(np.int64) + return np.random.randint( + 0, + attrs[3]['weight_size'][0], + size=(attrs[3]['batch_size'], attrs[3]['input_dim'], 1), + ).astype(np.int64) else: - return np.random.randint(0, - attrs[3]['weight_size'][0], - size=(attrs[3]['batch_size'], - attrs[3]['input_dim'])).astype( - np.int64) + return np.random.randint( + 0, + attrs[3]['weight_size'][0], + size=(attrs[3]['batch_size'], attrs[3]['input_dim']), + ).astype(np.int64) def generate_weight1(attrs): # set embedding weight by attrs @@ -102,116 +106,128 @@ def generate_weight1(attrs): def generate_weight2(attrs): # set layernorm weight by attrs if attrs[2]['begin_norm_axis'] == 1: - return np.random.random(attrs[3]['input_dim'] * - attrs[3]['weight_size'][1]).astype( - np.float32) + return np.random.random( + attrs[3]['input_dim'] * attrs[3]['weight_size'][1] + ).astype(np.float32) else: return np.random.random(attrs[3]['weight_size'][1]).astype( - np.float32) + np.float32 + ) - attrs = [{ - 'is_sparse': is_sparse, - 'is_distributed': is_distributed, - 'padding_idx': padding_idx, - 'op_type': op_type - }, { - 'axis': axis - }, { - 'begin_norm_axis': begin_norm_axis, - 'epsilon': epsilon - }, { - 'batch_size': batch_size, - 'input_dim': input_dim, - 'weight_size': weight_size - }] + attrs = [ + { + 'is_sparse': is_sparse, + 'is_distributed': is_distributed, + 'padding_idx': padding_idx, + 'op_type': op_type, + }, + {'axis': axis}, + {'begin_norm_axis': begin_norm_axis, 'epsilon': epsilon}, + { + 'batch_size': batch_size, + 'input_dim': input_dim, + 'weight_size': weight_size, + }, + ] - emb_op1 = OpConfig(type=attrs[0]['op_type'], - inputs={ - "Ids": ["input_data1"], - "W": ["embedding_weight1"] - }, - outputs={"Out": ["embedding_output1"]}, - attrs={ - 'is_sparse': attrs[0]['is_sparse'], - 'is_distributed': attrs[0]['is_distributed'], - 'padding_idx': attrs[0]['padding_idx'] - }) - emb_op2 = OpConfig(type=attrs[0]['op_type'], - inputs={ - "Ids": ["input_data2"], - "W": ["embedding_weight2"] - }, - outputs={"Out": ["embedding_output2"]}, - attrs={ - 'is_sparse': attrs[0]['is_sparse'], - 'is_distributed': attrs[0]['is_distributed'], - 'padding_idx': attrs[0]['padding_idx'] - }) - emb_op3 = OpConfig(type=attrs[0]['op_type'], - inputs={ - "Ids": ["input_data3"], - "W": ["embedding_weight3"] - }, - outputs={"Out": ["embedding_output3"]}, - attrs={ - 'is_sparse': attrs[0]['is_sparse'], - 'is_distributed': attrs[0]['is_distributed'], - 'padding_idx': attrs[0]['padding_idx'] - }) - add_op1 = OpConfig(type='elementwise_add', - inputs={ - "X": [emb_op2.outputs["Out"][0]], - "Y": [emb_op3.outputs["Out"][0]], - }, - outputs={"Out": ["elementwise_add_output1"]}, - attrs={"axis": attrs[1]['axis']}) - add_op2 = OpConfig(type='elementwise_add', - inputs={ - "X": [add_op1.outputs["Out"][0]], - "Y": [emb_op1.outputs["Out"][0]], - }, - outputs={"Out": ["elementwise_add_output2"]}, - attrs={"axis": attrs[1]['axis']}) - layer_norm_op = OpConfig(type='layer_norm', - inputs={ - "X": [add_op2.outputs["Out"][0]], - "Bias": ["layer_norm_bias"], - "Scale": ["layer_norm_scale"] - }, - outputs={ - "Y": ["layer_norm_output1"], - "Mean": ["layer_norm_output2"], - "Variance": ["layer_norm_output3"] - }, - attrs={ - 'begin_norm_axis': - attrs[2]['begin_norm_axis'], - 'epsilon': attrs[2]['epsilon'] - }) + emb_op1 = OpConfig( + type=attrs[0]['op_type'], + inputs={"Ids": ["input_data1"], "W": ["embedding_weight1"]}, + outputs={"Out": ["embedding_output1"]}, + attrs={ + 'is_sparse': attrs[0]['is_sparse'], + 'is_distributed': attrs[0]['is_distributed'], + 'padding_idx': attrs[0]['padding_idx'], + }, + ) + emb_op2 = OpConfig( + type=attrs[0]['op_type'], + inputs={"Ids": ["input_data2"], "W": ["embedding_weight2"]}, + outputs={"Out": ["embedding_output2"]}, + attrs={ + 'is_sparse': attrs[0]['is_sparse'], + 'is_distributed': attrs[0]['is_distributed'], + 'padding_idx': attrs[0]['padding_idx'], + }, + ) + emb_op3 = OpConfig( + type=attrs[0]['op_type'], + inputs={"Ids": ["input_data3"], "W": ["embedding_weight3"]}, + outputs={"Out": ["embedding_output3"]}, + attrs={ + 'is_sparse': attrs[0]['is_sparse'], + 'is_distributed': attrs[0]['is_distributed'], + 'padding_idx': attrs[0]['padding_idx'], + }, + ) + add_op1 = OpConfig( + type='elementwise_add', + inputs={ + "X": [emb_op2.outputs["Out"][0]], + "Y": [emb_op3.outputs["Out"][0]], + }, + outputs={"Out": ["elementwise_add_output1"]}, + attrs={"axis": attrs[1]['axis']}, + ) + add_op2 = OpConfig( + type='elementwise_add', + inputs={ + "X": [add_op1.outputs["Out"][0]], + "Y": [emb_op1.outputs["Out"][0]], + }, + outputs={"Out": ["elementwise_add_output2"]}, + attrs={"axis": attrs[1]['axis']}, + ) + layer_norm_op = OpConfig( + type='layer_norm', + inputs={ + "X": [add_op2.outputs["Out"][0]], + "Bias": ["layer_norm_bias"], + "Scale": ["layer_norm_scale"], + }, + outputs={ + "Y": ["layer_norm_output1"], + "Mean": ["layer_norm_output2"], + "Variance": ["layer_norm_output3"], + }, + attrs={ + 'begin_norm_axis': attrs[2]['begin_norm_axis'], + 'epsilon': attrs[2]['epsilon'], + }, + ) program_config = ProgramConfig( ops=[emb_op1, emb_op2, emb_op3, add_op1, add_op2, layer_norm_op], weights={ - "embedding_weight1": - TensorConfig(data_gen=partial(generate_weight1, attrs[3])), - "embedding_weight2": - TensorConfig(data_gen=partial(generate_weight1, attrs[3])), - "embedding_weight3": - TensorConfig(data_gen=partial(generate_weight1, attrs[3])), - "layer_norm_bias": - TensorConfig(data_gen=partial(generate_weight2, attrs)), - "layer_norm_scale": - TensorConfig(data_gen=partial(generate_weight2, attrs)) + "embedding_weight1": TensorConfig( + data_gen=partial(generate_weight1, attrs[3]) + ), + "embedding_weight2": TensorConfig( + data_gen=partial(generate_weight1, attrs[3]) + ), + "embedding_weight3": TensorConfig( + data_gen=partial(generate_weight1, attrs[3]) + ), + "layer_norm_bias": TensorConfig( + data_gen=partial(generate_weight2, attrs) + ), + "layer_norm_scale": TensorConfig( + data_gen=partial(generate_weight2, attrs) + ), }, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input, attrs)), - "input_data2": - TensorConfig(data_gen=partial(generate_input, attrs)), - "input_data3": - TensorConfig(data_gen=partial(generate_input, attrs)) + "input_data1": TensorConfig( + data_gen=partial(generate_input, attrs) + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, attrs) + ), + "input_data3": TensorConfig( + data_gen=partial(generate_input, attrs) + ), }, - outputs=["layer_norm_output1"]) + outputs=["layer_norm_output1"], + ) return program_config @@ -227,7 +243,8 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Half, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['fused_embedding_eltwise_layernorm'], (1e-5, 1e-5) # trt dynamic_shape config = self.create_trt_inference_config() @@ -237,61 +254,71 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Half, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) if program_config.ops[0].type == 'lookup_table': config.set_trt_dynamic_shape_info( { "input_data1": [1, 4, 1], "input_data2": [1, 4, 1], - "input_data3": [1, 4, 1] - }, { + "input_data3": [1, 4, 1], + }, + { "input_data1": [4, 512, 1], "input_data2": [4, 512, 1], - "input_data3": [4, 512, 1] - }, { + "input_data3": [4, 512, 1], + }, + { "input_data1": [2, 128, 1], "input_data2": [2, 128, 1], - "input_data3": [2, 128, 1] - }) + "input_data3": [2, 128, 1], + }, + ) else: config.set_trt_dynamic_shape_info( { "input_data1": [1, 4], "input_data2": [1, 4], - "input_data3": [1, 4] - }, { + "input_data3": [1, 4], + }, + { "input_data1": [4, 512], "input_data2": [4, 512], - "input_data3": [4, 512] - }, { + "input_data3": [4, 512], + }, + { "input_data1": [2, 128], "input_data2": [2, 128], - "input_data3": [2, 128] - }) + "input_data3": [2, 128], + }, + ) yield config, ['fused_embedding_eltwise_layernorm'], (1e-5, 1e-5) def add_ignore_pass_case(self): - def teller1(program_config, predictor_config): - if program_config.ops[3].attrs['axis'] in [ - -1, 2 - ] and program_config.ops[5].attrs[ - 'begin_norm_axis'] == 2 and program_config.weights[ - 'embedding_weight1'].shape in [(64, 32), (64, 64)]: + if ( + program_config.ops[3].attrs['axis'] in [-1, 2] + and program_config.ops[5].attrs['begin_norm_axis'] == 2 + and program_config.weights['embedding_weight1'].shape + in [(64, 32), (64, 64)] + ): return True return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The pass output has diff in a specific case. We need to fix it as soon as possible." + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The pass output has diff in a specific case. We need to fix it as soon as possible.", ) def test(self): # this fuse need to fix, now there's no program can ran successfully - self.run_and_statis(quant=False, - max_examples=50, - passes=["embedding_eltwise_layernorm_fuse_pass"], - min_success_num=0) + self.run_and_statis( + quant=False, + max_examples=50, + passes=["embedding_eltwise_layernorm_fuse_pass"], + min_success_num=0, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_elementwise_layernorm_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_elementwise_layernorm_fuse_pass.py index b15fb88d73d6e..5074fd1d1b740 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_elementwise_layernorm_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_elementwise_layernorm_fuse_pass.py @@ -21,7 +21,6 @@ class FcElementLayernormFusePassDataGen: - def __init__(self, min_v, max_v, shape, dtype): self.min_v = min_v self.max_v = max_v @@ -29,8 +28,9 @@ def __init__(self, min_v, max_v, shape, dtype): self.dtype = dtype def __call__(self): - return np.random.normal(self.min_v, self.max_v, - self.shape).astype(self.dtype) + return np.random.normal(self.min_v, self.max_v, self.shape).astype( + self.dtype + ) class TestFCElementwiseLayerNormFusePass(PassAutoScanTest): @@ -56,18 +56,20 @@ def sample_predictor_configs(self, program_config): def sample_program_config(self, draw): # 1. Generate shape of input:X of fc x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=5)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=5 + ) + ) x_shape = [2, 1] x_rank = len(x_shape) # 2. Generate attr:in_num_col_dims of fc in_num_col_dims = draw(st.integers(min_value=1, max_value=x_rank - 1)) # 3. Generate legal shape of input:W/bias of fc w_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) w_shape[0] = int(np.prod(x_shape[in_num_col_dims:])) w_shape = [1, 2] fc_bias_shape = [ @@ -84,17 +86,14 @@ def sample_program_config(self, draw): axis = draw(st.integers(min_value=-1, max_value=0)) # 5. Generate legal shape of layer_norm begin_norm_axis = draw( - st.integers(min_value=1, max_value=len(fc_out_shape) - 1)) + st.integers(min_value=1, max_value=len(fc_out_shape) - 1) + ) layer_norm_shape = [int(np.prod(fc_out_shape[begin_norm_axis:]))] epsilon = 1e-5 fc_op = OpConfig( "fc", - inputs={ - "Input": ["fc_x"], - "W": ["fc_w"], - "Bias": ["fc_bias"] - }, + inputs={"Input": ["fc_x"], "W": ["fc_w"], "Bias": ["fc_bias"]}, outputs={"Out": ["fc_out"]}, in_num_col_dims=in_num_col_dims, padding_weights=False, @@ -104,43 +103,40 @@ def sample_program_config(self, draw): ) add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["fc_out"], - "Y": ["add_bias"] - }, + inputs={"X": ["fc_out"], "Y": ["add_bias"]}, outputs={"Out": ["add_out"]}, axis=axis, ) - layer_norm_op = OpConfig("layer_norm", - inputs={ - "X": ["add_out"], - "Scale": ["scale"], - "Bias": ["layer_norm_bias"] - }, - outputs={ - "Y": ["layer_norm_out"], - "Mean": ["layer_norm_mean"], - "Variance": ["layer_norm_var"] - }, - begin_norm_axis=begin_norm_axis, - epsilon=epsilon) + layer_norm_op = OpConfig( + "layer_norm", + inputs={ + "X": ["add_out"], + "Scale": ["scale"], + "Bias": ["layer_norm_bias"], + }, + outputs={ + "Y": ["layer_norm_out"], + "Mean": ["layer_norm_mean"], + "Variance": ["layer_norm_var"], + }, + begin_norm_axis=begin_norm_axis, + epsilon=epsilon, + ) ops = [fc_op, add_op, layer_norm_op] program_config = ProgramConfig( ops=ops, weights={ - "fc_w": - TensorConfig(shape=w_shape), - "fc_bias": - TensorConfig(shape=fc_bias_shape), - "add_bias": - TensorConfig(shape=add_bias_shape), - "scale": - TensorConfig(shape=layer_norm_shape, - data_gen=FcElementLayernormFusePassDataGen( - 0.0, 0.5, layer_norm_shape, np.float32)), - "layer_norm_bias": - TensorConfig(shape=layer_norm_shape), + "fc_w": TensorConfig(shape=w_shape), + "fc_bias": TensorConfig(shape=fc_bias_shape), + "add_bias": TensorConfig(shape=add_bias_shape), + "scale": TensorConfig( + shape=layer_norm_shape, + data_gen=FcElementLayernormFusePassDataGen( + 0.0, 0.5, layer_norm_shape, np.float32 + ), + ), + "layer_norm_bias": TensorConfig(shape=layer_norm_shape), }, inputs={ "fc_x": TensorConfig(shape=x_shape), @@ -150,9 +146,11 @@ def sample_program_config(self, draw): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=300, - passes=["fc_elementwise_layernorm_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=300, + passes=["fc_elementwise_layernorm_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_fuse_pass.py index 58494fba43c25..73aa1738b846b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_fuse_pass.py @@ -50,7 +50,8 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['fc'], (1e-5, 1e-5) def add_ignore_pass_case(self): @@ -74,8 +75,10 @@ def teller1(program_config, predictor_config): def teller2(program_config, predictor_config): # TODO fuse has bug while axis != -1 axis = program_config.ops[1].attrs["axis"] - if axis != -1 and axis != program_config.ops[0].attrs[ - "x_num_col_dims"]: + if ( + axis != -1 + and axis != program_config.ops[0].attrs["x_num_col_dims"] + ): return True return False @@ -102,18 +105,21 @@ def is_program_valid(self, prog_config): def sample_program_config(self, draw): # 1. Generate shape of input:X of mul x_shape = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=4 + ) + ) # 2. Generate attr:x_num_col_dims/y_num_col_dims of mul x_num_col_dims = draw( - st.integers(min_value=1, max_value=len(x_shape) - 1)) + st.integers(min_value=1, max_value=len(x_shape) - 1) + ) y_num_col_dims = 1 # 3. Generate legal shape of input:Y of mul y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = int(np.prod(x_shape[x_num_col_dims:])) # 4. Generate legal attr:axis of elementwise_add mul_out_shape = x_shape[:x_num_col_dims] + y_shape[1:] @@ -122,12 +128,13 @@ def sample_program_config(self, draw): if axis >= 0: max_bias_rank = x_num_col_dims + 1 - axis bias_rank = draw(st.integers(min_value=1, max_value=max_bias_rank)) - bias_shape = mul_out_shape[axis:axis + bias_rank] + bias_shape = mul_out_shape[axis : axis + bias_rank] else: max_bias_rank = 1 bias_rank = draw( - st.integers(min_value=1, max_value=len(mul_out_shape))) - bias_shape = mul_out_shape[-1 * bias_rank:] + st.integers(min_value=1, max_value=len(mul_out_shape)) + ) + bias_shape = mul_out_shape[-1 * bias_rank :] # 6. Random choose if use broadcast for elementwise_add, e.g [3, 4] -> [1, 4] if draw(st.booleans()): broadcast_dims = draw(st.integers(min_value=1, max_value=bias_rank)) @@ -147,28 +154,22 @@ def sample_program_config(self, draw): # Use function `add_skip_pass_case` to ignore the programs even if they cause bug while runing mul_op = OpConfig( "mul", - inputs={ - "X": ["mul_x"], - "Y": ["mul_y"] - }, + inputs={"X": ["mul_x"], "Y": ["mul_y"]}, outputs={"Out": ["mul_out"]}, x_num_col_dims=x_num_col_dims, y_num_col_dims=y_num_col_dims, ) add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["mul_out"], - "Y": ["bias"] - }, + inputs={"X": ["mul_out"], "Y": ["bias"]}, outputs={"Out": ["add_out"]}, axis=axis, ) ops = [mul_op, add_op] if has_relu: - relu_op = OpConfig("relu", - inputs={"X": ["add_out"]}, - outputs={"Out": ["relu_out"]}) + relu_op = OpConfig( + "relu", inputs={"X": ["add_out"]}, outputs={"Out": ["relu_out"]} + ) ops.append(relu_op) program_config = ProgramConfig( ops=ops, @@ -184,9 +185,9 @@ def sample_program_config(self, draw): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=500, - passes=["fc_fuse_pass"]) + self.run_and_statis( + quant=False, max_examples=500, passes=["fc_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_gru_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_gru_fuse_pass.py index b722fb798c5b7..f0c2cbefc0c96 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_gru_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_gru_fuse_pass.py @@ -20,28 +20,29 @@ class FcGruFusePassTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): dict_dim, emb_dim = 128, 64 - data = fluid.data(name='step_data', - shape=[None], - dtype='int64', - lod_level=1) + data = fluid.data( + name='step_data', shape=[None], dtype='int64', lod_level=1 + ) emb = fluid.embedding(input=data, size=[dict_dim, emb_dim]) hidden_dim = 512 x = fluid.layers.fc(input=emb, size=hidden_dim * 3) - hidden = fluid.layers.dynamic_gru(input=x, - size=hidden_dim, - bias_attr=True, - origin_mode=False, - is_reverse=True) + hidden = fluid.layers.dynamic_gru( + input=x, + size=hidden_dim, + bias_attr=True, + origin_mode=False, + is_reverse=True, + ) batch = 16 lod_tensor = fluid.LoDTensor() lod_tensor.set( np.random.randint(0, dict_dim, size=[batch]).astype("int64"), - fluid.CPUPlace()) + fluid.CPUPlace(), + ) lod_tensor.set_lod([[0, batch]]) self.feeds = {"step_data": lod_tensor} self.fetch_list = [hidden] @@ -53,28 +54,29 @@ def test_check_output(self): class MulGruFusePassTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): dict_dim, emb_dim = 128, 64 - data = fluid.data(name='step_data', - shape=[None], - dtype='int64', - lod_level=1) + data = fluid.data( + name='step_data', shape=[None], dtype='int64', lod_level=1 + ) emb = fluid.embedding(input=data, size=[dict_dim, emb_dim]) hidden_dim = 512 x = fluid.layers.fc(input=emb, size=hidden_dim * 3, bias_attr=False) - hidden = fluid.layers.dynamic_gru(input=x, - size=hidden_dim, - bias_attr=True, - origin_mode=False, - is_reverse=True) + hidden = fluid.layers.dynamic_gru( + input=x, + size=hidden_dim, + bias_attr=True, + origin_mode=False, + is_reverse=True, + ) batch = 16 lod_tensor = fluid.LoDTensor() lod_tensor.set( np.random.randint(0, dict_dim, size=[batch]).astype("int64"), - fluid.CPUPlace()) + fluid.CPUPlace(), + ) lod_tensor.set_lod([[0, batch]]) self.feeds = {"step_data": lod_tensor} self.fetch_list = [hidden] diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_lstm_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_lstm_fuse_pass.py index 040cf05b02a44..acef755d57125 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_fc_lstm_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_fc_lstm_fuse_pass.py @@ -20,26 +20,26 @@ class MulLstmFusePassTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): dict_dim, emb_dim = 128, 64 hidden_dim = 512 - data = fluid.data(name='data', - shape=[1], - dtype='int64', - lod_level=1) + data = fluid.data( + name='data', shape=[1], dtype='int64', lod_level=1 + ) emb = fluid.embedding(input=data, size=[dict_dim, emb_dim]) x = fluid.layers.fc(input=emb, size=hidden_dim * 4, bias_attr=False) - forward, cell = fluid.layers.dynamic_lstm(input=x, - size=hidden_dim * 4) + forward, cell = fluid.layers.dynamic_lstm( + input=x, size=hidden_dim * 4 + ) batch = 16 lod_tensor = fluid.LoDTensor() lod_tensor.set( np.random.randint(0, dict_dim, size=[batch]).astype("int64"), - fluid.CPUPlace()) + fluid.CPUPlace(), + ) lod_tensor.set_lod([[0, batch]]) self.feeds = {"data": lod_tensor} self.fetch_list = [forward, cell] diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_flatten2_matmul_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_flatten2_matmul_fuse_pass.py index 21e0843a5aab6..2d84561b770f9 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_flatten2_matmul_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_flatten2_matmul_fuse_pass.py @@ -44,9 +44,10 @@ def sample_predictor_configs(self, program_config): def sample_program_config(self, draw): # 1. Generate shape and attr of flatten2 x_shape = draw( - st.lists(st.integers(min_value=1, max_value=10), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=10), min_size=4, max_size=4 + ) + ) # [a, b, c, d] => [a, b*c*d] flatten_axis = 1 flatten_shape = [x_shape[0], x_shape[1] * x_shape[2] * x_shape[3]] @@ -58,9 +59,10 @@ def sample_program_config(self, draw): # 3. Generate legal shape of input:Y of matmul y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = flatten_shape[1] # 4. Generate legal attr:axis of elementwise_add @@ -82,17 +84,11 @@ def sample_program_config(self, draw): "X": ["flatten2_x"], }, axis=flatten_axis, - outputs={ - "Out": ["flatten2_out"], - "XShape": ["xshape"] - }, + outputs={"Out": ["flatten2_out"], "XShape": ["xshape"]}, ) matmul_op = OpConfig( "matmul", - inputs={ - "X": ["flatten2_out"], - "Y": ["matmul_y"] - }, + inputs={"X": ["flatten2_out"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, transpose_X=transpose_X, @@ -107,10 +103,7 @@ def sample_program_config(self, draw): add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["matmul_out"], - "Y": ["bias"] - }, + inputs={"X": ["matmul_out"], "Y": ["bias"]}, outputs={"Out": ["add_out"]}, axis=axis, ) @@ -143,10 +136,12 @@ def sample_program_config(self, draw): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=50, - max_duration=1000, - passes=["gpu_cpu_flatten2_matmul_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=50, + max_duration=1000, + passes=["gpu_cpu_flatten2_matmul_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_identity_scale_clean_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_identity_scale_clean_pass.py index e97eac1652849..2261ba416d7f2 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_identity_scale_clean_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_identity_scale_clean_pass.py @@ -20,7 +20,6 @@ class TestIdentityScaleCleanPass(PassAutoScanTest): - def sample_predictor_configs(self, program_config): config = self.create_trt_inference_config() config.enable_tensorrt_engine( @@ -29,7 +28,8 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['relu'], (1e-5, 1e-5) def sample_program_config(self, draw): @@ -39,25 +39,29 @@ def sample_program_config(self, draw): h = draw(st.integers(min_value=1, max_value=20)) w = draw(st.integers(min_value=1, max_value=20)) - relu_op = OpConfig("relu", - inputs={"X": ["relu_x"]}, - outputs={"Out": ["relu_out"]}) - scale_op = OpConfig("scale", - inputs={"X": ["relu_out"]}, - outputs={"Out": ["scale_out"]}, - bias=0., - scale=1., - bias_after_scale=True) + relu_op = OpConfig( + "relu", inputs={"X": ["relu_x"]}, outputs={"Out": ["relu_out"]} + ) + scale_op = OpConfig( + "scale", + inputs={"X": ["relu_out"]}, + outputs={"Out": ["scale_out"]}, + bias=0.0, + scale=1.0, + bias_after_scale=True, + ) program_config = ProgramConfig( ops=[relu_op, scale_op], weights={}, inputs={"relu_x": TensorConfig(shape=[n, c, h, w])}, - outputs=["scale_out"]) + outputs=["scale_out"], + ) return program_config def test(self): - self.run_and_statis(max_examples=25, - passes=["identity_scale_op_clean_pass"]) + self.run_and_statis( + max_examples=25, passes=["identity_scale_op_clean_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_layer_norm_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_layer_norm_fuse_pass.py index 9b928436b5998..3c6bf6a1ca9fa 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_layer_norm_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_layer_norm_fuse_pass.py @@ -76,15 +76,17 @@ def teller1(program_config, predictor_config): def sample_program_config(self, draw): # 1. Generate shape of input:X x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=4, - max_size=5)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=4, max_size=5 + ) + ) x_shape_rank = len(x_shape) # 2. Generate attrs of reduce_mean keep_dim = draw(st.booleans()) reduce_all = False begin_norm_axis = draw( - st.integers(min_value=1, max_value=x_shape_rank - 1)) + st.integers(min_value=1, max_value=x_shape_rank - 1) + ) if begin_norm_axis == x_shape_rank - 1 and draw(st.booleans()): reduce_mean_dim = [-1] else: @@ -108,22 +110,30 @@ def sample_program_config(self, draw): pow_axis = -1 def generate_pow_data(): - return np.array([ - 2, - ], dtype="float32") + return np.array( + [ + 2, + ], + dtype="float32", + ) # 5. Generate attrs of elementwise_add if keep_dim: add_axis = draw( - st.integers(min_value=-1, max_value=x_shape_rank - 1)) + st.integers(min_value=-1, max_value=x_shape_rank - 1) + ) else: add_axis = draw( - st.integers(min_value=-1, max_value=begin_norm_axis - 1)) + st.integers(min_value=-1, max_value=begin_norm_axis - 1) + ) def generate_epsilon_data(): - return np.array([ - 1e-5, - ], dtype="float32") + return np.array( + [ + 1e-5, + ], + dtype="float32", + ) # 6. Generate attrs of elementwise_div div_axis = 0 @@ -151,19 +161,13 @@ def generate_epsilon_data(): ) sub_op = OpConfig( "elementwise_sub", - inputs={ - "X": ["x"], - "Y": ["mean_out"] - }, + inputs={"X": ["x"], "Y": ["mean_out"]}, outputs={"Out": ["sub_out"]}, axis=sub_axis, ) pow_op = OpConfig( "elementwise_pow", - inputs={ - "X": ["sub_out"], - "Y": ["pow_y"] - }, + inputs={"X": ["sub_out"], "Y": ["pow_y"]}, outputs={"Out": ["pow_out"]}, axis=pow_axis, ) @@ -179,10 +183,7 @@ def generate_epsilon_data(): ) add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["mean_out2"], - "Y": ["epsilon_var"] - }, + inputs={"X": ["mean_out2"], "Y": ["epsilon_var"]}, outputs={"Out": ["add_out"]}, axis=add_axis, ) @@ -195,35 +196,33 @@ def generate_epsilon_data(): ) div_op = OpConfig( "elementwise_div", - inputs={ - "X": ["sub_out"], - "Y": ["sqrt_out"] - }, + inputs={"X": ["sub_out"], "Y": ["sqrt_out"]}, outputs={"Out": ["div_out"]}, axis=div_axis, ) mul_op = OpConfig( "elementwise_mul", - inputs={ - "X": ["div_out"], - "Y": ["gamma_var"] - }, + inputs={"X": ["div_out"], "Y": ["gamma_var"]}, outputs={"Out": ["mul_out"]}, axis=mul_axis, ) add_op2 = OpConfig( "elementwise_add", - inputs={ - "X": ["mul_out"], - "Y": ["beta_var"] - }, + inputs={"X": ["mul_out"], "Y": ["beta_var"]}, outputs={"Out": ["add_out2"]}, axis=add_axis2, ) ops = [ - mean_op1, sub_op, pow_op, mean_op2, add_op, sqrt_op, div_op, mul_op, - add_op2 + mean_op1, + sub_op, + pow_op, + mean_op2, + add_op, + sqrt_op, + div_op, + mul_op, + add_op2, ] program_config = ProgramConfig( diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_layernorm_shift_partition_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_layernorm_shift_partition_pass.py index aeaff113f58d2..56436710f3332 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_layernorm_shift_partition_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_layernorm_shift_partition_pass.py @@ -49,14 +49,19 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) - config.set_trt_dynamic_shape_info({ - "input_data": [1, 9, 96], - }, { - "input_data": [4, 3136, 768], - }, { - "input_data": [1, 784, 384], - }) + use_calib_mode=False, + ) + config.set_trt_dynamic_shape_info( + { + "input_data": [1, 9, 96], + }, + { + "input_data": [4, 3136, 768], + }, + { + "input_data": [1, 784, 384], + }, + ) yield config, ['layernorm_shift_partition'], (1e-5, 1e-5) # trt dynamic_shape @@ -67,14 +72,19 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Half, use_static=False, - use_calib_mode=False) - config.set_trt_dynamic_shape_info({ - "input_data": [1, 9, 96], - }, { - "input_data": [4, 3136, 768], - }, { - "input_data": [1, 784, 384], - }) + use_calib_mode=False, + ) + config.set_trt_dynamic_shape_info( + { + "input_data": [1, 9, 96], + }, + { + "input_data": [4, 3136, 768], + }, + { + "input_data": [1, 784, 384], + }, + ) yield config, ['layernorm_shift_partition'], (1e-3, 1e-3) def sample_program_config(self, draw): @@ -90,95 +100,112 @@ def sample_program_config(self, draw): def generate_input(attrs): return np.random.random( - [attrs[1]["batch_size"], - *attrs[1]["input_dim"]]).astype(np.float32) + [attrs[1]["batch_size"], *attrs[1]["input_dim"]] + ).astype(np.float32) def generate_weight(attrs): return np.random.random(attrs[1]['input_dim'][-1]).astype( - np.float32) + np.float32 + ) - attrs = [{ - 'begin_norm_axis': begin_norm_axis, - 'epsilon': epsilon, - }, { - 'batch_size': batch_size, - 'input_dim': [(window_size * move_shape)**2, dim], - }, { - 'axis': axis, - 'input_resolution': window_size * move_shape, - 'move_shape': move_shape, - 'window_size': window_size, - }] + attrs = [ + { + 'begin_norm_axis': begin_norm_axis, + 'epsilon': epsilon, + }, + { + 'batch_size': batch_size, + 'input_dim': [(window_size * move_shape) ** 2, dim], + }, + { + 'axis': axis, + 'input_resolution': window_size * move_shape, + 'move_shape': move_shape, + 'window_size': window_size, + }, + ] - layer_norm_op = OpConfig(type="layer_norm", - inputs={ - "X": ["input_data"], - "Bias": ["layer_norm_bias"], - "Scale": ["layer_norm_scale"] - }, - outputs={ - "Y": ["layer_norm_output1"], - "Mean": ["layer_norm_output2"], - "Variance": ["layer_norm_output3"] - }, - attrs={ - "begin_norm_axis": - attrs[0]["begin_norm_axis"], - "epsilon": attrs[0]["epsilon"], - }) - reshape_op2 = OpConfig(type="reshape2", - inputs={ - "X": ["layer_norm_output1"], - }, - outputs={ - "Out": ["reshape_output2"], - "XShape": ["reshape_output2_xshape"], - }, - attrs={ - 'shape': [ - -1, attrs[2]["input_resolution"], - attrs[2]["input_resolution"], - attrs[1]["input_dim"][-1] - ] - }) - reshape_op3 = OpConfig(type="reshape2", - inputs={ - "X": ["reshape_output2"], - }, - outputs={ - "Out": ["reshape_output3"], - "XShape": ["reshape_output3_xshape"], - }, - attrs={ - 'shape': [ - -1, attrs[2]["move_shape"], - attrs[2]["window_size"], - attrs[2]["move_shape"], - attrs[2]["window_size"], - attrs[1]["input_dim"][-1] - ] - }) - transpose_op4 = OpConfig(type='transpose2', - inputs={ - "X": ["reshape_output3"], - }, - outputs={"Out": ["transpose_output4"]}, - attrs={"axis": attrs[2]['axis']}) - reshape_op5 = OpConfig(type="reshape2", - inputs={ - "X": ["transpose_output4"], - }, - outputs={ - "Out": ["reshape_output5"], - "XShape": ["reshape_output5_xshape"], - }, - attrs={ - 'shape': [ - -1, attrs[2]["window_size"], - attrs[2]["window_size"], - attrs[1]["input_dim"][-1] - ] - }) + layer_norm_op = OpConfig( + type="layer_norm", + inputs={ + "X": ["input_data"], + "Bias": ["layer_norm_bias"], + "Scale": ["layer_norm_scale"], + }, + outputs={ + "Y": ["layer_norm_output1"], + "Mean": ["layer_norm_output2"], + "Variance": ["layer_norm_output3"], + }, + attrs={ + "begin_norm_axis": attrs[0]["begin_norm_axis"], + "epsilon": attrs[0]["epsilon"], + }, + ) + reshape_op2 = OpConfig( + type="reshape2", + inputs={ + "X": ["layer_norm_output1"], + }, + outputs={ + "Out": ["reshape_output2"], + "XShape": ["reshape_output2_xshape"], + }, + attrs={ + 'shape': [ + -1, + attrs[2]["input_resolution"], + attrs[2]["input_resolution"], + attrs[1]["input_dim"][-1], + ] + }, + ) + reshape_op3 = OpConfig( + type="reshape2", + inputs={ + "X": ["reshape_output2"], + }, + outputs={ + "Out": ["reshape_output3"], + "XShape": ["reshape_output3_xshape"], + }, + attrs={ + 'shape': [ + -1, + attrs[2]["move_shape"], + attrs[2]["window_size"], + attrs[2]["move_shape"], + attrs[2]["window_size"], + attrs[1]["input_dim"][-1], + ] + }, + ) + transpose_op4 = OpConfig( + type='transpose2', + inputs={ + "X": ["reshape_output3"], + }, + outputs={"Out": ["transpose_output4"]}, + attrs={"axis": attrs[2]['axis']}, + ) + reshape_op5 = OpConfig( + type="reshape2", + inputs={ + "X": ["transpose_output4"], + }, + outputs={ + "Out": ["reshape_output5"], + "XShape": ["reshape_output5_xshape"], + }, + attrs={ + 'shape': [ + -1, + attrs[2]["window_size"], + attrs[2]["window_size"], + attrs[1]["input_dim"][-1], + ] + }, + ) reshape_op6 = OpConfig( type="reshape2", inputs={ @@ -189,35 +216,49 @@ def generate_weight(attrs): "XShape": ["reshape_output6_xshape"], }, attrs={ - 'shape': - [-1, attrs[2]["window_size"]**2, attrs[1]["input_dim"][-1]] - }) + 'shape': [ + -1, + attrs[2]["window_size"] ** 2, + attrs[1]["input_dim"][-1], + ] + }, + ) program_config = ProgramConfig( ops=[ - layer_norm_op, reshape_op2, reshape_op3, transpose_op4, - reshape_op5, reshape_op6 + layer_norm_op, + reshape_op2, + reshape_op3, + transpose_op4, + reshape_op5, + reshape_op6, ], weights={ - "layer_norm_bias": - TensorConfig(data_gen=partial(generate_weight, attrs)), - "layer_norm_scale": - TensorConfig(data_gen=partial(generate_weight, attrs)) + "layer_norm_bias": TensorConfig( + data_gen=partial(generate_weight, attrs) + ), + "layer_norm_scale": TensorConfig( + data_gen=partial(generate_weight, attrs) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, attrs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, attrs) + ), }, - outputs=["reshape_output6"]) + outputs=["reshape_output6"], + ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=50, - passes=["layernorm_shift_partition_fuse_pass"], - max_duration=250, - min_success_num=50) + self.run_and_statis( + quant=False, + max_examples=50, + passes=["layernorm_shift_partition_fuse_pass"], + max_duration=250, + min_success_num=50, + ) class TestLayernormShiftPartition2Pass(PassAutoScanTest): @@ -248,14 +289,19 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) - config.set_trt_dynamic_shape_info({ - "input_data": [1, 9, 96], - }, { - "input_data": [4, 3136, 768], - }, { - "input_data": [1, 784, 384], - }) + use_calib_mode=False, + ) + config.set_trt_dynamic_shape_info( + { + "input_data": [1, 9, 96], + }, + { + "input_data": [4, 3136, 768], + }, + { + "input_data": [1, 784, 384], + }, + ) yield config, ['layernorm_shift_partition'], (1e-5, 1e-5) # trt dynamic_shape @@ -266,14 +312,19 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Half, use_static=False, - use_calib_mode=False) - config.set_trt_dynamic_shape_info({ - "input_data": [1, 9, 96], - }, { - "input_data": [4, 3136, 768], - }, { - "input_data": [1, 784, 384], - }) + use_calib_mode=False, + ) + config.set_trt_dynamic_shape_info( + { + "input_data": [1, 9, 96], + }, + { + "input_data": [4, 3136, 768], + }, + { + "input_data": [1, 784, 384], + }, + ) yield config, ['layernorm_shift_partition'], (1e-3, 1e-3) def sample_program_config(self, draw): @@ -289,106 +340,124 @@ def sample_program_config(self, draw): def generate_input(attrs): return np.random.random( - [attrs[1]["batch_size"], - *attrs[1]["input_dim"]]).astype(np.float32) + [attrs[1]["batch_size"], *attrs[1]["input_dim"]] + ).astype(np.float32) def generate_weight(attrs): return np.random.random(attrs[1]['input_dim'][-1]).astype( - np.float32) + np.float32 + ) - attrs = [{ - 'begin_norm_axis': begin_norm_axis, - 'epsilon': epsilon, - }, { - 'batch_size': batch_size, - 'input_dim': [(window_size * move_shape)**2, dim], - }, { - 'axis': axis, - 'input_resolution': window_size * move_shape, - 'move_shape': move_shape, - 'window_size': window_size, - }] + attrs = [ + { + 'begin_norm_axis': begin_norm_axis, + 'epsilon': epsilon, + }, + { + 'batch_size': batch_size, + 'input_dim': [(window_size * move_shape) ** 2, dim], + }, + { + 'axis': axis, + 'input_resolution': window_size * move_shape, + 'move_shape': move_shape, + 'window_size': window_size, + }, + ] - layer_norm_op = OpConfig(type="layer_norm", - inputs={ - "X": ["input_data"], - "Bias": ["layer_norm_bias"], - "Scale": ["layer_norm_scale"] - }, - outputs={ - "Y": ["layer_norm_output1"], - "Mean": ["layer_norm_output2"], - "Variance": ["layer_norm_output3"] - }, - attrs={ - "begin_norm_axis": - attrs[0]["begin_norm_axis"], - "epsilon": attrs[0]["epsilon"], - }) - reshape_op2 = OpConfig(type="reshape2", - inputs={ - "X": ["layer_norm_output1"], - }, - outputs={ - "Out": ["reshape_output2"], - "XShape": ["reshape_output2_xshape"], - }, - attrs={ - 'shape': [ - -1, attrs[2]["input_resolution"], - attrs[2]["input_resolution"], - attrs[1]["input_dim"][-1] - ] - }) - roll_op1 = OpConfig(type="roll", - inputs={"X": ["reshape_output2"]}, - outputs={"Out": ["roll_output1"]}, - attrs={ - "axis": [1, 2], - "shifts": [ - -math.floor( - (attrs[2]["window_size"]) / 2.0), - -math.floor((attrs[2]["window_size"]) / 2.0) - ] - }) - reshape_op3 = OpConfig(type="reshape2", - inputs={ - "X": ["roll_output1"], - }, - outputs={ - "Out": ["reshape_output3"], - "XShape": ["reshape_output3_xshape"], - }, - attrs={ - 'shape': [ - -1, attrs[2]["move_shape"], - attrs[2]["window_size"], - attrs[2]["move_shape"], - attrs[2]["window_size"], - attrs[1]["input_dim"][-1] - ] - }) - transpose_op4 = OpConfig(type='transpose2', - inputs={ - "X": ["reshape_output3"], - }, - outputs={"Out": ["transpose_output4"]}, - attrs={"axis": attrs[2]['axis']}) - reshape_op5 = OpConfig(type="reshape2", - inputs={ - "X": ["transpose_output4"], - }, - outputs={ - "Out": ["reshape_output5"], - "XShape": ["reshape_output5_xshape"], - }, - attrs={ - 'shape': [ - -1, attrs[2]["window_size"], - attrs[2]["window_size"], - attrs[1]["input_dim"][-1] - ] - }) + layer_norm_op = OpConfig( + type="layer_norm", + inputs={ + "X": ["input_data"], + "Bias": ["layer_norm_bias"], + "Scale": ["layer_norm_scale"], + }, + outputs={ + "Y": ["layer_norm_output1"], + "Mean": ["layer_norm_output2"], + "Variance": ["layer_norm_output3"], + }, + attrs={ + "begin_norm_axis": attrs[0]["begin_norm_axis"], + "epsilon": attrs[0]["epsilon"], + }, + ) + reshape_op2 = OpConfig( + type="reshape2", + inputs={ + "X": ["layer_norm_output1"], + }, + outputs={ + "Out": ["reshape_output2"], + "XShape": ["reshape_output2_xshape"], + }, + attrs={ + 'shape': [ + -1, + attrs[2]["input_resolution"], + attrs[2]["input_resolution"], + attrs[1]["input_dim"][-1], + ] + }, + ) + roll_op1 = OpConfig( + type="roll", + inputs={"X": ["reshape_output2"]}, + outputs={"Out": ["roll_output1"]}, + attrs={ + "axis": [1, 2], + "shifts": [ + -math.floor((attrs[2]["window_size"]) / 2.0), + -math.floor((attrs[2]["window_size"]) / 2.0), + ], + }, + ) + reshape_op3 = OpConfig( + type="reshape2", + inputs={ + "X": ["roll_output1"], + }, + outputs={ + "Out": ["reshape_output3"], + "XShape": ["reshape_output3_xshape"], + }, + attrs={ + 'shape': [ + -1, + attrs[2]["move_shape"], + attrs[2]["window_size"], + attrs[2]["move_shape"], + attrs[2]["window_size"], + attrs[1]["input_dim"][-1], + ] + }, + ) + transpose_op4 = OpConfig( + type='transpose2', + inputs={ + "X": ["reshape_output3"], + }, + outputs={"Out": ["transpose_output4"]}, + attrs={"axis": attrs[2]['axis']}, + ) + reshape_op5 = OpConfig( + type="reshape2", + inputs={ + "X": ["transpose_output4"], + }, + outputs={ + "Out": ["reshape_output5"], + "XShape": ["reshape_output5_xshape"], + }, + attrs={ + 'shape': [ + -1, + attrs[2]["window_size"], + attrs[2]["window_size"], + attrs[1]["input_dim"][-1], + ] + }, + ) reshape_op6 = OpConfig( type="reshape2", inputs={ @@ -399,35 +468,50 @@ def generate_weight(attrs): "XShape": ["reshape_output6_xshape"], }, attrs={ - 'shape': - [-1, attrs[2]["window_size"]**2, attrs[1]["input_dim"][-1]] - }) + 'shape': [ + -1, + attrs[2]["window_size"] ** 2, + attrs[1]["input_dim"][-1], + ] + }, + ) program_config = ProgramConfig( ops=[ - layer_norm_op, reshape_op2, roll_op1, reshape_op3, - transpose_op4, reshape_op5, reshape_op6 + layer_norm_op, + reshape_op2, + roll_op1, + reshape_op3, + transpose_op4, + reshape_op5, + reshape_op6, ], weights={ - "layer_norm_bias": - TensorConfig(data_gen=partial(generate_weight, attrs)), - "layer_norm_scale": - TensorConfig(data_gen=partial(generate_weight, attrs)) + "layer_norm_bias": TensorConfig( + data_gen=partial(generate_weight, attrs) + ), + "layer_norm_scale": TensorConfig( + data_gen=partial(generate_weight, attrs) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, attrs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, attrs) + ), }, - outputs=["reshape_output6"]) + outputs=["reshape_output6"], + ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=50, - passes=["layernorm_shift_partition_fuse_pass"], - max_duration=250, - min_success_num=50) + self.run_and_statis( + quant=False, + max_examples=50, + passes=["layernorm_shift_partition_fuse_pass"], + max_duration=250, + min_success_num=50, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_to_mul_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_to_mul_pass.py index e6db1204897c5..138782f00e10a 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_to_mul_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_to_mul_pass.py @@ -21,9 +21,9 @@ class TestMapMatmulToMulPass(PassAutoScanTest): r""" - x_var y_var(persistable) - \ / - matmul + x_var y_var(persistable) + \ / + matmul """ def sample_predictor_configs(self, program_config): @@ -67,19 +67,23 @@ def teller1(program_config, predictor_config): return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The pass error on TRT while shape of mul_x > 5.") + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The pass error on TRT while shape of mul_x > 5.", + ) def sample_program_config(self, draw): # 1. Generate shape and attr of matmul x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=5)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=5 + ) + ) y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = x_shape[-1] alpha = 1.0 transpose_X = False @@ -87,10 +91,7 @@ def sample_program_config(self, draw): matmul_op = OpConfig( "matmul", - inputs={ - "X": ["matmul_x"], - "Y": ["matmul_y"] - }, + inputs={"X": ["matmul_x"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, transpose_X=transpose_X, @@ -121,10 +122,12 @@ def sample_program_config(self, draw): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=100, - passes=["gpu_cpu_map_matmul_to_mul_pass"], - max_duration=180) + self.run_and_statis( + quant=False, + max_examples=100, + passes=["gpu_cpu_map_matmul_to_mul_pass"], + max_duration=180, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_matmul_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_matmul_pass.py index 38f86dc71702c..d2dec6060734c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_matmul_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_matmul_pass.py @@ -21,9 +21,9 @@ class TestMapMatmulToMulPass(PassAutoScanTest): r""" - x_var y_var(persistable) - \ / - matmul_v2 + x_var y_var(persistable) + \ / + matmul_v2 """ def sample_predictor_configs(self, program_config): @@ -63,19 +63,23 @@ def teller1(program_config, predictor_config): return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The pass error on TRT while shape of mul_x > 5.") + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The pass error on TRT while shape of mul_x > 5.", + ) def sample_program_config(self, draw): # 1. Generate shape and attr of matmul x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=5)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=5 + ) + ) y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) transpose_X = draw(st.booleans()) transpose_Y = draw(st.booleans()) if transpose_X: @@ -89,15 +93,12 @@ def sample_program_config(self, draw): else: y_shape[0] = x_shape[-1] - y_shape = x_shape[0:len(x_shape) - 2] + y_shape + y_shape = x_shape[0 : len(x_shape) - 2] + y_shape alpha = 1.0 matmul_op = OpConfig( "matmul_v2", - inputs={ - "X": ["matmul_x"], - "Y": ["matmul_y"] - }, + inputs={"X": ["matmul_x"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, trans_x=transpose_X, @@ -128,9 +129,11 @@ def sample_program_config(self, draw): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=100, - passes=["gpu_cpu_map_matmul_v2_to_matmul_pass"]) + self.run_and_statis( + quant=False, + max_examples=100, + passes=["gpu_cpu_map_matmul_v2_to_matmul_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_mul_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_mul_pass.py index d4c6db1c7f5d4..148610edad134 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_mul_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_map_matmul_v2_to_mul_pass.py @@ -21,9 +21,9 @@ class TestMapMatmulToMulPass(PassAutoScanTest): r""" - x_var y_var(persistable) - \ / - matmul_v2 + x_var y_var(persistable) + \ / + matmul_v2 """ def sample_predictor_configs(self, program_config): @@ -63,19 +63,23 @@ def teller1(program_config, predictor_config): return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The pass error on TRT while shape of mul_x > 5.") + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The pass error on TRT while shape of mul_x > 5.", + ) def sample_program_config(self, draw): # 1. Generate shape and attr of matmul x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=5)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=5 + ) + ) y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = x_shape[-1] alpha = 1.0 transpose_X = False @@ -83,10 +87,7 @@ def sample_program_config(self, draw): matmul_op = OpConfig( "matmul_v2", - inputs={ - "X": ["matmul_x"], - "Y": ["matmul_y"] - }, + inputs={"X": ["matmul_x"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, trans_x=transpose_X, @@ -117,9 +118,11 @@ def sample_program_config(self, draw): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=100, - passes=["gpu_cpu_map_matmul_v2_to_mul_pass"]) + self.run_and_statis( + quant=False, + max_examples=100, + passes=["gpu_cpu_map_matmul_v2_to_mul_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_scale_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_scale_fuse_pass.py index 6db5468292c4e..2a60d62db3d57 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_scale_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_scale_fuse_pass.py @@ -21,11 +21,11 @@ class TestMatmulScaleFusePass(PassAutoScanTest): r""" - x_var y_var(persistable) - \ / - matmul - | - scale + x_var y_var(persistable) + \ / + matmul + | + scale """ def sample_predictor_configs(self, program_config): @@ -50,14 +50,18 @@ def sample_predictor_configs(self, program_config): def sample_program_config(self, draw): # 1. Generate shape and attr of matmul x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=5)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=5 + ) + ) x_shape_rank = len(x_shape) y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=x_shape_rank, - max_size=x_shape_rank)) + st.lists( + st.integers(min_value=1, max_value=8), + min_size=x_shape_rank, + max_size=x_shape_rank, + ) + ) y_shape_rank = len(y_shape) y_shape[-2] = x_shape[-1] for i in range(y_shape_rank - 3, -1, -1): @@ -75,10 +79,7 @@ def sample_program_config(self, draw): matmul_op = OpConfig( "matmul", - inputs={ - "X": ["matmul_x"], - "Y": ["matmul_y"] - }, + inputs={"X": ["matmul_x"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, transpose_X=transpose_X, transpose_Y=transpose_Y, @@ -95,10 +96,7 @@ def sample_program_config(self, draw): if is_scale_tensor: scale_op = OpConfig( "scale", - inputs={ - "X": ["matmul_out"], - "ScaleTensor": ["scale_tensor"] - }, + inputs={"X": ["matmul_out"], "ScaleTensor": ["scale_tensor"]}, outputs={"Out": ["scale_out"]}, scale=scale_value, bias=0.0, @@ -122,7 +120,7 @@ def sample_program_config(self, draw): if is_scale_tensor: weights = { "matmul_y": TensorConfig(shape=y_shape), - "scale_tensor": TensorConfig(shape=scale_shape) + "scale_tensor": TensorConfig(shape=scale_shape), } inputs = { "matmul_x": TensorConfig(shape=x_shape), diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_v2_scale_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_v2_scale_fuse_pass.py index a9b48e7cf3627..4b0c226e1a7ad 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_v2_scale_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_matmul_v2_scale_fuse_pass.py @@ -21,13 +21,13 @@ class TestMatmulV2ScaleFusePass(PassAutoScanTest): r""" - x_var y_var(persistable) x_var y_var*scale(persistable) - \ / \ / - matmul_v2 matmul_v2 - | => | - scale scale_out - ｜ - scale_out + x_var y_var(persistable) x_var y_var*scale(persistable) + \ / \ / + matmul_v2 matmul_v2 + | => | + scale scale_out + ｜ + scale_out """ def sample_predictor_configs(self, program_config): @@ -44,14 +44,18 @@ def sample_predictor_configs(self, program_config): def sample_program_config(self, draw): # 1. Generate shape and attr of matmul x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=5)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=5 + ) + ) x_shape_rank = len(x_shape) y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=x_shape_rank, - max_size=x_shape_rank)) + st.lists( + st.integers(min_value=1, max_value=8), + min_size=x_shape_rank, + max_size=x_shape_rank, + ) + ) y_shape_rank = len(y_shape) y_shape[-2] = x_shape[-1] for i in range(y_shape_rank - 3, -1, -1): @@ -68,10 +72,7 @@ def sample_program_config(self, draw): matmul_v2_op = OpConfig( "matmul_v2", - inputs={ - "X": ["matmul_x"], - "Y": ["matmul_y"] - }, + inputs={"X": ["matmul_x"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, trans_x=transpose_X, trans_y=transpose_Y, @@ -86,10 +87,7 @@ def sample_program_config(self, draw): if is_scale_tensor: scale_op = OpConfig( "scale", - inputs={ - "X": ["matmul_out"], - "ScaleTensor": ["scale_tensor"] - }, + inputs={"X": ["matmul_out"], "ScaleTensor": ["scale_tensor"]}, outputs={"Out": ["scale_out"]}, scale=scale_value, bias=0.0, diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_merge_layernorm_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_merge_layernorm_fuse_pass.py index cd25181f501e4..897e63e7cf5e9 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_merge_layernorm_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_merge_layernorm_fuse_pass.py @@ -47,10 +47,13 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) - config.set_trt_dynamic_shape_info({"input_data": [1, 196, 96]}, - {"input_data": [4, 3136, 384]}, - {"input_data": [1, 3136, 96]}) + use_calib_mode=False, + ) + config.set_trt_dynamic_shape_info( + {"input_data": [1, 196, 96]}, + {"input_data": [4, 3136, 384]}, + {"input_data": [1, 3136, 96]}, + ) yield config, ["merge_layernorm"], (1e-5, 1e-5) # trt dynamic_shape fp16 config = self.create_trt_inference_config() @@ -60,10 +63,13 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Half, use_static=False, - use_calib_mode=False) - config.set_trt_dynamic_shape_info({"input_data": [1, 196, 96]}, - {"input_data": [4, 3136, 384]}, - {"input_data": [1, 3136, 96]}) + use_calib_mode=False, + ) + config.set_trt_dynamic_shape_info( + {"input_data": [1, 196, 96]}, + {"input_data": [4, 3136, 384]}, + {"input_data": [1, 3136, 96]}, + ) yield config, ["merge_layernorm"], (1e-3, 1e-3) def sample_program_config(self, draw): @@ -72,39 +78,45 @@ def sample_program_config(self, draw): input_n = draw(st.sampled_from([96, 192, 384])) layernorm_40_begin_norm_axis = 2 layernorm_40_epsilon = draw( - st.floats(min_value=0.0000001, max_value=0.001)) + st.floats(min_value=0.0000001, max_value=0.001) + ) def generate_input(attrs): - return np.random.random([ - attrs[3]['batch_size'], - attrs[3]['input_H_W'] * attrs[3]['input_H_W'], - attrs[3]['input_n'] - ]).astype(np.float32) + return np.random.random( + [ + attrs[3]['batch_size'], + attrs[3]['input_H_W'] * attrs[3]['input_H_W'], + attrs[3]['input_n'], + ] + ).astype(np.float32) def generate_weight(attrs): - return np.random.random([attrs[3]['input_n'] * 4 - ]).astype(np.float32) + return np.random.random([attrs[3]['input_n'] * 4]).astype( + np.float32 + ) - attrs = [{ - 'shape': [-1, input_H_W, input_H_W, input_n] - }, { - 'shape': [-1, int(input_H_W * input_H_W / 4), - int(input_n * 4)] - }, { - 'begin_norm_axis': layernorm_40_begin_norm_axis, - 'epsilon': layernorm_40_epsilon - }, { - 'batch_size': batch_size, - 'input_H_W': input_H_W, - 'input_n': input_n - }] - reshape2_00_op = OpConfig(type="reshape2", - inputs={'X': ['input_data']}, - outputs={ - 'Out': ['reshape2_00_out'], - 'XShape': ['reshape2_00_outxshape'] - }, - attrs={'shape': attrs[0]['shape']}) + attrs = [ + {'shape': [-1, input_H_W, input_H_W, input_n]}, + {'shape': [-1, int(input_H_W * input_H_W / 4), int(input_n * 4)]}, + { + 'begin_norm_axis': layernorm_40_begin_norm_axis, + 'epsilon': layernorm_40_epsilon, + }, + { + 'batch_size': batch_size, + 'input_H_W': input_H_W, + 'input_n': input_n, + }, + ] + reshape2_00_op = OpConfig( + type="reshape2", + inputs={'X': ['input_data']}, + outputs={ + 'Out': ['reshape2_00_out'], + 'XShape': ['reshape2_00_outxshape'], + }, + attrs={'shape': attrs[0]['shape']}, + ) strided_slice_10_op = OpConfig( type="strided_slice", inputs={'Input': ['reshape2_00_out']}, @@ -114,8 +126,9 @@ def generate_weight(attrs): 'starts': [0, 0], 'infer_flags': [1, 1], 'ends': [attrs[3]['input_H_W'], attrs[3]['input_H_W']], - 'strides': [2, 2] - }) + 'strides': [2, 2], + }, + ) strided_slice_11_op = OpConfig( type="strided_slice", inputs={'Input': ['reshape2_00_out']}, @@ -125,8 +138,9 @@ def generate_weight(attrs): 'starts': [1, 0], 'infer_flags': [1, 1], 'ends': [attrs[3]['input_H_W'], attrs[3]['input_H_W']], - 'strides': [2, 2] - }) + 'strides': [2, 2], + }, + ) strided_slice_12_op = OpConfig( type="strided_slice", inputs={'Input': ['reshape2_00_out']}, @@ -136,8 +150,9 @@ def generate_weight(attrs): 'starts': [0, 1], 'infer_flags': [1, 1], 'ends': [attrs[3]['input_H_W'], attrs[3]['input_H_W']], - 'strides': [2, 2] - }) + 'strides': [2, 2], + }, + ) strided_slice_13_op = OpConfig( type="strided_slice", inputs={'Input': ['reshape2_00_out']}, @@ -147,68 +162,84 @@ def generate_weight(attrs): 'starts': [1, 1], 'infer_flags': [1, 1], 'ends': [attrs[3]['input_H_W'], attrs[3]['input_H_W']], - 'strides': [2, 2] - }) - concat_20_op = OpConfig(type="concat", - inputs={ - 'X': [ - 'strided_slice_10_out', - 'strided_slice_11_out', - 'strided_slice_12_out', - 'strided_slice_13_out' - ] - }, - outputs={'Out': ['concat_20_out']}, - attrs={'axis': -1}) - reshape2_30_op = OpConfig(type='reshape2', - inputs={'X': ['concat_20_out']}, - outputs={ - 'Out': ['reshape2_30_Out'], - 'XShape': ['reshape2_30_XShape'] - }, - attrs={'shape': attrs[1]['shape']}) - layernorm_40_op = OpConfig(type='layer_norm', - inputs={ - 'X': ['reshape2_30_Out'], - 'Bias': ['layer_norm_bias'], - 'Scale': ['layer_norm_scale'] - }, - outputs={ - "Y": ["layer_norm_out"], - "Mean": ["layer_norm_outMean"], - "Variance": ["layer_norm_outVariance"] - }, - attrs={ - 'begin_norm_axis': - attrs[2]['begin_norm_axis'], - 'epsilon': - attrs[2]['epsilon'] - }) + 'strides': [2, 2], + }, + ) + concat_20_op = OpConfig( + type="concat", + inputs={ + 'X': [ + 'strided_slice_10_out', + 'strided_slice_11_out', + 'strided_slice_12_out', + 'strided_slice_13_out', + ] + }, + outputs={'Out': ['concat_20_out']}, + attrs={'axis': -1}, + ) + reshape2_30_op = OpConfig( + type='reshape2', + inputs={'X': ['concat_20_out']}, + outputs={ + 'Out': ['reshape2_30_Out'], + 'XShape': ['reshape2_30_XShape'], + }, + attrs={'shape': attrs[1]['shape']}, + ) + layernorm_40_op = OpConfig( + type='layer_norm', + inputs={ + 'X': ['reshape2_30_Out'], + 'Bias': ['layer_norm_bias'], + 'Scale': ['layer_norm_scale'], + }, + outputs={ + "Y": ["layer_norm_out"], + "Mean": ["layer_norm_outMean"], + "Variance": ["layer_norm_outVariance"], + }, + attrs={ + 'begin_norm_axis': attrs[2]['begin_norm_axis'], + 'epsilon': attrs[2]['epsilon'], + }, + ) program_config = ProgramConfig( ops=[ - reshape2_00_op, strided_slice_10_op, strided_slice_11_op, - strided_slice_12_op, strided_slice_13_op, concat_20_op, - reshape2_30_op, layernorm_40_op + reshape2_00_op, + strided_slice_10_op, + strided_slice_11_op, + strided_slice_12_op, + strided_slice_13_op, + concat_20_op, + reshape2_30_op, + layernorm_40_op, ], weights={ - 'layer_norm_bias': - TensorConfig(data_gen=partial(generate_weight, attrs)), - 'layer_norm_scale': - TensorConfig(data_gen=partial(generate_weight, attrs)) + 'layer_norm_bias': TensorConfig( + data_gen=partial(generate_weight, attrs) + ), + 'layer_norm_scale': TensorConfig( + data_gen=partial(generate_weight, attrs) + ), }, inputs={ - 'input_data': - TensorConfig(data_gen=partial(generate_input, attrs)) + 'input_data': TensorConfig( + data_gen=partial(generate_input, attrs) + ) }, - outputs=['layer_norm_out']) + outputs=['layer_norm_out'], + ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=50, - passes=["merge_layernorm_fuse_pass"], - max_duration=250, - min_success_num=50) + self.run_and_statis( + quant=False, + max_examples=50, + passes=["merge_layernorm_fuse_pass"], + max_duration=250, + min_success_num=50, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_batch_norm_act_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_batch_norm_act_fuse_pass.py index 63bb9e7256d77..85c3aa2cd1c1d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_batch_norm_act_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_batch_norm_act_fuse_pass.py @@ -22,7 +22,6 @@ class TestScaleMatmulMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -55,40 +54,40 @@ def generate_input(): def generate_weight(): return np.random.random(channel).astype(np.float32) - batch_norm_op = OpConfig(type="batch_norm", - inputs={ - "X": ["input_data"], - "Bias": ["Bias"], - "Mean": ["Mean"], - "Scale": ["Scale"], - "Variance": ["Variance"] - }, - outputs={ - "Y": ["norm_output"], - "MeanOut": ["Mean"], - "VarianceOut": ["Variance"], - "SavedMean": ["SavedMean"], - "SavedVariance": ["SavedVariance"] - }, - attrs={ - "data_layout": data_layout, - "epsilon": epsilon, - "fuse_with_relu": fuse_with_relu, - "is_test": is_test, - "momentum": momentum, - "trainable_statistics": - trainable_statistics, - "use_global_stats": use_global_stats, - "use_mkldnn": use_mkldnn1 - }) - - relu_op = OpConfig(type="relu", - inputs={"X": ["norm_output"]}, - outputs={"Out": ["relu_output"]}, - attrs={ - "use_cudnn": use_cudnn, - "use_mkldnn": use_mkldnn2 - }) + batch_norm_op = OpConfig( + type="batch_norm", + inputs={ + "X": ["input_data"], + "Bias": ["Bias"], + "Mean": ["Mean"], + "Scale": ["Scale"], + "Variance": ["Variance"], + }, + outputs={ + "Y": ["norm_output"], + "MeanOut": ["Mean"], + "VarianceOut": ["Variance"], + "SavedMean": ["SavedMean"], + "SavedVariance": ["SavedVariance"], + }, + attrs={ + "data_layout": data_layout, + "epsilon": epsilon, + "fuse_with_relu": fuse_with_relu, + "is_test": is_test, + "momentum": momentum, + "trainable_statistics": trainable_statistics, + "use_global_stats": use_global_stats, + "use_mkldnn": use_mkldnn1, + }, + ) + + relu_op = OpConfig( + type="relu", + inputs={"X": ["norm_output"]}, + outputs={"Out": ["relu_output"]}, + attrs={"use_cudnn": use_cudnn, "use_mkldnn": use_mkldnn2}, + ) model_net = [batch_norm_op, relu_op] @@ -98,12 +97,13 @@ def generate_weight(): "Bias": TensorConfig(data_gen=partial(generate_weight)), "Mean": TensorConfig(data_gen=partial(generate_weight)), "Scale": TensorConfig(data_gen=partial(generate_weight)), - "Variance": TensorConfig(data_gen=partial(generate_weight)) + "Variance": TensorConfig(data_gen=partial(generate_weight)), }, inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)) }, - outputs=["relu_output"]) + outputs=["relu_output"], + ) return program_config diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_bias_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_bias_fuse_pass.py index 21d3234929081..9bf4fa55b0c53 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_bias_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_bias_fuse_pass.py @@ -22,7 +22,6 @@ class TestConv3dBiasMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -39,78 +38,80 @@ def sample_program_config(self, draw): def generate_input1(attrs): if attrs[0]['data_format'] == "NCDHW": return np.random.random( - [attrs[2]['batch_size'], 48, 64, 32, 64]).astype(np.float32) + [attrs[2]['batch_size'], 48, 64, 32, 64] + ).astype(np.float32) else: return np.random.random( - [attrs[2]['batch_size'], 64, 32, 64, 48]).astype(np.float32) + [attrs[2]['batch_size'], 64, 32, 64, 48] + ).astype(np.float32) def generate_weight1(): - return np.random.random([16, int(48 / groups), 3, 3, - 3]).astype(np.float32) + return np.random.random([16, int(48 / groups), 3, 3, 3]).astype( + np.float32 + ) def generate_weight2(): return np.random.random([16]).astype(np.float32) - attrs = [{ - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides - }, { - "axis": axis - }, { - 'batch_size': batch_size - }] - - ops_config = [{ - "op_type": "conv3d", - "op_inputs": { - "Input": ["input_data1"], - "Filter": ["conv_weight"] - }, - "op_outputs": { - "Output": ["conv_output"] - }, - "op_attrs": { - "data_format": attrs[0]['data_format'], - "dilations": attrs[0]['dilations'], - "padding_algorithm": attrs[0]['padding_algorithm'], - "groups": attrs[0]['groups'], - "paddings": attrs[0]['paddings'], - "strides": attrs[0]['strides'], - "is_test": True - } - }, { - "op_type": "elementwise_add", - "op_inputs": { - "X": ["conv_output"], - "Y": ["elementwise_weight"] + attrs = [ + { + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, }, - "op_outputs": { - "Out": ["elementwise_output"] + {"axis": axis}, + {'batch_size': batch_size}, + ] + + ops_config = [ + { + "op_type": "conv3d", + "op_inputs": { + "Input": ["input_data1"], + "Filter": ["conv_weight"], + }, + "op_outputs": {"Output": ["conv_output"]}, + "op_attrs": { + "data_format": attrs[0]['data_format'], + "dilations": attrs[0]['dilations'], + "padding_algorithm": attrs[0]['padding_algorithm'], + "groups": attrs[0]['groups'], + "paddings": attrs[0]['paddings'], + "strides": attrs[0]['strides'], + "is_test": True, + }, }, - "op_attrs": { - 'axis': attrs[1]['axis'] + { + "op_type": "elementwise_add", + "op_inputs": { + "X": ["conv_output"], + "Y": ["elementwise_weight"], + }, + "op_outputs": {"Out": ["elementwise_output"]}, + "op_attrs": {'axis': attrs[1]['axis']}, }, - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "conv_weight": - TensorConfig(data_gen=partial(generate_weight1)), - "elementwise_weight": - TensorConfig(data_gen=partial(generate_weight2)) + "conv_weight": TensorConfig(data_gen=partial(generate_weight1)), + "elementwise_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), }, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input1, attrs)) + "input_data1": TensorConfig( + data_gen=partial(generate_input1, attrs) + ) }, - outputs=["elementwise_output"]) + outputs=["elementwise_output"], + ) return program_config diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_op.py index 22454e55a139f..f9ea557a53ab5 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv3d_op.py @@ -23,53 +23,54 @@ class TestMkldnnConv3dOp(MkldnnAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): if kwargs["data_format"] == "NCDHW": - return np.random.random([kwargs["batch_size"], 48, 64, 32, - 64]).astype(np.float32) + return np.random.random( + [kwargs["batch_size"], 48, 64, 32, 64] + ).astype(np.float32) else: - return np.random.random([kwargs["batch_size"], 64, 32, 64, - 48]).astype(np.float32) + return np.random.random( + [kwargs["batch_size"], 64, 32, 64, 48] + ).astype(np.float32) def generate_weight(*args, **kwargs): - return np.random.random([16, - int(48 / kwargs["groups"]), 3, 3, - 3]).astype(np.float32) + return np.random.random( + [16, int(48 / kwargs["groups"]), 3, 3, 3] + ).astype(np.float32) - conv3d_op = OpConfig(type="conv3d", - inputs={ - "Input": ["input_data"], - "Filter": ["conv_weight"] - }, - outputs={"Output": ["conv_output"]}, - attrs={ - "data_format": kwargs["data_format"], - "dilations": kwargs["dilations"], - "padding_algorithm": - kwargs["padding_algorithm"], - "groups": kwargs["groups"], - "paddings": kwargs["paddings"], - "strides": kwargs["strides"], - "is_test": True - }) + conv3d_op = OpConfig( + type="conv3d", + inputs={"Input": ["input_data"], "Filter": ["conv_weight"]}, + outputs={"Output": ["conv_output"]}, + attrs={ + "data_format": kwargs["data_format"], + "dilations": kwargs["dilations"], + "padding_algorithm": kwargs["padding_algorithm"], + "groups": kwargs["groups"], + "paddings": kwargs["paddings"], + "strides": kwargs["strides"], + "is_test": True, + }, + ) program_config = ProgramConfig( ops=[conv3d_op], weights={ - "conv_weight": - TensorConfig(data_gen=partial(generate_weight, *args, **kwargs)) + "conv_weight": TensorConfig( + data_gen=partial(generate_weight, *args, **kwargs) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)) + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ) }, - outputs=["conv_output"]) + outputs=["conv_output"], + ) yield program_config diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py index fb70f2260a342..bc4d527be87dc 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_activation_fuse_pass.py @@ -20,18 +20,19 @@ class ConvActivationMkldnnFusePassTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") - conv_out = fluid.layers.conv2d(data, - num_filters=self.conv_num_filters, - filter_size=self.conv_filter_size, - bias_attr=self.conv_bias_attr, - act=self.act) + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) + conv_out = fluid.layers.conv2d( + data, + num_filters=self.conv_num_filters, + filter_size=self.conv_filter_size, + bias_attr=self.conv_bias_attr, + act=self.act, + ) self.feeds = { "data": np.random.random((1, 3, 100, 100)).astype("float32") @@ -55,7 +56,6 @@ def test_pass_compatible(self): class ConvActivationMkldnnFusePassTest_1(ConvActivationMkldnnFusePassTest): - def set_params(self): self.conv_num_filters = 5 self.conv_filter_size = 5 @@ -64,7 +64,6 @@ def set_params(self): class ConvActivationMkldnnFusePassTest_2(ConvActivationMkldnnFusePassTest): - def set_params(self): self.conv_num_filters = 3 self.conv_filter_size = 3 @@ -73,7 +72,6 @@ def set_params(self): class ConvActivationMkldnnFusePassTest_3(ConvActivationMkldnnFusePassTest): - def set_params(self): self.conv_num_filters = 5 self.conv_filter_size = 5 @@ -82,7 +80,6 @@ def set_params(self): class ConvActivationMkldnnFusePassTest_4(ConvActivationMkldnnFusePassTest): - def set_params(self): self.conv_num_filters = 3 self.conv_filter_size = 3 @@ -91,7 +88,6 @@ def set_params(self): class ConvActivationMkldnnFusePassTest_5(ConvActivationMkldnnFusePassTest): - def set_params(self): self.conv_num_filters = 5 self.conv_filter_size = 5 @@ -100,7 +96,6 @@ def set_params(self): class ConvActivationMkldnnFusePassTest_6(ConvActivationMkldnnFusePassTest): - def set_params(self): self.conv_num_filters = 5 self.conv_filter_size = 5 @@ -109,7 +104,6 @@ def set_params(self): class ConvHardSigmoidOneDNNFusePassTest(ConvActivationMkldnnFusePassTest): - def set_params(self): self.conv_num_filters = 5 self.conv_filter_size = 5 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_affine_channel_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_affine_channel_fuse_pass.py index 94b9ac66a3d60..ba6179a1ff41f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_affine_channel_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_affine_channel_fuse_pass.py @@ -22,7 +22,6 @@ class TestConvAffineChannelFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -38,22 +37,27 @@ def sample_program_config(self, draw): out_channel = groups * out_channel_factor batch_size = draw(st.integers(min_value=1, max_value=4)) dilations = draw( - st.lists(st.integers(min_value=1, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=2), min_size=2, max_size=2 + ) + ) paddings = draw( - st.lists(st.integers(min_value=0, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=0, max_value=2), min_size=2, max_size=2 + ) + ) strides = draw( - st.lists(st.integers(min_value=1, max_value=2), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=2), min_size=2, max_size=2 + ) + ) has_bias = draw(st.booleans()) - x_shape = [ - batch_size, in_channel, 64, 64 - ] if data_format == "NCHW" else [batch_size, 64, 64, in_channel] + x_shape = ( + [batch_size, in_channel, 64, 64] + if data_format == "NCHW" + else [batch_size, 64, 64, in_channel] + ) w_shape = [out_channel, filter_channel, filter_size, filter_size] scale_shape = [out_channel] bias_shape = [out_channel] @@ -70,28 +74,32 @@ def generate_bias(): def generate_scale_bias(): return np.random.random(bias_shape).astype(np.float32) - conv2d_op = OpConfig("conv2d", - inputs={ - "Input": ["input_data"], - "Filter": ["conv2d_weight"], - }, - outputs={"Output": ["conv_output"]}, - data_format=data_format, - dilations=dilations, - padding_algorithm=padding_algorithm, - groups=groups, - paddings=paddings, - strides=strides, - has_bias=has_bias, - is_test=True) - ac_op = OpConfig("affine_channel", - inputs={ - "X": ["conv_output"], - "Scale": ["affine_channel_scale"], - "Bias": ["affine_channel_bias"] - }, - outputs={"Out": ["affine_channel_ouput"]}, - data_layout=data_format) + conv2d_op = OpConfig( + "conv2d", + inputs={ + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + outputs={"Output": ["conv_output"]}, + data_format=data_format, + dilations=dilations, + padding_algorithm=padding_algorithm, + groups=groups, + paddings=paddings, + strides=strides, + has_bias=has_bias, + is_test=True, + ) + ac_op = OpConfig( + "affine_channel", + inputs={ + "X": ["conv_output"], + "Scale": ["affine_channel_scale"], + "Bias": ["affine_channel_bias"], + }, + outputs={"Out": ["affine_channel_ouput"]}, + data_layout=data_format, + ) if has_bias == True: conv2d_op.inputs["Bias"] = ["conv2d_bias"] ops = [conv2d_op, ac_op] @@ -102,19 +110,23 @@ def generate_scale_bias(): "input_data": TensorConfig(data_gen=partial(generate_input)), }, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial(generate_weight)), - "conv2d_bias": - TensorConfig(data_gen=partial(generate_bias)), - "affine_channel_scale": - TensorConfig(data_gen=partial(generate_scale_bias)), - "affine_channel_bias": - TensorConfig(data_gen=partial(generate_scale_bias)), + "conv2d_weight": TensorConfig( + data_gen=partial(generate_weight) + ), + "conv2d_bias": TensorConfig(data_gen=partial(generate_bias)), + "affine_channel_scale": TensorConfig( + data_gen=partial(generate_scale_bias) + ), + "affine_channel_bias": TensorConfig( + data_gen=partial(generate_scale_bias) + ), }, - outputs=["affine_channel_ouput"]) + outputs=["affine_channel_ouput"], + ) if has_bias == True: program_config.weights["conv2d_bias"] = TensorConfig( - data_gen=partial(generate_bias)) + data_gen=partial(generate_bias) + ) return program_config def sample_predictor_configs(self, program_config): @@ -131,18 +143,23 @@ def teller1(program_config, predictor_config): # mkldnn Output has diff with bias! def teller2(program_config, predictor_config): - return predictor_config.mkldnn_enabled( - ) and program_config.ops[0].attrs['has_bias'] == True + return ( + predictor_config.mkldnn_enabled() + and program_config.ops[0].attrs['has_bias'] == True + ) self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, "The output format of conv2d is wrong when data_format attribute is NHWC, \ - because currently its fused op (Conv2DFusion) only supports data format of channel first (NCHW)." + because currently its fused op (Conv2DFusion) only supports data format of channel first (NCHW).", ) self.add_ignore_check_case( - teller2, IgnoreReasons.PASS_ACCURACY_ERROR, - "Currently mkldnn Output has diff with bias!") + teller2, + IgnoreReasons.PASS_ACCURACY_ERROR, + "Currently mkldnn Output has diff with bias!", + ) def test(self): self.run_and_statis( diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_bias_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_bias_fuse_pass.py index 7a1a04ca06f8c..6c667ac08db1c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_bias_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_bias_fuse_pass.py @@ -19,22 +19,24 @@ from paddle.fluid.core import PassVersionChecker -#padding SAME +# padding SAME class ConvBiasMkldnnFusePassSamePadTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) param_attr = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) - conv_out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - padding="SAME", - bias_attr=param_attr) + learning_rate=0.001, + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + padding="SAME", + bias_attr=param_attr, + ) self.feeds = { "data": np.random.random((1, 3, 100, 100)).astype("float32") @@ -46,25 +48,28 @@ def test_check_output(self): use_gpu = False self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible("conv_bias_mkldnn_fuse_pass")) + PassVersionChecker.IsCompatible("conv_bias_mkldnn_fuse_pass") + ) -#padding VALID +# padding VALID class ConvBiasMkldnnFusePassValidPadTest(ConvBiasMkldnnFusePassSamePadTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) param_attr = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) - conv_out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - padding="VALID", - bias_attr=param_attr) + learning_rate=0.001, + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + padding="VALID", + bias_attr=param_attr, + ) self.feeds = { "data": np.random.random((1, 3, 100, 100)).astype("float32") @@ -73,22 +78,24 @@ def setUp(self): self.enable_mkldnn = True -#padding EXPLICT NUMBER +# padding EXPLICT NUMBER class ConvBiasMkldnnFusePassExplictPadTest(ConvBiasMkldnnFusePassSamePadTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) param_attr = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) - conv_out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - padding=[2, 4, 6, 8], - bias_attr=param_attr) + learning_rate=0.001, + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + padding=[2, 4, 6, 8], + bias_attr=param_attr, + ) self.feeds = { "data": np.random.random((1, 3, 100, 100)).astype("float32") @@ -98,24 +105,26 @@ def setUp(self): class ConvBiasMkldnnFusePassGroupTest(ConvBiasMkldnnFusePassSamePadTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) param_attr = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) - conv_out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - padding="VALID", - groups=3, - bias_attr=param_attr, - use_cudnn=False, - act="softmax", - data_format="NCHW") + learning_rate=0.001, + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + padding="VALID", + groups=3, + bias_attr=param_attr, + use_cudnn=False, + act="softmax", + data_format="NCHW", + ) self.feeds = { "data": np.random.random((1, 3, 100, 100)).astype("float32") @@ -125,26 +134,29 @@ def setUp(self): class ConvBiasMkldnnFusePassDialtionsGroupsTest( - ConvBiasMkldnnFusePassSamePadTest): - + ConvBiasMkldnnFusePassSamePadTest +): def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) param_attr = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) - conv_out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - padding="VALID", - dilation=2, - groups=3, - bias_attr=param_attr, - use_cudnn=False, - act="softmax", - data_format="NCHW") + learning_rate=0.001, + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + padding="VALID", + dilation=2, + groups=3, + bias_attr=param_attr, + use_cudnn=False, + act="softmax", + data_format="NCHW", + ) self.feeds = { "data": np.random.random((1, 3, 100, 100)).astype("float32") @@ -154,20 +166,22 @@ def setUp(self): class ConvTransposeMkldnnFusePassDialtionsGroupsTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[-1, 3, 5, 5], dtype="float32") param_attr = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) - conv_out = fluid.layers.conv2d_transpose(input=data, - num_filters=3, - filter_size=3, - padding="SAME", - dilation=1, - bias_attr=param_attr, - use_cudnn=False) + learning_rate=0.001, + ) + conv_out = fluid.layers.conv2d_transpose( + input=data, + num_filters=3, + filter_size=3, + padding="SAME", + dilation=1, + bias_attr=param_attr, + use_cudnn=False, + ) self.feeds = {"data": np.random.random((1, 3, 5, 5)).astype("float32")} self.fetch_list = [conv_out] @@ -178,7 +192,9 @@ def test_check_output(self): self.check_output_with_option(use_gpu) self.assertTrue( PassVersionChecker.IsCompatible( - "conv_transpose_bias_mkldnn_fuse_pass")) + "conv_transpose_bias_mkldnn_fuse_pass" + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_concat_relu_mkldnn_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_concat_relu_mkldnn_fuse_pass.py index a5d2738869fe7..2d76d2846e9ae 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_concat_relu_mkldnn_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_concat_relu_mkldnn_fuse_pass.py @@ -21,7 +21,6 @@ class TestConvConcatActivationMkldnnFusePass(PassAutoScanTest): - def sample_program_config(self, draw): data_format = draw(st.sampled_from(['NCHW', 'NHWC'])) dilations = draw(st.sampled_from([[2, 2]])) @@ -31,11 +30,24 @@ def sample_program_config(self, draw): strides = draw(st.sampled_from([[1, 2]])) axis = draw(st.sampled_from([0])) activation_type = draw( - st.sampled_from([ - 'relu', 'gelu', 'swish', 'mish', 'sqrt', 'hard_swish', - 'sigmoid', 'abs', 'relu6', 'clip', 'tanh', 'hard_sigmoid', - 'leaky_relu' - ])) + st.sampled_from( + [ + 'relu', + 'gelu', + 'swish', + 'mish', + 'sqrt', + 'hard_swish', + 'sigmoid', + 'abs', + 'relu6', + 'clip', + 'tanh', + 'hard_sigmoid', + 'leaky_relu', + ] + ) + ) def generate_data(input_type): if input_type == 'NCHW': @@ -43,94 +55,103 @@ def generate_data(input_type): elif input_type == 'NHWC': return np.random.random([16, 64, 64, 48]).astype(np.float32) elif input_type == 'weights': - return np.random.random([16, int(48 / groups), 3, - 3]).astype(np.float32) - - conv2d_op1 = OpConfig(type='conv2d', - inputs={ - 'Input': ['conv_input_1'], - 'Filter': ['conv_weights_1'] - }, - outputs={'Output': ['conv_output_1']}, - attrs={ - 'data_format': data_format, - 'dilations': dilations, - 'padding_algorithm': padding_algorithm, - 'groups': groups, - 'paddings': paddings, - 'strides': strides - }) - - conv2d_op2 = OpConfig(type='conv2d', - inputs={ - 'Input': ['conv_input_2'], - 'Filter': ['conv_weights_2'] - }, - outputs={'Output': ['conv_output_2']}, - attrs={ - 'data_format': data_format, - 'dilations': dilations, - 'padding_algorithm': padding_algorithm, - 'groups': groups, - 'paddings': paddings, - 'strides': strides - }) + return np.random.random([16, int(48 / groups), 3, 3]).astype( + np.float32 + ) + + conv2d_op1 = OpConfig( + type='conv2d', + inputs={'Input': ['conv_input_1'], 'Filter': ['conv_weights_1']}, + outputs={'Output': ['conv_output_1']}, + attrs={ + 'data_format': data_format, + 'dilations': dilations, + 'padding_algorithm': padding_algorithm, + 'groups': groups, + 'paddings': paddings, + 'strides': strides, + }, + ) + + conv2d_op2 = OpConfig( + type='conv2d', + inputs={'Input': ['conv_input_2'], 'Filter': ['conv_weights_2']}, + outputs={'Output': ['conv_output_2']}, + attrs={ + 'data_format': data_format, + 'dilations': dilations, + 'padding_algorithm': padding_algorithm, + 'groups': groups, + 'paddings': paddings, + 'strides': strides, + }, + ) - concat_op = OpConfig(type='concat', - inputs={'X': ['conv_output_1', 'conv_output_2']}, - outputs={'Out': ['concat_output']}, - attrs={'axis': axis}) + concat_op = OpConfig( + type='concat', + inputs={'X': ['conv_output_1', 'conv_output_2']}, + outputs={'Out': ['concat_output']}, + attrs={'axis': axis}, + ) if activation_type == 'relu6': - activation_op = OpConfig(activation_type, - inputs={'X': ['concat_output']}, - outputs={'Out': ['activation_output']}, - threshold=draw( - st.floats(min_value=1.0, - max_value=10.0))) + activation_op = OpConfig( + activation_type, + inputs={'X': ['concat_output']}, + outputs={'Out': ['activation_output']}, + threshold=draw(st.floats(min_value=1.0, max_value=10.0)), + ) elif activation_type == 'leaky_relu': - activation_op = OpConfig(activation_type, - inputs={'X': ['concat_output']}, - outputs={'Out': ['activation_output']}, - alpha=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={'X': ['concat_output']}, + outputs={'Out': ['activation_output']}, + alpha=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == 'swish': - activation_op = OpConfig(activation_type, - inputs={'X': ['concat_output']}, - outputs={'Out': ['activation_output']}, - beta=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={'X': ['concat_output']}, + outputs={'Out': ['activation_output']}, + beta=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == 'clip': activation_op = OpConfig( activation_type, inputs={'X': ['concat_output']}, outputs={'Out': ['activation_output']}, min=draw(st.floats(min_value=0.1, max_value=0.49)), - max=draw(st.floats(min_value=0.5, max_value=1.0))) + max=draw(st.floats(min_value=0.5, max_value=1.0)), + ) else: - activation_op = OpConfig(activation_type, - inputs={'X': ['concat_output']}, - outputs={'Out': ['activation_output']}) + activation_op = OpConfig( + activation_type, + inputs={'X': ['concat_output']}, + outputs={'Out': ['activation_output']}, + ) model_net = [conv2d_op1, conv2d_op2, concat_op, activation_op] program_config = ProgramConfig( ops=model_net, inputs={ - 'conv_input_1': - TensorConfig(data_gen=partial(generate_data, data_format)), - 'conv_input_2': - TensorConfig(data_gen=partial(generate_data, data_format)) + 'conv_input_1': TensorConfig( + data_gen=partial(generate_data, data_format) + ), + 'conv_input_2': TensorConfig( + data_gen=partial(generate_data, data_format) + ), }, weights={ - 'conv_weights_1': - TensorConfig(data_gen=partial(generate_data, 'weights')), - 'conv_weights_2': - TensorConfig(data_gen=partial(generate_data, 'weights')) + 'conv_weights_1': TensorConfig( + data_gen=partial(generate_data, 'weights') + ), + 'conv_weights_2': TensorConfig( + data_gen=partial(generate_data, 'weights') + ), }, - outputs=['activation_output']) + outputs=['activation_output'], + ) return program_config @@ -139,8 +160,9 @@ def sample_predictor_configs(self, program_config): yield config, ['conv2d', 'conv2d', 'concat'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=['conv_activation_mkldnn_fuse_pass']) + self.run_and_statis( + quant=False, passes=['conv_activation_mkldnn_fuse_pass'] + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_elementwise_add_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_elementwise_add_fuse_pass.py index 3d0ca34a5e005..edf8599442f16 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_elementwise_add_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_elementwise_add_fuse_pass.py @@ -23,7 +23,6 @@ # the two inputs of elementwise_add are tensor class TestConvElementwiseAddMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -46,58 +45,60 @@ def sample_program_config(self, draw): def generate_input(): if data_format == "NCHW": - return np.random.random([batch_size, 48, 64, - 64]).astype(np.float32) + return np.random.random([batch_size, 48, 64, 64]).astype( + np.float32 + ) else: - return np.random.random([batch_size, 64, 64, - 48]).astype(np.float32) + return np.random.random([batch_size, 64, 64, 48]).astype( + np.float32 + ) def generate_weight(): - return np.random.random([48, int(48 / groups), 3, - 3]).astype(np.float32) - - relu_op = OpConfig(type="relu", - inputs={"X": ["input_data"]}, - outputs={"Out": ["relu_out"]}, - attrs={}) - - conv2d_op1 = OpConfig(type="conv2d", - inputs={ - "Input": ["relu_out"], - "Filter": ["conv_weight1"] - }, - outputs={"Output": ["conv_output1"]}, - attrs={ - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides - }) - - conv2d_op2 = OpConfig(type="conv2d", - inputs={ - "Input": ["input_data"], - "Filter": ["conv_weight2"] - }, - outputs={"Output": ["conv_output2"]}, - attrs={ - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides - }) - - elt_op = OpConfig(type="elementwise_add", - inputs={ - "X": ["conv_output1"], - "Y": ["conv_output2"] - }, - outputs={"Out": ["elementwise_output"]}, - attrs={'axis': axis}) + return np.random.random([48, int(48 / groups), 3, 3]).astype( + np.float32 + ) + + relu_op = OpConfig( + type="relu", + inputs={"X": ["input_data"]}, + outputs={"Out": ["relu_out"]}, + attrs={}, + ) + + conv2d_op1 = OpConfig( + type="conv2d", + inputs={"Input": ["relu_out"], "Filter": ["conv_weight1"]}, + outputs={"Output": ["conv_output1"]}, + attrs={ + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + }, + ) + + conv2d_op2 = OpConfig( + type="conv2d", + inputs={"Input": ["input_data"], "Filter": ["conv_weight2"]}, + outputs={"Output": ["conv_output2"]}, + attrs={ + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + }, + ) + + elt_op = OpConfig( + type="elementwise_add", + inputs={"X": ["conv_output1"], "Y": ["conv_output2"]}, + outputs={"Out": ["elementwise_output"]}, + attrs={'axis': axis}, + ) model_net = [relu_op, conv2d_op1, conv2d_op2, elt_op] @@ -105,12 +106,13 @@ def generate_weight(): ops=model_net, weights={ "conv_weight1": TensorConfig(data_gen=partial(generate_weight)), - "conv_weight2": TensorConfig(data_gen=partial(generate_weight)) + "conv_weight2": TensorConfig(data_gen=partial(generate_weight)), }, inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)) }, - outputs=["elementwise_output"]) + outputs=["elementwise_output"], + ) return program_config @@ -119,8 +121,9 @@ def sample_predictor_configs(self, program_config): yield config, ["relu", "conv2d", "conv2d"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["conv_elementwise_add_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["conv_elementwise_add_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py index 3abb582365baa..8094f323a9047 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_gelu_fuse_pass.py @@ -23,7 +23,6 @@ class TestConvGeluMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -39,45 +38,45 @@ def sample_program_config(self, draw): def generate_input(): if data_format == "NCHW": - return np.random.random([batch_size, 48, 64, - 64]).astype(np.float32) + return np.random.random([batch_size, 48, 64, 64]).astype( + np.float32 + ) else: - return np.random.random([batch_size, 64, 64, - 48]).astype(np.float32) + return np.random.random([batch_size, 64, 64, 48]).astype( + np.float32 + ) def generate_weight(): - return np.random.random([16, int(48 / groups), 3, - 3]).astype(np.float32) - - ops_config = [{ - "op_type": "conv2d", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["input_weight"] - }, - "op_outputs": { - "Output": ["conv_output"] - }, - "op_attrs": { - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides - } - }, { - "op_type": "gelu", - "op_inputs": { - "X": ["conv_output"] - }, - "op_outputs": { - "Out": ["gelu_output"] + return np.random.random([16, int(48 / groups), 3, 3]).astype( + np.float32 + ) + + ops_config = [ + { + "op_type": "conv2d", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["input_weight"], + }, + "op_outputs": {"Output": ["conv_output"]}, + "op_attrs": { + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + }, }, - "op_attrs": { - "approximate": approximate, + { + "op_type": "gelu", + "op_inputs": {"X": ["conv_output"]}, + "op_outputs": {"Out": ["gelu_output"]}, + "op_attrs": { + "approximate": approximate, + }, }, - }] + ] ops = self.generate_op_config(ops_config) @@ -89,7 +88,8 @@ def generate_weight(): inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)), }, - outputs=["gelu_output"]) + outputs=["gelu_output"], + ) return program_config @@ -98,8 +98,9 @@ def sample_predictor_configs(self, program_config): yield config, ["conv2d"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["conv_activation_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["conv_activation_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py index 010d3e8f3aa86..2b76ba180f5ca 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_sigmoid_fuse_pass.py @@ -22,7 +22,6 @@ class TestConvHardSigmoidMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -39,46 +38,43 @@ def sample_program_config(self, draw): def generate_input(): if data_format == "NCHW": - return np.random.random([batch_size, 48, 64, - 64]).astype(np.float32) + return np.random.random([batch_size, 48, 64, 64]).astype( + np.float32 + ) else: - return np.random.random([batch_size, 64, 64, - 48]).astype(np.float32) + return np.random.random([batch_size, 64, 64, 48]).astype( + np.float32 + ) def generate_weight(): - return np.random.random([16, int(48 / groups), 3, - 3]).astype(np.float32) - - ops_config = [{ - "op_type": "conv2d", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["input_weight"] - }, - "op_outputs": { - "Output": ["conv_output"] - }, - "op_attrs": { - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides - } - }, { - "op_type": "hard_sigmoid", - "op_inputs": { - "X": ["conv_output"] - }, - "op_outputs": { - "Out": ["sigmoid_output"] + return np.random.random([16, int(48 / groups), 3, 3]).astype( + np.float32 + ) + + ops_config = [ + { + "op_type": "conv2d", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["input_weight"], + }, + "op_outputs": {"Output": ["conv_output"]}, + "op_attrs": { + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + }, }, - "op_attrs": { - "slope": slope, - "offset": offset + { + "op_type": "hard_sigmoid", + "op_inputs": {"X": ["conv_output"]}, + "op_outputs": {"Out": ["sigmoid_output"]}, + "op_attrs": {"slope": slope, "offset": offset}, }, - }] + ] ops = self.generate_op_config(ops_config) @@ -90,7 +86,8 @@ def generate_weight(): inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)), }, - outputs=["sigmoid_output"]) + outputs=["sigmoid_output"], + ) return program_config @@ -99,8 +96,9 @@ def sample_predictor_configs(self, program_config): yield config, ["conv2d"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["conv_activation_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["conv_activation_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py index 799d1cbaa1c9f..417fb95c72a1d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_hard_swish_fuse_pass.py @@ -22,7 +22,6 @@ class TestConvHardSwishMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -40,47 +39,47 @@ def sample_program_config(self, draw): def generate_input(): if data_format == "NCHW": - return np.random.random([batch_size, 48, 64, - 64]).astype(np.float32) + return np.random.random([batch_size, 48, 64, 64]).astype( + np.float32 + ) else: - return np.random.random([batch_size, 64, 64, - 48]).astype(np.float32) + return np.random.random([batch_size, 64, 64, 48]).astype( + np.float32 + ) def generate_weight(): - return np.random.random([16, int(48 / groups), 3, - 3]).astype(np.float32) - - ops_config = [{ - "op_type": "conv2d", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["input_weight"] - }, - "op_outputs": { - "Output": ["conv_output"] - }, - "op_attrs": { - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides - } - }, { - "op_type": "hard_swish", - "op_inputs": { - "X": ["conv_output"] - }, - "op_outputs": { - "Out": ["swish_output"] + return np.random.random([16, int(48 / groups), 3, 3]).astype( + np.float32 + ) + + ops_config = [ + { + "op_type": "conv2d", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["input_weight"], + }, + "op_outputs": {"Output": ["conv_output"]}, + "op_attrs": { + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + }, }, - "op_attrs": { - "threshold": threshold, - "scale": scale, - "offset": offset + { + "op_type": "hard_swish", + "op_inputs": {"X": ["conv_output"]}, + "op_outputs": {"Out": ["swish_output"]}, + "op_attrs": { + "threshold": threshold, + "scale": scale, + "offset": offset, + }, }, - }] + ] ops = self.generate_op_config(ops_config) @@ -92,7 +91,8 @@ def generate_weight(): inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)), }, - outputs=["swish_output"]) + outputs=["swish_output"], + ) return program_config @@ -101,8 +101,9 @@ def sample_predictor_configs(self, program_config): yield config, ["conv2d"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["conv_activation_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["conv_activation_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py index 6f5810cb802d9..93f58741ba8c6 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_mish_fuse_pass.py @@ -21,7 +21,6 @@ class TestConvMishMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [op.attrs for op in program_config.ops] # If the problem has been fixed, the judgment @@ -42,43 +41,43 @@ def sample_program_config(self, draw): def generate_input(): if data_format == "NCHW": - return np.random.random([batch_size, 48, 64, - 64]).astype(np.float32) + return np.random.random([batch_size, 48, 64, 64]).astype( + np.float32 + ) else: - return np.random.random([batch_size, 64, 64, - 48]).astype(np.float32) + return np.random.random([batch_size, 64, 64, 48]).astype( + np.float32 + ) def generate_weight(): - return np.random.random([16, int(48 / groups), 3, - 3]).astype(np.float32) - - ops_config = [{ - "op_type": "conv2d", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["input_weight"] - }, - "op_outputs": { - "Output": ["conv_output"] - }, - "op_attrs": { - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides - } - }, { - "op_type": "mish", - "op_inputs": { - "X": ["conv_output"] + return np.random.random([16, int(48 / groups), 3, 3]).astype( + np.float32 + ) + + ops_config = [ + { + "op_type": "conv2d", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["input_weight"], + }, + "op_outputs": {"Output": ["conv_output"]}, + "op_attrs": { + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + }, }, - "op_outputs": { - "Out": ["mish_output"] + { + "op_type": "mish", + "op_inputs": {"X": ["conv_output"]}, + "op_outputs": {"Out": ["mish_output"]}, + "op_attrs": {}, }, - "op_attrs": {}, - }] + ] ops = self.generate_op_config(ops_config) @@ -90,7 +89,8 @@ def generate_weight(): inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)), }, - outputs=["mish_output"]) + outputs=["mish_output"], + ) return program_config @@ -99,8 +99,9 @@ def sample_predictor_configs(self, program_config): yield config, ["conv2d"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["conv_activation_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["conv_activation_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_transpose_bias_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_transpose_bias_fuse_pass.py index 2eebd529554d3..01de124314895 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_transpose_bias_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_conv_transpose_bias_fuse_pass.py @@ -22,7 +22,6 @@ class TestConvTransposeMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -47,11 +46,13 @@ def sample_program_config(self, draw): def generate_input(): if data_format == "NCHW": - return np.random.random([batch_size, 16, 64, - 64]).astype(np.float32) + return np.random.random([batch_size, 16, 64, 64]).astype( + np.float32 + ) else: - return np.random.random([batch_size, 64, 64, - 16]).astype(np.float32) + return np.random.random([batch_size, 64, 64, 16]).astype( + np.float32 + ) def generate_weight1(): return np.random.random([16, 16, 3, 3]).astype(np.float32) @@ -59,46 +60,47 @@ def generate_weight1(): def generate_weight2(): return np.random.random([16 * groups]).astype(np.float32) - conv2d_op = OpConfig(type="conv2d_transpose", - inputs={ - "Input": ["input_data"], - "Filter": ["conv2d_weight"] - }, - outputs={"Output": ["conv_output"]}, - attrs={ - "data_format": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides, - "output_size": [], - "output_padding": [], - "is_test": True - }) - - elt_op = OpConfig(type="elementwise_add", - inputs={ - "X": ["conv_output"], - "Y": ["elementwise_weight"] - }, - outputs={"Out": ["elementwise_output"]}, - attrs={'axis': axis}) + conv2d_op = OpConfig( + type="conv2d_transpose", + inputs={"Input": ["input_data"], "Filter": ["conv2d_weight"]}, + outputs={"Output": ["conv_output"]}, + attrs={ + "data_format": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + "output_size": [], + "output_padding": [], + "is_test": True, + }, + ) + + elt_op = OpConfig( + type="elementwise_add", + inputs={"X": ["conv_output"], "Y": ["elementwise_weight"]}, + outputs={"Out": ["elementwise_output"]}, + attrs={'axis': axis}, + ) model_net = [conv2d_op, elt_op] program_config = ProgramConfig( ops=model_net, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial(generate_weight1)), - "elementwise_weight": - TensorConfig(data_gen=partial(generate_weight2)) + "conv2d_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "elementwise_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), }, inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)) }, - outputs=["elementwise_output"]) + outputs=["elementwise_output"], + ) return program_config @@ -107,9 +109,11 @@ def sample_predictor_configs(self, program_config): yield config, ['conv2d_transpose'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_duration=300, - passes=["conv_transpose_bias_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, + max_duration=300, + passes=["conv_transpose_bias_mkldnn_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_cpu_bfloat16_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_cpu_bfloat16_pass.py index 95996f22a86d7..2cd9cbcb05700 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_cpu_bfloat16_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_cpu_bfloat16_pass.py @@ -20,20 +20,20 @@ class TestMKLDNNCpuBfloat16Pass(InferencePassTest): - def setUp(self): self.init_data() with fluid.program_guard(self.main_program, self.startup_program): - x = fluid.data(name='x', - shape=[-1] + self.shape_x, - dtype=self.d_type) + x = fluid.data( + name='x', shape=[-1] + self.shape_x, dtype=self.d_type + ) out = fluid.layers.transpose(x, perm=[0, 1, 2, 3]) out = fluid.layers.reshape(out, [0, 0, 0, 0]) out = fluid.layers.fc(out, size=1) self.feeds = { - "x": - np.random.random([self.bs] + self.shape_x).astype(self.d_type) + "x": np.random.random([self.bs] + self.shape_x).astype( + self.d_type + ) } self.fetch_list = [out] diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_depthwise_conv_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_depthwise_conv_pass.py index 736c5fca7f8f0..3a9f520f1c766 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_depthwise_conv_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_depthwise_conv_pass.py @@ -42,24 +42,29 @@ def sample_program_config(self, draw): random_groups = draw(st.integers(min_value=1, max_value=3)) random_dilations = draw( - st.lists(st.integers(min_value=1, max_value=3), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=3), min_size=2, max_size=2 + ) + ) random_strides = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) random_paddings = draw( - st.lists(st.integers(min_value=0, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=0, max_value=4), min_size=2, max_size=2 + ) + ) random_padding_algorithm = draw( - st.sampled_from(["EXPLICIT", "SAME", "VALID"])) + st.sampled_from(["EXPLICIT", "SAME", "VALID"]) + ) random_data_layout = draw(st.sampled_from(["NCHW", "NHWC"])) random_filter = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) def generate_conv2d_Input(): shape = [random_input_dim1, random_input_dim2] @@ -78,23 +83,25 @@ def generate_conv2d_Filter(): return np.random.random(shape).astype(np.float32) # define op - conv2d_op = OpConfig(type="depthwise_conv2d", - inputs={ - "Input": ["conv2d_Input"], - "Filter": ["conv2d_Filter"], - }, - outputs={ - "Output": ["conv2d_Out"], - }, - attrs={ - 'groups': random_groups, - 'dilations': random_dilations, - 'strides': random_strides, - 'paddings': random_paddings, - 'padding_algorithm': random_padding_algorithm, - 'data_format': random_data_layout, - 'use_mkldnn': True, - }) + conv2d_op = OpConfig( + type="depthwise_conv2d", + inputs={ + "Input": ["conv2d_Input"], + "Filter": ["conv2d_Filter"], + }, + outputs={ + "Output": ["conv2d_Out"], + }, + attrs={ + 'groups': random_groups, + 'dilations': random_dilations, + 'strides': random_strides, + 'paddings': random_paddings, + 'padding_algorithm': random_padding_algorithm, + 'data_format': random_data_layout, + 'use_mkldnn': True, + }, + ) # define model_net model_net = [conv2d_op] @@ -108,7 +115,8 @@ def generate_conv2d_Filter(): weights={ "conv2d_Filter": TensorConfig(data_gen=generate_conv2d_Filter), }, - outputs=["conv2d_Out"]) + outputs=["conv2d_Out"], + ) return program_config @@ -128,13 +136,13 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def add_ignore_pass_case(self): - def teller1(program_config, predictor_config): if program_config.ops[0].attrs['data_format'] == "NHWC": return True return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The output format of depthwise_conv2d is wrong when data_format attribute is NHWC" + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The output format of depthwise_conv2d is wrong when data_format attribute is NHWC", ) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py index 0063fea59a7a1..c9647ec60b5cb 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass.py @@ -28,12 +28,12 @@ class ElementwiseActivationMkldnnFusePassTest(InferencePassTest): def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data_A = fluid.data(name="data_A", - shape=[-1, 3, 100, 100], - dtype="float32") - data_B = fluid.data(name="data_B", - shape=[-1, 3, 100, 100], - dtype="float32") + data_A = fluid.data( + name="data_A", shape=[-1, 3, 100, 100], dtype="float32" + ) + data_B = fluid.data( + name="data_B", shape=[-1, 3, 100, 100], dtype="float32" + ) elt_out = self.operand(data_A, data_B) if self.act is not None: if self.act_beta is not None: @@ -45,7 +45,7 @@ def setUp(self): self.feeds = { "data_A": np.random.random((1, 3, 100, 100)).astype("float32"), - "data_B": np.random.random((1, 3, 100, 100)).astype("float32") + "data_B": np.random.random((1, 3, 100, 100)).astype("float32"), } self.fetch_list = [elt_out] self.enable_mkldnn = True @@ -63,24 +63,24 @@ def test_pass_compatible(self): class ElementwiseActivationMkldnnFusePassTest_Add_Relu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.relu class ElementwiseActivationMkldnnFusePassTest_Add_Tanh( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.tanh class ElementwiseActivationMkldnnFusePassTest_Add_LeakyRelu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act_alpha = 0.2 @@ -88,8 +88,8 @@ def set_params(self): class ElementwiseActivationMkldnnFusePassTest_Add_Swish( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act_alpha = 4 @@ -97,32 +97,32 @@ def set_params(self): class ElementwiseActivationMkldnnFusePassTest_Add_HardSwish( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.hard_swish class ElementwiseActivationMkldnnFusePassTest_Add_SQRT( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.sqrt class ElementwiseActivationMkldnnFusePassTest_Add_ABS( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.abs class ElementwiseActivationMkldnnFusePassTest_Add_Clip( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.clip @@ -131,16 +131,16 @@ def set_params(self): class ElementwiseActivationMkldnnFusePassTest_Add_Gelu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.gelu class ElementwiseActivationMkldnnFusePassTest_Add_Gelu_Tanh( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.gelu @@ -148,8 +148,8 @@ def set_params(self): class ElementwiseActivationMkldnnFusePassTest_Add_Relu6( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.relu6 @@ -157,32 +157,32 @@ def set_params(self): class ElementwiseActivationMkldnnFusePassTest_Add_Sigmoid( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_add self.act = fluid.layers.sigmoid class ElementwiseActivationMkldnnFusePassTest_Sub_Relu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.relu class ElementwiseActivationMkldnnFusePassTest_Sub_Tanh( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.tanh class ElementwiseActivationMkldnnFusePassTest_Sub_LeakyRelu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act_alpha = 0.2 @@ -190,32 +190,32 @@ def set_params(self): class ElementwiseActivationMkldnnFusePassTest_Sub_Swish( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.swish class ElementwiseActivationMkldnnFusePassTest_Sub_HardSwish( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.hard_swish class ElementwiseActivationMkldnnFusePassTest_Sub_ABS( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.abs class ElementwiseActivationMkldnnFusePassTest_Sub_Clip( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.clip @@ -224,16 +224,16 @@ def set_params(self): class ElementwiseActivationMkldnnFusePassTest_Sub_Gelu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.gelu class ElementwiseActivationMkldnnFusePassTest_Sub_Gelu_Tanh( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.gelu @@ -241,8 +241,8 @@ def set_params(self): class ElementwiseActivationMkldnnFusePassTest_Sub_Relu6( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.relu6 @@ -250,32 +250,32 @@ def set_params(self): class ElementwiseActivationMkldnnFusePassTest_Sub_Sigmoid( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_sub self.act = fluid.layers.sigmoid class ElementwiseActivationMkldnnFusePassTest_Mul_Relu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.relu class ElementwiseActivationMkldnnFusePassTest_Mul_Tanh( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.tanh class ElementwiseActivationMkldnnFusePassTest_Mul_LeakyRelu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act_alpha = 0.2 @@ -283,40 +283,40 @@ def set_params(self): class ElementwiseActivationMkldnnFusePassTest_Mul_Swish( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.swish class ElementwiseActivationMkldnnFusePassTest_Mul_HardSwish( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.hard_swish class ElementwiseActivationMkldnnFusePassTest_Mul_SQRT( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.sqrt class ElementwiseActivationMkldnnFusePassTest_Mul_ABS( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.abs class ElementwiseActivationMkldnnFusePassTest_Mul_Clip( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.clip @@ -325,16 +325,16 @@ def set_params(self): class ElementwiseActivationMkldnnFusePassTest_Mul_Gelu( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.gelu class ElementwiseActivationMkldnnFusePassTest_Mul_Gelu_Tanh( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.gelu @@ -342,8 +342,8 @@ def set_params(self): class ElementwiseActivationMkldnnFusePassTest_Mul_Relu6( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.relu6 @@ -351,8 +351,8 @@ def set_params(self): class ElementwiseActivationMkldnnFusePassTest_Mul_Sigmoid( - ElementwiseActivationMkldnnFusePassTest): - + ElementwiseActivationMkldnnFusePassTest +): def set_params(self): self.operand = fluid.layers.elementwise_mul self.act = fluid.layers.sigmoid diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py index feef212b1f697..8144e02afa00f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_elt_act_fuse_pass_new.py @@ -22,7 +22,6 @@ class TestElementWiseAddReluFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -30,29 +29,24 @@ def sample_program_config(self, draw): batch_size = draw(st.integers(min_value=1, max_value=4)) def generate_input(): - return np.random.random([batch_size, 3, 100, - 100]).astype(np.float32) + return np.random.random([batch_size, 3, 100, 100]).astype( + np.float32 + ) - ops_config = [{ - "op_type": "elementwise_add", - "op_inputs": { - "X": ["A"], - "Y": ["B"] - }, - "op_outputs": { - "Out": ["add_output"] - }, - "op_attrs": {} - }, { - "op_type": "relu", - "op_inputs": { - "X": ["add_output"] + ops_config = [ + { + "op_type": "elementwise_add", + "op_inputs": {"X": ["A"], "Y": ["B"]}, + "op_outputs": {"Out": ["add_output"]}, + "op_attrs": {}, }, - "op_outputs": { - "Out": ["relu_output"] + { + "op_type": "relu", + "op_inputs": {"X": ["add_output"]}, + "op_outputs": {"Out": ["relu_output"]}, + "op_attrs": {}, }, - "op_attrs": {} - }] + ] ops = self.generate_op_config(ops_config) @@ -61,9 +55,10 @@ def generate_input(): weights={}, inputs={ "A": TensorConfig(data_gen=partial(generate_input)), - "B": TensorConfig(data_gen=partial(generate_input)) + "B": TensorConfig(data_gen=partial(generate_input)), }, - outputs=["relu_output"]) + outputs=["relu_output"], + ) return program_config @@ -72,9 +67,9 @@ def sample_predictor_configs(self, program_config): yield config, ["elementwise_add"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["elt_act_mkldnn_fuse_pass"], - min_success_num=4) + self.run_and_statis( + quant=False, passes=["elt_act_mkldnn_fuse_pass"], min_success_num=4 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_act_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_act_fuse_pass.py index c438c28370e19..4848a8c9770ff 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_act_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_act_fuse_pass.py @@ -25,13 +25,12 @@ class FCGeluTanhOneDnnFusePassTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 128, 768], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 128, 768], dtype="float32" + ) fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2) gelu_out = fluid.layers.gelu(fc_out, approximate=False) @@ -48,13 +47,12 @@ def test_check_output(self): class FCGeluErfOneDnnFusePassTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 128, 768], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 128, 768], dtype="float32" + ) fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2) gelu_out = fluid.layers.gelu(fc_out, approximate=True) @@ -72,13 +70,12 @@ def test_check_output(self): class FCTanhOneDnnFusePassTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 128, 768], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 128, 768], dtype="float32" + ) fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2) tanh_out = fluid.layers.tanh(fc_out) @@ -96,13 +93,12 @@ def test_check_output(self): class FCSigmoidOneDnnFusePassTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 128, 768], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 128, 768], dtype="float32" + ) fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2) sigmoid_out = fluid.layers.sigmoid(fc_out) @@ -120,13 +116,12 @@ def test_check_output(self): class FCHardSwishOneDnnFusePassTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 128, 768], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 128, 768], dtype="float32" + ) fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2) hardswish_out = fluid.layers.hard_swish(fc_out) @@ -144,13 +139,12 @@ def test_check_output(self): class FCMishOneDnnFusePassTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 128, 768], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 128, 768], dtype="float32" + ) fc_out = fluid.layers.fc(input=data, size=3072, num_flatten_dims=2) mish_out = fluid.layers.mish(fc_out) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_elementwise_add_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_elementwise_add_fuse_pass.py index d69a7b8a8bc47..85f877147fffc 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_elementwise_add_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_fc_elementwise_add_fuse_pass.py @@ -22,7 +22,6 @@ class TestFCElementwiseAddMkldnnFusePass(PassAutoScanTest): - def sample_program_config(self, draw): axis = draw(st.sampled_from([-1, 0, 1])) fc_as_x = draw(st.sampled_from([True, False])) @@ -38,34 +37,40 @@ def generate_fc_weight(): def generate_fc_bias(): return np.random.random([fc_wei]).astype(np.float32) - relu_op = OpConfig(type="relu", - inputs={"X": ["input_data"]}, - outputs={"Out": ["relu_out"]}, - attrs={}) - - fc_op = OpConfig(type="fc", - inputs={ - "Input": ["relu_out"], - "W": ["fc_weight"], - "Bias": ["fc_bias"] - }, - outputs={"Out": ["fc_output"]}, - attrs={ - "use_mkldnn": True, - "padding_weights": False, - "activation_type": "", - "in_num_col_dims": 1, - }) + relu_op = OpConfig( + type="relu", + inputs={"X": ["input_data"]}, + outputs={"Out": ["relu_out"]}, + attrs={}, + ) + + fc_op = OpConfig( + type="fc", + inputs={ + "Input": ["relu_out"], + "W": ["fc_weight"], + "Bias": ["fc_bias"], + }, + outputs={"Out": ["fc_output"]}, + attrs={ + "use_mkldnn": True, + "padding_weights": False, + "activation_type": "", + "in_num_col_dims": 1, + }, + ) if fc_as_x: inputs = {"X": ["fc_output"], "Y": ["input_data"]} else: inputs = {"X": ["input_data"], "Y": ["fc_output"]} - elt_add_op = OpConfig(type="elementwise_add", - inputs=inputs, - outputs={"Out": ["elementwise_output"]}, - attrs={'axis': axis}) + elt_add_op = OpConfig( + type="elementwise_add", + inputs=inputs, + outputs={"Out": ["elementwise_output"]}, + attrs={'axis': axis}, + ) model_net = [relu_op, fc_op, elt_add_op] @@ -78,7 +83,8 @@ def generate_fc_bias(): inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)) }, - outputs=["elementwise_output"]) + outputs=["elementwise_output"], + ) return program_config @@ -87,8 +93,9 @@ def sample_predictor_configs(self, program_config): yield config, ["relu", "fc"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["fc_elementwise_add_mkldnn_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["fc_elementwise_add_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_inplace_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_inplace_fuse_pass.py index 6b589821ba8a4..6dc911390b058 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_inplace_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_inplace_fuse_pass.py @@ -21,22 +21,20 @@ class MkldnnInplacePassTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): paddle.enable_static() - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") - conv_out_1 = fluid.layers.conv2d(data, - num_filters=3, - filter_size=3, - bias_attr=False) + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) + conv_out_1 = fluid.layers.conv2d( + data, num_filters=3, filter_size=3, bias_attr=False + ) softmax_out = fluid.layers.softmax(conv_out_1) relu_out = fluid.layers.relu(conv_out_1) - eltwise_out = fluid.layers.elementwise_add(softmax_out, - relu_out, - axis=-1) + eltwise_out = fluid.layers.elementwise_add( + softmax_out, relu_out, axis=-1 + ) self.pass_name = 'mkldnn_inplace_pass' self.feeds = { diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_int8_scale_calculation_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_int8_scale_calculation_pass.py index 3d2895cc619d4..7308f93d792a9 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_int8_scale_calculation_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_int8_scale_calculation_pass.py @@ -20,7 +20,6 @@ class TestInt8ScaleCalculationMkldnnPass(PassAutoScanTest): - def sample_predictor_configs(self, program_config): config = self.create_inference_config(use_gpu=False) config.pass_builder().append_pass("int8_scale_calculation_mkldnn_pass") @@ -36,12 +35,36 @@ def is_program_valid(self, prog_config): filter_shape = prog_config.weights["filter"].shape input_shape = prog_config.inputs["input_x"].shape if padding_algorithm == "VALID": - if ((input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + (input_shape[2] - (dilations[0] * (filter_shape[2] - 1) + 1)) + / strides[0] + + 1 + ) <= 1 or ( + (input_shape[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) + / strides[1] + + 1 + ) <= 1: return False if padding_algorithm == "EXPLICIT": - if ((input_shape[2] + paddings[0] + paddings[1] - (dilations[0] * (filter_shape[2] - 1) + 1)) / strides[0] + 1) <= 1 or \ - ((input_shape[3] + paddings[2] + paddings[3] - (dilations[1] * (filter_shape[3] - 1) + 1)) / strides[1] + 1) <= 1: + if ( + ( + input_shape[2] + + paddings[0] + + paddings[1] + - (dilations[0] * (filter_shape[2] - 1) + 1) + ) + / strides[0] + + 1 + ) <= 1 or ( + ( + input_shape[3] + + paddings[2] + + paddings[3] + - (dilations[1] * (filter_shape[3] - 1) + 1) + ) + / strides[1] + + 1 + ) <= 1: return False if data_format == "NCHW": if input_shape[1] != filter_shape[1] * groups: @@ -57,40 +80,45 @@ def is_program_valid(self, prog_config): def sample_program_config(self, draw): x_shape = draw( - st.lists(st.integers(min_value=5, max_value=100), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=5, max_value=100), min_size=4, max_size=4 + ) + ) x_shape[1] = draw(st.integers(min_value=5, max_value=10)) data_format = draw(st.sampled_from(["NCHW", "NHWC"])) f_shape = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=4, max_size=4 + ) + ) if data_format == "NCHW": f_shape[1] = x_shape[1] else: f_shape[1] = x_shape[3] strides = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) padding_algorithm = draw(st.sampled_from(["EXPLICIT", "SAME", "VALID"])) padding = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=4, max_size=4 + ) + ) groups = draw(st.integers(min_value=1, max_value=3)) dilations = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=2 + ) + ) bias_shape = [f_shape[0]] inputs = dict() @@ -116,17 +144,19 @@ def sample_program_config(self, draw): "filter": TensorConfig(shape=f_shape), } - conv2d_op = OpConfig("conv2d", - inputs=inputs, - outputs={"Output": ["conv2d_out"]}, - strides=strides, - padding_algorithm=padding_algorithm, - paddings=padding, - groups=groups, - dilations=dilations, - data_format=data_format, - use_mkldnn=use_mkldnn, - mkldnn_data_type="int8") + conv2d_op = OpConfig( + "conv2d", + inputs=inputs, + outputs={"Output": ["conv2d_out"]}, + strides=strides, + padding_algorithm=padding_algorithm, + paddings=padding, + groups=groups, + dilations=dilations, + data_format=data_format, + use_mkldnn=use_mkldnn, + mkldnn_data_type="int8", + ) ops = [conv2d_op] @@ -134,13 +164,16 @@ def sample_program_config(self, draw): ops=ops, weights=weights, inputs={"input_x": TensorConfig(shape=x_shape)}, - outputs=["conv2d_out"]) + outputs=["conv2d_out"], + ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=100, - passes=["int8_scale_calculation_mkldnn_pass"]) + self.run_and_statis( + quant=False, + max_examples=100, + passes=["int8_scale_calculation_mkldnn_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_log_softmax_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_log_softmax_op.py index 929863b42a730..814163d4ec3dc 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_log_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_log_softmax_op.py @@ -22,28 +22,30 @@ class TestMKLDNNLogSoftmaxOp(MkldnnAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): return np.random.random(kwargs['in_shape']).astype(np.float32) - logsoftmax_op = OpConfig(type="log_softmax", - inputs={"X": ["input_data"]}, - outputs={"Out": ["output_data"]}, - attrs={"axis": kwargs['axis']}) + logsoftmax_op = OpConfig( + type="log_softmax", + inputs={"X": ["input_data"]}, + outputs={"Out": ["output_data"]}, + attrs={"axis": kwargs['axis']}, + ) program_config = ProgramConfig( ops=[logsoftmax_op], weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config @@ -51,10 +53,12 @@ def sample_predictor_configs(self, program_config): config = self.create_inference_config(use_mkldnn=True) yield config, (1e-5, 1e-5) - @given(axis=st.sampled_from([-2, -1, 0, 1]), - in_shape=st.lists(st.integers(min_value=2, max_value=5), - min_size=3, - max_size=5)) + @given( + axis=st.sampled_from([-2, -1, 0, 1]), + in_shape=st.lists( + st.integers(min_value=2, max_value=5), min_size=3, max_size=5 + ), + ) def test(self, *args, **kwargs): self.run_test(quant=False, *args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py index b894fc708b424..a90129394e1bb 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_activation_fuse_pass.py @@ -21,7 +21,6 @@ class TestMatmulActivationMkldnnFusePass(PassAutoScanTest): - def sample_program_config(self, draw): transpose_X = draw(st.booleans()) transpose_Y = draw(st.booleans()) @@ -30,11 +29,24 @@ def sample_program_config(self, draw): channel = draw(st.sampled_from([8])) input_dim = draw(st.sampled_from([32])) activation_type = draw( - st.sampled_from([ - 'relu', 'gelu', 'swish', 'mish', 'sqrt', 'hard_swish', - 'sigmoid', 'abs', 'relu6', 'clip', 'tanh', 'hard_sigmoid', - 'leaky_relu' - ])) + st.sampled_from( + [ + 'relu', + 'gelu', + 'swish', + 'mish', + 'sqrt', + 'hard_swish', + 'sigmoid', + 'abs', + 'relu6', + 'clip', + 'tanh', + 'hard_sigmoid', + 'leaky_relu', + ] + ) + ) def generate_input(type): if transpose_X and transpose_Y: @@ -55,50 +67,52 @@ def generate_input(type): else: return np.random.random(shape_y).astype(np.float32) - matmul_op = OpConfig(type='matmul', - inputs={ - 'X': ['matmul_X'], - 'Y': ['matmul_Y'] - }, - outputs={'Out': ['matmul_output']}, - attrs={ - 'transpose_X': transpose_X, - 'transpose_Y': transpose_Y, - 'alpha': alpha - }) + matmul_op = OpConfig( + type='matmul', + inputs={'X': ['matmul_X'], 'Y': ['matmul_Y']}, + outputs={'Out': ['matmul_output']}, + attrs={ + 'transpose_X': transpose_X, + 'transpose_Y': transpose_Y, + 'alpha': alpha, + }, + ) if activation_type == "relu6": - activation_op = OpConfig(activation_type, - inputs={"X": ["matmul_output"]}, - outputs={"Out": ["activation_output"]}, - threshold=draw( - st.floats(min_value=1.0, - max_value=10.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["matmul_output"]}, + outputs={"Out": ["activation_output"]}, + threshold=draw(st.floats(min_value=1.0, max_value=10.0)), + ) elif activation_type == "leaky_relu": - activation_op = OpConfig(activation_type, - inputs={"X": ["matmul_output"]}, - outputs={"Out": ["activation_output"]}, - alpha=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["matmul_output"]}, + outputs={"Out": ["activation_output"]}, + alpha=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == "swish": - activation_op = OpConfig(activation_type, - inputs={"X": ["matmul_output"]}, - outputs={"Out": ["activation_output"]}, - beta=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["matmul_output"]}, + outputs={"Out": ["activation_output"]}, + beta=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == "clip": activation_op = OpConfig( activation_type, inputs={"X": ["matmul_output"]}, outputs={"Out": ["activation_output"]}, min=draw(st.floats(min_value=0.1, max_value=0.49)), - max=draw(st.floats(min_value=0.5, max_value=1.0))) + max=draw(st.floats(min_value=0.5, max_value=1.0)), + ) else: - activation_op = OpConfig(activation_type, - inputs={"X": ["matmul_output"]}, - outputs={"Out": ["activation_output"]}) + activation_op = OpConfig( + activation_type, + inputs={"X": ["matmul_output"]}, + outputs={"Out": ["activation_output"]}, + ) model_net = [matmul_op, activation_op] @@ -107,9 +121,10 @@ def generate_input(type): weights={}, inputs={ 'matmul_X': TensorConfig(data_gen=partial(generate_input, 'x')), - 'matmul_Y': TensorConfig(data_gen=partial(generate_input, 'y')) + 'matmul_Y': TensorConfig(data_gen=partial(generate_input, 'y')), }, - outputs=['activation_output']) + outputs=['activation_output'], + ) return program_config @@ -118,9 +133,11 @@ def sample_predictor_configs(self, program_config): yield config, ['matmul'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_examples=30, - passes=['matmul_activation_mkldnn_fuse_pass']) + self.run_and_statis( + quant=False, + max_examples=30, + passes=['matmul_activation_mkldnn_fuse_pass'], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_activation_fuse_pass.py index a27ed9dd9c99a..035cd1d38004c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_activation_fuse_pass.py @@ -22,7 +22,6 @@ class TestMatmulElementwiseAddActivationMkldnnFusePass(PassAutoScanTest): - def sample_program_config(self, draw): axis = draw(st.sampled_from([-1, 0, 1])) matmul_as_x = draw(st.booleans()) @@ -30,71 +29,88 @@ def sample_program_config(self, draw): channel = draw(st.sampled_from([16, 32, 64])) input_dim = draw(st.sampled_from([16, 32, 64])) activation_type = draw( - st.sampled_from([ - 'relu', 'gelu', 'tanh', 'sigmoid', 'swish', 'mish', 'sqrt', - 'hard_swish', 'sigmoid', 'abs', 'relu6', 'clip', 'tanh', - 'hard_sigmoid', 'leaky_relu' - ])) + st.sampled_from( + [ + 'relu', + 'gelu', + 'tanh', + 'sigmoid', + 'swish', + 'mish', + 'sqrt', + 'hard_swish', + 'sigmoid', + 'abs', + 'relu6', + 'clip', + 'tanh', + 'hard_sigmoid', + 'leaky_relu', + ] + ) + ) def generate_input(): - return np.random.random([batch_size, channel, input_dim, - input_dim]).astype(np.float32) - - matmul_op = OpConfig(type='matmul', - inputs={ - 'X': ['matmul_x'], - 'Y': ['matmul_y'] - }, - outputs={'Out': ['matmul_output']}, - attrs={ - 'use_mkldnn': True, - }) + return np.random.random( + [batch_size, channel, input_dim, input_dim] + ).astype(np.float32) + + matmul_op = OpConfig( + type='matmul', + inputs={'X': ['matmul_x'], 'Y': ['matmul_y']}, + outputs={'Out': ['matmul_output']}, + attrs={ + 'use_mkldnn': True, + }, + ) if matmul_as_x: inputs = {'X': ['matmul_output'], 'Y': ['elementwise_addend']} else: inputs = {'X': ['elementwise_addend'], 'Y': ['matmul_output']} - elt_add_op = OpConfig(type='elementwise_add', - inputs=inputs, - outputs={'Out': ['elementwise_add_output']}, - attrs={ - 'axis': axis, - 'use_mkldnn': True - }) + elt_add_op = OpConfig( + type='elementwise_add', + inputs=inputs, + outputs={'Out': ['elementwise_add_output']}, + attrs={'axis': axis, 'use_mkldnn': True}, + ) if activation_type == "relu6": - activation_op = OpConfig(activation_type, - inputs={"X": ["elementwise_add_output"]}, - outputs={"Out": ["activation_output"]}, - threshold=draw( - st.floats(min_value=1.0, - max_value=10.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["elementwise_add_output"]}, + outputs={"Out": ["activation_output"]}, + threshold=draw(st.floats(min_value=1.0, max_value=10.0)), + ) elif activation_type == "leaky_relu": - activation_op = OpConfig(activation_type, - inputs={"X": ["elementwise_add_output"]}, - outputs={"Out": ["activation_output"]}, - alpha=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["elementwise_add_output"]}, + outputs={"Out": ["activation_output"]}, + alpha=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == "swish": - activation_op = OpConfig(activation_type, - inputs={"X": ["elementwise_add_output"]}, - outputs={"Out": ["activation_output"]}, - beta=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["elementwise_add_output"]}, + outputs={"Out": ["activation_output"]}, + beta=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == "clip": activation_op = OpConfig( activation_type, inputs={"X": ["elementwise_add_output"]}, outputs={"Out": ["activation_output"]}, min=draw(st.floats(min_value=0.1, max_value=0.49)), - max=draw(st.floats(min_value=0.5, max_value=1.0))) + max=draw(st.floats(min_value=0.5, max_value=1.0)), + ) else: - activation_op = OpConfig(activation_type, - inputs={"X": ["elementwise_add_output"]}, - outputs={"Out": ["activation_output"]}) + activation_op = OpConfig( + activation_type, + inputs={"X": ["elementwise_add_output"]}, + outputs={"Out": ["activation_output"]}, + ) model_net = [matmul_op, elt_add_op, activation_op] @@ -104,10 +120,12 @@ def generate_input(): inputs={ 'matmul_x': TensorConfig(data_gen=partial(generate_input)), 'matmul_y': TensorConfig(data_gen=partial(generate_input)), - 'elementwise_addend': - TensorConfig(data_gen=partial(generate_input)) + 'elementwise_addend': TensorConfig( + data_gen=partial(generate_input) + ), }, - outputs=['activation_output']) + outputs=['activation_output'], + ) return program_config @@ -116,16 +134,19 @@ def sample_predictor_configs(self, program_config): use_mkldnn=True, passes=[ 'matmul_elementwise_add_mkldnn_fuse_pass', - 'matmul_activation_mkldnn_fuse_pass' - ]) + 'matmul_activation_mkldnn_fuse_pass', + ], + ) yield config, ['matmul'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=[ - 'matmul_elementwise_add_mkldnn_fuse_pass', - 'matmul_activation_mkldnn_fuse_pass' - ]) + self.run_and_statis( + quant=False, + passes=[ + 'matmul_elementwise_add_mkldnn_fuse_pass', + 'matmul_activation_mkldnn_fuse_pass', + ], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_fuse_pass.py index 38c8985dbad1f..95b79db463016 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_elementwise_add_fuse_pass.py @@ -22,7 +22,6 @@ class TestMatmulElementwiseAddMkldnnFusePass(PassAutoScanTest): - def sample_program_config(self, draw): axis = draw(st.sampled_from([-1, 0, 1])) matmul_as_x = draw(st.booleans()) @@ -31,31 +30,30 @@ def sample_program_config(self, draw): input_dim = draw(st.sampled_from([16, 32, 64])) def generate_input(): - return np.random.random([batch_size, channel, input_dim, - input_dim]).astype(np.float32) - - matmul_op = OpConfig(type='matmul', - inputs={ - 'X': ['matmul_x'], - 'Y': ['matmul_y'] - }, - outputs={'Out': ['matmul_output']}, - attrs={ - 'use_mkldnn': True, - }) + return np.random.random( + [batch_size, channel, input_dim, input_dim] + ).astype(np.float32) + + matmul_op = OpConfig( + type='matmul', + inputs={'X': ['matmul_x'], 'Y': ['matmul_y']}, + outputs={'Out': ['matmul_output']}, + attrs={ + 'use_mkldnn': True, + }, + ) if matmul_as_x: inputs = {'X': ['matmul_output'], 'Y': ['elementwise_addend']} else: inputs = {'X': ['elementwise_addend'], 'Y': ['matmul_output']} - elt_add_op = OpConfig(type='elementwise_add', - inputs=inputs, - outputs={'Out': ['elementwise_add_output']}, - attrs={ - 'axis': axis, - 'use_mkldnn': True - }) + elt_add_op = OpConfig( + type='elementwise_add', + inputs=inputs, + outputs={'Out': ['elementwise_add_output']}, + attrs={'axis': axis, 'use_mkldnn': True}, + ) model_net = [matmul_op, elt_add_op] @@ -65,21 +63,25 @@ def generate_input(): inputs={ 'matmul_x': TensorConfig(data_gen=partial(generate_input)), 'matmul_y': TensorConfig(data_gen=partial(generate_input)), - 'elementwise_addend': - TensorConfig(data_gen=partial(generate_input)) + 'elementwise_addend': TensorConfig( + data_gen=partial(generate_input) + ), }, - outputs=['elementwise_add_output']) + outputs=['elementwise_add_output'], + ) return program_config def sample_predictor_configs(self, program_config): config = self.create_inference_config( - use_mkldnn=True, passes=['matmul_elementwise_add_mkldnn_fuse_pass']) + use_mkldnn=True, passes=['matmul_elementwise_add_mkldnn_fuse_pass'] + ) yield config, ['matmul'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=['matmul_elementwise_add_mkldnn_fuse_pass']) + self.run_and_statis( + quant=False, passes=['matmul_elementwise_add_mkldnn_fuse_pass'] + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_op_output_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_op_output_fuse_pass.py index 1136867e5530a..064847dd7a0bf 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_op_output_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_op_output_fuse_pass.py @@ -20,7 +20,6 @@ class TestMKLDNNMatmulFuseOp(InferencePassTest): - def init_data(self): self.bs = 8 self.d_type = np.float32 @@ -30,16 +29,17 @@ def init_data(self): def make_network(self): with fluid.program_guard(self.main_program, self.startup_program): - x = fluid.data(name='x', - shape=[-1] + self.shape_x, - dtype=self.d_type) - y = fluid.data(name='y', - shape=[-1] + self.shape_y, - dtype=self.d_type) + x = fluid.data( + name='x', shape=[-1] + self.shape_x, dtype=self.d_type + ) + y = fluid.data( + name='y', shape=[-1] + self.shape_y, dtype=self.d_type + ) out = fluid.layers.matmul(x, y) out = fluid.layers.transpose(out, perm=[0, 2, 1, 3]) out = fluid.layers.reshape( - out, [0, 0, self.shape_y[0] * self.shape_y[2]]) + out, [0, 0, self.shape_y[0] * self.shape_y[2]] + ) out = fluid.layers.fc(out, size=1) return out @@ -51,7 +51,7 @@ def setUp(self): def set_feeds(self, out): self.feeds = { "x": np.random.random([self.bs] + self.shape_x).astype(self.d_type), - "y": np.random.random([self.bs] + self.shape_y).astype(self.d_type) + "y": np.random.random([self.bs] + self.shape_y).astype(self.d_type), } self.fetch_list = [out] @@ -61,7 +61,6 @@ def test_check_output(self): class TestMKLDNNMatmulOtherDimsFuseOp(TestMKLDNNMatmulFuseOp): - def init_data(self): self.bs = 8 self.d_type = np.float32 @@ -71,15 +70,14 @@ def init_data(self): class TestMKLDNNMatmulOpNotFusedWrongTransposeAxis(TestMKLDNNMatmulFuseOp): - def make_network(self): with fluid.program_guard(self.main_program, self.startup_program): - x = fluid.data(name='x', - shape=[-1] + self.shape_x, - dtype=self.d_type) - y = fluid.data(name='y', - shape=[-1] + self.shape_y, - dtype=self.d_type) + x = fluid.data( + name='x', shape=[-1] + self.shape_x, dtype=self.d_type + ) + y = fluid.data( + name='y', shape=[-1] + self.shape_y, dtype=self.d_type + ) out = fluid.layers.matmul(x, y) out = fluid.layers.transpose(out, perm=[0, 1, 2, 3]) out = fluid.layers.reshape(out, [0, 0, 0, 0]) @@ -88,7 +86,6 @@ def make_network(self): class TestMKLDNNMatmulOpNotFusedBreakPattern(TestMKLDNNMatmulFuseOp): - def init_data(self): self.bs = 7 self.d_type = np.float32 @@ -98,18 +95,20 @@ def init_data(self): def make_network(self): with fluid.program_guard(self.main_program, self.startup_program): - x = fluid.data(name='x', - shape=[-1] + self.shape_x, - dtype=self.d_type) - y = fluid.data(name='y', - shape=[-1] + self.shape_y, - dtype=self.d_type) + x = fluid.data( + name='x', shape=[-1] + self.shape_x, dtype=self.d_type + ) + y = fluid.data( + name='y', shape=[-1] + self.shape_y, dtype=self.d_type + ) out = fluid.layers.matmul(x, y) out = fluid.layers.transpose(out, perm=[0, 2, 1, 3]) - out = fluid.layers.transpose(out, perm=[0, 1, 2, - 3]) # breaks pattern + out = fluid.layers.transpose( + out, perm=[0, 1, 2, 3] + ) # breaks pattern out = fluid.layers.reshape( - out, [0, 0, self.shape_y[0] * self.shape_y[2]]) + out, [0, 0, self.shape_y[0] * self.shape_y[2]] + ) out = fluid.layers.fc(out, size=1) return out diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_transpose_reshape_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_transpose_reshape_fuse_pass.py index 56825149dd26c..6c9a9c315949b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_transpose_reshape_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_transpose_reshape_fuse_pass.py @@ -22,7 +22,6 @@ class TestMatmulTransposeReshapeMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -63,39 +62,39 @@ def generate_input(type): else: return np.random.random(shape_y).astype(np.float32) - matmul_op = OpConfig(type="matmul", - inputs={ - "X": ["input_data1"], - "Y": ["input_data2"] - }, - outputs={"Out": ["matmul_output"]}, - attrs={ - "transpose_X": transpose_X, - "transpose_Y": transpose_Y, - "alpha": alpha, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }) - - transpose2_op = OpConfig(type="transpose2", - inputs={"X": ["matmul_output"]}, - outputs={ - "Out": ["transpose2_output"], - "XShape": ["transpose2_xshape"] - }, - attrs={'axis': axis}) - - reshape2_op = OpConfig(type="reshape2", - inputs={"X": ["transpose2_output"]}, - outputs={ - "Out": ["reshape2_output"], - "XShape": ["reshape2_xshape"] - }, - attrs={'shape': shape}) + matmul_op = OpConfig( + type="matmul", + inputs={"X": ["input_data1"], "Y": ["input_data2"]}, + outputs={"Out": ["matmul_output"]}, + attrs={ + "transpose_X": transpose_X, + "transpose_Y": transpose_Y, + "alpha": alpha, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + ) + + transpose2_op = OpConfig( + type="transpose2", + inputs={"X": ["matmul_output"]}, + outputs={ + "Out": ["transpose2_output"], + "XShape": ["transpose2_xshape"], + }, + attrs={'axis': axis}, + ) + + reshape2_op = OpConfig( + type="reshape2", + inputs={"X": ["transpose2_output"]}, + outputs={"Out": ["reshape2_output"], "XShape": ["reshape2_xshape"]}, + attrs={'shape': shape}, + ) model_net = [matmul_op, transpose2_op, reshape2_op] @@ -103,12 +102,15 @@ def generate_input(type): ops=model_net, weights={}, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input, "x")), - "input_data2": - TensorConfig(data_gen=partial(generate_input, "y")) + "input_data1": TensorConfig( + data_gen=partial(generate_input, "x") + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, "y") + ), }, - outputs=["reshape2_output"]) + outputs=["reshape2_output"], + ) return program_config @@ -118,7 +120,8 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis( - quant=False, passes=["matmul_transpose_reshape_mkldnn_fuse_pass"]) + quant=False, passes=["matmul_transpose_reshape_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_activation_fuse_pass.py index 153b81fa797af..97b2c22b35e79 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_activation_fuse_pass.py @@ -21,7 +21,6 @@ class TestMatmulv2ActivationMkldnnFusePass(PassAutoScanTest): - def sample_program_config(self, draw): transpose_X = draw(st.booleans()) transpose_Y = draw(st.booleans()) @@ -29,11 +28,24 @@ def sample_program_config(self, draw): channel = draw(st.sampled_from([16, 32, 64])) input_dim = draw(st.sampled_from([16, 32, 64])) activation_type = draw( - st.sampled_from([ - 'relu', 'gelu', 'swish', 'mish', 'sqrt', 'hard_swish', - 'sigmoid', 'abs', 'relu6', 'clip', 'tanh', 'hard_sigmoid', - 'leaky_relu' - ])) + st.sampled_from( + [ + 'relu', + 'gelu', + 'swish', + 'mish', + 'sqrt', + 'hard_swish', + 'sigmoid', + 'abs', + 'relu6', + 'clip', + 'tanh', + 'hard_sigmoid', + 'leaky_relu', + ] + ) + ) def generate_input(type): broadcast_X = st.booleans() @@ -60,49 +72,48 @@ def generate_input(type): else: return np.random.random(shape_y).astype(np.float32) - matmul_op = OpConfig(type='matmul_v2', - inputs={ - 'X': ['matmul_X'], - 'Y': ['matmul_Y'] - }, - outputs={'Out': ['matmul_output']}, - attrs={ - 'trans_x': transpose_X, - 'trans_y': transpose_Y - }) + matmul_op = OpConfig( + type='matmul_v2', + inputs={'X': ['matmul_X'], 'Y': ['matmul_Y']}, + outputs={'Out': ['matmul_output']}, + attrs={'trans_x': transpose_X, 'trans_y': transpose_Y}, + ) if activation_type == 'relu6': - activation_op = OpConfig(activation_type, - inputs={'X': ['matmul_output']}, - outputs={'Out': ['activation_output']}, - threshold=draw( - st.floats(min_value=1.0, - max_value=10.0))) + activation_op = OpConfig( + activation_type, + inputs={'X': ['matmul_output']}, + outputs={'Out': ['activation_output']}, + threshold=draw(st.floats(min_value=1.0, max_value=10.0)), + ) elif activation_type == 'leaky_relu': - activation_op = OpConfig(activation_type, - inputs={'X': ['matmul_output']}, - outputs={'Out': ['activation_output']}, - alpha=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={'X': ['matmul_output']}, + outputs={'Out': ['activation_output']}, + alpha=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == 'swish': - activation_op = OpConfig(activation_type, - inputs={'X': ['matmul_output']}, - outputs={'Out': ['activation_output']}, - beta=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={'X': ['matmul_output']}, + outputs={'Out': ['activation_output']}, + beta=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == 'clip': activation_op = OpConfig( activation_type, inputs={'X': ['matmul_output']}, outputs={'Out': ['activation_output']}, min=draw(st.floats(min_value=0.1, max_value=0.49)), - max=draw(st.floats(min_value=0.5, max_value=1.0))) + max=draw(st.floats(min_value=0.5, max_value=1.0)), + ) else: - activation_op = OpConfig(activation_type, - inputs={'X': ['matmul_output']}, - outputs={'Out': ['activation_output']}) + activation_op = OpConfig( + activation_type, + inputs={'X': ['matmul_output']}, + outputs={'Out': ['activation_output']}, + ) model_net = [matmul_op, activation_op] @@ -111,9 +122,10 @@ def generate_input(type): weights={}, inputs={ 'matmul_X': TensorConfig(data_gen=partial(generate_input, 'X')), - 'matmul_Y': TensorConfig(data_gen=partial(generate_input, 'Y')) + 'matmul_Y': TensorConfig(data_gen=partial(generate_input, 'Y')), }, - outputs=['activation_output']) + outputs=['activation_output'], + ) return program_config @@ -122,9 +134,11 @@ def sample_predictor_configs(self, program_config): yield config, ['matmul_v2'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_examples=30, - passes=['matmul_activation_mkldnn_fuse_pass']) + self.run_and_statis( + quant=False, + max_examples=30, + passes=['matmul_activation_mkldnn_fuse_pass'], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_elementwise_add_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_elementwise_add_fuse_pass.py index 03f2867948e91..783a38783e683 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_elementwise_add_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_elementwise_add_fuse_pass.py @@ -21,7 +21,6 @@ class TestMatmulV2ElementwiseAddMkldnnFusePass(PassAutoScanTest): - def sample_program_config(self, draw): axis = draw(st.sampled_from([-1, 0, 1])) matmul_as_x = draw(st.booleans()) @@ -49,26 +48,24 @@ def generate_input(type): shape_out = [batch_size, channel, input_dim_X, input_dim_Y] return np.random.random(shape_out).astype(np.float32) - matmul_op = OpConfig(type='matmul_v2', - inputs={ - 'X': ['matmul_X'], - 'Y': ['matmul_Y'] - }, - outputs={'Out': ['matmul_output']}, - attrs={'use_mkldnn': True}) + matmul_op = OpConfig( + type='matmul_v2', + inputs={'X': ['matmul_X'], 'Y': ['matmul_Y']}, + outputs={'Out': ['matmul_output']}, + attrs={'use_mkldnn': True}, + ) if matmul_as_x: inputs = {'X': ['matmul_output'], 'Y': ['elementwise_addend']} else: inputs = {'X': ['elementwise_addend'], 'Y': ['matmul_output']} - elt_add_op = OpConfig(type='elementwise_add', - inputs=inputs, - outputs={'Out': ['elementwise_add_output']}, - attrs={ - 'axis': axis, - 'use_mkldnn': True - }) + elt_add_op = OpConfig( + type='elementwise_add', + inputs=inputs, + outputs={'Out': ['elementwise_add_output']}, + attrs={'axis': axis, 'use_mkldnn': True}, + ) model_net = [matmul_op, elt_add_op] @@ -76,14 +73,14 @@ def generate_input(type): ops=model_net, weights={}, inputs={ - 'matmul_X': - TensorConfig(data_gen=partial(generate_input, 'X')), - 'matmul_Y': - TensorConfig(data_gen=partial(generate_input, 'Y')), - 'elementwise_addend': - TensorConfig(data_gen=partial(generate_input, 'ElAdd')) + 'matmul_X': TensorConfig(data_gen=partial(generate_input, 'X')), + 'matmul_Y': TensorConfig(data_gen=partial(generate_input, 'Y')), + 'elementwise_addend': TensorConfig( + data_gen=partial(generate_input, 'ElAdd') + ), }, - outputs=['elementwise_add_output']) + outputs=['elementwise_add_output'], + ) return program_config @@ -92,9 +89,11 @@ def sample_predictor_configs(self, program_config): yield config, ['matmul_v2'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_examples=30, - passes=['matmul_elementwise_add_mkldnn_fuse_pass']) + self.run_and_statis( + quant=False, + max_examples=30, + passes=['matmul_elementwise_add_mkldnn_fuse_pass'], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_transpose_reshape_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_transpose_reshape_fuse_pass.py index 42f349cf4ae0a..317595d20a63e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_transpose_reshape_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmul_v2_transpose_reshape_fuse_pass.py @@ -22,18 +22,25 @@ class TestMatmulv2TransposeReshapeMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: - if program_config.inputs["input_data1"].shape[ - -4] != 1 and program_config.inputs["input_data2"].shape[-4] != 1: - if program_config.inputs["input_data1"].shape[ - -4] != program_config.inputs["input_data2"].shape[-4]: + if ( + program_config.inputs["input_data1"].shape[-4] != 1 + and program_config.inputs["input_data2"].shape[-4] != 1 + ): + if ( + program_config.inputs["input_data1"].shape[-4] + != program_config.inputs["input_data2"].shape[-4] + ): return False - if program_config.inputs["input_data1"].shape[ - -3] != 1 and program_config.inputs["input_data2"].shape[-3] != 1: - if program_config.inputs["input_data1"].shape[ - -3] != program_config.inputs["input_data2"].shape[-3]: + if ( + program_config.inputs["input_data1"].shape[-3] != 1 + and program_config.inputs["input_data2"].shape[-3] != 1 + ): + if ( + program_config.inputs["input_data1"].shape[-3] + != program_config.inputs["input_data2"].shape[-3] + ): return False attrs = [ @@ -76,38 +83,38 @@ def generate_input(type): else: return np.random.random(shape_y).astype(np.float32) - matmul_op = OpConfig(type="matmul_v2", - inputs={ - "X": ["input_data1"], - "Y": ["input_data2"] - }, - outputs={"Out": ["matmul_output"]}, - attrs={ - "trans_x": transpose_X, - "trans_y": transpose_Y, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }) - - transpose2_op = OpConfig(type="transpose2", - inputs={"X": ["matmul_output"]}, - outputs={ - "Out": ["transpose2_output"], - "XShape": ["transpose2_xshape"] - }, - attrs={'axis': axis}) - - reshape2_op = OpConfig(type="reshape2", - inputs={"X": ["transpose2_output"]}, - outputs={ - "Out": ["reshape2_output"], - "XShape": ["reshape2_xshape"] - }, - attrs={'shape': shape}) + matmul_op = OpConfig( + type="matmul_v2", + inputs={"X": ["input_data1"], "Y": ["input_data2"]}, + outputs={"Out": ["matmul_output"]}, + attrs={ + "trans_x": transpose_X, + "trans_y": transpose_Y, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + ) + + transpose2_op = OpConfig( + type="transpose2", + inputs={"X": ["matmul_output"]}, + outputs={ + "Out": ["transpose2_output"], + "XShape": ["transpose2_xshape"], + }, + attrs={'axis': axis}, + ) + + reshape2_op = OpConfig( + type="reshape2", + inputs={"X": ["transpose2_output"]}, + outputs={"Out": ["reshape2_output"], "XShape": ["reshape2_xshape"]}, + attrs={'shape': shape}, + ) model_net = [matmul_op, transpose2_op, reshape2_op] @@ -115,12 +122,15 @@ def generate_input(type): ops=model_net, weights={}, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input, "x")), - "input_data2": - TensorConfig(data_gen=partial(generate_input, "y")) + "input_data1": TensorConfig( + data_gen=partial(generate_input, "x") + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, "y") + ), }, - outputs=["reshape2_output"]) + outputs=["reshape2_output"], + ) return program_config @@ -139,7 +149,8 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis( - quant=False, passes=["matmul_transpose_reshape_mkldnn_fuse_pass"]) + quant=False, passes=["matmul_transpose_reshape_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmulv2_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmulv2_op.py index 06829c2d78b5e..29dee32df5566 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmulv2_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_matmulv2_op.py @@ -23,25 +23,30 @@ class TestMkldnnMatmulv2Op(MkldnnAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: if len(program_config.inputs["input_data2"].shape) == 4: - if program_config.inputs["input_data1"].shape[ - -4] != 1 and program_config.inputs["input_data2"].shape[ - -4] != 1: - if program_config.inputs["input_data1"].shape[ - -4] != program_config.inputs["input_data2"].shape[-4]: + if ( + program_config.inputs["input_data1"].shape[-4] != 1 + and program_config.inputs["input_data2"].shape[-4] != 1 + ): + if ( + program_config.inputs["input_data1"].shape[-4] + != program_config.inputs["input_data2"].shape[-4] + ): return False - if program_config.inputs["input_data1"].shape[ - -3] != 1 and program_config.inputs["input_data2"].shape[-3] != 1: - if program_config.inputs["input_data1"].shape[ - -3] != program_config.inputs["input_data2"].shape[-3]: + if ( + program_config.inputs["input_data1"].shape[-3] != 1 + and program_config.inputs["input_data2"].shape[-3] != 1 + ): + if ( + program_config.inputs["input_data1"].shape[-3] + != program_config.inputs["input_data2"].shape[-3] + ): return False return True def sample_program_configs(self, *args, **kwargs): - def generate_input(type, *args, **kwargs): transpose_X = kwargs["transpose_X"] transpose_Y = kwargs["transpose_Y"] @@ -81,35 +86,35 @@ def generate_input(type, *args, **kwargs): else: return np.random.random(shape_y).astype(np.float32) - matmul_op = OpConfig(type="matmul_v2", - inputs={ - "X": ["input_data1"], - "Y": ["input_data2"] - }, - outputs={"Out": ["matmul_output"]}, - attrs={ - "trans_x": kwargs["transpose_X"], - "trans_y": kwargs["transpose_Y"], - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }) + matmul_op = OpConfig( + type="matmul_v2", + inputs={"X": ["input_data1"], "Y": ["input_data2"]}, + outputs={"Out": ["matmul_output"]}, + attrs={ + "trans_x": kwargs["transpose_X"], + "trans_y": kwargs["transpose_Y"], + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + ) program_config = ProgramConfig( ops=[matmul_op], weights={}, inputs={ - "input_data1": - TensorConfig( - data_gen=partial(generate_input, "x", *args, **kwargs)), - "input_data2": - TensorConfig( - data_gen=partial(generate_input, "y", *args, **kwargs)) + "input_data1": TensorConfig( + data_gen=partial(generate_input, "x", *args, **kwargs) + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, "y", *args, **kwargs) + ), }, - outputs=["matmul_output"]) + outputs=["matmul_output"], + ) yield program_config @@ -117,14 +122,16 @@ def sample_predictor_configs(self, program_config): config = self.create_inference_config(use_mkldnn=True) yield config, (1e-5, 1e-5) - @given(transpose_X=st.booleans(), - transpose_Y=st.booleans(), - y_dim_len=st.sampled_from([3, 4]), - batch_size1=st.integers(min_value=1, max_value=4), - batch_size2=st.integers(min_value=1, max_value=4), - channel1=st.sampled_from([1, 16, 32, 64]), - channel2=st.sampled_from([1, 16, 32, 64]), - input_dim=st.sampled_from([16, 32, 64])) + @given( + transpose_X=st.booleans(), + transpose_Y=st.booleans(), + y_dim_len=st.sampled_from([3, 4]), + batch_size1=st.integers(min_value=1, max_value=4), + batch_size2=st.integers(min_value=1, max_value=4), + channel1=st.sampled_from([1, 16, 32, 64]), + channel2=st.sampled_from([1, 16, 32, 64]), + input_dim=st.sampled_from([16, 32, 64]), + ) def test(self, *args, **kwargs): self.run_test(*args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_mish_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_mish_op.py index 2b2759cc65151..a6062cb526b23 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_mish_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_mish_op.py @@ -22,35 +22,39 @@ class TestMkldnnMishOp(MkldnnAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: # if mode is channel, and in_shape is 1 rank - if len(program_config.inputs['input_data'].shape - ) == 1 and program_config.ops[0].attrs['mode'] == 'channel': + if ( + len(program_config.inputs['input_data'].shape) == 1 + and program_config.ops[0].attrs['mode'] == 'channel' + ): return False return True def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): return np.random.random(kwargs['in_shape']).astype(np.float32) - mish_op = OpConfig(type="mish", - inputs={"X": ["input_data"]}, - outputs={"Out": ["output_data"]}, - attrs={ - "mode": kwargs['mode'], - "data_format": kwargs['data_format'] - }) + mish_op = OpConfig( + type="mish", + inputs={"X": ["input_data"]}, + outputs={"Out": ["output_data"]}, + attrs={ + "mode": kwargs['mode'], + "data_format": kwargs['data_format'], + }, + ) program_config = ProgramConfig( ops=[mish_op], weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config @@ -58,11 +62,13 @@ def sample_predictor_configs(self, program_config): config = self.create_inference_config(use_mkldnn=True) yield config, (1e-5, 1e-5) - @given(mode=st.sampled_from(['all', 'channel', 'element']), - data_format=st.sampled_from(['NCHW', 'NHWC']), - in_shape=st.lists(st.integers(min_value=1, max_value=32), - min_size=1, - max_size=4)) + @given( + mode=st.sampled_from(['all', 'channel', 'element']), + data_format=st.sampled_from(['NCHW', 'NHWC']), + in_shape=st.lists( + st.integers(min_value=1, max_value=32), min_size=1, max_size=4 + ), + ) def test(self, *args, **kwargs): self.run_test(quant=False, *args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad2d_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad2d_op.py index e26cf2a8605a2..86fc1df8cc641 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad2d_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad2d_op.py @@ -22,29 +22,31 @@ class TestOneDNNPad2DOp(MkldnnAutoScanTest): - def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): return np.random.random(kwargs['in_shape']).astype(np.float32) - pad3d_op = OpConfig(type="pad2d", - inputs={"X": ["input_data"]}, - outputs={"Out": ["output_data"]}, - attrs={ - "mode": "constant", - "data_format": kwargs['data_format'], - "paddings": kwargs['paddings'], - }) + pad3d_op = OpConfig( + type="pad2d", + inputs={"X": ["input_data"]}, + outputs={"Out": ["output_data"]}, + attrs={ + "mode": "constant", + "data_format": kwargs['data_format'], + "paddings": kwargs['paddings'], + }, + ) program_config = ProgramConfig( ops=[pad3d_op], weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config @@ -52,11 +54,15 @@ def sample_predictor_configs(self, program_config): config = self.create_inference_config(use_mkldnn=True) yield config, (1e-5, 1e-5) - @given(data_format=st.sampled_from(['NCHW', 'NHWC']), - in_shape=st.sampled_from([[2, 3, 4, 5], [1, 4, 1, 3], [4, 3, 2, 1], - [1, 1, 1, 1]]), - paddings=st.sampled_from([[0, 0, 0, 0], [1, 2, 0, 1], [2, 5, 11, 3], - [0, 5, 0, 1]])) + @given( + data_format=st.sampled_from(['NCHW', 'NHWC']), + in_shape=st.sampled_from( + [[2, 3, 4, 5], [1, 4, 1, 3], [4, 3, 2, 1], [1, 1, 1, 1]] + ), + paddings=st.sampled_from( + [[0, 0, 0, 0], [1, 2, 0, 1], [2, 5, 11, 3], [0, 5, 0, 1]] + ), + ) def test(self, *args, **kwargs): self.run_test(quant=False, *args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad3d_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad3d_op.py index 8184bb149e1ac..cb4daa686534f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad3d_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_pad3d_op.py @@ -22,37 +22,35 @@ class TestOneDNNPad3DOp(MkldnnAutoScanTest): - def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): return np.random.random(kwargs['in_shape']).astype(np.float32) def generate_paddings(): return np.random.randint(0, 4, size=(6)).astype(np.int32) - pad3d_op = OpConfig(type="pad3d", - inputs={ - "X": ["input_data"], - "Paddings": ["paddings_data"] - }, - outputs={"Out": ["output_data"]}, - attrs={ - "mode": "constant", - "data_format": kwargs['data_format'], - "paddings": kwargs['paddings'], - }) + pad3d_op = OpConfig( + type="pad3d", + inputs={"X": ["input_data"], "Paddings": ["paddings_data"]}, + outputs={"Out": ["output_data"]}, + attrs={ + "mode": "constant", + "data_format": kwargs['data_format'], + "paddings": kwargs['paddings'], + }, + ) program_config = ProgramConfig( ops=[pad3d_op], weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)), - "paddings_data": - TensorConfig(data_gen=generate_paddings) + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ), + "paddings_data": TensorConfig(data_gen=generate_paddings), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config @@ -60,12 +58,21 @@ def sample_predictor_configs(self, program_config): config = self.create_inference_config(use_mkldnn=True) yield config, (1e-5, 1e-5) - @given(data_format=st.sampled_from(['NCDHW', 'NDHWC']), - use_paddings_tensor=st.sampled_from([True, False]), - in_shape=st.sampled_from([[2, 3, 4, 5, 6], [1, 4, 1, 3, 2], - [4, 3, 2, 1, 1], [1, 1, 1, 1, 1]]), - paddings=st.sampled_from([[0, 0, 0, 0, 0, 0], [1, 2, 0, 1, 2, 1], - [2, 5, 11, 3, 4, 3], [0, 5, 0, 1, 0, 2]])) + @given( + data_format=st.sampled_from(['NCDHW', 'NDHWC']), + use_paddings_tensor=st.sampled_from([True, False]), + in_shape=st.sampled_from( + [[2, 3, 4, 5, 6], [1, 4, 1, 3, 2], [4, 3, 2, 1, 1], [1, 1, 1, 1, 1]] + ), + paddings=st.sampled_from( + [ + [0, 0, 0, 0, 0, 0], + [1, 2, 0, 1, 2, 1], + [2, 5, 11, 3, 4, 3], + [0, 5, 0, 1, 0, 2], + ] + ), + ) def test(self, *args, **kwargs): self.run_test(quant=False, *args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_prelu_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_prelu_op.py index 646cf1170ee97..5a9aabe1b9b5c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_prelu_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_prelu_op.py @@ -23,16 +23,16 @@ class TestMkldnnPreluOp(MkldnnAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: # if mode is channel, and in_shape is 1 rank - if len(program_config.inputs['input_data'].shape - ) == 1 and program_config.ops[0].attrs['mode'] == 'channel': + if ( + len(program_config.inputs['input_data'].shape) == 1 + and program_config.ops[0].attrs['mode'] == 'channel' + ): return False return True def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): return np.random.random(kwargs['in_shape']).astype(np.float32) @@ -45,38 +45,42 @@ def generate_alpha(*args, **kwargs): return np.zeros((1)).astype(np.float32) if kwargs['data_format'] == 'NCHW': return np.random.random(kwargs['in_shape'][1]).astype( - np.float32) + np.float32 + ) else: return np.random.random(kwargs['in_shape'][-1]).astype( - np.float32) + np.float32 + ) else: if len(kwargs['in_shape']) <= 1: # not valid case, just return 0 return np.zeros((1)).astype(np.float32) return np.random.random(kwargs['in_shape']).astype(np.float32) - prelu_op = OpConfig(type="prelu", - inputs={ - "X": ["input_data"], - "Alpha": ["alpha_weight"] - }, - outputs={"Out": ["output_data"]}, - attrs={ - "mode": kwargs['mode'], - "data_format": kwargs['data_format'] - }) + prelu_op = OpConfig( + type="prelu", + inputs={"X": ["input_data"], "Alpha": ["alpha_weight"]}, + outputs={"Out": ["output_data"]}, + attrs={ + "mode": kwargs['mode'], + "data_format": kwargs['data_format'], + }, + ) program_config = ProgramConfig( ops=[prelu_op], weights={ - "alpha_weight": - TensorConfig(data_gen=partial(generate_alpha, *args, **kwargs)) + "alpha_weight": TensorConfig( + data_gen=partial(generate_alpha, *args, **kwargs) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config @@ -87,11 +91,13 @@ def sample_predictor_configs(self, program_config): def add_skip_pass_case(self): pass - @given(mode=st.sampled_from(['all', 'channel', 'element']), - data_format=st.sampled_from(['NCHW', 'NHWC']), - in_shape=st.lists(st.integers(min_value=1, max_value=32), - min_size=1, - max_size=4)) + @given( + mode=st.sampled_from(['all', 'channel', 'element']), + data_format=st.sampled_from(['NCHW', 'NHWC']), + in_shape=st.lists( + st.integers(min_value=1, max_value=32), min_size=1, max_size=4 + ), + ) def test(self, *args, **kwargs): self.add_skip_pass_case() self.run_test(quant=False, *args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_fuse_pass.py index 62615ce54c87c..7147c412618fd 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_fuse_pass.py @@ -25,7 +25,6 @@ class TestReshapeTransposeMatmulMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -35,8 +34,8 @@ def sample_program_config(self, draw): alpha = draw(st.floats(min_value=0.01, max_value=2)) axis = draw(st.sampled_from([[0, 2, 1, 3]])) shape = draw( - st.sampled_from([[0, 64, -1, 32], [0, 32, -1, 64], [-1, 32, 1, - 64]])) + st.sampled_from([[0, 64, -1, 32], [0, 32, -1, 64], [-1, 32, 1, 64]]) + ) batch_size = draw(st.integers(min_value=1, max_value=4)) channel = draw(st.integers(min_value=1, max_value=64)) input_dim = draw(st.sampled_from([32, 64])) @@ -65,8 +64,10 @@ def generate_input2(attrs): if attrs[2]['transpose_X'] and attrs[2]['transpose_Y']: shape_y = [ - matmul_shape[0], matmul_shape[1], matmul_shape[-1], - int(num / matmul_shape[-1]) + matmul_shape[0], + matmul_shape[1], + matmul_shape[-1], + int(num / matmul_shape[-1]), ] elif attrs[2]['transpose_X']: shape_y = matmul_shape @@ -74,70 +75,64 @@ def generate_input2(attrs): shape_y = matmul_shape else: shape_y = [ - matmul_shape[0], matmul_shape[1], matmul_shape[-1], - int(num / matmul_shape[-1]) + matmul_shape[0], + matmul_shape[1], + matmul_shape[-1], + int(num / matmul_shape[-1]), ] return np.random.random(shape_y).astype(np.float32) - attrs = [{ - "shape": shape - }, { - "axis": axis - }, { - "transpose_X": transpose_X, - "transpose_Y": transpose_Y, - "alpha": alpha - }, { - 'batch_size': batch_size, - 'channel': channel, - 'input_dim': input_dim - }] - - ops_config = [{ - "op_type": "reshape2", - "op_inputs": { - "X": ["input_data1"] - }, - "op_outputs": { - "Out": ["reshape2_output"], - "XShape": ["reshape2_xshape"] - }, - "op_attrs": { - 'shape': attrs[0]['shape'] - }, - }, { - "op_type": "transpose2", - "op_inputs": { - "X": ["reshape2_output"] + attrs = [ + {"shape": shape}, + {"axis": axis}, + { + "transpose_X": transpose_X, + "transpose_Y": transpose_Y, + "alpha": alpha, }, - "op_outputs": { - "Out": ["transpose2_output"], - "XShape": ["transpose2_xshape"] + { + 'batch_size': batch_size, + 'channel': channel, + 'input_dim': input_dim, }, - "op_attrs": { - 'axis': attrs[1]['axis'] + ] + + ops_config = [ + { + "op_type": "reshape2", + "op_inputs": {"X": ["input_data1"]}, + "op_outputs": { + "Out": ["reshape2_output"], + "XShape": ["reshape2_xshape"], + }, + "op_attrs": {'shape': attrs[0]['shape']}, }, - }, { - "op_type": "matmul", - "op_inputs": { - "X": ["transpose2_output"], - "Y": ["input_data2"] + { + "op_type": "transpose2", + "op_inputs": {"X": ["reshape2_output"]}, + "op_outputs": { + "Out": ["transpose2_output"], + "XShape": ["transpose2_xshape"], + }, + "op_attrs": {'axis': attrs[1]['axis']}, }, - "op_outputs": { - "Out": ["matmul_output"] + { + "op_type": "matmul", + "op_inputs": {"X": ["transpose2_output"], "Y": ["input_data2"]}, + "op_outputs": {"Out": ["matmul_output"]}, + "op_attrs": { + 'transpose_X': attrs[2]['transpose_X'], + 'transpose_Y': attrs[2]['transpose_Y'], + 'alpha': attrs[2]['alpha'], + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, }, - "op_attrs": { - 'transpose_X': attrs[2]['transpose_X'], - 'transpose_Y': attrs[2]['transpose_Y'], - 'alpha': attrs[2]['alpha'], - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - } - }] + ] ops = self.generate_op_config(ops_config) @@ -145,12 +140,15 @@ def generate_input2(attrs): ops=ops, weights={}, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input1, attrs)), - "input_data2": - TensorConfig(data_gen=partial(generate_input2, attrs)) + "input_data1": TensorConfig( + data_gen=partial(generate_input1, attrs) + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input2, attrs) + ), }, - outputs=["matmul_output"]) + outputs=["matmul_output"], + ) return program_config @@ -160,7 +158,8 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis( - quant=False, passes=["reshape_transpose_matmul_mkldnn_fuse_pass"]) + quant=False, passes=["reshape_transpose_matmul_mkldnn_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_v2_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_v2_fuse_pass.py index 9d6c0309e17d2..b62b3eaf51795 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_v2_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_reshape_transpose_matmul_v2_fuse_pass.py @@ -21,24 +21,26 @@ class TestReshapeTransposeMatmulV2OneDNNFusePass(InferencePassTest): - def setUp(self): self.set_params() self.tranpose_perm = [0, 2, 1, 3] self.pass_name = 'reshape_transpose_matmul_mkldnn_fuse_pass' with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=self.data_shape, - dtype="float32") - weight = fluid.layers.create_parameter(shape=self.weight_shape, - dtype="float32") + data = fluid.data( + name="data", shape=self.data_shape, dtype="float32" + ) + weight = fluid.layers.create_parameter( + shape=self.weight_shape, dtype="float32" + ) reshape = fluid.layers.reshape(data, shape=self.reshape_shape) transpose = fluid.layers.transpose(reshape, self.tranpose_perm) - matmul = paddle.matmul(transpose, - weight, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y) + matmul = paddle.matmul( + transpose, + weight, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + ) self.fetch_list = [matmul] self.enable_mkldnn = True @@ -60,8 +62,8 @@ def test_pass_compatible(self): class TestReshapeTransposeMatmulV2OneDNNFusePassBroadcast( - TestReshapeTransposeMatmulV2OneDNNFusePass): - + TestReshapeTransposeMatmulV2OneDNNFusePass +): def set_params(self): self.data_shape = [2, 64, 16] self.weight_shape = [1, 2, 8, 64] diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_scale_matmul_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_scale_matmul_fuse_pass.py index c12915c1eb150..109a56edbcb02 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_scale_matmul_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_scale_matmul_fuse_pass.py @@ -22,7 +22,6 @@ class TestScaleMatmulMkldnnFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -40,39 +39,55 @@ def sample_program_config(self, draw): def generate_input(attrs, type): if attrs[1]['transpose_X'] and attrs[1]['transpose_Y']: shape_x = [ - attrs[2]['batch_size'], attrs[2]['channel'], - attrs[2]['input_dim'], 32 + attrs[2]['batch_size'], + attrs[2]['channel'], + attrs[2]['input_dim'], + 32, ] shape_y = [ - attrs[2]['batch_size'], attrs[2]['channel'], 64, - attrs[2]['input_dim'] + attrs[2]['batch_size'], + attrs[2]['channel'], + 64, + attrs[2]['input_dim'], ] elif attrs[1]['transpose_X']: shape_x = [ - attrs[2]['batch_size'], attrs[2]['channel'], - attrs[2]['input_dim'], 32 + attrs[2]['batch_size'], + attrs[2]['channel'], + attrs[2]['input_dim'], + 32, ] shape_y = [ - attrs[2]['batch_size'], attrs[2]['channel'], - attrs[2]['input_dim'], 64 + attrs[2]['batch_size'], + attrs[2]['channel'], + attrs[2]['input_dim'], + 64, ] elif attrs[1]['transpose_Y']: shape_x = [ - attrs[2]['batch_size'], attrs[2]['channel'], 32, - attrs[2]['input_dim'] + attrs[2]['batch_size'], + attrs[2]['channel'], + 32, + attrs[2]['input_dim'], ] shape_y = [ - attrs[2]['batch_size'], attrs[2]['channel'], 8, - attrs[2]['input_dim'] + attrs[2]['batch_size'], + attrs[2]['channel'], + 8, + attrs[2]['input_dim'], ] else: shape_x = [ - attrs[2]['batch_size'], attrs[2]['channel'], 32, - attrs[2]['input_dim'] + attrs[2]['batch_size'], + attrs[2]['channel'], + 32, + attrs[2]['input_dim'], ] shape_y = [ - attrs[2]['batch_size'], attrs[2]['channel'], - attrs[2]['input_dim'], 16 + attrs[2]['batch_size'], + attrs[2]['channel'], + attrs[2]['input_dim'], + 16, ] if type == "x": @@ -80,54 +95,52 @@ def generate_input(attrs, type): else: return np.random.random(shape_y).astype(np.float32) - attrs = [{ - "scale": scale, - "bias": bias, - "bias_after_scale": bias_after_scale - }, { - "transpose_X": transpose_X, - "transpose_Y": transpose_Y, - "alpha": alpha - }, { - 'batch_size': batch_size, - 'channel': channel, - 'input_dim': input_dim - }] - - ops_config = [{ - "op_type": "scale", - "op_inputs": { - "X": ["input_data1"] + attrs = [ + { + "scale": scale, + "bias": bias, + "bias_after_scale": bias_after_scale, }, - "op_outputs": { - "Out": ["scale_output"] + { + "transpose_X": transpose_X, + "transpose_Y": transpose_Y, + "alpha": alpha, }, - "op_attrs": { - "scale": attrs[0]['scale'], - "bias": attrs[0]['bias'], - "bias_after_scale": attrs[0]['bias_after_scale'] + { + 'batch_size': batch_size, + 'channel': channel, + 'input_dim': input_dim, }, - }, { - "op_type": "matmul", - "op_inputs": { - "X": ["scale_output"], - "Y": ["input_data2"] + ] + + ops_config = [ + { + "op_type": "scale", + "op_inputs": {"X": ["input_data1"]}, + "op_outputs": {"Out": ["scale_output"]}, + "op_attrs": { + "scale": attrs[0]['scale'], + "bias": attrs[0]['bias'], + "bias_after_scale": attrs[0]['bias_after_scale'], + }, }, - "op_outputs": { - "Out": ["matmul_output"] + { + "op_type": "matmul", + "op_inputs": {"X": ["scale_output"], "Y": ["input_data2"]}, + "op_outputs": {"Out": ["matmul_output"]}, + "op_attrs": { + 'transpose_X': attrs[1]['transpose_X'], + 'transpose_Y': attrs[1]['transpose_Y'], + 'alpha': attrs[1]['alpha'], + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, }, - "op_attrs": { - 'transpose_X': attrs[1]['transpose_X'], - 'transpose_Y': attrs[1]['transpose_Y'], - 'alpha': attrs[1]['alpha'], - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - } - }] + ] ops = self.generate_op_config(ops_config) @@ -135,12 +148,15 @@ def generate_input(attrs, type): ops=ops, weights={}, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input, attrs, "x")), - "input_data2": - TensorConfig(data_gen=partial(generate_input, attrs, "y")) + "input_data1": TensorConfig( + data_gen=partial(generate_input, attrs, "x") + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, attrs, "y") + ), }, - outputs=["matmul_output"]) + outputs=["matmul_output"], + ) return program_config diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shape_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shape_op.py index 92111062b1298..15fae34372975 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shape_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shape_op.py @@ -22,28 +22,31 @@ class TestMkldnnShapeOp(MkldnnAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): return np.random.random(kwargs['in_shape']).astype( - kwargs['in_dtype']) + kwargs['in_dtype'] + ) - shape_op = OpConfig(type="shape", - inputs={"Input": ["input_data"]}, - outputs={"Out": ["output_data"]}) + shape_op = OpConfig( + type="shape", + inputs={"Input": ["input_data"]}, + outputs={"Out": ["output_data"]}, + ) program_config = ProgramConfig( ops=[shape_op], weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config @@ -51,10 +54,12 @@ def sample_predictor_configs(self, program_config): config = self.create_inference_config(use_mkldnn=True) yield config, (1e-5, 1e-5) - @given(in_shape=st.lists(st.integers(min_value=1, max_value=3), - min_size=1, - max_size=6), - in_dtype=st.sampled_from([np.float32, np.uint16, np.int8, np.uint8])) + @given( + in_shape=st.lists( + st.integers(min_value=1, max_value=3), min_size=1, max_size=6 + ), + in_dtype=st.sampled_from([np.float32, np.uint16, np.int8, np.uint8]), + ) def test(self, *args, **kwargs): self.run_test(quant=False, *args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_detect_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_detect_pass.py index 423bb0f0dbb45..b4d94c4e81519 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_detect_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_detect_pass.py @@ -31,7 +31,6 @@ def product(input): class TestShuffleChannelMKLDNNDetectPass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: input_shape = program_config.inputs['input_data'].shape first_reshape2_shape = program_config.ops[0].attrs['shape'] @@ -43,18 +42,26 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: img_w = input_shape[-1] if shape_prod != product(first_reshape2_shape) or shape_prod != product( - second_reshape2_shape): + second_reshape2_shape + ): return False - if len(input_shape) != 4 or len(first_reshape2_shape) != 5 or len( - second_reshape2_shape) != 4: + if ( + len(input_shape) != 4 + or len(first_reshape2_shape) != 5 + or len(second_reshape2_shape) != 4 + ): return False if transpose2_axis != [0, 2, 1, 3, 4]: return False - if first_reshape2_shape[-1] != img_w or first_reshape2_shape[ - -2] != img_h: + if ( + first_reshape2_shape[-1] != img_w + or first_reshape2_shape[-2] != img_h + ): return False - if second_reshape2_shape[-1] != img_w or second_reshape2_shape[ - -2] != img_h: + if ( + second_reshape2_shape[-1] != img_w + or second_reshape2_shape[-2] != img_h + ): return False return True @@ -62,10 +69,12 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: def sample_program_config(self, draw): input_shape = draw(st.sampled_from([[128, 32, 32]])) first_reshape2_shape = draw( - st.sampled_from([[2, 64, 32, 32], [8, 16, 32, 32]])) + st.sampled_from([[2, 64, 32, 32], [8, 16, 32, 32]]) + ) transpose2_axis = draw(st.sampled_from([[0, 2, 1, 3, 4], [0, 2, 1, 3]])) second_reshape2_shape = draw( - st.sampled_from([[128, 32, 32], [128, 31, 32]])) + st.sampled_from([[128, 32, 32], [128, 31, 32]]) + ) batch_size = draw(st.integers(min_value=1, max_value=10)) input_shape.insert(0, batch_size) @@ -75,43 +84,37 @@ def sample_program_config(self, draw): def generate_input(): return np.random.random(input_shape).astype(np.float32) - ops_config = [{ - "op_type": "reshape2", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["first_reshape2_output"], - "XShape": ["first_reshape2_xshape"] - }, - "op_attrs": { - 'shape': first_reshape2_shape - }, - }, { - "op_type": "transpose2", - "op_inputs": { - "X": ["first_reshape2_output"] - }, - "op_outputs": { - "Out": ["transpose2_output"], - "XShape": ["transpose2_xshape"] - }, - "op_attrs": { - 'axis': transpose2_axis + ops_config = [ + { + "op_type": "reshape2", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": { + "Out": ["first_reshape2_output"], + "XShape": ["first_reshape2_xshape"], + }, + "op_attrs": {'shape': first_reshape2_shape}, }, - }, { - "op_type": "reshape2", - "op_inputs": { - "X": ["transpose2_output"], + { + "op_type": "transpose2", + "op_inputs": {"X": ["first_reshape2_output"]}, + "op_outputs": { + "Out": ["transpose2_output"], + "XShape": ["transpose2_xshape"], + }, + "op_attrs": {'axis': transpose2_axis}, }, - "op_outputs": { - "Out": ["output_data"], - "XShape": ["second_reshape2_xshape"] + { + "op_type": "reshape2", + "op_inputs": { + "X": ["transpose2_output"], + }, + "op_outputs": { + "Out": ["output_data"], + "XShape": ["second_reshape2_xshape"], + }, + "op_attrs": {'shape': second_reshape2_shape}, }, - "op_attrs": { - 'shape': second_reshape2_shape - } - }] + ] ops = self.generate_op_config(ops_config) @@ -121,7 +124,8 @@ def generate_input(): inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)) }, - outputs=["output_data"]) + outputs=["output_data"], + ) return program_config @@ -130,8 +134,9 @@ def sample_predictor_configs(self, program_config): yield config, ["shuffle_channel"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["shuffle_channel_mkldnn_detect_pass"]) + self.run_and_statis( + quant=False, passes=["shuffle_channel_mkldnn_detect_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_op.py index d9050b58ee380..72dfe3f65611b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_shuffle_channel_op.py @@ -22,28 +22,30 @@ class TestMKLDNNShuffleChannelOp(MkldnnAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self, *args, **kwargs): - def generate_input(*args, **kwargs): return np.random.random(kwargs['in_shape']).astype(np.float32) - shuffle_channel_op = OpConfig(type="shuffle_channel", - inputs={"X": ["input_data"]}, - outputs={"Out": ["output_data"]}, - attrs={"group": kwargs['group']}) + shuffle_channel_op = OpConfig( + type="shuffle_channel", + inputs={"X": ["input_data"]}, + outputs={"Out": ["output_data"]}, + attrs={"group": kwargs['group']}, + ) program_config = ProgramConfig( ops=[shuffle_channel_op], weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, *args, **kwargs)), + "input_data": TensorConfig( + data_gen=partial(generate_input, *args, **kwargs) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config @@ -51,8 +53,10 @@ def sample_predictor_configs(self, program_config): config = self.create_inference_config(use_mkldnn=True) yield config, (1e-5, 1e-5) - @given(group=st.sampled_from([1, 2, 8, 32, 128]), - in_shape=st.sampled_from([[5, 512, 2, 3], [2, 256, 5, 4]])) + @given( + group=st.sampled_from([1, 2, 8, 32, 128]), + in_shape=st.sampled_from([[5, 512, 2, 3], [2, 256, 5, 4]]), + ) def test(self, *args, **kwargs): self.run_test(quant=False, *args, **kwargs) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py index 037234ccda1c5..67445e18444fb 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mkldnn_softplus_activation_fuse_pass.py @@ -28,17 +28,18 @@ class SoftplusActivationReluOneDNNFusePassTest(InferencePassTest): def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 100, 100], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 100, 100], dtype="float32" + ) softplus_out = fluid.layers.softplus(data) if self.fuse_beta is not None: - activation_out = self.fuse_activation(softplus_out, - self.fuse_alpha, - self.fuse_beta) + activation_out = self.fuse_activation( + softplus_out, self.fuse_alpha, self.fuse_beta + ) elif self.fuse_alpha is not None: - activation_out = self.fuse_activation(softplus_out, - self.fuse_alpha) + activation_out = self.fuse_activation( + softplus_out, self.fuse_alpha + ) else: activation_out = self.fuse_activation(softplus_out) @@ -60,52 +61,52 @@ def test_pass_compatible(self): class SoftplusActivationTanhOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.tanh class SoftplusActivationLeakyReluOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.leaky_relu self.fuse_alpha = 0.3 class SoftplusActivationSwishOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.swish self.fuse_alpha = 3 class SoftplusActivationHardSwishOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.hard_swish class SoftplusActivationSqrtOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.hard_swish class SoftplusActivationAbsOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.abs class SoftplusActivationClipOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.clip self.fuse_alpha = 1.1 @@ -113,30 +114,30 @@ def set_params(self): class SoftplusActivationGeluErfOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.gelu class SoftplusActivationGeluTanhOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.gelu self.fuse_alpha = True # simulated "Approximate" attr class SoftplusActivationRelu6OneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.relu6 class SoftplusActivationSigmoidOneDNNFusePassTest( - SoftplusActivationReluOneDNNFusePassTest): - + SoftplusActivationReluOneDNNFusePassTest +): def set_params(self): self.fuse_activation = fluid.layers.sigmoid diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mul_gru_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mul_gru_fuse_pass.py index 54f6b60ff86d8..ad0ba4c1fc9d5 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mul_gru_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mul_gru_fuse_pass.py @@ -22,7 +22,6 @@ class TestMulGruFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -43,83 +42,89 @@ def generate_input(): def generate_weight(shape): return np.full(shape, 0.0001).astype(np.float32) - im2sequence_op = OpConfig(type="im2sequence", - inputs={"X": ["input_data"]}, - outputs={"Out": ["seq_out"]}, - attrs={ - "kernels": [6, 1], - "out_stride": [1, 1], - "paddings": [0, 0, 0, 0], - "strides": [1, 1] - }) - - mul_op = OpConfig(type="mul", - inputs={ - "X": ["seq_out"], - "Y": ["mul_weight"] - }, - outputs={"Out": ["mul_out"]}, - attrs={ - "x_num_col_dims": x_col, - "y_num_col_dims": y_col - }) + im2sequence_op = OpConfig( + type="im2sequence", + inputs={"X": ["input_data"]}, + outputs={"Out": ["seq_out"]}, + attrs={ + "kernels": [6, 1], + "out_stride": [1, 1], + "paddings": [0, 0, 0, 0], + "strides": [1, 1], + }, + ) + + mul_op = OpConfig( + type="mul", + inputs={"X": ["seq_out"], "Y": ["mul_weight"]}, + outputs={"Out": ["mul_out"]}, + attrs={"x_num_col_dims": x_col, "y_num_col_dims": y_col}, + ) if has_origin_mode: - gru_op = OpConfig(type="gru", - inputs={ - "Input": ["mul_out"], - "Weight": ["gru_weight"], - "Bias": ["gru_bias"] - }, - outputs={ - "BatchGate": ["batch_gate"], - "BatchHidden": ["batch_hidden"], - "BatchResetHiddenPrev": ["batch_reset"], - "Hidden": ["hidden"] - }, - attrs={ - 'activation': activation, - 'is_reverse': is_reverse, - 'gate_activation': gate_activation, - 'is_test': True, - 'origin_mode': origin_mode - }) + gru_op = OpConfig( + type="gru", + inputs={ + "Input": ["mul_out"], + "Weight": ["gru_weight"], + "Bias": ["gru_bias"], + }, + outputs={ + "BatchGate": ["batch_gate"], + "BatchHidden": ["batch_hidden"], + "BatchResetHiddenPrev": ["batch_reset"], + "Hidden": ["hidden"], + }, + attrs={ + 'activation': activation, + 'is_reverse': is_reverse, + 'gate_activation': gate_activation, + 'is_test': True, + 'origin_mode': origin_mode, + }, + ) else: - gru_op = OpConfig(type="gru", - inputs={ - "Input": ["mul_out"], - "Weight": ["gru_weight"], - "Bias": ["gru_bias"] - }, - outputs={ - "BatchGate": ["batch_gate"], - "BatchHidden": ["batch_hidden"], - "BatchResetHiddenPrev": ["batch_reset"], - "Hidden": ["hidden"] - }, - attrs={ - 'activation': activation, - 'is_reverse': is_reverse, - 'gate_activation': gate_activation, - 'is_test': True - }) + gru_op = OpConfig( + type="gru", + inputs={ + "Input": ["mul_out"], + "Weight": ["gru_weight"], + "Bias": ["gru_bias"], + }, + outputs={ + "BatchGate": ["batch_gate"], + "BatchHidden": ["batch_hidden"], + "BatchResetHiddenPrev": ["batch_reset"], + "Hidden": ["hidden"], + }, + attrs={ + 'activation': activation, + 'is_reverse': is_reverse, + 'gate_activation': gate_activation, + 'is_test': True, + }, + ) model_net = [im2sequence_op, mul_op, gru_op] program_config = ProgramConfig( ops=model_net, weights={ - "mul_weight": - TensorConfig(data_gen=partial(generate_weight, [768, 600])), - "gru_weight": - TensorConfig(data_gen=partial(generate_weight, [200, 600])), - "gru_bias": - TensorConfig(data_gen=partial(generate_weight, [1, 600])) + "mul_weight": TensorConfig( + data_gen=partial(generate_weight, [768, 600]) + ), + "gru_weight": TensorConfig( + data_gen=partial(generate_weight, [200, 600]) + ), + "gru_bias": TensorConfig( + data_gen=partial(generate_weight, [1, 600]) + ), }, inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)) }, - outputs=["hidden"]) + outputs=["hidden"], + ) return program_config @@ -128,9 +133,9 @@ def sample_predictor_configs(self, program_config): yield config, ["im2sequence", "fusion_gru"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_duration=300, - passes=["mul_gru_fuse_pass"]) + self.run_and_statis( + quant=False, max_duration=300, passes=["mul_gru_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_mul_lstm_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_mul_lstm_fuse_pass.py index 2d65a7fad5e0e..3b6e72ecf4330 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_mul_lstm_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_mul_lstm_fuse_pass.py @@ -22,7 +22,6 @@ class TestMulLstmFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -34,7 +33,8 @@ def sample_program_config(self, draw): gate_activation = draw(st.sampled_from(["sigmoid"])) cell_activation = draw(st.sampled_from(["tanh", "relu", "identity"])) candidate_activation = draw( - st.sampled_from(["tanh", "relu", "identity"])) + st.sampled_from(["tanh", "relu", "identity"]) + ) batch_size = draw(st.integers(min_value=1, max_value=40)) def generate_input(): @@ -44,47 +44,47 @@ def generate_input(): def generate_weight(shape): return np.full(shape, 0.0001).astype(np.float32) - im2sequence_op = OpConfig(type="im2sequence", - inputs={"X": ["input_data"]}, - outputs={"Out": ["seq_out"]}, - attrs={ - "kernels": [6, 1], - "out_stride": [1, 1], - "paddings": [0, 0, 0, 0], - "strides": [1, 1] - }) - - mul_op = OpConfig(type="mul", - inputs={ - "X": ["seq_out"], - "Y": ["mul_weight"] - }, - outputs={"Out": ["mul_out"]}, - attrs={ - "x_num_col_dims": x_col, - "y_num_col_dims": y_col - }) - - lstm_op = OpConfig(type="lstm", - inputs={ - "Input": ["mul_out"], - "Weight": ["lstm_weight"], - "Bias": ["lstm_bias"] - }, - outputs={ - "Hidden": ["lstm_hidden"], - "Cell": ["lstm_cell"], - "BatchGate": ["lstm_gate"], - "BatchCellPreAct": ["lstm_batch_cell"] - }, - attrs={ - 'use_peepholes': use_peepholes, - 'is_reverse': is_reverse, - 'gate_activation': gate_activation, - 'cell_activation': cell_activation, - 'candidate_activation': candidate_activation, - 'is_test': True - }) + im2sequence_op = OpConfig( + type="im2sequence", + inputs={"X": ["input_data"]}, + outputs={"Out": ["seq_out"]}, + attrs={ + "kernels": [6, 1], + "out_stride": [1, 1], + "paddings": [0, 0, 0, 0], + "strides": [1, 1], + }, + ) + + mul_op = OpConfig( + type="mul", + inputs={"X": ["seq_out"], "Y": ["mul_weight"]}, + outputs={"Out": ["mul_out"]}, + attrs={"x_num_col_dims": x_col, "y_num_col_dims": y_col}, + ) + + lstm_op = OpConfig( + type="lstm", + inputs={ + "Input": ["mul_out"], + "Weight": ["lstm_weight"], + "Bias": ["lstm_bias"], + }, + outputs={ + "Hidden": ["lstm_hidden"], + "Cell": ["lstm_cell"], + "BatchGate": ["lstm_gate"], + "BatchCellPreAct": ["lstm_batch_cell"], + }, + attrs={ + 'use_peepholes': use_peepholes, + 'is_reverse': is_reverse, + 'gate_activation': gate_activation, + 'cell_activation': cell_activation, + 'candidate_activation': candidate_activation, + 'is_test': True, + }, + ) model_net = [im2sequence_op, mul_op, lstm_op] @@ -96,17 +96,21 @@ def generate_weight(shape): program_config = ProgramConfig( ops=model_net, weights={ - "mul_weight": - TensorConfig(data_gen=partial(generate_weight, [768, 600])), - "lstm_weight": - TensorConfig(data_gen=partial(generate_weight, [150, 600])), - "lstm_bias": - TensorConfig(data_gen=partial(generate_weight, lstm_bias_shape)) + "mul_weight": TensorConfig( + data_gen=partial(generate_weight, [768, 600]) + ), + "lstm_weight": TensorConfig( + data_gen=partial(generate_weight, [150, 600]) + ), + "lstm_bias": TensorConfig( + data_gen=partial(generate_weight, lstm_bias_shape) + ), }, inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)), }, - outputs=["lstm_hidden"]) + outputs=["lstm_hidden"], + ) return program_config @@ -115,9 +119,9 @@ def sample_predictor_configs(self, program_config): yield config, ["im2sequence", "fusion_lstm"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_duration=300, - passes=["mul_lstm_fuse_pass"]) + self.run_and_statis( + quant=False, max_duration=300, passes=["mul_lstm_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_multihead_matmul_fuse_pass_v3.py b/python/paddle/fluid/tests/unittests/ir/inference/test_multihead_matmul_fuse_pass_v3.py index c59da1a8913ea..25f1c551709b5 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_multihead_matmul_fuse_pass_v3.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_multihead_matmul_fuse_pass_v3.py @@ -20,171 +20,190 @@ class TestMultiheadMatmulFusePass(PassAutoScanTest): - def sample_predictor_configs(self, program_config): # gpu config = self.create_inference_config(use_gpu=True) yield config, ["multihead_matmul", "mul"], (1e-2, 1e-3) def sample_program_config(self, draw): - def generate_mul_input(): return np.random.random([1, 128, 768]).astype(np.float32) - 0.5 def generate_elewise_input(): return np.random.random([1, 12, 128, 128]).astype(np.float32) - mul_0 = OpConfig("mul", - inputs={ - "X": ["mul_x"], - "Y": ["mul_0_w"] - }, - outputs={"Out": ["mul_0_out"]}, - x_num_col_dims=2, - y_num_col_dims=1) - mul_1 = OpConfig("mul", - inputs={ - "X": ["mul_x"], - "Y": ["mul_1_w"] - }, - outputs={"Out": ["mul_1_out"]}, - x_num_col_dims=2, - y_num_col_dims=1) - mul_2 = OpConfig("mul", - inputs={ - "X": ["mul_x"], - "Y": ["mul_2_w"] - }, - outputs={"Out": ["mul_2_out"]}, - x_num_col_dims=2, - y_num_col_dims=1) - ele_0 = OpConfig("elementwise_add", - inputs={ - "X": [mul_0.outputs["Out"][0]], - "Y": ["ele_0_w"] - }, - outputs={"Out": ["ele_0_out"]}, - axis=-1) - ele_1 = OpConfig("elementwise_add", - inputs={ - "X": [mul_1.outputs["Out"][0]], - "Y": ["ele_1_w"] - }, - outputs={"Out": ["ele_1_out"]}, - axis=-1) - ele_2 = OpConfig("elementwise_add", - inputs={ - "X": [mul_2.outputs["Out"][0]], - "Y": ["ele_2_w"] - }, - outputs={"Out": ["ele_2_out"]}, - axis=-1) - reshape_0 = OpConfig("reshape2", - inputs={"X": [ele_0.outputs["Out"][0]]}, - outputs={ - "Out": ["reshape_0_out"], - "XShape": ["reshape_0_Xout"] - }, - shape=(1, 128, 12, 64)) - reshape_1 = OpConfig("reshape2", - inputs={"X": [ele_1.outputs["Out"][0]]}, - outputs={ - "Out": ["reshape_1_out"], - "XShape": ["reshape_1_Xout"] - }, - shape=(1, 128, 12, 64)) - reshape_2 = OpConfig("reshape2", - inputs={"X": [ele_2.outputs["Out"][0]]}, - outputs={ - "Out": ["reshape_2_out"], - "XShape": ["reshape_2_Xout"] - }, - shape=(1, 128, 12, 64)) - transpose_0 = OpConfig("transpose2", - inputs={"X": [reshape_0.outputs["Out"][0]]}, - outputs={"Out": ["transpose_0_out"]}, - axis=(0, 2, 1, 3)) - transpose_1 = OpConfig("transpose2", - inputs={"X": [reshape_1.outputs["Out"][0]]}, - outputs={"Out": ["transpose_1_out"]}, - axis=(0, 2, 3, 1)) - transpose_2 = OpConfig("transpose2", - inputs={"X": [reshape_2.outputs["Out"][0]]}, - outputs={"Out": ["transpose_2_out"]}, - axis=(0, 2, 1, 3)) - matmul_0 = OpConfig("matmul", - inputs={ - "X": [transpose_0.outputs["Out"][0]], - "Y": [transpose_1.outputs["Out"][0]] - }, - outputs={"Out": ["matmul_0_out"]}, - alpha=0.125, - transpose_X=False, - transpose_Y=False, - fused_reshape_Out=[], - fused_reshape_X=[], - fused_reshape_Y=[], - fused_transpose_Out=[], - fused_transpose_X=[], - fused_transpose_Y=[]) - ele_3 = OpConfig("elementwise_add", - inputs={ - "X": [matmul_0.outputs["Out"][0]], - "Y": ["eltadd_qk_b_var"] - }, - outputs={"Out": ["ele_3_out"]}, - axis=-1) - softmax_op = OpConfig("softmax", - inputs={"X": [ele_3.outputs["Out"][0]]}, - outputs={"Out": ["softmax_out"]}, - axis=3, - is_test=True) - matmul_1 = OpConfig("matmul", - inputs={ - "X": [softmax_op.outputs["Out"][0]], - "Y": [transpose_2.outputs["Out"][0]] - }, - outputs={"Out": ["matmul_1_out"]}, - alpha=1.0, - transpose_X=False, - transpose_Y=False, - fused_reshape_Out=[], - fused_reshape_X=[], - fused_reshape_Y=[], - fused_transpose_Out=[], - fused_transpose_X=[], - fused_transpose_Y=[]) - transpose_3 = OpConfig("transpose2", - inputs={"X": [matmul_1.outputs["Out"][0]]}, - outputs={"Out": ["transpose_3_out"]}, - axis=(0, 2, 1, 3)) - reshape_3 = OpConfig("reshape2", - inputs={"X": [transpose_3.outputs["Out"][0]]}, - outputs={ - "Out": ["reshape_3_out"], - "XShape": ["reshape_3_Xout"] - }, - shape=(1, 128, 768)) - mul_3 = OpConfig("mul", - inputs={ - "X": [reshape_3.outputs["Out"][0]], - "Y": ["mul_3_w"] - }, - outputs={"Out": ["mul_3_out"]}, - x_num_col_dims=2, - y_num_col_dims=1) + mul_0 = OpConfig( + "mul", + inputs={"X": ["mul_x"], "Y": ["mul_0_w"]}, + outputs={"Out": ["mul_0_out"]}, + x_num_col_dims=2, + y_num_col_dims=1, + ) + mul_1 = OpConfig( + "mul", + inputs={"X": ["mul_x"], "Y": ["mul_1_w"]}, + outputs={"Out": ["mul_1_out"]}, + x_num_col_dims=2, + y_num_col_dims=1, + ) + mul_2 = OpConfig( + "mul", + inputs={"X": ["mul_x"], "Y": ["mul_2_w"]}, + outputs={"Out": ["mul_2_out"]}, + x_num_col_dims=2, + y_num_col_dims=1, + ) + ele_0 = OpConfig( + "elementwise_add", + inputs={"X": [mul_0.outputs["Out"][0]], "Y": ["ele_0_w"]}, + outputs={"Out": ["ele_0_out"]}, + axis=-1, + ) + ele_1 = OpConfig( + "elementwise_add", + inputs={"X": [mul_1.outputs["Out"][0]], "Y": ["ele_1_w"]}, + outputs={"Out": ["ele_1_out"]}, + axis=-1, + ) + ele_2 = OpConfig( + "elementwise_add", + inputs={"X": [mul_2.outputs["Out"][0]], "Y": ["ele_2_w"]}, + outputs={"Out": ["ele_2_out"]}, + axis=-1, + ) + reshape_0 = OpConfig( + "reshape2", + inputs={"X": [ele_0.outputs["Out"][0]]}, + outputs={"Out": ["reshape_0_out"], "XShape": ["reshape_0_Xout"]}, + shape=(1, 128, 12, 64), + ) + reshape_1 = OpConfig( + "reshape2", + inputs={"X": [ele_1.outputs["Out"][0]]}, + outputs={"Out": ["reshape_1_out"], "XShape": ["reshape_1_Xout"]}, + shape=(1, 128, 12, 64), + ) + reshape_2 = OpConfig( + "reshape2", + inputs={"X": [ele_2.outputs["Out"][0]]}, + outputs={"Out": ["reshape_2_out"], "XShape": ["reshape_2_Xout"]}, + shape=(1, 128, 12, 64), + ) + transpose_0 = OpConfig( + "transpose2", + inputs={"X": [reshape_0.outputs["Out"][0]]}, + outputs={"Out": ["transpose_0_out"]}, + axis=(0, 2, 1, 3), + ) + transpose_1 = OpConfig( + "transpose2", + inputs={"X": [reshape_1.outputs["Out"][0]]}, + outputs={"Out": ["transpose_1_out"]}, + axis=(0, 2, 3, 1), + ) + transpose_2 = OpConfig( + "transpose2", + inputs={"X": [reshape_2.outputs["Out"][0]]}, + outputs={"Out": ["transpose_2_out"]}, + axis=(0, 2, 1, 3), + ) + matmul_0 = OpConfig( + "matmul", + inputs={ + "X": [transpose_0.outputs["Out"][0]], + "Y": [transpose_1.outputs["Out"][0]], + }, + outputs={"Out": ["matmul_0_out"]}, + alpha=0.125, + transpose_X=False, + transpose_Y=False, + fused_reshape_Out=[], + fused_reshape_X=[], + fused_reshape_Y=[], + fused_transpose_Out=[], + fused_transpose_X=[], + fused_transpose_Y=[], + ) + ele_3 = OpConfig( + "elementwise_add", + inputs={ + "X": [matmul_0.outputs["Out"][0]], + "Y": ["eltadd_qk_b_var"], + }, + outputs={"Out": ["ele_3_out"]}, + axis=-1, + ) + softmax_op = OpConfig( + "softmax", + inputs={"X": [ele_3.outputs["Out"][0]]}, + outputs={"Out": ["softmax_out"]}, + axis=3, + is_test=True, + ) + matmul_1 = OpConfig( + "matmul", + inputs={ + "X": [softmax_op.outputs["Out"][0]], + "Y": [transpose_2.outputs["Out"][0]], + }, + outputs={"Out": ["matmul_1_out"]}, + alpha=1.0, + transpose_X=False, + transpose_Y=False, + fused_reshape_Out=[], + fused_reshape_X=[], + fused_reshape_Y=[], + fused_transpose_Out=[], + fused_transpose_X=[], + fused_transpose_Y=[], + ) + transpose_3 = OpConfig( + "transpose2", + inputs={"X": [matmul_1.outputs["Out"][0]]}, + outputs={"Out": ["transpose_3_out"]}, + axis=(0, 2, 1, 3), + ) + reshape_3 = OpConfig( + "reshape2", + inputs={"X": [transpose_3.outputs["Out"][0]]}, + outputs={"Out": ["reshape_3_out"], "XShape": ["reshape_3_Xout"]}, + shape=(1, 128, 768), + ) + mul_3 = OpConfig( + "mul", + inputs={"X": [reshape_3.outputs["Out"][0]], "Y": ["mul_3_w"]}, + outputs={"Out": ["mul_3_out"]}, + x_num_col_dims=2, + y_num_col_dims=1, + ) ops = [ - mul_0, mul_1, mul_2, ele_0, ele_1, ele_2, reshape_0, reshape_1, - reshape_2, transpose_0, transpose_1, transpose_2, matmul_0, ele_3, - softmax_op, matmul_1, transpose_3, reshape_3, mul_3 + mul_0, + mul_1, + mul_2, + ele_0, + ele_1, + ele_2, + reshape_0, + reshape_1, + reshape_2, + transpose_0, + transpose_1, + transpose_2, + matmul_0, + ele_3, + softmax_op, + matmul_1, + transpose_3, + reshape_3, + mul_3, ] program_config = ProgramConfig( ops=ops, inputs={ - "mul_x": - TensorConfig(data_gen=partial(generate_mul_input)), - "eltadd_qk_b_var": - TensorConfig(data_gen=partial(generate_elewise_input)) + "mul_x": TensorConfig(data_gen=partial(generate_mul_input)), + "eltadd_qk_b_var": TensorConfig( + data_gen=partial(generate_elewise_input) + ), }, weights={ "mul_0_w": TensorConfig(shape=[768, 768]), @@ -193,16 +212,19 @@ def generate_elewise_input(): "mul_3_w": TensorConfig(shape=[768, 768]), "ele_0_w": TensorConfig(shape=[768]), "ele_1_w": TensorConfig(shape=[768]), - "ele_2_w": TensorConfig(shape=[768]) + "ele_2_w": TensorConfig(shape=[768]), }, - outputs=[ops[-1].outputs["Out"][0]]) + outputs=[ops[-1].outputs["Out"][0]], + ) return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=100, - min_success_num=1, - passes=["multihead_matmul_fuse_pass_v3"]) + self.run_and_statis( + quant=False, + max_examples=100, + min_success_num=1, + passes=["multihead_matmul_fuse_pass_v3"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_onednn_fc_activation_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_onednn_fc_activation_fuse_pass.py index 4e798a0ed57d5..770aac84ec0d3 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_onednn_fc_activation_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_onednn_fc_activation_fuse_pass.py @@ -21,32 +21,46 @@ class TestFCActivationOneDNNFusePass(PassAutoScanTest): - def sample_program_config(self, draw): fc_in = draw(st.sampled_from([32, 64])) fc_wei = draw(st.sampled_from([64])) activation_type = draw( - st.sampled_from([ - 'relu', 'gelu', 'swish', 'mish', 'sqrt', 'hard_swish', - 'sigmoid', 'abs', 'relu6', 'clip', 'tanh', 'hard_sigmoid', - 'leaky_relu' - ])) + st.sampled_from( + [ + 'relu', + 'gelu', + 'swish', + 'mish', + 'sqrt', + 'hard_swish', + 'sigmoid', + 'abs', + 'relu6', + 'clip', + 'tanh', + 'hard_sigmoid', + 'leaky_relu', + ] + ) + ) def generate_input(shape): return np.random.random(shape).astype(np.float32) - fc_op = OpConfig(type="fc", - inputs={ - "Input": ["fc_input"], - "W": ["fc_weight"], - "Bias": ["fc_bias"] - }, - outputs={"Out": ["fc_output"]}, - attrs={ - "use_mkldnn": True, - "padding_weights": False, - "in_num_col_dims": 1, - }) + fc_op = OpConfig( + type="fc", + inputs={ + "Input": ["fc_input"], + "W": ["fc_weight"], + "Bias": ["fc_bias"], + }, + outputs={"Out": ["fc_output"]}, + attrs={ + "use_mkldnn": True, + "padding_weights": False, + "in_num_col_dims": 1, + }, + ) if activation_type == "clip": activation_op = OpConfig( @@ -54,58 +68,69 @@ def generate_input(shape): inputs={"X": ["fc_output"]}, outputs={"Out": ["activation_output"]}, min=draw(st.floats(min_value=0.1, max_value=0.49)), - max=draw(st.floats(min_value=0.5, max_value=1.0))) + max=draw(st.floats(min_value=0.5, max_value=1.0)), + ) elif activation_type == "gelu": - activation_op = OpConfig(activation_type, - inputs={"X": ["fc_output"]}, - outputs={"Out": ["activation_output"]}, - approximate=draw(st.booleans())) + activation_op = OpConfig( + activation_type, + inputs={"X": ["fc_output"]}, + outputs={"Out": ["activation_output"]}, + approximate=draw(st.booleans()), + ) elif activation_type == "leaky_relu": - activation_op = OpConfig(activation_type, - inputs={"X": ["fc_output"]}, - outputs={"Out": ["activation_output"]}, - alpha=draw( - st.floats(min_value=0.1, - max_value=1.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["fc_output"]}, + outputs={"Out": ["activation_output"]}, + alpha=draw(st.floats(min_value=0.1, max_value=1.0)), + ) elif activation_type == "relu6": - activation_op = OpConfig(activation_type, - inputs={"X": ["fc_output"]}, - outputs={"Out": ["activation_output"]}, - threshold=6) + activation_op = OpConfig( + activation_type, + inputs={"X": ["fc_output"]}, + outputs={"Out": ["activation_output"]}, + threshold=6, + ) elif activation_type == "swish": - activation_op = OpConfig(activation_type, - inputs={"X": ["fc_output"]}, - outputs={"Out": ["activation_output"]}, - beta=draw( - st.floats(min_value=0.1, - max_value=10.0))) + activation_op = OpConfig( + activation_type, + inputs={"X": ["fc_output"]}, + outputs={"Out": ["activation_output"]}, + beta=draw(st.floats(min_value=0.1, max_value=10.0)), + ) else: - activation_op = OpConfig(activation_type, - inputs={"X": ["fc_output"]}, - outputs={"Out": ["activation_output"]}) + activation_op = OpConfig( + activation_type, + inputs={"X": ["fc_output"]}, + outputs={"Out": ["activation_output"]}, + ) model_net = [fc_op, activation_op] program_config = ProgramConfig( ops=model_net, weights={ - "fc_weight": - TensorConfig( - data_gen=partial(generate_input, [fc_wei, fc_wei])), - "fc_bias": - TensorConfig(data_gen=partial(generate_input, [fc_wei])), + "fc_weight": TensorConfig( + data_gen=partial(generate_input, [fc_wei, fc_wei]) + ), + "fc_bias": TensorConfig( + data_gen=partial(generate_input, [fc_wei]) + ), }, inputs={ - "fc_input": - TensorConfig(data_gen=partial(generate_input, [fc_in, fc_wei])) + "fc_input": TensorConfig( + data_gen=partial(generate_input, [fc_in, fc_wei]) + ) }, - outputs=["activation_output"]) + outputs=["activation_output"], + ) return program_config def sample_predictor_configs(self, program_config): config = self.create_inference_config( - use_mkldnn=True, passes=["fc_act_mkldnn_fuse_pass"]) + use_mkldnn=True, passes=["fc_act_mkldnn_fuse_pass"] + ) yield config, ["fc"], (1e-5, 1e-5) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_repeated_fc_relu_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_repeated_fc_relu_fuse_pass.py index f0cb8797f4ef0..1334407cfaf10 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_repeated_fc_relu_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_repeated_fc_relu_fuse_pass.py @@ -22,7 +22,6 @@ class TestRepeatedFcReluFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -39,82 +38,77 @@ def generate_input(): def generate_weight(shape): return np.random.random(shape).astype(np.float32) - attrs = [{ - "x_col": x_col, - "y_col": y_col - }, { - "axis": axis - }, { - 'batch_size': batch_size, - 'dim': dim - }] - - mul_op1 = OpConfig(type="mul", - inputs={ - "X": ["input_data"], - "Y": ["mul1_weight"] - }, - outputs={"Out": ["mul1_output"]}, - attrs={ - "x_num_col_dims": x_col, - "y_num_col_dims": y_col - }) - - elt_op1 = OpConfig(type="elementwise_add", - inputs={ - "X": ["mul1_output"], - "Y": ["elementwise1_weight"] - }, - outputs={"Out": ["elementwise1_output"]}, - attrs={"axis": axis}) - - relu_op1 = OpConfig(type="relu", - inputs={"X": ["elementwise1_output"]}, - outputs={"Out": ["relu1_output"]}, - attrs={}) - - mul_op2 = OpConfig(type="mul", - inputs={ - "X": ["relu1_output"], - "Y": ["mul2_weight"] - }, - outputs={"Out": ["mul2_output"]}, - attrs={ - "x_num_col_dims": x_col, - "y_num_col_dims": y_col - }) - - elt_op2 = OpConfig(type="elementwise_add", - inputs={ - "X": ["mul2_output"], - "Y": ["elementwise2_weight"] - }, - outputs={"Out": ["elementwise2_output"]}, - attrs={"axis": axis}) - - relu_op2 = OpConfig(type="relu", - inputs={"X": ["elementwise2_output"]}, - outputs={"Out": ["relu2_output"]}, - attrs={}) + attrs = [ + {"x_col": x_col, "y_col": y_col}, + {"axis": axis}, + {'batch_size': batch_size, 'dim': dim}, + ] + + mul_op1 = OpConfig( + type="mul", + inputs={"X": ["input_data"], "Y": ["mul1_weight"]}, + outputs={"Out": ["mul1_output"]}, + attrs={"x_num_col_dims": x_col, "y_num_col_dims": y_col}, + ) + + elt_op1 = OpConfig( + type="elementwise_add", + inputs={"X": ["mul1_output"], "Y": ["elementwise1_weight"]}, + outputs={"Out": ["elementwise1_output"]}, + attrs={"axis": axis}, + ) + + relu_op1 = OpConfig( + type="relu", + inputs={"X": ["elementwise1_output"]}, + outputs={"Out": ["relu1_output"]}, + attrs={}, + ) + + mul_op2 = OpConfig( + type="mul", + inputs={"X": ["relu1_output"], "Y": ["mul2_weight"]}, + outputs={"Out": ["mul2_output"]}, + attrs={"x_num_col_dims": x_col, "y_num_col_dims": y_col}, + ) + + elt_op2 = OpConfig( + type="elementwise_add", + inputs={"X": ["mul2_output"], "Y": ["elementwise2_weight"]}, + outputs={"Out": ["elementwise2_output"]}, + attrs={"axis": axis}, + ) + + relu_op2 = OpConfig( + type="relu", + inputs={"X": ["elementwise2_output"]}, + outputs={"Out": ["relu2_output"]}, + attrs={}, + ) model_net = [mul_op1, elt_op1, relu_op1, mul_op2, elt_op2, relu_op2] program_config = ProgramConfig( ops=model_net, weights={ - "mul1_weight": - TensorConfig(data_gen=partial(generate_weight, [dim, 32])), - "mul2_weight": - TensorConfig(data_gen=partial(generate_weight, [32, 128])), - "elementwise1_weight": - TensorConfig(data_gen=partial(generate_weight, [32])), - "elementwise2_weight": - TensorConfig(data_gen=partial(generate_weight, [128])) + "mul1_weight": TensorConfig( + data_gen=partial(generate_weight, [dim, 32]) + ), + "mul2_weight": TensorConfig( + data_gen=partial(generate_weight, [32, 128]) + ), + "elementwise1_weight": TensorConfig( + data_gen=partial(generate_weight, [32]) + ), + "elementwise2_weight": TensorConfig( + data_gen=partial(generate_weight, [128]) + ), }, inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)), }, - outputs=["relu2_output"]) + outputs=["relu2_output"], + ) return program_config diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_reshape2_matmul_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_reshape2_matmul_fuse_pass.py index 7de2491e69371..8efe56ed222fc 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_reshape2_matmul_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_reshape2_matmul_fuse_pass.py @@ -44,9 +44,10 @@ def sample_predictor_configs(self, program_config): def sample_program_config(self, draw): # 1. Generate shape and attr of reshape2 reshape = draw( - st.lists(st.integers(min_value=1, max_value=10), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=10), min_size=2, max_size=2 + ) + ) x_shape = reshape + [1, 1] # 2. Generate attr:transpose_X/transpose_Y/alpha of matmul @@ -56,9 +57,10 @@ def sample_program_config(self, draw): # 3. Generate legal shape of input:Y of matmul y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = x_shape[1] # 4. Generate legal attr:axis of elementwise_add @@ -91,17 +93,11 @@ def sample_program_config(self, draw): "X": ["reshape2_x"], }, shape=reshape, - outputs={ - "Out": ["reshape2_out"], - "XShape": ["xshape"] - }, + outputs={"Out": ["reshape2_out"], "XShape": ["xshape"]}, ) matmul_op = OpConfig( "matmul", - inputs={ - "X": ["reshape2_out"], - "Y": ["matmul_y"] - }, + inputs={"X": ["reshape2_out"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, transpose_X=transpose_X, @@ -116,10 +112,7 @@ def sample_program_config(self, draw): add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["matmul_out"], - "Y": ["bias"] - }, + inputs={"X": ["matmul_out"], "Y": ["bias"]}, outputs={"Out": ["add_out"]}, axis=axis, ) @@ -152,10 +145,12 @@ def sample_program_config(self, draw): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=50, - max_duration=1000, - passes=["gpu_cpu_reshape2_matmul_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=50, + max_duration=1000, + passes=["gpu_cpu_reshape2_matmul_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_seq_concat_fc_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_seq_concat_fc_fuse_pass.py index 53dc65f84b5de..52153788abedf 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_seq_concat_fc_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_seq_concat_fc_fuse_pass.py @@ -22,7 +22,6 @@ class TestSeqConcatFcFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -44,83 +43,83 @@ def generate_input(shape): def generate_weight(shape): return np.random.random(shape).astype(np.float32) - sequence_expand_op1 = OpConfig(type="sequence_expand", - inputs={ - "X": ["input_data1"], - "Y": ["input_data2"] - }, - outputs={"Out": ["seq_exp1_out"]}, - attrs={"ref_level": ref_level}) - - sequence_expand_op2 = OpConfig(type="sequence_expand", - inputs={ - "X": ["input_data1"], - "Y": ["input_data3"] - }, - outputs={"Out": ["seq_exp2_out"]}, - attrs={"ref_level": ref_level}) + sequence_expand_op1 = OpConfig( + type="sequence_expand", + inputs={"X": ["input_data1"], "Y": ["input_data2"]}, + outputs={"Out": ["seq_exp1_out"]}, + attrs={"ref_level": ref_level}, + ) + + sequence_expand_op2 = OpConfig( + type="sequence_expand", + inputs={"X": ["input_data1"], "Y": ["input_data3"]}, + outputs={"Out": ["seq_exp2_out"]}, + attrs={"ref_level": ref_level}, + ) concat_op = OpConfig( type="concat", inputs={"X": ["input_data1", "seq_exp1_out", "seq_exp2_out"]}, outputs={"Out": ["concat_output"]}, - attrs={'axis': axis1}) - - mul_op = OpConfig(type="mul", - inputs={ - "X": ["concat_output"], - "Y": ["mul_weight"] - }, - outputs={"Out": ["mul_out"]}, - attrs={ - "x_num_col_dims": x_col, - "y_num_col_dims": y_col - }) - - elt_op = OpConfig(type="elementwise_add", - inputs={ - "X": ["mul_out"], - "Y": ["elt_weight"] - }, - outputs={"Out": ["elt_out"]}, - attrs={"axis": axis2}) - - act_op = OpConfig(type=act_type, - inputs={"X": ["elt_out"]}, - outputs={"Out": ["act_out"]}, - attrs={ - "use_cudnn": use_cudnn, - "use_mkldnn": use_mkldnn - }) + attrs={'axis': axis1}, + ) + + mul_op = OpConfig( + type="mul", + inputs={"X": ["concat_output"], "Y": ["mul_weight"]}, + outputs={"Out": ["mul_out"]}, + attrs={"x_num_col_dims": x_col, "y_num_col_dims": y_col}, + ) + + elt_op = OpConfig( + type="elementwise_add", + inputs={"X": ["mul_out"], "Y": ["elt_weight"]}, + outputs={"Out": ["elt_out"]}, + attrs={"axis": axis2}, + ) + + act_op = OpConfig( + type=act_type, + inputs={"X": ["elt_out"]}, + outputs={"Out": ["act_out"]}, + attrs={"use_cudnn": use_cudnn, "use_mkldnn": use_mkldnn}, + ) model_net = [ - sequence_expand_op1, sequence_expand_op2, concat_op, mul_op, elt_op, - act_op + sequence_expand_op1, + sequence_expand_op2, + concat_op, + mul_op, + elt_op, + act_op, ] program_config = ProgramConfig( ops=model_net, weights={ - "mul_weight": - TensorConfig(data_gen=partial(generate_weight, [384, dim])), - "elt_weight": - TensorConfig(data_gen=partial(generate_weight, [dim])) + "mul_weight": TensorConfig( + data_gen=partial(generate_weight, [384, dim]) + ), + "elt_weight": TensorConfig( + data_gen=partial(generate_weight, [dim]) + ), }, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input, - [batch_size, 128]), - lod=[[0, 1]]), - "input_data2": - TensorConfig(data_gen=partial(generate_input, - [batch_size, 128]), - lod=[[0, 1]]), - "input_data3": - TensorConfig(data_gen=partial(generate_input, - [batch_size, 128]), - lod=[[0, 1]]) + "input_data1": TensorConfig( + data_gen=partial(generate_input, [batch_size, 128]), + lod=[[0, 1]], + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, [batch_size, 128]), + lod=[[0, 1]], + ), + "input_data3": TensorConfig( + data_gen=partial(generate_input, [batch_size, 128]), + lod=[[0, 1]], + ), }, - outputs=["act_out"]) + outputs=["act_out"], + ) return program_config @@ -129,15 +128,15 @@ def sample_predictor_configs(self, program_config): yield config, ["fusion_seqexpand_concat_fc"], (1e-5, 1e-5) def add_ignore_pass_case(self): - def teller1(program_config, predictor_config): if program_config.ops[-1].type == "relu": return True return False self.add_ignore_check_case( - teller1, IgnoreReasons.PASS_ACCURACY_ERROR, - "The pass output has diff in a specific case. We need to fix it as soon as possible." + teller1, + IgnoreReasons.PASS_ACCURACY_ERROR, + "The pass output has diff in a specific case. We need to fix it as soon as possible.", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_seqconv_eltadd_relu_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_seqconv_eltadd_relu_fuse_pass.py index ee5bb892b942e..08c95df169bac 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_seqconv_eltadd_relu_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_seqconv_eltadd_relu_fuse_pass.py @@ -22,7 +22,6 @@ class TestSeqconvEltaddReluFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -41,71 +40,80 @@ def generate_input(): def generate_weight(shape): return np.random.random(shape).astype(np.float32) - im2sequence_op = OpConfig(type="im2sequence", - inputs={"X": ["input_data"]}, - outputs={"Out": ["seq_out"]}, - attrs={ - "kernels": [6, 1], - "out_stride": [1, 1], - "paddings": [0, 0, 0, 0], - "strides": [1, 1] - }) - - sequence_conv_op = OpConfig(type="sequence_conv", - inputs={ - "X": ["seq_out"], - "Filter": ["conv_weight"] - }, - outputs={"Out": ["conv_out"]}, - attrs={ - "contextLength": contextLength, - "contextStart": contextStart, - "contextStride": contextStride, - "paddingTrainable": paddingTrainable - }) - - elementwise_add_op = OpConfig(type="elementwise_add", - inputs={ - "X": ["conv_out"], - "Y": ["elt_weight"] - }, - outputs={"Out": ["elt_output"]}, - attrs={'axis': axis}) - - relu_op = OpConfig(type="relu", - inputs={"X": ["elt_output"]}, - outputs={"Out": ["relu_output"]}, - attrs={}) + im2sequence_op = OpConfig( + type="im2sequence", + inputs={"X": ["input_data"]}, + outputs={"Out": ["seq_out"]}, + attrs={ + "kernels": [6, 1], + "out_stride": [1, 1], + "paddings": [0, 0, 0, 0], + "strides": [1, 1], + }, + ) + + sequence_conv_op = OpConfig( + type="sequence_conv", + inputs={"X": ["seq_out"], "Filter": ["conv_weight"]}, + outputs={"Out": ["conv_out"]}, + attrs={ + "contextLength": contextLength, + "contextStart": contextStart, + "contextStride": contextStride, + "paddingTrainable": paddingTrainable, + }, + ) + + elementwise_add_op = OpConfig( + type="elementwise_add", + inputs={"X": ["conv_out"], "Y": ["elt_weight"]}, + outputs={"Out": ["elt_output"]}, + attrs={'axis': axis}, + ) + + relu_op = OpConfig( + type="relu", + inputs={"X": ["elt_output"]}, + outputs={"Out": ["relu_output"]}, + attrs={}, + ) model_net = [ - im2sequence_op, sequence_conv_op, elementwise_add_op, relu_op + im2sequence_op, + sequence_conv_op, + elementwise_add_op, + relu_op, ] program_config = ProgramConfig( ops=model_net, weights={ - "conv_weight": - TensorConfig( - data_gen=partial(generate_weight, [768 * - contextLength, 16])), - "elt_weight": - TensorConfig(data_gen=partial(generate_weight, [16])) + "conv_weight": TensorConfig( + data_gen=partial(generate_weight, [768 * contextLength, 16]) + ), + "elt_weight": TensorConfig( + data_gen=partial(generate_weight, [16]) + ), }, inputs={ "input_data": TensorConfig(data_gen=partial(generate_input)) }, - outputs=["relu_output"]) + outputs=["relu_output"], + ) return program_config def sample_predictor_configs(self, program_config): config = self.create_inference_config() - yield config, ["im2sequence", - "fusion_seqconv_eltadd_relu"], (1e-5, 1e-5) + yield config, ["im2sequence", "fusion_seqconv_eltadd_relu"], ( + 1e-5, + 1e-5, + ) def test(self): - self.run_and_statis(quant=False, - passes=["seqconv_eltadd_relu_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["seqconv_eltadd_relu_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_seqpool_cvm_concat_fuse_pass_py.py b/python/paddle/fluid/tests/unittests/ir/inference/test_seqpool_cvm_concat_fuse_pass_py.py index d768b1c23ff51..08878c3cd40aa 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_seqpool_cvm_concat_fuse_pass_py.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_seqpool_cvm_concat_fuse_pass_py.py @@ -22,7 +22,6 @@ class TestSeqpoolCvmConcatFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -47,84 +46,88 @@ def generate_input2(): def generate_input3(): return np.random.random([1, 768]).astype(np.float32) - im2sequence_op = OpConfig(type="im2sequence", - inputs={"X": ["input_data1"]}, - outputs={"Out": ["seq_out"]}, - attrs={ - "kernels": [6, 1], - "out_stride": [1, 1], - "paddings": [0, 0, 0, 0], - "strides": [1, 1] - }) - - sequence_pool_op1 = OpConfig(type="sequence_pool", - inputs={"X": ["seq_out"]}, - outputs={ - "Out": ["seq_pool1_out"], - "MaxIndex": ["index1_out"] - }, - attrs={ - "is_test": is_test, - "pooltype": pooltype, - "pad_value": pad_value1 - }) - - sequence_pool_op2 = OpConfig(type="sequence_pool", - inputs={"X": ["seq_out"]}, - outputs={ - "Out": ["seq_pool2_out"], - "MaxIndex": ["index2_out"] - }, - attrs={ - "is_test": is_test, - "pooltype": pooltype, - "pad_value": pad_value2 - }) - - sequence_pool_op3 = OpConfig(type="sequence_pool", - inputs={"X": ["seq_out"]}, - outputs={ - "Out": ["seq_pool3_out"], - "MaxIndex": ["index3_out"] - }, - attrs={ - "is_test": is_test, - "pooltype": pooltype, - "pad_value": pad_value3 - }) - - cvm_op1 = OpConfig(type="cvm", - inputs={ - "X": ["seq_pool1_out"], - "CVM": ["input_data2"] - }, - outputs={"Y": ["cvm1_out"]}, - attrs={"use_cvm": use_cvm}) - - cvm_op2 = OpConfig(type="cvm", - inputs={ - "X": ["seq_pool2_out"], - "CVM": ["input_data2"] - }, - outputs={"Y": ["cvm2_out"]}, - attrs={"use_cvm": use_cvm}) - - cvm_op3 = OpConfig(type="cvm", - inputs={ - "X": ["seq_pool3_out"], - "CVM": ["input_data2"] - }, - outputs={"Y": ["cvm3_out"]}, - attrs={"use_cvm": use_cvm}) - - concat_op = OpConfig(type="concat", - inputs={"X": ["cvm1_out", "cvm2_out", "cvm3_out"]}, - outputs={"Out": ["concat_output"]}, - attrs={'axis': axis}) + im2sequence_op = OpConfig( + type="im2sequence", + inputs={"X": ["input_data1"]}, + outputs={"Out": ["seq_out"]}, + attrs={ + "kernels": [6, 1], + "out_stride": [1, 1], + "paddings": [0, 0, 0, 0], + "strides": [1, 1], + }, + ) + + sequence_pool_op1 = OpConfig( + type="sequence_pool", + inputs={"X": ["seq_out"]}, + outputs={"Out": ["seq_pool1_out"], "MaxIndex": ["index1_out"]}, + attrs={ + "is_test": is_test, + "pooltype": pooltype, + "pad_value": pad_value1, + }, + ) + + sequence_pool_op2 = OpConfig( + type="sequence_pool", + inputs={"X": ["seq_out"]}, + outputs={"Out": ["seq_pool2_out"], "MaxIndex": ["index2_out"]}, + attrs={ + "is_test": is_test, + "pooltype": pooltype, + "pad_value": pad_value2, + }, + ) + + sequence_pool_op3 = OpConfig( + type="sequence_pool", + inputs={"X": ["seq_out"]}, + outputs={"Out": ["seq_pool3_out"], "MaxIndex": ["index3_out"]}, + attrs={ + "is_test": is_test, + "pooltype": pooltype, + "pad_value": pad_value3, + }, + ) + + cvm_op1 = OpConfig( + type="cvm", + inputs={"X": ["seq_pool1_out"], "CVM": ["input_data2"]}, + outputs={"Y": ["cvm1_out"]}, + attrs={"use_cvm": use_cvm}, + ) + + cvm_op2 = OpConfig( + type="cvm", + inputs={"X": ["seq_pool2_out"], "CVM": ["input_data2"]}, + outputs={"Y": ["cvm2_out"]}, + attrs={"use_cvm": use_cvm}, + ) + + cvm_op3 = OpConfig( + type="cvm", + inputs={"X": ["seq_pool3_out"], "CVM": ["input_data2"]}, + outputs={"Y": ["cvm3_out"]}, + attrs={"use_cvm": use_cvm}, + ) + + concat_op = OpConfig( + type="concat", + inputs={"X": ["cvm1_out", "cvm2_out", "cvm3_out"]}, + outputs={"Out": ["concat_output"]}, + attrs={'axis': axis}, + ) model_net = [ - im2sequence_op, sequence_pool_op1, sequence_pool_op2, - sequence_pool_op3, cvm_op1, cvm_op2, cvm_op3, concat_op + im2sequence_op, + sequence_pool_op1, + sequence_pool_op2, + sequence_pool_op3, + cvm_op1, + cvm_op2, + cvm_op3, + concat_op, ] program_config = ProgramConfig( @@ -133,9 +136,10 @@ def generate_input3(): inputs={ "input_data1": TensorConfig(data_gen=partial(generate_input1)), "input_data2": TensorConfig(data_gen=partial(generate_input2)), - "input_data3": TensorConfig(data_gen=partial(generate_input3)) + "input_data3": TensorConfig(data_gen=partial(generate_input3)), }, - outputs=["concat_output"]) + outputs=["concat_output"], + ) return program_config @@ -144,8 +148,9 @@ def sample_predictor_configs(self, program_config): yield config, ["im2sequence", "fusion_seqpool_cvm_concat"], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - passes=["seqpool_cvm_concat_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["seqpool_cvm_concat_fuse_pass"] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_shuffle_channel_detect_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_shuffle_channel_detect_pass.py index 669b1906d2c10..5aa03a57f4bb3 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_shuffle_channel_detect_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_shuffle_channel_detect_pass.py @@ -23,7 +23,6 @@ class TestShuffleChannelDetectPass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -46,44 +45,52 @@ def sample_program_config(self, draw): def generate_reshape2_Input(): return np.random.random(x_shape).astype(np.float32) - reshape2_op1 = OpConfig("reshape2", - inputs={ - "X": ["reshape2_input1"], - }, - outputs={ - "Out": ["reshape2_output1"], - "XShape": ["reshape2_xshape1"] - }, - shape=shape, - input_shape=x_shape) - transpose2_op = OpConfig("transpose2", - inputs={ - "X": ["reshape2_output1"], - }, - outputs={ - "Out": ["transpose2_output"], - "XShape": ["transpose2_xshape"] - }, - axis=axis_v) - reshape2_op2 = OpConfig("reshape2", - inputs={ - "X": ["transpose2_output"], - }, - outputs={ - "Out": ["reshape2_output2"], - "XShape": ["reshape2_xshape2"] - }, - shape=x_shape) + reshape2_op1 = OpConfig( + "reshape2", + inputs={ + "X": ["reshape2_input1"], + }, + outputs={ + "Out": ["reshape2_output1"], + "XShape": ["reshape2_xshape1"], + }, + shape=shape, + input_shape=x_shape, + ) + transpose2_op = OpConfig( + "transpose2", + inputs={ + "X": ["reshape2_output1"], + }, + outputs={ + "Out": ["transpose2_output"], + "XShape": ["transpose2_xshape"], + }, + axis=axis_v, + ) + reshape2_op2 = OpConfig( + "reshape2", + inputs={ + "X": ["transpose2_output"], + }, + outputs={ + "Out": ["reshape2_output2"], + "XShape": ["reshape2_xshape2"], + }, + shape=x_shape, + ) ops = [reshape2_op1, transpose2_op, reshape2_op2] program_config = ProgramConfig( ops=ops, inputs={ - "reshape2_input1": - TensorConfig(data_gen=partial(generate_reshape2_Input)), + "reshape2_input1": TensorConfig( + data_gen=partial(generate_reshape2_Input) + ), }, weights={}, - outputs=["reshape2_output2"]) + outputs=["reshape2_output2"], + ) return program_config def sample_predictor_configs(self, program_config): @@ -94,7 +101,8 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=1, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['shuffle_channel'], (1e-5, 1e-5) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_simplify_with_basic_ops_pass_autoscan.py b/python/paddle/fluid/tests/unittests/ir/inference/test_simplify_with_basic_ops_pass_autoscan.py index 6dab206551c18..42c5a38a72af7 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_simplify_with_basic_ops_pass_autoscan.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_simplify_with_basic_ops_pass_autoscan.py @@ -21,47 +21,49 @@ class TestSimplifyWithBasicOpsPassUpscale(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_config(self, draw): - #scale = draw(st.floats(min_value=0.01, max_value=1.0)) - #bias = draw(st.floats(min_value=0.01, max_value=2.0)) - #bias_after_scale = draw(st.booleans()) + # scale = draw(st.floats(min_value=0.01, max_value=1.0)) + # bias = draw(st.floats(min_value=0.01, max_value=2.0)) + # bias_after_scale = draw(st.booleans()) fix_seed = draw(st.booleans()) dropout_implementation = "upscale_in_train" dropout_prob = draw(st.floats(min_value=0.0, max_value=1.0)) seed = draw(st.integers(min_value=0, max_value=512)) x_shape = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=4 + ) + ) is_test = True - dropout_op = OpConfig("dropout", - inputs={"X": ["input_data"]}, - outputs={ - "Out": ["dropout_output"], - "Mask": ["mask"] - }, - fix_seed=fix_seed, - dropout_implementation=dropout_implementation, - dropout_prob=dropout_prob, - seed=seed, - is_test=is_test) - relu_op = OpConfig("relu", - inputs={"X": ["dropout_output"]}, - outputs={"Out": ["relu_out"]}) + dropout_op = OpConfig( + "dropout", + inputs={"X": ["input_data"]}, + outputs={"Out": ["dropout_output"], "Mask": ["mask"]}, + fix_seed=fix_seed, + dropout_implementation=dropout_implementation, + dropout_prob=dropout_prob, + seed=seed, + is_test=is_test, + ) + relu_op = OpConfig( + "relu", + inputs={"X": ["dropout_output"]}, + outputs={"Out": ["relu_out"]}, + ) ops = [dropout_op, relu_op] - program_config = ProgramConfig(ops=ops, - weights={}, - inputs={ - "input_data": - TensorConfig(shape=x_shape), - }, - outputs=["relu_out"]) + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "input_data": TensorConfig(shape=x_shape), + }, + outputs=["relu_out"], + ) return program_config @@ -77,18 +79,20 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['relu'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_examples=30, - passes=["simplify_with_basic_ops_pass"], - min_success_num=30) + self.run_and_statis( + quant=False, + max_examples=30, + passes=["simplify_with_basic_ops_pass"], + min_success_num=30, + ) class TestSimplifyWithBasicOpsPassDowngrade(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -98,34 +102,37 @@ def sample_program_config(self, draw): dropout_prob = draw(st.floats(min_value=0.0, max_value=1.0)) seed = draw(st.integers(min_value=0, max_value=512)) x_shape = draw( - st.lists(st.integers(min_value=1, max_value=4), - min_size=2, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=4), min_size=2, max_size=4 + ) + ) is_test = True - dropout_op = OpConfig("dropout", - inputs={"X": ["input_data"]}, - outputs={ - "Out": ["dropout_output"], - "Mask": ["mask"] - }, - fix_seed=fix_seed, - dropout_implementation=dropout_implementation, - dropout_prob=dropout_prob, - seed=seed, - is_test=is_test) - relu_op = OpConfig("relu", - inputs={"X": ["dropout_output"]}, - outputs={"Out": ["relu_out"]}) + dropout_op = OpConfig( + "dropout", + inputs={"X": ["input_data"]}, + outputs={"Out": ["dropout_output"], "Mask": ["mask"]}, + fix_seed=fix_seed, + dropout_implementation=dropout_implementation, + dropout_prob=dropout_prob, + seed=seed, + is_test=is_test, + ) + relu_op = OpConfig( + "relu", + inputs={"X": ["dropout_output"]}, + outputs={"Out": ["relu_out"]}, + ) ops = [dropout_op, relu_op] - program_config = ProgramConfig(ops=ops, - weights={}, - inputs={ - "input_data": - TensorConfig(shape=x_shape), - }, - outputs=["relu_out"]) + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "input_data": TensorConfig(shape=x_shape), + }, + outputs=["relu_out"], + ) return program_config @@ -141,14 +148,17 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['scale', 'relu'], (1e-5, 1e-5) def test(self): - self.run_and_statis(quant=False, - max_examples=30, - passes=["simplify_with_basic_ops_pass"], - min_success_num=30) + self.run_and_statis( + quant=False, + max_examples=30, + passes=["simplify_with_basic_ops_pass"], + min_success_num=30, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_squared_mat_sub_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_squared_mat_sub_fuse_pass.py index 93a6e0fa32293..4e2287c9a65b8 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_squared_mat_sub_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_squared_mat_sub_fuse_pass.py @@ -22,7 +22,6 @@ class TestSquaredMatSubFusePass(PassAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -49,110 +48,125 @@ def generate_input(type): else: return np.random.random(shape_y).astype(np.float32) - matmul_op1 = OpConfig(type="matmul", - inputs={ - "X": ["input_data1"], - "Y": ["input_data2"] - }, - outputs={"Out": ["matmul1_output"]}, - attrs={ - "transpose_X": transpose_X, - "transpose_Y": transpose_Y, - "alpha": alpha1, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }) - - square_op1 = OpConfig(type="square", - inputs={"X": ["matmul1_output"]}, - outputs={"Out": ["square1_output"]}, - attrs={}) - - square_op2 = OpConfig(type="square", - inputs={"X": ["input_data1"]}, - outputs={"Out": ["square2_output"]}, - attrs={}) - - square_op3 = OpConfig(type="square", - inputs={"X": ["input_data2"]}, - outputs={"Out": ["square3_output"]}, - attrs={}) - - matmul_op2 = OpConfig(type="matmul", - inputs={ - "X": ["square2_output"], - "Y": ["square3_output"] - }, - outputs={"Out": ["matmul2_output"]}, - attrs={ - "transpose_X": transpose_X, - "transpose_Y": transpose_Y, - "alpha": alpha2, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }) - - elt_sub_op = OpConfig(type="elementwise_sub", - inputs={ - "X": ["square1_output"], - "Y": ["matmul2_output"] - }, - outputs={"Out": ["sub_out"]}, - attrs={"axis": axis1}) + matmul_op1 = OpConfig( + type="matmul", + inputs={"X": ["input_data1"], "Y": ["input_data2"]}, + outputs={"Out": ["matmul1_output"]}, + attrs={ + "transpose_X": transpose_X, + "transpose_Y": transpose_Y, + "alpha": alpha1, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + ) + + square_op1 = OpConfig( + type="square", + inputs={"X": ["matmul1_output"]}, + outputs={"Out": ["square1_output"]}, + attrs={}, + ) + + square_op2 = OpConfig( + type="square", + inputs={"X": ["input_data1"]}, + outputs={"Out": ["square2_output"]}, + attrs={}, + ) + + square_op3 = OpConfig( + type="square", + inputs={"X": ["input_data2"]}, + outputs={"Out": ["square3_output"]}, + attrs={}, + ) + + matmul_op2 = OpConfig( + type="matmul", + inputs={"X": ["square2_output"], "Y": ["square3_output"]}, + outputs={"Out": ["matmul2_output"]}, + attrs={ + "transpose_X": transpose_X, + "transpose_Y": transpose_Y, + "alpha": alpha2, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + ) + + elt_sub_op = OpConfig( + type="elementwise_sub", + inputs={"X": ["square1_output"], "Y": ["matmul2_output"]}, + outputs={"Out": ["sub_out"]}, + attrs={"axis": axis1}, + ) if has_str_value: - fill_constant_op = OpConfig(type="fill_constant", - inputs={}, - outputs={"Out": ["constant_out"]}, - attrs={ - "dtype": 5, - "place_type": place_type, - "str_value": str_value, - "value": value, - "shape": shape - }) + fill_constant_op = OpConfig( + type="fill_constant", + inputs={}, + outputs={"Out": ["constant_out"]}, + attrs={ + "dtype": 5, + "place_type": place_type, + "str_value": str_value, + "value": value, + "shape": shape, + }, + ) else: - fill_constant_op = OpConfig(type="fill_constant", - inputs={}, - outputs={"Out": ["constant_out"]}, - attrs={ - "dtype": 5, - "place_type": place_type, - "value": value, - "shape": shape - }) - - elt_mul_op = OpConfig(type="elementwise_mul", - inputs={ - "X": ["sub_out"], - "Y": ["constant_out"] - }, - outputs={"Out": ["mul_out"]}, - attrs={"axis": axis2}) + fill_constant_op = OpConfig( + type="fill_constant", + inputs={}, + outputs={"Out": ["constant_out"]}, + attrs={ + "dtype": 5, + "place_type": place_type, + "value": value, + "shape": shape, + }, + ) + + elt_mul_op = OpConfig( + type="elementwise_mul", + inputs={"X": ["sub_out"], "Y": ["constant_out"]}, + outputs={"Out": ["mul_out"]}, + attrs={"axis": axis2}, + ) model_net = [ - matmul_op1, square_op1, square_op2, square_op3, matmul_op2, - elt_sub_op, fill_constant_op, elt_mul_op + matmul_op1, + square_op1, + square_op2, + square_op3, + matmul_op2, + elt_sub_op, + fill_constant_op, + elt_mul_op, ] program_config = ProgramConfig( ops=model_net, weights={}, inputs={ - "input_data1": - TensorConfig(data_gen=partial(generate_input, "x")), - "input_data2": - TensorConfig(data_gen=partial(generate_input, "y")) + "input_data1": TensorConfig( + data_gen=partial(generate_input, "x") + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, "y") + ), }, - outputs=["mul_out"]) + outputs=["mul_out"], + ) return program_config diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_squeeze2_matmul_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_squeeze2_matmul_fuse_pass.py index 8deee7c1517e8..e7ef132f1abbd 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_squeeze2_matmul_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_squeeze2_matmul_fuse_pass.py @@ -44,9 +44,10 @@ def sample_predictor_configs(self, program_config): def sample_program_config(self, draw): # 1. Generate shape of input:X of squeeze2 x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) # axes of squeeze2 == [2, 3] x_shape += [1, 1] axes = [2, 3] @@ -58,9 +59,10 @@ def sample_program_config(self, draw): # 3. Generate legal shape of input:Y of matmul y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = x_shape[1] # 4. Generate legal attr:axis of elementwise_add @@ -93,17 +95,11 @@ def sample_program_config(self, draw): "X": ["squeeze2_x"], }, axes=axes, - outputs={ - "Out": ["squeeze2_out"], - "XShape": ["xshape"] - }, + outputs={"Out": ["squeeze2_out"], "XShape": ["xshape"]}, ) matmul_op = OpConfig( "matmul", - inputs={ - "X": ["squeeze2_out"], - "Y": ["matmul_y"] - }, + inputs={"X": ["squeeze2_out"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, transpose_X=transpose_X, @@ -118,10 +114,7 @@ def sample_program_config(self, draw): add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["matmul_out"], - "Y": ["bias"] - }, + inputs={"X": ["matmul_out"], "Y": ["bias"]}, outputs={"Out": ["add_out"]}, axis=axis, ) @@ -154,10 +147,12 @@ def sample_program_config(self, draw): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=50, - max_duration=1000, - passes=["gpu_cpu_squeeze2_matmul_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=50, + max_duration=1000, + passes=["gpu_cpu_squeeze2_matmul_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_transpose_flatten_concat_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_transpose_flatten_concat_fuse_pass.py index 41145be9bca0b..acdff3d9a86ce 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_transpose_flatten_concat_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_transpose_flatten_concat_fuse_pass.py @@ -94,14 +94,18 @@ def sample_program_config(self, draw): if draw(st.booleans()): trans_axis[j], trans_axis[-1] = trans_axis[-1], trans_axis[j] # Generate axis of flatten - flatten_axis = draw(st.integers(min_value=0, - max_value=x_shape_rank - 1)) + flatten_axis = draw( + st.integers(min_value=0, max_value=x_shape_rank - 1) + ) for i in range(times): # Generate x_shape of transpose x_shape = draw( - st.lists(st.integers(min_value=1, max_value=10), - min_size=x_shape_rank, - max_size=x_shape_rank)) + st.lists( + st.integers(min_value=1, max_value=10), + min_size=x_shape_rank, + max_size=x_shape_rank, + ) + ) str_i = str(i) transpose_op = OpConfig( @@ -112,7 +116,7 @@ def sample_program_config(self, draw): axis=trans_axis, outputs={ "Out": ["trans_out" + str_i], - "XShape": ["trans_shape" + str_i] + "XShape": ["trans_shape" + str_i], }, ) ops.append(transpose_op) @@ -124,7 +128,7 @@ def sample_program_config(self, draw): axis=flatten_axis, outputs={ "Out": ["flatten2_out" + str_i], - "XShape": ["xshape" + str_i] + "XShape": ["xshape" + str_i], }, ) concat_input.append("flatten2_out" + str_i) @@ -152,9 +156,11 @@ def sample_program_config(self, draw): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=300, - passes=["transpose_flatten_concat_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=300, + passes=["transpose_flatten_concat_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_activation_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_activation_pass.py index 7ba824360ad46..0f2a8a97430cd 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_activation_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_activation_pass.py @@ -24,18 +24,18 @@ class TensorRTSubgraphPassActivationTest(InferencePassTest): - def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) def setUp(self): self.setUpTensorRTParam() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 32, 32], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 32, 32], dtype="float32" + ) act_out = self.append_act(data) out = fluid.layers.batch_norm(act_out, is_test=True) self.feeds = { @@ -51,298 +51,340 @@ def test_check_output(self): use_gpu = True if os.path.exists(self.path + "_opt_cache"): shutil.rmtree(self.path + "_opt_cache") - if self.trt_parameters.precision == AnalysisConfig.Precision.Float32: + if ( + self.trt_parameters.precision + == AnalysisConfig.Precision.Float32 + ): self.check_output_with_option(use_gpu) else: self.check_output_with_option(use_gpu, 1e-3) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassLeakyReluTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.leaky_relu(x) class TensorRTSubgraphPassRelu6Test(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.relu6(x) class TensorRTSubgraphPassSoftMaxTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.softmax(x) class TensorRTSubgraphPassSigmoidTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.sigmoid(x) class TensorRTSubgraphPassHardSwishTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.hard_swish(x) class TensorRTSubgraphPassHardSigmoidTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.hard_sigmoid(x) -class TensorRTSubgraphPassHardSwishPluginTest(TensorRTSubgraphPassActivationTest - ): - +class TensorRTSubgraphPassHardSwishPluginTest( + TensorRTSubgraphPassActivationTest +): def append_act(self, x): return fluid.layers.hard_swish(x, threshold=4.0, scale=8.0) class TensorRTSubgraphPassClipTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.clip(x, 0, 1) class TensorRTSubgraphPassTanhTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.tanh(x) class TensorRTSubgraphPassSwishTest(TensorRTSubgraphPassActivationTest): - def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False + ) def append_act(self, x): return fluid.layers.swish(x) class TensorRTSubgraphPassSwishFp16SerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) def append_act(self, x): return fluid.layers.swish(x) class TensorRTSubgraphPassDynamicSwishFp16SerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.swish(x) class TensorRTSubgraphPassMishTest(TensorRTSubgraphPassActivationTest): - def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False + ) def append_act(self, x): return fluid.layers.mish(x) class TensorRTSubgraphPassMishFp16SerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) def append_act(self, x): return fluid.layers.mish(x) class TensorRTSubgraphPassDynamicMishFp16SerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.mish(x) class TensorRTSubgraphPassPreluAllTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.prelu(x, mode='all') class TensorRTSubgraphPassPreluChannelTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.prelu(x, mode='channel') class TensorRTSubgraphPassPreluElementTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.prelu(x, mode='element') class TensorRTSubgraphPassPreluDynamicTest(TensorRTSubgraphPassActivationTest): - def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.prelu(x, mode='all') class TensorRTSubgraphPassPreluFp16Test(TensorRTSubgraphPassActivationTest): - def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False + ) def append_act(self, x): return fluid.layers.prelu(x, mode='all') class TensorRTSubgraphPassPreluFp16SerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) def append_act(self, x): return fluid.layers.prelu(x, mode='all') class TensorRTSubgraphPassPreluFp16DynamicTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.prelu(x, mode='all') class TensorRTSubgraphPassPreluFp16DynamicSerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.prelu(x, mode='all') class TensorRTSubgraphPassGeluTest(TensorRTSubgraphPassActivationTest): - def append_act(self, x): return fluid.layers.gelu(x) class TensorRTSubgraphPassGeluDynamicTest(TensorRTSubgraphPassActivationTest): - def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.gelu(x) class TensorRTSubgraphPassGeluFp16Test(TensorRTSubgraphPassActivationTest): - def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False + ) def append_act(self, x): return fluid.layers.gelu(x) class TensorRTSubgraphPassGeluFp16SerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) def append_act(self, x): return fluid.layers.gelu(x) -class TensorRTSubgraphPassGeluFp16DynamicTest(TensorRTSubgraphPassActivationTest - ): - +class TensorRTSubgraphPassGeluFp16DynamicTest( + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, False, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.gelu(x) class TensorRTSubgraphPassGeluFp16DynamicSerializeTest( - TensorRTSubgraphPassActivationTest): - + TensorRTSubgraphPassActivationTest +): def setUpTensorRTParam(self): self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassActivationTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) - self.dynamic_shape_params = TensorRTSubgraphPassActivationTest.DynamicShapeParam( - {'data': [1, 6, 8, 8]}, {'data': [1, 6, 128, 128]}, - {'data': [1, 6, 64, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassActivationTest.DynamicShapeParam( + {'data': [1, 6, 8, 8]}, + {'data': [1, 6, 128, 128]}, + {'data': [1, 6, 64, 64]}, + False, + ) + ) def append_act(self, x): return fluid.layers.gelu(x) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_affine_channel_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_affine_channel_op.py index eb2360668f562..69e73d4359697 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_affine_channel_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_affine_channel_op.py @@ -23,7 +23,6 @@ class TRTAffineChannelTest(InferencePassTest): - def setUp(self): self.bs = 2 self.channel = 8 @@ -38,7 +37,8 @@ def build(self): # set min_graph_size to 2, # because affine channel doesn't support nhwc format self.trt_parameters = InferencePassTest.TensorRTParam( - 1 << 30, self.bs, 2, self.precision, self.serialize, False) + 1 << 30, self.bs, 2, self.precision, self.serialize, False + ) with fluid.program_guard(self.main_program, self.startup_program): if self.data_layout == 'NCHW': @@ -51,13 +51,16 @@ def build(self): scale = fluid.layers.create_parameter( shape=[self.channel], dtype='float32', - default_initializer=fluid.initializer.Constant(2.)) + default_initializer=fluid.initializer.Constant(2.0), + ) bias = fluid.layers.create_parameter( shape=[self.channel], dtype='float32', - default_initializer=fluid.initializer.Constant(.5)) + default_initializer=fluid.initializer.Constant(0.5), + ) affine_channel_out = fluid.layers.affine_channel( - data, scale=scale, bias=bias, data_layout=self.data_layout) + data, scale=scale, bias=bias, data_layout=self.data_layout + ) out = fluid.layers.batch_norm(affine_channel_out, is_test=True) shape[0] = self.bs @@ -74,7 +77,8 @@ def check_output(self): atol = 2e-2 self.check_output_with_option(use_gpu, atol, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def run_test(self): self.build() @@ -82,30 +86,39 @@ def run_test(self): def run_test_all(self): precision_opt = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_opt = [False, True] if self.data_layout == 'NCHW': min_shape = [ - self.bs, self.channel, self.height // 2, self.width // 2 + self.bs, + self.channel, + self.height // 2, + self.width // 2, ] max_shape = [self.bs, self.channel, self.height * 2, self.width * 2] opt_shape = [self.bs, self.channel, self.height, self.width] if self.data_layout == 'NHWC': min_shape = [ - self.bs, self.height // 2, self.width // 2, self.channel + self.bs, + self.height // 2, + self.width // 2, + self.channel, ] max_shape = [self.bs, self.height * 2, self.width * 2, self.channel] opt_shape = [self.bs, self.height, self.width, self.channel] dynamic_shape_profile = InferencePassTest.DynamicShapeParam( - {'in': min_shape}, {'in': max_shape}, {'in': opt_shape}, False) + {'in': min_shape}, {'in': max_shape}, {'in': opt_shape}, False + ) dynamic_shape_opt = [None, dynamic_shape_profile] for precision, serialize, dynamic_shape in itertools.product( - precision_opt, serialize_opt, dynamic_shape_opt): + precision_opt, serialize_opt, dynamic_shape_opt + ): self.precision = precision self.serialize = serialize self.dynamic_shape_params = dynamic_shape @@ -126,7 +139,9 @@ def test_dynamic(self): self.dynamic_shape_params = InferencePassTest.DynamicShapeParam( {'in': [self.bs, self.channel, self.height // 2, self.width // 2]}, {'in': [self.bs, self.channel, self.height * 2, self.width * 2]}, - {'in': [self.bs, self.channel, self.height, self.width]}, False) + {'in': [self.bs, self.channel, self.height, self.width]}, + False, + ) self.run_test() def test_nchw_all(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_anchor_generator_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_anchor_generator_op.py index 7e22604a374a6..cd05c8528bc1b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_anchor_generator_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_anchor_generator_op.py @@ -22,40 +22,47 @@ class TRTAnchorGeneratorBaseTest(InferencePassTest): - def setUp(self): self.bs = 1 self.channel = 16 self.height = 32 self.width = 32 - self.anchor_sizes = [64., 128., 256., 512.] - self.aspect_ratios = [.5, 1., 2.] - self.variance = [.1, .1, .2, .2] - self.stride = [8., 8.] + self.anchor_sizes = [64.0, 128.0, 256.0, 512.0] + self.aspect_ratios = [0.5, 1.0, 2.0] + self.variance = [0.1, 0.1, 0.2, 0.2] + self.stride = [8.0, 8.0] self.precision = AnalysisConfig.Precision.Float32 self.serialize = False self.enable_trt = True self.feeds = { - 'data': - np.random.random([self.bs, self.channel, self.height, - self.width]).astype('float32'), + 'data': np.random.random( + [self.bs, self.channel, self.height, self.width] + ).astype('float32'), } def build(self): min_graph_size = 3 if self.dynamic_shape_params is not None else 2 self.trt_parameters = InferencePassTest.TensorRTParam( - 1 << 30, self.bs, min_graph_size, self.precision, self.serialize, - False) + 1 << 30, + self.bs, + min_graph_size, + self.precision, + self.serialize, + False, + ) with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name='data', - shape=[-1, self.channel, self.height, self.width], - dtype='float32') + data = fluid.data( + name='data', + shape=[-1, self.channel, self.height, self.width], + dtype='float32', + ) anchor, var = fluid.layers.detection.anchor_generator( data, anchor_sizes=self.anchor_sizes, aspect_ratios=self.aspect_ratios, variance=self.variance, - stride=self.stride) + stride=self.stride, + ) if self.dynamic_shape_params is not None: anchor = fluid.layers.transpose(anchor, [2, 3, 0, 1]) out = fluid.layers.batch_norm(anchor, is_test=True) @@ -69,10 +76,17 @@ def run_test(self): def set_dynamic(self): self.dynamic_shape_params = InferencePassTest.DynamicShapeParam( { - 'data': - [self.bs, self.channel, self.height // 2, self.width // 2] - }, {'data': [self.bs, self.channel, self.height, self.width]}, - {'data': [self.bs, self.channel, self.height, self.width]}, False) + 'data': [ + self.bs, + self.channel, + self.height // 2, + self.width // 2, + ] + }, + {'data': [self.bs, self.channel, self.height, self.width]}, + {'data': [self.bs, self.channel, self.height, self.width]}, + False, + ) def test_base(self): self.run_test() @@ -113,7 +127,8 @@ def check_output(self): atol = 1e-3 self.check_output_with_option(use_gpu, atol, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_c_allreduce_infer_script.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_c_allreduce_infer_script.py index e90157c928cd2..d2d34e34cbc99 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_c_allreduce_infer_script.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_c_allreduce_infer_script.py @@ -39,20 +39,25 @@ def run(op_type, precision): lod_level=data.lod_level, persistable=False, is_data=False, - initializer=paddle.nn.initializer.Constant(value=1.0)) - block.append_op(type=op_type, - inputs={'X': data}, - outputs={'Out': c_data}, - attrs={ - 'ring_id': 0, - 'use_calc_stream': True, - 'use_model_parallel': True - }) + initializer=paddle.nn.initializer.Constant(value=1.0), + ) + block.append_op( + type=op_type, + inputs={'X': data}, + outputs={'Out': c_data}, + attrs={ + 'ring_id': 0, + 'use_calc_stream': True, + 'use_model_parallel': True, + }, + ) out = paddle.static.nn.fc( x=c_data, size=1, weight_attr=paddle.ParamAttr( - initializer=paddle.nn.initializer.Constant(value=0.5))) + initializer=paddle.nn.initializer.Constant(value=0.5) + ), + ) mean = paddle.mean(out) exe = paddle.static.Executor(paddle.CPUPlace()) exe.run(startup_program) @@ -68,12 +73,17 @@ def run(op_type, precision): dist_config.enable_dist_model(True) with tempfile.TemporaryDirectory(prefix="allreduce_") as tmpdir: - paddle.static.save_inference_model(os.path.join(tmpdir, "model"), - [data], [mean], - exe, - program=main_program) - config = Config(os.path.join(tmpdir, "model.pdmodel"), - os.path.join(tmpdir, "model.pdiparams")) + paddle.static.save_inference_model( + os.path.join(tmpdir, "model"), + [data], + [mean], + exe, + program=main_program, + ) + config = Config( + os.path.join(tmpdir, "model.pdmodel"), + os.path.join(tmpdir, "model.pdiparams"), + ) config.enable_memory_optim() config.enable_use_gpu(1000, fleet.worker_index()) config.set_dist_config(dist_config) @@ -82,11 +92,14 @@ def run(op_type, precision): max_batch_size=1, min_subgraph_size=1, precision_mode=PrecisionType.Half - if precision == "fp16" else PrecisionType.Int8, + if precision == "fp16" + else PrecisionType.Int8, use_static=False, - use_calib_mode=False) - config.set_trt_dynamic_shape_info({"data": [3, 4]}, {"data": [3, 4]}, - {"data": [3, 4]}) + use_calib_mode=False, + ) + config.set_trt_dynamic_shape_info( + {"data": [3, 4]}, {"data": [3, 4]}, {"data": [3, 4]} + ) predictor = create_predictor(config) input_names = predictor.get_input_names() input_tensor = predictor.get_input_handle("data") diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_op.py index 088b548f0184a..70e62f9eaf6c7 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_op.py @@ -22,29 +22,31 @@ class TensorRTSubgraphPassConv3dTest(InferencePassTest): - def setUp(self): self.init_params() self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 6, 32, 32], - dtype="float32") - conv_out = fluid.layers.conv3d(input=data, - num_filters=self.conv_num_filters, - filter_size=self.conv_filter_size, - groups=self.conv_groups, - padding=self.conv_padding, - bias_attr=False, - use_cudnn=self.use_cudnn, - stride=self.stride, - act=None) + data = fluid.data( + name="data", shape=[-1, 3, 6, 32, 32], dtype="float32" + ) + conv_out = fluid.layers.conv3d( + input=data, + num_filters=self.conv_num_filters, + filter_size=self.conv_filter_size, + groups=self.conv_groups, + padding=self.conv_padding, + bias_attr=False, + use_cudnn=self.use_cudnn, + stride=self.stride, + act=None, + ) self.feeds = { "data": np.random.random([1, 3, 6, 32, 32]).astype("float32"), } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassConv3dTest.TensorRTParam( - 1 << 30, 32, 1, self.precision, self.use_static, False) + 1 << 30, 32, 1, self.precision, self.use_static, False + ) self.fetch_list = [conv_out] def init_params(self): @@ -65,12 +67,13 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) - + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) -class TensorRTSubgraphPassConv3dValidPaddingTest(TensorRTSubgraphPassConv3dTest - ): +class TensorRTSubgraphPassConv3dValidPaddingTest( + TensorRTSubgraphPassConv3dTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -79,7 +82,6 @@ def set_params(self): class TensorRTSubgraphPassConv3dSamePaddingTest(TensorRTSubgraphPassConv3dTest): - def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -88,7 +90,6 @@ def set_params(self): class TensorRTSubgraphPassConv3dPaddingTest(TensorRTSubgraphPassConv3dTest): - def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -97,7 +98,6 @@ def set_params(self): class TensorRTSubgraphPassConv3dStrideTest(TensorRTSubgraphPassConv3dTest): - def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -107,39 +107,49 @@ def set_params(self): class DynamicShapeTensorRTSubgraphPassConv3dTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, -1, -1, -1], - dtype="float32") - conv_out = fluid.layers.conv3d(input=data, - num_filters=self.conv_num_filters, - filter_size=self.conv_filter_size, - groups=self.conv_groups, - padding=self.conv_padding, - bias_attr=False, - use_cudnn=self.use_cudnn, - stride=self.stride, - act=None) + data = fluid.data( + name="data", shape=[-1, 6, -1, -1, -1], dtype="float32" + ) + conv_out = fluid.layers.conv3d( + input=data, + num_filters=self.conv_num_filters, + filter_size=self.conv_filter_size, + groups=self.conv_groups, + padding=self.conv_padding, + bias_attr=False, + use_cudnn=self.use_cudnn, + stride=self.stride, + act=None, + ) self.feeds = { "data": np.random.random([1, 6, 32, 32, 8]).astype("float32"), } self.enable_trt = True - self.trt_parameters = DynamicShapeTensorRTSubgraphPassConv3dTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = DynamicShapeTensorRTSubgraphPassConv3dTest.DynamicShapeParam( - { - "data": [1, 6, 8, 8, 8], - "conv3d_0.tmp_0": [1, 6, 8, 8, 4], - }, { - "data": [32, 6, 32, 32, 8], - "conv3d_0.tmp_0": [32, 6, 32, 32, 8], - }, { - "data": [16, 6, 16, 16, 8], - "conv3d_0.tmp_0": [16, 6, 16, 16, 8], - }, False) + self.trt_parameters = ( + DynamicShapeTensorRTSubgraphPassConv3dTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + ) + self.dynamic_shape_params = ( + DynamicShapeTensorRTSubgraphPassConv3dTest.DynamicShapeParam( + { + "data": [1, 6, 8, 8, 8], + "conv3d_0.tmp_0": [1, 6, 8, 8, 4], + }, + { + "data": [32, 6, 32, 32, 8], + "conv3d_0.tmp_0": [32, 6, 32, 32, 8], + }, + { + "data": [16, 6, 16, 16, 8], + "conv3d_0.tmp_0": [16, 6, 16, 16, 8], + }, + False, + ) + ) self.fetch_list = [conv_out] def set_params(self): @@ -155,7 +165,8 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_transpose_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_transpose_op.py index 0cd4bd1a8e98f..9beabe5505230 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv3d_transpose_op.py @@ -22,13 +22,12 @@ class TensorRTSubgraphPassConv3dTransposeTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 4, 4, 32, 32], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 4, 4, 32, 32], dtype="float32" + ) conv_out = fluid.layers.conv3d_transpose( input=data, num_filters=self.conv_num_filters, @@ -38,13 +37,17 @@ def setUp(self): bias_attr=False, use_cudnn=self.use_cudnn, stride=1, - act=None) + act=None, + ) self.feeds = { "data": np.random.random([1, 4, 4, 32, 32]).astype("float32"), } self.enable_trt = True - self.trt_parameters = TensorRTSubgraphPassConv3dTransposeTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + self.trt_parameters = ( + TensorRTSubgraphPassConv3dTransposeTest.TensorRTParam( + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) + ) self.fetch_list = [conv_out] def set_params(self): @@ -59,12 +62,13 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassConv3dTransposeSamePaddingTest( - TensorRTSubgraphPassConv3dTransposeTest): - + TensorRTSubgraphPassConv3dTransposeTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -74,8 +78,8 @@ def set_params(self): class TensorRTSubgraphPassConv3dTransposeMultigroupTest( - TensorRTSubgraphPassConv3dTransposeTest): - + TensorRTSubgraphPassConv3dTransposeTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -85,13 +89,12 @@ def set_params(self): class DynamicShapeTensorRTSubgraphPassConv3dTransposeTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, -1, -1, -1], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, -1, -1, -1], dtype="float32" + ) conv_out = fluid.layers.conv3d_transpose( input=data, num_filters=self.conv_num_filters, @@ -101,24 +104,32 @@ def setUp(self): bias_attr=False, use_cudnn=self.use_cudnn, stride=self.stride, - act=None) + act=None, + ) self.feeds = { "data": np.random.random([1, 6, 32, 32, 8]).astype("float32"), } self.enable_trt = True - self.trt_parameters = DynamicShapeTensorRTSubgraphPassConv3dTransposeTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + self.trt_parameters = ( + DynamicShapeTensorRTSubgraphPassConv3dTransposeTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + ) self.dynamic_shape_params = DynamicShapeTensorRTSubgraphPassConv3dTransposeTest.DynamicShapeParam( { "data": [1, 6, 8, 8, 8], "conv3d_transpose_0.tmp_0": [1, 6, 8, 8, 1], - }, { + }, + { "data": [32, 6, 32, 32, 8], "conv3d_transpose_0.tmp_0": [32, 6, 64, 64, 16], - }, { + }, + { "data": [16, 6, 16, 16, 8], "conv3d_transpose_0.tmp_0": [16, 6, 16, 16, 8], - }, False) + }, + False, + ) self.fetch_list = [conv_out] def set_params(self): @@ -134,7 +145,8 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py index 5e163bce4073a..90dbed96f9066 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_pass.py @@ -25,27 +25,29 @@ class TensorRTSubgraphPassConvTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") - conv_out = fluid.layers.conv2d(input=data, - num_filters=self.conv_num_filters, - filter_size=self.conv_filter_size, - groups=self.conv_groups, - padding=self.conv_padding, - bias_attr=False, - use_cudnn=self.use_cudnn, - act=None) + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=self.conv_num_filters, + filter_size=self.conv_filter_size, + groups=self.conv_groups, + padding=self.conv_padding, + bias_attr=False, + use_cudnn=self.use_cudnn, + act=None, + ) self.feeds = { "data": np.random.random([1, 6, 64, 64]).astype("float32"), } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassConvTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [conv_out] def set_params(self): @@ -60,11 +62,11 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassConvValidPaddingTest(TensorRTSubgraphPassConvTest): - def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -74,7 +76,6 @@ def set_params(self): class TensorRTSubgraphPassConvSamePaddingTest(InferencePassTest): - def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -84,7 +85,6 @@ def set_params(self): class TensorRTSubgraphPassDepthwiseConvTest(TensorRTSubgraphPassConvTest): - def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -94,7 +94,6 @@ def set_params(self): class TensorRTSubgraphPassDepthwiseConv2Test(TensorRTSubgraphPassConvTest): - def set_params(self): self.conv_num_filters = 12 self.conv_filter_size = 6 @@ -104,13 +103,12 @@ def set_params(self): class TensorRTSubgraphPassConvTransposeTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) conv_out = fluid.layers.conv2d_transpose( input=data, num_filters=self.conv_num_filters, @@ -119,13 +117,17 @@ def setUp(self): padding=self.conv_padding, bias_attr=False, use_cudnn=self.use_cudnn, - act=None) + act=None, + ) self.feeds = { "data": np.random.random([1, 6, 64, 64]).astype("float32"), } self.enable_trt = True - self.trt_parameters = TensorRTSubgraphPassConvTransposeTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + self.trt_parameters = ( + TensorRTSubgraphPassConvTransposeTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + ) self.fetch_list = [conv_out] def set_params(self): @@ -140,12 +142,13 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassConvTransposeValidPaddingTest( - TensorRTSubgraphPassConvTransposeTest): - + TensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -155,8 +158,8 @@ def set_params(self): class TensorRTSubgraphPassConvTransposeSamePaddingTest( - TensorRTSubgraphPassConvTransposeTest): - + TensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -166,8 +169,8 @@ def set_params(self): class TensorRTSubgraphPassConvTransposeMultiGroupTest( - TensorRTSubgraphPassConvTransposeTest): - + TensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 @@ -177,8 +180,8 @@ def set_params(self): class TensorRTSubgraphPassConvTranspose2Test( - TensorRTSubgraphPassConvTransposeTest): - + TensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 12 self.conv_filter_size = 4 @@ -188,8 +191,8 @@ def set_params(self): class TensorRTSubgraphPassDepthwiseConvTransposeTest( - TensorRTSubgraphPassConvTransposeTest): - + TensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 4 @@ -199,42 +202,52 @@ def set_params(self): class DynamicShapeTensorRTSubgraphPassConvTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, -1, -1], - dtype="float32") - conv_out = fluid.layers.conv2d(input=data, - num_filters=self.conv_num_filters, - filter_size=self.conv_filter_size, - groups=self.conv_groups, - padding=self.conv_padding, - bias_attr=False, - use_cudnn=self.use_cudnn, - stride=self.stride, - act=None) + data = fluid.data( + name="data", shape=[-1, 6, -1, -1], dtype="float32" + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=self.conv_num_filters, + filter_size=self.conv_filter_size, + groups=self.conv_groups, + padding=self.conv_padding, + bias_attr=False, + use_cudnn=self.use_cudnn, + stride=self.stride, + act=None, + ) self.feeds = { "data": np.random.random([32, 6, 64, 64]).astype("float32"), } self.enable_trt = True - self.trt_parameters = DynamicShapeTensorRTSubgraphPassConvTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = DynamicShapeTensorRTSubgraphPassConvTest.DynamicShapeParam( - { - "conv2d_0.tmp_0": [1, 6, 8, 8], - "data": [1, 6, 8, 8], - "depthwise_conv2d_0.tmp_0": [1, 6, 8, 8] - }, { - "conv2d_0.tmp_0": [32, 6, 64, 64], - "data": [32, 6, 64, 64], - "depthwise_conv2d_0.tmp_0": [32, 6, 64, 64] - }, { - "conv2d_0.tmp_0": [16, 6, 16, 16], - "data": [16, 6, 16, 16], - "depthwise_conv2d_0.tmp_0": [16, 6, 16, 16] - }, False) + self.trt_parameters = ( + DynamicShapeTensorRTSubgraphPassConvTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + ) + self.dynamic_shape_params = ( + DynamicShapeTensorRTSubgraphPassConvTest.DynamicShapeParam( + { + "conv2d_0.tmp_0": [1, 6, 8, 8], + "data": [1, 6, 8, 8], + "depthwise_conv2d_0.tmp_0": [1, 6, 8, 8], + }, + { + "conv2d_0.tmp_0": [32, 6, 64, 64], + "data": [32, 6, 64, 64], + "depthwise_conv2d_0.tmp_0": [32, 6, 64, 64], + }, + { + "conv2d_0.tmp_0": [16, 6, 16, 16], + "data": [16, 6, 16, 16], + "depthwise_conv2d_0.tmp_0": [16, 6, 16, 16], + }, + False, + ) + ) self.fetch_list = [conv_out] def set_params(self): @@ -250,12 +263,13 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class DynamicShapeTensorRTSubgraphPassDepthwiseConvTransposeTest( - DynamicShapeTensorRTSubgraphPassConvTest): - + DynamicShapeTensorRTSubgraphPassConvTest +): def set_params(self): self.conv_num_filters = 6 self.conv_filter_size = 6 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_quant_dequant_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_quant_dequant_pass.py index fbaf6944be983..f19de2a3bb372 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_quant_dequant_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_conv_quant_dequant_pass.py @@ -23,25 +23,26 @@ class QuantDequantTensorRTSubgraphPassConvTest(QuantDequantTest): - def setUp(self): self.set_params() def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) data_reshape = fluid.layers.reshape(self.data, shape=[1, 4, 14, 14]) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') label_shape = fluid.layers.reshape(self.label, shape=[1, 1, 1]) - conv_out = fluid.layers.conv2d(input=data_reshape, - num_filters=self.conv_num_filters, - filter_size=self.conv_filter_size, - groups=self.conv_groups, - padding=self.conv_padding, - bias_attr=False, - use_cudnn=self.use_cudnn, - act=None) + conv_out = fluid.layers.conv2d( + input=data_reshape, + num_filters=self.conv_num_filters, + filter_size=self.conv_filter_size, + groups=self.conv_groups, + padding=self.conv_padding, + bias_attr=False, + use_cudnn=self.use_cudnn, + act=None, + ) if self.conv_padding == [1, 1]: cout = fluid.layers.reshape(conv_out, shape=[1, 1, 10816]) elif self.conv_padding == 'VALID': @@ -58,21 +59,25 @@ def network(): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random([1, 28, 28]).astype("float32")} self.fetch_list = [result] self.enable_trt = True - self.trt_parameters = QuantDequantTensorRTSubgraphPassConvTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) + self.trt_parameters = ( + QuantDequantTensorRTSubgraphPassConvTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' @@ -86,17 +91,17 @@ def set_params(self): def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1e-1, - flatten=False, - rtol=1e-1) + self.check_output_with_option( + use_gpu, atol=1e-1, flatten=False, rtol=1e-1 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class QuantDequantTensorRTSubgraphPassConvValidPaddingTest( - QuantDequantTensorRTSubgraphPassConvTest): - + QuantDequantTensorRTSubgraphPassConvTest +): def set_params(self): self.conv_num_filters = 64 self.conv_filter_size = 4 @@ -106,8 +111,8 @@ def set_params(self): class QuantDequantTensorRTSubgraphPassConvSamePaddingTest( - QuantDequantTensorRTSubgraphPassConvTest): - + QuantDequantTensorRTSubgraphPassConvTest +): def set_params(self): self.conv_num_filters = 64 self.conv_filter_size = 4 @@ -117,8 +122,8 @@ def set_params(self): class QuantDequantTensorRTSubgraphPassDWConvTest( - QuantDequantTensorRTSubgraphPassConvTest): - + QuantDequantTensorRTSubgraphPassConvTest +): def set_params(self): self.conv_num_filters = 64 self.conv_filter_size = 4 @@ -128,25 +133,26 @@ def set_params(self): class DynamicShapeQuantDequantTensorRTSubgraphPassConvTest(QuantDequantTest): - def setUp(self): self.set_params() def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) data_reshape = fluid.layers.reshape(self.data, shape=[1, 4, 14, 14]) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') label_shape = fluid.layers.reshape(self.label, shape=[1, 1, 1]) - conv_out = fluid.layers.conv2d(input=data_reshape, - num_filters=self.conv_num_filters, - filter_size=self.conv_filter_size, - groups=self.conv_groups, - padding=self.conv_padding, - bias_attr=False, - use_cudnn=self.use_cudnn, - act=None) + conv_out = fluid.layers.conv2d( + input=data_reshape, + num_filters=self.conv_num_filters, + filter_size=self.conv_filter_size, + groups=self.conv_groups, + padding=self.conv_padding, + bias_attr=False, + use_cudnn=self.use_cudnn, + act=None, + ) cout = fluid.layers.reshape(conv_out, shape=[1, 1, 10816]) result = fluid.layers.relu(cout) loss = fluid.layers.cross_entropy(input=result, label=label_shape) @@ -156,41 +162,49 @@ def network(): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random([1, 28, 28]).astype("float32")} self.fetch_list = [result] self.enable_trt = True - self.trt_parameters = DynamicShapeQuantDequantTensorRTSubgraphPassConvTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) + self.trt_parameters = ( + DynamicShapeQuantDequantTensorRTSubgraphPassConvTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) + ) self.dynamic_shape_params = DynamicShapeQuantDequantTensorRTSubgraphPassConvTest.DynamicShapeParam( { "conv2d_0.tmp_0": [1, 4, 14, 14], "data": [1, 28, 28], "depthwise_conv2d_0.tmp_0": [1, 4, 14, 14], "reshape2_0.tmp_0": [1, 4, 14, 14], - "reshape2_2.tmp_0": [1, 1, 10816] - }, { + "reshape2_2.tmp_0": [1, 1, 10816], + }, + { "conv2d_0.tmp_0": [4, 4, 14, 14], "data": [4, 28, 28], "depthwise_conv2d_0.tmp_0": [4, 4, 14, 14], "reshape2_0.tmp_0": [4, 4, 14, 14], - "reshape2_2.tmp_0": [1, 1, 43264] - }, { + "reshape2_2.tmp_0": [1, 1, 43264], + }, + { "conv2d_0.tmp_0": [1, 4, 14, 14], "data": [1, 28, 28], "depthwise_conv2d_0.tmp_0": [1, 4, 14, 14], "reshape2_0.tmp_0": [1, 4, 14, 14], - "reshape2_2.tmp_0": [1, 1, 10816] - }, False) + "reshape2_2.tmp_0": [1, 1, 10816], + }, + False, + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' @@ -204,23 +218,22 @@ def set_params(self): def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1e-1, - flatten=False, - rtol=1e-1) + self.check_output_with_option( + use_gpu, atol=1e-1, flatten=False, rtol=1e-1 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class QuantDequantTensorRTSubgraphPassConvTransposeTest(QuantDequantTest): - def setUp(self): self.set_params() def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) data_reshape = fluid.layers.reshape(self.data, shape=[1, 4, 14, 14]) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') label_shape = fluid.layers.reshape(self.label, shape=[1, 1, 1]) @@ -232,7 +245,8 @@ def network(): padding=self.conv_padding, bias_attr=False, use_cudnn=self.use_cudnn, - act=None) + act=None, + ) if self.conv_padding == [1, 1]: cout = fluid.layers.reshape(conv_out, shape=[1, 1, 14400]) elif self.conv_padding == 'VALID': @@ -249,21 +263,25 @@ def network(): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random([1, 28, 28]).astype("float32")} self.fetch_list = [result] self.enable_trt = True - self.trt_parameters = QuantDequantTensorRTSubgraphPassConvTransposeTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) + self.trt_parameters = ( + QuantDequantTensorRTSubgraphPassConvTransposeTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' @@ -277,17 +295,17 @@ def set_params(self): def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1e-1, - flatten=False, - rtol=1e-1) + self.check_output_with_option( + use_gpu, atol=1e-1, flatten=False, rtol=1e-1 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class QuantDequantTensorRTSubgraphPassConvTransValidPaddingTest( - QuantDequantTensorRTSubgraphPassConvTransposeTest): - + QuantDequantTensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 64 self.conv_filter_size = 4 @@ -297,8 +315,8 @@ def set_params(self): class QuantDequantTensorRTSubgraphPassConvTransSamePaddingTest( - QuantDequantTensorRTSubgraphPassConvTransposeTest): - + QuantDequantTensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 64 self.conv_filter_size = 4 @@ -308,8 +326,8 @@ def set_params(self): class QuantDequantTensorRTSubgraphPassTransDWConvTest( - QuantDequantTensorRTSubgraphPassConvTransposeTest): - + QuantDequantTensorRTSubgraphPassConvTransposeTest +): def set_params(self): self.conv_num_filters = 64 self.conv_filter_size = 4 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_activation.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_activation.py index 49367f78e2da4..c9bf048a044ba 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_activation.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_activation.py @@ -22,12 +22,10 @@ class TrtConvertActivationTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(dims, batch, attrs: List[Dict[str, Any]]): if dims == 1: return np.random.random([32]).astype(np.float32) @@ -41,11 +39,19 @@ def generate_input1(dims, batch, attrs: List[Dict[str, Any]]): for dims in [1, 2, 3, 4]: for batch in [1, 4]: for op_type in [ - "relu", "sigmoid", "tanh", "relu6", "elu", "selu", - "softsign", "stanh", "thresholded_relu", "softplus" + "relu", + "sigmoid", + "tanh", + "relu6", + "elu", + "selu", + "softsign", + "stanh", + "thresholded_relu", + "softplus", ]: # few samples to reduce time - #for beta in [-0.2, 0.5, 0.67, 3]: + # for beta in [-0.2, 0.5, 0.67, 3]: # for alpha in [-0.2, 0.5, 0.67, 3]: for beta in [0.67]: for alpha in [0.67]: @@ -62,33 +68,34 @@ def generate_input1(dims, batch, attrs: List[Dict[str, Any]]): if op_type == "softplus": dics = [{"beta": beta}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dims, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dims, batch, dics + ) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -131,19 +138,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_affine_channel.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_affine_channel.py index 34b033b4b7c80..a9ba3b11f8155 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_affine_channel.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_affine_channel.py @@ -22,12 +22,10 @@ class TrtConvertAffineChannelTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(batch, dims, attrs: List[Dict[str, Any]]): if dims == 2: return np.ones([batch, 64]).astype(np.float32) @@ -50,42 +48,45 @@ def generate_weight1(dims, attrs: List[Dict[str, Any]]): self.dims = dims dics = [{"data_layout": data_layout}] - ops_config = [{ - "op_type": "affine_channel", - "op_inputs": { - "X": ["input_data"], - "Scale": ["scale"], - "Bias": ["bias"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "affine_channel", + "op_inputs": { + "X": ["input_data"], + "Scale": ["scale"], + "Bias": ["bias"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "scale": - TensorConfig( - data_gen=partial(generate_weight1, dims, dics)), - "bias": - TensorConfig( - data_gen=partial(generate_weight1, dims, dics)) + "scale": TensorConfig( + data_gen=partial(generate_weight1, dims, dics) + ), + "bias": TensorConfig( + data_gen=partial(generate_weight1, dims, dics) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, batch, dims, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, batch, dims, dics + ) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 2: self.dynamic_shape.min_input_shape = {"input_data": [1, 32]} @@ -132,19 +133,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_anchor_generator.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_anchor_generator.py index b3e58c204f322..32cb6b64514d0 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_anchor_generator.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_anchor_generator.py @@ -22,60 +22,66 @@ class TrtConvertAnchorGeneratorTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(batch, attrs: List[Dict[str, Any]]): return np.random.random([batch, 3, 64, 64]).astype(np.float32) for batch in [1, 2, 4]: for anchor_sizes in [[64.0, 128.0, 256.0, 512.0]]: for aspect_ratios in [[0.5, 1, 2], [0.4, 1.2, 3]]: - for variances in [[1.0, 1.0, 1.0, 1.0], - [0.5, 1.0, 0.5, 1.0]]: + for variances in [ + [1.0, 1.0, 1.0, 1.0], + [0.5, 1.0, 0.5, 1.0], + ]: for stride in [[16.0, 16.0], [16.0, 32.0]]: for offset in [0.5, 0.8]: - dics = [{ - "anchor_sizes": anchor_sizes, - "aspect_ratios": aspect_ratios, - "variances": variances, - "stride": stride, - "offset": offset - }] - - ops_config = [{ - "op_type": "anchor_generator", - "op_inputs": { - "Input": ["input_data"] - }, - "op_outputs": { - "Anchors": ["output_anchors"], - "Variances": ["output_variances"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "anchor_sizes": anchor_sizes, + "aspect_ratios": aspect_ratios, + "variances": variances, + "stride": stride, + "offset": offset, + } + ] + + ops_config = [ + { + "op_type": "anchor_generator", + "op_inputs": {"Input": ["input_data"]}, + "op_outputs": { + "Anchors": ["output_anchors"], + "Variances": ["output_variances"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, batch, dics + ) + ) }, outputs=[ - "output_anchors", "output_variances" - ]) + "output_anchors", + "output_variances", + ], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -100,19 +106,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_arg_max.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_arg_max.py index 3d8d492e89124..a19132571468a 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_arg_max.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_arg_max.py @@ -22,7 +22,6 @@ class TrtConvertArgMaxTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: input_shape = program_config.inputs["arg_max_input"].shape axis = program_config.ops[0].attrs["axis"] @@ -33,7 +32,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(rank, batch): dims = [batch] for i in range(rank - 1): @@ -48,36 +46,37 @@ def generate_input(rank, batch): self.rank = rank flatten = False dtype = 2 - ops_config = [{ - "op_type": "arg_max", - "op_inputs": { - "X": ["arg_max_input"] - }, - "op_outputs": { - "Out": ["arg_max_out"] - }, - "op_attrs": { - "axis": axis, - "keepdims": keepdims, - "flatten": flatten, - "dtype": dtype + ops_config = [ + { + "op_type": "arg_max", + "op_inputs": {"X": ["arg_max_input"]}, + "op_outputs": {"Out": ["arg_max_out"]}, + "op_attrs": { + "axis": axis, + "keepdims": keepdims, + "flatten": flatten, + "dtype": dtype, + }, } - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "arg_max_input": - TensorConfig(data_gen=partial( - generate_input, rank, batch)) + "arg_max_input": TensorConfig( + data_gen=partial( + generate_input, rank, batch + ) + ) }, - outputs=["arg_max_out"]) + outputs=["arg_max_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.rank == 3: self.dynamic_shape.min_input_shape = { @@ -117,19 +116,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_batch_norm.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_batch_norm.py index 88d13c062a55d..806ead9bc1912 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_batch_norm.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_batch_norm.py @@ -22,12 +22,10 @@ class TrtConvertBatchNormTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: if attrs[0]['data_layout'] == "NCHW": @@ -62,88 +60,119 @@ def generate_MomentumTensor(attrs: List[Dict[str, Any]], batch): for momentum in [0.9, 0.8]: self.num_input = num_input self.dims = dims - dics = [{ - "epsilon": epsilon, - "data_layout": data_layout, - "momentum": momentum, - "is_test": True, - "trainable_statistics": False - }, {}] - dics_intput = [{ - "X": ["batch_norm_input"], - "Bias": ["Bias"], - "Mean": ["Mean"], - "Scale": ["Scale"], - "Variance": ["Variance"], - "MomentumTensor": ["MomentumTensor"] - }, { - "X": ["batch_norm_input"], - "Bias": ["Bias"], - "Mean": ["Mean"], - "Scale": ["Scale"], - "Variance": ["Variance"] - }] - dics_intputs = [{ - "Bias": - TensorConfig(data_gen=partial( - generate_bias, dics, batch)), - "Mean": - TensorConfig(data_gen=partial( - generate_mean, dics, batch)), - "Scale": - TensorConfig(data_gen=partial( - generate_scale, dics, batch)), - "Variance": - TensorConfig(data_gen=partial( - generate_variance, dics, batch)), - "MomentumTensor": - TensorConfig(data_gen=partial( - generate_MomentumTensor, dics, batch)), - }, { - "Bias": - TensorConfig(data_gen=partial( - generate_bias, dics, batch)), - "Mean": - TensorConfig(data_gen=partial( - generate_mean, dics, batch)), - "Scale": - TensorConfig(data_gen=partial( - generate_scale, dics, batch)), - "Variance": - TensorConfig(data_gen=partial( - generate_variance, dics, batch)) - }] - ops_config = [{ - "op_type": - "batch_norm", - "op_inputs": - dics_intput[num_input], - "op_outputs": { - "Y": ["batch_norm_out"], - "MeanOut": ["Mean"], - "VarianceOut": ["Variance"], - "SavedMean": ["SavedMean"], - "SavedVariance": ["SavedVariance"] + dics = [ + { + "epsilon": epsilon, + "data_layout": data_layout, + "momentum": momentum, + "is_test": True, + "trainable_statistics": False, + }, + {}, + ] + dics_intput = [ + { + "X": ["batch_norm_input"], + "Bias": ["Bias"], + "Mean": ["Mean"], + "Scale": ["Scale"], + "Variance": ["Variance"], + "MomentumTensor": ["MomentumTensor"], + }, + { + "X": ["batch_norm_input"], + "Bias": ["Bias"], + "Mean": ["Mean"], + "Scale": ["Scale"], + "Variance": ["Variance"], }, - "op_attrs": - dics[0] - }] + ] + dics_intputs = [ + { + "Bias": TensorConfig( + data_gen=partial( + generate_bias, dics, batch + ) + ), + "Mean": TensorConfig( + data_gen=partial( + generate_mean, dics, batch + ) + ), + "Scale": TensorConfig( + data_gen=partial( + generate_scale, dics, batch + ) + ), + "Variance": TensorConfig( + data_gen=partial( + generate_variance, dics, batch + ) + ), + "MomentumTensor": TensorConfig( + data_gen=partial( + generate_MomentumTensor, + dics, + batch, + ) + ), + }, + { + "Bias": TensorConfig( + data_gen=partial( + generate_bias, dics, batch + ) + ), + "Mean": TensorConfig( + data_gen=partial( + generate_mean, dics, batch + ) + ), + "Scale": TensorConfig( + data_gen=partial( + generate_scale, dics, batch + ) + ), + "Variance": TensorConfig( + data_gen=partial( + generate_variance, dics, batch + ) + ), + }, + ] + ops_config = [ + { + "op_type": "batch_norm", + "op_inputs": dics_intput[num_input], + "op_outputs": { + "Y": ["batch_norm_out"], + "MeanOut": ["Mean"], + "VarianceOut": ["Variance"], + "SavedMean": ["SavedMean"], + "SavedVariance": ["SavedVariance"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights=dics_intputs[num_input], inputs={ - "batch_norm_input": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)) + "batch_norm_input": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ) }, - outputs=["batch_norm_out"]) + outputs=["batch_norm_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: if attrs[0]['data_layout'] == "NCHW": @@ -202,29 +231,35 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(program_config.weights) == 5: return True return False - self.add_skip_case(teller1, SkipReasons.TRT_NOT_SUPPORT, - "INPUT MomentumTensor NOT SUPPORT") + self.add_skip_case( + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "INPUT MomentumTensor NOT SUPPORT", + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bilinear_interp_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bilinear_interp_v2.py index 83559d4fe863a..5015e7e36be06 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bilinear_interp_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bilinear_interp_v2.py @@ -22,7 +22,6 @@ class TrtConvertBilinearInterpV2Test(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -33,13 +32,13 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.ones([1, 3, 64, 64]).astype(np.float32) def generate_input2(attrs: List[Dict[str, Any]]): - return np.random.uniform(low=0.5, high=6.0, - size=(2)).astype("float32") + return np.random.uniform(low=0.5, high=6.0, size=(2)).astype( + "float32" + ) for data_layout in ["NCHW", "NHWC"]: for scale_y in [2.0, -1.0, 0.0]: @@ -47,48 +46,55 @@ def generate_input2(attrs: List[Dict[str, Any]]): scale = [scale_y, scale_x] for out_h in [32, 64, 128, 192]: for out_w in [32, 64]: - dics = [{ - "data_layout": data_layout, - "interp_method": "bilinear", - "align_corners": False, - "align_mode": 0, - "scale": scale, - "out_h": out_h, - "out_w": out_w - }] - - ops_config = [{ - "op_type": "bilinear_interp_v2", - "op_inputs": { - "X": ["input_data"], - "Scale": ["input_scale"] - }, - "op_outputs": { - "Out": ["bilinear_interp_v2_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "data_layout": data_layout, + "interp_method": "bilinear", + "align_corners": False, + "align_mode": 0, + "scale": scale, + "out_h": out_h, + "out_w": out_w, + } + ] + + ops_config = [ + { + "op_type": "bilinear_interp_v2", + "op_inputs": { + "X": ["input_data"], + "Scale": ["input_scale"], + }, + "op_outputs": { + "Out": [ + "bilinear_interp_v2_output_data" + ] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "input_scale": - TensorConfig( - data_gen=partial(generate_input2, dics)) + "input_scale": TensorConfig( + data_gen=partial(generate_input2, dics) + ) }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["bilinear_interp_v2_output_data"]) + outputs=["bilinear_interp_v2_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 64, 64]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -110,19 +116,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 + attrs, True + ), 1e-2 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bmm.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bmm.py index 54a2cffe3ad51..6a7aedc9c9e8d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bmm.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_bmm.py @@ -23,9 +23,7 @@ class TrtConvertBmmTest_dynamic(TrtLayerAutoScanTest): - def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -35,49 +33,50 @@ def generate_input(shape): input1_shape = [batch, 350, 75] input2_shape = [batch, 75, 25] dics = [{}] - ops_config = [{ - "op_type": "bmm", - "op_inputs": { - "X": ["input1_data"], - "Y": ["input2_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "bmm", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1_data": - TensorConfig( - data_gen=partial(generate_input, input1_shape)), - "input2_data": - TensorConfig( - data_gen=partial(generate_input, input2_shape)) + "input1_data": TensorConfig( + data_gen=partial(generate_input, input1_shape) + ), + "input2_data": TensorConfig( + data_gen=partial(generate_input, input2_shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input1_data": [10, 350, 75], - "input2_data": [10, 75, 25] + "input2_data": [10, 75, 25], } self.dynamic_shape.max_input_shape = { "input1_data": [100, 350, 75], - "input2_data": [100, 75, 25] + "input2_data": [100, 75, 25], } self.dynamic_shape.opt_input_shape = { "input1_data": [15, 350, 75], - "input2_data": [15, 75, 25] + "input2_data": [15, 75, 25], } def clear_dynamic_shape(): @@ -98,25 +97,29 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # The output has little diff between gpu and trt in CI-Windows-Inference tol_fp32 = 1e-4 tol_half = 1e-4 - if (os.name == 'nt'): + if os.name == 'nt': tol_fp32 = 1e-2 tol_half = 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), tol_fp32 + attrs, True + ), tol_fp32 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), tol_half + attrs, True + ), tol_half def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_c_allreduce.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_c_allreduce.py index a9f1964e49aeb..964234220cf98 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_c_allreduce.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_c_allreduce.py @@ -19,14 +19,13 @@ class TestDistTRT(unittest.TestCase): - def setUp(self): self.init_case() self.script = "test_trt_c_allreduce_infer_script.py" def init_case(self): self.op_type = "c_allreduce_sum" - self.target_value = 4. + self.target_value = 4.0 self.precision = "fp16" def test_run(self): @@ -45,14 +44,13 @@ def test_run(self): class TestMin(TestDistTRT): - def init_case(self): self.op_type = "c_allreduce_min" - self.target_value = 2. + self.target_value = 2.0 self.precision = "int8" -#class TestMax(TestDistTRT): +# class TestMax(TestDistTRT): # # def init_case(self): # self.op_type = "c_allreduce_max" @@ -60,7 +58,7 @@ def init_case(self): # self.precision = "fp16" # # -#class TestProd(TestDistTRT): +# class TestProd(TestDistTRT): # # def init_case(self): # self.op_type = "c_allreduce_prod" diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_cast.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_cast.py index 5c915db7250d2..76b46313f9590 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_cast.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_cast.py @@ -22,7 +22,6 @@ class TrtConvertCastTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -31,14 +30,13 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return False if attrs[0]['in_dtype'] in [4, 5] and attrs[0]['out_dtype'] == 4: return False - if attrs[0]['in_dtype'] not in [ - 2, 4, 5 - ] or attrs[0]['out_dtype'] not in [2, 4, 5]: + if attrs[0]['in_dtype'] not in [2, 4, 5] or attrs[0][ + 'out_dtype' + ] not in [2, 4, 5]: return False return True def sample_program_configs(self): - def generate_input(type): if type == 0: return np.ones([1, 3, 64, 64]).astype(np.bool) @@ -53,32 +51,32 @@ def generate_input(type): for out_dtype in [0, 2, 4, 5, 6]: dics = [{"in_dtype": in_dtype, "out_dtype": out_dtype}] - ops_config = [{ - "op_type": "cast", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["cast_output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "cast", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["cast_output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input, in_dtype)) + "input_data": TensorConfig( + data_gen=partial(generate_input, in_dtype) + ) }, - outputs=["cast_output_data"]) + outputs=["cast_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 64, 64]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -100,19 +98,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 + attrs, True + ), 1e-2 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_clip.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_clip.py index ad1dce6cdb2df..18d5adb284bf2 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_clip.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_clip.py @@ -22,12 +22,10 @@ class TrtConvertClipTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(dims, batch, attrs: List[Dict[str, Any]]): if dims == 1: return np.ones([32]).astype(np.float32) @@ -46,52 +44,52 @@ def generate_weight2(attrs: List[Dict[str, Any]]): for dims in [1, 2, 3, 4]: for batch in [1, 4]: - for op_inputs in [{ - "X": ["input_data"] - }, { - "X": ["input_data"], - "Min": ["Min_"], - "Max": ["Max_"] - }]: + for op_inputs in [ + {"X": ["input_data"]}, + {"X": ["input_data"], "Min": ["Min_"], "Max": ["Max_"]}, + ]: self.input_num = len(op_inputs) self.dims = dims - dics = [{ - "min": np.random.uniform(1, 10), - "max": np.random.uniform(10, 20) - }, { - "op_inputs": op_inputs - }] - ops_config = [{ - "op_type": "clip", - "op_inputs": op_inputs, - "op_outputs": { - "Out": ["output_data"] + dics = [ + { + "min": np.random.uniform(1, 10), + "max": np.random.uniform(10, 20), }, - "op_attrs": dics[0] - }] + {"op_inputs": op_inputs}, + ] + ops_config = [ + { + "op_type": "clip", + "op_inputs": op_inputs, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "Min_": - TensorConfig( - data_gen=partial(generate_weight1, dics)), - "Max_": - TensorConfig( - data_gen=partial(generate_weight2, dics)) + "Min_": TensorConfig( + data_gen=partial(generate_weight1, dics) + ), + "Max_": TensorConfig( + data_gen=partial(generate_weight2, dics) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dims, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dims, batch, dics + ) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs(self, program_config): - def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -135,19 +133,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_concat.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_concat.py index af4d479f8f21b..46a7b82ef4d78 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_concat.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_concat.py @@ -22,7 +22,6 @@ class TrtConvertConcatTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -31,14 +30,13 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - #The input dimension should be less than or equal to the set axis. + # The input dimension should be less than or equal to the set axis. if len(inputs['concat_input1'].shape) <= attrs[0]['axis']: return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: return np.ones([batch, 3, 24, 24]).astype(np.float32) @@ -79,58 +77,83 @@ def generate_weight1(attrs: List[Dict[str, Any]]): self.num_input = num_input self.dims = dims dics = [{"axis": axis}, {}] - dics_intput = [{ - "X": - ["concat_input1", "concat_input2", "concat_input3"], - "AxisTensor": ["AxisTensor"], - }, { - "X": - ["concat_input1", "concat_input2", "concat_input3"] - }] - dics_inputs = [{ - "concat_input1": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)), - "concat_input2": - TensorConfig( - data_gen=partial(generate_input2, dics, batch)), - "concat_input3": - TensorConfig( - data_gen=partial(generate_input3, dics, batch)), - "AxisTensor": - TensorConfig( - data_gen=partial(generate_weight1, dics)) - }, { - "concat_input1": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)), - "concat_input2": - TensorConfig( - data_gen=partial(generate_input2, dics, batch)), - "concat_input3": - TensorConfig( - data_gen=partial(generate_input3, dics, batch)) - }] - ops_config = [{ - "op_type": "concat", - "op_inputs": dics_intput[num_input], - "op_outputs": { - "Out": ["concat_output"] + dics_intput = [ + { + "X": [ + "concat_input1", + "concat_input2", + "concat_input3", + ], + "AxisTensor": ["AxisTensor"], + }, + { + "X": [ + "concat_input1", + "concat_input2", + "concat_input3", + ] + }, + ] + dics_inputs = [ + { + "concat_input1": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ), + "concat_input2": TensorConfig( + data_gen=partial( + generate_input2, dics, batch + ) + ), + "concat_input3": TensorConfig( + data_gen=partial( + generate_input3, dics, batch + ) + ), + "AxisTensor": TensorConfig( + data_gen=partial(generate_weight1, dics) + ), + }, + { + "concat_input1": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ), + "concat_input2": TensorConfig( + data_gen=partial( + generate_input2, dics, batch + ) + ), + "concat_input3": TensorConfig( + data_gen=partial( + generate_input3, dics, batch + ) + ), }, - "op_attrs": dics[0] - }] + ] + ops_config = [ + { + "op_type": "concat", + "op_inputs": dics_intput[num_input], + "op_outputs": {"Out": ["concat_output"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs=dics_inputs[num_input], - outputs=["concat_output"]) + outputs=["concat_output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.num_input == 0: if self.dims == 4: @@ -138,76 +161,76 @@ def generate_dynamic_shape(attrs): "concat_input1": [1, 3, 24, 24], "concat_input2": [1, 3, 24, 24], "concat_input3": [1, 3, 24, 24], - "AxisTensor": [1] + "AxisTensor": [1], } self.dynamic_shape.max_input_shape = { "concat_input1": [4, 3, 48, 48], "concat_input2": [4, 3, 48, 48], "concat_input3": [4, 3, 48, 48], - "AxisTensor": [1] + "AxisTensor": [1], } self.dynamic_shape.opt_input_shape = { "concat_input1": [1, 3, 24, 24], "concat_input2": [1, 3, 24, 24], "concat_input3": [1, 3, 24, 24], - "AxisTensor": [1] + "AxisTensor": [1], } elif self.dims == 3: self.dynamic_shape.min_input_shape = { "concat_input1": [1, 3, 24], "concat_input2": [1, 3, 24], "concat_input3": [1, 3, 24], - "AxisTensor": [1] + "AxisTensor": [1], } self.dynamic_shape.max_input_shape = { "concat_input1": [4, 12, 48], "concat_input2": [4, 12, 48], "concat_input3": [4, 12, 48], - "AxisTensor": [1] + "AxisTensor": [1], } self.dynamic_shape.opt_input_shape = { "concat_input1": [1, 3, 24], "concat_input2": [1, 3, 24], "concat_input3": [1, 3, 24], - "AxisTensor": [1] + "AxisTensor": [1], } elif self.dims == 2: self.dynamic_shape.min_input_shape = { "concat_input1": [1, 24], "concat_input2": [1, 24], "concat_input3": [1, 24], - "AxisTensor": [1] + "AxisTensor": [1], } self.dynamic_shape.max_input_shape = { "concat_input1": [4, 48], "concat_input2": [4, 48], "concat_input3": [4, 48], - "AxisTensor": [1] + "AxisTensor": [1], } self.dynamic_shape.opt_input_shape = { "concat_input1": [1, 24], "concat_input2": [1, 24], "concat_input3": [1, 24], - "AxisTensor": [1] + "AxisTensor": [1], } elif self.dims == 1: self.dynamic_shape.min_input_shape = { "concat_input1": [24], "concat_input2": [24], "concat_input3": [24], - "AxisTensor": [0] + "AxisTensor": [0], } self.dynamic_shape.max_input_shape = { "concat_input1": [48], "concat_input2": [48], "concat_input3": [48], - "AxisTensor": [0] + "AxisTensor": [0], } self.dynamic_shape.opt_input_shape = { "concat_input1": [24], "concat_input2": [24], "concat_input3": [24], - "AxisTensor": [0] + "AxisTensor": [0], } elif self.num_input == 1: if self.dims == 4: @@ -219,60 +242,60 @@ def generate_dynamic_shape(attrs): self.dynamic_shape.max_input_shape = { "concat_input1": [4, 3, 48, 48], "concat_input2": [4, 3, 48, 48], - "concat_input3": [4, 3, 48, 48] + "concat_input3": [4, 3, 48, 48], } self.dynamic_shape.opt_input_shape = { "concat_input1": [1, 3, 24, 24], "concat_input2": [1, 3, 24, 24], - "concat_input3": [1, 3, 24, 24] + "concat_input3": [1, 3, 24, 24], } elif self.dims == 3: self.dynamic_shape.min_input_shape = { "concat_input1": [1, 3, 24], "concat_input2": [1, 3, 24], - "concat_input3": [1, 3, 24] + "concat_input3": [1, 3, 24], } self.dynamic_shape.max_input_shape = { "concat_input1": [4, 12, 48], "concat_input2": [4, 12, 48], - "concat_input3": [4, 12, 48] + "concat_input3": [4, 12, 48], } self.dynamic_shape.opt_input_shape = { "concat_input1": [1, 3, 24], "concat_input2": [1, 3, 24], - "concat_input3": [1, 3, 24] + "concat_input3": [1, 3, 24], } elif self.dims == 2: self.dynamic_shape.min_input_shape = { "concat_input1": [1, 24], "concat_input2": [1, 24], - "concat_input3": [1, 24] + "concat_input3": [1, 24], } self.dynamic_shape.max_input_shape = { "concat_input1": [4, 48], "concat_input2": [4, 48], - "concat_input3": [4, 48] + "concat_input3": [4, 48], } self.dynamic_shape.opt_input_shape = { "concat_input1": [1, 24], "concat_input2": [1, 24], - "concat_input3": [1, 24] + "concat_input3": [1, 24], } elif self.dims == 1: self.dynamic_shape.min_input_shape = { "concat_input1": [24], "concat_input2": [24], - "concat_input3": [24] + "concat_input3": [24], } self.dynamic_shape.max_input_shape = { "concat_input1": [48], "concat_input2": [48], - "concat_input3": [48] + "concat_input3": [48], } self.dynamic_shape.opt_input_shape = { "concat_input1": [24], "concat_input2": [24], - "concat_input3": [24] + "concat_input3": [24], } def clear_dynamic_shape(): @@ -296,29 +319,33 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(program_config.inputs) == 4: return True return False - self.add_skip_case(teller1, SkipReasons.TRT_NOT_SUPPORT, - "INPUT AxisTensor NOT SUPPORT") + self.add_skip_case( + teller1, SkipReasons.TRT_NOT_SUPPORT, "INPUT AxisTensor NOT SUPPORT" + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d.py index 0682477357e0c..364d8a1b6bb8e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d.py @@ -23,7 +23,6 @@ class TrtConvertConv2dTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -31,14 +30,17 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - if inputs['input_data'].shape[ - 1] != weights['conv2d_weight'].shape[1] * attrs[0]['groups']: + if ( + inputs['input_data'].shape[1] + != weights['conv2d_weight'].shape[1] * attrs[0]['groups'] + ): return False ver = paddle_infer.get_trt_compile_version() if ver[0] * 1000 + ver[1] * 100 + ver[0] * 10 < 7000: if attrs[0]['padding_algorithm'] == 'SAME' and ( - attrs[0]['strides'][0] > 1 or attrs[0]['strides'][1] > 1): + attrs[0]['strides'][0] > 1 or attrs[0]['strides'][1] > 1 + ): return False return True @@ -47,8 +49,12 @@ def sample_program_configs(self): self.trt_param.workspace_size = 1073741824 def generate_input1(batch, attrs: List[Dict[str, Any]]): - return np.ones([batch, attrs[0]['groups'] * 3, 64, 64]).astype( - np.float32) / 4 + return ( + np.ones([batch, attrs[0]['groups'] * 3, 64, 64]).astype( + np.float32 + ) + / 4 + ) def generate_weight1(attrs: List[Dict[str, Any]]): return np.random.random([9, 3, 3, 3]).astype(np.float32) - 0.5 @@ -71,73 +77,82 @@ def generate_weight1(attrs: List[Dict[str, Any]]): data_format_options, ] - for batch, strides, paddings, groups, padding_algorithm, dilations, data_format in itertools.product( - *configurations): - - attrs = [{ - "data_fromat": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides, - "data_format": data_format, - }, {}] - - ops_config = [{ - "op_type": "conv2d", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["conv2d_weight"] - }, - "op_outputs": { - "Output": ["conv_output_data"] + for ( + batch, + strides, + paddings, + groups, + padding_algorithm, + dilations, + data_format, + ) in itertools.product(*configurations): + + attrs = [ + { + "data_fromat": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + "data_format": data_format, }, - "op_attrs": attrs[0] - }, { - "op_type": "relu", - "op_inputs": { - "X": ["conv_output_data"] + {}, + ] + + ops_config = [ + { + "op_type": "conv2d", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + "op_outputs": {"Output": ["conv_output_data"]}, + "op_attrs": attrs[0], }, - "op_outputs": { - "Out": ["output_data"] + { + "op_type": "relu", + "op_inputs": {"X": ["conv_output_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": attrs[1], }, - "op_attrs": attrs[1] - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial(generate_weight1, attrs)) + "conv2d_weight": TensorConfig( + data_gen=partial(generate_weight1, attrs) + ) }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, batch, attrs)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, attrs) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): input_groups = attrs[0]['groups'] * 3 self.dynamic_shape.min_input_shape = { "input_data": [1, input_groups, 32, 32], - "output_data": [1, 24, 32, 32] + "output_data": [1, 24, 32, 32], } self.dynamic_shape.max_input_shape = { "input_data": [4, input_groups, 64, 64], - "output_data": [4, 24, 64, 64] + "output_data": [4, 24, 64, 64], } self.dynamic_shape.opt_input_shape = { "input_data": [1, input_groups, 64, 64], - "output_data": [1, 24, 64, 64] + "output_data": [1, 24, 64, 64], } def clear_dynamic_shape(): @@ -156,25 +171,31 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) self.trt_param.precision = paddle_infer.PrecisionType.Int8 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-2, 1e-2) + attrs, False + ), (1e-2, 1e-2) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) self.trt_param.precision = paddle_infer.PrecisionType.Int8 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-2, 1e-2) + attrs, True + ), (1e-2, 1e-2) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_fusion.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_fusion.py index 420b3a5cfaa37..ab3af8e24aa2e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_fusion.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_fusion.py @@ -23,7 +23,6 @@ class TrtConvertConv2dFusionTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -31,8 +30,10 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - if inputs['input_data'].shape[ - 1] != weights['conv2d_weight'].shape[1] * attrs[0]['groups']: + if ( + inputs['input_data'].shape[1] + != weights['conv2d_weight'].shape[1] * attrs[0]['groups'] + ): return False if attrs[0]['groups'] <= 1: @@ -41,7 +42,8 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: ver = paddle_infer.get_trt_compile_version() if ver[0] * 1000 + ver[1] * 100 + ver[0] * 10 < 7000: if attrs[0]['padding_algorithm'] == 'SAME' and ( - attrs[0]['strides'][0] > 1 or attrs[0]['strides'][1] > 1): + attrs[0]['strides'][0] > 1 or attrs[0]['strides'][1] > 1 + ): return False return True @@ -50,8 +52,9 @@ def sample_program_configs(self): self.trt_param.workspace_size = 1073741824 def generate_input1(batch, attrs: List[Dict[str, Any]]): - return np.ones([batch, attrs[0]['groups'] * 3, 64, - 64]).astype(np.float32) + return np.ones([batch, attrs[0]['groups'] * 3, 64, 64]).astype( + np.float32 + ) def generate_weight1(attrs: List[Dict[str, Any]]): return np.random.random([24, 3, 3, 3]).astype(np.float32) @@ -77,77 +80,87 @@ def generate_weight2(attrs: List[Dict[str, Any]]): data_format_options, ] - for (batch, strides, paddings, groups, padding_algorithm, dilations, - data_format) in product(*configurations): - - attrs = [{ - "strides": strides, - "paddings": paddings, - "groups": groups, - "padding_algorithm": padding_algorithm, - "dilations": dilations, - "data_format": data_format, - }, { - "axis": 1 - }] - - ops_config = [{ - "op_type": "conv2d", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["conv2d_weight"] - }, - "op_outputs": { - "Output": ["conv_output_data"] + for ( + batch, + strides, + paddings, + groups, + padding_algorithm, + dilations, + data_format, + ) in product(*configurations): + + attrs = [ + { + "strides": strides, + "paddings": paddings, + "groups": groups, + "padding_algorithm": padding_algorithm, + "dilations": dilations, + "data_format": data_format, }, - "op_attrs": attrs[0] - }, { - "op_type": "elementwise_add", - "op_inputs": { - "X": ["conv_output_data"], - "Y": ["elementwise_weight"] + {"axis": 1}, + ] + + ops_config = [ + { + "op_type": "conv2d", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + "op_outputs": {"Output": ["conv_output_data"]}, + "op_attrs": attrs[0], }, - "op_outputs": { - "Out": ["output_data"] + { + "op_type": "elementwise_add", + "op_inputs": { + "X": ["conv_output_data"], + "Y": ["elementwise_weight"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": attrs[1], }, - "op_attrs": attrs[1] - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial(generate_weight1, attrs)), - "elementwise_weight": - TensorConfig(data_gen=partial(generate_weight2, attrs)) + "conv2d_weight": TensorConfig( + data_gen=partial(generate_weight1, attrs) + ), + "elementwise_weight": TensorConfig( + data_gen=partial(generate_weight2, attrs) + ), }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, batch, attrs)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, attrs) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): input_groups = attrs[0]['groups'] * 3 self.dynamic_shape.min_input_shape = { "input_data": [1, input_groups, 32, 32], - "output_data": [1, 24, 32, 32] + "output_data": [1, 24, 32, 32], } self.dynamic_shape.max_input_shape = { "input_data": [2, input_groups, 64, 64], - "output_data": [2, 24, 64, 64] + "output_data": [2, 24, 64, 64], } self.dynamic_shape.opt_input_shape = { "input_data": [1, input_groups, 64, 64], - "output_data": [1, 24, 64, 64] + "output_data": [1, 24, 64, 64], } def clear_dynamic_shape(): @@ -166,25 +179,31 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) self.trt_param.precision = paddle_infer.PrecisionType.Int8 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) self.trt_param.precision = paddle_infer.PrecisionType.Int8 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_transpose.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_transpose.py index 3103a105de2aa..bb0f3cfc3a80d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_transpose.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv2d_transpose.py @@ -22,7 +22,6 @@ class TrtConvertConv2dTransposeTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -30,8 +29,10 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - if inputs['input_data'].shape[ - 1] != weights['conv2d_weight'].shape[1] * attrs[0]['groups']: + if ( + inputs['input_data'].shape[1] + != weights['conv2d_weight'].shape[1] * attrs[0]['groups'] + ): return False if inputs['input_data'].shape[1] != weights['conv2d_weight'].shape[0]: @@ -54,12 +55,13 @@ def generate_input1(batch, num_channels, attrs: List[Dict[str, Any]]): def generate_weight1(num_channels, attrs: List[Dict[str, Any]]): if attrs[0]['groups'] == 1: - return np.random.random([num_channels, num_channels, 3, - 3]).astype(np.float32) + return np.random.random( + [num_channels, num_channels, 3, 3] + ).astype(np.float32) else: return np.random.random( - [num_channels, int(num_channels / 2), 3, - 3]).astype(np.float32) + [num_channels, int(num_channels / 2), 3, 3] + ).astype(np.float32) for num_channels in [2, 4, 6]: for batch in [1, 4]: @@ -67,99 +69,113 @@ def generate_weight1(num_channels, attrs: List[Dict[str, Any]]): for paddings in [[0, 3], [1, 2, 3, 4]]: for groups in [2]: for padding_algorithm in [ - 'EXPLICIT', 'SAME', 'VALID' + 'EXPLICIT', + 'SAME', + 'VALID', ]: for dilations in [[2, 2], [1, 2]]: for data_format in ['NCHW']: self.num_channels = num_channels - dics = [{ - "data_fromat": data_format, - "dilations": dilations, - "padding_algorithm": - padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides, - "data_format": data_format, - "output_size": [], - "output_padding": [] - }] - - ops_config = [{ - "op_type": "conv2d_transpose", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["conv2d_weight"] - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "data_fromat": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + "data_format": data_format, + "output_size": [], + "output_padding": [], + } + ] + + ops_config = [ + { + "op_type": "conv2d_transpose", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + "op_outputs": { + "Output": ["output_data"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config( - ops_config) + ops_config + ) program_config = ProgramConfig( ops=ops, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial( - generate_weight1, - num_channels, dics)) + "conv2d_weight": TensorConfig( + data_gen=partial( + generate_weight1, + num_channels, + dics, + ) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, batch, - num_channels, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, + batch, + num_channels, + dics, + ) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.num_channels == 2: self.dynamic_shape.min_input_shape = { "input_data": [1, 2, 32, 32], - "output_data": [1, 24, 32, 32] + "output_data": [1, 24, 32, 32], } self.dynamic_shape.max_input_shape = { "input_data": [4, 2, 64, 64], - "output_data": [4, 24, 64, 64] + "output_data": [4, 24, 64, 64], } self.dynamic_shape.opt_input_shape = { "input_data": [1, 2, 64, 64], - "output_data": [1, 24, 64, 64] + "output_data": [1, 24, 64, 64], } elif self.num_channels == 4: self.dynamic_shape.min_input_shape = { "input_data": [1, 4, 32, 32], - "output_data": [1, 24, 32, 32] + "output_data": [1, 24, 32, 32], } self.dynamic_shape.max_input_shape = { "input_data": [4, 4, 64, 64], - "output_data": [4, 24, 64, 64] + "output_data": [4, 24, 64, 64], } self.dynamic_shape.opt_input_shape = { "input_data": [1, 4, 64, 64], - "output_data": [1, 24, 64, 64] + "output_data": [1, 24, 64, 64], } else: self.dynamic_shape.min_input_shape = { "input_data": [1, 6, 32, 32], - "output_data": [1, 24, 32, 32] + "output_data": [1, 24, 32, 32], } self.dynamic_shape.max_input_shape = { "input_data": [4, 6, 64, 64], - "output_data": [4, 24, 64, 64] + "output_data": [4, 24, 64, 64], } self.dynamic_shape.opt_input_shape = { "input_data": [1, 6, 64, 64], - "output_data": [1, 24, 64, 64] + "output_data": [1, 24, 64, 64], } def clear_dynamic_shape(): @@ -178,10 +194,12 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # self.trt_param.precision = paddle_infer.PrecisionType.Int8 # yield self.create_inference_config(), generate_trt_nodes_num( # attrs, False), (1e-5, 1e-5) @@ -190,24 +208,26 @@ def generate_trt_nodes_num(attrs, dynamic_shape): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) # self.trt_param.precision = paddle_infer.PrecisionType.Int8 # yield self.create_inference_config(), generate_trt_nodes_num( # attrs, True), (1e-5, 1e-5) def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if self.trt_param.precision == paddle_infer.PrecisionType.Int8: return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_IMPLEMENTED, - "When precisionType is int8 without relu op, output is different between Trt and Paddle." + teller1, + SkipReasons.TRT_NOT_IMPLEMENTED, + "When precisionType is int8 without relu op, output is different between Trt and Paddle.", ) def test(self): @@ -221,7 +241,6 @@ def test_quant(self): # Special case class TrtConvertConv2dTransposeTest2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: ver = paddle_infer.get_trt_compile_version() if ver[0] * 1000 + ver[1] * 100 + ver[2] * 10 < 7000: @@ -241,49 +260,52 @@ def generate_weight1(num_channels, attrs: List[Dict[str, Any]]): batch = 1 self.num_channels = num_channels - dics = [{ - "data_fromat": 'NCHW', - "dilations": [1, 1], - "padding_algorithm": 'EXPLICIT', - "groups": 1, - "paddings": [1, 1], - "strides": [2, 2], - "output_padding": [1, 1], - "output_size": [], - }] - - ops_config = [{ - "op_type": "conv2d_transpose", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["conv2d_weight"] - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "data_fromat": 'NCHW', + "dilations": [1, 1], + "padding_algorithm": 'EXPLICIT', + "groups": 1, + "paddings": [1, 1], + "strides": [2, 2], + "output_padding": [1, 1], + "output_size": [], + } + ] + + ops_config = [ + { + "op_type": "conv2d_transpose", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + "op_outputs": {"Output": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "conv2d_weight": - TensorConfig( - data_gen=partial(generate_weight1, num_channels, dics)) + "conv2d_weight": TensorConfig( + data_gen=partial(generate_weight1, num_channels, dics) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, batch, - num_channels, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, num_channels, dics) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 128, 20, 30], @@ -311,19 +333,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-4 + attrs, False + ), 1e-4 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e0, 1e-3) + attrs, False + ), (1e0, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-4 + attrs, True + ), 1e-4 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e0, 1e-3) + attrs, True + ), (1e0, 1e-3) def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv3d_transpose.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv3d_transpose.py index a19405cdec917..d0edb4fba0e55 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv3d_transpose.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_conv3d_transpose.py @@ -23,7 +23,6 @@ # Special case class TrtConvertConv3dTransposeTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: ver = paddle_infer.get_trt_compile_version() if ver[0] * 1000 + ver[1] * 100 + ver[2] * 10 < 8400: @@ -37,56 +36,60 @@ def generate_input1(batch, num_channels, attrs: List[Dict[str, Any]]): return np.ones([batch, num_channels, 4, 20, 30]).astype(np.float32) def generate_weight1(num_channels, attrs: List[Dict[str, Any]]): - return np.random.random([num_channels, 64, 3, 3, - 3]).astype(np.float32) + return np.random.random([num_channels, 64, 3, 3, 3]).astype( + np.float32 + ) num_channels = 128 batch = 1 # in_channels self.num_channels = num_channels - dics = [{ - "data_fromat": 'NCHW', - "dilations": [1, 1, 1], - "padding_algorithm": 'EXPLICIT', - "groups": 1, - "paddings": [1, 1, 1], - "strides": [2, 2, 2], - "output_padding": [1, 1, 1], - "output_size": [], - }] - - ops_config = [{ - "op_type": "conv3d_transpose", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["conv3d_weight"] - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "data_fromat": 'NCHW', + "dilations": [1, 1, 1], + "padding_algorithm": 'EXPLICIT', + "groups": 1, + "paddings": [1, 1, 1], + "strides": [2, 2, 2], + "output_padding": [1, 1, 1], + "output_size": [], + } + ] + + ops_config = [ + { + "op_type": "conv3d_transpose", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["conv3d_weight"], + }, + "op_outputs": {"Output": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "conv3d_weight": - TensorConfig( - data_gen=partial(generate_weight1, num_channels, dics)) + "conv3d_weight": TensorConfig( + data_gen=partial(generate_weight1, num_channels, dics) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, batch, - num_channels, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, num_channels, dics) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 128, 4, 20, 30], @@ -114,13 +117,15 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_deformable_conv.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_deformable_conv.py index f50f1704875b0..e8fb7142b4d57 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_deformable_conv.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_deformable_conv.py @@ -22,7 +22,6 @@ class TrtConvertDeformableConvTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -30,114 +29,158 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - if inputs['input_data'].shape[ - 1] != weights['filter_data'].shape[1] * attrs[0]['groups']: + if ( + inputs['input_data'].shape[1] + != weights['filter_data'].shape[1] * attrs[0]['groups'] + ): return False return True def sample_program_configs(self): - - def compute_output_size(input_size: List[int], kernel_sizes: List[int], - attrs: List[Dict[str, Any]]): + def compute_output_size( + input_size: List[int], + kernel_sizes: List[int], + attrs: List[Dict[str, Any]], + ): strides = attrs[0]['strides'] paddings = attrs[0]['paddings'] dilations = attrs[0]['dilations'] output_size = [] - for i, k, s, p, d in zip(input_size, kernel_sizes, strides, - paddings, dilations): + for i, k, s, p, d in zip( + input_size, kernel_sizes, strides, paddings, dilations + ): k = d * (k - 1) + 1 output_size.append((i + 2 * p - k) // s + 1) return output_size - def generate_input1(batch: int, input_size: List[int], - kernel_sizes: List[int], attrs: List[Dict[str, - Any]]): + def generate_input1( + batch: int, + input_size: List[int], + kernel_sizes: List[int], + attrs: List[Dict[str, Any]], + ): return np.random.random([batch, 3] + input_size).astype(np.float32) - def generate_offset1(batch: int, input_size: List[int], - kernel_sizes: List[int], attrs: List[Dict[str, - Any]]): + def generate_offset1( + batch: int, + input_size: List[int], + kernel_sizes: List[int], + attrs: List[Dict[str, Any]], + ): output_size = compute_output_size(input_size, kernel_sizes, attrs) - return np.random.random([batch, 2 * np.prod(kernel_sizes)] + - output_size).astype(np.float32) - - def generate_mask1(batch: int, input_size: List[int], - kernel_sizes: List[int], attrs: List[Dict[str, - Any]]): + return np.random.random( + [batch, 2 * np.prod(kernel_sizes)] + output_size + ).astype(np.float32) + + def generate_mask1( + batch: int, + input_size: List[int], + kernel_sizes: List[int], + attrs: List[Dict[str, Any]], + ): output_size = compute_output_size(input_size, kernel_sizes, attrs) - return np.random.random([batch, np.prod(kernel_sizes)] + - output_size).astype(np.float32) - - def generate_filter1(batch: int, input_size: List[int], - kernel_sizes: List[int], attrs: List[Dict[str, - Any]]): + return np.random.random( + [batch, np.prod(kernel_sizes)] + output_size + ).astype(np.float32) + + def generate_filter1( + batch: int, + input_size: List[int], + kernel_sizes: List[int], + attrs: List[Dict[str, Any]], + ): return np.random.random([6, 3] + kernel_sizes).astype(np.float32) for batch in [ - 1, + 1, ]: for input_size in [[32, 32]]: for kernel_sizes in [[3, 3]]: for strides in [[1, 1], [2, 2]]: for paddings in [[1, 1], [0, 2]]: for groups in [ - 1, + 1, ]: for dilations in [[1, 1], [2, 2]]: - dics = [{ - "strides": strides, - "paddings": paddings, - "groups": groups, - "dilations": dilations, - "deformable_groups": 1, - "im2col_step": 1 - }] - - ops_config = [{ - "op_type": "deformable_conv", - "op_inputs": { - "Input": ["input_data"], - "Offset": ["offset_data"], - "Mask": ["mask_data"], - "Filter": ["filter_data"] - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "strides": strides, + "paddings": paddings, + "groups": groups, + "dilations": dilations, + "deformable_groups": 1, + "im2col_step": 1, + } + ] + + ops_config = [ + { + "op_type": "deformable_conv", + "op_inputs": { + "Input": ["input_data"], + "Offset": ["offset_data"], + "Mask": ["mask_data"], + "Filter": ["filter_data"], + }, + "op_outputs": { + "Output": ["output_data"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "filter_data": - TensorConfig(data_gen=partial( - generate_filter1, batch, input_size, - kernel_sizes, dics)) + "filter_data": TensorConfig( + data_gen=partial( + generate_filter1, + batch, + input_size, + kernel_sizes, + dics, + ) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, batch, input_size, - kernel_sizes, dics)), - "offset_data": - TensorConfig(data_gen=partial( - generate_offset1, batch, input_size, - kernel_sizes, dics)), - "mask_data": - TensorConfig(data_gen=partial( - generate_mask1, batch, input_size, - kernel_sizes, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, + batch, + input_size, + kernel_sizes, + dics, + ) + ), + "offset_data": TensorConfig( + data_gen=partial( + generate_offset1, + batch, + input_size, + kernel_sizes, + dics, + ) + ), + "mask_data": TensorConfig( + data_gen=partial( + generate_mask1, + batch, + input_size, + kernel_sizes, + dics, + ) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def clear_dynamic_shape(): self.dynamic_shape.min_input_shape = {} self.dynamic_shape.max_input_shape = {} @@ -158,7 +201,8 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 def test(self): self.trt_param.workspace_size = 1 << 28 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d.py index 90afa7fc1f05a..8fd03ae830a7d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d.py @@ -23,7 +23,6 @@ class TrtConvertDepthwiseConv2dTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -31,8 +30,10 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - if inputs['input_data'].shape[ - 1] != weights['conv2d_weight'].shape[1] * attrs[0]['groups']: + if ( + inputs['input_data'].shape[1] + != weights['conv2d_weight'].shape[1] * attrs[0]['groups'] + ): return False return True @@ -65,61 +66,72 @@ def generate_weight1(attrs: List[Dict[str, Any]]): data_format_options, ] - for (batch, strides, paddings, groups, padding_algorithm, dilations, - data_format) in itertools.product(*configurations): - attrs = [{ - "strides": strides, - "paddings": paddings, - "groups": groups, - "padding_algorithm": padding_algorithm, - "dilations": dilations, - "data_fromat": data_format, - }] - - ops_config = [{ - "op_type": "depthwise_conv2d", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["conv2d_weight"] - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": attrs[0] - }] + for ( + batch, + strides, + paddings, + groups, + padding_algorithm, + dilations, + data_format, + ) in itertools.product(*configurations): + attrs = [ + { + "strides": strides, + "paddings": paddings, + "groups": groups, + "padding_algorithm": padding_algorithm, + "dilations": dilations, + "data_fromat": data_format, + } + ] + + ops_config = [ + { + "op_type": "depthwise_conv2d", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + "op_outputs": {"Output": ["output_data"]}, + "op_attrs": attrs[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial(generate_weight1, attrs)) + "conv2d_weight": TensorConfig( + data_gen=partial(generate_weight1, attrs) + ) }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, batch, attrs)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, attrs) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): groups = attrs[0]['groups'] self.dynamic_shape.min_input_shape = { "input_data": [1, groups, 32, 32], - "output_data": [1, 24, 32, 32] + "output_data": [1, 24, 32, 32], } self.dynamic_shape.max_input_shape = { "input_data": [4, groups, 64, 64], - "output_data": [4, 24, 64, 64] + "output_data": [4, 24, 64, 64], } self.dynamic_shape.opt_input_shape = { "input_data": [1, groups, 64, 64], - "output_data": [1, 24, 64, 64] + "output_data": [1, 24, 64, 64], } def clear_dynamic_shape(): @@ -139,35 +151,44 @@ def generate_trt_nodes_num(): self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num(), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half - yield self.create_inference_config(), generate_trt_nodes_num(), (1e-3, - 1e-3) + yield self.create_inference_config(), generate_trt_nodes_num(), ( + 1e-3, + 1e-3, + ) self.trt_param.precision = paddle_infer.PrecisionType.Int8 - yield self.create_inference_config(), generate_trt_nodes_num(), (1e-3, - 1e-3) + yield self.create_inference_config(), generate_trt_nodes_num(), ( + 1e-3, + 1e-3, + ) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num(), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half - yield self.create_inference_config(), generate_trt_nodes_num(), (1e-3, - 1e-3) + yield self.create_inference_config(), generate_trt_nodes_num(), ( + 1e-3, + 1e-3, + ) self.trt_param.precision = paddle_infer.PrecisionType.Int8 - yield self.create_inference_config(), generate_trt_nodes_num(), (1e-3, - 1e-3) + yield self.create_inference_config(), generate_trt_nodes_num(), ( + 1e-3, + 1e-3, + ) def add_skip_trt_case(self): - def teller1(program_config, predictor_config): - if program_config.ops[0].attrs[ - 'padding_algorithm'] == "SAME" or program_config.ops[ - 0].attrs['padding_algorithm'] == "VALID": + if ( + program_config.ops[0].attrs['padding_algorithm'] == "SAME" + or program_config.ops[0].attrs['padding_algorithm'] == "VALID" + ): return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_IMPLEMENTED, - "When padding_algorithm is 'SAME' or 'VALID', Trt dose not support. In this case, trt build error is caused by scale op." + teller1, + SkipReasons.TRT_NOT_IMPLEMENTED, + "When padding_algorithm is 'SAME' or 'VALID', Trt dose not support. In this case, trt build error is caused by scale op.", ) def teller2(program_config, predictor_config): @@ -176,8 +197,9 @@ def teller2(program_config, predictor_config): return False self.add_skip_case( - teller2, SkipReasons.TRT_NOT_IMPLEMENTED, - "When precisionType is int8 without relu op, output is different between Trt and Paddle." + teller2, + SkipReasons.TRT_NOT_IMPLEMENTED, + "When precisionType is int8 without relu op, output is different between Trt and Paddle.", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d_transpose.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d_transpose.py index 417849ded555f..345d546ae640f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d_transpose.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_depthwise_conv2d_transpose.py @@ -22,7 +22,6 @@ class TrtConvertDepthwiseConv2dTransposeTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -30,8 +29,10 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - if inputs['input_data'].shape[ - 1] != weights['conv2d_weight'].shape[1] * attrs[0]['groups']: + if ( + inputs['input_data'].shape[1] + != weights['conv2d_weight'].shape[1] * attrs[0]['groups'] + ): return False if inputs['input_data'].shape[1] != weights['conv2d_weight'].shape[1]: @@ -53,12 +54,14 @@ def sample_program_configs(self): self.trt_param.workspace_size = 1073741824 def generate_input1(batch, attrs: List[Dict[str, Any]]): - return np.ones([batch, attrs[0]['groups'], 64, - 64]).astype(np.float32) + return np.ones([batch, attrs[0]['groups'], 64, 64]).astype( + np.float32 + ) def generate_weight1(attrs: List[Dict[str, Any]]): - return np.random.random([attrs[0]['groups'], 1, 3, - 3]).astype(np.float32) + return np.random.random([attrs[0]['groups'], 1, 3, 3]).astype( + np.float32 + ) for batch in [1, 2, 4]: for strides in [[1, 1], [2, 2], [1, 2]]: @@ -68,62 +71,71 @@ def generate_weight1(attrs: List[Dict[str, Any]]): for dilations in [[1, 1], [2, 2], [1, 2]]: for data_format in ['NCHW']: - dics = [{ - "data_fromat": data_format, - "dilations": dilations, - "padding_algorithm": padding_algorithm, - "groups": groups, - "paddings": paddings, - "strides": strides, - "data_format": data_format, - "output_size": [], - "output_padding": [] - }] - - ops_config = [{ - "op_type": "conv2d_transpose", - "op_inputs": { - "Input": ["input_data"], - "Filter": ["conv2d_weight"] - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "data_fromat": data_format, + "dilations": dilations, + "padding_algorithm": padding_algorithm, + "groups": groups, + "paddings": paddings, + "strides": strides, + "data_format": data_format, + "output_size": [], + "output_padding": [], + } + ] + + ops_config = [ + { + "op_type": "conv2d_transpose", + "op_inputs": { + "Input": ["input_data"], + "Filter": ["conv2d_weight"], + }, + "op_outputs": { + "Output": ["output_data"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "conv2d_weight": - TensorConfig(data_gen=partial( - generate_weight1, dics)) + "conv2d_weight": TensorConfig( + data_gen=partial( + generate_weight1, dics + ) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, batch, dics + ) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, attrs[0]['groups'], 32, 32], - "output_data": [1, attrs[0]['groups'], 32, 32] + "output_data": [1, attrs[0]['groups'], 32, 32], } self.dynamic_shape.max_input_shape = { "input_data": [4, attrs[0]['groups'], 64, 64], - "output_data": [4, attrs[0]['groups'], 64, 64] + "output_data": [4, attrs[0]['groups'], 64, 64], } self.dynamic_shape.opt_input_shape = { "input_data": [1, attrs[0]['groups'], 64, 64], - "output_data": [1, attrs[0]['groups'], 64, 64] + "output_data": [1, attrs[0]['groups'], 64, 64], } def clear_dynamic_shape(): @@ -142,10 +154,12 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # self.trt_param.precision = paddle_infer.PrecisionType.Int8 # yield self.create_inference_config(), generate_trt_nodes_num( # attrs, False), (1e-5, 1e-5) @@ -154,24 +168,26 @@ def generate_trt_nodes_num(attrs, dynamic_shape): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) # self.trt_param.precision = paddle_infer.PrecisionType.Int8 # yield self.create_inference_config(), generate_trt_nodes_num( # attrs, True), (1e-5, 1e-5) def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if self.trt_param.precision == paddle_infer.PrecisionType.Int8: return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_IMPLEMENTED, - "When precisionType is int8 without relu op, output is different between Trt and Paddle." + teller1, + SkipReasons.TRT_NOT_IMPLEMENTED, + "When precisionType is int8 without relu op, output is different between Trt and Paddle.", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_dropout.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_dropout.py index 3f843c773cb71..91b8380d7d612 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_dropout.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_dropout.py @@ -22,12 +22,10 @@ class TrtConvertDropoutTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(dims, batch, attrs: List[Dict[str, Any]]): if dims == 1: return np.ones([64]).astype(np.float32) @@ -42,47 +40,57 @@ def generate_input1(dims, batch, attrs: List[Dict[str, Any]]): for batch in [1, 2, 4]: for fix_seed in [False, True]: for dropout_implementation in [ - "downgrade_in_infer", "upscale_in_train" + "downgrade_in_infer", + "upscale_in_train", ]: for dropout_prob in [np.random.random()]: for seed in [0, 64, 128, 512]: self.dims = dims - dics = [{ - "fix_seed": fix_seed, - "dropout_implementation": - dropout_implementation, - "dropout_prob": dropout_prob, - "seed": seed, - "is_test": True - }] - - ops_config = [{ - "op_type": "dropout", - "op_inputs": { - "X": ["input_data"], - }, - "op_outputs": { - "Out": ["dropout_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "fix_seed": fix_seed, + "dropout_implementation": dropout_implementation, + "dropout_prob": dropout_prob, + "seed": seed, + "is_test": True, + } + ] + + ops_config = [ + { + "op_type": "dropout", + "op_inputs": { + "X": ["input_data"], + }, + "op_outputs": { + "Out": ["dropout_output_data"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dims, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, + dims, + batch, + dics, + ) + ) }, - outputs=["dropout_output_data"]) + outputs=["dropout_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -128,19 +136,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_elementwise.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_elementwise.py index f9d7f0b9079e3..3699055475507 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_elementwise.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_elementwise.py @@ -24,12 +24,10 @@ # This is the special test case with weight including batch dimension # I don't want to mess up the code written by others, so I wrote a class specifically class TrtConvertElementwiseTest_one_input_special_case0(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -39,44 +37,50 @@ def generate_weight(): for batch in [1, 4]: for shape in [[batch, 32, 16, 32]]: for op_type in [ - "elementwise_add", "elementwise_mul", "elementwise_sub", - "elementwise_div", "elementwise_pow", "elementwise_min", - "elementwise_max" + "elementwise_add", + "elementwise_mul", + "elementwise_sub", + "elementwise_div", + "elementwise_pow", + "elementwise_min", + "elementwise_max", ]: for axis in [-1]: self.dims = len(shape) dics = [{"axis": axis}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["input_data"], - "Y": ["weight"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": { + "X": ["input_data"], + "Y": ["weight"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "weight": - TensorConfig(data_gen=partial(generate_weight)) + "weight": TensorConfig( + data_gen=partial(generate_weight) + ) }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, shape)), + "input_data": TensorConfig( + data_gen=partial(generate_input, shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The input.dims[1] must be equal to the weight's length. if self.dims == 4: @@ -106,19 +110,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-5, 1e-5) + attrs, False + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-5, 1e-5) + attrs, True + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): pass @@ -130,12 +138,10 @@ def test(self): # This is the special test case class TrtConvertElementwiseTest_one_input_special_case1(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -144,44 +150,47 @@ def generate_weight(): for shape in [[32]]: for op_type in [ - "elementwise_add", "elementwise_mul", "elementwise_sub", - "elementwise_div", "elementwise_pow", "elementwise_min", - "elementwise_max" + "elementwise_add", + "elementwise_mul", + "elementwise_sub", + "elementwise_div", + "elementwise_pow", + "elementwise_min", + "elementwise_max", ]: for axis in [-1]: self.dims = len(shape) dics = [{"axis": axis}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["input_data"], - "Y": ["weight"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": {"X": ["input_data"], "Y": ["weight"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "weight": - TensorConfig(data_gen=partial(generate_weight)) + "weight": TensorConfig( + data_gen=partial(generate_weight) + ) }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, shape)), + "input_data": TensorConfig( + data_gen=partial(generate_input, shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [32]} self.dynamic_shape.max_input_shape = {"input_data": [64]} @@ -205,19 +214,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-5, 1e-5) + attrs, False + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-5, 1e-5) + attrs, True + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): pass @@ -228,12 +241,10 @@ def test(self): class TrtConvertElementwiseTest_one_input(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -241,47 +252,57 @@ def generate_weight(): return np.random.randn(32).astype(np.float32) for batch in [1, 4]: - for shape in [[32], [batch, 32], [batch, 32, 32], - [batch, 32, 16, 32]]: + for shape in [ + [32], + [batch, 32], + [batch, 32, 32], + [batch, 32, 16, 32], + ]: for op_type in [ - "elementwise_add", "elementwise_mul", "elementwise_sub", - "elementwise_div", "elementwise_pow", "elementwise_min", - "elementwise_max" + "elementwise_add", + "elementwise_mul", + "elementwise_sub", + "elementwise_div", + "elementwise_pow", + "elementwise_min", + "elementwise_max", ]: for axis in [-1 if len(shape) == 1 else 1]: self.dims = len(shape) dics = [{"axis": axis}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["input_data"], - "Y": ["weight"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": { + "X": ["input_data"], + "Y": ["weight"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "weight": - TensorConfig(data_gen=partial(generate_weight)) + "weight": TensorConfig( + data_gen=partial(generate_weight) + ) }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, shape)), + "input_data": TensorConfig( + data_gen=partial(generate_input, shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The input.dims[1] must be equal to the weight's length. if self.dims == 1: @@ -325,19 +346,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-5, 1e-5) + attrs, False + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-5, 1e-5) + attrs, True + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): pass @@ -348,108 +373,112 @@ def test(self): class TrtConvertElementwiseTest_two_input_without_broadcast( - TrtLayerAutoScanTest): - + TrtLayerAutoScanTest +): def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) for shape in [[4], [4, 32], [2, 64, 32], [1, 8, 16, 32]]: for op_type in [ - "elementwise_add", "elementwise_mul", "elementwise_sub", - "elementwise_div", "elementwise_pow", "elementwise_min", - "elementwise_max" + "elementwise_add", + "elementwise_mul", + "elementwise_sub", + "elementwise_div", + "elementwise_pow", + "elementwise_min", + "elementwise_max", ]: for axis in [0, -1]: self.dims = len(shape) dics = [{"axis": axis}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["input_data1"], - "Y": ["input_data2"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": { + "X": ["input_data1"], + "Y": ["input_data2"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data1": - TensorConfig( - data_gen=partial(generate_input, shape)), - "input_data2": - TensorConfig( - data_gen=partial(generate_input, shape)) + "input_data1": TensorConfig( + data_gen=partial(generate_input, shape) + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = { "input_data1": [1], - "input_data2": [1] + "input_data2": [1], } self.dynamic_shape.max_input_shape = { "input_data1": [128], - "input_data2": [128] + "input_data2": [128], } self.dynamic_shape.opt_input_shape = { "input_data1": [32], - "input_data2": [32] + "input_data2": [32], } elif self.dims == 2: self.dynamic_shape.min_input_shape = { "input_data1": [1, 4], - "input_data2": [1, 4] + "input_data2": [1, 4], } self.dynamic_shape.max_input_shape = { "input_data1": [128, 256], - "input_data2": [128, 256] + "input_data2": [128, 256], } self.dynamic_shape.opt_input_shape = { "input_data1": [32, 64], - "input_data2": [32, 64] + "input_data2": [32, 64], } elif self.dims == 3: self.dynamic_shape.min_input_shape = { "input_data1": [1, 4, 4], - "input_data2": [1, 4, 4] + "input_data2": [1, 4, 4], } self.dynamic_shape.max_input_shape = { "input_data1": [128, 128, 256], - "input_data2": [128, 128, 256] + "input_data2": [128, 128, 256], } self.dynamic_shape.opt_input_shape = { "input_data1": [2, 64, 64], - "input_data2": [2, 64, 64] + "input_data2": [2, 64, 64], } elif self.dims == 4: self.dynamic_shape.min_input_shape = { "input_data1": [1, 4, 4, 4], - "input_data2": [1, 4, 4, 4] + "input_data2": [1, 4, 4, 4], } self.dynamic_shape.max_input_shape = { "input_data1": [8, 128, 64, 128], - "input_data2": [8, 128, 64, 128] + "input_data2": [8, 128, 64, 128], } self.dynamic_shape.opt_input_shape = { "input_data1": [2, 64, 32, 32], - "input_data2": [2, 64, 32, 32] + "input_data2": [2, 64, 32, 32], } def clear_dynamic_shape(): @@ -470,10 +499,12 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-5, 1e-5) + attrs, False + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) @@ -491,7 +522,6 @@ def test(self): class TrtConvertElementwiseTest_two_input_with_broadcast(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs if len(inputs['input_data1'].shape) != len(inputs['input_data2'].shape): @@ -500,7 +530,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -512,8 +541,12 @@ def generate_input(shape): input2_shape5_list = [[32], [2, 1, 32], [4, 1, 1, 32]] input2_shape6_list = [[1, 32], [1, 32], [1, 1, 1, 32]] input2_shape_list = [ - input2_shape1_list, input2_shape2_list, input2_shape3_list, - input2_shape4_list, input2_shape5_list, input2_shape6_list + input2_shape1_list, + input2_shape2_list, + input2_shape3_list, + input2_shape4_list, + input2_shape5_list, + input2_shape6_list, ] axis1_list = [[-1], [1, -1], [1, -1]] axis2_list = [[-1], [0], [0]] @@ -522,8 +555,12 @@ def generate_input(shape): axis5_list = [[-1, 1], [-1, 0], [-1, 0]] axis6_list = [[-1, 0], [-1, 1], [-1, 0]] axis_list = [ - axis1_list, axis2_list, axis3_list, axis4_list, axis5_list, - axis6_list + axis1_list, + axis2_list, + axis3_list, + axis4_list, + axis5_list, + axis6_list, ] for i in range(3): @@ -531,66 +568,75 @@ def generate_input(shape): for j in range(6): input2_shape = input2_shape_list[j][i] for op_type in [ - "elementwise_add", - "elementwise_mul", - "elementwise_sub", - "elementwise_div", - "elementwise_pow", - "elementwise_min", - "elementwise_max", + "elementwise_add", + "elementwise_mul", + "elementwise_sub", + "elementwise_div", + "elementwise_pow", + "elementwise_min", + "elementwise_max", ]: for axis in axis_list[j][i]: self.shape1 = input1_shape self.shape2 = input2_shape dics = [{"axis": axis}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["input_data1"], - "Y": ["input_data2"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": { + "X": ["input_data1"], + "Y": ["input_data2"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data1": - TensorConfig(data_gen=partial( - generate_input, input1_shape)), - "input_data2": - TensorConfig(data_gen=partial( - generate_input, input2_shape)) + "input_data1": TensorConfig( + data_gen=partial( + generate_input, input1_shape + ) + ), + "input_data2": TensorConfig( + data_gen=partial( + generate_input, input2_shape + ) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): - max_shape = [[128], [128, 128], [128, 128, 128], - [128, 128, 128, 128]] + max_shape = [ + [128], + [128, 128], + [128, 128, 128], + [128, 128, 128, 128], + ] min_shape = [[1], [1, 1], [1, 1, 1], [1, 1, 1, 1]] opt_shape = [[32], [32, 32], [32, 32, 32], [32, 32, 32, 32]] self.dynamic_shape.min_input_shape = { "input_data1": min_shape[len(self.shape1) - 1], - "input_data2": min_shape[len(self.shape2) - 1] + "input_data2": min_shape[len(self.shape2) - 1], } self.dynamic_shape.max_input_shape = { "input_data1": max_shape[len(self.shape1) - 1], - "input_data2": max_shape[len(self.shape2) - 1] + "input_data2": max_shape[len(self.shape2) - 1], } self.dynamic_shape.opt_input_shape = { "input_data1": opt_shape[len(self.shape1) - 1], - "input_data2": opt_shape[len(self.shape2) - 1] + "input_data2": opt_shape[len(self.shape2) - 1], } def clear_dynamic_shape(): @@ -626,12 +672,10 @@ def test(self): class TrtConvertElementwiseTest_one_input_corner_case(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -640,52 +684,58 @@ def generate_weight(): return np.random.rand(32).astype(np.float32) for batch in [1, 2, 4]: - for shape in [[32], [batch, 32], [batch, 32, 32], - [batch, 32, 16, 32]]: + for shape in [ + [32], + [batch, 32], + [batch, 32, 32], + [batch, 32, 16, 32], + ]: for op_type in [ - "elementwise_add", - "elementwise_mul", - "elementwise_sub", - "elementwise_div", - "elementwise_pow", - "elementwise_min", - "elementwise_max", + "elementwise_add", + "elementwise_mul", + "elementwise_sub", + "elementwise_div", + "elementwise_pow", + "elementwise_min", + "elementwise_max", ]: self.op_type = op_type for axis in [-1 if len(shape) == 1 else 1]: self.dims = len(shape) dics = [{"axis": axis}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["weight"], - "Y": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": { + "X": ["weight"], + "Y": ["input_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "weight": - TensorConfig(data_gen=partial(generate_weight)) + "weight": TensorConfig( + data_gen=partial(generate_weight) + ) }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, shape)), + "input_data": TensorConfig( + data_gen=partial(generate_input, shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The input.dims[1] must be equal to the weight's length. if self.dims == 1: diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_emb_eltwise_layernorm.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_emb_eltwise_layernorm.py index 12b80e09a5515..c47992b52afe8 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_emb_eltwise_layernorm.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_emb_eltwise_layernorm.py @@ -22,15 +22,14 @@ class TrtConvertEmbEltwiseLayernormTest1(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch, input_size): - return np.random.randint(0, 7, size=(batch, input_size, - 1)).astype(np.int64) + return np.random.randint(0, 7, size=(batch, input_size, 1)).astype( + np.int64 + ) def generate_weight1(size11, size2): return np.random.randn(size11, size2).astype(np.float32) @@ -56,180 +55,228 @@ def generate_weight4(size2): for axis1 in [0, -1]: for axis2 in [0, -1]: for type in [ - "lookup_table", - "lookup_table_v2" + "lookup_table", + "lookup_table_v2", ]: - dics = [{ - "is_sparse": False, - "is_distributed": False, - "padding_idx": -1, - "is_test": True - }, { - "is_sparse": False, - "is_distributed": False, - "padding_idx": -1, - }, { - "axis": axis1 - }, { - "axis": axis2 - }, { - "begin_norm_axis": norm_axis, - "epsilon": epsilon - }] - ops_config = [{ - "op_type": - type, - "op_inputs": { - "Ids": ["input_data1"], - "W": ["embedding1_weight"] - }, - "op_outputs": { - "Out": - ["embedding1_output"] + dics = [ + { + "is_sparse": False, + "is_distributed": False, + "padding_idx": -1, + "is_test": True, }, - "op_attrs": - dics[0] if type - == "lookup_table" else dics[1] - }, { - "op_type": - type, - "op_inputs": { - "Ids": ["input_data2"], - "W": ["embedding2_weight"] + { + "is_sparse": False, + "is_distributed": False, + "padding_idx": -1, }, - "op_outputs": { - "Out": - ["embedding2_output"] + {"axis": axis1}, + {"axis": axis2}, + { + "begin_norm_axis": norm_axis, + "epsilon": epsilon, }, - "op_attrs": - dics[0] if type - == "lookup_table" else dics[1] - }, { - "op_type": - type, - "op_inputs": { - "Ids": ["input_data3"], - "W": ["embedding3_weight"] + ] + ops_config = [ + { + "op_type": type, + "op_inputs": { + "Ids": ["input_data1"], + "W": [ + "embedding1_weight" + ], + }, + "op_outputs": { + "Out": [ + "embedding1_output" + ] + }, + "op_attrs": dics[0] + if type == "lookup_table" + else dics[1], }, - "op_outputs": { - "Out": - ["embedding3_output"] + { + "op_type": type, + "op_inputs": { + "Ids": ["input_data2"], + "W": [ + "embedding2_weight" + ], + }, + "op_outputs": { + "Out": [ + "embedding2_output" + ] + }, + "op_attrs": dics[0] + if type == "lookup_table" + else dics[1], }, - "op_attrs": - dics[0] if type - == "lookup_table" else dics[1] - }, { - "op_type": "elementwise_add", - "op_inputs": { - "X": ["embedding2_output"], - "Y": ["embedding3_output"] + { + "op_type": type, + "op_inputs": { + "Ids": ["input_data3"], + "W": [ + "embedding3_weight" + ], + }, + "op_outputs": { + "Out": [ + "embedding3_output" + ] + }, + "op_attrs": dics[0] + if type == "lookup_table" + else dics[1], }, - "op_outputs": { - "Out": - ["elementwise_add1_output"] + { + "op_type": "elementwise_add", + "op_inputs": { + "X": [ + "embedding2_output" + ], + "Y": [ + "embedding3_output" + ], + }, + "op_outputs": { + "Out": [ + "elementwise_add1_output" + ] + }, + "op_attrs": dics[2], }, - "op_attrs": dics[2] - }, { - "op_type": "elementwise_add", - "op_inputs": { - "X": - ["elementwise_add1_output"], - "Y": ["embedding1_output"] + { + "op_type": "elementwise_add", + "op_inputs": { + "X": [ + "elementwise_add1_output" + ], + "Y": [ + "embedding1_output" + ], + }, + "op_outputs": { + "Out": [ + "elementwise_add2_output" + ] + }, + "op_attrs": dics[3], }, - "op_outputs": { - "Out": - ["elementwise_add2_output"] + { + "op_type": "layer_norm", + "op_inputs": { + "X": [ + "elementwise_add2_output" + ], + "Bias": [ + "layer_norm_bias" + ], + "Scale": [ + "layer_norm_scale" + ], + }, + "op_outputs": { + "Y": [ + "layer_norm_output1" + ], + "Mean": [ + "layer_norm_output2" + ], + "Variance": [ + "layer_norm_output3" + ], + }, + "op_attrs": dics[4], }, - "op_attrs": dics[3] - }, { - "op_type": "layer_norm", - "op_inputs": { - "X": - ["elementwise_add2_output"], - "Bias": ["layer_norm_bias"], - "Scale": - ["layer_norm_scale"] - }, - "op_outputs": { - "Y": ["layer_norm_output1"], - "Mean": - ["layer_norm_output2"], - "Variance": - ["layer_norm_output3"] - }, - "op_attrs": dics[4] - }] + ] ops = self.generate_op_config( - ops_config) + ops_config + ) program_config = ProgramConfig( ops=ops, weights={ - "embedding1_weight": - TensorConfig( + "embedding1_weight": TensorConfig( data_gen=partial( generate_weight1, - size11, size2)), - "embedding2_weight": - TensorConfig( + size11, + size2, + ) + ), + "embedding2_weight": TensorConfig( data_gen=partial( generate_weight2, - size12, size2)), - "embedding3_weight": - TensorConfig( + size12, + size2, + ) + ), + "embedding3_weight": TensorConfig( data_gen=partial( generate_weight3, - size13, size2)), - "layer_norm_bias": - TensorConfig( + size13, + size2, + ) + ), + "layer_norm_bias": TensorConfig( data_gen=partial( generate_weight4, - size2)), - "layer_norm_scale": - TensorConfig( + size2, + ) + ), + "layer_norm_scale": TensorConfig( data_gen=partial( generate_weight4, - size2)) + size2, + ) + ), }, inputs={ - "input_data1": - TensorConfig( + "input_data1": TensorConfig( data_gen=partial( generate_input, - batch, input_size)), - "input_data2": - TensorConfig( + batch, + input_size, + ) + ), + "input_data2": TensorConfig( data_gen=partial( generate_input, - batch, input_size)), - "input_data3": - TensorConfig( + batch, + input_size, + ) + ), + "input_data3": TensorConfig( data_gen=partial( generate_input, - batch, input_size)) + batch, + input_size, + ) + ), }, - outputs=["layer_norm_output1"]) + outputs=["layer_norm_output1"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data1": [1, 4, 1], "input_data2": [1, 4, 1], - "input_data3": [1, 4, 1] + "input_data3": [1, 4, 1], } self.dynamic_shape.max_input_shape = { "input_data1": [4, 512, 1], "input_data2": [4, 512, 1], - "input_data3": [4, 512, 1] + "input_data3": [4, 512, 1], } self.dynamic_shape.opt_input_shape = { "input_data1": [2, 128, 1], "input_data2": [2, 128, 1], - "input_data3": [2, 128, 1] + "input_data3": [2, 128, 1], } def clear_dynamic_shape(): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_equal.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_equal.py index 4ca0da9532bc4..ee73586e6e96a 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_equal.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_equal.py @@ -22,7 +22,6 @@ class TrtConvertElementwiseTest_one_input_corner_case(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -35,7 +34,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -44,86 +42,84 @@ def generate_input(shape): for axis in [-1 if len(shape) == 1 else 1]: self.dims = len(shape) dics = [{"axis": axis}, {"in_dtype": 0, "out_dtype": 5}] - ops_config = [{ - "op_type": "equal", - "op_inputs": { - "X": ["input_data1"], - "Y": ["input_data2"] - }, - "op_outputs": { - "Out": ["compare_output_data"] + ops_config = [ + { + "op_type": "equal", + "op_inputs": { + "X": ["input_data1"], + "Y": ["input_data2"], + }, + "op_outputs": {"Out": ["compare_output_data"]}, + "op_attrs": dics[0], }, - "op_attrs": dics[0] - }, { - "op_type": "cast", - "op_inputs": { - "X": ["compare_output_data"] + { + "op_type": "cast", + "op_inputs": {"X": ["compare_output_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[1], }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[1] - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data1": - TensorConfig( - data_gen=partial(generate_input, shape)), - "input_data2": - TensorConfig( - data_gen=partial(generate_input, shape)) + "input_data1": TensorConfig( + data_gen=partial(generate_input, shape) + ), + "input_data2": TensorConfig( + data_gen=partial(generate_input, shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The input.dims[1] must be equal to the weight's length. if self.dims == 2: self.dynamic_shape.min_input_shape = { "input_data1": [1, 1], - "input_data2": [1, 1] + "input_data2": [1, 1], } self.dynamic_shape.max_input_shape = { "input_data1": [4, 1], - "input_data2": [4, 1] + "input_data2": [4, 1], } self.dynamic_shape.opt_input_shape = { "input_data1": [2, 1], - "input_data2": [2, 1] + "input_data2": [2, 1], } elif self.dims == 3: self.dynamic_shape.min_input_shape = { "input_data1": [1, 1, 4], - "input_data2": [1, 1, 4] + "input_data2": [1, 1, 4], } self.dynamic_shape.max_input_shape = { "input_data1": [4, 1, 256], - "input_data2": [1, 1, 256] + "input_data2": [1, 1, 256], } self.dynamic_shape.opt_input_shape = { "input_data1": [2, 1, 16], - "input_data2": [2, 1, 16] + "input_data2": [2, 1, 16], } elif self.dims == 4: self.dynamic_shape.min_input_shape = { "input_data1": [1, 1, 4, 4], - "input_data2": [1, 1, 4, 4] + "input_data2": [1, 1, 4, 4], } self.dynamic_shape.max_input_shape = { "input_data1": [4, 1, 128, 256], - "input_data2": [4, 1, 128, 256] + "input_data2": [4, 1, 128, 256], } self.dynamic_shape.opt_input_shape = { "input_data1": [2, 1, 32, 16], - "input_data2": [2, 1, 32, 16] + "input_data2": [2, 1, 32, 16], } def clear_dynamic_shape(): @@ -144,19 +140,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_expand_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_expand_v2.py index 781264531d625..6eb0228103590 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_expand_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_expand_v2.py @@ -22,7 +22,6 @@ class TrtConvertExpandV2Test(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -35,7 +34,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): if self.dims == 4: self.input_shape = [1, 1, 4, 6] @@ -69,29 +67,31 @@ def generate_shapeT2_data(attrs: List[Dict[str, Any]]): self.dims = dims dics_intput = [{"X": ["expand_v2_input"]}] - ops_config = [{ - "op_type": "expand_v2", - "op_inputs": dics_intput[0], - "op_outputs": { - "Out": ["expand_v2_out"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "expand_v2", + "op_inputs": dics_intput[0], + "op_outputs": {"Out": ["expand_v2_out"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "expand_v2_input": - TensorConfig(data_gen=partial(generate_input1, dics)) + "expand_v2_input": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["expand_v2_out"]) + outputs=["expand_v2_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { @@ -146,19 +146,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass @@ -169,7 +173,6 @@ def test(self): class TrtConvertExpandV2Test2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -177,7 +180,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): if self.dims == 1: self.input_shape = [1] @@ -192,18 +194,13 @@ def generate_input1(attrs: List[Dict[str, Any]]): ] self.dims = dims dics_intput = [ - { - "X": ["expand_v2_input"], - "Shape": ["shapeT1_data"] - }, + {"X": ["expand_v2_input"], "Shape": ["shapeT1_data"]}, ] ops_config = [ { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shapeT1_data"] - }, + "op_outputs": {"Out": ["shapeT1_data"]}, "op_attrs": { "dtype": 2, "str_value": "10", @@ -213,10 +210,8 @@ def generate_input1(attrs: List[Dict[str, Any]]): { "op_type": "expand_v2", "op_inputs": dics_intput[0], - "op_outputs": { - "Out": ["expand_v2_out"] - }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["expand_v2_out"]}, + "op_attrs": dics[0], }, ] ops = self.generate_op_config(ops_config) @@ -224,16 +219,18 @@ def generate_input1(attrs: List[Dict[str, Any]]): ops=ops, weights={}, inputs={ - "expand_v2_input": - TensorConfig(data_gen=partial(generate_input1, dics)) + "expand_v2_input": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["expand_v2_out"]) + outputs=["expand_v2_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): if self.dims == 1: self.dynamic_shape.min_input_shape = {"expand_v2_input": [1]} @@ -262,7 +259,6 @@ def test(self): class TrtConvertExpandV2Test3(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -270,7 +266,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): if self.dims == 4: self.input_shape = [1, 1, 4, 6] @@ -291,18 +286,18 @@ def generate_input1(attrs: List[Dict[str, Any]]): { "X": ["expand_v2_input"], "expand_shapes_tensor": [ - "shapeT1_data", "shapeT2_data", "shapeT3_data", - "shapeT4_data" - ] + "shapeT1_data", + "shapeT2_data", + "shapeT3_data", + "shapeT4_data", + ], }, ] ops_config = [ { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shapeT1_data"] - }, + "op_outputs": {"Out": ["shapeT1_data"]}, "op_attrs": { "dtype": 2, "str_value": "10", @@ -312,9 +307,7 @@ def generate_input1(attrs: List[Dict[str, Any]]): { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shapeT2_data"] - }, + "op_outputs": {"Out": ["shapeT2_data"]}, "op_attrs": { "dtype": 2, "str_value": "12", @@ -324,9 +317,7 @@ def generate_input1(attrs: List[Dict[str, Any]]): { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shapeT3_data"] - }, + "op_outputs": {"Out": ["shapeT3_data"]}, "op_attrs": { "dtype": 2, "str_value": "4", @@ -336,9 +327,7 @@ def generate_input1(attrs: List[Dict[str, Any]]): { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shapeT4_data"] - }, + "op_outputs": {"Out": ["shapeT4_data"]}, "op_attrs": { "dtype": 2, "str_value": "6", @@ -348,10 +337,8 @@ def generate_input1(attrs: List[Dict[str, Any]]): { "op_type": "expand_v2", "op_inputs": dics_intput[0], - "op_outputs": { - "Out": ["expand_v2_out"] - }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["expand_v2_out"]}, + "op_attrs": dics[0], }, ] ops = self.generate_op_config(ops_config) @@ -359,16 +346,18 @@ def generate_input1(attrs: List[Dict[str, Any]]): ops=ops, weights={}, inputs={ - "expand_v2_input": - TensorConfig(data_gen=partial(generate_input1, dics)) + "expand_v2_input": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["expand_v2_out"]) + outputs=["expand_v2_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): if self.dims == 4: self.dynamic_shape.min_input_shape = { diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fc.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fc.py index 97045ee8a544e..04763c985887c 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fc.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fc.py @@ -23,10 +23,9 @@ class TrtConvertFcTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: # The output has diff between gpu and trt in CI windows - if (os.name == 'nt'): + if os.name == 'nt': return False return True @@ -34,12 +33,14 @@ def sample_program_configs(self): self.trt_param.workspace_size = 1073741824 def generate_input1(batch, attrs: List[Dict[str, Any]]): - return np.random.random([batch, 3, 64, (int)(attrs[0]["m"] / 2), - 2]).astype(np.float32) + return np.random.random( + [batch, 3, 64, (int)(attrs[0]["m"] / 2), 2] + ).astype(np.float32) def generate_w(batch, attrs: List[Dict[str, Any]]): - return np.random.random([attrs[0]["m"], - attrs[0]["n"]]).astype(np.float32) + return np.random.random([attrs[0]["m"], attrs[0]["n"]]).astype( + np.float32 + ) def generate_bias(batch, attrs: List[Dict[str, Any]]): return np.random.random([attrs[0]["n"]]).astype(np.float32) @@ -53,7 +54,7 @@ def generate_bias(batch, attrs: List[Dict[str, Any]]): "m": m, "n": n, }, - {} + {}, ] ops_config = [ @@ -62,12 +63,10 @@ def generate_bias(batch, attrs: List[Dict[str, Any]]): "op_inputs": { "Input": ["input_data"], "W": ["w_data"], - "Bias": ["bias_data"] - }, - "op_outputs": { - "Out": ["output_data"] + "Bias": ["bias_data"], }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], }, ] @@ -76,24 +75,26 @@ def generate_bias(batch, attrs: List[Dict[str, Any]]): program_config = ProgramConfig( ops=ops, weights={ - "w_data": - TensorConfig(data_gen=partial(generate_w, batch, dics)), - "bias_data": - TensorConfig( - data_gen=partial(generate_bias, batch, dics)) + "w_data": TensorConfig( + data_gen=partial(generate_w, batch, dics) + ), + "bias_data": TensorConfig( + data_gen=partial(generate_bias, batch, dics) + ), }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, batch, dics)), + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, dics) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 3, 32, 16, 2], @@ -121,19 +122,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): # clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() @@ -143,10 +148,9 @@ def test_quant(self): class TrtConvertFcTest2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: # The output has diff between gpu and trt in CI windows - if (os.name == 'nt'): + if os.name == 'nt': return False return True @@ -157,8 +161,9 @@ def generate_input1(batch, attrs: List[Dict[str, Any]]): return np.random.random([batch, 3, 64, 14]).astype(np.float32) def generate_w(batch, attrs: List[Dict[str, Any]]): - return np.random.random([attrs[0]["m"], - attrs[0]["n"]]).astype(np.float32) + return np.random.random([attrs[0]["m"], attrs[0]["n"]]).astype( + np.float32 + ) def generate_bias(batch, attrs: List[Dict[str, Any]]): return np.random.random([attrs[0]["n"]]).astype(np.float32) @@ -172,7 +177,7 @@ def generate_bias(batch, attrs: List[Dict[str, Any]]): "m": m, "n": n, }, - {} + {}, ] ops_config = [ @@ -181,12 +186,10 @@ def generate_bias(batch, attrs: List[Dict[str, Any]]): "op_inputs": { "Input": ["input_data"], "W": ["w_data"], - "Bias": ["bias_data"] + "Bias": ["bias_data"], }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], }, ] @@ -195,24 +198,26 @@ def generate_bias(batch, attrs: List[Dict[str, Any]]): program_config = ProgramConfig( ops=ops, weights={ - "w_data": - TensorConfig(data_gen=partial(generate_w, batch, dics)), - "bias_data": - TensorConfig( - data_gen=partial(generate_bias, batch, dics)) + "w_data": TensorConfig( + data_gen=partial(generate_w, batch, dics) + ), + "bias_data": TensorConfig( + data_gen=partial(generate_bias, batch, dics) + ), }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, batch, dics)), + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, dics) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): self.dynamic_shape.min_input_shape = { "input_data": [1, 3, 32, 14], @@ -277,7 +282,7 @@ def generate_bias(batch, attrs: List[Dict[str, Any]]): "m": m, "n": n, }, - {} + {}, ] ops_config = [ @@ -286,12 +291,10 @@ def generate_bias(batch, attrs: List[Dict[str, Any]]): "op_inputs": { "Input": ["input_data"], "W": ["w_data"], - "Bias": ["bias_data"] + "Bias": ["bias_data"], }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], }, ] @@ -300,24 +303,26 @@ def generate_bias(batch, attrs: List[Dict[str, Any]]): program_config = ProgramConfig( ops=ops, weights={ - "w_data": - TensorConfig(data_gen=partial(generate_w, batch, dics)), - "bias_data": - TensorConfig( - data_gen=partial(generate_bias, batch, dics)) + "w_data": TensorConfig( + data_gen=partial(generate_w, batch, dics) + ), + "bias_data": TensorConfig( + data_gen=partial(generate_bias, batch, dics) + ), }, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, batch, dics)), + "input_data": TensorConfig( + data_gen=partial(generate_input1, batch, dics) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): self.dynamic_shape.min_input_shape = { "input_data": [1, 14, 1, 2], diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fill_constant.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fill_constant.py index 5e7132d74c40b..6e22f5db13c18 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fill_constant.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fill_constant.py @@ -22,12 +22,10 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_value_data(attrs: List[Dict[str, Any]]): return np.array([1]).astype(np.int32) @@ -47,21 +45,28 @@ def generate_shapelist_data(attrs: List[Dict[str, Any]]): str_value = str_value else: str_value = "" - dics = [{ - "str_value": str_value, - "value": value, - "shape": shape, - "dtype": dtype - }, { - "axis": -1 - }] - dics_intput = [{ - "ValueTensor": ["value_data"] - }, { - "ShapeTensor": ["shape_data"], - }, { - "ShapeTensorList": ["shapeT1_data", "shapeT2_data"], - }, {}] + dics = [ + { + "str_value": str_value, + "value": value, + "shape": shape, + "dtype": dtype, + }, + {"axis": -1}, + ] + dics_intput = [ + {"ValueTensor": ["value_data"]}, + { + "ShapeTensor": ["shape_data"], + }, + { + "ShapeTensorList": [ + "shapeT1_data", + "shapeT2_data", + ], + }, + {}, + ] ops_config = [ { "op_type": "fill_constant", @@ -69,7 +74,7 @@ def generate_shapelist_data(attrs: List[Dict[str, Any]]): "op_outputs": { "Out": ["out_data"], }, - "op_attrs": dics[0] + "op_attrs": dics[0], }, ] @@ -81,26 +86,31 @@ def generate_input(): ops=ops, weights={}, inputs={ - "value_data": - TensorConfig(data_gen=partial( - generate_value_data, dics)), - "shape_data": - TensorConfig(data_gen=partial( - generate_shape_data, dics)), - "shapeT1_data": - TensorConfig(data_gen=partial( - generate_shapelist_data, dics)), - "shapeT2_data": - TensorConfig(data_gen=partial( - generate_shapelist_data, dics)), + "value_data": TensorConfig( + data_gen=partial(generate_value_data, dics) + ), + "shape_data": TensorConfig( + data_gen=partial(generate_shape_data, dics) + ), + "shapeT1_data": TensorConfig( + data_gen=partial( + generate_shapelist_data, dics + ) + ), + "shapeT2_data": TensorConfig( + data_gen=partial( + generate_shapelist_data, dics + ) + ), }, - outputs=["out_data"]) + outputs=["out_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.input_shape = [1, 1] max_shape = list(self.input_shape) @@ -118,7 +128,7 @@ def clear_dynamic_shape(): self.dynamic_shape.opt_input_shape = {} def generate_trt_nodes_num(attrs, dynamic_shape): - if (self.num_input < 3): + if self.num_input < 3: return 0, 6 return 1, 5 @@ -131,10 +141,12 @@ def generate_trt_nodes_num(attrs, dynamic_shape): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten.py index 314457e25e9ee..4027b8e1f48b8 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten.py @@ -22,12 +22,10 @@ class TrtConvertFlattenTest_dim_2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch): return np.random.random([batch, 32]).astype(np.float32) @@ -39,34 +37,35 @@ def generate_input(batch): else: op_outputs = { "Out": ["output_data"], - "XShape": ["xshape_data"] + "XShape": ["xshape_data"], } dics = [{"axis": axis}] - ops_config = [{ - "op_type": "flatten", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": op_outputs, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "flatten", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": op_outputs, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, batch)) + "input_data": TensorConfig( + data_gen=partial(generate_input, batch) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 8]} self.dynamic_shape.max_input_shape = {"input_data": [4, 64]} @@ -100,31 +99,33 @@ def generate_trt_nodes_num(attrs, dynamic_shape): # for static_shape clear_dynamic_shape() yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() class TrtConvertFlattenTest_dim_3(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch): return np.random.random([batch, 32, 64]).astype(np.float32) @@ -136,34 +137,35 @@ def generate_input(batch): else: op_outputs = { "Out": ["output_data"], - "XShape": ["xshape_data"] + "XShape": ["xshape_data"], } dics = [{"axis": axis}] - ops_config = [{ - "op_type": "flatten", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": op_outputs, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "flatten", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": op_outputs, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, batch)) + "input_data": TensorConfig( + data_gen=partial(generate_input, batch) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 8, 8]} self.dynamic_shape.max_input_shape = {"input_data": [4, 32, 64]} @@ -198,31 +200,33 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() class TrtConvertFlattenTest_dim_4(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch): return np.random.random([batch, 8, 8, 8]).astype(np.float32) @@ -234,34 +238,35 @@ def generate_input(batch): else: op_outputs = { "Out": ["output_data"], - "XShape": ["xshape_data"] + "XShape": ["xshape_data"], } dics = [{"axis": axis}] - ops_config = [{ - "op_type": "flatten", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": op_outputs, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "flatten", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": op_outputs, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, batch)) + "input_data": TensorConfig( + data_gen=partial(generate_input, batch) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 4, 4, 4]} self.dynamic_shape.max_input_shape = {"input_data": [4, 32, 32, 32]} @@ -296,31 +301,33 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() class TrtConvertFlattenTest_dim_5(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch): return np.random.random([batch, 8, 8, 8]).astype(np.float32) @@ -332,34 +339,35 @@ def generate_input(batch): else: op_outputs = { "Out": ["output_data"], - "XShape": ["xshape_data"] + "XShape": ["xshape_data"], } dics = [{"axis": axis}] - ops_config = [{ - "op_type": "flatten", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": op_outputs, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "flatten", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": op_outputs, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, batch)) + "input_data": TensorConfig( + data_gen=partial(generate_input, batch) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 4, 4, 4]} self.dynamic_shape.max_input_shape = {"input_data": [4, 16, 16, 8]} @@ -394,19 +402,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten_contiguous_range.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten_contiguous_range.py index e39d8bf10e2df..fb27f8e630e00 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten_contiguous_range.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_flatten_contiguous_range.py @@ -22,12 +22,10 @@ class TrtConvertFlattenContiguousRangeTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch): return np.random.random([2, batch, 4, 8, 3]).astype(np.float32) @@ -37,35 +35,36 @@ def generate_input(batch): type = "flatten_contiguous_range" op_outputs = { "Out": ["output_data"], - "XShape": ["xshape_data"] + "XShape": ["xshape_data"], } - ops_config = [{ - "op_type": type, - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": op_outputs, - "op_attrs": { - "start_axis": start_axis, - "stop_axis": stop_axis, + ops_config = [ + { + "op_type": type, + "op_inputs": {"X": ["input_data"]}, + "op_outputs": op_outputs, + "op_attrs": { + "start_axis": start_axis, + "stop_axis": stop_axis, + }, } - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, batch)) + "input_data": TensorConfig( + data_gen=partial(generate_input, batch) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [2, 1, 4, 8, 3]} self.dynamic_shape.max_input_shape = {"input_data": [2, 4, 4, 8, 3]} @@ -97,19 +96,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fused_token_prune.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fused_token_prune.py index 287f0af082369..5130660257c79 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fused_token_prune.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_fused_token_prune.py @@ -22,7 +22,6 @@ class TrtConvertFusedTokenPruneTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -40,65 +39,70 @@ def generate_new_mask(attrs: List[Dict[str, Any]]): for keep_first_token in [True, False]: for keep_order in [True, False]: - dics = [{ - "keep_first_token": keep_first_token, - "keep_order": keep_order - }] - ops_config = [{ - "op_type": "fused_token_prune", - "op_inputs": { - "Attn": ["attn"], - "X": ["x"], - "Mask": ["mask"], - "NewMask": ["new_mask"] - }, - "op_outputs": { - "SlimmedX": ["slimmed_x"], - "CLSInds": ["cls_inds"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "keep_first_token": keep_first_token, + "keep_order": keep_order, + } + ] + ops_config = [ + { + "op_type": "fused_token_prune", + "op_inputs": { + "Attn": ["attn"], + "X": ["x"], + "Mask": ["mask"], + "NewMask": ["new_mask"], + }, + "op_outputs": { + "SlimmedX": ["slimmed_x"], + "CLSInds": ["cls_inds"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "attn": - TensorConfig( - data_gen=partial(generate_attn_or_mask, dics)), - "x": - TensorConfig(data_gen=partial(generate_x, dics)), - "mask": - TensorConfig( - data_gen=partial(generate_attn_or_mask, dics)), - "new_mask": - TensorConfig(data_gen=partial(generate_new_mask, dics)) + "attn": TensorConfig( + data_gen=partial(generate_attn_or_mask, dics) + ), + "x": TensorConfig(data_gen=partial(generate_x, dics)), + "mask": TensorConfig( + data_gen=partial(generate_attn_or_mask, dics) + ), + "new_mask": TensorConfig( + data_gen=partial(generate_new_mask, dics) + ), }, - outputs=["slimmed_x", "cls_inds"]) + outputs=["slimmed_x", "cls_inds"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "attn": [4, 12, 64, 64], "x": [4, 64, 76], "mask": [4, 12, 64, 64], - "new_mask": [4, 12, 32, 32] + "new_mask": [4, 12, 32, 32], } self.dynamic_shape.max_input_shape = { "attn": [4, 12, 64, 64], "x": [4, 64, 76], "mask": [4, 12, 64, 64], - "new_mask": [4, 12, 32, 32] + "new_mask": [4, 12, 32, 32], } self.dynamic_shape.opt_input_shape = { "attn": [4, 12, 64, 64], "x": [4, 64, 76], "mask": [4, 12, 64, 64], - "new_mask": [4, 12, 32, 32] + "new_mask": [4, 12, 32, 32], } def clear_dynamic_shape(): @@ -116,10 +120,12 @@ def generate_trt_nodes_num(attrs, dynamic_shape): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-2, 1e-2) + attrs, True + ), (1e-2, 1e-2) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-1, 1e-2) + attrs, True + ), (1e-1, 1e-2) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather.py index d74cad229284b..ca2984fa18777 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather.py @@ -22,7 +22,6 @@ class TrtConvertGatherTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs attrs = [ @@ -34,7 +33,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(shape): return np.random.random(shape).astype(np.float32) @@ -51,112 +49,126 @@ def generate_input3(axis): for index in [[1, 4], [4, 8]]: for axis in [0, 1, 2, 3]: for overwrite in [True, False]: - for input in [{ - "X": ["input_data"], - "Index": ["index_data"] - }, { + for input in [ + {"X": ["input_data"], "Index": ["index_data"]}, + { "X": ["input_data"], "Index": ["index_data"], - "Axis": ["axis_data"] - }]: + "Axis": ["axis_data"], + }, + ]: for index_type_int32 in [True, False]: self.shape = shape self.axis = axis self.input_num = len(input) self.index_type_int32 = index_type_int32 dics = [{"overwrite": overwrite, "axis": axis}] - ops_config = [{ - "op_type": "gather", - "op_inputs": input, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "gather", + "op_inputs": input, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, shape)), - "index_data": - TensorConfig(data_gen=partial( - generate_input2 - if index_type_int32 == - True else generate_input4, index)), - } if len(input) == 2 else { - "input_data": - TensorConfig(data_gen=partial( - generate_input1, shape)), - "index_data": - TensorConfig(data_gen=partial( - generate_input2, index)), - "axis_data": - TensorConfig(data_gen=partial( - generate_input3, axis)), + "input_data": TensorConfig( + data_gen=partial( + generate_input1, shape + ) + ), + "index_data": TensorConfig( + data_gen=partial( + generate_input2 + if index_type_int32 == True + else generate_input4, + index, + ) + ), + } + if len(input) == 2 + else { + "input_data": TensorConfig( + data_gen=partial( + generate_input1, shape + ) + ), + "index_data": TensorConfig( + data_gen=partial( + generate_input2, index + ) + ), + "axis_data": TensorConfig( + data_gen=partial( + generate_input3, axis + ) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if len(self.shape) == 1: self.dynamic_shape.min_input_shape = { "input_data": [4], - "index_data": [1] + "index_data": [1], } self.dynamic_shape.max_input_shape = { "input_data": [128], - "index_data": [4] + "index_data": [4], } self.dynamic_shape.opt_input_shape = { "input_data": [16], - "index_data": [2] + "index_data": [2], } elif len(self.shape) == 2: self.dynamic_shape.min_input_shape = { "input_data": [2, 4], - "index_data": [1] + "index_data": [1], } self.dynamic_shape.max_input_shape = { "input_data": [256, 256], - "index_data": [4] + "index_data": [4], } self.dynamic_shape.opt_input_shape = { "input_data": [64, 32], - "index_data": [2] + "index_data": [2], } elif len(self.shape) == 3: self.dynamic_shape.min_input_shape = { "input_data": [2, 4, 4], - "index_data": [1] + "index_data": [1], } self.dynamic_shape.max_input_shape = { "input_data": [128, 256, 256], - "index_data": [4] + "index_data": [4], } self.dynamic_shape.opt_input_shape = { "input_data": [16, 64, 32], - "index_data": [2] + "index_data": [2], } elif len(self.shape) == 4: self.dynamic_shape.min_input_shape = { "input_data": [2, 4, 4, 2], - "index_data": [1] + "index_data": [1], } self.dynamic_shape.max_input_shape = { "input_data": [128, 256, 64, 128], - "index_data": [4] + "index_data": [4], } self.dynamic_shape.opt_input_shape = { "input_data": [16, 64, 16, 32], - "index_data": [2] + "index_data": [2], } def clear_dynamic_shape(): @@ -181,10 +193,12 @@ def generate_trt_nodes_num(dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-5 + False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-3 + False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) @@ -200,14 +214,17 @@ def add_skip_trt_case(self): def teller1(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0: inputs = program_config.inputs - if len(inputs['input_data'].shape) == 1 or len( - inputs['index_data'].shape) == 1: + if ( + len(inputs['input_data'].shape) == 1 + or len(inputs['index_data'].shape) == 1 + ): return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_SUPPORT, - "Need to repair the case: trt reshape out failed for dynamic shape mode when inputs' dims==1. under trt7.0 " + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "Need to repair the case: trt reshape out failed for dynamic shape mode when inputs' dims==1. under trt7.0 ", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather_nd.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather_nd.py index ea392357114e7..6ac8314ec7776 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather_nd.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gather_nd.py @@ -23,7 +23,6 @@ class TrtConvertGatherNdTest_dim_4_1(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: # The output has diff between gpu and trt in CI windows # if ( and self.trt_param.precision == paddle_infer.PrecisionType.Half): @@ -31,54 +30,53 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([2, 32, 64, 64]).astype(np.float32) def generate_input2(): return np.ones([1]).astype(np.int32) - ops_config = [{ - "op_type": "gather_nd", - "op_inputs": { - "X": ["input_data"], - "Index": ["index_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "gather_nd", + "op_inputs": {"X": ["input_data"], "Index": ["index_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) for i in range(10): program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1)), - "index_data": - TensorConfig(data_gen=partial(generate_input2)), + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ), + "index_data": TensorConfig( + data_gen=partial(generate_input2) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 8, 8, 8], - "index_data": [1] + "index_data": [1], } self.dynamic_shape.max_input_shape = { "input_data": [4, 32, 64, 64], - "index_data": [1] + "index_data": [1], } self.dynamic_shape.opt_input_shape = { "input_data": [2, 32, 64, 64], - "index_data": [1] + "index_data": [1], } def clear_dynamic_shape(): @@ -105,15 +103,16 @@ def clear_dynamic_shape(): yield self.create_inference_config(), (1, 3), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0 and os.name == 'nt': return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_SUPPORT, - "Under Windows Ci, this case will sporadically fail.") + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "Under Windows Ci, this case will sporadically fail.", + ) def test(self): self.add_skip_trt_case() @@ -121,29 +120,24 @@ def test(self): class TrtConvertGatherNdTest_dim_4_1_2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([2, 32, 64, 64]).astype(np.float32) def generate_input2(): return np.array([1, 2]).astype(np.int32) - ops_config = [{ - "op_type": "gather_nd", - "op_inputs": { - "X": ["input_data"], - "Index": ["index_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "gather_nd", + "op_inputs": {"X": ["input_data"], "Index": ["index_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( @@ -153,25 +147,26 @@ def generate_input2(): "input_data": TensorConfig(data_gen=partial(generate_input1)), "index_data": TensorConfig(data_gen=partial(generate_input2)), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 8, 8, 8], - "index_data": [2] + "index_data": [2], } self.dynamic_shape.max_input_shape = { "input_data": [4, 32, 64, 64], - "index_data": [2] + "index_data": [2], } self.dynamic_shape.opt_input_shape = { "input_data": [2, 32, 64, 64], - "index_data": [2] + "index_data": [2], } def clear_dynamic_shape(): @@ -198,15 +193,16 @@ def clear_dynamic_shape(): yield self.create_inference_config(), (1, 3), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0 and os.name == 'nt': return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_SUPPORT, - "Under Windows Ci, this case will sporadically fail.") + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "Under Windows Ci, this case will sporadically fail.", + ) def test(self): self.add_skip_trt_case() @@ -214,29 +210,24 @@ def test(self): class TrtConvertGatherNdTest_dim_4_2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([2, 32, 64, 64]).astype(np.float32) def generate_input2(): return np.ones([2, 2]).astype(np.int32) - ops_config = [{ - "op_type": "gather_nd", - "op_inputs": { - "X": ["input_data"], - "Index": ["index_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "gather_nd", + "op_inputs": {"X": ["input_data"], "Index": ["index_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( @@ -246,25 +237,26 @@ def generate_input2(): "input_data": TensorConfig(data_gen=partial(generate_input1)), "index_data": TensorConfig(data_gen=partial(generate_input2)), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 8, 8, 8], - "index_data": [2, 2] + "index_data": [2, 2], } self.dynamic_shape.max_input_shape = { "input_data": [4, 32, 64, 64], - "index_data": [2, 2] + "index_data": [2, 2], } self.dynamic_shape.opt_input_shape = { "input_data": [2, 32, 64, 64], - "index_data": [2, 2] + "index_data": [2, 2], } def clear_dynamic_shape(): @@ -291,15 +283,16 @@ def clear_dynamic_shape(): yield self.create_inference_config(), (1, 3), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0 and os.name == 'nt': return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_SUPPORT, - "Under Windows Ci, this case will sporadically fail.") + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "Under Windows Ci, this case will sporadically fail.", + ) def test(self): self.add_skip_trt_case() @@ -307,29 +300,24 @@ def test(self): class TrtConvertGatherNdTest_dim_4_3(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([2, 32, 64, 64]).astype(np.float32) def generate_input2(): return np.ones([2, 2, 4]).astype(np.int32) - ops_config = [{ - "op_type": "gather_nd", - "op_inputs": { - "X": ["input_data"], - "Index": ["index_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "gather_nd", + "op_inputs": {"X": ["input_data"], "Index": ["index_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( @@ -339,25 +327,26 @@ def generate_input2(): "input_data": TensorConfig(data_gen=partial(generate_input1)), "index_data": TensorConfig(data_gen=partial(generate_input2)), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 8, 8, 8], - "index_data": [2, 2, 4] + "index_data": [2, 2, 4], } self.dynamic_shape.max_input_shape = { "input_data": [4, 32, 64, 64], - "index_data": [2, 2, 4] + "index_data": [2, 2, 4], } self.dynamic_shape.opt_input_shape = { "input_data": [2, 32, 64, 64], - "index_data": [2, 2, 4] + "index_data": [2, 2, 4], } def clear_dynamic_shape(): @@ -384,15 +373,16 @@ def clear_dynamic_shape(): yield self.create_inference_config(), (1, 3), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0 and os.name == 'nt': return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_SUPPORT, - "Under Windows Ci, this case will sporadically fail.") + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "Under Windows Ci, this case will sporadically fail.", + ) def test(self): self.add_skip_trt_case() @@ -400,29 +390,24 @@ def test(self): class TrtConvertGatherNdTest_dim_2_2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([2, 32]).astype(np.float32) def generate_input2(): return np.array([[0, 3], [1, 9]]).astype(np.int32) - ops_config = [{ - "op_type": "gather_nd", - "op_inputs": { - "X": ["input_data"], - "Index": ["index_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "gather_nd", + "op_inputs": {"X": ["input_data"], "Index": ["index_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( @@ -432,25 +417,26 @@ def generate_input2(): "input_data": TensorConfig(data_gen=partial(generate_input1)), "index_data": TensorConfig(data_gen=partial(generate_input2)), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 4], - "index_data": [2, 2] + "index_data": [2, 2], } self.dynamic_shape.max_input_shape = { "input_data": [4, 64], - "index_data": [2, 2] + "index_data": [2, 2], } self.dynamic_shape.opt_input_shape = { "input_data": [2, 8], - "index_data": [2, 2] + "index_data": [2, 2], } def clear_dynamic_shape(): @@ -477,15 +463,16 @@ def clear_dynamic_shape(): yield self.create_inference_config(), (1, 3), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0 and os.name == 'nt': return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_SUPPORT, - "Under Windows Ci, this case will sporadically fail.") + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "Under Windows Ci, this case will sporadically fail.", + ) def test(self): self.add_skip_trt_case() @@ -493,30 +480,26 @@ def test(self): class TrtConvertGatherNdTest_dim_3_3(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([16, 32, 256]).astype(np.float32) def generate_input2(): - return np.array([[[2, 5], [3, 8]], [[0, 2], [0, - 3]]]).astype(np.int32) - - ops_config = [{ - "op_type": "gather_nd", - "op_inputs": { - "X": ["input_data"], - "Index": ["index_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + return np.array([[[2, 5], [3, 8]], [[0, 2], [0, 3]]]).astype( + np.int32 + ) + + ops_config = [ + { + "op_type": "gather_nd", + "op_inputs": {"X": ["input_data"], "Index": ["index_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( @@ -526,25 +509,26 @@ def generate_input2(): "input_data": TensorConfig(data_gen=partial(generate_input1)), "index_data": TensorConfig(data_gen=partial(generate_input2)), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 4, 4], - "index_data": [1, 1, 1] + "index_data": [1, 1, 1], } self.dynamic_shape.max_input_shape = { "input_data": [16, 64, 512], - "index_data": [4, 2, 4] + "index_data": [4, 2, 4], } self.dynamic_shape.opt_input_shape = { "input_data": [2, 8, 64], - "index_data": [2, 2, 2] + "index_data": [2, 2, 2], } def clear_dynamic_shape(): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gelu.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gelu.py index 4f8e5c6356abb..d6d2f876361e5 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gelu.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_gelu.py @@ -22,12 +22,10 @@ class TrtConvertGeluTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(dims, attrs: List[Dict[str, Any]]): if dims == 1: return np.ones([32]).astype(np.float32) @@ -43,33 +41,32 @@ def generate_input1(dims, attrs: List[Dict[str, Any]]): self.dims = dims dics = [{"approximate": approximate}] - ops_config = [{ - "op_type": "gelu", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "gelu", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, dims, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dims, dics) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -123,19 +120,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_grid_sampler.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_grid_sampler.py index e04c096d9498b..36b0c1638cb77 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_grid_sampler.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_grid_sampler.py @@ -22,29 +22,27 @@ class TrtConvertGridSampler(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([1, 3, 32, 32]).astype(np.float32) def generate_input2(): return np.random.random([1, 3, 3, 2]).astype(np.float32) - ops_config = [{ - "op_type": "grid_sampler", - "op_inputs": { - "X": ["input_data"], - "Grid": ["grid_data"], - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "grid_sampler", + "op_inputs": { + "X": ["input_data"], + "Grid": ["grid_data"], + }, + "op_outputs": {"Output": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) for i in range(10): @@ -52,30 +50,33 @@ def generate_input2(): ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1)), - "grid_data": - TensorConfig(data_gen=partial(generate_input2)), + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ), + "grid_data": TensorConfig( + data_gen=partial(generate_input2) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 3, 32, 32], - "grid_data": [1, 3, 3, 2] + "grid_data": [1, 3, 3, 2], } self.dynamic_shape.max_input_shape = { "input_data": [1, 3, 64, 64], - "grid_data": [1, 3, 4, 4] + "grid_data": [1, 3, 4, 4], } self.dynamic_shape.opt_input_shape = { "input_data": [1, 3, 32, 32], - "grid_data": [1, 3, 3, 2] + "grid_data": [1, 3, 3, 2], } def clear_dynamic_shape(): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_group_norm.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_group_norm.py index 0c8e6eeb87bab..18f40934172f6 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_group_norm.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_group_norm.py @@ -22,7 +22,6 @@ class TrtConvertGroupNormTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -36,7 +35,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(attrs: List[Dict[str, Any]], batch): if attrs[0]['data_layout'] == 'NCHW': return np.random.random([batch, 32, 64, 64]).astype(np.float32) @@ -53,47 +51,56 @@ def generate_bias(): for group in [1, 4, 32, -1]: for epsilon in [0.0001, 0.0007, -1, 1]: for data_layout in ['NCHW']: - dics = [{ - "epsilon": epsilon, - "groups": group, - "data_layout": data_layout - }] - ops_config = [{ - "op_type": "group_norm", - "op_inputs": { - "X": ["input_data"], - "Scale": ["scale_weight"], - "Bias": ["bias_weight"] - }, - "op_outputs": { - "Y": ["y_output"], - "Mean": ["mean_output"], - "Variance": ["variance_output"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "epsilon": epsilon, + "groups": group, + "data_layout": data_layout, + } + ] + ops_config = [ + { + "op_type": "group_norm", + "op_inputs": { + "X": ["input_data"], + "Scale": ["scale_weight"], + "Bias": ["bias_weight"], + }, + "op_outputs": { + "Y": ["y_output"], + "Mean": ["mean_output"], + "Variance": ["variance_output"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "scale_weight": - TensorConfig(data_gen=partial(generate_scale)), - "bias_weight": - TensorConfig(data_gen=partial(generate_bias)) + "scale_weight": TensorConfig( + data_gen=partial(generate_scale) + ), + "bias_weight": TensorConfig( + data_gen=partial(generate_bias) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input, dics, batch)) + "input_data": TensorConfig( + data_gen=partial( + generate_input, dics, batch + ) + ) }, - outputs=["y_output"]) + outputs=["y_output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 16, 16, 16]} self.dynamic_shape.max_input_shape = { @@ -117,19 +124,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_sigmoid.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_sigmoid.py index 0a1699139c970..3c8cd8948f415 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_sigmoid.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_sigmoid.py @@ -22,12 +22,10 @@ class TrtConvertHardSigmoidTest_dim_2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -37,33 +35,34 @@ def generate_input(shape): for slope in [0.1, 0.5]: for offset in [0.2, 0.7]: dics = [{"slope": slope, "offset": offset}] - ops_config = [{ - "op_type": "hard_sigmoid", - "op_inputs": { - "X": ["input_data"], - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "hard_sigmoid", + "op_inputs": { + "X": ["input_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input, shape)) + "input_data": TensorConfig( + data_gen=partial(generate_input, shape) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.input_dim == 2: self.dynamic_shape.min_input_shape = {"input_data": [1, 8]} diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_swish.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_swish.py index b014d6fa2b0c2..4180c5b083071 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_swish.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_hard_swish.py @@ -22,7 +22,6 @@ class TrtConvertHardSwishTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -36,46 +35,46 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.ones([1, 3, 32, 32]).astype(np.float32) for threshold in [6.0, 7.0, 100.0, 0.0, -1.0]: for scale in [5.0, 7.0, -1.0, 0.0, 100.0]: for offset in [3.0, 5.0, -1.0, 0.0, 100.0]: - dics = [{ - "threshold": threshold, - "scale": scale, - "offset": offset - }] - - ops_config = [{ - "op_type": "hard_swish", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["hard_swish_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "threshold": threshold, + "scale": scale, + "offset": offset, + } + ] + + ops_config = [ + { + "op_type": "hard_swish", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["hard_swish_output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["hard_swish_output_data"]) + outputs=["hard_swish_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 16, 16]} self.dynamic_shape.max_input_shape = {"input_data": [2, 3, 32, 32]} @@ -97,19 +96,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_instance_norm.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_instance_norm.py index 135858fec77c1..b3be8dfdc6653 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_instance_norm.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_instance_norm.py @@ -24,7 +24,6 @@ class TrtConvertInstanceNormTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -36,7 +35,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], shape_input): return np.random.random(shape_input).astype(np.float32) @@ -44,48 +42,60 @@ def generate_input2(attrs: List[Dict[str, Any]], shape_input): return np.random.random(shape_input[1]).astype(np.float32) for batch in [1, 2, 4]: - for shape_input in [[batch, 16], [batch, 32, 64], - [batch, 16, 32, 64]]: + for shape_input in [ + [batch, 16], + [batch, 32, 64], + [batch, 16, 32, 64], + ]: self.in_dim = len(shape_input) for epsilon in [0.0005, -1, 1]: dics = [{"epsilon": epsilon}] - ops_config = [{ - "op_type": "instance_norm", - "op_inputs": { - "X": ["input_data"], - "Scale": ["scale_data"], - "Bias": ["bias_data"] - }, - "op_outputs": { - "Y": ["y_data"], - "SavedMean": ["saved_mean_data"], - "SavedVariance": ["saved_variance_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "instance_norm", + "op_inputs": { + "X": ["input_data"], + "Scale": ["scale_data"], + "Bias": ["bias_data"], + }, + "op_outputs": { + "Y": ["y_data"], + "SavedMean": ["saved_mean_data"], + "SavedVariance": ["saved_variance_data"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "bias_data": - TensorConfig(data_gen=partial( - generate_input2, dics, shape_input)), - "scale_data": - TensorConfig(data_gen=partial( - generate_input2, dics, shape_input)) + "bias_data": TensorConfig( + data_gen=partial( + generate_input2, dics, shape_input + ) + ), + "scale_data": TensorConfig( + data_gen=partial( + generate_input2, dics, shape_input + ) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dics, shape_input)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dics, shape_input + ) + ) }, - outputs=["y_data"]) + outputs=["y_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.in_dim == 2: self.dynamic_shape.min_input_shape = {"input_data": [1, 4]} @@ -128,30 +138,35 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): - def teller2(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0 and os.name == 'nt': return True return False self.add_skip_case( - teller2, SkipReasons.TRT_NOT_SUPPORT, - "The output has diff between gpu and trt in Windows.") + teller2, + SkipReasons.TRT_NOT_SUPPORT, + "The output has diff between gpu and trt in Windows.", + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_inverse.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_inverse.py index 390eff3e1e84a..6ccb00d1a0f51 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_inverse.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_inverse.py @@ -22,41 +22,41 @@ class TrtConvertInverse(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([32, 32]).astype(np.float32) - ops_config = [{ - "op_type": "inverse", - "op_inputs": { - "Input": ["input_data"], - }, - "op_outputs": { - "Output": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "inverse", + "op_inputs": { + "Input": ["input_data"], + }, + "op_outputs": {"Output": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) for i in range(10): program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1)), + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 1], diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_layer_norm.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_layer_norm.py index f6e4dec9538fe..2d52f74be3823 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_layer_norm.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_layer_norm.py @@ -23,7 +23,6 @@ class TrtConvertLayerNormTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -34,13 +33,13 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: if attrs[0]['epsilon'] < 0 or attrs[0]['epsilon'] > 0.001: return False if attrs[0]['begin_norm_axis'] <= 0 or attrs[0]['begin_norm_axis'] >= ( - len(inputs['input_data'].shape) - 1): + len(inputs['input_data'].shape) - 1 + ): return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], shape_input): return np.ones(shape_input).astype(np.float32) @@ -53,49 +52,52 @@ def generate_input2(attrs: List[Dict[str, Any]], shape_input): for epsilon in [0.0005, -1, 1]: for begin_norm_axis in [1, 0, -1, 2, 3]: - dics = [{ - "epsilon": epsilon, - "begin_norm_axis": begin_norm_axis - }, {}] - - ops_config = [{ - "op_type": "layer_norm", - "op_inputs": { - "X": ["input_data"], - "Scale": ["scale_data"], - "Bias": ["bias_data"] - }, - "op_outputs": { - "Y": ["y_data"], - "Mean": ["saved_mean_data"], - "Variance": ["saved_variance_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + {"epsilon": epsilon, "begin_norm_axis": begin_norm_axis}, + {}, + ] + + ops_config = [ + { + "op_type": "layer_norm", + "op_inputs": { + "X": ["input_data"], + "Scale": ["scale_data"], + "Bias": ["bias_data"], + }, + "op_outputs": { + "Y": ["y_data"], + "Mean": ["saved_mean_data"], + "Variance": ["saved_variance_data"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) shape_input = [1, 3, 64, 64] program_config = ProgramConfig( ops=ops, weights={ - "bias_data": - TensorConfig(data_gen=partial(generate_input2, dics, - shape_input)), - "scale_data": - TensorConfig(data_gen=partial(generate_input2, dics, - shape_input)) + "bias_data": TensorConfig( + data_gen=partial(generate_input2, dics, shape_input) + ), + "scale_data": TensorConfig( + data_gen=partial(generate_input2, dics, shape_input) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, dics, - shape_input)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics, shape_input) + ) }, - outputs=["y_data"]) + outputs=["y_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -108,7 +110,7 @@ def clear_dynamic_shape(): def generate_trt_nodes_num(attrs, dynamic_shape): inputs = program_config.inputs - #if not dynamic_shape: + # if not dynamic_shape: # if attrs[0]["begin_norm_axis"] >= len(inputs["input_data"].shape) - 1: # print ("iiiiiii") # return 0, 3 @@ -122,26 +124,29 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 + attrs, True + ), 1e-2 def test(self): self.run_test() class TrtConvertLayerNormTest_2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -152,13 +157,13 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: if attrs[0]['epsilon'] < 0 or attrs[0]['epsilon'] > 0.001: return False if attrs[0]['begin_norm_axis'] <= 0 or attrs[0]['begin_norm_axis'] >= ( - len(inputs['input_data'].shape) - 1): + len(inputs['input_data'].shape) - 1 + ): return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], shape_input): return np.ones(shape_input).astype(np.float32) @@ -171,49 +176,52 @@ def generate_input2(attrs: List[Dict[str, Any]], shape_input): for epsilon in [0.0005, -1, 1]: for begin_norm_axis in [1, 0, -1, 2, 3]: - dics = [{ - "epsilon": epsilon, - "begin_norm_axis": begin_norm_axis - }, {}] - - ops_config = [{ - "op_type": "layer_norm", - "op_inputs": { - "X": ["input_data"], - "Scale": ["scale_data"], - "Bias": ["bias_data"] - }, - "op_outputs": { - "Y": ["y_data"], - "Mean": ["saved_mean_data"], - "Variance": ["saved_variance_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + {"epsilon": epsilon, "begin_norm_axis": begin_norm_axis}, + {}, + ] + + ops_config = [ + { + "op_type": "layer_norm", + "op_inputs": { + "X": ["input_data"], + "Scale": ["scale_data"], + "Bias": ["bias_data"], + }, + "op_outputs": { + "Y": ["y_data"], + "Mean": ["saved_mean_data"], + "Variance": ["saved_variance_data"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) shape_input = [2, 64, 3, 3] program_config = ProgramConfig( ops=ops, weights={ - "bias_data": - TensorConfig(data_gen=partial(generate_input2, dics, - shape_input)), - "scale_data": - TensorConfig(data_gen=partial(generate_input2, dics, - shape_input)) + "bias_data": TensorConfig( + data_gen=partial(generate_input2, dics, shape_input) + ), + "scale_data": TensorConfig( + data_gen=partial(generate_input2, dics, shape_input) + ), }, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, dics, - shape_input)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics, shape_input) + ) }, - outputs=["y_data"]) + outputs=["y_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 64, 3, 3]} self.dynamic_shape.max_input_shape = {"input_data": [4, 64, 3, 3]} @@ -226,7 +234,7 @@ def clear_dynamic_shape(): def generate_trt_nodes_num(attrs, dynamic_shape): inputs = program_config.inputs - #if not dynamic_shape: + # if not dynamic_shape: # if attrs[0]["begin_norm_axis"] >= len(inputs["input_data"].shape) - 1: # print ("iiiiiii") # return 0, 3 @@ -240,19 +248,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 + attrs, True + ), 1e-2 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_leaky_relu.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_leaky_relu.py index 2742a536de97b..93f7115ae4fbb 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_leaky_relu.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_leaky_relu.py @@ -23,12 +23,10 @@ class TrtConvertLeakyReluTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(shape): return np.random.random(shape).astype(np.float32) @@ -37,32 +35,35 @@ def generate_input1(shape): self.input_dim = len(shape) for alpha in [0.02, 1.0, 100.0, -1.0, 0.0]: dics = [{"alpha": alpha}] - ops_config = [{ - "op_type": "leaky_relu", - "op_inputs": { - "X": ["input_data"], - }, - "op_outputs": { - "Out": ["y_data"], - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "leaky_relu", + "op_inputs": { + "X": ["input_data"], + }, + "op_outputs": { + "Out": ["y_data"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, shape)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, shape) + ) }, - outputs=["y_data"]) + outputs=["y_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.input_dim == 2: self.dynamic_shape.min_input_shape = {"input_data": [1, 8]} @@ -101,25 +102,31 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) self.trt_param.precision = paddle_infer.PrecisionType.Int8 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) self.trt_param.precision = paddle_infer.PrecisionType.Int8 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul.py index b4895d805bc62..9175c9b4ae13b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul.py @@ -22,12 +22,10 @@ class TrtConvertMatmulTest_static(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -47,48 +45,55 @@ def generate_input(shape): input1_shape = [batch, 32, 6] input2_shape = [batch, 6, 11] for alpha in [0.3, 1.0]: - dics = [{ - "transpose_X": trans_x, - "transpose_Y": trans_y, - "alpha": alpha, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }] - ops_config = [{ - "op_type": "matmul", - "op_inputs": { - "X": ["input1_data"], - "Y": ["input2_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "transpose_X": trans_x, + "transpose_Y": trans_y, + "alpha": alpha, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + } + ] + ops_config = [ + { + "op_type": "matmul", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1_data": - TensorConfig(data_gen=partial( - generate_input, input1_shape)), - "input2_data": - TensorConfig(data_gen=partial( - generate_input, input2_shape)) + "input1_data": TensorConfig( + data_gen=partial( + generate_input, input1_shape + ) + ), + "input2_data": TensorConfig( + data_gen=partial( + generate_input, input2_shape + ) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): pass @@ -109,12 +114,10 @@ def test(self): class TrtConvertMatmulTest_dynamic(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -133,60 +136,63 @@ def generate_input(shape): # input1_shape = [batch, 32, 6] # input2_shape = [batch, 6, 11] for alpha in [0.3, 1.0]: - dics = [{ - "transpose_X": trans_x, - "transpose_Y": trans_y, - "alpha": alpha, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }] - ops_config = [{ - "op_type": "matmul", - "op_inputs": { - "X": ["input1_data"], - "Y": ["input2_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "transpose_X": trans_x, + "transpose_Y": trans_y, + "alpha": alpha, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + } + ] + ops_config = [ + { + "op_type": "matmul", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1_data": - TensorConfig( - data_gen=partial(generate_input, input1_shape)), - "input2_data": - TensorConfig( - data_gen=partial(generate_input, input2_shape)) + "input1_data": TensorConfig( + data_gen=partial(generate_input, input1_shape) + ), + "input2_data": TensorConfig( + data_gen=partial(generate_input, input2_shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input1_data": [1, 4, 4], - "input2_data": [1, 4, 4] + "input2_data": [1, 4, 4], } self.dynamic_shape.max_input_shape = { "input1_data": [16, 4, 4], - "input2_data": [16, 4, 4] + "input2_data": [16, 4, 4], } self.dynamic_shape.opt_input_shape = { "input1_data": [8, 4, 4], - "input2_data": [8, 4, 4] + "input2_data": [8, 4, 4], } attrs = [ diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py index 524262244d3ce..82d9143a0981f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_matmul_v2.py @@ -23,9 +23,7 @@ class TrtConvertMatmulTest_dynamic(TrtLayerAutoScanTest): - def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -34,53 +32,56 @@ def generate_input(shape): for trans_y in [False]: input1_shape = [batch, 64, 350, 75] input2_shape = [75, 25] - dics = [{ - "trans_x": trans_x, - "trans_y": trans_y, - }] - ops_config = [{ - "op_type": "matmul_v2", - "op_inputs": { - "X": ["input1_data"], - "Y": ["input2_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "trans_x": trans_x, + "trans_y": trans_y, + } + ] + ops_config = [ + { + "op_type": "matmul_v2", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1_data": - TensorConfig( - data_gen=partial(generate_input, input1_shape)), - "input2_data": - TensorConfig( - data_gen=partial(generate_input, input2_shape)) + "input1_data": TensorConfig( + data_gen=partial(generate_input, input1_shape) + ), + "input2_data": TensorConfig( + data_gen=partial(generate_input, input2_shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input1_data": [10, 64, 350, 75], - "input2_data": [75, 25] + "input2_data": [75, 25], } self.dynamic_shape.max_input_shape = { "input1_data": [100, 64, 350, 75], - "input2_data": [75, 25] + "input2_data": [75, 25], } self.dynamic_shape.opt_input_shape = { "input1_data": [15, 64, 350, 75], - "input2_data": [75, 25] + "input2_data": [75, 25], } attrs = [ @@ -90,7 +91,7 @@ def generate_dynamic_shape(attrs): # The output has little diff between gpu and trt in CI-Windows-Inference tol_fp32 = 1e-5 tol_half = 1e-5 - if (os.name == 'nt'): + if os.name == 'nt': tol_fp32 = 1e-3 tol_half = 1e-3 # for dynamic_shape @@ -109,9 +110,7 @@ def test(self): class TrtConvertMatmulTest_dynamic2(TrtLayerAutoScanTest): - def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -120,53 +119,56 @@ def generate_input(shape): for trans_y in [False]: input1_shape = [60, 40] input2_shape = [batch, 40, 90] - dics = [{ - "trans_x": trans_x, - "trans_y": trans_y, - }] - ops_config = [{ - "op_type": "matmul_v2", - "op_inputs": { - "X": ["input1_data"], - "Y": ["input2_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "trans_x": trans_x, + "trans_y": trans_y, + } + ] + ops_config = [ + { + "op_type": "matmul_v2", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1_data": - TensorConfig( - data_gen=partial(generate_input, input1_shape)), - "input2_data": - TensorConfig( - data_gen=partial(generate_input, input2_shape)) + "input1_data": TensorConfig( + data_gen=partial(generate_input, input1_shape) + ), + "input2_data": TensorConfig( + data_gen=partial(generate_input, input2_shape) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input1_data": [60, 40], - "input2_data": [10, 40, 90] + "input2_data": [10, 40, 90], } self.dynamic_shape.max_input_shape = { "input1_data": [60, 40], - "input2_data": [20, 40, 90] + "input2_data": [20, 40, 90], } self.dynamic_shape.opt_input_shape = { "input1_data": [60, 40], - "input2_data": [15, 40, 90] + "input2_data": [15, 40, 90], } attrs = [ @@ -175,7 +177,7 @@ def generate_dynamic_shape(attrs): # The output has little diff between gpu and trt in CI-Windows-Inference tol_fp32 = 1e-5 tol_half = 1e-5 - if (os.name == 'nt'): + if os.name == 'nt': tol_fp32 = 1e-3 tol_half = 1e-3 # for dynamic_shape @@ -194,9 +196,7 @@ def test(self): class TrtConvertMatmulTest_dynamic3(TrtLayerAutoScanTest): - def sample_program_configs(self): - def generate_input(shape): return np.random.random(shape).astype(np.float32) @@ -219,86 +219,95 @@ def generate_input(shape): elif case == 2: input1_shape = [50] input2_shape = [50] - if (case == 0 or case == 1): - dics = [{ - "trans_x": False, - "trans_y": False, - }] - elif (case == 2): - dics = [{ - "trans_x": trans_x, - "trans_y": trans_y, - }] - ops_config = [{ - "op_type": "matmul_v2", - "op_inputs": { - "X": ["input1_data"], - "Y": ["input2_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + if case == 0 or case == 1: + dics = [ + { + "trans_x": False, + "trans_y": False, + } + ] + elif case == 2: + dics = [ + { + "trans_x": trans_x, + "trans_y": trans_y, + } + ] + ops_config = [ + { + "op_type": "matmul_v2", + "op_inputs": { + "X": ["input1_data"], + "Y": ["input2_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1_data": - TensorConfig(data_gen=partial( - generate_input, input1_shape)), - "input2_data": - TensorConfig(data_gen=partial( - generate_input, input2_shape)) + "input1_data": TensorConfig( + data_gen=partial( + generate_input, input1_shape + ) + ), + "input2_data": TensorConfig( + data_gen=partial( + generate_input, input2_shape + ) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): - if (self.case == 0): + if self.case == 0: self.dynamic_shape.min_input_shape = { "input1_data": [20, 50], - "input2_data": [50] + "input2_data": [50], } self.dynamic_shape.max_input_shape = { "input1_data": [30, 50], - "input2_data": [50] + "input2_data": [50], } self.dynamic_shape.opt_input_shape = { "input1_data": [25, 50], - "input2_data": [50] + "input2_data": [50], } - elif (self.case == 1): + elif self.case == 1: self.dynamic_shape.min_input_shape = { "input2_data": [50, 20], - "input1_data": [50] + "input1_data": [50], } self.dynamic_shape.max_input_shape = { "input2_data": [50, 30], - "input1_data": [50] + "input1_data": [50], } self.dynamic_shape.opt_input_shape = { "input2_data": [50, 25], - "input1_data": [50] + "input1_data": [50], } - elif (self.case == 2): + elif self.case == 2: self.dynamic_shape.min_input_shape = { "input2_data": [30], - "input1_data": [50] + "input1_data": [50], } self.dynamic_shape.max_input_shape = { "input2_data": [50], - "input1_data": [50] + "input1_data": [50], } self.dynamic_shape.opt_input_shape = { "input2_data": [50], - "input1_data": [50] + "input1_data": [50], } generate_dynamic_shape() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_mish.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_mish.py index d6d62967cfb5d..baac4943f7c26 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_mish.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_mish.py @@ -21,12 +21,10 @@ class TrtConvertMishTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch, dim1, dim2, dim3): shape = [batch] if dim1 != 0: @@ -51,35 +49,36 @@ def generate_input(batch, dim1, dim2, dim3): if dim1 == 0 and dim2 == 0 and dim3 != 0: continue - ops_config = [{ - "op_type": "mish", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["mish_output_data"] - }, - "op_attrs": { - "threshold": thre + ops_config = [ + { + "op_type": "mish", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["mish_output_data"]}, + "op_attrs": {"threshold": thre}, } - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input, batch, dim1, dim2, - dim3)) + "input_data": TensorConfig( + data_gen=partial( + generate_input, + batch, + dim1, + dim2, + dim3, + ) + ) }, - outputs=["mish_output_data"]) + outputs=["mish_output_data"], + ) yield program_config def sample_predictor_configs(self, program_config): - def generate_dynamic_shape(attrs): if self.dim1 == 0: self.dynamic_shape.min_input_shape = { @@ -139,29 +138,35 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if self.dim1 == 0 and self.dim2 == 0 and self.dim3 == 0: return True return False - self.add_skip_case(teller1, SkipReasons.TRT_NOT_SUPPORT, - "Trt does not support 1-dimensional input.") + self.add_skip_case( + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "Trt does not support 1-dimensional input.", + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms.py index e52eb4d1ce9c7..2d98b339d2ad3 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms.py @@ -22,7 +22,6 @@ class TrtConvertMulticlassNMSTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -39,17 +38,21 @@ def create_inference_config(self, use_trt=True) -> paddle_infer.Config: min_subgraph_size=self.trt_param.min_subgraph_size, precision_mode=self.trt_param.precision, use_static=self.trt_param.use_static, - use_calib_mode=self.trt_param.use_calib_mode) - if len(self.dynamic_shape.min_input_shape - ) != 0 and self.dynamic_shape.min_input_shape.keys( - ) == self.dynamic_shape.max_input_shape.keys( - ) and self.dynamic_shape.min_input_shape.keys( - ) == self.dynamic_shape.opt_input_shape.keys(): + use_calib_mode=self.trt_param.use_calib_mode, + ) + if ( + len(self.dynamic_shape.min_input_shape) != 0 + and self.dynamic_shape.min_input_shape.keys() + == self.dynamic_shape.max_input_shape.keys() + and self.dynamic_shape.min_input_shape.keys() + == self.dynamic_shape.opt_input_shape.keys() + ): config.set_trt_dynamic_shape_info( self.dynamic_shape.min_input_shape, self.dynamic_shape.max_input_shape, self.dynamic_shape.opt_input_shape, - self.dynamic_shape.disable_trt_plugin_fp16) + self.dynamic_shape.disable_trt_plugin_fp16, + ) return config else: config = paddle_infer.Config() @@ -59,15 +62,15 @@ def create_inference_config(self, use_trt=True) -> paddle_infer.Config: return config def sample_program_configs(self): - def generate_boxes(batch, num_boxes): - return np.arange(batch * num_boxes * 4, - dtype=np.float32).reshape([batch, num_boxes, 4]) + return np.arange(batch * num_boxes * 4, dtype=np.float32).reshape( + [batch, num_boxes, 4] + ) def generate_scores(batch, num_boxes, num_classes): - return np.arange(batch * num_classes * num_boxes, - dtype=np.float32).reshape( - [batch, num_classes, num_boxes]) + return np.arange( + batch * num_classes * num_boxes, dtype=np.float32 + ).reshape([batch, num_classes, num_boxes]) # return np.random.rand(batch, num_classes, num_boxes).astype(np.float32) for batch in [1, 2]: @@ -76,46 +79,55 @@ def generate_scores(batch, num_boxes, num_classes): for num_boxes, num_classes in [[80, 100], [40, 200], [20, 400]]: self.num_boxes, self.num_classes = num_boxes, num_classes for score_threshold in [ - 0.01, + 0.01, ]: - ops_config = [{ - "op_type": "multiclass_nms", - "op_inputs": { - "BBoxes": ["input_bboxes"], - "Scores": ["input_scores"], - }, - "op_outputs": { - "Out": ["nms_output_boxes"], - }, - "op_attrs": { - "background_label": -1, - "score_threshold": score_threshold, - "nms_top_k": num_boxes, - "keep_top_k": num_boxes, - "nms_threshold": 0.3, - "normalized": False, - "nms_eta": nms_eta + ops_config = [ + { + "op_type": "multiclass_nms", + "op_inputs": { + "BBoxes": ["input_bboxes"], + "Scores": ["input_scores"], + }, + "op_outputs": { + "Out": ["nms_output_boxes"], + }, + "op_attrs": { + "background_label": -1, + "score_threshold": score_threshold, + "nms_top_k": num_boxes, + "keep_top_k": num_boxes, + "nms_threshold": 0.3, + "normalized": False, + "nms_eta": nms_eta, + }, } - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_bboxes": - TensorConfig(data_gen=partial( - generate_boxes, batch, num_boxes)), - "input_scores": - TensorConfig( - data_gen=partial(generate_scores, batch, - num_boxes, num_classes)) + "input_bboxes": TensorConfig( + data_gen=partial( + generate_boxes, batch, num_boxes + ) + ), + "input_scores": TensorConfig( + data_gen=partial( + generate_scores, + batch, + num_boxes, + num_classes, + ) + ), }, - outputs=["nms_output_boxes"]) + outputs=["nms_output_boxes"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The last dim of input_bboxes should be static. self.dynamic_shape.min_input_shape = { @@ -147,39 +159,53 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 # self.trt_param.precision = paddle_infer.PrecisionType.Half # yield self.create_inference_config(), generate_trt_nodes_num( # attrs, True), (1e-2, 1e-2) - def assert_tensors_near(self, atol: float, rtol: float, - tensor: Dict[str, np.array], - baseline: Dict[str, np.array]): + def assert_tensors_near( + self, + atol: float, + rtol: float, + tensor: Dict[str, np.array], + baseline: Dict[str, np.array], + ): # the order of tensorrt outputs are not consistent with paddle for key, arr in tensor.items(): if key == "nms_output_boxes": basline_arr = np.array( - sorted(baseline[key].reshape((-1, 6)), - key=lambda i: [i[0], i[1]])) + sorted( + baseline[key].reshape((-1, 6)), + key=lambda i: [i[0], i[1]], + ) + ) arr = np.array( - sorted(arr.reshape((-1, 6)), key=lambda i: [i[0], i[1]])) + sorted(arr.reshape((-1, 6)), key=lambda i: [i[0], i[1]]) + ) else: basline_arr = np.array(baseline[key].reshape((-1, 1))) arr = np.array(arr.reshape((-1, 1))) self.assertTrue( basline_arr.shape == arr.shape, - "The output shapes are not equal, the baseline shape is " + - str(basline_arr.shape) + ', but got ' + str(arr.shape)) + "The output shapes are not equal, the baseline shape is " + + str(basline_arr.shape) + + ', but got ' + + str(arr.shape), + ) diff = abs(basline_arr - arr) np.testing.assert_allclose( basline_arr, @@ -187,7 +213,9 @@ def assert_tensors_near(self, atol: float, rtol: float, rtol=rtol, atol=atol, err_msg='Output has diff, Maximum absolute error: {}'.format( - np.amax(diff))) + np.amax(diff) + ), + ) def assert_op_size(self, trt_engine_num, paddle_op_num): # tensorrt op num is not consistent with paddle diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms3.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms3.py index 7b743b00b3d12..3a04993f2cbbd 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms3.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multiclass_nms3.py @@ -22,7 +22,6 @@ class TrtConvertMulticlassNMS3Test(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -39,17 +38,21 @@ def create_inference_config(self, use_trt=True) -> paddle_infer.Config: min_subgraph_size=self.trt_param.min_subgraph_size, precision_mode=self.trt_param.precision, use_static=self.trt_param.use_static, - use_calib_mode=self.trt_param.use_calib_mode) - if len(self.dynamic_shape.min_input_shape - ) != 0 and self.dynamic_shape.min_input_shape.keys( - ) == self.dynamic_shape.max_input_shape.keys( - ) and self.dynamic_shape.min_input_shape.keys( - ) == self.dynamic_shape.opt_input_shape.keys(): + use_calib_mode=self.trt_param.use_calib_mode, + ) + if ( + len(self.dynamic_shape.min_input_shape) != 0 + and self.dynamic_shape.min_input_shape.keys() + == self.dynamic_shape.max_input_shape.keys() + and self.dynamic_shape.min_input_shape.keys() + == self.dynamic_shape.opt_input_shape.keys() + ): config.set_trt_dynamic_shape_info( self.dynamic_shape.min_input_shape, self.dynamic_shape.max_input_shape, self.dynamic_shape.opt_input_shape, - self.dynamic_shape.disable_trt_plugin_fp16) + self.dynamic_shape.disable_trt_plugin_fp16, + ) return config else: config = paddle_infer.Config() @@ -59,15 +62,15 @@ def create_inference_config(self, use_trt=True) -> paddle_infer.Config: return config def sample_program_configs(self): - def generate_boxes(batch, num_boxes): - return np.arange(batch * num_boxes * 4, - dtype=np.float32).reshape([batch, num_boxes, 4]) + return np.arange(batch * num_boxes * 4, dtype=np.float32).reshape( + [batch, num_boxes, 4] + ) def generate_scores(batch, num_boxes, num_classes): - return np.arange(batch * num_classes * num_boxes, - dtype=np.float32).reshape( - [batch, num_classes, num_boxes]) + return np.arange( + batch * num_classes * num_boxes, dtype=np.float32 + ).reshape([batch, num_classes, num_boxes]) # return np.random.rand(batch, num_classes, num_boxes).astype(np.float32) for batch in [1, 2]: @@ -76,51 +79,61 @@ def generate_scores(batch, num_boxes, num_classes): for num_boxes, num_classes in [[80, 100], [40, 200], [20, 400]]: self.num_boxes, self.num_classes = num_boxes, num_classes for score_threshold in [ - 0.01, + 0.01, ]: - ops_config = [{ - "op_type": "multiclass_nms3", - "op_inputs": { - "BBoxes": ["input_bboxes"], - "Scores": ["input_scores"], - }, - "op_outputs": { - "Out": ["nms_output_boxes"], - "Index": ["nms_output_index"], - "NmsRoisNum": ["nms_output_num"] - }, - "op_attrs": { - "background_label": -1, - "score_threshold": score_threshold, - "nms_top_k": num_boxes, - "keep_top_k": num_boxes, - "nms_threshold": 0.3, - "normalized": False, - "nms_eta": nms_eta + ops_config = [ + { + "op_type": "multiclass_nms3", + "op_inputs": { + "BBoxes": ["input_bboxes"], + "Scores": ["input_scores"], + }, + "op_outputs": { + "Out": ["nms_output_boxes"], + "Index": ["nms_output_index"], + "NmsRoisNum": ["nms_output_num"], + }, + "op_attrs": { + "background_label": -1, + "score_threshold": score_threshold, + "nms_top_k": num_boxes, + "keep_top_k": num_boxes, + "nms_threshold": 0.3, + "normalized": False, + "nms_eta": nms_eta, + }, } - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_bboxes": - TensorConfig(data_gen=partial( - generate_boxes, batch, num_boxes)), - "input_scores": - TensorConfig( - data_gen=partial(generate_scores, batch, - num_boxes, num_classes)) + "input_bboxes": TensorConfig( + data_gen=partial( + generate_boxes, batch, num_boxes + ) + ), + "input_scores": TensorConfig( + data_gen=partial( + generate_scores, + batch, + num_boxes, + num_classes, + ) + ), }, outputs=[ - "nms_output_boxes", "nms_output_num", - "nms_output_index" - ]) + "nms_output_boxes", + "nms_output_num", + "nms_output_index", + ], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The last dim of input_bboxes should be static. self.dynamic_shape.min_input_shape = { @@ -152,41 +165,55 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 # self.trt_param.precision = paddle_infer.PrecisionType.Half # yield self.create_inference_config(), generate_trt_nodes_num( # attrs, True), (1e-2, 1e-2) - def assert_tensors_near(self, atol: float, rtol: float, - tensor: Dict[str, np.array], - baseline: Dict[str, np.array]): + def assert_tensors_near( + self, + atol: float, + rtol: float, + tensor: Dict[str, np.array], + baseline: Dict[str, np.array], + ): # the order of tensorrt outputs are not consistent with paddle for key, arr in tensor.items(): if key == "nms_output_index": continue if key == "nms_output_boxes": basline_arr = np.array( - sorted(baseline[key].reshape((-1, 6)), - key=lambda i: [i[0], i[1]])) + sorted( + baseline[key].reshape((-1, 6)), + key=lambda i: [i[0], i[1]], + ) + ) arr = np.array( - sorted(arr.reshape((-1, 6)), key=lambda i: [i[0], i[1]])) + sorted(arr.reshape((-1, 6)), key=lambda i: [i[0], i[1]]) + ) else: basline_arr = np.array(baseline[key].reshape((-1, 1))) arr = np.array(arr.reshape((-1, 1))) self.assertTrue( basline_arr.shape == arr.shape, - "The output shapes are not equal, the baseline shape is " + - str(basline_arr.shape) + ', but got ' + str(arr.shape)) + "The output shapes are not equal, the baseline shape is " + + str(basline_arr.shape) + + ', but got ' + + str(arr.shape), + ) diff = abs(basline_arr - arr) np.testing.assert_allclose( basline_arr, @@ -194,7 +221,9 @@ def assert_tensors_near(self, atol: float, rtol: float, rtol=rtol, atol=atol, err_msg='Output has diff, Maximum absolute error: {}'.format( - np.amax(diff))) + np.amax(diff) + ), + ) def assert_op_size(self, trt_engine_num, paddle_op_num): # tensorrt op num is not consistent with paddle diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multihead_matmul.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multihead_matmul.py index a580828943bb9..db02ce1e0f4e7 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multihead_matmul.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_multihead_matmul.py @@ -22,12 +22,10 @@ class TrtConvertMultiHeadMatmulTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(batch, dim1): return np.random.random((batch, dim1, 768)).astype(np.float32) @@ -44,103 +42,86 @@ def generate_weight2(): self.batch = batch for reshape_shape in [[0, 0, 12, 64]]: for dim1 in [128]: - input2_shapes = [[batch, reshape_shape[2], dim1, dim1], - [batch, 1, 1, dim1]] + input2_shapes = [ + [batch, reshape_shape[2], dim1, dim1], + [batch, 1, 1, dim1], + ] for input2_shape in input2_shapes: for axis in [0]: - dics = [{ - "x_num_col_dims": 2, - "y_num_col_dims": 1 - }, { - "axis": 2 - }, { - "shape": reshape_shape - }, { - "axis": [0, 2, 1, 3] - }, { - "x_num_col_dims": 2, - "y_num_col_dims": 1 - }, { - "axis": 2 - }, { - "shape": reshape_shape - }, { - "axis": [0, 2, 1, 3] - }, { - "x_num_col_dims": 2, - "y_num_col_dims": 1 - }, { - "axis": 2 - }, { - "shape": reshape_shape - }, { - "axis": [0, 2, 1, 3] - }, { - "scale": 0.125, - "bias": 0.0, - "bias_after_scale": True - }, { - "alpha": 1.0, - "transpose_X": False, - "transpose_Y": True, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }, { - "axis": axis - }, { - "axis": -1, - "is_test": True - }, { - "seed": 0, - "dropout_prob": 0.10000000149011612, - "dropout_implementation": "upscale_in_train", - "fix_seed": False, - "is_test": True - }, { - "alpha": 1.0, - "transpose_X": False, - "transpose_Y": False, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }, { - "axis": [0, 2, 1, 3] - }, { - "shape": [0, 0, 768] - }, { - "x_num_col_dims": 2, - "y_num_col_dims": 1 - }] + dics = [ + {"x_num_col_dims": 2, "y_num_col_dims": 1}, + {"axis": 2}, + {"shape": reshape_shape}, + {"axis": [0, 2, 1, 3]}, + {"x_num_col_dims": 2, "y_num_col_dims": 1}, + {"axis": 2}, + {"shape": reshape_shape}, + {"axis": [0, 2, 1, 3]}, + {"x_num_col_dims": 2, "y_num_col_dims": 1}, + {"axis": 2}, + {"shape": reshape_shape}, + {"axis": [0, 2, 1, 3]}, + { + "scale": 0.125, + "bias": 0.0, + "bias_after_scale": True, + }, + { + "alpha": 1.0, + "transpose_X": False, + "transpose_Y": True, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + {"axis": axis}, + {"axis": -1, "is_test": True}, + { + "seed": 0, + "dropout_prob": 0.10000000149011612, + "dropout_implementation": "upscale_in_train", + "fix_seed": False, + "is_test": True, + }, + { + "alpha": 1.0, + "transpose_X": False, + "transpose_Y": False, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + {"axis": [0, 2, 1, 3]}, + {"shape": [0, 0, 768]}, + {"x_num_col_dims": 2, "y_num_col_dims": 1}, + ] ops_config = [ { "op_type": "mul", "op_inputs": { "X": ["input_data1"], - "Y": ["mul1_weight"] + "Y": ["mul1_weight"], }, - "op_outputs": { - "Out": ["mul1_output"] - }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["mul1_output"]}, + "op_attrs": dics[0], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["mul1_output"], - "Y": ["elementwise_add1_weight"] + "Y": ["elementwise_add1_weight"], }, "op_outputs": { "Out": ["elementwise_add1_output"] }, - "op_attrs": dics[1] + "op_attrs": dics[1], }, { "op_type": "reshape2", @@ -149,42 +130,38 @@ def generate_weight2(): }, "op_outputs": { "Out": ["reshape21_output"], - "XShape": ["reshape21_output_xshape"] + "XShape": ["reshape21_output_xshape"], }, - "op_attrs": dics[2] + "op_attrs": dics[2], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["reshape21_output"] - }, + "op_inputs": {"X": ["reshape21_output"]}, "op_outputs": { "Out": ["transpose21_output"], - "XShape": ["transpose21_output_xshape"] + "XShape": ["transpose21_output_xshape"], }, - "op_attrs": dics[3] + "op_attrs": dics[3], }, { "op_type": "mul", "op_inputs": { "X": ["input_data1"], - "Y": ["mul2_weight"] + "Y": ["mul2_weight"], }, - "op_outputs": { - "Out": ["mul2_output"] - }, - "op_attrs": dics[4] + "op_outputs": {"Out": ["mul2_output"]}, + "op_attrs": dics[4], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["mul2_output"], - "Y": ["elementwise_add2_weight"] + "Y": ["elementwise_add2_weight"], }, "op_outputs": { "Out": ["elementwise_add2_output"] }, - "op_attrs": dics[5] + "op_attrs": dics[5], }, { "op_type": "reshape2", @@ -193,42 +170,38 @@ def generate_weight2(): }, "op_outputs": { "Out": ["reshape22_output"], - "XShape": ["reshape22_output_xshape"] + "XShape": ["reshape22_output_xshape"], }, - "op_attrs": dics[6] + "op_attrs": dics[6], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["reshape22_output"] - }, + "op_inputs": {"X": ["reshape22_output"]}, "op_outputs": { "Out": ["transpose22_output"], - "XShape": ["transpose22_output_xshape"] + "XShape": ["transpose22_output_xshape"], }, - "op_attrs": dics[7] + "op_attrs": dics[7], }, { "op_type": "mul", "op_inputs": { "X": ["input_data1"], - "Y": ["mul3_weight"] + "Y": ["mul3_weight"], }, - "op_outputs": { - "Out": ["mul3_output"] - }, - "op_attrs": dics[8] + "op_outputs": {"Out": ["mul3_output"]}, + "op_attrs": dics[8], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["mul3_output"], - "Y": ["elementwise_add3_weight"] + "Y": ["elementwise_add3_weight"], }, "op_outputs": { "Out": ["elementwise_add3_output"] }, - "op_attrs": dics[9] + "op_attrs": dics[9], }, { "op_type": "reshape2", @@ -237,30 +210,26 @@ def generate_weight2(): }, "op_outputs": { "Out": ["reshape23_output"], - "XShape": ["reshape23_output_xshape"] + "XShape": ["reshape23_output_xshape"], }, - "op_attrs": dics[10] + "op_attrs": dics[10], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["reshape23_output"] - }, + "op_inputs": {"X": ["reshape23_output"]}, "op_outputs": { "Out": ["transpose23_output"], - "XShape": ["transpose23_output_xshape"] + "XShape": ["transpose23_output_xshape"], }, - "op_attrs": dics[11] + "op_attrs": dics[11], }, { "op_type": "scale", "op_inputs": { "X": ["transpose23_output"], }, - "op_outputs": { - "Out": ["scale_output"] - }, - "op_attrs": dics[12] + "op_outputs": {"Out": ["scale_output"]}, + "op_attrs": dics[12], }, { "op_type": "matmul", @@ -268,41 +237,35 @@ def generate_weight2(): "X": ["scale_output"], "Y": ["transpose22_output"], }, - "op_outputs": { - "Out": ["matmul1_output"] - }, - "op_attrs": dics[13] + "op_outputs": {"Out": ["matmul1_output"]}, + "op_attrs": dics[13], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["matmul1_output"], - "Y": ["input_data2"] + "Y": ["input_data2"], }, "op_outputs": { "Out": ["elementwise_add4_output"] }, - "op_attrs": dics[14] + "op_attrs": dics[14], }, { "op_type": "softmax", "op_inputs": { "X": ["elementwise_add4_output"] }, - "op_outputs": { - "Out": ["softmax_output"] - }, - "op_attrs": dics[15] + "op_outputs": {"Out": ["softmax_output"]}, + "op_attrs": dics[15], }, { "op_type": "dropout", "op_inputs": { "X": ["softmax_output"], }, - "op_outputs": { - "Out": ["dropout3_output"] - }, - "op_attrs": dics[16] + "op_outputs": {"Out": ["dropout3_output"]}, + "op_attrs": dics[16], }, { "op_type": "matmul", @@ -310,32 +273,26 @@ def generate_weight2(): "X": ["dropout3_output"], "Y": ["transpose21_output"], }, - "op_outputs": { - "Out": ["matmul2_output"] - }, - "op_attrs": dics[17] + "op_outputs": {"Out": ["matmul2_output"]}, + "op_attrs": dics[17], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["matmul2_output"] - }, + "op_inputs": {"X": ["matmul2_output"]}, "op_outputs": { "Out": ["transpose24_output"], - "XShape": ["transpose24_output_xshape"] + "XShape": ["transpose24_output_xshape"], }, - "op_attrs": dics[18] + "op_attrs": dics[18], }, { "op_type": "reshape2", - "op_inputs": { - "X": ["transpose24_output"] - }, + "op_inputs": {"X": ["transpose24_output"]}, "op_outputs": { "Out": ["reshape24_output"], - "XShape": ["reshape24_output_xshape"] + "XShape": ["reshape24_output_xshape"], }, - "op_attrs": dics[19] + "op_attrs": dics[19], }, # In order to fuse ops with # multihead_matmul_fuse_pass_v2, the last op @@ -344,72 +301,75 @@ def generate_weight2(): "op_type": "mul", "op_inputs": { "X": ["reshape24_output"], - "Y": ["mul4_weight"] + "Y": ["mul4_weight"], }, - "op_outputs": { - "Out": ["mul4_output"] - }, - "op_attrs": dics[20] - } + "op_outputs": {"Out": ["mul4_output"]}, + "op_attrs": dics[20], + }, ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "mul1_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "mul2_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "mul3_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "mul4_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "elementwise_add1_weight": - TensorConfig( - data_gen=partial(generate_weight2)), - "elementwise_add2_weight": - TensorConfig( - data_gen=partial(generate_weight2)), - "elementwise_add3_weight": - TensorConfig( - data_gen=partial(generate_weight2)), + "mul1_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "mul2_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "mul3_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "mul4_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "elementwise_add1_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), + "elementwise_add2_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), + "elementwise_add3_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), }, inputs={ - "input_data1": - TensorConfig(data_gen=partial( - generate_input1, batch, dim1)), - "input_data2": - TensorConfig(data_gen=partial( - generate_input2, input2_shape)), + "input_data1": TensorConfig( + data_gen=partial( + generate_input1, batch, dim1 + ) + ), + "input_data2": TensorConfig( + data_gen=partial( + generate_input2, input2_shape + ) + ), }, - outputs=["mul4_output"]) + outputs=["mul4_output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The last dim of input1 and input2 should be static. self.dynamic_shape.min_input_shape = { "input_data1": [1, 8, 768], "input_data2": [1, 1, 1, 128], - "reshape24_output": [1, 128, 768] + "reshape24_output": [1, 128, 768], } self.dynamic_shape.max_input_shape = { "input_data1": [16, 512, 768], "input_data2": [16, 256, 512, 128], - "reshape24_output": [1, 128, 768] + "reshape24_output": [1, 128, 768], } self.dynamic_shape.opt_input_shape = { "input_data1": [8, 128, 768], "input_data2": [8, 32, 64, 128], - "reshape24_output": [1, 128, 768] + "reshape24_output": [1, 128, 768], } def clear_dynamic_shape(): @@ -438,25 +398,30 @@ def clear_dynamic_shape(): yield self.create_inference_config(), (1, 3), (1e-3, 1e-3) def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if self.trt_param.precision == paddle_infer.PrecisionType.Half: return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_IMPLEMENTED, - "The output has diff between gpu and trt in fp16 mode.") + teller1, + SkipReasons.TRT_NOT_IMPLEMENTED, + "The output has diff between gpu and trt in fp16 mode.", + ) def teller2(program_config, predictor_config): - if self.trt_param.precision == paddle_infer.PrecisionType.Float32 and len( - self.dynamic_shape.min_input_shape) != 0 and self.batch > 2: + if ( + self.trt_param.precision == paddle_infer.PrecisionType.Float32 + and len(self.dynamic_shape.min_input_shape) != 0 + and self.batch > 2 + ): return True return False self.add_skip_case( - teller2, SkipReasons.TRT_NOT_IMPLEMENTED, - "The output has diff between gpu and trt when dynamic fp32 mode and batch size > 2." + teller2, + SkipReasons.TRT_NOT_IMPLEMENTED, + "The output has diff between gpu and trt when dynamic fp32 mode and batch size > 2.", ) def teller3(program_config, predictor_config): @@ -465,8 +430,10 @@ def teller3(program_config, predictor_config): return False self.add_skip_case( - teller3, SkipReasons.TRT_NOT_IMPLEMENTED, - "The output has diff between gpu and trt in int8 mode.") + teller3, + SkipReasons.TRT_NOT_IMPLEMENTED, + "The output has diff between gpu and trt in int8 mode.", + ) def test(self): self.add_skip_trt_case() @@ -474,9 +441,7 @@ def test(self): class TrtConvertMultiHeadMatmulTestInt8(TrtConvertMultiHeadMatmulTest): - def sample_program_configs(self): - def generate_input1(batch, dim1): return np.random.random((batch, dim1, 768)).astype(np.float32) @@ -493,112 +458,110 @@ def generate_weight2(): self.batch = batch for reshape_shape in [[0, 0, 12, 64]]: for dim1 in [128]: - input2_shapes = [[batch, reshape_shape[2], dim1, dim1], - [batch, 1, 1, dim1]] + input2_shapes = [ + [batch, reshape_shape[2], dim1, dim1], + [batch, 1, 1, dim1], + ] for input2_shape in input2_shapes: for axis in [0]: - dics = [{ - "x_num_col_dims": 2, - "y_num_col_dims": 1, - "enable_int8": True, - "Input_scale": 1.0, - }, { - "axis": 2, - "out_threshold": 1.0, - }, { - "shape": reshape_shape - }, { - "axis": [0, 2, 1, 3] - }, { - "x_num_col_dims": 2, - "y_num_col_dims": 1, - "enable_int8": True, - "Input_scale": 1.0, - }, { - "axis": 2, - "out_threshold": 1.0, - }, { - "shape": reshape_shape - }, { - "axis": [0, 2, 1, 3] - }, { - "x_num_col_dims": 2, - "y_num_col_dims": 1, - "enable_int8": True, - "Input_scale": 1.0, - }, { - "axis": 2, - "out_threshold": 1.0, - }, { - "shape": reshape_shape - }, { - "axis": [0, 2, 1, 3] - }, { - "scale": 0.125, - "bias": 0.0, - "bias_after_scale": True - }, { - "alpha": 1.0, - "transpose_X": False, - "transpose_Y": True, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }, { - "axis": axis - }, { - "axis": -1, - "is_test": True - }, { - "seed": 0, - "dropout_prob": 0.10000000149011612, - "dropout_implementation": "upscale_in_train", - "fix_seed": False, - "is_test": True - }, { - "alpha": 1.0, - "transpose_X": False, - "transpose_Y": False, - "fused_reshape_X": [], - "fused_reshape_Y": [], - "fused_transpose_X": [], - "fused_transpose_Y": [], - "fused_reshape_Out": [], - "fused_transpose_Out": [] - }, { - "axis": [0, 2, 1, 3] - }, { - "shape": [0, 0, 768] - }, { - "x_num_col_dims": 2, - "y_num_col_dims": 1 - }] + dics = [ + { + "x_num_col_dims": 2, + "y_num_col_dims": 1, + "enable_int8": True, + "Input_scale": 1.0, + }, + { + "axis": 2, + "out_threshold": 1.0, + }, + {"shape": reshape_shape}, + {"axis": [0, 2, 1, 3]}, + { + "x_num_col_dims": 2, + "y_num_col_dims": 1, + "enable_int8": True, + "Input_scale": 1.0, + }, + { + "axis": 2, + "out_threshold": 1.0, + }, + {"shape": reshape_shape}, + {"axis": [0, 2, 1, 3]}, + { + "x_num_col_dims": 2, + "y_num_col_dims": 1, + "enable_int8": True, + "Input_scale": 1.0, + }, + { + "axis": 2, + "out_threshold": 1.0, + }, + {"shape": reshape_shape}, + {"axis": [0, 2, 1, 3]}, + { + "scale": 0.125, + "bias": 0.0, + "bias_after_scale": True, + }, + { + "alpha": 1.0, + "transpose_X": False, + "transpose_Y": True, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + {"axis": axis}, + {"axis": -1, "is_test": True}, + { + "seed": 0, + "dropout_prob": 0.10000000149011612, + "dropout_implementation": "upscale_in_train", + "fix_seed": False, + "is_test": True, + }, + { + "alpha": 1.0, + "transpose_X": False, + "transpose_Y": False, + "fused_reshape_X": [], + "fused_reshape_Y": [], + "fused_transpose_X": [], + "fused_transpose_Y": [], + "fused_reshape_Out": [], + "fused_transpose_Out": [], + }, + {"axis": [0, 2, 1, 3]}, + {"shape": [0, 0, 768]}, + {"x_num_col_dims": 2, "y_num_col_dims": 1}, + ] ops_config = [ { "op_type": "mul", "op_inputs": { "X": ["input_data1"], - "Y": ["mul1_weight"] - }, - "op_outputs": { - "Out": ["mul1_output"] + "Y": ["mul1_weight"], }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["mul1_output"]}, + "op_attrs": dics[0], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["mul1_output"], - "Y": ["elementwise_add1_weight"] + "Y": ["elementwise_add1_weight"], }, "op_outputs": { "Out": ["elementwise_add1_output"] }, - "op_attrs": dics[1] + "op_attrs": dics[1], }, { "op_type": "reshape2", @@ -607,42 +570,38 @@ def generate_weight2(): }, "op_outputs": { "Out": ["reshape21_output"], - "XShape": ["reshape21_output_xshape"] + "XShape": ["reshape21_output_xshape"], }, - "op_attrs": dics[2] + "op_attrs": dics[2], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["reshape21_output"] - }, + "op_inputs": {"X": ["reshape21_output"]}, "op_outputs": { "Out": ["transpose21_output"], - "XShape": ["transpose21_output_xshape"] + "XShape": ["transpose21_output_xshape"], }, - "op_attrs": dics[3] + "op_attrs": dics[3], }, { "op_type": "mul", "op_inputs": { "X": ["input_data1"], - "Y": ["mul2_weight"] - }, - "op_outputs": { - "Out": ["mul2_output"] + "Y": ["mul2_weight"], }, - "op_attrs": dics[4] + "op_outputs": {"Out": ["mul2_output"]}, + "op_attrs": dics[4], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["mul2_output"], - "Y": ["elementwise_add2_weight"] + "Y": ["elementwise_add2_weight"], }, "op_outputs": { "Out": ["elementwise_add2_output"] }, - "op_attrs": dics[5] + "op_attrs": dics[5], }, { "op_type": "reshape2", @@ -651,42 +610,38 @@ def generate_weight2(): }, "op_outputs": { "Out": ["reshape22_output"], - "XShape": ["reshape22_output_xshape"] + "XShape": ["reshape22_output_xshape"], }, - "op_attrs": dics[6] + "op_attrs": dics[6], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["reshape22_output"] - }, + "op_inputs": {"X": ["reshape22_output"]}, "op_outputs": { "Out": ["transpose22_output"], - "XShape": ["transpose22_output_xshape"] + "XShape": ["transpose22_output_xshape"], }, - "op_attrs": dics[7] + "op_attrs": dics[7], }, { "op_type": "mul", "op_inputs": { "X": ["input_data1"], - "Y": ["mul3_weight"] - }, - "op_outputs": { - "Out": ["mul3_output"] + "Y": ["mul3_weight"], }, - "op_attrs": dics[8] + "op_outputs": {"Out": ["mul3_output"]}, + "op_attrs": dics[8], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["mul3_output"], - "Y": ["elementwise_add3_weight"] + "Y": ["elementwise_add3_weight"], }, "op_outputs": { "Out": ["elementwise_add3_output"] }, - "op_attrs": dics[9] + "op_attrs": dics[9], }, { "op_type": "reshape2", @@ -695,30 +650,26 @@ def generate_weight2(): }, "op_outputs": { "Out": ["reshape23_output"], - "XShape": ["reshape23_output_xshape"] + "XShape": ["reshape23_output_xshape"], }, - "op_attrs": dics[10] + "op_attrs": dics[10], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["reshape23_output"] - }, + "op_inputs": {"X": ["reshape23_output"]}, "op_outputs": { "Out": ["transpose23_output"], - "XShape": ["transpose23_output_xshape"] + "XShape": ["transpose23_output_xshape"], }, - "op_attrs": dics[11] + "op_attrs": dics[11], }, { "op_type": "scale", "op_inputs": { "X": ["transpose23_output"], }, - "op_outputs": { - "Out": ["scale_output"] - }, - "op_attrs": dics[12] + "op_outputs": {"Out": ["scale_output"]}, + "op_attrs": dics[12], }, { "op_type": "matmul", @@ -726,41 +677,35 @@ def generate_weight2(): "X": ["scale_output"], "Y": ["transpose22_output"], }, - "op_outputs": { - "Out": ["matmul1_output"] - }, - "op_attrs": dics[13] + "op_outputs": {"Out": ["matmul1_output"]}, + "op_attrs": dics[13], }, { "op_type": "elementwise_add", "op_inputs": { "X": ["matmul1_output"], - "Y": ["input_data2"] + "Y": ["input_data2"], }, "op_outputs": { "Out": ["elementwise_add4_output"] }, - "op_attrs": dics[14] + "op_attrs": dics[14], }, { "op_type": "softmax", "op_inputs": { "X": ["elementwise_add4_output"] }, - "op_outputs": { - "Out": ["softmax_output"] - }, - "op_attrs": dics[15] + "op_outputs": {"Out": ["softmax_output"]}, + "op_attrs": dics[15], }, { "op_type": "dropout", "op_inputs": { "X": ["softmax_output"], }, - "op_outputs": { - "Out": ["dropout3_output"] - }, - "op_attrs": dics[16] + "op_outputs": {"Out": ["dropout3_output"]}, + "op_attrs": dics[16], }, { "op_type": "matmul", @@ -768,32 +713,26 @@ def generate_weight2(): "X": ["dropout3_output"], "Y": ["transpose21_output"], }, - "op_outputs": { - "Out": ["matmul2_output"] - }, - "op_attrs": dics[17] + "op_outputs": {"Out": ["matmul2_output"]}, + "op_attrs": dics[17], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["matmul2_output"] - }, + "op_inputs": {"X": ["matmul2_output"]}, "op_outputs": { "Out": ["transpose24_output"], - "XShape": ["transpose24_output_xshape"] + "XShape": ["transpose24_output_xshape"], }, - "op_attrs": dics[18] + "op_attrs": dics[18], }, { "op_type": "reshape2", - "op_inputs": { - "X": ["transpose24_output"] - }, + "op_inputs": {"X": ["transpose24_output"]}, "op_outputs": { "Out": ["reshape24_output"], - "XShape": ["reshape24_output_xshape"] + "XShape": ["reshape24_output_xshape"], }, - "op_attrs": dics[19] + "op_attrs": dics[19], }, # In order to fuse ops with # multihead_matmul_fuse_pass_v2, the last op @@ -802,61 +741,62 @@ def generate_weight2(): "op_type": "mul", "op_inputs": { "X": ["reshape24_output"], - "Y": ["mul4_weight"] - }, - "op_outputs": { - "Out": ["mul4_output"] + "Y": ["mul4_weight"], }, - "op_attrs": dics[20] - } + "op_outputs": {"Out": ["mul4_output"]}, + "op_attrs": dics[20], + }, ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "mul1_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "mul2_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "mul3_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "mul4_weight": - TensorConfig( - data_gen=partial(generate_weight1)), - "elementwise_add1_weight": - TensorConfig( - data_gen=partial(generate_weight2)), - "elementwise_add2_weight": - TensorConfig( - data_gen=partial(generate_weight2)), - "elementwise_add3_weight": - TensorConfig( - data_gen=partial(generate_weight2)), + "mul1_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "mul2_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "mul3_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "mul4_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "elementwise_add1_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), + "elementwise_add2_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), + "elementwise_add3_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), }, inputs={ - "input_data1": - TensorConfig(data_gen=partial( - generate_input1, batch, dim1)), - "input_data2": - TensorConfig(data_gen=partial( - generate_input2, input2_shape)), + "input_data1": TensorConfig( + data_gen=partial( + generate_input1, batch, dim1 + ) + ), + "input_data2": TensorConfig( + data_gen=partial( + generate_input2, input2_shape + ) + ), }, - outputs=["mul4_output"]) + outputs=["mul4_output"], + ) yield program_config class TrtConvertVitToMultiHeadMatmulTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(batch, length): return np.zeros((batch, length, 768), dtype=np.float32) @@ -870,216 +810,190 @@ def generate_weight2(): self.batch = batch for length in [64, 384]: self.length = length - ops_config = [{ - "op_type": "matmul_v2", - "op_inputs": { - "X": ["input_data1"], - "Y": ["matmul1_weight"] - }, - "op_outputs": { - "Out": ["matmul1_output"] - }, - "op_attrs": { - "trans_x": False, - "trans_y": False - } - }, { - "op_type": "elementwise_add", - "op_inputs": { - "X": ["matmul1_output"], - "Y": ["elementwise_add1_weight"] - }, - "op_outputs": { - "Out": ["elementwise_add1_output"] + ops_config = [ + { + "op_type": "matmul_v2", + "op_inputs": { + "X": ["input_data1"], + "Y": ["matmul1_weight"], + }, + "op_outputs": {"Out": ["matmul1_output"]}, + "op_attrs": {"trans_x": False, "trans_y": False}, }, - "op_attrs": { - "Scale_out": 1.0, - "Scale_x": 1.0, - "Scale_y": 1.0, - "axis": 2 - } - }, { - "op_type": "reshape2", - "op_inputs": { - "X": ["elementwise_add1_output"], + { + "op_type": "elementwise_add", + "op_inputs": { + "X": ["matmul1_output"], + "Y": ["elementwise_add1_weight"], + }, + "op_outputs": {"Out": ["elementwise_add1_output"]}, + "op_attrs": { + "Scale_out": 1.0, + "Scale_x": 1.0, + "Scale_y": 1.0, + "axis": 2, + }, }, - "op_outputs": { - "Out": ["reshape1_output"], - "XShape": ["reshape1_output_xshape"] + { + "op_type": "reshape2", + "op_inputs": { + "X": ["elementwise_add1_output"], + }, + "op_outputs": { + "Out": ["reshape1_output"], + "XShape": ["reshape1_output_xshape"], + }, + "op_attrs": {"shape": [-1, self.length, 3, 12, 64]}, }, - "op_attrs": { - "shape": [-1, self.length, 3, 12, 64] - } - }, { - "op_type": "transpose2", - "op_inputs": { - "X": ["reshape1_output"] + { + "op_type": "transpose2", + "op_inputs": {"X": ["reshape1_output"]}, + "op_outputs": { + "Out": ["transpose1_output"], + "XShape": ["transpose1_output_xshape"], + }, + "op_attrs": { + "axis": [2, 0, 3, 1, 4], + "data_format": "AnyLayout", + }, }, - "op_outputs": { - "Out": ["transpose1_output"], - "XShape": ["transpose1_output_xshape"] + { + "op_type": "slice", + "op_inputs": { + "Input": ["transpose1_output"], + }, + "op_outputs": {"Out": ["slice1_output"]}, + "op_attrs": { + "axes": [0], + "starts": [0], + "ends": [1], + "decrease_axis": [0], + "infer_flags": [1], + }, }, - "op_attrs": { - "axis": [2, 0, 3, 1, 4], - "data_format": "AnyLayout" - } - }, { - "op_type": "slice", - "op_inputs": { - "Input": ["transpose1_output"], + { + "op_type": "slice", + "op_inputs": { + "Input": ["transpose1_output"], + }, + "op_outputs": {"Out": ["slice2_output"]}, + "op_attrs": { + "axes": [0], + "starts": [1], + "ends": [2], + "decrease_axis": [0], + "infer_flags": [1], + }, }, - "op_outputs": { - "Out": ["slice1_output"] + { + "op_type": "slice", + "op_inputs": { + "Input": ["transpose1_output"], + }, + "op_outputs": {"Out": ["slice3_output"]}, + "op_attrs": { + "axes": [0], + "starts": [2], + "ends": [3], + "decrease_axis": [0], + "infer_flags": [1], + }, }, - "op_attrs": { - "axes": [0], - "starts": [0], - "ends": [1], - "decrease_axis": [0], - "infer_flags": [1] - } - }, { - "op_type": "slice", - "op_inputs": { - "Input": ["transpose1_output"], + { + "op_type": "transpose2", + "op_inputs": {"X": ["slice2_output"]}, + "op_outputs": { + "Out": ["transpose2_output"], + }, + "op_attrs": { + "axis": [0, 1, 3, 2], + "data_format": "AnyLayout", + }, }, - "op_outputs": { - "Out": ["slice2_output"] + { + "op_type": "matmul_v2", + "op_inputs": { + "X": ["slice1_output"], + "Y": ["transpose2_output"], + }, + "op_outputs": {"Out": ["matmul2_output"]}, + "op_attrs": {"trans_x": False, "trans_y": False}, }, - "op_attrs": { - "axes": [0], - "starts": [1], - "ends": [2], - "decrease_axis": [0], - "infer_flags": [1] - } - }, { - "op_type": "slice", - "op_inputs": { - "Input": ["transpose1_output"], + { + "op_type": "scale", + "op_inputs": { + "X": ["matmul2_output"], + }, + "op_outputs": {"Out": ["scale_output"]}, + "op_attrs": { + "scale": 0.125, + "bias": 0.0, + "bias_after_scale": True, + }, }, - "op_outputs": { - "Out": ["slice3_output"] + { + "op_type": "softmax", + "op_inputs": {"X": ["scale_output"]}, + "op_outputs": {"Out": ["softmax_output"]}, + "op_attrs": {"axis": -1, "data_format": "AnyLayout"}, }, - "op_attrs": { - "axes": [0], - "starts": [2], - "ends": [3], - "decrease_axis": [0], - "infer_flags": [1] - } - }, { - "op_type": "transpose2", - "op_inputs": { - "X": ["slice2_output"] + { + "op_type": "matmul_v2", + "op_inputs": { + "X": ["softmax_output"], + "Y": ["slice3_output"], + }, + "op_outputs": {"Out": ["matmul3_output"]}, + "op_attrs": {"trans_x": False, "trans_y": False}, }, - "op_outputs": { - "Out": ["transpose2_output"], + { + "op_type": "transpose2", + "op_inputs": {"X": ["matmul3_output"]}, + "op_outputs": { + "Out": ["transpose3_output"], + "XShape": ["transpose3_output_xshape"], + }, + "op_attrs": { + "axis": [0, 2, 1, 3], + "data_format": "AnyLayout", + }, }, - "op_attrs": { - "axis": [0, 1, 3, 2], - "data_format": "AnyLayout" - } - }, { - "op_type": "matmul_v2", - "op_inputs": { - "X": ["slice1_output"], - "Y": ["transpose2_output"] + { + "op_type": "reshape2", + "op_inputs": {"X": ["transpose3_output"]}, + "op_outputs": { + "Out": ["reshape2_output"], + "XShape": ["reshape2_output_xshape"], + }, + "op_attrs": {"shape": [-1, self.length, 768]}, }, - "op_outputs": { - "Out": ["matmul2_output"] - }, - "op_attrs": { - "trans_x": False, - "trans_y": False - } - }, { - "op_type": "scale", - "op_inputs": { - "X": ["matmul2_output"], - }, - "op_outputs": { - "Out": ["scale_output"] - }, - "op_attrs": { - "scale": 0.125, - "bias": 0.0, - "bias_after_scale": True - } - }, { - "op_type": "softmax", - "op_inputs": { - "X": ["scale_output"] - }, - "op_outputs": { - "Out": ["softmax_output"] - }, - "op_attrs": { - "axis": -1, - "data_format": "AnyLayout" - } - }, { - "op_type": "matmul_v2", - "op_inputs": { - "X": ["softmax_output"], - "Y": ["slice3_output"] - }, - "op_outputs": { - "Out": ["matmul3_output"] - }, - "op_attrs": { - "trans_x": False, - "trans_y": False - } - }, { - "op_type": "transpose2", - "op_inputs": { - "X": ["matmul3_output"] - }, - "op_outputs": { - "Out": ["transpose3_output"], - "XShape": ["transpose3_output_xshape"] - }, - "op_attrs": { - "axis": [0, 2, 1, 3], - "data_format": "AnyLayout" - } - }, { - "op_type": "reshape2", - "op_inputs": { - "X": ["transpose3_output"] - }, - "op_outputs": { - "Out": ["reshape2_output"], - "XShape": ["reshape2_output_xshape"] - }, - "op_attrs": { - "shape": [-1, self.length, 768] - } - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "matmul1_weight": - TensorConfig(data_gen=partial(generate_weight1)), - "elementwise_add1_weight": - TensorConfig(data_gen=partial(generate_weight2)) + "matmul1_weight": TensorConfig( + data_gen=partial(generate_weight1) + ), + "elementwise_add1_weight": TensorConfig( + data_gen=partial(generate_weight2) + ), }, inputs={ - "input_data1": - TensorConfig( - data_gen=partial(generate_input1, batch, length)) + "input_data1": TensorConfig( + data_gen=partial(generate_input1, batch, length) + ) }, - outputs=["reshape2_output"]) + outputs=["reshape2_output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): # The last dim of input1 and input2 should be static. self.dynamic_shape.min_input_shape = { @@ -1111,11 +1025,15 @@ def generate_trt_nodes_num(): generate_dynamic_shape(attrs) self.trt_param.workspace_size = 2013265920 self.trt_param.precision = paddle_infer.PrecisionType.Half - yield self.create_inference_config(), generate_trt_nodes_num(), (1e-3, - 1e-3) + yield self.create_inference_config(), generate_trt_nodes_num(), ( + 1e-3, + 1e-3, + ) self.trt_param.precision = paddle_infer.PrecisionType.Float32 - yield self.create_inference_config(), generate_trt_nodes_num(), (1e-5, - 1e-5) + yield self.create_inference_config(), generate_trt_nodes_num(), ( + 1e-5, + 1e-5, + ) def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp.py index 71893db5e5f18..1df42024992cc 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp.py @@ -22,7 +22,6 @@ class TrtConvertNearestInterpTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -30,8 +29,9 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - if attrs[0]['scale'] <= 0 and (attrs[0]['out_h'] <= 0 - or attrs[0]['out_w'] <= 0): + if attrs[0]['scale'] <= 0 and ( + attrs[0]['out_h'] <= 0 or attrs[0]['out_w'] <= 0 + ): return False if (attrs[0]['out_h'] <= 0) ^ (attrs[0]['out_w'] <= 0): return False @@ -39,7 +39,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.ones([1, 3, 64, 64]).astype(np.float32) @@ -49,42 +48,49 @@ def generate_input1(attrs: List[Dict[str, Any]]): for scale in [2.0, -1.0, 0.0]: for out_h in [32, 64, 128 - 32]: for out_w in [32, -32]: - dics = [{ - "data_layout": data_layout, - "interp_method": interp_method, - "align_corners": align_corners, - "scale": scale, - "out_h": out_h, - "out_w": out_w - }] - - ops_config = [{ - "op_type": "nearest_interp", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["nearest_interp_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "data_layout": data_layout, + "interp_method": interp_method, + "align_corners": align_corners, + "scale": scale, + "out_h": out_h, + "out_w": out_w, + } + ] + + ops_config = [ + { + "op_type": "nearest_interp", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": { + "Out": [ + "nearest_interp_output_data" + ] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dics + ) + ) }, - outputs=["nearest_interp_output_data"]) + outputs=["nearest_interp_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -106,33 +112,39 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 + attrs, True + ), 1e-2 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): - if program_config.ops[0].attrs[ - 'scale'] <= 0 and self.dynamic_shape.min_input_shape: + if ( + program_config.ops[0].attrs['scale'] <= 0 + and self.dynamic_shape.min_input_shape + ): return True if program_config.ops[0].attrs['align_corners'] == True: return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_IMPLEMENTED, - "NOT Implemented: we need to add support scale <= 0 in dynamic shape in the future" + teller1, + SkipReasons.TRT_NOT_IMPLEMENTED, + "NOT Implemented: we need to add support scale <= 0 in dynamic shape in the future", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp_v2.py index a8156e37f9b60..775db8a3c5955 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_nearest_interp_v2.py @@ -21,47 +21,44 @@ class TrtConvertNearestInterpV2Test(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(): return np.ones([1, 3, 32, 32]).astype(np.float32) - ops_config = [{ - "op_type": "nearest_interp_v2", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["interp_output_data"] - }, - "op_attrs": { - "data_layout": "NCHW", - "interp_method": "nearest", - "align_corners": False, - "align_mode": 1, - "scale": [2., 2.], - "out_d": 0, - "out_h": 0, - "out_w": 0 + ops_config = [ + { + "op_type": "nearest_interp_v2", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["interp_output_data"]}, + "op_attrs": { + "data_layout": "NCHW", + "interp_method": "nearest", + "align_corners": False, + "align_mode": 1, + "scale": [2.0, 2.0], + "out_d": 0, + "out_h": 0, + "out_w": 0, + }, } - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={"input_data": TensorConfig(data_gen=generate_input)}, - outputs=["interp_output_data"]) + outputs=["interp_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -83,19 +80,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 + attrs, True + ), 1e-2 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad.py index dafb78961810c..12262a83022e3 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad.py @@ -22,7 +22,6 @@ class TrtConvertPadTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -39,7 +38,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.ones([1, 3, 64, 64]).astype(np.float32) @@ -47,37 +45,40 @@ def generate_weight1(attrs: List[Dict[str, Any]]): return np.random.random([24, 3, 3, 3]).astype(np.float32) for pad_value in [0.0, 1.0, 2.0, -100, 100.0]: - for paddings in [[0, 0, 0, 0, 1, 1, 1, 1], [0, 0, 0, 0, 1, 2, 3, 4], - [0, 0, 1, 1, 1, 1, 1, 1], - [0, 0, 0, 0, -1, -1, 1, 1]]: + for paddings in [ + [0, 0, 0, 0, 1, 1, 1, 1], + [0, 0, 0, 0, 1, 2, 3, 4], + [0, 0, 1, 1, 1, 1, 1, 1], + [0, 0, 0, 0, -1, -1, 1, 1], + ]: dics = [{"pad_value": pad_value, "paddings": paddings}, {}] - ops_config = [{ - "op_type": "pad", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["pad_output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "pad", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["pad_output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["pad_output_data"]) + outputs=["pad_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -102,22 +103,25 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-2 + attrs, False + ), 1e-2 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 + attrs, True + ), 1e-2 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): for x in range(len(program_config.ops[0].attrs['paddings']) - 4): if program_config.ops[0].attrs['paddings'][x] != 0: @@ -125,8 +129,9 @@ def teller1(program_config, predictor_config): return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_IMPLEMENTED, - "NOT Implemented: we need to add support pad not only inplement on h or w, such as paddings = [0, 0, 1, 1, 1, 1, 1, 1]" + teller1, + SkipReasons.TRT_NOT_IMPLEMENTED, + "NOT Implemented: we need to add support pad not only inplement on h or w, such as paddings = [0, 0, 1, 1, 1, 1, 1, 1]", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad3d.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad3d.py index d44da5534e6a2..02429bed44c04 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad3d.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pad3d.py @@ -22,30 +22,30 @@ class TrtConvertPad3d(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.ones([1, 1, 3, 64, 64]).astype(np.float32) for value in [True, False]: - for paddings in [[0, 0, 0, 0, 1, 1], [0, 0, 1, 2, 3, 4], - [1, 1, 1, 1, 1, 1], [0, 0, -1, -1, 1, 1]]: + for paddings in [ + [0, 0, 0, 0, 1, 1], + [0, 0, 1, 2, 3, 4], + [1, 1, 1, 1, 1, 1], + [0, 0, -1, -1, 1, 1], + ]: dics = [{"value": value, "paddings": paddings}, {}] - ops_config = [{ - "op_type": "pad3d", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "pad3d", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) for i in range(10): @@ -53,16 +53,18 @@ def generate_input1(): ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1)), + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 1, 3, 64, 64] diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pool2d.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pool2d.py index 49d24e863ca12..b4eacbb136f06 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pool2d.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_pool2d.py @@ -23,7 +23,6 @@ class TrtConvertPool2dTest(TrtLayerAutoScanTest): - def is_paddings_valid(self, program_config: ProgramConfig) -> bool: exclusive = program_config.ops[0].attrs['exclusive'] paddings = program_config.ops[0].attrs['paddings'] @@ -65,39 +64,54 @@ def generate_weight1(attrs: List[Dict[str, Any]]): ceil_mode_options = [True, False] configurations = [ - strides_options, paddings_options, pooling_type_options, - padding_algorithm_options, ksize_options, data_format_options, - global_pooling_options, exclusive_options, adaptive_option, - ceil_mode_options + strides_options, + paddings_options, + pooling_type_options, + padding_algorithm_options, + ksize_options, + data_format_options, + global_pooling_options, + exclusive_options, + adaptive_option, + ceil_mode_options, ] - for (strides, paddings, pooling_type, padding_algorithm, ksize, - data_format, global_pooling, exclusive, adaptive, - ceil_mode) in itertools.product(*configurations): - - attrs = [{ - "strides": strides, - "paddings": paddings, - "pooling_type": pooling_type, - "padding_algorithm": padding_algorithm, - "ksize": ksize, - "data_format": data_format, - "global_pooling": global_pooling, - "exclusive": exclusive, - "adaptive": adaptive, - "ceil_mode": ceil_mode, - }] - - ops_config = [{ - "op_type": "pool2d", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": attrs[0] - }] + for ( + strides, + paddings, + pooling_type, + padding_algorithm, + ksize, + data_format, + global_pooling, + exclusive, + adaptive, + ceil_mode, + ) in itertools.product(*configurations): + + attrs = [ + { + "strides": strides, + "paddings": paddings, + "pooling_type": pooling_type, + "padding_algorithm": padding_algorithm, + "ksize": ksize, + "data_format": data_format, + "global_pooling": global_pooling, + "exclusive": exclusive, + "adaptive": adaptive, + "ceil_mode": ceil_mode, + } + ] + + ops_config = [ + { + "op_type": "pool2d", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": attrs[0], + } + ] ops = self.generate_op_config(ops_config) @@ -105,16 +119,18 @@ def generate_weight1(attrs: List[Dict[str, Any]]): ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, attrs)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, attrs) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [1, 3, 64, 64]} @@ -136,34 +152,40 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): - def teller(program_config, predictor_config): - if program_config.ops[0].attrs['pooling_type'] == 'avg' and \ - program_config.ops[0].attrs['global_pooling'] == False and \ - program_config.ops[0].attrs['exclusive'] == True and \ - program_config.ops[0].attrs['adaptive'] == False and \ - program_config.ops[0].attrs['ceil_mode'] == True: + if ( + program_config.ops[0].attrs['pooling_type'] == 'avg' + and program_config.ops[0].attrs['global_pooling'] == False + and program_config.ops[0].attrs['exclusive'] == True + and program_config.ops[0].attrs['adaptive'] == False + and program_config.ops[0].attrs['ceil_mode'] == True + ): return True return False self.add_skip_case( - teller, SkipReasons.TRT_NOT_IMPLEMENTED, - "The results of some cases are Nan, but the results of TensorRT and GPU are the same." + teller, + SkipReasons.TRT_NOT_IMPLEMENTED, + "The results of some cases are Nan, but the results of TensorRT and GPU are the same.", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_bias.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_bias.py index 284fd9231d192..9390d12ca8b11 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_bias.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_bias.py @@ -22,7 +22,6 @@ class TrtConvertSkipLayernormTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -32,14 +31,13 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - #The input dimension should be less than or equal to the set axis. + # The input dimension should be less than or equal to the set axis. if 'begin_norm_axis' in attrs[0] and attrs[0]['begin_norm_axis'] >= 0: if len(inputs['inputX_data'].shape) <= attrs[0]['begin_norm_axis']: return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): return np.ones([batch, 128, 768]).astype(np.float32) @@ -56,96 +54,100 @@ def generate_weight2(attrs: List[Dict[str, Any]]): for epsilon in [1e-5]: for begin_norm_axis in [2]: for enable_int8 in [False, True]: - dics = [{ - "epsilon": epsilon, - "begin_norm_axis": begin_norm_axis, - }, {}] - - ops_config = [{ - "op_type": "elementwise_add", - "op_inputs": { - "X": ["inputX_data"], - "Y": ["EleBias"] - }, - "op_outputs": { - "Out": ["bias_out"] + dics = [ + { + "epsilon": epsilon, + "begin_norm_axis": begin_norm_axis, }, - "op_attrs": { - "axis": -1 - } - }, { - "op_type": "elementwise_add", - "op_inputs": { - "X": ["bias_out"], - "Y": ["inputY_data"] + {}, + ] + + ops_config = [ + { + "op_type": "elementwise_add", + "op_inputs": { + "X": ["inputX_data"], + "Y": ["EleBias"], + }, + "op_outputs": {"Out": ["bias_out"]}, + "op_attrs": {"axis": -1}, }, - "op_outputs": { - "Out": ["ele_out"] + { + "op_type": "elementwise_add", + "op_inputs": { + "X": ["bias_out"], + "Y": ["inputY_data"], + }, + "op_outputs": {"Out": ["ele_out"]}, + "op_attrs": {"axis": -1}, }, - "op_attrs": { - "axis": -1 - } - }, { - "op_type": "layer_norm", - "op_inputs": { - "X": ["ele_out"], - "Bias": ["Bias"], - "Scale": ["Scale"] + { + "op_type": "layer_norm", + "op_inputs": { + "X": ["ele_out"], + "Bias": ["Bias"], + "Scale": ["Scale"], + }, + "op_outputs": { + "Y": ["layernorm_out"], + "Mean": ["Mean"], + "Variance": ["Variance"], + }, + "op_attrs": dics[0], }, - "op_outputs": { - "Y": ["layernorm_out"], - "Mean": ["Mean"], - "Variance": ["Variance"] - }, - "op_attrs": dics[0] - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "Bias": - TensorConfig( - data_gen=partial(generate_weight1, dics)), - "Scale": - TensorConfig( - data_gen=partial(generate_weight2, dics)), - "EleBias": - TensorConfig( - data_gen=partial(generate_weight2, dics)) + "Bias": TensorConfig( + data_gen=partial(generate_weight1, dics) + ), + "Scale": TensorConfig( + data_gen=partial(generate_weight2, dics) + ), + "EleBias": TensorConfig( + data_gen=partial(generate_weight2, dics) + ), }, inputs={ - "inputX_data": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)), - "inputY_data": - TensorConfig(data_gen=partial( - generate_input2, dics, batch)) + "inputX_data": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ), + "inputY_data": TensorConfig( + data_gen=partial( + generate_input2, dics, batch + ) + ), }, - outputs=["ele_out", "layernorm_out"]) + outputs=["ele_out", "layernorm_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "inputX_data": [4, 128, 768], "inputY_data": [4, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } self.dynamic_shape.max_input_shape = { "inputX_data": [4, 128, 768], "inputY_data": [4, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } self.dynamic_shape.opt_input_shape = { "inputX_data": [4, 128, 768], "inputY_data": [4, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } def clear_dynamic_shape(): @@ -164,10 +166,12 @@ def generate_trt_nodes_num(attrs, dynamic_shape): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 # atol=1e-2 while rtol is 1e-8 + attrs, True + ), 1e-2 # atol=1e-2 while rtol is 1e-8 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 # atol=1e-2 while rtol is 1e-8 + attrs, True + ), 1e-2 # atol=1e-2 while rtol is 1e-8 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_no_bias.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_no_bias.py index 74582cffb715c..320bdd1925c31 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_no_bias.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_preln_residual_no_bias.py @@ -22,7 +22,6 @@ class TrtConvertSkipLayernormTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -32,14 +31,13 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - #The input dimension should be less than or equal to the set axis. + # The input dimension should be less than or equal to the set axis. if 'begin_norm_axis' in attrs[0] and attrs[0]['begin_norm_axis'] >= 0: if len(inputs['inputX_data'].shape) <= attrs[0]['begin_norm_axis']: return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): return np.ones([batch, 128, 768]).astype(np.float32) @@ -56,81 +54,88 @@ def generate_weight2(attrs: List[Dict[str, Any]]): for epsilon in [1e-5]: for begin_norm_axis in [2]: for enable_int8 in [False, True]: - dics = [{ - "epsilon": epsilon, - "begin_norm_axis": begin_norm_axis, - }, {}] - - ops_config = [{ - "op_type": "elementwise_add", - "op_inputs": { - "X": ["inputX_data"], - "Y": ["inputY_data"] + dics = [ + { + "epsilon": epsilon, + "begin_norm_axis": begin_norm_axis, }, - "op_outputs": { - "Out": ["ele_out"] + {}, + ] + + ops_config = [ + { + "op_type": "elementwise_add", + "op_inputs": { + "X": ["inputX_data"], + "Y": ["inputY_data"], + }, + "op_outputs": {"Out": ["ele_out"]}, + "op_attrs": {"axis": -1}, }, - "op_attrs": { - "axis": -1 - } - }, { - "op_type": "layer_norm", - "op_inputs": { - "X": ["ele_out"], - "Bias": ["Bias"], - "Scale": ["Scale"] + { + "op_type": "layer_norm", + "op_inputs": { + "X": ["ele_out"], + "Bias": ["Bias"], + "Scale": ["Scale"], + }, + "op_outputs": { + "Y": ["layernorm_out"], + "Mean": ["Mean"], + "Variance": ["Variance"], + }, + "op_attrs": dics[0], }, - "op_outputs": { - "Y": ["layernorm_out"], - "Mean": ["Mean"], - "Variance": ["Variance"] - }, - "op_attrs": dics[0] - }] + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "Bias": - TensorConfig( - data_gen=partial(generate_weight1, dics)), - "Scale": - TensorConfig( - data_gen=partial(generate_weight2, dics)) + "Bias": TensorConfig( + data_gen=partial(generate_weight1, dics) + ), + "Scale": TensorConfig( + data_gen=partial(generate_weight2, dics) + ), }, inputs={ - "inputX_data": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)), - "inputY_data": - TensorConfig(data_gen=partial( - generate_input2, dics, batch)) + "inputX_data": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ), + "inputY_data": TensorConfig( + data_gen=partial( + generate_input2, dics, batch + ) + ), }, - outputs=["ele_out", "layernorm_out"]) + outputs=["ele_out", "layernorm_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "inputX_data": [4, 128, 768], "inputY_data": [4, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } self.dynamic_shape.max_input_shape = { "inputX_data": [4, 128, 768], "inputY_data": [4, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } self.dynamic_shape.opt_input_shape = { "inputX_data": [4, 128, 768], "inputY_data": [4, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } def clear_dynamic_shape(): @@ -149,10 +154,12 @@ def generate_trt_nodes_num(attrs, dynamic_shape): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 # atol=1e-2 while rtol is 1e-8 + attrs, True + ), 1e-2 # atol=1e-2 while rtol is 1e-8 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-2 # atol=1e-2 while rtol is 1e-8 + attrs, True + ), 1e-2 # atol=1e-2 while rtol is 1e-8 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_prelu.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_prelu.py index 0bc26954f0f07..862bf1ff61089 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_prelu.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_prelu.py @@ -22,12 +22,10 @@ class TrtConvertPreluTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input(batch, dim1, dim2, dim3): shape = [batch] if dim1 != 0: @@ -41,8 +39,10 @@ def generate_input(batch, dim1, dim2, dim3): def generate_alpha(attrs: List[Dict[str, Any]], dim1, dim2, dim3): if attrs[0]["mode"] == "all": return np.random.random(size=(1)).astype(np.float32) - elif attrs[0]["mode"] == "channel" and attrs[0][ - "data_format"] == "NCHW": + elif ( + attrs[0]["mode"] == "channel" + and attrs[0]["data_format"] == "NCHW" + ): shape = [1] if dim1 != 0: shape.append(dim1) @@ -51,8 +51,10 @@ def generate_alpha(attrs: List[Dict[str, Any]], dim1, dim2, dim3): if dim3 != 0: shape.append(1) return np.random.random(size=shape).astype(np.float32) - elif attrs[0]["mode"] == "channel" and attrs[0][ - "data_format"] == "NHWC": + elif ( + attrs[0]["mode"] == "channel" + and attrs[0]["data_format"] == "NHWC" + ): shape = [1] if dim1 != 0: shape.append(1) @@ -86,48 +88,66 @@ def generate_alpha(attrs: List[Dict[str, Any]], dim1, dim2, dim3): for mode in ["all", "channel", "element"]: for data_format in ['NCHW', 'NHWC']: - if mode == "channel" and dim1 == 0 and data_format == "NCHW": + if ( + mode == "channel" + and dim1 == 0 + and data_format == "NCHW" + ): continue - if mode == "channel" and dim3 == 0 and data_format == "NHWC": + if ( + mode == "channel" + and dim3 == 0 + and data_format == "NHWC" + ): continue - dics = [{ - "mode": mode, - "data_format": data_format - }] - ops_config = [{ - "op_type": "prelu", - "op_inputs": { - "X": ["input_data"], - "Alpha": ["alpha_weight"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + {"mode": mode, "data_format": data_format} + ] + ops_config = [ + { + "op_type": "prelu", + "op_inputs": { + "X": ["input_data"], + "Alpha": ["alpha_weight"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "alpha_weight": - TensorConfig(data_gen=partial( - generate_alpha, dics, dim1, dim2, - dim3)) + "alpha_weight": TensorConfig( + data_gen=partial( + generate_alpha, + dics, + dim1, + dim2, + dim3, + ) + ) }, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input, batch, dim1, dim2, - dim3)), + "input_data": TensorConfig( + data_gen=partial( + generate_input, + batch, + dim1, + dim2, + dim3, + ) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dim1 == 0: self.dynamic_shape.min_input_shape = { @@ -181,7 +201,12 @@ def clear_dynamic_shape(): ] def generate_trt_nodes_num(attrs, dynamic_shape): - if not dynamic_shape and self.dim1 == 0 and self.dim2 == 0 and self.dim3 == 0: + if ( + not dynamic_shape + and self.dim1 == 0 + and self.dim2 == 0 + and self.dim3 == 0 + ): return 0, 3 return 1, 2 @@ -189,19 +214,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): ver = paddle_infer.get_trt_compile_version() @@ -213,8 +242,9 @@ def teller(program_config, predictor_config): return False self.add_skip_case( - teller, SkipReasons.TRT_NOT_IMPLEMENTED, - "Need to repair the case: the output of GPU and tensorrt has diff in trt6, the prelu static plugin has bug." + teller, + SkipReasons.TRT_NOT_IMPLEMENTED, + "Need to repair the case: the output of GPU and tensorrt has diff in trt6, the prelu static plugin has bug.", ) def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py index 95c9e24dcad1f..78ad7978b06c0 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py @@ -22,7 +22,6 @@ class TrtConvertReduceMeanTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs attrs = [ @@ -45,7 +44,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(dtype, attrs: List[Dict[str, Any]]): if dtype == -1 or dtype == 5: return np.random.random([1, 3, 64, 64]).astype(np.float32) @@ -53,39 +51,52 @@ def generate_input1(dtype, attrs: List[Dict[str, Any]]): return np.random.random([1, 3, 64, 64]).astype(np.int32) for keep_dim in [True, False]: - for dim in [[], [1], [0], [0, 1], [1, 2, 3], [-2, 0, 3], [-3], - [-4, 1], [3, 4, 5]]: + for dim in [ + [], + [1], + [0], + [0, 1], + [1, 2, 3], + [-2, 0, 3], + [-3], + [-4, 1], + [3, 4, 5], + ]: for reduce_all in [True, False]: for out_dtype in [-1, 2, 5]: - dics = [{ - "keep_dim": keep_dim, - "dim": dim, - "reduce_all": reduce_all, - "out_dtype": out_dtype, - "in_dtype": out_dtype, - }, {}] - - ops_config = [{ - "op_type": "reduce_mean", - "op_inputs": { - "X": ["input_data"] + dics = [ + { + "keep_dim": keep_dim, + "dim": dim, + "reduce_all": reduce_all, + "out_dtype": out_dtype, + "in_dtype": out_dtype, }, - "op_outputs": { - "Out": ["reduce_output_data"] - }, - "op_attrs": dics[0] - }] + {}, + ] + + ops_config = [ + { + "op_type": "reduce_mean", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["reduce_output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, out_dtype, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, out_dtype, dics + ) + ) }, - outputs=["reduce_output_data"]) + outputs=["reduce_output_data"], + ) if not self.is_program_valid(program_config): continue @@ -93,8 +104,8 @@ def generate_input1(dtype, attrs: List[Dict[str, Any]]): yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -125,19 +136,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (5e-4, 5e-4) + attrs, False + ), (5e-4, 5e-4) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (5e-4, 5e-4) + attrs, True + ), (5e-4, 5e-4) def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py index 41d491d32551a..67f3ab26f3750 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py @@ -23,7 +23,6 @@ class TrtConvertReduceSumTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs attrs = [ @@ -41,7 +40,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(dtype, attrs: List[Dict[str, Any]]): if dtype == -1 or dtype == 5: return np.random.random([1, 3, 32, 32]).astype(np.float32) @@ -49,39 +47,52 @@ def generate_input1(dtype, attrs: List[Dict[str, Any]]): return np.random.random([1, 3, 32, 32]).astype(np.int32) for keep_dim in [True, False]: - for dim in [[], [1], [0], [0, 1], [1, 2, 3], [-2, 0, 3], [-3], - [-4, 1], [3, 4, 5]]: + for dim in [ + [], + [1], + [0], + [0, 1], + [1, 2, 3], + [-2, 0, 3], + [-3], + [-4, 1], + [3, 4, 5], + ]: for reduce_all in [True, False]: for out_dtype in [-1, 2, 5]: - dics = [{ - "keep_dim": keep_dim, - "dim": dim, - "reduce_all": reduce_all, - "out_dtype": out_dtype, - "in_dtype": out_dtype, - }, {}] - - ops_config = [{ - "op_type": "reduce_sum", - "op_inputs": { - "X": ["input_data"] + dics = [ + { + "keep_dim": keep_dim, + "dim": dim, + "reduce_all": reduce_all, + "out_dtype": out_dtype, + "in_dtype": out_dtype, }, - "op_outputs": { - "Out": ["reduce_output_data"] - }, - "op_attrs": dics[0] - }] + {}, + ] + + ops_config = [ + { + "op_type": "reduce_sum", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["reduce_output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, out_dtype, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, out_dtype, dics + ) + ) }, - outputs=["reduce_output_data"]) + outputs=["reduce_output_data"], + ) if not self.is_program_valid(program_config): continue @@ -89,7 +100,6 @@ def generate_input1(dtype, attrs: List[Dict[str, Any]]): yield program_config def sample_predictor_configs(self, program_config): - def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -120,19 +130,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-5, 1e-5) + attrs, False + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-5, 1e-5) + attrs, True + ), (1e-5, 1e-5) self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reshape.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reshape.py index abd4be3f888ed..90aaddfcb403e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reshape.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reshape.py @@ -22,7 +22,6 @@ class TrtConvertReshapeTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) @@ -31,7 +30,7 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: if len(attrs[0]['shape']) != 1: return False - #To test if the shape contains 0 + # To test if the shape contains 0 if len(attrs[0]['shape']) == 3: if attrs[0]['shape'][1] == 0: if self.dims != 3: @@ -45,7 +44,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): if self.dims == 4: self.input_shape = [1, 2, 4, 6] @@ -70,9 +68,18 @@ def generate_shapeT2_data(attrs: List[Dict[str, Any]]): return np.array([24]).astype(np.int32) for dims in [4, 3, 2, 1]: - for shape in [[1, 6, 8], [1, 2, 4, 6], [1, 1, 0, 12], [1, 0, 6], - [1, -1, 12], [2, -1], [3, 16], [3, 4, 4], [48], - [-1, 48]]: + for shape in [ + [1, 6, 8], + [1, 2, 4, 6], + [1, 1, 0, 12], + [1, 0, 6], + [1, -1, 12], + [2, -1], + [3, 16], + [3, 4, 4], + [48], + [-1, 48], + ]: dics = [ { "shape": shape, @@ -81,29 +88,31 @@ def generate_shapeT2_data(attrs: List[Dict[str, Any]]): self.dims = dims dics_intput = [{"X": ["reshape_input"]}] - ops_config = [{ - "op_type": "reshape", - "op_inputs": dics_intput[0], - "op_outputs": { - "Out": ["reshape_out"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "reshape", + "op_inputs": dics_intput[0], + "op_outputs": {"Out": ["reshape_out"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "reshape_input": - TensorConfig(data_gen=partial(generate_input1, dics)) + "reshape_input": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["reshape_out"]) + outputs=["reshape_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { @@ -141,13 +150,14 @@ def clear_dynamic_shape(): def generate_trt_nodes_num(attrs, dynamic_shape): # in static shape mode, here is consistent with op_teller.cc - if (not dynamic_shape): - if (attrs[0]['shape'][0] == 0): + if not dynamic_shape: + if attrs[0]['shape'][0] == 0: return 1, 2 - elif (len(attrs[0]['shape']) == 1): + elif len(attrs[0]['shape']) == 1: return 0, 3 - elif (np.prod(attrs[0]['shape'][1:]) == np.prod( - self.input_shape[1:])): + elif np.prod(attrs[0]['shape'][1:]) == np.prod( + self.input_shape[1:] + ): return 1, 2 else: return 0, 3 @@ -161,19 +171,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass @@ -185,12 +199,10 @@ def test(self): # reshape having three inputs. class TrtConvertReshapeTest2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): if self.dims == 4: return np.random.random([1, 2, 4, 6]).astype(np.float32) @@ -203,9 +215,12 @@ def generate_input1(attrs: List[Dict[str, Any]]): for dims in [4, 3, 2, 1]: for shape in [[-1, 48]]: - dics = [{ - "shape": shape, - }, {}] + dics = [ + { + "shape": shape, + }, + {}, + ] self.dims = dims dics_intput = [ { @@ -217,9 +232,7 @@ def generate_input1(attrs: List[Dict[str, Any]]): { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shapeT1_data"] - }, + "op_outputs": {"Out": ["shapeT1_data"]}, "op_attrs": { "dtype": 2, "str_value": "2", @@ -229,9 +242,7 @@ def generate_input1(attrs: List[Dict[str, Any]]): { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shapeT2_data"] - }, + "op_outputs": {"Out": ["shapeT2_data"]}, "op_attrs": { "dtype": 2, "str_value": "24", @@ -241,10 +252,8 @@ def generate_input1(attrs: List[Dict[str, Any]]): { "op_type": "reshape", "op_inputs": dics_intput[0], - "op_outputs": { - "Out": ["reshape_out"] - }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["reshape_out"]}, + "op_attrs": dics[0], }, ] ops = self.generate_op_config(ops_config) @@ -252,16 +261,18 @@ def generate_input1(attrs: List[Dict[str, Any]]): ops=ops, weights={}, inputs={ - "reshape_input": - TensorConfig(data_gen=partial(generate_input1, dics)) + "reshape_input": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["reshape_out"]) + outputs=["reshape_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): if self.dims == 4: self.dynamic_shape.min_input_shape = { @@ -309,12 +320,10 @@ def test(self): # reshape having 2 inputs. class TrtConvertReshapeTest3(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): if self.dims == 4: return np.random.random([1, 2, 12, 6]).astype(np.float32) @@ -327,9 +336,12 @@ def generate_input1(attrs: List[Dict[str, Any]]): for dims in [4, 3, 2, 1]: for shape in [[-1, 144]]: - dics = [{ - "shape": shape, - }, {}] + dics = [ + { + "shape": shape, + }, + {}, + ] self.dims = dims dics_intput = [ { @@ -341,9 +353,7 @@ def generate_input1(attrs: List[Dict[str, Any]]): { "op_type": "fill_constant", "op_inputs": {}, - "op_outputs": { - "Out": ["shape_data"] - }, + "op_outputs": {"Out": ["shape_data"]}, "op_attrs": { "dtype": 2, "str_value": "12", @@ -353,10 +363,8 @@ def generate_input1(attrs: List[Dict[str, Any]]): { "op_type": "reshape", "op_inputs": dics_intput[0], - "op_outputs": { - "Out": ["reshape_out"] - }, - "op_attrs": dics[0] + "op_outputs": {"Out": ["reshape_out"]}, + "op_attrs": dics[0], }, ] ops = self.generate_op_config(ops_config) @@ -364,16 +372,18 @@ def generate_input1(attrs: List[Dict[str, Any]]): ops=ops, weights={}, inputs={ - "reshape_input": - TensorConfig(data_gen=partial(generate_input1, dics)) + "reshape_input": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["reshape_out"]) + outputs=["reshape_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): if self.dims == 4: self.dynamic_shape.min_input_shape = { diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_rnn.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_rnn.py index 2b76e273aa0a9..4f65ea6abf41d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_rnn.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_rnn.py @@ -23,7 +23,6 @@ class TrtConvertSliceTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -36,148 +35,168 @@ def sample_program_configs(self): for num_layers in [1, 2]: for is_bidirec in [True, False]: dics = [] - dics.append({ - "hidden_size": hidden_size, - "input_size": input_size, - "num_layers": num_layers, - "mode": "LSTM", - "is_bidirec": is_bidirec, - "is_test": True, - "dropout_prob": 0.0, - # for my convience - "batch": batch, - "seq_len": seq_len, - }) + dics.append( + { + "hidden_size": hidden_size, + "input_size": input_size, + "num_layers": num_layers, + "mode": "LSTM", + "is_bidirec": is_bidirec, + "is_test": True, + "dropout_prob": 0.0, + # for my convience + "batch": batch, + "seq_len": seq_len, + } + ) K = 1 - if (dics[0]["is_bidirec"]): + if dics[0]["is_bidirec"]: K = 2 def generate_input1(): - return np.random.random([ - batch, seq_len, input_size - ]).astype(np.float32) * 2 - 1 + return ( + np.random.random( + [batch, seq_len, input_size] + ).astype(np.float32) + * 2 + - 1 + ) # initial input -> hidden def generate_w0(): - return np.random.random([ - 4 * hidden_size, input_size - ]).astype(np.float32) * 2 - 1 + return ( + np.random.random( + [4 * hidden_size, input_size] + ).astype(np.float32) + * 2 + - 1 + ) # prev layer's output -> hidden def generate_w1(): - return np.random.random([ - 4 * hidden_size, K * hidden_size - ]).astype(np.float32) * 2 - 1 + return ( + np.random.random( + [4 * hidden_size, K * hidden_size] + ).astype(np.float32) + * 2 + - 1 + ) # def generate_w2(): - return np.random.random([ - 4 * hidden_size, hidden_size - ]).astype(np.float32) * 2 - 1 + return ( + np.random.random( + [4 * hidden_size, hidden_size] + ).astype(np.float32) + * 2 + - 1 + ) def generate_b(): - return np.random.random([ - 4 * hidden_size - ]).astype(np.float32) * 2 - 1 - - dics.append({ - "dtype": - 5, - "input_dim_idx": - 0, - "str_value": - "", - "value": - 0.0, - "shape": [K * num_layers, -1, hidden_size], - "output_dim_idx": - 1, - }) + return ( + np.random.random( + [4 * hidden_size] + ).astype(np.float32) + * 2 + - 1 + ) + + dics.append( + { + "dtype": 5, + "input_dim_idx": 0, + "str_value": "", + "value": 0.0, + "shape": [ + K * num_layers, + -1, + hidden_size, + ], + "output_dim_idx": 1, + } + ) dics.append({"axis": [1, 0, 2]}) # set weights WeightList = [ "weight" + str(i) - for i in range(4 * K * - dics[0]["num_layers"]) + for i in range( + 4 * K * dics[0]["num_layers"] + ) ] weights = {} for i in range((int)(len(WeightList) / 2)): # mean this weight : input->hidden # input has 2 case: initial input input_size, K * hidden form the prev layer. - if (i % 2 == 0): - if (i <= K): + if i % 2 == 0: + if i <= K: weights[ - WeightList[i]] = TensorConfig( - data_gen=partial( - generate_w0)) + WeightList[i] + ] = TensorConfig( + data_gen=partial(generate_w0) + ) else: weights[ - WeightList[i]] = TensorConfig( - data_gen=partial( - generate_w1)) + WeightList[i] + ] = TensorConfig( + data_gen=partial(generate_w1) + ) # mean this weight : hidden->hidden - if (i % 2 == 1): + if i % 2 == 1: weights[WeightList[i]] = TensorConfig( - data_gen=partial(generate_w2)) - for i in range((int)(len(WeightList) / 2), - len(WeightList)): + data_gen=partial(generate_w2) + ) + for i in range( + (int)(len(WeightList) / 2), len(WeightList) + ): weights[WeightList[i]] = TensorConfig( - data_gen=partial(generate_b)) + data_gen=partial(generate_b) + ) ops_config = [ { - "op_type": - "fill_constant_batch_size_like", - "op_inputs": { - "Input": ["input_data"] - }, - "op_outputs": { - "Out": ["prestate1"] - }, - "op_attrs": dics[1] + "op_type": "fill_constant_batch_size_like", + "op_inputs": {"Input": ["input_data"]}, + "op_outputs": {"Out": ["prestate1"]}, + "op_attrs": dics[1], }, { - "op_type": - "fill_constant_batch_size_like", - "op_inputs": { - "Input": ["input_data"] - }, - "op_outputs": { - "Out": ["prestate2"] - }, - "op_attrs": dics[1] + "op_type": "fill_constant_batch_size_like", + "op_inputs": {"Input": ["input_data"]}, + "op_outputs": {"Out": ["prestate2"]}, + "op_attrs": dics[1], }, { "op_type": "transpose2", - "op_inputs": { - "X": ["input_data"] - }, + "op_inputs": {"X": ["input_data"]}, "op_outputs": { "Out": ["rnn_input_data"] }, - "op_attrs": dics[2] + "op_attrs": dics[2], }, { "op_type": "rnn", "op_inputs": { "Input": ["rnn_input_data"], # prev_c, prev_h - "PreState": - ["prestate1", "prestate2"], + "PreState": [ + "prestate1", + "prestate2", + ], "WeightList": WeightList, }, "op_outputs": { "Out": ["rnn_output_data"], "State": [ "state_output_data0", - "state_output_data1" + "state_output_data1", ], "Reserve": ["reserve_data"], - "DropoutState": - ["DropoutState_data"] + "DropoutState": [ + "DropoutState_data" + ], }, - "op_attrs": dics[0] - } + "op_attrs": dics[0], + }, ] ops = self.generate_op_config(ops_config) @@ -185,16 +204,18 @@ def generate_b(): ops=ops, weights=weights, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1)) + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ) }, - outputs=["rnn_output_data"]) + outputs=["rnn_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): + self, program_config + ) -> (paddle_infer.Config, List[int], float): attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) ] @@ -234,7 +255,7 @@ def generate_trt_nodes_num(attrs, dynamic_shape): # The output has diff between gpu and trt in PR-CI-Windows-Inference tol_fp32 = 1e-5 tol_half = 1e-2 - if (os.name == 'nt'): + if os.name == 'nt': tol_fp32 = 1e-2 tol_half = 1e-1 @@ -242,10 +263,12 @@ def generate_trt_nodes_num(attrs, dynamic_shape): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), tol_fp32 + attrs, True + ), tol_fp32 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), tol_half + attrs, True + ), tol_half def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roi_align.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roi_align.py index ee84df151b5c6..f89527359d4d1 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roi_align.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roi_align.py @@ -22,12 +22,10 @@ class TrtConvertRoiAlignTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): return np.ones([batch, 256, 32, 32]).astype(np.float32) @@ -47,92 +45,111 @@ def generate_input3(attrs: List[Dict[str, Any]], batch): self.num_input = num_input if num_input == 1: batch = 1 - dics = [{ - "spatial_scale": spatial_scale, - "pooled_height": pooled_height, - "pooled_width": pooled_width, - "sampling_ratio": sampling_ratio, - "aligned": aligned - }, {}] - dics_input = [{ - "X": ["roi_align_input"], - "ROIs": ["ROIs"], - "RoisNum": ["RoisNum"] - }, { - "X": ["roi_align_input"], - "ROIs": ["ROIs"] - }] - program_input = [{ - "roi_align_input": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)), - "ROIs": - TensorConfig(data_gen=partial( - generate_input2, dics, batch)), - "RoisNum": - TensorConfig(data_gen=partial( - generate_input3, dics, batch)) - }, { - "roi_align_input": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)), - "ROIs": - TensorConfig(data_gen=partial( - generate_input2, dics, batch), - lod=[[32, 3]]) - }] - ops_config = [{ - "op_type": - "roi_align", - "op_inputs": - dics_input[num_input], - "op_outputs": { - "Out": ["roi_align_out"] + dics = [ + { + "spatial_scale": spatial_scale, + "pooled_height": pooled_height, + "pooled_width": pooled_width, + "sampling_ratio": sampling_ratio, + "aligned": aligned, + }, + {}, + ] + dics_input = [ + { + "X": ["roi_align_input"], + "ROIs": ["ROIs"], + "RoisNum": ["RoisNum"], + }, + { + "X": ["roi_align_input"], + "ROIs": ["ROIs"], + }, + ] + program_input = [ + { + "roi_align_input": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ), + "ROIs": TensorConfig( + data_gen=partial( + generate_input2, dics, batch + ) + ), + "RoisNum": TensorConfig( + data_gen=partial( + generate_input3, dics, batch + ) + ), + }, + { + "roi_align_input": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ), + "ROIs": TensorConfig( + data_gen=partial( + generate_input2, dics, batch + ), + lod=[[32, 3]], + ), }, - "op_attrs": - dics[0] - }] + ] + ops_config = [ + { + "op_type": "roi_align", + "op_inputs": dics_input[num_input], + "op_outputs": { + "Out": ["roi_align_out"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs=program_input[num_input], - outputs=["roi_align_out"]) + outputs=["roi_align_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.num_input == 0: self.dynamic_shape.min_input_shape = { "roi_align_input": [1, 256, 32, 32], "ROIs": [3, 4], - "RoisNum": [1] + "RoisNum": [1], } self.dynamic_shape.max_input_shape = { "roi_align_input": [1, 256, 64, 64], "ROIs": [3, 4], - "RoisNum": [1] + "RoisNum": [1], } self.dynamic_shape.opt_input_shape = { "roi_align_input": [1, 256, 64, 64], "ROIs": [3, 4], - "RoisNum": [1] + "RoisNum": [1], } elif self.num_input == 1: self.dynamic_shape.min_input_shape = { "roi_align_input": [1, 256, 32, 32], - "ROIs": [3, 4] + "ROIs": [3, 4], } self.dynamic_shape.max_input_shape = { "roi_align_input": [1, 256, 64, 64], - "ROIs": [3, 4] + "ROIs": [3, 4], } self.dynamic_shape.opt_input_shape = { "roi_align_input": [1, 256, 64, 64], - "ROIs": [3, 4] + "ROIs": [3, 4], } def clear_dynamic_shape(): @@ -159,29 +176,33 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(program_config.inputs) == 3: return True return False - self.add_skip_case(teller1, SkipReasons.TRT_NOT_SUPPORT, - "INPUT RoisNum NOT SUPPORT") + self.add_skip_case( + teller1, SkipReasons.TRT_NOT_SUPPORT, "INPUT RoisNum NOT SUPPORT" + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roll.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roll.py index 6b6e1e0c4bfe5..47d415b1a6b76 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roll.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_roll.py @@ -22,7 +22,6 @@ class TrtConvertRollTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -32,43 +31,44 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.ones([1, 56, 56, 192]).astype(np.float32) for axis in [[1, 2]]: for shifts in [[-1, -1], [-3, -3]]: - dics = [{ - "axis": axis, - "shifts": shifts, - }] - - ops_config = [{ - "op_type": "roll", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["roll_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "axis": axis, + "shifts": shifts, + } + ] + + ops_config = [ + { + "op_type": "roll", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["roll_output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["roll_output_data"]) + outputs=["roll_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 56, 56, 192] @@ -103,19 +103,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scale.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scale.py index 33a7c060ecc15..27658d9286367 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scale.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scale.py @@ -22,12 +22,10 @@ class TrtConvertScaleTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: return np.ones([batch, 3, 24, 24]).astype(np.float32) @@ -49,51 +47,60 @@ def generate_weight1(attrs: List[Dict[str, Any]]): for bias_after_scale in [False, True]: self.num_input = num_input self.dims = dims - dics = [{ - "scale": scale, - "bias": bias, - "bias_after_scale": bias_after_scale - }, {}] - - dics_intput = [{ - "X": ["scale_input"], - "ScaleTensor": ["ScaleTensor"], - }, { - "X": ["scale_input"] - }] - dics_intputs = [{ - "ScaleTensor": - TensorConfig(data_gen=partial( - generate_weight1, dics)) - }, {}] - - ops_config = [{ - "op_type": - "scale", - "op_inputs": - dics_intput[num_input], - "op_outputs": { - "Out": ["scale_out"] + dics = [ + { + "scale": scale, + "bias": bias, + "bias_after_scale": bias_after_scale, + }, + {}, + ] + + dics_intput = [ + { + "X": ["scale_input"], + "ScaleTensor": ["ScaleTensor"], }, - "op_attrs": - dics[0] - }] + {"X": ["scale_input"]}, + ] + dics_intputs = [ + { + "ScaleTensor": TensorConfig( + data_gen=partial( + generate_weight1, dics + ) + ) + }, + {}, + ] + + ops_config = [ + { + "op_type": "scale", + "op_inputs": dics_intput[num_input], + "op_outputs": {"Out": ["scale_out"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights=dics_intputs[num_input], inputs={ - "scale_input": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)) + "scale_input": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ) }, - outputs=["scale_out"]) + outputs=["scale_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { @@ -134,37 +141,46 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if self.num_input == 0: return True return False - self.add_skip_case(teller1, SkipReasons.TRT_NOT_SUPPORT, - "INPUT ScaleTensor and Shape NOT SUPPORT") + self.add_skip_case( + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "INPUT ScaleTensor and Shape NOT SUPPORT", + ) def teller2(program_config, predictor_config): if self.dims == 1 and len(self.dynamic_shape.min_input_shape) == 0: return True return False - self.add_skip_case(teller2, SkipReasons.TRT_NOT_SUPPORT, - "INPUT DIM EQUAL TO 1 OF STATIC SHAPE NOT SUPPORT") + self.add_skip_case( + teller2, + SkipReasons.TRT_NOT_SUPPORT, + "INPUT DIM EQUAL TO 1 OF STATIC SHAPE NOT SUPPORT", + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scatter_nd_add.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scatter_nd_add.py index d0e6cf867ce43..9376ade22a2be 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scatter_nd_add.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_scatter_nd_add.py @@ -22,12 +22,10 @@ class TrtConvertScatterNd(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([6]).astype(np.float32) @@ -37,38 +35,42 @@ def generate_input2(): def generate_input3(): return np.random.random([4]).astype(np.float32) - ops_config = [{ - "op_type": "scatter_nd_add", - "op_inputs": { - "X": ["input_data"], - "Index": ["index_data"], - "Updates": ["update_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "scatter_nd_add", + "op_inputs": { + "X": ["input_data"], + "Index": ["index_data"], + "Updates": ["update_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) for i in range(10): program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1)), - "index_data": - TensorConfig(data_gen=partial(generate_input2)), - "update_data": - TensorConfig(data_gen=partial(generate_input3)), + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ), + "index_data": TensorConfig( + data_gen=partial(generate_input2) + ), + "update_data": TensorConfig( + data_gen=partial(generate_input3) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1], diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shape.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shape.py index 778a097c3ab03..03df0287836aa 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shape.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shape.py @@ -22,12 +22,10 @@ class TrtConvertSumTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(batch): if self.dims == 4: return np.ones([batch, 3, 24, 24]).astype(np.float32) @@ -41,31 +39,31 @@ def generate_input1(batch): for dims in [1, 2, 3, 4]: for batch in [1, 4]: self.dims = dims - ops_config = [{ - "op_type": "shape", - "op_inputs": { - "Input": ["input1"] - }, - "op_outputs": { - "Out": ["output"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "shape", + "op_inputs": {"Input": ["input1"]}, + "op_outputs": {"Out": ["output"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1": - TensorConfig(data_gen=partial(generate_input1, batch)) + "input1": TensorConfig( + data_gen=partial(generate_input1, batch) + ) }, - outputs=["output"]) + outputs=["output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): if self.dims == 4: self.dynamic_shape.min_input_shape = {"input1": [1, 3, 24, 24]} @@ -87,7 +85,7 @@ def generate_dynamic_shape(): } def generate_trt_nodes_num(dynamic_shape): - if (not dynamic_shape): + if not dynamic_shape: return 0, 3 return 1, 2 @@ -100,10 +98,12 @@ def clear_dynamic_shape(): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-5 + False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-3 + False + ), 1e-3 # for dynamic_shape generate_dynamic_shape() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shuffle_channel.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shuffle_channel.py index bf5584e002943..04c1e3259fc12 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shuffle_channel.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_shuffle_channel.py @@ -22,44 +22,41 @@ class TrtConvertShuffleChannelTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): return np.ones([batch, 6, 24, 24]).astype(np.float32) for batch in [1, 2, 4]: for group in [1, 2, 3]: dics = [{"group": group}, {}] - ops_config = [{ - "op_type": "shuffle_channel", - "op_inputs": { - "X": ["shuffle_channel_input"] - }, - "op_outputs": { - "Out": ["shuffle_channel_out"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "shuffle_channel", + "op_inputs": {"X": ["shuffle_channel_input"]}, + "op_outputs": {"Out": ["shuffle_channel_out"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "shuffle_channel_input": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)) + "shuffle_channel_input": TensorConfig( + data_gen=partial(generate_input1, dics, batch) + ) }, - outputs=["shuffle_channel_out"]) + outputs=["shuffle_channel_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "shuffle_channel_input": [1, 6, 24, 24] @@ -78,8 +75,10 @@ def clear_dynamic_shape(): def generate_trt_nodes_num(attrs, dynamic_shape): ver = paddle_infer.get_trt_compile_version() - if ver[0] * 1000 + ver[1] * 100 + ver[ - 2] * 10 < 8000 and dynamic_shape == True: + if ( + ver[0] * 1000 + ver[1] * 100 + ver[2] * 10 < 8000 + and dynamic_shape == True + ): return 0, 3 else: return 1, 2 @@ -92,19 +91,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_silu.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_silu.py index 3b9ff8a0527d5..73df326fb01ad 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_silu.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_silu.py @@ -22,12 +22,10 @@ class TrtConvertSwishTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(dims, attrs: List[Dict[str, Any]]): if dims == 1: return np.ones([3]).astype(np.float32) @@ -42,33 +40,34 @@ def generate_input1(dims, attrs: List[Dict[str, Any]]): for beta in [1.0, 2.0, 3.0]: self.dims = dims - ops_config = [{ - "op_type": "silu", - "op_inputs": { - "X": ["input_data"], - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "silu", + "op_inputs": { + "X": ["input_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, dims, {})) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dims, {}) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -113,19 +112,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_skip_layernorm.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_skip_layernorm.py index 3a3b0603ec846..18ea2abe6bc35 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_skip_layernorm.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_skip_layernorm.py @@ -22,7 +22,6 @@ class TrtConvertSkipLayernormTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -32,19 +31,20 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - #The input dimension should be less than or equal to the set axis. + # The input dimension should be less than or equal to the set axis. if attrs[0]['begin_norm_axis'] >= 0: - if len(inputs['skip_layernorm_inputX_data'].shape - ) <= attrs[0]['begin_norm_axis']: + if ( + len(inputs['skip_layernorm_inputX_data'].shape) + <= attrs[0]['begin_norm_axis'] + ): return False - #2D input is not supported. + # 2D input is not supported. if self.dims == 2: return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: return np.ones([batch, 6, 128, 768]).astype(np.float32) @@ -73,107 +73,117 @@ def generate_weight2(attrs: List[Dict[str, Any]]): for begin_norm_axis in [0, 1, 2, -1]: for enable_int8 in [False, True]: self.dims = dims - dics = [{ - "epsilon": epsilon, - "begin_norm_axis": begin_norm_axis, - "enable_int8": enable_int8 - }, {}] - ops_config = [{ - "op_type": "skip_layernorm", - "op_inputs": { - "X": ["skip_layernorm_inputX_data"], - "Y": ["skip_layernorm_inputY_data"], - "Bias": ["Bias"], - "Scale": ["Scale"] + dics = [ + { + "epsilon": epsilon, + "begin_norm_axis": begin_norm_axis, + "enable_int8": enable_int8, }, - "op_outputs": { - "Out": ["skip_layernorm_out"] - }, - "op_attrs": dics[0] - }] + {}, + ] + ops_config = [ + { + "op_type": "skip_layernorm", + "op_inputs": { + "X": ["skip_layernorm_inputX_data"], + "Y": ["skip_layernorm_inputY_data"], + "Bias": ["Bias"], + "Scale": ["Scale"], + }, + "op_outputs": { + "Out": ["skip_layernorm_out"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={ - "Bias": - TensorConfig(data_gen=partial( - generate_weight1, dics)), - "Scale": - TensorConfig(data_gen=partial( - generate_weight2, dics)) + "Bias": TensorConfig( + data_gen=partial(generate_weight1, dics) + ), + "Scale": TensorConfig( + data_gen=partial(generate_weight2, dics) + ), }, inputs={ - "skip_layernorm_inputX_data": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)), - "skip_layernorm_inputY_data": - TensorConfig(data_gen=partial( - generate_input2, dics, batch)) + "skip_layernorm_inputX_data": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ), + "skip_layernorm_inputY_data": TensorConfig( + data_gen=partial( + generate_input2, dics, batch + ) + ), }, - outputs=["skip_layernorm_out"]) + outputs=["skip_layernorm_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { "skip_layernorm_inputX_data": [1, 6, 128, 768], "skip_layernorm_inputY_data": [1, 6, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } self.dynamic_shape.max_input_shape = { "skip_layernorm_inputX_data": [4, 6, 768, 3072], "skip_layernorm_inputY_data": [4, 6, 768, 3072], "Bias": [3072], - "Scale": [3072] + "Scale": [3072], } self.dynamic_shape.opt_input_shape = { "skip_layernorm_inputX_data": [2, 6, 128, 768], "skip_layernorm_inputY_data": [2, 6, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } elif self.dims == 3: self.dynamic_shape.min_input_shape = { "skip_layernorm_inputX_data": [1, 128, 768], "skip_layernorm_inputY_data": [1, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } self.dynamic_shape.max_input_shape = { "skip_layernorm_inputX_data": [4, 768, 3072], "skip_layernorm_inputY_data": [4, 768, 3072], "Bias": [3072], - "Scale": [3072] + "Scale": [3072], } self.dynamic_shape.opt_input_shape = { "skip_layernorm_inputX_data": [2, 128, 768], "skip_layernorm_inputY_data": [2, 128, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } elif self.dims == 2: self.dynamic_shape.min_input_shape = { "skip_layernorm_inputX_data": [1, 768], "skip_layernorm_inputY_data": [1, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } self.dynamic_shape.max_input_shape = { "skip_layernorm_inputX_data": [4, 3072], "skip_layernorm_inputY_data": [4, 3072], "Bias": [3072], - "Scale": [3072] + "Scale": [3072], } self.dynamic_shape.opt_input_shape = { "skip_layernorm_inputX_data": [2, 768], "skip_layernorm_inputY_data": [2, 768], "Bias": [768], - "Scale": [768] + "Scale": [768], } def clear_dynamic_shape(): @@ -204,10 +214,12 @@ def generate_trt_nodes_num(attrs, dynamic_shape): generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_slice.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_slice.py index 22eb7cef872b7..81d833f39f08f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_slice.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_slice.py @@ -22,7 +22,6 @@ class TrtConvertSliceTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -34,13 +33,17 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: start = 0 end = 0 if attrs[0]["starts"][x] < 0: - start = attrs[0]["starts"][x] + inputs['input_data'].shape[ - attrs[0]["axes"][x]] + start = ( + attrs[0]["starts"][x] + + inputs['input_data'].shape[attrs[0]["axes"][x]] + ) else: start = attrs[0]["starts"][x] if attrs[0]["ends"][x] < 0: - end = attrs[0]["ends"][x] + inputs['input_data'].shape[ - attrs[0]["axes"][x]] + end = ( + attrs[0]["ends"][x] + + inputs['input_data'].shape[attrs[0]["axes"][x]] + ) else: end = attrs[0]["ends"][x] start = max(0, start) @@ -51,12 +54,11 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: for x in attrs[0]["decrease_axis"]: if x < 0: return False - if (out_shape[x] != 1): + if out_shape[x] != 1: return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.random.random([6, 6, 64, 64]).astype(np.float32) @@ -65,41 +67,44 @@ def generate_input1(attrs: List[Dict[str, Any]]): for ends in [[2, 2], [5, 5], [1, -1]]: for decrease_axis in [[], [1], [2], [-1], [-100]]: for infer_flags in [[-1]]: - dics = [{ - "axes": axes, - "starts": starts, - "ends": ends, - "decrease_axis": decrease_axis, - "infer_flags": infer_flags - }] - - ops_config = [{ - "op_type": "slice", - "op_inputs": { - "Input": ["input_data"] - }, - "op_outputs": { - "Out": ["slice_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "axes": axes, + "starts": starts, + "ends": ends, + "decrease_axis": decrease_axis, + "infer_flags": infer_flags, + } + ] + + ops_config = [ + { + "op_type": "slice", + "op_inputs": {"Input": ["input_data"]}, + "op_outputs": { + "Out": ["slice_output_data"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["slice_output_data"]) + outputs=["slice_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [8, 8, 64, 64]} @@ -125,19 +130,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): # TODO(inference): fix. diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_softmax.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_softmax.py index 6662146331080..7f1ed848cb178 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_softmax.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_softmax.py @@ -22,7 +22,6 @@ class TrtConvertSoftmaxTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -32,14 +31,13 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - #The input dimension should be less than or equal to the set axis. + # The input dimension should be less than or equal to the set axis. if len(inputs['softmax_input'].shape) <= attrs[0]['axis']: return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: return np.ones([batch, 3, 24, 24]).astype(np.float32) @@ -53,32 +51,31 @@ def generate_input1(attrs: List[Dict[str, Any]], batch): for axis in [-1, 0, 1, 2, 3]: self.dims = dims dics = [{"axis": axis}, {}] - ops_config = [{ - "op_type": "softmax", - "op_inputs": { - "X": ["softmax_input"] - }, - "op_outputs": { - "Out": ["softmax_out"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "softmax", + "op_inputs": {"X": ["softmax_input"]}, + "op_outputs": {"Out": ["softmax_out"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "softmax_input": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)) + "softmax_input": TensorConfig( + data_gen=partial(generate_input1, dics, batch) + ) }, - outputs=["softmax_out"]) + outputs=["softmax_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { @@ -123,19 +120,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): else: self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_split.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_split.py index b60b9227395d7..ce9c533c62587 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_split.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_split.py @@ -22,7 +22,6 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -35,13 +34,13 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: if len(inputs['split_input'].shape) <= attrs[0]['axis']: return False - #Sections and num cannot both be equal to 0. + # Sections and num cannot both be equal to 0. if len(attrs[0]['sections']) == 0: if attrs[0]['num'] == 0: return False - #When sections and num are not both equal to 0, sections has higher priority. - #The sum of sections should be equal to the input size. + # When sections and num are not both equal to 0, sections has higher priority. + # The sum of sections should be equal to the input size. if len(attrs[0]['sections']) != 0: if attrs[0]['num'] != 0: return False @@ -53,16 +52,18 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: if sum != inputs['split_input'].shape[attrs[0]['axis']]: return False - #The size of num should be equal to the input dimension. + # The size of num should be equal to the input dimension. if attrs[0]['num'] != 0: if len(outputs) != attrs[0]['num']: return False - #Test AxisTensor and SectionsTensorList + # Test AxisTensor and SectionsTensorList if self.num_input == 0: - if self.dims == 2 and attrs[0]['sections'] == [ - 10, 14 - ] and len(outputs) == 2: + if ( + self.dims == 2 + and attrs[0]['sections'] == [10, 14] + and len(outputs) == 2 + ): return True else: return False @@ -70,7 +71,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: return np.random.random([batch, 3, 3, 24]).astype(np.float32) @@ -93,72 +93,95 @@ def generate_SectionsTensorList2(attrs: List[Dict[str, Any]]): for num_input in [0, 1]: for dims in [1, 2, 3, 4]: for batch in [3, 6, 9]: - for Out in [["output_var0", "output_var1"], - ["output_var0", "output_var1", "output_var2"]]: - for sections in [[], [1, 2], [2, 1], [10, 14], - [1, 1, 1], [2, 2, 2], [3, 3, 3], - [3, 7, 14]]: + for Out in [ + ["output_var0", "output_var1"], + ["output_var0", "output_var1", "output_var2"], + ]: + for sections in [ + [], + [1, 2], + [2, 1], + [10, 14], + [1, 1, 1], + [2, 2, 2], + [3, 3, 3], + [3, 7, 14], + ]: for num in [0, 3]: for axis in [0, 1, 2, 3]: self.batch = batch self.num_input = num_input self.dims = dims - dics = [{ - "sections": sections, - "num": num, - "axis": axis - }, {}] - - dics_intput = [{ - "X": ["split_input"], - "AxisTensor": ["AxisTensor"], - "SectionsTensorList": [ - "SectionsTensorList1", - "SectionsTensorList2" - ] - }, { - "X": ["split_input"] - }] - dics_intputs = [{ - "AxisTensor": - TensorConfig(data_gen=partial( - generate_AxisTensor, dics)), - "SectionsTensorList1": - TensorConfig(data_gen=partial( - generate_SectionsTensorList1, - dics)), - "SectionsTensorList2": - TensorConfig(data_gen=partial( - generate_SectionsTensorList2, dics)) - }, {}] - - ops_config = [{ - "op_type": - "split", - "op_inputs": - dics_intput[num_input], - "op_outputs": { - "Out": Out + dics = [ + { + "sections": sections, + "num": num, + "axis": axis, + }, + {}, + ] + + dics_intput = [ + { + "X": ["split_input"], + "AxisTensor": ["AxisTensor"], + "SectionsTensorList": [ + "SectionsTensorList1", + "SectionsTensorList2", + ], }, - "op_attrs": - dics[0] - }] + {"X": ["split_input"]}, + ] + dics_intputs = [ + { + "AxisTensor": TensorConfig( + data_gen=partial( + generate_AxisTensor, dics + ) + ), + "SectionsTensorList1": TensorConfig( + data_gen=partial( + generate_SectionsTensorList1, + dics, + ) + ), + "SectionsTensorList2": TensorConfig( + data_gen=partial( + generate_SectionsTensorList2, + dics, + ) + ), + }, + {}, + ] + + ops_config = [ + { + "op_type": "split", + "op_inputs": dics_intput[num_input], + "op_outputs": {"Out": Out}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights=dics_intputs[num_input], inputs={ - "split_input": - TensorConfig(data_gen=partial( - generate_input1, dics, batch)) + "split_input": TensorConfig( + data_gen=partial( + generate_input1, dics, batch + ) + ) }, - outputs=Out) + outputs=Out, + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { @@ -216,30 +239,35 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): - def teller1(program_config, predictor_config): if len(program_config.weights) == 3: return True return False self.add_skip_case( - teller1, SkipReasons.TRT_NOT_SUPPORT, - "INPUT AxisTensor AND SectionsTensorList NOT SUPPORT.") + teller1, + SkipReasons.TRT_NOT_SUPPORT, + "INPUT AxisTensor AND SectionsTensorList NOT SUPPORT.", + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_squeeze2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_squeeze2.py index 31bf484c59950..2786045209144 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_squeeze2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_squeeze2.py @@ -22,7 +22,6 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs attrs = [ @@ -40,25 +39,25 @@ def sample_program_configs(self): self.dims = dims self.axes = axes dics = [{"axes": axes}] - ops_config = [{ - "op_type": "squeeze2", - "op_inputs": { - "X": ["in_data"] - }, - "op_outputs": { - "Out": ["out_data"], - "XShape": ["XShape_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "squeeze2", + "op_inputs": {"X": ["in_data"]}, + "op_outputs": { + "Out": ["out_data"], + "XShape": ["XShape_data"], + }, + "op_attrs": dics[0], + } + ] # new_axes is the update of axes new_axes = list(axes) for i in range(len(new_axes)): - if (new_axes[i] < 0): + if new_axes[i] < 0: new_axes[i] += dims - if (max(new_axes) >= dims): + if max(new_axes) >= dims: continue - # generate input data + # generate input data self.input_shape = [1] * dims for i in range(dims): self.input_shape[i] = np.random.randint(1, 20) @@ -68,24 +67,26 @@ def generate_input1(attrs: List[Dict[str, Any]], batch): for i in new_axes: self.input_shape[i] = 1 return np.random.random(self.input_shape).astype( - np.float32) + np.float32 + ) ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "in_data": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)) + "in_data": TensorConfig( + data_gen=partial(generate_input1, dics, batch) + ) }, - outputs=["out_data"]) + outputs=["out_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): max_shape = list(self.input_shape) min_shape = list(self.input_shape) @@ -112,19 +113,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_stack.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_stack.py index dc2c8bb41626b..91e5e499b19b4 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_stack.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_stack.py @@ -22,7 +22,6 @@ class TrtConvertStackTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -31,14 +30,13 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: attrs = [ program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - #The input dimension should be less than the set axis. + # The input dimension should be less than the set axis. if len(inputs['stack_input1'].shape) < attrs[0]['axis']: return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: return np.random.random([batch, 3, 24, 24]).astype(np.float32) @@ -74,103 +72,107 @@ def generate_input3(attrs: List[Dict[str, Any]], batch): for axis in [-2, -1, 0, 1, 2, 3]: self.dims = dims dics = [{"axis": axis}, {}] - ops_config = [{ - "op_type": "stack", - "op_inputs": { - "X": - ["stack_input1", "stack_input2", "stack_input3"] - }, - "op_outputs": { - "Y": ["stack_output"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "stack", + "op_inputs": { + "X": [ + "stack_input1", + "stack_input2", + "stack_input3", + ] + }, + "op_outputs": {"Y": ["stack_output"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "stack_input1": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)), - "stack_input2": - TensorConfig( - data_gen=partial(generate_input2, dics, batch)), - "stack_input3": - TensorConfig( - data_gen=partial(generate_input3, dics, batch)) + "stack_input1": TensorConfig( + data_gen=partial(generate_input1, dics, batch) + ), + "stack_input2": TensorConfig( + data_gen=partial(generate_input2, dics, batch) + ), + "stack_input3": TensorConfig( + data_gen=partial(generate_input3, dics, batch) + ), }, - outputs=["stack_output"]) + outputs=["stack_output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { "stack_input1": [1, 3, 24, 24], "stack_input2": [1, 3, 24, 24], - "stack_input3": [1, 3, 24, 24] + "stack_input3": [1, 3, 24, 24], } self.dynamic_shape.max_input_shape = { "stack_input1": [4, 3, 48, 48], "stack_input2": [4, 3, 48, 48], - "stack_input3": [4, 3, 48, 48] + "stack_input3": [4, 3, 48, 48], } self.dynamic_shape.opt_input_shape = { "stack_input1": [1, 3, 24, 24], "stack_input2": [1, 3, 24, 24], - "stack_input3": [1, 3, 24, 24] + "stack_input3": [1, 3, 24, 24], } elif self.dims == 3: self.dynamic_shape.min_input_shape = { "stack_input1": [1, 3, 24], "stack_input2": [1, 3, 24], - "stack_input3": [1, 3, 24] + "stack_input3": [1, 3, 24], } self.dynamic_shape.max_input_shape = { "stack_input1": [4, 3, 48], "stack_input2": [4, 3, 48], - "stack_input3": [4, 3, 48] + "stack_input3": [4, 3, 48], } self.dynamic_shape.opt_input_shape = { "stack_input1": [1, 3, 24], "stack_input2": [1, 3, 24], - "stack_input3": [1, 3, 24] + "stack_input3": [1, 3, 24], } elif self.dims == 2: self.dynamic_shape.min_input_shape = { "stack_input1": [1, 24], "stack_input2": [1, 24], - "stack_input3": [1, 24] + "stack_input3": [1, 24], } self.dynamic_shape.max_input_shape = { "stack_input1": [4, 48], "stack_input2": [4, 48], - "stack_input3": [4, 48] + "stack_input3": [4, 48], } self.dynamic_shape.opt_input_shape = { "stack_input1": [1, 24], "stack_input2": [1, 24], - "stack_input3": [1, 24] + "stack_input3": [1, 24], } elif self.dims == 1: self.dynamic_shape.min_input_shape = { "stack_input1": [24], "stack_input2": [24], - "stack_input3": [24] + "stack_input3": [24], } self.dynamic_shape.max_input_shape = { "stack_input1": [48], "stack_input2": [48], - "stack_input3": [48] + "stack_input3": [48], } self.dynamic_shape.opt_input_shape = { "stack_input1": [24], "stack_input2": [24], - "stack_input3": [24] + "stack_input3": [24], } def clear_dynamic_shape(): @@ -191,19 +193,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_strided_slice.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_strided_slice.py index 4f517b447a191..b8549c918736a 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_strided_slice.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_strided_slice.py @@ -22,7 +22,6 @@ class TrtConvertStridedSliceTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -32,7 +31,6 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.random.random([1, 56, 56, 192]).astype(np.float32) @@ -42,42 +40,47 @@ def generate_input1(attrs: List[Dict[str, Any]]): for decrease_axis in [[]]: for infer_flags in [[1, 1]]: for strides in [[2, 2]]: - dics = [{ - "axes": axes, - "starts": starts, - "ends": ends, - "decrease_axis": decrease_axis, - "infer_flags": infer_flags, - "strides": strides - }] - - ops_config = [{ - "op_type": "strided_slice", - "op_inputs": { - "Input": ["input_data"] - }, - "op_outputs": { - "Out": ["slice_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "axes": axes, + "starts": starts, + "ends": ends, + "decrease_axis": decrease_axis, + "infer_flags": infer_flags, + "strides": strides, + } + ] + + ops_config = [ + { + "op_type": "strided_slice", + "op_inputs": {"Input": ["input_data"]}, + "op_outputs": { + "Out": ["slice_output_data"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dics + ) + ) }, - outputs=["slice_output_data"]) + outputs=["slice_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 56, 56, 192] @@ -118,71 +121,81 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 def test(self): self.run_test() class TrtConvertStridedSliceTest2(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): return np.random.random([1, 56, 56, 192]).astype(np.float32) for axes in [[1, 2], [2, 3], [1, 3]]: - for starts in [[-10, 1], [-10, 20], [-10, 15], [-10, 16], [-10, - 20]]: + for starts in [ + [-10, 1], + [-10, 20], + [-10, 15], + [-10, 16], + [-10, 20], + ]: for ends in [[-9, 10000], [-9, -1], [-9, 40]]: for decrease_axis in [[]]: for infer_flags in [[1, 1]]: for strides in [[2, 2]]: - dics = [{ - "axes": axes, - "starts": starts, - "ends": ends, - "decrease_axis": [axes[0]], - "infer_flags": infer_flags, - "strides": strides - }] - - ops_config = [{ - "op_type": "strided_slice", - "op_inputs": { - "Input": ["input_data"] - }, - "op_outputs": { - "Out": ["slice_output_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "axes": axes, + "starts": starts, + "ends": ends, + "decrease_axis": [axes[0]], + "infer_flags": infer_flags, + "strides": strides, + } + ] + + ops_config = [ + { + "op_type": "strided_slice", + "op_inputs": {"Input": ["input_data"]}, + "op_outputs": { + "Out": ["slice_output_data"] + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dics + ) + ) }, - outputs=["slice_output_data"]) + outputs=["slice_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): self.dynamic_shape.min_input_shape = { "input_data": [1, 56, 56, 192] diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_sum.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_sum.py index 673d20313e122..f7534215c6247 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_sum.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_sum.py @@ -22,12 +22,10 @@ class TrtConvertSumTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(batch): if self.dims == 4: return np.ones([batch, 3, 24, 24]).astype(np.float32) @@ -61,99 +59,101 @@ def generate_input3(batch): for dims in [1, 2, 3, 4]: for batch in [1, 4]: self.dims = dims - ops_config = [{ - "op_type": "sum", - "op_inputs": { - "X": ["input1", "input2", "input3"] - }, - "op_outputs": { - "Out": ["output"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "sum", + "op_inputs": {"X": ["input1", "input2", "input3"]}, + "op_outputs": {"Out": ["output"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1": - TensorConfig(data_gen=partial(generate_input1, batch)), - "input2": - TensorConfig(data_gen=partial(generate_input2, batch)), - "input3": - TensorConfig(data_gen=partial(generate_input3, batch)) + "input1": TensorConfig( + data_gen=partial(generate_input1, batch) + ), + "input2": TensorConfig( + data_gen=partial(generate_input2, batch) + ), + "input3": TensorConfig( + data_gen=partial(generate_input3, batch) + ), }, - outputs=["output"]) + outputs=["output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): if self.dims == 4: self.dynamic_shape.min_input_shape = { "input1": [1, 3, 24, 24], "input2": [1, 3, 24, 24], - "input3": [1, 3, 24, 24] + "input3": [1, 3, 24, 24], } self.dynamic_shape.max_input_shape = { "input1": [4, 3, 48, 48], "input2": [4, 3, 48, 48], - "input3": [4, 3, 48, 48] + "input3": [4, 3, 48, 48], } self.dynamic_shape.opt_input_shape = { "input1": [1, 3, 24, 24], "input2": [1, 3, 24, 24], - "input3": [1, 3, 24, 24] + "input3": [1, 3, 24, 24], } elif self.dims == 3: self.dynamic_shape.min_input_shape = { "input1": [1, 3, 24], "input2": [1, 3, 24], - "input3": [1, 3, 24] + "input3": [1, 3, 24], } self.dynamic_shape.max_input_shape = { "input1": [4, 3, 48], "input2": [4, 3, 48], - "input3": [4, 3, 48] + "input3": [4, 3, 48], } self.dynamic_shape.opt_input_shape = { "input1": [1, 3, 24], "input2": [1, 3, 24], - "input3": [1, 3, 24] + "input3": [1, 3, 24], } elif self.dims == 2: self.dynamic_shape.min_input_shape = { "input1": [1, 24], "input2": [1, 24], - "input3": [1, 24] + "input3": [1, 24], } self.dynamic_shape.max_input_shape = { "input1": [4, 48], "input2": [4, 48], - "input3": [4, 48] + "input3": [4, 48], } self.dynamic_shape.opt_input_shape = { "input1": [1, 24], "input2": [1, 24], - "input3": [1, 24] + "input3": [1, 24], } elif self.dims == 1: self.dynamic_shape.min_input_shape = { "input1": [24], "input2": [24], - "input3": [24] + "input3": [24], } self.dynamic_shape.max_input_shape = { "input1": [48], "input2": [48], - "input3": [48] + "input3": [48], } self.dynamic_shape.opt_input_shape = { "input1": [24], "input2": [24], - "input3": [24] + "input3": [24], } def clear_dynamic_shape(): @@ -162,7 +162,7 @@ def clear_dynamic_shape(): self.dynamic_shape.opt_input_shape = {} def generate_trt_nodes_num(dynamic_shape): - if (self.dims == 1 and not dynamic_shape): + if self.dims == 1 and not dynamic_shape: return 0, 5 return 1, 4 @@ -170,10 +170,12 @@ def generate_trt_nodes_num(dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-5 + False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-3 + False + ), 1e-3 # for dynamic_shape generate_dynamic_shape() @@ -188,12 +190,10 @@ def test(self): # special case when sum having olny one input class TrtConvertSumTest1(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(batch): if self.dims == 4: return np.ones([batch, 3, 24, 24]).astype(np.float32) @@ -207,31 +207,31 @@ def generate_input1(batch): for dims in [1, 2, 3, 4]: for batch in [1, 4]: self.dims = dims - ops_config = [{ - "op_type": "sum", - "op_inputs": { - "X": ["input1"] - }, - "op_outputs": { - "Out": ["output"] - }, - "op_attrs": {} - }] + ops_config = [ + { + "op_type": "sum", + "op_inputs": {"X": ["input1"]}, + "op_outputs": {"Out": ["output"]}, + "op_attrs": {}, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input1": - TensorConfig(data_gen=partial(generate_input1, batch)), + "input1": TensorConfig( + data_gen=partial(generate_input1, batch) + ), }, - outputs=["output"]) + outputs=["output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(): if self.dims == 4: self.dynamic_shape.min_input_shape = {"input1": [1, 3, 24, 24]} @@ -268,7 +268,7 @@ def clear_dynamic_shape(): self.dynamic_shape.opt_input_shape = {} def generate_trt_nodes_num(dynamic_shape): - if (self.dims == 1 and not dynamic_shape): + if self.dims == 1 and not dynamic_shape: return 0, 3 return 1, 2 @@ -276,10 +276,12 @@ def generate_trt_nodes_num(dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-5 + False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - False), 1e-3 + False + ), 1e-3 # for dynamic_shape generate_dynamic_shape() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_swish.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_swish.py index 1b5398bb483ad..b31e165c2e0f5 100755 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_swish.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_swish.py @@ -22,12 +22,10 @@ class TrtConvertSwishTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(dims, attrs: List[Dict[str, Any]]): if dims == 1: return np.ones([3]).astype(np.float32) @@ -44,33 +42,34 @@ def generate_input1(dims, attrs: List[Dict[str, Any]]): dics = [{"beta": beta}] - ops_config = [{ - "op_type": "swish", - "op_inputs": { - "X": ["input_data"], - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "swish", + "op_inputs": { + "X": ["input_data"], + }, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig( - data_gen=partial(generate_input1, dims, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dims, dics) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -115,19 +114,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), (1e-3, 1e-3) + attrs, False + ), (1e-3, 1e-3) # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), (1e-3, 1e-3) + attrs, True + ), (1e-3, 1e-3) def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_tile.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_tile.py index 87cb2c6630d05..d65d7e3c29f96 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_tile.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_tile.py @@ -25,7 +25,6 @@ class TrtConvertTileTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs attrs = [ @@ -38,38 +37,37 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self, *args, **kwargs): - def generate_input1(attrs: List[Dict[str, Any]]): return np.ones([1, 2, 3, 4]).astype(np.float32) dics = [{"repeat_times": kwargs['repeat_times']}] - ops_config = [{ - "op_type": "tile", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["tile_output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "tile", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["tile_output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1, dics)) + "input_data": TensorConfig( + data_gen=partial(generate_input1, dics) + ) }, - outputs=["tile_output_data"]) + outputs=["tile_output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} @@ -98,19 +96,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 @given(repeat_times=st.sampled_from([[100], [1, 2], [0, 3], [1, 2, 100]])) def test(self, *args, **kwargs): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k.py index 7e8caf4a7e457..b8150238407a2 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k.py @@ -22,7 +22,6 @@ class TrtConvertActivationTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -44,34 +43,37 @@ def generate_input1(dims, batch, attrs: List[Dict[str, Any]]): for k in [1, 3]: self.dims = dims dics = [{"k": k}] - ops_config = [{ - "op_type": "top_k", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"], - "Indices": ["indices_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "top_k", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": { + "Out": ["output_data"], + "Indices": ["indices_data"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dims, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dims, batch, dics + ) + ) }, - outputs=["output_data", "indices_data"]) + outputs=["output_data", "indices_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -114,19 +116,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 ## for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k_v2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k_v2.py index c4114e0d4c5a8..477ea649effd3 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k_v2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_top_k_v2.py @@ -22,7 +22,6 @@ class TrtConvertActivationTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs attrs = [ @@ -53,40 +52,48 @@ def generate_input1(dims, batch, attrs: List[Dict[str, Any]]): for sort in [True, False]: self.dims = dims self.sort = sort - dics = [{ - "k": k, - "axis": axis, - "largest": largest, - "sorted": sort - }] - ops_config = [{ - "op_type": "top_k_v2", - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"], - "Indices": ["indices_data"] - }, - "op_attrs": dics[0] - }] + dics = [ + { + "k": k, + "axis": axis, + "largest": largest, + "sorted": sort, + } + ] + ops_config = [ + { + "op_type": "top_k_v2", + "op_inputs": {"X": ["input_data"]}, + "op_outputs": { + "Out": ["output_data"], + "Indices": ["indices_data"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dims, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, + dims, + batch, + dics, + ) + ) }, - outputs=["output_data", "indices_data"]) + outputs=["output_data", "indices_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -131,19 +138,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_transpose.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_transpose.py index 684a47f423218..9ebcd87399230 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_transpose.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_transpose.py @@ -22,7 +22,6 @@ class TrtConvertTransposeTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: inputs = program_config.inputs weights = program_config.weights @@ -32,14 +31,13 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: program_config.ops[i].attrs for i in range(len(program_config.ops)) ] - #The shape of input and axis should be equal. + # The shape of input and axis should be equal. if len(inputs['transpose_input'].shape) != len(attrs[0]['axis']): return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch): if self.dims == 4: return np.ones([batch, 3, 24, 24]).astype(np.float32) @@ -50,37 +48,43 @@ def generate_input1(attrs: List[Dict[str, Any]], batch): for dims in [2, 3, 4]: for batch in [1, 2, 4]: - for axis in [[0, 1, 3, 2], [0, 3, 2, 1], [3, 2, 0, 1], - [0, 1, 2, 3], [0, 1, 2], [2, 0, 1], [1, 0], [0, - 1]]: + for axis in [ + [0, 1, 3, 2], + [0, 3, 2, 1], + [3, 2, 0, 1], + [0, 1, 2, 3], + [0, 1, 2], + [2, 0, 1], + [1, 0], + [0, 1], + ]: self.dims = dims dics = [{"axis": axis}, {}] - ops_config = [{ - "op_type": "transpose", - "op_inputs": { - "X": ["transpose_input"] - }, - "op_outputs": { - "Out": ["transpose_out"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "transpose", + "op_inputs": {"X": ["transpose_input"]}, + "op_outputs": {"Out": ["transpose_out"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "transpose_input": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)) + "transpose_input": TensorConfig( + data_gen=partial(generate_input1, dics, batch) + ) }, - outputs=["transpose_out"]) + outputs=["transpose_out"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 4: self.dynamic_shape.min_input_shape = { @@ -134,19 +138,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unary.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unary.py index 65656fd96d6c2..56b17df2dd04e 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unary.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unary.py @@ -22,7 +22,6 @@ class TrtConvertActivationTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -42,40 +41,54 @@ def generate_input1(dims, batch, attrs: List[Dict[str, Any]]): for dims in [1, 2, 3, 4]: for batch in [1, 4]: for op_type in [ - "exp", "log", "sqrt", "abs", "sin", "cos", "tan", - "sinh", "cosh", "asin", "acos", "atan", "asinh", - "atanh", "ceil", "floor" + "exp", + "log", + "sqrt", + "abs", + "sin", + "cos", + "tan", + "sinh", + "cosh", + "asin", + "acos", + "atan", + "asinh", + "atanh", + "ceil", + "floor", ]: self.dims = dims dics = [{}] - ops_config = [{ - "op_type": op_type, - "op_inputs": { - "X": ["input_data"] - }, - "op_outputs": { - "Out": ["output_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": op_type, + "op_inputs": {"X": ["input_data"]}, + "op_outputs": {"Out": ["output_data"]}, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial( - generate_input1, dims, batch, dics)) + "input_data": TensorConfig( + data_gen=partial( + generate_input1, dims, batch, dics + ) + ) }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if self.dims == 1: self.dynamic_shape.min_input_shape = {"input_data": [1]} @@ -118,19 +131,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def test(self): self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unfold.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unfold.py index 685658ca74600..5ec187daef4e9 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unfold.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unfold.py @@ -22,46 +22,46 @@ class TrtConvertUnfold(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(): return np.random.random([1, 3, 24, 24]).astype(np.float32) - ops_config = [{ - "op_type": "unfold", - "op_inputs": { - "X": ["input_data"], - }, - "op_outputs": { - "Y": ["output_data"] - }, - "op_attrs": { - "dilations": [1, 1], - "kernel_sizes": [4, 4], - "paddings": [0, 0, 0, 0], - "strides": [1, 1], + ops_config = [ + { + "op_type": "unfold", + "op_inputs": { + "X": ["input_data"], + }, + "op_outputs": {"Y": ["output_data"]}, + "op_attrs": { + "dilations": [1, 1], + "kernel_sizes": [4, 4], + "paddings": [0, 0, 0, 0], + "strides": [1, 1], + }, } - }] + ] ops = self.generate_op_config(ops_config) for i in range(10): program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "input_data": - TensorConfig(data_gen=partial(generate_input1)), + "input_data": TensorConfig( + data_gen=partial(generate_input1) + ), }, - outputs=["output_data"]) + outputs=["output_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): self.dynamic_shape.min_input_shape = { "input_data": [1, 3, 4, 4], diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unsqueeze2.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unsqueeze2.py index b18e98e969099..d79f4ddf7fc6d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unsqueeze2.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_unsqueeze2.py @@ -22,7 +22,6 @@ class TrtConvertSplitTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True @@ -34,17 +33,17 @@ def sample_program_configs(self): self.dims = dims self.axes = axes dics = [{"axes": axes}] - ops_config = [{ - "op_type": "unsqueeze2", - "op_inputs": { - "X": ["in_data"] - }, - "op_outputs": { - "Out": ["out_data"], - "XShape": ["XShape_data"] - }, - "op_attrs": dics[0] - }] + ops_config = [ + { + "op_type": "unsqueeze2", + "op_inputs": {"X": ["in_data"]}, + "op_outputs": { + "Out": ["out_data"], + "XShape": ["XShape_data"], + }, + "op_attrs": dics[0], + } + ] # generate input data self.input_shape = [1] * dims @@ -54,24 +53,26 @@ def sample_program_configs(self): def generate_input1(attrs: List[Dict[str, Any]], batch): self.input_shape[0] = batch return np.random.random(self.input_shape).astype( - np.float32) + np.float32 + ) ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "in_data": - TensorConfig( - data_gen=partial(generate_input1, dics, batch)) + "in_data": TensorConfig( + data_gen=partial(generate_input1, dics, batch) + ) }, - outputs=["out_data"]) + outputs=["out_data"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): max_shape = list(self.input_shape) min_shape = list(self.input_shape) @@ -98,19 +99,23 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): pass diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box.py index ed9bb722a8a6d..a0d089c69c934 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box.py @@ -23,19 +23,19 @@ class TrtConvertYoloBoxTest(TrtLayerAutoScanTest): - def is_program_valid(self, program_config: ProgramConfig) -> bool: return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]], batch, channel): if attrs[0]['iou_aware'] == True: - return np.ones([batch, 3 * (channel + 6), 13, - 13]).astype(np.float32) + return np.ones([batch, 3 * (channel + 6), 13, 13]).astype( + np.float32 + ) else: - return np.ones([batch, 3 * (channel + 5), 13, - 13]).astype(np.float32) + return np.ones([batch, 3 * (channel + 5), 13, 13]).astype( + np.float32 + ) def generate_input2(attrs: List[Dict[str, Any]], batch): return np.random.random([batch, 2]).astype(np.int32) @@ -49,89 +49,92 @@ def generate_input2(attrs: List[Dict[str, Any]], batch): for scale_x_y in [1.0, 0.9]: for iou_aware in [False, True]: for iou_aware_factor in [0.5]: - dics = [{ - "class_num": - class_num, - "anchors": - anchors, - "downsample_ratio": - downsample_ratio, - "conf_thresh": - conf_thresh, - "clip_bbox": - clip_bbox, - "scale_x_y": - scale_x_y, - "iou_aware": - iou_aware, - "iou_aware_factor": - iou_aware_factor - }, {}] - ops_config = [{ - "op_type": "yolo_box", - "op_inputs": { - "X": ["yolo_box_input"], - "ImgSize": ["imgsize"] - }, - "op_outputs": { - "Boxes": ["boxes"], - "Scores": ["scores"] + dics = [ + { + "class_num": class_num, + "anchors": anchors, + "downsample_ratio": downsample_ratio, + "conf_thresh": conf_thresh, + "clip_bbox": clip_bbox, + "scale_x_y": scale_x_y, + "iou_aware": iou_aware, + "iou_aware_factor": iou_aware_factor, }, - "op_attrs": dics[0] - }] + {}, + ] + ops_config = [ + { + "op_type": "yolo_box", + "op_inputs": { + "X": ["yolo_box_input"], + "ImgSize": ["imgsize"], + }, + "op_outputs": { + "Boxes": ["boxes"], + "Scores": ["scores"], + }, + "op_attrs": dics[0], + } + ] ops = self.generate_op_config( - ops_config) + ops_config + ) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "yolo_box_input": - TensorConfig( + "yolo_box_input": TensorConfig( data_gen=partial( generate_input1, - dics, batch, - class_num)), - "imgsize": - TensorConfig( + dics, + batch, + class_num, + ) + ), + "imgsize": TensorConfig( data_gen=partial( generate_input2, - dics, batch)) + dics, + batch, + ) + ), }, - outputs=["boxes", "scores"]) + outputs=["boxes", "scores"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): - + self, program_config + ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): if attrs[0]['iou_aware'] == True: channel = 3 * (attrs[0]['class_num'] + 6) self.dynamic_shape.min_input_shape = { "yolo_box_input": [1, channel, 12, 12], - "imgsize": [1, 2] + "imgsize": [1, 2], } self.dynamic_shape.max_input_shape = { "yolo_box_input": [4, channel, 24, 24], - "imgsize": [4, 2] + "imgsize": [4, 2], } self.dynamic_shape.opt_input_shape = { "yolo_box_input": [1, channel, 24, 24], - "imgsize": [1, 2] + "imgsize": [1, 2], } else: channel = 3 * (attrs[0]['class_num'] + 5) self.dynamic_shape.min_input_shape = { "yolo_box_input": [1, channel, 12, 12], - "imgsize": [1, 2] + "imgsize": [1, 2], } self.dynamic_shape.max_input_shape = { "yolo_box_input": [4, channel, 24, 24], - "imgsize": [4, 2] + "imgsize": [4, 2], } self.dynamic_shape.opt_input_shape = { "yolo_box_input": [1, channel, 24, 24], - "imgsize": [1, 2] + "imgsize": [1, 2], } def clear_dynamic_shape(): @@ -149,30 +152,35 @@ def generate_trt_nodes_num(attrs, dynamic_shape): clear_dynamic_shape() self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-5 + attrs, False + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, False), 1e-3 + attrs, False + ), 1e-3 # for dynamic_shape generate_dynamic_shape(attrs) self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-5 + attrs, True + ), 1e-5 self.trt_param.precision = paddle_infer.PrecisionType.Half yield self.create_inference_config(), generate_trt_nodes_num( - attrs, True), 1e-3 + attrs, True + ), 1e-3 def add_skip_trt_case(self): - def teller2(program_config, predictor_config): if len(self.dynamic_shape.min_input_shape) != 0 and os.name == 'nt': return True return False self.add_skip_case( - teller2, SkipReasons.TRT_NOT_SUPPORT, - "The output has diff between gpu and trt in Windows.") + teller2, + SkipReasons.TRT_NOT_SUPPORT, + "The output has diff between gpu and trt in Windows.", + ) def test(self): self.add_skip_trt_case() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box_head.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box_head.py index 08a09338bf27b..e8b4006933f57 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box_head.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_yolo_box_head.py @@ -22,17 +22,18 @@ class TrtConvertYoloBoxHeadTest(TrtLayerAutoScanTest): - def sample_program_configs(self): - def generate_input(attrs: List[Dict[str, Any]], batch, shape): gen_shape = shape.copy() gen_shape.insert(0, batch) return np.random.uniform(0, 1, gen_shape).astype("float32") input_shape = [[255, 19, 19], [255, 38, 38], [255, 76, 76]] - anchors = [[116, 90, 156, 198, 373, 326], [30, 61, 62, 45, 59, 119], - [10, 13, 16, 30, 33, 23]] + anchors = [ + [116, 90, 156, 198, 373, 326], + [30, 61, 62, 45, 59, 119], + [10, 13, 16, 30, 33, 23], + ] class_num = 80 for batch in [1, 4]: for i in range(len(anchors)): @@ -40,31 +41,40 @@ def generate_input(attrs: List[Dict[str, Any]], batch, shape): "anchors": anchors[i], "class_num": class_num, } - ops_config = [{ - "op_type": "yolo_box_head", - "op_inputs": { - "X": ["yolo_box_head_input"], - }, - "op_outputs": { - "Out": ["yolo_box_head_output"], - }, - "op_attrs": attrs_dict - }] + ops_config = [ + { + "op_type": "yolo_box_head", + "op_inputs": { + "X": ["yolo_box_head_input"], + }, + "op_outputs": { + "Out": ["yolo_box_head_output"], + }, + "op_attrs": attrs_dict, + } + ] ops = self.generate_op_config(ops_config) program_config = ProgramConfig( ops=ops, weights={}, inputs={ - "yolo_box_head_input": - TensorConfig(data_gen=partial( - generate_input, attrs_dict, batch, input_shape[i])) + "yolo_box_head_input": TensorConfig( + data_gen=partial( + generate_input, + attrs_dict, + batch, + input_shape[i], + ) + ) }, - outputs=["yolo_box_head_output"]) + outputs=["yolo_box_head_output"], + ) yield program_config def sample_predictor_configs( - self, program_config) -> (paddle_infer.Config, List[int], float): + self, program_config + ) -> (paddle_infer.Config, List[int], float): # for static_shape self.trt_param.precision = paddle_infer.PrecisionType.Float32 yield self.create_inference_config(), [1, 2], 1e-5 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_deformable_conv.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_deformable_conv.py index 21949e3d78048..eb79b4ccd1d98 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_deformable_conv.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_deformable_conv.py @@ -25,19 +25,18 @@ class TRTDeformableConvTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - input = fluid.data(name='input', - shape=self.input_size, - dtype=self.dtype) - offset = fluid.data(name='offset', - shape=self.offset_size, - dtype=self.dtype) - mask = fluid.data(name='mask', - shape=self.mask_size, - dtype=self.dtype) + input = fluid.data( + name='input', shape=self.input_size, dtype=self.dtype + ) + offset = fluid.data( + name='offset', shape=self.offset_size, dtype=self.dtype + ) + mask = fluid.data( + name='mask', shape=self.mask_size, dtype=self.dtype + ) output = fluid.layers.deformable_conv( input, @@ -50,19 +49,21 @@ def setUp(self): dilation=self.dilations, groups=self.groups, deformable_groups=self.deformable_groups, - im2col_step=self.im2col_step) + im2col_step=self.im2col_step, + ) self.feeds = { 'input': np.random.random(self.input_size).astype(self.dtype), 'offset': np.random.random(self.offset_size).astype(self.dtype), - 'mask': np.random.random(self.mask_size).astype(self.dtype) + 'mask': np.random.random(self.mask_size).astype(self.dtype), } self.enable_trt = True dtype = AnalysisConfig.Precision.Float32 if self.dtype == 'float16': dtype = AnalysisConfig.Precision.Half self.trt_parameters = TRTDeformableConvTest.TensorRTParam( - 1 << 30, self.bs, 0, dtype, False, False) + 1 << 30, self.bs, 0, dtype, False, False + ) self.fetch_list = [output] def set_params(self): @@ -77,13 +78,21 @@ def set_params(self): self.input_size = [self.bs, 8, 4, 4] self.num_filters = 8 self.filter_size = 3 - offset_c = 2 * self.deformable_groups * self.filter_size * self.filter_size + offset_c = ( + 2 * self.deformable_groups * self.filter_size * self.filter_size + ) mask_c = self.deformable_groups * self.filter_size * self.filter_size self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] self.dtype = 'float32' @@ -93,7 +102,8 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_dynamic_shape.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_dynamic_shape.py index ae5f9f94673ab..ee021b24a3bfa 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_dynamic_shape.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_dynamic_shape.py @@ -21,27 +21,32 @@ class TRTDynamicShapeTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 16, 16], - dtype="float32") - out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - groups=1, - padding=[1, 1], - bias_attr=False, - act=None) + data = fluid.data( + name="data", shape=[-1, 3, 16, 16], dtype="float32" + ) + out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + groups=1, + padding=[1, 1], + bias_attr=False, + act=None, + ) self.feeds = self.set_feeds() self.enable_trt = True self.trt_parameters = TRTDynamicShapeTest.TensorRTParam( - 1 << 30, 1, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 1, 1, AnalysisConfig.Precision.Float32, False, False + ) self.dynamic_shape_params = TRTDynamicShapeTest.DynamicShapeParam( - {'data': [1, 3, 8, 8]}, {'data': [1, 3, 32, 32]}, - {'data': [1, 3, 16, 16]}, False) + {'data': [1, 3, 8, 8]}, + {'data': [1, 3, 32, 32]}, + {'data': [1, 3, 16, 16]}, + False, + ) self.fetch_list = [out] def set_feeds(self): @@ -56,7 +61,6 @@ def test_check_output(self): class TRTDynamicShapeOutOfBound1Test(TRTDynamicShapeTest): - def set_feeds(self): return { "data": np.random.random([1, 3, 64, 16]).astype("float32"), @@ -84,7 +88,6 @@ def test_check_output(self): class TRTDynamicShapeOutOfBound3Test(TRTDynamicShapeTest): - def set_feeds(self): return { "data": np.random.random([1, 3, 4, 16]).astype("float32"), diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_elementwise_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_elementwise_op.py index b24595e379791..7fb0baf11ffb0 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_elementwise_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_elementwise_op.py @@ -24,15 +24,14 @@ class TensorRTSubgraphPassElementwiseBroadcastTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data1 = fluid.data(name="data1", - shape=[-1, 3, 64, 64], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[-1, 3, 64, 1], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[-1, 3, 64, 64], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[-1, 3, 64, 1], dtype="float32" + ) eltwise_out = self.append_eltwise(data1, data2) out = fluid.layers.batch_norm(eltwise_out, is_test=True) self.feeds = { @@ -40,8 +39,11 @@ def setUp(self): "data2": np.random.random([1, 3, 64, 1]).astype("float32"), } self.enable_trt = True - self.trt_parameters = TensorRTSubgraphPassElementwiseBroadcastTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False) + self.trt_parameters = ( + TensorRTSubgraphPassElementwiseBroadcastTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False + ) + ) self.fetch_list = [out] def append_eltwise(self, data1, data2): @@ -54,26 +56,27 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassElementwiseBroadcastTest1( - TensorRTSubgraphPassElementwiseBroadcastTest): - + TensorRTSubgraphPassElementwiseBroadcastTest +): def append_eltwise(self, data1, data2): return fluid.layers.elementwise_sub(x=data1, y=data2, axis=0) class TensorRTSubgraphPassElementwiseBroadcastTest2( - TensorRTSubgraphPassElementwiseBroadcastTest): - + TensorRTSubgraphPassElementwiseBroadcastTest +): def append_eltwise(self, data1, data2): return fluid.layers.elementwise_mul(x=data1, y=data2, axis=0) class TensorRTSubgraphPassElementwiseBroadcastTest3( - TensorRTSubgraphPassElementwiseBroadcastTest): - + TensorRTSubgraphPassElementwiseBroadcastTest +): def append_eltwise(self, data1, data2): return fluid.layers.elementwise_div(x=data1, y=data2, axis=0) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_pass.py index 78436f644b4f2..606836c8d0f57 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_pass.py @@ -21,16 +21,14 @@ class FCFusePassTRTTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[32, 128, 2, 2], - dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=128, - num_flatten_dims=1, - act="relu") + data = fluid.data( + name="data", shape=[32, 128, 2, 2], dtype="float32" + ) + fc_out1 = fluid.layers.fc( + input=data, size=128, num_flatten_dims=1, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = { @@ -53,16 +51,14 @@ def test_check_output(self): class FCFusePassTRTStaticDims4Cols1Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[32, 128, 32, 8], - dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=64, - num_flatten_dims=1, - act="relu") + data = fluid.data( + name="data", shape=[32, 128, 32, 8], dtype="float32" + ) + fc_out1 = fluid.layers.fc( + input=data, size=64, num_flatten_dims=1, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = { @@ -70,7 +66,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = FCFusePassTRTStaticDims4Cols1Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -82,16 +79,14 @@ def test_check_output(self): class FCFusePassTRTStaticDims4Cols2Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[3, 24, 16, 16], - dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=32, - num_flatten_dims=2, - act="relu") + data = fluid.data( + name="data", shape=[3, 24, 16, 16], dtype="float32" + ) + fc_out1 = fluid.layers.fc( + input=data, size=32, num_flatten_dims=2, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = { @@ -99,7 +94,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = FCFusePassTRTStaticDims4Cols2Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -111,22 +107,27 @@ def test_check_output(self): class FCFusePassTRTDynamicDims2Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[32, 128], dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=64, - num_flatten_dims=1, - act="relu") + fc_out1 = fluid.layers.fc( + input=data, size=64, num_flatten_dims=1, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = {"data": np.random.random((32, 128)).astype("float32")} self.enable_trt = True self.trt_parameters = FCFusePassTRTDynamicDims2Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = FCFusePassTRTDynamicDims2Test.DynamicShapeParam( - {'data': [1, 128]}, {'data': [64, 128]}, {'data': [32, 128]}, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + FCFusePassTRTDynamicDims2Test.DynamicShapeParam( + {'data': [1, 128]}, + {'data': [64, 128]}, + {'data': [32, 128]}, + False, + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -138,23 +139,27 @@ def test_check_output(self): class FCFusePassTRTDynamicDims3Cols1Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[32, 128, 32], dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=64, - num_flatten_dims=1, - act="relu") + fc_out1 = fluid.layers.fc( + input=data, size=64, num_flatten_dims=1, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = {"data": np.random.random((32, 128, 32)).astype("float32")} self.enable_trt = True self.trt_parameters = FCFusePassTRTDynamicDims3Cols1Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = FCFusePassTRTDynamicDims3Cols1Test.DynamicShapeParam( - {'data': [1, 128, 32]}, {'data': [64, 128, 32]}, - {'data': [32, 128, 32]}, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + FCFusePassTRTDynamicDims3Cols1Test.DynamicShapeParam( + {'data': [1, 128, 32]}, + {'data': [64, 128, 32]}, + {'data': [32, 128, 32]}, + False, + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -166,23 +171,27 @@ def test_check_output(self): class FCFusePassTRTDynamicDims3Cols2Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[32, 128, 32], dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=64, - num_flatten_dims=2, - act="relu") + fc_out1 = fluid.layers.fc( + input=data, size=64, num_flatten_dims=2, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = {"data": np.random.random((32, 128, 32)).astype("float32")} self.enable_trt = True self.trt_parameters = FCFusePassTRTDynamicDims3Cols2Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = FCFusePassTRTDynamicDims3Cols2Test.DynamicShapeParam( - {'data': [1, 32, 32]}, {'data': [64, 256, 32]}, - {'data': [32, 128, 32]}, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + FCFusePassTRTDynamicDims3Cols2Test.DynamicShapeParam( + {'data': [1, 32, 32]}, + {'data': [64, 256, 32]}, + {'data': [32, 128, 32]}, + False, + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -194,16 +203,14 @@ def test_check_output(self): class FCFusePassTRTDynamicDims4Cols1Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[32, 12, 4, 6], - dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=64, - num_flatten_dims=1, - act="relu") + data = fluid.data( + name="data", shape=[32, 12, 4, 6], dtype="float32" + ) + fc_out1 = fluid.layers.fc( + input=data, size=64, num_flatten_dims=1, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = { @@ -211,10 +218,16 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = FCFusePassTRTDynamicDims4Cols1Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = FCFusePassTRTDynamicDims4Cols1Test.DynamicShapeParam( - {'data': [1, 12, 4, 6]}, {'data': [64, 12, 4, 6]}, - {'data': [32, 12, 4, 6]}, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + FCFusePassTRTDynamicDims4Cols1Test.DynamicShapeParam( + {'data': [1, 12, 4, 6]}, + {'data': [64, 12, 4, 6]}, + {'data': [32, 12, 4, 6]}, + False, + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -226,16 +239,14 @@ def test_check_output(self): class FCFusePassTRTDynamicDims4Cols2Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[32, 128, 32, 32], - dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=64, - num_flatten_dims=2, - act="relu") + data = fluid.data( + name="data", shape=[32, 128, 32, 32], dtype="float32" + ) + fc_out1 = fluid.layers.fc( + input=data, size=64, num_flatten_dims=2, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = { @@ -243,10 +254,16 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = FCFusePassTRTDynamicDims4Cols2Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = FCFusePassTRTDynamicDims4Cols2Test.DynamicShapeParam( - {'data': [1, 64, 32, 32]}, {'data': [64, 256, 32, 32]}, - {'data': [32, 128, 32, 32]}, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + FCFusePassTRTDynamicDims4Cols2Test.DynamicShapeParam( + {'data': [1, 64, 32, 32]}, + {'data': [64, 256, 32, 32]}, + {'data': [32, 128, 32, 32]}, + False, + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -258,16 +275,14 @@ def test_check_output(self): class FCFusePassTRTDynamicDims4Cols3Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[32, 128, 32, 32], - dtype="float32") - fc_out1 = fluid.layers.fc(input=data, - size=64, - num_flatten_dims=3, - act="relu") + data = fluid.data( + name="data", shape=[32, 128, 32, 32], dtype="float32" + ) + fc_out1 = fluid.layers.fc( + input=data, size=64, num_flatten_dims=3, act="relu" + ) out = fluid.layers.softmax(input=fc_out1) self.feeds = { @@ -275,10 +290,16 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = FCFusePassTRTDynamicDims4Cols3Test.TensorRTParam( - 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False) - self.dynamic_shape_params = FCFusePassTRTDynamicDims4Cols3Test.DynamicShapeParam( - {'data': [1, 128, 32, 32]}, {'data': [64, 128, 32, 32]}, - {'data': [32, 128, 32, 32]}, False) + 1 << 30, 32, 2, AnalysisConfig.Precision.Float32, False, False + ) + self.dynamic_shape_params = ( + FCFusePassTRTDynamicDims4Cols3Test.DynamicShapeParam( + {'data': [1, 128, 32, 32]}, + {'data': [64, 128, 32, 32]}, + {'data': [32, 128, 32, 32]}, + False, + ) + ) self.fetch_list = [out] def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_quant_dequant_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_quant_dequant_pass.py index 847c836972fea..179dcd140d8a2 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_quant_dequant_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_fc_fuse_quant_dequant_pass.py @@ -23,19 +23,19 @@ class FCQuantDequantFusePassTRTDims3Cols1Test(QuantDequantTest): - def setUp(self): - def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') - fc_out = fluid.layers.fc(input=self.data, - size=10, - num_flatten_dims=1, - bias_attr=False, - act="relu") + fc_out = fluid.layers.fc( + input=self.data, + size=10, + num_flatten_dims=1, + bias_attr=False, + act="relu", + ) result = fluid.layers.relu(fc_out) loss = fluid.layers.cross_entropy(input=result, label=self.label) avg_loss = paddle.mean(loss) @@ -44,62 +44,64 @@ def network(): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random((1, 28, 28)).astype("float32")} self.fetch_list = [result] self.enable_trt = True - self.trt_parameters = FCQuantDequantFusePassTRTDims3Cols1Test.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) - self.dynamic_shape_params = FCQuantDequantFusePassTRTDims3Cols1Test.DynamicShapeParam( - { - 'data': [1, 28, 28], - 'reshape2_1.tmp_0': [1, 1, 10] - }, { - 'data': [2, 28, 28], - 'reshape2_1.tmp_0': [2, 1, 10] - }, { - 'data': [1, 28, 28], - 'reshape2_1.tmp_0': [1, 1, 10] - }, False) + self.trt_parameters = ( + FCQuantDequantFusePassTRTDims3Cols1Test.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) + ) + self.dynamic_shape_params = ( + FCQuantDequantFusePassTRTDims3Cols1Test.DynamicShapeParam( + {'data': [1, 28, 28], 'reshape2_1.tmp_0': [1, 1, 10]}, + {'data': [2, 28, 28], 'reshape2_1.tmp_0': [2, 1, 10]}, + {'data': [1, 28, 28], 'reshape2_1.tmp_0': [1, 1, 10]}, + False, + ) + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' def test_check_output(self): - #self.quant_dequant() + # self.quant_dequant() if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1e-2, - flatten=False, - rtol=1e-2) + self.check_output_with_option( + use_gpu, atol=1e-2, flatten=False, rtol=1e-2 + ) self.assertTrue( PassVersionChecker.IsCompatible( - 'quant_conv2d_dequant_fuse_pass')) + 'quant_conv2d_dequant_fuse_pass' + ) + ) class FCQuantDequantFusePassTRTDims3Cols2Test(QuantDequantTest): - def setUp(self): - def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') - fc_out = fluid.layers.fc(input=self.data, - size=28, - num_flatten_dims=2, - bias_attr=False, - act=None) + fc_out = fluid.layers.fc( + input=self.data, + size=28, + num_flatten_dims=2, + bias_attr=False, + act=None, + ) c_out = fluid.layers.reshape(fc_out, shape=[0, 784]) result = fluid.layers.relu(c_out) loss = fluid.layers.cross_entropy(input=result, label=self.label) @@ -109,64 +111,66 @@ def network(): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random((1, 28, 28)).astype("float32")} self.fetch_list = [result] self.enable_trt = True - self.trt_parameters = FCQuantDequantFusePassTRTDims3Cols2Test.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) - self.dynamic_shape_params = FCQuantDequantFusePassTRTDims3Cols2Test.DynamicShapeParam( - { - 'data': [1, 28, 28], - 'reshape2_0.tmp_0': [1, 784] - }, { - 'data': [4, 28, 28], - 'reshape2_0.tmp_0': [4, 784] - }, { - 'data': [1, 28, 28], - 'reshape2_0.tmp_0': [1, 784] - }, False) + self.trt_parameters = ( + FCQuantDequantFusePassTRTDims3Cols2Test.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) + ) + self.dynamic_shape_params = ( + FCQuantDequantFusePassTRTDims3Cols2Test.DynamicShapeParam( + {'data': [1, 28, 28], 'reshape2_0.tmp_0': [1, 784]}, + {'data': [4, 28, 28], 'reshape2_0.tmp_0': [4, 784]}, + {'data': [1, 28, 28], 'reshape2_0.tmp_0': [1, 784]}, + False, + ) + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' def test_check_output(self): - #self.quant_dequant() + # self.quant_dequant() if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1e-1, - flatten=False, - rtol=1e-1) + self.check_output_with_option( + use_gpu, atol=1e-1, flatten=False, rtol=1e-1 + ) self.assertTrue( PassVersionChecker.IsCompatible( - 'quant_conv2d_dequant_fuse_pass')) + 'quant_conv2d_dequant_fuse_pass' + ) + ) class FCQuantDequantFusePassTRTDims3Cols3Test(QuantDequantTest): - def setUp(self): - def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') label_shape = fluid.layers.reshape(self.label, shape=[1, 1, 1]) reshape_out = fluid.layers.reshape(self.data, shape=[1, 14, 14, 4]) - fc_out = fluid.layers.fc(input=reshape_out, - size=14, - num_flatten_dims=3, - bias_attr=False, - act=None) + fc_out = fluid.layers.fc( + input=reshape_out, + size=14, + num_flatten_dims=3, + bias_attr=False, + act=None, + ) c_out = fluid.layers.reshape(fc_out, shape=[1, 1, 2744]) result = fluid.layers.relu(c_out) loss = fluid.layers.cross_entropy(input=result, label=label_shape) @@ -176,49 +180,60 @@ def network(): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random((1, 28, 28)).astype("float32")} self.fetch_list = [result] self.enable_trt = True - self.trt_parameters = FCQuantDequantFusePassTRTDims3Cols3Test.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) - self.dynamic_shape_params = FCQuantDequantFusePassTRTDims3Cols3Test.DynamicShapeParam( - { - 'data': [1, 28, 28], - "reshape2_1.tmp_0": [1, 14, 14, 4], - "reshape2_2.tmp_0": [1, 1, 2744] - }, { - 'data': [4, 28, 28], - "reshape2_1.tmp_0": [4, 14, 14, 4], - "reshape2_2.tmp_0": [4, 1, 2744] - }, { - 'data': [1, 28, 28], - "reshape2_1.tmp_0": [1, 14, 14, 4], - "reshape2_2.tmp_0": [1, 1, 2744] - }, False) + self.trt_parameters = ( + FCQuantDequantFusePassTRTDims3Cols3Test.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) + ) + self.dynamic_shape_params = ( + FCQuantDequantFusePassTRTDims3Cols3Test.DynamicShapeParam( + { + 'data': [1, 28, 28], + "reshape2_1.tmp_0": [1, 14, 14, 4], + "reshape2_2.tmp_0": [1, 1, 2744], + }, + { + 'data': [4, 28, 28], + "reshape2_1.tmp_0": [4, 14, 14, 4], + "reshape2_2.tmp_0": [4, 1, 2744], + }, + { + 'data': [1, 28, 28], + "reshape2_1.tmp_0": [1, 14, 14, 4], + "reshape2_2.tmp_0": [1, 1, 2744], + }, + False, + ) + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' def test_check_output(self): - #self.quant_dequant() + # self.quant_dequant() if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1e0, - flatten=False, - rtol=1e0) + self.check_output_with_option( + use_gpu, atol=1e0, flatten=False, rtol=1e0 + ) self.assertTrue( PassVersionChecker.IsCompatible( - 'quant_conv2d_dequant_fuse_pass')) + 'quant_conv2d_dequant_fuse_pass' + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten2_matmul_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten2_matmul_fuse_pass.py index 698eae07ccf8e..b163d429e4778 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten2_matmul_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten2_matmul_fuse_pass.py @@ -42,7 +42,8 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['mul', 'elementwise_add'], (1e-4, 1e-1) def add_ignore_pass_case(self): @@ -65,9 +66,10 @@ def teller1(program_config, predictor_config): def sample_program_config(self, draw): # 1. Generate shape and attr of flatten2 x_shape = draw( - st.lists(st.integers(min_value=1, max_value=10), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=10), min_size=4, max_size=4 + ) + ) # [a, b, c, d] => [a, b*c*d] flatten_axis = 1 flatten_shape = [x_shape[0], x_shape[1] * x_shape[2] * x_shape[3]] @@ -79,9 +81,10 @@ def sample_program_config(self, draw): # 3. Generate legal shape of input:Y of matmul y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = flatten_shape[1] # 4. Generate legal attr:axis of elementwise_add @@ -98,17 +101,11 @@ def sample_program_config(self, draw): "X": ["flatten2_x"], }, axis=flatten_axis, - outputs={ - "Out": ["flatten2_out"], - "XShape": ["xshape"] - }, + outputs={"Out": ["flatten2_out"], "XShape": ["xshape"]}, ) matmul_op = OpConfig( "matmul", - inputs={ - "X": ["flatten2_out"], - "Y": ["matmul_y"] - }, + inputs={"X": ["flatten2_out"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, transpose_X=transpose_X, @@ -123,10 +120,7 @@ def sample_program_config(self, draw): add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["matmul_out"], - "Y": ["bias"] - }, + inputs={"X": ["matmul_out"], "Y": ["bias"]}, outputs={"Out": ["add_out"]}, axis=axis, ) @@ -148,9 +142,11 @@ def sample_program_config(self, draw): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=25, - passes=["trt_flatten2_matmul_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=25, + passes=["trt_flatten2_matmul_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten_op.py index 41c0c7621ac99..6d230134a3527 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_flatten_op.py @@ -22,12 +22,11 @@ class TRTFlattenTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) flatten_out = self.append_flatten(data) out = fluid.layers.batch_norm(flatten_out, is_test=True) self.feeds = { @@ -35,7 +34,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTFlattenTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def append_flatten(self, data): @@ -46,16 +46,16 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTFlattenDynamicTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) flatten_out = self.append_flatten(data) out = fluid.layers.batch_norm(flatten_out, is_test=True) self.feeds = { @@ -63,18 +63,14 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTFlattenDynamicTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.dynamic_shape_params = TRTFlattenDynamicTest.DynamicShapeParam( - { - 'data': [2, 6, 64, 64], - 'flatten_0.tmp_0': [2, 6 * 64 * 64] - }, { - 'data': [2, 6, 64, 64], - 'flatten_0.tmp_0': [2, 6 * 64 * 64] - }, { - 'data': [2, 6, 64, 64], - 'flatten_0.tmp_0': [2, 6 * 64 * 64] - }, False) + {'data': [2, 6, 64, 64], 'flatten_0.tmp_0': [2, 6 * 64 * 64]}, + {'data': [2, 6, 64, 64], 'flatten_0.tmp_0': [2, 6 * 64 * 64]}, + {'data': [2, 6, 64, 64], 'flatten_0.tmp_0': [2, 6 * 64 * 64]}, + False, + ) self.fetch_list = [out] def append_flatten(self, data): @@ -85,7 +81,8 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_nd_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_nd_op.py index 2f8a711287cd2..75dbf5477b034 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_nd_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_nd_op.py @@ -22,7 +22,6 @@ class TRTGatherNdTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[-1, 3, 4], dtype="float32") @@ -32,40 +31,37 @@ def setUp(self): self.feeds = { "data": np.random.random([2, 3, 4]).astype("float32"), - "index": np.array([[[0, 1], [1, 0]], [[1, 2], - [0, 1]]]).astype("int32"), + "index": np.array([[[0, 1], [1, 0]], [[1, 2], [0, 1]]]).astype( + "int32" + ), } self.enable_trt = True self.trt_parameters = TRTGatherNdTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] self.dynamic_shape_params = TRTGatherNdTest.DynamicShapeParam( - { - 'data': [1, 3, 4], - 'index': [1, 2, 2] - }, { - 'data': [3, 3, 4], - 'index': [3, 2, 2] - }, { - 'data': [3, 3, 4], - 'index': [3, 2, 2] - }, False) + {'data': [1, 3, 4], 'index': [1, 2, 2]}, + {'data': [3, 3, 4], 'index': [3, 2, 2]}, + {'data': [3, 3, 4], 'index': [3, 2, 2]}, + False, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTGatherNdFp16Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 1280, 192], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 1280, 192], dtype="float32" + ) index = fluid.data(name="index", shape=[-1, 1028, 2], dtype="int32") gather_nd = fluid.layers.gather_nd(data, index) out = fluid.layers.batch_norm(gather_nd, is_test=True) @@ -77,26 +73,23 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTGatherNdFp16Test.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False + ) self.fetch_list = [out] self.dynamic_shape_params = TRTGatherNdFp16Test.DynamicShapeParam( - { - 'data': [1, 1280, 192], - 'index': [1, 1028, 2] - }, { - 'data': [3, 1280, 192], - 'index': [3, 1028, 2] - }, { - 'data': [3, 1280, 192], - 'index': [3, 1028, 2] - }, False) + {'data': [1, 1280, 192], 'index': [1, 1028, 2]}, + {'data': [3, 1280, 192], 'index': [3, 1028, 2]}, + {'data': [3, 1280, 192], 'index': [3, 1028, 2]}, + False, + ) def test_check_output(self, atol=1e-3): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_op.py index c765d27f0bc1f..72c62ff4127a6 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_gather_op.py @@ -22,7 +22,6 @@ class TRTGatherTest1(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): @@ -33,23 +32,19 @@ def setUp(self): self.feeds = { "data": np.random.random([self.bs, 128]).astype("float32"), - "index": self.index + "index": self.index, } self.enable_trt = True self.trt_parameters = TRTGatherTest1.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.dynamic_shape_params = TRTGatherTest1.DynamicShapeParam( - { - 'data': [1, 1], - 'index': [1, 1] - }, { - 'data': [32, 128], - 'index': [3, 1] - }, { - 'data': [32, 128], - 'index': [3, 1] - }, False) + {'data': [1, 1], 'index': [1, 1]}, + {'data': [32, 128], 'index': [3, 1]}, + {'data': [32, 128], 'index': [3, 1]}, + False, + ) self.fetch_list = [out] def set_params(self): @@ -61,11 +56,11 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu, flatten=False) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTGatherTest2(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): @@ -76,23 +71,19 @@ def setUp(self): self.feeds = { "data": np.random.random([self.bs, 64]).astype("float32"), - "index": self.index + "index": self.index, } self.enable_trt = True self.trt_parameters = TRTGatherTest2.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.dynamic_shape_params = TRTGatherTest2.DynamicShapeParam( - { - 'data': [2, 4], - 'index': [1] - }, { - 'data': [256, 256], - 'index': [4] - }, { - 'data': [64, 32], - 'index': [2] - }, False) + {'data': [2, 4], 'index': [1]}, + {'data': [256, 256], 'index': [4]}, + {'data': [64, 32], 'index': [2]}, + False, + ) self.fetch_list = [out] def set_params(self): @@ -104,7 +95,8 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu, flatten=False) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_inspector.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_inspector.py index 2aba345ffc686..cab331aaa3c91 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_inspector.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_inspector.py @@ -24,16 +24,17 @@ class TensorRTInspectorTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[1, 16, 16], dtype="float32") - matmul_out = fluid.layers.matmul(x=data, - y=data, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y, - alpha=self.alpha) + matmul_out = fluid.layers.matmul( + x=data, + y=data, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + alpha=self.alpha, + ) out = fluid.layers.batch_norm(matmul_out, is_test=True) self.feeds = { @@ -41,7 +42,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = InferencePassTest.TensorRTParam( - 1 << 30, 1, 0, AnalysisConfig.Precision.Float32, False, False, True) + 1 << 30, 1, 0, AnalysisConfig.Precision.Float32, False, False, True + ) self.fetch_list = [out] def set_params(self): @@ -53,12 +55,16 @@ def test_check_output(self): if core.is_compiled_with_cuda(): build_engine = subprocess.run( [sys.executable, 'test_trt_inspector.py', '--build-engine'], - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + ) engine_info = build_engine.stderr.decode('ascii') trt_compile_version = paddle.inference.get_trt_compile_version() trt_runtime_version = paddle.inference.get_trt_runtime_version() valid_version = (8, 2, 0) - if trt_compile_version >= valid_version and trt_runtime_version >= valid_version: + if ( + trt_compile_version >= valid_version + and trt_runtime_version >= valid_version + ): self.assertTrue('====== engine info ======' in engine_info) self.assertTrue('====== engine info end ======' in engine_info) self.assertTrue('matmul' in engine_info) @@ -66,8 +72,9 @@ def test_check_output(self): self.assertTrue('batch_norm' in engine_info) else: self.assertTrue( - 'Inspector needs TensorRT version 8.2 and after.' in - engine_info) + 'Inspector needs TensorRT version 8.2 and after.' + in engine_info + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_instance_norm_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_instance_norm_op.py index 6939e1c1a0054..c69394d7adfa2 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_instance_norm_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_instance_norm_op.py @@ -25,7 +25,6 @@ class TRTInstanceNormTest(InferencePassTest): - def setUp(self): self.bs = 4 self.channel = 4 @@ -37,7 +36,8 @@ def setUp(self): def build(self): self.trt_parameters = InferencePassTest.TensorRTParam( - 1 << 30, self.bs, 2, self.precision, self.serialize, False) + 1 << 30, self.bs, 2, self.precision, self.serialize, False + ) with fluid.program_guard(self.main_program, self.startup_program): shape = [-1, self.channel, self.height, self.width] @@ -61,7 +61,8 @@ def check_output(self, remove_cache=False): atol = 2e-2 self.check_output_with_option(use_gpu, atol, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def run_test(self, remove_cache=False): self.build() @@ -69,12 +70,14 @@ def run_test(self, remove_cache=False): def run_all_tests(self): precision_opt = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_opt = [False, True] - for precision, serialize in itertools.product(precision_opt, - serialize_opt): + for precision, serialize in itertools.product( + precision_opt, serialize_opt + ): self.precision = precision self.serialize = serialize self.run_test() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul.py index 14b0e9fa1451d..9e24c0f3233b7 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul.py @@ -22,16 +22,17 @@ class TensorRTMatMulDims2Test(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[24, 24], dtype="float32") - matmul_out = fluid.layers.matmul(x=data, - y=data, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y, - alpha=self.alpha) + matmul_out = fluid.layers.matmul( + x=data, + y=data, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + alpha=self.alpha, + ) out = fluid.layers.batch_norm(matmul_out, is_test=True) self.feeds = { @@ -39,7 +40,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TensorRTMatMulDims2Test.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def set_params(self): @@ -52,22 +54,24 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTMatMulTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 24, 24], - dtype="float32") - matmul_out = fluid.layers.matmul(x=data, - y=data, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y, - alpha=self.alpha) + data = fluid.data( + name="data", shape=[-1, 6, 24, 24], dtype="float32" + ) + matmul_out = fluid.layers.matmul( + x=data, + y=data, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + alpha=self.alpha, + ) out = fluid.layers.batch_norm(matmul_out, is_test=True) self.feeds = { @@ -75,7 +79,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TensorRTMatMulTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def set_params(self): @@ -88,11 +93,11 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTMatMulTransposeXTest(TensorRTMatMulTest): - def set_params(self): self.transpose_x = True self.transpose_y = False @@ -100,7 +105,6 @@ def set_params(self): class TensorRTMatMulTransposeYTest(TensorRTMatMulTest): - def set_params(self): self.transpose_x = False self.transpose_y = True @@ -108,7 +112,6 @@ def set_params(self): class TensorRTMatMulScaleTest(TensorRTMatMulTest): - def set_params(self): self.transpose_x = False self.transpose_y = False @@ -116,29 +119,31 @@ def set_params(self): class TensorRTMatMulBroadcastTest(InferencePassTest): - def setUp(self): self.set_params() place = fluid.CPUPlace() with fluid.program_guard(self.main_program, self.startup_program): - data_x = fluid.data(name="data_x", - shape=[-1, 6, 24], - dtype="float32") + data_x = fluid.data( + name="data_x", shape=[-1, 6, 24], dtype="float32" + ) data_y = fluid.data(name="data_y", shape=[24, 16], dtype="float32") - matmul_out = fluid.layers.matmul(x=data_x, - y=data_y, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y, - alpha=self.alpha) + matmul_out = fluid.layers.matmul( + x=data_x, + y=data_y, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + alpha=self.alpha, + ) out = fluid.layers.batch_norm(matmul_out, is_test=True) self.feeds = { "data_x": np.ones([2, 6, 24]).astype("float32"), - "data_y": np.ones([24, 16]).astype("float32") + "data_y": np.ones([24, 16]).astype("float32"), } self.enable_trt = True self.trt_parameters = TensorRTMatMulBroadcastTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def set_params(self): @@ -151,7 +156,8 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul_quant_dequant.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul_quant_dequant.py index b075c65c82148..75db2ebb221f5 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul_quant_dequant.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_matmul_quant_dequant.py @@ -23,25 +23,28 @@ class TensorRTMatMulQuantDequantDims3Test(QuantDequantTest): - def setUp(self): self.set_params() def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') - matmul_out = fluid.layers.matmul(x=self.data, - y=self.data, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y, - alpha=self.alpha) - fc_out = fluid.layers.fc(input=matmul_out, - size=10, - num_flatten_dims=1, - bias_attr=False, - act=None) + matmul_out = fluid.layers.matmul( + x=self.data, + y=self.data, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + alpha=self.alpha, + ) + fc_out = fluid.layers.fc( + input=matmul_out, + size=10, + num_flatten_dims=1, + bias_attr=False, + act=None, + ) result = fluid.layers.relu(fc_out) loss = fluid.layers.cross_entropy(input=result, label=self.label) avg_loss = paddle.mean(loss) @@ -50,21 +53,23 @@ def network(): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random([1, 28, 28]).astype("float32")} self.fetch_list = [result] self.enable_trt = True self.trt_parameters = TensorRTMatMulQuantDequantDims3Test.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' @@ -74,20 +79,20 @@ def set_params(self): self.alpha = 1.0 def test_check_output(self): - #self.quant_dequant() + # self.quant_dequant() if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1, - flatten=False, - rtol=1e-1) + self.check_output_with_option( + use_gpu, atol=1, flatten=False, rtol=1e-1 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTMatMulQuantDequantDims3TransposeXTest( - TensorRTMatMulQuantDequantDims3Test): - + TensorRTMatMulQuantDequantDims3Test +): def set_params(self): self.transpose_x = True self.transpose_y = False @@ -95,8 +100,8 @@ def set_params(self): class TensorRTMatMulQuantDequantDims3TransposeYTest( - TensorRTMatMulQuantDequantDims3Test): - + TensorRTMatMulQuantDequantDims3Test +): def set_params(self): self.transpose_x = False self.transpose_y = True @@ -104,8 +109,8 @@ def set_params(self): class TensorRTMatMulQuantDequantDims3TransposeXYTest( - TensorRTMatMulQuantDequantDims3Test): - + TensorRTMatMulQuantDequantDims3Test +): def set_params(self): self.transpose_x = True self.transpose_y = True @@ -113,27 +118,30 @@ def set_params(self): class TensorRTMatMulQuantDequantDims4Test(QuantDequantTest): - def setUp(self): self.set_params() def network(): - self.data = fluid.data(name='data', - shape=[1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[1, 28, 28], dtype='float32' + ) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') reshape_out = fluid.layers.reshape(self.data, shape=[1, 4, 14, 14]) - matmul_out = fluid.layers.matmul(x=reshape_out, - y=reshape_out, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y, - alpha=self.alpha) + matmul_out = fluid.layers.matmul( + x=reshape_out, + y=reshape_out, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + alpha=self.alpha, + ) out = fluid.layers.batch_norm(matmul_out, is_test=True) - fc_out = fluid.layers.fc(input=matmul_out, - size=10, - num_flatten_dims=1, - bias_attr=False, - act=None) + fc_out = fluid.layers.fc( + input=matmul_out, + size=10, + num_flatten_dims=1, + bias_attr=False, + act=None, + ) result = fluid.layers.relu(fc_out) loss = fluid.layers.cross_entropy(input=result, label=self.label) avg_loss = paddle.mean(loss) @@ -142,21 +150,23 @@ def network(): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random([1, 28, 28]).astype("float32")} self.fetch_list = [result] self.enable_trt = True self.trt_parameters = TensorRTMatMulQuantDequantDims4Test.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' @@ -166,20 +176,20 @@ def set_params(self): self.alpha = 1.0 def test_check_output(self): - #self.quant_dequant() + # self.quant_dequant() if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1, - flatten=False, - rtol=1e-1) + self.check_output_with_option( + use_gpu, atol=1, flatten=False, rtol=1e-1 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTMatMulQuantDequantDims4TransposeXTest( - TensorRTMatMulQuantDequantDims4Test): - + TensorRTMatMulQuantDequantDims4Test +): def set_params(self): self.transpose_x = True self.transpose_y = False @@ -187,8 +197,8 @@ def set_params(self): class TensorRTMatMulQuantDequantDims4TransposeYTest( - TensorRTMatMulQuantDequantDims4Test): - + TensorRTMatMulQuantDequantDims4Test +): def set_params(self): self.transpose_x = False self.transpose_y = True @@ -196,8 +206,8 @@ def set_params(self): class TensorRTMatMulQuantDequantDims4TransposeXYTest( - TensorRTMatMulQuantDequantDims4Test): - + TensorRTMatMulQuantDequantDims4Test +): def set_params(self): self.transpose_x = True self.transpose_y = True @@ -205,26 +215,29 @@ def set_params(self): class TensorRTMatMulQuantDequantDims3DynamicTest(QuantDequantTest): - def setUp(self): self.set_params() def network(): - self.data = fluid.data(name='data', - shape=[-1, 28, 28], - dtype='float32') + self.data = fluid.data( + name='data', shape=[-1, 28, 28], dtype='float32' + ) self.label = fluid.data(name='label', shape=[1, 1], dtype='int64') - matmul_out = fluid.layers.matmul(x=self.data, - y=self.data, - transpose_x=self.transpose_x, - transpose_y=self.transpose_y, - alpha=self.alpha) + matmul_out = fluid.layers.matmul( + x=self.data, + y=self.data, + transpose_x=self.transpose_x, + transpose_y=self.transpose_y, + alpha=self.alpha, + ) out = fluid.layers.batch_norm(matmul_out, is_test=True) - fc_out = fluid.layers.fc(input=matmul_out, - size=10, - num_flatten_dims=1, - bias_attr=False, - act=None) + fc_out = fluid.layers.fc( + input=matmul_out, + size=10, + num_flatten_dims=1, + bias_attr=False, + act=None, + ) result = fluid.layers.relu(fc_out) loss = fluid.layers.cross_entropy(input=result, label=self.label) avg_loss = paddle.mean(loss) @@ -233,24 +246,33 @@ def network(): self.main_program.random_seed = 2 self.startup_program.random_seed = 2 self.test_main_program.random_seed = 2 - #self.test_startup_program.random_seed = 2 + # self.test_startup_program.random_seed = 2 with fluid.unique_name.guard(): with fluid.program_guard(self.main_program, self.startup_program): self.loss, result = network() opt = fluid.optimizer.Adam(learning_rate=0.0001) opt.minimize(self.loss) with fluid.unique_name.guard(): - with fluid.program_guard(self.test_main_program, - self.startup_program): + with fluid.program_guard( + self.test_main_program, self.startup_program + ): network() self.feeds = {"data": np.random.random([3, 28, 28]).astype("float32")} self.fetch_list = [result] self.enable_trt = True - self.trt_parameters = TensorRTMatMulQuantDequantDims3DynamicTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False) - self.dynamic_shape_params = TensorRTMatMulQuantDequantDims3DynamicTest.DynamicShapeParam( - {'data': [1, 28, 28]}, {'data': [4, 28, 28]}, {'data': [3, 28, 28]}, - False) + self.trt_parameters = ( + TensorRTMatMulQuantDequantDims3DynamicTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Int8, False, False + ) + ) + self.dynamic_shape_params = ( + TensorRTMatMulQuantDequantDims3DynamicTest.DynamicShapeParam( + {'data': [1, 28, 28]}, + {'data': [4, 28, 28]}, + {'data': [3, 28, 28]}, + False, + ) + ) self.activation_quantize_type = 'moving_average_abs_max' self.weight_quantize_type = 'channel_wise_abs_max' @@ -260,20 +282,20 @@ def set_params(self): self.alpha = 1.0 def test_check_output(self): - #self.quant_dequant() + # self.quant_dequant() if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - atol=1, - flatten=False, - rtol=1e-1) + self.check_output_with_option( + use_gpu, atol=1, flatten=False, rtol=1e-1 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTMatMulQuantDequantDims4TransposeXDynamicTest( - TensorRTMatMulQuantDequantDims3DynamicTest): - + TensorRTMatMulQuantDequantDims3DynamicTest +): def set_params(self): self.transpose_x = True self.transpose_y = False @@ -281,8 +303,8 @@ def set_params(self): class TensorRTMatMulQuantDequantDims4TransposeYDynamicTest( - TensorRTMatMulQuantDequantDims3DynamicTest): - + TensorRTMatMulQuantDequantDims3DynamicTest +): def set_params(self): self.transpose_x = False self.transpose_y = True @@ -290,8 +312,8 @@ def set_params(self): class TensorRTMatMulQuantDequantDims4TransposeXYDynamicTest( - TensorRTMatMulQuantDequantDims3DynamicTest): - + TensorRTMatMulQuantDequantDims3DynamicTest +): def set_params(self): self.transpose_x = True self.transpose_y = True diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms3_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms3_op.py index 91ca32ef8994e..05df3b65082b7 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms3_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms3_op.py @@ -24,19 +24,21 @@ from paddle.fluid.core import AnalysisConfig -def multiclass_nms(bboxes, - scores, - score_threshold, - nms_top_k, - keep_top_k, - nms_threshold=0.3, - normalized=True, - nms_eta=1., - background_label=-1, - return_index=False, - return_rois_num=True, - rois_num=None, - name=None): +def multiclass_nms( + bboxes, + scores, + score_threshold, + nms_top_k, + keep_top_k, + nms_threshold=0.3, + normalized=True, + nms_eta=1.0, + background_label=-1, + return_index=False, + return_rois_num=True, + rois_num=None, + name=None, +): """ This operator is to do multi-class non maximum suppression (NMS) on boxes and scores. @@ -127,12 +129,25 @@ class number return_index=True) """ if in_dygraph_mode(): - attrs = ('background_label', background_label, 'score_threshold', - score_threshold, 'nms_top_k', nms_top_k, 'nms_threshold', - nms_threshold, 'keep_top_k', keep_top_k, 'nms_eta', nms_eta, - 'normalized', normalized) + attrs = ( + 'background_label', + background_label, + 'score_threshold', + score_threshold, + 'nms_top_k', + nms_top_k, + 'nms_threshold', + nms_threshold, + 'keep_top_k', + keep_top_k, + 'nms_eta', + nms_eta, + 'normalized', + normalized, + ) output, index, nms_rois_num = core.ops.multiclass_nms3( - bboxes, scores, rois_num, *attrs) + bboxes, scores, rois_num, *attrs + ) if not return_index: index = None return output, nms_rois_num, index @@ -150,21 +165,24 @@ class number if return_rois_num: nms_rois_num = helper.create_variable_for_type_inference( - dtype='int32') + dtype='int32' + ) outputs['NmsRoisNum'] = nms_rois_num - helper.append_op(type="multiclass_nms3", - inputs=inputs, - attrs={ - 'background_label': background_label, - 'score_threshold': score_threshold, - 'nms_top_k': nms_top_k, - 'nms_threshold': nms_threshold, - 'keep_top_k': keep_top_k, - 'nms_eta': nms_eta, - 'normalized': normalized - }, - outputs=outputs) + helper.append_op( + type="multiclass_nms3", + inputs=inputs, + attrs={ + 'background_label': background_label, + 'score_threshold': score_threshold, + 'nms_top_k': nms_top_k, + 'nms_threshold': nms_threshold, + 'keep_top_k': keep_top_k, + 'nms_eta': nms_eta, + 'normalized': normalized, + }, + outputs=outputs, + ) output.stop_gradient = True index.stop_gradient = True if not return_index: @@ -176,7 +194,6 @@ class number class TensorRTMultiClassNMS3Test(InferencePassTest): - def setUp(self): self.enable_trt = True self.enable_tensorrt_varseqlen = True @@ -184,25 +201,28 @@ def setUp(self): self.serialize = False self.bs = 1 self.background_label = -1 - self.score_threshold = .5 + self.score_threshold = 0.5 self.nms_top_k = 8 - self.nms_threshold = .3 + self.nms_threshold = 0.3 self.keep_top_k = 8 self.normalized = False self.num_classes = 8 self.num_boxes = 8 self.nms_eta = 1.1 self.trt_parameters = InferencePassTest.TensorRTParam( - 1 << 30, self.bs, 2, self.precision, self.serialize, False) + 1 << 30, self.bs, 2, self.precision, self.serialize, False + ) def build(self): with fluid.program_guard(self.main_program, self.startup_program): - boxes = fluid.data(name='bboxes', - shape=[-1, self.num_boxes, 4], - dtype='float32') - scores = fluid.data(name='scores', - shape=[-1, self.num_classes, self.num_boxes], - dtype='float32') + boxes = fluid.data( + name='bboxes', shape=[-1, self.num_boxes, 4], dtype='float32' + ) + scores = fluid.data( + name='scores', + shape=[-1, self.num_classes, self.num_boxes], + dtype='float32', + ) multiclass_nms_out, _, _ = multiclass_nms( bboxes=boxes, scores=scores, @@ -212,17 +232,26 @@ def build(self): nms_threshold=self.nms_threshold, keep_top_k=self.keep_top_k, normalized=self.normalized, - nms_eta=self.nms_eta) - mutliclass_nms_out = multiclass_nms_out + 1. + nms_eta=self.nms_eta, + ) + mutliclass_nms_out = multiclass_nms_out + 1.0 multiclass_nms_out = fluid.layers.reshape( - multiclass_nms_out, [self.bs, 1, self.keep_top_k, 6], - name='reshape') + multiclass_nms_out, + [self.bs, 1, self.keep_top_k, 6], + name='reshape', + ) out = fluid.layers.batch_norm(multiclass_nms_out, is_test=True) - boxes_data = np.arange(self.num_boxes * 4).reshape( - [self.bs, self.num_boxes, 4]).astype('float32') - scores_data = np.arange(1 * self.num_classes * self.num_boxes).reshape( - [self.bs, self.num_classes, self.num_boxes]).astype('float32') + boxes_data = ( + np.arange(self.num_boxes * 4) + .reshape([self.bs, self.num_boxes, 4]) + .astype('float32') + ) + scores_data = ( + np.arange(1 * self.num_classes * self.num_boxes) + .reshape([self.bs, self.num_classes, self.num_boxes]) + .astype('float32') + ) self.feeds = { 'bboxes': boxes_data, 'scores': scores_data, @@ -235,7 +264,8 @@ def run_test(self): def run_test_all(self): precision_opt = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_opt = [False, True] max_shape = { @@ -246,13 +276,15 @@ def run_test_all(self): dynamic_shape_opt = [ None, InferencePassTest.DynamicShapeParam( - { - 'bboxes': [1, 1, 4], - 'scores': [1, 1, 1] - }, max_shape, opt_shape, False) + {'bboxes': [1, 1, 4], 'scores': [1, 1, 1]}, + max_shape, + opt_shape, + False, + ), ] for precision, serialize, dynamic_shape in itertools.product( - precision_opt, serialize_opt, dynamic_shape_opt): + precision_opt, serialize_opt, dynamic_shape_opt + ): self.precision = precision self.serialize = serialize self.dynamic_shape_params = dynamic_shape @@ -264,7 +296,8 @@ def check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def test_base(self): self.run_test() @@ -284,10 +317,11 @@ def test_dynamic(self): } opt_shape = max_shape self.dynamic_shape_params = InferencePassTest.DynamicShapeParam( - { - 'bboxes': [1, 1, 4], - 'scores': [1, 1, 1] - }, max_shape, opt_shape, False) + {'bboxes': [1, 1, 4], 'scores': [1, 1, 1]}, + max_shape, + opt_shape, + False, + ) self.run_test() def test_background(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms_op.py index 5e04241f14991..ead11ba7ae170 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_multiclass_nms_op.py @@ -23,7 +23,6 @@ class TensorRTMultiClassNMSTest(InferencePassTest): - def setUp(self): self.enable_trt = True self.enable_tensorrt_varseqlen = True @@ -31,24 +30,27 @@ def setUp(self): self.serialize = False self.bs = 1 self.background_label = -1 - self.score_threshold = .5 + self.score_threshold = 0.5 self.nms_top_k = 8 - self.nms_threshold = .3 + self.nms_threshold = 0.3 self.keep_top_k = 8 self.normalized = False self.num_classes = 8 self.num_boxes = 8 self.trt_parameters = InferencePassTest.TensorRTParam( - 1 << 30, self.bs, 2, self.precision, self.serialize, False) + 1 << 30, self.bs, 2, self.precision, self.serialize, False + ) def build(self): with fluid.program_guard(self.main_program, self.startup_program): - boxes = fluid.data(name='bboxes', - shape=[-1, self.num_boxes, 4], - dtype='float32') - scores = fluid.data(name='scores', - shape=[-1, self.num_classes, self.num_boxes], - dtype='float32') + boxes = fluid.data( + name='bboxes', shape=[-1, self.num_boxes, 4], dtype='float32' + ) + scores = fluid.data( + name='scores', + shape=[-1, self.num_classes, self.num_boxes], + dtype='float32', + ) multiclass_nms_out = fluid.layers.multiclass_nms( bboxes=boxes, scores=scores, @@ -57,17 +59,26 @@ def build(self): nms_top_k=self.nms_top_k, nms_threshold=self.nms_threshold, keep_top_k=self.keep_top_k, - normalized=self.normalized) - mutliclass_nms_out = multiclass_nms_out + 1. + normalized=self.normalized, + ) + mutliclass_nms_out = multiclass_nms_out + 1.0 multiclass_nms_out = fluid.layers.reshape( - multiclass_nms_out, [self.bs, 1, self.keep_top_k, 6], - name='reshape') + multiclass_nms_out, + [self.bs, 1, self.keep_top_k, 6], + name='reshape', + ) out = fluid.layers.batch_norm(multiclass_nms_out, is_test=True) - boxes_data = np.arange(self.num_boxes * 4).reshape( - [self.bs, self.num_boxes, 4]).astype('float32') - scores_data = np.arange(1 * self.num_classes * self.num_boxes).reshape( - [self.bs, self.num_classes, self.num_boxes]).astype('float32') + boxes_data = ( + np.arange(self.num_boxes * 4) + .reshape([self.bs, self.num_boxes, 4]) + .astype('float32') + ) + scores_data = ( + np.arange(1 * self.num_classes * self.num_boxes) + .reshape([self.bs, self.num_classes, self.num_boxes]) + .astype('float32') + ) self.feeds = { 'bboxes': boxes_data, 'scores': scores_data, @@ -80,7 +91,8 @@ def run_test(self): def run_test_all(self): precision_opt = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_opt = [False, True] max_shape = { @@ -91,13 +103,15 @@ def run_test_all(self): dynamic_shape_opt = [ None, InferencePassTest.DynamicShapeParam( - { - 'bboxes': [1, 1, 4], - 'scores': [1, 1, 1] - }, max_shape, opt_shape, False) + {'bboxes': [1, 1, 4], 'scores': [1, 1, 1]}, + max_shape, + opt_shape, + False, + ), ] for precision, serialize, dynamic_shape in itertools.product( - precision_opt, serialize_opt, dynamic_shape_opt): + precision_opt, serialize_opt, dynamic_shape_opt + ): self.precision = precision self.serialize = serialize self.dynamic_shape_params = dynamic_shape @@ -109,7 +123,8 @@ def check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def test_base(self): self.run_test() @@ -129,10 +144,11 @@ def test_dynamic(self): } opt_shape = max_shape self.dynamic_shape_params = InferencePassTest.DynamicShapeParam( - { - 'bboxes': [1, 1, 4], - 'scores': [1, 1, 1] - }, max_shape, opt_shape, False) + {'bboxes': [1, 1, 4], 'scores': [1, 1, 1]}, + max_shape, + opt_shape, + False, + ) self.run_test() def test_background(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_op.py index e5f5908fd0373..3582d9848d583 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_op.py @@ -22,20 +22,23 @@ class TRTNearestInterpTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): if self.data_layout == 'NCHW': shape = [ - -1, self.channels, self.origin_shape[0], - self.origin_shape[1] + -1, + self.channels, + self.origin_shape[0], + self.origin_shape[1], ] else: shape = [ - -1, self.origin_shape[0], self.origin_shape[1], - self.channels + -1, + self.origin_shape[0], + self.origin_shape[1], + self.channels, ] data = fluid.data(name='data', shape=shape, dtype='float32') resize_out = self.append_nearest_interp(data) @@ -43,13 +46,17 @@ def setUp(self): if self.data_layout == 'NCHW': shape = [ - self.bs, self.channels, self.origin_shape[0], - self.origin_shape[1] + self.bs, + self.channels, + self.origin_shape[0], + self.origin_shape[1], ] else: shape = [ - self.bs, self.origin_shape[0], self.origin_shape[1], - self.channels + self.bs, + self.origin_shape[0], + self.origin_shape[1], + self.channels, ] self.feeds = { @@ -57,7 +64,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTNearestInterpTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def set_params(self): @@ -71,26 +79,30 @@ def set_params(self): self.data_layout = 'NCHW' def append_nearest_interp(self, data): - if self.scale > 0.: - return fluid.layers.resize_nearest(data, - scale=self.scale, - align_corners=self.align_corners, - data_format=self.data_layout) - return fluid.layers.resize_nearest(data, - out_shape=self.resize_shape, - align_corners=self.align_corners, - data_format=self.data_layout) + if self.scale > 0.0: + return fluid.layers.resize_nearest( + data, + scale=self.scale, + align_corners=self.align_corners, + data_format=self.data_layout, + ) + return fluid.layers.resize_nearest( + data, + out_shape=self.resize_shape, + align_corners=self.align_corners, + data_format=self.data_layout, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTNearestInterpTest1(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 @@ -102,10 +114,9 @@ def set_params(self): class TRTNearestInterpTest2(TRTNearestInterpTest): - def set_params(self): self.bs = 4 - self.scale = 2. + self.scale = 2.0 self.channels = 3 self.origin_shape = (16, 16) # HW self.resize_shape = (32, 32) # HW @@ -114,7 +125,6 @@ def set_params(self): class TRTNearestInterpTest3(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = 0 @@ -126,7 +136,6 @@ def set_params(self): class TRTNearestInterpTest4(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 @@ -138,7 +147,6 @@ def set_params(self): class TRTNearestInterpTest5(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 @@ -150,10 +158,9 @@ def set_params(self): class TRTNearestInterpTest6(TRTNearestInterpTest): - def set_params(self): self.bs = 4 - self.scale = 2. + self.scale = 2.0 self.channels = 3 self.origin_shape = (16, 16) # HW self.resize_shape = (32, 32) # HW @@ -162,7 +169,6 @@ def set_params(self): class TRTNearestInterpTest7(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 @@ -174,7 +180,6 @@ def set_params(self): class TRTNearestInterpTest8(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 @@ -186,7 +191,6 @@ def set_params(self): class TRTNearestInterpTest9(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_v2_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_v2_op.py index 8e6a28c483db3..fc145fc410ad3 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_nearest_interp_v2_op.py @@ -23,20 +23,23 @@ class TRTNearestInterpTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): if self.data_layout == 'NCHW': shape = [ - -1, self.channels, self.origin_shape[0], - self.origin_shape[1] + -1, + self.channels, + self.origin_shape[0], + self.origin_shape[1], ] else: shape = [ - -1, self.origin_shape[0], self.origin_shape[1], - self.channels + -1, + self.origin_shape[0], + self.origin_shape[1], + self.channels, ] data = fluid.data(name='data', shape=shape, dtype='float32') resize_out = self.append_nearest_interp(data) @@ -44,13 +47,17 @@ def setUp(self): if self.data_layout == 'NCHW': shape = [ - self.bs, self.channels, self.origin_shape[0], - self.origin_shape[1] + self.bs, + self.channels, + self.origin_shape[0], + self.origin_shape[1], ] else: shape = [ - self.bs, self.origin_shape[0], self.origin_shape[1], - self.channels + self.bs, + self.origin_shape[0], + self.origin_shape[1], + self.channels, ] self.feeds = { @@ -58,7 +65,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTNearestInterpTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def set_params(self): @@ -71,31 +79,35 @@ def set_params(self): self.data_layout = 'NCHW' def append_nearest_interp(self, data): - if self.scale > 0.: - return F.interpolate(data, - scale_factor=self.scale, - align_corners=self.align_corners, - mode='nearest', - data_format=self.data_layout) - return F.interpolate(data, - size=self.resize_shape, - align_corners=self.align_corners, - mode='nearest', - data_format=self.data_layout) + if self.scale > 0.0: + return F.interpolate( + data, + scale_factor=self.scale, + align_corners=self.align_corners, + mode='nearest', + data_format=self.data_layout, + ) + return F.interpolate( + data, + size=self.resize_shape, + align_corners=self.align_corners, + mode='nearest', + data_format=self.data_layout, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTNearestInterpTest1(TRTNearestInterpTest): - def set_params(self): self.bs = 4 - self.scale = 2. + self.scale = 2.0 self.channels = 3 self.origin_shape = (16, 16) # HW self.resize_shape = (32, 32) # HW @@ -104,7 +116,6 @@ def set_params(self): class TRTNearestInterpTest2(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 @@ -116,7 +127,6 @@ def set_params(self): class TRTNearestInterpTest3(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 @@ -128,10 +138,9 @@ def set_params(self): class TRTNearestInterpTest4(TRTNearestInterpTest): - def set_params(self): self.bs = 4 - self.scale = 2. + self.scale = 2.0 self.channels = 3 self.origin_shape = (16, 16) # HW self.resize_shape = (32, 32) # HW @@ -140,7 +149,6 @@ def set_params(self): class TRTNearestInterpTest5(TRTNearestInterpTest): - def set_params(self): self.bs = 4 self.scale = -1 diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pad_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pad_op.py index ae6dbefa86da0..338d7d8a48107 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pad_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pad_op.py @@ -21,15 +21,14 @@ class PadOpTRTTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[1, 3, 128, 128], - dtype="float32") - pad_out = fluid.layers.pad(x=data, - paddings=[0, 0, 0, 0, 0, 1, 1, 2], - pad_value=0.0) + data = fluid.data( + name="data", shape=[1, 3, 128, 128], dtype="float32" + ) + pad_out = fluid.layers.pad( + x=data, paddings=[0, 0, 0, 0, 0, 1, 1, 2], pad_value=0.0 + ) out = fluid.layers.batch_norm(pad_out, is_test=True) self.feeds = { @@ -37,7 +36,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = PadOpTRTTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool3d_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool3d_op.py index c4488a57f9605..7e7499a19c718 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool3d_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool3d_op.py @@ -26,7 +26,6 @@ class TensorRTPool3dTest(InferencePassTest): - def setUp(self): self.bs = 1 self.channel = 3 @@ -44,10 +43,9 @@ def setUp(self): self.serialize = False self.precision = AnalysisConfig.Precision.Float32 self.feeds = { - 'data': - np.random.random( - [self.bs, self.channel, self.depth, self.height, - self.width]).astype('float32'), + 'data': np.random.random( + [self.bs, self.channel, self.depth, self.height, self.width] + ).astype('float32'), } def set_extra_config(self): @@ -56,22 +54,26 @@ def set_extra_config(self): def build_network(self): self.set_extra_config() self.trt_parameters = TensorRTPool3dTest.TensorRTParam( - 1 << 30, self.bs, 0, self.precision, self.serialize, False) + 1 << 30, self.bs, 0, self.precision, self.serialize, False + ) with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data( name='data', shape=[-1, self.channel, self.depth, self.height, self.width], - dtype='float32') - pool_out = fluid.layers.pool3d(input=data, - pool_size=self.pool_size, - pool_type=self.pool_type, - pool_stride=self.pool_stride, - pool_padding=self.pool_padding, - global_pooling=self.global_pooling, - ceil_mode=self.ceil_mode, - exclusive=self.exclusive) - #out = fluid.layers.batch_norm(pool_out, is_test=True) + dtype='float32', + ) + pool_out = fluid.layers.pool3d( + input=data, + pool_size=self.pool_size, + pool_type=self.pool_type, + pool_stride=self.pool_stride, + pool_padding=self.pool_padding, + global_pooling=self.global_pooling, + ceil_mode=self.ceil_mode, + exclusive=self.exclusive, + ) + # out = fluid.layers.batch_norm(pool_out, is_test=True) self.fetch_list = [pool_out] def check_output(self): @@ -84,11 +86,13 @@ def check_output(self): elif self.precision == AnalysisConfig.Precision.Half: atol, rtol = (1e-3, 1e-3) else: - raise ValueError("Unsupported precision {}".format( - self.precision)) + raise ValueError( + "Unsupported precision {}".format(self.precision) + ) self.check_output_with_option(use_gpu, atol=atol, rtol=rtol) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def run_test(self): self.build_network() @@ -96,30 +100,51 @@ def run_test(self): def test(self): precision_options = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_options = [False, True] dynamic_shape_profile = InferencePassTest.DynamicShapeParam( { 'data': [ - self.bs, self.channel, self.depth // 2, self.height // 2, - self.width // 2 + self.bs, + self.channel, + self.depth // 2, + self.height // 2, + self.width // 2, ] - }, { - 'data': - [self.bs, self.channel, self.depth, self.height, self.width] - }, { - 'data': - [self.bs, self.channel, self.depth, self.height, self.width] - }, False) + }, + { + 'data': [ + self.bs, + self.channel, + self.depth, + self.height, + self.width, + ] + }, + { + 'data': [ + self.bs, + self.channel, + self.depth, + self.height, + self.width, + ] + }, + False, + ) dynamic_shape_options = [None, dynamic_shape_profile] for precision, serialize, dynamic_shape in itertools.product( - precision_options, serialize_options, dynamic_shape_options): + precision_options, serialize_options, dynamic_shape_options + ): is_dynamic = True if dynamic_shape_options is not None else False with self.subTest( - 'Precision: {}, Serialize: {}, Dynamic: {}'.format( - precision, serialize, is_dynamic)): + 'Precision: {}, Serialize: {}, Dynamic: {}'.format( + precision, serialize, is_dynamic + ) + ): self.precision = precision self.serialize = serialize self.dynamic_shape_params = dynamic_shape @@ -127,7 +152,6 @@ def test(self): class TensorRTAvgPool3dTest(TensorRTPool3dTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'avg' @@ -139,7 +163,6 @@ def set_extra_config(self): class TensorRTGlobalPool3dTest(TensorRTPool3dTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -151,7 +174,6 @@ def set_extra_config(self): class TensorRTCeilPool3dTest(TensorRTPool3dTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -163,7 +185,6 @@ def set_extra_config(self): class TensorRTExclusivePool3dTest(TensorRTPool3dTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -175,7 +196,6 @@ def set_extra_config(self): class TensorRTSamePaddingPool3dTest(InferencePassTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -187,7 +207,6 @@ def set_extra_config(self): class TensorRTValidPaddingPool3dTest(InferencePassTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -199,7 +218,6 @@ def set_extra_config(self): class TensorRTAdaptiveAvgPool3DTest(InferencePassTest): - def setUp(self): self.bs = 1 self.channel = 3 @@ -210,24 +228,26 @@ def setUp(self): self.serialize = False self.precision = AnalysisConfig.Precision.Float32 self.feeds = { - 'data': - np.random.random( - [self.bs, self.channel, self.depth, self.height, - self.width]).astype('float32'), + 'data': np.random.random( + [self.bs, self.channel, self.depth, self.height, self.width] + ).astype('float32'), } def build_network(self): self.trt_parameters = TensorRTPool3dTest.TensorRTParam( - 1 << 30, self.bs, 0, self.precision, self.serialize, False) + 1 << 30, self.bs, 0, self.precision, self.serialize, False + ) with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data( name='data', shape=[-1, self.channel, self.depth, self.height, self.width], - dtype='float32') + dtype='float32', + ) pool_out = paddle.nn.functional.adaptive_avg_pool3d( - x=data, output_size=[3, 3, 3]) - #out = fluid.layers.batch_norm(pool_out, is_test=True) + x=data, output_size=[3, 3, 3] + ) + # out = fluid.layers.batch_norm(pool_out, is_test=True) self.fetch_list = [pool_out] def check_output(self): @@ -237,7 +257,8 @@ def check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def run_test(self): self.build_network() @@ -245,30 +266,51 @@ def run_test(self): def test(self): precision_options = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_options = [False, True] dynamic_shape_profile = InferencePassTest.DynamicShapeParam( { 'data': [ - self.bs, self.channel, self.depth // 2, self.height // 2, - self.width // 2 + self.bs, + self.channel, + self.depth // 2, + self.height // 2, + self.width // 2, ] - }, { - 'data': - [self.bs, self.channel, self.depth, self.height, self.width] - }, { - 'data': - [self.bs, self.channel, self.depth, self.height, self.width] - }, False) + }, + { + 'data': [ + self.bs, + self.channel, + self.depth, + self.height, + self.width, + ] + }, + { + 'data': [ + self.bs, + self.channel, + self.depth, + self.height, + self.width, + ] + }, + False, + ) dynamic_shape_options = [None, dynamic_shape_profile] for precision, serialize, dynamic_shape in itertools.product( - precision_options, serialize_options, dynamic_shape_options): + precision_options, serialize_options, dynamic_shape_options + ): is_dynamic = True if dynamic_shape_options is not None else False with self.subTest( - 'Precision: {}, Serialize: {}, Dynamic: {}'.format( - precision, serialize, is_dynamic)): + 'Precision: {}, Serialize: {}, Dynamic: {}'.format( + precision, serialize, is_dynamic + ) + ): self.precision = precision self.serialize = serialize self.dynamic_shape_params = dynamic_shape @@ -276,7 +318,6 @@ def test(self): class TensorRTAdaptiveMaxPool3DTest(InferencePassTest): - def setUp(self): self.bs = 1 self.channel = 3 @@ -287,24 +328,26 @@ def setUp(self): self.serialize = False self.precision = AnalysisConfig.Precision.Float32 self.feeds = { - 'data': - np.random.random( - [self.bs, self.channel, self.depth, self.height, - self.width]).astype('float32'), + 'data': np.random.random( + [self.bs, self.channel, self.depth, self.height, self.width] + ).astype('float32'), } def build_network(self): self.trt_parameters = TensorRTPool3dTest.TensorRTParam( - 1 << 30, self.bs, 0, self.precision, self.serialize, False) + 1 << 30, self.bs, 0, self.precision, self.serialize, False + ) with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data( name='data', shape=[-1, self.channel, self.depth, self.height, self.width], - dtype='float32') + dtype='float32', + ) pool_out = paddle.nn.functional.adaptive_max_pool3d( - x=data, output_size=[3, 3, 3]) - #out = fluid.layers.batch_norm(pool_out, is_test=True) + x=data, output_size=[3, 3, 3] + ) + # out = fluid.layers.batch_norm(pool_out, is_test=True) self.fetch_list = [pool_out] def check_output(self): @@ -314,7 +357,8 @@ def check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def run_test(self): self.build_network() @@ -322,30 +366,51 @@ def run_test(self): def test(self): precision_options = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_options = [False, True] dynamic_shape_profile = InferencePassTest.DynamicShapeParam( { 'data': [ - self.bs, self.channel, self.depth // 2, self.height // 2, - self.width // 2 + self.bs, + self.channel, + self.depth // 2, + self.height // 2, + self.width // 2, ] - }, { - 'data': - [self.bs, self.channel, self.depth, self.height, self.width] - }, { - 'data': - [self.bs, self.channel, self.depth, self.height, self.width] - }, False) + }, + { + 'data': [ + self.bs, + self.channel, + self.depth, + self.height, + self.width, + ] + }, + { + 'data': [ + self.bs, + self.channel, + self.depth, + self.height, + self.width, + ] + }, + False, + ) dynamic_shape_options = [None, dynamic_shape_profile] for precision, serialize, dynamic_shape in itertools.product( - precision_options, serialize_options, dynamic_shape_options): + precision_options, serialize_options, dynamic_shape_options + ): is_dynamic = True if dynamic_shape_options is not None else False with self.subTest( - 'Precision: {}, Serialize: {}, Dynamic: {}'.format( - precision, serialize, is_dynamic)): + 'Precision: {}, Serialize: {}, Dynamic: {}'.format( + precision, serialize, is_dynamic + ) + ): self.precision = precision self.serialize = serialize self.dynamic_shape_params = dynamic_shape diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool_op.py index f6eaa2fb8c75e..2ceb891024ee0 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_pool_op.py @@ -25,7 +25,6 @@ class TensorRTPoolTest(InferencePassTest): - def setUp(self): self.bs = 1 self.channel = 2 @@ -42,9 +41,9 @@ def setUp(self): self.serialize = False self.precision = AnalysisConfig.Precision.Float32 self.feeds = { - 'data': - np.random.random([self.bs, self.channel, self.height, - self.width]).astype('float32'), + 'data': np.random.random( + [self.bs, self.channel, self.height, self.width] + ).astype('float32'), } def set_extra_config(self): @@ -53,20 +52,25 @@ def set_extra_config(self): def build_network(self): self.set_extra_config() self.trt_parameters = TensorRTPoolTest.TensorRTParam( - 1 << 30, self.bs, 0, self.precision, self.serialize, False) + 1 << 30, self.bs, 0, self.precision, self.serialize, False + ) with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name='data', - shape=[-1, self.channel, self.height, self.width], - dtype='float32') - pool_out = fluid.layers.pool2d(input=data, - pool_size=self.pool_size, - pool_type=self.pool_type, - pool_stride=self.pool_stride, - pool_padding=self.pool_padding, - global_pooling=self.global_pooling, - ceil_mode=self.ceil_mode, - exclusive=self.exclusive) + data = fluid.data( + name='data', + shape=[-1, self.channel, self.height, self.width], + dtype='float32', + ) + pool_out = fluid.layers.pool2d( + input=data, + pool_size=self.pool_size, + pool_type=self.pool_type, + pool_stride=self.pool_stride, + pool_padding=self.pool_padding, + global_pooling=self.global_pooling, + ceil_mode=self.ceil_mode, + exclusive=self.exclusive, + ) out = fluid.layers.batch_norm(pool_out, is_test=True) self.fetch_list = [out] @@ -80,11 +84,13 @@ def check_output(self): elif self.precision == AnalysisConfig.Precision.Half: atol, rtol = (1e-3, 1e-3) else: - raise ValueError("Unsupported precision {}".format( - self.precision)) + raise ValueError( + "Unsupported precision {}".format(self.precision) + ) self.check_output_with_option(use_gpu, atol=atol, rtol=rtol) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def run_test(self): self.build_network() @@ -92,23 +98,34 @@ def run_test(self): def test(self): precision_options = [ - AnalysisConfig.Precision.Float32, AnalysisConfig.Precision.Half + AnalysisConfig.Precision.Float32, + AnalysisConfig.Precision.Half, ] serialize_options = [False, True] dynamic_shape_profile = InferencePassTest.DynamicShapeParam( { - 'data': - [self.bs, self.channel, self.height // 2, self.width // 2] - }, {'data': [self.bs, self.channel, self.height, self.width]}, - {'data': [self.bs, self.channel, self.height, self.width]}, False) + 'data': [ + self.bs, + self.channel, + self.height // 2, + self.width // 2, + ] + }, + {'data': [self.bs, self.channel, self.height, self.width]}, + {'data': [self.bs, self.channel, self.height, self.width]}, + False, + ) dynamic_shape_options = [None, dynamic_shape_profile] for precision, serialize, dynamic_shape in itertools.product( - precision_options, serialize_options, dynamic_shape_options): + precision_options, serialize_options, dynamic_shape_options + ): is_dynamic = True if dynamic_shape_options is not None else False with self.subTest( - 'Precision: {}, Serialize: {}, Dynamic: {}'.format( - precision, serialize, is_dynamic)): + 'Precision: {}, Serialize: {}, Dynamic: {}'.format( + precision, serialize, is_dynamic + ) + ): self.precision = precision self.serialize = serialize self.dynamic_shape = dynamic_shape @@ -116,7 +133,6 @@ def test(self): class TensorRTAvgPoolTest(TensorRTPoolTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'avg' @@ -128,7 +144,6 @@ def set_extra_config(self): class TensorRTAvgCeilPoolTest(TensorRTPoolTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'avg' @@ -140,7 +155,6 @@ def set_extra_config(self): class TensorRTGlobalPoolTest(TensorRTPoolTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -152,7 +166,6 @@ def set_extra_config(self): class TensorRTCeilPoolTest(TensorRTPoolTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -164,7 +177,6 @@ def set_extra_config(self): class TensorRTExclusivePoolTest(TensorRTPoolTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -176,7 +188,6 @@ def set_extra_config(self): class TensorRTSamePaddingPoolTest(InferencePassTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' @@ -188,7 +199,6 @@ def set_extra_config(self): class TensorRTValidPaddingPoolTest(InferencePassTest): - def set_extra_config(self): self.pool_size = 2 self.pool_type = 'max' diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_mean_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_mean_op.py index f699c5572faa0..05621516d0b5a 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_mean_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_mean_op.py @@ -22,15 +22,14 @@ class TRTReduceMeanTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, -1, -1], - dtype="float32") - reduce_mean = fluid.layers.reduce_mean(data, - dim=[2, -1], - keep_dim=True) + data = fluid.data( + name="data", shape=[-1, 3, -1, -1], dtype="float32" + ) + reduce_mean = fluid.layers.reduce_mean( + data, dim=[2, -1], keep_dim=True + ) out = fluid.layers.batch_norm(reduce_mean, is_test=True) self.feeds = { @@ -38,27 +37,31 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTReduceMeanTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] self.dynamic_shape_params = TRTReduceMeanTest.DynamicShapeParam( - {'data': [1, 3, 16, 16]}, {'data': [3, 3, 56, 56]}, - {'data': [3, 3, 56, 56]}, False) + {'data': [1, 3, 16, 16]}, + {'data': [3, 3, 56, 56]}, + {'data': [3, 3, 56, 56]}, + False, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceMeanAllNoBatchTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, -1, -1], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, -1, -1], dtype="float32" + ) reduce_mean = fluid.layers.reduce_mean(data, keep_dim=True) out = fluid.layers.batch_norm(reduce_mean, is_test=True) @@ -67,30 +70,36 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTReduceMeanAllNoBatchTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] - self.dynamic_shape_params = TRTReduceMeanAllNoBatchTest.DynamicShapeParam( - {'data': [1, 3, 16, 16]}, {'data': [3, 3, 56, 56]}, - {'data': [3, 3, 56, 56]}, False) + self.dynamic_shape_params = ( + TRTReduceMeanAllNoBatchTest.DynamicShapeParam( + {'data': [1, 3, 16, 16]}, + {'data': [3, 3, 56, 56]}, + {'data': [3, 3, 56, 56]}, + False, + ) + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceMeanTestFP16(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, -1, -1], - dtype="float32") - reduce_mean = fluid.layers.reduce_mean(data, - dim=[2, -1], - keep_dim=True) + data = fluid.data( + name="data", shape=[-1, 3, -1, -1], dtype="float32" + ) + reduce_mean = fluid.layers.reduce_mean( + data, dim=[2, -1], keep_dim=True + ) out = fluid.layers.batch_norm(reduce_mean, is_test=True) self.feeds = { @@ -98,27 +107,31 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTReduceMeanTestFP16.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False + ) self.fetch_list = [out] self.dynamic_shape_params = TRTReduceMeanTestFP16.DynamicShapeParam( - {'data': [1, 3, 16, 16]}, {'data': [3, 3, 56, 56]}, - {'data': [3, 3, 56, 56]}, False) + {'data': [1, 3, 16, 16]}, + {'data': [3, 3, 56, 56]}, + {'data': [3, 3, 56, 56]}, + False, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceMeanAllTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 56, 56], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 56, 56], dtype="float32" + ) reduce_mean = fluid.layers.reduce_mean(data, keep_dim=True) out = fluid.layers.batch_norm(reduce_mean, is_test=True) @@ -127,30 +140,34 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTReduceMeanAllTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] self.dynamic_shape_params = TRTReduceMeanAllTest.DynamicShapeParam( - {'data': [1, 3, 56, 56]}, {'data': [3, 3, 56, 56]}, - {'data': [3, 3, 56, 56]}, False) + {'data': [1, 3, 56, 56]}, + {'data': [3, 3, 56, 56]}, + {'data': [3, 3, 56, 56]}, + False, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceMeanTestStatic(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[3, 3, 56, 56], - dtype="float32") - reduce_mean = fluid.layers.reduce_mean(data, - dim=[2, -1], - keep_dim=True) + data = fluid.data( + name="data", shape=[3, 3, 56, 56], dtype="float32" + ) + reduce_mean = fluid.layers.reduce_mean( + data, dim=[2, -1], keep_dim=True + ) out = fluid.layers.batch_norm(reduce_mean, is_test=True) self.feeds = { @@ -158,7 +175,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTReduceMeanTestStatic.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -166,16 +184,16 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceMeanStaticAllTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[4, 3, 56, 56], - dtype="float32") + data = fluid.data( + name="data", shape=[4, 3, 56, 56], dtype="float32" + ) reduce_mean = fluid.layers.reduce_mean(data, keep_dim=True) out = fluid.layers.batch_norm(reduce_mean, is_test=True) @@ -184,7 +202,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTReduceMeanStaticAllTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -192,16 +211,16 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceMeanStaticFP16(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[4, 3, 56, 56], - dtype="float32") + data = fluid.data( + name="data", shape=[4, 3, 56, 56], dtype="float32" + ) reduce_mean = fluid.layers.reduce_mean(data, keep_dim=True) out = fluid.layers.batch_norm(reduce_mean, is_test=True) @@ -210,27 +229,27 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTReduceMeanStaticFP16.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False + ) self.fetch_list = [out] def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - flatten=True, - atol=1e-3, - rtol=1e-3) + self.check_output_with_option( + use_gpu, flatten=True, atol=1e-3, rtol=1e-3 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceMeanFP16Static(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[4, 3, 56, 56], - dtype="float32") + data = fluid.data( + name="data", shape=[4, 3, 56, 56], dtype="float32" + ) reduce_mean = fluid.layers.reduce_mean(data, keep_dim=True) out = fluid.layers.batch_norm(reduce_mean, is_test=True) @@ -239,18 +258,19 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTReduceMeanFP16Static.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, True, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, True, False + ) self.fetch_list = [out] def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, - flatten=True, - atol=1e-3, - rtol=1e-3) + self.check_output_with_option( + use_gpu, flatten=True, atol=1e-3, rtol=1e-3 + ) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_sum_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_sum_op.py index 13663676c6db1..09e9a652ca866 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_sum_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reduce_sum_op.py @@ -22,15 +22,14 @@ class TRTReduceSumTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 10, 192], - dtype="float32") - reduce_sum = fluid.layers.reduce_sum(data, - dim=[2, -1], - keep_dim=True) + data = fluid.data( + name="data", shape=[-1, 3, 10, 192], dtype="float32" + ) + reduce_sum = fluid.layers.reduce_sum( + data, dim=[2, -1], keep_dim=True + ) out = fluid.layers.batch_norm(reduce_sum, is_test=True) self.feeds = { @@ -38,27 +37,31 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTReduceSumTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] self.dynamic_shape_params = TRTReduceSumTest.DynamicShapeParam( - {'data': [1, 3, 8, 8]}, {'data': [3, 3, 10, 192]}, - {'data': [3, 3, 10, 192]}, False) + {'data': [1, 3, 8, 8]}, + {'data': [3, 3, 10, 192]}, + {'data': [3, 3, 10, 192]}, + False, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReduceSumAllTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 10, 192], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 10, 192], dtype="float32" + ) reduce_sum = fluid.layers.reduce_sum(data, keep_dim=True) out = fluid.layers.batch_norm(reduce_sum, is_test=True) @@ -67,18 +70,23 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTReduceSumAllTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] self.dynamic_shape_params = TRTReduceSumAllTest.DynamicShapeParam( - {'data': [1, 3, 8, 8]}, {'data': [3, 3, 10, 192]}, - {'data': [3, 3, 10, 192]}, False) + {'data': [1, 3, 8, 8]}, + {'data': [3, 3, 10, 192]}, + {'data': [3, 3, 10, 192]}, + False, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reshape_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reshape_op.py index ade320796ebef..075919b7bf6e3 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reshape_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_reshape_op.py @@ -22,19 +22,20 @@ class TRTReshapeTest(InferencePassTest): - def setUp(self): self.bs = 1 self.input_shape = [16, 3, 8] self.reshape = [-1, 4, 4, 24] self.data_shape = [ - self.bs, self.input_shape[0], self.input_shape[1], - self.input_shape[2] + self.bs, + self.input_shape[0], + self.input_shape[1], + self.input_shape[2], ] with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name='data', - shape=self.data_shape, - dtype='float32') + data = fluid.data( + name='data', shape=self.data_shape, dtype='float32' + ) reshape_out = self.append_reshape(data, self.reshape) out = fluid.layers.batch_norm(reshape_out, is_test=True) self.feeds = { @@ -42,7 +43,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTReshapeTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def append_reshape(self, data, reshape): @@ -53,23 +55,25 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTReshapeTest1(TRTReshapeTest): - def setUp(self): self.bs = 2 self.input_shape = [23, 13, 12] self.reshape = [2, 0, -1, 6] self.data_shape = [ - self.bs, self.input_shape[0], self.input_shape[1], - self.input_shape[2] + self.bs, + self.input_shape[0], + self.input_shape[1], + self.input_shape[2], ] with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name='data', - shape=self.data_shape, - dtype='float32') + data = fluid.data( + name='data', shape=self.data_shape, dtype='float32' + ) reshape_out = self.append_reshape(data, self.reshape) out = fluid.layers.batch_norm(reshape_out, is_test=True) self.feeds = { @@ -77,24 +81,26 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTReshapeTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] class TRTReshapeTest2(TRTReshapeTest): - def setUp(self): self.bs = 2 self.input_shape = [23, 13, 12] self.reshape = [2, 0, -1, 6] self.data_shape = [ - self.bs, self.input_shape[0], self.input_shape[1], - self.input_shape[2] + self.bs, + self.input_shape[0], + self.input_shape[1], + self.input_shape[2], ] with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name='data', - shape=self.data_shape, - dtype='float32') + data = fluid.data( + name='data', shape=self.data_shape, dtype='float32' + ) reshape_out = fluid.layers.reshape(x=data, shape=self.reshape) out = fluid.layers.batch_norm(reshape_out, is_test=True) self.feeds = { @@ -102,24 +108,26 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTReshapeTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] class TRTReshapeTest3(TRTReshapeTest): - def setUp(self): self.bs = 1 self.input_shape = [7, 16, 27] self.reshape = [1, 8, 14, 0] self.data_shape = [ - self.bs, self.input_shape[0], self.input_shape[1], - self.input_shape[2] + self.bs, + self.input_shape[0], + self.input_shape[1], + self.input_shape[2], ] with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name='data', - shape=self.data_shape, - dtype='float32') + data = fluid.data( + name='data', shape=self.data_shape, dtype='float32' + ) bn_out = fluid.layers.batch_norm(data, is_test=True) out = self.append_reshape(bn_out, self.reshape) self.feeds = { @@ -127,7 +135,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTReshapeTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) ''' self.dynamic_shape_params = TRTReshapeTest.DynamicShapeParam({ 'data': [1, 3, 8, 8] diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_roi_align_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_roi_align_op.py index a854e74110c63..3b1d75d695448 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_roi_align_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_roi_align_op.py @@ -22,7 +22,6 @@ class TRTRoiAlignTest(InferencePassTest): - def setUp(self): self.bs = 2 self.num_rois = 4 @@ -35,21 +34,27 @@ def setUp(self): def build(self): self.trt_parameters = TRTRoiAlignTest.TensorRTParam( - 1 << 30, self.bs * self.num_rois, 1, self.precision, self.serialize, - False) + 1 << 30, + self.bs * self.num_rois, + 1, + self.precision, + self.serialize, + False, + ) with fluid.program_guard(self.main_program, self.startup_program): data_shape = [-1, self.channel, self.height, self.width] data = fluid.data(name='data', shape=data_shape, dtype='float32') - rois = fluid.data(name='rois', - shape=[-1, 4], - dtype='float32', - lod_level=1) + rois = fluid.data( + name='rois', shape=[-1, 4], dtype='float32', lod_level=1 + ) roi_align_out = fluid.layers.roi_align(data, rois) out = fluid.layers.batch_norm(roi_align_out, is_test=True) rois_lod = fluid.create_lod_tensor( np.random.random([self.bs * self.num_rois, 4]).astype('float32'), - [[self.num_rois, self.num_rois]], fluid.CPUPlace()) + [[self.num_rois, self.num_rois]], + fluid.CPUPlace(), + ) data_shape[0] = self.bs self.feeds = { @@ -66,27 +71,38 @@ def check_output(self): atol = 1e-3 self.check_output_with_option(use_gpu, atol, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) def set_dynamic(self): min_shape_spec = dict() max_shape_spec = dict() opt_shape_spec = dict() min_shape_spec['data'] = [ - self.bs, self.channel, self.height // 2, self.width // 2 + self.bs, + self.channel, + self.height // 2, + self.width // 2, ] min_shape_spec['rois'] = [1, 4] max_shape_spec['data'] = [ - self.bs, self.channel, self.height * 2, self.width * 2 + self.bs, + self.channel, + self.height * 2, + self.width * 2, ] max_shape_spec['rois'] = [self.bs * self.num_rois, 4] opt_shape_spec['data'] = [ - self.bs, self.channel, self.height, self.width + self.bs, + self.channel, + self.height, + self.width, ] opt_shape_spec['rois'] = [self.bs * self.num_rois, 4] self.dynamic_shape_params = InferencePassTest.DynamicShapeParam( - min_shape_spec, max_shape_spec, opt_shape_spec, False) + min_shape_spec, max_shape_spec, opt_shape_spec, False + ) def run_test(self): self.build() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_scale_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_scale_op.py index 77f2170f3e28d..966e1e77e1242 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_scale_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_scale_op.py @@ -22,7 +22,6 @@ class TRTScaleTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[-1, 512], dtype="float32") @@ -34,30 +33,30 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTScaleTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def append_scale(self, data): - return fluid.layers.scale(x=data, - scale=2.0, - bias=-1.0, - bias_after_scale=False) + return fluid.layers.scale( + x=data, scale=2.0, bias=-1.0, bias_after_scale=False + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTScaleShape2Test(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 512, 512], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 512, 512], dtype="float32" + ) scale_out = self.append_scale(data) out = fluid.layers.batch_norm(scale_out, is_test=True) @@ -66,21 +65,22 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTScaleShape2Test.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def append_scale(self, data): - return fluid.layers.scale(x=data, - scale=2.0, - bias=-1.0, - bias_after_scale=False) + return fluid.layers.scale( + x=data, scale=2.0, bias=-1.0, bias_after_scale=False + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_shuffle_channel_detect_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_shuffle_channel_detect_pass.py index 0752599e76b8c..d5bbbcde1e163 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_shuffle_channel_detect_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_shuffle_channel_detect_pass.py @@ -21,12 +21,11 @@ class ShuffleChannelFuseTRTPassTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) reshape1 = fluid.layers.reshape(x=data, shape=[-1, 2, 3, 64, 64]) trans = fluid.layers.transpose(x=reshape1, perm=[0, 2, 1, 3, 4]) reshape2 = fluid.layers.reshape(x=trans, shape=[-1, 6, 64, 64]) @@ -37,7 +36,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = ShuffleChannelFuseTRTPassTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -45,7 +45,8 @@ def test_check_output(self): self.check_output() self.assertTrue( - PassVersionChecker.IsCompatible('shuffle_channel_detect_pass')) + PassVersionChecker.IsCompatible('shuffle_channel_detect_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_dynamic_plugin.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_dynamic_plugin.py index e4d3f5b418e0e..e9ea9948af795 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_dynamic_plugin.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_dynamic_plugin.py @@ -20,9 +20,8 @@ from paddle.fluid.core import AnalysisConfig -#normal starts && ends +# normal starts && ends class SlicePluginTRTDynamicTest(InferencePassTest): - def setUpSliceParams(self): self.params_axes = [1, 3] self.params_starts = [0, 1] @@ -30,11 +29,15 @@ def setUpSliceParams(self): def setUpTensorRTParams(self): self.trt_parameters = SlicePluginTRTDynamicTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.enable_trt = True self.dynamic_shape_params = SlicePluginTRTDynamicTest.DynamicShapeParam( - {'data': [1, 1, 1, 1]}, {'data': [8, 8, 8, 8]}, - {'data': [8, 8, 8, 8]}, False) + {'data': [1, 1, 1, 1]}, + {'data': [8, 8, 8, 8]}, + {'data': [8, 8, 8, 8]}, + False, + ) def setUp(self): self.setUpSliceParams() @@ -44,10 +47,9 @@ def setUp(self): axes = self.params_axes starts = self.params_starts ends = self.params_ends - slice_out = fluid.layers.slice(data, - axes=axes, - starts=starts, - ends=ends) + slice_out = fluid.layers.slice( + data, axes=axes, starts=starts, ends=ends + ) self.feeds = { "data": np.random.random((3, 3, 3, 3)).astype("float32"), @@ -66,7 +68,6 @@ def test_check_output(self): class SlicePluginTRTDynamicBoundTest(SlicePluginTRTDynamicTest): - def setUpSliceParams(self): self.params_axes = [1, 3] self.params_starts = [0, 1] @@ -74,27 +75,40 @@ def setUpSliceParams(self): def setUpTensorRTParams(self): self.trt_parameters = SlicePluginTRTDynamicBoundTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False + ) self.enable_trt = True - self.dynamic_shape_params = SlicePluginTRTDynamicBoundTest.DynamicShapeParam( - {'data': [1, 1, 1, 1]}, {'data': [8, 8, 8, 8]}, - {'data': [8, 8, 8, 8]}, False) + self.dynamic_shape_params = ( + SlicePluginTRTDynamicBoundTest.DynamicShapeParam( + {'data': [1, 1, 1, 1]}, + {'data': [8, 8, 8, 8]}, + {'data': [8, 8, 8, 8]}, + False, + ) + ) class SlicePluginTRTDynamicNegativeBoundTest(SlicePluginTRTDynamicTest): - def setUpSliceParams(self): self.params_axes = [1, 3] self.params_starts = [-5, 1] self.params_ends = [2, 1000] def setUpTensorRTParams(self): - self.trt_parameters = SlicePluginTRTDynamicNegativeBoundTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False) + self.trt_parameters = ( + SlicePluginTRTDynamicNegativeBoundTest.TensorRTParam( + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False + ) + ) self.enable_trt = True - self.dynamic_shape_params = SlicePluginTRTDynamicNegativeBoundTest.DynamicShapeParam( - {'data': [1, 1, 1, 1]}, {'data': [8, 8, 8, 8]}, - {'data': [8, 8, 8, 8]}, False) + self.dynamic_shape_params = ( + SlicePluginTRTDynamicNegativeBoundTest.DynamicShapeParam( + {'data': [1, 1, 1, 1]}, + {'data': [8, 8, 8, 8]}, + {'data': [8, 8, 8, 8]}, + False, + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_plugin.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_plugin.py index 70334d92cc4ed..ab93e3f647513 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_plugin.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_slice_plugin.py @@ -20,9 +20,8 @@ from paddle.fluid.core import AnalysisConfig -#normal starts && ends +# normal starts && ends class SlicePluginTRTTest(InferencePassTest): - def setUpSliceParams(self): self.params_axes = [1, 3] self.params_starts = [0, 1] @@ -30,7 +29,8 @@ def setUpSliceParams(self): def setUpTensorRTParams(self): self.trt_parameters = SlicePluginTRTTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, False, False + ) self.enable_trt = True def setUp(self): @@ -41,10 +41,9 @@ def setUp(self): axes = self.params_axes starts = self.params_starts ends = self.params_ends - slice_out = fluid.layers.slice(data, - axes=axes, - starts=starts, - ends=ends) + slice_out = fluid.layers.slice( + data, axes=axes, starts=starts, ends=ends + ) out = fluid.layers.batch_norm(slice_out, is_test=True) self.feeds = { @@ -63,51 +62,48 @@ def test_check_output(self): self.check_output_with_option(use_gpu[i], atol) -#negative starts && ends +# negative starts && ends class SlicePluginTRTTestNegativeStartsAndEnds(SlicePluginTRTTest): - def setUpSliceParams(self): self.params_axes = [2, 3] self.params_starts = [-3, -2] self.params_ends = [-1, 3] -#exceeded bound starts && ends +# exceeded bound starts && ends class SlicePluginTRTTestStartsAndEndsBoundCheck(SlicePluginTRTTest): - def setUpSliceParams(self): self.params_axes = [2, 3] self.params_starts = [-5, -2] self.params_ends = [-1, 8] -#fp16 +# fp16 class SlicePluginTRTTestFp16(SlicePluginTRTTest): - def setUpTensorRTParams(self): self.trt_parameters = SlicePluginTRTTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, False, False + ) self.enable_trt = True class StaticSlicePluginTRTTestFp16(SlicePluginTRTTest): - def setUpTensorRTParams(self): self.trt_parameters = SlicePluginTRTTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Half, True, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Half, True, False + ) self.enable_trt = True class StaticSlicePluginTRTTestFp32(SlicePluginTRTTest): - def setUpTensorRTParams(self): self.trt_parameters = SlicePluginTRTTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, True, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, True, False + ) self.enable_trt = True class SlicePluginTRTTestInt32(SlicePluginTRTTest): - def setUp(self): self.setUpSliceParams() self.setUpTensorRTParams() @@ -116,10 +112,9 @@ def setUp(self): axes = self.params_axes starts = self.params_starts ends = self.params_ends - slice_out = fluid.layers.slice(data, - axes=axes, - starts=starts, - ends=ends) + slice_out = fluid.layers.slice( + data, axes=axes, starts=starts, ends=ends + ) cast_out = fluid.layers.cast(slice_out, 'float32') out = fluid.layers.batch_norm(cast_out, is_test=True) @@ -130,10 +125,10 @@ def setUp(self): class StaticSlicePluginTRTTestInt32(SlicePluginTRTTest): - def setUpTensorRTParams(self): self.trt_parameters = SlicePluginTRTTest.TensorRTParam( - 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, True, False) + 1 << 30, 32, 1, AnalysisConfig.Precision.Float32, True, False + ) self.enable_trt = True def setUp(self): @@ -144,10 +139,9 @@ def setUp(self): axes = self.params_axes starts = self.params_starts ends = self.params_ends - slice_out = fluid.layers.slice(data, - axes=axes, - starts=starts, - ends=ends) + slice_out = fluid.layers.slice( + data, axes=axes, starts=starts, ends=ends + ) cast_out = fluid.layers.cast(slice_out, 'float32') out = fluid.layers.batch_norm(cast_out, is_test=True) diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_squeeze2_matmul_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_squeeze2_matmul_fuse_pass.py index 54c1f72268573..ce86d97afb0b1 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_squeeze2_matmul_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_squeeze2_matmul_fuse_pass.py @@ -42,7 +42,8 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, ['mul', 'elementwise_add'], (1e-4, 1e-1) def add_ignore_pass_case(self): @@ -65,9 +66,10 @@ def teller1(program_config, predictor_config): def sample_program_config(self, draw): # 1. Generate shape of input:X of squeeze2 x_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) # axes of squeeze2 == [2, 3] x_shape += [1, 1] axes = [2, 3] @@ -79,9 +81,10 @@ def sample_program_config(self, draw): # 3. Generate legal shape of input:Y of matmul y_shape = draw( - st.lists(st.integers(min_value=1, max_value=8), - min_size=2, - max_size=2)) + st.lists( + st.integers(min_value=1, max_value=8), min_size=2, max_size=2 + ) + ) y_shape[0] = x_shape[1] # 4. Generate legal attr:axis of elementwise_add @@ -103,17 +106,11 @@ def sample_program_config(self, draw): "X": ["squeeze2_x"], }, axes=axes, - outputs={ - "Out": ["squeeze2_out"], - "XShape": ["xshape"] - }, + outputs={"Out": ["squeeze2_out"], "XShape": ["xshape"]}, ) matmul_op = OpConfig( "matmul", - inputs={ - "X": ["squeeze2_out"], - "Y": ["matmul_y"] - }, + inputs={"X": ["squeeze2_out"], "Y": ["matmul_y"]}, outputs={"Out": ["matmul_out"]}, alpha=alpha, transpose_X=transpose_X, @@ -128,10 +125,7 @@ def sample_program_config(self, draw): add_op = OpConfig( "elementwise_add", - inputs={ - "X": ["matmul_out"], - "Y": ["bias"] - }, + inputs={"X": ["matmul_out"], "Y": ["bias"]}, outputs={"Out": ["add_out"]}, axis=axis, ) @@ -152,9 +146,11 @@ def sample_program_config(self, draw): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=25, - passes=["trt_squeeze2_matmul_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=25, + passes=["trt_squeeze2_matmul_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_subgraph_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_subgraph_pass.py index 2472ff027e3cc..7478f6a3734e7 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_subgraph_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_subgraph_pass.py @@ -24,12 +24,11 @@ class TensorRTSubgraphPassFcTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) fc_out = fluid.layers.fc(input=[data], act=None, size=1000) reshape_out = fluid.layers.reshape(x=fc_out, shape=[1, 1000]) self.feeds = { @@ -37,7 +36,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassFcTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [reshape_out] def test_check_output(self): @@ -46,19 +46,19 @@ def test_check_output(self): # TRT output shape of fc is (1, 1000, 1, 1). To compare the output value only, flatten the results. self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassConcatTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data1 = fluid.data(name="data1", - shape=[-1, 3, 64, 64], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[-1, 3, 64, 64], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[-1, 3, 64, 64], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[-1, 3, 64, 64], dtype="float32" + ) concat_out = fluid.layers.concat([data1, data2], axis=2) out = fluid.layers.batch_norm(concat_out, is_test=True) self.feeds = { @@ -67,7 +67,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassConcatTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -75,16 +76,16 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassSplitTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 64, 64], dtype="float32" + ) split_out = fluid.layers.split(data, dim=-1, num_or_sections=2) out = fluid.layers.batch_norm(split_out[0], is_test=True) self.feeds = { @@ -92,7 +93,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassSplitTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -100,16 +102,16 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassSplitSerializeTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 64, 64], dtype="float32" + ) split_out = fluid.layers.split(data, dim=-1, num_or_sections=2) out = fluid.layers.batch_norm(split_out[0], is_test=True) self.feeds = { @@ -117,7 +119,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassSplitTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False + ) self.fetch_list = [out] def test_check_output(self): @@ -127,16 +130,16 @@ def test_check_output(self): shutil.rmtree(self.path + "_opt_cache") self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassDynamicSplitFp16SerializeTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 64, 64], dtype="float32" + ) split_out = fluid.layers.split(data, dim=-1, num_or_sections=2) out = fluid.layers.batch_norm(split_out[0], is_test=True) self.feeds = { @@ -144,10 +147,16 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassSplitTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False) - self.dynamic_shape_params = TensorRTSubgraphPassDynamicSplitFp16SerializeTest.DynamicShapeParam( - {'data': [1, 3, 8, 64]}, {'data': [1, 3, 512, 64]}, - {'data': [1, 3, 256, 64]}, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassDynamicSplitFp16SerializeTest.DynamicShapeParam( + {'data': [1, 3, 8, 64]}, + {'data': [1, 3, 512, 64]}, + {'data': [1, 3, 256, 64]}, + False, + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -157,31 +166,36 @@ def test_check_output(self): shutil.rmtree(self.path + "_opt_cache") self.check_output_with_option(use_gpu, 1e-3) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassInstanceNormTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 3, 64, 64], dtype="float32" + ) param_attr = fluid.ParamAttr( name='instance_norm_w', - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0), + ) bias_attr = fluid.ParamAttr( name='instance_norm_b', - initializer=fluid.initializer.Constant(value=0.0)) - out = fluid.layers.instance_norm(input=data, - param_attr=param_attr, - bias_attr=bias_attr) + initializer=fluid.initializer.Constant(value=0.0), + ) + out = fluid.layers.instance_norm( + input=data, param_attr=param_attr, bias_attr=bias_attr + ) self.feeds = { "data": np.random.random([1, 3, 64, 64]).astype("float32"), } self.enable_trt = True - self.trt_parameters = TensorRTSubgraphPassInstanceNormTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + self.trt_parameters = ( + TensorRTSubgraphPassInstanceNormTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -189,16 +203,16 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu, atol=1e-4, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassTransposeTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) transpose_out = self.append_transpose(data) out = fluid.layers.batch_norm(transpose_out, is_test=True) self.feeds = { @@ -206,7 +220,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassTransposeTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def append_transpose(self, data): @@ -217,25 +232,27 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassLayerNormTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 64, 64], - dtype="float32") - out = fluid.layers.layer_norm(data, - begin_norm_axis=self.begin_norm_axis) + data = fluid.data( + name="data", shape=[-1, 3, 64, 64], dtype="float32" + ) + out = fluid.layers.layer_norm( + data, begin_norm_axis=self.begin_norm_axis + ) self.feeds = { "data": np.random.random([1, 3, 64, 64]).astype("float32"), } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassLayerNormTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def set_params(self): @@ -246,19 +263,20 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassLayerNormDynamicTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 3, 64, 64], - dtype="float32") - out = fluid.layers.layer_norm(data, - begin_norm_axis=self.begin_norm_axis) + data = fluid.data( + name="data", shape=[-1, 3, 64, 64], dtype="float32" + ) + out = fluid.layers.layer_norm( + data, begin_norm_axis=self.begin_norm_axis + ) self.feeds = { "data": np.random.random([1, 3, 64, 64]).astype("float32"), } @@ -267,16 +285,25 @@ def setUp(self): def set_trt_params(self): self.enable_trt = True - self.trt_parameters = TensorRTSubgraphPassLayerNormDynamicTest.TensorRTParam( - 1 << 30, 32, 0, self.precision, self.serialize, False) - self.dynamic_shape_params = TensorRTSubgraphPassLayerNormDynamicTest.DynamicShapeParam( - { - 'data': [1, 3, 64, 64], - }, { - 'data': [8, 8, 64, 64], - }, { - 'data': [4, 4, 64, 64], - }, False) + self.trt_parameters = ( + TensorRTSubgraphPassLayerNormDynamicTest.TensorRTParam( + 1 << 30, 32, 0, self.precision, self.serialize, False + ) + ) + self.dynamic_shape_params = ( + TensorRTSubgraphPassLayerNormDynamicTest.DynamicShapeParam( + { + 'data': [1, 3, 64, 64], + }, + { + 'data': [8, 8, 64, 64], + }, + { + 'data': [4, 4, 64, 64], + }, + False, + ) + ) def set_params(self): self.begin_norm_axis = 2 @@ -290,12 +317,13 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassLayerNormDynamicFP16Test( - TensorRTSubgraphPassLayerNormDynamicTest): - + TensorRTSubgraphPassLayerNormDynamicTest +): def set_params(self): self.begin_norm_axis = 2 self.precision = AnalysisConfig.Precision.Half @@ -308,33 +336,33 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu, atol=0.01, rtol=0.01) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassLayerNormBeginNormAxis2Test( - TensorRTSubgraphPassLayerNormTest): - + TensorRTSubgraphPassLayerNormTest +): def set_params(self): self.begin_norm_axis = 2 class TensorRTSubgraphPassLayerNormBeginNormAxis3Test( - TensorRTSubgraphPassLayerNormTest): - + TensorRTSubgraphPassLayerNormTest +): def set_params(self): self.begin_norm_axis = 3 class TensorRTSubgraphPassElementwiseTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data1 = fluid.data(name="data1", - shape=[-1, 3, 64, 64], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[-1, 3, 64, 64], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[-1, 3, 64, 64], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[-1, 3, 64, 64], dtype="float32" + ) eltwise_out = self.append_eltwise(data1, data2) out = fluid.layers.batch_norm(eltwise_out, is_test=True) self.feeds = { @@ -343,7 +371,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TensorRTSubgraphPassElementwiseTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def append_eltwise(self, data1, data2): @@ -354,38 +383,40 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) -class TensorRTSubgraphPassElementwiseMulTest(TensorRTSubgraphPassElementwiseTest - ): - +class TensorRTSubgraphPassElementwiseMulTest( + TensorRTSubgraphPassElementwiseTest +): def append_eltwise(self, data1, data2): return fluid.layers.elementwise_mul(x=data1, y=data2) class TensorRTSubgraphPassElementwiseSerializeTest( - TensorRTSubgraphPassElementwiseTest): - + TensorRTSubgraphPassElementwiseTest +): def setUp(self): super(TensorRTSubgraphPassElementwiseSerializeTest, self).setUp() self.trt_parameters = TensorRTSubgraphPassElementwiseTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False) + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False + ) def test_check_output(self): if os.path.exists(self.path + "_opt_cache"): shutil.rmtree(self.path + "_opt_cache") - super(TensorRTSubgraphPassElementwiseSerializeTest, - self).test_check_output() + super( + TensorRTSubgraphPassElementwiseSerializeTest, self + ).test_check_output() class TensorRTSubgraphPassElementwiseBroadcastDynamicTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data1 = fluid.data(name="data1", - shape=[-1, 3, 64, 64], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[-1, 3, 64, 64], dtype="float32" + ) data2 = fluid.data(name="data2", shape=[64, 64], dtype="float32") eltwise_out = self.append_eltwise(data1, data2) out = fluid.layers.batch_norm(eltwise_out, is_test=True) @@ -394,19 +425,17 @@ def setUp(self): "data2": np.random.random([64, 64]).astype("float32"), } self.enable_trt = True - self.trt_parameters = TensorRTSubgraphPassElementwiseBroadcastDynamicTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False) + self.trt_parameters = ( + TensorRTSubgraphPassElementwiseBroadcastDynamicTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False + ) + ) self.dynamic_shape_params = TensorRTSubgraphPassElementwiseBroadcastDynamicTest.DynamicShapeParam( - { - 'data1': [1, 3, 8, 64], - 'data2': [8, 64] - }, { - 'data1': [1, 3, 512, 64], - 'data2': [512, 64] - }, { - 'data1': [1, 3, 256, 64], - 'data2': [256, 64] - }, False) + {'data1': [1, 3, 8, 64], 'data2': [8, 64]}, + {'data1': [1, 3, 512, 64], 'data2': [512, 64]}, + {'data1': [1, 3, 256, 64], 'data2': [256, 64]}, + False, + ) self.fetch_list = [out] def append_eltwise(self, data1, data2): @@ -419,24 +448,27 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TensorRTSubgraphPassShuffleChannelTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) sc_out = fluid.layers.shuffle_channel(data, group=3) out = fluid.layers.batch_norm(sc_out, is_test=True) self.feeds = { "data": np.random.random([1, 6, 64, 64]).astype("float32"), } self.enable_trt = True - self.trt_parameters = TensorRTSubgraphPassShuffleChannelTest.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False) + self.trt_parameters = ( + TensorRTSubgraphPassShuffleChannelTest.TensorRTParam( + 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, False, False + ) + ) self.fetch_list = [out] def test_check_output(self): @@ -444,7 +476,8 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tile_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tile_op.py index bb3772ca5aeeb..52fa99fc81ad5 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tile_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tile_op.py @@ -23,12 +23,11 @@ class TRTTileTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[4, 3, 224, 256], - dtype="float32") + data = fluid.data( + name="data", shape=[4, 3, 224, 256], dtype="float32" + ) tile_out = paddle.tile(x=data, repeat_times=[1, 1, 1, 1]) out = fluid.layers.batch_norm(tile_out, is_test=True) @@ -37,7 +36,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTTileTest.TensorRTParam( - 1 << 30, 16, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 16, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -45,11 +45,11 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTTileExpandTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[1, 1, 1, 1], dtype="float32") @@ -61,7 +61,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTTileExpandTest.TensorRTParam( - 1 << 30, 1, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, 1, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -69,11 +70,11 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTTileExpandStaticTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[1, 1, 1, 1], dtype="float32") @@ -85,7 +86,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTTileExpandStaticTest.TensorRTParam( - 1 << 30, 1, 1, AnalysisConfig.Precision.Float32, True, False) + 1 << 30, 1, 1, AnalysisConfig.Precision.Float32, True, False + ) self.fetch_list = [out] def test_check_output(self): @@ -93,11 +95,11 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTTileExpandHalfTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): data = fluid.data(name="data", shape=[1, 1, 1, 1], dtype="float32") @@ -109,7 +111,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTTileExpandHalfTest.TensorRTParam( - 1 << 30, 1, 1, AnalysisConfig.Precision.Half, False, False) + 1 << 30, 1, 1, AnalysisConfig.Precision.Half, False, False + ) self.fetch_list = [out] def test_check_output(self): @@ -117,7 +120,8 @@ def test_check_output(self): use_gpu = True self.check_output_with_option(use_gpu, 1e-4, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_transpose_flatten_concat_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_transpose_flatten_concat_fuse_pass.py index 3a15f0dcf34a6..409d36600d28b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_transpose_flatten_concat_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_transpose_flatten_concat_fuse_pass.py @@ -21,15 +21,14 @@ class TransposeFlattenConcatFusePassTRTTest(InferencePassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data1 = fluid.data(name="data1", - shape=[8, 32, 128], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[8, 32, 128], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[8, 32, 128], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[8, 32, 128], dtype="float32" + ) trans1 = fluid.layers.transpose(data1, perm=[0, 2, 1]) trans2 = fluid.layers.transpose(data2, perm=[0, 2, 1]) flatt1 = fluid.layers.flatten(trans1) @@ -42,11 +41,14 @@ def setUp(self): self.feeds = { "data1": np.random.random([8, 32, 128]).astype("float32"), - "data2": np.random.random([8, 32, 128]).astype("float32") + "data2": np.random.random([8, 32, 128]).astype("float32"), } self.enable_trt = True - self.trt_parameters = TransposeFlattenConcatFusePassTRTTest.TensorRTParam( - 1 << 20, 8, 0, AnalysisConfig.Precision.Float32, False, False) + self.trt_parameters = ( + TransposeFlattenConcatFusePassTRTTest.TensorRTParam( + 1 << 20, 8, 0, AnalysisConfig.Precision.Float32, False, False + ) + ) self.fetch_list = [out] def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tuned_dynamic_shape.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tuned_dynamic_shape.py index 3ef7ca22acaa8..44345b89c1926 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tuned_dynamic_shape.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_tuned_dynamic_shape.py @@ -22,7 +22,6 @@ class TRTTunedDynamicShapeTest(unittest.TestCase): - def get_model(self): place = fluid.CUDAPlace(0) exe = fluid.Executor(place) @@ -30,21 +29,25 @@ def get_model(self): main_program = fluid.Program() startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): - data = fluid.data(name="data", - shape=[-1, 6, 64, 64], - dtype="float32") - conv_out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - groups=1, - padding=0, - bias_attr=False, - act=None) + data = fluid.data( + name="data", shape=[-1, 6, 64, 64], dtype="float32" + ) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + groups=1, + padding=0, + bias_attr=False, + act=None, + ) exe.run(startup_program) serialized_program = paddle.static.serialize_program( - data, conv_out, program=main_program) + data, conv_out, program=main_program + ) serialized_params = paddle.static.serialize_persistables( - data, conv_out, executor=exe, program=main_program) + data, conv_out, executor=exe, program=main_program + ) return serialized_program, serialized_params def get_config(self, model, params, tuned=False): @@ -61,9 +64,11 @@ def get_config(self, model, params, tuned=False): min_subgraph_size=0, precision_mode=paddle.inference.PrecisionType.Float32, use_static=True, - use_calib_mode=False) - config.enable_tuned_tensorrt_dynamic_shape('shape_range.pbtxt', - True) + use_calib_mode=False, + ) + config.enable_tuned_tensorrt_dynamic_shape( + 'shape_range.pbtxt', True + ) return config diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_yolo_box_op.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_yolo_box_op.py index 5eeafe0228a78..d2ad64cf72ef7 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_yolo_box_op.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_yolo_box_op.py @@ -22,26 +22,26 @@ class TRTYoloBoxTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): image_shape = [self.bs, self.channel, self.height, self.width] image = fluid.data(name='image', shape=image_shape, dtype='float32') - image_size = fluid.data(name='image_size', - shape=[self.bs, 2], - dtype='int32') + image_size = fluid.data( + name='image_size', shape=[self.bs, 2], dtype='int32' + ) boxes, scores = self.append_yolobox(image, image_size) self.feeds = { - 'image': - np.random.random(image_shape).astype('float32'), - 'image_size': - np.random.randint(32, 64, size=(self.bs, 2)).astype('int32'), + 'image': np.random.random(image_shape).astype('float32'), + 'image_size': np.random.randint(32, 64, size=(self.bs, 2)).astype( + 'int32' + ), } self.enable_trt = True self.trt_parameters = TRTYoloBoxTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [scores, boxes] def set_params(self): @@ -51,35 +51,37 @@ def set_params(self): self.width = 64 self.class_num = 80 self.anchors = [10, 13, 16, 30, 33, 23] - self.conf_thresh = .1 + self.conf_thresh = 0.1 self.downsample_ratio = 32 def append_yolobox(self, image, image_size): - return fluid.layers.yolo_box(x=image, - img_size=image_size, - class_num=self.class_num, - anchors=self.anchors, - conf_thresh=self.conf_thresh, - downsample_ratio=self.downsample_ratio) + return fluid.layers.yolo_box( + x=image, + img_size=image_size, + class_num=self.class_num, + anchors=self.anchors, + conf_thresh=self.conf_thresh, + downsample_ratio=self.downsample_ratio, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTYoloBoxFP16Test(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): image_shape = [self.bs, self.channel, self.height, self.width] image = fluid.data(name='image', shape=image_shape, dtype='float32') - image_size = fluid.data(name='image_size', - shape=[self.bs, 2], - dtype='int32') + image_size = fluid.data( + name='image_size', shape=[self.bs, 2], dtype='int32' + ) boxes, scores = self.append_yolobox(image, image_size) self.feeds = { @@ -88,7 +90,8 @@ def setUp(self): } self.enable_trt = True self.trt_parameters = TRTYoloBoxFP16Test.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Half, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Half, False, False + ) self.fetch_list = [scores, boxes] def set_params(self): @@ -98,46 +101,49 @@ def set_params(self): self.class_num = 1 self.anchors = [106, 148, 92, 300, 197, 334] self.channel = 18 - self.conf_thresh = .05 + self.conf_thresh = 0.05 self.downsample_ratio = 32 def append_yolobox(self, image, image_size): - return fluid.layers.yolo_box(x=image, - img_size=image_size, - class_num=self.class_num, - anchors=self.anchors, - conf_thresh=self.conf_thresh, - downsample_ratio=self.downsample_ratio) + return fluid.layers.yolo_box( + x=image, + img_size=image_size, + class_num=self.class_num, + anchors=self.anchors, + conf_thresh=self.conf_thresh, + downsample_ratio=self.downsample_ratio, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True, rtol=1e-1) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) class TRTYoloBoxIoUAwareTest(InferencePassTest): - def setUp(self): self.set_params() with fluid.program_guard(self.main_program, self.startup_program): image_shape = [self.bs, self.channel, self.height, self.width] image = fluid.data(name='image', shape=image_shape, dtype='float32') - image_size = fluid.data(name='image_size', - shape=[self.bs, 2], - dtype='int32') + image_size = fluid.data( + name='image_size', shape=[self.bs, 2], dtype='int32' + ) boxes, scores = self.append_yolobox(image, image_size) self.feeds = { - 'image': - np.random.random(image_shape).astype('float32'), - 'image_size': - np.random.randint(32, 64, size=(self.bs, 2)).astype('int32'), + 'image': np.random.random(image_shape).astype('float32'), + 'image_size': np.random.randint(32, 64, size=(self.bs, 2)).astype( + 'int32' + ), } self.enable_trt = True self.trt_parameters = TRTYoloBoxTest.TensorRTParam( - 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False) + 1 << 30, self.bs, 1, AnalysisConfig.Precision.Float32, False, False + ) self.fetch_list = [scores, boxes] def set_params(self): @@ -147,27 +153,30 @@ def set_params(self): self.width = 64 self.class_num = 80 self.anchors = [10, 13, 16, 30, 33, 23] - self.conf_thresh = .1 + self.conf_thresh = 0.1 self.downsample_ratio = 32 self.iou_aware = True self.iou_aware_factor = 0.5 def append_yolobox(self, image, image_size): - return fluid.layers.yolo_box(x=image, - img_size=image_size, - class_num=self.class_num, - anchors=self.anchors, - conf_thresh=self.conf_thresh, - downsample_ratio=self.downsample_ratio, - iou_aware=self.iou_aware, - iou_aware_factor=self.iou_aware_factor) + return fluid.layers.yolo_box( + x=image, + img_size=image_size, + class_num=self.class_num, + anchors=self.anchors, + conf_thresh=self.conf_thresh, + downsample_ratio=self.downsample_ratio, + iou_aware=self.iou_aware, + iou_aware_factor=self.iou_aware_factor, + ) def test_check_output(self): if core.is_compiled_with_cuda(): use_gpu = True self.check_output_with_option(use_gpu, flatten=True) self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass')) + PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_unsqueeze2_eltwise_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/inference/test_unsqueeze2_eltwise_fuse_pass.py index 7ea2b494f3d17..d9626ebd1de6f 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_unsqueeze2_eltwise_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_unsqueeze2_eltwise_fuse_pass.py @@ -40,7 +40,8 @@ def sample_predictor_configs(self, program_config): min_subgraph_size=0, precision_mode=paddle_infer.PrecisionType.Float32, use_static=False, - use_calib_mode=False) + use_calib_mode=False, + ) yield config, [ 'elementwise_mul', ], (1e-5, 1e-5) @@ -48,9 +49,10 @@ def sample_predictor_configs(self, program_config): def sample_program_config(self, draw): # 1. Generate shape and attr of mul x_shape = draw( - st.lists(st.integers(min_value=1, max_value=10), - min_size=4, - max_size=4)) + st.lists( + st.integers(min_value=1, max_value=10), min_size=4, max_size=4 + ) + ) axis = -1 # 2. Generate legal shape and attr of input:Y of unsqueeze2 @@ -62,20 +64,14 @@ def sample_program_config(self, draw): inputs={ "X": ["unsqueeze2_x"], "AxesTensor": [], - "AxesTensorList": [] + "AxesTensorList": [], }, axes=unsqueeze2_axes, - outputs={ - "Out": ["unsqueeze2_out"], - "XShape": ["xshape"] - }, + outputs={"Out": ["unsqueeze2_out"], "XShape": ["xshape"]}, ) mul_op = OpConfig( "elementwise_mul", - inputs={ - "Y": ["unsqueeze2_out"], - "X": ["mul_x"] - }, + inputs={"Y": ["unsqueeze2_out"], "X": ["mul_x"]}, axis=axis, outputs={"Out": ["mul_out"]}, ) @@ -97,9 +93,11 @@ def sample_program_config(self, draw): return program_config def test(self): - self.run_and_statis(quant=False, - max_examples=300, - passes=["unsqueeze2_eltwise_fuse_pass"]) + self.run_and_statis( + quant=False, + max_examples=300, + passes=["unsqueeze2_eltwise_fuse_pass"], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_yolo_box_post.py b/python/paddle/fluid/tests/unittests/ir/inference/test_yolo_box_post.py index 74f251ec62b6e..fd1d03a6a7229 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_yolo_box_post.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_yolo_box_post.py @@ -20,22 +20,24 @@ paddle.enable_static() -def yolo_box_post(box0, - box1, - box2, - im_shape, - im_scale, - anchors0=[116, 90, 156, 198, 373, 326], - anchors1=[30, 61, 62, 45, 59, 119], - anchors2=[10, 13, 16, 30, 33, 23], - class_num=80, - conf_thresh=0.005, - downsample_ratio0=32, - downsample_ratio1=16, - downsample_ratio2=8, - clip_bbox=True, - scale_x_y=1., - nms_threshold=0.45): +def yolo_box_post( + box0, + box1, + box2, + im_shape, + im_scale, + anchors0=[116, 90, 156, 198, 373, 326], + anchors1=[30, 61, 62, 45, 59, 119], + anchors2=[10, 13, 16, 30, 33, 23], + class_num=80, + conf_thresh=0.005, + downsample_ratio0=32, + downsample_ratio1=16, + downsample_ratio2=8, + clip_bbox=True, + scale_x_y=1.0, + nms_threshold=0.45, +): helper = LayerHelper('yolo_box_post', **locals()) output = helper.create_variable_for_type_inference(dtype=box0.dtype) nms_rois_num = helper.create_variable_for_type_inference(dtype='int32') @@ -44,35 +46,37 @@ def yolo_box_post(box0, 'Boxes1': box1, 'Boxes2': box2, "ImageShape": im_shape, - "ImageScale": im_scale + "ImageScale": im_scale, } outputs = {'Out': output, 'NmsRoisNum': nms_rois_num} - helper.append_op(type="yolo_box_post", - inputs=inputs, - attrs={ - 'anchors0': anchors0, - 'anchors1': anchors1, - 'anchors2': anchors2, - 'class_num': class_num, - 'conf_thresh': conf_thresh, - 'downsample_ratio0': downsample_ratio0, - 'downsample_ratio1': downsample_ratio1, - 'downsample_ratio2': downsample_ratio2, - 'clip_bbox': clip_bbox, - 'scale_x_y': scale_x_y, - 'nms_threshold': nms_threshold - }, - outputs=outputs) + helper.append_op( + type="yolo_box_post", + inputs=inputs, + attrs={ + 'anchors0': anchors0, + 'anchors1': anchors1, + 'anchors2': anchors2, + 'class_num': class_num, + 'conf_thresh': conf_thresh, + 'downsample_ratio0': downsample_ratio0, + 'downsample_ratio1': downsample_ratio1, + 'downsample_ratio2': downsample_ratio2, + 'clip_bbox': clip_bbox, + 'scale_x_y': scale_x_y, + 'nms_threshold': nms_threshold, + }, + outputs=outputs, + ) output.stop_gradient = True nms_rois_num.stop_gradient = True return output, nms_rois_num -@unittest.skipIf(not paddle.is_compiled_with_cuda(), - "only support cuda kernel.") +@unittest.skipIf( + not paddle.is_compiled_with_cuda(), "only support cuda kernel." +) class TestYoloBoxPost(unittest.TestCase): - def test_yolo_box_post(self): place = paddle.CUDAPlace(0) program = paddle.static.Program() @@ -90,8 +94,8 @@ def test_yolo_box_post(self): "box0": np.random.uniform(size=[1, 255, 19, 19]).astype("float32"), "box1": np.random.uniform(size=[1, 255, 38, 38]).astype("float32"), "box2": np.random.uniform(size=[1, 255, 76, 76]).astype("float32"), - "im_shape": np.array([[608., 608.]], "float32"), - "im_scale": np.array([[1., 1.]], "float32") + "im_shape": np.array([[608.0, 608.0]], "float32"), + "im_scale": np.array([[1.0, 1.0]], "float32"), } outs = exe.run(program, feed=feed, fetch_list=[out.name, rois_num.name]) diff --git a/python/paddle/fluid/tests/unittests/ir/pass_test.py b/python/paddle/fluid/tests/unittests/ir/pass_test.py index 0336dcb2da548..4cae2e7fec187 100644 --- a/python/paddle/fluid/tests/unittests/ir/pass_test.py +++ b/python/paddle/fluid/tests/unittests/ir/pass_test.py @@ -23,7 +23,6 @@ class PassTest(unittest.TestCase): - @classmethod def setUpClass(self): self.main_program = fluid.Program() @@ -68,10 +67,12 @@ def check_output(self, startup_on_cpu=False, atol=1e-5): self.check_output_with_place(place, startup_on_cpu, atol) def _run_program(self, executor, program): - outs = executor.run(program=program, - feed=self.feeds, - fetch_list=self.fetch_list, - return_numpy=False) + outs = executor.run( + program=program, + feed=self.feeds, + fetch_list=self.fetch_list, + return_numpy=False, + ) outs_np = [] outs_lod = [] for out in outs: @@ -128,8 +129,10 @@ def check_output_with_place(self, place, startup_on_cpu=False, atol=1e-5): outs, lods = self._run_program(executor, self.main_program) self.assertTrue( len(self.fetch_list) == len(outs), - "Checking the number of fetchs failed. Expected: {}, Received: {}". - format(len(self.fetch_list), len(outs))) + "Checking the number of fetchs failed. Expected: {}, Received: {}".format( + len(self.fetch_list), len(outs) + ), + ) # Parameters may be changed in ir passes. opt_program = self._apply_ir_passes() @@ -138,13 +141,16 @@ def check_output_with_place(self, place, startup_on_cpu=False, atol=1e-5): if startup_on_cpu and not isinstance(place, fluid.CPUPlace): warnings.warn( "Parameters are on CPU, and will be transferred to GPU " - "automatically by data transform.") + "automatically by data transform." + ) outs_opt, lods_opt = self._run_program(executor, opt_program) self.assertTrue( len(self.fetch_list) == len(outs_opt), - "Checking the number of fetchs failed. Expected: {}, Received: {}". - format(len(self.fetch_list), len(outs_opt))) + "Checking the number of fetchs failed. Expected: {}, Received: {}".format( + len(self.fetch_list), len(outs_opt) + ), + ) for i in range(len(self.fetch_list)): is_allclose = np.allclose(outs_opt[i], outs[i], atol=atol) if not is_allclose: @@ -156,9 +162,17 @@ def check_output_with_place(self, place, startup_on_cpu=False, atol=1e-5): self.assertTrue( is_allclose, "Output (name: %s, shape: %s, dtype: %s) has diff at %s. The maximum diff is %e, first error element is %d, expected %e, but got %e" - % (self.fetch_list[i].name, str(self.fetch_list[i].shape), - self.fetch_list[i].dtype, str(place), max_diff, offset, - a.flatten()[offset], b.flatten()[offset])) + % ( + self.fetch_list[i].name, + str(self.fetch_list[i].shape), + self.fetch_list[i].dtype, + str(place), + max_diff, + offset, + a.flatten()[offset], + b.flatten()[offset], + ), + ) def _check_fused_ops(self, program): ''' @@ -179,9 +193,10 @@ def _check_fused_ops(self, program): self.assertTrue( self.num_fused_ops == acctual_num_fused_ops, "Checking of the number of fused operator < {} > failed. " - "Expected: {}, Received: {}".format(self.fused_op_type, - self.num_fused_ops, - acctual_num_fused_ops)) + "Expected: {}, Received: {}".format( + self.fused_op_type, self.num_fused_ops, acctual_num_fused_ops + ), + ) def check_program(self, program=None): ''' @@ -196,13 +211,16 @@ def check_program(self, program=None): self.assertTrue( self.main_program.desc != program.desc, "The optimized program and the origin main_program hold the same " - "desc.") + "desc.", + ) self.assertTrue( self.main_program.num_blocks == program.num_blocks, "The number of blocks of the origin program and the optimized " "program are different ({} vs {}).".format( - self.main_program.num_blocks, program.num_blocks)) + self.main_program.num_blocks, program.num_blocks + ), + ) is_different = False for i in range(program.num_blocks): @@ -219,7 +237,8 @@ def check_program(self, program=None): break if len(self.main_program.block(i).vars) != len( - program.block(i).vars): + program.block(i).vars + ): # The number of vars in the block i of the origin program and # the optimized program is different. is_different = True @@ -235,7 +254,8 @@ def check_program(self, program=None): self.assertTrue( is_different, "The optimized program is logically the same with the origin " - "program.") + "program.", + ) def _find_op(self, specified_op, program, block_id): is_find = False diff --git a/python/paddle/fluid/tests/unittests/ir/test_convert_to_mixed_precision.py b/python/paddle/fluid/tests/unittests/ir/test_convert_to_mixed_precision.py index 20891b22fde3b..6169cb98b6b28 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_convert_to_mixed_precision.py +++ b/python/paddle/fluid/tests/unittests/ir/test_convert_to_mixed_precision.py @@ -23,55 +23,75 @@ from paddle.inference import convert_to_mixed_precision -@unittest.skipIf(not paddle.is_compiled_with_cuda() - or paddle.get_cudnn_version() < 8000, - 'should compile with cuda.') +@unittest.skipIf( + not paddle.is_compiled_with_cuda() or paddle.get_cudnn_version() < 8000, + 'should compile with cuda.', +) class TestConvertToMixedPrecision(unittest.TestCase): - def test_convert_to_fp16(self): model = resnet50(True) net = to_static( - model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')]) + model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')] + ) paddle.jit.save(net, 'resnet50/inference') - convert_to_mixed_precision('resnet50/inference.pdmodel', - 'resnet50/inference.pdiparams', - 'mixed/inference.pdmodel', - 'mixed/inference.pdiparams', - PrecisionType.Half, PlaceType.GPU, True) + convert_to_mixed_precision( + 'resnet50/inference.pdmodel', + 'resnet50/inference.pdiparams', + 'mixed/inference.pdmodel', + 'mixed/inference.pdiparams', + PrecisionType.Half, + PlaceType.GPU, + True, + ) def test_convert_to_fp16_with_fp16_input(self): model = resnet50(True) net = to_static( - model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')]) + model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')] + ) paddle.jit.save(net, 'resnet50/inference') - convert_to_mixed_precision('resnet50/inference.pdmodel', - 'resnet50/inference.pdiparams', - 'mixed1/inference.pdmodel', - 'mixed1/inference.pdiparams', - PrecisionType.Half, PlaceType.GPU, False) + convert_to_mixed_precision( + 'resnet50/inference.pdmodel', + 'resnet50/inference.pdiparams', + 'mixed1/inference.pdmodel', + 'mixed1/inference.pdiparams', + PrecisionType.Half, + PlaceType.GPU, + False, + ) def test_convert_to_fp16_with_blacklist(self): model = resnet50(True) net = to_static( - model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')]) + model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')] + ) paddle.jit.save(net, 'resnet50/inference') - convert_to_mixed_precision('resnet50/inference.pdmodel', - 'resnet50/inference.pdiparams', - 'mixed2/inference.pdmodel', - 'mixed2/inference.pdiparams', - PrecisionType.Half, PlaceType.GPU, False, - set('conv2d')) + convert_to_mixed_precision( + 'resnet50/inference.pdmodel', + 'resnet50/inference.pdiparams', + 'mixed2/inference.pdmodel', + 'mixed2/inference.pdiparams', + PrecisionType.Half, + PlaceType.GPU, + False, + set('conv2d'), + ) def test_convert_to_bf16(self): model = resnet50(True) net = to_static( - model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')]) + model, input_spec=[InputSpec(shape=[None, 3, 224, 224], name='x')] + ) paddle.jit.save(net, 'resnet50/inference') - convert_to_mixed_precision('resnet50/inference.pdmodel', - 'resnet50/inference.pdiparams', - 'mixed3/inference.pdmodel', - 'mixed3/inference.pdiparams', - PrecisionType.Bfloat16, PlaceType.GPU, True) + convert_to_mixed_precision( + 'resnet50/inference.pdmodel', + 'resnet50/inference.pdiparams', + 'mixed3/inference.pdmodel', + 'mixed3/inference.pdiparams', + PrecisionType.Bfloat16, + PlaceType.GPU, + True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/ir/test_fuse_resnet_unit.py b/python/paddle/fluid/tests/unittests/ir/test_fuse_resnet_unit.py index 58b227f4cb596..4dabcdbcf35d6 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_fuse_resnet_unit.py +++ b/python/paddle/fluid/tests/unittests/ir/test_fuse_resnet_unit.py @@ -22,13 +22,14 @@ np.random.seed(0) -@unittest.skipIf(not paddle.is_compiled_with_cuda() - or paddle.get_cudnn_version() < 8000 - or paddle.device.cuda.get_device_capability()[0] < 7, - "only support with cuda and cudnn version is at least 8.0 " - "and device's compute capability is at least 7.0") +@unittest.skipIf( + not paddle.is_compiled_with_cuda() + or paddle.get_cudnn_version() < 8000 + or paddle.device.cuda.get_device_capability()[0] < 7, + "only support with cuda and cudnn version is at least 8.0 " + "and device's compute capability is at least 7.0", +) class TestFuseResNetUnit(unittest.TestCase): - def test_fuse_resenet_unit(self): place = paddle.CUDAPlace(0) program = paddle.static.Program() @@ -36,14 +37,12 @@ def test_fuse_resenet_unit(self): with paddle.static.amp.fp16_guard(): with paddle.static.program_guard(program, startup_program): x = paddle.static.data("x", [1, 64, 64, 8]) - conv2d = paddle.nn.Conv2D(8, - 32, - 1, - bias_attr=False, - data_format='NHWC') - batch_norm = paddle.nn.BatchNorm(32, - act='relu', - data_layout='NHWC') + conv2d = paddle.nn.Conv2D( + 8, 32, 1, bias_attr=False, data_format='NHWC' + ) + batch_norm = paddle.nn.BatchNorm( + 32, act='relu', data_layout='NHWC' + ) out = batch_norm(conv2d(x)) graph = core.Graph(program.desc) core.get_pass("fuse_resnet_unit").apply(graph) @@ -52,15 +51,15 @@ def test_fuse_resenet_unit(self): after_params = paddle.static.amp.cast_model_to_fp16(after_program) exe = paddle.static.Executor(place) exe.run(startup_program) - paddle.static.amp.cast_parameters_to_fp16(place, - program, - to_fp16_var_names=params) paddle.static.amp.cast_parameters_to_fp16( - place, after_program, to_fp16_var_names=after_params) + place, program, to_fp16_var_names=params + ) + paddle.static.amp.cast_parameters_to_fp16( + place, after_program, to_fp16_var_names=after_params + ) feed = {"x": np.random.randn(1, 64, 64, 8).astype("float16")} before_out = exe.run(program, feed=feed, fetch_list=[out.name]) after_out = exe.run(after_program, feed=feed, fetch_list=[out.name]) - np.testing.assert_allclose(before_out[0], - after_out[0], - rtol=1e-05, - atol=0.005) + np.testing.assert_allclose( + before_out[0], after_out[0], rtol=1e-05, atol=0.005 + ) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_embedding_eltwise_layernorm_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_embedding_eltwise_layernorm_fuse_pass.py index 6a573e7beacce..6b0a69568a385 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_embedding_eltwise_layernorm_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_embedding_eltwise_layernorm_fuse_pass.py @@ -21,89 +21,102 @@ class EmbEltwiseLayerNormFusePassTest(PassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - word_id = fluid.layers.data(name="word_id", - shape=[1, 128, 1], - dtype="int64", - append_batch_size=False) - pos_id = fluid.layers.data(name="pos_id", - shape=[1, 128, 1], - dtype="int64", - append_batch_size=False) - sent_id = fluid.layers.data(name="sent_id", - shape=[1, 128, 1], - dtype="int64", - append_batch_size=False) - word_emb = fluid.layers.embedding(input=word_id, - size=(128, 768), - dtype='float32') - pos_emb = fluid.layers.embedding(input=pos_id, - size=(128, 768), - dtype='float32') - sent_emb = fluid.layers.embedding(input=sent_id, - size=(128, 768), - dtype='float32') + word_id = fluid.layers.data( + name="word_id", + shape=[1, 128, 1], + dtype="int64", + append_batch_size=False, + ) + pos_id = fluid.layers.data( + name="pos_id", + shape=[1, 128, 1], + dtype="int64", + append_batch_size=False, + ) + sent_id = fluid.layers.data( + name="sent_id", + shape=[1, 128, 1], + dtype="int64", + append_batch_size=False, + ) + word_emb = fluid.layers.embedding( + input=word_id, size=(128, 768), dtype='float32' + ) + pos_emb = fluid.layers.embedding( + input=pos_id, size=(128, 768), dtype='float32' + ) + sent_emb = fluid.layers.embedding( + input=sent_id, size=(128, 768), dtype='float32' + ) add1 = fluid.layers.elementwise_add(word_emb, pos_emb) add2 = fluid.layers.elementwise_add(add1, sent_emb) hidden1 = fluid.layers.layer_norm(input=add2, begin_norm_axis=2) - id1 = fluid.layers.data(name="id1", - shape=[1, 128, 1], - dtype="int64", - append_batch_size=False) - id2 = fluid.layers.data(name="id2", - shape=[1, 128, 1], - dtype="int64", - append_batch_size=False) - id3 = fluid.layers.data(name="id3", - shape=[1, 128, 1], - dtype="int64", - append_batch_size=False) - id4 = fluid.layers.data(name="id4", - shape=[1, 128, 1], - dtype="int64", - append_batch_size=False) - emb1 = fluid.layers.embedding(input=id1, - size=(128, 768), - dtype='float32') - emb2 = fluid.layers.embedding(input=id2, - size=(128, 768), - dtype='float32') - emb3 = fluid.layers.embedding(input=id3, - size=(128, 768), - dtype='float32') - emb4 = fluid.layers.embedding(input=id4, - size=(128, 768), - dtype='float32') + id1 = fluid.layers.data( + name="id1", + shape=[1, 128, 1], + dtype="int64", + append_batch_size=False, + ) + id2 = fluid.layers.data( + name="id2", + shape=[1, 128, 1], + dtype="int64", + append_batch_size=False, + ) + id3 = fluid.layers.data( + name="id3", + shape=[1, 128, 1], + dtype="int64", + append_batch_size=False, + ) + id4 = fluid.layers.data( + name="id4", + shape=[1, 128, 1], + dtype="int64", + append_batch_size=False, + ) + emb1 = fluid.layers.embedding( + input=id1, size=(128, 768), dtype='float32' + ) + emb2 = fluid.layers.embedding( + input=id2, size=(128, 768), dtype='float32' + ) + emb3 = fluid.layers.embedding( + input=id3, size=(128, 768), dtype='float32' + ) + emb4 = fluid.layers.embedding( + input=id4, size=(128, 768), dtype='float32' + ) add_1 = fluid.layers.elementwise_add(emb1, emb2) add_2 = fluid.layers.elementwise_add(add_1, emb3) add_3 = fluid.layers.elementwise_add(add_2, emb4) hidden_1 = fluid.layers.layer_norm(input=add_3, begin_norm_axis=2) self.feeds = { - "word_id": - np.random.randint(low=0, high=128, - size=(1, 128, 1)).astype("int64"), - "pos_id": - np.random.randint(low=0, high=128, - size=(1, 128, 1)).astype("int64"), - "sent_id": - np.random.randint(low=0, high=128, - size=(1, 128, 1)).astype("int64"), - "id1": - np.random.randint(low=0, high=128, - size=(1, 128, 1)).astype("int64"), - "id2": - np.random.randint(low=0, high=128, - size=(1, 128, 1)).astype("int64"), - "id3": - np.random.randint(low=0, high=128, - size=(1, 128, 1)).astype("int64"), - "id4": - np.random.randint(low=0, high=128, - size=(1, 128, 1)).astype("int64"), + "word_id": np.random.randint( + low=0, high=128, size=(1, 128, 1) + ).astype("int64"), + "pos_id": np.random.randint( + low=0, high=128, size=(1, 128, 1) + ).astype("int64"), + "sent_id": np.random.randint( + low=0, high=128, size=(1, 128, 1) + ).astype("int64"), + "id1": np.random.randint(low=0, high=128, size=(1, 128, 1)).astype( + "int64" + ), + "id2": np.random.randint(low=0, high=128, size=(1, 128, 1)).astype( + "int64" + ), + "id3": np.random.randint(low=0, high=128, size=(1, 128, 1)).astype( + "int64" + ), + "id4": np.random.randint(low=0, high=128, size=(1, 128, 1)).astype( + "int64" + ), } self.fetch_list = [hidden1, hidden_1] self.pass_names = "embedding_eltwise_layernorm_fuse_pass" @@ -115,9 +128,7 @@ def test_check_output(self): if not core.is_compiled_with_cuda(): return self.pass_attrs = { - "embedding_eltwise_layernorm_fuse_pass": { - "use_gpu": True - } + "embedding_eltwise_layernorm_fuse_pass": {"use_gpu": True} } place = fluid.CUDAPlace(0) self.check_output_with_place(place, startup_on_cpu=True) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_fc_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_fc_fuse_pass.py index 060d63cc332a5..6022f98fa399c 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_fc_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_fc_fuse_pass.py @@ -21,17 +21,14 @@ class FCFusePassTest(PassTest): - def setUp(self): with fluid.program_guard(self.main_program, self.startup_program): - data = fluid.data(name="data", - shape=[32, 128], - dtype="float32", - lod_level=0) - tmp_0 = fluid.layers.fc(input=data, - size=128, - num_flatten_dims=1, - act="relu") + data = fluid.data( + name="data", shape=[32, 128], dtype="float32", lod_level=0 + ) + tmp_0 = fluid.layers.fc( + input=data, size=128, num_flatten_dims=1, act="relu" + ) tmp_1 = fluid.layers.fc(input=tmp_0, size=32, num_flatten_dims=1) tmp_2 = fluid.layers.softmax(input=tmp_1) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_fusion_group_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_fusion_group_pass.py index 10b861fad54c4..3775a4d08d1aa 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_fusion_group_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_fusion_group_pass.py @@ -22,12 +22,12 @@ class FusionGroupPassTest(PassTest): - def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 2) self.feed_vars.append( - fluid.data(name="data2", shape=[128, 128], dtype=dtype)) + fluid.data(name="data2", shape=[128, 128], dtype=dtype) + ) # subgraph with only 1 op node tmp_0 = self.feed_vars[0] * self.feed_vars[1] @@ -78,7 +78,6 @@ def test_check_output(self): class FusionGroupPassComplicatedTest(FusionGroupPassTest): - def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 64], dtype, 5) @@ -87,7 +86,8 @@ def build_program(self, dtype): tmp_0 = one * self.feed_vars[0] # subgraph with 9 op nodes tmp_1 = tmp_0 * layers.sigmoid(self.feed_vars[1]) + layers.sigmoid( - self.feed_vars[2]) * layers.tanh(self.feed_vars[3]) + self.feed_vars[2] + ) * layers.tanh(self.feed_vars[3]) tmp_2 = layers.tanh(tmp_1) + layers.sigmoid(self.feed_vars[4]) self.append_gradients(tmp_2) @@ -97,12 +97,12 @@ def build_program(self, dtype): class FusionGroupPassInplaceTest(FusionGroupPassTest): - def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 3) self.feed_vars.append( - fluid.data(name="data3", shape=[128, 32], dtype=dtype)) + fluid.data(name="data3", shape=[128, 32], dtype=dtype) + ) # subgraph with 3 op node tmp_0 = self.feed_vars[0] - self.feed_vars[1] @@ -115,7 +115,6 @@ def build_program(self, dtype): class FusionGroupPassTestFP64(FusionGroupPassTest): - def setUp(self): self.build_program("float64") self.feeds = self._feed_random_data(self.feed_vars) @@ -124,12 +123,12 @@ def setUp(self): class FusionGroupPassTestCastAndFP16(FusionGroupPassTest): - def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 2) self.feed_vars.append( - fluid.data(name="data2", shape=[128, 128], dtype=dtype)) + fluid.data(name="data2", shape=[128, 128], dtype=dtype) + ) # subgraph with 2 op nodes tmp_0 = self.feed_vars[0] * self.feed_vars[1] @@ -152,16 +151,17 @@ def build_program(self, dtype): class FusionGroupPassSumTest(FusionGroupPassTest): - def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([32, 128], dtype, 3) self.feed_vars.append( - fluid.data(name="data3", shape=[128, 128], dtype=dtype)) + fluid.data(name="data3", shape=[128, 128], dtype=dtype) + ) # subgraph with 2 op nodes tmp_0 = layers.sum( - [self.feed_vars[0], self.feed_vars[1], self.feed_vars[2]]) + [self.feed_vars[0], self.feed_vars[1], self.feed_vars[2]] + ) tmp_1 = layers.sqrt(tmp_0) tmp_2 = layers.mul(tmp_0, self.feed_vars[3]) # subgraph with 2 op nodes @@ -174,7 +174,6 @@ def build_program(self, dtype): class FusionGroupPassCastTest(FusionGroupPassTest): - def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([2, 2], dtype, 2) @@ -196,17 +195,15 @@ def setUp(self): class FusionGroupPassFillConstantTest(FusionGroupPassTest): - def build_program(self, dtype): with fluid.program_guard(self.main_program, self.startup_program): self.feed_vars = self._prepare_feed_vars([2, 2], dtype, 2) tmp_0 = layers.elementwise_add(self.feed_vars[0], self.feed_vars[1]) tmp_1 = layers.fill_constant(shape=[2, 2], dtype=dtype, value=2.0) - tmp_2 = layers.scale(tmp_1, - scale=3.0, - bias=1.0, - bias_after_scale=True) + tmp_2 = layers.scale( + tmp_1, scale=3.0, bias=1.0, bias_after_scale=True + ) tmp_3 = layers.elementwise_mul(tmp_2, tmp_0) self.append_gradients(tmp_3) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_generate_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_generate_pass.py index e4d8f1b32a7fc..76b58d3ce9000 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_generate_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_generate_pass.py @@ -23,9 +23,7 @@ # 1: relu(X=ewadd(X=mul(X=x, Y=w), Y=b)) => fc(Input=x, W=w, Bias=b) @ir.RegisterPass def generate_fc_fuse(): - def create_pass_pair(with_relu): - def pattern(x, w, b): mul = ir.PassDesc.OP.mul(X=x, Y=w) ewadd = ir.PassDesc.OP.elementwise_add(X=mul, Y=b) @@ -36,8 +34,9 @@ def pattern(x, w, b): def replace(x, w, b): fc = ir.PassDesc.OP.fc(Input=x, W=w, Bias=b) - fc.Attr("in_num_col_dims").MappedPattern(op="mul", - name="x_num_col_dims") + fc.Attr("in_num_col_dims").MappedPattern( + op="mul", name="x_num_col_dims" + ) if with_relu: fc.SetAttr("activation_type", "relu") return fc @@ -57,7 +56,6 @@ def multi_add_to_sum_v1(): @ir.RegisterPass def multi_add_to_sum_v2(): - def pattern(x, y, z): ewadd1 = ir.PassDesc.OP.elementwise_add(X=x, Y=y) ewadd2 = ir.PassDesc.OP.elementwise_add(X=ewadd1, Y=z) @@ -75,13 +73,14 @@ def multi_add_to_sum_v3(): # mul(x, y1), mul(x, y2) => slice(mul(x, concat(y1, y2))) -@ir.RegisterPass(input_specs={ - 'x': InputSpec([16, 32]), - 'y1': InputSpec([32, 12]), - 'y2': InputSpec([32, 48]) -}) +@ir.RegisterPass( + input_specs={ + 'x': InputSpec([16, 32]), + 'y1': InputSpec([32, 12]), + 'y2': InputSpec([32, 48]), + } +) def generate_combine_mul_v1(): - def pattern(x, y1, y2): mul1 = paddle.matmul(x, y1) mul2 = paddle.matmul(x, y2) @@ -99,7 +98,6 @@ def replace(x, y1, y2): @ir.RegisterPass def generate_combine_mul_v2(): - def pattern(x, y1, y2): mul1 = ir.PassDesc.OP.matmul_v2(X=x, Y=y1) mul2 = ir.PassDesc.OP.matmul_v2(X=x, Y=y2) @@ -118,7 +116,6 @@ def replace(x, y1, y2): # reshape(reshape(x)) => x @ir.RegisterPass(input_specs={'x': InputSpec([10, 16, 16])}) def generate_simplify_inference_v1(): - def pattern(x): transpose = paddle.transpose(x, [0, 2, 1]) return paddle.transpose(transpose, [0, 2, 1]) @@ -128,7 +125,6 @@ def pattern(x): @ir.RegisterPass def generate_simplify_inference_v2(): - def pattern(x): op1 = ir.PassDesc.OP.transpose2 op2 = ir.PassDesc.OP.transpose2 @@ -140,7 +136,6 @@ def pattern(x): @ir.RegisterPass def generate_layer_norm_fuse_pass(): - def pattern(x, gamma, beta): gamma.Attr("shape").Size().EQ(1) gamma.Attr("shape")[0].EQ(x.Attr("shape")[-1]) @@ -175,7 +170,6 @@ def replace(x, gamma, beta): @ir.RegisterPass def unimplemented_operand_exception(): - def pattern(x, y): return ir.PassDesc.OP.elementwise_add(X=x, Y=y) @@ -189,7 +183,6 @@ def replace(x, y): @ir.RegisterPass def unimplemented_operation_exception(): - def pattern(x, y): return ir.PassDesc.OP.elementwise_add(X=x, Y=y) @@ -208,7 +201,6 @@ def get_multi_pass_desc_from_str(s): class TestGeneratePass(unittest.TestCase): - def convert_ops_to_op_dicts(self, ops): op_dicts = dict() for op in ops: @@ -237,13 +229,13 @@ def test_exception(self): core.get_pass("unimplemented_operation_exception").apply(graph) def test_generate_fc_fuse(self): - def _check_fc_fuse_pass(pass_desc, with_relu): pattern_op_dicts = self.convert_ops_to_op_dicts(pass_desc.pattern) replace_op_dicts = self.convert_ops_to_op_dicts(pass_desc.replace) self.assertEqual(len(pattern_op_dicts.get("mul", [])), 1) - self.assertEqual(len(pattern_op_dicts.get("elementwise_add", [])), - 1) + self.assertEqual( + len(pattern_op_dicts.get("elementwise_add", [])), 1 + ) if with_relu: self.assertEqual(len(pattern_op_dicts.get("relu", [])), 1) pattern_op_num = 3 # relu, ewadd, mul @@ -285,12 +277,12 @@ def check_multi_add_to_sum(self, pass_type): feed = { "x": np.random.random([10, 10, 10]).astype("float32"), "y": np.random.random([10, 10, 10]).astype("float32"), - "z": np.random.random([10, 10, 10]).astype("float32") + "z": np.random.random([10, 10, 10]).astype("float32"), } before_out = executor.run(program, feed=feed, fetch_list=[out.name]) - after_out = executor.run(after_program, - feed=feed, - fetch_list=[out.name]) + after_out = executor.run( + after_program, feed=feed, fetch_list=[out.name] + ) np.testing.assert_allclose(before_out, after_out, rtol=1e-05) def test_multi_add_to_sum(self): @@ -320,13 +312,14 @@ def test_generate_combine_mul_v1(self): feed = { "x": np.random.random([16, 32]).astype("float32"), "y": np.random.random([32, 12]).astype("float32"), - "z": np.random.random([32, 48]).astype("float32") + "z": np.random.random([32, 48]).astype("float32"), } before_out1, before_out2 = executor.run( - program, feed=feed, fetch_list=[out1.name, out2.name]) - after_out1, after_out2 = executor.run(after_program, - feed=feed, - fetch_list=[out1.name, out2.name]) + program, feed=feed, fetch_list=[out1.name, out2.name] + ) + after_out1, after_out2 = executor.run( + after_program, feed=feed, fetch_list=[out1.name, out2.name] + ) np.testing.assert_allclose(before_out1, after_out1, rtol=1e-05) np.testing.assert_allclose(before_out2, after_out2, rtol=1e-05) @@ -366,9 +359,9 @@ def check_generate_simplify_inference(self, pass_type): executor.run(startup_program) feed = {"x": np.random.random([10, 16, 16]).astype("float32")} before_out = executor.run(program, feed=feed, fetch_list=[out.name]) - after_out = executor.run(after_program, - feed=feed, - fetch_list=[out.name]) + after_out = executor.run( + after_program, feed=feed, fetch_list=[out.name] + ) np.testing.assert_allclose(before_out, after_out, rtol=1e-05) def test_generate_simplify_inference(self): @@ -381,12 +374,12 @@ def test_generate_layer_norm_fuse_pass(self): startup_program = paddle.static.Program() with paddle.static.program_guard(program, startup_program): x = paddle.static.data("x", [3, 64, 120], "float32") - gamma = paddle.static.create_parameter(shape=[120], - dtype="float32", - is_bias=True) - beta = paddle.static.create_parameter(shape=[120], - dtype="float32", - is_bias=True) + gamma = paddle.static.create_parameter( + shape=[120], dtype="float32", is_bias=True + ) + beta = paddle.static.create_parameter( + shape=[120], dtype="float32", is_bias=True + ) x_sub_mean = x - paddle.mean(x, axis=-1, keepdim=True) std_dev = paddle.mean(x_sub_mean.pow(2), axis=-1, keepdim=True) @@ -402,7 +395,7 @@ def test_generate_layer_norm_fuse_pass(self): executor.run(startup_program) feed = {"x": np.random.random([3, 64, 120]).astype("float32")} before_out = executor.run(program, feed=feed, fetch_list=[out.name]) - after_out = executor.run(after_program, - feed=feed, - fetch_list=[out.name]) + after_out = executor.run( + after_program, feed=feed, fetch_list=[out.name] + ) np.testing.assert_allclose(before_out, after_out, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_graph_to_program_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_graph_to_program_pass.py index c4f9147e33a42..1b68c64805cb7 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_graph_to_program_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_graph_to_program_pass.py @@ -31,7 +31,6 @@ def IRGraph_to_program(ir_graph): class GraphToProgramPassTest(unittest.TestCase): - def check_vars_equal(self, o_block, c_block): o_params = sorted(o_block.all_parameters(), key=lambda p: p.name) c_params = sorted(c_block.all_parameters(), key=lambda p: p.name) @@ -69,12 +68,12 @@ def check_op_attrs_equal(self, o_op, c_op): o_attr = o_attrs[attr_idx] c_attr = c_attrs[attr_idx] self.assertEqual(o_attr, c_attr) - self.assertEqual(o_op.desc.attr_type(o_attr), - c_op.desc.attr_type(c_attr)) + self.assertEqual( + o_op.desc.attr_type(o_attr), c_op.desc.attr_type(c_attr) + ) class SingleGraphToProgramPass(GraphToProgramPassTest): - def setUp(self): self.origin_program = self.build_program() ir_graph = program_to_IRGraph(self.origin_program) @@ -91,10 +90,12 @@ def build_program(): return program def test_check_parameter(self): - origin_parameter = sorted(self.origin_program.all_parameters(), - key=lambda p: p.name) - converted_parameter = sorted(self.converted_program.all_parameters(), - key=lambda p: p.name) + origin_parameter = sorted( + self.origin_program.all_parameters(), key=lambda p: p.name + ) + converted_parameter = sorted( + self.converted_program.all_parameters(), key=lambda p: p.name + ) self.assertEqual(len(origin_parameter), len(converted_parameter)) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_preln_residual_bias_fuse_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_preln_residual_bias_fuse_pass.py index a8d2cdc01c493..2f367b83ae640 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_preln_residual_bias_fuse_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_preln_residual_bias_fuse_pass.py @@ -19,20 +19,18 @@ class PrelnResidualBiasFusePassTest(PassTest): - def setUp(self): paddle.enable_static() - with paddle.static.program_guard(self.main_program, - self.startup_program): - x = paddle.static.data(name="x", - shape=[128, 768], - dtype="float32", - lod_level=0) + with paddle.static.program_guard( + self.main_program, self.startup_program + ): + x = paddle.static.data( + name="x", shape=[128, 768], dtype="float32", lod_level=0 + ) bias = paddle.static.create_parameter(shape=[768], dtype='float32') - y = paddle.static.data(name="y", - shape=[128, 768], - dtype="float32", - lod_level=0) + y = paddle.static.data( + name="y", shape=[128, 768], dtype="float32", lod_level=0 + ) x = x + bias elementwise_out = x + y out = paddle.static.nn.layer_norm(input=elementwise_out) @@ -57,19 +55,17 @@ def test_check_program(self): class PrelnResidualBiasFusePassNoBiasTest(PassTest): - def setUp(self): paddle.enable_static() - with paddle.static.program_guard(self.main_program, - self.startup_program): - x = paddle.static.data(name="x", - shape=[128, 768], - dtype="float32", - lod_level=0) - y = paddle.static.data(name="y", - shape=[128, 768], - dtype="float32", - lod_level=0) + with paddle.static.program_guard( + self.main_program, self.startup_program + ): + x = paddle.static.data( + name="x", shape=[128, 768], dtype="float32", lod_level=0 + ) + y = paddle.static.data( + name="y", shape=[128, 768], dtype="float32", lod_level=0 + ) elementwise_out = x + y out = paddle.static.nn.layer_norm(input=elementwise_out) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_skip_layernorm_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_skip_layernorm_pass.py index d53e7109d6a8b..82f53f95d9ead 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_skip_layernorm_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_skip_layernorm_pass.py @@ -21,18 +21,15 @@ class SkipLayerNormFusePassTest(PassTest): - def setUp(self): paddle.enable_static() with fluid.program_guard(self.main_program, self.startup_program): - x = fluid.data(name="x", - shape=[128, 768], - dtype="float32", - lod_level=0) - y = fluid.data(name="y", - shape=[128, 768], - dtype="float32", - lod_level=0) + x = fluid.data( + name="x", shape=[128, 768], dtype="float32", lod_level=0 + ) + y = fluid.data( + name="y", shape=[128, 768], dtype="float32", lod_level=0 + ) elementwise_out = fluid.layers.elementwise_add(x=x, y=y) out = fluid.layers.layer_norm(input=elementwise_out) @@ -42,7 +39,7 @@ def setUp(self): self.num_fused_ops = 1 self.graph_attrs = { "embedding_eltwise_layernorm_fuse_pass_flag": True, - "multihead_matmul_fuse_pass_flag": True + "multihead_matmul_fuse_pass_flag": True, } def test_check_program(self): diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_subgraph_python_interface.py b/python/paddle/fluid/tests/unittests/ir/test_ir_subgraph_python_interface.py index 56f2e5e0a2450..bc9e63826cdff 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_subgraph_python_interface.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_subgraph_python_interface.py @@ -27,13 +27,11 @@ class TestQuantizationSubGraph(unittest.TestCase): - def build_graph_with_sub_graph(self): - def linear_fc(num): - data = fluid.layers.data(name='image', - shape=[1, 32, 32], - dtype='float32') + data = fluid.layers.data( + name='image', shape=[1, 32, 32], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') hidden = data for _ in range(num): @@ -63,7 +61,8 @@ def false_func(): graph = IrGraph(core_graph, for_test=True) sub_graph = graph.get_sub_graph(0) all_sub_graphs = graph.all_sub_graphs( - for_test=True) # same reason for subgraph + for_test=True + ) # same reason for subgraph # Should return graph and sub_graphs at the same time. If only return sub_graph, the graph will # be destructed and the sub_graphs will be empty. return graph, all_sub_graphs @@ -75,7 +74,8 @@ def test_quant_sub_graphs(self, use_cuda=False): scope=fluid.global_scope(), place=place, activation_quantize_type='abs_max', - weight_quantize_type='range_abs_max') + weight_quantize_type='range_abs_max', + ) Find_inserted_quant_op = False for sub_graph in sub_graphs: transform_pass.apply(sub_graph) @@ -87,8 +87,9 @@ def test_quant_sub_graphs(self, use_cuda=False): def test_quant_sub_graphs_cpu(self): self.test_quant_sub_graphs(use_cuda=False) - @OpTestTool.skip_if(not paddle.is_compiled_with_cuda(), - "Not GPU version paddle") + @OpTestTool.skip_if( + not paddle.is_compiled_with_cuda(), "Not GPU version paddle" + ) def test_quant_sub_graphs_gpu(self): self.test_quant_sub_graphs(use_cuda=True) diff --git a/python/paddle/fluid/tests/unittests/ir/test_ir_yolo_box_pass.py b/python/paddle/fluid/tests/unittests/ir/test_ir_yolo_box_pass.py index e86327f62f8cf..58bda7ab8384a 100644 --- a/python/paddle/fluid/tests/unittests/ir/test_ir_yolo_box_pass.py +++ b/python/paddle/fluid/tests/unittests/ir/test_ir_yolo_box_pass.py @@ -20,15 +20,17 @@ paddle.enable_static() -def multiclass_nms(bboxes, - scores, - score_threshold, - nms_top_k, - keep_top_k, - nms_threshold=0.3, - normalized=True, - nms_eta=1., - background_label=-1): +def multiclass_nms( + bboxes, + scores, + score_threshold, + nms_top_k, + keep_top_k, + nms_threshold=0.3, + normalized=True, + nms_eta=1.0, + background_label=-1, +): helper = LayerHelper('multiclass_nms3', **locals()) output = helper.create_variable_for_type_inference(dtype=bboxes.dtype) index = helper.create_variable_for_type_inference(dtype='int32') @@ -36,18 +38,20 @@ def multiclass_nms(bboxes, inputs = {'BBoxes': bboxes, 'Scores': scores} outputs = {'Out': output, 'Index': index, 'NmsRoisNum': nms_rois_num} - helper.append_op(type="multiclass_nms3", - inputs=inputs, - attrs={ - 'background_label': background_label, - 'score_threshold': score_threshold, - 'nms_top_k': nms_top_k, - 'nms_threshold': nms_threshold, - 'keep_top_k': keep_top_k, - 'nms_eta': nms_eta, - 'normalized': normalized - }, - outputs=outputs) + helper.append_op( + type="multiclass_nms3", + inputs=inputs, + attrs={ + 'background_label': background_label, + 'score_threshold': score_threshold, + 'nms_top_k': nms_top_k, + 'nms_threshold': nms_threshold, + 'keep_top_k': keep_top_k, + 'nms_eta': nms_eta, + 'normalized': normalized, + }, + outputs=outputs, + ) output.stop_gradient = True index.stop_gradient = True @@ -55,7 +59,6 @@ def multiclass_nms(bboxes, class TestYoloBoxPass(unittest.TestCase): - def test_yolo_box_pass(self): program = paddle.static.Program() with paddle.static.program_guard(program): @@ -67,18 +70,22 @@ def test_yolo_box_pass(self): div = paddle.divide(im_shape, im_scale) cast = paddle.cast(div, "int32") boxes0, scores0 = paddle.vision.ops.yolo_box( - yolo_box0_x, cast, [116, 90, 156, 198, 373, 326], 80, 0.005, 32) + yolo_box0_x, cast, [116, 90, 156, 198, 373, 326], 80, 0.005, 32 + ) boxes1, scores1 = paddle.vision.ops.yolo_box( - yolo_box1_x, cast, [30, 61, 62, 45, 59, 119], 80, 0.005, 16) + yolo_box1_x, cast, [30, 61, 62, 45, 59, 119], 80, 0.005, 16 + ) boxes2, scores2 = paddle.vision.ops.yolo_box( - yolo_box2_x, cast, [10, 13, 16, 30, 33, 23], 80, 0.005, 8) + yolo_box2_x, cast, [10, 13, 16, 30, 33, 23], 80, 0.005, 8 + ) transpose0 = paddle.transpose(scores0, [0, 2, 1]) transpose1 = paddle.transpose(scores1, [0, 2, 1]) transpose2 = paddle.transpose(scores2, [0, 2, 1]) concat0 = paddle.concat([boxes0, boxes1, boxes2], 1) concat1 = paddle.concat([transpose0, transpose1, transpose2], 2) - out0, out1, out2 = multiclass_nms(concat0, concat1, 0.01, 1000, 100, - 0.45, True, 1., 80) + out0, out1, out2 = multiclass_nms( + concat0, concat1, 0.01, 1000, 100, 0.45, True, 1.0, 80 + ) graph = core.Graph(program.desc) core.get_pass("yolo_box_fuse_pass").apply(graph) graph = paddle.fluid.framework.IrGraph(graph) diff --git a/python/paddle/fluid/tests/unittests/ir_memory_optimize_net_base.py b/python/paddle/fluid/tests/unittests/ir_memory_optimize_net_base.py index ca518747821b0..88c2024cff151 100644 --- a/python/paddle/fluid/tests/unittests/ir_memory_optimize_net_base.py +++ b/python/paddle/fluid/tests/unittests/ir_memory_optimize_net_base.py @@ -31,20 +31,22 @@ class BuildIrMemOptBase(unittest.TestCase): - def setup_reader(self): self.batch_size = 32 self.word_dict = paddle.dataset.imdb.word_dict() - self.train_reader = paddle.batch(paddle.dataset.imdb.train( - self.word_dict), - batch_size=self.batch_size) - - def check_network_convergence(self, - network, - use_cuda=True, - use_ir_memory_optimize=True, - enable_inplace=True, - iter=5): + self.train_reader = paddle.batch( + paddle.dataset.imdb.train(self.word_dict), + batch_size=self.batch_size, + ) + + def check_network_convergence( + self, + network, + use_cuda=True, + use_ir_memory_optimize=True, + enable_inplace=True, + iter=5, + ): if use_cuda and not core.is_compiled_with_cuda(): print('Skip use_cuda=True because Paddle is not compiled with cuda') return @@ -57,10 +59,9 @@ def check_network_convergence(self, fluid.default_startup_program().random_seed = 100 fluid.default_main_program().random_seed = 100 - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") @@ -79,8 +80,9 @@ def check_network_convergence(self, exe.run(fluid.default_startup_program()) train_cp = compiler.CompiledProgram(fluid.default_main_program()) - train_cp = train_cp.with_data_parallel(loss_name=cost.name, - build_strategy=build_strategy) + train_cp = train_cp.with_data_parallel( + loss_name=cost.name, build_strategy=build_strategy + ) fetch_list = [cost.name] begin = time.time() @@ -100,21 +102,23 @@ def check_network_convergence(self, break end = time.time() - print("%.4f Instance per second" % ((self.batch_size * iter) / - (end - begin))) + print( + "%.4f Instance per second" + % ((self.batch_size * iter) / (end - begin)) + ) print(first_loss, last_loss) avg_last_loss_val = np.array(last_loss).mean() avg_first_loss_val = np.array(first_loss).mean() if math.isnan(float(avg_last_loss_val)) or math.isnan( - float(avg_first_loss_val)): + float(avg_first_loss_val) + ): sys.exit("got NaN loss, training failed.") return first_loss, last_loss class TestIrMemOptBase(BuildIrMemOptBase): - def setUp(self): self.network = None @@ -126,15 +130,22 @@ def test_network(self): with fluid.program_guard(fluid.Program(), fluid.Program()): with fluid.scope_guard(core.Scope()): - baseline_first_loss, baseline_last_loss = self.check_network_convergence( - self.network) + ( + baseline_first_loss, + baseline_last_loss, + ) = self.check_network_convergence(self.network) cur_first_loss, cur_last_loss = self.check_network_convergence( - self.network) - - self.assertAlmostEquals(np.mean(baseline_last_loss), - np.mean(cur_last_loss), - delta=1e-6) - self.assertAlmostEquals(np.mean(baseline_first_loss), - np.mean(cur_first_loss), - delta=1e-6) + self.network + ) + + self.assertAlmostEquals( + np.mean(baseline_last_loss), + np.mean(cur_last_loss), + delta=1e-6, + ) + self.assertAlmostEquals( + np.mean(baseline_first_loss), + np.mean(cur_first_loss), + delta=1e-6, + ) diff --git a/python/paddle/fluid/tests/unittests/jit_load_rename_var.py b/python/paddle/fluid/tests/unittests/jit_load_rename_var.py index caaca0716cdef..2ba1e5522d1e9 100644 --- a/python/paddle/fluid/tests/unittests/jit_load_rename_var.py +++ b/python/paddle/fluid/tests/unittests/jit_load_rename_var.py @@ -32,8 +32,9 @@ def rename_var_with_generator(names_old): else: temp_name = "_".join(temp_name) name_new = _generate_unique_var_name_sync_with_main_program( - temp_name) - if name_new not in names_old[:var_idx] + names_old[var_idx + 1:]: + temp_name + ) + if name_new not in names_old[:var_idx] + names_old[var_idx + 1 :]: break dict_rename_var_old_new[name_old] = name_new return dict_rename_var_old_new diff --git a/python/paddle/fluid/tests/unittests/launch_function_helper.py b/python/paddle/fluid/tests/unittests/launch_function_helper.py index 992bc58b5e490..b8bb4cbc35ab1 100644 --- a/python/paddle/fluid/tests/unittests/launch_function_helper.py +++ b/python/paddle/fluid/tests/unittests/launch_function_helper.py @@ -59,7 +59,6 @@ def wait(procs, timeout=30): def _find_free_port(port_set): - def __free_port(): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: s.bind(('', 0)) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/check_flags_mkldnn_ops_on_off.py b/python/paddle/fluid/tests/unittests/mkldnn/check_flags_mkldnn_ops_on_off.py index 4042b4da63338..133320c7f375e 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/check_flags_mkldnn_ops_on_off.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/check_flags_mkldnn_ops_on_off.py @@ -23,15 +23,23 @@ def check(): - print("check: _global_flags()['FLAGS_use_mkldnn']=", - _global_flags()["FLAGS_use_mkldnn"]) - print("check: fluid.get_flags('FLAGS_use_mkldnn')=", - fluid.get_flags(['FLAGS_use_mkldnn'])) + print( + "check: _global_flags()['FLAGS_use_mkldnn']=", + _global_flags()["FLAGS_use_mkldnn"], + ) + print( + "check: fluid.get_flags('FLAGS_use_mkldnn')=", + fluid.get_flags(['FLAGS_use_mkldnn']), + ) print("check: DNNL_VERBOSE=", os.environ['DNNL_VERBOSE']) - print("check: FLAGS_tracer_mkldnn_ops_on=", - _global_flags()['FLAGS_tracer_mkldnn_ops_on']) - print("check: FLAGS_tracer_mkldnn_ops_off=", - _global_flags()['FLAGS_tracer_mkldnn_ops_off']) + print( + "check: FLAGS_tracer_mkldnn_ops_on=", + _global_flags()['FLAGS_tracer_mkldnn_ops_on'], + ) + print( + "check: FLAGS_tracer_mkldnn_ops_off=", + _global_flags()['FLAGS_tracer_mkldnn_ops_off'], + ) a_np = np.random.uniform(-2, 2, (10, 20, 30)).astype(np.float32) b_np = np.random.uniform(-5, 5, (10, 20, 30)).astype(np.float32) helper = LayerHelper(fluid.unique_name.generate(str("test")), act="relu") diff --git a/python/paddle/fluid/tests/unittests/mkldnn/check_flags_use_mkldnn.py b/python/paddle/fluid/tests/unittests/mkldnn/check_flags_use_mkldnn.py index 03786e0aed7fe..1bdca9244cc2b 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/check_flags_use_mkldnn.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/check_flags_use_mkldnn.py @@ -20,10 +20,14 @@ def check(): - print("check: _global_flags()['FLAGS_use_mkldnn']=", - _global_flags()["FLAGS_use_mkldnn"]) - print("check: fluid.get_flags('FLAGS_use_mkldnn')=", - fluid.get_flags(['FLAGS_use_mkldnn'])) + print( + "check: _global_flags()['FLAGS_use_mkldnn']=", + _global_flags()["FLAGS_use_mkldnn"], + ) + print( + "check: fluid.get_flags('FLAGS_use_mkldnn')=", + fluid.get_flags(['FLAGS_use_mkldnn']), + ) print("check: DNNL_VERBOSE=", os.environ['DNNL_VERBOSE']) a_np = np.random.uniform(-2, 2, (10, 20, 30)).astype(np.float32) helper = LayerHelper(fluid.unique_name.generate(str("test")), act="relu") @@ -32,7 +36,7 @@ def check(): a = fluid.dygraph.to_variable(a_np) res1 = func(a) res2 = np.maximum(a_np, 0) - assert (np.array_equal(res1.numpy(), res2)) + assert np.array_equal(res1.numpy(), res2) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mkldnn/mkldnn_op_test.py b/python/paddle/fluid/tests/unittests/mkldnn/mkldnn_op_test.py index 3b6788e6ab126..9d79ee3bf6933 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/mkldnn_op_test.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/mkldnn_op_test.py @@ -18,12 +18,14 @@ def __assert_close(test_case, tensor, np_array, msg, atol=1e-4): - test_case.assertTrue(np.allclose(np.array(tensor), np_array, atol=atol), - msg) + test_case.assertTrue( + np.allclose(np.array(tensor), np_array, atol=atol), msg + ) -def check_if_mkldnn_primitives_exist_in_bwd(test_case, op_type, x, out, - out_grad, x_grad): +def check_if_mkldnn_primitives_exist_in_bwd( + test_case, op_type, x, out, out_grad, x_grad +): place = core.CPUPlace() var_dict = {'x': x, 'out': out, 'out@GRAD': out_grad, 'x@GRAD': x_grad} @@ -34,20 +36,23 @@ def check_if_mkldnn_primitives_exist_in_bwd(test_case, op_type, x, out, with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype=np.float32, - shape=ground_truth[name].shape) + block.create_var( + name=name, dtype=np.float32, shape=ground_truth[name].shape + ) - op = block.append_op(type=op_type, - inputs={ - 'X': block.var('x'), - }, - outputs={'Out': block.var('out')}, - attrs={'use_mkldnn': True}) + op = block.append_op( + type=op_type, + inputs={ + 'X': block.var('x'), + }, + outputs={'Out': block.var('out')}, + attrs={'use_mkldnn': True}, + ) # Generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - op.desc, set(), []) + op.desc, set(), [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -65,29 +70,35 @@ def check_if_mkldnn_primitives_exist_in_bwd(test_case, op_type, x, out, for i in range(2): out = exe.run( program, - feed={name: var_dict[name] - for name in ['x', 'out@GRAD']}, - fetch_list=['x@GRAD', 'out']) + feed={name: var_dict[name] for name in ['x', 'out@GRAD']}, + fetch_list=['x@GRAD', 'out'], + ) __assert_close(test_case, x_grad, out[0], 'x@GRAD') -def check_if_mkldnn_batchnorm_primitives_exist_in_bwd(test_case, var_dict, - place, shape, - data_layout): +def check_if_mkldnn_batchnorm_primitives_exist_in_bwd( + test_case, var_dict, place, shape, data_layout +): var_names = [ - 'x', 'scale', 'bias', 'mean', 'variance', 'y', 'saved_mean', - 'saved_variance' + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + 'y', + 'saved_mean', + 'saved_variance', ] ground_truth = {name: var_dict[name] for name in var_names} program = fluid.Program() with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, dtype='float32', shape=ground_truth[name].shape + ) bn_op = block.append_op( type="batch_norm", inputs={ @@ -95,14 +106,14 @@ def check_if_mkldnn_batchnorm_primitives_exist_in_bwd(test_case, var_dict, "Scale": block.var('scale'), "Bias": block.var('bias'), "Mean": block.var('mean'), - "Variance": block.var('variance') + "Variance": block.var('variance'), }, outputs={ "Y": block.var('y'), "MeanOut": block.var('mean'), # share memory "VarianceOut": block.var('variance'), # share memory "SavedMean": block.var('saved_mean'), - "SavedVariance": block.var('saved_variance') + "SavedVariance": block.var('saved_variance'), }, attrs={ "momentum": test_case.momentum, @@ -111,15 +122,17 @@ def check_if_mkldnn_batchnorm_primitives_exist_in_bwd(test_case, var_dict, "data_layout": data_layout, "use_mkldnn": test_case.use_mkldnn, "fuse_with_relu": test_case.fuse_with_relu, - "use_global_stats": test_case.use_global_stats - }) - block.create_var(name='y@GRAD', - dtype='float32', - shape=var_dict['y'].shape) + "use_global_stats": test_case.use_global_stats, + }, + ) + block.create_var( + name='y@GRAD', dtype='float32', shape=var_dict['y'].shape + ) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - bn_op.desc, test_case.no_grad_set, []) + bn_op.desc, test_case.no_grad_set, [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -139,10 +152,17 @@ def check_if_mkldnn_batchnorm_primitives_exist_in_bwd(test_case, var_dict, program, feed={ name: var_dict[name] - for name in - ['x', 'scale', 'bias', 'mean', 'variance', 'y@GRAD'] + for name in [ + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + 'y@GRAD', + ] }, - fetch_list=test_case.fetch_list) + fetch_list=test_case.fetch_list, + ) for id, name in enumerate(test_case.fetch_list): __assert_close(test_case, var_dict[name], out[id], name) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_activation_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_activation_bf16_mkldnn_op.py index f7d6763bddc60..de9c8dcf49542 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_activation_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_activation_bf16_mkldnn_op.py @@ -17,14 +17,16 @@ import numpy as np from scipy.special import erf import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTestTool, + convert_float_to_uint16, +) from paddle.fluid.tests.unittests.test_activation_op import TestActivation from paddle.fluid.tests.unittests.test_gelu_op import gelu @OpTestTool.skip_if_not_cpu_bf16() class MKLDNNBF16ActivationOp(metaclass=abc.ABCMeta): - @abc.abstractmethod def config(self): pass @@ -62,14 +64,15 @@ def test_check_output(self): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", user_defined_grads=[self.dx], - user_defined_grad_outputs=[convert_float_to_uint16(self.out)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.out)], + ) class TestMKLDNNSigmoidBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "sigmoid" @@ -81,7 +84,6 @@ def op_grad(self, dout, x): class TestMKLDNNSqrtBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "sqrt" @@ -96,7 +98,6 @@ def op_grad(self, dout, x): class TestMKLDNNGeluErfBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "gelu" @@ -104,19 +105,19 @@ def op_forward(self, x): return gelu(x, False) def op_grad(self, dout, x): - return (dout * - (0.5 + 0.5 * erf(x / np.sqrt(2)) + - (x / np.sqrt(2 * np.pi) * np.exp(-0.5 * np.power(x, 2))))) + return dout * ( + 0.5 + + 0.5 * erf(x / np.sqrt(2)) + + (x / np.sqrt(2 * np.pi) * np.exp(-0.5 * np.power(x, 2))) + ) class TestMKLDNNGeluErfDim2BF16Op(TestMKLDNNGeluErfBF16Op): - def init_data(self): self.x = np.random.uniform(-1, 1, [11, 17]).astype(np.float32) class TestMKLDNNGeluTanhBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "gelu" @@ -125,23 +126,30 @@ def op_forward(self, x): def op_grad(self, dout, x): grad_part = np.tanh( - np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3))) - return dout * 0.5 * (1 + grad_part) * (1 + np.sqrt(2 / np.pi) * - (x + 0.134145 * np.power(x, 3)) * - (1 - grad_part)) + np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)) + ) + return ( + dout + * 0.5 + * (1 + grad_part) + * ( + 1 + + np.sqrt(2 / np.pi) + * (x + 0.134145 * np.power(x, 3)) + * (1 - grad_part) + ) + ) def set_attrs(self): self.attrs = {"use_mkldnn": True, "approximate": True} class TestMKLDNNGeluTanhDim2BF16Op(TestMKLDNNGeluTanhBF16Op): - def init_data(self): self.x = np.random.uniform(-1, 1, [11, 17]).astype(np.float32) class TestMKLDNNReluBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "relu" @@ -153,7 +161,6 @@ def op_grad(self, dout, x): class TestMKLDNNMishBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "mish" @@ -161,14 +168,17 @@ def op_forward(self, x): return x * np.tanh(np.log(1 + np.exp(x))) def op_grad(self, dout, x): - omega = np.exp( - 3 * x) + 4 * np.exp(2 * x) + np.exp(x) * (4 * x + 6) + 4 * (x + 1) + omega = ( + np.exp(3 * x) + + 4 * np.exp(2 * x) + + np.exp(x) * (4 * x + 6) + + 4 * (x + 1) + ) delta = np.exp(2 * x) + 2 * np.exp(x) + 2 return dout * ((np.exp(x) * omega) / delta**2) class TestMKLDNNRelu6BF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "relu6" @@ -180,7 +190,6 @@ def op_grad(self, dout, x): class TestMKLDNNLeakyReluBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "leaky_relu" @@ -196,7 +205,6 @@ def set_attrs(self): class TestMKLDNNSwishBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "swish" @@ -215,7 +223,6 @@ def set_attrs(self): class TestMKLDNNHardSwishBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "hard_swish" @@ -229,7 +236,6 @@ def op_grad(self, dout, x): class TestMKLDNNTanhBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "tanh" @@ -237,11 +243,10 @@ def op_forward(self, x): return np.tanh(x) def op_grad(self, dout, x): - return dout * (1 - np.tanh(x)**2) + return dout * (1 - np.tanh(x) ** 2) class TestMKLDNNAbsBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "abs" @@ -253,7 +258,6 @@ def op_grad(self, dout, x): class TestMKLDNNEluBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "elu" @@ -269,7 +273,6 @@ def set_attrs(self): class TestMKLDNNExpBF16Op(MKLDNNBF16ActivationOp, TestActivation): - def config(self): self.op_type = "exp" diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_activation_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_activation_mkldnn_op.py index 0ca003b45f2ea..12ab822d3e1c2 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_activation_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_activation_mkldnn_op.py @@ -17,13 +17,23 @@ from scipy.special import expit import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16 -from paddle.fluid.tests.unittests.test_activation_op import TestActivation, TestRelu, TestTanh, TestSqrt, TestAbs, TestLeakyRelu, TestSwish, TestHardSwish, TestRelu6, TestSigmoid +from paddle.fluid.tests.unittests.test_activation_op import ( + TestActivation, + TestRelu, + TestTanh, + TestSqrt, + TestAbs, + TestLeakyRelu, + TestSwish, + TestHardSwish, + TestRelu6, + TestSigmoid, +) from paddle.fluid.tests.unittests.test_gelu_op import gelu from mkldnn_op_test import check_if_mkldnn_primitives_exist_in_bwd class TestMKLDNNReluDim2(TestRelu): - def setUp(self): super(TestMKLDNNReluDim2, self).setUp() @@ -34,7 +44,6 @@ def init_dtype(self): class TestMKLDNNRelu6Dim2(TestRelu6): - def setUp(self): super(TestMKLDNNRelu6Dim2, self).setUp() self.attrs.update({"use_mkldnn": True}) @@ -44,7 +53,6 @@ def init_dtype(self): class TestMKLDNNLeakyReluDim2(TestLeakyRelu): - def setUp(self): super(TestMKLDNNLeakyReluDim2, self).setUp() @@ -55,7 +63,6 @@ def init_dtype(self): class TestMKLDNNGeluDim2(TestActivation): - def setUp(self): self.op_type = "gelu" self.dtype = np.float32 @@ -69,7 +76,6 @@ def setUp(self): class TestMKLDNNGeluDim2Approx(TestActivation): - def setUp(self): self.op_type = "gelu" self.dtype = np.float32 @@ -83,7 +89,6 @@ def setUp(self): class TestMKLDNNTanhDim2(TestTanh): - def setUp(self): super(TestMKLDNNTanhDim2, self).setUp() @@ -94,7 +99,6 @@ def init_dtype(self): class TestMKLDNNSqrtDim2(TestSqrt): - def setUp(self): super(TestMKLDNNSqrtDim2, self).setUp() @@ -105,7 +109,6 @@ def init_dtype(self): class TestMKLDNNAbsDim2(TestAbs): - def setUp(self): super(TestMKLDNNAbsDim2, self).setUp() self.attrs = {"use_mkldnn": True} @@ -115,7 +118,6 @@ def init_dtype(self): class TestMKLDNNSwishDim2(TestSwish): - def setUp(self): super(TestMKLDNNSwishDim2, self).setUp() @@ -127,21 +129,18 @@ def init_dtype(self): class TestMKLDNNHardSwishDim2(TestHardSwish): - def setUp(self): super(TestMKLDNNHardSwishDim2, self).setUp() self.attrs = {"use_mkldnn": True} class TestMKLDNNSigmoidDim2(TestSigmoid): - def setUp(self): super(TestMKLDNNSigmoidDim2, self).setUp() self.attrs = {"use_mkldnn": True} class TestMKLDNNReluDim4(TestRelu): - def setUp(self): super(TestMKLDNNReluDim4, self).setUp() @@ -159,7 +158,6 @@ def init_dtype(self): class TestMKLDNNLeakyReluDim4(TestLeakyRelu): - def setUp(self): super(TestMKLDNNLeakyReluDim4, self).setUp() @@ -177,7 +175,6 @@ def init_dtype(self): class TestMKLDNNGeluDim4(TestActivation): - def setUp(self): self.op_type = "gelu" self.dtype = np.float32 @@ -191,7 +188,6 @@ def setUp(self): class TestMKLDNNGeluDim4Approx(TestActivation): - def setUp(self): self.op_type = "gelu" self.dtype = np.float32 @@ -204,10 +200,10 @@ def setUp(self): self.attrs = {"use_mkldnn": True, "approximate": True} -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestMKLDNNGeluBf16Dim4(TestActivation): - def setUp(self): self.op_type = "gelu" self.dtype = np.uint16 @@ -226,10 +222,10 @@ def test_check_grad(self): pass -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestMKLDNNGeluBf16Dim4Approx(TestActivation): - def setUp(self): self.op_type = "gelu" self.dtype = np.uint16 @@ -249,7 +245,6 @@ def test_check_grad(self): class TestMKLDNNTanhDim4(TestTanh): - def setUp(self): super(TestMKLDNNTanhDim4, self).setUp() @@ -261,7 +256,6 @@ def setUp(self): class TestMKLDNNSqrtDim4(TestSqrt): - def setUp(self): super(TestMKLDNNSqrtDim4, self).setUp() @@ -273,7 +267,6 @@ def setUp(self): class TestMKLDNNAbsDim4(TestAbs): - def setUp(self): super(TestMKLDNNAbsDim4, self).setUp() @@ -289,7 +282,6 @@ def init_dtype(self): class TestMKLDNNSwishDim4(TestSwish): - def setUp(self): super(TestMKLDNNSwishDim4, self).setUp() @@ -311,12 +303,12 @@ def ref_hardswish(x, threshold=6.0, scale=6.0, offset=3.0): if x_dtype == 'float16': x_dtype = 'float16' x = x.astype('float32') - return (x * np.minimum(np.maximum(x + offset, 0.), threshold) / - scale).astype(x_dtype) + return ( + x * np.minimum(np.maximum(x + offset, 0.0), threshold) / scale + ).astype(x_dtype) class TestMKLDNNHardSwishDim4(TestHardSwish): - def setUp(self): super(TestMKLDNNHardSwishDim4, self).setUp() @@ -338,7 +330,6 @@ def init_dtype(self): class TestMKLDNNMish(TestActivation): - def setUp(self): self.op_type = "mish" self.dtype = np.float32 @@ -352,7 +343,6 @@ def setUp(self): class TestMKLDNNRound(TestActivation): - def setUp(self): self.op_type = "round" @@ -365,7 +355,6 @@ def setUp(self): class TestMKLDNNSigmoidDim4(TestSigmoid): - def setUp(self): super(TestMKLDNNSigmoidDim4, self).setUp() @@ -377,7 +366,6 @@ def setUp(self): class TestMKLDNNEluDefaultAlpha(TestActivation): - def setUp(self): self.op_type = "elu" self.set_alpha() @@ -387,8 +375,8 @@ def setUp(self): self.inputs = {'X': x} self.attrs = {'use_mkldnn': True, 'alpha': self.alpha} self.outputs = { - 'Out': - np.maximum(0, x) + np.minimum(0, self.alpha * (np.exp(x) - 1)) + 'Out': np.maximum(0, x) + + np.minimum(0, self.alpha * (np.exp(x) - 1)) } def set_alpha(self): @@ -396,13 +384,11 @@ def set_alpha(self): class TestMKLDNNEluCustomAlpha(TestMKLDNNEluDefaultAlpha): - def set_alpha(self): self.alpha = 2.5 class TestMKLDNNExpOp(TestActivation): - def setUp(self): self.op_type = "exp" x = np.random.random((5, 5, 4)).astype("float32") @@ -414,7 +400,6 @@ def setUp(self): # Check if primitives already exist in backward class TestMKLDNNAbsPrimitivesAlreadyExist(unittest.TestCase): - def setUp(self): super(TestMKLDNNAbsPrimitivesAlreadyExist, self).setUp() @@ -430,9 +415,9 @@ def __abs_bwd(self, x, out_grad): return out_grad * np.sign(x) def test_check(self): - check_if_mkldnn_primitives_exist_in_bwd(self, self.op_type, self.x, - self.out, self.out_grad, - self.x_grad) + check_if_mkldnn_primitives_exist_in_bwd( + self, self.op_type, self.x, self.out, self.out_grad, self.x_grad + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_batch_norm_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_batch_norm_mkldnn_op.py index f8c5ca3d1be89..f4190f0663458 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_batch_norm_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_batch_norm_mkldnn_op.py @@ -16,47 +16,69 @@ import numpy as np import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import _set_use_system_allocator -from paddle.fluid.tests.unittests.test_batch_norm_op import TestBatchNormOpInference, TestBatchNormOpTraining, _reference_training, _reference_grad +from paddle.fluid.tests.unittests.test_batch_norm_op import ( + TestBatchNormOpInference, + TestBatchNormOpTraining, + _reference_training, + _reference_grad, +) from mkldnn_op_test import check_if_mkldnn_batchnorm_primitives_exist_in_bwd _set_use_system_allocator(True) class TestMKLDNNBatchNormOpTraining(TestBatchNormOpTraining): - def init_kernel_type(self): self.use_mkldnn = True self.data_formats = ["NCHW"] - def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, - epsilon, momentum, shape, data_layout): + def ref_forward_backward( + self, + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ): if data_layout != "NCHW" and data_layout != "NHWC": raise ValueError("Unknown data order.") # run forward y, saved_mean, saved_variance = _reference_training( - x, scale, bias, epsilon, data_layout) - mean_out = saved_mean * (1. - momentum) + momentum * mean - variance_out = saved_variance * (1. - momentum) + momentum * variance + x, scale, bias, epsilon, data_layout + ) + mean_out = saved_mean * (1.0 - momentum) + momentum * mean + variance_out = saved_variance * (1.0 - momentum) + momentum * variance # run backward - x_grad, scale_grad, bias_grad = _reference_grad(x, y_grad, scale, - saved_mean, - saved_variance, epsilon, - data_layout) - - return y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad + x_grad, scale_grad, bias_grad = _reference_grad( + x, y_grad, scale, saved_mean, saved_variance, epsilon, data_layout + ) + + return ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) class TestMKLDNNBatchNormOpTraining_NHWC(TestMKLDNNBatchNormOpTraining): - def init_kernel_type(self): self.use_mkldnn = True self.data_formats = ["NHWC"] class TestMKLDNNBatchNormOpExistedPrimitives(TestMKLDNNBatchNormOpTraining): - def init_test_case(self): TestMKLDNNBatchNormOpTraining.init_test_case(self) self.fetch_list = ['y', 'x@GRAD'] @@ -74,20 +96,38 @@ def test_forward_backward(self): mean, variance = self.set_mean_variance(scale_shape, x, data_layout) y_grad = np.random.random_sample(shape).astype(np.float32) - y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad = self.ref_forward_backward( - x, y_grad, scale, bias, mean, variance, self.epsilon, self.momentum, - shape, data_layout) + ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) = self.ref_forward_backward( + x, + y_grad, + scale, + bias, + mean, + variance, + self.epsilon, + self.momentum, + shape, + data_layout, + ) var_dict = locals() var_dict['y@GRAD'] = y_grad var_dict['x@GRAD'] = x_grad var_dict['scale@GRAD'] = scale_grad var_dict['bias@GRAD'] = bias_grad check_if_mkldnn_batchnorm_primitives_exist_in_bwd( - self, var_dict, place, shape, data_layout) + self, var_dict, place, shape, data_layout + ) class TestMKLDNNBatchNormOpInference(TestBatchNormOpInference): - def init_kernel_type(self): self.use_mkldnn = True @@ -98,7 +138,6 @@ def test_check_output(self): class TestMKLDNNBatchNormOpInference_NHWC(TestMKLDNNBatchNormOpInference): - def test_check_output(self): place = core.CPUPlace() data_format = "NHWC" @@ -106,7 +145,6 @@ def test_check_output(self): class TestMKLDNNBatchNormOpWithReluInference(TestBatchNormOpInference): - def init_kernel_type(self): self.use_mkldnn = True self.fuse_with_relu = True @@ -119,5 +157,6 @@ def test_check_output(self): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_mkldnn_op.py index 2a91d6257300d..273b232ecf830 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_mkldnn_op.py @@ -19,12 +19,9 @@ from paddle.fluid.tests.unittests.op_test import skip_check_grad_ci -def bilinear_interp_mkldnn_np(input, - out_h, - out_w, - out_size=None, - actual_shape=None, - data_layout='NCHW'): +def bilinear_interp_mkldnn_np( + input, out_h, out_w, out_size=None, actual_shape=None, data_layout='NCHW' +): """bilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -54,10 +51,12 @@ def bilinear_interp_mkldnn_np(input, input_h1_w0 = input[:, :, h1, w0] input_h0_w1 = input[:, :, h0, w1] input_h1_w1 = input[:, :, h1, w1] - out[:, :, oh, - ow] = input_h0_w0 * (1 - Wh) * (1 - Ww) + input_h1_w0 * Wh * ( - 1 - Ww) + input_h0_w1 * (1 - - Wh) * Ww + input_h1_w1 * Wh * Ww + out[:, :, oh, ow] = ( + input_h0_w0 * (1 - Wh) * (1 - Ww) + + input_h1_w0 * Wh * (1 - Ww) + + input_h0_w1 * (1 - Wh) * Ww + + input_h1_w1 * Wh * Ww + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC @@ -67,7 +66,6 @@ def bilinear_interp_mkldnn_np(input, @skip_check_grad_ci(reason="Haven not implement interpolate grad kernel.") class TestBilinearInterpMKLDNNOp(OpTest): - def init_test_case(self): pass @@ -102,9 +100,14 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_mkldnn_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.data_layout) + output_np = bilinear_interp_mkldnn_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: @@ -117,7 +120,7 @@ def setUp(self): 'out_w': self.out_w, 'scale': self.scale, 'data_layout': self.data_layout, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.outputs = {'Out': output_np} @@ -126,7 +129,6 @@ def test_check_output(self): class TestBilinearInterpOpMKLDNNNHWC(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [3, 2, 32, 16] self.out_h = 27 @@ -136,73 +138,67 @@ def init_test_case(self): class TestBilinearNeighborInterpMKLDNNCase2(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 1. + self.scale = 1.0 class TestBilinearNeighborInterpDataLayout(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [2, 4, 4, 5] self.out_h = 6 self.out_w = 7 - self.scale = 0. + self.scale = 0.0 self.data_layout = "NHWC" class TestBilinearNeighborInterpCase3(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 128 - self.scale = 0. + self.scale = 0.0 class TestBilinearNeighborInterpCase4(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") class TestBilinearNeighborInterpCase5(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([13, 13]).astype("int32") class TestBilinearNeighborInterpCase6(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 129]).astype("int32") class TestBilinearNeighborInterpSame(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_v2_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_v2_mkldnn_op.py index 55f57764b12f3..e6cf8381fa28a 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_v2_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_bilinear_interp_v2_mkldnn_op.py @@ -19,12 +19,9 @@ from paddle.fluid.tests.unittests.op_test import skip_check_grad_ci -def bilinear_interp_mkldnn_np(input, - out_h, - out_w, - out_size=None, - actual_shape=None, - data_layout='NCHW'): +def bilinear_interp_mkldnn_np( + input, out_h, out_w, out_size=None, actual_shape=None, data_layout='NCHW' +): """bilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -54,10 +51,12 @@ def bilinear_interp_mkldnn_np(input, input_h1_w0 = input[:, :, h1, w0] input_h0_w1 = input[:, :, h0, w1] input_h1_w1 = input[:, :, h1, w1] - out[:, :, oh, - ow] = input_h0_w0 * (1 - Wh) * (1 - Ww) + input_h1_w0 * Wh * ( - 1 - Ww) + input_h0_w1 * (1 - - Wh) * Ww + input_h1_w1 * Wh * Ww + out[:, :, oh, ow] = ( + input_h0_w0 * (1 - Wh) * (1 - Ww) + + input_h1_w0 * Wh * (1 - Ww) + + input_h0_w1 * (1 - Wh) * Ww + + input_h1_w1 * Wh * Ww + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC @@ -67,7 +66,6 @@ def bilinear_interp_mkldnn_np(input, @skip_check_grad_ci(reason="Haven not implement interpolate grad kernel.") class TestBilinearInterpMKLDNNOp(OpTest): - def init_test_case(self): pass @@ -116,9 +114,14 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_mkldnn_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.data_layout) + output_np = bilinear_interp_mkldnn_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.data_layout, + ) if isinstance(self.scale, float): self.scale = [self.scale, self.scale] @@ -134,7 +137,7 @@ def setUp(self): 'out_w': self.out_w, 'scale': self.scale, 'data_layout': self.data_layout, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.outputs = {'Out': output_np} @@ -143,7 +146,6 @@ def test_check_output(self): class TestBilinearInterpOpMKLDNNNHWC(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [3, 2, 32, 16] self.out_h = 27 @@ -153,7 +155,6 @@ def init_test_case(self): class TestBilinearNeighborInterpMKLDNNCase2(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [3, 3, 9, 6] self.out_h = 12 @@ -161,7 +162,6 @@ def init_test_case(self): class TestBilinearNeighborInterpCase3(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 @@ -170,7 +170,6 @@ def init_test_case(self): class TestBilinearNeighborInterpCase4(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 @@ -180,7 +179,6 @@ def init_test_case(self): class TestBilinearNeighborInterpCase5(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 9, 6] self.out_h = 12 @@ -189,7 +187,6 @@ def init_test_case(self): class TestBilinearNeighborInterpCase6(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 @@ -199,7 +196,6 @@ def init_test_case(self): class TestBilinearNeighborInterpSame(TestBilinearInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [2, 3, 32, 64] self.out_h = 32 @@ -210,5 +206,6 @@ def init_test_case(self): if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_cast_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_cast_mkldnn_op.py index 8f7267d212ebd..92b98d35d02e9 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_cast_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_cast_mkldnn_op.py @@ -20,10 +20,10 @@ from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16 -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestCastBF16ToFP32MKLDNNOp(OpTest): - def init_data(self): self.out = np.random.random(size=[10, 10]).astype("float32") self.x = convert_float_to_uint16(self.out) @@ -32,12 +32,15 @@ def setUp(self): self.init_data() self.inputs = {'X': self.x} self.outputs = {'Out': self.out} - prepare_dtype = lambda x: int(core.VarDesc.VarType.BF16 if x.dtype != np - .float32 else core.VarDesc.VarType.FP32) + prepare_dtype = lambda x: int( + core.VarDesc.VarType.BF16 + if x.dtype != np.float32 + else core.VarDesc.VarType.FP32 + ) self.attrs = { 'in_dtype': prepare_dtype(self.x), 'out_dtype': prepare_dtype(self.out), - 'use_mkldnn': True + 'use_mkldnn': True, } self.op_type = 'cast' @@ -46,29 +49,28 @@ def test_check_output(self): def test_check_grad(self): self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", check_dygraph=False, user_defined_grads=[self.inputs['X']], - user_defined_grad_outputs=[self.outputs['Out']]) + user_defined_grad_outputs=[self.outputs['Out']], + ) class TestCastFP32ToBF16MKLDNNOp(TestCastBF16ToFP32MKLDNNOp): - def init_data(self): self.x = np.random.random(size=[2, 6]).astype("float32") self.out = convert_float_to_uint16(self.x) class TestCastBF16ToBF16MKLDNNOp(TestCastBF16ToFP32MKLDNNOp): - def init_data(self): self.x = np.random.random(size=[6, 13]).astype("uint16") self.out = self.x class TestCastFP32ToFP32MKLDNNOp(TestCastBF16ToFP32MKLDNNOp): - def init_data(self): self.x = np.random.random(size=[7, 15]).astype("float32") self.out = self.x diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_clip_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_clip_mkldnn_op.py index b9a9119e5adfa..012c7cb018bbb 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_clip_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_clip_mkldnn_op.py @@ -14,14 +14,17 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) import paddle import paddle.fluid.core as core @OpTestTool.skip_if_not_cpu_bf16() class TestClipOneDNNOp(OpTest): - def setUp(self): self.op_type = "clip" self.set_inputs() @@ -29,10 +32,16 @@ def setUp(self): self.set_additional_inputs() self.adjust_op_settings() - self.min = self.attrs[ - 'min'] if 'Min' not in self.inputs else self.inputs['Min'] - self.max = self.attrs[ - 'max'] if 'Max' not in self.inputs else self.inputs['Max'] + self.min = ( + self.attrs['min'] + if 'Min' not in self.inputs + else self.inputs['Min'] + ) + self.max = ( + self.attrs['max'] + if 'Max' not in self.inputs + else self.inputs['Max'] + ) self.outputs = {'Out': np.clip(self.x_fp32, self.min, self.max)} @@ -57,19 +66,16 @@ def test_check_grad(self): class TestClipMinAsInputOneDNNOp(TestClipOneDNNOp): - def set_additional_inputs(self): self.inputs['Min'] = np.array([6.8]).astype('float32') class TestClipMaxAsInputOneDNNOp(TestClipOneDNNOp): - def set_additional_inputs(self): self.inputs['Max'] = np.array([9.1]).astype('float32') class TestClipMaxAndMinAsInputsOneDNNOp(TestClipOneDNNOp): - def set_additional_inputs(self): self.inputs['Max'] = np.array([8.5]).astype('float32') self.inputs['Min'] = np.array([7.1]).astype('float32') @@ -77,10 +83,8 @@ def set_additional_inputs(self): # BF16 TESTS def create_bf16_test_class(parent): - @OpTestTool.skip_if_not_cpu_bf16() class TestClipBF16OneDNNOp(parent): - def set_inputs(self): self.x_fp32 = np.random.random((10, 10)).astype(np.float32) * 25 self.inputs = {'X': convert_float_to_uint16(self.x_fp32)} @@ -95,8 +99,10 @@ def calculate_grads(self): for i in range(self.dx.shape[0]): for j in range(self.dx.shape[1]): - if self.x_fp32[j][i] > self.min and self.x_fp32[j][ - i] < self.max: + if ( + self.x_fp32[j][i] > self.min + and self.x_fp32[j][i] < self.max + ): self.dx[j][i] = self.dout[j][i] def test_check_output(self): @@ -105,10 +111,12 @@ def test_check_output(self): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", user_defined_grads=[self.dx], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) cls_name = "{0}_{1}".format(parent.__name__, "BF16") TestClipBF16OneDNNOp.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_concat_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_concat_bf16_mkldnn_op.py index f10b04bdb5be2..eefb3c766abec 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_concat_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_concat_bf16_mkldnn_op.py @@ -20,10 +20,10 @@ from paddle import enable_static -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestConcatBf16Op(OpTest): - def setUp(self): self.op_type = "concat" self.use_mkldnn = True @@ -35,14 +35,15 @@ def setUp(self): self.attrs = { 'axis': self.axis, 'use_mkldnn': True, - 'mkldnn_data_type': self.mkldnn_data_type + 'mkldnn_data_type': self.mkldnn_data_type, } self.sections = [self.x0.shape[self.axis]] * 2 self.sections[1] += self.x1.shape[self.axis] - self.output = np.concatenate((self.x0, self.x1, self.x2), - axis=self.axis).astype(np.uint16) + self.output = np.concatenate( + (self.x0, self.x1, self.x2), axis=self.axis + ).astype(np.uint16) self.outputs = {'Out': self.output} def calculate_grads(self): @@ -55,20 +56,25 @@ def test_check_output(self): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["x0", "x1", "x2"], + core.CPUPlace(), + ["x0", "x1", "x2"], "Out", user_defined_grads=[self.dxs[0], self.dxs[1], self.dxs[2]], - user_defined_grad_outputs=[self.dout]) + user_defined_grad_outputs=[self.dout], + ) -# --------------------test concat bf16 in with axis 0-------------------- + # --------------------test concat bf16 in with axis 0-------------------- def init_test_data(self): self.x0 = convert_float_to_uint16( - np.random.random(self.x0_shape).astype(np.float32)) + np.random.random(self.x0_shape).astype(np.float32) + ) self.x1 = convert_float_to_uint16( - np.random.random(self.x1_shape).astype(np.float32)) + np.random.random(self.x1_shape).astype(np.float32) + ) self.x2 = convert_float_to_uint16( - np.random.random(self.x2_shape).astype(np.float32)) + np.random.random(self.x2_shape).astype(np.float32) + ) def init_axis(self): self.axis = 0 @@ -83,7 +89,6 @@ def init_shape(self): class TestAxis1Case(TestConcatBf16Op): - def init_axis(self): self.axis = 1 @@ -97,7 +102,6 @@ def init_shape(self): class TestAxis2Case(TestConcatBf16Op): - def init_axis(self): self.axis = 2 @@ -111,7 +115,6 @@ def init_shape(self): class TestAxis3Case(TestConcatBf16Op): - def init_axis(self): self.axis = 3 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_concat_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_concat_int8_mkldnn_op.py index da07bce944726..5d275c7fb434f 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_concat_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_concat_int8_mkldnn_op.py @@ -18,7 +18,6 @@ class TestConcatOp(OpTest): - def setUp(self): self.op_type = "concat" self.use_mkldnn = True @@ -29,8 +28,9 @@ def setUp(self): self.inputs = {'X': [('x0', self.x0), ('x1', self.x1), ('x2', self.x2)]} self.attrs = {'axis': self.axis, 'use_mkldnn': True} - self.output = np.concatenate((self.x0, self.x1, self.x2), - axis=self.axis).astype('int') + self.output = np.concatenate( + (self.x0, self.x1, self.x2), axis=self.axis + ).astype('int') self.outputs = {'Out': self.output} @@ -38,7 +38,7 @@ def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode self.check_output(check_dygraph=False) -#--------------------test concat s8 in with axis 0-------------------- + # --------------------test concat s8 in with axis 0-------------------- def init_test_data(self): self.x0 = (np.random.randint(0, 100, self.x0_shape) - 50).astype('int8') @@ -54,11 +54,10 @@ def init_shape(self): self.x2_shape = [3, 2, 1, 2] -#--------------------test concat u8 in with axis 0-------------------- +# --------------------test concat u8 in with axis 0-------------------- class TestConcatOp2(TestConcatOp): - def init_test_data(self): self.x0 = (np.random.randint(0, 100, self.x0_shape)).astype('uint8') self.x1 = (np.random.randint(0, 50, self.x1_shape)).astype('uint8') @@ -75,10 +74,9 @@ def init_shape(self): def create_test_int8_class(parent): - #--------------------test concat s8/u8 in with axis 1-------------------- + # --------------------test concat s8/u8 in with axis 1-------------------- class TestAxis1Case(parent): - def init_axis(self): self.axis = 1 @@ -87,10 +85,9 @@ def init_shape(self): self.x1_shape = [1, 2, 5, 5] self.x2_shape = [1, 3, 5, 5] -#--------------------test concat s8/u8 in with axis 2-------------------- + # --------------------test concat s8/u8 in with axis 2-------------------- class TestAxis2Case(parent): - def init_axis(self): self.axis = 2 @@ -99,11 +96,9 @@ def init_shape(self): self.x1_shape = [2, 3, 5, 5] self.x2_shape = [2, 3, 6, 5] - -#--------------------test concat s8/u8 in with axis 3-------------------- + # --------------------test concat s8/u8 in with axis 3-------------------- class TestAxis3Case(parent): - def init_axis(self): self.axis = 3 @@ -122,10 +117,12 @@ def init_shape(self): globals()[cls_name_2] = TestAxis2Case globals()[cls_name_3] = TestAxis3Case + create_test_int8_class(TestConcatOp) create_test_int8_class(TestConcatOp2) if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_concat_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_concat_mkldnn_op.py index 1788923e20b2e..549c9620be8f7 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_concat_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_concat_mkldnn_op.py @@ -21,7 +21,6 @@ class TestConcatAxis0OneDNNOp(OpTest): - def setUp(self): self.op_type = "concat" self.mkldnn_data_type = "float32" @@ -33,11 +32,12 @@ def setUp(self): self.attrs = { 'axis': self.axis, 'use_mkldnn': True, - 'mkldnn_data_type': self.mkldnn_data_type + 'mkldnn_data_type': self.mkldnn_data_type, } - self.output = np.concatenate((self.x0, self.x1, self.x2), - axis=self.axis).astype(self.dtype) + self.output = np.concatenate( + (self.x0, self.x1, self.x2), axis=self.axis + ).astype(self.dtype) self.outputs = {'Out': self.output} @@ -68,7 +68,6 @@ def init_shape(self): class TestConcatAxis1OneDNNOp(TestConcatAxis0OneDNNOp): - def init_axis(self): self.axis = 1 @@ -79,7 +78,6 @@ def init_shape(self): class TestConcatAxis2OneDNNOp(TestConcatAxis0OneDNNOp): - def init_axis(self): self.axis = 2 @@ -90,7 +88,6 @@ def init_shape(self): class TestConcatAxis3OneDNNOp(TestConcatAxis0OneDNNOp): - def init_axis(self): self.axis = 3 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_bf16_mkldnn_op.py index e4f23f195d2f6..80657bad6317d 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_bf16_mkldnn_op.py @@ -16,8 +16,15 @@ import numpy as np import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16, OpTestTool -from paddle.fluid.tests.unittests.test_conv2d_op import conv2d_forward_naive, TestConv2DOp +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + convert_float_to_uint16, + OpTestTool, +) +from paddle.fluid.tests.unittests.test_conv2d_op import ( + conv2d_forward_naive, + TestConv2DOp, +) def conv2d_residual_naive(out, residual): @@ -26,10 +33,10 @@ def conv2d_residual_naive(out, residual): return out -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestConv2DBF16Op(TestConv2DOp): - def setUp(self): self.op_type = "conv2d" self.use_cudnn = False @@ -53,7 +60,7 @@ def setUp(self): self.conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } self.input = np.random.random(self.input_size).astype(np.float32) @@ -61,16 +68,18 @@ def setUp(self): self.inputs_fp32 = {'Input': self.input, 'Filter': self.filter} - conv_out, _, _, _, _ = conv2d_forward_naive(self.input, self.filter, - self.groups, - self.conv2d_param) + conv_out, _, _, _, _ = conv2d_forward_naive( + self.input, self.filter, self.groups, self.conv2d_param + ) self.conv_output_float = conv_out if self.fuse_residual: self.input_residual = np.random.random( - self.input_residual_size).astype(np.float32) + self.input_residual_size + ).astype(np.float32) self.conv_output_float = conv2d_residual_naive( - self.conv_output_float, self.input_residual) + self.conv_output_float, self.input_residual + ) self.conv_output = convert_float_to_uint16(self.conv_output_float) self.outputs = {'Output': self.conv_output} elif self.force_fp32_output: @@ -87,15 +96,16 @@ def setUp(self): self.filter = convert_float_to_uint16(self.filter) self.inputs = { - 'Input': - self.input, - 'Filter': - OpTest.np_dtype_to_fluid_dtype(self.filter.astype(self.weight_type)) + 'Input': self.input, + 'Filter': OpTest.np_dtype_to_fluid_dtype( + self.filter.astype(self.weight_type) + ), } if self.fuse_residual: self.inputs['ResidualData'] = OpTest.np_dtype_to_fluid_dtype( - convert_float_to_uint16(self.input_residual)) + convert_float_to_uint16(self.input_residual) + ) self.attrs = { 'strides': self.stride, @@ -106,7 +116,7 @@ def setUp(self): 'use_mkldnn': self.use_mkldnn, 'mkldnn_data_type': self.mkldnn_data_type, 'force_fp32_output': self.force_fp32_output, - 'fuse_residual_connection': self.fuse_residual + 'fuse_residual_connection': self.fuse_residual, } self.init_additional_attrs() @@ -156,7 +166,6 @@ def init_additional_attrs(self): @OpTestTool.skip_if_not_cpu_bf16() class TestConv2DWithGradBF16Op(TestConv2DBF16Op): - def init_fuse_relu(self): self.fuse_activation = None @@ -177,10 +186,12 @@ def test_check_grad(self): dx, dweights = conv_backward(dout, x, w, self.conv2d_param) self.check_grad_with_place( - core.CPUPlace(), ["Input", "Filter"], + core.CPUPlace(), + ["Input", "Filter"], "Output", user_defined_grads=[dx, dweights], - user_defined_grad_outputs=[convert_float_to_uint16(dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(dout)], + ) def test_check_grad_no_filter(self): dout = self.conv_output_float @@ -190,11 +201,13 @@ def test_check_grad_no_filter(self): dx, _ = conv_backward(dout, x, w, self.conv2d_param) self.check_grad_with_place( - core.CPUPlace(), ["Input"], + core.CPUPlace(), + ["Input"], "Output", set(['Filter']), user_defined_grads=[dx], - user_defined_grad_outputs=[convert_float_to_uint16(dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(dout)], + ) def test_check_grad_no_input(self): dout = self.conv_output_float @@ -204,11 +217,13 @@ def test_check_grad_no_input(self): _, dweights = conv_backward(dout, x, w, self.conv2d_param) self.check_grad_with_place( - core.CPUPlace(), ["Filter"], + core.CPUPlace(), + ["Filter"], "Output", set(['Input']), user_defined_grads=[dweights], - user_defined_grad_outputs=[convert_float_to_uint16(dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(dout)], + ) def conv_backward(dout, x, w, params): @@ -224,7 +239,7 @@ def conv_backward(dout, x, w, params): H_out = int(1 + (H + 2 * padding - KH) / stride[0]) W_out = int(1 + (W + 2 * padding - KW) / stride[1]) - x_padded = np.pad(x, ((0, ), (0, ), (padding, ), (padding, )), 'constant') + x_padded = np.pad(x, ((0,), (0,), (padding,), (padding,)), 'constant') for n in range(N): for oc in range(OC): @@ -233,11 +248,17 @@ def conv_backward(dout, x, w, params): for k in range(H_out): for l in range(W_out): for ic in range(IC): - dweights[oc, ic, i, j] += x_padded[ - n, ic, i + k * stride[0], - j + l * stride[1]] * dout[n, oc, k, l] - - dx_padded = np.pad(dx, ((0, ), (0, ), (padding, ), (padding, )), 'constant') + dweights[oc, ic, i, j] += ( + x_padded[ + n, + ic, + i + k * stride[0], + j + l * stride[1], + ] + * dout[n, oc, k, l] + ) + + dx_padded = np.pad(dx, ((0,), (0,), (padding,), (padding,)), 'constant') w_ = np.zeros_like(w) for i in range(KH): @@ -251,10 +272,14 @@ def conv_backward(dout, x, w, params): for kh in range(KH): for kw in range(KW): for ic in range(IC): - dx_padded[n, ic, stride[0] * i + kh, - stride[1] * j + - kw] += dout[n, oc, i, j] * w[oc, ic, - kh, kw] + dx_padded[ + n, + ic, + stride[0] * i + kh, + stride[1] * j + kw, + ] += ( + dout[n, oc, i, j] * w[oc, ic, kh, kw] + ) if padding == 0: dx = dx_padded @@ -265,21 +290,18 @@ def conv_backward(dout, x, w, params): class TestConv2DBF16WithPadding1(TestConv2DWithGradBF16Op): - def init_test_case(self): TestConv2DWithGradBF16Op.init_test_case(self) self.pad = [1, 1] class TestConv2DBF16WithStride2(TestConv2DWithGradBF16Op): - def init_test_case(self): TestConv2DWithGradBF16Op.init_test_case(self) self.stride = [2, 3] class TestConv2D(TestConv2DBF16Op): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -294,7 +316,6 @@ def init_data_type(self): class TestWithPad(TestConv2D): - def init_test_case(self): TestConv2D.init_test_case(self) self.pad = [1, 1] @@ -302,13 +323,11 @@ def init_test_case(self): class TestWithGroup(TestConv2D): - def init_group(self): self.groups = 3 class TestWithStride(TestConv2DBF16Op): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -323,7 +342,6 @@ def init_data_type(self): class TestWithDilations(TestConv2DBF16Op): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -339,7 +357,6 @@ def init_data_type(self): class TestWith1x1ForceFP32Output(TestConv2DBF16Op): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -356,7 +373,6 @@ def init_fuse_residual(self): class TestWithInput1x1Filter1x1(TestConv2DBF16Op): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -372,5 +388,6 @@ def init_group(self): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_int8_mkldnn_op.py index 4ec61b0bb0b8a..c4113c12c4cbb 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_int8_mkldnn_op.py @@ -17,7 +17,10 @@ import os import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import OpTest -from paddle.fluid.tests.unittests.test_conv2d_op import conv2d_forward_naive, TestConv2DOp +from paddle.fluid.tests.unittests.test_conv2d_op import ( + conv2d_forward_naive, + TestConv2DOp, +) def conv2d_forward_refer(input, filter, group, conv_param): @@ -25,10 +28,10 @@ def conv2d_forward_refer(input, filter, group, conv_param): return out -@unittest.skipIf(not core.supports_int8(), - "place does not support int8 computation") +@unittest.skipIf( + not core.supports_int8(), "place does not support int8 computation" +) class TestConv2DInt8Op(TestConv2DOp): - def setUp(self): self.op_type = "conv2d" self.use_cudnn = False @@ -49,85 +52,109 @@ def setUp(self): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } # This implementation of convolution quantization is based on OneDNN documentation # https://oneapi-src.github.io/oneDNN/dev_guide_int8_computations.html#doxid-dev-guide-int8-computations-1dg-i8-comp-s11 - inner_scale = 1. if self.fuse_activation != "" else self.scale_out - activation_scale = self.scale_out if self.fuse_activation != "" else 1. - scale_output_shift = (inner_scale / - (self.scale_in * self.scale_weights[0])) + inner_scale = 1.0 if self.fuse_activation != "" else self.scale_out + activation_scale = self.scale_out if self.fuse_activation != "" else 1.0 + scale_output_shift = inner_scale / ( + self.scale_in * self.scale_weights[0] + ) filter = np.random.random(self.filter_size).astype(self.weighttype) # When the Intel AVX2 or Intel AVX512 Instruction Set is used # the reorder additionally scales the weights by 0.5 # to overcome the potential overflow issue. If the processor supports VNNI instructions, # modification of the weights is not necessary. - avx_scale = 0.5 if not core.supports_vnni( - ) and self.srctype == np.int8 else 1. - filter_int = np.round(filter * self.scale_weights[0] * - avx_scale).astype(np.int32) + avx_scale = ( + 0.5 if not core.supports_vnni() and self.srctype == np.int8 else 1.0 + ) + filter_int = np.round( + filter * self.scale_weights[0] * avx_scale + ).astype(np.int32) scale_output_shift = scale_output_shift / avx_scale def conv2d_forward_refer_helper(input_): - return conv2d_forward_refer(input_.astype(np.int32), filter_int, - self.groups, conv2d_param).astype( - np.float32) * scale_output_shift + return ( + conv2d_forward_refer( + input_.astype(np.int32), + filter_int, + self.groups, + conv2d_param, + ).astype(np.float32) + * scale_output_shift + ) def residual_helper(init_low, init_high, output_): input_residual_ = np.random.randint( - init_low, init_high, - self.input_residual_size).astype(self.srctype) - return (output_ + input_residual_ * - (inner_scale / self.scale_in_eltwise)), input_residual_ + init_low, init_high, self.input_residual_size + ).astype(self.srctype) + return ( + output_ + + input_residual_ * (inner_scale / self.scale_in_eltwise) + ), input_residual_ if self.srctype == np.int8: init_low, init_high = (-5, 5) - input = np.random.randint(init_low, init_high, - self.input_size).astype(self.srctype) + input = np.random.randint( + init_low, init_high, self.input_size + ).astype(self.srctype) input_shift = (np.ones(self.input_size) * 128).astype(np.uint8) output1 = conv2d_forward_refer_helper( - np.round(input + input_shift).astype(np.int32)) + np.round(input + input_shift).astype(np.int32) + ) output2 = conv2d_forward_refer_helper( - np.round(input_shift).astype(np.int32)) + np.round(input_shift).astype(np.int32) + ) output = output1 - output2 else: init_low, init_high = (0, 10) - input = np.random.randint(init_low, init_high, - self.input_size).astype(self.srctype) + input = np.random.randint( + init_low, init_high, self.input_size + ).astype(self.srctype) output = conv2d_forward_refer_helper(input) if self.fuse_residual: - output, input_residual = residual_helper(init_low, init_high, - output) + output, input_residual = residual_helper( + init_low, init_high, output + ) if self.fuse_activation == "": pass elif self.fuse_activation == "relu": output = activation_scale * np.maximum(output, 0) elif self.fuse_activation == "hard_swish": - output = activation_scale * output / 6. * np.minimum( - np.maximum(0, output + 3.), 6) + output = ( + activation_scale + * output + / 6.0 + * np.minimum(np.maximum(0, output + 3.0), 6) + ) elif self.fuse_activation == "relu6": output = activation_scale * np.maximum(0, np.minimum(6, output)) elif self.fuse_activation == "swish": - output = activation_scale * output / (1. + np.exp(-1. * output)) + output = activation_scale * output / (1.0 + np.exp(-1.0 * output)) elif self.fuse_activation == "leaky_relu": output = activation_scale * np.maximum(output, 0.02 * output) else: - raise NotImplementedError("test for " + self.fuse_activation + - " activation not implemented") + raise NotImplementedError( + "test for " + + self.fuse_activation + + " activation not implemented" + ) output = np.round(output).astype(self.dsttype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input.astype(self.srctype)), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } if self.fuse_residual: self.inputs['ResidualData'] = OpTest.np_dtype_to_fluid_dtype( - input_residual) + input_residual + ) self.attrs = { 'strides': self.stride, @@ -146,15 +173,15 @@ def residual_helper(init_low, init_high, output_): 'fuse_alpha': self.fuse_alpha, 'fuse_beta': self.fuse_beta, 'fuse_residual_connection': self.fuse_residual, - 'mkldnn_data_type': self.mkldnn_data_type + 'mkldnn_data_type': self.mkldnn_data_type, } self.outputs = {'Output': output} def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output_with_place(core.CPUPlace(), - atol=0, - check_dygraph=False) + self.check_output_with_place( + core.CPUPlace(), atol=0, check_dygraph=False + ) def test_check_grad(self): pass @@ -193,7 +220,6 @@ def init_fuse_residual(self): class TestConv2D(TestConv2DInt8Op): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -209,7 +235,6 @@ def init_test_case(self): class TestWithHardSwish(TestConv2D): - def init_fuse_activation(self): self.fuse_activation = "hard_swish" self.fuse_alpha = 0 @@ -217,7 +242,6 @@ def init_fuse_activation(self): class TestWithRelu6(TestConv2D): - def init_fuse_activation(self): self.fuse_activation = "relu6" self.fuse_alpha = 6 @@ -225,7 +249,6 @@ def init_fuse_activation(self): class TestWithSwish(TestConv2D): - def init_fuse_activation(self): self.fuse_activation = "swish" self.fuse_alpha = 1 @@ -233,7 +256,6 @@ def init_fuse_activation(self): class TestWithLeakyRelu(TestConv2D): - def init_fuse_activation(self): self.fuse_activation = "leaky_relu" self.fuse_alpha = 0.02 @@ -241,7 +263,6 @@ def init_fuse_activation(self): class TestWithPad(TestConv2D): - def init_test_case(self): TestConv2D.init_test_case(self) self.pad = [1, 1] @@ -249,13 +270,11 @@ def init_test_case(self): class TestWithGroup(TestConv2D): - def init_group(self): self.groups = 3 class TestWithStride(TestConv2DInt8Op): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -271,7 +290,6 @@ def init_test_case(self): class TestWithDilations(TestConv2DInt8Op): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -288,7 +306,6 @@ def init_test_case(self): class TestWith1x1(TestConv2DInt8Op): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -304,7 +321,6 @@ def init_test_case(self): class TestWithInput1x1Filter1x1(TestConv2DInt8Op): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -335,37 +351,31 @@ def create_test_int8_class(parent): # --------------------test conv2d s8 in and u8 out-------------------- class TestS8U8Case(parent): - def init_data_type(self): init_data_type_with_fusion(self, np.int8, "relu", False) # --------------------test conv2d s8 in and s8 out-------------------- class TestS8S8Case(parent): - def init_data_type(self): init_data_type_with_fusion(self, np.int8, "", False) # --------------------test conv2d u8 in and s8 out-------------------- class TestU8S8Case(parent): - def init_data_type(self): init_data_type_with_fusion(self, np.uint8, "", False) # --------------------test conv2d u8 in and u8 out without residual fuse-------------------- class TestU8U8Case(parent): - def init_data_type(self): init_data_type_with_fusion(self, np.uint8, "relu", False) # --------------------test conv2d s8 in and s8 out with residual fuse-------------------- class TestS8S8ResCase(parent): - def init_data_type(self): init_data_type_with_fusion(self, np.int8, "", True) # --------------------test conv2d u8 in and s8 out with residual fuse-------------------- class TestU8S8ResCase(parent): - def init_data_type(self): init_data_type_with_fusion(self, np.uint8, "", True) @@ -375,9 +385,11 @@ def init_data_type(self): cls_name_u8u8 = "{0}_relu_{1}_residual_0".format(parent.__name__, "1") cls_name_s8s8_re_1 = "{0}_relu_{1}_residual_{2}".format( - parent.__name__, "0", "1") + parent.__name__, "0", "1" + ) cls_name_u8s8_re_1 = "{0}_relu_{1}_residual_{2}".format( - parent.__name__, "0", "1") + parent.__name__, "0", "1" + ) TestS8U8Case.__name__ = cls_name_s8u8 TestS8S8Case.__name__ = cls_name_s8s8 TestU8S8Case.__name__ = cls_name_u8s8 @@ -395,12 +407,12 @@ def init_data_type(self): if os.name != 'nt': # --------------------test conv2d s8 in and u8 out with residual fuse-------------------- class TestS8U8ResCase(parent): - def init_data_type(self): init_data_type_with_fusion(self, np.int8, "relu", True) cls_name_s8u8_re_1 = "{0}_relu_{1}_residual_{2}".format( - parent.__name__, "1", "1") + parent.__name__, "1", "1" + ) TestS8U8ResCase.__name__ = cls_name_s8u8_re_1 globals()[cls_name_s8u8_re_1] = TestS8U8ResCase @@ -415,7 +427,6 @@ def init_data_type(self): class TestConv2DOp_AsyPadding_INT_MKLDNN(TestConv2DInt8Op): - def init_kernel_type(self): self.use_mkldnn = True @@ -425,14 +436,12 @@ def init_paddings(self): class TestConv2DOp_Same_INT_MKLDNN(TestConv2DOp_AsyPadding_INT_MKLDNN): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" class TestConv2DOp_Valid_INT_MKLDNN(TestConv2DOp_AsyPadding_INT_MKLDNN): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" @@ -440,5 +449,6 @@ def init_paddings(self): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_mkldnn_op.py index d3e9bdde1a689..65147ea6c2bdd 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_mkldnn_op.py @@ -16,7 +16,10 @@ import numpy as np from paddle.fluid.tests.unittests.op_test import OpTest, skip_check_grad_ci -from paddle.fluid.tests.unittests.test_conv2d_op import TestConv2DOp, TestConv2DOp_v2 +from paddle.fluid.tests.unittests.test_conv2d_op import ( + TestConv2DOp, + TestConv2DOp_v2, +) def conv2d_bias_naive(out, bias): @@ -34,7 +37,6 @@ def conv2d_residual_naive(out, residual): class TestConv2DMKLDNNOp(TestConv2DOp): - def init_group(self): self.groups = 1 @@ -65,7 +67,7 @@ def setUp(self): output = self.outputs['Output'] - #mkldnn only support either conv-sum-relu, or conv-relu. + # mkldnn only support either conv-sum-relu, or conv-relu. if self.fuse_bias and self.bias_size is not None: bias = np.random.random(self.bias_size).astype(self.dtype) output = conv2d_bias_naive(output, bias) @@ -73,22 +75,29 @@ def setUp(self): self.attrs['fuse_bias'] = self.fuse_bias self.inputs['Bias'] = OpTest.np_dtype_to_fluid_dtype(bias) - if self.fuse_residual_connection and self.input_residual_size is not None: + if ( + self.fuse_residual_connection + and self.input_residual_size is not None + ): input_residual = np.random.random(self.input_residual_size).astype( - self.dtype) + self.dtype + ) output = conv2d_residual_naive(output, input_residual) self.attrs[ - 'fuse_residual_connection'] = self.fuse_residual_connection + 'fuse_residual_connection' + ] = self.fuse_residual_connection self.inputs['ResidualData'] = OpTest.np_dtype_to_fluid_dtype( - input_residual) + input_residual + ) if self.fuse_activation == "relu": output = np.maximum(output, 0).astype(self.dsttype) if self.fuse_activation == "relu6": - output = np.minimum(np.maximum(output, 0), - self.fuse_alpha).astype(self.dsttype) + output = np.minimum(np.maximum(output, 0), self.fuse_alpha).astype( + self.dsttype + ) output = output.astype(self.dtype) self.attrs['fuse_bias'] = self.fuse_bias @@ -101,9 +110,9 @@ def setUp(self): @skip_check_grad_ci( - reason="Fusion is for inference only, check_grad is not required.") + reason="Fusion is for inference only, check_grad is not required." +) class TestWithbreluFusion(TestConv2DMKLDNNOp): - def init_test_case(self): TestConv2DMKLDNNOp.init_test_case(self) self.fuse_activation = "relu6" @@ -112,9 +121,9 @@ def init_test_case(self): @skip_check_grad_ci( - reason="Fusion is for inference only, check_grad is not required.") + reason="Fusion is for inference only, check_grad is not required." +) class TestWithFuse(TestConv2DMKLDNNOp): - def init_test_case(self): TestConv2DMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -125,7 +134,6 @@ def init_test_case(self): class TestWithPadWithBias(TestConv2DMKLDNNOp): - def init_test_case(self): TestConv2DMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -133,7 +141,6 @@ def init_test_case(self): class TestWithStride(TestConv2DMKLDNNOp): - def init_test_case(self): TestConv2DMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -142,7 +149,6 @@ def init_test_case(self): class TestWithGroup(TestConv2DMKLDNNOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -156,14 +162,12 @@ def init_group(self): class TestWith1x1(TestConv2DMKLDNNOp): - def init_test_case(self): TestConv2DMKLDNNOp.init_test_case(self) self.filter_size = [40, 3, 1, 1] class TestWithInput1x1Filter1x1(TestConv2DMKLDNNOp): - def init_test_case(self): TestConv2DMKLDNNOp.init_test_case(self) self.input_size = [2, 60, 1, 1] # NCHW @@ -176,7 +180,6 @@ def init_group(self): class TestConv2DOp_AsyPadding_MKLDNN(TestConv2DOp_v2): - def init_kernel_type(self): self.use_mkldnn = True self.dtype = np.float32 @@ -187,21 +190,18 @@ def init_paddings(self): class TestConv2DOp_Same_MKLDNN(TestConv2DOp_AsyPadding_MKLDNN): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" class TestConv2DOp_Valid_MKLDNN(TestConv2DOp_AsyPadding_MKLDNN): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" class TestConv2DOp_Valid_NHWC_MKLDNN(TestConv2DOp_Valid_MKLDNN): - def init_data_format(self): self.data_format = "NHWC" @@ -211,21 +211,18 @@ def init_test_case_2(self): class TestConv2DOp_Same_NHWC_MKLDNN(TestConv2DOp_Valid_NHWC_MKLDNN): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" class TestConv2DOp_AsyPadding_NHWC_MKLDNN(TestConv2DOp_Valid_NHWC_MKLDNN): - def init_paddings(self): self.pad = [0, 0, 1, 2] self.padding_algorithm = "EXPLICIT" class TestMKLDNNDilations(TestConv2DMKLDNNOp): - def init_test_case(self): TestConv2DMKLDNNOp.init_test_case(self) self.pad = [0, 0] @@ -244,5 +241,6 @@ def init_group(self): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_bf16_mkldnn_op.py index e9f4f5a325198..e605dc6bf9551 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_bf16_mkldnn_op.py @@ -17,7 +17,9 @@ import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16 -from paddle.fluid.tests.unittests.test_conv2d_transpose_op import conv2dtranspose_forward_naive +from paddle.fluid.tests.unittests.test_conv2d_transpose_op import ( + conv2dtranspose_forward_naive, +) from paddle import enable_static @@ -29,10 +31,10 @@ def conv2d_bias_naive(out, bias): return out -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestConv2DTransposeBF16MKLDNNOp(OpTest): - def test_check_output(self): self.check_output_with_place(core.CPUPlace()) @@ -95,7 +97,7 @@ def setUp(self): 'data_format': self.data_format, 'fuse_activation': self.fuse_activation, 'fuse_alpha': self.fuse_alpha, - 'fuse_beta': self.fuse_beta + 'fuse_beta': self.fuse_beta, } if self.output_size is not None: self.attrs['output_size'] = self.output_size @@ -103,15 +105,16 @@ def setUp(self): if len(self.output_padding) > 0: self.attrs['output_padding'] = self.output_padding - output = conv2dtranspose_forward_naive(input, filter, - self.attrs).astype(np.float32) + output = conv2dtranspose_forward_naive( + input, filter, self.attrs + ).astype(np.float32) if self.input_type is not np.float32: input = convert_float_to_uint16(input) self.inputs = { 'Input': input.view(self.input_type), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } if self.fuse_bias and self.bias_size is not None: @@ -132,7 +135,6 @@ def setUp(self): class TestMKLDNNFuseBias(TestConv2DTransposeBF16MKLDNNOp): - def init_test_case(self): super(TestMKLDNNFuseBias, self).init_test_case() self.pad = [1, 1] @@ -141,7 +143,6 @@ def init_test_case(self): class TestMKLDNNWithPad(TestConv2DTransposeBF16MKLDNNOp): - def init_test_case(self): super(TestMKLDNNWithPad, self).init_test_case() self.pad = [1, 1] @@ -149,7 +150,6 @@ def init_test_case(self): class TestMKLDNNWithStride(TestConv2DTransposeBF16MKLDNNOp): - def init_test_case(self): super(TestMKLDNNWithStride, self).init_test_case() self.pad = [1, 1] @@ -158,7 +158,6 @@ def init_test_case(self): class TestMKLDNNWithAsymPad(TestConv2DTransposeBF16MKLDNNOp): - def init_test_case(self): super(TestMKLDNNWithAsymPad, self).init_test_case() self.pad = [0, 0, 1, 2] @@ -166,7 +165,6 @@ def init_test_case(self): class TestMKLDNNWithSamePad(TestConv2DTransposeBF16MKLDNNOp): - def init_test_case(self): super(TestMKLDNNWithSamePad, self).init_test_case() self.pad = [0, 0] @@ -174,7 +172,6 @@ def init_test_case(self): class TestMKLDNNWithValidPad(TestConv2DTransposeBF16MKLDNNOp): - def init_test_case(self): super(TestMKLDNNWithValidPad, self).init_test_case() self.pad = [1, 1] @@ -182,7 +179,6 @@ def init_test_case(self): class TestMKLDNNWithValidPad_NHWC(TestMKLDNNWithValidPad): - def init_test_case(self): super(TestMKLDNNWithValidPad_NHWC, self).init_test_case() self.data_format = 'NHWC' @@ -191,11 +187,12 @@ def init_test_case(self): class TestConv2DTransposeMKLDNNWithDilationsExplicitPad( - TestConv2DTransposeBF16MKLDNNOp): - + TestConv2DTransposeBF16MKLDNNOp +): def init_test_case(self): - super(TestConv2DTransposeMKLDNNWithDilationsExplicitPad, - self).init_test_case() + super( + TestConv2DTransposeMKLDNNWithDilationsExplicitPad, self + ).init_test_case() self.stride = [2, 1] self.dilations = [1, 2] self.groups = 1 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_mkldnn_op.py index 081f70a02e335..14e42fc50ce78 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_conv2d_transpose_mkldnn_op.py @@ -16,7 +16,9 @@ import numpy as np from paddle.fluid.tests.unittests.op_test import OpTest from paddle import enable_static -from paddle.fluid.tests.unittests.test_conv2d_transpose_op import TestConv2DTransposeOp +from paddle.fluid.tests.unittests.test_conv2d_transpose_op import ( + TestConv2DTransposeOp, +) def conv2d_bias_naive(out, bias): @@ -28,7 +30,6 @@ def conv2d_bias_naive(out, bias): class TestConv2DTransposeMKLDNNOp(TestConv2DTransposeOp): - def test_check_grad(self): return @@ -87,7 +88,6 @@ def setUp(self): class TestMKLDNNFuseBias(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -96,7 +96,6 @@ def init_test_case(self): class TestMKLDNNWithPad(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -104,7 +103,6 @@ def init_test_case(self): class TestMKLDNNWithStride(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -113,7 +111,6 @@ def init_test_case(self): class TestMKLDNNWithAsymPad(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [0, 0, 1, 2] @@ -121,7 +118,6 @@ def init_test_case(self): class TestMKLDNNWithSamePad(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [0, 0] @@ -129,7 +125,6 @@ def init_test_case(self): class TestMKLDNNWithValidPad(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -137,7 +132,6 @@ def init_test_case(self): class TestMKLDNNWithValidPad_NHWC(TestMKLDNNWithValidPad): - def init_test_case(self): super(TestMKLDNNWithValidPad_NHWC, self).init_test_case() self.data_format = "NHWC" @@ -146,8 +140,8 @@ def init_test_case(self): class TestConv2DTransposeMKLDNNWithDilationsExplicitPad( - TestConv2DTransposeMKLDNNOp): - + TestConv2DTransposeMKLDNNOp +): def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.stride = [2, 1] @@ -161,7 +155,6 @@ def init_test_case(self): class TestMKLDNNWithGroups(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [1, 1] @@ -172,7 +165,6 @@ def init_test_case(self): class TestMKLDNNWithGroups_NHWC(TestConv2DTransposeMKLDNNOp): - def init_test_case(self): TestConv2DTransposeMKLDNNOp.init_test_case(self) self.pad = [1, 1] diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_conv3d_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_conv3d_mkldnn_op.py index 6e31912a909b7..30561ec8f0f81 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_conv3d_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_conv3d_mkldnn_op.py @@ -14,11 +14,17 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.test_conv3d_op import TestCase1, TestConv3DOp, TestWith1x1, TestWithGroup1, TestWithGroup2, TestWithInput1x1Filter1x1 +from paddle.fluid.tests.unittests.test_conv3d_op import ( + TestCase1, + TestConv3DOp, + TestWith1x1, + TestWithGroup1, + TestWithGroup2, + TestWithInput1x1Filter1x1, +) class TestMKLDNN(TestConv3DOp): - def init_kernel_type(self): self.use_mkldnn = True self.data_format = "NCHW" @@ -26,7 +32,6 @@ def init_kernel_type(self): class TestMKLDNNCase1(TestCase1): - def init_kernel_type(self): self.use_mkldnn = True self.data_format = "NCHW" @@ -34,7 +39,6 @@ def init_kernel_type(self): class TestMKLDNNGroup1(TestWithGroup1): - def init_kernel_type(self): self.use_mkldnn = True self.data_format = "NCHW" @@ -42,7 +46,6 @@ def init_kernel_type(self): class TestMKLDNNGroup2(TestWithGroup2): - def init_kernel_type(self): self.use_mkldnn = True self.data_format = "NCHW" @@ -50,7 +53,6 @@ def init_kernel_type(self): class TestMKLDNNWith1x1(TestWith1x1): - def init_kernel_type(self): self.use_mkldnn = True self.data_format = "NCHW" @@ -58,7 +60,6 @@ def init_kernel_type(self): class TestMKLDNNWithInput1x1Filter1x1(TestWithInput1x1Filter1x1): - def init_kernel_type(self): self.use_mkldnn = True self.data_format = "NCHW" @@ -66,7 +67,6 @@ def init_kernel_type(self): class TestConv3DOp_AsyPadding_MKLDNN(TestConv3DOp): - def init_kernel_type(self): self.use_mkldnn = True self.data_format = "NCHW" @@ -78,7 +78,6 @@ def init_paddings(self): class TestConv3DOp_Same_MKLDNN(TestConv3DOp_AsyPadding_MKLDNN): - def init_paddings(self): self.pad = [0, 0, 0] self.padding_algorithm = "SAME" @@ -90,7 +89,6 @@ def init_kernel_type(self): class TestConv3DOp_Valid_MKLDNN(TestConv3DOp_AsyPadding_MKLDNN): - def init_paddings(self): self.pad = [1, 1, 1] self.padding_algorithm = "VALID" @@ -103,5 +101,6 @@ def init_kernel_type(self): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_dequantize_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_dequantize_mkldnn_op.py index 70a1fd823941b..2e3a460959967 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_dequantize_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_dequantize_mkldnn_op.py @@ -19,7 +19,6 @@ class TestDeQuantizeOp(OpTest): - def setUp(self): self.op_type = 'dequantize' self.scale = 127.0 @@ -45,19 +44,22 @@ def prepare_input_output_bf16(self): def prepare_input_int8(self): if self.data_type == 'int8': # input data values are integers from interval [-128, 128) - self.input = (np.random.randint(0, 256, self.input_size) - - 128).astype(self.data_type) + self.input = ( + np.random.randint(0, 256, self.input_size) - 128 + ).astype(self.data_type) else: # input data values are integers from interval [0, 256) self.input = (np.random.randint(0, 256, self.input_size)).astype( - self.data_type) + self.data_type + ) self.inputs = {'Input': OpTest.np_dtype_to_fluid_dtype(self.input)} self.attrs = {'Scale': self.scale, 'Shift': self.shift} def prepare_output_int8(self): - output = (self.input / self.scale - - (self.shift / self.scale)).astype('float') + output = (self.input / self.scale - (self.shift / self.scale)).astype( + 'float' + ) self.outputs = {'Output': output} def test_check_output(self): @@ -87,7 +89,6 @@ def set_input_size(self): class TestDeQuantizeOp1(TestDeQuantizeOp): - def set_scale(self): self.scale = 1.5 @@ -96,7 +97,6 @@ def set_data_type(self): class TestDeQuantizeOp2(TestDeQuantizeOp): - def set_scale(self): self.scale = 0.8 @@ -105,7 +105,6 @@ def set_data_type(self): class TestDeQuantizeOpBf16(TestDeQuantizeOp): - def set_scale(self): self.scale = 1.0 @@ -116,7 +115,6 @@ def set_data_type(self): # 2-dim input # P - positive input, with shift class TestDeQuantizeOpShift_2_P(TestDeQuantizeOp): - def set_data_type(self): self.data_type = 'uint8' @@ -133,7 +131,6 @@ def set_input_size(self): # 2-dim input # N - negative input, with shift class TestDeQuantizeOpShift_2_N(TestDeQuantizeOpShift_2_P): - def set_data_type(self): self.data_type = 'int8' @@ -149,26 +146,22 @@ def set_input_size(self): # 3-dim input class TestDeQuantizeOpShift_3_P(TestDeQuantizeOpShift_2_P): - def set_input_size(self): self.input_size = [2, 3, 4] class TestDeQuantizeOpShift_3_N(TestDeQuantizeOpShift_2_N): - def set_input_size(self): self.input_size = [2, 3, 4] # 4-dim input class TestDeQuantizeOpShift_4_P(TestDeQuantizeOpShift_2_P): - def set_input_size(self): self.input_size = [2, 3, 4, 5] class TestDeQuantizeOpShift_4_N(TestDeQuantizeOpShift_2_N): - def set_input_size(self): self.input_size = [2, 3, 4, 5] diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_bf16_mkldnn_op.py index 6511b1a153886..8e2ae04bb7c8f 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_bf16_mkldnn_op.py @@ -19,10 +19,10 @@ from paddle import enable_static -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestElementwiseAddBf16MklDNNOp(OpTest): - def setUp(self): self.op_type = "elementwise_add" self.use_mkldnn = True @@ -38,8 +38,12 @@ def setUp(self): self.outputs = {'Out': convert_float_to_uint16(self.out)} def generate_data(self): - self.x = np.random.random(100, ).astype(np.float32) - self.y = np.random.random(100, ).astype(np.float32) + self.x = np.random.random( + 100, + ).astype(np.float32) + self.y = np.random.random( + 100, + ).astype(np.float32) self.out = np.add(self.x, self.y) def test_check_output(self): @@ -47,30 +51,39 @@ def test_check_output(self): # elementwise_add grad (no braodcasting) is just passing upper gradients to either X or Y or both def test_check_grad_normal(self): - self.check_grad_with_place(core.CPUPlace(), ["X", "Y"], - "Out", - check_dygraph=False, - user_defined_grads=[self.x, self.x], - user_defined_grad_outputs=[self.x_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["X", "Y"], + "Out", + check_dygraph=False, + user_defined_grads=[self.x, self.x], + user_defined_grad_outputs=[self.x_bf16], + ) def test_check_grad_ingore_x(self): - self.check_grad_with_place(core.CPUPlace(), ["Y"], - "Out", - check_dygraph=False, - user_defined_grads=[self.y], - user_defined_grad_outputs=[self.y_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["Y"], + "Out", + check_dygraph=False, + user_defined_grads=[self.y], + user_defined_grad_outputs=[self.y_bf16], + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(core.CPUPlace(), ["X"], - "Out", - check_dygraph=False, - user_defined_grads=[self.x], - user_defined_grad_outputs=[self.x_bf16]) - + self.check_grad_with_place( + core.CPUPlace(), + ["X"], + "Out", + check_dygraph=False, + user_defined_grads=[self.x], + user_defined_grad_outputs=[self.x_bf16], + ) -class TestElementwiseAddBroadCastingBf16MklDNNOp(TestElementwiseAddBf16MklDNNOp - ): +class TestElementwiseAddBroadCastingBf16MklDNNOp( + TestElementwiseAddBf16MklDNNOp +): def generate_data(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(np.float32) self.y = np.random.uniform(1, 2, [100]).astype(np.float32) @@ -85,20 +98,23 @@ def compute_reduced_gradients(self, out_grads): def test_check_grad_normal(self): self.check_grad_with_place( - core.CPUPlace(), ["X", "Y"], + core.CPUPlace(), + ["X", "Y"], "Out", check_dygraph=False, - user_defined_grads=[self.x, - self.compute_reduced_gradients(self.x)], - user_defined_grad_outputs=[self.x_bf16]) + user_defined_grads=[self.x, self.compute_reduced_gradients(self.x)], + user_defined_grad_outputs=[self.x_bf16], + ) def test_check_grad_ingore_x(self): self.check_grad_with_place( - core.CPUPlace(), ["Y"], + core.CPUPlace(), + ["Y"], "Out", check_dygraph=False, user_defined_grads=[self.compute_reduced_gradients(self.x)], - user_defined_grad_outputs=[self.x_bf16]) + user_defined_grad_outputs=[self.x_bf16], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_mkldnn_op.py index 2b89d3c4b10da..4001e2ba76ba7 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_add_mkldnn_op.py @@ -15,12 +15,13 @@ import unittest import numpy as np from paddle.fluid.tests.unittests.op_test import skip_check_grad_ci -from paddle.fluid.tests.unittests.test_elementwise_add_op import TestElementwiseAddOp +from paddle.fluid.tests.unittests.test_elementwise_add_op import ( + TestElementwiseAddOp, +) from paddle import enable_static class TestMKLDNNElementwiseAddOp(TestElementwiseAddOp): - def init_kernel_type(self): self.use_mkldnn = True @@ -29,15 +30,13 @@ def init_dtype(self): class TestMKLDNNElementwiseAddOp2(TestMKLDNNElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestMKLDNNElementwiseAddOp3(TestMKLDNNElementwiseAddOp): - def init_input_output(self): self.x = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) self.y = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) @@ -45,7 +44,6 @@ def init_input_output(self): class TestMKLDNNElementwiseAddOp4(TestMKLDNNElementwiseAddOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 32]).astype(self.dtype) self.y = np.random.uniform(1, 2, [4, 32]).astype(self.dtype) @@ -60,7 +58,6 @@ def test_check_grad_ingore_y(self): class TestMKLDNNElementwiseAddOp5(TestMKLDNNElementwiseAddOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(self.dtype) self.y = np.random.uniform(1, 2, [100]).astype(self.dtype) @@ -68,7 +65,6 @@ def init_input_output(self): class TestMKLDNNElementwiseAddOpBroadcastXintoY(TestMKLDNNElementwiseAddOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 50, 1]).astype(self.dtype) self.y = np.random.uniform(1, 2, [2, 50, 160]).astype(self.dtype) @@ -76,7 +72,6 @@ def init_input_output(self): class TestMKLDNNElementwiseAddOp_broadcast_3(TestMKLDNNElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -87,7 +82,6 @@ def init_axis(self): class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestMKLDNNElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 12).astype(self.dtype) self.y = np.random.rand(2, 2, 10, 12).astype(self.dtype) @@ -111,9 +105,9 @@ def test_check_grad_ingore_x(self): @skip_check_grad_ci( - reason="oneDNN's int8 elementwise_ops don't implemend grad kernel.") + reason="oneDNN's int8 elementwise_ops don't implemend grad kernel." +) class TestInt8(TestElementwiseAddOp): - def init_kernel_type(self): self.use_mkldnn = True self._cpu_only = True @@ -147,7 +141,6 @@ def test_check_grad_ingore_y(self): class TestInt8Scales(TestInt8): - def quantize(self, tensor, dt="int8"): max_int = 127.0 if dt == "int8" else 255.0 scale = max_int / np.abs(np.amax(tensor)) @@ -155,8 +148,8 @@ def quantize(self, tensor, dt="int8"): return scale, quantized def init_input_output(self): - self.x_f = np.random.random((100, )).astype("float") - self.y_f = np.random.random((100, )).astype("float") + self.x_f = np.random.random((100,)).astype("float") + self.y_f = np.random.random((100,)).astype("float") self.out_f = np.add(self.x_f, self.y_f) self.scale_x, self.x = self.quantize(self.x_f) @@ -172,15 +165,15 @@ def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode self.init_scales() int_atol = 1 # different quantization techniques - self.check_output(check_dygraph=(self.use_mkldnn == False), - atol=int_atol) + self.check_output( + check_dygraph=(self.use_mkldnn == False), atol=int_atol + ) class TestUint8Scales(TestInt8Scales): - def init_input_output(self): - self.x_f = np.random.random((100, )).astype("float") - self.y_f = np.random.random((100, )).astype("float") + self.x_f = np.random.random((100,)).astype("float") + self.y_f = np.random.random((100,)).astype("float") self.out_f = np.add(self.x_f, self.y_f) self.scale_x, self.x = self.quantize(self.x_f, "uint8") diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_div_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_div_mkldnn_op.py index 9312e9459037a..8adeb32dc9f6c 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_div_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_div_mkldnn_op.py @@ -15,15 +15,20 @@ import unittest import numpy as np from paddle import enable_static -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) from paddle.fluid.framework import _current_expected_place import paddle.fluid.core as core -@OpTestTool.skip_if(not (isinstance(_current_expected_place(), core.CPUPlace)), - "GPU is not supported") +@OpTestTool.skip_if( + not (isinstance(_current_expected_place(), core.CPUPlace)), + "GPU is not supported", +) class TestMKLDNNElementwiseDivOp(OpTest): - def setUp(self): self.op_type = "elementwise_div" self.init_dtype() @@ -32,7 +37,7 @@ def setUp(self): self.init_axis() self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -65,7 +70,6 @@ def test_check_output(self): class TestMKLDNNElementwiseDivOp2(TestMKLDNNElementwiseDivOp): - def init_input_output(self): self.x = np.random.uniform(0.1, 1, [100]).astype(self.dtype) self.y = np.random.uniform(0.1, 1, [100]).astype(self.dtype) @@ -73,7 +77,6 @@ def init_input_output(self): class TestMKLDNNElementwiseDivOp3(TestMKLDNNElementwiseDivOp): - def init_input_output(self): self.x = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) self.y = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) @@ -81,7 +84,6 @@ def init_input_output(self): class TestMKLDNNElementwiseDivOp4(TestMKLDNNElementwiseDivOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 32]).astype(self.dtype) self.y = np.random.uniform(1, 2, [4, 32]).astype(self.dtype) @@ -96,7 +98,6 @@ def test_check_grad_ignore_x(self): class TestMKLDNNElementwiseDivOp5(TestMKLDNNElementwiseDivOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(self.dtype) self.y = np.random.uniform(1, 2, [100]).astype(self.dtype) @@ -112,7 +113,6 @@ def test_check_grad_ignore_x(self): @OpTestTool.skip_if_not_cpu_bf16() class TestBf16(TestMKLDNNElementwiseDivOp): - def setUp(self): self.op_type = "elementwise_div" self.init_dtype() @@ -139,34 +139,43 @@ def test_check_output(self): self.check_output_with_place(core.CPUPlace()) def test_check_grad_normal(self): - self.check_grad_with_place(core.CPUPlace(), ["X", "Y"], - "Out", - user_defined_grads=[ - np.divide(self.x, self.y), - np.divide((np.multiply(-self.x, self.x)), - np.multiply(self.y, self.y)) - ], - user_defined_grad_outputs=[self.x_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["X", "Y"], + "Out", + user_defined_grads=[ + np.divide(self.x, self.y), + np.divide( + (np.multiply(-self.x, self.x)), np.multiply(self.y, self.y) + ), + ], + user_defined_grad_outputs=[self.x_bf16], + ) def test_check_grad_ignore_x(self): - self.check_grad_with_place(core.CPUPlace(), ["Y"], - "Out", - user_defined_grads=[ - np.divide((np.multiply(-self.x, self.y)), - np.multiply(self.y, self.y)) - ], - user_defined_grad_outputs=[self.y_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["Y"], + "Out", + user_defined_grads=[ + np.divide( + (np.multiply(-self.x, self.y)), np.multiply(self.y, self.y) + ) + ], + user_defined_grad_outputs=[self.y_bf16], + ) def test_check_grad_ignore_y(self): self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", user_defined_grads=[np.divide(self.x, self.y)], - user_defined_grad_outputs=[self.x_bf16]) + user_defined_grad_outputs=[self.x_bf16], + ) class TestBf16Broadcasting(TestBf16): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(self.dtype) self.y = np.random.uniform(1, 2, [100]).astype(self.dtype) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_bf16_mkldnn_op.py index 26531d826b5be..d7e31b061552e 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_bf16_mkldnn_op.py @@ -19,10 +19,10 @@ from paddle import enable_static -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestElementwiseMulBf16MklDNNOp(OpTest): - def setUp(self): self.op_type = "elementwise_mul" self.use_mkldnn = True @@ -37,43 +37,54 @@ def setUp(self): self.outputs = {'Out': convert_float_to_uint16(self.out)} def generate_data(self): - self.x = np.random.random(100, ).astype(np.float32) - self.y = np.random.random(100, ).astype(np.float32) + self.x = np.random.random( + 100, + ).astype(np.float32) + self.y = np.random.random( + 100, + ).astype(np.float32) self.out = np.multiply(self.x, self.y) def test_check_output(self): self.check_output_with_place(core.CPUPlace()) def test_check_grad_normal(self): - self.check_grad_with_place(core.CPUPlace(), ["X", "Y"], - "Out", - check_dygraph=False, - user_defined_grads=[ - np.multiply(self.x, self.y), - np.multiply(self.x, self.x) - ], - user_defined_grad_outputs=[self.x_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["X", "Y"], + "Out", + check_dygraph=False, + user_defined_grads=[ + np.multiply(self.x, self.y), + np.multiply(self.x, self.x), + ], + user_defined_grad_outputs=[self.x_bf16], + ) def test_check_grad_ingore_x(self): self.check_grad_with_place( - core.CPUPlace(), ["Y"], + core.CPUPlace(), + ["Y"], "Out", check_dygraph=False, user_defined_grads=[np.multiply(self.y, self.x)], - user_defined_grad_outputs=[self.y_bf16]) + user_defined_grad_outputs=[self.y_bf16], + ) def test_check_grad_ingore_y(self): self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", check_dygraph=False, user_defined_grads=[np.multiply(self.x, self.y)], - user_defined_grad_outputs=[self.x_bf16]) - + user_defined_grad_outputs=[self.x_bf16], + ) -class TestElementwiseMulBroadcastingBf16MklDNNOp(TestElementwiseMulBf16MklDNNOp - ): +class TestElementwiseMulBroadcastingBf16MklDNNOp( + TestElementwiseMulBf16MklDNNOp +): def generate_data(self): self.x = np.random.uniform(1, 2, [1, 2, 3, 100]).astype(np.float32) self.y = np.random.uniform(1, 2, [100]).astype(np.float32) @@ -90,7 +101,7 @@ def compute_reduced_gradients(self, out_grads): # accuracy problems that need to be explained def test_check_grad_normal(self): pass - #self.check_grad_with_place( + # self.check_grad_with_place( # core.CPUPlace(), ["X", "Y"], # "Out", # check_dy_graph=False, @@ -102,7 +113,7 @@ def test_check_grad_normal(self): def test_check_grad_ingore_x(self): pass - #self.check_grad_with_place( + # self.check_grad_with_place( # core.CPUPlace(), ["Y"], # "Out", # check_dy_graph=False, diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_mkldnn_op.py index 7f3bd429eb388..3ca09093b814d 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_mul_mkldnn_op.py @@ -15,12 +15,13 @@ import unittest import numpy as np from paddle.fluid.tests.unittests.op_test import skip_check_grad_ci -from paddle.fluid.tests.unittests.test_elementwise_mul_op import ElementwiseMulOp +from paddle.fluid.tests.unittests.test_elementwise_mul_op import ( + ElementwiseMulOp, +) from paddle import enable_static class TestMKLDNNElementwiseMulOp(ElementwiseMulOp): - def init_kernel_type(self): self.use_mkldnn = True @@ -29,15 +30,13 @@ def init_dtype(self): class TestMKLDNNElementwiseMulOp2(TestMKLDNNElementwiseMulOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.multiply(self.x, self.y) class TestMKLDNNElementwiseMulOp3(TestMKLDNNElementwiseMulOp): - def init_input_output(self): self.x = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) self.y = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) @@ -45,7 +44,6 @@ def init_input_output(self): class TestMKLDNNElementwiseMulOp4(TestMKLDNNElementwiseMulOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 32]).astype(self.dtype) self.y = np.random.uniform(1, 2, [4, 32]).astype(self.dtype) @@ -60,7 +58,6 @@ def test_check_grad_ingore_y(self): class TestMKLDNNElementwiseMulOp5(TestMKLDNNElementwiseMulOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(self.dtype) self.y = np.random.uniform(1, 2, [100]).astype(self.dtype) @@ -81,9 +78,9 @@ def test_check_grad_ingore_x(self): @skip_check_grad_ci( - reason="oneDNN's int8 elementwise_ops don't implemend grad kernel.") + reason="oneDNN's int8 elementwise_ops don't implemend grad kernel." +) class TestInt8(ElementwiseMulOp): - def init_kernel_type(self): self.use_mkldnn = True self._cpu_only = True @@ -117,7 +114,6 @@ def test_check_grad_ingore_y(self): class TestInt8Scales(TestInt8): - def quantize(self, tensor, dt="int8"): max_int = 127.0 if dt == "int8" else 255.0 scale = max_int / np.abs(np.amax(tensor)) @@ -125,8 +121,8 @@ def quantize(self, tensor, dt="int8"): return scale, quantized def init_input_output(self): - self.x_f = np.random.random((100, )).astype("float") - self.y_f = np.random.random((100, )).astype("float") + self.x_f = np.random.random((100,)).astype("float") + self.y_f = np.random.random((100,)).astype("float") self.out_f = np.multiply(self.x_f, self.y_f) self.scale_x, self.x = self.quantize(self.x_f) @@ -142,15 +138,15 @@ def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode self.init_scales() int_atol = 1 # different quantization techniques - self.check_output(check_dygraph=(self.use_mkldnn == False), - atol=int_atol) + self.check_output( + check_dygraph=(self.use_mkldnn == False), atol=int_atol + ) class TestUint8Scales(TestInt8Scales): - def init_input_output(self): - self.x_f = np.random.random((100, )).astype("float") - self.y_f = np.random.random((100, )).astype("float") + self.x_f = np.random.random((100,)).astype("float") + self.y_f = np.random.random((100,)).astype("float") self.out_f = np.multiply(self.x_f, self.y_f) self.scale_x, self.x = self.quantize(self.x_f, "uint8") diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_sub_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_sub_mkldnn_op.py index b91e81550cdeb..6d1f12d1414cb 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_sub_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_elementwise_sub_mkldnn_op.py @@ -15,15 +15,20 @@ import unittest import numpy as np from paddle import enable_static -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) from paddle.fluid.framework import _current_expected_place import paddle.fluid.core as core -@OpTestTool.skip_if(not (isinstance(_current_expected_place(), core.CPUPlace)), - "GPU is not supported") +@OpTestTool.skip_if( + not (isinstance(_current_expected_place(), core.CPUPlace)), + "GPU is not supported", +) class TestMKLDNNElementwiseSubOp(OpTest): - def setUp(self): self.op_type = "elementwise_sub" self.init_dtype() @@ -32,7 +37,7 @@ def setUp(self): self.init_axis() self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -65,15 +70,13 @@ def test_check_output(self): class TestMKLDNNElementwiseSubOp2(TestMKLDNNElementwiseSubOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.subtract(self.x, self.y) class TestMKLDNNElementwiseSubOp3(TestMKLDNNElementwiseSubOp): - def init_input_output(self): self.x = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) self.y = np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) @@ -81,7 +84,6 @@ def init_input_output(self): class TestMKLDNNElementwiseSubOp4(TestMKLDNNElementwiseSubOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 32]).astype(self.dtype) self.y = np.random.uniform(1, 2, [4, 32]).astype(self.dtype) @@ -89,7 +91,6 @@ def init_input_output(self): class TestMKLDNNElementwiseSubOp40(TestMKLDNNElementwiseSubOp): - def init_input_output(self): self.x = np.random.uniform(0.1, 2, [180, 1]).astype(self.dtype) self.y = np.random.uniform(0.1, 1, [1, 256]).astype(self.dtype) @@ -106,7 +107,6 @@ def test_check_grad_ignore_y(self): class TestMKLDNNElementwiseSubOp5(TestMKLDNNElementwiseSubOp): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(self.dtype) self.y = np.random.uniform(1, 2, [100]).astype(self.dtype) @@ -114,7 +114,6 @@ def init_input_output(self): class TestMKLDNNElementwiseSubOp_broadcast(TestMKLDNNElementwiseSubOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -125,7 +124,6 @@ def init_axis(self): class TestElementwiseSubOp_xsize_lessthan_ysize_sub(TestMKLDNNElementwiseSubOp): - def init_input_output(self): self.x = np.random.rand(10, 12).astype(self.dtype) self.y = np.random.rand(2, 2, 10, 12).astype(self.dtype) @@ -146,7 +144,6 @@ def test_check_grad_ignore_x(self): @OpTestTool.skip_if_not_cpu_bf16() class TestBf16(TestMKLDNNElementwiseSubOp): - def setUp(self): self.op_type = "elementwise_sub" self.init_dtype() @@ -165,34 +162,46 @@ def init_dtype(self): self.mkldnn_data_type = "bfloat16" def init_input_output(self): - self.x = np.random.random(100, ).astype(self.dtype) - self.y = np.random.random(100, ).astype(self.dtype) + self.x = np.random.random( + 100, + ).astype(self.dtype) + self.y = np.random.random( + 100, + ).astype(self.dtype) self.out = np.subtract(self.x, self.y) def test_check_output(self): self.check_output_with_place(core.CPUPlace()) def test_check_grad_normal(self): - self.check_grad_with_place(core.CPUPlace(), ["X", "Y"], - "Out", - user_defined_grads=[self.x, -self.x], - user_defined_grad_outputs=[self.x_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["X", "Y"], + "Out", + user_defined_grads=[self.x, -self.x], + user_defined_grad_outputs=[self.x_bf16], + ) def test_check_grad_ignore_x(self): - self.check_grad_with_place(core.CPUPlace(), ["Y"], - "Out", - user_defined_grads=[-self.y], - user_defined_grad_outputs=[self.y_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["Y"], + "Out", + user_defined_grads=[-self.y], + user_defined_grad_outputs=[self.y_bf16], + ) def test_check_grad_ignore_y(self): - self.check_grad_with_place(core.CPUPlace(), ["X"], - "Out", - user_defined_grads=[self.x], - user_defined_grad_outputs=[self.x_bf16]) + self.check_grad_with_place( + core.CPUPlace(), + ["X"], + "Out", + user_defined_grads=[self.x], + user_defined_grad_outputs=[self.x_bf16], + ) class TestBf16Broadcasting(TestBf16): - def init_input_output(self): self.x = np.random.uniform(1, 2, [2, 3, 4, 100]).astype(self.dtype) self.y = np.random.uniform(1, 2, [100]).astype(self.dtype) @@ -206,22 +215,24 @@ def compute_reduced_gradients(self, out_grads): def test_check_grad_normal(self): self.check_grad_with_place( - core.CPUPlace(), ["X", "Y"], + core.CPUPlace(), + ["X", "Y"], "Out", - user_defined_grads=[self.x, - self.compute_reduced_gradients(self.x)], - user_defined_grad_outputs=[self.x_bf16]) + user_defined_grads=[self.x, self.compute_reduced_gradients(self.x)], + user_defined_grad_outputs=[self.x_bf16], + ) def test_check_grad_ignore_x(self): self.check_grad_with_place( - core.CPUPlace(), ["Y"], + core.CPUPlace(), + ["Y"], "Out", user_defined_grads=[self.compute_reduced_gradients(self.x)], - user_defined_grad_outputs=[self.x_bf16]) + user_defined_grad_outputs=[self.x_bf16], + ) class TestInt8(TestMKLDNNElementwiseSubOp): - def init_kernel_type(self): self.use_mkldnn = True self._cpu_only = True diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_expand_v2_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_expand_v2_mkldnn_op.py index cf98324809d33..f156cceb82312 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_expand_v2_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_expand_v2_mkldnn_op.py @@ -16,13 +16,18 @@ import numpy as np import paddle from paddle.fluid import core -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) -@OpTestTool.skip_if(core.is_compiled_with_cuda(), - "CUDA required dygraph so oneDNN UT must be skipped") +@OpTestTool.skip_if( + core.is_compiled_with_cuda(), + "CUDA required dygraph so oneDNN UT must be skipped", +) class TestExpandV2OneDNNOp(OpTest): - def setUp(self): self.op_type = "expand_v2" self.init_data() @@ -52,7 +57,6 @@ def test_check_grad(self): class TestExpandV2ExpandDimOneDNNOp(TestExpandV2OneDNNOp): - def init_data(self): self.ori_shape = [120] self.shape = [2, 120] @@ -60,7 +64,6 @@ def init_data(self): class TestExpandV2CopyScenarioOneDNNOp(TestExpandV2OneDNNOp): - def init_data(self): self.ori_shape = (2, 10, 5) self.shape = (2, 10, 5) @@ -68,7 +71,6 @@ def init_data(self): class TestExpandV2CopyScenarioShapeNotGivenOneDNNOp(TestExpandV2OneDNNOp): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.shape = (-1, -1, -1, -1) @@ -76,7 +78,6 @@ def init_data(self): class TestExpandV2ExpandShapesTensor1OneDNNOp(TestExpandV2OneDNNOp): - def init_data(self): self.ori_shape = [100, 1] self.expand_times = [1, 2] @@ -86,8 +87,9 @@ def init_data(self): def calc_expand_shapes_tensor(self): self.expand_shapes_tensor = [] for index, ele in enumerate(self.expand_shape): - self.expand_shapes_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + self.expand_shapes_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) def set_additional_inputs(self): self.calc_expand_shapes_tensor() @@ -95,8 +97,8 @@ def set_additional_inputs(self): class TestExpandV2ExpandShapesTensor2OneDNNOp( - TestExpandV2ExpandShapesTensor1OneDNNOp): - + TestExpandV2ExpandShapesTensor1OneDNNOp +): def init_data(self): self.ori_shape = [12, 14] self.expand_times = [1, 1] @@ -105,7 +107,6 @@ def init_data(self): class TestExpandV2ShapesTensorOneDNNOp(TestExpandV2OneDNNOp): - def init_data(self): self.ori_shape = [100] self.expand_times = [2, 1] @@ -118,10 +119,8 @@ def set_additional_inputs(self): # BF16 TESTS def create_expand_v2_bf16_test_class(parent): - @OpTestTool.skip_if_not_cpu_bf16() class TestExpandV2BF16OneDNNOp(parent): - def set_inputs(self): self.attrs['mkldnn_data_type'] = 'bfloat16' self.inputs = {"X": convert_float_to_uint16(self.x)} @@ -137,10 +136,12 @@ def calculate_grads(self): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", user_defined_grads=[convert_float_to_uint16(self.dx)], - user_defined_grad_outputs=[self.dout]) + user_defined_grad_outputs=[self.dout], + ) cls_name = "{0}_{1}".format(parent.__name__, "Expand_v2_BF16") TestExpandV2BF16OneDNNOp.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fc_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fc_bf16_mkldnn_op.py index b5b83bf47ea72..b35b06915b5fd 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fc_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fc_bf16_mkldnn_op.py @@ -25,16 +25,15 @@ def fully_connected_naive(input, weights, bias_data): class MatrixGenerate: - def __init__(self, mb, ic, oc, h, w): self.input = np.random.random((mb, ic * h * w)).astype(np.float32) self.weights = np.random.random((ic * h * w, oc)).astype(np.float32) -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestFcBf16MklDNNOp(OpTest): - def generate_data(self): self.matrix = MatrixGenerate(1, 10, 15, 3, 3) self.bias = np.random.random(15).astype("float32") @@ -46,20 +45,21 @@ def setUp(self): self.force_fp32_output = False self.generate_data() - self.output = fully_connected_naive(self.matrix.input, - self.matrix.weights, self.bias) + self.output = fully_connected_naive( + self.matrix.input, self.matrix.weights, self.bias + ) if not self.force_fp32_output: self.output = convert_float_to_uint16(self.output) self.inputs = { 'Input': convert_float_to_uint16(self.matrix.input), 'W': self.matrix.weights, - 'Bias': self.bias + 'Bias': self.bias, } self.attrs = { 'use_mkldnn': self.use_mkldnn, - 'force_fp32_output': self.force_fp32_output + 'force_fp32_output': self.force_fp32_output, } self.outputs = {'Out': self.output} @@ -75,7 +75,6 @@ def test_check_grad_no_weight(self): class TestFCMKLDNNOp1(TestFcBf16MklDNNOp): - def generate_data(self): self.matrix = MatrixGenerate(2, 15, 48, 2, 2) self.bias = np.random.random(48).astype(np.float32) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fc_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fc_int8_mkldnn_op.py index 5cecbf159cc05..9549325173527 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fc_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fc_int8_mkldnn_op.py @@ -19,7 +19,6 @@ @OpTestTool.skip_if_not_cpu() class TestFCINT8OneDNNOp(OpTest): - def setUp(self): self.op_type = "fc" self._cpu_only = True @@ -32,7 +31,7 @@ def setUp(self): 'Scale_in': self.x_scale, 'Scale_weights': [self.y_scale], 'Scale_out': self.out_scale, - 'force_fp32_output': self.force_fp32_output + 'force_fp32_output': self.force_fp32_output, } if self.force_fp32_output: @@ -50,7 +49,7 @@ def set_inputs(self): self.inputs = {'Input': self.x, 'W': self.y_float, 'Bias': self.bias} def quantize(self, tensor): - scale = 63. / np.abs(np.amax(tensor)) + scale = 63.0 / np.abs(np.amax(tensor)) quantized = np.round(scale * tensor).astype("int8") return scale, quantized @@ -74,7 +73,6 @@ def test_check_output(self): class TestFCINT8NoBiasOneDNNOp(TestFCINT8OneDNNOp): - def configure(self): self.use_bias = False self.force_fp32_output = False @@ -87,7 +85,6 @@ def set_inputs(self): class TestFCINT8ForceFP32OutputOneDNNOp(TestFCINT8NoBiasOneDNNOp): - def configure(self): self.use_bias = False self.force_fp32_output = True @@ -95,5 +92,6 @@ def configure(self): if __name__ == "__main__": import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fc_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fc_mkldnn_op.py index 0c1d9bef032bc..d64e0b441ee69 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fc_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fc_mkldnn_op.py @@ -23,14 +23,12 @@ def fully_connected_naive(input, weights, bias_data): class MatrixGenerate: - def __init__(self, mb, ic, oc, h, w): self.input = np.random.random((mb, ic * h * w)).astype("float32") self.weights = np.random.random((ic * h * w, oc)).astype("float32") class TestFCMKLDNNOp(OpTest): - def create_data(self): self.matrix = MatrixGenerate(1, 10, 15, 3, 3) self.bias = np.random.random(15).astype("float32") @@ -43,15 +41,15 @@ def setUp(self): self.inputs = { 'Input': self.matrix.input, 'W': self.matrix.weights, - 'Bias': self.bias + 'Bias': self.bias, } self.attrs = {'use_mkldnn': self.use_mkldnn} self.outputs = { - 'Out': - fully_connected_naive(self.matrix.input, self.matrix.weights, - self.bias) + 'Out': fully_connected_naive( + self.matrix.input, self.matrix.weights, self.bias + ) } def test_check_output(self): @@ -66,7 +64,6 @@ def test_check_grad_no_weight(self): class TestFCMKLDNNOp1(TestFCMKLDNNOp): - def create_data(self): self.matrix = MatrixGenerate(2, 15, 48, 2, 2) self.bias = np.random.random(48).astype("float32") @@ -74,5 +71,6 @@ def create_data(self): if __name__ == "__main__": import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fill_constant_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fill_constant_mkldnn_op.py index f4dfcee10bca2..92e274229b025 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fill_constant_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fill_constant_mkldnn_op.py @@ -20,7 +20,6 @@ @OpTestTool.skip_if_not_cpu_bf16() class TestFillConstant2DOneDNNOp(OpTest): - def setUp(self): self.op_type = "fill_constant" self.dtype = np.float32 @@ -62,59 +61,58 @@ def test_check_output(self): self.check_output() -class TestFillZerosLike4DShapeTensorPriorityOneDNNOp(TestFillConstant2DOneDNNOp - ): - +class TestFillZerosLike4DShapeTensorPriorityOneDNNOp( + TestFillConstant2DOneDNNOp +): def set_inputs(self): self.inputs = {'ShapeTensor': np.array([5, 6, 7, 8]).astype("int32")} class TestFillZerosLike4DShapeTensorListPriorityOneDNNOp( - TestFillConstant2DOneDNNOp): - + TestFillConstant2DOneDNNOp +): def set_inputs(self): shape = (4, 5, 6, 7) self.shape_tensor_list = [] for index, elem in enumerate(shape): - self.shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * elem)) + self.shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * elem) + ) self.inputs = {'ShapeTensorList': self.shape_tensor_list} class TestFillZerosLike2DStringValueInfOneDNNOp(TestFillConstant2DOneDNNOp): - def set_attrs(self): self.str_value = "inf" self.attrs = {'shape': (10, 13), 'use_mkldnn': True, 'str_value': "inf"} -class TestFillZerosLike2DStringValueMinusInfOneDNNOp(TestFillConstant2DOneDNNOp - ): - +class TestFillZerosLike2DStringValueMinusInfOneDNNOp( + TestFillConstant2DOneDNNOp +): def set_attrs(self): self.str_value = "-inf" self.attrs = { 'shape': (10, 13), 'use_mkldnn': True, - 'str_value': "-inf" + 'str_value': "-inf", } class TestFillZerosLike2DStringValueFloatOneDNNOp(TestFillConstant2DOneDNNOp): - def set_attrs(self): self.str_value = "0.123" self.attrs = { 'shape': (10, 13), 'use_mkldnn': True, - 'str_value': "0.123" + 'str_value': "0.123", } class TestFillZerosLike2DValueTensorPriorityOneDNNOp( - TestFillZerosLike2DStringValueFloatOneDNNOp): - + TestFillZerosLike2DStringValueFloatOneDNNOp +): def set_inputs(self): self.inputs = {'ValueTensor': np.atleast_1d(2.25).astype("float32")} diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_flags_mkldnn_ops_on_off.py b/python/paddle/fluid/tests/unittests/mkldnn/test_flags_mkldnn_ops_on_off.py index ec4e87918d7c2..b59ce2ee71498 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_flags_mkldnn_ops_on_off.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_flags_mkldnn_ops_on_off.py @@ -20,7 +20,6 @@ class TestFlagsUseMkldnn(unittest.TestCase): - def setUp(self): self._python_interp = sys.executable self._python_interp += " check_flags_mkldnn_ops_on_off.py" @@ -30,16 +29,22 @@ def setUp(self): self.env[str("FLAGS_use_mkldnn")] = str("1") self.relu_regex = b"^onednn_verbose,exec,cpu,eltwise,.+alg:eltwise_relu alpha:0 beta:0,10x20x20" - self.ew_add_regex = b"^onednn_verbose,exec,cpu,binary.+alg:binary_add,10x20x30:10x20x30" - self.matmul_regex = b"^onednn_verbose,exec,cpu,matmul,.*10x20x30:10x30x20:10x20x20" + self.ew_add_regex = ( + b"^onednn_verbose,exec,cpu,binary.+alg:binary_add,10x20x30:10x20x30" + ) + self.matmul_regex = ( + b"^onednn_verbose,exec,cpu,matmul,.*10x20x30:10x30x20:10x20x20" + ) def flags_use_mkl_dnn_common(self, e): cmd = self._python_interp env = dict(self.env, **e) - proc = subprocess.Popen(cmd.split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env) + proc = subprocess.Popen( + cmd.split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) out, err = proc.communicate() returncode = proc.returncode @@ -98,7 +103,7 @@ def test_flags_use_mkl_dnn_off_multiple(self): def test_flags_use_mkl_dnn_on_off(self): env = { str("FLAGS_tracer_mkldnn_ops_on"): str("elementwise_add"), - str("FLAGS_tracer_mkldnn_ops_off"): str("matmul") + str("FLAGS_tracer_mkldnn_ops_off"): str("matmul"), } out, err = self.flags_use_mkl_dnn_common(env) assert self.not_found(self.relu_regex, out, err) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_flags_use_mkldnn.py b/python/paddle/fluid/tests/unittests/mkldnn/test_flags_use_mkldnn.py index 1e79d8e82c5d3..35ca3188f1950 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_flags_use_mkldnn.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_flags_use_mkldnn.py @@ -20,7 +20,6 @@ class TestFlagsUseMkldnn(unittest.TestCase): - def setUp(self): self._python_interp = sys.executable self._python_interp += " check_flags_use_mkldnn.py" @@ -45,10 +44,12 @@ def found(self, regex, out, err): def test_flags_use_mkl_dnn(self): cmd = self._python_interp - proc = subprocess.Popen(cmd.split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=self.env) + proc = subprocess.Popen( + cmd.split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.env, + ) out, err = proc.communicate() returncode = proc.returncode diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_flatten_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_flatten_mkldnn_op.py index 32e7cec2999e0..0786b8d7d7a9c 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_flatten_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_flatten_mkldnn_op.py @@ -17,12 +17,15 @@ import paddle import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) @OpTestTool.skip_if_not_cpu_bf16() class TestFlattenOneDNNOp(OpTest): - def setUp(self): self.set_op_type() self.init_test_case() @@ -50,7 +53,6 @@ def init_test_case(self): class TestFlattenOneDNNOp1(TestFlattenOneDNNOp): - def init_test_case(self): self.in_shape = (3, 2, 2, 10) self.axis = 0 @@ -58,7 +60,6 @@ def init_test_case(self): class TestFlattenOneDNNOpSixDims(TestFlattenOneDNNOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 @@ -66,28 +67,23 @@ def init_test_case(self): class TestFlatten2OneDNNOp(TestFlattenOneDNNOp): - def set_op_type(self): self.op_type = "flatten2" class TestFlatten2OneDNNOp1(TestFlattenOneDNNOp1): - def set_op_type(self): self.op_type = "flatten2" class TestFlatten2OneDNNOpSixDims(TestFlattenOneDNNOpSixDims): - def set_op_type(self): self.op_type = "flatten2" # BF16 TESTS def create_flatten_bf16_test_classes(parent): - class TestFlatten2BF16OneDNNOp(parent): - def set_inputs(self): self.dtype = np.uint16 self.inputs = { @@ -99,22 +95,25 @@ def calculate_grads(self): self.dx = np.reshape(self.dout, self.ori_shape) def test_check_output(self): - self.check_output_with_place(core.CPUPlace(), - no_check_set=["XShape"]) + self.check_output_with_place( + core.CPUPlace(), no_check_set=["XShape"] + ) def test_check_grad(self): self.calculate_grads() - self.check_grad_with_place(core.CPUPlace(), ["X"], - "Out", - user_defined_grads=[self.dx], - user_defined_grad_outputs=[self.dout]) + self.check_grad_with_place( + core.CPUPlace(), + ["X"], + "Out", + user_defined_grads=[self.dx], + user_defined_grad_outputs=[self.dout], + ) cls_name = "{0}_{1}".format(parent.__name__, "Flatten2_BF16") TestFlatten2BF16OneDNNOp.__name__ = cls_name globals()[cls_name] = TestFlatten2BF16OneDNNOp class TestFlattenBF16OneDNNOp(parent): - def set_op_type(self): self.dtype = np.uint16 self.op_type = "flatten" @@ -138,10 +137,12 @@ def test_check_output(self): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", user_defined_grads=[self.dx], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) cls_name = "{0}_{1}".format(parent.__name__, "Flatten_BF16") TestFlattenBF16OneDNNOp.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_bf16_mkldnn_op.py index 62759913e8452..da8d514936090 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_bf16_mkldnn_op.py @@ -20,10 +20,10 @@ from paddle.fluid.tests.unittests.test_fusion_lstm_op import ACTIVATION -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestFusionGRUBF16MKLDNNOp(OpTest): - def set_confs(self): pass @@ -66,19 +66,30 @@ def setUp(self): wh_bf16 = convert_float_to_uint16(wh_fp32) # bias is fp32 despite other inputs being in bf16 - bias = np.random.rand( - 1, 3 * self.D).astype('float32') if self.with_bias else np.zeros( - (1, 3 * self.D), dtype='float32') - - h0_fp32 = np.random.rand( - N, self.D).astype('float32') if self.with_h0 else np.zeros( - (N, self.D), dtype='float32') - - _, _, _, hidden = fusion_gru(x_fp32, self.lod, h0_fp32, wx_fp32, - wh_fp32, bias, self.is_reverse, - self.origin_mode, - ACTIVATION[self.act_state], - ACTIVATION[self.act_gate]) + bias = ( + np.random.rand(1, 3 * self.D).astype('float32') + if self.with_bias + else np.zeros((1, 3 * self.D), dtype='float32') + ) + + h0_fp32 = ( + np.random.rand(N, self.D).astype('float32') + if self.with_h0 + else np.zeros((N, self.D), dtype='float32') + ) + + _, _, _, hidden = fusion_gru( + x_fp32, + self.lod, + h0_fp32, + wx_fp32, + wh_fp32, + bias, + self.is_reverse, + self.origin_mode, + ACTIVATION[self.act_state], + ACTIVATION[self.act_gate], + ) hidden_bf16 = convert_float_to_uint16(hidden) @@ -86,13 +97,13 @@ def setUp(self): self.inputs = { 'X': (x_bf16, self.lod), 'WeightX': wx_bf16, - 'WeightH': wh_bf16 + 'WeightH': wh_bf16, } elif self.weights_dtype == 'fp32': self.inputs = { 'X': (x_bf16, self.lod), 'WeightX': wx_fp32, - 'WeightH': wh_fp32 + 'WeightH': wh_fp32, } if self.with_bias: @@ -120,24 +131,22 @@ def setUp(self): class TestFusionGRUINT8MKLDNNOp2(TestFusionGRUBF16MKLDNNOp): - def set_confs(self): self.origin_mode = False class TestFusionGRUINT8MKLDNNOp3(TestFusionGRUBF16MKLDNNOp): - def set_confs(self): self.with_bias = False class TestFusionGRUINT8MKLDNNBF16WeightsOp(TestFusionGRUBF16MKLDNNOp): - def set_confs(self): self.weights_dtype = 'bf16' if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_int8_mkldnn_op.py index 9cbbddd6a42af..fdd9ef2b4cca9 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_int8_mkldnn_op.py @@ -20,7 +20,6 @@ class TestFusionGRUINT8MKLDNNOp(OpTest): - def set_confs(self): pass @@ -62,35 +61,60 @@ def setUp(self): # WeightH data shape in PP: [OC, 2 * OC] + [OC, OC] # Scales shape in oneDNN: [3, OC] s8_max = 127.0 - scale_ur = s8_max / np.max(np.abs( - np.concatenate([ - wx[:, :2 * self.OC], - wh.flatten()[:2 * self.OC * self.OC].reshape( - self.OC, 2 * self.OC) - ], - axis=0)), - axis=0) - scale_o = s8_max / np.max(np.abs( - np.concatenate([ - wx[:, 2 * self.OC:], - wh.flatten()[2 * self.OC * self.OC:].reshape(self.OC, self.OC) - ], - axis=0)), - axis=0) + scale_ur = s8_max / np.max( + np.abs( + np.concatenate( + [ + wx[:, : 2 * self.OC], + wh.flatten()[: 2 * self.OC * self.OC].reshape( + self.OC, 2 * self.OC + ), + ], + axis=0, + ) + ), + axis=0, + ) + scale_o = s8_max / np.max( + np.abs( + np.concatenate( + [ + wx[:, 2 * self.OC :], + wh.flatten()[2 * self.OC * self.OC :].reshape( + self.OC, self.OC + ), + ], + axis=0, + ) + ), + axis=0, + ) scale_weights = np.concatenate([scale_ur, scale_o]).astype('float') - bias = np.random.rand( - 1, 3 * self.OC).astype('float32') if self.with_bias else np.zeros( - (1, 3 * self.OC), dtype='float32') - h0 = np.random.rand( - N, self.OC).astype('float32') if self.with_h0 else np.zeros( - (N, self.OC), dtype='float32') - - _, _, _, hidden_f32 = fusion_gru(x_f32, self.lod, h0, wx, wh, bias, - self.is_reverse, self.origin_mode, - ACTIVATION[self.act_state], - ACTIVATION[self.act_gate]) + bias = ( + np.random.rand(1, 3 * self.OC).astype('float32') + if self.with_bias + else np.zeros((1, 3 * self.OC), dtype='float32') + ) + h0 = ( + np.random.rand(N, self.OC).astype('float32') + if self.with_h0 + else np.zeros((N, self.OC), dtype='float32') + ) + + _, _, _, hidden_f32 = fusion_gru( + x_f32, + self.lod, + h0, + wx, + wh, + bias, + self.is_reverse, + self.origin_mode, + ACTIVATION[self.act_state], + ACTIVATION[self.act_gate], + ) self.inputs = {'X': (x_u8, self.lod), 'WeightX': wx, 'WeightH': wh} @@ -106,7 +130,8 @@ def setUp(self): else: self.error_margin = 1 hidden_u8 = np.rint(hidden_f32 * scale_data + shift_data).astype( - np.uint8) + np.uint8 + ) # hidden_u8 = (hidden_f32 * scale_data + shift_data).astype(np.uint8) self.outputs = {'Hidden': (hidden_u8, self.lod)} @@ -120,7 +145,7 @@ def setUp(self): 'force_fp32_output': self.force_fp32_output, 'Scale_data': scale_data, 'Shift_data': shift_data, - 'Scale_weights': scale_weights + 'Scale_weights': scale_weights, } def test_check_output(self): @@ -128,30 +153,27 @@ def test_check_output(self): class TestFusionGRUINT8MKLDNNOp2(TestFusionGRUINT8MKLDNNOp): - def set_confs(self): self.force_fp32_output = False class TestFusionGRUINT8MKLDNNOp3(TestFusionGRUINT8MKLDNNOp): - def set_confs(self): self.origin_mode = False class TestFusionGRUINT8MKLDNNOp4(TestFusionGRUINT8MKLDNNOp): - def set_confs(self): self.with_bias = False class TestFusionGRUINT8MKLDNNOp5(TestFusionGRUINT8MKLDNNOp): - def set_confs(self): self.with_h0 = False if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_mkldnn_op.py index e47105e733cca..aa38c0047593f 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_mkldnn_op.py @@ -17,41 +17,35 @@ class TestFusionGRUMKLDNNOp(TestFusionGRUOp): - def set_confs(self): self.use_mkldnn = True class TestFusionGRUMKLDNNOpNoInitial(TestFusionGRUOp): - def set_confs(self): self.with_h0 = False self.use_mkldnn = True class TestFusionGRUMKLDNNOpNoBias(TestFusionGRUOp): - def set_confs(self): self.with_bias = False self.use_mkldnn = True class TestFusionGRUMKLDNNOpReverse(TestFusionGRUOp): - def set_confs(self): self.is_reverse = True self.use_mkldnn = True class TestFusionGRUMKLDNNOpOriginMode(TestFusionGRUOp): - def set_confs(self): self.origin_mode = True self.use_mkldnn = True class TestFusionGRUMKLDNNOpMD1(TestFusionGRUOp): - def set_confs(self): self.M = 36 self.D = 8 @@ -59,7 +53,6 @@ def set_confs(self): class TestFusionGRUMKLDNNOpMD2(TestFusionGRUOp): - def set_confs(self): self.M = 8 self.D = 8 @@ -67,7 +60,6 @@ def set_confs(self): class TestFusionGRUMKLDNNOpMD3(TestFusionGRUOp): - def set_confs(self): self.M = 17 self.D = 15 @@ -75,7 +67,6 @@ def set_confs(self): class TestFusionGRUMKLDNNOpBS1(TestFusionGRUOp): - def set_confs(self): self.lod = [[3]] self.D = 16 @@ -84,5 +75,6 @@ def set_confs(self): if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_bf16_mkldnn_op.py index a2fdf54da9ded..4d2842d26ede6 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_bf16_mkldnn_op.py @@ -16,22 +16,25 @@ import numpy as np import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16 -from paddle.fluid.tests.unittests.test_fusion_lstm_op import ACTIVATION, fusion_lstm +from paddle.fluid.tests.unittests.test_fusion_lstm_op import ( + ACTIVATION, + fusion_lstm, +) -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestFusionLSTMBF16ONEDNNOp(OpTest): - def set_confs(self): pass def test_check_output(self): for use_seq in {True, False}: self.attrs['use_seq'] = use_seq - self.check_output(check_dygraph=False, - no_check_set=["Cell"], - atol=2e-2) + self.check_output( + check_dygraph=False, no_check_set=["Cell"], atol=2e-2 + ) def setUp(self): self.op_type = 'fusion_lstm' @@ -75,8 +78,8 @@ def setUp(self): b = np.random.normal(size=(1, 7 * self.D)).astype('float32') else: b = np.random.normal(size=(1, 4 * self.D)).astype('float32') - w_b = np.copy(b[:, 0:4 * self.D]) - w_c = b[:, 4 * self.D:] if self.use_peepholes else None + w_b = np.copy(b[:, 0 : 4 * self.D]) + w_c = b[:, 4 * self.D :] if self.use_peepholes else None wx = np.random.normal(size=(self.M, 4 * self.D)).astype('float32') @@ -84,12 +87,23 @@ def setUp(self): wh_bf16 = convert_float_to_uint16(wh) bx = np.random.normal(size=(1, 4 * self.D)).astype('float32') - b[0, 0:4 * self.D] += bx[0, :] - - hidden, c = fusion_lstm(x, self.lod, wx, bx, h0, c0, wh, w_b, w_c, - self.is_reverse, ACTIVATION[self.act_gate], - ACTIVATION[self.act_cell], - ACTIVATION[self.act_cand]) + b[0, 0 : 4 * self.D] += bx[0, :] + + hidden, c = fusion_lstm( + x, + self.lod, + wx, + bx, + h0, + c0, + wh, + w_b, + w_c, + self.is_reverse, + ACTIVATION[self.act_gate], + ACTIVATION[self.act_cell], + ACTIVATION[self.act_cand], + ) hidden = hidden.astype('float32') hidden_bf16 = convert_float_to_uint16(hidden) @@ -99,14 +113,14 @@ def setUp(self): 'X': (x_bf16, self.lod), 'WeightX': wx_bf16, 'WeightH': wh_bf16, - 'Bias': b + 'Bias': b, } elif self.weights_dtype == 'fp32': self.inputs = { 'X': (x_bf16, self.lod), 'WeightX': wx, 'WeightH': wh, - 'Bias': b + 'Bias': b, } if self.has_initial_state: @@ -135,30 +149,27 @@ def setUp(self): class TestFusionLSTMBF16ONEDNNPeepholesOp(TestFusionLSTMBF16ONEDNNOp): - def set_confs(self): self.use_peepholes = True class TestFusionLSTMBF16ONEDNNInitializedStateOp(TestFusionLSTMBF16ONEDNNOp): - def set_confs(self): self.has_initial_state = True class TestFusionLSTMBF16ONEDNNReverseOp(TestFusionLSTMBF16ONEDNNOp): - def set_confs(self): self.is_reverse = True class TestFusionLSTMBF16ONEDNNBF16WeightsOp(TestFusionLSTMBF16ONEDNNOp): - def set_confs(self): self.weights_dtype = 'bf16' if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_int8_mkldnn_op.py index 968c01faa17b0..0b252e7cbafa3 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_int8_mkldnn_op.py @@ -15,11 +15,13 @@ import unittest import numpy as np from paddle.fluid.tests.unittests.op_test import OpTest -from paddle.fluid.tests.unittests.test_fusion_lstm_op import ACTIVATION, fusion_lstm +from paddle.fluid.tests.unittests.test_fusion_lstm_op import ( + ACTIVATION, + fusion_lstm, +) class TestFusionLSTMINT8MKLDNNOp(OpTest): - def set_confs(self): pass @@ -59,7 +61,8 @@ def setUp(self): s8_max = 127.0 scale_weights = s8_max / np.max( - np.abs(np.concatenate([wx[:, :], wh[:, :]], axis=0)), axis=0) + np.abs(np.concatenate([wx[:, :], wh[:, :]], axis=0)), axis=0 + ) scale_weights = scale_weights.astype('float') @@ -67,11 +70,11 @@ def setUp(self): b = np.random.rand(1, 7 * self.OC).astype('float32') else: b = np.random.rand(1, 4 * self.OC).astype('float32') - w_b = np.copy(b[:, 0:4 * self.OC]) - w_c = b[:, 4 * self.OC:] if self.use_peepholes else None + w_b = np.copy(b[:, 0 : 4 * self.OC]) + w_c = b[:, 4 * self.OC :] if self.use_peepholes else None bx = np.random.normal(size=(1, 4 * self.OC)).astype('float32') - b[0, 0:4 * self.OC] += bx[0, :] + b[0, 0 : 4 * self.OC] += bx[0, :] if self.has_initial_state: h0 = np.random.rand(N, self.OC).astype('float32') @@ -80,17 +83,27 @@ def setUp(self): h0 = np.zeros((N, self.OC)).astype('float32') c0 = np.zeros((N, self.OC)).astype('float32') - hidden_f32, c = fusion_lstm(x_f32, self.lod, wx, bx, h0, c0, wh, w_b, - w_c, self.is_reverse, - ACTIVATION[self.act_gate], - ACTIVATION[self.act_cell], - ACTIVATION[self.act_cand]) + hidden_f32, c = fusion_lstm( + x_f32, + self.lod, + wx, + bx, + h0, + c0, + wh, + w_b, + w_c, + self.is_reverse, + ACTIVATION[self.act_gate], + ACTIVATION[self.act_cell], + ACTIVATION[self.act_cand], + ) self.inputs = { 'X': (x_u8, self.lod), 'WeightX': wx, 'WeightH': wh, - 'Bias': b + 'Bias': b, } if self.has_initial_state: @@ -101,15 +114,16 @@ def setUp(self): self.error_margin = 1e-1 self.outputs = { 'Hidden': (hidden_f32, self.lod), - 'Cell': (c, self.lod) + 'Cell': (c, self.lod), } else: self.error_margin = 2 hidden_u8 = np.rint(hidden_f32 * scale_data + shift_data).astype( - np.uint8) + np.uint8 + ) self.outputs = { 'Hidden': (hidden_u8, self.lod), - 'Cell': (c, self.lod) + 'Cell': (c, self.lod), } self.attrs = { @@ -123,36 +137,36 @@ def setUp(self): 'force_fp32_output': self.force_fp32_output, 'Scale_data': scale_data, 'Shift_data': shift_data, - 'Scale_weights': scale_weights + 'Scale_weights': scale_weights, } def test_check_output(self): for use_seq in {True, False}: self.attrs['use_seq'] = use_seq - self.check_output(check_dygraph=False, - no_check_set=["Cell"], - atol=self.error_margin) + self.check_output( + check_dygraph=False, + no_check_set=["Cell"], + atol=self.error_margin, + ) class TestFusionLSTMINT8MKLDNNOp2(TestFusionLSTMINT8MKLDNNOp): - def set_confs(self): self.force_fp32_output = True class TestFusionLSTMINT8MKLDNNOp4(TestFusionLSTMINT8MKLDNNOp): - def set_confs(self): self.is_reverse = True class TestFusionLSTMINT8MKLDNNOp5(TestFusionLSTMINT8MKLDNNOp): - def set_confs(self): self.has_initial_state = True if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_mkldnn_op.py index 5ce307fe35ed5..8df2aedb3762d 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_lstm_mkldnn_op.py @@ -17,7 +17,6 @@ class TestFusionLSTMONEDNNOp(TestFusionLSTMOp): - def set_conf(self): self.use_mkldnn = True @@ -28,14 +27,12 @@ def test_check_output(self): class TestFusionLSTMONEDNNOpReverse(TestFusionLSTMONEDNNOp): - def set_conf(self): self.is_reverse = True self.use_mkldnn = True class TestFusionLSTMONEDNNOpInitReverse(TestFusionLSTMONEDNNOp): - def set_conf(self): self.has_initial_state = True self.is_reverse = True @@ -43,7 +40,6 @@ def set_conf(self): class TestFusionLSTMONEDNNOpMD1(TestFusionLSTMONEDNNOp): - def set_conf(self): self.M = 36 self.D = 8 @@ -51,7 +47,6 @@ def set_conf(self): class TestFusionLSTMONEDNNOpMD2(TestFusionLSTMONEDNNOp): - def set_conf(self): self.M = 8 self.D = 8 @@ -59,7 +54,6 @@ def set_conf(self): class TestFusionLSTMONEDNNOpMD3(TestFusionLSTMONEDNNOp): - def set_conf(self): self.M = 15 self.D = 3 @@ -67,7 +61,6 @@ def set_conf(self): class TestFusionLSTMONEDNNOpBS1(TestFusionLSTMONEDNNOp): - def set_conf(self): self.lod = [[3]] self.D = 16 @@ -75,7 +68,6 @@ def set_conf(self): class TestFusionLSTMONEDNNOpPeepholesInit(TestFusionLSTMONEDNNOp): - def set_conf(self): self.use_peepholes = True self.has_initial_state = True @@ -84,5 +76,6 @@ def set_conf(self): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_gaussian_random_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_gaussian_random_mkldnn_op.py index 545caa7b73f6b..8a7c7c23e40f1 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_gaussian_random_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_gaussian_random_mkldnn_op.py @@ -14,17 +14,17 @@ import unittest -from paddle.fluid.tests.unittests.test_gaussian_random_op import TestGaussianRandomOp +from paddle.fluid.tests.unittests.test_gaussian_random_op import ( + TestGaussianRandomOp, +) class TestMKLDNNGaussianRandomOpSeed10(TestGaussianRandomOp): - def init_kernel_type(self): self.use_mkldnn = True class TestMKLDNNGaussianRandomOpSeed0(TestGaussianRandomOp): - def setUp(self): TestGaussianRandomOp.setUp(self) self.use_mkldnn = True @@ -33,7 +33,7 @@ def setUp(self): "mean": 1.0, "std": 2.0, "seed": 10, - "use_mkldnn": self.use_mkldnn + "use_mkldnn": self.use_mkldnn, } diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_bf16_mkldnn_op.py index ab4cc17a713c6..29dd1a8f3609d 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_bf16_mkldnn_op.py @@ -22,8 +22,12 @@ from paddle import enable_static from functools import reduce -from paddle.fluid.tests.unittests.mkldnn.test_layer_norm_mkldnn_op import TestLayerNormMKLDNNOp -from paddle.fluid.tests.unittests.mkldnn.test_layer_norm_mkldnn_op import _reference_layer_norm_naive +from paddle.fluid.tests.unittests.mkldnn.test_layer_norm_mkldnn_op import ( + TestLayerNormMKLDNNOp, +) +from paddle.fluid.tests.unittests.mkldnn.test_layer_norm_mkldnn_op import ( + _reference_layer_norm_naive, +) from paddle.fluid.tests.unittests.op_test import convert_float_to_uint16 from paddle.fluid.tests.unittests.op_test import _set_use_system_allocator @@ -32,26 +36,22 @@ _set_use_system_allocator(True) -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestLayerNormBF16MKLDNNOp(TestLayerNormMKLDNNOp): - def __assert_close(self, tensor, np_array, msg, rtol=2e-02, atol=2): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=rtol, - atol=atol, - err_msg=msg) - - def check_forward(self, - shape, - begin_norm_axis, - with_scale_bias=True, - with_is_test=False): + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=rtol, atol=atol, err_msg=msg + ) + + def check_forward( + self, shape, begin_norm_axis, with_scale_bias=True, with_is_test=False + ): # attr epsilon = 0.00001 x_shape = shape - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) scale_shape = [D] np.random.seed(123) @@ -66,8 +66,9 @@ def check_forward(self, bias = np.array([]) # reference forward & backward - y, mean, variance = _reference_layer_norm_naive(x, scale, bias, epsilon, - begin_norm_axis) + y, mean, variance = _reference_layer_norm_naive( + x, scale, bias, epsilon, begin_norm_axis + ) y_bf16 = convert_float_to_uint16(y) @@ -85,13 +86,17 @@ def check_forward(self, # scale and bias are fp32 and other vars are of bf16 for name in ground_truth: if name == 'x_bf16' or name == 'y_bf16': - block.create_var(name=name, - dtype='uint16', - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype='uint16', + shape=ground_truth[name].shape, + ) else: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) inputs = {"X": block.var('x_bf16')} if with_scale_bias: @@ -110,8 +115,9 @@ def check_forward(self, "epsilon": epsilon, "begin_norm_axis": begin_norm_axis, "use_mkldnn": True, - "is_test": with_is_test - }) + "is_test": with_is_test, + }, + ) exe = fluid.Executor(core.CPUPlace()) @@ -120,19 +126,20 @@ def check_forward(self, input_list.append('scale') input_list.append('bias') - out = exe.run(program, - feed={name: var_dict[name] - for name in input_list}, - fetch_list=['y_bf16', 'mean', 'variance']) + out = exe.run( + program, + feed={name: var_dict[name] for name in input_list}, + fetch_list=['y_bf16', 'mean', 'variance'], + ) self.__assert_close(y_bf16, out[0], "y_bf16", 2) if not with_is_test: self.__assert_close(mean, out[1], "mean") self.__assert_close(variance, out[2], "variance", 1e-3) def test_check_forward_with_is_test(self): - self.check_forward(shape=[2, 3, 4, 5], - begin_norm_axis=3, - with_is_test=True) + self.check_forward( + shape=[2, 3, 4, 5], begin_norm_axis=3, with_is_test=True + ) # TODO (jczaja): Enable those to test when enabling training using bf16 def test_check_forward_with_scale_and_bias(self): diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_mkldnn_op.py index 6506ea41ccdc1..2233bdfda9576 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_layer_norm_mkldnn_op.py @@ -22,7 +22,10 @@ from paddle import enable_static from functools import reduce -from paddle.fluid.tests.unittests.op_test import _set_use_system_allocator, OpTestTool +from paddle.fluid.tests.unittests.op_test import ( + _set_use_system_allocator, + OpTestTool, +) np.random.random(123) @@ -32,7 +35,7 @@ def _reference_layer_norm_naive(x, scale, beta, epsilon, begin_norm_axis=1): x_shape = x.shape N = reduce(mul, x_shape[0:begin_norm_axis], 1) - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) x.shape = [N, D] if scale.size == 0 and beta.size == 0: scale = np.ones([1, D]) @@ -43,34 +46,32 @@ def _reference_layer_norm_naive(x, scale, beta, epsilon, begin_norm_axis=1): mean = np.mean(x, axis=1) var = np.var(x, axis=1) + epsilon - output = scale * np.divide((x - mean.reshape([N, 1])), - (np.sqrt(var)).reshape([N, 1])) + beta + output = ( + scale + * np.divide((x - mean.reshape([N, 1])), (np.sqrt(var)).reshape([N, 1])) + + beta + ) x.shape, output.shape = x_shape, x_shape return output, mean, var class TestLayerNormMKLDNNOp(unittest.TestCase): - def setUp(self): self.use_mkldnn = True def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=1e-05, - atol=atol, - err_msg=msg) - - def check_forward(self, - shape, - begin_norm_axis, - with_scale_bias=True, - with_is_test=False): + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=1e-05, atol=atol, err_msg=msg + ) + + def check_forward( + self, shape, begin_norm_axis, with_scale_bias=True, with_is_test=False + ): # attr epsilon = 0.00001 x_shape = shape - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) scale_shape = [D] np.random.seed(123) @@ -84,8 +85,9 @@ def check_forward(self, bias = np.array([]) # reference forward & backward - y, mean, variance = _reference_layer_norm_naive(x, scale, bias, epsilon, - begin_norm_axis) + y, mean, variance = _reference_layer_norm_naive( + x, scale, bias, epsilon, begin_norm_axis + ) var_dict = locals() var_names = ['x', 'mean', 'variance', 'y'] @@ -99,9 +101,9 @@ def check_forward(self, block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, dtype='float32', shape=ground_truth[name].shape + ) inputs = {"X": block.var('x')} if with_scale_bias: @@ -120,8 +122,9 @@ def check_forward(self, "epsilon": epsilon, "begin_norm_axis": begin_norm_axis, "use_mkldnn": True, - "is_test": with_is_test - }) + "is_test": with_is_test, + }, + ) exe = fluid.Executor(core.CPUPlace()) @@ -130,10 +133,11 @@ def check_forward(self, input_list.append('scale') input_list.append('bias') - out = exe.run(program, - feed={name: var_dict[name] - for name in input_list}, - fetch_list=['y', 'mean', 'variance']) + out = exe.run( + program, + feed={name: var_dict[name] for name in input_list}, + fetch_list=['y', 'mean', 'variance'], + ) self.__assert_close(y, out[0], "y") if not with_is_test: self.__assert_close(mean, out[1], "mean") @@ -147,14 +151,14 @@ def test_check_forward_with_scale_and_bias(self): self.check_forward(shape=[2, 3, 4, 5], begin_norm_axis=3) def test_check_forward_without_scale_and_bias(self): - self.check_forward(shape=[2, 3, 4, 5], - begin_norm_axis=3, - with_scale_bias=False) + self.check_forward( + shape=[2, 3, 4, 5], begin_norm_axis=3, with_scale_bias=False + ) def test_check_forward_with_is_test(self): - self.check_forward(shape=[2, 3, 4, 5], - begin_norm_axis=3, - with_is_test=True) + self.check_forward( + shape=[2, 3, 4, 5], begin_norm_axis=3, with_is_test=True + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_log_softmax_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_log_softmax_mkldnn_op.py index 89de5198101c2..4279ffc5915fc 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_log_softmax_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_log_softmax_mkldnn_op.py @@ -17,12 +17,15 @@ import paddle from paddle.fluid import core from paddle.fluid.tests.unittests.test_log_softmax import ref_log_softmax -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) @OpTestTool.skip_if_not_cpu_bf16() class TestLogSoftmaxOneDNNOp(OpTest): - def setUp(self): self.op_type = 'log_softmax' self.set_dtype() @@ -53,45 +56,39 @@ def test_check_output(self): class TestLogSoftmax1DOneDNNOp(TestLogSoftmaxOneDNNOp): - def set_shape(self): self.shape = [100] class TestLogSoftmax3DOneDNNOp(TestLogSoftmaxOneDNNOp): - def set_shape(self): self.shape = [12, 10, 3] class TestLogSoftmax5DOneDNNOp(TestLogSoftmaxOneDNNOp): - def set_shape(self): self.shape = [2, 3, 4, 5, 6] class TestLogSoftmaxPositiveAxisOneDNNOp(TestLogSoftmaxOneDNNOp): - def set_axis(self): self.axis = 2 # BF16 TESTS class TestLogSoftmax1DBF16OneDNNOp(TestLogSoftmax1DOneDNNOp): - def set_dtype(self): self.dtype = np.uint16 -class TestLogSoftmaxPositiveAxisBF16OneDNNOp(TestLogSoftmaxPositiveAxisOneDNNOp - ): - +class TestLogSoftmaxPositiveAxisBF16OneDNNOp( + TestLogSoftmaxPositiveAxisOneDNNOp +): def set_dtype(self): self.dtype = np.uint16 class TestLogSoftmax5DBF16OneDNNOp(TestLogSoftmax5DOneDNNOp): - def set_shape(self): self.shape = [2, 3, 4, 5, 6] diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_lrn_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_lrn_mkldnn_op.py index 8f3913a86f421..f81a0d22aa760 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_lrn_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_lrn_mkldnn_op.py @@ -17,7 +17,6 @@ class TestLRNMKLDNNOp(TestLRNOp): - def get_attrs(self): attrs = TestLRNOp.get_attrs(self) attrs['use_mkldnn'] = True @@ -26,36 +25,31 @@ def get_attrs(self): def test_check_output(self): # We cannot validate MidOut as LRN REF has diffrent meaning in it # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output(atol=0.002, - no_check_set=['MidOut'], - check_dygraph=False) + self.check_output( + atol=0.002, no_check_set=['MidOut'], check_dygraph=False + ) def test_check_grad_normal(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad(['X'], - 'Out', - max_relative_error=0.01, - check_dygraph=False) + self.check_grad( + ['X'], 'Out', max_relative_error=0.01, check_dygraph=False + ) class TestLRNMKLDNNOpWithIsTest(TestLRNMKLDNNOp): - def get_attrs(self): attrs = TestLRNMKLDNNOp.get_attrs(self) attrs['is_test'] = True return attrs def test_check_grad_normal(self): - def check_raise_is_test(): try: - self.check_grad(['X'], - 'Out', - max_relative_error=0.01, - check_dygraph=False) + self.check_grad( + ['X'], 'Out', max_relative_error=0.01, check_dygraph=False + ) except Exception as e: - t = \ - "is_test attribute should be set to False in training phase." + t = "is_test attribute should be set to False in training phase." if t in str(e): raise AttributeError @@ -63,12 +57,12 @@ def check_raise_is_test(): class TestLRNMKLDNNOpNHWC(TestLRNMKLDNNOp): - def init_test_case(self): self.data_format = 'NHWC' if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_bf16_mkldnn_op.py index 8940502b80662..cdb9dfd8ee838 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_bf16_mkldnn_op.py @@ -19,10 +19,10 @@ from paddle import enable_static -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestMatmulBf16MklDNNOp(OpTest): - def generate_data(self): self.x_fp32 = np.random.random((25, 2, 2)).astype(np.float32) self.y_fp32 = np.random.random((25, 2, 2)).astype(np.float32) @@ -35,7 +35,7 @@ def set_attributes(self): "mkldnn_data_type": self.mkldnn_data_type, "force_fp32_output": self.force_fp32_output, 'transpose_X': False, - 'transpose_Y': False + 'transpose_Y': False, } def setUp(self): @@ -62,11 +62,13 @@ def test_check_output(self): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X", "Y"], + core.CPUPlace(), + ["X", "Y"], "Out", check_dygraph=False, user_defined_grads=[self.dx, self.dy], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) def matmul_grad(self, x, transpose_x, y, transpose_y): x_transpose_axes = [1, 0] if x.ndim == 2 else [0, 2, 1] @@ -81,23 +83,35 @@ def calculate_grads(self): x_transpose_axes = [1, 0] if self.x_fp32.ndim == 2 else [0, 2, 1] y_transpose_axes = [1, 0] if self.y_fp32.ndim == 2 else [0, 2, 1] - x = np.transpose(self.x_fp32, x_transpose_axes - ) if self.attrs['transpose_X'] is True else self.x_fp32 - y = np.transpose(self.y_fp32, y_transpose_axes - ) if self.attrs['transpose_Y'] is True else self.y_fp32 + x = ( + np.transpose(self.x_fp32, x_transpose_axes) + if self.attrs['transpose_X'] is True + else self.x_fp32 + ) + y = ( + np.transpose(self.y_fp32, y_transpose_axes) + if self.attrs['transpose_Y'] is True + else self.y_fp32 + ) dout = self.alpha * np.matmul(x, y) - if self.attrs['transpose_X'] is True and self.attrs[ - 'transpose_Y'] is True: + if ( + self.attrs['transpose_X'] is True + and self.attrs['transpose_Y'] is True + ): self.dx = self.matmul_grad(self.y_fp32, True, dout, True) self.dy = self.matmul_grad(dout, True, self.x_fp32, True) - elif self.attrs['transpose_X'] is True and self.attrs[ - 'transpose_Y'] is False: + elif ( + self.attrs['transpose_X'] is True + and self.attrs['transpose_Y'] is False + ): self.dx = self.matmul_grad(self.y_fp32, False, dout, True) self.dy = self.matmul_grad(self.x_fp32, False, dout, False) - elif self.attrs['transpose_X'] is False and self.attrs[ - 'transpose_Y'] is True: + elif ( + self.attrs['transpose_X'] is False + and self.attrs['transpose_Y'] is True + ): self.dx = self.matmul_grad(dout, False, self.y_fp32, False) self.dy = self.matmul_grad(dout, True, self.x_fp32, False) else: @@ -108,7 +122,6 @@ def calculate_grads(self): class TestDnnlMatMulOpAlpha(TestMatmulBf16MklDNNOp): - def generate_data(self): self.x_fp32 = np.random.random((17, 2, 3)).astype(np.float32) self.y_fp32 = np.random.random((17, 3, 2)).astype(np.float32) @@ -117,7 +130,6 @@ def generate_data(self): class TestDnnlMatMulOp2D(TestMatmulBf16MklDNNOp): - def generate_data(self): self.x_fp32 = np.random.random((12, 9)).astype(np.float32) self.y_fp32 = np.random.random((9, 12)).astype(np.float32) @@ -125,7 +137,6 @@ def generate_data(self): class TestDnnlMatMulOpTransposeX(TestMatmulBf16MklDNNOp): - def generate_data(self): self.x_fp32 = np.random.random((12, 9)).astype(np.float32) self.y_fp32 = np.random.random((12, 9)).astype(np.float32) @@ -136,12 +147,11 @@ def set_attributes(self): "use_mkldnn": self.use_mkldnn, "mkldnn_data_type": self.mkldnn_data_type, 'transpose_X': True, - 'transpose_Y': False + 'transpose_Y': False, } class TestDnnlMatMulOpTransposeY(TestMatmulBf16MklDNNOp): - def generate_data(self): self.x_fp32 = np.random.random((12, 9)).astype(np.float32) self.y_fp32 = np.random.random((12, 9)).astype(np.float32) @@ -152,12 +162,11 @@ def set_attributes(self): "use_mkldnn": self.use_mkldnn, "mkldnn_data_type": self.mkldnn_data_type, 'transpose_Y': True, - 'transpose_X': False + 'transpose_X': False, } class TestMatmulBf16MklDNNForceFp32Output(TestMatmulBf16MklDNNOp): - def generate_data(self): self.x_fp32 = np.random.random((12, 9)).astype(np.float32) self.y_fp32 = np.random.random((9, 12)).astype(np.float32) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_mkldnn_op.py index ffaa725614b77..4d4b6a384a5ad 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_mkldnn_op.py @@ -18,7 +18,6 @@ class TestDnnlMatMulOp(OpTest): - def generate_data(self): self.x = np.random.random((25, 2, 2)).astype("float32") self.y = np.random.random((25, 2, 2)).astype("float32") @@ -47,13 +46,11 @@ def test_check_output(self): class TestDnnlMatMulWithGradOp(TestDnnlMatMulOp): - def test_check_grad(self): self.check_grad(['X', 'Y'], 'Out', max_relative_error=1e-2) class TestDnnlMatMulOpMixedDims1(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((17, 2, 3)).astype("float32") self.y = np.random.random((3, 4)).astype("float32") @@ -61,7 +58,6 @@ def generate_data(self): class TestDnnlMatMulOpMixedDimsYWiderTransposeY(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((8, 2, 3)).astype("float32") self.y = np.random.random((4, 3)).astype("float32") @@ -72,7 +68,6 @@ def set_attributes(self): class TestDnnlMatMulOpMixedDimsYWiderTransposeX(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((8, 3, 2)).astype("float32") self.y = np.random.random((3, 4)).astype("float32") @@ -83,31 +78,30 @@ def set_attributes(self): class TestDnnlMatMulOpMixedDimsXWiderTransposeXY(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((8, 3, 2)).astype("float32") self.y = np.random.random((4, 3)).astype("float32") - self.out = np.matmul(np.transpose(self.x, (0, 2, 1)), - np.transpose(self.y)) + self.out = np.matmul( + np.transpose(self.x, (0, 2, 1)), np.transpose(self.y) + ) def set_attributes(self): self.attrs = {'transpose_X': True, 'transpose_Y': True} class TestDnnlMatMulOpMixedDimsYWiderTransposeXY(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((3, 2)).astype("float32") self.y = np.random.random((8, 4, 3)).astype("float32") - self.out = np.matmul(np.transpose(self.x), - np.transpose(self.y, (0, 2, 1))) + self.out = np.matmul( + np.transpose(self.x), np.transpose(self.y, (0, 2, 1)) + ) def set_attributes(self): self.attrs = {'transpose_X': True, 'transpose_Y': True} class TestDnnlMatMulOpMixedDimsXWiderTransposeX(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((5, 4)).astype("float32") self.y = np.random.random((8, 5, 4)).astype("float32") @@ -118,7 +112,6 @@ def set_attributes(self): class TestDnnlMatMulOpVectorMultiply(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((5)).astype("float32") self.y = np.random.random((5)).astype("float32") @@ -126,7 +119,6 @@ def generate_data(self): class TestDnnlMatMulOpVectorMultiplyTranspose(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((5)).astype("float32") x_resized = np.copy(self.x) @@ -141,7 +133,6 @@ def set_attributes(self): class TestDnnlMatMulOpMixedDims2(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((2, 3)).astype("float32") self.y = np.random.random((17, 3, 4)).astype("float32") @@ -149,7 +140,6 @@ def generate_data(self): class TestDnnlMatMulOpAlpha(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((17, 2, 3)).astype("float32") self.y = np.random.random((17, 3, 2)).astype("float32") @@ -158,7 +148,6 @@ def generate_data(self): class TestDnnlMatMulOp2D(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((12, 9)).astype("float32") self.y = np.random.random((9, 12)).astype("float32") @@ -166,7 +155,6 @@ def generate_data(self): class TestDnnlMatMulOpTransposeX(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((12, 9)).astype("float32") self.y = np.random.random((12, 9)).astype("float32") @@ -177,7 +165,6 @@ def set_attributes(self): class TestDnnlMatMulOpTransposeY(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((12, 9)).astype("float32") self.y = np.random.random((12, 9)).astype("float32") @@ -188,7 +175,6 @@ def set_attributes(self): class TestDnnlMatMulOpTransposeY3D(TestDnnlMatMulWithGradOp): - def generate_data(self): self.x = np.random.random((17, 3, 2)).astype("float32") self.y = np.random.random((17, 3, 2)).astype("float32") @@ -199,7 +185,6 @@ def set_attributes(self): class TestDnnlMatMulOpInt8NoScales(TestDnnlMatMulOp): - def generate_data(self): self.x = np.random.random((12, 9)).astype("int8") self.y = np.random.random((9, 12)).astype("int8") @@ -211,7 +196,7 @@ class TestDnnlMatMulOpInt8(TestDnnlMatMulOp): # on older platforms (BDW, SKX) we needed to reduce # range from [-127, 127] to [-63, 63] def quantize(self, tensor): - scale = 63. / np.abs(np.amax(tensor)) + scale = 63.0 / np.abs(np.amax(tensor)) quantized = np.round(scale * tensor).astype("int8") return scale, quantized @@ -238,7 +223,6 @@ def test_check_output(self): class TestDnnlMatMulOpInt8ForceFP32(TestDnnlMatMulOpInt8): - def generate_data(self): x_float = np.random.random((12, 9)).astype("float32") self.x_scale, self.x = self.quantize(x_float) @@ -253,12 +237,11 @@ def set_attributes(self): self.attrs = { 'Scale_x': self.x_scale, 'Scale_y': self.y_scale, - 'force_fp32_output': True + 'force_fp32_output': True, } class TestDnnlMatMulOpInt8ForceFP32BasicScales(TestDnnlMatMulOp): - def generate_data(self): self.x = np.random.randint(0, 3, (12, 9)).astype("int8") self.y = np.random.randint(0, 3, (9, 12)).astype("int8") @@ -270,15 +253,22 @@ def set_attributes(self): @skip_check_grad_ci(reason="DNNL's MatMul doesn't implement grad kernel.") class TestReshapeTransposeMatMulOp(OpTest): - def init_data_type(self): self.data_type_ = 'float32' def generate_data(self): - self.x = np.random.random([2, 128, 768]).astype("float32").reshape( - [2, 128, 12, 64]).transpose([0, 2, 1, 3]) - self.y = np.random.random([2, 128, 768]).astype("float32").reshape( - [2, 128, 12, 64]).transpose([0, 2, 1, 3]) + self.x = ( + np.random.random([2, 128, 768]) + .astype("float32") + .reshape([2, 128, 12, 64]) + .transpose([0, 2, 1, 3]) + ) + self.y = ( + np.random.random([2, 128, 768]) + .astype("float32") + .reshape([2, 128, 12, 64]) + .transpose([0, 2, 1, 3]) + ) self.out = np.matmul(self.x, self.y.transpose([0, 1, 3, 2])) self.fused_reshape_X = [] self.fused_transpose_X = [] @@ -300,7 +290,7 @@ def setUp(self): self.inputs = {'X': self.x, 'Y': self.y} self.attrs = { 'use_mkldnn': self.use_mkldnn, - self.transpose_y_name: self.transpose_y + self.transpose_y_name: self.transpose_y, } if len(self.fused_transpose_X) > 0: self.attrs['fused_transpose_X'] = self.fused_transpose_X @@ -318,49 +308,53 @@ def test_check_output(self): class TestReshapeTransposeMatMulOp4DXFloat(TestReshapeTransposeMatMulOp): - def generate_data(self): self.x = np.random.random([2, 128, 768]).astype("float32") - self.y = np.random.random([2, 128, 768]).astype("float32").reshape( - [2, 128, 12, 64]).transpose([0, 2, 1, 3]) + self.y = ( + np.random.random([2, 128, 768]) + .astype("float32") + .reshape([2, 128, 12, 64]) + .transpose([0, 2, 1, 3]) + ) self.fused_transpose_X = [0, 2, 1, 3] self.fused_reshape_X = [0, 0, 12, 64] self.fused_transpose_Y = [] self.fused_reshape_Y = [] self.out = np.matmul( self.x.reshape([2, 128, 12, 64]).transpose([0, 2, 1, 3]), - self.y.transpose([0, 1, 3, 2])) + self.y.transpose([0, 1, 3, 2]), + ) class TestReshapeTransposeMatMulOp4DXInt8(TestReshapeTransposeMatMulOp4DXFloat): - def init_data_type(self): self.data_type_ = 'int8' class TestReshapeTransposeMatMulOp4DYFloat(TestReshapeTransposeMatMulOp): - def generate_data(self): - self.x = np.random.random([2, 128, 768]).astype("float32").reshape( - [2, 128, 12, 64]).transpose([0, 2, 1, 3]) + self.x = ( + np.random.random([2, 128, 768]) + .astype("float32") + .reshape([2, 128, 12, 64]) + .transpose([0, 2, 1, 3]) + ) self.y = np.random.random([2, 128, 768]).astype("float32") self.fused_transpose_X = [] self.fused_reshape_X = [] self.fused_transpose_Y = [0, 2, 1, 3] self.fused_reshape_Y = [0, 0, 12, 64] self.out = np.matmul( - self.x, - self.y.reshape([2, 128, 12, 64]).transpose([0, 2, 3, 1])) + self.x, self.y.reshape([2, 128, 12, 64]).transpose([0, 2, 3, 1]) + ) class TestReshapeTransposeMatMulOp4DYInt8(TestReshapeTransposeMatMulOp4DYFloat): - def init_data_type(self): self.data_type_ = 'int8' class TestReshapeTransposeMatMulOp4DXYFloat(TestReshapeTransposeMatMulOp): - def generate_data(self): self.x = np.random.random([2, 128, 768]).astype("float32") self.y = np.random.random([2, 128, 768]).astype("float32") @@ -370,42 +364,48 @@ def generate_data(self): self.fused_reshape_Y = [0, 0, 12, 64] self.out = np.matmul( self.x.reshape([2, 128, 12, 64]).transpose([0, 2, 1, 3]), - self.y.reshape([2, 128, 12, 64]).transpose([0, 2, 3, 1])) - + self.y.reshape([2, 128, 12, 64]).transpose([0, 2, 3, 1]), + ) -class TestReshapeTransposeMatMulOp4DXYInt8(TestReshapeTransposeMatMulOp4DXYFloat - ): +class TestReshapeTransposeMatMulOp4DXYInt8( + TestReshapeTransposeMatMulOp4DXYFloat +): def init_data_type(self): self.data_type_ = 'int8' class TestReshapeTransposeMatMulOp2DXFloat(TestReshapeTransposeMatMulOp): - def generate_data(self): self.x = np.random.random([2, 5, 10]).astype("float32") - self.y = np.random.random([2, 5, 10]).astype("float32").reshape( - [10, 10]).transpose([1, 0]) + self.y = ( + np.random.random([2, 5, 10]) + .astype("float32") + .reshape([10, 10]) + .transpose([1, 0]) + ) self.fused_transpose_X = [1, 0] self.fused_reshape_X = [10, 10] self.fused_transpose_Y = [] self.fused_reshape_Y = [] self.out = np.matmul( - self.x.reshape([10, 10]).transpose([1, 0]), self.y.transpose([1, - 0])) + self.x.reshape([10, 10]).transpose([1, 0]), self.y.transpose([1, 0]) + ) class TestReshapeTransposeMatMulOp2DXInt8(TestReshapeTransposeMatMulOp2DXFloat): - def init_data_type(self): self.data_type_ = 'int8' class TestReshapeTransposeMatMulOp2DYFloat(TestReshapeTransposeMatMulOp): - def generate_data(self): - self.x = np.random.random([2, 5, 10]).astype("float32").reshape( - [10, 10]).transpose([1, 0]) + self.x = ( + np.random.random([2, 5, 10]) + .astype("float32") + .reshape([10, 10]) + .transpose([1, 0]) + ) self.y = np.random.random([2, 5, 10]).astype("float32") self.fused_transpose_X = [] self.fused_reshape_X = [] @@ -415,37 +415,42 @@ def generate_data(self): class TestReshapeTransposeMatMulOp2DYInt8(TestReshapeTransposeMatMulOp2DYFloat): - def init_data_type(self): self.data_type_ = 'int8' class TestReshapeTransposeMatMulOp3DXFloat(TestReshapeTransposeMatMulOp): - def generate_data(self): self.x = np.random.random([2, 2, 5, 5]).astype("float32") - self.y = np.random.random([2, 2, 5, 5]).astype("float32").reshape( - [2, 10, 5]).transpose([0, 2, 1]) + self.y = ( + np.random.random([2, 2, 5, 5]) + .astype("float32") + .reshape([2, 10, 5]) + .transpose([0, 2, 1]) + ) self.fused_transpose_X = [0, 2, 1] self.fused_reshape_X = [2, 10, 5] self.fused_transpose_Y = [] self.fused_reshape_Y = [] self.out = np.matmul( self.x.reshape([2, 10, 5]).transpose(0, 2, 1), - self.y.transpose(0, 2, 1)) + self.y.transpose(0, 2, 1), + ) class TestReshapeTransposeMatMulOp3DXInt8(TestReshapeTransposeMatMulOp3DXFloat): - def init_data_type(self): self.data_type_ = 'int8' class TestReshapeTransposeMatMulOp3DYFloat(TestReshapeTransposeMatMulOp): - def generate_data(self): - self.x = np.random.random([2, 2, 5, 5]).astype(self.data_type_).reshape( - [2, 10, 5]).transpose([0, 2, 1]) + self.x = ( + np.random.random([2, 2, 5, 5]) + .astype(self.data_type_) + .reshape([2, 10, 5]) + .transpose([0, 2, 1]) + ) self.y = np.random.random([2, 2, 5, 5]).astype(self.data_type_) self.fused_transpose_X = [] self.fused_reshape_X = [] @@ -455,14 +460,12 @@ def generate_data(self): class TestReshapeTransposeMatMulOp3DYInt8(TestReshapeTransposeMatMulOp3DYFloat): - def init_data_type(self): self.data_type_ = 'int8' @skip_check_grad_ci(reason="Tests inference only optimization.") class TestMatMulOpTransposeReshapeEmptyFloat(OpTest): - def init_data_type(self): self.data_type_ = np.float32 @@ -512,39 +515,44 @@ def check_raise_error(self, msg): class TestMatMulOpTransposeReshapeIntEmptyInt( - TestMatMulOpTransposeReshapeEmptyFloat): - + TestMatMulOpTransposeReshapeEmptyFloat +): def init_data_type(self): self.data_type_ = np.int8 class TestMatMulOpTransposeReshapeBasicFloat( - TestMatMulOpTransposeReshapeEmptyFloat): - + TestMatMulOpTransposeReshapeEmptyFloat +): def generate_data(self): self.bs = 8 - self.x = np.random.random([self.bs, 12, 128, - 128]).astype(self.data_type_) - self.y = np.random.random([self.bs, 12, 128, - 64]).astype(self.data_type_) + self.x = np.random.random([self.bs, 12, 128, 128]).astype( + self.data_type_ + ) + self.y = np.random.random([self.bs, 12, 128, 64]).astype( + self.data_type_ + ) def init_params_and_out(self): self.transpose_out = [0, 2, 1, 3] self.reshape_out = [0, 0, self.x.shape[1] * self.y.shape[-1]] - self.out = np.matmul(self.x, self.y).transpose([0, 2, 1, 3]).reshape( - [self.bs, -1, self.x.shape[1] * self.y.shape[-1]]) + self.out = ( + np.matmul(self.x, self.y) + .transpose([0, 2, 1, 3]) + .reshape([self.bs, -1, self.x.shape[1] * self.y.shape[-1]]) + ) class TestMatMulOpTransposeReshapeBasicInt( - TestMatMulOpTransposeReshapeBasicFloat): - + TestMatMulOpTransposeReshapeBasicFloat +): def init_data_type(self): self.data_type_ = np.int8 class TestMatMulOpTransposeReshapeOtherDimFloat( - TestMatMulOpTransposeReshapeBasicFloat): - + TestMatMulOpTransposeReshapeBasicFloat +): def generate_data(self): self.bs = 11 self.x = np.random.random([self.bs, 12, 14, 18]).astype(self.data_type_) @@ -552,13 +560,14 @@ def generate_data(self): class TestMatMulOpTransposeReshapeOtherDimInt( - TestMatMulOpTransposeReshapeOtherDimFloat): - + TestMatMulOpTransposeReshapeOtherDimFloat +): def init_data_type(self): self.data_type_ = np.int8 if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_v2_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_v2_mkldnn_op.py index 5038114582116..4a4e0d3ac66ca 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_v2_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_matmul_v2_mkldnn_op.py @@ -15,17 +15,26 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) import paddle.fluid.core as core import paddle from paddle.fluid.tests.unittests.mkldnn.test_matmul_mkldnn_op import ( TestMatMulOpTransposeReshapeEmptyFloat, TestMatMulOpTransposeReshapeBasicFloat, - TestMatMulOpTransposeReshapeOtherDimFloat, TestReshapeTransposeMatMulOp, - TestReshapeTransposeMatMulOp4DXFloat, TestReshapeTransposeMatMulOp4DYFloat, - TestReshapeTransposeMatMulOp4DXYFloat, TestReshapeTransposeMatMulOp2DXFloat, - TestReshapeTransposeMatMulOp2DYFloat, TestReshapeTransposeMatMulOp3DXFloat, - TestReshapeTransposeMatMulOp3DYFloat) + TestMatMulOpTransposeReshapeOtherDimFloat, + TestReshapeTransposeMatMulOp, + TestReshapeTransposeMatMulOp4DXFloat, + TestReshapeTransposeMatMulOp4DYFloat, + TestReshapeTransposeMatMulOp4DXYFloat, + TestReshapeTransposeMatMulOp2DXFloat, + TestReshapeTransposeMatMulOp2DYFloat, + TestReshapeTransposeMatMulOp3DXFloat, + TestReshapeTransposeMatMulOp3DYFloat, +) def reference_matmul(X, Y, transpose_x=False, transpose_y=False): @@ -34,7 +43,7 @@ def reference_matmul(X, Y, transpose_x=False, transpose_y=False): # transpose X and Y appropriately. if transpose_x: if X.ndim == 1: - X = X.reshape((X.size, )) + X = X.reshape((X.size,)) elif X.ndim == 2: X = X.T else: @@ -43,7 +52,7 @@ def reference_matmul(X, Y, transpose_x=False, transpose_y=False): X = np.transpose(X, tuple(dim)) if transpose_y: if Y.ndim == 1: - Y = Y.reshape((Y.size, )) + Y = Y.reshape((Y.size,)) else: dim = [i for i in range(len(Y.shape))] dim[-1], dim[len(Y.shape) - 2] = dim[len(Y.shape) - 2], dim[-1] @@ -54,10 +63,9 @@ def reference_matmul(X, Y, transpose_x=False, transpose_y=False): class TestMatMulV2VectorXVectorOneDNNOp(OpTest): - def config(self): - self.x_shape = (100, ) - self.y_shape = (100, ) + self.x_shape = (100,) + self.y_shape = (100,) self.trans_x = False self.trans_y = False self._cpu_only = True @@ -77,14 +85,15 @@ def setUp(self): # -0.1 ~ 0.1 x = -0.1 + 0.2 * x y = -0.1 + 0.2 * y - result = reference_matmul(x, y, self.trans_x, - self.trans_y).astype("float32") + result = reference_matmul(x, y, self.trans_x, self.trans_y).astype( + "float32" + ) self.set_inputs(x, y) self.attrs = { 'trans_x': self.trans_x, 'trans_y': self.trans_y, - 'use_mkldnn': True + 'use_mkldnn': True, } self.set_dtype_attr() self.outputs = {'Out': result} @@ -97,45 +106,42 @@ def test_check_grad(self): class TestMatMulV2VectorXMatrixTransposeYOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 3, 2, 100) self.trans_x = False self.trans_y = True class TestMatMulV2VectorXMatrixOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 1, 100, 2) self.trans_x = False self.trans_y = False class TestMatMulV2MatrixXVectorTransposeXOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (1, 1, 100, 1) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = True self.trans_y = False class TestMatMulV2MatrixXVectorOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (1, 2, 1, 100) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = False self.trans_y = False class TestMatMulV2MatrixXMatrixOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (1, 1, 2, 100) self.y_shape = (1, 1, 100, 1) @@ -144,8 +150,8 @@ def config(self): class TestMatMulV2MatrixXMatrixTransposeYOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (1, 1, 1, 100) self.y_shape = (2, 1, 2, 100) @@ -154,7 +160,6 @@ def config(self): class TestMatMulV2MatrixXMatrix2OneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (2, 1, 12, 9) self.y_shape = (1, 3, 9, 12) @@ -163,7 +168,6 @@ def config(self): class TestMatMulV2MatrixXMatrix3OneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (2, 1, 2, 100) self.y_shape = (1, 1, 100, 2) @@ -172,8 +176,8 @@ def config(self): class TestMatMulV2MatrixXMatrixTranposeXOneDNNOp2( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (2, 1, 4, 25) self.y_shape = (1, 1, 4, 25) @@ -182,8 +186,8 @@ def config(self): class TestMatMulV2MatrixXMatrixTranposeX2OneDNNOp3( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (2, 2, 7, 4) self.y_shape = (2, 2, 7, 5) @@ -192,8 +196,8 @@ def config(self): class TestMatMulV2MatrixXMatrixTransposeX3OneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (3, 1, 6, 7) self.y_shape = (1, 2, 6, 9) @@ -202,7 +206,6 @@ def config(self): class TestMatMulV2MatrixXMatrix4OneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (3, 1, 6, 6) self.y_shape = (1, 2, 6, 9) @@ -211,26 +214,24 @@ def config(self): class TestMatMulV2VectorXMatrix5DOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): - self.x_shape = (100) + self.x_shape = 100 self.y_shape = (1, 2, 2, 100, 2) self.trans_x = False self.trans_y = False class TestMatMulV2Matrix3DXVectorOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (2, 1, 100) - self.y_shape = (100) + self.y_shape = 100 self.trans_x = False self.trans_y = False class TestMatMulV2MatrixXMatrixTransposeXTransposeYOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (3, 1, 10, 8) self.y_shape = (1, 2, 9, 10) @@ -239,8 +240,8 @@ def config(self): class TestMatMulV2MatrixXMatrixTransposeY2OneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (3, 1, 10, 10) self.y_shape = (1, 2, 9, 10) @@ -249,8 +250,8 @@ def config(self): class TestMatMulV2MatrixXMatrix5DTranposeYOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (1, 3, 1, 10, 10) self.y_shape = (3, 1, 2, 9, 10) @@ -259,7 +260,6 @@ def config(self): class TestMatMulV2MatrixXMatrix6Dx2DOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (1, 1, 2, 1, 8, 9) self.y_shape = (9, 12) @@ -268,7 +268,6 @@ def config(self): class TestMatMulV2MatrixXMatrix2Dx5DOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (20, 5) self.y_shape = (1, 2, 1, 5, 11) @@ -277,8 +276,8 @@ def config(self): class TestMatMulV2MatrixXMatrix4Dx3DTransposeXOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (5, 4, 15, 10) self.y_shape = (1, 15, 20) @@ -287,8 +286,8 @@ def config(self): class TestMatMulV2MatrixXMatrix3Dx4DTransposeYOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (2, 10, 15) self.y_shape = (4, 2, 20, 15) @@ -297,8 +296,8 @@ def config(self): class TestMatMulV2MatrixXMatrix5Dx3DTransposeXTransposeYOneDNNOp( - TestMatMulV2VectorXVectorOneDNNOp): - + TestMatMulV2VectorXVectorOneDNNOp +): def config(self): self.x_shape = (4, 3, 2, 15, 10) self.y_shape = (1, 20, 15) @@ -307,7 +306,6 @@ def config(self): class TestMatMulV2MatrixXMatrix3Dx4DOneDNNOp(TestMatMulV2VectorXVectorOneDNNOp): - def config(self): self.x_shape = (1, 1, 32, 16) self.y_shape = (16, 16, 16) @@ -317,14 +315,12 @@ def config(self): # BF16 TESTS def create_bf16_test_class(parent): - @OpTestTool.skip_if_not_cpu_bf16() class TestMatMulV2Bf16OneDNNOp(parent): - def set_inputs(self, x, y): self.inputs = { 'X': convert_float_to_uint16(x), - 'Y': convert_float_to_uint16(y) + 'Y': convert_float_to_uint16(y), } self.x_fp32 = x self.y_fp32 = y @@ -338,16 +334,24 @@ def test_check_output(self): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X", "Y"], + core.CPUPlace(), + ["X", "Y"], "Out", user_defined_grads=[self.dx, self.dy], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) def matmul_grad(self, x, transpose_x, y, transpose_y): - x = np.transpose( - x, self.shape_transpose_axes[x.ndim]) if transpose_x else x - y = np.transpose( - y, self.shape_transpose_axes[y.ndim]) if transpose_y else y + x = ( + np.transpose(x, self.shape_transpose_axes[x.ndim]) + if transpose_x + else x + ) + y = ( + np.transpose(y, self.shape_transpose_axes[y.ndim]) + if transpose_y + else y + ) return np.matmul(x, y) @@ -357,7 +361,7 @@ def calculate_grads(self): 3: [0, 2, 1], 4: [0, 1, 3, 2], 5: [0, 1, 2, 4, 3], - 6: [0, 1, 2, 3, 5, 4] + 6: [0, 1, 2, 3, 5, 4], } # expand vector so it will be a valid matrix for multiplication @@ -369,10 +373,16 @@ def calculate_grads(self): x_transpose_axes = self.shape_transpose_axes[self.x_fp32.ndim] y_transpose_axes = self.shape_transpose_axes[self.y_fp32.ndim] - x = np.transpose(self.x_fp32, x_transpose_axes - ) if self.attrs['trans_x'] is True else self.x_fp32 - y = np.transpose(self.y_fp32, y_transpose_axes - ) if self.attrs['trans_y'] is True else self.y_fp32 + x = ( + np.transpose(self.x_fp32, x_transpose_axes) + if self.attrs['trans_x'] is True + else self.x_fp32 + ) + y = ( + np.transpose(self.y_fp32, y_transpose_axes) + if self.attrs['trans_y'] is True + else self.y_fp32 + ) dout = np.matmul(x, y) @@ -389,12 +399,14 @@ def calculate_grads(self): if self.attrs['trans_x'] is True and self.attrs['trans_y'] is True: self.dx = self.matmul_grad(self.y_fp32, True, dout, True) self.dy = self.matmul_grad(dout, True, self.x_fp32, True) - elif self.attrs['trans_x'] is True and self.attrs[ - 'trans_y'] is False: + elif ( + self.attrs['trans_x'] is True and self.attrs['trans_y'] is False + ): self.dx = self.matmul_grad(self.y_fp32, False, dout, True) self.dy = self.matmul_grad(self.x_fp32, False, dout, False) - elif self.attrs['trans_x'] is False and self.attrs[ - 'trans_y'] is True: + elif ( + self.attrs['trans_x'] is False and self.attrs['trans_y'] is True + ): self.dx = self.matmul_grad(dout, False, self.y_fp32, False) self.dy = self.matmul_grad(dout, True, self.x_fp32, False) else: @@ -405,21 +417,25 @@ def calculate_grads(self): x_reduce_axis = [] y_reduce_axis = [] for index, (first, second) in enumerate( - zip(x_shape[0:-2], self.dx.shape[0:-2])): + zip(x_shape[0:-2], self.dx.shape[0:-2]) + ): if first != second: x_reduce_axis.append(index) for index, (first, second) in enumerate( - zip(y_shape[0:-2], self.dy.shape[0:-2])): + zip(y_shape[0:-2], self.dy.shape[0:-2]) + ): if first != second: y_reduce_axis.append(index) if x_reduce_axis: - self.dx = self.dx.sum(axis=tuple(x_reduce_axis), - keepdims=True) + self.dx = self.dx.sum( + axis=tuple(x_reduce_axis), keepdims=True + ) if y_reduce_axis: - self.dy = self.dy.sum(axis=tuple(y_reduce_axis), - keepdims=True) + self.dy = self.dy.sum( + axis=tuple(y_reduce_axis), keepdims=True + ) # after multiplying with vector one dimension is deleted from tensor if len(x_shape) == 2 and x_shape[0] == 1: @@ -456,84 +472,83 @@ def calculate_grads(self): class TestMatMulV2OpTransposeReshapeEmptyFloat( - TestMatMulOpTransposeReshapeEmptyFloat): - + TestMatMulOpTransposeReshapeEmptyFloat +): def set_op_type(self): self.op_type = "matmul_v2" class TestMatMulV2OpTransposeReshapeBasicFloat( - TestMatMulOpTransposeReshapeBasicFloat): - + TestMatMulOpTransposeReshapeBasicFloat +): def set_op_type(self): self.op_type = "matmul_v2" class TestMatMulV2OpTransposeReshapeOtherDimFloat( - TestMatMulOpTransposeReshapeOtherDimFloat): - + TestMatMulOpTransposeReshapeOtherDimFloat +): def set_op_type(self): self.op_type = "matmul_v2" class TestMatMulV2OpReshapeTranspose(TestReshapeTransposeMatMulOp): - def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" class TestMatMulV2OpReshapeTranspose4DXFloat( - TestReshapeTransposeMatMulOp4DXFloat): - + TestReshapeTransposeMatMulOp4DXFloat +): def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" class TestMatMulV2OpReshapeTranspose4DYFloat( - TestReshapeTransposeMatMulOp4DYFloat): - + TestReshapeTransposeMatMulOp4DYFloat +): def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" class TestMatMulV2OpReshapeTranspose4DXYFloat( - TestReshapeTransposeMatMulOp4DXYFloat): - + TestReshapeTransposeMatMulOp4DXYFloat +): def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" class TestMatMulV2OpReshapeTranspose2DXFloat( - TestReshapeTransposeMatMulOp2DXFloat): - + TestReshapeTransposeMatMulOp2DXFloat +): def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" class TestMatMulV2OpReshapeTranspose2DYFloat( - TestReshapeTransposeMatMulOp2DYFloat): - + TestReshapeTransposeMatMulOp2DYFloat +): def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" class TestMatMulV2OpReshapeTranspose3DXFloat( - TestReshapeTransposeMatMulOp3DXFloat): - + TestReshapeTransposeMatMulOp3DXFloat +): def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" class TestMatMulV2OpReshapeTranspose3DYFloat( - TestReshapeTransposeMatMulOp3DYFloat): - + TestReshapeTransposeMatMulOp3DYFloat +): def set_op_type_and_transpose_y_name(self): self.op_type = "matmul_v2" self.transpose_y_name = "trans_y" diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_mul_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_mul_int8_mkldnn_op.py index 944cab82c908e..9011b1229bdb0 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_mul_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_mul_int8_mkldnn_op.py @@ -17,17 +17,16 @@ import paddle import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import OpTest, skip_check_grad_ci + ''' test case for s8 * s8 ''' @skip_check_grad_ci( - reason= - "mul_mkldnn_op does not implement grad operator, check_grad is not required." + reason="mul_mkldnn_op does not implement grad operator, check_grad is not required." ) class TestMKLDNNMulOpS8S8(OpTest): - def setUp(self): self.op_type = "mul" self.init_kernel_type() @@ -65,10 +64,9 @@ def init_data(self): quant_B = np.round(B_data * self.scale_y[0]).astype(np.int_) output = np.dot(A_data, quant_B) - scale_output_shift = (self.scale_out) / \ - (self.scale_x * self.scale_y[0]) + scale_output_shift = (self.scale_out) / (self.scale_x * self.scale_y[0]) - if (self.force_fp32): + if self.force_fp32: output = (output * scale_output_shift).astype(self.dsttype) else: output = np.round(output * scale_output_shift).astype(self.dsttype) @@ -78,9 +76,9 @@ def init_data(self): def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output_with_place(core.CPUPlace(), - atol=0, - check_dygraph=False) + self.check_output_with_place( + core.CPUPlace(), atol=0, check_dygraph=False + ) ''' @@ -89,7 +87,6 @@ def test_check_output(self): class TestMKLDNNMulOpS8U8(TestMKLDNNMulOpS8S8): - def init_data_type(self): self.srctype = np.uint8 self.dsttype = np.float32 if self.force_fp32 else np.int8 @@ -101,7 +98,6 @@ def init_data_type(self): class TestMKLDNNMulOpS8S8WithFlatten(TestMKLDNNMulOpS8S8): - def setUp(self): self.op_type = "mul" self.init_kernel_type() @@ -128,8 +124,9 @@ def init_data(self): else: A_data = np.random.randint(0, 127, (3, 4, 4, 3)).astype(np.uint8) - B_data = np.random.uniform(-127, 127, - (2, 6, 1, 2, 3)).astype(np.float32) + B_data = np.random.uniform(-127, 127, (2, 6, 1, 2, 3)).astype( + np.float32 + ) A_data_reshape = A_data.reshape(3 * 4, 4 * 3) B_data_reshape = B_data.reshape(2 * 6, 1 * 2 * 3) @@ -137,10 +134,9 @@ def init_data(self): quant_B = np.round(B_data_reshape * self.scale_y[0]).astype(np.int_) output = np.dot(A_data_reshape, quant_B) - scale_output_shift = (self.scale_out) / \ - (self.scale_x * self.scale_y[0]) + scale_output_shift = (self.scale_out) / (self.scale_x * self.scale_y[0]) - if (self.force_fp32): + if self.force_fp32: output = (output * scale_output_shift).astype(self.dsttype) else: output = np.round(output * scale_output_shift).astype(self.dsttype) @@ -157,7 +153,6 @@ def init_data(self): class TestMKLDNNMulOpS8U8WithFlatten(TestMKLDNNMulOpS8S8WithFlatten): - def init_data_type(self): self.srctype = np.uint8 self.dsttype = np.float32 if self.force_fp32 else np.int8 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_mul_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_mul_mkldnn_op.py index 85a17094fdc36..0752d4a010433 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_mul_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_mul_mkldnn_op.py @@ -16,12 +16,15 @@ import numpy as np import paddle import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16, OpTestTool +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + convert_float_to_uint16, + OpTestTool, +) @OpTestTool.skip_if_not_cpu_bf16() class TestMulOneDNNOp(OpTest): - def setUp(self): self.op_type = "mul" self.attrs = {'use_mkldnn': True} @@ -37,8 +40,10 @@ def setUp(self): self.inputs = {'X': self.x, 'Y': self.y} - output = np.dot(np.reshape(self.x_fp32, self.np_x_shape), - np.reshape(self.y_fp32, self.np_y_shape)) + output = np.dot( + np.reshape(self.x_fp32, self.np_x_shape), + np.reshape(self.y_fp32, self.np_y_shape), + ) self.outputs = {'Out': np.reshape(output, self.out_shape)} def init_shapes_and_attrs(self): @@ -67,7 +72,6 @@ def test_check_grad_ingore_y(self): class TestMulXNumColDims2OneDNNOp(TestMulOneDNNOp): - def init_shapes_and_attrs(self): self.x_shape = (6, 7, 5) self.y_shape = (5, 21) @@ -81,7 +85,6 @@ def init_shapes_and_attrs(self): class TestMulYNumColDims2OneDNNOp(TestMulOneDNNOp): - def init_shapes_and_attrs(self): self.x_shape = (20, 6) self.y_shape = (2, 3, 21) @@ -95,7 +98,6 @@ def init_shapes_and_attrs(self): class TestMulYAndXNumColDims2OneDNNOp(TestMulOneDNNOp): - def init_shapes_and_attrs(self): self.x_shape = (10, 5, 6) self.y_shape = (2, 3, 21) @@ -110,7 +112,6 @@ def init_shapes_and_attrs(self): class TestMulBF16OneDNNOp(TestMulOneDNNOp): - def init_inputs_dtype(self): self.x = convert_float_to_uint16(self.x) self.y = convert_float_to_uint16(self.y) @@ -131,28 +132,34 @@ def calculate_grads(self): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ['X', 'Y'], + core.CPUPlace(), + ['X', 'Y'], 'Out', user_defined_grads=[self.dx, self.dy], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) def test_check_grad_ingore_x(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ['Y'], + core.CPUPlace(), + ['Y'], 'Out', set('X'), user_defined_grads=[self.dy], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) def test_check_grad_ingore_y(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ['X'], + core.CPUPlace(), + ['X'], 'Out', set('Y'), user_defined_grads=[self.dx], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_multi_gru_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_multi_gru_mkldnn_op.py index a5f4583c3e5f6..273138496fce1 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_multi_gru_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_multi_gru_mkldnn_op.py @@ -15,33 +15,55 @@ import unittest import numpy as np from paddle.fluid.tests.unittests.op_test import OpTest -from paddle.fluid.tests.unittests.test_fusion_gru_op import fusion_gru, ACTIVATION +from paddle.fluid.tests.unittests.test_fusion_gru_op import ( + fusion_gru, + ACTIVATION, +) def multi_gru( - x, # T x M - lod, # 1 x N - h0, # N x D - wx, # M x 3D - wh, # D x 3D - bias, # 1 x 3D - origin_mode, - layers): + x, # T x M + lod, # 1 x N + h0, # N x D + wx, # M x 3D + wh, # D x 3D + bias, # 1 x 3D + origin_mode, + layers, +): act_state = ACTIVATION['tanh'] act_gate = ACTIVATION['sigmoid'] input = x for i in range(0, layers * 2, 2): - _, _, _, gru1_out = fusion_gru(input, lod, h0[i], wx[i], wh[i], bias[i], - False, origin_mode, act_state, act_gate) - _, _, _, gru2_out = fusion_gru(input, lod, h0[i + 1], wx[i + 1], - wh[i + 1], bias[i + 1], True, - origin_mode, act_state, act_gate) + _, _, _, gru1_out = fusion_gru( + input, + lod, + h0[i], + wx[i], + wh[i], + bias[i], + False, + origin_mode, + act_state, + act_gate, + ) + _, _, _, gru2_out = fusion_gru( + input, + lod, + h0[i + 1], + wx[i + 1], + wh[i + 1], + bias[i + 1], + True, + origin_mode, + act_state, + act_gate, + ) input = np.concatenate((gru1_out, gru2_out), axis=1) return input class TestMultiGruMkldnnOp(OpTest): - def set_confs(self): pass @@ -96,17 +118,22 @@ def setUp(self): wx.append(np.random.rand(IC, 3 * OC).astype('float32')) wh.append(np.random.rand(OC, 3 * OC).astype('float32')) bias.append( - np.random.rand(1, 3 * OC).astype('float32') if self. - with_bias else np.zeros((1, 3 * OC), dtype='float32')) + np.random.rand(1, 3 * OC).astype('float32') + if self.with_bias + else np.zeros((1, 3 * OC), dtype='float32') + ) h0.append(np.zeros((N, OC), dtype='float32')) - self.inputs['WeightX'] = [('wx' + str(i), wx[i]) - for i in range(self.layers * 2)] - self.inputs['WeightH'] = [('wh' + str(i), wh[i]) - for i in range(self.layers * 2)] + self.inputs['WeightX'] = [ + ('wx' + str(i), wx[i]) for i in range(self.layers * 2) + ] + self.inputs['WeightH'] = [ + ('wh' + str(i), wh[i]) for i in range(self.layers * 2) + ] if self.with_bias: - self.inputs['Bias'] = [('b' + str(i), bias[i]) - for i in range(self.layers * 2)] + self.inputs['Bias'] = [ + ('b' + str(i), bias[i]) for i in range(self.layers * 2) + ] if is_int8: s8_max = 127.0 @@ -114,38 +141,54 @@ def setUp(self): for layer in range(self.layers): OC = self.OCs[layer] for j in range(2): - scale_ur = s8_max / np.max(np.abs( - np.concatenate([ - wx[2 * layer + j][:, :2 * OC], - wh[2 * layer + j].flatten()[:2 * OC * OC].reshape( - OC, 2 * OC) - ], - axis=0)), - axis=0) - scale_o = s8_max / np.max(np.abs( - np.concatenate([ - wx[2 * layer + j][:, 2 * OC:], - wh[2 * layer + j].flatten()[2 * OC * OC:].reshape( - OC, OC) - ], - axis=0)), - axis=0) + scale_ur = s8_max / np.max( + np.abs( + np.concatenate( + [ + wx[2 * layer + j][:, : 2 * OC], + wh[2 * layer + j] + .flatten()[: 2 * OC * OC] + .reshape(OC, 2 * OC), + ], + axis=0, + ) + ), + axis=0, + ) + scale_o = s8_max / np.max( + np.abs( + np.concatenate( + [ + wx[2 * layer + j][:, 2 * OC :], + wh[2 * layer + j] + .flatten()[2 * OC * OC :] + .reshape(OC, OC), + ], + axis=0, + ) + ), + axis=0, + ) scale_weights.append( - np.concatenate([scale_ur, scale_o]).astype('float32')) - self.inputs['Scale_weights'] = [('w_scale' + str(i), - scale_weights[i]) - for i in range(self.layers * 2)] + np.concatenate([scale_ur, scale_o]).astype('float32') + ) + self.inputs['Scale_weights'] = [ + ('w_scale' + str(i), scale_weights[i]) + for i in range(self.layers * 2) + ] self.error_margin = 1e-1 if self.force_fp32_output else 1 - hidden_f32 = multi_gru(x_f32, self.lod, h0, wx, wh, bias, - self.origin_mode, self.layers) + hidden_f32 = multi_gru( + x_f32, self.lod, h0, wx, wh, bias, self.origin_mode, self.layers + ) if self.dtype == 'float32' or self.force_fp32_output: self.outputs = {'Hidden': (hidden_f32, self.lod)} else: hidden_u8 = np.rint(hidden_f32 * scale_data + shift_data).astype( - np.uint8) + np.uint8 + ) self.outputs = {'Hidden': (hidden_u8, self.lod)} self.attrs = { @@ -166,13 +209,11 @@ def test_check_output(self): class TestMultiGruMkldnnOpNoBias(TestMultiGruMkldnnOp): - def set_confs(self): self.with_bias = False class TestMultiGruMkldnnOpLayers2(TestMultiGruMkldnnOp): - def set_confs(self): self.layers = 2 self.ICs = [2, 6] @@ -180,7 +221,6 @@ def set_confs(self): class TestMultiGruMkldnnOpLayers3(TestMultiGruMkldnnOp): - def set_confs(self): self.layers = 3 self.ICs = [2, 6, 12] @@ -188,71 +228,64 @@ def set_confs(self): class TestMultiGruMkldnnOpOriginMode(TestMultiGruMkldnnOp): - def set_confs(self): self.origin_mode = True class TestMultiGruMkldnnInt8Op(TestMultiGruMkldnnOp): - def set_dtype(self): self.dtype = 'int8' class TestMultiGruMkldnnInt8OpForceFP32Output(TestMultiGruMkldnnInt8Op): - def set_force_fp32_output(self): self.force_fp32_output = True class TestMultiGruMkldnnInt8OpNoBias(TestMultiGruMkldnnOpNoBias): - def set_dtype(self): self.dtype = 'int8' class TestMultiGruMkldnnInt8OpNoBiasForceFP32Output( - TestMultiGruMkldnnInt8OpNoBias): - + TestMultiGruMkldnnInt8OpNoBias +): def set_force_fp32_output(self): self.force_fp32_output = True class TestMultiGruMkldnnInt8OpLayers2(TestMultiGruMkldnnOpLayers2): - def set_dtype(self): self.dtype = 'int8' class TestMultiGruMkldnnInt8OpLayers2ForceFP32Output( - TestMultiGruMkldnnInt8OpLayers2): - + TestMultiGruMkldnnInt8OpLayers2 +): def set_force_fp32_output(self): self.force_fp32_output = True class TestMultiGruMkldnnInt8OpLayers3(TestMultiGruMkldnnOpLayers3): - def set_dtype(self): self.dtype = 'int8' class TestMultiGruMkldnnInt8OpLayers3ForceFP32Output( - TestMultiGruMkldnnInt8OpLayers3): - + TestMultiGruMkldnnInt8OpLayers3 +): def set_force_fp32_output(self): self.force_fp32_output = True class TestMultiGruMkldnnInt8OpOriginMode(TestMultiGruMkldnnOpOriginMode): - def set_dtype(self): self.dtype = 'int8' class TestMultiGruMkldnnInt8OpOriginModeForceFP32Output( - TestMultiGruMkldnnInt8OpOriginMode): - + TestMultiGruMkldnnInt8OpOriginMode +): def set_force_fp32_output(self): self.force_fp32_output = True diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_mkldnn_op.py index f60050509e6bd..e6a53f122bb88 100755 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_mkldnn_op.py @@ -18,12 +18,9 @@ from paddle.fluid.tests.unittests.op_test import skip_check_grad_ci -def nearest_neighbor_interp_mkldnn_np(X, - out_h, - out_w, - out_size=None, - actual_shape=None, - data_layout='NCHW'): +def nearest_neighbor_interp_mkldnn_np( + X, out_h, out_w, out_size=None, actual_shape=None, data_layout='NCHW' +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 3, 1, 2)) # NHWC => NCHW @@ -37,9 +34,9 @@ def nearest_neighbor_interp_mkldnn_np(X, n, c, in_h, in_w = X.shape fh = fw = 0.0 - if (out_h > 1): + if out_h > 1: fh = out_h * 1.0 / in_h - if (out_w > 1): + if out_w > 1: fw = out_w * 1.0 / in_w out = np.zeros((n, c, out_h, out_w)) @@ -58,7 +55,6 @@ def nearest_neighbor_interp_mkldnn_np(X, @skip_check_grad_ci(reason="Haven not implement interpolate grad kernel.") class TestNearestInterpMKLDNNOp(OpTest): - def init_test_case(self): pass @@ -86,8 +82,9 @@ def setUp(self): input_np = np.random.random(self.input_shape).astype(self.dtype) else: init_low, init_high = (-5, 5) if self.dtype == np.int8 else (0, 10) - input_np = np.random.randint(init_low, init_high, - self.input_shape).astype(self.dtype) + input_np = np.random.randint( + init_low, init_high, self.input_shape + ).astype(self.dtype) if self.data_layout == "NCHW": in_h = self.input_shape[2] @@ -103,10 +100,14 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = nearest_neighbor_interp_mkldnn_np(input_np, out_h, out_w, - self.out_size, - self.actual_shape, - self.data_layout) + output_np = nearest_neighbor_interp_mkldnn_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: @@ -119,7 +120,7 @@ def setUp(self): 'out_w': self.out_w, 'scale': self.scale, 'data_layout': self.data_layout, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.outputs = {'Out': output_np} @@ -128,7 +129,6 @@ def test_check_output(self): class TestNearestInterpOpMKLDNNNHWC(TestNearestInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [3, 2, 32, 16] self.out_h = 27 @@ -138,40 +138,36 @@ def init_test_case(self): class TestNearestNeighborInterpMKLDNNCase2(TestNearestInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 1. + self.scale = 1.0 class TestNearestNeighborInterpCase3(TestNearestInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 128 - self.scale = 0. + self.scale = 0.0 class TestNearestNeighborInterpCase4(TestNearestInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 129]).astype("int32") class TestNearestNeighborInterpSame(TestNearestInterpMKLDNNOp): - def init_test_case(self): self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 def create_test_class(parent): @@ -180,12 +176,10 @@ def create_test_class(parent): ''' class TestInt8Case(parent): - def init_data_type(self): self.dtype = np.int8 class TestUint8Case(parent): - def init_data_type(self): self.dtype = np.uint8 @@ -204,5 +198,6 @@ def init_data_type(self): if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_v2_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_v2_mkldnn_op.py index 6e5601e91b02f..6d5480a03f598 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_v2_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_nearest_interp_v2_mkldnn_op.py @@ -14,16 +14,17 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) from paddle.fluid.tests.unittests.op_test import skip_check_grad_ci -def nearest_neighbor_interp_mkldnn_np(X, - out_h, - out_w, - out_size=None, - actual_shape=None, - data_layout='NCHW'): +def nearest_neighbor_interp_mkldnn_np( + X, out_h, out_w, out_size=None, actual_shape=None, data_layout='NCHW' +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 3, 1, 2)) # NHWC => NCHW @@ -37,9 +38,9 @@ def nearest_neighbor_interp_mkldnn_np(X, n, c, in_h, in_w = X.shape fh = fw = 0.0 - if (out_h > 1): + if out_h > 1: fh = out_h * 1.0 / in_h - if (out_w > 1): + if out_w > 1: fw = out_w * 1.0 / in_w out = np.zeros((n, c, out_h, out_w)) @@ -59,7 +60,6 @@ def nearest_neighbor_interp_mkldnn_np(X, @skip_check_grad_ci(reason="Haven not implement interpolate grad kernel.") @OpTestTool.skip_if_not_cpu_bf16() class TestNearestInterpV2MKLDNNOp(OpTest): - def init_test_case(self): pass @@ -87,8 +87,9 @@ def setUp(self): input_np = np.random.random(self.input_shape).astype(self.dtype) else: init_low, init_high = (-5, 5) if self.dtype == np.int8 else (0, 10) - input_np = np.random.randint(init_low, init_high, - self.input_shape).astype(self.dtype) + input_np = np.random.randint( + init_low, init_high, self.input_shape + ).astype(self.dtype) if self.data_layout == "NCHW": in_h = self.input_shape[2] @@ -118,10 +119,14 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = nearest_neighbor_interp_mkldnn_np(input_np, out_h, out_w, - self.out_size, - self.actual_shape, - self.data_layout) + output_np = nearest_neighbor_interp_mkldnn_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.data_layout, + ) if isinstance(self.scale, float): self.scale = [self.scale] @@ -140,7 +145,7 @@ def setUp(self): 'out_w': self.out_w, 'scale': self.scale, 'data_layout': self.data_layout, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.outputs = {'Out': output_np} @@ -149,7 +154,6 @@ def test_check_output(self): class TestNearestInterpOpV2MKLDNNNHWC(TestNearestInterpV2MKLDNNOp): - def init_test_case(self): self.input_shape = [3, 2, 32, 16] self.out_h = 27 @@ -159,7 +163,6 @@ def init_test_case(self): class TestNearestNeighborInterpV2MKLDNNCase2(TestNearestInterpV2MKLDNNOp): - def init_test_case(self): self.input_shape = [3, 3, 9, 6] self.out_h = 12 @@ -167,7 +170,6 @@ def init_test_case(self): class TestNearestNeighborInterpV2MKLDNNCase3(TestNearestInterpV2MKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 @@ -176,7 +178,6 @@ def init_test_case(self): class TestNearestNeighborInterpV2MKLDNNCase4(TestNearestInterpV2MKLDNNOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 @@ -186,7 +187,6 @@ def init_test_case(self): class TestNearestNeighborInterpV2MKLDNNSame(TestNearestInterpV2MKLDNNOp): - def init_test_case(self): self.input_shape = [2, 3, 32, 64] self.out_h = 32 @@ -200,17 +200,14 @@ def create_test_class(parent): ''' class TestBf16Case(parent): - def init_data_type(self): self.dtype = np.uint16 class TestInt8Case(parent): - def init_data_type(self): self.dtype = np.int8 class TestUint8Case(parent): - def init_data_type(self): self.dtype = np.uint8 @@ -231,5 +228,6 @@ def init_data_type(self): if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_onnx_format_quantization_mobilenetv1.py b/python/paddle/fluid/tests/unittests/mkldnn/test_onnx_format_quantization_mobilenetv1.py index ae4a8c16d67de..1b27a39f2e956 100755 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_onnx_format_quantization_mobilenetv1.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_onnx_format_quantization_mobilenetv1.py @@ -74,13 +74,14 @@ def process_image(sample, mode, color_jitter, rotate): return img, sample[1] -def _reader_creator(file_list, - mode, - shuffle=False, - color_jitter=False, - rotate=False, - data_dir=DATA_DIR): - +def _reader_creator( + file_list, + mode, + shuffle=False, + color_jitter=False, + rotate=False, + data_dir=DATA_DIR, +): def reader(): with open(file_list) as flist: full_lines = [line.strip() for line in flist] @@ -95,10 +96,9 @@ def reader(): continue yield img_path, int(label) - mapper = functools.partial(process_image, - mode=mode, - color_jitter=color_jitter, - rotate=rotate) + mapper = functools.partial( + process_image, mode=mode, color_jitter=color_jitter, rotate=rotate + ) return paddle.reader.xmap_readers(mapper, reader, THREAD, BUF_SIZE) @@ -109,11 +109,11 @@ def val(data_dir=DATA_DIR): class TestPostTrainingQuantization(unittest.TestCase): - def setUp(self): self.int8_download = 'int8/download' - self.cache_folder = os.path.expanduser('~/.cache/paddle/dataset/' + - self.int8_download) + self.cache_folder = os.path.expanduser( + '~/.cache/paddle/dataset/' + self.int8_download + ) self.data_cache_folder = '' data_urls = [] data_md5s = [] @@ -126,27 +126,32 @@ def setUp(self): 'https://paddle-inference-dist.bj.bcebos.com/int8/ILSVRC2012_img_val.tar.gz.partab' ) data_md5s.append('1e9f15f64e015e58d6f9ec3210ed18b5') - self.data_cache_folder = self.download_data(data_urls, data_md5s, - "full_data", False) + self.data_cache_folder = self.download_data( + data_urls, data_md5s, "full_data", False + ) else: data_urls.append( 'http://paddle-inference-dist.bj.bcebos.com/int8/calibration_test_data.tar.gz' ) data_md5s.append('1b6c1c434172cca1bf9ba1e4d7a3157d') - self.data_cache_folder = self.download_data(data_urls, data_md5s, - "small_data", False) + self.data_cache_folder = self.download_data( + data_urls, data_md5s, "small_data", False + ) # reader/decorator.py requires the relative path to the data folder if not os.path.exists("./data/ILSVRC2012"): - cmd = 'rm -rf {0} && ln -s {1} {0}'.format("data", - self.data_cache_folder) + cmd = 'rm -rf {0} && ln -s {1} {0}'.format( + "data", self.data_cache_folder + ) os.system(cmd) self.batch_size = 1 if os.environ.get('DATASET') == 'full' else 50 - self.sample_iterations = 50 if os.environ.get( - 'DATASET') == 'full' else 2 - self.infer_iterations = 50000 if os.environ.get( - 'DATASET') == 'full' else 2 + self.sample_iterations = ( + 50 if os.environ.get('DATASET') == 'full' else 2 + ) + self.infer_iterations = ( + 50000 if os.environ.get('DATASET') == 'full' else 2 + ) self.int8_model = "post_training_quantization" print("self.int8_model: ", self.int8_model) @@ -159,7 +164,8 @@ def tearDown(self): def cache_unzipping(self, target_folder, zip_path): if not os.path.exists(target_folder): cmd = 'mkdir {0} && tar xf {1} -C {0}'.format( - target_folder, zip_path) + target_folder, zip_path + ) os.system(cmd) def download_data(self, data_urls, data_md5s, folder_name, is_model=True): @@ -171,13 +177,15 @@ def download_data(self, data_urls, data_md5s, folder_name, is_model=True): download(data_urls[i], self.int8_download, data_md5s[i]) file_names.append(data_urls[i].split('/')[-1]) - zip_path = os.path.join(self.cache_folder, - 'full_imagenet_val.tar.gz') + zip_path = os.path.join( + self.cache_folder, 'full_imagenet_val.tar.gz' + ) if not os.path.exists(zip_path): cat_command = 'cat' for file_name in file_names: - cat_command += ' ' + os.path.join(self.cache_folder, - file_name) + cat_command += ' ' + os.path.join( + self.cache_folder, file_name + ) cat_command += ' > ' + zip_path os.system(cat_command) @@ -193,11 +201,9 @@ def download_data(self, data_urls, data_md5s, folder_name, is_model=True): def download_model(self): pass - def run_program(self, - model_path, - batch_size, - infer_iterations, - is_quantized_model=False): + def run_program( + self, model_path, batch_size, infer_iterations, is_quantized_model=False + ): image_shape = [3, 224, 224] config = paddle.inference.Config(model_path) config.disable_gpu() @@ -223,8 +229,9 @@ def run_program(self, cnt = 0 periods = [] for batch_id, data in enumerate(val_reader()): - image = np.array([x[0].reshape(image_shape) - for x in data]).astype("float32") + image = np.array([x[0].reshape(image_shape) for x in data]).astype( + "float32" + ) label = np.array([x[1] for x in data]).astype("int64") label = label.reshape([-1, 1]) @@ -252,80 +259,106 @@ def run_program(self, acc1 = np.sum(test_info) / cnt return (throughput, latency, acc1) - def generate_quantized_model(self, - model_path, - quantizable_op_type, - algo="KL", - round_type="round", - is_full_quantize=False, - is_use_cache_file=False, - is_optimize_model=False, - onnx_format=False): + def generate_quantized_model( + self, + model_path, + quantizable_op_type, + algo="KL", + round_type="round", + is_full_quantize=False, + is_use_cache_file=False, + is_optimize_model=False, + onnx_format=False, + ): place = fluid.CPUPlace() exe = fluid.Executor(place) val_reader = val() - ptq = PostTrainingQuantization(executor=exe, - sample_generator=val_reader, - model_dir=model_path, - algo=algo, - quantizable_op_type=quantizable_op_type, - round_type=round_type, - is_full_quantize=is_full_quantize, - optimize_model=is_optimize_model, - onnx_format=onnx_format, - is_use_cache_file=is_use_cache_file) + ptq = PostTrainingQuantization( + executor=exe, + sample_generator=val_reader, + model_dir=model_path, + algo=algo, + quantizable_op_type=quantizable_op_type, + round_type=round_type, + is_full_quantize=is_full_quantize, + optimize_model=is_optimize_model, + onnx_format=onnx_format, + is_use_cache_file=is_use_cache_file, + ) ptq.quantize() ptq.save_quantized_model(self.int8_model) - def run_test(self, - model, - algo, - round_type, - data_urls, - data_md5s, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - onnx_format=True): + def run_test( + self, + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + onnx_format=True, + ): infer_iterations = self.infer_iterations batch_size = self.batch_size sample_iterations = self.sample_iterations model_cache_folder = self.download_data(data_urls, data_md5s, model) - print("Start INT8 post training quantization for {0} on {1} images ...". - format(model, sample_iterations * batch_size)) - self.generate_quantized_model(os.path.join(model_cache_folder, "model"), - quantizable_op_type, algo, round_type, - is_full_quantize, is_use_cache_file, - is_optimize_model, onnx_format) + print( + "Start INT8 post training quantization for {0} on {1} images ...".format( + model, sample_iterations * batch_size + ) + ) + self.generate_quantized_model( + os.path.join(model_cache_folder, "model"), + quantizable_op_type, + algo, + round_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + onnx_format, + ) - print("Start FP32 inference for {0} on {1} images ...".format( - model, infer_iterations * batch_size)) + print( + "Start FP32 inference for {0} on {1} images ...".format( + model, infer_iterations * batch_size + ) + ) (fp32_throughput, fp32_latency, fp32_acc1) = self.run_program( - os.path.join(model_cache_folder, "model"), batch_size, - infer_iterations) + os.path.join(model_cache_folder, "model"), + batch_size, + infer_iterations, + ) - print("Start INT8 inference for {0} on {1} images ...".format( - model, infer_iterations * batch_size)) - (int8_throughput, int8_latency, - int8_acc1) = self.run_program(self.int8_model, - batch_size, - infer_iterations, - is_quantized_model=True) + print( + "Start INT8 inference for {0} on {1} images ...".format( + model, infer_iterations * batch_size + ) + ) + (int8_throughput, int8_latency, int8_acc1) = self.run_program( + self.int8_model, + batch_size, + infer_iterations, + is_quantized_model=True, + ) print("---Post training quantization of {} method---".format(algo)) print( - "FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}." - .format(model, batch_size, fp32_throughput, fp32_latency, - fp32_acc1)) + "FP32 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.".format( + model, batch_size, fp32_throughput, fp32_latency, fp32_acc1 + ) + ) print( - "INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.\n" - .format(model, batch_size, int8_throughput, int8_latency, - int8_acc1)) + "INT8 {0}: batch_size {1}, throughput {2} images/second, latency {3} second, accuracy {4}.\n".format( + model, batch_size, int8_throughput, int8_latency, int8_acc1 + ) + ) sys.stdout.flush() delta_value = int8_latency - fp32_latency @@ -333,7 +366,6 @@ def run_test(self, class TestMKLDNNInt8ForResnet50AvgONNXFormat(TestPostTrainingQuantization): - def test_onnx_format_avg_resnet50(self): model = "resnet50" algo = "avg" @@ -351,17 +383,19 @@ def test_onnx_format_avg_resnet50(self): is_use_cache_file = False is_optimize_model = False diff_threshold = 0 - self.run_test(model, - algo, - round_type, - data_urls, - data_md5s, - quantizable_op_type, - is_full_quantize, - is_use_cache_file, - is_optimize_model, - diff_threshold, - onnx_format=True) + self.run_test( + model, + algo, + round_type, + data_urls, + data_md5s, + quantizable_op_type, + is_full_quantize, + is_use_cache_file, + is_optimize_model, + diff_threshold, + onnx_format=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_bf16_mkldnn_op.py index d92025344767f..81ca727b63895 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_bf16_mkldnn_op.py @@ -15,15 +15,23 @@ import unittest import numpy as np import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 -from paddle.fluid.tests.unittests.test_pool2d_op import TestPool2D_Op_Mixin, max_pool2D_forward_naive -from paddle.fluid.tests.unittests.npu.test_pool2d_op_npu import pool2d_backward_navie as pool2d_backward_naive +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) +from paddle.fluid.tests.unittests.test_pool2d_op import ( + TestPool2D_Op_Mixin, + max_pool2D_forward_naive, +) +from paddle.fluid.tests.unittests.npu.test_pool2d_op_npu import ( + pool2d_backward_navie as pool2d_backward_naive, +) from paddle import enable_static @OpTestTool.skip_if_not_cpu_bf16() class TestPoolBf16MklDNNOpGrad(TestPool2D_Op_Mixin, OpTest): - def init_kernel_type(self): self.use_mkldnn = True @@ -35,11 +43,17 @@ def setUp(self): self.attrs['mkldnn_data_type'] = "bfloat16" self.x_fp32 = np.random.random(self.shape).astype(np.float32) - output = self.pool2D_forward_naive(self.x_fp32, self.ksize, - self.strides, self.paddings, - self.global_pool, self.ceil_mode, - self.exclusive, self.adaptive, - "float32").astype(np.float32) + output = self.pool2D_forward_naive( + self.x_fp32, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + "float32", + ).astype(np.float32) self.inputs = {'X': convert_float_to_uint16(self.x_fp32)} self.outputs = {'Out': convert_float_to_uint16(output)} @@ -48,27 +62,27 @@ def test_check_output(self): self.check_output_with_place(core.CPUPlace()) def test_check_grad(self): - x_grad = pool2d_backward_naive(self.x_fp32, - ksize=self.ksize, - strides=self.strides, - paddings=self.paddings, - global_pool=self.global_pool, - ceil_mode=False, - exclusive=self.exclusive, - adaptive=self.adaptive, - data_format=self.data_format, - pool_type=self.pool_type, - padding_algorithm=self.padding_algorithm) + x_grad = pool2d_backward_naive( + self.x_fp32, + ksize=self.ksize, + strides=self.strides, + paddings=self.paddings, + global_pool=self.global_pool, + ceil_mode=False, + exclusive=self.exclusive, + adaptive=self.adaptive, + data_format=self.data_format, + pool_type=self.pool_type, + padding_algorithm=self.padding_algorithm, + ) x_grad = x_grad / np.prod(self.outputs['Out'].shape) - self.check_grad_with_place(core.CPUPlace(), - set(['X']), - 'Out', - user_defined_grads=[x_grad]) + self.check_grad_with_place( + core.CPUPlace(), set(['X']), 'Out', user_defined_grads=[x_grad] + ) @OpTestTool.skip_if_not_cpu_bf16() class TestPoolBf16MklDNNOp(TestPool2D_Op_Mixin, OpTest): - def init_kernel_type(self): self.use_mkldnn = True @@ -77,11 +91,19 @@ def setUp(self): self.dtype = np.uint16 input = np.random.random(self.shape).astype(np.float32) - output = (self.pool2D_forward_naive(input, self.ksize, self.strides, - self.paddings, self.global_pool, - self.ceil_mode, self.exclusive, - self.adaptive, - "float32")).astype(np.float32) + output = ( + self.pool2D_forward_naive( + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + "float32", + ) + ).astype(np.float32) self.inputs = {'X': convert_float_to_uint16(input)} self.outputs = {'Out': convert_float_to_uint16(output)} @@ -94,7 +116,6 @@ def test_check_grad(self): class TestCase1Avg(TestPoolBf16MklDNNOp): - def init_test_case(self): self.shape = [2, 3, 7, 7] self.ksize = [3, 3] @@ -109,7 +130,6 @@ def init_exclusive(self): class TestCase2Avg(TestPoolBf16MklDNNOp): - def init_test_case(self): self.shape = [2, 3, 7, 7] self.ksize = [3, 3] @@ -124,28 +144,24 @@ def init_exclusive(self): class TestCase0Max(TestPoolBf16MklDNNOp): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase1Max(TestCase1Avg): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase2Max(TestCase2Avg): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase1PadZeroExclusiveAvgGrad(TestPoolBf16MklDNNOpGrad): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -164,27 +180,23 @@ def init_exclusive(self): class TestCase2PadOneNonExclusiveAvgGrad(TestCase1PadZeroExclusiveAvgGrad): - def init_exclusive(self): self.exclusive = False class TestCase0InitialMaxGrad(TestPoolBf16MklDNNOpGrad): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase1PadZeroExclusiveMaxGrad(TestCase1PadZeroExclusiveAvgGrad): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase2PadOneNonExclusiveMaxGrad(TestCase2PadOneNonExclusiveAvgGrad): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_int8_mkldnn_op.py index 27595afac1f54..f21891f35e7d9 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_int8_mkldnn_op.py @@ -17,11 +17,13 @@ import paddle.fluid.core as core from paddle.fluid.tests.unittests.op_test import OpTest -from paddle.fluid.tests.unittests.test_pool2d_op import TestPool2D_Op, max_pool2D_forward_naive +from paddle.fluid.tests.unittests.test_pool2d_op import ( + TestPool2D_Op, + max_pool2D_forward_naive, +) class TestPool2DMKLDNNInt8_Op(TestPool2D_Op): - def init_kernel_type(self): self.use_mkldnn = True @@ -30,29 +32,38 @@ def init_data_type(self): def setUp(self): TestPool2D_Op.setUp(self) - assert self.dtype in [np.int8, - np.uint8], 'Dtype should be int8 or uint8' + assert self.dtype in [ + np.int8, + np.uint8, + ], 'Dtype should be int8 or uint8' input = np.random.randint(0, 100, self.shape).astype(self.dtype) - output = (self.pool2D_forward_naive(input, self.ksize, self.strides, - self.paddings, self.global_pool, - self.ceil_mode, self.exclusive, - self.adaptive, - self.dtype)).astype(self.dtype) + output = ( + self.pool2D_forward_naive( + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + self.dtype, + ) + ).astype(self.dtype) self.inputs = {'X': OpTest.np_dtype_to_fluid_dtype(input)} self.outputs = {'Out': output} def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output_with_place(core.CPUPlace(), - atol=1e-5, - check_dygraph=False) + self.check_output_with_place( + core.CPUPlace(), atol=1e-5, check_dygraph=False + ) def test_check_grad(self): pass class TestCase1Avg(TestPool2DMKLDNNInt8_Op): - def init_test_case(self): self.shape = [2, 3, 7, 7] self.ksize = [3, 3] @@ -67,7 +78,6 @@ def init_exclusive(self): class TestCase2Avg(TestPool2DMKLDNNInt8_Op): - def init_test_case(self): self.shape = [2, 3, 7, 7] self.ksize = [3, 3] @@ -82,35 +92,29 @@ def init_exclusive(self): class TestCase0Max(TestPool2DMKLDNNInt8_Op): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase1Max(TestCase1Avg): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase2Max(TestCase2Avg): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive def create_test_s8_u8_class(parent): - class TestS8Case(parent): - def init_data_type(self): self.dtype = np.int8 class TestU8Case(parent): - def init_data_type(self): self.dtype = np.uint8 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_mkldnn_op.py index 8a201a2e1e402..e4e8ecb5bef28 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_pool2d_mkldnn_op.py @@ -14,13 +14,19 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.test_pool2d_op import TestPool2D_Op, TestCase1, TestCase2, TestCase3, TestCase4, TestCase5, avg_pool2D_forward_naive +from paddle.fluid.tests.unittests.test_pool2d_op import ( + TestPool2D_Op, + TestCase1, + TestCase2, + TestCase3, + TestCase4, + TestCase5, + avg_pool2D_forward_naive, +) def create_test_mkldnn_use_ceil_class(parent): - class TestMKLDNNPool2DUseCeilCase(parent): - def init_kernel_type(self): self.use_mkldnn = True @@ -41,9 +47,7 @@ def init_data_type(self): def create_test_mkldnn_class(parent): - class TestMKLDNNCase(parent): - def init_kernel_type(self): self.use_mkldnn = True @@ -64,7 +68,6 @@ def init_data_type(self): class TestAvgPoolAdaptive(TestPool2D_Op): - def init_adaptive(self): self.adaptive = True @@ -87,7 +90,6 @@ def init_global_pool(self): class TestAvgPoolAdaptive2(TestAvgPoolAdaptive): - def init_test_case(self): self.ksize = [2, 3] self.strides = [1, 1] @@ -97,7 +99,6 @@ def init_shape(self): class TestAvgPoolAdaptive3(TestAvgPoolAdaptive): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -107,7 +108,6 @@ def init_shape(self): class TestAsymPad(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -133,81 +133,68 @@ def init_data_type(self): class TestAsymPadCase1(TestAsymPad): - def init_paddings(self): self.paddings = [1, 1, 0, 0] class TestAsymPadCase2(TestAsymPad): - def init_paddings(self): self.paddings = [1, 0, 1, 2] class TestAsymPadCase3(TestAsymPad): - def init_paddings(self): self.paddings = [1, 2, 1, 2] class TestAsymPadCase4(TestAsymPad): - def init_paddings(self): self.paddings = [1, 0, 1, 2] class TestAsymPadCase5(TestAsymPad): - def init_paddings(self): self.paddings = [2, 2, 1, 2] class TestAsymPadMaxCase1(TestAsymPadCase1): - def init_pool_type(self): self.pool_type = "max" class TestAsymPadMaxCase2(TestAsymPadCase2): - def init_pool_type(self): self.pool_type = "max" class TestAsymPadMaxCase3(TestAsymPadCase3): - def init_pool_type(self): self.pool_type = "max" class TestAsymPadMaxCase4(TestAsymPadCase4): - def init_pool_type(self): self.pool_type = "max" class TestAsymPadMaxCase5(TestAsymPadCase5): - def init_pool_type(self): self.pool_type = "max" class TestAsymPadSame(TestAsymPad): - def init_paddings(self): self.paddings = [0, 0] self.padding_algorithm = "SAME" class TestAsymPadValid(TestAsymPad): - def init_paddings(self): self.paddings = [0, 0, 0, 0] self.padding_algorithm = "VALID" class TestAsymPadValidNHWC(TestAsymPadValid): - def init_data_format(self): self.data_format = "NHWC" @@ -217,5 +204,6 @@ def init_shape(self): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_prelu_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_prelu_mkldnn_op.py index 809d5efd1e437..1b1c79248ce2d 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_prelu_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_prelu_mkldnn_op.py @@ -16,7 +16,11 @@ import numpy as np import paddle import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) def ref_prelu(x, weight, mode): @@ -27,12 +31,14 @@ def ref_prelu(x, weight, mode): elif mode == "channel": if len(weight.shape) > 1: for i in range(x.shape[1]): - result[:, i] = np.where(x[:, i] > 0, x[:, i], - x[:, i] * weight[0, i]) + result[:, i] = np.where( + x[:, i] > 0, x[:, i], x[:, i] * weight[0, i] + ) else: for i in range(x.shape[1]): - result[:, i] = np.where(x[:, i] > 0, x[:, i], - x[:, i] * weight[i]) + result[:, i] = np.where( + x[:, i] > 0, x[:, i], x[:, i] * weight[i] + ) elif mode == "element": result = np.where(x[:] > 0, x[:], x[:] * weight) @@ -40,7 +46,6 @@ def ref_prelu(x, weight, mode): class TestPReluModeChannelOneDNNOp(OpTest): - def init_attrs(self): self.mode = "element" self.alpha = np.random.random((1, 4, 5, 5)).astype("float32") @@ -69,7 +74,6 @@ def test_check_grad(self): class TestPReluModeAllOneDNNOp(TestPReluModeChannelOneDNNOp): - def init_attrs(self): self.mode = "all" self.alpha = np.random.random((1, 1, 1, 1)).astype("float32") @@ -81,14 +85,12 @@ def test_check_grad(self): class TestPReluModeElementOneDNNOp(TestPReluModeChannelOneDNNOp): - def init_attrs(self): self.mode = "element" self.alpha = np.random.random((1, 4, 5, 5)).astype("float32") class TestPReluModeChannel3DOneDNNOp(TestPReluModeChannelOneDNNOp): - def init_attrs(self): self.mode = "channel" self.x = np.random.random((1, 100, 1)).astype("float32") @@ -96,7 +98,6 @@ def init_attrs(self): class TestPReluModeChannelAlpha1DOneDNNOp(TestPReluModeChannelOneDNNOp): - def init_attrs(self): self.mode = "channel" self.x = np.random.random((1, 100, 1)).astype("float32") @@ -104,7 +105,6 @@ def init_attrs(self): class TestPReluModeAllAlpha1DOneDNNOp(TestPReluModeAllOneDNNOp): - def init_attrs(self): self.mode = "channel" self.x = np.random.random((1, 1, 100)).astype("float32") @@ -113,14 +113,14 @@ def init_attrs(self): # BF16 TESTS def create_bf16_test_class(parent): - @OpTestTool.skip_if_not_cpu_bf16() class TestPReluBF16OneDNNOp(parent): - - def set_inputs(self, ): + def set_inputs( + self, + ): self.inputs = { 'X': convert_float_to_uint16(self.x), - 'Alpha': convert_float_to_uint16(self.alpha) + 'Alpha': convert_float_to_uint16(self.alpha), } def set_dtype_attr(self): @@ -136,12 +136,18 @@ def calculate_grads(self): elif self.mode == "channel": if len(self.alpha.shape) > 1: for i in range(self.x.shape[1]): - self.dx[:, i] = np.where(self.x[:, i] > 0, dout[:, i], - dout[:, i] * self.alpha[0, i]) + self.dx[:, i] = np.where( + self.x[:, i] > 0, + dout[:, i], + dout[:, i] * self.alpha[0, i], + ) else: for i in range(self.x.shape[1]): - self.dx[:, i] = np.where(self.x[:, i] > 0, dout[:, i], - dout[:, i] * self.alpha[i]) + self.dx[:, i] = np.where( + self.x[:, i] > 0, + dout[:, i], + dout[:, i] * self.alpha[i], + ) self.dx elif self.mode == "element": self.dx = np.where(self.x[:] > 0, dout[:], dout[:] * self.alpha) @@ -155,10 +161,12 @@ def test_check_output(self): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X", "Alpha"], + core.CPUPlace(), + ["X", "Alpha"], "Out", user_defined_grads=[self.dx, self.dalpha], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) cls_name = "{0}_{1}".format(parent.__name__, "BF16") TestPReluBF16OneDNNOp.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_quantize_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_quantize_mkldnn_op.py index 52498793d58b5..4712cf4326934 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_quantize_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_quantize_mkldnn_op.py @@ -19,7 +19,6 @@ class TestQuantizeOp(OpTest): - def setUp(self): self.op_type = 'quantize' self.scale = 255.0 @@ -38,25 +37,28 @@ def setUp(self): def prepare_input(self): if self.is_negative: # input data values are from interval [-1.0, 1.0) - self.input = (2 * np.random.random_sample(self.input_size) - - 1).astype('float32') + self.input = ( + 2 * np.random.random_sample(self.input_size) - 1 + ).astype('float32') else: # input data values are from interval [0.0, 1.0) - self.input = (np.random.random_sample( - self.input_size)).astype('float32') + self.input = (np.random.random_sample(self.input_size)).astype( + 'float32' + ) self.inputs = {'Input': OpTest.np_dtype_to_fluid_dtype(self.input)} self.attrs = { 'Scale': self.scale, 'Shift': self.shift, 'is_negative_input': self.is_negative, - 'output_format': self.output_format + 'output_format': self.output_format, } def prepare_output(self): input_data_type = 'int8' if self.is_negative else 'uint8' - output = np.rint(self.input * self.scale + - self.shift).astype(input_data_type) + output = np.rint(self.input * self.scale + self.shift).astype( + input_data_type + ) self.outputs = {'Output': output} def test_check_output(self): @@ -89,7 +91,6 @@ def set_output_format(self): class TestQuantizeOp1(TestQuantizeOp): - def set_scale(self): self.scale = 127.0 @@ -98,7 +99,6 @@ def set_is_negative(self): class TestQuantizeOp2(TestQuantizeOp): - def set_scale(self): self.scale = 255.0 @@ -109,7 +109,6 @@ def set_is_negative(self): # 2-dim input # P - positive input class TestQuantizeOpShift_NCHW_2_P(TestQuantizeOp): - def set_output_format(self): self.output_format = 'NCHW' @@ -129,7 +128,6 @@ def set_input_size(self): # 2-dim input # N - negative input class TestQuantizeOpShift_NCHW_2_N(TestQuantizeOpShift_NCHW_2_P): - def set_is_negative(self): self.is_nagative = True @@ -141,63 +139,53 @@ def set_shift(self): class TestQuantizeOpShift_NHWC_2_P(TestQuantizeOpShift_NCHW_2_P): - def set_output_format(self): self.output_format = 'NHWC' class TestQuantizeOpShift_NHWC_2_N(TestQuantizeOpShift_NCHW_2_N): - def set_output_format(self): self.output_format = 'NHWC' # 3-dim input class TestQuantizeOpShift_NCHW_3_P(TestQuantizeOpShift_NCHW_2_P): - def set_input_size(self): self.input_size = [2, 3, 4] class TestQuantizeOpShift_NCHW_3_N(TestQuantizeOpShift_NCHW_2_N): - def set_input_size(self): self.input_size = [2, 3, 4] class TestQuantizeOpShift_NHWC_3_P(TestQuantizeOpShift_NCHW_3_P): - def set_output_format(self): self.output_format = 'NHWC' class TestQuantizeOpShift_NHWC_3_N(TestQuantizeOpShift_NCHW_3_N): - def set_output_format(self): self.output_format = 'NHWC' # 4-dim input class TestQuantizeOpShift_NCHW_4_P(TestQuantizeOpShift_NCHW_2_P): - def set_input_size(self): self.input_size = [2, 3, 4, 5] class TestQuantizeOpShift_NCHW_4_N(TestQuantizeOpShift_NCHW_2_N): - def set_input_size(self): self.input_size = [2, 3, 4, 5] class TestQuantizeOpShift_NHWC_4_P(TestQuantizeOpShift_NCHW_4_P): - def set_output_format(self): self.output_format = 'NHWC' class TestQuantizeOpShift_NHWC_4_N(TestQuantizeOpShift_NCHW_4_N): - def set_output_format(self): self.output_format = 'NHWC' diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_bf16_mkldnn_op.py index 12b8a468b46f5..fdd24498f88e0 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_bf16_mkldnn_op.py @@ -14,7 +14,12 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import OpTestTool, OpTest, skip_check_grad_ci, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTestTool, + OpTest, + skip_check_grad_ci, + convert_float_to_uint16, +) import paddle.fluid.core as core import paddle @@ -23,7 +28,6 @@ @OpTestTool.skip_if_not_cpu_bf16() class TestReduceSumDefaultBF16OneDNNOp(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -69,20 +73,21 @@ def calculate_grads(self): class TestReduceDefaultWithGradBF16OneDNNOp(TestReduceSumDefaultBF16OneDNNOp): - def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", check_dygraph=False, user_defined_grads=[self.grad_X], - user_defined_grad_outputs=[convert_float_to_uint16(self.grad_Out)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.grad_Out)], + ) class TestReduceSum4DReduceAllDimAttributeBF16OneDNNOp( - TestReduceDefaultWithGradBF16OneDNNOp): - + TestReduceDefaultWithGradBF16OneDNNOp +): def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -94,8 +99,8 @@ def setUp(self): class TestReduceSum4DReduceAllWithoutReduceAllAttributeNegativeDimsBF16OneDNNOp( - TestReduceDefaultWithGradBF16OneDNNOp): - + TestReduceDefaultWithGradBF16OneDNNOp +): def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -107,8 +112,8 @@ def setUp(self): class TestReduceSum5DReduceAllKeepDimsBF16OneDNNOp( - TestReduceDefaultWithGradBF16OneDNNOp): - + TestReduceDefaultWithGradBF16OneDNNOp +): def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -119,9 +124,9 @@ def setUp(self): self.outputs = {'Out': self.x_fp32.sum(keepdims=self.attrs['keep_dim'])} -class TestReduceSum4DReduceAllBF16OneDNNOp(TestReduceDefaultWithGradBF16OneDNNOp - ): - +class TestReduceSum4DReduceAllBF16OneDNNOp( + TestReduceDefaultWithGradBF16OneDNNOp +): def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -134,7 +139,8 @@ def setUp(self): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMax3DBF16OneDNNOp(TestReduceSumDefaultBF16OneDNNOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -150,9 +156,11 @@ def setUp(self): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMax4DNegativeAndPositiveDimsBF16OneDNNOp( - TestReduceSumDefaultBF16OneDNNOp): + TestReduceSumDefaultBF16OneDNNOp +): """Remove Max with subgradient from gradient check to confirm the success of CI.""" def setUp(self): @@ -167,7 +175,8 @@ def setUp(self): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMin3DBF16OneDNNOp(TestReduceSumDefaultBF16OneDNNOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -182,7 +191,6 @@ def setUp(self): class TestReduceMean3DBF16OneDNNOp(TestReduceDefaultWithGradBF16OneDNNOp): - def setUp(self): self.op_type = "reduce_mean" self.use_mkldnn = True @@ -194,7 +202,6 @@ def setUp(self): class TestReduceMean4DBF16OneDNNOp(TestReduceDefaultWithGradBF16OneDNNOp): - def setUp(self): self.op_type = "reduce_mean" self.use_mkldnn = True @@ -203,9 +210,8 @@ def setUp(self): self.inputs = {'X': self.x_bf16} self.attrs = {'use_mkldnn': self.use_mkldnn, 'dim': [0, 1]} self.outputs = { - 'Out': - self.x_fp32.sum(axis=tuple(self.attrs['dim'])) / - (self.x_fp32.shape[0] * self.x_fp32.shape[1]) + 'Out': self.x_fp32.sum(axis=tuple(self.attrs['dim'])) + / (self.x_fp32.shape[0] * self.x_fp32.shape[1]) } diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_mkldnn_op.py index 93e81070cb669..b6f2c30a68285 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_reduce_mkldnn_op.py @@ -14,12 +14,15 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, skip_check_grad_ci +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + skip_check_grad_ci, +) import paddle class TestReduceSumDefaultOneDNNOp(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -32,13 +35,11 @@ def test_check_output(self): class TestReduceDefaultWithGradOneDNNOp(TestReduceSumDefaultOneDNNOp): - def test_check_grad(self): self.check_grad(['X'], 'Out') class TestReduceSum4DOneDNNOp(TestReduceDefaultWithGradOneDNNOp): - def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -50,8 +51,8 @@ def setUp(self): class TestReduceSum4DReduceAllDimAttributeBF16OneDNNOp( - TestReduceDefaultWithGradOneDNNOp): - + TestReduceDefaultWithGradOneDNNOp +): def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -63,22 +64,21 @@ def setUp(self): class TestReduceSum5DKeepDimsOneDNNOp(TestReduceDefaultWithGradOneDNNOp): - def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True self.inputs = {'X': np.random.random((2, 5, 3, 2, 2)).astype("float32")} self.attrs = {'dim': (2, 3, 4), 'keep_dim': True, 'use_mkldnn': True} self.outputs = { - 'Out': - self.inputs['X'].sum(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].sum( + axis=tuple(self.attrs['dim']), keepdims=self.attrs['keep_dim'] + ) } -class TestReduceSum5DReduceAllKeepDimsOneDNNOp(TestReduceDefaultWithGradOneDNNOp - ): - +class TestReduceSum5DReduceAllKeepDimsOneDNNOp( + TestReduceDefaultWithGradOneDNNOp +): def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -90,7 +90,6 @@ def setUp(self): class TestReduceSum4DReduceAllOneDNNOp(TestReduceDefaultWithGradOneDNNOp): - def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -101,8 +100,8 @@ def setUp(self): @OpTestTool.skip_if_not_cpu() class TestReduceSum4DNoReduceSimpleCopyOneDNNOp( - TestReduceDefaultWithGradOneDNNOp): - + TestReduceDefaultWithGradOneDNNOp +): def setUp(self): self.op_type = "reduce_sum" self.use_mkldnn = True @@ -113,7 +112,8 @@ def setUp(self): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMax3DOneDNNOp(TestReduceSumDefaultOneDNNOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -129,9 +129,11 @@ def setUp(self): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMax4DNegativeAndPositiveDimsOneDNNOp( - TestReduceSumDefaultOneDNNOp): + TestReduceSumDefaultOneDNNOp +): """Remove Max with subgradient from gradient check to confirm the success of CI.""" def setUp(self): @@ -146,7 +148,8 @@ def setUp(self): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMin3DOneDNNOp(TestReduceSumDefaultOneDNNOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -161,7 +164,6 @@ def setUp(self): class TestReduceMean3DOneDNNOp(TestReduceDefaultWithGradOneDNNOp): - def setUp(self): self.op_type = "reduce_mean" self.use_mkldnn = True @@ -173,15 +175,14 @@ def setUp(self): class TestReduceMean4DReduceAllOneDNNOp(TestReduceDefaultWithGradOneDNNOp): - def setUp(self): self.op_type = "reduce_mean" self.use_mkldnn = True self.inputs = {'X': np.random.random((5, 6, 8, 10)).astype("float32")} self.attrs = {'reduce_all': True, 'use_mkldnn': self.use_mkldnn} self.outputs = { - 'Out': - self.inputs['X'].sum() / np.asarray(self.inputs['X'].shape).prod() + 'Out': self.inputs['X'].sum() + / np.asarray(self.inputs['X'].shape).prod() } diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_requantize_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_requantize_mkldnn_op.py index f49bb89e3e53e..a908b5f408d38 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_requantize_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_requantize_mkldnn_op.py @@ -21,7 +21,6 @@ class TestReQuantizeOp(OpTest): - def set_input_size(self): self.input_size = [1, 1, 10, 10] self.format_reorder = format_reorder @@ -43,32 +42,36 @@ def setUp(self): def prepare_input(self): if self.input_data_type == 'int8': # input data values are integers from interval [-128, 128) - self.input = (np.random.randint(0, 256, self.input_size) - - 128).astype(self.input_data_type) + self.input = ( + np.random.randint(0, 256, self.input_size) - 128 + ).astype(self.input_data_type) else: # input data values are integers from interval [0, 256) self.input = (np.random.randint(0, 256, self.input_size)).astype( - self.input_data_type) + self.input_data_type + ) self.inputs = {'Input': OpTest.np_dtype_to_fluid_dtype(self.input)} self.attrs = { 'Scale_in': self.scale_in, 'Scale_out': self.scale_out, 'Shift_in': self.shift_in, - 'Shift_out': self.shift_out + 'Shift_out': self.shift_out, } def prepare_output(self): scale_ratio = self.scale_out / self.scale_in - with_shift = (self.shift_in != 0.0 or self.shift_out != 0.0) + with_shift = self.shift_in != 0.0 or self.shift_out != 0.0 if with_shift or self.input_data_type == 'uint8': dst_type = 'uint8' type_min = 0 type_max = 255 new_shift = np.clip( - np.rint(self.shift_out - scale_ratio * self.shift_in), type_min, - type_max) + np.rint(self.shift_out - scale_ratio * self.shift_in), + type_min, + type_max, + ) else: dst_type = 'int8' type_min = -128 @@ -77,15 +80,19 @@ def prepare_output(self): output_tmp = np.clip( np.rint(self.input.astype('float32') * scale_ratio + new_shift), - type_min, type_max).astype(dst_type) + type_min, + type_max, + ).astype(dst_type) self.output = self.format_reorder(output_tmp, self.input_size) self.outputs = {'Output': self.output} def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.assertTrue(self.input_data_type == 'uint8' or self.shift_in == 0.0, - 'Input data must be unsigned if it has nonzero shift.') + self.assertTrue( + self.input_data_type == 'uint8' or self.shift_in == 0.0, + 'Input data must be unsigned if it has nonzero shift.', + ) self.check_output(check_dygraph=False) def check_raise_error(self, msg): @@ -111,28 +118,24 @@ def set_input_data_type(self): class TestReQuantizeOp_S8_SameScales(TestReQuantizeOp): - def set_scales(self): self.scale_in = 127.0 self.scale_out = 127.0 class TestReQuantizeOp_S8_DifferentScales_1(TestReQuantizeOp): - def set_scales(self): self.scale_in = 127.0 self.scale_out = 100.0 class TestReQuantizeOp_S8_DifferentScales_2(TestReQuantizeOp): - def set_scales(self): self.scale_in = 100.0 self.scale_out = 127.0 class TestReQuantizeOp_S8_ZeroInputScale(TestReQuantizeOp): - def set_scales(self): self.scale_in = 0.0 self.scale_out = 127.0 @@ -142,12 +145,14 @@ def prepare_output(self): self.outputs = {'Output': self.output} def test_check_output(self): - self.assertRaises(AttributeError, self.check_raise_error, - 'Scale of input cannot be 0.0') + self.assertRaises( + AttributeError, + self.check_raise_error, + 'Scale of input cannot be 0.0', + ) class TestReQuantizeOp_S8_ZeroOutputScale(TestReQuantizeOp): - def set_scales(self): self.scale_in = 127.0 self.scale_out = 0.0 @@ -157,29 +162,31 @@ def prepare_output(self): self.outputs = {'Output': self.output} def test_check_output(self): - self.assertRaises(AttributeError, self.check_raise_error, - 'Scale of output cannot be 0.0') + self.assertRaises( + AttributeError, + self.check_raise_error, + 'Scale of output cannot be 0.0', + ) # ---------------test requantize with u8 input, no shift-------------------- class TestReQuantizeOp_U8_SameScales(TestReQuantizeOp_S8_SameScales): - def set_input_data_type(self): self.input_data_type = 'uint8' class TestReQuantizeOp_U8_DifferentScales_1( - TestReQuantizeOp_S8_DifferentScales_1): - + TestReQuantizeOp_S8_DifferentScales_1 +): def set_input_data_type(self): self.input_data_type = 'uint8' class TestReQuantizeOp_U8_DifferentScales_2( - TestReQuantizeOp_S8_DifferentScales_2): - + TestReQuantizeOp_S8_DifferentScales_2 +): def set_input_data_type(self): self.input_data_type = 'uint8' @@ -188,7 +195,6 @@ def set_input_data_type(self): class TestReQuantizeOp_S8_WithShift(TestReQuantizeOp): - def set_scales(self): self.scale_in = 60.0 self.scale_out = 127.0 @@ -199,12 +205,13 @@ def set_shifts(self): def test_check_output(self): self.assertRaises( - AttributeError, self.check_raise_error, - 'Requantize does not support nonzero shift for signed input.') + AttributeError, + self.check_raise_error, + 'Requantize does not support nonzero shift for signed input.', + ) class TestReQuantizeOp_S8_WithOutputShift(TestReQuantizeOp): - def set_scales(self): self.scale_in = 127.0 self.scale_out = 60.0 @@ -218,71 +225,70 @@ def set_shifts(self): class TestReQuantizeOp_U8_SameScales_SameShift(TestReQuantizeOp_U8_SameScales): - def set_shifts(self): self.shift_in = 128.0 self.shift_out = 128.0 class TestReQuantizeOp_U8_SameScales_DifferentShift_1( - TestReQuantizeOp_U8_SameScales): - + TestReQuantizeOp_U8_SameScales +): def set_shifts(self): self.shift_in = 60.0 self.shift_out = 128.0 class TestReQuantizeOp_U8_SameScales_DifferentShift_2( - TestReQuantizeOp_U8_SameScales): - + TestReQuantizeOp_U8_SameScales +): def set_shifts(self): self.shift_in = 128.0 self.shift_out = 60.0 class TestReQuantizeOp_U8_DifferentScales_1_SameShift( - TestReQuantizeOp_U8_DifferentScales_1): - + TestReQuantizeOp_U8_DifferentScales_1 +): def set_shifts(self): self.shift_in = 128.0 self.shift_out = 128.0 class TestReQuantizeOp_U8_DifferentScales_2_SameShift( - TestReQuantizeOp_U8_DifferentScales_2): - + TestReQuantizeOp_U8_DifferentScales_2 +): def set_shifts(self): self.shift_in = 128.0 self.shift_out = 128.0 class TestReQuantizeOp_U8_DifferentScales_1_DifferentShift_1( - TestReQuantizeOp_U8_DifferentScales_1): - + TestReQuantizeOp_U8_DifferentScales_1 +): def set_shifts(self): self.shift_in = 128.0 self.shift_out = 60.0 class TestReQuantizeOp_U8_DifferentScales_2_DifferentShift_1( - TestReQuantizeOp_U8_DifferentScales_2): - + TestReQuantizeOp_U8_DifferentScales_2 +): def set_shifts(self): self.shift_in = 128.0 self.shift_out = 60.0 class TestReQuantizeOp_U8_DifferentScales_1_DifferentShift_2( - TestReQuantizeOp_U8_DifferentScales_1): - + TestReQuantizeOp_U8_DifferentScales_1 +): def set_shifts(self): self.shift_in = 60.0 self.shift_out = 128.0 class TestReQuantizeOp_U8_DifferentScales_2_DifferentShift_2( - TestReQuantizeOp_U8_DifferentScales_2): - + TestReQuantizeOp_U8_DifferentScales_2 +): def set_shifts(self): self.shift_in = 60.0 self.shift_out = 128.0 @@ -292,7 +298,6 @@ def set_shifts(self): class TestReQuantizeOp_2DimFormat(TestReQuantizeOp): - def format_reorder_2Dim(self, out, size): return out @@ -305,7 +310,6 @@ def set_input_size(self): class TestReQuantizeOpReused(TestReQuantizeOp): - def setUp(self): # self.input_size = [1, 1, 10, 10] self.input_size = [1, 1, 2, 2] @@ -337,38 +341,40 @@ def test_check_output(self): with fluid.program_guard(program): block = program.global_block() for name in variables: - block.create_var(name=name, - dtype="int8", - shape=variables[name].shape) - block.append_op(type="requantize", - inputs={ - 'Input': block.var('input'), - }, - outputs={"Output": block.var('output')}, - attrs={ - 'Scale_in': self.scale_in, - 'Scale_out': self.scale_out, - 'Shift_in': self.shift_in, - 'Shift_out': self.shift_out - }) + block.create_var( + name=name, dtype="int8", shape=variables[name].shape + ) + block.append_op( + type="requantize", + inputs={ + 'Input': block.var('input'), + }, + outputs={"Output": block.var('output')}, + attrs={ + 'Scale_in': self.scale_in, + 'Scale_out': self.scale_out, + 'Shift_in': self.shift_in, + 'Shift_out': self.shift_out, + }, + ) place = core.CPUPlace() exe = fluid.Executor(place) for i in range(2): - out = exe.run(program, - feed={'input': variables['input']}, - fetch_list=['output']) + out = exe.run( + program, + feed={'input': variables['input']}, + fetch_list=['output'], + ) - np.testing.assert_allclose(variables['output'], - out[0], - rtol=1e-05, - atol=1e-4) + np.testing.assert_allclose( + variables['output'], out[0], rtol=1e-05, atol=1e-4 + ) # ---------------test reused requantize op, no shift------------------------ class TestReQuantizeOpReused_WithShift(TestReQuantizeOpReused): - def set_input_data_type(self): self.input_data_type = 'uint8' diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_bf16_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_bf16_op.py index 7fbdf66ee7d75..efa1459b047c8 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_bf16_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_bf16_op.py @@ -20,10 +20,10 @@ from paddle import enable_static -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestReshapeBf16Op(OpTest): - def setUp(self): self.op_type = "reshape2" self.use_mkldnn = False @@ -35,11 +35,11 @@ def setUp(self): self.attrs = { 'shape': self.new_shape, 'use_mkldnn': self.use_mkldnn, - 'mkldnn_data_type': self.mkldnn_data_type + 'mkldnn_data_type': self.mkldnn_data_type, } self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype(np.float32) + 'XShape': np.random.random(self.ori_shape).astype(np.float32), } def init_data(self): @@ -49,21 +49,24 @@ def init_data(self): def init_input_data(self): self.input_data_fp32 = np.random.random(self.ori_shape).astype( - np.float32) + np.float32 + ) self.input_data = convert_float_to_uint16(self.input_data_fp32) def test_check_output(self): self.check_output_with_place(core.CPUPlace(), no_check_set=['XShape']) def test_check_grad(self): - self.check_grad_with_place(core.CPUPlace(), ["X"], - "Out", - check_dygraph=False, - user_defined_grads=[self.input_data_fp32], - user_defined_grad_outputs=[ - self.inputs["X"].reshape( - self.infered_shape) - ]) + self.check_grad_with_place( + core.CPUPlace(), + ["X"], + "Out", + check_dygraph=False, + user_defined_grads=[self.input_data_fp32], + user_defined_grad_outputs=[ + self.inputs["X"].reshape(self.infered_shape) + ], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_mkldnn_op.py index ef2748155e783..46d232cfc0809 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_reshape_mkldnn_op.py @@ -16,13 +16,18 @@ import numpy as np import paddle import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) -@OpTestTool.skip_if(core.is_compiled_with_cuda(), - "CUDA has to be skipped because it forces dygraph") +@OpTestTool.skip_if( + core.is_compiled_with_cuda(), + "CUDA has to be skipped because it forces dygraph", +) class TestReshape2OneDNNOp(OpTest): - def setUp(self): self.init_data() self.set_op_type() @@ -47,7 +52,7 @@ def set_attrs(self): def set_outputs(self): self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def init_data(self): @@ -63,7 +68,6 @@ def test_check_grad(self): class TestReshape2OneDNNOpDimInfer1(TestReshape2OneDNNOp): - def init_data(self): self.ori_shape = (5, 25) self.new_shape = (5, -1, 5) @@ -71,7 +75,6 @@ def init_data(self): class TestReshape2OneDNNOpDimInfer2(TestReshape2OneDNNOp): - def init_data(self): self.ori_shape = (6, 20) self.new_shape = (0, -1, 20) @@ -83,12 +86,11 @@ def set_additional_inputs(self): def set_outputs(self): self.outputs = { "Out": self.inputs["X"].reshape(self.actual_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } class TestReshape2OneDNNOp_attr_OnlyShape(TestReshape2OneDNNOp): - def set_additional_inputs(self): self.inputs["Shape"] = np.array(self.new_shape, dtype="int32") @@ -98,7 +100,7 @@ def set_attrs(self): def set_outputs(self): self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def init_data(self): @@ -108,8 +110,8 @@ def init_data(self): class TestReshape2OneDNNOpDimInfer1_attr_OnlyShape( - TestReshape2OneDNNOp_attr_OnlyShape): - + TestReshape2OneDNNOp_attr_OnlyShape +): def init_data(self): self.ori_shape = (5, 20) self.new_shape = (5, -1, 10) @@ -118,12 +120,12 @@ def init_data(self): class TestReshape2OneDNNOpDimInfer1_attr_ShapeTensor(TestReshape2OneDNNOp): - def set_additional_inputs(self): shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs["ShapeTensor"] = shape_tensor @@ -135,20 +137,20 @@ def init_data(self): class TestReshape2OneDNNOpDimInfer1_attr_ShapeTensorAndShape( - TestReshape2OneDNNOpDimInfer1_attr_ShapeTensor): - + TestReshape2OneDNNOpDimInfer1_attr_ShapeTensor +): def set_additional_inputs(self): shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs["Shape"] = np.array((1, 2, 3, 4), dtype="int32") self.inputs["ShapeTensor"] = shape_tensor class TestReshapeOneDNNOp(TestReshape2OneDNNOp): - def set_op_type(self): self.op_type = "reshape" @@ -160,7 +162,6 @@ def test_check_output(self): class TestReshapeOneDNNOpDimInfer1(TestReshapeOneDNNOp): - def init_data(self): self.ori_shape = (5, 25) self.new_shape = (5, -1, 5) @@ -168,7 +169,6 @@ def init_data(self): class TestReshapeOneDNNOp_attr_OnlyShape(TestReshape2OneDNNOp_attr_OnlyShape): - def set_op_type(self): self.op_type = "reshape" @@ -180,8 +180,8 @@ def test_check_output(self): class TestReshapeOneDNNOpDimInfer1_attr_OnlyShape( - TestReshapeOneDNNOp_attr_OnlyShape): - + TestReshapeOneDNNOp_attr_OnlyShape +): def init_data(self): self.ori_shape = (5, 20) self.new_shape = (5, -1, 10) @@ -191,10 +191,8 @@ def init_data(self): # BF16 TESTS def create_reshape_bf16_test_classes(parent): - @OpTestTool.skip_if_not_cpu_bf16() class TestReshape2BF16OneDNNOp(parent): - def set_inputs(self): self.dtype = np.uint16 self.inputs = {"X": convert_float_to_uint16(self.x)} @@ -204,22 +202,25 @@ def calculate_grads(self): self.dx = np.reshape(self.dout, self.ori_shape) def test_check_output(self): - self.check_output_with_place(core.CPUPlace(), - no_check_set=["XShape"]) + self.check_output_with_place( + core.CPUPlace(), no_check_set=["XShape"] + ) def test_check_grad(self): self.calculate_grads() - self.check_grad_with_place(core.CPUPlace(), ["X"], - "Out", - user_defined_grads=[self.dx], - user_defined_grad_outputs=[self.dout]) + self.check_grad_with_place( + core.CPUPlace(), + ["X"], + "Out", + user_defined_grads=[self.dx], + user_defined_grad_outputs=[self.dout], + ) cls_name = "{0}_{1}".format(parent.__name__, "Reshape2_BF16") TestReshape2BF16OneDNNOp.__name__ = cls_name globals()[cls_name] = TestReshape2BF16OneDNNOp class TestReshapeBF16OneDNNOp(TestReshape2BF16OneDNNOp): - def set_op_type(self): self.dtype = np.uint16 self.op_type = "reshape" @@ -233,10 +234,12 @@ def test_check_output(self): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", user_defined_grads=[self.dx], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) cls_name = "{0}_{1}".format(parent.__name__, "Reshape_BF16") TestReshapeBF16OneDNNOp.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_scale_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_scale_bf16_mkldnn_op.py index 18224e07d42b5..e22161672fe82 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_scale_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_scale_bf16_mkldnn_op.py @@ -19,12 +19,14 @@ import paddle.fluid.core as core -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") -@unittest.skipIf(core.is_compiled_with_cuda(), - "core is compiled with CUDA which has no BF implementation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) +@unittest.skipIf( + core.is_compiled_with_cuda(), + "core is compiled with CUDA which has no BF implementation", +) class TestScaleOpBF16(OpTest): - def setUp(self): self.op_type = "scale" self.x_fp32 = np.random.random((10, 10)).astype(np.float32) @@ -47,7 +49,7 @@ def calculate_grads(self): scale = self.attrs['ScaleTensor'] self.out = (self.x_fp32 * scale) + bias - self.dx = (self.out * scale) + self.dx = self.out * scale def test_check_output(self): self.check_output(check_dygraph=False) @@ -55,15 +57,16 @@ def test_check_output(self): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["X"], + core.CPUPlace(), + ["X"], "Out", check_dygraph=False, user_defined_grads=[self.dx], - user_defined_grad_outputs=[convert_float_to_uint16(self.out)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.out)], + ) class TestScaleOpBF16BiasNotAfterScale(TestScaleOpBF16): - def setUp(self): self.op_type = "scale" self.x_fp32 = np.random.random((10, 10)).astype(np.float32) @@ -74,7 +77,7 @@ def setUp(self): 'scale': self.scale, 'use_mkldnn': True, 'bias': 0.0, - 'bias_after_scale': False + 'bias_after_scale': False, } self.use_mkldnn = True self.outputs = { @@ -83,7 +86,6 @@ def setUp(self): class TestScaleOpBF16ScaleTensor(TestScaleOpBF16): - def setUp(self): self.op_type = "scale" self.scale = -2.3 @@ -92,14 +94,13 @@ def setUp(self): self.scale_tensor = np.array([self.scale]).astype(np.float32) self.inputs = { 'X': self.x_bf16, - 'ScaleTensor': convert_float_to_uint16(self.scale_tensor) + 'ScaleTensor': convert_float_to_uint16(self.scale_tensor), } self.attrs = {'use_mkldnn': True} self.outputs = {'Out': self.x_fp32 * self.scale} class TestScaleOpBF16ScaleTensorNotBiasAfterScale(TestScaleOpBF16): - def setUp(self): self.op_type = "scale" self.scale = 1.2 @@ -108,12 +109,12 @@ def setUp(self): self.scale_tensor = np.array([self.scale]).astype(np.float32) self.inputs = { 'X': self.x_bf16, - 'ScaleTensor': convert_float_to_uint16(self.scale_tensor) + 'ScaleTensor': convert_float_to_uint16(self.scale_tensor), } self.attrs = { 'bias': -1.1, 'bias_after_scale': False, - 'use_mkldnn': True + 'use_mkldnn': True, } self.outputs = {'Out': (self.x_fp32 + self.attrs['bias']) * self.scale} diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_scale_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_scale_mkldnn_op.py index 9fcbb98dfa287..b812f8d876d91 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_scale_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_scale_mkldnn_op.py @@ -19,7 +19,6 @@ class TestScaleOp(OpTest): - def setUp(self): self.op_type = "scale" self.inputs = {'X': np.random.random((10, 10)).astype(np.float32)} @@ -37,7 +36,6 @@ def test_check_grad(self): class TestScaleOpBiasNotAfterScale(OpTest): - def setUp(self): self.op_type = "scale" self.inputs = {'X': np.random.random((10, 10)).astype(np.float32)} @@ -45,7 +43,7 @@ def setUp(self): 'scale': 1.5, 'use_mkldnn': True, 'bias': 2.3, - 'bias_after_scale': False + 'bias_after_scale': False, } self.use_mkldnn = True self.outputs = { @@ -60,13 +58,12 @@ def test_check_grad(self): class TestScaleOpScaleTensor(OpTest): - def setUp(self): self.op_type = "scale" self.scale = -2.3 self.inputs = { 'X': np.random.random((10, 10)).astype(np.float32), - 'ScaleTensor': np.array([self.scale]).astype(np.float32) + 'ScaleTensor': np.array([self.scale]).astype(np.float32), } self.attrs = {} self.outputs = {'Out': self.inputs['X'] * self.scale} @@ -79,18 +76,17 @@ def test_check_grad(self): class TestScaleOpScaleTensorNotBiasAfterScale(OpTest): - def setUp(self): self.op_type = "scale" self.scale = -1.2 self.inputs = { 'X': np.random.random((10, 10)).astype(np.float32), - 'ScaleTensor': np.array([self.scale]).astype(np.float32) + 'ScaleTensor': np.array([self.scale]).astype(np.float32), } self.attrs = {'bias': -6.8, 'bias_after_scale': False} self.outputs = { - 'Out': - (self.inputs['X'] + self.attrs['bias']) * self.inputs['ScaleTensor'] + 'Out': (self.inputs['X'] + self.attrs['bias']) + * self.inputs['ScaleTensor'] } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_shape_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_shape_mkldnn_op.py index 573d4a9b2cf22..98401c3ca1754 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_shape_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_shape_mkldnn_op.py @@ -21,7 +21,6 @@ @OpTestTool.skip_if_not_cpu_bf16() class TestShape3DFP32OneDNNOp(OpTest): - def setUp(self): self.op_type = "shape" self.config() @@ -38,21 +37,18 @@ def test_check_output(self): class TestShape6DBF16OneDNNOp(TestShape3DFP32OneDNNOp): - def config(self): self.shape = [10, 2, 3, 4, 5, 2] self.dtype = np.uint16 class TestShape9DINT8OneDNNOp(TestShape3DFP32OneDNNOp): - def config(self): self.shape = [1, 2, 3, 4, 5, 6, 7, 8, 9] self.dtype = np.int8 class TestShape2DUINT8OneDNNOp(TestShape3DFP32OneDNNOp): - def config(self): self.shape = [7, 11] self.dtype = np.uint8 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_shuffle_channel_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_shuffle_channel_mkldnn_op.py index 0ed91e959242b..b5320c0653a26 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_shuffle_channel_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_shuffle_channel_mkldnn_op.py @@ -21,7 +21,6 @@ @OpTestTool.skip_if_not_cpu_bf16() class TestShuffleChannelOneDNNOp(OpTest): - def setUp(self): self.op_type = "shuffle_channel" self.set_dtype() @@ -30,8 +29,9 @@ def setUp(self): self.attrs = {'use_mkldnn': True, 'group': self.group} _, c, h, w = self.inputs['X'].shape - input_reshaped = np.reshape(self.inputs['X'], - (-1, self.group, c // self.group, h, w)) + input_reshaped = np.reshape( + self.inputs['X'], (-1, self.group, c // self.group, h, w) + ) input_transposed = np.transpose(input_reshaped, (0, 2, 1, 3, 4)) self.outputs = {'Out': np.reshape(input_transposed, (-1, c, h, w))} @@ -46,13 +46,11 @@ def test_check_output(self): class TestShuffleChannelSingleGroupOneDNNOp(TestShuffleChannelOneDNNOp): - def set_group(self): self.group = 1 class TestShuffleChannelBF16OneDNNOp(TestShuffleChannelOneDNNOp): - def set_dtype(self): self.dtype = np.uint16 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_slice_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_slice_mkldnn_op.py index 0812f4da8b275..8dca568bb1eff 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_slice_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_slice_mkldnn_op.py @@ -15,14 +15,19 @@ import unittest import numpy as np import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) import paddle -@OpTestTool.skip_if(core.is_compiled_with_cuda(), - "CUDA required dygraph so oneDNN UT must be skipped") +@OpTestTool.skip_if( + core.is_compiled_with_cuda(), + "CUDA required dygraph so oneDNN UT must be skipped", +) class TestSliceOneDNNOp(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -33,7 +38,7 @@ def setUp(self): 'starts': self.starts, 'ends': self.ends, 'infer_flags': self.infer_flags, - 'use_mkldnn': True + 'use_mkldnn': True, } self.set_attrs() @@ -59,7 +64,6 @@ def test_check_grad(self): class TestSliceOneDNNOp1(TestSliceOneDNNOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-3, 0, 2] @@ -70,7 +74,6 @@ def config(self): class TestSliceOneDNNOp2(TestSliceOneDNNOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-3, 0, 2] @@ -81,7 +84,6 @@ def config(self): class TestSliceDecrease1AxisOneDNNOp(TestSliceOneDNNOp): - def set_attrs(self): self.attrs['decrease_axis'] = self.decrease_axis @@ -96,7 +98,6 @@ def config(self): class TestSliceDecrease2AxesOneDNNOp(TestSliceDecrease1AxisOneDNNOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [1, 0, 2] @@ -108,7 +109,6 @@ def config(self): class TestSliceDecrease3AxesOneDNNOp(TestSliceDecrease1AxisOneDNNOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-1, 0, 2] @@ -120,7 +120,6 @@ def config(self): class TestSliceDecrease4AxesOneDNNOp(TestSliceDecrease1AxisOneDNNOp): - def config(self): self.input = np.random.random([3, 4, 5, 7]).astype("float32") self.starts = [0, 1, 2, 3] @@ -132,7 +131,6 @@ def config(self): class TestSlice5DOneDNNOp(TestSliceDecrease1AxisOneDNNOp): - def config(self): self.input = np.random.random([3, 4, 5, 6, 7]).astype("float32") self.starts = [-1] @@ -144,7 +142,6 @@ def config(self): class TestSlice3DOneDNNOp(TestSliceDecrease1AxisOneDNNOp): - def config(self): self.input = np.random.random([5, 4, 5]).astype("float32") self.starts = [-1] @@ -156,8 +153,8 @@ def config(self): class TestSliceOneDNNOp_decs_dim_starts_ListTensor( - TestSliceDecrease1AxisOneDNNOp): - + TestSliceDecrease1AxisOneDNNOp +): def set_inputs(self): starts_tensor = [] for index, ele in enumerate(self.starts): @@ -175,7 +172,6 @@ def config(self): class TestSlice4DInferDimsOneDNNOp(TestSliceDecrease1AxisOneDNNOp): - def config(self): self.input = np.random.random([1, 1, 10, 10]).astype("float32") self.starts = [1, 2] @@ -187,7 +183,6 @@ def config(self): class TestSlice4DInferDimsOneDNNOp2(TestSliceDecrease1AxisOneDNNOp): - def config(self): self.input = np.random.random([1, 1, 10, 10]).astype("float32") self.starts = [4, 2] @@ -200,10 +195,8 @@ def config(self): # BF16 TESTS def create_bf16_test_class(parent): - @OpTestTool.skip_if_not_cpu_bf16() class TestSliceBF16OneDNNOp(parent): - def set_inputs(self): self.dtype = np.uint16 self.inputs = {'Input': convert_float_to_uint16(self.input)} @@ -218,8 +211,12 @@ def calculate_grads(self): for i in range(len(self.axes)): begin[self.axes[i]] = self.starts[i] end[self.axes[i]] = self.ends[i] - self.dx[begin[0]:end[0], begin[1]:end[1], begin[2]:end[2], - begin[3]:end[3]] = self.dout + self.dx[ + begin[0] : end[0], + begin[1] : end[1], + begin[2] : end[2], + begin[3] : end[3], + ] = self.dout def test_check_output(self): self.check_output_with_place(core.CPUPlace()) @@ -227,10 +224,12 @@ def test_check_output(self): def test_check_grad(self): self.calculate_grads() self.check_grad_with_place( - core.CPUPlace(), ["Input"], + core.CPUPlace(), + ["Input"], "Out", user_defined_grads=[self.dx], - user_defined_grad_outputs=[convert_float_to_uint16(self.dout)]) + user_defined_grad_outputs=[convert_float_to_uint16(self.dout)], + ) cls_name = "{0}_{1}".format(parent.__name__, "BF16") TestSliceBF16OneDNNOp.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_bf16_mkldnn_op.py index 23643261836ea..e87214b6b27e7 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_bf16_mkldnn_op.py @@ -16,21 +16,28 @@ import numpy as np from paddle.fluid.tests.unittests.op_test import convert_float_to_uint16 import paddle.fluid.core as core -from paddle.fluid.tests.unittests.test_softmax_op import TestSoftmaxOp, TestSoftmaxOp2, TestSoftmaxOp3, TestSoftmaxOp4, TestSoftmaxOp5, TestSoftmaxOp6 +from paddle.fluid.tests.unittests.test_softmax_op import ( + TestSoftmaxOp, + TestSoftmaxOp2, + TestSoftmaxOp3, + TestSoftmaxOp4, + TestSoftmaxOp5, + TestSoftmaxOp6, +) from paddle import enable_static def stable_softmax(x): """Compute the softmax of vector x in a numerically stable way.""" - shiftx = x - np.max(x).clip(-64.) + shiftx = x - np.max(x).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestSoftmaxMKLDNNOp(TestSoftmaxOp): - def get_x_shape(self): return [10, 10] @@ -47,7 +54,8 @@ def setUp(self): x = np.random.uniform(0.1, 1, self.shape).astype(np.float64) out = convert_float_to_uint16( - np.apply_along_axis(stable_softmax, self.axis, x)) + np.apply_along_axis(stable_softmax, self.axis, x) + ) self.inputs = {'X': convert_float_to_uint16(x)} self.outputs = {'Out': out} @@ -64,31 +72,26 @@ def init_kernel_type(self): class TestSoftmaxMKLDNNOp2(TestSoftmaxOp2): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp3(TestSoftmaxOp3): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp4(TestSoftmaxOp4): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp5(TestSoftmaxOp5): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp6(TestSoftmaxOp6): - def init_kernel_type(self): self.use_mkldnn = True diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_mkldnn_op.py index 1422d1aeb342d..c181c1248867f 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_softmax_mkldnn_op.py @@ -16,19 +16,25 @@ import numpy as np from paddle.fluid.tests.unittests.op_test import OpTest import paddle.fluid.core as core -from paddle.fluid.tests.unittests.test_softmax_op import TestSoftmaxOp, TestSoftmaxOp2, TestSoftmaxOp3, TestSoftmaxOp4, TestSoftmaxOp5, TestSoftmaxOp6 +from paddle.fluid.tests.unittests.test_softmax_op import ( + TestSoftmaxOp, + TestSoftmaxOp2, + TestSoftmaxOp3, + TestSoftmaxOp4, + TestSoftmaxOp5, + TestSoftmaxOp6, +) from mkldnn_op_test import check_if_mkldnn_primitives_exist_in_bwd def stable_softmax(x): """Compute the softmax of vector x in a numerically stable way.""" - shiftx = x - np.max(x).clip(-64.) + shiftx = x - np.max(x).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) class TestSoftmaxMKLDNNOp(TestSoftmaxOp): - def get_x_shape(self): return [10, 10] @@ -52,7 +58,7 @@ def setUp(self): self.attrs = { 'axis': self.axis, 'use_cudnn': self.use_cudnn, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } def test_check_output(self): @@ -68,53 +74,49 @@ def test_check_grad(self): if self.use_cudnn or self.dtype == np.float16: place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ["X"], - "Out", - max_relative_error=0.01, - check_dygraph=False) + self.check_grad_with_place( + place, + ["X"], + "Out", + max_relative_error=0.01, + check_dygraph=False, + ) else: - self.check_grad(["X"], - "Out", - max_relative_error=0.01, - check_dygraph=False) + self.check_grad( + ["X"], "Out", max_relative_error=0.01, check_dygraph=False + ) def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp2(TestSoftmaxOp2): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp3(TestSoftmaxOp3): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp4(TestSoftmaxOp4): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp5(TestSoftmaxOp5): - def init_kernel_type(self): self.use_mkldnn = True class TestSoftmaxMKLDNNOp6(TestSoftmaxOp6): - def init_kernel_type(self): self.use_mkldnn = True # Check if primitives already exist in backward class TestSoftmaxMKLDNNPrimitivesAlreadyExist(unittest.TestCase): - def setUp(self): super(TestSoftmaxMKLDNNPrimitivesAlreadyExist, self).setUp() @@ -130,12 +132,13 @@ def __softmax_bwd(self, out, out_grad): return out * (out_grad - np.dot(out, out_grad)) def test_check(self): - check_if_mkldnn_primitives_exist_in_bwd(self, self.op_type, self.x, - self.out, self.out_grad, - self.x_grad) + check_if_mkldnn_primitives_exist_in_bwd( + self, self.op_type, self.x, self.out, self.out_grad, self.x_grad + ) if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_softplus_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_softplus_mkldnn_op.py index db7eeb88f1d0d..49846a2280cda 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_softplus_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_softplus_mkldnn_op.py @@ -14,20 +14,25 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) import paddle def ref_softplus(x, beta, threshold): x_beta = beta * x - out = np.select([x_beta <= threshold, x_beta > threshold], - [np.log(1 + np.exp(x_beta)) / beta, x]) + out = np.select( + [x_beta <= threshold, x_beta > threshold], + [np.log(1 + np.exp(x_beta)) / beta, x], + ) return out @OpTestTool.skip_if_not_cpu_bf16() class TestSoftplusOneDNNOp(OpTest): - def setUp(self): self.op_type = "softplus" self.beta = 1 @@ -57,52 +62,45 @@ def test_check_output(self): class TestSoftplus4DOneDNNOp(TestSoftplusOneDNNOp): - def config(self): self.x_shape = (10, 5, 4, 2) class TestSoftplus6DOneDNNOp(TestSoftplusOneDNNOp): - def config(self): self.x_shape = (3, 2, 2, 5, 4, 2) class TestSoftplus6DExtendedFunctorOneDNNOp(TestSoftplusOneDNNOp): - def config(self): self.x_shape = (3, 5, 2, 5, 4, 2) self.beta = 2.5 class TestSoftplus3DExtendedFunctorOneDNNOp(TestSoftplusOneDNNOp): - def config(self): self.x_shape = (20, 4, 2) self.beta = 0.4 class TestSoftplusBF16OneDNNOp(TestSoftplusOneDNNOp): - def set_dtype(self): self.dtype = np.uint16 class TestSoftplus4DBF16OneDNNOp(TestSoftplus4DOneDNNOp): - def set_dtype(self): self.dtype = np.uint16 class TestSoftplus6DBF16OneDNNOp(TestSoftplus6DOneDNNOp): - def set_dtype(self): self.dtype = np.uint16 class TestSoftplus3DExtendedFunctorBF16OneDNNOp( - TestSoftplus3DExtendedFunctorOneDNNOp): - + TestSoftplus3DExtendedFunctorOneDNNOp +): def set_dtype(self): self.dtype = np.uint16 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_split_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_split_bf16_mkldnn_op.py index 56416629d771d..a3345c1922d54 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_split_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_split_bf16_mkldnn_op.py @@ -19,12 +19,14 @@ from paddle.fluid.tests.unittests.op_test import OpTest -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") -@unittest.skipIf(core.is_compiled_with_cuda(), - "core is compiled with CUDA which has no BF implementation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) +@unittest.skipIf( + core.is_compiled_with_cuda(), + "core is compiled with CUDA which has no BF implementation", +) class TestSplitSectionsBF16OneDNNOp(OpTest): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("uint16") self.axis = 1 @@ -43,7 +45,7 @@ def setUp(self): self.attrs = { 'use_mkldnn': True, 'num': self.num, - 'mkldnn_data_type': "bfloat16" + 'mkldnn_data_type': "bfloat16", } if self.axis is not None: @@ -55,8 +57,9 @@ def setUp(self): if self.sections_tensor_list is not None: self.inputs['SectionsTensorList'] = self.sections_tensor_list - self.outputs = {'Out': [('out%d' % i, self.out[i]) \ - for i in range(len(self.out))]} + self.outputs = { + 'Out': [('out%d' % i, self.out[i]) for i in range(len(self.out))] + } def test_check_output(self): self.check_output_with_place(core.CPUPlace()) @@ -73,50 +76,47 @@ def test_check_output(self): class TestSplitNumBF16OneDNNOp(TestSplitSectionsBF16OneDNNOp): - def init_data(self): self.x = np.random.random((4, 8, 5, 3)).astype("uint16") self.axis = 1 self.sections = [] self.num = 4 - indices_or_sections = 4 #indices + indices_or_sections = 4 # indices self.out = np.split(self.x, indices_or_sections, self.axis) class TestSplitNumAxisTensorBF16OneDNNOp(TestSplitSectionsBF16OneDNNOp): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("uint16") self.axis = None self.sections = [] self.num = 3 - indices_or_sections = 3 #indices + indices_or_sections = 3 # indices self.axis_tensor = np.array([2]).astype("int32") self.out = np.split(self.x, indices_or_sections, 2) class TestSplitSectionsTensorBF16OneDNNOp(TestSplitSectionsBF16OneDNNOp): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("uint16") self.axis = 1 self.sections = [2, 1, 2] self.sections_tensor_list = [] for index, ele in enumerate(self.sections): - self.sections_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + self.sections_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.sections = [-1, -1, -1] - indices_or_sections = [2, 3] #sections + indices_or_sections = [2, 3] # sections self.out = np.split(self.x, indices_or_sections, self.axis) class TestSplitOpUnknownSectionBF16OneDNNOp(TestSplitSectionsBF16OneDNNOp): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("uint16") self.axis = 2 self.sections = [2, 2, -1] - indices_or_sections = [2, 4] #sections + indices_or_sections = [2, 4] # sections self.out = np.split(self.x, indices_or_sections, self.axis) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_split_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_split_mkldnn_op.py index 2b1e1fc9f226a..e148e0cdcab50 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_split_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_split_mkldnn_op.py @@ -19,7 +19,6 @@ class TestSplitSectionsOneDNNOp(OpTest): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("float32") self.axis = 1 @@ -46,8 +45,9 @@ def setUp(self): if self.sections_tensor_list is not None: self.inputs['SectionsTensorList'] = self.sections_tensor_list - self.outputs = {'Out': [('out%d' % i, self.out[i]) \ - for i in range(len(self.out))]} + self.outputs = { + 'Out': [('out%d' % i, self.out[i]) for i in range(len(self.out))] + } def test_check_output(self): self.check_output() @@ -58,13 +58,12 @@ def test_check_grad(self): # test with attr(num) class TestSplitNumOneDNNOp(TestSplitSectionsOneDNNOp): - def init_data(self): self.x = np.random.random((4, 8, 5, 3)).astype("float32") self.axis = 1 self.sections = [] self.num = 4 - indices_or_sections = 4 #indices + indices_or_sections = 4 # indices self.out = np.split(self.x, indices_or_sections, self.axis) def test_check_grad(self): @@ -72,40 +71,38 @@ def test_check_grad(self): class TestSplitNumAxisTensorOneDNNOp(TestSplitSectionsOneDNNOp): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("float32") self.axis = None self.sections = [] self.num = 3 - indices_or_sections = 3 #indices + indices_or_sections = 3 # indices self.axis_tensor = np.array([2]).astype("int32") self.out = np.split(self.x, indices_or_sections, 2) # attr(sections) is list containing Tensor class TestSplitSectionsTensorOneDNNOp(TestSplitSectionsOneDNNOp): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("float32") self.axis = 1 self.sections = [2, 1, 2] self.sections_tensor_list = [] for index, ele in enumerate(self.sections): - self.sections_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + self.sections_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.sections = [-1, -1, -1] - indices_or_sections = [2, 3] #sections + indices_or_sections = [2, 3] # sections self.out = np.split(self.x, indices_or_sections, self.axis) class TestSplitOpUnknownSectionOneDNNOp(TestSplitSectionsOneDNNOp): - def init_data(self): self.x = np.random.random((4, 5, 6)).astype("float32") self.axis = 2 self.sections = [2, 2, -1] - indices_or_sections = [2, 4] #sections + indices_or_sections = [2, 4] # sections self.out = np.split(self.x, indices_or_sections, self.axis) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_squeeze2_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_squeeze2_mkldnn_op.py index 9d8647b908670..0b5cca9d5ad01 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_squeeze2_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_squeeze2_mkldnn_op.py @@ -16,13 +16,18 @@ import numpy as np import paddle import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + OpTestTool, + convert_float_to_uint16, +) -@OpTestTool.skip_if(core.is_compiled_with_cuda(), - "CUDA has to be skipped because it forces dygraph") +@OpTestTool.skip_if( + core.is_compiled_with_cuda(), + "CUDA has to be skipped because it forces dygraph", +) class TestSqueeze2OneDNNOp(OpTest): - def set_op_type(self): self.op_type = "squeeze2" @@ -40,7 +45,7 @@ def init_attrs(self): def set_outputs(self): self.outputs = { "Out": self.x.reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float32") + "XShape": np.random.random(self.ori_shape).astype("float32"), } def setUp(self): @@ -59,7 +64,6 @@ def test_check_grad(self): class TestSqueezeOneDNNOp(TestSqueeze2OneDNNOp): - def set_op_type(self): self.op_type = "squeeze" @@ -71,7 +75,6 @@ def test_check_output(self): class TestSqueeze2OneDNNOp1(TestSqueeze2OneDNNOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = (0, -2) @@ -79,7 +82,6 @@ def init_test_case(self): class TestSqueezeOneDNNOp1(TestSqueezeOneDNNOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = (0, -2) @@ -87,7 +89,6 @@ def init_test_case(self): class TestSqueeze2OneDNNOp2(TestSqueeze2OneDNNOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -95,7 +96,6 @@ def init_test_case(self): class TestSqueezeOneDNNOp2(TestSqueezeOneDNNOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -103,7 +103,6 @@ def init_test_case(self): class TestSqueeze2OneDNNOp3(TestSqueeze2OneDNNOp): - def init_test_case(self): self.ori_shape = (25, 1, 1, 4, 1) self.axes = (1, -1) @@ -111,7 +110,6 @@ def init_test_case(self): class TestSqueezeOneDNNOp3(TestSqueezeOneDNNOp): - def init_test_case(self): self.ori_shape = (25, 1, 1, 4, 1) self.axes = (1, -1) @@ -120,10 +118,8 @@ def init_test_case(self): # BF16 TESTS def create_squeeze_bf16_test_classes(parent): - @OpTestTool.skip_if_not_cpu_bf16() class TestSqueeze2BF16OneDNNOp(parent): - def set_inputs(self): self.dtype = np.uint16 self.inputs = {"X": convert_float_to_uint16(self.x)} @@ -134,17 +130,19 @@ def calculate_grads(self): def test_check_grad(self): self.calculate_grads() - self.check_grad_with_place(core.CPUPlace(), ["X"], - "Out", - user_defined_grads=[self.dx], - user_defined_grad_outputs=[self.dout]) + self.check_grad_with_place( + core.CPUPlace(), + ["X"], + "Out", + user_defined_grads=[self.dx], + user_defined_grad_outputs=[self.dout], + ) cls_name = "{0}_{1}".format(parent.__name__, "Squeeze2_BF16") TestSqueeze2BF16OneDNNOp.__name__ = cls_name globals()[cls_name] = TestSqueeze2BF16OneDNNOp class TestSqueezeBF16OneDNNOp(TestSqueeze2BF16OneDNNOp): - def set_op_type(self): self.dtype = np.uint16 self.op_type = "squeeze" diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_stack_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_stack_mkldnn_op.py index 5638c14b9e488..4052d50a474a2 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_stack_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_stack_mkldnn_op.py @@ -21,7 +21,6 @@ @OpTestTool.skip_if_not_cpu() class TestStack2DOneDNNOp(OpTest): - def initDefaultParameters(self): self.num_inputs = 4 self.input_dim = (2, 2) @@ -45,7 +44,8 @@ def setUp(self): for i in range(self.num_inputs): self.op_inputs.append( - np.random.random(size=self.input_dim).astype(np.float32)) + np.random.random(size=self.input_dim).astype(np.float32) + ) input_list = [] input_names = self.getInputNames() @@ -65,21 +65,18 @@ def test_check_grad(self): class TestStack1DOneDNNOp(TestStack2DOneDNNOp): - def initParameters(self): - self.input_dim = (100) + self.input_dim = 100 self.axis = 0 class TestStack1DAxis1OneDNNOp(TestStack2DOneDNNOp): - def initParameters(self): - self.input_dim = (100) + self.input_dim = 100 self.axis = 1 class TestStack2DAxisLastOneDNNOp(TestStack2DOneDNNOp): - def initParameters(self): self.input_dim = (13, 24) self.num_inputs = 5 @@ -87,14 +84,12 @@ def initParameters(self): class TestStack3DAxisNegativeOneDNNOp(TestStack2DOneDNNOp): - def initParameters(self): self.input_dim = (10, 128, 128) self.axis = -2 class TestStack3DOneDNNOp(TestStack2DOneDNNOp): - def initParameters(self): self.input_dim = (10, 128, 128) self.num_inputs = 3 @@ -102,7 +97,6 @@ def initParameters(self): class TestStack4DOneDNNOp(TestStack2DOneDNNOp): - def initParameters(self): self.input_dim = (2, 2, 2, 2) self.num_inputs = 3 @@ -110,7 +104,6 @@ def initParameters(self): class TestStack5DOneDNNOp(TestStack2DOneDNNOp): - def initParameters(self): self.input_dim = (2, 3, 4, 5, 6) self.num_inputs = 6 diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_sum_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_sum_bf16_mkldnn_op.py index 6792eaf7e6944..435651f38a79c 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_sum_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_sum_bf16_mkldnn_op.py @@ -20,10 +20,10 @@ import numpy as np -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestSumBF16MKLDNN(TestSumOp): - def setUp(self): self.op_type = "sum" self.use_mkldnn = True diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_sum_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_sum_mkldnn_op.py index a9c345e9dfa92..ab26f10168787 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_sum_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_sum_mkldnn_op.py @@ -20,7 +20,6 @@ class TestSumMKLDNN(TestSumOp): - def setUp(self): self.op_type = "sum" self.init_data_type() @@ -46,7 +45,6 @@ def test_check_grad(self): class TestMKLDNNSumInplaceOp(unittest.TestCase): - def setUp(self): self.op_type = "sum" self.init_data_type() @@ -70,10 +68,9 @@ def test_check_output(self): tensor = var.get_tensor() tensor.set(var_value, place) - sum_op = fluid_op.Operator("sum", - X=["x0", "x1"], - Out=out_var_name, - use_mkldnn=True) + sum_op = fluid_op.Operator( + "sum", X=["x0", "x1"], Out=out_var_name, use_mkldnn=True + ) expected_out = np.array(self.x0 + self.x1) sum_op.run(scope, place) out = scope.find_var("x0").get_tensor() @@ -83,7 +80,7 @@ def test_check_output(self): out_array, rtol=1e-05, atol=1e-05, - err_msg='Inplace sum_mkldnn_op output has diff with expected output' + err_msg='Inplace sum_mkldnn_op output has diff with expected output', ) def test_check_grad(self): @@ -92,5 +89,6 @@ def test_check_grad(self): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_bf16_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_bf16_mkldnn_op.py index 2edab93d778de..352e35f66dd31 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_bf16_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_bf16_mkldnn_op.py @@ -19,10 +19,10 @@ from paddle import enable_static -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestTransposeOp(OpTest): - def setUp(self): self.op_type = "transpose2" self.use_mkldnn = True @@ -36,12 +36,12 @@ def setUp(self): self.attrs = { 'axis': list(self.axis), 'use_mkldnn': self.use_mkldnn, - 'mkldnn_data_type': self.mkldnn_data_type + 'mkldnn_data_type': self.mkldnn_data_type, } self.outputs = { 'XShape': np.random.random(self.shape).astype(np.uint16), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } def test_check_output(self): @@ -52,11 +52,11 @@ def init_test_case(self): def init_test_data(self): self.input_data = convert_float_to_uint16( - np.random.random(self.shape).astype(np.float32)) + np.random.random(self.shape).astype(np.float32) + ) class TestBF16Case(TestTransposeOp): - def init_test_case(self): self.shape = (2, 4, 6, 8) diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_int8_mkldnn_op.py index 5b3be06aa7505..15c89a37e0922 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_int8_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_int8_mkldnn_op.py @@ -20,7 +20,6 @@ class TestTransposeOp(OpTest): - def setUp(self): self.init_op_type() self.initTestCase() @@ -31,7 +30,7 @@ def setUp(self): self.inputs = { 'X': format_reorder(self.input_data, self.shape).astype(np.int8) - } #transform data format to 'NHWC' for INT8 transpose specially. + } # transform data format to 'NHWC' for INT8 transpose specially. self.attrs = { 'axis': list(self.axis), @@ -40,7 +39,7 @@ def setUp(self): self.outputs = { 'XShape': np.random.random(self.shape).astype(np.int8), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } def init_op_type(self): @@ -48,37 +47,37 @@ def init_op_type(self): def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output_with_place(core.CPUPlace(), - 1e-5, - no_check_set=['XShape'], - check_dygraph=False) + self.check_output_with_place( + core.CPUPlace(), 1e-5, no_check_set=['XShape'], check_dygraph=False + ) def initTestCase(self): self.shape = (2, 3, 4, 5) def initInputData(self): self.input_data = (np.random.randint(0, 100, self.shape) - 50).astype( - np.int8) + np.int8 + ) class TestINT8Case(TestTransposeOp): - def initTestCase(self): self.shape = (2, 4, 6, 8) def initInputData(self): self.input_data = (np.random.randint(0, 100, self.shape) - 50).astype( - np.int8) + np.int8 + ) class TestUINT8Case(TestTransposeOp): - def initTestCase(self): self.shape = (1, 3, 5, 7) def initDataType(self): - self.input_data = (np.random.randint(0, 100, - self.shape)).astype(np.uint8) + self.input_data = (np.random.randint(0, 100, self.shape)).astype( + np.uint8 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_mkldnn_op.py index 5e81358c688b1..292c98b4dae23 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_transpose_mkldnn_op.py @@ -19,7 +19,6 @@ class TestTransposeMKLDNN(TestTransposeOp): - def setUp(self): self.init_op_type() self.initTestCase() @@ -30,7 +29,7 @@ def setUp(self): } self.outputs = { 'XShape': np.random.random(self.shape).astype("float32"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } def init_op_type(self): @@ -52,42 +51,36 @@ def initTestCase(self): class TestCase0MKLDNN(TestTransposeMKLDNN): - def initTestCase(self): - self.shape = (100, ) - self.axis = (0, ) + self.shape = (100,) + self.axis = (0,) class TestCase1a(TestTransposeMKLDNN): - def initTestCase(self): self.shape = (3, 4, 10) self.axis = (0, 2, 1) class TestCase1b(TestTransposeMKLDNN): - def initTestCase(self): self.shape = (3, 4, 10) self.axis = (2, 1, 0) class TestCase2(TestTransposeMKLDNN): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.axis = (0, 2, 3, 1) class TestCase3(TestTransposeMKLDNN): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.axis = (4, 2, 3, 1, 0) class TestCase4(TestTransposeMKLDNN): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6, 1) self.axis = (4, 2, 3, 1, 0, 5) diff --git a/python/paddle/fluid/tests/unittests/mlu/c_comm_init_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/c_comm_init_op_mlu.py index 9e6385bda4a75..ca779db12c241 100644 --- a/python/paddle/fluid/tests/unittests/mlu/c_comm_init_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/c_comm_init_op_mlu.py @@ -16,14 +16,15 @@ import os import paddle.fluid.core as core import paddle.fluid as fluid -from paddle.distributed.fleet.base.private_helper_function import wait_server_ready +from paddle.distributed.fleet.base.private_helper_function import ( + wait_server_ready, +) import paddle paddle.enable_static() class TestCCommInitOp(unittest.TestCase): - def setUp(self): self.endpoints = os.getenv("PADDLE_TRAINER_ENDPOINTS").split(',') self.current_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT") @@ -43,24 +44,29 @@ def test_specifying_devices(self): cncl_id_var = block.create_var( name=fluid.unique_name.generate('cncl_id'), persistable=True, - type=fluid.core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_cncl_id', - inputs={}, - outputs={'Out': cncl_id_var}, - attrs={ - 'rank': self.rank, - 'endpoint': self.current_endpoint, - 'other_endpoints': self.other_endpoints - }) - block.append_op(type='c_comm_init', - inputs={'X': cncl_id_var}, - outputs={}, - attrs={ - 'nranks': self.nranks, - 'rank': self.rank, - 'ring_id': 0, - 'device_id': self.mlu_id - }) + type=fluid.core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_cncl_id', + inputs={}, + outputs={'Out': cncl_id_var}, + attrs={ + 'rank': self.rank, + 'endpoint': self.current_endpoint, + 'other_endpoints': self.other_endpoints, + }, + ) + block.append_op( + type='c_comm_init', + inputs={'X': cncl_id_var}, + outputs={}, + attrs={ + 'nranks': self.nranks, + 'rank': self.rank, + 'ring_id': 0, + 'device_id': self.mlu_id, + }, + ) self.exe.run(program) diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_allgather_api.py b/python/paddle/fluid/tests/unittests/mlu/collective_allgather_api.py index c9c681ae38b0e..1d8d11ad11200 100755 --- a/python/paddle/fluid/tests/unittests/mlu/collective_allgather_api.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_allgather_api.py @@ -30,22 +30,24 @@ from multiprocessing import Process import paddle.fluid.layers as layers from functools import reduce -from test_collective_api_base_mlu import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_api_base_mlu import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) paddle.enable_static() class TestCollectiveAllgatherAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): tensor_list = [] - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) paddle.distributed.all_gather(tensor_list, tindata) return tensor_list diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_allgather_op.py b/python/paddle/fluid/tests/unittests/mlu/collective_allgather_op.py index 727f3011ff14c..8040e834d65a9 100755 --- a/python/paddle/fluid/tests/unittests/mlu/collective_allgather_op.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_allgather_op.py @@ -35,7 +35,6 @@ class TestCollectiveAllgather(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -43,26 +42,28 @@ def get_model(self, main_prog, startup_program, col_type): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofallgather", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_allgather", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_allgather", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'nranks': nranks}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_api.py b/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_api.py index 7ea3a2f00c53e..22ca990c55afd 100644 --- a/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_api.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_api.py @@ -30,21 +30,23 @@ from multiprocessing import Process import paddle.fluid.layers as layers from functools import reduce -from test_collective_api_base_mlu import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_api_base_mlu import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) paddle.enable_static() class TestCollectiveAllreduceAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) paddle.distributed.all_reduce(tindata) return [tindata] diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_op.py b/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_op.py index 19da310741d6e..609d9e7c41688 100644 --- a/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_op.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_allreduce_op.py @@ -36,30 +36,34 @@ class TestCollectiveAllreduce(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, col_type): ring_id = 0 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outof" + col_type, dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_" + col_type, - inputs={'X': tindata}, - attrs={'ring_id': ring_id}, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_" + col_type, + inputs={'X': tindata}, + attrs={'ring_id': ring_id}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_api.py b/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_api.py index c14456704dfc8..0c1ae572251aa 100644 --- a/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_api.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_api.py @@ -30,21 +30,23 @@ from multiprocessing import Process import paddle.fluid.layers as layers from functools import reduce -from test_collective_api_base_mlu import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_api_base_mlu import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) paddle.enable_static() class TestCollectiveBroadcastAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype="float32") + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype="float32" + ) paddle.distributed.broadcast(tindata, src=1) return [tindata] diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_op.py b/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_op.py index 3dd50f09a8f5e..982d5d204fccc 100755 --- a/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_op.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_broadcast_op.py @@ -36,7 +36,6 @@ class TestCollectiveBroadcast(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -44,26 +43,28 @@ def get_model(self, main_prog, startup_program, col_type): ring_id = 0 rootid = 1 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofbroadcast", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_broadcast", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'root': rootid - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_broadcast", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'root': rootid}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_reduce_api.py b/python/paddle/fluid/tests/unittests/mlu/collective_reduce_api.py index 4f257d07fa331..33a0d9d0d51a7 100644 --- a/python/paddle/fluid/tests/unittests/mlu/collective_reduce_api.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_reduce_api.py @@ -30,21 +30,23 @@ from multiprocessing import Process import paddle.fluid.layers as layers from functools import reduce -from test_collective_api_base_mlu import TestCollectiveAPIRunnerBase, runtime_main +from test_collective_api_base_mlu import ( + TestCollectiveAPIRunnerBase, + runtime_main, +) paddle.enable_static() class TestCollectiveReduceAPI(TestCollectiveAPIRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program, rank): with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) paddle.distributed.reduce(tindata, dst=0) return [tindata] diff --git a/python/paddle/fluid/tests/unittests/mlu/collective_reduce_op.py b/python/paddle/fluid/tests/unittests/mlu/collective_reduce_op.py index 733440cb0de47..d9271376b0f31 100644 --- a/python/paddle/fluid/tests/unittests/mlu/collective_reduce_op.py +++ b/python/paddle/fluid/tests/unittests/mlu/collective_reduce_op.py @@ -36,7 +36,6 @@ class TestCollectiveReduce(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -44,26 +43,28 @@ def get_model(self, main_prog, startup_program, col_type): ring_id = 0 rootid = 1 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outof" + col_type, dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_" + col_type, - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'root_id': rootid - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_" + col_type, + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'root_id': rootid}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/mlu/multi_process_mlu.py b/python/paddle/fluid/tests/unittests/mlu/multi_process_mlu.py index 782475ff8cb5e..96148a7bacc4d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/multi_process_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/multi_process_mlu.py @@ -26,12 +26,18 @@ def train(prefix): worker_endpoints = worker_endpoints_env trainers_num = len(worker_endpoints.split(',')) - name = "selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_mlus, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_mlus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) - with open("multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + with open( + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(name) @@ -48,23 +54,34 @@ def train_abort(prefix): # train abort exit(1) except SystemExit: - name = "abort>>> selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_mlus, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "abort>>> selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_mlus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) with open( - "multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(name) raise else: # sleep 30s to make sure paddle.distributed.launch will terminate this process time.sleep(30) - name = "selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_mlus, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_mlus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) - with open("multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + with open( + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(name) diff --git a/python/paddle/fluid/tests/unittests/mlu/nproc_process_mlu.py b/python/paddle/fluid/tests/unittests/mlu/nproc_process_mlu.py index 9b2713532e41b..2a070bf67227f 100644 --- a/python/paddle/fluid/tests/unittests/mlu/nproc_process_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/nproc_process_mlu.py @@ -25,8 +25,13 @@ def train(prefix): worker_endpoints = worker_endpoints_env trainers_num = len(worker_endpoints.split(',')) - name = "selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_mlus, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "selected_mlus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_mlus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) with open("{}.check_{}.log".format(prefix, trainer_id), "w") as f: diff --git a/python/paddle/fluid/tests/unittests/mlu/parallel_dygraph_sync_batch_norm.py b/python/paddle/fluid/tests/unittests/mlu/parallel_dygraph_sync_batch_norm.py index b562f9ce4acaf..56abdfc26dda9 100644 --- a/python/paddle/fluid/tests/unittests/mlu/parallel_dygraph_sync_batch_norm.py +++ b/python/paddle/fluid/tests/unittests/mlu/parallel_dygraph_sync_batch_norm.py @@ -32,37 +32,42 @@ class TestLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(TestLayer, self).__init__() - self._conv = Conv2D(in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=False) + self._conv = Conv2D( + in_channels=num_channels, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + bias_attr=False, + ) self._sync_batch_norm = SyncBatchNorm(num_filters) - self._conv2 = Conv2D(in_channels=num_filters, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=False) + self._conv2 = Conv2D( + in_channels=num_filters, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + bias_attr=False, + ) - self._sync_batch_norm2 = SyncBatchNorm(num_filters, - weight_attr=False, - bias_attr=False) + self._sync_batch_norm2 = SyncBatchNorm( + num_filters, weight_attr=False, bias_attr=False + ) def forward(self, inputs): y = self._conv(inputs) @@ -74,20 +79,23 @@ def forward(self, inputs): class TestSyncBatchNorm(TestParallelDyGraphRunnerBase): - def get_model(self): model = TestLayer(3, 64, 7) - train_reader = paddle.batch(paddle.dataset.flowers.test(use_xmap=False), - batch_size=32, - drop_last=True) - opt = fluid.optimizer.Adam(learning_rate=1e-3, - parameter_list=model.parameters()) + train_reader = paddle.batch( + paddle.dataset.flowers.test(use_xmap=False), + batch_size=32, + drop_last=True, + ) + opt = fluid.optimizer.Adam( + learning_rate=1e-3, parameter_list=model.parameters() + ) return model, train_reader, opt def run_one_loop(self, model, opt, data): batch_size = len(data) - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') + dy_x_data = np.array([x[0].reshape(3, 224, 224) for x in data]).astype( + 'float32' + ) img = to_variable(dy_x_data) img.stop_gradient = False diff --git a/python/paddle/fluid/tests/unittests/mlu/sync_batch_norm_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/sync_batch_norm_op_mlu.py index eb410f14f540a..3ea79600954aa 100644 --- a/python/paddle/fluid/tests/unittests/mlu/sync_batch_norm_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/sync_batch_norm_op_mlu.py @@ -31,7 +31,10 @@ from multiprocessing import Process import paddle.fluid.layers as layers from functools import reduce -from test_sync_batch_norm_base_mlu import TestSyncBatchNormRunnerBase, runtime_main +from test_sync_batch_norm_base_mlu import ( + TestSyncBatchNormRunnerBase, + runtime_main, +) from op_test import OpTest, _set_use_system_allocator from test_sync_batch_norm_op import create_or_get_tensor @@ -41,7 +44,6 @@ class TestSyncBatchNormOpTraining(TestSyncBatchNormRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -54,29 +56,34 @@ def __init__(self): self.dshape = [self.N, self.C, self.H, self.W] self.atol = 1e-3 - def get_model(self, - main, - startup, - place, - layout, - seed, - sync_bn=False, - only_forward=False): + def get_model( + self, + main, + startup, + place, + layout, + seed, + sync_bn=False, + only_forward=False, + ): """Build program.""" use_cudnn = False with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - data = fluid.layers.data(name='input', - shape=self.dshape, - dtype=self.dtype, - append_batch_size=False) + data = fluid.layers.data( + name='input', + shape=self.dshape, + dtype=self.dtype, + append_batch_size=False, + ) conv = fluid.layers.conv2d( input=data, num_filters=32, filter_size=1, param_attr=fluid.ParamAttr(name='conv2d_weight'), bias_attr=False, - use_cudnn=use_cudnn) + use_cudnn=use_cudnn, + ) if self.bn_dtype == np.float16: conv = fluid.layers.cast(conv, 'float16') bn = fluid.layers.batch_norm( @@ -86,7 +93,8 @@ def get_model(self, moving_mean_name='bn_moving_mean', moving_variance_name='bn_moving_variance', data_layout=layout, - is_test=only_forward) + is_test=only_forward, + ) if self.bn_dtype == np.float16: bn = fluid.layers.cast(bn, 'float32') sigmoid = fluid.layers.sigmoid(bn) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_abs_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_abs_op_mlu.py index fe56236b8e25a..c9e09fccf20e3 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_abs_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_abs_op_mlu.py @@ -29,7 +29,6 @@ class TestAbs(OpTest): - def setUp(self): self.op_type = "abs" self.set_mlu() @@ -56,12 +55,12 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], ['Out'], - check_eager=False) + self.check_grad_with_place( + self.place, ['X'], ['Out'], check_eager=False + ) class TestAbsHalf(OpTest): - def setUp(self): self.op_type = "abs" self.set_mlu() @@ -88,8 +87,9 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], ['Out'], - check_eager=False) + self.check_grad_with_place( + self.place, ['X'], ['Out'], check_eager=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/mlu/test_accuracy_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_accuracy_op_mlu.py index 0cdc2045ec7b7..e37be2f9f3d11 100755 --- a/python/paddle/fluid/tests/unittests/mlu/test_accuracy_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_accuracy_op_mlu.py @@ -26,7 +26,6 @@ class TestAccuracyOp(OpTest): - def setUp(self): self.op_type = "accuracy" self.place = paddle.device.MLUPlace(0) @@ -47,7 +46,7 @@ def setUp(self): self.outputs = { 'Accuracy': np.array([num_correct / float(n)]).astype(self.dtype), 'Correct': np.array([num_correct]).astype("int32"), - 'Total': np.array([n]).astype("int32") + 'Total': np.array([n]).astype("int32"), } def init_dtype(self): @@ -58,7 +57,6 @@ def test_check_output(self): class TestAccuracyOpFp16(TestAccuracyOp): - def init_dtype(self): self.dtype = np.float16 @@ -67,15 +65,15 @@ def test_check_output(self): class TestAccuracyOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of accuracy_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.MLUPlace(0)) - label = fluid.layers.data(name='label', - shape=[-1, 1], - dtype="int32") + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.MLUPlace(0) + ) + label = fluid.layers.data( + name='label', shape=[-1, 1], dtype="int32" + ) self.assertRaises(TypeError, fluid.layers.accuracy, x1, label) self.assertRaises(TypeError, paddle.metric.accuracy, x1, label) # The input dtype of accuracy_op must be float32 or float64. @@ -88,40 +86,42 @@ def test_errors(self): class TestAccuracyAPI1(unittest.TestCase): - def setUp(self): - self.predictions = paddle.static.data(shape=[2, 5], - name="predictions", - dtype="float32") - self.label = paddle.static.data(shape=[2, 1], - name="labels", - dtype="int32") - self.result = paddle.static.accuracy(input=self.predictions, - label=self.label, - k=1) + self.predictions = paddle.static.data( + shape=[2, 5], name="predictions", dtype="float32" + ) + self.label = paddle.static.data( + shape=[2, 1], name="labels", dtype="int32" + ) + self.result = paddle.static.accuracy( + input=self.predictions, label=self.label, k=1 + ) self.input_predictions = np.array( [[0.2, 0.1, 0.4, 0.1, 0.1], [0.2, 0.3, 0.1, 0.15, 0.25]], - dtype="float32") + dtype="float32", + ) self.input_labels = np.array([[2], [0]], dtype="int32") self.expect_value = np.array([0.5], dtype='float32') def test_api(self): exe = paddle.static.Executor() - result, = exe.run(feed={ - "predictions": self.input_predictions, - 'labels': self.input_labels - }, - fetch_list=[self.result.name]) + (result,) = exe.run( + feed={ + "predictions": self.input_predictions, + 'labels': self.input_labels, + }, + fetch_list=[self.result.name], + ) self.assertEqual((result == self.expect_value).all(), True) class TestAccuracyAPI2(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): predictions = paddle.to_tensor( [[0.2, 0.1, 0.4, 0.1, 0.1], [0.2, 0.3, 0.1, 0.15, 0.25]], - dtype='float32') + dtype='float32', + ) label = paddle.to_tensor([[2], [0]], dtype="int32") result = paddle.static.accuracy(input=predictions, label=label, k=1) expect_value = np.array([0.5], dtype='float32') @@ -129,12 +129,12 @@ def test_api(self): class TestAccuracyAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): predictions = paddle.to_tensor( [[0.2, 0.1, 0.4, 0.1, 0.1], [0.2, 0.3, 0.1, 0.15, 0.25]], - dtype='float32') + dtype='float32', + ) label = paddle.to_tensor([[2], [0]], dtype="int32") result = paddle.metric.accuracy(input=predictions, label=label, k=1) expect_value = np.array([0.5], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/mlu/test_adam_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_adam_op_mlu.py index 3ac6e23d21a4d..70a3a2671b6b8 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_adam_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_adam_op_mlu.py @@ -28,7 +28,6 @@ class TestAdam(OpTest): - def setUp(self): self.set_mlu() self.op_type = "adam" @@ -52,20 +51,19 @@ def setUp(self): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } self.attrs = {'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def set_mlu(self): @@ -80,7 +78,6 @@ def test_check_output(self): class TestAdamWithEpsilonTensor(OpTest): - def setUp(self): self.set_mlu() self.op_type = "adam" @@ -112,15 +109,14 @@ def setUp(self): self.attrs = {'epsilon': epsilon} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def set_mlu(self): @@ -135,7 +131,6 @@ def test_check_output(self): class TestAdamOpWithSkipUpdate(OpTest): - def setUp(self): self.set_mlu() self.op_type = "adam" @@ -188,7 +183,6 @@ def test_check_output(self): class TestAdamOpWithGlobalBetaPow(OpTest): - def setUp(self): self.set_mlu() self.op_type = "adam" @@ -220,8 +214,7 @@ def setUp(self): attributes = {'epsilon': epsilon} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, attributes) + param_out, moment1_out, moment2_out = adam_step(self.inputs, attributes) self.attrs = {'use_global_beta_pow': True} @@ -231,7 +224,7 @@ def setUp(self): 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([]), - 'Beta2PowOut': np.array([]) + 'Beta2PowOut': np.array([]), } def set_mlu(self): @@ -246,7 +239,6 @@ def test_check_output(self): class TestNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -261,9 +253,9 @@ def _test(self, run_mlu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -287,16 +279,17 @@ def _test(self, run_mlu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_adamw_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_adamw_op_mlu.py index e60664f27a0e1..c9b3b527e72db 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_adamw_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_adamw_op_mlu.py @@ -28,7 +28,6 @@ class TestAdamW(OpTest): - def setUp(self): self.set_mlu() self.op_type = "adamw" @@ -52,7 +51,7 @@ def setUp(self): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } self.attrs = { @@ -60,18 +59,19 @@ def setUp(self): 'beta1': beta1, 'beta2': beta2, "coeff": 0.9, - "with_decay": True + "with_decay": True, } - param_out, moment1_out, \ - moment2_out = adamw_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adamw_step( + self.inputs, self.attrs + ) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def set_mlu(self): @@ -86,7 +86,6 @@ def test_check_output(self): class TestAdamOpWithSkipUpdate(OpTest): - def setUp(self): self.set_mlu() self.op_type = "adamw" @@ -139,7 +138,6 @@ def test_check_output(self): class TestAdamOpWithoutDecay(OpTest): - def setUp(self): self.set_mlu() self.op_type = "adamw" @@ -192,7 +190,6 @@ def test_check_output(self): class TestNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -207,9 +204,9 @@ def _test(self, run_mlu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -233,16 +230,17 @@ def _test(self, run_mlu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_amp_check_finite_and_scale_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_amp_check_finite_and_scale_op_mlu.py index 3e113847b7a78..7a2e22a18ad1d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_amp_check_finite_and_scale_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_amp_check_finite_and_scale_op_mlu.py @@ -25,7 +25,6 @@ class TestCheckFiniteAndUnscaleOp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "check_finite_and_unscale" @@ -54,7 +53,6 @@ def test_check_output(self): class TestCheckFiniteAndUnscaleOpWithNan(TestCheckFiniteAndUnscaleOp): - def init_test_case(self): x = np.random.random((129, 129)).astype(self.dtype) x[128][128] = np.nan @@ -73,7 +71,6 @@ def test_check_output(self): class TestCheckFiniteAndUnscaleOpWithInf(TestCheckFiniteAndUnscaleOp): - def init_test_case(self): x = np.random.random((129, 129)).astype(self.dtype) x[128][128] = np.inf @@ -92,7 +89,6 @@ def test_check_output(self): class TestCheckFiniteAndUnscaleOpMultiInput(TestCheckFiniteAndUnscaleOp): - def init_test_case(self): x0 = np.random.random((129, 129)).astype(self.dtype) x1 = np.random.random((129, 129)).astype(self.dtype) @@ -106,7 +102,6 @@ def init_test_case(self): class TestCheckFiniteAndUnscaleOpMultiInputWithNan(TestCheckFiniteAndUnscaleOp): - def init_test_case(self): x0 = np.random.random((129, 129)).astype(self.dtype) x0[128][128] = np.nan @@ -126,7 +121,6 @@ def test_check_output(self): class TestCheckFiniteAndUnscaleOpMultiInputWithInf(TestCheckFiniteAndUnscaleOp): - def init_test_case(self): x0 = np.random.random((129, 129)).astype(self.dtype) x0[128][128] = np.nan diff --git a/python/paddle/fluid/tests/unittests/mlu/test_arg_max_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_arg_max_op_mlu.py index e8a4101874322..81d854b4875e7 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_arg_max_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_arg_max_op_mlu.py @@ -27,7 +27,6 @@ class BaseTestCase(OpTest): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.MLUPlace(0) @@ -51,7 +50,6 @@ def test_check_output(self): class TestArgMaxSameValue1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dtype = 'float32' @@ -67,7 +65,6 @@ def setUp(self): class TestArgMaxSameValue2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dtype = 'float16' @@ -84,7 +81,6 @@ def setUp(self): # test argmax, dtype: float16 class TestArgMaxFloat16Case1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -93,7 +89,6 @@ def initTestCase(self): class TestArgMaxFloat16Case2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -102,7 +97,6 @@ def initTestCase(self): class TestArgMaxFloat16Case3(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -111,7 +105,6 @@ def initTestCase(self): class TestArgMaxFloat16Case4(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -120,7 +113,6 @@ def initTestCase(self): class TestArgMaxFloat16Case5(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -129,7 +121,6 @@ def initTestCase(self): class TestArgMaxFloat16Case6(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -138,7 +129,6 @@ def initTestCase(self): class TestArgMaxFloat16Case7(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -147,35 +137,31 @@ def initTestCase(self): class TestArgMaxFloat16Case8(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (1, ) + self.dims = (1,) self.dtype = 'float16' self.axis = 0 class TestArgMaxFloat16Case9(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (2, ) + self.dims = (2,) self.dtype = 'float16' self.axis = 0 class TestArgMaxFloat16Case10(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (3, ) + self.dims = (3,) self.dtype = 'float16' self.axis = 0 # test argmax, dtype: float32 class TestArgMaxFloat32Case1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -184,7 +170,6 @@ def initTestCase(self): class TestArgMaxFloat32Case2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -193,7 +178,6 @@ def initTestCase(self): class TestArgMaxFloat32Case3(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -202,7 +186,6 @@ def initTestCase(self): class TestArgMaxFloat32Case4(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -211,7 +194,6 @@ def initTestCase(self): class TestArgMaxFloat32Case5(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -220,7 +202,6 @@ def initTestCase(self): class TestArgMaxFloat32Case6(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -229,7 +210,6 @@ def initTestCase(self): class TestArgMaxFloat32Case7(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -238,34 +218,30 @@ def initTestCase(self): class TestArgMaxFloat32Case8(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (1, ) + self.dims = (1,) self.dtype = 'float32' self.axis = 0 class TestArgMaxFloat32Case9(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (2, ) + self.dims = (2,) self.dtype = 'float32' self.axis = 0 class TestArgMaxFloat32Case10(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (3, ) + self.dims = (3,) self.dtype = 'float32' self.axis = 0 class BaseTestComplex1_1(OpTest): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.MLUPlace(0) @@ -283,7 +259,7 @@ def setUp(self): self.inputs = {'X': self.x} self.attrs = { 'axis': self.axis, - 'dtype': int(core.VarDesc.VarType.INT32) + 'dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { 'Out': np.argmax(self.x, axis=self.axis).astype("int32") @@ -294,7 +270,6 @@ def test_check_output(self): class BaseTestComplex1_2(OpTest): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.MLUPlace(0) @@ -312,7 +287,7 @@ def setUp(self): self.inputs = {'X': self.x} self.attrs = { 'axis': self.axis, - 'dtype': int(core.VarDesc.VarType.INT32) + 'dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { 'Out': np.argmax(self.x, axis=self.axis).astype("int32") @@ -323,7 +298,6 @@ def test_check_output(self): class TestArgMaxAPI(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -335,7 +309,6 @@ def setUp(self): self.place = [paddle.MLUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2022) @@ -343,9 +316,9 @@ def run(place): tensor_input = paddle.to_tensor(numpy_input) numpy_output = np.argmax(numpy_input, axis=self.axis) paddle_output = paddle.argmax(tensor_input, axis=self.axis) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) paddle.enable_static() for place in self.place: @@ -353,7 +326,6 @@ def run(place): class TestArgMaxAPI_2(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -366,20 +338,20 @@ def setUp(self): self.place = [paddle.MLUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2022) numpy_input = (np.random.random(self.dims)).astype(self.dtype) tensor_input = paddle.to_tensor(numpy_input) - numpy_output = np.argmax(numpy_input, - axis=self.axis).reshape(1, 4, 5) - paddle_output = paddle.argmax(tensor_input, - axis=self.axis, - keepdim=self.keep_dims) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + numpy_output = np.argmax(numpy_input, axis=self.axis).reshape( + 1, 4, 5 + ) + paddle_output = paddle.argmax( + tensor_input, axis=self.axis, keepdim=self.keep_dims + ) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() @@ -388,7 +360,6 @@ def run(place): class TestArgMaxAPI_3(unittest.TestCase): - def initTestCase(self): self.dims = (1, 9) self.dtype = 'float32' @@ -399,7 +370,6 @@ def setUp(self): self.place = [paddle.MLUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2022) @@ -407,9 +377,9 @@ def run(place): tensor_input = paddle.to_tensor(numpy_input) numpy_output = np.argmax(numpy_input).reshape([1]) paddle_output = paddle.argmax(tensor_input) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/mlu/test_argsort_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_argsort_op_mlu.py index 6e474ca604e83..4f96e3b5ccdd8 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_argsort_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_argsort_op_mlu.py @@ -26,9 +26,7 @@ def gen_test_class(dtype, axis, descending): - class TestArgsortOp(OpTest): - def setUp(self): np.random.seed(SEED) self.set_mlu() @@ -48,9 +46,11 @@ def setUp(self): def get_output(self): if descending: self.indices = np.flip( - np.argsort(self.x, kind='heapsort', axis=axis), axis) + np.argsort(self.x, kind='heapsort', axis=axis), axis + ) self.sorted_x = np.flip( - np.sort(self.x, kind='heapsort', axis=axis), axis) + np.sort(self.x, kind='heapsort', axis=axis), axis + ) else: self.indices = np.argsort(self.x, kind='heapsort', axis=axis) self.sorted_x = np.sort(self.x, kind='heapsort', axis=axis) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_assign_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_assign_op_mlu.py index 91327f748459c..7ee93fa0097c0 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_assign_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_assign_op_mlu.py @@ -25,7 +25,6 @@ class TestAssign(OpTest): - def setUp(self): self.set_mlu() self.op_type = "assign" diff --git a/python/paddle/fluid/tests/unittests/mlu/test_assign_value_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_assign_value_op_mlu.py index 6f120560f9801..535ae9f8c86b2 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_assign_value_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_assign_value_op_mlu.py @@ -29,7 +29,6 @@ class TestAssignValueMLUOp(op_test.OpTest): - def setUp(self): self.set_mlu() self.op_type = "assign_value" @@ -39,7 +38,8 @@ def setUp(self): self.attrs["shape"] = self.value.shape self.attrs["dtype"] = framework.convert_np_dtype_to_dtype_( - self.value.dtype) + self.value.dtype + ) self.outputs = {"Out": self.value} def set_mlu(self): @@ -55,24 +55,22 @@ def test_check_output(self): class TestAssignValueMLUOp2(TestAssignValueMLUOp): - def init_data(self): self.value = numpy.random.random(size=(2, 5)).astype(numpy.int32) self.attrs["int32_values"] = [int(v) for v in self.value.flat] class TestAssignValueMLUOp3(TestAssignValueMLUOp): - def init_data(self): self.value = numpy.random.random(size=(2, 5)).astype(numpy.int64) self.attrs["int64_values"] = [int(v) for v in self.value.flat] class TestAssignValueMLUOp4(TestAssignValueMLUOp): - def init_data(self): - self.value = numpy.random.choice(a=[False, True], - size=(2, 5)).astype(numpy.bool) + self.value = numpy.random.choice(a=[False, True], size=(2, 5)).astype( + numpy.bool + ) self.attrs["bool_values"] = [int(v) for v in self.value.flat] diff --git a/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu.py index d97a1e074d609..53b78e18f8861 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu.py @@ -158,12 +158,20 @@ def _reference_grad(x, y_grad, scale, mean, var, epsilon, data_format): x = np.transpose(x, (0, 2, 3, 1)) y_grad = np.transpose(y_grad, (0, 2, 3, 1)) - x_grad = scale * (y_grad - np.mean(y_grad, axis=(0, 1, 2)) - - (x - mean) * np.mean(y_grad * - (x - mean), axis=(0, 1, 2)) / - (var + epsilon)) / np.sqrt(var + epsilon) - grad_scale = np.sum(y_grad * (x - mean) / np.sqrt(var + epsilon), - axis=(0, 1, 2)) + x_grad = ( + scale + * ( + y_grad + - np.mean(y_grad, axis=(0, 1, 2)) + - (x - mean) + * np.mean(y_grad * (x - mean), axis=(0, 1, 2)) + / (var + epsilon) + ) + / np.sqrt(var + epsilon) + ) + grad_scale = np.sum( + y_grad * (x - mean) / np.sqrt(var + epsilon), axis=(0, 1, 2) + ) grad_offset = np.sum(y_grad, axis=(0, 1, 2)) # transfer back to N, C, H, W @@ -188,7 +196,6 @@ def create_or_get_tensor(scope, var_name, var, place): def set_output_grad(scope, outputs, place, feed_dict=None): - def __set_tensor__(name, data=None): out_tensor = scope.find_var(name).get_tensor() grad_tensor = scope.var(grad_var_name(name)).get_tensor() @@ -210,17 +217,15 @@ def __set_tensor__(name, data=None): class TestBatchNormOpInference(unittest.TestCase): - def setUp(self): self.dtype = np.float32 self.fuse_with_relu = False self.init_kernel_type() def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - atol=atol, - err_msg=msg) + np.testing.assert_allclose( + np.array(tensor), np_array, atol=atol, err_msg=msg + ) def check_with_place(self, place, data_layout, dtype, shape): epsilon = 0.00001 @@ -246,34 +251,39 @@ def check_with_place(self, place, data_layout, dtype, shape): mean = np.zeros(scale_shape).astype(np.float32) variance = np.ones(scale_shape).astype(np.float32) - y_out = _reference_testing(x_val, scale_val, bias_val, mean, variance, - epsilon, data_layout).astype(dtype) + y_out = _reference_testing( + x_val, scale_val, bias_val, mean, variance, epsilon, data_layout + ).astype(dtype) if self.fuse_with_relu: y_out = np.maximum(y_out, 0) scope = core.Scope() # create input - x_tensor = create_or_get_tensor(scope, "x_val", - OpTest.np_dtype_to_fluid_dtype(x_val), - place) + x_tensor = create_or_get_tensor( + scope, "x_val", OpTest.np_dtype_to_fluid_dtype(x_val), place + ) scale_tensor = create_or_get_tensor( - scope, "scale_val", OpTest.np_dtype_to_fluid_dtype(scale_val), - place) + scope, "scale_val", OpTest.np_dtype_to_fluid_dtype(scale_val), place + ) bias_tensor = create_or_get_tensor( - scope, "bias_val", OpTest.np_dtype_to_fluid_dtype(bias_val), place) - mean_tensor = create_or_get_tensor(scope, "mean", - OpTest.np_dtype_to_fluid_dtype(mean), - place) + scope, "bias_val", OpTest.np_dtype_to_fluid_dtype(bias_val), place + ) + mean_tensor = create_or_get_tensor( + scope, "mean", OpTest.np_dtype_to_fluid_dtype(mean), place + ) variance_tensor = create_or_get_tensor( - scope, "variance", OpTest.np_dtype_to_fluid_dtype(variance), place) + scope, "variance", OpTest.np_dtype_to_fluid_dtype(variance), place + ) # create output y_tensor = create_or_get_tensor(scope, "y_out", None, place) - saved_mean_tensor = create_or_get_tensor(scope, "saved_mean", None, - place) - saved_variance_tensor = create_or_get_tensor(scope, "saved_variance", - None, place) + saved_mean_tensor = create_or_get_tensor( + scope, "saved_mean", None, place + ) + saved_variance_tensor = create_or_get_tensor( + scope, "saved_variance", None, place + ) mean_out_tensor = mean_tensor variance_out_tensor = variance_tensor @@ -296,17 +306,25 @@ def check_with_place(self, place, data_layout, dtype, shape): data_layout=data_layout, use_mkldnn=False, fuse_with_relu=self.fuse_with_relu, - epsilon=epsilon) + epsilon=epsilon, + ) batch_norm_op.run(scope, place) # check inference result - self.__assert_close(y_tensor, - y_out, - "inference output are different at " + str(place) + - ", " + data_layout + ", " + str(np.dtype(dtype)) + - str(np.array(y_tensor)) + str(y_out), - atol=1e-3) + self.__assert_close( + y_tensor, + y_out, + "inference output are different at " + + str(place) + + ", " + + data_layout + + ", " + + str(np.dtype(dtype)) + + str(np.array(y_tensor)) + + str(y_out), + atol=1e-3, + ) def test_check_output(self): places = [core.CPUPlace()] @@ -315,8 +333,9 @@ def test_check_output(self): for place in places: for data_format in ["NCHW", "NHWC"]: - self.check_with_place(place, data_format, self.dtype, - [2, 3, 4, 5]) + self.check_with_place( + place, data_format, self.dtype, [2, 3, 4, 5] + ) self.check_with_place(place, data_format, self.dtype, [2, 3]) def init_kernel_type(self): @@ -324,7 +343,6 @@ def init_kernel_type(self): class TestFP16BatchNormOpInference(TestBatchNormOpInference): - def setUp(self): self.dtype = np.float16 self.fuse_with_relu = False @@ -337,13 +355,13 @@ def test_check_output(self): for place in places: for data_format in ["NCHW", "NHWC"]: - self.check_with_place(place, data_format, self.dtype, - [2, 3, 4, 5]) + self.check_with_place( + place, data_format, self.dtype, [2, 3, 4, 5] + ) self.check_with_place(place, data_format, self.dtype, [2, 3]) class TestBatchNormOpTraining(unittest.TestCase): - def setUp(self): self.fuse_with_relu = False self.data_formats = ["NCHW", "NHWC"] @@ -357,27 +375,54 @@ def init_test_case(self): self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] def __assert_close(self, tensor, np_array, msg, atol=1e-4): np.allclose(np.array(tensor), np_array, atol=atol) - def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, - epsilon, momentum, shape, data_layout): + def ref_forward_backward( + self, + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ): # run forward - y, saved_mean, var_ref = _reference_training(x, scale, bias, epsilon, - data_layout) - mean_out = saved_mean * (1. - momentum) + momentum * mean - variance_out = var_ref * (1. - momentum) + momentum * variance - saved_variance = 1. / np.sqrt(var_ref + epsilon) + y, saved_mean, var_ref = _reference_training( + x, scale, bias, epsilon, data_layout + ) + mean_out = saved_mean * (1.0 - momentum) + momentum * mean + variance_out = var_ref * (1.0 - momentum) + momentum * variance + saved_variance = 1.0 / np.sqrt(var_ref + epsilon) # run backward - x_grad, scale_grad, bias_grad = _reference_grad(x, y_grad, scale, - saved_mean, var_ref, - epsilon, data_layout) - - return y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad + x_grad, scale_grad, bias_grad = _reference_grad( + x, y_grad, scale, saved_mean, var_ref, epsilon, data_layout + ) + + return ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) def set_mean_variance(self, scale_shape, x, data_layout): mean, variance = _cal_mean_variance(x, self.epsilon, data_layout) @@ -386,12 +431,11 @@ def set_mean_variance(self, scale_shape, x, data_layout): # computing global mean/variance for one step if self.use_global_stats: mom = self.momentum - mean = mean * (1. - mom) + mom * mean_pre - variance = variance * (1. - mom) + mom * variance_pre + mean = mean * (1.0 - mom) + mom * mean_pre + variance = variance * (1.0 - mom) + mom * variance_pre return mean, variance def test_forward_backward(self): - def test_with_place(place, data_layout, shape): # attr epsilon = self.epsilon @@ -410,9 +454,27 @@ def test_with_place(place, data_layout, shape): y_grad = np.random.random_sample(shape).astype(np.float32) momentum_var = np.array([momentum]).astype(np.float32) - y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad = self.ref_forward_backward( - x, y_grad, scale, bias, mean, variance, epsilon, momentum, - shape, data_layout) + ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) = self.ref_forward_backward( + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ) var_dict = locals() var_dict['y@GRAD'] = y_grad @@ -421,8 +483,15 @@ def test_with_place(place, data_layout, shape): var_dict['bias@GRAD'] = bias_grad var_names = [ - 'x', 'scale', 'bias', 'mean', 'variance', 'y', 'saved_mean', - 'saved_variance', 'momentum_var' + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + 'y', + 'saved_mean', + 'saved_variance', + 'momentum_var', ] ground_truth = {name: var_dict[name] for name in var_names} @@ -430,15 +499,17 @@ def test_with_place(place, data_layout, shape): with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) inputs = { "X": block.var('x'), "Scale": block.var('scale'), "Bias": block.var('bias'), "Mean": block.var('mean'), - "Variance": block.var('variance') + "Variance": block.var('variance'), } attrs = { "epsilon": epsilon, @@ -446,7 +517,7 @@ def test_with_place(place, data_layout, shape): "data_layout": data_layout, "use_mkldnn": False, "fuse_with_relu": self.fuse_with_relu, - "use_global_stats": self.use_global_stats + "use_global_stats": self.use_global_stats, } if self.use_momentum_variable: inputs['MomentumTensor'] = block.var('momentum_var') @@ -458,19 +529,22 @@ def test_with_place(place, data_layout, shape): "MeanOut": block.var('mean'), # share memory "VarianceOut": block.var('variance'), # share memory "SavedMean": block.var('saved_mean'), - "SavedVariance": block.var('saved_variance') + "SavedVariance": block.var('saved_variance'), } block.create_var(name="reserve_space", dtype='float32') outputs["ReserveSpace"] = block.var('reserve_space') - bn_op = block.append_op(type="batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + bn_op = block.append_op( + type="batch_norm", + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) block.create_var(name='y@GRAD', dtype='float32', shape=y.shape) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - bn_op.desc, self.no_grad_set, []) + bn_op.desc, self.no_grad_set, [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -485,22 +559,28 @@ def test_with_place(place, data_layout, shape): program._sync_with_cpp() exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in [ - 'x', 'scale', 'bias', 'mean', 'variance', - 'y@GRAD', 'momentum_var' - ] - }, - fetch_list=self.fetch_list) + out = exe.run( + program, + feed={ + name: var_dict[name] + for name in [ + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + 'y@GRAD', + 'momentum_var', + ] + }, + fetch_list=self.fetch_list, + ) for id, name in enumerate(self.fetch_list): if name == 'variance': - self.__assert_close(var_dict[name], - out[id], - name, - atol=1e-3) + self.__assert_close( + var_dict[name], out[id], name, atol=1e-3 + ) continue self.__assert_close(var_dict[name], out[id], name) print("op test forward passed: ", str(place), data_layout) @@ -519,7 +599,6 @@ def init_kernel_type(self): class TestBatchNormOpTrainingCase1(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = False self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -527,19 +606,23 @@ def init_test_case(self): class TestBatchNormOpTrainingCase2(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] os.environ['FLAGS_cudnn_batchnorm_spatial_persistent'] = "1" class TestBatchNormOpTrainingCase3(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = False self.no_grad_set = set(['x@GRAD']) @@ -547,24 +630,33 @@ def init_test_case(self): class TestBatchNormOpTrainingMomentumVariable(TestBatchNormOpTraining): - def init_test_case(self): self.use_momentum_variable = True self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = True self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'x@GRAD', 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] def reference_grad(self, x, y_grad, scale, mean, var, epsilon, data_format): @@ -573,8 +665,9 @@ def reference_grad(self, x, y_grad, scale, mean, var, epsilon, data_format): y_grad = np.transpose(y_grad, (0, 2, 3, 1)) x_grad = scale * y_grad / np.sqrt(var + epsilon) - grad_scale = np.sum(y_grad * (x - mean) / np.sqrt(var + epsilon), - axis=(0, 1, 2)) + grad_scale = np.sum( + y_grad * (x - mean) / np.sqrt(var + epsilon), axis=(0, 1, 2) + ) grad_offset = np.sum(y_grad, axis=(0, 1, 2)) # transfer back to N, C, H, W @@ -585,8 +678,19 @@ def reference_grad(self, x, y_grad, scale, mean, var, epsilon, data_format): return x_grad, grad_scale, grad_offset - def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, - epsilon, momentum, shape, data_layout): + def ref_forward_backward( + self, + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ): if data_layout != "NCHW" and data_layout != "NHWC": raise ValueError("Unknown data order.") @@ -604,17 +708,27 @@ def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, mean_out = mean variance_out = variance - saved_variance = 1. / np.sqrt(variance + epsilon) + saved_variance = 1.0 / np.sqrt(variance + epsilon) # run backward x_grad, scale_grad, bias_grad = self.reference_grad( - x, y_grad, scale, mean, variance, epsilon, data_layout) + x, y_grad, scale, mean, variance, epsilon, data_layout + ) - return y, mean_out, variance_out, mean, saved_variance, x_grad, scale_grad, bias_grad + return ( + y, + mean_out, + variance_out, + mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) class TestBatchNormOpFreezeStatsAndScaleBiasTraining( - TestBatchNormOpFreezeStatsTraining): - + TestBatchNormOpFreezeStatsTraining +): def init_test_case(self): self.use_global_stats = True self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -622,12 +736,12 @@ def init_test_case(self): class TestBatchNormOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of batch_norm must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.batch_norm, x1) # the input dtype of batch_norm must be float16 or float32 or float64 @@ -637,13 +751,13 @@ def test_errors(self): class TestDygraphBatchNormAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_norm = fluid.dygraph.BatchNorm(10) # the input of BatchNorm must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, batch_norm, x1) # the input dtype of BatchNorm must be float16 or float32 or float64 @@ -653,7 +767,6 @@ def test_errors(self): class TestDygraphBatchNormTrainableStats(unittest.TestCase): - def test_dygraph(self): places = [fluid.CPUPlace()] if core.is_compiled_with_mlu(): @@ -666,7 +779,8 @@ def compute(x, is_test, trainable_statistics): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) y = bn(fluid.dygraph.to_variable(x)) return y.numpy() @@ -688,7 +802,8 @@ def compute(x_np, is_test, trainable_statistics): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) x = fluid.data(name='x', shape=x_np.shape, dtype=x_np.dtype) y = bn(x) exe.run(fluid.default_startup_program()) @@ -702,7 +817,6 @@ def compute(x_np, is_test, trainable_statistics): class TestDygraphBatchNormOpenReserveSpace(unittest.TestCase): - def test_reservespace(self): with program_guard(Program(), Program()): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu_v2.py b/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu_v2.py index 27d25c6146e0f..72e7ac89caf36 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu_v2.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_batch_norm_op_mlu_v2.py @@ -31,7 +31,6 @@ class TestBatchNorm(unittest.TestCase): - def test_name(self): places = [fluid.CPUPlace()] if core.is_compiled_with_mlu(): @@ -45,7 +44,7 @@ def test_error(self): if core.is_compiled_with_mlu(): places.append(fluid.MLUPlace(0)) for p in places: - #paddle.disable_static() + # paddle.disable_static() x_data_4 = np.random.random(size=(2, 1, 3, 3)).astype('float32') x_data_3 = np.random.random(size=(2, 1, 3)).astype('float32') @@ -99,7 +98,8 @@ def compute_v1(x, is_test, trainable_statistics): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) y = bn(fluid.dygraph.to_variable(x)) return y.numpy() @@ -116,19 +116,22 @@ def compute_v3(x, is_test, trainable_statistics): is_test=is_test, param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(1.0), - trainable=False), + trainable=False, + ), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(0.0), - trainable=False), - trainable_statistics=trainable_statistics) + trainable=False, + ), + trainable_statistics=trainable_statistics, + ) y = bn(fluid.dygraph.to_variable(x)) return y.numpy() def compute_v4(x): with fluid.dygraph.guard(p): - bn = paddle.nn.BatchNorm2D(shape[1], - weight_attr=False, - bias_attr=False) + bn = paddle.nn.BatchNorm2D( + shape[1], weight_attr=False, bias_attr=False + ) y = bn(fluid.dygraph.to_variable(x)) return y.numpy() @@ -153,7 +156,8 @@ def compute_v1(x_np, is_test, trainable_statistics): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) x = fluid.data(name='x', shape=x_np.shape, dtype=x_np.dtype) y = bn(x) exe.run(fluid.default_startup_program()) @@ -176,7 +180,6 @@ def compute_v2(x_np): class TestBatchNormChannelLast(unittest.TestCase): - def setUp(self): self.original_dtyep = paddle.get_default_dtype() paddle.set_default_dtype("float32") @@ -199,10 +202,9 @@ def test_1d(self): channel_first_x = paddle.transpose(x, [0, 2, 1]) y2 = net2(channel_first_x) y2 = paddle.transpose(y2, [0, 2, 1]) - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05, atol=1e-07 + ) def test_2d(self): for p in self.places: @@ -216,10 +218,9 @@ def test_2d(self): channel_first_x = paddle.transpose(x, [0, 3, 1, 2]) y2 = net2(channel_first_x) y2 = paddle.transpose(y2, [0, 2, 3, 1]) - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05, atol=1e-07 + ) def test_3d(self): for p in self.places: @@ -233,10 +234,9 @@ def test_3d(self): channel_first_x = paddle.transpose(x, [0, 4, 1, 2, 3]) y2 = net2(channel_first_x) y2 = paddle.transpose(y2, [0, 2, 3, 4, 1]) - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05, atol=1e-07 + ) # res = np.allclose(y1.numpy(), y2.numpy()) # if res == False: # np.savetxt("./y1.txt", y1.numpy().flatten(), fmt='%.10f', delimiter='\n') @@ -245,7 +245,6 @@ def test_3d(self): class TestBatchNormUseGlobalStats(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_mlu(): @@ -264,11 +263,14 @@ def test_global_stats(self): net1 = paddle.fluid.dygraph.BatchNorm( 6, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(1.0)), + initializer=fluid.initializer.Constant(1.0) + ), use_global_stats=self.use_global_stats, - trainable_statistics=self.trainable_statistics) + trainable_statistics=self.trainable_statistics, + ) net2 = paddle.nn.BatchNorm2D( - 6, use_global_stats=self.use_global_stats) + 6, use_global_stats=self.use_global_stats + ) net2.weight = net1.weight net2.bias = net1.bias if self.trainable_statistics == True: diff --git a/python/paddle/fluid/tests/unittests/mlu/test_bce_loss_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_bce_loss_mlu.py index 3805d27a14f04..58ccf798136f4 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_bce_loss_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_bce_loss_mlu.py @@ -24,88 +24,76 @@ paddle.enable_static() -def test_static_layer(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_static_layer( + place, input_np, label_np, reduction='mean', weight_np=None +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=input_np.shape, - dtype='float32') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float32') + input = paddle.fluid.data( + name='input', shape=input_np.shape, dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float32' + ) if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float32') - bce_loss = paddle.nn.loss.BCELoss(weight=weight, - reduction=reduction) + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float32' + ) + bce_loss = paddle.nn.loss.BCELoss( + weight=weight, reduction=reduction + ) else: bce_loss = paddle.nn.loss.BCELoss(reduction=reduction) res = bce_loss(input, label) exe = paddle.static.Executor(place) - static_result = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - } if weight_np is None else { - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + static_result = exe.run( + prog, + feed={"input": input_np, "label": label_np} + if weight_np is None + else {"input": input_np, "label": label_np, "weight": weight_np}, + fetch_list=[res], + ) return static_result[0] -def test_static_functional(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_static_functional( + place, input_np, label_np, reduction='mean', weight_np=None +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=input_np.shape, - dtype='float32') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float32') + input = paddle.fluid.data( + name='input', shape=input_np.shape, dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float32' + ) if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float32') - res = paddle.nn.functional.binary_cross_entropy(input, - label, - weight=weight, - reduction=reduction) + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float32' + ) + res = paddle.nn.functional.binary_cross_entropy( + input, label, weight=weight, reduction=reduction + ) else: - res = paddle.nn.functional.binary_cross_entropy(input, - label, - reduction=reduction) + res = paddle.nn.functional.binary_cross_entropy( + input, label, reduction=reduction + ) exe = paddle.static.Executor(place) - static_result = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - } if weight_np is None else { - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + static_result = exe.run( + prog, + feed={"input": input_np, "label": label_np} + if weight_np is None + else {"input": input_np, "label": label_np, "weight": weight_np}, + fetch_list=[res], + ) return static_result[0] -def test_dygraph_layer(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_dygraph_layer( + place, input_np, label_np, reduction='mean', weight_np=None +): paddle.disable_static() if weight_np is not None: weight = paddle.to_tensor(weight_np) @@ -118,25 +106,22 @@ def test_dygraph_layer(place, return dy_result -def test_dygraph_functional(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_dygraph_functional( + place, input_np, label_np, reduction='mean', weight_np=None +): paddle.disable_static() input = paddle.to_tensor(input_np) label = paddle.to_tensor(label_np) if weight_np is not None: weight = paddle.to_tensor(weight_np) - dy_res = paddle.nn.functional.binary_cross_entropy(input, - label, - weight=weight, - reduction=reduction) + dy_res = paddle.nn.functional.binary_cross_entropy( + input, label, weight=weight, reduction=reduction + ) else: - dy_res = paddle.nn.functional.binary_cross_entropy(input, - label, - reduction=reduction) + dy_res = paddle.nn.functional.binary_cross_entropy( + input, label, reduction=reduction + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result @@ -144,11 +129,19 @@ def test_dygraph_functional(place, def calc_bceloss(input_np, label_np, reduction='mean', weight_np=None): if weight_np is None: - expected = -1 * (label_np * np.log(input_np) + - (1. - label_np) * np.log(1. - input_np)) + expected = -1 * ( + label_np * np.log(input_np) + + (1.0 - label_np) * np.log(1.0 - input_np) + ) else: - expected = -1 * weight_np * (label_np * np.log(input_np) + - (1. - label_np) * np.log(1. - input_np)) + expected = ( + -1 + * weight_np + * ( + label_np * np.log(input_np) + + (1.0 - label_np) * np.log(1.0 - input_np) + ) + ) if reduction == 'mean': expected = np.mean(expected) @@ -161,7 +154,6 @@ def calc_bceloss(input_np, label_np, reduction='mean', weight_np=None): class TestBCELoss(unittest.TestCase): - def test_BCELoss(self): input_np = np.random.uniform(0.1, 0.8, size=(20, 30)).astype(np.float32) label_np = np.random.randint(0, 2, size=(20, 30)).astype(np.float32) @@ -169,84 +161,82 @@ def test_BCELoss(self): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - static_result = test_static_layer(place, input_np, label_np, - reduction) - dy_result = test_dygraph_layer(place, input_np, label_np, - reduction) + static_result = test_static_layer( + place, input_np, label_np, reduction + ) + dy_result = test_dygraph_layer( + place, input_np, label_np, reduction + ) expected = calc_bceloss(input_np, label_np, reduction) np.testing.assert_allclose(static_result, expected, rtol=1e-6) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected, rtol=1e-6) static_functional = test_static_functional( - place, input_np, label_np, reduction) - dy_functional = test_dygraph_functional(place, input_np, - label_np, reduction) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-6) + place, input_np, label_np, reduction + ) + dy_functional = test_dygraph_functional( + place, input_np, label_np, reduction + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-6 + ) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected, rtol=1e-6) def test_BCELoss_weight(self): - input_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float32) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float32) + input_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float32 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float32 + ) weight_np = np.random.random(size=(3, 4, 10)).astype(np.float32) place = fluid.MLUPlace(0) for reduction in ['sum', 'mean', 'none']: - static_result = test_static_layer(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - dy_result = test_dygraph_layer(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - expected = calc_bceloss(input_np, - label_np, - reduction, - weight_np=weight_np) + static_result = test_static_layer( + place, input_np, label_np, reduction, weight_np=weight_np + ) + dy_result = test_dygraph_layer( + place, input_np, label_np, reduction, weight_np=weight_np + ) + expected = calc_bceloss( + input_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_result, expected, rtol=1e-6) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected, rtol=1e-6) - static_functional = test_static_functional(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - dy_functional = test_dygraph_functional(place, - input_np, - label_np, - reduction, - weight_np=weight_np) + static_functional = test_static_functional( + place, input_np, label_np, reduction, weight_np=weight_np + ) + dy_functional = test_dygraph_functional( + place, input_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_functional, expected, rtol=1e-6) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected, rtol=1e-6) def test_BCELoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.loss.BCELoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, paddle.nn.loss.BCELoss, reduction="unsupport reduction" + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([[0.0, 1.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.binary_cross_entropy, - input=input, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.binary_cross_entropy, + input=input, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() def bce_loss(input, label): - return -1 * (label * np.log(input) + (1. - label) * np.log(1. - input)) + return -1 * (label * np.log(input) + (1.0 - label) * np.log(1.0 - input)) class TestBceLossOp(OpTest): - def setUp(self): self.init_test_case() self.op_type = "bce_loss" @@ -270,13 +260,11 @@ def init_test_case(self): class TestBceLossOpCase1(TestBceLossOp): - def init_test_case(self): self.shape = [2, 3, 4, 5] class TestBceLossOpCase2(TestBceLossOp): - def init_test_case(self): self.shape = [2, 3, 20] diff --git a/python/paddle/fluid/tests/unittests/mlu/test_bce_with_logits_loss_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_bce_with_logits_loss_mlu.py index 6b0b91cce9383..0aafe99276fb5 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_bce_with_logits_loss_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_bce_with_logits_loss_mlu.py @@ -20,43 +20,51 @@ sys.path.append('..') from op_test import OpTest -from test_bce_with_logits_loss import call_bce_layer, call_bce_functional, test_dygraph, calc_bce_with_logits_loss - - -def test_static(place, - logit_np, - label_np, - weight_np=None, - reduction='mean', - pos_weight_np=None, - functional=False): +from test_bce_with_logits_loss import ( + call_bce_layer, + call_bce_functional, + test_dygraph, + calc_bce_with_logits_loss, +) + + +def test_static( + place, + logit_np, + label_np, + weight_np=None, + reduction='mean', + pos_weight_np=None, + functional=False, +): paddle.enable_static() prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - logit = paddle.fluid.data(name='logit', - shape=logit_np.shape, - dtype='float32') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float32') + logit = paddle.fluid.data( + name='logit', shape=logit_np.shape, dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float32' + ) feed_dict = {"logit": logit_np, "label": label_np} pos_weight = None weight = None if pos_weight_np is not None: - pos_weight = paddle.fluid.data(name='pos_weight', - shape=pos_weight_np.shape, - dtype='float32') + pos_weight = paddle.fluid.data( + name='pos_weight', shape=pos_weight_np.shape, dtype='float32' + ) feed_dict["pos_weight"] = pos_weight_np if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float32') + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float32' + ) feed_dict["weight"] = weight_np if functional: - res = call_bce_functional(logit, label, weight, reduction, - pos_weight) + res = call_bce_functional( + logit, label, weight, reduction, pos_weight + ) else: res = call_bce_layer(logit, label, weight, reduction, pos_weight) exe = paddle.static.Executor(place) @@ -68,7 +76,6 @@ def test_static(place, class TestBCEWithLogitsLoss(unittest.TestCase): - def test_BCEWithLogitsLoss(self): logit_np = np.random.uniform(0.1, 0.8, size=(20, 30)).astype(np.float32) label_np = np.random.randint(0, 2, size=(20, 30)).astype(np.float32) @@ -76,125 +83,150 @@ def test_BCEWithLogitsLoss(self): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - static_result = test_static(place, - logit_np, - label_np, - reduction=reduction) - dy_result = test_dygraph(place, - logit_np, - label_np, - reduction=reduction) - expected = calc_bce_with_logits_loss(logit_np, label_np, - reduction) + static_result = test_static( + place, logit_np, label_np, reduction=reduction + ) + dy_result = test_dygraph( + place, logit_np, label_np, reduction=reduction + ) + expected = calc_bce_with_logits_loss( + logit_np, label_np, reduction + ) np.testing.assert_allclose(static_result, expected, rtol=1e-6) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected, rtol=1e-6) - static_functional = test_static(place, - logit_np, - label_np, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place, - logit_np, - label_np, - reduction=reduction, - functional=True) - - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-6) + static_functional = test_static( + place, + logit_np, + label_np, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place, + logit_np, + label_np, + reduction=reduction, + functional=True, + ) + + np.testing.assert_allclose( + static_functional, expected, rtol=1e-6 + ) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected, rtol=1e-6) def test_BCEWithLogitsLoss_weight(self): - logit_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float32) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float32) + logit_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float32 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float32 + ) weight_np = np.random.random(size=(2, 3, 4, 10)).astype(np.float32) place = fluid.MLUPlace(0) for reduction in ['sum', 'mean', 'none']: - static_result = test_static(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction) - dy_result = test_dygraph(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction) - expected = calc_bce_with_logits_loss(logit_np, - label_np, - reduction, - weight_np=weight_np) + static_result = test_static( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + ) + dy_result = test_dygraph( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + ) + expected = calc_bce_with_logits_loss( + logit_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_result, expected, rtol=1e-6) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected, rtol=1e-6) - static_functional = test_static(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction, - functional=True) + static_functional = test_static( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + functional=True, + ) np.testing.assert_allclose(static_functional, expected, rtol=1e-6) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected, rtol=1e-6) def test_BCEWithLogitsLoss_pos_weight(self): - logit_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float32) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float32) + logit_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float32 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float32 + ) pos_weight_np = np.random.random(size=(3, 4, 10)).astype(np.float32) weight_np = np.random.random(size=(2, 3, 4, 10)).astype(np.float32) place = fluid.MLUPlace(0) reduction = "mean" - static_result = test_static(place, logit_np, label_np, weight_np, - reduction, pos_weight_np) - dy_result = test_dygraph(place, logit_np, label_np, weight_np, - reduction, pos_weight_np) - expected = calc_bce_with_logits_loss(logit_np, label_np, reduction, - weight_np, pos_weight_np) + static_result = test_static( + place, logit_np, label_np, weight_np, reduction, pos_weight_np + ) + dy_result = test_dygraph( + place, logit_np, label_np, weight_np, reduction, pos_weight_np + ) + expected = calc_bce_with_logits_loss( + logit_np, label_np, reduction, weight_np, pos_weight_np + ) np.testing.assert_allclose(static_result, expected) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected) - static_functional = test_static(place, - logit_np, - label_np, - weight_np, - reduction, - pos_weight_np, - functional=True) - dy_functional = test_dygraph(place, - logit_np, - label_np, - weight_np, - reduction, - pos_weight_np, - functional=True) + static_functional = test_static( + place, + logit_np, + label_np, + weight_np, + reduction, + pos_weight_np, + functional=True, + ) + dy_functional = test_dygraph( + place, + logit_np, + label_np, + weight_np, + reduction, + pos_weight_np, + functional=True, + ) np.testing.assert_allclose(static_functional, expected) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected) def test_BCEWithLogitsLoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.BCEWithLogitsLoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.BCEWithLogitsLoss, + reduction="unsupport reduction", + ) logit = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([[0.0, 1.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.binary_cross_entropy_with_logits, - logit=logit, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.binary_cross_entropy_with_logits, + logit=logit, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/mlu/test_bilinear_interp_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_bilinear_interp_v2_op_mlu.py index b4ae7cf6da0fa..912578cda8c5c 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_bilinear_interp_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_bilinear_interp_v2_op_mlu.py @@ -26,16 +26,18 @@ paddle.enable_static() -def bilinear_interp_np(input, - out_h, - out_w, - scale_w=0, - scale_h=0, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCHW'): +def bilinear_interp_np( + input, + out_h, + out_w, + scale_w=0, + scale_h=0, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCHW', +): """bilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -49,7 +51,7 @@ def bilinear_interp_np(input, ratio_h = ratio_w = 0.0 if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: @@ -57,7 +59,7 @@ def bilinear_interp_np(input, else: ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -68,37 +70,40 @@ def bilinear_interp_np(input, out = np.zeros((batch_size, channel, out_h, out_w)) for i in range(out_h): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: h = int(ratio_h * (i + 0.5) - 0.5) else: h = int(ratio_h * i) h = max(0, h) hid = 1 if h < in_h - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_h = max(ratio_h * (i + 0.5) - 0.5, 0) h1lambda = idx_src_h - h else: h1lambda = ratio_h * i - h h2lambda = 1.0 - h1lambda for j in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (j + 0.5) - 0.5) else: w = int(ratio_w * j) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (j + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * j - w w2lambda = 1.0 - w1lambda - out[:, :, i, j] = h2lambda*(w2lambda*input[:, :, h, w] + - w1lambda*input[:, :, h, w+wid]) + \ - h1lambda*(w2lambda*input[:, :, h+hid, w] + - w1lambda*input[:, :, h+hid, w+wid]) + out[:, :, i, j] = h2lambda * ( + w2lambda * input[:, :, h, w] + + w1lambda * input[:, :, h, w + wid] + ) + h1lambda * ( + w2lambda * input[:, :, h + hid, w] + + w1lambda * input[:, :, h + hid, w + wid] + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC @@ -107,7 +112,6 @@ def bilinear_interp_np(input, class TestBilinearInterpOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -129,7 +133,7 @@ def setUp(self): scale_w = 0 if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: scale_h = scale_w = float(self.scale) if isinstance(self.scale, list) and len(self.scale) == 1: scale_w = scale_h = self.scale[0] @@ -142,10 +146,18 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode, - self.data_layout) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -158,12 +170,12 @@ def setUp(self): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: self.scale = [self.scale] if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] @@ -181,44 +193,41 @@ def init_test_case(self): self.input_shape = [2, 3, 5, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 @@ -227,46 +236,42 @@ def test_check_output(self): class TestBilinearInterpCase4(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase5(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase6(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 33]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase7(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] @@ -278,38 +283,35 @@ def init_test_case(self): class TestBilinearInterpSame(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpActualShape(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpDataLayout(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 5, 5, 3] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True self.align_mode = 1 @@ -317,52 +319,46 @@ def init_test_case(self): class TestBilinearInterpOtherMethod1(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 1 class TestBilinearInterpWithMethod2(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 0 class TestBilinearInterpWithMethod3(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = True self.align_mode = 0 class TestBilinearInterpScale1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 2. + self.scale = 2.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 1. + self.scale = 1.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -374,7 +370,6 @@ def init_test_case(self): class TestBilinearInterpScale4(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -386,7 +381,6 @@ def init_test_case(self): class TestBilinearInterpZero(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -398,7 +392,6 @@ def init_test_case(self): class TestBilinearInterpOp_attr_tensor(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -438,8 +431,9 @@ def setUp(self): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h @@ -451,9 +445,16 @@ def setUp(self): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): @@ -467,33 +468,31 @@ def init_test_case(self): self.input_shape = [2, 3, 5, 5] self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [3, 3] self.align_corners = True # out_size is a 1-D tensor class TestBilinearInterp_attr_tensor_Case1(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = [8, 12] self.align_corners = True # scale is a 1-D tensor class TestBilinearInterp_attr_tensor_Case2(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.shape_by_1Dtensor = True @@ -501,7 +500,6 @@ def init_test_case(self): # scale is a 1-D tensor class TestBilinearInterp_attr_tensor_Case3(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] @@ -514,23 +512,22 @@ def init_test_case(self): class TestBilinearInterpOpAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[2], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) out1 = fluid.layers.resize_bilinear(x, out_shape=[12, 12]) out2 = fluid.layers.resize_bilinear(x, out_shape=[12, dim]) out3 = fluid.layers.resize_bilinear(x, out_shape=shape_tensor) - out4 = fluid.layers.resize_bilinear(x, - out_shape=[4, 4], - actual_shape=actual_size) + out4 = fluid.layers.resize_bilinear( + x, out_shape=[4, 4], actual_shape=actual_size + ) out5 = fluid.layers.resize_bilinear(x, scale=scale_tensor) x_data = np.random.random((2, 3, 6, 6)).astype("float32") @@ -545,29 +542,30 @@ def test_case(self): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = bilinear_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=True) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = bilinear_interp_np( + x_data, out_h=12, out_w=12, align_corners=True + ) for res in results: np.testing.assert_allclose(res, expect_res, rtol=1e-6) class TestBilinearInterpOpAPI_dy(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_mlu(): place = paddle.device.MLUPlace(0) else: @@ -575,21 +573,19 @@ def test_case(self): with fluid.dygraph.guard(place): input_data = np.random.random((2, 3, 6, 6)).astype("float32") input_x = paddle.to_tensor(input_data) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - size=[12, 12], - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, size=[12, 12], mode="bilinear", align_corners=False + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-6) class TestBilinearInterpOpAPI_dy2(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_mlu(): place = paddle.device.MLUPlace(0) else: @@ -599,21 +595,19 @@ def test_case(self): size_np = np.array([12, 12]).astype("int64") input_x = paddle.to_tensor(input_data) size = paddle.to_tensor(size_np) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - size=size, - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, size=size, mode="bilinear", align_corners=False + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-6) class TestBilinearInterpOpAPI_dy3(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_mlu(): place = paddle.device.MLUPlace(0) else: @@ -623,21 +617,22 @@ def test_case(self): size_1 = np.array([12]).astype("int64") input_x = paddle.to_tensor(input_data) size = paddle.to_tensor(size_1) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - size=[size, size], - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, + size=[size, size], + mode="bilinear", + align_corners=False, + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-6) class TestBilinearInterpOpAPI_dy4(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_mlu(): place = paddle.device.MLUPlace(0) else: @@ -647,14 +642,15 @@ def test_case(self): scale_np = np.array([2, 2]).astype("int64") input_x = paddle.to_tensor(input_data) scale = paddle.to_tensor(scale_np) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - scale_factor=scale, - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, + scale_factor=scale, + mode="bilinear", + align_corners=False, + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-6) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_cast_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_cast_op_mlu.py index 4b75767efe6ce..a5c5f7b8a2f2b 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_cast_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_cast_op_mlu.py @@ -28,14 +28,13 @@ class TestCastOpFp32ToFp16(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float32')} self.outputs = {'Out': ipt.astype('float16')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP32), - 'out_dtype': int(core.VarDesc.VarType.FP16) + 'out_dtype': int(core.VarDesc.VarType.FP16), } self.op_type = 'cast' self.place = paddle.device.MLUPlace(0) @@ -47,14 +46,13 @@ def test_check_output(self): class TestCastOpFp16ToFp32(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float16')} self.outputs = {'Out': ipt.astype('float32')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP16), - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.op_type = 'cast' self.place = paddle.device.MLUPlace(0) @@ -66,14 +64,13 @@ def test_check_output(self): class TestCastOpFp32ToFp64(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float32')} self.outputs = {'Out': ipt.astype('float64')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP32), - 'out_dtype': int(core.VarDesc.VarType.FP64) + 'out_dtype': int(core.VarDesc.VarType.FP64), } self.op_type = 'cast' self.place = paddle.device.MLUPlace(0) @@ -85,14 +82,13 @@ def test_check_output(self): class TestCastOpInt32ToInt32(OpTest): - def setUp(self): ipt = np.random.randint(1000, size=(10, 10)) self.inputs = {'X': ipt.astype('int32')} self.outputs = {'Out': ipt.astype('int32')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.INT32), - 'out_dtype': int(core.VarDesc.VarType.INT32) + 'out_dtype': int(core.VarDesc.VarType.INT32), } self.op_type = 'cast' self.place = paddle.device.MLUPlace(0) @@ -103,14 +99,13 @@ def test_check_output(self): class TestCastOpInt32ToFp32(OpTest): - def setUp(self): ipt = np.random.randint(1000, size=[10, 10]) self.inputs = {'X': ipt.astype('int32')} self.outputs = {'Out': ipt.astype('float32')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.INT32), - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.op_type = 'cast' self.place = paddle.device.MLUPlace(0) @@ -121,14 +116,13 @@ def test_check_output(self): class TestCastOpInt16ToFp64(OpTest): - def setUp(self): ipt = np.random.randint(1000, size=[10, 10]) self.inputs = {'X': ipt.astype('int16')} self.outputs = {'Out': ipt.astype('int64')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.INT16), - 'out_dtype': int(core.VarDesc.VarType.INT64) + 'out_dtype': int(core.VarDesc.VarType.INT64), } self.op_type = 'cast' self.place = paddle.device.MLUPlace(0) @@ -139,12 +133,12 @@ def test_check_output(self): class TestCastOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of cast_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.MLUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.MLUPlace(0) + ) self.assertRaises(TypeError, fluid.layers.cast, x1, 'int32') diff --git a/python/paddle/fluid/tests/unittests/mlu/test_clip_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_clip_op_mlu.py index 12c5dbc2cd217..c306ab96b61db 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_clip_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_clip_op_mlu.py @@ -27,7 +27,6 @@ class TestClipOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -73,7 +72,6 @@ def initTestCase(self): class TestCase1(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (8, 16, 8) @@ -82,7 +80,6 @@ def initTestCase(self): class TestCase2(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (8, 16) @@ -91,7 +88,6 @@ def initTestCase(self): class TestCase3(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (4, 8, 16) @@ -100,7 +96,6 @@ def initTestCase(self): class TestCase4(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (4, 8, 8) @@ -111,7 +106,6 @@ def initTestCase(self): class TestCase5(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (4, 8, 16) @@ -120,7 +114,6 @@ def initTestCase(self): class TestCase6(TestClipOp): - def initTestCase(self): self.dtype = np.float16 self.shape = (4, 8, 8) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_coalesce_tensor_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_coalesce_tensor_op_mlu.py index 8eacd633db8bd..97bc47971f475 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_coalesce_tensor_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_coalesce_tensor_op_mlu.py @@ -26,7 +26,6 @@ class TestAllocContinuousSpace(OpTest): - def setUp(self): self.op_type = "coalesce_tensor" self.dtype, self.fluid_dtype = self.init_dtype() @@ -36,7 +35,8 @@ def setUp(self): self.set_constant = attrs["set_constant"] self.Inputs = self.init_input() self.Outputs, self.FusedOutput = self.init_output( - self.Inputs, self.set_constant, self.constant) + self.Inputs, self.set_constant, self.constant + ) self.inputs = {'Input': self.Inputs} self.attrs = attrs self.outputs = {'Output': self.Outputs, 'FusedOutput': self.FusedOutput} @@ -59,7 +59,7 @@ def init_attr(self): "copy_data": True, "set_constant": False, "constant": 0.0, - "dtype": self.fluid_dtype + "dtype": self.fluid_dtype, } def init_output(self, input_list, set_constant, constant): @@ -76,32 +76,36 @@ def init_output(self, input_list, set_constant, constant): coalesce_tensor_var = np.concatenate([input for input in inputs]) if set_constant: coalesce_tensor_var = np.ones((len(coalesce_tensor_var))) * constant - outputs = [(out[0], - np.ones(out[1].shape).astype(self.dtype) * constant) - for out in outputs] + outputs = [ + (out[0], np.ones(out[1].shape).astype(self.dtype) * constant) + for out in outputs + ] return outputs, coalesce_tensor_var def test_check_output(self): - self.check_output_with_place(place=paddle.device.MLUPlace(0), - no_check_set=["FusedOutput"], - atol=1e-5) + self.check_output_with_place( + place=paddle.device.MLUPlace(0), + no_check_set=["FusedOutput"], + atol=1e-5, + ) class TestAllocContinuousSpace2(TestAllocContinuousSpace): - def init_attr(self): return { "copy_data": False, "set_constant": True, "constant": 5, "dtype": self.fluid_dtype, - "user_defined_size_of_dtype": 2 + "user_defined_size_of_dtype": 2, } def test_check_output(self): - self.check_output_with_place(place=paddle.device.MLUPlace(0), - no_check_set=["FusedOutput"], - atol=1e-5) + self.check_output_with_place( + place=paddle.device.MLUPlace(0), + no_check_set=["FusedOutput"], + atol=1e-5, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather.py index 236eb8f1003b8..d4c7779807aa5 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather.py @@ -23,32 +23,36 @@ class TestCAllgatherOp(TestDistBase): - def _setup_config(self): pass def test_allgather_fp32(self): - self.check_with_place("collective_allgather_op.py", "allgather", - "float32") + self.check_with_place( + "collective_allgather_op.py", "allgather", "float32" + ) def test_allgather_fp16(self): - self.check_with_place("collective_allgather_op.py", "allgather", - "float16") + self.check_with_place( + "collective_allgather_op.py", "allgather", "float16" + ) def test_allgather_int32(self): - self.check_with_place("collective_allgather_op.py", "allgather", - "int32") + self.check_with_place( + "collective_allgather_op.py", "allgather", "int32" + ) def test_allgather_int16(self): - self.check_with_place("collective_allgather_op.py", "allgather", - "int16") + self.check_with_place( + "collective_allgather_op.py", "allgather", "int16" + ) def test_allgather_int8(self): self.check_with_place("collective_allgather_op.py", "allgather", "int8") def test_allgather_uint8(self): - self.check_with_place("collective_allgather_op.py", "allgather", - "uint8") + self.check_with_place( + "collective_allgather_op.py", "allgather", "uint8" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather_api_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather_api_mlu.py index 42b8f92850d3d..dfd29e09a2395 100755 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather_api_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_allgather_api_mlu.py @@ -22,21 +22,23 @@ class TestCollectiveAllgatherAPI(TestDistBase): - def _setup_config(self): pass def test_allgather_cncl_fp16(self): - self.check_with_place("collective_allgather_api.py", "allgather", - "float16") + self.check_with_place( + "collective_allgather_api.py", "allgather", "float16" + ) def test_allgather_cncl_fp32(self): - self.check_with_place("collective_allgather_api.py", "allgather", - "float32") + self.check_with_place( + "collective_allgather_api.py", "allgather", "float32" + ) def test_allgather_cncl_int32(self): - self.check_with_place("collective_allgather_api.py", "allgather", - "int32") + self.check_with_place( + "collective_allgather_api.py", "allgather", "int32" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_api_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_api_mlu.py index 74ab7c76045f7..f7d7d140579a0 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_api_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_api_mlu.py @@ -22,21 +22,23 @@ class TestCollectiveAllreduceAPI(TestDistBase): - def _setup_config(self): pass def test_allreduce_cncl_fp16(self): - self.check_with_place("collective_allreduce_api.py", "allreduce", - "float16") + self.check_with_place( + "collective_allreduce_api.py", "allreduce", "float16" + ) def test_allreduce_cncl_fp32(self): - self.check_with_place("collective_allreduce_api.py", "allreduce", - "float32") + self.check_with_place( + "collective_allreduce_api.py", "allreduce", "float32" + ) def test_allreduce_cncl_int32(self): - self.check_with_place("collective_allreduce_api.py", "allreduce", - "int32") + self.check_with_place( + "collective_allreduce_api.py", "allreduce", "int32" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_max.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_max.py index ed05fed8db091..fe6cd747106ec 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_max.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_max.py @@ -23,33 +23,38 @@ class TestCAllreduceOp(TestDistBase): - def _setup_config(self): pass def test_allreduce_max_fp32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_max", - "float32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_max", "float32" + ) def test_allreduce_max_fp16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_max", - "float16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_max", "float16" + ) def test_allreduce_max_int32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_max", - "int32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_max", "int32" + ) def test_allreduce_max_int16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_max", - "int16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_max", "int16" + ) def test_allreduce_max_int8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_max", - "int8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_max", "int8" + ) def test_allreduce_max_uint8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_max", - "uint8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_max", "uint8" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_min.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_min.py index 2b79a7caafea5..c3b52a6112645 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_min.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_min.py @@ -23,33 +23,38 @@ class TestCAllreduceOp(TestDistBase): - def _setup_config(self): pass def test_allreduce_min_fp32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_min", - "float32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_min", "float32" + ) def test_allreduce_min_fp16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_min", - "float16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_min", "float16" + ) def test_allreduce_min_int32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_min", - "int32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_min", "int32" + ) def test_allreduce_min_int16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_min", - "int16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_min", "int16" + ) def test_allreduce_min_int8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_min", - "int8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_min", "int8" + ) def test_allreduce_min_uint8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_min", - "uint8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_min", "uint8" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_prod.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_prod.py index 96bee38af5004..a10275553c892 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_prod.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_prod.py @@ -23,33 +23,38 @@ class TestCAllreduceOp(TestDistBase): - def _setup_config(self): pass def test_allreduce_prod_fp32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_prod", - "float32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_prod", "float32" + ) def test_allreduce_prod_fp16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_prod", - "float16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_prod", "float16" + ) def test_allreduce_prod_int32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_prod", - "int32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_prod", "int32" + ) def test_allreduce_prod_int16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_prod", - "int16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_prod", "int16" + ) def test_allreduce_prod_int8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_prod", - "int8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_prod", "int8" + ) def test_allreduce_prod_uint8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_prod", - "uint8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_prod", "uint8" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_sum.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_sum.py index 302c51beaaffd..353e41b094ab4 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_sum.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_allreduce_sum.py @@ -23,33 +23,38 @@ class TestCAllreduceOp(TestDistBase): - def _setup_config(self): pass def test_allreduce_sum_fp32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_sum", - "float32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_sum", "float32" + ) def test_allreduce_sum_fp16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_sum", - "float16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_sum", "float16" + ) def test_allreduce_sum_int32(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_sum", - "int32") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_sum", "int32" + ) def test_allreduce_sum_int16(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_sum", - "int16") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_sum", "int16" + ) def test_allreduce_sum_int8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_sum", - "int8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_sum", "int8" + ) def test_allreduce_sum_uint8(self): - self.check_with_place("collective_allreduce_op.py", "allreduce_sum", - "uint8") + self.check_with_place( + "collective_allreduce_op.py", "allreduce_sum", "uint8" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_api_base_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_api_base_mlu.py index b405bdda8d78e..26e7c2972b331 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_api_base_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_api_base_mlu.py @@ -40,10 +40,10 @@ def DataTypeCast(date_type): class TestCollectiveAPIRunnerBase(object): - def get_model(self, train_prog, startup_prog, rank, indata=None): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def run_trainer(self, args): train_prog = fluid.Program() @@ -65,12 +65,12 @@ def run_trainer(self, args): fetch_list = [] for elem in result: fetch_list.append(elem.name) - out = exe.run(train_prog, - feed={'tindata': indata}, - fetch_list=fetch_list) + out = exe.run( + train_prog, feed={'tindata': indata}, fetch_list=fetch_list + ) else: out = self.get_model(train_prog, startup_prog, rank, indata) - #print(out, sys.stderr) + # print(out, sys.stderr) sys.stdout.buffer.write(pickle.dumps(out)) @@ -95,19 +95,20 @@ def runtime_main(test_class, col_type): class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -120,13 +121,13 @@ def __free_port(): def _run_cluster(self, model_file, envs): worker_endpoints = self._ps_endpoints.split(",") w0_ep, w1_ep = worker_endpoints - #print("w0_ep:",w0_ep," w1_ep:",w1_ep) + # print("w0_ep:",w0_ep," w1_ep:",w1_ep) env0 = { "FLAGS_selected_mlus": "0", "PADDLE_TRAINER_ID": "0", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w0_ep + "PADDLE_CURRENT_ENDPOINT": w0_ep, } env1 = { @@ -134,9 +135,9 @@ def _run_cluster(self, model_file, envs): "PADDLE_TRAINER_ID": "1", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w1_ep + "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) if os.getenv('WITH_COVERAGE', 'OFF') == 'ON': @@ -147,16 +148,20 @@ def _run_cluster(self, model_file, envs): tr1_cmd = tr_cmd % (self._python_interp, model_file) tr0_pipe = open("/tmp/tr0_err_%d.log" % os.getpid(), "w") tr1_pipe = open("/tmp/tr1_err_%d.log" % os.getpid(), "w") - #print(tr0_cmd) - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) - - tr1_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + # print(tr0_cmd) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) + + tr1_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -169,17 +174,23 @@ def _run_cluster(self, model_file, envs): sys.stderr.write('trainer 0 stderr file: %s\n' % f.read()) with open("/tmp/tr1_err_%d.log" % os.getpid(), "r") as f: sys.stderr.write('trainer 1 stderr file: %s\n' % f.read()) - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid - - def check_with_place(self, - model_file, - col_type, - data_type, - path_id="0", - static_mode="1", - check_error_log=False, - need_envs={}): + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) + + def check_with_place( + self, + model_file, + col_type, + data_type, + path_id="0", + static_mode="1", + check_error_log=False, + need_envs={}, + ): required_envs = { "FLAGS_fraction_of_gpu_memory_to_use": "0.15", "FLAGS_eager_delete_tensor_gb": "0.0", @@ -193,7 +204,7 @@ def check_with_place(self, "PADDLE_WITH_GLOO": '0', "BACKEND": "cncl", "PATH_ID": path_id, - "DATA_TYPE": data_type + "DATA_TYPE": data_type, } required_envs.update(need_envs) if check_error_log: @@ -201,7 +212,8 @@ def check_with_place(self, required_envs["GLOG_logtostderr"] = "1" required_envs["GLOO_LOG_LEVEL"] = "TRACE" tr0_out, tr1_out, pid0, pid1 = self._run_cluster( - model_file, required_envs) + model_file, required_envs + ) np_data_type = DataTypeCast(data_type) np.random.seed(pid0) input1 = np.random.random((10, 1000)).astype(np_data_type) @@ -213,14 +225,12 @@ def check_with_place(self, np.testing.assert_allclose(tr1_out[0], need_result) elif col_type == "allreduce": need_result = input1 + input2 - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "reduce": need_result = input1 + input2 np.testing.assert_allclose(tr0_out[0], need_result) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_base_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_base_mlu.py index c2e6f63f4d5d9..ab089aa26b774 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_base_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_base_mlu.py @@ -52,10 +52,10 @@ def DataTypeCast(date_type): class TestCollectiveRunnerBase(object): - def get_model(self, train_prog, startup_prog, col_type): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def wait_server_ready(self, endpoints): while True: @@ -63,13 +63,15 @@ def wait_server_ready(self, endpoints): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, - 1) + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT, 1 + ) result = sock.connect_ex((ip_port[0], int(ip_port[1]))) if result != 0: @@ -77,44 +79,51 @@ def wait_server_ready(self, endpoints): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + - str(not_ready_endpoints) + "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: break + # endpoints should be ["ip1:port1","ip2:port2"] -#endpoints should be ["ip1:port1","ip2:port2"] - - def initCommunicator(self, program, rank, nranks, wait_port, - current_endpoint, endpoints): + def initCommunicator( + self, program, rank, nranks, wait_port, current_endpoint, endpoints + ): other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) if rank == 0 and wait_port: self.wait_server_ready(other_endpoints) block = program.global_block() - cncl_id_var = block.create_var(name=nameGen.generate('cncl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) - - block.append_op(type='c_gen_cncl_id', - inputs={}, - outputs={'Out': cncl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) - - block.append_op(type='c_comm_init', - inputs={'X': cncl_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': self.global_ring_id - }) + cncl_id_var = block.create_var( + name=nameGen.generate('cncl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) + + block.append_op( + type='c_gen_cncl_id', + inputs={}, + outputs={'Out': cncl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) + + block.append_op( + type='c_comm_init', + inputs={'X': cncl_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': self.global_ring_id, + }, + ) def run_trainer(self, args): train_prog = fluid.Program() @@ -123,8 +132,9 @@ def run_trainer(self, args): rank = args["trainerid"] current_endpoint = args["currentendpoint"] nranks = 2 - self.initCommunicator(startup_prog, rank, nranks, True, - current_endpoint, endpoints) + self.initCommunicator( + startup_prog, rank, nranks, True, current_endpoint, endpoints + ) self.rank = rank result = self.get_model(train_prog, startup_prog, args["col_type"]) device_id = int(os.getenv("FLAGS_selected_mlus", "0")) @@ -134,9 +144,9 @@ def run_trainer(self, args): np.random.seed(os.getpid()) np_data_type = DataTypeCast(args["data_type"]) indata = np.random.random((10, 1000)).astype(np_data_type) - out = exe.run(train_prog, - feed={'tindata': indata}, - fetch_list=[result.name]) + out = exe.run( + train_prog, feed={'tindata': indata}, fetch_list=[result.name] + ) sys.stdout.buffer.write(pickle.dumps(out)) @@ -159,19 +169,20 @@ def runtime_main(test_class): class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -190,7 +201,7 @@ def _run_cluster(self, model_file, envs): "PADDLE_TRAINER_ID": "0", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w0_ep + "PADDLE_CURRENT_ENDPOINT": w0_ep, } env1 = { @@ -198,9 +209,9 @@ def _run_cluster(self, model_file, envs): "PADDLE_TRAINER_ID": "1", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w1_ep + "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) tr_cmd = "%s %s" @@ -209,15 +220,19 @@ def _run_cluster(self, model_file, envs): tr0_pipe = open("/tmp/tr0_err.log", "wb") tr1_pipe = open("/tmp/tr1_err.log", "wb") - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) - tr1_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + tr1_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -226,15 +241,21 @@ def _run_cluster(self, model_file, envs): # close trainer file tr0_pipe.close() tr1_pipe.close() - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid - - def check_with_place(self, - model_file, - col_type, - data_type, - check_error_log=False, - need_envs={}): + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) + + def check_with_place( + self, + model_file, + col_type, + data_type, + check_error_log=False, + need_envs={}, + ): required_envs = { "FLAGS_eager_delete_tensor_gb": "0.0", "PATH": os.getenv("PATH"), @@ -250,7 +271,8 @@ def check_with_place(self, required_envs["GLOG_v"] = "3" required_envs["GLOG_logtostderr"] = "1" tr0_out, tr1_out, pid0, pid1 = self._run_cluster( - model_file, required_envs) + model_file, required_envs + ) np_data_type = DataTypeCast(data_type) np.random.seed(pid0) input1 = np.random.random((10, 1000)).astype(np_data_type) @@ -262,44 +284,36 @@ def check_with_place(self, np.testing.assert_allclose(tr1_out[0], need_result) elif col_type == "allreduce_sum": need_result = input1 + input2 - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "allreduce_prod": need_result = input1 * input2 - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "allreduce_max": need_result = np.maximum(input1, input2) - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "allreduce_min": need_result = np.minimum(input1, input2) - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "reduce_sum": need_result = input1 + input2 np.testing.assert_allclose(tr1_out[0], need_result) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast.py index 723a14edb932e..d06a599243e5e 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast.py @@ -23,32 +23,36 @@ class TestCBroadcastOp(TestDistBase): - def _setup_config(self): pass def test_broadcast_fp32(self): - self.check_with_place("collective_broadcast_op.py", "broadcast", - "float32") + self.check_with_place( + "collective_broadcast_op.py", "broadcast", "float32" + ) def test_broadcast_fp16(self): - self.check_with_place("collective_broadcast_op.py", "broadcast", - "float16") + self.check_with_place( + "collective_broadcast_op.py", "broadcast", "float16" + ) def test_broadcast_int32(self): - self.check_with_place("collective_broadcast_op.py", "broadcast", - "int32") + self.check_with_place( + "collective_broadcast_op.py", "broadcast", "int32" + ) def test_broadcast_int16(self): - self.check_with_place("collective_broadcast_op.py", "broadcast", - "int16") + self.check_with_place( + "collective_broadcast_op.py", "broadcast", "int16" + ) def test_broadcast_int8(self): self.check_with_place("collective_broadcast_op.py", "broadcast", "int8") def test_broadcast_uint8(self): - self.check_with_place("collective_broadcast_op.py", "broadcast", - "uint8") + self.check_with_place( + "collective_broadcast_op.py", "broadcast", "uint8" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast_api_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast_api_mlu.py index 0619c5507142e..d40da0f1db66a 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast_api_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_broadcast_api_mlu.py @@ -22,21 +22,23 @@ class TestCollectiveBroadcastAPI(TestDistBase): - def _setup_config(self): pass def test_broadcast_cncl_fp16(self): - self.check_with_place("collective_broadcast_api.py", "broadcast", - "float16") + self.check_with_place( + "collective_broadcast_api.py", "broadcast", "float16" + ) def test_broadcast_cncl_fp32(self): - self.check_with_place("collective_broadcast_api.py", "broadcast", - "float32") + self.check_with_place( + "collective_broadcast_api.py", "broadcast", "float32" + ) def test_broadcast_cncl_int32(self): - self.check_with_place("collective_broadcast_api.py", "broadcast", - "int32") + self.check_with_place( + "collective_broadcast_api.py", "broadcast", "int32" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_api_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_api_mlu.py index d76429196cc28..69c1ebaf43cd8 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_api_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_api_mlu.py @@ -22,7 +22,6 @@ class TestCollectiveReduceAPI(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_max.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_max.py index fd84b1c1eab23..ac07b352e0cb6 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_max.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_max.py @@ -23,17 +23,18 @@ class TestCReduceOp(TestDistBase): - def _setup_config(self): pass def test_reduce_max_fp32(self): - self.check_with_place("collective_reduce_op.py", "reduce_max", - "float32") + self.check_with_place( + "collective_reduce_op.py", "reduce_max", "float32" + ) def test_reduce_max_fp16(self): - self.check_with_place("collective_reduce_op.py", "reduce_max", - "float16") + self.check_with_place( + "collective_reduce_op.py", "reduce_max", "float16" + ) def test_reduce_max_int32(self): self.check_with_place("collective_reduce_op.py", "reduce_max", "int32") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_min.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_min.py index 6976e047e91e2..abe970bafb507 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_min.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_min.py @@ -23,17 +23,18 @@ class TestCReduceOp(TestDistBase): - def _setup_config(self): pass def test_reduce_min_fp32(self): - self.check_with_place("collective_reduce_op.py", "reduce_min", - "float32") + self.check_with_place( + "collective_reduce_op.py", "reduce_min", "float32" + ) def test_reduce_min_fp16(self): - self.check_with_place("collective_reduce_op.py", "reduce_min", - "float16") + self.check_with_place( + "collective_reduce_op.py", "reduce_min", "float16" + ) def test_reduce_min_int32(self): self.check_with_place("collective_reduce_op.py", "reduce_min", "int32") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_prod.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_prod.py index a5ed2ff8b1e4a..6fc9bd331ca3b 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_prod.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_prod.py @@ -23,17 +23,18 @@ class TestCReduceOp(TestDistBase): - def _setup_config(self): pass def test_reduce_prod_fp32(self): - self.check_with_place("collective_reduce_op.py", "reduce_prod", - "float32") + self.check_with_place( + "collective_reduce_op.py", "reduce_prod", "float32" + ) def test_reduce_prod_fp16(self): - self.check_with_place("collective_reduce_op.py", "reduce_prod", - "float16") + self.check_with_place( + "collective_reduce_op.py", "reduce_prod", "float16" + ) def test_reduce_prod_int32(self): self.check_with_place("collective_reduce_op.py", "reduce_prod", "int32") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_sum.py b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_sum.py index a86749de5fb62..141a424a978d5 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_sum.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_collective_reduce_sum.py @@ -23,17 +23,18 @@ class TestCReduceOp(TestDistBase): - def _setup_config(self): pass def test_reduce_sum_fp32(self): - self.check_with_place("collective_reduce_op.py", "reduce_sum", - "float32") + self.check_with_place( + "collective_reduce_op.py", "reduce_sum", "float32" + ) def test_reduce_sum_fp16(self): - self.check_with_place("collective_reduce_op.py", "reduce_sum", - "float16") + self.check_with_place( + "collective_reduce_op.py", "reduce_sum", "float16" + ) def test_reduce_sum_int32(self): self.check_with_place("collective_reduce_op.py", "reduce_sum", "int32") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_compare_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_compare_op_mlu.py index f314a6759d95a..f994fb185f64e 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_compare_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_compare_op_mlu.py @@ -24,9 +24,7 @@ def create_test_class(op_type, typename, callback): - class Cls(OpTest): - def setUp(self): self.set_mlu() self.place = paddle.MLUPlace(0) @@ -77,46 +75,42 @@ def test_dynamic_api(self): def test_broadcast_api_1(self): paddle.enable_static() with program_guard(Program(), Program()): - x = paddle.static.data(name='x', - shape=[1, 2, 1, 3], - dtype=typename) - y = paddle.static.data(name='y', - shape=[1, 2, 3], - dtype=typename) + x = paddle.static.data( + name='x', shape=[1, 2, 1, 3], dtype=typename + ) + y = paddle.static.data( + name='y', shape=[1, 2, 3], dtype=typename + ) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) exe = paddle.static.Executor(self.place) input_x = np.arange(1, 7).reshape((1, 2, 1, 3)).astype(typename) input_y = np.arange(0, 6).reshape((1, 2, 3)).astype(typename) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) @unittest.skipIf(typename == 'float16', "float16 is not supported now") def test_broadcast_api_2(self): paddle.enable_static() with program_guard(Program(), Program()): - x = paddle.static.data(name='x', - shape=[1, 2, 3], - dtype=typename) - y = paddle.static.data(name='y', - shape=[1, 2, 1, 3], - dtype=typename) + x = paddle.static.data( + name='x', shape=[1, 2, 3], dtype=typename + ) + y = paddle.static.data( + name='y', shape=[1, 2, 1, 3], dtype=typename + ) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) exe = paddle.static.Executor(self.place) input_x = np.arange(0, 6).reshape((1, 2, 3)).astype(typename) input_y = np.arange(1, 7).reshape((1, 2, 1, 3)).astype(typename) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) @unittest.skipIf(typename == 'float16', "float16 is not supported now") @@ -131,11 +125,9 @@ def test_broadcast_api_3(self): input_x = np.arange(0, 5).reshape((5)).astype(typename) input_y = np.array([5, 3, 2]).reshape((3, 1)).astype(typename) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) @unittest.skipIf(typename == 'float16', "float16 is not supported now") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_concat_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_concat_op_mlu.py index 91bed72ff2867..d0caa993c33e9 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_concat_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_concat_op_mlu.py @@ -26,7 +26,6 @@ class TestConcatOp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "concat" @@ -43,8 +42,9 @@ def setUp(self): self.actual_axis = self.axis self.outputs = { - 'Out': - np.concatenate((self.x0, self.x1, self.x2), axis=self.actual_axis) + 'Out': np.concatenate( + (self.x0, self.x1, self.x2), axis=self.actual_axis + ) } def set_mlu(self): @@ -69,7 +69,6 @@ def init_test_data(self): class TestConcatOp2(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((2, 3, 4, 5)).astype(self.dtype) self.x1 = np.random.random((2, 3, 4, 5)).astype(self.dtype) @@ -78,9 +77,9 @@ def init_test_data(self): @skip_check_grad_ci( - reason="The function 'check_grad' for large inputs is too slow.") + reason="The function 'check_grad' for large inputs is too slow." +) class TestConcatOp3(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((1, 256, 170, 256)).astype(self.dtype) self.x1 = np.random.random((1, 128, 170, 256)).astype(self.dtype) @@ -92,11 +91,9 @@ def test_check_grad(self): @skip_check_grad_ci( - reason= - "This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." + reason="This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." ) class TestConcatOp4(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((2, 3, 4, 5)).astype(self.dtype) self.x1 = np.random.random((2, 3, 4, 5)).astype(self.dtype) @@ -108,7 +105,6 @@ def test_check_grad(self): class TestConcatOp5(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((5, 1, 4, 5)).astype(self.dtype) self.x1 = np.random.random((5, 2, 4, 5)).astype(self.dtype) @@ -116,11 +112,9 @@ def init_test_data(self): self.axis = -3 -#----------------Concat Fp16---------------- +# ----------------Concat Fp16---------------- def create_test_fp16(parent): - class TestConcatFp16(parent): - def init_dtype(self): self.dtype = np.float16 @@ -136,11 +130,9 @@ def init_dtype(self): create_test_fp16(TestConcatOp5) -#----------------Concat Int64---------------- +# ----------------Concat Int64---------------- def create_test_int64(parent): - class TestConcatInt64(parent): - def init_dtype(self): self.dtype = np.int64 @@ -159,11 +151,9 @@ def test_check_grad(self): create_test_int64(TestConcatOp5) -#----------------Concat Int32---------------- +# ----------------Concat Int32---------------- def create_test_int32(parent): - class TestConcatInt32(parent): - def init_dtype(self): self.dtype = np.int32 @@ -182,11 +172,9 @@ def test_check_grad(self): create_test_int32(TestConcatOp5) -#----------------Concat AxisTensor---------------- +# ----------------Concat AxisTensor---------------- def create_test_AxisTensor(parent): - class TestConcatAxisTensor(parent): - def setUp(self): self.op_type = "concat" self.init_dtype() @@ -194,20 +182,22 @@ def setUp(self): self.inputs = { 'X': [('x0', self.x0), ('x1', self.x1), ('x2', self.x2)], - 'AxisTensor': np.array([self.axis]).astype("int32") + 'AxisTensor': np.array([self.axis]).astype("int32"), } self.attrs = {} if self.axis < 0: self.actual_axis = self.axis + len(self.x0.shape) - self.actual_axis = self.actual_axis if self.actual_axis > 0 else 0 + self.actual_axis = ( + self.actual_axis if self.actual_axis > 0 else 0 + ) else: self.actual_axis = self.axis self.outputs = { - 'Out': - np.concatenate((self.x0, self.x1, self.x2), - axis=self.actual_axis) + 'Out': np.concatenate( + (self.x0, self.x1, self.x2), axis=self.actual_axis + ) } self.place = paddle.device.MLUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_depthwise_conv_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_depthwise_conv_mlu.py index 8fac2941e43c1..593bbc7096e5d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_depthwise_conv_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_depthwise_conv_mlu.py @@ -25,13 +25,19 @@ import paddle.fluid as fluid from op_test import OpTest from paddle.fluid import Program, program_guard -from test_conv2d_op_mlu import TestConv2DOp, TestConv2DOp_v2, create_test_padding_SAME_class, create_test_padding_VALID_class, create_test_channel_last_class, create_test_fp16_class +from test_conv2d_op_mlu import ( + TestConv2DOp, + TestConv2DOp_v2, + create_test_padding_SAME_class, + create_test_padding_VALID_class, + create_test_channel_last_class, + create_test_fp16_class, +) -#----------------TestDepthwiseConv ----- +# ----------------TestDepthwiseConv ----- class TestDepthwiseConv(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -44,7 +50,6 @@ def init_test_case(self): class TestDepthwiseConv2(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -57,7 +62,6 @@ def init_test_case(self): class TestDepthwiseConv3(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -70,7 +74,6 @@ def init_test_case(self): class TestDepthwiseConvandFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.pad = [1, 1] @@ -84,7 +87,6 @@ def init_test_case(self): class TestDepthwiseConv2andFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.pad = [1, 1] @@ -98,7 +100,6 @@ def init_test_case(self): class TestDepthwiseConv3andFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.pad = [1, 1] @@ -112,7 +113,6 @@ def init_test_case(self): class TestDepthwiseConv_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 3, 5, 5] # NCHW @@ -128,7 +128,6 @@ def init_paddings(self): class TestDepthwiseConv2_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -144,7 +143,6 @@ def init_paddings(self): class TestDepthwiseConv3_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -160,7 +158,6 @@ def init_paddings(self): class TestDepthwiseConvandFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.pad = [1, 1] @@ -178,7 +175,6 @@ def init_paddings(self): class TestDepthwiseConv2andFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.pad = [1, 1] @@ -196,7 +192,6 @@ def init_paddings(self): class TestDepthwiseConv3andFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.pad = [1, 1] diff --git a/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_mlu.py index 7f79caa549926..7153ea09ef2fc 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_conv2d_op_mlu.py @@ -28,9 +28,7 @@ def create_test_channel_last_class(parent): - class TestChannelLastCase(parent): - def init_data_format(self): self.data_format = "NHWC" @@ -44,9 +42,7 @@ def init_test_case_2(self): def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" @@ -57,9 +53,7 @@ def init_paddings(self): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" @@ -70,9 +64,7 @@ def init_paddings(self): def create_test_fp16_class(parent): - class TestFp16Case(parent): - def init_dtype(self): self.dtype = np.float16 @@ -82,7 +74,6 @@ def init_dtype(self): class TestConv2DOp(OpTest): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.device.MLUPlace(0) @@ -105,22 +96,24 @@ def setUp(self): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.uniform(-1, 1, self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input, - filter, - self.groups, - conv2d_param, - data_format=self.data_format) + output, _, _, _, _ = conv2d_forward_naive( + input, + filter, + self.groups, + conv2d_param, + data_format=self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -137,28 +130,37 @@ def test_check_output(self): def test_check_grad(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def init_test_case(self): self.pad = [0, 0] @@ -176,7 +178,6 @@ def init_group(self): class TestWithPad(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -187,7 +188,6 @@ def init_test_case(self): class TestWithStride(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -198,7 +198,6 @@ def init_test_case(self): class TestWithGroup(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -210,7 +209,6 @@ def init_test_case(self): class TestWith1x1(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -227,7 +225,6 @@ def init_group(self): class TestWithDepthWise5x5(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -241,7 +238,6 @@ def init_group(self): class TestWithDepthWise7x7(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -255,7 +251,6 @@ def init_group(self): class TestWithDilation(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -274,7 +269,6 @@ def init_dilation(self): class TestWithInput1x1Filter1x1(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -288,7 +282,6 @@ def init_group(self): class TestConv2DOp_v2(OpTest): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.device.MLUPlace(0) @@ -308,20 +301,24 @@ def setUp(self): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.uniform(-1, 1, self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input, filter, self.groups, - conv2d_param, - self.padding_algorithm, - self.data_format) + output, _, _, _, _ = conv2d_forward_naive( + input, + filter, + self.groups, + conv2d_param, + self.padding_algorithm, + self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -339,27 +336,36 @@ def test_check_output(self): def test_check_grad(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.02, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.02, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Input'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Filter']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Filter']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def init_test_case(self): self.pad = [0, 0] @@ -390,14 +396,12 @@ def init_test_case_2(self): class TestConv2DOp_AsyPadding(TestConv2DOp_v2): - def init_paddings(self): self.pad = [0, 0, 1, 2] self.padding_algorithm = "EXPLICIT" class TestWithPad_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -411,7 +415,6 @@ def init_paddings(self): class TestWithStride_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 3, 6, 6] # NCHW @@ -425,7 +428,6 @@ def init_paddings(self): class TestWithGroup_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 2] @@ -437,7 +439,6 @@ def init_test_case(self): class TestWith1x1_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -454,7 +455,6 @@ def init_paddings(self): class TestWithDepthWise3x3_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [3, 4, 10, 10] # NCHW @@ -476,7 +476,6 @@ def init_paddings(self): class TestWithDepthWise5x5_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 4, 10, 10] # NCHW @@ -493,7 +492,6 @@ def init_paddings(self): class TestWithDepthWise7x7_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 8, 10, 10] # NCHW @@ -510,7 +508,6 @@ def init_paddings(self): class TestWithDilation_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 10, 10] # NCHW @@ -532,7 +529,6 @@ def init_paddings(self): class TestWithInput1x1Filter1x1_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [100, 1, 1, 1] # NCHW diff --git a/python/paddle/fluid/tests/unittests/mlu/test_conv2d_transposed_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_conv2d_transposed_op_mlu.py index 0d33bf853691e..076c6e2ca3fcd 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_conv2d_transposed_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_conv2d_transposed_op_mlu.py @@ -27,9 +27,10 @@ def conv2dtranspose_forward_naive(input_, filter_, attrs): padding_algorithm = attrs['padding_algorithm'] if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if attrs['data_format'] == 'NHWC': input_ = np.transpose(input_, [0, 3, 1, 2]) @@ -40,18 +41,22 @@ def conv2dtranspose_forward_naive(input_, filter_, attrs): out_c = f_out_c * groups sub_in_c = in_c // groups - stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ - 'dilations'] + stride, pad, dilations = ( + attrs['strides'], + attrs['paddings'], + attrs['dilations'], + ) # update pad and dilation def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, - kernel_size, - kernel_stride): + for input_size, filter_size, stride_size in zip( + input_shape, kernel_size, kernel_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -85,37 +90,49 @@ def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride): if 'output_padding' in attrs: out_pad_h = attrs['output_padding'][0] out_pad_w = attrs['output_padding'][1] - out = np.zeros((in_n, out_c, out_h + out_pad_h, out_w + out_pad_w), - dtype=input_.dtype) + out = np.zeros( + (in_n, out_c, out_h + out_pad_h, out_w + out_pad_w), dtype=input_.dtype + ) for n in range(in_n): for i in range(in_h): for j in range(in_w): for g in range(groups): - input_masked = input_[n, g * sub_in_c:(g + 1) * sub_in_c, i, - j] # (c) + input_masked = input_[ + n, g * sub_in_c : (g + 1) * sub_in_c, i, j + ] # (c) input_masked = np.reshape(input_masked, (sub_in_c, 1, 1)) input_masked = np.tile(input_masked, (1, f_h, f_w)) for k in range(f_out_c): tmp_out = np.sum( - input_masked * - filter_[g * sub_in_c:(g + 1) * sub_in_c, k, :, :], - axis=0) + input_masked + * filter_[ + g * sub_in_c : (g + 1) * sub_in_c, k, :, : + ], + axis=0, + ) i1, i2 = i * stride[0], i * stride[0] + d_bolck_h j1, j2 = j * stride[1], j * stride[1] + d_bolck_w - out[n, g * f_out_c + k, i1:i2:dilations[0], - j1:j2:dilations[1]] += tmp_out - - out = out[:, :, pad_h_0:out_h - pad_h_1 + out_pad_h, - pad_w_0:out_w - pad_w_1 + out_pad_w] + out[ + n, + g * f_out_c + k, + i1 : i2 : dilations[0], + j1 : j2 : dilations[1], + ] += tmp_out + + out = out[ + :, + :, + pad_h_0 : out_h - pad_h_1 + out_pad_h, + pad_w_0 : out_w - pad_w_1 + out_pad_w, + ] if attrs['data_format'] == 'NHWC': out = np.transpose(out, [0, 2, 3, 1]) return out class TestConv2DTransposeOp(OpTest): - def setUp(self): # init as conv transpose self.dtype = np.float32 @@ -145,7 +162,7 @@ def setUp(self): 'use_cudnn': self.use_cudnn, 'is_test': self.is_test, 'use_mkldnn': self.use_mkldnn, - 'data_format': self.data_format + 'data_format': self.data_format, } if self.output_size is not None: self.attrs['output_size'] = self.output_size @@ -153,8 +170,9 @@ def setUp(self): if len(self.output_padding) > 0: self.attrs['output_padding'] = self.output_padding - output = conv2dtranspose_forward_naive(input_, filter_, - self.attrs).astype(self.dtype) + output = conv2dtranspose_forward_naive( + input_, filter_, self.attrs + ).astype(self.dtype) self.outputs = {'Output': output} @@ -167,23 +185,28 @@ def test_check_output(self): def test_check_grad_no_input(self): if self.need_check_grad: - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Input'])) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Input']), + ) def test_check_grad_no_filter(self): if self.need_check_grad: - self.check_grad_with_place(self.place, ['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + self.place, ['Input'], 'Output', no_grad_set=set(['Filter']) + ) def test_check_grad(self): if self.need_check_grad: - self.check_grad_with_place(self.place, - set(['Input', 'Filter']), - 'Output', - max_relative_error=0.02) + self.check_grad_with_place( + self.place, + set(['Input', 'Filter']), + 'Output', + max_relative_error=0.02, + ) def init_test_case(self): self.pad = [0, 0] @@ -199,7 +222,6 @@ def init_op_type(self): class TestWithSymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -211,7 +233,6 @@ def init_test_case(self): class TestWithAsymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -223,7 +244,6 @@ def init_test_case(self): class TestWithSAMEPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [2, 1] self.dilations = [1, 2] @@ -235,7 +255,6 @@ def init_test_case(self): class TestWithVALIDPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [1, 1] self.dilations = [1, 1] @@ -247,7 +266,6 @@ def init_test_case(self): class TestWithGroups(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -259,7 +277,6 @@ def init_test_case(self): class TestWithStride(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -271,7 +288,6 @@ def init_test_case(self): class TestWithDilation(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -283,7 +299,6 @@ def init_test_case(self): class TestWithEvenUpsample(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -296,7 +311,6 @@ def init_test_case(self): class TestWithEvenUpsampleOutputPadding(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -309,7 +323,6 @@ def init_test_case(self): class Test_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -322,7 +335,6 @@ def init_test_case(self): class TestWithSymmetricPad_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -335,7 +347,6 @@ def init_test_case(self): class TestWithAsymmetricPad_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -348,7 +359,6 @@ def init_test_case(self): class TestWithGroups_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -361,7 +371,6 @@ def init_test_case(self): class TestWithStride_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -374,7 +383,6 @@ def init_test_case(self): class TestWithDilation_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -387,7 +395,6 @@ def init_test_case(self): class TestWithEvenUpsample_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -401,7 +408,6 @@ def init_test_case(self): class TestWithEvenUpsample_NHWC_output_padding(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -415,7 +421,6 @@ def init_test_case(self): class TestMLU_FP16(TestConv2DTransposeOp): - def init_test_case(self): self.dtype = np.float16 self.set_mlu() @@ -440,7 +445,6 @@ def test_check_output(self): class TestMLU_NHWC_FP16(TestMLU_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [0, 0] @@ -454,7 +458,6 @@ def init_test_case(self): class TestMLUWithGroups_NHWC_FP16(TestMLU_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [1, 1] @@ -468,7 +471,6 @@ def init_test_case(self): class TestMLUWithEvenUpsample_NHWC_FP16(TestMLU_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [2, 2] @@ -483,7 +485,6 @@ def init_test_case(self): class TestConv2DTransposeAPI(unittest.TestCase): - def setUp(self): self.set_mlu() @@ -492,67 +493,78 @@ def set_mlu(self): self.place = paddle.device.MLUPlace(0) def test_case1(self): - data1 = fluid.layers.data(name='data1', - shape=[3, 5, 5], - dtype='float32') - data2 = fluid.layers.data(name='data2', - shape=[5, 5, 3], - dtype='float32') - out1 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - data_format='NCHW') - out2 = fluid.layers.conv2d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - data_format='NHWC') - out3 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - padding=[[0, 0], [1, 1], [1, 1], - [0, 0]], - data_format='NHWC') - out4 = fluid.layers.conv2d_transpose(input=data1, - groups=3, - num_filters=6, - filter_size=3, - padding=[[0, 0], [0, 0], [2, 1], - [0, 0]], - data_format='NCHW') - out5 = fluid.layers.conv2d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - padding='SAME', - data_format='NCHW') - out6 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - padding='VALID', - data_format='NHWC') - out7 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - output_size=[7, 7], - padding=[0, 0], - data_format='NHWC') + data1 = fluid.layers.data( + name='data1', shape=[3, 5, 5], dtype='float32' + ) + data2 = fluid.layers.data( + name='data2', shape=[5, 5, 3], dtype='float32' + ) + out1 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + data_format='NCHW', + ) + out2 = fluid.layers.conv2d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + data_format='NHWC', + ) + out3 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + padding=[[0, 0], [1, 1], [1, 1], [0, 0]], + data_format='NHWC', + ) + out4 = fluid.layers.conv2d_transpose( + input=data1, + groups=3, + num_filters=6, + filter_size=3, + padding=[[0, 0], [0, 0], [2, 1], [0, 0]], + data_format='NCHW', + ) + out5 = fluid.layers.conv2d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + padding='SAME', + data_format='NCHW', + ) + out6 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + padding='VALID', + data_format='NHWC', + ) + out7 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + output_size=[7, 7], + padding=[0, 0], + data_format='NHWC', + ) data1_np = np.random.random((2, 3, 5, 5)).astype("float32") data2_np = np.random.random((2, 5, 5, 3)).astype("float32") exe = fluid.Executor(self.place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "data1": data1_np, - "data2": data2_np - }, - fetch_list=[out1, out2, out3, out4, out5, out6, out7], - return_numpy=True) + results = exe.run( + fluid.default_main_program(), + feed={"data1": data1_np, "data2": data2_np}, + fetch_list=[out1, out2, out3, out4, out5, out6, out7], + return_numpy=True, + ) self.assertIsNotNone(results[0]) self.assertIsNotNone(results[1]) self.assertIsNotNone(results[2]) @@ -563,7 +575,6 @@ def test_case1(self): class TestConv2DTransposeOpException(unittest.TestCase): - def setUp(self): self.set_mlu() @@ -575,68 +586,74 @@ def test_exception(self): data = fluid.layers.data(name='data', shape=[3, 5, 5], dtype="float32") def attr_data_format(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - data_format="NCDHW") + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + data_format="NCDHW", + ) self.assertRaises(ValueError, attr_data_format) def attr_padding_str(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding='Vald') + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding='Vald', + ) self.assertRaises(ValueError, attr_padding_str) def attr_padding_list(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding=[[1, 1], [1, 1], [0, 0], - [0, 0]]) + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding=[[1, 1], [1, 1], [0, 0], [0, 0]], + ) self.assertRaises(ValueError, attr_padding_list) def attr_padding_with_data_format(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding=[[1, 1], [0, 0], [0, 0], - [1, 1]], - data_format='NHWC') + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding=[[1, 1], [0, 0], [0, 0], [1, 1]], + data_format='NHWC', + ) self.assertRaises(ValueError, attr_padding_with_data_format) - error_input = fluid.layers.data(name='error_data', - shape=[1], - dtype="float32") + error_input = fluid.layers.data( + name='error_data', shape=[1], dtype="float32" + ) def error_input_size(): - out = fluid.layers.conv2d_transpose(input=error_input, - groups=1, - num_filters=6, - filter_size=3) + out = fluid.layers.conv2d_transpose( + input=error_input, groups=1, num_filters=6, filter_size=3 + ) self.assertRaises(ValueError, error_input_size) def error_groups(): - out = fluid.layers.conv2d_transpose(input=data, - groups=0, - num_filters=6, - filter_size=3, - data_format='NHWC') + out = fluid.layers.conv2d_transpose( + input=data, + groups=0, + num_filters=6, + filter_size=3, + data_format='NHWC', + ) self.assertRaises(ValueError, error_groups) class TestConv2DTransposeRepr(unittest.TestCase): - def setUp(self): self.set_mlu() @@ -646,7 +663,7 @@ def set_mlu(self): def test_case(self): paddle.disable_static() - x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.) + x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0) conv = nn.Conv2DTranspose(4, 6, (3, 3), output_padding=1, stride=2) print(conv) y_var = conv(x_var) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_cumsum_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_cumsum_op_mlu.py index 77847a8f64d4d..ca1b99ec79c4e 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_cumsum_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_cumsum_op_mlu.py @@ -24,7 +24,6 @@ class TestMLUCumSumOp(OpTest): - def setUp(self): self.op_type = "cumsum" self.set_mlu() @@ -48,18 +47,17 @@ def init_testcase(self): class TestMLUCumSumOp2(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': -1, 'reverse': True} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.outputs = { - 'Out': np.flip(np.flip(self.inputs['X'], axis=2).cumsum(axis=2), - axis=2) + 'Out': np.flip( + np.flip(self.inputs['X'], axis=2).cumsum(axis=2), axis=2 + ) } class TestMLUCumSumOp3(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 1} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} @@ -67,7 +65,6 @@ def init_testcase(self): class TestMLUCumSumOp4(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 0} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} @@ -75,107 +72,115 @@ def init_testcase(self): class TestMLUCumSumOp5(TestMLUCumSumOp): - def init_testcase(self): self.inputs = {'X': np.random.random((5, 20)).astype(self.dtype)} self.outputs = {'Out': self.inputs['X'].cumsum(axis=1)} class TestMLUCumSumOp7(TestMLUCumSumOp): - def init_testcase(self): self.inputs = {'X': np.random.random((100)).astype(self.dtype)} self.outputs = {'Out': self.inputs['X'].cumsum(axis=0)} class TestNPUCumSumExclusive1(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 65)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive2(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((1, 1, 888)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (1, 1, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((1, 1, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive3(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 888)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive4(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((1, 1, 3049)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (1, 1, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((1, 1, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive5(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 3096)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumReverseExclusive(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'axis': 2, 'reverse': True, "exclusive": True} a = np.random.random((4, 5, 6)).astype(self.dtype) self.inputs = {'X': a} a = np.flip(a, axis=2) self.outputs = { - 'Out': - np.concatenate( - (np.flip(a[:, :, :-1].cumsum(axis=2), - axis=2), np.zeros((4, 5, 1), dtype=self.dtype)), - axis=2) + 'Out': np.concatenate( + ( + np.flip(a[:, :, :-1].cumsum(axis=2), axis=2), + np.zeros((4, 5, 1), dtype=self.dtype), + ), + axis=2, + ) } class TestNPUCumSumWithFlatten1(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'flatten': True} self.inputs = {'X': np.random.random((5, 6)).astype(self.dtype)} @@ -183,7 +188,6 @@ def init_testcase(self): class TestNPUCumSumWithFlatten2(TestMLUCumSumOp): - def init_testcase(self): self.attrs = {'flatten': True} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} diff --git a/python/paddle/fluid/tests/unittests/mlu/test_deformable_conv_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_deformable_conv_op_mlu.py index 352d9641827ae..5332299c44bcc 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_deformable_conv_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_deformable_conv_op_mlu.py @@ -28,7 +28,6 @@ class TestModulatedDeformableConvOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -42,7 +41,7 @@ def setUp(self): conv_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) @@ -50,15 +49,16 @@ def setUp(self): mask = 10 * np.random.random(self.mask_size).astype(self.dtype) filter = np.random.random(self.filter_size).astype(self.dtype) - output = dconv_im2col_gemm(input, offset, mask, filter, self.groups, - conv_param) + output = dconv_im2col_gemm( + input, offset, mask, filter, self.groups, conv_param + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), 'Offset': OpTest.np_dtype_to_fluid_dtype(offset), 'Mask': OpTest.np_dtype_to_fluid_dtype(mask), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -74,11 +74,13 @@ def test_check_output(self): self.check_output_with_place(self.place, check_eager=False) def test_check_grad(self): - self.check_grad_with_place(self.place, - {'Input', 'Offset', 'Mask', 'Filter'}, - 'Output', - max_relative_error=0.05, - check_eager=False) + self.check_grad_with_place( + self.place, + {'Input', 'Offset', 'Mask', 'Filter'}, + 'Output', + max_relative_error=0.05, + check_eager=False, + ) def init_test_case(self): self.pad = [1, 1] @@ -90,15 +92,26 @@ def init_test_case(self): self.filter_size = [4, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): @@ -112,7 +125,6 @@ def init_type(self): class TestWithStride(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [3, 3] self.stride = [2, 2] @@ -122,20 +134,30 @@ def init_test_case(self): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] class TestWithDilation(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [1, 1] @@ -145,15 +167,26 @@ def init_test_case(self): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): @@ -161,7 +194,6 @@ def init_dilation(self): class TestWith3x3(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -171,15 +203,26 @@ def init_test_case(self): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] diff --git a/python/paddle/fluid/tests/unittests/mlu/test_dropout_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_dropout_op_mlu.py index 0bbec2b7ae05e..9c115c3bd3894 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_dropout_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_dropout_op_mlu.py @@ -27,7 +27,6 @@ class TestDropoutOp(OpTest): - def setUp(self): self.set_mlu() self.init_dtype() @@ -39,7 +38,7 @@ def setUp(self): 'dropout_prob': self.dropout_prob, 'fix_seed': self.fix_seed, 'is_test': self.is_test, - 'dropout_implementation': self.dropout_implementation + 'dropout_implementation': self.dropout_implementation, } out = self.inputs['X'] * (1.0 - self.dropout_prob) @@ -74,21 +73,21 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - if hasattr(self.__class__, "no_need_check_grad" - ) and self.__class__.no_need_check_grad == True: + if ( + hasattr(self.__class__, "no_need_check_grad") + and self.__class__.no_need_check_grad == True + ): return self.check_grad_with_place(self.place, ['X'], 'Out') class TestDropoutOpInput1d(TestDropoutOp): - def init_inputs_shape(self): self.shape = [2000] class TestDropoutOp2(TestDropoutOp): - def init_inputs_shape(self): self.shape = [32, 64] @@ -100,13 +99,11 @@ def init_attrs(self): class TestDropoutOp3(TestDropoutOp): - def init_inputs_shape(self): self.shape = [32, 64, 2] class TestDropoutOp4(TestDropoutOp): - def init_attrs(self): self.__class__.no_need_check_grad = True self.dropout_prob = 0.35 @@ -116,7 +113,6 @@ def init_attrs(self): class TestDropoutOp5(TestDropoutOp): - def init_inputs_shape(self): self.shape = [32, 64, 3] @@ -129,7 +125,6 @@ def init_attrs(self): class TestDropoutOp6(TestDropoutOp): - def init_attrs(self): self.__class__.no_need_check_grad = True self.dropout_prob = 0.0 @@ -146,16 +141,16 @@ def setUp(self): self.dtype = np.float32 self.inputs = { "X": np.random.random((32, 64)).astype(self.dtype), - "Seed": np.asarray([125], dtype="int32") + "Seed": np.asarray([125], dtype="int32"), } self.attrs = { 'dropout_prob': 0.0, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64)).astype('uint8') + 'Mask': np.ones((32, 64)).astype('uint8'), } def set_mlu(self): @@ -192,7 +187,7 @@ def setUp(self): 'dropout_prob': 0.35, 'fix_seed': True, 'is_test': True, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = {'Out': self.inputs['X']} @@ -209,7 +204,6 @@ def test_check_output(self): @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestDropoutOpInference2(TestDropoutOpInference): - def setUp(self): self.op_type = "dropout" self.set_mlu() @@ -218,13 +212,12 @@ def setUp(self): self.attrs = { 'dropout_prob': 0.75, 'is_test': True, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = {'Out': self.inputs['X']} class TestDropoutAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace(), paddle.device.MLUPlace(0)] @@ -232,43 +225,44 @@ def setUp(self): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): input = fluid.data(name="input", shape=[40, 40], dtype="float32") - res1 = paddle.nn.functional.dropout(x=input, - p=0., - training=False, - mode='upscale_in_train') - res2 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=True, - mode='upscale_in_train') - res3 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=False, - mode='upscale_in_train') - res4 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=True, - mode='upscale_in_train') - res5 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=False, - mode='upscale_in_train') - res6 = paddle.nn.functional.dropout(x=input, - p=1., - training=True, - mode='upscale_in_train') + res1 = paddle.nn.functional.dropout( + x=input, p=0.0, training=False, mode='upscale_in_train' + ) + res2 = paddle.nn.functional.dropout( + x=input, p=0.0, axis=0, training=True, mode='upscale_in_train' + ) + res3 = paddle.nn.functional.dropout( + x=input, p=0.0, axis=0, training=False, mode='upscale_in_train' + ) + res4 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=True, + mode='upscale_in_train', + ) + res5 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=False, + mode='upscale_in_train', + ) + res6 = paddle.nn.functional.dropout( + x=input, p=1.0, training=True, mode='upscale_in_train' + ) res7 = paddle.fluid.layers.dropout( x=input, - dropout_prob=0., - dropout_implementation='upscale_in_train') - res8 = paddle.nn.functional.dropout(x=input, - p=0., - axis=(0, 1), - training=False, - mode='upscale_in_train') + dropout_prob=0.0, + dropout_implementation='upscale_in_train', + ) + res8 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=(0, 1), + training=False, + mode='upscale_in_train', + ) in_np = np.random.random([40, 40]).astype("float32") res_np = in_np @@ -277,13 +271,17 @@ def check_static_result(self, place): exe = fluid.Executor(place) res_list = [res1, res2, res3, res4, res5, res7, res8] for res in res_list: - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res], + ) np.testing.assert_allclose(fetches[0], res_np) - fetches2 = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res6]) + fetches2 = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res6], + ) np.testing.assert_allclose(fetches2[0], res_np2) def test_static(self): diff --git a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_add_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_add_op_mlu.py index b76969332dd38..f5e79bb64ec80 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_add_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_add_op_mlu.py @@ -27,7 +27,6 @@ class TestElementwiseAddOp(OpTest): - def set_mlu(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -41,7 +40,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis} self.outputs = {'Out': self.out} @@ -52,25 +51,31 @@ def test_check_output(self): def test_check_grad_normal(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', max_relative_error=0.01 + ) def test_check_grad_ingore_x(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - max_relative_error=0.01) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + max_relative_error=0.01, + ) def test_check_grad_ingore_y(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set('Y'), - max_relative_error=0.01) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set('Y'), + max_relative_error=0.01, + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -85,7 +90,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp(TestElementwiseAddOp): - def init_dtype(self): self.dtype = np.float16 @@ -94,9 +98,9 @@ def test_check_output(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_scalar(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -104,9 +108,9 @@ def init_input_output(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestFP16ElementwiseAddOp_scalar(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -114,9 +118,9 @@ def init_input_output(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." +) class TestElementwiseAddOp_scalar2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) @@ -124,9 +128,9 @@ def init_input_output(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." +) class TestFP16ElementwiseAddOp_scalar2(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) @@ -134,23 +138,20 @@ def init_input_output(self): class TestElementwiseAddOp_Vector(TestElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestFP16ElementwiseAddOp_Vector(TestFP16ElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestElementwiseAddOp_broadcast_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -161,7 +162,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_broadcast_0(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -172,7 +172,6 @@ def init_axis(self): class TestElementwiseAddOp_broadcast_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -183,7 +182,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_broadcast_1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -194,7 +192,6 @@ def init_axis(self): class TestElementwiseAddOp_broadcast_2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -202,7 +199,6 @@ def init_input_output(self): class TestFP16ElementwiseAddOp_broadcast_2(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -210,7 +206,6 @@ def init_input_output(self): class TestElementwiseAddOp_broadcast_3(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 1).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -221,7 +216,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_broadcast_3(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -232,7 +226,6 @@ def init_axis(self): class TestElementwiseAddOp_broadcast_4(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 1, 2).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -243,7 +236,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_broadcast_4(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 1, 2).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -254,7 +246,6 @@ def init_axis(self): class TestElementwiseAddOp_broadcast_5(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) @@ -262,7 +253,6 @@ def init_input_output(self): class TestFP16ElementwiseAddOp_broadcast_5(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) @@ -270,7 +260,6 @@ def init_input_output(self): class TestElementwiseAddOp_broadcast_6(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) @@ -278,7 +267,6 @@ def init_input_output(self): class TestElementwiseAddOp_broadcast_7(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(1, 1, 20, 5).astype(self.dtype) self.y = np.random.rand(20, 5, 1, 1).astype(self.dtype) @@ -286,7 +274,6 @@ def init_input_output(self): class TestFP16ElementwiseAddOp_broadcast_6(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) @@ -294,7 +281,6 @@ def init_input_output(self): class TestElementwiseAddOp_rowwise_add_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -305,7 +291,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_rowwise_add_0(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -316,9 +301,9 @@ def init_axis(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_rowwise_add_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -329,9 +314,9 @@ def init_axis(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestFP16ElementwiseAddOp_rowwise_add_1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -342,7 +327,6 @@ def init_axis(self): class TestElementwiseAddOp_channelwise_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -353,7 +337,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_channelwise_add(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -364,7 +347,6 @@ def init_axis(self): class TestElementwiseAddOp_commonuse_add1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -375,7 +357,6 @@ def init_axis(self): class TestElementwiseFP16AddOp_commonuse_add1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -386,7 +367,6 @@ def init_axis(self): class TestElementwiseAddOp_commonuse_add2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 1, 4).astype(self.dtype) self.y = np.random.rand(10, 1, 12, 1).astype(self.dtype) @@ -397,7 +377,6 @@ def init_axis(self): class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 12).astype(self.dtype) self.y = np.random.rand(2, 2, 10, 12).astype(self.dtype) @@ -408,7 +387,6 @@ def init_axis(self): class TestElementwiseAddOp_same_shape_ysize_large(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 1, 12).astype(self.dtype) self.y = np.random.rand(10, 2, 12).astype(self.dtype) @@ -419,14 +397,15 @@ def init_axis(self): class TestElementwiseAddOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_add must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.MLUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.MLUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.MLUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.MLUPlace(0) + ) self.assertRaises(TypeError, fluid.layers.elementwise_add, x1, y1) # the input dtype of elementwise_add must be float16 or float32 @@ -436,7 +415,6 @@ def test_errors(self): class TestAddApi(unittest.TestCase): - def _executed_api(self, x, y, name=None): return paddle.add(x, y, name) @@ -454,7 +432,7 @@ def test_declarative(self): def gen_data(): return { "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') + "y": np.array([1, 5, 2]).astype('float32'), } x = fluid.data(name="x", shape=[3], dtype='float32') @@ -464,7 +442,7 @@ def gen_data(): place = fluid.MLUPlace(0) exe = fluid.Executor(place) z_value = exe.run(feed=gen_data(), fetch_list=[z.name]) - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((z_value == z_expected).all(), True) def test_dygraph(self): @@ -475,18 +453,16 @@ def test_dygraph(self): y = fluid.dygraph.to_variable(np_y) z = self._executed_api(x, y) np_z = z.numpy() - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((np_z == z_expected).all(), True) class TestAddInplaceApi(TestAddApi): - def _executed_api(self, x, y, name=None): return x.add_(y, name) class TestAddInplaceBroadcastSuccess(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 4).astype('float32') self.y_numpy = np.random.rand(3, 4).astype('float32') @@ -503,21 +479,18 @@ def test_broadcast_success(self): class TestAddInplaceBroadcastSuccess2(TestAddInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(1, 2, 3, 1).astype('float32') self.y_numpy = np.random.rand(3, 1).astype('float32') class TestAddInplaceBroadcastSuccess3(TestAddInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 1, 5).astype('float32') self.y_numpy = np.random.rand(1, 3, 1, 5).astype('float32') class TestAddInplaceBroadcastError(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(3, 4).astype('float32') self.y_numpy = np.random.rand(2, 3, 4).astype('float32') @@ -536,21 +509,18 @@ def broadcast_shape_error(): class TestAddInplaceBroadcastError2(TestAddInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(2, 1, 4).astype('float32') self.y_numpy = np.random.rand(2, 3, 4).astype('float32') class TestAddInplaceBroadcastError3(TestAddInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(5, 2, 1, 4).astype('float32') self.y_numpy = np.random.rand(2, 3, 4).astype('float32') class TestBoolAddFloatElementwiseAddop(unittest.TestCase): - def test_static_add(self): paddle.enable_static() a = 1.5 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_div_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_div_op_mlu.py index 2171e5043f539..2ca8beed3b324 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_div_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_div_op_mlu.py @@ -27,7 +27,6 @@ class TestElementwiseDiv(OpTest): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" @@ -40,7 +39,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.attrs = {} self.outputs = {'Out': out} @@ -56,25 +55,30 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=0.05) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', max_relative_error=0.05 + ) def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - max_relative_error=0.05, - no_grad_set=set("X")) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + max_relative_error=0.05, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.05, - no_grad_set=set("Y")) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + max_relative_error=0.05, + no_grad_set=set("Y"), + ) class TestElementwiseDivFp16(OpTest): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" @@ -87,7 +91,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.attrs = {} self.outputs = {'Out': out} @@ -105,124 +109,120 @@ def test_check_output(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestTestElementwiseDiv_scalar(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [20, 3, 4]).astype(np.float32), - 'Y': np.random.uniform(0.1, 1, [1]).astype(np.float32) + 'Y': np.random.uniform(0.1, 1, [1]).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] / self.inputs['Y']} class TestTestElementwiseDiv_Vector(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [100]).astype("float32"), - 'Y': np.random.uniform(0.1, 1, [100]).astype("float32") + 'Y': np.random.uniform(0.1, 1, [100]).astype("float32"), } self.outputs = {'Out': np.divide(self.inputs['X'], self.inputs['Y'])} class TestTestElementwiseDiv_broadcast_0(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [100, 3, 4]).astype("float32"), - 'Y': np.random.uniform(0.1, 1, [100]).astype("float32") + 'Y': np.random.uniform(0.1, 1, [100]).astype("float32"), } self.attrs = {'axis': 0} self.outputs = { - 'Out': np.divide(self.inputs['X'], - self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestTestElementwiseDiv_broadcast_1(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 100, 4]).astype("float32"), - 'Y': np.random.uniform(0.1, 1, [100]).astype("float32") + 'Y': np.random.uniform(0.1, 1, [100]).astype("float32"), } self.attrs = {'axis': 1} self.outputs = { - 'Out': np.divide(self.inputs['X'], - self.inputs['Y'].reshape(1, 100, 1)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1) + ) } class TestTestElementwiseDiv_broadcast_2(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 100]).astype("float32"), - 'Y': np.random.uniform(0.1, 1, [100]).astype("float32") + 'Y': np.random.uniform(0.1, 1, [100]).astype("float32"), } self.outputs = { - 'Out': np.divide(self.inputs['X'], - self.inputs['Y'].reshape(1, 1, 100)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100) + ) } class TestTestElementwiseDiv_broadcast_3(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 10, 12, 5]).astype("float32"), - 'Y': np.random.uniform(0.1, 1, [10, 12]).astype("float32") + 'Y': np.random.uniform(0.1, 1, [10, 12]).astype("float32"), } self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.divide(self.inputs['X'], self.inputs['Y'].reshape(1, 10, 12, 1)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(1, 10, 12, 1) + ) } class TestTestElementwiseDiv_broadcast_4(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 50]).astype("float32"), - 'Y': np.random.uniform(0.1, 1, [2, 1, 50]).astype("float32") + 'Y': np.random.uniform(0.1, 1, [2, 1, 50]).astype("float32"), } self.outputs = {'Out': np.divide(self.inputs['X'], self.inputs['Y'])} class TestTestElementwiseDiv_broadcast_5(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 4, 20]).astype("float32"), - 'Y': np.random.uniform(0.1, 1, [2, 3, 1, 20]).astype("float32") + 'Y': np.random.uniform(0.1, 1, [2, 3, 1, 20]).astype("float32"), } self.outputs = {'Out': np.divide(self.inputs['X'], self.inputs['Y'])} class TestTestElementwiseDiv_commonuse_1(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" @@ -234,7 +234,6 @@ def setUp(self): class TestTestElementwiseDiv_commonuse_2(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" @@ -246,7 +245,6 @@ def setUp(self): class TestTestElementwiseDiv_xsize_lessthan_ysize(TestElementwiseDiv): - def setUp(self): self.set_mlu() self.op_type = "elementwise_div" diff --git a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_max_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_max_op_mlu.py index 4fc1533b1bced..cdc354acdcba5 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_max_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_max_op_mlu.py @@ -41,7 +41,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_x)) or ( - shape_out[ax] > 1 and shape_x[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_x[ax - src_axis] == 1 + ): reduce_axes_x.append(ax) if shape_y != shape_out: @@ -52,7 +53,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_y)) or ( - shape_out[ax] > 1 and shape_y[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_y[ax - src_axis] == 1 + ): reduce_axes_y.append(ax) if len(reduce_axes_x) > 0: @@ -79,7 +81,6 @@ def ComputeGrad(x, y, out, axis): class TestElementwiseMaxOp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "elementwise_max" @@ -90,7 +91,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis} self.outputs = {'Out': self.out} @@ -106,7 +107,8 @@ def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) sgn = np.random.choice([-1, 1], [13, 17]).astype(self.dtype) self.y = self.x + sgn * np.random.uniform(0.1, 1, [13, 17]).astype( - self.dtype) + self.dtype + ) self.out = np.maximum(self.x, self.y) def init_axis(self): @@ -117,9 +119,9 @@ def test_check_output(self): def test_check_grad_normal(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=0.5) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', max_relative_error=0.5 + ) else: self.check_grad_with_place( self.place, @@ -129,10 +131,13 @@ def test_check_grad_normal(self): def test_check_grad_ingore_x(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - max_relative_error=0.9) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + max_relative_error=0.9, + ) else: self.check_grad_with_place( self.place, @@ -143,10 +148,13 @@ def test_check_grad_ingore_x(self): def test_check_grad_ingore_y(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - max_relative_error=0.1) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + max_relative_error=0.1, + ) else: self.check_grad_with_place( self.place, @@ -157,7 +165,6 @@ def test_check_grad_ingore_y(self): class TestElementwiseMaxOp_int32(TestElementwiseMaxOp): - def init_dtype(self): self.dtype = np.int32 @@ -173,15 +180,14 @@ def test_check_grad_ingore_y(self): class TestElementwiseMaxOp_FP16(TestElementwiseMaxOp): - def init_dtype(self): self.dtype = np.float16 @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMaxOp_scalar(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.random_integers(-5, 5, [2, 3, 20]).astype(self.dtype) self.y = np.array([0.5]).astype(self.dtype) @@ -189,22 +195,22 @@ def init_input_output(self): class TestElementwiseMaxOp_vector(TestElementwiseMaxOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x + sgn * np.random.uniform(0.1, 1, - (100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x + sgn * np.random.uniform(0.1, 1, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y) class TestElementwiseMaxOp_broadcast_0(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (100, 5, 2)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[:, 0, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[:, 0, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y.reshape(100, 1, 1)) def init_axis(self): @@ -212,12 +218,12 @@ def init_axis(self): class TestElementwiseMaxOp_broadcast_1(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 100, 3)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[0, :, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[0, :, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y.reshape(1, 100, 1)) def init_axis(self): @@ -225,56 +231,68 @@ def init_axis(self): def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwiseMaxOp_broadcast_2(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 100)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[0, 0, :] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[0, 0, :] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y.reshape(1, 1, 100)) def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwiseMaxOp_broadcast_3(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 50, 2, 1)).astype(self.dtype) sgn = np.random.choice([-1, 1], (50, 2)).astype(self.dtype) - self.y = self.x[0, :, :, 0] + sgn * \ - np.random.uniform(1, 2, (50, 2)).astype(self.dtype) + self.y = self.x[0, :, :, 0] + sgn * np.random.uniform( + 1, 2, (50, 2) + ).astype(self.dtype) self.out = np.maximum(self.x, self.y.reshape(1, 50, 2, 1)) def init_axis(self): @@ -282,27 +300,26 @@ def init_axis(self): class TestElementwiseMaxOp_broadcast_4(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 4, 5)).astype(self.dtype) sgn = np.random.choice([-1, 1], (2, 3, 1, 5)).astype(self.dtype) - self.y = self.x + sgn * \ - np.random.uniform(1, 2, (2, 3, 1, 5)).astype(self.dtype) + self.y = self.x + sgn * np.random.uniform(1, 2, (2, 3, 1, 5)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y) class TestElementwiseMaxOp_broadcast_5(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 4, 5)).astype(self.dtype) sgn = np.random.choice([-1, 1], (2, 3, 1, 1)).astype(self.dtype) - self.y = self.x + sgn * \ - np.random.uniform(1, 2, (2, 3, 1, 1)).astype(self.dtype) + self.y = self.x + sgn * np.random.uniform(1, 2, (2, 3, 1, 1)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y) class TestElementwiseMaxNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -317,9 +334,9 @@ def _test(self, run_mlu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.maximum(a, b) @@ -342,16 +359,17 @@ def _test(self, run_mlu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_min_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_min_op_mlu.py index 31e22904bdeee..f1546b5ac63e3 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_min_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_min_op_mlu.py @@ -28,7 +28,6 @@ class TestElementwiseMinOp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "elementwise_min" @@ -36,7 +35,7 @@ def setUp(self): self.init_input_output() self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {'axis': self.axis} @@ -52,7 +51,8 @@ def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) self.sgn = np.random.choice([-1, 1], [13, 17]).astype(self.dtype) self.y = self.x + self.sgn * np.random.uniform(0.1, 1, [13, 17]).astype( - self.dtype) + self.dtype + ) self.out = np.minimum(self.x, self.y) self.axis = -1 @@ -64,9 +64,9 @@ def test_check_output(self): def test_check_grad_normal(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=0.5) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', max_relative_error=0.5 + ) else: self.check_grad_with_place( self.place, @@ -76,10 +76,13 @@ def test_check_grad_normal(self): def test_check_grad_ingore_x(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - max_relative_error=0.9) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + max_relative_error=0.9, + ) else: self.check_grad_with_place( self.place, @@ -90,10 +93,13 @@ def test_check_grad_ingore_x(self): def test_check_grad_ingore_y(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - max_relative_error=0.1) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + max_relative_error=0.1, + ) else: self.check_grad_with_place( self.place, @@ -104,32 +110,30 @@ def test_check_grad_ingore_y(self): class TestElementwiseMinOpFp16(TestElementwiseMinOp): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMinOp_Vector(TestElementwiseMinOp): - def init_input_output(self): - self.x = np.random.uniform(1, 2, (100, )).astype(self.dtype) - self.sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x + self.sgn * np.random.uniform(0.1, 1, (100, )).astype( - self.dtype) + self.x = np.random.uniform(1, 2, (100,)).astype(self.dtype) + self.sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x + self.sgn * np.random.uniform(0.1, 1, (100,)).astype( + self.dtype + ) self.out = np.minimum(self.x, self.y) self.axis = -1 class TestElementwiseMinOpFp16_Vector(TestElementwiseMinOp_Vector): - def init_dtype(self): self.dtype = np.float16 @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMinOp_scalar(TestElementwiseMinOp): - def init_input_output(self): self.x = np.random.random_integers(-5, 5, [10, 3, 4]).astype(self.dtype) self.y = np.array([0.5]).astype(self.dtype) @@ -138,32 +142,30 @@ def init_input_output(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMinOpFp16_scalar(TestElementwiseMinOp_scalar): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMinOp_broadcast(TestElementwiseMinOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 100)).astype(self.dtype) - self.sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[0, 0, :] + self.sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + self.sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[0, 0, :] + self.sgn * np.random.uniform( + 1, 2, (100,) + ).astype(self.dtype) self.out = np.minimum(self.x, self.y.reshape(1, 1, 100)) self.axis = -1 class TestElementwiseMinOpFp16_broadcast(TestElementwiseMinOp_broadcast): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMinOpNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -178,9 +180,9 @@ def _test(self, run_mlu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.minimum(a, b) @@ -203,16 +205,17 @@ def _test(self, run_mlu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_mul_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_mul_op_mlu.py index fc5e8e89254b2..cead808d42ca4 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_mul_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_mul_op_mlu.py @@ -30,7 +30,6 @@ class ElementwiseMulOp(OpTest): - def init_kernel_type(self): self.__class__.use_mlu = True self.place = paddle.device.MLUPlace(0) @@ -46,7 +45,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {'axis': self.axis} @@ -58,14 +57,14 @@ def test_check_grad_normal(self): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X")) + self.check_grad_with_place( + self.place, ['Y'], 'Out', no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set('Y')) + self.check_grad_with_place( + self.place, ['X'], 'Out', no_grad_set=set('Y') + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -80,33 +79,31 @@ def init_axis(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMulOp_scalar(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 3, 4).astype(np.float32), - 'Y': np.random.rand(1).astype(np.float32) + 'Y': np.random.rand(1).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_Vector(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { - 'X': np.random.random((100, )).astype("float32"), - 'Y': np.random.random((100, )).astype("float32") + 'X': np.random.random((100,)).astype("float32"), + 'Y': np.random.random((100,)).astype("float32"), } self.outputs = {'Out': np.multiply(self.inputs['X'], self.inputs['Y'])} self.init_kernel_type() class TestElementwiseMulOp_broadcast_0(ElementwiseMulOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -117,12 +114,11 @@ def init_axis(self): class TestElementwiseMulOp_broadcast_1(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 100, 3).astype(np.float32), - 'Y': np.random.rand(100).astype(np.float32) + 'Y': np.random.rand(100).astype(np.float32), } self.attrs = {'axis': 1} @@ -133,12 +129,11 @@ def setUp(self): class TestElementwiseMulOp_broadcast_2(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float32), - 'Y': np.random.rand(100).astype(np.float32) + 'Y': np.random.rand(100).astype(np.float32), } self.outputs = { @@ -148,12 +143,11 @@ def setUp(self): class TestElementwiseMulOp_broadcast_3(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 10, 12, 3).astype(np.float32), - 'Y': np.random.rand(10, 12).astype(np.float32) + 'Y': np.random.rand(10, 12).astype(np.float32), } self.attrs = {'axis': 1} @@ -164,66 +158,60 @@ def setUp(self): class TestElementwiseMulOp_broadcast_4(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 2, 11).astype(np.float32), - 'Y': np.random.rand(10, 1, 11).astype(np.float32) + 'Y': np.random.rand(10, 1, 11).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_broadcast_5(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 4, 2, 3).astype(np.float32), - 'Y': np.random.rand(10, 4, 1, 3).astype(np.float32) + 'Y': np.random.rand(10, 4, 1, 3).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOpFp16(ElementwiseMulOp): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMulOp_commonuse_1(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float32), - 'Y': np.random.rand(1, 1, 100).astype(np.float32) + 'Y': np.random.rand(1, 1, 100).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_commonuse_2(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(30, 3, 1, 5).astype(np.float32), - 'Y': np.random.rand(30, 1, 4, 1).astype(np.float32) + 'Y': np.random.rand(30, 1, 4, 1).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_xsize_lessthan_ysize(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 10).astype(np.float32), - 'Y': np.random.rand(2, 2, 10, 10).astype(np.float32) + 'Y': np.random.rand(2, 2, 10, 10).astype(np.float32), } self.attrs = {'axis': 2} @@ -235,14 +223,15 @@ def setUp(self): class TestElementwiseMulOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_mul must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.elementwise_mul, x1, y1) # the input dtype of elementwise_mul must be float16 or float32 or int32 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_pow_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_pow_op_mlu.py index 300323dba05fe..7215ce6f9ee9b 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_pow_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_pow_op_mlu.py @@ -42,7 +42,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_x)) or ( - shape_out[ax] > 1 and shape_x[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_x[ax - src_axis] == 1 + ): reduce_axes_x.append(ax) if shape_y != shape_out: @@ -53,7 +54,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_y)) or ( - shape_out[ax] > 1 and shape_y[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_y[ax - src_axis] == 1 + ): reduce_axes_y.append(ax) if len(reduce_axes_x) > 0: @@ -79,7 +81,6 @@ def ComputeGrad(x, y, out, axis): class TestElementwisePow(OpTest): - def setUp(self): self.set_mlu() self.op_type = "elementwise_pow" @@ -90,7 +91,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis} self.outputs = {'Out': self.out} @@ -116,27 +117,32 @@ def init_input_output(self): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwisePowFp16(TestElementwisePow): - def init_input_output(self): np.random.seed(SEED) self.x = np.random.uniform(1, 2, [11, 17]).astype(self.dtype) @@ -156,7 +162,6 @@ def test_check_output(self): class TestElementwisePowOp_broadcast_0(TestElementwisePow): - def init_axis(self): self.axis = 1 @@ -168,27 +173,32 @@ def init_input_output(self): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwisePowOp_broadcast_1(TestElementwisePow): - def init_axis(self): self.axis = 1 @@ -200,27 +210,32 @@ def init_input_output(self): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwisePowOp_broadcast_2(TestElementwisePow): - def init_axis(self): self.axis = 0 @@ -232,23 +247,29 @@ def init_input_output(self): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_sub_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_sub_op_mlu.py index 4062cbeaf95f4..5b2d0a4358833 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_sub_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_sub_op_mlu.py @@ -27,7 +27,6 @@ class TestElementwiseSubOp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" @@ -37,7 +36,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis} self.outputs = {'Out': self.out} @@ -64,52 +63,56 @@ def test_check_grad_normal(self): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + max_relative_error=0.005, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y')) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + max_relative_error=0.005, + no_grad_set=set('Y'), + ) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseSubOp_scalar(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(10, 3, 4).astype(np.float32), - 'Y': np.random.rand(1).astype(np.float32) + 'Y': np.random.rand(1).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_Vector(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { - 'X': np.random.random((100, )).astype("float32"), - 'Y': np.random.random((100, )).astype("float32") + 'X': np.random.random((100,)).astype("float32"), + 'Y': np.random.random((100,)).astype("float32"), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_broadcast_0(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(100, 3, 2).astype(np.float32), - 'Y': np.random.rand(100).astype(np.float32) + 'Y': np.random.rand(100).astype(np.float32), } self.attrs = {'axis': 0} self.outputs = { @@ -118,13 +121,12 @@ def setUp(self): class TestElementwiseSubOp_broadcast_1(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 100, 3).astype(np.float32), - 'Y': np.random.rand(100).astype(np.float32) + 'Y': np.random.rand(100).astype(np.float32), } self.attrs = {'axis': 1} self.outputs = { @@ -133,13 +135,12 @@ def setUp(self): class TestElementwiseSubOp_broadcast_2(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float32), - 'Y': np.random.rand(100).astype(np.float32) + 'Y': np.random.rand(100).astype(np.float32), } self.outputs = { 'Out': self.inputs['X'] - self.inputs['Y'].reshape(1, 1, 100) @@ -147,13 +148,12 @@ def setUp(self): class TestElementwiseSubOp_broadcast_3(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 10, 12, 3).astype(np.float32), - 'Y': np.random.rand(10, 12).astype(np.float32) + 'Y': np.random.rand(10, 12).astype(np.float32), } self.attrs = {'axis': 1} self.outputs = { @@ -162,49 +162,45 @@ def setUp(self): class TestElementwiseSubOp_broadcast_4(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 5, 3, 12).astype(np.float32), - 'Y': np.random.rand(2, 5, 1, 12).astype(np.float32) + 'Y': np.random.rand(2, 5, 1, 12).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_commonuse_1(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float32), - 'Y': np.random.rand(1, 1, 100).astype(np.float32) + 'Y': np.random.rand(1, 1, 100).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_commonuse_2(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(10, 3, 1, 4).astype(np.float32), - 'Y': np.random.rand(10, 1, 12, 1).astype(np.float32) + 'Y': np.random.rand(10, 1, 12, 1).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_xsize_lessthan_ysize(TestElementwiseSubOp): - def setUp(self): self.set_mlu() self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(10, 12).astype(np.float32), - 'Y': np.random.rand(2, 3, 10, 12).astype(np.float32) + 'Y': np.random.rand(2, 3, 10, 12).astype(np.float32), } self.attrs = {'axis': 2} self.outputs = { diff --git a/python/paddle/fluid/tests/unittests/mlu/test_exp_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_exp_op_mlu.py index b2d77a4c85ff0..e4c6192138655 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_exp_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_exp_op_mlu.py @@ -26,7 +26,6 @@ class TestExp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "exp" @@ -55,7 +54,6 @@ def test_check_grad(self): class TestExpFp16(OpTest): - def setUp(self): self.set_mlu() self.op_type = "exp" @@ -82,7 +80,6 @@ def test_check_output(self): class TestExpNeg(OpTest): - def setUp(self): self.set_mlu() self.op_type = "exp" diff --git a/python/paddle/fluid/tests/unittests/mlu/test_expand_as_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_expand_as_v2_op_mlu.py index 5143a5f83a7da..0a88f759831e4 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_expand_as_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_expand_as_v2_op_mlu.py @@ -25,9 +25,7 @@ def test_class1(op_type, typename): - class TestExpandAsBasic(OpTest): - def setUp(self): self.set_mlu() self.op_type = "expand_as_v2" @@ -54,9 +52,7 @@ def test_check_output(self): def test_class2(op_type, typename): - class TestExpandAsOpRank2(OpTest): - def setUp(self): self.set_mlu() self.op_type = "expand_as_v2" @@ -83,9 +79,7 @@ def test_check_output(self): def test_class3(op_type, typename): - class TestExpandAsOpRank3(OpTest): - def setUp(self): self.set_mlu() self.op_type = "expand_as_v2" @@ -112,9 +106,7 @@ def test_check_output(self): def test_class4(op_type, typename): - class TestExpandAsOpRank4(OpTest): - def setUp(self): self.set_mlu() self.op_type = "expand_as_v2" @@ -142,34 +134,39 @@ def test_check_output(self): # Test python API class TestExpandAsV2API(unittest.TestCase): - def test_api(self): input1 = np.random.random([12, 14]).astype("float32") input2 = np.random.random([2, 12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) - y = fluid.layers.data(name='target_tensor', - shape=[2, 12, 14], - append_batch_size=False, - dtype="float32") + y = fluid.layers.data( + name='target_tensor', + shape=[2, 12, 14], + append_batch_size=False, + dtype="float32", + ) out_1 = paddle.expand_as(x, y=y) exe = fluid.Executor(place=fluid.MLUPlace(0)) - res_1 = exe.run(fluid.default_main_program(), - feed={ - "x": input1, - "target_tensor": input2 - }, - fetch_list=[out_1]) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": input1, "target_tensor": input2}, + fetch_list=[out_1], + ) assert np.array_equal(res_1[0], np.tile(input1, (2, 1, 1))) for _typename in { - 'float16', 'float32', 'int64', 'int32', 'int8', 'uint8', 'bool' + 'float16', + 'float32', + 'int64', + 'int32', + 'int8', + 'uint8', + 'bool', }: test_class1('expand_as_v2', _typename) test_class2('expand_as_v2', _typename) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_expand_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_expand_v2_op_mlu.py index 527a9bd2e55fc..f1e8b06cd9ab6 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_expand_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_expand_v2_op_mlu.py @@ -28,7 +28,6 @@ # Situation 1: shape is a list(without tensor) class TestExpandV2OpRank1(OpTest): - def setUp(self): self.op_type = "expand_v2" self.place = paddle.device.MLUPlace(0) @@ -54,7 +53,6 @@ def test_check_grad(self): class TestExpandV2OpRank2_DimExpanding(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = [120] self.shape = [2, 120] @@ -62,7 +60,6 @@ def init_data(self): class TestExpandV2OpRank2(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = [1, 140] self.shape = [12, 140] @@ -70,7 +67,6 @@ def init_data(self): class TestExpandV2OpRank3_Corner(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.shape = (2, 10, 5) @@ -78,7 +74,6 @@ def init_data(self): class TestExpandV2OpRank4(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.shape = (-1, -1, -1, -1) @@ -86,7 +81,6 @@ def init_data(self): class TestExpandV2OpRank5(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = (2, 4, 1, 15) self.shape = (2, -1, 4, -1) @@ -94,7 +88,6 @@ def init_data(self): class TestExpandV2OpRank6(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = (4, 1, 30) self.shape = (2, -1, 4, 30) @@ -103,7 +96,6 @@ def init_data(self): # Situation 2: shape is a list(with tensor) class TestExpandV2OpRank1_tensor_attr(OpTest): - def setUp(self): self.op_type = "expand_v2" self.place = paddle.device.MLUPlace(0) @@ -111,8 +103,9 @@ def setUp(self): self.init_data() expand_shapes_tensor = [] for index, ele in enumerate(self.expand_shape): - expand_shapes_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + expand_shapes_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype("float32"), @@ -136,7 +129,6 @@ def test_check_grad(self): class TestExpandV2OpRank2_Corner_tensor_attr(TestExpandV2OpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.expand_times = [1, 1] @@ -146,7 +138,6 @@ def init_data(self): # Situation 3: shape is a tensor class TestExpandV2OpRank1_tensor(OpTest): - def setUp(self): self.op_type = "expand_v2" self.place = paddle.device.MLUPlace(0) @@ -175,7 +166,6 @@ def test_check_grad(self): # Situation 4: input x is Integer class TestExpandV2OpInteger(OpTest): - def setUp(self): self.op_type = "expand_v2" self.place = paddle.device.MLUPlace(0) @@ -193,7 +183,6 @@ def test_check_output(self): # Situation 5: input x is Bool class TestExpandV2OpBoolean(OpTest): - def setUp(self): self.op_type = "expand_v2" self.place = paddle.device.MLUPlace(0) @@ -209,7 +198,6 @@ def test_check_output(self): # Situation 56: input x is Integer class TestExpandV2OpInt64_t(OpTest): - def setUp(self): self.op_type = "expand_v2" self.place = paddle.device.MLUPlace(0) @@ -226,11 +214,11 @@ def test_check_output(self): class TestExpandV2Error(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - paddle.device.MLUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], paddle.device.MLUPlace(0) + ) shape = [2, 2] self.assertRaises(TypeError, paddle.tensor.expand, x1, shape) x2 = fluid.layers.data(name='x2', shape=[4], dtype="uint8") @@ -242,19 +230,19 @@ def test_errors(self): # Test python API class TestExpandV2API(unittest.TestCase): - def test_api(self): input = np.random.random([12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) positive_2 = fluid.layers.fill_constant([1], "int32", 12) - expand_shape = fluid.layers.data(name="expand_shape", - shape=[2], - append_batch_size=False, - dtype="int32") + expand_shape = fluid.layers.data( + name="expand_shape", + shape=[2], + append_batch_size=False, + dtype="int32", + ) out_1 = paddle.expand(x, shape=[12, 14]) out_2 = paddle.expand(x, shape=[positive_2, 14]) @@ -263,28 +251,28 @@ def test_api(self): g0 = fluid.backward.calc_gradient(out_2, x) exe = fluid.Executor(place=paddle.device.MLUPlace(0)) - res_1, res_2, res_3 = exe.run(fluid.default_main_program(), - feed={ - "x": - input, - "expand_shape": - np.array([12, 14]).astype("int32") - }, - fetch_list=[out_1, out_2, out_3]) + res_1, res_2, res_3 = exe.run( + fluid.default_main_program(), + feed={ + "x": input, + "expand_shape": np.array([12, 14]).astype("int32"), + }, + fetch_list=[out_1, out_2, out_3], + ) assert np.array_equal(res_1, np.tile(input, (1, 1))) assert np.array_equal(res_2, np.tile(input, (1, 1))) assert np.array_equal(res_3, np.tile(input, (1, 1))) class TestExpandInferShape(unittest.TestCase): - def test_shape_with_var(self): with program_guard(Program(), Program()): x = paddle.static.data(shape=[-1, 1, 3], name='x') fake_var = paddle.randn([2, 3]) target_shape = [ - -1, paddle.shape(fake_var)[0], - paddle.shape(fake_var)[1] + -1, + paddle.shape(fake_var)[0], + paddle.shape(fake_var)[1], ] out = paddle.expand(x, shape=target_shape) self.assertListEqual(list(out.shape), [-1, -1, -1]) @@ -292,7 +280,6 @@ def test_shape_with_var(self): # Test python Dygraph API class TestExpandV2DygraphAPI(unittest.TestCase): - def test_expand_times_is_tensor(self): with paddle.fluid.dygraph.guard(): paddle.seed(1) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_fill_any_like_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_fill_any_like_op_mlu.py index 249aa48e3d8ab..021f45aee0698 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_fill_any_like_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_fill_any_like_op_mlu.py @@ -25,7 +25,6 @@ class TestFillAnyLikeOp(OpTest): - def setUp(self): self.init_dtype() self.set_mlu() @@ -54,25 +53,21 @@ def test_check_output(self): class TestFillAnyLikeOp2(TestFillAnyLikeOp): - def set_value(self): self.value = -0.0 class TestFillAnyLikeOp3(TestFillAnyLikeOp): - def set_value(self): self.value = 1.0 class TestFillAnyLikeOp4(TestFillAnyLikeOp): - def set_value(self): self.value = 1e-9 class TestFillAnyLikeOp5(TestFillAnyLikeOp): - def set_value(self): if self.dtype == "float16": self.value = 0.05 @@ -81,7 +76,6 @@ def set_value(self): class TestFillAnyLikeOpInt32(TestFillAnyLikeOp): - def init_dtype(self): self.dtype = np.int32 @@ -90,7 +84,6 @@ def set_value(self): class TestFillAnyLikeOpInt64(TestFillAnyLikeOp): - def init_dtype(self): self.dtype = np.int64 @@ -99,7 +92,6 @@ def set_value(self): class TestFillAnyLikeOpFloat32(TestFillAnyLikeOp): - def init_dtype(self): self.dtype = np.float32 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_batch_size_like_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_batch_size_like_op_mlu.py index ba5983fbb1e01..79b5d7abe7d82 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_batch_size_like_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_batch_size_like_op_mlu.py @@ -27,20 +27,21 @@ paddle.enable_static() -def fill_constant_batch_size_like(input, - shape, - value, - data_type, - input_dim_idx=0, - output_dim_idx=0, - force_cpu=False): +def fill_constant_batch_size_like( + input, + shape, + value, + data_type, + input_dim_idx=0, + output_dim_idx=0, + force_cpu=False, +): return paddle.fluid.layers.fill_constant_batch_size_like( - input, shape, data_type, value, input_dim_idx, output_dim_idx, - force_cpu) + input, shape, data_type, value, input_dim_idx, output_dim_idx, force_cpu + ) class TestFillConstantBatchSizeLike(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -61,11 +62,12 @@ def setUp(self): 'dtype': self.dtype, 'force_cpu': self.force_cpu, 'input_dim_idx': self.input_dim_idx, - 'output_dim_idx': self.output_dim_idx + 'output_dim_idx': self.output_dim_idx, } self.outputs = { - 'Out': np.full(self.output_shape, self.output_value, - self.output_dtype) + 'Out': np.full( + self.output_shape, self.output_value, self.output_dtype + ) } def init_shape(self): @@ -94,7 +96,6 @@ def test_check_output(self): class TestFillConstantBatchSizeLike2(TestFillConstantBatchSizeLike): - def init_shape(self): # test shape self.input_shape = [4, 5, 6, 7] @@ -103,7 +104,6 @@ def init_shape(self): class TestFillConstantBatchSizeLike3(TestFillConstantBatchSizeLike): - def init_value(self): # use 'str_value' rather than 'value' self.value = 3.8 @@ -112,7 +112,6 @@ def init_value(self): class TestFillConstantBatchSizeLike4(TestFillConstantBatchSizeLike): - def init_value(self): # str_value = 'inf' self.value = 3.8 @@ -121,7 +120,6 @@ def init_value(self): class TestFillConstantBatchSizeLike5(TestFillConstantBatchSizeLike): - def init_value(self): # str_value = '-inf' self.value = 3.8 @@ -130,7 +128,6 @@ def init_value(self): class TestFillConstantBatchSizeLike6(TestFillConstantBatchSizeLike): - def init_dtype(self): self.dtype = core.VarDesc.VarType.FP16 self.output_dtype = np.float16 @@ -140,20 +137,17 @@ def test_check_output(self): class TestFillConstantBatchSizeLike7(TestFillConstantBatchSizeLike): - def init_dtype(self): self.dtype = core.VarDesc.VarType.INT32 self.output_dtype = np.int32 class TestFillConstantBatchSizeLike8(TestFillConstantBatchSizeLike): - def init_force_cpu(self): self.force_cpu = True class TestFillConstantBatchSizeLike9(TestFillConstantBatchSizeLike): - def init_shape(self): self.input_shape = [4, 5] self.shape = [123, 92] @@ -187,11 +181,12 @@ def setUp(self): 'dtype': self.dtype, 'force_cpu': self.force_cpu, 'input_dim_idx': self.input_dim_idx, - 'output_dim_idx': self.output_dim_idx + 'output_dim_idx': self.output_dim_idx, } self.outputs = { - 'Out': np.full(self.output_shape, self.output_value, - self.output_dtype) + 'Out': np.full( + self.output_shape, self.output_value, self.output_dtype + ) } def init_shape(self): @@ -201,7 +196,8 @@ def init_shape(self): class TestFillConstantBatchSizeLikeLodTensor2( - TestFillConstantBatchSizeLikeLodTensor): + TestFillConstantBatchSizeLikeLodTensor +): # test LodTensor with 'input_dim_idx' != 0 def init_shape(self): self.input_shape = [10, 20] diff --git a/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_op_mlu.py index b05f720b5f10c..7a19779b049ce 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_fill_constant_op_mlu.py @@ -31,10 +31,8 @@ # Situation 1: Attr(shape) is a list(without tensor) class TestFillConstantOp1(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.inputs = {} @@ -49,10 +47,8 @@ def test_check_output(self): class TestFillConstantOp2(OpTest): - def setUp(self): - '''Test fill_constant op with default value - ''' + '''Test fill_constant op with default value''' self.op_type = "fill_constant" self.inputs = {} @@ -67,10 +63,8 @@ def test_check_output(self): class TestFillConstantOp3(OpTest): - def setUp(self): - '''Test fill_constant op with specified int64 value - ''' + '''Test fill_constant op with specified int64 value''' self.op_type = "fill_constant" self.inputs = {} @@ -85,10 +79,8 @@ def test_check_output(self): class TestFillConstantOp4(OpTest): - def setUp(self): - '''Test fill_constant op with specified int value - ''' + '''Test fill_constant op with specified int value''' self.op_type = "fill_constant" self.inputs = {} @@ -103,17 +95,15 @@ def test_check_output(self): class TestFillConstantOpWithSelectedRows(unittest.TestCase): - def check_with_place(self, place): scope = core.Scope() # create Out Variable out = scope.var('Out').get_selected_rows() # create and run fill_constant_op operator - fill_constant_op = Operator("fill_constant", - shape=[123, 92], - value=3.8, - Out='Out') + fill_constant_op = Operator( + "fill_constant", shape=[123, 92], value=3.8, Out='Out' + ) fill_constant_op.run(scope, place) # get result from Out @@ -133,16 +123,15 @@ def test_fill_constant_with_selected_rows(self): # Situation 2: Attr(shape) is a list(with tensor) class TestFillConstantOp1_ShapeTensorList(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} self.attrs = {'shape': self.infer_shape, 'value': self.value} @@ -161,16 +150,15 @@ def test_check_output(self): class TestFillConstantOp2_ShapeTensorList(OpTest): - def setUp(self): - '''Test fill_constant op with default value - ''' + '''Test fill_constant op with default value''' self.op_type = "fill_constant" self.init_data() shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} self.attrs = {'shape': self.infer_shape} @@ -188,7 +176,6 @@ def test_check_output(self): class TestFillConstantOp3_ShapeTensorList(TestFillConstantOp1_ShapeTensorList): - def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -196,7 +183,6 @@ def init_data(self): class TestFillConstantOp4_ShapeTensorList(TestFillConstantOp1_ShapeTensorList): - def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -205,10 +191,8 @@ def init_data(self): # Situation 3: shape is a tensor class TestFillConstantOp1_ShapeTensor(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() @@ -229,16 +213,14 @@ def test_check_output(self): # Situation 4: value is a tensor class TestFillConstantOp1_ValueTensor(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() self.inputs = { "ShapeTensor": np.array(self.shape).astype("int32"), - 'ValueTensor': np.array([self.value]).astype("float32") + 'ValueTensor': np.array([self.value]).astype("float32"), } self.attrs = {'value': self.value + 1.0} self.outputs = {'Out': np.full(self.shape, self.value)} @@ -247,7 +229,7 @@ def setUp(self): self.__class__.use_mlu = True def init_data(self): - #self.shape = [123, 92] + # self.shape = [123, 92] self.shape = [2, 2] self.value = 3.8 self.dtype = np.float32 @@ -258,16 +240,14 @@ def test_check_output(self): # Situation 5: value is a tensor class TestFillConstantOp2_ValueTensor(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() self.inputs = { "ShapeTensor": np.array(self.shape).astype("int32"), - 'ValueTensor': np.array([self.value]).astype("int32") + 'ValueTensor': np.array([self.value]).astype("int32"), } self.attrs = {'value': self.value, 'dtype': 2} self.outputs = {'Out': np.full(self.shape, self.value)} @@ -286,56 +266,55 @@ def test_check_output(self): # Test python API class TestFillConstantAPI(unittest.TestCase): - def test_api(self): positive_2_int32 = fluid.layers.fill_constant([1], "int32", 2) positive_2_int64 = fluid.layers.fill_constant([1], "int64", 2) - shape_tensor_int32 = fluid.data(name="shape_tensor_int32", - shape=[2], - dtype="int32") - shape_tensor_int64 = fluid.data(name="shape_tensor_int64", - shape=[2], - dtype="int64") - - out_1 = fluid.layers.fill_constant(shape=[1, 2], - dtype="float32", - value=1.1) - - out_2 = fluid.layers.fill_constant(shape=[1, positive_2_int32], - dtype="float32", - value=1.1) - - out_3 = fluid.layers.fill_constant(shape=[1, positive_2_int64], - dtype="float32", - value=1.1) - - out_4 = fluid.layers.fill_constant(shape=shape_tensor_int32, - dtype="float32", - value=1.1) - - out_5 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype="float32", - value=1.1) - - out_6 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype=np.float32, - value=1.1) - - val1 = fluid.layers.fill_constant(shape=[1], - dtype=np.float32, - value=1.1) - val2 = fluid.layers.fill_constant(shape=[1], - dtype=np.float64, - value=1.1) - out_7 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype=np.float32, - value=val1) - - out_8 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype=np.float32, - value=val2) + shape_tensor_int32 = fluid.data( + name="shape_tensor_int32", shape=[2], dtype="int32" + ) + shape_tensor_int64 = fluid.data( + name="shape_tensor_int64", shape=[2], dtype="int64" + ) + + out_1 = fluid.layers.fill_constant( + shape=[1, 2], dtype="float32", value=1.1 + ) + + out_2 = fluid.layers.fill_constant( + shape=[1, positive_2_int32], dtype="float32", value=1.1 + ) + + out_3 = fluid.layers.fill_constant( + shape=[1, positive_2_int64], dtype="float32", value=1.1 + ) + + out_4 = fluid.layers.fill_constant( + shape=shape_tensor_int32, dtype="float32", value=1.1 + ) + + out_5 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype="float32", value=1.1 + ) + + out_6 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype=np.float32, value=1.1 + ) + + val1 = fluid.layers.fill_constant( + shape=[1], dtype=np.float32, value=1.1 + ) + val2 = fluid.layers.fill_constant( + shape=[1], dtype=np.float64, value=1.1 + ) + out_7 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype=np.float32, value=val1 + ) + + out_8 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype=np.float32, value=val2 + ) exe = fluid.Executor(place=fluid.CPUPlace()) res_1, res_2, res_3, res_4, res_5, res_6, res_7, res_8 = exe.run( @@ -344,7 +323,8 @@ def test_api(self): "shape_tensor_int32": np.array([1, 2]).astype("int32"), "shape_tensor_int64": np.array([1, 2]).astype("int64"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8], + ) assert np.array_equal(res_1, np.full([1, 2], 1.1, dtype="float32")) assert np.array_equal(res_2, np.full([1, 2], 1.1, dtype="float32")) @@ -357,7 +337,6 @@ def test_api(self): class TestFillConstantImperative(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): data1 = np.array([1, 2]).astype('int32') @@ -366,26 +345,30 @@ def test_api(self): shape = fluid.dygraph.to_variable(data1) val = fluid.dygraph.to_variable(data2) value = fluid.dygraph.to_variable(data3) - res1 = fluid.layers.fill_constant(shape=[1, 2], - dtype='float32', - value=1.1) - res2 = fluid.layers.fill_constant(shape=shape, - dtype='float32', - value=1.1) - res3 = fluid.layers.fill_constant(shape=shape, - dtype='float32', - value=val) - res4 = fluid.layers.fill_constant(shape=shape, - dtype='int32', - value=value) - assert np.array_equal(res1.numpy(), - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(res2.numpy(), - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(res3.numpy(), - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(res4.numpy(), - np.full([1, 2], 88, dtype="int32")) + res1 = fluid.layers.fill_constant( + shape=[1, 2], dtype='float32', value=1.1 + ) + res2 = fluid.layers.fill_constant( + shape=shape, dtype='float32', value=1.1 + ) + res3 = fluid.layers.fill_constant( + shape=shape, dtype='float32', value=val + ) + res4 = fluid.layers.fill_constant( + shape=shape, dtype='int32', value=value + ) + assert np.array_equal( + res1.numpy(), np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + res2.numpy(), np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + res3.numpy(), np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + res4.numpy(), np.full([1, 2], 88, dtype="int32") + ) def test_nan(self): with fluid.dygraph.guard(): @@ -405,42 +388,49 @@ def test_ninf(self): class TestFillConstantOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - #for ci coverage + # for ci coverage x1 = fluid.layers.data(name='x1', shape=[1], dtype="int16") - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[1], - value=5, - dtype='uint4') - - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[1.1], - value=5, - dtype='float32', - out=x1) + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[1], + value=5, + dtype='uint4', + ) + + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[1.1], + value=5, + dtype='float32', + out=x1, + ) # The argument dtype of fill_constant_op must be one of bool, float16, - #float32, float64, uint8, int16, int32 or int64 + # float32, float64, uint8, int16, int32 or int64 x2 = fluid.layers.data(name='x2', shape=[1], dtype="int32") - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[1], - value=5, - dtype='float64', - out=x2) + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[1], + value=5, + dtype='float64', + out=x2, + ) x3 = np.random.randn(100, 100).astype('int32') - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[100, 100], - value=5, - dtype='float64', - out=x3) + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[100, 100], + value=5, + dtype='float64', + out=x3, + ) # The argument shape's type of fill_constant_op must be list, tuple or Variable. def test_shape_type(): @@ -456,22 +446,22 @@ def test_shape_size(): # The shape dtype of fill_constant_op must be int32 or int64. def test_shape_tensor_dtype(): - shape = fluid.data(name="shape_tensor", - shape=[2], - dtype="float32") - fluid.layers.fill_constant(shape=shape, - dtype="float32", - value=1) + shape = fluid.data( + name="shape_tensor", shape=[2], dtype="float32" + ) + fluid.layers.fill_constant( + shape=shape, dtype="float32", value=1 + ) self.assertRaises(TypeError, test_shape_tensor_dtype) def test_shape_tensor_list_dtype(): - shape = fluid.data(name="shape_tensor_list", - shape=[1], - dtype="bool") - fluid.layers.fill_constant(shape=[shape, 2], - dtype="float32", - value=1) + shape = fluid.data( + name="shape_tensor_list", shape=[1], dtype="bool" + ) + fluid.layers.fill_constant( + shape=[shape, 2], dtype="float32", value=1 + ) self.assertRaises(TypeError, test_shape_tensor_list_dtype) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_flatten2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_flatten2_op_mlu.py index c799fd0114358..ace458ff42c05 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_flatten2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_flatten2_op_mlu.py @@ -25,7 +25,6 @@ class TestFlattenOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -35,7 +34,7 @@ def setUp(self): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype("float32") + "XShape": np.random.random(self.in_shape).astype("float32"), } def test_check_output(self): @@ -54,7 +53,6 @@ def init_attrs(self): class TestFlattenOp1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.axis = 0 @@ -62,7 +60,6 @@ def init_test_case(self): class TestFlattenOpWithDefaultAxis(TestFlattenOp): - def init_test_case(self): self.in_shape = (10, 2, 2, 3) self.new_shape = (10, 12) @@ -72,7 +69,6 @@ def init_attrs(self): class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 @@ -80,7 +76,6 @@ def init_test_case(self): class TestStaticFlattenInferShapePythonAPI(unittest.TestCase): - def execute_api(self, x, axis=1): return fluid.layers.flatten(x, axis=axis) @@ -88,15 +83,14 @@ def test_static_api(self): paddle.enable_static() main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, paddle.static.Program()): - x = paddle.static.data(name="x", - shape=[-1, 3, -1, -1], - dtype='float32') + x = paddle.static.data( + name="x", shape=[-1, 3, -1, -1], dtype='float32' + ) out = self.execute_api(x, axis=2) self.assertTrue((-1, -1) == out.shape) class TestFlatten2OpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): input_data = np.random.random((3, 2, 4, 5)).astype("float64") @@ -109,9 +103,9 @@ def test_Variable(): def test_type(): # dtype must be float32, float64, int8, int32, int64, uint8. - x2 = fluid.layers.data(name='x2', - shape=[3, 2, 4, 5], - dtype='float16') + x2 = fluid.layers.data( + name='x2', shape=[3, 2, 4, 5], dtype='float16' + ) fluid.layers.flatten(x2, axis=1) self.assertRaises(TypeError, test_type) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_flatten_contigous_range_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_flatten_contigous_range_op_mlu.py index 1a988f6104d25..942d5f9c89764 100755 --- a/python/paddle/fluid/tests/unittests/mlu/test_flatten_contigous_range_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_flatten_contigous_range_op_mlu.py @@ -25,7 +25,6 @@ class TestFlattenOp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "flatten_contiguous_range" @@ -39,7 +38,7 @@ def setUp(self): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype("float32") + "XShape": np.random.random(self.in_shape).astype("float32"), } def set_mlu(self): @@ -49,24 +48,23 @@ def test_check_output(self): self.check_output_with_place(self.place, no_check_set=["XShape"]) def test_check_grad(self): - #pass + # pass self.check_grad_with_place(self.place, ["X"], "Out") def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 self.stop_axis = -1 - self.new_shape = (120) + self.new_shape = 120 def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 1 @@ -76,12 +74,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_2(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -91,12 +88,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_3(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -106,12 +102,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_4(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = -2 @@ -121,12 +116,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_5(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 2 @@ -136,12 +130,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.start_axis = 3 @@ -151,12 +144,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_Float32(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -167,12 +159,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_int32(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -183,7 +174,7 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -191,7 +182,6 @@ def test_check_grad(self): class TestFlattenOp_uint8(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -202,7 +192,7 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -210,7 +200,6 @@ def test_check_grad(self): class TestFlattenOp_int8(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -221,7 +210,7 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -229,7 +218,6 @@ def test_check_grad(self): class TestFlattenOp_int64(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -240,7 +228,7 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -248,45 +236,58 @@ def test_check_grad(self): class TestFlatten2OpError(unittest.TestCase): - def test_errors(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_ValueError1(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) out = paddle.flatten(x_var, start_axis=2, stop_axis=1) self.assertRaises(ValueError, test_ValueError1) def test_ValueError2(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=10, stop_axis=1) self.assertRaises(ValueError, test_ValueError2) def test_ValueError3(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=2, stop_axis=10) self.assertRaises(ValueError, test_ValueError3) def test_type(): # dtype must be float32, float64, int8, int32, int64, uint8. - x2 = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x2 = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x2 = x2.astype('float16') - x2_var = paddle.fluid.data(name='x2', - shape=[3, 2, 4, 5], - dtype='float16') + x2_var = paddle.fluid.data( + name='x2', shape=[3, 2, 4, 5], dtype='float16' + ) paddle.flatten(x2_var) self.assertRaises(TypeError, test_type) @@ -298,7 +299,6 @@ def test_InputError(): class TestStaticFlattenPythonAPI(unittest.TestCase): - def execute_api(self, x, start_axis=0, stop_axis=-1): return paddle.flatten(x, start_axis, stop_axis) @@ -308,9 +308,9 @@ def test_static_api(self): main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, paddle.static.Program()): - x = paddle.static.data(name="x", - shape=[2, 3, 4, 4], - dtype='float32') + x = paddle.static.data( + name="x", shape=[2, 3, 4, 4], dtype='float32' + ) out = self.execute_api(x, start_axis=-2, stop_axis=-1) exe = paddle.static.Executor(place=paddle.MLUPlace(0)) @@ -319,17 +319,22 @@ def test_static_api(self): class TestStaticInplaceFlattenPythonAPI(TestStaticFlattenPythonAPI): - def execute_api(self, x, start_axis=0, stop_axis=-1): return x.flatten_(start_axis, stop_axis) class TestFlattenPython(unittest.TestCase): - def test_python_api(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_InputError(): diff --git a/python/paddle/fluid/tests/unittests/mlu/test_flatten_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_flatten_op_mlu.py index d06c165e8d446..0b8da1f18f062 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_flatten_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_flatten_op_mlu.py @@ -26,7 +26,6 @@ class TestFlattenOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -52,7 +51,6 @@ def init_attrs(self): class TestFlattenOp1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 2, 10) self.axis = 0 @@ -60,7 +58,6 @@ def init_test_case(self): class TestFlattenOpWithDefaultAxis(TestFlattenOp): - def init_test_case(self): self.in_shape = (10, 2, 2, 3) self.new_shape = (10, 12) @@ -70,7 +67,6 @@ def init_attrs(self): class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_floor_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_floor_op_mlu.py index bdee19091fbdd..2e661b217da8a 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_floor_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_floor_op_mlu.py @@ -25,7 +25,6 @@ class TestFloor(OpTest): - def setUp(self): self.op_type = "floor" self.place = paddle.device.MLUPlace(0) @@ -49,7 +48,6 @@ def init_dtype(self): class TestFloorFP16(TestFloor): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_gather_nd_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_gather_nd_op_mlu.py index ad903da68bc80..3fb5634df4532 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_gather_nd_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_gather_nd_op_mlu.py @@ -26,11 +26,11 @@ def gather_nd_grad(x, index): # for TestGatherNdOpWithLowIndex - dout_shape = index.shape[:-1] + x.shape[index.shape[-1]:] + dout_shape = index.shape[:-1] + x.shape[index.shape[-1] :] numel = 1 for i in dout_shape: numel = numel * i - dout = np.full(dout_shape, 1. / numel) + dout = np.full(dout_shape, 1.0 / numel) dx = np.full_like(x, 0) index = tuple(index.reshape(-1, index.shape[-1]).T) @@ -40,7 +40,6 @@ def gather_nd_grad(x, index): def test_class1(op_type, typename): - class TestGatherNdOpWithEmptyIndex(OpTest): # Index has empty element, which means copy entire tensor @@ -51,7 +50,7 @@ def setUp(self): xnp = np.random.random((5, 20)).astype(typename) self.inputs = { 'X': xnp, - 'Index': np.array([[], []]).astype("int32") + 'Index': np.array([[], []]).astype("int32"), } self.outputs = { 'Out': np.vstack((xnp[np.newaxis, :], xnp[np.newaxis, :])) @@ -76,9 +75,7 @@ def test_check_grad(self): def test_class2(op_type, typename): - class TestGatherNdOpWithIndex1(OpTest): - def setUp(self): self.set_mlu() self.op_type = "gather_nd" @@ -106,9 +103,8 @@ def test_check_grad(self): def test_class3(op_type, typename): - class TestGatherNdOpWithLowIndex(OpTest): - #Index has low rank, X has high rank + # Index has low rank, X has high rank def setUp(self): self.set_mlu() @@ -132,9 +128,9 @@ def test_check_grad(self): if typename == "float16": self.__class__.no_need_check_grad = True else: - self.check_grad_with_place(self.place, ['X'], - 'Out', - user_defined_grads=[self.x_grad]) + self.check_grad_with_place( + self.place, ['X'], 'Out', user_defined_grads=[self.x_grad] + ) cls_name = "{0}_{1}_3".format(op_type, typename) TestGatherNdOpWithLowIndex.__name__ = cls_name @@ -142,9 +138,8 @@ def test_check_grad(self): def test_class4(op_type, typename): - class TestGatherNdOpIndex1(OpTest): - #Index has low rank, X has high rank + # Index has low rank, X has high rank def setUp(self): self.set_mlu() @@ -176,9 +171,8 @@ def test_check_grad(self): def test_class5(op_type, typename): - class TestGatherNdOpWithSameIndexAsX(OpTest): - #Index has same rank as X's rank + # Index has same rank as X's rank def setUp(self): self.set_mlu() @@ -188,7 +182,7 @@ def setUp(self): index = np.array([[1, 1], [2, 1]]).astype("int64") self.inputs = {'X': xnp, 'Index': index} - self.outputs = {'Out': xnp[tuple(index.T)]} #[25, 22] + self.outputs = {'Out': xnp[tuple(index.T)]} # [25, 22] def set_mlu(self): self.__class__.use_mlu = True @@ -209,9 +203,8 @@ def test_check_grad(self): def test_class6(op_type, typename): - class TestGatherNdOpWithHighRankSame(OpTest): - #Both Index and X have high rank, and Rank(Index) = Rank(X) + # Both Index and X have high rank, and Rank(Index) = Rank(X) def setUp(self): self.set_mlu() @@ -219,8 +212,9 @@ def setUp(self): self.python_api = paddle.gather_nd shape = (5, 2, 3, 1, 10) xnp = np.random.rand(*shape).astype(typename) - index = np.vstack([np.random.randint(0, s, size=2) - for s in shape]).T + index = np.vstack( + [np.random.randint(0, s, size=2) for s in shape] + ).T self.inputs = {'X': xnp, 'Index': index.astype("int32")} self.outputs = {'Out': xnp[tuple(index.T)]} @@ -244,9 +238,8 @@ def test_check_grad(self): def test_class7(op_type, typename): - class TestGatherNdOpWithHighRankDiff(OpTest): - #Both Index and X have high rank, and Rank(Index) < Rank(X) + # Both Index and X have high rank, and Rank(Index) < Rank(X) def setUp(self): self.set_mlu() @@ -255,7 +248,8 @@ def setUp(self): shape = (2, 3, 4, 1, 10) xnp = np.random.rand(*shape).astype(typename) index = np.vstack( - [np.random.randint(0, s, size=200) for s in shape]).T + [np.random.randint(0, s, size=200) for s in shape] + ).T index_re = index.reshape([20, 5, 2, 5]) self.inputs = {'X': xnp, 'Index': index_re.astype("int32")} @@ -279,9 +273,8 @@ def test_check_grad(self): globals()[cls_name] = TestGatherNdOpWithHighRankDiff -#Test Python API +# Test Python API class TestGatherNdAPI2(unittest.TestCase): - def test_imperative(self): paddle.disable_static() input_1 = np.array([[1, 2], [3, 4], [5, 6]]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_gather_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_gather_op_mlu.py index 32881fe342722..2f50c10a62f26 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_gather_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_gather_op_mlu.py @@ -34,7 +34,6 @@ def gather_numpy(x, index, axis): class TestGatherOp(OpTest): - def setUp(self): self.op_type = "gather" self.place = paddle.MLUPlace(0) @@ -44,7 +43,7 @@ def setUp(self): xnp = np.random.random(self.x_shape).astype(self.x_type) self.inputs = { 'X': xnp, - 'Index': np.array(self.index).astype(self.index_type) + 'Index': np.array(self.index).astype(self.index_type), } self.outputs = {'Out': self.inputs["X"][self.inputs["Index"]]} @@ -65,31 +64,28 @@ def config(self): class TestCase1(TestGatherOp): - def config(self): """ For one dimension input """ - self.x_shape = (100) + self.x_shape = 100 self.x_type = "float32" self.index = [1, 3, 5] self.index_type = "int32" class TestCase2(TestGatherOp): - def config(self): """ For int64_t index type """ - self.x_shape = (100) + self.x_shape = 100 self.x_type = "float32" self.index = [1, 3, 5] self.index_type = "int64" class API_TestDygraphGather(unittest.TestCase): - def test_out1(self): paddle.disable_static() input_1 = np.array([[1, 2], [3, 4], [5, 6]]).astype('int32') @@ -127,18 +123,18 @@ def test_zero_index(self): class TestGathertError(unittest.TestCase): - def test_error1(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): shape = [8, 9, 6] x = paddle.fluid.data(shape=shape, dtype='int8', name='x') axis = paddle.fluid.data(shape=[1], dtype='float32', name='axis') index = paddle.fluid.data(shape=shape, dtype='int32', name='index') - index_float = paddle.fluid.data(shape=shape, - dtype='float32', - name='index_float') + index_float = paddle.fluid.data( + shape=shape, dtype='float32', name='index_float' + ) def test_x_type(): paddle.gather(x, index) @@ -166,9 +162,9 @@ def test_error2(self): shape = [8, 9, 6] x = fluid.data(shape=shape, dtype='int8', name='x') index = fluid.data(shape=shape, dtype='int32', name='mask') - index_float = fluid.data(shape=shape, - dtype='float32', - name='index_float') + index_float = fluid.data( + shape=shape, dtype='float32', name='index_float' + ) def test_x_type(): paddle.fluid.layers.gather(x, index) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_gaussian_random_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_gaussian_random_op_mlu.py index 83540bfadd2bf..4cac3a2e36b6c 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_gaussian_random_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_gaussian_random_op_mlu.py @@ -29,7 +29,6 @@ class TestGaussianRandomOp(OpTest): - def setUp(self): self.op_type = "gaussian_random" self.place = paddle.device.MLUPlace(0) @@ -48,7 +47,7 @@ def setUp(self): def set_attrs(self): self.mean = 1.0 - self.std = 2. + self.std = 2.0 def test_check_output(self): self.check_output_with_place_customized(self.verify_output, self.place) @@ -67,7 +66,6 @@ def verify_output(self, outs): class TestMeanStdAreInt(TestGaussianRandomOp): - def set_attrs(self): self.mean = 1 self.std = 2 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_gelu_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_gelu_op_mlu.py index b905eba2371a8..8a879aa22213a 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_gelu_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_gelu_op_mlu.py @@ -32,7 +32,6 @@ def np_gelu(x): class TestGelu(OpTest): - def setUp(self): self.set_mlu() self.op_type = "gelu" @@ -57,13 +56,12 @@ def test_check_output(self): self.check_output_with_place(self.place, atol=1e-3) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.007) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.007 + ) class TestGeluFp16(OpTest): - def setUp(self): self.set_mlu() self.op_type = "gelu" @@ -90,7 +88,6 @@ def test_check_output(self): class TestGeluNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -105,9 +102,9 @@ def _test(self, run_mlu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) @@ -131,16 +128,17 @@ def _test(self, run_mlu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_grid_sampler_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_grid_sampler_op_mlu.py index 96dbaab9ee157..df173ebf18c6e 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_grid_sampler_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_grid_sampler_op_mlu.py @@ -28,12 +28,15 @@ def AffineGrid(theta, grid_shape): n = grid_shape[0] h = grid_shape[1] w = grid_shape[2] - h_idx = np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, - axis=0).T[:, :, np.newaxis] - w_idx = np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, - axis=0)[:, :, np.newaxis] - grid = np.concatenate([w_idx, h_idx, np.ones([h, w, 1])], - axis=2) # h * w * 3 + h_idx = np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, axis=0).T[ + :, :, np.newaxis + ] + w_idx = np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, axis=0)[ + :, :, np.newaxis + ] + grid = np.concatenate( + [w_idx, h_idx, np.ones([h, w, 1])], axis=2 + ) # h * w * 3 grid = np.repeat(grid[np.newaxis, :], n, axis=0) # n * h * w *3 ret = np.zeros([n, h * w, 2]) @@ -53,13 +56,17 @@ def getGridPointValue(data, x, y): out_H = x.shape[1] out_W = x.shape[2] - #out = np.zeros(data_shape, dtype='float32') + # out = np.zeros(data_shape, dtype='float32') out = np.zeros([N, C, out_H, out_W], dtype='float32') for i in range(N): for j in range(out_H): for k in range(out_W): - if y[i, j, k] < 0 or y[i, j, k] > in_H - 1 or x[ - i, j, k] < 0 or x[i, j, k] > in_W - 1: + if ( + y[i, j, k] < 0 + or y[i, j, k] > in_H - 1 + or x[i, j, k] < 0 + or x[i, j, k] > in_W - 1 + ): out[i, :, j, k] = 0 else: out[i, :, j, k] = data[i, :, y[i, j, k], x[i, j, k]] @@ -75,27 +82,28 @@ def unnormalizeAndClip(grid_slice, max_val, align_corners, padding_mode): if align_corners: grid_slice = 0.5 * ((grid_slice.astype('float32') + 1.0) * max_val) else: - grid_slice = 0.5 * ((grid_slice.astype('float32') + 1.0) * - (max_val + 1)) - 0.5 + grid_slice = ( + 0.5 * ((grid_slice.astype('float32') + 1.0) * (max_val + 1)) - 0.5 + ) if padding_mode == "border": grid_slice = clip(grid_slice, 0, max_val) elif padding_mode == "reflection": double_range = 2 * max_val if align_corners else (max_val + 1) * 2 - grid_abs = np.abs(grid_slice) if align_corners else np.abs(grid_slice + - 0.5) + grid_abs = ( + np.abs(grid_slice) if align_corners else np.abs(grid_slice + 0.5) + ) extra = grid_abs - np.floor(grid_abs / double_range) * double_range grid_slice = np.minimum(extra, double_range - extra) - grid_slice = grid_slice if align_corners else clip( - grid_slice - 0.5, 0, max_val) + grid_slice = ( + grid_slice if align_corners else clip(grid_slice - 0.5, 0, max_val) + ) return grid_slice -def GridSampler(data, - grid, - align_corners=True, - mode="bilinear", - padding_mode="zeros"): +def GridSampler( + data, grid, align_corners=True, mode="bilinear", padding_mode="zeros" +): dims = data.shape N = dims[0] in_C = dims[1] @@ -119,14 +127,18 @@ def GridSampler(data, y0 = np.floor(y).astype('int32') y1 = y0 + 1 - wa = np.tile(((x1 - x) * (y1 - y)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wb = np.tile(((x1 - x) * (y - y0)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wc = np.tile(((x - x0) * (y1 - y)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wd = np.tile(((x - x0) * (y - y0)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) + wa = np.tile( + ((x1 - x) * (y1 - y)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wb = np.tile( + ((x1 - x) * (y - y0)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wc = np.tile( + ((x - x0) * (y1 - y)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wd = np.tile( + ((x - x0) * (y - y0)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) va = getGridPointValue(data, x0, y0) vb = getGridPointValue(data, x0, y1) @@ -142,7 +154,6 @@ def GridSampler(data, class TestGridSamplerOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -166,12 +177,12 @@ def setUp(self): 'use_cudnn': False, "align_corners": self.align_corners, "padding_mode": self.padding_mode, - "mode": self.mode + "mode": self.mode, } self.outputs = { - 'Output': - GridSampler(x, grid, self.align_corners, self.mode, - self.padding_mode) + 'Output': GridSampler( + x, grid, self.align_corners, self.mode, self.padding_mode + ) } def test_check_output(self): @@ -187,7 +198,6 @@ def initTestCase(self): class Case1(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -198,7 +208,6 @@ def initTestCase(self): class LargeInputCase(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 128, 128) self.grid_shape = (2, 130, 130, 2) @@ -209,7 +218,6 @@ def initTestCase(self): class Case2(LargeInputCase): - def initTestCase(self): self.x_shape = (2, 3, 128, 128) self.grid_shape = (2, 130, 130, 2) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_hard_sigmoid_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_hard_sigmoid_op_mlu.py index fca95416a861c..695d37ec54d6a 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_hard_sigmoid_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_hard_sigmoid_op_mlu.py @@ -28,11 +28,10 @@ def ref_hardsigmoid(x, slope=0.166666666666667, offset=0.5): - return np.maximum(np.minimum(x * slope + offset, 1.), 0.).astype(x.dtype) + return np.maximum(np.minimum(x * slope + offset, 1.0), 0.0).astype(x.dtype) class TestMLUHardSigmoid(OpTest): - def setUp(self): paddle.enable_static() @@ -43,7 +42,7 @@ def setUp(self): x = np.random.uniform(-5, 5, [10, 12]).astype(self.dtype) lower_threshold = -self.offset / self.slope - upper_threshold = (1. - self.offset) / self.slope + upper_threshold = (1.0 - self.offset) / self.slope # Same reason as TestAbs delta = 0.005 @@ -75,21 +74,18 @@ def set_attrs(self): class TestMLUHardSigmoid2(TestMLUHardSigmoid): - def set_attrs(self): self.slope = 0.2 self.offset = 0.5 class TestMLUHardSigmoid3(TestMLUHardSigmoid): - def set_attrs(self): self.slope = 0.2 self.offset = 0.4 class TestMLUHardSigmoidFp16(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -114,17 +110,21 @@ def test_check_output_and_grad_mlu(self): cpu_diff_1 = np.divide( np.sum(np.abs(self.float32_y.numpy() - self.float16_y)), - np.sum(np.abs(self.float32_y.numpy()))) + np.sum(np.abs(self.float32_y.numpy())), + ) mlu_diff_1 = np.divide( np.sum(np.abs(self.float32_y.numpy() - mlu_float16_y.numpy())), - np.sum(np.abs(self.float32_y.numpy()))) + np.sum(np.abs(self.float32_y.numpy())), + ) cpu_diff_2 = np.divide( np.sum(np.square(self.float32_y.numpy() - self.float16_y)), - np.sum(np.square(self.float32_y.numpy()))) + np.sum(np.square(self.float32_y.numpy())), + ) mlu_diff_2 = np.divide( np.sum(np.square(self.float32_y.numpy() - mlu_float16_y.numpy())), - np.sum(np.square(self.float32_y.numpy()))) + np.sum(np.square(self.float32_y.numpy())), + ) assert mlu_diff_1 <= cpu_diff_1 assert mlu_diff_2 <= cpu_diff_2 @@ -179,14 +179,14 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.hardsigmoid, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.hardsigmoid, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.hardsigmoid(x_fp16) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_hard_swish_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_hard_swish_op_mlu.py index a88103154e92d..2774162bc7958 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_hard_swish_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_hard_swish_op_mlu.py @@ -38,15 +38,17 @@ def ref_hard_swish_grad(x, threshold, scale, offset, data_type): threshold = scalarToType(threshold, data_type) scale = scalarToType(scale, data_type) offset = scalarToType(offset, data_type) - dout = np.full_like(x, fill_value=1. / x.size) + dout = np.full_like(x, fill_value=1.0 / x.size) tmp = ((x + offset) < threshold).astype(x.dtype) - dx = dout * (((x + offset) > 0).astype(x.dtype) * - (2 * x + offset) * tmp / scale + 1.0 - tmp) + dx = dout * ( + ((x + offset) > 0).astype(x.dtype) * (2 * x + offset) * tmp / scale + + 1.0 + - tmp + ) return dx class TestHardSwishMLU(OpTest): - def setUp(self): paddle.enable_static() @@ -63,11 +65,11 @@ def setUp(self): x[np.abs(x - threshold + offset) < 0.005] = threshold - offset + 0.02 out = ( - x * - (np.minimum(np.maximum(x + offset, 0.), threshold) / scale)).astype( - self.dtype) - self.x_grad = ref_hard_swish_grad(x, threshold, scale, offset, - self.dtype) + x * (np.minimum(np.maximum(x + offset, 0.0), threshold) / scale) + ).astype(self.dtype) + self.x_grad = ref_hard_swish_grad( + x, threshold, scale, offset, self.dtype + ) self.set_mlu() self.inputs = {'X': x} self.attrs = {'threshold': threshold, 'scale': scale, 'offset': offset} @@ -87,7 +89,6 @@ def test_check_grad(self): class TestHardSwishMLUWithCPUFloat16(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -111,11 +112,21 @@ def setUp(self): threshold = scalarToType(threshold, np.float16) scale = scalarToType(scale, np.float16) offset = scalarToType(offset, np.float16) - self.float16_y = (self.float16_x * (np.minimum( - np.maximum(self.float16_x + offset, scalarToType(0., np.float16)), - threshold) / scale)).astype(np.float16) - self.float16_grad = ref_hard_swish_grad(self.float16_x, threshold, - scale, offset, np.float16) + self.float16_y = ( + self.float16_x + * ( + np.minimum( + np.maximum( + self.float16_x + offset, scalarToType(0.0, np.float16) + ), + threshold, + ) + / scale + ) + ).astype(np.float16) + self.float16_grad = ref_hard_swish_grad( + self.float16_x, threshold, scale, offset, np.float16 + ) def test_check_output_and_grad_mlu(self): # mlu float16 @@ -127,36 +138,45 @@ def test_check_output_and_grad_mlu(self): cpu_diff_1 = np.divide( np.sum(np.abs(self.float32_y.numpy() - self.float16_y)), - np.sum(np.abs(self.float32_y.numpy()))) + np.sum(np.abs(self.float32_y.numpy())), + ) mlu_diff_1 = np.divide( np.sum(np.abs(self.float32_y.numpy() - mlu_float16_y.numpy())), - np.sum(np.abs(self.float32_y.numpy()))) + np.sum(np.abs(self.float32_y.numpy())), + ) cpu_diff_2 = np.divide( np.sum(np.square(self.float32_y.numpy() - self.float16_y)), - np.sum(np.square(self.float32_y.numpy()))) + np.sum(np.square(self.float32_y.numpy())), + ) mlu_diff_2 = np.divide( np.sum(np.square(self.float32_y.numpy() - mlu_float16_y.numpy())), - np.sum(np.square(self.float32_y.numpy()))) + np.sum(np.square(self.float32_y.numpy())), + ) assert mlu_diff_1 <= cpu_diff_1 assert mlu_diff_2 <= cpu_diff_2 cpu_diff_1 = np.divide( np.sum(np.abs(self.float32_grad.numpy() - self.float16_grad)), - np.sum(np.abs(self.float32_grad.numpy()))) + np.sum(np.abs(self.float32_grad.numpy())), + ) mlu_diff_1 = np.divide( - np.sum(np.abs(self.float32_grad.numpy() - - mlu_float16_grad.numpy())), - np.sum(np.abs(self.float32_grad.numpy()))) + np.sum( + np.abs(self.float32_grad.numpy() - mlu_float16_grad.numpy()) + ), + np.sum(np.abs(self.float32_grad.numpy())), + ) cpu_diff_2 = np.divide( np.sum(np.square(self.float32_grad.numpy() - self.float16_grad)), - np.sum(np.square(self.float32_grad.numpy()))) + np.sum(np.square(self.float32_grad.numpy())), + ) mlu_diff_2 = np.divide( np.sum( - np.square(self.float32_grad.numpy() - - mlu_float16_grad.numpy())), - np.sum(np.square(self.float32_grad.numpy()))) + np.square(self.float32_grad.numpy() - mlu_float16_grad.numpy()) + ), + np.sum(np.square(self.float32_grad.numpy())), + ) assert mlu_diff_1 <= cpu_diff_1 assert mlu_diff_2 <= cpu_diff_2 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_huber_loss_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_huber_loss_op_mlu.py index 40e0dab080302..35003418095cb 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_huber_loss_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_huber_loss_op_mlu.py @@ -34,7 +34,6 @@ def huber_loss_forward(val, delta): class TestHuberLossOp(OpTest): - def setUp(self): self.op_type = 'huber_loss' self.set_mlu() @@ -44,16 +43,17 @@ def setUp(self): self.init_input() shape = self.set_shape() residual = self.inputs['Y'] - self.inputs['X'] - loss = np.vectorize(huber_loss_forward)(residual, - self.delta).astype('float32') + loss = np.vectorize(huber_loss_forward)(residual, self.delta).astype( + 'float32' + ) self.attrs = {'delta': self.delta} self.outputs = {'Residual': residual, 'Out': loss.reshape(shape)} def init_input(self): shape = self.set_shape() self.inputs = { - 'X': np.random.uniform(0, 1., shape).astype('float32'), - 'Y': np.random.uniform(0, 1., shape).astype('float32'), + 'X': np.random.uniform(0, 1.0, shape).astype('float32'), + 'Y': np.random.uniform(0, 1.0, shape).astype('float32'), } def set_mlu(self): @@ -70,38 +70,40 @@ def test_check_grad_normal(self): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - max_relative_error=0.008, - no_grad_set=set("residual")) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + max_relative_error=0.008, + no_grad_set=set("residual"), + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.008, - no_grad_set=set('residual')) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + max_relative_error=0.008, + no_grad_set=set('residual'), + ) def TestHuberLossOp1(TestHuberLossOp): - def set_shape(self): - return (64) + return 64 def TestHuberLossOp2(TestHuberLossOp): - def set_shape(self): return (6, 6) def TestHuberLossOp3(TestHuberLossOp): - def set_shape(self): return (6, 6, 1) class TestHuberLossOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input and label must be Variable @@ -116,10 +118,12 @@ def test_errors(self): # the dtype of input and label must be float32 or float64 xw2 = fluid.data(name='xw2', shape=[None, 6], dtype="int32") lw2 = fluid.data(name='lw2', shape=[None, 6], dtype="int32") - self.assertRaises(TypeError, fluid.layers.huber_loss, xw2, lr, - delta) - self.assertRaises(TypeError, fluid.layers.huber_loss, xr, lw2, - delta) + self.assertRaises( + TypeError, fluid.layers.huber_loss, xw2, lr, delta + ) + self.assertRaises( + TypeError, fluid.layers.huber_loss, xr, lw2, delta + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_iou_similarity_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_iou_similarity_op_mlu.py index ac206d6649aca..a9035df040953 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_iou_similarity_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_iou_similarity_op_mlu.py @@ -28,7 +28,6 @@ class TestMluIouSimilarityOp(OpTest): - def setUp(self): self.op_type = "iou_similarity" self.set_mlu() @@ -58,7 +57,7 @@ def set_inputs(self): self.boxes1_lod = [[1 for _ in range(self.N)]] self.inputs = { 'X': (self.boxes1, self.boxes1_lod), - 'Y': self.boxes2 + 'Y': self.boxes2, } else: self.inputs = {'X': self.boxes1, 'Y': self.boxes2} @@ -74,7 +73,9 @@ def set_outputs(self): def test_check_output(self): self.check_output_with_place(self.place) - def _compute_iou(self, ): + def _compute_iou( + self, + ): for row in range(self.boxes1.shape[0]): for col in range(self.boxes2.shape[0]): xmin1, ymin1, xmax1, ymax1 = self.boxes1[row] @@ -104,7 +105,6 @@ def _compute_iou(self, ): class TestMluIouSimilarityOpWithLoD(TestMluIouSimilarityOp): - def set_init_config(self): super(TestMluIouSimilarityOpWithLoD, self).set_init_config() self.box_normalized = True @@ -112,7 +112,6 @@ def set_init_config(self): class TestMluIouSimilarityOpWithBoxNormalized(TestMluIouSimilarityOp): - def set_init_config(self): super(TestMluIouSimilarityOpWithBoxNormalized, self).set_init_config() self.box_normalized = True @@ -120,7 +119,6 @@ def set_init_config(self): def TestMluIouSimilarityOpFp16(TestMluIouSimilarityOp): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_label_smooth_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_label_smooth_op_mlu.py index dfc01e42202ec..0379c8c9cd8be 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_label_smooth_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_label_smooth_op_mlu.py @@ -26,7 +26,6 @@ class TestLabelSmoothOp(OpTest): - def init_dtype(self): self.dtype = np.float32 @@ -46,7 +45,8 @@ def setUp(self): self.init_dtype() self.config() smoothed_label = ( - 1 - self.epsilon) * self.label + self.epsilon / self.label_dim + 1 - self.epsilon + ) * self.label + self.epsilon / self.label_dim smoothed_label = smoothed_label.astype(self.dtype) self.inputs = {'X': self.label} self.attrs = {'epsilon': self.epsilon} @@ -57,7 +57,6 @@ def test_check_output(self): class TestLabelSmoothOpWithPriorDist(TestLabelSmoothOp): - def setUp(self): self.init_dtype() self.config() @@ -70,45 +69,43 @@ def setUp(self): class TestLabelSmoothOp3D(TestLabelSmoothOp): - def setUp(self): super(TestLabelSmoothOp3D, self).setUp() self.inputs['X'] = self.inputs['X'].reshape( - [2, -1, self.inputs['X'].shape[-1]]) + [2, -1, self.inputs['X'].shape[-1]] + ) self.outputs['Out'] = self.outputs['Out'].reshape( - self.inputs['X'].shape) + self.inputs['X'].shape + ) class TestLabelSmoothOpWithPriorDist3D(TestLabelSmoothOpWithPriorDist): - def setUp(self): super(TestLabelSmoothOpWithPriorDist3D, self).setUp() self.inputs['X'] = self.inputs['X'].reshape( - [2, -1, self.inputs['X'].shape[-1]]) + [2, -1, self.inputs['X'].shape[-1]] + ) self.outputs['Out'] = self.outputs['Out'].reshape( - self.inputs['X'].shape) + self.inputs['X'].shape + ) class TestLabelSmoothOpFP16(TestLabelSmoothOp): - def init_dtype(self): self.dtype = np.float16 class TestLabelSmoothOpWithPriorDistFP16(TestLabelSmoothOpWithPriorDist): - def init_dtype(self): self.dtype = np.float16 class TestLabelSmoothOp3DFP16(TestLabelSmoothOp3D): - def init_dtype(self): self.dtype = np.float16 class TestLabelSmoothOpWithPriorDist3DFP16(TestLabelSmoothOpWithPriorDist3D): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_layer_norm_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_layer_norm_op_mlu.py index 838ee571772fb..1a0eac49eccfb 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_layer_norm_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_layer_norm_op_mlu.py @@ -26,8 +26,13 @@ sys.path.append('..') from op_test import _set_use_system_allocator from paddle.fluid import Program, program_guard -from paddle.fluid.contrib.mixed_precision.fp16_utils import _keep_layer_norm_scale_bias_to_fp32 -from test_layer_norm_op import _reference_layer_norm_naive, _reference_layer_norm_grad +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + _keep_layer_norm_scale_bias_to_fp32, +) +from test_layer_norm_op import ( + _reference_layer_norm_naive, + _reference_layer_norm_grad, +) paddle.enable_static() @@ -37,51 +42,57 @@ class TestLayerNormOp(unittest.TestCase): - def setUp(self): self.use_cudnn = True self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=1e-5, - atol=atol, - err_msg=msg) - - def check_forward_backward(self, - shape, - begin_norm_axis, - has_scale=True, - has_bias=True, - y_grad_scale=1.0, - use_mkldnn=False): - - def test_with_place(place, - shape, - begin_norm_axis, - use_mkldnn=use_mkldnn): + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=1e-5, atol=atol, err_msg=msg + ) + + def check_forward_backward( + self, + shape, + begin_norm_axis, + has_scale=True, + has_bias=True, + y_grad_scale=1.0, + use_mkldnn=False, + ): + def test_with_place( + place, shape, begin_norm_axis, use_mkldnn=use_mkldnn + ): # attr epsilon = 0.00001 x_shape = shape - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) scale_shape = [D] np.random.seed(123) x = np.random.random_sample(x_shape).astype(np.float32) - scale = np.random.random_sample(scale_shape).astype( - np.float32) if has_scale else None - bias = np.random.random_sample(scale_shape).astype( - np.float32) if has_bias else None + scale = ( + np.random.random_sample(scale_shape).astype(np.float32) + if has_scale + else None + ) + bias = ( + np.random.random_sample(scale_shape).astype(np.float32) + if has_bias + else None + ) y_grad = (np.random.random_sample(x_shape) * y_grad_scale).astype( - np.float32) + np.float32 + ) # reference forward & backward y, mean, variance = _reference_layer_norm_naive( - x, scale, bias, epsilon, begin_norm_axis) + x, scale, bias, epsilon, begin_norm_axis + ) x_grad, scale_grad, bias_grad = _reference_layer_norm_grad( - x, y_grad, scale, bias, mean, variance, begin_norm_axis) + x, y_grad, scale, bias, mean, variance, begin_norm_axis + ) var_dict = locals() var_dict['y@GRAD'] = y_grad @@ -96,9 +107,11 @@ def test_with_place(place, with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) inputs = {"X": block.var('x')} fetch_list = [ 'y', @@ -118,17 +131,20 @@ def test_with_place(place, outputs={ "Y": block.var('y'), "Mean": block.var('mean'), # share the same memory - "Variance": - block.var('variance'), # share the same memory + "Variance": block.var( + 'variance' + ), # share the same memory }, attrs={ "epsilon": epsilon, "begin_norm_axis": begin_norm_axis, - "use_mkldnn": use_mkldnn - }) + "use_mkldnn": use_mkldnn, + }, + ) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - layer_norm_op.desc, set(), []) + layer_norm_op.desc, set(), [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -142,105 +158,130 @@ def test_with_place(place, program._sync_with_cpp() exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in ['x', 'scale', 'bias', 'y@GRAD'] - }, - fetch_list=fetch_list) + out = exe.run( + program, + feed={ + name: var_dict[name] + for name in ['x', 'scale', 'bias', 'y@GRAD'] + }, + fetch_list=fetch_list, + ) self.__assert_close(y, out[0], "y") self.__assert_close(mean, out[1], "mean") - self.__assert_close(1 / np.sqrt(variance), out[2], "variance", - 1e-3) + self.__assert_close( + 1 / np.sqrt(variance), out[2], "variance", 1e-3 + ) self.__assert_close(x_grad, out[3], "x_grad") if has_scale: - self.__assert_close(scale_grad.reshape(-1), - out[fetch_list.index('scale@GRAD')], - "scale_grad", 1e-3) + self.__assert_close( + scale_grad.reshape(-1), + out[fetch_list.index('scale@GRAD')], + "scale_grad", + 1e-3, + ) if has_bias: - self.__assert_close(bias_grad.reshape(-1), - out[fetch_list.index('bias@GRAD')], - "bias_grad") + self.__assert_close( + bias_grad.reshape(-1), + out[fetch_list.index('bias@GRAD')], + "bias_grad", + ) test_with_place(self.place, shape, begin_norm_axis) def test_check_forward_backward_with_scale_and_bias(self): self.check_forward_backward(shape=[1, 3, 4, 5], begin_norm_axis=1) self.check_forward_backward(shape=[2, 3, 4, 5], begin_norm_axis=1) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=False, - has_bias=True) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=True, - has_bias=False) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=False, - has_bias=False) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=False, + has_bias=True, + ) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=True, + has_bias=False, + ) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=False, + has_bias=False, + ) self.check_forward_backward(shape=[2, 3, 4, 5], begin_norm_axis=3) - self.check_forward_backward(shape=[92, 513, 129], - begin_norm_axis=2, - y_grad_scale=0.1) + self.check_forward_backward( + shape=[92, 513, 129], begin_norm_axis=2, y_grad_scale=0.1 + ) self.check_forward_backward(shape=[3, 34, 1134], begin_norm_axis=2) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - y_grad_scale=0.1) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - has_scale=False, - has_bias=True, - y_grad_scale=0.1) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - has_scale=True, - has_bias=False, - y_grad_scale=0.1) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - has_scale=False, - has_bias=False, - y_grad_scale=0.1) - self.check_forward_backward(shape=[512, 1024], - begin_norm_axis=1, - has_scale=True, - has_bias=True) + self.check_forward_backward( + shape=[92, 513, 1134], begin_norm_axis=2, y_grad_scale=0.1 + ) + self.check_forward_backward( + shape=[92, 513, 1134], + begin_norm_axis=2, + has_scale=False, + has_bias=True, + y_grad_scale=0.1, + ) + self.check_forward_backward( + shape=[92, 513, 1134], + begin_norm_axis=2, + has_scale=True, + has_bias=False, + y_grad_scale=0.1, + ) + self.check_forward_backward( + shape=[92, 513, 1134], + begin_norm_axis=2, + has_scale=False, + has_bias=False, + y_grad_scale=0.1, + ) + self.check_forward_backward( + shape=[512, 1024], begin_norm_axis=1, has_scale=True, has_bias=True + ) class TestLayerNormAPI(unittest.TestCase): - def test_case(self): - x = fluid.layers.data(name='x', - shape=[64, 32, 256], - dtype='float32', - append_batch_size=False) - x = fluid.layers.layer_norm(x, - scale=True, - shift=True, - begin_norm_axis=1, - epsilon=1e-05, - param_attr=None, - bias_attr=None) - x = fluid.layers.layer_norm(x, - scale=False, - shift=False, - begin_norm_axis=1, - epsilon=1e-05, - param_attr=None, - bias_attr=None) - x = fluid.layers.layer_norm(x, - scale=False, - shift=False, - begin_norm_axis=1, - epsilon=1e-05, - param_attr="scale", - bias_attr="shift") + x = fluid.layers.data( + name='x', + shape=[64, 32, 256], + dtype='float32', + append_batch_size=False, + ) + x = fluid.layers.layer_norm( + x, + scale=True, + shift=True, + begin_norm_axis=1, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + ) + x = fluid.layers.layer_norm( + x, + scale=False, + shift=False, + begin_norm_axis=1, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + ) + x = fluid.layers.layer_norm( + x, + scale=False, + shift=False, + begin_norm_axis=1, + epsilon=1e-05, + param_attr="scale", + bias_attr="shift", + ) class TestDygraphLayerNormAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): paddle.enable_static() @@ -256,7 +297,6 @@ def test_errors(self): class TestFP16ScaleBiasLayerNorm(unittest.TestCase): - def check_main(self, x_np, weight_np, bias_np, dtype): paddle.disable_static() @@ -285,9 +325,11 @@ def test_main(self): bias_np = np.random.random([20]).astype('float16') y_np_1, x_g_np_1, w_g_np_1, b_g_np_1 = self.check_main( - x_np, weight_np, bias_np, 'float16') + x_np, weight_np, bias_np, 'float16' + ) y_np_2, x_g_np_2, w_g_np_2, b_g_np_2 = self.check_main( - x_np, weight_np, bias_np, 'float32') + x_np, weight_np, bias_np, 'float32' + ) def assert_equal(x, y): np.testing.assert_allclose(x, y) @@ -299,7 +341,6 @@ def assert_equal(x, y): class TestGetSetKeepLayerNormScaleBiasFP32Flag(unittest.TestCase): - def test_main(self): self.assertTrue(_keep_layer_norm_scale_bias_to_fp32()) _keep_layer_norm_scale_bias_to_fp32(False) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_leaky_relu_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_leaky_relu_op_mlu.py index 23fd10ffbd887..05d78e7f31a1c 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_leaky_relu_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_leaky_relu_op_mlu.py @@ -27,7 +27,6 @@ class TestLeadyRelu(OpTest): - def setUp(self): self.set_mlu() self.op_type = "leaky_relu" @@ -63,33 +62,29 @@ def test_check_output(self): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.006 + ) else: self.check_grad_with_place(self.place, ['X'], 'Out') class TestLeadyReluFP16(TestLeadyRelu): - def init_dtype(self): self.dtype = np.float16 class TestLeadyRelu2(TestLeadyRelu): - def set_attrs(self): self.attrs = {'alpha': 0.5} class TestLeadyRelu3(TestLeadyRelu): - def set_attrs(self): self.attrs = {'alpha': -0.5} class TestLeakyReluNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -102,9 +97,9 @@ def _test(self, run_mlu=True): with paddle.static.program_guard(main_prog, startup_prog): x = paddle.static.data(name="x", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) y = paddle.nn.functional.leaky_relu(x) @@ -127,15 +122,17 @@ def _test(self, run_mlu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "x": x_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"x": x_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_log_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_log_op_mlu.py index 1dca8d9a0bcb4..11df0949a6f47 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_log_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_log_op_mlu.py @@ -26,7 +26,6 @@ class TestActivation(OpTest): - def setUp(self): self.set_mlu() self.op_type = "exp" @@ -57,7 +56,6 @@ def set_mlu(self): class TestLog(TestActivation): - def setUp(self): self.set_mlu() self.op_type = "log" @@ -72,21 +70,18 @@ def setUp(self): self.outputs = {'Out': out} def test_error(self): - in1 = fluid.layers.data(name="in1", - shape=[11, 17], - append_batch_size=False, - dtype="int32") - in2 = fluid.layers.data(name="in2", - shape=[11, 17], - append_batch_size=False, - dtype="int64") + in1 = fluid.layers.data( + name="in1", shape=[11, 17], append_batch_size=False, dtype="int32" + ) + in2 = fluid.layers.data( + name="in2", shape=[11, 17], append_batch_size=False, dtype="int64" + ) self.assertRaises(TypeError, fluid.layers.log, in1) self.assertRaises(TypeError, fluid.layers.log, in2) class TestLog2(TestActivation): - def setUp(self): self.set_mlu() self.op_type = "log2" @@ -107,19 +102,22 @@ def test_error(self): self.assertRaises(TypeError, paddle.log2, in2) def test_api(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): input_x = np.random.uniform(0.1, 1, [11, 17]).astype("float32") - data_x = paddle.static.data(name="data_x", - shape=[11, 17], - dtype="float32") + data_x = paddle.static.data( + name="data_x", shape=[11, 17], dtype="float32" + ) out1 = paddle.log2(data_x) exe = paddle.static.Executor(place=fluid.CPUPlace()) exe.run(paddle.static.default_startup_program()) - res1 = exe.run(paddle.static.default_main_program(), - feed={"data_x": input_x}, - fetch_list=[out1]) + res1 = exe.run( + paddle.static.default_main_program(), + feed={"data_x": input_x}, + fetch_list=[out1], + ) expected_res = np.log2(input_x) np.testing.assert_allclose(res1[0], expected_res, rtol=1e-6) @@ -136,7 +134,6 @@ def test_api(self): class TestLog10(TestActivation): - def setUp(self): self.set_mlu() self.op_type = "log10" @@ -157,19 +154,22 @@ def test_error(self): self.assertRaises(TypeError, paddle.log10, in2) def test_api(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): input_x = np.random.uniform(0.1, 1, [11, 17]).astype("float32") - data_x = paddle.static.data(name="data_x", - shape=[11, 17], - dtype="float32") + data_x = paddle.static.data( + name="data_x", shape=[11, 17], dtype="float32" + ) out1 = paddle.log10(data_x) exe = paddle.static.Executor(place=paddle.CPUPlace()) exe.run(paddle.static.default_startup_program()) - res1 = exe.run(paddle.static.default_main_program(), - feed={"data_x": input_x}, - fetch_list=[out1]) + res1 = exe.run( + paddle.static.default_main_program(), + feed={"data_x": input_x}, + fetch_list=[out1], + ) expected_res = np.log10(input_x) np.testing.assert_allclose(res1[0], expected_res, rtol=1e-6) @@ -184,7 +184,6 @@ def test_api(self): class TestLogHalf(TestLog): - def init_dtype(self): self.dtype = np.float16 @@ -193,7 +192,6 @@ def test_api(self): class TestLog2Half(TestLog2): - def init_dtype(self): self.dtype = np.float16 @@ -202,7 +200,6 @@ def test_api(self): class TestLog10Half(TestLog10): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_log_softmax_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_log_softmax_op_mlu.py index 91e8a86ce6856..82d22af893303 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_log_softmax_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_log_softmax_op_mlu.py @@ -24,7 +24,7 @@ def ref_log_softmax(x): - shiftx = (x - np.max(x)) + shiftx = x - np.max(x) out = shiftx - np.log(np.exp(shiftx).sum()) return out @@ -34,14 +34,14 @@ def ref_log_softmax_grad(x, axis): axis += len(x.shape) out = np.apply_along_axis(ref_log_softmax, axis, x) axis_dim = x.shape[axis] - dout = np.full_like(x, fill_value=1. / x.size) + dout = np.full_like(x, fill_value=1.0 / x.size) dx = dout - np.exp(out) * dout.copy().sum(axis=axis, keepdims=True).repeat( - axis_dim, axis=axis) + axis_dim, axis=axis + ) return dx class TestLogSoftmaxOp(OpTest): - def setUp(self): self.op_type = 'log_softmax' self.set_mlu() @@ -51,7 +51,7 @@ def setUp(self): self.axis = -1 self.set_attrs() - x = np.random.uniform(0.1, 1., self.shape).astype(self.dtype) + x = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) out = np.apply_along_axis(ref_log_softmax, self.axis, x) self.x_grad = ref_log_softmax_grad(x, self.axis) @@ -70,24 +70,22 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], ['Out'], - user_defined_grads=[self.x_grad]) + self.check_grad_with_place( + self.place, ['X'], ['Out'], user_defined_grads=[self.x_grad] + ) class TestLogSoftmaxShape(TestLogSoftmaxOp): - def set_attrs(self): self.shape = [12, 10] class TestLogSoftmaxAxis(TestLogSoftmaxOp): - def set_attrs(self): self.axis = 1 class TestLogSoftmaxOpFp16(OpTest): - def setUp(self): self.op_type = 'log_softmax' self.set_mlu() @@ -97,7 +95,7 @@ def setUp(self): self.axis = -1 self.set_attrs() - x = np.random.uniform(0.1, 1., self.shape).astype(self.dtype) + x = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) out = np.apply_along_axis(ref_log_softmax, self.axis, x) self.x_grad = ref_log_softmax_grad(x, self.axis) @@ -116,17 +114,20 @@ def test_check_output(self): self.check_output_with_place(self.place, atol=1e-2) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], ['Out'], - user_defined_grads=[self.x_grad], - max_relative_error=0.015) + self.check_grad_with_place( + self.place, + ['X'], + ['Out'], + user_defined_grads=[self.x_grad], + max_relative_error=0.015, + ) class TestNNLogSoftmaxAPI(unittest.TestCase): - def setUp(self): self.set_mlu() self.x_shape = [2, 3, 4, 5] - self.x = np.random.uniform(-1., 1., self.x_shape).astype(np.float32) + self.x = np.random.uniform(-1.0, 1.0, self.x_shape).astype(np.float32) def set_mlu(self): self.__class__.use_mlu = True @@ -158,7 +159,6 @@ def test_check_api(self): class TestNNFunctionalLogSoftmaxAPI(unittest.TestCase): - def setUp(self): self.set_mlu() self.x_shape = [2, 3, 4, 5] diff --git a/python/paddle/fluid/tests/unittests/mlu/test_logical_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_logical_op_mlu.py index c032748eb9aad..6923f66967815 100755 --- a/python/paddle/fluid/tests/unittests/mlu/test_logical_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_logical_op_mlu.py @@ -26,84 +26,32 @@ SUPPORTED_DTYPES = [bool, np.int8, np.int16, np.int32, np.float32] -TEST_META_OP_DATA = [{ - 'op_str': 'logical_and', - 'binary_op': True -}, { - 'op_str': 'logical_or', - 'binary_op': True -}, { - 'op_str': 'logical_xor', - 'binary_op': True -}, { - 'op_str': 'logical_not', - 'binary_op': False -}] +TEST_META_OP_DATA = [ + {'op_str': 'logical_and', 'binary_op': True}, + {'op_str': 'logical_or', 'binary_op': True}, + {'op_str': 'logical_xor', 'binary_op': True}, + {'op_str': 'logical_not', 'binary_op': False}, +] TEST_META_SHAPE_DATA = { - 'XDimLargerThanYDim1': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 5] - }, - 'XDimLargerThanYDim2': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 1] - }, - 'XDimLargerThanYDim3': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [1, 4, 1] - }, - 'XDimLargerThanYDim4': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [3, 4, 1] - }, - 'XDimLargerThanYDim5': { - 'x_shape': [2, 3, 1, 5], - 'y_shape': [3, 1, 1] - }, - 'XDimLessThanYDim1': { - 'x_shape': [4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim2': { - 'x_shape': [1, 4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim3': { - 'x_shape': [3, 4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim4': { - 'x_shape': [3, 1, 1], - 'y_shape': [2, 3, 1, 5] - }, - 'XDimLessThanYDim5': { - 'x_shape': [4, 5], - 'y_shape': [2, 3, 4, 5] - }, - 'Axis1InLargerDim': { - 'x_shape': [1, 4, 5], - 'y_shape': [2, 3, 1, 5] - }, - 'EqualDim1': { - 'x_shape': [10, 7], - 'y_shape': [10, 7] - }, - 'EqualDim2': { - 'x_shape': [1, 1, 4, 5], - 'y_shape': [2, 3, 1, 5] - } + 'XDimLargerThanYDim1': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 5]}, + 'XDimLargerThanYDim2': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 1]}, + 'XDimLargerThanYDim3': {'x_shape': [2, 3, 4, 5], 'y_shape': [1, 4, 1]}, + 'XDimLargerThanYDim4': {'x_shape': [2, 3, 4, 5], 'y_shape': [3, 4, 1]}, + 'XDimLargerThanYDim5': {'x_shape': [2, 3, 1, 5], 'y_shape': [3, 1, 1]}, + 'XDimLessThanYDim1': {'x_shape': [4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim2': {'x_shape': [1, 4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim3': {'x_shape': [3, 4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim4': {'x_shape': [3, 1, 1], 'y_shape': [2, 3, 1, 5]}, + 'XDimLessThanYDim5': {'x_shape': [4, 5], 'y_shape': [2, 3, 4, 5]}, + 'Axis1InLargerDim': {'x_shape': [1, 4, 5], 'y_shape': [2, 3, 1, 5]}, + 'EqualDim1': {'x_shape': [10, 7], 'y_shape': [10, 7]}, + 'EqualDim2': {'x_shape': [1, 1, 4, 5], 'y_shape': [2, 3, 1, 5]}, } TEST_META_WRONG_SHAPE_DATA = { - 'ErrorDim1': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [3, 4] - }, - 'ErrorDim2': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 3] - } + 'ErrorDim1': {'x_shape': [2, 3, 4, 5], 'y_shape': [3, 4]}, + 'ErrorDim2': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 3]}, } @@ -162,16 +110,20 @@ def test(unit_test, use_mlu=False, test_error=False): META_DATA = dict(TEST_META_WRONG_SHAPE_DATA) for shape_data in META_DATA.values(): for data_type in SUPPORTED_DTYPES: - meta_data['x_np'] = np_data_generator(shape_data['x_shape'], - dtype=data_type) - meta_data['y_np'] = np_data_generator(shape_data['y_shape'], - dtype=data_type) + meta_data['x_np'] = np_data_generator( + shape_data['x_shape'], dtype=data_type + ) + meta_data['y_np'] = np_data_generator( + shape_data['y_shape'], dtype=data_type + ) if meta_data['binary_op'] and test_error: # catch C++ Exception - unit_test.assertRaises(BaseException, run_static, - **meta_data) - unit_test.assertRaises(BaseException, run_dygraph, - **meta_data) + unit_test.assertRaises( + BaseException, run_static, **meta_data + ) + unit_test.assertRaises( + BaseException, run_dygraph, **meta_data + ) continue static_result = run_static(**meta_data) dygraph_result = run_dygraph(**meta_data) @@ -181,11 +133,11 @@ def test(unit_test, use_mlu=False, test_error=False): np_result = np_op(meta_data['x_np']) unit_test.assertTrue((static_result == np_result).all()) unit_test.assertTrue( - (dygraph_result.numpy() == np_result).all()) + (dygraph_result.numpy() == np_result).all() + ) def test_type_error(unit_test, use_mlu, type_str_map): - def check_type(op_str, x, y, binary_op): op = getattr(paddle, op_str) error_type = ValueError @@ -220,24 +172,24 @@ def check_type(op_str, x, y, binary_op): startup_program = paddle.static.Program() main_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - x = paddle.static.data(name='x', - shape=[10], - dtype=type_str_map['x']) - y = paddle.static.data(name='y', - shape=[10], - dtype=type_str_map['y']) + x = paddle.static.data( + name='x', shape=[10], dtype=type_str_map['x'] + ) + y = paddle.static.data( + name='y', shape=[10], dtype=type_str_map['y'] + ) check_type(meta_data['op_str'], x, y, binary_op) def type_map_factory(): - return [{ - 'x': x_type, - 'y': y_type - } for x_type in SUPPORTED_DTYPES for y_type in SUPPORTED_DTYPES] + return [ + {'x': x_type, 'y': y_type} + for x_type in SUPPORTED_DTYPES + for y_type in SUPPORTED_DTYPES + ] class TestMLU(unittest.TestCase): - def test(self): test(self, True) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_lookup_table_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_lookup_table_v2_op_mlu.py index 29b58a4bbec0b..f8b962d44d13c 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_lookup_table_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_lookup_table_v2_op_mlu.py @@ -26,7 +26,6 @@ class TestLookupTableV2(OpTest): - def setUp(self): self.set_mlu() self.op_type = "lookup_table_v2" @@ -36,22 +35,22 @@ def setUp(self): self.init_padding_idx() np.random.seed(SEED) w = np.random.random([self.vocab, self.dim]).astype(self.dtype) - x = np.random.randint(0, self.vocab, - size=(self.bsz, - self.seqlen)).astype(self.ids_dtype) + x = np.random.randint( + 0, self.vocab, size=(self.bsz, self.seqlen) + ).astype(self.ids_dtype) out = w[x] if self.padding_idx != -1: out[np.squeeze(x == self.padding_idx)] = np.zeros(self.dim) self.inputs = { 'W': OpTest.np_dtype_to_fluid_dtype(w), - 'Ids': OpTest.np_dtype_to_fluid_dtype(x) + 'Ids': OpTest.np_dtype_to_fluid_dtype(x), } self.attrs = { 'is_sparse': False, 'is_distributed': False, 'remote_prefetch': False, - 'padding_idx': self.padding_idx + 'padding_idx': self.padding_idx, } self.outputs = {'Out': out} @@ -78,9 +77,9 @@ def test_check_output(self): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['W'], - 'Out', - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ['W'], 'Out', max_relative_error=0.01 + ) else: self.check_grad_with_place(self.place, ['W'], 'Out') @@ -98,7 +97,6 @@ def set_mlu(self): class TestLookupTableV2Dim32(TestLookupTableV2): - def init_dims(self): self.bsz = 6 self.seqlen = 8 @@ -126,13 +124,11 @@ def set_mlu(self): class TestLookupTableV2WithPadding(TestLookupTableV2): - def init_padding_idx(self): self.padding_idx = np.random.randint(0, self.vocab) class TestLookupTableV2WithPadding1(TestLookupTableV2): - def init_padding_idx(self): self.padding_idx = np.random.randint(0, self.vocab) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_masked_select_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_masked_select_op_mlu.py index c60eea0787400..11e9dc86cd470 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_masked_select_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_masked_select_op_mlu.py @@ -39,7 +39,6 @@ def np_masked_select(shape, x, mask): class TestMaskedSelectOp(OpTest): - def setUp(self): self.init() self.__class__.use_mlu = True @@ -63,20 +62,17 @@ def init(self): class TestMaskedSelectOp1(TestMaskedSelectOp): - def init(self): self.shape = (6, 8, 9, 18) class TestMaskedSelectOp2(TestMaskedSelectOp): - def init(self): - self.shape = (168, ) + self.shape = (168,) @skip_check_grad_ci(reason="get_numeric_gradient not support int32") class TestMaskedSelectOpInt32(TestMaskedSelectOp): - def init_dtype(self): self.dtype = np.int32 @@ -85,20 +81,18 @@ def test_check_grad(self): class TestMaskedSelectOpFp16(TestMaskedSelectOp): - def init_dtype(self): self.dtype = np.float16 def test_check_grad(self): x_grad = self.inputs['Mask'].astype(self.dtype) x_grad = x_grad * (1 / x_grad.size) - self.check_grad_with_place(self.place, ['X'], - 'Y', - user_defined_grads=[x_grad]) + self.check_grad_with_place( + self.place, ['X'], 'Y', user_defined_grads=[x_grad] + ) class TestMaskedSelectAPI(unittest.TestCase): - def test_imperative_mode(self): paddle.disable_static() shape = (88, 6, 8) @@ -123,27 +117,26 @@ def test_static_mode(self): exe = paddle.static.Executor(place=paddle.device.MLUPlace(0)) - res = exe.run(paddle.static.default_main_program(), - feed={ - "x": np_x, - "mask": np_mask - }, - fetch_list=[out]) + res = exe.run( + paddle.static.default_main_program(), + feed={"x": np_x, "mask": np_mask}, + fetch_list=[out], + ) self.assertEqual(np.allclose(res, np_out), True) class TestMaskedSelectError(unittest.TestCase): - def test_error(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): shape = [8, 9, 6] x = paddle.fluid.data(shape=shape, dtype='float32', name='x') mask = paddle.fluid.data(shape=shape, dtype='bool', name='mask') - mask_float = paddle.fluid.data(shape=shape, - dtype='float32', - name='mask_float') + mask_float = paddle.fluid.data( + shape=shape, dtype='float32', name='mask_float' + ) np_x = np.random.random(shape).astype('float32') np_mask = np.array(np.random.randint(2, size=shape, dtype=bool)) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_matmul_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_matmul_op_mlu.py index 76daa775756cd..a33ef9d54127e 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_matmul_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_matmul_op_mlu.py @@ -31,7 +31,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False, scale=1.0): # transpose X and Y appropriately. if transpose_X: if X.ndim == 1: - X = X.reshape((X.size, )) + X = X.reshape((X.size,)) elif X.ndim == 2: X = X.T else: @@ -40,7 +40,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False, scale=1.0): X = np.transpose(X, tuple(dim)) if transpose_Y: if Y.ndim == 1: - Y = Y.reshape((Y.size, )) + Y = Y.reshape((Y.size,)) else: dim = [i for i in range(len(Y.shape))] dim[-1], dim[len(Y.shape) - 2] = dim[len(Y.shape) - 2], dim[-1] @@ -76,14 +76,15 @@ def setUp(self): X = -0.1 + 0.2 * X Y = -0.1 + 0.2 * Y - Out = reference_matmul(X, Y, self.transpose_X, self.transpose_Y, - self.alpha) + Out = reference_matmul( + X, Y, self.transpose_X, self.transpose_Y, self.alpha + ) Out = Out.astype(self.dtype) self.inputs = {'X': X, 'Y': Y} self.attrs = { 'transpose_X': self.transpose_X, 'transpose_Y': self.transpose_Y, - 'alpha': self.alpha + 'alpha': self.alpha, } self.outputs = {'Out': Out} @@ -92,8 +93,8 @@ def set_mlu(self): self.place = paddle.device.MLUPlace(0) def config(self): - self.x_shape = (100, ) - self.y_shape = (100, ) + self.x_shape = (100,) + self.y_shape = (100,) self.transpose_X = False self.transpose_Y = False @@ -116,7 +117,7 @@ class TestMatMulOp1(TestMatMulOp): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 3, 2, 100) self.transpose_X = False self.transpose_Y = True @@ -129,7 +130,7 @@ class TestMatMulOp2(TestMatMulOp): def config(self): self.x_shape = (1, 2, 100, 1) - self.y_shape = (100, ) + self.y_shape = (100,) self.transpose_X = True self.transpose_Y = False @@ -248,7 +249,7 @@ class TestMatMulOp12(TestMatMulOp): """ def config(self): - self.x_shape = (100) + self.x_shape = 100 self.y_shape = (1, 2, 2, 100, 2) self.transpose_X = False self.transpose_Y = False @@ -261,13 +262,13 @@ class TestMatMulOp13(TestMatMulOp): def config(self): self.x_shape = (2, 1, 100) - self.y_shape = (100) + self.y_shape = 100 self.transpose_X = False self.transpose_Y = False # TODO(mlu): alpha will be supported in next version -#--------------------test matmul alpha-------------------- +# --------------------test matmul alpha-------------------- # def create_test_alpha_class(parent): # class TestMatMulOpAlphaCase(parent): # def init_alpha(self): @@ -291,11 +292,9 @@ def config(self): # create_test_alpha_class(TestMatMulOp13) -#--------------------test matmul fp16-------------------- +# --------------------test matmul fp16-------------------- def create_test_fp16_class(parent, atol=0.001, max_relative_error=2.5): - class TestMatMulOpFp16Case(parent): - def init_kernel_type(self): self.dtype = np.float16 @@ -303,9 +302,12 @@ def test_check_output(self): self.check_output_with_place(self.place, atol=atol) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=max_relative_error) + self.check_grad_with_place( + self.place, + ['X', 'Y'], + 'Out', + max_relative_error=max_relative_error, + ) cls_name = "{0}_{1}".format(parent.__name__, "Fp16") TestMatMulOpFp16Case.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/mlu/test_matmul_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_matmul_v2_op_mlu.py index 4c782980b3db9..f1b142902b104 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_matmul_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_matmul_v2_op_mlu.py @@ -36,8 +36,8 @@ def set_mlu(self): self.place = paddle.device.MLUPlace(0) def config(self): - self.x_shape = (100, ) - self.y_shape = (100, ) + self.x_shape = (100,) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -76,7 +76,7 @@ class TestMatMuklOp2(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 3, 2, 100) self.trans_x = False self.trans_y = True @@ -88,7 +88,7 @@ class TestMatMuklOp3(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 1, 100, 2) self.trans_x = False self.trans_y = False @@ -100,7 +100,7 @@ class TestMatMuklOp4(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 2, 100, 2) self.trans_x = False self.trans_y = False @@ -113,7 +113,7 @@ class TestMatMuklOp5(TestMatMulV2Op): def config(self): self.x_shape = (1, 1, 100, 1) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = True self.trans_y = False @@ -125,7 +125,7 @@ class TestMatMuklOp6(TestMatMulV2Op): def config(self): self.x_shape = (1, 2, 102, 1) - self.y_shape = (102, ) + self.y_shape = (102,) self.trans_x = True self.trans_y = False @@ -137,7 +137,7 @@ class TestMatMuklOp7(TestMatMulV2Op): def config(self): self.x_shape = (1, 2, 1, 100) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -244,7 +244,7 @@ class TestMatMuklOp16(TestMatMulV2Op): """ def config(self): - self.x_shape = (100) + self.x_shape = 100 self.y_shape = (1, 2, 2, 100, 2) self.trans_x = False self.trans_y = False @@ -257,7 +257,7 @@ class TestMatMuklOp17(TestMatMulV2Op): def config(self): self.x_shape = (2, 1, 100) - self.y_shape = (100) + self.y_shape = 100 self.trans_x = False self.trans_y = False @@ -298,13 +298,11 @@ def config(self): self.trans_y = True -#--------------------test matmul fp16-------------------- +# --------------------test matmul fp16-------------------- def create_test_fp16_class(parent, atol=0.001, max_relative_error=2.5): - class TestMatMulOpFp16Case(parent): - def init_kernel_type(self): self.dtype = np.float16 @@ -312,9 +310,12 @@ def test_check_output(self): self.check_output_with_place(self.place, atol=atol) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=max_relative_error) + self.check_grad_with_place( + self.place, + ['X', 'Y'], + 'Out', + max_relative_error=max_relative_error, + ) cls_name = "{0}_{1}".format(parent.__name__, "Fp16") TestMatMulOpFp16Case.__name__ = cls_name @@ -342,7 +343,6 @@ def test_check_grad(self): class TestMatMulV2API(unittest.TestCase): - def setUp(self): self.places = [paddle.CPUPlace()] if paddle.is_compiled_with_mlu(): @@ -359,12 +359,11 @@ def check_static_result(self, place): y_np = np.random.random([3, 4]).astype("float32") exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input_x": x_np, - "input_y": y_np - }, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": x_np, "input_y": y_np}, + fetch_list=[result], + ) def test_static(self): for place in self.places: diff --git a/python/paddle/fluid/tests/unittests/mlu/test_mean_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_mean_op_mlu.py index 5dcdf1db8458d..60f9d7b8e45b2 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_mean_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_mean_op_mlu.py @@ -27,7 +27,6 @@ class TestMean(OpTest): - def setUp(self): self.set_mlu() self.place = paddle.device.MLUPlace(0) @@ -55,7 +54,6 @@ def test_check_grad(self): class TestMeanFP16(OpTest): - def setUp(self): self.set_mlu() self.place = paddle.MLUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_merged_adam_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_merged_adam_op_mlu.py index 3aa61e9f982fb..242e1c8e663f4 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_merged_adam_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_merged_adam_op_mlu.py @@ -21,20 +21,22 @@ from paddle.fluid.framework import in_dygraph_mode -def run_adam_op(params, - grads, - lrs, - moment1s, - moment2s, - beta1_pows, - beta2_pows, - master_params, - epsilon, - beta1, - beta2, - place, - multi_precision=False, - use_merged=False): +def run_adam_op( + params, + grads, + lrs, + moment1s, + moment2s, + beta1_pows, + beta2_pows, + master_params, + epsilon, + beta1, + beta2, + place, + multi_precision=False, + use_merged=False, +): assert len(params) == len(grads) assert len(params) == len(lrs) assert len(params) == len(moment1s) @@ -59,25 +61,71 @@ def run_adam_op(params, if not use_merged: for i in range(len(param_vars)): _, _, _, _, _, _ = _legacy_C_ops.adam( - param_vars[i], grad_vars[i], lr_vars[i], moment1_vars[i], - moment2_vars[i], beta1_pow_vars[i], beta2_pow_vars[i], - master_param_vars[i], param_vars[i], moment1_vars[i], - moment2_vars[i], beta1_pow_vars[i], beta2_pow_vars[i], - master_param_vars[i], 'epsilon', epsilon, 'beta1', beta1, - 'beta2', beta2, 'multi_precision', multi_precision) + param_vars[i], + grad_vars[i], + lr_vars[i], + moment1_vars[i], + moment2_vars[i], + beta1_pow_vars[i], + beta2_pow_vars[i], + master_param_vars[i], + param_vars[i], + moment1_vars[i], + moment2_vars[i], + beta1_pow_vars[i], + beta2_pow_vars[i], + master_param_vars[i], + 'epsilon', + epsilon, + 'beta1', + beta1, + 'beta2', + beta2, + 'multi_precision', + multi_precision, + ) else: if in_dygraph_mode(): _, _, _, _, _, _ = _C_ops.merged_adam_( - param_vars, grad_vars, lr_vars, moment1_vars, moment2_vars, - beta1_pow_vars, beta2_pow_vars, master_param_vars, beta1, beta2, - epsilon, multi_precision, False) + param_vars, + grad_vars, + lr_vars, + moment1_vars, + moment2_vars, + beta1_pow_vars, + beta2_pow_vars, + master_param_vars, + beta1, + beta2, + epsilon, + multi_precision, + False, + ) else: _, _, _, _, _, _ = _legacy_C_ops.merged_adam( - param_vars, grad_vars, lr_vars, moment1_vars, moment2_vars, - beta1_pow_vars, beta2_pow_vars, master_param_vars, param_vars, - moment1_vars, moment2_vars, beta1_pow_vars, beta2_pow_vars, - master_param_vars, 'epsilon', epsilon, 'beta1', beta1, 'beta2', - beta2, 'multi_precision', multi_precision) + param_vars, + grad_vars, + lr_vars, + moment1_vars, + moment2_vars, + beta1_pow_vars, + beta2_pow_vars, + master_param_vars, + param_vars, + moment1_vars, + moment2_vars, + beta1_pow_vars, + beta2_pow_vars, + master_param_vars, + 'epsilon', + epsilon, + 'beta1', + beta1, + 'beta2', + beta2, + 'multi_precision', + multi_precision, + ) outputs = { 'ParamOut': param_vars, @@ -85,14 +133,13 @@ def run_adam_op(params, 'Moment2Out': moment2_vars, 'Beta1PowOut': beta1_pow_vars, 'Beta2PowOut': beta2_pow_vars, - 'MasterParamOut': master_param_vars + 'MasterParamOut': master_param_vars, } return outputs class TestMergedAdam(unittest.TestCase): - def setUp(self): paddle.disable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -116,27 +163,46 @@ def prepare_data(self, shapes, multi_precision, seed, place): beta1_pows = self.gen_rand_data([[1], [1], [1], [1]], mp_dtype) beta2_pows = self.gen_rand_data([[1], [1], [1], [1]], mp_dtype) master_params = [p.astype(mp_dtype) for p in params] - return params, grads, lrs, moment1s, moment2s, beta1_pows, beta2_pows, master_params + return ( + params, + grads, + lrs, + moment1s, + moment2s, + beta1_pows, + beta2_pows, + master_params, + ) def check_with_place(self, place, multi_precision): - params, grads, lrs, moment1s, moment2s, beta1_pows, beta2_pows, master_params = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + lrs, + moment1s, + moment2s, + beta1_pows, + beta2_pows, + master_params, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_merged): - return run_adam_op(params=params, - grads=grads, - lrs=lrs, - moment1s=moment1s, - moment2s=moment2s, - beta1_pows=beta1_pows, - beta2_pows=beta2_pows, - master_params=master_params, - epsilon=0.9, - beta1=0.9, - beta2=0.99, - place=place, - multi_precision=multi_precision, - use_merged=use_merged) + return run_adam_op( + params=params, + grads=grads, + lrs=lrs, + moment1s=moment1s, + moment2s=moment2s, + beta1_pows=beta1_pows, + beta2_pows=beta2_pows, + master_params=master_params, + epsilon=0.9, + beta1=0.9, + beta2=0.99, + place=place, + multi_precision=multi_precision, + use_merged=use_merged, + ) outs1 = run_op(True) outs2 = run_op(False) @@ -149,10 +215,9 @@ def run_op(use_merged): if place == 'mlu': np.testing.assert_array_equal(value1[i], value2[i]) else: - np.testing.assert_allclose(value1[i], - value2[i], - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + value1[i], value2[i], rtol=1e-05, atol=1e-07 + ) def test_main(self): for multi_precision in [False, True]: diff --git a/python/paddle/fluid/tests/unittests/mlu/test_merged_momentum_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_merged_momentum_op_mlu.py index 7a1f590cf3da6..e7c4c4bb4ca53 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_merged_momentum_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_merged_momentum_op_mlu.py @@ -22,16 +22,18 @@ from collections import OrderedDict -def run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - mu=0.9, - rescale_grad=0.01, - use_merged=False): +def run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + mu=0.9, + rescale_grad=0.01, + use_merged=False, +): assert len(params) == len(grads) assert len(params) == len(velocitys) if multi_precision: @@ -48,48 +50,70 @@ def run_momentum_op(params, } param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in params ] grad_vars = [ helper.create_variable(shape=g.shape, dtype=g.dtype) for g in grads ] velocity_vars = [ - helper.create_variable(persistable=True, - shape=v.shape, - dtype=v.dtype) for v in velocitys + helper.create_variable( + persistable=True, shape=v.shape, dtype=v.dtype + ) + for v in velocitys ] - lr_var = helper.create_variable(persistable=True, - shape=learning_rate.shape, - dtype=learning_rate.dtype) + lr_var = helper.create_variable( + persistable=True, + shape=learning_rate.shape, + dtype=learning_rate.dtype, + ) feed_dict = OrderedDict() feed_dict.update( - OrderedDict([(p_var.name, p_val) - for p_var, p_val in zip(param_vars, params)])) + OrderedDict( + [ + (p_var.name, p_val) + for p_var, p_val in zip(param_vars, params) + ] + ) + ) feed_dict.update( - OrderedDict([(v_var.name, v_val) - for v_var, v_val in zip(velocity_vars, velocitys)])) + OrderedDict( + [ + (v_var.name, v_val) + for v_var, v_val in zip(velocity_vars, velocitys) + ] + ) + ) fetch_list = list(feed_dict.keys()) feed_dict.update( - OrderedDict([(g_var.name, g_val) - for g_var, g_val in zip(grad_vars, grads)])) + OrderedDict( + [(g_var.name, g_val) for g_var, g_val in zip(grad_vars, grads)] + ) + ) feed_dict.update({lr_var.name: learning_rate}) if multi_precision: master_param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in master_params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in master_params ] feed_dict.update( - OrderedDict([ - (mp_var.name, mp_val) - for mp_var, mp_val in zip(master_param_vars, master_params) - ])) + OrderedDict( + [ + (mp_var.name, mp_val) + for mp_var, mp_val in zip( + master_param_vars, master_params + ) + ] + ) + ) # CPUPlace does not use MasterParam if isinstance(place, paddle.CUDAPlace): fetch_list = fetch_list + [ @@ -99,8 +123,9 @@ def run_momentum_op(params, master_param_vars = None if not use_merged: - for i, (p, g, - v) in enumerate(zip(param_vars, grad_vars, velocity_vars)): + for i, (p, g, v) in enumerate( + zip(param_vars, grad_vars, velocity_vars) + ): inputs = { 'Param': p, 'Grad': g, @@ -111,10 +136,9 @@ def run_momentum_op(params, if multi_precision: inputs['MasterParam'] = master_param_vars[i] outputs['MasterParamOut'] = master_param_vars[i] - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) else: inputs = { 'Param': param_vars, @@ -126,10 +150,9 @@ def run_momentum_op(params, if multi_precision: inputs['MasterParam'] = master_param_vars outputs['MasterParamOut'] = master_param_vars - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): @@ -137,17 +160,19 @@ def run_momentum_op(params, return exe.run(main, feed=feed_dict, fetch_list=fetch_list) -def run_momentum_op2(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - mu=0.9, - rescale_grad=0.01, - use_merged=False, - use_nesterov=True): +def run_momentum_op2( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + mu=0.9, + rescale_grad=0.01, + use_merged=False, + use_nesterov=True, +): assert len(params) == len(grads) assert len(params) == len(velocitys) if multi_precision: @@ -159,48 +184,70 @@ def run_momentum_op2(params, helper = LayerHelper(op_type, **locals()) param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in params ] grad_vars = [ helper.create_variable(shape=g.shape, dtype=g.dtype) for g in grads ] velocity_vars = [ - helper.create_variable(persistable=True, - shape=v.shape, - dtype=v.dtype) for v in velocitys + helper.create_variable( + persistable=True, shape=v.shape, dtype=v.dtype + ) + for v in velocitys ] - lr_var = helper.create_variable(persistable=True, - shape=learning_rate.shape, - dtype=learning_rate.dtype) + lr_var = helper.create_variable( + persistable=True, + shape=learning_rate.shape, + dtype=learning_rate.dtype, + ) feed_dict = OrderedDict() feed_dict.update( - OrderedDict([(p_var.name, p_val) - for p_var, p_val in zip(param_vars, params)])) + OrderedDict( + [ + (p_var.name, p_val) + for p_var, p_val in zip(param_vars, params) + ] + ) + ) feed_dict.update( - OrderedDict([(v_var.name, v_val) - for v_var, v_val in zip(velocity_vars, velocitys)])) + OrderedDict( + [ + (v_var.name, v_val) + for v_var, v_val in zip(velocity_vars, velocitys) + ] + ) + ) fetch_list = list(feed_dict.keys()) feed_dict.update( - OrderedDict([(g_var.name, g_val) - for g_var, g_val in zip(grad_vars, grads)])) + OrderedDict( + [(g_var.name, g_val) for g_var, g_val in zip(grad_vars, grads)] + ) + ) feed_dict.update({lr_var.name: learning_rate}) if multi_precision: master_param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in master_params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in master_params ] feed_dict.update( - OrderedDict([ - (mp_var.name, mp_val) - for mp_var, mp_val in zip(master_param_vars, master_params) - ])) + OrderedDict( + [ + (mp_var.name, mp_val) + for mp_var, mp_val in zip( + master_param_vars, master_params + ) + ] + ) + ) # CPUPlace does not use MasterParam if isinstance(place, paddle.CUDAPlace): fetch_list = fetch_list + [ @@ -210,8 +257,9 @@ def run_momentum_op2(params, master_param_vars = None if not use_merged: - for i, (p, g, - v) in enumerate(zip(param_vars, grad_vars, velocity_vars)): + for i, (p, g, v) in enumerate( + zip(param_vars, grad_vars, velocity_vars) + ): inputs = { 'Param': p, 'Grad': g, @@ -230,10 +278,9 @@ def run_momentum_op2(params, 'regularization_method': 'l2_decay', 'regularization_coeff': 2.0, } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) else: inputs = { 'Param': param_vars, @@ -246,22 +293,18 @@ def run_momentum_op2(params, inputs['MasterParam'] = master_param_vars outputs['MasterParamOut'] = master_param_vars attrs = { - 'mu': - mu, - 'multi_precision': - multi_precision, - 'rescale_grad': - rescale_grad, - 'use_nesterov': - use_nesterov, - 'regularization_method': - ['l2_decay' for i in range(len(param_vars))], + 'mu': mu, + 'multi_precision': multi_precision, + 'rescale_grad': rescale_grad, + 'use_nesterov': use_nesterov, + 'regularization_method': [ + 'l2_decay' for i in range(len(param_vars)) + ], 'regularization_coeff': [2.0 for i in range(len(param_vars))], } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): @@ -270,7 +313,6 @@ def run_momentum_op2(params, class TestMergedMomentum(unittest.TestCase): - def setUp(self): paddle.enable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -296,21 +338,28 @@ def prepare_data(self, shapes, multi_precision, seed, place): return params, grads, velocitys, master_params, learning_rate def check_with_place(self, place, multi_precision): - params, grads, velocitys, master_params, learning_rate = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + velocitys, + master_params, + learning_rate, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_merged): # MLU Momentum Op does not support rescale_grad rescale_grad = 1.0 - return run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - rescale_grad=rescale_grad, - use_merged=use_merged) + return run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + rescale_grad=rescale_grad, + use_merged=use_merged, + ) outs1 = run_op(True) outs2 = run_op(False) @@ -323,7 +372,6 @@ def test_main(self): class TestMergedMomentum2(unittest.TestCase): - def setUp(self): paddle.enable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -349,22 +397,29 @@ def prepare_data(self, shapes, multi_precision, seed, place): return params, grads, velocitys, master_params, learning_rate def check_with_place(self, place, multi_precision): - params, grads, velocitys, master_params, learning_rate = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + velocitys, + master_params, + learning_rate, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_nesterov, use_merged): # MLU Momentum Op does not support rescale_grad rescale_grad = 1.0 - return run_momentum_op2(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - rescale_grad=rescale_grad, - use_merged=use_merged, - use_nesterov=use_nesterov) + return run_momentum_op2( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + rescale_grad=rescale_grad, + use_merged=use_merged, + use_nesterov=use_nesterov, + ) outs1 = run_op(use_nesterov=True, use_merged=True) outs2 = run_op(use_nesterov=True, use_merged=False) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_meshgrid_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_meshgrid_op_mlu.py index d0c5969226524..6353ca638a5ce 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_meshgrid_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_meshgrid_op_mlu.py @@ -27,7 +27,6 @@ class TestMeshgridOp(OpTest): - def setUp(self): self.op_type = "meshgrid" self.place = paddle.device.MLUPlace(0) @@ -50,7 +49,7 @@ def init_test_data(self): ins = [] outs = [] for i in range(len(self.shape)): - ins.append(np.random.random((self.shape[i], )).astype(self.dtype)) + ins.append(np.random.random((self.shape[i],)).astype(self.dtype)) for i in range(len(self.shape)): out_reshape = [1] * len(self.shape) @@ -64,23 +63,29 @@ def get_x_shape(self): class TestMeshgridOp2(TestMeshgridOp): - def get_x_shape(self): return [100, 300] class TestMeshgridOp3(unittest.TestCase): - def test_api(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, + 100, + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -89,28 +94,34 @@ def test_api(self): exe = fluid.Executor(place=fluid.MLUPlace(0)) grid_x, grid_y = paddle.tensor.meshgrid(x, y) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) assert np.array_equal(res_1, out_1) assert np.array_equal(res_2, out_2) class TestMeshgridOp4(unittest.TestCase): - def test_list_input(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -119,29 +130,35 @@ def test_list_input(self): exe = fluid.Executor(place=fluid.MLUPlace(0)) grid_x, grid_y = paddle.tensor.meshgrid([x, y]) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) assert np.array_equal(res_1, out_1) assert np.array_equal(res_2, out_2) class TestMeshgridOp5(unittest.TestCase): - def test_tuple_input(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -150,26 +167,32 @@ def test_tuple_input(self): exe = fluid.Executor(place=fluid.MLUPlace(0)) grid_x, grid_y = paddle.tensor.meshgrid((x, y)) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) assert np.array_equal(res_1, out_1) assert np.array_equal(res_2, out_2) class TestMeshgridOp7(unittest.TestCase): - def test_api_with_dygraph_list_input(self): - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, + 100, + [ + 200, + ], + ).astype('int32') with fluid.dygraph.guard(): tensor_3 = fluid.dygraph.to_variable(input_3) @@ -181,14 +204,21 @@ def test_api_with_dygraph_list_input(self): class TestMeshgridOp8(unittest.TestCase): - def test_api_with_dygraph_tuple_input(self): - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') with fluid.dygraph.guard(): tensor_3 = fluid.dygraph.to_variable(input_3) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_momentum_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_momentum_op_mlu.py index 542675ada9f4b..8efc129b68885 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_momentum_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_momentum_op_mlu.py @@ -29,7 +29,6 @@ class TestMomentumOp1(OpTest): - def setUp(self): self.op_type = "momentum" self.dtype = np.float32 @@ -47,7 +46,7 @@ def setUp(self): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = {'mu': mu} @@ -58,7 +57,8 @@ def setUp(self): mu=mu, velocity=velocity, use_nesterov=use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -74,7 +74,6 @@ def test_check_output(self): class TestMomentumOpFp16(TestMomentumOp1): - def init_dtype(self): self.dtype = np.float16 @@ -83,8 +82,7 @@ def test_check_output(self): class TestMomentumOp2(OpTest): - '''Test Momentum with default values for attributes - ''' + '''Test Momentum with default values for attributes''' def setUp(self): self.op_type = "momentum" @@ -102,7 +100,7 @@ def setUp(self): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = {'mu': mu, 'use_nesterov': use_nesterov} @@ -113,7 +111,8 @@ def setUp(self): mu=mu, velocity=velocity, use_nesterov=use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -122,16 +121,15 @@ def test_check_output(self): class TestMomentumV2(unittest.TestCase): - def test_momentum_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Momentum(learning_rate=0.01, - momentum=0.9, - parameters=linear.parameters()) + adam = paddle.optimizer.Momentum( + learning_rate=0.01, momentum=0.9, parameters=linear.parameters() + ) out = linear(a) out.backward() adam.step() @@ -148,13 +146,15 @@ def test_momentum(self): cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) - rms_optimizer = paddle.optimizer.Momentum(learning_rate=0.1, - momentum=0.9) + rms_optimizer = paddle.optimizer.Momentum( + learning_rate=0.1, momentum=0.9 + ) rms_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -162,14 +162,13 @@ def test_momentum(self): exe.run(main, feed=feeder.feed(data), fetch_list=fetch_list) def test_raise_error(self): - self.assertRaises(ValueError, - paddle.optimizer.Momentum, - learning_rate=None) + self.assertRaises( + ValueError, paddle.optimizer.Momentum, learning_rate=None + ) self.assertRaises(ValueError, paddle.optimizer.Momentum, momentum=None) class TestMomentumOpWithDecay(OpTest): - def setUp(self): self.op_type = "momentum" self.place = paddle.device.MLUPlace(0) @@ -193,14 +192,14 @@ def setUp(self): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = { 'mu': mu, 'use_nesterov': use_nesterov, 'regularization_method': regularization_method, - 'regularization_coeff': regularization_coeff + 'regularization_coeff': regularization_coeff, } grad = grad + regularization_coeff * param @@ -211,7 +210,8 @@ def setUp(self): mu=mu, velocity=velocity, use_nesterov=use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -224,7 +224,6 @@ def test_check_output(self): class TestMomentumOpWithDecayFP16(TestMomentumOpWithDecay): - def init_config(self): self.dtype = np.float16 @@ -233,13 +232,11 @@ def test_check_output(self): class TestMomentumOpWithDecay2(TestMomentumOpWithDecay): - def init_config(self): self.use_nesterov = False class TestMomentumOpWithDecayAPI(unittest.TestCase): - def _test_momentum_dygraph_common(self, regularization): paddle.disable_static() inp = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32") @@ -252,13 +249,16 @@ def _test_momentum_dygraph_common(self, regularization): learning_rate=0.01, momentum=0.9, parameter_list=linear.parameters(), - regularization=regularization) + regularization=regularization, + ) momentum.minimize(loss) def test_momentum_dygraph_1(self): self._test_momentum_dygraph_common( regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ) + ) def test_momentum_static(self): paddle.enable_static() @@ -272,12 +272,14 @@ def test_momentum_static(self): avg_cost = paddle.mean(cost) momentum_optimizer = paddle.fluid.contrib.optimizer.Momentum( - learning_rate=0.1, momentum=0.9) + learning_rate=0.1, momentum=0.9 + ) momentum_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -286,23 +288,23 @@ def test_momentum_static(self): class TestFusedMomentumWithDecayAPI(unittest.TestCase): - def get_program(self, weight_attr, bias_attr=False): main_program = paddle.static.Program() startup_program = paddle.static.Program() - with paddle.static.program_guard(main_program=main_program, - startup_program=startup_program): + with paddle.static.program_guard( + main_program=main_program, startup_program=startup_program + ): x = paddle.static.data(name='x', shape=[10, 10]) - linear = paddle.nn.Linear(10, - 10, - weight_attr=weight_attr, - bias_attr=bias_attr) + linear = paddle.nn.Linear( + 10, 10, weight_attr=weight_attr, bias_attr=bias_attr + ) out = linear(x) loss = paddle.mean(out) optimizer = paddle.optimizer.Momentum( learning_rate=0.01, momentum=0.9, - weight_decay=paddle.regularizer.L2Decay(0.5)) + weight_decay=paddle.regularizer.L2Decay(0.5), + ) optimizer.minimize(loss) return main_program @@ -311,7 +313,8 @@ def test_param_has_l2decay(self): weight_attr = paddle.ParamAttr( name="weight", initializer=paddle.nn.initializer.Constant(value=0.5), - regularizer=paddle.regularizer.L2Decay(0.1)) + regularizer=paddle.regularizer.L2Decay(0.1), + ) program = self.get_program(weight_attr, bias_attr=False) ops = program.global_block().ops @@ -326,11 +329,13 @@ def test_param_has_l1decay(self): weight_attr = paddle.ParamAttr( name="weight", initializer=paddle.nn.initializer.Constant(value=0.5), - regularizer=paddle.regularizer.L1Decay(0.1)) + regularizer=paddle.regularizer.L1Decay(0.1), + ) bias_attr = paddle.ParamAttr( name="bias", - initializer=paddle.nn.initializer.Constant(value=0.), - regularizer=None) + initializer=paddle.nn.initializer.Constant(value=0.0), + regularizer=None, + ) program = self.get_program(weight_attr, bias_attr) ops = program.global_block().ops @@ -345,8 +350,9 @@ def test_param_has_l1decay(self): self.assertEqual(ops[-1].attr('regularization_coeff'), 0) if 'bias' in ops[-2].input('Param'): self.assertEqual(ops[-2].attr('regularization_method'), 'l2_decay') - self.assertEqual(ops[-2].attr('regularization_coeff'), - np.float32(0.5)) + self.assertEqual( + ops[-2].attr('regularization_coeff'), np.float32(0.5) + ) def test_param_has_no_regularizer(self): paddle.enable_static() @@ -360,11 +366,11 @@ def test_param_has_no_regularizer(self): class TestMomentumOpVsMomentumOpWithDecayAPI(unittest.TestCase): - def __update_params(self, momentum, linear): for i in range(10): - inp = paddle.full(shape=[2, 2], fill_value=i, - dtype='float32').astype("float32") + inp = paddle.full( + shape=[2, 2], fill_value=i, dtype='float32' + ).astype("float32") inp = paddle.to_tensor(inp) out = linear(inp) loss = paddle.mean(out) @@ -379,32 +385,39 @@ def __test_vs(self, place=fluid.MLUPlace(0)): 2, 2, weight_attr=paddle.nn.initializer.Constant(value=2.0), - bias_attr=paddle.nn.initializer.Constant(value=2.0)) + bias_attr=paddle.nn.initializer.Constant(value=2.0), + ) momentum_old = paddle.fluid.optimizer.Momentum( learning_rate=0.01, momentum=0.9, parameter_list=linear_old.parameters(), regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ), + ) self.__update_params(momentum=momentum_old, linear=linear_old) linear_new = paddle.nn.Linear( 2, 2, weight_attr=paddle.nn.initializer.Constant(value=2.0), - bias_attr=paddle.nn.initializer.Constant(value=2.0)) + bias_attr=paddle.nn.initializer.Constant(value=2.0), + ) momentum_new = paddle.fluid.contrib.optimizer.Momentum( learning_rate=0.01, momentum=0.9, parameter_list=linear_new.parameters(), regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ), + ) self.__update_params(momentum=momentum_new, linear=linear_new) self.assertEqual( (linear_old.weight.numpy() == linear_new.weight.numpy()).all(), True, - 'the param weight updated by two Momentum optimizers should equal') + 'the param weight updated by two Momentum optimizers should equal', + ) def test_vs(self, place=fluid.MLUPlace(0)): places = [fluid.MLUPlace(0)] @@ -413,7 +426,6 @@ def test_vs(self, place=fluid.MLUPlace(0)): class TestMomentumV2Group(TestMomentumV2): - def test_momentum_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") @@ -421,22 +433,20 @@ def test_momentum_dygraph(self): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Momentum(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001, - 'learning_rate': - 0.1, - 'momentum': - 0.99 - }], - weight_decay=0.1, - momentum=0.9) + adam = paddle.optimizer.Momentum( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + 'learning_rate': 0.1, + 'momentum': 0.99, + }, + ], + weight_decay=0.1, + momentum=0.9, + ) out = linear_1(a) out = linear_2(out) out.backward() @@ -445,13 +455,14 @@ def test_momentum_dygraph(self): class TestMultiTensorMomentumDygraph(unittest.TestCase): - - def _momentum_optimize_dygraph(self, - place, - use_param_attr=False, - use_param_group=False, - use_amp=False, - use_multi_tensor=False): + def _momentum_optimize_dygraph( + self, + place, + use_param_attr=False, + use_param_group=False, + use_amp=False, + use_multi_tensor=False, + ): paddle.disable_static() paddle.seed(10) paddle.set_device(place) @@ -459,7 +470,8 @@ def _momentum_optimize_dygraph(self, weight_attr = paddle.ParamAttr( learning_rate=0.5, regularizer=paddle.regularizer.L2Decay(1.0), - trainable=True) + trainable=True, + ) if use_param_attr: model = paddle.nn.Linear(5, 5, weight_attr) else: @@ -468,17 +480,21 @@ def _momentum_optimize_dygraph(self, optimizer = paddle.optimizer.Momentum( parameters=model.parameters(), use_multi_tensor=use_multi_tensor, - multi_precision=use_amp) + multi_precision=use_amp, + ) else: optimizer = paddle.optimizer.Momentum( - parameters=[{ - 'params': model.parameters(), - 'weight_decay': 0.001, - 'learning_rate': 0.1, - 'momentum': 0.99 - }], + parameters=[ + { + 'params': model.parameters(), + 'weight_decay': 0.001, + 'learning_rate': 0.1, + 'momentum': 0.99, + } + ], use_multi_tensor=use_multi_tensor, - multi_precision=use_amp) + multi_precision=use_amp, + ) for idx in range(5): if place == 'mlu' and use_amp == True: model = paddle.amp.decorate(models=model, level='O2') @@ -506,9 +522,11 @@ def _get_places(self): def _check_with_place_amp(self, place, use_amp): output1, params1 = self._momentum_optimize_dygraph( - place=place, use_amp=use_amp, use_multi_tensor=True) + place=place, use_amp=use_amp, use_multi_tensor=True + ) output2, params2 = self._momentum_optimize_dygraph( - place=place, use_amp=use_amp, use_multi_tensor=False) + place=place, use_amp=use_amp, use_multi_tensor=False + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): np.testing.assert_allclose(params1[idx], params2[idx], rtol=1e-05) @@ -518,12 +536,14 @@ def _check_with_param_arrt(self, place, use_amp): place=place, use_amp=use_amp, use_param_attr=True, - use_multi_tensor=True) + use_multi_tensor=True, + ) output2, params2 = self._momentum_optimize_dygraph( place=place, use_amp=use_amp, use_param_attr=True, - use_multi_tensor=False) + use_multi_tensor=False, + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): np.testing.assert_allclose(params1[idx], params2[idx], rtol=1e-05) @@ -533,12 +553,14 @@ def _check_with_param_group(self, place, use_amp): place=place, use_amp=use_amp, use_param_group=True, - use_multi_tensor=True) + use_multi_tensor=True, + ) output2, params2 = self._momentum_optimize_dygraph( place=place, use_amp=use_amp, use_param_group=True, - use_multi_tensor=False) + use_multi_tensor=False, + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): np.testing.assert_allclose(params1[idx], params2[idx], rtol=1e-05) @@ -554,11 +576,9 @@ def test_main(self): class TestMultiTensorMomentumStatic(unittest.TestCase): - - def _momentum_optimize_static(self, - place, - use_amp=False, - use_multi_tensor=False): + def _momentum_optimize_static( + self, place, use_amp=False, use_multi_tensor=False + ): paddle.enable_static() paddle.seed(10) np.random.seed(10) @@ -567,24 +587,26 @@ def _momentum_optimize_static(self, exe = paddle.static.Executor(place=paddle.device.MLUPlace(0)) train_program = paddle.static.Program() startup_program = paddle.static.Program() - optimizer = paddle.optimizer.Momentum(multi_precision=use_amp, - use_multi_tensor=use_multi_tensor) + optimizer = paddle.optimizer.Momentum( + multi_precision=use_amp, use_multi_tensor=use_multi_tensor + ) if use_amp: optimizer = paddle.static.amp.decorate( optimizer, init_loss_scaling=128.0, use_dynamic_loss_scaling=True, use_pure_fp16=True, - use_fp16_guard=False) + use_fp16_guard=False, + ) with paddle.static.program_guard(train_program, startup_program): if use_amp: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float16') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float16' + ) else: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float32') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float32' + ) hidden = paddle.static.nn.fc(x=data, size=10) loss = paddle.mean(hidden) optimizer.minimize(loss) @@ -596,9 +618,9 @@ def _momentum_optimize_static(self, x = numpy.random.random(size=(2, 2)).astype('float32') out = [] for idx in range(5): - loss_data, = exe.run(train_program, - feed={"X": x}, - fetch_list=[loss.name]) + (loss_data,) = exe.run( + train_program, feed={"X": x}, fetch_list=[loss.name] + ) out.append(loss_data) return out @@ -607,12 +629,12 @@ def _get_places(self): return places def _check_with_place_amp(self, place, use_amp): - output1 = self._momentum_optimize_static(place=place, - use_amp=use_amp, - use_multi_tensor=True) - output2 = self._momentum_optimize_static(place=place, - use_amp=use_amp, - use_multi_tensor=False) + output1 = self._momentum_optimize_static( + place=place, use_amp=use_amp, use_multi_tensor=True + ) + output2 = self._momentum_optimize_static( + place=place, use_amp=use_amp, use_multi_tensor=False + ) for idx in range(len(output1)): np.testing.assert_allclose(output1[idx], output2[idx], rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_nearest_interp_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_nearest_interp_v2_op_mlu.py index 6cf320d56b144..1ab286740ef2d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_nearest_interp_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_nearest_interp_v2_op_mlu.py @@ -27,15 +27,17 @@ paddle.enable_static() -def nearest_neighbor_interp_np(X, - out_h, - out_w, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='NCHW'): +def nearest_neighbor_interp_np( + X, + out_h, + out_w, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='NCHW', +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 3, 1, 2)) # NHWC => NCHW @@ -48,16 +50,16 @@ def nearest_neighbor_interp_np(X, n, c, in_h, in_w = X.shape ratio_h = ratio_w = 0.0 - if (out_h > 1): - if (align_corners): + if out_h > 1: + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: ratio_h = 1.0 / scale_h else: ratio_h = 1.0 * in_h / out_h - if (out_w > 1): - if (align_corners): + if out_w > 1: + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -85,17 +87,19 @@ def nearest_neighbor_interp_np(X, return out.astype(X.dtype) -def nearest_neighbor_interp3d_np(X, - out_d, - out_h, - out_w, - scale_d=0, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='NCHW'): +def nearest_neighbor_interp3d_np( + X, + out_d, + out_h, + out_w, + scale_d=0, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='NCHW', +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 4, 1, 2, 3)) # NDHWC => NCDHW @@ -110,24 +114,24 @@ def nearest_neighbor_interp3d_np(X, n, c, in_d, in_h, in_w = X.shape ratio_d = ratio_h = ratio_w = 0.0 - if (out_d > 1): - if (align_corners): + if out_d > 1: + if align_corners: ratio_d = (in_d - 1.0) / (out_d - 1.0) else: if scale_d > 0: ratio_d = 1.0 / scale_d else: ratio_d = 1.0 * in_d / out_d - if (out_h > 1): - if (align_corners): + if out_h > 1: + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: ratio_h = 1.0 / scale_h else: ratio_h = 1.0 * in_h / out_h - if (out_w > 1): - if (align_corners): + if out_w > 1: + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -159,7 +163,6 @@ def nearest_neighbor_interp3d_np(X, class TestNearestInterpOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -216,15 +219,30 @@ def setUp(self): if len(self.input_shape) == 4: output_np = nearest_neighbor_interp_np( - input_np, out_h, out_w, scale_h, scale_w, self.out_size, - self.actual_shape, self.align_corners, self.data_layout) + input_np, + out_h, + out_w, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) elif len(self.input_shape) == 5: - output_np = nearest_neighbor_interp3d_np(input_np, out_d, out_h, - out_w, scale_d, scale_h, - scale_w, self.out_size, - self.actual_shape, - self.align_corners, - self.data_layout) + output_np = nearest_neighbor_interp3d_np( + input_np, + out_d, + out_h, + out_w, + scale_d, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -237,7 +255,7 @@ def setUp(self): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } else: self.attrs = { @@ -245,7 +263,7 @@ def setUp(self): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -267,7 +285,7 @@ def init_test_case(self): self.input_shape = [2, 3, 4, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True @@ -285,119 +303,108 @@ def init_test_case(self): class TestNearestNeighborInterpCase2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpCase3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpCase4(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = True class TestNearestNeighborInterpCase5(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") self.align_corners = True class TestNearestNeighborInterpCase6(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 129]).astype("int32") self.align_corners = True class TestNearestNeighborInterpSame(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpActualShape(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True class TestNearestNeighborInterpDataLayout(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 4, 4, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 8]).astype("int32") self.align_corners = True self.data_layout = "NHWC" class TestNearestInterpWithoutCorners(TestNearestInterpOp): - def set_align_corners(self): self.align_corners = False class TestNearestNeighborInterpScale1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 2. + self.scale = 2.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True class TestNearestNeighborInterpScale2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 5, 7] @@ -409,7 +416,6 @@ def init_test_case(self): class TestNearestNeighborInterpScale3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] @@ -421,7 +427,6 @@ def init_test_case(self): class TestNearestInterpOp_attr_tensor(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -461,8 +466,9 @@ def setUp(self): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h @@ -474,9 +480,16 @@ def setUp(self): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): @@ -490,33 +503,31 @@ def init_test_case(self): self.input_shape = [2, 5, 4, 4] self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [3, 3] self.align_corners = True # out_size is a tensor list class TestNearestInterp_attr_tensor_Case1(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = [8, 12] self.align_corners = True # out_size is a 1-D tensor class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.shape_by_1Dtensor = True @@ -524,7 +535,6 @@ def init_test_case(self): # scale is a 1-D tensor class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] @@ -537,7 +547,6 @@ def init_test_case(self): class TestNearestAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") y = fluid.data(name="y", shape=[2, 6, 6, 3], dtype="float32") @@ -545,27 +554,25 @@ def test_case(self): dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[2], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") - - out1 = fluid.layers.resize_nearest(y, - out_shape=[12, 12], - data_format='NHWC', - align_corners=False) - out2 = fluid.layers.resize_nearest(x, - out_shape=[12, dim], - align_corners=False) - out3 = fluid.layers.resize_nearest(x, - out_shape=shape_tensor, - align_corners=False) - out4 = fluid.layers.resize_nearest(x, - out_shape=[4, 4], - actual_shape=actual_size, - align_corners=False) - out5 = fluid.layers.resize_nearest(x, - scale=scale_tensor, - align_corners=False) + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) + + out1 = fluid.layers.resize_nearest( + y, out_shape=[12, 12], data_format='NHWC', align_corners=False + ) + out2 = fluid.layers.resize_nearest( + x, out_shape=[12, dim], align_corners=False + ) + out3 = fluid.layers.resize_nearest( + x, out_shape=shape_tensor, align_corners=False + ) + out4 = fluid.layers.resize_nearest( + x, out_shape=[4, 4], actual_shape=actual_size, align_corners=False + ) + out5 = fluid.layers.resize_nearest( + x, scale=scale_tensor, align_corners=False + ) x_data = np.random.random((2, 3, 6, 6)).astype("float32") dim_data = np.array([12]).astype("int32") @@ -576,39 +583,41 @@ def test_case(self): place = paddle.MLUPlace(0) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": np.transpose(x_data, (0, 2, 3, 1)), - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = nearest_neighbor_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=False) - np.testing.assert_allclose(results[0], - np.transpose(expect_res, (0, 2, 3, 1))) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": np.transpose(x_data, (0, 2, 3, 1)), + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = nearest_neighbor_interp_np( + x_data, out_h=12, out_w=12, align_corners=False + ) + np.testing.assert_allclose( + results[0], np.transpose(expect_res, (0, 2, 3, 1)) + ) for i in range(len(results) - 1): np.testing.assert_allclose(results[i + 1], expect_res) class TestNearestInterpException(unittest.TestCase): - def test_exception(self): import paddle + input = fluid.data(name="input", shape=[1, 3, 6, 6], dtype="float32") def attr_data_format(): # for 4-D input, data_format can only be NCHW or NHWC - out = fluid.layers.resize_nearest(input, - out_shape=[4, 8], - data_format='NDHWC') + out = fluid.layers.resize_nearest( + input, out_shape=[4, 8], data_format='NDHWC' + ) def attr_scale_type(): out = fluid.layers.resize_nearest(input, scale='scale') @@ -622,9 +631,9 @@ def input_shape_error(): def mode_error(): x = paddle.randn([1, 3]) - out = paddle.nn.functional.interpolate(x, - scale_factor='scale', - mode="BILINEAR") + out = paddle.nn.functional.interpolate( + x, scale_factor='scale', mode="BILINEAR" + ) self.assertRaises(ValueError, attr_data_format) self.assertRaises(TypeError, attr_scale_type) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_one_hot_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_one_hot_v2_op_mlu.py index cc6db21571376..3605ab254168d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_one_hot_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_one_hot_v2_op_mlu.py @@ -29,7 +29,6 @@ class TestOneHotOp(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -55,7 +54,6 @@ def test_check_output(self): class TestOneHotOp_attr(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -66,8 +64,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -83,7 +82,6 @@ def test_check_output(self): class TestOneHotOp_default_dtype(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -109,7 +107,6 @@ def test_check_output(self): class TestOneHotOp_default_dtype_attr(OpTest): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -120,8 +117,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -135,7 +133,6 @@ def test_check_output(self): class TestOneHotOp_exception(unittest.TestCase): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -153,30 +150,34 @@ def setUp(self): def test_check_output(self): program = Program() with program_guard(program): - x = fluid.layers.data(name='x', - shape=[self.dimension], - dtype='float32', - lod_level=1) + x = fluid.layers.data( + name='x', shape=[self.dimension], dtype='float32', lod_level=1 + ) block = program.current_block() - one_hot_out = block.create_var(name="one_hot_out", - type=core.VarDesc.VarType.LOD_TENSOR, - dtype='float32') - block.append_op(type='one_hot', - inputs={'X': x}, - attrs={'depth': self.depth}, - outputs={'Out': one_hot_out}) + one_hot_out = block.create_var( + name="one_hot_out", + type=core.VarDesc.VarType.LOD_TENSOR, + dtype='float32', + ) + block.append_op( + type='one_hot', + inputs={'X': x}, + attrs={'depth': self.depth}, + outputs={'Out': one_hot_out}, + ) exe = fluid.Executor(self.place) def run(): - exe.run(feed={'x': self.x}, - fetch_list=[one_hot_out], - return_numpy=False) + exe.run( + feed={'x': self.x}, + fetch_list=[one_hot_out], + return_numpy=False, + ) self.assertRaises(ValueError, run) class TestOneHotOpApi(unittest.TestCase): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -191,14 +192,17 @@ def test_api_with_depthTensor(self): def test_api_with_dygraph(self): depth = 10 - label = np.array([np.random.randint(0, depth - 1) - for i in range(6)]).reshape([6, 1]) + label = np.array( + [np.random.randint(0, depth - 1) for i in range(6)] + ).reshape([6, 1]) with fluid.dygraph.guard(): one_hot_label = fluid.one_hot( - input=fluid.dygraph.to_variable(label), depth=depth) + input=fluid.dygraph.to_variable(label), depth=depth + ) one_hot_label = paddle.nn.functional.one_hot( - fluid.dygraph.to_variable(label), depth) + fluid.dygraph.to_variable(label), depth + ) # with _test_eager_guard(): # one_hot_label = paddle.nn.functional.one_hot( # paddle.to_tensor(label), depth) @@ -207,20 +211,22 @@ def _run(self, depth): label = fluid.layers.data(name="label", shape=[1], dtype="int64") one_hot_label = fluid.one_hot(input=label, depth=depth) - label_data = np.array([np.random.randint(0, 10 - 1) - for i in range(6)]).reshape([6, 1]) + label_data = np.array( + [np.random.randint(0, 10 - 1) for i in range(6)] + ).reshape([6, 1]) exe = fluid.Executor(self.place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'label': label_data, - }, - fetch_list=[one_hot_label], - return_numpy=False) + ret = exe.run( + feed={ + 'label': label_data, + }, + fetch_list=[one_hot_label], + return_numpy=False, + ) class BadInputTestOnehotV2(unittest.TestCase): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -229,10 +235,12 @@ def test_error(self): with fluid.program_guard(fluid.Program()): def test_bad_x(): - label = fluid.layers.data(name="label", - shape=[4], - append_batch_size=False, - dtype="float32") + label = fluid.layers.data( + name="label", + shape=[4], + append_batch_size=False, + dtype="float32", + ) one_hot_label = fluid.one_hot(input=label, depth=4) self.assertRaises(TypeError, test_bad_x) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_parallel_dygraph_sync_batch_norm_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_parallel_dygraph_sync_batch_norm_mlu.py index b42567c5d0c5a..4d39cd6cdd745 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_parallel_dygraph_sync_batch_norm_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_parallel_dygraph_sync_batch_norm_mlu.py @@ -31,7 +31,6 @@ class TestParallelDygraphMnistMLU(TestDistBase): - def _setup_config(self): self._sync_mode = False self._cncl_mode = True @@ -49,7 +48,7 @@ def _get_required_envs(self, check_error_log=False, need_envs={}): "FLAGS_call_stack_level": "2", "GLOG_v": "2", "PADDLE_WITH_GLOO": '0', - "BACKEND": "cncl" + "BACKEND": "cncl", } if check_error_log: @@ -60,14 +59,16 @@ def _get_required_envs(self, check_error_log=False, need_envs={}): required_envs.update(need_envs) return required_envs - def _run_local(self, - model, - envs, - check_error_log=False, - batch_size=DEFAULT_BATCH_SIZE, - batch_merge_repeat=1, - log_name="", - devices="1"): + def _run_local( + self, + model, + envs, + check_error_log=False, + batch_size=DEFAULT_BATCH_SIZE, + batch_merge_repeat=1, + log_name="", + devices="1", + ): cmd = self._python_interp @@ -75,8 +76,10 @@ def _run_local(self, envs['COVERAGE_FILE'] = os.getenv('COVERAGE_FILE', '') cmd += " -m coverage run --branch -p" - cmd += " %s --role trainer --update_method local --lr %f" % (model, - self._lr) + cmd += " %s --role trainer --update_method local --lr %f" % ( + model, + self._lr, + ) if batch_size != DEFAULT_BATCH_SIZE: cmd += " --batch_size %d" % batch_size @@ -90,7 +93,7 @@ def _run_local(self, env_local = { "FLAGS_selected_mlus": devices, "PADDLE_TRAINERS_NUM": "1", - "PADDLE_TRAINER_ID": "0" + "PADDLE_TRAINER_ID": "0", } else: env_local = {'CPU_NUM': '1'} @@ -111,30 +114,36 @@ def _run_local(self, if check_error_log: path = "/tmp/local_err_%d.log" % os.getpid() err_log = open(path, "w") - local_proc = subprocess.Popen(cmd.split(" "), - stdout=subprocess.PIPE, - stderr=err_log, - env=env_local) + local_proc = subprocess.Popen( + cmd.split(" "), + stdout=subprocess.PIPE, + stderr=err_log, + env=env_local, + ) else: - local_proc = subprocess.Popen(cmd.split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env_local) + local_proc = subprocess.Popen( + cmd.split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env_local, + ) local_out, local_err = local_proc.communicate() if check_error_log: err_log.close() sys.stderr.write( - '\n--run_local-- trainer 0 stderr file saved in: %s\n' % (path)) + '\n--run_local-- trainer 0 stderr file saved in: %s\n' % (path) + ) sys.stderr.write('local_stderr: %s\n' % local_err) sys.stderr.write('local_stdout: %s\n' % pickle.loads(local_out)) return pickle.loads(local_out) - def _run_cluster_nccl2(self, model, envs, update_method, check_error_log, - log_name): + def _run_cluster_nccl2( + self, model, envs, update_method, check_error_log, log_name + ): # NOTE: we reuse ps_endpoints as nccl2 worker endpoints worker_endpoints = self._ps_endpoints.split(",") @@ -144,20 +153,28 @@ def _run_cluster_nccl2(self, model, envs, update_method, check_error_log, pipes = [] for i in range(0, trainer_num): tr_cmd, tr_env = self._get_nccl2_trainer_cmd( - model, worker_endpoints[i], update_method, i, trainer_num) + model, worker_endpoints[i], update_method, i, trainer_num + ) tr_env.update(envs) - print("use_hallreduce:{} \ntr{}_cmd:{}, env: {}".format( - self._use_hallreduce, i, tr_cmd, tr_env)) + print( + "use_hallreduce:{} \ntr{}_cmd:{}, env: {}".format( + self._use_hallreduce, i, tr_cmd, tr_env + ) + ) tr_pipe = open("/tmp/tr%d_err_%d.log" % (i, os.getpid()), "w") sys.stderr.write( "\n{} going to start process {} with nccl2\n".format( - type(self).__name__, i)) - tr_proc = subprocess.Popen(tr_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr_pipe, - env=tr_env) + type(self).__name__, i + ) + ) + tr_proc = subprocess.Popen( + tr_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr_pipe, + env=tr_env, + ) procs.append(tr_proc) pipes.append(tr_pipe) @@ -170,7 +187,9 @@ def _run_cluster_nccl2(self, model, envs, update_method, check_error_log, sys.stderr.write('trainer {} stderr: {}\n'.format(i, tr_err)) sys.stderr.write( 'trainer {} glog file saved in: /tmp/tr{}_err_{}.log \n'.format( - i, i, os.getpid())) + i, i, os.getpid() + ) + ) if check_error_log: print("outs[0]:", pickle.loads(outs[0])) @@ -184,7 +203,8 @@ def test_mnist(self): os.path.abspath("parallel_dygraph_sync_batch_norm.py"), delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/mlu/test_pool2d_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_pool2d_op_mlu.py index e9add5e6a5c39..57202e62a4333 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_pool2d_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_pool2d_op_mlu.py @@ -23,30 +23,40 @@ sys.path.append('..') from op_test import OpTest -from test_pool2d_op import pool2D_forward_naive, avg_pool2D_forward_naive, max_pool2D_forward_naive, adaptive_start_index, adaptive_end_index +from test_pool2d_op import ( + pool2D_forward_naive, + avg_pool2D_forward_naive, + max_pool2D_forward_naive, + adaptive_start_index, + adaptive_end_index, +) paddle.enable_static() -def pool2d_backward_navie(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_format='NCHW', - pool_type="max", - padding_algorithm="EXPLICIT"): +def pool2d_backward_navie( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_format='NCHW', + pool_type="max", + padding_algorithm="EXPLICIT", +): # update paddings def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -56,9 +66,10 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): if isinstance(padding_algorithm, str): padding_algorithm = padding_algorithm.upper() if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if padding_algorithm == "VALID": paddings = [0, 0, 0, 0] @@ -66,7 +77,8 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): raise ValueError( "When Attr(pool_padding) is \"VALID\", Attr(ceil_mode)" " must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) elif padding_algorithm == "SAME": input_data_shape = [] if data_format == "NCHW": @@ -95,10 +107,20 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) // strides[0] + 1 \ - if ceil_mode else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 - W_out = (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) // strides[1] + 1 \ - if ceil_mode else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 + H_out = ( + (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) + // strides[0] + + 1 + if ceil_mode + else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) + // strides[1] + + 1 + if ceil_mode + else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 + ) x_grad = np.zeros_like(x) for i in range(H_out): @@ -126,20 +148,26 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): in_w_end = np.min((in_w_end, W)) if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (in_h_end - in_h_start) * (in_w_end - - in_w_start) - x_grad[:, :, in_h_start:in_h_end, - in_w_start:in_w_end] += 1 / field_size + if exclusive or adaptive: + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) + x_grad[:, :, in_h_start:in_h_end, in_w_start:in_w_end] += ( + 1 / field_size + ) elif pool_type == 'max': for n in range(N): for c in range(C): - idx = np.argmax(x[n, c, in_h_start:in_h_end, - in_w_start:in_w_end].flatten()) + idx = np.argmax( + x[ + n, c, in_h_start:in_h_end, in_w_start:in_w_end + ].flatten() + ) idx_h = idx // (in_w_end - in_w_start) idx_w = idx % (in_w_end - in_w_start) - x_grad[n, c, in_h_start + idx_h, - in_w_start + idx_w] += 1 + x_grad[ + n, c, in_h_start + idx_h, in_w_start + idx_w + ] += 1 if data_format == "NHWC": x_grad = x_grad.transpose([0, 2, 3, 1]) @@ -147,7 +175,6 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): class TestPool2D_Op_Mixin(object): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -165,12 +192,19 @@ def setUp(self): self.init_shape() input = np.random.random(self.shape).astype(self.dtype) - output = pool2D_forward_naive(input, self.ksize, self.strides, - self.paddings, self.global_pool, - self.ceil_mode, self.exclusive, - self.adaptive, self.data_format, - self.pool_type, - self.padding_algorithm).astype(self.dtype) + output = pool2D_forward_naive( + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + self.data_format, + self.pool_type, + self.padding_algorithm, + ).astype(self.dtype) self.inputs = {'X': OpTest.np_dtype_to_fluid_dtype(input)} self.attrs = { @@ -192,23 +226,27 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad(self): - x_grad = pool2d_backward_navie(self.inputs["X"], - ksize=self.ksize, - strides=self.strides, - paddings=self.paddings, - global_pool=self.global_pool, - ceil_mode=False, - exclusive=self.exclusive, - adaptive=self.adaptive, - data_format=self.data_format, - pool_type=self.pool_type, - padding_algorithm=self.padding_algorithm) + x_grad = pool2d_backward_navie( + self.inputs["X"], + ksize=self.ksize, + strides=self.strides, + paddings=self.paddings, + global_pool=self.global_pool, + ceil_mode=False, + exclusive=self.exclusive, + adaptive=self.adaptive, + data_format=self.data_format, + pool_type=self.pool_type, + padding_algorithm=self.padding_algorithm, + ) x_grad = x_grad / np.prod(self.outputs['Out'].shape) - self.check_grad_with_place(self.place, - set(['X']), - 'Out', - max_relative_error=0.06, - user_defined_grads=[x_grad]) + self.check_grad_with_place( + self.place, + set(['X']), + 'Out', + max_relative_error=0.06, + user_defined_grads=[x_grad], + ) def init_data_format(self): self.data_format = "NCHW" @@ -249,7 +287,6 @@ class TestPool2D_Op(TestPool2D_Op_Mixin, OpTest): class TestCase1(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -269,7 +306,6 @@ def init_shape(self): class TestCase2(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -289,30 +325,25 @@ def init_shape(self): class TestCase3(TestPool2D_Op): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase4(TestCase1): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase5(TestCase2): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive def create_test_fp16_class(parent): - class TestFp16Case(parent): - def init_data_type(self): self.dtype = np.float16 @@ -332,13 +363,11 @@ def test_check_output(self): create_test_fp16_class(TestCase4) create_test_fp16_class(TestCase5) -#--------------------test pool2d use ceil mode-------------------- +# --------------------test pool2d use ceil mode-------------------- def create_test_use_ceil_class(parent): - class TestPool2DUseCeilCase(parent): - def init_ceil_mode(self): self.ceil_mode = True @@ -352,19 +381,16 @@ def init_ceil_mode(self): class TestAvgInclude(TestCase2): - def init_exclusive(self): self.exclusive = False class TestAvgPoolAdaptive(TestCase1): - def init_adaptive(self): self.adaptive = True class TestAvgPoolAdaptiveAsyOutSize(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -377,11 +403,10 @@ def init_test_case(self): self.paddings = [0, 0, 0, 0] -#-------test pool2d with asymmetric padding----- +# -------test pool2d with asymmetric padding----- class TestPool2D_AsyPadding(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -392,7 +417,6 @@ def init_shape(self): class TestCase1_AsyPadding(TestCase1): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -403,7 +427,6 @@ def init_shape(self): class TestCase2_AsyPadding(TestCase2): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -414,7 +437,6 @@ def init_shape(self): class TestCase3_AsyPadding(TestCase3): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -425,7 +447,6 @@ def init_shape(self): class TestCase4_AsyPadding(TestCase4): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -436,7 +457,6 @@ def init_shape(self): class TestCase5_AsyPadding((TestCase5)): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -451,7 +471,6 @@ def init_shape(self): class TestAvgInclude_AsyPadding(TestCase2): - def init_exclusive(self): self.exclusive = False @@ -465,7 +484,6 @@ def init_shape(self): class TestAvgPoolAdaptive_AsyPadding(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -478,9 +496,8 @@ def init_shape(self): self.shape = [2, 3, 7, 7] -#----------- test channel_last -------------- +# ----------- test channel_last -------------- class TestPool2D_channel_last(TestPool2D_Op): - def init_data_format(self): self.data_format = "NHWC" @@ -489,7 +506,6 @@ def init_shape(self): class TestCase1_channel_last(TestCase1): - def init_data_format(self): self.data_format = "NHWC" @@ -498,7 +514,6 @@ def init_shape(self): class TestCase2_channel_last(TestCase2): - def init_data_format(self): self.data_format = "NHWC" @@ -507,7 +522,6 @@ def init_shape(self): class TestCase3_channel_last(TestCase3): - def init_data_format(self): self.data_format = "NHWC" @@ -516,7 +530,6 @@ def init_shape(self): class TestCase4_channel_last(TestCase4): - def init_data_format(self): self.data_format = "NHWC" @@ -525,7 +538,6 @@ def init_shape(self): class TestCase5_channel_last(TestCase5): - def init_data_format(self): self.data_format = "NHWC" @@ -538,13 +550,11 @@ def init_shape(self): class TestCase5_Max(TestCase2): - def init_pool_type(self): self.pool_type = "max" class TestCase5_channel_last_Max(TestCase5_Max): - def init_data_format(self): self.data_format = "NHWC" @@ -553,19 +563,16 @@ def init_shape(self): class TestAvgInclude_channel_last(TestCase2_channel_last): - def init_exclusive(self): self.exclusive = False class TestAvgPoolAdaptive_channel_last(TestCase1_channel_last): - def init_adaptive(self): self.adaptive = True class TestPool2D_AsyPadding_channel_last(TestPool2D_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -574,7 +581,6 @@ def init_shape(self): class TestCase1_AsyPadding_channel_last(TestCase1_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -583,7 +589,6 @@ def init_shape(self): class TestCase2_AsyPadding_channel_last(TestCase2_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -592,7 +597,6 @@ def init_shape(self): class TestCase3_AsyPadding_channel_last(TestCase3_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -601,7 +605,6 @@ def init_shape(self): class TestCase4_AsyPadding_channel_last(TestCase4_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -610,7 +613,6 @@ def init_shape(self): class TestCase5_AsyPadding_channel_last(TestCase5_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -623,7 +625,6 @@ def init_shape(self): class TestAvgInclude_AsyPadding_channel_last(TestAvgInclude_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -635,9 +636,7 @@ def init_shape(self): def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.paddings = [0, 0] self.padding_algorithm = "SAME" @@ -663,9 +662,7 @@ def init_paddings(self): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.paddings = [1, 1] self.padding_algorithm = "VALID" @@ -691,7 +688,6 @@ def init_paddings(self): class TestCase1_strides(TestCase1): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 2] @@ -705,96 +701,121 @@ def init_shape(self): # ----- test API class TestPool2DAPI(unittest.TestCase): - def test_api(self): x_NHWC = np.random.random([2, 5, 5, 3]).astype("float32") x_NCHW = np.random.random([2, 3, 5, 5]).astype("float32") - input_NHWC = fluid.layers.data(name="input_NHWC", - shape=[2, 5, 5, 3], - append_batch_size=False, - dtype="float32") - - input_NCHW = fluid.layers.data(name="input_NCHW", - shape=[2, 3, 5, 5], - append_batch_size=False, - dtype="float32") - - input_NHWC_negetive = fluid.layers.data(name="input_NHWC_negetive", - shape=[2, -1, 5, 3], - append_batch_size=False, - dtype="float32") - - input_NCHW_negetive = fluid.layers.data(name="input_NCHW_negetive", - shape=[2, 3, -1, -1], - append_batch_size=False, - dtype="float32") + input_NHWC = fluid.layers.data( + name="input_NHWC", + shape=[2, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + input_NCHW = fluid.layers.data( + name="input_NCHW", + shape=[2, 3, 5, 5], + append_batch_size=False, + dtype="float32", + ) + + input_NHWC_negetive = fluid.layers.data( + name="input_NHWC_negetive", + shape=[2, -1, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + input_NCHW_negetive = fluid.layers.data( + name="input_NCHW_negetive", + shape=[2, 3, -1, -1], + append_batch_size=False, + dtype="float32", + ) ksize = [3, 3] - out_1 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1], - data_format="NHWC") - - out_2 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="avg", - pool_padding=[[0, 0], [1, 1], [1, 1], - [0, 0]], - data_format="NHWC") - - out_3 = fluid.layers.pool2d(input=input_NCHW, - pool_size=ksize, - pool_type="avg", - pool_padding=[[0, 0], [0, 0], [1, 1], - [1, 1]], - data_format="NCHW") - - out_4 = fluid.layers.pool2d(input=input_NCHW, - pool_size=ksize, - pool_type="avg", - pool_padding=[1, 2, 1, 0], - data_format="NCHW") + out_1 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1], + data_format="NHWC", + ) + + out_2 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="avg", + pool_padding=[[0, 0], [1, 1], [1, 1], [0, 0]], + data_format="NHWC", + ) + + out_3 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=ksize, + pool_type="avg", + pool_padding=[[0, 0], [0, 0], [1, 1], [1, 1]], + data_format="NCHW", + ) + + out_4 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=ksize, + pool_type="avg", + pool_padding=[1, 2, 1, 0], + data_format="NCHW", + ) # test VALID - out_5 = fluid.layers.pool2d(input=input_NCHW, - pool_size=ksize, - pool_type="avg", - pool_padding="VALID", - data_format="NCHW") - - out_6 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALID", - data_format="NHWC") + out_5 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=ksize, + pool_type="avg", + pool_padding="VALID", + data_format="NCHW", + ) + + out_6 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALID", + data_format="NHWC", + ) # test SAME - out_7 = fluid.layers.pool2d(input=input_NCHW, - pool_size=[4, 4], - pool_type="avg", - pool_padding="SAME", - data_format="NCHW") - - out_8 = fluid.layers.pool2d(input=input_NHWC, - pool_size=[4, 4], - pool_type="max", - pool_padding="SAME", - data_format="NHWC") + out_7 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=[4, 4], + pool_type="avg", + pool_padding="SAME", + data_format="NCHW", + ) + + out_8 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=[4, 4], + pool_type="max", + pool_padding="SAME", + data_format="NHWC", + ) # test negetive - out_9 = fluid.layers.pool2d(input=input_NHWC_negetive, - pool_size=ksize, - pool_type="avg", - pool_padding=[0, 0], - data_format="NHWC") + out_9 = fluid.layers.pool2d( + input=input_NHWC_negetive, + pool_size=ksize, + pool_type="avg", + pool_padding=[0, 0], + data_format="NHWC", + ) assert out_9.shape == (2, -1, 3, 3) - out_10 = fluid.layers.pool2d(input=input_NCHW_negetive, - pool_size=ksize, - pool_type="avg", - pool_padding=[0, 0], - data_format="NCHW") + out_10 = fluid.layers.pool2d( + input=input_NCHW_negetive, + pool_size=ksize, + pool_type="avg", + pool_padding=[0, 0], + data_format="NCHW", + ) assert out_10.shape == (2, 3, -1, -1) exe = fluid.Executor(place=fluid.MLUPlace(0)) @@ -804,46 +825,61 @@ def test_api(self): "input_NHWC": x_NHWC, "input_NCHW": x_NCHW, "input_NHWC_negetive": x_NHWC, - "input_NCHW_negetive": x_NCHW + "input_NCHW_negetive": x_NCHW, }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8], + ) assert np.allclose( res_1, - pool2D_forward_naive(x=x_NHWC, - ksize=ksize, - pool_type="max", - strides=[1, 1], - paddings=[1, 1], - data_format="NHWC")) + pool2D_forward_naive( + x=x_NHWC, + ksize=ksize, + pool_type="max", + strides=[1, 1], + paddings=[1, 1], + data_format="NHWC", + ), + ) assert np.allclose( res_2, - pool2D_forward_naive(x=x_NHWC, - ksize=ksize, - pool_type="avg", - strides=[1, 1], - paddings=[1, 1, 1, 1], - data_format="NHWC")) - assert np.allclose(res_3, - pool2D_forward_naive(x=x_NCHW, - ksize=ksize, - pool_type="avg", - strides=[1, 1], - paddings=[1, 1, 1, 1], - data_format="NCHW"), - rtol=0.07, - atol=1e-05) - - assert np.allclose(res_4, - pool2D_forward_naive(x=x_NCHW, - ksize=ksize, - pool_type="avg", - strides=[1, 1], - paddings=[1, 2, 1, 0], - data_format="NCHW"), - rtol=0.07, - atol=1e-05) + pool2D_forward_naive( + x=x_NHWC, + ksize=ksize, + pool_type="avg", + strides=[1, 1], + paddings=[1, 1, 1, 1], + data_format="NHWC", + ), + ) + assert np.allclose( + res_3, + pool2D_forward_naive( + x=x_NCHW, + ksize=ksize, + pool_type="avg", + strides=[1, 1], + paddings=[1, 1, 1, 1], + data_format="NCHW", + ), + rtol=0.07, + atol=1e-05, + ) + + assert np.allclose( + res_4, + pool2D_forward_naive( + x=x_NCHW, + ksize=ksize, + pool_type="avg", + strides=[1, 1], + paddings=[1, 2, 1, 0], + data_format="NCHW", + ), + rtol=0.07, + atol=1e-05, + ) # VALID assert np.allclose( @@ -855,175 +891,215 @@ def test_api(self): strides=[1, 1], paddings=[10, 20], # any ele is ok padding_algorithm="VALID", - data_format="NCHW"), + data_format="NCHW", + ), rtol=0.07, - atol=1e-05) + atol=1e-05, + ) assert np.allclose( res_6, - pool2D_forward_naive(x=x_NHWC, - ksize=ksize, - pool_type="max", - strides=[1, 1], - paddings=[10, 20], - padding_algorithm="VALID", - data_format="NHWC")) + pool2D_forward_naive( + x=x_NHWC, + ksize=ksize, + pool_type="max", + strides=[1, 1], + paddings=[10, 20], + padding_algorithm="VALID", + data_format="NHWC", + ), + ) # SAME - assert np.allclose(res_7, - pool2D_forward_naive(x=x_NCHW, - ksize=[4, 4], - pool_type="avg", - strides=[1, 1], - paddings=[10, 20], - padding_algorithm="SAME", - data_format="NCHW"), - rtol=0.07, - atol=1e-05) + assert np.allclose( + res_7, + pool2D_forward_naive( + x=x_NCHW, + ksize=[4, 4], + pool_type="avg", + strides=[1, 1], + paddings=[10, 20], + padding_algorithm="SAME", + data_format="NCHW", + ), + rtol=0.07, + atol=1e-05, + ) assert np.allclose( res_8, - pool2D_forward_naive(x=x_NHWC, - ksize=[4, 4], - pool_type="max", - strides=[1, 1], - paddings=[10, 20], - padding_algorithm="SAME", - data_format="NHWC")) + pool2D_forward_naive( + x=x_NHWC, + ksize=[4, 4], + pool_type="max", + strides=[1, 1], + paddings=[10, 20], + padding_algorithm="SAME", + data_format="NHWC", + ), + ) class TestPool2DAPI_Error(unittest.TestCase): - def test_api(self): - input_NHWC = fluid.layers.data(name="input_NHWC", - shape=[2, 5, 5, 3], - append_batch_size=False, - dtype="float32") + input_NHWC = fluid.layers.data( + name="input_NHWC", + shape=[2, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) ksize = [3, 3] # data_format value error def run_2(): - out_2 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1], - data_format="NHWCC") + out_2 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1], + data_format="NHWCC", + ) self.assertRaises(ValueError, run_2) # padding str value error def run_3(): - out_3 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALIDSAME", - data_format="NHWC") + out_3 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALIDSAME", + data_format="NHWC", + ) self.assertRaises(ValueError, run_3) # padding str valid and ceil_mode value error def run_4(): - out_4 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALID", - ceil_mode=True, - data_format="NHWC") + out_4 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALID", + ceil_mode=True, + data_format="NHWC", + ) self.assertRaises(ValueError, run_4) # padding with 8 ele. value error def run_5(): - out_5 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[[1, 1], [0, 0], [0, 0], - [1, 1]], - data_format="NHWC") + out_5 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[[1, 1], [0, 0], [0, 0], [1, 1]], + data_format="NHWC", + ) self.assertRaises(ValueError, run_5) class TestDygraphPool2DAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of Pool2D must be Variable. data1 = np.random.random((3, 32, 32, 5)).astype('float32') - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='max', - pool_stride=1, - global_pooling=False) + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='max', + pool_stride=1, + global_pooling=False, + ) self.assertRaises(TypeError, pool2d, data1) # the input dtype of mlu Pool2D must be float16 or float32 - data2 = fluid.layers.data(name='x1', - shape=[3, 32, 32, 5], - dtype="int32") + data2 = fluid.layers.data( + name='x1', shape=[3, 32, 32, 5], dtype="int32" + ) self.assertRaises(TypeError, pool2d, data2) def test_data_format_error(self): with program_guard(Program(), Program()): # the data_format must be 'NCHW' or 'NHWC' data1 = np.random.random((3, 32, 32, 5)).astype('float32') - self.assertRaises(ValueError, - fluid.dygraph.Pool2D, - pool_size=2, - pool_type='max', - pool_stride=1, - global_pooling=False, - data_format='NWHC') + self.assertRaises( + ValueError, + fluid.dygraph.Pool2D, + pool_size=2, + pool_type='max', + pool_stride=1, + global_pooling=False, + data_format='NWHC', + ) class TestDygraphPool2DAPI(unittest.TestCase): - def test_nhwc(self): with fluid.dygraph.guard(): data = np.random.random((3, 32, 32, 5)).astype('float32') x = fluid.dygraph.to_variable(data) - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='max', - pool_stride=1, - pool_padding=[0, 0], - global_pooling=False, - data_format='NHWC') + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='max', + pool_stride=1, + pool_padding=[0, 0], + global_pooling=False, + data_format='NHWC', + ) out1 = pool2d(x) - out2 = pool2D_forward_naive(data, [2, 2], [1, 1], - paddings=[0, 0], - pool_type='max', - data_format='NHWC') + out2 = pool2D_forward_naive( + data, + [2, 2], + [1, 1], + paddings=[0, 0], + pool_type='max', + data_format='NHWC', + ) np.testing.assert_allclose(out1.numpy(), out2) def test_lower_case(self): with fluid.dygraph.guard(): data = np.random.random((3, 32, 32, 5)).astype('float32') x = fluid.dygraph.to_variable(data) - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='max', - pool_stride=1, - pool_padding=[0, 0], - global_pooling=False, - data_format='nhwc') + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='max', + pool_stride=1, + pool_padding=[0, 0], + global_pooling=False, + data_format='nhwc', + ) out1 = pool2d(x) - out2 = pool2D_forward_naive(data, [2, 2], [1, 1], - paddings=[0, 0], - pool_type='max', - data_format='NHWC') + out2 = pool2D_forward_naive( + data, + [2, 2], + [1, 1], + paddings=[0, 0], + pool_type='max', + data_format='NHWC', + ) np.testing.assert_allclose(out1.numpy(), out2) def test_upper_case(self): with fluid.dygraph.guard(): data = np.random.random((3, 32, 32, 5)).astype('float32') x = fluid.dygraph.to_variable(data) - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='MAX', - pool_stride=1, - pool_padding=[0, 0], - global_pooling=False, - data_format='nhwc') + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='MAX', + pool_stride=1, + pool_padding=[0, 0], + global_pooling=False, + data_format='nhwc', + ) out1 = pool2d(x) - out2 = pool2D_forward_naive(data, [2, 2], [1, 1], - paddings=[0, 0], - pool_type='max', - data_format='NHWC') + out2 = pool2D_forward_naive( + data, + [2, 2], + [1, 1], + paddings=[0, 0], + pool_type='max', + data_format='NHWC', + ) np.testing.assert_allclose(out1.numpy(), out2) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_prior_box_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_prior_box_op_mlu.py index 2e2b96c9e7734..8f8473cceab7d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_prior_box_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_prior_box_op_mlu.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import unittest import sys @@ -28,7 +26,6 @@ class TestMLUPriorBox(OpTest): - def setUp(self): self.op_type = "prior_box" self.set_mlu() @@ -60,7 +57,7 @@ def set_data(self): 'min_max_aspect_ratios_order': self.min_max_aspect_ratios_order, 'step_w': self.step_w, 'step_h': self.step_h, - 'offset': self.offset + 'offset': self.offset, } if len(self.max_sizes) > 0: self.attrs['max_sizes'] = self.max_sizes @@ -95,8 +92,9 @@ def init_test_params(self): self.flip = True self.set_min_max_aspect_ratios_order() self.real_aspect_ratios = [1, 2.0, 1.0 / 2.0, 3.0, 1.0 / 3.0] - self.aspect_ratios = np.array(self.aspect_ratios, - dtype=np.float64).flatten() + self.aspect_ratios = np.array( + self.aspect_ratios, dtype=np.float64 + ).flatten() self.variances = [0.1, 0.1, 0.2, 0.2] self.variances = np.array(self.variances, dtype=np.float64).flatten() @@ -108,12 +106,12 @@ def init_test_params(self): def init_test_input(self): self.image = np.random.random( - (self.batch_size, self.image_channels, self.image_w, - self.image_h)).astype('float32') + (self.batch_size, self.image_channels, self.image_w, self.image_h) + ).astype('float32') self.input = np.random.random( - (self.batch_size, self.input_channels, self.layer_w, - self.layer_h)).astype('float32') + (self.batch_size, self.input_channels, self.layer_w, self.layer_h) + ).astype('float32') def init_test_output(self): out_dim = (self.layer_h, self.layer_w, self.num_priors, 4) @@ -134,73 +132,78 @@ def init_test_output(self): ar = self.real_aspect_ratios[r] c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 else: - c_w = c_h = min_size / 2. - out_boxes[h, w, idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + c_w = c_h = min_size / 2.0 + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # rest of priors for r in range(len(self.real_aspect_ratios)): ar = self.real_aspect_ratios[r] - if abs(ar - 1.) < 1e-6: + if abs(ar - 1.0) < 1e-6: continue c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # clip the prior's coordidate such that it is within[0, 1] if self.clip: out_boxes = np.clip(out_boxes, 0.0, 1.0) # set the variance. - out_var = np.tile(self.variances, - (self.layer_h, self.layer_w, self.num_priors, 1)) + out_var = np.tile( + self.variances, (self.layer_h, self.layer_w, self.num_priors, 1) + ) self.out_boxes = out_boxes.astype('float32') self.out_var = out_var.astype('float32') class TestMLUPriorBoxWithoutMaxSize(TestMLUPriorBox): - def set_max_sizes(self): self.max_sizes = [] class TestMLUPriorBoxWithoutSpecifiedOutOrder(TestMLUPriorBox): - def set_min_max_aspect_ratios_order(self): self.min_max_aspect_ratios_order = False diff --git a/python/paddle/fluid/tests/unittests/mlu/test_randperm_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_randperm_op_mlu.py index 5412e6c4a7b60..3ee3144905ef5 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_randperm_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_randperm_op_mlu.py @@ -27,31 +27,38 @@ def check_randperm_out(n, data_np): - assert isinstance(data_np, np.ndarray), \ - "The input data_np should be np.ndarray." + assert isinstance( + data_np, np.ndarray + ), "The input data_np should be np.ndarray." gt_sorted = np.arange(n) out_sorted = np.sort(data_np) return list(gt_sorted == out_sorted) def error_msg(data_np): - return "The sorted ground truth and sorted out should " + \ - "be equal, out = " + str(data_np) + return ( + "The sorted ground truth and sorted out should " + + "be equal, out = " + + str(data_np) + ) def convert_dtype(dtype_str): dtype_str_list = ["int32", "int64", "float32", "float64"] dtype_num_list = [ - core.VarDesc.VarType.INT32, core.VarDesc.VarType.INT64, - core.VarDesc.VarType.FP32, core.VarDesc.VarType.FP64 + core.VarDesc.VarType.INT32, + core.VarDesc.VarType.INT64, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP64, ] - assert dtype_str in dtype_str_list, dtype_str + \ - " should in " + str(dtype_str_list) + assert dtype_str in dtype_str_list, ( + dtype_str + " should in " + str(dtype_str_list) + ) return dtype_num_list[dtype_str_list.index(dtype_str)] class TestRandpermOp(OpTest): - """ Test randperm op.""" + """Test randperm op.""" def setUp(self): self.op_type = "randperm" @@ -77,36 +84,32 @@ def test_check_output(self): def verify_output(self, outs): out_np = np.array(outs[0]) - self.assertTrue(check_randperm_out(self.n, out_np), - msg=error_msg(out_np)) + self.assertTrue( + check_randperm_out(self.n, out_np), msg=error_msg(out_np) + ) class TestRandpermOpN(TestRandpermOp): - def init_attrs(self): self.n = 10000 class TestRandpermOpInt32(TestRandpermOp): - def init_attrs(self): self.dtype = "int32" class TestRandpermOpFloat32(TestRandpermOp): - def init_attrs(self): self.dtype = "float32" class TestRandpermOpFloat64(TestRandpermOp): - def init_attrs(self): self.dtype = "float64" class TestRandpermOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): self.assertRaises(ValueError, paddle.randperm, -3) @@ -114,7 +117,6 @@ def test_errors(self): class TestRandpermAPI(unittest.TestCase): - def test_out(self): n = 10 place = paddle.MLUPlace(0) @@ -132,20 +134,19 @@ def test_out(self): class TestRandpermImperative(unittest.TestCase): - def test_out(self): paddle.disable_static() n = 10 for dtype in ['int32', np.int64, 'float32', 'float64']: data_p = paddle.randperm(n, dtype) data_np = data_p.numpy() - self.assertTrue(check_randperm_out(n, data_np), - msg=error_msg(data_np)) + self.assertTrue( + check_randperm_out(n, data_np), msg=error_msg(data_np) + ) paddle.enable_static() class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -158,25 +159,70 @@ def test_fixed_random_number(self): x = paddle.randperm(30000, dtype='int32').numpy() expect = [ - 24562, 8409, 9379, 10328, 20503, 18059, 9681, 21883, 11783, 27413 + 24562, + 8409, + 9379, + 10328, + 20503, + 18059, + 9681, + 21883, + 11783, + 27413, ] np.testing.assert_allclose(x[0:10], expect) expect = [ - 29477, 27100, 9643, 16637, 8605, 16892, 27767, 2724, 1612, 13096 + 29477, + 27100, + 9643, + 16637, + 8605, + 16892, + 27767, + 2724, + 1612, + 13096, ] np.testing.assert_allclose(x[10000:10010], expect) expect = [ - 298, 4104, 16479, 22714, 28684, 7510, 14667, 9950, 15940, 28343 + 298, + 4104, + 16479, + 22714, + 28684, + 7510, + 14667, + 9950, + 15940, + 28343, ] np.testing.assert_allclose(x[20000:20010], expect) x = paddle.randperm(30000, dtype='int64').numpy() expect = [ - 6587, 1909, 5525, 23001, 6488, 14981, 14355, 3083, 29561, 8171 + 6587, + 1909, + 5525, + 23001, + 6488, + 14981, + 14355, + 3083, + 29561, + 8171, ] np.testing.assert_allclose(x[0:10], expect) expect = [ - 23460, 12394, 22501, 5427, 20185, 9100, 5127, 1651, 25806, 4818 + 23460, + 12394, + 22501, + 5427, + 20185, + 9100, + 5127, + 1651, + 25806, + 4818, ] np.testing.assert_allclose(x[10000:10010], expect) expect = [5829, 4508, 16193, 24836, 8526, 242, 9984, 9243, 1977, 11839] @@ -184,35 +230,83 @@ def test_fixed_random_number(self): x = paddle.randperm(30000, dtype='float32').numpy() expect = [ - 5154., 10537., 14362., 29843., 27185., 28399., 27561., 4144., - 22906., 10705. + 5154.0, + 10537.0, + 14362.0, + 29843.0, + 27185.0, + 28399.0, + 27561.0, + 4144.0, + 22906.0, + 10705.0, ] np.testing.assert_allclose(x[0:10], expect) expect = [ - 1958., 18414., 20090., 21910., 22746., 27346., 22347., 3002., 4564., - 26991. + 1958.0, + 18414.0, + 20090.0, + 21910.0, + 22746.0, + 27346.0, + 22347.0, + 3002.0, + 4564.0, + 26991.0, ] np.testing.assert_allclose(x[10000:10010], expect) expect = [ - 25580., 12606., 553., 16387., 29536., 4241., 20946., 16899., 16339., - 4662. + 25580.0, + 12606.0, + 553.0, + 16387.0, + 29536.0, + 4241.0, + 20946.0, + 16899.0, + 16339.0, + 4662.0, ] np.testing.assert_allclose(x[20000:20010], expect) x = paddle.randperm(30000, dtype='float64').numpy() expect = [ - 19051., 2449., 21940., 11121., 282., 7330., 13747., 24321., 21147., - 9163. + 19051.0, + 2449.0, + 21940.0, + 11121.0, + 282.0, + 7330.0, + 13747.0, + 24321.0, + 21147.0, + 9163.0, ] np.testing.assert_allclose(x[0:10], expect) expect = [ - 15483., 1315., 5723., 20954., 13251., 25539., 5074., 1823., 14945., - 17624. + 15483.0, + 1315.0, + 5723.0, + 20954.0, + 13251.0, + 25539.0, + 5074.0, + 1823.0, + 14945.0, + 17624.0, ] np.testing.assert_allclose(x[10000:10010], expect) expect = [ - 10516., 2552., 29970., 5941., 986., 8007., 24805., 26753., 12202., - 21404. + 10516.0, + 2552.0, + 29970.0, + 5941.0, + 986.0, + 8007.0, + 24805.0, + 26753.0, + 12202.0, + 21404.0, ] np.testing.assert_allclose(x[20000:20010], expect) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/mlu/test_range_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_range_op_mlu.py index d65040ecb394a..6ebbb089e580f 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_range_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_range_op_mlu.py @@ -29,7 +29,6 @@ def arange_wrapper(start, end, step, dtype=None): class TestRangeOp(OpTest): - def setUp(self): self.op_type = "range" self.place = paddle.device.MLUPlace(0) @@ -38,13 +37,13 @@ def setUp(self): self.inputs = { 'Start': np.array([self.case[0]]).astype(self.dtype), 'End': np.array([self.case[1]]).astype(self.dtype), - 'Step': np.array([self.case[2]]).astype(self.dtype) + 'Step': np.array([self.case[2]]).astype(self.dtype), } self.outputs = { - 'Out': - np.arange(self.case[0], self.case[1], - self.case[2]).astype(self.dtype) + 'Out': np.arange(self.case[0], self.case[1], self.case[2]).astype( + self.dtype + ) } def init_config(self): @@ -57,7 +56,6 @@ def test_check_output(self): class TestFloatRangeOpCase0(TestRangeOp): - def init_config(self): self.dtype = np.float32 self.python_api = partial(arange_wrapper, dtype=self.dtype) @@ -65,7 +63,6 @@ def init_config(self): class TestInt32RangeOpCase0(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.python_api = partial(arange_wrapper, dtype=self.dtype) @@ -73,7 +70,6 @@ def init_config(self): class TestInt32RangeOpCase1(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.python_api = partial(arange_wrapper, dtype=self.dtype) @@ -81,7 +77,6 @@ def init_config(self): class TestInt32RangeOpCase2(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.python_api = partial(arange_wrapper, dtype=self.dtype) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_reciprocal_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_reciprocal_op_mlu.py index e93afd7cc29e7..0b6d08bdfd37d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_reciprocal_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_reciprocal_op_mlu.py @@ -24,7 +24,6 @@ class TestMLUReciprocal(OpTest): - def setUp(self): self.op_type = "reciprocal" self.set_mlu() @@ -41,9 +40,9 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.01 + ) def set_mlu(self): self.__class__.use_mlu = True @@ -54,7 +53,6 @@ def init_dtype(self): class TestMLUReciprocalFp16(TestMLUReciprocal): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.MLUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_reduce_max_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_reduce_max_op_mlu.py index dc90a521d379d..b225e25b06baa 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_reduce_max_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_reduce_max_op_mlu.py @@ -26,7 +26,8 @@ @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestMLUReduceMaxOp(OpTest): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -54,7 +55,8 @@ def init_dtype(self): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpMultiAxises(TestMLUReduceMaxOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -72,7 +74,8 @@ def setUp(self): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceAll(TestMLUReduceMaxOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -88,7 +91,8 @@ def setUp(self): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_int32(TestMLUReduceMaxOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -100,11 +104,12 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT32) + 'out_dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype(np.int32) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.int32) } def init_dtype(self): @@ -113,7 +118,8 @@ def init_dtype(self): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_fp16(TestMLUReduceMaxOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -125,12 +131,12 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP16) + 'out_dtype': int(core.VarDesc.VarType.FP16), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype( - np.float16) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.float16) } def init_dtype(self): @@ -142,7 +148,8 @@ def test_check_output(self): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_fp32(TestMLUReduceMaxOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -154,12 +161,12 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } def init_dtype(self): diff --git a/python/paddle/fluid/tests/unittests/mlu/test_reduce_mean_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_reduce_mean_op_mlu.py index 588dc949aeba3..bbb35ef40d311 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_reduce_mean_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_reduce_mean_op_mlu.py @@ -25,7 +25,6 @@ class TestMeanOp(OpTest): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.device.MLUPlace(0) @@ -44,7 +43,6 @@ def test_check_grad(self): class TestMeanOp5D(TestMeanOp): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -55,7 +53,6 @@ def setUp(self): class TestMeanOp6D(TestMeanOp): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -66,7 +63,6 @@ def setUp(self): class TestMeanOp8D(TestMeanOp): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -78,7 +74,6 @@ def setUp(self): class Test1DReduce(TestMeanOp): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -87,7 +82,6 @@ def setUp(self): class Test2DReduce0(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -97,7 +91,6 @@ def setUp(self): class Test2DReduce1(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -109,7 +102,6 @@ def setUp(self): class Test3DReduce0(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -121,7 +113,6 @@ def setUp(self): class Test3DReduce1(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -133,7 +124,6 @@ def setUp(self): class Test3DReduce2(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -145,7 +135,6 @@ def setUp(self): class Test3DReduce3(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -157,21 +146,19 @@ def setUp(self): class TestKeepDimReduce(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" self.inputs = {'X': np.random.random((5, 6, 10)).astype("float32")} self.attrs = {'dim': [1], 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].mean(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].mean( + axis=tuple(self.attrs['dim']), keepdims=self.attrs['keep_dim'] + ) } class TestKeepDim8DReduce(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" @@ -180,14 +167,13 @@ def setUp(self): } self.attrs = {'dim': (3, 4, 5), 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].mean(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].mean( + axis=tuple(self.attrs['dim']), keepdims=self.attrs['keep_dim'] + ) } class TestReduceAll(Test1DReduce): - def setUp(self): self.set_mlu() self.op_type = "reduce_mean" diff --git a/python/paddle/fluid/tests/unittests/mlu/test_reduce_min_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_reduce_min_op_mlu.py index 9d69d6d8cccef..d6649f214c469 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_reduce_min_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_reduce_min_op_mlu.py @@ -26,7 +26,8 @@ @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestMLUReduceMinOp(OpTest): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -54,7 +55,8 @@ def init_dtype(self): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpMultiAxises(TestMLUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -72,7 +74,8 @@ def setUp(self): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceAll(TestMLUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -88,7 +91,8 @@ def setUp(self): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_int32(TestMLUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -100,11 +104,12 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT32) + 'out_dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype(np.int32) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.int32) } def init_dtype(self): @@ -113,7 +118,8 @@ def init_dtype(self): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_fp16(TestMLUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -125,12 +131,12 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP16) + 'out_dtype': int(core.VarDesc.VarType.FP16), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype( - np.float16) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.float16) } def init_dtype(self): @@ -142,7 +148,8 @@ def test_check_output(self): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_fp32(TestMLUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -154,12 +161,12 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } def init_dtype(self): diff --git a/python/paddle/fluid/tests/unittests/mlu/test_reduce_prod_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_reduce_prod_op_mlu.py index 85179a72f0302..fe690daabea33 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_reduce_prod_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_reduce_prod_op_mlu.py @@ -29,7 +29,6 @@ def raw_reduce_prod(x, dim=[0], keep_dim=False): class TestProdOp(OpTest): - def setUp(self): self.op_type = "reduce_prod" self.place = paddle.device.MLUPlace(0) @@ -47,7 +46,6 @@ def test_check_output(self): class TestProd6DOp(OpTest): - def setUp(self): self.op_type = "reduce_prod" self.place = paddle.device.MLUPlace(0) @@ -70,7 +68,6 @@ def test_check_output(self): class TestProd8DOp(OpTest): - def setUp(self): self.op_type = "reduce_prod" self.place = paddle.device.MLUPlace(0) @@ -78,8 +75,9 @@ def setUp(self): self.python_api = raw_reduce_prod self.init_data_type() self.inputs = { - 'X': np.random.random( - (2, 5, 3, 2, 2, 3, 4, 2)).astype(self.data_type) + 'X': np.random.random((2, 5, 3, 2, 2, 3, 4, 2)).astype( + self.data_type + ) } self.attrs = {'dim': [2, 3, 4]} self.outputs = { diff --git a/python/paddle/fluid/tests/unittests/mlu/test_reduce_sum_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_reduce_sum_op_mlu.py index 823a7398f0709..219b17a039e15 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_reduce_sum_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_reduce_sum_op_mlu.py @@ -24,7 +24,6 @@ class TestMLUReduceSumOp(OpTest): - def setUp(self): self.init_op_type() self.initTestCase() @@ -32,16 +31,16 @@ def setUp(self): self.attrs = { 'dim': self.axis, 'keep_dim': self.keep_dim, - 'reduce_all': self.reduce_all + 'reduce_all': self.reduce_all, } self.inputs = {'X': np.random.random(self.shape).astype("float32")} if self.attrs['reduce_all']: self.outputs = {'Out': self.inputs['X'].sum()} else: self.outputs = { - 'Out': - self.inputs['X'].sum(axis=self.axis, - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].sum( + axis=self.axis, keepdims=self.attrs['keep_dim'] + ) } def set_mlu(self): @@ -62,105 +61,92 @@ def init_op_type(self): def initTestCase(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) class TestSumOp5D(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (1, 2, 5, 6, 10) - self.axis = (0, ) + self.axis = (0,) class TestSumOp6D(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (1, 1, 2, 5, 6, 10) - self.axis = (0, ) + self.axis = (0,) class TestSumOp8D(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (1, 3, 1, 2, 1, 4, 3, 10) self.axis = (0, 3) class Test1DReduce(TestMLUReduceSumOp): - def initTestCase(self): self.shape = 120 - self.axis = (0, ) + self.axis = (0,) class Test2DReduce0(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (20, 10) - self.axis = (0, ) + self.axis = (0,) class Test2DReduce1(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (20, 10) - self.axis = (1, ) + self.axis = (1,) class Test3DReduce0(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (5, 6, 7) - self.axis = (1, ) + self.axis = (1,) class Test3DReduce1(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (5, 6, 7) - self.axis = (2, ) + self.axis = (2,) class Test3DReduce2(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (5, 6, 7) - self.axis = (-2, ) + self.axis = (-2,) class Test3DReduce3(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (5, 6, 7) self.axis = (1, 2) class TestKeepDimReduce(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (5, 6, 10) - self.axis = (1, ) + self.axis = (1,) self.keep_dim = True class TestKeepDim8DReduce(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (2, 5, 3, 2, 2, 3, 4, 2) self.axis = (3, 4, 5) self.keep_dim = True def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.03) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.03 + ) class TestReduceAll(TestMLUReduceSumOp): - def initTestCase(self): self.shape = (5, 6, 2, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = True diff --git a/python/paddle/fluid/tests/unittests/mlu/test_relu6_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_relu6_op_mlu.py index 45a1a825c435a..4e8ebacf0eb69 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_relu6_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_relu6_op_mlu.py @@ -34,7 +34,6 @@ def ref_relu6(x, threshold=6.0): class TestRelu6(OpTest): - def setUp(self): self.set_mlu() self.op_type = "relu6" @@ -64,7 +63,6 @@ def init_dtype(self): class TestRelu6Float16(TestRelu6): - def set_mlu(self): self.__class__.use_mlu = True self.__class__.no_need_check_grad = True @@ -77,7 +75,6 @@ def test_check_output(self): class TestReluNeg(TestRelu6): - def setUp(self): self.set_mlu() self.op_type = "relu6" @@ -104,7 +101,6 @@ def test_check_output(self): class TestRelu6Net(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -119,9 +115,9 @@ def _test(self, run_mlu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.nn.functional.relu6(sum) @@ -145,16 +141,17 @@ def _test(self, run_mlu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_relu_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_relu_op_mlu.py index a6c4303a2085a..db62e592ac2dd 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_relu_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_relu_op_mlu.py @@ -26,7 +26,6 @@ class TestRelu(OpTest): - def setUp(self): self.set_mlu() self.op_type = "relu" @@ -52,7 +51,6 @@ def test_check_output(self): class TestReluFp16(OpTest): - def setUp(self): self.set_mlu() self.op_type = "relu" @@ -79,7 +77,6 @@ def test_check_output(self): class TestReluNeg(OpTest): - def setUp(self): self.set_mlu() self.op_type = "relu" @@ -105,7 +102,6 @@ def test_check_output(self): class TestReluNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -120,9 +116,9 @@ def _test(self, run_mlu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.nn.functional.relu(sum) @@ -146,16 +142,17 @@ def _test(self, run_mlu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_reshape2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_reshape2_op_mlu.py index 3354acf6d0cbe..94090b9a1776d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_reshape2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_reshape2_op_mlu.py @@ -26,7 +26,6 @@ class TestReshape2(OpTest): - def setUp(self): self.set_mlu() self.op_type = "reshape2" @@ -37,7 +36,7 @@ def setUp(self): self.attrs = {"shape": self.new_shape} self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def set_mlu(self): @@ -56,7 +55,6 @@ def test_check_grad_normal(self): class TestReshape2_case2(TestReshape2): - def init_data(self): self.ori_shape = (2, 100) self.new_shape = (-1, 10) @@ -64,7 +62,6 @@ def init_data(self): class TestReshape2_case3(TestReshape2): - def init_data(self): self.ori_shape = (100, 5, 6) self.new_shape = (-1, 0, 3) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_rnn_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_rnn_op_mlu.py index e5f9d154bfd49..22597342f7fb4 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_rnn_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_rnn_op_mlu.py @@ -35,7 +35,6 @@ class TestRNNOp(OpTest): - def get_weight_names(self): weight_names = [] for i in range(self.num_layers): @@ -64,41 +63,53 @@ def setUp(self): self.direction_num = 2 if self.is_bidirec else 1 direction = "bidirectional" if self.is_bidirec else "forward" - input = np.random.uniform(low=-0.1, - high=0.1, - size=(self.seq_length, self.batch_size, - self.input_size)).astype(self.dtype) + input = np.random.uniform( + low=-0.1, + high=0.1, + size=(self.seq_length, self.batch_size, self.input_size), + ).astype(self.dtype) input[11][1:][:] = 0 input[10][2:][:] = 0 input[9][3:][:] = 0 input[8][4:][:] = 0 - rnn1 = LSTM(self.input_size, - self.hidden_size, - num_layers=self.num_layers, - time_major=True, - direction=direction, - dropout=self.dropout, - dtype=self.dtype) + rnn1 = LSTM( + self.input_size, + self.hidden_size, + num_layers=self.num_layers, + time_major=True, + direction=direction, + dropout=self.dropout, + dtype=self.dtype, + ) flat_w = get_params_for_net(rnn1) - output, (last_hidden, - last_cell) = rnn1(input, sequence_length=self.sequence_length) + output, (last_hidden, last_cell) = rnn1( + input, sequence_length=self.sequence_length + ) init_h = np.zeros( - (self.num_layers * self.direction_num, self.batch_size, - self.hidden_size)).astype(self.dtype) + ( + self.num_layers * self.direction_num, + self.batch_size, + self.hidden_size, + ) + ).astype(self.dtype) init_c = np.zeros( - (self.num_layers * self.direction_num, self.batch_size, - self.hidden_size)).astype(self.dtype) + ( + self.num_layers * self.direction_num, + self.batch_size, + self.hidden_size, + ) + ).astype(self.dtype) state_out = np.ndarray((300)).astype("uint8") self.inputs = { 'Input': input, 'WeightList': flat_w, 'PreState': [('init_h', init_h), ('init_c', init_c)], - 'SequenceLength': self.sequence_length + 'SequenceLength': self.sequence_length, } if self.sequence_length is None: self.inputs = { @@ -113,13 +124,13 @@ def setUp(self): 'hidden_size': self.hidden_size, 'num_layers': self.num_layers, 'mode': self.mode, - 'is_test': self.is_test + 'is_test': self.is_test, } self.outputs = { 'Out': output, "State": [('last_hidden', last_hidden), ('last_cell', last_cell)], 'Reserve': np.ndarray((400)).astype("uint8"), - 'DropoutState': state_out + 'DropoutState': state_out, } def init_dtype(self): @@ -135,7 +146,8 @@ def test_output(self): self.check_output_with_place( self.place, atol=1e-4, - no_check_set=['Reserve', 'DropoutState', 'State']) + no_check_set=['Reserve', 'DropoutState', 'State'], + ) def set_attrs(self): pass @@ -146,39 +158,38 @@ def test_grad(self): var_name_list = self.get_weight_names() grad_check_list = ['Input', 'init_h', 'init_c'] grad_check_list.extend(var_name_list) - self.check_grad_with_place(self.place, set(grad_check_list), - ['Out', 'last_hidden', 'last_cell']) + self.check_grad_with_place( + self.place, + set(grad_check_list), + ['Out', 'last_hidden', 'last_cell'], + ) class TestRNNOp1(TestRNNOp): - def set_attrs(self): self.sequence_length = None class TestRNNOp2(TestRNNOp): - def set_attrs(self): self.sequence_length = None self.is_bidirec = True class TestRNNOp3(TestRNNOp): - def set_attrs(self): self.is_test = True self.sequence_length = None class TestRNNOp4(TestRNNOp): - def set_attrs(self): self.is_test = True self.sequence_length = None self.is_bidirec = True -#TODO(chenxiao): cnnl doesn't support num_layers > 1 case +# TODO(chenxiao): cnnl doesn't support num_layers > 1 case # class TestRNNOp5(TestRNNOp): # def set_attrs(self): diff --git a/python/paddle/fluid/tests/unittests/mlu/test_roi_align_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_roi_align_op_mlu.py index 61ca7b38a8059..0f7b347315fc6 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_roi_align_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_roi_align_op_mlu.py @@ -26,7 +26,6 @@ class TestROIAlignMLUOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() @@ -37,7 +36,7 @@ def set_data(self): self.inputs = { 'X': self.x, 'ROIs': self.rois[:, 1:5], - 'RoisNum': np.asarray(seq_len).astype('int32') + 'RoisNum': np.asarray(seq_len).astype('int32'), } # print("self.inputs: ",self.inputs) @@ -46,7 +45,7 @@ def set_data(self): 'pooled_height': self.pooled_height, 'pooled_width': self.pooled_width, 'sampling_ratio': self.sampling_ratio, - 'aligned': self.aligned + 'aligned': self.aligned, } self.outputs = {'Out': self.out_data} @@ -68,25 +67,45 @@ def init_test_case(self): self.x = np.random.random(self.x_dim).astype('float32') - def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, roi_bin_grid_w, - bin_size_h, bin_size_w): + def pre_calc( + self, + x_i, + roi_xmin, + roi_ymin, + roi_bin_grid_h, + roi_bin_grid_w, + bin_size_h, + bin_size_w, + ): count = roi_bin_grid_h * roi_bin_grid_w bilinear_pos = np.zeros( [self.channels, self.pooled_height, self.pooled_width, count, 4], - np.float32) - bilinear_w = np.zeros([self.pooled_height, self.pooled_width, count, 4], - np.float32) + np.float32, + ) + bilinear_w = np.zeros( + [self.pooled_height, self.pooled_width, count, 4], np.float32 + ) for ph in range(self.pooled_width): for pw in range(self.pooled_height): c = 0 for iy in range(roi_bin_grid_h): - y = roi_ymin + ph * bin_size_h + (iy + 0.5) * \ - bin_size_h / roi_bin_grid_h + y = ( + roi_ymin + + ph * bin_size_h + + (iy + 0.5) * bin_size_h / roi_bin_grid_h + ) for ix in range(roi_bin_grid_w): - x = roi_xmin + pw * bin_size_w + (ix + 0.5) * \ - bin_size_w / roi_bin_grid_w - if y < -1.0 or y > self.height or \ - x < -1.0 or x > self.width: + x = ( + roi_xmin + + pw * bin_size_w + + (ix + 0.5) * bin_size_w / roi_bin_grid_w + ) + if ( + y < -1.0 + or y > self.height + or x < -1.0 + or x > self.width + ): continue if y <= 0: y = 0 @@ -107,14 +126,18 @@ def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, roi_bin_grid_w, hy = 1 - ly hx = 1 - lx for ch in range(self.channels): - bilinear_pos[ch, ph, pw, c, 0] = x_i[ch, y_low, - x_low] - bilinear_pos[ch, ph, pw, c, 1] = x_i[ch, y_low, - x_high] - bilinear_pos[ch, ph, pw, c, 2] = x_i[ch, y_high, - x_low] - bilinear_pos[ch, ph, pw, c, 3] = x_i[ch, y_high, - x_high] + bilinear_pos[ch, ph, pw, c, 0] = x_i[ + ch, y_low, x_low + ] + bilinear_pos[ch, ph, pw, c, 1] = x_i[ + ch, y_low, x_high + ] + bilinear_pos[ch, ph, pw, c, 2] = x_i[ + ch, y_high, x_low + ] + bilinear_pos[ch, ph, pw, c, 3] = x_i[ + ch, y_high, x_high + ] bilinear_w[ph, pw, c, 0] = hy * hx bilinear_w[ph, pw, c, 1] = hy * lx bilinear_w[ph, pw, c, 2] = ly * hx @@ -124,10 +147,15 @@ def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, roi_bin_grid_w, def calc_roi_align(self): self.out_data = np.zeros( - (self.rois_num, self.channels, self.pooled_height, - self.pooled_width)).astype('float32') - - offset = 0.5 if self.aligned else 0. + ( + self.rois_num, + self.channels, + self.pooled_height, + self.pooled_width, + ) + ).astype('float32') + + offset = 0.5 if self.aligned else 0.0 for i in range(self.rois_num): roi = self.rois[i] roi_batch_id = int(roi[0]) @@ -145,16 +173,27 @@ def calc_roi_align(self): bin_size_h = float(roi_height) / float(self.pooled_height) bin_size_w = float(roi_width) / float(self.pooled_width) - roi_bin_grid_h = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_height / self.pooled_height) - roi_bin_grid_w = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_width / self.pooled_width) + roi_bin_grid_h = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_height / self.pooled_height) + ) + roi_bin_grid_w = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_width / self.pooled_width) + ) count = max(int(roi_bin_grid_h * roi_bin_grid_w), 1) pre_size = count * self.pooled_width * self.pooled_height - bilinear_pos, bilinear_w = self.pre_calc(x_i, roi_xmin, roi_ymin, - int(roi_bin_grid_h), - int(roi_bin_grid_w), - bin_size_h, bin_size_w) + bilinear_pos, bilinear_w = self.pre_calc( + x_i, + roi_xmin, + roi_ymin, + int(roi_bin_grid_h), + int(roi_bin_grid_w), + bin_size_h, + bin_size_w, + ) for ch in range(self.channels): align_per_bin = (bilinear_pos[ch] * bilinear_w).sum(axis=-1) output_val = align_per_bin.mean(axis=-1) @@ -168,14 +207,18 @@ def make_rois(self): # self.rois_lod[0].append(bno) self.rois_lod[0].append(1) x1 = np.random.randint( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.randint( - 0, self.height // self.spatial_scale - self.pooled_height) + 0, self.height // self.spatial_scale - self.pooled_height + ) - x2 = np.random.randint(x1 + self.pooled_width, - self.width // self.spatial_scale) - y2 = np.random.randint(y1 + self.pooled_height, - self.height // self.spatial_scale) + x2 = np.random.randint( + x1 + self.pooled_width, self.width // self.spatial_scale + ) + y2 = np.random.randint( + y1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) @@ -197,7 +240,6 @@ def test_check_grad(self): class TestROIAlignOpWithMinusSample(TestROIAlignMLUOp): - def init_test_case(self): self.batch_size = 3 self.channels = 3 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_scale_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_scale_op_mlu.py index 9b04042458d0a..c7858a482295d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_scale_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_scale_op_mlu.py @@ -28,7 +28,6 @@ class TestScaleOp(OpTest): - def setUp(self): self.op_type = "scale" self.place = paddle.device.MLUPlace(0) @@ -49,7 +48,6 @@ def test_check_output(self): class TestScaleOpScaleVariable(OpTest): - def setUp(self): self.op_type = "scale" self.place = paddle.device.MLUPlace(0) @@ -59,7 +57,7 @@ def setUp(self): self.scale = -2.3 self.inputs = { 'X': np.random.random((10, 10)).astype(self.dtype), - 'ScaleTensor': np.array([self.scale]).astype('float32') + 'ScaleTensor': np.array([self.scale]).astype('float32'), } self.attrs = {} self.outputs = {'Out': self.inputs['X'] * self.dtype(self.scale)} @@ -72,7 +70,6 @@ def test_check_output(self): class TestScaleOpSelectedRows(unittest.TestCase): - def init_dtype_type(self): pass @@ -91,8 +88,9 @@ def check_with_place(self, place, in_name, out_name): in_selected_rows = scope.var(in_name).get_selected_rows() in_selected_rows.set_height(in_height) in_selected_rows.set_rows(in_rows) - in_array = np.random.random( - (len(in_rows), in_row_numel)).astype(self.dtype) + in_array = np.random.random((len(in_rows), in_row_numel)).astype( + self.dtype + ) in_tensor = in_selected_rows.get_tensor() in_tensor.set(in_array, place) @@ -131,9 +129,7 @@ def test_scale_selected_rows_inplace(self): class TestScaleRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.scale([10]) @@ -141,10 +137,10 @@ def test_type(): # Add FP16 test -@unittest.skipIf(not core.is_compiled_with_mlu(), - "core is not compiled with MLU") +@unittest.skipIf( + not core.is_compiled_with_mlu(), "core is not compiled with MLU" +) class TestScaleFp16Op(TestScaleOp): - def init_dtype_type(self): self.dtype = np.float16 @@ -152,10 +148,10 @@ def test_check_output(self): self.check_output_with_place(self.place, atol=0.002) -@unittest.skipIf(not core.is_compiled_with_mlu(), - "core is not compiled with MLU") +@unittest.skipIf( + not core.is_compiled_with_mlu(), "core is not compiled with MLU" +) class TestScaleFp16OpSelectedRows(TestScaleOpSelectedRows): - def init_dtype_type(self): self.dtype = np.float16 @@ -169,7 +165,6 @@ def test_scale_selected_rows_inplace(self): class TestScaleApiStatic(unittest.TestCase): - def _executed_api(self, x, scale=1.0, bias=0.0): return paddle.scale(x, scale, bias) @@ -187,13 +182,11 @@ def test_api(self): class TestScaleInplaceApiStatic(TestScaleApiStatic): - def _executed_api(self, x, scale=1.0, bias=0.0): return x.scale_(scale, bias) class TestScaleApiDygraph(unittest.TestCase): - def _executed_api(self, x, scale=1.0, bias=0.0): return paddle.scale(x, scale, bias) @@ -207,7 +200,6 @@ def test_api(self): class TestScaleInplaceApiDygraph(TestScaleApiDygraph): - def _executed_api(self, x, scale=1.0, bias=0.0): return x.scale_(scale, bias) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_scatter_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_scatter_op_mlu.py index 76042074b0ea0..cb13f305caeba 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_scatter_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_scatter_op_mlu.py @@ -28,7 +28,6 @@ class TestScatterOp(OpTest): - def setUp(self): self.op_type = "scatter" self.place = paddle.device.MLUPlace(0) @@ -50,7 +49,6 @@ def test_check_grad(self): class TestScatterOp0(OpTest): - def setUp(self): self.op_type = "scatter" self.place = paddle.device.MLUPlace(0) @@ -73,7 +71,6 @@ def test_check_grad(self): class TestScatterOp1(OpTest): - def setUp(self): self.op_type = "scatter" self.place = paddle.device.MLUPlace(0) @@ -99,7 +96,6 @@ def test_check_grad(self): class TestScatterOp2(OpTest): - def setUp(self): self.op_type = "scatter" self.place = paddle.device.MLUPlace(0) @@ -121,7 +117,6 @@ def test_check_grad(self): class TestScatterAPI(unittest.TestCase): - def setUp(self): self.places = [paddle.device.MLUPlace(0)] self.__class__.use_mlu = True @@ -139,19 +134,26 @@ def check_static_result(self, place): input_data = np.array([[1, 1], [2, 2], [3, 3]]).astype(np.float32) index_data = np.array([2, 1, 0, 1]).astype(np.int64) - updates_data = np.array([[1, 1], [2, 2], [3, 3], - [4, 4]]).astype(np.float32) + updates_data = np.array([[1, 1], [2, 2], [3, 3], [4, 4]]).astype( + np.float32 + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input": input_data, - "index": index_data, - "updates": updates_data - }, - fetch_list=[result]) - self.assertEqual((fetches[0] == \ - np.array([[3., 3.],[6., 6.],[1., 1.]])).all(), True) + fetches = exe.run( + fluid.default_main_program(), + feed={ + "input": input_data, + "index": index_data, + "updates": updates_data, + }, + fetch_list=[result], + ) + self.assertEqual( + ( + fetches[0] == np.array([[3.0, 3.0], [6.0, 6.0], [1.0, 1.0]]) + ).all(), + True, + ) def test_static(self): for place in self.places: @@ -162,16 +164,22 @@ def test_dygraph(self): with fluid.dygraph.guard(place): x_data = np.array([[1, 1], [2, 2], [3, 3]]).astype(np.float32) index_data = np.array([2, 1, 0, 1]).astype(np.int64) - updates_data = np.array([[1, 1], [2, 2], [3, 3], - [4, 4]]).astype(np.float32) + updates_data = np.array( + [[1, 1], [2, 2], [3, 3], [4, 4]] + ).astype(np.float32) x = fluid.dygraph.to_variable(x_data) index = fluid.dygraph.to_variable(index_data) updates = fluid.dygraph.to_variable(updates_data) output1 = self.scatter(x, index, updates, overwrite=False) - self.assertEqual((output1.numpy() == \ - np.array([[3., 3.],[6., 6.],[1., 1.]])).all(), True) + self.assertEqual( + ( + output1.numpy() + == np.array([[3.0, 3.0], [6.0, 6.0], [1.0, 1.0]]) + ).all(), + True, + ) def test_large_data(self): if os.name == "nt": @@ -183,27 +191,30 @@ def test_large_data(self): def test_dygraph(): with fluid.dygraph.guard(): - mlu_out = paddle.scatter(paddle.to_tensor(x), - paddle.to_tensor(index), - paddle.to_tensor(updates)) + mlu_out = paddle.scatter( + paddle.to_tensor(x), + paddle.to_tensor(index), + paddle.to_tensor(updates), + ) return mlu_out.numpy() @switch_to_static_graph def test_static_graph(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x_t = paddle.static.data(name="x", dtype=x.dtype, shape=x.shape) - index_t = paddle.static.data(name="index", - dtype=index.dtype, - shape=index.shape) - updates_t = paddle.static.data(name="updates", - dtype=updates.dtype, - shape=updates.shape) + index_t = paddle.static.data( + name="index", dtype=index.dtype, shape=index.shape + ) + updates_t = paddle.static.data( + name="updates", dtype=updates.dtype, shape=updates.shape + ) out_t = paddle.scatter(x_t, index_t, updates_t) feed = { x_t.name: x, index_t.name: index, - updates_t.name: updates + updates_t.name: updates, } fetch = [out_t] @@ -215,7 +226,6 @@ def test_static_graph(): class TestScatterOpFp16(OpTest): - def setUp(self): self.op_type = "scatter" self.place = paddle.device.MLUPlace(0) @@ -238,7 +248,6 @@ def test_check_grad(self): class TestScatterInplaceAPI(TestScatterAPI): - def executed_api(self): self.scatter = paddle.scatter_ diff --git a/python/paddle/fluid/tests/unittests/mlu/test_set_value_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_set_value_op_mlu.py index d621aab9507b1..d9db940b02943 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_set_value_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_set_value_op_mlu.py @@ -24,7 +24,6 @@ class TestSetValueBase(unittest.TestCase): - def set_mlu(self): self.__class__.use_mlu = True self.place = paddle.device.MLUPlace(0) @@ -55,7 +54,6 @@ def _get_answer(self): class TestSetValueApi(TestSetValueBase): - def _run_static(self): paddle.enable_static() with paddle.static.program_guard(self.program): @@ -71,15 +69,18 @@ def test_api(self): static_out = self._run_static() self._get_answer() - error_msg = "\nIn {} mode: \nExpected res = \n{}, \n\nbut received : \n{}" - self.assertTrue((self.data == static_out).all(), - msg=error_msg.format("static", self.data, static_out)) + error_msg = ( + "\nIn {} mode: \nExpected res = \n{}, \n\nbut received : \n{}" + ) + self.assertTrue( + (self.data == static_out).all(), + msg=error_msg.format("static", self.data, static_out), + ) # 1. Test different type of item: int, Python slice, Paddle Tensor # 1.1 item is int class TestSetValueItemInt(TestSetValueApi): - def _call_setitem(self, x): x[0] = self.value @@ -90,7 +91,6 @@ def _get_answer(self): # 1.2 item is slice # 1.2.1 step is 1 class TestSetValueItemSlice(TestSetValueApi): - def _call_setitem(self, x): x[0:2] = self.value @@ -99,7 +99,6 @@ def _get_answer(self): class TestSetValueItemSlice2(TestSetValueApi): - def _call_setitem(self, x): x[0:-1] = self.value @@ -108,7 +107,6 @@ def _get_answer(self): class TestSetValueItemSlice3(TestSetValueApi): - def _call_setitem(self, x): x[0:-1, 0:2] = self.value @@ -117,7 +115,6 @@ def _get_answer(self): class TestSetValueItemSlice4(TestSetValueApi): - def _call_setitem(self, x): x[0:, 1:2, :] = self.value @@ -126,7 +123,6 @@ def _get_answer(self): class TestSetValueItemSlice5(TestSetValueApi): - def set_shape(self): self.shape = [100, 426, 640] @@ -137,8 +133,8 @@ def _get_answer(self): self.data[0:-1] = self.value -#TODO: Fix this after MLU support while_loop -#class TestSetValueItemSliceInWhile(TestSetValueApi): +# TODO: Fix this after MLU support while_loop +# class TestSetValueItemSliceInWhile(TestSetValueApi): # def _call_setitem(self, x): # def cond(i, x): # return i < 1 @@ -157,7 +153,6 @@ def _get_answer(self): # 1.2.2 step > 1 class TestSetValueItemSliceStep(TestSetValueApi): - def set_shape(self): self.shape = [5, 5, 5] @@ -169,7 +164,6 @@ def _get_answer(self): class TestSetValueItemSliceStep2(TestSetValueApi): - def set_shape(self): self.shape = [7, 5, 5] @@ -181,7 +175,6 @@ def _get_answer(self): class TestSetValueItemSliceStep3(TestSetValueApi): - def _call_setitem(self, x): x[0:-1, 0:2, ::2] = self.value @@ -190,7 +183,6 @@ def _get_answer(self): class TestSetValueItemSliceStep4(TestSetValueApi): - def _call_setitem(self, x): x[0:, 1:2:2, :] = self.value @@ -200,7 +192,6 @@ def _get_answer(self): # 1.2.3 step < 0 class TestSetValueItemSliceNegetiveStep(TestSetValueApi): - def set_shape(self): self.shape = [5, 2] @@ -215,7 +206,6 @@ def _get_answer(self): class TestSetValueItemSliceNegetiveStep2(TestSetValueApi): - def set_shape(self): self.shape = [5] @@ -230,7 +220,6 @@ def _get_answer(self): class TestSetValueItemSliceNegetiveStep3(TestSetValueApi): - def set_shape(self): self.shape = [3] @@ -245,7 +234,6 @@ def _get_answer(self): class TestSetValueItemSliceNegetiveStep4(TestSetValueApi): - def set_shape(self): self.shape = [3, 4, 5] @@ -260,7 +248,6 @@ def _get_answer(self): class TestSetValueItemEllipsis1(TestSetValueApi): - def _call_setitem(self, x): x[0:, ..., 1:] = self.value @@ -269,7 +256,6 @@ def _get_answer(self): class TestSetValueItemEllipsis2(TestSetValueApi): - def _call_setitem(self, x): x[0:, ...] = self.value @@ -278,7 +264,6 @@ def _get_answer(self): class TestSetValueItemEllipsis3(TestSetValueApi): - def _call_setitem(self, x): x[..., 1:] = self.value @@ -287,7 +272,6 @@ def _get_answer(self): class TestSetValueItemEllipsis4(TestSetValueApi): - def _call_setitem(self, x): x[...] = self.value @@ -297,7 +281,6 @@ def _get_answer(self): # 1.4 item is Paddle Tensor class TestSetValueItemTensor(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") x[zero] = self.value @@ -307,7 +290,6 @@ def _get_answer(self): class TestSetValueItemTensor2(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -318,7 +300,6 @@ def _get_answer(self): class TestSetValueItemTensor3(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -329,7 +310,6 @@ def _get_answer(self): class TestSetValueItemTensor4(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -340,7 +320,6 @@ def _get_answer(self): class TestSetValueItemTensor5(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -351,7 +330,6 @@ def _get_answer(self): class TestSetValueItemTensor6(TestSetValueApi): - def set_shape(self): self.shape = [3, 4, 5] @@ -366,7 +344,6 @@ def _get_answer(self): # 1.5 item is None class TestSetValueItemNone1(TestSetValueApi): - def _call_setitem(self, x): x[None] = self.value @@ -375,7 +352,6 @@ def _get_answer(self): class TestSetValueItemNone2(TestSetValueApi): - def _call_setitem(self, x): x[0, None, 1] = self.value @@ -384,7 +360,6 @@ def _get_answer(self): class TestSetValueItemNone3(TestSetValueApi): - def _call_setitem(self, x): x[:, None, None, 1] = self.value @@ -393,7 +368,6 @@ def _get_answer(self): class TestSetValueItemNone4(TestSetValueApi): - def _call_setitem(self, x): x[0, 0, None, 1] = self.value @@ -402,7 +376,6 @@ def _get_answer(self): class TestSetValueItemNone5(TestSetValueApi): - def _call_setitem(self, x): x[0, None, 0, None, 1] = self.value @@ -411,7 +384,6 @@ def _get_answer(self): class TestSetValueItemNone6(TestSetValueApi): - def _call_setitem(self, x): x[None, 0, 0, None, 0] = self.value @@ -420,7 +392,6 @@ def _get_answer(self): class TestSetValueItemNone7(TestSetValueApi): - def _call_setitem(self, x): x[:, None, 1] = np.zeros(self.shape)[:, None, 0] @@ -429,7 +400,6 @@ def _get_answer(self): class TestSetValueItemNone8(TestSetValueApi): - def _call_setitem(self, x): x[:, 1, None] = np.zeros(self.shape)[:, 0, None] @@ -438,7 +408,6 @@ def _get_answer(self): class TestSetValueItemNone9(TestSetValueApi): - def _call_setitem(self, x): x[None, :, 1, ..., None] = np.zeros(self.shape)[0, 0, :, None] @@ -448,7 +417,6 @@ def _get_answer(self): # 1.5 item is list or Tensor of bol class TestSetValueItemBool1(TestSetValueApi): - def _call_setitem(self, x): x[[True, False]] = self.value @@ -457,7 +425,6 @@ def _get_answer(self): class TestSetValueItemBool2(TestSetValueApi): - def _call_setitem(self, x): x[[False, False]] = self.value @@ -466,7 +433,6 @@ def _get_answer(self): class TestSetValueItemBool3(TestSetValueApi): - def _call_setitem(self, x): x[[False, True]] = np.zeros(self.shape[2]) @@ -475,7 +441,6 @@ def _get_answer(self): class TestSetValueItemBool4(TestSetValueApi): - def _call_setitem(self, x): idx = paddle.assign(np.array([False, True])) x[idx] = np.zeros(self.shape[2]) @@ -485,19 +450,19 @@ def _get_answer(self): class TestSetValueItemBool5(TestSetValueApi): - def _call_setitem(self, x): idx = paddle.assign( - np.array([[False, True, False], [True, True, False]])) + np.array([[False, True, False], [True, True, False]]) + ) x[idx] = self.value def _get_answer(self): - self.data[np.array([[False, True, False], [True, True, - False]])] = self.value + self.data[ + np.array([[False, True, False], [True, True, False]]) + ] = self.value class TestSetValueItemBool6(TestSetValueApi): - def _call_setitem(self, x): x[0, ...] = 0 x[x > 0] = self.value @@ -508,9 +473,7 @@ def _get_answer(self): def create_test_value_int32(parent): - class TestValueInt(parent): - def set_value(self): self.value = 7 @@ -531,9 +494,7 @@ def set_dtype(self): def create_test_value_tensor_fp32(parent): - class TestValueInt(parent): - def set_dtype(self): self.dtype = "float32" @@ -559,7 +520,6 @@ def _get_answer(self): # 3. Test different shape of value class TestSetValueValueShape1(TestSetValueApi): - def set_value(self): self.value = np.array([3, 4, 5, 6]) # shape is (4,) @@ -571,7 +531,6 @@ def _get_answer(self): class TestSetValueValueShape2(TestSetValueApi): - def set_value(self): self.value = np.array([[3, 4, 5, 6]]) # shape is (1,4) @@ -583,10 +542,10 @@ def _get_answer(self): class TestSetValueValueShape3(TestSetValueApi): - def set_value(self): - self.value = np.array([[1, 1, 1, 1], [2, 2, 2, 2], - [3, 3, 3, 3]]) # shape is (3,4) + self.value = np.array( + [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]] + ) # shape is (3,4) def _call_setitem(self, x): x[0] = self.value @@ -596,11 +555,12 @@ def _get_answer(self): class TestSetValueValueShape4(TestSetValueApi): - def set_value(self): - self.value = np.array([[1, 1, 1, 1], [2, 2, 2, 2], - [3, 3, 3, - 3]]).astype(self.dtype) # shape is (3,4) + self.value = np.array( + [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]] + ).astype( + self.dtype + ) # shape is (3,4) def _call_setitem(self, x): x[0] = paddle.assign(self.value) # x is Paddle.Tensor @@ -610,7 +570,6 @@ def _get_answer(self): class TestSetValueValueShape5(TestSetValueApi): - def set_value(self): self.value = np.array([3, 3, 3]).astype(self.dtype) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_shape_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_shape_op_mlu.py index 7e743dc13c4f6..f18ba5532b47b 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_shape_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_shape_op_mlu.py @@ -19,6 +19,7 @@ sys.path.append("..") from op_test import OpTest import paddle + # from paddle.fluid import core # from paddle.fluid.op import Operator @@ -27,7 +28,6 @@ class TestShape(OpTest): - def setUp(self): self.set_mlu() self.op_type = "shape" @@ -53,43 +53,36 @@ def test_check_output(self): class TestShape_fp16(TestShape): - def init_dtype(self): self.dtype = np.float16 class TestShape_double(TestShape): - def init_dtype(self): self.dtype = np.float64 class TestShape_int32(TestShape): - def init_dtype(self): self.dtype = np.int32 class TestShape_int64(TestShape): - def init_dtype(self): self.dtype = np.int64 class TestShape_int8(TestShape): - def init_dtype(self): self.dtype = np.int8 class TestShape_uint8(TestShape): - def init_dtype(self): self.dtype = np.uint8 class TestShape_bool(TestShape): - def init_dtype(self): self.dtype = bool diff --git a/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_cross_entropy_with_logits_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_cross_entropy_with_logits_op_mlu.py index ead15510e19f0..7e594cb3db5fe 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_cross_entropy_with_logits_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_cross_entropy_with_logits_op_mlu.py @@ -29,8 +29,7 @@ class TestSigmoidCrossEntropyWithLogitsOp1(OpTest): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -40,13 +39,14 @@ def setUp(self): batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.randint(0, 2, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.randint(0, 2, (batch_size, num_classes)).astype( + self.dtype + ), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -71,10 +71,10 @@ def init_dtype(self): self.dtype = np.float32 -class TestSigmoidCrossEntropyWithLogitsOp3(TestSigmoidCrossEntropyWithLogitsOp1 - ): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ +class TestSigmoidCrossEntropyWithLogitsOp3( + TestSigmoidCrossEntropyWithLogitsOp1 +): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -84,13 +84,14 @@ def setUp(self): batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.uniform(0, 1, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.uniform(0, 1, (batch_size, num_classes)).astype( + self.dtype + ), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -102,10 +103,10 @@ def setUp(self): self.outputs = {'Out': -term1 - term2} -class TestSigmoidCrossEntropyWithLogitsOp5(TestSigmoidCrossEntropyWithLogitsOp1 - ): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ +class TestSigmoidCrossEntropyWithLogitsOp5( + TestSigmoidCrossEntropyWithLogitsOp1 +): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -115,14 +116,14 @@ def setUp(self): batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + [num_classes])).astype( - self.dtype)), - 'Label': - np.random.uniform(0, 1, tuple(batch_size + [num_classes])).astype( - self.dtype) + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype) + ), + 'Label': np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -134,10 +135,10 @@ def setUp(self): self.outputs = {'Out': -term1 - term2} -class TestSigmoidCrossEntropyWithLogitsOp6(TestSigmoidCrossEntropyWithLogitsOp1 - ): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ +class TestSigmoidCrossEntropyWithLogitsOp6( + TestSigmoidCrossEntropyWithLogitsOp1 +): + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -147,14 +148,14 @@ def setUp(self): batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + [num_classes])).astype( - self.dtype)), - 'Label': - np.random.randint(0, 2, tuple(batch_size + [num_classes])).astype( - self.dtype) + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype) + ), + 'Label': np.random.randint( + 0, 2, tuple(batch_size + [num_classes]) + ).astype(self.dtype), } # Fw Pass is implemented as elementwise sigmoid followed by diff --git a/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_op_mlu.py index b45dba73c5d65..0d7e5b08ee148 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_sigmoid_op_mlu.py @@ -24,7 +24,6 @@ class TestMLUSigmoid(OpTest): - def setUp(self): self.op_type = "sigmoid" self.set_mlu() @@ -41,9 +40,9 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.01 + ) def set_mlu(self): self.__class__.use_mlu = True @@ -54,7 +53,6 @@ def init_dtype(self): class TestMLUSigmoidFp16(TestMLUSigmoid): - def test_check_output(self): self.check_output_with_place(self.place, atol=1e-3) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_size_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_size_op_mlu.py index a56ca8fe5c91f..977359f8a9634 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_size_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_size_op_mlu.py @@ -25,7 +25,6 @@ class TestSizeOp(OpTest): - def setUp(self): self.op_type = "size" self.shape = [] @@ -42,31 +41,26 @@ def test_check_output(self): class TestRank1Tensor(TestSizeOp): - def config(self): self.shape = [2] class TestRank2Tensor(TestSizeOp): - def config(self): self.shape = [2, 3] class TestRank3Tensor(TestSizeOp): - def config(self): self.shape = [2, 3, 100] class TestLargeTensor(TestSizeOp): - def config(self): self.shape = [2**10] class TestSizeAPI(unittest.TestCase): - def test_size_static(self): main_program = fluid.Program() startup_program = fluid.Program() @@ -80,17 +74,19 @@ def test_size_static(self): out_1 = paddle.fluid.layers.size(x_1) out_2 = paddle.fluid.layers.size(x_2) exe = paddle.static.Executor(place=paddle.MLUPlace(0)) - res_1, res_2 = exe.run(feed={ - "x_1": input_1, - "x_2": input_2, - }, - fetch_list=[out_1, out_2]) - assert (np.array_equal(res_1, - np.array([np.size(input_1) - ]).astype("int64"))) - assert (np.array_equal(res_2, - np.array([np.size(input_2) - ]).astype("int64"))) + res_1, res_2 = exe.run( + feed={ + "x_1": input_1, + "x_2": input_2, + }, + fetch_list=[out_1, out_2], + ) + assert np.array_equal( + res_1, np.array([np.size(input_1)]).astype("int64") + ) + assert np.array_equal( + res_2, np.array([np.size(input_2)]).astype("int64") + ) def test_size_imperative(self): paddle.disable_static(paddle.MLUPlace(0)) @@ -100,8 +96,8 @@ def test_size_imperative(self): x_2 = paddle.to_tensor(input_2) out_1 = paddle.fluid.layers.size(x_1) out_2 = paddle.fluid.layers.size(x_2) - assert (np.array_equal(out_1.numpy().item(0), np.size(input_1))) - assert (np.array_equal(out_2.numpy().item(0), np.size(input_2))) + assert np.array_equal(out_1.numpy().item(0), np.size(input_1)) + assert np.array_equal(out_2.numpy().item(0), np.size(input_2)) paddle.enable_static() def test_error(self): diff --git a/python/paddle/fluid/tests/unittests/mlu/test_slice_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_slice_op_mlu.py index a9f21a24e68d4..bc23ea270b5f4 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_slice_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_slice_op_mlu.py @@ -29,7 +29,6 @@ # Situation 1: starts(list, no tensor), ends(list, no tensor) # 1.1 without attr(decrease) class TestSliceOp(OpTest): - def setUp(self): self.op_type = "slice" self.set_mlu() @@ -40,7 +39,7 @@ def setUp(self): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -55,9 +54,9 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) def set_mlu(self): self.__class__.use_mlu = True @@ -65,7 +64,6 @@ def set_mlu(self): class TestCase1(TestSliceOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-3, 0, 2] @@ -76,7 +74,6 @@ def config(self): class TestCase2(TestSliceOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-3, 0, 2] @@ -88,7 +85,6 @@ def config(self): # 1.2 with attr(decrease) class TestSliceOp_decs_dim(OpTest): - def setUp(self): self.op_type = "slice" self.set_mlu() @@ -116,9 +112,9 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) def set_mlu(self): self.__class__.use_mlu = True @@ -126,7 +122,6 @@ def set_mlu(self): class TestSliceOp_decs_dim_2(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [1, 0, 2] @@ -138,7 +133,6 @@ def config(self): class TestSliceOp_decs_dim_3(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-1, 0, 2] @@ -150,7 +144,6 @@ def config(self): class TestSliceOp_decs_dim_4(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 7]).astype("float32") self.starts = [0, 1, 2, 3] @@ -162,7 +155,6 @@ def config(self): class TestSliceOp_decs_dim_5(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-1] @@ -174,7 +166,6 @@ def config(self): class TestSliceOp_decs_dim_6(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [0, 1, 2, 3] @@ -188,7 +179,6 @@ def config(self): # Situation 2: starts(list, have tensor), ends(list, no tensor) # without attr(decrease) class TestSliceOp_starts_ListTensor(OpTest): - def setUp(self): self.op_type = "slice" self.set_mlu() @@ -196,8 +186,9 @@ def setUp(self): starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int64') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int64') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} self.outputs = {'Out': self.out} @@ -205,7 +196,7 @@ def setUp(self): 'axes': self.axes, 'starts': self.starts_infer, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -222,9 +213,9 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) def set_mlu(self): self.__class__.use_mlu = True @@ -234,7 +225,6 @@ def set_mlu(self): # Situation 2: starts(list, have tensor), ends(list, no tensor) # with attr(decrease) class TestSliceOp_decs_dim_starts_ListTensor(OpTest): - def setUp(self): self.op_type = "slice" self.set_mlu() @@ -242,8 +232,9 @@ def setUp(self): starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} @@ -271,9 +262,9 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) def set_mlu(self): self.__class__.use_mlu = True @@ -281,8 +272,8 @@ def set_mlu(self): class TestSliceOp_decs_dim_5_starts_ListTensor( - TestSliceOp_decs_dim_starts_ListTensor): - + TestSliceOp_decs_dim_starts_ListTensor +): def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float32") self.starts = [-1] @@ -298,7 +289,6 @@ def config(self): # Situation 3: starts(tensor), ends(list, no tensor) # with attr(decrease) class TestSliceOp_decs_dim_starts_OneTensor(OpTest): - def setUp(self): self.op_type = "slice" self.__class__.use_mlu = True @@ -306,7 +296,7 @@ def setUp(self): self.config() self.inputs = { 'Input': self.input, - "StartsTensor": np.array(self.starts, dtype="int32") + "StartsTensor": np.array(self.starts, dtype="int32"), } self.outputs = {'Out': self.out} self.attrs = { @@ -330,15 +320,14 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) # Situation 4: starts(tensor), ends(tensor) # without attr(decrease) class TestSliceOp_starts_OneTensor_ends_OneTensor(OpTest): - def setUp(self): self.op_type = "slice" self.__class__.use_mlu = True @@ -348,14 +337,14 @@ def setUp(self): self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int64"), - "EndsTensor": np.array(self.ends, dtype="int32") + "EndsTensor": np.array(self.ends, dtype="int32"), } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, #'starts': self.starts, #'ends': self.ends_infer, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -370,15 +359,14 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) # Situation 5: starts(tensor), ends(tensor) # with attr(decrease) class TestSliceOp_decs_dim_starts_and_ends_OneTensor(OpTest): - def setUp(self): self.op_type = "slice" self.__class__.use_mlu = True @@ -387,7 +375,7 @@ def setUp(self): self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int32"), - "EndsTensor": np.array(self.ends, dtype="int32") + "EndsTensor": np.array(self.ends, dtype="int32"), } self.outputs = {'Out': self.out} self.attrs = { @@ -411,15 +399,14 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) # Situation 6: starts(tensor), ends(list, have tensor) # without attr(decrease) class TestSliceOp_starts_OneTensor_ends_ListTensor(OpTest): - def setUp(self): self.op_type = "slice" self.__class__.use_mlu = True @@ -428,20 +415,21 @@ def setUp(self): ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("y" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("y" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int32"), - 'EndsTensorList': ends_tensor + 'EndsTensorList': ends_tensor, } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, #'starts': self.starts, 'ends': self.ends_infer, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -458,14 +446,13 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) # Test float16 class TestFP16(OpTest): - def setUp(self): self.op_type = "slice" self.__class__.use_mlu = True @@ -477,7 +464,7 @@ def setUp(self): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -493,13 +480,12 @@ def test_check_output(self): self.check_output_with_place(self.place, atol=1e-5) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) class TestFP16_2(OpTest): - def setUp(self): self.op_type = "slice" self.__class__.use_mlu = True @@ -511,7 +497,7 @@ def setUp(self): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -527,24 +513,28 @@ def test_check_output(self): self.check_output_with_place(self.place, atol=1e-5) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006, - numeric_grad_delta=0.5) + self.check_grad_with_place( + self.place, + ['Input'], + 'Out', + max_relative_error=0.006, + numeric_grad_delta=0.5, + ) class TestSliceApiWithTensor(unittest.TestCase): - def test_starts_ends_is_tensor(self): with paddle.fluid.dygraph.guard(): a = paddle.rand(shape=[4, 5, 6], dtype='float32') axes = [0, 1, 2] starts = [-3, 0, 2] ends = [3, 2, 4] - a_1 = paddle.slice(a, - axes=axes, - starts=paddle.to_tensor(starts, dtype='int32'), - ends=paddle.to_tensor(ends, dtype='int32')) + a_1 = paddle.slice( + a, + axes=axes, + starts=paddle.to_tensor(starts, dtype='int32'), + ends=paddle.to_tensor(ends, dtype='int32'), + ) a_2 = paddle.slice(a, axes=axes, starts=starts, ends=ends) np.testing.assert_allclose(a_1.numpy(), a_2.numpy()) @@ -567,24 +557,22 @@ def test_bool_tensor(self): class TestImperativeVarBaseGetItem(unittest.TestCase): - def test_getitem_with_long(self): with fluid.dygraph.guard(): data = np.random.random((2, 80, 16128)).astype('float32') var = fluid.dygraph.to_variable(data) - sliced = var[:, 10:, :var.shape[1]] # var.shape[1] is 80L here + sliced = var[:, 10:, : var.shape[1]] # var.shape[1] is 80L here self.assertEqual(sliced.shape, [2, 70, 80]) - sliced = var[:, var.shape[0]:, var.shape[0]:var.shape[1]] + sliced = var[:, var.shape[0] :, var.shape[0] : var.shape[1]] self.assertEqual(sliced.shape, [2, 78, 78]) def test_getitem_with_float(self): - def test_float_in_slice_item(): with fluid.dygraph.guard(): data = np.random.random((2, 80, 16128)).astype('float32') var = fluid.dygraph.to_variable(data) - sliced = var[:, 1.1:, :var.shape[1]] + sliced = var[:, 1.1:, : var.shape[1]] self.assertRaises(Exception, test_float_in_slice_item) @@ -598,7 +586,6 @@ def test_float_in_index(): class TestInferShape(unittest.TestCase): - def test_axis_less_than_zero(self): # Using paddle.disable_static will make other unittests fail. @@ -606,13 +593,18 @@ def test_axis_less_than_zero(self): x_arr = np.arange(0, 24, dtype=np.float32).reshape([2, 3, 4]) x = paddle.to_tensor(x_arr) - pp_slice = paddle.slice(x, [ - 100, - ], [0], [1]) + pp_slice = paddle.slice( + x, + [ + 100, + ], + [0], + [1], + ) np_slice = x_arr[:, :, 0:1] np.testing.assert_allclose(pp_slice, np_slice) - pp_slice = paddle.slice(x, (-100, ), [0], [1]) + pp_slice = paddle.slice(x, (-100,), [0], [1]) np_slice = x_arr[0:1] np.testing.assert_allclose(pp_slice, np_slice) @@ -620,9 +612,11 @@ def test_axis_less_than_zero(self): x = paddle.to_tensor(np.reshape(x_arr, (0, 0, 0))) starts = paddle.to_tensor( - np.reshape(np.array([], dtype=np.int32), (0, ))) + np.reshape(np.array([], dtype=np.int32), (0,)) + ) ends = paddle.to_tensor( - np.reshape(np.array([], dtype=np.int32), (0, ))) + np.reshape(np.array([], dtype=np.int32), (0,)) + ) with self.assertRaises(ValueError): paddle.slice(x, [-1000000], starts, ends) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_softmax_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_softmax_op_mlu.py index b9362c9a27d49..8c1ebbe01e5cf 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_softmax_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_softmax_op_mlu.py @@ -32,7 +32,7 @@ def stable_softmax(x): """Compute the softmax of vector x in a numerically stable way.""" # clip to shiftx, otherwise, when calc loss with # log(exp(shiftx)), may get log(0)=INF - shiftx = (x - np.max(x)).clip(-64.) + shiftx = (x - np.max(x)).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) @@ -47,7 +47,6 @@ def ref_softmax(x, axis=None, dtype=None): class TestSoftmaxOp(OpTest): - def get_x_shape(self): return [10, 10] @@ -79,19 +78,17 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ["X"], - "Out", - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ["X"], "Out", max_relative_error=0.01 + ) class TestSoftmaxOp2(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] class TestSoftmaxOp3(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -100,7 +97,6 @@ def get_axis(self): class TestSoftmaxOp4(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -109,7 +105,6 @@ def get_axis(self): class TestSoftmaxOp5(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -118,7 +113,6 @@ def get_axis(self): class TestSoftmaxOp6(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -127,10 +121,9 @@ def get_axis(self): class TestSoftmaxAPI(unittest.TestCase): - def setUp(self): self.place = paddle.MLUPlace(0) - self.x_np = np.random.uniform(-1., 1., [2, 3, 4, 5]).astype('float32') + self.x_np = np.random.uniform(-1.0, 1.0, [2, 3, 4, 5]).astype('float32') self.out_ref = np.apply_along_axis(stable_softmax, -1, self.x_np) self.executed_api() @@ -180,19 +173,18 @@ def test_error(self): # The input type must be Variable. self.assertRaises(TypeError, self.softmax, 1) # The input dtype must be float16, float32 - x_int32 = paddle.fluid.data(name='x_int32', - shape=[2, 3], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[2, 3], dtype='int32' + ) self.assertRaises(TypeError, self.softmax, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[2, 3], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[2, 3], dtype='float16' + ) self.softmax(x_fp16) class TestSoftmaxInplaceAPI(TestSoftmaxAPI): - def executed_api(self): self.softmax = F.softmax_ diff --git a/python/paddle/fluid/tests/unittests/mlu/test_softmax_with_cross_entropy_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_softmax_with_cross_entropy_op_mlu.py index e7ae6dd511177..bf77d52532926 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_softmax_with_cross_entropy_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_softmax_with_cross_entropy_op_mlu.py @@ -28,7 +28,6 @@ class TestSoftmaxWithCrossEntropyOp(OpTest): - def set_mlu(self): self.__class__.use_mlu = True @@ -51,8 +50,10 @@ def setUp(self): self.initParams() logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, logits) if self.soft_label: @@ -63,8 +64,9 @@ def setUp(self): self.shape[self.axis] = 1 labels = np.random.randint(0, axis_dim, self.shape, dtype="int64") - loss = cross_entropy(softmax, labels, self.soft_label, self.axis, - self.ignore_index) + loss = cross_entropy( + softmax, labels, self.soft_label, self.axis, self.ignore_index + ) one_hot_label = np.eye(axis_dim)[labels.reshape(-1)] @@ -72,7 +74,7 @@ def setUp(self): self.outputs = { "Backprop": (softmax - one_hot_label).astype(self.dtype), "Softmax": softmax.astype(self.dtype), - "Loss": loss.astype(self.dtype) + "Loss": loss.astype(self.dtype), } self.attrs = { "numeric_stable_mode": self.numeric_stable_mode, @@ -90,14 +92,16 @@ def test_check_grad(self): if self.dtype == np.float16: return # fp32 has low precision, cpu and mlu both need to relax the max_relative_error if using fp32 - self.check_grad_with_place(self.place, ['Logits'], - 'Loss', - numeric_grad_delta=0.001, - max_relative_error=0.5) + self.check_grad_with_place( + self.place, + ['Logits'], + 'Loss', + numeric_grad_delta=0.001, + max_relative_error=0.5, + ) class TestPowNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -112,9 +116,9 @@ def _test(self, run_mlu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -138,16 +142,17 @@ def _test(self, run_mlu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_spawn_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_spawn_mlu.py index e97f4deead5f1..605c2693afc0e 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_spawn_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_spawn_mlu.py @@ -19,12 +19,15 @@ import paddle.nn as nn import paddle.optimizer as opt import paddle.distributed as dist -from paddle.distributed.spawn import _get_subprocess_env_list, _options_valid_check, _get_default_nprocs +from paddle.distributed.spawn import ( + _get_subprocess_env_list, + _options_valid_check, + _get_default_nprocs, +) from paddle.fluid import core class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear1 = nn.Linear(10, 10) @@ -62,7 +65,6 @@ def train(print_result=False): class TestSpawn(unittest.TestCase): - def test_nprocs_greater_than_device_num_error(self): with self.assertRaises(RuntimeError): _get_subprocess_env_list(nprocs=100, options=dict()) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_split_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_split_op_mlu.py index 28585629dca3e..072b985613e08 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_split_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_split_op_mlu.py @@ -27,7 +27,6 @@ class TestCase1(OpTest): - def setUp(self): self.set_mlu() self.set_example() @@ -35,9 +34,9 @@ def setUp(self): self.place = paddle.device.MLUPlace(0) ipt = self.x.astype(self.dtype) axis = self.axis if isinstance(self.axis, int) else int(self.axis[0]) - tmp_outs = np.split(ipt, - axis=axis, - indices_or_sections=self.num_or_sections) + tmp_outs = np.split( + ipt, axis=axis, indices_or_sections=self.num_or_sections + ) tmp_outs = [o.astype(self.dtype) for o in tmp_outs] self.outputs = {'Out': []} self.outs = [] @@ -64,7 +63,6 @@ def set_example(self): class TestCase2(TestCase1): - def set_example(self): self.dtype = "float32" self.x = np.random.random((20, 4, 50)) @@ -73,7 +71,6 @@ def set_example(self): class TestCase4(TestCase1): - def set_example(self): self.dtype = "float16" self.x = np.random.random((4, 50, 20)) @@ -83,7 +80,6 @@ def set_example(self): # Test Sections class TestCase5(TestCase1): - def set_example(self): super().set_example() self.x = np.random.random((2, 10, 4)) @@ -96,7 +92,6 @@ def setUp(self): class API_TestSplit(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.layers.data('data', shape=[-1, 10], dtype='float32') @@ -105,13 +100,12 @@ def test_out(self): exe = fluid.Executor(place) input1 = np.random.random([1, 10]).astype('float32') r0, r1 = exe.run(feed={"data": input1}, fetch_list=[x0, x1]) - ex_x0, ex_x1 = np.split(input1, (3, ), axis=1) + ex_x0, ex_x1 = np.split(input1, (3,), axis=1) np.testing.assert_allclose(ex_x0, r0) np.testing.assert_allclose(ex_x1, r1) class API_TestSplit2(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.layers.data('data', shape=[-1, 10], dtype='float32') @@ -126,7 +120,6 @@ def test_out(self): class API_TestDygraphSplit(unittest.TestCase): - def test_out1(self): with fluid.dygraph.guard(paddle.MLUPlace(0)): input_1 = np.random.random([4, 6, 6]).astype("int32") @@ -158,20 +151,18 @@ def test_out2(self): # attr(axis) is Tensor class TestSplitOp_AxisTensor(OpTest): - def setUp(self): self._set_op_type() self.dtype = self.get_dtype() self.init_data() self.inputs = { 'X': self.x, - 'AxisTensor': np.array([self.axis]).astype("int32") + 'AxisTensor': np.array([self.axis]).astype("int32"), } self.attrs = {'sections': self.sections, 'num': self.num} out = np.split(self.x, self.indices_or_sections, self.axis) - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = {'Out': [('out%d' % i, out[i]) for i in range(len(out))]} self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -194,7 +185,6 @@ def test_check_output(self): class TestSplitOp_SectionsTensor(OpTest): - def setUp(self): self._set_op_type() self.dtype = self.get_dtype() @@ -203,20 +193,20 @@ def setUp(self): sections_tensor = [] for index, ele in enumerate(self.sections): - sections_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + sections_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SectionsTensorList'] = sections_tensor self.attrs = { 'axis': self.axis, 'sections': self.sections_infer, - 'num': self.num + 'num': self.num, } out = np.split(self.x, self.indices_or_sections, self.axis) - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = {'Out': [('out%d' % i, out[i]) for i in range(len(out))]} self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True diff --git a/python/paddle/fluid/tests/unittests/mlu/test_sqrt_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_sqrt_op_mlu.py index cc02f975abbea..211c8e349eb99 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_sqrt_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_sqrt_op_mlu.py @@ -29,7 +29,6 @@ class TestSqrt(OpTest): - def setUp(self): self.op_type = "sqrt" self.dtype = 'float32' @@ -55,7 +54,6 @@ def test_check_output(self): class TestSqrtHalf(OpTest): - def setUp(self): self.op_type = "sqrt" self.dtype = 'float16' @@ -74,10 +72,9 @@ def set_mlu(self): self.place = paddle.device.MLUPlace(0) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - check_eager=False, - max_relative_error=0.85) + self.check_grad_with_place( + self.place, ['X'], 'Out', check_eager=False, max_relative_error=0.85 + ) def test_check_output(self): self.check_output_with_place(self.place) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_squared_l2_norm_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_squared_l2_norm_op_mlu.py index a08b2f263af67..27642b92c3d5d 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_squared_l2_norm_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_squared_l2_norm_op_mlu.py @@ -26,8 +26,7 @@ class TestL2LossOp(OpTest): - """Test squared_l2_norm - """ + """Test squared_l2_norm""" def setUp(self): self.place = paddle.device.MLUPlace(0) @@ -44,13 +43,12 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=self.max_relative_error) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=self.max_relative_error + ) class TestL2LossDeterministic(unittest.TestCase): - def check_place(self, place): with paddle.fluid.dygraph.guard(place): x_np = np.random.rand(5, 11, 13).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/mlu/test_squeeze2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_squeeze2_op_mlu.py index bddd71ce2e161..62bd3d86b459b 100755 --- a/python/paddle/fluid/tests/unittests/mlu/test_squeeze2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_squeeze2_op_mlu.py @@ -25,7 +25,6 @@ # Correct: General. class TestSqueezeOp(OpTest): - def setUp(self): self.init_test_case() self.set_mlu() @@ -34,7 +33,7 @@ def setUp(self): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float32") + "XShape": np.random.random(self.ori_shape).astype("float32"), } def set_mlu(self): @@ -58,7 +57,6 @@ def init_attrs(self): # Correct: There is mins axis. class TestSqueezeOp1(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = (0, -2) @@ -67,7 +65,6 @@ def init_test_case(self): # Correct: No axes input. class TestSqueezeOp2(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -76,7 +73,6 @@ def init_test_case(self): # Correct: Just part of axes be squeezed. class TestSqueezeOp3(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_squeeze_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_squeeze_op_mlu.py index 24f02492c63cd..9df87453929de 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_squeeze_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_squeeze_op_mlu.py @@ -30,7 +30,6 @@ # Correct: General. class TestSqueezeOp(OpTest): - def setUp(self): self.op_type = "squeeze" self.init_test_case() @@ -61,7 +60,6 @@ def init_attrs(self): class TestSqueezeBF16Op(OpTest): - def setUp(self): self.op_type = "squeeze" self.dtype = np.uint16 @@ -89,7 +87,6 @@ def init_attrs(self): # Correct: There is mins axis. class TestSqueezeOp1(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 3, 1, 40) self.axes = (0, -2) @@ -98,7 +95,6 @@ def init_test_case(self): # Correct: No axes input. class TestSqueezeOp2(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -107,7 +103,6 @@ def init_test_case(self): # Correct: Just part of axes be squeezed. class TestSqueezeOp3(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) @@ -116,7 +111,6 @@ def init_test_case(self): # Correct: The demension of axis is not of size 1 remains unchanged. class TestSqueezeOp4(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, 2) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_stack_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_stack_op_mlu.py index 1d4550a11a5f3..eefe1d7d691fc 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_stack_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_stack_op_mlu.py @@ -25,7 +25,6 @@ class TestStackOpBase(OpTest): - def initDefaultParameters(self): self.num_inputs = 4 self.input_dim = (5, 6, 7) @@ -49,7 +48,8 @@ def setUp(self): self.x = [] for i in range(self.num_inputs): self.x.append( - np.random.random(size=self.input_dim).astype(self.dtype)) + np.random.random(size=self.input_dim).astype(self.dtype) + ) tmp = [] x_names = self.get_x_names() @@ -73,61 +73,51 @@ def test_check_output(self): class TestStackOp1(TestStackOpBase): - def initParameters(self): self.num_inputs = 16 class TestStackOp2(TestStackOpBase): - def initParameters(self): self.num_inputs = 20 class TestStackOp3(TestStackOpBase): - def initParameters(self): self.axis = -1 class TestStackOp4(TestStackOpBase): - def initParameters(self): self.axis = -4 class TestStackOp5(TestStackOpBase): - def initParameters(self): self.axis = 1 class TestStackOp6(TestStackOpBase): - def initParameters(self): self.axis = 3 class TestStackOpINT32(TestStackOpBase): - def init_dtype(self): self.dtype = np.int32 class TestStackOpINT64(TestStackOpBase): - def init_dtype(self): self.dtype = np.int64 class TestStackOpHalf(TestStackOpBase): - def init_dtype(self): self.dtype = np.float16 class API_test(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[1, 2], dtype='float32') @@ -139,12 +129,10 @@ def test_out(self): input1 = np.random.random([1, 2]).astype('float32') input2 = np.random.random([1, 2]).astype('float32') input3 = np.random.random([1, 2]).astype('float32') - result, = exe.run(feed={ - "data1": input1, - "data2": input2, - "data3": input3 - }, - fetch_list=[result_stack]) + (result,) = exe.run( + feed={"data1": input1, "data2": input2, "data3": input3}, + fetch_list=[result_stack], + ) expected_result = np.stack([input1, input2, input3], axis=0) np.testing.assert_allclose(expected_result, result) @@ -155,7 +143,6 @@ def test_single_tensor_error(self): class API_DygraphTest(unittest.TestCase): - def test_out(self): data1 = np.array([[1.0, 2.0]]).astype("float32") data2 = np.array([[3.0, 4.0]]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_strided_slice_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_strided_slice_op_mlu.py index 21388f1131f72..35069485054f5 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_strided_slice_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_strided_slice_op_mlu.py @@ -26,16 +26,15 @@ class TestStrideSliceOp(OpTest): - def setUp(self): self.initTestCase() self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True self.op_type = 'strided_slice' self.python_api = paddle.strided_slice - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.inputs = {'Input': self.input} self.outputs = {'Out': self.output} @@ -44,17 +43,16 @@ def setUp(self): 'starts': self.starts, 'ends': self.ends, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def test_check_output(self): self.check_output_with_place(self.place, check_eager=False) def test_check_grad(self): - self.check_grad_with_place(self.place, - set(['Input']), - 'Out', - check_eager=False) + self.check_grad_with_place( + self.place, set(['Input']), 'Out', check_eager=False + ) def initTestCase(self): self.input = np.random.rand(100).astype(np.float32) @@ -66,7 +64,6 @@ def initTestCase(self): class TestStrideSliceOp1(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(100).astype(np.float32) self.axes = [0] @@ -77,7 +74,6 @@ def initTestCase(self): class TestStrideSliceOp2(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(100).astype(np.float32) self.axes = [0] @@ -88,7 +84,6 @@ def initTestCase(self): class TestStrideSliceOp3(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(100).astype(np.float32) self.axes = [0] @@ -99,7 +94,6 @@ def initTestCase(self): class TestStrideSliceOp4(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 4, 10).astype(np.float32) self.axes = [0, 1, 2] @@ -110,7 +104,6 @@ def initTestCase(self): class TestStrideSliceOp5(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5).astype(np.float32) self.axes = [0, 1, 2] @@ -121,7 +114,6 @@ def initTestCase(self): class TestStrideSliceOp6(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5).astype(np.float32) self.axes = [0, 1, 2] @@ -132,7 +124,6 @@ def initTestCase(self): class TestStrideSliceOp7(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5).astype(np.float32) self.axes = [0, 1, 2] @@ -143,7 +134,6 @@ def initTestCase(self): class TestStrideSliceOp8(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(1, 100, 1).astype(np.float32) self.axes = [1] @@ -154,7 +144,6 @@ def initTestCase(self): class TestStrideSliceOp9(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(1, 100, 1).astype(np.float32) self.axes = [1] @@ -165,7 +154,6 @@ def initTestCase(self): class TestStrideSliceOp10(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(10, 10).astype(np.float32) self.axes = [0, 1] @@ -176,7 +164,6 @@ def initTestCase(self): class TestStrideSliceOp11(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4).astype(np.float32) self.axes = [0, 1, 2, 3] @@ -187,7 +174,6 @@ def initTestCase(self): class TestStrideSliceOp12(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4, 5).astype(np.float32) self.axes = [0, 1, 2, 3, 4] @@ -198,7 +184,6 @@ def initTestCase(self): class TestStrideSliceOp13(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 6, 7, 8).astype(np.float32) self.axes = [0, 1, 2, 3, 4, 5] @@ -209,7 +194,6 @@ def initTestCase(self): class TestStrideSliceOp14(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(4, 4, 4, 4).astype(np.float32) self.axes = [1, 2, 3] @@ -220,13 +204,11 @@ def initTestCase(self): class TestStrideSliceOpBool(TestStrideSliceOp): - def test_check_grad(self): pass class TestStrideSliceOpBool1D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(100).astype("bool") self.axes = [0] @@ -237,7 +219,6 @@ def initTestCase(self): class TestStrideSliceOpBool2D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(10, 10).astype("bool") self.axes = [0, 1] @@ -248,7 +229,6 @@ def initTestCase(self): class TestStrideSliceOpBool3D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 4, 10).astype("bool") self.axes = [0, 1, 2] @@ -259,7 +239,6 @@ def initTestCase(self): class TestStrideSliceOpBool4D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4).astype("bool") self.axes = [0, 1, 2, 3] @@ -270,7 +249,6 @@ def initTestCase(self): class TestStrideSliceOpBool5D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4, 5).astype("bool") self.axes = [0, 1, 2, 3, 4] @@ -281,7 +259,6 @@ def initTestCase(self): class TestStrideSliceOpBool6D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 6, 7, 8).astype("bool") self.axes = [0, 1, 2, 3, 4, 5] @@ -292,7 +269,6 @@ def initTestCase(self): class TestStridedSliceOp_starts_ListTensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.place = paddle.device.MLUPlace(0) @@ -301,8 +277,9 @@ def setUp(self): starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} self.outputs = {'Out': self.output} @@ -311,7 +288,7 @@ def setUp(self): 'starts': self.starts_infer, 'ends': self.ends, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -321,9 +298,9 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [1, -1, 1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.starts_infer = [1, 10, 2] @@ -331,13 +308,12 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) class TestStridedSliceOp_ends_ListTensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.place = paddle.device.MLUPlace(0) @@ -346,8 +322,9 @@ def setUp(self): ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'EndsTensorList': ends_tensor} self.outputs = {'Out': self.output} @@ -356,7 +333,7 @@ def setUp(self): 'starts': self.starts, 'ends': self.ends_infer, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -366,9 +343,9 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, 1, 2] self.infer_flags = [1, -1, 1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.ends_infer = [3, 1, 4] @@ -376,13 +353,12 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) class TestStridedSliceOp_starts_Tensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.place = paddle.device.MLUPlace(0) @@ -390,7 +366,7 @@ def setUp(self): self.config() self.inputs = { 'Input': self.input, - "StartsTensor": np.array(self.starts, dtype="int32") + "StartsTensor": np.array(self.starts, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -408,21 +384,20 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) class TestStridedSliceOp_ends_Tensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.place = paddle.device.MLUPlace(0) @@ -430,7 +405,7 @@ def setUp(self): self.config() self.inputs = { 'Input': self.input, - "EndsTensor": np.array(self.ends, dtype="int32") + "EndsTensor": np.array(self.ends, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -448,35 +423,35 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) class TestStridedSliceOp_listTensor_Tensor(OpTest): - def setUp(self): self.config() self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.op_type = "strided_slice" self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int32"), - "EndsTensorList": ends_tensor + "EndsTensorList": ends_tensor, } self.outputs = {'Out': self.output} self.attrs = { @@ -494,21 +469,20 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) class TestStridedSliceOp_strides_Tensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.place = paddle.device.MLUPlace(0) @@ -516,7 +490,7 @@ def setUp(self): self.config() self.inputs = { 'Input': self.input, - "StridesTensor": np.array(self.strides, dtype="int32") + "StridesTensor": np.array(self.strides, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -534,63 +508,65 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, -1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.006 + ) # Test python API class TestStridedSliceAPI(unittest.TestCase): - def test_1(self): input = np.random.random([3, 4, 5, 6]).astype("float32") minus_1 = fluid.layers.fill_constant([1], "int32", -1) minus_3 = fluid.layers.fill_constant([1], "int32", -3) - starts = fluid.layers.data(name='starts', - shape=[3], - dtype='int32', - append_batch_size=False) - ends = fluid.layers.data(name='ends', - shape=[3], - dtype='int32', - append_batch_size=False) - strides = fluid.layers.data(name='strides', - shape=[3], - dtype='int32', - append_batch_size=False) - - x = fluid.layers.data(name="x", - shape=[3, 4, 5, 6], - append_batch_size=False, - dtype="float32") - out_1 = paddle.strided_slice(x, - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[3, 100, -1], - strides=[1, 1, 1]) - out_2 = paddle.strided_slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, -1], - strides=[1, 1, 1]) - out_3 = paddle.strided_slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, minus_1], - strides=[1, 1, 1]) - out_4 = paddle.strided_slice(x, - axes=[0, 1, 2], - starts=starts, - ends=ends, - strides=strides) + starts = fluid.layers.data( + name='starts', shape=[3], dtype='int32', append_batch_size=False + ) + ends = fluid.layers.data( + name='ends', shape=[3], dtype='int32', append_batch_size=False + ) + strides = fluid.layers.data( + name='strides', shape=[3], dtype='int32', append_batch_size=False + ) + + x = fluid.layers.data( + name="x", + shape=[3, 4, 5, 6], + append_batch_size=False, + dtype="float32", + ) + out_1 = paddle.strided_slice( + x, + axes=[0, 1, 2], + starts=[-3, 0, 2], + ends=[3, 100, -1], + strides=[1, 1, 1], + ) + out_2 = paddle.strided_slice( + x, + axes=[0, 1, 3], + starts=[minus_3, 0, 2], + ends=[3, 100, -1], + strides=[1, 1, 1], + ) + out_3 = paddle.strided_slice( + x, + axes=[0, 1, 3], + starts=[minus_3, 0, 2], + ends=[3, 100, minus_1], + strides=[1, 1, 1], + ) + out_4 = paddle.strided_slice( + x, axes=[0, 1, 2], starts=starts, ends=ends, strides=strides + ) out_5 = x[-3:3, 0:100:2, -1:2:-1] out_6 = x[minus_3:3:1, 0:100:2, :, minus_1:2:minus_1] @@ -603,9 +579,10 @@ def test_1(self): "x": input, 'starts': np.array([-3, 0, 2]).astype("int32"), 'ends': np.array([3, 2147483648, -1]).astype("int64"), - 'strides': np.array([1, 1, 1]).astype("int32") + 'strides': np.array([1, 1, 1]).astype("int32"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7], + ) assert np.array_equal(res_1, input[-3:3, 0:100, 2:-1, :]) assert np.array_equal(res_2, input[-3:3, 0:100, :, 2:-1]) assert np.array_equal(res_3, input[-3:3, 0:100, :, 2:-1]) @@ -620,11 +597,9 @@ def test_dygraph_op(self): starts = [-3, 0, 2] ends = [3, 2, 4] strides_1 = [1, 1, 1] - sliced_1 = paddle.strided_slice(x, - axes=axes, - starts=starts, - ends=ends, - strides=strides_1) + sliced_1 = paddle.strided_slice( + x, axes=axes, starts=starts, ends=ends, strides=strides_1 + ) assert sliced_1.shape == (3, 2, 2, 2) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_sum_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_sum_op_mlu.py index ebf00a9db08f4..0dae84dc79b71 100755 --- a/python/paddle/fluid/tests/unittests/mlu/test_sum_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_sum_op_mlu.py @@ -27,7 +27,6 @@ class TestSum1(OpTest): - def setUp(self): self.set_mlu() self.init_dtype() @@ -54,7 +53,6 @@ def test_check_output(self): class TestSum2(OpTest): - def setUp(self): self.set_mlu() self.init_dtype() @@ -72,8 +70,12 @@ def setUp(self): # For example, the results of `x0+x1+x2+x3` is different from that of # `x3+x2+x1+x0` if the dtype is fp16. # Therefore, converting the input to fp32 for calculation. - y = (x0.astype(np.float32) + x1.astype(np.float32) + - x2.astype(np.float32) + x3.astype(np.float32)).astype(self.dtype) + y = ( + x0.astype(np.float32) + + x1.astype(np.float32) + + x2.astype(np.float32) + + x3.astype(np.float32) + ).astype(self.dtype) self.outputs = {'Out': y} self.attrs = {'use_mkldnn': False} @@ -89,7 +91,6 @@ def test_check_output(self): class TestSum3(OpTest): - def setUp(self): self.set_mlu() self.init_dtype() diff --git a/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_base_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_base_mlu.py index 0d73e5af5c5b8..dd842800198cb 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_base_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_base_mlu.py @@ -41,17 +41,19 @@ class TestSyncBatchNormRunnerBase(object): - - def get_model(self, - main, - startup, - place, - layout, - seed, - sync_bn=False, - only_forward=False): + def get_model( + self, + main, + startup, + place, + layout, + seed, + sync_bn=False, + only_forward=False, + ): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def wait_server_ready(self, endpoints): assert not isinstance(endpoints, str) @@ -60,13 +62,15 @@ def wait_server_ready(self, endpoints): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, - 1) + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT, 1 + ) result = sock.connect_ex((ip_port[0], int(ip_port[1]))) if result != 0: @@ -74,39 +78,47 @@ def wait_server_ready(self, endpoints): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + - str(not_ready_endpoints) + "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: break - def initCommunicator(self, program, rank, nranks, wait_port, - current_endpoint, endpoints): + def initCommunicator( + self, program, rank, nranks, wait_port, current_endpoint, endpoints + ): other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) if rank == 0 and wait_port: self.wait_server_ready(other_endpoints) block = program.global_block() - cncl_id_var = block.create_var(name=nameGen.generate('cncl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_cncl_id', - inputs={}, - outputs={'Out': cncl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) - block.append_op(type='c_comm_init', - inputs={'X': cncl_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': self.global_ring_id - }) + cncl_id_var = block.create_var( + name=nameGen.generate('cncl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_cncl_id', + inputs={}, + outputs={'Out': cncl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) + block.append_op( + type='c_comm_init', + inputs={'X': cncl_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': self.global_ring_id, + }, + ) def run_trainer(self, args): device_id = int(os.getenv("FLAGS_selected_mlus", "0")) @@ -139,24 +151,30 @@ def run_trainer(self, args): sys.stdout.buffer.write( pickle.dumps( - 'training, inference, fp32, fp16, NCHW, NHWC all passed')) + 'training, inference, fp32, fp16, NCHW, NHWC all passed' + ) + ) def _compare(self, args, place, layout, only_forward): scope = core.Scope() np.random.seed(SEED) - data = np.random.random(size=self.dshape).astype(self.dtype) * 4. - 2 + data = np.random.random(size=self.dshape).astype(self.dtype) * 4.0 - 2 sys.stderr.write("data: " + str(data) + "\n") - data = create_or_get_tensor(scope, "input", - OpTest.np_dtype_to_fluid_dtype(data), place) + data = create_or_get_tensor( + scope, "input", OpTest.np_dtype_to_fluid_dtype(data), place + ) - bn_fetches = self._cal_single_card(args, data, place, layout, - only_forward) + bn_fetches = self._cal_single_card( + args, data, place, layout, only_forward + ) fetch_names, sync_bn_fetches = self._cal_multiple_cards( - args, data, place, layout, only_forward) + args, data, place, layout, only_forward + ) - sys.stderr.write("len(sync_bn_fetches): " + str(len(sync_bn_fetches)) + - "\n") + sys.stderr.write( + "len(sync_bn_fetches): " + str(len(sync_bn_fetches)) + "\n" + ) for i in range(0, len(sync_bn_fetches)): sys.stderr.write("i: " + str(i) + "\n") sys.stderr.write("fetch_names[i]): " + fetch_names[i] + "\n") @@ -164,13 +182,14 @@ def _compare(self, args, place, layout, only_forward): bn_val = bn_fetches[i] sync_bn_val = sync_bn_fetches[i] if sync_bn_val.shape != bn_val.shape: - sync_bn_val = sync_bn_val[:bn_val.shape[0]] + sync_bn_val = sync_bn_val[: bn_val.shape[0]] # i = 0 if fetch_names[i] == 'reduce_sum_0.tmp_0': # sys.stderr.write("skip reduce_sum_0.tmp_0 (Out of reduce_sum op)" + "\n") - sys.stderr.write("reduce_sum_0.tmp_0 (Out of reduce_sum op)" + - "\n") + sys.stderr.write( + "reduce_sum_0.tmp_0 (Out of reduce_sum op)" + "\n" + ) sys.stderr.write("bn_val: " + str(bn_val) + "\n") sys.stderr.write("sync_bn_val: " + str(sync_bn_val) + "\n") @@ -198,7 +217,8 @@ def _compare(self, args, place, layout, only_forward): if fetch_names[i] == 'batch_norm_0.tmp_2': # sys.stderr.write("skip batch_norm_0.tmp_2 (ReserveSpace of batch_norm)" + "\n") sys.stderr.write( - "batch_norm_0.tmp_2 (ReserveSpace of batch_norm)" + "\n") + "batch_norm_0.tmp_2 (ReserveSpace of batch_norm)" + "\n" + ) sys.stderr.write("bn_val: " + str(bn_val) + "\n") sys.stderr.write("sync_bn_val: " + str(sync_bn_val) + "\n") @@ -231,8 +251,9 @@ def _compare(self, args, place, layout, only_forward): # i = 8 if fetch_names[i] == 'batch_norm_0.tmp_1': - sys.stderr.write("skip batch_norm_0.tmp_1 (SavedVariance)" + - "\n") + sys.stderr.write( + "skip batch_norm_0.tmp_1 (SavedVariance)" + "\n" + ) sys.stderr.write("bn_val: " + str(bn_val) + "\n") sys.stderr.write("sync_bn_val: " + str(sync_bn_val) + "\n") @@ -278,10 +299,16 @@ def _compare(self, args, place, layout, only_forward): if fetch_names[i] == 'conv2d_0.tmp_0@GRAD': atol = 1e-2 - assert np.allclose( - bn_val, sync_bn_val, atol=atol), "Output (" + fetch_names[ - i] + ") has diff. \n" + "\nBN " + str( - bn_val) + "\n" + "Sync BN " + str(sync_bn_val) + assert np.allclose(bn_val, sync_bn_val, atol=atol), ( + "Output (" + + fetch_names[i] + + ") has diff. \n" + + "\nBN " + + str(bn_val) + + "\n" + + "Sync BN " + + str(sync_bn_val) + ) def _cal_single_card(self, args, data, place, layout, only_forward): # Single-MLU, N = 32 per MLU @@ -291,23 +318,31 @@ def _cal_single_card(self, args, data, place, layout, only_forward): startup_prog.global_seed(SEED) paddle.seed(SEED) - outs = self.get_model(train_prog, startup_prog, place, layout, SEED, - False, only_forward) + outs = self.get_model( + train_prog, startup_prog, place, layout, SEED, False, only_forward + ) exe = fluid.Executor(place) exe.run(startup_prog) fetch_names = [v.name for v in outs] + [ - 'bn_moving_mean', 'bn_moving_variance', 'bn_scale', 'bn_bias' + 'bn_moving_mean', + 'bn_moving_variance', + 'bn_scale', + 'bn_bias', ] if not only_forward: others = [ - 'batch_norm_0.tmp_0', 'batch_norm_0.tmp_1', 'bn_scale@GRAD', - 'bn_bias@GRAD', 'batch_norm_0.tmp_3@GRAD', 'conv2d_0.tmp_0@GRAD' + 'batch_norm_0.tmp_0', + 'batch_norm_0.tmp_1', + 'bn_scale@GRAD', + 'bn_bias@GRAD', + 'batch_norm_0.tmp_3@GRAD', + 'conv2d_0.tmp_0@GRAD', ] fetch_names += others - bn_fetches = exe.run(program=train_prog, - feed={'input': data}, - fetch_list=fetch_names) + bn_fetches = exe.run( + program=train_prog, feed={'input': data}, fetch_list=fetch_names + ) return bn_fetches @@ -328,8 +363,9 @@ def _cal_multiple_cards(self, args, data, place, layout, only_forward): current_endpoint = args["currentendpoint"] nranks = 2 - self.initCommunicator(startup_prog, rank, nranks, True, - current_endpoint, endpoints) + self.initCommunicator( + startup_prog, rank, nranks, True, current_endpoint, endpoints + ) # sys.stderr.write("after init, startup_prog: " + # startup_prog.to_string(True) + "\n") train_prog.global_seed(SEED) @@ -339,8 +375,9 @@ def _cal_multiple_cards(self, args, data, place, layout, only_forward): paddle.seed(SEED) self.rank = rank - outs = self.get_model(train_prog, startup_prog, place, layout, SEED, - True, only_forward) + outs = self.get_model( + train_prog, startup_prog, place, layout, SEED, True, only_forward + ) # sys.stderr.write("after get_model, train_prog: " + # train_prog.to_string(True) + "\n") # sys.stderr.write("after get_model, startup_prog: " + @@ -363,17 +400,24 @@ def _cal_multiple_cards(self, args, data, place, layout, only_forward): exe = fluid.Executor(place) exe.run(startup_prog) fetch_names = [v.name for v in outs] + [ - 'bn_moving_mean', 'bn_moving_variance', 'bn_scale', 'bn_bias' + 'bn_moving_mean', + 'bn_moving_variance', + 'bn_scale', + 'bn_bias', ] if not only_forward: others = [ - 'batch_norm_0.tmp_0', 'batch_norm_0.tmp_1', 'bn_scale@GRAD', - 'bn_bias@GRAD', 'batch_norm_0.tmp_3@GRAD', 'conv2d_0.tmp_0@GRAD' + 'batch_norm_0.tmp_0', + 'batch_norm_0.tmp_1', + 'bn_scale@GRAD', + 'bn_bias@GRAD', + 'batch_norm_0.tmp_3@GRAD', + 'conv2d_0.tmp_0@GRAD', ] fetch_names += others - sync_bn_fetches = exe.run(program=train_prog, - feed={'input': data}, - fetch_list=fetch_names) + sync_bn_fetches = exe.run( + program=train_prog, feed={'input': data}, fetch_list=fetch_names + ) return fetch_names, sync_bn_fetches @@ -396,19 +440,20 @@ def runtime_main(test_class, col_type, sub_type): class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -437,7 +482,7 @@ def _run_cluster(self, model_file, envs): "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) @@ -448,15 +493,19 @@ def _run_cluster(self, model_file, envs): tr1_pipe = open("/tmp/tr1_err_%d.log" % os.getpid(), "w") print("tr0_cmd: {}, env: {}\n".format(tr0_cmd, env0)) print("tr1_cmd: {}, env: {}\n".format(tr1_cmd, env1)) - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) - - tr1_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) + + tr1_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -470,14 +519,16 @@ def _run_cluster(self, model_file, envs): sys.stderr.write('trainer 0 stderr file: %s\n' % f.read()) with open("/tmp/tr1_err_%d.log" % os.getpid(), "r") as f: sys.stderr.write('trainer 1 stderr file: %s\n' % f.read()) - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid - - def check_with_place(self, - model_file, - col_type, - check_error_log=False, - need_envs={}): + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) + + def check_with_place( + self, model_file, col_type, check_error_log=False, need_envs={} + ): required_envs = { "FLAGS_fraction_of_gpu_memory_to_use": "0.15", "FLAGS_eager_delete_tensor_gb": "0.0", @@ -488,7 +539,7 @@ def check_with_place(self, "FLAGS_call_stack_level": "2", "GLOG_v": "3", "PADDLE_WITH_GLOO": '0', - "BACKEND": "cncl" + "BACKEND": "cncl", } required_envs.update(need_envs) if check_error_log: @@ -496,8 +547,11 @@ def check_with_place(self, required_envs["GLOG_logtostderr"] = "1" required_envs["GLOO_LOG_LEVEL"] = "TRACE" tr0_out, tr1_out, pid0, pid1 = self._run_cluster( - model_file, required_envs) + model_file, required_envs + ) self.assertEqual( - tr0_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed') + tr0_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed' + ) self.assertEqual( - tr1_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed') + tr1_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed' + ) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_baseline.py b/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_baseline.py index 44dc8cf235984..597c033ba78fb 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_baseline.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_baseline.py @@ -28,14 +28,13 @@ class TestSyncBatchNormOp(TestDistBase): - def _setup_config(self): pass def test_identity(self, col_type="identity"): - self.check_with_place("sync_batch_norm_op_mlu.py", - col_type, - check_error_log=True) + self.check_with_place( + "sync_batch_norm_op_mlu.py", col_type, check_error_log=True + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_extra.py b/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_extra.py index 6488f757c50f8..fd19512528b6b 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_extra.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_sync_batch_norm_op_mlu_extra.py @@ -34,15 +34,15 @@ class TestDygraphSyncBatchNormAPIError(unittest.TestCase): - def test_errors(self): if not core.is_compiled_with_mlu(): return with program_guard(Program(), Program()): my_sync_batch_norm = paddle.nn.SyncBatchNorm(10) - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.MLUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.MLUPlace(0) + ) self.assertRaises(TypeError, my_sync_batch_norm, x1) # the input dtype of SyncBatchNorm must be float16 or float32 @@ -51,36 +51,39 @@ def test_errors(self): class TestConvertSyncBatchNorm(unittest.TestCase): - def test_convert(self): if not core.is_compiled_with_mlu(): return with program_guard(Program(), Program()): - compare_model = paddle.nn.Sequential(paddle.nn.Conv2D(3, 5, 3), - paddle.nn.BatchNorm2D(5), - paddle.nn.BatchNorm2D(5)) + compare_model = paddle.nn.Sequential( + paddle.nn.Conv2D(3, 5, 3), + paddle.nn.BatchNorm2D(5), + paddle.nn.BatchNorm2D(5), + ) model = paddle.nn.Sequential( - paddle.nn.Conv2D(3, 5, 3), paddle.nn.BatchNorm2D(5), + paddle.nn.Conv2D(3, 5, 3), + paddle.nn.BatchNorm2D(5), paddle.nn.BatchNorm2D( 5, weight_attr=fluid.ParamAttr(name='bn.scale'), - bias_attr=fluid.ParamAttr(name='bn.bias'))) + bias_attr=fluid.ParamAttr(name='bn.bias'), + ), + ) model = paddle.nn.SyncBatchNorm.convert_sync_batchnorm(model) for idx, sublayer in enumerate(compare_model.sublayers()): if isinstance(sublayer, paddle.nn.BatchNorm2D): self.assertEqual( - isinstance(model[idx], paddle.nn.SyncBatchNorm), True) + isinstance(model[idx], paddle.nn.SyncBatchNorm), True + ) class TestConvertSyncBatchNormCast1(unittest.TestCase): - def test_convert(self): if not core.is_compiled_with_mlu(): return class Net(nn.Layer): - def __init__(self): super(Net, self).__init__() self.conv1 = nn.Conv2D(3, 5, 3) @@ -105,7 +108,6 @@ def forward(self, x): class TestConvertSyncBatchNormCase2(unittest.TestCase): - def test_convert(self): if not core.is_compiled_with_mlu(): return @@ -113,16 +115,19 @@ def test_convert(self): with fluid.dygraph.guard(fluid.MLUPlace(0)): class SyBNNet(paddle.nn.Layer): - def __init__(self, in_ch=3, out_ch=3, dirate=1): super(SyBNNet, self).__init__() self.bn_s1 = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( paddle.nn.BatchNorm3D( out_ch, weight_attr=paddle.ParamAttr( - regularizer=paddle.regularizer.L2Decay(0.)))) + regularizer=paddle.regularizer.L2Decay(0.0) + ), + ) + ) self.bn_s2 = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( - paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC')) + paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC') + ) def forward(self, x): x = self.bn_s1(x) @@ -130,15 +135,17 @@ def forward(self, x): return out class BNNet(paddle.nn.Layer): - def __init__(self, in_ch=3, out_ch=3, dirate=1): super(BNNet, self).__init__() self.bn_s1 = paddle.nn.BatchNorm3D( out_ch, weight_attr=paddle.ParamAttr( - regularizer=paddle.regularizer.L2Decay(0.))) + regularizer=paddle.regularizer.L2Decay(0.0) + ), + ) self.bn_s2 = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( - paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC')) + paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC') + ) def forward(self, x): x = self.bn_s1(x) @@ -156,12 +163,16 @@ def forward(self, x): bn_out.numpy(), sybn_out.numpy(), rtol=1e-05, - err_msg='Output has diff. \n' + '\nBN ' + - str(bn_out.numpy()) + '\n' + 'Sync BN ' + str(sybn_out.numpy())) + err_msg='Output has diff. \n' + + '\nBN ' + + str(bn_out.numpy()) + + '\n' + + 'Sync BN ' + + str(sybn_out.numpy()), + ) class TestDygraphSyncBatchNormDataFormatError(unittest.TestCase): - def test_errors(self): if not core.is_compiled_with_mlu(): return diff --git a/python/paddle/fluid/tests/unittests/mlu/test_tanh_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_tanh_op_mlu.py index a50ad86523afd..c346dd0867a5b 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_tanh_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_tanh_op_mlu.py @@ -26,7 +26,6 @@ class TestTanh(OpTest): - def setUp(self): self.set_mlu() self.op_type = "tanh" @@ -58,7 +57,6 @@ def test_check_grad(self): class TestTanhFp16(OpTest): - def setUp(self): self.set_mlu() self.op_type = "tanh" @@ -85,7 +83,6 @@ def test_check_output(self): class TestTanhNet(unittest.TestCase): - def _test(self, run_mlu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -100,9 +97,9 @@ def _test(self, run_mlu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) d = paddle.tanh(c) @@ -126,16 +123,17 @@ def _test(self, run_mlu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/mlu/test_tile_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_tile_op_mlu.py index c42695f2b5d31..2f1a0d990bf01 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_tile_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_tile_op_mlu.py @@ -25,9 +25,8 @@ paddle.enable_static() -#Situation 1: repeat_times is a list (without tensor) +# Situation 1: repeat_times is a list (without tensor) class TestTileOpRank1(OpTest): - def setUp(self): self.op_type = "tile" self.place = paddle.device.MLUPlace(0) @@ -51,42 +50,36 @@ def test_check_grad(self): # with dimension expanding class TestTileOpRank2Expanding(TestTileOpRank1): - def init_data(self): self.ori_shape = [120] self.repeat_times = [2, 2] class TestTileOpRank2(TestTileOpRank1): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] class TestTileOpRank3_Corner(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (1, 1, 1) class TestTileOpRank3_Corner2(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (2, 2) class TestTileOpRank3(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 15) self.repeat_times = (2, 1, 4) class TestTileOpRank4(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.repeat_times = (3, 2, 1, 2) @@ -94,7 +87,6 @@ def init_data(self): # Situation 2: repeat_times is a list (with tensor) class TestTileOpRank1_tensor_attr(OpTest): - def setUp(self): self.op_type = "tile" self.place = paddle.device.MLUPlace(0) @@ -102,8 +94,9 @@ def setUp(self): self.init_data() repeat_times_tensor = [] for index, ele in enumerate(self.repeat_times): - repeat_times_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + repeat_times_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype("float32"), @@ -126,7 +119,6 @@ def test_check_grad(self): class TestTileOpRank2_Corner_tensor_attr(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [1, 1] @@ -134,7 +126,6 @@ def init_data(self): class TestTileOpRank2_attr_tensor(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -143,7 +134,6 @@ def init_data(self): # Situation 3: repeat_times is a tensor class TestTileOpRank1_tensor(OpTest): - def setUp(self): self.op_type = "tile" self.place = paddle.device.MLUPlace(0) @@ -169,7 +159,6 @@ def test_check_grad(self): class TestTileOpRank2_tensor(TestTileOpRank1_tensor): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -177,7 +166,6 @@ def init_data(self): # Situation 4: input x is Integer class TestTileOpInteger(OpTest): - def setUp(self): self.op_type = "tile" self.place = paddle.device.MLUPlace(0) @@ -195,7 +183,6 @@ def test_check_output(self): # Situation 5: input x is Bool class TestTileOpBoolean(OpTest): - def setUp(self): self.op_type = "tile" self.place = paddle.device.MLUPlace(0) @@ -211,7 +198,6 @@ def test_check_output(self): # Situation 56: input x is Integer class TestTileOpInt64_t(OpTest): - def setUp(self): self.op_type = "tile" self.place = paddle.device.MLUPlace(0) @@ -228,11 +214,11 @@ def test_check_output(self): class TestTileError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) repeat_times = [2, 2] self.assertRaises(TypeError, paddle.tile, x1, repeat_times) x2 = fluid.layers.data(name='x2', shape=[4], dtype="uint8") @@ -243,7 +229,6 @@ def test_errors(self): class TestTileAPIStatic(unittest.TestCase): - def test_api(self): with program_guard(Program(), Program()): repeat_times = [2, 2] @@ -255,7 +240,6 @@ def test_api(self): # Test python API class TestTileAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): np_x = np.random.random([12, 14]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/mlu/test_top_k_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_top_k_op_mlu.py index 8d7a5eb2c60ad..8e4a9d483892e 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_top_k_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_top_k_op_mlu.py @@ -25,7 +25,6 @@ class TestTopkOp(OpTest): - def setUp(self): self.variable_k = False self.set_args() @@ -66,7 +65,6 @@ def test_check_output(self): class TestTopkFP16Op(TestTopkOp): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_top_k_v2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_top_k_v2_op_mlu.py index 8516fa224fa6a..4c9d426e82d90 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_top_k_v2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_top_k_v2_op_mlu.py @@ -39,7 +39,6 @@ def numpy_topk(x, k=1, axis=-1, largest=True): class TestTopkOp(OpTest): - def init_args(self): self.k = 3 self.axis = 1 @@ -55,10 +54,9 @@ def setUp(self): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} def test_check_output(self): @@ -67,7 +65,6 @@ def test_check_output(self): class TestTopkOp1(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 0 @@ -75,7 +72,6 @@ def init_args(self): class TestTopkOp2(TestTopkOp): - def init_args(self): self.k = 4 self.axis = 0 @@ -83,7 +79,6 @@ def init_args(self): class TestTopkOp3(OpTest): - def init_args(self): self.k = 6 self.axis = 1 @@ -96,15 +91,13 @@ def setUp(self): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopkOp4(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -119,15 +112,13 @@ def setUp(self): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopkOp5(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -142,15 +133,13 @@ def setUp(self): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopkOp6(OpTest): - def init_args(self): self.k = 100 self.axis = 1 @@ -165,15 +154,13 @@ def setUp(self): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopKAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.dtype = np.float32 @@ -219,22 +206,23 @@ def run_dygraph(self, place): np.testing.assert_allclose(paddle_result[1].numpy(), numpy_result[1]) # test case for basic test case 7 for the unsorted paddle_result = paddle.topk(input_tensor, k=2, axis=1, sorted=False) - sort_paddle = numpy_topk(np.array(paddle_result[0].numpy()), - axis=1, - k=2) + sort_paddle = numpy_topk( + np.array(paddle_result[0].numpy()), axis=1, k=2 + ) numpy_result = numpy_topk(self.input_data, k=2, axis=1) np.testing.assert_allclose(sort_paddle[0], numpy_result[0]) def run_static(self, place): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - input_tensor = paddle.static.data(name="x", - shape=[6, 7, 8], - dtype="float32") - large_input_tensor = paddle.static.data(name="large_x", - shape=[2, 1030], - dtype="float32") + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + input_tensor = paddle.static.data( + name="x", shape=[6, 7, 8], dtype="float32" + ) + large_input_tensor = paddle.static.data( + name="large_x", shape=[2, 1030], dtype="float32" + ) k_tensor = paddle.static.data(name="k", shape=[1], dtype="int32") result1 = paddle.topk(input_tensor, k=2) result2 = paddle.topk(input_tensor, k=2, axis=-1) @@ -248,18 +236,29 @@ def run_static(self, place): exe = paddle.static.Executor(place) input_data = np.random.rand(10, 20).astype("float32") large_input_data = np.random.rand(2, 100).astype("float32") - paddle_result = exe.run(feed={ - "x": self.input_data, - "large_x": self.large_input_data, - "k": np.array([2]).astype("int32") - }, - fetch_list=[ - result1[0], result1[1], result2[0], - result2[1], result3[0], result3[1], - result4[0], result4[1], result5[0], - result5[1], result6[0], result6[1], - result7[0], result7[1] - ]) + paddle_result = exe.run( + feed={ + "x": self.input_data, + "large_x": self.large_input_data, + "k": np.array([2]).astype("int32"), + }, + fetch_list=[ + result1[0], + result1[1], + result2[0], + result2[1], + result3[0], + result3[1], + result4[0], + result4[1], + result5[0], + result5[1], + result6[0], + result6[1], + result7[0], + result7[1], + ], + ) numpy_result = numpy_topk(self.input_data, k=2) np.testing.assert_allclose(paddle_result[0], numpy_result[0]) np.testing.assert_allclose(paddle_result[1], numpy_result[1]) @@ -269,16 +268,14 @@ def run_static(self, place): numpy_result = numpy_topk(self.input_data, k=2, axis=1) np.testing.assert_allclose(paddle_result[4], numpy_result[0]) np.testing.assert_allclose(paddle_result[5], numpy_result[1]) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=1, - largest=False) + numpy_result = numpy_topk( + self.input_data, k=2, axis=1, largest=False + ) np.testing.assert_allclose(paddle_result[6], numpy_result[0]) np.testing.assert_allclose(paddle_result[7], numpy_result[1]) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=-1, - largest=False) + numpy_result = numpy_topk( + self.input_data, k=2, axis=-1, largest=False + ) np.testing.assert_allclose(paddle_result[8], numpy_result[0]) np.testing.assert_allclose(paddle_result[9], numpy_result[1]) numpy_result = numpy_topk(self.large_input_data, k=1, axis=-1) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_transpose_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_transpose_op_mlu.py index 6aca81d0ba933..30e4519887956 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_transpose_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_transpose_op_mlu.py @@ -27,7 +27,6 @@ class TestTransposeOp(OpTest): - def setUp(self): self.init_op_type() self.initKernelType() @@ -57,92 +56,79 @@ def initKernelType(self): class TestCase0(TestTransposeOp): - def initTestCase(self): - self.shape = (100, ) - self.axis = (0, ) + self.shape = (100,) + self.axis = (0,) class TestCase1(TestTransposeOp): - def initTestCase(self): self.shape = (3, 4, 10) self.axis = (0, 2, 1) class TestCase2(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.axis = (0, 2, 3, 1) class TestCase3(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.axis = (4, 2, 3, 1, 0) class TestCase4(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6, 1) self.axis = (4, 2, 3, 1, 0, 5) class TestCase5(TestTransposeOp): - def initTestCase(self): self.shape = (2, 16, 96) self.axis = (0, 2, 1) class TestCase6(TestTransposeOp): - def initTestCase(self): self.shape = (2, 10, 12, 16) self.axis = (3, 1, 2, 0) class TestCase7(TestTransposeOp): - def initTestCase(self): self.shape = (2, 10, 2, 16) self.axis = (0, 1, 3, 2) class TestCase8(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (0, 1, 3, 2, 4, 5, 6, 7) class TestCase9(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (6, 1, 3, 5, 0, 2, 4, 7) class TestTransposeOpBool(TestTransposeOp): - def test_check_grad(self): pass class TestTransposeOpBool1D(TestTransposeOpBool): - def initTestCase(self): - self.shape = (100, ) - self.axis = (0, ) + self.shape = (100,) + self.axis = (0,) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = {'Out': self.inputs['X'].transpose(self.axis)} class TestTransposeOpBool2D(TestTransposeOpBool): - def initTestCase(self): self.shape = (3, 40) self.axis = (1, 0) @@ -151,7 +137,6 @@ def initTestCase(self): class TestTransposeOpBool3D(TestTransposeOpBool): - def initTestCase(self): self.shape = (3, 4, 10) self.axis = (0, 2, 1) @@ -160,7 +145,6 @@ def initTestCase(self): class TestTransposeOpBool4D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.axis = (0, 2, 3, 1) @@ -169,7 +153,6 @@ def initTestCase(self): class TestTransposeOpBool5D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.axis = (4, 2, 3, 1, 0) @@ -178,7 +161,6 @@ def initTestCase(self): class TestTransposeOpBool6D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6, 1) self.axis = (4, 2, 3, 1, 0, 5) @@ -187,7 +169,6 @@ def initTestCase(self): class TestTransposeOpBool7D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3) self.axis = (0, 1, 3, 2, 4, 5, 6) @@ -196,7 +177,6 @@ def initTestCase(self): class TestTransposeOpBool8D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (6, 1, 3, 5, 0, 2, 4, 7) @@ -205,7 +185,6 @@ def initTestCase(self): class TestTransposeOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -238,7 +217,6 @@ def test_each_elem_value_check(): class TestTransposeApi(unittest.TestCase): - def test_static_out(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): @@ -248,8 +226,9 @@ def test_static_out(self): place = paddle.MLUPlace(0) exe = paddle.static.Executor(place) x_np = np.random.random([2, 3, 4]).astype("float32") - result1, result2 = exe.run(feed={"x": x_np}, - fetch_list=[x_trans1, x_trans2]) + result1, result2 = exe.run( + feed={"x": x_np}, fetch_list=[x_trans1, x_trans2] + ) expected_result1 = np.transpose(x_np, [1, 0, 2]) expected_result2 = np.transpose(x_np, (2, 1, 0)) @@ -275,7 +254,6 @@ def test_dygraph_out(self): class TestTAPI(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program()): data = fluid.data(shape=[10], dtype="float32", name="data") @@ -283,7 +261,7 @@ def test_out(self): place = fluid.MLUPlace(0) exe = fluid.Executor(place) data_np = np.random.random([10]).astype("float32") - result, = exe.run(feed={"data": data_np}, fetch_list=[data_t]) + (result,) = exe.run(feed={"data": data_np}, fetch_list=[data_t]) expected_result = np.transpose(data_np) self.assertEqual((result == expected_result).all(), True) @@ -293,7 +271,7 @@ def test_out(self): place = fluid.MLUPlace(0) exe = fluid.Executor(place) data_np = np.random.random([10, 5]).astype("float32") - result, = exe.run(feed={"data": data_np}, fetch_list=[data_t]) + (result,) = exe.run(feed={"data": data_np}, fetch_list=[data_t]) expected_result = np.transpose(data_np) self.assertEqual((result == expected_result).all(), True) @@ -303,7 +281,7 @@ def test_out(self): place = fluid.MLUPlace(0) exe = fluid.Executor(place) data_np = np.random.random([1, 5]).astype("float32") - result, = exe.run(feed={"data": data_np}, fetch_list=[data_t]) + (result,) = exe.run(feed={"data": data_np}, fetch_list=[data_t]) expected_result = np.transpose(data_np) self.assertEqual((result == expected_result).all(), True) @@ -342,7 +320,6 @@ def test_x_dimension_check(): class TestMoveAxis(unittest.TestCase): - def test_moveaxis1(self): x_np = np.random.randn(2, 3, 4, 5, 7).astype('float32') expected = np.moveaxis(x_np, [0, 4, 3, 2], [1, 3, 2, 0]) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_tril_triu_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_tril_triu_op_mlu.py index fe74cbb05b0e3..2ed56de3c6d31 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_tril_triu_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_tril_triu_op_mlu.py @@ -27,14 +27,15 @@ class TrilTriuOpDefaultTest(OpTest): - """ the base class of other op testcases - """ + """the base class of other op testcases""" def setUp(self): self.initTestCase() self.__class__.use_mlu = True self.place = paddle.device.MLUPlace(0) - self.python_api = paddle.tril if self.real_op_type == 'tril' else paddle.triu + self.python_api = ( + paddle.tril if self.real_op_type == 'tril' else paddle.triu + ) self.real_np_op = getattr(np, self.real_op_type) self.op_type = "tril_triu" @@ -44,9 +45,9 @@ def setUp(self): 'lower': True if self.real_op_type == 'tril' else False, } self.outputs = { - 'Out': - self.real_np_op(self.X, self.diagonal) - if self.diagonal else self.real_np_op(self.X) + 'Out': self.real_np_op(self.X, self.diagonal) + if self.diagonal + else self.real_np_op(self.X) } def test_check_output(self): @@ -64,27 +65,29 @@ def case_generator(op_type, Xshape, diagonal, expected): If arg`expercted` is 'success', it will register an Optest case and expect to pass. Otherwise, it will register an API case and check the expect failure. """ - cls_name = "{0}_{1}_shape_{2}_diag_{3}".format(expected, op_type, Xshape, - diagonal) + cls_name = "{0}_{1}_shape_{2}_diag_{3}".format( + expected, op_type, Xshape, diagonal + ) errmsg = { - "diagonal: TypeError": - "diagonal in {} must be a python Int".format(op_type), - "input: ValueError": - "x shape in {} must be at least 2-D".format(op_type), + "diagonal: TypeError": "diagonal in {} must be a python Int".format( + op_type + ), + "input: ValueError": "x shape in {} must be at least 2-D".format( + op_type + ), } class FailureCase(unittest.TestCase): - def test_failure(self): paddle.enable_static() data = fluid.data(shape=Xshape, dtype='float64', name=cls_name) - with self.assertRaisesRegexp(eval(expected.split(':')[-1]), - errmsg[expected]): + with self.assertRaisesRegexp( + eval(expected.split(':')[-1]), errmsg[expected] + ): getattr(tensor, op_type)(x=data, diagonal=diagonal) class SuccessCase(TrilTriuOpDefaultTest): - def initTestCase(self): paddle.enable_static() @@ -109,15 +112,13 @@ def initTestCase(self): (20, 20): [ '2020', [20], - { - 20: 20 - }, + {20: 20}, (20, 20), 20.20, ], # str, list, dict, tuple, float }, 'input: ValueError': { - (2020, ): [None], + (2020,): [None], }, } for _op_type in ['tril', 'triu']: @@ -126,12 +127,15 @@ def initTestCase(self): list( map( lambda _diagonal: case_generator( - _op_type, _Xshape, _diagonal, _expected), _diaglist)) + _op_type, _Xshape, _diagonal, _expected + ), + _diaglist, + ) + ) class TestTrilTriuOpAPI(unittest.TestCase): - """ test case by using API and has -1 dimension - """ + """test case by using API and has -1 dimension""" def test_api(self): paddle.enable_static() @@ -163,8 +167,10 @@ def test_api_with_dygraph(self): with fluid.dygraph.guard(): data = np.random.random([1, 9, 9, 4]).astype(dtype) x = fluid.dygraph.to_variable(data) - tril_out, triu_out = tensor.tril(x).numpy(), tensor.triu( - x).numpy() + tril_out, triu_out = ( + tensor.tril(x).numpy(), + tensor.triu(x).numpy(), + ) np.testing.assert_allclose(tril_out, np.tril(data)) np.testing.assert_allclose(triu_out, np.triu(data)) @@ -182,9 +188,11 @@ def test_fluid_api(self): place = fluid.MLUPlace(0) exe = fluid.Executor(place) - triu_out = exe.run(fluid.default_main_program(), - feed={"x": data}, - fetch_list=[triu_out]) + triu_out = exe.run( + fluid.default_main_program(), + feed={"x": data}, + fetch_list=[triu_out], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/mlu/test_truncated_gaussian_random_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_truncated_gaussian_random_op_mlu.py index 50e5537fbffd7..98bcd7109d037 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_truncated_gaussian_random_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_truncated_gaussian_random_op_mlu.py @@ -30,14 +30,13 @@ class TestTrunctedGaussianRandomOp(unittest.TestCase): - def setUp(self): self.op_type = "truncated_gaussian_random" self.inputs = {} self.attrs = { "shape": [10000], - "mean": .0, - "std": 1., + "mean": 0.0, + "std": 1.0, "seed": 10, } self.outputs = ["Out"] @@ -56,9 +55,9 @@ def gaussian_random_test(self, place): program = fluid.Program() block = program.global_block() vout = block.create_var(name="Out") - op = block.append_op(type=self.op_type, - outputs={"Out": vout}, - attrs=self.attrs) + op = block.append_op( + type=self.op_type, outputs={"Out": vout}, attrs=self.attrs + ) op.desc.infer_var_type(block.desc) op.desc.infer_shape(block.desc) @@ -70,7 +69,7 @@ def gaussian_random_test(self, place): exe = Executor(place) outs = exe.run(program, fetch_list=fetch_list) tensor = outs[0] - self.assertAlmostEqual(numpy.mean(tensor), .0, delta=0.1) + self.assertAlmostEqual(numpy.mean(tensor), 0.0, delta=0.1) self.assertAlmostEqual(numpy.var(tensor), 0.773, delta=0.1) # TruncatedNormal.__call__ has no return value, so here call _C_ops api @@ -79,9 +78,14 @@ def gaussian_random_test_eager(self, place): with fluid.dygraph.guard(place): with _test_eager_guard(): out = paddle._C_ops.truncated_gaussian_random( - self.attrs["shape"], self.attrs["mean"], self.attrs["std"], - self.attrs["seed"], core.VarDesc.VarType.FP32, place) - self.assertAlmostEqual(numpy.mean(out.numpy()), .0, delta=0.1) + self.attrs["shape"], + self.attrs["mean"], + self.attrs["std"], + self.attrs["seed"], + core.VarDesc.VarType.FP32, + place, + ) + self.assertAlmostEqual(numpy.mean(out.numpy()), 0.0, delta=0.1) self.assertAlmostEqual(numpy.var(out.numpy()), 0.773, delta=0.1) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_uniform_random_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_uniform_random_op_mlu.py index 0dc702b5a221d..20abed4245c89 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_uniform_random_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_uniform_random_op_mlu.py @@ -25,7 +25,10 @@ from paddle.fluid.op import Operator import paddle.fluid as fluid from paddle.fluid import Program, program_guard -from test_uniform_random_op import TestUniformRandomOp, TestUniformRandomOpSelectedRows +from test_uniform_random_op import ( + TestUniformRandomOp, + TestUniformRandomOpSelectedRows, +) paddle.enable_static() @@ -39,7 +42,6 @@ def output_hist(out): class TestMLUUniformRandomOp(OpTest): - def setUp(self): self.set_mlu() self.op_type = "uniform_random" @@ -53,7 +55,7 @@ def init_attrs(self): "shape": [1000, 784], "min": -5.0, "max": 10.0, - "seed": 10 + "seed": 10, } self.output_hist = output_hist @@ -73,7 +75,6 @@ def verify_output(self, outs): class TestMLUUniformRandomOpSelectedRows(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_mlu(): @@ -88,12 +89,14 @@ def check_with_place(self, place): scope = core.Scope() out = scope.var("X").get_selected_rows() paddle.seed(10) - op = Operator("uniform_random", - Out="X", - shape=[1000, 784], - min=-5.0, - max=10.0, - seed=10) + op = Operator( + "uniform_random", + Out="X", + shape=[1000, 784], + min=-5.0, + max=10.0, + seed=10, + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) hist, prob = output_hist(np.array(out.get_tensor())) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze2_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze2_op_mlu.py index 3a52019357517..281a9a1ca2d08 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze2_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze2_op_mlu.py @@ -27,7 +27,6 @@ # Correct: General. class TestUnsqueezeOp(OpTest): - def setUp(self): self.init_test_case() self.set_mlu() @@ -36,7 +35,7 @@ def setUp(self): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float32") + "XShape": np.random.random(self.ori_shape).astype("float32"), } def set_mlu(self): @@ -60,16 +59,14 @@ def init_attrs(self): # Correct: Single input index. class TestUnsqueezeOp1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) # Correct: Mixed input axis. class TestUnsqueezeOp2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -78,7 +75,6 @@ def init_test_case(self): # Correct: There is duplicated axis. class TestUnsqueezeOp3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -87,7 +83,6 @@ def init_test_case(self): # Correct: Reversed axes. class TestUnsqueezeOp4(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -96,7 +91,6 @@ def init_test_case(self): # axes is a list(with tensor) class TestUnsqueezeOp_AxesTensorList(OpTest): - def setUp(self): self.init_test_case() self.set_mlu() @@ -104,17 +98,18 @@ def setUp(self): axes_tensor_list = [] for index, ele in enumerate(self.axes): - axes_tensor_list.append(("axes" + str(index), np.ones( - (1)).astype('int32') * ele)) + axes_tensor_list.append( + ("axes" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { "X": np.random.random(self.ori_shape).astype("float32"), - "AxesTensorList": axes_tensor_list + "AxesTensorList": axes_tensor_list, } self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float32") + "XShape": np.random.random(self.ori_shape).astype("float32"), } def set_mlu(self): @@ -137,15 +132,13 @@ def init_attrs(self): class TestUnsqueezeOp1_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) class TestUnsqueezeOp2_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -153,7 +146,6 @@ def init_test_case(self): class TestUnsqueezeOp3_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -161,7 +153,6 @@ def init_test_case(self): class TestUnsqueezeOp4_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -170,7 +161,6 @@ def init_test_case(self): # axes is a Tensor class TestUnsqueezeOp_AxesTensor(OpTest): - def setUp(self): self.init_test_case() self.set_mlu() @@ -178,12 +168,12 @@ def setUp(self): self.inputs = { "X": np.random.random(self.ori_shape).astype("float32"), - "AxesTensor": np.array(self.axes).astype("int32") + "AxesTensor": np.array(self.axes).astype("int32"), } self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float32") + "XShape": np.random.random(self.ori_shape).astype("float32"), } def set_mlu(self): @@ -206,15 +196,13 @@ def init_attrs(self): class TestUnsqueezeOp1_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) class TestUnsqueezeOp2_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -222,7 +210,6 @@ def init_test_case(self): class TestUnsqueezeOp3_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -230,7 +217,6 @@ def init_test_case(self): class TestUnsqueezeOp4_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze_op_mlu.py index 11317b61219e9..033f58d0e72f1 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_unsqueeze_op_mlu.py @@ -27,7 +27,6 @@ # Correct: General. class TestUnsqueezeOp(OpTest): - def setUp(self): self.init_test_case() self.set_mlu() @@ -57,16 +56,14 @@ def init_attrs(self): # Correct: Single input index. class TestUnsqueezeOp1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) # Correct: Mixed input axis. class TestUnsqueezeOp2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -75,7 +72,6 @@ def init_test_case(self): # Correct: There is duplicated axis. class TestUnsqueezeOp3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -84,7 +80,6 @@ def init_test_case(self): # Correct: Reversed axes. class TestUnsqueezeOp4(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_unstack_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_unstack_op_mlu.py index 2ec9726bcf2ad..6e46a9ff0b2d9 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_unstack_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_unstack_op_mlu.py @@ -24,7 +24,6 @@ class TestUnStackOpBase(OpTest): - def initDefaultParameters(self): self.input_dim = (5, 6, 7) self.axis = 0 @@ -74,25 +73,21 @@ def test_check_grad(self): class TestStackOp3(TestUnStackOpBase): - def initParameters(self): self.axis = -1 class TestStackOp4(TestUnStackOpBase): - def initParameters(self): self.axis = -3 class TestStackOp5(TestUnStackOpBase): - def initParameters(self): self.axis = 1 class TestStackOp6(TestUnStackOpBase): - def initParameters(self): self.axis = 2 diff --git a/python/paddle/fluid/tests/unittests/mlu/test_where_index_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_where_index_op_mlu.py index 2486bd8324ad3..163d47e33c2fb 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_where_index_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_where_index_op_mlu.py @@ -28,7 +28,6 @@ class TestWhereIndexOp(OpTest): - def setUp(self): self.op_type = "where_index" self.place = paddle.device.MLUPlace(0) @@ -47,7 +46,6 @@ def init_config(self): class TestAllFalse(unittest.TestCase): - def setUp(self): self.op_type = "where_index" self.place = paddle.device.MLUPlace(0) @@ -78,7 +76,6 @@ def test_all_false(self): class TestRank2(TestWhereIndexOp): - def init_config(self): self.inputs = { 'Condition': np.array([[True, False], [False, True]]), @@ -88,24 +85,26 @@ def init_config(self): class TestRank3(TestWhereIndexOp): - def init_config(self): self.inputs = { - 'Condition': - np.array([[[True, False], [False, True]], - [[False, True], [True, False]], - [[False, False], [False, True]]]), + 'Condition': np.array( + [ + [[True, False], [False, True]], + [[False, True], [True, False]], + [[False, False], [False, True]], + ] + ), } self.outputs = { - 'Out': - np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], [2, 1, 1]], - dtype='int64') + 'Out': np.array( + [[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], [2, 1, 1]], + dtype='int64', + ) } class TestWhereOpError(unittest.TestCase): - def test_api(self): with program_guard(Program(), Program()): cond = fluid.layers.data(name='cond', shape=[4], dtype='bool') @@ -118,9 +117,7 @@ def test_api(self): class TestWhereRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.where([10]) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_where_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_where_op_mlu.py index 5f757015656df..2bb780e1f268b 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_where_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_where_op_mlu.py @@ -31,7 +31,6 @@ class TestWhereOp(OpTest): - def setUp(self): self.op_type = 'where' self.place = paddle.device.MLUPlace(0) @@ -55,7 +54,6 @@ def init_config(self): class TestWhereOp2(TestWhereOp): - def init_config(self): self.x = np.random.uniform((-5), 5, (60, 2)).astype('float32') self.y = np.random.uniform((-5), 5, (60, 2)).astype('float32') @@ -63,7 +61,6 @@ def init_config(self): class TestWhereOp3(TestWhereOp): - def init_config(self): self.x = np.random.uniform((-3), 5, (20, 2, 4)).astype('float32') self.y = np.random.uniform((-3), 5, (20, 2, 4)).astype('float32') @@ -71,7 +68,6 @@ def init_config(self): class TestWhereAPI(unittest.TestCase): - def setUp(self): self.place = paddle.device.MLUPlace(0) self.__class__.use_mlu = True @@ -95,45 +91,49 @@ def test_api(self, use_mlu=False): for x_stop_gradient in [False, True]: for y_stop_gradient in [False, True]: with fluid.program_guard(Program(), Program()): - cond = fluid.layers.data(name='cond', - shape=self.shape, - dtype='bool') - x = fluid.layers.data(name='x', - shape=self.shape, - dtype='float32') - y = fluid.layers.data(name='y', - shape=self.shape, - dtype='float32') + cond = fluid.layers.data( + name='cond', shape=self.shape, dtype='bool' + ) + x = fluid.layers.data( + name='x', shape=self.shape, dtype='float32' + ) + y = fluid.layers.data( + name='y', shape=self.shape, dtype='float32' + ) x.stop_gradient = x_stop_gradient y.stop_gradient = y_stop_gradient result = paddle.where(cond, x, y) append_backward(paddle.mean(result)) for use_mlu in [False, True]: - place = (paddle.device.MLUPlace(0) - if use_mlu else fluid.CPUPlace()) + place = ( + paddle.device.MLUPlace(0) + if use_mlu + else fluid.CPUPlace() + ) exe = fluid.Executor(place) fetch_list = [result, result.grad_name] - if (x_stop_gradient is False): + if x_stop_gradient is False: fetch_list.append(x.grad_name) - if (y_stop_gradient is False): + if y_stop_gradient is False: fetch_list.append(y.grad_name) - out = exe.run(fluid.default_main_program(), - feed={ - 'cond': self.cond, - 'x': self.x, - 'y': self.y - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={'cond': self.cond, 'x': self.x, 'y': self.y}, + fetch_list=fetch_list, + ) assert np.array_equal(out[0], self.out) - if (x_stop_gradient is False): - assert np.array_equal(out[2], - self.ref_x_backward(out[1])) - if (y.stop_gradient is False): + if x_stop_gradient is False: + assert np.array_equal( + out[2], self.ref_x_backward(out[1]) + ) + if y.stop_gradient is False: assert np.array_equal( - out[3], self.ref_y_backward(out[1])) - elif (y.stop_gradient is False): - assert np.array_equal(out[2], - self.ref_y_backward(out[1])) + out[3], self.ref_y_backward(out[1]) + ) + elif y.stop_gradient is False: + assert np.array_equal( + out[2], self.ref_y_backward(out[1]) + ) def test_api_broadcast(self, use_mlu=False): main_program = Program() @@ -141,19 +141,20 @@ def test_api_broadcast(self, use_mlu=False): x = fluid.layers.data(name='x', shape=[4, 1], dtype='float32') y = fluid.layers.data(name='y', shape=[4, 2], dtype='float32') x_i = np.array([[0.9383, 0.1983, 3.2, 1.2]]).astype('float32') - y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, - 1.0]]).astype('float32') + y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]).astype( + 'float32' + ) result = paddle.where((x > 1), x=x, y=y) for use_mlu in [False, True]: - place = (paddle.device.MLUPlace(0) - if use_mlu else fluid.CPUPlace()) + place = ( + paddle.device.MLUPlace(0) if use_mlu else fluid.CPUPlace() + ) exe = fluid.Executor(place) - out = exe.run(fluid.default_main_program(), - feed={ - 'x': x_i, - 'y': y_i - }, - fetch_list=[result]) + out = exe.run( + fluid.default_main_program(), + feed={'x': x_i, 'y': y_i}, + fetch_list=[result], + ) assert np.array_equal(out[0], np.where((x_i > 1), x_i, y_i)) def test_scalar(self): @@ -161,20 +162,23 @@ def test_scalar(self): main_program = Program() with fluid.program_guard(main_program): cond_shape = [2, 4] - cond = fluid.layers.data(name='cond', - shape=cond_shape, - dtype='bool') + cond = fluid.layers.data( + name='cond', shape=cond_shape, dtype='bool' + ) x_data = 1.0 y_data = 2.0 cond_data = np.array([False, False, True, True]).astype('bool') result = paddle.where(condition=cond, x=x_data, y=y_data) for use_mlu in [False, True]: - place = (paddle.device.MLUPlace(0) - if use_mlu else fluid.CPUPlace()) + place = ( + paddle.device.MLUPlace(0) if use_mlu else fluid.CPUPlace() + ) exe = fluid.Executor(place) - out = exe.run(fluid.default_main_program(), - feed={'cond': cond_data}, - fetch_list=[result]) + out = exe.run( + fluid.default_main_program(), + feed={'cond': cond_data}, + fetch_list=[result], + ) expect = np.where(cond_data, x_data, y_data) assert np.array_equal(out[0], expect) @@ -182,27 +186,26 @@ def __test_where_with_broadcast_static(self, cond_shape, x_shape, y_shape): paddle.enable_static() main_program = Program() with fluid.program_guard(main_program): - cond = fluid.layers.data(name='cond', - shape=cond_shape, - dtype='bool') + cond = fluid.layers.data( + name='cond', shape=cond_shape, dtype='bool' + ) x = fluid.layers.data(name='x', shape=x_shape, dtype='float32') y = fluid.layers.data(name='y', shape=y_shape, dtype='float32') cond_data_tmp = np.random.random(size=cond_shape).astype('float32') - cond_data = (cond_data_tmp < 0.3) + cond_data = cond_data_tmp < 0.3 x_data = np.random.random(size=x_shape).astype('float32') y_data = np.random.random(size=y_shape).astype('float32') result = paddle.where(condition=cond, x=x, y=y) for use_mlu in [False, True]: - place = (paddle.device.MLUPlace(0) - if use_mlu else fluid.CPUPlace()) + place = ( + paddle.device.MLUPlace(0) if use_mlu else fluid.CPUPlace() + ) exe = fluid.Executor(place) - out = exe.run(fluid.default_main_program(), - feed={ - 'cond': cond_data, - 'x': x_data, - 'y': y_data - }, - fetch_list=[result]) + out = exe.run( + fluid.default_main_program(), + feed={'cond': cond_data, 'x': x_data, 'y': y_data}, + fetch_list=[result], + ) expect = np.where(cond_data, x_data, y_data) assert np.array_equal(out[0], expect) @@ -256,7 +259,6 @@ def test_static_api_broadcast_8(self): class TestWhereDygraphAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): x_i = np.array([0.9383, 0.1983, 3.2, 1.2]).astype('float32') @@ -280,7 +282,7 @@ def test_scalar(self): def __test_where_with_broadcast_dygraph(self, cond_shape, a_shape, b_shape): with fluid.dygraph.guard(): cond_tmp = paddle.rand(cond_shape) - cond = (cond_tmp < 0.3) + cond = cond_tmp < 0.3 a = paddle.rand(a_shape) b = paddle.rand(b_shape) result = paddle.where(cond, a, b) @@ -345,9 +347,9 @@ def test_where_condition(self): self.assertEqual(len(y), 2) z = fluid.layers.concat(list(y), axis=1) exe = fluid.Executor(paddle.device.MLUPlace(0)) - (res, ) = exe.run(feed={'x': data}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[z.name], return_numpy=False + ) expect_out = np.array([[0, 0], [1, 1]]) np.testing.assert_allclose(expect_out, np.array(res)) data = np.array([True, True, False]) @@ -358,15 +360,14 @@ def test_where_condition(self): self.assertEqual(len(y), 1) z = fluid.layers.concat(list(y), axis=1) exe = fluid.Executor(paddle.device.MLUPlace(0)) - (res, ) = exe.run(feed={'x': data}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[z.name], return_numpy=False + ) expect_out = np.array([[0], [1]]) np.testing.assert_allclose(expect_out, np.array(res)) class TestWhereOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x_i = np.array([0.9383, 0.1983, 3.2, 1.2]).astype('float32') @@ -390,7 +391,7 @@ def test_value_error(self): with fluid.dygraph.guard(): cond_shape = [2, 2, 4] cond_tmp = paddle.rand(cond_shape) - cond = (cond_tmp < 0.3) + cond = cond_tmp < 0.3 a = paddle.rand(cond_shape) self.assertRaises(ValueError, paddle.where, cond, a) diff --git a/python/paddle/fluid/tests/unittests/mlu/test_yolo_box_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_yolo_box_op_mlu.py index 86bc314eafcd8..fd8855aeb5706 100644 --- a/python/paddle/fluid/tests/unittests/mlu/test_yolo_box_op_mlu.py +++ b/python/paddle/fluid/tests/unittests/mlu/test_yolo_box_op_mlu.py @@ -29,7 +29,7 @@ def sigmoid(x): - return (1.0 / (1.0 + np.exp(((-1.0) * x)))) + return 1.0 / (1.0 + np.exp(((-1.0) * x))) def YoloBox(x, img_size, attrs): @@ -43,9 +43,9 @@ def YoloBox(x, img_size, attrs): scale_x_y = attrs['scale_x_y'] iou_aware = attrs['iou_aware'] iou_aware_factor = attrs['iou_aware_factor'] - bias_x_y = ((-0.5) * (scale_x_y - 1.0)) - input_h = (downsample * h) - input_w = (downsample * w) + bias_x_y = (-0.5) * (scale_x_y - 1.0) + input_h = downsample * h + input_w = downsample * w if iou_aware: ioup = x[:, :an_num, :, :] ioup = np.expand_dims(ioup, axis=(-1)) @@ -54,49 +54,54 @@ def YoloBox(x, img_size, attrs): pred_box = x[:, :, :, :, :4].copy() grid_x = np.tile(np.arange(w).reshape((1, w)), (h, 1)) grid_y = np.tile(np.arange(h).reshape((h, 1)), (1, w)) - pred_box[:, :, :, :, 0] = (( - (grid_x + (sigmoid(pred_box[:, :, :, :, 0]) * scale_x_y)) + bias_x_y) / - w) - pred_box[:, :, :, :, 1] = (( - (grid_y + (sigmoid(pred_box[:, :, :, :, 1]) * scale_x_y)) + bias_x_y) / - h) - anchors = [(anchors[i], anchors[(i + 1)]) - for i in range(0, len(anchors), 2)] - anchors_s = np.array([((an_w / input_w), (an_h / input_h)) - for (an_w, an_h) in anchors]) + pred_box[:, :, :, :, 0] = ( + (grid_x + (sigmoid(pred_box[:, :, :, :, 0]) * scale_x_y)) + bias_x_y + ) / w + pred_box[:, :, :, :, 1] = ( + (grid_y + (sigmoid(pred_box[:, :, :, :, 1]) * scale_x_y)) + bias_x_y + ) / h + anchors = [ + (anchors[i], anchors[(i + 1)]) for i in range(0, len(anchors), 2) + ] + anchors_s = np.array( + [((an_w / input_w), (an_h / input_h)) for (an_w, an_h) in anchors] + ) anchor_w = anchors_s[:, 0:1].reshape((1, an_num, 1, 1)) anchor_h = anchors_s[:, 1:2].reshape((1, an_num, 1, 1)) - pred_box[:, :, :, :, 2] = (np.exp(pred_box[:, :, :, :, 2]) * anchor_w) - pred_box[:, :, :, :, 3] = (np.exp(pred_box[:, :, :, :, 3]) * anchor_h) + pred_box[:, :, :, :, 2] = np.exp(pred_box[:, :, :, :, 2]) * anchor_w + pred_box[:, :, :, :, 3] = np.exp(pred_box[:, :, :, :, 3]) * anchor_h if iou_aware: - pred_conf = ((sigmoid(x[:, :, :, :, 4:5])**(1 - iou_aware_factor)) * - (sigmoid(ioup)**iou_aware_factor)) + pred_conf = (sigmoid(x[:, :, :, :, 4:5]) ** (1 - iou_aware_factor)) * ( + sigmoid(ioup) ** iou_aware_factor + ) else: pred_conf = sigmoid(x[:, :, :, :, 4:5]) pred_conf[(pred_conf < conf_thresh)] = 0.0 - pred_score = (sigmoid(x[:, :, :, :, 5:]) * pred_conf) - pred_box = (pred_box * (pred_conf > 0.0).astype('float32')) + pred_score = sigmoid(x[:, :, :, :, 5:]) * pred_conf + pred_box = pred_box * (pred_conf > 0.0).astype('float32') pred_box = pred_box.reshape((n, (-1), 4)) - (pred_box[:, :, :2], - pred_box[:, :, 2:4]) = ((pred_box[:, :, :2] - (pred_box[:, :, 2:4] / 2.0)), - (pred_box[:, :, :2] + (pred_box[:, :, 2:4] / 2.0))) - pred_box[:, :, 0] = (pred_box[:, :, 0] * img_size[:, 1][:, np.newaxis]) - pred_box[:, :, 1] = (pred_box[:, :, 1] * img_size[:, 0][:, np.newaxis]) - pred_box[:, :, 2] = (pred_box[:, :, 2] * img_size[:, 1][:, np.newaxis]) - pred_box[:, :, 3] = (pred_box[:, :, 3] * img_size[:, 0][:, np.newaxis]) + (pred_box[:, :, :2], pred_box[:, :, 2:4]) = ( + (pred_box[:, :, :2] - (pred_box[:, :, 2:4] / 2.0)), + (pred_box[:, :, :2] + (pred_box[:, :, 2:4] / 2.0)), + ) + pred_box[:, :, 0] = pred_box[:, :, 0] * img_size[:, 1][:, np.newaxis] + pred_box[:, :, 1] = pred_box[:, :, 1] * img_size[:, 0][:, np.newaxis] + pred_box[:, :, 2] = pred_box[:, :, 2] * img_size[:, 1][:, np.newaxis] + pred_box[:, :, 3] = pred_box[:, :, 3] * img_size[:, 0][:, np.newaxis] if clip_bbox: for i in range(len(pred_box)): pred_box[i, :, 0] = np.clip(pred_box[i, :, 0], 0, np.inf) pred_box[i, :, 1] = np.clip(pred_box[i, :, 1], 0, np.inf) - pred_box[i, :, 2] = np.clip(pred_box[i, :, 2], (-np.inf), - (img_size[(i, 1)] - 1)) - pred_box[i, :, 3] = np.clip(pred_box[i, :, 3], (-np.inf), - (img_size[(i, 0)] - 1)) + pred_box[i, :, 2] = np.clip( + pred_box[i, :, 2], (-np.inf), (img_size[(i, 1)] - 1) + ) + pred_box[i, :, 3] = np.clip( + pred_box[i, :, 3], (-np.inf), (img_size[(i, 0)] - 1) + ) return (pred_box, pred_score.reshape((n, (-1), class_num))) class TestYoloBoxOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'yolo_box' @@ -114,7 +119,7 @@ def setUp(self): 'clip_bbox': self.clip_bbox, 'scale_x_y': self.scale_x_y, 'iou_aware': self.iou_aware, - 'iou_aware_factor': self.iou_aware_factor + 'iou_aware_factor': self.iou_aware_factor, } self.inputs = {'X': x, 'ImgSize': img_size} (boxes, scores) = YoloBox(x, img_size, self.attrs) @@ -131,8 +136,12 @@ def initTestCase(self): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = True - self.x_shape = (self.batch_size, (an_num * (5 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (5 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.0 self.iou_aware = False @@ -140,7 +149,6 @@ def initTestCase(self): class TestYoloBoxOpNoClipBbox(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) @@ -149,8 +157,12 @@ def initTestCase(self): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = False - self.x_shape = (self.batch_size, (an_num * (5 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (5 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.0 self.iou_aware = False @@ -158,7 +170,6 @@ def initTestCase(self): class TestYoloBoxOpScaleXY(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) @@ -167,8 +178,12 @@ def initTestCase(self): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = True - self.x_shape = (self.batch_size, (an_num * (5 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (5 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.2 self.iou_aware = False @@ -176,7 +191,6 @@ def initTestCase(self): class TestYoloBoxOpIoUAware(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) @@ -185,8 +199,12 @@ def initTestCase(self): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = True - self.x_shape = (self.batch_size, (an_num * (6 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (6 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.0 self.iou_aware = True @@ -194,67 +212,72 @@ def initTestCase(self): class TestYoloBoxDygraph(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() img_size = np.ones((2, 2)).astype('int32') img_size = paddle.to_tensor(img_size) x1 = np.random.random([2, 14, 8, 8]).astype('float32') x1 = paddle.to_tensor(x1) - (boxes, scores) = paddle.vision.ops.yolo_box(x1, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0) - assert ((boxes is not None) and (scores is not None)) + (boxes, scores) = paddle.vision.ops.yolo_box( + x1, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + ) + assert (boxes is not None) and (scores is not None) x2 = np.random.random([2, 16, 8, 8]).astype('float32') x2 = paddle.to_tensor(x2) - (boxes, scores) = paddle.vision.ops.yolo_box(x2, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0, - iou_aware=True, - iou_aware_factor=0.5) + (boxes, scores) = paddle.vision.ops.yolo_box( + x2, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + iou_aware=True, + iou_aware_factor=0.5, + ) paddle.enable_static() class TestYoloBoxStatic(unittest.TestCase): - def test_static(self): x1 = paddle.static.data('x1', [2, 14, 8, 8], 'float32') img_size = paddle.static.data('img_size', [2, 2], 'int32') - (boxes, scores) = paddle.vision.ops.yolo_box(x1, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0) - assert ((boxes is not None) and (scores is not None)) + (boxes, scores) = paddle.vision.ops.yolo_box( + x1, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + ) + assert (boxes is not None) and (scores is not None) x2 = paddle.static.data('x2', [2, 16, 8, 8], 'float32') - (boxes, scores) = paddle.vision.ops.yolo_box(x2, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0, - iou_aware=True, - iou_aware_factor=0.5) - assert ((boxes is not None) and (scores is not None)) + (boxes, scores) = paddle.vision.ops.yolo_box( + x2, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + iou_aware=True, + iou_aware_factor=0.5, + ) + assert (boxes is not None) and (scores is not None) class TestYoloBoxOpHW(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) diff --git a/python/paddle/fluid/tests/unittests/multi_process.py b/python/paddle/fluid/tests/unittests/multi_process.py index fa6b7200f32f9..74283be7b3726 100644 --- a/python/paddle/fluid/tests/unittests/multi_process.py +++ b/python/paddle/fluid/tests/unittests/multi_process.py @@ -25,12 +25,18 @@ def train(prefix): worker_endpoints = worker_endpoints_env trainers_num = len(worker_endpoints.split(',')) - name = "selected_gpus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_gpus, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "selected_gpus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_gpus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) - with open("multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + with open( + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(name) @@ -47,23 +53,34 @@ def train_abort(prefix): # train abort exit(1) except SystemExit: - name = "abort>>> selected_gpus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_gpus, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "abort>>> selected_gpus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_gpus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) with open( - "multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(name) raise else: # sleep 30s to make sure paddle.distributed.launch will terminate this process time.sleep(30) - name = "selected_gpus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_gpus, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "selected_gpus:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_gpus, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) - with open("multi_process_{}.check_{}.log".format(prefix, trainer_id), - "w") as f: + with open( + "multi_process_{}.check_{}.log".format(prefix, trainer_id), "w" + ) as f: f.write(name) diff --git a/python/paddle/fluid/tests/unittests/my_data_generator.py b/python/paddle/fluid/tests/unittests/my_data_generator.py index 5279973e01acf..b08e5c53815d3 100644 --- a/python/paddle/fluid/tests/unittests/my_data_generator.py +++ b/python/paddle/fluid/tests/unittests/my_data_generator.py @@ -16,14 +16,14 @@ class MyDataset(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def data_iter(): elements = line.strip().split()[0:] - output = [("show", [int(elements[0])]), - ("click", [int(elements[1])]), - ("slot1", [int(elements[2])])] + output = [ + ("show", [int(elements[0])]), + ("click", [int(elements[1])]), + ("slot1", [int(elements[2])]), + ] yield output return data_iter diff --git a/python/paddle/fluid/tests/unittests/nproc_process.py b/python/paddle/fluid/tests/unittests/nproc_process.py index bbd0b6a0e5ee6..7a01005fe2e88 100644 --- a/python/paddle/fluid/tests/unittests/nproc_process.py +++ b/python/paddle/fluid/tests/unittests/nproc_process.py @@ -28,8 +28,13 @@ def train(prefix): worker_endpoints = worker_endpoints_env trainers_num = len(worker_endpoints.split(',')) - name = "selected_devices:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}"\ - .format(selected_devices, worker_endpoints, trainers_num, current_endpoint,trainer_id) + name = "selected_devices:{} worker_endpoints:{} trainers_num:{} current_endpoint:{} trainer_id:{}".format( + selected_devices, + worker_endpoints, + trainers_num, + current_endpoint, + trainer_id, + ) print(name) with open("{}.check_{}.log".format(prefix, trainer_id), "w") as f: diff --git a/python/paddle/fluid/tests/unittests/npu/collective_identity_op_npu.py b/python/paddle/fluid/tests/unittests/npu/collective_identity_op_npu.py index e4ed24a73f5ea..f8f8612ca89e3 100644 --- a/python/paddle/fluid/tests/unittests/npu/collective_identity_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/collective_identity_op_npu.py @@ -35,7 +35,6 @@ class TestCollectiveIdentity(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -43,22 +42,22 @@ def get_model(self, main_prog, startup_program): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofgather", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_identity", - inputs={'X': tindata}, - outputs={'Out': toutdata}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_identity", + inputs={'X': tindata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id, 'nranks': nranks}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/npu/process_group_hccl.py b/python/paddle/fluid/tests/unittests/npu/process_group_hccl.py index 4b96002536ee1..474d4104b04eb 100644 --- a/python/paddle/fluid/tests/unittests/npu/process_group_hccl.py +++ b/python/paddle/fluid/tests/unittests/npu/process_group_hccl.py @@ -37,7 +37,6 @@ def init_process_group(strategy=None): class TestProcessGroupFp32(unittest.TestCase): - def setUp(self): paddle.seed(2022) random.seed(2022) @@ -50,8 +49,9 @@ def config(self): def test_create_process_group_nccl(self): with _test_eager_guard(): - paddle.set_device('npu:%d' % - paddle.distributed.ParallelEnv().dev_id) + paddle.set_device( + 'npu:%d' % paddle.distributed.ParallelEnv().dev_id + ) pg = init_process_group() @@ -147,8 +147,9 @@ def test_create_process_group_nccl(self): task.wait() paddle.device.cuda.synchronize() out_1 = paddle.slice(tensor_out, [0], [0], [out_shape[0] // 2]) - out_2 = paddle.slice(tensor_out, [0], [out_shape[0] // 2], - [out_shape[0]]) + out_2 = paddle.slice( + tensor_out, [0], [out_shape[0] // 2], [out_shape[0]] + ) assert np.array_equal(tensor_x, out_1) assert np.array_equal(tensor_y, out_2) print("test allgather api ok\n") @@ -163,10 +164,12 @@ def test_create_process_group_nccl(self): tensor_y = paddle.to_tensor(y) tensor_out1 = paddle.to_tensor(out1) tensor_out2 = paddle.to_tensor(out2) - raw_tensor_x_2 = paddle.slice(tensor_x, [0], [self.shape[0] // 2], - [self.shape[0]]) - raw_tensor_y_1 = paddle.slice(tensor_y, [0], [0], - [self.shape[0] // 2]) + raw_tensor_x_2 = paddle.slice( + tensor_x, [0], [self.shape[0] // 2], [self.shape[0]] + ) + raw_tensor_y_1 = paddle.slice( + tensor_y, [0], [0], [self.shape[0] // 2] + ) if pg.rank() == 0: task = pg.alltoall(tensor_x, tensor_out1) task.wait() @@ -176,8 +179,9 @@ def test_create_process_group_nccl(self): task = pg.alltoall(tensor_y, tensor_out2) task.wait() paddle.device.cuda.synchronize() - out1_2 = paddle.slice(tensor_out1, [0], [self.shape[0] // 2], - [self.shape[0]]) + out1_2 = paddle.slice( + tensor_out1, [0], [self.shape[0] // 2], [self.shape[0]] + ) out2_1 = paddle.slice(tensor_out2, [0], [0], [self.shape[0] // 2]) if pg.rank() == 0: assert np.array_equal(out1_2.numpy(), raw_tensor_y_1.numpy()) @@ -223,8 +227,9 @@ def test_create_process_group_nccl(self): task.wait() paddle.device.cuda.synchronize() out1 = paddle.slice(tensor_x, [0], [0], [self.shape[0]]) - out2 = paddle.slice(tensor_x, [0], [self.shape[0]], - [self.shape[0] * 2]) + out2 = paddle.slice( + tensor_x, [0], [self.shape[0]], [self.shape[0] * 2] + ) if pg.rank() == 0: assert np.array_equal(tensor_y, out1) else: @@ -233,7 +238,6 @@ def test_create_process_group_nccl(self): class TestProcessGroupFp16(TestProcessGroupFp32): - def setUp(self): paddle.seed(2022) random.seed(2022) diff --git a/python/paddle/fluid/tests/unittests/npu/sync_batch_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/sync_batch_norm_op_npu.py index 84f93bc88a18c..4960fee949b61 100644 --- a/python/paddle/fluid/tests/unittests/npu/sync_batch_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/sync_batch_norm_op_npu.py @@ -31,17 +31,24 @@ from multiprocessing import Process import paddle.fluid.layers as layers from functools import reduce -from test_sync_batch_norm_base_npu import TestSyncBatchNormRunnerBase, runtime_main -from paddle.fluid.tests.unittests.op_test import OpTest, _set_use_system_allocator +from test_sync_batch_norm_base_npu import ( + TestSyncBatchNormRunnerBase, + runtime_main, +) +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + _set_use_system_allocator, +) -from paddle.fluid.tests.unittests.test_sync_batch_norm_op import create_or_get_tensor +from paddle.fluid.tests.unittests.test_sync_batch_norm_op import ( + create_or_get_tensor, +) _set_use_system_allocator(False) paddle.enable_static() class TestSyncBatchNormOpTraining(TestSyncBatchNormRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -53,29 +60,34 @@ def __init__(self): self.dshape = [self.N, self.C, self.H, self.W] self.atol = 1e-3 - def get_model(self, - main, - startup, - place, - layout, - seed, - sync_bn=False, - only_forward=False): + def get_model( + self, + main, + startup, + place, + layout, + seed, + sync_bn=False, + only_forward=False, + ): """Build program.""" use_cudnn = False with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - data = fluid.layers.data(name='input', - shape=self.dshape, - dtype=self.dtype, - append_batch_size=False) + data = fluid.layers.data( + name='input', + shape=self.dshape, + dtype=self.dtype, + append_batch_size=False, + ) conv = fluid.layers.conv2d( input=data, num_filters=32, filter_size=1, param_attr=fluid.ParamAttr(name='conv2d_weight'), bias_attr=False, - use_cudnn=use_cudnn) + use_cudnn=use_cudnn, + ) bn = fluid.layers.batch_norm( conv, param_attr=fluid.ParamAttr(name='bn_scale'), @@ -83,7 +95,8 @@ def get_model(self, moving_mean_name='bn_moving_mean', moving_variance_name='bn_moving_variance', data_layout=layout, - is_test=only_forward) + is_test=only_forward, + ) # if self.dtype == np.float16: # bn = fluid.layers.cast(bn, 'float32') sigmoid = fluid.layers.sigmoid(bn) diff --git a/python/paddle/fluid/tests/unittests/npu/test_abs_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_abs_op_npu.py index 8926937509482..f28552f14e749 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_abs_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_abs_op_npu.py @@ -25,7 +25,6 @@ class TestNPUAbs(OpTest): - def setUp(self): self.op_type = "abs" self.set_npu() diff --git a/python/paddle/fluid/tests/unittests/npu/test_accuracy_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_accuracy_op_npu.py index c46a51dfe3ae6..54d029384c6c0 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_accuracy_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_accuracy_op_npu.py @@ -27,7 +27,6 @@ class TestAccuracy(OpTest): - def setUp(self): self.op_type = "accuracy" self.set_npu() @@ -47,7 +46,7 @@ def setUp(self): self.outputs = { 'Accuracy': np.array([num_correct / float(n)]).astype(self.dtype), 'Correct': np.array([num_correct]).astype("int32"), - 'Total': np.array([n]).astype("int32") + 'Total': np.array([n]).astype("int32"), } def set_npu(self): @@ -62,7 +61,6 @@ def test_check_output(self): class TestAccuracy2(TestAccuracy): - def setUp(self): self.op_type = "accuracy" self.set_npu() @@ -82,12 +80,11 @@ def setUp(self): self.outputs = { 'Accuracy': np.array([num_correct / float(n)]).astype(self.dtype), 'Correct': np.array([num_correct]).astype("int32"), - 'Total': np.array([n]).astype("int32") + 'Total': np.array([n]).astype("int32"), } class TestAccuracyType(TestAccuracy): - def setUp(self): self.op_type = "accuracy" self.set_npu() @@ -107,12 +104,11 @@ def setUp(self): self.outputs = { 'Accuracy': np.array([num_correct / float(n)]).astype(self.dtype), 'Correct': np.array([num_correct]).astype("int32"), - 'Total': np.array([n]).astype("int32") + 'Total': np.array([n]).astype("int32"), } class TestAccuracyType2(TestAccuracy): - def setUp(self): self.op_type = "accuracy" self.set_npu() @@ -132,7 +128,7 @@ def setUp(self): self.outputs = { 'Accuracy': np.array([num_correct / float(n)]).astype(self.dtype), 'Correct': np.array([num_correct]).astype("int32"), - 'Total': np.array([n]).astype("int32") + 'Total': np.array([n]).astype("int32"), } diff --git a/python/paddle/fluid/tests/unittests/npu/test_adam_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_adam_op_npu.py index 70ab75ef5f242..49589d5d9dc80 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_adam_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_adam_op_npu.py @@ -28,7 +28,6 @@ class TestAdam(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -53,20 +52,19 @@ def setUp(self): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } self.attrs = {'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def set_npu(self): @@ -80,7 +78,6 @@ def test_check_output(self): class TestAdamWithEpsilonTensor(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -113,15 +110,14 @@ def setUp(self): self.attrs = {'epsilon': epsilon} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def set_npu(self): @@ -135,7 +131,6 @@ def test_check_output(self): class TestAdamOpWithSkipUpdate(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -188,7 +183,6 @@ def test_check_output(self): class TestAdamOpWithGlobalBetaPow(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -221,8 +215,7 @@ def setUp(self): attributes = {'epsilon': epsilon} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, attributes) + param_out, moment1_out, moment2_out = adam_step(self.inputs, attributes) self.attrs = {'use_global_beta_pow': True} @@ -232,7 +225,7 @@ def setUp(self): 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([]), - 'Beta2PowOut': np.array([]) + 'Beta2PowOut': np.array([]), } def set_npu(self): @@ -246,7 +239,6 @@ def test_check_output(self): class TestNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -261,9 +253,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -287,16 +279,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -309,13 +302,14 @@ def test_npu(self): class TestNetWithEpsilonTensor(unittest.TestCase): - - def _test(self, - place, - use_tensor=True, - use_fluid_api=True, - use_global_beta_pow=False, - flatten_param_grads=False): + def _test( + self, + place, + use_tensor=True, + use_fluid_api=True, + use_global_beta_pow=False, + flatten_param_grads=False, + ): paddle.enable_static() main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -329,29 +323,30 @@ def _test(self, weight_attr1 = paddle.ParamAttr( name="weight1", initializer=fluid.initializer.Constant(value=1.0), - trainable=True) + trainable=True, + ) weight_attr2 = paddle.ParamAttr( name="weight2", initializer=fluid.initializer.Constant(value=2.0), - trainable=True) + trainable=True, + ) clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) with paddle.static.program_guard(main_prog, startup_prog): with paddle.utils.unique_name.guard(): a = paddle.static.data(name="a", shape=[2, 2], dtype='float32') b = paddle.static.data(name="b", shape=[2, 2], dtype='float32') - label = paddle.static.data(name="label", - shape=[2, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[2, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) fc_1 = fluid.layers.fc(input=z, size=2, param_attr=weight_attr1) - prediction = fluid.layers.fc(input=fc_1, - size=2, - param_attr=weight_attr2, - act='softmax') + prediction = fluid.layers.fc( + input=fc_1, size=2, param_attr=weight_attr2, act='softmax' + ) cost = fluid.layers.cross_entropy(input=prediction, label=label) loss = fluid.layers.reduce_mean(cost) @@ -364,19 +359,22 @@ def _test(self, value=float(beta1_init), dtype='float32', persistable=True, - name="beta1") + name="beta1", + ) beta2 = fluid.layers.create_global_var( shape=[1], value=float(beta2_init), dtype='float32', persistable=True, - name="beta2") + name="beta2", + ) epsilon = fluid.layers.create_global_var( shape=[1], value=float(epsilon_init), dtype='float32', persistable=True, - name="epsilon") + name="epsilon", + ) if use_fluid_api: adam = fluid.optimizer.Adam( learning_rate=0.01, @@ -386,13 +384,16 @@ def _test(self, use_global_beta_pow=use_global_beta_pow, flatten_param_grads=flatten_param_grads, align_size=256, - grad_clip=clip) + grad_clip=clip, + ) else: - adam = paddle.optimizer.Adam(learning_rate=0.01, - beta1=beta1, - beta2=beta2, - epsilon=epsilon, - grad_clip=clip) + adam = paddle.optimizer.Adam( + learning_rate=0.01, + beta1=beta1, + beta2=beta2, + epsilon=epsilon, + grad_clip=clip, + ) else: if use_fluid_api: adam = fluid.optimizer.Adam( @@ -403,13 +404,16 @@ def _test(self, use_global_beta_pow=use_global_beta_pow, flatten_param_grads=flatten_param_grads, align_size=256, - grad_clip=clip) + grad_clip=clip, + ) else: - adam = fluid.optimizer.Adam(learning_rate=0.01, - beta1=beta1_init, - beta2=beta2_init, - epsilon=epsilon_init, - grad_clip=clip) + adam = fluid.optimizer.Adam( + learning_rate=0.01, + beta1=beta1_init, + beta2=beta2_init, + epsilon=epsilon_init, + grad_clip=clip, + ) adam.minimize(loss) @@ -420,15 +424,16 @@ def _test(self, print("Start run on {}".format(place)) for epoch in range(10): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) paddle.disable_static() return pred_res, loss_res @@ -440,10 +445,13 @@ def _test_with_place(self, place): for use_fluid_api in [True, False]: for use_global_beta_pow in [True, False]: for flatten_param_grads in [True, False]: - pred, loss = self._test(place, use_tensor, - use_fluid_api, - use_global_beta_pow, - flatten_param_grads) + pred, loss = self._test( + place, + use_tensor, + use_fluid_api, + use_global_beta_pow, + flatten_param_grads, + ) preds.append(pred) losses.append(loss) for pred in preds: diff --git a/python/paddle/fluid/tests/unittests/npu/test_adamw_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_adamw_op_npu.py index 579892dee3dce..0211eb196d58e 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_adamw_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_adamw_op_npu.py @@ -28,7 +28,6 @@ class TestAdamW(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -53,7 +52,7 @@ def setUp(self): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } self.attrs = { @@ -61,18 +60,19 @@ def setUp(self): 'beta1': beta1, 'beta2': beta2, "coeff": 0.9, - "with_decay": True + "with_decay": True, } - param_out, moment1_out, \ - moment2_out = adamw_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adamw_step( + self.inputs, self.attrs + ) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def set_npu(self): @@ -86,7 +86,6 @@ def test_check_output(self): class TestAdamOpWithSkipUpdate(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -139,7 +138,6 @@ def test_check_output(self): class TestAdamOpWithoutDecay(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -192,7 +190,6 @@ def test_check_output(self): class TestNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -207,9 +204,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -233,16 +230,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_amp_check_finite_and_scale_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_amp_check_finite_and_scale_op_npu.py index ee43d18ae2f01..ff11f1e68f0ca 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_amp_check_finite_and_scale_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_amp_check_finite_and_scale_op_npu.py @@ -27,7 +27,6 @@ class TestCheckFiniteAndUnscale(unittest.TestCase): - def get_prog(self): paddle.enable_static() main_program = paddle.static.Program() @@ -35,20 +34,21 @@ def get_prog(self): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') scale = paddle.static.data(name="scale", shape=[1], dtype='float32') - float_status = paddle.static.data(name="status", - shape=[8], - dtype='float32') + float_status = paddle.static.data( + name="status", shape=[8], dtype='float32' + ) main_program.global_block().append_op( - type="alloc_float_status", - outputs={"FloatStatus": float_status}) + type="alloc_float_status", outputs={"FloatStatus": float_status} + ) main_program.global_block().append_op( type="clear_float_status", inputs={"FloatStatus": float_status}, - outputs={"FloatStatusOut": float_status}) + outputs={"FloatStatusOut": float_status}, + ) c = paddle.fluid.layers.elementwise_div(a, b) - out, found_inf = check_finite_and_unscale([c], - scale, - float_status=float_status) + out, found_inf = check_finite_and_unscale( + [c], scale, float_status=float_status + ) return main_program, out, found_inf, float_status @@ -58,12 +58,9 @@ def run_prog(self, a, b, scale): exe = fluid.Executor(place) out_, founf_inf_, float_status_ = exe.run( main_program, - feed={ - "a": a, - "b": b, - "scale": scale - }, - fetch_list=[out, found_inf, float_status]) + feed={"a": a, "b": b, "scale": scale}, + fetch_list=[out, found_inf, float_status], + ) print(float_status_) return out_, founf_inf_ @@ -100,7 +97,6 @@ def test_not_contains_nan_inf(self): class TestCheckFiniteAndUnscaleClearFloatStatus(unittest.TestCase): - def get_prog(self): paddle.enable_static() main_program = paddle.static.Program() @@ -108,31 +104,33 @@ def get_prog(self): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') scale = paddle.static.data(name="scale", shape=[1], dtype='float32') - float_status = paddle.static.data(name="status", - shape=[8], - dtype='float32') + float_status = paddle.static.data( + name="status", shape=[8], dtype='float32' + ) main_program.global_block().append_op( - type="alloc_float_status", - outputs={"FloatStatus": float_status}) + type="alloc_float_status", outputs={"FloatStatus": float_status} + ) main_program.global_block().append_op( type="clear_float_status", inputs={"FloatStatus": float_status}, - outputs={"FloatStatusOut": float_status}) + outputs={"FloatStatusOut": float_status}, + ) c = paddle.fluid.layers.elementwise_div(a, b) - out, found_inf = check_finite_and_unscale([c], - scale, - float_status=float_status) + out, found_inf = check_finite_and_unscale( + [c], scale, float_status=float_status + ) main_program.global_block().append_op( - type="alloc_float_status", - outputs={"FloatStatus": float_status}) + type="alloc_float_status", outputs={"FloatStatus": float_status} + ) main_program.global_block().append_op( type="clear_float_status", inputs={"FloatStatus": float_status}, - outputs={"FloatStatusOut": float_status}) + outputs={"FloatStatusOut": float_status}, + ) d = paddle.fluid.layers.elementwise_add(a, b) - out, found_inf = check_finite_and_unscale([d], - scale, - float_status=float_status) + out, found_inf = check_finite_and_unscale( + [d], scale, float_status=float_status + ) return main_program, out, found_inf, float_status @@ -142,12 +140,9 @@ def run_prog(self, a, b, scale): exe = fluid.Executor(place) out_, founf_inf_, float_status_ = exe.run( main_program, - feed={ - "a": a, - "b": b, - "scale": scale - }, - fetch_list=[out, found_inf, float_status]) + feed={"a": a, "b": b, "scale": scale}, + fetch_list=[out, found_inf, float_status], + ) print(float_status_) return out_, founf_inf_ diff --git a/python/paddle/fluid/tests/unittests/npu/test_arg_max_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_arg_max_op_npu.py index 46e21378b848f..9b089462b01b5 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_arg_max_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_arg_max_op_npu.py @@ -27,7 +27,6 @@ class BaseTestCase(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -52,7 +51,6 @@ def test_check_output(self): # test argmax, dtype: float16 class TestArgMaxFloat16Case1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -61,7 +59,6 @@ def initTestCase(self): class TestArgMaxFloat16Case2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -70,7 +67,6 @@ def initTestCase(self): class TestArgMaxFloat16Case3(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -79,7 +75,6 @@ def initTestCase(self): class TestArgMaxFloat16Case4(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -88,7 +83,6 @@ def initTestCase(self): class TestArgMaxFloat16Case5(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -97,7 +91,6 @@ def initTestCase(self): class TestArgMaxFloat16Case6(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -106,7 +99,6 @@ def initTestCase(self): class TestArgMaxFloat16Case7(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -115,35 +107,31 @@ def initTestCase(self): class TestArgMaxFloat16Case8(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (1, ) + self.dims = (1,) self.dtype = 'float16' self.axis = 0 class TestArgMaxFloat16Case9(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (2, ) + self.dims = (2,) self.dtype = 'float16' self.axis = 0 class TestArgMaxFloat16Case10(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (3, ) + self.dims = (3,) self.dtype = 'float16' self.axis = 0 # test argmax, dtype: float32 class TestArgMaxFloat32Case1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -152,7 +140,6 @@ def initTestCase(self): class TestArgMaxFloat32Case2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -161,7 +148,6 @@ def initTestCase(self): class TestArgMaxFloat32Case3(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -170,7 +156,6 @@ def initTestCase(self): class TestArgMaxFloat32Case4(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -179,7 +164,6 @@ def initTestCase(self): class TestArgMaxFloat32Case5(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -188,7 +172,6 @@ def initTestCase(self): class TestArgMaxFloat32Case6(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -197,7 +180,6 @@ def initTestCase(self): class TestArgMaxFloat32Case7(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -206,34 +188,30 @@ def initTestCase(self): class TestArgMaxFloat32Case8(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (1, ) + self.dims = (1,) self.dtype = 'float32' self.axis = 0 class TestArgMaxFloat32Case9(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (2, ) + self.dims = (2,) self.dtype = 'float32' self.axis = 0 class TestArgMaxFloat32Case10(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (3, ) + self.dims = (3,) self.dtype = 'float32' self.axis = 0 class BaseTestComplex1_1(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -251,7 +229,7 @@ def setUp(self): self.inputs = {'X': self.x} self.attrs = { 'axis': self.axis, - 'dtype': int(core.VarDesc.VarType.INT32) + 'dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { 'Out': np.argmax(self.x, axis=self.axis).astype("int32") @@ -262,7 +240,6 @@ def test_check_output(self): class BaseTestComplex1_2(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -280,7 +257,7 @@ def setUp(self): self.inputs = {'X': self.x} self.attrs = { 'axis': self.axis, - 'dtype': int(core.VarDesc.VarType.INT32) + 'dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { 'Out': np.argmax(self.x, axis=self.axis).astype("int32") @@ -291,7 +268,6 @@ def test_check_output(self): class TestArgMaxAPI(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -303,7 +279,6 @@ def setUp(self): self.place = [paddle.NPUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -311,9 +286,9 @@ def run(place): tensor_input = paddle.to_tensor(numpy_input) numpy_output = np.argmax(numpy_input, axis=self.axis) paddle_output = paddle.argmax(tensor_input, axis=self.axis) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) paddle.enable_static() for place in self.place: @@ -321,7 +296,6 @@ def run(place): class TestArgMaxAPI_2(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -334,20 +308,20 @@ def setUp(self): self.place = [paddle.NPUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) numpy_input = (np.random.random(self.dims)).astype(self.dtype) tensor_input = paddle.to_tensor(numpy_input) - numpy_output = np.argmax(numpy_input, - axis=self.axis).reshape(1, 4, 5) - paddle_output = paddle.argmax(tensor_input, - axis=self.axis, - keepdim=self.keep_dims) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + numpy_output = np.argmax(numpy_input, axis=self.axis).reshape( + 1, 4, 5 + ) + paddle_output = paddle.argmax( + tensor_input, axis=self.axis, keepdim=self.keep_dims + ) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() @@ -356,7 +330,6 @@ def run(place): class TestArgMaxAPI_3(unittest.TestCase): - def initTestCase(self): self.dims = (1, 9) self.dtype = 'float32' @@ -367,7 +340,6 @@ def setUp(self): self.place = [paddle.NPUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -375,9 +347,9 @@ def run(place): tensor_input = paddle.to_tensor(numpy_input) numpy_output = np.argmax(numpy_input).reshape([1]) paddle_output = paddle.argmax(tensor_input) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/npu/test_arg_min_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_arg_min_op_npu.py index 8aeb32e5e5d9b..af8c038ac7606 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_arg_min_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_arg_min_op_npu.py @@ -25,7 +25,6 @@ class BaseTestCase(OpTest): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -51,7 +50,6 @@ def test_check_output(self): # test argmin, dtype: float16 class TestArgMinFloat16Case1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -60,7 +58,6 @@ def initTestCase(self): class TestArgMinFloat16Case2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -69,7 +66,6 @@ def initTestCase(self): class TestArgMinFloat16Case3(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -78,7 +74,6 @@ def initTestCase(self): class TestArgMinFloat16Case4(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -87,7 +82,6 @@ def initTestCase(self): class TestArgMinFloat16Case5(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -96,7 +90,6 @@ def initTestCase(self): class TestArgMinFloat16Case6(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -105,7 +98,6 @@ def initTestCase(self): class TestArgMinFloat16Case7(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -114,35 +106,31 @@ def initTestCase(self): class TestArgMinFloat16Case8(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' - self.dims = (1, ) + self.dims = (1,) self.dtype = 'float16' self.axis = 0 class TestArgMinFloat16Case9(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' - self.dims = (2, ) + self.dims = (2,) self.dtype = 'float16' self.axis = 0 class TestArgMinFloat16Case10(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' - self.dims = (3, ) + self.dims = (3,) self.dtype = 'float16' self.axis = 0 # test argmin, dtype: float32 class TestArgMinFloat32Case1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -151,7 +139,6 @@ def initTestCase(self): class TestArgMinFloat32Case2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -160,7 +147,6 @@ def initTestCase(self): class TestArgMinFloat32Case3(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -169,7 +155,6 @@ def initTestCase(self): class TestArgMinFloat32Case4(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -178,7 +163,6 @@ def initTestCase(self): class TestArgMinFloat32Case5(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -187,7 +171,6 @@ def initTestCase(self): class TestArgMinFloat32Case6(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -196,7 +179,6 @@ def initTestCase(self): class TestArgMinFloat32Case7(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -205,34 +187,30 @@ def initTestCase(self): class TestArgMinFloat32Case8(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' - self.dims = (1, ) + self.dims = (1,) self.dtype = 'float32' self.axis = 0 class TestArgMinFloat32Case9(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' - self.dims = (2, ) + self.dims = (2,) self.dtype = 'float32' self.axis = 0 class TestArgMinFloat32Case10(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' - self.dims = (3, ) + self.dims = (3,) self.dtype = 'float32' self.axis = 0 class TestArgMinAPI(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -244,7 +222,6 @@ def setUp(self): self.place = [paddle.NPUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -252,9 +229,9 @@ def run(place): tensor_input = paddle.to_tensor(numpy_input) numpy_output = np.argmin(numpy_input, axis=self.axis) paddle_output = paddle.argmin(tensor_input, axis=self.axis) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) paddle.enable_static() for place in self.place: @@ -262,7 +239,6 @@ def run(place): class TestArgMaxAPI_2(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -275,20 +251,20 @@ def setUp(self): self.place = [paddle.NPUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) numpy_input = (np.random.random(self.dims)).astype(self.dtype) tensor_input = paddle.to_tensor(numpy_input) - numpy_output = np.argmin(numpy_input, - axis=self.axis).reshape(1, 4, 5) - paddle_output = paddle.argmin(tensor_input, - axis=self.axis, - keepdim=self.keep_dims) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + numpy_output = np.argmin(numpy_input, axis=self.axis).reshape( + 1, 4, 5 + ) + paddle_output = paddle.argmin( + tensor_input, axis=self.axis, keepdim=self.keep_dims + ) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/npu/test_argsort_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_argsort_op_npu.py index 36f0e5e55f949..5aaf1de24516e 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_argsort_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_argsort_op_npu.py @@ -31,7 +31,6 @@ class TestArgsortOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "argsort" @@ -50,9 +49,11 @@ def setUp(self): def get_output(self): if self.descending: self.indices = np.flip( - np.argsort(self.x, kind='heapsort', axis=self.axis), self.axis) + np.argsort(self.x, kind='heapsort', axis=self.axis), self.axis + ) self.sorted_x = np.flip( - np.sort(self.x, kind='heapsort', axis=self.axis), self.axis) + np.sort(self.x, kind='heapsort', axis=self.axis), self.axis + ) else: self.indices = np.argsort(self.x, kind='heapsort', axis=self.axis) self.sorted_x = np.sort(self.x, kind='heapsort', axis=self.axis) @@ -78,67 +79,56 @@ def init_direction(self): class TestArgsortOpAxis0NPU(TestArgsortOp): - def init_axis(self): self.axis = 0 class TestArgsortOpAxis1NPU(TestArgsortOp): - def init_axis(self): self.axis = 1 class TestArgsortOpAxis2NPU(TestArgsortOp): - def init_axis(self): self.axis = 2 class TestArgsortOpAxisNeg1NPU(TestArgsortOp): - def init_axis(self): self.axis = -1 class TestArgsortOpAxisNeg2NPU(TestArgsortOp): - def init_axis(self): self.axis = -2 class TestArgsortOpDescendingAxisNPU(TestArgsortOp): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis0NPU(TestArgsortOpAxis0NPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis1NPU(TestArgsortOpAxis1NPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis2NPU(TestArgsortOpAxis2NPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg1NPU(TestArgsortOpAxisNeg1NPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg2NPU(TestArgsortOpAxisNeg2NPU): - def init_direction(self): self.descending = True @@ -151,7 +141,6 @@ def init_direction(self): class TestArgsortOpAxis0NPUFP32(TestArgsortOp): - def init_axis(self): self.axis = 0 @@ -165,74 +154,63 @@ def set_npu(self): self.__class__.use_npu = True def test_check_grad(self): - self.check_grad_with_place(self.place, ["X"], - "Out", - max_relative_error=0.03) + self.check_grad_with_place( + self.place, ["X"], "Out", max_relative_error=0.03 + ) class TestArgsortOpAxis1NPUFP32(TestArgsortOpAxis0NPUFP32): - def init_axis(self): self.axis = 1 class TestArgsortOpAxis2NPUFP32(TestArgsortOpAxis0NPUFP32): - def init_axis(self): self.axis = 2 class TestArgsortOpAxisNeg1NPUFP32(TestArgsortOpAxis0NPUFP32): - def init_axis(self): self.axis = -1 class TestArgsortOpAxisNeg2NPUFP32(TestArgsortOpAxis0NPUFP32): - def init_axis(self): self.axis = -2 class TestArgsortOpDescendingAxisNPUFP32(TestArgsortOpAxis0NPUFP32): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis0NPUFP32(TestArgsortOpAxis0NPUFP32): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis1NPUFP32(TestArgsortOpAxis1NPUFP32): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis2NPUFP32(TestArgsortOpAxis2NPUFP32): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg1NPUFP32(TestArgsortOpAxisNeg1NPUFP32): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg2NPUFP32(TestArgsortOpAxisNeg2NPUFP32): - def init_direction(self): self.descending = True # test cases for int64 class TestArgsortOpAxis0NPUINT64(TestArgsortOp): - def setUp(self): self.set_npu() self.op_type = "argsort" @@ -242,10 +220,9 @@ def setUp(self): self.init_axis() self.init_direction() - self.x = np.random.randint(low=-100, - high=100, - size=self.input_shape, - dtype=self.dtype).astype(self.dtype) + self.x = np.random.randint( + low=-100, high=100, size=self.input_shape, dtype=self.dtype + ).astype(self.dtype) self.inputs = {"X": self.x} self.attrs = {"axis": self.axis, "descending": self.descending} self.get_output() @@ -265,61 +242,51 @@ def set_npu(self): class TestArgsortOpAxis1NPUINT64(TestArgsortOpAxis0NPUINT64): - def init_axis(self): self.axis = 1 class TestArgsortOpAxis2NPUINT64(TestArgsortOpAxis0NPUINT64): - def init_axis(self): self.axis = 2 class TestArgsortOpAxisNeg1NPUINT64(TestArgsortOpAxis0NPUINT64): - def init_axis(self): self.axis = -1 class TestArgsortOpAxisNeg2NPUINT64(TestArgsortOpAxis0NPUINT64): - def init_axis(self): self.axis = -2 class TestArgsortOpDescendingAxisNPUINT64(TestArgsortOpAxis0NPUINT64): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis0NPUINT64(TestArgsortOpAxis0NPUINT64): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis1NPUINT64(TestArgsortOpAxis1NPUINT64): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis2NPUINT64(TestArgsortOpAxis2NPUINT64): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg1NPUINT64(TestArgsortOpAxisNeg1NPUINT64): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg2NPUINT64(TestArgsortOpAxisNeg2NPUINT64): - def init_direction(self): self.descending = True diff --git a/python/paddle/fluid/tests/unittests/npu/test_assign_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_assign_op_npu.py index 82969805dc168..98da7c4141dee 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_assign_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_assign_op_npu.py @@ -27,7 +27,6 @@ class TestAssign(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/npu/test_assign_value_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_assign_value_op_npu.py index 7ffbcd1db13a7..402b90bc49bbd 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_assign_value_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_assign_value_op_npu.py @@ -29,7 +29,6 @@ class TestAssignValueNPUOp(op_test.OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -41,7 +40,8 @@ def setUp(self): self.attrs["shape"] = self.value.shape self.attrs["dtype"] = framework.convert_np_dtype_to_dtype_( - self.value.dtype) + self.value.dtype + ) self.outputs = {"Out": self.value} def set_npu(self): @@ -56,35 +56,36 @@ def test_forward(self): class TestAssignValueNPUOp2(TestAssignValueNPUOp): - def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int32) self.attrs["int32_values"] = [int(v) for v in self.value.flat] class TestAssignValueNPUOp3(TestAssignValueNPUOp): - def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int64) self.attrs["int64_values"] = [int(v) for v in self.value.flat] class TestAssignValueNPUOp4(TestAssignValueNPUOp): - def init_data(self): - self.value = np.random.choice(a=[False, True], - size=(2, 5)).astype(np.bool) + self.value = np.random.choice(a=[False, True], size=(2, 5)).astype( + np.bool + ) self.attrs["bool_values"] = [int(v) for v in self.value.flat] class TestAssignApi(unittest.TestCase): - def setUp(self): self.init_dtype() self.value = (-100 + 200 * np.random.random(size=(2, 5))).astype( - self.dtype) - self.place = fluid.NPUPlace( - 0) if fluid.core.is_compiled_with_npu() else fluid.CPUPlace() + self.dtype + ) + self.place = ( + fluid.NPUPlace(0) + if fluid.core.is_compiled_with_npu() + else fluid.CPUPlace() + ) def init_dtype(self): self.dtype = "float32" @@ -102,25 +103,26 @@ def test_assign(self): class TestAssignApi2(TestAssignApi): - def init_dtype(self): self.dtype = "int32" class TestAssignApi3(TestAssignApi): - def init_dtype(self): self.dtype = "int64" class TestAssignApi4(TestAssignApi): - def setUp(self): self.init_dtype() - self.value = np.random.choice(a=[False, True], - size=(2, 5)).astype(np.bool) - self.place = fluid.NPUPlace( - 0) if fluid.core.is_compiled_with_npu() else fluid.CPUPlace() + self.value = np.random.choice(a=[False, True], size=(2, 5)).astype( + np.bool + ) + self.place = ( + fluid.NPUPlace(0) + if fluid.core.is_compiled_with_npu() + else fluid.CPUPlace() + ) def init_dtype(self): self.dtype = "bool" diff --git a/python/paddle/fluid/tests/unittests/npu/test_atan_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_atan_op_npu.py index bb13d82366935..fe22453f4fa45 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_atan_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_atan_op_npu.py @@ -26,7 +26,6 @@ class TestAtan(OpTest): - def setUp(self): self.set_npu() self.op_type = "atan" @@ -57,7 +56,7 @@ def test_out_name(self): out = paddle.atan(data, name='Y') place = paddle.NPUPlace(0) exe = fluid.Executor(place) - result, = exe.run(feed={"X": np_x}, fetch_list=[out]) + (result,) = exe.run(feed={"X": np_x}, fetch_list=[out]) expected = np.arctan(np_x) self.assertEqual(result, expected) @@ -74,13 +73,11 @@ def test_check_output(self): class TestAtanShape(TestAtan): - def set_attrs(self): self.shape = [12, 23, 10] class TestAtanFloat16(TestAtan): - def set_attrs(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/npu/test_batch_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_batch_norm_op_npu.py index 37c6fcce1e1ac..e39506eed7a9b 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_batch_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_batch_norm_op_npu.py @@ -24,24 +24,27 @@ from op_test import OpTest, _set_use_system_allocator from paddle.fluid import Program, program_guard -from test_batch_norm_op import _reference_testing, _cal_mean_variance, _reference_training, _reference_grad +from test_batch_norm_op import ( + _reference_testing, + _cal_mean_variance, + _reference_training, + _reference_grad, +) _set_use_system_allocator(False) paddle.enable_static() class TestBatchNormOpInference(unittest.TestCase): - def setUp(self): self.dtype = np.float32 self.init_kernel_type() self.data_formats = ["NCHW", "NHWC"] def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - atol=atol, - err_msg=msg) + np.testing.assert_allclose( + np.array(tensor), np_array, atol=atol, err_msg=msg + ) def check_with_place(self, place, data_layout, dtype, shape): epsilon = epsilon = 0.00001 @@ -72,8 +75,9 @@ def check_with_place(self, place, data_layout, dtype, shape): bias = np.random.random_sample(scale_shape).astype(np.float32) mean = np.zeros(scale_shape).astype(np.float32) variance = np.ones(scale_shape).astype(np.float32) - y = _reference_testing(x, scale, bias, mean, variance, epsilon, - data_layout).astype(dtype) + y = _reference_testing( + x, scale, bias, mean, variance, epsilon, data_layout + ).astype(dtype) var_dict = locals() var_names = ["x", "scale", "bias", "mean", "variance", "y"] ground_truth = {name: var_dict[name] for name in var_names} @@ -84,15 +88,15 @@ def check_with_place(self, place, data_layout, dtype, shape): with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype="float32", - shape=ground_truth[name].shape) + block.create_var( + name=name, dtype="float32", shape=ground_truth[name].shape + ) inputs = { "X": block.var("x"), "Scale": block.var("scale"), "Bias": block.var("bias"), "Mean": block.var("mean"), - "Variance": block.var("variance") + "Variance": block.var("variance"), } attrs = { "epsilon": epsilon, @@ -106,14 +110,13 @@ def check_with_place(self, place, data_layout, dtype, shape): "MeanOut": block.var("mean"), # share memory "VarianceOut": block.var("variance"), # share memory "SavedMean": block.var("saved_mean"), - "SavedVariance": block.var("saved_variance") + "SavedVariance": block.var("saved_variance"), } block.create_var(name="reserve_space", dtype='float32') outputs["ReserveSpace"] = block.var('reserve_space') - bn_op = block.append_op(type="batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + bn_op = block.append_op( + type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs + ) program._sync_with_cpp() @@ -124,7 +127,8 @@ def check_with_place(self, place, data_layout, dtype, shape): name: ground_truth[name] for name in ["x", "scale", "bias", "mean", "variance"] }, - fetch_list=["y"]) + fetch_list=["y"], + ) self.__assert_close(var_dict["y"], out[0], "y", atol=1e-3) def test_check_output(self): @@ -138,7 +142,6 @@ def init_kernel_type(self): class TestFP16BatchNormOpInference(TestBatchNormOpInference): - def setUp(self): self.dtype = np.float16 self.init_kernel_type() @@ -146,7 +149,6 @@ def setUp(self): class TestBatchNormOpTraining(unittest.TestCase): - def set_npu(self): self.__class__.use_npu = True @@ -169,27 +171,54 @@ def init_test_case(self): self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - "y", 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + "y", + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] def __assert_close(self, tensor, np_array, msg, atol=1e-4): np.allclose(np.array(tensor), np_array, atol=atol) - def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, - epsilon, momentum, shape, data_layout): + def ref_forward_backward( + self, + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ): # run forward - y, saved_mean, var_ref = _reference_training(x, scale, bias, epsilon, - data_layout) - mean_out = saved_mean * (1. - momentum) + momentum * mean - variance_out = var_ref * (1. - momentum) + momentum * variance - saved_variance = 1. / np.sqrt(var_ref + epsilon) + y, saved_mean, var_ref = _reference_training( + x, scale, bias, epsilon, data_layout + ) + mean_out = saved_mean * (1.0 - momentum) + momentum * mean + variance_out = var_ref * (1.0 - momentum) + momentum * variance + saved_variance = 1.0 / np.sqrt(var_ref + epsilon) # run backward - x_grad, scale_grad, bias_grad = _reference_grad(x, y_grad, scale, - saved_mean, var_ref, - epsilon, data_layout) - - return y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad + x_grad, scale_grad, bias_grad = _reference_grad( + x, y_grad, scale, saved_mean, var_ref, epsilon, data_layout + ) + + return ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) def set_mean_variance(self, scale_shape, x, data_layout): mean, variance = _cal_mean_variance(x, self.epsilon, data_layout) @@ -198,12 +227,11 @@ def set_mean_variance(self, scale_shape, x, data_layout): # computing global mean/variance for one step if self.use_global_stats: mom = self.momentum - mean = mean * (1. - mom) + mom * mean_pre - variance = variance * (1. - mom) + mom * variance_pre + mean = mean * (1.0 - mom) + mom * mean_pre + variance = variance * (1.0 - mom) + mom * variance_pre return mean, variance def test_forward_backward(self): - def test_with_place(place, data_layout, shape): # attr epsilon = self.epsilon @@ -236,9 +264,27 @@ def test_with_place(place, data_layout, shape): y_grad = np.random.random_sample(shape).astype(self.dtype) momentum_var = np.array([momentum]).astype(np.float32) - y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad = self.ref_forward_backward( - x, y_grad, scale, bias, mean, variance, epsilon, momentum, - shape, data_layout) + ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) = self.ref_forward_backward( + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ) var_dict = locals() var_dict['y@GRAD'] = y_grad @@ -247,8 +293,15 @@ def test_with_place(place, data_layout, shape): var_dict['bias@GRAD'] = bias_grad var_names = [ - 'x', 'scale', 'bias', 'mean', 'variance', "y", 'saved_mean', - 'saved_variance', 'momentum_var' + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + "y", + 'saved_mean', + 'saved_variance', + 'momentum_var', ] ground_truth = {name: var_dict[name] for name in var_names} @@ -256,15 +309,17 @@ def test_with_place(place, data_layout, shape): with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) inputs = { "X": block.var('x'), "Scale": block.var('scale'), "Bias": block.var('bias'), "Mean": block.var('mean'), - "Variance": block.var('variance') + "Variance": block.var('variance'), } attrs = { "epsilon": epsilon, @@ -272,7 +327,7 @@ def test_with_place(place, data_layout, shape): "data_layout": data_layout, "use_mkldnn": self.use_mkldnn, "fuse_with_relu": self.fuse_with_relu, - "use_global_stats": self.use_global_stats + "use_global_stats": self.use_global_stats, } if self.use_momentum_variable: inputs['MomentumTensor'] = block.var('momentum_var') @@ -284,19 +339,22 @@ def test_with_place(place, data_layout, shape): "MeanOut": block.var('mean'), # share memory "VarianceOut": block.var('variance'), # share memory "SavedMean": block.var('saved_mean'), - "SavedVariance": block.var('saved_variance') + "SavedVariance": block.var('saved_variance'), } block.create_var(name="reserve_space", dtype='float32') outputs["ReserveSpace"] = block.var('reserve_space') - bn_op = block.append_op(type="batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + bn_op = block.append_op( + type="batch_norm", + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) block.create_var(name='y@GRAD', dtype=self.dtype, shape=y.shape) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - bn_op.desc, self.no_grad_set, []) + bn_op.desc, self.no_grad_set, [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -311,22 +369,28 @@ def test_with_place(place, data_layout, shape): program._sync_with_cpp() exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in [ - 'x', 'scale', 'bias', 'mean', 'variance', - 'y@GRAD', 'momentum_var' - ] - }, - fetch_list=self.fetch_list) + out = exe.run( + program, + feed={ + name: var_dict[name] + for name in [ + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + 'y@GRAD', + 'momentum_var', + ] + }, + fetch_list=self.fetch_list, + ) for id, name in enumerate(self.fetch_list): if name == 'variance': - self.__assert_close(var_dict[name], - out[id], - name, - atol=1e-3) + self.__assert_close( + var_dict[name], out[id], name, atol=1e-3 + ) continue self.__assert_close(var_dict[name], out[id], name) print("op test forward passed: ", str(place), data_layout) @@ -340,13 +404,11 @@ def init_kernel_type(self): class TestFP16BatchNormOpTraining(TestBatchNormOpTraining): - def init_dtype(self): self.dtype = np.float16 class TestBatchNormOpTrainingCase1(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = False self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -354,24 +416,33 @@ def init_test_case(self): class TestBatchNormOpTrainingMomentumVariable(TestBatchNormOpTraining): - def init_test_case(self): self.use_momentum_variable = True self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = True self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'x@GRAD', 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] def reference_grad(self, x, y_grad, scale, mean, var, epsilon, data_format): @@ -379,20 +450,23 @@ def reference_grad(self, x, y_grad, scale, mean, var, epsilon, data_format): if len(x_shape) == 3: if data_format == "NCHW": # NCL -> NCL1 x = np.reshape(x, (x_shape[0], x_shape[1], x_shape[2], 1)) - y_grad = np.reshape(y_grad, - (x_shape[0], x_shape[1], x_shape[2], 1)) + y_grad = np.reshape( + y_grad, (x_shape[0], x_shape[1], x_shape[2], 1) + ) else: # NLC -> NL1C x = np.reshape(x, (x_shape[0], x_shape[1], 1, x_shape[2])) - y_grad = np.reshape(y_grad, - (x_shape[0], x_shape[1], 1, x_shape[2])) + y_grad = np.reshape( + y_grad, (x_shape[0], x_shape[1], 1, x_shape[2]) + ) if data_format == "NCHW": x = np.transpose(x, (0, 2, 3, 1)) y_grad = np.transpose(y_grad, (0, 2, 3, 1)) x_grad = scale * y_grad / np.sqrt(var + epsilon) - grad_scale = np.sum(y_grad * (x - mean) / np.sqrt(var + epsilon), - axis=(0, 1, 2)) + grad_scale = np.sum( + y_grad * (x - mean) / np.sqrt(var + epsilon), axis=(0, 1, 2) + ) grad_offset = np.sum(y_grad, axis=(0, 1, 2)) # transfer back to N, C, H, W @@ -406,8 +480,19 @@ def reference_grad(self, x, y_grad, scale, mean, var, epsilon, data_format): return x_grad, grad_scale, grad_offset - def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, - epsilon, momentum, shape, data_layout): + def ref_forward_backward( + self, + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ): if data_layout != "NCHW" and data_layout != "NHWC": raise ValueError("Unknown data order.") @@ -415,12 +500,14 @@ def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, if len(x_shape) == 3: if data_layout == "NCHW": # NCL -> NCL1 x = np.reshape(x, (x_shape[0], x_shape[1], x_shape[2], 1)) - y_grad = np.reshape(y_grad, - (x_shape[0], x_shape[1], x_shape[2], 1)) + y_grad = np.reshape( + y_grad, (x_shape[0], x_shape[1], x_shape[2], 1) + ) else: # NLC -> NL1C x = np.reshape(x, (x_shape[0], x_shape[1], 1, x_shape[2])) - y_grad = np.reshape(y_grad, - (x_shape[0], x_shape[1], 1, x_shape[2])) + y_grad = np.reshape( + y_grad, (x_shape[0], x_shape[1], 1, x_shape[2]) + ) if data_layout == "NCHW": x = np.transpose(x, (0, 2, 3, 1)) @@ -436,21 +523,31 @@ def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, mean_out = mean variance_out = variance - saved_variance = 1. / np.sqrt(variance + epsilon) + saved_variance = 1.0 / np.sqrt(variance + epsilon) # run backward x_grad, scale_grad, bias_grad = self.reference_grad( - x, y_grad, scale, mean, variance, epsilon, data_layout) + x, y_grad, scale, mean, variance, epsilon, data_layout + ) if len(x_shape) == 3: y = np.reshape(y, x_shape) x_grad = np.reshape(x_grad, x_shape) - return y, mean_out, variance_out, mean, saved_variance, x_grad, scale_grad, bias_grad + return ( + y, + mean_out, + variance_out, + mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) class TestBatchNormOpFreezeStatsAndScaleBiasTraining( - TestBatchNormOpFreezeStatsTraining): - + TestBatchNormOpFreezeStatsTraining +): def init_test_case(self): self.use_global_stats = True self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -458,7 +555,6 @@ def init_test_case(self): class TestDygraphBatchNormTrainableStats(unittest.TestCase): - def test_dygraph(self): places = [fluid.NPUPlace(0)] for p in places: @@ -469,7 +565,8 @@ def compute(x, is_test, trainable_statistics): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) y = bn(fluid.dygraph.to_variable(x)) return y.numpy() @@ -489,7 +586,8 @@ def compute(x_np, is_test, trainable_statistics): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) x = fluid.data(name='x', shape=x_np.shape, dtype=x_np.dtype) y = bn(x) exe.run(fluid.default_startup_program()) diff --git a/python/paddle/fluid/tests/unittests/npu/test_bce_loss_npu.py b/python/paddle/fluid/tests/unittests/npu/test_bce_loss_npu.py index 10ec6f3fae1ed..d9e968964afb6 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_bce_loss_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_bce_loss_npu.py @@ -24,88 +24,76 @@ paddle.enable_static() -def test_static_layer(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_static_layer( + place, input_np, label_np, reduction='mean', weight_np=None +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=input_np.shape, - dtype='float32') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float32') + input = paddle.fluid.data( + name='input', shape=input_np.shape, dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float32' + ) if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float32') - bce_loss = paddle.nn.loss.BCELoss(weight=weight, - reduction=reduction) + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float32' + ) + bce_loss = paddle.nn.loss.BCELoss( + weight=weight, reduction=reduction + ) else: bce_loss = paddle.nn.loss.BCELoss(reduction=reduction) res = bce_loss(input, label) exe = paddle.static.Executor(place) - static_result = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - } if weight_np is None else { - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + static_result = exe.run( + prog, + feed={"input": input_np, "label": label_np} + if weight_np is None + else {"input": input_np, "label": label_np, "weight": weight_np}, + fetch_list=[res], + ) return static_result[0] -def test_static_functional(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_static_functional( + place, input_np, label_np, reduction='mean', weight_np=None +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=input_np.shape, - dtype='float32') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float32') + input = paddle.fluid.data( + name='input', shape=input_np.shape, dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float32' + ) if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float32') - res = paddle.nn.functional.binary_cross_entropy(input, - label, - weight=weight, - reduction=reduction) + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float32' + ) + res = paddle.nn.functional.binary_cross_entropy( + input, label, weight=weight, reduction=reduction + ) else: - res = paddle.nn.functional.binary_cross_entropy(input, - label, - reduction=reduction) + res = paddle.nn.functional.binary_cross_entropy( + input, label, reduction=reduction + ) exe = paddle.static.Executor(place) - static_result = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - } if weight_np is None else { - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + static_result = exe.run( + prog, + feed={"input": input_np, "label": label_np} + if weight_np is None + else {"input": input_np, "label": label_np, "weight": weight_np}, + fetch_list=[res], + ) return static_result[0] -def test_dygraph_layer(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_dygraph_layer( + place, input_np, label_np, reduction='mean', weight_np=None +): paddle.disable_static(place) if weight_np is not None: weight = paddle.to_tensor(weight_np) @@ -118,25 +106,22 @@ def test_dygraph_layer(place, return dy_result -def test_dygraph_functional(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_dygraph_functional( + place, input_np, label_np, reduction='mean', weight_np=None +): paddle.disable_static(place) input = paddle.to_tensor(input_np) label = paddle.to_tensor(label_np) if weight_np is not None: weight = paddle.to_tensor(weight_np) - dy_res = paddle.nn.functional.binary_cross_entropy(input, - label, - weight=weight, - reduction=reduction) + dy_res = paddle.nn.functional.binary_cross_entropy( + input, label, weight=weight, reduction=reduction + ) else: - dy_res = paddle.nn.functional.binary_cross_entropy(input, - label, - reduction=reduction) + dy_res = paddle.nn.functional.binary_cross_entropy( + input, label, reduction=reduction + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result @@ -144,11 +129,19 @@ def test_dygraph_functional(place, def calc_bceloss(input_np, label_np, reduction='mean', weight_np=None): if weight_np is None: - expected = -1 * (label_np * np.log(input_np) + - (1. - label_np) * np.log(1. - input_np)) + expected = -1 * ( + label_np * np.log(input_np) + + (1.0 - label_np) * np.log(1.0 - input_np) + ) else: - expected = -1 * weight_np * (label_np * np.log(input_np) + - (1. - label_np) * np.log(1. - input_np)) + expected = ( + -1 + * weight_np + * ( + label_np * np.log(input_np) + + (1.0 - label_np) * np.log(1.0 - input_np) + ) + ) if reduction == 'mean': expected = np.mean(expected) @@ -161,7 +154,6 @@ def calc_bceloss(input_np, label_np, reduction='mean', weight_np=None): class TestBCELoss(unittest.TestCase): - def test_BCELoss(self): input_np = np.random.uniform(0.1, 0.8, size=(20, 30)).astype(np.float32) label_np = np.random.randint(0, 2, size=(20, 30)).astype(np.float32) @@ -171,87 +163,88 @@ def test_BCELoss(self): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - static_result = test_static_layer(place, input_np, label_np, - reduction) - dy_result = test_dygraph_layer(place, input_np, label_np, - reduction) + static_result = test_static_layer( + place, input_np, label_np, reduction + ) + dy_result = test_dygraph_layer( + place, input_np, label_np, reduction + ) expected = calc_bceloss(input_np, label_np, reduction) np.testing.assert_allclose(static_result, expected, rtol=1e-6) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected, rtol=1e-6) static_functional = test_static_functional( - place, input_np, label_np, reduction) - dy_functional = test_dygraph_functional(place, input_np, - label_np, reduction) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-6) + place, input_np, label_np, reduction + ) + dy_functional = test_dygraph_functional( + place, input_np, label_np, reduction + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-6 + ) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected, rtol=1e-6) def test_BCELoss_weight(self): - input_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float32) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float32) + input_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float32 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float32 + ) weight_np = np.random.random(size=(3, 4, 10)).astype(np.float32) - place = fluid.NPUPlace( - 0) if fluid.core.is_compiled_with_npu() else fluid.CPUPlace() + place = ( + fluid.NPUPlace(0) + if fluid.core.is_compiled_with_npu() + else fluid.CPUPlace() + ) for reduction in ['sum', 'mean', 'none']: - static_result = test_static_layer(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - dy_result = test_dygraph_layer(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - expected = calc_bceloss(input_np, - label_np, - reduction, - weight_np=weight_np) + static_result = test_static_layer( + place, input_np, label_np, reduction, weight_np=weight_np + ) + dy_result = test_dygraph_layer( + place, input_np, label_np, reduction, weight_np=weight_np + ) + expected = calc_bceloss( + input_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_result, expected, rtol=1e-6) np.testing.assert_allclose(static_result, dy_result, rtol=1e-6) np.testing.assert_allclose(dy_result, expected, rtol=1e-6) - static_functional = test_static_functional(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - dy_functional = test_dygraph_functional(place, - input_np, - label_np, - reduction, - weight_np=weight_np) + static_functional = test_static_functional( + place, input_np, label_np, reduction, weight_np=weight_np + ) + dy_functional = test_dygraph_functional( + place, input_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_functional, expected, rtol=1e-6) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-6) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-6 + ) np.testing.assert_allclose(dy_functional, expected, rtol=1e-6) def test_BCELoss_error(self): paddle.disable_static(paddle.NPUPlace(0)) - self.assertRaises(ValueError, - paddle.nn.loss.BCELoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, paddle.nn.loss.BCELoss, reduction="unsupport reduction" + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([[0.0, 1.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.binary_cross_entropy, - input=input, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.binary_cross_entropy, + input=input, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() def bce_loss(input, label): - return -1 * (label * np.log(input) + (1. - label) * np.log(1. - input)) + return -1 * (label * np.log(input) + (1.0 - label) * np.log(1.0 - input)) class TestBceLossOp(OpTest): - def setUp(self): self.set_npu() self.init_test_case() @@ -278,13 +271,11 @@ def init_test_case(self): class TestBceLossOpCase1(OpTest): - def init_test_cast(self): self.shape = [2, 3, 4, 5] class TestBceLossOpCase2(OpTest): - def init_test_cast(self): self.shape = [2, 3, 20] diff --git a/python/paddle/fluid/tests/unittests/npu/test_beam_search_decode_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_beam_search_decode_op_npu.py index f00fc60fb42ec..7f5588f34aca3 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_beam_search_decode_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_beam_search_decode_op_npu.py @@ -42,28 +42,41 @@ def test_get_set(self): # beam_size = 2, end_id = 1 # start with start_id [ - self.append_lod_tensor(array, [[0, 1, 2], [0, 1, 2]], - np.array([0, 0], dtype=dtype)) + self.append_lod_tensor( + array, [[0, 1, 2], [0, 1, 2]], np.array([0, 0], dtype=dtype) + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 1, 2], [0, 2, 4]], - np.array([2, 3, 4, 5], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 1, 2], [0, 2, 4]], + np.array([2, 3, 4, 5], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 2, 4], [0, 2, 2, 4, 4]], - np.array([3, 1, 5, 4], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 2, 4], [0, 2, 2, 4, 4]], + np.array([3, 1, 5, 4], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 2, 4], [0, 1, 2, 3, 4]], - np.array([1, 1, 3, 5], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 2, 4], [0, 1, 2, 3, 4]], + np.array([1, 1, 3, 5], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 2, 4], [0, 0, 0, 2, 2]], - np.array([5, 1], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 2, 4], [0, 0, 0, 2, 2]], + np.array([5, 1], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] @@ -89,7 +102,8 @@ def test_get_set(self): self.assertEqual(sentence_scores.lod(), expected_lod) expected_data = np.array( - [0, 2, 3, 1, 0, 2, 1, 0, 4, 5, 3, 5, 0, 4, 5, 3, 1], "int64") + [0, 2, 3, 1, 0, 2, 1, 0, 4, 5, 3, 5, 0, 4, 5, 3, 1], "int64" + ) np.testing.assert_array_equal(np.array(sentence_ids), expected_data) np.testing.assert_array_equal(np.array(sentence_scores), expected_data) diff --git a/python/paddle/fluid/tests/unittests/npu/test_beam_search_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_beam_search_op_npu.py index d185260a86a26..df340b97cdbc7 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_beam_search_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_beam_search_op_npu.py @@ -25,7 +25,6 @@ class TestBeamSearchNPUOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -35,19 +34,19 @@ def setUp(self): 'pre_ids': (self.pre_ids, self.lod), 'pre_scores': (self.pre_score, self.lod), 'ids': (self.ids, self.lod), - 'scores': (self.score, self.lod) + 'scores': (self.score, self.lod), } # The `target_lod` attribute is still based on offset self.attrs = { 'level': 0, 'beam_size': self.beam_size, 'end_id': 0, - 'is_accumulated': self.is_accumulated + 'is_accumulated': self.is_accumulated, } self.outputs = { 'selected_ids': (self.selected_ids, self.out_lod), 'selected_scores': (self.selected_scores, self.out_lod), - 'parent_idx': self.parent_idx + 'parent_idx': self.parent_idx, } def set_npu(self): @@ -57,18 +56,21 @@ def init_data(self): self.beam_size = 2 self.is_accumulated = True self.pre_ids = np.array([[1], [2], [3], [4]], dtype='int64') - self.ids = np.array([[4, 2, 5], [2, 1, 3], [3, 5, 2], [8, 2, 1]], - dtype='int64') + self.ids = np.array( + [[4, 2, 5], [2, 1, 3], [3, 5, 2], [8, 2, 1]], dtype='int64' + ) self.lod = [[2, 2], [1, 1, 1, 1]] self.out_lod = [[2, 2], [1, 1, 1, 1]] self.offset_lod = [[0, 2, 4], [0, 1, 2, 3, 4]] - self.score = np.array([ - [0.5, 0.3, 0.2], - [0.6, 0.3, 0.1], - [0.9, 0.5, 0.1], - [0.7, 0.5, 0.1], - ], - dtype='float32') + self.score = np.array( + [ + [0.5, 0.3, 0.2], + [0.6, 0.3, 0.1], + [0.9, 0.5, 0.1], + [0.7, 0.5, 0.1], + ], + dtype='float32', + ) self.pre_score = np.array([[0.1], [0.2], [0.3], [0.4]], dtype='float32') self.selected_ids = np.array([4, 2, 3, 8])[:, np.newaxis] self.selected_scores = np.array([0.5, 0.6, 0.9, 0.7])[:, np.newaxis] @@ -79,7 +81,6 @@ def test_check_output(self): class TestBeamSearchNPUOp2(TestBeamSearchNPUOp): - def init_data(self): self.beam_size = 2 self.is_accumulated = True @@ -88,13 +89,15 @@ def init_data(self): self.lod = [[2, 2], [1, 1, 1, 1]] self.out_lod = [[2, 2], [2, 0, 1, 1]] self.offset_lod = [[0, 2, 4], [0, 2, 2, 3, 4]] - self.score = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.1, 0.7], - ], - dtype='float32') + self.score = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.1, 0.7], + ], + dtype='float32', + ) self.pre_score = np.array([[0.1], [0.2], [0.3], [0.4]], dtype='float32') self.selected_ids = np.array([4, 2, 3, 1])[:, np.newaxis] self.selected_scores = np.array([0.6, 0.9, 0.9, 0.7])[:, np.newaxis] @@ -102,7 +105,6 @@ def init_data(self): class TestBeamSearchNPUOp3(TestBeamSearchNPUOp): - def init_data(self): # end_id = 0 self.beam_size = 2 @@ -112,13 +114,15 @@ def init_data(self): self.lod = [[2, 2], [1, 1, 1, 1]] self.out_lod = [[2, 2], [1, 1, 0, 2]] self.offset_lod = [[0, 2, 4], [0, 1, 2, 2, 4]] - self.score = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.6, 0.7], - ], - dtype='float32') + self.score = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.6, 0.7], + ], + dtype='float32', + ) self.pre_score = np.array([[0.1], [1.2], [0.5], [0.4]], dtype='float32') self.selected_ids = np.array([2, 0, 8, 1])[:, np.newaxis] self.selected_scores = np.array([0.9, 1.2, 0.6, 0.7])[:, np.newaxis] @@ -126,7 +130,6 @@ def init_data(self): class TestBeamSearchNPUOp4(TestBeamSearchNPUOp): - def init_data(self): # is_accumulated = False self.beam_size = 2 @@ -136,22 +139,24 @@ def init_data(self): self.lod = [[2, 2], [1, 1, 1, 1]] self.out_lod = [[2, 2], [0, 2, 1, 1]] self.offset_lod = [[0, 2, 4], [0, 0, 2, 3, 4]] - self.score = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.1, 0.7], - ], - dtype='float32') + self.score = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.1, 0.7], + ], + dtype='float32', + ) self.pre_score = np.array([[0.1], [2.2], [0.3], [0.4]], dtype='float32') self.selected_ids = np.array([7, 3, 3, 1])[:, np.newaxis] - self.selected_scores = np.array([1.50685, 0.996027, 0.194639, - 0.043325])[:, np.newaxis] + self.selected_scores = np.array( + [1.50685, 0.996027, 0.194639, 0.043325] + )[:, np.newaxis] self.parent_idx = np.array([1, 1, 2, 3]) class TestBeamSearchNPUOp5(TestBeamSearchNPUOp): - def init_data(self): # beam_size = 1 self.beam_size = 1 @@ -161,13 +166,15 @@ def init_data(self): self.lod = [[1, 1, 1, 1], [1, 1, 1, 1]] self.out_lod = [[1, 1, 1, 1], [1, 1, 1, 1]] self.offset_lod = [[0, 1, 2, 3, 4], [0, 1, 2, 3, 4]] - self.score = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.1, 0.7], - ], - dtype='float32') + self.score = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.1, 0.7], + ], + dtype='float32', + ) self.pre_score = np.array([[0.1], [0.2], [0.3], [0.4]], dtype='float32') self.selected_ids = np.array([2, 7, 3, 1])[:, np.newaxis] self.selected_scores = np.array([0.9, 0.5, 0.9, 0.7])[:, np.newaxis] diff --git a/python/paddle/fluid/tests/unittests/npu/test_bilinear_interp_v2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_bilinear_interp_v2_op_npu.py index 5bcd736f5405b..5429125b4ef35 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_bilinear_interp_v2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_bilinear_interp_v2_op_npu.py @@ -29,7 +29,6 @@ class TestBilinearInterpOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -53,7 +52,7 @@ def setUp(self): scale_w = 0 if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: scale_h = scale_w = float(self.scale) if isinstance(self.scale, list) and len(self.scale) == 1: scale_w = scale_h = self.scale[0] @@ -66,10 +65,18 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, scale_w, scale_h, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode, - self.data_layout) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + scale_w, + scale_h, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: @@ -83,11 +90,11 @@ def setUp(self): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: self.scale = [self.scale] if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] @@ -106,13 +113,19 @@ def test_check_grad(self): output_names = ['Out'] no_grad_set = set() cpu_place = fluid.CPUPlace() - cpu_grads = self._get_gradient(inputs_to_check, cpu_place, output_names, - no_grad_set) - npu_grads = self._get_gradient(inputs_to_check, self.place, - output_names, no_grad_set) - self._assert_is_close(cpu_grads, npu_grads, inputs_to_check, - self.max_relative_error, - "Gradient Check between places") + cpu_grads = self._get_gradient( + inputs_to_check, cpu_place, output_names, no_grad_set + ) + npu_grads = self._get_gradient( + inputs_to_check, self.place, output_names, no_grad_set + ) + self._assert_is_close( + cpu_grads, + npu_grads, + inputs_to_check, + self.max_relative_error, + "Gradient Check between places", + ) def init_test_case(self): self.interp_method = 'bilinear' @@ -127,7 +140,6 @@ def init_test_case(self): class TestBilinearInterpCaseFP16(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCaseFP16, self).init_test_case() self.dtype = 'float16' @@ -135,70 +147,63 @@ def init_test_case(self): class TestBilinearInterpCase1(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCase1, self).init_test_case() self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 class TestBilinearInterpCase2(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCase2, self).init_test_case() self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 class TestBilinearInterpCase3(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCase3, self).init_test_case() self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 class TestBilinearInterpCase4(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCase4, self).init_test_case() self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") class TestBilinearInterpCase5(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCase5, self).init_test_case() self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") class TestBilinearInterpCase6(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCase6, self).init_test_case() self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 33]).astype("int32") class TestBilinearInterpCase7(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpCase7, self).init_test_case() self.input_shape = [1, 1, 32, 64] @@ -208,81 +213,72 @@ def init_test_case(self): class TestBilinearInterpSame(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpSame, self).init_test_case() self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 class TestBilinearInterpActualShape(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpActualShape, self).init_test_case() self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") class TestBilinearInterpDataLayout(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpDataLayout, self).init_test_case() self.input_shape = [2, 5, 5, 3] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.data_layout = "NHWC" class TestBilinearInterpOtherMethod1(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 1 class TestBilinearInterpWithMethod2(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 0 class TestBilinearInterpWithMethod3(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = True self.align_mode = 0 class TestBilinearInterpScale1(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpScale1, self).init_test_case() self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 2. + self.scale = 2.0 class TestBilinearInterpScale2(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpScale2, self).init_test_case() self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 1. + self.scale = 1.0 class TestBilinearInterpZero(TestBilinearInterpOp): - def init_test_case(self): super(TestBilinearInterpZero, self).init_test_case() self.input_shape = [2, 3, 5, 7] diff --git a/python/paddle/fluid/tests/unittests/npu/test_box_coder_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_box_coder_op_npu.py index 6655c8f09f601..a8409474e066b 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_box_coder_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_box_coder_op_npu.py @@ -40,9 +40,11 @@ def box_decoder(t_box, p_box, pb_v, output_box, norm, axis=0): pb_y = pb_y.reshape(shape) if pb_v.ndim == 2: - var_shape = (1, pb_v.shape[0], - pb_v.shape[1]) if axis == 0 else (pb_v.shape[0], 1, - pb_v.shape[1]) + var_shape = ( + (1, pb_v.shape[0], pb_v.shape[1]) + if axis == 0 + else (pb_v.shape[0], 1, pb_v.shape[1]) + ) pb_v = pb_v.reshape(var_shape) if pb_v.ndim == 1: tb_x = pb_v[0] * t_box[:, :, 0] * pb_w + pb_x @@ -99,20 +101,24 @@ def batch_box_coder(p_box, pb_v, t_box, lod, code_type, norm, axis=0): cur_offset = 0 for i in range(len(lod)): - if (code_type == "encode_center_size"): - box_encoder(t_box[cur_offset:(cur_offset + lod[i]), :], p_box, pb_v, - output_box[cur_offset:(cur_offset + lod[i]), :, :], - norm) - elif (code_type == "decode_center_size"): + if code_type == "encode_center_size": + box_encoder( + t_box[cur_offset : (cur_offset + lod[i]), :], + p_box, + pb_v, + output_box[cur_offset : (cur_offset + lod[i]), :, :], + norm, + ) + elif code_type == "decode_center_size": box_decoder(t_box, p_box, pb_v, output_box, norm, axis) cur_offset += lod[i] return output_box -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestBoxCoderOp(OpTest): - def setUp(self): self.op_type = "box_coder" self.set_npu() @@ -143,10 +149,10 @@ def set_init_config(self): def set_inputs(self): self.inputs = {} - assert (self.code_type in ['decode_center_size', 'encode_center_size']) - assert (self.axis in [0, 1]) + assert self.code_type in ['decode_center_size', 'encode_center_size'] + assert self.axis in [0, 1] if self.code_type == 'decode_center_size': - assert (not self.use_variance or not self.without_prior_box_var) + assert not self.use_variance or not self.without_prior_box_var self.prior_box = np.random.random((self.M, 4)).astype(self.dtype) @@ -156,50 +162,60 @@ def set_inputs(self): if self.without_prior_box_var: self.prior_box_var = np.ones((self.M, 4)).astype(self.dtype) else: - self.prior_box_var = np.random.random( - (self.M, 4)).astype(self.dtype) + self.prior_box_var = np.random.random((self.M, 4)).astype( + self.dtype + ) if self.axis == 0: - self.target_box = np.random.random( - (self.N, self.M, 4)).astype(self.dtype) + self.target_box = np.random.random((self.N, self.M, 4)).astype( + self.dtype + ) else: - self.target_box = np.random.random( - (self.M, self.N, 4)).astype(self.dtype) + self.target_box = np.random.random((self.M, self.N, 4)).astype( + self.dtype + ) self.inputs['PriorBox'] = self.prior_box self.inputs['TargetBox'] = self.target_box - if (not self.use_variance and not self.without_prior_box_var): + if not self.use_variance and not self.without_prior_box_var: self.inputs['PriorBoxVar'] = self.prior_box_var else: - #encode_center_size + # encode_center_size self.prior_box = np.random.random((self.M, 4)).astype(self.dtype) if self.use_variance: self.prior_box_var = np.random.random(4).astype(self.dtype) else: - self.prior_box_var = np.random.random( - (self.M, 4)).astype(self.dtype) + self.prior_box_var = np.random.random((self.M, 4)).astype( + self.dtype + ) self.target_box = np.random.random((self.N, 4)).astype(self.dtype) self.inputs['PriorBox'] = self.prior_box - #self.inputs['PriorBoxVar'] = self.prior_box_var + # self.inputs['PriorBoxVar'] = self.prior_box_var self.inputs['TargetBox'] = (self.target_box, self.lod) - if (not self.use_variance): + if not self.use_variance: self.inputs['PriorBoxVar'] = self.prior_box_var def set_attrs(self): self.attrs = { 'code_type': self.code_type, - 'box_normalized': self.box_normalized + 'box_normalized': self.box_normalized, } if self.use_variance: self.attrs['variance'] = self.prior_box_var.astype( - np.float64).flatten() + np.float64 + ).flatten() if self.axis != 0: self.attrs['axis'] = self.axis def set_outputs(self): - output_box = batch_box_coder(self.prior_box, self.prior_box_var, - self.target_box, self.lod[0], - self.code_type, self.box_normalized, - self.axis) + output_box = batch_box_coder( + self.prior_box, + self.prior_box_var, + self.target_box, + self.lod[0], + self.code_type, + self.box_normalized, + self.axis, + ) self.outputs = {'OutputBox': output_box.astype(self.dtype)} def test_check_output(self): @@ -207,7 +223,6 @@ def test_check_output(self): class TestBoxCoderOpWithoutBoxVar(TestBoxCoderOp): - def set_init_config(self): super(TestBoxCoderOpWithoutBoxVar, self).set_init_config() self.without_prior_box_var = True @@ -215,7 +230,6 @@ def set_init_config(self): class TestBoxCoderOpWithLoD(TestBoxCoderOp): - def set_init_config(self): super(TestBoxCoderOpWithLoD, self).set_init_config() self.M = 20 @@ -226,28 +240,24 @@ def set_init_config(self): class TestBoxCoderOpWithLoDWithVariance(TestBoxCoderOpWithLoD): - def set_init_config(self): super(TestBoxCoderOpWithLoDWithVariance, self).set_init_config() self.use_variance = True class TestBoxCoderOpWithAxis(TestBoxCoderOp): - def set_init_config(self): super(TestBoxCoderOpWithAxis, self).set_init_config() self.axis = 1 class TestBoxCoderOpWithVariance(TestBoxCoderOp): - def set_init_config(self): super(TestBoxCoderOpWithVariance, self).set_init_config() self.use_variance = True class TestBoxCoderOpFP16(TestBoxCoderOp): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/npu/test_c_embedding_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_c_embedding_op_npu.py index fb9447cc2d4e3..eaceefe7e6d93 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_c_embedding_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_c_embedding_op_npu.py @@ -21,7 +21,11 @@ import paddle import paddle.fluid as fluid import paddle.fluid.core as core -from paddle.fluid.tests.unittests.c_embedding_op_base import TestCEmbeddingCPU, TestCEmbeddingOpBase, TestCEmbeddingOpFP32 +from paddle.fluid.tests.unittests.c_embedding_op_base import ( + TestCEmbeddingCPU, + TestCEmbeddingOpBase, + TestCEmbeddingOpFP32, +) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/npu/test_c_identity_npu.py b/python/paddle/fluid/tests/unittests/npu/test_c_identity_npu.py index 5701739c883be..381fd5baa4632 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_c_identity_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_c_identity_npu.py @@ -23,15 +23,14 @@ class TestIdentityOp(TestDistBase): - def _setup_config(self): pass def test_identity(self, col_type="identity"): dist_env = os.environ - self.check_with_place("collective_identity_op_npu.py", - col_type, - need_envs=dist_env) + self.check_with_place( + "collective_identity_op_npu.py", col_type, need_envs=dist_env + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_cast_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_cast_op_npu.py index 4b3e877fef12c..dfc651074632a 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_cast_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_cast_op_npu.py @@ -28,7 +28,6 @@ @skip_check_grad_ci(reason="[skip NPU cast grad check] not implemented yet.") class TestCast1(OpTest): - def setUp(self): self.set_npu() self.op_type = "cast" @@ -40,7 +39,7 @@ def setUp(self): self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP32), - 'out_dtype': int(core.VarDesc.VarType.FP16) + 'out_dtype': int(core.VarDesc.VarType.FP16), } def set_npu(self): @@ -52,7 +51,6 @@ def test_check_output(self): @skip_check_grad_ci(reason="[skip NPU cast grad check] not implemented yet.") class TestCast2(OpTest): - def setUp(self): self.set_npu() self.op_type = "cast" @@ -64,7 +62,7 @@ def setUp(self): self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP16), - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } def set_npu(self): @@ -76,7 +74,6 @@ def test_check_output(self): @skip_check_grad_ci(reason="[skip NPU cast grad check] not implemented yet.") class TestCast3(OpTest): - def setUp(self): self.set_npu() self.op_type = "cast" @@ -88,7 +85,7 @@ def setUp(self): self.attrs = { 'in_dtype': int(core.VarDesc.VarType.INT32), - 'out_dtype': int(core.VarDesc.VarType.INT32) + 'out_dtype': int(core.VarDesc.VarType.INT32), } def set_npu(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_clip_by_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_clip_by_norm_op_npu.py index a1c397796cc0f..bff2e674e5425 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_clip_by_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_clip_by_norm_op_npu.py @@ -26,7 +26,6 @@ class TestClipByNormOp(OpTest): - def setUp(self): self.set_npu() self.max_relative_error = 0.006 @@ -55,7 +54,7 @@ def test_check_output(self): self.check_output_with_place(self.place) def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1.0 def init_dtype(self): @@ -63,28 +62,24 @@ def init_dtype(self): class TestCase1(TestClipByNormOp): - def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1e20 class TestCase2(TestClipByNormOp): - def initTestCase(self): self.shape = (16, 16) self.max_norm = 0.1 class TestCase3(TestClipByNormOp): - def initTestCase(self): self.shape = (4, 8, 16) self.max_norm = 1.0 class TestClipByNormOpFp16(TestClipByNormOp): - def init_dtype(self): self.dtype = np.float16 @@ -93,21 +88,18 @@ def test_check_output(self): class TestClipByNormOpFp16Case1(TestClipByNormOpFp16): - def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1e20 class TestClipByNormOpFp16Case2(TestClipByNormOpFp16): - def initTestCase(self): self.shape = (16, 16) self.max_norm = 0.1 class TestClipByNormOpFp16Case3(TestClipByNormOpFp16): - def initTestCase(self): self.shape = (4, 8, 16) self.max_norm = 1.0 diff --git a/python/paddle/fluid/tests/unittests/npu/test_clip_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_clip_op_npu.py index 3c6b6cb8949de..8b13546d9a285 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_clip_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_clip_op_npu.py @@ -24,7 +24,6 @@ class TestClipOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -75,7 +74,6 @@ def initTestCase(self): class TestCase1(TestClipOp): - def initTestCase(self): self.shape = (8, 16, 8) self.max = 0.7 @@ -83,7 +81,6 @@ def initTestCase(self): class TestCase2(TestClipOp): - def initTestCase(self): self.shape = (8, 16) self.max = 1.0 @@ -91,7 +88,6 @@ def initTestCase(self): class TestCase3(TestClipOp): - def initTestCase(self): self.shape = (4, 8, 16) self.max = 0.7 @@ -99,7 +95,6 @@ def initTestCase(self): class TestCase4(TestClipOp): - def initTestCase(self): self.shape = (4, 8, 8) self.max = 0.7 @@ -109,7 +104,6 @@ def initTestCase(self): class TestCase5(TestClipOp): - def initTestCase(self): self.shape = (4, 8, 16) self.max = 0.5 @@ -117,7 +111,6 @@ def initTestCase(self): class TestClipOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -137,7 +130,6 @@ def test_dtype(): class TestClipAPI(unittest.TestCase): - def _executed_api(self, x, min=None, max=None): return paddle.clip(x, min, max) @@ -149,8 +141,11 @@ def test_clip(self): min = fluid.data(name='min', shape=[1], dtype='float32') max = fluid.data(name='max', shape=[1], dtype='float32') - place = fluid.NPUPlace( - 0) if fluid.core.is_compiled_with_npu() else fluid.CPUPlace() + place = ( + fluid.NPUPlace(0) + if fluid.core.is_compiled_with_npu() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) out_1 = self._executed_api(images, min=min, max=max) @@ -159,7 +154,7 @@ def test_clip(self): out_4 = self._executed_api(images, max=0.7) out_5 = self._executed_api(images, min=min) out_6 = self._executed_api(images, max=max) - out_7 = self._executed_api(images, max=-1.) + out_7 = self._executed_api(images, max=-1.0) out_8 = self._executed_api(images) res1, res2, res3, res4, res5, res6, res7, res8 = exe.run( @@ -167,9 +162,10 @@ def test_clip(self): feed={ "image": data, "min": np.array([0.2]).astype('float32'), - "max": np.array([0.8]).astype('float32') + "max": np.array([0.8]).astype('float32'), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8], + ) np.testing.assert_allclose(res1, data.clip(0.2, 0.8)) np.testing.assert_allclose(res2, data.clip(0.2, 0.9)) @@ -183,8 +179,11 @@ def test_clip(self): def test_clip_dygraph(self): paddle.disable_static() - place = fluid.NPUPlace( - 0) if fluid.core.is_compiled_with_npu() else fluid.CPUPlace() + place = ( + fluid.NPUPlace(0) + if fluid.core.is_compiled_with_npu() + else fluid.CPUPlace() + ) paddle.disable_static(place) data_shape = [1, 9, 9, 4] data = np.random.random(data_shape).astype('float32') @@ -212,7 +211,6 @@ def test_errors(self): class TestInplaceClipAPI(TestClipAPI): - def _executed_api(self, x, min=None, max=None): return x.clip_(min, max) diff --git a/python/paddle/fluid/tests/unittests/npu/test_coalesce_tensor_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_coalesce_tensor_op_npu.py index ec257b84d4375..8f6ef52107048 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_coalesce_tensor_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_coalesce_tensor_op_npu.py @@ -28,7 +28,6 @@ class TestAllocContinuousSpace(OpTest): - def setUp(self): self.__class__.use_npu = True self.op_type = "coalesce_tensor" @@ -39,7 +38,8 @@ def setUp(self): self.set_constant = attrs["set_constant"] self.Inputs = self.init_input() self.Outputs, self.FusedOutput = self.init_output( - self.Inputs, self.set_constant, self.constant) + self.Inputs, self.set_constant, self.constant + ) self.inputs = {'Input': self.Inputs} self.attrs = attrs self.outputs = {'Output': self.Outputs, 'FusedOutput': self.FusedOutput} @@ -59,7 +59,7 @@ def init_attr(self): "set_constant": False, "constant": 0.0, "use_align": True, - "dtype": self.fluid_dtype + "dtype": self.fluid_dtype, } def init_output(self, input_list, set_constant, constant): @@ -85,7 +85,6 @@ def test_check_output(self): class TestAllocContinuousSpace2(TestAllocContinuousSpace): - def init_attr(self): return { "copy_data": True, @@ -93,7 +92,7 @@ def init_attr(self): "constant": 0.5, "use_align": True, "dtype": self.fluid_dtype, - "user_defined_size_of_dtype": 2 + "user_defined_size_of_dtype": 2, } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_collective_base_npu.py b/python/paddle/fluid/tests/unittests/npu/test_collective_base_npu.py index fbbb64aaf078b..6b7e056a7f86a 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_collective_base_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_collective_base_npu.py @@ -29,10 +29,10 @@ class TestCollectiveRunnerBase(object): - def get_model(self, train_prog, startup_prog): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def wait_server_ready(self, endpoints): assert not isinstance(endpoints, str) @@ -41,13 +41,15 @@ def wait_server_ready(self, endpoints): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, - 1) + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT, 1 + ) result = sock.connect_ex((ip_port[0], int(ip_port[1]))) if result != 0: @@ -55,43 +57,50 @@ def wait_server_ready(self, endpoints): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + - str(not_ready_endpoints) + "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: break + # endpoints should be ["ip1:port1","ip2:port2"] -#endpoints should be ["ip1:port1","ip2:port2"] - - def initCommunicator(self, program, rank, nranks, wait_port, - current_endpoint, endpoints): + def initCommunicator( + self, program, rank, nranks, wait_port, current_endpoint, endpoints + ): other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) if rank == 0 and wait_port: self.wait_server_ready(other_endpoints) block = program.global_block() - hccl_id_var = block.create_var(name=nameGen.generate('hccl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_hccl_id', - inputs={}, - outputs={'Out': hccl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) - block.append_op(type='c_comm_init_hccl', - inputs={'X': hccl_id_var}, - outputs={}, - attrs={ - 'rank': rank, - 'ring_id': self.global_ring_id, - 'device_id': int(os.getenv("FLAGS_selected_npus")), - 'rank_ids': nranks - }) + hccl_id_var = block.create_var( + name=nameGen.generate('hccl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_hccl_id', + inputs={}, + outputs={'Out': hccl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) + block.append_op( + type='c_comm_init_hccl', + inputs={'X': hccl_id_var}, + outputs={}, + attrs={ + 'rank': rank, + 'ring_id': self.global_ring_id, + 'device_id': int(os.getenv("FLAGS_selected_npus")), + 'rank_ids': nranks, + }, + ) def run_trainer(self, args): train_prog = fluid.Program() @@ -100,8 +109,9 @@ def run_trainer(self, args): rank = args["trainerid"] current_endpoint = args["currentendpoint"] nranks = 2 - self.initCommunicator(startup_prog, rank, nranks, True, - current_endpoint, endpoints) + self.initCommunicator( + startup_prog, rank, nranks, True, current_endpoint, endpoints + ) self.rank = rank result = self.get_model(train_prog, startup_prog) device_id = int(os.getenv("FLAGS_selected_npus", "0")) @@ -110,9 +120,9 @@ def run_trainer(self, args): exe.run(startup_prog) np.random.seed(os.getpid()) indata = np.random.random((10, 1000)) - out = exe.run(train_prog, - feed={'tindata': indata}, - fetch_list=[result.name]) + out = exe.run( + train_prog, feed={'tindata': indata}, fetch_list=[result.name] + ) sys.stdout.buffer.write(pickle.dumps(out)) @@ -134,19 +144,20 @@ def runtime_main(test_class, col_type, sub_type): class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -159,7 +170,7 @@ def __free_port(): def _run_cluster(self, model_file, envs): worker_endpoints = self._ps_endpoints.split(",") w0_ep, w1_ep = worker_endpoints - #print("w0_ep:",w0_ep," w1_ep:",w1_ep) + # print("w0_ep:",w0_ep," w1_ep:",w1_ep) env0 = { "FLAGS_selected_npus": "0", "PADDLE_TRAINER_ID": "0", @@ -175,7 +186,7 @@ def _run_cluster(self, model_file, envs): "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) tr_cmd = "%s %s" @@ -183,16 +194,20 @@ def _run_cluster(self, model_file, envs): tr1_cmd = tr_cmd % (self._python_interp, model_file) tr0_pipe = open("/tmp/tr0_err.log", "wb") tr1_pipe = open("/tmp/tr1_err.log", "wb") - #print(tr0_cmd) - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) - - tr1_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + # print(tr0_cmd) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) + + tr1_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -201,8 +216,12 @@ def _run_cluster(self, model_file, envs): # close trainer file tr0_pipe.close() tr1_pipe.close() - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) def check_with_place(self, model_file, col_type, need_envs={}): diff --git a/python/paddle/fluid/tests/unittests/npu/test_collective_process_group_hccl.py b/python/paddle/fluid/tests/unittests/npu/test_collective_process_group_hccl.py index f26d33840ae88..181ba75c11b7e 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_collective_process_group_hccl.py +++ b/python/paddle/fluid/tests/unittests/npu/test_collective_process_group_hccl.py @@ -20,7 +20,6 @@ class TestProcessGroup(TestMultipleGpus): - def test_process_group_nccl(self): self.run_mnist_2gpu('process_group_hccl.py') diff --git a/python/paddle/fluid/tests/unittests/npu/test_compare_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_compare_op_npu.py index b31b805fb771f..4fd0be14c12b5 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_compare_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_compare_op_npu.py @@ -24,9 +24,7 @@ def create_test_class(op_type, typename, callback): - class Cls(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -77,46 +75,42 @@ def test_dynamic_api(self): def test_broadcast_api_1(self): paddle.enable_static() with program_guard(Program(), Program()): - x = paddle.static.data(name='x', - shape=[1, 2, 1, 3], - dtype=typename) - y = paddle.static.data(name='y', - shape=[1, 2, 3], - dtype=typename) + x = paddle.static.data( + name='x', shape=[1, 2, 1, 3], dtype=typename + ) + y = paddle.static.data( + name='y', shape=[1, 2, 3], dtype=typename + ) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) exe = paddle.static.Executor(self.place) input_x = np.arange(1, 7).reshape((1, 2, 1, 3)).astype(typename) input_y = np.arange(0, 6).reshape((1, 2, 3)).astype(typename) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) @unittest.skipIf(typename == 'float16', "float16 is not supported now") def test_broadcast_api_2(self): paddle.enable_static() with program_guard(Program(), Program()): - x = paddle.static.data(name='x', - shape=[1, 2, 3], - dtype=typename) - y = paddle.static.data(name='y', - shape=[1, 2, 1, 3], - dtype=typename) + x = paddle.static.data( + name='x', shape=[1, 2, 3], dtype=typename + ) + y = paddle.static.data( + name='y', shape=[1, 2, 1, 3], dtype=typename + ) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) exe = paddle.static.Executor(self.place) input_x = np.arange(0, 6).reshape((1, 2, 3)).astype(typename) input_y = np.arange(1, 7).reshape((1, 2, 1, 3)).astype(typename) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) @unittest.skipIf(typename == 'float16', "float16 is not supported now") @@ -131,11 +125,9 @@ def test_broadcast_api_3(self): input_x = np.arange(0, 5).reshape((5)).astype(typename) input_y = np.array([5, 3, 2]).reshape((3, 1)).astype(typename) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) @unittest.skipIf(typename == 'float16', "float16 is not supported now") diff --git a/python/paddle/fluid/tests/unittests/npu/test_concat_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_concat_op_npu.py index a9c43a868f564..301ce9c47736f 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_concat_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_concat_op_npu.py @@ -26,7 +26,6 @@ class TestConcatOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "concat" @@ -43,8 +42,9 @@ def setUp(self): self.actual_axis = self.axis self.outputs = { - 'Out': - np.concatenate((self.x0, self.x1, self.x2), axis=self.actual_axis) + 'Out': np.concatenate( + (self.x0, self.x1, self.x2), axis=self.actual_axis + ) } def set_npu(self): @@ -69,7 +69,6 @@ def init_test_data(self): class TestConcatOp2(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((2, 3, 4, 5)).astype(self.dtype) self.x1 = np.random.random((2, 3, 4, 5)).astype(self.dtype) @@ -78,9 +77,9 @@ def init_test_data(self): @skip_check_grad_ci( - reason="The function 'check_grad' for large inputs is too slow.") + reason="The function 'check_grad' for large inputs is too slow." +) class TestConcatOp3(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((1, 256, 170, 256)).astype(self.dtype) self.x1 = np.random.random((1, 128, 170, 256)).astype(self.dtype) @@ -92,11 +91,9 @@ def test_check_grad(self): @skip_check_grad_ci( - reason= - "This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." + reason="This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." ) class TestConcatOp4(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((2, 3, 4, 5)).astype(self.dtype) self.x1 = np.random.random((2, 3, 4, 5)).astype(self.dtype) @@ -108,7 +105,6 @@ def test_check_grad(self): class TestConcatOp5(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((5, 1, 4, 5)).astype(self.dtype) self.x1 = np.random.random((5, 2, 4, 5)).astype(self.dtype) @@ -116,11 +112,9 @@ def init_test_data(self): self.axis = -3 -#----------------Concat Fp16---------------- +# ----------------Concat Fp16---------------- def create_test_fp16(parent): - class TestConcatFp16(parent): - def init_dtype(self): self.dtype = np.float16 @@ -136,11 +130,9 @@ def init_dtype(self): create_test_fp16(TestConcatOp5) -#----------------Concat Int64---------------- +# ----------------Concat Int64---------------- def create_test_int64(parent): - class TestConcatInt64(parent): - def init_dtype(self): self.dtype = np.int64 @@ -179,9 +171,9 @@ def set_program(self, use_fluid_api): with fluid.program_guard(self.program): input = fluid.layers.assign(self.x) tensor_array = fluid.layers.create_array(dtype='float32') - zero = fluid.layers.fill_constant(shape=[1], - value=0, - dtype="int64") + zero = fluid.layers.fill_constant( + shape=[1], value=0, dtype="int64" + ) for i in range(self.iter_num): fluid.layers.array_write(input, zero + i, tensor_array) @@ -217,7 +209,8 @@ def _run_static_mode(self, use_fluid_api): exe = fluid.Executor(self.place) res = exe.run(self.program, fetch_list=self.out_var) np.testing.assert_allclose( - res[0], np.concatenate([self.x] * self.iter_num, axis=self.axis)) + res[0], np.concatenate([self.x] * self.iter_num, axis=self.axis) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_depthwise_conv_npu.py b/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_depthwise_conv_npu.py index 0161f6bcdce70..537912edb4606 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_depthwise_conv_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_depthwise_conv_npu.py @@ -31,9 +31,7 @@ def create_test_channel_last_class(parent): - class TestChannelLastCase(parent): - def init_data_format(self): self.data_format = "NHWC" @@ -47,9 +45,7 @@ def init_test_case_2(self): def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" @@ -60,9 +56,7 @@ def init_paddings(self): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" @@ -73,9 +67,7 @@ def init_paddings(self): def create_test_fp16_class(parent): - class TestFp16Case(parent): - def init_data_type(self): self.dtype = np.float16 @@ -85,7 +77,6 @@ def init_data_type(self): class TestDepthwiseConvNPU(OpTest): - def setUp(self): self.set_npu() self.op_type = "depthwise_conv2d" @@ -97,21 +88,26 @@ def setUp(self): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.uniform(-1, 1, self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input, filter, self.groups, - conv2d_param, "EXPLICIT", - self.data_format) + output, _, _, _, _ = conv2d_forward_naive( + input, + filter, + self.groups, + conv2d_param, + "EXPLICIT", + self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { @@ -143,41 +139,59 @@ def test_check_output(self): def test_check_grad(self): if self.dilations[0] == 1 and self.dilations[1] == 1: if self.dtype == np.float16: - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.9) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.9, + ) else: - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Input'], - 'Output', - no_grad_set=set(['Filter']), - max_relative_error=0.9) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + no_grad_set=set(['Filter']), + max_relative_error=0.9, + ) else: - self.check_grad_with_place(self.place, ['Input'], - 'Output', - no_grad_set=set(['Filter']), - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + no_grad_set=set(['Filter']), + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): if self.dilations[0] == 1 and self.dilations[1] == 1: if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - no_grad_set=set(['Input']), - max_relative_error=0.9) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + max_relative_error=0.9, + ) else: - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - no_grad_set=set(['Input']), - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def init_data_format(self): self.data_format = "NCHW" @@ -190,7 +204,6 @@ def init_test_case_2(self): class TestDepthwiseConvNPU2(TestDepthwiseConvNPU): - def init_test_case(self): self.pad = [1, 1] self.dilations = [1, 1] @@ -203,7 +216,6 @@ def init_test_case(self): class TestDepthwiseConvNPU3(TestDepthwiseConvNPU): - def init_test_case(self): self.pad = [1, 1] self.dilations = [1, 1] @@ -216,7 +228,6 @@ def init_test_case(self): class TestDepthwiseConvNPU4(TestDepthwiseConvNPU): - def init_test_case(self): self.pad = [1, 1] self.dilations = [1, 1] @@ -229,7 +240,6 @@ def init_test_case(self): class TestDepthwiseConvNPU_Padding(OpTest): - def setUp(self): self.op_type = "depthwise_conv2d" self.dtype = np.float32 @@ -243,21 +253,25 @@ def setUp(self): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.uniform(-1, 1, self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input, filter, self.groups, - conv2d_param, - self.padding_algorithm, - self.data_format) + output, _, _, _, _ = conv2d_forward_naive( + input, + filter, + self.groups, + conv2d_param, + self.padding_algorithm, + self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { @@ -266,7 +280,7 @@ def setUp(self): 'padding_algorithm': self.padding_algorithm, 'groups': self.groups, 'dilations': self.dilations, - 'data_format': self.data_format + 'data_format': self.data_format, } self.outputs = {'Output': output} @@ -289,40 +303,58 @@ def test_check_output(self): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=1.2) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=1.2, + ) else: - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Input'], - 'Output', - max_relative_error=0.7, - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + max_relative_error=0.7, + no_grad_set=set(['Filter']), + ) else: - self.check_grad_with_place(self.place, ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - max_relative_error=0.8, - no_grad_set=set(['Input'])) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + max_relative_error=0.8, + no_grad_set=set(['Input']), + ) else: - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def init_data_format(self): self.data_format = "NCHW" @@ -339,7 +371,6 @@ def init_test_case_2(self): class TestDepthwiseConvNPU2_Padding(TestDepthwiseConvNPU_Padding): - def init_test_case(self): self.pad = [1, 1, 0, 1] self.dilations = [1, 1] @@ -356,7 +387,6 @@ def init_paddings(self): class TestDepthwiseConvNPU3_Padding(TestDepthwiseConvNPU_Padding): - def init_test_case(self): self.pad = [1, 1, 0, 1] self.dilations = [1, 1] diff --git a/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_npu.py index 974607bc25226..d30a68ac91646 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_conv2d_op_npu.py @@ -28,9 +28,7 @@ def create_test_channel_last_class(parent): - class TestChannelLastCase(parent): - def init_data_format(self): self.data_format = "NHWC" @@ -44,9 +42,7 @@ def init_test_case_2(self): def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" @@ -57,9 +53,7 @@ def init_paddings(self): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" @@ -70,9 +64,7 @@ def init_paddings(self): def create_test_fp16_class(parent): - class TestFp16Case(parent): - def init_dtype(self): self.dtype = np.float16 @@ -82,7 +74,6 @@ def init_dtype(self): class TestConv2DOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -104,22 +95,24 @@ def setUp(self): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.uniform(-1, 1, self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input, - filter, - self.groups, - conv2d_param, - data_format=self.data_format) + output, _, _, _, _ = conv2d_forward_naive( + input, + filter, + self.groups, + conv2d_param, + data_format=self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -134,24 +127,33 @@ def test_check_output(self): self.check_output_with_place(fluid.NPUPlace(0), atol=1e-2) def test_check_grad(self): - self.check_grad_with_place(fluid.NPUPlace(0), {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + fluid.NPUPlace(0), + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): - self.check_grad_with_place(fluid.NPUPlace(0), ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + fluid.NPUPlace(0), + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): - self.check_grad_with_place(fluid.NPUPlace(0), ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + fluid.NPUPlace(0), + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def init_test_case(self): self.pad = [0, 0] @@ -169,7 +171,6 @@ def init_group(self): class TestWithPad(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -180,7 +181,6 @@ def init_test_case(self): class TestWithStride(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -191,7 +191,6 @@ def init_test_case(self): class TestWithGroup(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -203,7 +202,6 @@ def init_test_case(self): class TestWith1x1(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -220,7 +218,6 @@ def init_group(self): class TestWithDepthWise5x5(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -234,7 +231,6 @@ def init_group(self): class TestWithDepthWise7x7(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -248,7 +244,6 @@ def init_group(self): class TestWithDilation(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -265,7 +260,6 @@ def init_group(self): class TestWithInput1x1Filter1x1(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -279,7 +273,6 @@ def init_group(self): class TestConv2DOp_v2(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -301,20 +294,24 @@ def setUp(self): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.uniform(-1, 1, self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input, filter, self.groups, - conv2d_param, - self.padding_algorithm, - self.data_format) + output, _, _, _, _ = conv2d_forward_naive( + input, + filter, + self.groups, + conv2d_param, + self.padding_algorithm, + self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -331,39 +328,57 @@ def test_check_output(self): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(paddle.NPUPlace(0), {'Input', 'Filter'}, - 'Output', - max_relative_error=1.1) + self.check_grad_with_place( + paddle.NPUPlace(0), + {'Input', 'Filter'}, + 'Output', + max_relative_error=1.1, + ) else: - self.check_grad_with_place(paddle.NPUPlace(0), {'Input', 'Filter'}, - 'Output', - max_relative_error=0.02, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + paddle.NPUPlace(0), + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.02, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: - self.check_grad_with_place(paddle.NPUPlace(0), ['Input'], - 'Output', - max_relative_error=0.99, - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + paddle.NPUPlace(0), + ['Input'], + 'Output', + max_relative_error=0.99, + no_grad_set=set(['Filter']), + ) else: - self.check_grad_with_place(paddle.NPUPlace(0), ['Input'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Filter']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + paddle.NPUPlace(0), + ['Input'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Filter']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: - self.check_grad_with_place(paddle.NPUPlace(0), ['Filter'], - 'Output', - max_relative_error=0.99, - no_grad_set=set(['Input'])) + self.check_grad_with_place( + paddle.NPUPlace(0), + ['Filter'], + 'Output', + max_relative_error=0.99, + no_grad_set=set(['Input']), + ) else: - self.check_grad_with_place(paddle.NPUPlace(0), ['Filter'], - 'Output', - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + paddle.NPUPlace(0), + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def init_test_case(self): self.pad = [0, 0] @@ -394,14 +409,12 @@ def init_test_case_2(self): class TestConv2DOp_AsyPadding(TestConv2DOp_v2): - def init_paddings(self): self.pad = [0, 0, 1, 2] self.padding_algorithm = "EXPLICIT" class TestWithPad_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -415,7 +428,6 @@ def init_paddings(self): class TestWithStride_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 3, 6, 6] # NCHW @@ -429,7 +441,6 @@ def init_paddings(self): class TestWithGroup_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 2] @@ -441,7 +452,6 @@ def init_test_case(self): class TestWith1x1_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -458,7 +468,6 @@ def init_paddings(self): class TestWithDepthWise3x3_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [3, 4, 10, 10] # NCHW @@ -478,7 +487,6 @@ def init_paddings(self): class TestWithDepthWise5x5_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 4, 10, 10] # NCHW @@ -495,7 +503,6 @@ def init_paddings(self): class TestWithDepthWise7x7_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 8, 10, 10] # NCHW @@ -512,7 +519,6 @@ def init_paddings(self): class TestWithDilation_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 10, 10] # NCHW @@ -532,7 +538,6 @@ def init_paddings(self): class TestWithInput1x1Filter1x1_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [100, 1, 1, 1] # NCHW diff --git a/python/paddle/fluid/tests/unittests/npu/test_conv2d_transpose_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_conv2d_transpose_op_npu.py index ff66e35b0c68c..d27c98b270bfb 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_conv2d_transpose_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_conv2d_transpose_op_npu.py @@ -29,7 +29,6 @@ class TestConv2DTransposeOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -62,15 +61,16 @@ def setUp(self): 'use_cudnn': False, 'is_test': False, 'use_mkldnn': False, - 'data_format': self.data_format + 'data_format': self.data_format, } if self.output_size is not None: self.attrs['output_size'] = self.output_size if len(self.output_padding) > 0: self.attrs['output_padding'] = self.output_padding - output = conv2dtranspose_forward_naive(input_, filter_, - self.attrs).astype(self.dtype) + output = conv2dtranspose_forward_naive( + input_, filter_, self.attrs + ).astype(self.dtype) self.outputs = {'Output': output} @@ -79,26 +79,34 @@ def test_check_output(self): def test_check_grad_no_input(self): if self.need_check_grad: - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.need_check_grad: - self.check_grad_with_place(self.place, ['Input'], - 'Output', - no_grad_set=set(['Filter']), - max_relative_error=0.006, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + no_grad_set=set(['Filter']), + max_relative_error=0.006, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad(self): if self.need_check_grad: - self.check_grad_with_place(self.place, - set(['Input', 'Filter']), - 'Output', - max_relative_error=0.02, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + set(['Input', 'Filter']), + 'Output', + max_relative_error=0.02, + numeric_place=paddle.CPUPlace(), + ) def init_test_case(self): self.pad = [0, 0] @@ -117,7 +125,6 @@ def init_dtype(self): class TestWithSymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -129,14 +136,12 @@ def init_test_case(self): class TestWithSymmetricPad_FP16(TestWithSymmetricPad): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithAsymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -148,14 +153,12 @@ def init_test_case(self): class TestWithAsymmetricPad_FP16(TestWithAsymmetricPad): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithSAMEPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [2, 1] self.dilations = [1, 2] @@ -167,14 +170,12 @@ def init_test_case(self): class TestWithSAMEPad_FP16(TestWithSAMEPad): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithVALIDPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [1, 1] self.dilations = [1, 1] @@ -186,14 +187,12 @@ def init_test_case(self): class TestWithVALIDPad_FP16(TestWithVALIDPad): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithGroups(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -205,14 +204,12 @@ def init_test_case(self): class TestWithGroups_FP16(TestWithGroups): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithStride(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -224,14 +221,12 @@ def init_test_case(self): class TestWithStride_FP16(TestWithStride): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithDilation(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -243,14 +238,12 @@ def init_test_case(self): class TestWithDilation_FP16(TestWithDilation): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithEvenUpsample(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -263,14 +256,12 @@ def init_test_case(self): class TestWithEvenUpsample_FP16(TestWithEvenUpsample): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithEvenUpsampleOutputPadding(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -283,14 +274,12 @@ def init_test_case(self): class TestWithEvenUpsampleOutputPadding_FP16(TestWithEvenUpsampleOutputPadding): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class Test_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -303,14 +292,12 @@ def init_test_case(self): class Test_NHWC_FP16(Test_NHWC): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithSymmetricPad_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -323,14 +310,12 @@ def init_test_case(self): class TestWithSymmetricPad_NHWC_FP16(TestWithSymmetricPad_NHWC): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithAsymmetricPad_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -343,14 +328,12 @@ def init_test_case(self): class TestWithAsymmetricPad_NHWC_FP16(TestWithAsymmetricPad_NHWC): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithGroups_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -363,14 +346,12 @@ def init_test_case(self): class TestWithGroups_NHWC_FP16(TestWithGroups_NHWC): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithStride_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -383,14 +364,12 @@ def init_test_case(self): class TestWithStride_NHWC_FP16(TestWithStride_NHWC): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithDilation_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -403,14 +382,12 @@ def init_test_case(self): class TestWithDilation_NHWC_FP16(TestWithDilation_NHWC): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithEvenUpsample_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -424,14 +401,12 @@ def init_test_case(self): class TestWithEvenUpsample_NHWC_FP16(TestWithEvenUpsample_NHWC): - def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestWithEvenUpsample_NHWC_output_padding(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -445,64 +420,75 @@ def init_test_case(self): class TestWithEvenUpsample_NHWC_output_padding_FP16( - TestWithEvenUpsample_NHWC_output_padding): - + TestWithEvenUpsample_NHWC_output_padding +): def init_dtype(self): self.dtype = np.float16 self.need_check_grad = False class TestConv2DTransposeAPI(unittest.TestCase): - def test_case1(self): - data1 = fluid.layers.data(name='data1', - shape=[3, 5, 5], - dtype='float32') - data2 = fluid.layers.data(name='data2', - shape=[5, 5, 3], - dtype='float32') - out1 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - data_format='NCHW') - out2 = fluid.layers.conv2d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - data_format='NHWC') - out3 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - padding=[[0, 0], [1, 1], [1, 1], - [0, 0]], - data_format='NHWC') - out4 = fluid.layers.conv2d_transpose(input=data1, - groups=3, - num_filters=6, - filter_size=3, - padding=[[0, 0], [0, 0], [2, 1], - [0, 0]], - data_format='NCHW') - out5 = fluid.layers.conv2d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - padding='SAME', - data_format='NCHW') - out6 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - padding='VALID', - data_format='NHWC') - out7 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - output_size=[7, 7], - padding=[0, 0], - data_format='NHWC') + data1 = fluid.layers.data( + name='data1', shape=[3, 5, 5], dtype='float32' + ) + data2 = fluid.layers.data( + name='data2', shape=[5, 5, 3], dtype='float32' + ) + out1 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + data_format='NCHW', + ) + out2 = fluid.layers.conv2d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + data_format='NHWC', + ) + out3 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + padding=[[0, 0], [1, 1], [1, 1], [0, 0]], + data_format='NHWC', + ) + out4 = fluid.layers.conv2d_transpose( + input=data1, + groups=3, + num_filters=6, + filter_size=3, + padding=[[0, 0], [0, 0], [2, 1], [0, 0]], + data_format='NCHW', + ) + out5 = fluid.layers.conv2d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + padding='SAME', + data_format='NCHW', + ) + out6 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + padding='VALID', + data_format='NHWC', + ) + out7 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + output_size=[7, 7], + padding=[0, 0], + data_format='NHWC', + ) data1_np = np.random.random((2, 3, 5, 5)).astype("float32") data2_np = np.random.random((2, 5, 5, 3)).astype("float32") @@ -510,13 +496,12 @@ def test_case1(self): place = core.NPUPlace(0) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "data1": data1_np, - "data2": data2_np - }, - fetch_list=[out1, out2, out3, out4, out5, out6, out7], - return_numpy=True) + results = exe.run( + fluid.default_main_program(), + feed={"data1": data1_np, "data2": data2_np}, + fetch_list=[out1, out2, out3, out4, out5, out6, out7], + return_numpy=True, + ) self.assertIsNotNone(results[0]) self.assertIsNotNone(results[1]) self.assertIsNotNone(results[2]) @@ -527,10 +512,9 @@ def test_case1(self): class TestConv2DTransposeRepr(unittest.TestCase): - def test_case(self): paddle.disable_static(paddle.NPUPlace(0)) - x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.) + x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0) conv = nn.Conv2DTranspose(4, 6, (3, 3), output_padding=1, stride=2) print(conv) y_var = conv(x_var) diff --git a/python/paddle/fluid/tests/unittests/npu/test_conv3d_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_conv3d_op_npu.py index fcbfde79e3f43..81af05b156c86 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_conv3d_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_conv3d_op_npu.py @@ -29,9 +29,7 @@ def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.pad = [0, 0, 0] self.padding_algorithm = "SAME" @@ -42,9 +40,7 @@ def init_paddings(self): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1, 1] self.padding_algorithm = "VALID" @@ -55,9 +51,7 @@ def init_paddings(self): def create_test_channel_last_class(parent): - class TestChannelLastCase(parent): - def init_data_format(self): self.data_format = "NDHWC" @@ -71,9 +65,7 @@ def init_test_case_2(self): def create_test_fp16_class(parent): - class TestFp16Case(parent): - def init_dtype(self): self.dtype = np.float16 @@ -83,7 +75,6 @@ def init_dtype(self): class TestConv3DOp(OpTest): - def setUp(self): self.op_type = "conv3d" self.set_npu() @@ -96,7 +87,7 @@ def setUp(self): conv3d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilations': self.dilations + 'dilations': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) @@ -110,14 +101,14 @@ def setUp(self): self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, 'paddings': self.pad, 'groups': self.groups, 'dilations': self.dilations, - 'data_format': self.data_format + 'data_format': self.data_format, } self.outputs = {'Output': output} @@ -128,30 +119,39 @@ def test_check_grad(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def set_npu(self): self.__class__.use_npu = True @@ -179,7 +179,6 @@ def init_test_case(self): class TestCase1(TestConv3DOp): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -193,7 +192,6 @@ def init_test_case(self): class TestConv3DOp_2(OpTest): - def setUp(self): self.op_type = "conv3d" self.set_npu() @@ -209,18 +207,23 @@ def setUp(self): conv3d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilations': self.dilations + 'dilations': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.random(self.filter_size).astype(self.dtype) - output = conv3d_forward_naive(input, filter, self.groups, conv3d_param, - self.padding_algorithm, - self.data_format).astype(self.dtype) + output = conv3d_forward_naive( + input, + filter, + self.groups, + conv3d_param, + self.padding_algorithm, + self.data_format, + ).astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -228,7 +231,7 @@ def setUp(self): 'padding_algorithm': self.padding_algorithm, 'groups': self.groups, 'dilations': self.dilations, - 'data_format': self.data_format + 'data_format': self.data_format, } self.outputs = {'Output': output} @@ -239,30 +242,39 @@ def test_check_grad(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03, - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.03, + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + numeric_place=paddle.CPUPlace(), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: return - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input']), - numeric_place=paddle.CPUPlace()) + self.check_grad_with_place( + self.place, + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + numeric_place=paddle.CPUPlace(), + ) def set_npu(self): self.__class__.use_npu = True @@ -296,7 +308,6 @@ def init_test_case_2(self): class TestConv3DOp_AsyPadding(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 2] self.input_size = [2, 3, 4, 4, 4] # NCDHW @@ -310,7 +321,6 @@ def init_paddings(self): class TestConv3DOp_DiffDataInDiffDim(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 2] self.input_size = [2, 3, 4, 5, 5] # NCDHW @@ -324,7 +334,6 @@ def init_paddings(self): class TestCase1_AsyPadding(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 1] self.input_size = [2, 3, 4, 4, 4] # NCDHW @@ -339,196 +348,228 @@ def init_paddings(self): # --------- test python API --------------- class TestConv3DAPI(unittest.TestCase): - def test_api(self): - input_NDHWC = fluid.layers.data(name="input_NDHWC", - shape=[2, 5, 5, 5, 3], - append_batch_size=False, - dtype="float32") - - input_NCDHW = fluid.layers.data(name="input_NCDHW", - shape=[2, 3, 5, 5, 3], - append_batch_size=False, - dtype="float32") - - fluid.layers.conv3d(input=input_NDHWC, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=0, - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=[1, 2, 1, 0, 1, 0], - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=[[0, 0], [0, 0], [1, 1], [1, 1], [1, 1]], - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NDHWC, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=[[0, 0], [1, 1], [1, 1], [1, 1], [0, 0]], - dilation=[1, 1, 1], - groups=1, - data_format="NDHWC") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding="SAME", - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding="VALID", - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") + input_NDHWC = fluid.layers.data( + name="input_NDHWC", + shape=[2, 5, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + input_NCDHW = fluid.layers.data( + name="input_NCDHW", + shape=[2, 3, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + fluid.layers.conv3d( + input=input_NDHWC, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=0, + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=[1, 2, 1, 0, 1, 0], + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=[[0, 0], [0, 0], [1, 1], [1, 1], [1, 1]], + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NDHWC, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=[[0, 0], [1, 1], [1, 1], [1, 1], [0, 0]], + dilation=[1, 1, 1], + groups=1, + data_format="NDHWC", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding="SAME", + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding="VALID", + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) class TestConv3DAPI_Error(unittest.TestCase): - def test_api(self): - input = fluid.layers.data(name="input", - shape=[2, 5, 5, 5, 4], - append_batch_size=False, - dtype="float32") + input = fluid.layers.data( + name="input", + shape=[2, 5, 5, 5, 4], + append_batch_size=False, + dtype="float32", + ) # ValueError: cudnn def run_1(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding=0, - dilation=1, - groups=1, - use_cudnn=[0], - data_format="NCDHW") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding=0, + dilation=1, + groups=1, + use_cudnn=[0], + data_format="NCDHW", + ) self.assertRaises(ValueError, run_1) # ValueError: data_format def run_2(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=0, - dilation=[1, 1, 1], - groups=1, - use_cudnn=False, - data_format="NCHWC") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=0, + dilation=[1, 1, 1], + groups=1, + use_cudnn=False, + data_format="NCHWC", + ) self.assertRaises(ValueError, run_2) # ValueError: padding def run_3(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding="SAMEE", - dilation=1, - groups=1, - use_cudnn=False, - data_format="NCDHW") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding="SAMEE", + dilation=1, + groups=1, + use_cudnn=False, + data_format="NCDHW", + ) self.assertRaises(ValueError, run_3) def run_4(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding=[[0, 1], [0, 0], [0, 1], [0, 1], [0, - 1]], - dilation=1, - groups=1, - use_cudnn=False, - data_format="NCDHW") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding=[[0, 1], [0, 0], [0, 1], [0, 1], [0, 1]], + dilation=1, + groups=1, + use_cudnn=False, + data_format="NCDHW", + ) self.assertRaises(ValueError, run_4) def run_5(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=0, - stride=0, - padding=[[0, 1], [0, 1], [0, 1], [0, 1], [0, - 1]], - dilation=1, - groups=1, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=0, + stride=0, + padding=[[0, 1], [0, 1], [0, 1], [0, 1], [0, 1]], + dilation=1, + groups=1, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_5) # ValueError: channel dimmention - x = fluid.layers.data(name="x", - shape=[2, 5, 5, 5, -1], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name="x", + shape=[2, 5, 5, 5, -1], + append_batch_size=False, + dtype="float32", + ) def run_6(): - fluid.layers.conv3d(input=x, - num_filters=3, - filter_size=3, - stride=1, - padding=0, - dilation=1, - groups=1, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=x, + num_filters=3, + filter_size=3, + stride=1, + padding=0, + dilation=1, + groups=1, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_6) # ValueError: groups def run_7(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding=0, - dilation=1, - groups=3, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding=0, + dilation=1, + groups=3, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_7) # ValueError: filter num def run_8(): - fluid.layers.conv3d(input=input, - num_filters=0, - filter_size=0, - stride=0, - padding=0, - dilation=0, - groups=1, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=input, + num_filters=0, + filter_size=0, + stride=0, + padding=0, + dilation=0, + groups=1, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_8) diff --git a/python/paddle/fluid/tests/unittests/npu/test_conv3d_transpose_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_conv3d_transpose_op_npu.py index da6966cf50a1e..29113544c3b12 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_conv3d_transpose_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_conv3d_transpose_op_npu.py @@ -29,9 +29,10 @@ def conv3dtranspose_forward_naive(input_, filter_, attrs): padding_algorithm = attrs['padding_algorithm'] if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if attrs['data_format'] == 'NHWC': input_ = np.transpose(input_, [0, 4, 1, 2, 3]) @@ -42,17 +43,21 @@ def conv3dtranspose_forward_naive(input_, filter_, attrs): out_c = f_out_c * groups sub_in_c = in_c // groups - stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ - 'dilations'] + stride, pad, dilations = ( + attrs['strides'], + attrs['paddings'], + attrs['dilations'], + ) def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, - kernel_size, - kernel_stride): + for input_size, filter_size, stride_size in zip( + input_shape, kernel_size, kernel_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -88,34 +93,50 @@ def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride): for i in range(in_h): for j in range(in_w): for g in range(groups): - input_masked = input_[n, - g * sub_in_c:(g + 1) * sub_in_c, - d, i, j] # (c) - input_masked = np.reshape(input_masked, - (sub_in_c, 1, 1, 1)) + input_masked = input_[ + n, g * sub_in_c : (g + 1) * sub_in_c, d, i, j + ] # (c) + input_masked = np.reshape( + input_masked, (sub_in_c, 1, 1, 1) + ) input_masked = np.tile(input_masked, (1, f_d, f_h, f_w)) for k in range(f_out_c): - tmp_out = np.sum(input_masked * - filter_[g * sub_in_c:(g + 1) * - sub_in_c, k, :, :, :], - axis=0) + tmp_out = np.sum( + input_masked + * filter_[ + g * sub_in_c : (g + 1) * sub_in_c, + k, + :, + :, + :, + ], + axis=0, + ) d1, d2 = d * stride[0], d * stride[0] + d_bolck_d i1, i2 = i * stride[1], i * stride[1] + d_bolck_h j1, j2 = j * stride[2], j * stride[2] + d_bolck_w - out[n, g * f_out_c + k, d1:d2:dilations[0], - i1:i2:dilations[1], - j1:j2:dilations[2]] += tmp_out - - out = out[:, :, pad_d_0:out_d - pad_d_1, pad_h_0:out_h - pad_h_1, - pad_w_0:out_w - pad_w_1] + out[ + n, + g * f_out_c + k, + d1 : d2 : dilations[0], + i1 : i2 : dilations[1], + j1 : j2 : dilations[2], + ] += tmp_out + + out = out[ + :, + :, + pad_d_0 : out_d - pad_d_1, + pad_h_0 : out_h - pad_h_1, + pad_w_0 : out_w - pad_w_1, + ] if attrs['data_format'] == 'NHWC': out = np.transpose(out, [0, 2, 3, 4, 1]) return out class TestConv3DTransposeOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -141,11 +162,12 @@ def setUp(self): 'padding_algorithm': self.padding_algorithm, 'dilations': self.dilations, 'groups': self.groups, - 'data_format': self.data_format + 'data_format': self.data_format, } - output = conv3dtranspose_forward_naive(input_, filter_, - self.attrs).astype("float32") + output = conv3dtranspose_forward_naive( + input_, filter_, self.attrs + ).astype("float32") self.outputs = {'Output': output} @@ -166,7 +188,6 @@ def init_op_type(self): class TestWithSymmetricPad(TestConv3DTransposeOp): - def init_test_case(self): self.check_no_input = True self.pad = [1, 1, 1] @@ -179,7 +200,6 @@ def init_test_case(self): class TestWithAsymmetricPad(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 0, 1, 2] self.stride = [1, 1, 1] @@ -191,7 +211,6 @@ def init_test_case(self): class TestWithSAMEPad(TestConv3DTransposeOp): - def init_test_case(self): self.stride = [1, 1, 2] self.dilations = [1, 2, 1] @@ -203,7 +222,6 @@ def init_test_case(self): class TestWithVALIDPad(TestConv3DTransposeOp): - def init_test_case(self): self.stride = [2, 1, 1] self.dilations = [1, 1, 1] @@ -215,7 +233,6 @@ def init_test_case(self): class TestWithStride(TestConv3DTransposeOp): - def init_test_case(self): self.check_no_filter = True self.pad = [1, 1, 1] @@ -228,7 +245,6 @@ def init_test_case(self): class TestWithDilation(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -240,7 +256,6 @@ def init_test_case(self): class Test_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [0, 0, 0] self.stride = [1, 1, 1] diff --git a/python/paddle/fluid/tests/unittests/npu/test_cos_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_cos_op_npu.py index f3e73b0c65e00..1ab4edef71080 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_cos_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_cos_op_npu.py @@ -26,7 +26,6 @@ class TestCos(OpTest): - def setUp(self): self.set_npu() self.op_type = "cos" @@ -55,7 +54,6 @@ def test_check_grad(self): class TestCosFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "cos" @@ -82,7 +80,6 @@ def test_check_output(self): class TestCosNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -97,9 +94,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) d = paddle.cos(c) @@ -123,16 +120,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_crop_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_crop_op_npu.py index 446ba19c2a498..1ab9f21fdce8b 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_crop_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_crop_op_npu.py @@ -28,7 +28,6 @@ class TestCropOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -41,7 +40,7 @@ def setUp(self): if self.crop_by_input: self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype), - 'Y': np.random.random(self.crop_shape).astype(self.dtype) + 'Y': np.random.random(self.crop_shape).astype(self.dtype), } else: self.attrs['shape'] = self.crop_shape @@ -74,7 +73,6 @@ def test_check_output(self): class TestCase1(TestCropOp): - def initTestCase(self): self.x_shape = (16, 8, 32) self.crop_shape = [2, 2, 3] @@ -82,7 +80,6 @@ def initTestCase(self): class TestCase2(TestCropOp): - def initTestCase(self): self.x_shape = (15, 8) self.crop_shape = [15, 8] @@ -90,7 +87,6 @@ def initTestCase(self): class TestCase3(TestCropOp): - def initTestCase(self): self.x_shape = (4, 10) self.crop_shape = [2, 3] @@ -99,7 +95,6 @@ def initTestCase(self): class TestCase4(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] @@ -107,7 +102,6 @@ def initTestCase(self): class TestCase5(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] @@ -116,7 +110,6 @@ def initTestCase(self): class TestCase6(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] @@ -127,7 +120,6 @@ def initTestCase(self): class TestCase7(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] @@ -137,7 +129,6 @@ def initTestCase(self): class TestCase8(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] @@ -146,7 +137,6 @@ def initTestCase(self): class TestCase9(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] @@ -155,7 +145,6 @@ def initTestCase(self): class TestCase10(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] diff --git a/python/paddle/fluid/tests/unittests/npu/test_cumsum_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_cumsum_op_npu.py index 64cfef9ced651..4f9badc230aed 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_cumsum_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_cumsum_op_npu.py @@ -24,7 +24,6 @@ class TestCumsumOp(unittest.TestCase): - def run_cases(self): data_np = np.arange(12).reshape(3, 4) data = paddle.to_tensor(data_np) @@ -65,11 +64,17 @@ def run_static(self, use_npu=False): place = fluid.NPUPlace(0) if use_npu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - out = exe.run(feed={'X': data_np}, - fetch_list=[ - y.name, y2.name, y3.name, y4.name, y5.name, - y6.name - ]) + out = exe.run( + feed={'X': data_np}, + fetch_list=[ + y.name, + y2.name, + y3.name, + y4.name, + y5.name, + y6.name, + ], + ) z = np.cumsum(data_np) np.testing.assert_allclose(z, out[0]) @@ -95,7 +100,6 @@ def test_name(self): class TestNPUCumSumOp1(OpTest): - def setUp(self): self.op_type = "cumsum" self.set_npu() @@ -119,18 +123,17 @@ def init_testcase(self): class TestNPUCumSumOp2(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': -1, 'reverse': True} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.outputs = { - 'Out': np.flip(np.flip(self.inputs['X'], axis=2).cumsum(axis=2), - axis=2) + 'Out': np.flip( + np.flip(self.inputs['X'], axis=2).cumsum(axis=2), axis=2 + ) } class TestNPUCumSumOp3(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 1} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} @@ -138,7 +141,6 @@ def init_testcase(self): class TestNPUCumSumOp4(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 0} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} @@ -146,107 +148,115 @@ def init_testcase(self): class TestNPUCumSumOp5(TestNPUCumSumOp1): - def init_testcase(self): self.inputs = {'X': np.random.random((5, 20)).astype(self.dtype)} self.outputs = {'Out': self.inputs['X'].cumsum(axis=1)} class TestNPUCumSumOp7(TestNPUCumSumOp1): - def init_testcase(self): self.inputs = {'X': np.random.random((100)).astype(self.dtype)} self.outputs = {'Out': self.inputs['X'].cumsum(axis=0)} class TestNPUCumSumExclusive1(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 65)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive2(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((1, 1, 888)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (1, 1, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((1, 1, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive3(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 888)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive4(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((1, 1, 3049)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (1, 1, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((1, 1, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumExclusive5(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 3096)).astype(self.dtype) self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=self.dtype), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=self.dtype), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } class TestNPUCumSumReverseExclusive(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': 2, 'reverse': True, "exclusive": True} a = np.random.random((4, 5, 6)).astype(self.dtype) self.inputs = {'X': a} a = np.flip(a, axis=2) self.outputs = { - 'Out': - np.concatenate( - (np.flip(a[:, :, :-1].cumsum(axis=2), - axis=2), np.zeros((4, 5, 1), dtype=self.dtype)), - axis=2) + 'Out': np.concatenate( + ( + np.flip(a[:, :, :-1].cumsum(axis=2), axis=2), + np.zeros((4, 5, 1), dtype=self.dtype), + ), + axis=2, + ) } class TestNPUCumSumWithFlatten1(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'flatten': True} self.inputs = {'X': np.random.random((5, 6)).astype(self.dtype)} @@ -254,31 +264,28 @@ def init_testcase(self): class TestNPUCumSumWithFlatten2(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'flatten': True} self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.outputs = {'Out': self.inputs['X'].cumsum()} -#----------------Cumsum Int64---------------- +# ----------------Cumsum Int64---------------- class TestNPUCumSumOpInt64(TestNPUCumSumOp1): - def init_testcase(self): self.attrs = {'axis': -1, 'reverse': True} self.inputs = { 'X': np.random.randint(1, 10000, size=(5, 6, 10)).astype(self.dtype) } self.outputs = { - 'Out': np.flip(np.flip(self.inputs['X'], axis=2).cumsum(axis=2), - axis=2) + 'Out': np.flip( + np.flip(self.inputs['X'], axis=2).cumsum(axis=2), axis=2 + ) } def create_test_int64(parent): - class TestCumSumInt64(parent): - def init_dtype(self): self.dtype = np.int64 diff --git a/python/paddle/fluid/tests/unittests/npu/test_density_prior_box_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_density_prior_box_op_npu.py index c3bb0880299d6..8c7ea643edc74 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_density_prior_box_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_density_prior_box_op_npu.py @@ -27,12 +27,11 @@ class TestNpuDensityPriorBoxOp(OpTest): - def set_data(self): self.init_test_params() self.init_test_input() self.init_test_output() - #self.init_test_output2() + # self.init_test_output2() self.inputs = {'Input': self.input, 'Image': self.image} self.attrs = { @@ -44,7 +43,7 @@ def set_data(self): 'densities': self.densities, 'fixed_sizes': self.fixed_sizes, 'fixed_ratios': self.fixed_ratios, - 'flatten_to_2d': self.flatten_to_2d + 'flatten_to_2d': self.flatten_to_2d, } self.outputs = {'Boxes': self.out_boxes, 'Variances': self.out_var} @@ -89,19 +88,20 @@ def init_test_params(self): if len(self.fixed_sizes) > 0 and len(self.densities) > 0: for density in self.densities: if len(self.fixed_ratios) > 0: - self.num_priors += len(self.fixed_ratios) * (pow( - density, 2)) + self.num_priors += len(self.fixed_ratios) * ( + pow(density, 2) + ) self.offset = 0.5 self.atol = 1e-5 def init_test_input(self): self.image = np.random.random( - (self.batch_size, self.image_channels, self.image_h, - self.image_w)).astype(self.dtype) + (self.batch_size, self.image_channels, self.image_h, self.image_w) + ).astype(self.dtype) self.input = np.random.random( - (self.batch_size, self.input_channels, self.layer_h, - self.layer_w)).astype(self.dtype) + (self.batch_size, self.input_channels, self.layer_h, self.layer_w) + ).astype(self.dtype) def init_test_output(self): out_dim = (self.layer_h, self.layer_w, self.num_priors, 4) @@ -115,32 +115,56 @@ def init_test_output(self): c_x = (w + self.offset) * self.step_w c_y = (h + self.offset) * self.step_h # Generate density prior boxes with fixed size - for density, fixed_size in zip(self.densities, - self.fixed_sizes): - if (len(self.fixed_ratios) > 0): + for density, fixed_size in zip( + self.densities, self.fixed_sizes + ): + if len(self.fixed_ratios) > 0: for ar in self.fixed_ratios: shift = int(step_average / density) box_width_ratio = fixed_size * math.sqrt(ar) box_height_ratio = fixed_size / math.sqrt(ar) for di in range(density): for dj in range(density): - c_x_temp = c_x - step_average / 2.0 + shift / 2.0 + dj * shift - c_y_temp = c_y - step_average / 2.0 + shift / 2.0 + di * shift + c_x_temp = ( + c_x + - step_average / 2.0 + + shift / 2.0 + + dj * shift + ) + c_y_temp = ( + c_y + - step_average / 2.0 + + shift / 2.0 + + di * shift + ) out_boxes[h, w, idx, :] = [ - max((c_x_temp - box_width_ratio / 2.0) / - self.image_w, 0), - max((c_y_temp - box_height_ratio / 2.0) - / self.image_h, 0), - min((c_x_temp + box_width_ratio / 2.0) / - self.image_w, 1), - min((c_y_temp + box_height_ratio / 2.0) - / self.image_h, 1) + max( + (c_x_temp - box_width_ratio / 2.0) + / self.image_w, + 0, + ), + max( + (c_y_temp - box_height_ratio / 2.0) + / self.image_h, + 0, + ), + min( + (c_x_temp + box_width_ratio / 2.0) + / self.image_w, + 1, + ), + min( + (c_y_temp + box_height_ratio / 2.0) + / self.image_h, + 1, + ), ] idx += 1 if self.clip: out_boxes = np.clip(out_boxes, 0.0, 1.0) - out_var = np.tile(self.variances, - (self.layer_h, self.layer_w, self.num_priors, 1)) + out_var = np.tile( + self.variances, (self.layer_h, self.layer_w, self.num_priors, 1) + ) self.out_boxes = out_boxes.astype(self.dtype) self.out_var = out_var.astype(self.dtype) if self.flatten_to_2d: @@ -149,7 +173,6 @@ def init_test_output(self): class TestNpuDensityPriorBoxFlatten(TestNpuDensityPriorBoxOp): - def set_density(self): self.densities = [3, 4] self.fixed_sizes = [1.0, 2.0] @@ -162,7 +185,6 @@ def set_density(self): class TestNpuDensityPriorBoxOp1(TestNpuDensityPriorBoxOp): - def set_density(self): super(TestNpuDensityPriorBoxOp1, self).set_density() self.layer_w = 1 @@ -170,7 +192,6 @@ def set_density(self): class TestNpuDensityPriorBoxOp2(TestNpuDensityPriorBoxOp): - def set_density(self): super(TestNpuDensityPriorBoxOp2, self).set_density() self.layer_w = 15 @@ -180,14 +201,12 @@ def set_density(self): class TestNpuDensityPriorBoxOp3(TestNpuDensityPriorBoxOp): - def set_density(self): super(TestNpuDensityPriorBoxOp3, self).set_density() self.fixed_ratios = [1.0, 4.0] class TestNpuDensityPriorBoxOpFP16(TestNpuDensityPriorBoxOp): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/npu/test_dropout_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_dropout_op_npu.py index 3c55d7537b066..b9233f3fd685c 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_dropout_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_dropout_op_npu.py @@ -28,7 +28,6 @@ class TestDropoutOp(OpTest): - def setUp(self): self.op_type = "dropout" self.set_npu() @@ -38,11 +37,11 @@ def setUp(self): 'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64)).astype('uint8') + 'Mask': np.ones((32, 64)).astype('uint8'), } def init_dtype(self): @@ -70,11 +69,11 @@ def setUp(self): 'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((3, 62)).astype('uint8') + 'Mask': np.ones((3, 62)).astype('uint8'), } @@ -89,11 +88,11 @@ def setUp(self): 'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((2000)).astype('uint8') + 'Mask': np.ones((2000)).astype('uint8'), } @@ -108,11 +107,11 @@ def setUp(self): 'dropout_prob': 1.0, 'fix_seed': True, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': np.zeros((32, 64)).astype('float32'), - 'Mask': np.zeros((32, 64)).astype('uint8') + 'Mask': np.zeros((32, 64)).astype('uint8'), } @@ -127,11 +126,11 @@ def setUp(self): 'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64, 2)).astype('uint8') + 'Mask': np.ones((32, 64, 2)).astype('uint8'), } @@ -147,7 +146,7 @@ def setUp(self): 'dropout_prob': 0.35, 'fix_seed': True, 'is_test': True, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = {'Out': self.inputs['X']} @@ -164,7 +163,6 @@ def test_check_output(self): @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestDropoutOpInference2(TestDropoutOpInference): - def setUp(self): self.op_type = "dropout" self.set_npu() @@ -173,7 +171,7 @@ def setUp(self): self.attrs = { 'dropout_prob': 0.75, 'is_test': True, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = {'Out': self.inputs['X']} @@ -186,16 +184,16 @@ def setUp(self): self.init_dtype() self.inputs = { "X": np.random.random((32, 64)).astype(self.dtype), - "Seed": np.asarray([125], dtype="int32") + "Seed": np.asarray([125], dtype="int32"), } self.attrs = { 'dropout_prob': 0.0, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64)).astype('uint8') + 'Mask': np.ones((32, 64)).astype('uint8'), } @@ -211,7 +209,6 @@ def set_npu(self): class TestDropoutAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace(), paddle.NPUPlace(0)] @@ -219,43 +216,44 @@ def setUp(self): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): input = fluid.data(name="input", shape=[40, 40], dtype="float32") - res1 = paddle.nn.functional.dropout(x=input, - p=0., - training=False, - mode='upscale_in_train') - res2 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=True, - mode='upscale_in_train') - res3 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=False, - mode='upscale_in_train') - res4 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=True, - mode='upscale_in_train') - res5 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=False, - mode='upscale_in_train') - res6 = paddle.nn.functional.dropout(x=input, - p=1., - training=True, - mode='upscale_in_train') + res1 = paddle.nn.functional.dropout( + x=input, p=0.0, training=False, mode='upscale_in_train' + ) + res2 = paddle.nn.functional.dropout( + x=input, p=0.0, axis=0, training=True, mode='upscale_in_train' + ) + res3 = paddle.nn.functional.dropout( + x=input, p=0.0, axis=0, training=False, mode='upscale_in_train' + ) + res4 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=True, + mode='upscale_in_train', + ) + res5 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=False, + mode='upscale_in_train', + ) + res6 = paddle.nn.functional.dropout( + x=input, p=1.0, training=True, mode='upscale_in_train' + ) res7 = paddle.fluid.layers.dropout( x=input, - dropout_prob=0., - dropout_implementation='upscale_in_train') - res8 = paddle.nn.functional.dropout(x=input, - p=0., - axis=(0, 1), - training=False, - mode='upscale_in_train') + dropout_prob=0.0, + dropout_implementation='upscale_in_train', + ) + res8 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=(0, 1), + training=False, + mode='upscale_in_train', + ) in_np = np.random.random([40, 40]).astype("float32") res_np = in_np @@ -264,13 +262,17 @@ def check_static_result(self, place): exe = fluid.Executor(place) res_list = [res1, res2, res3, res4, res5, res7, res8] for res in res_list: - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res], + ) np.testing.assert_allclose(fetches[0], res_np) - fetches2 = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res6]) + fetches2 = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res6], + ) np.testing.assert_allclose(fetches2[0], res_np2) def test_static(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_add_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_add_op_npu.py index b4da50a47acc9..735b1d0ac0613 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_add_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_add_op_npu.py @@ -28,7 +28,6 @@ class TestElementwiseAddOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "elementwise_add" @@ -40,7 +39,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -128,21 +127,19 @@ def test_check_grad_ingore_y(self): class TestFP16ElementwiseAddOp(TestElementwiseAddOp): - def init_dtype(self): self.dtype = np.float16 class TestINT64ElementwiseAddOp(TestElementwiseAddOp): - def init_dtype(self): self.dtype = np.int64 @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_scalar(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -150,9 +147,9 @@ def init_input_output(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestFP16ElementwiseAddOp_scalar(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -160,9 +157,9 @@ def init_input_output(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." +) class TestElementwiseAddOp_scalar2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) @@ -170,9 +167,9 @@ def init_input_output(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." +) class TestFP16ElementwiseAddOp_scalar2(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) @@ -180,7 +177,6 @@ def init_input_output(self): class TestAddAPI(unittest.TestCase): - def test_name(self): with paddle.static.program_guard(paddle.static.Program()): x = paddle.static.data(name="x", shape=[2, 3], dtype="float32") @@ -205,66 +201,65 @@ def test_static(self): place = paddle.NPUPlace(0) exe = paddle.static.Executor(place) - x_value, y_value, z_value = exe.run(feed={ - "x": x_np, - "y": y_np - }, - fetch_list=[x, y, z]) + x_value, y_value, z_value = exe.run( + feed={"x": x_np, "y": y_np}, fetch_list=[x, y, z] + ) - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual( (x_value == x_np).all(), True, - msg="x_value = {}, but expected {}".format(x_value, x_np)) + msg="x_value = {}, but expected {}".format(x_value, x_np), + ) self.assertEqual( (y_value == y_np).all(), True, - msg="y_value = {}, but expected {}".format(y_value, y_np)) + msg="y_value = {}, but expected {}".format(y_value, y_np), + ) self.assertEqual( (z_value == z_expected).all(), True, - msg="z_value = {}, but expected {}".format(z_value, z_expected)) + msg="z_value = {}, but expected {}".format(z_value, z_expected), + ) class TestAddError(unittest.TestCase): - def test_errors(self): with paddle.static.program_guard(paddle.static.Program()): # the input of elementwise_add must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.NPUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.NPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.NPUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.NPUPlace(0) + ) self.assertRaises(TypeError, paddle.add, x1, y1) # the input dtype must be float16 or float32 or float64 or int32 or int64 - x2 = paddle.static.data(name='x2', - shape=[3, 4, 5, 6], - dtype="uint8") - y2 = paddle.static.data(name='y2', - shape=[3, 4, 5, 6], - dtype="uint8") + x2 = paddle.static.data( + name='x2', shape=[3, 4, 5, 6], dtype="uint8" + ) + y2 = paddle.static.data( + name='y2', shape=[3, 4, 5, 6], dtype="uint8" + ) self.assertRaises(TypeError, paddle.add, x2, y2) class TestElementwiseAddOp_Vector(TestElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestFP16ElementwiseAddOp_Vector(TestFP16ElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestElementwiseAddOp_broadcast_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -275,7 +270,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_broadcast_0(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -286,7 +280,6 @@ def init_axis(self): class TestElementwiseAddOp_broadcast_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -297,7 +290,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_broadcast_1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -308,7 +300,6 @@ def init_axis(self): class TestElementwiseAddOp_broadcast_2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -316,7 +307,6 @@ def init_input_output(self): class TestFP16ElementwiseAddOp_broadcast_2(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -324,7 +314,6 @@ def init_input_output(self): class TestElementwiseAddOp_broadcast_3(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 1).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -335,7 +324,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_broadcast_3(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -346,7 +334,6 @@ def init_axis(self): class TestElementwiseAddOp_broadcast_4(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 1, 2).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -357,7 +344,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_broadcast_4(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 1, 2).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -368,7 +354,6 @@ def init_axis(self): class TestElementwiseAddOp_broadcast_5(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) @@ -376,7 +361,6 @@ def init_input_output(self): class TestFP16ElementwiseAddOp_broadcast_5(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) @@ -384,7 +368,6 @@ def init_input_output(self): class TestElementwiseAddOp_broadcast_6(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) @@ -392,7 +375,6 @@ def init_input_output(self): class TestElementwiseAddOp_broadcast_7(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(1, 1, 20, 5).astype(self.dtype) self.y = np.random.rand(20, 5, 1, 1).astype(self.dtype) @@ -400,7 +382,6 @@ def init_input_output(self): class TestFP16ElementwiseAddOp_broadcast_6(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) @@ -408,7 +389,6 @@ def init_input_output(self): class TestElementwiseAddOp_rowwise_add_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -419,7 +399,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_rowwise_add_0(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -430,9 +409,9 @@ def init_axis(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_rowwise_add_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -443,9 +422,9 @@ def init_axis(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestFP16ElementwiseAddOp_rowwise_add_1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -456,7 +435,6 @@ def init_axis(self): class TestElementwiseAddOp_channelwise_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -467,7 +445,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_channelwise_add(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -478,7 +455,6 @@ def init_axis(self): class TestElementwiseAddOp_commonuse_add1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -489,7 +465,6 @@ def init_axis(self): class TestElementwiseFP16AddOp_commonuse_add1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -500,7 +475,6 @@ def init_axis(self): class TestElementwiseAddOp_commonuse_add2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 1, 4).astype(self.dtype) self.y = np.random.rand(10, 1, 12, 1).astype(self.dtype) @@ -511,7 +485,6 @@ def init_axis(self): class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 12).astype(self.dtype) self.y = np.random.rand(2, 2, 10, 12).astype(self.dtype) @@ -522,7 +495,6 @@ def init_axis(self): class TestElementwiseAddOp_same_shape_ysize_large(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 1, 12).astype(self.dtype) self.y = np.random.rand(10, 2, 12).astype(self.dtype) @@ -533,14 +505,15 @@ def init_axis(self): class TestElementwiseAddOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_add must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.NPUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.NPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.NPUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.NPUPlace(0) + ) self.assertRaises(TypeError, fluid.layers.elementwise_add, x1, y1) # the input dtype of elementwise_add must be float16 or float32 or float64 or int32 or int64 @@ -551,7 +524,6 @@ def test_errors(self): class TestAddApi(unittest.TestCase): - def _executed_api(self, x, y, name=None): return paddle.add(x, y, name) @@ -569,7 +541,7 @@ def test_declarative(self): def gen_data(): return { "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') + "y": np.array([1, 5, 2]).astype('float32'), } x = fluid.data(name="x", shape=[3], dtype='float32') @@ -579,7 +551,7 @@ def gen_data(): place = fluid.NPUPlace(0) exe = fluid.Executor(place) z_value = exe.run(feed=gen_data(), fetch_list=[z.name]) - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((z_value == z_expected).all(), True) def test_dygraph(self): @@ -590,18 +562,16 @@ def test_dygraph(self): y = fluid.dygraph.to_variable(np_y) z = self._executed_api(x, y) np_z = z.numpy() - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((np_z == z_expected).all(), True) class TestAddInplaceApi(TestAddApi): - def _executed_api(self, x, y, name=None): return x.add_(y, name) class TestAddInplaceBroadcastSuccess(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 4).astype('float') self.y_numpy = np.random.rand(3, 4).astype('float') @@ -618,21 +588,18 @@ def test_broadcast_success(self): class TestAddInplaceBroadcastSuccess2(TestAddInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(1, 2, 3, 1).astype('float') self.y_numpy = np.random.rand(3, 1).astype('float') class TestAddInplaceBroadcastSuccess3(TestAddInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 1, 5).astype('float') self.y_numpy = np.random.rand(1, 3, 1, 5).astype('float') class TestAddInplaceBroadcastError(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(3, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') @@ -651,14 +618,12 @@ def broadcast_shape_error(): class TestAddInplaceBroadcastError2(TestAddInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(2, 1, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') class TestAddInplaceBroadcastError3(TestAddInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(5, 2, 1, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_div_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_div_op_npu.py index ab9c74fcf7006..acdb8c75db561 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_div_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_div_op_npu.py @@ -27,7 +27,6 @@ class TestElementwiseDiv(OpTest): - def setUp(self): self.set_npu() self.op_type = "elementwise_div" @@ -41,7 +40,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.attrs = {} self.outputs = {'Out': out} @@ -73,13 +72,12 @@ def test_check_grad_ingore_x(self): ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y")) + self.check_grad_with_place( + self.place, ['X'], 'Out', no_grad_set=set("Y") + ) class TestElementwiseDivFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "elementwise_div" @@ -93,7 +91,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.attrs = {} self.outputs = {'Out': out} @@ -110,7 +108,6 @@ def test_check_output(self): class TestElementwiseDivNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -129,9 +126,9 @@ def _test(self, run_npu=True): b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') c = paddle.static.data(name="c", shape=[32, 32], dtype='float32') d = paddle.static.data(name="d", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) e = paddle.multiply(a, b) f = paddle.multiply(c, d) @@ -157,18 +154,23 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "c": c_np, - "d": d_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={ + "a": a_np, + "b": b_np, + "c": c_np, + "d": d_np, + "label": label_np, + }, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -181,7 +183,6 @@ def test_npu(self): class TestFloatStatus(unittest.TestCase): - def test_overflow(self): paddle.disable_static() paddle.set_device('npu') @@ -191,9 +192,9 @@ def test_overflow(self): self.assertEqual(flag.numpy().sum(), 0.0) x = paddle.to_tensor([12.564], stop_gradient=False) - y = paddle.to_tensor([2.], stop_gradient=False) + y = paddle.to_tensor([2.0], stop_gradient=False) z = x / y - out = 32768. * z + out = 32768.0 * z ops.get_float_status(flag, flag) self.assertEqual(flag.numpy().sum(), 0.0) diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_floordiv_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_floordiv_op_npu.py index 7301f582fca67..6762c51ca3898 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_floordiv_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_floordiv_op_npu.py @@ -24,7 +24,6 @@ class TestElementwiseFloorDiv(OpTest): - def setUp(self): self.op_type = "elementwise_floordiv" self.set_npu() @@ -33,7 +32,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {} self.outputs = {'Out': self.out} @@ -55,7 +54,6 @@ def test_check_output(self): class TestElementwiseFloorDiv2(TestElementwiseFloorDiv): - def init_dtype(self): self.dtype = "int32" diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_max_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_max_op_npu.py index 92a91b384fcb7..fe3d584792944 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_max_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_max_op_npu.py @@ -41,7 +41,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_x)) or ( - shape_out[ax] > 1 and shape_x[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_x[ax - src_axis] == 1 + ): reduce_axes_x.append(ax) if shape_y != shape_out: @@ -52,7 +53,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_y)) or ( - shape_out[ax] > 1 and shape_y[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_y[ax - src_axis] == 1 + ): reduce_axes_y.append(ax) if len(reduce_axes_x) > 0: @@ -79,7 +81,6 @@ def ComputeGrad(x, y, out, axis): class TestElementwiseMaxOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "elementwise_max" @@ -91,7 +92,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis} self.outputs = {'Out': self.out} @@ -106,7 +107,8 @@ def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) sgn = np.random.choice([-1, 1], [13, 17]).astype(self.dtype) self.y = self.x + sgn * np.random.uniform(0.1, 1, [13, 17]).astype( - self.dtype) + self.dtype + ) self.out = np.maximum(self.x, self.y) def init_axis(self): @@ -119,18 +121,17 @@ def test_check_grad_normal(self): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X")) + self.check_grad_with_place( + self.place, ['Y'], 'Out', no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y")) + self.check_grad_with_place( + self.place, ['X'], 'Out', no_grad_set=set("Y") + ) class TestElementwiseMaxOp_int32(TestElementwiseMaxOp): - def init_dtype(self): self.dtype = np.int32 @@ -146,7 +147,6 @@ def test_check_grad_ingore_y(self): class TestElementwiseMaxOp_scalar(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.random_integers(-5, 5, [2, 3, 20]).astype(self.dtype) self.y = np.array([0.5]).astype(self.dtype) @@ -154,22 +154,22 @@ def init_input_output(self): class TestElementwiseMaxOp_vector(TestElementwiseMaxOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x + sgn * np.random.uniform(0.1, 1, - (100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x + sgn * np.random.uniform(0.1, 1, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y) class TestElementwiseMaxOp_broadcast_0(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (100, 5, 2)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[:, 0, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[:, 0, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y.reshape(100, 1, 1)) def init_axis(self): @@ -177,12 +177,12 @@ def init_axis(self): class TestElementwiseMaxOp_broadcast_1(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 100, 3)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[0, :, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[0, :, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y.reshape(1, 100, 1)) def init_axis(self): @@ -190,56 +190,68 @@ def init_axis(self): def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwiseMaxOp_broadcast_2(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 100)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[0, 0, :] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[0, 0, :] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y.reshape(1, 1, 100)) def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwiseMaxOp_broadcast_3(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 50, 2, 1)).astype(self.dtype) sgn = np.random.choice([-1, 1], (50, 2)).astype(self.dtype) - self.y = self.x[0, :, :, 0] + sgn * \ - np.random.uniform(1, 2, (50, 2)).astype(self.dtype) + self.y = self.x[0, :, :, 0] + sgn * np.random.uniform( + 1, 2, (50, 2) + ).astype(self.dtype) self.out = np.maximum(self.x, self.y.reshape(1, 50, 2, 1)) def init_axis(self): @@ -247,27 +259,26 @@ def init_axis(self): class TestElementwiseMaxOp_broadcast_4(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 4, 5)).astype(self.dtype) sgn = np.random.choice([-1, 1], (2, 3, 1, 5)).astype(self.dtype) - self.y = self.x + sgn * \ - np.random.uniform(1, 2, (2, 3, 1, 5)).astype(self.dtype) + self.y = self.x + sgn * np.random.uniform(1, 2, (2, 3, 1, 5)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y) class TestElementwiseMaxOp_broadcast_5(TestElementwiseMaxOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 4, 5)).astype(self.dtype) sgn = np.random.choice([-1, 1], (2, 3, 1, 1)).astype(self.dtype) - self.y = self.x + sgn * \ - np.random.uniform(1, 2, (2, 3, 1, 1)).astype(self.dtype) + self.y = self.x + sgn * np.random.uniform(1, 2, (2, 3, 1, 1)).astype( + self.dtype + ) self.out = np.maximum(self.x, self.y) class TestElementwiseMaxNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -282,9 +293,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.maximum(a, b) @@ -307,16 +318,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_min_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_min_op_npu.py index 5956faba0e1e9..8cd51765bd829 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_min_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_min_op_npu.py @@ -28,7 +28,6 @@ class TestElementwiseMinOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "elementwise_min" @@ -37,7 +36,7 @@ def setUp(self): self.init_input_output() self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {'axis': self.axis} @@ -52,7 +51,8 @@ def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) self.sgn = np.random.choice([-1, 1], [13, 17]).astype(self.dtype) self.y = self.x + self.sgn * np.random.uniform(0.1, 1, [13, 17]).astype( - self.dtype) + self.dtype + ) self.out = np.minimum(self.x, self.y) self.axis = -1 @@ -64,9 +64,9 @@ def test_check_output(self): def test_check_grad_normal(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=0.5) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', max_relative_error=0.5 + ) else: self.check_grad_with_place( self.place, @@ -76,10 +76,13 @@ def test_check_grad_normal(self): def test_check_grad_ingore_x(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - max_relative_error=0.9) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + max_relative_error=0.9, + ) else: self.check_grad_with_place( self.place, @@ -90,10 +93,13 @@ def test_check_grad_ingore_x(self): def test_check_grad_ingore_y(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - max_relative_error=0.1) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + max_relative_error=0.1, + ) else: self.check_grad_with_place( self.place, @@ -104,32 +110,30 @@ def test_check_grad_ingore_y(self): class TestElementwiseMinOpFp16(TestElementwiseMinOp): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMinOp_Vector(TestElementwiseMinOp): - def init_input_output(self): - self.x = np.random.uniform(1, 2, (100, )).astype(self.dtype) - self.sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x + self.sgn * np.random.uniform(0.1, 1, (100, )).astype( - self.dtype) + self.x = np.random.uniform(1, 2, (100,)).astype(self.dtype) + self.sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x + self.sgn * np.random.uniform(0.1, 1, (100,)).astype( + self.dtype + ) self.out = np.minimum(self.x, self.y) self.axis = -1 class TestElementwiseMinOpFp16_Vector(TestElementwiseMinOp_Vector): - def init_dtype(self): self.dtype = np.float16 @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMinOp_scalar(TestElementwiseMinOp): - def init_input_output(self): self.x = np.random.random_integers(-5, 5, [10, 3, 4]).astype(self.dtype) self.y = np.array([0.5]).astype(self.dtype) @@ -138,32 +142,30 @@ def init_input_output(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMinOpFp16_scalar(TestElementwiseMinOp_scalar): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMinOp_broadcast(TestElementwiseMinOp): - def init_input_output(self): self.x = np.random.uniform(0.5, 1, (2, 3, 100)).astype(self.dtype) - self.sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - self.y = self.x[0, 0, :] + self.sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + self.sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + self.y = self.x[0, 0, :] + self.sgn * np.random.uniform( + 1, 2, (100,) + ).astype(self.dtype) self.out = np.minimum(self.x, self.y.reshape(1, 1, 100)) self.axis = -1 class TestElementwiseMinOpFp16_broadcast(TestElementwiseMinOp_broadcast): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMinOpNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -178,9 +180,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.minimum(a, b) @@ -203,16 +205,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_mod_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_mod_op_npu.py index fb5c773c825dc..efb81fbad6ec0 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_mod_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_mod_op_npu.py @@ -29,7 +29,6 @@ class TestElementwiseModOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -42,7 +41,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -69,13 +68,11 @@ def test_check_output(self): class TestElementwiseModOpInt64(TestElementwiseModOp): - def init_dtype(self): self.dtype = np.int64 class TestElementwiseModOp_scalar(TestElementwiseModOp): - def init_input_output(self): scale_x = random.randint(0, 100000000) scale_y = random.randint(1, 100000000) @@ -85,7 +82,6 @@ def init_input_output(self): class TestElementwiseModOpFloat(TestElementwiseModOp): - def init_dtype(self): self.dtype = np.float32 @@ -99,7 +95,6 @@ def test_check_output(self): class TestElementwiseModOpDouble(TestElementwiseModOpFloat): - def init_dtype(self): self.dtype = np.float64 @@ -108,7 +103,6 @@ def test_check_output(self): class TestElementwiseModOpFP16(TestElementwiseModOpFloat): - def init_dtype(self): self.dtype = np.float16 @@ -117,7 +111,6 @@ def test_check_output(self): class TestElementwiseModOp_broadcast_0(TestElementwiseModOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -128,7 +121,6 @@ def init_axis(self): class TestElementwiseModOp_broadcast_1(TestElementwiseModOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -139,7 +131,6 @@ def init_axis(self): class TestElementwiseModOp_broadcast_2(TestElementwiseModOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -150,7 +141,6 @@ def init_axis(self): class TestRemainderOp(unittest.TestCase): - def test_name(self): paddle.set_device('npu:0') with fluid.program_guard(fluid.Program()): @@ -172,7 +162,7 @@ def test_dygraph(self): self.assertEqual((np_z == z_expected).all(), True) np_x = np.array([-3.3, 11.5, -2, 3.5]) - np_y = np.array([-1.2, 2., 3.3, -2.3]) + np_y = np.array([-1.2, 2.0, 3.3, -2.3]) x = paddle.to_tensor(np_x) y = paddle.to_tensor(np_y) z = x % y diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_mul_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_mul_op_npu.py index a3fe2dcf453a2..fa0b80d633898 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_mul_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_mul_op_npu.py @@ -25,7 +25,6 @@ class ElementwiseMulOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -41,7 +40,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {'axis': self.axis} @@ -53,14 +52,14 @@ def test_check_grad_normal(self): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X")) + self.check_grad_with_place( + self.place, ['Y'], 'Out', no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set('Y')) + self.check_grad_with_place( + self.place, ['X'], 'Out', no_grad_set=set('Y') + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -75,33 +74,31 @@ def init_axis(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMulOp_scalar(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 3, 4).astype(np.float32), - 'Y': np.random.rand(1).astype(np.float32) + 'Y': np.random.rand(1).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_Vector(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { - 'X': np.random.random((100, )).astype("float32"), - 'Y': np.random.random((100, )).astype("float32") + 'X': np.random.random((100,)).astype("float32"), + 'Y': np.random.random((100,)).astype("float32"), } self.outputs = {'Out': np.multiply(self.inputs['X'], self.inputs['Y'])} class TestElementwiseMulOp_broadcast_0(ElementwiseMulOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -112,13 +109,12 @@ def init_axis(self): class TestElementwiseMulOp_broadcast_1(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 100, 3).astype(np.float32), - 'Y': np.random.rand(100).astype(np.float32) + 'Y': np.random.rand(100).astype(np.float32), } self.attrs = {'axis': 1} @@ -128,13 +124,12 @@ def setUp(self): class TestElementwiseMulOp_broadcast_2(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float32), - 'Y': np.random.rand(100).astype(np.float32) + 'Y': np.random.rand(100).astype(np.float32), } self.outputs = { @@ -143,13 +138,12 @@ def setUp(self): class TestElementwiseMulOp_broadcast_3(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 10, 12, 3).astype(np.float32), - 'Y': np.random.rand(10, 12).astype(np.float32) + 'Y': np.random.rand(10, 12).astype(np.float32), } self.attrs = {'axis': 1} @@ -159,69 +153,64 @@ def setUp(self): class TestElementwiseMulOp_broadcast_4(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 2, 11).astype(np.float32), - 'Y': np.random.rand(10, 1, 11).astype(np.float32) + 'Y': np.random.rand(10, 1, 11).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_broadcast_5(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 4, 2, 3).astype(np.float32), - 'Y': np.random.rand(10, 4, 1, 3).astype(np.float32) + 'Y': np.random.rand(10, 4, 1, 3).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "paddle is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "paddle is not compiled with NPU" +) class TestElementwiseMulOpFp16(ElementwiseMulOp): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMulOp_commonuse_1(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float32), - 'Y': np.random.rand(1, 1, 100).astype(np.float32) + 'Y': np.random.rand(1, 1, 100).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_commonuse_2(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(30, 3, 1, 5).astype(np.float32), - 'Y': np.random.rand(30, 1, 4, 1).astype(np.float32) + 'Y': np.random.rand(30, 1, 4, 1).astype(np.float32), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_xsize_lessthan_ysize(ElementwiseMulOp): - def setUp(self): self.set_npu() self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 10).astype(np.float32), - 'Y': np.random.rand(2, 2, 10, 10).astype(np.float32) + 'Y': np.random.rand(2, 2, 10, 10).astype(np.float32), } self.attrs = {'axis': 2} diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_pow_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_pow_op_npu.py index 715289942384a..b872c5bf83edf 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_pow_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_pow_op_npu.py @@ -42,7 +42,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_x)) or ( - shape_out[ax] > 1 and shape_x[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_x[ax - src_axis] == 1 + ): reduce_axes_x.append(ax) if shape_y != shape_out: @@ -53,7 +54,8 @@ def ComputeGrad(x, y, out, axis): for ax in range(len(shape_out)): if (ax < src_axis or ax >= src_axis + len(shape_y)) or ( - shape_out[ax] > 1 and shape_y[ax - src_axis] == 1): + shape_out[ax] > 1 and shape_y[ax - src_axis] == 1 + ): reduce_axes_y.append(ax) if len(reduce_axes_x) > 0: @@ -79,7 +81,6 @@ def ComputeGrad(x, y, out, axis): class TestElementwisePow(OpTest): - def setUp(self): self.set_npu() self.op_type = "elementwise_pow" @@ -91,7 +92,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis} self.outputs = {'Out': self.out} @@ -116,27 +117,32 @@ def init_input_output(self): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwisePowFp16(TestElementwisePow): - def init_input_output(self): np.random.seed(SEED) self.x = np.random.uniform(1, 2, [11, 17]).astype(self.dtype) @@ -155,7 +161,6 @@ def test_check_output(self): class TestElementwisePowDouble(TestElementwisePow): - def init_input_output(self): np.random.seed(SEED) self.x = np.random.uniform(1, 2, [11, 17]).astype(self.dtype) @@ -174,7 +179,6 @@ def test_check_output(self): class TestElementwisePowOp_broadcast_0(TestElementwisePow): - def init_axis(self): self.axis = 1 @@ -186,27 +190,32 @@ def init_input_output(self): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwisePowOp_broadcast_1(TestElementwisePow): - def init_axis(self): self.axis = 1 @@ -218,27 +227,32 @@ def init_input_output(self): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwisePowOp_broadcast_2(TestElementwisePow): - def init_axis(self): self.axis = 0 @@ -250,27 +264,32 @@ def init_input_output(self): def test_check_grad_normal(self): dx, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy]) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', user_defined_grads=[dx, dy] + ) def test_check_grad_ingore_x(self): _, dy = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[dy]) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[dy], + ) def test_check_grad_ingore_y(self): dx, _ = ComputeGrad(self.x, self.y, self.out, self.axis) - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set("Y"), - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + no_grad_set=set("Y"), + user_defined_grads=[dx], + ) class TestElementwisePowNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -285,9 +304,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.pow(a, b) @@ -310,16 +329,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_elementwise_sub_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_elementwise_sub_op_npu.py index ac3658f0c34ae..8542ed6bdc396 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_elementwise_sub_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_elementwise_sub_op_npu.py @@ -27,7 +27,6 @@ class TestElementwiseSubOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "elementwise_sub" @@ -39,7 +38,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -91,19 +90,16 @@ def test_check_output(self): class TestElementwiseSubOpInt32(TestElementwiseSubOp): - def init_dtype(self): self.dtype = np.int32 class TestElementwiseSubOpInt64(TestElementwiseSubOp): - def init_dtype(self): self.dtype = np.int64 class TestSubtractAPI(unittest.TestCase): - def test_name(self): with paddle.static.program_guard(paddle.static.Program()): x = paddle.static.data(name="x", shape=[2, 3], dtype="float32") @@ -128,50 +124,51 @@ def test_static(self): place = paddle.NPUPlace(0) exe = paddle.static.Executor(place) - x_value, y_value, z_value = exe.run(feed={ - "x": x_np, - "y": y_np - }, - fetch_list=[x, y, z]) + x_value, y_value, z_value = exe.run( + feed={"x": x_np, "y": y_np}, fetch_list=[x, y, z] + ) - z_expected = np.array([1., -2., 2.]) + z_expected = np.array([1.0, -2.0, 2.0]) self.assertEqual( (x_value == x_np).all(), True, - msg="x_value = {}, but expected {}".format(x_value, x_np)) + msg="x_value = {}, but expected {}".format(x_value, x_np), + ) self.assertEqual( (y_value == y_np).all(), True, - msg="y_value = {}, but expected {}".format(y_value, y_np)) + msg="y_value = {}, but expected {}".format(y_value, y_np), + ) self.assertEqual( (z_value == z_expected).all(), True, - msg="z_value = {}, but expected {}".format(z_value, z_expected)) + msg="z_value = {}, but expected {}".format(z_value, z_expected), + ) class TestSubtractError(unittest.TestCase): - def test_errors(self): with paddle.static.program_guard(paddle.static.Program()): # the input of elementwise_add must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.NPUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.NPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.NPUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.NPUPlace(0) + ) self.assertRaises(TypeError, paddle.subtract, x1, y1) # the input dtype must be float16 or float32 or float64 or int32 or int64 - x2 = paddle.static.data(name='x2', - shape=[3, 4, 5, 6], - dtype="uint8") - y2 = paddle.static.data(name='y2', - shape=[3, 4, 5, 6], - dtype="uint8") + x2 = paddle.static.data( + name='x2', shape=[3, 4, 5, 6], dtype="uint8" + ) + y2 = paddle.static.data( + name='y2', shape=[3, 4, 5, 6], dtype="uint8" + ) self.assertRaises(TypeError, paddle.subtract, x2, y2) class TestSubtractNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -186,9 +183,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) c = paddle.assign(b) @@ -212,16 +209,17 @@ def _test(self, run_npu=True): for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_exp_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_exp_op_npu.py index 44ba219d40fc7..ca43877d45315 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_exp_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_exp_op_npu.py @@ -30,7 +30,6 @@ class TestExpNPUOP(OpTest): - def setUp(self): self.set_npu() @@ -63,7 +62,6 @@ def set_npu(self): class TestExpNPUOPFloat64(TestExpNPUOP): - def init_dtype(self): self.dtype = np.float64 diff --git a/python/paddle/fluid/tests/unittests/npu/test_expand_as_v2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_expand_as_v2_op_npu.py index 76ab1d2959a24..ca2b0195b4280 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_expand_as_v2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_expand_as_v2_op_npu.py @@ -26,7 +26,6 @@ class TestExpandAsOpRank1(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -50,7 +49,6 @@ def test_check_grad(self): class TestExpandAsOpRank2(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -74,7 +72,6 @@ def test_check_grad(self): class TestExpandAsOpRank3(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -98,7 +95,6 @@ def test_check_grad(self): class TestExpandAsOpRank4(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -123,29 +119,28 @@ def test_check_grad(self): # Test python API class TestExpandAsV2API(unittest.TestCase): - def test_api(self): input1 = np.random.random([12, 14]).astype("float32") input2 = np.random.random([2, 12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) - y = fluid.layers.data(name='target_tensor', - shape=[2, 12, 14], - append_batch_size=False, - dtype="float32") + y = fluid.layers.data( + name='target_tensor', + shape=[2, 12, 14], + append_batch_size=False, + dtype="float32", + ) out_1 = paddle.expand_as(x, y=y) exe = fluid.Executor(place=fluid.NPUPlace(0)) - res_1 = exe.run(fluid.default_main_program(), - feed={ - "x": input1, - "target_tensor": input2 - }, - fetch_list=[out_1]) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": input1, "target_tensor": input2}, + fetch_list=[out_1], + ) assert np.array_equal(res_1[0], np.tile(input1, (2, 1, 1))) diff --git a/python/paddle/fluid/tests/unittests/npu/test_expand_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_expand_op_npu.py index 3096369b1966a..0ac8cfc5d3c13 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_expand_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_expand_op_npu.py @@ -26,7 +26,6 @@ class TestExpand(OpTest): - def setUp(self): self.set_npu() self.op_type = "expand" @@ -55,7 +54,6 @@ def test_check_grad(self): class TestExpandV2(TestExpand): - def setUp(self): self.set_npu() self.op_type = "expand" @@ -69,7 +67,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'ExpandTimes': OpTest.np_dtype_to_fluid_dtype(expand_times) + 'ExpandTimes': OpTest.np_dtype_to_fluid_dtype(expand_times), } self.attrs = {} self.outputs = {'Out': out} @@ -83,7 +81,6 @@ def init_dtype(self): class TestExpandNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -96,9 +93,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 1], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) res = paddle.fluid.layers.expand(a, [1, 32]) loss = res.sum() @@ -115,12 +112,11 @@ def _test(self, run_npu=True): for epoch in range(100): - loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "label": label_np - }, - fetch_list=[loss]) + loss_res = exe.run( + main_prog, + feed={"a": a_np, "label": label_np}, + fetch_list=[loss], + ) if epoch % 10 == 0: print("Epoch {} | Loss: {}".format(epoch, loss)) @@ -139,7 +135,6 @@ def test_npu(self): class TestExpand_expand_times_all_one(TestExpand): - def setUp(self): self.set_npu() self.op_type = "expand" diff --git a/python/paddle/fluid/tests/unittests/npu/test_expand_v2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_expand_v2_op_npu.py index d922cef46c83c..0ea52f04d9d76 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_expand_v2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_expand_v2_op_npu.py @@ -29,7 +29,6 @@ # CANN Op Support X: float16, float32, int32, int8 ,uint8 # Situation 1: shape is a list(without tensor) class TestExpandV2NPUOpRank1(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -58,7 +57,6 @@ def test_check_grad(self): class TestExpandV2OpRank2_DimExpanding(TestExpandV2NPUOpRank1): - def init_data(self): self.ori_shape = [120] self.shape = [2, 120] @@ -66,7 +64,6 @@ def init_data(self): class TestExpandV2OpRank2(TestExpandV2NPUOpRank1): - def init_data(self): self.ori_shape = [1, 140] self.shape = [12, 140] @@ -74,7 +71,6 @@ def init_data(self): class TestExpandV2OpRank3_Corner(TestExpandV2NPUOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.shape = (2, 10, 5) @@ -82,7 +78,6 @@ def init_data(self): class TestExpandV2OpRank4(TestExpandV2NPUOpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.shape = (-1, -1, -1, -1) @@ -90,7 +85,6 @@ def init_data(self): class TestExpandV2OpRank5(TestExpandV2NPUOpRank1): - def init_data(self): self.ori_shape = (2, 4, 1, 15) self.shape = (2, -1, 4, -1) @@ -98,7 +92,6 @@ def init_data(self): class TestExpandV2OpRank6(TestExpandV2NPUOpRank1): - def init_data(self): self.ori_shape = (4, 1, 30) self.shape = (2, -1, 4, 30) @@ -107,7 +100,6 @@ def init_data(self): # Situation 2: shape is a list(with tensor) class TestExpandV2OpNPURank1_tensor_attr(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -116,8 +108,9 @@ def setUp(self): self.dtype = np.float32 expand_shapes_tensor = [] for index, ele in enumerate(self.expand_shape): - expand_shapes_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + expand_shapes_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype(self.dtype), @@ -143,9 +136,9 @@ def test_check_grad(self): self.check_grad_with_place(self.place, ['X'], 'Out') -class TestExpandV2OpRank2_Corner_tensor_attr(TestExpandV2OpNPURank1_tensor_attr - ): - +class TestExpandV2OpRank2_Corner_tensor_attr( + TestExpandV2OpNPURank1_tensor_attr +): def init_data(self): self.ori_shape = [12, 14] self.expand_times = [1, 1] @@ -155,7 +148,6 @@ def init_data(self): # Situation 3: shape is a tensor class TestExpandV2NPUOpRank1_tensor(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -189,7 +181,6 @@ def test_check_grad(self): # Situation 4: input x is float16 # skip grad check for float16 class TestExpandV2OpFloat(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -212,7 +203,6 @@ def test_check_output(self): # Situation 5: input x is int32 # skip grad check for int32 class TestExpandV2OpInteger(OpTest): - def init_dtype(self): self.dtype = 'int32' @@ -236,13 +226,11 @@ def test_check_output(self): class TesstExpandV2OpInt64(TestExpandV2OpInteger): - def init_dtype(self): self.dtype = 'int64' class TesstExpandV2OpBool(TestExpandV2OpInteger): - def init_dtype(self): self.dtype = 'bool' @@ -257,11 +245,11 @@ def setUp(self): class TestExpandV2Error(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - paddle.NPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], paddle.NPUPlace(0) + ) shape = [2, 2] self.assertRaises(TypeError, paddle.tensor.expand, x1, shape) x2 = fluid.layers.data(name='x2', shape=[2], dtype="uint8") @@ -273,20 +261,23 @@ def test_errors(self): # Test python API class TestExpandV2API(unittest.TestCase): - def test_static(self): with fluid.program_guard(fluid.Program(), fluid.Program()): input = np.random.random([12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', + shape=[12, 14], + append_batch_size=False, + dtype="float32", + ) positive_2 = fluid.layers.fill_constant([1], "int32", 12) - expand_shape = fluid.layers.data(name="expand_shape", - shape=[2], - append_batch_size=False, - dtype="int32") + expand_shape = fluid.layers.data( + name="expand_shape", + shape=[2], + append_batch_size=False, + dtype="int32", + ) out_1 = paddle.expand(x, shape=[12, 14]) out_2 = paddle.expand(x, shape=[positive_2, 14]) @@ -295,14 +286,14 @@ def test_static(self): g0 = fluid.backward.calc_gradient(out_2, x) exe = fluid.Executor(place=paddle.NPUPlace(0)) - res_1, res_2, res_3 = exe.run(fluid.default_main_program(), - feed={ - "x": - input, - "expand_shape": - np.array([12, 14]).astype("int32") - }, - fetch_list=[out_1, out_2, out_3]) + res_1, res_2, res_3 = exe.run( + fluid.default_main_program(), + feed={ + "x": input, + "expand_shape": np.array([12, 14]).astype("int32"), + }, + fetch_list=[out_1, out_2, out_3], + ) assert np.array_equal(res_1, np.tile(input, (1, 1))) assert np.array_equal(res_2, np.tile(input, (1, 1))) diff --git a/python/paddle/fluid/tests/unittests/npu/test_eye_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_eye_op_npu.py index fe0ba17d20101..e80be96f2cdcb 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_eye_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_eye_op_npu.py @@ -28,7 +28,6 @@ class TestEyeOp(OpTest): - def setUp(self): ''' Test eye op with specified shape @@ -47,14 +46,14 @@ def setUp(self): if self.num_columns == 0: self.attrs = { 'num_rows': self.num_rows, - 'dtype': framework.convert_np_dtype_to_dtype_(self.dtype) + 'dtype': framework.convert_np_dtype_to_dtype_(self.dtype), } self.outputs = {'Out': np.eye(self.num_rows, dtype=self.dtype)} else: self.attrs = { 'num_rows': self.num_rows, 'num_columns': self.num_columns, - 'dtype': framework.convert_np_dtype_to_dtype_(self.dtype) + 'dtype': framework.convert_np_dtype_to_dtype_(self.dtype), } self.outputs = { 'Out': np.eye(self.num_rows, self.num_columns, dtype=self.dtype) @@ -73,41 +72,35 @@ def test_check_output(self): class TestEyeOp1(TestEyeOp): - def initTestCase(self): self.num_rows = 50 class TestEyeOp2(TestEyeOp): - def initTestCase(self): self.num_rows = 50 self.dtype = np.int32 class TestEyeOp3(TestEyeOp): - def initTestCase(self): self.num_rows = 50 self.dtype = np.float16 class TestEyeOp4(TestEyeOp): - def initTestCase(self): self.num_rows = 1 self.num_columns = 99 class TestEyeOp5(TestEyeOp): - def initTestCase(self): self.num_rows = 100 self.num_columns = 100 class TestEyeOp6(TestEyeOp): - def initTestCase(self): self.num_rows = 100 self.num_columns = 100 @@ -115,13 +108,12 @@ def initTestCase(self): class API_TestTensorEye(unittest.TestCase): - def test_out(self): with paddle.static.program_guard(paddle.static.Program()): data = paddle.eye(10) place = paddle.NPUPlace(0) exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[data]) + (result,) = exe.run(fetch_list=[data]) expected_result = np.eye(10, dtype="float32") self.assertEqual((result == expected_result).all(), True) @@ -129,7 +121,7 @@ def test_out(self): data = paddle.eye(10, num_columns=7, dtype="float16") place = paddle.NPUPlace(0) exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[data]) + (result,) = exe.run(fetch_list=[data]) expected_result = np.eye(10, 7, dtype="float16") self.assertEqual((result == expected_result).all(), True) @@ -137,7 +129,7 @@ def test_out(self): data = paddle.eye(10, dtype="int32") place = paddle.NPUPlace(0) exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[data]) + (result,) = exe.run(fetch_list=[data]) expected_result = np.eye(10, dtype="int32") self.assertEqual((result == expected_result).all(), True) @@ -159,8 +151,9 @@ def test_out(self): result = tmp_result expected_result = np.stack(result, axis=0) paddle.enable_static() - self.assertEqual(out.numpy().shape == np.array(expected_result).shape, - True) + self.assertEqual( + out.numpy().shape == np.array(expected_result).shape, True + ) self.assertEqual((out.numpy() == expected_result).all(), True) paddle.disable_static(paddle.NPUPlace(0)) @@ -175,8 +168,9 @@ def test_out(self): result = tmp_result expected_result = np.stack(result, axis=0) paddle.enable_static() - self.assertEqual(out.numpy().shape == np.array(expected_result).shape, - True) + self.assertEqual( + out.numpy().shape == np.array(expected_result).shape, True + ) self.assertEqual((out.numpy() == expected_result).all(), True) def test_errors(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_fill_any_like_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_fill_any_like_op_npu.py index 23f07d299e289..09e38fe0665d9 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_fill_any_like_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_fill_any_like_op_npu.py @@ -26,7 +26,6 @@ class TestFillAnyLikeNPUOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -52,47 +51,40 @@ def test_check_output(self): class TestFillAnyLikeNPUOpInt32(TestFillAnyLikeNPUOp): - def init(self): self.dtype = np.int32 self.value = -1 class TestFillAnyLikeNPUOpInt64(TestFillAnyLikeNPUOp): - def init(self): self.dtype = np.int64 self.value = -1 class TestFillAnyLikeNPUOpFloat32(TestFillAnyLikeNPUOp): - def init(self): self.dtype = np.float32 self.value = 0.09 class TestFillAnyLikeNPUOpFloat16(TestFillAnyLikeNPUOp): - def init(self): self.dtype = np.float16 self.value = 0.05 class TestFillAnyLikeNPUOpValue1(TestFillAnyLikeNPUOp): - def init(self): self.value = 1.0 class TestFillAnyLikeNPUOpValue2(TestFillAnyLikeNPUOp): - def init(self): self.value = 1e-9 class TestFillAnyLikeNPUOpShape(TestFillAnyLikeNPUOp): - def init(self): self.shape = [12, 10] diff --git a/python/paddle/fluid/tests/unittests/npu/test_fill_constant_batch_size_like_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_fill_constant_batch_size_like_op_npu.py index f157845d24b10..0ceaa0faf343e 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_fill_constant_batch_size_like_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_fill_constant_batch_size_like_op_npu.py @@ -27,7 +27,6 @@ class TestFillConstantBatchSizeLike(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -48,11 +47,12 @@ def setUp(self): 'dtype': self.dtype, 'force_cpu': self.force_cpu, 'input_dim_idx': self.input_dim_idx, - 'output_dim_idx': self.output_dim_idx + 'output_dim_idx': self.output_dim_idx, } self.outputs = { - 'Out': np.full(self.output_shape, self.output_value, - self.output_dtype) + 'Out': np.full( + self.output_shape, self.output_value, self.output_dtype + ) } def set_npu(self): @@ -84,7 +84,6 @@ def test_check_output(self): class TestFillConstantBatchSizeLike2(TestFillConstantBatchSizeLike): - def init_shape(self): # test shape self.input_shape = [4, 5, 6, 7] @@ -93,7 +92,6 @@ def init_shape(self): class TestFillConstantBatchSizeLike3(TestFillConstantBatchSizeLike): - def init_value(self): # use 'str_value' rather than 'value' self.value = 3.8 @@ -102,7 +100,6 @@ def init_value(self): class TestFillConstantBatchSizeLike4(TestFillConstantBatchSizeLike): - def init_value(self): # str_value = 'inf' self.value = 3.8 @@ -111,7 +108,6 @@ def init_value(self): class TestFillConstantBatchSizeLike5(TestFillConstantBatchSizeLike): - def init_value(self): # str_value = '-inf' self.value = 3.8 @@ -120,7 +116,6 @@ def init_value(self): class TestFillConstantBatchSizeLike6(TestFillConstantBatchSizeLike): - def init_dtype(self): self.dtype = core.VarDesc.VarType.FP16 self.output_dtype = np.float16 @@ -130,20 +125,17 @@ def test_check_output(self): class TestFillConstantBatchSizeLike7(TestFillConstantBatchSizeLike): - def init_dtype(self): self.dtype = core.VarDesc.VarType.INT32 self.output_dtype = np.int32 class TestFillConstantBatchSizeLike8(TestFillConstantBatchSizeLike): - def init_force_cpu(self): self.force_cpu = True class TestFillConstantBatchSizeLike9(TestFillConstantBatchSizeLike): - def init_shape(self): self.input_shape = [4, 5] self.shape = [123, 92] @@ -177,11 +169,12 @@ def setUp(self): 'dtype': self.dtype, 'force_cpu': self.force_cpu, 'input_dim_idx': self.input_dim_idx, - 'output_dim_idx': self.output_dim_idx + 'output_dim_idx': self.output_dim_idx, } self.outputs = { - 'Out': np.full(self.output_shape, self.output_value, - self.output_dtype) + 'Out': np.full( + self.output_shape, self.output_value, self.output_dtype + ) } def init_shape(self): @@ -191,7 +184,8 @@ def init_shape(self): class TestFillConstantBatchSizeLikeLodTensor2( - TestFillConstantBatchSizeLikeLodTensor): + TestFillConstantBatchSizeLikeLodTensor +): # test LodTensor with 'input_dim_idx' != 0 def init_shape(self): self.input_shape = [10, 20] diff --git a/python/paddle/fluid/tests/unittests/npu/test_fill_constant_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_fill_constant_op_npu.py index 3d0f770b4eb64..d3087514c6374 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_fill_constant_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_fill_constant_op_npu.py @@ -27,7 +27,6 @@ class TestFillConstant(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -49,7 +48,6 @@ def test_check_output(self): class TestFillConstantInt(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -59,7 +57,7 @@ def setUp(self): self.attrs = { 'shape': [123, 92], 'value': 1, - 'dtype': core.VarDesc.VarType.INT32 + 'dtype': core.VarDesc.VarType.INT32, } self.outputs = {'Out': np.full((123, 92), 1).astype(self.dtype)} @@ -74,7 +72,6 @@ def test_check_output(self): class TestFillConstantInt64(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -84,7 +81,7 @@ def setUp(self): self.attrs = { 'shape': [123, 92], 'value': 1, - 'dtype': core.VarDesc.VarType.INT64 + 'dtype': core.VarDesc.VarType.INT64, } self.outputs = {'Out': np.full((123, 92), 1).astype(self.dtype)} @@ -99,7 +96,6 @@ def test_check_output(self): class TestFillConstantFP16(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -109,7 +105,7 @@ def setUp(self): self.attrs = { 'shape': [123, 92], 'value': 1.0, - 'dtype': core.VarDesc.VarType.FP16 + 'dtype': core.VarDesc.VarType.FP16, } self.outputs = {'Out': np.full((123, 92), 1.0).astype(self.dtype)} @@ -124,7 +120,6 @@ def test_check_output(self): class TestFillConstantBool(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -134,7 +129,7 @@ def setUp(self): self.attrs = { 'shape': [123, 92], 'value': True, - 'dtype': core.VarDesc.VarType.BOOL + 'dtype': core.VarDesc.VarType.BOOL, } self.outputs = {'Out': np.full((123, 92), True).astype(self.dtype)} @@ -149,7 +144,6 @@ def test_check_output(self): class TestFillConstantWithPlaceType(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/npu/test_fill_zeros_like_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_fill_zeros_like_op_npu.py index 3095c18179256..59b0c43c15768 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_fill_zeros_like_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_fill_zeros_like_op_npu.py @@ -24,7 +24,6 @@ class TestFillZerosLikeOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -44,31 +43,26 @@ def test_check_output(self): class TestFillZerosLikeOpBool(TestFillZerosLikeOp): - def init_dtype(self): self.dtype = np.bool_ class TestFillZerosLikeOpFp16(TestFillZerosLikeOp): - def init_dtype(self): self.dtype = np.float16 class TestFillZerosLikeOpFp64(TestFillZerosLikeOp): - def init_dtype(self): self.dtype = np.float64 class TestFillZerosLikeOpInt32(TestFillZerosLikeOp): - def init_dtype(self): self.dtype = np.int32 class TestFillZerosLikeOpInt64(TestFillZerosLikeOp): - def init_dtype(self): self.dtype = np.int64 diff --git a/python/paddle/fluid/tests/unittests/npu/test_flags_check_nan_inf_npu.py b/python/paddle/fluid/tests/unittests/npu/test_flags_check_nan_inf_npu.py index 69c586fb2d884..88949de237cd2 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_flags_check_nan_inf_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_flags_check_nan_inf_npu.py @@ -27,7 +27,6 @@ class TestCheckFiniteAndUnscale(unittest.TestCase): - def setUp(self): fluid.set_flags({'FLAGS_check_nan_inf': True}) diff --git a/python/paddle/fluid/tests/unittests/npu/test_flatten2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_flatten2_op_npu.py index f384620f2a20d..caf370ed09826 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_flatten2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_flatten2_op_npu.py @@ -25,7 +25,6 @@ class TestFlatten2Op(OpTest): - def setUp(self): self.set_npu() self.op_type = "flatten2" @@ -35,7 +34,7 @@ def setUp(self): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype("float32") + "XShape": np.random.random(self.in_shape).astype("float32"), } def set_npu(self): @@ -57,7 +56,6 @@ def init_attrs(self): class TestFlatten2OpWithCornerAxis(TestFlatten2Op): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.axis = 0 @@ -65,7 +63,6 @@ def init_test_case(self): class TestFlatten2OpWithDefaultAxis(TestFlatten2Op): - def init_test_case(self): self.in_shape = (10, 2, 2, 3) self.new_shape = (10, 12) @@ -75,7 +72,6 @@ def init_attrs(self): class TestFlatten2OpSixDims(TestFlatten2Op): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 diff --git a/python/paddle/fluid/tests/unittests/npu/test_flatten_contiguous_range_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_flatten_contiguous_range_op_npu.py index 7c1b6f1776a63..1ee72cbac8c09 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_flatten_contiguous_range_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_flatten_contiguous_range_op_npu.py @@ -25,7 +25,6 @@ class TestFlattenOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "flatten_contiguous_range" @@ -39,7 +38,7 @@ def setUp(self): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype("float32") + "XShape": np.random.random(self.in_shape).astype("float32"), } def set_npu(self): @@ -55,17 +54,16 @@ def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 self.stop_axis = -1 - self.new_shape = (120) + self.new_shape = 120 def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 1 @@ -75,12 +73,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_2(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -90,12 +87,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_3(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -105,12 +101,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_4(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = -2 @@ -120,12 +115,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_5(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 2 @@ -135,12 +129,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.start_axis = 3 @@ -150,12 +143,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_Float32(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -166,12 +158,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_int32(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -182,7 +173,7 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -190,7 +181,6 @@ def test_check_grad(self): class TestFlattenOp_uint8(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -201,7 +191,7 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -209,7 +199,6 @@ def test_check_grad(self): class TestFlattenOp_int8(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -220,7 +209,7 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -228,7 +217,6 @@ def test_check_grad(self): class TestFlattenOp_int64(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -239,7 +227,7 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -247,45 +235,58 @@ def test_check_grad(self): class TestFlatten2OpError(unittest.TestCase): - def test_errors(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_ValueError1(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) out = paddle.flatten(x_var, start_axis=2, stop_axis=1) self.assertRaises(ValueError, test_ValueError1) def test_ValueError2(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=10, stop_axis=1) self.assertRaises(ValueError, test_ValueError2) def test_ValueError3(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=2, stop_axis=10) self.assertRaises(ValueError, test_ValueError3) def test_type(): # dtype must be float32, float64, int8, int32, int64, uint8. - x2 = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x2 = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x2 = x2.astype('float16') - x2_var = paddle.fluid.data(name='x2', - shape=[3, 2, 4, 5], - dtype='float16') + x2_var = paddle.fluid.data( + name='x2', shape=[3, 2, 4, 5], dtype='float16' + ) paddle.flatten(x2_var) self.assertRaises(TypeError, test_type) @@ -297,7 +298,6 @@ def test_InputError(): class TestStaticFlattenPythonAPI(unittest.TestCase): - def execute_api(self, x, start_axis=0, stop_axis=-1): return paddle.flatten(x, start_axis, stop_axis) @@ -307,9 +307,9 @@ def test_static_api(self): main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, paddle.static.Program()): - x = paddle.static.data(name="x", - shape=[2, 3, 4, 4], - dtype='float32') + x = paddle.static.data( + name="x", shape=[2, 3, 4, 4], dtype='float32' + ) out = self.execute_api(x, start_axis=-2, stop_axis=-1) exe = paddle.static.Executor(place=paddle.NPUPlace(0)) @@ -318,17 +318,22 @@ def test_static_api(self): class TestStaticInplaceFlattenPythonAPI(TestStaticFlattenPythonAPI): - def execute_api(self, x, start_axis=0, stop_axis=-1): return x.flatten_(start_axis, stop_axis) class TestFlattenPython(unittest.TestCase): - def test_python_api(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_InputError(): diff --git a/python/paddle/fluid/tests/unittests/npu/test_float_status_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_float_status_op_npu.py index 3db98ea8f3849..fdc9b91dd4dfa 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_float_status_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_float_status_op_npu.py @@ -23,7 +23,6 @@ class TestGetFloatStatusOp(unittest.TestCase): - def setUp(self): device = paddle.set_device('npu') @@ -64,7 +63,6 @@ def test_not_contains_nan_inf(self): class TestClearFloatStatusOp(unittest.TestCase): - def setUp(self): device = paddle.set_device('npu') diff --git a/python/paddle/fluid/tests/unittests/npu/test_gather_nd_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_gather_nd_op_npu.py index fa6da7817de02..4d66c28d6fa40 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_gather_nd_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_gather_nd_op_npu.py @@ -23,11 +23,11 @@ def gather_nd_grad(x, index): - dout_shape = index.shape[:-1] + x.shape[index.shape[-1]:] + dout_shape = index.shape[:-1] + x.shape[index.shape[-1] :] numel = 1 for i in dout_shape: numel = numel * i - dout = np.full(dout_shape, 1. / numel) + dout = np.full(dout_shape, 1.0 / numel) dx = np.full_like(x, 0) index = tuple(index.reshape(-1, index.shape[-1]).T) @@ -37,9 +37,8 @@ def gather_nd_grad(x, index): def test_class1(op_type, typename): - class TestGatherNdOpWithEmptyIndex(OpTest): - #Index has empty element, which means copy entire tensor + # Index has empty element, which means copy entire tensor def setUp(self): self.set_npu() @@ -48,7 +47,7 @@ def setUp(self): xnp = np.random.random((5, 20)).astype(typename) self.inputs = { 'X': xnp, - 'Index': np.array([[], []]).astype("int32") + 'Index': np.array([[], []]).astype("int32"), } self.outputs = { 'Out': np.vstack((xnp[np.newaxis, :], xnp[np.newaxis, :])) @@ -72,9 +71,7 @@ def test_check_grad(self): def test_class2(op_type, typename): - class TestGatherNdOpWithIndex1(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -101,9 +98,8 @@ def test_check_grad(self): def test_class3(op_type, typename): - class TestGatherNdOpWithLowIndex(OpTest): - #Index has low rank, X has high rank + # Index has low rank, X has high rank def setUp(self): self.set_npu() @@ -126,9 +122,9 @@ def test_check_grad(self): if typename == "float16" or typename == "int64": self.__class__.no_need_check_grad = True else: - self.check_grad_with_place(self.place, ['X'], - 'Out', - user_defined_grads=[self.x_grad]) + self.check_grad_with_place( + self.place, ['X'], 'Out', user_defined_grads=[self.x_grad] + ) cls_name = "{0}_{1}_3".format(op_type, typename) TestGatherNdOpWithLowIndex.__name__ = cls_name @@ -136,9 +132,8 @@ def test_check_grad(self): def test_class4(op_type, typename): - class TestGatherNdOpIndex1(OpTest): - #Index has low rank, X has high rank + # Index has low rank, X has high rank def setUp(self): self.set_npu() @@ -169,9 +164,8 @@ def test_check_grad(self): def test_class5(op_type, typename): - class TestGatherNdOpWithSameIndexAsX(OpTest): - #Index has same rank as X's rank + # Index has same rank as X's rank def setUp(self): self.set_npu() @@ -181,7 +175,7 @@ def setUp(self): index = np.array([[1, 1], [2, 1]]).astype("int64") self.inputs = {'X': xnp, 'Index': index} - self.outputs = {'Out': xnp[tuple(index.T)]} #[25, 22] + self.outputs = {'Out': xnp[tuple(index.T)]} # [25, 22] def set_npu(self): self.__class__.use_npu = True @@ -201,9 +195,8 @@ def test_check_grad(self): def test_class6(op_type, typename): - class TestGatherNdOpWithHighRankSame(OpTest): - #Both Index and X have high rank, and Rank(Index) = Rank(X) + # Both Index and X have high rank, and Rank(Index) = Rank(X) def setUp(self): self.set_npu() @@ -211,8 +204,9 @@ def setUp(self): self.op_type = "gather_nd" shape = (5, 2, 3, 1, 10) xnp = np.random.rand(*shape).astype(typename) - index = np.vstack([np.random.randint(0, s, size=2) - for s in shape]).T + index = np.vstack( + [np.random.randint(0, s, size=2) for s in shape] + ).T self.inputs = {'X': xnp, 'Index': index.astype("int32")} self.outputs = {'Out': xnp[tuple(index.T)]} @@ -235,9 +229,8 @@ def test_check_grad(self): def test_class7(op_type, typename): - class TestGatherNdOpWithHighRankDiff(OpTest): - #Both Index and X have high rank, Rank(Index) < Rank(X) + # Both Index and X have high rank, Rank(Index) < Rank(X) def setUp(self): self.set_npu() @@ -246,7 +239,8 @@ def setUp(self): shape = (2, 3, 4, 1, 10) xnp = np.random.rand(*shape).astype(typename) index = np.vstack( - [np.random.randint(0, s, size=200) for s in shape]).T + [np.random.randint(0, s, size=200) for s in shape] + ).T index_re = index.reshape([20, 5, 2, 5]) self.inputs = {'X': xnp, 'Index': index_re.astype("int32")} @@ -270,7 +264,6 @@ def test_check_grad(self): class TestGatherNdAPI(unittest.TestCase): - def test_imperative(self): paddle.disable_static() input_1 = np.array([[1, 2], [3, 4], [5, 6]]) diff --git a/python/paddle/fluid/tests/unittests/npu/test_gather_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_gather_op_npu.py index bedc6f1efc7f9..5e783b952af9c 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_gather_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_gather_op_npu.py @@ -34,7 +34,6 @@ def gather_numpy(x, index, axis): class TestGatherOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -43,7 +42,7 @@ def setUp(self): xnp = np.random.random(self.x_shape).astype(self.x_type) self.inputs = { 'X': xnp, - 'Index': np.array(self.index).astype(self.index_type) + 'Index': np.array(self.index).astype(self.index_type), } self.outputs = {'Out': self.inputs["X"][self.inputs["Index"]]} @@ -72,19 +71,17 @@ def config(self): class TestCase1(TestGatherOp): - def config(self): """ For one dimension input """ - self.x_shape = (100) + self.x_shape = 100 self.x_type = "float32" self.index = [1, 3, 5] self.index_type = "int32" class API_TestGather(unittest.TestCase): - def test_out1(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[-1, 2], dtype='float32') @@ -94,17 +91,16 @@ def test_out1(self): exe = fluid.Executor(place) input = np.array([[1, 2], [3, 4], [5, 6]]) index_1 = np.array([1, 2]) - result, = exe.run(feed={ - "data1": input, - "index": index_1 - }, - fetch_list=[out]) + (result,) = exe.run( + feed={"data1": input, "index": index_1}, fetch_list=[out] + ) expected_output = np.array([[3, 4], [5, 6]]) np.testing.assert_allclose(result, expected_output, rtol=1e-5) def test_out2(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data('x', shape=[-1, 2], dtype='float32') index = paddle.fluid.data('index', shape=[-1, 1], dtype='int32') out = paddle.gather(x, index) @@ -112,17 +108,14 @@ def test_out2(self): exe = paddle.static.Executor(place) x_np = np.array([[1, 2], [3, 4], [5, 6]]).astype('float32') index_np = np.array([1, 1]).astype('int32') - result, = exe.run(feed={ - "x": x_np, - "index": index_np - }, - fetch_list=[out]) + (result,) = exe.run( + feed={"x": x_np, "index": index_np}, fetch_list=[out] + ) expected_output = gather_numpy(x_np, index_np, axis=0) np.testing.assert_allclose(result, expected_output, rtol=1e-5) class TestGatherGrad(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -135,9 +128,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[8192, 768], dtype='float32') - index = paddle.static.data(name="index", - shape=[1232, 1], - dtype='int32') + index = paddle.static.data( + name="index", shape=[1232, 1], dtype='int32' + ) a.stop_gradient = False b = paddle.gather(a, index) @@ -156,15 +149,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "index": index_np - }, - fetch_list=[b, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "index": index_np}, + fetch_list=[b, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res[0])) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res[0] + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_gaussian_random_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_gaussian_random_op_npu.py index 5f6b756f17554..326945bb47826 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_gaussian_random_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_gaussian_random_op_npu.py @@ -26,7 +26,6 @@ class TestNPUGaussianRandomOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "gaussian_random" @@ -39,7 +38,7 @@ def setUp(self): "mean": self.mean, "std": self.std, "seed": 10, - "use_mkldnn": self.use_mkldnn + "use_mkldnn": self.use_mkldnn, } paddle.seed(10) @@ -47,7 +46,7 @@ def setUp(self): def set_attrs(self): self.mean = 1.0 - self.std = 2. + self.std = 2.0 def set_npu(self): self.__class__.use_npu = True @@ -68,12 +67,13 @@ def verify_output(self, outs): hist2, _ = np.histogram(data, range=(-3, 5)) hist2 = hist2.astype("float32") hist2 /= float(outs[0].size) - np.testing.assert_allclose(hist, - hist2, - rtol=0, - atol=0.01, - err_msg="hist: " + str(hist) + " hist2: " + - str(hist2)) + np.testing.assert_allclose( + hist, + hist2, + rtol=0, + atol=0.01, + err_msg="hist: " + str(hist) + " hist2: " + str(hist2), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/npu/test_gelu_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_gelu_op_npu.py index 2a03c36622d2c..0c2d163becadd 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_gelu_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_gelu_op_npu.py @@ -32,7 +32,6 @@ def np_gelu(x): class TestGelu(OpTest): - def setUp(self): self.set_npu() self.op_type = "gelu" @@ -57,13 +56,12 @@ def test_check_output(self): self.check_output_with_place(self.place, atol=1e-3) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.007) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.007 + ) class TestGeluFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "gelu" @@ -90,7 +88,6 @@ def test_check_output(self): class TestGeluNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -105,9 +102,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) @@ -131,16 +128,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_group_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_group_norm_op_npu.py index e995e7f9475e9..06d0f5dd1d029 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_group_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_group_norm_op_npu.py @@ -46,7 +46,6 @@ def group_norm_naive(x, scale, bias, epsilon, groups, data_layout): class TestGroupNormOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): @@ -58,9 +57,9 @@ def test_x_type(): self.assertRaises(TypeError, test_x_type) def test_x_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[2, 100, 3, 5], - dtype='int32') + x2 = fluid.layers.data( + name='x2', shape=[2, 100, 3, 5], dtype='int32' + ) groups = 2 fluid.layers.group_norm(x2, groups) @@ -68,7 +67,6 @@ def test_x_dtype(): class TestGroupNormOp(OpTest): - def setUp(self): self.set_npu() self.op_type = 'group_norm' @@ -89,15 +87,19 @@ def setUp(self): input = np.transpose(input, (0, 2, 3, 1)) scale = np.random.random([self.shape[1]]).astype(self.dtype) bias = np.random.random([self.shape[1]]).astype(self.dtype) - output, mean, var = group_norm_naive(input, scale, bias, - self.attrs['epsilon'], - self.attrs['groups'], - self.data_format) + output, mean, var = group_norm_naive( + input, + scale, + bias, + self.attrs['epsilon'], + self.attrs['groups'], + self.data_format, + ) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(input), 'Scale': OpTest.np_dtype_to_fluid_dtype(scale), - 'Bias': OpTest.np_dtype_to_fluid_dtype(bias) + 'Bias': OpTest.np_dtype_to_fluid_dtype(bias), } self.outputs = {'Y': output, 'Mean': mean, 'Variance': var} self.attrs['data_layout'] = self.data_format @@ -120,67 +122,65 @@ def test_check_grad(self): output_names = 'Y' no_grad_set = set() cpu_place = fluid.CPUPlace() - cpu_grads = self._get_gradient(inputs_to_check, cpu_place, output_names, - no_grad_set) - npu_grads = self._get_gradient(inputs_to_check, self.place, - output_names, no_grad_set) - - self._assert_is_close(cpu_grads, npu_grads, inputs_to_check, - self.max_relative_error, - "Gradient Check between places") + cpu_grads = self._get_gradient( + inputs_to_check, cpu_place, output_names, no_grad_set + ) + npu_grads = self._get_gradient( + inputs_to_check, self.place, output_names, no_grad_set + ) + + self._assert_is_close( + cpu_grads, + npu_grads, + inputs_to_check, + self.max_relative_error, + "Gradient Check between places", + ) def init_test_case(self): pass class TestGroupNormOp1(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 class TestGroupNormOp2(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 class TestGroupNormOpBigEps1(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 self.attrs['epsilon'] = 0.5 class TestGroupNormOpBigEps2(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 self.attrs['epsilon'] = 0.5 class TestGroupNormOpBigEps3(TestGroupNormOp): - def init_test_case(self): self.attrs['epsilon'] = 0.5 class TestGroupNormOp1_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 self.data_format = "NHWC" class TestGroupNormOp2_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 self.data_format = "NHWC" class TestGroupNormOpBigEps1_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 self.attrs['epsilon'] = 0.5 @@ -188,7 +188,6 @@ def init_test_case(self): class TestGroupNormOpBigEps2_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 self.attrs['epsilon'] = 0.5 @@ -196,20 +195,17 @@ def init_test_case(self): class TestGroupNormOpBigEps3_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['epsilon'] = 0.5 self.data_format = "NHWC" class TestGroupNormOpFP16(TestGroupNormOp): - def init_dtype(self): self.dtype = np.float16 class TestGroupNormOpFP16_With_NHWC(TestGroupNormOp): - def init_dtype(self): self.dtype = np.float16 @@ -223,9 +219,9 @@ def test_exception(self): data = fluid.data(name='data', shape=[None, 3, 3, 4], dtype="float64") def attr_data_format(): - out = fluid.layers.group_norm(input=data, - groups=2, - data_layout="NDHW") + out = fluid.layers.group_norm( + input=data, groups=2, data_layout="NDHW" + ) self.assertRaises(ValueError, attr_data_format) diff --git a/python/paddle/fluid/tests/unittests/npu/test_hard_sigmoid_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_hard_sigmoid_op_npu.py index c7c69eef5e59a..44155e4388062 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_hard_sigmoid_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_hard_sigmoid_op_npu.py @@ -26,11 +26,10 @@ def ref_hardsigmoid(x, slope=0.166666666666667, offset=0.5): - return np.maximum(np.minimum(x * slope + offset, 1.), 0.).astype(x.dtype) + return np.maximum(np.minimum(x * slope + offset, 1.0), 0.0).astype(x.dtype) class TestNPUHardSigmoid(OpTest): - def setUp(self): paddle.enable_static() @@ -41,7 +40,7 @@ def setUp(self): x = np.random.uniform(-5, 5, [10, 12]).astype(self.dtype) lower_threshold = -self.offset / self.slope - upper_threshold = (1. - self.offset) / self.slope + upper_threshold = (1.0 - self.offset) / self.slope # Same reason as TestAbs delta = 0.005 @@ -73,21 +72,18 @@ def set_attrs(self): class TestNPUHardSigmoid2(TestNPUHardSigmoid): - def set_attrs(self): self.slope = 0.2 self.offset = 0.5 class TestNPUHardSigmoid3(TestNPUHardSigmoid): - def set_attrs(self): self.slope = 0.2 self.offset = 0.4 class TestNPUHardSigmoidFp16(TestNPUHardSigmoid): - def test_check_output(self): self.check_output_with_place(self.place, atol=1e-3) @@ -144,14 +140,14 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.hardsigmoid, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.hardsigmoid, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.hardsigmoid(x_fp16) diff --git a/python/paddle/fluid/tests/unittests/npu/test_hard_swish_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_hard_swish_op_npu.py index 3b0b5a712c20c..86c4bf5402d1b 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_hard_swish_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_hard_swish_op_npu.py @@ -24,15 +24,17 @@ def ref_hard_swish_grad(x, threshold=6.0, scale=6.0, offset=3.0): - dout = np.full_like(x, fill_value=1. / x.size) + dout = np.full_like(x, fill_value=1.0 / x.size) tmp = ((x + offset) < threshold).astype(x.dtype) - dx = dout * (((x + offset) > 0).astype(x.dtype) * - (2 * x + offset) * tmp / scale + 1.0 - tmp) + dx = dout * ( + ((x + offset) > 0).astype(x.dtype) * (2 * x + offset) * tmp / scale + + 1.0 + - tmp + ) return dx class TestHardSwishNPU(OpTest): - def setUp(self): paddle.enable_static() @@ -45,13 +47,12 @@ def setUp(self): threshold = 6.0 scale = 6.0 offset = 3.0 - #the same with TestAbs + # the same with TestAbs x[np.abs(x + offset) < 0.005] = 0.02 x[np.abs(x - threshold + offset) < 0.005] = threshold - offset + 0.02 out = ( - x * - (np.minimum(np.maximum(x + offset, 0.), threshold) / scale)).astype( - self.dtype) + x * (np.minimum(np.maximum(x + offset, 0.0), threshold) / scale) + ).astype(self.dtype) self.x_grad = ref_hard_swish_grad(x, threshold, scale, offset) self.inputs = {'X': x} @@ -72,13 +73,12 @@ def test_check_grad(self): # can't satisfy the default precision requirement # when compared with numeric_grads, but the results on # NPU and CPU are same (verified in TestHardSwishNPUWithCPU) - self.check_grad_with_place(self.place, ['X'], - 'Out', - user_defined_grads=[self.x_grad]) + self.check_grad_with_place( + self.place, ['X'], 'Out', user_defined_grads=[self.x_grad] + ) class TestHardSwishNPUFp16(TestHardSwishNPU): - def test_check_output(self): self.check_output_with_place(self.place) @@ -88,7 +88,6 @@ def init_dtype(self): # test the result of hard_swish and hard_swish_grad on CPU and NPU class TestHardSwishNPUWithCPU(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -116,17 +115,33 @@ def test_check_output_and_grad_npu(self): np.testing.assert_allclose( self.out_y.numpy(), y.numpy(), - err_msg="Output of NPU HardSwish forward has diff at " + - str(self.place) + "\nExpect " + str(self.out_y) + "\n" + "But Got" + - str(y) + " in class " + self.__class__.__name__ + ".", - rtol=1e-5) + err_msg="Output of NPU HardSwish forward has diff at " + + str(self.place) + + "\nExpect " + + str(self.out_y) + + "\n" + + "But Got" + + str(y) + + " in class " + + self.__class__.__name__ + + ".", + rtol=1e-5, + ) np.testing.assert_allclose( self.out_g.numpy(), data.grad.numpy(), - err_msg="Output of NPU HardSwish backward has diff at " + - str(self.place) + "\nExpect " + str(self.out_g) + "\n" + "But Got" + - str(data.grad) + " in class " + self.__class__.__name__ + ".", - rtol=1e-5) + err_msg="Output of NPU HardSwish backward has diff at " + + str(self.place) + + "\nExpect " + + str(self.out_g) + + "\n" + + "But Got" + + str(data.grad) + + " in class " + + self.__class__.__name__ + + ".", + rtol=1e-5, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_huber_loss_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_huber_loss_op_npu.py index d9a83d74e1f12..0e81f00de62c8 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_huber_loss_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_huber_loss_op_npu.py @@ -33,10 +33,10 @@ def huber_loss_forward(val, delta): return delta * (abs_val - 0.5 * delta) -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestHuberLossOp(OpTest): - def setUp(self): self.set_npu() self.op_type = 'huber_loss' @@ -50,11 +50,11 @@ def setUp(self): def set_inputs(self): shape = self.set_shape() - x = np.random.uniform(0, 1., shape).astype(self.dtype) - y = np.random.uniform(0, 1., shape).astype(self.dtype) + x = np.random.uniform(0, 1.0, shape).astype(self.dtype) + y = np.random.uniform(0, 1.0, shape).astype(self.dtype) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } def set_attrs(self): @@ -64,8 +64,9 @@ def set_outputs(self): delta = self.attrs['delta'] shape = self.set_shape() residual = self.inputs['Y'] - self.inputs['X'] - loss = np.vectorize(huber_loss_forward)(residual, - delta).astype(self.dtype) + loss = np.vectorize(huber_loss_forward)(residual, delta).astype( + self.dtype + ) self.outputs = {'Residual': residual, 'Out': loss.reshape(shape)} def set_shape(self): @@ -84,46 +85,48 @@ def test_check_grad_normal(self): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - max_relative_error=0.008, - no_grad_set=set("residual")) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + max_relative_error=0.008, + no_grad_set=set("residual"), + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.008, - no_grad_set=set('residual')) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + max_relative_error=0.008, + no_grad_set=set('residual'), + ) def TestHuberLossOp1(TestHuberLossOp): - def set_shape(self): - return (64) + return 64 def TestHuberLossOp2(TestHuberLossOp): - def set_shape(self): return (6, 6) def TestHuberLossOp3(TestHuberLossOp): - def set_shape(self): return (6, 6, 1) def TestHuberLossOpFP16(TestHuberLossOp): - def init_dtype(self): self.dtype = np.float16 -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestHuberLossOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input and label must be Variable @@ -138,10 +141,12 @@ def test_errors(self): # the dtype of input and label must be float32 or float64 xw2 = fluid.data(name='xw2', shape=[None, 6], dtype="int32") lw2 = fluid.data(name='lw2', shape=[None, 6], dtype="int32") - self.assertRaises(TypeError, fluid.layers.huber_loss, xw2, lr, - delta) - self.assertRaises(TypeError, fluid.layers.huber_loss, xr, lw2, - delta) + self.assertRaises( + TypeError, fluid.layers.huber_loss, xw2, lr, delta + ) + self.assertRaises( + TypeError, fluid.layers.huber_loss, xr, lw2, delta + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_increment_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_increment_op_npu.py index c84abc5d479e5..39f11737bce6c 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_increment_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_increment_op_npu.py @@ -29,7 +29,6 @@ class TestIncrement(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(NPUPlace) @@ -37,8 +36,9 @@ def setUp(self): self.init_dtype() self.inputs = { - 'X': - OpTest.np_dtype_to_fluid_dtype(np.array([1]).astype(self.dtype)), + 'X': OpTest.np_dtype_to_fluid_dtype( + np.array([1]).astype(self.dtype) + ), } self.attrs = {"Step": 1} @@ -56,7 +56,6 @@ def test_check_output(self): class TestIncrementFP16(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(NPUPlace) @@ -64,8 +63,9 @@ def setUp(self): self.init_dtype() self.inputs = { - 'X': - OpTest.np_dtype_to_fluid_dtype(np.array([1]).astype(self.dtype)), + 'X': OpTest.np_dtype_to_fluid_dtype( + np.array([1]).astype(self.dtype) + ), } self.pre_input_id = id(self.inputs['X']) @@ -83,7 +83,6 @@ def test_check_output(self): class TestIncrementINT64(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(NPUPlace) @@ -91,8 +90,9 @@ def setUp(self): self.init_dtype() self.inputs = { - 'X': - OpTest.np_dtype_to_fluid_dtype(np.array([1]).astype(self.dtype)), + 'X': OpTest.np_dtype_to_fluid_dtype( + np.array([1]).astype(self.dtype) + ), } self.pre_input_id = id(self.inputs['X']) @@ -110,7 +110,6 @@ def test_check_output(self): class TestIncrementInplace(unittest.TestCase): - def test_npu(self): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -129,9 +128,13 @@ def test_npu(self): exe = paddle.static.Executor(place) exe.run(startup_prog) - b_value = exe.run(main_prog, feed={ - "a": a_np, - }, fetch_list=[b]) + b_value = exe.run( + main_prog, + feed={ + "a": a_np, + }, + fetch_list=[b], + ) print('input a id is : {}'.format(id(a))) print('input b id is : {}'.format(id(b))) diff --git a/python/paddle/fluid/tests/unittests/npu/test_index_sample_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_index_sample_op_npu.py index e64d556ebdc6d..5883ef7b56701 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_index_sample_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_index_sample_op_npu.py @@ -26,7 +26,6 @@ class TestIndexSampleOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -35,10 +34,9 @@ def setUp(self): self.op_type = "index_sample" self.config() xnp = np.random.random(self.x_shape).astype(self.dtype) - indexnp = np.random.randint(low=0, - high=self.x_shape[1], - size=self.index_shape).astype( - self.index_type) + indexnp = np.random.randint( + low=0, high=self.x_shape[1], size=self.index_shape + ).astype(self.index_type) self.inputs = {'X': xnp, 'Index': indexnp} index_array = [] for i in range(self.index_shape[0]): @@ -65,7 +63,6 @@ def config(self): class TestCase1(TestIndexSampleOp): - def config(self): """ For one dimension input @@ -77,7 +74,6 @@ def config(self): class TestCase2(TestIndexSampleOp): - def config(self): """ For int64_t index type @@ -89,7 +85,6 @@ def config(self): class TestCase3(TestIndexSampleOp): - def config(self): """ For int index type @@ -101,7 +96,6 @@ def config(self): class TestCase4(TestIndexSampleOp): - def config(self): """ For int64 index type @@ -113,7 +107,6 @@ def config(self): class TestCase5(TestIndexSampleOp): - def config(self): """ For float16 x type @@ -129,7 +122,6 @@ def test_check_grad(self): class TestCase6(TestCase5): - def config(self): """ For int32 x type @@ -142,7 +134,6 @@ def config(self): class TestCase7(TestCase5): - def config(self): """ For int64 x type @@ -155,7 +146,6 @@ def config(self): class TestIndexSampleShape(unittest.TestCase): - def test_shape(self): paddle.enable_static() # create x value @@ -166,8 +156,9 @@ def test_shape(self): # create index value index_shape = (2, 3) index_type = "int32" - index_np = np.random.randint(low=0, high=x_shape[1], - size=index_shape).astype(index_type) + index_np = np.random.randint( + low=0, high=x_shape[1], size=index_shape + ).astype(index_type) x = fluid.data(name='x', shape=[-1, 5], dtype='float32') index = fluid.data(name='index', shape=[-1, 3], dtype='int32') @@ -182,18 +173,24 @@ def test_shape(self): class TestIndexSampleDynamic(unittest.TestCase): - def test_result(self): with fluid.dygraph.guard(paddle.NPUPlace(0)): - x = paddle.to_tensor([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [9.0, 10.0, 11.0, 12.0]], - dtype='float32') - index = paddle.to_tensor([[0, 1, 2], [1, 2, 3], [0, 0, 0]], - dtype='int32') + x = paddle.to_tensor( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ], + dtype='float32', + ) + index = paddle.to_tensor( + [[0, 1, 2], [1, 2, 3], [0, 0, 0]], dtype='int32' + ) out_z1 = paddle.index_sample(x, index) - except_output = np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0], - [9.0, 9.0, 9.0]]) + except_output = np.array( + [[1.0, 2.0, 3.0], [6.0, 7.0, 8.0], [9.0, 9.0, 9.0]] + ) assert out_z1.numpy().all() == except_output.all() diff --git a/python/paddle/fluid/tests/unittests/npu/test_index_select_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_index_select_op_npu.py index 7050afa441101..39b0a01189313 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_index_select_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_index_select_op_npu.py @@ -26,7 +26,6 @@ class TestNPUIndexSelect(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -34,15 +33,17 @@ def setUp(self): self.config() x_np = np.random.random(self.x_shape).astype(self.x_type) - index_np = np.random.randint(low=0, - high=self.x_shape[self.dim], - size=self.index_size, - dtype=self.index_type) + index_np = np.random.randint( + low=0, + high=self.x_shape[self.dim], + size=self.index_size, + dtype=self.index_type, + ) # compute real output as baseline. - outer_loop = np.prod(self.x_shape[:self.dim]) + outer_loop = np.prod(self.x_shape[: self.dim]) outer_loop = outer_loop.astype(self.index_type) - x_reshape = [outer_loop] + list(self.x_shape[self.dim:]) + x_reshape = [outer_loop] + list(self.x_shape[self.dim :]) x_np_reshape = np.reshape(x_np, tuple(x_reshape)) out_list = [] @@ -76,7 +77,6 @@ def config(self): class TestNPUIndexSelectCase2(TestNPUIndexSelect): - def config(self): self.dim = -2 self.x_type = np.float32 @@ -86,7 +86,6 @@ def config(self): class TestNPUIndexSelectCase3(TestNPUIndexSelect): - def config(self): self.dim = 0 self.x_type = np.float32 @@ -96,7 +95,6 @@ def config(self): class TestNPUIndexSelectCase4(TestNPUIndexSelect): - def config(self): self.dim = -1 self.x_type = np.float32 @@ -106,10 +104,14 @@ def config(self): class TestNPUIndexSelectAPI(unittest.TestCase): - def input_data(self): - self.data_x = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [9.0, 10.0, 11.0, 12.0]]).astype('float32') + self.data_x = np.array( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ] + ).astype('float32') self.data_index = np.array([0, 1, 1]).astype('int32') def test_index_select_api(self): @@ -123,14 +125,14 @@ def test_index_select_api(self): index = paddle.static.data(name='index', shape=[3], dtype='int32') z = paddle.index_select(x, index, axis=1) exe = paddle.static.Executor(paddle.NPUPlace(0)) - res, = exe.run(feed={ - 'x': self.data_x, - 'index': self.data_index - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], - [9.0, 10.0, 10.0]]).astype('float32') + (res,) = exe.run( + feed={'x': self.data_x, 'index': self.data_index}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], [9.0, 10.0, 10.0]] + ).astype('float32') np.testing.assert_allclose(expect_out, np.array(res)) # case 2: @@ -139,14 +141,14 @@ def test_index_select_api(self): index = paddle.static.data(name='index', shape=[3], dtype='int32') z = paddle.index_select(x, index) exe = paddle.static.Executor(paddle.NPUPlace(0)) - res, = exe.run(feed={ - 'x': self.data_x, - 'index': self.data_index - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [5.0, 6.0, 7.0, 8.0]]).astype('float32') + (res,) = exe.run( + feed={'x': self.data_x, 'index': self.data_index}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [5.0, 6.0, 7.0, 8.0]] + ).astype('float32') np.testing.assert_allclose(expect_out, np.array(res)) def test_dygraph_index_select_api(self): @@ -159,8 +161,9 @@ def test_dygraph_index_select_api(self): index = paddle.to_tensor(self.data_index) z = paddle.index_select(x, index) np_z = z.numpy() - expect_out = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [5.0, 6.0, 7.0, 8.0]]).astype('float32') + expect_out = np.array( + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [5.0, 6.0, 7.0, 8.0]] + ).astype('float32') np.testing.assert_allclose(expect_out, np_z) # case 2: @@ -168,8 +171,9 @@ def test_dygraph_index_select_api(self): index = paddle.to_tensor(self.data_index) z = paddle.index_select(x, index, axis=1) np_z = z.numpy() - expect_out = np.array([[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], - [9.0, 10.0, 10.0]]).astype('float32') + expect_out = np.array( + [[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], [9.0, 10.0, 10.0]] + ).astype('float32') np.testing.assert_allclose(expect_out, np_z) diff --git a/python/paddle/fluid/tests/unittests/npu/test_instance_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_instance_norm_op_npu.py index 761c38a8576bb..1abc4af24cd4f 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_instance_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_instance_norm_op_npu.py @@ -31,7 +31,6 @@ class TestInstanceNorm(unittest.TestCase): - def test_dygraph(self): places = [fluid.NPUPlace(0)] for p in places: diff --git a/python/paddle/fluid/tests/unittests/npu/test_iou_similarity_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_iou_similarity_op_npu.py index 2485c8308c1c3..9b7b005dd3a7c 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_iou_similarity_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_iou_similarity_op_npu.py @@ -28,7 +28,6 @@ class TestNpuIouSimilarityOp(OpTest): - def setUp(self): self.op_type = "iou_similarity" self.set_npu() @@ -58,7 +57,7 @@ def set_inputs(self): self.boxes1_lod = [[1 for _ in range(self.N)]] self.inputs = { 'X': (self.boxes1, self.boxes1_lod), - 'Y': self.boxes2 + 'Y': self.boxes2, } else: self.inputs = {'X': self.boxes1, 'Y': self.boxes2} @@ -74,7 +73,9 @@ def set_outputs(self): def test_check_output(self): self.check_output_with_place(self.place) - def _compute_iou(self, ): + def _compute_iou( + self, + ): for row in range(self.boxes1.shape[0]): for col in range(self.boxes2.shape[0]): xmin1, ymin1, xmax1, ymax1 = self.boxes1[row] @@ -104,7 +105,6 @@ def _compute_iou(self, ): class TestNpuIouSimilarityOpWithLoD(TestNpuIouSimilarityOp): - def set_init_config(self): super(TestNpuIouSimilarityOpWithLoD, self).set_init_config() self.box_normalized = True @@ -112,7 +112,6 @@ def set_init_config(self): class TestNpuIouSimilarityOpWithBoxNormalized(TestNpuIouSimilarityOp): - def set_init_config(self): super(TestNpuIouSimilarityOpWithBoxNormalized, self).set_init_config() self.box_normalized = True @@ -120,7 +119,6 @@ def set_init_config(self): def TestNpuIouSimilarityOpFp16(TestNpuIouSimilarityOp): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/npu/test_is_empty_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_is_empty_op_npu.py index f939716f37fbb..9834ea0d5f2df 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_is_empty_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_is_empty_op_npu.py @@ -23,10 +23,10 @@ paddle.enable_static() -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestEmpty(OpTest): - def setUp(self): self.set_npu() self.init_dtype() @@ -48,23 +48,24 @@ def test_check_output(self): self.check_output_with_place(self.place) -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestNotEmpty(TestEmpty): - def set_data(self): self.inputs = {'X': np.array([])} self.outputs = {'Out': np.array([True])} -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestIsEmptyOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): input_data = np.random.random((3, 2)).astype("float32") def test_Variable(): @@ -75,27 +76,27 @@ def test_Variable(): def test_type(): # dtype must be float32, float16 in NPU - x3 = paddle.static.data(name="x3", - shape=[4, 32, 32], - dtype="bool") + x3 = paddle.static.data( + name="x3", shape=[4, 32, 32], dtype="bool" + ) res = paddle.is_empty(x=x3) self.assertRaises(TypeError, test_type) def test_name_type(): # name type must be string. - x4 = paddle.static.data(name="x4", - shape=[3, 2], - dtype="float32") + x4 = paddle.static.data( + name="x4", shape=[3, 2], dtype="float32" + ) res = paddle.is_empty(x=x4, name=1) self.assertRaises(TypeError, test_name_type) -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestIsEmptyOpDygraph(unittest.TestCase): - def test_dygraph(self): paddle.disable_static(paddle.NPUPlace(0)) input = paddle.rand(shape=[4, 32, 32], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/npu/test_kldiv_loss_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_kldiv_loss_op_npu.py index bf5042c01d227..3a55d9973af97 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_kldiv_loss_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_kldiv_loss_op_npu.py @@ -26,7 +26,6 @@ class TestKLDivLossOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -55,10 +54,13 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Loss', - no_grad_set=set(["Target"]), - max_relative_error=0.15) + self.check_grad_with_place( + self.place, + ['X'], + 'Loss', + no_grad_set=set(["Target"]), + max_relative_error=0.15, + ) def initTestCase(self): self.x_shape = (4, 5, 5) @@ -66,28 +68,24 @@ def initTestCase(self): class TestKLDivLossOp2(TestKLDivLossOp): - def initTestCase(self): self.x_shape = (3, 2, 7, 7) self.reduction = 'none' class TestKLDivLossOp3(TestKLDivLossOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 7, 9) self.reduction = 'mean' class TestKLDivLossOp4(TestKLDivLossOp): - def initTestCase(self): self.x_shape = (5, 20) self.reduction = 'sum' class TestKLDivLossOp_fp16(TestKLDivLossOp): - def init_dtype(self): self.dtype = 'float16' @@ -95,17 +93,22 @@ def test_check_output(self): self.check_output_with_place(self.place, atol=3e-1) def test_check_grad(self): - input_grad = -self.inputs['Target'] * ( - self.inputs['Target'] > 0) / self.inputs['Target'].shape[0] - self.check_grad_with_place(self.place, ['X'], - 'Loss', - no_grad_set=set(["Target"]), - max_relative_error=0.2, - user_defined_grads=[input_grad]) + input_grad = ( + -self.inputs['Target'] + * (self.inputs['Target'] > 0) + / self.inputs['Target'].shape[0] + ) + self.check_grad_with_place( + self.place, + ['X'], + 'Loss', + no_grad_set=set(["Target"]), + max_relative_error=0.2, + user_defined_grads=[input_grad], + ) class TestKLDivLossDygraph(unittest.TestCase): - def run_kl_loss(self, reduction, shape=(5, 20)): x = np.random.uniform(-10, 10, shape).astype('float32') target = np.random.uniform(-10, 10, shape).astype('float32') @@ -113,8 +116,9 @@ def run_kl_loss(self, reduction, shape=(5, 20)): with paddle.fluid.dygraph.guard(paddle.NPUPlace(0)): kldiv_criterion = paddle.nn.KLDivLoss(reduction) - pred_loss = kldiv_criterion(paddle.to_tensor(x), - paddle.to_tensor(target)) + pred_loss = kldiv_criterion( + paddle.to_tensor(x), paddle.to_tensor(target) + ) np.testing.assert_allclose(pred_loss.numpy(), gt_loss, rtol=1e-6) def test_kl_loss_batchmean(self): @@ -140,7 +144,6 @@ def test_kl_loss_static_api(self): class TestKLDivLossTypePromotion(unittest.TestCase): - def test_kl_div_promotion(self): with paddle.fluid.dygraph.guard(paddle.NPUPlace(0)): x1 = paddle.rand([5, 20], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/npu/test_label_smooth_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_label_smooth_op_npu.py index 26bcf0f6f860a..8bc5c9f5439a6 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_label_smooth_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_label_smooth_op_npu.py @@ -25,10 +25,10 @@ SEED = 2021 -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestLabelSmoothOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "label_smooth" @@ -62,8 +62,9 @@ def set_attrs(self): self.attrs = {"epsilon": epsilon} def set_outputs(self): - dist = None if 'PriorDist' not in self.inputs else self.inputs[ - 'PriorDist'] + dist = ( + None if 'PriorDist' not in self.inputs else self.inputs['PriorDist'] + ) out = self.calc_out(self.inputs['X'], self.attrs['epsilon'], dist) self.outputs = {'Out': out} @@ -78,15 +79,14 @@ def test_check_output(self): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.5) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.5 + ) else: self.check_grad_with_place(self.place, ['X'], 'Out') class TestLabelSmoothOpWithPriorDist(TestLabelSmoothOp): - def set_inputs(self): super(TestLabelSmoothOpWithPriorDist, self).set_inputs() label_dim = self.inputs['X'].shape[-1] @@ -95,39 +95,33 @@ def set_inputs(self): class TestLabelSmoothOp3D(TestLabelSmoothOp): - def set_inputs(self): super(TestLabelSmoothOp3D, self).set_inputs() self.inputs['X'].reshape([2, -1, self.inputs['X'].shape[-1]]) class TestLabelSmoothOpWithPriorDist3D(TestLabelSmoothOpWithPriorDist): - def set_inputs(self): super(TestLabelSmoothOpWithPriorDist3D, self).set_inputs() self.inputs['X'].reshape([2, -1, self.inputs['X'].shape[-1]]) class TestLabelSmoothOpFP16(TestLabelSmoothOp): - def init_dtype(self): self.dtype = np.float16 class TestLabelSmoothOpWithPriorDistFP16(TestLabelSmoothOpWithPriorDist): - def init_dtype(self): self.dtype = np.float16 class TestLabelSmoothOp3DFP16(TestLabelSmoothOp3D): - def init_dtype(self): self.dtype = np.float16 class TestLabelSmoothOpWithPriorDist3DFP16(TestLabelSmoothOpWithPriorDist3D): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/npu/test_layer_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_layer_norm_op_npu.py index f87cf8c805303..6ca621b647d57 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_layer_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_layer_norm_op_npu.py @@ -23,7 +23,10 @@ import paddle import paddle.fluid as fluid import paddle.fluid.core as core -from test_layer_norm_op import _reference_layer_norm_naive, _reference_layer_norm_grad +from test_layer_norm_op import ( + _reference_layer_norm_naive, + _reference_layer_norm_grad, +) paddle.enable_static() @@ -36,7 +39,6 @@ class TestLayerNormOp(unittest.TestCase): - def setUp(self): self.use_cudnn = True self.set_npu() @@ -51,44 +53,54 @@ def init_dtype(self): self.atol = 1e-4 def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor).astype( - np_array.dtype).reshape(np_array.shape), - np_array, - atol=atol, - err_msg=msg) - - def check_forward_backward(self, - shape, - begin_norm_axis, - has_scale=True, - has_bias=True, - y_grad_scale=1.0, - use_mkldnn=False): - - def test_with_place(place, - shape, - begin_norm_axis, - use_mkldnn=use_mkldnn): + np.testing.assert_allclose( + np.array(tensor).astype(np_array.dtype).reshape(np_array.shape), + np_array, + atol=atol, + err_msg=msg, + ) + + def check_forward_backward( + self, + shape, + begin_norm_axis, + has_scale=True, + has_bias=True, + y_grad_scale=1.0, + use_mkldnn=False, + ): + def test_with_place( + place, shape, begin_norm_axis, use_mkldnn=use_mkldnn + ): # attr epsilon = 0.00001 x_shape = shape - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) scale_shape = [D] np.random.seed(123) x = np.random.random_sample(x_shape).astype(self.dtype) - scale = np.random.random_sample(scale_shape).astype( - np.float32) if has_scale else None - bias = np.random.random_sample(scale_shape).astype( - np.float32) if has_bias else None + scale = ( + np.random.random_sample(scale_shape).astype(np.float32) + if has_scale + else None + ) + bias = ( + np.random.random_sample(scale_shape).astype(np.float32) + if has_bias + else None + ) y_grad = (np.random.random_sample(x_shape) * y_grad_scale).astype( - self.dtype) + self.dtype + ) # reference forward & backward y, mean, variance = _reference_layer_norm_naive( - x, scale, bias, epsilon, begin_norm_axis) + x, scale, bias, epsilon, begin_norm_axis + ) x_grad, scale_grad, bias_grad = _reference_layer_norm_grad( - x, y_grad, scale, bias, mean, variance, begin_norm_axis) + x, y_grad, scale, bias, mean, variance, begin_norm_axis + ) var_dict = locals() var_dict['y@GRAD'] = y_grad @@ -103,9 +115,11 @@ def test_with_place(place, with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype=self.dtype, - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype=self.dtype, + shape=ground_truth[name].shape, + ) inputs = {"X": block.var('x')} fetch_list = [ 'y', @@ -125,17 +139,20 @@ def test_with_place(place, outputs={ "Y": block.var('y'), "Mean": block.var('mean'), # share the same memory - "Variance": - block.var('variance'), # share the same memory + "Variance": block.var( + 'variance' + ), # share the same memory }, attrs={ "epsilon": epsilon, "begin_norm_axis": begin_norm_axis, - "use_mkldnn": use_mkldnn - }) + "use_mkldnn": use_mkldnn, + }, + ) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - layer_norm_op.desc, set(), []) + layer_norm_op.desc, set(), [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -149,46 +166,59 @@ def test_with_place(place, program._sync_with_cpp() exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in ['x', 'scale', 'bias', 'y@GRAD'] - }, - fetch_list=fetch_list) + out = exe.run( + program, + feed={ + name: var_dict[name] + for name in ['x', 'scale', 'bias', 'y@GRAD'] + }, + fetch_list=fetch_list, + ) self.__assert_close(y, out[0], "y", self.atol) self.__assert_close(mean, out[1], "mean") self.__assert_close(variance, out[2], "variance", 1e-3) self.__assert_close(x_grad, out[3], "x_grad", 1e-2) if has_scale: - self.__assert_close(scale_grad, - out[fetch_list.index('scale@GRAD')], - "scale_grad", 1e-2) + self.__assert_close( + scale_grad, + out[fetch_list.index('scale@GRAD')], + "scale_grad", + 1e-2, + ) if has_bias: - self.__assert_close(bias_grad, - out[fetch_list.index('bias@GRAD')], - "bias_grad", self.atol) + self.__assert_close( + bias_grad, + out[fetch_list.index('bias@GRAD')], + "bias_grad", + self.atol, + ) test_with_place(self.place, shape, begin_norm_axis) def test_check_forward_backward_with_scale_and_bias(self): self.check_forward_backward(shape=[2, 3, 4, 5], begin_norm_axis=1) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=False, - has_bias=True) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=True, - has_bias=False) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=False, - has_bias=False) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=False, + has_bias=True, + ) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=True, + has_bias=False, + ) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=False, + has_bias=False, + ) self.check_forward_backward(shape=[2, 3, 4, 5], begin_norm_axis=3) class TestLayerNormOpFP16(TestLayerNormOp): - def init_dtype(self): self.dtype = np.float16 self.atol = 1e-2 diff --git a/python/paddle/fluid/tests/unittests/npu/test_leaky_relu_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_leaky_relu_op_npu.py index 07067ee2519ae..e91a65faeec16 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_leaky_relu_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_leaky_relu_op_npu.py @@ -27,7 +27,6 @@ class TestLeadyRelu(OpTest): - def setUp(self): self.set_npu() self.op_type = "leaky_relu" @@ -63,33 +62,29 @@ def test_check_output(self): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.006 + ) else: self.check_grad_with_place(self.place, ['X'], 'Out') class TestLeadyReluFP16(TestLeadyRelu): - def init_dtype(self): self.dtype = np.float16 class TestLeadyRelu2(TestLeadyRelu): - def set_attrs(self): self.attrs = {'alpha': 0.5} class TestLeadyRelu3(TestLeadyRelu): - def set_attrs(self): self.attrs = {'alpha': -0.5} class TestLeakyReluNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -102,9 +97,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): x = paddle.static.data(name="x", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) y = paddle.nn.functional.leaky_relu(x) @@ -127,15 +122,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "x": x_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"x": x_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_log_loss_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_log_loss_op_npu.py index febc99093f18b..87cd872e8cc91 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_log_loss_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_log_loss_op_npu.py @@ -28,10 +28,10 @@ def sigmoid_array(x): return 1 / (1 + np.exp(-x)) -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestLogLossOp(OpTest): - def setUp(self): self.set_npu() self.op_type = 'log_loss' @@ -58,8 +58,9 @@ def set_outputs(self): epsilon = self.attrs['epsilon'] labels = self.inputs['Labels'] predicted = self.inputs['Predicted'] - loss = -labels * np.log(predicted + epsilon) - ( - 1 - labels) * np.log(1 - predicted + epsilon) + loss = -labels * np.log(predicted + epsilon) - (1 - labels) * np.log( + 1 - predicted + epsilon + ) self.outputs = {'Loss': loss} def set_npu(self): @@ -75,10 +76,10 @@ def test_check_grad(self): self.check_grad_with_place(self.place, ['Predicted'], 'Loss') -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestLogLossOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program()): diff --git a/python/paddle/fluid/tests/unittests/npu/test_log_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_log_op_npu.py index ea7dd3abde928..8745a66b45a4b 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_log_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_log_op_npu.py @@ -26,7 +26,6 @@ class TestLog(OpTest): - def setUp(self): self.set_npu() self.op_type = "log" @@ -55,7 +54,6 @@ def test_check_grad(self): class TestLogFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "log" @@ -82,7 +80,6 @@ def test_check_output(self): class TestLogNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -97,9 +94,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) d = paddle.log(c) @@ -123,16 +120,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_log_softmax_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_log_softmax_op_npu.py index d5dabdeb2018b..fc0c428983fa9 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_log_softmax_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_log_softmax_op_npu.py @@ -29,7 +29,6 @@ class TestLogSoftmaxNPUOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -39,7 +38,7 @@ def setUp(self): self.axis = -1 self.set_attrs() self.set_dtype() - x = np.random.uniform(0.1, 1., self.shape).astype(self.dtype) + x = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) out = np.apply_along_axis(ref_log_softmax, self.axis, x) self.x_grad = ref_log_softmax_grad(x, self.axis) self.inputs = {'X': x} @@ -64,18 +63,21 @@ def test_check_output(self): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], ['Out'], - user_defined_grads=[self.x_grad], - max_relative_error=0.02) + self.check_grad_with_place( + self.place, + ['X'], + ['Out'], + user_defined_grads=[self.x_grad], + max_relative_error=0.02, + ) else: - self.check_grad_with_place(self.place, ['X'], ['Out'], - user_defined_grads=[self.x_grad]) + self.check_grad_with_place( + self.place, ['X'], ['Out'], user_defined_grads=[self.x_grad] + ) def test_class(op_type, typename): - class TestLogSoftmaxShape(TestLogSoftmaxNPUOp): - def set_attrs(self): self.shape = [12, 10] @@ -88,9 +90,7 @@ def set_dtype(self): def test_class2(op_type, typename): - class TestLogSoftmaxAxis(TestLogSoftmaxNPUOp): - def set_attrs(self): self.axis = 0 @@ -109,13 +109,14 @@ def set_dtype(self): class TestNNLogSoftmaxAPI(unittest.TestCase): - def setUp(self): self.x_shape = [2, 3, 4, 5] - self.x = np.random.uniform(-1., 1., self.x_shape).astype(np.float32) - self.place = paddle.NPUPlace(0) \ - if paddle.fluid.core.is_compiled_with_npu() \ + self.x = np.random.uniform(-1.0, 1.0, self.x_shape).astype(np.float32) + self.place = ( + paddle.NPUPlace(0) + if paddle.fluid.core.is_compiled_with_npu() else paddle.CPUPlace() + ) def check_api(self, axis=-1): ref_out = np.apply_along_axis(ref_log_softmax, axis, self.x) @@ -142,13 +143,14 @@ def test_check_api(self): class TestNNFunctionalLogSoftmaxAPI(unittest.TestCase): - def setUp(self): self.x_shape = [2, 3, 4, 5] self.x = np.random.uniform(-1, 1, self.x_shape).astype(np.float32) - self.place = paddle.NPUPlace(0) \ - if paddle.fluid.core.is_compiled_with_npu() \ + self.place = ( + paddle.NPUPlace(0) + if paddle.fluid.core.is_compiled_with_npu() else paddle.CPUPlace() + ) def check_api(self, axis=-1, dtype=None): x = self.x.copy() diff --git a/python/paddle/fluid/tests/unittests/npu/test_logical_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_logical_op_npu.py index 14422e8565a9e..a783e04c67634 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_logical_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_logical_op_npu.py @@ -24,81 +24,31 @@ SUPPORTED_DTYPES = [bool] -TEST_META_OP_DATA = [{ - 'op_str': 'logical_and', - 'binary_op': True -}, { - 'op_str': 'logical_or', - 'binary_op': True -}, { - 'op_str': 'logical_not', - 'binary_op': False -}] +TEST_META_OP_DATA = [ + {'op_str': 'logical_and', 'binary_op': True}, + {'op_str': 'logical_or', 'binary_op': True}, + {'op_str': 'logical_not', 'binary_op': False}, +] TEST_META_SHAPE_DATA = { - 'XDimLargerThanYDim1': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 5] - }, - 'XDimLargerThanYDim2': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 1] - }, - 'XDimLargerThanYDim3': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [1, 4, 1] - }, - 'XDimLargerThanYDim4': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [3, 4, 1] - }, - 'XDimLargerThanYDim5': { - 'x_shape': [2, 3, 1, 5], - 'y_shape': [3, 1, 1] - }, - 'XDimLessThanYDim1': { - 'x_shape': [4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim2': { - 'x_shape': [1, 4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim3': { - 'x_shape': [3, 4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim4': { - 'x_shape': [3, 1, 1], - 'y_shape': [2, 3, 1, 5] - }, - 'XDimLessThanYDim5': { - 'x_shape': [4, 5], - 'y_shape': [2, 3, 4, 5] - }, - 'Axis1InLargerDim': { - 'x_shape': [1, 4, 5], - 'y_shape': [2, 3, 1, 5] - }, - 'EqualDim1': { - 'x_shape': [10, 7], - 'y_shape': [10, 7] - }, - 'EqualDim2': { - 'x_shape': [1, 1, 4, 5], - 'y_shape': [2, 3, 1, 5] - } + 'XDimLargerThanYDim1': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 5]}, + 'XDimLargerThanYDim2': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 1]}, + 'XDimLargerThanYDim3': {'x_shape': [2, 3, 4, 5], 'y_shape': [1, 4, 1]}, + 'XDimLargerThanYDim4': {'x_shape': [2, 3, 4, 5], 'y_shape': [3, 4, 1]}, + 'XDimLargerThanYDim5': {'x_shape': [2, 3, 1, 5], 'y_shape': [3, 1, 1]}, + 'XDimLessThanYDim1': {'x_shape': [4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim2': {'x_shape': [1, 4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim3': {'x_shape': [3, 4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim4': {'x_shape': [3, 1, 1], 'y_shape': [2, 3, 1, 5]}, + 'XDimLessThanYDim5': {'x_shape': [4, 5], 'y_shape': [2, 3, 4, 5]}, + 'Axis1InLargerDim': {'x_shape': [1, 4, 5], 'y_shape': [2, 3, 1, 5]}, + 'EqualDim1': {'x_shape': [10, 7], 'y_shape': [10, 7]}, + 'EqualDim2': {'x_shape': [1, 1, 4, 5], 'y_shape': [2, 3, 1, 5]}, } TEST_META_WRONG_SHAPE_DATA = { - 'ErrorDim1': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [3, 4] - }, - 'ErrorDim2': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 3] - } + 'ErrorDim1': {'x_shape': [2, 3, 4, 5], 'y_shape': [3, 4]}, + 'ErrorDim2': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 3]}, } @@ -157,16 +107,20 @@ def test(unit_test, use_npu=False, test_error=False): META_DATA = dict(TEST_META_WRONG_SHAPE_DATA) for shape_data in META_DATA.values(): for data_type in SUPPORTED_DTYPES: - meta_data['x_np'] = np_data_generator(shape_data['x_shape'], - dtype=data_type) - meta_data['y_np'] = np_data_generator(shape_data['y_shape'], - dtype=data_type) + meta_data['x_np'] = np_data_generator( + shape_data['x_shape'], dtype=data_type + ) + meta_data['y_np'] = np_data_generator( + shape_data['y_shape'], dtype=data_type + ) if meta_data['binary_op'] and test_error: # catch C++ Exception - unit_test.assertRaises(BaseException, run_static, - **meta_data) - unit_test.assertRaises(BaseException, run_dygraph, - **meta_data) + unit_test.assertRaises( + BaseException, run_static, **meta_data + ) + unit_test.assertRaises( + BaseException, run_dygraph, **meta_data + ) continue static_result = run_static(**meta_data) dygraph_result = run_dygraph(**meta_data) @@ -176,11 +130,11 @@ def test(unit_test, use_npu=False, test_error=False): np_result = np_op(meta_data['x_np']) unit_test.assertTrue((static_result == np_result).all()) unit_test.assertTrue( - (dygraph_result.numpy() == np_result).all()) + (dygraph_result.numpy() == np_result).all() + ) def test_type_error(unit_test, use_npu, type_str_map): - def check_type(op_str, x, y, binary_op): op = getattr(paddle, op_str) error_type = ValueError @@ -215,24 +169,24 @@ def check_type(op_str, x, y, binary_op): startup_program = paddle.static.Program() main_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - x = paddle.static.data(name='x', - shape=[10], - dtype=type_str_map['x']) - y = paddle.static.data(name='y', - shape=[10], - dtype=type_str_map['y']) + x = paddle.static.data( + name='x', shape=[10], dtype=type_str_map['x'] + ) + y = paddle.static.data( + name='y', shape=[10], dtype=type_str_map['y'] + ) check_type(meta_data['op_str'], x, y, binary_op) def type_map_factory(): - return [{ - 'x': x_type, - 'y': y_type - } for x_type in SUPPORTED_DTYPES for y_type in SUPPORTED_DTYPES] + return [ + {'x': x_type, 'y': y_type} + for x_type in SUPPORTED_DTYPES + for y_type in SUPPORTED_DTYPES + ] class TestCPU(unittest.TestCase): - def test(self): test(self) @@ -246,7 +200,6 @@ def test_type_error(self): class TestNPU(unittest.TestCase): - def test(self): test(self, True) diff --git a/python/paddle/fluid/tests/unittests/npu/test_lookup_table_v2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_lookup_table_v2_op_npu.py index 9efec50ddc59c..0670b69851a26 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_lookup_table_v2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_lookup_table_v2_op_npu.py @@ -26,7 +26,6 @@ class TestLookupTableV2(OpTest): - def setUp(self): self.set_npu() self.op_type = "lookup_table_v2" @@ -37,22 +36,22 @@ def setUp(self): self.init_padding_idx() np.random.seed(SEED) w = np.random.random([self.vocab, self.dim]).astype(self.dtype) - x = np.random.randint(0, self.vocab, - size=(self.bsz, - self.seqlen)).astype(self.ids_dtype) + x = np.random.randint( + 0, self.vocab, size=(self.bsz, self.seqlen) + ).astype(self.ids_dtype) out = w[x] if self.padding_idx != -1: out[np.squeeze(x == self.padding_idx)] = np.zeros(self.dim) self.inputs = { 'W': OpTest.np_dtype_to_fluid_dtype(w), - 'Ids': OpTest.np_dtype_to_fluid_dtype(x) + 'Ids': OpTest.np_dtype_to_fluid_dtype(x), } self.attrs = { 'is_sparse': False, 'is_distributed': False, 'remote_prefetch': False, - 'padding_idx': self.padding_idx + 'padding_idx': self.padding_idx, } self.outputs = {'Out': out} @@ -78,9 +77,9 @@ def test_check_output(self): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['W'], - 'Out', - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ['W'], 'Out', max_relative_error=0.01 + ) else: self.check_grad_with_place(self.place, ['W'], 'Out') @@ -98,7 +97,6 @@ def set_npu(self): class TestLookupTableV2Dim32(TestLookupTableV2): - def init_dims(self): self.bsz = 6 self.seqlen = 8 @@ -126,13 +124,11 @@ def set_npu(self): class TestLookupTableV2WithPadding(TestLookupTableV2): - def init_padding_idx(self): self.padding_idx = np.random.randint(0, self.vocab) class TestLookupTableV2WithPadding1(TestLookupTableV2): - def init_padding_idx(self): self.padding_idx = np.random.randint(0, self.vocab) diff --git a/python/paddle/fluid/tests/unittests/npu/test_masked_select_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_masked_select_op_npu.py index 50c9ca56f6663..8943b5ba95f42 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_masked_select_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_masked_select_op_npu.py @@ -33,7 +33,6 @@ def np_masked_select(x, mask): class TestMaskedSelectOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -63,33 +62,29 @@ def init_dtype(self): class TestMaskedSelectOp1(TestMaskedSelectOp): - def init(self): self.shape = (6, 8, 9, 18) class TestMaskedSelectOp2(TestMaskedSelectOp): - def init(self): - self.shape = (168, ) + self.shape = (168,) class TestMaskedSelectOpFp16(TestMaskedSelectOp): - def init_dtype(self): self.dtype = np.float16 def test_check_grad(self): x_grad = self.inputs['Mask'].astype(self.dtype) x_grad = x_grad * (1 / x_grad.sum()) - self.check_grad_with_place(self.place, ['X'], - 'Y', - user_defined_grads=[x_grad]) + self.check_grad_with_place( + self.place, ['X'], 'Y', user_defined_grads=[x_grad] + ) @skip_check_grad_ci(reason="get_numeric_gradient not support int32") class TestMaskedSelectOpInt32(TestMaskedSelectOp): - def init_dtype(self): self.dtype = np.int32 @@ -99,7 +94,6 @@ def test_check_grad(self): @skip_check_grad_ci(reason="get_numeric_gradient not support int64") class TestMaskedSelectOpInt64(TestMaskedSelectOp): - def init_dtype(self): self.dtype = np.int64 @@ -108,7 +102,6 @@ def test_check_grad(self): class TestMaskedSelectAPI(unittest.TestCase): - def test_imperative_mode(self): paddle.disable_static(paddle.NPUPlace(0)) shape = (88, 6, 8) @@ -133,27 +126,26 @@ def test_static_mode(self): exe = paddle.static.Executor(place=paddle.NPUPlace(0)) - res = exe.run(paddle.static.default_main_program(), - feed={ - "x": np_x, - "mask": np_mask - }, - fetch_list=[out]) + res = exe.run( + paddle.static.default_main_program(), + feed={"x": np_x, "mask": np_mask}, + fetch_list=[out], + ) self.assertEqual(np.allclose(res, np_out), True) class TestMaskedSelectError(unittest.TestCase): - def test_error(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): shape = [8, 9, 6] x = paddle.fluid.data(shape=shape, dtype='float32', name='x') mask = paddle.fluid.data(shape=shape, dtype='bool', name='mask') - mask_float = paddle.fluid.data(shape=shape, - dtype='float32', - name='mask_float') + mask_float = paddle.fluid.data( + shape=shape, dtype='float32', name='mask_float' + ) np_x = np.random.random(shape).astype('float32') np_mask = np.array(np.random.randint(2, size=shape, dtype=bool)) diff --git a/python/paddle/fluid/tests/unittests/npu/test_matmul_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_matmul_op_npu.py index d00c8ab867400..9a2563ad306fa 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_matmul_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_matmul_op_npu.py @@ -31,7 +31,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False, scale=1.0): # transpose X and Y appropriately. if transpose_X: if X.ndim == 1: - X = X.reshape((X.size, )) + X = X.reshape((X.size,)) elif X.ndim == 2: X = X.T else: @@ -40,7 +40,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False, scale=1.0): X = np.transpose(X, tuple(dim)) if transpose_Y: if Y.ndim == 1: - Y = Y.reshape((Y.size, )) + Y = Y.reshape((Y.size,)) else: dim = [i for i in range(len(Y.shape))] dim[-1], dim[len(Y.shape) - 2] = dim[len(Y.shape) - 2], dim[-1] @@ -76,14 +76,15 @@ def setUp(self): X = -0.1 + 0.2 * X Y = -0.1 + 0.2 * Y - Out = reference_matmul(X, Y, self.transpose_X, self.transpose_Y, - self.alpha) + Out = reference_matmul( + X, Y, self.transpose_X, self.transpose_Y, self.alpha + ) Out = Out.astype(self.dtype) self.inputs = {'X': X, 'Y': Y} self.attrs = { 'transpose_X': self.transpose_X, 'transpose_Y': self.transpose_Y, - 'alpha': self.alpha + 'alpha': self.alpha, } self.outputs = {'Out': Out} @@ -92,8 +93,8 @@ def set_npu(self): self.place = paddle.NPUPlace(0) def config(self): - self.x_shape = (100, ) - self.y_shape = (100, ) + self.x_shape = (100,) + self.y_shape = (100,) self.transpose_X = False self.transpose_Y = False @@ -116,7 +117,7 @@ class TestMatMulOp1(TestMatMulOp): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 3, 2, 100) self.transpose_X = False self.transpose_Y = True @@ -129,7 +130,7 @@ class TestMatMulOp2(TestMatMulOp): def config(self): self.x_shape = (1, 2, 100, 1) - self.y_shape = (100, ) + self.y_shape = (100,) self.transpose_X = True self.transpose_Y = False @@ -248,7 +249,7 @@ class TestMatMulOp12(TestMatMulOp): """ def config(self): - self.x_shape = (100) + self.x_shape = 100 self.y_shape = (1, 2, 2, 100, 2) self.transpose_X = False self.transpose_Y = False @@ -261,16 +262,14 @@ class TestMatMulOp13(TestMatMulOp): def config(self): self.x_shape = (2, 1, 100) - self.y_shape = (100) + self.y_shape = 100 self.transpose_X = False self.transpose_Y = False -#--------------------test matmul alpha-------------------- +# --------------------test matmul alpha-------------------- def create_test_alpha_class(parent): - class TestMatMulOpAlphaCase(parent): - def init_alpha(self): self.alpha = 0.125 @@ -293,11 +292,9 @@ def init_alpha(self): create_test_alpha_class(TestMatMulOp13) -#--------------------test matmul fp16-------------------- +# --------------------test matmul fp16-------------------- def create_test_fp16_class(parent, atol=0.001, max_relative_error=2.5): - class TestMatMulOpFp16Case(parent): - def init_kernel_type(self): self.dtype = np.float16 @@ -305,9 +302,12 @@ def test_check_output(self): self.check_output_with_place(self.place, atol=atol) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=max_relative_error) + self.check_grad_with_place( + self.place, + ['X', 'Y'], + 'Out', + max_relative_error=max_relative_error, + ) cls_name = "{0}_{1}".format(parent.__name__, "Fp16") TestMatMulOpFp16Case.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/npu/test_matmulv2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_matmulv2_op_npu.py index 1d3cdaf591523..91883824cf5b6 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_matmulv2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_matmulv2_op_npu.py @@ -36,8 +36,8 @@ def set_npu(self): self.place = paddle.NPUPlace(0) def config(self): - self.x_shape = (100, ) - self.y_shape = (100, ) + self.x_shape = (100,) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -76,7 +76,7 @@ class TestMatMulOp2(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 3, 2, 100) self.trans_x = False self.trans_y = True @@ -88,7 +88,7 @@ class TestMatMulOp3(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 1, 100, 2) self.trans_x = False self.trans_y = False @@ -100,7 +100,7 @@ class TestMatMulOp4(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 2, 100, 2) self.trans_x = False self.trans_y = False @@ -113,7 +113,7 @@ class TestMatMulOp5(TestMatMulV2Op): def config(self): self.x_shape = (1, 1, 100, 1) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = True self.trans_y = False @@ -125,7 +125,7 @@ class TestMatMulOp6(TestMatMulV2Op): def config(self): self.x_shape = (1, 2, 102, 1) - self.y_shape = (102, ) + self.y_shape = (102,) self.trans_x = True self.trans_y = False @@ -137,7 +137,7 @@ class TestMatMulOp7(TestMatMulV2Op): def config(self): self.x_shape = (1, 2, 1, 100) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -244,7 +244,7 @@ class TestMatMulOp16(TestMatMulV2Op): """ def config(self): - self.x_shape = (100) + self.x_shape = 100 self.y_shape = (1, 2, 2, 100, 2) self.trans_x = False self.trans_y = False @@ -257,7 +257,7 @@ class TestMatMulOp17(TestMatMulV2Op): def config(self): self.x_shape = (2, 1, 100) - self.y_shape = (100) + self.y_shape = 100 self.trans_x = False self.trans_y = False @@ -286,13 +286,11 @@ def config(self): self.trans_y = True -#--------------------test matmul fp16-------------------- +# --------------------test matmul fp16-------------------- def create_test_fp16_class(parent, atol=0.001, max_relative_error=2.5): - class TestMatMulOpFp16Case(parent): - def init_kernel_type(self): self.dtype = np.float16 @@ -300,9 +298,12 @@ def test_check_output(self): self.check_output_with_place(self.place, atol=atol) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=max_relative_error) + self.check_grad_with_place( + self.place, + ['X', 'Y'], + 'Out', + max_relative_error=max_relative_error, + ) cls_name = "{0}_{1}".format(parent.__name__, "Fp16") TestMatMulOpFp16Case.__name__ = cls_name @@ -329,7 +330,6 @@ def test_check_grad(self): class TestMatMulV2API(unittest.TestCase): - def setUp(self): self.places = [paddle.CPUPlace()] if paddle.is_compiled_with_npu(): @@ -346,12 +346,11 @@ def check_static_result(self, place): y_np = np.random.random([3, 4]).astype("float32") exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input_x": x_np, - "input_y": y_np - }, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": x_np, "input_y": y_np}, + fetch_list=[result], + ) def test_static(self): for place in self.places: diff --git a/python/paddle/fluid/tests/unittests/npu/test_mean_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_mean_op_npu.py index 37e2afbb892d6..4041ee9a2c182 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_mean_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_mean_op_npu.py @@ -27,7 +27,6 @@ class TestMean(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -55,7 +54,6 @@ def test_check_grad(self): class TestMeanFP16(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/npu/test_memcpy_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_memcpy_op_npu.py index 5367ea8a92be5..6ed36b710ab38 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_memcpy_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_memcpy_op_npu.py @@ -28,66 +28,77 @@ class TestMemcpy_FillConstant(unittest.TestCase): - def get_prog(self): paddle.enable_static() main_program = Program() with program_guard(main_program): cpu_var_name = "tensor@Cpu" npu_var_name = "tensor@Npu" - cpu_var = main_program.global_block().create_var(name=cpu_var_name, - shape=[10, 10], - dtype='float32', - persistable=False, - stop_gradient=True) - npu_var = main_program.global_block().create_var(name=npu_var_name, - shape=[10, 10], - dtype='float32', - persistable=False, - stop_gradient=True) - main_program.global_block().append_op(type="fill_constant", - outputs={"Out": npu_var_name}, - attrs={ - "shape": [10, 10], - "dtype": npu_var.dtype, - "value": 1.0, - "place_type": 4 - }) - main_program.global_block().append_op(type="fill_constant", - outputs={"Out": cpu_var_name}, - attrs={ - "shape": [10, 10], - "dtype": cpu_var.dtype, - "value": 0.0, - "place_type": 0 - }) + cpu_var = main_program.global_block().create_var( + name=cpu_var_name, + shape=[10, 10], + dtype='float32', + persistable=False, + stop_gradient=True, + ) + npu_var = main_program.global_block().create_var( + name=npu_var_name, + shape=[10, 10], + dtype='float32', + persistable=False, + stop_gradient=True, + ) + main_program.global_block().append_op( + type="fill_constant", + outputs={"Out": npu_var_name}, + attrs={ + "shape": [10, 10], + "dtype": npu_var.dtype, + "value": 1.0, + "place_type": 4, + }, + ) + main_program.global_block().append_op( + type="fill_constant", + outputs={"Out": cpu_var_name}, + attrs={ + "shape": [10, 10], + "dtype": cpu_var.dtype, + "value": 0.0, + "place_type": 0, + }, + ) return main_program, npu_var, cpu_var def test_npu_cpoy_to_cpu(self): main_program, npu_var, cpu_var = self.get_prog() - main_program.global_block().append_op(type='memcpy', - inputs={'X': npu_var}, - outputs={'Out': cpu_var}, - attrs={'dst_place_type': 0}) + main_program.global_block().append_op( + type='memcpy', + inputs={'X': npu_var}, + outputs={'Out': cpu_var}, + attrs={'dst_place_type': 0}, + ) place = fluid.NPUPlace(0) exe = fluid.Executor(place) - npu_, cpu_ = exe.run(main_program, - feed={}, - fetch_list=[npu_var.name, cpu_var.name]) + npu_, cpu_ = exe.run( + main_program, feed={}, fetch_list=[npu_var.name, cpu_var.name] + ) np.testing.assert_allclose(npu_, cpu_) np.testing.assert_allclose(cpu_, np.ones((10, 10))) def test_cpu_cpoy_npu(self): main_program, npu_var, cpu_var = self.get_prog() - main_program.global_block().append_op(type='memcpy', - inputs={'X': cpu_var}, - outputs={'Out': npu_var}, - attrs={'dst_place_type': 4}) + main_program.global_block().append_op( + type='memcpy', + inputs={'X': cpu_var}, + outputs={'Out': npu_var}, + attrs={'dst_place_type': 4}, + ) place = fluid.NPUPlace(0) exe = fluid.Executor(place) - npu_, cpu_ = exe.run(main_program, - feed={}, - fetch_list=[npu_var.name, cpu_var.name]) + npu_, cpu_ = exe.run( + main_program, feed={}, fetch_list=[npu_var.name, cpu_var.name] + ) np.testing.assert_allclose(npu_, cpu_) np.testing.assert_allclose(npu_, np.zeros((10, 10))) diff --git a/python/paddle/fluid/tests/unittests/npu/test_merged_momentum_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_merged_momentum_op_npu.py index 086911a56dbae..d530438bd581a 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_merged_momentum_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_merged_momentum_op_npu.py @@ -22,16 +22,18 @@ from collections import OrderedDict -def run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - mu=0.9, - rescale_grad=0.01, - use_merged=False): +def run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + mu=0.9, + rescale_grad=0.01, + use_merged=False, +): assert len(params) == len(grads) assert len(params) == len(velocitys) if multi_precision: @@ -48,48 +50,70 @@ def run_momentum_op(params, } param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in params ] grad_vars = [ helper.create_variable(shape=g.shape, dtype=g.dtype) for g in grads ] velocity_vars = [ - helper.create_variable(persistable=True, - shape=v.shape, - dtype=v.dtype) for v in velocitys + helper.create_variable( + persistable=True, shape=v.shape, dtype=v.dtype + ) + for v in velocitys ] - lr_var = helper.create_variable(persistable=True, - shape=learning_rate.shape, - dtype=learning_rate.dtype) + lr_var = helper.create_variable( + persistable=True, + shape=learning_rate.shape, + dtype=learning_rate.dtype, + ) feed_dict = OrderedDict() feed_dict.update( - OrderedDict([(p_var.name, p_val) - for p_var, p_val in zip(param_vars, params)])) + OrderedDict( + [ + (p_var.name, p_val) + for p_var, p_val in zip(param_vars, params) + ] + ) + ) feed_dict.update( - OrderedDict([(v_var.name, v_val) - for v_var, v_val in zip(velocity_vars, velocitys)])) + OrderedDict( + [ + (v_var.name, v_val) + for v_var, v_val in zip(velocity_vars, velocitys) + ] + ) + ) fetch_list = list(feed_dict.keys()) feed_dict.update( - OrderedDict([(g_var.name, g_val) - for g_var, g_val in zip(grad_vars, grads)])) + OrderedDict( + [(g_var.name, g_val) for g_var, g_val in zip(grad_vars, grads)] + ) + ) feed_dict.update({lr_var.name: learning_rate}) if multi_precision: master_param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in master_params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in master_params ] feed_dict.update( - OrderedDict([ - (mp_var.name, mp_val) - for mp_var, mp_val in zip(master_param_vars, master_params) - ])) + OrderedDict( + [ + (mp_var.name, mp_val) + for mp_var, mp_val in zip( + master_param_vars, master_params + ) + ] + ) + ) # CPUPlace does not use MasterParam if isinstance(place, paddle.CUDAPlace): fetch_list = fetch_list + [ @@ -99,8 +123,9 @@ def run_momentum_op(params, master_param_vars = None if not use_merged: - for i, (p, g, - v) in enumerate(zip(param_vars, grad_vars, velocity_vars)): + for i, (p, g, v) in enumerate( + zip(param_vars, grad_vars, velocity_vars) + ): inputs = { 'Param': p, 'Grad': g, @@ -111,10 +136,9 @@ def run_momentum_op(params, if multi_precision: inputs['MasterParam'] = master_param_vars[i] outputs['MasterParamOut'] = master_param_vars[i] - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) else: inputs = { 'Param': param_vars, @@ -126,10 +150,9 @@ def run_momentum_op(params, if multi_precision: inputs['MasterParam'] = master_param_vars outputs['MasterParamOut'] = master_param_vars - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): @@ -137,17 +160,19 @@ def run_momentum_op(params, return exe.run(main, feed=feed_dict, fetch_list=fetch_list) -def run_momentum_op2(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - mu=0.9, - rescale_grad=0.01, - use_merged=False, - use_nesterov=True): +def run_momentum_op2( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + mu=0.9, + rescale_grad=0.01, + use_merged=False, + use_nesterov=True, +): assert len(params) == len(grads) assert len(params) == len(velocitys) if multi_precision: @@ -159,48 +184,70 @@ def run_momentum_op2(params, helper = LayerHelper(op_type, **locals()) param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in params ] grad_vars = [ helper.create_variable(shape=g.shape, dtype=g.dtype) for g in grads ] velocity_vars = [ - helper.create_variable(persistable=True, - shape=v.shape, - dtype=v.dtype) for v in velocitys + helper.create_variable( + persistable=True, shape=v.shape, dtype=v.dtype + ) + for v in velocitys ] - lr_var = helper.create_variable(persistable=True, - shape=learning_rate.shape, - dtype=learning_rate.dtype) + lr_var = helper.create_variable( + persistable=True, + shape=learning_rate.shape, + dtype=learning_rate.dtype, + ) feed_dict = OrderedDict() feed_dict.update( - OrderedDict([(p_var.name, p_val) - for p_var, p_val in zip(param_vars, params)])) + OrderedDict( + [ + (p_var.name, p_val) + for p_var, p_val in zip(param_vars, params) + ] + ) + ) feed_dict.update( - OrderedDict([(v_var.name, v_val) - for v_var, v_val in zip(velocity_vars, velocitys)])) + OrderedDict( + [ + (v_var.name, v_val) + for v_var, v_val in zip(velocity_vars, velocitys) + ] + ) + ) fetch_list = list(feed_dict.keys()) feed_dict.update( - OrderedDict([(g_var.name, g_val) - for g_var, g_val in zip(grad_vars, grads)])) + OrderedDict( + [(g_var.name, g_val) for g_var, g_val in zip(grad_vars, grads)] + ) + ) feed_dict.update({lr_var.name: learning_rate}) if multi_precision: master_param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in master_params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in master_params ] feed_dict.update( - OrderedDict([ - (mp_var.name, mp_val) - for mp_var, mp_val in zip(master_param_vars, master_params) - ])) + OrderedDict( + [ + (mp_var.name, mp_val) + for mp_var, mp_val in zip( + master_param_vars, master_params + ) + ] + ) + ) # CPUPlace does not use MasterParam if isinstance(place, paddle.CUDAPlace): fetch_list = fetch_list + [ @@ -210,8 +257,9 @@ def run_momentum_op2(params, master_param_vars = None if not use_merged: - for i, (p, g, - v) in enumerate(zip(param_vars, grad_vars, velocity_vars)): + for i, (p, g, v) in enumerate( + zip(param_vars, grad_vars, velocity_vars) + ): inputs = { 'Param': p, 'Grad': g, @@ -230,10 +278,9 @@ def run_momentum_op2(params, 'regularization_method': 'l2_decay', 'regularization_coeff': 2.0, } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) else: inputs = { 'Param': param_vars, @@ -246,22 +293,18 @@ def run_momentum_op2(params, inputs['MasterParam'] = master_param_vars outputs['MasterParamOut'] = master_param_vars attrs = { - 'mu': - mu, - 'multi_precision': - multi_precision, - 'rescale_grad': - rescale_grad, - 'use_nesterov': - use_nesterov, - 'regularization_method': - ['l2_decay' for i in range(len(param_vars))], + 'mu': mu, + 'multi_precision': multi_precision, + 'rescale_grad': rescale_grad, + 'use_nesterov': use_nesterov, + 'regularization_method': [ + 'l2_decay' for i in range(len(param_vars)) + ], 'regularization_coeff': [2.0 for i in range(len(param_vars))], } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): @@ -270,7 +313,6 @@ def run_momentum_op2(params, class TestMergedMomentum(unittest.TestCase): - def setUp(self): paddle.enable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -296,21 +338,28 @@ def prepare_data(self, shapes, multi_precision, seed, place): return params, grads, velocitys, master_params, learning_rate def check_with_place(self, place, multi_precision): - params, grads, velocitys, master_params, learning_rate = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + velocitys, + master_params, + learning_rate, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_merged): # NPU Momentum Op does not support rescale_grad rescale_grad = 1.0 - return run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - rescale_grad=rescale_grad, - use_merged=use_merged) + return run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + rescale_grad=rescale_grad, + use_merged=use_merged, + ) outs1 = run_op(True) outs2 = run_op(False) @@ -323,7 +372,6 @@ def test_main(self): class TestMergedMomentum2(unittest.TestCase): - def setUp(self): paddle.enable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -349,22 +397,29 @@ def prepare_data(self, shapes, multi_precision, seed, place): return params, grads, velocitys, master_params, learning_rate def check_with_place(self, place, multi_precision): - params, grads, velocitys, master_params, learning_rate = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + velocitys, + master_params, + learning_rate, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_nesterov, use_merged): # NPU Momentum Op does not support rescale_grad rescale_grad = 1.0 - return run_momentum_op2(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - rescale_grad=rescale_grad, - use_merged=use_merged, - use_nesterov=use_nesterov) + return run_momentum_op2( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + rescale_grad=rescale_grad, + use_merged=use_merged, + use_nesterov=use_nesterov, + ) outs1 = run_op(use_nesterov=True, use_merged=True) outs2 = run_op(use_nesterov=True, use_merged=False) diff --git a/python/paddle/fluid/tests/unittests/npu/test_meshgrid_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_meshgrid_op_npu.py index 8ff2fb65aa73e..8af8f89924458 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_meshgrid_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_meshgrid_op_npu.py @@ -26,7 +26,6 @@ class TestMeshgridOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "meshgrid" @@ -55,7 +54,7 @@ def init_test_data(self): ins = [] outs = [] for i in range(len(self.shape)): - ins.append(np.random.random((self.shape[i], )).astype(self.dtype)) + ins.append(np.random.random((self.shape[i],)).astype(self.dtype)) for i in range(len(self.shape)): out_reshape = [1] * len(self.shape) @@ -69,43 +68,47 @@ def get_x_shape(self): @skip_check_grad_ci( - reason="The backward test is not supported for float16 type on NPU.") + reason="The backward test is not supported for float16 type on NPU." +) class TestMeshgridOpFP16(TestMeshgridOp): - def get_dtype(self): return "float16" class TestMeshgridOpINT32(TestMeshgridOp): - def get_dtype(self): return "int32" class TestMeshgridOpINT64(TestMeshgridOp): - def get_dtype(self): return "int64" class TestMeshgridOp2(TestMeshgridOp): - def get_x_shape(self): return [100, 300] class TestMeshgridOp3(unittest.TestCase): - def test_api(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -114,29 +117,35 @@ def test_api(self): exe = fluid.Executor(place=fluid.NPUPlace(0)) grid_x, grid_y = paddle.tensor.meshgrid(x, y) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) np.testing.assert_allclose(res_1, out_1) np.testing.assert_allclose(res_2, out_2) class TestMeshgridOp4(unittest.TestCase): - def test_list_input(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -145,29 +154,35 @@ def test_list_input(self): exe = fluid.Executor(place=fluid.NPUPlace(0)) grid_x, grid_y = paddle.tensor.meshgrid([x, y]) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) np.testing.assert_allclose(res_1, out_1) np.testing.assert_allclose(res_2, out_2) class TestMeshgridOp5(unittest.TestCase): - def test_tuple_input(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -176,27 +191,33 @@ def test_tuple_input(self): exe = fluid.Executor(place=fluid.NPUPlace(0)) grid_x, grid_y = paddle.tensor.meshgrid((x, y)) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) np.testing.assert_allclose(res_1, out_1) np.testing.assert_allclose(res_2, out_2) class TestMeshgridOp6(unittest.TestCase): - def test_api_with_dygraph(self): paddle.disable_static(paddle.NPUPlace(0)) - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_3 = np.reshape(input_3, [100, 1]) out_3 = np.broadcast_to(out_3, [100, 200]) @@ -213,15 +234,22 @@ def test_api_with_dygraph(self): class TestMeshgridOp7(unittest.TestCase): - def test_api_with_dygraph_list_input(self): paddle.disable_static(paddle.NPUPlace(0)) - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_3 = np.reshape(input_3, [100, 1]) out_3 = np.broadcast_to(out_3, [100, 200]) @@ -238,15 +266,22 @@ def test_api_with_dygraph_list_input(self): class TestMeshgridOp8(unittest.TestCase): - def test_api_with_dygraph_tuple_input(self): paddle.disable_static(paddle.NPUPlace(0)) - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_3 = np.reshape(input_3, [100, 1]) out_3 = np.broadcast_to(out_3, [100, 200]) diff --git a/python/paddle/fluid/tests/unittests/npu/test_mixed_precision_npu.py b/python/paddle/fluid/tests/unittests/npu/test_mixed_precision_npu.py index 9927316fddc08..3685ab9e1e1db 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_mixed_precision_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_mixed_precision_npu.py @@ -28,7 +28,6 @@ class SimpleNet(nn.Layer): - def __init__(self, input_size, output_size): super(SimpleNet, self).__init__() self.linear1 = nn.Linear(input_size, output_size) @@ -42,16 +41,15 @@ def forward(self, x): x = self.linear1(x) # currently, paddle's relu may hide nan/inf, relu(nan) = 0, relu(inf)= inf # so, do not use it here. - #x = self.relu1(x) + # x = self.relu1(x) x = self.linear2(x) - #x = self.relu2(x) + # x = self.relu2(x) x = self.linear3(x) return x class AMPTestNpu(unittest.TestCase): - def setUp(self): self.place = paddle.NPUPlace(0) @@ -67,10 +65,11 @@ def net(self): loss = mse(out, label) opt = paddle.fluid.optimizer.Adam( - learning_rate=0.0001, parameter_list=model.parameters()) # 定义优化器 - opt = paddle.static.amp.decorate(opt, - init_loss_scaling=128.0, - use_dynamic_loss_scaling=True) + learning_rate=0.0001, parameter_list=model.parameters() + ) # 定义优化器 + opt = paddle.static.amp.decorate( + opt, init_loss_scaling=128.0, use_dynamic_loss_scaling=True + ) opt.minimize(loss) return model, loss, opt @@ -85,11 +84,17 @@ def test_skip_update(self): model, loss, opt = self.net() weight = model.linear1.weight moment1 = opt._optimizer._get_accumulator( - opt._optimizer._moment1_acc_str, weight) + opt._optimizer._moment1_acc_str, weight + ) beta_pow1 = opt._optimizer._get_accumulator( - opt._optimizer._beta1_pow_acc_str, weight) + opt._optimizer._beta1_pow_acc_str, weight + ) fetch_list = [ - loss, weight, moment1, beta_pow1, 'find_infinite_scale.tmp_0' + loss, + weight, + moment1, + beta_pow1, + 'find_infinite_scale.tmp_0', ] exe = paddle.static.Executor(self.place) @@ -104,20 +109,24 @@ def test_skip_update(self): ] weight_, moment1_, beta_pow1_ = exe.run( - startup_prog, fetch_list=[weight, moment1, beta_pow1]) - pre_weight_, pre_moment1_, pre_beta_pow1_ = weight_, moment1_, beta_pow1_ + startup_prog, fetch_list=[weight, moment1, beta_pow1] + ) + pre_weight_, pre_moment1_, pre_beta_pow1_ = ( + weight_, + moment1_, + beta_pow1_, + ) for i in range(nums_batch): if i % 2: train_data[i][10] = np.inf loss_, weight_, moment1_, beta_pow1_, found_inf = exe.run( main_prog, - feed={ - "X": train_data[i], - "Y": labels[i] - }, - fetch_list=fetch_list) - print(loss_, weight_[0][0], moment1_[0][0], beta_pow1_, - found_inf) + feed={"X": train_data[i], "Y": labels[i]}, + fetch_list=fetch_list, + ) + print( + loss_, weight_[0][0], moment1_[0][0], beta_pow1_, found_inf + ) if i % 2: self.assertTrue(found_inf) np.testing.assert_array_equal(weight_, pre_weight_) @@ -128,7 +137,11 @@ def test_skip_update(self): self.assertFalse(np.array_equal(weight_, pre_weight_)) self.assertFalse(np.array_equal(moment1_, pre_moment1_)) self.assertFalse(np.array_equal(beta_pow1_, pre_beta_pow1_)) - pre_weight_, pre_moment1_, pre_beta_pow1_ = weight_, moment1_, beta_pow1_ + pre_weight_, pre_moment1_, pre_beta_pow1_ = ( + weight_, + moment1_, + beta_pow1_, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_momentum_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_momentum_op_npu.py index 852f4fd3dc2c4..9719c5582bc8c 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_momentum_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_momentum_op_npu.py @@ -28,7 +28,6 @@ class TestMomentumOp1(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -48,7 +47,7 @@ def setUp(self): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = {'mu': mu, 'use_nesterov': self.use_nesterov} @@ -59,7 +58,8 @@ def setUp(self): mu=mu, velocity=velocity, use_nesterov=self.use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -75,7 +75,6 @@ def test_check_output(self): class TestMomentumOpFp16(TestMomentumOp1): - def init_dtype(self): self.dtype = np.float16 @@ -84,23 +83,21 @@ def test_check_output(self): class TestMomentumOp2(TestMomentumOp1): - def init_case(self): self.shape = (123, 321) self.use_nesterov = True class TestMomentumV2(unittest.TestCase): - def test_momentum_dygraph(self): paddle.disable_static(place=fluid.NPUPlace(0)) value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Momentum(learning_rate=0.01, - momentum=0.9, - parameters=linear.parameters()) + adam = paddle.optimizer.Momentum( + learning_rate=0.01, momentum=0.9, parameters=linear.parameters() + ) out = linear(a) out.backward() adam.step() @@ -117,13 +114,15 @@ def test_momentum(self): cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) - rms_optimizer = paddle.optimizer.Momentum(learning_rate=0.1, - momentum=0.9) + rms_optimizer = paddle.optimizer.Momentum( + learning_rate=0.1, momentum=0.9 + ) rms_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -131,14 +130,13 @@ def test_momentum(self): exe.run(main, feed=feeder.feed(data), fetch_list=fetch_list) def test_raise_error(self): - self.assertRaises(ValueError, - paddle.optimizer.Momentum, - learning_rate=None) + self.assertRaises( + ValueError, paddle.optimizer.Momentum, learning_rate=None + ) self.assertRaises(ValueError, paddle.optimizer.Momentum, momentum=None) class TestMomentumOpWithDecay(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -164,14 +162,14 @@ def setUp(self): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = { 'mu': mu, 'use_nesterov': use_nesterov, 'regularization_method': regularization_method, - 'regularization_coeff': regularization_coeff + 'regularization_coeff': regularization_coeff, } grad = grad + regularization_coeff * param @@ -182,7 +180,8 @@ def setUp(self): mu=mu, velocity=velocity, use_nesterov=use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -195,7 +194,6 @@ def test_check_output(self): class TestMomentumOpWithDecayFP16(TestMomentumOpWithDecay): - def init_config(self): self.dtype = np.float16 @@ -205,13 +203,11 @@ def test_check_output(self): class TestMomentumOpWithDecay2(TestMomentumOpWithDecay): - def init_config(self): self.use_nesterov = False class TestMomentumOpWithDecayAPI(unittest.TestCase): - def _test_momentum_dygraph_common(self, regularization): paddle.disable_static(fluid.NPUPlace(0)) inp = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32") @@ -224,13 +220,16 @@ def _test_momentum_dygraph_common(self, regularization): learning_rate=0.01, momentum=0.9, parameter_list=linear.parameters(), - regularization=regularization) + regularization=regularization, + ) momentum.minimize(loss) def test_momentum_dygraph_1(self): self._test_momentum_dygraph_common( regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ) + ) def test_momentum_static(self): paddle.enable_static() @@ -244,12 +243,14 @@ def test_momentum_static(self): avg_cost = paddle.mean(cost) momentum_optimizer = paddle.fluid.contrib.optimizer.Momentum( - learning_rate=0.1, momentum=0.9) + learning_rate=0.1, momentum=0.9 + ) momentum_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -258,11 +259,11 @@ def test_momentum_static(self): class TestMomentumOpVsMomentumOpWithDecayAPI(unittest.TestCase): - def __update_params(self, momentum, linear): for i in range(10): - inp = paddle.full(shape=[2, 2], fill_value=i, - dtype='float32').astype("float32") + inp = paddle.full( + shape=[2, 2], fill_value=i, dtype='float32' + ).astype("float32") inp = paddle.to_tensor(inp) out = linear(inp) loss = paddle.mean(out) @@ -276,39 +277,45 @@ def __test_vs(self, place=fluid.NPUPlace(0)): 2, 2, weight_attr=paddle.nn.initializer.Constant(value=2.0), - bias_attr=paddle.nn.initializer.Constant(value=2.0)) + bias_attr=paddle.nn.initializer.Constant(value=2.0), + ) momentum_old = paddle.fluid.optimizer.Momentum( learning_rate=0.01, momentum=0.9, parameter_list=linear_old.parameters(), regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ), + ) self.__update_params(momentum=momentum_old, linear=linear_old) linear_new = paddle.nn.Linear( 2, 2, weight_attr=paddle.nn.initializer.Constant(value=2.0), - bias_attr=paddle.nn.initializer.Constant(value=2.0)) + bias_attr=paddle.nn.initializer.Constant(value=2.0), + ) momentum_new = paddle.fluid.contrib.optimizer.Momentum( learning_rate=0.01, momentum=0.9, parameter_list=linear_new.parameters(), regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ), + ) self.__update_params(momentum=momentum_new, linear=linear_new) self.assertEqual( (linear_old.weight.numpy() == linear_new.weight.numpy()).all(), True, - 'the param weight updated by two Momentum optimizers should equal') + 'the param weight updated by two Momentum optimizers should equal', + ) def test_vs(self, place=fluid.NPUPlace(0)): self.__test_vs(place=place) class TestMomentumV2Group(TestMomentumV2): - def test_momentum_dygraph(self): paddle.disable_static(place=fluid.NPUPlace(0)) value = np.arange(26).reshape(2, 13).astype("float32") @@ -316,22 +323,20 @@ def test_momentum_dygraph(self): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Momentum(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001, - 'learning_rate': - 0.1, - 'momentum': - 0.99 - }], - weight_decay=0.1, - momentum=0.9) + adam = paddle.optimizer.Momentum( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + 'learning_rate': 0.1, + 'momentum': 0.99, + }, + ], + weight_decay=0.1, + momentum=0.9, + ) out = linear_1(a) out = linear_2(out) out.backward() diff --git a/python/paddle/fluid/tests/unittests/npu/test_mul_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_mul_op_npu.py index 0b3a0c322cde0..8f8abea53964a 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_mul_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_mul_op_npu.py @@ -40,7 +40,7 @@ def setUp(self): np.random.seed(SEED) self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype), - 'Y': np.random.random(self.y_shape).astype(self.dtype) + 'Y': np.random.random(self.y_shape).astype(self.dtype), } self.outputs = {'Out': np.dot(self.inputs['X'], self.inputs['Y'])} @@ -81,9 +81,9 @@ def test_check_grad_ingore_y(self): @skip_check_grad_ci( - reason="Don't support grad checking for NPU OP with FP16 data type.") + reason="Don't support grad checking for NPU OP with FP16 data type." +) class TestMulFP16(TestMul): - def init_dtype(self): self.dtype = np.float16 @@ -112,7 +112,7 @@ def setUp(self): np.random.seed(SEED) self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype), - 'Y': np.random.random(self.y_shape).astype(self.dtype) + 'Y': np.random.random(self.y_shape).astype(self.dtype), } self.outputs = { 'Out': np.dot(self.inputs['X'].reshape(20, 10), self.inputs['Y']) @@ -120,9 +120,9 @@ def setUp(self): @skip_check_grad_ci( - reason="Don't support grad checking for NPU OP with FP16 data type.") + reason="Don't support grad checking for NPU OP with FP16 data type." +) class TestMul2FP16(TestMul2): - def init_dtype(self): self.dtype = np.float16 @@ -152,16 +152,16 @@ def setUp(self): np.random.seed(SEED) self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype), - 'Y': np.random.random(self.y_shape).astype(self.dtype) + 'Y': np.random.random(self.y_shape).astype(self.dtype), } self.attrs = {"x_num_col_dims": 2} self.outputs = {'Out': np.matmul(self.inputs['X'], self.inputs['Y'])} @skip_check_grad_ci( - reason="Don't support grad checking for NPU OP with FP16 data type.") + reason="Don't support grad checking for NPU OP with FP16 data type." +) class TestMul3FP16(TestMul3): - def init_dtype(self): self.dtype = np.float16 @@ -190,7 +190,7 @@ def setUp(self): np.random.seed(SEED) self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype), - 'Y': np.random.random(self.y_shape).astype(self.dtype) + 'Y': np.random.random(self.y_shape).astype(self.dtype), } self.outputs = { 'Out': np.dot(self.inputs['X'].reshape(20, 12), self.inputs['Y']) @@ -198,9 +198,9 @@ def setUp(self): @skip_check_grad_ci( - reason="Don't support grad checking for NPU OP with FP16 data type.") + reason="Don't support grad checking for NPU OP with FP16 data type." +) class TestMul4FP16(TestMul4): - def init_dtype(self): self.dtype = np.float16 @@ -215,7 +215,6 @@ def test_check_grad_ingore_y(self): class TestMulNet(unittest.TestCase): - def init_dtype(self): self.dtype = np.float32 @@ -237,9 +236,9 @@ def _test(self, run_npu=True): b = paddle.static.data(name="b", shape=[2, 3], dtype=self.dtype) c = paddle.static.data(name="c", shape=[3, 2], dtype=self.dtype) d = paddle.static.data(name="d", shape=[3, 2], dtype=self.dtype) - label = paddle.static.data(name="label", - shape=[2, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[2, 1], dtype='int64' + ) sum_1 = paddle.add(a, b) sum_2 = paddle.add(c, d) @@ -263,18 +262,23 @@ def _test(self, run_npu=True): print("TestMulNet Start run on {} . ".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "c": c_np, - "d": d_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={ + "a": a_np, + "b": b_np, + "c": c_np, + "d": d_np, + "label": label_np, + }, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -288,7 +292,6 @@ def test_npu(self): class TestMulNet3_2(unittest.TestCase): - def init_dtype(self): self.dtype = np.float32 @@ -310,9 +313,9 @@ def _test(self, run_npu=True): b = paddle.static.data(name="b", shape=[2, 3, 4], dtype=self.dtype) c = paddle.static.data(name="c", shape=[12, 5], dtype=self.dtype) d = paddle.static.data(name="d", shape=[12, 5], dtype=self.dtype) - label = paddle.static.data(name="label", - shape=[2, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[2, 1], dtype='int64' + ) sum_1 = paddle.add(a, b) sum_2 = paddle.add(c, d) @@ -336,18 +339,23 @@ def _test(self, run_npu=True): print("testMulNet3_2 tart run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "c": c_np, - "d": d_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={ + "a": a_np, + "b": b_np, + "c": c_np, + "d": d_np, + "label": label_np, + }, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -356,13 +364,13 @@ def test_npu(self): cpu_pred, cpu_loss = self._test(False) npu_pred, npu_loss = self._test(True) - np.testing.assert_allclose(npu_pred, cpu_pred, - atol=1e-5) # atol needed on cann 20.3 + np.testing.assert_allclose( + npu_pred, cpu_pred, atol=1e-5 + ) # atol needed on cann 20.3 np.testing.assert_allclose(npu_loss, cpu_loss, atol=1e-5) class TestMulNet3_2_xc2(unittest.TestCase): - def init_dtype(self): self.dtype = np.float32 @@ -384,9 +392,9 @@ def _test(self, run_npu=True): b = paddle.static.data(name="b", shape=[2, 3, 4], dtype=self.dtype) c = paddle.static.data(name="c", shape=[4, 5], dtype=self.dtype) d = paddle.static.data(name="d", shape=[4, 5], dtype=self.dtype) - label = paddle.static.data(name="label", - shape=[2, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[2, 1], dtype='int64' + ) sum_1 = paddle.add(a, b) sum_2 = paddle.add(c, d) @@ -411,18 +419,23 @@ def _test(self, run_npu=True): print("TestMulNet3_2_xc2. Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "c": c_np, - "d": d_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={ + "a": a_np, + "b": b_np, + "c": c_np, + "d": d_np, + "label": label_np, + }, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -436,7 +449,6 @@ def test_npu(self): class TestMulNet4_2(unittest.TestCase): - def init_dtype(self): self.dtype = np.float32 @@ -458,17 +470,18 @@ def _test(self, run_npu=True): b = paddle.static.data(name="b", shape=[12, 5], dtype=self.dtype) c = paddle.static.data(name="c", shape=[12, 5], dtype=self.dtype) d = paddle.static.data(name="d", shape=[12, 5], dtype=self.dtype) - label = paddle.static.data(name="label", - shape=[2, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[2, 1], dtype='int64' + ) sum_1 = paddle.add(a, b) # [12, 5] sum_2 = paddle.add(c, d) # [12, 5] fc_1 = fluid.layers.fc(input=sum_1, size=2) # [12, 2] fc_1_re_shape = paddle.reshape(fc_1, shape=[2, 3, 2, 2]) fc_2 = fluid.layers.fc(input=sum_2, size=2) # [12, 2] - result = paddle.fluid.layers.mul(fc_1_re_shape, - fc_2) # [2, 3, 2, 2] * [12, 2] + result = paddle.fluid.layers.mul( + fc_1_re_shape, fc_2 + ) # [2, 3, 2, 2] * [12, 2] prediction = fluid.layers.fc(input=result, size=2, act='softmax') @@ -487,18 +500,23 @@ def _test(self, run_npu=True): print("testMulNet4_2 tart run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "c": c_np, - "d": d_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={ + "a": a_np, + "b": b_np, + "c": c_np, + "d": d_np, + "label": label_np, + }, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -507,8 +525,9 @@ def test_npu(self): cpu_pred, cpu_loss = self._test(False) npu_pred, npu_loss = self._test(True) - np.testing.assert_allclose(npu_pred, cpu_pred, - atol=1e-5) # atol needed on cann 20.3 + np.testing.assert_allclose( + npu_pred, cpu_pred, atol=1e-5 + ) # atol needed on cann 20.3 np.testing.assert_allclose(npu_loss, cpu_loss, atol=1e-5) diff --git a/python/paddle/fluid/tests/unittests/npu/test_multinomial_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_multinomial_op_npu.py index d1599766756fb..ccff4ffd0cfb5 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_multinomial_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_multinomial_op_npu.py @@ -47,7 +47,6 @@ def sample_output_two_dimension(out, shape): class TestMultinomialOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "multinomial" @@ -65,8 +64,9 @@ def init_data(self): self.attrs = {"num_samples": 100000, "replacement": True} def test_check_output(self): - self.check_output_customized(self.verify_output, - custom_place=self.place) + self.check_output_customized( + self.verify_output, custom_place=self.place + ) def sample_output(self, out): return sample_output_one_dimension(out, 4) @@ -79,7 +79,6 @@ def verify_output(self, outs): class TestMultinomialOp2(TestMultinomialOp): - def init_data(self): # input probability is a matrix self.input_np = np.random.rand(3, 4) @@ -91,7 +90,6 @@ def sample_output(self, out): class TestMultinomialOp3(TestMultinomialOp): - def init_data(self): # replacement is False. number of samples must be less than number of categories. self.input_np = np.random.rand(1000) @@ -102,12 +100,13 @@ def verify_output(self, outs): out = np.array(outs[0]) unique_out = np.unique(out) self.assertEqual( - len(unique_out), 100, - "replacement is False. categories can't be sampled repeatedly") + len(unique_out), + 100, + "replacement is False. categories can't be sampled repeatedly", + ) class TestMultinomialApi(unittest.TestCase): - def test_dygraph(self): # input probability is a vector, and replacement is True paddle.set_device('npu:0') @@ -144,8 +143,10 @@ def test_dygraph3(self): unique_out = np.unique(out.numpy()) self.assertEqual( - len(unique_out), 100, - "replacement is False. categories can't be sampled repeatedly") + len(unique_out), + 100, + "replacement is False. categories can't be sampled repeatedly", + ) paddle.enable_static() def test_dygraph4(self): @@ -178,19 +179,17 @@ def test_static(self): class TestMultinomialAlias(unittest.TestCase): - def test_alias(self): paddle.set_device('npu:0') x = paddle.rand([4]) out1 = paddle.multinomial(x, num_samples=10, replacement=True) out2 = paddle.tensor.multinomial(x, num_samples=10, replacement=True) - out3 = paddle.tensor.random.multinomial(x, - num_samples=10, - replacement=True) + out3 = paddle.tensor.random.multinomial( + x, num_samples=10, replacement=True + ) class TestMultinomialError(unittest.TestCase): - def setUp(self): paddle.set_device('npu:0') paddle.disable_static() @@ -199,7 +198,6 @@ def tearDown(self): paddle.enable_static() def test_num_sample(self): - def test_num_sample_less_than_0(): x = paddle.rand([4]) out = paddle.multinomial(x, num_samples=-2) @@ -207,7 +205,6 @@ def test_num_sample_less_than_0(): self.assertRaises(ValueError, test_num_sample_less_than_0) def test_input_probs_dim(self): - def test_dim_larger_than_2(): x = paddle.rand([2, 3, 3]) out = paddle.multinomial(x) diff --git a/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_op_npu.py index 9c9d50e1ddac2..82a2ff48251df 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_op_npu.py @@ -27,7 +27,6 @@ class TestNearestInterpOp(OpTest): - def setUp(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -52,10 +51,15 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners, - self.data_layout) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -67,7 +71,7 @@ def setUp(self): 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } self.outputs = {'Out': output_np} @@ -83,124 +87,113 @@ def init_test_case(self): self.input_shape = [2, 3, 4, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = False class TestNearestNeighborInterpCase1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpCase2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = False def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - in_place=True, - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['X'], 'Out', in_place=True, max_relative_error=0.006 + ) class TestNearestNeighborInterpCase3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpCase4(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = False class TestNearestNeighborInterpCase5(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") self.align_corners = False class TestNearestNeighborInterpCase6(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 129]).astype("int32") self.align_corners = False class TestNearestNeighborInterpSame(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpActualShape(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = False class TestNearestNeighborInterpDataLayout(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 4, 4, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 8]).astype("int32") self.align_corners = False self.data_layout = "NHWC" class TestNearestInterpOpUint8(OpTest): - def setUp(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -209,8 +202,9 @@ def setUp(self): self.actual_shape = None self.init_test_case() self.op_type = "nearest_interp" - input_np = np.random.randint(low=0, high=256, - size=self.input_shape).astype("uint8") + input_np = np.random.randint( + low=0, high=256, size=self.input_shape + ).astype("uint8") if self.scale > 0: out_h = int(self.input_shape[2] * self.scale) @@ -219,9 +213,14 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -230,7 +229,7 @@ def setUp(self): 'out_w': self.out_w, 'scale': self.scale, 'interp_method': self.interp_method, - 'align_corners': self.align_corners + 'align_corners': self.align_corners, } self.outputs = {'Out': output_np} @@ -242,47 +241,43 @@ def init_test_case(self): self.input_shape = [1, 3, 9, 6] self.out_h = 10 self.out_w = 9 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpCase1Uint8(TestNearestInterpOpUint8): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] self.out_h = 80 self.out_w = 40 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpCase2Uint8(TestNearestInterpOpUint8): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 5 self.out_w = 13 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([6, 15]).astype("int32") self.align_corners = False class TestNearestNeighborInterpScale1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 2. + self.scale = 2.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = False class TestNearestNeighborInterpScale2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 5, 7] @@ -294,19 +289,17 @@ def init_test_case(self): class TestNearestNeighborInterpScale3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 1. + self.scale = 1.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = False class TestNearestInterpOp_attr_tensor(OpTest): - def setUp(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -340,15 +333,21 @@ def setUp(self): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h self.attrs['out_w'] = self.out_w - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): @@ -362,33 +361,31 @@ def init_test_case(self): self.input_shape = [2, 5, 4, 4] self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [3, 3] self.align_corners = False # out_size is a tensor list class TestNearestInterp_attr_tensor_Case1(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = [8, 12] self.align_corners = False # out_size is a 1-D tensor class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = False self.shape_by_1Dtensor = True @@ -396,7 +393,6 @@ def init_test_case(self): # scale is a 1-D tensor class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] @@ -409,7 +405,6 @@ def init_test_case(self): class TestNearestAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") y = fluid.data(name="y", shape=[2, 6, 6, 3], dtype="float32") @@ -417,27 +412,25 @@ def test_case(self): dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[2], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") - - out1 = fluid.layers.resize_nearest(y, - out_shape=[12, 12], - data_format='NHWC', - align_corners=False) - out2 = fluid.layers.resize_nearest(x, - out_shape=[12, dim], - align_corners=False) - out3 = fluid.layers.resize_nearest(x, - out_shape=shape_tensor, - align_corners=False) - out4 = fluid.layers.resize_nearest(x, - out_shape=[4, 4], - actual_shape=actual_size, - align_corners=False) - out5 = fluid.layers.resize_nearest(x, - scale=scale_tensor, - align_corners=False) + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) + + out1 = fluid.layers.resize_nearest( + y, out_shape=[12, 12], data_format='NHWC', align_corners=False + ) + out2 = fluid.layers.resize_nearest( + x, out_shape=[12, dim], align_corners=False + ) + out3 = fluid.layers.resize_nearest( + x, out_shape=shape_tensor, align_corners=False + ) + out4 = fluid.layers.resize_nearest( + x, out_shape=[4, 4], actual_shape=actual_size, align_corners=False + ) + out5 = fluid.layers.resize_nearest( + x, scale=scale_tensor, align_corners=False + ) x_data = np.random.random((2, 3, 6, 6)).astype("float32") dim_data = np.array([12]).astype("int32") @@ -448,38 +441,39 @@ def test_case(self): place = paddle.NPUPlace(0) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": np.transpose(x_data, (0, 2, 3, 1)), - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = nearest_neighbor_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=False) - np.testing.assert_allclose(results[0], - np.transpose(expect_res, (0, 2, 3, 1))) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": np.transpose(x_data, (0, 2, 3, 1)), + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = nearest_neighbor_interp_np( + x_data, out_h=12, out_w=12, align_corners=False + ) + np.testing.assert_allclose( + results[0], np.transpose(expect_res, (0, 2, 3, 1)) + ) for i in range(len(results) - 1): np.testing.assert_allclose(results[i + 1], expect_res) class TestNearestInterpException(unittest.TestCase): - def test_exception(self): input = fluid.data(name="input", shape=[1, 3, 6, 6], dtype="float32") def attr_data_format(): # for 4-D input, data_format can only be NCHW or NHWC - out = fluid.layers.resize_nearest(input, - out_shape=[4, 8], - data_format='NDHWC') + out = fluid.layers.resize_nearest( + input, out_shape=[4, 8], data_format='NDHWC' + ) def attr_scale_type(): out = fluid.layers.resize_nearest(input, scale='scale') diff --git a/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_v2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_v2_op_npu.py index 6c4b6f97b03eb..4e0b6089835f0 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_v2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_nearest_interp_v2_op_npu.py @@ -30,7 +30,6 @@ class TestNearestInterpOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -68,11 +67,17 @@ def setUp(self): output_h = self.out_h output_w = self.out_w - output_np = nearest_neighbor_interp_np(input_np, output_h, output_w, - scale_h, scale_w, self.out_size, - self.actual_shape, - self.align_corners, - self.data_layout) + output_np = nearest_neighbor_interp_np( + input_np, + output_h, + output_w, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -83,7 +88,7 @@ def setUp(self): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -99,15 +104,17 @@ def test_check_output(self): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - in_place=True, - max_relative_error=0.02) + self.check_grad_with_place( + self.place, ['X'], 'Out', in_place=True, max_relative_error=0.02 + ) else: - self.check_grad_with_place(self.place, ['X'], - 'Out', - in_place=True, - max_relative_error=0.006) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + in_place=True, + max_relative_error=0.006, + ) def init_dtype(self): self.dtype = np.float32 @@ -117,123 +124,112 @@ def init_test_case(self): self.input_shape = [2, 3, 4, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = False class TestNearestNeighborInterpFP16(TestNearestInterpOp): - def init_dtype(self): self.dtype = np.float16 class TestNearestNeighborInterpCase1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpCase2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpCase3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpCase4(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = False class TestNearestNeighborInterpCase5(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") self.align_corners = False class TestNearestNeighborInterpCase6(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 129]).astype("int32") self.align_corners = False class TestNearestNeighborInterpSame(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestNearestNeighborInterpActualShape(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = False class TestNearestNeighborInterpScale1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 2. + self.scale = 2.0 self.out_size = None self.align_corners = False class TestNearestNeighborInterpScale2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 5, 7] @@ -245,7 +241,6 @@ def init_test_case(self): class TestNearestNeighborInterpScale3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] @@ -257,7 +252,6 @@ def init_test_case(self): class TestNearestInterpOp_attr_tensor(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -300,8 +294,9 @@ def setUp(self): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h @@ -313,9 +308,16 @@ def setUp(self): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): @@ -329,33 +331,31 @@ def init_test_case(self): self.input_shape = [2, 5, 4, 4] self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [3, 3] self.align_corners = False # out_size is a tensor list class TestNearestInterp_attr_tensor_Case1(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = [8, 12] self.align_corners = False # out_size is a 1-D tensor class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = False self.shape_by_1Dtensor = True @@ -363,7 +363,6 @@ def init_test_case(self): # scale is a 1-D tensor class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] @@ -376,9 +375,9 @@ def init_test_case(self): class TestNearestInterpOpAPI_dy(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_npu(): place = core.NPUPlace(0) else: @@ -388,14 +387,15 @@ def test_case(self): scale_np = np.array([2, 2]).astype("int64") input_x = paddle.to_tensor(input_data) scale = paddle.to_tensor(scale_np) - expect_res = nearest_neighbor_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - scale_factor=scale, - mode="nearest", - align_corners=False) + expect_res = nearest_neighbor_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, + scale_factor=scale, + mode="nearest", + align_corners=False, + ) np.testing.assert_allclose(out.numpy(), expect_res) diff --git a/python/paddle/fluid/tests/unittests/npu/test_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_norm_op_npu.py index e0a62d5641acb..c59daf979fb89 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_norm_op_npu.py @@ -24,7 +24,6 @@ class TestNPUNormOp(OpTest): - def setUp(self): paddle.enable_static() self.set_npu() @@ -54,13 +53,12 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.006 + ) class TestNPUNormOp2(TestNPUNormOp): - def init_test_case(self): self.shape = [5, 3, 9, 7] self.axis = 0 @@ -68,17 +66,17 @@ def init_test_case(self): class TestNPUNormOp3(TestNPUNormOp): - def init_test_case(self): self.shape = [5, 3, 2, 7] self.axis = -1 self.epsilon = 1e-8 -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class TestNPUNormOp4(TestNPUNormOp): - def init_test_case(self): self.shape = [128, 1024, 14, 14] self.axis = 2 @@ -88,10 +86,11 @@ def test_check_grad(self): pass -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class TestNPUNormOp5(TestNPUNormOp): - def init_test_case(self): self.shape = [2048, 2048] self.axis = 1 @@ -102,7 +101,6 @@ def test_check_grad(self): class API_NormTest(unittest.TestCase): - def test_errors(self): paddle.enable_static() with fluid.program_guard(fluid.Program()): @@ -115,7 +113,6 @@ def test_norm_x_type(): class TestNPUNormOpFP16(TestNPUNormOp): - def set_npu(self): self.__class__.use_npu = True self.__class__.no_need_check_grad = True diff --git a/python/paddle/fluid/tests/unittests/npu/test_npu_place.py b/python/paddle/fluid/tests/unittests/npu/test_npu_place.py index 0075fa49e0440..8f4a9ee03e39d 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_npu_place.py +++ b/python/paddle/fluid/tests/unittests/npu/test_npu_place.py @@ -21,7 +21,6 @@ class TestNpuPlace(unittest.TestCase): - def test(self): p = core.Place() p.set_place(paddle.NPUPlace(0)) @@ -31,7 +30,6 @@ def test(self): class TestNpuPlaceError(unittest.TestCase): - def test_static(self): # NPU is not supported in ParallelExecutor prog = paddle.static.Program() @@ -48,8 +46,9 @@ def test_static(self): place = paddle.NPUPlace(0) exe = paddle.static.Executor(place) - with self.assertRaisesRegex(RuntimeError, - "NPU is not supported in ParallelExecutor"): + with self.assertRaisesRegex( + RuntimeError, "NPU is not supported in ParallelExecutor" + ): exe.run(compiled_prog, feed={"x": x_np, "y": y_np}) diff --git a/python/paddle/fluid/tests/unittests/npu/test_one_hot_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_one_hot_op_npu.py index 5379e9505dfd9..008645deca359 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_one_hot_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_one_hot_op_npu.py @@ -28,7 +28,6 @@ class TestOneHotOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -42,8 +41,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -57,7 +57,6 @@ def test_check_output(self): class TestOneHotOp_attr(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -70,8 +69,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -85,7 +85,6 @@ def test_check_output(self): class TestOneHotOp_default_dtype(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -99,8 +98,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -114,7 +114,6 @@ def test_check_output(self): class TestOneHotOp_default_dtype_attr(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -127,8 +126,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -142,7 +142,6 @@ def test_check_output(self): class TestOneHotOp_out_of_range(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -154,8 +153,9 @@ def setUp(self): x = [np.random.choice([-1, depth]) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) self.inputs = {'X': (x, x_lod)} self.attrs = {'depth': depth, 'allow_out_of_range': True} @@ -166,7 +166,6 @@ def test_check_output(self): class TestOneHotOp_dtype_int64(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -179,8 +178,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int64').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 diff --git a/python/paddle/fluid/tests/unittests/npu/test_one_hot_v2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_one_hot_v2_op_npu.py index b3fd2e6922135..4ccd33134bed7 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_one_hot_v2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_one_hot_v2_op_npu.py @@ -28,7 +28,6 @@ class TestOneHotOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -56,7 +55,6 @@ def test_check_output(self): class TestOneHotOp_non_lod(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -80,7 +78,6 @@ def test_check_output(self): class TestOneHotOp_attr(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -93,8 +90,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -108,7 +106,6 @@ def test_check_output(self): class TestOneHotOp_default_dtype(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -136,7 +133,6 @@ def test_check_output(self): class TestOneHotOp_default_dtype_attr(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -149,8 +145,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -164,7 +161,6 @@ def test_check_output(self): class TestOneHotOp_out_of_range(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -187,7 +183,6 @@ def test_check_output(self): class TestOneHotOp_dtype_int64(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -210,7 +205,6 @@ def test_check_output(self): class TestOneHotOpApi(unittest.TestCase): - def test_api(self): depth = 10 self._run(depth) @@ -221,27 +215,32 @@ def test_api_with_depthTensor(self): def test_api_with_dygraph(self): depth = 10 - label = np.array([np.random.randint(0, depth - 1) - for i in range(6)]).reshape([6, 1]) + label = np.array( + [np.random.randint(0, depth - 1) for i in range(6)] + ).reshape([6, 1]) with fluid.dygraph.guard(paddle.NPUPlace(0)): one_hot_label = fluid.one_hot( - input=fluid.dygraph.to_variable(label), depth=depth) + input=fluid.dygraph.to_variable(label), depth=depth + ) def _run(self, depth): label = fluid.layers.data(name="label", shape=[1], dtype="int64") one_hot_label = fluid.one_hot(input=label, depth=depth) place = fluid.NPUPlace(0) - label_data = np.array([np.random.randint(0, 10 - 1) - for i in range(6)]).reshape([6, 1]) + label_data = np.array( + [np.random.randint(0, 10 - 1) for i in range(6)] + ).reshape([6, 1]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'label': label_data, - }, - fetch_list=[one_hot_label], - return_numpy=False) + ret = exe.run( + feed={ + 'label': label_data, + }, + fetch_list=[one_hot_label], + return_numpy=False, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_p_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_p_norm_op_npu.py index 5560b8bbd143a..8cb1d0b11a8d9 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_p_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_p_norm_op_npu.py @@ -26,7 +26,6 @@ class TestPnormOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -41,7 +40,7 @@ def setUp(self): 'epsilon': self.epsilon, 'axis': self.axis, 'keepdim': self.keepdim, - 'porder': float(self.porder) + 'porder': float(self.porder), } self.outputs = {'Out': norm} self.gradient = self.calc_gradient() @@ -53,9 +52,9 @@ def test_check_output(self): self.check_output_with_place(paddle.NPUPlace(0)) def test_check_grad(self): - self.check_grad_with_place(paddle.NPUPlace(0), ['X'], - 'Out', - user_defined_grads=self.gradient) + self.check_grad_with_place( + paddle.NPUPlace(0), ['X'], 'Out', user_defined_grads=self.gradient + ) def init_test_case(self): self.shape = [2, 3, 4, 5] @@ -73,7 +72,7 @@ def calc_gradient(self): 'epsilon': self.epsilon, 'axis': self.axis, 'keepdim': self.keepdim, - 'porder': float(self.porder) + 'porder': float(self.porder), } x = self.inputs["X"] porder = self.attrs["porder"] @@ -87,8 +86,11 @@ def calc_gradient(self): grad[x_abs != norm] = 0.0 else: norm = p_norm(x, axis=axis, porder=porder, keepdims=True) - grad = np.power(norm, 1 - porder) * np.power( - np.abs(x), porder - 1) * np.sign(x) + grad = ( + np.power(norm, 1 - porder) + * np.power(np.abs(x), porder - 1) + * np.sign(x) + ) numel = 1 for s in x.shape: @@ -98,7 +100,6 @@ def calc_gradient(self): class TestPnormOp2(TestPnormOp): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -109,7 +110,6 @@ def init_test_case(self): class TestPnormOp3(TestPnormOp): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -120,7 +120,6 @@ def init_test_case(self): class TestPnormOp4(TestPnormOp3): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -131,7 +130,6 @@ def init_test_case(self): class TestPnormOp5(TestPnormOp3): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -142,7 +140,6 @@ def init_test_case(self): class TestPnormOp6(TestPnormOp3): - def init_test_case(self): self.shape = [2, 3, 4, 5] self.axis = 1 @@ -153,31 +150,26 @@ def init_test_case(self): class TestPnormOpfp16(TestPnormOp): - def init_dtype(self): self.dtype = "float16" class TestPnormOp2fp16(TestPnormOp2): - def init_dtype(self): self.dtype = "float16" class TestPnormOp3fp16(TestPnormOp3): - def init_dtype(self): self.dtype = "float16" class TestPnormOp4fp16(TestPnormOp4): - def init_dtype(self): self.dtype = "float16" class TestPnormOp5fp16(TestPnormOp5): - def init_dtype(self): self.dtype = "float16" diff --git a/python/paddle/fluid/tests/unittests/npu/test_pad3d_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_pad3d_op_npu.py index f388e66881c87..5d0d25e1f2081 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_pad3d_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_pad3d_op_npu.py @@ -27,7 +27,6 @@ class TestPad3dNPUOp(op_test.OpTest): - def setUp(self): paddle.enable_static() self.__class__.use_npu = True @@ -39,16 +38,18 @@ def setUp(self): self.variable_paddings = False self.initTestCase() - self.value = 0 #Asend npu only support constant_values = 0 right now. + self.value = 0 # Asend npu only support constant_values = 0 right now. self.inputs = {'X': np.random.random(self.shape).astype(self.x_type)} self.attrs = {} if self.variable_paddings: self.attrs['paddings'] = [] - self.inputs['Paddings'] = np.array( - self.paddings).flatten().astype("int32") + self.inputs['Paddings'] = ( + np.array(self.paddings).flatten().astype("int32") + ) else: - self.attrs['paddings'] = np.array( - self.paddings).flatten().astype("int32") + self.attrs['paddings'] = ( + np.array(self.paddings).flatten().astype("int32") + ) self.attrs['value'] = self.value self.attrs['mode'] = self.mode self.attrs['data_format'] = self.data_format @@ -69,10 +70,12 @@ def setUp(self): (0, 0), ] - out = np.pad(self.inputs['X'], - paddings, - mode=self.mode, - constant_values=self.value) + out = np.pad( + self.inputs['X'], + paddings, + mode=self.mode, + constant_values=self.value, + ) self.outputs = {'Out': out} @@ -89,7 +92,6 @@ def initTestCase(self): class TestCase1(TestPad3dNPUOp): - def initTestCase(self): self.shape = (3, 4, 5, 6, 7) self.paddings = [0, 1, 2, 3, 4, 5] @@ -102,7 +104,6 @@ def test_check_grad(self): class TestCase2(TestPad3dNPUOp): - def initTestCase(self): self.shape = (4, 5, 6, 7, 8) self.paddings = [1, 1, 1, 1, 1, 1] @@ -111,13 +112,9 @@ def initTestCase(self): class TestPadAPI(unittest.TestCase): - - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0, - data_format="NCDHW"): + def _get_numpy_out( + self, input_data, pad, mode, value=0, data_format="NCDHW" + ): if mode == "constant" and len(pad) == len(input_data.shape) * 2: pad = np.reshape(pad, (-1, 2)).tolist() elif data_format == "NCDHW": @@ -168,8 +165,11 @@ def _get_numpy_out(self, def test_static(self): paddle.enable_static() - self.place = fluid.NPUPlace( - 0) if fluid.core.is_compiled_with_npu() else fluid.CPUPlace() + self.place = ( + fluid.NPUPlace(0) + if fluid.core.is_compiled_with_npu() + else fluid.CPUPlace() + ) with program_guard(Program(), Program()): input_shape = (1, 2, 3, 4, 5) pad = [1, 2, 1, 1, 3, 4] @@ -177,31 +177,25 @@ def test_static(self): value = 0 input_data = np.random.rand(*input_shape).astype(np.float32) x = paddle.fluid.data(name="x", shape=input_shape) - result1 = F.pad(x=x, - pad=pad, - value=value, - mode=mode, - data_format="NCDHW") - result2 = F.pad(x=x, - pad=pad, - value=value, - mode=mode, - data_format="NDHWC") + result1 = F.pad( + x=x, pad=pad, value=value, mode=mode, data_format="NCDHW" + ) + result2 = F.pad( + x=x, pad=pad, value=value, mode=mode, data_format="NDHWC" + ) exe = Executor(self.place) - fetches = exe.run(default_main_program(), - feed={"x": input_data}, - fetch_list=[result1, result2]) - - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NCDHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NDHWC") + fetches = exe.run( + default_main_program(), + feed={"x": input_data}, + fetch_list=[result1, result2], + ) + + np_out1 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NCDHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NDHWC" + ) np.testing.assert_allclose(fetches[0], np_out1) np.testing.assert_allclose(fetches[1], np_out2) @@ -216,27 +210,19 @@ def test_dygraph_1(self): input_data = np.random.rand(*input_shape).astype(np.float32) tensor_data = paddle.to_tensor(input_data) - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NCDHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NDHWC") - - y1 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NCDHW") - y2 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NDHWC") + np_out1 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NCDHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NDHWC" + ) + + y1 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NCDHW" + ) + y2 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NDHWC" + ) np.testing.assert_allclose(y1.numpy(), np_out1) np.testing.assert_allclose(y2.numpy(), np_out2) @@ -252,27 +238,19 @@ def test_dygraph_2(self): input_data = np.random.rand(*input_shape).astype(np.float32) tensor_data = paddle.to_tensor(input_data) - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NCHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NHWC") - - y1 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NCHW") - y2 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NHWC") + np_out1 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NCHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NHWC" + ) + + y1 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NCHW" + ) + y2 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NHWC" + ) np.testing.assert_allclose(y1.numpy(), np_out1) np.testing.assert_allclose(y2.numpy(), np_out2) @@ -288,40 +266,28 @@ def test_dygraph_3(self): input_data = np.random.rand(*input_shape).astype(np.float32) tensor_data = paddle.to_tensor(input_data) - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NCL") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NLC") - - y1 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NCL") - y2 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NLC") + np_out1 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NCL" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NLC" + ) + + y1 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NCL" + ) + y2 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NLC" + ) np.testing.assert_allclose(y1.numpy(), np_out1) np.testing.assert_allclose(y2.numpy(), np_out2) class TestPad1dAPI(unittest.TestCase): - - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0.0, - data_format="NCL"): + def _get_numpy_out( + self, input_data, pad, mode, value=0.0, data_format="NCL" + ): if data_format == "NCL": pad = [ (0, 0), @@ -348,36 +314,33 @@ def test_class(self): input_data = np.random.rand(*input_shape).astype(np.float32) pad_constant = nn.Pad1D(padding=pad, mode="constant", value=value) - pad_constant_int = nn.Pad1D(padding=pad_int, - mode="constant", - value=value) + pad_constant_int = nn.Pad1D( + padding=pad_int, mode="constant", value=value + ) data = paddle.to_tensor(input_data) output = pad_constant(data) - np_out = self._get_numpy_out(input_data, - pad, - "constant", - value=value, - data_format="NCL") + np_out = self._get_numpy_out( + input_data, pad, "constant", value=value, data_format="NCL" + ) np.testing.assert_allclose(output.numpy(), np_out) output = pad_constant_int(data) - np_out = self._get_numpy_out(input_data, [pad_int] * 2, - "constant", - value=value, - data_format="NCL") + np_out = self._get_numpy_out( + input_data, + [pad_int] * 2, + "constant", + value=value, + data_format="NCL", + ) np.testing.assert_allclose(output.numpy(), np_out) class TestPad2dAPI(unittest.TestCase): - - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0.0, - data_format="NCHW"): + def _get_numpy_out( + self, input_data, pad, mode, value=0.0, data_format="NCHW" + ): if data_format == "NCHW": pad = [ (0, 0), @@ -406,36 +369,33 @@ def test_class(self): input_data = np.random.rand(*input_shape).astype(np.float32) pad_constant = nn.Pad2D(padding=pad, mode="constant", value=value) - pad_constant_int = nn.Pad2D(padding=pad_int, - mode="constant", - value=value) + pad_constant_int = nn.Pad2D( + padding=pad_int, mode="constant", value=value + ) data = paddle.to_tensor(input_data) output = pad_constant(data) - np_out = self._get_numpy_out(input_data, - pad, - "constant", - value=value, - data_format="NCHW") + np_out = self._get_numpy_out( + input_data, pad, "constant", value=value, data_format="NCHW" + ) np.testing.assert_allclose(output.numpy(), np_out) output = pad_constant_int(data) - np_out = self._get_numpy_out(input_data, [pad_int] * 4, - "constant", - value=value, - data_format="NCHW") + np_out = self._get_numpy_out( + input_data, + [pad_int] * 4, + "constant", + value=value, + data_format="NCHW", + ) np.testing.assert_allclose(output.numpy(), np_out) class TestPad3dAPI(unittest.TestCase): - - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0.0, - data_format="NCDHW"): + def _get_numpy_out( + self, input_data, pad, mode, value=0.0, data_format="NCDHW" + ): if data_format == "NCDHW": pad = [ (0, 0), @@ -466,32 +426,31 @@ def test_class(self): input_data = np.random.rand(*input_shape).astype(np.float32) pad_constant = nn.Pad3D(padding=pad, mode="constant", value=value) - pad_constant_int = nn.Pad3D(padding=pad_int, - mode="constant", - value=value) + pad_constant_int = nn.Pad3D( + padding=pad_int, mode="constant", value=value + ) data = paddle.to_tensor(input_data) output = pad_constant(data) - np_out = self._get_numpy_out(input_data, - pad, - "constant", - value=value, - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, pad, "constant", value=value, data_format="NCDHW" + ) np.testing.assert_allclose(output.numpy(), np_out) output = pad_constant_int(data) - np_out = self._get_numpy_out(input_data, [pad_int] * 6, - "constant", - value=value, - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, + [pad_int] * 6, + "constant", + value=value, + data_format="NCDHW", + ) np.testing.assert_allclose(output.numpy(), np_out) class TestPad3dOpNpuError(unittest.TestCase): - def test_errors(self): - def test_value(): input_shape = (1, 2, 3, 4, 5) data = np.random.rand(*input_shape).astype(np.float32) @@ -538,14 +497,16 @@ def test_mode_3(): class TestPadDataformatError(unittest.TestCase): - def test_errors(self): - def test_ncl(): input_shape = (1, 2, 3, 4) pad = paddle.to_tensor(np.array([2, 1, 2, 1]).astype('int32')) - data = np.arange(np.prod(input_shape), - dtype=np.float64).reshape(input_shape) + 1 + data = ( + np.arange(np.prod(input_shape), dtype=np.float64).reshape( + input_shape + ) + + 1 + ) my_pad = nn.Pad1D(padding=pad, mode="replicate", data_format="NCL") data = paddle.to_tensor(data) result = my_pad(data) @@ -553,8 +514,12 @@ def test_ncl(): def test_nchw(): input_shape = (1, 2, 4) pad = paddle.to_tensor(np.array([2, 1, 2, 1]).astype('int32')) - data = np.arange(np.prod(input_shape), - dtype=np.float64).reshape(input_shape) + 1 + data = ( + np.arange(np.prod(input_shape), dtype=np.float64).reshape( + input_shape + ) + + 1 + ) my_pad = nn.Pad1D(padding=pad, mode="replicate", data_format="NCHW") data = paddle.to_tensor(data) result = my_pad(data) @@ -562,11 +527,15 @@ def test_nchw(): def test_ncdhw(): input_shape = (1, 2, 3, 4) pad = paddle.to_tensor(np.array([2, 1, 2, 1]).astype('int32')) - data = np.arange(np.prod(input_shape), - dtype=np.float64).reshape(input_shape) + 1 - my_pad = nn.Pad1D(padding=pad, - mode="replicate", - data_format="NCDHW") + data = ( + np.arange(np.prod(input_shape), dtype=np.float64).reshape( + input_shape + ) + + 1 + ) + my_pad = nn.Pad1D( + padding=pad, mode="replicate", data_format="NCDHW" + ) data = paddle.to_tensor(data) result = my_pad(data) diff --git a/python/paddle/fluid/tests/unittests/npu/test_pad_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_pad_op_npu.py index 7b16b19f21477..97d2a83260146 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_pad_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_pad_op_npu.py @@ -27,7 +27,6 @@ class TestPadOp(OpTest): - def setUp(self): self.op_type = "pad" self.set_npu() @@ -41,11 +40,12 @@ def setUp(self): self.attrs['paddings'] = np.array(self.paddings).flatten() self.attrs['pad_value'] = self.pad_value self.outputs = { - 'Out': - np.pad(self.inputs['X'], - self.paddings, - mode='constant', - constant_values=self.pad_value) + 'Out': np.pad( + self.inputs['X'], + self.paddings, + mode='constant', + constant_values=self.pad_value, + ) } def test_check_output(self): @@ -53,9 +53,9 @@ def test_check_output(self): def test_check_grad_normal(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.6) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.6 + ) else: self.check_grad_with_place(self.place, ['X'], 'Out') @@ -73,7 +73,6 @@ def initTestCase(self): class TestCase1(TestPadOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.paddings = [(0, 1), (2, 3), (2, 1), (1, 1)] @@ -81,7 +80,6 @@ def initTestCase(self): class TestCase2(TestPadOp): - def initTestCase(self): self.shape = (5, 5, 5) self.paddings = [(0, 0), (0, 0), (1, 2)] @@ -89,20 +87,17 @@ def initTestCase(self): class TestCase3(TestPadOp): - def initTestCase(self): - self.shape = (100) + self.shape = 100 self.paddings = [(0, 1)] self.pad_value = 0.0 -#----------------Pad Fp16---------------- +# ----------------Pad Fp16---------------- def create_test_fp16(parent): - class TestPadFp16(parent): - def init_dtype(self): self.dtype = np.float16 @@ -118,7 +113,6 @@ def init_dtype(self): class TestPadOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.random((2, 2)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/npu/test_parallel_dygraph_mnist_npu.py b/python/paddle/fluid/tests/unittests/npu/test_parallel_dygraph_mnist_npu.py index a2be1ccfa67be..0eb98ff7a68d7 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_parallel_dygraph_mnist_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_parallel_dygraph_mnist_npu.py @@ -46,23 +46,20 @@ }""" need_envs = { - "ASCEND_AICPU_PATH": - os.getenv("ASCEND_AICPU_PATH", "/usr/local/Ascend/nnae/latest"), - "ASCEND_OPP_PATH": - os.getenv("ASCEND_OPP_PATH", "/usr/local/Ascend/nnae/latest/opp"), - "HCCL_CONNECT_TIMEOUT": - "7200", - "HCCL_WHITELIST_DISABLE": - "1", - "HCCL_SECURITY_MODE": - "1", - "RANK_TABLE_FILE": - "rank_table_file.json", + "ASCEND_AICPU_PATH": os.getenv( + "ASCEND_AICPU_PATH", "/usr/local/Ascend/nnae/latest" + ), + "ASCEND_OPP_PATH": os.getenv( + "ASCEND_OPP_PATH", "/usr/local/Ascend/nnae/latest/opp" + ), + "HCCL_CONNECT_TIMEOUT": "7200", + "HCCL_WHITELIST_DISABLE": "1", + "HCCL_SECURITY_MODE": "1", + "RANK_TABLE_FILE": "rank_table_file.json", } class TestParallelDygraphMnistNPU(TestDistBase): - def _setup_config(self): self._sync_mode = False self._hccl_mode = True @@ -78,11 +75,11 @@ def test_mnist(self): delta=1e-3, check_error_log=True, need_envs=need_envs, - log_name=flag_name) + log_name=flag_name, + ) class TestFleetDygraphMnistNPU(TestParallelDygraphMnistNPU): - def _setup_config(self): self._sync_mode = False self._hccl_mode = True diff --git a/python/paddle/fluid/tests/unittests/npu/test_pool2d_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_pool2d_op_npu.py index cb554d3c1d489..3abe452b4e56b 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_pool2d_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_pool2d_op_npu.py @@ -22,16 +22,20 @@ import paddle.fluid as fluid import paddle.fluid.core as core from op_test import OpTest -from test_pool2d_op import pool2D_forward_naive, avg_pool2D_forward_naive, max_pool2D_forward_naive, adaptive_start_index, adaptive_end_index +from test_pool2d_op import ( + pool2D_forward_naive, + avg_pool2D_forward_naive, + max_pool2D_forward_naive, + adaptive_start_index, + adaptive_end_index, +) from paddle.nn.functional import avg_pool2d, max_pool2d paddle.enable_static() def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.paddings = [0, 0] self.padding_algorithm = "SAME" @@ -42,9 +46,7 @@ def init_paddings(self): def create_test_use_ceil_class(parent): - class TestPool2DUseCeilCase(parent): - def init_ceil_mode(self): self.ceil_mode = True @@ -54,9 +56,7 @@ def init_ceil_mode(self): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.paddings = [1, 1] self.padding_algorithm = "VALID" @@ -67,9 +67,7 @@ def init_paddings(self): def create_test_fp16_class(parent): - class TestFp16Case(parent): - def init_kernel_type(self): self.use_cudnn = False self.dtype = np.float16 @@ -79,25 +77,29 @@ def init_kernel_type(self): globals()[cls_name] = TestFp16Case -def pool2d_backward_navie(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_format='NCHW', - pool_type="max", - padding_algorithm="EXPLICIT"): +def pool2d_backward_navie( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_format='NCHW', + pool_type="max", + padding_algorithm="EXPLICIT", +): # update paddings def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -107,9 +109,10 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): if isinstance(padding_algorithm, str): padding_algorithm = padding_algorithm.upper() if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if padding_algorithm == "VALID": paddings = [0, 0, 0, 0] @@ -117,7 +120,8 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): raise ValueError( "When Attr(pool_padding) is \"VALID\", Attr(ceil_mode)" " must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) elif padding_algorithm == "SAME": input_data_shape = [] if data_format == "NCHW": @@ -146,10 +150,20 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) // strides[0] + 1 \ - if ceil_mode else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 - W_out = (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) // strides[1] + 1 \ - if ceil_mode else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 + H_out = ( + (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) + // strides[0] + + 1 + if ceil_mode + else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) + // strides[1] + + 1 + if ceil_mode + else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 + ) x_grad = np.zeros_like(x) for i in range(H_out): @@ -177,20 +191,26 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): in_w_end = np.min((in_w_end, W)) if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (in_h_end - in_h_start) * (in_w_end - - in_w_start) - x_grad[:, :, in_h_start:in_h_end, - in_w_start:in_w_end] += 1 / field_size + if exclusive or adaptive: + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) + x_grad[:, :, in_h_start:in_h_end, in_w_start:in_w_end] += ( + 1 / field_size + ) elif pool_type == 'max': for n in range(N): for c in range(C): - idx = np.argmax(x[n, c, in_h_start:in_h_end, - in_w_start:in_w_end].flatten()) + idx = np.argmax( + x[ + n, c, in_h_start:in_h_end, in_w_start:in_w_end + ].flatten() + ) idx_h = idx // (in_w_end - in_w_start) idx_w = idx % (in_w_end - in_w_start) - x_grad[n, c, in_h_start + idx_h, - in_w_start + idx_w] += 1 + x_grad[ + n, c, in_h_start + idx_h, in_w_start + idx_w + ] += 1 if data_format == "NHWC": x_grad = x_grad.transpose([0, 2, 3, 1]) @@ -198,7 +218,6 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): class TestPool2D_Op(OpTest): - def setUp(self): self.set_npu() self.op_type = "pool2d" @@ -218,14 +237,24 @@ def setUp(self): input = np.random.random(self.shape).astype(self.dtype) if self.pool_type == "max": - input = np.array([x for x in range(np.prod(self.shape)) - ]).reshape(self.shape).astype(self.dtype) - output = pool2D_forward_naive(input, self.ksize, self.strides, - self.paddings, self.global_pool, - self.ceil_mode, self.exclusive, - self.adaptive, self.data_format, - self.pool_type, - self.padding_algorithm).astype(self.dtype) + input = ( + np.array([x for x in range(np.prod(self.shape))]) + .reshape(self.shape) + .astype(self.dtype) + ) + output = pool2D_forward_naive( + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + self.data_format, + self.pool_type, + self.padding_algorithm, + ).astype(self.dtype) self.inputs = {'X': OpTest.np_dtype_to_fluid_dtype(input)} self.attrs = { @@ -288,27 +317,30 @@ def test_check_output(self): self.check_output_with_place(fluid.NPUPlace(0), atol=1e-3) def test_check_grad(self): - x_grad = pool2d_backward_navie(self.inputs["X"], - ksize=self.ksize, - strides=self.strides, - paddings=self.paddings, - global_pool=self.global_pool, - ceil_mode=False, - exclusive=self.exclusive, - adaptive=self.adaptive, - data_format=self.data_format, - pool_type=self.pool_type, - padding_algorithm=self.padding_algorithm) + x_grad = pool2d_backward_navie( + self.inputs["X"], + ksize=self.ksize, + strides=self.strides, + paddings=self.paddings, + global_pool=self.global_pool, + ceil_mode=False, + exclusive=self.exclusive, + adaptive=self.adaptive, + data_format=self.data_format, + pool_type=self.pool_type, + padding_algorithm=self.padding_algorithm, + ) x_grad = x_grad / np.prod(self.outputs['Out'].shape) - self.check_grad_with_place(fluid.NPUPlace(0), - set(['X']), - 'Out', - max_relative_error=0.06, - user_defined_grads=[x_grad]) + self.check_grad_with_place( + fluid.NPUPlace(0), + set(['X']), + 'Out', + max_relative_error=0.06, + user_defined_grads=[x_grad], + ) class TestCase1(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -328,7 +360,6 @@ def init_shape(self): class TestCase2(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -348,34 +379,29 @@ def init_shape(self): class TestCase3(TestPool2D_Op): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase4(TestCase1): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase5(TestCase2): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestAvgInclude(TestCase2): - def init_exclusive(self): self.exclusive = False class TestAvgPoolAdaptive(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -389,7 +415,6 @@ def init_test_case(self): class TestAvgPoolAdaptiveAsyOutSize(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -404,9 +429,8 @@ def init_test_case(self): self.paddings = [0, 0, 0, 0] -#-------test pool2d with asymmetric padding----- +# -------test pool2d with asymmetric padding----- class TestPool2D_AsyPadding(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -417,7 +441,6 @@ def init_shape(self): class TestCase1_AsyPadding(TestCase1): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -428,7 +451,6 @@ def init_shape(self): class TestCase2_AsyPadding(TestCase2): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -439,7 +461,6 @@ def init_shape(self): class TestCase3_AsyPadding(TestCase3): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -450,7 +471,6 @@ def init_shape(self): class TestCase4_AsyPadding(TestCase4): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -461,7 +481,6 @@ def init_shape(self): class TestCase5_AsyPadding((TestCase5)): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -472,7 +491,6 @@ def init_shape(self): class TestAvgInclude_AsyPadding(TestCase2): - def init_exclusive(self): self.exclusive = False @@ -486,7 +504,6 @@ def init_shape(self): class TestAvgPoolAdaptive_AsyPadding(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -499,9 +516,8 @@ def init_shape(self): self.shape = [2, 3, 8, 8] -#----------- test channel_last -------------- +# ----------- test channel_last -------------- class TestPool2D_channel_last(TestPool2D_Op): - def init_data_format(self): self.data_format = "NHWC" @@ -510,7 +526,6 @@ def init_shape(self): class TestCase1_channel_last(TestCase1): - def init_data_format(self): self.data_format = "NHWC" @@ -519,7 +534,6 @@ def init_shape(self): class TestCase2_channel_last(TestCase2): - def init_data_format(self): self.data_format = "NHWC" @@ -528,7 +542,6 @@ def init_shape(self): class TestCase3_channel_last(TestCase3): - def init_data_format(self): self.data_format = "NHWC" @@ -537,7 +550,6 @@ def init_shape(self): class TestCase4_channel_last(TestCase4): - def init_data_format(self): self.data_format = "NHWC" @@ -546,7 +558,6 @@ def init_shape(self): class TestCase5_channel_last(TestCase5): - def init_data_format(self): self.data_format = "NHWC" @@ -555,13 +566,11 @@ def init_shape(self): class TestCase5_Max(TestCase2): - def init_pool_type(self): self.pool_type = "max" class TestCase5_channel_last_Max(TestCase5_Max): - def init_data_format(self): self.data_format = "NHWC" @@ -570,13 +579,11 @@ def init_shape(self): class TestAvgInclude_channel_last(TestCase2_channel_last): - def init_exclusive(self): self.exclusive = False class TestAvgPoolAdaptive_channel_last(TestCase1_channel_last): - def init_adaptive(self): self.adaptive = True @@ -589,7 +596,6 @@ def init_test_case(self): class TestPool2D_AsyPadding_channel_last(TestPool2D_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -598,7 +604,6 @@ def init_shape(self): class TestCase1_AsyPadding_channel_last(TestCase1_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -607,7 +612,6 @@ def init_shape(self): class TestCase2_AsyPadding_channel_last(TestCase2_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -616,7 +620,6 @@ def init_shape(self): class TestCase3_AsyPadding_channel_last(TestCase3_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -625,7 +628,6 @@ def init_shape(self): class TestCase4_AsyPadding_channel_last(TestCase4_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -634,7 +636,6 @@ def init_shape(self): class TestCase5_AsyPadding_channel_last(TestCase5_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -643,7 +644,6 @@ def init_shape(self): class TestAvgInclude_AsyPadding_channel_last(TestAvgInclude_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -651,9 +651,9 @@ def init_shape(self): self.shape = [2, 7, 7, 3] -class TestAvgPoolAdaptive_AsyPadding_channel_last(TestAvgPoolAdaptive_AsyPadding - ): - +class TestAvgPoolAdaptive_AsyPadding_channel_last( + TestAvgPoolAdaptive_AsyPadding +): def init_data_format(self): self.data_format = "NHWC" @@ -662,7 +662,6 @@ def init_shape(self): class TestCase1_strides(TestCase1): - def init_test_case(self): self.ksize = [3, 3] # fixme: CANN AvgPoolGradV3 dose not support asymmetric strides diff --git a/python/paddle/fluid/tests/unittests/npu/test_pow_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_pow_op_npu.py index 67bb2f9c40fe0..09f2d0fc055c1 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_pow_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_pow_op_npu.py @@ -26,7 +26,6 @@ class TestPow(OpTest): - def setUp(self): self.set_npu() self.op_type = "pow" @@ -55,7 +54,6 @@ def test_check_grad(self): class TestPowFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "pow" @@ -82,7 +80,6 @@ def test_check_output(self): class TestPowNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -97,9 +94,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -123,16 +120,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_prior_box_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_prior_box_op_npu.py index 4af37416af5c1..08e9226a37cb1 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_prior_box_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_prior_box_op_npu.py @@ -18,13 +18,15 @@ import sys import math -from paddle.fluid.tests.unittests.op_test import OpTest, _set_use_system_allocator +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + _set_use_system_allocator, +) paddle.enable_static() class TestNPUPriorBox(OpTest): - def setUp(self): self.op_type = "prior_box" self.set_npu() @@ -56,7 +58,7 @@ def set_data(self): 'min_max_aspect_ratios_order': self.min_max_aspect_ratios_order, 'step_w': self.step_w, 'step_h': self.step_h, - 'offset': self.offset + 'offset': self.offset, } if len(self.max_sizes) > 0: self.attrs['max_sizes'] = self.max_sizes @@ -92,8 +94,9 @@ def init_test_params(self): self.flip = True self.set_min_max_aspect_ratios_order() self.real_aspect_ratios = [1, 2.0, 1.0 / 2.0, 3.0, 1.0 / 3.0] - self.aspect_ratios = np.array(self.aspect_ratios, - dtype=np.float64).flatten() + self.aspect_ratios = np.array( + self.aspect_ratios, dtype=np.float64 + ).flatten() self.variances = [0.1, 0.1, 0.2, 0.2] self.variances = np.array(self.variances, dtype=np.float64).flatten() @@ -105,12 +108,12 @@ def init_test_params(self): def init_test_input(self): self.image = np.random.random( - (self.batch_size, self.image_channels, self.image_w, - self.image_h)).astype('float32') + (self.batch_size, self.image_channels, self.image_w, self.image_h) + ).astype('float32') self.input = np.random.random( - (self.batch_size, self.input_channels, self.layer_w, - self.layer_h)).astype('float32') + (self.batch_size, self.input_channels, self.layer_w, self.layer_h) + ).astype('float32') def init_test_output(self): out_dim = (self.layer_h, self.layer_w, self.num_priors, 4) @@ -131,73 +134,78 @@ def init_test_output(self): ar = self.real_aspect_ratios[r] c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 else: - c_w = c_h = min_size / 2. - out_boxes[h, w, idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + c_w = c_h = min_size / 2.0 + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # rest of priors for r in range(len(self.real_aspect_ratios)): ar = self.real_aspect_ratios[r] - if abs(ar - 1.) < 1e-6: + if abs(ar - 1.0) < 1e-6: continue c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # clip the prior's coordidate such that it is within[0, 1] if self.clip: out_boxes = np.clip(out_boxes, 0.0, 1.0) # set the variance. - out_var = np.tile(self.variances, - (self.layer_h, self.layer_w, self.num_priors, 1)) + out_var = np.tile( + self.variances, (self.layer_h, self.layer_w, self.num_priors, 1) + ) self.out_boxes = out_boxes.astype('float32') self.out_var = out_var.astype('float32') class TestNPUPriorBoxWithoutMaxSize(TestNPUPriorBox): - def set_max_sizes(self): self.max_sizes = [] class TestNPUPriorBoxWithoutSpecifiedOutOrder(TestNPUPriorBox): - def set_min_max_aspect_ratios_order(self): self.min_max_aspect_ratios_order = False self.atol = 1e-1 diff --git a/python/paddle/fluid/tests/unittests/npu/test_randperm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_randperm_op_npu.py index ca2d41e763333..71d955f196087 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_randperm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_randperm_op_npu.py @@ -27,7 +27,7 @@ class TestRandpermOp(OpTest): - """ Test randperm op.""" + """Test randperm op.""" def setUp(self): self.set_npu() @@ -57,36 +57,32 @@ def test_check_output(self): def verify_output(self, outs): out_np = np.array(outs[0]) - self.assertTrue(check_randperm_out(self.n, out_np), - msg=error_msg(out_np)) + self.assertTrue( + check_randperm_out(self.n, out_np), msg=error_msg(out_np) + ) class TestRandpermOpN(TestRandpermOp): - def init_attrs(self): self.n = 10000 class TestRandpermOpInt32(TestRandpermOp): - def init_attrs(self): self.dtype = "int32" class TestRandpermOpFloat32(TestRandpermOp): - def init_attrs(self): self.dtype = "float32" class TestRandpermOpFloat64(TestRandpermOp): - def init_attrs(self): self.dtype = "float64" class TestRandpermOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): self.assertRaises(ValueError, paddle.randperm, -3) @@ -94,7 +90,6 @@ def test_errors(self): class TestRandpermAPI(unittest.TestCase): - def test_out(self): n = 10 place = paddle.NPUPlace(0) @@ -112,15 +107,15 @@ def test_out(self): class TestRandpermImperative(unittest.TestCase): - def test_out(self): paddle.disable_static(paddle.NPUPlace(0)) n = 10 for dtype in ['int32', np.int64, 'float32', 'float64']: data_p = paddle.randperm(n, dtype) data_np = data_p.numpy() - self.assertTrue(check_randperm_out(n, data_np), - msg=error_msg(data_np)) + self.assertTrue( + check_randperm_out(n, data_np), msg=error_msg(data_np) + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/npu/test_range_npu.py b/python/paddle/fluid/tests/unittests/npu/test_range_npu.py index 8a89b57e38a3c..b0f4ad242f777 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_range_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_range_npu.py @@ -24,7 +24,6 @@ class TestRangeOp(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -36,13 +35,13 @@ def setUp(self): self.inputs = { 'Start': np.array([self.case[0]]).astype(self.dtype), 'End': np.array([self.case[1]]).astype(self.dtype), - 'Step': np.array([self.case[2]]).astype(self.dtype) + 'Step': np.array([self.case[2]]).astype(self.dtype), } self.outputs = { - 'Out': - np.arange(self.case[0], self.case[1], - self.case[2]).astype(self.dtype) + 'Out': np.arange(self.case[0], self.case[1], self.case[2]).astype( + self.dtype + ) } def init_config(self): @@ -54,49 +53,42 @@ def test_check_output(self): class TestFloatRangeOpCase0(TestRangeOp): - def init_config(self): self.dtype = np.float32 self.case = (0, 5, 1) class TestInt32RangeOpCase0(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.case = (0, 5, 2) class TestInt32RangeOpCase1(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.case = (10, 1, -2) class TestInt32RangeOpCase2(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.case = (-1, -10, -2) class TestInt64RangeOpCase0(TestRangeOp): - def init_config(self): self.dtype = np.int64 self.case = (0, 5, 2) class TestInt64RangeOpCase1(TestRangeOp): - def init_config(self): self.dtype = np.int64 self.case = (10, 1, -2) class TestInt64RangeOpCase2(TestRangeOp): - def init_config(self): self.dtype = np.int64 self.case = (-1, -10, -2) diff --git a/python/paddle/fluid/tests/unittests/npu/test_reciprocal_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reciprocal_op_npu.py index b67cc43efcfa1..2344b22cdea3f 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reciprocal_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reciprocal_op_npu.py @@ -24,7 +24,6 @@ class TestNPUReciprocal(OpTest): - def setUp(self): self.op_type = "reciprocal" self.set_npu() @@ -41,9 +40,9 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.01 + ) def set_npu(self): self.__class__.use_npu = True @@ -54,7 +53,6 @@ def init_dtype(self): class TestNPUReciprocalFp64(TestNPUReciprocal): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -64,9 +62,9 @@ def init_dtype(self): @skip_check_grad_ci( - reason="The backward test is not supported for float16 type on NPU.") + reason="The backward test is not supported for float16 type on NPU." +) class TestNPUReciprocalFp16(TestNPUReciprocal): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/npu/test_reduce_any_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reduce_any_op_npu.py index 9ab2e58228c19..ece002e47992a 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reduce_any_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reduce_any_op_npu.py @@ -28,14 +28,14 @@ class TestAny8DOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "reduce_any" self.place = paddle.NPUPlace(0) self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.attrs = {'dim': (3, 5, 4)} self.outputs = {'Out': self.inputs['X'].any(axis=self.attrs['dim'])} @@ -48,7 +48,6 @@ def test_check_output(self): class TestAnyOpWithDim(OpTest): - def setUp(self): self.set_npu() self.op_type = "reduce_any" @@ -65,14 +64,14 @@ def test_check_output(self): class TestAny8DOpWithDim(OpTest): - def setUp(self): self.set_npu() self.op_type = "reduce_any" self.place = paddle.NPUPlace(0) self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.attrs = {'dim': (3, 6)} self.outputs = {'Out': self.inputs['X'].any(axis=self.attrs['dim'])} @@ -85,7 +84,6 @@ def test_check_output(self): class TestAnyOpWithKeepDim(OpTest): - def setUp(self): self.set_npu() self.op_type = "reduce_any" @@ -93,8 +91,9 @@ def setUp(self): self.inputs = {'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool")} self.attrs = {'dim': (1), 'keep_dim': True} self.outputs = { - 'Out': - np.expand_dims(self.inputs['X'].any(axis=self.attrs['dim']), axis=1) + 'Out': np.expand_dims( + self.inputs['X'].any(axis=self.attrs['dim']), axis=1 + ) } def set_npu(self): @@ -105,19 +104,20 @@ def test_check_output(self): class TestAny8DOpWithKeepDim(OpTest): - def setUp(self): self.set_npu() self.op_type = "reduce_any" self.place = paddle.NPUPlace(0) self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.attrs = {'dim': (1), 'keep_dim': True} self.outputs = { - 'Out': - np.expand_dims(self.inputs['X'].any(axis=self.attrs['dim']), axis=1) + 'Out': np.expand_dims( + self.inputs['X'].any(axis=self.attrs['dim']), axis=1 + ) } def set_npu(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_reduce_max_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reduce_max_op_npu.py index 22ababf99b066..8e0cdd711b4f2 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reduce_max_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reduce_max_op_npu.py @@ -26,7 +26,8 @@ @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestNPUReduceMaxOp(OpTest): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -54,7 +55,8 @@ def init_dtype(self): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpMultiAxises(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -72,7 +74,8 @@ def setUp(self): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceAll(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -88,7 +91,8 @@ def setUp(self): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_bool(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -100,17 +104,19 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.BOOL) + 'out_dtype': int(core.VarDesc.VarType.BOOL), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype(np.bool_) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.bool_) } @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_int16(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -122,18 +128,20 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT16) + 'out_dtype': int(core.VarDesc.VarType.INT16), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype(np.int16) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.int16) } @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_int32(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -145,17 +153,19 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT32) + 'out_dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype(np.int32) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.int32) } @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_int64(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -167,17 +177,19 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT64) + 'out_dtype': int(core.VarDesc.VarType.INT64), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype(np.int64) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.int64) } @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_fp16(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -189,12 +201,12 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP16) + 'out_dtype': int(core.VarDesc.VarType.FP16), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype( - np.float16) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.float16) } def test_check_output(self): @@ -203,7 +215,8 @@ def test_check_output(self): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_fp32(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -215,18 +228,19 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_fp64(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -238,18 +252,19 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP64) + 'out_dtype': int(core.VarDesc.VarType.FP64), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype( - np.float64) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.float64) } @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpWithOutDtype_fp32_2(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -261,12 +276,12 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } def init_dtype(self): @@ -275,7 +290,8 @@ def init_dtype(self): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpInt64(TestNPUReduceMaxOp): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -287,12 +303,12 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT64) + 'out_dtype': int(core.VarDesc.VarType.INT64), } self.outputs = { - 'Out': - self.inputs['X'].max(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .max(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } def init_dtype(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_reduce_mean_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reduce_mean_op_npu.py index 232d53d49395a..fe56cb152c6e2 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reduce_mean_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reduce_mean_op_npu.py @@ -25,7 +25,6 @@ class TestMeanOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -43,7 +42,6 @@ def test_check_grad(self): class TestMeanOp5D(TestMeanOp): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -54,7 +52,6 @@ def setUp(self): class TestMeanOp6D(TestMeanOp): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -65,7 +62,6 @@ def setUp(self): class TestMeanOp8D(TestMeanOp): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -77,7 +73,6 @@ def setUp(self): class Test1DReduce(TestMeanOp): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -86,7 +81,6 @@ def setUp(self): class Test2DReduce0(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -96,7 +90,6 @@ def setUp(self): class Test2DReduce1(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -108,7 +101,6 @@ def setUp(self): class Test3DReduce0(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -120,7 +112,6 @@ def setUp(self): class Test3DReduce1(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -132,7 +123,6 @@ def setUp(self): class Test3DReduce2(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -144,7 +134,6 @@ def setUp(self): class Test3DReduce3(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -156,21 +145,19 @@ def setUp(self): class TestKeepDimReduce(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" self.inputs = {'X': np.random.random((5, 6, 10)).astype("float32")} self.attrs = {'dim': [1], 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].mean(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].mean( + axis=tuple(self.attrs['dim']), keepdims=self.attrs['keep_dim'] + ) } class TestKeepDim8DReduce(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" @@ -179,14 +166,13 @@ def setUp(self): } self.attrs = {'dim': (3, 4, 5), 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].mean(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].mean( + axis=tuple(self.attrs['dim']), keepdims=self.attrs['keep_dim'] + ) } class TestReduceAll(Test1DReduce): - def setUp(self): self.set_npu() self.op_type = "reduce_mean" diff --git a/python/paddle/fluid/tests/unittests/npu/test_reduce_min_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reduce_min_op_npu.py index da9d8a1893b94..2ced99b6020f3 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reduce_min_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reduce_min_op_npu.py @@ -26,7 +26,8 @@ @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestNPUReduceMinOp(OpTest): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -54,7 +55,8 @@ def init_dtype(self): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpMultiAxises(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -72,7 +74,8 @@ def setUp(self): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceAll(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -88,7 +91,8 @@ def setUp(self): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_bool(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -100,17 +104,19 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.BOOL) + 'out_dtype': int(core.VarDesc.VarType.BOOL), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype(np.bool_) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.bool_) } @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_int16(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -122,18 +128,20 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT16) + 'out_dtype': int(core.VarDesc.VarType.INT16), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype(np.int16) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.int16) } @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_int32(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -145,17 +153,19 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT32) + 'out_dtype': int(core.VarDesc.VarType.INT32), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype(np.int32) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.int32) } @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_int64(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -167,17 +177,19 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT64) + 'out_dtype': int(core.VarDesc.VarType.INT64), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype(np.int64) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.int64) } @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_fp16(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -189,12 +201,12 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP16) + 'out_dtype': int(core.VarDesc.VarType.FP16), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype( - np.float16) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.float16) } def test_check_output(self): @@ -203,7 +215,8 @@ def test_check_output(self): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_fp32(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -215,18 +228,19 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_fp64(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -238,18 +252,19 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP64) + 'out_dtype': int(core.VarDesc.VarType.FP64), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype( - np.float64) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.float64) } @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpWithOutDtype_fp32_2(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -261,12 +276,12 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } def init_dtype(self): @@ -275,7 +290,8 @@ def init_dtype(self): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpInt64(TestNPUReduceMinOp): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -287,12 +303,12 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = { 'dim': [-2, -1], - 'out_dtype': int(core.VarDesc.VarType.INT64) + 'out_dtype': int(core.VarDesc.VarType.INT64), } self.outputs = { - 'Out': - self.inputs['X'].min(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .min(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } def init_dtype(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_reduce_prod_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reduce_prod_op_npu.py index e8f4430d655f4..956a2258224c3 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reduce_prod_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reduce_prod_op_npu.py @@ -25,7 +25,6 @@ class TestNPUReduceProd(OpTest): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -49,7 +48,6 @@ def init_dtype(self): class TestNPUReduceProd2(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -61,7 +59,6 @@ def setUp(self): class TestNPUReduceProd3(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -73,7 +70,6 @@ def setUp(self): class TestNPUReduceProd6D(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -89,7 +85,6 @@ def setUp(self): class TestNPUReduceProd8D(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -105,7 +100,6 @@ def setUp(self): class TestReduceAll(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -117,7 +111,6 @@ def setUp(self): class TestNPUReduceProdWithOutDtype_bool(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -126,14 +119,13 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.BOOL)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.bool_) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.bool_) } class TestNPUReduceProdWithOutDtype_int16(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -142,14 +134,13 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.INT16)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.int16) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.int16) } class TestNPUReduceProdWithOutDtype_int32(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -158,14 +149,13 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.INT32)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.int32) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.int32) } class TestNPUReduceProdWithOutDtype_int64(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -174,14 +164,13 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.INT64)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.int64) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.int64) } class TestNPUReduceProdWithOutDtype_fp16(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -190,9 +179,9 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.FP16)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.float16) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.float16) } def test_check_output(self): @@ -200,7 +189,6 @@ def test_check_output(self): class TestNPUReduceProdWithOutDtype_fp32(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -209,14 +197,13 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.FP32)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } class TestNPUReduceProdWithOutDtype_fp64(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -225,15 +212,14 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.FP64)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.float64) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.float64) } @skip_check_grad_ci(reason="right now not implement grad op") class TestNPUReduceProdWithOutDtype_fp32_2(TestNPUReduceProd): - def setUp(self): self.op_type = "reduce_prod" self.set_npu() @@ -242,9 +228,9 @@ def setUp(self): self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [0], 'out_dtype': int(core.VarDesc.VarType.FP32)} self.outputs = { - 'Out': - self.inputs['X'].prod(axis=tuple(self.attrs['dim'])).astype( - np.float32) + 'Out': self.inputs['X'] + .prod(axis=tuple(self.attrs['dim'])) + .astype(np.float32) } def init_dtype(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_reduce_sum_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reduce_sum_op_npu.py index 498cc2d97ace0..d5cd5febfb9fe 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reduce_sum_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reduce_sum_op_npu.py @@ -26,7 +26,6 @@ class TestReduceSum(OpTest): - def setUp(self): np.random.seed(SEED) self.set_npu() @@ -39,16 +38,16 @@ def setUp(self): self.attrs = { 'dim': self.axis, 'keep_dim': self.keep_dim, - 'reduce_all': self.reduce_all + 'reduce_all': self.reduce_all, } self.inputs = {'X': np.random.random(self.shape).astype(self.dtype)} if self.attrs['reduce_all']: self.outputs = {'Out': self.inputs['X'].sum()} else: self.outputs = { - 'Out': - self.inputs['X'].sum(axis=self.axis, - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].sum( + axis=self.axis, keepdims=self.attrs['keep_dim'] + ) } def set_npu(self): @@ -65,7 +64,7 @@ def init_op_type(self): def initTestCase(self): self.shape = (5, 6) - self.axis = (0) + self.axis = 0 def test_check_output(self): self.check_output_with_place(self.place) @@ -79,13 +78,11 @@ def test_check_output(self): class TestReduceSum2(OpTest): - def init_dtype(self): self.dtype = np.int32 class TestReduceSumNet(unittest.TestCase): - def set_reduce_sum_function(self, x): # keep_dim = False return paddle.fluid.layers.reduce_sum(x, dim=-1) @@ -104,9 +101,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[2, 3, 4], dtype='float32') b = paddle.static.data(name="b", shape=[2, 3, 4], dtype='float32') - label = paddle.static.data(name="label", - shape=[2, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[2, 1], dtype='int64' + ) a_1 = fluid.layers.fc(input=a, size=4, num_flatten_dims=2, act=None) b_1 = fluid.layers.fc(input=b, size=4, num_flatten_dims=2, act=None) @@ -131,16 +128,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -153,14 +151,12 @@ def test_npu(self): class TestReduceSumNet2(TestReduceSumNet): - def set_reduce_sum_function(self, x): # keep_dim = True return paddle.fluid.layers.reduce_sum(x, dim=-1, keep_dim=True) class TestReduceSumNet3(TestReduceSumNet): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -191,12 +187,9 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np - }, - fetch_list=[loss]) + loss_res = exe.run( + main_prog, feed={"a": a_np, "b": b_np}, fetch_list=[loss] + ) if epoch % 10 == 0: print("Epoch {} | Loss: {}".format(epoch, loss_res)) diff --git a/python/paddle/fluid/tests/unittests/npu/test_relu6_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_relu6_op_npu.py index 0fe796a0b0da0..3988b4a6a9395 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_relu6_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_relu6_op_npu.py @@ -34,7 +34,6 @@ def ref_relu6(x, threshold=6.0): class TestRelu6(OpTest): - def setUp(self): self.set_npu() self.op_type = "relu6" @@ -64,7 +63,6 @@ def init_dtype(self): class TestRelu6Float16(TestRelu6): - def set_npu(self): self.__class__.use_npu = True self.__class__.no_need_check_grad = True @@ -77,7 +75,6 @@ def test_check_output(self): class TestReluNeg(TestRelu6): - def setUp(self): self.set_npu() self.op_type = "relu6" @@ -104,7 +101,6 @@ def test_check_output(self): class TestRelu6Net(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -119,9 +115,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.nn.functional.relu6(sum) @@ -145,16 +141,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_relu_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_relu_op_npu.py index 98c4632935026..a52e4d39cfd77 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_relu_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_relu_op_npu.py @@ -26,7 +26,6 @@ class TestRelu(OpTest): - def setUp(self): self.set_npu() self.op_type = "relu" @@ -53,21 +52,19 @@ def test_check_output(self): def test_check_grad(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.006 + ) else: self.check_grad_with_place(self.place, ['X'], 'Out') class TestReluFp16(TestRelu): - def init_dtype(self): self.dtype = np.float16 class TestReluNeg(OpTest): - def setUp(self): self.set_npu() self.op_type = "relu" @@ -97,7 +94,6 @@ def test_check_output(self): class TestReluNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -112,9 +108,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.nn.functional.relu(sum) @@ -138,16 +134,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_reshape2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_reshape2_op_npu.py index 10ce920fb903f..74175ca6bbc51 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_reshape2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_reshape2_op_npu.py @@ -26,7 +26,6 @@ class TestReshape2(OpTest): - def setUp(self): self.set_npu() self.op_type = "reshape2" @@ -37,7 +36,7 @@ def setUp(self): self.attrs = {"shape": self.new_shape} self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def set_npu(self): @@ -56,7 +55,6 @@ def test_check_grad_normal(self): class TestReshape2_case2(TestReshape2): - def init_data(self): self.ori_shape = (2, 100) self.new_shape = (-1, 10) @@ -64,7 +62,6 @@ def init_data(self): class TestReshape2_case3(TestReshape2): - def init_data(self): self.ori_shape = (100, 5, 6) self.new_shape = (-1, 0, 3) diff --git a/python/paddle/fluid/tests/unittests/npu/test_rmsprop_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_rmsprop_op_npu.py index ed872aa0dea7f..9274d8daa1d42 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_rmsprop_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_rmsprop_op_npu.py @@ -28,7 +28,6 @@ class TestNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -43,9 +42,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -69,16 +68,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -91,7 +91,6 @@ def test_npu(self): class TestCenteredNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -106,9 +105,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -132,16 +131,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_roi_align_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_roi_align_op_npu.py index 8e1529e43f5d3..fde9824d95d3b 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_roi_align_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_roi_align_op_npu.py @@ -26,7 +26,6 @@ class TestROIAlignNPUOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() @@ -37,7 +36,7 @@ def set_data(self): self.inputs = { 'X': self.x, 'ROIs': self.rois[:, 1:5], - 'RoisNum': np.asarray(seq_len).astype('int32') + 'RoisNum': np.asarray(seq_len).astype('int32'), } self.attrs = { @@ -45,7 +44,7 @@ def set_data(self): 'pooled_height': self.pooled_height, 'pooled_width': self.pooled_width, 'sampling_ratio': self.sampling_ratio, - 'aligned': self.aligned + 'aligned': self.aligned, } self.outputs = {'Out': self.out_data} @@ -67,25 +66,45 @@ def init_test_case(self): self.x = np.random.random(self.x_dim).astype('float32') - def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, roi_bin_grid_w, - bin_size_h, bin_size_w): + def pre_calc( + self, + x_i, + roi_xmin, + roi_ymin, + roi_bin_grid_h, + roi_bin_grid_w, + bin_size_h, + bin_size_w, + ): count = roi_bin_grid_h * roi_bin_grid_w bilinear_pos = np.zeros( [self.channels, self.pooled_height, self.pooled_width, count, 4], - np.float32) - bilinear_w = np.zeros([self.pooled_height, self.pooled_width, count, 4], - np.float32) + np.float32, + ) + bilinear_w = np.zeros( + [self.pooled_height, self.pooled_width, count, 4], np.float32 + ) for ph in range(self.pooled_width): for pw in range(self.pooled_height): c = 0 for iy in range(roi_bin_grid_h): - y = roi_ymin + ph * bin_size_h + (iy + 0.5) * \ - bin_size_h / roi_bin_grid_h + y = ( + roi_ymin + + ph * bin_size_h + + (iy + 0.5) * bin_size_h / roi_bin_grid_h + ) for ix in range(roi_bin_grid_w): - x = roi_xmin + pw * bin_size_w + (ix + 0.5) * \ - bin_size_w / roi_bin_grid_w - if y < -1.0 or y > self.height or \ - x < -1.0 or x > self.width: + x = ( + roi_xmin + + pw * bin_size_w + + (ix + 0.5) * bin_size_w / roi_bin_grid_w + ) + if ( + y < -1.0 + or y > self.height + or x < -1.0 + or x > self.width + ): continue if y <= 0: y = 0 @@ -106,14 +125,18 @@ def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, roi_bin_grid_w, hy = 1 - ly hx = 1 - lx for ch in range(self.channels): - bilinear_pos[ch, ph, pw, c, 0] = x_i[ch, y_low, - x_low] - bilinear_pos[ch, ph, pw, c, 1] = x_i[ch, y_low, - x_high] - bilinear_pos[ch, ph, pw, c, 2] = x_i[ch, y_high, - x_low] - bilinear_pos[ch, ph, pw, c, 3] = x_i[ch, y_high, - x_high] + bilinear_pos[ch, ph, pw, c, 0] = x_i[ + ch, y_low, x_low + ] + bilinear_pos[ch, ph, pw, c, 1] = x_i[ + ch, y_low, x_high + ] + bilinear_pos[ch, ph, pw, c, 2] = x_i[ + ch, y_high, x_low + ] + bilinear_pos[ch, ph, pw, c, 3] = x_i[ + ch, y_high, x_high + ] bilinear_w[ph, pw, c, 0] = hy * hx bilinear_w[ph, pw, c, 1] = hy * lx bilinear_w[ph, pw, c, 2] = ly * hx @@ -123,10 +146,15 @@ def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, roi_bin_grid_w, def calc_roi_align(self): self.out_data = np.zeros( - (self.rois_num, self.channels, self.pooled_height, - self.pooled_width)).astype('float32') - - offset = 0.5 if self.aligned else 0. + ( + self.rois_num, + self.channels, + self.pooled_height, + self.pooled_width, + ) + ).astype('float32') + + offset = 0.5 if self.aligned else 0.0 for i in range(self.rois_num): roi = self.rois[i] roi_batch_id = int(roi[0]) @@ -144,16 +172,27 @@ def calc_roi_align(self): bin_size_h = float(roi_height) / float(self.pooled_height) bin_size_w = float(roi_width) / float(self.pooled_width) - roi_bin_grid_h = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_height / self.pooled_height) - roi_bin_grid_w = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_width / self.pooled_width) + roi_bin_grid_h = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_height / self.pooled_height) + ) + roi_bin_grid_w = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_width / self.pooled_width) + ) count = max(int(roi_bin_grid_h * roi_bin_grid_w), 1) pre_size = count * self.pooled_width * self.pooled_height - bilinear_pos, bilinear_w = self.pre_calc(x_i, roi_xmin, roi_ymin, - int(roi_bin_grid_h), - int(roi_bin_grid_w), - bin_size_h, bin_size_w) + bilinear_pos, bilinear_w = self.pre_calc( + x_i, + roi_xmin, + roi_ymin, + int(roi_bin_grid_h), + int(roi_bin_grid_w), + bin_size_h, + bin_size_w, + ) for ch in range(self.channels): align_per_bin = (bilinear_pos[ch] * bilinear_w).sum(axis=-1) output_val = align_per_bin.mean(axis=-1) @@ -166,14 +205,18 @@ def make_rois(self): # for i in range(bno + 1): self.rois_lod[0].append(bno) x1 = np.random.randint( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.randint( - 0, self.height // self.spatial_scale - self.pooled_height) + 0, self.height // self.spatial_scale - self.pooled_height + ) - x2 = np.random.randint(x1 + self.pooled_width, - self.width // self.spatial_scale) - y2 = np.random.randint(y1 + self.pooled_height, - self.height // self.spatial_scale) + x2 = np.random.randint( + x1 + self.pooled_width, self.width // self.spatial_scale + ) + y2 = np.random.randint( + y1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) @@ -195,7 +238,6 @@ def test_check_grad(self): class TestROIAlignOpWithMinusSample(TestROIAlignNPUOp): - def init_test_case(self): self.batch_size = 3 self.channels = 3 diff --git a/python/paddle/fluid/tests/unittests/npu/test_run_program_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_run_program_op_npu.py index 52b95c7d9dcff..899a3fdf03cd7 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_run_program_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_run_program_op_npu.py @@ -50,10 +50,10 @@ def program_scope_guard(): # when create Operator, so here compare gradients with static graph # NOTE: Here rewrite a simple unittest framework for RunProgramOp class RunProgramNPUOpTest(unittest.TestCase): - def build_model(self): raise NotImplementedError( - "RunProgramOp test should implement build_model") + "RunProgramOp test should implement build_model" + ) def check_output(self): places = [fluid.NPUPlace(0)] @@ -84,9 +84,9 @@ def run_static_model(self, place, is_test=True): else: fetch_list = self.get_param_grad_names() - outs = exe.run(main_program, - feed=self.inputs['X'], - fetch_list=fetch_list) + outs = exe.run( + main_program, feed=self.inputs['X'], fetch_list=fetch_list + ) return outs def get_program_desc(self): @@ -95,9 +95,16 @@ def get_program_desc(self): return fluid.default_main_program().desc, fwd_op_num def prepare_attrs(self): - return ('global_block', self.program_desc.block(0), 'start_op_index', 0, - 'end_op_index', self.fwd_op_num, 'program_id', - _hash_with_id(self.program_desc, self)) + return ( + 'global_block', + self.program_desc.block(0), + 'start_op_index', + 0, + 'end_op_index', + self.fwd_op_num, + 'program_id', + _hash_with_id(self.program_desc, self), + ) def get_param_grad_names(self): grad_names = [] @@ -111,10 +118,9 @@ def check_output_with_place(self, place): # Step 2. compare output for expect_v, actual_v in zip(self.expect_outs, actual_outs): - np.testing.assert_allclose(expect_v, - actual_v.numpy(), - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + expect_v, actual_v.numpy(), rtol=1e-05, atol=1e-05 + ) def check_grad_with_place(self, place): # Step 1. calc grads @@ -123,24 +129,20 @@ def check_grad_with_place(self, place): # Step 2. compare grads for expect_v, actual_v in zip(self.expect_grads, actual_grads): np.testing.assert_array_almost_equal(expect_v, actual_v) - np.testing.assert_allclose(expect_v, - actual_v, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + expect_v, actual_v, rtol=1e-05, atol=1e-05 + ) def prepare_dygraph_input(self, place, return_param_list=False): - def create_var_base(is_input, name, np_value, stop_gradient): if _in_eager_mode_: - var = core.eager.Tensor(value=np_value, - name=name, - place=place, - zero_copy=True) + var = core.eager.Tensor( + value=np_value, name=name, place=place, zero_copy=True + ) else: - var = core.VarBase(value=np_value, - name=name, - place=place, - zero_copy=True) + var = core.VarBase( + value=np_value, name=name, place=place, zero_copy=True + ) var.stop_gradient = stop_gradient return var @@ -163,7 +165,6 @@ def create_var_base(is_input, name, np_value, stop_gradient): return inputs def prepare_dygraph_output(self): - def create_var_base(is_input, name): var = framework._varbase_creator(dtype=None, shape=None, name=name) var.stop_gradient = False @@ -181,7 +182,8 @@ def create_var_base(is_input, name): outputs['OutScope'] = framework._varbase_creator( type=core.VarDesc.VarType.STEP_SCOPES, name="program_out_scope", - persistable=True) + persistable=True, + ) inner_scope = core.Scope() outputs['OutScope'].value().set_scope(inner_scope) @@ -196,9 +198,15 @@ def calc_dygraph_output(self, place): inputs = self.prepare_dygraph_input(place) outputs = self.prepare_dygraph_output() - _legacy_C_ops.run_program(inputs['X'], inputs['Params'], - outputs['Out'], outputs['OutScope'], - outputs['DOut'], None, *self.attrs) + _legacy_C_ops.run_program( + inputs['X'], + inputs['Params'], + outputs['Out'], + outputs['OutScope'], + outputs['DOut'], + None, + *self.attrs + ) return outputs['Out'] def calc_dygraph_grad(self, place): @@ -210,9 +218,15 @@ def calc_dygraph_grad(self, place): inputs, input_param_list = self.prepare_dygraph_input(place, True) outputs = self.prepare_dygraph_output() - _legacy_C_ops.run_program(inputs['X'], inputs['Params'], - outputs['Out'], outputs['OutScope'], - outputs['DOut'], None, *self.attrs) + _legacy_C_ops.run_program( + inputs['X'], + inputs['Params'], + outputs['Out'], + outputs['OutScope'], + outputs['DOut'], + None, + *self.attrs + ) for param in input_param_list: var_type = self._get_grad_vartype(param.name) @@ -243,27 +257,29 @@ def _get_grad_vartype(self, name): class TestRunProgramOpWithFC(RunProgramNPUOpTest): - def setUp(self): self.op_type = "run_program" self.dtype = np.float32 self.input_names = { 'X': ['img'], - 'Params': ['weight_param', 'bias_param'] + 'Params': ['weight_param', 'bias_param'], } self.output_names = {'Out': ['fc_0.tmp_2']} self.inputs = { 'X': { - self.input_names['X'][0]: - np.random.random((32, 1, 28, 28)).astype(self.dtype) + self.input_names['X'][0]: np.random.random( + (32, 1, 28, 28) + ).astype(self.dtype) }, 'Params': { - self.input_names['Params'][0]: - np.random.random((784, 10)).astype(self.dtype), - self.input_names['Params'][1]: - np.random.random((32, 10)).astype(self.dtype) - } + self.input_names['Params'][0]: np.random.random( + (784, 10) + ).astype(self.dtype), + self.input_names['Params'][1]: np.random.random( + (32, 10) + ).astype(self.dtype), + }, } def test_check_output(self): @@ -274,26 +290,34 @@ def test_check_grad(self): def build_model(self): # 1. simple model - img = fluid.data(name=self.input_names['X'][0], - shape=[None, 1, 28, 28], - dtype='float32') + img = fluid.data( + name=self.input_names['X'][0], + shape=[None, 1, 28, 28], + dtype='float32', + ) weight_attr = fluid.ParamAttr( name=self.input_names['Params'][0], learning_rate=0.5, initializer=fluid.initializer.NumpyArrayInitializer( - self.inputs['Params'][self.input_names['Params'][0]]), - trainable=True) + self.inputs['Params'][self.input_names['Params'][0]] + ), + trainable=True, + ) bias_attr = fluid.ParamAttr( name=self.input_names['Params'][1], learning_rate=0.5, initializer=fluid.initializer.NumpyArrayInitializer( - self.inputs['Params'][self.input_names['Params'][1]]), - trainable=True) - pred = fluid.layers.fc(input=img, - size=10, - param_attr=weight_attr, - bias_attr=bias_attr, - act='relu') + self.inputs['Params'][self.input_names['Params'][1]] + ), + trainable=True, + ) + pred = fluid.layers.fc( + input=img, + size=10, + param_attr=weight_attr, + bias_attr=bias_attr, + act='relu', + ) # 2. get forward op num fwd_op_num = fluid.default_main_program().global_block().desc.op_size() # 3. append backward diff --git a/python/paddle/fluid/tests/unittests/npu/test_sampling_id_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sampling_id_op_npu.py index 399ac0d25225a..d354e39bcf08b 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sampling_id_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sampling_id_op_npu.py @@ -28,7 +28,6 @@ class TestSamplingIdShape(unittest.TestCase): - def test_shape(self): paddle.enable_static() x = fluid.layers.data(name='x', shape=[3], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/npu/test_save_load_npu.py b/python/paddle/fluid/tests/unittests/npu/test_save_load_npu.py index b6a6a0464a02c..09f25273696ef 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_save_load_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_save_load_npu.py @@ -36,62 +36,78 @@ class TestNPUSaveLoadBase(TestSaveLoadBase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) class TestNPUSaveLoadPartial(TestSaveLoadPartial): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) class TestNPUSaveLoadSetStateDict(TestSaveLoadSetStateDict): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) class TestNPUProgramStatePartial(TestProgramStatePartial): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) class TestNPULoadFromOldInterface(TestLoadFromOldInterface): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) class TestNPULoadFromOldInterfaceSingleFile(TestLoadFromOldInterfaceSingleFile): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) class TestNPUProgramStateOldSave(TestProgramStateOldSave): - def setUp(self): self.test_dygraph = False def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) class TestNPUProgramStateOldSaveSingleModel(TestProgramStateOldSaveSingleModel): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_npu() else paddle.NPUPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_npu() + else paddle.NPUPlace(0) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_scale_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_scale_op_npu.py index c7166159f9a8f..94477432e57a8 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_scale_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_scale_op_npu.py @@ -26,7 +26,6 @@ class TestScale(OpTest): - def setUp(self): self.set_npu() self.op_type = "scale" @@ -34,14 +33,15 @@ def setUp(self): self.init_dtype() self.inputs = { - 'X': - OpTest.np_dtype_to_fluid_dtype( - np.random.random((10, 10)).astype(self.dtype)) + 'X': OpTest.np_dtype_to_fluid_dtype( + np.random.random((10, 10)).astype(self.dtype) + ) } self.attrs = {'scale': -2.3, 'bias': 0, 'bias_after_scale': True} self.outputs = { 'Out': (self.inputs['X'] * self.dtype(self.attrs['scale'])).astype( - self.dtype) + self.dtype + ) } def set_npu(self): @@ -55,25 +55,21 @@ def test_check_output(self): class TestFP16Scale(TestScale): - def init_dtype(self): self.dtype = np.float16 class TestScaleInt(TestScale): - def init_dtype(self): self.dtype = np.int32 class TestScaleInt64(TestScale): - def init_dtype(self): self.dtype = np.int64 class TestBiasAfterScale(OpTest): - def setUp(self): self.set_npu() self.op_type = "scale" @@ -81,9 +77,9 @@ def setUp(self): self.init_dtype() self.inputs = { - 'X': - OpTest.np_dtype_to_fluid_dtype( - np.random.random((10, 10)).astype(self.dtype)) + 'X': OpTest.np_dtype_to_fluid_dtype( + np.random.random((10, 10)).astype(self.dtype) + ) } self.attrs = {'scale': -2.3, 'bias': 0, 'bias_after_scale': False} self.outputs = { diff --git a/python/paddle/fluid/tests/unittests/npu/test_scatter_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_scatter_op_npu.py index 8bb28b10997c8..bfa591e3f10d9 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_scatter_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_scatter_op_npu.py @@ -27,7 +27,6 @@ class TestCast1_FP32(OpTest): - def setUp(self): self.set_npu() self.op_type = "scatter" @@ -51,7 +50,6 @@ def test_check_output(self): class TestCast_INT32(OpTest): - def setUp(self): self.set_npu() self.op_type = "scatter" @@ -75,7 +73,6 @@ def test_check_output(self): class TestCast2_FP32(OpTest): - def setUp(self): self.set_npu() self.op_type = "scatter" @@ -99,7 +96,6 @@ def test_check_output(self): class TestCast3_FP32(OpTest): - def setUp(self): self.set_npu() self.op_type = "scatter" @@ -124,7 +120,6 @@ def test_check_output(self): class TestCast_INT64(OpTest): - def setUp(self): self.set_npu() self.op_type = "scatter" diff --git a/python/paddle/fluid/tests/unittests/npu/test_seed_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_seed_op_npu.py index 72402bd2b8667..dd688a29c442a 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_seed_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_seed_op_npu.py @@ -26,7 +26,6 @@ class TestSeedOpFixSeed(OpTest): - def setUp(self): self.set_npu() self.op_type = "seed" @@ -42,7 +41,6 @@ def test_check_output(self): class TestSeedOpDiffSeed(OpTest): - def setUp(self): self.set_npu() self.op_type = "seed" diff --git a/python/paddle/fluid/tests/unittests/npu/test_sequence_mask_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sequence_mask_npu.py index df0cb03d58bfd..66642cd4bc66f 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sequence_mask_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sequence_mask_npu.py @@ -21,13 +21,16 @@ import paddle import paddle.fluid as fluid import paddle.fluid.core as core -from paddle.fluid.framework import convert_np_dtype_to_dtype_, Program, program_guard +from paddle.fluid.framework import ( + convert_np_dtype_to_dtype_, + Program, + program_guard, +) paddle.enable_static() class SequenceMaskTestBase(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -51,19 +54,19 @@ def setUp(self): self.outputs = {'Y': self.calc_ground_truth_mask()} self.attrs = { 'maxlen': self.maxlen, - 'out_dtype': convert_np_dtype_to_dtype_(self.mask_dtype) + 'out_dtype': convert_np_dtype_to_dtype_(self.mask_dtype), } def calc_ground_truth_mask(self): maxlen = np.max(self.x) if self.maxlen < 0 else self.maxlen - shape = self.x.shape + (maxlen, ) - index_broadcast = np.broadcast_to(np.reshape( - range(maxlen), newshape=[1] * self.x.ndim + [-1]), - shape=shape) - x_broadcast = np.broadcast_to(np.reshape(self.x, - newshape=self.x.shape + - (-1, )), - shape=shape) + shape = self.x.shape + (maxlen,) + index_broadcast = np.broadcast_to( + np.reshape(range(maxlen), newshape=[1] * self.x.ndim + [-1]), + shape=shape, + ) + x_broadcast = np.broadcast_to( + np.reshape(self.x, newshape=self.x.shape + (-1,)), shape=shape + ) return (index_broadcast < x_broadcast).astype(self.mask_dtype) def test_check_output(self): @@ -71,43 +74,36 @@ def test_check_output(self): class SequenceMaskTest1(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'bool' class SequenceMaskTest2(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'uint8' class SequenceMaskTest3(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'int32' class SequenceMaskTest4(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'float32' class SequenceMaskTest5(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'float64' class SequenceMaskTest6(SequenceMaskTestBase): - def initParameters(self): self.maxlen = -1 class SequenceMaskTestBase_tensor_attr(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -134,14 +130,14 @@ def setUp(self): def calc_ground_truth_mask(self): maxlen = np.max(self.x) if self.maxlen < 0 else self.maxlen - shape = self.x.shape + (maxlen, ) - index_broadcast = np.broadcast_to(np.reshape( - range(maxlen), newshape=[1] * self.x.ndim + [-1]), - shape=shape) - x_broadcast = np.broadcast_to(np.reshape(self.x, - newshape=self.x.shape + - (-1, )), - shape=shape) + shape = self.x.shape + (maxlen,) + index_broadcast = np.broadcast_to( + np.reshape(range(maxlen), newshape=[1] * self.x.ndim + [-1]), + shape=shape, + ) + x_broadcast = np.broadcast_to( + np.reshape(self.x, newshape=self.x.shape + (-1,)), shape=shape + ) return (index_broadcast < x_broadcast).astype(self.mask_dtype) def test_check_output(self): @@ -149,37 +145,31 @@ def test_check_output(self): class SequenceMaskTest1_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'bool' class SequenceMaskTest2_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'uint8' class SequenceMaskTest3_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'int32' class SequenceMaskTest4_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'float32' class SequenceMaskTest5_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'float64' class TestSequenceMaskOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.uniform(1, 5, [4]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/npu/test_set_value_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_set_value_op_npu.py index e1b866f31f506..96af414883319 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_set_value_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_set_value_op_npu.py @@ -24,7 +24,6 @@ class TestSetValueBase(unittest.TestCase): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -55,7 +54,6 @@ def _get_answer(self): class TestSetValueApi(TestSetValueBase): - def _run_static(self): paddle.enable_static() with paddle.static.program_guard(self.program): @@ -80,17 +78,22 @@ def test_api(self): dynamic_out = self._run_dynamic() self._get_answer() - error_msg = "\nIn {} mode: \nExpected res = \n{}, \n\nbut received : \n{}" - self.assertTrue((self.data == static_out).all(), - msg=error_msg.format("static", self.data, static_out)) - self.assertTrue((self.data == dynamic_out).all(), - msg=error_msg.format("dynamic", self.data, dynamic_out)) + error_msg = ( + "\nIn {} mode: \nExpected res = \n{}, \n\nbut received : \n{}" + ) + self.assertTrue( + (self.data == static_out).all(), + msg=error_msg.format("static", self.data, static_out), + ) + self.assertTrue( + (self.data == dynamic_out).all(), + msg=error_msg.format("dynamic", self.data, dynamic_out), + ) # 1. Test different type of item: int, Python slice, Paddle Tensor # 1.1 item is int class TestSetValueItemInt(TestSetValueApi): - def _call_setitem(self, x): x[0] = self.value @@ -101,7 +104,6 @@ def _get_answer(self): # 1.2 item is slice # 1.2.1 step is 1 class TestSetValueItemSlice(TestSetValueApi): - def _call_setitem(self, x): x[0:2] = self.value @@ -110,7 +112,6 @@ def _get_answer(self): class TestSetValueItemSlice2(TestSetValueApi): - def _call_setitem(self, x): x[0:-1] = self.value @@ -119,7 +120,6 @@ def _get_answer(self): class TestSetValueItemSlice3(TestSetValueApi): - def _call_setitem(self, x): x[0:-1, 0:2] = self.value @@ -128,7 +128,6 @@ def _get_answer(self): class TestSetValueItemSlice4(TestSetValueApi): - def _call_setitem(self, x): x[0:, 1:2, :] = self.value @@ -156,7 +155,6 @@ def _get_answer(self): # 1.2.2 step > 1 class TestSetValueItemSliceStep(TestSetValueApi): - def set_shape(self): self.shape = [5, 5, 5] @@ -168,7 +166,6 @@ def _get_answer(self): class TestSetValueItemSliceStep2(TestSetValueApi): - def set_shape(self): self.shape = [7, 5, 5] @@ -180,7 +177,6 @@ def _get_answer(self): class TestSetValueItemSliceStep3(TestSetValueApi): - def _call_setitem(self, x): x[0:-1, 0:2, ::2] = self.value @@ -189,7 +185,6 @@ def _get_answer(self): class TestSetValueItemSliceStep4(TestSetValueApi): - def _call_setitem(self, x): x[0:, 1:2:2, :] = self.value @@ -199,7 +194,6 @@ def _get_answer(self): # 1.2.3 step < 0 class TestSetValueItemSliceNegetiveStep(TestSetValueApi): - def set_shape(self): self.shape = [5, 2] @@ -214,7 +208,6 @@ def _get_answer(self): class TestSetValueItemSliceNegetiveStep2(TestSetValueApi): - def set_shape(self): self.shape = [5] @@ -229,7 +222,6 @@ def _get_answer(self): class TestSetValueItemSliceNegetiveStep3(TestSetValueApi): - def set_shape(self): self.shape = [3] @@ -244,7 +236,6 @@ def _get_answer(self): class TestSetValueItemSliceNegetiveStep4(TestSetValueApi): - def set_shape(self): self.shape = [3, 4, 5] @@ -259,7 +250,6 @@ def _get_answer(self): class TestSetValueItemEllipsis1(TestSetValueApi): - def _call_setitem(self, x): x[0:, ..., 1:] = self.value @@ -268,7 +258,6 @@ def _get_answer(self): class TestSetValueItemEllipsis2(TestSetValueApi): - def _call_setitem(self, x): x[0:, ...] = self.value @@ -277,7 +266,6 @@ def _get_answer(self): class TestSetValueItemEllipsis3(TestSetValueApi): - def _call_setitem(self, x): x[..., 1:] = self.value @@ -286,7 +274,6 @@ def _get_answer(self): class TestSetValueItemEllipsis4(TestSetValueApi): - def _call_setitem(self, x): x[...] = self.value @@ -296,7 +283,6 @@ def _get_answer(self): # 1.4 item is Paddle Tensor class TestSetValueItemTensor(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") x[zero] = self.value @@ -306,7 +292,6 @@ def _get_answer(self): class TestSetValueItemTensor2(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -317,7 +302,6 @@ def _get_answer(self): class TestSetValueItemTensor3(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -328,7 +312,6 @@ def _get_answer(self): class TestSetValueItemTensor4(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -339,7 +322,6 @@ def _get_answer(self): class TestSetValueItemTensor5(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -350,7 +332,6 @@ def _get_answer(self): class TestSetValueItemTensor6(TestSetValueApi): - def set_shape(self): self.shape = [3, 4, 5] @@ -365,7 +346,6 @@ def _get_answer(self): # 1.5 item is None class TestSetValueItemNone1(TestSetValueApi): - def _call_setitem(self, x): x[None] = self.value @@ -374,7 +354,6 @@ def _get_answer(self): class TestSetValueItemNone2(TestSetValueApi): - def _call_setitem(self, x): x[0, None, 1] = self.value @@ -383,7 +362,6 @@ def _get_answer(self): class TestSetValueItemNone3(TestSetValueApi): - def _call_setitem(self, x): x[:, None, None, 1] = self.value @@ -392,7 +370,6 @@ def _get_answer(self): class TestSetValueItemNone4(TestSetValueApi): - def _call_setitem(self, x): x[0, 0, None, 1] = self.value @@ -401,7 +378,6 @@ def _get_answer(self): class TestSetValueItemNone5(TestSetValueApi): - def _call_setitem(self, x): x[0, None, 0, None, 1] = self.value @@ -410,7 +386,6 @@ def _get_answer(self): class TestSetValueItemNone6(TestSetValueApi): - def _call_setitem(self, x): x[None, 0, 0, None, 0] = self.value @@ -419,7 +394,6 @@ def _get_answer(self): class TestSetValueItemNone7(TestSetValueApi): - def _call_setitem(self, x): x[:, None, 1] = np.zeros(self.shape)[:, None, 0] @@ -428,7 +402,6 @@ def _get_answer(self): class TestSetValueItemNone8(TestSetValueApi): - def _call_setitem(self, x): x[:, 1, None] = np.zeros(self.shape)[:, 0, None] @@ -437,7 +410,6 @@ def _get_answer(self): class TestSetValueItemNone9(TestSetValueApi): - def _call_setitem(self, x): x[None, :, 1, ..., None] = np.zeros(self.shape)[0, 0, :, None] @@ -447,7 +419,6 @@ def _get_answer(self): # 1.5 item is list or Tensor of bol class TestSetValueItemBool1(TestSetValueApi): - def _call_setitem(self, x): x[[True, False]] = self.value @@ -456,7 +427,6 @@ def _get_answer(self): class TestSetValueItemBool2(TestSetValueApi): - def _call_setitem(self, x): x[[False, False]] = self.value @@ -465,7 +435,6 @@ def _get_answer(self): class TestSetValueItemBool3(TestSetValueApi): - def _call_setitem(self, x): x[[False, True]] = np.zeros(self.shape[2]) @@ -474,7 +443,6 @@ def _get_answer(self): class TestSetValueItemBool4(TestSetValueApi): - def _call_setitem(self, x): idx = paddle.assign(np.array([False, True])) x[idx] = np.zeros(self.shape[2]) @@ -484,19 +452,19 @@ def _get_answer(self): class TestSetValueItemBool5(TestSetValueApi): - def _call_setitem(self, x): idx = paddle.assign( - np.array([[False, True, False], [True, True, False]])) + np.array([[False, True, False], [True, True, False]]) + ) x[idx] = self.value def _get_answer(self): - self.data[np.array([[False, True, False], [True, True, - False]])] = self.value + self.data[ + np.array([[False, True, False], [True, True, False]]) + ] = self.value class TestSetValueItemBool6(TestSetValueApi): - def _call_setitem(self, x): x[0, ...] = 0 x[x > 0] = self.value @@ -507,9 +475,7 @@ def _get_answer(self): def create_test_value_int32(parent): - class TestValueInt(parent): - def set_value(self): self.value = 7 @@ -529,9 +495,7 @@ def set_dtype(self): def create_test_value_int64(parent): - class TestValueInt(parent): - def set_value(self): self.value = 7 @@ -551,9 +515,7 @@ def set_dtype(self): def create_test_value_tensor_fp32(parent): - class TestValueInt(parent): - def set_dtype(self): self.dtype = "float32" @@ -578,7 +540,6 @@ def _get_answer(self): # 3. Test different shape of value class TestSetValueValueShape1(TestSetValueApi): - def set_value(self): self.value = np.array([3, 4, 5, 6]) # shape is (4,) @@ -590,7 +551,6 @@ def _get_answer(self): class TestSetValueValueShape2(TestSetValueApi): - def set_value(self): self.value = np.array([[3, 4, 5, 6]]) # shape is (1,4) @@ -602,10 +562,10 @@ def _get_answer(self): class TestSetValueValueShape3(TestSetValueApi): - def set_value(self): - self.value = np.array([[1, 1, 1, 1], [2, 2, 2, 2], - [3, 3, 3, 3]]) # shape is (3,4) + self.value = np.array( + [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]] + ) # shape is (3,4) def _call_setitem(self, x): x[0] = self.value @@ -615,11 +575,12 @@ def _get_answer(self): class TestSetValueValueShape4(TestSetValueApi): - def set_value(self): - self.value = np.array([[1, 1, 1, 1], [2, 2, 2, 2], - [3, 3, 3, - 3]]).astype(self.dtype) # shape is (3,4) + self.value = np.array( + [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]] + ).astype( + self.dtype + ) # shape is (3,4) def _call_setitem(self, x): x[0] = paddle.assign(self.value) # x is Paddle.Tensor @@ -629,7 +590,6 @@ def _get_answer(self): class TestSetValueValueShape5(TestSetValueApi): - def set_value(self): self.value = np.array([3, 3, 3]).astype(self.dtype) diff --git a/python/paddle/fluid/tests/unittests/npu/test_sgd_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sgd_op_npu.py index 03c48b415dd4c..9747953862508 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sgd_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sgd_op_npu.py @@ -26,7 +26,6 @@ class TestSGD(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -54,7 +53,6 @@ def test_check_output(self): class TestNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -69,9 +67,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -95,16 +93,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_shape_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_shape_op_npu.py index 136bedea8226c..fdee5eced0cbc 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_shape_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_shape_op_npu.py @@ -26,7 +26,6 @@ class TestShape(OpTest): - def setUp(self): self.set_npu() self.op_type = "shape" @@ -52,25 +51,21 @@ def test_check_output(self): class TestShape_fp16(TestShape): - def init_dtype(self): self.dtype = np.float16 class TestShape_double(TestShape): - def init_dtype(self): self.dtype = np.float64 class TestShape_int32(TestShape): - def init_dtype(self): self.dtype = np.int32 class TestShape_int64(TestShape): - def init_dtype(self): self.dtype = np.int64 diff --git a/python/paddle/fluid/tests/unittests/npu/test_shard_index_op.py b/python/paddle/fluid/tests/unittests/npu/test_shard_index_op.py index 3ed6051b46fd2..2fc3d9beeb81e 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_shard_index_op.py +++ b/python/paddle/fluid/tests/unittests/npu/test_shard_index_op.py @@ -52,13 +52,12 @@ def common_setup(self, index_num, nshards, shard_id, ignore_value): 'index_num': index_num, 'nshards': nshards, 'shard_id': shard_id, - 'ignore_value': ignore_value + 'ignore_value': ignore_value, } self.outputs = {'Out': (out, x_lod)} class TestShardIndexShardId0Op(OpTest): - def setUp(self): common_setup(self, 20, 2, 0, -1) @@ -67,19 +66,16 @@ def test_check_output(self): class TestShardIndexShardId1Op(TestShardIndexShardId0Op): - def setUp(self): common_setup(self, 20, 2, 1, -1) class TestShardIndexIgnoreValueOp(TestShardIndexShardId0Op): - def setUp(self): common_setup(self, 20, 2, 0, -2) class TestShardIndexNotEvenlyDividedOp(TestShardIndexShardId0Op): - def setUp(self): common_setup(self, 15, 2, 1, -1) diff --git a/python/paddle/fluid/tests/unittests/npu/test_sigmoid_cross_entropy_with_logits_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sigmoid_cross_entropy_with_logits_op_npu.py index 1454edd7a7538..e06c132df251f 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sigmoid_cross_entropy_with_logits_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sigmoid_cross_entropy_with_logits_op_npu.py @@ -25,11 +25,11 @@ paddle.enable_static() -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestSigmoidCrossEntropyWithLogitsOp1(OpTest): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -39,13 +39,14 @@ def setUp(self): batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.randint(0, 2, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.randint(0, 2, (batch_size, num_classes)).astype( + self.dtype + ), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -70,12 +71,13 @@ def init_dtype(self): self.dtype = np.float32 -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") -class TestSigmoidCrossEntropyWithLogitsOp3(TestSigmoidCrossEntropyWithLogitsOp1 - ): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) +class TestSigmoidCrossEntropyWithLogitsOp3( + TestSigmoidCrossEntropyWithLogitsOp1 +): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -85,13 +87,14 @@ def setUp(self): batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.uniform(0, 1, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.uniform(0, 1, (batch_size, num_classes)).astype( + self.dtype + ), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -103,12 +106,13 @@ def setUp(self): self.outputs = {'Out': -term1 - term2} -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") -class TestSigmoidCrossEntropyWithLogitsOp5(TestSigmoidCrossEntropyWithLogitsOp1 - ): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) +class TestSigmoidCrossEntropyWithLogitsOp5( + TestSigmoidCrossEntropyWithLogitsOp1 +): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -118,14 +122,14 @@ def setUp(self): batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + [num_classes])).astype( - self.dtype)), - 'Label': - np.random.uniform(0, 1, tuple(batch_size + [num_classes])).astype( - self.dtype) + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype) + ), + 'Label': np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -137,12 +141,13 @@ def setUp(self): self.outputs = {'Out': -term1 - term2} -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") -class TestSigmoidCrossEntropyWithLogitsOp6(TestSigmoidCrossEntropyWithLogitsOp1 - ): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) +class TestSigmoidCrossEntropyWithLogitsOp6( + TestSigmoidCrossEntropyWithLogitsOp1 +): + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -152,14 +157,14 @@ def setUp(self): batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + [num_classes])).astype( - self.dtype)), - 'Label': - np.random.randint(0, 2, tuple(batch_size + [num_classes])).astype( - self.dtype) + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype) + ), + 'Label': np.random.randint( + 0, 2, tuple(batch_size + [num_classes]) + ).astype(self.dtype), } # Fw Pass is implemented as elementwise sigmoid followed by diff --git a/python/paddle/fluid/tests/unittests/npu/test_sigmoid_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sigmoid_op_npu.py index 47d881ee013ed..caac4ed6ba426 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sigmoid_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sigmoid_op_npu.py @@ -23,10 +23,10 @@ SEED = 2021 -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestNPUSigmoid(OpTest): - def setUp(self): self.op_type = "sigmoid" self.set_npu() @@ -43,9 +43,9 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.01) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=0.01 + ) def set_npu(self): self.__class__.use_npu = True @@ -55,10 +55,10 @@ def init_dtype(self): self.dtype = np.float32 -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestNPUSigmoidFp16(TestNPUSigmoid): - def test_check_output(self): self.check_output_with_place(self.place, atol=1e-3) diff --git a/python/paddle/fluid/tests/unittests/npu/test_sin_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sin_op_npu.py index 26e6ae9255d42..971184956e6d8 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sin_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sin_op_npu.py @@ -17,7 +17,11 @@ import numpy as np from scipy.special import expit, erf -from paddle.fluid.tests.unittests.op_test import OpTest, convert_float_to_uint16, skip_check_grad_ci +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + convert_float_to_uint16, + skip_check_grad_ci, +) import paddle import paddle.nn as nn import paddle.nn.functional as F @@ -29,9 +33,7 @@ def test_class(op_type, typename): - class TestSin(OpTest): - def setUp(self): self.op_type = "sin" self.__class__.use_npu = True @@ -57,7 +59,7 @@ def test_out_name(self): out = eval("paddle.%s(data, name='Y')" % self.op_type) place = fluid.NPUPlace(0) exe = fluid.Executor(place) - result, = exe.run(feed={"X": np_x}, fetch_list=[out]) + (result,) = exe.run(feed={"X": np_x}, fetch_list=[out]) expected = eval("np.%s(np_x)" % self.op_type) self.assertEqual(result, expected) diff --git a/python/paddle/fluid/tests/unittests/npu/test_size_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_size_op_npu.py index e5fd042674204..1e768a5dd185a 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_size_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_size_op_npu.py @@ -25,7 +25,6 @@ class TestSizeOp(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -48,49 +47,42 @@ def set_npu(self): class TestSizeOp1(TestSizeOp): - def config(self): self.shape = [2] self.dtype = np.float64 class TestSizeOp2(TestSizeOp): - def config(self): self.shape = [2, 3] self.dtype = np.float32 class TestSizeOp3(TestSizeOp): - def config(self): self.shape = [2, 3, 100] self.dtype = np.float16 class TestSizeOp4(TestSizeOp): - def config(self): self.shape = [2**10] self.dtype = np.bool_ class TestSizeOp5(TestSizeOp): - def config(self): self.shape = [7, 8, 9, 10] self.dtype = np.int64 class TestSizeOp6(TestSizeOp): - def config(self): self.shape = [] self.dtype = np.int64 class TestSizeAPI(unittest.TestCase): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -111,17 +103,19 @@ def test_size_static(self): out_1 = paddle.fluid.layers.size(x_1) out_2 = paddle.fluid.layers.size(x_2) exe = paddle.static.Executor(place=self.place) - res_1, res_2 = exe.run(feed={ - "x_1": input_1, - "x_2": input_2, - }, - fetch_list=[out_1, out_2]) - assert (np.array_equal(res_1, - np.array([np.size(input_1) - ]).astype("int64"))) - assert (np.array_equal(res_2, - np.array([np.size(input_2) - ]).astype("int64"))) + res_1, res_2 = exe.run( + feed={ + "x_1": input_1, + "x_2": input_2, + }, + fetch_list=[out_1, out_2], + ) + assert np.array_equal( + res_1, np.array([np.size(input_1)]).astype("int64") + ) + assert np.array_equal( + res_2, np.array([np.size(input_2)]).astype("int64") + ) def test_size_imperative(self): paddle.disable_static(self.place) @@ -131,8 +125,8 @@ def test_size_imperative(self): x_2 = paddle.to_tensor(input_2) out_1 = paddle.fluid.layers.size(x_1) out_2 = paddle.fluid.layers.size(x_2) - assert (np.array_equal(out_1.numpy().item(0), np.size(input_1))) - assert (np.array_equal(out_2.numpy().item(0), np.size(input_2))) + assert np.array_equal(out_1.numpy().item(0), np.size(input_1)) + assert np.array_equal(out_2.numpy().item(0), np.size(input_2)) paddle.enable_static() def test_error(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_slice_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_slice_op_npu.py index ba2b45ab9f4ac..5bc1700cc16f4 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_slice_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_slice_op_npu.py @@ -27,7 +27,6 @@ class TestSliceOp(OpTest): - def setUp(self): self.op_type = "slice" self.set_npu() @@ -39,7 +38,7 @@ def setUp(self): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -65,15 +64,14 @@ def test_check_output(self): def test_check_grad_normal(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.02) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.02 + ) else: self.check_grad_with_place(self.place, ['Input'], 'Out') class TestSliceOp2(TestSliceOp): - def config(self): self.input = np.random.random([10, 5, 6]).astype(self.dtype) self.starts = [0] @@ -84,7 +82,6 @@ def config(self): class TestSliceOpFp16(TestSliceOp): - def init_dtype(self): self.dtype = np.float16 @@ -95,7 +92,6 @@ def set_npu(self): class TestSliceOpTensor(TestSliceOp): - def setUp(self): self.op_type = "slice" self.set_npu() @@ -104,14 +100,14 @@ def setUp(self): self.inputs = { 'Input': self.input, 'StartsTensor': self.starts, - 'EndsTensor': self.ends + 'EndsTensor': self.ends, } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, 'starts': [-1, -1, -1], 'ends': [-1, -1, -1], - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -124,7 +120,6 @@ def config(self): class TestSliceOpTensor2(TestSliceOpTensor): - def setUp(self): self.op_type = "slice" self.set_npu() @@ -133,14 +128,14 @@ def setUp(self): self.inputs = { 'Input': self.input, 'StartsTensor': self.starts, - 'EndsTensor': self.ends + 'EndsTensor': self.ends, } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, 'starts': [-1], 'ends': [-1], - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -153,7 +148,6 @@ def config(self): class TestSliceOpFp16Tensor(TestSliceOpTensor): - def init_dtype(self): self.dtype = np.float16 @@ -164,7 +158,6 @@ def set_npu(self): class TestSliceOpTensorList(TestSliceOp): - def setUp(self): self.op_type = "slice" self.set_npu() @@ -173,25 +166,27 @@ def setUp(self): self.starts_tensor_list = [] for index, ele in enumerate(self.starts): - self.starts_tensor_list.append(("start" + str(index), np.ones( - (1)).astype('int32') * ele)) + self.starts_tensor_list.append( + ("start" + str(index), np.ones((1)).astype('int32') * ele) + ) self.ends_tensor_list = [] for index, ele in enumerate(self.ends): - self.ends_tensor_list.append(("end" + str(index), np.ones( - (1)).astype('int32') * ele)) + self.ends_tensor_list.append( + ("end" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'Input': self.input, 'StartsTensorList': self.starts_tensor_list, - 'EndsTensorList': self.ends_tensor_list + 'EndsTensorList': self.ends_tensor_list, } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, 'starts': [-1, -1, -1], 'ends': [-1, -1, -1], - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -204,7 +199,6 @@ def config(self): class TestSliceOpTensorList2(TestSliceOpTensorList): - def setUp(self): self.op_type = "slice" self.set_npu() @@ -213,25 +207,27 @@ def setUp(self): self.starts_tensor_list = [] for index, ele in enumerate(self.starts): - self.starts_tensor_list.append(("start" + str(index), np.ones( - (1)).astype('int32') * ele)) + self.starts_tensor_list.append( + ("start" + str(index), np.ones((1)).astype('int32') * ele) + ) self.ends_tensor_list = [] for index, ele in enumerate(self.ends): - self.ends_tensor_list.append(("end" + str(index), np.ones( - (1)).astype('int32') * ele)) + self.ends_tensor_list.append( + ("end" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'Input': self.input, 'StartsTensorList': self.starts_tensor_list, - 'EndsTensorList': self.ends_tensor_list + 'EndsTensorList': self.ends_tensor_list, } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, 'starts': [-1], 'ends': [-1], - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -244,7 +240,6 @@ def config(self): class TestSliceOpFp16TensorList(TestSliceOpTensorList): - def init_dtype(self): self.dtype = np.float16 @@ -255,7 +250,6 @@ def set_npu(self): class TestSliceNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -272,9 +266,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=data_shape, dtype='float32') b = paddle.static.data(name="b", shape=data_shape, dtype='float32') - label = paddle.static.data(name="label", - shape=[batch_size, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[batch_size, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.slice(sum, axes=[0, 1], starts=[0, 0], ends=[33, 2]) @@ -282,7 +276,8 @@ def _test(self, run_npu=True): prediction = paddle.static.nn.fc(z, size=2, activation='softmax') cost = paddle.fluid.layers.softmax_with_cross_entropy( - logits=prediction, label=label) + logits=prediction, label=label + ) loss = paddle.mean(cost) sgd = paddle.optimizer.SGD(learning_rate=0.01) sgd.minimize(loss) @@ -297,16 +292,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(EPOCH): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res @@ -319,7 +315,6 @@ def test_npu(self): class TestSliceOpDecsDim(OpTest): - def setUp(self): self.op_type = "slice" self.set_npu() @@ -365,21 +360,19 @@ def test_check_output(self): def test_check_grad_normal(self): if self.dtype == np.float16: - self.check_grad_with_place(self.place, ['Input'], - 'Out', - max_relative_error=0.5) + self.check_grad_with_place( + self.place, ['Input'], 'Out', max_relative_error=0.5 + ) else: self.check_grad_with_place(self.place, ['Input'], 'Out') class TestSliceOpDecsDimFp16(TestSliceOpDecsDim): - def init_dtype(self): self.dtype = np.float16 class TestSliceOpDecsDim2(TestSliceOpDecsDim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [1, 0, 2] @@ -391,7 +384,6 @@ def config(self): class TestSliceOpDecsDim3(TestSliceOpDecsDim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [-1, 0, 2] @@ -403,7 +395,6 @@ def config(self): class TestSliceOpDecsDim4(TestSliceOpDecsDim): - def config(self): self.input = np.random.random([3, 4, 5, 7]).astype(self.dtype) self.starts = [0, 1, 2, 3] @@ -415,7 +406,6 @@ def config(self): class TestSliceOpDecsDim5(TestSliceOpDecsDim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [-1] @@ -427,7 +417,6 @@ def config(self): class TestSliceOpDecsDim6(TestSliceOpDecsDim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [0, 1, 2, 3] @@ -439,11 +428,10 @@ def config(self): class TestSliceOpDecsDimStartsTensor(TestSliceOpDecsDim): - def set_inputs(self): self.inputs = { 'Input': self.input, - "StartsTensor": np.array(self.starts, dtype='int32') + "StartsTensor": np.array(self.starts, dtype='int32'), } def set_attrs(self): @@ -466,18 +454,16 @@ def config(self): class TestSliceOpDecsDimStartsTensorFP16(TestSliceOpDecsDimStartsTensor): - def init_dtype(self): self.dtype = np.float16 class TestSliceOpDecsDimStartsTensorStartsAndEndsTensor(TestSliceOpDecsDim): - def set_inputs(self): self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype='int64'), - "EndsTensor": np.array(self.ends, dtype='int32') + "EndsTensor": np.array(self.ends, dtype='int32'), } def set_attrs(self): @@ -500,19 +486,19 @@ def config(self): class TestSliceOpDecsDimStartsTensorStartsAndEndsTensorFP16( - TestSliceOpDecsDimStartsTensorStartsAndEndsTensor): - + TestSliceOpDecsDimStartsTensorStartsAndEndsTensor +): def init_dtype(self): self.dtype = np.float16 class TestSliceOpDecsDimStartsListTensor(TestSliceOpDecsDim): - def set_inputs(self): starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} @@ -538,7 +524,6 @@ def config(self): class TestSliceOpDecsDimStartsListTensor2(TestSliceOpDecsDimStartsListTensor): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [-1] @@ -551,15 +536,14 @@ def config(self): self.starts_infer = [-1] -class TestSliceOpDecsDimStartsListTensorFP16(TestSliceOpDecsDimStartsListTensor - ): - +class TestSliceOpDecsDimStartsListTensorFP16( + TestSliceOpDecsDimStartsListTensor +): def init_dtype(self): self.dtype = np.float16 class TestSliceOpInt64(OpTest): - def set_npu(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -575,12 +559,13 @@ def setUp(self): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): - self.input = np.random.randint(100, - size=(3, 4, 5, 6)).astype(self.dtype) + self.input = np.random.randint(100, size=(3, 4, 5, 6)).astype( + self.dtype + ) self.starts = [1, 0, 2] self.ends = [3, 3, 4] self.axes = [0, 1, 2] @@ -595,7 +580,6 @@ def test_check_output(self): class TestSliceOpTensorInt64(TestSliceOpInt64): - def setUp(self): self.op_type = "slice" self.set_npu() @@ -604,19 +588,20 @@ def setUp(self): self.inputs = { 'Input': self.input, 'StartsTensor': self.starts, - 'EndsTensor': self.ends + 'EndsTensor': self.ends, } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, 'starts': [-1, -1, -1], 'ends': [-1, -1, -1], - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): - self.input = np.random.randint(100, - size=(3, 4, 5, 6)).astype(self.dtype) + self.input = np.random.randint(100, size=(3, 4, 5, 6)).astype( + self.dtype + ) self.starts = np.array([1, 0, 2]).astype('int32') self.ends = np.array([3, 3, 4]).astype('int32') self.axes = [0, 1, 2] diff --git a/python/paddle/fluid/tests/unittests/npu/test_smooth_l1_loss_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_smooth_l1_loss_op_npu.py index f133e7865cc3f..54e625a9f4a91 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_smooth_l1_loss_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_smooth_l1_loss_op_npu.py @@ -33,7 +33,6 @@ def smooth_l1_loss_forward(val, sigma2): class TestSmoothL1LossOp1(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -41,7 +40,7 @@ def setUp(self): dims = (5, 20) self.inputs = { 'X': np.random.random(dims).astype("float32"), - 'Y': np.random.random(dims).astype("float32") + 'Y': np.random.random(dims).astype("float32"), } sigma = 3.0 self.attrs = {'sigma': sigma} @@ -51,7 +50,7 @@ def setUp(self): loss = loss.reshape((dims[0], 1)) self.outputs = { 'Diff': diff.astype('float32'), - 'Out': loss.astype('float32') + 'Out': loss.astype('float32'), } def set_npu(self): @@ -61,25 +60,30 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=0.02) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', max_relative_error=0.02 + ) def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - max_relative_error=0.03, - no_grad_set=set("X")) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + max_relative_error=0.03, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.03, - no_grad_set=set('Y')) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + max_relative_error=0.03, + no_grad_set=set('Y'), + ) class TestSmoothL1LossOp2(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -89,7 +93,7 @@ def setUp(self): 'X': np.random.random(dims).astype("float32"), 'Y': np.random.random(dims).astype("float32"), 'InsideWeight': np.random.random(dims).astype("float32"), - 'OutsideWeight': np.random.random(dims).astype("float32") + 'OutsideWeight': np.random.random(dims).astype("float32"), } sigma = 3.0 self.attrs = {'sigma': sigma} @@ -101,7 +105,7 @@ def setUp(self): loss = loss.sum(1).reshape((dims[0], 1)) self.outputs = { 'Diff': diff.astype('float32'), - 'Out': loss.astype('float32') + 'Out': loss.astype('float32'), } def set_npu(self): @@ -111,34 +115,39 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad_normal(self): - self.check_grad_with_place(self.place, ['X', 'Y'], - 'Out', - max_relative_error=0.03) + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', max_relative_error=0.03 + ) def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - max_relative_error=0.03, - no_grad_set=set( - ['X', 'InsideWeight', 'OutsideWeight'])) + self.check_grad_with_place( + self.place, + ['Y'], + 'Out', + max_relative_error=0.03, + no_grad_set=set(['X', 'InsideWeight', 'OutsideWeight']), + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.03, - no_grad_set=set( - ['Y', 'InsideWeight', 'OutsideWeight'])) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + max_relative_error=0.03, + no_grad_set=set(['Y', 'InsideWeight', 'OutsideWeight']), + ) class TestSmoothL1LossOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): # The input type of accuracy_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.NPUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.NPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.NPUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.NPUPlace(0) + ) self.assertRaises(TypeError, fluid.layers.smooth_l1, x1, y1) # The input dtype of accuracy_op must be float32 or float64. x2 = fluid.layers.data(name='x2', shape=[4], dtype="int32") diff --git a/python/paddle/fluid/tests/unittests/npu/test_softmax_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_softmax_op_npu.py index 1c145c4972146..41ccda3dba762 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_softmax_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_softmax_op_npu.py @@ -27,7 +27,6 @@ class TestSoftmax(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -53,7 +52,6 @@ def test_check_output(self): class TestSoftmaxNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -68,9 +66,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[4, 32], dtype='float32') b = paddle.static.data(name="b", shape=[4, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[4, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[4, 1], dtype='int64' + ) c = paddle.multiply(a, b) d = paddle.sqrt(c) @@ -99,16 +97,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_softmax_with_cross_entropy_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_softmax_with_cross_entropy_op_npu.py index 79654bf170edd..f19e892f9a37e 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_softmax_with_cross_entropy_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_softmax_with_cross_entropy_op_npu.py @@ -28,7 +28,6 @@ class TestSoftmaxWithCrossEntropyOp(OpTest): - def set_npu(self): self.__class__.use_npu = True @@ -51,8 +50,10 @@ def setUp(self): self.initParams() logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, logits) if self.soft_label: @@ -63,8 +64,9 @@ def setUp(self): self.shape[self.axis] = 1 labels = np.random.randint(0, axis_dim, self.shape, dtype="int64") - loss = cross_entropy(softmax, labels, self.soft_label, self.axis, - self.ignore_index) + loss = cross_entropy( + softmax, labels, self.soft_label, self.axis, self.ignore_index + ) one_hot_label = np.eye(axis_dim)[labels.reshape(-1)] @@ -72,7 +74,7 @@ def setUp(self): self.outputs = { "Backprop": (softmax - one_hot_label).astype(self.dtype), "Softmax": softmax.astype(self.dtype), - "Loss": loss.astype(self.dtype) + "Loss": loss.astype(self.dtype), } self.attrs = { "numeric_stable_mode": self.numeric_stable_mode, @@ -88,14 +90,16 @@ def test_check_output(self): def test_check_grad(self): # fp32 has low precision, cpu and npu both need to relax the max_relative_error if using fp32 - self.check_grad_with_place(self.place, ['Logits'], - 'Loss', - numeric_grad_delta=0.001, - max_relative_error=0.5) + self.check_grad_with_place( + self.place, + ['Logits'], + 'Loss', + numeric_grad_delta=0.001, + max_relative_error=0.5, + ) class TestPowNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -110,9 +114,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) @@ -136,16 +140,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_split_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_split_op_npu.py index 7550735c49696..d6a6cf9d57323 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_split_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_split_op_npu.py @@ -26,10 +26,10 @@ SEED = 2021 -@unittest.skipIf(not paddle.is_compiled_with_npu(), - "core is not compiled with NPU") +@unittest.skipIf( + not paddle.is_compiled_with_npu(), "core is not compiled with NPU" +) class TestCase1(OpTest): - def setUp(self): self.set_npu() self.set_example() @@ -37,9 +37,9 @@ def setUp(self): self.place = paddle.NPUPlace(0) ipt = self.x.astype(self.dtype) axis = self.axis if isinstance(self.axis, int) else int(self.axis[0]) - tmp_outs = np.split(ipt, - axis=axis, - indices_or_sections=self.num_or_sections) + tmp_outs = np.split( + ipt, axis=axis, indices_or_sections=self.num_or_sections + ) tmp_outs = [o.astype(self.dtype) for o in tmp_outs] self.outputs = {'Out': []} self.outs = [] @@ -69,7 +69,6 @@ def set_example(self): class TestCase2(TestCase1): - def set_example(self): self.dtype = "float32" self.x = np.random.random((20, 4, 50)) @@ -78,7 +77,6 @@ def set_example(self): class TestCase4(TestCase1): - def set_example(self): self.dtype = "float16" self.x = np.random.random((4, 50, 20)) @@ -88,7 +86,6 @@ def set_example(self): # Test Sections class TestCase5(TestCase1): - def set_example(self): super().set_example() self.x = np.random.random((2, 10, 4)) @@ -101,7 +98,6 @@ def setUp(self): class API_TestSplit(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.layers.data('data', shape=[-1, 10], dtype='float32') @@ -110,13 +106,12 @@ def test_out(self): exe = fluid.Executor(place) input1 = np.random.random([1, 10]).astype('float32') r0, r1 = exe.run(feed={"data": input1}, fetch_list=[x0, x1]) - ex_x0, ex_x1 = np.split(input1, (3, ), axis=1) + ex_x0, ex_x1 = np.split(input1, (3,), axis=1) np.testing.assert_allclose(ex_x0, r0) np.testing.assert_allclose(ex_x1, r1) class API_TestSplit2(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.layers.data('data', shape=[-1, 10], dtype='float32') @@ -131,7 +126,6 @@ def test_out(self): class API_TestDygraphSplit(unittest.TestCase): - def test_out1(self): with fluid.dygraph.guard(paddle.NPUPlace(0)): input_1 = np.random.random([4, 6, 6]).astype("int32") diff --git a/python/paddle/fluid/tests/unittests/npu/test_sqrt_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sqrt_op_npu.py index 5aca464aeba9a..d28f67a51e3e5 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sqrt_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sqrt_op_npu.py @@ -26,7 +26,6 @@ class TestSqrt(OpTest): - def setUp(self): self.set_npu() self.op_type = "sqrt" @@ -58,7 +57,6 @@ def test_check_grad(self): class TestSqrtFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "sqrt" @@ -85,7 +83,6 @@ def test_check_output(self): class TestSqrtNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -100,9 +97,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) d = paddle.sqrt(c) @@ -126,16 +123,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_square_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_square_op_npu.py index 86849cdb9c282..f6dbefee32a6f 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_square_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_square_op_npu.py @@ -26,7 +26,6 @@ class TestSquare(OpTest): - def setUp(self): self.set_npu() self.op_type = "square" @@ -55,7 +54,6 @@ def test_check_grad(self): class TestSquareFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "square" @@ -82,7 +80,6 @@ def test_check_output(self): class TestSquareNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -97,9 +94,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) d = paddle.square(c) @@ -123,16 +120,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_squared_l2_norm_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_squared_l2_norm_op_npu.py index 6c4f66958acde..ac1e57f295993 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_squared_l2_norm_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_squared_l2_norm_op_npu.py @@ -25,8 +25,7 @@ class TestL2LossOp(OpTest): - """Test npu squared_l2_norm - """ + """Test npu squared_l2_norm""" def setUp(self): self.set_npu() @@ -46,9 +45,9 @@ def test_check_output(self): self.check_output_with_place(place=self.place) def test_check_grad(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=self.max_relative_error) + self.check_grad_with_place( + self.place, ['X'], 'Out', max_relative_error=self.max_relative_error + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/npu/test_squeeze_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_squeeze_op_npu.py index 372c13fb90083..35940f62b8333 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_squeeze_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_squeeze_op_npu.py @@ -29,7 +29,6 @@ class TestSqueezeOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "squeeze" @@ -62,7 +61,6 @@ def init_attrs(self): class TestSqueezeOp1(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 3, 1, 40) self.axes = (0, -2) @@ -73,7 +71,6 @@ def init_test_case(self): class TestSqueezeOp2(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -84,7 +81,6 @@ def init_test_case(self): class TestSqueezeOp3(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) @@ -95,7 +91,6 @@ def init_test_case(self): class TestSqueezeOp4(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, 2) @@ -103,13 +98,13 @@ def init_test_case(self): class TestSqueezeOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): # The input type of softmax_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - paddle.NPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], paddle.NPUPlace(0) + ) self.assertRaises(TypeError, paddle.squeeze, x1) # The input axes of squeeze must be list. x2 = paddle.static.data(name='x2', shape=[4], dtype="int32") @@ -120,7 +115,6 @@ def test_errors(self): class API_TestSqueeze(unittest.TestCase): - def setUp(self): self.executed_api() @@ -129,29 +123,29 @@ def executed_api(self): def test_out(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - data1 = paddle.static.data('data1', - shape=[-1, 1, 10], - dtype='float64') + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + data1 = paddle.static.data( + 'data1', shape=[-1, 1, 10], dtype='float64' + ) result_squeeze = self.squeeze(data1, axis=[1]) place = paddle.NPUPlace(0) exe = paddle.static.Executor(place) input1 = np.random.random([5, 1, 10]).astype('float64') - result, = exe.run(feed={"data1": input1}, - fetch_list=[result_squeeze]) + (result,) = exe.run( + feed={"data1": input1}, fetch_list=[result_squeeze] + ) expected_result = np.squeeze(input1, axis=1) np.testing.assert_allclose(expected_result, result) class API_TestStaticSqueeze_(API_TestSqueeze): - def executed_api(self): self.squeeze = paddle.squeeze_ class API_TestDygraphSqueeze(unittest.TestCase): - def setUp(self): self.executed_api() @@ -210,14 +204,12 @@ def test_dimension_not_1(self): class API_TestDygraphSqueezeInplace(API_TestDygraphSqueeze): - def executed_api(self): self.squeeze = paddle.squeeze_ # Correct: General. class TestSqueeze2Op(OpTest): - def setUp(self): self.set_npu() self.op_type = "squeeze2" @@ -226,15 +218,16 @@ def setUp(self): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float32") + "XShape": np.random.random(self.ori_shape).astype("float32"), } def set_npu(self): self.__class__.use_npu = True def test_check_output(self): - self.check_output_with_place(paddle.NPUPlace(0), - no_check_set=['XShape']) + self.check_output_with_place( + paddle.NPUPlace(0), no_check_set=['XShape'] + ) def test_check_grad(self): self.check_grad_with_place(paddle.NPUPlace(0), ["X"], "Out") @@ -250,7 +243,6 @@ def init_attrs(self): # Correct: There is mins axis. class TestSqueeze2Op1(TestSqueeze2Op): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = (0, -2) @@ -259,7 +251,6 @@ def init_test_case(self): # Correct: No axes input. class TestSqueeze2Op2(TestSqueeze2Op): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -268,7 +259,6 @@ def init_test_case(self): # Correct: Just part of axes be squeezed. class TestSqueeze2Op3(TestSqueeze2Op): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) diff --git a/python/paddle/fluid/tests/unittests/npu/test_stack_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_stack_op_npu.py index 5f768c78af764..8a9e8879b8226 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_stack_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_stack_op_npu.py @@ -26,7 +26,6 @@ class TestStackOpBase(OpTest): - def initDefaultParameters(self): self.num_inputs = 4 self.input_dim = (5, 6, 7) @@ -50,7 +49,8 @@ def setUp(self): self.x = [] for i in range(self.num_inputs): self.x.append( - np.random.random(size=self.input_dim).astype(self.dtype)) + np.random.random(size=self.input_dim).astype(self.dtype) + ) tmp = [] x_names = self.get_x_names() @@ -78,49 +78,41 @@ def test_check_grad(self): class TestStackOp1(TestStackOpBase): - def initParameters(self): self.num_inputs = 16 class TestStackOp2(TestStackOpBase): - def initParameters(self): self.num_inputs = 20 class TestStackOp3(TestStackOpBase): - def initParameters(self): self.axis = -1 class TestStackOp4(TestStackOpBase): - def initParameters(self): self.axis = -4 class TestStackOp5(TestStackOpBase): - def initParameters(self): self.axis = 1 class TestStackOp6(TestStackOpBase): - def initParameters(self): self.axis = 3 class TestStackOpINT32(TestStackOpBase): - def init_dtype(self): self.dtype = np.int32 class TestStackOpINT64(TestStackOpBase): - def init_dtype(self): self.dtype = np.int64 @@ -135,8 +127,11 @@ def setUp(self): self.iter_num = 3 self.input_shape = [2, 3] self.x = np.random.random(self.input_shape).astype("float32") - self.place = paddle.NPUPlace(0) \ - if paddle.is_compiled_with_npu() else paddle.CPUPlace() + self.place = ( + paddle.NPUPlace(0) + if paddle.is_compiled_with_npu() + else paddle.CPUPlace() + ) self.set_program() def set_program(self): @@ -156,7 +151,8 @@ def test_case(self): exe = fluid.Executor(self.place) res = exe.run(self.program, fetch_list=self.out_var) np.testing.assert_allclose( - res[0], np.stack([self.x] * self.iter_num, axis=self.axis)) + res[0], np.stack([self.x] * self.iter_num, axis=self.axis) + ) class TestTensorStackAPIWithLoDTensorArray(unittest.TestCase): @@ -169,8 +165,11 @@ def setUp(self): self.iter_num = 3 self.input_shape = [2, 3] self.x = np.random.random(self.input_shape).astype("float32") - self.place = paddle.NPUPlace(0) \ - if paddle.is_compiled_with_npu() else paddle.CPUPlace() + self.place = ( + paddle.NPUPlace(0) + if paddle.is_compiled_with_npu() + else paddle.CPUPlace() + ) self.set_program() def set_program(self): @@ -190,11 +189,11 @@ def test_case(self): exe = fluid.Executor(self.place) res = exe.run(self.program, fetch_list=self.out_var) np.testing.assert_allclose( - res[0], np.stack([self.x] * self.iter_num, axis=self.axis)) + res[0], np.stack([self.x] * self.iter_num, axis=self.axis) + ) class API_test(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[1, 2], dtype='float32') @@ -206,12 +205,10 @@ def test_out(self): input1 = np.random.random([1, 2]).astype('float32') input2 = np.random.random([1, 2]).astype('float32') input3 = np.random.random([1, 2]).astype('float32') - result, = exe.run(feed={ - "data1": input1, - "data2": input2, - "data3": input3 - }, - fetch_list=[result_stack]) + (result,) = exe.run( + feed={"data1": input1, "data2": input2, "data3": input3}, + fetch_list=[result_stack], + ) expected_result = np.stack([input1, input2, input3], axis=0) np.testing.assert_allclose(expected_result, result) @@ -222,7 +219,6 @@ def test_single_tensor_error(self): class API_DygraphTest(unittest.TestCase): - def test_out(self): data1 = np.array([[1.0, 2.0]]) data2 = np.array([[3.0, 4.0]]) diff --git a/python/paddle/fluid/tests/unittests/npu/test_strided_slice_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_strided_slice_op_npu.py index bf32653455c22..6dcd8fcdae2ba 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_strided_slice_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_strided_slice_op_npu.py @@ -40,33 +40,52 @@ def strided_slice_native_forward(input, axes, starts, ends, strides): stride[axes[i]] = strides[i] result = { - 1: lambda input, start, end, stride: input[start[0]:end[0]:stride[0]], - 2: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1]], - 3: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2]], - 4: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2], start[3]:end[3]:stride[3]], - 5: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2], start[3]:end[3]:stride[3], start[4]:end[4]:stride[4]], - 6: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2], start[3]:end[3]:stride[3], \ - start[4]:end[4]:stride[4], start[5]:end[5]:stride[5]] + 1: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0] + ], + 2: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], start[1] : end[1] : stride[1] + ], + 3: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + ], + 4: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + start[3] : end[3] : stride[3], + ], + 5: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + start[3] : end[3] : stride[3], + start[4] : end[4] : stride[4], + ], + 6: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + start[3] : end[3] : stride[3], + start[4] : end[4] : stride[4], + start[5] : end[5] : stride[5], + ], }[dim](input, start, end, stride) return result class TestStridedSliceOp(OpTest): - def setUp(self): self.initTestCase() self.set_npu() self.place = paddle.NPUPlace(0) self.op_type = 'strided_slice' - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.inputs = {'Input': self.input} self.outputs = {'Out': self.output} @@ -75,7 +94,7 @@ def setUp(self): 'starts': self.starts, 'ends': self.ends, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def set_npu(self): @@ -97,7 +116,6 @@ def initTestCase(self): class TestStridedSliceOp1(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(100) self.axes = [0] @@ -108,7 +126,6 @@ def initTestCase(self): class TestStridedSliceOp2(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(100) self.axes = [0] @@ -119,7 +136,6 @@ def initTestCase(self): class TestStridedSliceOp3(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(100) self.axes = [0] @@ -130,7 +146,6 @@ def initTestCase(self): class TestStridedSliceOp4(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 4, 10) self.axes = [0, 1, 2] @@ -141,7 +156,6 @@ def initTestCase(self): class TestStridedSliceOp5(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5) self.axes = [0, 1, 2] @@ -152,7 +166,6 @@ def initTestCase(self): class TestStridedSliceOp6(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5) self.axes = [0, 1, 2] @@ -163,7 +176,6 @@ def initTestCase(self): class TestStridedSliceOp7(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5) self.axes = [0, 1, 2] @@ -174,7 +186,6 @@ def initTestCase(self): class TestStridedSliceOp8(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(1, 100, 1) self.axes = [1] @@ -185,7 +196,6 @@ def initTestCase(self): class TestStridedSliceOp9(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(1, 100, 1) self.axes = [1] @@ -196,7 +206,6 @@ def initTestCase(self): class TestStridedSliceOp10(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(10, 10) self.axes = [0, 1] @@ -207,7 +216,6 @@ def initTestCase(self): class TestStridedSliceOp11(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4) self.axes = [0, 1, 2, 3] @@ -218,7 +226,6 @@ def initTestCase(self): class TestStridedSliceOp12(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4, 5) self.axes = [0, 1, 2, 3, 4] @@ -229,7 +236,6 @@ def initTestCase(self): class TestStridedSliceOp13(TestStridedSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 6, 7, 8) self.axes = [0, 1, 2, 3, 4, 5] @@ -240,13 +246,11 @@ def initTestCase(self): class TestStridedSliceOpBool(TestStridedSliceOp): - def test_check_grad(self): pass class TestStridedSliceOpBool1D(TestStridedSliceOpBool): - def initTestCase(self): self.input = np.random.rand(100).astype("bool") self.axes = [0] @@ -257,7 +261,6 @@ def initTestCase(self): class TestStridedSliceOpBool2D(TestStridedSliceOpBool): - def initTestCase(self): self.input = np.random.rand(10, 10).astype("bool") self.axes = [0, 1] @@ -268,7 +271,6 @@ def initTestCase(self): class TestStridedSliceOpBool3D(TestStridedSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 4, 10).astype("bool") self.axes = [0, 1, 2] @@ -279,7 +281,6 @@ def initTestCase(self): class TestStridedSliceOpBool4D(TestStridedSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4).astype("bool") self.axes = [0, 1, 2, 3] @@ -290,7 +291,6 @@ def initTestCase(self): class TestStridedSliceOpBool5D(TestStridedSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4, 5).astype("bool") self.axes = [0, 1, 2, 3, 4] @@ -301,7 +301,6 @@ def initTestCase(self): class TestStridedSliceOpBool6D(TestStridedSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 6, 7, 8).astype("bool") self.axes = [0, 1, 2, 3, 4, 5] @@ -312,7 +311,6 @@ def initTestCase(self): class TestStridedSliceOp_starts_ListTensor(OpTest): - def setUp(self): self.place = paddle.NPUPlace(0) self.op_type = "strided_slice" @@ -321,8 +319,9 @@ def setUp(self): starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} self.outputs = {'Out': self.output} @@ -331,7 +330,7 @@ def setUp(self): 'starts': self.starts_infer, 'ends': self.ends, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def set_npu(self): @@ -344,9 +343,9 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [1, -1, 1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.starts_infer = [1, 10, 2] @@ -358,7 +357,6 @@ def test_check_grad_normal(self): class TestStridedSliceOp_ends_ListTensor(OpTest): - def setUp(self): self.place = paddle.NPUPlace(0) self.op_type = "strided_slice" @@ -367,8 +365,9 @@ def setUp(self): ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'EndsTensorList': ends_tensor} self.outputs = {'Out': self.output} @@ -377,7 +376,7 @@ def setUp(self): 'starts': self.starts, 'ends': self.ends_infer, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def set_npu(self): @@ -390,9 +389,9 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, 1, 2] self.infer_flags = [1, -1, 1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.ends_infer = [3, 1, 4] @@ -404,7 +403,6 @@ def test_check_grad_normal(self): class TestStridedSliceOp_starts_Tensor(OpTest): - def setUp(self): self.place = paddle.NPUPlace(0) self.op_type = "strided_slice" @@ -413,7 +411,7 @@ def setUp(self): self.inputs = { 'Input': self.input, - "StartsTensor": np.array(self.starts, dtype="int32") + "StartsTensor": np.array(self.starts, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -434,9 +432,9 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) @@ -446,7 +444,6 @@ def test_check_grad_normal(self): class TestStridedSliceOp_ends_Tensor(OpTest): - def setUp(self): self.place = paddle.NPUPlace(0) self.op_type = "strided_slice" @@ -455,7 +452,7 @@ def setUp(self): self.inputs = { 'Input': self.input, - "EndsTensor": np.array(self.ends, dtype="int32") + "EndsTensor": np.array(self.ends, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -476,9 +473,9 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) @@ -488,7 +485,6 @@ def test_check_grad_normal(self): class TestStridedSliceOp_listTensor_Tensor(OpTest): - def setUp(self): self.place = paddle.NPUPlace(0) self.op_type = "strided_slice" @@ -497,13 +493,14 @@ def setUp(self): ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int32"), - "EndsTensorList": ends_tensor + "EndsTensorList": ends_tensor, } self.outputs = {'Out': self.output} self.attrs = { @@ -524,9 +521,9 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) @@ -536,7 +533,6 @@ def test_check_grad_normal(self): class TestStridedSliceOp_strides_Tensor(OpTest): - def setUp(self): self.place = paddle.NPUPlace(0) self.op_type = "strided_slice" @@ -545,7 +541,7 @@ def setUp(self): self.inputs = { 'Input': self.input, - "StridesTensor": np.array(self.strides, dtype="int32") + "StridesTensor": np.array(self.strides, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -566,9 +562,9 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, -1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output_with_place(self.place) @@ -576,51 +572,54 @@ def test_check_output(self): def test_check_grad_normal(self): self.check_grad_with_place(self.place, ['Input'], 'Out') - # Test python API -class TestStridedSliceAPI(unittest.TestCase): + +class TestStridedSliceAPI(unittest.TestCase): def test_1(self): input = np.random.random([3, 4, 5, 6]).astype("float64") minus_1 = fluid.layers.fill_constant([1], "int32", -1) minus_3 = fluid.layers.fill_constant([1], "int32", -3) - starts = fluid.layers.data(name='starts', - shape=[3], - dtype='int32', - append_batch_size=False) - ends = fluid.layers.data(name='ends', - shape=[3], - dtype='int32', - append_batch_size=False) - strides = fluid.layers.data(name='strides', - shape=[3], - dtype='int32', - append_batch_size=False) - - x = fluid.layers.data(name="x", - shape=[3, 4, 5, 6], - append_batch_size=False, - dtype="float64") - out_1 = fluid.layers.strided_slice(x, - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[3, 100, -1], - strides=[1, 1, 1]) - out_2 = fluid.layers.strided_slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, -1], - strides=[1, 1, 1]) - out_3 = fluid.layers.strided_slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, minus_1], - strides=[1, 1, 1]) - out_4 = fluid.layers.strided_slice(x, - axes=[0, 1, 2], - starts=starts, - ends=ends, - strides=strides) + starts = fluid.layers.data( + name='starts', shape=[3], dtype='int32', append_batch_size=False + ) + ends = fluid.layers.data( + name='ends', shape=[3], dtype='int32', append_batch_size=False + ) + strides = fluid.layers.data( + name='strides', shape=[3], dtype='int32', append_batch_size=False + ) + + x = fluid.layers.data( + name="x", + shape=[3, 4, 5, 6], + append_batch_size=False, + dtype="float64", + ) + out_1 = fluid.layers.strided_slice( + x, + axes=[0, 1, 2], + starts=[-3, 0, 2], + ends=[3, 100, -1], + strides=[1, 1, 1], + ) + out_2 = fluid.layers.strided_slice( + x, + axes=[0, 1, 3], + starts=[minus_3, 0, 2], + ends=[3, 100, -1], + strides=[1, 1, 1], + ) + out_3 = fluid.layers.strided_slice( + x, + axes=[0, 1, 3], + starts=[minus_3, 0, 2], + ends=[3, 100, minus_1], + strides=[1, 1, 1], + ) + out_4 = fluid.layers.strided_slice( + x, axes=[0, 1, 2], starts=starts, ends=ends, strides=strides + ) out_5 = x[-3:3, 0:100:2, -1:2:-1] out_6 = x[minus_3:3:1, 0:100:2, :, minus_1:2:minus_1] @@ -633,9 +632,10 @@ def test_1(self): "x": input, 'starts': np.array([-3, 0, 2]).astype("int32"), 'ends': np.array([3, 2147483648, -1]).astype("int64"), - 'strides': np.array([1, 1, 1]).astype("int32") + 'strides': np.array([1, 1, 1]).astype("int32"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7], + ) assert np.array_equal(res_1, input[-3:3, 0:100, 2:-1, :]) assert np.array_equal(res_2, input[-3:3, 0:100, :, 2:-1]) assert np.array_equal(res_3, input[-3:3, 0:100, :, 2:-1]) @@ -650,11 +650,9 @@ def test_dygraph_op(self): starts = [-3, 0, 2] ends = [3, 2, 4] strides_1 = [1, 1, 1] - sliced_1 = paddle.strided_slice(x, - axes=axes, - starts=starts, - ends=ends, - strides=strides_1) + sliced_1 = paddle.strided_slice( + x, axes=axes, starts=starts, ends=ends, strides=strides_1 + ) assert sliced_1.shape == (3, 2, 2, 2) diff --git a/python/paddle/fluid/tests/unittests/npu/test_sum_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sum_op_npu.py index bda9ebe9d1114..8471fa09e2082 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_sum_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sum_op_npu.py @@ -27,7 +27,6 @@ class TestSum1(OpTest): - def setUp(self): self.set_npu() self.init_dtype() @@ -54,7 +53,6 @@ def test_check_output(self): class TestSum2(OpTest): - def setUp(self): self.set_npu() self.init_dtype() @@ -72,8 +70,12 @@ def setUp(self): # For example, the results of `x0+x1+x2+x3` is different from that of # `x3+x2+x1+x0` if the dtype is fp16. # Therefore, converting the input to fp32 for calculation. - y = (x0.astype(np.float32) + x1.astype(np.float32) + - x2.astype(np.float32) + x3.astype(np.float32)).astype(self.dtype) + y = ( + x0.astype(np.float32) + + x1.astype(np.float32) + + x2.astype(np.float32) + + x3.astype(np.float32) + ).astype(self.dtype) self.outputs = {'Out': y} self.attrs = {'use_mkldnn': False} @@ -89,7 +91,6 @@ def test_check_output(self): class TestSum3(OpTest): - def setUp(self): self.set_npu() self.init_dtype() diff --git a/python/paddle/fluid/tests/unittests/npu/test_swish_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_swish_op_npu.py index 9c45c84c43bd7..d967e1fc06d08 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_swish_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_swish_op_npu.py @@ -27,7 +27,6 @@ class TestSwishOp(OpTest): - def setUp(self): self.op_type = "swish" self.set_npu() @@ -49,10 +48,13 @@ def test_check_grad(self): dx = beta * out + expit(x) * (1 - beta * out) dx = dx / x.size - self.check_grad_with_place(self.place, ['X'], - 'Out', - max_relative_error=0.01, - user_defined_grads=[dx]) + self.check_grad_with_place( + self.place, + ['X'], + 'Out', + max_relative_error=0.01, + user_defined_grads=[dx], + ) def set_npu(self): self.__class__.use_npu = True @@ -63,7 +65,6 @@ def init_dtype(self): class TestSwishOpFp16(TestSwishOp): - def test_check_output(self): self.check_output_with_place(self.place, atol=1e-3) diff --git a/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_base_npu.py b/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_base_npu.py index 30aa594114307..1629c8e8fb20a 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_base_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_base_npu.py @@ -30,9 +30,14 @@ from paddle.fluid import core import paddle -from paddle.fluid.tests.unittests.op_test import OpTest, _set_use_system_allocator +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + _set_use_system_allocator, +) -from paddle.fluid.tests.unittests.test_sync_batch_norm_op import create_or_get_tensor +from paddle.fluid.tests.unittests.test_sync_batch_norm_op import ( + create_or_get_tensor, +) _set_use_system_allocator(False) paddle.enable_static() @@ -41,17 +46,19 @@ class TestSyncBatchNormRunnerBase(object): - - def get_model(self, - main, - startup, - place, - layout, - seed, - sync_bn=False, - only_forward=False): + def get_model( + self, + main, + startup, + place, + layout, + seed, + sync_bn=False, + only_forward=False, + ): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def wait_server_ready(self, endpoints): assert not isinstance(endpoints, str) @@ -60,13 +67,15 @@ def wait_server_ready(self, endpoints): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, - 1) + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT, 1 + ) result = sock.connect_ex((ip_port[0], int(ip_port[1]))) if result != 0: @@ -74,43 +83,50 @@ def wait_server_ready(self, endpoints): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + - str(not_ready_endpoints) + "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: break + # endpoints should be ["ip1:port1","ip2:port2"] -#endpoints should be ["ip1:port1","ip2:port2"] - - def initCommunicator(self, program, rank, nranks, wait_port, - current_endpoint, endpoints): + def initCommunicator( + self, program, rank, nranks, wait_port, current_endpoint, endpoints + ): other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) if rank == 0 and wait_port: self.wait_server_ready(other_endpoints) block = program.global_block() - hccl_id_var = block.create_var(name=nameGen.generate('hccl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_hccl_id', - inputs={}, - outputs={'Out': hccl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) - block.append_op(type='c_comm_init_hccl', - inputs={'X': hccl_id_var}, - outputs={}, - attrs={ - 'rank': rank, - 'ring_id': self.global_ring_id, - 'device_id': int(os.getenv("FLAGS_selected_npus")), - 'rank_ids': nranks - }) + hccl_id_var = block.create_var( + name=nameGen.generate('hccl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_hccl_id', + inputs={}, + outputs={'Out': hccl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) + block.append_op( + type='c_comm_init_hccl', + inputs={'X': hccl_id_var}, + outputs={}, + attrs={ + 'rank': rank, + 'ring_id': self.global_ring_id, + 'device_id': int(os.getenv("FLAGS_selected_npus")), + 'rank_ids': nranks, + }, + ) def run_trainer(self, args): device_id = int(os.getenv("FLAGS_selected_npus", "0")) @@ -143,24 +159,30 @@ def run_trainer(self, args): sys.stdout.buffer.write( pickle.dumps( - 'training, inference, fp32, fp16, NCHW, NHWC all passed')) + 'training, inference, fp32, fp16, NCHW, NHWC all passed' + ) + ) def _compare(self, args, place, layout, only_forward): scope = core.Scope() np.random.seed(SEED) - data = np.random.random(size=self.dshape).astype(self.dtype) * 4. - 2 + data = np.random.random(size=self.dshape).astype(self.dtype) * 4.0 - 2 sys.stderr.write("data: " + str(data) + "\n") - data = create_or_get_tensor(scope, "input", - OpTest.np_dtype_to_fluid_dtype(data), place) + data = create_or_get_tensor( + scope, "input", OpTest.np_dtype_to_fluid_dtype(data), place + ) - bn_fetches = self._cal_single_card(args, data, place, layout, - only_forward) + bn_fetches = self._cal_single_card( + args, data, place, layout, only_forward + ) fetch_names, sync_bn_fetches = self._cal_multiple_cards( - args, data, place, layout, only_forward) + args, data, place, layout, only_forward + ) - sys.stderr.write("len(sync_bn_fetches): " + str(len(sync_bn_fetches)) + - "\n") + sys.stderr.write( + "len(sync_bn_fetches): " + str(len(sync_bn_fetches)) + "\n" + ) for i in range(0, len(sync_bn_fetches)): sys.stderr.write("i: " + str(i) + "\n") sys.stderr.write("fetch_names[i]): " + fetch_names[i] + "\n") @@ -168,13 +190,14 @@ def _compare(self, args, place, layout, only_forward): bn_val = bn_fetches[i] sync_bn_val = sync_bn_fetches[i] if sync_bn_val.shape != bn_val.shape: - sync_bn_val = sync_bn_val[:bn_val.shape[0]] + sync_bn_val = sync_bn_val[: bn_val.shape[0]] # i = 0 if fetch_names[i] == 'reduce_sum_0.tmp_0': # sys.stderr.write("skip reduce_sum_0.tmp_0 (Out of reduce_sum op)" + "\n") - sys.stderr.write("reduce_sum_0.tmp_0 (Out of reduce_sum op)" + - "\n") + sys.stderr.write( + "reduce_sum_0.tmp_0 (Out of reduce_sum op)" + "\n" + ) sys.stderr.write("bn_val: " + str(bn_val) + "\n") sys.stderr.write("sync_bn_val: " + str(sync_bn_val) + "\n") @@ -202,7 +225,8 @@ def _compare(self, args, place, layout, only_forward): if fetch_names[i] == 'batch_norm_0.tmp_2': # sys.stderr.write("skip batch_norm_0.tmp_2 (ReserveSpace of batch_norm)" + "\n") sys.stderr.write( - "batch_norm_0.tmp_2 (ReserveSpace of batch_norm)" + "\n") + "batch_norm_0.tmp_2 (ReserveSpace of batch_norm)" + "\n" + ) sys.stderr.write("bn_val: " + str(bn_val) + "\n") sys.stderr.write("sync_bn_val: " + str(sync_bn_val) + "\n") @@ -235,8 +259,9 @@ def _compare(self, args, place, layout, only_forward): # i = 8 if fetch_names[i] == 'batch_norm_0.tmp_1': - sys.stderr.write("skip batch_norm_0.tmp_1 (SavedVariance)" + - "\n") + sys.stderr.write( + "skip batch_norm_0.tmp_1 (SavedVariance)" + "\n" + ) sys.stderr.write("bn_val: " + str(bn_val) + "\n") sys.stderr.write("sync_bn_val: " + str(sync_bn_val) + "\n") @@ -282,10 +307,16 @@ def _compare(self, args, place, layout, only_forward): if fetch_names[i] == 'conv2d_0.tmp_0@GRAD': atol = 1e-2 - assert np.allclose( - bn_val, sync_bn_val, atol=atol), "Output (" + fetch_names[ - i] + ") has diff. \n" + "\nBN " + str( - bn_val) + "\n" + "Sync BN " + str(sync_bn_val) + assert np.allclose(bn_val, sync_bn_val, atol=atol), ( + "Output (" + + fetch_names[i] + + ") has diff. \n" + + "\nBN " + + str(bn_val) + + "\n" + + "Sync BN " + + str(sync_bn_val) + ) def _cal_single_card(self, args, data, place, layout, only_forward): # Single-NPU, N = 32 per NPU @@ -295,23 +326,31 @@ def _cal_single_card(self, args, data, place, layout, only_forward): startup_prog.global_seed(SEED) paddle.seed(SEED) - outs = self.get_model(train_prog, startup_prog, place, layout, SEED, - False, only_forward) + outs = self.get_model( + train_prog, startup_prog, place, layout, SEED, False, only_forward + ) exe = fluid.Executor(place) exe.run(startup_prog) fetch_names = [v.name for v in outs] + [ - 'bn_moving_mean', 'bn_moving_variance', 'bn_scale', 'bn_bias' + 'bn_moving_mean', + 'bn_moving_variance', + 'bn_scale', + 'bn_bias', ] if not only_forward: others = [ - 'batch_norm_0.tmp_0', 'batch_norm_0.tmp_1', 'bn_scale@GRAD', - 'bn_bias@GRAD', 'batch_norm_0.tmp_3@GRAD', 'conv2d_0.tmp_0@GRAD' + 'batch_norm_0.tmp_0', + 'batch_norm_0.tmp_1', + 'bn_scale@GRAD', + 'bn_bias@GRAD', + 'batch_norm_0.tmp_3@GRAD', + 'conv2d_0.tmp_0@GRAD', ] fetch_names += others - bn_fetches = exe.run(program=train_prog, - feed={'input': data}, - fetch_list=fetch_names) + bn_fetches = exe.run( + program=train_prog, feed={'input': data}, fetch_list=fetch_names + ) return bn_fetches @@ -333,10 +372,12 @@ def _cal_multiple_cards(self, args, data, place, layout, only_forward): current_endpoint = args["currentendpoint"] nranks = 2 - self.initCommunicator(startup_prog, rank, nranks, True, - current_endpoint, endpoints) - sys.stderr.write("after init, startup_prog: " + - startup_prog.to_string(True) + "\n") + self.initCommunicator( + startup_prog, rank, nranks, True, current_endpoint, endpoints + ) + sys.stderr.write( + "after init, startup_prog: " + startup_prog.to_string(True) + "\n" + ) train_prog.global_seed(SEED) train_prog._sync_with_cpp() startup_prog.global_seed(SEED) @@ -344,12 +385,17 @@ def _cal_multiple_cards(self, args, data, place, layout, only_forward): paddle.seed(SEED) self.rank = rank - outs = self.get_model(train_prog, startup_prog, place, layout, SEED, - True, only_forward) - sys.stderr.write("after get_model, train_prog: " + - train_prog.to_string(True) + "\n") - sys.stderr.write("after get_model, startup_prog: " + - startup_prog.to_string(True) + "\n") + outs = self.get_model( + train_prog, startup_prog, place, layout, SEED, True, only_forward + ) + sys.stderr.write( + "after get_model, train_prog: " + train_prog.to_string(True) + "\n" + ) + sys.stderr.write( + "after get_model, startup_prog: " + + startup_prog.to_string(True) + + "\n" + ) ops = train_prog.blocks[0].ops for i, op in enumerate(ops): @@ -362,23 +408,33 @@ def _cal_multiple_cards(self, args, data, place, layout, only_forward): sys.stderr.write("op type: " + op.type + "\n") op.desc.set_type('sync_batch_norm_grad') - sys.stderr.write("after update sync_batch_norm, train_prog: " + - train_prog.to_string(True) + "\n") + sys.stderr.write( + "after update sync_batch_norm, train_prog: " + + train_prog.to_string(True) + + "\n" + ) exe = fluid.Executor(place) exe.run(startup_prog) fetch_names = [v.name for v in outs] + [ - 'bn_moving_mean', 'bn_moving_variance', 'bn_scale', 'bn_bias' + 'bn_moving_mean', + 'bn_moving_variance', + 'bn_scale', + 'bn_bias', ] if not only_forward: others = [ - 'batch_norm_0.tmp_0', 'batch_norm_0.tmp_1', 'bn_scale@GRAD', - 'bn_bias@GRAD', 'batch_norm_0.tmp_3@GRAD', 'conv2d_0.tmp_0@GRAD' + 'batch_norm_0.tmp_0', + 'batch_norm_0.tmp_1', + 'bn_scale@GRAD', + 'bn_bias@GRAD', + 'batch_norm_0.tmp_3@GRAD', + 'conv2d_0.tmp_0@GRAD', ] fetch_names += others - sync_bn_fetches = exe.run(program=train_prog, - feed={'input': data}, - fetch_list=fetch_names) + sync_bn_fetches = exe.run( + program=train_prog, feed={'input': data}, fetch_list=fetch_names + ) return fetch_names, sync_bn_fetches @@ -401,19 +457,20 @@ def runtime_main(test_class, col_type, sub_type): class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -442,7 +499,7 @@ def _run_cluster(self, model_file, envs): "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) @@ -453,15 +510,19 @@ def _run_cluster(self, model_file, envs): tr1_pipe = open("/tmp/tr1_err.log", "wb") # print(tr0_cmd) # print(tr1_cmd) - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) - - tr1_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) + + tr1_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -471,12 +532,18 @@ def _run_cluster(self, model_file, envs): # close trainer file tr0_pipe.close() tr1_pipe.close() - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) def check_with_place(self, model_file, col_type, need_envs={}): tr0_out, tr1_out, pid0, pid1 = self._run_cluster(model_file, need_envs) self.assertEqual( - tr0_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed') + tr0_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed' + ) self.assertEqual( - tr1_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed') + tr1_out, 'training, inference, fp32, fp16, NCHW, NHWC all passed' + ) diff --git a/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_baseline.py b/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_baseline.py index d66cb04e009c7..2efe1f91cb87e 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_baseline.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_baseline.py @@ -20,7 +20,10 @@ sys.path.append("..") -from paddle.fluid.tests.unittests.op_test import OpTest, _set_use_system_allocator +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + _set_use_system_allocator, +) from test_sync_batch_norm_base_npu import TestDistBase @@ -29,15 +32,14 @@ class TestSyncBatchNormOp(TestDistBase): - def _setup_config(self): pass def test_identity(self, col_type="identity"): dist_env = os.environ - self.check_with_place("sync_batch_norm_op_npu.py", - col_type, - need_envs=dist_env) + self.check_with_place( + "sync_batch_norm_op_npu.py", col_type, need_envs=dist_env + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_extra.py b/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_extra.py index 44644ef245ee4..15c4917042958 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_extra.py +++ b/python/paddle/fluid/tests/unittests/npu/test_sync_batch_norm_op_npu_extra.py @@ -26,19 +26,22 @@ import paddle.nn as nn from paddle.fluid import Program, program_guard -from paddle.fluid.tests.unittests.op_test import OpTest, _set_use_system_allocator +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + _set_use_system_allocator, +) # _set_use_system_allocator(False) paddle.enable_static() class TestDygraphSyncBatchNormAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): my_sync_batch_norm = paddle.nn.SyncBatchNorm(10) - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.NPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.NPUPlace(0) + ) self.assertRaises(TypeError, my_sync_batch_norm, x1) # the input dtype of SyncBatchNorm must be float16 or float32 @@ -48,31 +51,33 @@ def test_errors(self): class TestConvertSyncBatchNorm(unittest.TestCase): - def test_convert(self): with program_guard(Program(), Program()): - compare_model = paddle.nn.Sequential(paddle.nn.Conv2D(3, 5, 3), - paddle.nn.BatchNorm2D(5), - paddle.nn.BatchNorm2D(5)) + compare_model = paddle.nn.Sequential( + paddle.nn.Conv2D(3, 5, 3), + paddle.nn.BatchNorm2D(5), + paddle.nn.BatchNorm2D(5), + ) model = paddle.nn.Sequential( - paddle.nn.Conv2D(3, 5, 3), paddle.nn.BatchNorm2D(5), + paddle.nn.Conv2D(3, 5, 3), + paddle.nn.BatchNorm2D(5), paddle.nn.BatchNorm2D( 5, weight_attr=fluid.ParamAttr(name='bn.scale'), - bias_attr=fluid.ParamAttr(name='bn.bias'))) + bias_attr=fluid.ParamAttr(name='bn.bias'), + ), + ) model = paddle.nn.SyncBatchNorm.convert_sync_batchnorm(model) for idx, sublayer in enumerate(compare_model.sublayers()): if isinstance(sublayer, paddle.nn.BatchNorm2D): self.assertEqual( - isinstance(model[idx], paddle.nn.SyncBatchNorm), True) + isinstance(model[idx], paddle.nn.SyncBatchNorm), True + ) class TestConvertSyncBatchNormCast1(unittest.TestCase): - def test_convert(self): - class Net(nn.Layer): - def __init__(self): super(Net, self).__init__() self.conv1 = nn.Conv2D(3, 5, 3) @@ -97,7 +102,6 @@ def forward(self, x): class TestDygraphSyncBatchNormDataFormatError(unittest.TestCase): - def test_errors(self): with fluid.dygraph.guard(fluid.NPUPlace(0)): my_sync_batch_norm = paddle.nn.SyncBatchNorm(10, data_format='CN') diff --git a/python/paddle/fluid/tests/unittests/npu/test_take_along_axis_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_take_along_axis_op_npu.py index d0cc8e3abde46..d3e34c3936612 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_take_along_axis_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_take_along_axis_op_npu.py @@ -27,7 +27,6 @@ @unittest.skip(reason="Skip unsupported ut, need paddle surpport cann 5.0.4+") class TestTakeAlongAxisOp(OpTest): - def setUp(self): self.set_npu() self.init_data() @@ -59,14 +58,14 @@ def init_data(self): self.x_type = "float64" self.x_shape = (5, 5, 5) self.index_type = "int32" - self.index = np.array([[[1]], [[1]], [[2]], [[4]], - [[3]]]).astype(self.index_type) + self.index = np.array([[[1]], [[1]], [[2]], [[4]], [[3]]]).astype( + self.index_type + ) self.axis = 2 self.axis_type = "int64" class TestCase1(TestTakeAlongAxisOp): - def init_data(self): self.x_type = "float64" self.x_shape = (5, 5, 5) @@ -78,7 +77,6 @@ def init_data(self): @unittest.skip(reason="Skip unsupported ut, need paddle surpport cann 5.0.4+") class TestTakeAlongAxisAPI(unittest.TestCase): - def setUp(self): np.random.seed(0) self.shape = [3, 3] @@ -95,13 +93,12 @@ def test_api_static(self): index = paddle.fluid.data('Index', self.index_shape, "int64") out = paddle.take_along_axis(x, index, self.axis) exe = paddle.static.Executor(self.place) - res = exe.run(feed={ - 'X': self.x_np, - 'Index': self.index_np - }, - fetch_list=[out]) + res = exe.run( + feed={'X': self.x_np, 'Index': self.index_np}, fetch_list=[out] + ) out_ref = np.array( - np.take_along_axis(self.x_np, self.index_np, self.axis)) + np.take_along_axis(self.x_np, self.index_np, self.axis) + ) for out in res: np.testing.assert_allclose(out, out_ref, rtol=0.001) @@ -111,20 +108,21 @@ def test_api_dygraph(self): self.index = paddle.to_tensor(self.index_np) out = paddle.take_along_axis(x_tensor, self.index, self.axis) out_ref = np.array( - np.take_along_axis(self.x_np, self.index_np, self.axis)) + np.take_along_axis(self.x_np, self.index_np, self.axis) + ) np.testing.assert_allclose(out.numpy(), out_ref, rtol=0.001) paddle.enable_static() @unittest.skip(reason="Skip unsupported ut, need paddle surpport cann 5.0.4+") class TestTakeAlongAxisAPICase1(TestTakeAlongAxisAPI): - def setUp(self): np.random.seed(0) self.shape = [2, 2] self.index_shape = [4, 2] - self.index_np = np.array([[0, 0], [1, 0], [0, 0], [1, - 0]]).astype('int64') + self.index_np = np.array([[0, 0], [1, 0], [0, 0], [1, 0]]).astype( + 'int64' + ) self.x_np = np.random.random(self.shape).astype(np.float32) self.place = paddle.NPUPlace(0) self.axis = 0 diff --git a/python/paddle/fluid/tests/unittests/npu/test_tanh_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_tanh_op_npu.py index 0b1563282c6d5..8cbb0d217eb37 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_tanh_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_tanh_op_npu.py @@ -26,7 +26,6 @@ class TestTanh(OpTest): - def setUp(self): self.set_npu() self.op_type = "tanh" @@ -58,7 +57,6 @@ def test_check_grad(self): class TestTanhFp16(OpTest): - def setUp(self): self.set_npu() self.op_type = "tanh" @@ -85,7 +83,6 @@ def test_check_output(self): class TestTanhNet(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -100,9 +97,9 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): a = paddle.static.data(name="a", shape=[32, 32], dtype='float32') b = paddle.static.data(name="b", shape=[32, 32], dtype='float32') - label = paddle.static.data(name="label", - shape=[32, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[32, 1], dtype='int64' + ) c = paddle.multiply(a, b) d = paddle.tanh(c) @@ -126,16 +123,17 @@ def _test(self, run_npu=True): print("Start run on {}".format(place)) for epoch in range(100): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) if epoch % 10 == 0: - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) return pred_res, loss_res diff --git a/python/paddle/fluid/tests/unittests/npu/test_tile_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_tile_op_npu.py index 723beb31c9afb..e8003f82aa950 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_tile_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_tile_op_npu.py @@ -27,9 +27,8 @@ np.random.seed(10) -#Situation 1: repeat_times is a list (without tensor) +# Situation 1: repeat_times is a list (without tensor) class TestTileOpRank1(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -55,44 +54,38 @@ def test_check_grad(self): pass -#with dimension expanding +# with dimension expanding class TestTileOpRank2Expanding(TestTileOpRank1): - def init_data(self): self.ori_shape = [120] self.repeat_times = [2, 2] class TestTileOpRank2(TestTileOpRank1): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] class TestTileOpRank3_Corner(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (1, 1, 1) class TestTileOpRank3_Corner2(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (2, 2) class TestTileOpRank3(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 15) self.repeat_times = (2, 1, 4) class TestTileOpRank4(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.repeat_times = (3, 2, 1, 2) @@ -100,7 +93,6 @@ def init_data(self): # Situation 2: repeat_times is a list (with tensor) class TestTileOpRank1_tensor_attr(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -108,8 +100,9 @@ def setUp(self): self.init_data() repeat_times_tensor = [] for index, ele in enumerate(self.repeat_times): - repeat_times_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + repeat_times_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype("float32"), @@ -135,7 +128,6 @@ def test_check_grad(self): class TestTileOpRank2_Corner_tensor_attr(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [1, 1] @@ -143,7 +135,6 @@ def init_data(self): class TestTileOpRank2_attr_tensor(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -152,7 +143,6 @@ def init_data(self): # Situation 3: repeat_times is a tensor class TestTileOpRank1_tensor(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -182,7 +172,6 @@ def test_check_grad(self): class TestTileOpRank2_tensor(TestTileOpRank1_tensor): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -190,7 +179,6 @@ def init_data(self): # Situation 4: input x is Integer class TestTileOpInteger(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -211,7 +199,6 @@ def test_check_output(self): # Situation 5: input x is Integer class TestTileOpInt64_t(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -232,7 +219,6 @@ def test_check_output(self): # Situation 6: input x is Bool class TestTileOpBool(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -251,7 +237,6 @@ def test_check_output(self): # Test python API class TestTileAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(paddle.NPUPlace(0)): np_x = np.random.random([12, 14]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/npu/test_top_k_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_top_k_op_npu.py index 6e73f282c8c32..f4c7bab1105c4 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_top_k_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_top_k_op_npu.py @@ -28,20 +28,24 @@ class TestTopk(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) self.op_type = "top_k" self.init_dtype() - x = np.array([[0.78104149, 0.88745828, 0.32362268], - [0.82196718, 0.48763277, 0.42826136], - [0.96527182, 0.34851612, 0.12959783]]).astype(self.dtype) + x = np.array( + [ + [0.78104149, 0.88745828, 0.32362268], + [0.82196718, 0.48763277, 0.42826136], + [0.96527182, 0.34851612, 0.12959783], + ] + ).astype(self.dtype) self.inputs = {'X': x} - np_out = np.array([[0.88745828], [0.82196718], - [0.96527182]]).astype(self.dtype) + np_out = np.array([[0.88745828], [0.82196718], [0.96527182]]).astype( + self.dtype + ) np_indices = np.array([[1], [0], [0]]) self.attrs = {'k': 1, "axis": -1} @@ -59,20 +63,28 @@ def test_check_output(self): class TestTopkV2(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) self.op_type = "top_k" self.init_dtype() - x = np.array([[0.78104149, 0.88745828, 0.32362268], - [0.82196718, 0.48763277, 0.42826136], - [0.96527182, 0.34851612, 0.12959783]]).astype(self.dtype) + x = np.array( + [ + [0.78104149, 0.88745828, 0.32362268], + [0.82196718, 0.48763277, 0.42826136], + [0.96527182, 0.34851612, 0.12959783], + ] + ).astype(self.dtype) self.inputs = {'X': x} - np_out = np.array([[0.88745828, 0.78104149], [0.82196718, 0.48763277], - [0.96527182, 0.34851612]]).astype(self.dtype) + np_out = np.array( + [ + [0.88745828, 0.78104149], + [0.82196718, 0.48763277], + [0.96527182, 0.34851612], + ] + ).astype(self.dtype) np_indices = np.array([[1, 0], [0, 1], [0, 1]]) self.attrs = {'k': 2, "axis": -1} @@ -90,7 +102,6 @@ def test_check_output(self): class TestTopkV3(OpTest): - def setUp(self): self.set_npu() self.place = paddle.NPUPlace(0) @@ -99,10 +110,9 @@ def setUp(self): self.init_dtype() self.set_input_data() self.set_attrs() - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=True) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=True + ) self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis} @@ -123,8 +133,9 @@ def set_attrs(self): self.axis = 1 def set_input_data(self): - self.input_data = np.random.choice(10000, size=(10, 20), - replace=False).astype(self.dtype) + self.input_data = np.random.choice( + 10000, size=(10, 20), replace=False + ).astype(self.dtype) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/npu/test_top_k_v2_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_top_k_v2_op_npu.py index c72ce4831f7c8..6b46a41fe1d61 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_top_k_v2_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_top_k_v2_op_npu.py @@ -39,7 +39,6 @@ def numpy_topk(x, k=1, axis=-1, largest=True): class TestTopkV2NPUOp(OpTest): - def setUp(self): paddle.enable_static() self.op_type = "top_k_v2" @@ -48,10 +47,9 @@ def setUp(self): self.set_dtype() self.set_input_data() self.set_attrs() - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} @@ -66,8 +64,9 @@ def set_attrs(self): self.largest = True def set_input_data(self): - self.input_data = np.random.choice(10000, size=(10, 20), - replace=False).astype(self.dtype) + self.input_data = np.random.choice( + 10000, size=(10, 20), replace=False + ).astype(self.dtype) def test_check_output(self): self.__class__.no_need_check_grad = True @@ -82,7 +81,6 @@ def set_npu(self): class TestTopkV2OpFloat16(TestTopkV2NPUOp): - def set_attrs(self): self.k = 3 self.axis = 1 @@ -96,7 +94,6 @@ def set_input_data(self): class TestTopkV2OP1Int32(TestTopkV2NPUOp): - def set_attrs(self): self.k = 3 self.axis = 0 @@ -104,7 +101,6 @@ def set_attrs(self): class TestTopkV2OP2Int32(TestTopkV2NPUOp): - def set_attrs(self): self.k = 4 self.axis = 0 @@ -112,7 +108,6 @@ def set_attrs(self): class TestTopkV2OP3Int32(TestTopkV2NPUOp): - def set_attrs(self): self.k = 6 self.axis = 1 @@ -120,7 +115,6 @@ def set_attrs(self): class TestTopkV2OP4Int32(TestTopkV2NPUOp): - def set_attrs(self): self.k = 3 self.axis = 1 @@ -128,31 +122,26 @@ def set_attrs(self): class TestTopkV2Op1Int64(TestTopkV2OP1Int32): - def set_dtype(self): self.dtype = np.int64 class TestTopkV2Op2Int64(TestTopkV2OP2Int32): - def set_dtype(self): self.dtype = np.int64 class TestTopkV2Op3Int64(TestTopkV2OP3Int32): - def set_dtype(self): self.dtype = np.int64 class TestTopkV2Op4Int64(TestTopkV2OP4Int32): - def set_dtype(self): self.dtype = np.int64 class TestTopkV2Op1Float32(TestTopkV2OP1Int32): - def set_dtype(self): self.dtype = np.float32 @@ -161,7 +150,6 @@ def set_input_data(self): class TestTopkV2Op2Float32(TestTopkV2OP2Int32): - def set_dtype(self): self.dtype = np.float32 @@ -170,7 +158,6 @@ def set_input_data(self): class TestTopkV2Op3Float32(TestTopkV2OP3Int32): - def set_dtype(self): self.dtype = np.float32 @@ -179,7 +166,6 @@ def set_input_data(self): class TestTopkV2Op4Float32(TestTopkV2OP4Int32): - def set_dtype(self): self.dtype = np.float32 @@ -188,7 +174,6 @@ def set_input_data(self): class TestTopkV2Op1Float64(TestTopkV2OP1Int32): - def set_dtype(self): self.dtype = np.float64 @@ -197,7 +182,6 @@ def set_input_data(self): class TestTopkV2Op2Float64(TestTopkV2OP2Int32): - def set_dtype(self): self.dtype = np.float64 @@ -206,7 +190,6 @@ def set_input_data(self): class TestTopkV2Op3Float64(TestTopkV2OP3Int32): - def set_dtype(self): self.dtype = np.float64 @@ -215,7 +198,6 @@ def set_input_data(self): class TestTopkV2Op4Float64(TestTopkV2OP4Int32): - def set_dtype(self): self.dtype = np.float64 @@ -224,7 +206,6 @@ def set_input_data(self): class TestTopKAPI(unittest.TestCase): - def setUp(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -275,22 +256,23 @@ def run_dygraph(self, place): np.testing.assert_allclose(paddle_result[1].numpy(), numpy_result[1]) # test case for basic test case 7 for the unsorted paddle_result = paddle.topk(input_tensor, k=2, axis=1, sorted=False) - sort_paddle = numpy_topk(np.array(paddle_result[0].numpy()), - axis=1, - k=2) + sort_paddle = numpy_topk( + np.array(paddle_result[0].numpy()), axis=1, k=2 + ) numpy_result = numpy_topk(self.input_data, k=2, axis=1) np.testing.assert_allclose(sort_paddle[0], numpy_result[0]) def run_static(self, place): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - input_tensor = paddle.static.data(name="x", - shape=[6, 7, 8], - dtype="float64") - large_input_tensor = paddle.static.data(name="large_x", - shape=[2, 1030], - dtype="float64") + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + input_tensor = paddle.static.data( + name="x", shape=[6, 7, 8], dtype="float64" + ) + large_input_tensor = paddle.static.data( + name="large_x", shape=[2, 1030], dtype="float64" + ) k_tensor = paddle.static.data(name="k", shape=[1], dtype="int32") result1 = paddle.topk(input_tensor, k=2) result2 = paddle.topk(input_tensor, k=2, axis=-1) @@ -304,18 +286,29 @@ def run_static(self, place): exe = paddle.static.Executor(place) input_data = np.random.rand(10, 20).astype("float64") large_input_data = np.random.rand(2, 100).astype("float64") - paddle_result = exe.run(feed={ - "x": self.input_data, - "large_x": self.large_input_data, - "k": np.array([2]).astype("int32") - }, - fetch_list=[ - result1[0], result1[1], result2[0], - result2[1], result3[0], result3[1], - result4[0], result4[1], result5[0], - result5[1], result6[0], result6[1], - result7[0], result7[1] - ]) + paddle_result = exe.run( + feed={ + "x": self.input_data, + "large_x": self.large_input_data, + "k": np.array([2]).astype("int32"), + }, + fetch_list=[ + result1[0], + result1[1], + result2[0], + result2[1], + result3[0], + result3[1], + result4[0], + result4[1], + result5[0], + result5[1], + result6[0], + result6[1], + result7[0], + result7[1], + ], + ) numpy_result = numpy_topk(self.input_data, k=2) np.testing.assert_allclose(paddle_result[0], numpy_result[0]) np.testing.assert_allclose(paddle_result[1], numpy_result[1]) @@ -328,17 +321,15 @@ def run_static(self, place): np.testing.assert_allclose(paddle_result[4], numpy_result[0]) np.testing.assert_allclose(paddle_result[5], numpy_result[1]) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=1, - largest=False) + numpy_result = numpy_topk( + self.input_data, k=2, axis=1, largest=False + ) np.testing.assert_allclose(paddle_result[6], numpy_result[0]) np.testing.assert_allclose(paddle_result[7], numpy_result[1]) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=-1, - largest=False) + numpy_result = numpy_topk( + self.input_data, k=2, axis=-1, largest=False + ) np.testing.assert_allclose(paddle_result[8], numpy_result[0]) np.testing.assert_allclose(paddle_result[9], numpy_result[1]) diff --git a/python/paddle/fluid/tests/unittests/npu/test_transpose_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_transpose_op_npu.py index e6a457ca3e642..9503e28208eed 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_transpose_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_transpose_op_npu.py @@ -25,7 +25,6 @@ class TestTransposeOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "transpose2" @@ -55,83 +54,72 @@ def test_check_grad(self): class TestCase_ZeroDim(TestTransposeOp): - def init_shape_axis(self): self.shape = () self.axis = () -class TestCase0(TestTransposeOp): +class TestCase0(TestTransposeOp): def init_shape_axis(self): - self.shape = (100, ) - self.axis = (0, ) + self.shape = (100,) + self.axis = (0,) class TestCase1(TestTransposeOp): - def init_shape_axis(self): self.shape = (3, 4, 10) self.axis = (0, 2, 1) class TestCase2(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 3, 4, 5) self.axis = (0, 2, 3, 1) class TestCase3(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 3, 4, 5, 6) self.axis = (4, 2, 3, 1, 0) class TestCase4(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 3, 4, 5, 6, 1) self.axis = (4, 2, 3, 1, 0, 5) class TestCase5(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 16, 96) self.axis = (0, 2, 1) class TestCase6(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 10, 12, 16) self.axis = (3, 1, 2, 0) class TestCase7(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 10, 2, 16) self.axis = (0, 1, 3, 2) class TestCase8(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (0, 1, 3, 2, 4, 5, 6, 7) class TestCase9(TestTransposeOp): - def init_shape_axis(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (6, 1, 3, 5, 0, 2, 4, 7) class TestTransposeOpFP16(TestTransposeOp): - def init_dtype(self): self.dtype = np.float16 @@ -140,7 +128,6 @@ def test_check_grad(self): class TestTransposeOpInt64(TestTransposeOp): - def init_dtype(self): self.dtype = np.int64 diff --git a/python/paddle/fluid/tests/unittests/npu/test_tril_triu_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_tril_triu_op_npu.py index 2aecede417a5f..aca61989b7b63 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_tril_triu_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_tril_triu_op_npu.py @@ -23,8 +23,7 @@ class TestNPUTrilTriu(OpTest): - """ the base class of other op testcases - """ + """the base class of other op testcases""" def setUp(self): self.op_type = "tril_triu" @@ -40,9 +39,9 @@ def setUp(self): 'lower': True if self.real_op_type == 'tril' else False, } self.outputs = { - 'Out': - self.real_np_op(self.X, self.diagonal) - if self.diagonal else self.real_np_op(self.X) + 'Out': self.real_np_op(self.X, self.diagonal) + if self.diagonal + else self.real_np_op(self.X) } def test_check_output(self): @@ -67,27 +66,29 @@ def case_generator(op_type, Xshape, diagonal, expected): If arg `expercted` is 'success', it will register an Optest case and expect to pass. Otherwise, it will register an API case and check the expect failure. """ - cls_name = "{0}_{1}_shape_{2}_diag_{3}".format(expected, op_type, Xshape, - diagonal) + cls_name = "{0}_{1}_shape_{2}_diag_{3}".format( + expected, op_type, Xshape, diagonal + ) errmsg = { - "diagonal: TypeError": - "diagonal in {} must be a python Int".format(op_type), - "input: ValueError": - "x shape in {} must be at least 2-D".format(op_type), + "diagonal: TypeError": "diagonal in {} must be a python Int".format( + op_type + ), + "input: ValueError": "x shape in {} must be at least 2-D".format( + op_type + ), } class FailureCase(unittest.TestCase): - def test_failure(self): paddle.enable_static() data = fluid.data(shape=Xshape, dtype='float32', name=cls_name) - with self.assertRaisesRegexp(eval(expected.split(':')[-1]), - errmsg[expected]): + with self.assertRaisesRegexp( + eval(expected.split(':')[-1]), errmsg[expected] + ): getattr(tensor, op_type)(x=data, diagonal=diagonal) class SuccessCase(TestNPUTrilTriu): - def initTestCase(self): paddle.enable_static() @@ -112,15 +113,13 @@ def initTestCase(self): (20, 20): [ '2020', [20], - { - 20: 20 - }, + {20: 20}, (20, 20), 20.20, ], # str, list, dict, tuple, float }, 'input: ValueError': { - (2020, ): [None], + (2020,): [None], }, } for _op_type in ['tril', 'triu']: @@ -129,12 +128,15 @@ def initTestCase(self): list( map( lambda _diagonal: case_generator( - _op_type, _Xshape, _diagonal, _expected), _diaglist)) + _op_type, _Xshape, _diagonal, _expected + ), + _diaglist, + ) + ) class TestTrilTriuOpAPI(unittest.TestCase): - """ test case by using API and has -1 dimension - """ + """test case by using API and has -1 dimension""" def test_api(self): paddle.enable_static() @@ -166,8 +168,10 @@ def test_api_with_dygraph(self): with fluid.dygraph.guard(): data = np.random.random([1, 9, 9, 4]).astype(dtype) x = fluid.dygraph.to_variable(data) - tril_out, triu_out = tensor.tril(x).numpy(), tensor.triu( - x).numpy() + tril_out, triu_out = ( + tensor.tril(x).numpy(), + tensor.triu(x).numpy(), + ) np.testing.assert_allclose(tril_out, np.tril(data)) np.testing.assert_allclose(triu_out, np.triu(data)) @@ -185,14 +189,15 @@ def test_fluid_api(self): place = fluid.NPUPlace(0) exe = fluid.Executor(place) - triu_out = exe.run(fluid.default_main_program(), - feed={"x": data}, - fetch_list=[triu_out]) + triu_out = exe.run( + fluid.default_main_program(), + feed={"x": data}, + fetch_list=[triu_out], + ) # @skip_check_grad_ci(reason="[NPU does not support grad right now.") class TestNPUTrilTriu_bool(TestNPUTrilTriu): - def test_check_output(self): self.check_output_with_place(self.place) diff --git a/python/paddle/fluid/tests/unittests/npu/test_truncated_gaussian_random_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_truncated_gaussian_random_op_npu.py index 98bc517b90878..dfa70ec0b65ad 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_truncated_gaussian_random_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_truncated_gaussian_random_op_npu.py @@ -29,7 +29,6 @@ class TestTruncatedNormal(unittest.TestCase): - def _test(self, run_npu=True): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -44,12 +43,13 @@ def _test(self, run_npu=True): with paddle.static.program_guard(main_prog, startup_prog): weight_attr = paddle.framework.ParamAttr( name="linear_weight", - initializer=paddle.nn.initializer.TruncatedNormal(mean=0.0, - std=2.0)) - linear = paddle.nn.Linear(2, - 2, - weight_attr=weight_attr, - bias_attr=False) + initializer=paddle.nn.initializer.TruncatedNormal( + mean=0.0, std=2.0 + ), + ) + linear = paddle.nn.Linear( + 2, 2, weight_attr=weight_attr, bias_attr=False + ) if run_npu: place = paddle.NPUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/npu/test_uniform_random_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_uniform_random_op_npu.py index 0c3e37a54a85b..2fdf53b23a7ba 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_uniform_random_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_uniform_random_op_npu.py @@ -25,7 +25,10 @@ from paddle.fluid.op import Operator import paddle.fluid as fluid from paddle.fluid import Program, program_guard -from test_uniform_random_op import TestUniformRandomOp, TestUniformRandomOpSelectedRows +from test_uniform_random_op import ( + TestUniformRandomOp, + TestUniformRandomOpSelectedRows, +) paddle.enable_static() @@ -39,7 +42,6 @@ def output_hist(out): class TestNPUUniformRandomOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "uniform_random" @@ -53,7 +55,7 @@ def init_attrs(self): "shape": [1000, 784], "min": -5.0, "max": 10.0, - "seed": 10 + "seed": 10, } self.output_hist = output_hist @@ -73,7 +75,6 @@ def verify_output(self, outs): class TestNPUUniformRandomOpSelectedRows(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_npu(): @@ -88,12 +89,14 @@ def check_with_place(self, place): scope = core.Scope() out = scope.var("X").get_selected_rows() paddle.seed(10) - op = Operator("uniform_random", - Out="X", - shape=[1000, 784], - min=-5.0, - max=10.0, - seed=10) + op = Operator( + "uniform_random", + Out="X", + shape=[1000, 784], + min=-5.0, + max=10.0, + seed=10, + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) hist, prob = output_hist(np.array(out.get_tensor())) diff --git a/python/paddle/fluid/tests/unittests/npu/test_unsqueeze_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_unsqueeze_op_npu.py index 4861c26792277..0905e928e041e 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_unsqueeze_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_unsqueeze_op_npu.py @@ -28,7 +28,6 @@ # unsqueeze class TestUnsqueezeOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "unsqueeze" @@ -60,7 +59,6 @@ def init_attrs(self): class TestUnsqueezeOp1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (3, 40) self.axes = (0, -2) @@ -69,7 +67,6 @@ def init_test_case(self): # No axes input. class TestUnsqueezeOp2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = () @@ -78,7 +75,6 @@ def init_test_case(self): # Just part of axes be squeezed. class TestUnsqueezeOp3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (6, 5, 1, 4) self.axes = (1, -1) @@ -87,7 +83,6 @@ def init_test_case(self): # unsqueeze 2 class TestUnsqueeze2Op(OpTest): - def setUp(self): self.set_npu() self.op_type = "unsqueeze2" @@ -98,7 +93,7 @@ def setUp(self): self.init_attrs() self.outputs = { "Out": self.x.reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float32") + "XShape": np.random.random(self.ori_shape).astype("float32"), } def set_npu(self): @@ -121,7 +116,6 @@ def init_attrs(self): # Correct: There is mins axis. class TestUnsqueeze2Op1(TestUnsqueeze2Op): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -2) @@ -130,7 +124,6 @@ def init_test_case(self): # Correct: No axes input. class TestUnsqueeze2Op2(TestUnsqueeze2Op): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = () @@ -139,7 +132,6 @@ def init_test_case(self): # Correct: Just part of axes be squeezed. class TestUnsqueeze2Op3(TestUnsqueeze2Op): - def init_test_case(self): self.ori_shape = (6, 5, 1, 4) self.axes = (1, -1) diff --git a/python/paddle/fluid/tests/unittests/npu/test_unstack_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_unstack_op_npu.py index 91f2fb047fe72..5c20b083bfcb4 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_unstack_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_unstack_op_npu.py @@ -24,7 +24,6 @@ class TestUnStackOpBase(OpTest): - def initDefaultParameters(self): self.input_dim = (5, 6, 7) self.axis = 0 @@ -74,25 +73,21 @@ def test_check_grad(self): class TestStackOp3(TestUnStackOpBase): - def initParameters(self): self.axis = -1 class TestStackOp4(TestUnStackOpBase): - def initParameters(self): self.axis = -3 class TestStackOp5(TestUnStackOpBase): - def initParameters(self): self.axis = 1 class TestStackOp6(TestUnStackOpBase): - def initParameters(self): self.axis = 2 diff --git a/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_min_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_min_op_npu.py index 48df4ad454aad..48edac9ab331f 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_min_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_min_op_npu.py @@ -29,7 +29,6 @@ class TestUpdateLossScalingOpMinLossScalingBad(TestUpdateLossScalingOpBad): - def setUp(self): self.set_npu() self.op_type = "update_loss_scaling" @@ -48,14 +47,14 @@ def setUp(self): 'FoundInfinite': found_inf, 'PrevLossScaling': self.prev_loss_scaling, 'InGoodSteps': self.num_good_steps, - 'InBadSteps': self.num_bad_steps + 'InBadSteps': self.num_bad_steps, } self.outputs = { 'Out': [('out0', np.zeros_like(x))], 'LossScaling': np.array([1639.0]).astype(self.dtype), 'OutGoodSteps': self.zero_steps, - 'OutBadSteps': self.zero_steps + 'OutBadSteps': self.zero_steps, } def init(self): diff --git a/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_op_npu.py index 678e50247afc8..8d364af89e3cb 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_update_loss_scaling_op_npu.py @@ -27,7 +27,6 @@ class TestUpdateLossScalingOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "update_loss_scaling" @@ -42,14 +41,14 @@ def setUp(self): 'FoundInfinite': found_inf, 'PrevLossScaling': self.prev_loss_scaling, 'InGoodSteps': self.num_good_steps, - 'InBadSteps': self.num_bad_steps + 'InBadSteps': self.num_bad_steps, } self.outputs = { 'Out': [('out0', x)], 'LossScaling': self.prev_loss_scaling * self.incr_ratio, 'OutGoodSteps': self.zero_steps, - 'OutBadSteps': self.zero_steps + 'OutBadSteps': self.zero_steps, } def set_npu(self): @@ -75,7 +74,6 @@ def test_check_output(self): class TestUpdateLossScalingOpBad(TestUpdateLossScalingOp): - def setUp(self): self.set_npu() self.op_type = "update_loss_scaling" @@ -93,33 +91,32 @@ def setUp(self): 'FoundInfinite': found_inf, 'PrevLossScaling': self.prev_loss_scaling, 'InGoodSteps': self.num_good_steps, - 'InBadSteps': self.num_bad_steps + 'InBadSteps': self.num_bad_steps, } self.outputs = { 'Out': [('out0', np.zeros_like(x))], 'LossScaling': self.prev_loss_scaling * self.decr_ratio, 'OutGoodSteps': self.zero_steps, - 'OutBadSteps': self.zero_steps + 'OutBadSteps': self.zero_steps, } class TestUpdateLossScalingLayer(unittest.TestCase): - def loss_scaling_check(self, use_npu=True, scope=fluid.Scope()): a = fluid.data(name="a", shape=[1024, 1024], dtype='float32') b = fluid.data(name="b", shape=[512, 128], dtype='float32') x = [a, b] found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') - prev_loss_scaling = fluid.data(name="prev_loss_scaling", - shape=[1], - dtype='float32') - num_good_steps = fluid.data(name="num_good_steps", - shape=[1], - dtype='int32') - num_bad_steps = fluid.data(name="num_bad_steps", - shape=[1], - dtype='int32') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32' + ) + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32' + ) + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32' + ) a_v = np.random.random([1024, 1024]).astype('float32') b_v = np.random.random([512, 128]).astype('float32') @@ -133,33 +130,41 @@ def loss_scaling_check(self, use_npu=True, scope=fluid.Scope()): incr_ratio = 2 decr_ratio = 0.8 - result = amp_nn.update_loss_scaling(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - name="update_loss_scaling") + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling", + ) place = paddle.NPUPlace(0) if use_npu else fluid.CPUPlace() exe = fluid.Executor(place) with fluid.scope_guard(scope): exe.run(fluid.default_startup_program()) - result_v = exe.run(feed={ - 'a': a_v, - 'b': b_v, - 'found_inf': found_inf_v, - 'prev_loss_scaling': prev_loss_scaling_v, - 'num_good_steps': num_good_steps_v, - 'num_bad_steps': num_bad_steps_v - }, - fetch_list=[ - result, x, found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps - ]) + result_v = exe.run( + feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v, + }, + fetch_list=[ + result, + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + ], + ) assert np.array_equal(result_v[0], a_v) assert np.array_equal(result_v[1], b_v) assert np.array_equal(result_v[0], result_v[2]) @@ -174,15 +179,15 @@ def loss_scaling_check_inf(self, use_npu=True, scope=fluid.Scope()): b = fluid.data(name="b", shape=[512, 128], dtype='float32') x = [a, b] found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') - prev_loss_scaling = fluid.data(name="prev_loss_scaling", - shape=[1], - dtype='float32') - num_good_steps = fluid.data(name="num_good_steps", - shape=[1], - dtype='int32') - num_bad_steps = fluid.data(name="num_bad_steps", - shape=[1], - dtype='int32') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32' + ) + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32' + ) + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32' + ) a_v = np.random.random([1024, 1024]).astype('float32') b_v = np.random.random([512, 128]).astype('float32') @@ -199,33 +204,41 @@ def loss_scaling_check_inf(self, use_npu=True, scope=fluid.Scope()): incr_ratio = 2 decr_ratio = 0.8 - result = amp_nn.update_loss_scaling(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - name="update_loss_scaling") + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling", + ) place = paddle.NPUPlace(0) if use_npu else fluid.CPUPlace() exe = fluid.Executor(place) with fluid.scope_guard(scope): exe.run(fluid.default_startup_program()) - result_v = exe.run(feed={ - 'a': a_v, - 'b': b_v, - 'found_inf': found_inf_v, - 'prev_loss_scaling': prev_loss_scaling_v, - 'num_good_steps': num_good_steps_v, - 'num_bad_steps': num_bad_steps_v - }, - fetch_list=[ - result, x, found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps - ]) + result_v = exe.run( + feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v, + }, + fetch_list=[ + result, + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + ], + ) assert np.array_equal(result_v[0], np.zeros_like(a_v)) assert np.array_equal(result_v[1], np.zeros_like(b_v)) assert np.array_equal(result_v[2], np.zeros_like(a_v)) diff --git a/python/paddle/fluid/tests/unittests/npu/test_where_index_npu.py b/python/paddle/fluid/tests/unittests/npu/test_where_index_npu.py index ff159c1473f86..370a1934bff96 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_where_index_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_where_index_npu.py @@ -27,7 +27,6 @@ class TestWhereIndexOp(OpTest): - def setUp(self): self.set_npu() self.op_type = "where_index" @@ -49,7 +48,6 @@ def set_npu(self): class TestNotBool(TestWhereIndexOp): - def init_config(self): self.inputs = { 'Condition': np.array([1, 0, 8]), @@ -59,7 +57,6 @@ def init_config(self): class TestAllFalse(TestWhereIndexOp): - def init_config(self): self.inputs = { 'Condition': np.array([False, False, False]), @@ -69,7 +66,6 @@ def init_config(self): class TestRank2(TestWhereIndexOp): - def init_config(self): self.inputs = { 'Condition': np.array([[True, False], [False, True]]), @@ -79,24 +75,26 @@ def init_config(self): class TestRank3(TestWhereIndexOp): - def init_config(self): self.inputs = { - 'Condition': - np.array([[[True, False], [False, True]], - [[False, True], [True, False]], - [[False, False], [False, True]]]), + 'Condition': np.array( + [ + [[True, False], [False, True]], + [[False, True], [True, False]], + [[False, False], [False, True]], + ] + ), } self.outputs = { - 'Out': - np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], [2, 1, 1]], - dtype='int64') + 'Out': np.array( + [[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], [2, 1, 1]], + dtype='int64', + ) } class TestWhereOpError(unittest.TestCase): - def test_api(self): with program_guard(Program(), Program()): cond = fluid.layers.data(name='cond', shape=[4], dtype='bool') @@ -109,9 +107,7 @@ def test_api(self): class TestWhereRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.where([10]) diff --git a/python/paddle/fluid/tests/unittests/npu/test_where_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_where_op_npu.py index ab26d506c2fa8..9e1126d0aa043 100755 --- a/python/paddle/fluid/tests/unittests/npu/test_where_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_where_op_npu.py @@ -27,7 +27,6 @@ class TestNPUWhereOp(OpTest): - def setUp(self): self.op_type = "where" self.set_npu() @@ -52,7 +51,6 @@ def test_check_grad_normal(self): class TestNPUWhereOp2(TestNPUWhereOp): - def init_config(self): self.x = np.random.uniform(-5, 5, (60, 2)).astype("float64") self.y = np.random.uniform(-5, 5, (60, 2)).astype("float64") @@ -60,7 +58,6 @@ def init_config(self): class TestNPUWhereOp3(TestNPUWhereOp): - def init_config(self): self.x = np.random.uniform(-3, 5, (20, 2, 4)).astype("float64") self.y = np.random.uniform(-3, 5, (20, 2, 4)).astype("float64") @@ -68,7 +65,6 @@ def init_config(self): class TestNPUWhereAPI(unittest.TestCase): - def setUp(self): self.__class__.use_npu = True self.place = paddle.NPUPlace(0) @@ -93,9 +89,9 @@ def test_api(self): train_prog = fluid.Program() startup = fluid.Program() with fluid.program_guard(train_prog, startup): - cond = fluid.data(name='cond', - shape=self.shape, - dtype='bool') + cond = fluid.data( + name='cond', shape=self.shape, dtype='bool' + ) x = fluid.data(name='x', shape=self.shape, dtype='float32') y = fluid.data(name='y', shape=self.shape, dtype='float32') @@ -113,24 +109,25 @@ def test_api(self): fetch_list.append(x.grad_name) if y_stop_gradient is False: fetch_list.append(y.grad_name) - out = exe.run(train_prog, - feed={ - 'cond': self.cond, - 'x': self.x, - 'y': self.y - }, - fetch_list=fetch_list) + out = exe.run( + train_prog, + feed={'cond': self.cond, 'x': self.x, 'y': self.y}, + fetch_list=fetch_list, + ) assert np.array_equal(out[0], self.out) if x_stop_gradient is False: - assert np.array_equal(out[2], - self.ref_x_backward(out[1])) + assert np.array_equal( + out[2], self.ref_x_backward(out[1]) + ) if y.stop_gradient is False: - assert np.array_equal(out[3], - self.ref_y_backward(out[1])) + assert np.array_equal( + out[3], self.ref_y_backward(out[1]) + ) elif y.stop_gradient is False: - assert np.array_equal(out[2], - self.ref_y_backward(out[1])) + assert np.array_equal( + out[2], self.ref_y_backward(out[1]) + ) def test_api_broadcast(self, use_cuda=False): train_prog = fluid.Program() @@ -139,24 +136,21 @@ def test_api_broadcast(self, use_cuda=False): x = fluid.layers.data(name='x', shape=[4, 1], dtype='float32') y = fluid.layers.data(name='y', shape=[4, 2], dtype='float32') x_i = np.array([[0.9383, 0.1983, 3.2, 1.2]]).astype("float32") - y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, - 1.0]]).astype("float32") + y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]).astype( + "float32" + ) result = paddle.where(x > 1, x=x, y=y) exe = fluid.Executor(self.place) exe.run(startup) - out = exe.run(train_prog, - feed={ - 'x': x_i, - 'y': y_i - }, - fetch_list=[result]) + out = exe.run( + train_prog, feed={'x': x_i, 'y': y_i}, fetch_list=[result] + ) assert np.array_equal(out[0], np.where(x_i > 1, x_i, y_i)) class TestWhereDygraphAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(paddle.NPUPlace(0)): x_i = np.array([0.9383, 0.1983, 3.2, 1.2]).astype("float64") diff --git a/python/paddle/fluid/tests/unittests/npu/test_while_op_npu.py b/python/paddle/fluid/tests/unittests/npu/test_while_op_npu.py index 54e11413fe700..45aba89a84ee8 100644 --- a/python/paddle/fluid/tests/unittests/npu/test_while_op_npu.py +++ b/python/paddle/fluid/tests/unittests/npu/test_while_op_npu.py @@ -26,20 +26,16 @@ class TestWhileOp(unittest.TestCase): - def simple_net(self): - d0 = layers.data("d0", - shape=[10], - append_batch_size=False, - dtype='float32') - d1 = layers.data("d1", - shape=[10], - append_batch_size=False, - dtype='float32') - d2 = layers.data("d2", - shape=[10], - append_batch_size=False, - dtype='float32') + d0 = layers.data( + "d0", shape=[10], append_batch_size=False, dtype='float32' + ) + d1 = layers.data( + "d1", shape=[10], append_batch_size=False, dtype='float32' + ) + d2 = layers.data( + "d2", shape=[10], append_batch_size=False, dtype='float32' + ) # fill_constant npu op doesn't support int64 i = layers.zeros(shape=[1], dtype='int32') i = layers.cast(i, 'int64') @@ -107,12 +103,10 @@ def test_simple_net(self): for i in range(3): d.append(numpy.random.random(size=[10]).astype('float32')) - outs = exe.run(feed={ - 'd0': d[0], - 'd1': d[1], - 'd2': d[2] - }, - fetch_list=[sum_result]) + outs = exe.run( + feed={'d0': d[0], 'd1': d[1], 'd2': d[2]}, + fetch_list=[sum_result], + ) self.assertAlmostEqual(numpy.sum(d), numpy.sum(outs[0]), delta=0.01) def test_simple_net_forward(self): diff --git a/python/paddle/fluid/tests/unittests/op_test.py b/python/paddle/fluid/tests/unittests/op_test.py index 01a4d78890115..6147c88dc5680 100644 --- a/python/paddle/fluid/tests/unittests/op_test.py +++ b/python/paddle/fluid/tests/unittests/op_test.py @@ -27,12 +27,21 @@ import paddle.fluid as fluid from paddle.fluid.framework import _dygraph_tracer import paddle.fluid.core as core -from paddle.fluid.framework import _in_legacy_dygraph, _enable_legacy_dygraph, _in_eager_without_dygraph_check, _disable_legacy_dygraph +from paddle.fluid.framework import ( + _in_legacy_dygraph, + _enable_legacy_dygraph, + _in_eager_without_dygraph_check, + _disable_legacy_dygraph, +) from paddle.fluid.framework import _test_eager_guard from paddle.fluid.backward import append_backward from paddle.fluid.op import Operator from paddle.fluid.executor import Executor -from paddle.fluid.framework import OpProtoHolder, Program, _current_expected_place +from paddle.fluid.framework import ( + OpProtoHolder, + Program, + _current_expected_place, +) from paddle.fluid import unique_name from paddle.fluid.dygraph.dygraph_to_static.utils import parse_arg_and_kwargs @@ -54,8 +63,12 @@ # For switch new eager mode globally g_is_in_eager = _in_eager_without_dygraph_check() -g_enable_legacy_dygraph = _enable_legacy_dygraph if g_is_in_eager else lambda: None -g_disable_legacy_dygraph = _disable_legacy_dygraph if g_is_in_eager else lambda: None +g_enable_legacy_dygraph = ( + _enable_legacy_dygraph if g_is_in_eager else lambda: None +) +g_disable_legacy_dygraph = ( + _disable_legacy_dygraph if g_is_in_eager else lambda: None +) def check_out_dtype(api_fn, in_specs, expect_dtypes, target_index=0, **configs): @@ -85,12 +98,15 @@ def check_out_dtype(api_fn, in_specs, expect_dtypes, target_index=0, **configs): shape, dtype = spec else: raise ValueError( - "Value of in_specs[{}] should contains two elements: [shape, dtype]" - .format(index)) + "Value of in_specs[{}] should contains two elements: [shape, dtype]".format( + index + ) + ) input_t.append( - paddle.static.data(name='data_%s' % index, - shape=shape, - dtype=dtype)) + paddle.static.data( + name='data_%s' % index, shape=shape, dtype=dtype + ) + ) out = api_fn(*input_t, **configs) out_dtype = fluid.data_feeder.convert_dtype(out.dtype) @@ -98,7 +114,9 @@ def check_out_dtype(api_fn, in_specs, expect_dtypes, target_index=0, **configs): if out_dtype != expect_dtype: raise ValueError( "Expected out.dtype is {}, but got {} from {}.".format( - expect_dtype, out_dtype, api_fn.__name__)) + expect_dtype, out_dtype, api_fn.__name__ + ) + ) def _set_use_system_allocator(value=None): @@ -110,22 +128,25 @@ def _set_use_system_allocator(value=None): def randomize_probability(batch_size, class_num, dtype='float32'): - prob = np.random.uniform(0.1, 1.0, - size=(batch_size, class_num)).astype(dtype) + prob = np.random.uniform(0.1, 1.0, size=(batch_size, class_num)).astype( + dtype + ) prob_sum = prob.sum(axis=1) for i in range(len(prob)): prob[i] /= prob_sum[i] return prob -def get_numeric_gradient(place, - scope, - op, - inputs, - input_to_check, - output_names, - delta=0.005, - in_place=False): +def get_numeric_gradient( + place, + scope, + op, + inputs, + input_to_check, + output_names, + delta=0.005, + in_place=False, +): # FIXME: change this method by compile time concepts set_input(scope, op, inputs, place) @@ -150,9 +171,12 @@ def product(dim): elif tensor_to_check_dtype == core.VarDesc.VarType.COMPLEX128: tensor_to_check_dtype = np.complex128 else: - raise ValueError("Not supported data type " + - str(tensor_to_check_dtype) + ", tensor name : " + - str(input_to_check)) + raise ValueError( + "Not supported data type " + + str(tensor_to_check_dtype) + + ", tensor name : " + + str(input_to_check) + ) def get_output(): sum = [] @@ -167,7 +191,7 @@ def get_output(): sum.append(output_numpy.astype(tensor_to_check_dtype).mean()) return tensor_to_check_dtype(np.array(sum).sum() / len(output_names)) - gradient_flat = np.zeros(shape=(tensor_size, ), dtype=tensor_to_check_dtype) + gradient_flat = np.zeros(shape=(tensor_size,), dtype=tensor_to_check_dtype) def __get_elem__(tensor, i): if tensor_to_check_dtype == np.float16: @@ -179,15 +203,16 @@ def __get_elem__(tensor, i): numpy_tensor = numpy_tensor.flatten() return struct.unpack( ' 1 and is_np_data( - sub_val_value[1]): # case 3 + sub_val_value[1] + ): # case 3 dtype_set.add(sub_val_value[1].dtype) - elif len(sub_val_value) > 1 and isinstance(sub_val_value[1], (list, tuple)) \ - and is_np_data(sub_val_value[1][0]): # case 4 + elif ( + len(sub_val_value) > 1 + and isinstance(sub_val_value[1], (list, tuple)) + and is_np_data(sub_val_value[1][0]) + ): # case 4 dtype_set.add(sub_val_value[1][0].dtype) # infer dtype from inputs, and dtype means the precision of the test @@ -466,7 +525,7 @@ def infer_dtype(numpy_dict, dtype_set): np.dtype(np.int16), np.dtype(np.int8), np.dtype(np.uint8), - np.dtype(np.bool_) + np.dtype(np.bool_), ] # check the dtype in dtype_list in order, select the first dtype that in dtype_set for dtype in dtype_list: @@ -501,7 +560,8 @@ def feed_var(self, input_vars, place): if isinstance(self.inputs[var_name], tuple): tensor.set(self.inputs[var_name][0], place) tensor.set_recursive_sequence_lengths( - self.inputs[var_name][1]) + self.inputs[var_name][1] + ) else: tensor.set(self.inputs[var_name], place) feed_map[var_name] = tensor @@ -509,7 +569,9 @@ def feed_var(self, input_vars, place): return feed_map def _append_ops(self, block): - self.__class__.op_type = self.op_type # for ci check, please not delete it for now + self.__class__.op_type = ( + self.op_type + ) # for ci check, please not delete it for now if self.is_mkldnn_op(): self.__class__.use_mkldnn = True @@ -524,23 +586,28 @@ def _append_ops(self, block): self.output_dtype = np.uint16 else: self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs) - inputs = append_input_output(block, op_proto, self.inputs, True, - self.dtype) - outputs = append_input_output(block, op_proto, self.outputs, False, - self.dtype) + inputs = append_input_output( + block, op_proto, self.inputs, True, self.dtype + ) + outputs = append_input_output( + block, op_proto, self.outputs, False, self.dtype + ) if hasattr(self, "cache_name_list"): for name in self.cache_name_list: - inputs[name] = block.create_var(name=name, - persistable=True, - type=core.VarDesc.VarType.RAW, - stop_gradient=True) + inputs[name] = block.create_var( + name=name, + persistable=True, + type=core.VarDesc.VarType.RAW, + stop_gradient=True, + ) op = block.append_op( type=self.op_type, inputs=inputs, outputs=outputs, - attrs=copy(self.attrs) if hasattr(self, "attrs") else dict()) + attrs=copy(self.attrs) if hasattr(self, "attrs") else dict(), + ) # infer variable type and infer shape in compile-time op.desc.infer_var_type(block.desc) op.desc.infer_shape(block.desc) @@ -605,9 +672,12 @@ def lod_has_single_zero(self, lod): def lod_has_continuous_zero(self, lod): for i in range(len(lod) - 3): - if lod[i] != 0 and lod[i + - 1] == 0 and lod[i + - 2] == 0 and lod[i + 3] != 0: + if ( + lod[i] != 0 + and lod[i + 1] == 0 + and lod[i + 2] == 0 + and lod[i + 3] != 0 + ): return True return False @@ -625,8 +695,12 @@ def get_sequence_instance_size_0_input(self, lod=None, shape=None): if shape is None: shape = [12, 10] assert len(lod[0]) >= 8 - assert lod[0][0] == 0 and lod[0][1] == 0 and lod[0][-1] == 0 and lod[0][ - -2] == 0 + assert ( + lod[0][0] == 0 + and lod[0][1] == 0 + and lod[0][-1] == 0 + and lod[0][-2] == 0 + ) assert self.lod_has_single_zero(lod[0]) is True assert self.lod_has_continuous_zero(lod[0]) is True assert sum(lod[0]) == shape[0] @@ -634,9 +708,9 @@ def get_sequence_instance_size_0_input(self, lod=None, shape=None): x = np.random.uniform(0.1, 1, shape).astype('float32') return (x, lod) - def append_input_output_for_dygraph(self, op_proto, np_list, is_input, - if_return_inputs_grad_dict, block): - + def append_input_output_for_dygraph( + self, op_proto, np_list, is_input, if_return_inputs_grad_dict, block + ): def create_var(np_value, name, is_input, if_return_inputs_grad_dict): np_value_temp = np_value has_lod = False @@ -656,13 +730,16 @@ def create_var(np_value, name, is_input, if_return_inputs_grad_dict): if has_lod: v.value().get_tensor().set_recursive_sequence_lengths( - lod_temp) + lod_temp + ) else: - v = block.create_var(name=name, - dtype=np_value_temp.dtype, - type=core.VarDesc.VarType.LOD_TENSOR, - persistable=False, - stop_gradient=False) + v = block.create_var( + name=name, + dtype=np_value_temp.dtype, + type=core.VarDesc.VarType.LOD_TENSOR, + persistable=False, + stop_gradient=False, + ) return v # prepare variable for input or output @@ -676,21 +753,23 @@ def create_var(np_value, name, is_input, if_return_inputs_grad_dict): continue if name not in np_list: assert var_proto.intermediate, "{} not found".format(name) - v = block.create_var(dtype='float32', - type=core.VarDesc.VarType.LOD_TENSOR) + v = block.create_var( + dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR + ) var_dict[name].append(v) if if_return_inputs_grad_dict: inputs_grad_dict[name] = v continue if var_proto.duplicable: assert isinstance( - np_list[name], - list), "Duplicable {} should be set as list".format(name) + np_list[name], list + ), "Duplicable {} should be set as list".format(name) var_list = [] slot_name = name for (name, np_value) in np_list[name]: - v = create_var(np_value, name, is_input, - if_return_inputs_grad_dict) + v = create_var( + np_value, name, is_input, if_return_inputs_grad_dict + ) var_list.append(v) if if_return_inputs_grad_dict: inputs_grad_dict[name] = v @@ -704,8 +783,12 @@ def create_var(np_value, name, is_input, if_return_inputs_grad_dict): else: nplist_value_temp = np_list[name] name_temp = unique_name.generate("%s_out" % (name)) - v = create_var(nplist_value_temp, name_temp, is_input, - if_return_inputs_grad_dict) + v = create_var( + nplist_value_temp, + name_temp, + is_input, + if_return_inputs_grad_dict, + ) var_dict[name].append(v) if if_return_inputs_grad_dict: inputs_grad_dict[name] = v @@ -716,10 +799,10 @@ def create_var(np_value, name, is_input, if_return_inputs_grad_dict): return var_dict def _check_api_outs_by_dygraph_outs(self, api_outs, dygraph_outs, place): - """ for quick verify, here we take a simplest strategy: - 1. we only check variable in api_outs. - 2. we simply check the numpy (tensor) . - 3. we set atol and rtol as 1e-5, because they are unrelated to dtype. + """for quick verify, here we take a simplest strategy: + 1. we only check variable in api_outs. + 2. we simply check the numpy (tensor) . + 3. we set atol and rtol as 1e-5, because they are unrelated to dtype. """ for name in api_outs: np_api = np.array(api_outs[name]) @@ -729,19 +812,28 @@ def _check_api_outs_by_dygraph_outs(self, api_outs, dygraph_outs, place): np_dyg, rtol=1e-05, equal_nan=False, - err_msg='Output (' + name + ') has diff at ' + str(place) + - '\nExpect ' + str(np_dyg) + '\n' + 'But Got' + str(np_api) + - ' in class ' + self.__class__.__name__) + err_msg='Output (' + + name + + ') has diff at ' + + str(place) + + '\nExpect ' + + str(np_dyg) + + '\n' + + 'But Got' + + str(np_api) + + ' in class ' + + self.__class__.__name__, + ) def _calc_python_api_output(self, place, egr_inps=None, egr_oups=None): - """ set egr_inps and egr_oups = None if you want to create it by yourself. - """ + """set egr_inps and egr_oups = None if you want to create it by yourself.""" - def prepare_python_api_arguments(api, op_proto_ins, op_proto_attrs, - kernel_sig): - """ map from `op proto inputs and attrs` to `api input list and api attrs dict` + def prepare_python_api_arguments( + api, op_proto_ins, op_proto_attrs, kernel_sig + ): + """map from `op proto inputs and attrs` to `api input list and api attrs dict` - NOTE: the op_proto_attrs and op_proto_ins is a default dict. default value is [] + NOTE: the op_proto_attrs and op_proto_ins is a default dict. default value is [] """ class Empty: @@ -751,19 +843,19 @@ def is_empty(a): return isinstance(a, Empty) def get_default(idx, defaults): - assert not isinstance( - defaults[idx], Empty - ), "%d-th params of python api don't have default value." % idx + assert not isinstance(defaults[idx], Empty), ( + "%d-th params of python api don't have default value." % idx + ) return defaults[idx] def to_defaults_list(params, defaults): return [defaults[p] for p in params if p in defaults] def parse_attri_value(name, op_inputs, op_attrs): - """ parse true value from inputs and attrs, if there is no name passed by OpTest, return Empty - 1. if the name in op_attrs, use the op_attrs[name] - 2. if the name in op_inputs, convert the op_inputs to [type of default value] - 3. if the name not in op_attrs ans op_inputs, return Empty. (this will use the default value from python api) + """parse true value from inputs and attrs, if there is no name passed by OpTest, return Empty + 1. if the name in op_attrs, use the op_attrs[name] + 2. if the name in op_inputs, convert the op_inputs to [type of default value] + 3. if the name not in op_attrs ans op_inputs, return Empty. (this will use the default value from python api) """ if name in op_proto_attrs: return op_proto_attrs[name] @@ -771,8 +863,9 @@ def parse_attri_value(name, op_inputs, op_attrs): if len(op_inputs[name]) == 1: # why don't use numpy().item() : if the Tensor is float64, we will change it to python.float32, where we loss accuracy: [allclose_op] # why we reconstruct a tensor: because we want the tensor in cpu. - return paddle.to_tensor(op_inputs[name][0].numpy(), - place='cpu') + return paddle.to_tensor( + op_inputs[name][0].numpy(), place='cpu' + ) else: # if this is a list (test_unsqueeze2_op): we just pass it into the python api. return op_inputs[name] @@ -799,7 +892,8 @@ def parse_attri_value(name, op_inputs, op_attrs): Empty() for i in range(len(api_params) - len(api_defaults)) ] + api_defaults assert len(api_defaults) == len( - api_params), "Error happens. contack xiongkun03 to solve." + api_params + ), "Error happens. contack xiongkun03 to solve." inputs_sig, attrs_sig, outputs_sig = kernel_sig inputs_and_attrs = inputs_sig + attrs_sig input_arguments = [ @@ -815,7 +909,7 @@ def parse_attri_value(name, op_inputs, op_attrs): if arg_name in api_ignore_param_list: results.append(get_default(idx, api_defaults)) else: - if (idx_of_op_proto_arguments < len(input_arguments)): + if idx_of_op_proto_arguments < len(input_arguments): tmp = input_arguments[idx_of_op_proto_arguments] idx_of_op_proto_arguments += 1 else: @@ -838,9 +932,9 @@ def construct_output_dict_by_kernel_sig(ret_tuple, output_sig): return {a: [b] for a, b in zip(output_sig, ret_tuple)} else: # [assumption]: return multi-Tensor in a single output. such as paddle.split() - assert len( - output_sig - ) == 1, "Don't support multi-output with multi-tensor output. (May be you can use set `python_out_sig`, see `test_squeeze2_op` as a example.)" + assert ( + len(output_sig) == 1 + ), "Don't support multi-output with multi-tensor output. (May be you can use set `python_out_sig`, see `test_squeeze2_op` as a example.)" return {output_sig[0]: ret_tuple} def assumption_assert_and_transform(args, inp_num): @@ -853,29 +947,35 @@ def assumption_assert_and_transform(args, inp_num): only support "X" is list of Tensor, currently don't support other structure like dict. """ - inp_args = [[inp] if inp is None else inp - for inp in args[:inp_num]] # convert None -> [None] + inp_args = [ + [inp] if inp is None else inp for inp in args[:inp_num] + ] # convert None -> [None] for inp in inp_args: assert isinstance( inp, list ), "currently only support `X` is [Tensor], don't support other structure." - args = [inp[0] if len(inp) == 1 else inp - for inp in inp_args] + args[inp_num:] + args = [ + inp[0] if len(inp) == 1 else inp for inp in inp_args + ] + args[inp_num:] return args - def _get_kernel_signature(eager_tensor_inputs, eager_tensor_outputs, - attrs_outputs): + def _get_kernel_signature( + eager_tensor_inputs, eager_tensor_outputs, attrs_outputs + ): try: kernel_sig = _dygraph_tracer()._get_kernel_signature( - self.op_type, eager_tensor_inputs, eager_tensor_outputs, - attrs_outputs) + self.op_type, + eager_tensor_inputs, + eager_tensor_outputs, + attrs_outputs, + ) except RuntimeError as re: - """ we think the kernel_sig is missing. - """ + """we think the kernel_sig is missing.""" kernel_sig = None print( "[Warning: op_test.py] Kernel Signature is not found for %s, fall back to intermediate state." - % self.op_type) + % self.op_type + ) return kernel_sig def cal_python_api(python_api, args, kernel_sig): @@ -888,11 +988,21 @@ def cal_python_api(python_api, args, kernel_sig): block = fluid.default_main_program().global_block() op_proto = OpProtoHolder.instance().get_op_proto(self.op_type) # prepare input variable - eager_tensor_inputs = egr_inps if egr_inps else self.append_input_output_for_dygraph( - op_proto, self.inputs, True, False, block) + eager_tensor_inputs = ( + egr_inps + if egr_inps + else self.append_input_output_for_dygraph( + op_proto, self.inputs, True, False, block + ) + ) # prepare output variable - eager_tensor_outputs = egr_oups if egr_oups else self.append_input_output_for_dygraph( - op_proto, self.outputs, False, False, block) + eager_tensor_outputs = ( + egr_oups + if egr_oups + else self.append_input_output_for_dygraph( + op_proto, self.outputs, False, False, block + ) + ) # prepare attributes attrs_outputs = {} @@ -901,23 +1011,26 @@ def cal_python_api(python_api, args, kernel_sig): if self.attrs[attrs_name] is not None: attrs_outputs[attrs_name] = self.attrs[attrs_name] - kernel_sig = _get_kernel_signature(eager_tensor_inputs, - eager_tensor_outputs, - attrs_outputs) + kernel_sig = _get_kernel_signature( + eager_tensor_inputs, eager_tensor_outputs, attrs_outputs + ) if not kernel_sig: return None - assert hasattr( - self, "python_api" - ), "Detect there is KernelSignature for `%s` op, please set the `self.python_api` if you set check_eager = True" % self.op_type - args = prepare_python_api_arguments(self.python_api, - eager_tensor_inputs, - attrs_outputs, kernel_sig) + assert hasattr(self, "python_api"), ( + "Detect there is KernelSignature for `%s` op, please set the `self.python_api` if you set check_eager = True" + % self.op_type + ) + args = prepare_python_api_arguments( + self.python_api, eager_tensor_inputs, attrs_outputs, kernel_sig + ) """ we directly return the cal_python_api value because the value is already tensor. """ return cal_python_api(self.python_api, args, kernel_sig) def _calc_dygraph_output(self, place, parallel=False, no_check_set=None): - self.__class__.op_type = self.op_type # for ci check, please not delete it for now + self.__class__.op_type = ( + self.op_type + ) # for ci check, please not delete it for now with fluid.dygraph.base.guard(place=place): block = fluid.default_main_program().global_block() @@ -925,10 +1038,12 @@ def _calc_dygraph_output(self, place, parallel=False, no_check_set=None): # prepare input variable inputs = self.append_input_output_for_dygraph( - op_proto, self.inputs, True, False, block) + op_proto, self.inputs, True, False, block + ) # prepare output variable outputs = self.append_input_output_for_dygraph( - op_proto, self.outputs, False, False, block) + op_proto, self.outputs, False, False, block + ) # prepare attributes attrs_outputs = {} @@ -941,16 +1056,19 @@ def _calc_dygraph_output(self, place, parallel=False, no_check_set=None): type=self.op_type, inputs=inputs, outputs=outputs, - attrs=attrs_outputs if hasattr(self, "attrs") else None) + attrs=attrs_outputs if hasattr(self, "attrs") else None, + ) return outputs - def _calc_output(self, - place, - parallel=False, - no_check_set=None, - loss=None, - enable_inplace=None, - for_inplace_test=None): + def _calc_output( + self, + place, + parallel=False, + no_check_set=None, + loss=None, + enable_inplace=None, + for_inplace_test=None, + ): program = Program() block = program.global_block() op = self._append_ops(block) @@ -974,7 +1092,8 @@ def _calc_output(self, if isinstance(place, fluid.CUDAPlace): use_cuda = True compiled_prog = fluid.CompiledProgram(program).with_data_parallel( - loss_name=loss.name if loss else None, places=place) + loss_name=loss.name if loss else None, places=place + ) program = compiled_prog fetch_list = getattr(self, "fetch_list", []) # if the fetch_list is customized by user, we use it directly. @@ -998,14 +1117,14 @@ def _calc_output(self, build_strategy.enable_inplace = enable_inplace compiled_prog = fluid.CompiledProgram(program).with_data_parallel( - build_strategy=build_strategy, places=place) + build_strategy=build_strategy, places=place + ) program = compiled_prog executor = Executor(place) - outs = executor.run(program, - feed=feed_map, - fetch_list=fetch_list, - return_numpy=False) + outs = executor.run( + program, feed=feed_map, fetch_list=fetch_list, return_numpy=False + ) self.op = op self.program = original_program if for_inplace_test: @@ -1013,12 +1132,9 @@ def _calc_output(self, else: return outs, fetch_list - def _compare_expect_and_actual_outputs(self, - place, - fetch_list, - expect_outs, - actual_outs, - inplace_atol=None): + def _compare_expect_and_actual_outputs( + self, place, fetch_list, expect_outs, actual_outs, inplace_atol=None + ): """Compare expect outs and actual outs of an tested op. Args: @@ -1045,21 +1161,41 @@ def _compare_expect_and_actual_outputs(self, actual_out, rtol=1e-05, atol=inplace_atol, - err_msg='Output (' + name + ') has diff at ' + str(place) + - ' when using and not using inplace' + '\nExpect ' + - str(expect_out) + '\n' + 'But Got' + str(actual_out) + - ' in class ' + self.__class__.__name__) + err_msg='Output (' + + name + + ') has diff at ' + + str(place) + + ' when using and not using inplace' + + '\nExpect ' + + str(expect_out) + + '\n' + + 'But Got' + + str(actual_out) + + ' in class ' + + self.__class__.__name__, + ) else: np.testing.assert_array_equal( expect_out, actual_out, - err_msg='Output (' + name + ') has diff at ' + str(place) + - ' when using and not using inplace' + '\nExpect ' + - str(expect_out) + '\n' + 'But Got' + str(actual_out) + - ' in class ' + self.__class__.__name__ + '\n') - - def _construct_grad_program_from_forward(self, fwd_program, grad_op_desc, - op_grad_to_var): + err_msg='Output (' + + name + + ') has diff at ' + + str(place) + + ' when using and not using inplace' + + '\nExpect ' + + str(expect_out) + + '\n' + + 'But Got' + + str(actual_out) + + ' in class ' + + self.__class__.__name__ + + '\n', + ) + + def _construct_grad_program_from_forward( + self, fwd_program, grad_op_desc, op_grad_to_var + ): """Generate grad_program which contains the grad_op. Args: @@ -1077,19 +1213,23 @@ def _construct_grad_program_from_forward(self, fwd_program, grad_op_desc, grad_program._sync_with_cpp() # Create grad vars based on fwd vars (shape and dtype) - for arg in grad_op_desc.input_arg_names( - ) + grad_op_desc.output_arg_names(): + for arg in ( + grad_op_desc.input_arg_names() + grad_op_desc.output_arg_names() + ): fwd_var_name = op_grad_to_var.get(arg, None) if fwd_var_name is None: fwd_var_name = arg fwd_var = fwd_program.global_block().vars.get(fwd_var_name) assert fwd_var is not None, "{} cannot be found".format( - fwd_var_name) - grad_var = grad_block.create_var(name=arg, - dtype=fwd_var.dtype, - shape=fwd_var.shape, - type=fwd_var.type, - persistable=False) + fwd_var_name + ) + grad_var = grad_block.create_var( + name=arg, + dtype=fwd_var.dtype, + shape=fwd_var.shape, + type=fwd_var.type, + persistable=False, + ) # Some variables' tensors hold no buffer (tensor's _holder is NULL), like XShape in reshape2 op, # and the shapes of those variables contain 0 (eg. Xshape.shape = [0, 2, 5]). @@ -1100,8 +1240,9 @@ def _construct_grad_program_from_forward(self, fwd_program, grad_op_desc, grad_program._sync_with_cpp() return grad_program - def _construct_grad_feed_map_from_forward(self, place, fwd_res, - grad_op_desc, op_grad_to_var): + def _construct_grad_feed_map_from_forward( + self, place, fwd_res, grad_op_desc, op_grad_to_var + ): """Generate grad_feed_map for grad_program. since we don`t really check gradient accuracy, but check the consistency when using and not using inplace, @@ -1117,7 +1258,13 @@ def _construct_grad_feed_map_from_forward(self, place, fwd_res, Returns: grad_feed_map (dict): The feed_map of grad_op. """ - fwd_outs, fwd_fetch_list, fwd_feed_map, fwd_program, fwd_op_desc = fwd_res + ( + fwd_outs, + fwd_fetch_list, + fwd_feed_map, + fwd_program, + fwd_op_desc, + ) = fwd_res p = core.Place() p.set_place(place) grad_feed_map = {} @@ -1167,14 +1314,16 @@ def _dfs_grad_op(op_desc, fwd_op_desc=None): else: # get grad_op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - op_desc, set(), []) + op_desc, set(), [] + ) if not grad_op_desc_list: has_infer_inplace_in_grad_descendants = False else: for i, grad_op_desc in enumerate(grad_op_desc_list): - if grad_op_desc.type( - ) not in visited_ops and _dfs_grad_op( - grad_op_desc, fwd_op_desc=op_desc): + if ( + grad_op_desc.type() not in visited_ops + and _dfs_grad_op(grad_op_desc, fwd_op_desc=op_desc) + ): has_infer_inplace_in_grad_descendants = True if has_infer_inplace or has_infer_inplace_in_grad_descendants: need_run_ops.append((op_desc, fwd_op_desc)) @@ -1185,10 +1334,9 @@ def _dfs_grad_op(op_desc, fwd_op_desc=None): _dfs_grad_op(op_desc, fwd_op_desc=fwd_op_desc) return need_run_ops - def _check_forward_inplace(self, - place, - no_check_set=None, - inplace_atol=None): + def _check_forward_inplace( + self, place, no_check_set=None, inplace_atol=None + ): """Check the inplace correctness of given op (self.op_type). Run the op twice with same inputs, one enable inplace and another disable, compare their outputs. @@ -1202,27 +1350,31 @@ def _check_forward_inplace(self, We return this to construct grad_program and grad_feed_map for grad inplace check. """ # _calc_output() returns in the form tuple(outs, fetch_list, feed_map, program, op_desc) when for_inplace_test=True. - expect_res = self._calc_output(place, - no_check_set=no_check_set, - enable_inplace=False, - for_inplace_test=True) - actual_res = self._calc_output(place, - no_check_set=no_check_set, - enable_inplace=True, - for_inplace_test=True) + expect_res = self._calc_output( + place, + no_check_set=no_check_set, + enable_inplace=False, + for_inplace_test=True, + ) + actual_res = self._calc_output( + place, + no_check_set=no_check_set, + enable_inplace=True, + for_inplace_test=True, + ) # compare expect_outs and actual_outs - self._compare_expect_and_actual_outputs(place, - expect_res[1], - expect_res[0], - actual_res[0], - inplace_atol=inplace_atol) + self._compare_expect_and_actual_outputs( + place, + expect_res[1], + expect_res[0], + actual_res[0], + inplace_atol=inplace_atol, + ) return expect_res - def _calc_grad_output(self, - place, - fwd_res, - grad_op_desc, - enable_inplace=None): + def _calc_grad_output( + self, place, fwd_res, grad_op_desc, enable_inplace=None + ): """Calculate grad_output for given grad_op_desc. since we don`t really check gradient accuracy, but check the consistency when using and not using inplace, @@ -1238,13 +1390,22 @@ def _calc_grad_output(self, Returns: res (tuple(outs, fetch_list, feed_map, program, op_desc)): The results of given grad_op_desc. """ - fwd_outs, fwd_fetch_list, fwd_feed_map, fwd_program, fwd_op_desc = fwd_res + ( + fwd_outs, + fwd_fetch_list, + fwd_feed_map, + fwd_program, + fwd_op_desc, + ) = fwd_res grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - fwd_op_desc, set(), []) + fwd_op_desc, set(), [] + ) grad_program = self._construct_grad_program_from_forward( - fwd_program, grad_op_desc, op_grad_to_var) + fwd_program, grad_op_desc, op_grad_to_var + ) grad_feed_map = self._construct_grad_feed_map_from_forward( - place, fwd_res, grad_op_desc, op_grad_to_var) + place, fwd_res, grad_op_desc, op_grad_to_var + ) grad_fetch_list = grad_op_desc.output_arg_names() exe = Executor(place) program = grad_program @@ -1252,22 +1413,23 @@ def _calc_grad_output(self, build_strategy = fluid.BuildStrategy() build_strategy.enable_inplace = enable_inplace compiled_program = fluid.CompiledProgram( - grad_program).with_data_parallel(loss_name="", - build_strategy=build_strategy, - places=place) + grad_program + ).with_data_parallel( + loss_name="", build_strategy=build_strategy, places=place + ) program = compiled_program - outs = exe.run(program, - feed=grad_feed_map, - fetch_list=grad_fetch_list, - return_numpy=False) + outs = exe.run( + program, + feed=grad_feed_map, + fetch_list=grad_fetch_list, + return_numpy=False, + ) return outs, grad_fetch_list, grad_feed_map, grad_program, grad_op_desc - def _check_grad_inplace(self, - place, - fwd_res, - grad_op_desc, - inplace_atol=None): + def _check_grad_inplace( + self, place, fwd_res, grad_op_desc, inplace_atol=None + ): """Check the inplace correctness of given grad_op_desc. Run the grad op twice with same inputs, one enable inplace and another disable, compare their outputs. @@ -1285,26 +1447,25 @@ def _check_grad_inplace(self, expect_res (tuple(outs, fetch_list, feed_map, program, op_desc)): The results of given op. We return this to construct grad_program and grad_feed_map for grad inplace check. """ - expect_res = self._calc_grad_output(place, - fwd_res, - grad_op_desc, - enable_inplace=False) - actual_res = self._calc_grad_output(place, - fwd_res, - grad_op_desc, - enable_inplace=True) - - self._compare_expect_and_actual_outputs(place, - expect_res[1], - expect_res[0], - actual_res[0], - inplace_atol=inplace_atol) + expect_res = self._calc_grad_output( + place, fwd_res, grad_op_desc, enable_inplace=False + ) + actual_res = self._calc_grad_output( + place, fwd_res, grad_op_desc, enable_inplace=True + ) + + self._compare_expect_and_actual_outputs( + place, + expect_res[1], + expect_res[0], + actual_res[0], + inplace_atol=inplace_atol, + ) return expect_res - def check_inplace_output_with_place(self, - place, - no_check_set=None, - inplace_atol=None): + def check_inplace_output_with_place( + self, place, no_check_set=None, inplace_atol=None + ): """Chech the inplace correctness of given op, its grad op, its grad_grad op, etc. (1) Get all ops need to run. (see conditions in _get_need_run_ops()) @@ -1324,9 +1485,9 @@ def check_inplace_output_with_place(self, has_infer_inplace = fluid.core.has_infer_inplace(self.op_type) has_grad_op_maker = fluid.core.has_grad_op_maker(self.op_type) - fwd_res = self._calc_output(place, - no_check_set=no_check_set, - for_inplace_test=True) + fwd_res = self._calc_output( + place, no_check_set=no_check_set, for_inplace_test=True + ) op_desc = fwd_res[4] need_run_ops = self._get_need_run_ops(op_desc) @@ -1341,17 +1502,19 @@ def check_inplace_output_with_place(self, res[op_desc] = self._check_forward_inplace( place, no_check_set=no_check_set, - inplace_atol=inplace_atol) + inplace_atol=inplace_atol, + ) else: - res[op_desc] = self._calc_output(place, - no_check_set=no_check_set, - for_inplace_test=True) + res[op_desc] = self._calc_output( + place, no_check_set=no_check_set, for_inplace_test=True + ) else: # TODO(zhiqiu): enhance inplace_grad test for ops (sum and activation) using mkldnn # skip op that use_mkldnn currently flags_use_mkldnn = fluid.core.globals()["FLAGS_use_mkldnn"] attrs_use_mkldnn = hasattr(self, 'attrs') and bool( - self.attrs.get('use_mkldnn', False)) + self.attrs.get('use_mkldnn', False) + ) if flags_use_mkldnn or attrs_use_mkldnn: warnings.warn( "check inplace_grad for ops using mkldnn is not supported" @@ -1360,19 +1523,23 @@ def check_inplace_output_with_place(self, if has_infer_inplace: fwd_res = res[father_op_desc] res[op_desc] = self._check_grad_inplace( - place, fwd_res, op_desc, inplace_atol=inplace_atol) + place, fwd_res, op_desc, inplace_atol=inplace_atol + ) else: res[op_desc] = self._calc_grad_output( - place, fwd_res, op_desc) + place, fwd_res, op_desc + ) - def check_output_with_place(self, - place, - atol=0, - no_check_set=None, - equal_nan=False, - check_dygraph=True, - inplace_atol=None, - check_eager=False): + def check_output_with_place( + self, + place, + atol=0, + no_check_set=None, + equal_nan=False, + check_dygraph=True, + inplace_atol=None, + check_eager=False, + ): # disable legacy dygraph check when check_eager is True if check_eager == True: @@ -1387,26 +1554,29 @@ def find_imperative_actual(target_name, dygraph_outs, place): if var.name == target_name: return dygraph_outs[name][i] self.assertTrue( - False, "Found failed {} {}".format(dygraph_outs.keys(), - target_name)) + False, + "Found failed {} {}".format(dygraph_outs.keys(), target_name), + ) def find_actual(target_name, fetch_list): found = [ - i for i, var_name in enumerate(fetch_list) + i + for i, var_name in enumerate(fetch_list) if var_name == target_name ] self.assertTrue( - len(found) == 1, "Found {} {}".format(len(found), target_name)) + len(found) == 1, "Found {} {}".format(len(found), target_name) + ) return found[0] class Checker(object): - """ base class for check with self.outputs. - currently don't support check between checkers. + """base class for check with self.outputs. + currently don't support check between checkers. """ def __init__(self, op_test, expect_dict): - """ expect_dict is the self.outputs - support : {str: [numpy]} and {str: [(str, numpy), (str, numpy)]} + """expect_dict is the self.outputs + support : {str: [numpy]} and {str: [(str, numpy), (str, numpy)]} """ self.expects = expect_dict self.checker_name = "checker" @@ -1433,8 +1603,7 @@ def _is_skip_name(self, name): return False def find_actual_value(self, name): - """ return: (actual_tensor(var_base), actual_numpy) - """ + """return: (actual_tensor(var_base), actual_numpy)""" raise NotImplementedError("base class, not implement!") def _compare_numpy(self, name, actual_np, expect_np): @@ -1444,20 +1613,26 @@ def _compare_numpy(self, name, actual_np, expect_np): expect_np, atol=atol, rtol=self.rtol if hasattr(self, 'rtol') else 1e-5, - equal_nan=equal_nan), "Output (" + name + - ") has diff at " + str(place) + " in " + self.checker_name) + equal_nan=equal_nan, + ), + "Output (" + + name + + ") has diff at " + + str(place) + + " in " + + self.checker_name, + ) def _compare_list(self, name, actual, expect): - """ if expect is a tuple, we need to compare list. - """ + """if expect is a tuple, we need to compare list.""" raise NotImplementedError("base class, not implement!") def compare_single_output_with_expect(self, name, expect): actual, actual_np = self.find_actual_value(name) - expect_np = expect[0] \ - if isinstance(expect, tuple) else expect + expect_np = expect[0] if isinstance(expect, tuple) else expect actual_np, expect_np = self.convert_uint16_to_float_ifneed( - actual_np, expect_np) + actual_np, expect_np + ) # NOTE(zhiqiu): np.allclose([], [1.]) returns True # see details: https://stackoverflow.com/questions/38331703/why-does-numpys-broadcasting-sometimes-allow-comparing-arrays-of-different-leng if expect_np.size == 0: @@ -1468,17 +1643,20 @@ def compare_single_output_with_expect(self, name, expect): def compare_outputs_with_expects(self): for out_name, out_dup in Operator.get_op_outputs(self.op_type): - if self._is_skip_name(out_name): continue + if self._is_skip_name(out_name): + continue if out_dup: # if self.output = {'name': [(subname, Tensor), (subname, Tensor)]} sub_out = self.expects[out_name] if not isinstance(sub_out, list): - raise AssertionError("sub_out type %s is not list", - type(sub_out)) + raise AssertionError( + "sub_out type %s is not list", type(sub_out) + ) for item in sub_out: sub_out_name, expect = item[0], item[1] self.compare_single_output_with_expect( - sub_out_name, expect) + sub_out_name, expect + ) else: expect = self.expects[out_name] self.compare_single_output_with_expect(out_name, expect) @@ -1494,13 +1672,13 @@ def check(self): self.compare_outputs_with_expects() class StaticChecker(Checker): - def init(self): self.checker_name = "static checker" def calculate_output(self): outs, fetch_list = self.op_test._calc_output( - place, no_check_set=no_check_set) + place, no_check_set=no_check_set + ) self.outputs = outs self.fetch_list = fetch_list @@ -1516,13 +1694,17 @@ def convert_uint16_to_float_ifneed(self, actual_np, expect_np): return True | False """ if actual_np.dtype == np.uint16 and expect_np.dtype in [ - np.float32, np.float64 + np.float32, + np.float64, ]: actual_np = convert_uint16_to_float(actual_np) - self.rtol = 1.e-2 + self.rtol = 1.0e-2 else: - self.rtol = 1.e-5 - if expect_np.dtype == np.uint16 and actual_np.dtype == np.uint16: + self.rtol = 1.0e-5 + if ( + expect_np.dtype == np.uint16 + and actual_np.dtype == np.uint16 + ): nonlocal atol expect_np = convert_uint16_to_float(expect_np) actual_np = convert_uint16_to_float(actual_np) @@ -1530,36 +1712,40 @@ def convert_uint16_to_float_ifneed(self, actual_np, expect_np): return actual_np, expect_np def _compare_list(self, name, actual, expect): - """ if expect is a tuple, we need to compare list. - """ + """if expect is a tuple, we need to compare list.""" self.op_test.assertListEqual( - actual.recursive_sequence_lengths(), expect[1], - "Output (" + name + ") has different lod at " + str(place)) + actual.recursive_sequence_lengths(), + expect[1], + "Output (" + name + ") has different lod at " + str(place), + ) class DygraphChecker(Checker): - def init(self): self.checker_name = "dygraph checker" def calculate_output(self): self.outputs = self.op_test._calc_dygraph_output( - place, no_check_set=no_check_set) + place, no_check_set=no_check_set + ) def find_actual_value(self, name): with fluid.dygraph.base.guard(place=place): imperative_actual = find_imperative_actual( - name, self.outputs, place) + name, self.outputs, place + ) imperative_actual_t = np.array( - imperative_actual.value().get_tensor()) + imperative_actual.value().get_tensor() + ) return imperative_actual, imperative_actual_t def convert_uint16_to_float_ifneed(self, actual_np, expect_np): if actual_np.dtype == np.uint16 and expect_np.dtype in [ - np.float32, np.float64 + np.float32, + np.float64, ]: - self.rtol = 1.e-2 + self.rtol = 1.0e-2 else: - self.rtol = 1.e-5 + self.rtol = 1.0e-5 if self.op_test.is_bfloat16_op(): if actual_np.dtype == np.uint16: actual_np = convert_uint16_to_float(actual_np) @@ -1568,20 +1754,27 @@ def convert_uint16_to_float_ifneed(self, actual_np, expect_np): return actual_np, expect_np def _compare_list(self, name, actual, expect): - """ if expect is a tuple, we need to compare list. - """ + """if expect is a tuple, we need to compare list.""" with fluid.dygraph.base.guard(place=place): self.op_test.assertListEqual( - actual.value().get_tensor().recursive_sequence_lengths( - ), expect[1], - "Output (" + name + ") has different lod at " + - str(place) + " in dygraph mode") + actual.value() + .get_tensor() + .recursive_sequence_lengths(), + expect[1], + "Output (" + + name + + ") has different lod at " + + str(place) + + " in dygraph mode", + ) def _compare_numpy(self, name, actual_np, expect_np): - if functools.reduce(lambda x, y: x * y, actual_np.shape, - 1) == 0 and functools.reduce( - lambda x, y: x * y, expect_np.shape, - 1) == 0: + if ( + functools.reduce(lambda x, y: x * y, actual_np.shape, 1) + == 0 + and functools.reduce(lambda x, y: x * y, expect_np.shape, 1) + == 0 + ): pass else: self.op_test.assertTrue( @@ -1590,12 +1783,17 @@ def _compare_numpy(self, name, actual_np, expect_np): expect_np, atol=atol, rtol=self.rtol if hasattr(self, 'rtol') else 1e-5, - equal_nan=equal_nan), - "Output (" + name + ") has diff at " + str(place) + - " in " + self.checker_name) + equal_nan=equal_nan, + ), + "Output (" + + name + + ") has diff at " + + str(place) + + " in " + + self.checker_name, + ) class EagerChecker(DygraphChecker): - def init(self): self.checker_name = "eager checker" @@ -1604,12 +1802,14 @@ def calculate_output(self): with _test_eager_guard(): self.is_python_api_test = True eager_dygraph_outs = self.op_test._calc_python_api_output( - place) + place + ) if eager_dygraph_outs is None: self.is_python_api_test = False # missing KernelSignature, fall back to eager middle output. eager_dygraph_outs = self.op_test._calc_dygraph_output( - place, no_check_set=no_check_set) + place, no_check_set=no_check_set + ) self.outputs = eager_dygraph_outs def _compare_numpy(self, name, actual_np, expect_np): @@ -1619,30 +1819,35 @@ def _compare_numpy(self, name, actual_np, expect_np): def convert_uint16_to_float_ifneed(self, actual_np, expect_np): with _test_eager_guard(): return super().convert_uint16_to_float_ifneed( - actual_np, expect_np) + actual_np, expect_np + ) def find_actual_value(self, name): with _test_eager_guard(): return super().find_actual_value(name) def _compare_list(self, name, actual, expect): - """ if expect is a tuple, we need to compare list. - """ + """if expect is a tuple, we need to compare list.""" with _test_eager_guard(): super()._compare_list(name, actual, expect) def _is_skip_name(self, name): # if in final state and kernel signature don't have name, then skip it. - if self.is_python_api_test and hasattr( - self.op_test, "python_out_sig" - ) and name not in self.op_test.python_out_sig: + if ( + self.is_python_api_test + and hasattr(self.op_test, "python_out_sig") + and name not in self.op_test.python_out_sig + ): return True return super()._is_skip_name(name) # set some flags by the combination of arguments. self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs) - if self.dtype == np.float64 and \ - self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_OUTPUT_THRESHOLD_OP_LIST: + if ( + self.dtype == np.float64 + and self.op_type + not in op_threshold_white_list.NEED_FIX_FP64_CHECK_OUTPUT_THRESHOLD_OP_LIST + ): atol = 0 if self.is_bfloat16_op(): @@ -1650,7 +1855,8 @@ def _is_skip_name(self, name): check_dygraph = False check_eager = False if hasattr(self, 'force_fp32_output') and getattr( - self, 'force_fp32_output'): + self, 'force_fp32_output' + ): atol = 1e-2 else: atol = 2 @@ -1658,9 +1864,13 @@ def _is_skip_name(self, name): atol = 1e-1 if no_check_set is not None: - if self.op_type not in no_check_set_white_list.no_check_set_white_list: + if ( + self.op_type + not in no_check_set_white_list.no_check_set_white_list + ): raise AssertionError( - "no_check_set of op %s must be set to None." % self.op_type) + "no_check_set of op %s must be set to None." % self.op_type + ) static_checker = StaticChecker(self, self.outputs) static_checker.check() outs, fetch_list = static_checker.outputs, static_checker.fetch_list @@ -1691,13 +1901,15 @@ def _is_skip_name(self, name): # Check inplace for given op, its grad op, its grad_grad op, etc. # No effect on original OpTest # Currently not support ParallelExecutor on XPUPlace. - if not paddle.is_compiled_with_xpu( - ) and not paddle.is_compiled_with_npu( - ) and not paddle.is_compiled_with_mlu() and not isinstance( - place, core.CustomPlace): - self.check_inplace_output_with_place(place, - no_check_set=no_check_set, - inplace_atol=inplace_atol) + if ( + not paddle.is_compiled_with_xpu() + and not paddle.is_compiled_with_npu() + and not paddle.is_compiled_with_mlu() + and not isinstance(place, core.CustomPlace) + ): + self.check_inplace_output_with_place( + place, no_check_set=no_check_set, inplace_atol=inplace_atol + ) if check_eager: assert check_dygraph == False @@ -1708,10 +1920,10 @@ def _is_skip_name(self, name): return outs, fetch_list def check_compile_vs_runtime(self, fetch_list, fetch_outs): - def find_fetch_index(target_name, fetch_list): found = [ - i for i, var_name in enumerate(fetch_list) + i + for i, var_name in enumerate(fetch_list) if var_name == target_name ] if len(found) == 0: @@ -1719,7 +1931,8 @@ def find_fetch_index(target_name, fetch_list): else: self.assertTrue( len(found) == 1, - "Found {} {}".format(len(found), target_name)) + "Found {} {}".format(len(found), target_name), + ) return found[0] for name in self.op.desc.output_names(): @@ -1745,16 +1958,22 @@ def find_fetch_index(target_name, fetch_list): else: lod_level_compile = 0 self.assertEqual( - lod_level_compile, lod_level_runtime, - "The lod_level of Output (" + name + - ") is different between compile-time and runtime (" + - str(lod_level_compile) + " vs " + str(lod_level_runtime) + - ")") + lod_level_compile, + lod_level_runtime, + "The lod_level of Output (" + + name + + ") is different between compile-time and runtime (" + + str(lod_level_compile) + + " vs " + + str(lod_level_runtime) + + ")", + ) def _get_places(self): if self.dtype == np.float16: if core.is_compiled_with_cuda() and core.op_support_gpu( - self.op_type): + self.op_type + ): place = core.CUDAPlace(0) if core.is_float16_supported(place): return [place] @@ -1764,18 +1983,23 @@ def _get_places(self): return [] places = [fluid.CPUPlace()] cpu_only = self._cpu_only if hasattr(self, '_cpu_only') else False - if core.is_compiled_with_cuda() and core.op_support_gpu(self.op_type)\ - and not cpu_only: + if ( + core.is_compiled_with_cuda() + and core.op_support_gpu(self.op_type) + and not cpu_only + ): places.append(core.CUDAPlace(0)) return places - def check_output(self, - atol=1e-5, - no_check_set=None, - equal_nan=False, - check_dygraph=True, - inplace_atol=None, - check_eager=False): + def check_output( + self, + atol=1e-5, + no_check_set=None, + equal_nan=False, + check_dygraph=True, + inplace_atol=None, + check_eager=False, + ): # disable legacy dygraph check when check_eager is True if check_eager == True: @@ -1790,13 +2014,15 @@ def check_output(self, places = self._get_places() for place in places: - res = self.check_output_with_place(place, - atol, - no_check_set, - equal_nan, - check_dygraph, - inplace_atol, - check_eager=check_eager) + res = self.check_output_with_place( + place, + atol, + no_check_set, + equal_nan, + check_dygraph, + inplace_atol, + check_eager=check_eager, + ) if check_eager: assert check_dygraph == False outs, eager_dygraph_outs, fetch_list = res @@ -1804,7 +2030,10 @@ def check_output(self, outs, dygraph_outs, fetch_list = res else: outs, fetch_list = res - if self.op_type not in compile_vs_runtime_white_list.COMPILE_RUN_OP_WHITE_LIST: + if ( + self.op_type + not in compile_vs_runtime_white_list.COMPILE_RUN_OP_WHITE_LIST + ): self.check_compile_vs_runtime(fetch_list, outs) def check_output_customized(self, checker, custom_place=None): @@ -1823,8 +2052,14 @@ def check_output_with_place_customized(self, checker, place): outs.sort(key=len) checker(outs) - def _assert_is_close(self, numeric_grads, analytic_grads, names, - max_relative_error, msg_prefix): + def _assert_is_close( + self, + numeric_grads, + analytic_grads, + names, + max_relative_error, + msg_prefix, + ): for a, b, name in zip(numeric_grads, analytic_grads, names): # It asserts np.abs(a - b) / np.abs(a) < max_relative_error, in which # max_relative_error is 1e-7. According to the value of np.abs(a), we @@ -1834,8 +2069,11 @@ def _assert_is_close(self, numeric_grads, analytic_grads, names, # which is the same as np.abs(a - b) / np.abs(a) < max_relative_error*1e4. abs_a = np.abs(a) if abs_a.ndim > 0: - if self.dtype == np.float64 and \ - self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST: + if ( + self.dtype == np.float64 + and self.op_type + not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST + ): abs_a[abs_a < 1e-10] = 1e-3 abs_a[np.logical_and(abs_a > 1e-10, abs_a <= 1e-8)] *= 1e4 abs_a[np.logical_and(abs_a > 1e-8, abs_a <= 1e-6)] *= 1e2 @@ -1844,8 +2082,11 @@ def _assert_is_close(self, numeric_grads, analytic_grads, names, else: abs_a[abs_a < 1e-3] = 1 elif abs_a.ndim == 0: - if self.dtype == np.float64 and \ - self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST: + if ( + self.dtype == np.float64 + and self.op_type + not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST + ): if abs_a < 1e-10: abs_a = 1e-3 elif abs_a > 1e-10 and abs_a <= 1e-8: @@ -1862,10 +2103,21 @@ def _assert_is_close(self, numeric_grads, analytic_grads, names, def err_msg(): offset = np.argmax(diff_mat > max_relative_error) - return ("Operator %s error, %s variable %s (shape: %s, dtype: %s) max gradient diff %e over limit %e, " - "the first error element is %d, expected %e, but got %e.") \ - % (self.op_type, msg_prefix, name, str(a.shape), self.dtype, max_diff, max_relative_error, - offset, a.flatten()[offset], b.flatten()[offset]) + return ( + "Operator %s error, %s variable %s (shape: %s, dtype: %s) max gradient diff %e over limit %e, " + "the first error element is %d, expected %e, but got %e." + ) % ( + self.op_type, + msg_prefix, + name, + str(a.shape), + self.dtype, + max_diff, + max_relative_error, + offset, + a.flatten()[offset], + b.flatten()[offset], + ) self.assertLessEqual(max_diff, max_relative_error, err_msg()) @@ -1876,17 +2128,19 @@ def _check_grad_helper(self): if self.dtype == np.float64: self.__class__.exist_fp64_check_grad = True - def check_grad(self, - inputs_to_check, - output_names, - no_grad_set=None, - numeric_grad_delta=0.005, - in_place=False, - max_relative_error=0.005, - user_defined_grads=None, - user_defined_grad_outputs=None, - check_dygraph=True, - check_eager=False): + def check_grad( + self, + inputs_to_check, + output_names, + no_grad_set=None, + numeric_grad_delta=0.005, + in_place=False, + max_relative_error=0.005, + user_defined_grads=None, + user_defined_grad_outputs=None, + check_dygraph=True, + check_eager=False, + ): # disable legacy dygraph check when check_eager is True if check_eager == True: @@ -1895,31 +2149,35 @@ def check_grad(self, self._check_grad_helper() places = self._get_places() for place in places: - self.check_grad_with_place(place, - inputs_to_check, - output_names, - no_grad_set, - numeric_grad_delta, - in_place, - max_relative_error, - user_defined_grads, - user_defined_grad_outputs, - check_dygraph, - check_eager=check_eager) - - def check_grad_with_place(self, - place, - inputs_to_check, - output_names, - no_grad_set=None, - numeric_grad_delta=0.005, - in_place=False, - max_relative_error=0.005, - user_defined_grads=None, - user_defined_grad_outputs=None, - check_dygraph=True, - numeric_place=None, - check_eager=False): + self.check_grad_with_place( + place, + inputs_to_check, + output_names, + no_grad_set, + numeric_grad_delta, + in_place, + max_relative_error, + user_defined_grads, + user_defined_grad_outputs, + check_dygraph, + check_eager=check_eager, + ) + + def check_grad_with_place( + self, + place, + inputs_to_check, + output_names, + no_grad_set=None, + numeric_grad_delta=0.005, + in_place=False, + max_relative_error=0.005, + user_defined_grads=None, + user_defined_grad_outputs=None, + check_dygraph=True, + numeric_place=None, + check_eager=False, + ): # disable legacy dygraph check when check_eager is True if check_eager == True: @@ -1935,8 +2193,11 @@ def check_grad_with_place(self, check_dygraph = False check_eager = False - if self.dtype == np.float64 and \ - self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST: + if ( + self.dtype == np.float64 + and self.op_type + not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST + ): numeric_grad_delta = 1e-5 max_relative_error = 1e-7 @@ -1950,12 +2211,14 @@ def check_grad_with_place(self, op_attrs["use_mkldnn"] = False use_onednn = True - self.op = create_op(self.scope, - self.op_type, - op_inputs, - op_outputs, - op_attrs, - cache_list=cache_list) + self.op = create_op( + self.scope, + self.op_type, + op_inputs, + op_outputs, + op_attrs, + cache_list=cache_list, + ) if use_onednn: op_attrs["use_mkldnn"] = True @@ -1963,18 +2226,25 @@ def check_grad_with_place(self, if no_grad_set is None: no_grad_set = set() else: - if (self.op_type not in no_grad_set_white_list.NEED_TO_FIX_OP_LIST - ) and (self.op_type - not in no_grad_set_white_list.NOT_CHECK_OP_LIST) and ( - not self.is_bfloat16_op()): - raise AssertionError("no_grad_set must be None, op_type is " + - self.op_type + " Op.") + if ( + (self.op_type not in no_grad_set_white_list.NEED_TO_FIX_OP_LIST) + and ( + self.op_type not in no_grad_set_white_list.NOT_CHECK_OP_LIST + ) + and (not self.is_bfloat16_op()) + ): + raise AssertionError( + "no_grad_set must be None, op_type is " + + self.op_type + + " Op." + ) for input_to_check in inputs_to_check: set_input(self.scope, self.op, self.inputs, place) tensor_to_check = self.scope.find_var(input_to_check).get_tensor() - tensor_size = functools.reduce(lambda a, b: a * b, - tensor_to_check.shape(), 1) + tensor_size = functools.reduce( + lambda a, b: a * b, tensor_to_check.shape(), 1 + ) tensor_ndim = len(tensor_to_check.shape()) # for 0D Tensor, it's additional case for OP, so not raise error if tensor_ndim > 0 and tensor_size < 100: @@ -1987,26 +2257,34 @@ def check_grad_with_place(self, numeric_place = place numeric_grads = user_defined_grads or [ - get_numeric_gradient(numeric_place, - self.scope, - self.op, - self.inputs, - input_to_check, - output_names, - delta=numeric_grad_delta, - in_place=in_place) + get_numeric_gradient( + numeric_place, + self.scope, + self.op, + self.inputs, + input_to_check, + output_names, + delta=numeric_grad_delta, + in_place=in_place, + ) for input_to_check in inputs_to_check ] - analytic_grads = self._get_gradient(inputs_to_check, place, - output_names, no_grad_set, - user_defined_grad_outputs) + analytic_grads = self._get_gradient( + inputs_to_check, + place, + output_names, + no_grad_set, + user_defined_grad_outputs, + ) # comparison of bf16 results will happen as fp32 # loop over list of grads and convert bf16 to fp32 fp32_analytic_grads = [] for grad in analytic_grads: if grad.dtype == np.uint16: grad = convert_uint16_to_float(grad) - max_relative_error = 0.04 if max_relative_error < 0.04 else max_relative_error + max_relative_error = ( + 0.04 if max_relative_error < 0.04 else max_relative_error + ) fp32_analytic_grads.append(grad) analytic_grads = fp32_analytic_grads @@ -2014,32 +2292,50 @@ def check_grad_with_place(self, for grad in numeric_grads: if grad.dtype == np.uint16: grad = convert_uint16_to_float(grad) - max_relative_error = 0.04 if max_relative_error < 0.04 else max_relative_error + max_relative_error = ( + 0.04 if max_relative_error < 0.04 else max_relative_error + ) fp32_numeric_grads.append(grad) numeric_grads = fp32_numeric_grads - self._assert_is_close(numeric_grads, analytic_grads, inputs_to_check, - max_relative_error, - "Gradient Check On %s" % str(place)) + self._assert_is_close( + numeric_grads, + analytic_grads, + inputs_to_check, + max_relative_error, + "Gradient Check On %s" % str(place), + ) if check_dygraph: # ensure switch into legacy dygraph g_enable_legacy_dygraph() - dygraph_grad = self._get_dygraph_grad(inputs_to_check, place, - output_names, - user_defined_grad_outputs, - no_grad_set, False) + dygraph_grad = self._get_dygraph_grad( + inputs_to_check, + place, + output_names, + user_defined_grad_outputs, + no_grad_set, + False, + ) fp32_grads = [] for grad in dygraph_grad: if grad.dtype == np.uint16: grad = convert_uint16_to_float(grad) - max_relative_error = 0.03 if max_relative_error < 0.03 else max_relative_error + max_relative_error = ( + 0.03 + if max_relative_error < 0.03 + else max_relative_error + ) fp32_grads.append(grad) dygraph_grad = fp32_grads - self._assert_is_close(numeric_grads, dygraph_grad, inputs_to_check, - max_relative_error, - "Gradient Check On %s" % str(place)) + self._assert_is_close( + numeric_grads, + dygraph_grad, + inputs_to_check, + max_relative_error, + "Gradient Check On %s" % str(place), + ) # ensure switch back eager dygraph g_disable_legacy_dygraph() @@ -2047,18 +2343,31 @@ def check_grad_with_place(self, with fluid.dygraph.base.guard(place): with _test_eager_guard(): eager_dygraph_grad = self._get_dygraph_grad( - inputs_to_check, place, output_names, - user_defined_grad_outputs, no_grad_set, check_eager) + inputs_to_check, + place, + output_names, + user_defined_grad_outputs, + no_grad_set, + check_eager, + ) fp32_grads = [] for grad in eager_dygraph_grad: if grad.dtype == np.uint16: grad = convert_uint16_to_float(grad) - max_relative_error = 0.03 if max_relative_error < 0.03 else max_relative_error + max_relative_error = ( + 0.03 + if max_relative_error < 0.03 + else max_relative_error + ) fp32_grads.append(grad) eager_dygraph_grad = fp32_grads - self._assert_is_close(numeric_grads, eager_dygraph_grad, - inputs_to_check, max_relative_error, - "Gradient Check On %s" % str(place)) + self._assert_is_close( + numeric_grads, + eager_dygraph_grad, + inputs_to_check, + max_relative_error, + "Gradient Check On %s" % str(place), + ) def _find_var_in_dygraph(self, output_vars, name): if name in output_vars: @@ -2069,13 +2378,15 @@ def _find_var_in_dygraph(self, output_vars, name): if output_vars_selected.name == name: return output_vars_selected - def _get_dygraph_grad(self, - inputs_to_check, - place, - output_names, - user_defined_grad_outputs=None, - no_grad_set=None, - check_eager=False): + def _get_dygraph_grad( + self, + inputs_to_check, + place, + output_names, + user_defined_grad_outputs=None, + no_grad_set=None, + check_eager=False, + ): with fluid.dygraph.base.guard(place=place): block = fluid.default_main_program().global_block() @@ -2083,11 +2394,13 @@ def _get_dygraph_grad(self, # prepare input variable inputs, inputs_grad_dict = self.append_input_output_for_dygraph( - op_proto, self.inputs, True, True, block) + op_proto, self.inputs, True, True, block + ) # prepare output variable outputs = self.append_input_output_for_dygraph( - op_proto, self.outputs, False, False, block) + op_proto, self.outputs, False, False, block + ) # prepare attributes attrs_outputs = {} @@ -2098,37 +2411,42 @@ def _get_dygraph_grad(self, if check_eager: eager_outputs = self._calc_python_api_output( - place, inputs, outputs) + place, inputs, outputs + ) # if outputs is None, kernel sig is empty or other error is happens. if not check_eager or eager_outputs is None: block.append_op( type=self.op_type, inputs=inputs, outputs=outputs, - attrs=attrs_outputs if hasattr(self, "attrs") else None) + attrs=attrs_outputs if hasattr(self, "attrs") else None, + ) else: outputs = eager_outputs if self.dtype == np.uint16: - cast_inputs = self._find_var_in_dygraph(outputs, - output_names[0]) - cast_outputs = block.create_var(dtype="float32", - shape=cast_inputs[0].shape) - cast_op = block.append_op(inputs={"X": cast_inputs}, - outputs={"Out": cast_outputs}, - type="cast", - attrs={ - "in_dtype": - core.VarDesc.VarType.BF16, - "out_dtype": - core.VarDesc.VarType.FP32 - }) + cast_inputs = self._find_var_in_dygraph( + outputs, output_names[0] + ) + cast_outputs = block.create_var( + dtype="float32", shape=cast_inputs[0].shape + ) + cast_op = block.append_op( + inputs={"X": cast_inputs}, + outputs={"Out": cast_outputs}, + type="cast", + attrs={ + "in_dtype": core.VarDesc.VarType.BF16, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) outputs = {output_names[0]: cast_outputs} outputs_valid = {} for output_name in output_names: outputs_valid[output_name] = self._find_var_in_dygraph( - outputs, output_name) + outputs, output_name + ) if user_defined_grad_outputs is None: if len(outputs_valid) == 1: @@ -2137,13 +2455,15 @@ def _get_dygraph_grad(self, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, stop_gradient=False, - shape=[1]) + shape=[1], + ) for outputs_valid_key in outputs_valid: block.append_op( type="mean", inputs={"X": outputs_valid[outputs_valid_key]}, outputs={"Out": [loss]}, - attrs=None) + attrs=None, + ) else: avg_sum = [] for cur_loss in outputs_valid: @@ -2151,32 +2471,41 @@ def _get_dygraph_grad(self, dtype=self.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - block.append_op(type="mean", - inputs={"X": outputs_valid[cur_loss]}, - outputs={"Out": [cur_avg_loss]}, - attrs=None) + stop_gradient=False, + ) + block.append_op( + type="mean", + inputs={"X": outputs_valid[cur_loss]}, + outputs={"Out": [cur_avg_loss]}, + attrs=None, + ) avg_sum.append(cur_avg_loss) loss_sum = block.create_var( dtype=self.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, stop_gradient=False, - shape=[1]) - block.append_op(type='sum', - inputs={"X": avg_sum}, - outputs={"Out": loss_sum}, - attrs=None) + shape=[1], + ) + block.append_op( + type='sum', + inputs={"X": avg_sum}, + outputs={"Out": loss_sum}, + attrs=None, + ) loss = block.create_var( dtype=self.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, stop_gradient=False, - shape=[1]) - block.append_op(type='scale', - inputs={"X": loss_sum}, - outputs={"Out": loss}, - attrs={'scale': 1.0 / float(len(avg_sum))}) + shape=[1], + ) + block.append_op( + type='scale', + inputs={"X": loss_sum}, + outputs={"Out": loss}, + attrs={'scale': 1.0 / float(len(avg_sum))}, + ) loss.backward() fetch_list_grad = [] @@ -2193,11 +2522,12 @@ def _get_dygraph_grad(self, grad_outputs.append(paddle.to_tensor(grad_out_value)) # delete the inputs which no need to calculate grad for no_grad_val in no_grad_set: - del (inputs[no_grad_val]) + del inputs[no_grad_val] if not _in_legacy_dygraph(): - core.eager.run_backward(fluid.layers.utils.flatten(outputs), - grad_outputs, False) + core.eager.run_backward( + fluid.layers.utils.flatten(outputs), grad_outputs, False + ) grad_inputs = [] for inputs_list in inputs.values(): for inp in inputs_list: @@ -2207,7 +2537,8 @@ def _get_dygraph_grad(self, grad_inputs = paddle.grad( outputs=fluid.layers.utils.flatten(outputs), inputs=fluid.layers.utils.flatten(inputs), - grad_outputs=grad_outputs) + grad_outputs=grad_outputs, + ) return [grad.numpy() for grad in grad_inputs] @staticmethod @@ -2230,13 +2561,15 @@ def fluid_dtype_to_np_dtype(self, dtype): def np_value_to_fluid_value(input): return input - def _get_gradient(self, - input_to_check, - place, - output_names, - no_grad_set, - user_defined_grad_outputs=None, - parallel=False): + def _get_gradient( + self, + input_to_check, + place, + output_names, + no_grad_set, + user_defined_grad_outputs=None, + parallel=False, + ): prog = Program() scope = core.Scope() block = prog.global_block() @@ -2249,36 +2582,43 @@ def _get_gradient(self, if user_defined_grad_outputs is None: if self.dtype == np.uint16: cast_inputs = list(map(block.var, output_names)) - cast_outputs = block.create_var(dtype="float32", - shape=cast_inputs[0].shape) - cast_op = block.append_op(inputs={"X": cast_inputs}, - outputs={"Out": cast_outputs}, - type="cast", - attrs={ - "in_dtype": - core.VarDesc.VarType.BF16, - "out_dtype": - core.VarDesc.VarType.FP32 - }) + cast_outputs = block.create_var( + dtype="float32", shape=cast_inputs[0].shape + ) + cast_op = block.append_op( + inputs={"X": cast_inputs}, + outputs={"Out": cast_outputs}, + type="cast", + attrs={ + "in_dtype": core.VarDesc.VarType.BF16, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) cast_op.desc.infer_var_type(block.desc) cast_op.desc.infer_shape(block.desc) output_names = [cast_outputs.name] loss = append_loss_ops(block, output_names) - param_grad_list = append_backward(loss=loss, - parameter_list=input_to_check, - no_grad_set=no_grad_set) + param_grad_list = append_backward( + loss=loss, + parameter_list=input_to_check, + no_grad_set=no_grad_set, + ) fetch_list = [g for p, g in param_grad_list] else: - assert parallel is False, "unsupported parallel mode when giving custom grad outputs." + assert ( + parallel is False + ), "unsupported parallel mode when giving custom grad outputs." # user_defined_grad_outputs here are numpy arrays if not isinstance(user_defined_grad_outputs, list): user_defined_grad_outputs = [user_defined_grad_outputs] grad_outputs = [] for grad_out_value in user_defined_grad_outputs: # `presistable` is used to avoid executor create new var in local scope - var = block.create_var(shape=grad_out_value.shape, - dtype=grad_out_value.dtype, - persistable=True) + var = block.create_var( + shape=grad_out_value.shape, + dtype=grad_out_value.dtype, + persistable=True, + ) true_var = scope.var(var.name) tensor = true_var.get_tensor() tensor.set(grad_out_value, place) @@ -2287,8 +2627,9 @@ def _get_gradient(self, outputs[name] for name in outputs if name in output_names ] inputs = [inputs[name] for name in input_to_check if name in inputs] - grad_inputs = paddle.static.gradients(targets, inputs, grad_outputs, - no_grad_set) + grad_inputs = paddle.static.gradients( + targets, inputs, grad_outputs, no_grad_set + ) fetch_list = grad_inputs if parallel: @@ -2296,21 +2637,21 @@ def _get_gradient(self, if isinstance(place, fluid.CUDAPlace): use_cuda = True compiled_prog = fluid.CompiledProgram(prog).with_data_parallel( - loss_name=loss.name, places=place) + loss_name=loss.name, places=place + ) prog = compiled_prog executor = fluid.Executor(place) return list( map( np.array, - executor.run(prog, - feed_dict, - fetch_list, - scope=scope, - return_numpy=False))) + executor.run( + prog, feed_dict, fetch_list, scope=scope, return_numpy=False + ), + ) + ) class OpTestTool: - @classmethod def skip_if(cls, condition: object, reason: str): return unittest.skipIf(condition, reason) @@ -2318,12 +2659,16 @@ def skip_if(cls, condition: object, reason: str): @classmethod def skip_if_not_cpu_bf16(cls): return OpTestTool.skip_if( - not (isinstance(_current_expected_place(), core.CPUPlace) - and core.supports_bfloat16()), - "Place does not support BF16 evaluation") + not ( + isinstance(_current_expected_place(), core.CPUPlace) + and core.supports_bfloat16() + ), + "Place does not support BF16 evaluation", + ) @classmethod def skip_if_not_cpu(cls): return OpTestTool.skip_if( not isinstance(_current_expected_place(), core.CPUPlace), - "OneDNN supports only CPU for now") + "OneDNN supports only CPU for now", + ) diff --git a/python/paddle/fluid/tests/unittests/op_test_xpu.py b/python/paddle/fluid/tests/unittests/op_test_xpu.py index 82c8eb73969f1..220bd09f2cafa 100644 --- a/python/paddle/fluid/tests/unittests/op_test_xpu.py +++ b/python/paddle/fluid/tests/unittests/op_test_xpu.py @@ -22,11 +22,14 @@ from testsuite import append_loss_ops, create_op, set_input from white_list import op_threshold_white_list, no_grad_set_white_list from op_test import OpTest -from xpu.get_test_cover_info import is_empty_grad_op_type, get_xpu_op_support_types, type_dict_str_to_numpy +from xpu.get_test_cover_info import ( + is_empty_grad_op_type, + get_xpu_op_support_types, + type_dict_str_to_numpy, +) class XPUOpTest(OpTest): - @classmethod def setUpClass(cls): '''Fix random seeds to remove randomness from tests''' @@ -60,25 +63,36 @@ def _get_places(self): places = [paddle.XPUPlace(0)] return places - def check_output(self, - atol=0.001, - no_check_set=None, - equal_nan=False, - check_dygraph=True, - inplace_atol=None, - check_eager=False): + def check_output( + self, + atol=0.001, + no_check_set=None, + equal_nan=False, + check_dygraph=True, + inplace_atol=None, + check_eager=False, + ): place = paddle.XPUPlace(0) - self.check_output_with_place(place, atol, no_check_set, equal_nan, - check_dygraph, inplace_atol, check_eager) - - def check_output_with_place(self, - place, - atol=0.001, - no_check_set=None, - equal_nan=False, - check_dygraph=True, - inplace_atol=None, - check_eager=False): + self.check_output_with_place( + place, + atol, + no_check_set, + equal_nan, + check_dygraph, + inplace_atol, + check_eager, + ) + + def check_output_with_place( + self, + place, + atol=0.001, + no_check_set=None, + equal_nan=False, + check_dygraph=True, + inplace_atol=None, + check_eager=False, + ): self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs) if self.dtype == np.float64: return @@ -89,46 +103,60 @@ def check_output_with_place(self, if self.dtype == np.float16: atol = 0.1 - return super().check_output_with_place(place, atol, no_check_set, - equal_nan, check_dygraph, - inplace_atol) - - def check_grad(self, - inputs_to_check, - output_names, - no_grad_set=None, - numeric_grad_delta=0.005, - in_place=False, - max_relative_error=0.005, - user_defined_grads=None, - user_defined_grad_outputs=None, - check_dygraph=True, - numeric_place=None, - check_eager=False): + return super().check_output_with_place( + place, atol, no_check_set, equal_nan, check_dygraph, inplace_atol + ) + + def check_grad( + self, + inputs_to_check, + output_names, + no_grad_set=None, + numeric_grad_delta=0.005, + in_place=False, + max_relative_error=0.005, + user_defined_grads=None, + user_defined_grad_outputs=None, + check_dygraph=True, + numeric_place=None, + check_eager=False, + ): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, inputs_to_check, output_names, - no_grad_set, numeric_grad_delta, in_place, - max_relative_error, user_defined_grads, - user_defined_grad_outputs, check_dygraph, - numeric_place, check_eager) - - def check_grad_with_place(self, - place, - inputs_to_check, - output_names, - no_grad_set=None, - numeric_grad_delta=0.005, - in_place=False, - max_relative_error=0.005, - user_defined_grads=None, - user_defined_grad_outputs=None, - check_dygraph=True, - numeric_place=None, - check_eager=False): + self.check_grad_with_place( + place, + inputs_to_check, + output_names, + no_grad_set, + numeric_grad_delta, + in_place, + max_relative_error, + user_defined_grads, + user_defined_grad_outputs, + check_dygraph, + numeric_place, + check_eager, + ) + + def check_grad_with_place( + self, + place, + inputs_to_check, + output_names, + no_grad_set=None, + numeric_grad_delta=0.005, + in_place=False, + max_relative_error=0.005, + user_defined_grads=None, + user_defined_grad_outputs=None, + check_dygraph=True, + numeric_place=None, + check_eager=False, + ): if hasattr(self, 'op_type_need_check_grad'): xpu_version = core.get_xpu_device_version(0) - if is_empty_grad_op_type(xpu_version, self.op_type, - self.in_type_str): + if is_empty_grad_op_type( + xpu_version, self.op_type, self.in_type_str + ): self._check_grad_helper() return @@ -137,7 +165,7 @@ def check_grad_with_place(self, for ctype in cast_grad_op_types: cast_grad_op_types_np.append(type_dict_str_to_numpy[ctype]) - if (self.dtype not in cast_grad_op_types_np): + if self.dtype not in cast_grad_op_types_np: return if self.dtype == np.float64: @@ -150,51 +178,73 @@ def check_grad_with_place(self, if self.dtype == np.float16: max_relative_error = 1.0 return super().check_grad_with_place( - place, inputs_to_check, output_names, no_grad_set, - numeric_grad_delta, in_place, max_relative_error, - user_defined_grads, user_defined_grad_outputs, check_dygraph) + place, + inputs_to_check, + output_names, + no_grad_set, + numeric_grad_delta, + in_place, + max_relative_error, + user_defined_grads, + user_defined_grad_outputs, + check_dygraph, + ) a1 = self.get_grad_with_place( place, inputs_to_check, output_names, no_grad_set=no_grad_set, - user_defined_grad_outputs=user_defined_grad_outputs) + user_defined_grad_outputs=user_defined_grad_outputs, + ) a2 = self.get_grad_with_place( place, inputs_to_check, output_names, no_grad_set=no_grad_set, - user_defined_grad_outputs=user_defined_grad_outputs) + user_defined_grad_outputs=user_defined_grad_outputs, + ) a3 = self.get_grad_with_place( paddle.CPUPlace(), inputs_to_check, output_names, no_grad_set=no_grad_set, - user_defined_grad_outputs=user_defined_grad_outputs) - self._assert_is_close(a1, a2, inputs_to_check, 0.00000001, - "Gradient Check On two xpu") - self._assert_is_close(a1, a3, inputs_to_check, max_relative_error, - "Gradient Check On cpu & xpu") - - def get_grad_with_place(self, - place, - inputs_to_check, - output_names, - no_grad_set=None, - numeric_grad_delta=0.005, - in_place=False, - max_relative_error=0.005, - user_defined_grad_outputs=None, - check_dygraph=True): + user_defined_grad_outputs=user_defined_grad_outputs, + ) + self._assert_is_close( + a1, a2, inputs_to_check, 0.00000001, "Gradient Check On two xpu" + ) + self._assert_is_close( + a1, + a3, + inputs_to_check, + max_relative_error, + "Gradient Check On cpu & xpu", + ) + + def get_grad_with_place( + self, + place, + inputs_to_check, + output_names, + no_grad_set=None, + numeric_grad_delta=0.005, + in_place=False, + max_relative_error=0.005, + user_defined_grad_outputs=None, + check_dygraph=True, + ): self.scope = core.Scope() op_inputs = self.inputs if hasattr(self, "inputs") else dict() op_outputs = self.outputs if hasattr(self, "outputs") else dict() op_attrs = self.attrs if hasattr(self, "attrs") else dict() self._check_grad_helper() - if self.dtype == np.float64 and \ - self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST: + if ( + self.dtype == np.float64 + and self.op_type + not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST + ): numeric_grad_delta = 1e-5 max_relative_error = 1e-7 @@ -213,12 +263,14 @@ def get_grad_with_place(self, for mtype in mean_grad_op_types: mean_grad_op_types_np.append(type_dict_str_to_numpy[mtype]) - self.op = create_op(self.scope, - self.op_type, - op_inputs, - op_outputs, - op_attrs, - cache_list=cache_list) + self.op = create_op( + self.scope, + self.op_type, + op_inputs, + op_outputs, + op_attrs, + cache_list=cache_list, + ) if use_onednn: op_attrs["use_mkldnn"] = True @@ -226,12 +278,18 @@ def get_grad_with_place(self, if no_grad_set is None: no_grad_set = set() else: - if (self.op_type not in no_grad_set_white_list.NEED_TO_FIX_OP_LIST - ) and (self.op_type - not in no_grad_set_white_list.NOT_CHECK_OP_LIST) and ( - not self.is_bfloat16_op()): - raise AssertionError("no_grad_set must be None, op_type is " + - self.op_type + " Op.") + if ( + (self.op_type not in no_grad_set_white_list.NEED_TO_FIX_OP_LIST) + and ( + self.op_type not in no_grad_set_white_list.NOT_CHECK_OP_LIST + ) + and (not self.is_bfloat16_op()) + ): + raise AssertionError( + "no_grad_set must be None, op_type is " + + self.op_type + + " Op." + ) for input_to_check in inputs_to_check: set_input(self.scope, self.op, self.inputs, place) @@ -239,7 +297,7 @@ def get_grad_with_place(self, if not type(output_names) is list: output_names = [output_names] - if (self.dtype not in mean_grad_op_types_np): + if self.dtype not in mean_grad_op_types_np: prog = Program() block = prog.global_block() @@ -250,18 +308,18 @@ def get_grad_with_place(self, outputs = self._get_outputs(block) feed_dict = self.feed_var(inputs, place) cast_inputs = list(map(block.var, output_names)) - cast_outputs = block.create_var(dtype="float32", - shape=cast_inputs[0].shape) - cast_op = block.append_op(type="cast", - inputs={"X": cast_inputs}, - outputs={"Out": cast_outputs}, - attrs={ - "in_dtype": - convert_np_dtype_to_dtype_( - self.dtype), - "out_dtype": - core.VarDesc.VarType.FP32 - }) + cast_outputs = block.create_var( + dtype="float32", shape=cast_inputs[0].shape + ) + cast_op = block.append_op( + type="cast", + inputs={"X": cast_inputs}, + outputs={"Out": cast_outputs}, + attrs={ + "in_dtype": convert_np_dtype_to_dtype_(self.dtype), + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) cast_op.desc.infer_var_type(block.desc) cast_op.desc.infer_shape(block.desc) @@ -270,41 +328,48 @@ def get_grad_with_place(self, loss = append_loss_ops(block, output_names) loss_names = [loss.name] recast_inputs = list(map(block.var, loss_names)) - recast_loss = block.create_var(dtype=self.dtype, - shape=recast_inputs[0].shape) - - recast_op = block.append_op(type="cast", - inputs={"X": recast_inputs}, - outputs={"Out": recast_loss}, - attrs={ - "in_dtype": - core.VarDesc.VarType.FP32, - "out_dtype": - convert_np_dtype_to_dtype_( - self.dtype) - }) + recast_loss = block.create_var( + dtype=self.dtype, shape=recast_inputs[0].shape + ) + + recast_op = block.append_op( + type="cast", + inputs={"X": recast_inputs}, + outputs={"Out": recast_loss}, + attrs={ + "in_dtype": core.VarDesc.VarType.FP32, + "out_dtype": convert_np_dtype_to_dtype_(self.dtype), + }, + ) recast_op.desc.infer_var_type(block.desc) recast_op.desc.infer_shape(block.desc) - param_grad_list = append_backward(loss=recast_loss, - parameter_list=[input_to_check], - no_grad_set=no_grad_set) + param_grad_list = append_backward( + loss=recast_loss, + parameter_list=[input_to_check], + no_grad_set=no_grad_set, + ) fetch_list = [g for p, g in param_grad_list] executor = fluid.Executor(place) return list( map( np.array, - executor.run(prog, - feed_dict, - fetch_list, - scope=scope, - return_numpy=False))) + executor.run( + prog, + feed_dict, + fetch_list, + scope=scope, + return_numpy=False, + ), + ) + ) analytic_grads = self._get_gradient( inputs_to_check, place, output_names, no_grad_set, - user_defined_grad_outputs=user_defined_grad_outputs) + user_defined_grad_outputs=user_defined_grad_outputs, + ) return analytic_grads diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_dataparallel_with_pylayer.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_dataparallel_with_pylayer.py index b85caf2c966ec..d49202def5b27 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_dataparallel_with_pylayer.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_dataparallel_with_pylayer.py @@ -18,7 +18,9 @@ import numpy as np import paddle.distributed as dist from paddle.autograd import PyLayer -from paddle.distributed.fleet.utils.hybrid_parallel_util import fused_allreduce_gradients +from paddle.distributed.fleet.utils.hybrid_parallel_util import ( + fused_allreduce_gradients, +) batch = 5 in_dim = 20 @@ -26,7 +28,6 @@ class cus_tanh(PyLayer): - @staticmethod def forward(ctx, x): y = paddle.tanh(x) @@ -35,13 +36,12 @@ def forward(ctx, x): @staticmethod def backward(ctx, dy): - y, = ctx.saved_tensor() + (y,) = ctx.saved_tensor() grad = dy * (1 - paddle.square(y)) return grad class SimpleNet(paddle.nn.Layer): - def __init__(self, train_id, model_id): super(SimpleNet, self).__init__() self.w = self.create_parameter(shape=[in_dim, batch], dtype="float32") @@ -62,7 +62,6 @@ def forward(self, inputs): class TestDistTraning(unittest.TestCase): - def test_multiple_gpus(self): self.trainer_id = dist.get_rank() dist.init_parallel_env() diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check.py index bb66e117f339e..d33bde8e4060e 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check.py @@ -29,17 +29,19 @@ class SimpleNet(fluid.Layer): - def __init__(self, train_id): super(SimpleNet, self).__init__() - self.w1 = self.create_parameter(shape=[in_dim, out_dim], - dtype="float32") - self.w2 = self.create_parameter(shape=[in_dim, out_dim], - dtype="float32") + self.w1 = self.create_parameter( + shape=[in_dim, out_dim], dtype="float32" + ) + self.w2 = self.create_parameter( + shape=[in_dim, out_dim], dtype="float32" + ) self.share_net = Linear(out_dim, 10) - self.unused_param = self.create_parameter(shape=[out_dim, in_dim], - dtype="float64") + self.unused_param = self.create_parameter( + shape=[out_dim, in_dim], dtype="float64" + ) # just for test sync_params_buffers self.register_buffer("queue", paddle.randn([10, 5])) @@ -49,9 +51,10 @@ def __init__(self, train_id): self.trainer_id = train_id def forward(self, x): - is_use = (paddle.equal_all( - x, paddle.ones(shape=(batch, in_dim))).numpy()[0] - and self.trainer_id == 1) + is_use = ( + paddle.equal_all(x, paddle.ones(shape=(batch, in_dim))).numpy()[0] + and self.trainer_id == 1 + ) if is_use: tmp = paddle.matmul(x, self.w1) @@ -62,7 +65,6 @@ def forward(self, x): class TestDistTraning(unittest.TestCase): - def test_multiple_gpus(self): dist.init_parallel_env() self.trainer_id = dist.get_rank() @@ -100,10 +102,12 @@ def test_multiple_gpus(self): self.check_gradient(model_b.parameters()) # test acc gradient - w1_grad_sum = self.check_acc(model_a._layers.w1.grad, w1_grad_sum, - model_b._layers.w1.grad) - w2_grad_sum = self.check_acc(model_a._layers.w2.grad, w2_grad_sum, - model_b._layers.w2.grad) + w1_grad_sum = self.check_acc( + model_a._layers.w1.grad, w1_grad_sum, model_b._layers.w1.grad + ) + w2_grad_sum = self.check_acc( + model_a._layers.w2.grad, w2_grad_sum, model_b._layers.w2.grad + ) model_a.clear_gradients() diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check_in_eager_mode.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check_in_eager_mode.py index 506e025225d42..595aa09db2620 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check_in_eager_mode.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_gradient_check_in_eager_mode.py @@ -30,17 +30,19 @@ class SimpleNet(fluid.Layer): - def __init__(self, train_id): super(SimpleNet, self).__init__() - self.w1 = self.create_parameter(shape=[in_dim, out_dim], - dtype="float32") - self.w2 = self.create_parameter(shape=[in_dim, out_dim], - dtype="float32") + self.w1 = self.create_parameter( + shape=[in_dim, out_dim], dtype="float32" + ) + self.w2 = self.create_parameter( + shape=[in_dim, out_dim], dtype="float32" + ) self.share_net = Linear(out_dim, 10) - self.unused_param = self.create_parameter(shape=[out_dim, in_dim], - dtype="float64") + self.unused_param = self.create_parameter( + shape=[out_dim, in_dim], dtype="float64" + ) # just for test sync_params_buffers # self.register_buffer("queue", paddle.randn([10, 5])) @@ -50,9 +52,10 @@ def __init__(self, train_id): self.trainer_id = train_id def forward(self, x): - is_use = (paddle.equal_all( - x, paddle.ones(shape=(batch, in_dim))).numpy()[0] - and self.trainer_id == 1) + is_use = ( + paddle.equal_all(x, paddle.ones(shape=(batch, in_dim))).numpy()[0] + and self.trainer_id == 1 + ) if is_use: tmp = paddle.matmul(x, self.w1) @@ -63,7 +66,6 @@ def forward(self, x): class TestDistTraning(unittest.TestCase): - def test_multiple_gpus(self): self.trainer_id = dist.get_rank() with _test_eager_guard(): @@ -75,12 +77,12 @@ def test_multiple_gpus(self): state_dict = model_a.state_dict() model_b.set_state_dict(state_dict) - model_a = paddle.DataParallel(model_a, - find_unused_parameters=True, - group=self.pg) - model_b = paddle.DataParallel(model_b, - find_unused_parameters=True, - group=self.pg) + model_a = paddle.DataParallel( + model_a, find_unused_parameters=True, group=self.pg + ) + model_b = paddle.DataParallel( + model_b, find_unused_parameters=True, group=self.pg + ) ones_input = paddle.ones(shape=(batch, in_dim)) ones_input.stop_gradient = True @@ -106,12 +108,16 @@ def test_multiple_gpus(self): self.check_gradient(model_b.parameters()) # test acc gradient - w1_grad_sum = self.check_acc(model_a._layers.w1.grad, - w1_grad_sum, - model_b._layers.w1.grad) - w2_grad_sum = self.check_acc(model_a._layers.w2.grad, - w2_grad_sum, - model_b._layers.w2.grad) + w1_grad_sum = self.check_acc( + model_a._layers.w1.grad, + w1_grad_sum, + model_b._layers.w1.grad, + ) + w2_grad_sum = self.check_acc( + model_a._layers.w2.grad, + w2_grad_sum, + model_b._layers.w2.grad, + ) model_a.clear_gradients() diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_mnist.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_mnist.py index 8b07142ac2efb..694e1cf0e62a1 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_mnist.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_mnist.py @@ -23,43 +23,48 @@ class SimpleImgConvPool(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - pool_size, - pool_stride, - pool_padding=0, - pool_type='max', - global_pooling=False, - conv_stride=1, - conv_padding=0, - conv_dilation=1, - conv_groups=1, - act=None, - use_cudnn=False, - param_attr=None, - bias_attr=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + pool_size, + pool_stride, + pool_padding=0, + pool_type='max', + global_pooling=False, + conv_stride=1, + conv_padding=0, + conv_dilation=1, + conv_groups=1, + act=None, + use_cudnn=False, + param_attr=None, + bias_attr=None, + ): super(SimpleImgConvPool, self).__init__() - self._conv2d = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=conv_stride, - padding=conv_padding, - dilation=conv_dilation, - groups=conv_groups, - param_attr=None, - bias_attr=None, - use_cudnn=use_cudnn) - - self._pool2d = Pool2D(pool_size=pool_size, - pool_type=pool_type, - pool_stride=pool_stride, - pool_padding=pool_padding, - global_pooling=global_pooling, - use_cudnn=use_cudnn) + self._conv2d = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=conv_stride, + padding=conv_padding, + dilation=conv_dilation, + groups=conv_groups, + param_attr=None, + bias_attr=None, + use_cudnn=use_cudnn, + ) + + self._pool2d = Pool2D( + pool_size=pool_size, + pool_type=pool_type, + pool_stride=pool_stride, + pool_padding=pool_padding, + global_pooling=global_pooling, + use_cudnn=use_cudnn, + ) def forward(self, inputs): x = self._conv2d(inputs) @@ -68,33 +73,30 @@ def forward(self, inputs): class MNIST(fluid.dygraph.Layer): - def __init__(self): super(MNIST, self).__init__() - self._simple_img_conv_pool_1 = SimpleImgConvPool(1, - 20, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_1 = SimpleImgConvPool( + 1, 20, 5, 2, 2, act="relu" + ) - self._simple_img_conv_pool_2 = SimpleImgConvPool(20, - 50, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_2 = SimpleImgConvPool( + 20, 50, 5, 2, 2, act="relu" + ) self.pool_2_shape = 50 * 4 * 4 SIZE = 10 - scale = (2.0 / (self.pool_2_shape**2 * SIZE))**0.5 - self._fc = Linear(self.pool_2_shape, - 10, - param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.NormalInitializer( - loc=0.0, scale=scale)), - act="softmax") + scale = (2.0 / (self.pool_2_shape**2 * SIZE)) ** 0.5 + self._fc = Linear( + self.pool_2_shape, + 10, + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=scale + ) + ), + act="softmax", + ) def forward(self, inputs, label): x = self._simple_img_conv_pool_1(inputs) @@ -107,22 +109,26 @@ def forward(self, inputs, label): class TestMnist(TestParallelDyGraphRunnerBase): - def get_model(self): model = MNIST() - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=2, - drop_last=True) - opt = paddle.optimizer.Adam(learning_rate=1e-3, - parameters=model.parameters()) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=2, drop_last=True + ) + opt = paddle.optimizer.Adam( + learning_rate=1e-3, parameters=model.parameters() + ) return model, train_reader, opt def run_one_loop(self, model, opt, data): batch_size = len(data) - dy_x_data = np.array([x[0].reshape(1, 28, 28) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(batch_size, 1) + dy_x_data = np.array([x[0].reshape(1, 28, 28) for x in data]).astype( + 'float32' + ) + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(batch_size, 1) + ) img = to_variable(dy_x_data) label = to_variable(y_data) label.stop_gradient = True diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_none_var.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_none_var.py index 5713c93920f75..a46b3344333ad 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_none_var.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_none_var.py @@ -27,15 +27,18 @@ class SimpleNet(fluid.Layer): - def __init__(self): super(SimpleNet, self).__init__() - self.net_a = paddle.nn.Sequential(paddle.nn.Linear(10, 20), - paddle.nn.Linear(20, 20), - paddle.nn.Linear(20, 5)) - self.net_b = paddle.nn.Sequential(paddle.nn.Linear(10, 20), - paddle.nn.Linear(20, 20), - paddle.nn.Linear(20, 5)) + self.net_a = paddle.nn.Sequential( + paddle.nn.Linear(10, 20), + paddle.nn.Linear(20, 20), + paddle.nn.Linear(20, 5), + ) + self.net_b = paddle.nn.Sequential( + paddle.nn.Linear(10, 20), + paddle.nn.Linear(20, 20), + paddle.nn.Linear(20, 5), + ) self.step = 0 def forward(self, x): @@ -43,24 +46,23 @@ def forward(self, x): def fake_sample_reader(): - def __reader__(): for i in range(batch_num): - x_data = np.random.random_sample((10, )).astype('float32') + x_data = np.random.random_sample((10,)).astype('float32') yield x_data return __reader__ class TestSimpleNet(TestParallelDyGraphRunnerBase): - def get_model(self): model = SimpleNet() - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer def run_one_loop(self, model, optimizer, batch): diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_shared_unused_var.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_shared_unused_var.py index 851bf3b2fdc89..60b6601b8423b 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_shared_unused_var.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_shared_unused_var.py @@ -24,7 +24,6 @@ class SimpleNet(fluid.Layer): - def __init__(self): # bias is unused parameters, and it share with net_a super(SimpleNet, self).__init__() @@ -41,24 +40,23 @@ def forward(self, x): def fake_sample_reader(): - def __reader__(): for i in range(batch_num): - x_data = np.random.random_sample((10, )).astype('float32') + x_data = np.random.random_sample((10,)).astype('float32') yield x_data return __reader__ class TestSimpleNet(TestParallelDyGraphRunnerBase): - def get_model(self): model = SimpleNet() - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer def run_one_loop(self, model, optimizer, batch): diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding.py index c8e9bacad6208..f2e5ff7751c62 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding.py @@ -23,14 +23,15 @@ class SimpleNet(fluid.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_steps=20, - init_scale=0.1, - is_sparse=False, - dtype="float32"): + def __init__( + self, + hidden_size, + vocab_size, + num_steps=20, + init_scale=0.1, + is_sparse=False, + dtype="float32", + ): super(SimpleNet, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -42,28 +43,35 @@ def __init__(self, is_sparse=is_sparse, param_attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ) + ), + ) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype=dtype, default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype=dtype, default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label): x_emb = self.embedding(input) fc = fluid.layers.matmul(x_emb, self.softmax_weight) fc = fluid.layers.elementwise_add(fc, self.softmax_bias) projection = fluid.layers.reshape(fc, shape=[-1, self.vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -81,7 +89,6 @@ def forward(self, input, label): def fake_sample_reader(): - def __reader__(): for i in range(batch_num): x_data = np.arange(num_steps).astype('int64') @@ -92,20 +99,22 @@ def __reader__(): class TestSparseEmbedding(TestParallelDyGraphRunnerBase): - def get_model(self): - model = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=True) - - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + model = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=True, + ) + + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_fp64.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_fp64.py index 599c0754e7f98..d2a07868e8860 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_fp64.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_fp64.py @@ -22,14 +22,15 @@ class SimpleNet(Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_steps=20, - init_scale=0.1, - is_sparse=False, - dtype="float64"): + def __init__( + self, + hidden_size, + vocab_size, + num_steps=20, + init_scale=0.1, + is_sparse=False, + dtype="float64", + ): super(SimpleNet, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -40,26 +41,35 @@ def __init__(self, self.hidden_size, sparse=True, weight_attr=paddle.ParamAttr( - initializer=paddle.nn.initializer.Uniform(low=-init_scale, - high=init_scale))) + initializer=paddle.nn.initializer.Uniform( + low=-init_scale, high=init_scale + ) + ), + ) self.softmax_weight = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype=dtype, default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.vocab_size], dtype=dtype, default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.tmp = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype=dtype, default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label): x_emb = self.embedding(input) @@ -67,7 +77,8 @@ def forward(self, input, label): fc = paddle.add(fc, self.softmax_bias) projection = paddle.reshape(fc, shape=[-1, self.vocab_size]) loss = paddle.nn.functional.softmax_with_cross_entropy( - logits=projection, label=label, soft_label=False) + logits=projection, label=label, soft_label=False + ) loss = paddle.reshape(loss, shape=[-1, self.num_steps]) loss = paddle.mean(loss, axis=[0]) loss = paddle.sum(loss) @@ -85,7 +96,6 @@ def forward(self, input, label): def fake_sample_reader(): - def __reader__(): for i in range(batch_num): x_data = np.arange(num_steps).astype('int64') @@ -96,20 +106,22 @@ def __reader__(): class TestSparseEmbeddingFP64(TestParallelDyGraphRunnerBase): - def get_model(self): - model = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=True) - - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + model = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=True, + ) + + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_over_height.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_over_height.py index 7bd30deea6407..9d70390b1356e 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_over_height.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_sparse_embedding_over_height.py @@ -14,7 +14,11 @@ import paddle import paddle.fluid as fluid -from parallel_dygraph_sparse_embedding import SimpleNet, fake_sample_reader, TestSparseEmbedding +from parallel_dygraph_sparse_embedding import ( + SimpleNet, + fake_sample_reader, + TestSparseEmbedding, +) from test_dist_base import runtime_main @@ -29,20 +33,22 @@ class TestSparseEmbeddingOverHeight(TestSparseEmbedding): - def get_model(self): - model = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=True) - - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - - optimizer = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=model.parameters()) + model = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=True, + ) + + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + + optimizer = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=model.parameters() + ) return model, train_reader, optimizer diff --git a/python/paddle/fluid/tests/unittests/parallel_dygraph_unused_variables.py b/python/paddle/fluid/tests/unittests/parallel_dygraph_unused_variables.py index bcd8d765a27f3..1c8c30b856d82 100644 --- a/python/paddle/fluid/tests/unittests/parallel_dygraph_unused_variables.py +++ b/python/paddle/fluid/tests/unittests/parallel_dygraph_unused_variables.py @@ -20,14 +20,15 @@ class SimpleNet(Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_steps=20, - init_scale=0.1, - is_sparse=False, - dtype="float32"): + def __init__( + self, + hidden_size, + vocab_size, + num_steps=20, + init_scale=0.1, + is_sparse=False, + dtype="float32", + ): super(SimpleNet, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -38,27 +39,36 @@ def __init__(self, self.hidden_size, sparse=is_sparse, weight_attr=paddle.ParamAttr( - initializer=paddle.nn.initializer.Uniform(low=-init_scale, - high=init_scale))) + initializer=paddle.nn.initializer.Uniform( + low=-init_scale, high=init_scale + ) + ), + ) self.softmax_weight = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype=dtype, default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.vocab_size], dtype=dtype, default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) # add tmp var self.tmp = self.create_parameter( attr=paddle.ParamAttr(), shape=[self.vocab_size], dtype=dtype, default_initializer=paddle.nn.initializer.Uniform( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label): x_emb = self.embedding(input) @@ -69,7 +79,8 @@ def forward(self, input, label): fc = paddle.add(fc, self.softmax_bias) projection = paddle.reshape(fc, shape=[-1, self.vocab_size]) loss = paddle.nn.functional.softmax_with_cross_entropy( - logits=projection, label=label, soft_label=False) + logits=projection, label=label, soft_label=False + ) loss = paddle.reshape(loss, shape=[-1, self.num_steps]) loss = paddle.mean(loss, axis=[0]) loss = paddle.sum(loss) @@ -87,7 +98,6 @@ def forward(self, input, label): def fake_sample_reader(): - def __reader__(): for i in range(batch_num): x_data = np.arange(num_steps).astype('int64') @@ -98,20 +108,22 @@ def __reader__(): class TestSparseEmbeddingUnusedVars(TestParallelDyGraphRunnerBase): - def get_model(self): - model = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=False) - - train_reader = paddle.batch(fake_sample_reader(), - batch_size=batch_size, - drop_last=True) - - optimizer = paddle.optimizer.SGD(learning_rate=0.001, - parameters=model.parameters()) + model = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=False, + ) + + train_reader = paddle.batch( + fake_sample_reader(), batch_size=batch_size, drop_last=True + ) + + optimizer = paddle.optimizer.SGD( + learning_rate=0.001, parameters=model.parameters() + ) return model, train_reader, optimizer diff --git a/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py b/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py index 2d467ef50ef3e..88caad052ad2c 100644 --- a/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py +++ b/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py @@ -30,35 +30,37 @@ class TestParallelExecutorBase(unittest.TestCase): - @classmethod - def check_network_convergence(cls, - method, - use_device=DeviceType.CUDA, - iter=5, - batch_size=None, - feed_dict=None, - feed_data_reader=None, - get_data_from_feeder=None, - use_parallel_executor=True, - use_reduce=False, - use_ir_memory_optimize=False, - enable_inplace=True, - fuse_elewise_add_act_ops=False, - fuse_all_optimizer_ops=False, - fuse_all_reduce_ops=False, - fuse_relu_depthwise_conv=False, - optimizer=fluid.optimizer.Adam, - use_fast_executor=False, - enable_sequential_execution=False): - + def check_network_convergence( + cls, + method, + use_device=DeviceType.CUDA, + iter=5, + batch_size=None, + feed_dict=None, + feed_data_reader=None, + get_data_from_feeder=None, + use_parallel_executor=True, + use_reduce=False, + use_ir_memory_optimize=False, + enable_inplace=True, + fuse_elewise_add_act_ops=False, + fuse_all_optimizer_ops=False, + fuse_all_reduce_ops=False, + fuse_relu_depthwise_conv=False, + optimizer=fluid.optimizer.Adam, + use_fast_executor=False, + enable_sequential_execution=False, + ): def run_executor(exe, binary, feed, fetch_list): if feed_data_reader is None: res = exe.run(binary, feed=feed, fetch_list=fetch_list) else: - res = exe.run(binary, - feed=feed_data_reader.get_next(exe, binary), - fetch_list=fetch_list) + res = exe.run( + binary, + feed=feed_data_reader.get_next(exe, binary), + fetch_list=fetch_list, + ) return res if feed_data_reader is not None: @@ -72,63 +74,79 @@ def run_executor(exe, binary, feed, fetch_list): startup = fluid.Program() with fluid.program_guard(main, startup): - feed_dict, loss = cls.build_model(feed_dict, get_data_from_feeder, - main, method, optimizer) + feed_dict, loss = cls.build_model( + feed_dict, get_data_from_feeder, main, method, optimizer + ) - place = fluid.CUDAPlace( - 0) if use_device == DeviceType.CUDA else fluid.XPUPlace( - 0) if use_device == DeviceType.XPU else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if use_device == DeviceType.CUDA + else fluid.XPUPlace(0) + if use_device == DeviceType.XPU + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(startup) build_strategy, exec_strategy = cls.set_strategy( - enable_inplace, enable_sequential_execution, fuse_all_optimizer_ops, - fuse_all_reduce_ops, fuse_elewise_add_act_ops, - fuse_relu_depthwise_conv, use_fast_executor, use_ir_memory_optimize, - use_reduce, use_device) + enable_inplace, + enable_sequential_execution, + fuse_all_optimizer_ops, + fuse_all_reduce_ops, + fuse_elewise_add_act_ops, + fuse_relu_depthwise_conv, + use_fast_executor, + use_ir_memory_optimize, + use_reduce, + use_device, + ) if use_parallel_executor: binary = compiler.CompiledProgram(main).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy, - exec_strategy=exec_strategy) + exec_strategy=exec_strategy, + ) else: binary = main if batch_size is not None: - batch_size *= fluid.core.get_cuda_device_count( - ) if use_device == DeviceType.CUDA else fluid.core.get_xpu_device_count( - ) if use_device == DeviceType.XPU else int( - os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + batch_size *= ( + fluid.core.get_cuda_device_count() + if use_device == DeviceType.CUDA + else fluid.core.get_xpu_device_count() + if use_device == DeviceType.XPU + else int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + ) area_below_loss = 0 begin = time.time() - first_loss, = run_executor(exe=exe, - binary=binary, - feed=feed_dict, - fetch_list=[loss.name]) + (first_loss,) = run_executor( + exe=exe, binary=binary, feed=feed_dict, fetch_list=[loss.name] + ) area_below_loss += 0.5 * first_loss.mean() for _ in range(iter): - mid_loss = run_executor(exe=exe, - binary=binary, - feed=feed_dict, - fetch_list=[loss.name]) + mid_loss = run_executor( + exe=exe, binary=binary, feed=feed_dict, fetch_list=[loss.name] + ) area_below_loss += mid_loss[0].mean() - last_loss, = run_executor(exe=exe, - binary=binary, - feed=feed_dict, - fetch_list=[loss.name]) + (last_loss,) = run_executor( + exe=exe, binary=binary, feed=feed_dict, fetch_list=[loss.name] + ) area_below_loss += 0.5 * last_loss.mean() end = time.time() if batch_size is not None: - print("%.4f Instance per second" % ((batch_size * iter + 2) / - (end - begin))) + print( + "%.4f Instance per second" + % ((batch_size * iter + 2) / (end - begin)) + ) avg_last_loss_val = np.array(last_loss).mean() avg_first_loss_val = np.array(first_loss).mean() if math.isnan(float(avg_last_loss_val)) or math.isnan( - float(avg_first_loss_val)): + float(avg_first_loss_val) + ): sys.exit("got NaN loss, training failed.") print(first_loss, last_loss, area_below_loss) @@ -136,59 +154,85 @@ def run_executor(exe, binary, feed, fetch_list): return first_loss, last_loss, area_below_loss @classmethod - def check_pass_conflict(cls, - method, - use_device=DeviceType.CUDA, - feed_dict=None, - get_data_from_feeder=None, - use_reduce=False, - use_ir_memory_optimize=True, - enable_inplace=True, - fuse_elewise_add_act_ops=False, - fuse_all_optimizer_ops=False, - fuse_all_reduce_ops=False, - fuse_relu_depthwise_conv=False, - optimizer=fluid.optimizer.Adam, - use_fast_executor=True, - enable_sequential_execution=False): + def check_pass_conflict( + cls, + method, + use_device=DeviceType.CUDA, + feed_dict=None, + get_data_from_feeder=None, + use_reduce=False, + use_ir_memory_optimize=True, + enable_inplace=True, + fuse_elewise_add_act_ops=False, + fuse_all_optimizer_ops=False, + fuse_all_reduce_ops=False, + fuse_relu_depthwise_conv=False, + optimizer=fluid.optimizer.Adam, + use_fast_executor=True, + enable_sequential_execution=False, + ): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): - feed_dict, loss = cls.build_model(feed_dict, get_data_from_feeder, - main, method, optimizer) + feed_dict, loss = cls.build_model( + feed_dict, get_data_from_feeder, main, method, optimizer + ) - place = fluid.CUDAPlace( - 0) if use_device == DeviceType.CUDA else fluid.XPUPlace( - 0) if use_device == DeviceType.XPU else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if use_device == DeviceType.CUDA + else fluid.XPUPlace(0) + if use_device == DeviceType.XPU + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(startup) build_strategy, exec_strategy = cls.set_strategy( - enable_inplace, enable_sequential_execution, fuse_all_optimizer_ops, - fuse_all_reduce_ops, fuse_elewise_add_act_ops, - fuse_relu_depthwise_conv, use_fast_executor, use_ir_memory_optimize, - use_reduce, use_device) + enable_inplace, + enable_sequential_execution, + fuse_all_optimizer_ops, + fuse_all_reduce_ops, + fuse_elewise_add_act_ops, + fuse_relu_depthwise_conv, + use_fast_executor, + use_ir_memory_optimize, + use_reduce, + use_device, + ) binary = compiler.CompiledProgram(main).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy, - exec_strategy=exec_strategy) + exec_strategy=exec_strategy, + ) exe.run(binary, feed=feed_dict, fetch_list=[loss.name]) @classmethod - def set_strategy(cls, enable_inplace, enable_sequential_execution, - fuse_all_optimizer_ops, fuse_all_reduce_ops, - fuse_elewise_add_act_ops, fuse_relu_depthwise_conv, - use_fast_executor, use_ir_memory_optimize, use_reduce, - use_device): + def set_strategy( + cls, + enable_inplace, + enable_sequential_execution, + fuse_all_optimizer_ops, + fuse_all_reduce_ops, + fuse_elewise_add_act_ops, + fuse_relu_depthwise_conv, + use_fast_executor, + use_ir_memory_optimize, + use_reduce, + use_device, + ): exec_strategy = fluid.ExecutionStrategy() if use_fast_executor: exec_strategy.use_experimental_executor = True build_strategy = fluid.BuildStrategy() - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce \ - if use_reduce else fluid.BuildStrategy.ReduceStrategy.AllReduce + build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + if use_reduce + else fluid.BuildStrategy.ReduceStrategy.AllReduce + ) build_strategy.fuse_elewise_add_act_ops = fuse_elewise_add_act_ops build_strategy.fuse_relu_depthwise_conv = fuse_relu_depthwise_conv build_strategy.fuse_all_optimizer_ops = fuse_all_optimizer_ops @@ -210,8 +254,9 @@ def set_strategy(cls, enable_inplace, enable_sequential_execution, return build_strategy, exec_strategy @classmethod - def build_model(cls, feed_dict, get_data_from_feeder, main, method, - optimizer): + def build_model( + cls, feed_dict, get_data_from_feeder, main, method, optimizer + ): loss = method(use_feed=feed_dict is not None) # NOTE(zjl): memory_optimize/inplace pass would not require # that loss.persistable = True. diff --git a/python/paddle/fluid/tests/unittests/ps/dataset_generator_A.py b/python/paddle/fluid/tests/unittests/ps/dataset_generator_A.py index 1ab4b3580d6ad..a53da27593907 100755 --- a/python/paddle/fluid/tests/unittests/ps/dataset_generator_A.py +++ b/python/paddle/fluid/tests/unittests/ps/dataset_generator_A.py @@ -23,7 +23,6 @@ class CriteoDataset(dg.MultiSlotDataGenerator): - def generate_sample(self, line): """ Read the data line by line and process it as a dictionary @@ -38,7 +37,8 @@ def reader(): sparse_feature = [] for idx in categorical_range_: sparse_feature.append( - [hash(str(idx) + features[idx]) % hash_dim_]) + [hash(str(idx) + features[idx]) % hash_dim_] + ) for idx in categorical_range_: feature_name.append("C" + str(idx - 13)) yield list(zip(feature_name, sparse_feature)) diff --git a/python/paddle/fluid/tests/unittests/ps/dataset_generator_B.py b/python/paddle/fluid/tests/unittests/ps/dataset_generator_B.py index 76b2468592dff..77178ee4c333d 100755 --- a/python/paddle/fluid/tests/unittests/ps/dataset_generator_B.py +++ b/python/paddle/fluid/tests/unittests/ps/dataset_generator_B.py @@ -23,7 +23,6 @@ class CriteoDataset(dg.MultiSlotDataGenerator): - def generate_sample(self, line): """ Read the data line by line and process it as a dictionary @@ -40,8 +39,9 @@ def reader(): dense_feature.append(0.0) else: dense_feature.append( - (float(features[idx]) - cont_min_[idx - 1]) / - cont_diff_[idx - 1]) + (float(features[idx]) - cont_min_[idx - 1]) + / cont_diff_[idx - 1] + ) label = [int(features[0])] feature_name = ["dense_feature"] feature_name.append("label") diff --git a/python/paddle/fluid/tests/unittests/ps/fl_ps_trainer.py b/python/paddle/fluid/tests/unittests/ps/fl_ps_trainer.py index aebf38a458538..66e6b1cb62e0b 100755 --- a/python/paddle/fluid/tests/unittests/ps/fl_ps_trainer.py +++ b/python/paddle/fluid/tests/unittests/ps/fl_ps_trainer.py @@ -44,16 +44,22 @@ def get_dataset(inputs, config, pipe_cmd, role="worker"): def fl_ps_train(): # 0. get role import paddle.distributed.fleet.base.role_maker as role_maker + role_maker = role_maker.PaddleCloudRoleMaker() role_maker._generate_role() fleet.util._set_role_maker(role_maker) # 1. load yaml-config to dict-config - from ps_dnn_trainer import YamlHelper, StaticModel, get_user_defined_strategy + from ps_dnn_trainer import ( + YamlHelper, + StaticModel, + get_user_defined_strategy, + ) + yaml_helper = YamlHelper() config_yaml_path = '../ps/fl_async_ps_config.yaml' config = yaml_helper.load_yaml(config_yaml_path) - #yaml_helper.print_yaml(config) + # yaml_helper.print_yaml(config) # 2. get static model paddle.enable_static() @@ -67,18 +73,25 @@ def fl_ps_train(): a_sync_configs = user_defined_strategy.a_sync_configs a_sync_configs["launch_barrier"] = True user_defined_strategy.a_sync_configs = a_sync_configs - print("launch_barrier: ", - user_defined_strategy.a_sync_configs["launch_barrier"]) + print( + "launch_barrier: ", + user_defined_strategy.a_sync_configs["launch_barrier"], + ) learning_rate = config.get("hyper_parameters.optimizer.learning_rate") inner_optimizer = paddle.optimizer.Adam(learning_rate, lazy_mode=True) - from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ParameterServerOptimizer + from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ( + ParameterServerOptimizer, + ) + ps_optimizer = ParameterServerOptimizer(inner_optimizer) - ps_optimizer._set_basic_info(loss, role_maker, inner_optimizer, - user_defined_strategy) + ps_optimizer._set_basic_info( + loss, role_maker, inner_optimizer, user_defined_strategy + ) ps_optimizer.minimize_impl(loss) # 4. runtime from paddle.distributed.ps.the_one_ps import TheOnePSRuntime + _runtime_handle = TheOnePSRuntime() # ps 目录下重构版的 TheOnePSRuntime _runtime_handle._set_basic_info(ps_optimizer.pass_ctx._attrs) epoch_num = int(config.get('runner.epoch_num')) @@ -94,21 +107,29 @@ def fl_ps_train(): _runtime_handle._init_worker() print('trainer get dataset') inputs = feeds_list[1:-1] - dataset, file_list = get_dataset(inputs, config, - "python dataset_generator_A.py") - print("fluid.default_main_program: {}".format( - fluid.default_main_program()._heter_pipeline_opt)) + dataset, file_list = get_dataset( + inputs, config, "python dataset_generator_A.py" + ) + print( + "fluid.default_main_program: {}".format( + fluid.default_main_program()._heter_pipeline_opt + ) + ) for epoch in range(epoch_num): # A 方和 B 方如果要以文件粒度 shuffle 时，则需要固定同一个种子 dataset.set_filelist(file_list) start_time = time.time() - exe.train_from_dataset(program=fluid.default_main_program(), - dataset=dataset, - print_period=2, - debug=False) + exe.train_from_dataset( + program=fluid.default_main_program(), + dataset=dataset, + print_period=2, + debug=False, + ) end_time = time.time() - print("trainer epoch %d finished, use time=%d\n" % - ((epoch), end_time - start_time)) + print( + "trainer epoch %d finished, use time=%d\n" + % ((epoch), end_time - start_time) + ) exe.close() _runtime_handle._stop_worker() print("Fl partyA Trainer Success!") @@ -116,19 +137,26 @@ def fl_ps_train(): exe = fluid.Executor() exe.run(fluid.default_startup_program()) _runtime_handle._init_worker() - inputs = [feeds_list[0], - feeds_list[-1]] # 顺序务必要和 dataset_generator_B.py 中保持一致 - dataset, file_list = get_dataset(inputs, config, - "python dataset_generator_B.py", - "heter_worker") - print("fluid.default_main_program: {}".format( - fluid.default_main_program()._heter_pipeline_opt)) + inputs = [ + feeds_list[0], + feeds_list[-1], + ] # 顺序务必要和 dataset_generator_B.py 中保持一致 + dataset, file_list = get_dataset( + inputs, config, "python dataset_generator_B.py", "heter_worker" + ) + print( + "fluid.default_main_program: {}".format( + fluid.default_main_program()._heter_pipeline_opt + ) + ) for epoch in range(epoch_num): dataset.set_filelist(file_list) - exe.train_from_dataset(program=fluid.default_main_program(), - dataset=dataset, - print_period=2, - debug=False) + exe.train_from_dataset( + program=fluid.default_main_program(), + dataset=dataset, + print_period=2, + debug=False, + ) exe.close() _runtime_handle._stop_worker() print("Fl partB Trainer Success!") diff --git a/python/paddle/fluid/tests/unittests/ps/ps_dnn_trainer.py b/python/paddle/fluid/tests/unittests/ps/ps_dnn_trainer.py index 8d63c9aa0acca..08055ae170393 100755 --- a/python/paddle/fluid/tests/unittests/ps/ps_dnn_trainer.py +++ b/python/paddle/fluid/tests/unittests/ps/ps_dnn_trainer.py @@ -13,7 +13,12 @@ # limitations under the License. import paddle.distributed.fleet.base.role_maker as role_maker -from paddle.distributed.ps.utils.ps_program_builder import debug_program, logger, new_pass, ps_log_root_dir +from paddle.distributed.ps.utils.ps_program_builder import ( + debug_program, + logger, + new_pass, + ps_log_root_dir, +) import paddle.distributed.fleet as fleet import argparse import sys @@ -42,7 +47,6 @@ def is_distributed_env(): class YamlHelper(object): - def load_yaml(self, yaml_file, other_part=None): part_list = ["runner", "hyper_parameters"] if other_part: @@ -114,7 +118,8 @@ def pretty_print_envs(self, envs, header=None): max_k = max(max_k, len(k)) h_format = " " + "|{{:>{}s}}{}{{:^{}s}}|\n".format( - max_k, " " * spacing, max_v) + max_k, " " * spacing, max_v + ) l_format = " " + "|{{:>{}s}}{{}}{{:^{}s}}|\n".format(max_k, max_v) length = max_k + max_v + spacing @@ -150,7 +155,7 @@ def get_user_defined_strategy(config): logger.warn( "Not Find Distributed env, Change To local train mode. If you want train with fleet, please use [fleetrun] command." ) - #return None + # return None sync_mode = config.get("runner.sync_mode") assert sync_mode in ["async", "sync", "geo", "heter", "gpubox"] if sync_mode == "sync": @@ -159,8 +164,9 @@ def get_user_defined_strategy(config): elif sync_mode == "async": strategy = paddle.distributed.fleet.DistributedStrategy() strategy.a_sync = True - strategy.is_fl_ps_mode = True if config.get( - "runner.is_fl_ps_mode") == 1 else False + strategy.is_fl_ps_mode = ( + True if config.get("runner.is_fl_ps_mode") == 1 else False + ) if strategy.is_fl_ps_mode == True: strategy.pipeline = False micro_num = 1 @@ -191,7 +197,7 @@ def get_user_defined_strategy(config): "dump_param": config.get("runner.dump_param", []), "stat_var_names": config.get("stat_var_names", []), "local_sparse": config.get("runner.local_sparse", []), - "remote_sparse": config.get("runner.remote_sparse", []) + "remote_sparse": config.get("runner.remote_sparse", []), } print("strategy:", strategy.trainer_desc_configs) @@ -200,7 +206,7 @@ def get_user_defined_strategy(config): "uri": config.get("runner.fs_client.uri", ""), "user": config.get("runner.fs_client.user", ""), "passwd": config.get("runner.fs_client.passwd", ""), - "hadoop_bin": config.get("runner.fs_client.hadoop_bin", "hadoop") + "hadoop_bin": config.get("runner.fs_client.hadoop_bin", "hadoop"), } print("strategy:", strategy.fs_client_param) @@ -225,7 +231,9 @@ def get_user_defined_strategy(config): def get_distributed_strategy(user_defined_strategy): # pslib - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + StrategyFactory, + ) k_steps = user_defined_strategy.a_sync_configs["k_steps"] strategy = None @@ -254,45 +262,38 @@ def get_model(config): def parse_args(): parser = argparse.ArgumentParser("PsTest train script") - parser.add_argument('-m', - '--config_yaml', - type=str, - required=True, - help='config file path') - parser.add_argument('-bf16', - '--pure_bf16', - type=ast.literal_eval, - default=False, - help="whether use bf16") - - parser.add_argument('--run_minimize', - type=int, - default=0, - help="test single pass") - parser.add_argument('--run_single_pass', - type=int, - default=0, - help="test single pass") - parser.add_argument('--run_the_one_ps', - type=int, - default=0, - help="test the_one_ps") - parser.add_argument('--debug_new_minimize', - type=int, - default=0, - help="test single pass") - parser.add_argument('--debug_new_pass', - type=int, - default=0, - help="test single pass") - parser.add_argument('--applied_pass_name', - type=str, - default="", - help="test single pass") - parser.add_argument('--debug_the_one_ps', - type=int, - default=0, - help="test the_one_ps") + parser.add_argument( + '-m', '--config_yaml', type=str, required=True, help='config file path' + ) + parser.add_argument( + '-bf16', + '--pure_bf16', + type=ast.literal_eval, + default=False, + help="whether use bf16", + ) + + parser.add_argument( + '--run_minimize', type=int, default=0, help="test single pass" + ) + parser.add_argument( + '--run_single_pass', type=int, default=0, help="test single pass" + ) + parser.add_argument( + '--run_the_one_ps', type=int, default=0, help="test the_one_ps" + ) + parser.add_argument( + '--debug_new_minimize', type=int, default=0, help="test single pass" + ) + parser.add_argument( + '--debug_new_pass', type=int, default=0, help="test single pass" + ) + parser.add_argument( + '--applied_pass_name', type=str, default="", help="test single pass" + ) + parser.add_argument( + '--debug_the_one_ps', type=int, default=0, help="test the_one_ps" + ) args = parser.parse_args() args.abs_dir = os.path.dirname(os.path.abspath(args.config_yaml)) @@ -317,7 +318,6 @@ def bf16_to_fp32(val): class DnnTrainer(object): - def __init__(self, config): self.metrics = {} self.config = config @@ -351,36 +351,59 @@ def run_minimize(self): loss = self.model._cost user_defined_strategy = get_user_defined_strategy(self.config) learning_rate = self.config.get( - "hyper_parameters.optimizer.learning_rate") + "hyper_parameters.optimizer.learning_rate" + ) sync_mode = self.config.get("runner.sync_mode") inner_optimizer = paddle.optimizer.Adam(learning_rate, lazy_mode=True) self.role_maker._generate_role() # 必要 if self.config['debug_new_minimize'] == 1: print("entering run_minimize -- new") - from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ParameterServerOptimizer + from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ( + ParameterServerOptimizer, + ) + ps_optimizer = ParameterServerOptimizer(inner_optimizer) - ps_optimizer._set_basic_info(loss, self.role_maker, inner_optimizer, - user_defined_strategy) + ps_optimizer._set_basic_info( + loss, self.role_maker, inner_optimizer, user_defined_strategy + ) ps_optimizer.minimize_impl(loss) else: print("entering run_minimize -- old") fleet_obj = fleet.distributed_optimizer( - inner_optimizer, user_defined_strategy) ## Fleet 对象 + inner_optimizer, user_defined_strategy + ) ## Fleet 对象 fleet_obj.minimize(loss) if fleet.is_server(): - _main_file = ps_log_root_dir + sync_mode + '_run_minimize' + '_debug:_' + str( - self.config['debug_new_minimize']) + '_server_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + '_run_minimize' + + '_debug:_' + + str(self.config['debug_new_minimize']) + + '_server_main.prototxt' + ) debug_program(_main_file, loss.block.program) elif fleet.is_worker(): - _main_file = ps_log_root_dir + sync_mode + '_run_minimize' + '_debug:_' + str( - self.config['debug_new_minimize']) + '_worker_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + '_run_minimize' + + '_debug:_' + + str(self.config['debug_new_minimize']) + + '_worker_main.prototxt' + ) debug_program(_main_file, loss.block.program) elif self.role_maker._is_heter_worker(): - _main_file = ps_log_root_dir + sync_mode + '_run_minimize' + '_debug:_' + str( - self.config['debug_new_minimize'] - ) + '_heter_worker_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + '_run_minimize' + + '_debug:_' + + str(self.config['debug_new_minimize']) + + '_heter_worker_main.prototxt' + ) debug_program(_main_file, loss.block.program) def run_single_pass(self): @@ -396,42 +419,70 @@ def run_single_pass(self): startup_program = paddle.static.default_startup_program() inner_optimizer.minimize(loss, startup_program) if self.config['debug_new_pass'] == 1: - print("entering run {} - new".format( - str(config["applied_pass_name"]))) - from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ParameterServerOptimizer + print( + "entering run {} - new".format(str(config["applied_pass_name"])) + ) + from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ( + ParameterServerOptimizer, + ) + ps_optimizer = ParameterServerOptimizer(inner_optimizer) - ps_optimizer._set_basic_info(loss, self.role_maker, inner_optimizer, - user_defined_strategy) + ps_optimizer._set_basic_info( + loss, self.role_maker, inner_optimizer, user_defined_strategy + ) ps_optimizer._set_origin_programs([loss]) ps_optimizer._init_ps_pass_context(loss, startup_program) _main = ps_optimizer.pass_ctx._attrs['cloned_main'] - append_send_ops_pass = new_pass(config["applied_pass_name"], - ps_optimizer.pass_ctx._attrs) + append_send_ops_pass = new_pass( + config["applied_pass_name"], ps_optimizer.pass_ctx._attrs + ) append_send_ops_pass.apply([_main], [None], ps_optimizer.pass_ctx) else: - print("entering run {} - old".format( - str(config["applied_pass_name"]))) - from paddle.fluid.incubate.fleet.parameter_server.ir import public as public + print( + "entering run {} - old".format(str(config["applied_pass_name"])) + ) + from paddle.fluid.incubate.fleet.parameter_server.ir import ( + public as public, + ) + dist_strategy = get_distributed_strategy(user_defined_strategy) compiled_config = public.CompileTimeStrategy( - loss.block.program, startup_program, dist_strategy, - self.role_maker) + loss.block.program, + startup_program, + dist_strategy, + self.role_maker, + ) _main = compiled_config.origin_main_program.clone() _startup = compiled_config.origin_startup_program.clone() - from paddle.fluid.incubate.fleet.parameter_server.ir import trainer_pass as worker + from paddle.fluid.incubate.fleet.parameter_server.ir import ( + trainer_pass as worker, + ) + _main = worker.append_send_ops_pass(_main, compiled_config) if fleet.is_server(): - _main_file = ps_log_root_dir + sync_mode + "_" + str( - config["applied_pass_name"]) + '_debug:_' + str( - self.config['debug_new_pass']) + '_server_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + "_" + + str(config["applied_pass_name"]) + + '_debug:_' + + str(self.config['debug_new_pass']) + + '_server_main.prototxt' + ) debug_program(_main_file, _main) elif fleet.is_worker(): - _main_file = ps_log_root_dir + sync_mode + "_" + str( - config["applied_pass_name"]) + '_debug:_' + str( - self.config['debug_new_pass']) + '_worker_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + "_" + + str(config["applied_pass_name"]) + + '_debug:_' + + str(self.config['debug_new_pass']) + + '_worker_main.prototxt' + ) debug_program(_main_file, _main) def run_the_one_ps(self): @@ -442,7 +493,8 @@ def run_the_one_ps(self): loss = self.model._cost user_defined_strategy = get_user_defined_strategy(self.config) learning_rate = self.config.get( - "hyper_parameters.optimizer.learning_rate") + "hyper_parameters.optimizer.learning_rate" + ) sync_mode = self.config.get("runner.sync_mode") inner_optimizer = paddle.optimizer.Adam(learning_rate, lazy_mode=True) @@ -450,28 +502,37 @@ def run_the_one_ps(self): if self.config['debug_the_one_ps'] == 1: print("entering run_the_one_ps -- new") - from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ParameterServerOptimizer + from paddle.distributed.fleet.meta_optimizers.ps_optimizer import ( + ParameterServerOptimizer, + ) + ps_optimizer = ParameterServerOptimizer(inner_optimizer) - ps_optimizer._set_basic_info(loss, self.role_maker, inner_optimizer, - user_defined_strategy) + ps_optimizer._set_basic_info( + loss, self.role_maker, inner_optimizer, user_defined_strategy + ) ps_optimizer.minimize_impl(loss) from paddle.distributed.ps.the_one_ps import TheOnePSRuntime + _runtime_handle = TheOnePSRuntime() # ps 目录下重构版的 TheOnePSRuntime _runtime_handle._set_basic_info(ps_optimizer.pass_ctx._attrs) if fleet.is_worker(): - worker_desc = _runtime_handle.ps_desc_builder.build_worker_desc( + worker_desc = ( + _runtime_handle.ps_desc_builder.build_worker_desc() ) with open( - ps_log_root_dir + sync_mode + '_' + - 'new_worker_ps_desc', 'w') as f: + ps_log_root_dir + sync_mode + '_' + 'new_worker_ps_desc', + 'w', + ) as f: f.write(worker_desc) if fleet.is_server(): - server_desc = _runtime_handle.ps_desc_builder.build_server_desc( + server_desc = ( + _runtime_handle.ps_desc_builder.build_server_desc() ) with open( - ps_log_root_dir + sync_mode + '_' + - 'new_server_ps_desc', 'w') as f: + ps_log_root_dir + sync_mode + '_' + 'new_server_ps_desc', + 'w', + ) as f: f.write(server_desc) else: @@ -492,16 +553,34 @@ def run_the_one_ps(self): f.write(str(server_desc) + str(fleet_obj._runtime_handle._get_fs_client_desc().to_string())) ''' if fleet.is_server(): - _main_file = ps_log_root_dir + sync_mode + '_run_the_one_ps' + '_debug:_' + str( - self.config['debug_the_one_ps']) + '_server_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + '_run_the_one_ps' + + '_debug:_' + + str(self.config['debug_the_one_ps']) + + '_server_main.prototxt' + ) debug_program(_main_file, loss.block.program) elif fleet.is_worker(): - _main_file = ps_log_root_dir + sync_mode + '_run_the_one_ps' + '_debug:_' + str( - self.config['debug_the_one_ps']) + '_worker_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + '_run_the_one_ps' + + '_debug:_' + + str(self.config['debug_the_one_ps']) + + '_worker_main.prototxt' + ) debug_program(_main_file, loss.block.program) elif self.role_maker._is_heter_worker(): - _main_file = ps_log_root_dir + sync_mode + '_run_the_one_ps' + '_debug:_' + str( - self.config['debug_the_one_ps']) + '_heter_worker_main.prototxt' + _main_file = ( + ps_log_root_dir + + sync_mode + + '_run_the_one_ps' + + '_debug:_' + + str(self.config['debug_the_one_ps']) + + '_heter_worker_main.prototxt' + ) debug_program(_main_file, loss.block.program) diff --git a/python/paddle/fluid/tests/unittests/ps/test_fl_ps.py b/python/paddle/fluid/tests/unittests/ps/test_fl_ps.py index 2726d7c10cdbe..dc315b7df29ad 100755 --- a/python/paddle/fluid/tests/unittests/ps/test_fl_ps.py +++ b/python/paddle/fluid/tests/unittests/ps/test_fl_ps.py @@ -16,12 +16,13 @@ import unittest import shlex # noqa: F401 -from paddle.fluid.tests.unittests.distributed_passes.dist_pass_test_base import remove_path_if_exists # noqa: F401 +from paddle.fluid.tests.unittests.distributed_passes.dist_pass_test_base import ( + remove_path_if_exists, +) # noqa: F401 import os class FlPsTest(unittest.TestCase): - def test_launch_fl_ps(self): ''' cmd = [ diff --git a/python/paddle/fluid/tests/unittests/ps/test_the_one_ps.py b/python/paddle/fluid/tests/unittests/ps/test_the_one_ps.py index b561d82fc5f30..c9d0c9fc943dc 100755 --- a/python/paddle/fluid/tests/unittests/ps/test_the_one_ps.py +++ b/python/paddle/fluid/tests/unittests/ps/test_the_one_ps.py @@ -14,14 +14,16 @@ import unittest -from paddle.fluid.tests.unittests.distributed_passes.ps_pass_test_base import PsPassTestBase, remove_path_if_exists +from paddle.fluid.tests.unittests.distributed_passes.ps_pass_test_base import ( + PsPassTestBase, + remove_path_if_exists, +) from paddle.distributed.ps.utils.public import logger, ps_log_root_dir import paddle.distributed.fleet.proto.the_one_ps_pb2 as ps_pb2 # noqa: F401 from google.protobuf import text_format # noqa: F401 class TestTheOnePs(PsPassTestBase): - def setUp(self): pass @@ -56,14 +58,16 @@ def test_ps_cpu_async(self): self.config['run_the_one_ps'] = '1' self.config['debug_the_one_ps'] = '0' - self.config[ - 'log_dir'] = ps_log_root_dir + "async_cpu_log_old_the_one_ps" + self.config['log_dir'] = ( + ps_log_root_dir + "async_cpu_log_old_the_one_ps" + ) remove_path_if_exists(self.config['log_dir']) self.ps_launch() self.config['debug_the_one_ps'] = '1' - self.config[ - 'log_dir'] = ps_log_root_dir + "async_cpu_log_new_the_one_ps" + self.config['log_dir'] = ( + ps_log_root_dir + "async_cpu_log_new_the_one_ps" + ) remove_path_if_exists(self.config['log_dir']) self.ps_launch() diff --git a/python/paddle/fluid/tests/unittests/ps_dnn_model.py b/python/paddle/fluid/tests/unittests/ps_dnn_model.py index 89472c73bd31d..c2775fbca3125 100755 --- a/python/paddle/fluid/tests/unittests/ps_dnn_model.py +++ b/python/paddle/fluid/tests/unittests/ps_dnn_model.py @@ -18,14 +18,15 @@ class DNNLayer(nn.Layer): - - def __init__(self, - sparse_feature_number, - sparse_feature_dim, - dense_feature_dim, - num_field, - layer_sizes, - sync_mode=None): + def __init__( + self, + sparse_feature_number, + sparse_feature_dim, + dense_feature_dim, + num_field, + layer_sizes, + sync_mode=None, + ): super(DNNLayer, self).__init__() self.sync_mode = sync_mode self.sparse_feature_number = sparse_feature_number @@ -40,10 +41,15 @@ def __init__(self, sparse=True, weight_attr=paddle.ParamAttr( name="SparseFeatFactors", - initializer=paddle.nn.initializer.Uniform())) - - sizes = [sparse_feature_dim * num_field + dense_feature_dim - ] + self.layer_sizes + [2] + initializer=paddle.nn.initializer.Uniform(), + ), + ) + + sizes = ( + [sparse_feature_dim * num_field + dense_feature_dim] + + self.layer_sizes + + [2] + ) acts = ["relu" for _ in range(len(self.layer_sizes))] + [None] self._mlp_layers = [] for i in range(len(layer_sizes) + 1): @@ -52,7 +58,10 @@ def __init__(self, out_features=sizes[i + 1], weight_attr=paddle.ParamAttr( initializer=paddle.nn.initializer.Normal( - std=1.0 / math.sqrt(sizes[i])))) + std=1.0 / math.sqrt(sizes[i]) + ) + ), + ) self.add_sublayer('linear_%d' % i, linear) self._mlp_layers.append(linear) if acts[i] == 'relu': @@ -68,7 +77,8 @@ def forward(self, sparse_inputs, dense_inputs): emb = paddle.fluid.contrib.sparse_embedding( input=s_input, size=[self.sparse_feature_number, self.sparse_feature_dim], - param_attr=paddle.ParamAttr(name="embedding")) + param_attr=paddle.ParamAttr(name="embedding"), + ) else: emb = self.embedding(s_input) emb = paddle.reshape(emb, shape=[-1, self.sparse_feature_dim]) @@ -89,13 +99,14 @@ def forward(self, sparse_inputs, dense_inputs): class FlDNNLayer(nn.Layer): - - def __init__(self, - sparse_feature_number, - sparse_feature_dim, - dense_feature_dim, - sparse_number, - sync_mode=None): + def __init__( + self, + sparse_feature_number, + sparse_feature_dim, + dense_feature_dim, + sparse_number, + sync_mode=None, + ): super(FlDNNLayer, self).__init__() self.PART_A_DEVICE_FlAG = 'gpu:0' @@ -109,8 +120,11 @@ def __init__(self, self.slot_num = sparse_number self.dense_feature_dim = dense_feature_dim - layer_sizes_a = [self.slot_num * self.sparse_feature_dim, 5, - 7] # for test + layer_sizes_a = [ + self.slot_num * self.sparse_feature_dim, + 5, + 7, + ] # for test layer_sizes_b = [self.dense_feature_dim, 6, 7] layer_sizes_top = [7, 2] @@ -120,7 +134,9 @@ def __init__(self, sparse=True, weight_attr=paddle.ParamAttr( name="SparseFeatFactors", - initializer=paddle.nn.initializer.Uniform())) + initializer=paddle.nn.initializer.Uniform(), + ), + ) # part_a fc acts = ["relu" for _ in range(len(layer_sizes_a))] @@ -131,7 +147,10 @@ def __init__(self, out_features=layer_sizes_a[i + 1], weight_attr=paddle.ParamAttr( initializer=paddle.nn.initializer.Normal( - std=1.0 / math.sqrt(layer_sizes_a[i])))) + std=1.0 / math.sqrt(layer_sizes_a[i]) + ) + ), + ) self.add_sublayer('linear_%d' % i, linear) self._mlp_layers_a.append(linear) act = paddle.nn.ReLU() @@ -147,7 +166,10 @@ def __init__(self, out_features=layer_sizes_b[i + 1], weight_attr=paddle.ParamAttr( initializer=paddle.nn.initializer.Normal( - std=1.0 / math.sqrt(layer_sizes_b[i])))) + std=1.0 / math.sqrt(layer_sizes_b[i]) + ) + ), + ) self.add_sublayer('linear_%d' % i, linear) self._mlp_layers_b.append(linear) act = paddle.nn.ReLU() @@ -163,7 +185,10 @@ def __init__(self, out_features=layer_sizes_top[i + 1], weight_attr=paddle.ParamAttr( initializer=paddle.nn.initializer.Normal( - std=1.0 / math.sqrt(layer_sizes_top[i])))) + std=1.0 / math.sqrt(layer_sizes_top[i]) + ) + ), + ) self.add_sublayer('linear_%d' % i, linear) self._mlp_layers_top.append(linear) act = paddle.nn.ReLU() @@ -184,7 +209,8 @@ def bottom_a_layer(self, sparse_inputs): y = self._mlp_layers_a[2](y) with paddle.fluid.device_guard( - self.PART_A_JOINT_OP_DEVICE_FlAG): # joint point + self.PART_A_JOINT_OP_DEVICE_FlAG + ): # joint point bottom_a = self._mlp_layers_a[3](y) return bottom_a @@ -201,9 +227,11 @@ def bottom_b_layer(self, dense_inputs): def interactive_layer(self, bottom_a, bottom_b): with paddle.fluid.device_guard( - self.PART_B_JOINT_OP_DEVICE_FlAG): # joint point + self.PART_B_JOINT_OP_DEVICE_FlAG + ): # joint point interactive = paddle.fluid.layers.elementwise_add( - bottom_a, bottom_b) + bottom_a, bottom_b + ) return interactive def top_layer(self, interactive, label_input): @@ -211,16 +239,25 @@ def top_layer(self, interactive, label_input): y = self._mlp_layers_top[0](interactive) y_top = self._mlp_layers_top[1](y) predict_2d = paddle.nn.functional.softmax(y_top) - auc, batch_auc, [ - self.batch_stat_pos, self.batch_stat_neg, self.stat_pos, - self.stat_neg - ] = paddle.static.auc(input=predict_2d, - label=label_input, - num_thresholds=2**12, - slide_steps=20) - - cost = paddle.nn.functional.cross_entropy(input=y_top, - label=label_input) + ( + auc, + batch_auc, + [ + self.batch_stat_pos, + self.batch_stat_neg, + self.stat_pos, + self.stat_neg, + ], + ) = paddle.static.auc( + input=predict_2d, + label=label_input, + num_thresholds=2**12, + slide_steps=20, + ) + + cost = paddle.nn.functional.cross_entropy( + input=y_top, label=label_input + ) avg_cost = paddle.mean(x=cost) return auc, avg_cost @@ -237,8 +274,7 @@ def forward(self, sparse_inputs, dense_inputs, label_input): return auc, avg_cost -class StaticModel(): - +class StaticModel: def __init__(self, config): self.cost = None self.infer_target_var = None @@ -254,21 +290,28 @@ def _init_hyper_parameters(self): self.distributed_embedding = True self.sparse_feature_number = self.config.get( - "hyper_parameters.sparse_feature_number") + "hyper_parameters.sparse_feature_number" + ) self.sparse_feature_dim = self.config.get( - "hyper_parameters.sparse_feature_dim") + "hyper_parameters.sparse_feature_dim" + ) self.sparse_inputs_slots = self.config.get( - "hyper_parameters.sparse_inputs_slots") + "hyper_parameters.sparse_inputs_slots" + ) self.dense_input_dim = self.config.get( - "hyper_parameters.dense_input_dim") + "hyper_parameters.dense_input_dim" + ) self.learning_rate = self.config.get( - "hyper_parameters.optimizer.learning_rate") + "hyper_parameters.optimizer.learning_rate" + ) self.fc_sizes = self.config.get("hyper_parameters.fc_sizes") def create_feeds(self, is_infer=False): - dense_input = paddle.static.data(name="dense_input", - shape=[None, self.dense_input_dim], - dtype="float32") + dense_input = paddle.static.data( + name="dense_input", + shape=[None, self.dense_input_dim], + dtype="float32", + ) sparse_input_ids = [ paddle.static.data(name=str(i), shape=[None, 1], dtype="int64") @@ -282,33 +325,44 @@ def create_feeds(self, is_infer=False): def net(self, input, is_infer=False): self.label_input = input[0] - self.sparse_inputs = input[1:self.sparse_inputs_slots] + self.sparse_inputs = input[1 : self.sparse_inputs_slots] self.dense_input = input[-1] sparse_number = self.sparse_inputs_slots - 1 - dnn_model = DNNLayer(self.sparse_feature_number, - self.sparse_feature_dim, - self.dense_input_dim, - sparse_number, - self.fc_sizes, - sync_mode=self.sync_mode) + dnn_model = DNNLayer( + self.sparse_feature_number, + self.sparse_feature_dim, + self.dense_input_dim, + sparse_number, + self.fc_sizes, + sync_mode=self.sync_mode, + ) raw_predict_2d = dnn_model.forward(self.sparse_inputs, self.dense_input) predict_2d = paddle.nn.functional.softmax(raw_predict_2d) self.predict = predict_2d - auc, batch_auc, [ - self.batch_stat_pos, self.batch_stat_neg, self.stat_pos, - self.stat_neg - ] = paddle.static.auc(input=self.predict, - label=self.label_input, - num_thresholds=2**12, - slide_steps=20) + ( + auc, + batch_auc, + [ + self.batch_stat_pos, + self.batch_stat_neg, + self.stat_pos, + self.stat_neg, + ], + ) = paddle.static.auc( + input=self.predict, + label=self.label_input, + num_thresholds=2**12, + slide_steps=20, + ) self.inference_target_var = auc if is_infer: fetch_dict = {'auc': auc} return fetch_dict - cost = paddle.nn.functional.cross_entropy(input=raw_predict_2d, - label=self.label_input) + cost = paddle.nn.functional.cross_entropy( + input=raw_predict_2d, label=self.label_input + ) avg_cost = paddle.mean(x=cost) self._cost = avg_cost @@ -317,18 +371,21 @@ def net(self, input, is_infer=False): def fl_net(self, input, is_infer=False): self.label_input = input[0] - self.sparse_inputs = input[1:self.sparse_inputs_slots] + self.sparse_inputs = input[1 : self.sparse_inputs_slots] self.dense_input = input[-1] self.sparse_number = self.sparse_inputs_slots - 1 - fl_dnn_model = FlDNNLayer(self.sparse_feature_number, - self.sparse_feature_dim, - self.dense_input_dim, - self.sparse_number, - sync_mode=self.sync_mode) - - auc, avg_cost = fl_dnn_model.forward(self.sparse_inputs, - self.dense_input, self.label_input) + fl_dnn_model = FlDNNLayer( + self.sparse_feature_number, + self.sparse_feature_dim, + self.dense_input_dim, + self.sparse_number, + sync_mode=self.sync_mode, + ) + + auc, avg_cost = fl_dnn_model.forward( + self.sparse_inputs, self.dense_input, self.label_input + ) fetch_dict = {'cost': avg_cost, 'auc': auc} self._cost = avg_cost return fetch_dict diff --git a/python/paddle/fluid/tests/unittests/py_precise_roi_pool.py b/python/paddle/fluid/tests/unittests/py_precise_roi_pool.py index 29721c86200aa..e18c302fd7b41 100644 --- a/python/paddle/fluid/tests/unittests/py_precise_roi_pool.py +++ b/python/paddle/fluid/tests/unittests/py_precise_roi_pool.py @@ -17,7 +17,6 @@ class PyPrRoIPool(object): - def __init__(self): pass @@ -28,50 +27,65 @@ def _PrRoIPoolingGetData(self, data, h, w, height, width): else: return data[h][w] - def _PrRoIPoolingMatCalculation(self, this_data, s_h, s_w, e_h, e_w, y0, x0, - y1, x1, h0, w0): + def _PrRoIPoolingMatCalculation( + self, this_data, s_h, s_w, e_h, e_w, y0, x0, y1, x1, h0, w0 + ): sum_out = 0.0 alpha = x0 - float(s_w) beta = y0 - float(s_h) lim_alpha = x1 - float(s_w) lim_beta = y1 - float(s_h) - tmp = (lim_alpha - 0.5 * lim_alpha * lim_alpha - alpha + - 0.5 * alpha * alpha) * (lim_beta - 0.5 * lim_beta * lim_beta - - beta + 0.5 * beta * beta) + tmp = ( + lim_alpha + - 0.5 * lim_alpha * lim_alpha + - alpha + + 0.5 * alpha * alpha + ) * (lim_beta - 0.5 * lim_beta * lim_beta - beta + 0.5 * beta * beta) sum_out += self._PrRoIPoolingGetData(this_data, s_h, s_w, h0, w0) * tmp alpha = float(e_w) - x1 lim_alpha = float(e_w) - x0 - tmp = (lim_alpha - 0.5 * lim_alpha * lim_alpha - alpha + - 0.5 * alpha * alpha) * (lim_beta - 0.5 * lim_beta * lim_beta - - beta + 0.5 * beta * beta) + tmp = ( + lim_alpha + - 0.5 * lim_alpha * lim_alpha + - alpha + + 0.5 * alpha * alpha + ) * (lim_beta - 0.5 * lim_beta * lim_beta - beta + 0.5 * beta * beta) sum_out += self._PrRoIPoolingGetData(this_data, s_h, e_w, h0, w0) * tmp alpha = x0 - float(s_w) beta = float(e_h) - y1 lim_alpha = x1 - float(s_w) lim_beta = float(e_h) - y0 - tmp = (lim_alpha - 0.5 * lim_alpha * lim_alpha - alpha + - 0.5 * alpha * alpha) * (lim_beta - 0.5 * lim_beta * lim_beta - - beta + 0.5 * beta * beta) + tmp = ( + lim_alpha + - 0.5 * lim_alpha * lim_alpha + - alpha + + 0.5 * alpha * alpha + ) * (lim_beta - 0.5 * lim_beta * lim_beta - beta + 0.5 * beta * beta) sum_out += self._PrRoIPoolingGetData(this_data, e_h, s_w, h0, w0) * tmp alpha = float(e_w) - x1 lim_alpha = float(e_w) - x0 - tmp = (lim_alpha - 0.5 * lim_alpha * lim_alpha - alpha + - 0.5 * alpha * alpha) * (lim_beta - 0.5 * lim_beta * lim_beta - - beta + 0.5 * beta * beta) + tmp = ( + lim_alpha + - 0.5 * lim_alpha * lim_alpha + - alpha + + 0.5 * alpha * alpha + ) * (lim_beta - 0.5 * lim_beta * lim_beta - beta + 0.5 * beta * beta) sum_out += self._PrRoIPoolingGetData(this_data, e_h, e_w, h0, w0) * tmp return sum_out - def compute(self, - x, - rois, - output_channels, - spatial_scale=0.1, - pooled_height=1, - pooled_width=1): + def compute( + self, + x, + rois, + output_channels, + spatial_scale=0.1, + pooled_height=1, + pooled_width=1, + ): ''' calculate the precise roi pooling values Note: This function is implements as pure python without any paddle concept involved @@ -134,14 +148,18 @@ def compute(self, for w_iter in range(int(s_w), int(e_w)): for h_iter in range(int(s_h), int(e_h)): sum_out += self._PrRoIPoolingMatCalculation( - x_i[c_in], h_iter, w_iter, h_iter + 1, + x_i[c_in], + h_iter, + w_iter, + h_iter + 1, w_iter + 1, max(win_start_h, float(h_iter)), max(win_start_w, float(w_iter)), - min(win_end_h, - float(h_iter) + 1.0), - min(win_end_w, - float(w_iter + 1.0)), height, width) + min(win_end_h, float(h_iter) + 1.0), + min(win_end_w, float(w_iter + 1.0)), + height, + width, + ) out_data[i, c, ph, pw] = sum_out / win_size diff --git a/python/paddle/fluid/tests/unittests/rnn/convert.py b/python/paddle/fluid/tests/unittests/rnn/convert.py index b26db11e1d34f..c41f80c7eabbf 100644 --- a/python/paddle/fluid/tests/unittests/rnn/convert.py +++ b/python/paddle/fluid/tests/unittests/rnn/convert.py @@ -41,22 +41,28 @@ def convert_params_for_net(np_net, paddle_net): def convert_params_for_net_static(np_net, paddle_net, place): for np_layer, paddle_layer in zip(np_net, paddle_net): if hasattr(np_layer, "cell"): - convert_params_for_cell_static(np_layer.cell, paddle_layer.cell, - place) + convert_params_for_cell_static( + np_layer.cell, paddle_layer.cell, place + ) else: - convert_params_for_cell_static(np_layer.cell_fw, - paddle_layer.cell_fw, place) - convert_params_for_cell_static(np_layer.cell_bw, - paddle_layer.cell_bw, place) + convert_params_for_cell_static( + np_layer.cell_fw, paddle_layer.cell_fw, place + ) + convert_params_for_cell_static( + np_layer.cell_bw, paddle_layer.cell_bw, place + ) def get_params_for_cell(np_cell, num_layers, idx): state = np_cell.parameters - weight_list = [('{}.weight_{}'.format(num_layers, idx), state['weight_ih']), - ('{}.weight_{}'.format(num_layers, - idx + 1), state['weight_hh'])] - bias_list = [('{}.bias_{}'.format(num_layers, idx), state['bias_ih']), - ('{}.bias_{}'.format(num_layers, idx + 1), state['bias_hh'])] + weight_list = [ + ('{}.weight_{}'.format(num_layers, idx), state['weight_ih']), + ('{}.weight_{}'.format(num_layers, idx + 1), state['weight_hh']), + ] + bias_list = [ + ('{}.bias_{}'.format(num_layers, idx), state['bias_ih']), + ('{}.bias_{}'.format(num_layers, idx + 1), state['bias_hh']), + ] return weight_list, bias_list diff --git a/python/paddle/fluid/tests/unittests/rnn/rnn_numpy.py b/python/paddle/fluid/tests/unittests/rnn/rnn_numpy.py index fbdc3ec8a4854..20859f896d13f 100644 --- a/python/paddle/fluid/tests/unittests/rnn/rnn_numpy.py +++ b/python/paddle/fluid/tests/unittests/rnn/rnn_numpy.py @@ -17,13 +17,11 @@ class LayerMixin(object): - def __call__(self, *args, **kwargs): return self.forward(*args, **kwargs) class LayerListMixin(LayerMixin): - def __init__(self, layers=None): self._layers = list(layers) if layers else [] @@ -35,34 +33,39 @@ def __iter__(self): class SimpleRNNCell(LayerMixin): - - def __init__(self, - input_size, - hidden_size, - bias=True, - nonlinearity="RNN_TANH", - dtype="float64"): + def __init__( + self, + input_size, + hidden_size, + bias=True, + nonlinearity="RNN_TANH", + dtype="float64", + ): self.input_size = input_size self.hidden_size = hidden_size self.bias = bias if nonlinearity == 'RNN_TANH': self.nonlinearity = np.tanh else: - self.nonlinearity = lambda x: np.maximum(x, 0.) + self.nonlinearity = lambda x: np.maximum(x, 0.0) self.parameters = dict() std = 1.0 / math.sqrt(hidden_size) self.weight_ih = np.random.uniform( - -std, std, (hidden_size, input_size)).astype(dtype) + -std, std, (hidden_size, input_size) + ).astype(dtype) self.weight_hh = np.random.uniform( - -std, std, (hidden_size, hidden_size)).astype(dtype) + -std, std, (hidden_size, hidden_size) + ).astype(dtype) self.parameters['weight_ih'] = self.weight_ih self.parameters['weight_hh'] = self.weight_hh if bias: - self.bias_ih = np.random.uniform(-std, std, - (hidden_size, )).astype(dtype) - self.bias_hh = np.random.uniform(-std, std, - (hidden_size, )).astype(dtype) + self.bias_ih = np.random.uniform(-std, std, (hidden_size,)).astype( + dtype + ) + self.bias_hh = np.random.uniform(-std, std, (hidden_size,)).astype( + dtype + ) self.parameters['bias_ih'] = self.bias_ih self.parameters['bias_hh'] = self.bias_hh else: @@ -88,7 +91,6 @@ def forward(self, inputs, hx=None): class GRUCell(LayerMixin): - def __init__(self, input_size, hidden_size, bias=True, dtype="float64"): self.input_size = input_size self.hidden_size = hidden_size @@ -96,16 +98,20 @@ def __init__(self, input_size, hidden_size, bias=True, dtype="float64"): self.parameters = dict() std = 1.0 / math.sqrt(hidden_size) self.weight_ih = np.random.uniform( - -std, std, (3 * hidden_size, input_size)).astype(dtype) + -std, std, (3 * hidden_size, input_size) + ).astype(dtype) self.weight_hh = np.random.uniform( - -std, std, (3 * hidden_size, hidden_size)).astype(dtype) + -std, std, (3 * hidden_size, hidden_size) + ).astype(dtype) self.parameters['weight_ih'] = self.weight_ih self.parameters['weight_hh'] = self.weight_hh if bias: - self.bias_ih = np.random.uniform(-std, std, - (3 * hidden_size)).astype(dtype) - self.bias_hh = np.random.uniform(-std, std, - (3 * hidden_size)).astype(dtype) + self.bias_ih = np.random.uniform( + -std, std, (3 * hidden_size) + ).astype(dtype) + self.bias_hh = np.random.uniform( + -std, std, (3 * hidden_size) + ).astype(dtype) self.parameters['bias_ih'] = self.bias_ih self.parameters['bias_hh'] = self.bias_hh else: @@ -137,7 +143,6 @@ def forward(self, inputs, hx=None): class LSTMCell(LayerMixin): - def __init__(self, input_size, hidden_size, bias=True, dtype="float64"): self.input_size = input_size self.hidden_size = hidden_size @@ -145,16 +150,20 @@ def __init__(self, input_size, hidden_size, bias=True, dtype="float64"): self.parameters = dict() std = 1.0 / math.sqrt(hidden_size) self.weight_ih = np.random.uniform( - -std, std, (4 * hidden_size, input_size)).astype(dtype) + -std, std, (4 * hidden_size, input_size) + ).astype(dtype) self.weight_hh = np.random.uniform( - -std, std, (4 * hidden_size, hidden_size)).astype(dtype) + -std, std, (4 * hidden_size, hidden_size) + ).astype(dtype) self.parameters['weight_ih'] = self.weight_ih self.parameters['weight_hh'] = self.weight_hh if bias: - self.bias_ih = np.random.uniform(-std, std, - (4 * hidden_size)).astype(dtype) - self.bias_hh = np.random.uniform(-std, std, - (4 * hidden_size)).astype(dtype) + self.bias_ih = np.random.uniform( + -std, std, (4 * hidden_size) + ).astype(dtype) + self.bias_hh = np.random.uniform( + -std, std, (4 * hidden_size) + ).astype(dtype) self.parameters['bias_ih'] = self.bias_ih self.parameters['bias_hh'] = self.bias_hh else: @@ -204,12 +213,14 @@ def update_state(mask, new, old): return tuple(map(lambda x, y: np.where(mask, x, y), new, old)) -def rnn(cell, - inputs, - initial_states, - sequence_length=None, - time_major=False, - is_reverse=False): +def rnn( + cell, + inputs, + initial_states, + sequence_length=None, + time_major=False, + is_reverse=False, +): if not time_major: inputs = np.transpose(inputs, [1, 0, 2]) if is_reverse: @@ -234,7 +245,7 @@ def rnn(cell, if mask is not None: m_t = mask[t] y, new_state = cell(x_t, state) - y = np.where(m_t, y, 0.) + y = np.where(m_t, y, 0.0) outputs.append(y) state = update_state(m_t, new_state, state) else: @@ -252,25 +263,27 @@ def rnn(cell, return outputs, final_state -def birnn(cell_fw, - cell_bw, - inputs, - initial_states, - sequence_length=None, - time_major=False): +def birnn( + cell_fw, + cell_bw, + inputs, + initial_states, + sequence_length=None, + time_major=False, +): states_fw, states_bw = initial_states - outputs_fw, states_fw = rnn(cell_fw, - inputs, - states_fw, - sequence_length, - time_major=time_major) - - outputs_bw, states_bw = rnn(cell_bw, - inputs, - states_bw, - sequence_length, - time_major=time_major, - is_reverse=True) + outputs_fw, states_fw = rnn( + cell_fw, inputs, states_fw, sequence_length, time_major=time_major + ) + + outputs_bw, states_bw = rnn( + cell_bw, + inputs, + states_bw, + sequence_length, + time_major=time_major, + is_reverse=True, + ) outputs = np.concatenate((outputs_fw, outputs_bw), -1) final_states = (states_fw, states_bw) @@ -332,7 +345,6 @@ def concat_states(states, bidirectional=False, state_components=1): class RNN(LayerMixin): - def __init__(self, cell, is_reverse=False, time_major=False): super(RNN, self).__init__() self.cell = cell @@ -343,124 +355,135 @@ def __init__(self, cell, is_reverse=False, time_major=False): self.time_major = time_major def forward(self, inputs, initial_states=None, sequence_length=None): - final_outputs, final_states = rnn(self.cell, - inputs, - initial_states=initial_states, - sequence_length=sequence_length, - time_major=self.time_major, - is_reverse=self.is_reverse) + final_outputs, final_states = rnn( + self.cell, + inputs, + initial_states=initial_states, + sequence_length=sequence_length, + time_major=self.time_major, + is_reverse=self.is_reverse, + ) return final_outputs, final_states class BiRNN(LayerMixin): - def __init__(self, cell_fw, cell_bw, time_major=False): super(BiRNN, self).__init__() self.cell_fw = cell_fw self.cell_bw = cell_bw self.time_major = time_major - def forward(self, - inputs, - initial_states=None, - sequence_length=None, - **kwargs): + def forward( + self, inputs, initial_states=None, sequence_length=None, **kwargs + ): if isinstance(initial_states, (list, tuple)): - assert len(initial_states) == 2, \ - "length of initial_states should be 2 when it is a list/tuple" + assert ( + len(initial_states) == 2 + ), "length of initial_states should be 2 when it is a list/tuple" else: initial_states = [initial_states, initial_states] - outputs, final_states = birnn(self.cell_fw, self.cell_bw, inputs, - initial_states, sequence_length, - self.time_major) + outputs, final_states = birnn( + self.cell_fw, + self.cell_bw, + inputs, + initial_states, + sequence_length, + self.time_major, + ) return outputs, final_states class RNNMixin(LayerListMixin): - def forward(self, inputs, initial_states=None, sequence_length=None): batch_index = 1 if self.time_major else 0 batch_size = inputs.shape[batch_index] dtype = inputs.dtype if initial_states is None: - state_shape = (self.num_layers * self.num_directions, batch_size, - self.hidden_size) + state_shape = ( + self.num_layers * self.num_directions, + batch_size, + self.hidden_size, + ) if self.state_components == 1: initial_states = np.zeros(state_shape, dtype) else: - initial_states = tuple([ - np.zeros(state_shape, dtype) - for _ in range(self.state_components) - ]) - - states = split_states(initial_states, self.num_directions == 2, - self.state_components) + initial_states = tuple( + [ + np.zeros(state_shape, dtype) + for _ in range(self.state_components) + ] + ) + + states = split_states( + initial_states, self.num_directions == 2, self.state_components + ) final_states = [] input_temp = inputs for i, rnn_layer in enumerate(self): if i > 0: input_temp = dropout(inputs, self.dropout) - outputs, final_state = rnn_layer(input_temp, states[i], - sequence_length) + outputs, final_state = rnn_layer( + input_temp, states[i], sequence_length + ) final_states.append(final_state) inputs = outputs - final_states = concat_states(final_states, self.num_directions == 2, - self.state_components) + final_states = concat_states( + final_states, self.num_directions == 2, self.state_components + ) return outputs, final_states class SimpleRNN(RNNMixin): - - def __init__(self, - input_size, - hidden_size, - num_layers=1, - nonlinearity="RNN_TANH", - direction="forward", - dropout=0., - time_major=False, - dtype="float64"): + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + nonlinearity="RNN_TANH", + direction="forward", + dropout=0.0, + time_major=False, + dtype="float64", + ): super(SimpleRNN, self).__init__() bidirectional_list = ["bidirectional", "bidirect"] if direction in ["forward"]: is_reverse = False - cell = SimpleRNNCell(input_size, - hidden_size, - nonlinearity=nonlinearity, - dtype=dtype) + cell = SimpleRNNCell( + input_size, hidden_size, nonlinearity=nonlinearity, dtype=dtype + ) self.append(RNN(cell, is_reverse, time_major)) for i in range(1, num_layers): - cell = SimpleRNNCell(hidden_size, - hidden_size, - nonlinearity=nonlinearity, - dtype=dtype) + cell = SimpleRNNCell( + hidden_size, + hidden_size, + nonlinearity=nonlinearity, + dtype=dtype, + ) self.append(RNN(cell, is_reverse, time_major)) elif direction in bidirectional_list: - cell_fw = SimpleRNNCell(input_size, - hidden_size, - nonlinearity=nonlinearity, - dtype=dtype) - cell_bw = SimpleRNNCell(input_size, - hidden_size, - nonlinearity=nonlinearity, - dtype=dtype) + cell_fw = SimpleRNNCell( + input_size, hidden_size, nonlinearity=nonlinearity, dtype=dtype + ) + cell_bw = SimpleRNNCell( + input_size, hidden_size, nonlinearity=nonlinearity, dtype=dtype + ) self.append(BiRNN(cell_fw, cell_bw, time_major)) for i in range(1, num_layers): - cell_fw = SimpleRNNCell(2 * hidden_size, - hidden_size, - nonlinearity, - dtype=dtype) - cell_bw = SimpleRNNCell(2 * hidden_size, - hidden_size, - nonlinearity, - dtype=dtype) + cell_fw = SimpleRNNCell( + 2 * hidden_size, hidden_size, nonlinearity, dtype=dtype + ) + cell_bw = SimpleRNNCell( + 2 * hidden_size, hidden_size, nonlinearity, dtype=dtype + ) self.append(BiRNN(cell_fw, cell_bw, time_major)) else: raise ValueError( "direction should be forward, backward or bidirectional, " - "received direction = {}".format(direction)) + "received direction = {}".format(direction) + ) self.input_size = input_size self.hidden_size = hidden_size @@ -472,15 +495,16 @@ def __init__(self, class LSTM(RNNMixin): - - def __init__(self, - input_size, - hidden_size, - num_layers=1, - direction="forward", - dropout=0., - time_major=False, - dtype="float64"): + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + direction="forward", + dropout=0.0, + time_major=False, + dtype="float64", + ): super(LSTM, self).__init__() bidirectional_list = ["bidirectional", "bidirect"] @@ -502,7 +526,8 @@ def __init__(self, else: raise ValueError( "direction should be forward, backward or bidirectional, " - "received direction = {}".format(direction)) + "received direction = {}".format(direction) + ) self.input_size = input_size self.hidden_size = hidden_size @@ -514,15 +539,16 @@ def __init__(self, class GRU(RNNMixin): - - def __init__(self, - input_size, - hidden_size, - num_layers=1, - direction="forward", - dropout=0., - time_major=False, - dtype="float64"): + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + direction="forward", + dropout=0.0, + time_major=False, + dtype="float64", + ): super(GRU, self).__init__() bidirectional_list = ["bidirectional", "bidirect"] @@ -544,7 +570,8 @@ def __init__(self, else: raise ValueError( "direction should be forward, backward or bidirectional, " - "received direction = {}".format(direction)) + "received direction = {}".format(direction) + ) self.input_size = input_size self.hidden_size = hidden_size diff --git a/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells.py b/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells.py index 33dca32b76cd5..ee2e7cfaa0f9b 100644 --- a/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells.py +++ b/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells.py @@ -24,20 +24,19 @@ class TestSimpleRNNCell(unittest.TestCase): - def __init__(self, bias=True, place="cpu"): super(TestSimpleRNNCell, self).__init__(methodName="runTest") self.bias = bias - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): paddle.disable_static(self.place) rnn1 = SimpleRNNCell(16, 32, bias=self.bias) - rnn2 = paddle.nn.SimpleRNNCell(16, - 32, - bias_ih_attr=self.bias, - bias_hh_attr=self.bias) + rnn2 = paddle.nn.SimpleRNNCell( + 16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias + ) convert_params_for_cell(rnn1, rnn2) self.rnn1 = rnn1 @@ -65,7 +64,6 @@ def test_with_zero_state(self): np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5) def test_errors(self): - def test_zero_hidden_size(): cell = paddle.nn.SimpleRNNCell(-1, 0) @@ -78,20 +76,19 @@ def runTest(self): class TestGRUCell(unittest.TestCase): - def __init__(self, bias=True, place="cpu"): super(TestGRUCell, self).__init__(methodName="runTest") self.bias = bias - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): paddle.disable_static(self.place) rnn1 = GRUCell(16, 32, bias=self.bias) - rnn2 = paddle.nn.GRUCell(16, - 32, - bias_ih_attr=self.bias, - bias_hh_attr=self.bias) + rnn2 = paddle.nn.GRUCell( + 16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias + ) convert_params_for_cell(rnn1, rnn2) self.rnn1 = rnn1 @@ -119,7 +116,6 @@ def test_with_zero_state(self): np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5) def test_errors(self): - def test_zero_hidden_size(): cell = paddle.nn.GRUCell(-1, 0) @@ -132,19 +128,18 @@ def runTest(self): class TestLSTMCell(unittest.TestCase): - def __init__(self, bias=True, place="cpu"): super(TestLSTMCell, self).__init__(methodName="runTest") self.bias = bias - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): rnn1 = LSTMCell(16, 32, bias=self.bias) - rnn2 = paddle.nn.LSTMCell(16, - 32, - bias_ih_attr=self.bias, - bias_hh_attr=self.bias) + rnn2 = paddle.nn.LSTMCell( + 16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias + ) convert_params_for_cell(rnn1, rnn2) self.rnn1 = rnn1 @@ -159,9 +154,10 @@ def test_with_initial_state(self): prev_c = np.random.randn(4, 32) y1, (h1, c1) = rnn1(x, (prev_h, prev_c)) - y2, (h2, - c2) = rnn2(paddle.to_tensor(x), - (paddle.to_tensor(prev_h), paddle.to_tensor(prev_c))) + y2, (h2, c2) = rnn2( + paddle.to_tensor(x), + (paddle.to_tensor(prev_h), paddle.to_tensor(prev_c)), + ) np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5) np.testing.assert_allclose(c1, c2.numpy(), atol=1e-8, rtol=1e-5) @@ -177,7 +173,6 @@ def test_with_zero_state(self): np.testing.assert_allclose(c1, c2.numpy(), atol=1e-8, rtol=1e-5) def test_errors(self): - def test_zero_hidden_size(): cell = paddle.nn.LSTMCell(-1, 0) @@ -191,8 +186,9 @@ def runTest(self): def load_tests(loader, tests, pattern): suite = unittest.TestSuite() - devices = ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() \ - else ["cpu"] + devices = ( + ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() else ["cpu"] + ) for bias in [True, False]: for device in devices: for test_class in [TestSimpleRNNCell, TestGRUCell, TestLSTMCell]: diff --git a/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells_static.py b/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells_static.py index b4a5887c593bd..a9e2a013a25d5 100644 --- a/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells_static.py +++ b/python/paddle/fluid/tests/unittests/rnn/test_rnn_cells_static.py @@ -25,12 +25,12 @@ class TestSimpleRNNCell(unittest.TestCase): - def __init__(self, bias=True, place="cpu"): super(TestSimpleRNNCell, self).__init__(methodName="runTest") self.bias = bias - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): rnn1 = SimpleRNNCell(16, 32, bias=self.bias) @@ -39,10 +39,9 @@ def setUp(self): sp = paddle.static.Program() with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): - rnn2 = paddle.nn.SimpleRNNCell(16, - 32, - bias_ih_attr=self.bias, - bias_hh_attr=self.bias) + rnn2 = paddle.nn.SimpleRNNCell( + 16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias + ) place = self.place exe = paddle.static.Executor(place) @@ -75,11 +74,15 @@ def test_with_initial_state(self): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, 16], + dtype=paddle.framework.get_default_dtype(), + ) init_h = paddle.fluid.data( - "init_h", [-1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_h", + [-1, 32], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data, init_h) feed_dict = {x_data.name: x, init_h.name: prev_h} @@ -103,17 +106,18 @@ def test_with_zero_state(self): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, 16], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data) feed_dict = {x_data.name: x} with paddle.static.scope_guard(scope): - y2, h2 = exe.run(mp, - feed=feed_dict, - fetch_list=[y, h], - use_prune=True) + y2, h2 = exe.run( + mp, feed=feed_dict, fetch_list=[y, h], use_prune=True + ) np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5) @@ -123,12 +127,12 @@ def runTest(self): class TestGRUCell(unittest.TestCase): - def __init__(self, bias=True, place="cpu"): super(TestGRUCell, self).__init__(methodName="runTest") self.bias = bias - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): rnn1 = GRUCell(16, 32, bias=self.bias) @@ -137,10 +141,9 @@ def setUp(self): sp = paddle.static.Program() with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): - rnn2 = paddle.nn.GRUCell(16, - 32, - bias_ih_attr=self.bias, - bias_hh_attr=self.bias) + rnn2 = paddle.nn.GRUCell( + 16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias + ) place = self.place exe = paddle.static.Executor(place) @@ -174,11 +177,15 @@ def test_with_initial_state(self): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, 16], + dtype=paddle.framework.get_default_dtype(), + ) init_h = paddle.fluid.data( - "init_h", [-1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_h", + [-1, 32], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data, init_h) feed_dict = {x_data.name: x, init_h.name: prev_h} @@ -202,17 +209,18 @@ def test_with_zero_state(self): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, 16], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data) feed_dict = {x_data.name: x} with paddle.static.scope_guard(scope): - y2, h2 = exe.run(mp, - feed=feed_dict, - fetch_list=[y, h], - use_prune=True) + y2, h2 = exe.run( + mp, feed=feed_dict, fetch_list=[y, h], use_prune=True + ) np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5) @@ -222,12 +230,12 @@ def runTest(self): class TestLSTMCell(unittest.TestCase): - def __init__(self, bias=True, place="cpu"): super(TestLSTMCell, self).__init__(methodName="runTest") self.bias = bias - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): rnn1 = LSTMCell(16, 32, bias=self.bias) @@ -236,10 +244,9 @@ def setUp(self): sp = paddle.static.Program() with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): - rnn2 = paddle.nn.LSTMCell(16, - 32, - bias_ih_attr=self.bias, - bias_hh_attr=self.bias) + rnn2 = paddle.nn.LSTMCell( + 16, 32, bias_ih_attr=self.bias, bias_hh_attr=self.bias + ) place = self.place exe = paddle.static.Executor(place) @@ -274,14 +281,20 @@ def test_with_initial_state(self): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, 16], + dtype=paddle.framework.get_default_dtype(), + ) init_h = paddle.fluid.data( - "init_h", [-1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_h", + [-1, 32], + dtype=paddle.framework.get_default_dtype(), + ) init_c = paddle.fluid.data( - "init_c", [-1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_c", + [-1, 32], + dtype=paddle.framework.get_default_dtype(), + ) y, (h, c) = rnn2(x_data, (init_h, init_c)) feed_dict = {x_data.name: x, init_h.name: prev_h, init_c.name: prev_c} @@ -306,17 +319,18 @@ def test_with_zero_state(self): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, 16], + dtype=paddle.framework.get_default_dtype(), + ) y, (h, c) = rnn2(x_data) feed_dict = {x_data.name: x} with paddle.static.scope_guard(scope): - y2, h2, c2 = exe.run(mp, - feed=feed_dict, - fetch_list=[y, h, c], - use_prune=True) + y2, h2, c2 = exe.run( + mp, feed=feed_dict, fetch_list=[y, h, c], use_prune=True + ) np.testing.assert_allclose(h1, h2, atol=1e-8, rtol=1e-5) np.testing.assert_allclose(c1, c2, atol=1e-8, rtol=1e-5) @@ -328,8 +342,9 @@ def runTest(self): def load_tests(loader, tests, pattern): suite = unittest.TestSuite() - devices = ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() \ - else ["cpu"] + devices = ( + ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() else ["cpu"] + ) for bias in [True, False]: for device in devices: for test_class in [TestSimpleRNNCell, TestGRUCell, TestLSTMCell]: diff --git a/python/paddle/fluid/tests/unittests/rnn/test_rnn_cudnn_params_packing.py b/python/paddle/fluid/tests/unittests/rnn/test_rnn_cudnn_params_packing.py index f4dbc3bbbc7c8..5949addbea16c 100644 --- a/python/paddle/fluid/tests/unittests/rnn/test_rnn_cudnn_params_packing.py +++ b/python/paddle/fluid/tests/unittests/rnn/test_rnn_cudnn_params_packing.py @@ -18,15 +18,13 @@ def create_model(): hidden_size = 32 - bilstm = paddle.nn.LSTM(hidden_size, - hidden_size, - num_layers=1, - direction='bidirectional') + bilstm = paddle.nn.LSTM( + hidden_size, hidden_size, num_layers=1, direction='bidirectional' + ) return bilstm class TestRNNProgramClone(TestCase): - def setUp(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets.py b/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets.py index bf33d5532014f..0e3407631dcc6 100755 --- a/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets.py +++ b/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets.py @@ -29,7 +29,6 @@ class TestSimpleRNN(unittest.TestCase): - def __init__(self, time_major=True, direction="forward", place="cpu"): super(TestSimpleRNN, self).__init__("runTest") self.time_major = time_major @@ -42,16 +41,12 @@ def setUp(self): # `__init__` to avoid using an error device set by another test case. place = paddle.set_device(self.place) paddle.disable_static(place) - rnn1 = SimpleRNN(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) - rnn2 = paddle.nn.SimpleRNN(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn1 = SimpleRNN( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) + rnn2 = paddle.nn.SimpleRNN( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) convert_params_for_net(rnn1, rnn2) self.rnn1 = rnn1 @@ -117,7 +112,6 @@ def runTest(self): class TestGRU(unittest.TestCase): - def __init__(self, time_major=True, direction="forward", place="cpu"): super(TestGRU, self).__init__("runTest") self.time_major = time_major @@ -130,16 +124,12 @@ def setUp(self): # `__init__` to avoid using an error device set by another test case. place = paddle.set_device(self.place) paddle.disable_static(place) - rnn1 = GRU(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) - rnn2 = paddle.nn.GRU(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn1 = GRU( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) + rnn2 = paddle.nn.GRU( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) convert_params_for_net(rnn1, rnn2) self.rnn1 = rnn1 @@ -205,7 +195,6 @@ def runTest(self): class TestLSTM(unittest.TestCase): - def __init__(self, time_major=True, direction="forward", place="cpu"): super(TestLSTM, self).__init__("runTest") self.time_major = time_major @@ -218,16 +207,12 @@ def setUp(self): # `__init__` to avoid using an error device set by another test case. place = paddle.set_device(self.place) paddle.disable_static(place) - rnn1 = LSTM(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) - rnn2 = paddle.nn.LSTM(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn1 = LSTM( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) + rnn2 = paddle.nn.LSTM( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) convert_params_for_net(rnn1, rnn2) self.rnn1 = rnn1 @@ -244,9 +229,10 @@ def test_with_initial_state(self): prev_c = np.random.randn(2 * self.num_directions, 4, 32) y1, (h1, c1) = rnn1(x, (prev_h, prev_c)) - y2, (h2, - c2) = rnn2(paddle.to_tensor(x), - (paddle.to_tensor(prev_h), paddle.to_tensor(prev_c))) + y2, (h2, c2) = rnn2( + paddle.to_tensor(x), + (paddle.to_tensor(prev_h), paddle.to_tensor(prev_c)), + ) np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5) np.testing.assert_allclose(h1, h2.numpy(), atol=1e-8, rtol=1e-5) np.testing.assert_allclose(c1, c2.numpy(), atol=1e-8, rtol=1e-5) @@ -305,14 +291,11 @@ def predict_test_util(place, mode, stop_gradient=True): np.random.seed(123) class Net(paddle.nn.Layer): - def __init__(self): super(Net, self).__init__() - self.rnn = getattr(paddle.nn, mode)(16, - 32, - 2, - direction="bidirectional", - dropout=0.1) + self.rnn = getattr(paddle.nn, mode)( + 16, 32, 2, direction="bidirectional", dropout=0.1 + ) def forward(self, input): return self.rnn(input) @@ -327,8 +310,9 @@ def forward(self, input): y = y * mask loss = paddle.mean(y) loss.backward() - optimizer = paddle.optimizer.Adam(learning_rate=0.1, - parameters=rnn.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=0.1, parameters=rnn.parameters() + ) optimizer.step() rnn.eval() y, _ = rnn(x) @@ -337,7 +321,8 @@ def forward(self, input): rnn.train() rnn = paddle.jit.to_static( - rnn, [paddle.static.InputSpec(shape=[None, None, 16], dtype=x.dtype)]) + rnn, [paddle.static.InputSpec(shape=[None, None, 16], dtype=x.dtype)] + ) temp_dir = tempfile.TemporaryDirectory() save_dirname = os.path.join(temp_dir.name, "./inference/%s_infer" % mode) @@ -348,13 +333,19 @@ def forward(self, input): new_scope = paddle.static.Scope() with paddle.static.scope_guard(new_scope): exe = paddle.static.Executor(place) - [inference_program, feed_target_names, - fetch_targets] = paddle.static.load_inference_model(save_dirname, exe) - results = exe.run(inference_program, - feed={feed_target_names[0]: x.numpy()}, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(save_dirname, exe) + results = exe.run( + inference_program, + feed={feed_target_names[0]: x.numpy()}, + fetch_list=fetch_targets, + ) np.testing.assert_equal( - y.numpy(), results[0]) # eval results equal predict results + y.numpy(), results[0] + ) # eval results equal predict results paddle.disable_static() temp_dir.cleanup() @@ -362,8 +353,9 @@ def forward(self, input): def load_tests(loader, tests, pattern): suite = unittest.TestSuite() - devices = ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() \ - else ["cpu"] + devices = ( + ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() else ["cpu"] + ) for direction in ["forward", "bidirectional", "bidirect"]: for time_major in [True, False]: for device in devices: diff --git a/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets_static.py b/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets_static.py index 436bf0b6ea01d..e186f28c35cf9 100755 --- a/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets_static.py +++ b/python/paddle/fluid/tests/unittests/rnn/test_rnn_nets_static.py @@ -29,7 +29,6 @@ class TestSimpleRNN(unittest.TestCase): - def __init__(self, time_major=True, direction="forward", place="cpu"): super(TestSimpleRNN, self).__init__("runTest") self.time_major = time_major @@ -41,21 +40,21 @@ def setUp(self): # Since `set_device` is global, set `set_device` in `setUp` rather than # `__init__` to avoid using an error device set by another test case. place = paddle.set_device(self.place) - rnn1 = SimpleRNN(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn1 = SimpleRNN( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) mp = paddle.static.Program() sp = paddle.static.Program() with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): - rnn2 = paddle.nn.SimpleRNN(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn2 = paddle.nn.SimpleRNN( + 16, + 32, + 2, + time_major=self.time_major, + direction=self.direction, + ) exe = paddle.static.Executor(place) scope = paddle.fluid.Scope() @@ -90,11 +89,15 @@ def test_with_initial_state(self): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) init_h = paddle.fluid.data( - "init_h", [2 * self.num_directions, -1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_h", + [2 * self.num_directions, -1, 32], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data, init_h) feed_dict = {x_data.name: x, init_h.name: prev_h} @@ -121,8 +124,10 @@ def test_with_zero_state(self): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data) feed_dict = {x_data.name: x} @@ -151,8 +156,10 @@ def test_with_input_lengths(self): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) seq_len = paddle.fluid.data("seq_len", [-1], dtype="int64") mask = sequence_mask(seq_len, dtype=paddle.get_default_dtype()) if self.time_major: @@ -176,7 +183,6 @@ def runTest(self): class TestGRU(unittest.TestCase): - def __init__(self, time_major=True, direction="forward", place="cpu"): super(TestGRU, self).__init__("runTest") self.time_major = time_major @@ -188,21 +194,21 @@ def setUp(self): # Since `set_device` is global, set `set_device` in `setUp` rather than # `__init__` to avoid using an error device set by another test case. place = paddle.set_device(self.place) - rnn1 = GRU(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn1 = GRU( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) mp = paddle.static.Program() sp = paddle.static.Program() with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): - rnn2 = paddle.nn.GRU(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn2 = paddle.nn.GRU( + 16, + 32, + 2, + time_major=self.time_major, + direction=self.direction, + ) exe = paddle.static.Executor(place) scope = paddle.fluid.Scope() @@ -238,11 +244,15 @@ def test_with_initial_state(self): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) init_h = paddle.fluid.data( - "init_h", [2 * self.num_directions, -1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_h", + [2 * self.num_directions, -1, 32], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data, init_h) feed_dict = {x_data.name: x, init_h.name: prev_h} @@ -269,8 +279,10 @@ def test_with_zero_state(self): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) y, h = rnn2(x_data) feed_dict = {x_data.name: x} @@ -299,8 +311,10 @@ def test_with_input_lengths(self): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) seq_len = paddle.fluid.data("seq_len", [-1], dtype="int64") mask = sequence_mask(seq_len, dtype=paddle.get_default_dtype()) if self.time_major: @@ -323,7 +337,6 @@ def runTest(self): class TestLSTM(unittest.TestCase): - def __init__(self, time_major=True, direction="forward", place="cpu"): super(TestLSTM, self).__init__("runTest") self.time_major = time_major @@ -335,21 +348,21 @@ def setUp(self): # Since `set_device` is global, set `set_device` in `setUp` rather than # `__init__` to avoid using an error device set by another test case. place = paddle.set_device(self.place) - rnn1 = LSTM(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn1 = LSTM( + 16, 32, 2, time_major=self.time_major, direction=self.direction + ) mp = paddle.static.Program() sp = paddle.static.Program() with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): - rnn2 = paddle.nn.LSTM(16, - 32, - 2, - time_major=self.time_major, - direction=self.direction) + rnn2 = paddle.nn.LSTM( + 16, + 32, + 2, + time_major=self.time_major, + direction=self.direction, + ) exe = paddle.static.Executor(place) scope = paddle.fluid.Scope() @@ -385,14 +398,20 @@ def test_with_initial_state(self): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) init_h = paddle.fluid.data( - "init_h", [2 * self.num_directions, -1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_h", + [2 * self.num_directions, -1, 32], + dtype=paddle.framework.get_default_dtype(), + ) init_c = paddle.fluid.data( - "init_c", [2 * self.num_directions, -1, 32], - dtype=paddle.framework.get_default_dtype()) + "init_c", + [2 * self.num_directions, -1, 32], + dtype=paddle.framework.get_default_dtype(), + ) y, (h, c) = rnn2(x_data, (init_h, init_c)) feed_dict = {x_data.name: x, init_h.name: prev_h, init_c.name: prev_c} @@ -420,8 +439,10 @@ def test_with_zero_state(self): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) y, (h, c) = rnn2(x_data) feed_dict = {x_data.name: x} @@ -451,8 +472,10 @@ def test_with_input_lengths(self): with paddle.fluid.unique_name.guard(): with paddle.static.program_guard(mp, sp): x_data = paddle.fluid.data( - "input", [-1, -1, 16], - dtype=paddle.framework.get_default_dtype()) + "input", + [-1, -1, 16], + dtype=paddle.framework.get_default_dtype(), + ) seq_len = paddle.fluid.data("seq_len", [-1], dtype="int64") mask = sequence_mask(seq_len, dtype=paddle.get_default_dtype()) if self.time_major: @@ -478,8 +501,9 @@ def runTest(self): def load_tests(loader, tests, pattern): suite = unittest.TestSuite() - devices = ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() \ - else ["cpu"] + devices = ( + ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() else ["cpu"] + ) for direction in ["forward", "bidirectional", "bidirect"]: for time_major in [True, False]: for device in devices: diff --git a/python/paddle/fluid/tests/unittests/rnn/test_wrappers.py b/python/paddle/fluid/tests/unittests/rnn/test_wrappers.py index 2442e6b7a3b8c..27e1293cb6895 100755 --- a/python/paddle/fluid/tests/unittests/rnn/test_wrappers.py +++ b/python/paddle/fluid/tests/unittests/rnn/test_wrappers.py @@ -25,25 +25,29 @@ class TestRNNWrapper(unittest.TestCase): - def __init__(self, time_major=True, direction="forward", place="cpu"): super(TestRNNWrapper, self).__init__("runTest") self.time_major = time_major self.direction = direction - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): paddle.disable_static(self.place) cell1 = GRUCell(16, 32) cell2 = paddle.nn.GRUCell(16, 32) convert_params_for_cell(cell1, cell2) - rnn1 = RNN(cell1, - is_reverse=self.direction == "backward", - time_major=self.time_major) - rnn2 = paddle.nn.RNN(cell2, - is_reverse=self.direction == "backward", - time_major=self.time_major) + rnn1 = RNN( + cell1, + is_reverse=self.direction == "backward", + time_major=self.time_major, + ) + rnn2 = paddle.nn.RNN( + cell2, + is_reverse=self.direction == "backward", + time_major=self.time_major, + ) self.rnn1 = rnn1 self.rnn2 = rnn2 @@ -104,12 +108,12 @@ def runTest(self): class TestBiRNNWrapper(unittest.TestCase): - def __init__(self, time_major=True, place="cpu"): super(TestBiRNNWrapper, self).__init__("runTest") self.time_major = time_major - self.place = paddle.CPUPlace() if place == "cpu" \ - else paddle.CUDAPlace(0) + self.place = ( + paddle.CPUPlace() if place == "cpu" else paddle.CUDAPlace(0) + ) def setUp(self): paddle.disable_static(self.place) @@ -138,7 +142,8 @@ def test_with_initial_state(self): y1, (fw_h1, bw_h1) = rnn1(x, (fw_prev_h, bw_prev_h)) y2, (fw_h2, bw_h2) = rnn2( paddle.to_tensor(x), - (paddle.to_tensor(fw_prev_h), paddle.to_tensor(bw_prev_h))) + (paddle.to_tensor(fw_prev_h), paddle.to_tensor(bw_prev_h)), + ) np.testing.assert_allclose(y1, y2.numpy(), atol=1e-8, rtol=1e-5) np.testing.assert_allclose(fw_h1, fw_h2.numpy(), atol=1e-8, rtol=1e-5) np.testing.assert_allclose(bw_h1, bw_h2.numpy(), atol=1e-8, rtol=1e-5) @@ -188,8 +193,9 @@ def runTest(self): def load_tests(loader, tests, pattern): suite = unittest.TestSuite() - devices = ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() \ - else ["cpu"] + devices = ( + ["cpu", "gpu"] if paddle.fluid.is_compiled_with_cuda() else ["cpu"] + ) for direction in ["forward", "backward"]: for device in devices: for time_major in [False]: diff --git a/python/paddle/fluid/tests/unittests/rpc/rpc_launch_sync_add.py b/python/paddle/fluid/tests/unittests/rpc/rpc_launch_sync_add.py index 3294f159b7770..794efb22fc7ae 100644 --- a/python/paddle/fluid/tests/unittests/rpc/rpc_launch_sync_add.py +++ b/python/paddle/fluid/tests/unittests/rpc/rpc_launch_sync_add.py @@ -59,11 +59,11 @@ def main(): shape=(10 * world_size, 100), ) for i in range(world_size): - a = mmap_data1[i * 10:(i + 1) * 10, :] - b = mmap_data2[i * 10:(i + 1) * 10, :] + a = mmap_data1[i * 10 : (i + 1) * 10, :] + b = mmap_data2[i * 10 : (i + 1) * 10, :] args = (a, b) out = rpc_add(worker_name(i), args) - mmap_out[i * 10:(i + 1) * 10, :] = out[:] + mmap_out[i * 10 : (i + 1) * 10, :] = out[:] dist.rpc.shutdown() diff --git a/python/paddle/fluid/tests/unittests/rpc/test_rpc.py b/python/paddle/fluid/tests/unittests/rpc/test_rpc.py index c6c2eb43be204..67e8b65423c94 100644 --- a/python/paddle/fluid/tests/unittests/rpc/test_rpc.py +++ b/python/paddle/fluid/tests/unittests/rpc/test_rpc.py @@ -35,7 +35,6 @@ def paddle_add(a, b): class TestMultiProcessRpc(RpcTestBase): - def test_one_server_sync_paddle_add(self): a = np.random.random((10, 100)) b = np.random.random((10, 100)) @@ -78,7 +77,6 @@ def test_two_server_async_paddle_add(self): class TestSingleProcessRpc(RpcTestBase): - def setUp(self): self._port_set = set() master_endpoint = "127.0.0.1:{}".format(self._find_free_port()) @@ -123,7 +121,6 @@ def test_get_current_worker_info(self): class RpcLaunchTest(RpcLaunchTestBase): - def test_sync_rpc_paddle_add1(self): nnodes = 2 nproc_per_node = 1 diff --git a/python/paddle/fluid/tests/unittests/rpc/test_rpc_base.py b/python/paddle/fluid/tests/unittests/rpc/test_rpc_base.py index 95a59267d530c..daf50b8884505 100644 --- a/python/paddle/fluid/tests/unittests/rpc/test_rpc_base.py +++ b/python/paddle/fluid/tests/unittests/rpc/test_rpc_base.py @@ -64,10 +64,9 @@ def run_rpc_sync_master_working( ) if dist.get_rank() == 0: for i in range(1, dist.get_rank()): - res = dist.rpc.rpc_sync(worker_name(i), - fn, - args=args, - kwargs=kwargs) + res = dist.rpc.rpc_sync( + worker_name(i), fn, args=args, kwargs=kwargs + ) queue.put(res) dist.rpc.shutdown() @@ -109,16 +108,14 @@ def run_rpc_async_master_working( ) if dist.get_rank() == 0: for i in range(1, dist.get_rank()): - res = dist.rpc.rpc_async(worker_name(i), - fn, - args=args, - kwargs=kwargs) + res = dist.rpc.rpc_async( + worker_name(i), fn, args=args, kwargs=kwargs + ) queue.put(res.wait()) dist.rpc.shutdown() class RpcTestBase(unittest.TestCase): - def setUp(self): self._port_set = set() print("RPC setUp...") @@ -129,10 +126,10 @@ def tearDown(self): print("RPC tearDown...") def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(("", 0)) return s.getsockname()[1] @@ -162,7 +159,8 @@ def run_rpc(self, sync, world_size, fn, fn_args=None, fn_kwargs=None): fn_args, fn_kwargs, ), - )) + ) + ) else: self.processes.append( Process( @@ -176,13 +174,13 @@ def run_rpc(self, sync, world_size, fn, fn_args=None, fn_kwargs=None): fn_args, fn_kwargs, ), - )) + ) + ) [p.start() for p in self.processes] return queues class RpcLaunchTestBase(unittest.TestCase): - def setUp(self): self._port_set = set() print("Launch RPC setUp...") @@ -192,10 +190,10 @@ def tearDown(self): print("Launch RPC tearDown...") def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(("", 0)) return s.getsockname()[1] @@ -221,8 +219,8 @@ def create_data(self, nnodes, nproc_per_node): for i in range(nnodes * nproc_per_node): a = np.random.random((10, 100)).astype(np.float32) b = np.random.random((10, 100)).astype(np.float32) - mmap_data1[i * 10:(i + 1) * 10, :] = a - mmap_data2[i * 10:(i + 1) * 10, :] = b + mmap_data1[i * 10 : (i + 1) * 10, :] = a + mmap_data2[i * 10 : (i + 1) * 10, :] = b return mmap_data1, mmap_data2 def remove_data(self): @@ -234,8 +232,15 @@ def launch_rpc(self, nnodes, nproc_per_node, model_file): log_dir = "log" tr_cmd = "python -m paddle.distributed.launch --master {} --rank {} --nnodes {} --nproc_per_node {} --run_mode rpc {} --log_dir {}" cmds = [ - tr_cmd.format(master_endpoint, rank, nnodes, nproc_per_node, - model_file, log_dir) for rank in range(nnodes) + tr_cmd.format( + master_endpoint, + rank, + nnodes, + nproc_per_node, + model_file, + log_dir, + ) + for rank in range(nnodes) ] processes = [subprocess.Popen(cmd.strip().split()) for cmd in cmds] [proc.communicate() for proc in processes] diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_concat.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_concat.py index 4b60a0cbcfc51..80c975221ad49 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_concat.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_concat.py @@ -24,7 +24,6 @@ class TestSequenceConcat(OpTest): - def setLoD(self): self.lod1 = [7, 3] self.lod2 = [12, 8] @@ -35,8 +34,14 @@ def setUp(self): x2 = np.random.random(size=(20, 80)).astype('float64') self.setLoD() - out = np.concatenate((x1[0:self.lod1[0]], x2[0:self.lod2[0]], - x1[self.lod1[0]:], x2[self.lod2[0]:])) + out = np.concatenate( + ( + x1[0 : self.lod1[0]], + x2[0 : self.lod2[0]], + x1[self.lod1[0] :], + x2[self.lod2[0] :], + ) + ) self.op_type = "sequence_concat" self.inputs = { @@ -52,7 +57,6 @@ def test_dx(self): class TestSequenceConcatCase2(TestSequenceConcat): - def setLoD(self): self.lod1 = [10, 0] self.lod2 = [12, 8] @@ -60,7 +64,6 @@ def setLoD(self): class TestSequenceConcatCase3(TestSequenceConcat): - def setLoD(self): self.lod1 = [10, 0] self.lod2 = [20, 0] @@ -68,7 +71,6 @@ def setLoD(self): class TestSequenceConcatCase4(TestSequenceConcat): - def setLoD(self): self.lod1 = [0, 10] self.lod2 = [0, 20] @@ -76,7 +78,6 @@ def setLoD(self): class TestSequenceConcatCase5(TestSequenceConcat): - def setLoD(self): self.lod1 = [0, 10] self.lod2 = [20, 0] @@ -84,9 +85,7 @@ def setLoD(self): class TestSequenceConcatOpError(unittest.TestCase): - def test_errors(self): - def test_input_list(): # the input type must be list x_data = fluid.layers.data(name='x', shape=[4], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_conv.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_conv.py index 047ec6e294358..78fc6089cfe73 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_conv.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_conv.py @@ -21,13 +21,15 @@ from op_test import OpTest -def seqconv(x, - lod, - filter, - context_length, - context_start, - padding_trainable=False, - padding_data=None): +def seqconv( + x, + lod, + filter, + context_length, + context_start, + padding_trainable=False, + padding_data=None, +): [T, M] = x.shape col = np.zeros((T, context_length * M)).astype('float32') offset = [0] @@ -42,58 +44,79 @@ def seqconv(x, out_end = offset[i + 1] if in_begin < offset[i]: pad_size = np.min( - [offset[i] - in_begin, offset[i + 1] - offset[i]]) + [offset[i] - in_begin, offset[i + 1] - offset[i]] + ) if padding_trainable: - sub_w = padding_data[j:j + pad_size, :] - col[offset[i]:offset[i] + pad_size, - j * M:(j + 1) * M] = sub_w + sub_w = padding_data[j : j + pad_size, :] + col[ + offset[i] : offset[i] + pad_size, j * M : (j + 1) * M + ] = sub_w out_begin = offset[i] + pad_size in_begin = offset[i] if in_end > offset[i + 1]: pad_size = np.min( - [in_end - offset[i + 1], offset[i + 1] - offset[i]]) + [in_end - offset[i + 1], offset[i + 1] - offset[i]] + ) if padding_trainable: - sub_w = padding_data[begin_pad + context_start + j - - pad_size:begin_pad + context_start + - j, :] - col[offset[i + 1] - pad_size:offset[i + 1], - j * M:(j + 1) * M] = sub_w + sub_w = padding_data[ + begin_pad + + context_start + + j + - pad_size : begin_pad + + context_start + + j, + :, + ] + col[ + offset[i + 1] - pad_size : offset[i + 1], + j * M : (j + 1) * M, + ] = sub_w in_end = offset[i + 1] out_end = offset[i + 1] - pad_size if in_end <= in_begin: continue in_sub = x[in_begin:in_end, :] - col[out_begin:out_end, j * M:(j + 1) * M] += in_sub + col[out_begin:out_end, j * M : (j + 1) * M] += in_sub return np.dot(col, filter) class TestSeqProject(OpTest): - def setUp(self): self.init_test_case() self.op_type = 'sequence_conv' - if self.context_length == 1 \ - and self.context_start == 0 \ - and self.padding_trainable: - print("If context_start is 0 " \ - "and context_length is 1," \ - " padding_trainable should be false.") + if ( + self.context_length == 1 + and self.context_start == 0 + and self.padding_trainable + ): + print( + "If context_start is 0 " + "and context_length is 1," + " padding_trainable should be false." + ) return # one level, batch size x = np.random.uniform( - 0.1, 1, [self.input_size[0], self.input_size[1]]).astype('float32') - w = np.random.uniform(0.1, 1, [ - self.context_length * self.input_size[1], self.output_represention - ]).astype('float32') + 0.1, 1, [self.input_size[0], self.input_size[1]] + ).astype('float32') + w = np.random.uniform( + 0.1, + 1, + [ + self.context_length * self.input_size[1], + self.output_represention, + ], + ).astype('float32') begin_pad = np.max([0, -self.context_start]) end_pad = np.max([0, self.context_start + self.context_length - 1]) total_pad = begin_pad + end_pad padding_data = np.random.uniform( - 0.1, 1, [total_pad, self.input_size[1]]).astype('float32') + 0.1, 1, [total_pad, self.input_size[1]] + ).astype('float32') self.pad_data = padding_data self.inputs = { 'X': (x, self.lod), @@ -113,10 +136,17 @@ def setUp(self): 'contextStart': self.context_start, 'contextLength': self.context_length, 'paddingTrainable': self.padding_trainable, - 'contextStride': self.context_stride + 'contextStride': self.context_stride, } - out = seqconv(x, self.lod, w, self.context_length, self.context_start, - self.padding_trainable, self.pad_data) + out = seqconv( + x, + self.lod, + w, + self.context_length, + self.context_start, + self.padding_trainable, + self.pad_data, + ) self.outputs = {'Out': out} def test_check_output(self): @@ -124,48 +154,58 @@ def test_check_output(self): def test_check_grad(self): if self.padding_trainable: - self.check_grad(set(self.inputs_val), - 'Out', - max_relative_error=0.05) + self.check_grad( + set(self.inputs_val), 'Out', max_relative_error=0.05 + ) def test_check_grad_input(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.05, - no_grad_set=set(self.inputs_val_no_x)) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.05, + no_grad_set=set(self.inputs_val_no_x), + ) def test_check_grad_padding_data(self): if self.padding_trainable: - self.check_grad(['PaddingData'], - 'Out', - no_grad_set=set(['X', 'Filter'])) + self.check_grad( + ['PaddingData'], 'Out', no_grad_set=set(['X', 'Filter']) + ) def test_check_grad_Filter(self): - self.check_grad(['Filter'], - 'Out', - max_relative_error=0.05, - no_grad_set=set(self.inputs_val_no_f)) + self.check_grad( + ['Filter'], + 'Out', + max_relative_error=0.05, + no_grad_set=set(self.inputs_val_no_f), + ) def test_check_grad_input_filter(self): if self.padding_trainable: - self.check_grad(['X', 'Filter'], - 'Out', - max_relative_error=0.05, - no_grad_set=set(['PaddingData'])) + self.check_grad( + ['X', 'Filter'], + 'Out', + max_relative_error=0.05, + no_grad_set=set(['PaddingData']), + ) def test_check_grad_padding_input(self): if self.padding_trainable: - self.check_grad(self.inputs_val_no_f, - 'Out', - max_relative_error=0.05, - no_grad_set=set(['Filter'])) + self.check_grad( + self.inputs_val_no_f, + 'Out', + max_relative_error=0.05, + no_grad_set=set(['Filter']), + ) def test_check_grad_padding_filter(self): if self.padding_trainable: - self.check_grad(self.inputs_val_no_x, - 'Out', - max_relative_error=0.05, - no_grad_set=set(['X'])) + self.check_grad( + self.inputs_val_no_x, + 'Out', + max_relative_error=0.05, + no_grad_set=set(['X']), + ) def init_test_case(self): self.input_row = 11 @@ -184,7 +224,6 @@ def init_test_case(self): class TestSeqProjectCase1(TestSeqProject): - def init_test_case(self): self.input_row = 11 self.context_start = -1 @@ -202,7 +241,6 @@ def init_test_case(self): class TestSeqProjectCase2Len0(TestSeqProject): - def init_test_case(self): self.input_row = 11 self.context_start = -1 @@ -220,7 +258,6 @@ def init_test_case(self): class TestSeqProjectCase3(TestSeqProject): - def init_test_case(self): self.input_row = 25 self.context_start = 2 @@ -231,8 +268,9 @@ def init_test_case(self): self.input_size = [self.input_row, 25] idx = list(range(self.input_size[0])) del idx[0] - offset_lod = [[0] + np.sort(random.sample(idx, 8)).tolist() + - [self.input_size[0]]] + offset_lod = [ + [0] + np.sort(random.sample(idx, 8)).tolist() + [self.input_size[0]] + ] self.lod = [[]] # convert from offset-based lod to length-based lod for i in range(len(offset_lod[0]) - 1): @@ -241,19 +279,18 @@ def init_test_case(self): class TestSeqConvApi(unittest.TestCase): - def test_api(self): import paddle.fluid as fluid x = fluid.layers.data('x', shape=[32], lod_level=1) - y = fluid.layers.sequence_conv(input=x, - num_filters=2, - filter_size=3, - padding_start=None) + y = fluid.layers.sequence_conv( + input=x, num_filters=2, filter_size=3, padding_start=None + ) place = fluid.CPUPlace() x_tensor = fluid.create_lod_tensor( - np.random.rand(10, 32).astype("float32"), [[2, 3, 1, 4]], place) + np.random.rand(10, 32).astype("float32"), [[2, 3, 1, 4]], place + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) ret = exe.run(feed={'x': x_tensor}, fetch_list=[y], return_numpy=False) diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_enumerate_op.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_enumerate_op.py index 68c6d7d75359d..4f75fc08bef69 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_enumerate_op.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_enumerate_op.py @@ -30,15 +30,15 @@ def sequence_enumerate(input_seq, in_lod, win_size, pad_value): single_seq = [] for word_idx in range(win_size): word_pos = idx + word_idx - dat = input_seq[word_pos] if word_pos < lod0[i+1] \ - else pad_value + dat = ( + input_seq[word_pos] if word_pos < lod0[i + 1] else pad_value + ) single_seq.append(dat) out_seq.append(single_seq) return out_seq class TestSequenceEnumerateOp(OpTest): - def setUp(self): self.op_type = "sequence_enumerate" self.init_test_case() @@ -54,68 +54,69 @@ def init_test_case(self): self.lod = [[9, 4, 11, 6]] self.win_size = 2 self.pad_value = 0 - out_seq = sequence_enumerate(self.in_seq, self.lod, self.win_size, - self.pad_value) + out_seq = sequence_enumerate( + self.in_seq, self.lod, self.win_size, self.pad_value + ) self.out_seq = np.array(out_seq).astype("int32") class TesSequenceEnumerateOpInt64(TestSequenceEnumerateOp): - def init_test_case(self): self.in_seq = np.random.randint(0, 10, (30, 1)).astype("int64") self.lod = [[9, 4, 11, 6]] self.win_size = 2 self.pad_value = 0 - out_seq = sequence_enumerate(self.in_seq, self.lod, self.win_size, - self.pad_value) + out_seq = sequence_enumerate( + self.in_seq, self.lod, self.win_size, self.pad_value + ) self.out_seq = np.array(out_seq).astype("int64") class TestSequenceEnumerateOpLargeWinSize(TestSequenceEnumerateOp): - def init_test_case(self): self.in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") self.lod = [[9, 4, 11, 6]] self.win_size = 5 self.pad_value = 0 - out_seq = sequence_enumerate(self.in_seq, self.lod, self.win_size, - self.pad_value) + out_seq = sequence_enumerate( + self.in_seq, self.lod, self.win_size, self.pad_value + ) self.out_seq = np.array(out_seq).astype("int32") class TestSequenceEnumerateOpMaxWinSize(TestSequenceEnumerateOp): - def init_test_case(self): self.in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") self.lod = [[9, 4, 11, 6]] self.win_size = 30 self.pad_value = 0 - out_seq = sequence_enumerate(self.in_seq, self.lod, self.win_size, - self.pad_value) + out_seq = sequence_enumerate( + self.in_seq, self.lod, self.win_size, self.pad_value + ) self.out_seq = np.array(out_seq).astype("int32") class TestSequenceEnumerateOpLargePadValue(TestSequenceEnumerateOp): - def init_test_case(self): self.in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") self.lod = [[9, 4, 11, 6]] self.win_size = 5 self.pad_value = 5 - out_seq = sequence_enumerate(self.in_seq, self.lod, self.win_size, - self.pad_value) + out_seq = sequence_enumerate( + self.in_seq, self.lod, self.win_size, self.pad_value + ) self.out_seq = np.array(out_seq).astype("int32") class TestSequenceEnumerateOpLargePadValueSeqLen0(TestSequenceEnumerateOp): - def init_test_case(self): self.in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") self.lod = [[0, 14, 0, 16, 0]] self.win_size = 5 self.pad_value = 5 - out_seq = sequence_enumerate(self.in_seq, self.lod, self.win_size, - self.pad_value) + out_seq = sequence_enumerate( + self.in_seq, self.lod, self.win_size, self.pad_value + ) self.out_seq = np.array(out_seq).astype("int32") diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_erase_op.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_erase_op.py index 99d72e11ff179..4c4ea068117aa 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_erase_op.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_erase_op.py @@ -26,7 +26,7 @@ def sequence_erase(in_seq, lod0, tokens): offset = 0 for i in range(0, len(lod0)): num_out = 0 - for dat in in_seq[offset:(offset + lod0[i])]: + for dat in in_seq[offset : (offset + lod0[i])]: if dat not in tokens: out_seq.append(dat) num_out += 1 @@ -36,7 +36,6 @@ def sequence_erase(in_seq, lod0, tokens): class TestSequenceEraseOpInt32(OpTest): - def setUp(self): self.op_type = "sequence_erase" in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") @@ -52,7 +51,6 @@ def test_check_output(self): class TestSequenceEraseOpInt32LoD2(OpTest): - def setUp(self): self.op_type = "sequence_erase" in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") @@ -68,7 +66,6 @@ def test_check_output(self): class TestSequenceEraseOpInt64(OpTest): - def setUp(self): self.op_type = "sequence_erase" in_seq = np.random.randint(0, 10, (30, 1)).astype("int64") @@ -84,7 +81,6 @@ def test_check_output(self): class TestSequenceEraseOpInt64SeqLen0(OpTest): - def setUp(self): self.op_type = "sequence_erase" in_seq = np.random.randint(0, 10, (30, 1)).astype("int64") @@ -100,7 +96,6 @@ def test_check_output(self): class TestSequenceEraseOpEmpty(OpTest): - def setUp(self): self.op_type = "sequence_erase" in_seq = np.random.randint(0, 10, (30, 1)).astype("int32") diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand.py index 3d6f66f2a5546..4144812fb68e3 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand.py @@ -21,7 +21,6 @@ class TestSequenceExpand(OpTest): - def set_data(self): x_data = np.random.uniform(0.1, 1, [3, 40]).astype('float64') y_data = np.random.uniform(0.1, 1, [8, 1]).astype('float64') @@ -38,7 +37,7 @@ def compute(self): else: ref_level = len(y_lod) - 1 - out = np.zeros(shape=((0, ) + x_data.shape[1:]), dtype=x_data.dtype) + out = np.zeros(shape=((0,) + x_data.shape[1:]), dtype=x_data.dtype) if x_lod is None: # x_idx = [i for i in xrange(x_data.shape[0] + 1)] @@ -53,7 +52,7 @@ def compute(self): x_len = x_idx[i] if repeat_num > 0: - x_sub = x_data[offset:(offset + x_len), :] + x_sub = x_data[offset : (offset + x_len), :] stacked_x_sub = x_sub for r in range(repeat_num - 1): stacked_x_sub = np.vstack((stacked_x_sub, x_sub)) @@ -81,7 +80,6 @@ def test_check_grad(self): class TestSequenceExpandCase1(TestSequenceExpand): - def set_data(self): x_data = np.random.uniform(0.1, 1, [5, 20]).astype('float64') y_data = np.random.uniform(0.1, 1, [13, 1]).astype('float64') @@ -91,7 +89,6 @@ def set_data(self): class TestSequenceExpandCase2(TestSequenceExpand): - def set_data(self): x_data = np.random.uniform(0.1, 1, [1, 2, 50]).astype('float64') x_lod = [[1]] @@ -102,7 +99,6 @@ def set_data(self): class TestSequenceExpandCase3(TestSequenceExpand): - def set_data(self): x_data = np.random.uniform(0.1, 1, [4, 25]).astype('float64') x_lod = [[1, 1, 1, 1]] @@ -112,7 +108,6 @@ def set_data(self): class TestSequenceExpandCase4(TestSequenceExpand): - def set_data(self): data = np.random.uniform(0.1, 1, [5 * 20, 1]) x_data = np.array(data).reshape([5, 20]).astype('float64') @@ -123,7 +118,6 @@ def set_data(self): class TestSequenceExpandCase5(TestSequenceExpand): - def set_data(self): x_data = np.random.uniform(0.1, 1, [6, 20]).astype('float64') y_data = np.random.uniform(0.1, 1, [13, 1]).astype('float64') @@ -133,7 +127,6 @@ def set_data(self): class TestSequenceExpandCase6(TestSequenceExpand): - def set_data(self): x_data = np.random.uniform(0.1, 1, [4, 25]).astype('float64') x_lod = [[1, 1, 0, 1, 1]] diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand_as.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand_as.py index 00b6ca518e8fd..b1d6bc915ec82 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand_as.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_expand_as.py @@ -23,7 +23,6 @@ class TestSequenceExpandAs(OpTest): - def setUp(self): self.op_type = 'sequence_expand_as' self.set_data() @@ -60,7 +59,6 @@ def test_check_grad(self): class TestSequenceExpandAsCase1(TestSequenceExpandAs): - def set_data(self): x_data = np.random.uniform(0.1, 1, [5, 20]).astype('float64') x_lod = [[2, 3]] @@ -70,7 +68,6 @@ def set_data(self): class TestSequenceExpandAsCase2(TestSequenceExpandAs): - def set_data(self): x_data = np.random.uniform(0.1, 1, [5, 20]).astype('float64') x_lod = [[2, 3]] @@ -80,7 +77,6 @@ def set_data(self): class TestSequenceExpandAsCase3(TestSequenceExpandAs): - def set_data(self): x_data = np.random.uniform(0.1, 1, [1, 2, 50]).astype('float64') x_lod = [[1]] @@ -90,7 +86,6 @@ def set_data(self): class TestSequenceExpandAsOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input x must be Variable diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_first_step.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_first_step.py index 25589f2ad9b3e..a7771634980c2 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_first_step.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_first_step.py @@ -21,7 +21,6 @@ class TestSequenceFirstStepOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -34,11 +33,13 @@ def test_Variable(): def test_input_dtype(): # the dtype of input must be int64 - type_data = fluid.layers.data(name='type_data', - shape=[7, 1], - append_batch_size=False, - dtype='int64', - lod_level=1) + type_data = fluid.layers.data( + name='type_data', + shape=[7, 1], + append_batch_size=False, + dtype='int64', + lod_level=1, + ) fluid.layers.sequence_last_step(type_data) self.assertRaises(TypeError, test_input_dtype) diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_last_step.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_last_step.py index 0ddc348044d63..ca5ff76a8e47e 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_last_step.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_last_step.py @@ -21,7 +21,6 @@ class TestSequenceLastStepOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -34,11 +33,13 @@ def test_Variable(): def test_input_dtype(): # the dtype of input must be int64 - type_data = fluid.layers.data(name='type_data', - shape=[7, 1], - append_batch_size=False, - dtype='int64', - lod_level=1) + type_data = fluid.layers.data( + name='type_data', + shape=[7, 1], + append_batch_size=False, + dtype='int64', + lod_level=1, + ) fluid.layers.sequence_last_step(type_data) self.assertRaises(TypeError, test_input_dtype) diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_mask.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_mask.py index f240bc9900480..24210c318c284 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_mask.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_mask.py @@ -13,7 +13,11 @@ # limitations under the License. import paddle.fluid as fluid -from paddle.fluid.framework import convert_np_dtype_to_dtype_, Program, program_guard +from paddle.fluid.framework import ( + convert_np_dtype_to_dtype_, + Program, + program_guard, +) import numpy as np import unittest import sys @@ -23,7 +27,6 @@ class SequenceMaskTestBase(OpTest): - def initDefaultParameters(self): self.op_type = 'sequence_mask' self.maxlen = 10 @@ -43,19 +46,19 @@ def setUp(self): self.outputs = {'Y': self.calc_ground_truth_mask()} self.attrs = { 'maxlen': self.maxlen, - 'out_dtype': convert_np_dtype_to_dtype_(self.mask_dtype) + 'out_dtype': convert_np_dtype_to_dtype_(self.mask_dtype), } def calc_ground_truth_mask(self): maxlen = np.max(self.x) if self.maxlen < 0 else self.maxlen - shape = self.x.shape + (maxlen, ) - index_broadcast = np.broadcast_to(np.reshape( - range(maxlen), newshape=[1] * self.x.ndim + [-1]), - shape=shape) - x_broadcast = np.broadcast_to(np.reshape(self.x, - newshape=self.x.shape + - (-1, )), - shape=shape) + shape = self.x.shape + (maxlen,) + index_broadcast = np.broadcast_to( + np.reshape(range(maxlen), newshape=[1] * self.x.ndim + [-1]), + shape=shape, + ) + x_broadcast = np.broadcast_to( + np.reshape(self.x, newshape=self.x.shape + (-1,)), shape=shape + ) return (index_broadcast < x_broadcast).astype(self.mask_dtype) def test_check_output(self): @@ -63,43 +66,36 @@ def test_check_output(self): class SequenceMaskTest1(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'bool' class SequenceMaskTest2(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'uint8' class SequenceMaskTest3(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'int32' class SequenceMaskTest4(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'float32' class SequenceMaskTest5(SequenceMaskTestBase): - def initParameters(self): self.mask_dtype = 'float64' class SequenceMaskTest6(SequenceMaskTestBase): - def initParameters(self): self.maxlen = -1 class SequenceMaskTestBase_tensor_attr(OpTest): - def initDefaultParameters(self): self.op_type = 'sequence_mask' self.maxlen = 10 @@ -122,14 +118,14 @@ def setUp(self): def calc_ground_truth_mask(self): maxlen = np.max(self.x) if self.maxlen < 0 else self.maxlen - shape = self.x.shape + (maxlen, ) - index_broadcast = np.broadcast_to(np.reshape( - range(maxlen), newshape=[1] * self.x.ndim + [-1]), - shape=shape) - x_broadcast = np.broadcast_to(np.reshape(self.x, - newshape=self.x.shape + - (-1, )), - shape=shape) + shape = self.x.shape + (maxlen,) + index_broadcast = np.broadcast_to( + np.reshape(range(maxlen), newshape=[1] * self.x.ndim + [-1]), + shape=shape, + ) + x_broadcast = np.broadcast_to( + np.reshape(self.x, newshape=self.x.shape + (-1,)), shape=shape + ) return (index_broadcast < x_broadcast).astype(self.mask_dtype) def test_check_output(self): @@ -137,37 +133,31 @@ def test_check_output(self): class SequenceMaskTest1_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'bool' class SequenceMaskTest2_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'uint8' class SequenceMaskTest3_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'int32' class SequenceMaskTest4_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'float32' class SequenceMaskTest5_tensor_attr(SequenceMaskTestBase_tensor_attr): - def initParameters(self): self.mask_dtype = 'float64' class TestSequenceMaskOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.uniform(1, 5, [4]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_pad_op.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_pad_op.py index 934e0ebe8fd78..d04091d9332b9 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_pad_op.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_pad_op.py @@ -24,7 +24,6 @@ class TestSequencePadOp(OpTest): - def set_attr(self): self.x_shape = [12, 10] self.x_len_lod = [[2, 3, 4, 3]] @@ -37,7 +36,7 @@ def set_data(self): pad_value_data = np.array(self.pad_value).astype(self.dtype) self.inputs = { 'X': (x_data, self.x_len_lod), - 'PadValue': pad_value_data + 'PadValue': pad_value_data, } self.attrs = {'padded_length': self.padded_length} @@ -54,9 +53,10 @@ def compute(self): # do padding x_data = self.inputs['X'][0] pad_value_data = self.inputs['PadValue'] - if pad_value_data.shape == (1, ): - pad_value_data = np.broadcast_to(pad_value_data, - shape=x_data.shape[1:]) + if pad_value_data.shape == (1,): + pad_value_data = np.broadcast_to( + pad_value_data, shape=x_data.shape[1:] + ) padded_sequences = [] start_idx = 0 for l in x_len_lod_0: @@ -86,7 +86,6 @@ def test_check_grad(self): class TestSequencePadOp2(TestSequencePadOp): - def set_attr(self): self.x_shape = [12, 10] self.x_len_lod = [[2, 3, 4, 3]] @@ -96,7 +95,6 @@ def set_attr(self): class TestSequencePadOp3(TestSequencePadOp): - def set_attr(self): self.x_shape = [12, 10] self.x_len_lod = [[2, 3, 4, 3]] @@ -106,7 +104,6 @@ def set_attr(self): class TestSequencePadOp4(TestSequencePadOp): - def set_attr(self): self.x_shape = [12, 10] self.x_len_lod = [[2, 3, 4, 3]] @@ -116,7 +113,6 @@ def set_attr(self): class TestSequencePadOp5(TestSequencePadOp): - def set_attr(self): self.x_shape = [12, 2, 5] self.x_len_lod = [[2, 3, 4, 3]] @@ -126,7 +122,6 @@ def set_attr(self): class TestSequencePadOp6(TestSequencePadOp): - def set_attr(self): self.x_shape = [12, 2, 5] self.x_len_lod = [[2, 3, 4, 3]] @@ -136,7 +131,6 @@ def set_attr(self): class TestSequencePadOp7(TestSequencePadOp): - def set_attr(self): self.x_shape = [12, 2, 5] self.x_len_lod = [[2, 3, 4, 3]] @@ -146,7 +140,6 @@ def set_attr(self): class TestSequencePadOp8(TestSequencePadOp): - def set_attr(self): self.x_shape = [12, 2, 5] self.x_len_lod = [[0, 8, 0, 4, 0]] @@ -156,35 +149,33 @@ def set_attr(self): class TestSequencePadOpError(unittest.TestCase): - def test_error(self): - def test_x_variable(): # the input x type must be Variable x = np.random.random((2, 4)).astype("float32") pad_value = fluid.layers.assign( - input=np.array([0.0], dtype=np.float32)) + input=np.array([0.0], dtype=np.float32) + ) fluid.layers.sequence_pad(x=x, pad_value=pad_value) self.assertRaises(TypeError, test_x_variable) def test_pad_value_variable(): - x1 = fluid.layers.data(name='x1', - shape=[10, 5], - dtype='float32', - lod_level=1) + x1 = fluid.layers.data( + name='x1', shape=[10, 5], dtype='float32', lod_level=1 + ) pad_value1 = np.array([0.0], dtype=np.float32) fluid.layers.sequence_pad(x=x1, pad_value=pad_value1) self.assertRaises(TypeError, test_pad_value_variable) def test_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[10, 5], - dtype='int16', - lod_level=1) + x2 = fluid.layers.data( + name='x2', shape=[10, 5], dtype='int16', lod_level=1 + ) pad_value2 = fluid.layers.assign( - input=np.array([0.0], dtype=np.int32)) + input=np.array([0.0], dtype=np.int32) + ) fluid.layers.sequence_pad(x=x2, pad_value=pad_value2) self.assertRaises(TypeError, test_dtype) diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_pool.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_pool.py index ee4f4fa086034..b4af6f239df1c 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_pool.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_pool.py @@ -27,7 +27,7 @@ def compute_seqpool_sum(x, offset, out, pad_value=0.0): if offset[level][i] == offset[level][i + 1]: out[i] = pad_value else: - sub_x = x[offset[level][i]:offset[level][i + 1], :] + sub_x = x[offset[level][i] : offset[level][i + 1], :] out[i] = sub_x.sum(axis=0) @@ -37,7 +37,7 @@ def compute_seqpool_avg(x, offset, out, pad_value=0.0): if offset[level][i] == offset[level][i + 1]: out[i] = pad_value else: - sub_x = x[offset[level][i]:offset[level][i + 1], :] + sub_x = x[offset[level][i] : offset[level][i + 1], :] out[i] = sub_x.mean(axis=0) @@ -47,13 +47,12 @@ def compute_seqpool_sqrt(x, offset, out, pad_value=0.0): if offset[level][i] == offset[level][i + 1]: out[i] = pad_value else: - sub_x = x[offset[level][i]:offset[level][i + 1], :] + sub_x = x[offset[level][i] : offset[level][i + 1], :] seq_len = offset[level][i + 1] - offset[level][i] out[i] = sub_x.sum(axis=0) / np.sqrt(seq_len) class TestSeqAvgPool(OpTest): - def set_lod(self): return [[11]] @@ -88,69 +87,63 @@ def test_check_output(self): def test_check_grad(self): # Remove MaxIndex after check_grad is refined. out = self.outputs['Out'] - if isinstance(out, tuple): out = out[0] - self.outputs['MaxIndex'] = \ - np.zeros(out.shape).astype('int32') + if isinstance(out, tuple): + out = out[0] + self.outputs['MaxIndex'] = np.zeros(out.shape).astype('int32') self.check_grad(["X"], "Out", check_dygraph=False) class TestSeqAvgPoolBatch1(TestSeqAvgPool): - def set_lod(self): return [[11]] def set_lod_data(self): lod = self.set_lod() - x, _ = self.get_sequence_batch_size_1_input(lod=lod, - shape=[lod[0][0], 23]) + x, _ = self.get_sequence_batch_size_1_input( + lod=lod, shape=[lod[0][0], 23] + ) return x class TestSeqAvgPoolInstance0(TestSeqAvgPool): - def set_lod(self): return [[0, 0, 4, 0, 3, 0, 0, 5, 0, 0]] def set_lod_data(self): lod = self.set_lod() - x, _ = self.get_sequence_instance_size_0_input(lod=lod, - shape=[sum(lod[0]), 10]) + x, _ = self.get_sequence_instance_size_0_input( + lod=lod, shape=[sum(lod[0]), 10] + ) return x class TestSeqAvgPoolLen0(TestSeqAvgPool): - def set_lod(self): return [[0, 4, 0, 7, 0]] class TestSeqAvgPoolLen0LoDLevel2(TestSeqAvgPool): - def set_lod(self): return [[2, 0, 1, 2], [0, 4, 0, 7, 0]] class TestSeqSumPool(TestSeqAvgPool): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.1, 'pooltype': "SUM"} compute_seqpool_sum(x, offset, out, self.attrs["pad_value"]) class TestSeqSumPoolLen0(TestSeqSumPool): - def set_lod(self): return [[0, 4, 0, 7, 0]] class TestSeqSumPoolLen0LoDLevel2(TestSeqSumPool): - def set_lod(self): return [[2, 0, 1, 2], [0, 4, 0, 7, 0]] class TestSeqMaxPool(TestSeqAvgPool): - def set_lod(self): return [[13]] @@ -178,43 +171,37 @@ def compute(self, x, offset, out): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] else: - sub_x = x[offset[level][i]:offset[level][i + 1], :] + sub_x = x[offset[level][i] : offset[level][i + 1], :] out[i] = np.amax(sub_x, axis=0) class TestSeqMaxPoolLen0(TestSeqMaxPool): - def set_lod(self): return [[0, 1, 1, 5, 6, 0]] class TestSeqMaxPoolLen0LoDLevel2(TestSeqMaxPool): - def set_lod(self): return [[2, 0, 3, 1], [0, 1, 1, 5, 6, 0]] class TestSeqSqrtPool(TestSeqAvgPool): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.0, 'pooltype': "SQRT"} compute_seqpool_sqrt(x, offset, out, self.attrs["pad_value"]) class TestSeqSqrtPoolLen0(TestSeqSqrtPool): - def set_lod(self): return [[0, 7, 0, 2, 2, 0]] class TestSeqSqrtPoolLen0LoDLevel2(TestSeqSqrtPool): - def set_lod(self): return [[1, 2, 0, 3], [0, 7, 0, 2, 2, 0]] class TestSeqLastPool(TestSeqAvgPool): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.0, 'pooltype': "LAST"} level = len(offset) - 1 @@ -222,24 +209,21 @@ def compute(self, x, offset, out): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] else: - sub_x = x[offset[level][i]:offset[level][i + 1], :] + sub_x = x[offset[level][i] : offset[level][i + 1], :] out[i] = sub_x[-1, :] class TestSeqLastPoolLen0(TestSeqLastPool): - def set_lod(self): return [[0, 3, 4, 0, 4, 0]] class TestSeqLastPoolLen0LoDLevel2(TestSeqLastPool): - def set_lod(self): return [[1, 0, 2, 3], [0, 3, 4, 0, 4, 0]] class TestSeqFirstPool(TestSeqAvgPool): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.3, 'pooltype': "FIRST"} level = len(offset) - 1 @@ -247,24 +231,21 @@ def compute(self, x, offset, out): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] else: - sub_x = x[offset[level][i]:offset[level][i + 1], :] + sub_x = x[offset[level][i] : offset[level][i + 1], :] out[i] = sub_x[0, :] class TestSeqFirstPoolLen0(TestSeqFirstPool): - def set_lod(self): return [[0, 2, 0, 3, 6, 0]] class TestSeqFirstPoolLen0LoDLevel2(TestSeqFirstPool): - def set_lod(self): return [[1, 0, 2, 3], [0, 2, 0, 3, 6, 0]] class TestSeqAvgPool2D(TestSeqAvgPool): - def set_lod(self): return [[4, 1, 3, 5]] @@ -287,25 +268,23 @@ def compute(self, x, offset, out): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] * np.ones((3, 17)) else: - sub_x = np.reshape(x[offset[level][i]:offset[level][i + 1], :], - (-1, 3 * 17)) + sub_x = np.reshape( + x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 17) + ) out[i] = np.reshape(sub_x.mean(axis=0), (3, 17)) class TestSeqAvgPool2DLen0(TestSeqAvgPool2D): - def set_lod(self): return [[0, 5, 0, 8, 0]] class TestSeqAvgPool2DLen0LoDLevel2(TestSeqAvgPool2D): - def set_lod(self): return [[1, 0, 4], [0, 5, 0, 8, 0]] class TestSeqSumPool2D(TestSeqAvgPool2D): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.2, 'pooltype': "SUM"} level = len(offset) - 1 @@ -313,25 +292,23 @@ def compute(self, x, offset, out): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] * np.ones((3, 17)) else: - sub_x = np.reshape(x[offset[level][i]:offset[level][i + 1], :], - (-1, 3 * 17)) + sub_x = np.reshape( + x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 17) + ) out[i] = np.reshape(sub_x.sum(axis=0), (3, 17)) class TestSeqSumPool2DLen0(TestSeqSumPool2D): - def set_lod(self): return [[0, 8, 0, 5, 0]] class TestSeqSumPool2DLen0LoDLevel2(TestSeqSumPool2D): - def set_lod(self): return [[1, 0, 4], [0, 8, 0, 5, 0]] class TestSeqSqrtPool2D(TestSeqAvgPool2D): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.0, 'pooltype': "SQRT"} level = len(offset) - 1 @@ -339,39 +316,36 @@ def compute(self, x, offset, out): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] * np.ones((3, 17)) else: - sub_x = np.reshape(x[offset[level][i]:offset[level][i + 1], :], - (-1, 3 * 17)) + sub_x = np.reshape( + x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 17) + ) seq_len = offset[level][i + 1] - offset[level][i] out[i] = np.reshape( - sub_x.sum(axis=0) / np.sqrt(seq_len), (3, 17)) + sub_x.sum(axis=0) / np.sqrt(seq_len), (3, 17) + ) def test_check_grad(self): # Remove MaxIndex after check_grad is refined. out = self.outputs['Out'] if isinstance(out, tuple): out = out[0] - self.outputs['MaxIndex'] = \ - np.zeros(out.shape).astype('int32') - self.check_grad(["X"], - "Out", - max_relative_error=0.06, - check_dygraph=False) + self.outputs['MaxIndex'] = np.zeros(out.shape).astype('int32') + self.check_grad( + ["X"], "Out", max_relative_error=0.06, check_dygraph=False + ) class TestSeqSqrtPool2DLen0(TestSeqSqrtPool2D): - def set_lod(self): return [[0, 8, 0, 5, 0]] class TestSeqSqrtPool2DLen0LoDLevel2(TestSeqSqrtPool2D): - def set_lod(self): return [[1, 0, 2, 2], [0, 8, 0, 5, 0]] class TestSeqMaxPool2D(TestSeqAvgPool2D): - def set_lod(self): return [[4, 1, 3, 5]] @@ -399,27 +373,27 @@ def compute(self, x, offset, out): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] * np.ones((3, 11)) continue - sub_x = np.reshape(x[offset[level][i]:offset[level][i + 1], :], - (-1, 3 * 11)) + sub_x = np.reshape( + x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 11) + ) out[i] = np.reshape(np.amax(sub_x, axis=0), (3, 11)) class TestSeqMaxPool2DLen0(TestSeqMaxPool2D): - def set_lod(self): return [[0, 3, 0, 10, 0]] class TestSeqMaxPool2DLen0LoDLevel2(TestSeqMaxPool2D): - def set_lod(self): return [[1, 0, 2, 2], [0, 3, 0, 10, 0]] -@skip_check_grad_ci(reason="Grad computation does not apply to Sequence MAX " - "Pool executed when is_test is true.") +@skip_check_grad_ci( + reason="Grad computation does not apply to Sequence MAX " + "Pool executed when is_test is true." +) class TestSeqMaxPool2DInference(TestSeqMaxPool2D): - def compute(self, x, offset, out): self.attrs = {"pad_value": 1.0, 'pooltype': "MAX", 'is_test': True} level = len(offset) - 1 @@ -427,30 +401,28 @@ def compute(self, x, offset, out): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] * np.ones((3, 11)) else: - sub_x = np.reshape(x[offset[level][i]:offset[level][i + 1], :], - (-1, 3 * 11)) + sub_x = np.reshape( + x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 11) + ) out[i] = np.reshape(np.amax(sub_x, axis=0), (3, 11)) def test_check_grad(self): """Grad computation does not apply to Sequence MAX - Pool executed when is_test is true """ + Pool executed when is_test is true""" return class TestSeqMaxPool2DInferenceLen0(TestSeqMaxPool2DInference): - def set_lod(self): return [[0, 3, 0, 10, 0]] class TestSeqMaxPool2DInferenceLen0LoDLevel2(TestSeqMaxPool2DInference): - def set_lod(self): return [[1, 0, 2, 2], [0, 3, 0, 10, 0]] class TestSeqLastPool2D(TestSeqAvgPool2D): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.0, 'pooltype': "LAST"} level = len(offset) - 1 @@ -458,25 +430,23 @@ def compute(self, x, offset, out): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] * np.ones((3, 17)) else: - sub_x = np.reshape(x[offset[level][i]:offset[level][i + 1], :], - (-1, 3 * 17)) + sub_x = np.reshape( + x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 17) + ) out[i] = np.reshape(sub_x[-1, :], (3, 17)) class TestSeqLastPool2DLen0(TestSeqLastPool2D): - def set_lod(self): return [[0, 3, 0, 1, 9, 0]] class TestSeqLastPool2DLen0LoDLevel2(TestSeqLastPool2D): - def set_lod(self): return [[1, 0, 2, 3], [0, 3, 0, 1, 9, 0]] class TestSeqFirstPool2D(TestSeqAvgPool2D): - def compute(self, x, offset, out): self.attrs = {"pad_value": 0.0, 'pooltype': "FIRST"} level = len(offset) - 1 @@ -484,19 +454,18 @@ def compute(self, x, offset, out): if offset[level][i] == offset[level][i + 1]: out[i] = self.attrs["pad_value"] * np.ones((3, 17)) else: - sub_x = np.reshape(x[offset[level][i]:offset[level][i + 1], :], - (-1, 3 * 17)) + sub_x = np.reshape( + x[offset[level][i] : offset[level][i + 1], :], (-1, 3 * 17) + ) out[i] = np.reshape(sub_x[0, :], (3, 17)) class TestSeqFirstPool2DLen0(TestSeqFirstPool2D): - def set_lod(self): return [[0, 3, 0, 3, 7, 0]] class TestSeqFirstPool2DLen0LoDLevel2(TestSeqFirstPool2D): - def set_lod(self): return [[1, 0, 2, 3], [0, 3, 0, 3, 7, 0]] diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_reshape.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_reshape.py index 5577ad052c04d..ea1b5bbf916b0 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_reshape.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_reshape.py @@ -23,7 +23,6 @@ class TestSequenceReshape(OpTest): - def init_data(self): self.dimension = 12 self.x_lod = [[4, 1, 3, 3]] @@ -57,7 +56,6 @@ def test_check_grad(self): class TestSequenceReshape_reduce(TestSequenceReshape): - def init_data(self): self.dimension = 24 self.x_lod = [[4, 2, 2, 4]] @@ -65,7 +63,6 @@ def init_data(self): class TestSequenceReshape_same(TestSequenceReshape): - def init_data(self): self.dimension = 12 self.x_lod = [[4, 2, 2, 4]] @@ -73,7 +70,6 @@ def init_data(self): class TestSequenceReshape_reduce_seq_len0(TestSequenceReshape): - def init_data(self): self.dimension = 24 self.x_lod = [[0, 6, 0, 2, 4]] @@ -81,7 +77,6 @@ def init_data(self): class TestSequenceReshape_reduce_seq_len0_case1(TestSequenceReshape): - def init_data(self): self.dimension = 24 self.x_lod = [[0, 2, 8, 2, 0]] @@ -89,9 +84,7 @@ def init_data(self): class TestSequenceReshapeOpError(unittest.TestCase): - def test_error(self): - def test_variable(): x = np.random.random((2, 4)).astype("float32") fluid.layers.sequence_reshape(x=x, new_dim=4) @@ -99,11 +92,13 @@ def test_variable(): self.assertRaises(TypeError, test_variable) def test_dtype(): - x1 = fluid.layers.data(name='x1', - shape=[2, 6], - append_batch_size=False, - dtype='float16', - lod_level=1) + x1 = fluid.layers.data( + name='x1', + shape=[2, 6], + append_batch_size=False, + dtype='float16', + lod_level=1, + ) fluid.layers.sequence_reshape(x=x1, new_dim=4) self.assertRaises(TypeError, test_dtype) diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_reverse.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_reverse.py index e4612b064eb32..80b03dd01b456 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_reverse.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_reverse.py @@ -22,7 +22,6 @@ class TestSequenceReverseBase(OpTest): - def initParameters(self): pass @@ -36,14 +35,20 @@ def setUp(self): self.y = self.get_output() self.inputs = { - 'X': (self.x, [ - self.lod, - ]), + 'X': ( + self.x, + [ + self.lod, + ], + ), } self.outputs = { - 'Y': (self.y, [ - self.lod, - ]), + 'Y': ( + self.y, + [ + self.lod, + ], + ), } def get_output(self): @@ -65,37 +70,31 @@ def test_grad(self): class TestSequenceReserve1(TestSequenceReverseBase): - def initParameters(self): self.size = (12, 10) self.lod = [4, 5, 3] class TestSequenceReverse2(TestSequenceReverseBase): - def initParameters(self): self.size = (12, 10) self.lod = [12] class TestSequenceReverse3(TestSequenceReverseBase): - def initParameters(self): self.size = (12, 10) self.lod = [3, 0, 6, 3] class TestSequenceReverse4(TestSequenceReverseBase): - def initParameters(self): self.size = (12, 10) self.lod = [0, 2, 10, 0] class TestSequenceReverseOpError(unittest.TestCase): - def test_error(self): - def test_variable(): # the input type must be Variable x_data = np.random.random((2, 4)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_scatter_op.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_scatter_op.py index 2b8e826182909..ad4025a349d20 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_scatter_op.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_scatter_op.py @@ -21,7 +21,6 @@ class TestSequenceScatterOp(OpTest): - def init_lod(self): return [[30, 50, 40]] @@ -45,7 +44,7 @@ def setUp(self): self.inputs = { 'X': X_data, 'Ids': (Ids_data, Ids_lod), - 'Updates': (Updates_data, Updates_lod) + 'Updates': (Updates_data, Updates_lod), } self.outputs = {'Out': Out_data} @@ -57,37 +56,31 @@ def test_check_grad(self): class TestSequenceScatterOpSeqLen0(TestSequenceScatterOp): - def init_lod(self): return [[60, 60, 00]] class TestSequenceScatterOpSeqLen0Case1(TestSequenceScatterOp): - def init_lod(self): return [[0, 60, 60]] class TestSequenceScatterOpSeqLen0Case2(TestSequenceScatterOp): - def init_lod(self): return [[60, 0, 60]] class TestSequenceScatterOpSeqLen0Case3(TestSequenceScatterOp): - def init_lod(self): return [[120, 0, 0]] class TestSequenceScatterOpSeqLen0Case4(TestSequenceScatterOp): - def init_lod(self): return [[0, 120, 0]] class TestSequenceScatterOpSeqLen0Case5(TestSequenceScatterOp): - def init_lod(self): return [[0, 0, 120]] diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_slice_op.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_slice_op.py index 052e0fad06259..c3b1e291de866 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_slice_op.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_slice_op.py @@ -21,7 +21,6 @@ class TestSequenceSliceOp(OpTest): - def set_data(self): self.init_test_case() # only supprot one level LoD @@ -31,12 +30,17 @@ def set_data(self): length = np.array(self.length).astype("int64") self.inputs = {'X': (x, lod), 'Offset': offset, 'Length': length} - outs = [] #np.zeros((100, 3, 2)).astype('float32') + outs = [] # np.zeros((100, 3, 2)).astype('float32') out_lod = [[]] lod_offset = 0 for i in range(len(offset)): - sub_x = x[lod_offset + offset[i, 0]:lod_offset + offset[i, 0] + - length[i, 0], :] + sub_x = x[ + lod_offset + + offset[i, 0] : lod_offset + + offset[i, 0] + + length[i, 0], + :, + ] outs.append(sub_x) out_lod[0].append(len(sub_x)) lod_offset += lod[0][i] @@ -61,7 +65,6 @@ def test_check_grad(self): class TestSequenceSliceOpSeqlen0Case0(TestSequenceSliceOp): - def init_test_case(self): self.x_dim = (100, 3, 2) self.x_lod = [[20, 30, 0, 30, 20]] @@ -70,7 +73,6 @@ def init_test_case(self): class TestSequenceSliceOpSeqlen0Case1(TestSequenceSliceOp): - def init_test_case(self): self.x_dim = (100, 3, 2) self.x_lod = [[0, 70, 0, 30, 0]] @@ -79,7 +81,6 @@ def init_test_case(self): class TestSequenceSliceOpSeqlen0Case2(TestSequenceSliceOp): - def init_test_case(self): self.x_dim = (100, 3, 2) self.x_lod = [[0, 100, 0, 0, 0]] diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_softmax_op.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_softmax_op.py index 1918dc37a0be2..a08cde6af4dd1 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_softmax_op.py @@ -23,7 +23,6 @@ class TestSequenceSoftmaxOp(OpTest): - def setUp(self): self.op_type = "sequence_softmax" self.use_cudnn = False @@ -34,13 +33,14 @@ def setUp(self): out = np.zeros((110, 1)).astype(self.dtype) offset = 0 for i in range(len(self.lod[0])): - if (self.lod[0][i] == 0): + if self.lod[0][i] == 0: continue - sub_x = x[offset:offset + self.lod[0][i], :] + sub_x = x[offset : offset + self.lod[0][i], :] sub_x = sub_x.reshape(1, self.lod[0][i]) sub_out = stable_softmax(sub_x) - out[offset:offset + self.lod[0][i], :] = sub_out.reshape( - self.lod[0][i], 1) + out[offset : offset + self.lod[0][i], :] = sub_out.reshape( + self.lod[0][i], 1 + ) offset += self.lod[0][i] self.inputs = {"X": (x, self.lod)} @@ -71,28 +71,25 @@ def test_check_grad(self): # ----------------cudnn Sequencesoftmax---------------- -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSequenceSoftmaxCUDNNOp(TestSequenceSoftmaxOp): - def init_op_type(self): self.use_cudnn = True class TestSequenceSoftmaxOpSeqLen0Case0(TestSequenceSoftmaxOp): - def init_lod(self): self.lod = [[40, 0, 40, 30]] class TestSequenceSoftmaxOpSeqLen0Case1(TestSequenceSoftmaxOp): - def init_lod(self): self.lod = [[0, 40, 70, 0]] class TestSequenceSoftmaxOpSeqLen0Case2(TestSequenceSoftmaxOp): - def init_lod(self): self.lod = [[0, 0, 0, 110]] diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_topk_avg_pooling.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_topk_avg_pooling.py index 415e9688cfbd1..651ce9213da30 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_topk_avg_pooling.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_topk_avg_pooling.py @@ -22,7 +22,6 @@ class TestSequenceTopkAvgPoolingOp(OpTest): - def setUp(self): self.init_op_type() self.set_data() @@ -51,7 +50,7 @@ def init_data(self, topks, channel_num, row, col, dim=10): self.inputs = { 'X': (x_data, x_lod), 'ROW': (row_data, [row]), - 'COLUMN': (col_data, [col]) + 'COLUMN': (col_data, [col]), } def calc_gradient(self, pos_data, topks, channel_num, row, col): @@ -60,7 +59,7 @@ def calc_gradient(self, pos_data, topks, channel_num, row, col): in_numel = sum([row[i] * col[i] for i in range(len(row))]) * channel_num out_numel = sum(row) * len(topks) * channel_num gradient = np.zeros(shape=(in_numel), dtype="float32") - dout_val = 1. / out_numel + dout_val = 1.0 / out_numel pos_offset, in_offset = 0, 0 for bs_idx in range(len(row)): # batch row_size = row[bs_idx] @@ -72,9 +71,9 @@ def calc_gradient(self, pos_data, topks, channel_num, row, col): for k_idx in range(len(topks)): for k in range(topks[k_idx]): if pos_data[pos_idx + k] != -1: - gradient[in_idx + - pos_data[pos_idx + - k]] += dout_val / topks[k_idx] + gradient[in_idx + pos_data[pos_idx + k]] += ( + dout_val / topks[k_idx] + ) in_offset += row_size * col_size pos_offset += row_size * max_k return gradient @@ -87,7 +86,7 @@ def compute(self): col_data, col_lod = self.inputs['COLUMN'] channel_num = self.attrs['channel_num'] out = np.zeros((0, len(topks) * channel_num), dtype=x_data.dtype) - pos = np.zeros((0, ), dtype='int32') + pos = np.zeros((0,), dtype='int32') out_lod = deepcopy(row_lod) offset = 0 @@ -95,45 +94,51 @@ def compute(self): x_len = x_lod[0][idx] self.assertTrue( x_len == channel_num * row_lod[0][idx] * col_lod[0][idx], - "x_len: %s can't mod channel_num: %s" % (x_len, channel_num)) - out_tmp = np.zeros((0, ), dtype=x_data.dtype) - pos_tmp = np.zeros((0, ), dtype='int32') + "x_len: %s can't mod channel_num: %s" % (x_len, channel_num), + ) + out_tmp = np.zeros((0,), dtype=x_data.dtype) + pos_tmp = np.zeros((0,), dtype='int32') for ch in range(channel_num): for r_id in range(row_lod[0][idx]): - x_sub = x_data[offset:(offset + col_lod[0][idx])] + x_sub = x_data[offset : (offset + col_lod[0][idx])] topk_val, topk_pos = self.get_topk(x_sub, max_k) sum_data = self.topk_sum(topk_val, topk_pos, max_k) new_feature = np.array( - [sum_data[topk] / topk for topk in topks]) + [sum_data[topk] / topk for topk in topks] + ) out_tmp = np.hstack((out_tmp, new_feature)) pos_tmp = np.hstack((pos_tmp, topk_pos)) offset += col_lod[0][idx] - out_tmp = out_tmp.reshape([channel_num, -1, - len(topks)]).transpose(1, 0, 2) - pos_tmp = pos_tmp.reshape([channel_num, -1, - max_k]).transpose(1, 0, 2) + out_tmp = out_tmp.reshape([channel_num, -1, len(topks)]).transpose( + 1, 0, 2 + ) + pos_tmp = pos_tmp.reshape([channel_num, -1, max_k]).transpose( + 1, 0, 2 + ) out = np.vstack( - (out, out_tmp.reshape([-1, len(topks) * channel_num]))) + (out, out_tmp.reshape([-1, len(topks) * channel_num])) + ) pos = np.hstack((pos, pos_tmp.flatten())) self.outputs = {'Out': (out.astype('float32'), out_lod), 'pos': pos} - self.gradient = self.calc_gradient(pos, topks, channel_num, row_lod[0], - col_lod[0]) + self.gradient = self.calc_gradient( + pos, topks, channel_num, row_lod[0], col_lod[0] + ) def get_topk(self, x, topk): real_topk = topk if topk < len(x) else len(x) topk_pos = np.array(x).argsort()[-topk:][::-1] topk_val = np.array(x)[topk_pos] if real_topk < topk: - topk_pos = np.hstack((topk_pos, np.full((topk - real_topk, ), -1))) - topk_val = np.hstack((topk_val, np.full((topk - real_topk, ), 0.0))) + topk_pos = np.hstack((topk_pos, np.full((topk - real_topk,), -1))) + topk_val = np.hstack((topk_val, np.full((topk - real_topk,), 0.0))) return topk_val, topk_pos def topk_sum(self, x, pos, max_k): - sum_data = [0.] * (max_k + 1) + sum_data = [0.0] * (max_k + 1) for i in range(1, max_k + 1): if pos[i - 1] == -1: sum_data[i] = sum_data[i - 1] @@ -149,7 +154,6 @@ def test_check_grad(self): class TestSequenceTopkAvgPoolingOpCase1(TestSequenceTopkAvgPoolingOp): - def set_data(self): topks = [2, 3] channel_num = 5 @@ -160,32 +164,32 @@ def set_data(self): def test_api(self): import paddle.fluid as fluid + x = fluid.layers.data(name='x', shape=[1], lod_level=1) row = fluid.layers.data(name='row', shape=[10], lod_level=1) col = fluid.layers.data(name='col', shape=[10], lod_level=1) - topk_avg = fluid.contrib.sequence_topk_avg_pooling(input=x, - row=row, - col=col, - topks=[1, 3, 5], - channel_num=5) + topk_avg = fluid.contrib.sequence_topk_avg_pooling( + input=x, row=row, col=col, topks=[1, 3, 5], channel_num=5 + ) place = fluid.CPUPlace() x_tensor = fluid.create_lod_tensor( - np.random.rand(45, 1).astype('float32'), [[30, 15]], place) + np.random.rand(45, 1).astype('float32'), [[30, 15]], place + ) row_tensor = fluid.create_lod_tensor( - np.random.rand(5, 10).astype('float32'), [[2, 3]], place) + np.random.rand(5, 10).astype('float32'), [[2, 3]], place + ) col_tensor = fluid.create_lod_tensor( - np.random.rand(4, 10).astype('float32'), [[3, 1]], place) + np.random.rand(4, 10).astype('float32'), [[3, 1]], place + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'x': x_tensor, - 'row': row_tensor, - 'col': col_tensor - }, - fetch_list=[topk_avg], - return_numpy=False) + ret = exe.run( + feed={'x': x_tensor, 'row': row_tensor, 'col': col_tensor}, + fetch_list=[topk_avg], + return_numpy=False, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/sequence/test_sequence_unpad_op.py b/python/paddle/fluid/tests/unittests/sequence/test_sequence_unpad_op.py index 7345274caeb77..07dfd70ca2bca 100644 --- a/python/paddle/fluid/tests/unittests/sequence/test_sequence_unpad_op.py +++ b/python/paddle/fluid/tests/unittests/sequence/test_sequence_unpad_op.py @@ -23,7 +23,6 @@ class TestSequenceUnpadOp(OpTest): - def init(self): self.length = [2, 3, 4] self.x_shape = (3, 40) @@ -34,13 +33,13 @@ def compute(self): x = np.random.random(self.x_shape).astype(self.dtype) out_lod = [self.length] - out = x[0, 0:self.length[0]] + out = x[0, 0 : self.length[0]] for i in range(1, x.shape[0]): - out = np.append(out, x[i, 0:self.length[i]], axis=0) + out = np.append(out, x[i, 0 : self.length[i]], axis=0) - out_shape = (sum(self.length), ) + out_shape = (sum(self.length),) if len(self.x_shape) == 2: - out_shape = out_shape + (1, ) + out_shape = out_shape + (1,) else: out_shape = out_shape + self.x_shape[2:] @@ -60,7 +59,6 @@ def test_check_grad(self): class TestSequenceUnpadOp2(TestSequenceUnpadOp): - def init(self): self.length = [2, 3, 4] self.x_shape = (3, 5, 4, 3) @@ -68,7 +66,6 @@ def init(self): class TestSequenceUnpadOp3(TestSequenceUnpadOp): - def init(self): self.length = [5, 2, 3, 4] self.x_shape = (4, 5, 3, 3, 6) @@ -76,7 +73,6 @@ def init(self): class TestSequenceUnpadOp4(TestSequenceUnpadOp): - def init(self): self.length = [5, 0, 0, 4] self.x_shape = (4, 5, 3, 3, 6) @@ -84,7 +80,6 @@ def init(self): class TestSequenceUnpadOp5(TestSequenceUnpadOp): - def init(self): self.length = [0, 4, 3, 0] self.x_shape = (4, 5, 3, 3, 6) @@ -92,9 +87,7 @@ def init(self): class TestSequenceUnpadOpError(unittest.TestCase): - def test_error(self): - def test_x_variable(): x = np.random.random((10, 5)).astype("float64") len = fluid.data(name='length2', shape=[10], dtype='int64') diff --git a/python/paddle/fluid/tests/unittests/seresnext_net.py b/python/paddle/fluid/tests/unittests/seresnext_net.py index d1676380afefb..b2bc25e35ae3e 100644 --- a/python/paddle/fluid/tests/unittests/seresnext_net.py +++ b/python/paddle/fluid/tests/unittests/seresnext_net.py @@ -48,37 +48,40 @@ def squeeze_excitation(input, num_channels, reduction_ratio): # input=input, pool_size=0, pool_type='avg', global_pooling=True) conv = input shape = conv.shape - reshape = fluid.layers.reshape(x=conv, - shape=[-1, shape[1], shape[2] * shape[3]]) + reshape = fluid.layers.reshape( + x=conv, shape=[-1, shape[1], shape[2] * shape[3]] + ) pool = fluid.layers.reduce_mean(input=reshape, dim=2) - squeeze = fluid.layers.fc(input=pool, - size=num_channels // reduction_ratio, - act='relu') - excitation = fluid.layers.fc(input=squeeze, - size=num_channels, - act='sigmoid') + squeeze = fluid.layers.fc( + input=pool, size=num_channels // reduction_ratio, act='relu' + ) + excitation = fluid.layers.fc( + input=squeeze, size=num_channels, act='sigmoid' + ) scale = fluid.layers.elementwise_mul(x=input, y=excitation, axis=0) return scale -def conv_bn_layer(input, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): - conv = fluid.layers.conv2d(input=input, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - use_cudnn=(not remove_cudnn_conv), - bias_attr=False) - return conv if remove_bn else fluid.layers.batch_norm( - input=conv, act=act, momentum=0.1) +def conv_bn_layer( + input, num_filters, filter_size, stride=1, groups=1, act=None +): + conv = fluid.layers.conv2d( + input=input, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + use_cudnn=(not remove_cudnn_conv), + bias_attr=False, + ) + return ( + conv + if remove_bn + else fluid.layers.batch_norm(input=conv, act=act, momentum=0.1) + ) def shortcut(input, ch_out, stride): @@ -96,23 +99,25 @@ def shortcut(input, ch_out, stride): def bottleneck_block(input, num_filters, stride, cardinality, reduction_ratio): # The number of first 1x1 convolutional channels for each bottleneck build block # was halved to reduce the compution cost. - conv0 = conv_bn_layer(input=input, - num_filters=num_filters, - filter_size=1, - act='relu') - conv1 = conv_bn_layer(input=conv0, - num_filters=num_filters * 2, - filter_size=3, - stride=stride, - groups=cardinality, - act='relu') - conv2 = conv_bn_layer(input=conv1, - num_filters=num_filters * 2, - filter_size=1, - act=None) - scale = squeeze_excitation(input=conv2, - num_channels=num_filters * 2, - reduction_ratio=reduction_ratio) + conv0 = conv_bn_layer( + input=input, num_filters=num_filters, filter_size=1, act='relu' + ) + conv1 = conv_bn_layer( + input=conv0, + num_filters=num_filters * 2, + filter_size=3, + stride=stride, + groups=cardinality, + act='relu', + ) + conv2 = conv_bn_layer( + input=conv1, num_filters=num_filters * 2, filter_size=1, act=None + ) + scale = squeeze_excitation( + input=conv2, + num_channels=num_filters * 2, + reduction_ratio=reduction_ratio, + ) short = shortcut(input, num_filters * 2, stride) @@ -127,26 +132,18 @@ def SE_ResNeXt50Small(use_feed): img = fluid.layers.data(name='image', shape=img_shape, dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') - conv = conv_bn_layer(input=img, - num_filters=16, - filter_size=3, - stride=2, - act='relu') - conv = conv_bn_layer(input=conv, - num_filters=16, - filter_size=3, - stride=1, - act='relu') - conv = conv_bn_layer(input=conv, - num_filters=16, - filter_size=3, - stride=1, - act='relu') - conv = fluid.layers.pool2d(input=conv, - pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + conv = conv_bn_layer( + input=img, num_filters=16, filter_size=3, stride=2, act='relu' + ) + conv = conv_bn_layer( + input=conv, num_filters=16, filter_size=3, stride=1, act='relu' + ) + conv = conv_bn_layer( + input=conv, num_filters=16, filter_size=3, stride=1, act='relu' + ) + conv = fluid.layers.pool2d( + input=conv, pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) cardinality = 32 reduction_ratio = 16 @@ -155,18 +152,24 @@ def SE_ResNeXt50Small(use_feed): for block in range(len(depth)): for i in range(depth[block]): - conv = bottleneck_block(input=conv, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - cardinality=cardinality, - reduction_ratio=reduction_ratio) + conv = bottleneck_block( + input=conv, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + cardinality=cardinality, + reduction_ratio=reduction_ratio, + ) shape = conv.shape - reshape = fluid.layers.reshape(x=conv, - shape=[-1, shape[1], shape[2] * shape[3]]) + reshape = fluid.layers.reshape( + x=conv, shape=[-1, shape[1], shape[2] * shape[3]] + ) pool = fluid.layers.reduce_mean(input=reshape, dim=2) - dropout = pool if remove_dropout else fluid.layers.dropout( - x=pool, dropout_prob=0.2, seed=1) + dropout = ( + pool + if remove_dropout + else fluid.layers.dropout(x=pool, dropout_prob=0.2, seed=1) + ) # Classifier layer: prediction = fluid.layers.fc(input=dropout, size=1000, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) @@ -176,11 +179,12 @@ def SE_ResNeXt50Small(use_feed): def optimizer(learning_rate=0.01): optimizer = fluid.optimizer.Momentum( - learning_rate=cosine_decay(learning_rate=learning_rate, - step_each_epoch=2, - epochs=1), + learning_rate=cosine_decay( + learning_rate=learning_rate, step_each_epoch=2, epochs=1 + ), momentum=0.9, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) return optimizer @@ -203,10 +207,13 @@ def iter(use_device): gpu_img, gpu_label = init_data( batch_size=batch_size(use_device=DeviceType.CUDA), img_shape=img_shape, - label_range=999) -cpu_img, cpu_label = init_data(batch_size=batch_size(use_device=DeviceType.CPU), - img_shape=img_shape, - label_range=999) + label_range=999, +) +cpu_img, cpu_label = init_data( + batch_size=batch_size(use_device=DeviceType.CPU), + img_shape=img_shape, + label_range=999, +) feed_dict_gpu = {"image": gpu_img, "label": gpu_label} feed_dict_cpu = {"image": cpu_img, "label": cpu_label} diff --git a/python/paddle/fluid/tests/unittests/seresnext_test_base.py b/python/paddle/fluid/tests/unittests/seresnext_test_base.py index 739de92db60c4..2199d1ed20a59 100644 --- a/python/paddle/fluid/tests/unittests/seresnext_test_base.py +++ b/python/paddle/fluid/tests/unittests/seresnext_test_base.py @@ -20,30 +20,33 @@ class TestResnetBase(TestParallelExecutorBase): - - def _compare_result_with_origin_model(self, - check_func, - use_device, - delta2=1e-5, - compare_separately=True): + def _compare_result_with_origin_model( + self, check_func, use_device, delta2=1e-5, compare_separately=True + ): if use_device == DeviceType.CUDA and not core.is_compiled_with_cuda(): return - func_1_first_loss, func_1_last_loss, func_1_loss_area = self.check_network_convergence( + ( + func_1_first_loss, + func_1_last_loss, + func_1_loss_area, + ) = self.check_network_convergence( seresnext_net.model, feed_dict=seresnext_net.feed_dict(use_device), iter=seresnext_net.iter(use_device), batch_size=seresnext_net.batch_size(use_device), use_device=use_device, use_reduce=False, - optimizer=seresnext_net.optimizer) + optimizer=seresnext_net.optimizer, + ) func_2_first_loss, func_2_last_loss, func_2_loss_area = check_func( seresnext_net.model, feed_dict=seresnext_net.feed_dict(use_device), iter=seresnext_net.iter(use_device), batch_size=seresnext_net.batch_size(use_device), - use_device=use_device) + use_device=use_device, + ) if compare_separately: for loss in zip(func_1_first_loss, func_2_first_loss): @@ -51,12 +54,12 @@ def _compare_result_with_origin_model(self, for loss in zip(func_1_last_loss, func_2_last_loss): self.assertAlmostEquals(loss[0], loss[1], delta=delta2) else: - np.testing.assert_allclose(func_1_loss_area, - func_2_loss_area, - rtol=delta2) - self.assertAlmostEquals(np.mean(func_1_first_loss), - func_2_first_loss[0], - delta=1e-5) - self.assertAlmostEquals(np.mean(func_1_last_loss), - func_2_last_loss[0], - delta=delta2) + np.testing.assert_allclose( + func_1_loss_area, func_2_loss_area, rtol=delta2 + ) + self.assertAlmostEquals( + np.mean(func_1_first_loss), func_2_first_loss[0], delta=1e-5 + ) + self.assertAlmostEquals( + np.mean(func_1_last_loss), func_2_last_loss[0], delta=delta2 + ) diff --git a/python/paddle/fluid/tests/unittests/simnet_dataset_reader.py b/python/paddle/fluid/tests/unittests/simnet_dataset_reader.py index 5eae0ff22e14d..5b4a49e693ad7 100644 --- a/python/paddle/fluid/tests/unittests/simnet_dataset_reader.py +++ b/python/paddle/fluid/tests/unittests/simnet_dataset_reader.py @@ -22,6 +22,5 @@ class DatasetSimnetReader(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): pass diff --git a/python/paddle/fluid/tests/unittests/simple_nets.py b/python/paddle/fluid/tests/unittests/simple_nets.py index 9326d51591576..f85a0f9e135e2 100644 --- a/python/paddle/fluid/tests/unittests/simple_nets.py +++ b/python/paddle/fluid/tests/unittests/simple_nets.py @@ -24,8 +24,10 @@ def simple_fc_net_with_inputs(img, label, class_num=10): hidden, size=100, act='relu', - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) prediction = fluid.layers.fc(hidden, size=class_num, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) loss = paddle.mean(loss) @@ -45,8 +47,10 @@ def batchnorm_fc_with_inputs(img, label, class_num=10): hidden, size=200, act='relu', - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) hidden = fluid.layers.batch_norm(input=hidden) @@ -62,26 +66,27 @@ def fc_with_batchnorm(use_feed=None): return batchnorm_fc_with_inputs(img, label, class_num=10) -def bow_net(use_feed, - dict_dim, - is_sparse=False, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2): +def bow_net( + use_feed, + dict_dim, + is_sparse=False, + emb_dim=128, + hid_dim=128, + hid_dim2=96, + class_dim=2, +): """ BOW net This model is from https://github.com/PaddlePaddle/models: fluid/PaddleNLP/text_classification/nets.py """ - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") - emb = fluid.layers.embedding(input=data, - is_sparse=is_sparse, - size=[dict_dim, emb_dim]) + emb = fluid.layers.embedding( + input=data, is_sparse=is_sparse, size=[dict_dim, emb_dim] + ) bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') bow_tanh = fluid.layers.tanh(bow) fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh") @@ -98,7 +103,9 @@ def init_data(batch_size=32, img_shape=[784], label_range=9): assert isinstance(img_shape, list) input_shape = [batch_size] + img_shape img = np.random.random(size=input_shape).astype(np.float32) - label = np.array( - [np.random.randint(0, label_range) for _ in range(batch_size)]).reshape( - (-1, 1)).astype("int64") + label = ( + np.array([np.random.randint(0, label_range) for _ in range(batch_size)]) + .reshape((-1, 1)) + .astype("int64") + ) return img, label diff --git a/python/paddle/fluid/tests/unittests/spawn_runner_base.py b/python/paddle/fluid/tests/unittests/spawn_runner_base.py index d832781ffbc4c..2b35fe70cf935 100644 --- a/python/paddle/fluid/tests/unittests/spawn_runner_base.py +++ b/python/paddle/fluid/tests/unittests/spawn_runner_base.py @@ -30,7 +30,6 @@ class SpawnAssistTestArgs(object): class TestDistSpawnRunner(unittest.TestCase): - def setUp(self): # NOTE(chenweihang): keep consistent with # TestDistBase.check_with_place @@ -42,10 +41,12 @@ def _run(self, model, args): def _run_parallel(self, model, args): args.update_method = "nccl2" - context = paddle.distributed.spawn(func=model.run_trainer_with_spawn, - args=(args, ), - nprocs=self.nprocs, - join=True) + context = paddle.distributed.spawn( + func=model.run_trainer_with_spawn, + args=(args,), + nprocs=self.nprocs, + join=True, + ) result_list = [] for res_queue in context.return_queues: result_list.append(res_queue.get()) @@ -53,10 +54,12 @@ def _run_parallel(self, model, args): def check_dist_result_with_spawn(self, test_class, delta=1e-3): with _test_eager_guard(): - self.check_dist_result_with_spawn_func(test_class=test_class, - delta=delta) - self.check_dist_result_with_spawn_func(test_class=test_class, - delta=delta) + self.check_dist_result_with_spawn_func( + test_class=test_class, delta=delta + ) + self.check_dist_result_with_spawn_func( + test_class=test_class, delta=delta + ) def check_dist_result_with_spawn_func(self, test_class, delta=1e-3): # 0. prepare model and args @@ -83,7 +86,8 @@ def check_dist_result_with_spawn_func(self, test_class, delta=1e-3): loss, dist_loss, delta=delta, - msg= - "The results of single-card execution and multi-card execution are inconsistent." - "signal-card loss is:\n{}\nmulti-card average loss is:\n{}\n". - format(loss, dist_loss)) + msg="The results of single-card execution and multi-card execution are inconsistent." + "signal-card loss is:\n{}\nmulti-card average loss is:\n{}\n".format( + loss, dist_loss + ), + ) diff --git a/python/paddle/fluid/tests/unittests/static_model_parallel_fused_attention.py b/python/paddle/fluid/tests/unittests/static_model_parallel_fused_attention.py index c6c318536164c..2d825af4b3362 100644 --- a/python/paddle/fluid/tests/unittests/static_model_parallel_fused_attention.py +++ b/python/paddle/fluid/tests/unittests/static_model_parallel_fused_attention.py @@ -25,9 +25,11 @@ def get_param_attr(weight, bias): weight_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(weight)) + initializer=fluid.initializer.NumpyArrayInitializer(weight) + ) bias_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(bias)) + initializer=fluid.initializer.NumpyArrayInitializer(bias) + ) return weight_attr, bias_attr @@ -40,14 +42,16 @@ def get_param_attr(weight, bias): def create_model(data, rank): np.random.seed(2021) - pre_ln_w = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) - pre_ln_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) - qkv_w = np.random.uniform(-1, 1, - size=(3, n_head, d_key, hidden)).astype(DTYPE) + pre_ln_w = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) + pre_ln_b = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) + qkv_w = np.random.uniform(-1, 1, size=(3, n_head, d_key, hidden)).astype( + DTYPE + ) qkv_b = np.random.uniform(-1, 1, size=(3, n_head, d_key)).astype(DTYPE) - linear_w = np.random.uniform(-1, 1, - size=(n_head * d_key, hidden)).astype(DTYPE) - linear_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) + linear_w = np.random.uniform(-1, 1, size=(n_head * d_key, hidden)).astype( + DTYPE + ) + linear_b = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) data.stop_gradient = False if rank is not None: @@ -55,46 +59,50 @@ def create_model(data, rank): end = start + n_head // MODEL_PARALLEL_SIZE col_qkv_w = qkv_w[:, start:end, :, :] col_qkv_b = qkv_b[:, start:end, :] - row_linear_w = linear_w[(start * d_key):(end * d_key), :] + row_linear_w = linear_w[(start * d_key) : (end * d_key), :] pre_ln_w_attr, pre_ln_b_attr = get_param_attr(pre_ln_w, pre_ln_b) qkv_w_attr, qkv_b_attr = get_param_attr(col_qkv_w, col_qkv_b) linear_w_attr, linear_b_attr = get_param_attr(row_linear_w, linear_b) - attn = FusedMultiHeadAttention(hidden, - n_head, - dropout_rate=0.0, - attn_dropout_rate=0.0, - normalize_before=False, - qkv_weight_attr=qkv_w_attr, - qkv_bias_attr=qkv_b_attr, - linear_weight_attr=linear_w_attr, - linear_bias_attr=linear_b_attr, - pre_ln_scale_attr=pre_ln_w_attr, - pre_ln_bias_attr=pre_ln_b_attr, - ln_scale_attr=pre_ln_w_attr, - ln_bias_attr=pre_ln_b_attr, - nranks=MODEL_PARALLEL_SIZE, - ring_id=0) + attn = FusedMultiHeadAttention( + hidden, + n_head, + dropout_rate=0.0, + attn_dropout_rate=0.0, + normalize_before=False, + qkv_weight_attr=qkv_w_attr, + qkv_bias_attr=qkv_b_attr, + linear_weight_attr=linear_w_attr, + linear_bias_attr=linear_b_attr, + pre_ln_scale_attr=pre_ln_w_attr, + pre_ln_bias_attr=pre_ln_b_attr, + ln_scale_attr=pre_ln_w_attr, + ln_bias_attr=pre_ln_b_attr, + nranks=MODEL_PARALLEL_SIZE, + ring_id=0, + ) result = attn(data) else: pre_ln_w_attr, pre_ln_b_attr = get_param_attr(pre_ln_w, pre_ln_b) qkv_w_attr, qkv_b_attr = get_param_attr(qkv_w, qkv_b) linear_w_attr, linear_b_attr = get_param_attr(linear_w, linear_b) - attn = FusedMultiHeadAttention(hidden, - n_head, - dropout_rate=0.0, - attn_dropout_rate=0.0, - normalize_before=False, - qkv_weight_attr=qkv_w_attr, - qkv_bias_attr=qkv_b_attr, - linear_weight_attr=linear_w_attr, - linear_bias_attr=linear_b_attr, - pre_ln_scale_attr=pre_ln_w_attr, - pre_ln_bias_attr=pre_ln_b_attr, - ln_scale_attr=pre_ln_w_attr, - ln_bias_attr=pre_ln_b_attr) + attn = FusedMultiHeadAttention( + hidden, + n_head, + dropout_rate=0.0, + attn_dropout_rate=0.0, + normalize_before=False, + qkv_weight_attr=qkv_w_attr, + qkv_bias_attr=qkv_b_attr, + linear_weight_attr=linear_w_attr, + linear_bias_attr=linear_b_attr, + pre_ln_scale_attr=pre_ln_w_attr, + pre_ln_bias_attr=pre_ln_b_attr, + ln_scale_attr=pre_ln_w_attr, + ln_bias_attr=pre_ln_b_attr, + ) result = attn(data) predict = paddle.sum(result) @@ -102,20 +110,20 @@ def create_model(data, rank): class TestModelParallel(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data seq_len = 2 - data_in = fluid.data(name='data_in', - shape=[batch_size, seq_len, hidden], - dtype=DTYPE) + data_in = fluid.data( + name='data_in', shape=[batch_size, seq_len, hidden], dtype=DTYPE + ) if dist_strategy: data_loader = fluid.io.DataLoader.from_generator( feed_list=[data_in], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) if dist_strategy: fleet.init(is_collective=True) @@ -128,8 +136,9 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): opt = fluid.optimizer.SGD(0.1) if dist_strategy: - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) diff --git a/python/paddle/fluid/tests/unittests/static_model_parallel_fused_feedforward.py b/python/paddle/fluid/tests/unittests/static_model_parallel_fused_feedforward.py index 409c84eaab583..f15dd57123f8c 100644 --- a/python/paddle/fluid/tests/unittests/static_model_parallel_fused_feedforward.py +++ b/python/paddle/fluid/tests/unittests/static_model_parallel_fused_feedforward.py @@ -30,20 +30,22 @@ def get_param_attr(weight, bias): weight_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(weight)) + initializer=fluid.initializer.NumpyArrayInitializer(weight) + ) bias_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(bias)) + initializer=fluid.initializer.NumpyArrayInitializer(bias) + ) return weight_attr, bias_attr def create_model(data, rank): np.random.seed(2021) - ln_w = np.random.uniform(-1, 1, size=(IN_SIZE, )).astype(DTYPE) - ln_b = np.random.uniform(-1, 1, size=(IN_SIZE, )).astype(DTYPE) + ln_w = np.random.uniform(-1, 1, size=(IN_SIZE,)).astype(DTYPE) + ln_b = np.random.uniform(-1, 1, size=(IN_SIZE,)).astype(DTYPE) w0 = np.random.uniform(-1, 1, size=(IN_SIZE, OUT_SIZE)).astype(DTYPE) - b0 = np.random.uniform(-1, 1, size=(OUT_SIZE, )).astype(DTYPE) + b0 = np.random.uniform(-1, 1, size=(OUT_SIZE,)).astype(DTYPE) w1 = np.random.uniform(-1, 1, size=(OUT_SIZE, IN_SIZE)).astype(DTYPE) - b1 = np.random.uniform(-1, 1, size=(IN_SIZE, )).astype(DTYPE) + b1 = np.random.uniform(-1, 1, size=(IN_SIZE,)).astype(DTYPE) data.stop_gradient = False if rank is not None: start = 0 if rank == 0 else OUT_SIZE // MODEL_PARALLEL_SIZE @@ -56,38 +58,42 @@ def create_model(data, rank): w0_attr, b0_attr = get_param_attr(col_w0, col_b0) w1_attr, b1_attr = get_param_attr(row_w1, b1) - ffn = FusedFeedForward(IN_SIZE, - OUT_SIZE, - dropout_rate=0.0, - activation='gelu', - normalize_before=True, - linear1_weight_attr=w0_attr, - linear1_bias_attr=b0_attr, - linear2_weight_attr=w1_attr, - linear2_bias_attr=b1_attr, - ln1_scale_attr=ln_w_attr, - ln1_bias_attr=ln_b_attr, - nranks=MODEL_PARALLEL_SIZE, - ring_id=0) - #ffn.eval() + ffn = FusedFeedForward( + IN_SIZE, + OUT_SIZE, + dropout_rate=0.0, + activation='gelu', + normalize_before=True, + linear1_weight_attr=w0_attr, + linear1_bias_attr=b0_attr, + linear2_weight_attr=w1_attr, + linear2_bias_attr=b1_attr, + ln1_scale_attr=ln_w_attr, + ln1_bias_attr=ln_b_attr, + nranks=MODEL_PARALLEL_SIZE, + ring_id=0, + ) + # ffn.eval() result = ffn(data) else: ln_w_attr, ln_b_attr = get_param_attr(ln_w, ln_b) w0_attr, b0_attr = get_param_attr(w0, b0) w1_attr, b1_attr = get_param_attr(w1, b1) - ffn = FusedFeedForward(IN_SIZE, - OUT_SIZE, - dropout_rate=0.0, - activation='gelu', - normalize_before=True, - linear1_weight_attr=w0_attr, - linear1_bias_attr=b0_attr, - linear2_weight_attr=w1_attr, - linear2_bias_attr=b1_attr, - ln1_scale_attr=ln_w_attr, - ln1_bias_attr=ln_b_attr) - #ffn.eval() + ffn = FusedFeedForward( + IN_SIZE, + OUT_SIZE, + dropout_rate=0.0, + activation='gelu', + normalize_before=True, + linear1_weight_attr=w0_attr, + linear1_bias_attr=b0_attr, + linear2_weight_attr=w1_attr, + linear2_bias_attr=b1_attr, + ln1_scale_attr=ln_w_attr, + ln1_bias_attr=ln_b_attr, + ) + # ffn.eval() result = ffn(data) predict = paddle.sum(result) @@ -95,20 +101,20 @@ def create_model(data, rank): class TestModelParallel(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data seq_len = 2 - data_in = fluid.data(name='data_in', - shape=[batch_size, seq_len, IN_SIZE], - dtype=DTYPE) + data_in = fluid.data( + name='data_in', shape=[batch_size, seq_len, IN_SIZE], dtype=DTYPE + ) if dist_strategy: data_loader = fluid.io.DataLoader.from_generator( feed_list=[data_in], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) if dist_strategy: fleet.init(is_collective=True) @@ -121,8 +127,9 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): opt = fluid.optimizer.SGD(0.1) if dist_strategy: - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) diff --git a/python/paddle/fluid/tests/unittests/static_model_parallel_fused_multi_transformer.py b/python/paddle/fluid/tests/unittests/static_model_parallel_fused_multi_transformer.py index 0ab441d296316..f9125dae34567 100644 --- a/python/paddle/fluid/tests/unittests/static_model_parallel_fused_multi_transformer.py +++ b/python/paddle/fluid/tests/unittests/static_model_parallel_fused_multi_transformer.py @@ -25,9 +25,11 @@ def get_param_attr(weight, bias): weight_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(weight)) + initializer=fluid.initializer.NumpyArrayInitializer(weight) + ) bias_attr = paddle.ParamAttr( - initializer=fluid.initializer.NumpyArrayInitializer(bias)) + initializer=fluid.initializer.NumpyArrayInitializer(bias) + ) return weight_attr, bias_attr @@ -41,28 +43,30 @@ def get_param_attr(weight, bias): def create_model(data, rank): np.random.seed(2021) - ln_w = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) - ln_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) - qkv_w = np.random.uniform(-1, 1, size=(3, num_head, dim_head, - hidden)).astype(DTYPE) + ln_w = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) + ln_b = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) + qkv_w = np.random.uniform( + -1, 1, size=(3, num_head, dim_head, hidden) + ).astype(DTYPE) qkv_b = np.random.uniform(-1, 1, size=(3, num_head, dim_head)).astype(DTYPE) - linear_w = np.random.uniform(-1, 1, size=(num_head * dim_head, - hidden)).astype(DTYPE) - linear_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) + linear_w = np.random.uniform( + -1, 1, size=(num_head * dim_head, hidden) + ).astype(DTYPE) + linear_b = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) - ffn_ln_w = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) - ffn_ln_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) + ffn_ln_w = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) + ffn_ln_b = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) ffn1_w = np.random.uniform(-1, 1, size=(hidden, dim_ffn)).astype(DTYPE) - ffn1_b = np.random.uniform(-1, 1, size=(dim_ffn, )).astype(DTYPE) + ffn1_b = np.random.uniform(-1, 1, size=(dim_ffn,)).astype(DTYPE) ffn2_w = np.random.uniform(-1, 1, size=(dim_ffn, hidden)).astype(DTYPE) - ffn2_b = np.random.uniform(-1, 1, size=(hidden, )).astype(DTYPE) + ffn2_b = np.random.uniform(-1, 1, size=(hidden,)).astype(DTYPE) if rank is not None: start = 0 if rank == 0 else (num_head // MODEL_PARALLEL_SIZE) end = start + (num_head // MODEL_PARALLEL_SIZE) col_qkv_w = qkv_w[:, start:end, :, :] col_qkv_b = qkv_b[:, start:end, :] - row_linear_w = linear_w[(start * dim_head):(end * dim_head), :] + row_linear_w = linear_w[(start * dim_head) : (end * dim_head), :] ln_w_attr, ln_b_attr = get_param_attr(ln_w, ln_b) qkv_w_attr, qkv_b_attr = get_param_attr(col_qkv_w, col_qkv_b) @@ -98,7 +102,8 @@ def create_model(data, rank): ffn2_weight_attrs=[ffn2_w_attr], ffn2_bias_attrs=[ffn2_b_attr], nranks=MODEL_PARALLEL_SIZE, - ring_id=0) + ring_id=0, + ) result = multi_transformer(data) else: ln_w_attr, ln_b_attr = get_param_attr(ln_w, ln_b) @@ -127,7 +132,8 @@ def create_model(data, rank): ffn1_weight_attrs=[ffn1_w_attr], ffn1_bias_attrs=[ffn1_b_attr], ffn2_weight_attrs=[ffn2_w_attr], - ffn2_bias_attrs=[ffn2_b_attr]) + ffn2_bias_attrs=[ffn2_b_attr], + ) result = multi_transformer(data) # fused_multi_transformer have no backward @@ -137,20 +143,20 @@ def create_model(data, rank): class TestModelParallel(TestDistRunnerBase): - def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): # Input data seq_len = 2 - data_in = fluid.data(name='data_in', - shape=[batch_size, seq_len, hidden], - dtype=DTYPE) + data_in = fluid.data( + name='data_in', shape=[batch_size, seq_len, hidden], dtype=DTYPE + ) if dist_strategy: data_loader = fluid.io.DataLoader.from_generator( feed_list=[data_in], capacity=64, use_double_buffer=False, - iterable=False) + iterable=False, + ) if dist_strategy: fleet.init(is_collective=True) @@ -163,8 +169,9 @@ def get_model(self, batch_size=2, use_dgc=False, dist_strategy=None): opt = fluid.optimizer.SGD(0.1) if dist_strategy: - dist_opt = fleet.distributed_optimizer(optimizer=opt, - strategy=strategy) + dist_opt = fleet.distributed_optimizer( + optimizer=opt, strategy=strategy + ) dist_opt.minimize(avg_cost) else: opt.minimize(avg_cost) diff --git a/python/paddle/fluid/tests/unittests/test_Tensor_type.py b/python/paddle/fluid/tests/unittests/test_Tensor_type.py index 2211aed945e31..f18818b8ef5d9 100644 --- a/python/paddle/fluid/tests/unittests/test_Tensor_type.py +++ b/python/paddle/fluid/tests/unittests/test_Tensor_type.py @@ -20,7 +20,6 @@ class TensorTypeTest(unittest.TestCase): - def func_type_totensor(self): paddle.disable_static() inx = np.array([1, 2]) diff --git a/python/paddle/fluid/tests/unittests/test_accuracy_op.py b/python/paddle/fluid/tests/unittests/test_accuracy_op.py index 834ab8f77d864..0de81034d3373 100755 --- a/python/paddle/fluid/tests/unittests/test_accuracy_op.py +++ b/python/paddle/fluid/tests/unittests/test_accuracy_op.py @@ -21,7 +21,6 @@ class TestAccuracyOp(OpTest): - def setUp(self): self.op_type = "accuracy" self.dtype = np.float32 @@ -40,7 +39,7 @@ def setUp(self): self.outputs = { 'Accuracy': np.array([num_correct / float(n)]).astype(self.dtype), 'Correct': np.array([num_correct]).astype("int32"), - 'Total': np.array([n]).astype("int32") + 'Total': np.array([n]).astype("int32"), } def init_dtype(self): @@ -51,7 +50,6 @@ def test_check_output(self): class TestAccuracyOpFp16(TestAccuracyOp): - def init_dtype(self): self.dtype = np.float16 @@ -60,15 +58,15 @@ def test_check_output(self): class TestAccuracyOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of accuracy_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - label = fluid.layers.data(name='label', - shape=[-1, 1], - dtype="int32") + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + label = fluid.layers.data( + name='label', shape=[-1, 1], dtype="int32" + ) self.assertRaises(TypeError, fluid.layers.accuracy, x1, label) self.assertRaises(TypeError, paddle.metric.accuracy, x1, label) # The input dtype of accuracy_op must be float32 or float64. @@ -81,40 +79,42 @@ def test_errors(self): class TestAccuracyAPI1(unittest.TestCase): - def setUp(self): - self.predictions = paddle.static.data(shape=[2, 5], - name="predictions", - dtype="float32") - self.label = paddle.static.data(shape=[2, 1], - name="labels", - dtype="int64") - self.result = paddle.static.accuracy(input=self.predictions, - label=self.label, - k=1) + self.predictions = paddle.static.data( + shape=[2, 5], name="predictions", dtype="float32" + ) + self.label = paddle.static.data( + shape=[2, 1], name="labels", dtype="int64" + ) + self.result = paddle.static.accuracy( + input=self.predictions, label=self.label, k=1 + ) self.input_predictions = np.array( [[0.2, 0.1, 0.4, 0.1, 0.1], [0.2, 0.3, 0.1, 0.15, 0.25]], - dtype="float32") + dtype="float32", + ) self.input_labels = np.array([[2], [0]], dtype="int64") self.expect_value = np.array([0.5], dtype='float32') def test_api(self): exe = paddle.static.Executor() - result, = exe.run(feed={ - "predictions": self.input_predictions, - 'labels': self.input_labels - }, - fetch_list=[self.result.name]) + (result,) = exe.run( + feed={ + "predictions": self.input_predictions, + 'labels': self.input_labels, + }, + fetch_list=[self.result.name], + ) self.assertEqual((result == self.expect_value).all(), True) class TestAccuracyAPI2(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): predictions = paddle.to_tensor( [[0.2, 0.1, 0.4, 0.1, 0.1], [0.2, 0.3, 0.1, 0.15, 0.25]], - dtype='float32') + dtype='float32', + ) label = paddle.to_tensor([[2], [0]], dtype="int64") result = paddle.static.accuracy(input=predictions, label=label, k=1) expect_value = np.array([0.5], dtype='float32') @@ -122,12 +122,12 @@ def test_api(self): class TestAccuracyAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): predictions = paddle.to_tensor( [[0.2, 0.1, 0.4, 0.1, 0.1], [0.2, 0.3, 0.1, 0.15, 0.25]], - dtype='float32') + dtype='float32', + ) label = paddle.to_tensor([[2], [0]], dtype="int64") result = paddle.metric.accuracy(input=predictions, label=label, k=1) expect_value = np.array([0.5], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py b/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py index cee8310cd1805..bc5a14070e5ce 100644 --- a/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py @@ -26,7 +26,6 @@ class TestSigmoidTripleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -37,11 +36,9 @@ def func(self, place): y = layers.sigmoid(x) x_arr = np.random.random(shape).astype(dtype) x_arr[np.abs(x_arr) < 0.005] = 0.002 - gradient_checker.triple_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -53,7 +50,6 @@ def test_grad(self): class TestSigmoidDoubleGradCheck(unittest.TestCase): - def sigmoid_wrapper(self, x): return fluid.layers.sigmoid(x[0]) @@ -67,17 +63,13 @@ def func(self, place): y = layers.sigmoid(x) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) x_arr[np.abs(x_arr) < 0.005] = 0.002 - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.sigmoid_wrapper, - [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.sigmoid_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -90,7 +82,6 @@ def test_grad(self): class TestTanhTripleGradCheck(unittest.TestCase): - def tanh_wrapper(self, x): return paddle.tanh(x[0]) @@ -104,16 +95,13 @@ def func(self, place): y = layers.tanh(x) x_arr = np.random.random(shape).astype(dtype) x_arr[np.abs(x_arr) < 0.005] = 0.002 - gradient_checker.triple_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.tanh_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.tanh_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -126,7 +114,6 @@ def test_grad(self): class TestTanhDoubleGradCheck(unittest.TestCase): - def tanh_wrapper(self, x): return paddle.tanh(x[0]) @@ -140,16 +127,13 @@ def func(self, place): y = paddle.tanh(x) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) x_arr[np.abs(x_arr) < 0.005] = 0.002 - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.tanh_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.tanh_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -162,7 +146,6 @@ def test_grad(self): class TestAbsDoubleGradCheck(unittest.TestCase): - def abs_wrapper(self, x): return paddle.abs(x[0]) @@ -176,16 +159,13 @@ def func(self, place): y = paddle.abs(x) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) x_arr[np.abs(x_arr) < 0.005] = 0.002 - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.abs_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.abs_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -198,7 +178,6 @@ def test_grad(self): class TestReluDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -211,11 +190,9 @@ def func(self, place): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) x_arr[np.abs(x_arr) < 0.005] = 0.02 - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -227,7 +204,6 @@ def test_grad(self): class TestLeakyReluDoubleGradCheck(unittest.TestCase): - def leaky_relu_wrapper(self, x): return paddle.nn.functional.leaky_relu(x[0], negative_slope=0.2) @@ -245,16 +221,12 @@ def func(self, place): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) x_arr[np.abs(x_arr) < 0.005] = 0.02 - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.leaky_relu_wrapper, - [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.leaky_relu_wrapper, [x], y, x_init=x_arr, place=place + ) def test_grad(self): paddle.enable_static() @@ -266,7 +238,6 @@ def test_grad(self): class TestELUDoubleGradCheck(unittest.TestCase): - def elu_wrapper(self, x): return paddle.nn.functional.elu(x[0], alpha=0.2) @@ -284,16 +255,13 @@ def func(self, place): y = layers.elu(x, alpha=alpha) np.random.RandomState(SEED) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.elu_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.elu_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -306,7 +274,6 @@ def test_grad(self): class TestCELUDoubleGradCheck(unittest.TestCase): - def celu_wrapper(self, x): return paddle.nn.functional.celu(x[0], alpha=0.2) @@ -324,16 +291,13 @@ def func(self, place): y = F.celu(x, alpha=alpha) np.random.RandomState(SEED) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.celu_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.celu_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -346,7 +310,6 @@ def test_grad(self): class TestSqrtDoubleGradCheck(unittest.TestCase): - def sqrt_wrapper(self, x): return paddle.sqrt(x[0]) @@ -362,15 +325,12 @@ def func(self, place): y = layers.sqrt(x) x_arr = np.random.uniform(0.1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.sqrt_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.sqrt_wrapper, [x], y, x_init=x_arr, place=place + ) def test_grad(self): paddle.enable_static() @@ -382,7 +342,6 @@ def test_grad(self): class TestRsqrtDoubleGradCheck(unittest.TestCase): - def rsqrt_wrapper(self, x): return paddle.rsqrt(x[0]) @@ -398,15 +357,12 @@ def func(self, place): y = layers.rsqrt(x) x_arr = np.random.uniform(0.1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.rsqrt_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.rsqrt_wrapper, [x], y, x_init=x_arr, place=place + ) def test_grad(self): paddle.enable_static() @@ -418,7 +374,6 @@ def test_grad(self): class TestSquareDoubleGradCheck(unittest.TestCase): - def square_wrapper(self, x): return paddle.square(x[0]) @@ -434,16 +389,13 @@ def func(self, place): y = layers.square(x) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.square_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.square_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -456,7 +408,6 @@ def test_grad(self): class TestAbsDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -473,11 +424,9 @@ def func(self, place): # we should avoid this x_arr[np.abs(x_arr) < 0.005] = 0.02 - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -489,7 +438,6 @@ def test_grad(self): class TestLogDoubleGradCheck(unittest.TestCase): - def log_wrapper(self, x): return paddle.log(x[0]) @@ -505,16 +453,13 @@ def func(self, place): x_arr = np.random.uniform(0.1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.log_wrapper, [x], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.log_wrapper, [x], y, x_init=x_arr, place=place + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): diff --git a/python/paddle/fluid/tests/unittests/test_activation_op.py b/python/paddle/fluid/tests/unittests/test_activation_op.py index 49c19f1cbff9a..a933d591a8798 100755 --- a/python/paddle/fluid/tests/unittests/test_activation_op.py +++ b/python/paddle/fluid/tests/unittests/test_activation_op.py @@ -29,26 +29,24 @@ class TestSqrtOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of sqrt op must be Variable or numpy.ndarray. in1 = 1 self.assertRaises(TypeError, fluid.layers.sqrt, in1) # The input dtype of sqrt op must be float16, float32, float64. - in2 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") + in2 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) self.assertRaises(TypeError, fluid.layers.sqrt, in2) - in3 = fluid.layers.data(name='input3', - shape=[12, 10], - dtype="float16") + in3 = fluid.layers.data( + name='input3', shape=[12, 10], dtype="float16" + ) fluid.layers.sqrt(x=in3) class TestActivation(OpTest): - def setUp(self): self.op_type = "exp" self.init_dtype() @@ -89,13 +87,11 @@ def init_kernel_type(self): class TestActivation_ZeroDim(TestActivation): - def init_shape(self): self.shape = [] class TestExpm1(TestActivation): - def setUp(self): self.op_type = "expm1" self.python_api = paddle.expm1 @@ -117,13 +113,11 @@ def test_check_output(self): class TestExpm1_ZeroDim(TestExpm1): - def init_shape(self): self.shape = [] class TestExpm1API(unittest.TestCase): - def init_dtype(self): self.dtype = 'float64' self.shape = [11, 17] @@ -153,7 +147,6 @@ def run(place): run(place) def test_dygraph_api(self): - def run(place): paddle.disable_static(place) X = paddle.to_tensor(self.x) @@ -173,7 +166,6 @@ def test_errors(self): class TestParameter(object): - def test_out_name(self): with fluid.program_guard(fluid.Program()): np_x = np.array([0.1]) @@ -181,7 +173,7 @@ def test_out_name(self): out = eval("paddle.%s(data, name='Y')" % self.op_type) place = fluid.CPUPlace() exe = fluid.Executor(place) - result, = exe.run(feed={"X": np_x}, fetch_list=[out]) + (result,) = exe.run(feed={"X": np_x}, fetch_list=[out]) expected = eval("np.%s(np_x)" % self.op_type) np.testing.assert_allclose(result, expected, rtol=1e-05) @@ -195,7 +187,6 @@ def test_dygraph(self): class TestSigmoid(TestActivation): - def setUp(self): self.op_type = "sigmoid" self.init_dtype() @@ -218,15 +209,14 @@ def test_check_grad(self): class TestSigmoid_ZeroDim(TestSigmoid): - def init_shape(self): self.shape = [] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSigmoidBF16(OpTest): - def setUp(self): self.op_type = "sigmoid" self.init_dtype() @@ -265,7 +255,6 @@ def init_shape(self): class TestSilu(TestActivation): - def setUp(self): self.op_type = "silu" self.init_dtype() @@ -288,7 +277,6 @@ def test_check_grad(self): class TestSilu_ZeroDim(TestSilu): - def init_shape(self): self.shape = [] @@ -297,8 +285,11 @@ class TestSiluAPI(unittest.TestCase): # test paddle.nn.Silu, paddle.nn.functional.silu def setUp(self): self.x_np = np.random.uniform(-1, 1, [11, 17]).astype('float32') - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -329,19 +320,18 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.silu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[11, 17], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[11, 17], dtype='int32' + ) self.assertRaises(TypeError, F.silu, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[11, 17], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[11, 17], dtype='float16' + ) F.silu(x_fp16) class TestLogSigmoid(TestActivation): - def setUp(self): self.op_type = "logsigmoid" self.init_dtype() @@ -361,7 +351,6 @@ def test_check_grad(self): class TestLogSigmoid_ZeroDim(TestLogSigmoid): - def init_shape(self): self.shape = [] @@ -371,8 +360,11 @@ class TestLogSigmoidAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [11, 17]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -414,19 +406,18 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.log_sigmoid, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[11, 17], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[11, 17], dtype='int32' + ) self.assertRaises(TypeError, F.log_sigmoid, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[11, 17], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[11, 17], dtype='float16' + ) F.log_sigmoid(x_fp16) class TestTanh(TestActivation, TestParameter): - def setUp(self): self.op_type = "tanh" self.init_dtype() @@ -445,14 +436,13 @@ def test_check_grad(self): self.check_grad(['X'], 'Out') def init_dtype(self): - #TODO If dtype is float64, the output (Out) has diff at CPUPlace + # TODO If dtype is float64, the output (Out) has diff at CPUPlace # when using and not using inplace. Therefore, set dtype as float32 # for now. self.dtype = np.float32 class TestTanh_ZeroDim(TestTanh): - def init_shape(self): self.shape = [] @@ -463,8 +453,11 @@ def setUp(self): self.dtype = 'float32' np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype(self.dtype) - self.place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) self.executed_api() def executed_api(self): @@ -511,14 +504,14 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, self.tanh, 1) # The input dtype must be float16, float32. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, self.tanh, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) self.tanh(x_fp16) @@ -529,7 +522,6 @@ def executed_api(self): class TestAtan(TestActivation, TestParameter): - def setUp(self): self.op_type = "atan" self.init_dtype() @@ -554,7 +546,7 @@ def test_out_name(self): out = paddle.atan(data, name='Y') place = fluid.CPUPlace() exe = fluid.Executor(place) - result, = exe.run(feed={"X": np_x}, fetch_list=[out]) + (result,) = exe.run(feed={"X": np_x}, fetch_list=[out]) expected = np.arctan(np_x) self.assertEqual(result, expected) @@ -568,13 +560,11 @@ def test_dygraph(self): class TestAtan_ZeroDim(TestTanh): - def init_shape(self): self.shape = [] class TestSinh(TestActivation): - def setUp(self): self.op_type = "sinh" self.init_dtype() @@ -594,13 +584,11 @@ def test_check_grad(self): class TestSinh_ZeroDim(TestSinh): - def init_shape(self): self.shape = [] class TestSinhAPI(unittest.TestCase): - def test_dygraph(self): with fluid.dygraph.guard(): np_x = np.array([0.1]) @@ -612,19 +600,24 @@ def test_dygraph(self): def test_api(self): test_data_shape = [11, 17] with fluid.program_guard(fluid.Program(), fluid.Program()): - input_x = np.random.uniform(0.1, 1, - test_data_shape).astype("float32") - data_x = fluid.layers.data(name="data_x", - shape=test_data_shape, - append_batch_size=False, - dtype="float32") + input_x = np.random.uniform(0.1, 1, test_data_shape).astype( + "float32" + ) + data_x = fluid.layers.data( + name="data_x", + shape=test_data_shape, + append_batch_size=False, + dtype="float32", + ) pd_sinh_out = fluid.layers.sinh(data_x) exe = fluid.Executor(place=fluid.CPUPlace()) exe.run(fluid.default_startup_program()) - np_sinh_res, = exe.run(fluid.default_main_program(), - feed={"data_x": input_x}, - fetch_list=[pd_sinh_out]) + (np_sinh_res,) = exe.run( + fluid.default_main_program(), + feed={"data_x": input_x}, + fetch_list=[pd_sinh_out], + ) expected_res = np.sinh(input_x) np.testing.assert_allclose(np_sinh_res, expected_res, rtol=1e-05) @@ -632,8 +625,9 @@ def test_api(self): def test_backward(self): test_data_shape = [11, 17] with fluid.dygraph.guard(): - input_x = np.random.uniform(0.1, 1, - test_data_shape).astype("float32") + input_x = np.random.uniform(0.1, 1, test_data_shape).astype( + "float32" + ) var = fluid.dygraph.to_variable(input_x) var.stop_gradient = False loss = fluid.layers.sinh(var) @@ -643,7 +637,6 @@ def test_backward(self): class TestSinhOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program()): # The input type must be Variable. @@ -657,7 +650,6 @@ def test_errors(self): class TestCosh(TestActivation): - def setUp(self): self.op_type = "cosh" self.init_dtype() @@ -677,13 +669,11 @@ def test_check_grad(self): class TestCosh_ZeroDim(TestCosh): - def init_shape(self): self.shape = [] class TestCoshAPI(unittest.TestCase): - def test_dygraph(self): with fluid.dygraph.guard(): np_x = np.array([0.1]) @@ -695,19 +685,24 @@ def test_dygraph(self): def test_api(self): test_data_shape = [11, 17] with fluid.program_guard(fluid.Program(), fluid.Program()): - input_x = np.random.uniform(0.1, 1, - test_data_shape).astype("float32") - data_x = fluid.layers.data(name="data_x", - shape=test_data_shape, - append_batch_size=False, - dtype="float32") + input_x = np.random.uniform(0.1, 1, test_data_shape).astype( + "float32" + ) + data_x = fluid.layers.data( + name="data_x", + shape=test_data_shape, + append_batch_size=False, + dtype="float32", + ) pd_cosh_out = paddle.cosh(data_x) exe = fluid.Executor(place=fluid.CPUPlace()) exe.run(fluid.default_startup_program()) - np_cosh_res, = exe.run(fluid.default_main_program(), - feed={"data_x": input_x}, - fetch_list=[pd_cosh_out]) + (np_cosh_res,) = exe.run( + fluid.default_main_program(), + feed={"data_x": input_x}, + fetch_list=[pd_cosh_out], + ) expected_res = np.cosh(input_x) np.testing.assert_allclose(np_cosh_res, expected_res, rtol=1e-05) @@ -715,8 +710,9 @@ def test_api(self): def test_backward(self): test_data_shape = [11, 17] with fluid.dygraph.guard(): - input_x = np.random.uniform(0.1, 1, - test_data_shape).astype("float32") + input_x = np.random.uniform(0.1, 1, test_data_shape).astype( + "float32" + ) var = fluid.dygraph.to_variable(input_x) var.stop_gradient = False loss = fluid.layers.cosh(var) @@ -726,7 +722,6 @@ def test_backward(self): class TestCoshOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program()): # The input type must be Variable. @@ -745,7 +740,6 @@ def ref_tanhshrink(x): class TestTanhshrink(TestActivation): - def setUp(self): self.op_type = "tanh_shrink" self.init_dtype() @@ -765,7 +759,6 @@ def test_check_grad(self): class TestTanhshrink_ZeroDim(TestTanhshrink): - def init_shape(self): self.shape = [] @@ -775,8 +768,11 @@ class TestTanhshrinkAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(10, 20, [10, 17]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -818,14 +814,14 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.tanhshrink, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.tanhshrink, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.tanhshrink(x_fp16) @@ -836,7 +832,6 @@ def ref_hardshrink(x, threshold): class TestHardShrink(TestActivation): - def setUp(self): self.op_type = "hard_shrink" self.init_dtype() @@ -865,7 +860,6 @@ def test_check_grad(self): class TestHardShrink_threshold_negative(TestHardShrink): - def set_attrs(self): self.threshold = -0.1 @@ -883,8 +877,11 @@ class TestHardShrinkAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -933,14 +930,14 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.hardshrink, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.hardshrink, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.hardshrink(x_fp16) @@ -957,8 +954,11 @@ class TestHardtanhAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-3, 3, [10, 12]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -997,26 +997,26 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.hardtanh, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.hardtanh, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.hardtanh(x_fp16) def ref_softshrink(x, threshold=0.5): out = np.copy(x) out = (out < -threshold) * (out + threshold) + (out > threshold) * ( - out - threshold) + out - threshold + ) return out class TestSoftshrink(TestActivation): - def setUp(self): self.op_type = "softshrink" self.check_eager = True @@ -1040,7 +1040,6 @@ def test_check_grad(self): class TestSoftshrink_ZeroDim(TestSoftshrink): - def init_shape(self): self.shape = [] @@ -1051,8 +1050,11 @@ def setUp(self): self.threshold = 0.8 np.random.seed(1024) self.x_np = np.random.uniform(0.25, 10, [10, 12]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -1094,24 +1096,23 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.softshrink, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.softshrink, x_int32) # The threshold must be no less than zero - x_fp32 = paddle.fluid.data(name='x_fp32', - shape=[12, 10], - dtype='float32') + x_fp32 = paddle.fluid.data( + name='x_fp32', shape=[12, 10], dtype='float32' + ) self.assertRaises(ValueError, F.softshrink, x_fp32, -1.0) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.softshrink(x_fp16) class TestSqrt(TestActivation, TestParameter): - def setUp(self): self.op_type = "sqrt" self.python_api = paddle.sqrt @@ -1135,15 +1136,14 @@ def test_check_output(self): class TestSqrt_ZeroDim(TestSqrt): - def init_shape(self): self.shape = [] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSqrtBF16(OpTest): - def setUp(self): self.op_type = "sqrt" self.python_api = paddle.sqrt @@ -1175,7 +1175,6 @@ def test_check_grad(self): class TestRsqrt(TestActivation): - def setUp(self): self.op_type = "rsqrt" self.python_api = paddle.rsqrt @@ -1195,10 +1194,9 @@ def init_shape(self): def test_check_grad(self): if self.dtype == np.float16: return - self.check_grad(['X'], - 'Out', - max_relative_error=0.0005, - check_eager=True) + self.check_grad( + ['X'], 'Out', max_relative_error=0.0005, check_eager=True + ) ''' @@ -1210,7 +1208,6 @@ def init_shape(self): class TestAbs(TestActivation): - def setUp(self): self.op_type = "abs" self.init_dtype() @@ -1238,13 +1235,11 @@ def test_check_grad(self): class TestAbs_ZeroDim(TestAbs): - def init_shape(self): self.shape = [] class TestCeil(TestActivation): - def setUp(self): self.op_type = "ceil" self.check_eager = True @@ -1268,13 +1263,11 @@ def test_check_grad(self): class TestCeil_ZeroDim(TestCeil): - def init_shape(self): self.shape = [] class TestFloor(TestActivation): - def setUp(self): self.op_type = "floor" self.check_eager = True @@ -1300,13 +1293,11 @@ def test_check_grad(self): class TestFloor_ZeroDim(TestFloor): - def init_shape(self): self.shape = [] class TestCos(TestActivation): - def setUp(self): self.op_type = "cos" self.init_dtype() @@ -1329,13 +1320,11 @@ def test_check_grad(self): class TestCos_ZeroDim(TestCos): - def init_shape(self): self.shape = [] class TestTan(TestActivation): - def setUp(self): np.random.seed(1024) self.op_type = "tan" @@ -1344,8 +1333,11 @@ def setUp(self): self.dtype = 'float32' self.x_np = np.random.uniform(-1, 1, self.shape).astype(self.dtype) - self.place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) out = np.tan(self.x_np) @@ -1362,19 +1354,20 @@ def test_check_grad(self): class TestTan_ZeroDim(TestTan): - def init_shape(self): self.shape = [] class TestTanAPI(unittest.TestCase): - def setUp(self): np.random.seed(1024) self.dtype = 'float32' self.x_np = np.random.uniform(-1, 1, [11, 17]).astype(self.dtype) - self.place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_dygraph_api(self): paddle.disable_static(self.place) @@ -1397,8 +1390,9 @@ def test_static_api(self): def test_backward(self): test_data_shape = [11, 17] with fluid.dygraph.guard(): - input_x = np.random.uniform(0.1, 1, - test_data_shape).astype("float32") + input_x = np.random.uniform(0.1, 1, test_data_shape).astype( + "float32" + ) var = paddle.to_tensor(input_x) var.stop_gradient = False loss = paddle.tan(var) @@ -1408,7 +1402,6 @@ def test_backward(self): class TestAcos(TestActivation): - def setUp(self): self.op_type = "acos" self.init_dtype() @@ -1431,13 +1424,11 @@ def test_check_grad(self): class TestAcos_ZeroDim(TestAcos): - def init_shape(self): self.shape = [] class TestSin(TestActivation, TestParameter): - def setUp(self): self.op_type = "sin" self.init_dtype() @@ -1460,13 +1451,11 @@ def test_check_grad(self): class TestSin_ZeroDim(TestSin): - def init_shape(self): self.shape = [] class TestAsin(TestActivation): - def setUp(self): self.op_type = "asin" self.init_dtype() @@ -1489,13 +1478,11 @@ def test_check_grad(self): class TestAsin_ZeroDim(TestAsin): - def init_shape(self): self.shape = [] class TestAcosh(TestActivation): - def setUp(self): self.op_type = "acosh" self.init_dtype() @@ -1518,13 +1505,11 @@ def test_check_grad(self): class TestAcosh_ZeroDim(TestAcosh): - def init_shape(self): self.shape = [] class TestAsinh(TestActivation): - def setUp(self): self.op_type = "asinh" self.init_dtype() @@ -1547,13 +1532,11 @@ def test_check_grad(self): class TestAsinh_ZeroDim(TestAsinh): - def init_shape(self): self.shape = [] class TestAtanh(TestActivation): - def setUp(self): self.op_type = "atanh" self.init_dtype() @@ -1576,13 +1559,11 @@ def test_check_grad(self): class TestAtanh_ZeroDim(TestAtanh): - def init_shape(self): self.shape = [] class TestRound(TestActivation): - def setUp(self): self.op_type = "round" self.check_eager = True @@ -1605,13 +1586,11 @@ def test_check_grad(self): class TestRound_ZeroDim(TestRound): - def init_shape(self): self.shape = [] class TestRelu(TestActivation): - def setUp(self): self.op_type = "relu" self.init_dtype() @@ -1640,7 +1619,6 @@ def test_check_grad(self): class TestRelu_ZeroDim(TestRelu): - def init_shape(self): self.shape = [] @@ -1650,8 +1628,11 @@ class TestReluAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) self.executed_api() def executed_api(self): @@ -1687,14 +1668,14 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, self.relu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[10, 12], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[10, 12], dtype='int32' + ) self.assertRaises(TypeError, self.relu, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[10, 12], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[10, 12], dtype='float16' + ) self.relu(x_fp16) @@ -1711,7 +1692,6 @@ def ref_leaky_relu(x, alpha=0.01): class TestLeakyRelu(TestActivation): - def get_alpha(self): return 0.02 @@ -1738,25 +1718,21 @@ def test_check_grad(self): class TestLeakyReluAlpha1(TestLeakyRelu): - def get_alpha(self): return 2 class TestLeakyReluAlpha2(TestLeakyRelu): - def get_alpha(self): return -0.01 class TestLeakyReluAlpha3(TestLeakyRelu): - def get_alpha(self): return -2.0 class TestLeakyRelu_ZeroDim(TestLeakyRelu): - def init_shape(self): self.shape = [] @@ -1767,8 +1743,11 @@ class TestLeakyReluAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -1817,28 +1796,33 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.leaky_relu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.leaky_relu, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.leaky_relu(x_fp16) def gelu(x, approximate): if approximate: - y_ref = 0.5 * x * ( - 1.0 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)))) + y_ref = ( + 0.5 + * x + * ( + 1.0 + + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3))) + ) + ) else: y_ref = 0.5 * x * (1 + erf(x / np.sqrt(2))) return y_ref.astype(x.dtype) class TestGeluApproximate(TestActivation): - def setUp(self): self.op_type = "gelu" self.init_dtype() @@ -1859,7 +1843,6 @@ def test_check_grad(self): class TestGelu(TestActivation): - def setUp(self): self.op_type = "gelu" self.init_dtype() @@ -1880,7 +1863,6 @@ def test_check_grad(self): class TestGelu_ZeroDim(TestGelu): - def init_shape(self): self.shape = [] @@ -1890,8 +1872,11 @@ class TestGELUAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [11, 17]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -1930,19 +1915,18 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.gelu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[11, 17], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[11, 17], dtype='int32' + ) self.assertRaises(TypeError, F.gelu, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[11, 17], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[11, 17], dtype='float16' + ) F.gelu(x_fp16) class TestBRelu(TestActivation): - def setUp(self): self.op_type = "brelu" self.init_dtype() @@ -1972,15 +1956,18 @@ class TestBreluAPI(unittest.TestCase): # test paddle.fluid.layers.brelu def setUp(self): np.random.seed(1024) - self.t_min = 0. - self.t_max = 24. + self.t_min = 0.0 + self.t_max = 24.0 self.x_np = np.random.uniform(-1, 30, [10, 12]).astype('float32') self.out_ref = np.copy(self.x_np) self.out_ref[self.out_ref < self.t_min] = self.t_min self.out_ref[self.out_ref > self.t_max] = self.t_max self.out_ref = self.out_ref.astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_fluid_api(self): with paddle.static.program_guard(paddle.static.Program()): @@ -2004,9 +1991,9 @@ def test_errors(self): x_int32 = fluid.data(name='x_int32', shape=[12, 10], dtype='int32') self.assertRaises(TypeError, fluid.layers.brelu, x_int32) # support the input dtype is float16 - x_fp16 = fluid.layers.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = fluid.layers.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) fluid.layers.brelu(x_fp16) @@ -2018,7 +2005,6 @@ def ref_relu6(x, threshold=6.0): class TestRelu6(TestActivation): - def setUp(self): self.op_type = "relu6" self.init_dtype() @@ -2044,7 +2030,6 @@ def test_check_grad(self): class TestRelu6_ZeroDim(TestRelu6): - def init_shape(self): self.shape = [] @@ -2055,8 +2040,11 @@ def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 10, [10, 12]).astype(np.float64) self.x_np[np.abs(self.x_np) < 0.005] = 0.02 - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -2098,14 +2086,14 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.relu6, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.relu6, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.relu6(x_fp16) @@ -2114,12 +2102,12 @@ def ref_hardswish(x, threshold=6.0, scale=6.0, offset=3.0): if x_dtype == 'float16': x_dtype = 'float16' x = x.astype('float32') - return (x * np.minimum(np.maximum(x + offset, 0.), threshold) / - scale).astype(x_dtype) + return ( + x * np.minimum(np.maximum(x + offset, 0.0), threshold) / scale + ).astype(x_dtype) class TestHardSwish(TestActivation): - def setUp(self): self.op_type = 'hard_swish' self.init_dtype() @@ -2131,7 +2119,7 @@ def setUp(self): threshold = 6.0 scale = 6.0 offset = 3.0 - #the same with TestAbs + # the same with TestAbs x[np.abs(x + offset) < 0.005] = 0.02 x[np.abs(x - threshold + offset) < 0.005] = threshold - offset + 0.02 out = ref_hardswish(x, threshold, scale, offset) @@ -2151,7 +2139,6 @@ def test_check_output(self): class TestHardSwish_ZeroDim(TestHardSwish): - def init_shape(self): self.shape = [] @@ -2160,8 +2147,11 @@ class TestHardswishAPI(unittest.TestCase): # test paddle.nn.Hardswish, paddle.nn.functional.hardswish def setUp(self): self.x_np = np.random.uniform(-1, 1, [10, 12]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): with paddle.static.program_guard(paddle.static.Program()): @@ -2177,11 +2167,11 @@ def test_static_api(self): def test_dygraph_api(self): paddle.disable_static(self.place) - x = paddle.to_tensor([11648., 11448.]) + x = paddle.to_tensor([11648.0, 11448.0]) out1 = F.hardswish(x) m = paddle.nn.Hardswish() out2 = m(x) - out_ref = [11648., 11448.] + out_ref = [11648.0, 11448.0] for r in [out1, out2]: np.testing.assert_allclose(out_ref, r.numpy(), rtol=1e-05) paddle.enable_static() @@ -2206,14 +2196,14 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.hardswish, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.hardswish, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.hardswish(x_fp16) def test_api_eager_dygraph(self): @@ -2223,7 +2213,6 @@ def test_api_eager_dygraph(self): class TestSoftRelu(TestActivation): - def setUp(self): self.op_type = "soft_relu" self.init_dtype() @@ -2250,7 +2239,6 @@ def test_check_grad(self): class TestSoftReluOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program()): # The input type must be Variable. @@ -2269,7 +2257,6 @@ def elu(x, alpha): class TestELU(TestActivation): - def setUp(self): self.op_type = "elu" self.init_dtype() @@ -2294,17 +2281,15 @@ def test_check_grad(self): self.check_grad(['X'], 'Out') def get_alpha(self): - return 1. + return 1.0 class TestELUAlpha(TestELU): - def get_alpha(self): return -0.2 class TestELU_ZeroDim(TestELU): - def init_shape(self): self.shape = [] @@ -2314,8 +2299,11 @@ class TestELUAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-3, 3, [10, 12]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) self.executed_api() def executed_api(self): @@ -2360,14 +2348,14 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, self.elu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[10, 12], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[10, 12], dtype='int32' + ) self.assertRaises(TypeError, self.elu, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[10, 12], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[10, 12], dtype='float16' + ) self.elu(x_fp16) @@ -2389,7 +2377,6 @@ def celu(x, alpha): class TestCELU(TestActivation): - def setUp(self): self.op_type = "celu" self.init_dtype() @@ -2414,7 +2401,6 @@ def test_check_grad(self): class TestCELU_ZeroDim(TestCELU): - def init_shape(self): self.shape = [] @@ -2424,8 +2410,11 @@ class TestCELUAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-3, 3, [10, 12]).astype('float32') - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) self.executed_api() def executed_api(self): @@ -2470,19 +2459,19 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, self.celu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[10, 12], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[10, 12], dtype='int32' + ) self.assertRaises(TypeError, self.celu, x_int32) # The alpha must be not equal 0 - x_fp32 = paddle.fluid.data(name='x_fp32', - shape=[10, 12], - dtype='float32') + x_fp32 = paddle.fluid.data( + name='x_fp32', shape=[10, 12], dtype='float32' + ) self.assertRaises(ZeroDivisionError, F.celu, x_fp32, 0) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[10, 12], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[10, 12], dtype='float16' + ) self.celu(x_fp16) def test_api_eager_dygraph(self): @@ -2492,7 +2481,6 @@ def test_api_eager_dygraph(self): class TestReciprocal(TestActivation): - def setUp(self): self.op_type = "reciprocal" self.python_api = paddle.reciprocal @@ -2516,13 +2504,11 @@ def test_check_output(self): class TestReciprocal_ZeroDim(TestReciprocal): - def init_shape(self): self.shape = [] class TestLog(TestActivation): - def setUp(self): self.op_type = "log" self.check_eager = True @@ -2543,27 +2529,23 @@ def test_check_grad(self): self.check_grad(['X'], 'Out', check_eager=True) def test_error(self): - in1 = fluid.layers.data(name="in1", - shape=[11, 17], - append_batch_size=False, - dtype="int32") - in2 = fluid.layers.data(name="in2", - shape=[11, 17], - append_batch_size=False, - dtype="int64") + in1 = fluid.layers.data( + name="in1", shape=[11, 17], append_batch_size=False, dtype="int32" + ) + in2 = fluid.layers.data( + name="in2", shape=[11, 17], append_batch_size=False, dtype="int64" + ) self.assertRaises(TypeError, fluid.layers.log, in1) self.assertRaises(TypeError, fluid.layers.log, in2) class TestLog_ZeroDim(TestLog): - def init_shape(self): self.shape = [] class TestLog2(TestActivation): - def setUp(self): self.op_type = "log2" self.check_eager = True @@ -2590,19 +2572,22 @@ def test_error(self): self.assertRaises(TypeError, paddle.log2, in2) def test_api(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): input_x = np.random.uniform(0.1, 1, [11, 17]).astype("float64") - data_x = paddle.static.data(name="data_x", - shape=[11, 17], - dtype="float64") + data_x = paddle.static.data( + name="data_x", shape=[11, 17], dtype="float64" + ) out1 = paddle.log2(data_x) exe = paddle.static.Executor(place=fluid.CPUPlace()) exe.run(paddle.static.default_startup_program()) - res1, = exe.run(paddle.static.default_main_program(), - feed={"data_x": input_x}, - fetch_list=[out1]) + (res1,) = exe.run( + paddle.static.default_main_program(), + feed={"data_x": input_x}, + fetch_list=[out1], + ) expected_res = np.log2(input_x) np.testing.assert_allclose(res1, expected_res, rtol=1e-05) @@ -2617,13 +2602,11 @@ def test_api(self): class TestLog2_ZeroDim(TestLog2): - def init_shape(self): self.shape = [] class TestLog10(TestActivation): - def setUp(self): self.op_type = "log10" self.check_eager = True @@ -2644,13 +2627,11 @@ def test_check_grad(self): class TestLog10_ZeroDim(TestLog10): - def init_shape(self): self.shape = [] class TestLog10API(unittest.TestCase): - def test_error(self): in1 = paddle.static.data(name="in1", shape=[11, 17], dtype="int32") in2 = paddle.static.data(name="in2", shape=[11, 17], dtype="int64") @@ -2659,19 +2640,22 @@ def test_error(self): self.assertRaises(TypeError, paddle.log10, in2) def test_api(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): input_x = np.random.uniform(0.1, 1, [11, 17]).astype("float64") - data_x = paddle.static.data(name="data_x", - shape=[11, 17], - dtype="float64") + data_x = paddle.static.data( + name="data_x", shape=[11, 17], dtype="float64" + ) out1 = paddle.log10(data_x) exe = paddle.static.Executor(place=paddle.CPUPlace()) exe.run(paddle.static.default_startup_program()) - res1, = exe.run(paddle.static.default_main_program(), - feed={"data_x": input_x}, - fetch_list=[out1]) + (res1,) = exe.run( + paddle.static.default_main_program(), + feed={"data_x": input_x}, + fetch_list=[out1], + ) expected_res = np.log10(input_x) np.testing.assert_allclose(res1, expected_res, rtol=1e-05) @@ -2686,7 +2670,6 @@ def test_api(self): class TestLog1p(TestActivation): - def setUp(self): self.op_type = "log1p" self.check_eager = True @@ -2708,27 +2691,29 @@ def test_check_grad(self): class TestLog1p_ZeroDim(TestLog1p): - def init_shape(self): self.shape = [] class TestLog1pAPI(unittest.TestCase): - def test_api(self): with fluid.program_guard(fluid.Program(), fluid.Program()): input_x = np.random.uniform(0.1, 1, [11, 17]).astype("float64") - data_x = fluid.layers.data(name="data_x", - shape=[11, 17], - append_batch_size=False, - dtype="float64") + data_x = fluid.layers.data( + name="data_x", + shape=[11, 17], + append_batch_size=False, + dtype="float64", + ) out1 = paddle.log1p(data_x) exe = fluid.Executor(place=fluid.CPUPlace()) exe.run(fluid.default_startup_program()) - res1, = exe.run(fluid.default_main_program(), - feed={"data_x": input_x}, - fetch_list=[out1]) + (res1,) = exe.run( + fluid.default_main_program(), + feed={"data_x": input_x}, + fetch_list=[out1], + ) expected_res = np.log1p(input_x) np.testing.assert_allclose(res1, expected_res, rtol=1e-05) @@ -2743,7 +2728,6 @@ def test_api(self): class TestSquare(TestActivation): - def setUp(self): self.op_type = "square" self.python_api = paddle.square @@ -2760,25 +2744,23 @@ def setUp(self): def test_check_grad(self): if self.dtype == np.float16: return - self.check_grad(['X'], - 'Out', - max_relative_error=0.007, - check_eager=True) + self.check_grad( + ['X'], 'Out', max_relative_error=0.007, check_eager=True + ) def test_check_output(self): self.check_output(check_eager=True) class TestSquare_ZeroDim(TestSquare): - def init_shape(self): self.shape = [] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSquareBF16(OpTest): - def setUp(self): self.op_type = "square" self.python_api = paddle.square @@ -2802,14 +2784,12 @@ def test_check_output(self): def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - numeric_grad_delta=0.5, - check_eager=True) + self.check_grad_with_place( + place, ['X'], 'Out', numeric_grad_delta=0.5, check_eager=True + ) class TestPow(TestActivation): - def setUp(self): self.op_type = "pow" self.python_api = paddle.pow @@ -2835,13 +2815,11 @@ def test_check_grad(self): class TestPow_ZeroDim(TestPow): - def init_shape(self): self.shape = [] class TestPow_factor_tensor(TestActivation): - def setUp(self): self.op_type = "pow" self.check_eager = False @@ -2854,7 +2832,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'FactorTensor': np.array([3.0]).astype("float32") + 'FactorTensor': np.array([3.0]).astype("float32"), } self.attrs = {} @@ -2870,14 +2848,12 @@ def test_check_grad(self): def test_api(self): input = np.random.uniform(1, 2, [11, 17]).astype("float32") - x = fluid.layers.data(name="x", - shape=[11, 17], - append_batch_size=False, - dtype="float32") - res = fluid.layers.data(name="res", - shape=[11, 17], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name="x", shape=[11, 17], append_batch_size=False, dtype="float32" + ) + res = fluid.layers.data( + name="res", shape=[11, 17], append_batch_size=False, dtype="float32" + ) factor_1 = 2.0 factor_2 = fluid.layers.fill_constant([1], "float32", 3.0) @@ -2891,29 +2867,26 @@ def test_api(self): res_1, res_2, res, res_6 = exe.run( fluid.default_main_program(), feed={"x": input}, - fetch_list=[out_1, out_2, res, out_6]) + fetch_list=[out_1, out_2, res, out_6], + ) assert np.allclose(res_1, np.power(input, 2)) assert np.allclose(res_2, np.power(input, 3)) assert np.allclose(res_6, np.power(input, 3)) def test_error(self): - in1 = fluid.layers.data(name="in1", - shape=[11, 17], - append_batch_size=False, - dtype="int32") - in2 = fluid.layers.data(name="in2", - shape=[11, 17], - append_batch_size=False, - dtype="int64") - in3 = fluid.layers.data(name="in3", - shape=[11, 17], - append_batch_size=False, - dtype="float32") - in4 = fluid.layers.data(name="in4", - shape=[11, 17], - append_batch_size=False, - dtype="float64") + in1 = fluid.layers.data( + name="in1", shape=[11, 17], append_batch_size=False, dtype="int32" + ) + in2 = fluid.layers.data( + name="in2", shape=[11, 17], append_batch_size=False, dtype="int64" + ) + in3 = fluid.layers.data( + name="in3", shape=[11, 17], append_batch_size=False, dtype="float32" + ) + in4 = fluid.layers.data( + name="in4", shape=[11, 17], append_batch_size=False, dtype="float64" + ) factor_1 = fluid.layers.fill_constant([1], "float64", 3.0) @@ -2929,7 +2902,6 @@ def ref_stanh(x, scale_a=0.67, scale_b=1.7159): class TestSTanh(TestActivation): - def get_scale_a(self): return 0.67 @@ -2960,19 +2932,16 @@ def test_check_grad(self): class TestSTanhScaleA(TestSTanh): - def get_scale_a(self): return 2.0 class TestSTanhScaleB(TestSTanh): - def get_scale_b(self): return 0.5 class TestSTanh_ZeroDim(TestSTanh): - def init_shape(self): self.shape = [] @@ -2990,8 +2959,11 @@ def setUp(self): self.x_np = np.random.uniform(-1, 1, [10, 12]).astype('float32') self.scale_a = self.get_scale_a() self.scale_b = self.get_scale_b() - self.place=paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -3029,38 +3001,37 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, paddle.stanh, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, paddle.stanh, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) paddle.stanh(x_fp16) class TestSTanhAPIScaleA(TestSTanhAPI): - def get_scale_a(self): return 2.0 class TestSTanhAPIScaleB(TestSTanhAPI): - def get_scale_b(self): return 0.5 def ref_softplus(x, beta=1, threshold=20): x_beta = beta * x - out = np.select([x_beta <= threshold, x_beta > threshold], - [np.log(1 + np.exp(x_beta)) / beta, x]) + out = np.select( + [x_beta <= threshold, x_beta > threshold], + [np.log(1 + np.exp(x_beta)) / beta, x], + ) return out class TestSoftplus(TestActivation): - def setUp(self): self.op_type = "softplus" self.python_api = paddle.nn.functional.softplus @@ -3091,15 +3062,14 @@ def test_check_grad(self): class TestSoftplus_ZeroDim(TestSoftplus): - def init_shape(self): self.shape = [] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftplusBF16(OpTest): - def setUp(self): self.op_type = "softplus" self.init_dtype() @@ -3133,8 +3103,11 @@ def setUp(self): self.threshold = 15 np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -3176,14 +3149,14 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.softplus, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.softplus, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.softplus(x_fp16) @@ -3193,7 +3166,6 @@ def ref_softsign(x): class TestSoftsign(TestActivation): - def setUp(self): self.op_type = "softsign" self.init_dtype() @@ -3217,7 +3189,6 @@ def test_check_grad(self): class TestSoftsign_ZeroDim(TestSoftsign): - def init_shape(self): self.shape = [] @@ -3227,8 +3198,11 @@ class TestSoftsignAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -3270,14 +3244,14 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.softsign, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.softsign, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.softsign(x_fp16) @@ -3287,7 +3261,6 @@ def ref_thresholded_relu(x, threshold=1.0): class TestThresholdedRelu(TestActivation): - def setUp(self): self.op_type = "thresholded_relu" self.init_dtype() @@ -3313,7 +3286,6 @@ def test_check_grad(self): class TestThresholdedRelu_ZeroDim(TestThresholdedRelu): - def init_shape(self): self.shape = [] @@ -3325,8 +3297,11 @@ def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-20, 20, [10, 12]).astype(np.float64) self.x_np[np.abs(self.x_np) < 0.005] = 0.02 - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -3368,23 +3343,22 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.thresholded_relu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.thresholded_relu, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.thresholded_relu(x_fp16) def ref_hardsigmoid(x, slope=0.166666666666667, offset=0.5): - return np.maximum(np.minimum(x * slope + offset, 1.), 0.).astype(x.dtype) + return np.maximum(np.minimum(x * slope + offset, 1.0), 0.0).astype(x.dtype) class TestHardSigmoid(TestActivation): - def setUp(self): self.op_type = "hard_sigmoid" self.dtype = 'float64' @@ -3395,7 +3369,7 @@ def setUp(self): x = np.random.uniform(-5, 5, self.shape).astype(self.dtype) lower_threshold = -self.offset / self.slope - upper_threshold = (1. - self.offset) / self.slope + upper_threshold = (1.0 - self.offset) / self.slope # Same reason as TestAbs delta = 0.005 @@ -3416,20 +3390,17 @@ def set_attrs(self): class TestHardSigmoidFP32(TestHardSigmoid): - def set_attrs(self): self.dtype = 'float32' class TestHardSigmoidSlopeOffset(TestHardSigmoid): - def set_attrs(self): self.slope = 0.2 self.offset = 0.4 class TestHardSigmoid_ZeroDim(TestHardSigmoid): - def init_shape(self): self.shape = [] @@ -3438,8 +3409,11 @@ class TestHardsigmoidAPI(unittest.TestCase): # test paddle.nn.Hardsigmoid, paddle.nn.functional.hardsigmoid def setUp(self): self.x_np = np.random.uniform(-1, 1, [10, 12]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): with paddle.static.program_guard(paddle.static.Program()): @@ -3484,14 +3458,14 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.hardsigmoid, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.hardsigmoid, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.hardsigmoid(x_fp16) @@ -3501,7 +3475,6 @@ def ref_swish(x): class TestSwish(TestActivation): - def setUp(self): self.op_type = "swish" self.python_api = paddle.nn.functional.swish @@ -3530,7 +3503,6 @@ def test_check_grad(self): class TestSwish_ZeroDim(TestSwish): - def init_shape(self): self.shape = [] @@ -3540,8 +3512,11 @@ class TestSwishAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -3588,25 +3563,25 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.swish, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.swish, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.swish(x_fp16) -def ref_mish(x, threshold=20.): - softplus = np.select([x <= threshold, x > threshold], - [np.log(1 + np.exp(x)), x]) +def ref_mish(x, threshold=20.0): + softplus = np.select( + [x <= threshold, x > threshold], [np.log(1 + np.exp(x)), x] + ) return x * np.tanh(softplus) class TestMish(TestActivation): - def setUp(self): self.op_type = "mish" self.python_api = paddle.fluid.layers.nn.mish @@ -3632,7 +3607,6 @@ def test_check_grad(self): class TestMish_ZeroDim(TestMish): - def init_shape(self): self.shape = [] @@ -3642,8 +3616,11 @@ class TestMishAPI(unittest.TestCase): def setUp(self): np.random.seed(1024) self.x_np = np.random.uniform(-1, 1, [10, 12]).astype(np.float64) - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -3685,32 +3662,30 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.mish, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.mish, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.mish(x_fp16) -#------------------ Test Error Activation---------------------- +# ------------------ Test Error Activation---------------------- def create_test_error_class(op_type): - class TestOpErrors(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): op = getattr(fluid.layers, op_type) # The input dtype of op_type must be float32, float64. - in1 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") - in2 = fluid.layers.data(name='input3', - shape=[12, 10], - dtype="int64") + in1 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) + in2 = fluid.layers.data( + name='input3', shape=[12, 10], dtype="int64" + ) self.assertRaises(TypeError, op, in1) self.assertRaises(TypeError, op, in2) @@ -3737,13 +3712,12 @@ def test_errors(self): create_test_error_class('atanh') -#------------------ Test Cudnn Activation---------------------- +# ------------------ Test Cudnn Activation---------------------- def create_test_act_cudnn_class(parent, atol=1e-3, grad_atol=1e-3): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestActCudnn(parent): - def init_kernel_type(self): self.attrs = {"use_cudnn": True} @@ -3758,16 +3732,14 @@ def init_kernel_type(self): create_test_act_cudnn_class(TestTanh) -#------------------ Test Fp16 ---------------------- -def create_test_act_fp16_class(parent, - atol=1e-3, - grad_check=True, - grad_atol=0.80): - - @unittest.skipIf(not paddle.is_compiled_with_cuda(), - "core is not compiled with CUDA") +# ------------------ Test Fp16 ---------------------- +def create_test_act_fp16_class( + parent, atol=1e-3, grad_check=True, grad_atol=0.80 +): + @unittest.skipIf( + not paddle.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestActFp16(parent): - def init_dtype(self): self.dtype = np.float16 @@ -3781,9 +3753,9 @@ def test_check_grad(self): place = core.CUDAPlace(0) support_fp16 = core.is_float16_supported(place) if support_fp16 and grad_check: - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=grad_atol) + self.check_grad_with_place( + place, ['X'], 'Out', max_relative_error=grad_atol + ) cls_name = "{0}_{1}".format(parent.__name__, "fp16") TestActFp16.__name__ = cls_name @@ -3843,15 +3815,13 @@ def test_check_grad(self): create_test_act_fp16_class(TestMish, grad_atol=0.9) -def create_test_act_bf16_class(parent, - atol=1e-2, - grad_check=True, - grad_atol=0.80): - - @unittest.skipIf(not paddle.is_compiled_with_cuda(), - "core is not compiled with CUDA") +def create_test_act_bf16_class( + parent, atol=1e-2, grad_check=True, grad_atol=0.80 +): + @unittest.skipIf( + not paddle.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestActBF16(parent): - def init_dtype(self): self.dtype = np.uint16 @@ -3861,9 +3831,9 @@ def test_check_output(self): def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=grad_atol) + self.check_grad_with_place( + place, ['X'], 'Out', max_relative_error=grad_atol + ) cls_name = "{0}_{1}".format(parent.__name__, "bf16") TestActBF16.__name__ = cls_name diff --git a/python/paddle/fluid/tests/unittests/test_activation_sparse_op.py b/python/paddle/fluid/tests/unittests/test_activation_sparse_op.py index ceab2d48ecf74..472dc334d6652 100644 --- a/python/paddle/fluid/tests/unittests/test_activation_sparse_op.py +++ b/python/paddle/fluid/tests/unittests/test_activation_sparse_op.py @@ -20,7 +20,6 @@ class TestSparseSquareOp(unittest.TestCase): - def check_with_place(self, place): scope = core.Scope() @@ -58,7 +57,6 @@ def test_sparse_acti(self): class TestSparseSqrtOp(unittest.TestCase): - def check_with_place(self, place): scope = core.Scope() diff --git a/python/paddle/fluid/tests/unittests/test_adadelta_op.py b/python/paddle/fluid/tests/unittests/test_adadelta_op.py index 3e246517ccba0..12122c8e05f37 100644 --- a/python/paddle/fluid/tests/unittests/test_adadelta_op.py +++ b/python/paddle/fluid/tests/unittests/test_adadelta_op.py @@ -20,7 +20,6 @@ class TestAdadeltaOp1(OpTest): - def setUp(self): self.op_type = "adadelta" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -37,27 +36,33 @@ def setUp(self): 'Param': param, 'Grad': grad, 'AvgSquaredGrad': avg_squared_grad, - 'AvgSquaredUpdate': avg_squared_update + 'AvgSquaredUpdate': avg_squared_update, } self.attrs = {'rho': rho, 'epsilon': epsilon} - avg_squared_grad_out = rho * avg_squared_grad + \ - (1 - rho) * np.square(grad) + avg_squared_grad_out = rho * avg_squared_grad + (1 - rho) * np.square( + grad + ) update = -np.multiply( np.sqrt( - np.divide(avg_squared_update + epsilon, - avg_squared_grad_out + epsilon)), grad) + np.divide( + avg_squared_update + epsilon, avg_squared_grad_out + epsilon + ) + ), + grad, + ) - avg_squared_update_out = rho * avg_squared_update + \ - (1 - rho) * np.square(update) + avg_squared_update_out = rho * avg_squared_update + ( + 1 - rho + ) * np.square(update) param_out = param + update self.outputs = { 'ParamOut': param_out, 'AvgSquaredGradOut': avg_squared_grad_out, - 'AvgSquaredUpdateOut': avg_squared_update_out + 'AvgSquaredUpdateOut': avg_squared_update_out, } def test_check_output(self): @@ -65,8 +70,7 @@ def test_check_output(self): class TestAdadeltaOp2(OpTest): - '''Test Adadelta op with default attribute values - ''' + '''Test Adadelta op with default attribute values''' def setUp(self): self.op_type = "adadelta" @@ -84,25 +88,31 @@ def setUp(self): 'Param': param, 'Grad': grad, 'AvgSquaredGrad': avg_squared_grad, - 'AvgSquaredUpdate': avg_squared_update + 'AvgSquaredUpdate': avg_squared_update, } - avg_squared_grad_out = rho * avg_squared_grad + \ - (1 - rho) * np.square(grad) + avg_squared_grad_out = rho * avg_squared_grad + (1 - rho) * np.square( + grad + ) update = -np.multiply( np.sqrt( - np.divide(avg_squared_update + epsilon, - avg_squared_grad_out + epsilon)), grad) + np.divide( + avg_squared_update + epsilon, avg_squared_grad_out + epsilon + ) + ), + grad, + ) - avg_squared_update_out = rho * avg_squared_update + \ - (1 - rho) * np.square(update) + avg_squared_update_out = rho * avg_squared_update + ( + 1 - rho + ) * np.square(update) param_out = param + update self.outputs = { 'ParamOut': param_out, 'AvgSquaredGradOut': avg_squared_grad_out, - 'AvgSquaredUpdateOut': avg_squared_update_out + 'AvgSquaredUpdateOut': avg_squared_update_out, } def test_check_output(self): @@ -110,16 +120,17 @@ def test_check_output(self): class TestAdadeltaV2(unittest.TestCase): - def test_adadelta_dygraph(self): paddle.disable_static(paddle.CPUPlace()) value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Adadelta(learning_rate=0.01, - parameters=linear.parameters(), - weight_decay=0.01) + adam = paddle.optimizer.Adadelta( + learning_rate=0.01, + parameters=linear.parameters(), + weight_decay=0.01, + ) out = linear(a) out.backward() adam.step() @@ -140,8 +151,9 @@ def test_adadelta(self): rms_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -150,18 +162,18 @@ def test_adadelta(self): def test_raise_error(self): self.assertRaises(ValueError, paddle.optimizer.Adadelta, None) - self.assertRaises(ValueError, - paddle.optimizer.Adadelta, - learning_rate=0.1, - rho=None) - self.assertRaises(ValueError, - paddle.optimizer.Adadelta, - learning_rate=0.1, - epsilon=None) + self.assertRaises( + ValueError, paddle.optimizer.Adadelta, learning_rate=0.1, rho=None + ) + self.assertRaises( + ValueError, + paddle.optimizer.Adadelta, + learning_rate=0.1, + epsilon=None, + ) class TestAdadeltaV2Group(TestAdadeltaV2): - def test_adadelta_dygraph(self): paddle.disable_static(paddle.CPUPlace()) value = np.arange(26).reshape(2, 13).astype("float32") @@ -169,17 +181,17 @@ def test_adadelta_dygraph(self): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 5) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Adadelta(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001, - }], - weight_decay=0.1) + adam = paddle.optimizer.Adadelta( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + }, + ], + weight_decay=0.1, + ) out = linear_1(a) out = linear_2(out) out.backward() diff --git a/python/paddle/fluid/tests/unittests/test_adagrad_op.py b/python/paddle/fluid/tests/unittests/test_adagrad_op.py index fcac8599d0c22..036322ee62d61 100644 --- a/python/paddle/fluid/tests/unittests/test_adagrad_op.py +++ b/python/paddle/fluid/tests/unittests/test_adagrad_op.py @@ -22,8 +22,7 @@ class TestAdagradOp1(OpTest): - ''' Test Adagrad operator with explicit attributes - ''' + '''Test Adagrad operator with explicit attributes''' def setUp(self): self.op_type = "adagrad" @@ -37,7 +36,7 @@ def setUp(self): 'Param': param, 'Grad': grad, 'Moment': moment, - 'LearningRate': np.array([lr]).astype("float32") + 'LearningRate': np.array([lr]).astype("float32"), } self.attrs = {'epsilon': epsilon} @@ -52,8 +51,7 @@ def test_check_output(self): class TestAdagradOp2(OpTest): - ''' Test Adagrad operator with default attributes - ''' + '''Test Adagrad operator with default attributes''' def setUp(self): self.op_type = "adagrad" @@ -68,7 +66,7 @@ def setUp(self): 'Param': param, 'Grad': grad, 'Moment': moment, - 'LearningRate': np.array([lr]).astype("float32") + 'LearningRate': np.array([lr]).astype("float32"), } self.attrs = {'epsilon': epsilon} @@ -83,7 +81,6 @@ def test_check_output(self): class TestSparseAdagradOp(unittest.TestCase): - def check_with_place(self, place): scope = core.Scope() @@ -118,14 +115,16 @@ def check_with_place(self, place): moment.set(moment_np_array, place) # create and run sgd operator - adagrad_op = Operator("adagrad", - Param='Param', - Grad='Grad', - ParamOut='Param', - Moment='Moment', - MomentOut='Moment', - LearningRate='LearningRate', - epsilon=2.0) + adagrad_op = Operator( + "adagrad", + Param='Param', + Grad='Grad', + ParamOut='Param', + Moment='Moment', + MomentOut='Moment', + LearningRate='LearningRate', + epsilon=2.0, + ) adagrad_op.run(scope, place) @@ -149,31 +148,35 @@ def check_with_place(self, place): def get_out(param, lr, grad, m, epsilon): return param - lr * grad / (math.sqrt(m) + epsilon) - self.assertAlmostEqual(get_out(5.0, 2.0, 2.0, 6.0, 2.0), - result_array[rows[0], 0], - places=5) - self.assertAlmostEqual(get_out(5.0, 2.0, 1.0, 3.0, 2.0), - result_array[rows[0], 2], - places=5) - self.assertAlmostEqual(get_out(5.0, 2.0, 0.0, 2.0, 2.0), - result_array[1, 0], - places=5) + self.assertAlmostEqual( + get_out(5.0, 2.0, 2.0, 6.0, 2.0), result_array[rows[0], 0], places=5 + ) + self.assertAlmostEqual( + get_out(5.0, 2.0, 1.0, 3.0, 2.0), result_array[rows[0], 2], places=5 + ) + self.assertAlmostEqual( + get_out(5.0, 2.0, 0.0, 2.0, 2.0), result_array[1, 0], places=5 + ) # grad_merge = 1.0 + 1.0 # m = 6.0 - self.assertAlmostEqual(get_out(5.0, 2.0, 2.0, 6.0, 2.0), - result_array[rows[1], 10], - places=5) - - self.assertAlmostEqual(get_out(5.0, 2.0, 0.0, 2.0, 2.0), - result_array[5, 8], - places=5) - self.assertAlmostEqual(get_out(5.0, 2.0, 1.0, 3.0, 2.0), - result_array[rows[2], 1], - places=5) - self.assertAlmostEqual(get_out(5.0, 2.0, 4.0, 18.0, 2.0), - result_array[rows[2], 8], - places=5) + self.assertAlmostEqual( + get_out(5.0, 2.0, 2.0, 6.0, 2.0), + result_array[rows[1], 10], + places=5, + ) + + self.assertAlmostEqual( + get_out(5.0, 2.0, 0.0, 2.0, 2.0), result_array[5, 8], places=5 + ) + self.assertAlmostEqual( + get_out(5.0, 2.0, 1.0, 3.0, 2.0), result_array[rows[2], 1], places=5 + ) + self.assertAlmostEqual( + get_out(5.0, 2.0, 4.0, 18.0, 2.0), + result_array[rows[2], 8], + places=5, + ) def test_sparse_adagrad(self): places = [core.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_adagrad_op_v2.py b/python/paddle/fluid/tests/unittests/test_adagrad_op_v2.py index e483dc1fa8949..c3eee954b610b 100644 --- a/python/paddle/fluid/tests/unittests/test_adagrad_op_v2.py +++ b/python/paddle/fluid/tests/unittests/test_adagrad_op_v2.py @@ -17,22 +17,21 @@ class TestAdagradOpV2(unittest.TestCase): - def test_v20_coverage(self): paddle.disable_static() inp = paddle.rand(shape=[10, 10]) linear = paddle.nn.Linear(10, 10) out = linear(inp) loss = paddle.mean(out) - adagrad = paddle.optimizer.Adagrad(learning_rate=0.1, - parameters=linear.parameters()) + adagrad = paddle.optimizer.Adagrad( + learning_rate=0.1, parameters=linear.parameters() + ) out.backward() adagrad.step() adagrad.clear_grad() class TestAdagradOpV2Group(TestAdagradOpV2): - def test_v20_coverage(self): paddle.disable_static() inp = paddle.rand(shape=[10, 10]) @@ -41,17 +40,17 @@ def test_v20_coverage(self): out = linear_1(inp) out = linear_2(out) loss = paddle.mean(out) - adagrad = paddle.optimizer.Adagrad(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001, - }], - weight_decay=0.1) + adagrad = paddle.optimizer.Adagrad( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + }, + ], + weight_decay=0.1, + ) out.backward() adagrad.step() adagrad.clear_grad() diff --git a/python/paddle/fluid/tests/unittests/test_adam_op.py b/python/paddle/fluid/tests/unittests/test_adam_op.py index 76309ab15eb97..bfe1429342f29 100644 --- a/python/paddle/fluid/tests/unittests/test_adam_op.py +++ b/python/paddle/fluid/tests/unittests/test_adam_op.py @@ -23,10 +23,8 @@ class TestAdamOp1(OpTest): - def setUp(self): - '''Test Adam Op with supplied attributes - ''' + '''Test Adam Op with supplied attributes''' self.op_type = "adam" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -48,20 +46,19 @@ def setUp(self): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } self.attrs = {'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def test_check_output(self): @@ -69,13 +66,11 @@ def test_check_output(self): class TestAdamOp2(OpTest): - def set_shape(self): self.shape = (102, 105) def setUp(self): - '''Test Adam Op with supplied attributes - ''' + '''Test Adam Op with supplied attributes''' self.op_type = "adam" self.set_shape() param = np.random.uniform(-1, 1, self.shape).astype("float32") @@ -98,20 +93,19 @@ def setUp(self): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } attributes = {'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, attributes) + param_out, moment1_out, moment2_out = adam_step(self.inputs, attributes) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def test_check_output(self): @@ -119,16 +113,13 @@ def test_check_output(self): class TestAdamOnlyTailOp(TestAdamOp2): - def set_shape(self): - self.shape = (3) + self.shape = 3 class TestAdamOpMultipleSteps(OpTest): - def setUp(self): - '''Test Adam Operator with supplied attributes - ''' + '''Test Adam Operator with supplied attributes''' self.op_type = "adam" self.num_steps = 10 @@ -152,19 +143,20 @@ def setUp(self): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([self.beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([self.beta2_pow]).astype("float32") + 'Beta2Pow': np.array([self.beta2_pow]).astype("float32"), } self.attrs = { 'epsilon': epsilon, 'beta1': self.beta1, - 'beta2': self.beta2 + 'beta2': self.beta2, } def test_check_output(self): for _ in range(self.num_steps): - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step( + self.inputs, self.attrs + ) beta1_pow_out = self.inputs['Beta1Pow'] * self.beta1 beta2_pow_out = self.inputs['Beta2Pow'] * self.beta2 @@ -173,7 +165,7 @@ def test_check_output(self): 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out + 'Beta2PowOut': beta2_pow_out, } # Verify output for this step @@ -189,8 +181,9 @@ def test_check_output(self): self.inputs['Beta2Pow'] = beta2_pow_out # Randomize gradient for next step - self.inputs['Grad'] = np.random.uniform( - -1, 1, (102, 105)).astype("float32") + self.inputs['Grad'] = np.random.uniform(-1, 1, (102, 105)).astype( + "float32" + ) def test_api_eager_dygraph(self): with _test_eager_guard(): @@ -270,8 +263,9 @@ def adamw_step(inputs, attributes): return param_out, moment1_out, moment2_out -def adam_step_sparse(inputs, attributes, height, rows, row_numel, np_grad, - lazy_mode): +def adam_step_sparse( + inputs, attributes, height, rows, row_numel, np_grad, lazy_mode +): ''' Simulate one step of the adam optimizer :param inputs: dict of inputs @@ -296,13 +290,16 @@ def adam_step_sparse(inputs, attributes, height, rows, row_numel, np_grad, param_out = np.zeros(shape=[height, row_numel]) def update_row(row_id, update_value): - moment1_out[row_id] = beta1 * moment1[row_id] + (1 - - beta1) * update_value - moment2_out[row_id] = beta2 * moment2[row_id] + ( - 1 - beta2) * np.square(update_value) + moment1_out[row_id] = ( + beta1 * moment1[row_id] + (1 - beta1) * update_value + ) + moment2_out[row_id] = beta2 * moment2[row_id] + (1 - beta2) * np.square( + update_value + ) lr_t = lr * np.sqrt(1 - beta2_pow) / (1 - beta1_pow) param_out[row_id] = param[row_id] - lr_t * ( - moment1_out[row_id] / (np.sqrt(moment2_out[row_id]) + epsilon)) + moment1_out[row_id] / (np.sqrt(moment2_out[row_id]) + epsilon) + ) if lazy_mode: for idx, row_id in enumerate(rows): @@ -318,7 +315,6 @@ def update_row(row_id, update_value): class TestSparseAdamOp(unittest.TestCase): - def setup(self, scope, place, lazy_mode): beta1 = 0.78 beta2 = 0.836 @@ -337,14 +333,14 @@ def setup(self, scope, place, lazy_mode): "Moment2": np.full((height, row_numel), 5.0).astype("float32"), 'Beta1Pow': beta1_pow, 'Beta2Pow': beta2_pow, - "LearningRate": np.full((1), 2.0).astype("float32") + "LearningRate": np.full((1), 2.0).astype("float32"), } self.init_output = np.full((height, row_numel), 0.0).astype("float32") self.attrs = { 'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2, - 'min_row_size_to_use_multithread': 2 + 'min_row_size_to_use_multithread': 2, } grad_selected_rows = scope.var('Grad').get_selected_rows() @@ -359,15 +355,21 @@ def setup(self, scope, place, lazy_mode): self.sparse_inputs = ["Grad"] - param_out, mom1, mom2 = adam_step_sparse(self.dense_inputs, self.attrs, - height, rows, row_numel, - np_array, lazy_mode) + param_out, mom1, mom2 = adam_step_sparse( + self.dense_inputs, + self.attrs, + height, + rows, + row_numel, + np_array, + lazy_mode, + ) self.outputs = { "ParamOut": param_out, "Moment1Out": mom1, "Moment2Out": mom2, 'Beta1PowOut': beta1_pow * beta1, - 'Beta2PowOut': beta2_pow * beta2 + 'Beta2PowOut': beta2_pow * beta2, } def check_with_place(self, place, lazy_mode): @@ -412,10 +414,8 @@ def test_sparse_adam(self): class TestAdamOpBetaVariable(OpTest): - def setUp(self): - '''Test Adam Op with beta as Variable - ''' + '''Test Adam Op with beta as Variable''' self.op_type = "adam" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -444,15 +444,14 @@ def setUp(self): attributes = {'epsilon': epsilon} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, attributes) + param_out, moment1_out, moment2_out = adam_step(self.inputs, attributes) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def test_check_output(self): @@ -460,10 +459,8 @@ def test_check_output(self): class TestAdamOpBetaEpsilonVariable(OpTest): - def setUp(self): - '''Test Adam Op with beta/epsilon as Variable - ''' + '''Test Adam Op with beta/epsilon as Variable''' self.op_type = "adam" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -493,15 +490,14 @@ def setUp(self): attributes = {'epsilon': epsilon} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, attributes) + param_out, moment1_out, moment2_out = adam_step(self.inputs, attributes) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def test_check_output(self): @@ -509,10 +505,8 @@ def test_check_output(self): class TestAdamOpWithGlobalBetaPow(OpTest): - def setUp(self): - '''Test Adam Op with global_beta_pow - ''' + '''Test Adam Op with global_beta_pow''' self.op_type = "adam" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -542,8 +536,7 @@ def setUp(self): attributes = {'epsilon': epsilon} - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, attributes) + param_out, moment1_out, moment2_out = adam_step(self.inputs, attributes) self.attrs = {'use_global_beta_pow': True} @@ -553,7 +546,7 @@ def setUp(self): 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([]), - 'Beta2PowOut': np.array([]) + 'Beta2PowOut': np.array([]), } def test_check_output(self): @@ -561,10 +554,8 @@ def test_check_output(self): class TestAdamOpWithSkipUpdate(OpTest): - def setUp(self): - '''Test Adam Op with global_beta_pow - ''' + '''Test Adam Op with global_beta_pow''' self.op_type = "adam" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -611,7 +602,6 @@ def test_check_output(self): class TestAdamOpV2(unittest.TestCase): - def test_adam_op(self): place = fluid.CPUPlace() shape = [2, 3, 8, 8] @@ -624,20 +614,20 @@ def test_adam_op(self): conv = fluid.layers.conv2d(data, 8, 3) loss = fluid.layers.reduce_mean(conv) - beta1 = fluid.layers.create_global_var(shape=[1], - value=0.85, - dtype='float32', - persistable=True) - beta2 = fluid.layers.create_global_var(shape=[1], - value=0.95, - dtype='float32', - persistable=True) + beta1 = fluid.layers.create_global_var( + shape=[1], value=0.85, dtype='float32', persistable=True + ) + beta2 = fluid.layers.create_global_var( + shape=[1], value=0.95, dtype='float32', persistable=True + ) betas = [beta1, beta2] - opt = paddle.optimizer.Adam(learning_rate=1e-5, - beta1=beta1, - beta2=beta2, - weight_decay=0.01, - epsilon=1e-8) + opt = paddle.optimizer.Adam( + learning_rate=1e-5, + beta1=beta1, + beta2=beta2, + weight_decay=0.01, + epsilon=1e-8, + ) opt.minimize(loss) exe.run(startup) @@ -651,8 +641,9 @@ def test_adam_op_dygraph(self): a = fluid.dygraph.to_variable(value) linear = fluid.Linear(13, 5, dtype="float32") - adam = paddle.optimizer.Adam(learning_rate=0.01, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + learning_rate=0.01, parameters=linear.parameters() + ) out = linear(a) out.backward() adam.step() @@ -668,26 +659,29 @@ def test_adam_op_with_state_dict(self): state_dict = adam.state_dict() adam.set_state_dict(state_dict) - #learning_rate is LRScheduler + # learning_rate is LRScheduler learning_rate = paddle.optimizer.lr.CosineAnnealingDecay( - learning_rate=0.1, T_max=10) + learning_rate=0.1, T_max=10 + ) adam = paddle.optimizer.Adam( learning_rate=learning_rate, weight_decay=fluid.regularizer.L2Decay(0.001), - parameters=emb.parameters()) + parameters=emb.parameters(), + ) lr = adam.get_lr() state_dict = adam.state_dict() adam.set_state_dict(state_dict) - #leanrning_rate is Tensor + # leanrning_rate is Tensor with self.assertRaises(TypeError): learning_rate = np.array([0.01]).astype("float32") learning_rate = paddle.to_tensor(learning_rate) - adam = paddle.optimizer.Adam(learning_rate=learning_rate, - parameters=emb.parameters()) + adam = paddle.optimizer.Adam( + learning_rate=learning_rate, parameters=emb.parameters() + ) params = adam.get_opti_var_name_list() - assert (params is not None) + assert params is not None paddle.enable_static() def test_adam_with_grad_clip(self): @@ -696,9 +690,9 @@ def test_adam_with_grad_clip(self): a = fluid.dygraph.to_variable(value) linear = fluid.Linear(13, 5, dtype="float32") clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0) - adam = paddle.optimizer.Adam(0.1, - parameters=linear.parameters(), - grad_clip=clip) + adam = paddle.optimizer.Adam( + 0.1, parameters=linear.parameters(), grad_clip=clip + ) out = linear(a) out.backward() adam.step() @@ -713,11 +707,11 @@ def test_adam_op_with_set_lr(self): lr = 0.01 adam.set_lr(lr) cur_lr = adam.get_lr() - assert (lr == cur_lr) + assert lr == cur_lr with self.assertRaises(TypeError): - lr_var = paddle.fluid.layers.create_global_var(shape=[1], - value=lr, - dtype='float32') + lr_var = paddle.fluid.layers.create_global_var( + shape=[1], value=lr, dtype='float32' + ) adam.set_lr(lr_var) paddle.enable_static() @@ -725,17 +719,17 @@ def test_adam_op_invalid_input(self): paddle.disable_static() linear = paddle.nn.Linear(10, 10) with self.assertRaises(ValueError): - adam = paddle.optimizer.Adam(0.1, - beta1=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + 0.1, beta1=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.Adam(0.1, - beta2=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + 0.1, beta2=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.Adam(0.1, - epsilon=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + 0.1, epsilon=-1, parameters=linear.parameters() + ) paddle.enable_static() def test_adam_op_with_sparse_input_and_weight_decay(self): @@ -744,9 +738,9 @@ def test_adam_op_with_sparse_input_and_weight_decay(self): x_data = np.arange(0, 10).reshape((10, 1)).astype(np.int64) x = paddle.to_tensor(x_data, stop_gradient=False) emb = paddle.nn.Embedding(10, 10, sparse=True) - adam = paddle.optimizer.Adam(0.001, - parameters=emb.parameters(), - weight_decay=0.01) + adam = paddle.optimizer.Adam( + 0.001, parameters=emb.parameters(), weight_decay=0.01 + ) with self.assertRaises(RuntimeError): out = emb(x) @@ -764,13 +758,14 @@ def test_api_eager_dygraph(self): class TestAdamOptimizer(unittest.TestCase): - - def _test(self, - place, - use_tensor=True, - use_fluid_api=True, - use_global_beta_pow=False, - flatten_param_grads=False): + def _test( + self, + place, + use_tensor=True, + use_fluid_api=True, + use_global_beta_pow=False, + flatten_param_grads=False, + ): paddle.enable_static() main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -784,29 +779,30 @@ def _test(self, weight_attr1 = paddle.ParamAttr( name="weight1", initializer=fluid.initializer.Constant(value=1.0), - trainable=True) + trainable=True, + ) weight_attr2 = paddle.ParamAttr( name="weight2", initializer=fluid.initializer.Constant(value=2.0), - trainable=True) + trainable=True, + ) clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) with paddle.static.program_guard(main_prog, startup_prog): with paddle.utils.unique_name.guard(): a = paddle.static.data(name="a", shape=[2, 2], dtype='float32') b = paddle.static.data(name="b", shape=[2, 2], dtype='float32') - label = paddle.static.data(name="label", - shape=[2, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[2, 1], dtype='int64' + ) sum = paddle.add(a, b) z = paddle.pow(sum, 2.0) fc_1 = fluid.layers.fc(input=z, size=2, param_attr=weight_attr1) - prediction = fluid.layers.fc(input=fc_1, - size=2, - param_attr=weight_attr2, - act='softmax') + prediction = fluid.layers.fc( + input=fc_1, size=2, param_attr=weight_attr2, act='softmax' + ) cost = fluid.layers.cross_entropy(input=prediction, label=label) loss = fluid.layers.reduce_mean(cost) @@ -819,19 +815,22 @@ def _test(self, value=float(beta1_init), dtype='float32', persistable=True, - name="beta1") + name="beta1", + ) beta2 = fluid.layers.create_global_var( shape=[1], value=float(beta2_init), dtype='float32', persistable=True, - name="beta2") + name="beta2", + ) epsilon = fluid.layers.create_global_var( shape=[1], value=float(epsilon_init), dtype='float32', persistable=True, - name="epsilon") + name="epsilon", + ) if use_fluid_api: adam = fluid.optimizer.Adam( learning_rate=0.01, @@ -841,13 +840,16 @@ def _test(self, use_global_beta_pow=use_global_beta_pow, flatten_param_grads=flatten_param_grads, align_size=256, - grad_clip=clip) + grad_clip=clip, + ) else: - adam = paddle.optimizer.Adam(learning_rate=0.01, - beta1=beta1, - beta2=beta2, - epsilon=epsilon, - grad_clip=clip) + adam = paddle.optimizer.Adam( + learning_rate=0.01, + beta1=beta1, + beta2=beta2, + epsilon=epsilon, + grad_clip=clip, + ) else: if use_fluid_api: adam = fluid.optimizer.Adam( @@ -858,13 +860,16 @@ def _test(self, use_global_beta_pow=use_global_beta_pow, flatten_param_grads=flatten_param_grads, align_size=256, - grad_clip=clip) + grad_clip=clip, + ) else: - adam = fluid.optimizer.Adam(learning_rate=0.01, - beta1=beta1_init, - beta2=beta2_init, - epsilon=epsilon_init, - grad_clip=clip) + adam = fluid.optimizer.Adam( + learning_rate=0.01, + beta1=beta1_init, + beta2=beta2_init, + epsilon=epsilon_init, + grad_clip=clip, + ) adam.minimize(loss) @@ -875,15 +880,16 @@ def _test(self, print("Start run on {}".format(place)) for epoch in range(10): - pred_res, loss_res = exe.run(main_prog, - feed={ - "a": a_np, - "b": b_np, - "label": label_np - }, - fetch_list=[prediction, loss]) - print("Epoch {} | Prediction[0]: {}, Loss: {}".format( - epoch, pred_res[0], loss_res)) + pred_res, loss_res = exe.run( + main_prog, + feed={"a": a_np, "b": b_np, "label": label_np}, + fetch_list=[prediction, loss], + ) + print( + "Epoch {} | Prediction[0]: {}, Loss: {}".format( + epoch, pred_res[0], loss_res + ) + ) paddle.disable_static() return pred_res, loss_res @@ -895,10 +901,13 @@ def _test_with_place(self, place): for use_fluid_api in [True, False]: for use_global_beta_pow in [True, False]: for flatten_param_grads in [True, False]: - pred, loss = self._test(place, use_tensor, - use_fluid_api, - use_global_beta_pow, - flatten_param_grads) + pred, loss = self._test( + place, + use_tensor, + use_fluid_api, + use_global_beta_pow, + flatten_param_grads, + ) preds.append(pred) losses.append(loss) for pred in preds: @@ -920,21 +929,22 @@ def test_adam_flatten_param_grads_with_regularizer(self): name="weight1", initializer=fluid.initializer.Constant(value=1.0), regularizer=fluid.regularizer.L1DecayRegularizer( - regularization_coeff=0.1), - trainable=True) + regularization_coeff=0.1 + ), + trainable=True, + ) with fluid.program_guard(main): x = fluid.data(name='x', shape=[None, 13], dtype='float32') y = fluid.data(name='y', shape=[None, 1], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1, - act=None, - param_attr=weight_attr) + y_predict = fluid.layers.fc( + input=x, size=1, act=None, param_attr=weight_attr + ) cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) - adam = fluid.optimizer.AdamOptimizer(0.01, - flatten_param_grads=True, - align_size=256) + adam = fluid.optimizer.AdamOptimizer( + 0.01, flatten_param_grads=True, align_size=256 + ) adam.minimize(avg_cost) paddle.disable_static() @@ -957,13 +967,16 @@ def test_adam_exception(self): adam = fluid.optimizer.Adam(use_global_beta_pow=True) adam.minimize(loss) self.assertRaises(Exception, adam._get_global_accumulator, 'tmp') - adam._add_global_accumulator('tmp', - type=core.VarDesc.VarType.LOD_TENSOR) + adam._add_global_accumulator( + 'tmp', type=core.VarDesc.VarType.LOD_TENSOR + ) adam._get_global_accumulator('tmp') - self.assertRaises(Exception, - adam._add_global_accumulator, - adam._beta1_pow_acc_str, - type=core.VarDesc.VarType.LOD_TENSOR) + self.assertRaises( + Exception, + adam._add_global_accumulator, + adam._beta1_pow_acc_str, + type=core.VarDesc.VarType.LOD_TENSOR, + ) paddle.disable_static() def test_adam_save_load(self): @@ -974,12 +987,14 @@ def test_adam_save_load(self): state_dict = linear.state_dict() fluid.save_dygraph(state_dict, "paddle_dy") - scheduler = paddle.optimizer.lr.NoamDecay(d_model=0.01, - warmup_steps=100, - verbose=True) - adam = paddle.fluid.optimizer.Adam(learning_rate=scheduler, - parameter_list=linear.parameters(), - use_global_beta_pow=True) + scheduler = paddle.optimizer.lr.NoamDecay( + d_model=0.01, warmup_steps=100, verbose=True + ) + adam = paddle.fluid.optimizer.Adam( + learning_rate=scheduler, + parameter_list=linear.parameters(), + use_global_beta_pow=True, + ) adam.minimize(b) state_dict = adam.state_dict() fluid.save_dygraph(state_dict, "paddle_dy") @@ -1000,13 +1015,14 @@ def get_opt(dtype, shape): state_dict = linear.state_dict() fluid.save_dygraph(state_dict, "paddle_dy") - scheduler = paddle.optimizer.lr.NoamDecay(d_model=0.01, - warmup_steps=100, - verbose=True) + scheduler = paddle.optimizer.lr.NoamDecay( + d_model=0.01, warmup_steps=100, verbose=True + ) adam = paddle.fluid.optimizer.Adam( learning_rate=scheduler, parameter_list=linear.parameters(), - use_global_beta_pow=True) + use_global_beta_pow=True, + ) adam.minimize(b) return adam @@ -1021,14 +1037,14 @@ def get_opt(dtype, shape): self.assertRaises(AssertionError, adam2.set_state_dict, opt_state_dict) adam3 = get_opt('float32', [10, 10]) # shape not match - opt_state_dict['beta1_pow_acc_0'] = np.array([0.9, 0.9], - dtype='float32') + opt_state_dict['beta1_pow_acc_0'] = np.array( + [0.9, 0.9], dtype='float32' + ) self.assertRaises(AssertionError, adam3.set_state_dict, opt_state_dict) paddle.enable_static() class TestAdamOpV2Group(TestAdamOpV2): - def test_adam_op(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") @@ -1036,16 +1052,19 @@ def test_adam_op(self): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Adam(learning_rate=0.01, - parameters=[{ - 'params': linear_1.parameters() - }, { - 'params': linear_2.parameters(), - 'weight_decay': 0.001, - 'beta1': 0.1, - 'beta2': 0.99 - }], - weight_decay=0.1) + adam = paddle.optimizer.Adam( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + 'beta1': 0.1, + 'beta2': 0.99, + }, + ], + weight_decay=0.1, + ) out = linear_1(a) out = linear_2(out) out.backward() @@ -1054,13 +1073,14 @@ def test_adam_op(self): class TestMultiTensorAdam(unittest.TestCase): - - def _adam_optimize_dygraph(self, - place, - use_param_attr=False, - use_param_group=False, - use_amp=False, - use_multi_tensor=False): + def _adam_optimize_dygraph( + self, + place, + use_param_attr=False, + use_param_group=False, + use_amp=False, + use_multi_tensor=False, + ): paddle.disable_static() paddle.seed(10) paddle.set_device(place) @@ -1070,29 +1090,32 @@ def _adam_optimize_dygraph(self, weight_attr = paddle.ParamAttr( learning_rate=0.5, regularizer=paddle.regularizer.L2Decay(1.0), - trainable=True) + trainable=True, + ) if use_param_attr: model = paddle.nn.Linear(5, 5, weight_attr) else: model = paddle.nn.Linear(5, 5) if not use_param_group: - optimizer = paddle.optimizer.Adam(parameters=model.parameters(), - use_multi_tensor=use_multi_tensor, - multi_precision=use_amp) + optimizer = paddle.optimizer.Adam( + parameters=model.parameters(), + use_multi_tensor=use_multi_tensor, + multi_precision=use_amp, + ) else: - optimizer = paddle.optimizer.Adam(parameters=[{ - 'params': - model.parameters(), - 'weight_decay': - 0.001, - 'beta1': - 0.1, - 'beta2': - 0.99 - }], - use_multi_tensor=use_multi_tensor, - multi_precision=use_amp) + optimizer = paddle.optimizer.Adam( + parameters=[ + { + 'params': model.parameters(), + 'weight_decay': 0.001, + 'beta1': 0.1, + 'beta2': 0.99, + } + ], + use_multi_tensor=use_multi_tensor, + multi_precision=use_amp, + ) for idx in range(2): if place == 'gpu' and use_amp == True: @@ -1116,10 +1139,9 @@ def _adam_optimize_dygraph(self, return output, model.parameters() - def _adam_optimize_static(self, - place, - use_amp=False, - use_multi_tensor=False): + def _adam_optimize_static( + self, place, use_amp=False, use_multi_tensor=False + ): paddle.enable_static() paddle.seed(10) np.random.seed(10) @@ -1128,24 +1150,26 @@ def _adam_optimize_static(self, exe = paddle.static.Executor(place=place) train_program = paddle.static.Program() startup_program = paddle.static.Program() - optimizer = paddle.optimizer.Adam(multi_precision=use_amp, - use_multi_tensor=use_multi_tensor) + optimizer = paddle.optimizer.Adam( + multi_precision=use_amp, use_multi_tensor=use_multi_tensor + ) if use_amp: optimizer = paddle.static.amp.decorate( optimizer, init_loss_scaling=128.0, use_dynamic_loss_scaling=True, use_pure_fp16=True, - use_fp16_guard=False) + use_fp16_guard=False, + ) with paddle.static.program_guard(train_program, startup_program): if use_amp: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float16') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float16' + ) else: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float32') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float32' + ) hidden = paddle.static.nn.fc(x=data, size=10) loss = paddle.mean(hidden) optimizer.minimize(loss) @@ -1157,9 +1181,9 @@ def _adam_optimize_static(self, x = np.random.random(size=(2, 2)).astype('float32') out = [] for idx in range(5): - loss_data, = exe.run(train_program, - feed={"X": x}, - fetch_list=[loss.name]) + (loss_data,) = exe.run( + train_program, feed={"X": x}, fetch_list=[loss.name] + ) out.append(loss_data) return out @@ -1172,49 +1196,59 @@ def _get_places(self): def _check_with_place_amp(self, place, use_amp): # test dygraph mode output_dygraph1, params_dygraph1 = self._adam_optimize_dygraph( - place=place, use_amp=use_amp, use_multi_tensor=True) + place=place, use_amp=use_amp, use_multi_tensor=True + ) output_dygraph2, params_dygraph2 = self._adam_optimize_dygraph( - place=place, use_amp=use_amp, use_multi_tensor=False) + place=place, use_amp=use_amp, use_multi_tensor=False + ) np.testing.assert_allclose(output_dygraph1, output_dygraph2, rtol=1e-05) for idx in range(len(params_dygraph1)): - np.testing.assert_allclose(params_dygraph1[idx], - params_dygraph2[idx], - rtol=1e-05) + np.testing.assert_allclose( + params_dygraph1[idx], params_dygraph2[idx], rtol=1e-05 + ) # test static mode - output_static1 = self._adam_optimize_static(place=place, - use_amp=use_amp, - use_multi_tensor=True) - output_static2 = self._adam_optimize_static(place=place, - use_amp=use_amp, - use_multi_tensor=False) + output_static1 = self._adam_optimize_static( + place=place, use_amp=use_amp, use_multi_tensor=True + ) + output_static2 = self._adam_optimize_static( + place=place, use_amp=use_amp, use_multi_tensor=False + ) for idx in range(len(output_static1)): - np.testing.assert_allclose(output_static1[idx], - output_static2[idx], - rtol=1e-05) + np.testing.assert_allclose( + output_static1[idx], output_static2[idx], rtol=1e-05 + ) def _check_with_param_arrt(self, place, use_amp): - output1, params1 = self._adam_optimize_dygraph(place=place, - use_amp=use_amp, - use_param_attr=True, - use_multi_tensor=True) - output2, params2 = self._adam_optimize_dygraph(place=place, - use_amp=use_amp, - use_param_attr=True, - use_multi_tensor=False) + output1, params1 = self._adam_optimize_dygraph( + place=place, + use_amp=use_amp, + use_param_attr=True, + use_multi_tensor=True, + ) + output2, params2 = self._adam_optimize_dygraph( + place=place, + use_amp=use_amp, + use_param_attr=True, + use_multi_tensor=False, + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): np.testing.assert_allclose(params1[idx], params2[idx], rtol=1e-05) def _check_with_param_group(self, place, use_amp): - output1, params1 = self._adam_optimize_dygraph(place=place, - use_amp=use_amp, - use_param_group=True, - use_multi_tensor=True) - output2, params2 = self._adam_optimize_dygraph(place=place, - use_amp=use_amp, - use_param_group=True, - use_multi_tensor=False) + output1, params1 = self._adam_optimize_dygraph( + place=place, + use_amp=use_amp, + use_param_group=True, + use_multi_tensor=True, + ) + output2, params2 = self._adam_optimize_dygraph( + place=place, + use_amp=use_amp, + use_param_group=True, + use_multi_tensor=False, + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): diff --git a/python/paddle/fluid/tests/unittests/test_adam_optimizer_fp32_fp64.py b/python/paddle/fluid/tests/unittests/test_adam_optimizer_fp32_fp64.py index 1f08eb085a3c5..904b02e778c25 100644 --- a/python/paddle/fluid/tests/unittests/test_adam_optimizer_fp32_fp64.py +++ b/python/paddle/fluid/tests/unittests/test_adam_optimizer_fp32_fp64.py @@ -39,8 +39,9 @@ def main_test_func(place, dtype): adam_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = fluid.io.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = fluid.io.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -49,7 +50,6 @@ def main_test_func(place, dtype): class AdamFp32Test(unittest.TestCase): - def setUp(self): self.dtype = 'float32' @@ -59,7 +59,6 @@ def test_main(self): class AdamFp64Test(AdamFp32Test): - def setUp(self): self.dtype = 'float64' diff --git a/python/paddle/fluid/tests/unittests/test_adamax_api.py b/python/paddle/fluid/tests/unittests/test_adamax_api.py index 87aea099ac5fc..78e9553a806d6 100644 --- a/python/paddle/fluid/tests/unittests/test_adamax_api.py +++ b/python/paddle/fluid/tests/unittests/test_adamax_api.py @@ -20,15 +20,16 @@ class TestAdamaxAPI(unittest.TestCase): - def func_adamax_api_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) - adam = paddle.optimizer.Adamax(learning_rate=0.01, - parameters=linear.parameters(), - weight_decay=0.01) + adam = paddle.optimizer.Adamax( + learning_rate=0.01, + parameters=linear.parameters(), + weight_decay=0.01, + ) out = linear(a) out.backward() adam.step() @@ -53,11 +54,13 @@ def func_adamax_api(self): loss = paddle.mean(conv) beta1 = 0.85 beta2 = 0.95 - opt = paddle.optimizer.Adamax(learning_rate=1e-5, - beta1=beta1, - beta2=beta2, - weight_decay=0.01, - epsilon=1e-8) + opt = paddle.optimizer.Adamax( + learning_rate=1e-5, + beta1=beta1, + beta2=beta2, + weight_decay=0.01, + epsilon=1e-8, + ) opt.minimize(loss) exe.run(startup) @@ -72,7 +75,6 @@ def test_adamax_api(self): class TestAdamaxAPIGroup(TestAdamaxAPI): - def func_adamax_api_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") @@ -80,21 +82,19 @@ def func_adamax_api_dygraph(self): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Adamax(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001, - 'beta1': - 0.1, - 'beta2': - 0.99 - }], - weight_decay=0.1) + adam = paddle.optimizer.Adamax( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + 'beta1': 0.1, + 'beta2': 0.99, + }, + ], + weight_decay=0.1, + ) out = linear_1(a) out = linear_2(out) out.backward() diff --git a/python/paddle/fluid/tests/unittests/test_adamax_op.py b/python/paddle/fluid/tests/unittests/test_adamax_op.py index 6ea12544d8706..b643271443469 100644 --- a/python/paddle/fluid/tests/unittests/test_adamax_op.py +++ b/python/paddle/fluid/tests/unittests/test_adamax_op.py @@ -18,10 +18,8 @@ class TestAdamaxOp1(OpTest): - def setUp(self): - '''Test Adamax Operator with supplied attributes - ''' + '''Test Adamax Operator with supplied attributes''' self.op_type = "adamax" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -41,18 +39,19 @@ def setUp(self): 'Moment': moment, 'InfNorm': inf_norm, 'LearningRate': np.array([learning_rate]).astype("float32"), - 'Beta1Pow': np.array([beta1_pow]).astype("float32") + 'Beta1Pow': np.array([beta1_pow]).astype("float32"), } self.attrs = {'beta1': beta1, 'beta2': beta2, 'epsilon': epsilon} param_out, moment_out, inf_norm_out = adamax_step( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = { 'ParamOut': param_out, 'MomentOut': moment_out, - 'InfNormOut': inf_norm_out + 'InfNormOut': inf_norm_out, } def test_check_output(self): @@ -60,8 +59,7 @@ def test_check_output(self): class TestAdamaxOp2(OpTest): - '''Test Adamax Operator with default attributes - ''' + '''Test Adamax Operator with default attributes''' def setUp(self): self.op_type = "adamax" @@ -83,7 +81,7 @@ def setUp(self): 'Moment': moment, 'InfNorm': inf_norm, 'LearningRate': np.array([learning_rate]).astype("float32"), - 'Beta1Pow': np.array([beta1_pow]).astype("float32") + 'Beta1Pow': np.array([beta1_pow]).astype("float32"), } attrs = {'beta1': beta1, 'beta2': beta2, 'epsilon': epsilon} @@ -92,7 +90,7 @@ def setUp(self): self.outputs = { 'ParamOut': param_out, 'MomentOut': moment_out, - 'InfNormOut': inf_norm_out + 'InfNormOut': inf_norm_out, } def test_check_output(self): @@ -100,10 +98,8 @@ def test_check_output(self): class TestAdamaxOpMultipleSteps(OpTest): - def setUp(self): - '''Test Adamax Operator with supplied attributes - ''' + '''Test Adamax Operator with supplied attributes''' self.op_type = "adamax" self.num_steps = 10 @@ -125,7 +121,7 @@ def setUp(self): 'Moment': moment, 'InfNorm': inf_norm, 'LearningRate': np.array([learning_rate]).astype("float32"), - 'Beta1Pow': np.array([beta1_pow]).astype("float32") + 'Beta1Pow': np.array([beta1_pow]).astype("float32"), } self.attrs = {'beta1': beta1, 'beta2': beta2, 'epsilon': epsilon} @@ -133,12 +129,13 @@ def setUp(self): def test_check_output(self): for _ in range(self.num_steps): param_out, moment_out, inf_norm_out = adamax_step( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = { 'ParamOut': param_out, 'MomentOut': moment_out, - 'InfNormOut': inf_norm_out + 'InfNormOut': inf_norm_out, } # Verify output for this step @@ -153,8 +150,9 @@ def test_check_output(self): self.inputs['Beta1Pow'] *= self.attrs['beta1'] # Randomize gradient for next step - self.inputs['Grad'] = np.random.uniform( - -1, 1, (102, 105)).astype("float32") + self.inputs['Grad'] = np.random.uniform(-1, 1, (102, 105)).astype( + "float32" + ) def adamax_step(inputs, attributes): @@ -178,30 +176,30 @@ def adamax_step(inputs, attributes): moment_out = beta1 * moment + (1 - beta1) * grad inf_norm_out = np.maximum(beta2 * inf_norm + epsilon, np.abs(grad)) - lr_t = (lr / (1 - beta1_pow)) + lr_t = lr / (1 - beta1_pow) param_out = param - lr_t * np.divide(moment_out, inf_norm_out) return param_out, moment_out, inf_norm_out class TestAdamaxOpV2(unittest.TestCase): - def test_adamax_op_invalid_input(self): import paddle + paddle.disable_static() linear = paddle.nn.Linear(10, 10) with self.assertRaises(ValueError): - adam = paddle.optimizer.Adamax(0.1, - beta1=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.Adamax( + 0.1, beta1=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.Adamax(0.1, - beta2=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.Adamax( + 0.1, beta2=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.Adamax(0.1, - epsilon=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.Adamax( + 0.1, epsilon=-1, parameters=linear.parameters() + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_adamw_op.py b/python/paddle/fluid/tests/unittests/test_adamw_op.py index e39638d86555e..6e4a7b43f20bf 100644 --- a/python/paddle/fluid/tests/unittests/test_adamw_op.py +++ b/python/paddle/fluid/tests/unittests/test_adamw_op.py @@ -60,10 +60,8 @@ def adamw_step(inputs, attributes): class TestAdamW(OpTest): - def setUp(self): - '''Test AdamW Op with supplied attributes - ''' + '''Test AdamW Op with supplied attributes''' self.op_type = "adamw" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -85,7 +83,7 @@ def setUp(self): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } self.attrs = { @@ -93,31 +91,31 @@ def setUp(self): 'beta1': beta1, 'beta2': beta2, "coeff": 0.5, - "with_decay": True + "with_decay": True, } - param_out, moment1_out, \ - moment2_out = adamw_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adamw_step( + self.inputs, self.attrs + ) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def test_check_output(self): self.check_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestAdamW2(OpTest): - def setUp(self): - '''Test AdamW Op with supplied attributes - ''' + '''Test AdamW Op with supplied attributes''' self.op_type = "adamw" param = np.random.uniform(-1, 1, (2, 2)).astype("float32") grad = np.random.uniform(-1, 1, (2, 2)).astype("float32") @@ -139,7 +137,7 @@ def setUp(self): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } self.attrs = { @@ -148,18 +146,19 @@ def setUp(self): 'beta2': beta2, "lr_ratio": 0.1, "coeff": 0.5, - "with_decay": True + "with_decay": True, } param_out, moment1_out, moment2_out = adamw_step( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype("float32") * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype("float32") * beta2, } def test_check_output(self): @@ -167,16 +166,17 @@ def test_check_output(self): class TestAdamWOp(unittest.TestCase): - def test_adamw_op_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) - adam = paddle.optimizer.AdamW(learning_rate=0.01, - parameters=linear.parameters(), - apply_decay_param_fun=lambda name: True, - weight_decay=0.01) + adam = paddle.optimizer.AdamW( + learning_rate=0.01, + parameters=linear.parameters(), + apply_decay_param_fun=lambda name: True, + weight_decay=0.01, + ) for _ in range(2): out = linear(a) @@ -189,11 +189,13 @@ def test_adamw_op_coverage(self): value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) - adam = paddle.optimizer.AdamW(learning_rate=0.0, - parameters=linear.parameters(), - apply_decay_param_fun=lambda name: True, - weight_decay=0.01) - assert (adam.__str__() is not None) + adam = paddle.optimizer.AdamW( + learning_rate=0.0, + parameters=linear.parameters(), + apply_decay_param_fun=lambda name: True, + weight_decay=0.01, + ) + assert adam.__str__() is not None def test_adamw_op(self): paddle.enable_static() @@ -208,20 +210,20 @@ def test_adamw_op(self): conv = fluid.layers.conv2d(data, 8, 3) loss = paddle.mean(conv) - beta1 = fluid.layers.create_global_var(shape=[1], - value=0.85, - dtype='float32', - persistable=True) - beta2 = fluid.layers.create_global_var(shape=[1], - value=0.95, - dtype='float32', - persistable=True) + beta1 = fluid.layers.create_global_var( + shape=[1], value=0.85, dtype='float32', persistable=True + ) + beta2 = fluid.layers.create_global_var( + shape=[1], value=0.95, dtype='float32', persistable=True + ) betas = [beta1, beta2] - opt = paddle.optimizer.AdamW(learning_rate=1e-5, - beta1=beta1, - beta2=beta2, - weight_decay=0.01, - epsilon=1e-8) + opt = paddle.optimizer.AdamW( + learning_rate=1e-5, + beta1=beta1, + beta2=beta2, + weight_decay=0.01, + epsilon=1e-8, + ) opt.minimize(loss) exe.run(startup) @@ -234,17 +236,17 @@ def test_adamw_op_invalid_input(self): paddle.disable_static() linear = paddle.nn.Linear(10, 10) with self.assertRaises(ValueError): - adam = paddle.optimizer.AdamW(0.1, - beta1=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.AdamW( + 0.1, beta1=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.AdamW(0.1, - beta2=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.AdamW( + 0.1, beta2=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.AdamW(0.1, - epsilon=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.AdamW( + 0.1, epsilon=-1, parameters=linear.parameters() + ) def test_api_eager_dygraph(self): with _test_eager_guard(): @@ -253,25 +255,21 @@ def test_api_eager_dygraph(self): class TestAdamWOpGroup(TestAdamWOp): - def test_adamw_op_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) - adam = paddle.optimizer.AdamW(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001 - }], - apply_decay_param_fun=lambda name: True, - weight_decay=0.01) + adam = paddle.optimizer.AdamW( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + {'params': linear_2.parameters(), 'weight_decay': 0.001}, + ], + apply_decay_param_fun=lambda name: True, + weight_decay=0.01, + ) for _ in range(2): out = linear_1(a) @@ -282,7 +280,6 @@ def test_adamw_op_dygraph(self): class TestAdamWOpMultiPrecison(unittest.TestCase): - def _test_adamw_op_dygraph_place_amp(self, place, use_amp=False): paddle.disable_static() paddle.seed(10) @@ -292,17 +289,17 @@ def _test_adamw_op_dygraph_place_amp(self, place, use_amp=False): model = paddle.nn.Linear(5, 5) - optimizer = paddle.optimizer.AdamW(parameters=[{ - 'params': - model.parameters(), - 'weight_decay': - 0.001, - 'beta1': - 0.1, - 'beta2': - 0.99 - }], - multi_precision=use_amp) + optimizer = paddle.optimizer.AdamW( + parameters=[ + { + 'params': model.parameters(), + 'weight_decay': 0.001, + 'beta1': 0.1, + 'beta2': 0.99, + } + ], + multi_precision=use_amp, + ) for idx in range(2): if place == 'gpu' and use_amp == True: @@ -338,51 +335,59 @@ def test_main(self): class TestAdamWOpError(unittest.TestCase): - def test_api_errors(self): - def test_weight_decay_dtype(): linear = paddle.nn.Linear(13, 5) - adam = paddle.optimizer.AdamW(learning_rate=0.01, - parameters=linear.parameters(), - weight_decay=1) + adam = paddle.optimizer.AdamW( + learning_rate=0.01, + parameters=linear.parameters(), + weight_decay=1, + ) def test_parameters_dtype1(): - adam = paddle.optimizer.AdamW(learning_rate=0.01, - parameters=paddle.randn((5, 5)), - weight_decay=0.1) + adam = paddle.optimizer.AdamW( + learning_rate=0.01, + parameters=paddle.randn((5, 5)), + weight_decay=0.1, + ) def test_parameters_dtype2(): linear = paddle.nn.Linear(13, 5) adam = paddle.optimizer.AdamW( learning_rate=0.01, parameters={'params': linear.parameters()}, - weight_decay=0.1) + weight_decay=0.1, + ) def test_parameters_dtype3(): - adam = paddle.optimizer.AdamW(learning_rate=0.01, - parameters=None, - weight_decay=0.1) + adam = paddle.optimizer.AdamW( + learning_rate=0.01, parameters=None, weight_decay=0.1 + ) def test_parameters_dtype4(): linear = paddle.nn.Linear(13, 5) adam = paddle.optimizer.AdamW( learning_rate=0.01, parameters={'params': set(linear.parameters())}, - weight_decay=0.1) + weight_decay=0.1, + ) def test_learning_rate_dtype(): linear = paddle.nn.Linear(13, 5) - adam = paddle.optimizer.AdamW(learning_rate=1, - parameters=linear.parameters(), - weight_decay=0.1) + adam = paddle.optimizer.AdamW( + learning_rate=1, + parameters=linear.parameters(), + weight_decay=0.1, + ) def test_grad_clip_dtype(): linear = paddle.nn.Linear(13, 5) - adam = paddle.optimizer.AdamW(learning_rate=0.01, - parameters=linear.parameters(), - weight_decay=0.1, - grad_clip=0.1) + adam = paddle.optimizer.AdamW( + learning_rate=0.01, + parameters=linear.parameters(), + weight_decay=0.1, + grad_clip=0.1, + ) self.assertRaises(TypeError, test_weight_decay_dtype) self.assertRaises(TypeError, test_parameters_dtype1) @@ -394,7 +399,6 @@ def test_grad_clip_dtype(): class TestAdamWOpGroupWithLR(TestAdamWOp): - def test_adamw_op_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") @@ -403,16 +407,21 @@ def test_adamw_op_dygraph(self): linear_2 = paddle.nn.Linear(5, 3) adam = paddle.optimizer.AdamW( learning_rate=paddle.optimizer.lr.PiecewiseDecay( - boundaries=[3, 6], values=[0.1, 0.2, 0.3]), - parameters=[{ - 'params': linear_1.parameters(), - 'learning_rate': 0.1, - }, { - 'params': linear_2.parameters(), - 'weight_decay': 0.001, - }], + boundaries=[3, 6], values=[0.1, 0.2, 0.3] + ), + parameters=[ + { + 'params': linear_1.parameters(), + 'learning_rate': 0.1, + }, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + }, + ], apply_decay_param_fun=lambda name: True, - weight_decay=0.01) + weight_decay=0.01, + ) for _ in range(2): out = linear_1(a) @@ -430,13 +439,13 @@ def simple_lr_setting(param, decay_rate, n_layers): else: depth = 0 - return decay_rate**(n_layers + 2 - depth) + return decay_rate ** (n_layers + 2 - depth) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestAdamWOpLayerwiseLR(TestAdamWOp): - def setUp(self): random.seed(2022) np.random.seed(2022) @@ -445,9 +454,11 @@ def setUp(self): def test_adamw_op_dygraph(self): paddle.disable_static() linear1 = paddle.nn.Linear( - 13, 8, bias_attr=paddle.nn.initializer.Constant(value=1.0)) + 13, 8, bias_attr=paddle.nn.initializer.Constant(value=1.0) + ) linear2 = paddle.nn.Linear( - 8, 5, bias_attr=paddle.nn.initializer.Constant(value=1.0)) + 8, 5, bias_attr=paddle.nn.initializer.Constant(value=1.0) + ) # fix the linear name, simple_lr_setting function will use the name linear1.weight.name = "linear_1.w_0" @@ -475,15 +486,18 @@ def test_adamw_op_dygraph(self): beta1 = 0.9 beta2 = 0.999 - opt = paddle.optimizer.AdamW(learning_rate=learning_rate, - parameters=[{ - 'params': linear1.parameters() - }, { - 'params': linear2.parameters(), - }], - apply_decay_param_fun=lambda name: True, - weight_decay=weight_decay, - lr_ratio=simple_lr_fun) + opt = paddle.optimizer.AdamW( + learning_rate=learning_rate, + parameters=[ + {'params': linear1.parameters()}, + { + 'params': linear2.parameters(), + }, + ], + apply_decay_param_fun=lambda name: True, + weight_decay=weight_decay, + lr_ratio=simple_lr_fun, + ) def get_numpy_output(param, grad, moment1, moment2, lr_ratio, t): np_inputs = { @@ -493,7 +507,7 @@ def get_numpy_output(param, grad, moment1, moment2, lr_ratio, t): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1**t]).astype("float32"), - 'Beta2Pow': np.array([beta2**t]).astype("float32") + 'Beta2Pow': np.array([beta2**t]).astype("float32"), } np_attrs = { @@ -502,32 +516,54 @@ def get_numpy_output(param, grad, moment1, moment2, lr_ratio, t): 'beta2': beta2, "lr_ratio": lr_ratio, "coeff": weight_decay, - "with_decay": True + "with_decay": True, } param_out, moment1_out, moment2_out = adamw_step( - np_inputs, np_attrs) + np_inputs, np_attrs + ) return param_out, moment1_out, moment2_out for i in range(5): a = paddle.to_tensor( - np.random.uniform(-1, 1, (2, 13)).astype("float32")) + np.random.uniform(-1, 1, (2, 13)).astype("float32") + ) a1 = linear1(a) out = linear2(a1) out = paddle.mean(out) out.backward() fc1_w, fc1_w_mon1, fc1_w_mon2 = get_numpy_output( - fc1_w, np.array(linear1.weight.grad), fc1_w_mon1, fc1_w_mon2, - simple_lr_fun(linear1.weight), i + 1) + fc1_w, + np.array(linear1.weight.grad), + fc1_w_mon1, + fc1_w_mon2, + simple_lr_fun(linear1.weight), + i + 1, + ) fc1_b, fc1_b_mon1, fc1_b_mon2 = get_numpy_output( - fc1_b, np.array(linear1.bias.grad), fc1_b_mon1, fc1_b_mon2, - simple_lr_fun(linear1.bias), i + 1) + fc1_b, + np.array(linear1.bias.grad), + fc1_b_mon1, + fc1_b_mon2, + simple_lr_fun(linear1.bias), + i + 1, + ) fc2_w, fc2_w_mon1, fc2_w_mon2 = get_numpy_output( - fc2_w, np.array(linear2.weight.grad), fc2_w_mon1, fc2_w_mon2, - simple_lr_fun(linear2.weight), i + 1) + fc2_w, + np.array(linear2.weight.grad), + fc2_w_mon1, + fc2_w_mon2, + simple_lr_fun(linear2.weight), + i + 1, + ) fc2_b, fc2_b_mon1, fc2_b_mon2 = get_numpy_output( - fc2_b, np.array(linear2.bias.grad), fc2_b_mon1, fc2_b_mon2, - simple_lr_fun(linear2.bias), i + 1) + fc2_b, + np.array(linear2.bias.grad), + fc2_b_mon1, + fc2_b_mon2, + simple_lr_fun(linear2.bias), + i + 1, + ) opt.step() opt.clear_gradients() @@ -557,19 +593,19 @@ def test_adamw_op(self): weight_attr1 = paddle.framework.ParamAttr(name="linear_0.w_0") bias_attr1 = paddle.framework.ParamAttr( name="linear_0.b_0", - initializer=paddle.nn.initializer.Constant(value=1.0)) + initializer=paddle.nn.initializer.Constant(value=1.0), + ) weight_attr2 = paddle.framework.ParamAttr(name="linear_1.w_0") bias_attr2 = paddle.framework.ParamAttr( name="linear_1.b_0", - initializer=paddle.nn.initializer.Constant(value=1.0)) - linear1 = paddle.nn.Linear(10, - 32, - weight_attr=weight_attr1, - bias_attr=bias_attr1) - linear2 = paddle.nn.Linear(32, - 1, - weight_attr=weight_attr2, - bias_attr=bias_attr2) + initializer=paddle.nn.initializer.Constant(value=1.0), + ) + linear1 = paddle.nn.Linear( + 10, 32, weight_attr=weight_attr1, bias_attr=bias_attr1 + ) + linear2 = paddle.nn.Linear( + 32, 1, weight_attr=weight_attr2, bias_attr=bias_attr2 + ) out = linear1(x) out = linear2(out) @@ -586,16 +622,18 @@ def test_adamw_op(self): cost = fluid.layers.square_error_cost(input=out, label=y) avg_cost = paddle.mean(cost) - simple_lr_fun = partial(simple_lr_setting, - decay_rate=0.8, - n_layers=2) - - opt = paddle.optimizer.AdamW(learning_rate=learning_rate, - beta1=beta1, - beta2=beta2, - weight_decay=weight_decay, - epsilon=epsilon, - lr_ratio=simple_lr_fun) + simple_lr_fun = partial( + simple_lr_setting, decay_rate=0.8, n_layers=2 + ) + + opt = paddle.optimizer.AdamW( + learning_rate=learning_rate, + beta1=beta1, + beta2=beta2, + weight_decay=weight_decay, + epsilon=epsilon, + lr_ratio=simple_lr_fun, + ) opt.minimize(avg_cost) def get_numpy_output(param, grad, moment1, moment2, lr_ratio, t): @@ -606,7 +644,7 @@ def get_numpy_output(param, grad, moment1, moment2, lr_ratio, t): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1**t]).astype("float32"), - 'Beta2Pow': np.array([beta2**t]).astype("float32") + 'Beta2Pow': np.array([beta2**t]).astype("float32"), } np_attrs = { @@ -615,19 +653,28 @@ def get_numpy_output(param, grad, moment1, moment2, lr_ratio, t): 'beta2': beta2, "lr_ratio": lr_ratio, "coeff": weight_decay, - "with_decay": True + "with_decay": True, } param_out, moment1_out, moment2_out = adamw_step( - np_inputs, np_attrs) + np_inputs, np_attrs + ) return param_out, moment1_out, moment2_out fetch_list1 = [ - "linear_0.w_0", "linear_0.b_0", "linear_1.w_0", "linear_1.b_0" + "linear_0.w_0", + "linear_0.b_0", + "linear_1.w_0", + "linear_1.b_0", ] fetch_list2 = [ - "linear_0.w_0", "linear_0.w_0@GRAD", "linear_0.b_0", - "linear_0.b_0@GRAD", "linear_1.w_0", "linear_1.w_0@GRAD", - "linear_1.b_0", "linear_1.b_0@GRAD" + "linear_0.w_0", + "linear_0.w_0@GRAD", + "linear_0.b_0", + "linear_0.b_0@GRAD", + "linear_1.w_0", + "linear_1.w_0@GRAD", + "linear_1.b_0", + "linear_1.b_0@GRAD", ] exe = fluid.Executor(place) @@ -638,18 +685,16 @@ def get_numpy_output(param, grad, moment1, moment2, lr_ratio, t): inputs = np.random.random(size=[8, 10]).astype('float32') outputs = np.random.random(size=[8, 1]).astype('float32') - param = exe.run(test_prog, - feed={ - "x": inputs, - "y": outputs - }, - fetch_list=fetch_list1) - params_and_gras = exe.run(train_prog, - feed={ - "x": inputs, - "y": outputs - }, - fetch_list=fetch_list2) + param = exe.run( + test_prog, + feed={"x": inputs, "y": outputs}, + fetch_list=fetch_list1, + ) + params_and_gras = exe.run( + train_prog, + feed={"x": inputs, "y": outputs}, + fetch_list=fetch_list2, + ) fc1_w = param[0] fc1_w_grad = params_and_gras[1] @@ -661,17 +706,37 @@ def get_numpy_output(param, grad, moment1, moment2, lr_ratio, t): fc2_b_grad = params_and_gras[7] fc1_w, fc1_w_mon1, fc1_w_mon2 = get_numpy_output( - fc1_w, fc1_w_grad, fc1_w_mon1, fc1_w_mon2, - simple_lr_fun(linear1.weight), i + 1) + fc1_w, + fc1_w_grad, + fc1_w_mon1, + fc1_w_mon2, + simple_lr_fun(linear1.weight), + i + 1, + ) fc1_b, fc1_b_mon1, fc1_b_mon2 = get_numpy_output( - fc1_b, fc1_b_grad, fc1_b_mon1, fc1_b_mon2, - simple_lr_fun(linear1.bias), i + 1) + fc1_b, + fc1_b_grad, + fc1_b_mon1, + fc1_b_mon2, + simple_lr_fun(linear1.bias), + i + 1, + ) fc2_w, fc2_w_mon1, fc2_w_mon2 = get_numpy_output( - fc2_w, fc2_w_grad, fc2_w_mon1, fc2_w_mon2, - simple_lr_fun(linear2.weight), i + 1) + fc2_w, + fc2_w_grad, + fc2_w_mon1, + fc2_w_mon2, + simple_lr_fun(linear2.weight), + i + 1, + ) fc2_b, fc2_b_mon1, fc2_b_mon2 = get_numpy_output( - fc2_b, fc2_b_grad, fc2_b_mon1, fc2_b_mon2, - simple_lr_fun(linear2.bias), i + 1) + fc2_b, + fc2_b_grad, + fc2_b_mon1, + fc2_b_mon2, + simple_lr_fun(linear2.bias), + i + 1, + ) np.testing.assert_allclose(params_and_gras[0], fc1_w, rtol=1e-6) np.testing.assert_allclose(params_and_gras[2], fc1_b, rtol=1e-6) diff --git a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool1d.py b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool1d.py index 36c102128cba0..532c2e8a45b0a 100644 --- a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool1d.py +++ b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool1d.py @@ -30,24 +30,28 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def avg_pool1D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=False, - adaptive=False, - data_type=np.float64): +def avg_pool1D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=False, + adaptive=False, + data_type=np.float64, +): N, C, L = x.shape if global_pool == 1: ksize = [L] if adaptive: L_out = ksize[0] else: - L_out = (L - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + L_out = ( + (L - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) out = np.zeros((N, C, L_out)) for i in range(L_out): @@ -59,19 +63,21 @@ def avg_pool1D_forward_naive(x, r_end = np.min((i * strides[0] + ksize[0] - paddings[0], L)) x_masked = x[:, :, r_start:r_end] - field_size = (r_end - r_start) \ - if (exclusive or adaptive) else (ksize[0]) + field_size = ( + (r_end - r_start) if (exclusive or adaptive) else (ksize[0]) + ) if data_type == np.int8 or data_type == np.uint8: - out[:, :, i] = (np.rint(np.sum(x_masked, axis=(2, 3)) / - field_size)).astype(data_type) + out[:, :, i] = ( + np.rint(np.sum(x_masked, axis=(2, 3)) / field_size) + ).astype(data_type) else: - out[:, :, - i] = (np.sum(x_masked, axis=(2)) / field_size).astype(data_type) + out[:, :, i] = (np.sum(x_masked, axis=(2)) / field_size).astype( + data_type + ) return out class TestPool1D_API(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -83,22 +89,21 @@ def check_adaptive_avg_dygraph_results(self, place): input_np = np.random.random([2, 3, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) result = F.adaptive_avg_pool1d(input, output_size=16) - result_np = avg_pool1D_forward_naive(input_np, - ksize=[16], - strides=[0], - paddings=[0], - adaptive=True) + result_np = avg_pool1D_forward_naive( + input_np, ksize=[16], strides=[0], paddings=[0], adaptive=True + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) ada_max_pool1d_dg = paddle.nn.layer.AdaptiveAvgPool1D( - output_size=16) + output_size=16 + ) result = ada_max_pool1d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - result = paddle.nn.functional.common.interpolate(input, - mode="area", - size=16) + result = paddle.nn.functional.common.interpolate( + input, mode="area", size=16 + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) def check_adaptive_avg_static_results(self, place): @@ -107,16 +112,16 @@ def check_adaptive_avg_static_results(self, place): result = F.adaptive_avg_pool1d(input, output_size=16) input_np = np.random.random([2, 3, 32]).astype("float32") - result_np = avg_pool1D_forward_naive(input_np, - ksize=[16], - strides=[2], - paddings=[0], - adaptive=True) + result_np = avg_pool1D_forward_naive( + input_np, ksize=[16], strides=[2], paddings=[0], adaptive=True + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def test_adaptive_avg_pool1d(self): diff --git a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool2d.py b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool2d.py index 9ca75c6b8b1eb..9c12d5977d42b 100644 --- a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool2d.py +++ b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool2d.py @@ -32,16 +32,18 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def adaptive_pool2d_forward(x, - output_size, - data_format='NCHW', - pool_type="avg"): +def adaptive_pool2d_forward( + x, output_size, data_format='NCHW', pool_type="avg" +): N = x.shape[0] - C, H, W = [x.shape[1], x.shape[2], x.shape[3]] if data_format == 'NCHW' \ + C, H, W = ( + [x.shape[1], x.shape[2], x.shape[3]] + if data_format == 'NCHW' else [x.shape[3], x.shape[1], x.shape[2]] + ) - if (isinstance(output_size, int) or output_size == None): + if isinstance(output_size, int) or output_size == None: H_out = output_size W_out = output_size output_size = [H_out, W_out] @@ -55,8 +57,11 @@ def adaptive_pool2d_forward(x, output_size[1] = W W_out = W - out = np.zeros((N, C, H_out, W_out)) if data_format=='NCHW' \ + out = ( + np.zeros((N, C, H_out, W_out)) + if data_format == 'NCHW' else np.zeros((N, H_out, W_out, C)) + ) for i in range(H_out): in_h_start = adaptive_start_index(i, H, output_size[0]) @@ -69,16 +74,18 @@ def adaptive_pool2d_forward(x, if data_format == 'NCHW': x_masked = x[:, :, in_h_start:in_h_end, in_w_start:in_w_end] if pool_type == 'avg': - field_size = ((in_h_end - in_h_start) * - (in_w_end - in_w_start)) + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) out[:, :, i, j] = np.sum(x_masked, axis=(2, 3)) / field_size elif pool_type == 'max': out[:, :, i, j] = np.max(x_masked, axis=(2, 3)) elif data_format == 'NHWC': x_masked = x[:, in_h_start:in_h_end, in_w_start:in_w_end, :] if pool_type == 'avg': - field_size = ((in_h_end - in_h_start) * - (in_w_end - in_w_start)) + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) out[:, i, j, :] = np.sum(x_masked, axis=(1, 2)) / field_size elif pool_type == 'max': out[:, i, j, :] = np.max(x_masked, axis=(1, 2)) @@ -86,57 +93,60 @@ def adaptive_pool2d_forward(x, class TestAdaptiveAvgPool2DAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[3, 3], - pool_type="avg") + self.res_1_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[3, 3], pool_type="avg" + ) - self.res_2_np = adaptive_pool2d_forward(x=self.x_np, - output_size=5, - pool_type="avg") + self.res_2_np = adaptive_pool2d_forward( + x=self.x_np, output_size=5, pool_type="avg" + ) - self.res_3_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[2, 5], - pool_type="avg") + self.res_3_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[2, 5], pool_type="avg" + ) - self.res_4_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[3, 3], - pool_type="avg", - data_format="NHWC") + self.res_4_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[3, 3], pool_type="avg", data_format="NHWC" + ) - self.res_5_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[None, 3], - pool_type="avg") + self.res_5_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[None, 3], pool_type="avg" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() x = paddle.fluid.data(name="x", shape=[2, 3, 7, 7], dtype="float32") - out_1 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=[3, 3]) + out_1 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=[3, 3] + ) out_2 = paddle.nn.functional.adaptive_avg_pool2d(x=x, output_size=5) - out_3 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=[2, 5]) + out_3 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=[2, 5] + ) - out_4 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=[3, 3], - data_format="NHWC") + out_4 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=[3, 3], data_format="NHWC" + ) out_5 = paddle.nn.functional.adaptive_avg_pool2d( - x=x, output_size=[None, 3]) + x=x, output_size=[None, 3] + ) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, res_4, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_4, out_5]) + [res_1, res_2, res_3, res_4, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_4, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -149,30 +159,34 @@ def test_static_graph(self): assert np.allclose(res_5, self.res_5_np) def test_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) - out_1 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=[3, 3]) + out_1 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=[3, 3] + ) out_2 = paddle.nn.functional.adaptive_avg_pool2d(x=x, output_size=5) - out_3 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=[2, 5]) + out_3 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=[2, 5] + ) - out_4 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=[3, 3], - data_format="NHWC") + out_4 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=[3, 3], data_format="NHWC" + ) out_5 = paddle.nn.functional.adaptive_avg_pool2d( - x=x, output_size=[None, 3]) + x=x, output_size=[None, 3] + ) - out_6 = paddle.nn.functional.interpolate(x=x, - mode="area", - size=[2, 5]) + out_6 = paddle.nn.functional.interpolate( + x=x, mode="area", size=[2, 5] + ) assert np.allclose(out_1.numpy(), self.res_1_np) @@ -188,33 +202,32 @@ def test_dynamic_graph(self): class TestAdaptiveAvgPool2DClassAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[3, 3], - pool_type="avg") + self.res_1_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[3, 3], pool_type="avg" + ) - self.res_2_np = adaptive_pool2d_forward(x=self.x_np, - output_size=5, - pool_type="avg") + self.res_2_np = adaptive_pool2d_forward( + x=self.x_np, output_size=5, pool_type="avg" + ) - self.res_3_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[2, 5], - pool_type="avg") + self.res_3_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[2, 5], pool_type="avg" + ) - self.res_4_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[3, 3], - pool_type="avg", - data_format="NHWC") + self.res_4_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[3, 3], pool_type="avg", data_format="NHWC" + ) - self.res_5_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[None, 3], - pool_type="avg") + self.res_5_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[None, 3], pool_type="avg" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() x = paddle.fluid.data(name="x", shape=[2, 3, 7, 7], dtype="float32") @@ -228,19 +241,22 @@ def test_static_graph(self): adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D(output_size=[2, 5]) out_3 = adaptive_avg_pool(x=x) - adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D(output_size=[3, 3], - data_format="NHWC") + adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D( + output_size=[3, 3], data_format="NHWC" + ) out_4 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D( - output_size=[None, 3]) + output_size=[None, 3] + ) out_5 = adaptive_avg_pool(x=x) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, res_4, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_4, out_5]) + [res_1, res_2, res_3, res_4, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_4, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -253,8 +269,9 @@ def test_static_graph(self): assert np.allclose(res_5, self.res_5_np) def test_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) @@ -268,12 +285,14 @@ def test_dynamic_graph(self): adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D(output_size=[2, 5]) out_3 = adaptive_avg_pool(x=x) - adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D(output_size=[3, 3], - data_format="NHWC") + adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D( + output_size=[3, 3], data_format="NHWC" + ) out_4 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool2D( - output_size=[None, 3]) + output_size=[None, 3] + ) out_5 = adaptive_avg_pool(x=x) assert np.allclose(out_1.numpy(), self.res_1_np) @@ -288,7 +307,6 @@ def test_dynamic_graph(self): class TestOutputSizeTensor(UnittestBase): - def init_info(self): self.shapes = [[1, 3, 6, 6]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -313,8 +331,9 @@ def test_static(self): exe.run(starup_prog) res = exe.run(fetch_list=[out1, out2]) np.testing.assert_allclose(res[0], res[1]) - paddle.static.save_inference_model(self.save_path, [x], - [out1, out2], exe) + paddle.static.save_inference_model( + self.save_path, [x], [out1, out2], exe + ) # Test for Inference Predictor infer_outs = self.infer_prog() np.testing.assert_array_equal(infer_outs[0].shape, (1, 3, 3, 3)) @@ -330,13 +349,13 @@ def call_func(self, x): # list[Tensor] output_size = [paddle.assign([3]), paddle.assign([3])] out1 = paddle.nn.functional.adaptive_avg_pool2d(x=x, output_size=[3, 3]) - out2 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=output_size) + out2 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=output_size + ) return out1, out2 class TestOutputSizeListTensor(TestOutputSizeTensor): - def path_prefix(self): return 'pool2d_tensors' @@ -344,13 +363,13 @@ def call_func(self, x): # list[int, Tensor] output_size = [paddle.assign([3]), 3] out1 = paddle.nn.functional.adaptive_avg_pool2d(x=x, output_size=[3, 3]) - out2 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=output_size) + out2 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=output_size + ) return out1, out2 class TestOutputSizeListTensor2(TestOutputSizeTensor): - def path_prefix(self): return 'pool2d_tensor2' @@ -358,8 +377,9 @@ def call_func(self, x): # A Tensor output_size = paddle.assign([3, 3]) out1 = paddle.nn.functional.adaptive_avg_pool2d(x=x, output_size=[3, 3]) - out2 = paddle.nn.functional.adaptive_avg_pool2d(x=x, - output_size=output_size) + out2 = paddle.nn.functional.adaptive_avg_pool2d( + x=x, output_size=output_size + ) return out1, out2 diff --git a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool3d.py b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool3d.py index 5f3e2d38eb144..e7a8685a8f164 100755 --- a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool3d.py +++ b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool3d.py @@ -28,17 +28,18 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def adaptive_pool3d_forward(x, - output_size, - adaptive=True, - data_format='NCDHW', - pool_type='avg'): +def adaptive_pool3d_forward( + x, output_size, adaptive=True, data_format='NCDHW', pool_type='avg' +): N = x.shape[0] - C, D, H, W = [x.shape[1], x.shape[2], x.shape[3], x.shape[4]] \ - if data_format == 'NCDHW' else [x.shape[4], x.shape[1], x.shape[2],x.shape[3]] + C, D, H, W = ( + [x.shape[1], x.shape[2], x.shape[3], x.shape[4]] + if data_format == 'NCDHW' + else [x.shape[4], x.shape[1], x.shape[2], x.shape[3]] + ) - if (isinstance(output_size, int) or output_size == None): + if isinstance(output_size, int) or output_size == None: H_out = output_size W_out = output_size D_out = output_size @@ -56,8 +57,11 @@ def adaptive_pool3d_forward(x, output_size[2] = W W_out = W - out = np.zeros((N, C, D_out, H_out, W_out)) if data_format=='NCDHW' \ + out = ( + np.zeros((N, C, D_out, H_out, W_out)) + if data_format == 'NCDHW' else np.zeros((N, D_out, H_out, W_out, C)) + ) for k in range(D_out): d_start = adaptive_start_index(k, D, output_size[0]) d_end = adaptive_end_index(k, D, output_size[0]) @@ -71,82 +75,99 @@ def adaptive_pool3d_forward(x, w_end = adaptive_end_index(j, W, output_size[2]) if data_format == 'NCDHW': - x_masked = x[:, :, d_start:d_end, h_start:h_end, - w_start:w_end] + x_masked = x[ + :, :, d_start:d_end, h_start:h_end, w_start:w_end + ] if pool_type == 'avg': - field_size = (d_end - d_start) * (h_end - h_start) * ( - w_end - w_start) - out[:, :, k, i, - j] = np.sum(x_masked, axis=(2, 3, 4)) / field_size + field_size = ( + (d_end - d_start) + * (h_end - h_start) + * (w_end - w_start) + ) + out[:, :, k, i, j] = ( + np.sum(x_masked, axis=(2, 3, 4)) / field_size + ) elif pool_type == 'max': out[:, :, k, i, j] = np.max(x_masked, axis=(2, 3, 4)) elif data_format == 'NDHWC': - x_masked = x[:, d_start:d_end, h_start:h_end, - w_start:w_end, :] + x_masked = x[ + :, d_start:d_end, h_start:h_end, w_start:w_end, : + ] if pool_type == 'avg': - field_size = (d_end - d_start) * (h_end - h_start) * ( - w_end - w_start) - out[:, k, i, j, :] = np.sum(x_masked, - axis=(1, 2, 3)) / field_size + field_size = ( + (d_end - d_start) + * (h_end - h_start) + * (w_end - w_start) + ) + out[:, k, i, j, :] = ( + np.sum(x_masked, axis=(1, 2, 3)) / field_size + ) elif pool_type == 'max': out[:, k, i, j, :] = np.max(x_masked, axis=(1, 2, 3)) return out class TestAdaptiveAvgPool3DAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 5, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[3, 3, 3], - pool_type="avg") + self.res_1_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[3, 3, 3], pool_type="avg" + ) - self.res_2_np = adaptive_pool3d_forward(x=self.x_np, - output_size=5, - pool_type="avg") + self.res_2_np = adaptive_pool3d_forward( + x=self.x_np, output_size=5, pool_type="avg" + ) - self.res_3_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[2, 3, 5], - pool_type="avg") + self.res_3_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[2, 3, 5], pool_type="avg" + ) - self.res_4_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[3, 3, 3], - pool_type="avg", - data_format="NDHWC") + self.res_4_np = adaptive_pool3d_forward( + x=self.x_np, + output_size=[3, 3, 3], + pool_type="avg", + data_format="NDHWC", + ) - self.res_5_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[None, 3, None], - pool_type="avg") + self.res_5_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[None, 3, None], pool_type="avg" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x = paddle.fluid.data(name="x", - shape=[2, 3, 5, 7, 7], - dtype="float32") + x = paddle.fluid.data( + name="x", shape=[2, 3, 5, 7, 7], dtype="float32" + ) out_1 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[3, 3, 3]) + x=x, output_size=[3, 3, 3] + ) out_2 = paddle.nn.functional.adaptive_avg_pool3d(x=x, output_size=5) out_3 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[2, 3, 5]) + x=x, output_size=[2, 3, 5] + ) out_4 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[3, 3, 3], data_format="NDHWC") + x=x, output_size=[3, 3, 3], data_format="NDHWC" + ) out_5 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[None, 3, None]) + x=x, output_size=[None, 3, None] + ) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, res_4, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_4, out_5]) + [res_1, res_2, res_3, res_4, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_4, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -159,29 +180,34 @@ def test_static_graph(self): assert np.allclose(res_5, self.res_5_np) def test_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) out_1 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[3, 3, 3]) + x=x, output_size=[3, 3, 3] + ) out_2 = paddle.nn.functional.adaptive_avg_pool3d(x=x, output_size=5) out_3 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[2, 3, 5]) + x=x, output_size=[2, 3, 5] + ) out_4 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[3, 3, 3], data_format="NDHWC") + x=x, output_size=[3, 3, 3], data_format="NDHWC" + ) out_5 = paddle.nn.functional.adaptive_avg_pool3d( - x=x, output_size=[None, 3, None]) + x=x, output_size=[None, 3, None] + ) - out_6 = paddle.nn.functional.interpolate(x=x, - mode="area", - size=[2, 3, 5]) + out_6 = paddle.nn.functional.interpolate( + x=x, mode="area", size=[2, 3, 5] + ) assert np.allclose(out_1.numpy(), self.res_1_np) @@ -197,63 +223,70 @@ def test_dynamic_graph(self): class TestAdaptiveAvgPool3DClassAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 5, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[3, 3, 3], - pool_type="avg") + self.res_1_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[3, 3, 3], pool_type="avg" + ) - self.res_2_np = adaptive_pool3d_forward(x=self.x_np, - output_size=5, - pool_type="avg") + self.res_2_np = adaptive_pool3d_forward( + x=self.x_np, output_size=5, pool_type="avg" + ) - self.res_3_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[2, 3, 5], - pool_type="avg") + self.res_3_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[2, 3, 5], pool_type="avg" + ) - self.res_4_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[3, 3, 3], - pool_type="avg", - data_format="NDHWC") + self.res_4_np = adaptive_pool3d_forward( + x=self.x_np, + output_size=[3, 3, 3], + pool_type="avg", + data_format="NDHWC", + ) - self.res_5_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[None, 3, None], - pool_type="avg") + self.res_5_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[None, 3, None], pool_type="avg" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x = paddle.fluid.data(name="x", - shape=[2, 3, 5, 7, 7], - dtype="float32") + x = paddle.fluid.data( + name="x", shape=[2, 3, 5, 7, 7], dtype="float32" + ) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[3, 3, 3]) + output_size=[3, 3, 3] + ) out_1 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D(output_size=5) out_2 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[2, 3, 5]) + output_size=[2, 3, 5] + ) out_3 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[3, 3, 3], data_format="NDHWC") + output_size=[3, 3, 3], data_format="NDHWC" + ) out_4 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[None, 3, None]) + output_size=[None, 3, None] + ) out_5 = adaptive_avg_pool(x=x) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, res_4, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_4, out_5]) + [res_1, res_2, res_3, res_4, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_4, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -266,29 +299,34 @@ def test_static_graph(self): assert np.allclose(res_5, self.res_5_np) def test_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[3, 3, 3]) + output_size=[3, 3, 3] + ) out_1 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D(output_size=5) out_2 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[2, 3, 5]) + output_size=[2, 3, 5] + ) out_3 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[3, 3, 3], data_format="NDHWC") + output_size=[3, 3, 3], data_format="NDHWC" + ) out_4 = adaptive_avg_pool(x=x) adaptive_avg_pool = paddle.nn.AdaptiveAvgPool3D( - output_size=[None, 3, None]) + output_size=[None, 3, None] + ) out_5 = adaptive_avg_pool(x=x) assert np.allclose(out_1.numpy(), self.res_1_np) diff --git a/python/paddle/fluid/tests/unittests/test_adaptive_max_pool1d.py b/python/paddle/fluid/tests/unittests/test_adaptive_max_pool1d.py index d64ff0a3c76d3..a83ef2bfd5bae 100644 --- a/python/paddle/fluid/tests/unittests/test_adaptive_max_pool1d.py +++ b/python/paddle/fluid/tests/unittests/test_adaptive_max_pool1d.py @@ -29,24 +29,28 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def max_pool1D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=False, - adaptive=False, - data_type=np.float64): +def max_pool1D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=False, + adaptive=False, + data_type=np.float64, +): N, C, L = x.shape if global_pool == 1: ksize = [L] if adaptive: L_out = ksize[0] else: - L_out = (L - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + L_out = ( + (L - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) out = np.zeros((N, C, L_out)) for i in range(L_out): @@ -63,7 +67,6 @@ def max_pool1D_forward_naive(x, class TestPool1D_API(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -76,15 +79,14 @@ def check_adaptive_max_dygraph_results(self, place): input = fluid.dygraph.to_variable(input_np) result = F.adaptive_max_pool1d(input, output_size=16) - result_np = max_pool1D_forward_naive(input_np, - ksize=[16], - strides=[0], - paddings=[0], - adaptive=True) + result_np = max_pool1D_forward_naive( + input_np, ksize=[16], strides=[0], paddings=[0], adaptive=True + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) ada_max_pool1d_dg = paddle.nn.layer.AdaptiveMaxPool1D( - output_size=16) + output_size=16 + ) result = ada_max_pool1d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -94,16 +96,16 @@ def check_adaptive_max_static_results(self, place): result = F.adaptive_max_pool1d(input, output_size=16) input_np = np.random.random([2, 3, 32]).astype("float32") - result_np = max_pool1D_forward_naive(input_np, - ksize=[16], - strides=[2], - paddings=[0], - adaptive=True) + result_np = max_pool1D_forward_naive( + input_np, ksize=[16], strides=[2], paddings=[0], adaptive=True + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def test_adaptive_max_pool1d(self): @@ -113,14 +115,15 @@ def test_adaptive_max_pool1d(self): class TestOutDtype(unittest.TestCase): - def test_max_pool(self): api_fn = F.adaptive_max_pool1d shape = [1, 3, 32] - check_out_dtype(api_fn, - in_specs=[(shape, )], - expect_dtypes=['float32', 'float64'], - output_size=16) + check_out_dtype( + api_fn, + in_specs=[(shape,)], + expect_dtypes=['float32', 'float64'], + output_size=16, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_adaptive_max_pool2d.py b/python/paddle/fluid/tests/unittests/test_adaptive_max_pool2d.py index 5ee4918a753c0..9884cb6c90da8 100644 --- a/python/paddle/fluid/tests/unittests/test_adaptive_max_pool2d.py +++ b/python/paddle/fluid/tests/unittests/test_adaptive_max_pool2d.py @@ -30,16 +30,18 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def adaptive_pool2d_forward(x, - output_size, - data_format='NCHW', - pool_type="max"): +def adaptive_pool2d_forward( + x, output_size, data_format='NCHW', pool_type="max" +): N = x.shape[0] - C, H, W = [x.shape[1], x.shape[2], x.shape[3]] if data_format == 'NCHW' \ + C, H, W = ( + [x.shape[1], x.shape[2], x.shape[3]] + if data_format == 'NCHW' else [x.shape[3], x.shape[1], x.shape[2]] + ) - if (isinstance(output_size, int) or output_size == None): + if isinstance(output_size, int) or output_size == None: H_out = output_size W_out = output_size output_size = [H_out, W_out] @@ -53,8 +55,11 @@ def adaptive_pool2d_forward(x, output_size[1] = W W_out = W - out = np.zeros((N, C, H_out, W_out)) if data_format=='NCHW' \ + out = ( + np.zeros((N, C, H_out, W_out)) + if data_format == 'NCHW' else np.zeros((N, H_out, W_out, C)) + ) for i in range(H_out): in_h_start = adaptive_start_index(i, H, output_size[0]) @@ -67,16 +72,18 @@ def adaptive_pool2d_forward(x, if data_format == 'NCHW': x_masked = x[:, :, in_h_start:in_h_end, in_w_start:in_w_end] if pool_type == 'avg': - field_size = ((in_h_end - in_h_start) * - (in_w_end - in_w_start)) + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) out[:, :, i, j] = np.sum(x_masked, axis=(2, 3)) / field_size elif pool_type == 'max': out[:, :, i, j] = np.max(x_masked, axis=(2, 3)) elif data_format == 'NHWC': x_masked = x[:, in_h_start:in_h_end, in_w_start:in_w_end, :] if pool_type == 'avg': - field_size = ((in_h_end - in_h_start) * - (in_w_end - in_w_start)) + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) out[:, i, j, :] = np.sum(x_masked, axis=(1, 2)) / field_size elif pool_type == 'max': out[:, i, j, :] = np.max(x_masked, axis=(1, 2)) @@ -84,20 +91,19 @@ def adaptive_pool2d_forward(x, class TestAdaptiveMaxPool2DAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[3, 3], - pool_type="max") + self.res_1_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[3, 3], pool_type="max" + ) - self.res_2_np = adaptive_pool2d_forward(x=self.x_np, - output_size=5, - pool_type="max") + self.res_2_np = adaptive_pool2d_forward( + x=self.x_np, output_size=5, pool_type="max" + ) - self.res_3_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[2, 5], - pool_type="max") + self.res_3_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[2, 5], pool_type="max" + ) """ self.res_4_np = adaptive_pool2d_forward( x=self.x_np, @@ -105,36 +111,41 @@ def setUp(self): pool_type="max", data_format="NHWC") """ - self.res_5_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[None, 3], - pool_type="max") + self.res_5_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[None, 3], pool_type="max" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() x = paddle.fluid.data(name="x", shape=[2, 3, 7, 7], dtype="float32") - out_1 = paddle.nn.functional.adaptive_max_pool2d(x=x, - output_size=[3, 3]) + out_1 = paddle.nn.functional.adaptive_max_pool2d( + x=x, output_size=[3, 3] + ) out_2 = paddle.nn.functional.adaptive_max_pool2d(x=x, output_size=5) - out_3 = paddle.nn.functional.adaptive_max_pool2d(x=x, - output_size=[2, 5]) + out_3 = paddle.nn.functional.adaptive_max_pool2d( + x=x, output_size=[2, 5] + ) - #out_4 = paddle.nn.functional.adaptive_max_pool2d( + # out_4 = paddle.nn.functional.adaptive_max_pool2d( # x=x, output_size=[3, 3], data_format="NHWC") out_5 = paddle.nn.functional.adaptive_max_pool2d( - x=x, output_size=[None, 3]) + x=x, output_size=[None, 3] + ) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_5]) + [res_1, res_2, res_3, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -142,31 +153,34 @@ def test_static_graph(self): assert np.allclose(res_3, self.res_3_np) - #assert np.allclose(res_4, self.res_4_np) + # assert np.allclose(res_4, self.res_4_np) assert np.allclose(res_5, self.res_5_np) def test_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) - out_1 = paddle.nn.functional.adaptive_max_pool2d(x=x, - return_mask=False, - output_size=[3, 3]) + out_1 = paddle.nn.functional.adaptive_max_pool2d( + x=x, return_mask=False, output_size=[3, 3] + ) out_2 = paddle.nn.functional.adaptive_max_pool2d(x=x, output_size=5) - out_3 = paddle.nn.functional.adaptive_max_pool2d(x=x, - output_size=[2, 5]) + out_3 = paddle.nn.functional.adaptive_max_pool2d( + x=x, output_size=[2, 5] + ) - #out_4 = paddle.nn.functional.adaptive_max_pool2d( + # out_4 = paddle.nn.functional.adaptive_max_pool2d( # x=x, output_size=[3, 3], data_format="NHWC") out_5 = paddle.nn.functional.adaptive_max_pool2d( - x=x, output_size=[None, 3]) + x=x, output_size=[None, 3] + ) assert np.allclose(out_1.numpy(), self.res_1_np) @@ -174,40 +188,40 @@ def test_dynamic_graph(self): assert np.allclose(out_3.numpy(), self.res_3_np) - #assert np.allclose(out_4.numpy(), self.res_4_np) + # assert np.allclose(out_4.numpy(), self.res_4_np) assert np.allclose(out_5.numpy(), self.res_5_np) class TestAdaptiveMaxPool2DClassAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[3, 3], - pool_type="max") + self.res_1_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[3, 3], pool_type="max" + ) - self.res_2_np = adaptive_pool2d_forward(x=self.x_np, - output_size=5, - pool_type="max") + self.res_2_np = adaptive_pool2d_forward( + x=self.x_np, output_size=5, pool_type="max" + ) - self.res_3_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[2, 5], - pool_type="max") + self.res_3_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[2, 5], pool_type="max" + ) - #self.res_4_np = adaptive_pool2d_forward( + # self.res_4_np = adaptive_pool2d_forward( # x=self.x_np, # output_size=[3, 3], # pool_type="max", # data_format="NHWC") - self.res_5_np = adaptive_pool2d_forward(x=self.x_np, - output_size=[None, 3], - pool_type="max") + self.res_5_np = adaptive_pool2d_forward( + x=self.x_np, output_size=[None, 3], pool_type="max" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() x = paddle.fluid.data(name="x", shape=[2, 3, 7, 7], dtype="float32") @@ -226,14 +240,16 @@ def test_static_graph(self): # out_4 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool2D( - output_size=[None, 3]) + output_size=[None, 3] + ) out_5 = adaptive_max_pool(x=x) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_5]) + [res_1, res_2, res_3, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -241,13 +257,14 @@ def test_static_graph(self): assert np.allclose(res_3, self.res_3_np) - #assert np.allclose(res_4, self.res_4_np) + # assert np.allclose(res_4, self.res_4_np) assert np.allclose(res_5, self.res_5_np) def test_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) @@ -261,12 +278,13 @@ def test_dynamic_graph(self): adaptive_max_pool = paddle.nn.AdaptiveMaxPool2D(output_size=[2, 5]) out_3 = adaptive_max_pool(x=x) - #adaptive_max_pool = paddle.nn.AdaptiveMaxPool2D( + # adaptive_max_pool = paddle.nn.AdaptiveMaxPool2D( # output_size=[3, 3], data_format="NHWC") - #out_4 = adaptive_max_pool(x=x) + # out_4 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool2D( - output_size=[None, 3]) + output_size=[None, 3] + ) out_5 = adaptive_max_pool(x=x) assert np.allclose(out_1.numpy(), self.res_1_np) @@ -275,20 +293,21 @@ def test_dynamic_graph(self): assert np.allclose(out_3.numpy(), self.res_3_np) - #assert np.allclose(out_4.numpy(), self.res_4_np) + # assert np.allclose(out_4.numpy(), self.res_4_np) assert np.allclose(out_5.numpy(), self.res_5_np) class TestOutDtype(unittest.TestCase): - def test_max_pool(self): api_fn = F.adaptive_max_pool2d shape = [1, 3, 32, 32] - check_out_dtype(api_fn, - in_specs=[(shape, )], - expect_dtypes=['float32', 'float64'], - output_size=16) + check_out_dtype( + api_fn, + in_specs=[(shape,)], + expect_dtypes=['float32', 'float64'], + output_size=16, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_adaptive_max_pool3d.py b/python/paddle/fluid/tests/unittests/test_adaptive_max_pool3d.py index 214b6d90b61bb..e90c406198038 100755 --- a/python/paddle/fluid/tests/unittests/test_adaptive_max_pool3d.py +++ b/python/paddle/fluid/tests/unittests/test_adaptive_max_pool3d.py @@ -30,17 +30,18 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def adaptive_pool3d_forward(x, - output_size, - adaptive=True, - data_format='NCDHW', - pool_type='max'): +def adaptive_pool3d_forward( + x, output_size, adaptive=True, data_format='NCDHW', pool_type='max' +): N = x.shape[0] - C, D, H, W = [x.shape[1], x.shape[2], x.shape[3], x.shape[4]] \ - if data_format == 'NCDHW' else [x.shape[4], x.shape[1], x.shape[2],x.shape[3]] + C, D, H, W = ( + [x.shape[1], x.shape[2], x.shape[3], x.shape[4]] + if data_format == 'NCDHW' + else [x.shape[4], x.shape[1], x.shape[2], x.shape[3]] + ) - if (isinstance(output_size, int) or output_size == None): + if isinstance(output_size, int) or output_size == None: H_out = output_size W_out = output_size D_out = output_size @@ -58,8 +59,11 @@ def adaptive_pool3d_forward(x, output_size[2] = W W_out = W - out = np.zeros((N, C, D_out, H_out, W_out)) if data_format=='NCDHW' \ + out = ( + np.zeros((N, C, D_out, H_out, W_out)) + if data_format == 'NCDHW' else np.zeros((N, D_out, H_out, W_out, C)) + ) for k in range(D_out): d_start = adaptive_start_index(k, D, output_size[0]) d_end = adaptive_end_index(k, D, output_size[0]) @@ -73,82 +77,98 @@ def adaptive_pool3d_forward(x, w_end = adaptive_end_index(j, W, output_size[2]) if data_format == 'NCDHW': - x_masked = x[:, :, d_start:d_end, h_start:h_end, - w_start:w_end] + x_masked = x[ + :, :, d_start:d_end, h_start:h_end, w_start:w_end + ] if pool_type == 'avg': - field_size = (d_end - d_start) * (h_end - h_start) * ( - w_end - w_start) - out[:, :, k, i, - j] = np.sum(x_masked, axis=(2, 3, 4)) / field_size + field_size = ( + (d_end - d_start) + * (h_end - h_start) + * (w_end - w_start) + ) + out[:, :, k, i, j] = ( + np.sum(x_masked, axis=(2, 3, 4)) / field_size + ) elif pool_type == 'max': out[:, :, k, i, j] = np.max(x_masked, axis=(2, 3, 4)) elif data_format == 'NDHWC': - x_masked = x[:, d_start:d_end, h_start:h_end, - w_start:w_end, :] + x_masked = x[ + :, d_start:d_end, h_start:h_end, w_start:w_end, : + ] if pool_type == 'avg': - field_size = (d_end - d_start) * (h_end - h_start) * ( - w_end - w_start) - out[:, k, i, j, :] = np.sum(x_masked, - axis=(1, 2, 3)) / field_size + field_size = ( + (d_end - d_start) + * (h_end - h_start) + * (w_end - w_start) + ) + out[:, k, i, j, :] = ( + np.sum(x_masked, axis=(1, 2, 3)) / field_size + ) elif pool_type == 'max': out[:, k, i, j, :] = np.max(x_masked, axis=(1, 2, 3)) return out class TestAdaptiveMaxPool3DAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 5, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[3, 3, 3], - pool_type="max") + self.res_1_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[3, 3, 3], pool_type="max" + ) - self.res_2_np = adaptive_pool3d_forward(x=self.x_np, - output_size=5, - pool_type="max") + self.res_2_np = adaptive_pool3d_forward( + x=self.x_np, output_size=5, pool_type="max" + ) - self.res_3_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[2, 3, 5], - pool_type="max") + self.res_3_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[2, 3, 5], pool_type="max" + ) - self.res_4_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[3, 3, 3], - pool_type="max", - data_format="NDHWC") + self.res_4_np = adaptive_pool3d_forward( + x=self.x_np, + output_size=[3, 3, 3], + pool_type="max", + data_format="NDHWC", + ) - self.res_5_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[None, 3, None], - pool_type="max") + self.res_5_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[None, 3, None], pool_type="max" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x = paddle.fluid.data(name="x", - shape=[2, 3, 5, 7, 7], - dtype="float32") + x = paddle.fluid.data( + name="x", shape=[2, 3, 5, 7, 7], dtype="float32" + ) out_1 = paddle.nn.functional.adaptive_max_pool3d( - x=x, output_size=[3, 3, 3]) + x=x, output_size=[3, 3, 3] + ) out_2 = paddle.nn.functional.adaptive_max_pool3d(x=x, output_size=5) out_3 = paddle.nn.functional.adaptive_max_pool3d( - x=x, output_size=[2, 3, 5]) + x=x, output_size=[2, 3, 5] + ) - #out_4 = paddle.nn.functional.adaptive_max_pool3d( + # out_4 = paddle.nn.functional.adaptive_max_pool3d( # x=x, output_size=[3, 3, 3], data_format="NDHWC") out_5 = paddle.nn.functional.adaptive_max_pool3d( - x=x, output_size=[None, 3, None]) + x=x, output_size=[None, 3, None] + ) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_5]) + [res_1, res_2, res_3, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -156,30 +176,34 @@ def test_static_graph(self): assert np.allclose(res_3, self.res_3_np) - #assert np.allclose(res_4, self.res_4_np) + # assert np.allclose(res_4, self.res_4_np) assert np.allclose(res_5, self.res_5_np) def func_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) out_1 = paddle.nn.functional.adaptive_max_pool3d( - x=x, output_size=[3, 3, 3]) + x=x, output_size=[3, 3, 3] + ) out_2 = paddle.nn.functional.adaptive_max_pool3d(x=x, output_size=5) out_3 = paddle.nn.functional.adaptive_max_pool3d( - x=x, output_size=[2, 3, 5]) + x=x, output_size=[2, 3, 5] + ) - #out_4 = paddle.nn.functional.adaptive_max_pool3d( + # out_4 = paddle.nn.functional.adaptive_max_pool3d( # x=x, output_size=[3, 3, 3], data_format="NDHWC") out_5 = paddle.nn.functional.adaptive_max_pool3d( - x=x, output_size=[None, 3, None]) + x=x, output_size=[None, 3, None] + ) assert np.allclose(out_1.numpy(), self.res_1_np) @@ -187,7 +211,7 @@ def func_dynamic_graph(self): assert np.allclose(out_3.numpy(), self.res_3_np) - #assert np.allclose(out_4.numpy(), self.res_4_np) + # assert np.allclose(out_4.numpy(), self.res_4_np) assert np.allclose(out_5.numpy(), self.res_5_np) @@ -198,20 +222,19 @@ def test_dynamic_graph(self): class TestAdaptiveMaxPool3DClassAPI(unittest.TestCase): - def setUp(self): self.x_np = np.random.random([2, 3, 5, 7, 7]).astype("float32") - self.res_1_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[3, 3, 3], - pool_type="max") + self.res_1_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[3, 3, 3], pool_type="max" + ) - self.res_2_np = adaptive_pool3d_forward(x=self.x_np, - output_size=5, - pool_type="max") + self.res_2_np = adaptive_pool3d_forward( + x=self.x_np, output_size=5, pool_type="max" + ) - self.res_3_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[2, 3, 5], - pool_type="max") + self.res_3_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[2, 3, 5], pool_type="max" + ) # self.res_4_np = adaptive_pool3d_forward( # x=self.x_np, @@ -219,28 +242,31 @@ def setUp(self): # pool_type="max", # data_format="NDHWC") - self.res_5_np = adaptive_pool3d_forward(x=self.x_np, - output_size=[None, 3, None], - pool_type="max") + self.res_5_np = adaptive_pool3d_forward( + x=self.x_np, output_size=[None, 3, None], pool_type="max" + ) def test_static_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x = paddle.fluid.data(name="x", - shape=[2, 3, 5, 7, 7], - dtype="float32") + x = paddle.fluid.data( + name="x", shape=[2, 3, 5, 7, 7], dtype="float32" + ) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( - output_size=[3, 3, 3]) + output_size=[3, 3, 3] + ) out_1 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D(output_size=5) out_2 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( - output_size=[2, 3, 5]) + output_size=[2, 3, 5] + ) out_3 = adaptive_max_pool(x=x) # adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( @@ -248,14 +274,16 @@ def test_static_graph(self): # out_4 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( - output_size=[None, 3, None]) + output_size=[None, 3, None] + ) out_5 = adaptive_max_pool(x=x) exe = paddle.static.Executor(place=place) - [res_1, res_2, res_3, - res_5] = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=[out_1, out_2, out_3, out_5]) + [res_1, res_2, res_3, res_5] = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=[out_1, out_2, out_3, out_5], + ) assert np.allclose(res_1, self.res_1_np) @@ -268,21 +296,24 @@ def test_static_graph(self): assert np.allclose(res_5, self.res_5_np) def test_dynamic_graph(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) x = paddle.to_tensor(self.x_np) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( - output_size=[3, 3, 3]) + output_size=[3, 3, 3] + ) out_1 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D(output_size=5) out_2 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( - output_size=[2, 3, 5]) + output_size=[2, 3, 5] + ) out_3 = adaptive_max_pool(x=x) # adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( @@ -290,7 +321,8 @@ def test_dynamic_graph(self): # out_4 = adaptive_max_pool(x=x) adaptive_max_pool = paddle.nn.AdaptiveMaxPool3D( - output_size=[None, 3, None]) + output_size=[None, 3, None] + ) out_5 = adaptive_max_pool(x=x) assert np.allclose(out_1.numpy(), self.res_1_np) @@ -305,14 +337,15 @@ def test_dynamic_graph(self): class TestOutDtype(unittest.TestCase): - def test_max_pool(self): api_fn = F.adaptive_max_pool3d shape = [1, 3, 32, 32, 32] - check_out_dtype(api_fn, - in_specs=[(shape, )], - expect_dtypes=['float32', 'float64'], - output_size=16) + check_out_dtype( + api_fn, + in_specs=[(shape,)], + expect_dtypes=['float32', 'float64'], + output_size=16, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_add_position_encoding_op.py b/python/paddle/fluid/tests/unittests/test_add_position_encoding_op.py index aa510efc59db8..791c2351d7340 100644 --- a/python/paddle/fluid/tests/unittests/test_add_position_encoding_op.py +++ b/python/paddle/fluid/tests/unittests/test_add_position_encoding_op.py @@ -31,13 +31,15 @@ def add_position_encoding(input, alpha=1.0, beta=1.0): for i in range(batch_size): for j in range(max_length): for k in range(half_shape): - val = j / pow( - 10000.0, k * 1.0 / - (half_shape - 1)) if half_shape > 1 else j / 10000.0 - out[i, j, k] = \ - input[i, j, k] * alpha + math.sin(val) * beta - out[i, j, half_shape + k] = \ + val = ( + j / pow(10000.0, k * 1.0 / (half_shape - 1)) + if half_shape > 1 + else j / 10000.0 + ) + out[i, j, k] = input[i, j, k] * alpha + math.sin(val) * beta + out[i, j, half_shape + k] = ( input[i, j, half_shape + k] * alpha + math.cos(val) * beta + ) return out @@ -133,45 +135,49 @@ def init_input_output(self): max_length = self.lod[0][i] for j in range(max_length): for k in range(half_shape): - val = j / pow( - 10000.0, k * 1.0 / - (half_shape - 1)) if half_shape > 1 else j / 10000.0 + val = ( + j / pow(10000.0, k * 1.0 / (half_shape - 1)) + if half_shape > 1 + else j / 10000.0 + ) pos = start + j - self.out[pos, k] = \ + self.out[pos, k] = ( self.x[pos, k] * self.alpha + math.sin(val) * self.beta - self.out[pos, half_shape + k] = \ - self.x[pos, half_shape + k] * self.alpha + math.cos(val) * self.beta + ) + self.out[pos, half_shape + k] = ( + self.x[pos, half_shape + k] * self.alpha + + math.cos(val) * self.beta + ) start += max_length class TestAddPositionEncodingOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.random((4, 16, 8)).astype("float32") def test_Variable(): # the input type must be Variable - fluid.layers.add_position_encoding(input=input_data, - alpha=1.0, - beta=1.0) + fluid.layers.add_position_encoding( + input=input_data, alpha=1.0, beta=1.0 + ) self.assertRaises(TypeError, test_Variable) class TestAddPositionEncodingOpDygraph(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() tensor = np.random.randn(16, 32, 64) position_tensor = paddle.fluid.layers.add_position_encoding( - input=paddle.to_tensor(tensor), alpha=1.0, beta=1.0).numpy() + input=paddle.to_tensor(tensor), alpha=1.0, beta=1.0 + ).numpy() paddle.enable_static() position_tensor_np = add_position_encoding(tensor, 1.0, 1.0) - np.testing.assert_allclose(position_tensor, - position_tensor_np, - rtol=1e-05) + np.testing.assert_allclose( + position_tensor, position_tensor_np, rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_add_reader_dependency.py b/python/paddle/fluid/tests/unittests/test_add_reader_dependency.py index c7479e059b48f..1604445a87357 100644 --- a/python/paddle/fluid/tests/unittests/test_add_reader_dependency.py +++ b/python/paddle/fluid/tests/unittests/test_add_reader_dependency.py @@ -21,15 +21,16 @@ def inplace_add(x, bias): helper = LayerHelper('scale', **locals()) - helper.append_op(type='scale', - inputs={'X': [x]}, - outputs={'Out': [x]}, - attrs={'bias': bias}) + helper.append_op( + type='scale', + inputs={'X': [x]}, + outputs={'Out': [x]}, + attrs={'bias': bias}, + ) return x class TestAddReaderDependency(unittest.TestCase): - def setUp(self): self.batch_num = 3 self.sleep_time = 2 @@ -49,17 +50,19 @@ def run_main(self, place): feed_list=[tmp_in], capacity=16, iterable=False, - use_double_buffer=self.use_double_buffer) + use_double_buffer=self.use_double_buffer, + ) def data_source(): for _ in range(self.batch_num): time.sleep(self.sleep_time) # sleep some times - yield np.random.uniform(low=-1, high=1, - size=[1]).astype('float32'), + yield np.random.uniform( + low=-1, high=1, size=[1] + ).astype('float32'), - persistable_in = fluid.data(name='persistable_in', - dtype='float32', - shape=[1]) + persistable_in = fluid.data( + name='persistable_in', dtype='float32', shape=[1] + ) persistable_in.persistable = True persistable_in = inplace_add(persistable_in, bias=1) @@ -74,31 +77,34 @@ def data_source(): while True: if batch_id == 0: feed = { - persistable_in.name: - np.array([-1]).astype('float32') + persistable_in.name: np.array([-1]).astype( + 'float32' + ) } else: feed = None - ret, = exe.run(prog, - feed=feed, - fetch_list=[persistable_in]) - self.assertEqual(ret.shape, (1, )) + (ret,) = exe.run( + prog, feed=feed, fetch_list=[persistable_in] + ) + self.assertEqual(ret.shape, (1,)) self.assertEqual(ret[0], batch_id) batch_id += 1 except fluid.core.EOFException: loader.reset() self.assertEqual(batch_id, self.batch_num) - t = fluid.global_scope().find_var( - persistable_in.name).get_tensor() + t = ( + fluid.global_scope() + .find_var(persistable_in.name) + .get_tensor() + ) t_val = np.array(t) - self.assertEqual(t_val.shape, (1, )) + self.assertEqual(t_val.shape, (1,)) self.assertEqual(t_val[0] + 1, batch_id) class TestAddReaderDependencyWithoutDoubleBuffer(TestAddReaderDependency): - def setUp(self): self.batch_num = 3 self.sleep_time = 2 diff --git a/python/paddle/fluid/tests/unittests/test_addmm_op.py b/python/paddle/fluid/tests/unittests/test_addmm_op.py index de87233cdb59b..f101f55dc739e 100644 --- a/python/paddle/fluid/tests/unittests/test_addmm_op.py +++ b/python/paddle/fluid/tests/unittests/test_addmm_op.py @@ -33,8 +33,8 @@ def setUp(self): 'Y': np.random.random((10, 20)).astype(self.dtype), } self.outputs = { - 'Out': - self.inputs['Input'] + np.dot(self.inputs['X'], self.inputs['Y']) + 'Out': self.inputs['Input'] + + np.dot(self.inputs['X'], self.inputs['Y']) } def init_dtype_type(self): @@ -62,91 +62,122 @@ def test_errors(self): with program_guard(Program(), Program()): # The input type of addmm_op must be Variable. - input = fluid.create_lod_tensor(np.array([[-1, -1], [-1, -1]]), - [[2]], fluid.CPUPlace()) - x1 = fluid.create_lod_tensor(np.array([[-1, -1], [-1, -1]]), [[2]], - fluid.CPUPlace()) - x2 = fluid.create_lod_tensor(np.array([[-1, -1], [-1, -1]]), [[2]], - fluid.CPUPlace()) + input = fluid.create_lod_tensor( + np.array([[-1, -1], [-1, -1]]), [[2]], fluid.CPUPlace() + ) + x1 = fluid.create_lod_tensor( + np.array([[-1, -1], [-1, -1]]), [[2]], fluid.CPUPlace() + ) + x2 = fluid.create_lod_tensor( + np.array([[-1, -1], [-1, -1]]), [[2]], fluid.CPUPlace() + ) self.assertRaises(TypeError, paddle.addmm, input, x1, x2) # The input dtype of mul_op must be float32 or float64. - input = fluid.layers.data(name='input', - shape=[4, 4], - dtype="int32", - append_batch_size=False) - x3 = fluid.layers.data(name='x3', - shape=[4, 4], - dtype="int32", - append_batch_size=False) - x4 = fluid.layers.data(name='x4', - shape=[4, 4], - dtype="int32", - append_batch_size=False) + input = fluid.layers.data( + name='input', + shape=[4, 4], + dtype="int32", + append_batch_size=False, + ) + x3 = fluid.layers.data( + name='x3', shape=[4, 4], dtype="int32", append_batch_size=False + ) + x4 = fluid.layers.data( + name='x4', shape=[4, 4], dtype="int32", append_batch_size=False + ) self.assertRaises(TypeError, paddle.addmm, input, x3, x4) # x and y dimension mismatch - x5 = fluid.layers.data(name='x5', - shape=[4, 5], - dtype="float32", - append_batch_size=False) - x6 = fluid.layers.data(name='x6', - shape=[4, 4], - dtype="float32", - append_batch_size=False) + x5 = fluid.layers.data( + name='x5', + shape=[4, 5], + dtype="float32", + append_batch_size=False, + ) + x6 = fluid.layers.data( + name='x6', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) self.assertRaises(ValueError, paddle.addmm, input, x5, x6) # input and x are not broadcastable - x7 = fluid.layers.data(name='x7', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - x8 = fluid.layers.data(name='x8', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - input1 = fluid.layers.data(name='input1', - shape=[2, 4], - dtype="float32", - append_batch_size=False) + x7 = fluid.layers.data( + name='x7', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + x8 = fluid.layers.data( + name='x8', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + input1 = fluid.layers.data( + name='input1', + shape=[2, 4], + dtype="float32", + append_batch_size=False, + ) self.assertRaises(ValueError, paddle.addmm, input1, x7, x8) # input and x are not broadcastable - x9 = fluid.layers.data(name='x9', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - x10 = fluid.layers.data(name='x10', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - input2 = fluid.layers.data(name='input2', - shape=[1, 2], - dtype="float32", - append_batch_size=False) + x9 = fluid.layers.data( + name='x9', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + x10 = fluid.layers.data( + name='x10', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + input2 = fluid.layers.data( + name='input2', + shape=[1, 2], + dtype="float32", + append_batch_size=False, + ) self.assertRaises(ValueError, paddle.addmm, input2, x9, x10) - x11 = fluid.layers.data(name='x11', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - x12 = fluid.layers.data(name='x12', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - input3 = fluid.layers.data(name='input3', - shape=[4, 2], - dtype="float32", - append_batch_size=False) + x11 = fluid.layers.data( + name='x11', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + x12 = fluid.layers.data( + name='x12', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + input3 = fluid.layers.data( + name='input3', + shape=[4, 2], + dtype="float32", + append_batch_size=False, + ) self.assertRaises(ValueError, paddle.addmm, input3, x11, x12) - x13 = fluid.layers.data(name='x13', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - x14 = fluid.layers.data(name='x14', - shape=[4, 4], - dtype="float32", - append_batch_size=False) - input4 = fluid.layers.data(name='input4', - shape=[3, 1], - dtype="float32", - append_batch_size=False) + x13 = fluid.layers.data( + name='x13', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + x14 = fluid.layers.data( + name='x14', + shape=[4, 4], + dtype="float32", + append_batch_size=False, + ) + input4 = fluid.layers.data( + name='input4', + shape=[3, 1], + dtype="float32", + append_batch_size=False, + ) self.assertRaises(ValueError, paddle.addmm, input4, x13, x14) @@ -166,8 +197,10 @@ def setUp(self): 'Alpha': 0.1, 'Beta': 1.0, } - self.outputs = {'Out': self.attrs['Beta'] * self.inputs['Input'] + \ - self.attrs['Alpha'] * np.dot(self.inputs['X'], self.inputs['Y'])} + self.outputs = { + 'Out': self.attrs['Beta'] * self.inputs['Input'] + + self.attrs['Alpha'] * np.dot(self.inputs['X'], self.inputs['Y']) + } class TestAddMMOp3(OpTest): @@ -185,8 +218,10 @@ def setUp(self): 'Alpha': 0.5, 'Beta': 2.0, } - self.outputs = {'Out': self.attrs['Beta'] * self.inputs['Input'] + \ - self.attrs['Alpha'] * np.dot(self.inputs['X'], self.inputs['Y'])} + self.outputs = { + 'Out': self.attrs['Beta'] * self.inputs['Input'] + + self.attrs['Alpha'] * np.dot(self.inputs['X'], self.inputs['Y']) + } def init_dtype_type(self): pass @@ -222,8 +257,10 @@ def setUp(self): 'Alpha': 0.5, 'Beta': 2.0, } - self.outputs = {'Out': self.attrs['Beta'] * self.inputs['Input'] + \ - self.attrs['Alpha'] * np.dot(self.inputs['X'], self.inputs['Y'])} + self.outputs = { + 'Out': self.attrs['Beta'] * self.inputs['Input'] + + self.attrs['Alpha'] * np.dot(self.inputs['X'], self.inputs['Y']) + } def init_dtype_type(self): pass @@ -245,7 +282,6 @@ def test_check_grad_input(self): class TestAddMMOp5(unittest.TestCase): - def test_api_with_dygraph(self): np_input = np.random.random((20, 30)).astype(np.float32) np_x = np.random.random((20, 6)).astype(np.float32) @@ -260,7 +296,6 @@ def test_api_with_dygraph(self): class TestAddMMAPI(unittest.TestCase): - def test_api_error(self): data_x = np.ones((2, 2)).astype(np.float32) data_y = np.ones((2, 2)).astype(np.float32) @@ -273,11 +308,9 @@ def test_error1(): x = paddle.to_tensor(data_x_wrong) y = paddle.to_tensor(data_y) input = paddle.to_tensor(data_input) - out = paddle.tensor.addmm(input=input, - x=x, - y=y, - beta=0.5, - alpha=5.0) + out = paddle.tensor.addmm( + input=input, x=x, y=y, beta=0.5, alpha=5.0 + ) self.assertRaises(ValueError, test_error1) @@ -286,11 +319,9 @@ def test_error2(): x = paddle.to_tensor(data_x_wrong) y = paddle.to_tensor(data_y) input = paddle.to_tensor(data_input) - out = paddle.tensor.addmm(input=input, - x=x, - y=y, - beta=0.5, - alpha=5.0) + out = paddle.tensor.addmm( + input=input, x=x, y=y, beta=0.5, alpha=5.0 + ) self.assertRaises(ValueError, test_error2) @@ -299,11 +330,9 @@ def test_error3(): x = paddle.to_tensor(data_x) y = paddle.to_tensor(data_y) input = paddle.to_tensor(data_input_wrong) - out = paddle.tensor.addmm(input=input, - x=x, - y=y, - beta=0.5, - alpha=5.0) + out = paddle.tensor.addmm( + input=input, x=x, y=y, beta=0.5, alpha=5.0 + ) self.assertRaises(ValueError, test_error3) @@ -312,11 +341,9 @@ def test_error4(): x = paddle.to_tensor(data_x) y = paddle.to_tensor(data_y) input = paddle.to_tensor(data_input_wrong) - out = paddle.tensor.addmm(input=input, - x=x, - y=y, - beta=0.5, - alpha=5.0) + out = paddle.tensor.addmm( + input=input, x=x, y=y, beta=0.5, alpha=5.0 + ) self.assertRaises(ValueError, test_error4) @@ -334,17 +361,16 @@ def test_api_normal_1(self): x = paddle.to_tensor(data_x) y = paddle.to_tensor(data_y) input = paddle.to_tensor(data_input) - paddle_output = paddle.tensor.addmm(input=input, - x=x, - y=y, - beta=data_beta, - alpha=data_alpha) + paddle_output = paddle.tensor.addmm( + input=input, x=x, y=y, beta=data_beta, alpha=data_alpha + ) numpy_output = data_beta * data_input + data_alpha * np.dot( - data_x, data_y) + data_x, data_y + ) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) paddle.enable_static() @@ -360,17 +386,16 @@ def test_api_normal_2(self): x = paddle.to_tensor(data_x) y = paddle.to_tensor(data_y) input = paddle.to_tensor(data_input) - paddle_output = paddle.tensor.addmm(input=input, - x=x, - y=y, - beta=data_beta, - alpha=data_alpha) + paddle_output = paddle.tensor.addmm( + input=input, x=x, y=y, beta=data_beta, alpha=data_alpha + ) numpy_output = data_beta * data_input + data_alpha * np.dot( - data_x, data_y) + data_x, data_y + ) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) paddle.enable_static() @@ -386,17 +411,16 @@ def test_api_normal_3(self): x = paddle.to_tensor(data_x) y = paddle.to_tensor(data_y) input = paddle.to_tensor(data_input) - paddle_output = paddle.tensor.addmm(input=input, - x=x, - y=y, - beta=data_beta, - alpha=data_alpha) + paddle_output = paddle.tensor.addmm( + input=input, x=x, y=y, beta=data_beta, alpha=data_alpha + ) numpy_output = data_beta * data_input + data_alpha * np.dot( - data_x, data_y) + data_x, data_y + ) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_affine_channel_op.py b/python/paddle/fluid/tests/unittests/test_affine_channel_op.py index 4bd9100225604..f09e5be577506 100644 --- a/python/paddle/fluid/tests/unittests/test_affine_channel_op.py +++ b/python/paddle/fluid/tests/unittests/test_affine_channel_op.py @@ -33,7 +33,6 @@ def affine_channel(x, scale, bias, layout): class TestAffineChannelOp(OpTest): - def setUp(self): self.op_type = "affine_channel" self.init_test_case() @@ -67,7 +66,6 @@ def init_test_case(self): class TestAffineChannelOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program()): @@ -78,32 +76,31 @@ def test_x_type(): self.assertRaises(TypeError, test_x_type) def test_x_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[None, 1, 2, 2], - dtype='int32') + x2 = fluid.layers.data( + name='x2', shape=[None, 1, 2, 2], dtype='int32' + ) fluid.layers.affine_channel(x2) self.assertRaises(TypeError, test_x_dtype) def test_scale_type(): - x3 = fluid.layers.data(name='x3', - shape=[None, 1, 2, 2], - dtype='float32') + x3 = fluid.layers.data( + name='x3', shape=[None, 1, 2, 2], dtype='float32' + ) fluid.layers.affine_channel(x3, scale=1) self.assertRaises(TypeError, test_scale_type) def test_bias_type(): - x4 = fluid.layers.data(name='x4', - shape=[None, 1, 2, 2], - dtype='float32') + x4 = fluid.layers.data( + name='x4', shape=[None, 1, 2, 2], dtype='float32' + ) fluid.layers.affine_channel(x4, bias=1) self.assertRaises(TypeError, test_bias_type) class TestAffineChannelNHWC(TestAffineChannelOp): - def init_test_case(self): self.shape = [2, 3, 3, 100] self.C = 100 @@ -117,7 +114,6 @@ def test_check_grad_stopgrad_dscale_dbias(self): class TestAffineChannel2D(TestAffineChannelOp): - def init_test_case(self): self.shape = [2, 100] self.C = 100 @@ -131,7 +127,7 @@ def test_check_grad_stopgrad_dscale_dbias(self): # TODO(qingqing): disable unit testing for large shape -#class TestAffineChannelNCHWLargeShape(TestAffineChannelOp): +# class TestAffineChannelNCHWLargeShape(TestAffineChannelOp): # def init_test_case(self): # self.shape = [4, 128, 112, 112] # self.C = 128 @@ -147,7 +143,7 @@ def test_check_grad_stopgrad_dscale_dbias(self): # def test_check_grad_stopgrad_dscale_dbias(self): # pass -#class TestAffineChannelNHWCLargeShape(TestAffineChannelNCHWLargeShape): +# class TestAffineChannelNHWCLargeShape(TestAffineChannelNCHWLargeShape): # def init_test_case(self): # self.shape = [64, 32, 32, 128] # self.C = 128 diff --git a/python/paddle/fluid/tests/unittests/test_affine_grid_function.py b/python/paddle/fluid/tests/unittests/test_affine_grid_function.py index 9091a1c9b6ddd..cd1cecfb618ca 100644 --- a/python/paddle/fluid/tests/unittests/test_affine_grid_function.py +++ b/python/paddle/fluid/tests/unittests/test_affine_grid_function.py @@ -21,15 +21,16 @@ class AffineGridTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - theta_shape=(20, 2, 3), - output_shape=[20, 2, 5, 7], - align_corners=True, - dtype="float32", - invalid_theta=False, - variable_output_shape=False): + def __init__( + self, + methodName='runTest', + theta_shape=(20, 2, 3), + output_shape=[20, 2, 5, 7], + align_corners=True, + dtype="float32", + invalid_theta=False, + variable_output_shape=False, + ): super(AffineGridTestCase, self).__init__(methodName) self.theta_shape = theta_shape @@ -48,14 +49,14 @@ def fluid_layer(self, place): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - theta_var = fluid.data("input", - self.theta_shape, - dtype=self.dtype) + theta_var = fluid.data( + "input", self.theta_shape, dtype=self.dtype + ) y_var = fluid.layers.affine_grid(theta_var, self.output_shape) feed_dict = {"input": self.theta} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def functional(self, place): @@ -64,28 +65,33 @@ def functional(self, place): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - theta_var = fluid.data("input", - self.theta_shape, - dtype=self.dtype) - y_var = F.affine_grid(theta_var, - self.output_shape, - align_corners=self.align_corners) + theta_var = fluid.data( + "input", self.theta_shape, dtype=self.dtype + ) + y_var = F.affine_grid( + theta_var, + self.output_shape, + align_corners=self.align_corners, + ) feed_dict = {"input": self.theta} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_dygraph_layer(self): paddle.disable_static() - theta_var = dg.to_variable( - self.theta) if not self.invalid_theta else "invalid" - output_shape = dg.to_variable( - self.output_shape - ) if self.variable_output_shape else self.output_shape - y_var = F.affine_grid(theta_var, - output_shape, - align_corners=self.align_corners) + theta_var = ( + dg.to_variable(self.theta) if not self.invalid_theta else "invalid" + ) + output_shape = ( + dg.to_variable(self.output_shape) + if self.variable_output_shape + else self.output_shape + ) + y_var = F.affine_grid( + theta_var, output_shape, align_corners=self.align_corners + ) y_np = y_var.numpy() return y_np @@ -108,7 +114,6 @@ def runTest(self): class AffineGridErrorTestCase(AffineGridTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -122,22 +127,26 @@ def add_cases(suite): suite.addTest(AffineGridTestCase(methodName='runTest', align_corners=False)) suite.addTest( - AffineGridTestCase(methodName='runTest', variable_output_shape=True)) + AffineGridTestCase(methodName='runTest', variable_output_shape=True) + ) suite.addTest( - AffineGridTestCase(methodName='runTest', - theta_shape=(20, 2, 3), - output_shape=[20, 1, 7, 7], - align_corners=True)) + AffineGridTestCase( + methodName='runTest', + theta_shape=(20, 2, 3), + output_shape=[20, 1, 7, 7], + align_corners=True, + ) + ) def add_error_cases(suite): suite.addTest( - AffineGridErrorTestCase(methodName='runTest', output_shape="not_valid")) + AffineGridErrorTestCase(methodName='runTest', output_shape="not_valid") + ) suite.addTest( - AffineGridErrorTestCase( - methodName='runTest', - invalid_theta=True)) # to test theta not variable error checking + AffineGridErrorTestCase(methodName='runTest', invalid_theta=True) + ) # to test theta not variable error checking def load_tests(loader, standard_tests, pattern): diff --git a/python/paddle/fluid/tests/unittests/test_affine_grid_op.py b/python/paddle/fluid/tests/unittests/test_affine_grid_op.py index f8c0dedc18423..096b4fe12777d 100644 --- a/python/paddle/fluid/tests/unittests/test_affine_grid_op.py +++ b/python/paddle/fluid/tests/unittests/test_affine_grid_op.py @@ -26,12 +26,21 @@ def AffineGrid4D(theta, size, align_corners): if not align_corners: h_factor = (h - 1) / float(h) w_factor = (w - 1) / float(w) - h_idx = np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, - axis=0).T[:, :, np.newaxis] * h_factor - w_idx = np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, - axis=0)[:, :, np.newaxis] * w_factor - grid = np.concatenate([w_idx, h_idx, np.ones([h, w, 1])], - axis=2) # h * w * 3 + h_idx = ( + np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, axis=0).T[ + :, :, np.newaxis + ] + * h_factor + ) + w_idx = ( + np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, axis=0)[ + :, :, np.newaxis + ] + * w_factor + ) + grid = np.concatenate( + [w_idx, h_idx, np.ones([h, w, 1])], axis=2 + ) # h * w * 3 grid = np.repeat(grid[np.newaxis, :], size[0], axis=0) # n * h * w *3 ret = np.zeros([n, h * w, 2]) @@ -51,20 +60,39 @@ def AffineGrid5D(theta, size, align_corners): d_factor = (d - 1) / float(d) h_factor = (h - 1) / float(h) w_factor = (w - 1) / float(w) - d_idx = np.repeat(np.repeat( - np.linspace(-1, 1, d)[:, np.newaxis, np.newaxis], h, axis=1), - w, - axis=2)[:, :, :, np.newaxis] * d_factor - h_idx = np.repeat(np.repeat( - np.linspace(-1, 1, h)[np.newaxis, :, np.newaxis], w, axis=2), - d, - axis=0)[:, :, :, np.newaxis] * h_factor - w_idx = np.repeat(np.repeat( - np.linspace(-1, 1, w)[np.newaxis, np.newaxis, :], h, axis=1), - d, - axis=0)[:, :, :, np.newaxis] * w_factor + d_idx = ( + np.repeat( + np.repeat( + np.linspace(-1, 1, d)[:, np.newaxis, np.newaxis], h, axis=1 + ), + w, + axis=2, + )[:, :, :, np.newaxis] + * d_factor + ) + h_idx = ( + np.repeat( + np.repeat( + np.linspace(-1, 1, h)[np.newaxis, :, np.newaxis], w, axis=2 + ), + d, + axis=0, + )[:, :, :, np.newaxis] + * h_factor + ) + w_idx = ( + np.repeat( + np.repeat( + np.linspace(-1, 1, w)[np.newaxis, np.newaxis, :], h, axis=1 + ), + d, + axis=0, + )[:, :, :, np.newaxis] + * w_factor + ) grid = np.concatenate( - [w_idx, h_idx, d_idx, np.ones([d, h, w, 1])], axis=3) # d * h * w * 4 + [w_idx, h_idx, d_idx, np.ones([d, h, w, 1])], axis=3 + ) # d * h * w * 4 grid = np.repeat(grid[np.newaxis, :], size[0], axis=0) # n * d * h * w * 4 ret = np.zeros([n, d * h * w, 3]) @@ -75,7 +103,6 @@ def AffineGrid5D(theta, size, align_corners): class TestAffineGridOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = "affine_grid" @@ -84,31 +111,32 @@ def setUp(self): self.inputs = {'Theta': theta} self.attrs = { "use_cudnn": self.use_cudnn, - "align_corners": self.align_corners + "align_corners": self.align_corners, } if self.dynamic_shape: self.inputs['OutputShape'] = self.output_shape else: self.attrs['output_shape'] = self.output_shape - if (self.theta_shape[1] == 2 and self.theta_shape[2] == 3): + if self.theta_shape[1] == 2 and self.theta_shape[2] == 3: self.outputs = { - 'Output': AffineGrid4D(theta, self.output_shape, - self.align_corners) + 'Output': AffineGrid4D( + theta, self.output_shape, self.align_corners + ) } else: self.outputs = { - 'Output': AffineGrid5D(theta, self.output_shape, - self.align_corners) + 'Output': AffineGrid5D( + theta, self.output_shape, self.align_corners + ) } def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['Theta'], - 'Output', - no_grad_set=['OutputShape'], - check_eager=True) + self.check_grad( + ['Theta'], 'Output', no_grad_set=['OutputShape'], check_eager=True + ) def initTestCase(self): self.theta_shape = (17, 2, 3) @@ -119,19 +147,19 @@ def initTestCase(self): class TestAffineGridOpCase1(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (20, 2, 3) self.output_shape = np.array([20, 2, 5, 7]).astype("int32") self.dynamic_shape = True self.use_cudnn = True if paddle.fluid.core.is_compiled_with_rocm(): - self.use_cudnn = False # ROCM platform do not have MIOPEN kernel for affine_grid + self.use_cudnn = ( + False # ROCM platform do not have MIOPEN kernel for affine_grid + ) self.align_corners = True class TestAffineGridOpCase2(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (20, 2, 3) self.output_shape = np.array([20, 2, 5, 7]).astype("int32") @@ -141,7 +169,6 @@ def initTestCase(self): class TestAffineGridOpCase3(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (20, 2, 3) self.output_shape = np.array([20, 2, 5, 7]).astype("int32") @@ -151,7 +178,6 @@ def initTestCase(self): class TestAffineGridOpCase4(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (25, 2, 3) self.output_shape = np.array([25, 2, 5, 6]).astype("int32") @@ -161,7 +187,6 @@ def initTestCase(self): class TestAffineGridOp5DCase1(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (20, 3, 4) self.output_shape = np.array([20, 1, 2, 5, 7]).astype("int32") @@ -171,7 +196,6 @@ def initTestCase(self): class TestAffineGridOp5DCase2(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (20, 3, 4) self.output_shape = np.array([20, 1, 2, 5, 7]).astype("int32") @@ -181,7 +205,6 @@ def initTestCase(self): class TestAffineGridOp5DCase3(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (20, 3, 4) self.output_shape = np.array([20, 1, 2, 5, 7]).astype("int32") @@ -191,7 +214,6 @@ def initTestCase(self): class TestAffineGridOp5DCase4(TestAffineGridOp): - def initTestCase(self): self.theta_shape = (25, 3, 4) self.output_shape = np.array([25, 1, 2, 5, 6]).astype("int32") diff --git a/python/paddle/fluid/tests/unittests/test_allclose_layer.py b/python/paddle/fluid/tests/unittests/test_allclose_layer.py index 66afbcfe20947..c406241c65be1 100644 --- a/python/paddle/fluid/tests/unittests/test_allclose_layer.py +++ b/python/paddle/fluid/tests/unittests/test_allclose_layer.py @@ -20,68 +20,53 @@ class TestAllcloseLayer(unittest.TestCase): - def allclose_check(self, use_cuda, dtype='float32'): a = fluid.data(name="a", shape=[2], dtype=dtype) b = fluid.data(name="b", shape=[2], dtype=dtype) - result = paddle.allclose(a, - b, - rtol=1e-05, - atol=1e-08, - equal_nan=False, - name="ignore_nan") - result_nan = paddle.allclose(a, - b, - rtol=1e-05, - atol=1e-08, - equal_nan=True, - name="equal_nan") - result_corner = paddle.allclose(a, - b, - rtol=0.01, - atol=0.0, - name="corner_case") + result = paddle.allclose( + a, b, rtol=1e-05, atol=1e-08, equal_nan=False, name="ignore_nan" + ) + result_nan = paddle.allclose( + a, b, rtol=1e-05, atol=1e-08, equal_nan=True, name="equal_nan" + ) + result_corner = paddle.allclose( + a, b, rtol=0.01, atol=0.0, name="corner_case" + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - x = np.array([10000., 1e-07]).astype(dtype) + x = np.array([10000.0, 1e-07]).astype(dtype) y = np.array([10000.1, 1e-08]).astype(dtype) - result_v, result_nan_v = exe.run(feed={ - 'a': x, - 'b': y - }, - fetch_list=[result, result_nan]) + result_v, result_nan_v = exe.run( + feed={'a': x, 'b': y}, fetch_list=[result, result_nan] + ) self.assertEqual(result_v[0], False) self.assertEqual(result_nan_v[0], False) - x = np.array([10000., 1e-08]).astype(dtype) + x = np.array([10000.0, 1e-08]).astype(dtype) y = np.array([10000.1, 1e-09]).astype(dtype) - result_v, result_nan_v = exe.run(feed={ - 'a': x, - 'b': y - }, - fetch_list=[result, result_nan]) + result_v, result_nan_v = exe.run( + feed={'a': x, 'b': y}, fetch_list=[result, result_nan] + ) self.assertEqual(result_v[0], True) self.assertEqual(result_nan_v[0], True) x = np.array([1.0, float('nan')]).astype(dtype) y = np.array([1.0, float('nan')]).astype(dtype) - result_v, result_nan_v = exe.run(feed={ - 'a': x, - 'b': y - }, - fetch_list=[result, result_nan]) + result_v, result_nan_v = exe.run( + feed={'a': x, 'b': y}, fetch_list=[result, result_nan] + ) self.assertEqual(result_v[0], False) self.assertEqual(result_nan_v[0], True) # for corner case x = np.array([10.1, 10.1]).astype(dtype) y = np.array([10, 10]).astype(dtype) - result_c, = exe.run(feed={'a': x, 'b': y}, fetch_list=[result_corner]) - corner_res = (dtype == 'float64') + (result_c,) = exe.run(feed={'a': x, 'b': y}, fetch_list=[result_corner]) + corner_res = dtype == 'float64' self.assertEqual(result_c[0], corner_res) def test_allclose_cpu_fp32(self): @@ -115,9 +100,9 @@ def test_allclose_gpu_fp64(self): self.allclose_check(use_cuda=True, dtype='float64') def func_dygraph_mode(self): - x_1 = np.array([10000., 1e-07]).astype("float32") + x_1 = np.array([10000.0, 1e-07]).astype("float32") y_1 = np.array([10000.1, 1e-08]).astype("float32") - x_2 = np.array([10000., 1e-08]).astype("float32") + x_2 = np.array([10000.0, 1e-08]).astype("float32") y_2 = np.array([10000.1, 1e-09]).astype("float32") x_3 = np.array([1.0, float('nan')]).astype("float32") y_3 = np.array([1.0, float('nan')]).astype("float32") @@ -129,68 +114,76 @@ def func_dygraph_mode(self): with fluid.dygraph.guard(): x_v_1 = paddle.to_tensor(x_1) y_v_1 = paddle.to_tensor(y_1) - ret_1 = paddle.allclose(x_v_1, - y_v_1, - rtol=1e-05, - atol=1e-08, - equal_nan=False, - name='test_1') + ret_1 = paddle.allclose( + x_v_1, + y_v_1, + rtol=1e-05, + atol=1e-08, + equal_nan=False, + name='test_1', + ) self.assertEqual(ret_1.numpy()[0], False) - ret_1 = paddle.allclose(x_v_1, - y_v_1, - rtol=1e-05, - atol=1e-08, - equal_nan=True, - name='test_2') + ret_1 = paddle.allclose( + x_v_1, + y_v_1, + rtol=1e-05, + atol=1e-08, + equal_nan=True, + name='test_2', + ) self.assertEqual(ret_1.numpy()[0], False) x_v_2 = paddle.to_tensor(x_2) y_v_2 = paddle.to_tensor(y_2) - ret_2 = paddle.allclose(x_v_2, - y_v_2, - rtol=1e-05, - atol=1e-08, - equal_nan=False, - name='test_3') + ret_2 = paddle.allclose( + x_v_2, + y_v_2, + rtol=1e-05, + atol=1e-08, + equal_nan=False, + name='test_3', + ) self.assertEqual(ret_2.numpy()[0], True) - ret_2 = paddle.allclose(x_v_2, - y_v_2, - rtol=1e-05, - atol=1e-08, - equal_nan=True, - name='test_4') + ret_2 = paddle.allclose( + x_v_2, + y_v_2, + rtol=1e-05, + atol=1e-08, + equal_nan=True, + name='test_4', + ) self.assertEqual(ret_2.numpy()[0], True) x_v_3 = paddle.to_tensor(x_3) y_v_3 = paddle.to_tensor(y_3) - ret_3 = paddle.allclose(x_v_3, - y_v_3, - rtol=1e-05, - atol=1e-08, - equal_nan=False, - name='test_5') + ret_3 = paddle.allclose( + x_v_3, + y_v_3, + rtol=1e-05, + atol=1e-08, + equal_nan=False, + name='test_5', + ) self.assertEqual(ret_3.numpy()[0], False) - ret_3 = paddle.allclose(x_v_3, - y_v_3, - rtol=1e-05, - atol=1e-08, - equal_nan=True, - name='test_6') + ret_3 = paddle.allclose( + x_v_3, + y_v_3, + rtol=1e-05, + atol=1e-08, + equal_nan=True, + name='test_6', + ) self.assertEqual(ret_3.numpy()[0], True) # for corner case x_v_4 = paddle.to_tensor(x_4) y_v_4 = paddle.to_tensor(y_4) - ret_4 = paddle.allclose(x_v_4, - y_v_4, - rtol=0.01, - atol=0.0, - name='test_7') + ret_4 = paddle.allclose( + x_v_4, y_v_4, rtol=0.01, atol=0.0, name='test_7' + ) self.assertEqual(ret_4.numpy()[0], False) x_v_5 = paddle.to_tensor(x_5) y_v_5 = paddle.to_tensor(y_5) - ret_5 = paddle.allclose(x_v_5, - y_v_5, - rtol=0.015, - atol=0.0, - name='test_8') + ret_5 = paddle.allclose( + x_v_5, y_v_5, rtol=0.015, atol=0.0, name='test_8' + ) self.assertEqual(ret_5.numpy()[0], True) def test_dygraph_mode(self): diff --git a/python/paddle/fluid/tests/unittests/test_allclose_op.py b/python/paddle/fluid/tests/unittests/test_allclose_op.py index 26351abe802dc..0eb47aa4441b9 100644 --- a/python/paddle/fluid/tests/unittests/test_allclose_op.py +++ b/python/paddle/fluid/tests/unittests/test_allclose_op.py @@ -19,9 +19,8 @@ class TestAllcloseOp(OpTest): - def set_args(self): - self.input = np.array([10000., 1e-07]).astype("float32") + self.input = np.array([10000.0, 1e-07]).astype("float32") self.other = np.array([10000.1, 1e-08]).astype("float32") self.rtol = np.array([1e-05]).astype("float64") self.atol = np.array([1e-08]).astype("float64") @@ -35,18 +34,21 @@ def setUp(self): 'Input': self.input, 'Other': self.other, "Rtol": self.rtol, - "Atol": self.atol + "Atol": self.atol, } self.attrs = {'equal_nan': self.equal_nan} self.outputs = { - 'Out': - np.array([ - np.allclose(self.inputs['Input'], - self.inputs['Other'], - rtol=self.rtol, - atol=self.atol, - equal_nan=self.equal_nan) - ]) + 'Out': np.array( + [ + np.allclose( + self.inputs['Input'], + self.inputs['Other'], + rtol=self.rtol, + atol=self.atol, + equal_nan=self.equal_nan, + ) + ] + ) } def test_check_output(self): @@ -54,9 +56,7 @@ def test_check_output(self): class TestAllcloseOpException(TestAllcloseOp): - def test_check_output(self): - def test_rtol_num(): self.inputs['Rtol'] = np.array([1e-05, 1e-05]).astype("float64") self.inputs['Atol'] = np.array([1e-08]).astype("float64") @@ -87,9 +87,8 @@ def test_atol_type(): class TestAllcloseOpSmallNum(TestAllcloseOp): - def set_args(self): - self.input = np.array([10000., 1e-08]).astype("float32") + self.input = np.array([10000.0, 1e-08]).astype("float32") self.other = np.array([10000.1, 1e-09]).astype("float32") self.rtol = np.array([1e-05]).astype("float64") self.atol = np.array([1e-08]).astype("float64") @@ -97,7 +96,6 @@ def set_args(self): class TestAllcloseOpNanFalse(TestAllcloseOp): - def set_args(self): self.input = np.array([1.0, float('nan')]).astype("float32") self.other = np.array([1.0, float('nan')]).astype("float32") @@ -107,7 +105,6 @@ def set_args(self): class TestAllcloseOpNanTrue(TestAllcloseOp): - def set_args(self): self.input = np.array([1.0, float('nan')]).astype("float32") self.other = np.array([1.0, float('nan')]).astype("float32") @@ -117,7 +114,6 @@ def set_args(self): class TestAllcloseDygraph(unittest.TestCase): - def test_api_case(self): paddle.disable_static() x_data = np.random.rand(10, 10) @@ -131,12 +127,11 @@ def test_api_case(self): class TestAllcloseError(unittest.TestCase): - def test_input_dtype(self): - def test_x_dtype(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float16') y = paddle.fluid.data(name='y', shape=[10, 10], dtype='float64') result = paddle.allclose(x, y) @@ -144,8 +139,9 @@ def test_x_dtype(): self.assertRaises(TypeError, test_x_dtype) def test_y_dtype(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float64') y = paddle.fluid.data(name='y', shape=[10, 10], dtype='int32') result = paddle.allclose(x, y) @@ -173,7 +169,6 @@ def test_equal_nan(): class TestAllcloseOpFloat32(TestAllcloseOp): - def set_args(self): self.input = np.array([10.1]).astype("float32") self.other = np.array([10]).astype("float32") @@ -183,7 +178,6 @@ def set_args(self): class TestAllcloseOpFloat64(TestAllcloseOp): - def set_args(self): self.input = np.array([10.1]).astype("float64") self.other = np.array([10]).astype("float64") @@ -193,7 +187,6 @@ def set_args(self): class TestAllcloseOpLargeDimInput(TestAllcloseOp): - def set_args(self): self.input = np.array(np.zeros([2048, 1024])).astype("float64") self.other = np.array(np.zeros([2048, 1024])).astype("float64") diff --git a/python/paddle/fluid/tests/unittests/test_allgather.py b/python/paddle/fluid/tests/unittests/test_allgather.py index 90da7128dcf43..89441f394cff7 100644 --- a/python/paddle/fluid/tests/unittests/test_allgather.py +++ b/python/paddle/fluid/tests/unittests/test_allgather.py @@ -21,7 +21,6 @@ class TestAllGatherOp(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/test_amp_check_finite_and_scale_op.py b/python/paddle/fluid/tests/unittests/test_amp_check_finite_and_scale_op.py index b44c294f3acb6..7101d3439f642 100644 --- a/python/paddle/fluid/tests/unittests/test_amp_check_finite_and_scale_op.py +++ b/python/paddle/fluid/tests/unittests/test_amp_check_finite_and_scale_op.py @@ -24,7 +24,6 @@ def check_finite_and_unscale_wrapper(x, scale): class TestCheckFiniteAndUnscaleOp(OpTest): - def setUp(self): self.op_type = "check_finite_and_unscale" self.python_api = check_finite_and_unscale_wrapper @@ -47,7 +46,6 @@ def test_check_output(self): class TestCheckFiniteAndUnscaleOpWithNan(OpTest): - def setUp(self): self.op_type = "check_finite_and_unscale" self.init_dtype() @@ -73,7 +71,6 @@ def test_check_output(self): class TestCheckFiniteAndUnscaleOpWithInf(OpTest): - def setUp(self): self.op_type = "check_finite_and_unscale" self.init_dtype() diff --git a/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py b/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py index b5ce42ee7582c..3766dc95ab93e 100644 --- a/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py +++ b/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py @@ -17,8 +17,9 @@ from op_test import OpTest -def anchor_generator_in_python(input_feat, anchor_sizes, aspect_ratios, - variances, stride, offset): +def anchor_generator_in_python( + input_feat, anchor_sizes, aspect_ratios, variances, stride, offset +): num_anchors = len(aspect_ratios) * len(anchor_sizes) layer_h = input_feat.shape[2] layer_w = input_feat.shape[3] @@ -42,11 +43,12 @@ def anchor_generator_in_python(input_feat, anchor_sizes, aspect_ratios, scale_h = anchor_size / stride[1] w = scale_w * base_w h = scale_h * base_h - out_anchors[h_idx, w_idx, - idx, :] = [(x_ctr - 0.5 * (w - 1)), - (y_ctr - 0.5 * (h - 1)), - (x_ctr + 0.5 * (w - 1)), - (y_ctr + 0.5 * (h - 1))] + out_anchors[h_idx, w_idx, idx, :] = [ + (x_ctr - 0.5 * (w - 1)), + (y_ctr - 0.5 * (h - 1)), + (x_ctr + 0.5 * (w - 1)), + (y_ctr + 0.5 * (h - 1)), + ] idx += 1 # set the variance. @@ -57,7 +59,6 @@ def anchor_generator_in_python(input_feat, anchor_sizes, aspect_ratios, class TestAnchorGeneratorOp(OpTest): - def set_data(self): self.init_test_params() self.init_test_input() @@ -87,9 +88,9 @@ def init_test_params(self): self.layer_h = 2 self.layer_w = 2 - self.anchor_sizes = [64., 128., 256., 512.] - self.aspect_ratios = [0.5, 1., 2.] - self.stride = [16., 16.] + self.anchor_sizes = [64.0, 128.0, 256.0, 512.0] + self.aspect_ratios = [0.5, 1.0, 2.0] + self.stride = [16.0, 16.0] self.offset = 0.5 @@ -97,13 +98,18 @@ def init_test_params(self): def init_test_input(self): self.input = np.random.random( - (self.batch_size, self.input_channels, self.layer_h, - self.layer_w)).astype('float32') + (self.batch_size, self.input_channels, self.layer_h, self.layer_w) + ).astype('float32') def init_test_output(self): self.out_anchors, self.out_var = anchor_generator_in_python( - self.input, self.anchor_sizes, self.aspect_ratios, self.variances, - self.stride, self.offset) + self.input, + self.anchor_sizes, + self.aspect_ratios, + self.variances, + self.stride, + self.offset, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_angle_op.py b/python/paddle/fluid/tests/unittests/test_angle_op.py index 9d4282150105c..767a274ad3224 100644 --- a/python/paddle/fluid/tests/unittests/test_angle_op.py +++ b/python/paddle/fluid/tests/unittests/test_angle_op.py @@ -29,7 +29,7 @@ def angle_grad(x, dout): def angle_grad_element(xi, douti): if xi == 0: return 0 - rsquare = np.abs(xi)**2 + rsquare = np.abs(xi) ** 2 return -douti * xi.imag / rsquare + 1j * douti * xi.real / rsquare return np.vectorize(angle_grad_element)(x, dout) @@ -38,7 +38,6 @@ def angle_grad_element(xi, douti): class TestAngleOpFloat(OpTest): - def setUp(self): self.op_type = "angle" self.python_api = paddle.angle @@ -52,17 +51,17 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[ - angle_grad(self.x, - np.ones_like(self.x) / self.x.size) - ], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[ + angle_grad(self.x, np.ones_like(self.x) / self.x.size) + ], + check_eager=True, + ) class TestAngleOpComplex(OpTest): - def setUp(self): self.op_type = "angle" self.python_api = paddle.angle @@ -78,17 +77,17 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[ - angle_grad(self.x, - np.ones_like(self.x) / self.x.size) - ], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[ + angle_grad(self.x, np.ones_like(self.x) / self.x.size) + ], + check_eager=True, + ) class TestAngleAPI(unittest.TestCase): - def setUp(self): self.x = np.random.randn(2, 3) + 1j * np.random.randn(2, 3) self.out = np.angle(self.x) diff --git a/python/paddle/fluid/tests/unittests/test_apply_pass_to_program.py b/python/paddle/fluid/tests/unittests/test_apply_pass_to_program.py index 54a83d2a5ec75..fff492e917673 100644 --- a/python/paddle/fluid/tests/unittests/test_apply_pass_to_program.py +++ b/python/paddle/fluid/tests/unittests/test_apply_pass_to_program.py @@ -26,9 +26,9 @@ def get_resnet50_model(): main = paddle.static.Program() startup = paddle.static.Program() with paddle.static.program_guard(main, startup): - image = paddle.static.data(name="image", - shape=[None, 3, 224, 224], - dtype="float32") + image = paddle.static.data( + name="image", shape=[None, 3, 224, 224], dtype="float32" + ) label = paddle.static.data(name="label", shape=[None, 1], dtype="int64") model = resnet50() loss_fn = CrossEntropyLoss() @@ -48,7 +48,6 @@ def global_block_contains_op(program, op_type): class TestApplyPassToProgram(unittest.TestCase): - def setUp(self): paddle.enable_static() @@ -67,21 +66,23 @@ def test_case(self): "size_t_attr": "size_t", "float32_attr": "float32", } - ret_attrs = _apply_pass(main, startup, "fuse_elewise_add_act_pass", - attrs, attr_types) + ret_attrs = _apply_pass( + main, startup, "fuse_elewise_add_act_pass", attrs, attr_types + ) self.assertEqual(attrs, ret_attrs) self.assertTrue(global_block_contains_op(main, fused_op)) class TestIRPassBase(unittest.TestCase): - def setUp(self): paddle.enable_static() if paddle.is_compiled_with_cuda(): - fluid.set_flags({ - 'FLAGS_cudnn_deterministic': 1, - 'FLAGS_max_inplace_grad_add': 6, - }) + fluid.set_flags( + { + 'FLAGS_cudnn_deterministic': 1, + 'FLAGS_max_inplace_grad_add': 6, + } + ) self.place = paddle.CUDAPlace(0) else: self.place = paddle.CPUPlace() @@ -104,7 +105,8 @@ def check_before_applied(self, main, startup): self.assertFalse(global_block_contains_op(main, "share_buffer")) self.assertFalse(global_block_contains_op(main, "coalesce_tensor")) self.assertFalse( - global_block_contains_op(main, "fused_elemwise_add_activation")) + global_block_contains_op(main, "fused_elemwise_add_activation") + ) adam_cnt = 0 for op in main.global_block().ops: @@ -119,7 +121,8 @@ def check_after_applied(self, main, startup): self.assertTrue(global_block_contains_op(main, "coalesce_tensor")) self.assertTrue(global_block_contains_op(main, "depend")) self.assertTrue( - global_block_contains_op(main, "fused_elemwise_add_activation")) + global_block_contains_op(main, "fused_elemwise_add_activation") + ) share_dims_cnt = 0 non_share_dims_cnt = 0 @@ -167,8 +170,9 @@ def test_main(self): setattr(build_strategy, k, v) self.check_before_applied(main2, startup2) - apply_build_strategy(main2, startup2, build_strategy, - {"use_cuda": self.use_cuda}) + apply_build_strategy( + main2, startup2, build_strategy, {"use_cuda": self.use_cuda} + ) self.check_after_applied(main2, startup2) image_shape = [batch_size] + list(image.shape)[1:] @@ -186,22 +190,22 @@ def test_main(self): for idx in range(batch_num): feed = { - image.name: - np.random.rand(*image_shape).astype('float32'), - label.name: - np.random.randint(low=0, - high=self.num_classes, - size=label_shape, - dtype='int64'), + image.name: np.random.rand(*image_shape).astype('float32'), + label.name: np.random.randint( + low=0, + high=self.num_classes, + size=label_shape, + dtype='int64', + ), } with paddle.static.scope_guard(scope1): - loss_value1 = self.executor.run(main1, - feed=feed, - fetch_list=[loss1])[0] + loss_value1 = self.executor.run( + main1, feed=feed, fetch_list=[loss1] + )[0] with paddle.static.scope_guard(scope2): - loss_value2 = self.executor.run(main2, - feed=feed, - fetch_list=[loss2])[0] + loss_value2 = self.executor.run( + main2, feed=feed, fetch_list=[loss2] + )[0] self.assertEqual(loss_value1, loss_value2, "batch {}".format(idx)) diff --git a/python/paddle/fluid/tests/unittests/test_arange.py b/python/paddle/fluid/tests/unittests/test_arange.py index 277b6d67d94d9..a7c5c4231d44d 100644 --- a/python/paddle/fluid/tests/unittests/test_arange.py +++ b/python/paddle/fluid/tests/unittests/test_arange.py @@ -21,20 +21,19 @@ class TestArangeOp(OpTest): - def setUp(self): self.op_type = "range" self.init_config() self.inputs = { 'Start': np.array([self.case[0]]).astype(self.dtype), 'End': np.array([self.case[1]]).astype(self.dtype), - 'Step': np.array([self.case[2]]).astype(self.dtype) + 'Step': np.array([self.case[2]]).astype(self.dtype), } self.outputs = { - 'Out': - np.arange(self.case[0], self.case[1], - self.case[2]).astype(self.dtype) + 'Out': np.arange(self.case[0], self.case[1], self.case[2]).astype( + self.dtype + ) } def init_config(self): @@ -46,48 +45,45 @@ def test_check_output(self): class TestFloatArangeOp(TestArangeOp): - def init_config(self): self.dtype = np.float32 self.case = (0, 5, 1) class TestInt32ArangeOp(TestArangeOp): - def init_config(self): self.dtype = np.int32 self.case = (0, 5, 2) class TestFloat64ArangeOp(TestArangeOp): - def init_config(self): self.dtype = np.float64 self.case = (10, 1, -2) class TestInt64ArangeOp(TestArangeOp): - def init_config(self): self.dtype = np.int64 self.case = (-1, -10, -2) class TestArangeOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): self.assertRaises(TypeError, paddle.arange, 10, dtype='int8') class TestArangeAPI(unittest.TestCase): - def test_out(self): with program_guard(Program(), Program()): x1 = paddle.arange(0, 5, 1, 'float32') - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) out = exe.run(fetch_list=[x1]) @@ -96,10 +92,12 @@ def test_out(self): class TestArangeImperative(unittest.TestCase): - def test_out(self): - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) paddle.disable_static(place) x1 = paddle.arange(0, 5, 1) x2 = paddle.tensor.arange(5) diff --git a/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py b/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py index 05e38f4706e04..6c9d09bbab9df 100644 --- a/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py @@ -23,7 +23,6 @@ class BaseTestCase(OpTest): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4, 5) @@ -45,7 +44,6 @@ def test_check_output(self): class TestCase0(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -54,7 +52,6 @@ def initTestCase(self): class TestCase1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -63,7 +60,6 @@ def initTestCase(self): class TestCase2(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -71,10 +67,10 @@ def initTestCase(self): self.axis = 0 -@unittest.skipIf(not paddle.is_compiled_with_cuda(), - "FP16 test runs only on GPU") +@unittest.skipIf( + not paddle.is_compiled_with_cuda(), "FP16 test runs only on GPU" +) class TestCase0FP16(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4, 5) @@ -82,10 +78,10 @@ def initTestCase(self): self.axis = 0 -@unittest.skipIf(not paddle.is_compiled_with_cuda(), - "FP16 test runs only on GPU") +@unittest.skipIf( + not paddle.is_compiled_with_cuda(), "FP16 test runs only on GPU" +) class TestCase1FP16(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (3, 4) @@ -94,7 +90,6 @@ def initTestCase(self): class TestCase2_1(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (3, 4) @@ -103,33 +98,29 @@ def initTestCase(self): class TestCase3(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (3, ) + self.dims = (3,) self.dtype = 'int64' self.axis = 0 class TestCase4(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_min' - self.dims = (1, ) + self.dims = (1,) self.dtype = 'int32' self.axis = 0 class TestCase3_(BaseTestCase): - def initTestCase(self): self.op_type = 'arg_max' - self.dims = (3, ) + self.dims = (3,) self.axis = 0 class BaseTestComplex1_1(OpTest): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (4, 5, 6) @@ -153,7 +144,6 @@ def setUp(self): class BaseTestComplex1_2(OpTest): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (4, 5, 6) @@ -177,7 +167,6 @@ def setUp(self): class BaseTestComplex2_1(OpTest): - def initTestCase(self): self.op_type = 'arg_max' self.dims = (4, 5, 6) @@ -193,20 +182,19 @@ def setUp(self): self.attrs = {'keep_dims': True} if self.op_type == "arg_min": self.outputs = { - 'Out': - np.argmin(self.x, - axis=self.axis).asdtype("int32").reshape(4, 5, 1) + 'Out': np.argmin(self.x, axis=self.axis) + .asdtype("int32") + .reshape(4, 5, 1) } else: self.outputs = { - 'Out': - np.argmax(self.x, - axis=self.axis).asdtype("int32").reshape(4, 5, 1) + 'Out': np.argmax(self.x, axis=self.axis) + .asdtype("int32") + .reshape(4, 5, 1) } class BaseTestComplex2_2(OpTest): - def initTestCase(self): self.op_type = 'arg_min' self.dims = (4, 5, 6) @@ -222,20 +210,19 @@ def setUp(self): self.attrs = {'keep_dims': True} if self.op_type == "arg_min": self.outputs = { - 'Out': - np.argmin(self.x, - axis=self.axis).asdtype("int32").reshape(4, 5, 1) + 'Out': np.argmin(self.x, axis=self.axis) + .asdtype("int32") + .reshape(4, 5, 1) } else: self.outputs = { - 'Out': - np.argmax(self.x, - axis=self.axis).asdtype("int32").reshape(4, 5, 1) + 'Out': np.argmax(self.x, axis=self.axis) + .asdtype("int32") + .reshape(4, 5, 1) } class TestArgMaxTensorAxis(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.x = [np.random.randn(*shape) for shape in self.shapes] @@ -259,8 +246,9 @@ def test_static(self): exe = paddle.static.Executor() exe.run(starup_prog) res = exe.run(fetch_list=[feat, out]) - paddle.static.save_inference_model(self.save_path, [x], [feat, out], - exe) + paddle.static.save_inference_model( + self.save_path, [x], [feat, out], exe + ) gt = np.argmax(res[0], 0) np.testing.assert_allclose(res[1], gt) @@ -282,7 +270,6 @@ def call_func(self, x): class TestArgMinTensorAxis(TestArgMaxTensorAxis): - def test_static(self): main_prog = Program() starup_prog = Program() @@ -301,8 +288,9 @@ def test_static(self): exe = paddle.static.Executor() exe.run(starup_prog) res = exe.run(fetch_list=[feat, out]) - paddle.static.save_inference_model(self.save_path, [x], [feat, out], - exe) + paddle.static.save_inference_model( + self.save_path, [x], [feat, out], exe + ) gt = np.argmin(res[0], 1) np.testing.assert_allclose(np.squeeze(res[1]), gt) diff --git a/python/paddle/fluid/tests/unittests/test_arg_min_max_v2_op.py b/python/paddle/fluid/tests/unittests/test_arg_min_max_v2_op.py index b59a4d169982b..a6f3fee21095d 100644 --- a/python/paddle/fluid/tests/unittests/test_arg_min_max_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_arg_min_max_v2_op.py @@ -22,9 +22,7 @@ def create_kernel_case(op_type, numpy_op_type): - class ArgMinMaxKernelBaseCase(OpTest): - def initTestCase(self): self.op_type = op_type self.numpy_op_type = numpy_op_type @@ -35,7 +33,7 @@ def setUp(self): self.initTestCase() self.dims = (4, 5, 6) self.dtype = "float64" - self.x = (1000 * np.random.random(self.dims).astype(self.dtype)) + self.x = 1000 * np.random.random(self.dims).astype(self.dtype) self.inputs = {'X': self.x} self.attrs = {"axis": self.axis} self.numpy_op = eval("np.%s" % (numpy_op_type)) @@ -46,40 +44,35 @@ def test_check_output(self): self.check_output() class ArgMinMaxKernelCase0(ArgMinMaxKernelBaseCase): - def initTestCase(self): self.op_type = op_type self.numpy_op_type = numpy_op_type self.axis = 1 class ArgMinMaxKernelCase1(ArgMinMaxKernelBaseCase): - def initTestCase(self): self.op_type = op_type self.numpy_op_type = numpy_op_type self.axis = 2 class ArgMinMaxKernelCase2(ArgMinMaxKernelBaseCase): - def initTestCase(self): self.op_type = op_type self.numpy_op_type = numpy_op_type self.axis = -1 class ArgMinMaxKernelCase3(ArgMinMaxKernelBaseCase): - def initTestCase(self): self.op_type = op_type self.numpy_op_type = numpy_op_type self.axis = -2 class ArgMinMaxKernelCase4(ArgMinMaxKernelBaseCase): - def setUp(self): self.initTestCase() self.dims = (4, 5, 6) self.dtype = "float64" - self.x = (1000 * np.random.random(self.dims).astype(self.dtype)) + self.x = 1000 * np.random.random(self.dims).astype(self.dtype) self.inputs = {'X': self.x} self.attrs = {"axis": self.axis, "keepdims": True} self.numpy_op = eval("np.%s" % (numpy_op_type)) @@ -88,12 +81,11 @@ def setUp(self): } class ArgMinMaxKernelCase5(ArgMinMaxKernelBaseCase): - def setUp(self): self.initTestCase() - self.dims = (4) + self.dims = 4 self.dtype = "float64" - self.x = (1000 * np.random.random(self.dims).astype(self.dtype)) + self.x = 1000 * np.random.random(self.dims).astype(self.dtype) self.inputs = {'X': self.x} self.attrs = {"axis": self.axis, "flatten": True} self.numpy_op = eval("np.%s" % (numpy_op_type)) @@ -102,12 +94,11 @@ def setUp(self): } class ArgMinMaxKernelCase6(ArgMinMaxKernelBaseCase): - def setUp(self): self.initTestCase() - self.dims = (4) + self.dims = 4 self.dtype = "float64" - self.x = (1000 * np.random.random(self.dims).astype(self.dtype)) + self.x = 1000 * np.random.random(self.dims).astype(self.dtype) self.inputs = {'X': self.x} self.attrs = {"axis": self.axis, "flatten": True, "keepdims": True} self.numpy_op = eval("np.%s" % (numpy_op_type)) @@ -153,9 +144,7 @@ def setUp(self): def create_test_case(op_type): - class ArgMaxMinTestCase(unittest.TestCase): - def setUp(self): np.random.seed(123) self.input_data = np.random.rand(10, 10).astype("float32") @@ -169,61 +158,67 @@ def setUp(self): def run_static(self, place): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - data_var = paddle.static.data(name="data", - shape=[10, 10], - dtype="float32") + data_var = paddle.static.data( + name="data", shape=[10, 10], dtype="float32" + ) op = eval("paddle.%s" % (op_type)) result = op(data_var) exe = paddle.static.Executor(place) - result_data = exe.run(feed={"data": self.input_data}, - fetch_list=[result]) + result_data = exe.run( + feed={"data": self.input_data}, fetch_list=[result] + ) expected_data = self.numpy_op(self.input_data) - self.assertTrue((result_data == np.array(expected_data)).all(), - True) + self.assertTrue( + (result_data == np.array(expected_data)).all(), True + ) with paddle.static.program_guard(paddle.static.Program()): - data_var = paddle.static.data(name="data", - shape=[10, 10], - dtype="float32") + data_var = paddle.static.data( + name="data", shape=[10, 10], dtype="float32" + ) op = eval("paddle.%s" % (op_type)) result = op(data_var, axis=1) exe = paddle.static.Executor(place) - result_data = exe.run(feed={"data": self.input_data}, - fetch_list=[result]) + result_data = exe.run( + feed={"data": self.input_data}, fetch_list=[result] + ) expected_data = self.numpy_op(self.input_data, axis=1) self.assertTrue((result_data == expected_data).all(), True) with paddle.static.program_guard(paddle.static.Program()): - data_var = paddle.static.data(name="data", - shape=[10, 10], - dtype="float32") + data_var = paddle.static.data( + name="data", shape=[10, 10], dtype="float32" + ) op = eval("paddle.%s" % (op_type)) result = op(data_var, axis=-1) exe = paddle.static.Executor(place) - result_data = exe.run(feed={"data": self.input_data}, - fetch_list=[result]) + result_data = exe.run( + feed={"data": self.input_data}, fetch_list=[result] + ) expected_data = self.numpy_op(self.input_data, axis=-1) self.assertTrue((result_data == expected_data).all(), True) with paddle.static.program_guard(paddle.static.Program()): - data_var = paddle.static.data(name="data", - shape=[10, 10], - dtype="float32") + data_var = paddle.static.data( + name="data", shape=[10, 10], dtype="float32" + ) op = eval("paddle.%s" % (op_type)) result = op(data_var, axis=-1, keepdim=True) exe = paddle.static.Executor(place) - result_data = exe.run(feed={"data": self.input_data}, - fetch_list=[result]) + result_data = exe.run( + feed={"data": self.input_data}, fetch_list=[result] + ) expected_data = self.numpy_op(self.input_data, axis=-1).reshape( - (10, 1)) + (10, 1) + ) self.assertTrue((result_data == expected_data).all(), True) with paddle.static.program_guard(paddle.static.Program()): op = eval("paddle.%s" % (op_type)) - data_var = paddle.static.data(name="data", - shape=[10, 10], - dtype="float32") + data_var = paddle.static.data( + name="data", shape=[10, 10], dtype="float32" + ) result = op(data_var, axis=-1, name="test_arg_api") self.assertTrue("test_arg_api" in result.name) @@ -232,28 +227,28 @@ def run_dygraph(self, place): op = eval("paddle.%s" % (op_type)) data_tensor = paddle.to_tensor(self.input_data) - #case 1 + # case 1 result_data = op(data_tensor) excepted_data = self.numpy_op(self.input_data) self.assertTrue((result_data.numpy() == excepted_data).all(), True) - #case 2 + # case 2 result_data = op(data_tensor, axis=1) excepted_data = self.numpy_op(self.input_data, axis=1) self.assertTrue((result_data.numpy() == excepted_data).all(), True) - #case 3 + # case 3 result_data = op(data_tensor, axis=-1) excepted_data = self.numpy_op(self.input_data, axis=-1) self.assertTrue((result_data.numpy() == excepted_data).all(), True) - #case 4 + # case 4 result_data = op(data_tensor, axis=-1, keepdim=True) excepted_data = self.numpy_op(self.input_data, axis=-1) excepted_data = excepted_data.reshape((10, 1)) self.assertTrue((result_data.numpy() == excepted_data).all(), True) - #case 5 + # case 5 result_data = op(data_tensor, axis=-1, keepdim=True, dtype="int32") self.assertTrue(result_data.numpy().dtype == np.int32) @@ -288,7 +283,6 @@ def test_case(self): class TestArgMinMaxOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -306,49 +300,49 @@ def test_argmin_x_type(): self.assertRaises(TypeError, test_argmin_x_type) def test_argmax_attr_type(): - data = paddle.static.data(name="test_argmax", - shape=[10], - dtype="float32") + data = paddle.static.data( + name="test_argmax", shape=[10], dtype="float32" + ) output = paddle.argmax(x=data, dtype="float32") self.assertRaises(TypeError, test_argmax_attr_type) def test_argmin_attr_type(): - data = paddle.static.data(name="test_argmax", - shape=[10], - dtype="float32") + data = paddle.static.data( + name="test_argmax", shape=[10], dtype="float32" + ) output = paddle.argmin(x=data, dtype="float32") self.assertRaises(TypeError, test_argmin_attr_type) def test_argmax_axis_type(): - data = paddle.static.data(name="test_argmax", - shape=[10], - dtype="float32") + data = paddle.static.data( + name="test_argmax", shape=[10], dtype="float32" + ) output = paddle.argmax(x=data, axis=1.2) self.assertRaises(TypeError, test_argmax_axis_type) def test_argmin_axis_type(): - data = paddle.static.data(name="test_argmin", - shape=[10], - dtype="float32") + data = paddle.static.data( + name="test_argmin", shape=[10], dtype="float32" + ) output = paddle.argmin(x=data, axis=1.2) self.assertRaises(TypeError, test_argmin_axis_type) def test_argmax_dtype_type(): - data = paddle.static.data(name="test_argmax", - shape=[10], - dtype="float32") + data = paddle.static.data( + name="test_argmax", shape=[10], dtype="float32" + ) output = paddle.argmax(x=data, dtype=None) self.assertRaises(ValueError, test_argmax_dtype_type) def test_argmin_dtype_type(): - data = paddle.static.data(name="test_argmin", - shape=[10], - dtype="float32") + data = paddle.static.data( + name="test_argmin", shape=[10], dtype="float32" + ) output = paddle.argmin(x=data, dtype=None) self.assertRaises(ValueError, test_argmin_dtype_type) diff --git a/python/paddle/fluid/tests/unittests/test_argsort_op.py b/python/paddle/fluid/tests/unittests/test_argsort_op.py index d9106e26627be..d8df9fefbbd43 100644 --- a/python/paddle/fluid/tests/unittests/test_argsort_op.py +++ b/python/paddle/fluid/tests/unittests/test_argsort_op.py @@ -26,7 +26,6 @@ class PyArgsort(object): - def __init__(self, input_shape, axis, descending, dtype): self.x = np.random.random(input_shape).astype(dtype) self.label = np.random.random(input_shape).astype(dtype) @@ -39,17 +38,21 @@ def __init__(self, input_shape, axis, descending, dtype): def forward(self): if self.descending: self.indices = np.flip( - np.argsort(self.x, kind='quicksort', axis=self.axis), self.axis) + np.argsort(self.x, kind='quicksort', axis=self.axis), self.axis + ) self.sorted_x = np.flip( - np.sort(self.x, kind='quicksort', axis=self.axis), self.axis) + np.sort(self.x, kind='quicksort', axis=self.axis), self.axis + ) else: self.indices = np.argsort(self.x, kind='quicksort', axis=self.axis) self.sorted_x = np.sort(self.x, kind='quicksort', axis=self.axis) self.loss = self.sorted_x * self.label self.loss = np.sum(self.loss) - out = (np.array(self.indices, dtype=self.indices.dtype), - np.array(self.sorted_x, dtype=self.sorted_x.dtype), - np.array([self.loss], dtype=self.loss.dtype)) + out = ( + np.array(self.indices, dtype=self.indices.dtype), + np.array(self.sorted_x, dtype=self.sorted_x.dtype), + np.array([self.loss], dtype=self.loss.dtype), + ) return out @@ -60,7 +63,6 @@ def create_tensor(np_data, place): class TestArgsortOpCPU(unittest.TestCase): - def setup_program(self): self.main_program = Program() self.startup_program = Program() @@ -76,19 +78,21 @@ def setUp(self): self.feed_data_field = {"x", "label"} self.grad_data_field = {"x"} - self.py_argsort = PyArgsort(self.input_shape, self.axis, - self.descending, self.dtype) + self.py_argsort = PyArgsort( + self.input_shape, self.axis, self.descending, self.dtype + ) with fluid.program_guard(self.main_program, self.startup_program): - x = fluid.layers.data(name="x", - shape=self.input_shape, - dtype=self.dtype) + x = fluid.layers.data( + name="x", shape=self.input_shape, dtype=self.dtype + ) x.stop_gradient = False - label = fluid.layers.data(name="label", - shape=self.input_shape, - dtype=self.dtype) + label = fluid.layers.data( + name="label", shape=self.input_shape, dtype=self.dtype + ) self.sorted_x, self.index = fluid.layers.argsort( - input=x, axis=self.axis, descending=self.descending) + input=x, axis=self.axis, descending=self.descending + ) self.sorted_x.stop_gradient = False loss = fluid.layers.elementwise_mul(self.sorted_x, label) self.loss = fluid.layers.reduce_sum(loss) @@ -99,9 +103,11 @@ def forward(self): for x in self.feed_data_field } exe = Executor(self.place) - out = exe.run(self.main_program, - feed=self.feed_map, - fetch_list=[self.index, self.sorted_x, self.loss]) + out = exe.run( + self.main_program, + feed=self.feed_map, + fetch_list=[self.index, self.sorted_x, self.loss], + ) return out def backward(self): @@ -114,10 +120,12 @@ def backward(self): for x in self.grad_data_field ] exe = Executor(self.place) - out = exe.run(self.main_program, - feed=self.feed_map, - fetch_list=fetch_list, - return_numpy=False) + out = exe.run( + self.main_program, + feed=self.feed_map, + fetch_list=fetch_list, + return_numpy=False, + ) return out def test_backward(self, numeric_grad_delta=1e-5, max_relative_error=1e-7): @@ -129,23 +137,22 @@ def test_backward(self, numeric_grad_delta=1e-5, max_relative_error=1e-7): ana_grad = [np.array(x) for x in self.backward()] num_grad = self.get_numerical_gradient(delta=numeric_grad_delta) - self.assert_is_close(num_grad, - ana_grad, - 'x', - max_relative_error=max_relative_error, - msg_prefix="Gradient Check On %s" % - str(self.place)) + self.assert_is_close( + num_grad, + ana_grad, + 'x', + max_relative_error=max_relative_error, + msg_prefix="Gradient Check On %s" % str(self.place), + ) def check_forward(self): pd_outputs = self.forward() py_outputs = self.py_argsort.forward() for pd_output, py_output in zip(pd_outputs, py_outputs): self.assertEqual(pd_output.shape, py_output.shape) - np.testing.assert_allclose(pd_output, - py_output, - rtol=1e-05, - atol=0, - equal_nan=False) + np.testing.assert_allclose( + pd_output, py_output, rtol=1e-05, atol=0, equal_nan=False + ) def get_numerical_gradient(self, delta=1e-7): if self.dtype == 'float16': @@ -167,8 +174,14 @@ def get_numerical_gradient(self, delta=1e-7): return grad_list - def assert_is_close(self, numeric_grads, analytic_grads, names, - max_relative_error, msg_prefix): + def assert_is_close( + self, + numeric_grads, + analytic_grads, + names, + max_relative_error, + msg_prefix, + ): for a, b, name in zip(numeric_grads, analytic_grads, names): abs_a = np.abs(a) abs_a[abs_a < 1e-3] = 1 @@ -178,10 +191,19 @@ def assert_is_close(self, numeric_grads, analytic_grads, names, def err_msg(): offset = np.argmax(diff_mat > max_relative_error) - return ("%s error, %s variable %s max gradient diff %f over limit %f, " - "the first error element is %d, expected %f, but got %f.") \ - % ('argsort', msg_prefix, name, max_diff, max_relative_error, - offset, a.flatten()[offset], b.flatten()[offset]) + return ( + "%s error, %s variable %s max gradient diff %f over limit %f, " + "the first error element is %d, expected %f, but got %f." + ) % ( + 'argsort', + msg_prefix, + name, + max_diff, + max_relative_error, + offset, + a.flatten()[offset], + b.flatten()[offset], + ) self.assertLessEqual(max_diff, max_relative_error, err_msg()) @@ -202,7 +224,6 @@ def init_place(self): class TestArgsortOpGPU(TestArgsortOpCPU): - def init_place(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -211,144 +232,120 @@ def init_place(self): class TestArgsortOpAxis0CPU(TestArgsortOpCPU): - def init_axis(self): self.axis = 0 class TestArgsortOpAxis0GPU(TestArgsortOpGPU): - def init_axis(self): self.axis = 0 class TestArgsortOpAxis1CPU(TestArgsortOpCPU): - def init_axis(self): self.axis = 1 class TestArgsortOpAxis1GPU(TestArgsortOpGPU): - def init_axis(self): self.axis = 1 class TestArgsortOpAxis2CPU(TestArgsortOpCPU): - def init_axis(self): self.axis = 2 class TestArgsortOpAxis2GPU(TestArgsortOpGPU): - def init_axis(self): self.axis = 2 class TestArgsortOpAxisNeg1CPU(TestArgsortOpCPU): - def init_axis(self): self.axis = -1 class TestArgsortOpAxisNeg1GPU(TestArgsortOpGPU): - def init_axis(self): self.axis = -1 class TestArgsortOpAxisNeg2CPU(TestArgsortOpCPU): - def init_axis(self): self.axis = -2 class TestArgsortOpAxisNeg2GPU(TestArgsortOpGPU): - def init_axis(self): self.axis = -2 class TestArgsortOpDescendingAxisCPU(TestArgsortOpCPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisGPU(TestArgsortOpGPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis0CPU(TestArgsortOpAxis0CPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis0GPU(TestArgsortOpAxis0GPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis1CPU(TestArgsortOpAxis1CPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis1GPU(TestArgsortOpAxis1GPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis2CPU(TestArgsortOpAxis2CPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis2GPU(TestArgsortOpAxis2GPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg1CPU(TestArgsortOpAxisNeg1CPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg1GPU(TestArgsortOpAxisNeg1GPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg2CPU(TestArgsortOpAxisNeg2CPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg2GPU(TestArgsortOpAxisNeg2GPU): - def init_direction(self): self.descending = True class TestArgsortErrorOnCPU(unittest.TestCase): - def setUp(self): self.place = core.CPUPlace() def test_error(self): - def test_fluid_var_type(): with fluid.program_guard(fluid.Program()): x = [1] @@ -363,7 +360,6 @@ def test_paddle_var_type(): class TestArgsortErrorOnGPU(TestArgsortErrorOnCPU): - def setUp(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -372,7 +368,6 @@ def setUp(self): class TestArgsort(unittest.TestCase): - def init(self): self.input_shape = [ 10000, @@ -389,16 +384,17 @@ def setUp(self): def test_api(self): with fluid.program_guard(fluid.Program()): - input = fluid.data(name="input", - shape=self.input_shape, - dtype="float64") + input = fluid.data( + name="input", shape=self.input_shape, dtype="float64" + ) output = paddle.argsort(input, axis=self.axis) output2 = paddle.argsort(input, axis=self.axis, descending=True) exe = fluid.Executor(self.place) - result, result2 = exe.run(feed={'input': self.data}, - fetch_list=[output, output2]) + result, result2 = exe.run( + feed={'input': self.data}, fetch_list=[output, output2] + ) np_result = np.argsort(self.data, axis=self.axis) self.assertEqual((result == np_result).all(), True) @@ -408,28 +404,24 @@ def test_api(self): class TestArgsort2(TestArgsort): - def init(self): self.input_shape = [10000, 1] self.axis = 0 class TestArgsort3(TestArgsort): - def init(self): self.input_shape = [1, 10000] self.axis = 1 class TestArgsort4(TestArgsort): - def init(self): self.input_shape = [2, 3, 4] self.axis = 1 class TestArgsortImperative(unittest.TestCase): - def init(self): self.input_shape = [ 10000, @@ -459,28 +451,24 @@ def test_api(self): class TestArgsortImperative2(TestArgsortImperative): - def init(self): self.input_shape = [10000, 1] self.axis = 0 class TestArgsortImperative3(TestArgsortImperative): - def init(self): self.input_shape = [1, 10000] self.axis = 1 class TestArgsortImperative4(TestArgsortImperative): - def init(self): self.input_shape = [2, 3, 4] self.axis = 1 class TestArgsortWithInputNaN(unittest.TestCase): - def init(self): self.axis = 0 diff --git a/python/paddle/fluid/tests/unittests/test_array_read_write_op.py b/python/paddle/fluid/tests/unittests/test_array_read_write_op.py index 4c1c7563d8b3d..f5f54017d9983 100644 --- a/python/paddle/fluid/tests/unittests/test_array_read_write_op.py +++ b/python/paddle/fluid/tests/unittests/test_array_read_write_op.py @@ -58,12 +58,11 @@ def _test_read_write(x): class TestArrayReadWrite(unittest.TestCase): - def test_read_write(self): x = [ layers.data(name='x0', shape=[100]), layers.data(name='x1', shape=[100]), - layers.data(name='x2', shape=[100]) + layers.data(name='x2', shape=[100]), ] for each_x in x: each_x.stop_gradient = False @@ -73,13 +72,11 @@ def test_read_write(self): place = core.CPUPlace() exe = Executor(place) - outs = exe.run(feed={ - 'x0': tensor, - 'x1': tensor, - 'x2': tensor - }, - fetch_list=[a_sum, x_sum], - scope=core.Scope()) + outs = exe.run( + feed={'x0': tensor, 'x1': tensor, 'x2': tensor}, + fetch_list=[a_sum, x_sum], + scope=core.Scope(), + ) self.assertEqual(outs[0], outs[1]) total_sum = layers.sums(input=[a_sum, x_sum]) @@ -88,15 +85,17 @@ def test_read_write(self): append_backward(total_sum_scaled) g_vars = list( - map(default_main_program().global_block().var, - [each_x.name + "@GRAD" for each_x in x])) + map( + default_main_program().global_block().var, + [each_x.name + "@GRAD" for each_x in x], + ) + ) g_out = [ - item.sum() for item in exe.run(feed={ - 'x0': tensor, - 'x1': tensor, - 'x2': tensor - }, - fetch_list=g_vars) + item.sum() + for item in exe.run( + feed={'x0': tensor, 'x1': tensor, 'x2': tensor}, + fetch_list=g_vars, + ) ] g_out_sum = np.array(g_out).sum() @@ -116,9 +115,11 @@ def test_read_write(self): self.assertEqual(a_sum_dygraph, x_sum_dygraph) total_sum_dygraph = layers.sums( - input=[a_sum_dygraph, x_sum_dygraph]) - total_sum_scaled_dygraph = layers.scale(x=total_sum_dygraph, - scale=1 / 6.0) + input=[a_sum_dygraph, x_sum_dygraph] + ) + total_sum_scaled_dygraph = layers.scale( + x=total_sum_dygraph, scale=1 / 6.0 + ) total_sum_scaled_dygraph.backward() g_out_dygraph = [ item._grad_ivar().numpy().sum() for item in x_dygraph @@ -129,40 +130,33 @@ def test_read_write(self): class TestArrayReadWriteOpError(unittest.TestCase): - def _test_errors(self, use_fluid_api=True): if use_fluid_api: with program_guard(Program(), Program()): x1 = np.random.randn(2, 4).astype('int32') - x2 = fluid.layers.fill_constant(shape=[1], - dtype='int32', - value=1) + x2 = fluid.layers.fill_constant( + shape=[1], dtype='int32', value=1 + ) x3 = np.random.randn(2, 4).astype('int32') - self.assertRaises(TypeError, - fluid.layers.array_read, - array=x1, - i=x2) - self.assertRaises(TypeError, - fluid.layers.array_write, - array=x1, - i=x2, - out=x3) + self.assertRaises( + TypeError, fluid.layers.array_read, array=x1, i=x2 + ) + self.assertRaises( + TypeError, fluid.layers.array_write, array=x1, i=x2, out=x3 + ) else: with program_guard(Program(), Program()): x1 = np.random.randn(2, 4).astype('int32') x2 = paddle.ones(shape=[1], dtype='int32') x3 = np.random.randn(2, 4).astype('int32') - self.assertRaises(TypeError, - paddle.tensor.array_read, - array=x1, - i=x2) - self.assertRaises(TypeError, - paddle.tensor.array_write, - array=x1, - i=x2, - out=x3) + self.assertRaises( + TypeError, paddle.tensor.array_read, array=x1, i=x2 + ) + self.assertRaises( + TypeError, paddle.tensor.array_write, array=x1, i=x2, out=x3 + ) def test_fluid_api(self): self._test_errors(use_fluid_api=True) @@ -172,7 +166,6 @@ def test_paddle_api(self): class TestArrayReadWriteApi(unittest.TestCase): - def test_api(self): paddle.disable_static() arr = paddle.tensor.create_array(dtype="float32") diff --git a/python/paddle/fluid/tests/unittests/test_ascend_trigger.py b/python/paddle/fluid/tests/unittests/test_ascend_trigger.py index 8d8a896a42442..16b38865593a6 100644 --- a/python/paddle/fluid/tests/unittests/test_ascend_trigger.py +++ b/python/paddle/fluid/tests/unittests/test_ascend_trigger.py @@ -18,7 +18,7 @@ class TestAscendTriggerOP(unittest.TestCase): - """ TestCases for ascend_trigger op""" + """TestCases for ascend_trigger op""" def test_ascend_trigger_op(self): paddle.enable_static() @@ -27,10 +27,12 @@ def test_ascend_trigger_op(self): with fluid.program_guard(program): x = fluid.data(name='x', shape=[1], dtype='int64', lod_level=0) y = fluid.data(name='y', shape=[1], dtype='int64', lod_level=0) - block.append_op(type="ascend_trigger", - inputs={"FeedList": [x]}, - outputs={"FetchList": [y]}, - attrs={'graph_idx': 0}) + block.append_op( + type="ascend_trigger", + inputs={"FeedList": [x]}, + outputs={"FetchList": [y]}, + attrs={'graph_idx': 0}, + ) exe = paddle.static.Executor(paddle.CPUPlace()) try: diff --git a/python/paddle/fluid/tests/unittests/test_assert_op.py b/python/paddle/fluid/tests/unittests/test_assert_op.py index 481a58b441e55..9a91ebca5d89c 100644 --- a/python/paddle/fluid/tests/unittests/test_assert_op.py +++ b/python/paddle/fluid/tests/unittests/test_assert_op.py @@ -19,7 +19,6 @@ class TestAssertOp(unittest.TestCase): - def run_network(self, net_func): main_program = fluid.Program() startup_program = fluid.Program() @@ -29,39 +28,35 @@ def run_network(self, net_func): exe.run(main_program) def test_assert_true(self): - def net_func(): - condition = layers.fill_constant(shape=[1], - dtype='bool', - value=True) + condition = layers.fill_constant( + shape=[1], dtype='bool', value=True + ) layers.Assert(condition, []) self.run_network(net_func) def test_assert_false(self): - def net_func(): - condition = layers.fill_constant(shape=[1], - dtype='bool', - value=False) + condition = layers.fill_constant( + shape=[1], dtype='bool', value=False + ) layers.Assert(condition) with self.assertRaises(ValueError): self.run_network(net_func) def test_assert_cond_numel_error(self): - def net_func(): - condition = layers.fill_constant(shape=[1, 2], - dtype='bool', - value=True) + condition = layers.fill_constant( + shape=[1, 2], dtype='bool', value=True + ) layers.Assert(condition, []) with self.assertRaises(ValueError): self.run_network(net_func) def test_assert_print_data(self): - def net_func(): zero = layers.fill_constant(shape=[1], dtype='int64', value=0) one = layers.fill_constant(shape=[1], dtype='int64', value=1) @@ -73,18 +68,16 @@ def net_func(): self.run_network(net_func) def test_assert_summary(self): - def net_func(): x = layers.fill_constant(shape=[10], dtype='float32', value=2.0) condition = layers.reduce_max(x) < 1.0 - layers.Assert(condition, (x, ), 5) + layers.Assert(condition, (x,), 5) print("test_assert_summary") with self.assertRaises(ValueError): self.run_network(net_func) def test_assert_summary_greater_than_size(self): - def net_func(): x = layers.fill_constant(shape=[2, 3], dtype='float32', value=2.0) condition = layers.reduce_max(x) < 1.0 diff --git a/python/paddle/fluid/tests/unittests/test_assign_op.py b/python/paddle/fluid/tests/unittests/test_assign_op.py index f1ae5362e085d..a59d9fb6688ff 100644 --- a/python/paddle/fluid/tests/unittests/test_assign_op.py +++ b/python/paddle/fluid/tests/unittests/test_assign_op.py @@ -27,7 +27,6 @@ class TestAssignOp(op_test.OpTest): - def setUp(self): self.python_api = paddle.assign self.op_type = "assign" @@ -53,7 +52,6 @@ def test_backward(self): class TestAssignFP16Op(op_test.OpTest): - def setUp(self): self.python_api = paddle.assign self.op_type = "assign" @@ -79,7 +77,6 @@ def test_backward(self): class TestAssignOpWithLoDTensorArray(unittest.TestCase): - def test_assign_LoDTensorArray(self): paddle.enable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -88,9 +85,9 @@ def test_assign_LoDTensorArray(self): with program_guard(main_program): x = fluid.data(name='x', shape=[100, 10], dtype='float32') x.stop_gradient = False - y = fluid.layers.fill_constant(shape=[100, 10], - dtype='float32', - value=1) + y = fluid.layers.fill_constant( + shape=[100, 10], dtype='float32', value=1 + ) z = fluid.layers.elementwise_add(x=x, y=y) i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=0) init_array = fluid.layers.array_write(x=z, i=i) @@ -100,28 +97,33 @@ def test_assign_LoDTensorArray(self): append_backward(mean) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feed_x = np.random.random(size=(100, 10)).astype('float32') ones = np.ones((100, 10)).astype('float32') feed_add = feed_x + ones - res = exe.run(main_program, - feed={'x': feed_x}, - fetch_list=[sums.name, x.grad_name]) + res = exe.run( + main_program, + feed={'x': feed_x}, + fetch_list=[sums.name, x.grad_name], + ) np.testing.assert_allclose(res[0], feed_add, rtol=1e-05) np.testing.assert_allclose(res[1], ones / 1000.0, rtol=1e-05) paddle.disable_static() class TestAssignOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): # The type of input must be Variable or numpy.ndarray. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.assign, x1) # When the type of input is numpy.ndarray, the dtype of input must be float32, int32. x2 = np.array([[2.5, 2.5]], dtype='uint8') @@ -130,7 +132,6 @@ def test_errors(self): class TestAssignOApi(unittest.TestCase): - def test_assign_LoDTensorArray(self): paddle.enable_static() main_program = Program() @@ -138,9 +139,9 @@ def test_assign_LoDTensorArray(self): with program_guard(main_program): x = fluid.data(name='x', shape=[100, 10], dtype='float32') x.stop_gradient = False - y = fluid.layers.fill_constant(shape=[100, 10], - dtype='float32', - value=1) + y = fluid.layers.fill_constant( + shape=[100, 10], dtype='float32', value=1 + ) z = fluid.layers.elementwise_add(x=x, y=y) i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=0) init_array = fluid.layers.array_write(x=z, i=i) @@ -149,15 +150,20 @@ def test_assign_LoDTensorArray(self): mean = paddle.mean(sums) append_backward(mean) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feed_x = np.random.random(size=(100, 10)).astype('float32') ones = np.ones((100, 10)).astype('float32') feed_add = feed_x + ones - res = exe.run(main_program, - feed={'x': feed_x}, - fetch_list=[sums.name, x.grad_name]) + res = exe.run( + main_program, + feed={'x': feed_x}, + fetch_list=[sums.name, x.grad_name], + ) np.testing.assert_allclose(res[0], feed_add, rtol=1e-05) np.testing.assert_allclose(res[1], ones / 1000.0, rtol=1e-05) paddle.disable_static() @@ -225,23 +231,25 @@ def test_clone(self): x = paddle.static.data("X", shape=[2, 3]) clone_x = paddle.clone(x) exe = paddle.static.Executor() - y_np = exe.run(paddle.static.default_main_program(), - feed={'X': x_np}, - fetch_list=[clone_x])[0] + y_np = exe.run( + paddle.static.default_main_program(), + feed={'X': x_np}, + fetch_list=[clone_x], + )[0] np.testing.assert_array_equal(y_np, x_np) paddle.disable_static() class TestAssignOpErrorApi(unittest.TestCase): - def test_errors(self): paddle.enable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) with program_guard(Program(), Program()): # The type of input must be Variable or numpy.ndarray. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, paddle.assign, x1) # When the type of input is numpy.ndarray, the dtype of input must be float32, int32. x2 = np.array([[2.5, 2.5]], dtype='uint8') @@ -259,7 +267,6 @@ def test_type_error(self): class TestAssignDoubleGradCheck(unittest.TestCase): - def assign_wrapper(self, x): return paddle.fluid.layers.assign(x[0]) @@ -274,17 +281,13 @@ def func(self, place): out = paddle.fluid.layers.assign(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.assign_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.assign_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -296,7 +299,6 @@ def test_grad(self): class TestAssignTripleGradCheck(unittest.TestCase): - def assign_wrapper(self, x): return paddle.fluid.layers.assign(x[0]) @@ -311,17 +313,13 @@ def func(self, place): out = paddle.fluid.layers.assign(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.assign_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.assign_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_assign_pos_op.py b/python/paddle/fluid/tests/unittests/test_assign_pos_op.py index 2440d349e0a3a..d04fecd7b0500 100644 --- a/python/paddle/fluid/tests/unittests/test_assign_pos_op.py +++ b/python/paddle/fluid/tests/unittests/test_assign_pos_op.py @@ -24,7 +24,7 @@ def assign_pos(x, _cum_count): cum_count = np.copy(_cum_count) x = x.reshape(-1) - res = np.zeros((cum_count[-1], ), dtype=np.int64) + res = np.zeros((cum_count[-1],), dtype=np.int64) for i, idx in enumerate(x): p = cum_count[idx] cum_count[idx] -= 1 @@ -34,7 +34,7 @@ def assign_pos(x, _cum_count): def count(x, upper_num): - res = np.zeros((upper_num, )).astype(int) + res = np.zeros((upper_num,)).astype(int) for i in x.reshape(-1): if i >= 0 and i < len(res): res[i] += 1 @@ -62,17 +62,16 @@ def assert_allclose(res, out, cum_count): def get_redefined_allclose(cum_count): - def redefined_allclose(x, y, *args, **kwargs): return assert_allclose(x, y, cum_count) return redefined_allclose -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestAssignPosOpInt64(op_test.OpTest): - def setUp(self): x = np.random.randint(0, 16, size=(100, 2)).astype("int64") y = count(x, 16) @@ -81,7 +80,7 @@ def setUp(self): self.inputs = { 'X': x, "cum_count": cum_count, - "eff_num_len": np.array([cum_count[-1]]) + "eff_num_len": np.array([cum_count[-1]]), } self.outputs = {'Out': assign_pos(x, cum_count)} self.cum_count = cum_count @@ -91,10 +90,10 @@ def test_forward(self): self.check_output_with_place(paddle.CUDAPlace(0)) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestAssignPosAPI(unittest.TestCase): - def setUp(self): self.x = np.random.randint(0, 16, size=(100, 2)).astype("int64") y = count(self.x, 16) @@ -106,16 +105,15 @@ def test_api_static(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): x = paddle.fluid.data('x', self.x.shape, dtype="int64") - cum_count = paddle.fluid.data('cum_count', - self.cum_count.shape, - dtype="int64") + cum_count = paddle.fluid.data( + 'cum_count', self.cum_count.shape, dtype="int64" + ) out = utils._assign_pos(x, cum_count) exe = paddle.static.Executor(self.place) - res = exe.run(feed={ - 'x': self.x, - "cum_count": self.cum_count - }, - fetch_list=[out]) + res = exe.run( + feed={'x': self.x, "cum_count": self.cum_count}, + fetch_list=[out], + ) assert_allclose(res[0], self.out, self.cum_count) def func_api_dygraph(self): diff --git a/python/paddle/fluid/tests/unittests/test_assign_value_op.py b/python/paddle/fluid/tests/unittests/test_assign_value_op.py index df3becbbd9c40..cac321c2a27d3 100644 --- a/python/paddle/fluid/tests/unittests/test_assign_value_op.py +++ b/python/paddle/fluid/tests/unittests/test_assign_value_op.py @@ -25,7 +25,6 @@ class TestAssignValueOp(op_test.OpTest): - def setUp(self): self.op_type = "assign_value" self.inputs = {} @@ -33,7 +32,8 @@ def setUp(self): self.init_data() self.attrs["shape"] = self.value.shape self.attrs["dtype"] = framework.convert_np_dtype_to_dtype_( - self.value.dtype) + self.value.dtype + ) self.outputs = {"Out": self.value} def init_data(self): @@ -45,35 +45,36 @@ def test_forward(self): class TestAssignValueOp2(TestAssignValueOp): - def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int32) self.attrs["int32_values"] = [int(v) for v in self.value.flat] class TestAssignValueOp3(TestAssignValueOp): - def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int64) self.attrs["int64_values"] = [int(v) for v in self.value.flat] class TestAssignValueOp4(TestAssignValueOp): - def init_data(self): - self.value = np.random.choice(a=[False, True], - size=(2, 5)).astype(np.bool) + self.value = np.random.choice(a=[False, True], size=(2, 5)).astype( + np.bool + ) self.attrs["bool_values"] = [int(v) for v in self.value.flat] class TestAssignApi(unittest.TestCase): - def setUp(self): self.init_dtype() self.value = (-100 + 200 * np.random.random(size=(2, 5))).astype( - self.dtype) - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.dtype + ) + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def init_dtype(self): self.dtype = "float32" @@ -91,25 +92,26 @@ def test_assign(self): class TestAssignApi2(TestAssignApi): - def init_dtype(self): self.dtype = "int32" class TestAssignApi3(TestAssignApi): - def init_dtype(self): self.dtype = "int64" class TestAssignApi4(TestAssignApi): - def setUp(self): self.init_dtype() - self.value = np.random.choice(a=[False, True], - size=(2, 5)).astype(np.bool) - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.value = np.random.choice(a=[False, True], size=(2, 5)).astype( + np.bool + ) + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def init_dtype(self): self.dtype = "bool" diff --git a/python/paddle/fluid/tests/unittests/test_async_ssa_graph_executor_mnist.py b/python/paddle/fluid/tests/unittests/test_async_ssa_graph_executor_mnist.py index f46d5303f52a2..c56235cd0d129 100644 --- a/python/paddle/fluid/tests/unittests/test_async_ssa_graph_executor_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_async_ssa_graph_executor_mnist.py @@ -34,21 +34,26 @@ def convolutional_neural_network(use_py_reader): capacity=64, feed_list=[img, label], iterable=False, - use_double_buffer=False) - - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") + use_double_buffer=False, + ) + + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) prediction = fluid.layers.fc(input=conv_pool_2, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) @@ -65,20 +70,30 @@ def test(): place = fluid.CPUPlace() exe = fluid.Executor(place) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=BATCH_SIZE) - - array, img, label, prediction, avg_loss, acc, py_reader = convolutional_neural_network( - use_py_reader=False) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=BATCH_SIZE + ) + + ( + array, + img, + label, + prediction, + avg_loss, + acc, + py_reader, + ) = convolutional_neural_network(use_py_reader=False) feeder = fluid.DataFeeder(feed_list=[img, label], place=place) def train_test(train_test_program, train_test_feed, train_test_reader): acc_set = [] avg_loss_set = [] for test_data in train_test_reader(): - acc_np, avg_loss_np = exe.run(program=train_test_program, - feed=train_test_feed.feed(test_data), - fetch_list=[acc, avg_loss]) + acc_np, avg_loss_np = exe.run( + program=train_test_program, + feed=train_test_feed.feed(test_data), + fetch_list=[acc, avg_loss], + ) acc_set.append(float(acc_np)) avg_loss_set.append(float(avg_loss_np)) # get test acc and loss @@ -90,7 +105,8 @@ def train_test(train_test_program, train_test_feed, train_test_reader): avg_loss_val, acc_val = train_test( train_test_program=fluid.default_main_program(), train_test_reader=test_reader, - train_test_feed=feeder) + train_test_feed=feeder, + ) print("Test: avg_cost: %s, acc: %s" % (avg_loss_val, acc_val)) assert acc_val > 0.96 @@ -101,17 +117,25 @@ def train(use_cuda, thread_num, cpu_num): print("paddle is not compiled with cuda, exit!") return - array, img, label, prediction, avg_loss, acc, py_reader = convolutional_neural_network( - use_py_reader=True) + ( + array, + img, + label, + prediction, + avg_loss, + acc, + py_reader, + ) = convolutional_neural_network(use_py_reader=True) print("build convolutional neural network done.") optimizer = fluid.optimizer.Adam(learning_rate=0.001) optimizer.minimize(avg_loss) print("Adam optimizer minimize done.") - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=BATCH_SIZE) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=BATCH_SIZE, + ) print("declared train reader done.") place = fluid.CPUPlace() @@ -133,11 +157,13 @@ def train(use_cuda, thread_num, cpu_num): exec_strategy.num_iteration_per_run = 10 main_program = fluid.default_main_program() - pe = fluid.ParallelExecutor(use_cuda=False, - loss_name=avg_loss.name, - main_program=main_program, - build_strategy=build_strategy, - exec_strategy=exec_strategy) + pe = fluid.ParallelExecutor( + use_cuda=False, + loss_name=avg_loss.name, + main_program=main_program, + build_strategy=build_strategy, + exec_strategy=exec_strategy, + ) print("declare parallel executor done.") py_reader.set_sample_list_generator(train_reader) @@ -148,15 +174,18 @@ def train(use_cuda, thread_num, cpu_num): try: while True: array_v, acc_v, prediction_v, loss_val = pe.run( - fetch_list=[array, acc, prediction, avg_loss.name]) + fetch_list=[array, acc, prediction, avg_loss.name] + ) assert numpy.allclose(array_v[0], prediction_v) == True assert numpy.allclose(array_v[1], acc_v) == True loss_val = numpy.mean(loss_val) if step % 10 == 0: - print("Pass %d, Batch %d, Cost %f, queue size %d" % - (pass_id, step, loss_val, py_reader.queue.size())) + print( + "Pass %d, Batch %d, Cost %f, queue size %d" + % (pass_id, step, loss_val, py_reader.queue.size()) + ) step += 1 except fluid.core.EOFException: print("train end pass = " + str(pass_id)) @@ -166,24 +195,33 @@ def train(use_cuda, thread_num, cpu_num): class TestAsyncSSAGraphExecutor(unittest.TestCase): - def test_check_async_ssa_exe_train(self): step_list = [] for cpu_num in [1, 2, 4]: print("run cpu_num -> " + str(cpu_num)) with fluid.scope_guard(fluid.core.Scope()): - with fluid.program_guard(main_program=fluid.Program(), - startup_program=fluid.Program()): + with fluid.program_guard( + main_program=fluid.Program(), + startup_program=fluid.Program(), + ): start_time = time.time() - step = train(use_cuda=False, - thread_num=cpu_num, - cpu_num=cpu_num) + step = train( + use_cuda=False, thread_num=cpu_num, cpu_num=cpu_num + ) end_time = time.time() step_list.append(step) - print("cpu_num -> " + str(cpu_num) + " step -> " + str(step) + - " time -> " + str(end_time - start_time)) - with fluid.program_guard(main_program=fluid.Program(), - startup_program=fluid.Program()): + print( + "cpu_num -> " + + str(cpu_num) + + " step -> " + + str(step) + + " time -> " + + str(end_time - start_time) + ) + with fluid.program_guard( + main_program=fluid.Program(), + startup_program=fluid.Program(), + ): test() assert abs(int(step_list[0] / 2) - int(step_list[1])) < 5 assert abs(int(step_list[1] / 2) - int(step_list[2])) < 5 diff --git a/python/paddle/fluid/tests/unittests/test_atan2_op.py b/python/paddle/fluid/tests/unittests/test_atan2_op.py index c0200d3c4a166..b74aeb9057004 100644 --- a/python/paddle/fluid/tests/unittests/test_atan2_op.py +++ b/python/paddle/fluid/tests/unittests/test_atan2_op.py @@ -30,7 +30,6 @@ def atan2_grad(x1, x2, dout): class TestAtan2(OpTest): - def setUp(self): self.op_type = "atan2" self.python_api = paddle.atan2 @@ -54,40 +53,39 @@ def init_dtype(self): class TestAtan2_float(TestAtan2): - def init_dtype(self): self.dtype = np.float32 def test_check_grad(self): if self.dtype not in [np.int32, np.int64]: - self.check_grad(['X1', 'X2'], - 'Out', - user_defined_grads=atan2_grad( - self.inputs['X1'], self.inputs['X2'], - 1 / self.inputs['X1'].size), - check_eager=True) + self.check_grad( + ['X1', 'X2'], + 'Out', + user_defined_grads=atan2_grad( + self.inputs['X1'], + self.inputs['X2'], + 1 / self.inputs['X1'].size, + ), + check_eager=True, + ) class TestAtan2_float16(TestAtan2_float): - def init_dtype(self): self.dtype = np.float16 class TestAtan2_int32(TestAtan2_float): - def init_dtype(self): self.dtype = np.int32 class TestAtan2_int64(TestAtan2_float): - def init_dtype(self): self.dtype = np.int64 class TestAtan2API(unittest.TestCase): - def init_dtype(self): self.dtype = 'float64' self.shape = [11, 17] @@ -118,7 +116,6 @@ def run(place): run(place) def test_dygraph_api(self): - def run(place): paddle.disable_static(place) X1 = paddle.to_tensor(self.x1) diff --git a/python/paddle/fluid/tests/unittests/test_attention_lstm_op.py b/python/paddle/fluid/tests/unittests/test_attention_lstm_op.py index b67ab48494a0f..053912005cc1b 100644 --- a/python/paddle/fluid/tests/unittests/test_attention_lstm_op.py +++ b/python/paddle/fluid/tests/unittests/test_attention_lstm_op.py @@ -20,17 +20,18 @@ def attention_lstm( - x, # T x M - lod, # 1 x N - h0, # N x D - c0, # N x D - fcws, # (M+D) x 1, 1x1 - fcbs, # 1 x 1, 1x1 - w, # (M+D) x 4D - b, # 1 x 4D - act_gate, - act_cell, - act_cand): + x, # T x M + lod, # 1 x N + h0, # N x D + c0, # N x D + fcws, # (M+D) x 1, 1x1 + fcbs, # 1 x 1, 1x1 + w, # (M+D) x 4D + b, # 1 x 4D + act_gate, + act_cell, + act_cand, +): T = sum(lod[0]) N = len(lod[0]) @@ -44,8 +45,9 @@ def attention_lstm( start_offset = 0 for bid in range(N): seq_len = lod[0][bid] - xi = np.copy(x[start_offset:start_offset + seq_len, :]).reshape( - seq_len, M) + xi = np.copy(x[start_offset : start_offset + seq_len, :]).reshape( + seq_len, M + ) prev_cell = np.copy(c0[bid]).reshape([1, D]) prev_hidden = np.copy(h0[bid]).reshape([1, D]) for step in range(seq_len): @@ -86,7 +88,6 @@ def attention_lstm( class TestAttentionLSTMOp(OpTest): - def set_conf(self): pass @@ -117,14 +118,24 @@ def setUp(self): fcb2 = np.random.normal(size=(1, 1)).astype('float32') # lstm weight and bias - w = np.random.normal(size=(self.M + self.D, - self.D * 4)).astype('float32') + w = np.random.normal(size=(self.M + self.D, self.D * 4)).astype( + 'float32' + ) b = np.random.normal(size=(1, self.D * 4)).astype('float32') - h, c = attention_lstm(x, self.lod, h0, c0, [fcw1, fcw2], [fcb1, fcb2], - w, b, ACTIVATION[self.act_gate], - ACTIVATION[self.act_cell], - ACTIVATION[self.act_cand]) + h, c = attention_lstm( + x, + self.lod, + h0, + c0, + [fcw1, fcw2], + [fcb1, fcb2], + w, + b, + ACTIVATION[self.act_gate], + ACTIVATION[self.act_cell], + ACTIVATION[self.act_cand], + ) self.inputs = { 'X': (x, self.lod), @@ -134,7 +145,7 @@ def setUp(self): 'AttentionScalar': fcw2, 'AttentionScalarBias': fcb2, 'LSTMWeight': w, - 'LSTMBias': b + 'LSTMBias': b, } if self.has_initial_hidden: @@ -147,7 +158,7 @@ def setUp(self): self.attrs = { 'gate_activation': self.act_gate, 'cell_activation': self.act_cell, - 'candidate_activation': self.act_cand + 'candidate_activation': self.act_cand, } def test_check_output(self): @@ -155,13 +166,11 @@ def test_check_output(self): class TestAttentionOpNonInit(TestAttentionLSTMOp): - def set_conf(self): self.has_initial_hidden = False class TestAttentionOpAct(TestAttentionLSTMOp): - def set_conf(self): self.M = 3 self.D = 2 @@ -171,28 +180,24 @@ def set_conf(self): class TestAttentionOpMD1(TestAttentionLSTMOp): - def set_conf(self): self.M = 36 self.D = 8 class TestAttentionOpMD2(TestAttentionLSTMOp): - def set_conf(self): self.M = 8 self.D = 8 class TestAttentionOpMD3(TestAttentionLSTMOp): - def set_conf(self): self.M = 15 self.D = 30 class TestAttentionOpBS1(TestAttentionLSTMOp): - def set_conf(self): self.lod = [[5]] self.M = 16 @@ -200,13 +205,11 @@ def set_conf(self): class TestAttentionOpBS2(TestAttentionLSTMOp): - def set_conf(self): self.lod = [[3, 6]] class TestAttentionOpBS5(TestAttentionLSTMOp): - def set_conf(self): self.lod = [[3, 2, 4, 7, 5]] diff --git a/python/paddle/fluid/tests/unittests/test_attribute_var.py b/python/paddle/fluid/tests/unittests/test_attribute_var.py index 6e8e3c6675087..60d202eb1b589 100644 --- a/python/paddle/fluid/tests/unittests/test_attribute_var.py +++ b/python/paddle/fluid/tests/unittests/test_attribute_var.py @@ -25,7 +25,6 @@ class UnittestBase(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.init_info() @@ -41,8 +40,9 @@ def path_prefix(self): return type(self).__name__ def infer_prog(self): - config = paddle_infer.Config(self.save_path + '.pdmodel', - self.save_path + '.pdiparams') + config = paddle_infer.Config( + self.save_path + '.pdmodel', self.save_path + '.pdiparams' + ) predictor = paddle_infer.create_predictor(config) input_names = predictor.get_input_names() for i, shape in enumerate(self.shapes): @@ -65,7 +65,6 @@ def infer_prog(self): class TestDropout(UnittestBase): - def init_info(self): self.shapes = [[10, 10]] self.save_path = os.path.join(self.temp_dir.name, 'dropout') @@ -98,11 +97,11 @@ def test_static(self): self.assertEqual( main_prog.block(0).ops[4].all_attrs()['dropout_prob'].name, - p.name) + p.name, + ) class TestTileTensorList(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.save_path = os.path.join(self.temp_dir.name, 'tile_tensors') @@ -136,7 +135,6 @@ def _test_static(self): class TestTileTensor(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.save_path = os.path.join(self.temp_dir.name, 'tile_tensor') @@ -169,13 +167,12 @@ def _test_static(self): class TestRegiterSupportTensorInOpMaker(unittest.TestCase): - def setUp(self): self.all_protos = OpProtoHolder.instance() self.support_tensor_attrs = { 'dropout': ['dropout_prob'], 'tile': ['repeat_times'], - 'concat': ['axis'] + 'concat': ['axis'], } # Just add a op example to test not support tensor self.not_support_tensor_attrs = {'svd': ['full_matrices']} @@ -189,8 +186,9 @@ def test_support_tensor(self): # All Attribute not tagged with .SupportTensor() in OpMaker will return False for op_type, attr_names in self.not_support_tensor_attrs.items(): for attr_name in attr_names: - self.assertFalse(self.is_support_tensor_attr( - op_type, attr_name)) + self.assertFalse( + self.is_support_tensor_attr(op_type, attr_name) + ) def is_support_tensor_attr(self, op_type, attr_name): proto = self.all_protos.get_op_proto(op_type) diff --git a/python/paddle/fluid/tests/unittests/test_auc_op.py b/python/paddle/fluid/tests/unittests/test_auc_op.py index 56c0b1c69778a..0ea67b19b13fc 100644 --- a/python/paddle/fluid/tests/unittests/test_auc_op.py +++ b/python/paddle/fluid/tests/unittests/test_auc_op.py @@ -21,7 +21,6 @@ class TestAucOp(OpTest): - def setUp(self): self.op_type = "auc" pred = np.random.random((128, 2)).astype("float32") @@ -30,25 +29,27 @@ def setUp(self): slide_steps = 1 stat_pos = np.zeros( - (1 + slide_steps) * (num_thresholds + 1) + 1, ).astype("int64") + (1 + slide_steps) * (num_thresholds + 1) + 1, + ).astype("int64") stat_neg = np.zeros( - (1 + slide_steps) * (num_thresholds + 1) + 1, ).astype("int64") + (1 + slide_steps) * (num_thresholds + 1) + 1, + ).astype("int64") self.inputs = { 'Predict': pred, 'Label': labels, "StatPos": stat_pos, - "StatNeg": stat_neg + "StatNeg": stat_neg, } self.attrs = { 'curve': 'ROC', 'num_thresholds': num_thresholds, - "slide_steps": slide_steps + "slide_steps": slide_steps, } - python_auc = metrics.Auc(name="auc", - curve='ROC', - num_thresholds=num_thresholds) + python_auc = metrics.Auc( + name="auc", curve='ROC', num_thresholds=num_thresholds + ) python_auc.update(pred, labels) pos = python_auc._stat_pos * 2 @@ -58,7 +59,7 @@ def setUp(self): self.outputs = { 'AUC': np.array(python_auc.eval()), 'StatPosOut': np.array(pos), - 'StatNegOut': np.array(neg) + 'StatNegOut': np.array(neg), } def test_check_output(self): @@ -66,7 +67,6 @@ def test_check_output(self): class TestGlobalAucOp(OpTest): - def setUp(self): self.op_type = "auc" pred = np.random.random((128, 2)).astype("float32") @@ -81,17 +81,17 @@ def setUp(self): 'Predict': pred, 'Label': labels, "StatPos": stat_pos, - "StatNeg": stat_neg + "StatNeg": stat_neg, } self.attrs = { 'curve': 'ROC', 'num_thresholds': num_thresholds, - "slide_steps": slide_steps + "slide_steps": slide_steps, } - python_auc = metrics.Auc(name="auc", - curve='ROC', - num_thresholds=num_thresholds) + python_auc = metrics.Auc( + name="auc", curve='ROC', num_thresholds=num_thresholds + ) python_auc.update(pred, labels) pos = python_auc._stat_pos @@ -99,7 +99,7 @@ def setUp(self): self.outputs = { 'AUC': np.array(python_auc.eval()), 'StatPosOut': np.array(pos), - 'StatNegOut': np.array(neg) + 'StatNegOut': np.array(neg), } def test_check_output(self): @@ -107,17 +107,16 @@ def test_check_output(self): class TestAucAPI(unittest.TestCase): - def test_static(self): paddle.enable_static() data = paddle.static.data(name="input", shape=[-1, 1], dtype="float32") label = paddle.static.data(name="label", shape=[4], dtype="int64") - ins_tag_weight = paddle.static.data(name="ins_tag_weight", - shape=[4], - dtype="float32") - result = paddle.static.auc(input=data, - label=label, - ins_tag_weight=ins_tag_weight) + ins_tag_weight = paddle.static.data( + name="ins_tag_weight", shape=[4], dtype="float32" + ) + result = paddle.static.auc( + input=data, label=label, ins_tag_weight=ins_tag_weight + ) place = paddle.CPUPlace() exe = paddle.static.Executor(place) @@ -128,37 +127,34 @@ def test_static(self): y = np.array([0, 0, 1, 0]).astype('int64') z = np.array([1, 1, 1, 1]).astype('float32') - output, = exe.run(feed={ - "input": x, - "label": y, - "ins_tag_weight": z - }, - fetch_list=[result[0]]) + (output,) = exe.run( + feed={"input": x, "label": y, "ins_tag_weight": z}, + fetch_list=[result[0]], + ) auc_np = np.array([0.66666667]).astype("float32") np.testing.assert_allclose(output, auc_np, rtol=1e-05) class TestAucOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): def test_type1(): data1 = fluid.data(name="input1", shape=[-1, 2], dtype="int") label1 = fluid.data(name="label1", shape=[-1], dtype="int") - ins_tag_w1 = paddle.static.data(name="label1", - shape=[-1], - dtype="int") - result1 = paddle.static.auc(input=data1, - label=label1, - ins_tag_weight=ins_tag_w1) + ins_tag_w1 = paddle.static.data( + name="label1", shape=[-1], dtype="int" + ) + result1 = paddle.static.auc( + input=data1, label=label1, ins_tag_weight=ins_tag_w1 + ) self.assertRaises(TypeError, test_type1) def test_type2(): - data2 = fluid.data(name="input2", - shape=[-1, 2], - dtype="float32") + data2 = fluid.data( + name="input2", shape=[-1, 2], dtype="float32" + ) label2 = fluid.data(name="label2", shape=[-1], dtype="float32") result2 = fluid.layers.auc(input=data2, label=label2) diff --git a/python/paddle/fluid/tests/unittests/test_auc_single_pred_op.py b/python/paddle/fluid/tests/unittests/test_auc_single_pred_op.py index 28e2808be350d..13482d0af3aba 100644 --- a/python/paddle/fluid/tests/unittests/test_auc_single_pred_op.py +++ b/python/paddle/fluid/tests/unittests/test_auc_single_pred_op.py @@ -19,7 +19,6 @@ class TestAucSinglePredOp(OpTest): - def setUp(self): self.op_type = "auc" pred = np.random.random((128, 2)).astype("float32") @@ -29,25 +28,27 @@ def setUp(self): slide_steps = 1 stat_pos = np.zeros( - (1 + slide_steps) * (num_thresholds + 1) + 1, ).astype("int64") + (1 + slide_steps) * (num_thresholds + 1) + 1, + ).astype("int64") stat_neg = np.zeros( - (1 + slide_steps) * (num_thresholds + 1) + 1, ).astype("int64") + (1 + slide_steps) * (num_thresholds + 1) + 1, + ).astype("int64") self.inputs = { 'Predict': pred0, 'Label': labels, "StatPos": stat_pos, - "StatNeg": stat_neg + "StatNeg": stat_neg, } self.attrs = { 'curve': 'ROC', 'num_thresholds': num_thresholds, - "slide_steps": slide_steps + "slide_steps": slide_steps, } - python_auc = metrics.Auc(name="auc", - curve='ROC', - num_thresholds=num_thresholds) + python_auc = metrics.Auc( + name="auc", curve='ROC', num_thresholds=num_thresholds + ) for i in range(128): pred[i][1] = pred[i][0] python_auc.update(pred, labels) @@ -59,7 +60,7 @@ def setUp(self): self.outputs = { 'AUC': np.array(python_auc.eval()), 'StatPosOut': np.array(pos), - 'StatNegOut': np.array(neg) + 'StatNegOut': np.array(neg), } def test_check_output(self): @@ -67,7 +68,6 @@ def test_check_output(self): class TestAucGlobalSinglePredOp(OpTest): - def setUp(self): self.op_type = "auc" pred = np.random.random((128, 2)).astype("float32") @@ -83,17 +83,17 @@ def setUp(self): 'Predict': pred0, 'Label': labels, "StatPos": stat_pos, - "StatNeg": stat_neg + "StatNeg": stat_neg, } self.attrs = { 'curve': 'ROC', 'num_thresholds': num_thresholds, - "slide_steps": slide_steps + "slide_steps": slide_steps, } - python_auc = metrics.Auc(name="auc", - curve='ROC', - num_thresholds=num_thresholds) + python_auc = metrics.Auc( + name="auc", curve='ROC', num_thresholds=num_thresholds + ) for i in range(128): pred[i][1] = pred[i][0] python_auc.update(pred, labels) @@ -103,7 +103,7 @@ def setUp(self): self.outputs = { 'AUC': np.array(python_auc.eval()), 'StatPosOut': np.array(pos), - 'StatNegOut': np.array(neg) + 'StatNegOut': np.array(neg), } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/test_auto_growth_gpu_memory_limit.py b/python/paddle/fluid/tests/unittests/test_auto_growth_gpu_memory_limit.py index 948aa58990d0c..7f4f6c78c1945 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_growth_gpu_memory_limit.py +++ b/python/paddle/fluid/tests/unittests/test_auto_growth_gpu_memory_limit.py @@ -23,7 +23,6 @@ class TestBase(unittest.TestCase): - def setUp(self): if fluid.is_compiled_with_cuda(): self._limit = fluid.core.globals()['FLAGS_gpu_memory_limit_mb'] @@ -36,8 +35,10 @@ def test_allocate(self): place = fluid.CUDAPlace(0) t = fluid.LoDTensor() - t.set(np.ndarray([int(self._limit / 2), other_dim], dtype='float32'), - place) + t.set( + np.ndarray([int(self._limit / 2), other_dim], dtype='float32'), + place, + ) del t t = fluid.LoDTensor() diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_autoconvert.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_autoconvert.py index 0777f5217b573..82969299efd7d 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_autoconvert.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_autoconvert.py @@ -18,7 +18,6 @@ class TestAutoParallelAutoConvert(TestMultipleGpus): - def test_auto_parallel_autoconvert(self): self.run_mnist_2gpu('auto_parallel_autoconvert.py') diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_cluster.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_cluster.py index df11100755ef3..a632d29f6915b 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_cluster.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_cluster.py @@ -199,7 +199,6 @@ class TestAutoParallelCluster(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -207,8 +206,9 @@ def tearDown(self): self.temp_dir.cleanup() def test_cluster(self): - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) @@ -295,8 +295,9 @@ def test_cluster(self): self.assertEqual(device2_machine0.global_id, 2) self.assertEqual(device2_machine0.local_id, 0) self.assertEqual(device2_machine0.type, DeviceType.CPU) - self.assertEqual(device2_machine0.model, - "Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GH") + self.assertEqual( + device2_machine0.model, "Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GH" + ) self.assertAlmostEqual(device2_machine0.sp_gflops, 150) self.assertAlmostEqual(device2_machine0.dp_gflops, 75) self.assertAlmostEqual(device2_machine0.memory, 1510) @@ -401,8 +402,9 @@ def test_cluster(self): self.assertEqual(device6_machine1.global_id, 6) self.assertEqual(device6_machine1.local_id, 0) self.assertEqual(device6_machine1.type, DeviceType.CPU) - self.assertEqual(device6_machine1.model, - "Intel(R) Xeon(R) Gold 6271C CPU @ 2.60G") + self.assertEqual( + device6_machine1.model, "Intel(R) Xeon(R) Gold 6271C CPU @ 2.60G" + ) self.assertAlmostEqual(device6_machine1.sp_gflops, 150) self.assertAlmostEqual(device6_machine1.dp_gflops, 75) self.assertAlmostEqual(device6_machine1.memory, 503) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_completion.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_completion.py index c757e3a45afba..5e1659b49c1b5 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_completion.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_completion.py @@ -33,38 +33,42 @@ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") def forward(self, input): if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.linear0.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.linear1.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.linear0.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.linear1.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) out = self.norm(input) out = self.linear0(out) @@ -76,79 +80,93 @@ def forward(self, input): def mlp_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input, - process_mesh=_global_process_mesh, - shard_spec=["dp", None, None]) - - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + auto.shard_tensor( + input, + process_mesh=_global_process_mesh, + shard_spec=["dp", None, None], + ) + + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) out = mlp(input) return train_program, start_program class TestMLPAutoCompletion(unittest.TestCase): - def test_mlp_dp(self): global _global_parallel_strategy _global_parallel_strategy = "dp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["dp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["dp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = mlp_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_mlp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = mlp_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_mlp_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = mlp_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) # def test_mlp_misc(self): @@ -203,14 +221,15 @@ def test_mlp_dp_mp(self): class AttentionLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - sequence_len=512, - intermediate_size=4 * 1024, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + sequence_len=512, + intermediate_size=4 * 1024, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(AttentionLayer, self).__init__() self.hidden_size = hidden_size self.sequence_len = sequence_len @@ -219,38 +238,38 @@ def __init__(self, self.vdim = self.embed_dim self.num_heads = num_heads self.head_dim = self.embed_dim // self.num_heads - assert self.head_dim * self.num_heads == self.embed_dim, \ - "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * self.num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" self.dropout_ratio = dropout_ratio self.initializer_range = initializer_range self.training = True self.attn_mask = None weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.q_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.k_proj = nn.Linear(self.kdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.v_proj = nn.Linear(self.vdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.out_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) + self.q_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.k_proj = nn.Linear( + self.kdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.v_proj = nn.Linear( + self.vdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.out_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) def forward(self, input): if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input, - process_mesh=_global_process_mesh, - shard_spec=["dp", None, None]) + auto.shard_tensor( + input, + process_mesh=_global_process_mesh, + shard_spec=["dp", None, None], + ) q = self.q_proj(input) q = tensor.reshape(x=q, shape=[0, 0, self.num_heads, self.head_dim]) @@ -260,15 +279,21 @@ def forward(self, input): v = self.v_proj(input) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.q_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.k_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.v_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.q_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.k_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.v_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) k = tensor.transpose(x=k, perm=[0, 2, 1, 3]) @@ -276,10 +301,9 @@ def forward(self, input): v = tensor.transpose(x=v, perm=[0, 2, 1, 3]) # scale dot product attention - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.head_dim**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.head_dim**-0.5 + ) if self.attn_mask is not None: product = product + self.attn_mask @@ -287,10 +311,12 @@ def forward(self, input): weights = F.softmax(product) if self.dropout_ratio: - weights = F.dropout(weights, - self.dropout_ratio, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout_ratio, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) @@ -301,98 +327,113 @@ def forward(self, input): # project to output out = self.out_proj(out) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.out_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.out_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) return out def attn_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="query", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - attn = AttentionLayer(hidden_size=hidden_size, - sequence_len=sequence_len, - intermediate_size=4 * hidden_size, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02) + input = static.data( + name="query", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + attn = AttentionLayer( + hidden_size=hidden_size, + sequence_len=sequence_len, + intermediate_size=4 * hidden_size, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ) out = attn(input) return train_program, start_program class TestAttentionAutoCompletion(unittest.TestCase): - def test_attn_dp(self): global _global_parallel_strategy _global_parallel_strategy = "dp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["dp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["dp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = attn_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_attn_mp(self): global _global_parallel_strategy _global_parallel_strategy = "mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = attn_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_attn_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = attn_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) class DecoderLayer(nn.Layer): - - def __init__(self, - vocab_size=32768, - hidden_size=1024, - sequence_len=512, - max_position_embeddings=512, - intermediate_size=4 * 1024, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + vocab_size=32768, + hidden_size=1024, + sequence_len=512, + max_position_embeddings=512, + intermediate_size=4 * 1024, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(DecoderLayer, self).__init__() self.vocab_size = vocab_size self.hidden_size = hidden_size @@ -408,57 +449,64 @@ def __init__(self, self.attn_mask = None self.head_dim = self.embed_dim // self.num_heads - assert self.head_dim * self.num_heads == self.embed_dim, \ - "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * self.num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" self.word_embeddings = nn.Embedding( self.vocab_size, self.hidden_size, - weight_attr=paddle.ParamAttr(name="word_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, - std=self.initializer_range))) + weight_attr=paddle.ParamAttr( + name="word_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ), + ), + ) self.position_embeddings = nn.Embedding( self.max_position_embeddings, self.hidden_size, - weight_attr=paddle.ParamAttr(name="pos_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, - std=self.initializer_range))) + weight_attr=paddle.ParamAttr( + name="pos_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ), + ), + ) - weight_attr = paddle.ParamAttr(initializer=nn.initializer.Normal( - mean=0.0, std=self.initializer_range)) + weight_attr = paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ) + ) bias_attr = None - self.q_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.k_proj = nn.Linear(self.kdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.v_proj = nn.Linear(self.vdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.out_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) + self.q_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.k_proj = nn.Linear( + self.kdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.v_proj = nn.Linear( + self.vdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.out_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) intermediate_size = 4 * self.hidden_size d_model = self.hidden_size dim_feedforward = intermediate_size - weight_attr = paddle.ParamAttr(initializer=nn.initializer.Normal( - mean=0.0, std=self.initializer_range)) + weight_attr = paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm1 = nn.LayerNorm(d_model, epsilon=1e-5) self.norm2 = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout1 = nn.Dropout(self.dropout_ratio) @@ -467,17 +515,21 @@ def __init__(self, def forward(self, input_ids, position_ids): if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input_ids, - process_mesh=_global_process_mesh, - shard_spec=["dp", None]) + auto.shard_tensor( + input_ids, + process_mesh=_global_process_mesh, + shard_spec=["dp", None], + ) input_embeddings = self.word_embeddings(input_ids) position_embeddings = self.position_embeddings(position_ids) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.word_embeddings.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.word_embeddings.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) embeddings = input_embeddings + position_embeddings embeddings = self.dropout1(embeddings) @@ -494,15 +546,21 @@ def forward(self, input_ids, position_ids): v = self.v_proj(target) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.q_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.k_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.v_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.q_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.k_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.v_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) k = tensor.transpose(x=k, perm=[0, 2, 1, 3]) @@ -510,10 +568,9 @@ def forward(self, input_ids, position_ids): v = tensor.transpose(x=v, perm=[0, 2, 1, 3]) # scale dot product attention - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.head_dim**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.head_dim**-0.5 + ) if self.attn_mask is not None: product = product + self.attn_mask @@ -521,10 +578,12 @@ def forward(self, input_ids, position_ids): weights = F.softmax(product) if self.dropout_ratio: - weights = F.dropout(weights, - self.dropout_ratio, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout_ratio, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) @@ -536,9 +595,11 @@ def forward(self, input_ids, position_ids): out = self.out_proj(out) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.out_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.out_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) # Add residual residual = embeddings + self.dropout2(out) @@ -552,12 +613,16 @@ def forward(self, input_ids, position_ids): out3 = self.linear1(out2) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.linear0.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.linear1.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.linear0.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.linear1.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) # Add residual final = residual + self.dropout3(out3) @@ -565,81 +630,91 @@ def forward(self, input_ids, position_ids): def decoder_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input_ids = static.data(name="input_ids", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') - decoder = DecoderLayer(vocab_size=32768, - hidden_size=hidden_size, - sequence_len=sequence_len, - max_position_embeddings=512, - intermediate_size=4 * hidden_size, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02) + input_ids = static.data( + name="input_ids", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) + decoder = DecoderLayer( + vocab_size=32768, + hidden_size=hidden_size, + sequence_len=sequence_len, + max_position_embeddings=512, + intermediate_size=4 * hidden_size, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ) out = decoder(input_ids, position_ids) return train_program, start_program class TestDecoderLayerAutoCompletion(unittest.TestCase): - def test_decoder_dp(self): global _global_parallel_strategy _global_parallel_strategy = "dp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["dp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["dp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = decoder_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_decoder_mp(self): global _global_parallel_strategy _global_parallel_strategy = "mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = decoder_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_decoder_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = decoder_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_completion_gpt.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_completion_gpt.py index 239033e7c8b94..0511ee97e55eb 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_completion_gpt.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_completion_gpt.py @@ -42,17 +42,19 @@ class MultiHeadAttention(nn.Layer): Cache = collections.namedtuple("Cache", ["k", "v"]) StaticCache = collections.namedtuple("StaticCache", ["k", "v"]) - def __init__(self, - embed_dim, - num_heads, - dropout=0., - kdim=None, - vdim=None, - need_weights=False, - weight_attr=None, - bias_attr=None, - topo=None, - fuse=False): + def __init__( + self, + embed_dim, + num_heads, + dropout=0.0, + kdim=None, + vdim=None, + need_weights=False, + weight_attr=None, + bias_attr=None, + topo=None, + fuse=False, + ): super(MultiHeadAttention, self).__init__() self.embed_dim = embed_dim self.kdim = kdim if kdim is not None else embed_dim @@ -63,38 +65,36 @@ def __init__(self, self.fuse = fuse self.head_dim = embed_dim // num_heads - assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" if topo is None or topo.mp_info.size == 1: if self.fuse: assert self.kdim == embed_dim assert self.vdim == embed_dim - self.qkv_proj = nn.Linear(embed_dim, - 3 * embed_dim, - weight_attr, - bias_attr=bias_attr) + self.qkv_proj = nn.Linear( + embed_dim, 3 * embed_dim, weight_attr, bias_attr=bias_attr + ) else: - self.q_proj = nn.Linear(embed_dim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.k_proj = nn.Linear(self.kdim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.v_proj = nn.Linear(self.vdim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.out_proj = nn.Linear(embed_dim, - embed_dim, - weight_attr, - bias_attr=bias_attr) + self.q_proj = nn.Linear( + embed_dim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.k_proj = nn.Linear( + self.kdim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.v_proj = nn.Linear( + self.vdim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.out_proj = nn.Linear( + embed_dim, embed_dim, weight_attr, bias_attr=bias_attr + ) def _fuse_prepare_qkv(self, query): mix_layer = self.qkv_proj(query) - mix_layer = paddle.reshape_(mix_layer, - [0, 0, self.num_heads, 3 * self.head_dim]) + mix_layer = paddle.reshape_( + mix_layer, [0, 0, self.num_heads, 3 * self.head_dim] + ) mix_layer = paddle.transpose(mix_layer, [0, 2, 1, 3]) q, k, v = paddle.split(mix_layer, num_or_sections=3, axis=-1) return q, k, v @@ -108,9 +108,11 @@ def _prepare_qkv(self, query, key, value, use_cache=False, cache=None): q = self.q_proj(query) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.q_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.q_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) q = tensor.reshape(x=q, shape=[0, 0, self.num_heads, self.head_dim]) q = tensor.transpose(x=q, perm=[0, 2, 1, 3]) @@ -144,12 +146,16 @@ def compute_kv(self, key, value): v = self.v_proj(value) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.k_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.v_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.k_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.v_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) k = tensor.transpose(x=k, perm=[0, 2, 1, 3]) @@ -171,24 +177,22 @@ def gen_cache(self, key, value=None, type=Cache): input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) v = layers.fill_constant_batch_size_like( input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) return self.Cache(k, v) else: # incremental_state with initial value, mainly for usage like UniLM return self.Cache(key, value) - def forward(self, - query, - key, - value, - attn_mask=None, - use_cache=False, - cache=None): + def forward( + self, query, key, value, attn_mask=None, use_cache=False, cache=None + ): r""" Applies multi-head attention to map queries and a set of key-value pairs to outputs. @@ -202,23 +206,25 @@ def forward(self, else: q, k, v = self._prepare_qkv(query, key, value, use_cache, cache) else: - q, k, v, cache = self._prepare_qkv(query, key, value, use_cache, - cache) + q, k, v, cache = self._prepare_qkv( + query, key, value, use_cache, cache + ) # scale dot product attention - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.head_dim**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.head_dim**-0.5 + ) if attn_mask is not None: product = product + attn_mask weights = F.softmax(product) if self.dropout: - weights = F.dropout(weights, - self.dropout, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) @@ -230,9 +236,11 @@ def forward(self, out = self.out_proj(out) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.out_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.out_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) outs = [out] if self.need_weights: @@ -247,12 +255,9 @@ class TransformerDecoder(nn.Layer): TransformerDecoder is a stack of N decoder layers. """ - def __init__(self, - decoder_layers, - num_layers, - norm=None, - hidden_size=None, - topo=None): + def __init__( + self, decoder_layers, num_layers, norm=None, hidden_size=None, topo=None + ): super(TransformerDecoder, self).__init__() self.topo = topo @@ -265,13 +270,15 @@ def __init__(self, raise ValueError("Only support LayerNorm") self.checkpoints = [] - def forward(self, - tgt, - memory, - tgt_mask=None, - memory_mask=None, - use_cache=False, - cache=None): + def forward( + self, + tgt, + memory, + tgt_mask=None, + memory_mask=None, + use_cache=False, + cache=None, + ): r""" Applies a stack of N Transformer decoder layers on inputs. If `norm` is provided, also applies layer normalization on the output of last decoder @@ -284,25 +291,31 @@ def forward(self, for i, mod in enumerate(self.layers): if cache is None: if use_cache: - output, new_cache = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache) + output, new_cache = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache, + ) new_caches.append(new_cache) else: - output = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache) + output = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache, + ) else: - output, new_cache = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache[i]) + output, new_cache = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache[i], + ) new_caches.append(new_cache) self.checkpoints.append(output.name) @@ -317,7 +330,7 @@ def gen_cache(self, memory, do_zip=False): produced by `TransformerDecoderLayer.gen_cache`. See `TransformerDecoderLayer.gen_cache` for more details. If `do_zip` is True, apply `zip` on these tuples to get a list with two elements. - """ + """ cache = [layer.gen_cache(memory) for layer in self.layers] if do_zip: cache = list(zip(*cache)) @@ -330,18 +343,20 @@ class TransformerDecoderLayer(nn.Layer): It contains multiheadattention and some linear layers. """ - def __init__(self, - d_model, - nhead, - dim_feedforward, - dropout=0.1, - activation="gelu", - attn_dropout=None, - act_dropout=None, - normalize_before=True, - weight_attr=None, - bias_attr=None, - topo=None): + def __init__( + self, + d_model, + nhead, + dim_feedforward, + dropout=0.1, + activation="gelu", + attn_dropout=None, + act_dropout=None, + normalize_before=True, + weight_attr=None, + bias_attr=None, + topo=None, + ): self._config = locals() self._config.pop("self") self._config.pop("__class__", None) # py3 @@ -354,21 +369,27 @@ def __init__(self, weight_attrs = _convert_param_attr_to_list(weight_attr, 3) bias_attrs = _convert_param_attr_to_list(bias_attr, 3) - self.self_attn = MultiHeadAttention(d_model, - nhead, - dropout=attn_dropout, - weight_attr=weight_attrs[0], - bias_attr=bias_attrs[0], - topo=topo) + self.self_attn = MultiHeadAttention( + d_model, + nhead, + dropout=attn_dropout, + weight_attr=weight_attrs[0], + bias_attr=bias_attrs[0], + topo=topo, + ) if topo is None or topo.mp_info.size == 1: - self.linear1 = nn.Linear(d_model, - dim_feedforward, - weight_attrs[2], - bias_attr=bias_attrs[2]) - self.linear2 = nn.Linear(dim_feedforward, - d_model, - weight_attrs[2], - bias_attr=bias_attrs[2]) + self.linear1 = nn.Linear( + d_model, + dim_feedforward, + weight_attrs[2], + bias_attr=bias_attrs[2], + ) + self.linear2 = nn.Linear( + dim_feedforward, + d_model, + weight_attrs[2], + bias_attr=bias_attrs[2], + ) self.norm1 = nn.LayerNorm(d_model, epsilon=1e-5) self.norm2 = nn.LayerNorm(d_model, epsilon=1e-5) @@ -385,8 +406,9 @@ def forward(self, tgt, memory, tgt_mask=None, use_cache=False, cache=None): if use_cache is False: tgt = self.self_attn(tgt, tgt, tgt, tgt_mask, use_cache, cache) else: - tgt, incremental_cache = self.self_attn(tgt, tgt, tgt, tgt_mask, - use_cache, cache) + tgt, incremental_cache = self.self_attn( + tgt, tgt, tgt, tgt_mask, use_cache, cache + ) tgt = residual + self.dropout1(tgt) if not self.normalize_before: tgt = self.norm1(tgt) @@ -396,12 +418,16 @@ def forward(self, tgt, memory, tgt_mask=None, use_cache=False, cache=None): tgt = self.norm2(tgt) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.linear1.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.linear2.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.linear1.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.linear2.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) # tgt = self.dropout2( # self.linear2(F.gelu( @@ -417,8 +443,9 @@ def forward(self, tgt, memory, tgt_mask=None, use_cache=False, cache=None): return tgt if use_cache is False else (tgt, incremental_cache) def gen_cache(self, memory): - incremental_cache = self.self_attn.gen_cache(memory, - type=self.self_attn.Cache) + incremental_cache = self.self_attn.gen_cache( + memory, type=self.self_attn.Cache + ) return incremental_cache @@ -427,29 +454,38 @@ class GPTEmbeddings(nn.Layer): Include embeddings from word, position and token_type embeddings """ - def __init__(self, - vocab_size, - hidden_size=768, - hidden_dropout_prob=0.1, - max_position_embeddings=512, - type_vocab_size=16, - initializer_range=0.02, - topo=None): + def __init__( + self, + vocab_size, + hidden_size=768, + hidden_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + topo=None, + ): super(GPTEmbeddings, self).__init__() if topo is None or topo.mp_info.size == 1: self.word_embeddings = nn.Embedding( vocab_size, hidden_size, - weight_attr=paddle.ParamAttr(name="word_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, - std=initializer_range))) + weight_attr=paddle.ParamAttr( + name="word_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ), + ), + ) self.position_embeddings = nn.Embedding( max_position_embeddings, hidden_size, - weight_attr=paddle.ParamAttr(name="pos_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, std=initializer_range))) + weight_attr=paddle.ParamAttr( + name="pos_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ), + ), + ) self.dropout = nn.Dropout(hidden_dropout_prob) @@ -462,9 +498,11 @@ def forward(self, input_ids, position_ids=None): input_embedings = self.word_embeddings(input_ids) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.word_embeddings.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.word_embeddings.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) position_embeddings = self.position_embeddings(position_ids) embeddings = input_embedings + position_embeddings @@ -477,20 +515,22 @@ class GPTModel(nn.Layer): The base model of gpt. """ - def __init__(self, - vocab_size, - hidden_size=768, - num_hidden_layers=12, - num_attention_heads=12, - intermediate_size=3072, - hidden_act="gelu", - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - max_position_embeddings=512, - type_vocab_size=16, - initializer_range=0.02, - pad_token_id=0, - topo=None): + def __init__( + self, + vocab_size, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + pad_token_id=0, + topo=None, + ): super(GPTModel, self).__init__() self.pad_token_id = pad_token_id @@ -503,71 +543,94 @@ def __init__(self, if self.pipline_mode: self.layer_per_stage = num_hidden_layers // self.topo.pp_info.size - self.embeddings = GPTEmbeddings(vocab_size, hidden_size, - hidden_dropout_prob, - max_position_embeddings, - type_vocab_size, self.initializer_range, - topo) + self.embeddings = GPTEmbeddings( + vocab_size, + hidden_size, + hidden_dropout_prob, + max_position_embeddings, + type_vocab_size, + self.initializer_range, + topo, + ) decoder_layers = nn.LayerList() for i in range(num_hidden_layers): DecoderLayer = TransformerDecoderLayer decoder_layers.append( - DecoderLayer(d_model=hidden_size, - nhead=num_attention_heads, - dim_feedforward=intermediate_size, - dropout=hidden_dropout_prob, - activation=hidden_act, - attn_dropout=attention_probs_dropout_prob, - act_dropout=hidden_dropout_prob, - weight_attr=paddle.ParamAttr( - initializer=nn.initializer.Normal( - mean=0.0, std=self.initializer_range)), - bias_attr=None, - topo=topo)) + DecoderLayer( + d_model=hidden_size, + nhead=num_attention_heads, + dim_feedforward=intermediate_size, + dropout=hidden_dropout_prob, + activation=hidden_act, + attn_dropout=attention_probs_dropout_prob, + act_dropout=hidden_dropout_prob, + weight_attr=paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ) + ), + bias_attr=None, + topo=topo, + ) + ) Decoder = TransformerDecoder - self.decoder = Decoder(decoder_layers, - num_hidden_layers, - norm="LayerNorm", - hidden_size=hidden_size, - topo=topo) + self.decoder = Decoder( + decoder_layers, + num_hidden_layers, + norm="LayerNorm", + hidden_size=hidden_size, + topo=topo, + ) self.checkpoints = [] - def forward(self, - input_ids, - position_ids=None, - attention_mask=None, - use_cache=False, - cache=None): + def forward( + self, + input_ids, + position_ids=None, + attention_mask=None, + use_cache=False, + cache=None, + ): self.checkpoints = [] if attention_mask is None: length = paddle.shape(input_ids)[1] # Use bool mask attention_mask = paddle.tensor.tril( - paddle.ones((length, length), - dtype=self.embeddings.word_embeddings.weight.dtype)) + paddle.ones( + (length, length), + dtype=self.embeddings.word_embeddings.weight.dtype, + ) + ) if position_ids is None: past_length = 0 if cache is not None: past_length = paddle.shape(cache[0].k)[-2] - position_ids = paddle.arange(past_length, - paddle.shape(input_ids)[-1] + - past_length, - dtype='int64') + position_ids = paddle.arange( + past_length, + paddle.shape(input_ids)[-1] + past_length, + dtype='int64', + ) position_ids = position_ids.unsqueeze(0) # .expand_as(input_ids) position_ids = paddle.fluid.layers.expand_as( - position_ids, input_ids) - embedding_output = self.embeddings(input_ids=input_ids, - position_ids=position_ids) + position_ids, input_ids + ) + embedding_output = self.embeddings( + input_ids=input_ids, position_ids=position_ids + ) # TODO, use registered buffer - causal_mask = paddle.tensor.triu(paddle.ones( - (paddle.shape(input_ids)[-1], paddle.shape(input_ids)[-1])) * -1e9, - diagonal=1) + causal_mask = paddle.tensor.triu( + paddle.ones( + (paddle.shape(input_ids)[-1], paddle.shape(input_ids)[-1]) + ) + * -1e9, + diagonal=1, + ) if attention_mask is not None: attention_mask = attention_mask + causal_mask @@ -577,11 +640,13 @@ def forward(self, # The tensor returned by triu not in static graph. attention_mask.stop_gradient = True - encoder_outputs = self.decoder(embedding_output, - memory=None, - tgt_mask=attention_mask, - use_cache=use_cache, - cache=cache) + encoder_outputs = self.decoder( + embedding_output, + memory=None, + tgt_mask=attention_mask, + use_cache=use_cache, + cache=cache, + ) self.checkpoints.extend(self.decoder.checkpoints) return encoder_outputs @@ -603,11 +668,12 @@ def __init__(self, gpt): def parallel_matmul(self, lm_output, logit_weights, parallel_output, topo): if topo is not None and topo.mp_info.size > 1: input_parallel = paddle.distributed.collective._c_identity( - lm_output, group=None) + lm_output, group=None + ) - logits = paddle.matmul(input_parallel, - logit_weights, - transpose_y=True) + logits = paddle.matmul( + input_parallel, logit_weights, transpose_y=True + ) if parallel_output: return logits @@ -617,24 +683,29 @@ def parallel_matmul(self, lm_output, logit_weights, parallel_output, topo): logits = paddle.matmul(lm_output, logit_weights, transpose_y=True) return logits - def forward(self, - input_ids, - position_ids=None, - attention_mask=None, - masked_positions=None, - use_cache=False, - cache=None): - outputs = self.gpt(input_ids, - position_ids=position_ids, - attention_mask=attention_mask, - use_cache=use_cache, - cache=cache) + def forward( + self, + input_ids, + position_ids=None, + attention_mask=None, + masked_positions=None, + use_cache=False, + cache=None, + ): + outputs = self.gpt( + input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + use_cache=use_cache, + cache=cache, + ) if use_cache: encoder_outputs, cached_kvs = outputs[:2] else: encoder_outputs = outputs - logits = self.parallel_matmul(encoder_outputs, self.weight, True, - self.gpt.topo) + logits = self.parallel_matmul( + encoder_outputs, self.weight, True, self.gpt.topo + ) if use_cache: return logits, cached_kvs @@ -653,11 +724,14 @@ def __init__(self, topo=None): if topo is None or topo.mp_info.size == 1: self.loss_func = paddle.nn.CrossEntropyLoss(reduction="none") else: - self.loss_func = paddle.distributed.collective._c_softmax_with_cross_entropy + self.loss_func = ( + paddle.distributed.collective._c_softmax_with_cross_entropy + ) def forward(self, prediction_scores, masked_lm_labels, loss_mask): - masked_lm_loss = self.loss_func(prediction_scores, - masked_lm_labels.unsqueeze(2)) + masked_lm_loss = self.loss_func( + prediction_scores, masked_lm_labels.unsqueeze(2) + ) loss_mask = loss_mask.reshape([-1]) masked_lm_loss = paddle.sum(masked_lm_loss.reshape([-1]) * loss_mask) @@ -666,45 +740,51 @@ def forward(self, prediction_scores, masked_lm_labels, loss_mask): def gpt_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 16 sequence_len = 512 - input_ids = static.data(name="input_ids", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + input_ids = static.data( + name="input_ids", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = static.data( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float64') - labels = static.data(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = static.data(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float64') + dtype='float64', + ) + labels = static.data( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = static.data( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float64' + ) if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input_ids, - process_mesh=_global_process_mesh, - shard_spec=["dp", None]) - - gpt = GPTModel(vocab_size=32768, - hidden_size=1024, - num_hidden_layers=2, - num_attention_heads=16, - intermediate_size=4096, - hidden_act="gelu", - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - max_position_embeddings=1024, - type_vocab_size=16, - initializer_range=0.02, - pad_token_id=0, - topo=None) + auto.shard_tensor( + input_ids, + process_mesh=_global_process_mesh, + shard_spec=["dp", None], + ) + + gpt = GPTModel( + vocab_size=32768, + hidden_size=1024, + num_hidden_layers=2, + num_attention_heads=16, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=1024, + type_vocab_size=16, + initializer_range=0.02, + pad_token_id=0, + topo=None, + ) model = GPTForPretraining(gpt) @@ -718,57 +798,64 @@ def gpt_pretrain_forward(train_program, start_program): class TestGPTAutoCompletion(unittest.TestCase): - def test_gpt_dp(self): global _global_parallel_strategy _global_parallel_strategy = "dp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["dp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["dp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = gpt_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_gpt_mp(self): global _global_parallel_strategy _global_parallel_strategy = "mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = gpt_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) def test_gpt_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) train_program = static.Program() start_program = static.Program() dist_context = DistributedContext() train_program, start_program = gpt_pretrain_forward( - train_program, start_program) + train_program, start_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) self.assertTrue(dist_context.validate_dist_attr_for_program()) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_cost_model.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_cost_model.py index 821fee4df16a7..bef9619ead3ea 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_cost_model.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_cost_model.py @@ -42,27 +42,27 @@ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=256, - intermediate_size=4 * 256, - initializer_range=0.02, - is_distributed=True): + def __init__( + self, + hidden_size=256, + intermediate_size=4 * 256, + initializer_range=0.02, + is_distributed=True, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.is_distributed = is_distributed @@ -84,13 +84,14 @@ def get_single_node_data(): train_program = paddle.static.Program() startup_program = paddle.static.Program() - loss, train_program, startup_program = mlp_forward(train_program, - startup_program, - is_distributed=False) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program, is_distributed=False + ) cost_model = core.CostModel() - cost_data = cost_model.profile_measure(train_program, startup_program, - device, ["time"]) + cost_data = cost_model.profile_measure( + train_program, startup_program, device, ["time"] + ) op_name2cost = [{}, {}] for idx, op in enumerate(train_program.blocks[0].ops): @@ -102,34 +103,37 @@ def get_single_node_data(): def mlp_forward(train_program, start_program, is_distributed=True): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 256 sequence_len = 128 if is_distributed: - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) else: - input = paddle.ones(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = paddle.ones(name="label", - shape=[batch_size, 1], - dtype='float32') + input = paddle.ones( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = paddle.ones( + name="label", shape=[batch_size, 1], dtype='float32' + ) if is_distributed: auto.shard_tensor(input, PP_MESH_0, ["x", None]) auto.shard_tensor(label, PP_MESH_1, ["x", None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02, - is_distributed=is_distributed) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + is_distributed=is_distributed, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -139,8 +143,9 @@ def mlp_forward(train_program, start_program, is_distributed=True): def get_dist_prog(train_program, startup_program, dist_context, rank_id): - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) fleet._user_defined_strategy = fleet.DistributedStrategy() fleet.user_defined_optimizer = paddle.fluid.optimizer.AdamOptimizer() @@ -150,25 +155,38 @@ def get_dist_prog(train_program, startup_program, dist_context, rank_id): # serial forward & backward completion completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) # logical partition partitioner = Partitioner(dist_context, rank_id) - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads = partitioner.partition( - complete_train_program, startup_program, params_grads) + ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) = partitioner.partition( + complete_train_program, startup_program, params_grads + ) partitioned_optimize_ops = parallelizer._apply_optimize( - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads) + auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + ) - return auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + return ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) def check_runtime_estimation(cost): @@ -199,18 +217,19 @@ def check_empty_program_memory(cost): class TestCostModel(unittest.TestCase): - def test_empty_program_cost_model(self): empty_program = paddle.static.Program() startup_program = paddle.static.Program() standalone_cost_data = [{}] empty_pp_cfg = None cluster = None - cost = estimate_cost([empty_program], - cluster=cluster, - pipeline_config=empty_pp_cfg, - standalone_cost_data=standalone_cost_data, - batch_size=1) + cost = estimate_cost( + [empty_program], + cluster=cluster, + pipeline_config=empty_pp_cfg, + standalone_cost_data=standalone_cost_data, + batch_size=1, + ) self.assertTrue(check_empty_program_runtime(cost)) self.assertTrue(check_empty_program_memory(cost)) @@ -222,18 +241,30 @@ def test_auto_parallel_cost_model(self): train_program = paddle.static.Program() startup_program = paddle.static.Program() dist_context = DistributedContext() - distributed_program, dist_startup_prog, dist_params_grads = get_dist_prog( - train_program, startup_program, dist_context, rank_id) - resharder = Resharder(distributed_program, dist_startup_prog, - rank_id, dist_context, dist_params_grads) + ( + distributed_program, + dist_startup_prog, + dist_params_grads, + ) = get_dist_prog( + train_program, startup_program, dist_context, rank_id + ) + resharder = Resharder( + distributed_program, + dist_startup_prog, + rank_id, + dist_context, + dist_params_grads, + ) resharder.reshard() dist_program.append(distributed_program) cluster = None - cost = estimate_cost(dist_program, - cluster=cluster, - pipeline_config=pp_cfg, - standalone_cost_data=standalone_cost_data, - batch_size=4) + cost = estimate_cost( + dist_program, + cluster=cluster, + pipeline_config=pp_cfg, + standalone_cost_data=standalone_cost_data, + batch_size=4, + ) self.assertTrue(check_runtime_estimation(cost)) self.assertTrue(check_memory_estimation(cost)) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_data_unshard.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_data_unshard.py index 34b1f5dd6d4a1..f641961b5afbc 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_data_unshard.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_data_unshard.py @@ -18,7 +18,6 @@ class TestAutoParallelDataUnshard(TestMultipleGpus): - def test_auto_parallel_data_unshard(self): self.run_mnist_2gpu('auto_parallel_data_unshard.py') diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_dist_tensor.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_dist_tensor.py index a3b842bed8f45..e1f504602398e 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_dist_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_dist_tensor.py @@ -23,18 +23,23 @@ from paddle.distributed.auto_parallel.partitioner import Partitioner from paddle.distributed.auto_parallel.dist_context import DistributedContext from paddle.distributed.auto_parallel.dist_tensor import DistributedTensor -from paddle.distributed.auto_parallel.dist_attribute import TensorDistributedAttribute +from paddle.distributed.auto_parallel.dist_attribute import ( + TensorDistributedAttribute, +) import test_auto_parallel_reshard from test_auto_parallel_reshard import mlp_forward -def get_dist_prog(train_program, - startup_program, - dist_context, - rank_id, - complete_train_program=None): - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) +def get_dist_prog( + train_program, + startup_program, + dist_context, + rank_id, + complete_train_program=None, +): + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) fleet._user_defined_strategy = fleet.DistributedStrategy() fleet.user_defined_optimizer = paddle.fluid.optimizer.AdamOptimizer() @@ -43,49 +48,68 @@ def get_dist_prog(train_program, # serial forward & backward completion completer = Completer(dist_context) - complete_train_program = completer.complete_forward_annotation( - train_program - ) if complete_train_program is None else complete_train_program + complete_train_program = ( + completer.complete_forward_annotation(train_program) + if complete_train_program is None + else complete_train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) # logical partition partitioner = Partitioner(dist_context, rank_id) - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads = partitioner.partition( - complete_train_program, startup_program, params_grads) + ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) = partitioner.partition( + complete_train_program, startup_program, params_grads + ) partitioned_optimize_ops = parallelizer._apply_optimize( - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads) + auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + ) - return auto_parallel_main_prog, auto_parallel_startup_prog, complete_train_program + return ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + complete_train_program, + ) class TestDistributedTensor(unittest.TestCase): - def test_new_local_tensor(self): test_auto_parallel_reshard._global_process_mesh = auto.ProcessMesh( - mesh=[0, 1], dim_names=["x"]) + mesh=[0, 1], dim_names=["x"] + ) test_auto_parallel_reshard._global_parallel_strategy = "dp" train_program = paddle.static.Program() startup_program = paddle.static.Program() dist_context = DistributedContext() rank_id = 0 - dist_main_prog, dist_startup_prog, complete_train_program = get_dist_prog( - train_program, startup_program, dist_context, rank_id) + ( + dist_main_prog, + dist_startup_prog, + complete_train_program, + ) = get_dist_prog(train_program, startup_program, dist_context, rank_id) dist_context.dist_main_programs[rank_id] = dist_main_prog dist_context.dist_startup_programs[rank_id] = dist_startup_prog name = "layer_norm_1.tmp_2" dist_tensor = dist_context.get_dist_tensor_for_program( - complete_train_program.global_block().vars[name]) + complete_train_program.global_block().vars[name] + ) dist_tensor._dist_context = dist_context intermediate_var_0 = dist_tensor.new_local_tensor( - name="intermediate_var_0") + name="intermediate_var_0" + ) self.assertEqual(intermediate_var_0.shape, (2, 1024)) self.assertEqual(intermediate_var_0.name, "intermediate_var_0") @@ -93,25 +117,34 @@ def test_new_local_tensor(self): train_program = paddle.static.Program() startup_program = paddle.static.Program() dist_context = DistributedContext() - dist_main_prog, dist_startup_prog, complete_train_program = get_dist_prog( - train_program, startup_program, dist_context, rank_id, None) + ( + dist_main_prog, + dist_startup_prog, + complete_train_program, + ) = get_dist_prog( + train_program, startup_program, dist_context, rank_id, None + ) dist_context.dist_main_programs[rank_id] = dist_main_prog dist_context.dist_startup_programs[rank_id] = dist_startup_prog name = "layer_norm_1.tmp_2" dist_tensor = dist_context.get_dist_tensor_for_program( - complete_train_program.global_block().vars[name]) + complete_train_program.global_block().vars[name] + ) dist_tensor._dist_context = dist_context intermediate_var_1 = dist_tensor.new_local_tensor( - rank=rank_id, name="intermediate_var_1") + rank=rank_id, name="intermediate_var_1" + ) self.assertEqual(intermediate_var_0.shape, (2, 1024)) self.assertEqual(intermediate_var_1.name, "intermediate_var_1") name = "linear_0.w_0" dist_tensor = dist_context.get_dist_tensor_for_program( - complete_train_program.global_block().vars[name]) + complete_train_program.global_block().vars[name] + ) dist_tensor._dist_context = dist_context intermediate_var_1 = dist_tensor.new_local_tensor( - rank=rank_id, name="linear_0.w_0_intermediate") + rank=rank_id, name="linear_0.w_0_intermediate" + ) self.assertEqual(intermediate_var_1.shape, (1024, 4096)) self.assertEqual(intermediate_var_1.name, "linear_0.w_0_intermediate") @@ -121,7 +154,10 @@ def test_new_local_tensor(self): id(copied_dist_context), id( copied_dist_context.get_dist_tensor_for_program( - dist_tensor.serial_tensor).dist_context)) + dist_tensor.serial_tensor + ).dist_context + ), + ) def test_static_method(self): dims_mapping = [1, 0] @@ -133,58 +169,63 @@ def test_static_method(self): # rank 1 [(2, 4), (0, 3)] # rank 4 [(2, 4), (3, 6)] rank = 0 - local_sizes = DistributedTensor.get_local_sizes(global_sizes, - dims_mapping, topology, - processes) + local_sizes = DistributedTensor.get_local_sizes( + global_sizes, dims_mapping, topology, processes + ) self.assertEqual(local_sizes, [2, 3]) local_offsets = DistributedTensor.get_local_offsets( - global_sizes, dims_mapping, topology, processes, rank) + global_sizes, dims_mapping, topology, processes, rank + ) self.assertEqual(local_offsets, [0, 0]) - local_shard = DistributedTensor.get_local_shard(global_sizes, - dims_mapping, topology, - processes, rank) + local_shard = DistributedTensor.get_local_shard( + global_sizes, dims_mapping, topology, processes, rank + ) self.assertEqual(local_shard, [(0, 2), (0, 3)]) rank = 1 - local_sizes = DistributedTensor.get_local_sizes(global_sizes, - dims_mapping, topology, - processes) + local_sizes = DistributedTensor.get_local_sizes( + global_sizes, dims_mapping, topology, processes + ) self.assertEqual(local_sizes, [2, 3]) local_offsets = DistributedTensor.get_local_offsets( - global_sizes, dims_mapping, topology, processes, rank) + global_sizes, dims_mapping, topology, processes, rank + ) self.assertEqual(local_offsets, [2, 0]) - local_shard = DistributedTensor.get_local_shard(global_sizes, - dims_mapping, topology, - processes, rank) + local_shard = DistributedTensor.get_local_shard( + global_sizes, dims_mapping, topology, processes, rank + ) self.assertEqual(local_shard, [(2, 4), (0, 3)]) rank = 4 - local_sizes = DistributedTensor.get_local_sizes(global_sizes, - dims_mapping, topology, - processes) + local_sizes = DistributedTensor.get_local_sizes( + global_sizes, dims_mapping, topology, processes + ) self.assertEqual(local_sizes, [2, 3]) local_offsets = DistributedTensor.get_local_offsets( - global_sizes, dims_mapping, topology, processes, rank) + global_sizes, dims_mapping, topology, processes, rank + ) self.assertEqual(local_offsets, [2, 3]) - local_shard = DistributedTensor.get_local_shard(global_sizes, - dims_mapping, topology, - processes, rank) + local_shard = DistributedTensor.get_local_shard( + global_sizes, dims_mapping, topology, processes, rank + ) self.assertEqual(local_shard, [(2, 4), (3, 6)]) # global sizes local_sizes = [2, 3] global_sizes = DistributedTensor.get_global_sizes( - local_sizes, dims_mapping, topology, processes) + local_sizes, dims_mapping, topology, processes + ) self.assertEqual(global_sizes, [6, 6]) def test_instance_method(self): tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.dims_mapping = [1, 0] tensor_dist_attr.process_mesh = auto.ProcessMesh( - mesh=[[0, 1, 2], [3, 4, 5]]) - serial_tensor = paddle.static.data(name="data", - shape=[6, 6], - dtype='float32') + mesh=[[0, 1, 2], [3, 4, 5]] + ) + serial_tensor = paddle.static.data( + name="data", shape=[6, 6], dtype='float32' + ) dist_tensor = DistributedTensor(serial_tensor, tensor_dist_attr) # rank 0 [(0, 2), (0, 3)] diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_graph.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_graph.py index d8abad85ccaa9..2e43bf6f928cb 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_graph.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_graph.py @@ -17,7 +17,6 @@ class TestAutoParallelGraph(unittest.TestCase): - def test_graph(self): graph = Graph(name="foo") self.assertEqual(graph.attrs["name"], "foo") diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_mapper.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_mapper.py index 5d473fe55075e..e18a585d33f38 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_mapper.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_mapper.py @@ -361,11 +361,9 @@ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=64, - intermediate_size=4 * 64, - initializer_range=0.02): + def __init__( + self, hidden_size=64, intermediate_size=4 * 64, initializer_range=0.02 + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size @@ -379,37 +377,37 @@ def __init__(self, weight_attr2 = paddle.ParamAttr(initializer=NumpyArrayInitializer(arr2)) weight_attr3 = paddle.ParamAttr(initializer=NumpyArrayInitializer(arr3)) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr0, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr1, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr0, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr1, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) - self.linear2 = nn.Linear(d_model, - dim_feedforward, - weight_attr2, - bias_attr=bias_attr) - self.linear3 = nn.Linear(dim_feedforward, - d_model, - weight_attr3, - bias_attr=bias_attr) + self.linear2 = nn.Linear( + d_model, dim_feedforward, weight_attr2, bias_attr=bias_attr + ) + self.linear3 = nn.Linear( + dim_feedforward, d_model, weight_attr3, bias_attr=bias_attr + ) def forward(self, input): if _global_parallel_strategy == "dp_mp_pp": - auto.shard_tensor(self.linear0.weight, _global_process_mesh[0], - [None, "y"]) + auto.shard_tensor( + self.linear0.weight, _global_process_mesh[0], [None, "y"] + ) - auto.shard_tensor(self.linear1.weight, _global_process_mesh[0], - ["y", None]) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh[0], ["y", None] + ) - auto.shard_tensor(self.linear2.weight, _global_process_mesh[1], - [None, "y"]) + auto.shard_tensor( + self.linear2.weight, _global_process_mesh[1], [None, "y"] + ) - auto.shard_tensor(self.linear3.weight, _global_process_mesh[1], - ["y", None]) + auto.shard_tensor( + self.linear3.weight, _global_process_mesh[1], ["y", None] + ) out = self.norm(input) out = self.linear0(out) @@ -425,22 +423,25 @@ def forward(self, input): def mlp_forward(train_program, start_program): - with static.program_guard(train_program,start_program), \ - utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 64 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) if _global_parallel_strategy == "dp_mp_pp": auto.shard_tensor(input, _global_process_mesh[0], ["x", None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) loss = paddle.mean(error_cost) @@ -448,8 +449,9 @@ def mlp_forward(train_program, start_program): def get_dist_prog(train_program, startup_program, dist_context, rank_id): - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) fleet._user_defined_strategy = fleet.DistributedStrategy() fleet.user_defined_optimizer = paddle.fluid.optimizer.AdamOptimizer() @@ -459,24 +461,38 @@ def get_dist_prog(train_program, startup_program, dist_context, rank_id): # auto completion completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) partitioner = Partitioner(dist_context, rank_id) - dist_train_program, dist_startup_prog, dist_params_grads = partitioner.partition( - complete_train_program, startup_program, params_grads) + ( + dist_train_program, + dist_startup_prog, + dist_params_grads, + ) = partitioner.partition( + complete_train_program, startup_program, params_grads + ) partitioned_optimize_ops = parallelizer._apply_optimize( - dist_train_program, dist_startup_prog, dist_params_grads) - - resharder = Resharder(dist_train_program, dist_startup_prog, rank_id, - dist_context, dist_params_grads) + dist_train_program, dist_startup_prog, dist_params_grads + ) + + resharder = Resharder( + dist_train_program, + dist_startup_prog, + rank_id, + dist_context, + dist_params_grads, + ) resharder.reshard() return dist_train_program, dist_startup_prog @@ -496,7 +512,6 @@ def get_device_local_ids(machine): class TestAutoParallelMapper(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -504,8 +519,9 @@ def tearDown(self): self.temp_dir.cleanup() def test_mapper_dp_mp_pp(self): - cluster_json_path = os.path.join(self.temp_dir.name, - "auto_parallel_cluster.json") + cluster_json_path = os.path.join( + self.temp_dir.name, "auto_parallel_cluster.json" + ) cluster_json_object = json.loads(cluster_json) with open(cluster_json_path, "w") as cluster_json_file: json.dump(cluster_json_object, cluster_json_file) @@ -519,7 +535,7 @@ def test_mapper_dp_mp_pp(self): global _global_process_mesh _global_process_mesh = [ auto.ProcessMesh([[0, 1], [2, 3]], dim_names=["x", "y"]), - auto.ProcessMesh([[4, 5], [6, 7]], dim_names=["x", "y"]) + auto.ProcessMesh([[4, 5], [6, 7]], dim_names=["x", "y"]), ] processes = [0, 1, 2, 3, 4, 5, 6, 7] @@ -529,7 +545,8 @@ def test_mapper_dp_mp_pp(self): startup_program = static.Program() dist_context = DistributedContext() dist_train_program, dist_startup_prog = get_dist_prog( - train_program, startup_program, dist_context, rank_id) + train_program, startup_program, dist_context, rank_id + ) # if rank_id == 0: # print_program_with_dist_attr(dist_train_program, dist_context) dist_programs[rank_id] = [dist_train_program, None] @@ -547,8 +564,9 @@ def test_mapper_dp_mp_pp(self): self.assertTrue(is_in_machine(device_ids[0], machine)) machine_mapped_ranks.add(rank) machine_mapped_device_local_ids.add(device_ids[0]) - self.assertEqual(len(machine_mapped_ranks), - len(machine_mapped_device_local_ids)) + self.assertEqual( + len(machine_mapped_ranks), len(machine_mapped_device_local_ids) + ) all_mapped_ranks.update(machine_mapped_ranks) self.assertEqual(set(processes), all_mapped_ranks) @@ -575,35 +593,30 @@ def test_mapper_misc(self): dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) broadcast_op = train_program.global_block().append_op( type="c_broadcast", inputs={'X': input}, - attrs={ - 'ring_id': ring_id, - 'root': root_id - }, - outputs={'Out': output}) + attrs={'ring_id': ring_id, 'root': root_id}, + outputs={'Out': output}, + ) self.assertEqual(get_comm_volume(broadcast_op, 0, 1), 400) self.assertEqual(get_comm_volume(broadcast_op, 1, 0), None) allgather_op = train_program.global_block().append_op( type="c_allgather", inputs={'X': input}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }, - outputs={'Out': output}) + attrs={'ring_id': ring_id, 'nranks': nranks}, + outputs={'Out': output}, + ) self.assertEqual(get_comm_volume(allgather_op, 0, 1), 400) self.assertEqual(get_comm_volume(allgather_op, 0, 0), None) reduce_op = train_program.global_block().append_op( type="c_reduce_sum", inputs={'X': input}, - attrs={ - 'ring_id': ring_id, - 'root_id': root_id - }, - outputs={'Out': output}) + attrs={'ring_id': ring_id, 'root_id': root_id}, + outputs={'Out': output}, + ) self.assertEqual(get_comm_volume(reduce_op, 0, 1), None) self.assertEqual(get_comm_volume(reduce_op, 1, 0), 400) cast_op = train_program.global_block().append_op( @@ -612,8 +625,9 @@ def test_mapper_misc(self): outputs={"Out": output}, attrs={ "in_dtype": fluid.core.VarDesc.VarType.FP32, - "out_dtype": fluid.core.VarDesc.VarType.FP32 - }) + "out_dtype": fluid.core.VarDesc.VarType.FP32, + }, + ) self.assertRaises(ValueError, get_comm_volume, cast_op, 0, 1) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner.py index b5af410a2d0d2..366e42cc68672 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner.py @@ -44,16 +44,26 @@ def get_programs(annotated_func): train_program, start_program = annotated_func(train_program, start_program) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) rank_id = 3 dist_strategy = fleet.DistributedStrategy() partitioner = Partitioner(dist_context, rank_id) - test_auto_parallel_dist_main_prog, test_auto_parallel_dist_startup_prog, _ = partitioner.partition( - complete_train_program, start_program, []) + ( + test_auto_parallel_dist_main_prog, + test_auto_parallel_dist_startup_prog, + _, + ) = partitioner.partition(complete_train_program, start_program, []) - return complete_train_program, start_program, test_auto_parallel_dist_main_prog, test_auto_parallel_dist_startup_prog, dist_context + return ( + complete_train_program, + start_program, + test_auto_parallel_dist_main_prog, + test_auto_parallel_dist_startup_prog, + dist_context, + ) def is_all_parameters_shape_equal(prog1, prog2): @@ -84,41 +94,62 @@ def check_tensor_split(prog1, varnames1, prog2, varnames2, axis, nsplit): return True -def initialization_check(mode, dist_context, dist_startup_prog, - serial_startup_prog, var_need_broadcast, process_mesh, - mp_parallel_axis, dp_parallel_axis): +def initialization_check( + mode, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + process_mesh, + mp_parallel_axis, + dp_parallel_axis, +): if 'mp' in mode: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, mp_parallel_axis, - 3) + group_ranks = _get_comm_group( + process_mesh.processes, process_mesh.topology, mp_parallel_axis, 3 + ) mp_ring_id = new_process_group(group_ranks).id broadcast_ops = [ - op for op in dist_startup_prog.global_block().ops if - (op.type == "c_broadcast" and op.desc.attr("ring_id") == mp_ring_id) + op + for op in dist_startup_prog.global_block().ops + if ( + op.type == "c_broadcast" + and op.desc.attr("ring_id") == mp_ring_id + ) ] broadcast_varnames = sorted( - [op.desc.output_arg_names()[0] for op in broadcast_ops]) + [op.desc.output_arg_names()[0] for op in broadcast_ops] + ) if broadcast_varnames != var_need_broadcast: return False if 'dp' in mode: - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, dp_parallel_axis, - 3) + group_ranks = _get_comm_group( + process_mesh.processes, process_mesh.topology, dp_parallel_axis, 3 + ) dp_ring_id = new_process_group(group_ranks).id nparam = len(serial_startup_prog.all_parameters()) - nbroadcast_dp = len([ - op for op in dist_startup_prog.global_block().ops if - (op.type == "c_broadcast" and op.desc.attr("ring_id") == dp_ring_id) - ]) + nbroadcast_dp = len( + [ + op + for op in dist_startup_prog.global_block().ops + if ( + op.type == "c_broadcast" + and op.desc.attr("ring_id") == dp_ring_id + ) + ] + ) if nparam != nbroadcast_dp: return False if "dp" in mode and 'mp' in mode: - nbroadcast = len([ - op for op in dist_startup_prog.global_block().ops - if op.type == "c_broadcast" - ]) + nbroadcast = len( + [ + op + for op in dist_startup_prog.global_block().ops + if op.type == "c_broadcast" + ] + ) if len(var_need_broadcast) + nbroadcast_dp != nbroadcast: return False @@ -141,14 +172,17 @@ def get_output_var_dist_attr(op, main_program, dist_context): def check_equal_var_dist_attr(serial_dist_attr, dist_attr): equal = True - if serial_dist_attr.process_mesh != dist_attr.process_mesh or \ - serial_dist_attr.dims_mapping != dist_attr.dims_mapping: + if ( + serial_dist_attr.process_mesh != dist_attr.process_mesh + or serial_dist_attr.dims_mapping != dist_attr.dims_mapping + ): equal = False return equal -def check_equal_dist_op_attr(dist_context, dist_main_prog, serial_op, dist_ops, - dist_op_idx): +def check_equal_dist_op_attr( + dist_context, dist_main_prog, serial_op, dist_ops, dist_op_idx +): equal = True # get serial op's process_mesh and impl_idx serial_op_dist_attr = dist_context.get_op_dist_attr_for_program(serial_op) @@ -161,34 +195,40 @@ def check_equal_dist_op_attr(dist_context, dist_main_prog, serial_op, dist_ops, for in_varname in dist_ops[i].desc.input_arg_names(): in_var = dist_main_prog.global_block().var(in_varname) tensor_dist_attr = dist_context.get_tensor_dist_attr_for_program( - in_var) + in_var + ) tensor_dims_mapping = tensor_dist_attr.dims_mapping in_var_dims_mapping = op_dist_attr.get_input_dims_mapping( - in_varname) + in_varname + ) if tensor_dims_mapping != in_var_dims_mapping: equal = False for out_varname in dist_ops[i].desc.output_arg_names(): out_var = dist_main_prog.global_block().var(out_varname) tensor_dist_attr = dist_context.get_tensor_dist_attr_for_program( - out_var) + out_var + ) tensor_dims_mapping = tensor_dist_attr.dims_mapping out_var_dims_mapping = op_dist_attr.get_output_dims_mapping( - out_varname) + out_varname + ) if tensor_dims_mapping != out_var_dims_mapping: equal = False dist_op_process_mesh = op_dist_attr.process_mesh dist_op_impl_idx = op_dist_attr.impl_idx - if serial_op.desc.id() == dist_ops[i].desc.id() or \ - serial_process_mesh != dist_op_process_mesh or \ - serial_impl_idx != dist_op_impl_idx: + if ( + serial_op.desc.id() == dist_ops[i].desc.id() + or serial_process_mesh != dist_op_process_mesh + or serial_impl_idx != dist_op_impl_idx + ): equal = False return equal -def distributed_attr_check_for_dist_op(serial_main_prog, dist_main_prog, - dist_context, serial_op_idx, - dist_op_idx): +def distributed_attr_check_for_dist_op( + serial_main_prog, dist_main_prog, dist_context, serial_op_idx, dist_op_idx +): equal = True serial_ops = serial_main_prog.global_block().ops @@ -200,27 +240,34 @@ def distributed_attr_check_for_dist_op(serial_main_prog, dist_main_prog, if dist_op_0.type == "c_identity": # serial op input's dist_attr serial_in_dist_attr = get_input_var_dist_attr( - serial_op, serial_main_prog, dist_context) + serial_op, serial_main_prog, dist_context + ) # c_identity output's(new var) dist_attr identity_out_dist_attr = get_output_var_dist_attr( - dist_op_0, dist_main_prog, dist_context) + dist_op_0, dist_main_prog, dist_context + ) # check var dist_attr - equal = check_equal_var_dist_attr(serial_in_dist_attr, - identity_out_dist_attr) + equal = check_equal_var_dist_attr( + serial_in_dist_attr, identity_out_dist_attr + ) else: # serial op output's dist_attr serial_out_dist_attr = get_output_var_dist_attr( - serial_op, serial_main_prog, dist_context) + serial_op, serial_main_prog, dist_context + ) # dist op output's(new var) dist_attr - out_dist_attr = get_output_var_dist_attr(dist_op_0, dist_main_prog, - dist_context) + out_dist_attr = get_output_var_dist_attr( + dist_op_0, dist_main_prog, dist_context + ) # check var dist_attr - equal = check_equal_var_dist_attr(serial_out_dist_attr, - out_dist_attr) + equal = check_equal_var_dist_attr( + serial_out_dist_attr, out_dist_attr + ) # check op's dist_attr - equal = check_equal_dist_op_attr(dist_context, dist_main_prog, - serial_op, dist_ops, dist_op_idx[i]) + equal = check_equal_dist_op_attr( + dist_context, dist_main_prog, serial_op, dist_ops, dist_op_idx[i] + ) return equal @@ -242,45 +289,53 @@ def distributed_attr_check_for_program(dist_main_prog, dist_context): class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") def forward(self, input): if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.linear0.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.linear1.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.linear0.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.linear1.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) else: - auto.shard_tensor(self.linear0.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, None]) - auto.shard_tensor(self.linear1.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, None]) + auto.shard_tensor( + self.linear0.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, None], + ) + auto.shard_tensor( + self.linear1.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, None], + ) out = self.norm(input) out = self.linear0(out) @@ -292,46 +347,61 @@ def forward(self, input): def mlp_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') + input = static.data( + name="input", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input, - process_mesh=_global_process_mesh, - shard_spec=["dp", None, None]) - - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - dropout_ratio=0.1, - initializer_range=0.02) + auto.shard_tensor( + input, + process_mesh=_global_process_mesh, + shard_spec=["dp", None, None], + ) + + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + dropout_ratio=0.1, + initializer_range=0.02, + ) out = mlp(input) return train_program, start_program class TestMLPAutoPartitioner(unittest.TestCase): - def test_mlp_dp(self): global _global_parallel_strategy _global_parallel_strategy = "dp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["dp"]) - - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - mlp_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["dp"] + ) + + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(mlp_pretrain_forward) # parameter should not be partitioned self.assertTrue( - is_all_parameters_shape_equal(serial_main_prog, dist_main_prog)) + is_all_parameters_shape_equal(serial_main_prog, dist_main_prog) + ) self.assertTrue( - is_all_parameters_shape_equal(serial_startup_prog, - dist_startup_prog)) + is_all_parameters_shape_equal( + serial_startup_prog, dist_startup_prog + ) + ) # op in main prog should be the same serial_ops = serial_main_prog.global_block().ops @@ -343,152 +413,217 @@ def test_mlp_dp(self): # parameter initialization var_need_broadcast = [] self.assertTrue( - initialization_check(_global_parallel_strategy, - dist_context, - dist_startup_prog, - serial_startup_prog, - var_need_broadcast, - _global_process_mesh, - mp_parallel_axis=None, - dp_parallel_axis=0)) + initialization_check( + _global_parallel_strategy, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + _global_process_mesh, + mp_parallel_axis=None, + dp_parallel_axis=0, + ) + ) def test_mlp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["mp"]) - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - mlp_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["mp"] + ) + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(mlp_pretrain_forward) # param should be partition nrank = 4 # col parallel weights = ['linear_0.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 1, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 1, nrank + ) + ) weights = ['linear_0.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) # row parallel weights = ['linear_1.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) weights = ['linear_1.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, 1)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, 1 + ) + ) # row and col allreduce dist_ops = dist_main_prog.global_block().ops dist_ops = [op.type for op in dist_ops] ref_ops = [ - 'layer_norm', 'c_identity', 'matmul_v2', 'elementwise_add', 'gelu', - 'matmul_v2', 'c_allreduce_sum', 'elementwise_add', 'dropout' + 'layer_norm', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'gelu', + 'matmul_v2', + 'c_allreduce_sum', + 'elementwise_add', + 'dropout', ] self.assertTrue(dist_ops == ref_ops) # parameter initialization var_need_broadcast = sorted( - ['layer_norm_0.b_0', 'layer_norm_0.w_0', 'linear_1.b_0']) + ['layer_norm_0.b_0', 'layer_norm_0.w_0', 'linear_1.b_0'] + ) self.assertTrue( - initialization_check(_global_parallel_strategy, - dist_context, - dist_startup_prog, - serial_startup_prog, - var_need_broadcast, - _global_process_mesh, - mp_parallel_axis=0, - dp_parallel_axis=None)) + initialization_check( + _global_parallel_strategy, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + _global_process_mesh, + mp_parallel_axis=0, + dp_parallel_axis=None, + ) + ) # check var and op all have dist_attr in dist_main_program self.assertTrue( - distributed_attr_check_for_program(dist_main_prog, dist_context)) + distributed_attr_check_for_program(dist_main_prog, dist_context) + ) # check distribured attr for dist op serial_op_idx = [1, 4] dist_op_idx = [[1, 2], [5, 6]] self.assertTrue( - distributed_attr_check_for_dist_op(serial_main_prog, dist_main_prog, - dist_context, serial_op_idx, - dist_op_idx)) + distributed_attr_check_for_dist_op( + serial_main_prog, + dist_main_prog, + dist_context, + serial_op_idx, + dist_op_idx, + ) + ) def test_mlp_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - mlp_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(mlp_pretrain_forward) # param should be partition nrank = 4 # col parallel weights = ['linear_0.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 1, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 1, nrank + ) + ) weights = ['linear_0.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) # row parallel weights = ['linear_1.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) weights = ['linear_1.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, 1)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, 1 + ) + ) # row and col allreduce dist_ops = dist_main_prog.global_block().ops dist_ops = [op.type for op in dist_ops] ref_ops = [ - 'layer_norm', 'c_identity', 'matmul_v2', 'elementwise_add', 'gelu', - 'matmul_v2', 'c_allreduce_sum', 'elementwise_add', 'dropout' + 'layer_norm', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'gelu', + 'matmul_v2', + 'c_allreduce_sum', + 'elementwise_add', + 'dropout', ] self.assertTrue(dist_ops == ref_ops) # parameter initialization var_need_broadcast = sorted( - ['layer_norm_0.b_0', 'layer_norm_0.w_0', 'linear_1.b_0']) + ['layer_norm_0.b_0', 'layer_norm_0.w_0', 'linear_1.b_0'] + ) self.assertTrue( - initialization_check(_global_parallel_strategy, - dist_context, - dist_startup_prog, - serial_startup_prog, - var_need_broadcast, - _global_process_mesh, - mp_parallel_axis=1, - dp_parallel_axis=0)) + initialization_check( + _global_parallel_strategy, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + _global_process_mesh, + mp_parallel_axis=1, + dp_parallel_axis=0, + ) + ) # check var and op all have dist_attr in dist_main_program self.assertTrue( - distributed_attr_check_for_program(dist_main_prog, dist_context)) + distributed_attr_check_for_program(dist_main_prog, dist_context) + ) # check distribured attr for dist op serial_op_idx = [1, 4] dist_op_idx = [[1, 2], [5, 6]] self.assertTrue( - distributed_attr_check_for_dist_op(serial_main_prog, dist_main_prog, - dist_context, serial_op_idx, - dist_op_idx)) + distributed_attr_check_for_dist_op( + serial_main_prog, + dist_main_prog, + dist_context, + serial_op_idx, + dist_op_idx, + ) + ) class AttentionLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - sequence_len=512, - intermediate_size=4 * 1024, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + sequence_len=512, + intermediate_size=4 * 1024, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(AttentionLayer, self).__init__() self.hidden_size = hidden_size self.sequence_len = sequence_len @@ -497,38 +632,38 @@ def __init__(self, self.vdim = self.embed_dim self.num_heads = num_heads self.head_dim = self.embed_dim // self.num_heads - assert self.head_dim * self.num_heads == self.embed_dim, \ - "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * self.num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" self.dropout_ratio = dropout_ratio self.initializer_range = initializer_range self.training = True self.attn_mask = None weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.q_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.k_proj = nn.Linear(self.kdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.v_proj = nn.Linear(self.vdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.out_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) + self.q_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.k_proj = nn.Linear( + self.kdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.v_proj = nn.Linear( + self.vdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.out_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) def forward(self, input): if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input, - process_mesh=_global_process_mesh, - shard_spec=["dp", None, None]) + auto.shard_tensor( + input, + process_mesh=_global_process_mesh, + shard_spec=["dp", None, None], + ) q = self.q_proj(input) q = tensor.reshape(x=q, shape=[0, 0, self.num_heads, self.head_dim]) @@ -538,15 +673,21 @@ def forward(self, input): v = self.v_proj(input) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.q_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.k_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.v_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.q_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.k_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.v_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) k = tensor.transpose(x=k, perm=[0, 2, 1, 3]) @@ -554,10 +695,9 @@ def forward(self, input): v = tensor.transpose(x=v, perm=[0, 2, 1, 3]) # scale dot product attention - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.head_dim**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.head_dim**-0.5 + ) if self.attn_mask is not None: product = product + self.attn_mask @@ -565,10 +705,12 @@ def forward(self, input): weights = F.softmax(product) if self.dropout_ratio: - weights = F.dropout(weights, - self.dropout_ratio, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout_ratio, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) @@ -580,50 +722,65 @@ def forward(self, input): out = self.out_proj(out) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.out_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.out_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) return out def attn_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="query", - shape=[batch_size, sequence_len, hidden_size], - dtype='float32') - attn = AttentionLayer(hidden_size=hidden_size, - sequence_len=sequence_len, - intermediate_size=4 * hidden_size, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02) + input = static.data( + name="query", + shape=[batch_size, sequence_len, hidden_size], + dtype='float32', + ) + attn = AttentionLayer( + hidden_size=hidden_size, + sequence_len=sequence_len, + intermediate_size=4 * hidden_size, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ) out = attn(input) return train_program, start_program class TestAttentionAutoPartitioner(unittest.TestCase): - def test_attn_dp(self): global _global_parallel_strategy _global_parallel_strategy = "dp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["dp"]) - - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - attn_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["dp"] + ) + + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(attn_pretrain_forward) # parameter should not be partitioned self.assertTrue( - is_all_parameters_shape_equal(serial_main_prog, dist_main_prog)) + is_all_parameters_shape_equal(serial_main_prog, dist_main_prog) + ) self.assertTrue( - is_all_parameters_shape_equal(serial_startup_prog, - dist_startup_prog)) + is_all_parameters_shape_equal( + serial_startup_prog, dist_startup_prog + ) + ) # op in main prog should be the same serial_ops = serial_main_prog.global_block().ops @@ -635,162 +792,247 @@ def test_attn_dp(self): # parameter initialization var_need_broadcast = [] self.assertTrue( - initialization_check(_global_parallel_strategy, - dist_context, - dist_startup_prog, - serial_startup_prog, - var_need_broadcast, - _global_process_mesh, - mp_parallel_axis=None, - dp_parallel_axis=0)) + initialization_check( + _global_parallel_strategy, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + _global_process_mesh, + mp_parallel_axis=None, + dp_parallel_axis=0, + ) + ) def test_attn_mp(self): global _global_parallel_strategy _global_parallel_strategy = "mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[0, 1, 2, 3], - dim_names=["mp"]) - - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - attn_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[0, 1, 2, 3], dim_names=["mp"] + ) + + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(attn_pretrain_forward) # param should be partition nrank = 4 # col parallel weights = ['linear_0.w_0', 'linear_1.w_0', 'linear_2.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 1, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 1, nrank + ) + ) weights = ['linear_0.b_0', 'linear_1.b_0', 'linear_2.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) # row parallel weights = ['linear_3.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) weights = ['linear_3.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, 1)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, 1 + ) + ) # row and col allreduce dist_ops = dist_main_prog.global_block().ops dist_ops = [op.type for op in dist_ops] ref_ops = [ - 'c_identity', 'matmul_v2', 'elementwise_add', 'reshape2', - 'transpose2', 'c_identity', 'matmul_v2', 'elementwise_add', - 'c_identity', 'matmul_v2', 'elementwise_add', 'reshape2', - 'transpose2', 'reshape2', 'transpose2', 'matmul', 'softmax', - 'dropout', 'matmul_v2', 'transpose2', 'reshape2', 'matmul_v2', - 'c_allreduce_sum', 'elementwise_add' + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'reshape2', + 'transpose2', + 'matmul', + 'softmax', + 'dropout', + 'matmul_v2', + 'transpose2', + 'reshape2', + 'matmul_v2', + 'c_allreduce_sum', + 'elementwise_add', ] self.assertTrue(dist_ops == ref_ops) # parameter initialization var_need_broadcast = ['linear_3.b_0'] self.assertTrue( - initialization_check(_global_parallel_strategy, - dist_context, - dist_startup_prog, - serial_startup_prog, - var_need_broadcast, - _global_process_mesh, - mp_parallel_axis=0, - dp_parallel_axis=None)) + initialization_check( + _global_parallel_strategy, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + _global_process_mesh, + mp_parallel_axis=0, + dp_parallel_axis=None, + ) + ) # check var and op all have dist_attr in dist_main_program self.assertTrue( - distributed_attr_check_for_program(dist_main_prog, dist_context)) + distributed_attr_check_for_program(dist_main_prog, dist_context) + ) # check distribured attr for dist op serial_op_idx = [0, 4, 6, 18] dist_op_idx = [[0, 1], [5, 6], [8, 9], [21, 22]] self.assertTrue( - distributed_attr_check_for_dist_op(serial_main_prog, dist_main_prog, - dist_context, serial_op_idx, - dist_op_idx)) + distributed_attr_check_for_dist_op( + serial_main_prog, + dist_main_prog, + dist_context, + serial_op_idx, + dist_op_idx, + ) + ) def test_attn_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) - - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - attn_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) + + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(attn_pretrain_forward) # param should be partition nrank = 4 # col parallel weights = ['linear_0.w_0', 'linear_1.w_0', 'linear_2.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 1, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 1, nrank + ) + ) weights = ['linear_0.b_0', 'linear_1.b_0', 'linear_2.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) # row parallel weights = ['linear_3.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) weights = ['linear_3.b_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, 1)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, 1 + ) + ) # row and col allreduce dist_ops = dist_main_prog.global_block().ops dist_ops = [op.type for op in dist_ops] ref_ops = [ - 'c_identity', 'matmul_v2', 'elementwise_add', 'reshape2', - 'transpose2', 'c_identity', 'matmul_v2', 'elementwise_add', - 'c_identity', 'matmul_v2', 'elementwise_add', 'reshape2', - 'transpose2', 'reshape2', 'transpose2', 'matmul', 'softmax', - 'dropout', 'matmul_v2', 'transpose2', 'reshape2', 'matmul_v2', - 'c_allreduce_sum', 'elementwise_add' + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'reshape2', + 'transpose2', + 'matmul', + 'softmax', + 'dropout', + 'matmul_v2', + 'transpose2', + 'reshape2', + 'matmul_v2', + 'c_allreduce_sum', + 'elementwise_add', ] self.assertTrue(dist_ops == ref_ops) # parameter initialization var_need_broadcast = ['linear_3.b_0'] self.assertTrue( - initialization_check(_global_parallel_strategy, - dist_context, - dist_startup_prog, - serial_startup_prog, - var_need_broadcast, - _global_process_mesh, - mp_parallel_axis=1, - dp_parallel_axis=0)) + initialization_check( + _global_parallel_strategy, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + _global_process_mesh, + mp_parallel_axis=1, + dp_parallel_axis=0, + ) + ) # check var and op all have dist_attr in dist_main_program self.assertTrue( - distributed_attr_check_for_program(dist_main_prog, dist_context)) + distributed_attr_check_for_program(dist_main_prog, dist_context) + ) # check distribured attr for dist op serial_op_idx = [0, 4, 6, 18] dist_op_idx = [[0, 1], [5, 6], [8, 9], [21, 22]] self.assertTrue( - distributed_attr_check_for_dist_op(serial_main_prog, dist_main_prog, - dist_context, serial_op_idx, - dist_op_idx)) + distributed_attr_check_for_dist_op( + serial_main_prog, + dist_main_prog, + dist_context, + serial_op_idx, + dist_op_idx, + ) + ) class DecoderLayer(nn.Layer): - - def __init__(self, - vocab_size=32768, - hidden_size=1024, - sequence_len=512, - max_position_embeddings=512, - intermediate_size=4 * 1024, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02): + def __init__( + self, + vocab_size=32768, + hidden_size=1024, + sequence_len=512, + max_position_embeddings=512, + intermediate_size=4 * 1024, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ): super(DecoderLayer, self).__init__() self.vocab_size = vocab_size self.hidden_size = hidden_size @@ -806,57 +1048,64 @@ def __init__(self, self.attn_mask = None self.head_dim = self.embed_dim // self.num_heads - assert self.head_dim * self.num_heads == self.embed_dim, \ - "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * self.num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" self.word_embeddings = nn.Embedding( self.vocab_size, self.hidden_size, - weight_attr=paddle.ParamAttr(name="word_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, - std=self.initializer_range))) + weight_attr=paddle.ParamAttr( + name="word_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ), + ), + ) self.position_embeddings = nn.Embedding( self.max_position_embeddings, self.hidden_size, - weight_attr=paddle.ParamAttr(name="pos_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, - std=self.initializer_range))) + weight_attr=paddle.ParamAttr( + name="pos_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ), + ), + ) - weight_attr = paddle.ParamAttr(initializer=nn.initializer.Normal( - mean=0.0, std=self.initializer_range)) + weight_attr = paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ) + ) bias_attr = None - self.q_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.k_proj = nn.Linear(self.kdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.v_proj = nn.Linear(self.vdim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) - self.out_proj = nn.Linear(self.embed_dim, - self.embed_dim, - weight_attr, - bias_attr=bias_attr) + self.q_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.k_proj = nn.Linear( + self.kdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.v_proj = nn.Linear( + self.vdim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) + self.out_proj = nn.Linear( + self.embed_dim, self.embed_dim, weight_attr, bias_attr=bias_attr + ) intermediate_size = 4 * self.hidden_size d_model = self.hidden_size dim_feedforward = intermediate_size - weight_attr = paddle.ParamAttr(initializer=nn.initializer.Normal( - mean=0.0, std=self.initializer_range)) + weight_attr = paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout1 = nn.Dropout(self.dropout_ratio) self.dropout2 = nn.Dropout(self.dropout_ratio, mode="upscale_in_train") @@ -864,17 +1113,21 @@ def __init__(self, def forward(self, input_ids, position_ids): if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input_ids, - process_mesh=_global_process_mesh, - shard_spec=["dp", None]) + auto.shard_tensor( + input_ids, + process_mesh=_global_process_mesh, + shard_spec=["dp", None], + ) input_embeddings = self.word_embeddings(input_ids) position_embeddings = self.position_embeddings(position_ids) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.word_embeddings.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.word_embeddings.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) embeddings = input_embeddings + position_embeddings embeddings = self.dropout1(embeddings) @@ -891,15 +1144,21 @@ def forward(self, input_ids, position_ids): v = self.v_proj(target) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.q_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.k_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.v_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.q_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.k_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.v_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) k = tensor.transpose(x=k, perm=[0, 2, 1, 3]) @@ -907,10 +1166,9 @@ def forward(self, input_ids, position_ids): v = tensor.transpose(x=v, perm=[0, 2, 1, 3]) # scale dot product attention - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.head_dim**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.head_dim**-0.5 + ) if self.attn_mask is not None: product = product + self.attn_mask @@ -918,10 +1176,12 @@ def forward(self, input_ids, position_ids): weights = F.softmax(product) if self.dropout_ratio: - weights = F.dropout(weights, - self.dropout_ratio, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout_ratio, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) @@ -933,13 +1193,17 @@ def forward(self, input_ids, position_ids): out = self.out_proj(out) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.out_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.out_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) else: - auto.shard_tensor(self.out_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, None]) + auto.shard_tensor( + self.out_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, None], + ) # Add residual residual = embeddings + self.dropout2(out) @@ -953,12 +1217,16 @@ def forward(self, input_ids, position_ids): out3 = self.linear1(out2) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.linear0.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.linear1.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.linear0.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.linear1.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) # Add residual final = residual + self.dropout3(out3) @@ -966,184 +1234,343 @@ def forward(self, input_ids, position_ids): def decoder_pretrain_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input_ids = static.data(name="input_ids", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') - decoder = DecoderLayer(vocab_size=32768, - hidden_size=hidden_size, - sequence_len=sequence_len, - max_position_embeddings=512, - intermediate_size=4 * hidden_size, - num_heads=16, - dropout_ratio=0.1, - initializer_range=0.02) + input_ids = static.data( + name="input_ids", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) + decoder = DecoderLayer( + vocab_size=32768, + hidden_size=hidden_size, + sequence_len=sequence_len, + max_position_embeddings=512, + intermediate_size=4 * hidden_size, + num_heads=16, + dropout_ratio=0.1, + initializer_range=0.02, + ) out = decoder(input_ids, position_ids) return train_program, start_program class TestDecoderLayerPartitioner(unittest.TestCase): - def test_decoder_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - decoder_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(decoder_pretrain_forward) # param should be partition nrank = 4 # col parallel weights = [ - 'linear_0.w_0', 'linear_1.w_0', 'linear_2.w_0', 'linear_4.w_0' + 'linear_0.w_0', + 'linear_1.w_0', + 'linear_2.w_0', + 'linear_4.w_0', ] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 1, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 1, nrank + ) + ) weights = [ - 'linear_0.b_0', 'linear_1.b_0', 'linear_2.b_0', 'linear_4.b_0' + 'linear_0.b_0', + 'linear_1.b_0', + 'linear_2.b_0', + 'linear_4.b_0', ] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) # row parallel weights = ['word_embeddings', 'linear_3.w_0', 'linear_5.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) weights = [ - 'linear_3.b_0', 'pos_embeddings', 'layer_norm_0.b_0', - 'layer_norm_0.w_0', 'linear_5.b_0' + 'linear_3.b_0', + 'pos_embeddings', + 'layer_norm_0.b_0', + 'layer_norm_0.w_0', + 'linear_5.b_0', ] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, 1)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, 1 + ) + ) # row and col allreduce dist_ops = dist_main_prog.global_block().ops dist_ops = [op.type for op in dist_ops] ref_ops = [ - 'c_embedding', 'c_allreduce_sum', 'lookup_table_v2', - 'elementwise_add', 'dropout', 'layer_norm', 'c_identity', - 'matmul_v2', 'elementwise_add', 'reshape2', 'transpose2', - 'c_identity', 'matmul_v2', 'elementwise_add', 'c_identity', - 'matmul_v2', 'elementwise_add', 'reshape2', 'transpose2', - 'reshape2', 'transpose2', 'matmul', 'softmax', 'dropout', - 'matmul_v2', 'transpose2', 'reshape2', 'matmul_v2', - 'c_allreduce_sum', 'elementwise_add', 'dropout', 'elementwise_add', - 'layer_norm', 'c_identity', 'matmul_v2', 'elementwise_add', 'gelu', - 'matmul_v2', 'c_allreduce_sum', 'elementwise_add', 'dropout', - 'elementwise_add' + 'c_embedding', + 'c_allreduce_sum', + 'lookup_table_v2', + 'elementwise_add', + 'dropout', + 'layer_norm', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'reshape2', + 'transpose2', + 'matmul', + 'softmax', + 'dropout', + 'matmul_v2', + 'transpose2', + 'reshape2', + 'matmul_v2', + 'c_allreduce_sum', + 'elementwise_add', + 'dropout', + 'elementwise_add', + 'layer_norm', + 'c_identity', + 'matmul_v2', + 'elementwise_add', + 'gelu', + 'matmul_v2', + 'c_allreduce_sum', + 'elementwise_add', + 'dropout', + 'elementwise_add', ] self.assertTrue(dist_ops == ref_ops) # parameter initialization - var_need_broadcast = sorted([ - 'linear_3.b_0', 'pos_embeddings', 'layer_norm_0.b_0', - 'layer_norm_0.w_0', 'linear_5.b_0' - ]) + var_need_broadcast = sorted( + [ + 'linear_3.b_0', + 'pos_embeddings', + 'layer_norm_0.b_0', + 'layer_norm_0.w_0', + 'linear_5.b_0', + ] + ) self.assertTrue( - initialization_check(_global_parallel_strategy, - dist_context, - dist_startup_prog, - serial_startup_prog, - var_need_broadcast, - _global_process_mesh, - mp_parallel_axis=1, - dp_parallel_axis=0)) + initialization_check( + _global_parallel_strategy, + dist_context, + dist_startup_prog, + serial_startup_prog, + var_need_broadcast, + _global_process_mesh, + mp_parallel_axis=1, + dp_parallel_axis=0, + ) + ) # check var and op all have dist_attr in dist_main_program self.assertTrue( - distributed_attr_check_for_program(dist_main_prog, dist_context)) + distributed_attr_check_for_program(dist_main_prog, dist_context) + ) # check distribured attr serial_op_idx = [0, 5, 9, 11, 23, 28, 31] - dist_op_idx = [[0, 1], [6, 7], [11, 12], [14, 15], [27, 28], [33, 34], - [37, 38]] + dist_op_idx = [ + [0, 1], + [6, 7], + [11, 12], + [14, 15], + [27, 28], + [33, 34], + [37, 38], + ] self.assertTrue( - distributed_attr_check_for_dist_op(serial_main_prog, dist_main_prog, - dist_context, serial_op_idx, - dist_op_idx)) + distributed_attr_check_for_dist_op( + serial_main_prog, + dist_main_prog, + dist_context, + serial_op_idx, + dist_op_idx, + ) + ) def test_decoder_noparallel(self): global _global_parallel_strategy _global_parallel_strategy = "None" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["x", "y"]) - serial_main_prog, serial_startup_prog, dist_main_prog, dist_startup_prog, dist_context = get_programs( - decoder_pretrain_forward) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["x", "y"] + ) + ( + serial_main_prog, + serial_startup_prog, + dist_main_prog, + dist_startup_prog, + dist_context, + ) = get_programs(decoder_pretrain_forward) # param should be partition nrank = 1 # col parallel weights = [ - 'linear_0.w_0', 'linear_1.w_0', 'linear_2.w_0', 'linear_4.w_0' + 'linear_0.w_0', + 'linear_1.w_0', + 'linear_2.w_0', + 'linear_4.w_0', ] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 1, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 1, nrank + ) + ) weights = [ - 'linear_0.b_0', 'linear_1.b_0', 'linear_2.b_0', 'linear_4.b_0' + 'linear_0.b_0', + 'linear_1.b_0', + 'linear_2.b_0', + 'linear_4.b_0', ] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) # row parallel weights = ['word_embeddings', 'linear_3.w_0', 'linear_5.w_0'] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, nrank)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, nrank + ) + ) weights = [ - 'linear_3.b_0', 'pos_embeddings', 'layer_norm_0.b_0', - 'layer_norm_0.w_0', 'linear_5.b_0' + 'linear_3.b_0', + 'pos_embeddings', + 'layer_norm_0.b_0', + 'layer_norm_0.w_0', + 'linear_5.b_0', ] self.assertTrue( - check_tensor_split(dist_main_prog, weights, serial_main_prog, - weights, 0, 1)) + check_tensor_split( + dist_main_prog, weights, serial_main_prog, weights, 0, 1 + ) + ) # row and col allreduce dist_ops = dist_main_prog.global_block().ops dist_ops = [op.type for op in dist_ops] ref_ops = [ - 'lookup_table_v2', 'lookup_table_v2', 'elementwise_add', 'dropout', - 'layer_norm', 'matmul_v2', 'elementwise_add', 'reshape2', - 'transpose2', 'matmul_v2', 'elementwise_add', 'matmul_v2', - 'elementwise_add', 'reshape2', 'transpose2', 'reshape2', - 'transpose2', 'matmul', 'softmax', 'dropout', 'matmul_v2', - 'transpose2', 'reshape2', 'matmul_v2', 'elementwise_add', 'dropout', - 'elementwise_add', 'layer_norm', 'matmul_v2', 'elementwise_add', - 'gelu', 'matmul_v2', 'elementwise_add', 'dropout', 'elementwise_add' + 'lookup_table_v2', + 'lookup_table_v2', + 'elementwise_add', + 'dropout', + 'layer_norm', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'matmul_v2', + 'elementwise_add', + 'matmul_v2', + 'elementwise_add', + 'reshape2', + 'transpose2', + 'reshape2', + 'transpose2', + 'matmul', + 'softmax', + 'dropout', + 'matmul_v2', + 'transpose2', + 'reshape2', + 'matmul_v2', + 'elementwise_add', + 'dropout', + 'elementwise_add', + 'layer_norm', + 'matmul_v2', + 'elementwise_add', + 'gelu', + 'matmul_v2', + 'elementwise_add', + 'dropout', + 'elementwise_add', ] self.assertTrue(dist_ops == ref_ops) dist_ops = dist_startup_prog.global_block().ops dist_ops = [op.type for op in dist_ops] ref_ops = [ - 'gaussian_random', 'gaussian_random', 'gaussian_random', - 'fill_constant', 'gaussian_random', 'fill_constant', - 'gaussian_random', 'fill_constant', 'gaussian_random', - 'fill_constant', 'gaussian_random', 'fill_constant', - 'gaussian_random', 'fill_constant', 'fill_constant', - 'fill_constant', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast', 'c_broadcast', 'c_broadcast', 'c_broadcast', - 'c_broadcast' + 'gaussian_random', + 'gaussian_random', + 'gaussian_random', + 'fill_constant', + 'gaussian_random', + 'fill_constant', + 'gaussian_random', + 'fill_constant', + 'gaussian_random', + 'fill_constant', + 'gaussian_random', + 'fill_constant', + 'gaussian_random', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', + 'c_broadcast', ] self.assertTrue(dist_ops == ref_ops) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner_gpt.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner_gpt.py index 990b42947e6dd..852c6ab74b128 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner_gpt.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_partitioner_gpt.py @@ -88,17 +88,19 @@ class MultiHeadAttention(nn.Layer): Cache = collections.namedtuple("Cache", ["k", "v"]) StaticCache = collections.namedtuple("StaticCache", ["k", "v"]) - def __init__(self, - embed_dim, - num_heads, - dropout=0., - kdim=None, - vdim=None, - need_weights=False, - weight_attr=None, - bias_attr=None, - topo=None, - fuse=False): + def __init__( + self, + embed_dim, + num_heads, + dropout=0.0, + kdim=None, + vdim=None, + need_weights=False, + weight_attr=None, + bias_attr=None, + topo=None, + fuse=False, + ): super(MultiHeadAttention, self).__init__() self.embed_dim = embed_dim self.kdim = kdim if kdim is not None else embed_dim @@ -109,38 +111,36 @@ def __init__(self, self.fuse = fuse self.head_dim = embed_dim // num_heads - assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" if topo is None or topo.mp_info.size == 1: if self.fuse: assert self.kdim == embed_dim assert self.vdim == embed_dim - self.qkv_proj = nn.Linear(embed_dim, - 3 * embed_dim, - weight_attr, - bias_attr=bias_attr) + self.qkv_proj = nn.Linear( + embed_dim, 3 * embed_dim, weight_attr, bias_attr=bias_attr + ) else: - self.q_proj = nn.Linear(embed_dim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.k_proj = nn.Linear(self.kdim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.v_proj = nn.Linear(self.vdim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.out_proj = nn.Linear(embed_dim, - embed_dim, - weight_attr, - bias_attr=bias_attr) + self.q_proj = nn.Linear( + embed_dim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.k_proj = nn.Linear( + self.kdim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.v_proj = nn.Linear( + self.vdim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.out_proj = nn.Linear( + embed_dim, embed_dim, weight_attr, bias_attr=bias_attr + ) def _fuse_prepare_qkv(self, query): mix_layer = self.qkv_proj(query) - mix_layer = paddle.reshape_(mix_layer, - [0, 0, self.num_heads, 3 * self.head_dim]) + mix_layer = paddle.reshape_( + mix_layer, [0, 0, self.num_heads, 3 * self.head_dim] + ) mix_layer = paddle.transpose(mix_layer, [0, 2, 1, 3]) q, k, v = paddle.split(mix_layer, num_or_sections=3, axis=-1) return q, k, v @@ -154,9 +154,11 @@ def _prepare_qkv(self, query, key, value, use_cache=False, cache=None): q = self.q_proj(query) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.q_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.q_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) q = tensor.reshape(x=q, shape=[0, 0, self.num_heads, self.head_dim]) q = tensor.transpose(x=q, perm=[0, 2, 1, 3]) @@ -190,12 +192,16 @@ def compute_kv(self, key, value): v = self.v_proj(value) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.k_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.v_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) + auto.shard_tensor( + self.k_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.v_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) k = tensor.transpose(x=k, perm=[0, 2, 1, 3]) @@ -217,24 +223,22 @@ def gen_cache(self, key, value=None, type=Cache): input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) v = layers.fill_constant_batch_size_like( input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) return self.Cache(k, v) else: # incremental_state with initial value, mainly for usage like UniLM return self.Cache(key, value) - def forward(self, - query, - key, - value, - attn_mask=None, - use_cache=False, - cache=None): + def forward( + self, query, key, value, attn_mask=None, use_cache=False, cache=None + ): r""" Applies multi-head attention to map queries and a set of key-value pairs to outputs. @@ -248,23 +252,25 @@ def forward(self, else: q, k, v = self._prepare_qkv(query, key, value, use_cache, cache) else: - q, k, v, cache = self._prepare_qkv(query, key, value, use_cache, - cache) + q, k, v, cache = self._prepare_qkv( + query, key, value, use_cache, cache + ) # scale dot product attention - product = layers.matmul(x=q, - y=k, - transpose_y=True, - alpha=self.head_dim**-0.5) + product = layers.matmul( + x=q, y=k, transpose_y=True, alpha=self.head_dim**-0.5 + ) if attn_mask is not None: product = product + attn_mask weights = F.softmax(product) if self.dropout: - weights = F.dropout(weights, - self.dropout, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) @@ -276,9 +282,11 @@ def forward(self, out = self.out_proj(out) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.out_proj.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.out_proj.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) outs = [out] if self.need_weights: @@ -293,12 +301,9 @@ class TransformerDecoder(nn.Layer): TransformerDecoder is a stack of N decoder layers. """ - def __init__(self, - decoder_layers, - num_layers, - norm=None, - hidden_size=None, - topo=None): + def __init__( + self, decoder_layers, num_layers, norm=None, hidden_size=None, topo=None + ): super(TransformerDecoder, self).__init__() self.topo = topo @@ -311,13 +316,15 @@ def __init__(self, raise ValueError("Only support LayerNorm") self.checkpoints = [] - def forward(self, - tgt, - memory, - tgt_mask=None, - memory_mask=None, - use_cache=False, - cache=None): + def forward( + self, + tgt, + memory, + tgt_mask=None, + memory_mask=None, + use_cache=False, + cache=None, + ): r""" Applies a stack of N Transformer decoder layers on inputs. If `norm` is provided, also applies layer normalization on the output of last decoder @@ -330,25 +337,31 @@ def forward(self, for i, mod in enumerate(self.layers): if cache is None: if use_cache: - output, new_cache = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache) + output, new_cache = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache, + ) new_caches.append(new_cache) else: - output = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache) + output = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache, + ) else: - output, new_cache = mod(output, - memory, - tgt_mask=tgt_mask, - use_cache=use_cache, - cache=cache[i]) + output, new_cache = mod( + output, + memory, + tgt_mask=tgt_mask, + use_cache=use_cache, + cache=cache[i], + ) new_caches.append(new_cache) self.checkpoints.append(output.name) @@ -363,7 +376,7 @@ def gen_cache(self, memory, do_zip=False): produced by `TransformerDecoderLayer.gen_cache`. See `TransformerDecoderLayer.gen_cache` for more details. If `do_zip` is True, apply `zip` on these tuples to get a list with two elements. - """ + """ cache = [layer.gen_cache(memory) for layer in self.layers] if do_zip: cache = list(zip(*cache)) @@ -376,18 +389,20 @@ class TransformerDecoderLayer(nn.Layer): It contains multiheadattention and some linear layers. """ - def __init__(self, - d_model, - nhead, - dim_feedforward, - dropout=0.1, - activation="gelu", - attn_dropout=None, - act_dropout=None, - normalize_before=True, - weight_attr=None, - bias_attr=None, - topo=None): + def __init__( + self, + d_model, + nhead, + dim_feedforward, + dropout=0.1, + activation="gelu", + attn_dropout=None, + act_dropout=None, + normalize_before=True, + weight_attr=None, + bias_attr=None, + topo=None, + ): self._config = locals() self._config.pop("self") self._config.pop("__class__", None) # py3 @@ -400,21 +415,27 @@ def __init__(self, weight_attrs = _convert_param_attr_to_list(weight_attr, 3) bias_attrs = _convert_param_attr_to_list(bias_attr, 3) - self.self_attn = MultiHeadAttention(d_model, - nhead, - dropout=attn_dropout, - weight_attr=weight_attrs[0], - bias_attr=bias_attrs[0], - topo=topo) + self.self_attn = MultiHeadAttention( + d_model, + nhead, + dropout=attn_dropout, + weight_attr=weight_attrs[0], + bias_attr=bias_attrs[0], + topo=topo, + ) if topo is None or topo.mp_info.size == 1: - self.linear1 = nn.Linear(d_model, - dim_feedforward, - weight_attrs[2], - bias_attr=bias_attrs[2]) - self.linear2 = nn.Linear(dim_feedforward, - d_model, - weight_attrs[2], - bias_attr=bias_attrs[2]) + self.linear1 = nn.Linear( + d_model, + dim_feedforward, + weight_attrs[2], + bias_attr=bias_attrs[2], + ) + self.linear2 = nn.Linear( + dim_feedforward, + d_model, + weight_attrs[2], + bias_attr=bias_attrs[2], + ) self.norm1 = nn.LayerNorm(d_model, epsilon=1e-5) self.norm2 = nn.LayerNorm(d_model, epsilon=1e-5) @@ -431,8 +452,9 @@ def forward(self, tgt, memory, tgt_mask=None, use_cache=False, cache=None): if use_cache is False: tgt = self.self_attn(tgt, tgt, tgt, tgt_mask, use_cache, cache) else: - tgt, incremental_cache = self.self_attn(tgt, tgt, tgt, tgt_mask, - use_cache, cache) + tgt, incremental_cache = self.self_attn( + tgt, tgt, tgt, tgt_mask, use_cache, cache + ) tgt = residual + self.dropout1(tgt) if not self.normalize_before: tgt = self.norm1(tgt) @@ -442,12 +464,16 @@ def forward(self, tgt, memory, tgt_mask=None, use_cache=False, cache=None): tgt = self.norm2(tgt) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.linear1.weight, - process_mesh=_global_process_mesh, - shard_spec=[None, "mp"]) - auto.shard_tensor(self.linear2.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.linear1.weight, + process_mesh=_global_process_mesh, + shard_spec=[None, "mp"], + ) + auto.shard_tensor( + self.linear2.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) # tgt = self.dropout2( # self.linear2(F.gelu( @@ -463,8 +489,9 @@ def forward(self, tgt, memory, tgt_mask=None, use_cache=False, cache=None): return tgt if use_cache is False else (tgt, incremental_cache) def gen_cache(self, memory): - incremental_cache = self.self_attn.gen_cache(memory, - type=self.self_attn.Cache) + incremental_cache = self.self_attn.gen_cache( + memory, type=self.self_attn.Cache + ) return incremental_cache @@ -473,29 +500,38 @@ class GPTEmbeddings(nn.Layer): Include embeddings from word, position and token_type embeddings """ - def __init__(self, - vocab_size, - hidden_size=768, - hidden_dropout_prob=0.1, - max_position_embeddings=512, - type_vocab_size=16, - initializer_range=0.02, - topo=None): + def __init__( + self, + vocab_size, + hidden_size=768, + hidden_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + topo=None, + ): super(GPTEmbeddings, self).__init__() if topo is None or topo.mp_info.size == 1: self.word_embeddings = nn.Embedding( vocab_size, hidden_size, - weight_attr=paddle.ParamAttr(name="word_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, - std=initializer_range))) + weight_attr=paddle.ParamAttr( + name="word_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ), + ), + ) self.position_embeddings = nn.Embedding( max_position_embeddings, hidden_size, - weight_attr=paddle.ParamAttr(name="pos_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, std=initializer_range))) + weight_attr=paddle.ParamAttr( + name="pos_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ), + ), + ) self.dropout = nn.Dropout(hidden_dropout_prob) @@ -508,9 +544,11 @@ def forward(self, input_ids, position_ids=None): input_embedings = self.word_embeddings(input_ids) if _global_parallel_strategy in ["mp", "dp_mp"]: - auto.shard_tensor(self.word_embeddings.weight, - process_mesh=_global_process_mesh, - shard_spec=["mp", None]) + auto.shard_tensor( + self.word_embeddings.weight, + process_mesh=_global_process_mesh, + shard_spec=["mp", None], + ) position_embeddings = self.position_embeddings(position_ids) embeddings = input_embedings + position_embeddings @@ -523,20 +561,22 @@ class GPTModel(nn.Layer): The base model of gpt. """ - def __init__(self, - vocab_size, - hidden_size=768, - num_hidden_layers=4, - num_attention_heads=12, - intermediate_size=3072, - hidden_act="gelu", - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - max_position_embeddings=512, - type_vocab_size=16, - initializer_range=0.02, - pad_token_id=0, - topo=None): + def __init__( + self, + vocab_size, + hidden_size=768, + num_hidden_layers=4, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + pad_token_id=0, + topo=None, + ): super(GPTModel, self).__init__() self.pad_token_id = pad_token_id @@ -549,71 +589,94 @@ def __init__(self, if self.pipline_mode: self.layer_per_stage = num_hidden_layers // self.topo.pp_info.size - self.embeddings = GPTEmbeddings(vocab_size, hidden_size, - hidden_dropout_prob, - max_position_embeddings, - type_vocab_size, self.initializer_range, - topo) + self.embeddings = GPTEmbeddings( + vocab_size, + hidden_size, + hidden_dropout_prob, + max_position_embeddings, + type_vocab_size, + self.initializer_range, + topo, + ) decoder_layers = nn.LayerList() for i in range(num_hidden_layers): DecoderLayer = TransformerDecoderLayer decoder_layers.append( - DecoderLayer(d_model=hidden_size, - nhead=num_attention_heads, - dim_feedforward=intermediate_size, - dropout=hidden_dropout_prob, - activation=hidden_act, - attn_dropout=attention_probs_dropout_prob, - act_dropout=hidden_dropout_prob, - weight_attr=paddle.ParamAttr( - initializer=nn.initializer.Normal( - mean=0.0, std=self.initializer_range)), - bias_attr=None, - topo=topo)) + DecoderLayer( + d_model=hidden_size, + nhead=num_attention_heads, + dim_feedforward=intermediate_size, + dropout=hidden_dropout_prob, + activation=hidden_act, + attn_dropout=attention_probs_dropout_prob, + act_dropout=hidden_dropout_prob, + weight_attr=paddle.ParamAttr( + initializer=nn.initializer.Normal( + mean=0.0, std=self.initializer_range + ) + ), + bias_attr=None, + topo=topo, + ) + ) Decoder = TransformerDecoder - self.decoder = Decoder(decoder_layers, - num_hidden_layers, - norm="LayerNorm", - hidden_size=hidden_size, - topo=topo) + self.decoder = Decoder( + decoder_layers, + num_hidden_layers, + norm="LayerNorm", + hidden_size=hidden_size, + topo=topo, + ) self.checkpoints = [] - def forward(self, - input_ids, - position_ids=None, - attention_mask=None, - use_cache=False, - cache=None): + def forward( + self, + input_ids, + position_ids=None, + attention_mask=None, + use_cache=False, + cache=None, + ): self.checkpoints = [] if attention_mask is None: length = paddle.shape(input_ids)[1] # Use bool mask attention_mask = paddle.tensor.tril( - paddle.ones((length, length), - dtype=self.embeddings.word_embeddings.weight.dtype)) + paddle.ones( + (length, length), + dtype=self.embeddings.word_embeddings.weight.dtype, + ) + ) if position_ids is None: past_length = 0 if cache is not None: past_length = paddle.shape(cache[0].k)[-2] - position_ids = paddle.arange(past_length, - paddle.shape(input_ids)[-1] + - past_length, - dtype='int64') + position_ids = paddle.arange( + past_length, + paddle.shape(input_ids)[-1] + past_length, + dtype='int64', + ) position_ids = position_ids.unsqueeze(0) # .expand_as(input_ids) position_ids = paddle.fluid.layers.expand_as( - position_ids, input_ids) - embedding_output = self.embeddings(input_ids=input_ids, - position_ids=position_ids) + position_ids, input_ids + ) + embedding_output = self.embeddings( + input_ids=input_ids, position_ids=position_ids + ) # TODO, use registered buffer - causal_mask = paddle.tensor.triu(paddle.ones( - (paddle.shape(input_ids)[-1], paddle.shape(input_ids)[-1])) * -1e9, - diagonal=1) + causal_mask = paddle.tensor.triu( + paddle.ones( + (paddle.shape(input_ids)[-1], paddle.shape(input_ids)[-1]) + ) + * -1e9, + diagonal=1, + ) if attention_mask is not None: attention_mask = attention_mask + causal_mask @@ -623,11 +686,13 @@ def forward(self, # The tensor returned by triu not in static graph. attention_mask.stop_gradient = True - encoder_outputs = self.decoder(embedding_output, - memory=None, - tgt_mask=attention_mask, - use_cache=use_cache, - cache=cache) + encoder_outputs = self.decoder( + embedding_output, + memory=None, + tgt_mask=attention_mask, + use_cache=use_cache, + cache=cache, + ) self.checkpoints.extend(self.decoder.checkpoints) return encoder_outputs @@ -649,11 +714,12 @@ def __init__(self, gpt): def parallel_matmul(self, lm_output, logit_weights, parallel_output, topo): if topo is not None and topo.mp_info.size > 1: input_parallel = paddle.distributed.collective._c_identity( - lm_output, group=None) + lm_output, group=None + ) - logits = paddle.matmul(input_parallel, - logit_weights, - transpose_y=True) + logits = paddle.matmul( + input_parallel, logit_weights, transpose_y=True + ) if parallel_output: return logits @@ -663,24 +729,29 @@ def parallel_matmul(self, lm_output, logit_weights, parallel_output, topo): logits = paddle.matmul(lm_output, logit_weights, transpose_y=True) return logits - def forward(self, - input_ids, - position_ids=None, - attention_mask=None, - masked_positions=None, - use_cache=False, - cache=None): - outputs = self.gpt(input_ids, - position_ids=position_ids, - attention_mask=attention_mask, - use_cache=use_cache, - cache=cache) + def forward( + self, + input_ids, + position_ids=None, + attention_mask=None, + masked_positions=None, + use_cache=False, + cache=None, + ): + outputs = self.gpt( + input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + use_cache=use_cache, + cache=cache, + ) if use_cache: encoder_outputs, cached_kvs = outputs[:2] else: encoder_outputs = outputs - logits = self.parallel_matmul(encoder_outputs, self.weight, True, - self.gpt.topo) + logits = self.parallel_matmul( + encoder_outputs, self.weight, True, self.gpt.topo + ) if use_cache: return logits, cached_kvs @@ -699,11 +770,14 @@ def __init__(self, topo=None): if topo is None or topo.mp_info.size == 1: self.loss_func = paddle.nn.CrossEntropyLoss(reduction="none") else: - self.loss_func = paddle.distributed.collective._c_softmax_with_cross_entropy + self.loss_func = ( + paddle.distributed.collective._c_softmax_with_cross_entropy + ) def forward(self, prediction_scores, masked_lm_labels, loss_mask): - masked_lm_loss = self.loss_func(prediction_scores, - masked_lm_labels.unsqueeze(2)) + masked_lm_loss = self.loss_func( + prediction_scores, masked_lm_labels.unsqueeze(2) + ) loss_mask = loss_mask.reshape([-1]) masked_lm_loss = paddle.sum(masked_lm_loss.reshape([-1]) * loss_mask) @@ -712,45 +786,51 @@ def forward(self, prediction_scores, masked_lm_labels, loss_mask): def gpt_pretrain_forward(train_program, startup_program): - with static.program_guard(train_program, - startup_program), utils.unique_name.guard(): + with static.program_guard( + train_program, startup_program + ), utils.unique_name.guard(): batch_size = 16 sequence_len = 512 - input_ids = static.data(name="input_ids", - shape=[batch_size, sequence_len], - dtype='int64') - position_ids = static.data(name="position_ids", - shape=[batch_size, sequence_len], - dtype='int64') + input_ids = static.data( + name="input_ids", shape=[batch_size, sequence_len], dtype='int64' + ) + position_ids = static.data( + name="position_ids", shape=[batch_size, sequence_len], dtype='int64' + ) attention_mask = static.data( name="attention_mask", shape=[batch_size, 1, sequence_len, sequence_len], - dtype='float64') - labels = static.data(name="labels", - shape=[batch_size, sequence_len], - dtype='int64') - loss_mask = static.data(name="loss_mask", - shape=[batch_size, sequence_len], - dtype='float64') + dtype='float64', + ) + labels = static.data( + name="labels", shape=[batch_size, sequence_len], dtype='int64' + ) + loss_mask = static.data( + name="loss_mask", shape=[batch_size, sequence_len], dtype='float64' + ) if _global_parallel_strategy in ["dp", "dp_mp"]: - auto.shard_tensor(input_ids, - process_mesh=_global_process_mesh, - shard_spec=["dp", None]) - - gpt = GPTModel(vocab_size=32768, - hidden_size=768, - num_hidden_layers=2, - num_attention_heads=12, - intermediate_size=4096, - hidden_act="gelu", - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - max_position_embeddings=1024, - type_vocab_size=16, - initializer_range=0.02, - pad_token_id=0, - topo=None) + auto.shard_tensor( + input_ids, + process_mesh=_global_process_mesh, + shard_spec=["dp", None], + ) + + gpt = GPTModel( + vocab_size=32768, + hidden_size=768, + num_hidden_layers=2, + num_attention_heads=12, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=1024, + type_vocab_size=16, + initializer_range=0.02, + pad_token_id=0, + topo=None, + ) model = GPTForPretraining(gpt) @@ -764,29 +844,26 @@ def gpt_pretrain_forward(train_program, startup_program): class FakeStrategy(object): - def __init__(self): self.amp = False self.recompute = False class FakeFleet(object): - def __init__(self): self.user_defined_optimizer = None self._user_defined_strategy = FakeStrategy() class TestGPTPartitioner(unittest.TestCase): - def test_gpt_dp_mp(self): global _global_parallel_strategy _global_parallel_strategy = "dp_mp" global _global_process_mesh - _global_process_mesh = auto.ProcessMesh(mesh=[[0, 1, 2, 3], - [4, 5, 6, 7]], - dim_names=["dp", "mp"]) + _global_process_mesh = auto.ProcessMesh( + mesh=[[0, 1, 2, 3], [4, 5, 6, 7]], dim_names=["dp", "mp"] + ) train_program = static.Program() startup_program = static.Program() @@ -795,24 +872,33 @@ def test_gpt_dp_mp(self): dist_context.process_mesh = _global_process_mesh train_program, startup_program, loss = gpt_pretrain_forward( - train_program, startup_program) + train_program, startup_program + ) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) # serial backward pass - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) rank_id = 3 partitioner = Partitioner(dist_context, rank_id) - auto_parallel_main_prog, auto_parallel_startup_prog, params_grads = partitioner.partition( - complete_train_program, startup_program, params_grads) + ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + params_grads, + ) = partitioner.partition( + complete_train_program, startup_program, params_grads + ) nrank = 4 # col parallel @@ -822,59 +908,96 @@ def test_gpt_dp_mp(self): 'linear_10.w_0', ] self.assertTrue( - check_tensor_split(auto_parallel_main_prog, weights, - complete_train_program, weights, 1, nrank)) + check_tensor_split( + auto_parallel_main_prog, + weights, + complete_train_program, + weights, + 1, + nrank, + ) + ) # row parallel weights = ['word_embeddings', 'linear_9.w_0', 'linear_11.w_0'] self.assertTrue( - check_tensor_split(auto_parallel_main_prog, weights, - complete_train_program, weights, 0, nrank)) + check_tensor_split( + auto_parallel_main_prog, + weights, + complete_train_program, + weights, + 0, + nrank, + ) + ) weights = ['pos_embeddings', 'layer_norm_0.b_0', 'layer_norm_4.w_0'] self.assertTrue( - check_tensor_split(auto_parallel_main_prog, weights, - complete_train_program, weights, 0, 1)) + check_tensor_split( + auto_parallel_main_prog, + weights, + complete_train_program, + weights, + 0, + 1, + ) + ) all_params = sorted( - [param.name for param in startup_program.all_parameters()]) + [param.name for param in startup_program.all_parameters()] + ) allreduce_grads = [ - 'layer_norm_5.tmp_2', 'layer_norm_5.tmp_2', 'layer_norm_5.tmp_2', - 'layer_norm_6.tmp_2', 'layer_norm_7.tmp_2', 'layer_norm_7.tmp_2', - 'layer_norm_7.tmp_2', 'layer_norm_8.tmp_2' + 'layer_norm_5.tmp_2', + 'layer_norm_5.tmp_2', + 'layer_norm_5.tmp_2', + 'layer_norm_6.tmp_2', + 'layer_norm_7.tmp_2', + 'layer_norm_7.tmp_2', + 'layer_norm_7.tmp_2', + 'layer_norm_8.tmp_2', ] process_mesh = _global_process_mesh mp_parallel_axis = 1 dp_parallel_axis = 0 - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, mp_parallel_axis, - 3) + group_ranks = _get_comm_group( + process_mesh.processes, process_mesh.topology, mp_parallel_axis, 3 + ) mp_ring_id = new_process_group(group_ranks).id - group_ranks = _get_comm_group(process_mesh.processes, - process_mesh.topology, dp_parallel_axis, - 3) + group_ranks = _get_comm_group( + process_mesh.processes, process_mesh.topology, dp_parallel_axis, 3 + ) dp_ring_id = new_process_group(group_ranks).id - tensor_parallel_allreduce_vars = sorted([ - op.desc.output_arg_names()[0].split("@")[0] - for op in auto_parallel_main_prog.global_block().ops - if (op.type == "c_allreduce_sum" and op.attr('op_role') == 1 - and op.desc.attr("ring_id") == mp_ring_id) - ]) - data_parallel_allreduce_vars = sorted([ - op.desc.output_arg_names()[0].split("@")[0] - for op in auto_parallel_main_prog.global_block().ops - if (op.type == "c_allreduce_sum" - and op.desc.attr("ring_id") == dp_ring_id) - ]) + tensor_parallel_allreduce_vars = sorted( + [ + op.desc.output_arg_names()[0].split("@")[0] + for op in auto_parallel_main_prog.global_block().ops + if ( + op.type == "c_allreduce_sum" + and op.attr('op_role') == 1 + and op.desc.attr("ring_id") == mp_ring_id + ) + ] + ) + data_parallel_allreduce_vars = sorted( + [ + op.desc.output_arg_names()[0].split("@")[0] + for op in auto_parallel_main_prog.global_block().ops + if ( + op.type == "c_allreduce_sum" + and op.desc.attr("ring_id") == dp_ring_id + ) + ] + ) self.assertTrue(all_params == data_parallel_allreduce_vars) self.assertTrue(allreduce_grads == tensor_parallel_allreduce_vars) self.assertTrue( - is_valid_completed_program(dist_context, auto_parallel_main_prog)) + is_valid_completed_program(dist_context, auto_parallel_main_prog) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard.py index c604bf299afac..e47cc35d6afa7 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard.py @@ -26,7 +26,10 @@ from paddle.distributed.auto_parallel.parallelizer import AutoParallelizer from paddle.distributed.auto_parallel.partitioner import Partitioner from paddle.distributed.auto_parallel.reshard import Resharder -from paddle.distributed.auto_parallel.process_group import _g_process_group_map, ProcessGroup +from paddle.distributed.auto_parallel.process_group import ( + _g_process_group_map, + ProcessGroup, +) paddle.enable_static() _global_parallel_strategy = None @@ -36,26 +39,26 @@ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -63,10 +66,12 @@ def forward(self, input): auto.shard_tensor(self.linear0.weight, PP_MESH_0, [None, None]) auto.shard_tensor(self.linear1.weight, PP_MESH_1, [None, None]) else: - auto.shard_tensor(self.linear0.weight, _global_process_mesh, - [None, None]) - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - [None, None]) + auto.shard_tensor( + self.linear0.weight, _global_process_mesh, [None, None] + ) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, [None, None] + ) out = self.norm(input) out = self.linear0(out) @@ -77,17 +82,18 @@ def forward(self, input): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) if _global_parallel_strategy == "pp": auto.shard_tensor(input, PP_MESH_0, [None, None]) @@ -97,9 +103,11 @@ def mlp_forward(train_program, start_program): else: auto.shard_tensor(input, _global_process_mesh, [None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -108,13 +116,16 @@ def mlp_forward(train_program, start_program): return loss, train_program, start_program -def get_dist_prog(train_program, - startup_program, - dist_context, - rank_id, - change_process_mesh=False): - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) +def get_dist_prog( + train_program, + startup_program, + dist_context, + rank_id, + change_process_mesh=False, +): + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) fleet._user_defined_strategy = fleet.DistributedStrategy() fleet.user_defined_optimizer = paddle.fluid.optimizer.AdamOptimizer() @@ -124,30 +135,43 @@ def get_dist_prog(train_program, # serial forward & backward completion completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) if change_process_mesh: global PP_MESH_1 dist_context.get_tensor_dist_attr_for_program( - train_program.global_block( - ).vars["gelu_0.tmp_0"]).process_mesh = PP_MESH_1 - - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + train_program.global_block().vars["gelu_0.tmp_0"] + ).process_mesh = PP_MESH_1 + + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) # logical partition partitioner = Partitioner(dist_context, rank_id) - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads = partitioner.partition( - complete_train_program, startup_program, params_grads) + ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) = partitioner.partition( + complete_train_program, startup_program, params_grads + ) partitioned_optimize_ops = parallelizer._apply_optimize( - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads) + auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + ) - return auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + return ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) def check_backward_dist_attr(dist_context, dist_main_prog, op_need_check): @@ -159,16 +183,28 @@ def check_backward_dist_attr(dist_context, dist_main_prog, op_need_check): has_dist_attr = False for var_name in op_need_check.input_arg_names: - if not op_dist_attr.get_input_dims_mapping(var_name) or \ - not dist_context.get_tensor_dist_attr_for_program(vars[var_name]).dims_mapping or \ - not dist_context.get_tensor_dist_attr_for_program(vars[var_name]).process_mesh: + if ( + not op_dist_attr.get_input_dims_mapping(var_name) + or not dist_context.get_tensor_dist_attr_for_program( + vars[var_name] + ).dims_mapping + or not dist_context.get_tensor_dist_attr_for_program( + vars[var_name] + ).process_mesh + ): has_dist_attr = False break if has_dist_attr: for var_name in op_need_check.output_arg_names: - if not dist_context.get_tensor_dist_attr_for_program(vars[var_name]).dims_mapping or \ - not dist_context.get_tensor_dist_attr_for_program(vars[var_name]).process_mesh: + if ( + not dist_context.get_tensor_dist_attr_for_program( + vars[var_name] + ).dims_mapping + or not dist_context.get_tensor_dist_attr_for_program( + vars[var_name] + ).process_mesh + ): has_dist_attr = False break @@ -184,14 +220,22 @@ def check_send_recv_result(dist_main_prog, rank_id): for idx, op in enumerate(ops): if op.type == "send_v2" and "gelu_0.tmp_0" in op.input_arg_names: send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0@GRAD" in op.output_arg_names[ - 0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0@GRAD" in op.output_arg_names[0] + ): recv_result = True else: for idx, op in enumerate(ops): - if op.type == "send_v2" and "gelu_0.tmp_0@GRAD" in op.input_arg_names: + if ( + op.type == "send_v2" + and "gelu_0.tmp_0@GRAD" in op.input_arg_names + ): send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0" in op.output_arg_names[0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0" in op.output_arg_names[0] + ): recv_result = True return send_result and recv_result @@ -200,8 +244,10 @@ def check_send_recv_result(dist_main_prog, rank_id): def check_initialization(dist_startup_prog, rank_id): if rank_id == 0: need_check_params = [ - "layer_norm_0.b_0", "layer_norm_0.w_0", "linear_0.w_0", - "linear_0.b_0" + "layer_norm_0.b_0", + "layer_norm_0.w_0", + "linear_0.w_0", + "linear_0.b_0", ] else: need_check_params = ['linear_1.w_0', 'linear_1.b_0'] @@ -216,7 +262,10 @@ def check_initialization(dist_startup_prog, rank_id): def check_initialization_for_dp(dist_startup_prog): need_check_params = [ - "layer_norm_0.b_0", "layer_norm_0.w_0", "linear_0.w_0", "linear_0.b_0" + "layer_norm_0.b_0", + "layer_norm_0.w_0", + "linear_0.w_0", + "linear_0.b_0", ] + ['linear_1.w_0', 'linear_1.b_0'] params = [] for var_name, var in dist_startup_prog.global_block().vars.items(): @@ -227,12 +276,14 @@ def check_initialization_for_dp(dist_startup_prog): if op.type == "c_broadcast": broadcast_varnames.append(op.output_arg_names[0]) - return sorted(params) == sorted(need_check_params) == sorted( - broadcast_varnames) + return ( + sorted(params) + == sorted(need_check_params) + == sorted(broadcast_varnames) + ) class TestMLPReshard(unittest.TestCase): - def test_complete_backward_annotation(self): global _global_process_mesh _global_process_mesh = auto.ProcessMesh(mesh=[0, 1]) @@ -242,7 +293,8 @@ def test_complete_backward_annotation(self): dist_context = DistributedContext() rank_id = 0 dist_main_prog, dist_startup_prog, dist_params_grads = get_dist_prog( - train_program, startup_program, dist_context, 0) + train_program, startup_program, dist_context, 0 + ) op_need_check = None for op in dist_main_prog.global_block().ops: @@ -252,8 +304,10 @@ def test_complete_backward_annotation(self): # grad op should have dist attr self.assertTrue( - check_backward_dist_attr(dist_context, dist_main_prog, - op_need_check)) + check_backward_dist_attr( + dist_context, dist_main_prog, op_need_check + ) + ) # clear _g_process_group_map _g_process_group_map.clear() @@ -274,9 +328,15 @@ def test_mlp_pp(self): dist_context = DistributedContext() rank_id = 1 dist_main_prog, dist_startup_prog, dist_params_grads = get_dist_prog( - train_program, startup_program, dist_context, rank_id) - resharder = Resharder(dist_main_prog, dist_startup_prog, rank_id, - dist_context, dist_params_grads) + train_program, startup_program, dist_context, rank_id + ) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank_id, + dist_context, + dist_params_grads, + ) resharder.reshard() # check send and recv result @@ -303,9 +363,15 @@ def test_mlp_pp_diff_process_mesh(self): dist_context = DistributedContext() rank_id = 1 dist_main_prog, dist_startup_prog, dist_params_grads = get_dist_prog( - train_program, startup_program, dist_context, rank_id, True) - resharder = Resharder(dist_main_prog, dist_startup_prog, rank_id, - dist_context, dist_params_grads) + train_program, startup_program, dist_context, rank_id, True + ) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank_id, + dist_context, + dist_params_grads, + ) resharder.reshard() # check send and recv result self.assertTrue(check_send_recv_result(dist_main_prog, rank_id)) @@ -326,9 +392,15 @@ def test_mlp_dp(self): dist_context = DistributedContext() rank_id = 0 dist_main_prog, dist_startup_prog, dist_params_grads = get_dist_prog( - train_program, startup_program, dist_context, rank_id) - resharder = Resharder(dist_main_prog, dist_startup_prog, rank_id, - dist_context, dist_params_grads) + train_program, startup_program, dist_context, rank_id + ) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank_id, + dist_context, + dist_params_grads, + ) resharder.reshard() # send and recv should not exist in dp scene. diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_dpmppp.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_dpmppp.py index 4e4991579e9c7..502d5dae1728d 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_dpmppp.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_dpmppp.py @@ -31,33 +31,34 @@ paddle.enable_static() _global_parallel_strategy = "dp_mp_pp" -_global_process_mesh = auto.ProcessMesh([[[0, 1], [4, 5]], [[2, 3], [6, 7]]], - dim_names=["x", "y", "z"]) +_global_process_mesh = auto.ProcessMesh( + [[[0, 1], [4, 5]], [[2, 3], [6, 7]]], dim_names=["x", "y", "z"] +) PP_MESH_0 = auto.ProcessMesh([[0, 1], [4, 5]], dim_names=["x", "y"]) PP_MESH_1 = auto.ProcessMesh([[2, 3], [6, 7]], dim_names=["x", "y"]) class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -68,8 +69,9 @@ def forward(self, input): out = self.linear0(out) out = F.gelu(out, approximate=True) out = self.linear1(out) - param = paddle.fluid.layers.create_parameter([1024, 4096], - paddle.float32) + param = paddle.fluid.layers.create_parameter( + [1024, 4096], paddle.float32 + ) auto.shard_tensor(param, PP_MESH_1, [None, "y"]) out = paddle.fluid.layers.mul(out, param) @@ -77,24 +79,27 @@ def forward(self, input): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) auto.shard_tensor(input, PP_MESH_0, ["x", None]) auto.shard_tensor(label, PP_MESH_1, ["x", None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -106,8 +111,9 @@ def mlp_forward(train_program, start_program): def get_dist_prog(train_program, startup_program, dist_context, rank_id): global _global_process_mesh dist_context.process_mesh = _global_process_mesh - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) fleet._user_defined_strategy = fleet.DistributedStrategy() fleet.user_defined_optimizer = paddle.fluid.optimizer.AdamOptimizer() @@ -117,24 +123,37 @@ def get_dist_prog(train_program, startup_program, dist_context, rank_id): # serial forward & backward completion completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) # logical partition partitioner = Partitioner(dist_context, rank_id) - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads = partitioner.partition( - complete_train_program, startup_program, params_grads) + ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) = partitioner.partition( + complete_train_program, startup_program, params_grads + ) partitioned_optimize_ops = parallelizer._apply_optimize( - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads) + auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + ) - return auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + return ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) def check_send_recv_result(dist_main_prog, rank_id): @@ -145,14 +164,22 @@ def check_send_recv_result(dist_main_prog, rank_id): for idx, op in enumerate(ops): if op.type == "send_v2" and "gelu_0.tmp_0" in op.input_arg_names: send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0@GRAD" in op.output_arg_names[ - 0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0@GRAD" in op.output_arg_names[0] + ): recv_result = True else: for idx, op in enumerate(ops): - if op.type == "send_v2" and "gelu_0.tmp_0@GRAD" in op.input_arg_names: + if ( + op.type == "send_v2" + and "gelu_0.tmp_0@GRAD" in op.input_arg_names + ): send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0" in op.output_arg_names[0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0" in op.output_arg_names[0] + ): recv_result = True return send_result and recv_result @@ -168,14 +195,14 @@ def check_initialization_for_dpmppp(dist_startup_prog): class TestMLPReshard(unittest.TestCase): - def test_mlp_dpmppp(self): train_program = paddle.static.Program() startup_program = paddle.static.Program() dist_context = DistributedContext() rank_id = 2 dist_main_prog, dist_startup_prog, dist_params_grads = get_dist_prog( - train_program, startup_program, dist_context, rank_id) + train_program, startup_program, dist_context, rank_id + ) # test estimator cluster = Cluster() @@ -183,16 +210,23 @@ def test_mlp_dpmppp(self): cost_estimator = CostEstimator(train_program, cluster) global_cost = cost_estimator.estimate(dist_context) max_memory = cost_estimator._estimate_max_memory_by_dist_op( - dist_context) + dist_context + ) # test cache global_cost = cost_estimator.estimate(dist_context) max_memory = cost_estimator._estimate_max_memory_by_dist_op( - dist_context) + dist_context + ) assert global_cost.time > 0 assert max_memory > 0 - resharder = Resharder(dist_main_prog, dist_startup_prog, rank_id, - dist_context, dist_params_grads) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank_id, + dist_context, + dist_params_grads, + ) resharder.reshard() # print_program_with_dist_attr(dist_main_prog, dist_context) # check send and recv result diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_mppp.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_mppp.py index 21efd284b5d69..62ebc76c6fa7b 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_mppp.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_mppp.py @@ -37,37 +37,40 @@ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None self.word_embeddings = nn.Embedding( hidden_size, hidden_size, - weight_attr=paddle.ParamAttr(name="word_embeddings", - initializer=nn.initializer.Normal( - mean=0.0, std=initializer_range))) - - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) - self.linear2 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + weight_attr=paddle.ParamAttr( + name="word_embeddings", + initializer=nn.initializer.Normal( + mean=0.0, std=initializer_range + ), + ), + ) + + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) + self.linear2 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) def forward(self, input): auto.shard_tensor(self.word_embeddings.weight, PP_MESH_0, ["x", None]) @@ -76,8 +79,9 @@ def forward(self, input): auto.shard_tensor(self.linear2.weight, PP_MESH_1, ["x", None]) w_out = self.word_embeddings(input) out = self.linear0(w_out) - param = paddle.fluid.layers.create_parameter([4096, 4096], - paddle.float32) + param = paddle.fluid.layers.create_parameter( + [4096, 4096], paddle.float32 + ) auto.shard_tensor(param, PP_MESH_0, ["x", None]) out = paddle.fluid.layers.mul(out, param) gelu_out = F.gelu(out, approximate=True) @@ -89,22 +93,25 @@ def forward(self, input): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 input = static.data(name="input", shape=[batch_size], dtype='int32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) auto.shard_tensor(input, PP_MESH_0, [None]) auto.shard_tensor(label, PP_MESH_1, [None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -116,8 +123,9 @@ def mlp_forward(train_program, start_program): def get_dist_prog(train_program, startup_program, dist_context, rank_id): global _global_process_mesh dist_context.process_mesh = _global_process_mesh - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) fleet._user_defined_strategy = fleet.DistributedStrategy() fleet.user_defined_optimizer = paddle.fluid.optimizer.AdamOptimizer() @@ -127,23 +135,36 @@ def get_dist_prog(train_program, startup_program, dist_context, rank_id): # serial forward & backward completion completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) - params_grads = parallelizer._generate_backward(complete_train_program, - startup_program, - loss, - parameter_list=None, - no_grad_set=None, - callbacks=None) + params_grads = parallelizer._generate_backward( + complete_train_program, + startup_program, + loss, + parameter_list=None, + no_grad_set=None, + callbacks=None, + ) # logical partition partitioner = Partitioner(dist_context, rank_id) - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads = partitioner.partition( - complete_train_program, startup_program, params_grads) + ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) = partitioner.partition( + complete_train_program, startup_program, params_grads + ) partitioned_optimize_ops = parallelizer._apply_optimize( - auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads) - return auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + auto_parallel_main_prog, auto_parallel_startup_prog, dist_params_grads + ) + return ( + auto_parallel_main_prog, + auto_parallel_startup_prog, + dist_params_grads, + ) def check_send_recv_result(dist_main_prog, rank_id): @@ -154,15 +175,22 @@ def check_send_recv_result(dist_main_prog, rank_id): for idx, op in enumerate(ops): if op.type == "send_v2" and "gelu_0.tmp_0" in op.input_arg_names: send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0@GRAD" in op.output_arg_names[ - 0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0@GRAD" in op.output_arg_names[0] + ): recv_result = True else: for idx, op in enumerate(ops): - if op.type == "send_v2" and "gelu_0.tmp_0@GRAD" in op.input_arg_names[ - 0]: + if ( + op.type == "send_v2" + and "gelu_0.tmp_0@GRAD" in op.input_arg_names[0] + ): send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0" in op.output_arg_names[0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0" in op.output_arg_names[0] + ): recv_result = True return send_result and recv_result @@ -196,23 +224,29 @@ def check_allgather(dist_main_program): class TestMLPReshard(unittest.TestCase): - def test_mlp_mppp(self): train_program = paddle.static.Program() startup_program = paddle.static.Program() dist_context = DistributedContext() rank_id = 2 dist_main_prog, dist_startup_prog, dist_params_grads = get_dist_prog( - train_program, startup_program, dist_context, rank_id) - resharder = Resharder(dist_main_prog, dist_startup_prog, rank_id, - dist_context, dist_params_grads) + train_program, startup_program, dist_context, rank_id + ) + resharder = Resharder( + dist_main_prog, + dist_startup_prog, + rank_id, + dist_context, + dist_params_grads, + ) resharder.reshard() # check send and recv result self.assertTrue(check_send_recv_result(dist_main_prog, rank_id)) # parameter which not been sliced should be the same in the mp scene self.assertTrue( - check_initialization_for_mppp(dist_startup_prog, rank_id)) + check_initialization_for_mppp(dist_startup_prog, rank_id) + ) def test_allgather(self): train_program = paddle.static.Program() @@ -224,8 +258,9 @@ def test_allgather(self): w = paddle.static.data(name="w", shape=[4, 4], dtype='float32') w = auto.shard_tensor(w, process_mesh, [None, None]) - y = paddle.distributed.shard_op(paddle.matmul, process_mesh, - [[None, None], [None, None]])(x, w) + y = paddle.distributed.shard_op( + paddle.matmul, process_mesh, [[None, None], [None, None]] + )(x, w) rank_id = 0 dist_context = DistributedContext() @@ -233,10 +268,14 @@ def test_allgather(self): partitioner = Partitioner(dist_context, rank_id) completer = Completer(dist_context) complete_train_program = completer.complete_forward_annotation( - train_program) + train_program + ) dist_context.block_state.parse_forward_blocks(complete_train_program) - partitioned_main_prog, partitioned_startup_prog, partitioned_params_grads = partitioner.partition( - complete_train_program, startup_program, []) + ( + partitioned_main_prog, + partitioned_startup_prog, + partitioned_params_grads, + ) = partitioner.partition(complete_train_program, startup_program, []) # test estimator cluster = Cluster() @@ -244,16 +283,23 @@ def test_allgather(self): cost_estimator = CostEstimator(train_program, cluster) global_cost = cost_estimator.estimate(dist_context) max_memory = cost_estimator._estimate_max_memory_by_dist_op( - dist_context) + dist_context + ) # test cache global_cost = cost_estimator.estimate(dist_context) max_memory = cost_estimator._estimate_max_memory_by_dist_op( - dist_context) + dist_context + ) assert global_cost.time > 0 assert max_memory > 0 - resharder = Resharder(partitioned_main_prog, partitioned_startup_prog, - rank_id, dist_context, partitioned_params_grads) + resharder = Resharder( + partitioned_main_prog, + partitioned_startup_prog, + rank_id, + dist_context, + partitioned_params_grads, + ) resharder.reshard() # the x should not be slice self.assertTrue(check_allgather(partitioned_main_prog)) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_serial.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_serial.py index ff91bfe19f5ba..697998b9de00f 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_serial.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_reshard_serial.py @@ -15,6 +15,7 @@ import unittest import os + if os.getenv("CUDA_VISIBLE_DEVICES", None) is None: os.environ["CUDA_VISIBLE_DEVICES"] = '0' @@ -24,7 +25,9 @@ import paddle.nn.functional as F import paddle.utils as utils from paddle.distributed.fleet import auto -from paddle.distributed.auto_parallel.dist_context import get_default_distributed_context +from paddle.distributed.auto_parallel.dist_context import ( + get_default_distributed_context, +) from paddle.distributed import fleet paddle.enable_static() @@ -33,26 +36,26 @@ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -60,10 +63,12 @@ def forward(self, input): auto.shard_tensor(self.linear0.weight, PP_MESH_0, [None, None]) auto.shard_tensor(self.linear1.weight, PP_MESH_1, [None, None]) else: - auto.shard_tensor(self.linear0.weight, _global_process_mesh, - [None, None]) - auto.shard_tensor(self.linear1.weight, _global_process_mesh, - [None, None]) + auto.shard_tensor( + self.linear0.weight, _global_process_mesh, [None, None] + ) + auto.shard_tensor( + self.linear1.weight, _global_process_mesh, [None, None] + ) out = self.norm(input) out = self.linear0(out) @@ -74,17 +79,18 @@ def forward(self, input): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) if _global_parallel_strategy == "pp": auto.shard_tensor(input, PP_MESH_0, [None, None]) @@ -94,9 +100,11 @@ def mlp_forward(train_program, start_program): else: auto.shard_tensor(input, _global_process_mesh, [None, None]) - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -105,8 +113,9 @@ def mlp_forward(train_program, start_program): return loss, train_program, start_program -def get_dist_prog_with_parallelizer(train_program, startup_program, - dist_context): +def get_dist_prog_with_parallelizer( + train_program, startup_program, dist_context +): global _global_process_mesh dist_strategy = fleet.DistributedStrategy() @@ -118,18 +127,25 @@ def get_dist_prog_with_parallelizer(train_program, startup_program, dist_strategy.semi_auto = True fleet.init(is_collective=True, strategy=dist_strategy) - loss, train_program, startup_program = mlp_forward(train_program, - startup_program) - - optimizer = paddle.fluid.optimizer.AdamOptimizer(learning_rate=0.00001, - beta1=0.9, - beta2=0.999, - epsilon=1e-08, - grad_clip=None) + loss, train_program, startup_program = mlp_forward( + train_program, startup_program + ) + + optimizer = paddle.fluid.optimizer.AdamOptimizer( + learning_rate=0.00001, + beta1=0.9, + beta2=0.999, + epsilon=1e-08, + grad_clip=None, + ) optimizer = fleet.distributed_optimizer(optimizer) - _, _, distributed_startup_program, distributed_main_program = optimizer.minimize( - loss, startup_program) + ( + _, + _, + distributed_startup_program, + distributed_main_program, + ) = optimizer.minimize(loss, startup_program) return distributed_main_program, distributed_startup_program @@ -142,21 +158,28 @@ def check_send_recv_result(dist_main_prog, rank_id): for idx, op in enumerate(ops): if op.type == "send_v2" and "gelu_0.tmp_0" in op.input_arg_names: send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0@GRAD" in op.output_arg_names[ - 0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0@GRAD" in op.output_arg_names[0] + ): recv_result = True else: for idx, op in enumerate(ops): - if op.type == "send_v2" and "gelu_0.tmp_0@GRAD" in op.input_arg_names: + if ( + op.type == "send_v2" + and "gelu_0.tmp_0@GRAD" in op.input_arg_names + ): send_result = True - if op.type == "recv_v2" and "gelu_0.tmp_0" in op.output_arg_names[0]: + if ( + op.type == "recv_v2" + and "gelu_0.tmp_0" in op.output_arg_names[0] + ): recv_result = True return send_result and recv_result class TestMLPReshard(unittest.TestCase): - def test_mlp_serial(self): global _global_parallel_strategy _global_parallel_strategy = None @@ -168,7 +191,8 @@ def test_mlp_serial(self): dist_context = get_default_distributed_context() rank_id = 0 dist_main_prog, dist_startup_prog = get_dist_prog_with_parallelizer( - train_program, startup_program, dist_context) + train_program, startup_program, dist_context + ) # send and recv should not exist in serial scene. self.assertFalse(check_send_recv_result(dist_main_prog, rank_id)) diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_save_load.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_save_load.py index b2a478b365b1b..7b9cf4c0004a4 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_save_load.py @@ -18,7 +18,6 @@ class TestAutoParallelSaveLoad(TestMultipleGpus): - def test_auto_parallel_save_load(self): self.run_mnist_2gpu('auto_parallel_save_load.py') diff --git a/python/paddle/fluid/tests/unittests/test_auto_parallel_searcher.py b/python/paddle/fluid/tests/unittests/test_auto_parallel_searcher.py index cb5e4e00e9ac2..98880667e3410 100755 --- a/python/paddle/fluid/tests/unittests/test_auto_parallel_searcher.py +++ b/python/paddle/fluid/tests/unittests/test_auto_parallel_searcher.py @@ -22,35 +22,43 @@ from paddle.distributed.fleet import auto from paddle.distributed.auto_parallel.planner import PlanSpace from paddle.distributed.auto_parallel.dist_context import DistributedContext -from paddle.distributed.auto_parallel.dist_attribute import TensorDistributedAttribute -from paddle.distributed.auto_parallel.dist_attribute import OperatorDistributedAttribute -from paddle.distributed.auto_parallel.utils import update_op_dims_mapping_by_default_dist_impl -from paddle.distributed.auto_parallel.utils import update_op_dims_mapping_by_elementwise_like_dist_impl +from paddle.distributed.auto_parallel.dist_attribute import ( + TensorDistributedAttribute, +) +from paddle.distributed.auto_parallel.dist_attribute import ( + OperatorDistributedAttribute, +) +from paddle.distributed.auto_parallel.utils import ( + update_op_dims_mapping_by_default_dist_impl, +) +from paddle.distributed.auto_parallel.utils import ( + update_op_dims_mapping_by_elementwise_like_dist_impl, +) paddle.enable_static() class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -64,21 +72,24 @@ def forward(self, input): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sequence_len = 512 - input = static.data(name="input", - shape=[batch_size, hidden_size], - dtype='float32') - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') + input = static.data( + name="input", shape=[batch_size, hidden_size], dtype='float32' + ) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) loss_func = paddle.nn.CrossEntropyLoss(reduction="none") - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = loss_func(predict, label) @@ -98,18 +109,22 @@ def set_default_dist_attr(program, dist_context, process_mesh): tensor_dist_attr.process_mesh = process_mesh tensor_dist_attr.dims_mapping = [-1 for i in vars[var_name].shape] dist_context.set_tensor_dist_attr_for_program( - vars[var_name], tensor_dist_attr) - op_dist_attr.set_input_dims_mapping(var_name, - tensor_dist_attr.dims_mapping) + vars[var_name], tensor_dist_attr + ) + op_dist_attr.set_input_dims_mapping( + var_name, tensor_dist_attr.dims_mapping + ) for var_name in op.output_arg_names: tensor_dist_attr = TensorDistributedAttribute() tensor_dist_attr.process_mesh = process_mesh tensor_dist_attr.dims_mapping = [-1 for i in vars[var_name].shape] dist_context.set_tensor_dist_attr_for_program( - vars[var_name], tensor_dist_attr) - op_dist_attr.set_output_dims_mapping(var_name, - tensor_dist_attr.dims_mapping) + vars[var_name], tensor_dist_attr + ) + op_dist_attr.set_output_dims_mapping( + var_name, tensor_dist_attr.dims_mapping + ) dist_context.set_op_dist_attr_for_program(op, op_dist_attr) dist_context.add_process_mesh(process_mesh) @@ -123,73 +138,111 @@ def check_process_meshes(processes): def check_pipeline_enumerater(program, process_mesh_topology): - valid_dist_attr_dict, pipeline_process_meshes, global_process_mesh = PlanSpace.enum_valid_dist_attr_for_program( - program, process_mesh_topology, True) - if valid_dist_attr_dict and len( - pipeline_process_meshes) > 1 and not global_process_mesh: + ( + valid_dist_attr_dict, + pipeline_process_meshes, + global_process_mesh, + ) = PlanSpace.enum_valid_dist_attr_for_program( + program, process_mesh_topology, True + ) + if ( + valid_dist_attr_dict + and len(pipeline_process_meshes) > 1 + and not global_process_mesh + ): return True return False def check_nonpipeline_enumerater(program, process_mesh_topology): - valid_dist_attr_dict, pipeline_process_meshes, global_process_mesh = PlanSpace.enum_valid_dist_attr_for_program( - program, process_mesh_topology, False) - if valid_dist_attr_dict and not pipeline_process_meshes and global_process_mesh: + ( + valid_dist_attr_dict, + pipeline_process_meshes, + global_process_mesh, + ) = PlanSpace.enum_valid_dist_attr_for_program( + program, process_mesh_topology, False + ) + if ( + valid_dist_attr_dict + and not pipeline_process_meshes + and global_process_mesh + ): return True return False class TestMLPSearcher(unittest.TestCase): - def test_update(self): train_program = paddle.static.Program() startup_program = paddle.static.Program() - _, train_program, startup_program = mlp_forward(train_program, - startup_program) + _, train_program, startup_program = mlp_forward( + train_program, startup_program + ) global_process_mesh = auto.ProcessMesh(mesh=[0, 1]) dist_context = DistributedContext() set_default_dist_attr(train_program, dist_context, global_process_mesh) ops = train_program.global_block().ops vars = train_program.global_block().vars - from paddle.distributed.auto_parallel.operators.common import get_distributed_operator_impl_container - from paddle.distributed.auto_parallel.operators.common import is_elementwise_op + from paddle.distributed.auto_parallel.operators.common import ( + get_distributed_operator_impl_container, + ) + from paddle.distributed.auto_parallel.operators.common import ( + is_elementwise_op, + ) from paddle.distributed.auto_parallel.dist_op import DistributedOperator for op in ops: dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + op.type + ) if dist_op_impl_container is None: op_dist_attr = dist_context.get_op_dist_attr_for_program(op) dist_op = DistributedOperator(op, op_dist_attr) if is_elementwise_op(op.type): - changed = update_op_dims_mapping_by_elementwise_like_dist_impl( - dist_op) + changed = ( + update_op_dims_mapping_by_elementwise_like_dist_impl( + dist_op + ) + ) self.assertFalse(changed) dist_op.dist_attr.set_output_dims_mapping( - op.output_arg_names[0], [0] + [ - -1 for i in range( - 1, len(vars[op.output_arg_names[0]].shape)) - ]) + op.output_arg_names[0], + [0] + + [ + -1 + for i in range( + 1, len(vars[op.output_arg_names[0]].shape) + ) + ], + ) try: changed = update_op_dims_mapping_by_elementwise_like_dist_impl( - dist_op) + dist_op + ) except: continue self.assertTrue(changed) else: changed = update_op_dims_mapping_by_default_dist_impl( - dist_op) + dist_op + ) self.assertFalse(changed) dist_op.dist_attr.set_output_dims_mapping( - op.output_arg_names[0], [0] + [ - -1 for i in range( - 1, len(vars[op.output_arg_names[0]].shape)) - ]) + op.output_arg_names[0], + [0] + + [ + -1 + for i in range( + 1, len(vars[op.output_arg_names[0]].shape) + ) + ], + ) try: changed = update_op_dims_mapping_by_default_dist_impl( - dist_op) + dist_op + ) except: continue self.assertTrue(changed) @@ -200,13 +253,16 @@ def test_enumerater_and_checker(self): train_program = paddle.static.Program() startup_program = paddle.static.Program() - _, train_program, startup_program = mlp_forward(train_program, - startup_program) + _, train_program, startup_program = mlp_forward( + train_program, startup_program + ) process_mesh_topology = [4] self.assertTrue( - check_pipeline_enumerater(train_program, process_mesh_topology)) + check_pipeline_enumerater(train_program, process_mesh_topology) + ) self.assertTrue( - check_nonpipeline_enumerater(train_program, process_mesh_topology)) + check_nonpipeline_enumerater(train_program, process_mesh_topology) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_auto_search_dist_matmul_op.py b/python/paddle/fluid/tests/unittests/test_auto_search_dist_matmul_op.py index fd84c9e6f0c99..0962b2e8d5bf2 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_search_dist_matmul_op.py +++ b/python/paddle/fluid/tests/unittests/test_auto_search_dist_matmul_op.py @@ -20,8 +20,12 @@ import paddle.utils as utils import paddle.fluid.core as core from paddle.fluid import layers -from paddle.distributed.auto_parallel.operators.common import get_distributed_operator_impl_container -from paddle.distributed.auto_parallel.dist_attribute import OperatorDistributedAttribute +from paddle.distributed.auto_parallel.operators.common import ( + get_distributed_operator_impl_container, +) +from paddle.distributed.auto_parallel.dist_attribute import ( + OperatorDistributedAttribute, +) from paddle.distributed.auto_parallel.dist_op import DistributedOperator paddle.enable_static() @@ -29,26 +33,26 @@ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -61,8 +65,9 @@ def forward(self, input): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sqrt_hidden_size = 32 @@ -75,16 +80,20 @@ def mlp_forward(train_program, start_program): input = embedding(input) input = paddle.reshape(input, [hidden_size, batch_size]) input = paddle.transpose(input, perm=[1, 0]) - matmulinput = static.data(name="matmulinput", - shape=[hidden_size, hidden_size], - dtype='float32') + matmulinput = static.data( + name="matmulinput", + shape=[hidden_size, hidden_size], + dtype='float32', + ) input = layers.matmul(x=input, y=matmulinput) - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -95,37 +104,38 @@ def mlp_forward(train_program, start_program): class TestCompatible(unittest.TestCase): - def test_matmulv2_matmul_2_compatible(self): valid_op_dist_attr_list = [] program = paddle.static.Program() startup_program = paddle.static.Program() loss, program, start_program = mlp_forward(program, startup_program) - with static.program_guard(program, - start_program), utils.unique_name.guard(): - matmulx3 = static.data(name="matmulx3", - shape=[6, 2, 6], - dtype='float32') - matmuly3 = static.data(name="matmuly3", - shape=[6, 6], - dtype='float32') + with static.program_guard( + program, start_program + ), utils.unique_name.guard(): + matmulx3 = static.data( + name="matmulx3", shape=[6, 2, 6], dtype='float32' + ) + matmuly3 = static.data( + name="matmuly3", shape=[6, 6], dtype='float32' + ) output1 = paddle.matmul(x=matmulx3, y=matmuly3) output_1 = layers.matmul(x=matmulx3, y=matmuly3) - matmulx4 = static.data(name="matmulx4", - shape=[6, 6, 2, 6], - dtype='float32') - matmuly4 = static.data(name="matmuly4", - shape=[6, 6, 6, 6], - dtype='float32') + matmulx4 = static.data( + name="matmulx4", shape=[6, 6, 2, 6], dtype='float32' + ) + matmuly4 = static.data( + name="matmuly4", shape=[6, 6, 6, 6], dtype='float32' + ) output2 = paddle.matmul(x=matmulx4, y=matmuly4) output_2 = layers.matmul(x=matmulx4, y=matmuly4) ops = program.global_block().ops vars = program.global_block().vars for idx, op in enumerate(ops): if op.type == 'matmul_v2' or op.type == 'matmul': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() X = op.input_arg_names[0] @@ -135,98 +145,157 @@ def test_matmulv2_matmul_2_compatible(self): op_dist_attr.set_input_dims_mapping(X, [-1, -1]) op_dist_attr.set_input_dims_mapping(Y, [-1, -1]) op_dist_attr.set_output_dims_mapping(out, [-1, -1]) - self.assertTrue(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [1, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, 1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [1, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, 1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, 1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [1, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) if len(vars[X].shape) == 3 and len(vars[Y].shape) == 2: op_dist_attr.set_input_dims_mapping(X, [-1, -1, -1]) op_dist_attr.set_input_dims_mapping(Y, [-1, -1]) op_dist_attr.set_output_dims_mapping(out, [-1, -1, -1]) - self.assertTrue(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [1, -1, -1]) op_dist_attr.set_input_dims_mapping(X, [-1, -1, 1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [1, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, 1, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) if len(vars[X].shape) == 4 and len(vars[Y].shape) == 4: op_dist_attr.set_input_dims_mapping(X, [-1, -1, -1, -1]) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, -1, -1]) op_dist_attr.set_output_dims_mapping(out, [-1, -1, -1, -1]) - self.assertTrue(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [0, -1, -1, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [0, -1, -1, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, 0, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, -1, 0, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, -1, 1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, -1, 0, -1]) - self.assertFalse(impls[2].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[2].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) def test_matmulv2_matmul_1_compatible(self): valid_op_dist_attr_list = [] program = paddle.static.Program() startup_program = paddle.static.Program() loss, program, start_program = mlp_forward(program, startup_program) - with static.program_guard(program, - start_program), utils.unique_name.guard(): - matmulx3 = static.data(name="matmulx3", - shape=[6, 2, 6], - dtype='float32') - matmuly3 = static.data(name="matmuly3", - shape=[6, 6], - dtype='float32') + with static.program_guard( + program, start_program + ), utils.unique_name.guard(): + matmulx3 = static.data( + name="matmulx3", shape=[6, 2, 6], dtype='float32' + ) + matmuly3 = static.data( + name="matmuly3", shape=[6, 6], dtype='float32' + ) output1 = paddle.matmul(x=matmulx3, y=matmuly3) output_1 = layers.matmul(x=matmulx3, y=matmuly3) - matmulx4 = static.data(name="matmulx4", - shape=[6, 6, 6, 6], - dtype='float32') - matmuly4 = static.data(name="matmuly4", - shape=[6, 6, 6, 6], - dtype='float32') + matmulx4 = static.data( + name="matmulx4", shape=[6, 6, 6, 6], dtype='float32' + ) + matmuly4 = static.data( + name="matmuly4", shape=[6, 6, 6, 6], dtype='float32' + ) output2 = paddle.matmul(x=matmulx4, y=matmuly4) output_2 = layers.matmul(x=matmulx4, y=matmuly4) ops = program.global_block().ops vars = program.global_block().vars for idx, op in enumerate(ops): if op.type == 'matmul_v2' or op.type == 'matmul': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() X = op.input_arg_names[0] @@ -238,86 +307,133 @@ def test_matmulv2_matmul_1_compatible(self): op_dist_attr.set_output_dims_mapping(out, [-1, -1]) dist_op = DistributedOperator(op, op_dist_attr) op_dist_attr.set_output_dims_mapping(out, [1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) if len(vars[X].shape) == 3 and len(vars[Y].shape) == 2: op_dist_attr.set_input_dims_mapping(X, [-1, -1, 1]) op_dist_attr.set_input_dims_mapping(Y, [1, -1]) op_dist_attr.set_output_dims_mapping(out, [-1, -1, -1]) - self.assertTrue(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [1, -1, 1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(out, [-1, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, 0, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) if len(vars[X].shape) == 4 and len(vars[Y].shape) == 4: op_dist_attr.set_input_dims_mapping(X, [-1, -1, -1, 1]) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, 1, -1]) op_dist_attr.set_output_dims_mapping(out, [-1, -1, -1, -1]) - self.assertTrue(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [0, -1, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [0, -1, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, 0, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, -1, 0, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, -1, 1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, -1, 0, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) def test_matmulv2_matmul_0_compatible(self): valid_op_dist_attr_list = [] program = paddle.static.Program() startup_program = paddle.static.Program() loss, program, start_program = mlp_forward(program, startup_program) - with static.program_guard(program, - start_program), utils.unique_name.guard(): - matmulx3 = static.data(name="matmulx3", - shape=[6, 2, 6], - dtype='float32') - matmuly3 = static.data(name="matmuly3", - shape=[6, 6], - dtype='float32') + with static.program_guard( + program, start_program + ), utils.unique_name.guard(): + matmulx3 = static.data( + name="matmulx3", shape=[6, 2, 6], dtype='float32' + ) + matmuly3 = static.data( + name="matmuly3", shape=[6, 6], dtype='float32' + ) output1 = paddle.matmul(x=matmulx3, y=matmuly3) output_1 = layers.matmul(x=matmulx3, y=matmuly3) - matmulx4 = static.data(name="matmulx4", - shape=[6, 6, 2, 6], - dtype='float32') - matmuly4 = static.data(name="matmuly4", - shape=[6, 6, 6, 6], - dtype='float32') + matmulx4 = static.data( + name="matmulx4", shape=[6, 6, 2, 6], dtype='float32' + ) + matmuly4 = static.data( + name="matmuly4", shape=[6, 6, 6, 6], dtype='float32' + ) output2 = paddle.matmul(x=matmulx4, y=matmuly4) output_2 = layers.matmul(x=matmulx4, y=matmuly4) ops = program.global_block().ops vars = program.global_block().vars for idx, op in enumerate(ops): if op.type == 'matmul_v2' or op.type == 'matmul': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() X = op.input_arg_names[0] @@ -327,81 +443,150 @@ def test_matmulv2_matmul_0_compatible(self): op_dist_attr.set_input_dims_mapping(X, [-1, -1]) op_dist_attr.set_input_dims_mapping(Y, [-1, 1]) op_dist_attr.set_output_dims_mapping(out, [-1, 1]) - self.assertTrue(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [0, 0]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [0, -1]) op_dist_attr.set_output_dims_mapping(out, [1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) if len(vars[X].shape) == 3 and len(vars[Y].shape) == 2: op_dist_attr.set_input_dims_mapping(X, [-1, -1, -1]) op_dist_attr.set_input_dims_mapping(Y, [-1, 1]) op_dist_attr.set_output_dims_mapping(out, [-1, -1, 1]) - self.assertTrue(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, 0, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, 1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [1, -1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, -1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, 1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) if len(vars[X].shape) == 4 and len(vars[Y].shape) == 4: op_dist_attr.set_input_dims_mapping(X, [-1, -1, -1, -1]) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, -1, 1]) op_dist_attr.set_output_dims_mapping(out, [-1, -1, -1, 1]) - self.assertTrue(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertTrue( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [0, -1, -1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, 1, 1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, 1, -1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(X, [-1, -1, 1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [0, -1, -1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, 1, 1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, -1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_output_dims_mapping(out, [-1, -1, 1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(Y, [-1, -1, 1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_auto_search_dist_op.py b/python/paddle/fluid/tests/unittests/test_auto_search_dist_op.py index 8c80a7ae8087a..169926038965d 100644 --- a/python/paddle/fluid/tests/unittests/test_auto_search_dist_op.py +++ b/python/paddle/fluid/tests/unittests/test_auto_search_dist_op.py @@ -20,8 +20,12 @@ import paddle.utils as utils import paddle.fluid.core as core from paddle.fluid import layers -from paddle.distributed.auto_parallel.operators.common import get_distributed_operator_impl_container -from paddle.distributed.auto_parallel.dist_attribute import OperatorDistributedAttribute +from paddle.distributed.auto_parallel.operators.common import ( + get_distributed_operator_impl_container, +) +from paddle.distributed.auto_parallel.dist_attribute import ( + OperatorDistributedAttribute, +) from paddle.distributed.auto_parallel.dist_op import DistributedOperator paddle.enable_static() @@ -29,26 +33,26 @@ class MLPLayer(nn.Layer): - - def __init__(self, - hidden_size=1024, - intermediate_size=4 * 1024, - initializer_range=0.02): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + initializer_range=0.02, + ): super(MLPLayer, self).__init__() d_model = hidden_size dim_feedforward = intermediate_size weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range)) + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) + ) bias_attr = None - self.linear0 = nn.Linear(d_model, - dim_feedforward, - weight_attr, - bias_attr=bias_attr) - self.linear1 = nn.Linear(dim_feedforward, - d_model, - weight_attr, - bias_attr=bias_attr) + self.linear0 = nn.Linear( + d_model, dim_feedforward, weight_attr, bias_attr=bias_attr + ) + self.linear1 = nn.Linear( + dim_feedforward, d_model, weight_attr, bias_attr=bias_attr + ) self.norm = nn.LayerNorm(d_model, epsilon=1e-5) def forward(self, input): @@ -61,8 +65,9 @@ def forward(self, input): def mlp_forward(train_program, start_program): - with static.program_guard(train_program, - start_program), utils.unique_name.guard(): + with static.program_guard( + train_program, start_program + ), utils.unique_name.guard(): batch_size = 4 hidden_size = 1024 sqrt_hidden_size = 32 @@ -75,16 +80,20 @@ def mlp_forward(train_program, start_program): input = embedding(input) input = paddle.reshape(input, [hidden_size, batch_size]) input = paddle.transpose(input, perm=[1, 0]) - matmulinput = static.data(name="matmulinput", - shape=[hidden_size, hidden_size], - dtype='float32') + matmulinput = static.data( + name="matmulinput", + shape=[hidden_size, hidden_size], + dtype='float32', + ) input = layers.matmul(x=input, y=matmulinput) - label = static.data(name="label", - shape=[batch_size, 1], - dtype='float32') - mlp = MLPLayer(hidden_size=hidden_size, - intermediate_size=4 * hidden_size, - initializer_range=0.02) + label = static.data( + name="label", shape=[batch_size, 1], dtype='float32' + ) + mlp = MLPLayer( + hidden_size=hidden_size, + intermediate_size=4 * hidden_size, + initializer_range=0.02, + ) predict = mlp(input) error_cost = paddle.nn.functional.square_error_cost(predict, label) @@ -95,7 +104,6 @@ def mlp_forward(train_program, start_program): class TestCompatible(unittest.TestCase): - def test_reshape_remove_compatible(self): valid_op_dist_attr_list = [] program = paddle.static.Program() @@ -104,55 +112,95 @@ def test_reshape_remove_compatible(self): ops = program.global_block().ops for idx, op in enumerate(ops): if op.type == 'reshape2': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, -1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, -1, -1, -1]) - self.assertTrue(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, -1, -1, 1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [0, -1, -1, 1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, 1, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, -1, 1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [1, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [0, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [0, -1, -1, -1]) - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, 0, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1]) - - self.assertFalse(impls[1].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, -1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, -1, -1, -1] + ) + self.assertTrue( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, -1, -1, 1] + ) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [0, -1, -1, 1] + ) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, 1, -1, -1] + ) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, -1, 1, -1] + ) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [1, -1, -1] + ) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [0, -1, -1] + ) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [0, -1, -1, -1] + ) + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, 0, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1] + ) + + self.assertFalse( + impls[1].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) def test_reshape_add_compatible(self): valid_op_dist_attr_list = [] @@ -162,55 +210,91 @@ def test_reshape_add_compatible(self): ops = program.global_block().ops for idx, op in enumerate(ops): if op.type == 'reshape2': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], [-1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, -1]) - self.assertTrue(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, 0]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, -1] + ) + self.assertTrue( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, 0] + ) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], [-1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [0, -1]) - - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [1, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1, 1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [0, -1] + ) + + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1] + ) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, 1] + ) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [1, -1] + ) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [1, 1] + ) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1, 1] + ) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], [-1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [0, -1]) - self.assertFalse(impls[0].is_auto_compatible( - DistributedOperator(op, op_dist_attr))) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [0, -1] + ) + self.assertFalse( + impls[0].is_auto_compatible( + DistributedOperator(op, op_dist_attr) + ) + ) def test_transpose_compatible(self): valid_op_dist_attr_list = [] @@ -220,47 +304,58 @@ def test_transpose_compatible(self): ops = program.global_block().ops for idx, op in enumerate(ops): if op.type == 'transpose2': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, -1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, -1, -1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertTrue(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, 0, 0]) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, 0, 0] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [0, 0, 0]) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [0, 0, 0] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [1, -1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [-1, 0, 0]) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [-1, 0, 0] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [0, -1, -1]) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [0, -1, -1] + ) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, -1] + ) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[1], - [0, 1, 1]) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[1], [0, 1, 1] + ) self.assertFalse(impls[0].is_auto_compatible(dist_op)) def test_softmax_compatible(self): @@ -271,23 +366,28 @@ def test_softmax_compatible(self): ops = program.global_block().ops for idx, op in enumerate(ops): if op.type == 'softmax': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertTrue(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [1]) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, 1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, 1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) op.all_attrs()['axis'] = 2 @@ -301,52 +401,66 @@ def test_embedding_compatible(self): ops = program.global_block().ops for idx, op in enumerate(ops): if op.type == 'c_embedding' or op.type == 'lookup_table_v2': - dist_op_impl_container = get_distributed_operator_impl_container( - op.type) + dist_op_impl_container = ( + get_distributed_operator_impl_container(op.type) + ) impls = dist_op_impl_container.impls op_dist_attr = OperatorDistributedAttribute() - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, -1]) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[1], - [1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, -1] + ) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[1], [1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1, -1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertTrue(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, 0, 0]) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, 0, 0] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, 1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, 1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[1], - [-1, 1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[1], [-1, 1] + ) dist_op = DistributedOperator(op, op_dist_attr) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[1], - [1, 1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[1], [1, 1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1, -1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, 1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, 1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[1], - [1, 1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [-1, -1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[1], [1, 1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [-1, -1, -1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) - op_dist_attr.set_input_dims_mapping(op.input_arg_names[0], - [-1, -1]) - op_dist_attr.set_output_dims_mapping(op.output_arg_names[0], - [1, 1, -1]) + op_dist_attr.set_input_dims_mapping( + op.input_arg_names[0], [-1, -1] + ) + op_dist_attr.set_output_dims_mapping( + op.output_arg_names[0], [1, 1, -1] + ) dist_op = DistributedOperator(op, op_dist_attr) self.assertFalse(impls[0].is_auto_compatible(dist_op)) diff --git a/python/paddle/fluid/tests/unittests/test_avoid_twice_initialization.py b/python/paddle/fluid/tests/unittests/test_avoid_twice_initialization.py index 87a074c2b5064..43a2d08731d11 100644 --- a/python/paddle/fluid/tests/unittests/test_avoid_twice_initialization.py +++ b/python/paddle/fluid/tests/unittests/test_avoid_twice_initialization.py @@ -17,7 +17,6 @@ class TestAvoidTwiceInitialization(unittest.TestCase): - def test_avoid_twice_initialization(self): cur_program = fluid.Program() cur_block = cur_program.current_block() @@ -25,24 +24,26 @@ def test_avoid_twice_initialization(self): initializer=fluid.initializer.Constant(value=0.01), shape=[2, 2], dtype='float32', - name='var_a') - cur_block.append_op(type="c_broadcast", - inputs={"X": [var]}, - outputs={"Out": [var]}, - attrs={ - 'root': 0, - 'ring_id': 0, - 'use_calc_stream': False - }) - cur_block.append_op(type="c_sync_comm_stream", - inputs={'X': [var]}, - outputs={'Out': [var]}, - attrs={'ring_id': 0}) + name='var_a', + ) + cur_block.append_op( + type="c_broadcast", + inputs={"X": [var]}, + outputs={"Out": [var]}, + attrs={'root': 0, 'ring_id': 0, 'use_calc_stream': False}, + ) + cur_block.append_op( + type="c_sync_comm_stream", + inputs={'X': [var]}, + outputs={'Out': [var]}, + attrs={'ring_id': 0}, + ) var2 = cur_block.create_parameter( initializer=fluid.initializer.Constant(value=0.01), shape=[2, 2], dtype='float32', - name='var_a') + name='var_a', + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_backward.py b/python/paddle/fluid/tests/unittests/test_backward.py index 28c42b49d3371..a186d57a1fea5 100644 --- a/python/paddle/fluid/tests/unittests/test_backward.py +++ b/python/paddle/fluid/tests/unittests/test_backward.py @@ -56,8 +56,11 @@ class TestBackward(unittest.TestCase): """ def _check_all(self, net): - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) main = fluid.Program() @@ -79,21 +82,28 @@ def _check_backward(self, loss, main_program): no_grad_dict = self._check_stop_gradient(main_program) # 1.2 find_op_path op_path, block_no_grad_set = self._check_op_path( - main_program.block(global_block_idx), [loss], [], no_grad_dict) + main_program.block(global_block_idx), [loss], [], no_grad_dict + ) # 1.3 _find_no_grad_vars no_grad_vars = self._check_find_no_grad_vars( - main_program.block(global_block_idx), op_path, [loss], - block_no_grad_set) + main_program.block(global_block_idx), + op_path, + [loss], + block_no_grad_set, + ) # update no_grad_dict block_no_grad_set.update(no_grad_vars) no_grad_dict[global_block_idx].update( - list(map(fluid.backward._append_grad_suffix_, block_no_grad_set))) + list(map(fluid.backward._append_grad_suffix_, block_no_grad_set)) + ) def _check_params_grad(self, loss, parameter_list=None, no_grad_set=None): - params_grads = fluid.backward.append_backward(loss, parameter_list, - no_grad_set) + params_grads = fluid.backward.append_backward( + loss, parameter_list, no_grad_set + ) params_names = set( - [param_var.name for (param_var, grad_var) in params_grads]) + [param_var.name for (param_var, grad_var) in params_grads] + ) self.assertSetEqual(params_names, self.net.params_names) return params_grads @@ -101,8 +111,10 @@ def _check_params_grad(self, loss, parameter_list=None, no_grad_set=None): def _check_stop_gradient(self, program): no_grad_dict = fluid.backward._get_stop_gradients_(program) if no_grad_dict is not None and isinstance(no_grad_dict, dict): - self.assertSetEqual(no_grad_dict[self.global_block_idx], - self.net.stop_gradient_grad_vars) + self.assertSetEqual( + no_grad_dict[self.global_block_idx], + self.net.stop_gradient_grad_vars, + ) return no_grad_dict @@ -111,26 +123,35 @@ def _check_op_path(self, root_block, outputs, inputs=[], no_grad_dict=None): block_no_grad_set = None else: block_no_grad_set = set( - map(fluid.backward._strip_grad_suffix_, - no_grad_dict[self.global_block_idx])) - op_path = fluid.backward._find_op_path_(root_block, outputs, inputs, - block_no_grad_set) + map( + fluid.backward._strip_grad_suffix_, + no_grad_dict[self.global_block_idx], + ) + ) + op_path = fluid.backward._find_op_path_( + root_block, outputs, inputs, block_no_grad_set + ) op_types = [op.type for op in op_path] self.assertListEqual(op_types, self.net.op_path) return op_path, block_no_grad_set - def _check_find_no_grad_vars(self, root_block, op_path, targets, - block_no_grad_set): + def _check_find_no_grad_vars( + self, root_block, op_path, targets, block_no_grad_set + ): no_grad_vars = fluid.backward._find_no_grad_vars( - root_block, op_path, targets, block_no_grad_set) + root_block, op_path, targets, block_no_grad_set + ) self.assertSetEqual(no_grad_vars, self.net.no_grad_vars) return no_grad_vars def _check_error_param_list(self, net, parameter_list): - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) main = fluid.Program() @@ -144,8 +165,11 @@ def _check_error_param_list(self, net, parameter_list): exe.run(feed=net.init_data()) def _check_error_no_grad_set(self, net, no_grad_set): - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) main = fluid.Program() @@ -160,13 +184,16 @@ def _check_error_no_grad_set(self, net, no_grad_set): class SimpleNet(BackwardNet): - def __init__(self): super(SimpleNet, self).__init__() - self.stop_gradient_grad_vars = set([ - u'x_no_grad@GRAD', u'x2_no_grad@GRAD', u'x3_no_grad@GRAD', - u'label_no_grad@GRAD' - ]) + self.stop_gradient_grad_vars = set( + [ + u'x_no_grad@GRAD', + u'x2_no_grad@GRAD', + u'x3_no_grad@GRAD', + u'label_no_grad@GRAD', + ] + ) self.no_grad_vars = set() self.params_names = set([u'w2v', u'fc_predict.b_0', u'fc_w']) self.op_path = [ @@ -178,7 +205,7 @@ def __init__(self): u'softmax', # fc u'elementwise_sub', u'square', - u'reduce_mean' + u'reduce_mean', ] # loss self.shape = [16, 50] @@ -192,7 +219,7 @@ def init_data(self): 'x_no_grad': x, 'x2_no_grad': x2, 'x3_no_grad': x3, - 'label_no_grad': label + 'label_no_grad': label, } def build_model(self): @@ -200,37 +227,41 @@ def build_model(self): x = fluid.data(name='x_no_grad', shape=self.shape, dtype='int64') x2 = fluid.data(name='x2_no_grad', shape=self.shape, dtype='int64') x3 = fluid.data(name='x3_no_grad', shape=self.shape, dtype='int64') - label = fluid.data(name='label_no_grad', - shape=[self.shape[0], 1], - dtype='float32') + label = fluid.data( + name='label_no_grad', shape=[self.shape[0], 1], dtype='float32' + ) # shared layer, the grad of 'w2v' will be summed and renamed. # To test _addup_repetitive_outputs_ - x_emb = fluid.embedding(x, - size=[100, 64], - param_attr=fluid.ParamAttr(name='w2v')) - x2_emb = fluid.embedding(x2, - size=[100, 64], - param_attr=fluid.ParamAttr(name='w2v')) - x3_emb = fluid.embedding(x3, - size=[100, 64], - param_attr=fluid.ParamAttr(name='w2v')) + x_emb = fluid.embedding( + x, size=[100, 64], param_attr=fluid.ParamAttr(name='w2v') + ) + x2_emb = fluid.embedding( + x2, size=[100, 64], param_attr=fluid.ParamAttr(name='w2v') + ) + x3_emb = fluid.embedding( + x3, size=[100, 64], param_attr=fluid.ParamAttr(name='w2v') + ) # merge layers x_merge = fluid.layers.elementwise_add(x_emb, x2_emb, name='x_add_x2') - x2_merge = fluid.layers.elementwise_add(x2_emb, - x3_emb, - name='x2_add_x3') + x2_merge = fluid.layers.elementwise_add( + x2_emb, x3_emb, name='x2_add_x3' + ) # shared fc_w - predict = fluid.layers.fc(input=x_merge, - size=1, - act='softmax', - param_attr=fluid.ParamAttr(name='fc_w'), - name='fc_predict') + predict = fluid.layers.fc( + input=x_merge, + size=1, + act='softmax', + param_attr=fluid.ParamAttr(name='fc_w'), + name='fc_predict', + ) # useless layer for calculating loss - fc_no_use = fluid.layers.fc(input=x2_merge, - size=1, - act='sigmoid', - param_attr=fluid.ParamAttr(name='fc_w'), - name='fc_no_use') + fc_no_use = fluid.layers.fc( + input=x2_merge, + size=1, + act='sigmoid', + param_attr=fluid.ParamAttr(name='fc_w'), + name='fc_no_use', + ) # loss cost = fluid.layers.square_error_cost(input=predict, label=label) loss = paddle.mean(cost, name='mean_loss') @@ -239,7 +270,6 @@ def build_model(self): class TestSimpleNet(TestBackward): - def test_backward(self): """ Instantiate each NetClass to test backward. @@ -250,7 +280,6 @@ def test_backward(self): class TestGradientsError(unittest.TestCase): - def test_error(self): x = fluid.data(name='x', shape=[None, 2, 8, 8], dtype='float32') x.stop_gradient = False @@ -271,7 +300,6 @@ def test_error(self): class TestSimpleNetWithErrorParamList(TestBackward): - def test_parameter_list_type_error(self): self.global_block_idx = 0 self.net = SimpleNet() @@ -285,7 +313,6 @@ def test_parameter_list_type_error(self): class TestSimpleNetWithErrorNoGradSet(TestBackward): - def test_no_grad_set_type_error(self): self.global_block_idx = 0 self.net = SimpleNet() @@ -299,7 +326,6 @@ def test_no_grad_set_type_error(self): class TestAppendBackwardWithError(unittest.TestCase): - def build_net(self): x = fluid.data(name='x', shape=[None, 13], dtype='int64') y = fluid.data(name='y', shape=[None, 1], dtype='float32') @@ -326,8 +352,9 @@ def test_loss_type_error(self): def test_parameter_list_type_error(self): with self.assertRaises(TypeError): self.param_names[0] = np.random.random([10]) - fluid.backward.append_backward(loss=self.avg_loss, - parameter_list=self.param_names) + fluid.backward.append_backward( + loss=self.avg_loss, parameter_list=self.param_names + ) def test_callback_type_error(self): with self.assertRaises(TypeError): @@ -335,18 +362,19 @@ def test_callback_type_error(self): def callback(block, context): return - fluid.backward.append_backward(loss=self.avg_loss, - callbacks=callback) + fluid.backward.append_backward( + loss=self.avg_loss, callbacks=callback + ) class TestGradientsWithOptimizer(unittest.TestCase): - def _check_grad_op_name(self, forward_list, optimiezed_list): backward_list = [op + "_grad" for op in reversed(forward_list)] idx = optimiezed_list.index(backward_list[0], len(backward_list)) - self.assertListEqual(backward_list, - optimiezed_list[idx:idx + len(backward_list)]) + self.assertListEqual( + backward_list, optimiezed_list[idx : idx + len(backward_list)] + ) def test_gradient_with_optimizer(self): main = fluid.Program() @@ -359,8 +387,12 @@ def test_gradient_with_optimizer(self): opt = paddle.optimizer.Momentum(learning_rate=0.01, momentum=0.9) forward_list = [o.type for o in main.current_block().ops] - optimize_ops, pram_grads = paddle.autograd.backward_mode.gradients_with_optimizer( - main, opt) + ( + optimize_ops, + pram_grads, + ) = paddle.autograd.backward_mode.gradients_with_optimizer( + main, opt + ) optimized_list = [o.type for o in main.current_block().ops] @@ -371,7 +403,6 @@ def test_gradient_with_optimizer(self): # TODO(Aurelius84): add conditional network test class ConditionalNet(BackwardNet): - def __init__(self): super(ConditionalNet, self).__init__() diff --git a/python/paddle/fluid/tests/unittests/test_backward_infer_var_data_type_shape.py b/python/paddle/fluid/tests/unittests/test_backward_infer_var_data_type_shape.py index 9f94124fca6fd..2a16d6d57e3ba 100644 --- a/python/paddle/fluid/tests/unittests/test_backward_infer_var_data_type_shape.py +++ b/python/paddle/fluid/tests/unittests/test_backward_infer_var_data_type_shape.py @@ -20,17 +20,16 @@ class TestBackwardInferVarDataTypeShape(unittest.TestCase): - def test_backward_infer_var_data_type_shape(self): paddle.enable_static() program = fluid.default_main_program() - dy = program.global_block().create_var(name="Tmp@GRAD", - shape=[1, 1], - dtype=np.float32, - persistable=True) + dy = program.global_block().create_var( + name="Tmp@GRAD", shape=[1, 1], dtype=np.float32, persistable=True + ) # invoke warning - fluid.backward._infer_var_data_type_shape_("Tmp@GRAD", - program.global_block()) + fluid.backward._infer_var_data_type_shape_( + "Tmp@GRAD", program.global_block() + ) res = False with warnings.catch_warnings(): res = True diff --git a/python/paddle/fluid/tests/unittests/test_base_layer.py b/python/paddle/fluid/tests/unittests/test_base_layer.py index 993bad8363013..66cf1b489f6e6 100644 --- a/python/paddle/fluid/tests/unittests/test_base_layer.py +++ b/python/paddle/fluid/tests/unittests/test_base_layer.py @@ -24,26 +24,23 @@ class L1(fluid.Layer): - def __init__(self): super(L1, self).__init__() self._param_attr = fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.1)) - self.w1 = self.create_parameter(attr=self._param_attr, - shape=[2, 2], - dtype='float32', - is_bias=False) - self.w2 = self.create_parameter(attr=self._param_attr, - shape=[2, 2], - dtype='float32', - is_bias=False) + initializer=fluid.initializer.Constant(value=0.1) + ) + self.w1 = self.create_parameter( + attr=self._param_attr, shape=[2, 2], dtype='float32', is_bias=False + ) + self.w2 = self.create_parameter( + attr=self._param_attr, shape=[2, 2], dtype='float32', is_bias=False + ) def forward(self): return self.w1 + self.w2 class L2(fluid.Layer): - def __init__(self): super(L2, self).__init__() self.layer1 = L1() @@ -54,7 +51,6 @@ def forward(self): class L3(fluid.Layer): - def __init__(self): super(L3, self).__init__() self.layer1 = L2() @@ -65,7 +61,6 @@ def forward(self): class TestBaseLayer(unittest.TestCase): - def func_test_one_level(self): with fluid.dygraph.guard(): l = L1() @@ -75,9 +70,9 @@ def func_test_one_level(self): for name, _ in l.named_parameters(prefix='l1'): self.assertEqual(name, expected_names[idx]) idx += 1 - np.testing.assert_allclose(ret.numpy(), - 0.2 * np.ones([2, 2]), - rtol=1e-05) + np.testing.assert_allclose( + ret.numpy(), 0.2 * np.ones([2, 2]), rtol=1e-05 + ) def test_one_level(self): with _test_eager_guard(): @@ -102,9 +97,9 @@ def func_test_three_level(self): self.assertEqual(name, expected_names[idx]) idx += 1 ret = l() - np.testing.assert_allclose(ret.numpy(), - 0.8 * np.ones([2, 2]), - rtol=1e-05) + np.testing.assert_allclose( + ret.numpy(), 0.8 * np.ones([2, 2]), rtol=1e-05 + ) def test_three_level(self): with _test_eager_guard(): @@ -143,7 +138,6 @@ def test_add_parameter_with_error(self): class BufferLayer(fluid.Layer): - def __init__(self): super(BufferLayer, self).__init__() buffer_var = to_variable(np.zeros([2, 4]).astype('int32')) @@ -154,13 +148,12 @@ def forward(self): class BufferNet(fluid.Layer): - def __init__(self): super(BufferNet, self).__init__() self.buffer_layer = BufferLayer() - self.w1 = self.create_parameter(shape=[2, 2], - dtype='float32', - is_bias=False) + self.w1 = self.create_parameter( + shape=[2, 2], dtype='float32', is_bias=False + ) buffer_var = to_variable(np.ones([2, 4]).astype('int32')) self.register_buffer("net_buffer", buffer_var) @@ -171,9 +164,7 @@ def forward(self): class TestBuffer(unittest.TestCase): - def func_test_buffers_and_named_buffers(self): - def names(named_buffers): return [name for name, _ in named_buffers] @@ -187,11 +178,14 @@ def names(named_buffers): self.assertEqual(len(net.buffers()), 3) self.assertEqual( names(net.named_buffers()), - ['net_buffer', 'new_buffer', 'buffer_layer.layer_buffer']) + ['net_buffer', 'new_buffer', 'buffer_layer.layer_buffer'], + ) self.assertEqual(len(net.buffers(include_sublayers=False)), 2) - self.assertEqual(names(net.named_buffers(include_sublayers=False)), - ['net_buffer', 'new_buffer']) + self.assertEqual( + names(net.named_buffers(include_sublayers=False)), + ['net_buffer', 'new_buffer'], + ) def test_buffers_and_named_buffers(self): with _test_eager_guard(): @@ -203,25 +197,31 @@ def func_test_register_buffer_with_error(self): net = fluid.Layer() var = to_variable(np.zeros([1])) - with self.assertRaisesRegexp(TypeError, - "name of buffer should be a string"): + with self.assertRaisesRegexp( + TypeError, "name of buffer should be a string" + ): net.register_buffer(12, var) - with self.assertRaisesRegexp(TypeError, - "buffer should be a Paddle.Tensor"): + with self.assertRaisesRegexp( + TypeError, "buffer should be a Paddle.Tensor" + ): if in_dygraph_mode(): - net.register_buffer("buffer_name", - EagerParamBase([2, 2], 'float32')) + net.register_buffer( + "buffer_name", EagerParamBase([2, 2], 'float32') + ) else: - net.register_buffer("buffer_name", - ParamBase([2, 2], 'float32')) + net.register_buffer( + "buffer_name", ParamBase([2, 2], 'float32') + ) - with self.assertRaisesRegexp(KeyError, - "name of buffer can not contain"): + with self.assertRaisesRegexp( + KeyError, "name of buffer can not contain" + ): net.register_buffer("buffer.name", var) - with self.assertRaisesRegexp(KeyError, - "name of buffer can not be empty"): + with self.assertRaisesRegexp( + KeyError, "name of buffer can not be empty" + ): net.register_buffer("", var) net.attr_name = 10 @@ -358,8 +358,9 @@ def func_test_buffer_state_dict(self): net.register_buffer("buffer_var2", var2, persistable=False) self.assertEqual(len(net.state_dict()), 1) - self.assertEqual([name for name, _ in net.state_dict().items()], - ["buffer_var1"]) + self.assertEqual( + [name for name, _ in net.state_dict().items()], ["buffer_var1"] + ) # load state_dict net_load = fluid.Layer() @@ -379,7 +380,6 @@ def assert_var_base_equal(self, var1, var2): class BufferNetWithModification(paddle.nn.Layer): - def __init__(self, shape): super(BufferNetWithModification, self).__init__() @@ -397,7 +397,6 @@ def forward(self, x): class TestModifiedBuffer(unittest.TestCase): - def funcsetUp(self): paddle.disable_static() self.prog_trans = ProgramTranslator() @@ -418,8 +417,9 @@ def func_test_modified(self): st_outs = self._run(True) for i in range(len(dy_outs)): - np.testing.assert_array_equal(dy_outs[i].numpy(), - st_outs[i].numpy()) + np.testing.assert_array_equal( + dy_outs[i].numpy(), st_outs[i].numpy() + ) def test_modified(self): with _test_eager_guard(): @@ -428,7 +428,6 @@ def test_modified(self): class TestLayerTo(unittest.TestCase): - def funcsetUp(self): paddle.disable_static() self.linear = paddle.nn.Linear(2, 2) @@ -442,30 +441,39 @@ def funcsetUp(self): def func_test_to_api(self): self.linear.to(dtype='double') - self.assertEqual(self.linear.weight.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - self.assertEqual(self.linear.buf_name.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - np.testing.assert_allclose(self.linear.weight.grad.numpy(), - self.new_grad, - rtol=1e-05) - self.assertEqual(self.linear.weight._grad_ivar().dtype, - paddle.fluid.core.VarDesc.VarType.FP64) + self.assertEqual( + self.linear.weight.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + self.assertEqual( + self.linear.buf_name.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + np.testing.assert_allclose( + self.linear.weight.grad.numpy(), self.new_grad, rtol=1e-05 + ) + self.assertEqual( + self.linear.weight._grad_ivar().dtype, + paddle.fluid.core.VarDesc.VarType.FP64, + ) self.linear.to() - self.assertEqual(self.linear.weight.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - self.assertEqual(self.linear.buf_name.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - np.testing.assert_allclose(self.linear.weight.grad.numpy(), - self.new_grad, - rtol=1e-05) - self.assertEqual(self.linear.weight._grad_ivar().dtype, - paddle.fluid.core.VarDesc.VarType.FP64) + self.assertEqual( + self.linear.weight.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + self.assertEqual( + self.linear.buf_name.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + np.testing.assert_allclose( + self.linear.weight.grad.numpy(), self.new_grad, rtol=1e-05 + ) + self.assertEqual( + self.linear.weight._grad_ivar().dtype, + paddle.fluid.core.VarDesc.VarType.FP64, + ) for p in self.linear.parameters(): if in_dygraph_mode(): self.assertTrue( - isinstance(p, paddle.fluid.framework.EagerParamBase)) + isinstance(p, paddle.fluid.framework.EagerParamBase) + ) else: self.assertTrue(isinstance(p, paddle.fluid.framework.ParamBase)) @@ -476,9 +484,11 @@ def func_test_to_api(self): self.assertTrue(self.linear.buf_name.place.is_gpu_place()) self.assertEqual(self.linear.buf_name.place.gpu_device_id(), 0) self.assertTrue( - self.linear.weight._grad_ivar().place.is_gpu_place()) + self.linear.weight._grad_ivar().place.is_gpu_place() + ) self.assertEqual( - self.linear.weight._grad_ivar().place.gpu_device_id(), 0) + self.linear.weight._grad_ivar().place.gpu_device_id(), 0 + ) self.linear.to(device='gpu:0') self.assertTrue(self.linear.weight.place.is_gpu_place()) @@ -486,16 +496,20 @@ def func_test_to_api(self): self.assertTrue(self.linear.buf_name.place.is_gpu_place()) self.assertEqual(self.linear.buf_name.place.gpu_device_id(), 0) self.assertTrue( - self.linear.weight._grad_ivar().place.is_gpu_place()) + self.linear.weight._grad_ivar().place.is_gpu_place() + ) self.assertEqual( - self.linear.weight._grad_ivar().place.gpu_device_id(), 0) + self.linear.weight._grad_ivar().place.gpu_device_id(), 0 + ) for p in self.linear.parameters(): if in_dygraph_mode(): self.assertTrue( - isinstance(p, paddle.fluid.framework.EagerParamBase)) + isinstance(p, paddle.fluid.framework.EagerParamBase) + ) else: self.assertTrue( - isinstance(p, paddle.fluid.framework.ParamBase)) + isinstance(p, paddle.fluid.framework.ParamBase) + ) self.linear.to(device=paddle.CPUPlace()) self.assertTrue(self.linear.weight.place.is_cpu_place()) @@ -513,59 +527,77 @@ def func_test_to_api(self): def func_test_to_api_paddle_dtype(self): self.linear.to(dtype=paddle.float64) - self.assertEqual(self.linear.weight.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - self.assertEqual(self.linear.buf_name.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - np.testing.assert_allclose(self.linear.weight.grad.numpy(), - self.new_grad, - rtol=1e-05) - self.assertEqual(self.linear.weight._grad_ivar().dtype, - paddle.fluid.core.VarDesc.VarType.FP64) + self.assertEqual( + self.linear.weight.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + self.assertEqual( + self.linear.buf_name.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + np.testing.assert_allclose( + self.linear.weight.grad.numpy(), self.new_grad, rtol=1e-05 + ) + self.assertEqual( + self.linear.weight._grad_ivar().dtype, + paddle.fluid.core.VarDesc.VarType.FP64, + ) self.linear.to() - self.assertEqual(self.linear.weight.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - self.assertEqual(self.linear.buf_name.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - np.testing.assert_allclose(self.linear.weight.grad.numpy(), - self.new_grad, - rtol=1e-05) - self.assertEqual(self.linear.weight._grad_ivar().dtype, - paddle.fluid.core.VarDesc.VarType.FP64) + self.assertEqual( + self.linear.weight.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + self.assertEqual( + self.linear.buf_name.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + np.testing.assert_allclose( + self.linear.weight.grad.numpy(), self.new_grad, rtol=1e-05 + ) + self.assertEqual( + self.linear.weight._grad_ivar().dtype, + paddle.fluid.core.VarDesc.VarType.FP64, + ) for p in self.linear.parameters(): if in_dygraph_mode(): self.assertTrue( - isinstance(p, paddle.fluid.framework.EagerParamBase)) + isinstance(p, paddle.fluid.framework.EagerParamBase) + ) else: self.assertTrue(isinstance(p, paddle.fluid.framework.ParamBase)) def func_test_to_api_numpy_dtype(self): self.linear.to(dtype=np.float64) - self.assertEqual(self.linear.weight.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - self.assertEqual(self.linear.buf_name.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - np.testing.assert_allclose(self.linear.weight.grad.numpy(), - self.new_grad, - rtol=1e-05) - self.assertEqual(self.linear.weight._grad_ivar().dtype, - paddle.fluid.core.VarDesc.VarType.FP64) + self.assertEqual( + self.linear.weight.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + self.assertEqual( + self.linear.buf_name.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + np.testing.assert_allclose( + self.linear.weight.grad.numpy(), self.new_grad, rtol=1e-05 + ) + self.assertEqual( + self.linear.weight._grad_ivar().dtype, + paddle.fluid.core.VarDesc.VarType.FP64, + ) self.linear.to() - self.assertEqual(self.linear.weight.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - self.assertEqual(self.linear.buf_name.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) - np.testing.assert_allclose(self.linear.weight.grad.numpy(), - self.new_grad, - rtol=1e-05) - self.assertEqual(self.linear.weight._grad_ivar().dtype, - paddle.fluid.core.VarDesc.VarType.FP64) + self.assertEqual( + self.linear.weight.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + self.assertEqual( + self.linear.buf_name.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) + np.testing.assert_allclose( + self.linear.weight.grad.numpy(), self.new_grad, rtol=1e-05 + ) + self.assertEqual( + self.linear.weight._grad_ivar().dtype, + paddle.fluid.core.VarDesc.VarType.FP64, + ) for p in self.linear.parameters(): if in_dygraph_mode(): self.assertTrue( - isinstance(p, paddle.fluid.framework.EagerParamBase)) + isinstance(p, paddle.fluid.framework.EagerParamBase) + ) else: self.assertTrue(isinstance(p, paddle.fluid.framework.ParamBase)) diff --git a/python/paddle/fluid/tests/unittests/test_basic_gru_api.py b/python/paddle/fluid/tests/unittests/test_basic_gru_api.py index c0719d7643c13..339323110a116 100644 --- a/python/paddle/fluid/tests/unittests/test_basic_gru_api.py +++ b/python/paddle/fluid/tests/unittests/test_basic_gru_api.py @@ -34,27 +34,28 @@ def sigmoid(x): y = np.copy(x) y[x < SIGMOID_THRESHOLD_MIN] = SIGMOID_THRESHOLD_MIN y[x > SIGMOID_THRESHOLD_MAX] = SIGMOID_THRESHOLD_MAX - return 1. / (1. + np.exp(-y)) + return 1.0 / (1.0 + np.exp(-y)) def tanh(x): - y = -2. * x + y = -2.0 * x y[y > EXP_MAX_INPUT] = EXP_MAX_INPUT - return (2. / (1. + np.exp(y))) - 1. - - -def gru_np(input, - init_h, - hidden_size, - gate_weight, - gate_bias, - candidate_weight, - candidate_bias, - num_layers=1, - batch_first=False, - is_bidirect=False, - sequence_length=None): - + return (2.0 / (1.0 + np.exp(y))) - 1.0 + + +def gru_np( + input, + init_h, + hidden_size, + gate_weight, + gate_bias, + candidate_weight, + candidate_bias, + num_layers=1, + batch_first=False, + is_bidirect=False, + sequence_length=None, +): def step(step_in, pre_hidden, gate_w, gate_b, candidate_w, candidate_b): concat_1 = np.concatenate([step_in, pre_hidden], 1) @@ -65,8 +66,9 @@ def step(step_in, pre_hidden, gate_w, gate_b, candidate_w, candidate_b): r_hidden = r * pre_hidden - candidate = np.matmul(np.concatenate([step_in, r_hidden], 1), - candidate_w) + candidate = np.matmul( + np.concatenate([step_in, r_hidden], 1), candidate_w + ) candidate += candidate_b c = tanh(candidate) @@ -94,8 +96,9 @@ def step(step_in, pre_hidden, gate_w, gate_b, candidate_w, candidate_b): if is_bidirect: direc_num = 2 if init_h: - init_h = np.reshape(init_h, - shape=[num_layers, direc_num, -1, hidden_size]) + init_h = np.reshape( + init_h, shape=[num_layers, direc_num, -1, hidden_size] + ) else: init_h = np.zeros([num_layers, direc_num, batch_size, hidden_size]) @@ -117,15 +120,19 @@ def get_single_direction_output(rnn_input, mask=None, direc_index=0): for i in range(num_layers): new_hidden = step( - step_input, pre_hidden_array[i], + step_input, + pre_hidden_array[i], gate_weight[direc_index * num_layers + i], gate_bias[direc_index * num_layers + i], candidate_weight[direc_index * num_layers + i], - candidate_bias[direc_index * num_layers + i]) + candidate_bias[direc_index * num_layers + i], + ) if mask is not None: - new_hidden = new_hidden * step_mask + ( - 1 - step_mask) * pre_hidden_array[i] + new_hidden = ( + new_hidden * step_mask + + (1 - step_mask) * pre_hidden_array[i] + ) pre_hidden_array[i] = new_hidden @@ -135,14 +142,15 @@ def get_single_direction_output(rnn_input, mask=None, direc_index=0): rnn_out = np.reshape(rnn_out, [seq_len, -1, hidden_size]) last_hidden_out = np.concatenate(pre_hidden_array, 0) - last_hidden_out = np.reshape(last_hidden_out, - [num_layers, -1, hidden_size]) + last_hidden_out = np.reshape( + last_hidden_out, [num_layers, -1, hidden_size] + ) return rnn_out, last_hidden_out - fw_rnn_out, fw_last_hidden = get_single_direction_output(input, - mask, - direc_index=0) + fw_rnn_out, fw_last_hidden = get_single_direction_output( + input, mask, direc_index=0 + ) if is_bidirect: bw_input = input[::-1] @@ -150,16 +158,17 @@ def get_single_direction_output(rnn_input, mask=None, direc_index=0): if mask is not None: bw_mask = mask[::-1] - bw_rnn_out, bw_last_hidden = get_single_direction_output(bw_input, - bw_mask, - direc_index=1) + bw_rnn_out, bw_last_hidden = get_single_direction_output( + bw_input, bw_mask, direc_index=1 + ) bw_rnn_out = bw_rnn_out[::-1] rnn_out = np.concatenate([fw_rnn_out, bw_rnn_out], 2) last_hidden = np.concatenate([fw_last_hidden, bw_last_hidden], 1) - last_hidden = np.reshape(last_hidden, - [num_layers * direc_num, -1, hidden_size]) + last_hidden = np.reshape( + last_hidden, [num_layers * direc_num, -1, hidden_size] + ) if batch_first: rnn_out = np.transpose(rnn_out, [1, 0, 2]) @@ -176,7 +185,6 @@ def get_single_direction_output(rnn_input, mask=None, direc_index=0): class TestBasicGRUApi(unittest.TestCase): - def setUp(self): self.hidden_size = 10 self.batch_size = 5 @@ -186,15 +194,24 @@ def setUp(self): self.batch_first = False def test_run(self): - x = layers.data(name='x', - shape=[-1, self.batch_size, self.hidden_size], - dtype='float32') - sequence_length = layers.data(name="sequence_length", - shape=[-1], - dtype='float32') - - rnn_out, last_hidden = basic_gru( x, None, self.hidden_size, num_layers=self.num_layers, \ - batch_first = self.batch_first, bidirectional=self.is_bidirect, sequence_length=sequence_length ) + x = layers.data( + name='x', + shape=[-1, self.batch_size, self.hidden_size], + dtype='float32', + ) + sequence_length = layers.data( + name="sequence_length", shape=[-1], dtype='float32' + ) + + rnn_out, last_hidden = basic_gru( + x, + None, + self.hidden_size, + num_layers=self.num_layers, + batch_first=self.batch_first, + bidirectional=self.is_bidirect, + sequence_length=sequence_length, + ) last_hidden.persisbale = True rnn_out.persisbale = True @@ -218,38 +235,52 @@ def test_run(self): for i in range(self.num_layers): gate_w_name = "basic_gru_layers_" + str(i) + "/BasicGRUUnit_0.w_0" gate_b_name = "basic_gru_layers_" + str(i) + "/BasicGRUUnit_0.b_0" - candidate_w_name = "basic_gru_layers_" + str( - i) + "/BasicGRUUnit_0.w_1" - candidate_b_name = "basic_gru_layers_" + str( - i) + "/BasicGRUUnit_0.b_1" + candidate_w_name = ( + "basic_gru_layers_" + str(i) + "/BasicGRUUnit_0.w_1" + ) + candidate_b_name = ( + "basic_gru_layers_" + str(i) + "/BasicGRUUnit_0.b_1" + ) gate_w = np.array( - fluid.global_scope().find_var(gate_w_name).get_tensor()) - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float32') + fluid.global_scope().find_var(gate_w_name).get_tensor() + ) + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_w_name).get_tensor().set( - gate_w, place) + gate_w, place + ) gate_b = np.array( - fluid.global_scope().find_var(gate_b_name).get_tensor()) - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float32') + fluid.global_scope().find_var(gate_b_name).get_tensor() + ) + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_b_name).get_tensor().set( - gate_b, place) + gate_b, place + ) candidate_w = np.array( - fluid.global_scope().find_var(candidate_w_name).get_tensor()) + fluid.global_scope().find_var(candidate_w_name).get_tensor() + ) candidate_w = np.random.uniform( - -0.1, 0.1, size=candidate_w.shape).astype('float32') + -0.1, 0.1, size=candidate_w.shape + ).astype('float32') fluid.global_scope().find_var(candidate_w_name).get_tensor().set( - candidate_w, place) + candidate_w, place + ) candidate_b = np.array( - fluid.global_scope().find_var(candidate_b_name).get_tensor()) + fluid.global_scope().find_var(candidate_b_name).get_tensor() + ) candidate_b = np.random.uniform( - -0.1, 0.1, size=candidate_b.shape).astype('float32') + -0.1, 0.1, size=candidate_b.shape + ).astype('float32') fluid.global_scope().find_var(candidate_b_name).get_tensor().set( - candidate_b, place) + candidate_b, place + ) gate_weight.append(gate_w) gate_bias.append(gate_b) @@ -258,42 +289,58 @@ def test_run(self): if self.is_bidirect: for i in range(self.num_layers): - gate_w_name = "basic_gru_reverse_layers_" + str( - i) + "/BasicGRUUnit_0.w_0" - gate_b_name = "basic_gru_reverse_layers_" + str( - i) + "/BasicGRUUnit_0.b_0" - candidate_w_name = "basic_gru_reverse_layers_" + str( - i) + "/BasicGRUUnit_0.w_1" - candidate_b_name = "basic_gru_reverse_layers_" + str( - i) + "/BasicGRUUnit_0.b_1" + gate_w_name = ( + "basic_gru_reverse_layers_" + str(i) + "/BasicGRUUnit_0.w_0" + ) + gate_b_name = ( + "basic_gru_reverse_layers_" + str(i) + "/BasicGRUUnit_0.b_0" + ) + candidate_w_name = ( + "basic_gru_reverse_layers_" + str(i) + "/BasicGRUUnit_0.w_1" + ) + candidate_b_name = ( + "basic_gru_reverse_layers_" + str(i) + "/BasicGRUUnit_0.b_1" + ) gate_w = np.array( - fluid.global_scope().find_var(gate_w_name).get_tensor()) - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float32') + fluid.global_scope().find_var(gate_w_name).get_tensor() + ) + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_w_name).get_tensor().set( - gate_w, place) + gate_w, place + ) gate_b = np.array( - fluid.global_scope().find_var(gate_b_name).get_tensor()) - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float32') + fluid.global_scope().find_var(gate_b_name).get_tensor() + ) + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_b_name).get_tensor().set( - gate_b, place) + gate_b, place + ) - candidate_w = np.array(fluid.global_scope().find_var( - candidate_w_name).get_tensor()) + candidate_w = np.array( + fluid.global_scope().find_var(candidate_w_name).get_tensor() + ) candidate_w = np.random.uniform( - -0.1, 0.1, size=candidate_w.shape).astype('float32') + -0.1, 0.1, size=candidate_w.shape + ).astype('float32') fluid.global_scope().find_var( - candidate_w_name).get_tensor().set(candidate_w, place) + candidate_w_name + ).get_tensor().set(candidate_w, place) - candidate_b = np.array(fluid.global_scope().find_var( - candidate_b_name).get_tensor()) + candidate_b = np.array( + fluid.global_scope().find_var(candidate_b_name).get_tensor() + ) candidate_b = np.random.uniform( - -0.1, 0.1, size=candidate_b.shape).astype('float32') + -0.1, 0.1, size=candidate_b.shape + ).astype('float32') fluid.global_scope().find_var( - candidate_b_name).get_tensor().set(candidate_b, place) + candidate_b_name + ).get_tensor().set(candidate_b, place) gate_weight.append(gate_w) gate_bias.append(gate_b) @@ -301,39 +348,39 @@ def test_run(self): candidate_bias.append(candidate_b) step_input_np = np.random.uniform( - -0.1, 0.1, - (self.seq_len, self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.seq_len, self.batch_size, self.hidden_size) + ).astype('float32') sequence_length_np = np.random.randint( - self.seq_len // 2, self.seq_len, - size=(self.batch_size)).astype('int64') + self.seq_len // 2, self.seq_len, size=(self.batch_size) + ).astype('int64') - out = exe.run(feed={ - 'x': step_input_np, - 'sequence_length': sequence_length_np - }, - fetch_list=[rnn_out, last_hidden]) + out = exe.run( + feed={'x': step_input_np, 'sequence_length': sequence_length_np}, + fetch_list=[rnn_out, last_hidden], + ) api_rnn_out = out[0] api_last_hidden = out[1] - np_out = gru_np(step_input_np, - None, - self.hidden_size, - gate_weight, - gate_bias, - candidate_weight, - candidate_bias, - num_layers=self.num_layers, - batch_first=self.batch_first, - is_bidirect=self.is_bidirect, - sequence_length=sequence_length_np) + np_out = gru_np( + step_input_np, + None, + self.hidden_size, + gate_weight, + gate_bias, + candidate_weight, + candidate_bias, + num_layers=self.num_layers, + batch_first=self.batch_first, + is_bidirect=self.is_bidirect, + sequence_length=sequence_length_np, + ) np.testing.assert_allclose(api_rnn_out, np_out[0], rtol=0.0001, atol=0) - np.testing.assert_allclose(api_last_hidden, - np_out[1], - rtol=0.0001, - atol=0) + np.testing.assert_allclose( + api_last_hidden, np_out[1], rtol=0.0001, atol=0 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_basic_gru_unit_op.py b/python/paddle/fluid/tests/unittests/test_basic_gru_unit_op.py index ae3e6dc4f77d2..660aa184f5788 100644 --- a/python/paddle/fluid/tests/unittests/test_basic_gru_unit_op.py +++ b/python/paddle/fluid/tests/unittests/test_basic_gru_unit_op.py @@ -34,13 +34,13 @@ def sigmoid(x): y = np.copy(x) y[x < SIGMOID_THRESHOLD_MIN] = SIGMOID_THRESHOLD_MIN y[x > SIGMOID_THRESHOLD_MAX] = SIGMOID_THRESHOLD_MAX - return 1. / (1. + np.exp(-y)) + return 1.0 / (1.0 + np.exp(-y)) def tanh(x): - y = -2. * x + y = -2.0 * x y[y > EXP_MAX_INPUT] = EXP_MAX_INPUT - return (2. / (1. + np.exp(y))) - 1. + return (2.0 / (1.0 + np.exp(y))) - 1.0 def step(step_in, pre_hidden, gate_w, gate_b, candidate_w, candidate_b): @@ -64,16 +64,15 @@ def step(step_in, pre_hidden, gate_w, gate_b, candidate_w, candidate_b): class TestBasicGRUUnit(unittest.TestCase): - def setUp(self): self.hidden_size = 5 self.batch_size = 5 def test_run(self): x = layers.data(name='x', shape=[-1, self.hidden_size], dtype='float32') - pre_hidden = layers.data(name="pre_hidden", - shape=[-1, self.hidden_size], - dtype='float32') + pre_hidden = layers.data( + name="pre_hidden", shape=[-1, self.hidden_size], dtype='float32' + ) gru_unit = BasicGRUUnit("gru_unit", self.hidden_size) new_hidden = gru_unit(x, pre_hidden) @@ -98,48 +97,67 @@ def test_run(self): candidate_b_name = "gru_unit/BasicGRUUnit_0.b_1" gate_w = np.array( - fluid.global_scope().find_var(gate_w_name).get_tensor()) - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float32') + fluid.global_scope().find_var(gate_w_name).get_tensor() + ) + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_w_name).get_tensor().set( - gate_w, place) + gate_w, place + ) gate_b = np.array( - fluid.global_scope().find_var(gate_b_name).get_tensor()) - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float32') + fluid.global_scope().find_var(gate_b_name).get_tensor() + ) + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_b_name).get_tensor().set( - gate_b, place) + gate_b, place + ) candidate_w = np.array( - fluid.global_scope().find_var(candidate_w_name).get_tensor()) + fluid.global_scope().find_var(candidate_w_name).get_tensor() + ) candidate_w = np.random.uniform( - -0.1, 0.1, size=candidate_w.shape).astype('float32') + -0.1, 0.1, size=candidate_w.shape + ).astype('float32') fluid.global_scope().find_var(candidate_w_name).get_tensor().set( - candidate_w, place) + candidate_w, place + ) candidate_b = np.array( - fluid.global_scope().find_var(candidate_b_name).get_tensor()) + fluid.global_scope().find_var(candidate_b_name).get_tensor() + ) candidate_b = np.random.uniform( - -0.1, 0.1, size=candidate_b.shape).astype('float32') + -0.1, 0.1, size=candidate_b.shape + ).astype('float32') fluid.global_scope().find_var(candidate_b_name).get_tensor().set( - candidate_b, place) + candidate_b, place + ) step_input_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') pre_hidden_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') - out = exe.run(feed={ - 'x': step_input_np, - 'pre_hidden': pre_hidden_np - }, - fetch_list=[new_hidden]) + out = exe.run( + feed={'x': step_input_np, 'pre_hidden': pre_hidden_np}, + fetch_list=[new_hidden], + ) api_out = out[0] - np_out = step(step_input_np, pre_hidden_np, gate_w, gate_b, candidate_w, - candidate_b) + np_out = step( + step_input_np, + pre_hidden_np, + gate_w, + gate_b, + candidate_w, + candidate_b, + ) np.testing.assert_allclose(api_out, np_out, rtol=0.0001, atol=0) diff --git a/python/paddle/fluid/tests/unittests/test_basic_lstm_api.py b/python/paddle/fluid/tests/unittests/test_basic_lstm_api.py index 9574a1ea973ff..befc7e2c5ffa5 100644 --- a/python/paddle/fluid/tests/unittests/test_basic_lstm_api.py +++ b/python/paddle/fluid/tests/unittests/test_basic_lstm_api.py @@ -34,27 +34,28 @@ def sigmoid(x): y = np.copy(x) y[x < SIGMOID_THRESHOLD_MIN] = SIGMOID_THRESHOLD_MIN y[x > SIGMOID_THRESHOLD_MAX] = SIGMOID_THRESHOLD_MAX - return 1. / (1. + np.exp(-y)) + return 1.0 / (1.0 + np.exp(-y)) def tanh(x): - y = -2. * x + y = -2.0 * x y[y > EXP_MAX_INPUT] = EXP_MAX_INPUT - return (2. / (1. + np.exp(y))) - 1. - - -def lstm_np(input, - init_h, - init_c, - hidden_size, - gate_weight, - gate_bias, - num_layers=1, - batch_first=False, - is_bidirect=False, - sequence_length=None, - forget_bias=1.0): - + return (2.0 / (1.0 + np.exp(y))) - 1.0 + + +def lstm_np( + input, + init_h, + init_c, + hidden_size, + gate_weight, + gate_bias, + num_layers=1, + batch_first=False, + is_bidirect=False, + sequence_length=None, + forget_bias=1.0, +): def step(step_in, pre_hidden, pre_cell, gate_w, gate_b): concat_1 = np.concatenate([step_in, pre_hidden], 1) @@ -112,23 +113,27 @@ def get_single_direction_output(rnn_input, mask=None, direc_index=0): if mask is not None: step_mask = mask[i] step_mask = np.reshape(step_mask, [-1, 1]) - #print("np mask", step_mask.shape ) + # print("np mask", step_mask.shape ) for i in range(num_layers): new_hidden, new_cell = step( - step_input, pre_hidden_array[i], pre_cell_array[i], + step_input, + pre_hidden_array[i], + pre_cell_array[i], gate_weight[direc_index * num_layers + i], - gate_bias[direc_index * num_layers + i]) + gate_bias[direc_index * num_layers + i], + ) if mask is not None: new_hidden = np.multiply( - new_hidden, step_mask) - np.multiply( - pre_hidden_array[i], (step_mask - 1.0)) - #new_hidden = new_hidden * step_mask - pre_hidden_array[i] * ( step_mask -1 ) - #new_cell = new_cell * step_mask - pre_cell_array[i] * (step_mask -1) + new_hidden, step_mask + ) - np.multiply(pre_hidden_array[i], (step_mask - 1.0)) + # new_hidden = new_hidden * step_mask - pre_hidden_array[i] * ( step_mask -1 ) + # new_cell = new_cell * step_mask - pre_cell_array[i] * (step_mask -1) new_cell = np.multiply(new_cell, step_mask) - np.multiply( - pre_cell_array[i], (step_mask - 1.0)) + pre_cell_array[i], (step_mask - 1.0) + ) pre_hidden_array[i] = new_hidden pre_cell_array[i] = new_cell @@ -139,8 +144,9 @@ def get_single_direction_output(rnn_input, mask=None, direc_index=0): rnn_out = np.reshape(rnn_out, [seq_len, -1, hidden_size]) last_hidden_out = np.concatenate(pre_hidden_array, 0) - last_hidden_out = np.reshape(last_hidden_out, - [num_layers, -1, hidden_size]) + last_hidden_out = np.reshape( + last_hidden_out, [num_layers, -1, hidden_size] + ) last_cell_out = np.concatenate(pre_cell_array, 0) last_cell_out = np.reshape(last_cell_out, [num_layers, -1, hidden_size]) @@ -148,7 +154,8 @@ def get_single_direction_output(rnn_input, mask=None, direc_index=0): return rnn_out, last_hidden_out, last_cell_out fw_rnn_out, fw_last_hidden, fw_last_cell = get_single_direction_output( - input, mask, direc_index=0) + input, mask, direc_index=0 + ) if is_bidirect: bw_input = input[::-1] @@ -157,18 +164,21 @@ def get_single_direction_output(rnn_input, mask=None, direc_index=0): bw_mask = mask[::-1] bw_rnn_out, bw_last_hidden, bw_last_cell = get_single_direction_output( - bw_input, bw_mask, direc_index=1) + bw_input, bw_mask, direc_index=1 + ) bw_rnn_out = bw_rnn_out[::-1] rnn_out = np.concatenate([fw_rnn_out, bw_rnn_out], 2) last_hidden = np.concatenate([fw_last_hidden, bw_last_hidden], 1) - last_hidden = np.reshape(last_hidden, - [num_layers * direc_num, -1, hidden_size]) + last_hidden = np.reshape( + last_hidden, [num_layers * direc_num, -1, hidden_size] + ) last_cell = np.concatenate([fw_last_cell, bw_last_cell], 1) - last_cell = np.reshape(last_cell, - [num_layers * direc_num, -1, hidden_size]) + last_cell = np.reshape( + last_cell, [num_layers * direc_num, -1, hidden_size] + ) if batch_first: rnn_out = np.transpose(rnn_out, [1, 0, 2]) @@ -186,7 +196,6 @@ def get_single_direction_output(rnn_input, mask=None, direc_index=0): class TestBasicLSTMApi(unittest.TestCase): - def setUp(self): self.hidden_size = 10 self.batch_size = 5 @@ -197,15 +206,26 @@ def setUp(self): self.forget_bias = 1.0 def test_run(self): - x = layers.data(name='x', - shape=[-1, self.batch_size, self.hidden_size], - dtype='float32') - sequence_length = layers.data(name="sequence_length", - shape=[-1], - dtype='float32') - - rnn_out, last_hidden, last_cell = basic_lstm( x, None, None, self.hidden_size, num_layers=self.num_layers, \ - batch_first = self.batch_first, bidirectional=self.is_bidirect, sequence_length=sequence_length, forget_bias = self.forget_bias ) + x = layers.data( + name='x', + shape=[-1, self.batch_size, self.hidden_size], + dtype='float32', + ) + sequence_length = layers.data( + name="sequence_length", shape=[-1], dtype='float32' + ) + + rnn_out, last_hidden, last_cell = basic_lstm( + x, + None, + None, + self.hidden_size, + num_layers=self.num_layers, + batch_first=self.batch_first, + bidirectional=self.is_bidirect, + sequence_length=sequence_length, + forget_bias=self.forget_bias, + ) last_hidden.persisbale = True rnn_out.persisbale = True @@ -228,83 +248,100 @@ def test_run(self): gate_b_name = "basic_lstm_layers_" + str(i) + "/BasicLSTMUnit_0.b_0" gate_w = np.array( - fluid.global_scope().find_var(gate_w_name).get_tensor()) - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float32') + fluid.global_scope().find_var(gate_w_name).get_tensor() + ) + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_w_name).get_tensor().set( - gate_w, place) + gate_w, place + ) gate_b = np.array( - fluid.global_scope().find_var(gate_b_name).get_tensor()) - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float32') + fluid.global_scope().find_var(gate_b_name).get_tensor() + ) + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_b_name).get_tensor().set( - gate_b, place) + gate_b, place + ) gate_weight.append(gate_w) gate_bias.append(gate_b) if self.is_bidirect: for i in range(self.num_layers): - gate_w_name = "basic_lstm_reverse_layers_" + str( - i) + "/BasicLSTMUnit_0.w_0" - gate_b_name = "basic_lstm_reverse_layers_" + str( - i) + "/BasicLSTMUnit_0.b_0" + gate_w_name = ( + "basic_lstm_reverse_layers_" + + str(i) + + "/BasicLSTMUnit_0.w_0" + ) + gate_b_name = ( + "basic_lstm_reverse_layers_" + + str(i) + + "/BasicLSTMUnit_0.b_0" + ) gate_w = np.array( - fluid.global_scope().find_var(gate_w_name).get_tensor()) - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float32') + fluid.global_scope().find_var(gate_w_name).get_tensor() + ) + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_w_name).get_tensor().set( - gate_w, place) + gate_w, place + ) gate_b = np.array( - fluid.global_scope().find_var(gate_b_name).get_tensor()) - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float32') + fluid.global_scope().find_var(gate_b_name).get_tensor() + ) + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_b_name).get_tensor().set( - gate_b, place) + gate_b, place + ) gate_weight.append(gate_w) gate_bias.append(gate_b) step_input_np = np.random.uniform( - -0.1, 0.1, - (self.seq_len, self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.seq_len, self.batch_size, self.hidden_size) + ).astype('float32') sequence_length_np = np.random.randint( - self.seq_len // 2, self.seq_len, - size=(self.batch_size)).astype('int64') + self.seq_len // 2, self.seq_len, size=(self.batch_size) + ).astype('int64') - out = exe.run(feed={ - 'x': step_input_np, - 'sequence_length': sequence_length_np - }, - fetch_list=[rnn_out, last_hidden, last_cell]) + out = exe.run( + feed={'x': step_input_np, 'sequence_length': sequence_length_np}, + fetch_list=[rnn_out, last_hidden, last_cell], + ) api_rnn_out = out[0] api_last_hidden = out[1] api_last_cell = out[2] - np_out = lstm_np(step_input_np, - None, - None, - self.hidden_size, - gate_weight, - gate_bias, - num_layers=self.num_layers, - batch_first=self.batch_first, - is_bidirect=self.is_bidirect, - sequence_length=sequence_length_np) + np_out = lstm_np( + step_input_np, + None, + None, + self.hidden_size, + gate_weight, + gate_bias, + num_layers=self.num_layers, + batch_first=self.batch_first, + is_bidirect=self.is_bidirect, + sequence_length=sequence_length_np, + ) np.testing.assert_allclose(api_rnn_out, np_out[0], rtol=0.0001, atol=0) - np.testing.assert_allclose(api_last_hidden, - np_out[1], - rtol=0.0001, - atol=0) - np.testing.assert_allclose(api_last_cell, - np_out[2], - rtol=0.0001, - atol=0) + np.testing.assert_allclose( + api_last_hidden, np_out[1], rtol=0.0001, atol=0 + ) + np.testing.assert_allclose( + api_last_cell, np_out[2], rtol=0.0001, atol=0 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_basic_lstm_unit_op.py b/python/paddle/fluid/tests/unittests/test_basic_lstm_unit_op.py index 50ef22467dfe1..cf32b95d76f36 100644 --- a/python/paddle/fluid/tests/unittests/test_basic_lstm_unit_op.py +++ b/python/paddle/fluid/tests/unittests/test_basic_lstm_unit_op.py @@ -34,13 +34,13 @@ def sigmoid(x): y = np.copy(x) y[x < SIGMOID_THRESHOLD_MIN] = SIGMOID_THRESHOLD_MIN y[x > SIGMOID_THRESHOLD_MAX] = SIGMOID_THRESHOLD_MAX - return 1. / (1. + np.exp(-y)) + return 1.0 / (1.0 + np.exp(-y)) def tanh(x): - y = -2. * x + y = -2.0 * x y[y > EXP_MAX_INPUT] = EXP_MAX_INPUT - return (2. / (1. + np.exp(y))) - 1. + return (2.0 / (1.0 + np.exp(y))) - 1.0 def step(step_in, pre_hidden, pre_cell, gate_w, gate_b, forget_bias=1.0): @@ -57,19 +57,18 @@ def step(step_in, pre_hidden, pre_cell, gate_w, gate_b, forget_bias=1.0): class TestBasicGRUUnit(unittest.TestCase): - def setUp(self): self.hidden_size = 5 self.batch_size = 5 def test_run(self): x = layers.data(name='x', shape=[-1, self.hidden_size], dtype='float32') - pre_hidden = layers.data(name="pre_hidden", - shape=[-1, self.hidden_size], - dtype='float32') - pre_cell = layers.data(name="pre_cell", - shape=[-1, self.hidden_size], - dtype='float32') + pre_hidden = layers.data( + name="pre_hidden", shape=[-1, self.hidden_size], dtype='float32' + ) + pre_cell = layers.data( + name="pre_cell", shape=[-1, self.hidden_size], dtype='float32' + ) lstm_unit = BasicLSTMUnit("lstm_unit", self.hidden_size) @@ -94,44 +93,57 @@ def test_run(self): gate_b_name = "lstm_unit/BasicLSTMUnit_0.b_0" gate_w = np.array( - fluid.global_scope().find_var(gate_w_name).get_tensor()) - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float32') + fluid.global_scope().find_var(gate_w_name).get_tensor() + ) + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_w_name).get_tensor().set( - gate_w, place) + gate_w, place + ) gate_b = np.array( - fluid.global_scope().find_var(gate_b_name).get_tensor()) - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float32') + fluid.global_scope().find_var(gate_b_name).get_tensor() + ) + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float32' + ) fluid.global_scope().find_var(gate_b_name).get_tensor().set( - gate_b, place) + gate_b, place + ) step_input_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') pre_hidden_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') pre_cell_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') - - out = exe.run( feed={ 'x' : step_input_np, 'pre_hidden' : pre_hidden_np, \ - 'pre_cell' : pre_cell_np }, - fetch_list=[ new_hidden, new_cell]) + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') + + out = exe.run( + feed={ + 'x': step_input_np, + 'pre_hidden': pre_hidden_np, + 'pre_cell': pre_cell_np, + }, + fetch_list=[new_hidden, new_cell], + ) api_hidden_out = out[0] api_cell_out = out[1] - np_hidden_out, np_cell_out = step(step_input_np, pre_hidden_np, - pre_cell_np, gate_w, gate_b) - - np.testing.assert_allclose(api_hidden_out, - np_hidden_out, - rtol=0.0001, - atol=0) - np.testing.assert_allclose(api_cell_out, - np_cell_out, - rtol=0.0001, - atol=0) + np_hidden_out, np_cell_out = step( + step_input_np, pre_hidden_np, pre_cell_np, gate_w, gate_b + ) + + np.testing.assert_allclose( + api_hidden_out, np_hidden_out, rtol=0.0001, atol=0 + ) + np.testing.assert_allclose( + api_cell_out, np_cell_out, rtol=0.0001, atol=0 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_basic_rnn_name.py b/python/paddle/fluid/tests/unittests/test_basic_rnn_name.py index ca56cd62a46b5..17bec037e842b 100644 --- a/python/paddle/fluid/tests/unittests/test_basic_rnn_name.py +++ b/python/paddle/fluid/tests/unittests/test_basic_rnn_name.py @@ -20,14 +20,19 @@ class TestBasicGRUApiName(unittest.TestCase): - def setUp(self): - self.name_set = set([ - "test1_fw_w_0_gate", "test1_fw_w_0_candidate", "test1_fw_b_0_gate", - "test1_fw_b_0_candidate", "test1_bw_w_0_gate", - "test1_bw_w_0_candidate", "test1_bw_b_0_gate", - "test1_bw_b_0_candidate" - ]) + self.name_set = set( + [ + "test1_fw_w_0_gate", + "test1_fw_w_0_candidate", + "test1_fw_b_0_gate", + "test1_fw_b_0_candidate", + "test1_bw_w_0_gate", + "test1_bw_w_0_candidate", + "test1_bw_b_0_gate", + "test1_bw_b_0_candidate", + ] + ) def test_name(self): batch_size = 20 @@ -39,35 +44,54 @@ def test_name(self): batch_first = False with new_program_scope(): - input = layers.data(name="input", - shape=[-1, batch_size, input_size], - dtype='float32') - pre_hidden = layers.data(name="pre_hidden", - shape=[-1, hidden_size], - dtype='float32') - sequence_length = layers.data(name="sequence_length", - shape=[-1], - dtype='int32') - - - rnn_out, last_hidden = basic_gru( input, pre_hidden, hidden_size, num_layers = num_layers, \ - sequence_length = sequence_length, dropout_prob=dropout, bidirectional = bidirectional, \ - batch_first = batch_first, param_attr=fluid.ParamAttr( name ="test1"), bias_attr=fluid.ParamAttr( name="test1"), name="basic_gru") + input = layers.data( + name="input", + shape=[-1, batch_size, input_size], + dtype='float32', + ) + pre_hidden = layers.data( + name="pre_hidden", shape=[-1, hidden_size], dtype='float32' + ) + sequence_length = layers.data( + name="sequence_length", shape=[-1], dtype='int32' + ) + + rnn_out, last_hidden = basic_gru( + input, + pre_hidden, + hidden_size, + num_layers=num_layers, + sequence_length=sequence_length, + dropout_prob=dropout, + bidirectional=bidirectional, + batch_first=batch_first, + param_attr=fluid.ParamAttr(name="test1"), + bias_attr=fluid.ParamAttr(name="test1"), + name="basic_gru", + ) var_list = fluid.io.get_program_parameter( - fluid.default_main_program()) + fluid.default_main_program() + ) for var in var_list: self.assertTrue(var.name in self.name_set) class TestBasicLSTMApiName(unittest.TestCase): - def setUp(self): - self.name_set = set([ - "test1_fw_w_0", "test1_fw_b_0", "test1_fw_w_1", "test1_fw_b_1", - "test1_bw_w_0", "test1_bw_b_0", "test1_bw_w_1", "test1_bw_b_1" - ]) + self.name_set = set( + [ + "test1_fw_w_0", + "test1_fw_b_0", + "test1_fw_w_1", + "test1_fw_b_1", + "test1_bw_w_0", + "test1_bw_b_0", + "test1_bw_w_1", + "test1_bw_b_1", + ] + ) def test_name(self): batch_size = 20 @@ -79,27 +103,38 @@ def test_name(self): batch_first = False with new_program_scope(): - input = layers.data(name="input", - shape=[-1, batch_size, input_size], - dtype='float32') - pre_hidden = layers.data(name="pre_hidden", - shape=[-1, hidden_size], - dtype='float32') - pre_cell = layers.data(name="pre_cell", - shape=[-1, hidden_size], - dtype='float32') - sequence_length = layers.data(name="sequence_length", - shape=[-1], - dtype='int32') - - rnn_out, last_hidden, last_cell = basic_lstm( input, pre_hidden, pre_cell, \ - hidden_size, num_layers = num_layers, \ - sequence_length = sequence_length, dropout_prob=dropout, bidirectional = bidirectional, \ - param_attr=fluid.ParamAttr( name ="test1"), bias_attr=fluid.ParamAttr( name = "test1"), \ - batch_first = batch_first) + input = layers.data( + name="input", + shape=[-1, batch_size, input_size], + dtype='float32', + ) + pre_hidden = layers.data( + name="pre_hidden", shape=[-1, hidden_size], dtype='float32' + ) + pre_cell = layers.data( + name="pre_cell", shape=[-1, hidden_size], dtype='float32' + ) + sequence_length = layers.data( + name="sequence_length", shape=[-1], dtype='int32' + ) + + rnn_out, last_hidden, last_cell = basic_lstm( + input, + pre_hidden, + pre_cell, + hidden_size, + num_layers=num_layers, + sequence_length=sequence_length, + dropout_prob=dropout, + bidirectional=bidirectional, + param_attr=fluid.ParamAttr(name="test1"), + bias_attr=fluid.ParamAttr(name="test1"), + batch_first=batch_first, + ) var_list = fluid.io.get_program_parameter( - fluid.default_main_program()) + fluid.default_main_program() + ) for var in var_list: self.assertTrue(var.name in self.name_set) diff --git a/python/paddle/fluid/tests/unittests/test_batch_fc_op.py b/python/paddle/fluid/tests/unittests/test_batch_fc_op.py index 6214859931f80..2414514b2bd25 100644 --- a/python/paddle/fluid/tests/unittests/test_batch_fc_op.py +++ b/python/paddle/fluid/tests/unittests/test_batch_fc_op.py @@ -32,7 +32,6 @@ def np_cal_batchfc(input, w, bias): class TestBatchFCOp(OpTest): - def config(self): self.slot_pairs_num = 10 self.batch_size = 5 @@ -42,12 +41,15 @@ def config(self): def setUp(self): self.config() - self.input = np.random.random((self.slot_pairs_num, self.batch_size, - self.in_dim)).astype(self.dtype) + self.input = np.random.random( + (self.slot_pairs_num, self.batch_size, self.in_dim) + ).astype(self.dtype) self.w = np.random.random( - (self.slot_pairs_num, self.in_dim, self.out_dim)).astype(self.dtype) + (self.slot_pairs_num, self.in_dim, self.out_dim) + ).astype(self.dtype) self.bias = np.random.random( - (self.slot_pairs_num, self.out_dim)).astype(self.dtype) + (self.slot_pairs_num, self.out_dim) + ).astype(self.dtype) self.op_type = "batch_fc" np_out = np_cal_batchfc(self.input, self.w, self.bias) np_out = np_out.astype(self.dtype) @@ -60,12 +62,12 @@ def test_check_output_gpu(self): def test_check_grad_gpu(self): if core.is_compiled_with_cuda(): - self.check_grad_with_place(core.CUDAPlace(0), - ["Bias", "W", "Input"], "Out") + self.check_grad_with_place( + core.CUDAPlace(0), ["Bias", "W", "Input"], "Out" + ) class TestBatchFCOp1(OpTest): - def config(self): self.slot_pairs_num = 10 self.batch_size = 5 @@ -75,12 +77,15 @@ def config(self): def setUp(self): self.config() - self.input = np.random.random((self.slot_pairs_num, self.batch_size, - self.in_dim)).astype(self.dtype) + self.input = np.random.random( + (self.slot_pairs_num, self.batch_size, self.in_dim) + ).astype(self.dtype) self.w = np.random.random( - (self.slot_pairs_num, self.in_dim, self.out_dim)).astype(self.dtype) + (self.slot_pairs_num, self.in_dim, self.out_dim) + ).astype(self.dtype) self.bias = np.random.random( - (self.slot_pairs_num, self.out_dim)).astype(self.dtype) + (self.slot_pairs_num, self.out_dim) + ).astype(self.dtype) self.op_type = "batch_fc" np_out = np_cal_batchfc(self.input, self.w, self.bias) np_out = np_out.astype(self.dtype) @@ -95,8 +100,9 @@ def test_check_output_cpu(self): def test_check_grad_cpu(self): try: - self.check_grad_with_place(core.CPUPlace(), ["Bias", "W", "Input"], - "Out") + self.check_grad_with_place( + core.CPUPlace(), ["Bias", "W", "Input"], "Out" + ) except: print("do not support cpu test, skip") diff --git a/python/paddle/fluid/tests/unittests/test_batch_norm_op.py b/python/paddle/fluid/tests/unittests/test_batch_norm_op.py index 33265b33bd1b7..381640621cb1c 100644 --- a/python/paddle/fluid/tests/unittests/test_batch_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_batch_norm_op.py @@ -154,12 +154,20 @@ def _reference_grad(x, y_grad, scale, mean, var, epsilon, data_format): x = np.transpose(x, (0, 2, 3, 1)) y_grad = np.transpose(y_grad, (0, 2, 3, 1)) - x_grad = scale * (y_grad - np.mean(y_grad, axis=(0, 1, 2)) - - (x - mean) * np.mean(y_grad * - (x - mean), axis=(0, 1, 2)) / - (var + epsilon)) / np.sqrt(var + epsilon) - grad_scale = np.sum(y_grad * (x - mean) / np.sqrt(var + epsilon), - axis=(0, 1, 2)) + x_grad = ( + scale + * ( + y_grad + - np.mean(y_grad, axis=(0, 1, 2)) + - (x - mean) + * np.mean(y_grad * (x - mean), axis=(0, 1, 2)) + / (var + epsilon) + ) + / np.sqrt(var + epsilon) + ) + grad_scale = np.sum( + y_grad * (x - mean) / np.sqrt(var + epsilon), axis=(0, 1, 2) + ) grad_offset = np.sum(y_grad, axis=(0, 1, 2)) # transfer back to N, C, H, W @@ -184,7 +192,6 @@ def create_or_get_tensor(scope, var_name, var, place): def set_output_grad(scope, outputs, place, feed_dict=None): - def __set_tensor__(name, data=None): out_tensor = scope.find_var(name).get_tensor() grad_tensor = scope.var(grad_var_name(name)).get_tensor() @@ -206,7 +213,6 @@ def __set_tensor__(name, data=None): class TestBatchNormOpInference(unittest.TestCase): - def setUp(self): self.dtype = np.float32 self.use_mkldnn = False @@ -214,11 +220,9 @@ def setUp(self): self.init_kernel_type() def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=1e-05, - atol=atol, - err_msg=msg) + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=1e-05, atol=atol, err_msg=msg + ) def check_with_place(self, place, data_layout, dtype, shape): epsilon = 0.00001 @@ -244,34 +248,39 @@ def check_with_place(self, place, data_layout, dtype, shape): mean = np.zeros(scale_shape).astype(np.float32) variance = np.ones(scale_shape).astype(np.float32) - y_out = _reference_testing(x_val, scale_val, bias_val, mean, variance, - epsilon, data_layout).astype(dtype) + y_out = _reference_testing( + x_val, scale_val, bias_val, mean, variance, epsilon, data_layout + ).astype(dtype) if self.fuse_with_relu: y_out = np.maximum(y_out, 0) scope = core.Scope() # create input - x_tensor = create_or_get_tensor(scope, "x_val", - OpTest.np_dtype_to_fluid_dtype(x_val), - place) + x_tensor = create_or_get_tensor( + scope, "x_val", OpTest.np_dtype_to_fluid_dtype(x_val), place + ) scale_tensor = create_or_get_tensor( - scope, "scale_val", OpTest.np_dtype_to_fluid_dtype(scale_val), - place) + scope, "scale_val", OpTest.np_dtype_to_fluid_dtype(scale_val), place + ) bias_tensor = create_or_get_tensor( - scope, "bias_val", OpTest.np_dtype_to_fluid_dtype(bias_val), place) - mean_tensor = create_or_get_tensor(scope, "mean", - OpTest.np_dtype_to_fluid_dtype(mean), - place) + scope, "bias_val", OpTest.np_dtype_to_fluid_dtype(bias_val), place + ) + mean_tensor = create_or_get_tensor( + scope, "mean", OpTest.np_dtype_to_fluid_dtype(mean), place + ) variance_tensor = create_or_get_tensor( - scope, "variance", OpTest.np_dtype_to_fluid_dtype(variance), place) + scope, "variance", OpTest.np_dtype_to_fluid_dtype(variance), place + ) # create output y_tensor = create_or_get_tensor(scope, "y_out", None, place) - saved_mean_tensor = create_or_get_tensor(scope, "saved_mean", None, - place) - saved_variance_tensor = create_or_get_tensor(scope, "saved_variance", - None, place) + saved_mean_tensor = create_or_get_tensor( + scope, "saved_mean", None, place + ) + saved_variance_tensor = create_or_get_tensor( + scope, "saved_variance", None, place + ) mean_out_tensor = mean_tensor variance_out_tensor = variance_tensor @@ -294,7 +303,8 @@ def check_with_place(self, place, data_layout, dtype, shape): data_layout=data_layout, use_mkldnn=self.use_mkldnn, fuse_with_relu=self.fuse_with_relu, - epsilon=epsilon) + epsilon=epsilon, + ) batch_norm_op.run(scope, place) @@ -314,12 +324,19 @@ def check_with_place(self, place, data_layout, dtype, shape): y_tensor._set_dims(dims) # check inference result - self.__assert_close(y_tensor, - y_out, - "inference output are different at " + str(place) + - ", " + data_layout + ", " + str(np.dtype(dtype)) + - str(np.array(y_tensor)) + str(y_out), - atol=1e-3) + self.__assert_close( + y_tensor, + y_out, + "inference output are different at " + + str(place) + + ", " + + data_layout + + ", " + + str(np.dtype(dtype)) + + str(np.array(y_tensor)) + + str(y_out), + atol=1e-3, + ) def test_check_output(self): places = [core.CPUPlace()] @@ -328,8 +345,9 @@ def test_check_output(self): for place in places: for data_format in ["NCHW", "NHWC"]: - self.check_with_place(place, data_format, self.dtype, - [2, 3, 4, 5]) + self.check_with_place( + place, data_format, self.dtype, [2, 3, 4, 5] + ) self.check_with_place(place, data_format, self.dtype, [2, 3]) def init_kernel_type(self): @@ -337,7 +355,6 @@ def init_kernel_type(self): class TestFP16BatchNormOpInference(TestBatchNormOpInference): - def setUp(self): self.dtype = np.float16 self.use_mkldnn = False @@ -351,15 +368,15 @@ def test_check_output(self): if core.is_float16_supported(place): places.append(place) for place in places: - #for data_format in ["NCHW", "NHWC"]: + # for data_format in ["NCHW", "NHWC"]: for data_format in ["NCHW"]: - self.check_with_place(place, data_format, self.dtype, - [2, 3, 4, 5]) + self.check_with_place( + place, data_format, self.dtype, [2, 3, 4, 5] + ) self.check_with_place(place, data_format, self.dtype, [2, 3]) class TestBatchNormOpTraining(unittest.TestCase): - def setUp(self): self.use_mkldnn = False self.fuse_with_relu = False @@ -374,27 +391,54 @@ def init_test_case(self): self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] def __assert_close(self, tensor, np_array, msg, atol=1e-4): np.allclose(np.array(tensor), np_array, atol=atol) - def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, - epsilon, momentum, shape, data_layout): + def ref_forward_backward( + self, + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ): # run forward - y, saved_mean, var_ref = _reference_training(x, scale, bias, epsilon, - data_layout) - mean_out = saved_mean * (1. - momentum) + momentum * mean - variance_out = var_ref * (1. - momentum) + momentum * variance - saved_variance = 1. / np.sqrt(var_ref + epsilon) + y, saved_mean, var_ref = _reference_training( + x, scale, bias, epsilon, data_layout + ) + mean_out = saved_mean * (1.0 - momentum) + momentum * mean + variance_out = var_ref * (1.0 - momentum) + momentum * variance + saved_variance = 1.0 / np.sqrt(var_ref + epsilon) # run backward - x_grad, scale_grad, bias_grad = _reference_grad(x, y_grad, scale, - saved_mean, var_ref, - epsilon, data_layout) - - return y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad + x_grad, scale_grad, bias_grad = _reference_grad( + x, y_grad, scale, saved_mean, var_ref, epsilon, data_layout + ) + + return ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) def set_mean_variance(self, scale_shape, x, data_layout): mean, variance = _cal_mean_variance(x, self.epsilon, data_layout) @@ -403,12 +447,11 @@ def set_mean_variance(self, scale_shape, x, data_layout): # computing global mean/variance for one step if self.use_global_stats: mom = self.momentum - mean = mean * (1. - mom) + mom * mean_pre - variance = variance * (1. - mom) + mom * variance_pre + mean = mean * (1.0 - mom) + mom * mean_pre + variance = variance * (1.0 - mom) + mom * variance_pre return mean, variance def test_forward_backward(self): - def test_with_place(place, data_layout, shape): # attr epsilon = self.epsilon @@ -427,9 +470,27 @@ def test_with_place(place, data_layout, shape): y_grad = np.random.random_sample(shape).astype(np.float32) momentum_var = np.array([momentum]).astype(np.float32) - y, mean_out, variance_out, saved_mean, saved_variance, x_grad, scale_grad, bias_grad = self.ref_forward_backward( - x, y_grad, scale, bias, mean, variance, epsilon, momentum, - shape, data_layout) + ( + y, + mean_out, + variance_out, + saved_mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) = self.ref_forward_backward( + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ) var_dict = locals() var_dict['y@GRAD'] = y_grad @@ -438,8 +499,15 @@ def test_with_place(place, data_layout, shape): var_dict['bias@GRAD'] = bias_grad var_names = [ - 'x', 'scale', 'bias', 'mean', 'variance', 'y', 'saved_mean', - 'saved_variance', 'momentum_var' + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + 'y', + 'saved_mean', + 'saved_variance', + 'momentum_var', ] ground_truth = {name: var_dict[name] for name in var_names} @@ -447,15 +515,17 @@ def test_with_place(place, data_layout, shape): with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) inputs = { "X": block.var('x'), "Scale": block.var('scale'), "Bias": block.var('bias'), "Mean": block.var('mean'), - "Variance": block.var('variance') + "Variance": block.var('variance'), } attrs = { "epsilon": epsilon, @@ -463,7 +533,7 @@ def test_with_place(place, data_layout, shape): "data_layout": data_layout, "use_mkldnn": self.use_mkldnn, "fuse_with_relu": self.fuse_with_relu, - "use_global_stats": self.use_global_stats + "use_global_stats": self.use_global_stats, } if self.use_momentum_variable: inputs['MomentumTensor'] = block.var('momentum_var') @@ -475,19 +545,22 @@ def test_with_place(place, data_layout, shape): "MeanOut": block.var('mean'), # share memory "VarianceOut": block.var('variance'), # share memory "SavedMean": block.var('saved_mean'), - "SavedVariance": block.var('saved_variance') + "SavedVariance": block.var('saved_variance'), } block.create_var(name="reserve_space", dtype='float32') outputs["ReserveSpace"] = block.var('reserve_space') - bn_op = block.append_op(type="batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + bn_op = block.append_op( + type="batch_norm", + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) block.create_var(name='y@GRAD', dtype='float32', shape=y.shape) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - bn_op.desc, self.no_grad_set, []) + bn_op.desc, self.no_grad_set, [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -502,22 +575,28 @@ def test_with_place(place, data_layout, shape): program._sync_with_cpp() exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in [ - 'x', 'scale', 'bias', 'mean', 'variance', - 'y@GRAD', 'momentum_var' - ] - }, - fetch_list=self.fetch_list) + out = exe.run( + program, + feed={ + name: var_dict[name] + for name in [ + 'x', + 'scale', + 'bias', + 'mean', + 'variance', + 'y@GRAD', + 'momentum_var', + ] + }, + fetch_list=self.fetch_list, + ) for id, name in enumerate(self.fetch_list): if name == 'variance': - self.__assert_close(var_dict[name], - out[id], - name, - atol=1e-3) + self.__assert_close( + var_dict[name], out[id], name, atol=1e-3 + ) continue self.__assert_close(var_dict[name], out[id], name) print("op test forward passed: ", str(place), data_layout) @@ -536,7 +615,6 @@ def init_kernel_type(self): class TestBatchNormOpTrainingCase1(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = False self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -544,19 +622,23 @@ def init_test_case(self): class TestBatchNormOpTrainingCase2(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] os.environ['FLAGS_cudnn_batchnorm_spatial_persistent'] = "1" class TestBatchNormOpTrainingCase3(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = False self.no_grad_set = set(['x@GRAD']) @@ -564,24 +646,33 @@ def init_test_case(self): class TestBatchNormOpTrainingMomentumVariable(TestBatchNormOpTraining): - def init_test_case(self): self.use_momentum_variable = True self.use_global_stats = False self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'saved_mean', 'saved_variance', 'x@GRAD', - 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] class TestBatchNormOpFreezeStatsTraining(TestBatchNormOpTraining): - def init_test_case(self): self.use_global_stats = True self.no_grad_set = set() self.fetch_list = [ - 'y', 'mean', 'variance', 'x@GRAD', 'scale@GRAD', 'bias@GRAD' + 'y', + 'mean', + 'variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] def reference_grad(self, x, y_grad, scale, mean, var, epsilon, data_format): @@ -590,8 +681,9 @@ def reference_grad(self, x, y_grad, scale, mean, var, epsilon, data_format): y_grad = np.transpose(y_grad, (0, 2, 3, 1)) x_grad = scale * y_grad / np.sqrt(var + epsilon) - grad_scale = np.sum(y_grad * (x - mean) / np.sqrt(var + epsilon), - axis=(0, 1, 2)) + grad_scale = np.sum( + y_grad * (x - mean) / np.sqrt(var + epsilon), axis=(0, 1, 2) + ) grad_offset = np.sum(y_grad, axis=(0, 1, 2)) # transfer back to N, C, H, W @@ -602,8 +694,19 @@ def reference_grad(self, x, y_grad, scale, mean, var, epsilon, data_format): return x_grad, grad_scale, grad_offset - def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, - epsilon, momentum, shape, data_layout): + def ref_forward_backward( + self, + x, + y_grad, + scale, + bias, + mean, + variance, + epsilon, + momentum, + shape, + data_layout, + ): if data_layout != "NCHW" and data_layout != "NHWC": raise ValueError("Unknown data order.") @@ -621,17 +724,27 @@ def ref_forward_backward(self, x, y_grad, scale, bias, mean, variance, mean_out = mean variance_out = variance - saved_variance = 1. / np.sqrt(variance + epsilon) + saved_variance = 1.0 / np.sqrt(variance + epsilon) # run backward x_grad, scale_grad, bias_grad = self.reference_grad( - x, y_grad, scale, mean, variance, epsilon, data_layout) + x, y_grad, scale, mean, variance, epsilon, data_layout + ) - return y, mean_out, variance_out, mean, saved_variance, x_grad, scale_grad, bias_grad + return ( + y, + mean_out, + variance_out, + mean, + saved_variance, + x_grad, + scale_grad, + bias_grad, + ) class TestBatchNormOpFreezeStatsAndScaleBiasTraining( - TestBatchNormOpFreezeStatsTraining): - + TestBatchNormOpFreezeStatsTraining +): def init_test_case(self): self.use_global_stats = True self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -639,12 +752,12 @@ def init_test_case(self): class TestBatchNormOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of batch_norm must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.batch_norm, x1) # the input dtype of batch_norm must be float16 or float32 or float64 @@ -654,13 +767,13 @@ def test_errors(self): class TestDygraphBatchNormAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_norm = fluid.dygraph.BatchNorm(10) # the input of BatchNorm must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, batch_norm, x1) # the input dtype of BatchNorm must be float16 or float32 or float64 @@ -670,7 +783,6 @@ def test_errors(self): class TestDygraphBatchNormTrainableStats(unittest.TestCase): - def test_dygraph(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -683,7 +795,8 @@ def compute(x, is_test, trainable_statistics): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) y = bn(fluid.dygraph.to_variable(x)) return y.numpy() @@ -705,7 +818,8 @@ def compute(x_np, is_test, trainable_statistics): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) x = fluid.data(name='x', shape=x_np.shape, dtype=x_np.dtype) y = bn(x) exe.run(fluid.default_startup_program()) @@ -719,7 +833,6 @@ def compute(x_np, is_test, trainable_statistics): class TestDygraphBatchNormOpenReserveSpace(unittest.TestCase): - def test_reservespace(self): with program_guard(Program(), Program()): paddle.enable_static() @@ -734,5 +847,6 @@ def test_reservespace(self): if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_batch_norm_op_v2.py b/python/paddle/fluid/tests/unittests/test_batch_norm_op_v2.py index ed3b0560b5294..778489eb668dc 100644 --- a/python/paddle/fluid/tests/unittests/test_batch_norm_op_v2.py +++ b/python/paddle/fluid/tests/unittests/test_batch_norm_op_v2.py @@ -23,7 +23,6 @@ class TestBatchNorm(unittest.TestCase): - def test_name(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -37,7 +36,7 @@ def test_error(self): if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) for p in places: - #paddle.disable_static() + # paddle.disable_static() x_data_4 = np.random.random(size=(2, 1, 3, 3)).astype('float32') x_data_3 = np.random.random(size=(2, 1, 3)).astype('float32') @@ -80,7 +79,6 @@ def error3d(): self.assertRaises(ValueError, error3d_dataformat) def test_large_batch(self): - def compute_baseline(x): with fluid.dygraph.guard(p): bn = fluid.dygraph.BatchNorm(shape[1]) @@ -130,7 +128,7 @@ def test_eager_api(self): def compute_v1(x): with fluid.dygraph.guard(p): bn = fluid.dygraph.BatchNorm(shape[1]) - #bn = paddle.nn.BatchNorm2D(shape[1]) + # bn = paddle.nn.BatchNorm2D(shape[1]) x1 = paddle.to_tensor(x) x1.stop_gradient = False y = bn(x1) @@ -166,7 +164,8 @@ def compute_v1(x, is_test, trainable_statistics): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) y = bn(paddle.to_tensor(x)) return y.numpy() @@ -188,19 +187,22 @@ def compute_v3(x, is_test, trainable_statistics): is_test=is_test, param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(1.0), - trainable=False), + trainable=False, + ), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(0.0), - trainable=False), - trainable_statistics=trainable_statistics) + trainable=False, + ), + trainable_statistics=trainable_statistics, + ) y = bn(paddle.to_tensor(x)) return y.numpy() def compute_v4(x): with fluid.dygraph.guard(p): - bn = paddle.nn.BatchNorm2D(shape[1], - weight_attr=False, - bias_attr=False) + bn = paddle.nn.BatchNorm2D( + shape[1], weight_attr=False, bias_attr=False + ) y = bn(paddle.to_tensor(x)) return y.numpy() @@ -225,7 +227,8 @@ def compute_v1(x_np, is_test, trainable_statistics): bn = fluid.dygraph.BatchNorm( shape[1], is_test=is_test, - trainable_statistics=trainable_statistics) + trainable_statistics=trainable_statistics, + ) x = fluid.data(name='x', shape=x_np.shape, dtype=x_np.dtype) y = bn(x) exe.run(fluid.default_startup_program()) @@ -248,7 +251,6 @@ def compute_v2(x_np): class TestBatchNormChannelLast(unittest.TestCase): - def setUp(self): self.original_dtyep = paddle.get_default_dtype() # MIOPEN not support data type of double @@ -277,14 +279,13 @@ def test_1d(self): y2 = paddle.transpose(y2, [0, 2, 1]) if core.is_compiled_with_rocm(): # HIP will fail if no atol - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05, atol=1e-07 + ) else: - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05 + ) def test_2d(self): for p in self.places: @@ -300,14 +301,13 @@ def test_2d(self): y2 = paddle.transpose(y2, [0, 2, 3, 1]) if core.is_compiled_with_rocm(): # HIP will fail if no atol - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05, atol=1e-07 + ) else: - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05 + ) def test_3d(self): for p in self.places: @@ -323,14 +323,13 @@ def test_3d(self): y2 = paddle.transpose(y2, [0, 2, 3, 4, 1]) if core.is_compiled_with_rocm(): # HIP will fail if no atol - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05, atol=1e-07 + ) else: - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05 + ) def test_1d_opt(self): with fluid.dygraph.guard(): @@ -352,18 +351,18 @@ def test_1d_opt(self): y.backward() y2.backward() - assert np.allclose(y.numpy().flatten(), - y2.numpy().flatten(), - atol=1e-5, - rtol=1e-5) - assert np.allclose(bn1d.weight.grad.numpy().flatten(), - bn2d.weight.grad.numpy().flatten(), - atol=1e-5, - rtol=1e-5) + assert np.allclose( + y.numpy().flatten(), y2.numpy().flatten(), atol=1e-5, rtol=1e-5 + ) + assert np.allclose( + bn1d.weight.grad.numpy().flatten(), + bn2d.weight.grad.numpy().flatten(), + atol=1e-5, + rtol=1e-5, + ) class TestBatchNormUseGlobalStats(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -382,11 +381,14 @@ def test_global_stats(self): net1 = paddle.fluid.dygraph.BatchNorm( 6, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(1.0)), + initializer=fluid.initializer.Constant(1.0) + ), use_global_stats=self.use_global_stats, - trainable_statistics=self.trainable_statistics) + trainable_statistics=self.trainable_statistics, + ) net2 = paddle.nn.BatchNorm2D( - 6, use_global_stats=self.use_global_stats) + 6, use_global_stats=self.use_global_stats + ) net2.weight = net1.weight net2.bias = net1.bias if self.trainable_statistics == True: @@ -420,5 +422,6 @@ def init_test(self): if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_batch_sampler.py b/python/paddle/fluid/tests/unittests/test_batch_sampler.py index 833d96e8e3c4d..a848898c53f89 100644 --- a/python/paddle/fluid/tests/unittests/test_batch_sampler.py +++ b/python/paddle/fluid/tests/unittests/test_batch_sampler.py @@ -15,14 +15,19 @@ import unittest import numpy as np -from paddle.io import BatchSampler, Dataset, Sampler, SequenceSampler, \ - RandomSampler, WeightedRandomSampler +from paddle.io import ( + BatchSampler, + Dataset, + Sampler, + SequenceSampler, + RandomSampler, + WeightedRandomSampler, +) IMAGE_SIZE = 32 class RandomDataset(Dataset): - def __init__(self, sample_num, class_num): self.sample_num = sample_num self.class_num = class_num @@ -30,7 +35,7 @@ def __init__(self, sample_num, class_num): def __getitem__(self, idx): np.random.seed(idx) image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, self.class_num - 1, (1, )).astype('int64') + label = np.random.randint(0, self.class_num - 1, (1,)).astype('int64') return image, label def __len__(self): @@ -38,7 +43,6 @@ def __len__(self): class TestSampler(unittest.TestCase): - def test_main(self): dataset = RandomDataset(100, 10) sampler = Sampler(dataset) @@ -50,7 +54,6 @@ def test_main(self): class TestSequenceSampler(unittest.TestCase): - def test_main(self): dataset = RandomDataset(100, 10) sampler = SequenceSampler(dataset) @@ -61,7 +64,6 @@ def test_main(self): class TestRandomSampler(unittest.TestCase): - def test_main(self): dataset = RandomDataset(100, 10) sampler = RandomSampler(dataset) @@ -96,10 +98,9 @@ def test_with_generator(self): def test_with_generator_num_samples(self): dataset = RandomDataset(100, 10) generator = iter(range(0, 60)) - sampler = RandomSampler(dataset, - generator=generator, - num_samples=50, - replacement=True) + sampler = RandomSampler( + dataset, generator=generator, num_samples=50, replacement=True + ) assert len(sampler) == 50 rets = [] @@ -109,7 +110,6 @@ def test_with_generator_num_samples(self): class TestBatchSampler(unittest.TestCase): - def setUp(self): self.num_samples = 1000 self.num_classes = 10 @@ -119,17 +119,20 @@ def setUp(self): def init_batch_sampler(self): dataset = RandomDataset(self.num_samples, self.num_classes) - bs = BatchSampler(dataset=dataset, - batch_size=self.batch_size, - shuffle=self.shuffle, - drop_last=self.drop_last) + bs = BatchSampler( + dataset=dataset, + batch_size=self.batch_size, + shuffle=self.shuffle, + drop_last=self.drop_last, + ) return bs def test_main(self): bs = self.init_batch_sampler() # length check - bs_len = (self.num_samples + int(not self.drop_last) \ - * (self.batch_size - 1)) // self.batch_size + bs_len = ( + self.num_samples + int(not self.drop_last) * (self.batch_size - 1) + ) // self.batch_size self.assertTrue(bs_len == len(bs)) # output indices check @@ -142,7 +145,6 @@ def test_main(self): class TestBatchSamplerDropLast(TestBatchSampler): - def setUp(self): self.num_samples = 1000 self.num_classes = 10 @@ -152,7 +154,6 @@ def setUp(self): class TestBatchSamplerShuffle(TestBatchSampler): - def setUp(self): self.num_samples = 1000 self.num_classes = 10 @@ -162,18 +163,18 @@ def setUp(self): class TestBatchSamplerWithSampler(TestBatchSampler): - def init_batch_sampler(self): dataset = RandomDataset(1000, 10) sampler = SequenceSampler(dataset) - bs = BatchSampler(sampler=sampler, - batch_size=self.batch_size, - drop_last=self.drop_last) + bs = BatchSampler( + sampler=sampler, + batch_size=self.batch_size, + drop_last=self.drop_last, + ) return bs class TestBatchSamplerWithSamplerDropLast(unittest.TestCase): - def setUp(self): self.num_samples = 1000 self.num_classes = 10 @@ -183,7 +184,6 @@ def setUp(self): class TestBatchSamplerWithSamplerShuffle(unittest.TestCase): - def setUp(self): self.num_samples = 1000 self.num_classes = 10 @@ -195,20 +195,21 @@ def test_main(self): try: dataset = RandomDataset(self.num_samples, self.num_classes) sampler = RandomSampler(dataset) - bs = BatchSampler(sampler=sampler, - shuffle=self.shuffle, - batch_size=self.batch_size, - drop_last=self.drop_last) + bs = BatchSampler( + sampler=sampler, + shuffle=self.shuffle, + batch_size=self.batch_size, + drop_last=self.drop_last, + ) self.assertTrue(False) except AssertionError: pass class TestWeightedRandomSampler(unittest.TestCase): - def init_probs(self, total, pos): - pos_probs = np.random.random((pos, )).astype('float32') - probs = np.zeros((total, )).astype('float32') + pos_probs = np.random.random((pos,)).astype('float32') + probs = np.zeros((total,)).astype('float32') probs[:pos] = pos_probs np.random.shuffle(probs) return probs @@ -218,7 +219,7 @@ def test_replacement(self): sampler = WeightedRandomSampler(probs, 30, True) assert len(sampler) == 30 for idx in iter(sampler): - assert probs[idx] > 0. + assert probs[idx] > 0.0 def test_no_replacement(self): probs = self.init_probs(20, 10) @@ -226,13 +227,13 @@ def test_no_replacement(self): assert len(sampler) == 10 idxs = [] for idx in iter(sampler): - assert probs[idx] > 0. + assert probs[idx] > 0.0 idxs.append(idx) assert len(set(idxs)) == len(idxs) def test_assert(self): # all zeros - probs = np.zeros((10, )).astype('float32') + probs = np.zeros((10,)).astype('float32') sampler = WeightedRandomSampler(probs, 10, True) try: for idx in iter(sampler): @@ -252,7 +253,7 @@ def test_assert(self): self.assertTrue(True) # neg probs - probs = -1.0 * np.ones((10, )).astype('float32') + probs = -1.0 * np.ones((10,)).astype('float32') sampler = WeightedRandomSampler(probs, 10, True) try: for idx in iter(sampler): diff --git a/python/paddle/fluid/tests/unittests/test_bce_loss.py b/python/paddle/fluid/tests/unittests/test_bce_loss.py index 417f4795c4e7e..c77196c1d0d94 100644 --- a/python/paddle/fluid/tests/unittests/test_bce_loss.py +++ b/python/paddle/fluid/tests/unittests/test_bce_loss.py @@ -19,88 +19,76 @@ from op_test import OpTest -def test_static_layer(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_static_layer( + place, input_np, label_np, reduction='mean', weight_np=None +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=input_np.shape, - dtype='float64') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float64') + input = paddle.fluid.data( + name='input', shape=input_np.shape, dtype='float64' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float64' + ) if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float64') - bce_loss = paddle.nn.loss.BCELoss(weight=weight, - reduction=reduction) + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float64' + ) + bce_loss = paddle.nn.loss.BCELoss( + weight=weight, reduction=reduction + ) else: bce_loss = paddle.nn.loss.BCELoss(reduction=reduction) res = bce_loss(input, label) exe = paddle.static.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - } if weight_np is None else { - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np} + if weight_np is None + else {"input": input_np, "label": label_np, "weight": weight_np}, + fetch_list=[res], + ) return static_result -def test_static_functional(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_static_functional( + place, input_np, label_np, reduction='mean', weight_np=None +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=input_np.shape, - dtype='float64') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float64') + input = paddle.fluid.data( + name='input', shape=input_np.shape, dtype='float64' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float64' + ) if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float64') - res = paddle.nn.functional.binary_cross_entropy(input, - label, - weight=weight, - reduction=reduction) + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float64' + ) + res = paddle.nn.functional.binary_cross_entropy( + input, label, weight=weight, reduction=reduction + ) else: - res = paddle.nn.functional.binary_cross_entropy(input, - label, - reduction=reduction) + res = paddle.nn.functional.binary_cross_entropy( + input, label, reduction=reduction + ) exe = paddle.static.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - } if weight_np is None else { - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np} + if weight_np is None + else {"input": input_np, "label": label_np, "weight": weight_np}, + fetch_list=[res], + ) return static_result -def test_dygraph_layer(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_dygraph_layer( + place, input_np, label_np, reduction='mean', weight_np=None +): paddle.disable_static() if weight_np is not None: weight = paddle.to_tensor(weight_np) @@ -113,25 +101,22 @@ def test_dygraph_layer(place, return dy_result -def test_dygraph_functional(place, - input_np, - label_np, - reduction='mean', - weight_np=None): +def test_dygraph_functional( + place, input_np, label_np, reduction='mean', weight_np=None +): paddle.disable_static() input = paddle.to_tensor(input_np) label = paddle.to_tensor(label_np) if weight_np is not None: weight = paddle.to_tensor(weight_np) - dy_res = paddle.nn.functional.binary_cross_entropy(input, - label, - weight=weight, - reduction=reduction) + dy_res = paddle.nn.functional.binary_cross_entropy( + input, label, weight=weight, reduction=reduction + ) else: - dy_res = paddle.nn.functional.binary_cross_entropy(input, - label, - reduction=reduction) + dy_res = paddle.nn.functional.binary_cross_entropy( + input, label, reduction=reduction + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result @@ -139,11 +124,19 @@ def test_dygraph_functional(place, def calc_bceloss(input_np, label_np, reduction='mean', weight_np=None): if weight_np is None: - expected = -1 * (label_np * np.log(input_np) + - (1. - label_np) * np.log(1. - input_np)) + expected = -1 * ( + label_np * np.log(input_np) + + (1.0 - label_np) * np.log(1.0 - input_np) + ) else: - expected = -1 * weight_np * (label_np * np.log(input_np) + - (1. - label_np) * np.log(1. - input_np)) + expected = ( + -1 + * weight_np + * ( + label_np * np.log(input_np) + + (1.0 - label_np) * np.log(1.0 - input_np) + ) + ) if reduction == 'mean': expected = np.mean(expected) @@ -156,7 +149,6 @@ def calc_bceloss(input_np, label_np, reduction='mean', weight_np=None): class TestBCELoss(unittest.TestCase): - def test_BCELoss(self): input_np = np.random.uniform(0.1, 0.8, size=(20, 30)).astype(np.float64) label_np = np.random.randint(0, 2, size=(20, 30)).astype(np.float64) @@ -166,89 +158,90 @@ def test_BCELoss(self): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - static_result = test_static_layer(place, input_np, label_np, - reduction) - dy_result = test_dygraph_layer(place, input_np, label_np, - reduction) + static_result = test_static_layer( + place, input_np, label_np, reduction + ) + dy_result = test_dygraph_layer( + place, input_np, label_np, reduction + ) expected = calc_bceloss(input_np, label_np, reduction) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) static_functional = test_static_functional( - place, input_np, label_np, reduction) - dy_functional = test_dygraph_functional(place, input_np, - label_np, reduction) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-05) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-05) + place, input_np, label_np, reduction + ) + dy_functional = test_dygraph_functional( + place, input_np, label_np, reduction + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-05 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-05 + ) np.testing.assert_allclose(dy_functional, expected, rtol=1e-05) def test_BCELoss_weight(self): - input_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float64) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float64) + input_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float64 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float64 + ) weight_np = np.random.random(size=(3, 4, 10)).astype(np.float64) - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) for reduction in ['sum', 'mean', 'none']: - static_result = test_static_layer(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - dy_result = test_dygraph_layer(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - expected = calc_bceloss(input_np, - label_np, - reduction, - weight_np=weight_np) + static_result = test_static_layer( + place, input_np, label_np, reduction, weight_np=weight_np + ) + dy_result = test_dygraph_layer( + place, input_np, label_np, reduction, weight_np=weight_np + ) + expected = calc_bceloss( + input_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) - static_functional = test_static_functional(place, - input_np, - label_np, - reduction, - weight_np=weight_np) - dy_functional = test_dygraph_functional(place, - input_np, - label_np, - reduction, - weight_np=weight_np) + static_functional = test_static_functional( + place, input_np, label_np, reduction, weight_np=weight_np + ) + dy_functional = test_dygraph_functional( + place, input_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_functional, expected, rtol=1e-05) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-05) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-05 + ) np.testing.assert_allclose(dy_functional, expected, rtol=1e-05) def test_BCELoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.loss.BCELoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, paddle.nn.loss.BCELoss, reduction="unsupport reduction" + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([[0.0, 1.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.binary_cross_entropy, - input=input, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.binary_cross_entropy, + input=input, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() def bce_loss(input, label): - return -1 * (label * np.log(input) + (1. - label) * np.log(1. - input)) + return -1 * (label * np.log(input) + (1.0 - label) * np.log(1.0 - input)) class TestBceLossOp(OpTest): - def setUp(self): self.init_test_case() self.op_type = "bce_loss" @@ -270,13 +263,11 @@ def init_test_case(self): class TestBceLossOpCase1(OpTest): - def init_test_cast(self): self.shape = [2, 3, 4, 5] class TestBceLossOpCase2(OpTest): - def init_test_cast(self): self.shape = [2, 3, 20] diff --git a/python/paddle/fluid/tests/unittests/test_bce_with_logits_loss.py b/python/paddle/fluid/tests/unittests/test_bce_with_logits_loss.py index 55da8aea9b560..8d921215bfa32 100644 --- a/python/paddle/fluid/tests/unittests/test_bce_with_logits_loss.py +++ b/python/paddle/fluid/tests/unittests/test_bce_with_logits_loss.py @@ -19,76 +19,78 @@ from paddle.fluid.framework import _test_eager_guard -def call_bce_layer(logit, - label, - weight=None, - reduction='mean', - pos_weight=None): - bce_logit_loss = paddle.nn.loss.BCEWithLogitsLoss(weight=weight, - reduction=reduction, - pos_weight=pos_weight) +def call_bce_layer( + logit, label, weight=None, reduction='mean', pos_weight=None +): + bce_logit_loss = paddle.nn.loss.BCEWithLogitsLoss( + weight=weight, reduction=reduction, pos_weight=pos_weight + ) res = bce_logit_loss(logit, label) return res -def call_bce_functional(logit, - label, - weight=None, - reduction='mean', - pos_weight=None): +def call_bce_functional( + logit, label, weight=None, reduction='mean', pos_weight=None +): res = paddle.nn.functional.binary_cross_entropy_with_logits( - logit, label, weight=weight, reduction=reduction, pos_weight=pos_weight) + logit, label, weight=weight, reduction=reduction, pos_weight=pos_weight + ) return res -def test_static(place, - logit_np, - label_np, - weight_np=None, - reduction='mean', - pos_weight_np=None, - functional=False): +def test_static( + place, + logit_np, + label_np, + weight_np=None, + reduction='mean', + pos_weight_np=None, + functional=False, +): paddle.enable_static() prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - logit = paddle.fluid.data(name='logit', - shape=logit_np.shape, - dtype='float64') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float64') + logit = paddle.fluid.data( + name='logit', shape=logit_np.shape, dtype='float64' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float64' + ) feed_dict = {"logit": logit_np, "label": label_np} pos_weight = None weight = None if pos_weight_np is not None: - pos_weight = paddle.fluid.data(name='pos_weight', - shape=pos_weight_np.shape, - dtype='float64') + pos_weight = paddle.fluid.data( + name='pos_weight', shape=pos_weight_np.shape, dtype='float64' + ) feed_dict["pos_weight"] = pos_weight_np if weight_np is not None: - weight = paddle.fluid.data(name='weight', - shape=weight_np.shape, - dtype='float64') + weight = paddle.fluid.data( + name='weight', shape=weight_np.shape, dtype='float64' + ) feed_dict["weight"] = weight_np if functional: - res = call_bce_functional(logit, label, weight, reduction, - pos_weight) + res = call_bce_functional( + logit, label, weight, reduction, pos_weight + ) else: res = call_bce_layer(logit, label, weight, reduction, pos_weight) exe = paddle.static.Executor(place) - static_result, = exe.run(prog, feed=feed_dict, fetch_list=[res]) + (static_result,) = exe.run(prog, feed=feed_dict, fetch_list=[res]) return static_result -def test_dygraph(place, - logit_np, - label_np, - weight_np=None, - reduction='mean', - pos_weight_np=None, - functional=False): +def test_dygraph( + place, + logit_np, + label_np, + weight_np=None, + reduction='mean', + pos_weight_np=None, + functional=False, +): with paddle.fluid.dygraph.base.guard(): logit = paddle.to_tensor(logit_np) label = paddle.to_tensor(label_np) @@ -99,22 +101,23 @@ def test_dygraph(place, if pos_weight_np is not None: pos_weight = paddle.to_tensor(pos_weight_np) if functional: - dy_res = call_bce_functional(logit, label, weight, reduction, - pos_weight) + dy_res = call_bce_functional( + logit, label, weight, reduction, pos_weight + ) else: dy_res = call_bce_layer(logit, label, weight, reduction, pos_weight) dy_result = dy_res.numpy() return dy_result -def calc_bce_with_logits_loss(logit_np, - label_np, - reduction='mean', - weight_np=None, - pos_weight=None): - expected = np.maximum( - logit_np, - 0) - logit_np * label_np + np.log(1 + np.exp(-np.abs(logit_np))) +def calc_bce_with_logits_loss( + logit_np, label_np, reduction='mean', weight_np=None, pos_weight=None +): + expected = ( + np.maximum(logit_np, 0) + - logit_np * label_np + + np.log(1 + np.exp(-np.abs(logit_np))) + ) if pos_weight is not None: expected = expected * ((pos_weight - 1) * label_np + 1) if weight_np is not None: @@ -131,7 +134,6 @@ def calc_bce_with_logits_loss(logit_np, class TestBCEWithLogitsLoss(unittest.TestCase): - def test_BCEWithLogitsLoss(self): logit_np = np.random.uniform(0.1, 0.8, size=(20, 30)).astype(np.float64) label_np = np.random.randint(0, 2, size=(20, 30)).astype(np.float64) @@ -141,141 +143,174 @@ def test_BCEWithLogitsLoss(self): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - static_result = test_static(place, - logit_np, - label_np, - reduction=reduction) - dy_result = test_dygraph(place, - logit_np, - label_np, - reduction=reduction) - expected = calc_bce_with_logits_loss(logit_np, label_np, - reduction) + static_result = test_static( + place, logit_np, label_np, reduction=reduction + ) + dy_result = test_dygraph( + place, logit_np, label_np, reduction=reduction + ) + expected = calc_bce_with_logits_loss( + logit_np, label_np, reduction + ) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) - static_functional = test_static(place, - logit_np, - label_np, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place, - logit_np, - label_np, - reduction=reduction, - functional=True) + static_functional = test_static( + place, + logit_np, + label_np, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place, + logit_np, + label_np, + reduction=reduction, + functional=True, + ) with _test_eager_guard(): - eager_functional = test_dygraph(place, - logit_np, - label_np, - reduction=reduction, - functional=True) + eager_functional = test_dygraph( + place, + logit_np, + label_np, + reduction=reduction, + functional=True, + ) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-05) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-05) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-05 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-05 + ) np.testing.assert_allclose(dy_functional, expected, rtol=1e-05) - np.testing.assert_allclose(eager_functional, - expected, - rtol=1e-05) + np.testing.assert_allclose( + eager_functional, expected, rtol=1e-05 + ) def test_BCEWithLogitsLoss_weight(self): - logit_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float64) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float64) + logit_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float64 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float64 + ) weight_np = np.random.random(size=(2, 3, 4, 10)).astype(np.float64) - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) for reduction in ['sum', 'mean', 'none']: - static_result = test_static(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction) - dy_result = test_dygraph(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction) - expected = calc_bce_with_logits_loss(logit_np, - label_np, - reduction, - weight_np=weight_np) + static_result = test_static( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + ) + dy_result = test_dygraph( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + ) + expected = calc_bce_with_logits_loss( + logit_np, label_np, reduction, weight_np=weight_np + ) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) - static_functional = test_static(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place, - logit_np, - label_np, - weight_np=weight_np, - reduction=reduction, - functional=True) + static_functional = test_static( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place, + logit_np, + label_np, + weight_np=weight_np, + reduction=reduction, + functional=True, + ) np.testing.assert_allclose(static_functional, expected, rtol=1e-05) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-05) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-05 + ) np.testing.assert_allclose(dy_functional, expected, rtol=1e-05) def test_BCEWithLogitsLoss_pos_weight(self): - logit_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float64) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float64) + logit_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float64 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float64 + ) pos_weight_np = np.random.random(size=(3, 4, 10)).astype(np.float64) weight_np = np.random.random(size=(2, 3, 4, 10)).astype(np.float64) - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) reduction = "mean" - static_result = test_static(place, logit_np, label_np, weight_np, - reduction, pos_weight_np) - dy_result = test_dygraph(place, logit_np, label_np, weight_np, - reduction, pos_weight_np) - expected = calc_bce_with_logits_loss(logit_np, label_np, reduction, - weight_np, pos_weight_np) + static_result = test_static( + place, logit_np, label_np, weight_np, reduction, pos_weight_np + ) + dy_result = test_dygraph( + place, logit_np, label_np, weight_np, reduction, pos_weight_np + ) + expected = calc_bce_with_logits_loss( + logit_np, label_np, reduction, weight_np, pos_weight_np + ) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) - static_functional = test_static(place, - logit_np, - label_np, - weight_np, - reduction, - pos_weight_np, - functional=True) - dy_functional = test_dygraph(place, - logit_np, - label_np, - weight_np, - reduction, - pos_weight_np, - functional=True) + static_functional = test_static( + place, + logit_np, + label_np, + weight_np, + reduction, + pos_weight_np, + functional=True, + ) + dy_functional = test_dygraph( + place, + logit_np, + label_np, + weight_np, + reduction, + pos_weight_np, + functional=True, + ) np.testing.assert_allclose(static_functional, expected, rtol=1e-05) np.testing.assert_allclose(static_functional, dy_functional, rtol=1e-05) np.testing.assert_allclose(dy_functional, expected, rtol=1e-05) def test_BCEWithLogitsLoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.BCEWithLogitsLoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.BCEWithLogitsLoss, + reduction="unsupport reduction", + ) logit = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([[0.0, 1.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.binary_cross_entropy_with_logits, - logit=logit, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.binary_cross_entropy_with_logits, + logit=logit, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py b/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py index c2a379f9982ca..ee1e2fd64585f 100644 --- a/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py +++ b/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py @@ -41,28 +41,41 @@ def test_get_set(self): # beam_size = 2, end_id = 1 # start with start_id [ - self.append_lod_tensor(array, [[0, 1, 2], [0, 1, 2]], - np.array([0, 0], dtype=dtype)) + self.append_lod_tensor( + array, [[0, 1, 2], [0, 1, 2]], np.array([0, 0], dtype=dtype) + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 1, 2], [0, 2, 4]], - np.array([2, 3, 4, 5], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 1, 2], [0, 2, 4]], + np.array([2, 3, 4, 5], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 2, 4], [0, 2, 2, 4, 4]], - np.array([3, 1, 5, 4], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 2, 4], [0, 2, 2, 4, 4]], + np.array([3, 1, 5, 4], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 2, 4], [0, 1, 2, 3, 4]], - np.array([1, 1, 3, 5], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 2, 4], [0, 1, 2, 3, 4]], + np.array([1, 1, 3, 5], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] [ - self.append_lod_tensor(array, [[0, 2, 4], [0, 0, 0, 2, 2]], - np.array([5, 1], dtype=dtype)) + self.append_lod_tensor( + array, + [[0, 2, 4], [0, 0, 0, 2, 2]], + np.array([5, 1], dtype=dtype), + ) for array, dtype in ((ids, "int64"), (scores, "float32")) ] @@ -88,22 +101,22 @@ def test_get_set(self): self.assertEqual(sentence_scores.lod(), expected_lod) expected_data = np.array( - [0, 2, 3, 1, 0, 2, 1, 0, 4, 5, 3, 5, 0, 4, 5, 3, 1], "int64") + [0, 2, 3, 1, 0, 2, 1, 0, 4, 5, 3, 5, 0, 4, 5, 3, 1], "int64" + ) np.testing.assert_array_equal(np.array(sentence_ids), expected_data) np.testing.assert_array_equal(np.array(sentence_scores), expected_data) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestBeamSearchDecodeOpGPU(TestBeamSearchDecodeOp): - def setUp(self): self.scope = core.Scope() self.place = core.CUDAPlace(0) class TestBeamSearchDecodeOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -111,10 +124,9 @@ def test_id_Variable(): # the input pre_ids must be Variable test_ids = np.random.randint(1, 5, [5, 1]).astype("int64") scores = fluid.layers.create_array(dtype='float32') - fluid.layers.beam_search_decode(test_ids, - scores, - beam_size=5, - end_id=0) + fluid.layers.beam_search_decode( + test_ids, scores, beam_size=5, end_id=0 + ) self.assertRaises(TypeError, test_id_Variable) @@ -122,10 +134,9 @@ def test_score_Variable(): # the input pre_scores must be Variable ids = fluid.layers.create_array(dtype='int64') test_scores = np.random.uniform(1, 5, [5, 1]).astype("float32") - fluid.layers.beam_search_decode(ids, - test_scores, - beam_size=5, - end_id=0) + fluid.layers.beam_search_decode( + ids, test_scores, beam_size=5, end_id=0 + ) self.assertRaises(TypeError, test_score_Variable) @@ -133,10 +144,9 @@ def test_id_dtype(): # the dtype of input pre_ids must be int64 type_ids = fluid.layers.create_array(dtype='float32') scores = fluid.layers.create_array(dtype='float32') - fluid.layers.beam_search_decode(type_ids, - scores, - beam_size=5, - end_id=0) + fluid.layers.beam_search_decode( + type_ids, scores, beam_size=5, end_id=0 + ) self.assertRaises(TypeError, test_id_dtype) @@ -144,10 +154,9 @@ def test_score_dtype(): # the dtype of input pre_scores must be float32 ids = fluid.layers.create_array(dtype='int64') type_scores = fluid.layers.create_array(dtype='int64') - fluid.layers.beam_search_decode(ids, - type_scores, - beam_size=5, - end_id=0) + fluid.layers.beam_search_decode( + ids, type_scores, beam_size=5, end_id=0 + ) self.assertRaises(TypeError, test_score_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_beam_search_op.py b/python/paddle/fluid/tests/unittests/test_beam_search_op.py index a274ae92d3b26..7d99fad642292 100644 --- a/python/paddle/fluid/tests/unittests/test_beam_search_op.py +++ b/python/paddle/fluid/tests/unittests/test_beam_search_op.py @@ -41,32 +41,34 @@ def setUp(self): self.scope.var('parent_idx').get_tensor() def test_run(self): - op = Operator('beam_search', - pre_ids='pre_ids', - pre_scores='pre_scores', - ids='ids', - scores='scores', - selected_ids='selected_ids', - selected_scores='selected_scores', - parent_idx='parent_idx', - level=0, - beam_size=self.beam_size, - end_id=0, - is_accumulated=self.is_accumulated) + op = Operator( + 'beam_search', + pre_ids='pre_ids', + pre_scores='pre_scores', + ids='ids', + scores='scores', + selected_ids='selected_ids', + selected_scores='selected_scores', + parent_idx='parent_idx', + level=0, + beam_size=self.beam_size, + end_id=0, + is_accumulated=self.is_accumulated, + ) op.run(self.scope, core.CPUPlace()) selected_ids = self.scope.find_var("selected_ids").get_tensor() selected_scores = self.scope.find_var("selected_scores").get_tensor() parent_idx = self.scope.find_var("parent_idx").get_tensor() - np.testing.assert_allclose(np.array(selected_ids), - self.output_ids, - rtol=1e-05) - np.testing.assert_allclose(np.array(selected_scores), - self.output_scores, - rtol=1e-05) + np.testing.assert_allclose( + np.array(selected_ids), self.output_ids, rtol=1e-05 + ) + np.testing.assert_allclose( + np.array(selected_scores), self.output_scores, rtol=1e-05 + ) self.assertEqual(selected_ids.lod(), self.output_lod) - np.testing.assert_allclose(np.array(parent_idx), - self.output_parent_idx, - rtol=1e-05) + np.testing.assert_allclose( + np.array(parent_idx), self.output_parent_idx, rtol=1e-05 + ) def _create_pre_ids(self): np_data = np.array([[1, 2, 3, 4]], dtype='int64') @@ -78,19 +80,22 @@ def _create_pre_scores(self): def _create_ids(self): self.lod = [[0, 2, 4], [0, 1, 2, 3, 4]] - np_data = np.array([[4, 2, 5], [2, 1, 3], [3, 5, 2], [8, 2, 1]], - dtype='int64') + np_data = np.array( + [[4, 2, 5], [2, 1, 3], [3, 5, 2], [8, 2, 1]], dtype='int64' + ) tensor = create_tensor(self.scope, "ids", np_data) tensor.set_lod(self.lod) def _create_scores(self): - np_data = np.array([ - [0.5, 0.3, 0.2], - [0.6, 0.3, 0.1], - [0.9, 0.5, 0.1], - [0.7, 0.5, 0.1], - ], - dtype='float32') + np_data = np.array( + [ + [0.5, 0.3, 0.2], + [0.6, 0.3, 0.1], + [0.9, 0.5, 0.1], + [0.7, 0.5, 0.1], + ], + dtype='float32', + ) tensor = create_tensor(self.scope, "scores", np_data) tensor.set_lod(self.lod) @@ -104,7 +109,6 @@ def set_outputs(self): class BeamSearchOpTester2(BeamSearchOpTester): - def _create_pre_ids(self): np_data = np.array([[1], [2], [3], [4]], dtype='int64') tensor = create_tensor(self.scope, 'pre_ids', np_data) @@ -120,13 +124,15 @@ def _create_ids(self): tensor.set_lod(self.lod) def _create_scores(self): - np_data = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.1, 0.7], - ], - dtype='float32') + np_data = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.1, 0.7], + ], + dtype='float32', + ) tensor = create_tensor(self.scope, "scores", np_data) tensor.set_lod(self.lod) @@ -156,13 +162,15 @@ def _create_ids(self): tensor.set_lod(self.lod) def _create_scores(self): - np_data = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.6, 0.7], - ], - dtype='float32') + np_data = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.6, 0.7], + ], + dtype='float32', + ) tensor = create_tensor(self.scope, "scores", np_data) tensor.set_lod(self.lod) @@ -192,13 +200,15 @@ def _create_ids(self): tensor.set_lod(self.lod) def _create_scores(self): - np_data = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.6, 0.7], - ], - dtype='float32') + np_data = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.6, 0.7], + ], + dtype='float32', + ) tensor = create_tensor(self.scope, "scores", np_data) tensor.set_lod(self.lod) @@ -228,13 +238,15 @@ def _create_ids(self): tensor.set_lod(self.lod) def _create_scores(self): - np_data = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.1, 0.7], - ], - dtype='float32') + np_data = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.1, 0.7], + ], + dtype='float32', + ) tensor = create_tensor(self.scope, "scores", np_data) tensor.set_lod(self.lod) @@ -242,8 +254,9 @@ def set_outputs(self): self.beam_size = 2 self.is_accumulated = False self.output_ids = np.array([7, 3, 3, 1])[:, np.newaxis] - self.output_scores = np.array([1.50685, 0.996027, 0.194639, - 0.043325])[:, np.newaxis] + self.output_scores = np.array([1.50685, 0.996027, 0.194639, 0.043325])[ + :, np.newaxis + ] self.output_lod = [[0, 2, 4], [0, 0, 2, 3, 4]] self.output_parent_idx = np.array([1, 1, 2, 3]) @@ -265,13 +278,15 @@ def _create_ids(self): tensor.set_lod(self.lod) def _create_scores(self): - np_data = np.array([ - [0.6, 0.9], - [0.5, 0.3], - [0.9, 0.5], - [0.1, 0.7], - ], - dtype='float32') + np_data = np.array( + [ + [0.6, 0.9], + [0.5, 0.3], + [0.9, 0.5], + [0.1, 0.7], + ], + dtype='float32', + ) tensor = create_tensor(self.scope, "scores", np_data) tensor.set_lod(self.lod) @@ -285,100 +300,115 @@ def set_outputs(self): class TestBeamSearchOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - pre_ids = fluid.data(name='pre_id', - shape=[1], - lod_level=2, - dtype='int64') - pre_scores = fluid.data(name='pre_scores', - shape=[1], - lod_level=2, - dtype='float32') + pre_ids = fluid.data( + name='pre_id', shape=[1], lod_level=2, dtype='int64' + ) + pre_scores = fluid.data( + name='pre_scores', shape=[1], lod_level=2, dtype='float32' + ) probs = fluid.data(name='probs', shape=[10000], dtype='float32') topk_scores, topk_indices = fluid.layers.topk(probs, k=4) accu_scores = fluid.layers.elementwise_add( x=fluid.layers.log(x=topk_scores), y=fluid.layers.reshape(pre_scores, shape=[-1]), - axis=0) + axis=0, + ) def test_preids_Variable(): # the input pre_ids must be Variable preids_data = np.random.randint(1, 5, [5, 1]).astype("int64") - fluid.layers.beam_search(pre_ids=preids_data, - pre_scores=pre_scores, - ids=topk_indices, - scores=accu_scores, - beam_size=4, - end_id=1) + fluid.layers.beam_search( + pre_ids=preids_data, + pre_scores=pre_scores, + ids=topk_indices, + scores=accu_scores, + beam_size=4, + end_id=1, + ) self.assertRaises(TypeError, test_preids_Variable) def test_prescores_Variable(): # the input pre_scores must be Variable - prescores_data = np.random.uniform(1, 5, - [5, 1]).astype("float32") - fluid.layers.beam_search(pre_ids=pre_ids, - pre_scores=prescores_data, - ids=topk_indices, - scores=accu_scores, - beam_size=4, - end_id=1) + prescores_data = np.random.uniform(1, 5, [5, 1]).astype( + "float32" + ) + fluid.layers.beam_search( + pre_ids=pre_ids, + pre_scores=prescores_data, + ids=topk_indices, + scores=accu_scores, + beam_size=4, + end_id=1, + ) self.assertRaises(TypeError, test_prescores_Variable) def test_ids_Variable(): # the input ids must be Variable or None ids_data = np.random.randint(1, 5, [5, 1]).astype("int64") - fluid.layers.beam_search(pre_ids=pre_ids, - pre_scores=pre_scores, - ids=ids_data, - scores=accu_scores, - beam_size=4, - end_id=1) + fluid.layers.beam_search( + pre_ids=pre_ids, + pre_scores=pre_scores, + ids=ids_data, + scores=accu_scores, + beam_size=4, + end_id=1, + ) self.assertRaises(TypeError, test_ids_Variable) def test_scores_Variable(): # the input scores must be Variable scores_data = np.random.uniform(1, 5, [5, 1]).astype("float32") - fluid.layers.beam_search(pre_ids=pre_ids, - pre_scores=pre_scores, - ids=topk_indices, - scores=scores_data, - beam_size=4, - end_id=1) + fluid.layers.beam_search( + pre_ids=pre_ids, + pre_scores=pre_scores, + ids=topk_indices, + scores=scores_data, + beam_size=4, + end_id=1, + ) self.assertRaises(TypeError, test_scores_Variable) def test_preids_dtype(): # the dtype of input pre_ids must be int64 - preids_type_data = fluid.data(name='preids_type_data', - shape=[1], - lod_level=2, - dtype='float32') - fluid.layers.beam_search(pre_ids=preids_type_data, - pre_scores=pre_scores, - ids=topk_indices, - scores=accu_scores, - beam_size=4, - end_id=1) + preids_type_data = fluid.data( + name='preids_type_data', + shape=[1], + lod_level=2, + dtype='float32', + ) + fluid.layers.beam_search( + pre_ids=preids_type_data, + pre_scores=pre_scores, + ids=topk_indices, + scores=accu_scores, + beam_size=4, + end_id=1, + ) self.assertRaises(TypeError, test_preids_dtype) def test_prescores_dtype(): # the dtype of input pre_scores must be float32 - prescores_type_data = fluid.data(name='prescores_type_data', - shape=[1], - lod_level=2, - dtype='int64') - fluid.layers.beam_search(pre_ids=pre_ids, - pre_scores=prescores_type_data, - ids=topk_indices, - scores=accu_scores, - beam_size=4, - end_id=1) + prescores_type_data = fluid.data( + name='prescores_type_data', + shape=[1], + lod_level=2, + dtype='int64', + ) + fluid.layers.beam_search( + pre_ids=pre_ids, + pre_scores=prescores_type_data, + ids=topk_indices, + scores=accu_scores, + beam_size=4, + end_id=1, + ) self.assertRaises(TypeError, test_prescores_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_bernoulli_op.py b/python/paddle/fluid/tests/unittests/test_bernoulli_op.py index 662fbfffe88e2..027afdb177a29 100644 --- a/python/paddle/fluid/tests/unittests/test_bernoulli_op.py +++ b/python/paddle/fluid/tests/unittests/test_bernoulli_op.py @@ -27,7 +27,6 @@ def output_hist(out): class TestBernoulliOp(OpTest): - def setUp(self): self.op_type = "bernoulli" self.inputs = {"X": np.random.uniform(size=(1000, 784))} @@ -43,7 +42,6 @@ def verify_output(self, outs): class TestBernoulliApi(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() x = paddle.rand([1024, 1024]) @@ -56,14 +54,14 @@ def test_static(self): x = paddle.rand([1024, 1024]) out = paddle.bernoulli(x) exe = paddle.static.Executor(paddle.CPUPlace()) - out = exe.run(paddle.static.default_main_program(), - fetch_list=[out.name]) + out = exe.run( + paddle.static.default_main_program(), fetch_list=[out.name] + ) hist, prob = output_hist(out[0]) np.testing.assert_allclose(hist, prob, rtol=0, atol=0.01) class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -83,7 +81,7 @@ def test_fixed_random_number(self): self.assertEqual(np.sum(index0), 260028995) self.assertEqual(np.sum(index1), 8582429431) self.assertEqual(np.sum(index2), 8581445798) - expect = [0., 0., 0., 0., 0., 0., 0., 1., 1., 1.] + expect = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0] np.testing.assert_array_equal(y[16, 500, 500:510], expect) x = paddle.to_tensor(x_np, dtype='float32') @@ -92,7 +90,7 @@ def test_fixed_random_number(self): self.assertEqual(np.sum(index0), 260092343) self.assertEqual(np.sum(index1), 8583509076) self.assertEqual(np.sum(index2), 8582778540) - expect = [0., 0., 1., 1., 1., 1., 0., 1., 1., 1.] + expect = [0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0] np.testing.assert_array_equal(y[16, 500, 500:510], expect) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_bfgs.py b/python/paddle/fluid/tests/unittests/test_bfgs.py index f7023fba86b57..07b15d3aefca7 100644 --- a/python/paddle/fluid/tests/unittests/test_bfgs.py +++ b/python/paddle/fluid/tests/unittests/test_bfgs.py @@ -45,37 +45,35 @@ def test_static_graph_H0(func, x0, H0, dtype='float32'): startup = paddle.static.Program() with paddle.static.program_guard(main, startup): X = paddle.static.data(name='x', shape=[x0.shape[0]], dtype=dtype) - H = paddle.static.data(name='h', - shape=[H0.shape[0], H0.shape[1]], - dtype=dtype) - Y = minimize_bfgs(func, - X, - initial_inverse_hessian_estimate=H, - dtype=dtype) + H = paddle.static.data( + name='h', shape=[H0.shape[0], H0.shape[1]], dtype=dtype + ) + Y = minimize_bfgs( + func, X, initial_inverse_hessian_estimate=H, dtype=dtype + ) exe = paddle.static.Executor() exe.run(startup) return exe.run(main, feed={'x': x0, 'h': H0}, fetch_list=[Y]) -def test_dynamic_graph(func, - x0, - H0=None, - line_search_fn='strong_wolfe', - dtype='float32'): +def test_dynamic_graph( + func, x0, H0=None, line_search_fn='strong_wolfe', dtype='float32' +): paddle.disable_static() x0 = paddle.to_tensor(x0) if H0 is not None: H0 = paddle.to_tensor(H0) - return minimize_bfgs(func, - x0, - initial_inverse_hessian_estimate=H0, - line_search_fn=line_search_fn, - dtype=dtype) + return minimize_bfgs( + func, + x0, + initial_inverse_hessian_estimate=H0, + line_search_fn=line_search_fn, + dtype=dtype, + ) class TestBfgs(unittest.TestCase): - def test_quadratic_nd(self): for dimension in [1, 10]: minimum = np.random.random(size=[dimension]).astype('float32') @@ -85,20 +83,19 @@ def func(x): minimum_ = paddle.assign(minimum) scale_ = paddle.assign(scale) return paddle.sum( - paddle.multiply(scale_, (F.square_error_cost(x, minimum_)))) + paddle.multiply(scale_, (F.square_error_cost(x, minimum_))) + ) x0 = np.random.random(size=[dimension]).astype('float32') results = test_static_graph(func=func, x0=x0) - np.testing.assert_allclose(minimum, - results[2], - rtol=1e-05, - atol=1e-8) + np.testing.assert_allclose( + minimum, results[2], rtol=1e-05, atol=1e-8 + ) results = test_dynamic_graph(func=func, x0=x0) - np.testing.assert_allclose(minimum, - results[2].numpy(), - rtol=1e-05, - atol=1e-8) + np.testing.assert_allclose( + minimum, results[2].numpy(), rtol=1e-05, atol=1e-8 + ) def test_inf_minima(self): extream_point = np.array([-1, 2]).astype('float32') @@ -106,16 +103,17 @@ def test_inf_minima(self): def func(x): # df = 3(x - 1.01)(x - 0.99) # f = x^3 - 3x^2 + 3*1.01*0.99x - return x * x * x / 3.0 - ( - extream_point[0] + extream_point[1] - ) * x * x / 2 + extream_point[0] * extream_point[1] * x + return ( + x * x * x / 3.0 + - (extream_point[0] + extream_point[1]) * x * x / 2 + + extream_point[0] * extream_point[1] * x + ) x0 = np.array([-1.7]).astype('float32') results = test_static_graph(func, x0) self.assertFalse(results[0][0]) def test_multi_minima(self): - def func(x): # df = 12(x + 1.1)(x - 0.2)(x - 0.8) # f = 3*x^4+0.4*x^3-5.46*x^2+2.112*x @@ -138,7 +136,7 @@ def func(position): # f(x, y) = (a - x)^2 + b (y - x^2)^2 # minimum = (a, a^2) x, y = position[0], position[1] - c = (a - x)**2 + b * (y - x**2)**2 + c = (a - x) ** 2 + b * (y - x**2) ** 2 # the return cant be np array[1], or in jacobin will cause flat error return c[0] @@ -153,7 +151,6 @@ def test_rosenbrock(self): self.func_rosenbrock() def test_exception(self): - def func(x): return paddle.dot(x, x) @@ -162,10 +159,9 @@ def func(x): # test initial_inverse_hessian_estimate is good results = test_static_graph_H0(func, x0, H0, dtype='float32') - np.testing.assert_allclose([0.0, 0.0], - results[2], - rtol=1e-05, - atol=1e-8) + np.testing.assert_allclose( + [0.0, 0.0], results[2], rtol=1e-05, atol=1e-8 + ) self.assertTrue(results[0][0]) # test initial_inverse_hessian_estimate is bad @@ -173,11 +169,13 @@ def func(x): self.assertRaises(ValueError, test_dynamic_graph, func, x0, H0=H1) # test line_search_fn is bad - self.assertRaises(NotImplementedError, - test_static_graph, - func, - x0, - line_search_fn='other') + self.assertRaises( + NotImplementedError, + test_static_graph, + func, + x0, + line_search_fn='other', + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_bicubic_interp_op.py b/python/paddle/fluid/tests/unittests/test_bicubic_interp_op.py index ed8bc9113784a..886441429c25d 100644 --- a/python/paddle/fluid/tests/unittests/test_bicubic_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_bicubic_interp_op.py @@ -47,13 +47,15 @@ def value_bound(input, w, h, x, y): return input[:, :, access_y, access_x] -def bicubic_interp_np(input, - out_h, - out_w, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='kNCHW'): +def bicubic_interp_np( + input, + out_h, + out_w, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='kNCHW', +): """trilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -67,13 +69,13 @@ def bicubic_interp_np(input, ratio_h = ratio_w = 0.0 if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: ratio_w = 1.0 * in_w / out_w @@ -81,14 +83,14 @@ def bicubic_interp_np(input, out = np.zeros((batch_size, channel, out_h, out_w)) for k in range(out_h): - if (align_corners): + if align_corners: h = ratio_h * k else: h = ratio_h * (k + 0.5) - 0.5 input_y = np.floor(h) y_t = h - input_y for l in range(out_w): - if (align_corners): + if align_corners: w = ratio_w * l else: w = ratio_w * (l + 0.5) - 0.5 @@ -107,20 +109,23 @@ def bicubic_interp_np(input, coefficients[ii] = cubic_interp1d( input[i, j, access_y, access_x_0], input[i, j, access_y, access_x_1], - input[i, j, access_y, - access_x_2], input[i, j, access_y, - access_x_3], x_t) - out[i, j, k, - l] = cubic_interp1d(coefficients[0], coefficients[1], - coefficients[2], coefficients[3], - y_t) + input[i, j, access_y, access_x_2], + input[i, j, access_y, access_x_3], + x_t, + ) + out[i, j, k, l] = cubic_interp1d( + coefficients[0], + coefficients[1], + coefficients[2], + coefficients[3], + y_t, + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC return out.astype(input.dtype) class TestBicubicInterpOp(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -146,9 +151,15 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = bicubic_interp_np(input_np, out_h, out_w, self.out_size, - self.actual_shape, self.align_corners, - self.data_layout) + output_np = bicubic_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -163,7 +174,7 @@ def setUp(self): 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } self.outputs = {'Out': output_np} @@ -171,80 +182,73 @@ def test_check_output(self): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [2, 3, 5, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True class TestBicubicInterpCase1(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestBicubicInterpCase2(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [3, 3, 9, 6] self.out_h = 10 self.out_w = 8 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestBicubicInterpCase3(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = False class TestBicubicInterpCase4(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = True class TestBicubicInterpCase5(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [3, 3, 9, 6] self.out_h = 11 self.out_w = 11 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([6, 4]).astype("int32") self.align_corners = False class TestBicubicInterpCase6(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [1, 1, 32, 64] @@ -256,31 +260,28 @@ def init_test_case(self): class TestBicubicInterpSame(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestBicubicInterpDataLayout(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [2, 5, 5, 3] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True self.data_layout = "NHWC" class TestBicubicInterpOpAPI(unittest.TestCase): - def test_case(self): np.random.seed(200) x_data = np.random.random((2, 3, 6, 6)).astype("float32") @@ -291,103 +292,101 @@ def test_case(self): prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") dim = fluid.data(name="dim", shape=[1], dtype="int32") - shape_tensor = fluid.data(name="shape_tensor", - shape=[2], - dtype="int32") - actual_size = fluid.data(name="actual_size", - shape=[2], - dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") - - out1 = interpolate(x, - size=[12, 12], - mode='bicubic', - align_corners=False) - out2 = interpolate(x, - size=[12, dim], - mode='bicubic', - align_corners=False) - out3 = interpolate(x, - size=shape_tensor, - mode='bicubic', - align_corners=False) - out4 = interpolate(x, - size=[12, 12], - mode='bicubic', - align_corners=False) - out5 = interpolate(x, - scale_factor=scale_tensor, - mode='bicubic', - align_corners=False) + shape_tensor = fluid.data( + name="shape_tensor", shape=[2], dtype="int32" + ) + actual_size = fluid.data( + name="actual_size", shape=[2], dtype="int32" + ) + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) + + out1 = interpolate( + x, size=[12, 12], mode='bicubic', align_corners=False + ) + out2 = interpolate( + x, size=[12, dim], mode='bicubic', align_corners=False + ) + out3 = interpolate( + x, size=shape_tensor, mode='bicubic', align_corners=False + ) + out4 = interpolate( + x, size=[12, 12], mode='bicubic', align_corners=False + ) + out5 = interpolate( + x, + scale_factor=scale_tensor, + mode='bicubic', + align_corners=False, + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = bicubic_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=False) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = bicubic_interp_np( + x_data, out_h=12, out_w=12, align_corners=False + ) for res in results: np.testing.assert_allclose(res, expect_res, rtol=1e-05) with fluid.dygraph.guard(): x = fluid.dygraph.to_variable(x_data) - interp = interpolate(x, - size=[12, 12], - mode='bicubic', - align_corners=False) + interp = interpolate( + x, size=[12, 12], mode='bicubic', align_corners=False + ) dy_result = interp.numpy() - expect = bicubic_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=False) + expect = bicubic_interp_np( + x_data, out_h=12, out_w=12, align_corners=False + ) np.testing.assert_allclose(dy_result, expect, rtol=1e-05) class TestBicubicOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of interpoalte must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, interpolate, x1) def test_mode_type(): # mode must be "BILINEAR" "TRILINEAR" "NEAREST" "BICUBIC" x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=[12, 12], - mode='UNKONWN', - align_corners=False) + out = interpolate( + x, size=[12, 12], mode='UNKONWN', align_corners=False + ) def test_input_shape(): x = fluid.data(name="x", shape=[2], dtype="float32") - out = interpolate(x, - size=[12, 12], - mode='BICUBIC', - align_corners=False) + out = interpolate( + x, size=[12, 12], mode='BICUBIC', align_corners=False + ) def test_align_corcers(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") @@ -395,77 +394,88 @@ def test_align_corcers(): def test_out_shape(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=[12], - mode='bicubic', - align_corners=False) + out = interpolate( + x, size=[12], mode='bicubic', align_corners=False + ) def test_attr_data_format(): # for 5-D input, data_format only can be NCDHW or NDHWC - input = fluid.data(name="input", - shape=[2, 3, 6, 9, 4], - dtype="float32") - out = interpolate(input, - size=[4, 8, 4, 5], - mode='trilinear', - data_format='NHWC') + input = fluid.data( + name="input", shape=[2, 3, 6, 9, 4], dtype="float32" + ) + out = interpolate( + input, + size=[4, 8, 4, 5], + mode='trilinear', + data_format='NHWC', + ) def test_actual_shape(): # the actual_shape must be Variable. - x = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) - out = interpolate(x, - size=[12, 12], - mode='BICUBIC', - align_corners=False) + x = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + out = interpolate( + x, size=[12, 12], mode='BICUBIC', align_corners=False + ) def test_scale_value(): # the scale must be greater than zero. x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='BICUBIC', - align_corners=False, - scale_factor=-2.0) + out = interpolate( + x, + size=None, + mode='BICUBIC', + align_corners=False, + scale_factor=-2.0, + ) def test_attr_5D_input(): # for 5-D input, data_format only can be NCDHW or NDHWC - input = fluid.data(name="input", - shape=[2, 3, 6, 9, 4], - dtype="float32") - out = interpolate(input, - size=[4, 8, 4, 5], - mode='trilinear', - data_format='NDHWC') + input = fluid.data( + name="input", shape=[2, 3, 6, 9, 4], dtype="float32" + ) + out = interpolate( + input, + size=[4, 8, 4, 5], + mode='trilinear', + data_format='NDHWC', + ) def test_scale_type(): # the scale must be greater than zero. x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - scale = fluid.create_lod_tensor(np.array([-1, 3, 5, - 5]), [[1, 1, 1, 1]], - fluid.CPUPlace()) - out = interpolate(x, - size=None, - mode='bicubic', - align_corners=False, - scale_factor=scale) + scale = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + out = interpolate( + x, + size=None, + mode='bicubic', + align_corners=False, + scale_factor=scale, + ) def test_align_mode(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='nearest', - align_corners=False, - align_mode=2, - scale_factor=1.0) + out = interpolate( + x, + size=None, + mode='nearest', + align_corners=False, + align_mode=2, + scale_factor=1.0, + ) def test_outshape_and_scale(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='bicubic', - align_corners=False, - scale_factor=None) + out = interpolate( + x, + size=None, + mode='bicubic', + align_corners=False, + scale_factor=None, + ) self.assertRaises(ValueError, test_mode_type) self.assertRaises(ValueError, test_input_shape) diff --git a/python/paddle/fluid/tests/unittests/test_bicubic_interp_v2_op.py b/python/paddle/fluid/tests/unittests/test_bicubic_interp_v2_op.py index 8a58f8ec3694a..2bfe40abe53cb 100644 --- a/python/paddle/fluid/tests/unittests/test_bicubic_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_bicubic_interp_v2_op.py @@ -22,18 +22,20 @@ from paddle.nn.functional import interpolate -def bicubic_interp_test(x, - OutSize=None, - SizeTensor=None, - Scale=None, - data_layout='kNCHW', - out_d=-1, - out_h=-1, - out_w=-1, - scale=[], - interp_method='bicubic', - align_corners=True, - align_mode=0): +def bicubic_interp_test( + x, + OutSize=None, + SizeTensor=None, + Scale=None, + data_layout='kNCHW', + out_d=-1, + out_h=-1, + out_w=-1, + scale=[], + interp_method='bicubic', + align_corners=True, + align_mode=0, +): if isinstance(scale, float) or isinstance(scale, int): scale_list = [] for _ in range(len(x.shape) - 2): @@ -43,12 +45,23 @@ def bicubic_interp_test(x, scale = list(map(float, scale)) if SizeTensor is not None: if not isinstance(SizeTensor, list) and not isinstance( - SizeTensor, tuple): + SizeTensor, tuple + ): SizeTensor = [SizeTensor] - return paddle._C_ops.bicubic_interp(x, OutSize, SizeTensor, Scale, - data_layout, out_d, out_h, out_w, scale, - interp_method, align_corners, - align_mode) + return paddle._C_ops.bicubic_interp( + x, + OutSize, + SizeTensor, + Scale, + data_layout, + out_d, + out_h, + out_w, + scale, + interp_method, + align_corners, + align_mode, + ) def cubic_1(x, a): @@ -77,15 +90,17 @@ def value_bound(input, w, h, x, y): return input[:, :, access_y, access_x] -def bicubic_interp_np(input, - out_h, - out_w, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='kNCHW'): +def bicubic_interp_np( + input, + out_h, + out_w, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='kNCHW', +): """trilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -99,7 +114,7 @@ def bicubic_interp_np(input, ratio_h = ratio_w = 0.0 if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: @@ -108,7 +123,7 @@ def bicubic_interp_np(input, ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -119,14 +134,14 @@ def bicubic_interp_np(input, out = np.zeros((batch_size, channel, out_h, out_w)) for k in range(out_h): - if (align_corners): + if align_corners: h = ratio_h * k else: h = ratio_h * (k + 0.5) - 0.5 input_y = np.floor(h) y_t = h - input_y for l in range(out_w): - if (align_corners): + if align_corners: w = ratio_w * l else: w = ratio_w * (l + 0.5) - 0.5 @@ -145,20 +160,23 @@ def bicubic_interp_np(input, coefficients[ii] = cubic_interp1d( input[i, j, access_y, access_x_0], input[i, j, access_y, access_x_1], - input[i, j, access_y, - access_x_2], input[i, j, access_y, - access_x_3], x_t) - out[i, j, k, - l] = cubic_interp1d(coefficients[0], coefficients[1], - coefficients[2], coefficients[3], - y_t) + input[i, j, access_y, access_x_2], + input[i, j, access_y, access_x_3], + x_t, + ) + out[i, j, k, l] = cubic_interp1d( + coefficients[0], + coefficients[1], + coefficients[2], + coefficients[3], + y_t, + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC return out.astype(input.dtype) class TestBicubicInterpOp(OpTest): - def setUp(self): self.python_api = bicubic_interp_test self.out_size = None @@ -181,7 +199,7 @@ def setUp(self): if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: scale_h = scale_w = float(self.scale) if isinstance(self.scale, list) and len(self.scale) == 1: scale_w = scale_h = self.scale[0] @@ -194,9 +212,17 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = bicubic_interp_np(input_np, out_h, out_w, scale_h, scale_w, - self.out_size, self.actual_shape, - self.align_corners, self.data_layout) + output_np = bicubic_interp_np( + input_np, + out_h, + out_w, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -210,11 +236,11 @@ def setUp(self): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: self.scale = [self.scale] if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] @@ -225,10 +251,9 @@ def test_check_output(self): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'bicubic' @@ -241,7 +266,6 @@ def init_test_case(self): class TestBicubicInterpCase1(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [4, 1, 7, 8] @@ -252,7 +276,6 @@ def init_test_case(self): class TestBicubicInterpCase2(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [3, 3, 9, 6] @@ -263,7 +286,6 @@ def init_test_case(self): class TestBicubicInterpCase3(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [1, 1, 32, 64] @@ -274,7 +296,6 @@ def init_test_case(self): class TestBicubicInterpCase4(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [4, 1, 7, 8] @@ -286,7 +307,6 @@ def init_test_case(self): class TestBicubicInterpCase5(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [3, 3, 9, 6] @@ -298,7 +318,6 @@ def init_test_case(self): class TestBicubicInterpCase6(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [1, 1, 32, 64] @@ -310,7 +329,6 @@ def init_test_case(self): class TestBicubicInterpSame(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [2, 3, 32, 64] @@ -321,18 +339,16 @@ def init_test_case(self): class TestBicubicInterpScale(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = [1., 1.] + self.scale = [1.0, 1.0] self.align_corners = True class TestBicubicInterpDataLayout(TestBicubicInterpOp): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [2, 5, 5, 3] @@ -345,7 +361,6 @@ def init_test_case(self): class TestBicubicInterpOpAPI(unittest.TestCase): - def test_imperative_case(self): with _test_eager_guard(): self.func_case() @@ -361,52 +376,51 @@ def func_case(self): prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") dim = fluid.data(name="dim", shape=[1], dtype="int32") - shape_tensor = fluid.data(name="shape_tensor", - shape=[2], - dtype="int32") - actual_size = fluid.data(name="actual_size", - shape=[2], - dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") - - out1 = interpolate(x, - size=[12, 12], - mode='bicubic', - align_corners=False) - out2 = interpolate(x, - size=[12, dim], - mode='bicubic', - align_corners=False) - out3 = interpolate(x, - size=shape_tensor, - mode='bicubic', - align_corners=False) - out4 = interpolate(x, - size=[12, 12], - mode='bicubic', - align_corners=False) - out5 = interpolate(x, - scale_factor=scale_tensor, - mode='bicubic', - align_corners=False) - out6 = interpolate(x, - scale_factor=2.0, - mode='bicubic', - align_corners=False) - out7 = interpolate(x, - scale_factor=[2.0, 2.0], - mode='bicubic', - align_corners=False) + shape_tensor = fluid.data( + name="shape_tensor", shape=[2], dtype="int32" + ) + actual_size = fluid.data( + name="actual_size", shape=[2], dtype="int32" + ) + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) + + out1 = interpolate( + x, size=[12, 12], mode='bicubic', align_corners=False + ) + out2 = interpolate( + x, size=[12, dim], mode='bicubic', align_corners=False + ) + out3 = interpolate( + x, size=shape_tensor, mode='bicubic', align_corners=False + ) + out4 = interpolate( + x, size=[12, 12], mode='bicubic', align_corners=False + ) + out5 = interpolate( + x, + scale_factor=scale_tensor, + mode='bicubic', + align_corners=False, + ) + out6 = interpolate( + x, scale_factor=2.0, mode='bicubic', align_corners=False + ) + out7 = interpolate( + x, scale_factor=[2.0, 2.0], mode='bicubic', align_corners=False + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -417,55 +431,51 @@ def func_case(self): "dim": dim_data, "shape_tensor": shape_data, "actual_size": actual_size_data, - "scale_tensor": scale_data + "scale_tensor": scale_data, }, fetch_list=[out1, out2, out3, out4, out5, out6, out7], - return_numpy=True) + return_numpy=True, + ) - expect_res = bicubic_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=False) + expect_res = bicubic_interp_np( + x_data, out_h=12, out_w=12, align_corners=False + ) for res in results: np.testing.assert_allclose(res, expect_res, rtol=1e-05) with fluid.dygraph.guard(): x = fluid.dygraph.to_variable(x_data) - interp = interpolate(x, - size=[12, 12], - mode='bicubic', - align_corners=False) + interp = interpolate( + x, size=[12, 12], mode='bicubic', align_corners=False + ) dy_result = interp.numpy() - expect = bicubic_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=False) + expect = bicubic_interp_np( + x_data, out_h=12, out_w=12, align_corners=False + ) np.testing.assert_allclose(dy_result, expect, rtol=1e-05) class TestBicubicOpError(unittest.TestCase): - def test_imperative_errors(self): # the input of interpoalte must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, interpolate, x1) def test_mode_type(): # mode must be "BILINEAR" "TRILINEAR" "NEAREST" "BICUBIC" x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=[12, 12], - mode='UNKONWN', - align_corners=False) + out = interpolate( + x, size=[12, 12], mode='UNKONWN', align_corners=False + ) def test_input_shape(): x = fluid.data(name="x", shape=[2], dtype="float32") - out = interpolate(x, - size=[12, 12], - mode='BICUBIC', - align_corners=False) + out = interpolate( + x, size=[12, 12], mode='BICUBIC', align_corners=False + ) def test_align_corcers(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") @@ -477,123 +487,138 @@ def test_out_shape(): def test_attr_data_format(): # for 5-D input, data_format only can be NCDHW or NDHWC - input = fluid.data(name="input", - shape=[2, 3, 6, 9, 4], - dtype="float32") - out = interpolate(input, - size=[4, 8, 4, 5], - mode='trilinear', - data_format='NHWC') + input = fluid.data( + name="input", shape=[2, 3, 6, 9, 4], dtype="float32" + ) + out = interpolate( + input, size=[4, 8, 4, 5], mode='trilinear', data_format='NHWC' + ) def test_actual_shape(): # the actual_shape must be Variable. - x = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], - fluid.CPUPlace()) - out = interpolate(x, - size=[12, 12], - mode='BICUBIC', - align_corners=False) + x = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + out = interpolate( + x, size=[12, 12], mode='BICUBIC', align_corners=False + ) def test_scale_value(): # the scale must be greater than zero. x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='BICUBIC', - align_corners=False, - scale_factor=-2.0) + out = interpolate( + x, + size=None, + mode='BICUBIC', + align_corners=False, + scale_factor=-2.0, + ) def test_attr_5D_input(): # for 5-D input, data_format only can be NCDHW or NDHWC - input = fluid.data(name="input", - shape=[2, 3, 6, 9, 4], - dtype="float32") - out = interpolate(input, - size=[4, 8, 4, 5], - mode='trilinear', - data_format='NDHWC') + input = fluid.data( + name="input", shape=[2, 3, 6, 9, 4], dtype="float32" + ) + out = interpolate( + input, size=[4, 8, 4, 5], mode='trilinear', data_format='NDHWC' + ) def test_scale_type(): # the scale must be greater than zero. x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - scale = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) - out = interpolate(x, - size=None, - mode='bicubic', - align_corners=False, - scale_factor=scale) + scale = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + out = interpolate( + x, + size=None, + mode='bicubic', + align_corners=False, + scale_factor=scale, + ) def test_align_mode(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='nearest', - align_corners=False, - align_mode=2, - scale_factor=1.0) + out = interpolate( + x, + size=None, + mode='nearest', + align_corners=False, + align_mode=2, + scale_factor=1.0, + ) def test_outshape_and_scale(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='bicubic', - align_corners=False, - scale_factor=None) + out = interpolate( + x, + size=None, + mode='bicubic', + align_corners=False, + scale_factor=None, + ) def test_align_corners_and_nearest(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='nearest', - align_corners=True, - scale_factor=None) + out = interpolate( + x, + size=None, + mode='nearest', + align_corners=True, + scale_factor=None, + ) def test_scale_shape(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='nearest', - align_corners=False, - scale_factor=[1, 2, 2]) + out = interpolate( + x, + size=None, + mode='nearest', + align_corners=False, + scale_factor=[1, 2, 2], + ) def test_scale_value_1(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='bicubic', - align_corners=False, - scale_factor=[1, 2, 2]) + out = interpolate( + x, + size=None, + mode='bicubic', + align_corners=False, + scale_factor=[1, 2, 2], + ) def test_size_and_scale(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size=None, - mode='bicubic', - align_corners=False, - scale_factor=None) + out = interpolate( + x, + size=None, + mode='bicubic', + align_corners=False, + scale_factor=None, + ) def test_size_and_scale2(): x = fluid.data(name="input", shape=[2, 3, 6, 9, 4], dtype="float32") - out = interpolate(x, - size=[2, 2, 2], - mode='trilinear', - align_corners=False, - scale_factor=2.0) + out = interpolate( + x, + size=[2, 2, 2], + mode='trilinear', + align_corners=False, + scale_factor=2.0, + ) def test_size_type(): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") - out = interpolate(x, - size={2, 2}, - mode='bicubic', - align_corners=False) + out = interpolate( + x, size={2, 2}, mode='bicubic', align_corners=False + ) def test_input_shape_1(): x = fluid.data(name="x", shape=[2, 1, 0, 0], dtype="float32") - out = interpolate(x, - size=[3, 3], - mode="bicubic", - align_corners=False) + out = interpolate( + x, size=[3, 3], mode="bicubic", align_corners=False + ) self.assertRaises(ValueError, test_mode_type) self.assertRaises(ValueError, test_input_shape) @@ -619,10 +644,10 @@ def test_errors(self): self.test_imperative_errors() -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestBicubicInterpOpForFloat16(unittest.TestCase): - def init_test_case(self): self.interp_method = 'bicubic' self.input_shape = [2, 3, 5, 5] @@ -635,11 +660,13 @@ def check_main(self, x_np, dtype): x_np = x_np.astype(dtype) x = paddle.to_tensor(x_np) x.stop_gradient = False - y = interpolate(x, - size=self.out_size.tolist(), - mode=self.interp_method, - align_corners=self.align_corners, - data_format=self.data_layout) + y = interpolate( + x, + size=self.out_size.tolist(), + mode=self.interp_method, + align_corners=self.align_corners, + data_format=self.data_layout, + ) x_g = paddle.grad(y, x) y_np = y[0].numpy().astype('float32') x_g_np = x_g[0].numpy().astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_bilateral_slice_op.py b/python/paddle/fluid/tests/unittests/test_bilateral_slice_op.py index 976e7df60b80c..db0b8c182de92 100644 --- a/python/paddle/fluid/tests/unittests/test_bilateral_slice_op.py +++ b/python/paddle/fluid/tests/unittests/test_bilateral_slice_op.py @@ -20,7 +20,6 @@ class Gsz: - def __init__(self, h, w, gd, gh, gw, input_chans): self.h = h self.w = w @@ -53,8 +52,9 @@ def d_weight_z(x): return d_diff_abs(x) -def naive_bilateral_slice_forward(output, grid, guide, input, gsz, has_offset, - total_count, output_chans): +def naive_bilateral_slice_forward( + output, grid, guide, input, gsz, has_offset, total_count, output_chans +): h = gsz.h w = gsz.w gd = gsz.gd @@ -72,7 +72,7 @@ def naive_bilateral_slice_forward(output, grid, guide, input, gsz, has_offset, x = idx % w y = idx // w % h out_c = (idx // (h * w)) % output_chans - b = (idx // (output_chans * w * h)) + b = idx // (output_chans * w * h) gx = (x + 0.5) * gw / (1.0 * w) gy = (y + 0.5) * gh / (1.0 * h) @@ -99,11 +99,12 @@ def naive_bilateral_slice_forward(output, grid, guide, input, gsz, has_offset, wz = weight_z(zz + 0.5 - gz) c_ = coeff_stride * out_c + in_c - coeff_sample += grid[int(b), - int(c_), - int(z_), - int(y_), - int(x_)] * wx * wy * wz + coeff_sample += ( + grid[int(b), int(c_), int(z_), int(y_), int(x_)] + * wx + * wy + * wz + ) if in_c < input_chans: value += coeff_sample * input[int(b), int(in_c), int(y), int(x)] @@ -132,15 +133,16 @@ def naive_bilateral_slice(x, guide, grid, has_offset): gsz = Gsz(h, w, gd, gh, gw, input_chans) total_count = bs * h * w * output.shape[1] - naive_bilateral_slice_forward(output, grid, guide, x, gsz, has_offset, - total_count, output.shape[1]) + naive_bilateral_slice_forward( + output, grid, guide, x, gsz, has_offset, total_count, output.shape[1] + ) return output -@unittest.skipIf(not paddle.fluid.is_compiled_with_cuda(), - 'CPU testing is not supported') +@unittest.skipIf( + not paddle.fluid.is_compiled_with_cuda(), 'CPU testing is not supported' +) class TestBilateralSliceOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'bilateral_slice' @@ -177,27 +179,26 @@ def initTestCase(self): self.data_type = 'float64' -@unittest.skipIf(not paddle.fluid.is_compiled_with_cuda(), - 'CPU testing is not supported') +@unittest.skipIf( + not paddle.fluid.is_compiled_with_cuda(), 'CPU testing is not supported' +) class TestBilateralSliceOp1(TestBilateralSliceOp): - def initTestCase(self): self.has_offset = True self.data_type = 'float32' class TestBilateralSliceApi(unittest.TestCase): - def test_api(self): - x = paddle.fluid.data(name='x', - shape=[None, 3, 25, 15], - dtype='float32') - guide = paddle.fluid.data(name='guide', - shape=[None, 25, 15], - dtype='float32') - grid = paddle.fluid.data(name='grid', - shape=[None, None, 8, 5, 3], - dtype='float32') + x = paddle.fluid.data( + name='x', shape=[None, 3, 25, 15], dtype='float32' + ) + guide = paddle.fluid.data( + name='guide', shape=[None, 25, 15], dtype='float32' + ) + grid = paddle.fluid.data( + name='grid', shape=[None, None, 8, 5, 3], dtype='float32' + ) paddle.fluid.contrib.layers.bilateral_slice(x, guide, grid, False) if not paddle.fluid.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_bilinear_api.py b/python/paddle/fluid/tests/unittests/test_bilinear_api.py index 031163cdc435d..dcdb03d05bad9 100644 --- a/python/paddle/fluid/tests/unittests/test_bilinear_api.py +++ b/python/paddle/fluid/tests/unittests/test_bilinear_api.py @@ -21,10 +21,10 @@ class TestBilinearAPI(unittest.TestCase): - def test_api(self): - with fluid.program_guard(fluid.default_startup_program(), - fluid.default_main_program()): + with fluid.program_guard( + fluid.default_startup_program(), fluid.default_main_program() + ): if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -37,29 +37,26 @@ def test_api(self): layer1 = np.random.random((5, 5)).astype('float32') layer2 = np.random.random((5, 4)).astype('float32') - bilinear = paddle.nn.Bilinear(in1_features=5, - in2_features=4, - out_features=1000) + bilinear = paddle.nn.Bilinear( + in1_features=5, in2_features=4, out_features=1000 + ) ret = bilinear(data1, data2) exe.run(fluid.default_startup_program()) - ret_fetch = exe.run(feed={ - 'X1': layer1, - 'X2': layer2 - }, - fetch_list=[ret.name]) + ret_fetch = exe.run( + feed={'X1': layer1, 'X2': layer2}, fetch_list=[ret.name] + ) self.assertEqual(ret_fetch[0].shape, (5, 1000)) class TestBilinearAPIDygraph(unittest.TestCase): - def test_api(self): paddle.disable_static() layer1 = np.random.random((5, 5)).astype('float32') layer2 = np.random.random((5, 4)).astype('float32') - bilinear = paddle.nn.Bilinear(in1_features=5, - in2_features=4, - out_features=1000) + bilinear = paddle.nn.Bilinear( + in1_features=5, in2_features=4, out_features=1000 + ) ret = bilinear(paddle.to_tensor(layer1), paddle.to_tensor(layer2)) self.assertEqual(ret.shape, [5, 1000]) diff --git a/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py b/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py index bb46558c09395..b281d30a2221f 100755 --- a/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py @@ -22,14 +22,16 @@ paddle.enable_static() -def bilinear_interp_np(input, - out_h, - out_w, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCHW'): +def bilinear_interp_np( + input, + out_h, + out_w, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCHW', +): """bilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -43,12 +45,12 @@ def bilinear_interp_np(input, ratio_h = ratio_w = 0.0 if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: ratio_w = 1.0 * in_w / out_w @@ -56,37 +58,40 @@ def bilinear_interp_np(input, out = np.zeros((batch_size, channel, out_h, out_w)) for i in range(out_h): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: h = int(ratio_h * (i + 0.5) - 0.5) else: h = int(ratio_h * i) h = max(0, h) hid = 1 if h < in_h - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_h = max(ratio_h * (i + 0.5) - 0.5, 0) h1lambda = idx_src_h - h else: h1lambda = ratio_h * i - h h2lambda = 1.0 - h1lambda for j in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (j + 0.5) - 0.5) else: w = int(ratio_w * j) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (j + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * j - w w2lambda = 1.0 - w1lambda - out[:, :, i, j] = h2lambda*(w2lambda*input[:, :, h, w] + - w1lambda*input[:, :, h, w+wid]) + \ - h1lambda*(w2lambda*input[:, :, h+hid, w] + - w1lambda*input[:, :, h+hid, w+wid]) + out[:, :, i, j] = h2lambda * ( + w2lambda * input[:, :, h, w] + + w1lambda * input[:, :, h, w + wid] + ) + h1lambda * ( + w2lambda * input[:, :, h + hid, w] + + w1lambda * input[:, :, h + hid, w + wid] + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC @@ -95,7 +100,6 @@ def bilinear_interp_np(input, class TestBilinearInterpOp(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -121,9 +125,16 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode, self.data_layout) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -139,7 +150,7 @@ def setUp(self): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } self.outputs = {'Out': output_np} @@ -147,130 +158,120 @@ def test_check_output(self): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase4(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase5(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase6(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 33]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpSame(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpActualShape(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpDataLayout(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 5, 5, 3] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True self.align_mode = 1 @@ -278,15 +279,15 @@ def init_test_case(self): class TestBilinearInterpOpUint8(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None self.init_test_case() self.op_type = "bilinear_interp" self.check_eager = True - input_np = np.random.randint(low=0, high=256, - size=self.input_shape).astype("uint8") + input_np = np.random.randint( + low=0, high=256, size=self.input_shape + ).astype("uint8") if self.scale > 0: out_h = int(self.input_shape[2] * self.scale) @@ -295,9 +296,15 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -309,97 +316,89 @@ def setUp(self): 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } self.outputs = {'Out': output_np} def test_check_output(self): - self.check_output_with_place(place=core.CPUPlace(), - atol=1, - check_eager=self.check_eager) + self.check_output_with_place( + place=core.CPUPlace(), atol=1, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 3, 9, 6] self.out_h = 10 self.out_w = 9 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase1Uint8(TestBilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase2Uint8(TestBilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] self.out_h = 5 self.out_w = 13 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([6, 15]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpOtherMethod1(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 1 class TestBilinearInterpWithMethod2(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 0 class TestBilinearInterpWithMethod3(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = True self.align_mode = 0 class TestBilinearInterpScale1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 2. + self.scale = 2.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 1. + self.scale = 1.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -411,7 +410,6 @@ def init_test_case(self): class TestBilinearInterpZero(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -423,7 +421,6 @@ def init_test_case(self): class TestBilinearInterpOp_attr_tensor(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -456,58 +453,62 @@ def setUp(self): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.check_eager = False self.attrs['out_h'] = self.out_h self.attrs['out_w'] = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, self.out_size, - self.actual_shape, self.align_corners) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 5] self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [3, 3] self.align_corners = True # out_size is a 1-D tensor class TestBilinearInterp_attr_tensor_Case1(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = [8, 12] self.align_corners = True # scale is a 1-D tensor class TestBilinearInterp_attr_tensor_Case2(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.shape_by_1Dtensor = True @@ -515,7 +516,6 @@ def init_test_case(self): # scale is a 1-D tensor class TestBilinearInterp_attr_tensor_Case3(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] @@ -528,23 +528,22 @@ def init_test_case(self): class TestBilinearInterpOpAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[2], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) out1 = fluid.layers.resize_bilinear(x, out_shape=[12, 12]) out2 = fluid.layers.resize_bilinear(x, out_shape=[12, dim]) out3 = fluid.layers.resize_bilinear(x, out_shape=shape_tensor) - out4 = fluid.layers.resize_bilinear(x, - out_shape=[4, 4], - actual_shape=actual_size) + out4 = fluid.layers.resize_bilinear( + x, out_shape=[4, 4], actual_shape=actual_size + ) out5 = fluid.layers.resize_bilinear(x, scale=scale_tensor) x_data = np.random.random((2, 3, 6, 6)).astype("float32") @@ -559,21 +558,22 @@ def test_case(self): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = bilinear_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=True) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = bilinear_interp_np( + x_data, out_h=12, out_w=12, align_corners=True + ) for res in results: np.testing.assert_allclose(res, expect_res, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_bilinear_interp_v2_op.py b/python/paddle/fluid/tests/unittests/test_bilinear_interp_v2_op.py index e25ffeaee53a9..1a3786dbdd5d5 100755 --- a/python/paddle/fluid/tests/unittests/test_bilinear_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_bilinear_interp_v2_op.py @@ -21,18 +21,20 @@ import paddle -def bilinear_interp_test(x, - OutSize=None, - SizeTensor=None, - Scale=None, - data_layout='NCHW', - out_d=-1, - out_h=-1, - out_w=-1, - scale=[], - interp_method='bilinear', - align_corners=True, - align_mode=0): +def bilinear_interp_test( + x, + OutSize=None, + SizeTensor=None, + Scale=None, + data_layout='NCHW', + out_d=-1, + out_h=-1, + out_w=-1, + scale=[], + interp_method='bilinear', + align_corners=True, + align_mode=0, +): if isinstance(scale, float) or isinstance(scale, int): scale_list = [] for _ in range(len(x.shape) - 2): @@ -42,24 +44,37 @@ def bilinear_interp_test(x, scale = list(map(float, scale)) if SizeTensor is not None: if not isinstance(SizeTensor, list) and not isinstance( - SizeTensor, tuple): + SizeTensor, tuple + ): SizeTensor = [SizeTensor] - return paddle._C_ops.bilinear_interp(x, OutSize, SizeTensor, Scale, - data_layout, out_d, out_h, out_w, - scale, interp_method, align_corners, - align_mode) - - -def bilinear_interp_np(input, - out_h, - out_w, - scale_w=0, - scale_h=0, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCHW'): + return paddle._C_ops.bilinear_interp( + x, + OutSize, + SizeTensor, + Scale, + data_layout, + out_d, + out_h, + out_w, + scale, + interp_method, + align_corners, + align_mode, + ) + + +def bilinear_interp_np( + input, + out_h, + out_w, + scale_w=0, + scale_h=0, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCHW', +): """bilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -73,7 +88,7 @@ def bilinear_interp_np(input, ratio_h = ratio_w = 0.0 if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: @@ -81,7 +96,7 @@ def bilinear_interp_np(input, else: ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -92,37 +107,40 @@ def bilinear_interp_np(input, out = np.zeros((batch_size, channel, out_h, out_w)) for i in range(out_h): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: h = int(ratio_h * (i + 0.5) - 0.5) else: h = int(ratio_h * i) h = max(0, h) hid = 1 if h < in_h - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_h = max(ratio_h * (i + 0.5) - 0.5, 0) h1lambda = idx_src_h - h else: h1lambda = ratio_h * i - h h2lambda = 1.0 - h1lambda for j in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (j + 0.5) - 0.5) else: w = int(ratio_w * j) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (j + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * j - w w2lambda = 1.0 - w1lambda - out[:, :, i, j] = h2lambda*(w2lambda*input[:, :, h, w] + - w1lambda*input[:, :, h, w+wid]) + \ - h1lambda*(w2lambda*input[:, :, h+hid, w] + - w1lambda*input[:, :, h+hid, w+wid]) + out[:, :, i, j] = h2lambda * ( + w2lambda * input[:, :, h, w] + + w1lambda * input[:, :, h, w + wid] + ) + h1lambda * ( + w2lambda * input[:, :, h + hid, w] + + w1lambda * input[:, :, h + hid, w + wid] + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC @@ -131,7 +149,6 @@ def bilinear_interp_np(input, class TestBilinearInterpOp(OpTest): - def setUp(self): self.python_api = bilinear_interp_test self.out_size = None @@ -151,7 +168,7 @@ def setUp(self): scale_w = 0 if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: scale_h = scale_w = float(self.scale) if isinstance(self.scale, list) and len(self.scale) == 1: scale_w = scale_h = self.scale[0] @@ -164,10 +181,18 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode, - self.data_layout) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -180,11 +205,11 @@ def setUp(self): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: self.scale = [self.scale] if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] @@ -209,7 +234,6 @@ def init_test_case(self): class TestBilinearInterpCase1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] @@ -221,7 +245,6 @@ def init_test_case(self): class TestBilinearInterpCase2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] @@ -233,7 +256,6 @@ def init_test_case(self): class TestBilinearInterpCase3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] @@ -245,7 +267,6 @@ def init_test_case(self): class TestBilinearInterpCase4(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] @@ -258,7 +279,6 @@ def init_test_case(self): class TestBilinearInterpCase5(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] @@ -271,7 +291,6 @@ def init_test_case(self): class TestBilinearInterpCase6(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] @@ -284,7 +303,6 @@ def init_test_case(self): class TestBilinearInterpCase7(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] @@ -296,7 +314,6 @@ def init_test_case(self): class TestBilinearInterpSame(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 32, 64] @@ -308,7 +325,6 @@ def init_test_case(self): class TestBilinearInterpActualShape(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] @@ -321,7 +337,6 @@ def init_test_case(self): class TestBilinearInterpDataLayout(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 5, 5, 3] @@ -335,15 +350,15 @@ def init_test_case(self): class TestBilinearInterpOpUint8(OpTest): - def setUp(self): self.python_api = bilinear_interp_test self.out_size = None self.actual_shape = None self.init_test_case() self.op_type = "bilinear_interp_v2" - input_np = np.random.randint(low=0, high=256, - size=self.input_shape).astype("uint8") + input_np = np.random.randint( + low=0, high=256, size=self.input_shape + ).astype("uint8") if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -360,9 +375,17 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -372,7 +395,7 @@ def setUp(self): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -384,9 +407,9 @@ def setUp(self): self.outputs = {'Out': output_np} def test_check_output(self): - self.check_output_with_place(place=core.CPUPlace(), - atol=1, - check_eager=True) + self.check_output_with_place( + place=core.CPUPlace(), atol=1, check_eager=True + ) def init_test_case(self): self.interp_method = 'bilinear' @@ -399,7 +422,6 @@ def init_test_case(self): class TestBilinearInterpCase1Uint8(TestBilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 32, 64] @@ -411,7 +433,6 @@ def init_test_case(self): class TestBilinearInterpCase2Uint8(TestBilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] @@ -424,52 +445,46 @@ def init_test_case(self): class TestBilinearInterpOtherMethod1(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 1 class TestBilinearInterpWithMethod2(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 0 class TestBilinearInterpWithMethod3(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = True self.align_mode = 0 class TestBilinearInterpScale1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 2. + self.scale = 2.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 1. + self.scale = 1.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -481,7 +496,6 @@ def init_test_case(self): class TestBilinearInterpScale4(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -493,7 +507,6 @@ def init_test_case(self): class TestBilinearInterpZero(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -505,7 +518,6 @@ def init_test_case(self): class TestBilinearInterpOp_attr_tensor(OpTest): - def setUp(self): self.python_api = bilinear_interp_test self.out_size = None @@ -544,8 +556,9 @@ def setUp(self): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h @@ -557,9 +570,16 @@ def setUp(self): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): @@ -580,7 +600,6 @@ def init_test_case(self): # out_size is a 1-D tensor class TestBilinearInterp_attr_tensor_Case1(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] @@ -593,7 +612,6 @@ def init_test_case(self): # scale is a 1-D tensor class TestBilinearInterp_attr_tensor_Case2(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] @@ -607,7 +625,6 @@ def init_test_case(self): # scale is a 1-D tensor class TestBilinearInterp_attr_tensor_Case3(TestBilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] @@ -620,23 +637,22 @@ def init_test_case(self): class TestBilinearInterpOpAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[2], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) out1 = fluid.layers.resize_bilinear(x, out_shape=[12, 12]) out2 = fluid.layers.resize_bilinear(x, out_shape=[12, dim]) out3 = fluid.layers.resize_bilinear(x, out_shape=shape_tensor) - out4 = fluid.layers.resize_bilinear(x, - out_shape=[4, 4], - actual_shape=actual_size) + out4 = fluid.layers.resize_bilinear( + x, out_shape=[4, 4], actual_shape=actual_size + ) out5 = fluid.layers.resize_bilinear(x, scale=scale_tensor) x_data = np.random.random((2, 3, 6, 6)).astype("float32") @@ -651,29 +667,30 @@ def test_case(self): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = bilinear_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=True) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = bilinear_interp_np( + x_data, out_h=12, out_w=12, align_corners=True + ) for res in results: np.testing.assert_allclose(res, expect_res, rtol=1e-05) class TestBilinearInterpOpAPI_dy(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -681,21 +698,19 @@ def test_case(self): with fluid.dygraph.guard(place): input_data = np.random.random((2, 3, 6, 6)).astype("float32") input_x = paddle.to_tensor(input_data) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - size=[12, 12], - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, size=[12, 12], mode="bilinear", align_corners=False + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) class TestBilinearInterpOpAPI_dy2(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -705,21 +720,19 @@ def test_case(self): size_np = np.array([12, 12]).astype("int64") input_x = paddle.to_tensor(input_data) size = paddle.to_tensor(size_np) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - size=size, - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, size=size, mode="bilinear", align_corners=False + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) class TestBilinearInterpOpAPI_dy3(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -729,21 +742,22 @@ def test_case(self): size_1 = np.array([12]).astype("int64") input_x = paddle.to_tensor(input_data) size = paddle.to_tensor(size_1) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - size=[size, size], - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, + size=[size, size], + mode="bilinear", + align_corners=False, + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) class TestBilinearInterpOpAPI_dy4(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -753,21 +767,22 @@ def test_case(self): scale_np = np.array([2, 2]).astype("int64") input_x = paddle.to_tensor(input_data) scale = paddle.to_tensor(scale_np) - expect_res = bilinear_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - scale_factor=scale, - mode="bilinear", - align_corners=False) + expect_res = bilinear_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, + scale_factor=scale, + mode="bilinear", + align_corners=False, + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestBilinearInterpOpForFloat16(unittest.TestCase): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 5] @@ -781,12 +796,14 @@ def check_main(self, x_np, dtype): x_np = x_np.astype(dtype) x = paddle.to_tensor(x_np) x.stop_gradient = False - y = interpolate(x, - size=self.out_size.tolist(), - mode=self.interp_method, - align_mode=self.align_mode, - align_corners=self.align_corners, - data_format=self.data_layout) + y = interpolate( + x, + size=self.out_size.tolist(), + mode=self.interp_method, + align_mode=self.align_mode, + align_corners=self.align_corners, + data_format=self.data_layout, + ) x_g = paddle.grad(y, x) y_np = y[0].numpy().astype('float32') x_g_np = x_g[0].numpy().astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py b/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py index 6ffbbfdb99911..79ef2ffabbc1c 100644 --- a/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py +++ b/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py @@ -20,15 +20,15 @@ class TestDygraphBilinearTensorProductAPIError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): - layer = fluid.dygraph.nn.BilinearTensorProduct(input1_dim=5, - input2_dim=4, - output_dim=1000) + layer = fluid.dygraph.nn.BilinearTensorProduct( + input1_dim=5, input2_dim=4, output_dim=1000 + ) # the input must be Variable. - x0 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x0 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, layer, x0) # the input dtype must be float32 or float64 x1 = fluid.data(name='x1', shape=[-1, 5], dtype="float16") @@ -37,7 +37,6 @@ def test_errors(self): class TestBilinearTensorProductOp(OpTest): - def setUp(self): self.op_type = "bilinear_tensor_product" self.python_api = paddle.nn.functional.bilinear diff --git a/python/paddle/fluid/tests/unittests/test_bincount_op.py b/python/paddle/fluid/tests/unittests/test_bincount_op.py index e191b09fe4291..c5e39f1b13fa3 100644 --- a/python/paddle/fluid/tests/unittests/test_bincount_op.py +++ b/python/paddle/fluid/tests/unittests/test_bincount_op.py @@ -42,16 +42,17 @@ def test_static_graph(self): exe.run(startup_program) img = np.array([0, 1, 1, 3, 2, 1, 7]).astype(np.int64) w = np.array([0, 1, 1, 2, 2, 1, 0]).astype(np.int64) - res = exe.run(train_program, - feed={ - 'input': img, - 'weights': w - }, - fetch_list=[output]) + res = exe.run( + train_program, + feed={'input': img, 'weights': w}, + fetch_list=[output], + ) actual = np.array(res[0]) expected = np.bincount(img, weights=w) - self.assertTrue((actual == expected).all(), - msg='bincount output is wrong, out =' + str(actual)) + self.assertTrue( + (actual == expected).all(), + msg='bincount output is wrong, out =' + str(actual), + ) def test_dygraph(self): with fluid.dygraph.guard(): @@ -61,7 +62,8 @@ def test_dygraph(self): expected = np.bincount(inputs) self.assertTrue( (actual.numpy() == expected).all(), - msg='bincount output is wrong, out =' + str(actual.numpy())) + msg='bincount output is wrong, out =' + str(actual.numpy()), + ) class TestBincountOpError(unittest.TestCase): @@ -112,7 +114,7 @@ def test_input_type_errors(self): """Test input tensor should only contain non-negative ints.""" def net_func(): - input_value = paddle.to_tensor([1., 2., 3., 4., 5.]) + input_value = paddle.to_tensor([1.0, 2.0, 3.0, 4.0, 5.0]) paddle.bincount(input_value) with self.assertRaises(TypeError): @@ -161,12 +163,13 @@ def setUp(self): def init_test_case(self): self.minlength = 0 - self.np_weights = np.random.randint(low=0, high=20, - size=10).astype(np.float32) + self.np_weights = np.random.randint(low=0, high=20, size=10).astype( + np.float32 + ) self.np_input = np.random.randint(low=0, high=20, size=10) - self.Out = np.bincount(self.np_input, - weights=self.np_weights, - minlength=self.minlength).astype(np.float32) + self.Out = np.bincount( + self.np_input, weights=self.np_weights, minlength=self.minlength + ).astype(np.float32) class TestCase2(TestBincountOp): @@ -183,9 +186,9 @@ def init_test_case(self): self.minlength = 0 self.np_weights = np.random.randint(low=0, high=20, size=10) self.np_input = np.random.randint(low=0, high=20, size=10) - self.Out = np.bincount(self.np_input, - weights=self.np_weights, - minlength=self.minlength) + self.Out = np.bincount( + self.np_input, weights=self.np_weights, minlength=self.minlength + ) class TestCase3(TestBincountOp): @@ -200,8 +203,9 @@ class TestCase4(TestBincountOp): # with input(INT32) def init_test_case(self): self.minlength = 0 - self.np_input = np.random.randint(low=0, high=20, - size=10).astype(np.int32) + self.np_input = np.random.randint(low=0, high=20, size=10).astype( + np.int32 + ) self.Out = np.bincount(self.np_input, minlength=self.minlength) @@ -214,23 +218,27 @@ def init_test_case(self): class TestTensorMinlength(unittest.TestCase): - def setUp(self): paddle.disable_static() paddle.seed(2022) self.temp_dir = tempfile.TemporaryDirectory() - self.save_path = os.path.join(self.temp_dir.name, - 'tensor_minlength_bincount') - self.place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + self.save_path = os.path.join( + self.temp_dir.name, 'tensor_minlength_bincount' + ) + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) def test_dygraph(self): paddle.disable_static() x = np.random.randint(0, 10, [20]) minlength = 2 np_out = np.bincount(x, minlength=minlength) - pd_out = paddle.bincount(paddle.to_tensor(x), - minlength=paddle.to_tensor([2], dtype='int32')) + pd_out = paddle.bincount( + paddle.to_tensor(x), minlength=paddle.to_tensor([2], dtype='int32') + ) np.testing.assert_allclose(np_out, pd_out.numpy()) def test_static_and_infer(self): @@ -245,8 +253,9 @@ def test_static_and_infer(self): linear_out = linear(x) relu_out = paddle.nn.functional.relu(linear_out) minlength = paddle.full([1], 3, dtype='int32') - out = paddle.bincount(paddle.cast(relu_out, 'int32'), - minlength=minlength) + out = paddle.bincount( + paddle.cast(relu_out, 'int32'), minlength=minlength + ) exe = paddle.static.Executor(self.place) exe.run(starup_prog) @@ -254,8 +263,9 @@ def test_static_and_infer(self): # run infer paddle.static.save_inference_model(self.save_path, [x], [out], exe) - config = paddle_infer.Config(self.save_path + '.pdmodel', - self.save_path + '.pdiparams') + config = paddle_infer.Config( + self.save_path + '.pdmodel', self.save_path + '.pdiparams' + ) if paddle.is_compiled_with_cuda(): config.enable_use_gpu(100, 0) else: diff --git a/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py b/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py index 39bed077ebae2..0412c04d8ca0d 100644 --- a/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py +++ b/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np @@ -34,7 +34,7 @@ def bipartite_match(distance, match_indices, match_dist): match_sorted = sorted(match_pair, key=lambda tup: tup[2], reverse=True) - row_indices = -1 * np.ones((row, ), dtype=np.int_) + row_indices = -1 * np.ones((row,), dtype=np.int_) idx = 0 for i, j, dist in match_sorted: @@ -72,18 +72,25 @@ def batch_bipartite_match(distance, lod, match_type=None, dist_threshold=None): match_dist = np.zeros((n, m), dtype=np.float32) cur_offset = 0 for i in range(n): - if lod[i] == 0: continue - bipartite_match(distance[cur_offset:(cur_offset + lod[i]), :], - match_indices[i, :], match_dist[i, :]) + if lod[i] == 0: + continue + bipartite_match( + distance[cur_offset : (cur_offset + lod[i]), :], + match_indices[i, :], + match_dist[i, :], + ) if match_type == 'per_prediction': - argmax_match(distance[cur_offset:(cur_offset + lod[i]), :], - match_indices[i, :], match_dist[i, :], dist_threshold) + argmax_match( + distance[cur_offset : (cur_offset + lod[i]), :], + match_indices[i, :], + match_dist[i, :], + dist_threshold, + ) cur_offset += lod[i] return match_indices, match_dist class TestBipartiteMatchOpWithLoD(OpTest): - def setUp(self): self.op_type = 'bipartite_match' lod = [[5, 6, 12]] @@ -101,7 +108,6 @@ def test_check_output(self): class TestBipartiteMatchOpWithoutLoD(OpTest): - def setUp(self): self.op_type = 'bipartite_match' lod = [[8]] @@ -119,7 +125,6 @@ def test_check_output(self): class TestBipartiteMatchOpWithoutLoDLargeScaleInput(OpTest): - def setUp(self): self.op_type = 'bipartite_match' lod = [[300]] @@ -137,13 +142,13 @@ def test_check_output(self): class TestBipartiteMatchOpWithPerPredictionType(OpTest): - def setUp(self): self.op_type = 'bipartite_match' lod = [[5, 6, 12]] dist = np.random.random((23, 237)).astype('float32') match_indices, match_dist = batch_bipartite_match( - dist, lod[0], 'per_prediction', 0.5) + dist, lod[0], 'per_prediction', 0.5 + ) self.inputs = {'DistMat': (dist, lod)} self.outputs = { @@ -160,7 +165,6 @@ def test_check_output(self): class TestBipartiteMatchOpWithEmptyLoD(OpTest): - def setUp(self): self.op_type = 'bipartite_match' lod = [[5, 6, 0, 12]] diff --git a/python/paddle/fluid/tests/unittests/test_bitwise_op.py b/python/paddle/fluid/tests/unittests/test_bitwise_op.py index c387555ccda64..6a7b039380b9e 100644 --- a/python/paddle/fluid/tests/unittests/test_bitwise_op.py +++ b/python/paddle/fluid/tests/unittests/test_bitwise_op.py @@ -22,21 +22,18 @@ ################## TEST OP: BitwiseAnd ################## class TestBitwiseAnd(OpTest): - def setUp(self): self.op_type = "bitwise_and" self.init_dtype() self.init_shape() self.init_bound() - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.bitwise_and(x, y) self.inputs = {'X': x, 'Y': y} @@ -61,7 +58,6 @@ def init_bound(self): class TestBitwiseAndUInt8(TestBitwiseAnd): - def init_dtype(self): self.dtype = np.uint8 @@ -71,7 +67,6 @@ def init_bound(self): class TestBitwiseAndInt8(TestBitwiseAnd): - def init_dtype(self): self.dtype = np.int8 @@ -81,7 +76,6 @@ def init_shape(self): class TestBitwiseAndInt16(TestBitwiseAnd): - def init_dtype(self): self.dtype = np.int16 @@ -91,7 +85,6 @@ def init_shape(self): class TestBitwiseAndInt64(TestBitwiseAnd): - def init_dtype(self): self.dtype = np.int64 @@ -101,7 +94,6 @@ def init_shape(self): class TestBitwiseAndBool(TestBitwiseAnd): - def setUp(self): self.op_type = "bitwise_and" self.init_shape() @@ -116,21 +108,18 @@ def setUp(self): ################## TEST OP: BitwiseOr ################## class TestBitwiseOr(OpTest): - def setUp(self): self.op_type = "bitwise_or" self.init_dtype() self.init_shape() self.init_bound() - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.bitwise_or(x, y) self.inputs = {'X': x, 'Y': y} @@ -155,7 +144,6 @@ def init_bound(self): class TestBitwiseOrUInt8(TestBitwiseOr): - def init_dtype(self): self.dtype = np.uint8 @@ -165,7 +153,6 @@ def init_bound(self): class TestBitwiseOrInt8(TestBitwiseOr): - def init_dtype(self): self.dtype = np.int8 @@ -175,7 +162,6 @@ def init_shape(self): class TestBitwiseOrInt16(TestBitwiseOr): - def init_dtype(self): self.dtype = np.int16 @@ -185,7 +171,6 @@ def init_shape(self): class TestBitwiseOrInt64(TestBitwiseOr): - def init_dtype(self): self.dtype = np.int64 @@ -195,7 +180,6 @@ def init_shape(self): class TestBitwiseOrBool(TestBitwiseOr): - def setUp(self): self.op_type = "bitwise_or" self.init_shape() @@ -210,21 +194,18 @@ def setUp(self): ################## TEST OP: BitwiseXor ################## class TestBitwiseXor(OpTest): - def setUp(self): self.op_type = "bitwise_xor" self.init_dtype() self.init_shape() self.init_bound() - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.bitwise_xor(x, y) self.inputs = {'X': x, 'Y': y} @@ -249,7 +230,6 @@ def init_bound(self): class TestBitwiseXorUInt8(TestBitwiseXor): - def init_dtype(self): self.dtype = np.uint8 @@ -259,7 +239,6 @@ def init_bound(self): class TestBitwiseXorInt8(TestBitwiseXor): - def init_dtype(self): self.dtype = np.int8 @@ -269,7 +248,6 @@ def init_shape(self): class TestBitwiseXorInt16(TestBitwiseXor): - def init_dtype(self): self.dtype = np.int16 @@ -279,7 +257,6 @@ def init_shape(self): class TestBitwiseXorInt64(TestBitwiseXor): - def init_dtype(self): self.dtype = np.int64 @@ -289,7 +266,6 @@ def init_shape(self): class TestBitwiseXorBool(TestBitwiseXor): - def setUp(self): self.op_type = "bitwise_xor" self.init_shape() @@ -304,17 +280,15 @@ def setUp(self): ################## TEST OP: BitwiseNot ################## class TestBitwiseNot(OpTest): - def setUp(self): self.op_type = "bitwise_not" self.init_dtype() self.init_shape() self.init_bound() - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) out = np.bitwise_not(x) self.inputs = {'X': x} @@ -338,7 +312,6 @@ def init_bound(self): class TestBitwiseNotUInt8(TestBitwiseNot): - def init_dtype(self): self.dtype = np.uint8 @@ -348,7 +321,6 @@ def init_bound(self): class TestBitwiseNotInt8(TestBitwiseNot): - def init_dtype(self): self.dtype = np.int8 @@ -357,7 +329,6 @@ def init_shape(self): class TestBitwiseNotInt16(TestBitwiseNot): - def init_dtype(self): self.dtype = np.int16 @@ -367,7 +338,6 @@ def init_shape(self): class TestBitwiseNotInt64(TestBitwiseNot): - def init_dtype(self): self.dtype = np.int64 @@ -376,7 +346,6 @@ def init_shape(self): class TestBitwiseNotBool(TestBitwiseNot): - def setUp(self): self.op_type = "bitwise_not" self.init_shape() diff --git a/python/paddle/fluid/tests/unittests/test_block_rename_var.py b/python/paddle/fluid/tests/unittests/test_block_rename_var.py index 322cb8bc4471f..02908052278eb 100644 --- a/python/paddle/fluid/tests/unittests/test_block_rename_var.py +++ b/python/paddle/fluid/tests/unittests/test_block_rename_var.py @@ -18,16 +18,15 @@ class TestBlockRenameVar(unittest.TestCase): - def setUp(self): self.program = paddle.static.Program() self.block = self.program.current_block() - self.var = self.block.create_var(name="X", - shape=[-1, 23, 48], - dtype='float32') - self.op = self.block.append_op(type="abs", - inputs={"X": [self.var]}, - outputs={"Out": [self.var]}) + self.var = self.block.create_var( + name="X", shape=[-1, 23, 48], dtype='float32' + ) + self.op = self.block.append_op( + type="abs", inputs={"X": [self.var]}, outputs={"Out": [self.var]} + ) self.new_var_name = self.get_new_var_name() def get_new_var_name(self): @@ -35,19 +34,20 @@ def get_new_var_name(self): def test_rename_var(self): self.block._rename_var(self.var.name, self.new_var_name) - new_var_name_str = self.new_var_name if isinstance( - self.new_var_name, str) else self.new_var_name.decode() + new_var_name_str = ( + self.new_var_name + if isinstance(self.new_var_name, str) + else self.new_var_name.decode() + ) self.assertTrue(new_var_name_str in self.block.vars) class TestBlockRenameVarStrCase2(TestBlockRenameVar): - def get_new_var_name(self): return "ABC" class TestBlockRenameVarBytes(TestBlockRenameVar): - def get_new_var_name(self): return b"Y" diff --git a/python/paddle/fluid/tests/unittests/test_bmm_op.py b/python/paddle/fluid/tests/unittests/test_bmm_op.py index 44a6be83b32ee..5c99a2e62cc59 100644 --- a/python/paddle/fluid/tests/unittests/test_bmm_op.py +++ b/python/paddle/fluid/tests/unittests/test_bmm_op.py @@ -20,7 +20,6 @@ class TestBmmOp(OpTest): - def setUp(self): self.op_type = "bmm" self.python_api = paddle.tensor.bmm @@ -38,36 +37,40 @@ def test_checkout_grad(self): class API_TestBmm(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): - data1 = fluid.layers.data('data1', - shape=[-1, 3, 4], - dtype='float64') - data2 = fluid.layers.data('data2', - shape=[-1, 4, 5], - dtype='float64') + data1 = fluid.layers.data( + 'data1', shape=[-1, 3, 4], dtype='float64' + ) + data2 = fluid.layers.data( + 'data2', shape=[-1, 4, 5], dtype='float64' + ) result_bmm = paddle.bmm(data1, data2) place = fluid.CPUPlace() exe = fluid.Executor(place) input1 = np.random.random([10, 3, 4]).astype('float64') input2 = np.random.random([10, 4, 5]).astype('float64') - result, = exe.run(feed={ - "data1": input1, - "data2": input2 - }, - fetch_list=[result_bmm]) + (result,) = exe.run( + feed={"data1": input1, "data2": input2}, fetch_list=[result_bmm] + ) expected_result = np.matmul(input1, input2) np.testing.assert_allclose(expected_result, result, rtol=1e-05) class API_TestDygraphBmm(unittest.TestCase): - def test_out(self): - input1 = np.array([[[1.0, 1.0, 1.0], [2.0, 2.0, 2.0]], - [[3.0, 3.0, 3.0], [4.0, 4.0, 4.0]]]) - input2 = np.array([[[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]], - [[4.0, 4.0], [5.0, 5.0], [6.0, 6.0]]]) + input1 = np.array( + [ + [[1.0, 1.0, 1.0], [2.0, 2.0, 2.0]], + [[3.0, 3.0, 3.0], [4.0, 4.0, 4.0]], + ] + ) + input2 = np.array( + [ + [[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]], + [[4.0, 4.0], [5.0, 5.0], [6.0, 6.0]], + ] + ) with fluid.dygraph.guard(): x = fluid.dygraph.to_variable(input1) y = fluid.dygraph.to_variable(input2) @@ -78,7 +81,6 @@ def test_out(self): class TestBmmAPIError(unittest.TestCase): - def test_api_error(self): x_data = np.arange(24, dtype='float32').reshape((2, 3, 4)) y_data = np.arange(16, dtype='float32').reshape((2, 4, 2)) diff --git a/python/paddle/fluid/tests/unittests/test_box_clip_op.py b/python/paddle/fluid/tests/unittests/test_box_clip_op.py index e3c9c8e6eb4d6..2ee356976f6c6 100644 --- a/python/paddle/fluid/tests/unittests/test_box_clip_op.py +++ b/python/paddle/fluid/tests/unittests/test_box_clip_op.py @@ -20,14 +20,18 @@ def box_clip(input_box, im_info, output_box): im_w = round(im_info[1] / im_info[2]) im_h = round(im_info[0] / im_info[2]) - output_box[:, :, 0] = np.maximum(np.minimum(input_box[:, :, 0], im_w - 1), - 0) - output_box[:, :, 1] = np.maximum(np.minimum(input_box[:, :, 1], im_h - 1), - 0) - output_box[:, :, 2] = np.maximum(np.minimum(input_box[:, :, 2], im_w - 1), - 0) - output_box[:, :, 3] = np.maximum(np.minimum(input_box[:, :, 3], im_h - 1), - 0) + output_box[:, :, 0] = np.maximum( + np.minimum(input_box[:, :, 0], im_w - 1), 0 + ) + output_box[:, :, 1] = np.maximum( + np.minimum(input_box[:, :, 1], im_h - 1), 0 + ) + output_box[:, :, 2] = np.maximum( + np.minimum(input_box[:, :, 2], im_w - 1), 0 + ) + output_box[:, :, 3] = np.maximum( + np.minimum(input_box[:, :, 3], im_h - 1), 0 + ) def batch_box_clip(input_boxes, im_info, lod): @@ -36,15 +40,16 @@ def batch_box_clip(input_boxes, im_info, lod): output_boxes = np.zeros((n, m, 4), dtype=np.float32) cur_offset = 0 for i in range(len(lod)): - box_clip(input_boxes[cur_offset:(cur_offset + lod[i]), :, :], - im_info[i, :], - output_boxes[cur_offset:(cur_offset + lod[i]), :, :]) + box_clip( + input_boxes[cur_offset : (cur_offset + lod[i]), :, :], + im_info[i, :], + output_boxes[cur_offset : (cur_offset + lod[i]), :, :], + ) cur_offset += lod[i] return output_boxes class TestBoxClipOp(OpTest): - def test_check_output(self): self.check_output() @@ -52,7 +57,7 @@ def setUp(self): self.op_type = "box_clip" lod = [[1, 2, 3]] input_boxes = np.random.random((6, 10, 4)) * 5 - im_info = np.array([[5, 8, 1.], [6, 6, 1.], [7, 5, 1.]]) + im_info = np.array([[5, 8, 1.0], [6, 6, 1.0], [7, 5, 1.0]]) output_boxes = batch_box_clip(input_boxes, im_info, lod[0]) self.inputs = { diff --git a/python/paddle/fluid/tests/unittests/test_box_coder_op.py b/python/paddle/fluid/tests/unittests/test_box_coder_op.py index 0e70f9904b50a..6813a88070d21 100644 --- a/python/paddle/fluid/tests/unittests/test_box_coder_op.py +++ b/python/paddle/fluid/tests/unittests/test_box_coder_op.py @@ -32,9 +32,11 @@ def box_decoder(t_box, p_box, pb_v, output_box, norm, axis=0): pb_y = pb_y.reshape(shape) if pb_v.ndim == 2: - var_shape = (1, pb_v.shape[0], - pb_v.shape[1]) if axis == 0 else (pb_v.shape[0], 1, - pb_v.shape[1]) + var_shape = ( + (1, pb_v.shape[0], pb_v.shape[1]) + if axis == 0 + else (pb_v.shape[0], 1, pb_v.shape[1]) + ) pb_v = pb_v.reshape(var_shape) if pb_v.ndim == 1: tb_x = pb_v[0] * t_box[:, :, 0] * pb_w + pb_x @@ -90,18 +92,21 @@ def batch_box_coder(p_box, pb_v, t_box, lod, code_type, norm, axis=0): output_box = np.zeros((n, m, 4), dtype=np.float32) cur_offset = 0 for i in range(len(lod)): - if (code_type == "EncodeCenterSize"): - box_encoder(t_box[cur_offset:(cur_offset + lod[i]), :], p_box, pb_v, - output_box[cur_offset:(cur_offset + lod[i]), :, :], - norm) - elif (code_type == "DecodeCenterSize"): + if code_type == "EncodeCenterSize": + box_encoder( + t_box[cur_offset : (cur_offset + lod[i]), :], + p_box, + pb_v, + output_box[cur_offset : (cur_offset + lod[i]), :, :], + norm, + ) + elif code_type == "DecodeCenterSize": box_decoder(t_box, p_box, pb_v, output_box, norm, axis) cur_offset += lod[i] return output_box class TestBoxCoderOp(OpTest): - def test_check_output(self): self.check_output(check_eager=True) @@ -114,8 +119,14 @@ def setUp(self): target_box = np.random.random((20, 81, 4)).astype('float32') code_type = "DecodeCenterSize" box_normalized = False - output_box = batch_box_coder(prior_box, prior_box_var, target_box, - lod[0], code_type, box_normalized) + output_box = batch_box_coder( + prior_box, + prior_box_var, + target_box, + lod[0], + code_type, + box_normalized, + ) self.inputs = { 'PriorBox': prior_box, 'PriorBoxVar': prior_box_var, @@ -123,13 +134,12 @@ def setUp(self): } self.attrs = { 'code_type': 'decode_center_size', - 'box_normalized': False + 'box_normalized': False, } self.outputs = {'OutputBox': output_box} class TestBoxCoderOpWithoutBoxVar(OpTest): - def test_check_output(self): self.check_output(check_eager=True) @@ -142,8 +152,14 @@ def setUp(self): target_box = np.random.random((20, 81, 4)).astype('float32') code_type = "DecodeCenterSize" box_normalized = False - output_box = batch_box_coder(prior_box, prior_box_var, target_box, - lod[0], code_type, box_normalized) + output_box = batch_box_coder( + prior_box, + prior_box_var, + target_box, + lod[0], + code_type, + box_normalized, + ) self.inputs = { 'PriorBox': prior_box, @@ -152,13 +168,12 @@ def setUp(self): } self.attrs = { 'code_type': 'decode_center_size', - 'box_normalized': False + 'box_normalized': False, } self.outputs = {'OutputBox': output_box} class TestBoxCoderOpWithLoD(OpTest): - def test_check_output(self): self.check_output(check_eager=True) @@ -171,8 +186,14 @@ def setUp(self): target_box = np.random.random((50, 4)).astype('float32') code_type = "EncodeCenterSize" box_normalized = True - output_box = batch_box_coder(prior_box, prior_box_var, target_box, - lod[0], code_type, box_normalized) + output_box = batch_box_coder( + prior_box, + prior_box_var, + target_box, + lod[0], + code_type, + box_normalized, + ) self.inputs = { 'PriorBox': prior_box, @@ -184,7 +205,6 @@ def setUp(self): class TestBoxCoderOpWithAxis(OpTest): - def test_check_output(self): self.check_output(check_eager=True) @@ -198,8 +218,15 @@ def setUp(self): code_type = "DecodeCenterSize" box_normalized = False axis = 1 - output_box = batch_box_coder(prior_box, prior_box_var, target_box, - lod[0], code_type, box_normalized, axis) + output_box = batch_box_coder( + prior_box, + prior_box_var, + target_box, + lod[0], + code_type, + box_normalized, + axis, + ) self.inputs = { 'PriorBox': prior_box, @@ -209,13 +236,12 @@ def setUp(self): self.attrs = { 'code_type': 'decode_center_size', 'box_normalized': False, - 'axis': axis + 'axis': axis, } self.outputs = {'OutputBox': output_box} class TestBoxCoderOpWithVariance(OpTest): - def test_check_output(self): self.check_output() @@ -228,8 +254,15 @@ def setUp(self): code_type = "DecodeCenterSize" box_normalized = False axis = 1 - output_box = batch_box_coder(prior_box, prior_box_var, target_box, - lod[0], code_type, box_normalized, axis) + output_box = batch_box_coder( + prior_box, + prior_box_var, + target_box, + lod[0], + code_type, + box_normalized, + axis, + ) self.inputs = { 'PriorBox': prior_box, @@ -239,13 +272,12 @@ def setUp(self): 'code_type': 'decode_center_size', 'box_normalized': False, 'variance': prior_box_var.astype(np.float64).flatten(), - 'axis': axis + 'axis': axis, } self.outputs = {'OutputBox': output_box} class TestBoxCoderOpWithVarianceDygraphAPI(unittest.TestCase): - def setUp(self): self.lod = [[1, 1, 1, 1, 1]] self.prior_box = np.random.random((30, 4)).astype('float32') @@ -254,16 +286,20 @@ def setUp(self): self.code_type = "DecodeCenterSize" self.box_normalized = False self.axis = 1 - self.output_ref = batch_box_coder(self.prior_box, self.prior_box_var, - self.target_box, self.lod[0], - self.code_type, self.box_normalized, - self.axis) + self.output_ref = batch_box_coder( + self.prior_box, + self.prior_box_var, + self.target_box, + self.lod[0], + self.code_type, + self.box_normalized, + self.axis, + ) self.place = [paddle.CPUPlace()] if core.is_compiled_with_cuda(): self.place.append(paddle.CUDAPlace(0)) def test_dygraph_api(self): - def run(place): paddle.disable_static(place) output_box = paddle.fluid.layers.box_coder( @@ -272,10 +308,11 @@ def run(place): paddle.to_tensor(self.target_box), "decode_center_size", self.box_normalized, - axis=self.axis) - np.testing.assert_allclose(np.sum(self.output_ref), - np.sum(output_box.numpy()), - rtol=1e-05) + axis=self.axis, + ) + np.testing.assert_allclose( + np.sum(self.output_ref), np.sum(output_box.numpy()), rtol=1e-05 + ) paddle.enable_static() for place in self.place: diff --git a/python/paddle/fluid/tests/unittests/test_box_decoder_and_assign_op.py b/python/paddle/fluid/tests/unittests/test_box_decoder_and_assign_op.py index 41158533a62d8..0b826e362e7bd 100644 --- a/python/paddle/fluid/tests/unittests/test_box_decoder_and_assign_op.py +++ b/python/paddle/fluid/tests/unittests/test_box_decoder_and_assign_op.py @@ -60,7 +60,6 @@ def box_decoder_and_assign(deltas, weights, boxes, box_score, box_clip): class TestBoxDecoderAndAssignOpWithLoD(OpTest): - def test_check_output(self): self.check_output() @@ -74,7 +73,8 @@ def setUp(self): box_score = np.random.random((20, num_classes)).astype('float32') box_clip = 4.135 output_box, output_assign_box = box_decoder_and_assign( - target_box, prior_box_var, prior_box, box_score, box_clip) + target_box, prior_box_var, prior_box, box_score, box_clip + ) self.inputs = { 'PriorBox': (prior_box, lod), @@ -85,7 +85,7 @@ def setUp(self): self.attrs = {'box_clip': box_clip} self.outputs = { 'DecodeBox': output_box, - 'OutputAssignBox': output_assign_box + 'OutputAssignBox': output_assign_box, } diff --git a/python/paddle/fluid/tests/unittests/test_boxps.py b/python/paddle/fluid/tests/unittests/test_boxps.py index a3d3607fd8e0f..a51f86a7b2b72 100644 --- a/python/paddle/fluid/tests/unittests/test_boxps.py +++ b/python/paddle/fluid/tests/unittests/test_boxps.py @@ -21,7 +21,7 @@ class TestTranspile(unittest.TestCase): - """ TestCases for BoxPS Preload """ + """TestCases for BoxPS Preload""" def get_transpile(self, mode, trainers="127.0.0.1:6174"): config = fluid.DistributeTranspilerConfig() @@ -34,44 +34,52 @@ def test_transpile(self): main_program = fluid.Program() startup_program = fluid.Program() t = self.get_transpile("single_process_multi_thread") - t.transpile(trainer_id=0, - startup_program=startup_program, - trainers="127.0.0.1:6174", - program=main_program) + t.transpile( + trainer_id=0, + startup_program=startup_program, + trainers="127.0.0.1:6174", + program=main_program, + ) t = self.get_transpile("grad_allreduce") try: - t.transpile(trainer_id=0, - startup_program=startup_program, - trainers="127.0.0.1:6174", - program=main_program) + t.transpile( + trainer_id=0, + startup_program=startup_program, + trainers="127.0.0.1:6174", + program=main_program, + ) except ValueError as e: print(e) def test_single_trainers(self): transpiler = collective.GradAllReduce(0) try: - transpiler.transpile(startup_program=fluid.Program(), - main_program=fluid.Program(), - rank=1, - endpoints="127.0.0.1:6174", - current_endpoint="127.0.0.1:6174", - wait_port="6174") + transpiler.transpile( + startup_program=fluid.Program(), + main_program=fluid.Program(), + rank=1, + endpoints="127.0.0.1:6174", + current_endpoint="127.0.0.1:6174", + wait_port="6174", + ) except ValueError as e: print(e) transpiler = collective.LocalSGD(0) try: - transpiler.transpile(startup_program=fluid.Program(), - main_program=fluid.Program(), - rank=1, - endpoints="127.0.0.1:6174", - current_endpoint="127.0.0.1:6174", - wait_port="6174") + transpiler.transpile( + startup_program=fluid.Program(), + main_program=fluid.Program(), + rank=1, + endpoints="127.0.0.1:6174", + current_endpoint="127.0.0.1:6174", + wait_port="6174", + ) except ValueError as e: print(e) class TestRunCmd(unittest.TestCase): - """ TestCases for run_cmd""" + """TestCases for run_cmd""" def test_run_cmd(self): ret1 = int(core.run_cmd("ls; echo $?").strip().split('\n')[-1]) @@ -81,20 +89,18 @@ def test_run_cmd(self): class TestPullBoxSparseOP(unittest.TestCase): - """ TestCases for _pull_box_sparse op""" + """TestCases for _pull_box_sparse op""" def test_pull_box_sparse_op(self): paddle.enable_static() program = fluid.Program() with fluid.program_guard(program): - x = fluid.layers.data(name='x', - shape=[1], - dtype='int64', - lod_level=0) - y = fluid.layers.data(name='y', - shape=[1], - dtype='int64', - lod_level=0) + x = fluid.layers.data( + name='x', shape=[1], dtype='int64', lod_level=0 + ) + y = fluid.layers.data( + name='y', shape=[1], dtype='int64', lod_level=0 + ) emb_x, emb_y = _pull_box_sparse([x, y], size=1) diff --git a/python/paddle/fluid/tests/unittests/test_bpr_loss_op.py b/python/paddle/fluid/tests/unittests/test_bpr_loss_op.py index d527f4579ed73..5f8953aaea363 100644 --- a/python/paddle/fluid/tests/unittests/test_bpr_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_bpr_loss_op.py @@ -18,8 +18,7 @@ class TestBprLossOp1(OpTest): - """Test BprLoss with discrete one-hot labels. - """ + """Test BprLoss with discrete one-hot labels.""" def setUp(self): self.op_type = "bpr_loss" @@ -33,7 +32,7 @@ def setUp(self): for j in range(class_num): if j == label[i][0]: continue - sum += (-np.log(1.0 + np.exp(X[i][j] - X[i][label[i][0]]))) + sum += -np.log(1.0 + np.exp(X[i][j] - X[i][label[i][0]])) bpr_loss_result.append(-sum / (class_num - 1)) bpr_loss = np.asmatrix([[x] for x in bpr_loss_result], dtype="float64") self.inputs = {"X": X, "Label": label} diff --git a/python/paddle/fluid/tests/unittests/test_broadcast_error.py b/python/paddle/fluid/tests/unittests/test_broadcast_error.py index bc1d026815856..517de67fd6ddd 100644 --- a/python/paddle/fluid/tests/unittests/test_broadcast_error.py +++ b/python/paddle/fluid/tests/unittests/test_broadcast_error.py @@ -19,7 +19,6 @@ class TestBroadcastOpCpu(OpTest): - def setUp(self): self.op_type = "broadcast" input = np.random.random((100, 2)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_broadcast_shape.py b/python/paddle/fluid/tests/unittests/test_broadcast_shape.py index fedcb9bb465d5..1afb046b68b46 100644 --- a/python/paddle/fluid/tests/unittests/test_broadcast_shape.py +++ b/python/paddle/fluid/tests/unittests/test_broadcast_shape.py @@ -17,18 +17,19 @@ class TestBroadcastShape(unittest.TestCase): - def test_result(self): shape = paddle.broadcast_shape([2, 1, 3], [1, 3, 1]) self.assertEqual(shape, [2, 3, 3]) shape = paddle.broadcast_shape( - [-1, 1, 3], [1, 3, 1]) #support compile time infershape + [-1, 1, 3], [1, 3, 1] + ) # support compile time infershape self.assertEqual(shape, [-1, 3, 3]) def test_error(self): - self.assertRaises(ValueError, paddle.broadcast_shape, [2, 1, 3], - [3, 3, 1]) + self.assertRaises( + ValueError, paddle.broadcast_shape, [2, 1, 3], [3, 3, 1] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_broadcast_tensors_op.py b/python/paddle/fluid/tests/unittests/test_broadcast_tensors_op.py index 3c262048561f5..3872dadebd82d 100644 --- a/python/paddle/fluid/tests/unittests/test_broadcast_tensors_op.py +++ b/python/paddle/fluid/tests/unittests/test_broadcast_tensors_op.py @@ -80,7 +80,6 @@ def gen_mixed_tensors_test(dtype): class TestCPUBroadcastTensorsOp(OpTest): - def set_place(self): self.place = core.CPUPlace() @@ -92,7 +91,9 @@ def setUp(self): self.use_mkldnn = False self.attrs = {'use_mkldnn': self.use_mkldnn} self.test_gen_func_list = [ - gen_rank_diff_test, gen_no_broadcast_test, gen_mixed_tensors_test + gen_rank_diff_test, + gen_no_broadcast_test, + gen_mixed_tensors_test, ] self.set_place() self.set_dtypes() @@ -117,35 +118,38 @@ def run_triple_in_test(self, test_func, args): test_func(**args) def test_check_output(self): - self.run_dual_test(self.check_output_with_place, { - "place": self.place, - "atol": 1e-1, - "check_eager": True - }) + self.run_dual_test( + self.check_output_with_place, + {"place": self.place, "atol": 1e-1, "check_eager": True}, + ) def test_check_grad_normal(self): self.run_dual_test( - self.check_grad_with_place, { + self.check_grad_with_place, + { "place": self.place, "inputs_to_check": ['x0', 'x1'], "output_names": ['out0', 'out1'], "max_relative_error": 0.05, - "check_eager": True - }) + "check_eager": True, + }, + ) self.run_triple_in_test( - self.check_grad_with_place, { + self.check_grad_with_place, + { "place": self.place, "inputs_to_check": ['x0', 'x1', 'x2'], "output_names": ['out0', 'out1', "out2"], "max_relative_error": 0.05, - "check_eager": True - }) + "check_eager": True, + }, + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDABroadcastTensorsOp(TestCPUBroadcastTensorsOp): - def set_place(self): self.place = core.CUDAPlace(0) @@ -156,17 +160,15 @@ def set_dtypes(self): class TestBroadcastTensorsAPI(unittest.TestCase): - def test_api(self): - def test_static(): inputs = [ - paddle.fluid.layers.data(shape=[4, 1, 4, 1], - dtype='float32', - name="x0"), - paddle.fluid.layers.data(shape=[1, 4, 1, 4], - dtype='float32', - name="x1") + paddle.fluid.layers.data( + shape=[4, 1, 4, 1], dtype='float32', name="x0" + ), + paddle.fluid.layers.data( + shape=[1, 4, 1, 4], dtype='float32', name="x1" + ), ] paddle.broadcast_tensors(inputs) @@ -175,9 +177,11 @@ def test_dynamic(): try: inputs = [ paddle.to_tensor( - np.random.random([4, 1, 4, 1]).astype("float32")), + np.random.random([4, 1, 4, 1]).astype("float32") + ), paddle.to_tensor( - np.random.random([1, 4, 1, 4]).astype("float32")) + np.random.random([1, 4, 1, 4]).astype("float32") + ), ] paddle.broadcast_tensors(inputs) finally: @@ -188,39 +192,37 @@ def test_dynamic(): class TestRaiseBroadcastTensorsError(unittest.TestCase): - def test_errors(self): - def test_type(): inputs = [ - paddle.fluid.layers.data(shape=[1, 1, 1, 1], - dtype='float32', - name="x4"), - paddle.fluid.layers.data(shape=[1, 4, 1, 1], - dtype='float64', - name="x5") + paddle.fluid.layers.data( + shape=[1, 1, 1, 1], dtype='float32', name="x4" + ), + paddle.fluid.layers.data( + shape=[1, 4, 1, 1], dtype='float64', name="x5" + ), ] paddle.broadcast_tensors(inputs) def test_dtype(): inputs = [ - paddle.fluid.layers.data(shape=[1, 1, 1, 1], - dtype='int8', - name="x6"), - paddle.fluid.layers.data(shape=[1, 4, 1, 1], - dtype='int8', - name="x7") + paddle.fluid.layers.data( + shape=[1, 1, 1, 1], dtype='int8', name="x6" + ), + paddle.fluid.layers.data( + shape=[1, 4, 1, 1], dtype='int8', name="x7" + ), ] paddle.broadcast_tensors(inputs) def test_bcast_semantics(): inputs = [ - paddle.fluid.layers.data(shape=[1, 3, 1, 1], - dtype='float32', - name="x9"), - paddle.fluid.layers.data(shape=[1, 8, 1, 1], - dtype='float32', - name="x10") + paddle.fluid.layers.data( + shape=[1, 3, 1, 1], dtype='float32', name="x9" + ), + paddle.fluid.layers.data( + shape=[1, 8, 1, 1], dtype='float32', name="x10" + ), ] paddle.broadcast_tensors(inputs) @@ -230,33 +232,37 @@ def test_bcast_semantics(): class TestRaiseBroadcastTensorsErrorDyGraph(unittest.TestCase): - def test_errors(self): - def test_type(): inputs = [ paddle.to_tensor( - np.ones(shape=[1, 1, 1, 1], dtype='float32', name="x4")), + np.ones(shape=[1, 1, 1, 1], dtype='float32', name="x4") + ), paddle.to_tensor( - np.ones(shape=[1, 4, 1, 1], dtype='float64', name="x5")) + np.ones(shape=[1, 4, 1, 1], dtype='float64', name="x5") + ), ] paddle.broadcast_tensors(inputs) def test_dtype(): inputs = [ paddle.to_tensor( - np.ones(shape=[1, 1, 1, 1], dtype='int8', name="x6")), + np.ones(shape=[1, 1, 1, 1], dtype='int8', name="x6") + ), paddle.to_tensor( - np.ones(shape=[1, 4, 1, 1], dtype='int8', name="x7")) + np.ones(shape=[1, 4, 1, 1], dtype='int8', name="x7") + ), ] paddle.broadcast_tensors(inputs) def test_bcast_semantics(): inputs = [ paddle.to_tensor( - np.ones(shape=[1, 3, 1, 1], dtype='float32', name="x9")), + np.ones(shape=[1, 3, 1, 1], dtype='float32', name="x9") + ), paddle.to_tensor( - np.ones(shape=[1, 8, 1, 1], dtype='float32', name="x10")) + np.ones(shape=[1, 8, 1, 1], dtype='float32', name="x10") + ), ] paddle.broadcast_tensors(inputs) diff --git a/python/paddle/fluid/tests/unittests/test_broadcast_to_op.py b/python/paddle/fluid/tests/unittests/test_broadcast_to_op.py index aa97e316a3d49..e8e876766c332 100644 --- a/python/paddle/fluid/tests/unittests/test_broadcast_to_op.py +++ b/python/paddle/fluid/tests/unittests/test_broadcast_to_op.py @@ -22,11 +22,11 @@ class TestBroadcastToError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) shape = [2, 2] self.assertRaises(TypeError, paddle.tensor.broadcast_to, x1, shape) x2 = fluid.layers.data(name='x2', shape=[4], dtype="uint8") @@ -38,19 +38,19 @@ def test_errors(self): # Test python API class TestBroadcastToAPI(unittest.TestCase): - def test_api(self): input = np.random.random([12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) positive_2 = fluid.layers.fill_constant([1], "int32", 12) - expand_shape = fluid.layers.data(name="expand_shape", - shape=[2], - append_batch_size=False, - dtype="int32") + expand_shape = fluid.layers.data( + name="expand_shape", + shape=[2], + append_batch_size=False, + dtype="int32", + ) out_1 = paddle.broadcast_to(x, shape=[12, 14]) out_2 = paddle.broadcast_to(x, shape=[positive_2, 14]) @@ -59,14 +59,14 @@ def test_api(self): g0 = fluid.backward.calc_gradient(out_2, x) exe = fluid.Executor(place=fluid.CPUPlace()) - res_1, res_2, res_3 = exe.run(fluid.default_main_program(), - feed={ - "x": - input, - "expand_shape": - np.array([12, 14]).astype("int32") - }, - fetch_list=[out_1, out_2, out_3]) + res_1, res_2, res_3 = exe.run( + fluid.default_main_program(), + feed={ + "x": input, + "expand_shape": np.array([12, 14]).astype("int32"), + }, + fetch_list=[out_1, out_2, out_3], + ) assert np.array_equal(res_1, np.tile(input, (1, 1))) assert np.array_equal(res_2, np.tile(input, (1, 1))) assert np.array_equal(res_3, np.tile(input, (1, 1))) diff --git a/python/paddle/fluid/tests/unittests/test_bucketize_api.py b/python/paddle/fluid/tests/unittests/test_bucketize_api.py index e62c1a6028f42..e82d6e9d15c86 100644 --- a/python/paddle/fluid/tests/unittests/test_bucketize_api.py +++ b/python/paddle/fluid/tests/unittests/test_bucketize_api.py @@ -38,20 +38,20 @@ def run(place): sorted_sequence = paddle.static.data( 'SortedSequence', shape=self.sorted_sequence.shape, - dtype="float64") + dtype="float64", + ) x = paddle.static.data('x', shape=self.x.shape, dtype="float64") out1 = paddle.bucketize(x, sorted_sequence) out2 = paddle.bucketize(x, sorted_sequence, right=True) exe = paddle.static.Executor(place) - res = exe.run(feed={ - 'SortedSequence': self.sorted_sequence, - 'x': self.x - }, - fetch_list=[out1, out2]) + res = exe.run( + feed={'SortedSequence': self.sorted_sequence, 'x': self.x}, + fetch_list=[out1, out2], + ) out_ref = np.searchsorted(self.sorted_sequence, self.x) - out_ref1 = np.searchsorted(self.sorted_sequence, - self.x, - side='right') + out_ref1 = np.searchsorted( + self.sorted_sequence, self.x, side='right' + ) np.testing.assert_allclose(out_ref, res[0], rtol=1e-05) np.testing.assert_allclose(out_ref1, res[1], rtol=1e-05) @@ -59,7 +59,6 @@ def run(place): run(place) def test_api_dygraph(self): - def run(place): paddle.disable_static(place) sorted_sequence = paddle.to_tensor(self.sorted_sequence) @@ -67,9 +66,9 @@ def run(place): out1 = paddle.bucketize(x, sorted_sequence) out2 = paddle.bucketize(x, sorted_sequence, right=True) out_ref1 = np.searchsorted(self.sorted_sequence, self.x) - out_ref2 = np.searchsorted(self.sorted_sequence, - self.x, - side='right') + out_ref2 = np.searchsorted( + self.sorted_sequence, self.x, side='right' + ) np.testing.assert_allclose(out_ref1, out1.numpy(), rtol=1e-05) np.testing.assert_allclose(out_ref2, out2.numpy(), rtol=1e-05) paddle.enable_static() @@ -86,9 +85,9 @@ def test_out_int32(self): def test_bucketize_dims_error(self): with paddle.static.program_guard(paddle.static.Program()): - sorted_sequence = paddle.static.data('SortedSequence', - shape=[2, 2], - dtype="float64") + sorted_sequence = paddle.static.data( + 'SortedSequence', shape=[2, 2], dtype="float64" + ) x = paddle.static.data('x', shape=[2, 5], dtype="float64") self.assertRaises(ValueError, paddle.bucketize, x, sorted_sequence) @@ -96,16 +95,18 @@ def test_input_error(self): for place in self.place: paddle.disable_static(place) sorted_sequence = paddle.to_tensor(self.sorted_sequence) - self.assertRaises(ValueError, paddle.bucketize, self.x, - sorted_sequence) + self.assertRaises( + ValueError, paddle.bucketize, self.x, sorted_sequence + ) def test_empty_input_error(self): for place in self.place: paddle.disable_static(place) sorted_sequence = paddle.to_tensor(self.sorted_sequence) x = paddle.to_tensor(self.x) - self.assertRaises(ValueError, paddle.bucketize, None, - sorted_sequence) + self.assertRaises( + ValueError, paddle.bucketize, None, sorted_sequence + ) self.assertRaises(AttributeError, paddle.bucketize, x, None) diff --git a/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass.py b/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass.py index e7d9b8e485d54..5534ff67915b2 100644 --- a/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass.py +++ b/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass.py @@ -22,16 +22,14 @@ batch_size = 32 feed_dict = { - 'image': - np.random.random([batch_size, 784]).astype('float32'), - 'label': - np.random.random_integers(low=0, high=9, size=[batch_size, - 1]).astype('int64') + 'image': np.random.random([batch_size, 784]).astype('float32'), + 'label': np.random.random_integers( + low=0, high=9, size=[batch_size, 1] + ).astype('int64'), } class InplaceTestBase(unittest.TestCase): - def initParameter(self): self.use_cuda = True self.fuse_all_optimizer_ops = False @@ -61,8 +59,10 @@ def build_program_and_scope(self): with fluid.scope_guard(scope): exe = fluid.Executor( - fluid.CUDAPlace(0) if self.use_cuda else fluid.CPUPlace( - )) + fluid.CUDAPlace(0) + if self.use_cuda + else fluid.CPUPlace() + ) exe.run(startup_program) return main_program, scope, exe, loss @@ -93,11 +93,14 @@ def check_single_card_fetch_var(self): build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = memory_optimize build_strategy.enable_inplace = enable_inplace - build_strategy.fuse_all_optimizer_ops = self.fuse_all_optimizer_ops + build_strategy.fuse_all_optimizer_ops = ( + self.fuse_all_optimizer_ops + ) compiled_prog = fluid.CompiledProgram(prog).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy, - places=self.place) + places=self.place, + ) compiled_programs.append(compiled_prog) all_vars_name = self.get_all_vars(prog1) @@ -107,24 +110,26 @@ def check_single_card_fetch_var(self): for fetch_var in repeated_var_names[:4]: for _ in range(2): with fluid.scope_guard(scope1): - fetch_val1, = exe.run(prog1, - feed=feed_dict, - fetch_list=[fetch_var]) + (fetch_val1,) = exe.run( + prog1, feed=feed_dict, fetch_list=[fetch_var] + ) for scope, compiled_prog in zip(scopes, compiled_programs): with fluid.scope_guard(scope): - fetch_val2, = exe.run(compiled_prog, - feed=feed_dict, - fetch_list=[fetch_var]) + (fetch_val2,) = exe.run( + compiled_prog, + feed=feed_dict, + fetch_list=[fetch_var], + ) np.testing.assert_array_equal( fetch_val1, fetch_val2, - err_msg= - 'error var name: {}, fetch_val1: {}, fetch_val2: {}' - .format( + err_msg='error var name: {}, fetch_val1: {}, fetch_val2: {}'.format( fetch_var, fetch_val1[~np.equal(fetch_val1, fetch_val2)], - fetch_val2[~np.equal(fetch_val1, fetch_val2)])) + fetch_val2[~np.equal(fetch_val1, fetch_val2)], + ), + ) def check_multi_card_fetch_var(self): if self.is_invalid_test(): @@ -146,11 +151,16 @@ def check_multi_card_fetch_var(self): build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = memory_optimize build_strategy.enable_inplace = enable_inplace - build_strategy.fuse_all_optimizer_ops = self.fuse_all_optimizer_ops + build_strategy.fuse_all_optimizer_ops = ( + self.fuse_all_optimizer_ops + ) compiled_program = fluid.CompiledProgram( - prog).with_data_parallel(loss_name=loss.name, - build_strategy=build_strategy, - places=places) + prog + ).with_data_parallel( + loss_name=loss.name, + build_strategy=build_strategy, + places=places, + ) compiled_programs.append(compiled_program) repeated_var_names = self.get_all_vars(prog1) @@ -161,9 +171,11 @@ def check_multi_card_fetch_var(self): fetch_vals = [] for scope, compiled_prog in zip(scopes, compiled_programs): with fluid.scope_guard(scope): - fetch_val, = exe.run(compiled_prog, - feed=feed_dict, - fetch_list=[fetch_var]) + (fetch_val,) = exe.run( + compiled_prog, + feed=feed_dict, + fetch_list=[fetch_var], + ) fetch_vals.append(fetch_val) for item in fetch_vals: @@ -171,14 +183,15 @@ def check_multi_card_fetch_var(self): np.testing.assert_array_equal( fetch_vals[0], item, - err_msg='error var name: {}, fetch_vals[0]: {}, item: {}' - .format(fetch_var, - fetch_vals[0][~np.equal(fetch_vals[0], item)], - item[~np.equal(fetch_vals[0], item)])) + err_msg='error var name: {}, fetch_vals[0]: {}, item: {}'.format( + fetch_var, + fetch_vals[0][~np.equal(fetch_vals[0], item)], + item[~np.equal(fetch_vals[0], item)], + ), + ) class CUDAInplaceTest(InplaceTestBase): - def initParameter(self): self.use_cuda = True self.fuse_all_optimizer_ops = False @@ -191,7 +204,6 @@ def test_single_card_fetch_var(self): class CPUInplaceTest(InplaceTestBase): - def initParameter(self): self.use_cuda = False self.fuse_all_optimizer_ops = False diff --git a/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass_and_fuse_optimization_op_pass.py b/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass_and_fuse_optimization_op_pass.py index 6ce5e64b0ee57..e9e62bee00680 100644 --- a/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass_and_fuse_optimization_op_pass.py +++ b/python/paddle/fluid/tests/unittests/test_buffer_shared_memory_reuse_pass_and_fuse_optimization_op_pass.py @@ -17,7 +17,6 @@ class CUDAInplaceTestWithFuseOptimizationOps(InplaceTestBase): - def initParameter(self): self.use_cuda = True self.fuse_all_optimizer_ops = True @@ -31,7 +30,6 @@ def test_single_card_fetch_var(self): class CPUInplaceTestWithFuseOptimizationOps(InplaceTestBase): - def initParameter(self): self.use_cuda = False self.fuse_all_optimizer_ops = True diff --git a/python/paddle/fluid/tests/unittests/test_c_comm_init_all_op.py b/python/paddle/fluid/tests/unittests/test_c_comm_init_all_op.py index eb03c13602fd0..7f4eac0d3fb36 100644 --- a/python/paddle/fluid/tests/unittests/test_c_comm_init_all_op.py +++ b/python/paddle/fluid/tests/unittests/test_c_comm_init_all_op.py @@ -18,7 +18,6 @@ class TestCCommInitAllOp(unittest.TestCase): - def setUp(self): self.place = fluid.CUDAPlace(0) self.exe = fluid.Executor(self.place) @@ -39,11 +38,9 @@ def test_init_with_same_ring_id(self): def test_specifying_devices(self): program = fluid.Program() block = program.global_block() - block.append_op(type='c_comm_init_all', - attrs={ - 'devices': [0], - 'ring_id': 1 - }) + block.append_op( + type='c_comm_init_all', attrs={'devices': [0], 'ring_id': 1} + ) self.exe.run(program) diff --git a/python/paddle/fluid/tests/unittests/test_c_embedding_op.py b/python/paddle/fluid/tests/unittests/test_c_embedding_op.py index 8b32a1857b255..66c68097127a2 100644 --- a/python/paddle/fluid/tests/unittests/test_c_embedding_op.py +++ b/python/paddle/fluid/tests/unittests/test_c_embedding_op.py @@ -13,7 +13,11 @@ # limitations under the License. import unittest -from paddle.fluid.tests.unittests.c_embedding_op_base import TestCEmbeddingCPU, TestCEmbeddingOpBase, TestCEmbeddingOpFP32 +from paddle.fluid.tests.unittests.c_embedding_op_base import ( + TestCEmbeddingCPU, + TestCEmbeddingOpBase, + TestCEmbeddingOpFP32, +) TestCEmbeddingCPU() diff --git a/python/paddle/fluid/tests/unittests/test_calc_gradient.py b/python/paddle/fluid/tests/unittests/test_calc_gradient.py index 350f30aaf1a55..59bc1f574e443 100644 --- a/python/paddle/fluid/tests/unittests/test_calc_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_calc_gradient.py @@ -21,7 +21,6 @@ class TestCalcGradient(unittest.TestCase): - def test_calc_gradient(self): main = fluid.Program() startup = fluid.Program() @@ -39,7 +38,6 @@ def test_calc_gradient(self): class TestDoubleGrad(unittest.TestCase): - def test1(self): main = fluid.Program() startup = fluid.Program() @@ -49,10 +47,11 @@ def test1(self): name='x', shape=[1], dtype='float32', - default_initializer=fluid.initializer.Constant(3)) - grad1, = fluid.gradients(net(x), x) # 2x = 6 + default_initializer=fluid.initializer.Constant(3), + ) + (grad1,) = fluid.gradients(net(x), x) # 2x = 6 z = net(x - grad1) - grad2, = fluid.gradients(z, x) # gradients( (x - 2x)^2) = 2x = 6 + (grad2,) = fluid.gradients(z, x) # gradients( (x - 2x)^2) = 2x = 6 place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -69,21 +68,21 @@ def test2(self): name='x', shape=[1], dtype='float32', - default_initializer=fluid.initializer.Constant(1)) + default_initializer=fluid.initializer.Constant(1), + ) y = x * x - dx1, = fluid.gradients(y, x) + (dx1,) = fluid.gradients(y, x) z = dx1 * dx1 + y * y - dx2, = fluid.gradients(z, x) + (dx2,) = fluid.gradients(z, x) place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup) - out, = exe.run(main, fetch_list=[dx2]) + (out,) = exe.run(main, fetch_list=[dx2]) self.assertEqual(12, out[0]) class TestGradientWithPrune(unittest.TestCase): - def test_prune(self): with paddle.fluid.scope_guard(paddle.static.Scope()): x = fluid.data(name='x', shape=[3], dtype='float32') @@ -95,14 +94,15 @@ def test_prune(self): exe = fluid.Executor(fluid.CPUPlace()) main = fluid.default_main_program() exe.run(fluid.default_startup_program()) - out = exe.run(main, - feed={'x': np.ones([3]).astype('float32')}, - fetch_list=[x1_grad]) + out = exe.run( + main, + feed={'x': np.ones([3]).astype('float32')}, + fetch_list=[x1_grad], + ) np.testing.assert_array_equal(out[0], [2.0, 0.0, 0.0]) class TestDoubleGradient(unittest.TestCase): - def build_program(self): start_prog = paddle.static.Program() main_prog = paddle.static.Program() @@ -128,16 +128,17 @@ def test_calc_gradient(self): start_prog, main_prog, fetch_list = self.build_program() exe = paddle.static.Executor() exe.run(start_prog) - ans = exe.run(main_prog, - feed={'x': np.ones([2, 2]).astype(np.float32)}, - fetch_list=fetch_list) + ans = exe.run( + main_prog, + feed={'x': np.ones([2, 2]).astype(np.float32)}, + fetch_list=fetch_list, + ) self.assertEqual(len(ans), 2) - self.assertListEqual(ans[0].tolist(), [[0., 0.], [0., 0.]]) - self.assertListEqual(ans[1].tolist(), [[2., 2.], [2., 2.]]) + self.assertListEqual(ans[0].tolist(), [[0.0, 0.0], [0.0, 0.0]]) + self.assertListEqual(ans[1].tolist(), [[2.0, 2.0], [2.0, 2.0]]) class TestDoubleGradient2(unittest.TestCase): - def build_program(self): start_prog = paddle.static.Program() main_prog = paddle.static.Program() @@ -156,8 +157,9 @@ def build_program(self): grad_x = paddle.static.gradients(y, x, grad_y) grad_x2 = paddle.static.gradients(y2, x, grad_y) # test with multi targets - jvp = paddle.static.gradients([grad_x[0], grad_x2[0]], grad_y, - [v, v]) + jvp = paddle.static.gradients( + [grad_x[0], grad_x2[0]], grad_y, [v, v] + ) return start_prog, main_prog, [grad_x, jvp] @@ -166,12 +168,14 @@ def test_calc_gradient(self): start_prog, main_prog, fetch_list = self.build_program() exe = paddle.static.Executor() exe.run(start_prog) - ans = exe.run(main_prog, - feed={'x': np.ones([2, 2]).astype(np.float32)}, - fetch_list=fetch_list) + ans = exe.run( + main_prog, + feed={'x': np.ones([2, 2]).astype(np.float32)}, + fetch_list=fetch_list, + ) self.assertEqual(len(ans), 2) - self.assertListEqual(ans[0].tolist(), [[0., 0.], [0., 0.]]) - self.assertListEqual(ans[1].tolist(), [[5., 5.], [5., 5.]]) + self.assertListEqual(ans[0].tolist(), [[0.0, 0.0], [0.0, 0.0]]) + self.assertListEqual(ans[1].tolist(), [[5.0, 5.0], [5.0, 5.0]]) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_case.py b/python/paddle/fluid/tests/unittests/test_case.py index e550adc6a9ae1..62a3898fb9789 100644 --- a/python/paddle/fluid/tests/unittests/test_case.py +++ b/python/paddle/fluid/tests/unittests/test_case.py @@ -25,9 +25,7 @@ class TestAPICase(unittest.TestCase): - def test_return_single_var(self): - def fn_1(): return layers.fill_constant(shape=[4, 2], dtype='int32', value=1) @@ -47,16 +45,19 @@ def fn_3(): pred_1 = layers.less_than(z, x) # true: 0.2 < 0.3 # call fn_1 - out_0 = layers.case(pred_fn_pairs=[(pred_1, fn_1), (pred_1, fn_2)], - default=fn_3) + out_0 = layers.case( + pred_fn_pairs=[(pred_1, fn_1), (pred_1, fn_2)], default=fn_3 + ) # call fn_2 - out_1 = layers.case(pred_fn_pairs=[(pred_2, fn_1), (pred_1, fn_2)], - default=fn_3) + out_1 = layers.case( + pred_fn_pairs=[(pred_2, fn_1), (pred_1, fn_2)], default=fn_3 + ) # call default fn_3 - out_2 = layers.case(pred_fn_pairs=((pred_2, fn_1), (pred_2, fn_2)), - default=fn_3) + out_2 = layers.case( + pred_fn_pairs=((pred_2, fn_1), (pred_2, fn_2)), default=fn_3 + ) # no default, call fn_2 out_3 = layers.case(pred_fn_pairs=[(pred_1, fn_2)]) @@ -64,12 +65,16 @@ def fn_3(): # no default, call fn_2. but pred_2 is false out_4 = layers.case(pred_fn_pairs=[(pred_2, fn_2)]) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(main_program, - fetch_list=[out_0, out_1, out_2, out_3, out_4]) + res = exe.run( + main_program, fetch_list=[out_0, out_1, out_2, out_3, out_4] + ) np.testing.assert_allclose(res[0], 1, rtol=1e-05) np.testing.assert_allclose(res[1], 2, rtol=1e-05) @@ -78,27 +83,20 @@ def fn_3(): np.testing.assert_allclose(res[4], 2, rtol=1e-05) def test_return_var_tuple(self): - def fn_1(): - return layers.fill_constant(shape=[1, 2], dtype='int32', - value=1), layers.fill_constant( - shape=[2, 3], - dtype='float32', - value=2) + return layers.fill_constant( + shape=[1, 2], dtype='int32', value=1 + ), layers.fill_constant(shape=[2, 3], dtype='float32', value=2) def fn_2(): - return layers.fill_constant(shape=[3, 4], dtype='int32', - value=3), layers.fill_constant( - shape=[4, 5], - dtype='float32', - value=4) + return layers.fill_constant( + shape=[3, 4], dtype='int32', value=3 + ), layers.fill_constant(shape=[4, 5], dtype='float32', value=4) def fn_3(): - return layers.fill_constant(shape=[5], dtype='int32', - value=5), layers.fill_constant( - shape=[5, 6], - dtype='float32', - value=6) + return layers.fill_constant( + shape=[5], dtype='int32', value=5 + ), layers.fill_constant(shape=[5, 6], dtype='float32', value=6) main_program = Program() startup_program = Program() @@ -112,56 +110,87 @@ def fn_3(): out = layers.case(((pred_1, fn_1), (pred_2, fn_2)), fn_3) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) ret = exe.run(main_program, fetch_list=out) - np.testing.assert_allclose(np.asarray(ret[0]), - np.full((1, 2), 1, np.int32), - rtol=1e-05) - np.testing.assert_allclose(np.asarray(ret[1]), - np.full((2, 3), 2, np.float32), - rtol=1e-05) + np.testing.assert_allclose( + np.asarray(ret[0]), np.full((1, 2), 1, np.int32), rtol=1e-05 + ) + np.testing.assert_allclose( + np.asarray(ret[1]), np.full((2, 3), 2, np.float32), rtol=1e-05 + ) class TestAPICase_Nested(unittest.TestCase): - def test_nested_case(self): - def fn_1(x=1): var_5 = layers.fill_constant(shape=[1], dtype='int32', value=5) var_6 = layers.fill_constant(shape=[1], dtype='int32', value=6) - out = layers.case(pred_fn_pairs=[ - (var_5 < var_6, - partial( - layers.fill_constant, shape=[1], dtype='int32', value=x)), - (var_5 == var_6, - partial( - layers.fill_constant, shape=[2], dtype='int32', value=x)) - ]) + out = layers.case( + pred_fn_pairs=[ + ( + var_5 < var_6, + partial( + layers.fill_constant, + shape=[1], + dtype='int32', + value=x, + ), + ), + ( + var_5 == var_6, + partial( + layers.fill_constant, + shape=[2], + dtype='int32', + value=x, + ), + ), + ] + ) return out def fn_2(x=2): var_5 = layers.fill_constant(shape=[1], dtype='int32', value=5) var_6 = layers.fill_constant(shape=[1], dtype='int32', value=6) - out = layers.case(pred_fn_pairs=[ - (var_5 < var_6, partial(fn_1, x=x)), - (var_5 == var_6, - partial( - layers.fill_constant, shape=[2], dtype='int32', value=x)) - ]) + out = layers.case( + pred_fn_pairs=[ + (var_5 < var_6, partial(fn_1, x=x)), + ( + var_5 == var_6, + partial( + layers.fill_constant, + shape=[2], + dtype='int32', + value=x, + ), + ), + ] + ) return out def fn_3(): var_5 = layers.fill_constant(shape=[1], dtype='int32', value=5) var_6 = layers.fill_constant(shape=[1], dtype='int32', value=6) - out = layers.case(pred_fn_pairs=[ - (var_5 < var_6, partial(fn_2, x=3)), - (var_5 == var_6, - partial( - layers.fill_constant, shape=[2], dtype='int32', value=7)) - ]) + out = layers.case( + pred_fn_pairs=[ + (var_5 < var_6, partial(fn_2, x=3)), + ( + var_5 == var_6, + partial( + layers.fill_constant, + shape=[2], + dtype='int32', + value=7, + ), + ), + ] + ) return out main_program = Program() @@ -173,17 +202,23 @@ def fn_3(): pred_2 = layers.less_than(x, y) # false: 0.3 < 0.1 pred_1 = layers.less_than(z, x) # true: 0.2 < 0.3 - out_1 = layers.case(pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], - default=fn_3) + out_1 = layers.case( + pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], default=fn_3 + ) - out_2 = layers.case(pred_fn_pairs=[(pred_2, fn_1), (pred_1, fn_2)], - default=fn_3) + out_2 = layers.case( + pred_fn_pairs=[(pred_2, fn_1), (pred_1, fn_2)], default=fn_3 + ) - out_3 = layers.case(pred_fn_pairs=[(x == y, fn_1), (x == z, fn_2)], - default=fn_3) + out_3 = layers.case( + pred_fn_pairs=[(x == y, fn_1), (x == z, fn_2)], default=fn_3 + ) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) res = exe.run(main_program, fetch_list=[out_1, out_2, out_3]) @@ -194,9 +229,7 @@ def fn_3(): class TestAPICase_Error(unittest.TestCase): - def test_error(self): - def fn_1(): return layers.fill_constant(shape=[4, 2], dtype='int32', value=1) @@ -246,18 +279,17 @@ def type_error_default(): # when optimizer in case class TestMutiTask(unittest.TestCase): - def test_optimizer_in_case(self): BATCH_SIZE = 1 INPUT_SIZE = 784 EPOCH_NUM = 2 - x = fluid.data(name='x', - shape=[BATCH_SIZE, INPUT_SIZE], - dtype='float32') - y = fluid.data(name='y', - shape=[BATCH_SIZE, INPUT_SIZE], - dtype='float32') + x = fluid.data( + name='x', shape=[BATCH_SIZE, INPUT_SIZE], dtype='float32' + ) + y = fluid.data( + name='y', shape=[BATCH_SIZE, INPUT_SIZE], dtype='float32' + ) switch_id = fluid.data(name='switch_id', shape=[1], dtype='int32') @@ -282,16 +314,19 @@ def fn_2(): for epoch in range(EPOCH_NUM): np.random.seed(epoch) - feed_image = np.random.random( - size=[BATCH_SIZE, INPUT_SIZE]).astype('float32') + feed_image = np.random.random(size=[BATCH_SIZE, INPUT_SIZE]).astype( + 'float32' + ) main_program = fluid.default_main_program() - out = exe.run(main_program, - feed={ - 'x': feed_image, - 'y': feed_image, - 'switch_id': np.array([epoch]).astype('int32') - }, - fetch_list=[]) + out = exe.run( + main_program, + feed={ + 'x': feed_image, + 'y': feed_image, + 'switch_id': np.array([epoch]).astype('int32'), + }, + fetch_list=[], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_cast_op.py b/python/paddle/fluid/tests/unittests/test_cast_op.py index f4cabd6519c2e..8665e285fa5cf 100644 --- a/python/paddle/fluid/tests/unittests/test_cast_op.py +++ b/python/paddle/fluid/tests/unittests/test_cast_op.py @@ -27,14 +27,13 @@ class TestCastOpFp32ToFp64(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float32')} self.outputs = {'Out': ipt.astype('float64')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP32), - 'out_dtype': int(core.VarDesc.VarType.FP64) + 'out_dtype': int(core.VarDesc.VarType.FP64), } self.op_type = 'cast' @@ -46,14 +45,13 @@ def test_grad(self): class TestCastOpFp16ToFp32(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float16')} self.outputs = {'Out': ipt.astype('float32')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP16), - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.op_type = 'cast' self.__class__.no_need_check_grad = True @@ -63,14 +61,13 @@ def test_check_output(self): class TestCastOpFp32ToFp16(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float32')} self.outputs = {'Out': ipt.astype('float16')} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP32), - 'out_dtype': int(core.VarDesc.VarType.FP16) + 'out_dtype': int(core.VarDesc.VarType.FP16), } self.op_type = 'cast' self.__class__.no_need_check_grad = True @@ -80,14 +77,13 @@ def test_check_output(self): class TestCastOpBf16ToFp32(OpTest): - def setUp(self): ipt = np.array(np.random.randint(10, size=[10, 10])).astype('uint16') self.inputs = {'X': ipt} self.outputs = {'Out': convert_uint16_to_float(ipt)} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.BF16), - 'out_dtype': int(core.VarDesc.VarType.FP32) + 'out_dtype': int(core.VarDesc.VarType.FP32), } self.op_type = 'cast' self.__class__.no_need_check_grad = True @@ -97,14 +93,13 @@ def test_check_output(self): class TestCastOpFp32ToBf16(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]).astype('float32') self.inputs = {'X': ipt} self.outputs = {'Out': convert_float_to_uint16(ipt)} self.attrs = { 'in_dtype': int(core.VarDesc.VarType.FP32), - 'out_dtype': int(core.VarDesc.VarType.BF16) + 'out_dtype': int(core.VarDesc.VarType.BF16), } self.op_type = 'cast' self.__class__.no_need_check_grad = True @@ -114,32 +109,31 @@ def test_check_output(self): class TestCastOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of cast_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.cast, x1, 'int32') class TestCastOpEager(unittest.TestCase): - def test_eager(self): with paddle.fluid.dygraph.base.guard(): with _test_eager_guard(): x = paddle.ones([2, 2], dtype="float16") x.stop_gradient = False out = paddle.cast(x, "float32") - np.testing.assert_array_equal(out, - np.ones([2, 2]).astype('float32')) + np.testing.assert_array_equal( + out, np.ones([2, 2]).astype('float32') + ) out.backward() np.testing.assert_array_equal(x.gradient(), x.numpy()) self.assertTrue(x.gradient().dtype == np.float16) class TestCastDoubleGradCheck(unittest.TestCase): - def cast_wrapper(self, x): return paddle.cast(x[0], 'float64') @@ -154,17 +148,13 @@ def func(self, place): out = paddle.cast(data, 'float64') data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.cast_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.cast_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -176,7 +166,6 @@ def test_grad(self): class TestCastTripleGradCheck(unittest.TestCase): - def cast_wrapper(self, x): return paddle.cast(x[0], 'float64') @@ -191,17 +180,13 @@ def func(self, place): out = paddle.cast(data, 'float64') data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.cast_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.cast_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_center_loss.py b/python/paddle/fluid/tests/unittests/test_center_loss.py index 59c8b417d1002..b7eda71c0217b 100644 --- a/python/paddle/fluid/tests/unittests/test_center_loss.py +++ b/python/paddle/fluid/tests/unittests/test_center_loss.py @@ -19,7 +19,6 @@ class TestCenterLossOp(OpTest): - def setUp(self): self.op_type = "center_loss" self.dtype = np.float64 @@ -47,7 +46,7 @@ def setUp(self): cout[labels[i]] += 1 var_sum[labels[i]] += output[i] for i in range(cluster_num): - var_sum[i] /= (1 + cout[i]) + var_sum[i] /= 1 + cout[i] var_sum *= 0.1 result = centers + var_sum rate = np.array([0.1]).astype(np.float64) @@ -56,20 +55,20 @@ def setUp(self): 'X': feat, 'Label': labels, 'Centers': centers, - 'CenterUpdateRate': rate + 'CenterUpdateRate': rate, } if self.need_update == True: self.outputs = { 'SampleCenterDiff': output, 'Loss': loss, - 'CentersOut': result + 'CentersOut': result, } else: self.outputs = { 'SampleCenterDiff': output, 'Loss': loss, - 'CentersOut': centers + 'CentersOut': centers, } def config(self): @@ -86,35 +85,34 @@ def test_check_grad(self): class TestCenterLossOpNoUpdate(TestCenterLossOp): - def config(self): self.need_update = False class BadInputTestCenterLoss(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): def test_bad_x(): data = [[1, 2, 3, 4], [5, 6, 7, 8]] - label = fluid.layers.data(name='label', - shape=[2, 1], - dtype='int32') + label = fluid.layers.data( + name='label', shape=[2, 1], dtype='int32' + ) res = fluid.layers.center_loss( data, label, num_classes=1000, alpha=0.2, param_attr=fluid.initializer.Xavier(uniform=False), - update_center=True) + update_center=True, + ) self.assertRaises(TypeError, test_bad_x) def test_bad_y(): - data = fluid.layers.data(name='data', - shape=[2, 32], - dtype='float32') + data = fluid.layers.data( + name='data', shape=[2, 32], dtype='float32' + ) label = [[2], [3]] res = fluid.layers.center_loss( data, @@ -122,30 +120,38 @@ def test_bad_y(): num_classes=1000, alpha=0.2, param_attr=fluid.initializer.Xavier(uniform=False), - update_center=True) + update_center=True, + ) self.assertRaises(TypeError, test_bad_y) def test_bad_alpha(): - data = fluid.layers.data(name='data2', - shape=[2, 32], - dtype='float32', - append_batch_size=False) - label = fluid.layers.data(name='label2', - shape=[2, 1], - dtype='int32', - append_batch_size=False) - alpha = fluid.layers.data(name='alpha', - shape=[1], - dtype='int64', - append_batch_size=False) + data = fluid.layers.data( + name='data2', + shape=[2, 32], + dtype='float32', + append_batch_size=False, + ) + label = fluid.layers.data( + name='label2', + shape=[2, 1], + dtype='int32', + append_batch_size=False, + ) + alpha = fluid.layers.data( + name='alpha', + shape=[1], + dtype='int64', + append_batch_size=False, + ) res = fluid.layers.center_loss( data, label, num_classes=1000, alpha=alpha, param_attr=fluid.initializer.Xavier(uniform=False), - update_center=True) + update_center=True, + ) self.assertRaises(TypeError, test_bad_alpha) diff --git a/python/paddle/fluid/tests/unittests/test_channel_shuffle.py b/python/paddle/fluid/tests/unittests/test_channel_shuffle.py index 50df6bbc024cf..0da2a651477e2 100644 --- a/python/paddle/fluid/tests/unittests/test_channel_shuffle.py +++ b/python/paddle/fluid/tests/unittests/test_channel_shuffle.py @@ -42,7 +42,6 @@ def channel_shuffle_np(x, groups, data_format="NCHW"): class TestChannelShuffleOp(OpTest): - def setUp(self): self.op_type = "channel_shuffle" self.init_data_format() @@ -73,13 +72,11 @@ def test_check_grad(self): class TestChannelLast(TestChannelShuffleOp): - def init_data_format(self): self.format = "NHWC" class TestChannelShuffleAPI(unittest.TestCase): - def setUp(self): self.x_1_np = np.random.random([2, 9, 4, 4]).astype("float64") self.x_2_np = np.random.random([2, 4, 4, 9]).astype("float64") @@ -87,47 +84,53 @@ def setUp(self): self.out_2_np = channel_shuffle_np(self.x_2_np, 3, "NHWC") def test_static_graph_functional(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=[2, 9, 4, 4], - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=[2, 4, 4, 9], - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=[2, 9, 4, 4], dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=[2, 4, 4, 9], dtype="float64" + ) out_1 = F.channel_shuffle(x_1, 3) out_2 = F.channel_shuffle(x_2, 3, "NHWC") exe = paddle.static.Executor(place=place) - res_1 = exe.run(fluid.default_main_program(), - feed={"x": self.x_1_np}, - fetch_list=out_1, - use_prune=True) - - res_2 = exe.run(fluid.default_main_program(), - feed={"x2": self.x_2_np}, - fetch_list=out_2, - use_prune=True) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True, + ) + + res_2 = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True, + ) assert np.allclose(res_1, self.out_1_np) assert np.allclose(res_2, self.out_2_np) # same test between layer and functional in this op. def test_static_graph_layer(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=[2, 9, 4, 4], - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=[2, 4, 4, 9], - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=[2, 9, 4, 4], dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=[2, 4, 4, 9], dtype="float64" + ) # init instance ps_1 = paddle.nn.ChannelShuffle(3) ps_2 = paddle.nn.ChannelShuffle(3, "NHWC") @@ -137,15 +140,19 @@ def test_static_graph_layer(self): out_2_np = channel_shuffle_np(self.x_2_np, 3, "NHWC") exe = paddle.static.Executor(place=place) - res_1 = exe.run(fluid.default_main_program(), - feed={"x": self.x_1_np}, - fetch_list=out_1, - use_prune=True) - - res_2 = exe.run(fluid.default_main_program(), - feed={"x2": self.x_2_np}, - fetch_list=out_2, - use_prune=True) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True, + ) + + res_2 = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True, + ) assert np.allclose(res_1, out_1_np) assert np.allclose(res_2, out_2_np) @@ -163,23 +170,26 @@ def run_dygraph(self, groups, data_format): npresult = channel_shuffle_np(x, groups, data_format) - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) - channel_shuffle = paddle.nn.ChannelShuffle(groups, - data_format=data_format) + channel_shuffle = paddle.nn.ChannelShuffle( + groups, data_format=data_format + ) result = channel_shuffle(paddle.to_tensor(x)) np.testing.assert_allclose(result.numpy(), npresult, rtol=1e-05) - result_functional = F.channel_shuffle(paddle.to_tensor(x), 3, - data_format) - np.testing.assert_allclose(result_functional.numpy(), - npresult, - rtol=1e-05) + result_functional = F.channel_shuffle( + paddle.to_tensor(x), 3, data_format + ) + np.testing.assert_allclose( + result_functional.numpy(), npresult, rtol=1e-05 + ) channel_shuffle_str = 'groups={}'.format(groups) if data_format != 'NCHW': @@ -194,9 +204,7 @@ def test_dygraph2(self): class TestChannelShuffleError(unittest.TestCase): - def test_error_functional(self): - def error_input(): with paddle.fluid.dygraph.guard(): x = np.random.random([9, 4, 4]).astype("float64") @@ -221,13 +229,13 @@ def error_groups_2(): def error_data_format(): with paddle.fluid.dygraph.guard(): x = np.random.random([2, 9, 4, 4]).astype("float64") - channel_shuffle = F.channel_shuffle(paddle.to_tensor(x), 3, - "WOW") + channel_shuffle = F.channel_shuffle( + paddle.to_tensor(x), 3, "WOW" + ) self.assertRaises(ValueError, error_data_format) def test_error_layer(self): - def error_input_layer(): with paddle.fluid.dygraph.guard(): x = np.random.random([9, 4, 4]).astype("float64") diff --git a/python/paddle/fluid/tests/unittests/test_check_import_scipy.py b/python/paddle/fluid/tests/unittests/test_check_import_scipy.py index 1707ff232a1a4..f06b5c97e3f90 100644 --- a/python/paddle/fluid/tests/unittests/test_check_import_scipy.py +++ b/python/paddle/fluid/tests/unittests/test_check_import_scipy.py @@ -21,7 +21,6 @@ def my_import(name, globals=None, locals=None, fromlist=(), level=0): class importTest(unittest.TestCase): - def test_import(self): testOsName = 'nt' old_import = builtins.__import__ diff --git a/python/paddle/fluid/tests/unittests/test_checkpoint_saver.py b/python/paddle/fluid/tests/unittests/test_checkpoint_saver.py index b9851a283fe09..f3b539d8057ae 100644 --- a/python/paddle/fluid/tests/unittests/test_checkpoint_saver.py +++ b/python/paddle/fluid/tests/unittests/test_checkpoint_saver.py @@ -20,7 +20,6 @@ class CheckpointerSaverTest(unittest.TestCase): - def test(self): fs = HDFSClient("/usr/local/hadoop-2.7.7", None) dir_path = "./checkpointsaver_test" diff --git a/python/paddle/fluid/tests/unittests/test_cholesky_op.py b/python/paddle/fluid/tests/unittests/test_cholesky_op.py index b903cd9159da2..1f5d60843139a 100644 --- a/python/paddle/fluid/tests/unittests/test_cholesky_op.py +++ b/python/paddle/fluid/tests/unittests/test_cholesky_op.py @@ -24,16 +24,15 @@ @skip_check_grad_ci( - reason= - "The input of cholesky_op should always be symmetric positive-definite. " + reason="The input of cholesky_op should always be symmetric positive-definite. " "However, OpTest calculates the numeric gradient of each element in input " "via small finite difference, which makes the input no longer symmetric " "positive-definite thus can not compute the Cholesky decomposition. " "While we can use the gradient_checker.grad_check to perform gradient " "check of cholesky_op, since it supports check gradient with a program " - "and we can construct symmetric positive-definite matrices in the program") + "and we can construct symmetric positive-definite matrices in the program" +) class TestCholeskyOp(OpTest): - def setUp(self): self.op_type = "cholesky" self._input_shape = (2, 32, 32) @@ -41,12 +40,14 @@ def setUp(self): self.init_config() self.trans_dims = list(range(len(self._input_shape) - 2)) + [ len(self._input_shape) - 1, - len(self._input_shape) - 2 + len(self._input_shape) - 2, ] self.root_data = np.random.random(self._input_shape).astype("float64") # construct symmetric positive-definite matrice - input_data = np.matmul( - self.root_data, self.root_data.transpose(self.trans_dims)) + 1e-05 + input_data = ( + np.matmul(self.root_data, self.root_data.transpose(self.trans_dims)) + + 1e-05 + ) output_data = np.linalg.cholesky(input_data).astype("float64") if self._upper: output_data = output_data.transpose(self.trans_dims) @@ -70,8 +71,9 @@ def func(self, place): root_data = self.root_data[..., :3, :3] prog = fluid.Program() with fluid.program_guard(prog): - root = layers.create_parameter(dtype=root_data.dtype, - shape=root_data.shape) + root = layers.create_parameter( + dtype=root_data.dtype, shape=root_data.shape + ) root_t = layers.transpose(root, self.trans_dims) x = layers.matmul(x=root, y=root_t) + 1e-05 out = paddle.cholesky(x, upper=self.attrs["upper"]) @@ -82,19 +84,16 @@ def init_config(self): class TestCholeskyOpLower(TestCholeskyOp): - def init_config(self): self._upper = False class TestCholeskyOp2D(TestCholeskyOp): - def init_config(self): self._input_shape = (64, 64) class TestDygraph(unittest.TestCase): - def test_dygraph(self): if core.is_compiled_with_rocm(): paddle.disable_static(place=fluid.CPUPlace()) @@ -108,7 +107,6 @@ def test_dygraph(self): class TestCholeskySingularAPI(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and (not core.is_compiled_with_rocm()): @@ -123,9 +121,11 @@ def check_static_result(self, place, with_out=False): exe = fluid.Executor(place) try: - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) except RuntimeError as ex: print("The mat is singular") except ValueError as ex: @@ -138,9 +138,12 @@ def test_static(self): def test_dygraph(self): for place in self.places: with fluid.dygraph.guard(place): - input_np = np.array([[[1, 2, 3], [4, 5, 6], [7, 8, 9]], - [[10, 11, 12], [13, 14, 15], - [16, 17, 18]]]).astype("float64") + input_np = np.array( + [ + [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + [[10, 11, 12], [13, 14, 15], [16, 17, 18]], + ] + ).astype("float64") input = fluid.dygraph.to_variable(input_np) try: result = paddle.cholesky(input) diff --git a/python/paddle/fluid/tests/unittests/test_cholesky_solve_op.py b/python/paddle/fluid/tests/unittests/test_cholesky_solve_op.py index fda558f13bcb4..5ed62fe2e0076 100644 --- a/python/paddle/fluid/tests/unittests/test_cholesky_solve_op.py +++ b/python/paddle/fluid/tests/unittests/test_cholesky_solve_op.py @@ -28,7 +28,7 @@ paddle.enable_static() -#cholesky_solve implement 1 +# cholesky_solve implement 1 def cholesky_solution(X, B, upper=True): if upper: A = np.triu(X) @@ -39,10 +39,11 @@ def cholesky_solution(X, B, upper=True): L = A U = A.T return scipy.linalg.solve_triangular( - U, scipy.linalg.solve_triangular(L, B, lower=True)) + U, scipy.linalg.solve_triangular(L, B, lower=True) + ) -#cholesky_solve implement 2 +# cholesky_solve implement 2 def scipy_cholesky_solution(X, B, upper=True): if upper: umat = np.triu(X) @@ -54,7 +55,7 @@ def scipy_cholesky_solution(X, B, upper=True): return scipy.linalg.cho_solve(K, B) -#broadcast function used by cholesky_solve +# broadcast function used by cholesky_solve def broadcast_shape(matA, matB): shapeA = matA.shape shapeB = matB.shape @@ -67,14 +68,16 @@ def broadcast_shape(matA, matB): Broadshape.append(max(shapeA[idx], shapeB[idx])) else: raise Exception( - 'shapeA and shapeB should be broadcasted, but got {} and {}'. - format(shapeA, shapeB)) + 'shapeA and shapeB should be broadcasted, but got {} and {}'.format( + shapeA, shapeB + ) + ) bsA = Broadshape + list(shapeA[-2:]) bsB = Broadshape + list(shapeB[-2:]) return np.broadcast_to(matA, bsA), np.broadcast_to(matB, bsB) -#cholesky_solve implement in batch +# cholesky_solve implement in batch def scipy_cholesky_solution_batch(bumat, bB, upper=True): bumat, bB = broadcast_shape(bumat, bB) ushape = bumat.shape @@ -99,19 +102,21 @@ class TestCholeskySolveOp(OpTest): case 1 """ - #test condition set + # test condition set def config(self): self.y_shape = [15, 15] self.x_shape = [15, 5] self.upper = False - self.dtype = np.float64 #Here cholesky_solve Op only supports float64/float32 type, please check others if Op supports more types. + self.dtype = ( + np.float64 + ) # Here cholesky_solve Op only supports float64/float32 type, please check others if Op supports more types. - #get scipy result + # get scipy result def set_output(self): umat = self.inputs['Y'] - self.output = scipy_cholesky_solution_batch(umat, - self.inputs['X'], - upper=self.upper) + self.output = scipy_cholesky_solution_batch( + umat, self.inputs['X'], upper=self.upper + ) def setUp(self): self.op_type = "cholesky_solve" @@ -124,17 +129,17 @@ def setUp(self): self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype), - 'Y': umat + 'Y': umat, } self.attrs = {'upper': self.upper} self.set_output() self.outputs = {'Out': self.output} - #check Op forward result + # check Op forward result def test_check_output(self): self.check_output() - #check Op grad + # check Op grad def test_check_grad_normal(self): self.check_grad(['Y'], 'Out', max_relative_error=0.01) @@ -152,9 +157,8 @@ def config(self): self.dtype = np.float64 -#API function test +# API function test class TestCholeskySolveAPI(unittest.TestCase): - def setUp(self): np.random.seed(2021) self.place = [paddle.CPUPlace()] @@ -180,22 +184,20 @@ def check_static_result(self, place): z2_np = scipy_cholesky_solution(umat, x_np, upper=self.upper) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "x": x_np, - "y": umat - }, - fetch_list=[z]) + fetches = exe.run( + fluid.default_main_program(), + feed={"x": x_np, "y": umat}, + fetch_list=[z], + ) np.testing.assert_allclose(fetches[0], z_np, rtol=1e-05) - #test in static mode + # test in static mode def test_static(self): for place in self.place: self.check_static_result(place=place) - #test in dynamic mode + # test in dynamic mode def test_dygraph(self): - def run(place): paddle.disable_static(place) x_np = np.random.random([20, 2]).astype(self.dtype) @@ -213,9 +215,8 @@ def run(place): for idx, place in enumerate(self.place): run(place) - #test input with broadcast + # test input with broadcast def test_broadcast(self): - def run(place): paddle.disable_static() x_np = np.random.random([1, 30, 2]).astype(self.dtype) @@ -234,17 +235,18 @@ def run(place): run(place) -#test condition out of bounds +# test condition out of bounds class TestCholeskySolveOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): # The input type of solve_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, paddle.linalg.cholesky_solve, x1, y1) # The data type of input must be float32 or float64. diff --git a/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py b/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py index b6110a296f031..b82404e01a403 100644 --- a/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py +++ b/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py @@ -21,15 +21,17 @@ class Segment(object): - def __init__(self, chunk_type, start_idx, end_idx): self.chunk_type = chunk_type self.start_idx = start_idx self.end_idx = end_idx def __str__(self): - return '(Segment: %s, %s, %s)' % (self.chunk_type, self.start_idx, - self.end_idx) + return '(Segment: %s, %s, %s)' % ( + self.chunk_type, + self.start_idx, + self.end_idx, + ) __repr__ = __str__ @@ -48,17 +50,24 @@ def fill_with_chunks(self, data, chunks): for chunk in chunks: if self.scheme == 'IOB': data[chunk.start_idx] = chunk.chunk_type * self.num_tag_types - data[chunk.start_idx + 1:chunk. - end_idx] = chunk.chunk_type * self.num_tag_types + ( - self.num_tag_types - 1) - data[chunk.end_idx] = chunk.chunk_type * self.num_tag_types + ( + data[ + chunk.start_idx + 1 : chunk.end_idx + ] = chunk.chunk_type * self.num_tag_types + ( self.num_tag_types - 1 - ) if chunk.start_idx < chunk.end_idx else data[chunk.start_idx] + ) + data[chunk.end_idx] = ( + chunk.chunk_type * self.num_tag_types + + (self.num_tag_types - 1) + if chunk.start_idx < chunk.end_idx + else data[chunk.start_idx] + ) elif self.scheme == 'IOE': - data[chunk.start_idx:chunk. - end_idx] = chunk.chunk_type * self.num_tag_types + data[chunk.start_idx : chunk.end_idx] = ( + chunk.chunk_type * self.num_tag_types + ) data[chunk.end_idx] = chunk.chunk_type * self.num_tag_types + ( - self.num_tag_types - 1) + self.num_tag_types - 1 + ) def rand_chunks(self, starts, num_chunks): if num_chunks < 0: @@ -66,15 +75,17 @@ def rand_chunks(self, starts, num_chunks): chunks = [] # generate chunk beginnings chunk_begins = sorted( - np.random.choice(list(range(starts[-1])), num_chunks, - replace=False)) + np.random.choice(list(range(starts[-1])), num_chunks, replace=False) + ) seq_chunk_begins = [] begin_idx = 0 # divide chunks into sequences for i in range(len(starts) - 1): tmp_chunk_begins = [] - while begin_idx < len( - chunk_begins) and chunk_begins[begin_idx] < starts[i + 1]: + while ( + begin_idx < len(chunk_begins) + and chunk_begins[begin_idx] < starts[i + 1] + ): tmp_chunk_begins.append(chunk_begins[begin_idx]) begin_idx += 1 seq_chunk_begins.append(tmp_chunk_begins) @@ -83,9 +94,11 @@ def rand_chunks(self, starts, num_chunks): for i in range(len(seq_chunk_begins)): for j in range(len(seq_chunk_begins[i])): low = seq_chunk_begins[i][j] - high = seq_chunk_begins[i][ - j + 1] if j < len(seq_chunk_begins[i]) - 1 else starts[i + - 1] + high = ( + seq_chunk_begins[i][j + 1] + if j < len(seq_chunk_begins[i]) - 1 + else starts[i + 1] + ) chunk_ends.append(np.random.randint(low, high)) # generate chunks for chunk_pos in zip(chunk_begins, chunk_ends): @@ -95,20 +108,25 @@ def rand_chunks(self, starts, num_chunks): def gen_chunks(self, infer, label, starts): chunks = self.rand_chunks( - starts, self.num_infer_chunks + self.num_label_chunks - - self.num_correct_chunks) - correct_chunks = np.random.choice(list(range(len(chunks))), - self.num_correct_chunks, - replace=False) + starts, + self.num_infer_chunks + + self.num_label_chunks + - self.num_correct_chunks, + ) + correct_chunks = np.random.choice( + list(range(len(chunks))), self.num_correct_chunks, replace=False + ) infer_chunks = np.random.choice( [x for x in range(len(chunks)) if x not in correct_chunks], self.num_infer_chunks - self.num_correct_chunks, - replace=False) + replace=False, + ) infer_chunks = sorted(correct_chunks.tolist() + infer_chunks.tolist()) label_chunks = np.random.choice( [x for x in range(len(chunks)) if x not in infer_chunks], self.num_label_chunks - self.num_correct_chunks, - replace=False) + replace=False, + ) label_chunks = sorted(correct_chunks.tolist() + label_chunks.tolist()) self.fill_with_chunks(infer, [chunks[idx] for idx in infer_chunks]) self.fill_with_chunks(label, [chunks[idx] for idx in label_chunks]) @@ -123,7 +141,11 @@ def gen_chunks(self, infer, label, starts): for idx in label_chunks: if chunks[idx].chunk_type in self.excluded_chunk_types: self.num_label_chunks -= 1 - return self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks + return ( + self.num_correct_chunks, + self.num_infer_chunks, + self.num_label_chunks, + ) def set_confs(self): # Use the IOB scheme and labels with 2 chunk types @@ -134,43 +156,63 @@ def set_confs(self): self.attrs = { 'num_chunk_types': self.num_chunk_types, 'chunk_scheme': self.scheme, - 'excluded_chunk_types': self.excluded_chunk_types + 'excluded_chunk_types': self.excluded_chunk_types, } self.parse_scheme() - self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks = 4, 5, 9 + ( + self.num_correct_chunks, + self.num_infer_chunks, + self.num_label_chunks, + ) = (4, 5, 9) def set_data(self): - infer = np.zeros((self.batch_size, )).astype('int64') + infer = np.zeros((self.batch_size,)).astype('int64') infer.fill(self.num_chunk_types * self.num_tag_types) label = np.copy(infer) - starts = np.random.choice(list(range(1, self.batch_size)), - self.num_sequences - 1, - replace=False).tolist() + starts = np.random.choice( + list(range(1, self.batch_size)), + self.num_sequences - 1, + replace=False, + ).tolist() starts.extend([0, self.batch_size]) starts = sorted(starts) - self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks = self.gen_chunks( - infer, label, starts) + ( + self.num_correct_chunks, + self.num_infer_chunks, + self.num_label_chunks, + ) = self.gen_chunks(infer, label, starts) lod = [] for i in range(len(starts) - 1): lod.append(starts[i + 1] - starts[i]) self.set_input(infer, label, lod) - precision = float( - self.num_correct_chunks - ) / self.num_infer_chunks if self.num_infer_chunks else 0 - recall = float(self.num_correct_chunks - ) / self.num_label_chunks if self.num_label_chunks else 0 - f1 = float(2 * precision * recall) / ( - precision + recall) if self.num_correct_chunks else 0 + precision = ( + float(self.num_correct_chunks) / self.num_infer_chunks + if self.num_infer_chunks + else 0 + ) + recall = ( + float(self.num_correct_chunks) / self.num_label_chunks + if self.num_label_chunks + else 0 + ) + f1 = ( + float(2 * precision * recall) / (precision + recall) + if self.num_correct_chunks + else 0 + ) self.outputs = { 'Precision': np.asarray([precision], dtype='float32'), 'Recall': np.asarray([recall], dtype='float32'), 'F1-Score': np.asarray([f1], dtype='float32'), - 'NumInferChunks': np.asarray([self.num_infer_chunks], - dtype='int64'), - 'NumLabelChunks': np.asarray([self.num_label_chunks], - dtype='int64'), - 'NumCorrectChunks': np.asarray([self.num_correct_chunks], - dtype='int64') + 'NumInferChunks': np.asarray( + [self.num_infer_chunks], dtype='int64' + ), + 'NumLabelChunks': np.asarray( + [self.num_label_chunks], dtype='int64' + ), + 'NumCorrectChunks': np.asarray( + [self.num_correct_chunks], dtype='int64' + ), } def set_input(self, infer, label, lod): @@ -186,7 +228,6 @@ def test_check_output(self): class TestChunkEvalOpWithExclude(TestChunkEvalOp): - def set_confs(self): # Use the IOE scheme and labels with 3 chunk types self.scheme = 'IOE' @@ -196,14 +237,17 @@ def set_confs(self): self.attrs = { 'num_chunk_types': self.num_chunk_types, 'chunk_scheme': self.scheme, - 'excluded_chunk_types': self.excluded_chunk_types + 'excluded_chunk_types': self.excluded_chunk_types, } self.parse_scheme() - self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks = 15, 18, 20 + ( + self.num_correct_chunks, + self.num_infer_chunks, + self.num_label_chunks, + ) = (15, 18, 20) class TestChunkEvalOpWithTensorInput(TestChunkEvalOp): - def set_input(self, infer, label, lod): max_len = np.max(lod) pad_infer = [] @@ -212,13 +256,21 @@ def set_input(self, infer, label, lod): for i in range(len(lod)): end = lod[i] + start pad_infer.append( - np.pad(infer[start:end], (0, max_len - lod[i]), - 'constant', - constant_values=(-1, ))) + np.pad( + infer[start:end], + (0, max_len - lod[i]), + 'constant', + constant_values=(-1,), + ) + ) pad_label.append( - np.pad(label[start:end], (0, max_len - lod[i]), - 'constant', - constant_values=(-1, ))) + np.pad( + label[start:end], + (0, max_len - lod[i]), + 'constant', + constant_values=(-1,), + ) + ) start = end pad_infer = np.expand_dims(np.array(pad_infer, dtype='int64'), 2) @@ -227,46 +279,51 @@ def set_input(self, infer, label, lod): self.inputs = { 'Inference': pad_infer, 'Label': pad_label, - 'SeqLength': lod + 'SeqLength': lod, } class TestChunkEvalOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_input(): input_data = np.random.random(1, 1).astype("int64") label_data = np.random.random(1).astype("int64") - fluid.layers.chunk_eval(input=input_data, - label=label_data, - chunk_scheme="IOB", - num_chunk_types=3) + fluid.layers.chunk_eval( + input=input_data, + label=label_data, + chunk_scheme="IOB", + num_chunk_types=3, + ) self.assertRaises(TypeError, test_input) def test_label(): - input_ = fluid.data(name="input", - shape=[None, 1], - dtype="int64") + input_ = fluid.data( + name="input", shape=[None, 1], dtype="int64" + ) label_data = np.random.random(1).astype("int64") - fluid.layers.chunk_eval(input=input_, - label=label_data, - chunk_scheme="IOB", - num_chunk_types=3) + fluid.layers.chunk_eval( + input=input_, + label=label_data, + chunk_scheme="IOB", + num_chunk_types=3, + ) self.assertRaises(TypeError, test_label) def test_type(): - in_data = fluid.data(name="input_", - shape=[None, 1], - dtype="int32") + in_data = fluid.data( + name="input_", shape=[None, 1], dtype="int32" + ) label = fluid.data(name="label_", shape=[1], dtype="int64") - fluid.layers.chunk_eval(input=in_data, - label=label, - chunk_scheme="IOB", - num_chunk_types=3) + fluid.layers.chunk_eval( + input=in_data, + label=label, + chunk_scheme="IOB", + num_chunk_types=3, + ) self.assertRaises(TypeError, test_type) diff --git a/python/paddle/fluid/tests/unittests/test_chunk_op.py b/python/paddle/fluid/tests/unittests/test_chunk_op.py index 896da7d4c1d2e..f29d90ceb73dc 100644 --- a/python/paddle/fluid/tests/unittests/test_chunk_op.py +++ b/python/paddle/fluid/tests/unittests/test_chunk_op.py @@ -21,7 +21,6 @@ class TestChunkOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The type of axis in chunk_op should be int or Variable. @@ -54,7 +53,6 @@ def test_axis_type_tensor(): class API_TestChunk(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = paddle.fluid.data('data1', shape=[4, 6, 6], dtype='float64') @@ -64,11 +62,9 @@ def test_out(self): exe = paddle.static.Executor(place) input1 = np.random.random([4, 6, 6]).astype('float64') input2 = np.array([2]).astype('int32') - r0, r1, r2, = exe.run(feed={ - "data1": input1, - "data2": input2 - }, - fetch_list=[x0, x1, x2]) + r0, r1, r2, = exe.run( + feed={"data1": input1, "data2": input2}, fetch_list=[x0, x1, x2] + ) ex_x0, ex_x1, ex_x2 = np.array_split(input1, 3, axis=2) np.testing.assert_allclose(ex_x0, r0, rtol=1e-05) np.testing.assert_allclose(ex_x1, r1, rtol=1e-05) @@ -76,7 +72,6 @@ def test_out(self): class API_TestChunk1(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = paddle.fluid.data('data1', shape=[4, 6, 6], dtype='float64') @@ -84,8 +79,11 @@ def test_out(self): place = paddle.CPUPlace() exe = paddle.static.Executor(place) input1 = np.random.random([4, 6, 6]).astype('float64') - r0, r1, r2, = exe.run(feed={"data1": input1}, - fetch_list=[x0, x1, x2]) + ( + r0, + r1, + r2, + ) = exe.run(feed={"data1": input1}, fetch_list=[x0, x1, x2]) ex_x0, ex_x1, ex_x2 = np.array_split(input1, 3, axis=2) np.testing.assert_allclose(ex_x0, r0, rtol=1e-05) np.testing.assert_allclose(ex_x1, r1, rtol=1e-05) @@ -93,7 +91,6 @@ def test_out(self): class API_TestDygraphChunk(unittest.TestCase): - def test_out1(self): with fluid.dygraph.guard(): input_1 = np.random.random([4, 6, 6]).astype("int32") diff --git a/python/paddle/fluid/tests/unittests/test_class_center_sample_op.py b/python/paddle/fluid/tests/unittests/test_class_center_sample_op.py index 28b9ffd88b406..53b9cfad34530 100644 --- a/python/paddle/fluid/tests/unittests/test_class_center_sample_op.py +++ b/python/paddle/fluid/tests/unittests/test_class_center_sample_op.py @@ -28,10 +28,13 @@ def class_center_sample_numpy(label, classes_list, num_samples): unique_label_per_device = [] for i in range(nranks): - index = np.logical_and(unique_label >= class_interval[i], - unique_label < class_interval[i + 1]) - pos_class_center_per_device.append(unique_label[index] - - class_interval[i]) + index = np.logical_and( + unique_label >= class_interval[i], + unique_label < class_interval[i + 1], + ) + pos_class_center_per_device.append( + unique_label[index] - class_interval[i] + ) unique_label_per_device.append(unique_label[index]) num_samples_per_device = [] @@ -41,8 +44,9 @@ def class_center_sample_numpy(label, classes_list, num_samples): remapped_dict = {} for i in range(nranks): - for idx, v in enumerate(unique_label_per_device[i], - sampled_class_interval[i]): + for idx, v in enumerate( + unique_label_per_device[i], sampled_class_interval[i] + ): remapped_dict[v] = idx remapped_label = [] @@ -62,14 +66,12 @@ def python_api( fix_seed=False, seed=0, ): - return paddle.nn.functional.class_center_sample(label, - num_classes=num_classes, - num_samples=num_samples, - group=None) + return paddle.nn.functional.class_center_sample( + label, num_classes=num_classes, num_samples=num_samples, group=None + ) class TestClassCenterSampleOp(OpTest): - def initParams(self): self.op_type = "class_center_sample" self.python_api = python_api @@ -88,17 +90,18 @@ def setUp(self): self.initParams() self.init_dtype() self.init_fix_seed() - label = np.random.randint(0, - self.num_classes, (self.batch_size, ), - dtype=self.dtype) + label = np.random.randint( + 0, self.num_classes, (self.batch_size,), dtype=self.dtype + ) remapped_label, sampled_class_center = class_center_sample_numpy( - label, [self.num_classes], self.num_samples) + label, [self.num_classes], self.num_samples + ) self.inputs = {'Label': label} self.outputs = { 'RemappedLabel': remapped_label.astype(self.dtype), - 'SampledLocalClassCenter': sampled_class_center.astype(self.dtype) + 'SampledLocalClassCenter': sampled_class_center.astype(self.dtype), } self.attrs = { @@ -109,24 +112,22 @@ def setUp(self): } def test_check_output(self): - self.check_output(no_check_set=['SampledLocalClassCenter'], - check_eager=True) + self.check_output( + no_check_set=['SampledLocalClassCenter'], check_eager=True + ) class TestClassCenterSampleOpINT32(TestClassCenterSampleOp): - def init_dtype(self): self.dtype = np.int32 class TestClassCenterSampleOpFixSeed(TestClassCenterSampleOp): - def init_fix_seed(self): self.fix_seed = True class TestClassCenterSampleV2(unittest.TestCase): - def setUp(self): self.initParams() np.random.seed(self.seed) @@ -151,27 +152,37 @@ def test_static(self): def check_static_result(self, place): with program_guard(Program(), Program()): - label_np = np.random.randint(0, - self.num_classes, (self.batch_size, ), - dtype=self.dtype) - - label = paddle.static.data(name='label', - shape=[self.batch_size], - dtype=self.dtype) - remapped_label, sampled_class_index = paddle.nn.functional.class_center_sample( - label, self.num_classes, self.num_samples) - - remapped_label_np, sampled_class_center_np = class_center_sample_numpy( - label_np, [self.num_classes], self.num_samples) + label_np = np.random.randint( + 0, self.num_classes, (self.batch_size,), dtype=self.dtype + ) + + label = paddle.static.data( + name='label', shape=[self.batch_size], dtype=self.dtype + ) + ( + remapped_label, + sampled_class_index, + ) = paddle.nn.functional.class_center_sample( + label, self.num_classes, self.num_samples + ) + + ( + remapped_label_np, + sampled_class_center_np, + ) = class_center_sample_numpy( + label_np, [self.num_classes], self.num_samples + ) exe = paddle.fluid.Executor(place) - [remapped_label_res, sampled_class_index_res - ] = exe.run(paddle.fluid.default_main_program(), - feed={'label': label_np}, - fetch_list=[remapped_label, sampled_class_index]) + [remapped_label_res, sampled_class_index_res] = exe.run( + paddle.fluid.default_main_program(), + feed={'label': label_np}, + fetch_list=[remapped_label, sampled_class_index], + ) np.testing.assert_allclose(remapped_label_res, remapped_label_np) np.testing.assert_allclose( - sampled_class_index_res[:len(sampled_class_center_np[0])], - sampled_class_center_np[0]) + sampled_class_index_res[: len(sampled_class_center_np[0])], + sampled_class_center_np[0], + ) def test_dynamic(self): for place in self.places: @@ -179,33 +190,40 @@ def test_dynamic(self): def check_dynamic_result(self, place): with paddle.fluid.dygraph.guard(place): - label_np = np.random.randint(0, - self.num_classes, (self.batch_size, ), - dtype=self.dtype) + label_np = np.random.randint( + 0, self.num_classes, (self.batch_size,), dtype=self.dtype + ) label = paddle.to_tensor(label_np, dtype=self.dtype) - remapped_label, sampled_class_index = paddle.nn.functional.class_center_sample( - label, self.num_classes, self.num_samples) + ( + remapped_label, + sampled_class_index, + ) = paddle.nn.functional.class_center_sample( + label, self.num_classes, self.num_samples + ) - remapped_label_np, sampled_class_center_np = class_center_sample_numpy( - label_np, [self.num_classes], self.num_samples) + ( + remapped_label_np, + sampled_class_center_np, + ) = class_center_sample_numpy( + label_np, [self.num_classes], self.num_samples + ) remapped_label_res = remapped_label.numpy() sampled_class_index_res = sampled_class_index.numpy() np.testing.assert_allclose(remapped_label_res, remapped_label_np) np.testing.assert_allclose( - sampled_class_index_res[:len(sampled_class_center_np[0])], - sampled_class_center_np[0]) + sampled_class_index_res[: len(sampled_class_center_np[0])], + sampled_class_center_np[0], + ) class TestClassCenterSampleV2INT32(TestClassCenterSampleV2): - def init_dtype(self): self.dtype = np.int32 class TestClassCenterSampleAPIError(unittest.TestCase): - def setUp(self): self.initParams() np.random.seed(self.seed) @@ -224,24 +242,28 @@ def init_dtype(self): self.dtype = np.int64 def test_dynamic_errors(self): - def test_num_samples(): for place in self.places: with paddle.fluid.dygraph.guard(place): - label_np = np.random.randint(0, - self.num_classes, - (self.batch_size, ), - dtype=self.dtype) + label_np = np.random.randint( + 0, + self.num_classes, + (self.batch_size,), + dtype=self.dtype, + ) label = paddle.to_tensor(label_np) - remapped_label, sampled_class_index = paddle.nn.functional.class_center_sample( - label, self.num_classes, self.num_samples) + ( + remapped_label, + sampled_class_index, + ) = paddle.nn.functional.class_center_sample( + label, self.num_classes, self.num_samples + ) self.assertRaises(ValueError, test_num_samples) class TestClassCenterSampleAPIError1(unittest.TestCase): - def setUp(self): self.initParams() np.random.seed(self.seed) @@ -260,26 +282,35 @@ def init_dtype(self): self.dtype = np.int64 def test_dynamic_errors(self): - def test_empty_label(): for place in self.places: with paddle.fluid.dygraph.guard(place): label = paddle.to_tensor(np.array([], dtype=self.dtype)) - remapped_label, sampled_class_index = paddle.nn.functional.class_center_sample( - label, self.num_classes, self.num_samples) + ( + remapped_label, + sampled_class_index, + ) = paddle.nn.functional.class_center_sample( + label, self.num_classes, self.num_samples + ) def test_group_value(): for place in self.places: with paddle.fluid.dygraph.guard(place): - label_np = np.random.randint(0, - self.num_classes, - (self.batch_size, ), - dtype=self.dtype) + label_np = np.random.randint( + 0, + self.num_classes, + (self.batch_size,), + dtype=self.dtype, + ) label = paddle.to_tensor(label_np) - remapped_label, sampled_class_index = paddle.nn.functional.class_center_sample( - label, self.num_classes, self.num_samples, group=True) + ( + remapped_label, + sampled_class_index, + ) = paddle.nn.functional.class_center_sample( + label, self.num_classes, self.num_samples, group=True + ) self.assertRaises(ValueError, test_empty_label) self.assertRaises(ValueError, test_group_value) diff --git a/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py b/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py index ec95442a37cb3..d7b9c6cd52855 100644 --- a/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py @@ -22,7 +22,6 @@ class TestClipByNormOp(OpTest): - def setUp(self): self.max_relative_error = 0.006 self.python_api = fluid.layers.clip_by_norm @@ -47,7 +46,7 @@ def test_check_output(self): self.check_output(check_eager=True) def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1.0 def init_dtype(self): @@ -55,28 +54,24 @@ def init_dtype(self): class TestCase1(TestClipByNormOp): - def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1e20 class TestCase2(TestClipByNormOp): - def initTestCase(self): self.shape = (16, 16) self.max_norm = 0.1 class TestCase3(TestClipByNormOp): - def initTestCase(self): self.shape = (4, 8, 16) self.max_norm = 1.0 class TestClipByNormOpFp16(TestClipByNormOp): - def init_dtype(self): self.dtype = np.float16 @@ -84,34 +79,30 @@ def test_check_output(self): if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_output_with_place(place, - atol=0.001, - check_eager=True) + self.check_output_with_place( + place, atol=0.001, check_eager=True + ) class TestClipByNormOpFp16Case1(TestClipByNormOpFp16): - def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1e20 class TestClipByNormOpFp16Case2(TestClipByNormOpFp16): - def initTestCase(self): self.shape = (16, 16) self.max_norm = 0.1 class TestClipByNormOpFp16Case3(TestClipByNormOpFp16): - def initTestCase(self): self.shape = (4, 8, 16) self.max_norm = 1.0 class TestClipByNormOpWithSelectedRows(unittest.TestCase): - def check_with_place(self, place): self.config_test_case() scope = core.Scope() @@ -128,10 +119,9 @@ def check_with_place(self, place): out_selected_rows = scope.var('Out').get_selected_rows() # run clip_by_norm_op - clip_by_norm_op = fluid.op.Operator("clip_by_norm", - max_norm=self.max_norm, - X='X', - Out='Out') + clip_by_norm_op = fluid.op.Operator( + "clip_by_norm", max_norm=self.max_norm, X='X', Out='Out' + ) clip_by_norm_op.run(scope, place) # check output @@ -146,11 +136,13 @@ def check_with_place(self, place): output = self.max_norm * y_np / norm else: output = y_np - np.testing.assert_allclose(np.array(out_tensor), - output, - rtol=1e-05, - atol=1e-05, - equal_nan=False) + np.testing.assert_allclose( + np.array(out_tensor), + output, + rtol=1e-05, + atol=1e-05, + equal_nan=False, + ) def test_clip_by_norm_with_selected_ros(self): places = [core.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_clip_op.py b/python/paddle/fluid/tests/unittests/test_clip_op.py index a8e3b90af90f8..e955bd71a6430 100644 --- a/python/paddle/fluid/tests/unittests/test_clip_op.py +++ b/python/paddle/fluid/tests/unittests/test_clip_op.py @@ -22,7 +22,6 @@ class TestClipOp(OpTest): - def setUp(self): self.max_relative_error = 0.006 self.python_api = paddle.clip @@ -70,7 +69,6 @@ def initTestCase(self): class TestCase1(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (8, 16, 8) @@ -79,7 +77,6 @@ def initTestCase(self): class TestCase2(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (8, 16) @@ -88,7 +85,6 @@ def initTestCase(self): class TestCase3(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (4, 8, 16) @@ -97,7 +93,6 @@ def initTestCase(self): class TestCase4(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (4, 8, 8) @@ -108,7 +103,6 @@ def initTestCase(self): class TestCase5(TestClipOp): - def initTestCase(self): self.dtype = np.float32 self.shape = (4, 8, 16) @@ -117,7 +111,6 @@ def initTestCase(self): class TestCase6(TestClipOp): - def initTestCase(self): self.dtype == np.float16 self.shape = (4, 8, 8) @@ -128,7 +121,6 @@ def initTestCase(self): class TestClipOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -148,7 +140,6 @@ def test_dtype(): class TestClipAPI(unittest.TestCase): - def _executed_api(self, x, min=None, max=None): return paddle.clip(x, min, max) @@ -160,8 +151,11 @@ def test_clip(self): min = fluid.data(name='min', shape=[1], dtype='float32') max = fluid.data(name='max', shape=[1], dtype='float32') - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) out_1 = self._executed_api(images, min=min, max=max) @@ -170,29 +164,51 @@ def test_clip(self): out_4 = self._executed_api(images, max=0.7) out_5 = self._executed_api(images, min=min) out_6 = self._executed_api(images, max=max) - out_7 = self._executed_api(images, max=-1.) + out_7 = self._executed_api(images, max=-1.0) out_8 = self._executed_api(images) - out_9 = self._executed_api(paddle.cast(images, 'float64'), - min=0.2, - max=0.9) - out_10 = self._executed_api(paddle.cast(images * 10, 'int32'), - min=2, - max=8) - out_11 = self._executed_api(paddle.cast(images * 10, 'int64'), - min=2, - max=8) - - res1, res2, res3, res4, res5, res6, res7, res8, res9, res10, res11 = exe.run( + out_9 = self._executed_api( + paddle.cast(images, 'float64'), min=0.2, max=0.9 + ) + out_10 = self._executed_api( + paddle.cast(images * 10, 'int32'), min=2, max=8 + ) + out_11 = self._executed_api( + paddle.cast(images * 10, 'int64'), min=2, max=8 + ) + + ( + res1, + res2, + res3, + res4, + res5, + res6, + res7, + res8, + res9, + res10, + res11, + ) = exe.run( fluid.default_main_program(), feed={ "image": data, "min": np.array([0.2]).astype('float32'), - "max": np.array([0.8]).astype('float32') + "max": np.array([0.8]).astype('float32'), }, fetch_list=[ - out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8, out_9, - out_10, out_11 - ]) + out_1, + out_2, + out_3, + out_4, + out_5, + out_6, + out_7, + out_8, + out_9, + out_10, + out_11, + ], + ) np.testing.assert_allclose(res1, data.clip(0.2, 0.8), rtol=1e-05) np.testing.assert_allclose(res2, data.clip(0.2, 0.9), rtol=1e-05) @@ -202,21 +218,24 @@ def test_clip(self): np.testing.assert_allclose(res6, data.clip(max=0.8), rtol=1e-05) np.testing.assert_allclose(res7, data.clip(max=-1), rtol=1e-05) np.testing.assert_allclose(res8, data, rtol=1e-05) - np.testing.assert_allclose(res9, - data.astype(np.float64).clip(0.2, 0.9), - rtol=1e-05) - np.testing.assert_allclose(res10, - (data * 10).astype(np.int32).clip(2, 8), - rtol=1e-05) - np.testing.assert_allclose(res11, - (data * 10).astype(np.int64).clip(2, 8), - rtol=1e-05) + np.testing.assert_allclose( + res9, data.astype(np.float64).clip(0.2, 0.9), rtol=1e-05 + ) + np.testing.assert_allclose( + res10, (data * 10).astype(np.int32).clip(2, 8), rtol=1e-05 + ) + np.testing.assert_allclose( + res11, (data * 10).astype(np.int64).clip(2, 8), rtol=1e-05 + ) paddle.disable_static() def func_clip_dygraph(self): paddle.disable_static() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) paddle.disable_static(place) data_shape = [1, 9, 9, 4] data = np.random.random(data_shape).astype('float32') @@ -230,33 +249,33 @@ def func_clip_dygraph(self): images = paddle.to_tensor(data, dtype='float32') out_3 = self._executed_api(images, min=v_min, max=v_max) - out_4 = self._executed_api(paddle.cast(images * 10, 'int32'), - min=2, - max=8) - out_5 = self._executed_api(paddle.cast(images * 10, 'int64'), - min=2, - max=8) + out_4 = self._executed_api( + paddle.cast(images * 10, 'int32'), min=2, max=8 + ) + out_5 = self._executed_api( + paddle.cast(images * 10, 'int64'), min=2, max=8 + ) # test with numpy.generic out_6 = self._executed_api(images, min=np.abs(0.2), max=np.abs(0.8)) - np.testing.assert_allclose(out_1.numpy(), - data.clip(0.2, 0.8), - rtol=1e-05) - np.testing.assert_allclose(out_2.numpy(), - data.clip(0.2, 0.9), - rtol=1e-05) - np.testing.assert_allclose(out_3.numpy(), - data.clip(0.2, 0.8), - rtol=1e-05) - np.testing.assert_allclose(out_4.numpy(), - (data * 10).astype(np.int32).clip(2, 8), - rtol=1e-05) - np.testing.assert_allclose(out_5.numpy(), - (data * 10).astype(np.int64).clip(2, 8), - rtol=1e-05) - np.testing.assert_allclose(out_6.numpy(), - data.clip(0.2, 0.8), - rtol=1e-05) + np.testing.assert_allclose( + out_1.numpy(), data.clip(0.2, 0.8), rtol=1e-05 + ) + np.testing.assert_allclose( + out_2.numpy(), data.clip(0.2, 0.9), rtol=1e-05 + ) + np.testing.assert_allclose( + out_3.numpy(), data.clip(0.2, 0.8), rtol=1e-05 + ) + np.testing.assert_allclose( + out_4.numpy(), (data * 10).astype(np.int32).clip(2, 8), rtol=1e-05 + ) + np.testing.assert_allclose( + out_5.numpy(), (data * 10).astype(np.int64).clip(2, 8), rtol=1e-05 + ) + np.testing.assert_allclose( + out_6.numpy(), data.clip(0.2, 0.8), rtol=1e-05 + ) def test_clip_dygraph(self): with _test_eager_guard(): @@ -292,7 +311,6 @@ def test_errors(self): class TestInplaceClipAPI(TestClipAPI): - def _executed_api(self, x, min=None, max=None): return x.clip_(min, max) diff --git a/python/paddle/fluid/tests/unittests/test_coalesce_tensor_op.py b/python/paddle/fluid/tests/unittests/test_coalesce_tensor_op.py index 2bbac7a09e9b3..bcd9eb412bc99 100644 --- a/python/paddle/fluid/tests/unittests/test_coalesce_tensor_op.py +++ b/python/paddle/fluid/tests/unittests/test_coalesce_tensor_op.py @@ -20,30 +20,40 @@ import paddle -def coalesce_tensor_eager_api(Input, - datatype=core.VarDesc.VarType.FP32, - copy_data=False, - set_constant=False, - persist_output=False, - constant=0.0, - use_align=True, - align_size=-1, - user_defined_size_of_dtype=-1, - concated_shapes=[], - concated_ranks=[]): +def coalesce_tensor_eager_api( + Input, + datatype=core.VarDesc.VarType.FP32, + copy_data=False, + set_constant=False, + persist_output=False, + constant=0.0, + use_align=True, + align_size=-1, + user_defined_size_of_dtype=-1, + concated_shapes=[], + concated_ranks=[], +): if datatype == int(core.VarDesc.VarType.FP32): datatype = core.VarDesc.VarType.FP32 - return paddle._C_ops.coalesce_tensor(Input, datatype, copy_data, - set_constant, persist_output, constant, - use_align, align_size, - user_defined_size_of_dtype, - concated_shapes, concated_ranks) - - -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + return paddle._C_ops.coalesce_tensor( + Input, + datatype, + copy_data, + set_constant, + persist_output, + constant, + use_align, + align_size, + user_defined_size_of_dtype, + concated_shapes, + concated_ranks, + ) + + +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestAllocContinuousSpace(OpTest): - def setUp(self): self.python_api = coalesce_tensor_eager_api self.op_type = "coalesce_tensor" @@ -51,7 +61,8 @@ def setUp(self): self.attrs = self.init_attr() self.Inputs = self.init_input() self.Outputs, self.FusedOutput = self.init_output( - self.Inputs, self.attrs["set_constant"], self.attrs["constant"]) + self.Inputs, self.attrs["set_constant"], self.attrs["constant"] + ) self.inputs = {'Input': self.Inputs} self.outputs = {'Output': self.Outputs, 'FusedOutput': self.FusedOutput} @@ -73,7 +84,7 @@ def init_attr(self): "copy_data": True, "set_constant": False, "constant": 0.0, - "dtype": self.fluid_dtype + "dtype": self.fluid_dtype, } def init_output(self, input_list, set_constant, constant): @@ -94,9 +105,10 @@ def init_output(self, input_list, set_constant, constant): coalesce_tensor_var = np.concatenate([input for input in inputs]) if set_constant: coalesce_tensor_var = np.ones((len(coalesce_tensor_var))) * constant - outputs = [(out[0], - np.ones(out[1].shape).astype(self.dtype) * constant) - for out in outputs] + outputs = [ + (out[0], np.ones(out[1].shape).astype(self.dtype) * constant) + for out in outputs + ] return outputs, coalesce_tensor_var def verify_output(self, place): @@ -109,54 +121,65 @@ def verify_output(self, place): tensor_input, datatype=self.attrs["dtype"], copy_data=self.attrs["copy_data"] - if "copy_data" in self.attrs else False, + if "copy_data" in self.attrs + else False, set_constant=self.attrs["set_constant"] - if "set_constant" in self.attrs else False, + if "set_constant" in self.attrs + else False, persist_output=False, constant=self.attrs["constant"] - if "constant" in self.attrs else 0.0, + if "constant" in self.attrs + else 0.0, use_align=True, align_size=-1, - user_defined_size_of_dtype=self. - attrs["user_defined_size_of_dtype"] - if "user_defined_size_of_dtype" in self.attrs else -1, + user_defined_size_of_dtype=self.attrs[ + "user_defined_size_of_dtype" + ] + if "user_defined_size_of_dtype" in self.attrs + else -1, concated_shapes=[], - concated_ranks=[]) + concated_ranks=[], + ) for idx, (expected, eager_output) in enumerate( - zip(self.outputs['Output'], eager_outputs)): - np.testing.assert_allclose(expected[1], - eager_output, - atol=1e-5, - err_msg=f'not equal {idx}') - np.testing.assert_allclose(self.outputs['FusedOutput'], - eager_fused_output, - atol=1e-5, - err_msg='not equal fusedoutput') + zip(self.outputs['Output'], eager_outputs) + ): + np.testing.assert_allclose( + expected[1], + eager_output, + atol=1e-5, + err_msg=f'not equal {idx}', + ) + np.testing.assert_allclose( + self.outputs['FusedOutput'], + eager_fused_output, + atol=1e-5, + err_msg='not equal fusedoutput', + ) def test_check_output(self): - self.check_output_with_place(place=core.CUDAPlace(0), - no_check_set=["FusedOutput"], - atol=1e-5) + self.check_output_with_place( + place=core.CUDAPlace(0), no_check_set=["FusedOutput"], atol=1e-5 + ) self.verify_output(core.CUDAPlace(0)) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestAllocContinuousSpace2(TestAllocContinuousSpace): - def init_attr(self): return { "copy_data": False, "set_constant": True, "constant": 0.5, "dtype": self.fluid_dtype, - "user_defined_size_of_dtype": 2 + "user_defined_size_of_dtype": 2, } def test_check_output(self): - self.check_output_with_place(place=core.CUDAPlace(0), - no_check_set=["FusedOutput"], - atol=1e-5) + self.check_output_with_place( + place=core.CUDAPlace(0), no_check_set=["FusedOutput"], atol=1e-5 + ) self.verify_output(core.CUDAPlace(0)) diff --git a/python/paddle/fluid/tests/unittests/test_collect_fpn_proposals_op.py b/python/paddle/fluid/tests/unittests/test_collect_fpn_proposals_op.py index f06102fb2f2b6..06011ee62d8df 100644 --- a/python/paddle/fluid/tests/unittests/test_collect_fpn_proposals_op.py +++ b/python/paddle/fluid/tests/unittests/test_collect_fpn_proposals_op.py @@ -18,27 +18,29 @@ class TestCollectFPNProposalstOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() - self.scores_input = [('y%d' % i, (self.scores[i].reshape(-1, 1), - self.rois_lod[i])) - for i in range(self.num_level)] + self.scores_input = [ + ('y%d' % i, (self.scores[i].reshape(-1, 1), self.rois_lod[i])) + for i in range(self.num_level) + ] self.rois, self.lod = self.calc_rois_collect() - inputs_x = [('x%d' % i, (self.roi_inputs[i][:, 1:], self.rois_lod[i])) - for i in range(self.num_level)] + inputs_x = [ + ('x%d' % i, (self.roi_inputs[i][:, 1:], self.rois_lod[i])) + for i in range(self.num_level) + ] self.inputs = { 'MultiLevelRois': inputs_x, "MultiLevelScores": self.scores_input, - 'MultiLevelRoIsNum': [] + 'MultiLevelRoIsNum': [], } self.attrs = { 'post_nms_topN': self.post_nms_top_n, } self.outputs = { 'FpnRois': (self.rois, [self.lod]), - 'RoisNum': np.array(self.lod).astype('int32') + 'RoisNum': np.array(self.lod).astype('int32'), } def init_test_case(self): @@ -61,7 +63,7 @@ def resort_roi_by_batch_id(self, rois): def calc_rois_collect(self): roi_inputs = np.concatenate(self.roi_inputs) scores = np.concatenate(self.scores) - inds = np.argsort(-scores)[:self.post_nms_top_n] + inds = np.argsort(-scores)[: self.post_nms_top_n] rois = roi_inputs[inds, :] new_rois, new_lod = self.resort_roi_by_batch_id(rois) return new_rois, new_lod @@ -100,31 +102,34 @@ def test_check_output(self): class TestCollectFPNProposalstOpWithRoisNum(TestCollectFPNProposalstOp): - def set_data(self): self.init_test_case() self.make_rois() - self.scores_input = [('y%d' % i, (self.scores[i].reshape(-1, 1), - self.rois_lod[i])) - for i in range(self.num_level)] + self.scores_input = [ + ('y%d' % i, (self.scores[i].reshape(-1, 1), self.rois_lod[i])) + for i in range(self.num_level) + ] self.rois, self.lod = self.calc_rois_collect() - inputs_x = [('x%d' % i, (self.roi_inputs[i][:, 1:], self.rois_lod[i])) - for i in range(self.num_level)] - rois_num_per_level = [('rois%d' % i, - np.array(self.rois_lod[i][0]).astype('int32')) - for i in range(self.num_level)] + inputs_x = [ + ('x%d' % i, (self.roi_inputs[i][:, 1:], self.rois_lod[i])) + for i in range(self.num_level) + ] + rois_num_per_level = [ + ('rois%d' % i, np.array(self.rois_lod[i][0]).astype('int32')) + for i in range(self.num_level) + ] self.inputs = { 'MultiLevelRois': inputs_x, "MultiLevelScores": self.scores_input, - 'MultiLevelRoIsNum': rois_num_per_level + 'MultiLevelRoIsNum': rois_num_per_level, } self.attrs = { 'post_nms_topN': self.post_nms_top_n, } self.outputs = { 'FpnRois': (self.rois, [self.lod]), - 'RoisNum': np.array(self.lod).astype('int32') + 'RoisNum': np.array(self.lod).astype('int32'), } diff --git a/python/paddle/fluid/tests/unittests/test_collective_api_base.py b/python/paddle/fluid/tests/unittests/test_collective_api_base.py index 3a3e5d99ca709..4f4be4e2dcf4d 100644 --- a/python/paddle/fluid/tests/unittests/test_collective_api_base.py +++ b/python/paddle/fluid/tests/unittests/test_collective_api_base.py @@ -82,7 +82,12 @@ def create_test_data(shape=None, dtype=None, seed=None): return create_float_test_data(shape=shape, dtype=bfloat16, seed=seed) elif dtype == "bool": return create_bool_test_data(shape=shape, seed=seed) - elif dtype == "int32" or dtype == "int64" or dtype == "int8" or dtype == "uint8": + elif ( + dtype == "int32" + or dtype == "int64" + or dtype == "int8" + or dtype == "uint8" + ): return create_int_test_data(shape=shape, dtype=dtype, seed=seed) elif dtype == "complex64" or dtype == "complex128": return create_complex_test_data(shape=shape, dtype=dtype, seed=seed) @@ -95,15 +100,12 @@ def create_test_data(shape=None, dtype=None, seed=None): class TestCollectiveAPIRunnerBase(object): - - def get_model(self, - train_prog, - startup_prog, - rank, - indata=None, - dtype=None): + def get_model( + self, train_prog, startup_prog, rank, indata=None, dtype=None + ): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def run_trainer(self, args): train_prog = fluid.Program() @@ -116,15 +118,16 @@ def run_trainer(self, args): if args['backend'] == 'nccl': device_id = int(os.getenv("FLAGS_selected_gpus", "0")) place = fluid.CUDAPlace( - device_id) #if args.use_gpu else fluid.CPUPlace() + device_id + ) # if args.use_gpu else fluid.CPUPlace() elif args['backend'] == 'bkcl': device_id = int(os.getenv("FLAGS_selected_xpus", "0")) place = fluid.XPUPlace(device_id) else: place = fluid.CPUPlace() - indata = create_test_data(shape=(10, 1000), - dtype=args["dtype"], - seed=os.getpid()) + indata = create_test_data( + shape=(10, 1000), dtype=args["dtype"], seed=os.getpid() + ) if args['static_mode']: result = self.get_model(train_prog, startup_prog, rank) exe = fluid.Executor(place) @@ -132,12 +135,12 @@ def run_trainer(self, args): fetch_list = [] for elem in result: fetch_list.append(elem.name) - out = exe.run(train_prog, - feed={'tindata': indata}, - fetch_list=fetch_list) + out = exe.run( + train_prog, feed={'tindata': indata}, fetch_list=fetch_list + ) else: out = self.get_model(train_prog, startup_prog, rank, indata) - #print(out, sys.stderr) + # print(out, sys.stderr) sys.stdout.buffer.write(pickle.dumps(out)) @@ -161,12 +164,13 @@ def runtime_main(test_class, col_type): class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable self.temp_dir = tempfile.TemporaryDirectory() @@ -176,18 +180,20 @@ def setUp(self): nccl_version_str = subprocess.check_output( r"ldconfig -v | grep 'libnccl.so' | tail -n1 | sed -r 's/^.*\.so\.//'", stderr=subprocess.DEVNULL, - shell=True).decode('utf-8') - self._nccl_version = int("".join( - nccl_version_str.split("."))) if nccl_version_str else 0 + shell=True, + ).decode('utf-8') + self._nccl_version = ( + int("".join(nccl_version_str.split("."))) if nccl_version_str else 0 + ) def tearDown(self): self.temp_dir.cleanup() def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -200,14 +206,14 @@ def __free_port(): def _run_cluster(self, model_file, envs): worker_endpoints = self._ps_endpoints.split(",") w0_ep, w1_ep = worker_endpoints - #print("w0_ep:",w0_ep," w1_ep:",w1_ep) + # print("w0_ep:",w0_ep," w1_ep:",w1_ep) if core.is_compiled_with_cuda(): env0 = { "FLAGS_selected_gpus": "0", "PADDLE_TRAINER_ID": "0", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w0_ep + "PADDLE_CURRENT_ENDPOINT": w0_ep, } env1 = { @@ -215,7 +221,7 @@ def _run_cluster(self, model_file, envs): "PADDLE_TRAINER_ID": "1", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w1_ep + "PADDLE_CURRENT_ENDPOINT": w1_ep, } elif core.is_compiled_with_xpu(): env0 = { @@ -223,7 +229,7 @@ def _run_cluster(self, model_file, envs): "PADDLE_TRAINER_ID": "0", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w0_ep + "PADDLE_CURRENT_ENDPOINT": w0_ep, } env1 = { @@ -231,9 +237,9 @@ def _run_cluster(self, model_file, envs): "PADDLE_TRAINER_ID": "1", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w1_ep + "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) if os.getenv('WITH_COVERAGE', 'OFF') == 'ON': @@ -242,22 +248,28 @@ def _run_cluster(self, model_file, envs): tr_cmd = "%s %s" tr0_cmd = tr_cmd % (self._python_interp, model_file) tr1_cmd = tr_cmd % (self._python_interp, model_file) - path0 = os.path.join(self.temp_dir.name, - "/tmp/tr0_err_%d.log" % os.getpid()) - path1 = os.path.join(self.temp_dir.name, - "/tmp/tr1_err_%d.log" % os.getpid()) + path0 = os.path.join( + self.temp_dir.name, "/tmp/tr0_err_%d.log" % os.getpid() + ) + path1 = os.path.join( + self.temp_dir.name, "/tmp/tr1_err_%d.log" % os.getpid() + ) tr0_pipe = open(path0, "w") tr1_pipe = open(path1, "w") - #print(tr0_cmd) - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) - - tr1_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + # print(tr0_cmd) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) + + tr1_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -270,19 +282,25 @@ def _run_cluster(self, model_file, envs): sys.stderr.write('trainer 0 stderr file: %s\n' % f.read()) with open(path1, "r") as f: sys.stderr.write('trainer 1 stderr file: %s\n' % f.read()) - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid - - def check_with_place(self, - model_file, - col_type, - backend="nccl", - path_id="0", - static_mode="1", - check_error_log=False, - need_envs={}, - eager_mode=True, - dtype=None): + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) + + def check_with_place( + self, + model_file, + col_type, + backend="nccl", + path_id="0", + static_mode="1", + check_error_log=False, + need_envs={}, + eager_mode=True, + dtype=None, + ): if backend == "nccl" or backend == "bkcl": with_gloo = '0' else: @@ -296,7 +314,7 @@ def check_with_place(self, "PADDLE_DISTRI_BACKEND": backend, "BACKEND": backend, "PATH_ID": path_id, - "DTYPE": dtype + "DTYPE": dtype, } required_envs.update(additional_envs) required_envs.update(need_envs) @@ -307,7 +325,8 @@ def check_with_place(self, if os.getenv('NVIDIA_TF32_OVERRIDE', '') is not None: required_envs['NVIDIA_TF32_OVERRIDE'] = os.getenv( - 'NVIDIA_TF32_OVERRIDE', '') + 'NVIDIA_TF32_OVERRIDE', '' + ) if eager_mode: required_envs["FLAGS_enable_eager_mode"] = "%d" % 1 @@ -315,7 +334,8 @@ def check_with_place(self, required_envs["FLAGS_enable_eager_mode"] = "%d" % 0 tr0_out, tr1_out, pid0, pid1 = self._run_cluster( - model_file, required_envs) + model_file, required_envs + ) input1 = create_test_data(shape=(10, 1000), dtype=dtype, seed=pid0) input2 = create_test_data(shape=(10, 1000), dtype=dtype, seed=pid1) # cast bfloat16 to float32 for numeric comparison @@ -347,14 +367,14 @@ def check_with_place(self, np.testing.assert_allclose(tr0_out[0], need_result, rtol=rtol) elif col_type == "scatter": need_result = input2 - need_result1 = need_result[0:need_result.shape[0] // 2] - need_result2 = need_result[need_result.shape[0] // 2:] + need_result1 = need_result[0 : need_result.shape[0] // 2] + need_result2 = need_result[need_result.shape[0] // 2 :] np.testing.assert_allclose(tr0_out[0], need_result1, rtol=1e-05) np.testing.assert_allclose(tr1_out[0], need_result2, rtol=1e-05) elif col_type == "reduce_scatter": need_result = input1 + input2 - need_result1 = need_result[0:need_result.shape[0] // 2] - need_result2 = need_result[need_result.shape[0] // 2:] + need_result1 = need_result[0 : need_result.shape[0] // 2] + need_result2 = need_result[need_result.shape[0] // 2 :] if dtype == "bfloat16": rtol = 8e-03 else: @@ -369,14 +389,12 @@ def check_with_place(self, else: rtol = 1e-05 atol = 1e-05 - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=rtol, - atol=atol) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=rtol, - atol=atol) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=rtol, atol=atol + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=rtol, atol=atol + ) elif col_type == "parallel_embedding": result_data = tr0_out[0] np.random.seed(2020) @@ -384,48 +402,51 @@ def check_with_place(self, for i in range(result_data.shape[0]): for j in range(result_data.shape[1]): data = result_data[i][j] - assert np.allclose(tr0_out[1][i][j], - need_result[data], - atol=1e-08) + assert np.allclose( + tr0_out[1][i][j], need_result[data], atol=1e-08 + ) elif col_type == "row_parallel_linear": result_data = tr0_out[0] np.random.seed(2020) weight = np.random.rand(1000, 16) need_result = np.matmul(input1, weight) - np.testing.assert_allclose(result_data, - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + result_data, need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "column_parallel_linear": result_data = tr0_out[0] np.random.seed(2020) weight = np.random.rand(1000, 16) need_result = np.matmul(input1, weight) - np.testing.assert_allclose(result_data, - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + result_data, need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "alltoall": - need_result1 = np.vstack((input1[0:input1.shape[0] // 2, :], - input2[0:input2.shape[0] // 2, :])) - need_result2 = np.vstack((input1[input1.shape[0] // 2:, :], - input2[input2.shape[0] // 2:, :])) + need_result1 = np.vstack( + ( + input1[0 : input1.shape[0] // 2, :], + input2[0 : input2.shape[0] // 2, :], + ) + ) + need_result2 = np.vstack( + ( + input1[input1.shape[0] // 2 :, :], + input2[input2.shape[0] // 2 :, :], + ) + ) tr0_out = np.vstack(tr0_out) tr1_out = np.vstack(tr1_out) - np.testing.assert_allclose(tr0_out, - need_result1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out, - need_result2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out, need_result1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out, need_result2, rtol=1e-05, atol=1e-05 + ) elif col_type == "sendrecv": result_data = tr1_out[0] - np.testing.assert_allclose(input1, - result_data, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + input1, result_data, rtol=1e-05, atol=1e-05 + ) elif col_type == "global_gather": in_feat = 2 n_expert = 2 @@ -434,7 +455,8 @@ def check_with_place(self, np.random.seed(pid0) local_expert_count1 = np.random.randint( - 1, 4, size=tot_expert).astype("int") + 1, 4, size=tot_expert + ).astype("int") expert_ptr1 = np.ones(tot_expert, dtype=np.int32) expert_ptr1[0] = 0 for i in range(1, tot_expert): @@ -442,7 +464,8 @@ def check_with_place(self, np.random.seed(pid1) local_expert_count2 = np.random.randint( - 1, 4, size=tot_expert).astype("int") + 1, 4, size=tot_expert + ).astype("int") expert_ptr2 = np.ones(tot_expert, dtype=np.int32) expert_ptr2[0] = 0 for i in range(1, tot_expert): @@ -457,12 +480,14 @@ def check_with_place(self, np.random.seed(pid0) fwd_expert_count = sum(global_expert_count1).astype("int") - local_input_buf1 = np.random.rand(fwd_expert_count, - in_feat).astype("float32") + local_input_buf1 = np.random.rand(fwd_expert_count, in_feat).astype( + "float32" + ) np.random.seed(pid1) fwd_expert_count = sum(global_expert_count2).astype("int") - local_input_buf2 = np.random.rand(fwd_expert_count, - in_feat).astype("float32") + local_input_buf2 = np.random.rand(fwd_expert_count, in_feat).astype( + "float32" + ) output1 = [[], [], [], []] output2 = [[], [], [], []] send_ptr1 = 0 @@ -472,17 +497,21 @@ def check_with_place(self, for j in range(world_size): idx = j * n_expert + i if j == 0: - output1_part1 = local_input_buf1[send_ptr1: \ - send_ptr1 + global_expert_count1[idx], :] - output1_part2 = local_input_buf2[send_ptr2: \ - send_ptr2 + global_expert_count2[idx], :] + output1_part1 = local_input_buf1[ + send_ptr1 : send_ptr1 + global_expert_count1[idx], : + ] + output1_part2 = local_input_buf2[ + send_ptr2 : send_ptr2 + global_expert_count2[idx], : + ] output1[i].extend(output1_part1) output1[i + n_expert].extend(output1_part2) else: - output2_part1 = local_input_buf1[send_ptr1: \ - send_ptr1 + global_expert_count1[idx]] - output2_part2 = local_input_buf2[send_ptr2: \ - send_ptr2 + global_expert_count2[idx]] + output2_part1 = local_input_buf1[ + send_ptr1 : send_ptr1 + global_expert_count1[idx] + ] + output2_part2 = local_input_buf2[ + send_ptr2 : send_ptr2 + global_expert_count2[idx] + ] output2[i].extend(output2_part1) output2[i + n_expert].extend(output2_part2) send_ptr1 = send_ptr1 + global_expert_count1[idx] @@ -503,12 +532,14 @@ def check_with_place(self, output1 = np.array([]) else: output1 = np.concatenate(result1, axis=0).reshape( - sum(local_expert_count1), in_feat) + sum(local_expert_count1), in_feat + ) if result2 == []: output2 = np.array([]) else: output2 = np.concatenate(result2, axis=0).reshape( - sum(local_expert_count2), in_feat) + sum(local_expert_count2), in_feat + ) if tr0_out[0] is None or tr0_out[0].shape[0] == 0: tr0_out[0] = np.array([]) @@ -516,30 +547,27 @@ def check_with_place(self, if tr1_out[0] is None or tr1_out[0].shape[0] == 0: tr1_out[0] = np.array([]) - np.testing.assert_allclose(tr0_out[0], - output1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - output2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], output1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], output2, rtol=1e-05, atol=1e-05 + ) if static_mode == 0: - np.testing.assert_allclose(tr0_out[1], - 2 * local_input_buf1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[1], - 2 * local_input_buf2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[1], 2 * local_input_buf1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[1], 2 * local_input_buf2, rtol=1e-05, atol=1e-05 + ) elif col_type == "global_scatter": np.random.seed(pid0) local_expert_count1 = np.random.randint(1, 4, size=4).astype("int") fwd_expert_count = sum(local_expert_count1) - local_input_buf1 = np.random.rand(fwd_expert_count, - 2).astype("float32") + local_input_buf1 = np.random.rand(fwd_expert_count, 2).astype( + "float32" + ) expert_ptr1 = np.ones(4, dtype=np.int32) expert_ptr1[0] = 0 for i in range(1, 4): @@ -547,8 +575,9 @@ def check_with_place(self, np.random.seed(pid1) local_expert_count2 = np.random.randint(1, 4, size=4).astype("int") fwd_expert_count = sum(local_expert_count2) - local_input_buf2 = np.random.rand(fwd_expert_count, - 2).astype("float32") + local_input_buf2 = np.random.rand(fwd_expert_count, 2).astype( + "float32" + ) expert_ptr2 = np.ones(4, dtype=np.int32) expert_ptr2[0] = 0 for i in range(1, 4): @@ -561,15 +590,31 @@ def check_with_place(self, idx = j * 2 + i if j == 0: # send data to 0 card - output1.append(local_input_buf1[expert_ptr1[idx]: \ - expert_ptr1[idx]+local_expert_count1[idx]]) - output1.append(local_input_buf2[expert_ptr2[idx]:\ - expert_ptr2[idx]+local_expert_count2[idx]]) + output1.append( + local_input_buf1[ + expert_ptr1[idx] : expert_ptr1[idx] + + local_expert_count1[idx] + ] + ) + output1.append( + local_input_buf2[ + expert_ptr2[idx] : expert_ptr2[idx] + + local_expert_count2[idx] + ] + ) else: - output2.append(local_input_buf1[expert_ptr1[idx]: \ - expert_ptr1[idx]+local_expert_count1[idx]]) - output2.append(local_input_buf2[expert_ptr2[idx]:\ - expert_ptr2[idx]+local_expert_count2[idx]]) + output2.append( + local_input_buf1[ + expert_ptr1[idx] : expert_ptr1[idx] + + local_expert_count1[idx] + ] + ) + output2.append( + local_input_buf2[ + expert_ptr2[idx] : expert_ptr2[idx] + + local_expert_count2[idx] + ] + ) if output1 == []: output1 = np.array([]) else: @@ -585,22 +630,18 @@ def check_with_place(self, if tr1_out[0] is None or tr1_out[0].shape[0] == 0: tr1_out[0] = np.array([]) - np.testing.assert_allclose(tr0_out[0], - output1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - output2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], output1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], output2, rtol=1e-05, atol=1e-05 + ) if static_mode == 0: - np.testing.assert_allclose(tr0_out[1], - 2 * local_input_buf1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[1], - 2 * local_input_buf2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[1], 2 * local_input_buf1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[1], 2 * local_input_buf2, rtol=1e-05, atol=1e-05 + ) else: pass diff --git a/python/paddle/fluid/tests/unittests/test_collective_base.py b/python/paddle/fluid/tests/unittests/test_collective_base.py index 7300c0d13dc48..2104f1865f4a9 100644 --- a/python/paddle/fluid/tests/unittests/test_collective_base.py +++ b/python/paddle/fluid/tests/unittests/test_collective_base.py @@ -27,10 +27,10 @@ class TestCollectiveRunnerBase(object): - def get_model(self, train_prog, startup_prog): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def wait_server_ready(self, endpoints): while True: @@ -38,13 +38,15 @@ def wait_server_ready(self, endpoints): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, - 1) + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT, 1 + ) result = sock.connect_ex((ip_port[0], int(ip_port[1]))) if result != 0: @@ -52,44 +54,51 @@ def wait_server_ready(self, endpoints): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + - str(not_ready_endpoints) + "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: break + # endpoints should be ["ip1:port1","ip2:port2"] -#endpoints should be ["ip1:port1","ip2:port2"] - - def initCommunicator(self, program, rank, nranks, wait_port, - current_endpoint, endpoints): + def initCommunicator( + self, program, rank, nranks, wait_port, current_endpoint, endpoints + ): other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) if rank == 0 and wait_port: self.wait_server_ready(other_endpoints) block = program.global_block() - nccl_id_var = block.create_var(name=nameGen.generate('nccl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) + nccl_id_var = block.create_var( + name=nameGen.generate('nccl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) - block.append_op(type='c_gen_nccl_id', - inputs={}, - outputs={'Out': nccl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) + block.append_op( + type='c_gen_nccl_id', + inputs={}, + outputs={'Out': nccl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) - block.append_op(type='c_comm_init', - inputs={'X': nccl_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': self.global_ring_id - }) + block.append_op( + type='c_comm_init', + inputs={'X': nccl_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': self.global_ring_id, + }, + ) def run_trainer(self, args): train_prog = fluid.Program() @@ -98,20 +107,22 @@ def run_trainer(self, args): rank = args["trainerid"] current_endpoint = args["currentendpoint"] nranks = 2 - self.initCommunicator(startup_prog, rank, nranks, True, - current_endpoint, endpoints) + self.initCommunicator( + startup_prog, rank, nranks, True, current_endpoint, endpoints + ) self.rank = rank result = self.get_model(train_prog, startup_prog) device_id = int(os.getenv("FLAGS_selected_gpus", "0")) place = fluid.CUDAPlace( - device_id) #if args.use_gpu else fluid.CPUPlace() + device_id + ) # if args.use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup_prog) np.random.seed(os.getpid()) indata = np.random.random((10, 1000)) - out = exe.run(train_prog, - feed={'tindata': indata}, - fetch_list=[result.name]) + out = exe.run( + train_prog, feed={'tindata': indata}, fetch_list=[result.name] + ) sys.stdout.buffer.write(pickle.dumps(out)) @@ -132,12 +143,13 @@ def runtime_main(test_class, col_type, sub_type): class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable self.temp_dir = tempfile.TemporaryDirectory() @@ -146,10 +158,10 @@ def tearDown(self): self.temp_dir.cleanup() def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -162,13 +174,13 @@ def __free_port(): def _run_cluster(self, model_file, envs): worker_endpoints = self._ps_endpoints.split(",") w0_ep, w1_ep = worker_endpoints - #print("w0_ep:",w0_ep," w1_ep:",w1_ep) + # print("w0_ep:",w0_ep," w1_ep:",w1_ep) env0 = { "FLAGS_selected_gpus": "0", "PADDLE_TRAINER_ID": "0", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w0_ep + "PADDLE_CURRENT_ENDPOINT": w0_ep, } env1 = { @@ -176,9 +188,9 @@ def _run_cluster(self, model_file, envs): "PADDLE_TRAINER_ID": "1", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w1_ep + "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) tr_cmd = "%s %s" @@ -188,16 +200,20 @@ def _run_cluster(self, model_file, envs): path1 = os.path.join(self.temp_dir.name, "/tmp/tr1_err.log") tr0_pipe = open(path0, "wb") tr1_pipe = open(path1, "wb") - #print(tr0_cmd) - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) + # print(tr0_cmd) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) - tr1_proc = subprocess.Popen(tr0_cmd.strip().split(), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + tr1_proc = subprocess.Popen( + tr0_cmd.strip().split(), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -206,14 +222,16 @@ def _run_cluster(self, model_file, envs): # close trainer file tr0_pipe.close() tr1_pipe.close() - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) - def check_with_place(self, - model_file, - col_type, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, col_type, check_error_log=False, need_envs={} + ): required_envs = { "FLAGS_fraction_of_gpu_memory_to_use": "0.15", "FLAGS_eager_delete_tensor_gb": "0.0", @@ -222,14 +240,15 @@ def check_with_place(self, "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), "LD_PRELOAD": os.getenv("LD_PRELOAD", ""), "GLOG_v": "3", - "NCCL_P2P_DISABLE": "1" + "NCCL_P2P_DISABLE": "1", } required_envs.update(need_envs) if check_error_log: required_envs["GLOG_v"] = "3" required_envs["GLOG_logtostderr"] = "1" tr0_out, tr1_out, pid0, pid1 = self._run_cluster( - model_file, required_envs) + model_file, required_envs + ) np.random.seed(pid0) input1 = np.random.random((10, 1000)) np.random.seed(pid1) @@ -247,38 +266,33 @@ def check_with_place(self, np.testing.assert_allclose(tr1_out[0], need_result, rtol=1e-05) elif col_type == "scatter": need_result = input2 - need_result1 = need_result[0:need_result.shape[0] // 2] - need_result2 = need_result[need_result.shape[0] // 2:] + need_result1 = need_result[0 : need_result.shape[0] // 2] + need_result2 = need_result[need_result.shape[0] // 2 :] np.testing.assert_allclose(tr0_out[0], need_result1, rtol=1e-05) np.testing.assert_allclose(tr1_out[0], need_result2, rtol=1e-05) elif col_type == "allreduce": need_result = input1 + input2 - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "reduce_scatter": tmp = input1 + input2 - need_result1 = tmp[0:tmp.shape[0] // 2] - need_result2 = tmp[tmp.shape[0] // 2:] - np.testing.assert_allclose(tr0_out[0], - need_result1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result2, - rtol=1e-05, - atol=1e-05) + need_result1 = tmp[0 : tmp.shape[0] // 2] + need_result2 = tmp[tmp.shape[0] // 2 :] + np.testing.assert_allclose( + tr0_out[0], need_result1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result2, rtol=1e-05, atol=1e-05 + ) elif col_type == "sendrecv": need_result = input1 - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "identity": need_result1 = input1 need_result2 = input2 @@ -296,35 +310,29 @@ def check_with_place(self, np.testing.assert_allclose(tr1_out, need_result2, rtol=1e-05) elif col_type == "concat": need_result = np.concatenate((input1, input2), axis=1) - np.testing.assert_allclose(tr0_out[0], - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "split": need_result1 = np.split(input1, 2, axis=1)[0] need_result2 = np.split(input2, 2, axis=1)[1] - np.testing.assert_allclose(tr0_out[0], - need_result1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0], - need_result2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out[0], need_result1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0], need_result2, rtol=1e-05, atol=1e-05 + ) elif col_type == "sendrecv_array": need_result1 = np.array([[0, 1, 2]]) need_result2 = np.array([[3, 4, 5]]) - np.testing.assert_allclose(tr1_out[0][0], - need_result1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0][1], - need_result2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr1_out[0][0], need_result1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0][1], need_result2, rtol=1e-05, atol=1e-05 + ) else: pass diff --git a/python/paddle/fluid/tests/unittests/test_communicator_async.py b/python/paddle/fluid/tests/unittests/test_communicator_async.py index 0908dbc8df1ee..33a98a5c894c2 100644 --- a/python/paddle/fluid/tests/unittests/test_communicator_async.py +++ b/python/paddle/fluid/tests/unittests/test_communicator_async.py @@ -26,7 +26,6 @@ class TestCommunicator(unittest.TestCase): - def net(self): x = fluid.layers.data(name='x', shape=[1], dtype='float32') y = fluid.layers.data(name='y', shape=[1], dtype='float32') @@ -40,7 +39,8 @@ def test_communicator_async(self): current_id=0, role=role_maker.Role.WORKER, worker_num=2, - server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"]) + server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"], + ) fleet.init(role) avg_cost = self.net() diff --git a/python/paddle/fluid/tests/unittests/test_communicator_geo.py b/python/paddle/fluid/tests/unittests/test_communicator_geo.py index 6c99eaac737d9..0efea743f3314 100644 --- a/python/paddle/fluid/tests/unittests/test_communicator_geo.py +++ b/python/paddle/fluid/tests/unittests/test_communicator_geo.py @@ -31,7 +31,6 @@ class TestCommunicatorGeoEnd2End(unittest.TestCase): - def net(self): x = fluid.layers.data(name='x', shape=[13], dtype='float32') x1 = fluid.layers.data(name='x1', shape=[1], dtype='int64', lod_level=1) @@ -41,8 +40,10 @@ def net(self): size=[10000, 10], param_attr=fluid.ParamAttr( name="embedding", - initializer=fluid.initializer.Constant(value=0.01)), - is_sparse=True) + initializer=fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) pool = fluid.layers.sequence_pool(input=emb, pool_type="sum") z = fluid.layers.concat(input=[x, pool], axis=1) @@ -54,7 +55,6 @@ def net(self): return avg_cost, x, x1, y def fake_reader(self): - def reader(): for i in range(10000): x = numpy.random.random((1, 13)).astype('float32') @@ -91,9 +91,11 @@ def run_trainer(self, role, strategy): feeder = fluid.DataFeeder(place=place, feed_list=[x, z, y]) for batch_id, data in enumerate(train_reader()): - exe.run(fluid.default_main_program(), - feed=feeder.feed(data), - fetch_list=[]) + exe.run( + fluid.default_main_program(), + feed=feeder.feed(data), + fetch_list=[], + ) fleet.stop_worker() @@ -166,9 +168,11 @@ def runTest(self): ps_cmd = "{} {}".format(_python, server_file) - ps_proc = subprocess.Popen(ps_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ps_proc = subprocess.Popen( + ps_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) time.sleep(5) diff --git a/python/paddle/fluid/tests/unittests/test_communicator_ps_gpu.py b/python/paddle/fluid/tests/unittests/test_communicator_ps_gpu.py index 23af083af566f..328874b3f5565 100644 --- a/python/paddle/fluid/tests/unittests/test_communicator_ps_gpu.py +++ b/python/paddle/fluid/tests/unittests/test_communicator_ps_gpu.py @@ -27,7 +27,6 @@ class TestCommunicator(unittest.TestCase): - def test_communicator_ps_gpu(self): temp_dir = tempfile.TemporaryDirectory() path = os.path.join(temp_dir.name, "test_communicator_ps_gpu.txt") @@ -42,9 +41,11 @@ def test_communicator_ps_gpu(self): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.2:36001" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36002,127.0.0.2:36002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36002,127.0.0.2:36002" os.environ["TRAINING_ROLE"] = "TRAINER" os.environ["FLAGS_selected_gpus"] = "0" role = role_maker.PaddleCloudRoleMaker() @@ -71,10 +72,9 @@ def test_communicator_ps_gpu(self): optimizer.minimize(avg_cost) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=1, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=1, pipe_command="cat", use_var=slots_vars + ) dataset.set_filelist(["test_communicator_ps_gpu.txt"]) dataset.set_date("20211111") dataset.load_into_memory(is_shuffle=True) diff --git a/python/paddle/fluid/tests/unittests/test_compare_op.py b/python/paddle/fluid/tests/unittests/test_compare_op.py index a06ac3a9b0218..d9636972a13e3 100755 --- a/python/paddle/fluid/tests/unittests/test_compare_op.py +++ b/python/paddle/fluid/tests/unittests/test_compare_op.py @@ -23,9 +23,7 @@ def create_test_class(op_type, typename, callback): - class Cls(op_test.OpTest): - def setUp(self): a = numpy.random.random(size=(10, 7)).astype(typename) b = numpy.random.random(size=(10, 7)).astype(typename) @@ -45,11 +43,9 @@ def test_errors(self): y = fluid.layers.data(name='y', shape=[2], dtype='int32') a = fluid.layers.data(name='a', shape=[2], dtype='int16') if self.op_type == "less_than": - self.assertRaises(TypeError, - fluid.layers.less_than, - x=x, - y=y, - force_cpu=1) + self.assertRaises( + TypeError, fluid.layers.less_than, x=x, y=y, force_cpu=1 + ) op = eval("fluid.layers.%s" % self.op_type) self.assertRaises(TypeError, op, x=x, y=y, cond=1) self.assertRaises(TypeError, op, x=x, y=a) @@ -75,9 +71,7 @@ def test_errors(self): def create_paddle_case(op_type, callback): - class PaddleCls(unittest.TestCase): - def setUp(self): self.op_type = op_type self.input_x = np.array([1, 2, 3, 4]).astype(np.int64) @@ -95,11 +89,10 @@ def test_api(self): op = eval("paddle.%s" % (self.op_type)) out = op(x, y) exe = fluid.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "y": self.input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": self.input_x, "y": self.input_y}, + fetch_list=[out], + ) self.assertEqual((res == self.real_result).all(), True) def test_api_float(self): @@ -111,11 +104,9 @@ def test_api_float(self): op = eval("paddle.%s" % (self.op_type)) out = op(x, y) exe = fluid.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "y": 1.0 - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": self.input_x, "y": 1.0}, fetch_list=[out] + ) self.real_result = np.array([1, 0, 0, 0]).astype(np.int64) self.assertEqual((res == self.real_result).all(), True) @@ -160,31 +151,36 @@ def test_dynamic_api_inf_1(self): self.real_result = (x1 == y1).astype(np.int64) self.assertEqual( (out.numpy().astype(np.int64) == self.real_result).all(), - True) + True, + ) paddle.enable_static() def test_dynamic_api_inf_2(self): if self.op_type == "equal": paddle.disable_static() - x1 = np.array([1, float('inf'), - float('inf')]).astype(np.float32) + x1 = np.array([1, float('inf'), float('inf')]).astype( + np.float32 + ) x = paddle.to_tensor(x1) - y1 = np.array([1, float('-inf'), - float('inf')]).astype(np.float32) + y1 = np.array([1, float('-inf'), float('inf')]).astype( + np.float32 + ) y = paddle.to_tensor(y1) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) self.real_result = (x1 == y1).astype(np.int64) self.assertEqual( (out.numpy().astype(np.int64) == self.real_result).all(), - True) + True, + ) paddle.enable_static() def test_dynamic_api_inf_3(self): if self.op_type == "equal": paddle.disable_static() - x1 = np.array([1, float('inf'), - float('-inf')]).astype(np.float32) + x1 = np.array([1, float('inf'), float('-inf')]).astype( + np.float32 + ) x = paddle.to_tensor(x1) y1 = np.array([1, 2, 3]).astype(np.float32) y = paddle.to_tensor(y1) @@ -193,7 +189,8 @@ def test_dynamic_api_inf_3(self): self.real_result = (x1 == y1).astype(np.int64) self.assertEqual( (out.numpy().astype(np.int64) == self.real_result).all(), - True) + True, + ) paddle.enable_static() def test_dynamic_api_nan_1(self): @@ -208,31 +205,36 @@ def test_dynamic_api_nan_1(self): self.real_result = (x1 == y1).astype(np.int64) self.assertEqual( (out.numpy().astype(np.int64) == self.real_result).all(), - True) + True, + ) paddle.enable_static() def test_dynamic_api_nan_2(self): if self.op_type == "equal": paddle.disable_static() - x1 = np.array([1, float('nan'), - float('nan')]).astype(np.float32) + x1 = np.array([1, float('nan'), float('nan')]).astype( + np.float32 + ) x = paddle.to_tensor(x1) - y1 = np.array([1, float('-nan'), - float('nan')]).astype(np.float32) + y1 = np.array([1, float('-nan'), float('nan')]).astype( + np.float32 + ) y = paddle.to_tensor(y1) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) self.real_result = (x1 == y1).astype(np.int64) self.assertEqual( (out.numpy().astype(np.int64) == self.real_result).all(), - True) + True, + ) paddle.enable_static() def test_dynamic_api_nan_3(self): if self.op_type == "equal": paddle.disable_static() - x1 = np.array([1, float('-nan'), - float('nan')]).astype(np.float32) + x1 = np.array([1, float('-nan'), float('nan')]).astype( + np.float32 + ) x = paddle.to_tensor(x1) y1 = np.array([1, 2, 1]).astype(np.float32) y = paddle.to_tensor(y1) @@ -241,16 +243,19 @@ def test_dynamic_api_nan_3(self): self.real_result = (x1 == y1).astype(np.int64) self.assertEqual( (out.numpy().astype(np.int64) == self.real_result).all(), - True) + True, + ) paddle.enable_static() def test_not_equal(self): if self.op_type == "not_equal": paddle.disable_static() - x = paddle.to_tensor(np.array([1.2e-8, 2, 2, 1]), - dtype="float32") - y = paddle.to_tensor(np.array([1.1e-8, 2, 2, 1]), - dtype="float32") + x = paddle.to_tensor( + np.array([1.2e-8, 2, 2, 1]), dtype="float32" + ) + y = paddle.to_tensor( + np.array([1.1e-8, 2, 2, 1]), dtype="float32" + ) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) self.real_result = np.array([0, 0, 0, 0]).astype(np.int64) @@ -258,7 +263,6 @@ def test_not_equal(self): paddle.enable_static() def test_assert(self): - def test_dynamic_api_string(self): if self.op_type == "equal": paddle.disable_static() @@ -282,9 +286,9 @@ def test_dynamic_api_bool(self): def test_broadcast_api_1(self): paddle.enable_static() with program_guard(Program(), Program()): - x = paddle.static.data(name='x', - shape=[1, 2, 1, 3], - dtype='int32') + x = paddle.static.data( + name='x', shape=[1, 2, 1, 3], dtype='int32' + ) y = paddle.static.data(name='y', shape=[1, 2, 3], dtype='int32') op = eval("paddle.%s" % (self.op_type)) out = op(x, y) @@ -292,31 +296,27 @@ def test_broadcast_api_1(self): input_x = np.arange(1, 7).reshape((1, 2, 1, 3)).astype(np.int32) input_y = np.arange(0, 6).reshape((1, 2, 3)).astype(np.int32) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) def test_broadcast_api_2(self): paddle.enable_static() with program_guard(Program(), Program()): x = paddle.static.data(name='x', shape=[1, 2, 3], dtype='int32') - y = paddle.static.data(name='y', - shape=[1, 2, 1, 3], - dtype='int32') + y = paddle.static.data( + name='y', shape=[1, 2, 1, 3], dtype='int32' + ) op = eval("paddle.%s" % (self.op_type)) out = op(x, y) exe = paddle.static.Executor(self.place) input_x = np.arange(0, 6).reshape((1, 2, 3)).astype(np.int32) input_y = np.arange(1, 7).reshape((1, 2, 1, 3)).astype(np.int32) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) def test_broadcast_api_3(self): @@ -330,11 +330,9 @@ def test_broadcast_api_3(self): input_x = np.arange(0, 5).reshape((5)).astype(np.int32) input_y = np.array([5, 3, 2]).reshape((3, 1)).astype(np.int32) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) def test_bool_api_4(self): @@ -348,11 +346,9 @@ def test_bool_api_4(self): input_x = np.array([True, False, True]).astype(np.bool_) input_y = np.array([True, True, False]).astype(np.bool_) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) def test_bool_broadcast_api_4(self): @@ -366,11 +362,9 @@ def test_bool_broadcast_api_4(self): input_x = np.array([True, False, True]).astype(np.bool_) input_y = np.array([True]).astype(np.bool_) real_result = callback(input_x, input_y) - res, = exe.run(feed={ - "x": input_x, - "y": input_y - }, - fetch_list=[out]) + (res,) = exe.run( + feed={"x": input_x, "y": input_y}, fetch_list=[out] + ) self.assertEqual((res == real_result).all(), True) def test_attr_name(self): @@ -396,19 +390,18 @@ def test_attr_name(self): class TestCompareOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): # The input x and y of compare_op must be Variable. x = fluid.layers.data(name='x', shape=[1], dtype="float32") - y = fluid.create_lod_tensor(numpy.array([[-1]]), [[1]], - fluid.CPUPlace()) + y = fluid.create_lod_tensor( + numpy.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.greater_equal, x, y) class API_TestElementwise_Equal(unittest.TestCase): - def test_api(self): paddle.enable_static() with fluid.program_guard(fluid.Program(), fluid.Program()): @@ -417,7 +410,7 @@ def test_api(self): out = paddle.equal(x=label, y=limit) place = fluid.CPUPlace() exe = fluid.Executor(place) - res, = exe.run(fetch_list=[out]) + (res,) = exe.run(fetch_list=[out]) self.assertEqual((res == np.array([True, False])).all(), True) with fluid.program_guard(fluid.Program(), fluid.Program()): @@ -426,12 +419,11 @@ def test_api(self): out = paddle.equal(x=label, y=limit) place = fluid.CPUPlace() exe = fluid.Executor(place) - res, = exe.run(fetch_list=[out]) + (res,) = exe.run(fetch_list=[out]) self.assertEqual((res == np.array([True, True])).all(), True) class TestCompareOpPlace(unittest.TestCase): - def test_place_1(self): paddle.enable_static() place = paddle.CPUPlace() @@ -441,7 +433,7 @@ def test_place_1(self): limit = fluid.layers.assign(np.array([3, 2], dtype="int32")) out = fluid.layers.less_than(label, limit, force_cpu=True) exe = fluid.Executor(place) - res, = exe.run(fetch_list=[out]) + (res,) = exe.run(fetch_list=[out]) self.assertEqual((res == np.array([False, False])).all(), True) def test_place_2(self): diff --git a/python/paddle/fluid/tests/unittests/test_compare_reduce_op.py b/python/paddle/fluid/tests/unittests/test_compare_reduce_op.py index 5508f17a3bc88..bc88cba96eb6e 100644 --- a/python/paddle/fluid/tests/unittests/test_compare_reduce_op.py +++ b/python/paddle/fluid/tests/unittests/test_compare_reduce_op.py @@ -20,9 +20,7 @@ def create_test_not_equal_class(op_type, typename, callback): - class Cls(op_test.OpTest): - def setUp(self): x = np.random.random(size=(10, 7)).astype(typename) y = np.random.random(size=(10, 7)).astype(typename) @@ -41,9 +39,7 @@ def test_output(self): def create_test_not_shape_equal_class(op_type, typename, callback): - class Cls(op_test.OpTest): - def setUp(self): x = np.random.random(size=(10, 7)).astype(typename) y = np.random.random(size=(10)).astype(typename) @@ -62,9 +58,7 @@ def test_output(self): def create_test_equal_class(op_type, typename, callback): - class Cls(op_test.OpTest): - def setUp(self): x = y = np.random.random(size=(10, 7)).astype(typename) z = callback(x, y) @@ -82,9 +76,7 @@ def test_output(self): def create_test_dim1_class(op_type, typename, callback): - class Cls(op_test.OpTest): - def setUp(self): x = y = np.random.random(size=(1)).astype(typename) x = np.array([True, False, True]).astype(typename) @@ -112,7 +104,6 @@ def test_output(self): class TestEqualReduceAPI(unittest.TestCase): - def test_name(self): x = fluid.layers.assign(np.array([3, 4], dtype="int32")) y = fluid.layers.assign(np.array([3, 4], dtype="int32")) diff --git a/python/paddle/fluid/tests/unittests/test_compat.py b/python/paddle/fluid/tests/unittests/test_compat.py index e4fedede47ce3..d4ad8a8274e3a 100644 --- a/python/paddle/fluid/tests/unittests/test_compat.py +++ b/python/paddle/fluid/tests/unittests/test_compat.py @@ -17,7 +17,6 @@ class TestCompatible(unittest.TestCase): - def test_to_text(self): self.assertIsNone(cpt.to_text(None)) diff --git a/python/paddle/fluid/tests/unittests/test_compiled_program.py b/python/paddle/fluid/tests/unittests/test_compiled_program.py index 9406d4d898ee1..8418de88fd865 100644 --- a/python/paddle/fluid/tests/unittests/test_compiled_program.py +++ b/python/paddle/fluid/tests/unittests/test_compiled_program.py @@ -22,84 +22,87 @@ class TestCompiledProgram(unittest.TestCase): - def setUp(self): self.seed = 100 self.img = np.random.random(size=(16, 784)).astype('float32') - self.label = np.random.randint(low=0, - high=10, - size=[16, 1], - dtype=np.int64) + self.label = np.random.randint( + low=0, high=10, size=[16, 1], dtype=np.int64 + ) with new_program_scope(): paddle.seed(self.seed) paddle.framework.random._manual_program_seed(self.seed) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) loss = simple_fc_net() exe.run(fluid.default_startup_program()) - loss_data, = exe.run(fluid.default_main_program(), - feed={ - "image": self.img, - "label": self.label - }, - fetch_list=[loss.name]) + (loss_data,) = exe.run( + fluid.default_main_program(), + feed={"image": self.img, "label": self.label}, + fetch_list=[loss.name], + ) self.loss = loss_data[0] def test_compiled_program_base(self): with new_program_scope(): paddle.seed(self.seed) paddle.framework.random._manual_program_seed(self.seed) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) loss = simple_fc_net() exe.run(fluid.default_startup_program()) compiled_prog = fluid.CompiledProgram(fluid.default_main_program()) - loss_data, = exe.run(compiled_prog, - feed={ - "image": self.img, - "label": self.label - }, - fetch_list=[loss.name]) + (loss_data,) = exe.run( + compiled_prog, + feed={"image": self.img, "label": self.label}, + fetch_list=[loss.name], + ) np.testing.assert_array_equal(loss_data[0], self.loss) def test_compiled_program_with_data_parallel(self): with new_program_scope(): paddle.seed(self.seed) paddle.framework.random._manual_program_seed(self.seed) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) loss = simple_fc_net() exe.run(fluid.default_startup_program()) compiled_prog = fluid.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - loss_name=loss.name, places=[place]) - - loss_data, = exe.run(compiled_prog, - feed={ - "image": self.img, - "label": self.label - }, - fetch_list=[loss.name]) + fluid.default_main_program() + ).with_data_parallel(loss_name=loss.name, places=[place]) + + (loss_data,) = exe.run( + compiled_prog, + feed={"image": self.img, "label": self.label}, + fetch_list=[loss.name], + ) np.testing.assert_array_equal(loss_data[0], self.loss) class TestCompiledProgramError(unittest.TestCase): - def test_program_or_graph_error(self): self.assertRaises(TypeError, fluid.CompiledProgram, "program") def build_simple_model(self): - img = fluid.layers.data(name='image', - shape=[1, 28, 28], - dtype='float32') + img = fluid.layers.data( + name='image', shape=[1, 28, 28], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') prediction = fluid.layers.fc(input=img, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) @@ -112,7 +115,8 @@ def compile_program_not_compiled(self): # compile program program = fluid.default_main_program() compiled_program = fluid.CompiledProgram( - program).with_data_parallel() + program + ).with_data_parallel() return compiled_program def compile_program(self): @@ -148,7 +152,8 @@ def test_share_vars_from_error_no_parallel(self): # compile program program = fluid.default_main_program() compiled_program = fluid.CompiledProgram( - program).with_data_parallel(share_vars_from=source_program) + program + ).with_data_parallel(share_vars_from=source_program) scope = fluid.global_scope() place = fluid.CPUPlace() with self.assertRaises(ValueError): @@ -161,7 +166,8 @@ def test_share_vars_from_error_no_executor(self): # compile program program = fluid.default_main_program() compiled_program = fluid.CompiledProgram( - program).with_data_parallel(share_vars_from=source_program) + program + ).with_data_parallel(share_vars_from=source_program) scope = fluid.global_scope() place = fluid.CPUPlace() with self.assertRaises(ValueError): diff --git a/python/paddle/fluid/tests/unittests/test_complex_abs.py b/python/paddle/fluid/tests/unittests/test_complex_abs.py index e61732eba57b8..968bcb7961318 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_abs.py +++ b/python/paddle/fluid/tests/unittests/test_complex_abs.py @@ -22,7 +22,6 @@ class TestComplexAbsOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = paddle.abs @@ -37,7 +36,8 @@ def setUp(self): def init_input_output(self): self.x = np.random.random(self.shape).astype( - self.dtype) + 1J * np.random.random(self.shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.shape).astype(self.dtype) self.out = np.abs(self.x) def init_grad_input_output(self): @@ -48,15 +48,16 @@ def test_check_output(self): self.check_output(check_eager=False) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) class TestComplexAbsOpZeroValues(OpTest): - def setUp(self): paddle.enable_static() self.op_type = "abs" @@ -70,8 +71,9 @@ def setUp(self): self.outputs = {'Out': self.out} def init_input_output(self): - self.x = np.zeros(self.shape).astype( - self.dtype) + 1J * np.zeros(self.shape).astype(self.dtype) + self.x = np.zeros(self.shape).astype(self.dtype) + 1j * np.zeros( + self.shape + ).astype(self.dtype) self.out = np.abs(self.x) def init_grad_input_output(self): @@ -82,15 +84,16 @@ def test_check_output(self): self.check_output(check_eager=False) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) class TestAbs(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -111,7 +114,6 @@ def test_eager(self): class TestRealAbsOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = paddle.abs @@ -136,11 +138,13 @@ def test_check_output(self): self.check_output(check_eager=False) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_complex_cast.py b/python/paddle/fluid/tests/unittests/test_complex_cast.py index 015c56965df64..9065b2d008d33 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_cast.py +++ b/python/paddle/fluid/tests/unittests/test_complex_cast.py @@ -20,12 +20,11 @@ class TestComplexCastOp(unittest.TestCase): - def test_complex_to_real(self): r = np.random.random(size=[10, 10]) * 10 i = np.random.random(size=[10, 10]) - c_t = paddle.to_tensor(r + i * 1J, dtype='complex64') + c_t = paddle.to_tensor(r + i * 1j, dtype='complex64') self.assertEqual(c_t.cast('int64').dtype, paddle.int64) self.assertEqual(c_t.cast('int32').dtype, paddle.int32) @@ -33,21 +32,21 @@ def test_complex_to_real(self): self.assertEqual(c_t.cast('float64').dtype, paddle.float64) self.assertEqual(c_t.cast('bool').dtype, paddle.bool) - np.testing.assert_allclose(c_t.cast('int64').numpy(), - r.astype('int64'), - rtol=1e-05) - np.testing.assert_allclose(c_t.cast('int32').numpy(), - r.astype('int32'), - rtol=1e-05) - np.testing.assert_allclose(c_t.cast('float32').numpy(), - r.astype('float32'), - rtol=1e-05) - np.testing.assert_allclose(c_t.cast('float64').numpy(), - r.astype('float64'), - rtol=1e-05) - np.testing.assert_allclose(c_t.cast('bool').numpy(), - r.astype('bool'), - rtol=1e-05) + np.testing.assert_allclose( + c_t.cast('int64').numpy(), r.astype('int64'), rtol=1e-05 + ) + np.testing.assert_allclose( + c_t.cast('int32').numpy(), r.astype('int32'), rtol=1e-05 + ) + np.testing.assert_allclose( + c_t.cast('float32').numpy(), r.astype('float32'), rtol=1e-05 + ) + np.testing.assert_allclose( + c_t.cast('float64').numpy(), r.astype('float64'), rtol=1e-05 + ) + np.testing.assert_allclose( + c_t.cast('bool').numpy(), r.astype('bool'), rtol=1e-05 + ) def test_real_to_complex(self): r = np.random.random(size=[10, 10]) * 10 @@ -56,29 +55,29 @@ def test_real_to_complex(self): self.assertEqual(r_t.cast('complex64').dtype, paddle.complex64) self.assertEqual(r_t.cast('complex128').dtype, paddle.complex128) - np.testing.assert_allclose(r_t.cast('complex64').real().numpy(), - r, - rtol=1e-05) - np.testing.assert_allclose(r_t.cast('complex128').real().numpy(), - r, - rtol=1e-05) + np.testing.assert_allclose( + r_t.cast('complex64').real().numpy(), r, rtol=1e-05 + ) + np.testing.assert_allclose( + r_t.cast('complex128').real().numpy(), r, rtol=1e-05 + ) def test_complex64_complex128(self): r = np.random.random(size=[10, 10]) i = np.random.random(size=[10, 10]) - c = r + i * 1J + c = r + i * 1j c_64 = paddle.to_tensor(c, dtype='complex64') c_128 = paddle.to_tensor(c, dtype='complex128') self.assertTrue(c_64.cast('complex128').dtype, paddle.complex128) self.assertTrue(c_128.cast('complex128').dtype, paddle.complex64) - np.testing.assert_allclose(c_64.cast('complex128').numpy(), - c_128.numpy(), - rtol=1e-05) - np.testing.assert_allclose(c_128.cast('complex128').numpy(), - c_64.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + c_64.cast('complex128').numpy(), c_128.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + c_128.cast('complex128').numpy(), c_64.numpy(), rtol=1e-05 + ) def test_eager(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_complex_elementwise_layers.py b/python/paddle/fluid/tests/unittests/test_complex_elementwise_layers.py index cd66246bfd5e7..14bdec6101ab2 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_elementwise_layers.py +++ b/python/paddle/fluid/tests/unittests/test_complex_elementwise_layers.py @@ -30,7 +30,6 @@ class TestComplexElementwiseLayers(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -48,21 +47,27 @@ def assert_check(self, pd_result, np_result, place): pd_result, np_result, rtol=1e-05, - err_msg= - '\nplace: {}\npaddle diff result:\n {}\nnumpy diff result:\n {}\n'. - format(place, pd_result[~np.isclose(pd_result, np_result)], - np_result[~np.isclose(pd_result, np_result)])) + err_msg='\nplace: {}\npaddle diff result:\n {}\nnumpy diff result:\n {}\n'.format( + place, + pd_result[~np.isclose(pd_result, np_result)], + np_result[~np.isclose(pd_result, np_result)], + ), + ) def compare_by_basic_api(self, x, y): for place in self._places: - self.assert_check(self.paddle_calc(x, y, "add", place), x + y, - place) - self.assert_check(self.paddle_calc(x, y, "sub", place), x - y, - place) - self.assert_check(self.paddle_calc(x, y, "mul", place), x * y, - place) - self.assert_check(self.paddle_calc(x, y, "div", place), x / y, - place) + self.assert_check( + self.paddle_calc(x, y, "add", place), x + y, place + ) + self.assert_check( + self.paddle_calc(x, y, "sub", place), x - y, place + ) + self.assert_check( + self.paddle_calc(x, y, "mul", place), x * y, place + ) + self.assert_check( + self.paddle_calc(x, y, "div", place), x / y, place + ) def compare_op_by_basic_api(self, x, y): for place in self._places: @@ -76,18 +81,21 @@ def compare_op_by_basic_api(self, x, y): def test_complex_xy(self): for dtype in self._dtypes: - x = rand([2, 3, 4, 5 - ]).astype(dtype) + 1j * rand([2, 3, 4, 5]).astype(dtype) - y = rand([2, 3, 4, 5 - ]).astype(dtype) + 1j * rand([2, 3, 4, 5]).astype(dtype) + x = rand([2, 3, 4, 5]).astype(dtype) + 1j * rand( + [2, 3, 4, 5] + ).astype(dtype) + y = rand([2, 3, 4, 5]).astype(dtype) + 1j * rand( + [2, 3, 4, 5] + ).astype(dtype) self.compare_by_basic_api(x, y) self.compare_op_by_basic_api(x, y) def test_complex_x_real_y(self): for dtype in self._dtypes: - x = rand([2, 3, 4, 5 - ]).astype(dtype) + 1j * rand([2, 3, 4, 5]).astype(dtype) + x = rand([2, 3, 4, 5]).astype(dtype) + 1j * rand( + [2, 3, 4, 5] + ).astype(dtype) y = rand([4, 5]).astype(dtype) # promote types cases diff --git a/python/paddle/fluid/tests/unittests/test_complex_getitem.py b/python/paddle/fluid/tests/unittests/test_complex_getitem.py index 2d5428ce93353..7446e8f6dfefa 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_getitem.py +++ b/python/paddle/fluid/tests/unittests/test_complex_getitem.py @@ -20,7 +20,6 @@ class TestComplexGetitemLayer(unittest.TestCase): - def setUp(self): self._places = [fluid.CPUPlace()] if fluid.core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_complex_grad_accumulated.py b/python/paddle/fluid/tests/unittests/test_complex_grad_accumulated.py index 10c4eed62fdd2..9505ea3011afc 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_grad_accumulated.py +++ b/python/paddle/fluid/tests/unittests/test_complex_grad_accumulated.py @@ -22,52 +22,56 @@ class Optimization_ex1(paddle.nn.Layer): - - def __init__(self, - shape, - dtype, - param_attr=paddle.nn.initializer.Uniform(low=-5., high=5.)): + def __init__( + self, + shape, + dtype, + param_attr=paddle.nn.initializer.Uniform(low=-5.0, high=5.0), + ): super(Optimization_ex1, self).__init__() - self.theta0 = self.create_parameter(shape=shape, - attr=param_attr, - dtype=dtype, - is_bias=False) - self.theta1 = self.create_parameter(shape=shape, - attr=param_attr, - dtype=dtype, - is_bias=False) + self.theta0 = self.create_parameter( + shape=shape, attr=param_attr, dtype=dtype, is_bias=False + ) + self.theta1 = self.create_parameter( + shape=shape, attr=param_attr, dtype=dtype, is_bias=False + ) self.A = paddle.to_tensor( - np.random.random((4, 4)).astype(dtype) + - np.random.random((4, 4)).astype(dtype) * 1j) - self.B = paddle.to_tensor(np.random.random( - (4, 4)).astype(dtype) + np.random.random((4, 4)).astype(dtype) * 1j, - stop_gradient=False) + np.random.random((4, 4)).astype(dtype) + + np.random.random((4, 4)).astype(dtype) * 1j + ) + self.B = paddle.to_tensor( + np.random.random((4, 4)).astype(dtype) + + np.random.random((4, 4)).astype(dtype) * 1j, + stop_gradient=False, + ) def forward(self, mode=1): jj = paddle.to_tensor(np.array([1j]).astype(np.complex64)) if mode == 1: # run all calc in one step loss = paddle.sum(self.A + (self.theta0 + self.theta1 * jj)) * ( - paddle.sum(self.A + (self.theta0 + self.theta1 * jj)).conj()) + paddle.sum(self.A + (self.theta0 + self.theta1 * jj)).conj() + ) return loss.real() elif mode == 2: # run in two step self.theta = self.theta0 + self.theta1 * jj loss = paddle.sum(self.A + self.theta) * ( - paddle.sum(self.A + self.theta).conj()) + paddle.sum(self.A + self.theta).conj() + ) return loss.real() elif mode == 3: # run without param - loss = paddle.sum(self.A + self.B) * (paddle.sum(self.A + - self.B).conj()) + loss = paddle.sum(self.A + self.B) * ( + paddle.sum(self.A + self.B).conj() + ) return loss.real() else: raise NotImplementedError class TestComplexGradAccumulated(unittest.TestCase): - def setUp(self): self.devices = ['cpu'] if core.is_compiled_with_cuda(): @@ -81,8 +85,9 @@ def train(self, device, dtype, mode): paddle.set_device(device) myLayer = Optimization_ex1(self.theta_size, dtype) - optimizer = paddle.optimizer.SGD(learning_rate=self.learning_rate, - parameters=myLayer.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=self.learning_rate, parameters=myLayer.parameters() + ) for iter in range(self.iter): loss = myLayer(mode) @@ -95,8 +100,9 @@ def train_no_clear_grad(self, device, dtype, mode): paddle.set_device(device) myLayer = Optimization_ex1(self.theta_size, dtype) - optimizer = paddle.optimizer.SGD(learning_rate=self.learning_rate, - parameters=myLayer.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=self.learning_rate, parameters=myLayer.parameters() + ) for iter in range(self.iter): loss = myLayer(mode) diff --git a/python/paddle/fluid/tests/unittests/test_complex_kron.py b/python/paddle/fluid/tests/unittests/test_complex_kron.py index f216e6d037b25..7be6a3737edb4 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_kron.py +++ b/python/paddle/fluid/tests/unittests/test_complex_kron.py @@ -21,7 +21,6 @@ class ComplexKronTestCase(unittest.TestCase): - def __init__(self, methodName='runTest', x=None, y=None): super(ComplexKronTestCase, self).__init__(methodName) self.x = x @@ -43,9 +42,9 @@ def test_kron_api(self, place): x_var = dg.to_variable(self.x) y_var = dg.to_variable(self.y) out_var = paddle.kron(x_var, y_var) - np.testing.assert_allclose(out_var.numpy(), - self.ref_result, - rtol=1e-05) + np.testing.assert_allclose( + out_var.numpy(), self.ref_result, rtol=1e-05 + ) def test_eager(self, place): with _test_eager_guard(): @@ -56,23 +55,35 @@ def load_tests(loader, standard_tests, pattern): suite = unittest.TestSuite() for dtype in ["float32", "float64"]: suite.addTest( - ComplexKronTestCase(x=np.random.randn(2, 2).astype(dtype) + - 1j * np.random.randn(2, 2).astype(dtype), - y=np.random.randn(3, 3).astype(dtype) + - 1j * np.random.randn(3, 3).astype(dtype))) + ComplexKronTestCase( + x=np.random.randn(2, 2).astype(dtype) + + 1j * np.random.randn(2, 2).astype(dtype), + y=np.random.randn(3, 3).astype(dtype) + + 1j * np.random.randn(3, 3).astype(dtype), + ) + ) suite.addTest( - ComplexKronTestCase(x=np.random.randn(2, 2).astype(dtype), - y=np.random.randn(3, 3).astype(dtype) + - 1j * np.random.randn(3, 3).astype(dtype))) + ComplexKronTestCase( + x=np.random.randn(2, 2).astype(dtype), + y=np.random.randn(3, 3).astype(dtype) + + 1j * np.random.randn(3, 3).astype(dtype), + ) + ) suite.addTest( - ComplexKronTestCase(x=np.random.randn(2, 2).astype(dtype) + - 1j * np.random.randn(2, 2).astype(dtype), - y=np.random.randn(3, 3).astype(dtype))) + ComplexKronTestCase( + x=np.random.randn(2, 2).astype(dtype) + + 1j * np.random.randn(2, 2).astype(dtype), + y=np.random.randn(3, 3).astype(dtype), + ) + ) suite.addTest( - ComplexKronTestCase(x=np.random.randn(2, 2).astype(dtype) + - 1j * np.random.randn(2, 2).astype(dtype), - y=np.random.randn(2, 2, 3).astype(dtype))) + ComplexKronTestCase( + x=np.random.randn(2, 2).astype(dtype) + + 1j * np.random.randn(2, 2).astype(dtype), + y=np.random.randn(2, 2, 3).astype(dtype), + ) + ) return suite diff --git a/python/paddle/fluid/tests/unittests/test_complex_matmul.py b/python/paddle/fluid/tests/unittests/test_complex_matmul.py index 55cf8516078fa..f803930d2ad3e 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_matmul.py +++ b/python/paddle/fluid/tests/unittests/test_complex_matmul.py @@ -21,7 +21,6 @@ class TestComplexMatMulLayer(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [fluid.CPUPlace()] @@ -39,10 +38,12 @@ def compare_by_basic_api(self, x, y, np_result): pd_result, np_result, rtol=1e-05, - err_msg= - '\nplace: {}\npaddle diff result:\n {}\nnumpy diff result:\n {}\n' - .format(place, pd_result[~np.isclose(pd_result, np_result)], - np_result[~np.isclose(pd_result, np_result)])) + err_msg='\nplace: {}\npaddle diff result:\n {}\nnumpy diff result:\n {}\n'.format( + place, + pd_result[~np.isclose(pd_result, np_result)], + np_result[~np.isclose(pd_result, np_result)], + ), + ) def compare_op_by_basic_api(self, x, y, np_result): for place in self._places: @@ -55,19 +56,21 @@ def compare_op_by_basic_api(self, x, y, np_result): pd_result, np_result, rtol=1e-05, - err_msg= - '\nplace: {}\npaddle diff result:\n {}\nnumpy diff result:\n {}\n' - .format(place, pd_result[~np.isclose(pd_result, np_result)], - np_result[~np.isclose(pd_result, np_result)])) + err_msg='\nplace: {}\npaddle diff result:\n {}\nnumpy diff result:\n {}\n'.format( + place, + pd_result[~np.isclose(pd_result, np_result)], + np_result[~np.isclose(pd_result, np_result)], + ), + ) def test_complex_xy(self): for dtype in self._dtypes: - x = np.random.random( - (2, 3, 4, 5)).astype(dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(dtype) - y = np.random.random( - (2, 3, 5, 4)).astype(dtype) + 1J * np.random.random( - (2, 3, 5, 4)).astype(dtype) + x = np.random.random((2, 3, 4, 5)).astype( + dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(dtype) + y = np.random.random((2, 3, 5, 4)).astype( + dtype + ) + 1j * np.random.random((2, 3, 5, 4)).astype(dtype) np_result = np.matmul(x, y) @@ -76,9 +79,9 @@ def test_complex_xy(self): def test_complex_x_real_y(self): for dtype in self._dtypes: - x = np.random.random( - (2, 3, 4, 5)).astype(dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(dtype) + x = np.random.random((2, 3, 4, 5)).astype( + dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(dtype) y = np.random.random((2, 3, 5, 4)).astype(dtype) np_result = np.matmul(x, y) @@ -90,9 +93,9 @@ def test_complex_x_real_y(self): def test_real_x_complex_y(self): for dtype in self._dtypes: x = np.random.random((2, 3, 4, 5)).astype(dtype) - y = np.random.random( - (2, 3, 5, 4)).astype(dtype) + 1J * np.random.random( - (2, 3, 5, 4)).astype(dtype) + y = np.random.random((2, 3, 5, 4)).astype( + dtype + ) + 1j * np.random.random((2, 3, 5, 4)).astype(dtype) np_result = np.matmul(x, y) @@ -103,11 +106,12 @@ def test_real_x_complex_y(self): # for coverage def test_complex_xy_gemv(self): for dtype in self._dtypes: - x = np.random.random( - (2, 1, 100)).astype(dtype) + 1J * np.random.random( - (2, 1, 100)).astype(dtype) - y = np.random.random((100)).astype(dtype) + 1J * np.random.random( - (100)).astype(dtype) + x = np.random.random((2, 1, 100)).astype( + dtype + ) + 1j * np.random.random((2, 1, 100)).astype(dtype) + y = np.random.random((100)).astype(dtype) + 1j * np.random.random( + (100) + ).astype(dtype) np_result = np.matmul(x, y) @@ -117,12 +121,12 @@ def test_complex_xy_gemv(self): # for coverage def test_complex_xy_gemm(self): for dtype in self._dtypes: - x = np.random.random( - (1, 2, 50)).astype(dtype) + 1J * np.random.random( - (1, 2, 50)).astype(dtype) - y = np.random.random( - (1, 50, 2)).astype(dtype) + 1J * np.random.random( - (1, 50, 2)).astype(dtype) + x = np.random.random((1, 2, 50)).astype( + dtype + ) + 1j * np.random.random((1, 2, 50)).astype(dtype) + y = np.random.random((1, 50, 2)).astype( + dtype + ) + 1j * np.random.random((1, 50, 2)).astype(dtype) np_result = np.matmul(x, y) diff --git a/python/paddle/fluid/tests/unittests/test_complex_op.py b/python/paddle/fluid/tests/unittests/test_complex_op.py index 498e8b43a9e2f..01c6b55921bd2 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_op.py +++ b/python/paddle/fluid/tests/unittests/test_complex_op.py @@ -48,7 +48,6 @@ def ref_complex_grad(x, y, dout): class TestComplexOp(OpTest): - def init_spec(self): self.x_shape = [10, 10] self.y_shape = [10, 10] @@ -61,8 +60,9 @@ def setUp(self): x = np.random.randn(*self.x_shape).astype(self.dtype) y = np.random.randn(*self.y_shape).astype(self.dtype) out_ref = ref_complex(x, y) - self.out_grad = np.random.randn(*self.x_shape).astype(self.dtype) \ - + 1j * np.random.randn(*self.y_shape).astype(self.dtype) + self.out_grad = np.random.randn(*self.x_shape).astype( + self.dtype + ) + 1j * np.random.randn(*self.y_shape).astype(self.dtype) self.inputs = {'X': x, 'Y': y} self.outputs = {'Out': out_ref} @@ -71,41 +71,49 @@ def test_check_output(self): def test_check_grad(self): dout = self.out_grad - dx, dy = ref_complex_grad(self.inputs['X'], self.inputs['Y'], - self.out_grad) - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[dx, dy], - user_defined_grad_outputs=[dout], - check_eager=True) + dx, dy = ref_complex_grad( + self.inputs['X'], self.inputs['Y'], self.out_grad + ) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[dx, dy], + user_defined_grad_outputs=[dout], + check_eager=True, + ) def test_check_grad_ignore_x(self): dout = self.out_grad - dx, dy = ref_complex_grad(self.inputs['X'], self.inputs['Y'], - self.out_grad) + dx, dy = ref_complex_grad( + self.inputs['X'], self.inputs['Y'], self.out_grad + ) self.assertTupleEqual(dx.shape, tuple(self.x_shape)) self.assertTupleEqual(dy.shape, tuple(self.y_shape)) - self.check_grad(['Y'], - 'Out', - no_grad_set=set('X'), - user_defined_grads=[dy], - user_defined_grad_outputs=[dout], - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set('X'), + user_defined_grads=[dy], + user_defined_grad_outputs=[dout], + check_eager=True, + ) def test_check_grad_ignore_y(self): dout = self.out_grad - dx, dy = ref_complex_grad(self.inputs['X'], self.inputs['Y'], - self.out_grad) - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[dx], - user_defined_grad_outputs=[dout], - check_eager=True) + dx, dy = ref_complex_grad( + self.inputs['X'], self.inputs['Y'], self.out_grad + ) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[dx], + user_defined_grad_outputs=[dout], + check_eager=True, + ) class TestComplexOpBroadcast1(TestComplexOp): - def init_spec(self): self.x_shape = [10, 3, 1, 4] self.y_shape = [100, 1] @@ -113,7 +121,6 @@ def init_spec(self): class TestComplexOpBroadcast2(TestComplexOp): - def init_spec(self): self.x_shape = [100, 1] self.y_shape = [10, 3, 1, 4] @@ -121,7 +128,6 @@ def init_spec(self): class TestComplexOpBroadcast3(TestComplexOp): - def init_spec(self): self.x_shape = [1, 100] self.y_shape = [100] @@ -129,7 +135,6 @@ def init_spec(self): class TestComplexAPI(unittest.TestCase): - def setUp(self): self.x = np.random.randn(10, 10) self.y = np.random.randn(10, 10) @@ -151,12 +156,9 @@ def test_static(self): exe = static.Executor() exe.run(sp) - [out_np] = exe.run(mp, - feed={ - "x": self.x, - "y": self.y - }, - fetch_list=[out]) + [out_np] = exe.run( + mp, feed={"x": self.x, "y": self.y}, fetch_list=[out] + ) np.testing.assert_allclose(self.out, out_np, rtol=1e-05) def test_eager(self): diff --git a/python/paddle/fluid/tests/unittests/test_complex_reshape.py b/python/paddle/fluid/tests/unittests/test_complex_reshape.py index 29bdb5d7c89ac..0e8398cf816ea 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_reshape.py +++ b/python/paddle/fluid/tests/unittests/test_complex_reshape.py @@ -21,7 +21,6 @@ class TestComplexReshape(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -30,24 +29,24 @@ def setUp(self): def test_shape_norm_dims(self): for dtype in self._dtypes: - x_np = np.random.randn( - 2, 3, - 4).astype(dtype) + 1j * np.random.randn(2, 3, 4).astype(dtype) + x_np = np.random.randn(2, 3, 4).astype( + dtype + ) + 1j * np.random.randn(2, 3, 4).astype(dtype) shape = (2, -1) for place in self._places: with dg.guard(place): x_var = dg.to_variable(x_np) y_var = paddle.reshape(x_var, shape) y_np = y_var.numpy() - np.testing.assert_allclose(np.reshape(x_np, shape), - y_np, - rtol=1e-05) + np.testing.assert_allclose( + np.reshape(x_np, shape), y_np, rtol=1e-05 + ) def test_shape_omit_dims(self): for dtype in self._dtypes: - x_np = np.random.randn( - 2, 3, - 4).astype(dtype) + 1j * np.random.randn(2, 3, 4).astype(dtype) + x_np = np.random.randn(2, 3, 4).astype( + dtype + ) + 1j * np.random.randn(2, 3, 4).astype(dtype) shape = (0, -1) shape_ = (2, 12) for place in self._places: @@ -55,9 +54,9 @@ def test_shape_omit_dims(self): x_var = dg.to_variable(x_np) y_var = paddle.reshape(x_var, shape) y_np = y_var.numpy() - np.testing.assert_allclose(np.reshape(x_np, shape_), - y_np, - rtol=1e-05) + np.testing.assert_allclose( + np.reshape(x_np, shape_), y_np, rtol=1e-05 + ) def test_eager(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_complex_simplenet.py b/python/paddle/fluid/tests/unittests/test_complex_simplenet.py index ead80c158905d..125b21888985a 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_simplenet.py +++ b/python/paddle/fluid/tests/unittests/test_complex_simplenet.py @@ -22,19 +22,20 @@ class Optimization_ex1(paddle.nn.Layer): - - def __init__(self, - shape, - param_attr=paddle.nn.initializer.Uniform(low=-5., high=5.), - dtype='float32'): + def __init__( + self, + shape, + param_attr=paddle.nn.initializer.Uniform(low=-5.0, high=5.0), + dtype='float32', + ): super(Optimization_ex1, self).__init__() - self.theta = self.create_parameter(shape=shape, - attr=param_attr, - dtype=dtype, - is_bias=False) + self.theta = self.create_parameter( + shape=shape, attr=param_attr, dtype=dtype, is_bias=False + ) self.A = paddle.to_tensor( - np.random.randn(4, 4) + np.random.randn(4, 4) * 1j) + np.random.randn(4, 4) + np.random.randn(4, 4) * 1j + ) def forward(self): loss = paddle.add(self.theta, self.A) @@ -42,7 +43,6 @@ def forward(self): class TestComplexSimpleNet(unittest.TestCase): - def setUp(self): self.devices = ['cpu'] if core.is_compiled_with_cuda(): @@ -55,8 +55,9 @@ def train(self, device): paddle.set_device(device) myLayer = Optimization_ex1(self.theta_size) - optimizer = paddle.optimizer.Adam(learning_rate=self.learning_rate, - parameters=myLayer.parameters()) + optimizer = paddle.optimizer.Adam( + learning_rate=self.learning_rate, parameters=myLayer.parameters() + ) for itr in range(self.iter): loss = myLayer() diff --git a/python/paddle/fluid/tests/unittests/test_complex_sum_layer.py b/python/paddle/fluid/tests/unittests/test_complex_sum_layer.py index 8dc3b433cd55a..7de44b33b0fce 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_sum_layer.py +++ b/python/paddle/fluid/tests/unittests/test_complex_sum_layer.py @@ -23,7 +23,6 @@ class TestComplexSumLayer(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -32,9 +31,9 @@ def setUp(self): def test_complex_basic_api(self): for dtype in self._dtypes: - input = rand([ - 2, 10, 10 - ]).astype(dtype) + 1j * rand([2, 10, 10]).astype(dtype) + input = rand([2, 10, 10]).astype(dtype) + 1j * rand( + [2, 10, 10] + ).astype(dtype) for place in self._places: with dg.guard(place): var_x = dg.to_variable(input) diff --git a/python/paddle/fluid/tests/unittests/test_complex_trace_layer.py b/python/paddle/fluid/tests/unittests/test_complex_trace_layer.py index c0400d6fc5d1f..bb8bc7f445c19 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_trace_layer.py +++ b/python/paddle/fluid/tests/unittests/test_complex_trace_layer.py @@ -22,7 +22,6 @@ class TestComplexTraceLayer(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [fluid.CPUPlace()] @@ -31,14 +30,15 @@ def setUp(self): def test_basic_api(self): for dtype in self._dtypes: - input = rand([ - 2, 20, 2, 3 - ]).astype(dtype) + 1j * rand([2, 20, 2, 3]).astype(dtype) + input = rand([2, 20, 2, 3]).astype(dtype) + 1j * rand( + [2, 20, 2, 3] + ).astype(dtype) for place in self._places: with dg.guard(place): var_x = dg.to_variable(input) - result = tensor.trace(var_x, offset=1, axis1=0, - axis2=2).numpy() + result = tensor.trace( + var_x, offset=1, axis1=0, axis2=2 + ).numpy() target = np.trace(input, offset=1, axis1=0, axis2=2) np.testing.assert_allclose(result, target, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_complex_transpose.py b/python/paddle/fluid/tests/unittests/test_complex_transpose.py index cfb8040674fa3..b08f7e1be0716 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_transpose.py +++ b/python/paddle/fluid/tests/unittests/test_complex_transpose.py @@ -21,7 +21,6 @@ class TestComplexTransposeLayer(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -30,9 +29,9 @@ def setUp(self): def test_transpose_by_complex_api(self): for dtype in self._dtypes: - data = np.random.random( - (2, 3, 4, 5)).astype(dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(dtype) + data = np.random.random((2, 3, 4, 5)).astype( + dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(dtype) perm = [3, 2, 0, 1] np_trans = np.transpose(data, perm) for place in self._places: diff --git a/python/paddle/fluid/tests/unittests/test_complex_variable.py b/python/paddle/fluid/tests/unittests/test_complex_variable.py index c7173a7b814b9..b3afc1d9df9df 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_variable.py +++ b/python/paddle/fluid/tests/unittests/test_complex_variable.py @@ -23,10 +23,10 @@ class TestComplexVariable(unittest.TestCase): - def compare(self): - a = np.array([[1.0 + 1.0j, 2.0 + 1.0j], - [3.0 + 1.0j, 4.0 + 1.0j]]).astype(self._dtype) + a = np.array( + [[1.0 + 1.0j, 2.0 + 1.0j], [3.0 + 1.0j, 4.0 + 1.0j]] + ).astype(self._dtype) b = np.array([[1.0 + 1.0j, 1.0 + 1.0j]]).astype(self._dtype) with dg.guard(): @@ -46,16 +46,22 @@ def test_attrs(self): self.compare() def test_convert_np_dtype_to_dtype(self): - self.assertEqual(convert_np_dtype_to_dtype_(np.complex64), - core.VarDesc.VarType.COMPLEX64) - self.assertEqual(convert_np_dtype_to_dtype_(np.complex64), - core.VarDesc.VarType.COMPLEX64) + self.assertEqual( + convert_np_dtype_to_dtype_(np.complex64), + core.VarDesc.VarType.COMPLEX64, + ) + self.assertEqual( + convert_np_dtype_to_dtype_(np.complex64), + core.VarDesc.VarType.COMPLEX64, + ) def test_convert_dtype(self): - self.assertEqual(convert_dtype(core.VarDesc.VarType.COMPLEX64), - "complex64") - self.assertEqual(convert_dtype(core.VarDesc.VarType.COMPLEX128), - "complex128") + self.assertEqual( + convert_dtype(core.VarDesc.VarType.COMPLEX64), "complex64" + ) + self.assertEqual( + convert_dtype(core.VarDesc.VarType.COMPLEX128), "complex128" + ) def test_eager(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_complex_view_op.py b/python/paddle/fluid/tests/unittests/test_complex_view_op.py index ad283a058396b..451469dfa2101 100644 --- a/python/paddle/fluid/tests/unittests/test_complex_view_op.py +++ b/python/paddle/fluid/tests/unittests/test_complex_view_op.py @@ -34,14 +34,14 @@ def ref_view_as_real(x): class TestViewAsComplexOp(OpTest): - def setUp(self): self.op_type = "as_complex" self.python_api = paddle.as_complex x = np.random.randn(10, 10, 2).astype("float64") out_ref = ref_view_as_complex(x) - self.out_grad = np.ones( - [10, 10], dtype="float64") + 1j * np.ones([10, 10], dtype="float64") + self.out_grad = np.ones([10, 10], dtype="float64") + 1j * np.ones( + [10, 10], dtype="float64" + ) self.inputs = {'X': x} self.outputs = {'Out': out_ref} @@ -49,15 +49,16 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[ref_view_as_real(self.out_grad)], - user_defined_grad_outputs=[self.out_grad], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[ref_view_as_real(self.out_grad)], + user_defined_grad_outputs=[self.out_grad], + check_eager=True, + ) class TestViewAsRealOp(OpTest): - def setUp(self): self.op_type = "as_real" real = np.random.randn(10, 10).astype("float64") @@ -73,15 +74,16 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[ref_view_as_complex(self.out_grad)], - user_defined_grad_outputs=[self.out_grad], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[ref_view_as_complex(self.out_grad)], + user_defined_grad_outputs=[self.out_grad], + check_eager=True, + ) class TestViewAsComplexAPI(unittest.TestCase): - def setUp(self): self.x = np.random.randn(10, 10, 2) self.out = ref_view_as_complex(self.x) @@ -109,7 +111,6 @@ def test_eager(self): class TestViewAsRealAPI(unittest.TestCase): - def setUp(self): self.x = np.random.randn(10, 10) + 1j * np.random.randn(10, 10) self.out = ref_view_as_real(self.x) diff --git a/python/paddle/fluid/tests/unittests/test_concat_op.py b/python/paddle/fluid/tests/unittests/test_concat_op.py index 76397a2083bc5..624f2be4a9f8e 100644 --- a/python/paddle/fluid/tests/unittests/test_concat_op.py +++ b/python/paddle/fluid/tests/unittests/test_concat_op.py @@ -14,7 +14,11 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import OpTest, skip_check_grad_ci, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + skip_check_grad_ci, + convert_float_to_uint16, +) import paddle.fluid as fluid from paddle.fluid import Program, core, program_guard from paddle.fluid.framework import _test_eager_guard @@ -25,7 +29,6 @@ class TestConcatOp(OpTest): - def setUp(self): self.op_type = "concat" self.python_api = paddle.concat @@ -40,8 +43,9 @@ def setUp(self): self.actual_axis = self.axis self.outputs = { - 'Out': - np.concatenate((self.x0, self.x1, self.x2), axis=self.actual_axis) + 'Out': np.concatenate( + (self.x0, self.x1, self.x2), axis=self.actual_axis + ) } def get_dtype(self): @@ -81,7 +85,6 @@ def init_test_data(self): class TestConcatOp2(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((2, 3, 4, 5)).astype(self.dtype) self.x1 = np.random.random((2, 3, 4, 5)).astype(self.dtype) @@ -90,9 +93,9 @@ def init_test_data(self): @skip_check_grad_ci( - reason="The function 'check_grad' for large inputs is too slow.") + reason="The function 'check_grad' for large inputs is too slow." +) class TestConcatOp3(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((1, 256, 170, 256)).astype(self.dtype) self.x1 = np.random.random((1, 128, 170, 256)).astype(self.dtype) @@ -104,11 +107,9 @@ def test_check_grad(self): @skip_check_grad_ci( - reason= - "This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." + reason="This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." ) class TestConcatOp4(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((2, 3, 4, 5)).astype(self.dtype) self.x1 = np.random.random((2, 3, 4, 5)).astype(self.dtype) @@ -120,7 +121,6 @@ def test_check_grad(self): class TestConcatOp5(TestConcatOp): - def init_test_data(self): self.x0 = np.random.random((5, 1, 4, 5)).astype(self.dtype) self.x1 = np.random.random((5, 2, 4, 5)).astype(self.dtype) @@ -129,7 +129,6 @@ def init_test_data(self): class TestConcatOp6(TestConcatOp): - def setUp(self): self.op_type = "concat" self.dtype = self.get_dtype() @@ -138,8 +137,11 @@ def setUp(self): self.lod = [[20, 80]] self.out_lod = [[20, 80, 20, 80, 20, 80]] self.inputs = { - 'X': [('x0', (self.x0, self.lod)), ('x1', (self.x1, self.lod)), - ('x2', (self.x2, self.lod))] + 'X': [ + ('x0', (self.x0, self.lod)), + ('x1', (self.x1, self.lod)), + ('x2', (self.x2, self.lod)), + ] } self.attrs = {'axis': self.axis} if self.axis < 0: @@ -166,9 +168,7 @@ def init_test_data(self): def create_test_AxisTensor(parent): - class TestConcatAxisTensor(parent): - def setUp(self): self.op_type = "concat" self.python_api = paddle.concat @@ -177,20 +177,22 @@ def setUp(self): self.inputs = { 'X': [('x0', self.x0), ('x1', self.x1), ('x2', self.x2)], - 'AxisTensor': np.array([self.axis]).astype("int32") + 'AxisTensor': np.array([self.axis]).astype("int32"), } self.attrs = {} if self.axis < 0: self.actual_axis = self.axis + len(self.x0.shape) - self.actual_axis = self.actual_axis if self.actual_axis > 0 else 0 + self.actual_axis = ( + self.actual_axis if self.actual_axis > 0 else 0 + ) else: self.actual_axis = self.axis self.outputs = { - 'Out': - np.concatenate((self.x0, self.x1, self.x2), - axis=self.actual_axis) + 'Out': np.concatenate( + (self.x0, self.x1, self.x2), axis=self.actual_axis + ) } cls_name = "{0}_{1}".format(parent.__name__, "AxisTensor") @@ -205,13 +207,11 @@ def setUp(self): create_test_AxisTensor(TestConcatOp5) create_test_AxisTensor(TestConcatOp6) -#----------------Concat Fp16---------------- +# ----------------Concat Fp16---------------- def create_test_fp16(parent): - class TestConcatFp16(parent): - def get_dtype(self): return np.float16 @@ -228,13 +228,12 @@ def get_dtype(self): create_test_fp16(TestConcatOp6) -#----------------Concat Bf16---------------- +# ----------------Concat Bf16---------------- def create_test_bf16(parent): - - @unittest.skipIf(not paddle.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not paddle.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestConcatBf16(parent): - def get_dtype(self): return np.uint16 @@ -247,17 +246,18 @@ def get_dtype(self): class TestConcatOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of concat_op should be list. x1 = fluid.layers.data(shape=[4], dtype='int32', name='x1') fluid.layers.concat(x1) # The item in input must be Variable. - x2 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - x3 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x2 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + x3 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.concat, [x2]) # The input dtype of concat_op must be float16, float32, float64, int32, int64. x4 = fluid.layers.data(shape=[4], dtype='uint8', name='x4') @@ -281,7 +281,6 @@ def test_input_same_dtype(): class TestConcatAPI(unittest.TestCase): - def test_fluid_api(self): paddle.enable_static() x_1 = fluid.data(shape=[None, 1, 4, 5], dtype='int32', name='x_1') @@ -298,22 +297,20 @@ def test_fluid_api(self): out_3 = fluid.layers.concat(input=[x_2, x_3], axis=positive_1_int64) exe = fluid.Executor(place=fluid.CPUPlace()) - [res_1, res_2, res_3] = exe.run(fluid.default_main_program(), - feed={ - "x_1": input_2, - "x_2": input_2, - "x_3": input_3 - }, - fetch_list=[out_1, out_2, out_3]) + [res_1, res_2, res_3] = exe.run( + fluid.default_main_program(), + feed={"x_1": input_2, "x_2": input_2, "x_3": input_3}, + fetch_list=[out_1, out_2, out_3], + ) assert np.array_equal(res_1, np.concatenate((input_2, input_3), axis=1)) assert np.array_equal(res_2, np.concatenate((input_2, input_3), axis=1)) assert np.array_equal(res_3, np.concatenate((input_2, input_3), axis=1)) def test_api(self): paddle.enable_static() - x_1 = paddle.fluid.data(shape=[None, 1, 4, 5], - dtype='int32', - name='x_1') + x_1 = paddle.fluid.data( + shape=[None, 1, 4, 5], dtype='int32', name='x_1' + ) paddle.concat([x_1, x_1], 0) input_2 = np.random.random([2, 1, 4, 5]).astype("int32") @@ -329,14 +326,11 @@ def test_api(self): out_4 = paddle.concat(x=[x_2, x_3], axis=negative_int64) exe = paddle.static.Executor(place=paddle.CPUPlace()) - [res_1, res_2, res_3, - res_4] = exe.run(paddle.static.default_main_program(), - feed={ - "x_1": input_2, - "x_2": input_2, - "x_3": input_3 - }, - fetch_list=[out_1, out_2, out_3, out_4]) + [res_1, res_2, res_3, res_4] = exe.run( + paddle.static.default_main_program(), + feed={"x_1": input_2, "x_2": input_2, "x_3": input_3}, + fetch_list=[out_1, out_2, out_3, out_4], + ) assert np.array_equal(res_1, np.concatenate((input_2, input_3), axis=1)) assert np.array_equal(res_2, np.concatenate((input_2, input_3), axis=1)) assert np.array_equal(res_3, np.concatenate((input_2, input_3), axis=1)) @@ -367,10 +361,12 @@ def test_eager(self): def test_errors(self): with program_guard(Program(), Program()): # The item in input must be Variable. - x2 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - x3 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x2 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + x3 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, paddle.concat, [x2]) # The input dtype of concat_op must be float16, float32, float64, int32, int64. x4 = paddle.fluid.data(shape=[4], dtype='uint8', name='x4') @@ -404,8 +400,11 @@ def setUp(self): self.iter_num = 3 self.input_shape = [2, 3] self.x = np.random.random(self.input_shape).astype("float32") - self.place = fluid.CUDAPlace(0) \ - if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def set_program(self, use_fluid_api): paddle.enable_static() @@ -414,9 +413,9 @@ def set_program(self, use_fluid_api): with fluid.program_guard(self.program): input = fluid.layers.assign(self.x) tensor_array = fluid.layers.create_array(dtype='float32') - zero = fluid.layers.fill_constant(shape=[1], - value=0, - dtype="int64") + zero = fluid.layers.fill_constant( + shape=[1], value=0, dtype="int64" + ) for i in range(self.iter_num): fluid.layers.array_write(input, zero + i, tensor_array) @@ -449,11 +448,11 @@ def _run_static_mode(self, use_fluid_api): exe = fluid.Executor(self.place) res = exe.run(self.program, fetch_list=self.out_var) np.testing.assert_array_equal( - res[0], np.concatenate([self.x] * self.iter_num, axis=self.axis)) + res[0], np.concatenate([self.x] * self.iter_num, axis=self.axis) + ) class TestConcatDoubleGradCheck(unittest.TestCase): - def concat_wrapper(self, x): return paddle.concat(x) @@ -470,17 +469,21 @@ def func(self, place): out = paddle.concat([data1, data2]) data1_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) data2_arr = np.random.uniform(-1, 1, data2.shape).astype(dtype) - gradient_checker.double_grad_check([data1, data2], - out, - x_init=[data1_arr, data2_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data1, data2], + out, + x_init=[data1_arr, data2_arr], + place=place, + eps=eps, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) gradient_checker.double_grad_check_for_dygraph( - self.concat_wrapper, [data1, data2], + self.concat_wrapper, + [data1, data2], out, x_init=[data1_arr, data2_arr], - place=place) + place=place, + ) def test_grad(self): paddle.enable_static() @@ -492,7 +495,6 @@ def test_grad(self): class TestConcatTripleGradCheck(unittest.TestCase): - def concat_wrapper(self, x): return paddle.concat(x, 1) @@ -509,17 +511,21 @@ def func(self, place): out = paddle.concat([data1, data2], 1) data1_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) data2_arr = np.random.uniform(-1, 1, data2.shape).astype(dtype) - gradient_checker.double_grad_check([data1, data2], - out, - x_init=[data1_arr, data2_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data1, data2], + out, + x_init=[data1_arr, data2_arr], + place=place, + eps=eps, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) gradient_checker.double_grad_check_for_dygraph( - self.concat_wrapper, [data1, data2], + self.concat_wrapper, + [data1, data2], out, x_init=[data1_arr, data2_arr], - place=place) + place=place, + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_cond.py b/python/paddle/fluid/tests/unittests/test_cond.py index f0c5516eef46a..3d05d7694fa21 100644 --- a/python/paddle/fluid/tests/unittests/test_cond.py +++ b/python/paddle/fluid/tests/unittests/test_cond.py @@ -29,7 +29,6 @@ class TestCondInputOutput(unittest.TestCase): - def test_return_single_var(self): """ pseudocode: @@ -57,13 +56,16 @@ def false_func(): out = layers.cond(pred, true_func, false_func) # out is one tensor - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - ret, = exe.run(main_program, fetch_list=[out.name]) - np.testing.assert_allclose(np.asarray(ret), - np.full((3, 2), -1, np.int32), - rtol=1e-05) + (ret,) = exe.run(main_program, fetch_list=[out.name]) + np.testing.assert_allclose( + np.asarray(ret), np.full((3, 2), -1, np.int32), rtol=1e-05 + ) def test_return_var_tuple(self): """ @@ -78,18 +80,14 @@ def test_return_var_tuple(self): paddle.enable_static() def true_func(): - return layers.fill_constant(shape=[1, 2], dtype='int32', - value=1), layers.fill_constant( - shape=[2, 3], - dtype='bool', - value=True) + return layers.fill_constant( + shape=[1, 2], dtype='int32', value=1 + ), layers.fill_constant(shape=[2, 3], dtype='bool', value=True) def false_func(): - return layers.fill_constant(shape=[3, 4], dtype='float32', - value=3), layers.fill_constant( - shape=[4, 5], - dtype='int64', - value=2) + return layers.fill_constant( + shape=[3, 4], dtype='float32', value=3 + ), layers.fill_constant(shape=[4, 5], dtype='int64', value=2) main_program = Program() startup_program = Program() @@ -98,16 +96,19 @@ def false_func(): out = layers.cond(pred, true_func, false_func) # out is a tuple containing 2 tensors - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) ret = exe.run(main_program, fetch_list=out) - np.testing.assert_allclose(np.asarray(ret[0]), - np.full((1, 2), 1, np.int32), - rtol=1e-05) - np.testing.assert_allclose(np.asarray(ret[1]), - np.full((2, 3), True, bool), - rtol=1e-05) + np.testing.assert_allclose( + np.asarray(ret[0]), np.full((1, 2), 1, np.int32), rtol=1e-05 + ) + np.testing.assert_allclose( + np.asarray(ret[1]), np.full((2, 3), True, bool), rtol=1e-05 + ) def test_pass_and_modify_var(self): """ @@ -135,20 +136,28 @@ def false_func(a, i): with program_guard(main_program, startup_program): a = layers.fill_constant(shape=[3, 2, 1], dtype='int32', value=7) i = fluid.data(name="i", shape=[1], dtype='int32') - pred = ((i % 2) == 0) - a = layers.cond(pred, lambda: true_func(a, i), - lambda: false_func(a, i)) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + pred = (i % 2) == 0 + a = layers.cond( + pred, lambda: true_func(a, i), lambda: false_func(a, i) + ) + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) for feed_i in range(5): expected_a = 7 * (feed_i + 1) if feed_i % 2 == 0 else 8 - feed_i - ret, = exe.run(main_program, - feed={'i': np.full((1), feed_i, np.int32)}, - fetch_list=[a]) - np.testing.assert_allclose(np.asarray(ret), - np.full((3, 2, 1), expected_a, np.int32), - rtol=1e-05) + (ret,) = exe.run( + main_program, + feed={'i': np.full((1), feed_i, np.int32)}, + fetch_list=[a], + ) + np.testing.assert_allclose( + np.asarray(ret), + np.full((3, 2, 1), expected_a, np.int32), + rtol=1e-05, + ) def test_return_none(self): """ @@ -172,12 +181,15 @@ def false_func(): startup_program = Program() with program_guard(main_program, startup_program): i = fluid.data(name="i", shape=[1], dtype='int32') - pred = ((i % 2) == 0) + pred = (i % 2) == 0 out1 = layers.cond(pred, true_func, false_func) out2 = layers.cond(pred, None, false_func) out3 = layers.cond(pred, true_func, None) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) for feed_i in range(5): # Test that output is None is runnable @@ -200,17 +212,15 @@ def func_return_one_tensor(): return layers.fill_constant(shape=[2, 7], dtype='int32', value=3) def func_return_two_tensors(): - return layers.fill_constant(shape=[3, 1], dtype='int32', - value=7), layers.fill_constant( - shape=[3, 1], - dtype='int32', - value=8) + return layers.fill_constant( + shape=[3, 1], dtype='int32', value=7 + ), layers.fill_constant(shape=[3, 1], dtype='int32', value=8) main_program = Program() startup_program = Program() with program_guard(main_program, startup_program): i = fluid.data(name="i", shape=[1], dtype='int32') - pred = ((i % 2) == 0) + pred = (i % 2) == 0 with self.assertRaises(TypeError): out = layers.cond(pred, i, func_return_one_tensor) @@ -218,47 +228,57 @@ def func_return_two_tensors(): out = layers.cond(pred, func_return_one_tensor, np.asarray([3])) with self.assertRaises(Exception) as e: - out = layers.cond(pred, func_return_none, - func_return_one_tensor) + out = layers.cond( + pred, func_return_none, func_return_one_tensor + ) self.assertTrue( - "Incompatible return values of true_fn and false_fn in cond" in - str(e.exception)) + "Incompatible return values of true_fn and false_fn in cond" + in str(e.exception) + ) with self.assertRaises(Exception) as e: - out = layers.cond(pred, func_return_two_tensors, - func_return_none) + out = layers.cond( + pred, func_return_two_tensors, func_return_none + ) self.assertTrue( - "Incompatible return values of true_fn and false_fn in cond" in - str(e.exception)) + "Incompatible return values of true_fn and false_fn in cond" + in str(e.exception) + ) with self.assertRaises(Exception) as e: - out = layers.cond(pred, func_return_one_tensor, - func_return_two_tensors) + out = layers.cond( + pred, func_return_one_tensor, func_return_two_tensors + ) self.assertTrue( "true fn returns 1 vars, but false fn returns 2 vars, which is not equals" - in str(e.exception)) + in str(e.exception) + ) def test_extremely_simple_net_with_op_in_condition(self): paddle.enable_static() main_program = fluid.Program() startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): - a = fluid.layers.fill_constant(shape=[1], - dtype='float32', - value=1.23) + a = fluid.layers.fill_constant( + shape=[1], dtype='float32', value=1.23 + ) a.stop_gradient = False - b = fluid.layers.fill_constant(shape=[1], - dtype='float32', - value=1.25) + b = fluid.layers.fill_constant( + shape=[1], dtype='float32', value=1.25 + ) b.stop_gradient = False out = layers.cond(a - b < -1.0, lambda: a, lambda: b) append_backward(out) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - ret = exe.run(main_program, - fetch_list=[out, b, a.grad_name, b.grad_name]) + ret = exe.run( + main_program, fetch_list=[out, b, a.grad_name, b.grad_name] + ) # Note: fill_constant has loss of precision, you have to assertEqual # with values doens't lose precision in float-point number. self.assertEqual(ret[0][0], ret[1][0]) @@ -267,7 +287,6 @@ def test_extremely_simple_net_with_op_in_condition(self): class TestCondNestedControlFlow(unittest.TestCase): - def test_cond_inside_cond(self): """ pseudocode: @@ -288,25 +307,37 @@ def test_cond_inside_cond(self): paddle.enable_static() def less_than_branch(i, a): - return layers.cond(i >= 3.0, lambda: layers.elementwise_add(a, a), - lambda: layers.elementwise_sub(a, a)) + return layers.cond( + i >= 3.0, + lambda: layers.elementwise_add(a, a), + lambda: layers.elementwise_sub(a, a), + ) def greater_equal_branch(i, a): - return layers.cond(i < 8.0, lambda: layers.elementwise_mul(a, a), - lambda: layers.elementwise_div(a, a)) + return layers.cond( + i < 8.0, + lambda: layers.elementwise_mul(a, a), + lambda: layers.elementwise_div(a, a), + ) main_program = Program() startup_program = Program() with program_guard(main_program, startup_program): i = fluid.data(name="i", shape=[1], dtype='float32') a = 2.0 * i - out = layers.cond(i < 5.0, lambda: less_than_branch(i, a), - lambda: greater_equal_branch(i, a)) + out = layers.cond( + i < 5.0, + lambda: less_than_branch(i, a), + lambda: greater_equal_branch(i, a), + ) mean = paddle.mean(out) append_backward(mean) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) for feed_i in range(0, 10): expected_a = 2.0 * feed_i @@ -316,9 +347,11 @@ def greater_equal_branch(i, a): else: expected_ret = expected_a * expected_a if feed_i < 8 else 1.0 expected_a_grad = 2.0 * expected_a if feed_i < 8 else 0.0 - ret = exe.run(main_program, - feed={'i': np.full((1), feed_i, np.float32)}, - fetch_list=[out.name, a.grad_name]) + ret = exe.run( + main_program, + feed={'i': np.full((1), feed_i, np.float32)}, + fetch_list=[out.name, a.grad_name], + ) self.assertEqual(ret[0][0], expected_ret) self.assertEqual(ret[1][0], expected_a_grad) @@ -328,24 +361,34 @@ def test_cond_op_in_condition(self): startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): - a = fluid.layers.fill_constant(shape=[1], - dtype='float32', - value=1.23) + a = fluid.layers.fill_constant( + shape=[1], dtype='float32', value=1.23 + ) a.stop_gradient = False - b = fluid.layers.fill_constant(shape=[1], - dtype='float32', - value=1.24) + b = fluid.layers.fill_constant( + shape=[1], dtype='float32', value=1.24 + ) b.stop_gradient = False out = fluid.layers.cond( - a < b, lambda: fluid.layers.cond( - a - b < -1.0, lambda: fluid.layers.elementwise_add(a, b), - lambda: fluid.layers.elementwise_mul(a, b)), lambda: - fluid.layers.cond(a == b, lambda: fluid.layers.elementwise_sub( - a, b), lambda: fluid.layers.elementwise_pow(a, b))) + a < b, + lambda: fluid.layers.cond( + a - b < -1.0, + lambda: fluid.layers.elementwise_add(a, b), + lambda: fluid.layers.elementwise_mul(a, b), + ), + lambda: fluid.layers.cond( + a == b, + lambda: fluid.layers.elementwise_sub(a, b), + lambda: fluid.layers.elementwise_pow(a, b), + ), + ) append_backward(out) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) ret = exe.run(main_program, fetch_list=[out, a.grad_name, b.grad_name]) # Note: fill_constant has loss of precision, so we assertAlmostEqual. @@ -355,7 +398,6 @@ def test_cond_op_in_condition(self): class TestCondBackward(unittest.TestCase): - def backward_value_helper(self, cond_func, use_cuda, use_parallel_exe): """ Helper function that compares calculated backward value is close to dy/dx @@ -379,70 +421,76 @@ def backward_value_helper(self, cond_func, use_cuda, use_parallel_exe): num_devices = 1 if use_parallel_exe: os.environ['CPU_NUM'] = str(2) - exe = fluid.ParallelExecutor(use_cuda=use_cuda, - main_program=main_program, - loss_name=loss.name) + exe = fluid.ParallelExecutor( + use_cuda=use_cuda, + main_program=main_program, + loss_name=loss.name, + ) num_devices = exe.device_count delta = 0.005 for feed_i in range(0, 10): feed_img = np.random.random(size=[1, 9]).astype(np.float32) - feed_label = np.random.randint(low=0, - high=10, - size=[1, 1], - dtype=np.int64) + feed_label = np.random.randint( + low=0, high=10, size=[1, 1], dtype=np.int64 + ) if use_parallel_exe: img_grad, loss_value = exe.run( feed={ 'i': np.full((num_devices), feed_i, np.int32), 'image': np.repeat(feed_img, num_devices, axis=0), - 'label': np.repeat(feed_label, num_devices, axis=0) + 'label': np.repeat(feed_label, num_devices, axis=0), }, - fetch_list=[img.grad_name, loss.name]) + fetch_list=[img.grad_name, loss.name], + ) else: img_grad, loss_value = exe.run( main_program, feed={ 'i': np.full((1), feed_i, np.int32), 'image': feed_img, - 'label': feed_label + 'label': feed_label, }, - fetch_list=[img.grad_name, loss.name]) + fetch_list=[img.grad_name, loss.name], + ) numerical_grad = np.zeros(shape=[num_devices, 9], dtype=np.float32) feed_img_delta = np.copy(feed_img) for j in range(9): feed_img_delta[0][j] = feed_img[0][j] + delta if use_parallel_exe: - loss_delta = exe.run(feed={ - 'i': - np.full((num_devices), feed_i, np.int32), - 'image': - np.repeat(feed_img_delta, num_devices, axis=0), - 'label': - np.repeat(feed_label, num_devices, axis=0) - }, - fetch_list=[loss.name]) - multi_device_grad = (loss_delta[0] - - loss_value[0]) / delta / num_devices + loss_delta = exe.run( + feed={ + 'i': np.full((num_devices), feed_i, np.int32), + 'image': np.repeat( + feed_img_delta, num_devices, axis=0 + ), + 'label': np.repeat(feed_label, num_devices, axis=0), + }, + fetch_list=[loss.name], + ) + multi_device_grad = ( + (loss_delta[0] - loss_value[0]) / delta / num_devices + ) for d in range(num_devices): numerical_grad[d][j] = multi_device_grad[d] else: - loss_delta = exe.run(main_program, - feed={ - 'i': np.full((1), feed_i, - np.int32), - 'image': feed_img_delta, - 'label': feed_label - }, - fetch_list=[loss.name]) - numerical_grad[0][j] = (loss_delta[0] - - loss_value[0]) / delta + loss_delta = exe.run( + main_program, + feed={ + 'i': np.full((1), feed_i, np.int32), + 'image': feed_img_delta, + 'label': feed_label, + }, + fetch_list=[loss.name], + ) + numerical_grad[0][j] = ( + loss_delta[0] - loss_value[0] + ) / delta feed_img_delta[0][j] = feed_img[0][j] - np.testing.assert_allclose(img_grad, - numerical_grad, - rtol=0.05, - atol=0.05) + np.testing.assert_allclose( + img_grad, numerical_grad, rtol=0.05, atol=0.05 + ) def add_optimizer_helper(self, cond_func, use_cuda, use_parallel_exe): """ @@ -463,43 +511,49 @@ def add_optimizer_helper(self, cond_func, use_cuda, use_parallel_exe): exe.run(startup_program) if use_parallel_exe: os.environ['CPU_NUM'] = str(2) - exe = fluid.ParallelExecutor(use_cuda=use_cuda, - main_program=main_program, - loss_name=loss.name) + exe = fluid.ParallelExecutor( + use_cuda=use_cuda, + main_program=main_program, + loss_name=loss.name, + ) num_devices = exe.device_count for feed_i in range(0, 10): feed_img = np.random.random(size=[16, 784]).astype(np.float32) - feed_label = np.random.randint(low=0, - high=10, - size=[16, 1], - dtype=np.int64) + feed_label = np.random.randint( + low=0, high=10, size=[16, 1], dtype=np.int64 + ) if use_parallel_exe: - exe.run(feed={ - 'i': np.full((num_devices), feed_i, np.int32), - 'image': np.repeat(feed_img, num_devices, axis=0), - 'label': np.repeat(feed_label, num_devices, axis=0) - }, - fetch_list=[loss.name]) + exe.run( + feed={ + 'i': np.full((num_devices), feed_i, np.int32), + 'image': np.repeat(feed_img, num_devices, axis=0), + 'label': np.repeat(feed_label, num_devices, axis=0), + }, + fetch_list=[loss.name], + ) else: - exe.run(main_program, - feed={ - 'i': np.full((1), feed_i, np.int32), - 'image': feed_img, - 'label': feed_label - }, - fetch_list=[loss]) + exe.run( + main_program, + feed={ + 'i': np.full((1), feed_i, np.int32), + 'image': feed_img, + 'label': feed_label, + }, + fetch_list=[loss], + ) def test_cond_backward(self): paddle.enable_static() def cond_func(i, img, label): - predicate = ((i % 2) == 0) + predicate = (i % 2) == 0 return layers.cond( predicate, lambda: simple_fc_net_with_inputs(img, label, class_num=10), - lambda: batchnorm_fc_with_inputs(img, label, class_num=10)) + lambda: batchnorm_fc_with_inputs(img, label, class_num=10), + ) for use_parallel_exe in [False, True]: if use_parallel_exe and os.name == "nt": @@ -508,10 +562,12 @@ def cond_func(i, img, label): ) continue - self.backward_value_helper(cond_func, core.is_compiled_with_cuda(), - use_parallel_exe) - self.add_optimizer_helper(cond_func, core.is_compiled_with_cuda(), - use_parallel_exe) + self.backward_value_helper( + cond_func, core.is_compiled_with_cuda(), use_parallel_exe + ) + self.add_optimizer_helper( + cond_func, core.is_compiled_with_cuda(), use_parallel_exe + ) def test_half_nested_cond_backward(self): paddle.enable_static() @@ -520,15 +576,18 @@ def branch(i, img, label): return layers.cond( (i % 2) == 0, lambda: simple_fc_net_with_inputs(img, label, class_num=10), - lambda: batchnorm_fc_with_inputs(img, label, class_num=10)) + lambda: batchnorm_fc_with_inputs(img, label, class_num=10), + ) def cond_func_simple_net_at_true(i, img, label): - return layers.cond(i < 5, lambda: branch(i, img, label), - lambda: paddle.mean(img)) + return layers.cond( + i < 5, lambda: branch(i, img, label), lambda: paddle.mean(img) + ) def cond_func_simple_net_at_false(i, img, label): - return layers.cond(i < 5, lambda: paddle.mean(img), - lambda: branch(i, img, label)) + return layers.cond( + i < 5, lambda: paddle.mean(img), lambda: branch(i, img, label) + ) for use_parallel_exe in [False, True]: if use_parallel_exe and os.name == "nt": @@ -537,35 +596,47 @@ def cond_func_simple_net_at_false(i, img, label): ) continue - self.backward_value_helper(cond_func_simple_net_at_true, - core.is_compiled_with_cuda(), - use_parallel_exe) - self.add_optimizer_helper(cond_func_simple_net_at_true, - core.is_compiled_with_cuda(), - use_parallel_exe) - self.backward_value_helper(cond_func_simple_net_at_false, - core.is_compiled_with_cuda(), - use_parallel_exe) - self.add_optimizer_helper(cond_func_simple_net_at_false, - core.is_compiled_with_cuda(), - use_parallel_exe) + self.backward_value_helper( + cond_func_simple_net_at_true, + core.is_compiled_with_cuda(), + use_parallel_exe, + ) + self.add_optimizer_helper( + cond_func_simple_net_at_true, + core.is_compiled_with_cuda(), + use_parallel_exe, + ) + self.backward_value_helper( + cond_func_simple_net_at_false, + core.is_compiled_with_cuda(), + use_parallel_exe, + ) + self.add_optimizer_helper( + cond_func_simple_net_at_false, + core.is_compiled_with_cuda(), + use_parallel_exe, + ) def test_nested_cond_backward(self): paddle.enable_static() def branch(i, img, label, mod_two): if mod_two: - predicate = ((i % 2) == 0) + predicate = (i % 2) == 0 else: - predicate = ((i % 2) != 0) + predicate = (i % 2) != 0 return layers.cond( predicate, lambda: simple_fc_net_with_inputs(img, label, class_num=10), - lambda: batchnorm_fc_with_inputs(img, label, class_num=10)) + lambda: batchnorm_fc_with_inputs(img, label, class_num=10), + ) def cond_func(i, img, label): - return layers.cond(i < 5, lambda: branch(i, img, label, True), - lambda: branch(i, img, label, False)) + return layers.cond( + i < 5, + lambda: branch(i, img, label, True), + lambda: branch(i, img, label, False), + ) for use_parallel_exe in [False, True]: if use_parallel_exe and os.name == "nt": @@ -573,14 +644,15 @@ def cond_func(i, img, label): "Skip use_parallel_exe=True in Windows because of flaky test when using PE under old Windows machine" ) continue - self.backward_value_helper(cond_func, core.is_compiled_with_cuda(), - use_parallel_exe) - self.add_optimizer_helper(cond_func, core.is_compiled_with_cuda(), - use_parallel_exe) + self.backward_value_helper( + cond_func, core.is_compiled_with_cuda(), use_parallel_exe + ) + self.add_optimizer_helper( + cond_func, core.is_compiled_with_cuda(), use_parallel_exe + ) class TestCondWithError(unittest.TestCase): - def test_input_type_error(self): paddle.enable_static() main_program = framework.Program() diff --git a/python/paddle/fluid/tests/unittests/test_conditional_block.py b/python/paddle/fluid/tests/unittests/test_conditional_block.py index 6c0a290794cc5..fc8d926070e3e 100644 --- a/python/paddle/fluid/tests/unittests/test_conditional_block.py +++ b/python/paddle/fluid/tests/unittests/test_conditional_block.py @@ -24,7 +24,6 @@ class ConditionalBlockTest(unittest.TestCase): - def test_forward(self): main_program = fluid.Program() startup_program = fluid.Program() @@ -50,12 +49,12 @@ def test_forward(self): outs = exe.run( main_program, feed={'X': x}, - fetch_list=[main_program.block(0).var(data.name + "@GRAD")])[0] + fetch_list=[main_program.block(0).var(data.name + "@GRAD")], + )[0] print(outs) class TestConditionalBlockOpInferShape(unittest.TestCase): - def test_infer_shape(self): main_program = fluid.Program() startup_program = fluid.Program() @@ -64,24 +63,21 @@ def test_infer_shape(self): sub_block = main_program._create_block() main_program._rollback() step_scope = global_block.create_var( - type=core.VarDesc.VarType.STEP_SCOPES) - cond_var = layers.fill_constant(shape=[1], - dtype='bool', - value=False) + type=core.VarDesc.VarType.STEP_SCOPES + ) + cond_var = layers.fill_constant( + shape=[1], dtype='bool', value=False + ) - op = global_block.append_op(type='conditional_block', - inputs={ - 'Cond': [cond_var], - 'Input': [], - }, - outputs={ - 'Out': [], - 'Scope': [step_scope] - }, - attrs={ - 'sub_block': sub_block, - 'is_scalar_condition': True - }) + op = global_block.append_op( + type='conditional_block', + inputs={ + 'Cond': [cond_var], + 'Input': [], + }, + outputs={'Out': [], 'Scope': [step_scope]}, + attrs={'sub_block': sub_block, 'is_scalar_condition': True}, + ) op.desc.infer_shape(global_block.desc) diff --git a/python/paddle/fluid/tests/unittests/test_conj_op.py b/python/paddle/fluid/tests/unittests/test_conj_op.py index 577d8afbef036..67647458f5da7 100644 --- a/python/paddle/fluid/tests/unittests/test_conj_op.py +++ b/python/paddle/fluid/tests/unittests/test_conj_op.py @@ -27,7 +27,6 @@ class TestConjOp(OpTest): - def setUp(self): self.op_type = "conj" self.python_api = paddle.tensor.conj @@ -39,31 +38,34 @@ def init_dtype_type(self): self.dtype = np.complex64 def init_input_output(self): - x = (np.random.random((12, 14)) + 1j * np.random.random( - (12, 14))).astype(self.dtype) + x = ( + np.random.random((12, 14)) + 1j * np.random.random((12, 14)) + ).astype(self.dtype) out = np.conj(x) self.inputs = {'X': OpTest.np_dtype_to_fluid_dtype(x)} self.outputs = {'Out': out} def init_grad_input_output(self): - self.grad_out = (np.ones((12, 14)) + 1j * np.ones( - (12, 14))).astype(self.dtype) + self.grad_out = (np.ones((12, 14)) + 1j * np.ones((12, 14))).astype( + self.dtype + ) self.grad_in = np.conj(self.grad_out) def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.grad_in], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[self.grad_in], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) class TestComplexConjOp(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -72,9 +74,9 @@ def setUp(self): def test_conj_api(self): for dtype in self._dtypes: - input = rand([ - 2, 20, 2, 3 - ]).astype(dtype) + 1j * rand([2, 20, 2, 3]).astype(dtype) + input = rand([2, 20, 2, 3]).astype(dtype) + 1j * rand( + [2, 20, 2, 3] + ).astype(dtype) for place in self._places: with dg.guard(place): var_x = paddle.to_tensor(input) @@ -84,9 +86,9 @@ def test_conj_api(self): def test_conj_operator(self): for dtype in self._dtypes: - input = rand([ - 2, 20, 2, 3 - ]).astype(dtype) + 1j * rand([2, 20, 2, 3]).astype(dtype) + input = rand([2, 20, 2, 3]).astype(dtype) + 1j * rand( + [2, 20, 2, 3] + ).astype(dtype) for place in self._places: with dg.guard(place): var_x = paddle.to_tensor(input) @@ -95,21 +97,22 @@ def test_conj_operator(self): np.testing.assert_array_equal(result, target) def test_conj_static_mode(self): - def init_input_output(dtype): - input = rand([ - 2, 20, 2, 3 - ]).astype(dtype) + 1j * rand([2, 20, 2, 3]).astype(dtype) + input = rand([2, 20, 2, 3]).astype(dtype) + 1j * rand( + [2, 20, 2, 3] + ).astype(dtype) return {'x': input}, np.conj(input) for dtype in self._dtypes: input_dict, np_res = init_input_output(dtype) for place in self._places: with static.program_guard(static.Program()): - x_dtype = np.complex64 if dtype == "float32" else np.complex128 - x = static.data(name="x", - shape=[2, 20, 2, 3], - dtype=x_dtype) + x_dtype = ( + np.complex64 if dtype == "float32" else np.complex128 + ) + x = static.data( + name="x", shape=[2, 20, 2, 3], dtype=x_dtype + ) out = paddle.conj(x) exe = static.Executor(place) diff --git a/python/paddle/fluid/tests/unittests/test_const_value.py b/python/paddle/fluid/tests/unittests/test_const_value.py index e346f24ba6849..58ac6fa0a9a30 100644 --- a/python/paddle/fluid/tests/unittests/test_const_value.py +++ b/python/paddle/fluid/tests/unittests/test_const_value.py @@ -17,7 +17,6 @@ class ConstantTest(unittest.TestCase): - def test_const_value(self): self.assertEqual(framework.GRAD_VAR_SUFFIX, "@GRAD") self.assertEqual(framework.TEMP_VAR_NAME, "@TEMP@") diff --git a/python/paddle/fluid/tests/unittests/test_context_manager.py b/python/paddle/fluid/tests/unittests/test_context_manager.py index 93c1d7fa1a499..9ae5630b0aff9 100644 --- a/python/paddle/fluid/tests/unittests/test_context_manager.py +++ b/python/paddle/fluid/tests/unittests/test_context_manager.py @@ -19,7 +19,6 @@ class TestContextManagerRaiseException(unittest.TestCase): # When exception raised in 'with' context, we should safely exit the context def test_func1(self): - def foo(): with fluid.dygraph.guard(): print("raise error in context manager") diff --git a/python/paddle/fluid/tests/unittests/test_conv1d_layer.py b/python/paddle/fluid/tests/unittests/test_conv1d_layer.py index 0c1ffc6ddbeeb..bc0353218443d 100644 --- a/python/paddle/fluid/tests/unittests/test_conv1d_layer.py +++ b/python/paddle/fluid/tests/unittests/test_conv1d_layer.py @@ -21,22 +21,23 @@ class Conv1DTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - batch_size=4, - spartial_shape=(16, ), - num_channels=6, - num_filters=8, - filter_size=3, - padding=0, - padding_mode="zeros", - stride=1, - dilation=1, - groups=1, - no_bias=False, - dtype="float32", - data_format="NCL"): + def __init__( + self, + methodName='runTest', + batch_size=4, + spartial_shape=(16,), + num_channels=6, + num_filters=8, + filter_size=3, + padding=0, + padding_mode="zeros", + stride=1, + dilation=1, + groups=1, + no_bias=False, + dtype="float32", + data_format="NCL", + ): super(Conv1DTestCase, self).__init__(methodName) self.batch_size = batch_size self.num_channels = num_channels @@ -44,7 +45,7 @@ def __init__(self, self.spartial_shape = spartial_shape self.filter_size = filter_size self.data_format = data_format - self.channel_last = (self.data_format == "NLC") + self.channel_last = self.data_format == "NLC" self.padding = padding self.padding_mode = padding_mode @@ -55,23 +56,28 @@ def __init__(self, self.dtype = dtype def setUp(self): - input_shape = (self.batch_size, self.num_channels - ) + self.spartial_shape if not self.channel_last else ( - self.batch_size, ) + self.spartial_shape + ( - self.num_channels, ) + input_shape = ( + (self.batch_size, self.num_channels) + self.spartial_shape + if not self.channel_last + else (self.batch_size,) + self.spartial_shape + (self.num_channels,) + ) self.input = np.random.randn(*input_shape).astype(self.dtype) if isinstance(self.filter_size, int): filter_size = [self.filter_size] else: filter_size = self.filter_size - self.weight_shape = weight_shape = (self.num_filters, self.num_channels - // self.groups) + tuple(filter_size) - self.weight = np.random.uniform(-1, 1, - size=weight_shape).astype(self.dtype) + self.weight_shape = weight_shape = ( + self.num_filters, + self.num_channels // self.groups, + ) + tuple(filter_size) + self.weight = np.random.uniform(-1, 1, size=weight_shape).astype( + self.dtype + ) if not self.no_bias: self.bias = np.random.uniform( - -1, 1, size=(self.num_filters, )).astype(self.dtype) + -1, 1, size=(self.num_filters,) + ).astype(self.dtype) else: self.bias = None @@ -80,42 +86,49 @@ def functional(self, place): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, self.num_channels, - -1) if not self.channel_last else ( - -1, -1, self.num_channels) + input_shape = ( + (-1, self.num_channels, -1) + if not self.channel_last + else (-1, -1, self.num_channels) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) - w_var = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) - b_var = fluid.data("bias", (self.num_filters, ), - dtype=self.dtype) - y_var = F.conv1d(x_var, - w_var, - b_var if not self.no_bias else None, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + w_var = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) + b_var = fluid.data( + "bias", (self.num_filters,), dtype=self.dtype + ) + y_var = F.conv1d( + x_var, + w_var, + b_var if not self.no_bias else None, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) feed_dict = {"input": self.input, "weight": self.weight} if self.bias is not None: feed_dict["bias"] = self.bias exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_nn_layer(self): x_var = paddle.to_tensor(self.input) - conv = nn.Conv1D(self.num_channels, - self.num_filters, - self.filter_size, - padding=self.padding, - padding_mode=self.padding_mode, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + conv = nn.Conv1D( + self.num_channels, + self.num_filters, + self.filter_size, + padding=self.padding, + padding_mode=self.padding_mode, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) conv.weight.set_value(self.weight) if not self.no_bias: conv.bias.set_value(self.bias) @@ -139,7 +152,6 @@ def runTest(self): class Conv1DErrorTestCase(Conv1DTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -148,7 +160,6 @@ def runTest(self): class Conv1DTypeErrorTestCase(Conv1DTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -161,58 +172,75 @@ def add_cases(suite): suite.addTest(Conv1DTestCase(methodName='runTest', stride=[1], dilation=2)) suite.addTest(Conv1DTestCase(methodName='runTest', stride=2, dilation=(1))) suite.addTest( - Conv1DTestCase(methodName='runTest', padding="same", no_bias=True)) + Conv1DTestCase(methodName='runTest', padding="same", no_bias=True) + ) suite.addTest( - Conv1DTestCase(methodName='runTest', filter_size=3, padding='valid')) + Conv1DTestCase(methodName='runTest', filter_size=3, padding='valid') + ) suite.addTest( - Conv1DTestCase(methodName='runTest', num_filters=512, padding='valid')) + Conv1DTestCase(methodName='runTest', num_filters=512, padding='valid') + ) suite.addTest( - Conv1DTestCase(methodName='runTest', num_filters=512, padding=[1, 2])) + Conv1DTestCase(methodName='runTest', num_filters=512, padding=[1, 2]) + ) suite.addTest( - Conv1DTestCase(methodName='runTest', padding=2, data_format='NLC')) + Conv1DTestCase(methodName='runTest', padding=2, data_format='NLC') + ) suite.addTest(Conv1DTestCase(methodName='runTest', padding=[1])) suite.addTest(Conv1DTestCase(methodName='runTest', padding=[1, 2])) suite.addTest( - Conv1DTestCase(methodName='runTest', padding=[1, 2], data_format='NLC')) + Conv1DTestCase(methodName='runTest', padding=[1, 2], data_format='NLC') + ) suite.addTest(Conv1DTestCase(methodName='runTest', padding=2)) suite.addTest(Conv1DTestCase(methodName='runTest')) suite.addTest( - Conv1DTestCase(methodName='runTest', groups=2, padding="valid")) + Conv1DTestCase(methodName='runTest', groups=2, padding="valid") + ) suite.addTest( - Conv1DTestCase(methodName='runTest', - num_filters=6, - num_channels=3, - groups=3, - padding="valid", - data_format='NLC')) + Conv1DTestCase( + methodName='runTest', + num_filters=6, + num_channels=3, + groups=3, + padding="valid", + data_format='NLC', + ) + ) def add_error_cases(suite): suite.addTest( - Conv1DTypeErrorTestCase(methodName='runTest', - padding_mode="reflect", - padding="valid")) - suite.addTest(Conv1DErrorTestCase(methodName='runTest', - data_format="VALID")) + Conv1DTypeErrorTestCase( + methodName='runTest', padding_mode="reflect", padding="valid" + ) + ) + suite.addTest( + Conv1DErrorTestCase(methodName='runTest', data_format="VALID") + ) suite.addTest( - Conv1DErrorTestCase(methodName='runTest', padding_mode="VALID")) + Conv1DErrorTestCase(methodName='runTest', padding_mode="VALID") + ) suite.addTest( - Conv1DErrorTestCase(methodName='runTest', num_channels=5, groups=2)) + Conv1DErrorTestCase(methodName='runTest', num_channels=5, groups=2) + ) suite.addTest( - Conv1DErrorTestCase(methodName='runTest', - num_filters=8, - num_channels=15, - groups=3)) + Conv1DErrorTestCase( + methodName='runTest', num_filters=8, num_channels=15, groups=3 + ) + ) suite.addTest( - Conv1DErrorTestCase(methodName='runTest', padding=[1, 2, 3, 4, 5])) + Conv1DErrorTestCase(methodName='runTest', padding=[1, 2, 3, 4, 5]) + ) suite.addTest( - Conv1DErrorTestCase(methodName='runTest', - padding=[1, 2, 3, 4, 5], - data_format='NLC')) + Conv1DErrorTestCase( + methodName='runTest', padding=[1, 2, 3, 4, 5], data_format='NLC' + ) + ) suite.addTest( - Conv1DErrorTestCase(methodName='runTest', - num_filters=512, - padding=[1, 2, 3, 4, 5])) + Conv1DErrorTestCase( + methodName='runTest', num_filters=512, padding=[1, 2, 3, 4, 5] + ) + ) suite.addTest(Conv1DErrorTestCase(methodName='runTest', dilation=-10)) diff --git a/python/paddle/fluid/tests/unittests/test_conv1d_transpose_layer.py b/python/paddle/fluid/tests/unittests/test_conv1d_transpose_layer.py index fbd65ebbf7c90..ec8b7ee6ef78a 100644 --- a/python/paddle/fluid/tests/unittests/test_conv1d_transpose_layer.py +++ b/python/paddle/fluid/tests/unittests/test_conv1d_transpose_layer.py @@ -21,23 +21,24 @@ class Conv1DTransposeTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - batch_size=4, - spartial_shape=16, - in_channels=6, - out_channels=8, - filter_size=3, - output_size=None, - padding=0, - output_padding=0, - stride=1, - dilation=1, - groups=1, - no_bias=False, - data_format="NCL", - dtype="float32"): + def __init__( + self, + methodName='runTest', + batch_size=4, + spartial_shape=16, + in_channels=6, + out_channels=8, + filter_size=3, + output_size=None, + padding=0, + output_padding=0, + stride=1, + dilation=1, + groups=1, + no_bias=False, + data_format="NCL", + dtype="float32", + ): super(Conv1DTransposeTestCase, self).__init__(methodName) self.batch_size = batch_size self.in_channels = in_channels @@ -58,25 +59,32 @@ def __init__(self, def setUp(self): self.channel_last = False if self.data_format == "NCL" else True - input_shape = (self.batch_size, self.in_channels, - self.spartial_shape) if not self.channel_last else ( - self.batch_size, - self.spartial_shape, - self.in_channels, - ) + input_shape = ( + (self.batch_size, self.in_channels, self.spartial_shape) + if not self.channel_last + else ( + self.batch_size, + self.spartial_shape, + self.in_channels, + ) + ) self.input = np.random.randn(*input_shape).astype(self.dtype) if isinstance(self.filter_size, int): filter_size = [self.filter_size] else: filter_size = self.filter_size - self.weight_shape = weight_shape = (self.in_channels, self.out_channels - // self.groups) + tuple(filter_size) - self.weight = np.random.uniform(-1, 1, - size=weight_shape).astype(self.dtype) + self.weight_shape = weight_shape = ( + self.in_channels, + self.out_channels // self.groups, + ) + tuple(filter_size) + self.weight = np.random.uniform(-1, 1, size=weight_shape).astype( + self.dtype + ) if not self.no_bias: self.bias = np.random.uniform( - -1, 1, size=(self.out_channels, )).astype(self.dtype) + -1, 1, size=(self.out_channels,) + ).astype(self.dtype) else: self.bias = None @@ -85,44 +93,51 @@ def functional(self, place): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, self.in_channels, - -1) if not self.channel_last else ( - -1, -1, self.in_channels) + input_shape = ( + (-1, self.in_channels, -1) + if not self.channel_last + else (-1, -1, self.in_channels) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) - w_var = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) - b_var = fluid.data("bias", (self.out_channels, ), - dtype=self.dtype) - y_var = F.conv1d_transpose(x_var, - w_var, - None if self.no_bias else b_var, - output_size=self.output_size, - padding=self.padding, - output_padding=self.output_padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + w_var = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) + b_var = fluid.data( + "bias", (self.out_channels,), dtype=self.dtype + ) + y_var = F.conv1d_transpose( + x_var, + w_var, + None if self.no_bias else b_var, + output_size=self.output_size, + padding=self.padding, + output_padding=self.output_padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) feed_dict = {"input": self.input, "weight": self.weight} if self.bias is not None: feed_dict["bias"] = self.bias exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_nn_layer(self): x_var = paddle.to_tensor(self.input) - conv = nn.Conv1DTranspose(self.in_channels, - self.out_channels, - self.filter_size, - padding=self.padding, - output_padding=self.output_padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + conv = nn.Conv1DTranspose( + self.in_channels, + self.out_channels, + self.filter_size, + padding=self.padding, + output_padding=self.output_padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) conv.weight.set_value(self.weight) if not self.no_bias: conv.bias.set_value(self.bias) @@ -146,7 +161,6 @@ def runTest(self): class Conv1DTransposeErrorTestCase(Conv1DTransposeTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -157,64 +171,84 @@ def runTest(self): def add_cases(suite): suite.addTest(Conv1DTransposeTestCase(methodName='runTest')) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', - stride=[2], - no_bias=True, - dilation=2)) + Conv1DTransposeTestCase( + methodName='runTest', stride=[2], no_bias=True, dilation=2 + ) + ) + suite.addTest( + Conv1DTransposeTestCase( + methodName='runTest', + filter_size=(3), + output_size=[36], + stride=[2], + dilation=2, + ) + ) + suite.addTest( + Conv1DTransposeTestCase(methodName='runTest', stride=2, dilation=(2)) + ) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', - filter_size=(3), - output_size=[36], - stride=[2], - dilation=2)) + Conv1DTransposeTestCase(methodName='runTest', padding="valid") + ) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', stride=2, dilation=(2))) - suite.addTest(Conv1DTransposeTestCase(methodName='runTest', - padding="valid")) - suite.addTest(Conv1DTransposeTestCase(methodName='runTest', - padding='valid')) + Conv1DTransposeTestCase(methodName='runTest', padding='valid') + ) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', filter_size=1, padding=3)) + Conv1DTransposeTestCase(methodName='runTest', filter_size=1, padding=3) + ) suite.addTest(Conv1DTransposeTestCase(methodName='runTest', padding=[2])) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', data_format="NLC")) + Conv1DTransposeTestCase(methodName='runTest', data_format="NLC") + ) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', groups=2, - padding="valid")) + Conv1DTransposeTestCase(methodName='runTest', groups=2, padding="valid") + ) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', - out_channels=6, - in_channels=3, - groups=3, - padding="valid")) + Conv1DTransposeTestCase( + methodName='runTest', + out_channels=6, + in_channels=3, + groups=3, + padding="valid", + ) + ) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', - data_format="NLC", - spartial_shape=16, - output_size=18)) + Conv1DTransposeTestCase( + methodName='runTest', + data_format="NLC", + spartial_shape=16, + output_size=18, + ) + ) suite.addTest( - Conv1DTransposeTestCase(methodName='runTest', - data_format="NLC", - stride=3, - output_padding=2)) + Conv1DTransposeTestCase( + methodName='runTest', data_format="NLC", stride=3, output_padding=2 + ) + ) suite.addTest(Conv1DTransposeTestCase(methodName='runTest', padding=[1, 2])) def add_error_cases(suite): suite.addTest( - Conv1DTransposeErrorTestCase(methodName='runTest', - data_format="not_valid")) + Conv1DTransposeErrorTestCase( + methodName='runTest', data_format="not_valid" + ) + ) suite.addTest( - Conv1DTransposeErrorTestCase(methodName='runTest', - in_channels=5, - groups=2)) + Conv1DTransposeErrorTestCase( + methodName='runTest', in_channels=5, groups=2 + ) + ) suite.addTest( - Conv1DTransposeErrorTestCase(methodName='runTest', - stride=2, - output_padding=3)) + Conv1DTransposeErrorTestCase( + methodName='runTest', stride=2, output_padding=3 + ) + ) suite.addTest( - Conv1DTransposeErrorTestCase(methodName='runTest', - output_size="not_valid")) + Conv1DTransposeErrorTestCase( + methodName='runTest', output_size="not_valid" + ) + ) def load_tests(loader, standard_tests, pattern): diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_api.py b/python/paddle/fluid/tests/unittests/test_conv2d_api.py index d7b9339c39cfc..b95bf27cdc884 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_api.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_api.py @@ -23,251 +23,295 @@ class TestConv2DAPI(unittest.TestCase): - def test_api(self): - input_NHWC = fluid.layers.data(name="input_NHWC", - shape=[2, 5, 5, 3], - append_batch_size=False, - dtype="float32") - - input_NCHW = fluid.layers.data(name="input_NCHW", - shape=[2, 3, 5, 5], - append_batch_size=False, - dtype="float32") - - fluid.layers.conv2d(input=input_NHWC, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=0, - dilation=[1, 1], - groups=1, - data_format="NCHW") - - fluid.layers.conv2d(input=input_NCHW, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=[1, 2, 1, 0], - dilation=[1, 1], - groups=1, - data_format="NCHW") - - fluid.layers.conv2d(input=input_NCHW, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=[[0, 0], [0, 0], [1, 1], [1, 1]], - dilation=[1, 1], - groups=1, - data_format="NCHW") - - fluid.layers.conv2d(input=input_NHWC, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=[[0, 0], [1, 1], [1, 1], [0, 0]], - dilation=[1, 1], - groups=1, - data_format="NHWC") - - fluid.layers.conv2d(input=input_NCHW, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding="SAME", - dilation=[1, 1], - groups=1, - data_format="NCHW") - - fluid.layers.conv2d(input=input_NCHW, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding="VALID", - dilation=[1, 1], - groups=1, - data_format="NCHW") + input_NHWC = fluid.layers.data( + name="input_NHWC", + shape=[2, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + input_NCHW = fluid.layers.data( + name="input_NCHW", + shape=[2, 3, 5, 5], + append_batch_size=False, + dtype="float32", + ) + + fluid.layers.conv2d( + input=input_NHWC, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=0, + dilation=[1, 1], + groups=1, + data_format="NCHW", + ) + + fluid.layers.conv2d( + input=input_NCHW, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=[1, 2, 1, 0], + dilation=[1, 1], + groups=1, + data_format="NCHW", + ) + + fluid.layers.conv2d( + input=input_NCHW, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=[[0, 0], [0, 0], [1, 1], [1, 1]], + dilation=[1, 1], + groups=1, + data_format="NCHW", + ) + + fluid.layers.conv2d( + input=input_NHWC, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=[[0, 0], [1, 1], [1, 1], [0, 0]], + dilation=[1, 1], + groups=1, + data_format="NHWC", + ) + + fluid.layers.conv2d( + input=input_NCHW, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding="SAME", + dilation=[1, 1], + groups=1, + data_format="NCHW", + ) + + fluid.layers.conv2d( + input=input_NCHW, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding="VALID", + dilation=[1, 1], + groups=1, + data_format="NCHW", + ) def test_depthwise_conv2d(self): - x_var = paddle.uniform((2, 8, 8, 4), dtype='float32', min=-1., max=1.) - conv = paddle.nn.Conv2D(in_channels=4, - out_channels=4, - kernel_size=(3, 3), - groups=4, - data_format='NHWC') + x_var = paddle.uniform((2, 8, 8, 4), dtype='float32', min=-1.0, max=1.0) + conv = paddle.nn.Conv2D( + in_channels=4, + out_channels=4, + kernel_size=(3, 3), + groups=4, + data_format='NHWC', + ) y_var = conv(x_var) class TestConv2DAPI_Error(unittest.TestCase): - def test_api(self): - input = fluid.layers.data(name="input", - shape=[2, 5, 5, 5], - append_batch_size=False, - dtype="float32") + input = fluid.layers.data( + name="input", + shape=[2, 5, 5, 5], + append_batch_size=False, + dtype="float32", + ) # ValueError: cudnn def run_1(): - fluid.layers.conv2d(input=input, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=0, - dilation=[1, 1], - groups=1, - use_cudnn=[0], - data_format="NCHW") + fluid.layers.conv2d( + input=input, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=0, + dilation=[1, 1], + groups=1, + use_cudnn=[0], + data_format="NCHW", + ) self.assertRaises(ValueError, run_1) # ValueError: data_format def run_2(): - fluid.layers.conv2d(input=input, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=0, - dilation=[1, 1], - groups=1, - use_cudnn=False, - data_format="NCHWC") + fluid.layers.conv2d( + input=input, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=0, + dilation=[1, 1], + groups=1, + use_cudnn=False, + data_format="NCHWC", + ) self.assertRaises(ValueError, run_2) # ValueError: padding def run_3(): - fluid.layers.conv2d(input=input, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding="SAMEE", - dilation=[1, 1], - groups=1, - use_cudnn=False, - data_format="NCHW") + fluid.layers.conv2d( + input=input, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding="SAMEE", + dilation=[1, 1], + groups=1, + use_cudnn=False, + data_format="NCHW", + ) self.assertRaises(ValueError, run_3) def run_4(): - fluid.layers.conv2d(input=input, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=[[0, 1], [0, 1], [0, 1], [0, 1]], - dilation=[1, 1], - groups=1, - use_cudnn=False, - data_format="NCHW") + fluid.layers.conv2d( + input=input, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=[[0, 1], [0, 1], [0, 1], [0, 1]], + dilation=[1, 1], + groups=1, + use_cudnn=False, + data_format="NCHW", + ) self.assertRaises(ValueError, run_4) def run_5(): - fluid.layers.conv2d(input=input, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=[[0, 1], [0, 1], [0, 1], [0, 1]], - dilation=[1, 1], - groups=1, - use_cudnn=False, - data_format="NHWC") + fluid.layers.conv2d( + input=input, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=[[0, 1], [0, 1], [0, 1], [0, 1]], + dilation=[1, 1], + groups=1, + use_cudnn=False, + data_format="NHWC", + ) self.assertRaises(ValueError, run_5) # ValueError: channel dimmention - x = fluid.layers.data(name="x", - shape=[2, 5, 5, -1], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name="x", + shape=[2, 5, 5, -1], + append_batch_size=False, + dtype="float32", + ) def run_6(): - fluid.layers.conv2d(input=x, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=0, - dilation=[1, 1], - groups=1, - use_cudnn=False, - data_format="NHWC") + fluid.layers.conv2d( + input=x, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=0, + dilation=[1, 1], + groups=1, + use_cudnn=False, + data_format="NHWC", + ) self.assertRaises(ValueError, run_6) # ValueError: groups def run_7(): - fluid.layers.conv2d(input=input, - num_filters=3, - filter_size=[3, 3], - stride=[1, 1], - padding=0, - dilation=[1, 1], - groups=3, - use_cudnn=False, - data_format="NHWC") + fluid.layers.conv2d( + input=input, + num_filters=3, + filter_size=[3, 3], + stride=[1, 1], + padding=0, + dilation=[1, 1], + groups=3, + use_cudnn=False, + data_format="NHWC", + ) self.assertRaises(ValueError, run_7) # ValueError: filter num def run_8(): - fluid.layers.conv2d(input=input, - num_filters=0, - filter_size=0, - stride=0, - padding=0, - dilation=0, - groups=1, - use_cudnn=False, - data_format="NCHW") + fluid.layers.conv2d( + input=input, + num_filters=0, + filter_size=0, + stride=0, + padding=0, + dilation=0, + groups=1, + use_cudnn=False, + data_format="NCHW", + ) self.assertRaises(ValueError, run_8) # ValueError: groups def run_9(): - fluid.layers.conv2d(input=input, - num_filters=0, - filter_size=0, - stride=0, - padding=0, - dilation=0, - groups=0, - use_cudnn=False, - data_format="NCHW") + fluid.layers.conv2d( + input=input, + num_filters=0, + filter_size=0, + stride=0, + padding=0, + dilation=0, + groups=0, + use_cudnn=False, + data_format="NCHW", + ) self.assertRaises(ValueError, run_9) # ValueError: stride def run_10(): - fluid.layers.conv2d(input=input, - num_filters=1, - filter_size=1, - stride=0, - padding=0, - dilation=0, - groups=1, - use_cudnn=False, - data_format="NCHW") + fluid.layers.conv2d( + input=input, + num_filters=1, + filter_size=1, + stride=0, + padding=0, + dilation=0, + groups=1, + use_cudnn=False, + data_format="NCHW", + ) self.assertRaises(ValueError, run_10) def test_api_with_error_input(self): - input = fluid.layers.data(name="error_input", - shape=[1], - append_batch_size=False, - dtype="float32") + input = fluid.layers.data( + name="error_input", + shape=[1], + append_batch_size=False, + dtype="float32", + ) # ValueError: cudnn def run_1(): - fluid.layers.conv2d(input=input, - num_filters=0, - filter_size=0, - stride=0, - padding=0, - dilation=0, - groups=0, - use_cudnn=False, - data_format="NCHW") + fluid.layers.conv2d( + input=input, + num_filters=0, + filter_size=0, + stride=0, + padding=0, + dilation=0, + groups=0, + use_cudnn=False, + data_format="NCHW", + ) self.assertRaises(ValueError, run_1) @@ -275,36 +319,44 @@ def run_1(): # --------- test environment variable ------ @unittest.skipIf( not (core.is_compiled_with_cuda() or core.is_compiled_with_rocm()), - "core is not compiled with CUDA or ROCM") + "core is not compiled with CUDA or ROCM", +) class TestConv2DEnviron(unittest.TestCase): - def run1(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - inputs = fluid.layers.data(shape=[2, 3, 5, 5], - append_batch_size=False, - name="inputs", - dtype="float32") - result = fluid.layers.conv2d(input=inputs, - num_filters=4, - filter_size=[3, 3], - stride=[1, 1], - padding=0, - dilation=[1, 1], - groups=1, - data_format="NCHW") + inputs = fluid.layers.data( + shape=[2, 3, 5, 5], + append_batch_size=False, + name="inputs", + dtype="float32", + ) + result = fluid.layers.conv2d( + input=inputs, + num_filters=4, + filter_size=[3, 3], + stride=[1, 1], + padding=0, + dilation=[1, 1], + groups=1, + data_format="NCHW", + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - fetches = exe.run(fluid.default_main_program(), - feed={"inputs": self.input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"inputs": self.input_np}, + fetch_list=[result], + ) def run2(self, place): with fluid.dygraph.guard(place): inputs = fluid.dygraph.to_variable(self.input_np) - conv = paddle.nn.Conv2D(in_channels=3, - out_channels=4, - kernel_size=(3, 3), - data_format="NCHW") + conv = paddle.nn.Conv2D( + in_channels=3, + out_channels=4, + kernel_size=(3, 3), + data_format="NCHW", + ) result = conv(inputs) def run3(self, place): diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_fusion_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_fusion_op.py index f92c0c7090ee6..7e4a01331578b 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_fusion_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_fusion_op.py @@ -22,9 +22,7 @@ def create_test_padding_SAME_class(parent): - class TestPaddingSAMECase(parent): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" @@ -35,9 +33,7 @@ def init_paddings(self): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" @@ -48,7 +44,6 @@ def init_paddings(self): class TestConv2DFusionOp(OpTest): - def setUp(self): self.op_type = "conv2d_fusion" self.exhaustive_search = False @@ -71,32 +66,37 @@ def setUp(self): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.random(self.filter_size).astype(self.dtype) bias = np.random.random(self.filter_size[0]).astype(self.dtype) - self.output, _, _, _, _ = conv2d_forward_naive(input, filter, - self.groups, - conv2d_param, - self.padding_algorithm, - self.data_format) + self.output, _, _, _, _ = conv2d_forward_naive( + input, + filter, + self.groups, + conv2d_param, + self.padding_algorithm, + self.data_format, + ) self.output = self.output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), - 'Bias': OpTest.np_dtype_to_fluid_dtype(bias) + 'Bias': OpTest.np_dtype_to_fluid_dtype(bias), } if self.add_residual_data: residual_data = np.random.random(self.output.shape).astype( - self.dtype) + self.dtype + ) self.inputs['ResidualData'] = OpTest.np_dtype_to_fluid_dtype( - residual_data) + residual_data + ) self.output += residual_data # Add bias @@ -114,7 +114,7 @@ def setUp(self): 'data_format': self.data_format, 'exhaustive_search': self.exhaustive_search, 'activation': self.activation, - 'padding_algorithm': self.padding_algorithm + 'padding_algorithm': self.padding_algorithm, } if self.split_channels is not None: self.attrs['split_channels'] = self.split_channels @@ -163,32 +163,27 @@ def init_paddings(self): class TestWithoutResidual(TestConv2DFusionOp): - def init_residual(self): self.add_residual_data = False class TestIdentityActivation(TestConv2DFusionOp): - def init_activation(self): self.activation = 'identity' class TestIdentityActivation1(TestConv2DFusionOp): - def init_activation(self): self.activation = 'identity' self.add_residual_data = False class TestWithGroup(TestConv2DFusionOp): - def init_group(self): self.groups = 3 class TestWithDilation(TestConv2DFusionOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -205,13 +200,11 @@ def init_group(self): class TestCUDNNExhaustiveSearch(TestConv2DFusionOp): - def set_search_method(self): self.exhaustive_search = True class TestMultipleOutputs(TestConv2DFusionOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -228,14 +221,12 @@ def set_outputs(self): class TestAsyPadding(TestConv2DFusionOp): - def init_paddings(self): self.pad = [0, 0, 1, 2] self.padding_algorithm = "EXPLICIT" class TestWithPad_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 10, 10] # NCHW @@ -249,7 +240,6 @@ def init_paddings(self): class TestWithStride_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 3, 6, 6] # NCHW @@ -263,7 +253,6 @@ def init_paddings(self): class TestWith1x1_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -280,13 +269,11 @@ def init_paddings(self): class TestWithGroup_AsyPadding(TestConv2DFusionOp): - def init_group(self): self.groups = 3 class TestWithDepthWise3x3_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [1, 1] self.input_size = [3, 4, 10, 10] # NCHW @@ -306,7 +293,6 @@ def init_paddings(self): class TestWithDepthWise5x5_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 4, 10, 10] # NCHW @@ -323,7 +309,6 @@ def init_paddings(self): class TestWithDepthWise7x7_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 8, 10, 10] # NCHW @@ -340,7 +325,6 @@ def init_paddings(self): class TestWithDilation_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 10, 10] # NCHW @@ -360,7 +344,6 @@ def init_paddings(self): class TestWithInput1x1Filter1x1_AsyPadding(TestConv2DFusionOp): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 1, 1] # NCHW diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_layer.py b/python/paddle/fluid/tests/unittests/test_conv2d_layer.py index 8782ee84a26a0..22e6f0a56342c 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_layer.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_layer.py @@ -27,22 +27,23 @@ def _reverse_repeat_list(t, n): class Conv2DTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - batch_size=4, - spartial_shape=(16, 16), - num_channels=6, - num_filters=8, - filter_size=3, - padding=0, - padding_mode='zeros', - stride=1, - dilation=1, - groups=1, - no_bias=False, - data_format="NCHW", - dtype="float32"): + def __init__( + self, + methodName='runTest', + batch_size=4, + spartial_shape=(16, 16), + num_channels=6, + num_filters=8, + filter_size=3, + padding=0, + padding_mode='zeros', + stride=1, + dilation=1, + groups=1, + no_bias=False, + data_format="NCHW", + dtype="float32", + ): super(Conv2DTestCase, self).__init__(methodName) self.batch_size = batch_size self.num_channels = num_channels @@ -53,9 +54,11 @@ def __init__(self, self.padding = padding if padding_mode in {'reflect', 'replicate', 'circular'}: _paired_padding = fluid.layers.utils.convert_to_list( - padding, 2, 'padding') + padding, 2, 'padding' + ) self._reversed_padding_repeated_twice = _reverse_repeat_list( - _paired_padding, 2) + _paired_padding, 2 + ) self.padding_mode = padding_mode self.stride = stride self.dilation = dilation @@ -67,24 +70,31 @@ def __init__(self, def setUp(self): self.channel_last = self.data_format == "NHWC" if self.channel_last: - input_shape = (self.batch_size, ) + self.spartial_shape + ( - self.num_channels, ) + input_shape = ( + (self.batch_size,) + self.spartial_shape + (self.num_channels,) + ) else: - input_shape = (self.batch_size, - self.num_channels) + self.spartial_shape + input_shape = ( + self.batch_size, + self.num_channels, + ) + self.spartial_shape self.input = np.random.randn(*input_shape).astype(self.dtype) if isinstance(self.filter_size, int): filter_size = [self.filter_size] * 2 else: filter_size = self.filter_size - self.weight_shape = weight_shape = (self.num_filters, self.num_channels - // self.groups) + tuple(filter_size) - self.weight = np.random.uniform(-1, 1, - size=weight_shape).astype(self.dtype) + self.weight_shape = weight_shape = ( + self.num_filters, + self.num_channels // self.groups, + ) + tuple(filter_size) + self.weight = np.random.uniform(-1, 1, size=weight_shape).astype( + self.dtype + ) if not self.no_bias: self.bias = np.random.uniform( - -1, 1, size=(self.num_filters, )).astype(self.dtype) + -1, 1, size=(self.num_filters,) + ).astype(self.dtype) else: self.bias = None @@ -93,8 +103,11 @@ def fluid_layer(self, place): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1,self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1) + input_shape = ( + (-1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) weight_attr = I.NumpyArrayInitializer(self.weight) if self.bias is None: @@ -102,29 +115,33 @@ def fluid_layer(self, place): else: bias_attr = I.NumpyArrayInitializer(self.bias) if self.padding_mode != 'zeros': - x_var = F.pad(x_var, - self._reversed_padding_repeated_twice, - mode=self.padding_mode, - data_format=self.data_format) + x_var = F.pad( + x_var, + self._reversed_padding_repeated_twice, + mode=self.padding_mode, + data_format=self.data_format, + ) padding = 0 else: padding = self.padding - y_var = fluid.layers.conv2d(x_var, - self.num_filters, - self.filter_size, - padding=padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - param_attr=weight_attr, - bias_attr=bias_attr, - data_format=self.data_format) + y_var = fluid.layers.conv2d( + x_var, + self.num_filters, + self.filter_size, + padding=padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + param_attr=weight_attr, + bias_attr=bias_attr, + data_format=self.data_format, + ) feed_dict = {"input": self.input} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def functional(self, place): @@ -132,52 +149,62 @@ def functional(self, place): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1,self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1) + input_shape = ( + (-1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) - w_var = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) - b_var = fluid.data("bias", (self.num_filters, ), - dtype=self.dtype) + w_var = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) + b_var = fluid.data( + "bias", (self.num_filters,), dtype=self.dtype + ) if self.padding_mode != 'zeros': - x_var = F.pad(x_var, - self._reversed_padding_repeated_twice, - mode=self.padding_mode, - data_format=self.data_format) + x_var = F.pad( + x_var, + self._reversed_padding_repeated_twice, + mode=self.padding_mode, + data_format=self.data_format, + ) padding = 0 else: padding = self.padding - y_var = F.conv2d(x_var, - w_var, - b_var if not self.no_bias else None, - padding=padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y_var = F.conv2d( + x_var, + w_var, + b_var if not self.no_bias else None, + padding=padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) feed_dict = {"input": self.input, "weight": self.weight} if self.bias is not None: feed_dict["bias"] = self.bias exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_nn_layer(self): x_var = paddle.to_tensor(self.input) x_var.stop_gradient = False - conv = nn.Conv2D(self.num_channels, - self.num_filters, - self.filter_size, - padding=self.padding, - padding_mode=self.padding_mode, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + conv = nn.Conv2D( + self.num_channels, + self.num_filters, + self.filter_size, + padding=self.padding, + padding_mode=self.padding_mode, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) conv.weight.set_value(self.weight) if not self.no_bias: conv.bias.set_value(self.bias) @@ -210,7 +237,6 @@ def runTest(self): class Conv2DErrorTestCase(Conv2DTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -221,63 +247,86 @@ def runTest(self): def add_cases(suite): suite.addTest(Conv2DTestCase(methodName='runTest')) suite.addTest( - Conv2DTestCase(methodName='runTest', stride=[1, 2], dilation=2)) + Conv2DTestCase(methodName='runTest', stride=[1, 2], dilation=2) + ) suite.addTest( - Conv2DTestCase(methodName='runTest', stride=2, dilation=(2, 1))) + Conv2DTestCase(methodName='runTest', stride=2, dilation=(2, 1)) + ) suite.addTest( - Conv2DTestCase(methodName='runTest', padding="same", no_bias=True)) + Conv2DTestCase(methodName='runTest', padding="same", no_bias=True) + ) suite.addTest( - Conv2DTestCase(methodName='runTest', - filter_size=(3, 3), - padding='valid')) + Conv2DTestCase( + methodName='runTest', filter_size=(3, 3), padding='valid' + ) + ) suite.addTest(Conv2DTestCase(methodName='runTest', padding=(2, 3))) suite.addTest(Conv2DTestCase(methodName='runTest', padding=[1, 2, 2, 1])) suite.addTest( - Conv2DTestCase(methodName='runTest', - padding=[[0, 0], [0, 0], [1, 2], [2, 1]])) + Conv2DTestCase( + methodName='runTest', padding=[[0, 0], [0, 0], [1, 2], [2, 1]] + ) + ) suite.addTest(Conv2DTestCase(methodName='runTest', data_format="NHWC")) suite.addTest( - Conv2DTestCase(methodName='runTest', - data_format="NHWC", - padding=[[0, 0], [1, 1], [2, 2], [0, 0]])) + Conv2DTestCase( + methodName='runTest', + data_format="NHWC", + padding=[[0, 0], [1, 1], [2, 2], [0, 0]], + ) + ) suite.addTest( - Conv2DTestCase(methodName='runTest', groups=2, padding="valid")) + Conv2DTestCase(methodName='runTest', groups=2, padding="valid") + ) suite.addTest( - Conv2DTestCase(methodName='runTest', - num_filters=6, - num_channels=3, - groups=3, - padding="valid")) + Conv2DTestCase( + methodName='runTest', + num_filters=6, + num_channels=3, + groups=3, + padding="valid", + ) + ) suite.addTest( - Conv2DTestCase(methodName='runTest', - filter_size=(3, 3), - padding=1, - padding_mode='reflect')) + Conv2DTestCase( + methodName='runTest', + filter_size=(3, 3), + padding=1, + padding_mode='reflect', + ) + ) suite.addTest( - Conv2DTestCase(methodName='runTest', - filter_size=(3, 3), - padding=1, - padding_mode='replicate')) + Conv2DTestCase( + methodName='runTest', + filter_size=(3, 3), + padding=1, + padding_mode='replicate', + ) + ) suite.addTest( - Conv2DTestCase(methodName='runTest', - filter_size=(3, 3), - padding=1, - padding_mode='circular')) + Conv2DTestCase( + methodName='runTest', + filter_size=(3, 3), + padding=1, + padding_mode='circular', + ) + ) def add_error_cases(suite): suite.addTest( - Conv2DErrorTestCase(methodName='runTest', num_channels=5, groups=2)) + Conv2DErrorTestCase(methodName='runTest', num_channels=5, groups=2) + ) suite.addTest( - Conv2DErrorTestCase(methodName='runTest', - num_channels=5, - groups=2, - stride=0)) + Conv2DErrorTestCase( + methodName='runTest', num_channels=5, groups=2, stride=0 + ) + ) suite.addTest( - Conv2DErrorTestCase(methodName='runTest', - num_channels=5, - groups=2, - padding=[-1, -1])) + Conv2DErrorTestCase( + methodName='runTest', num_channels=5, groups=2, padding=[-1, -1] + ) + ) def load_tests(loader, standard_tests, pattern): diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_op.py index 056a7c52efb9f..34a34f062efb8 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_op.py @@ -19,29 +19,36 @@ import paddle import paddle.fluid.core as core import paddle.fluid as fluid -from paddle.fluid.tests.unittests.op_test import (OpTest, - convert_float_to_uint16, - get_numeric_gradient) +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + convert_float_to_uint16, + get_numeric_gradient, +) from paddle.fluid.tests.unittests.testsuite import create_op from paddle.fluid import Program, program_guard -def conv2d_forward_naive(input, - filter, - group, - conv_param, - padding_algorithm='EXPLICIT', - data_format='NCHW'): +def conv2d_forward_naive( + input, + filter, + group, + conv_param, + padding_algorithm='EXPLICIT', + data_format='NCHW', +): if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if data_format not in ["NCHW", "NHWC"]: - raise ValueError("Unknown Attr(data_format): '%s' ." - "It can only be 'NCHW' or 'NHWC'." % str(data_format)) + raise ValueError( + "Unknown Attr(data_format): '%s' ." + "It can only be 'NCHW' or 'NHWC'." % str(data_format) + ) - channel_last = (data_format == "NHWC") + channel_last = data_format == "NHWC" if channel_last: input = np.transpose(input, [0, 3, 1, 2]) @@ -54,17 +61,22 @@ def conv2d_forward_naive(input, sub_out_c = out_c // group sub_f_n = f_n // group - stride, pad, dilation = conv_param['stride'], conv_param['pad'], conv_param[ - 'dilation'] + stride, pad, dilation = ( + conv_param['stride'], + conv_param['pad'], + conv_param['dilation'], + ) # update pad and dilation def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -84,39 +96,52 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): if len(pad) == 4: pad_h_0, pad_h_1 = pad[0], pad[1] pad_w_0, pad_w_1 = pad[2], pad[3] - out_h = 1 + (in_h + pad_h_0 + pad_h_1 - (dilation[0] * - (f_h - 1) + 1)) // stride[0] - out_w = 1 + (in_w + pad_w_0 + pad_w_1 - (dilation[1] * - (f_w - 1) + 1)) // stride[1] + out_h = ( + 1 + + (in_h + pad_h_0 + pad_h_1 - (dilation[0] * (f_h - 1) + 1)) + // stride[0] + ) + out_w = ( + 1 + + (in_w + pad_w_0 + pad_w_1 - (dilation[1] * (f_w - 1) + 1)) + // stride[1] + ) out = np.zeros((out_n, out_c, out_h, out_w)) - d_bolck_h = (dilation[0] * (f_h - 1) + 1) - d_bolck_w = (dilation[1] * (f_w - 1) + 1) + d_bolck_h = dilation[0] * (f_h - 1) + 1 + d_bolck_w = dilation[1] * (f_w - 1) + 1 - input_pad = np.pad(input, - ((0, 0), (0, 0), (pad_h_0, pad_h_1), (pad_w_0, pad_w_1)), - mode='constant', - constant_values=0) + input_pad = np.pad( + input, + ((0, 0), (0, 0), (pad_h_0, pad_h_1), (pad_w_0, pad_w_1)), + mode='constant', + constant_values=0, + ) filter_dilation = np.zeros((f_n, f_c, d_bolck_h, d_bolck_w)) - filter_dilation[:, :, 0:d_bolck_h:dilation[0], - 0:d_bolck_w:dilation[1]] = filter + filter_dilation[ + :, :, 0 : d_bolck_h : dilation[0], 0 : d_bolck_w : dilation[1] + ] = filter for i in range(out_h): for j in range(out_w): for g in range(group): - input_pad_masked = \ - input_pad[:, g * f_c:(g + 1) * f_c, - i * stride[0]:i * stride[0] + d_bolck_h, - j * stride[1]:j * stride[1] + d_bolck_w] - - f_sub = filter_dilation[g * sub_f_n:(g + 1) * sub_f_n, :, :, :] + input_pad_masked = input_pad[ + :, + g * f_c : (g + 1) * f_c, + i * stride[0] : i * stride[0] + d_bolck_h, + j * stride[1] : j * stride[1] + d_bolck_w, + ] + + f_sub = filter_dilation[ + g * sub_f_n : (g + 1) * sub_f_n, :, :, : + ] # sub_f_n == sub_out_c for k in range(sub_out_c): # Multiplication of Corresponding Elements, then sum all - out[:, g * sub_out_c + k, i, j] = \ - np.sum(input_pad_masked * f_sub[k, :, :, :], - axis=(1, 2, 3)) + out[:, g * sub_out_c + k, i, j] = np.sum( + input_pad_masked * f_sub[k, :, :, :], axis=(1, 2, 3) + ) if channel_last: out = np.transpose(out, [0, 2, 3, 1]) @@ -125,15 +150,15 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): def create_test_cudnn_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNCase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) cls_name = "{0}_{1}".format(parent.__name__, "CUDNN") TestCUDNNCase.__name__ = cls_name @@ -141,11 +166,10 @@ def init_kernel_type(self): def create_test_cudnn_fp16_class(parent, grad_check=True): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestConv2DCUDNNFp16(parent): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -159,16 +183,16 @@ def test_check_output(self): def test_check_grad_no_filter(self): place = core.CUDAPlace(0) if core.is_float16_supported(place) and grad_check: - self.check_grad_with_place(place, ['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + place, ['Input'], 'Output', no_grad_set=set(['Filter']) + ) def test_check_grad_no_input(self): place = core.CUDAPlace(0) if core.is_float16_supported(place) and grad_check: - self.check_grad_with_place(place, ['Filter'], - 'Output', - no_grad_set=set(['Input'])) + self.check_grad_with_place( + place, ['Filter'], 'Output', no_grad_set=set(['Input']) + ) cls_name = "{0}_{1}".format(parent.__name__, "CUDNNFp16") TestConv2DCUDNNFp16.__name__ = cls_name @@ -176,20 +200,21 @@ def test_check_grad_no_input(self): def create_test_cudnn_bf16_class(parent): - @unittest.skipIf( not core.is_compiled_with_cuda() or not core.is_bfloat16_supported(core.CUDAPlace(0)), - "core is not compiled with CUDA and do not support bfloat16") + "core is not compiled with CUDA and do not support bfloat16", + ) class TestConv2DCUDNNBF16(parent): - def get_numeric_grad(self, place, check_name): scope = core.Scope() self._check_grad_helper() - op = create_op(scope, self.op_type, self.inputs, self.outputs, - self.attrs) - return get_numeric_gradient(place, scope, op, self.inputs_fp32, - check_name, ['Output']) + op = create_op( + scope, self.op_type, self.inputs, self.outputs, self.attrs + ) + return get_numeric_gradient( + place, scope, op, self.inputs_fp32, check_name, ['Output'] + ) def init_kernel_type(self): self.use_cudnn = True @@ -203,18 +228,24 @@ def test_check_output(self): def test_check_grad_no_filter(self): place = core.CUDAPlace(0) numeric_grads = self.get_numeric_grad(place, 'Input') - self.check_grad_with_place(place, ['Input'], - 'Output', - no_grad_set=set(['Filter']), - user_defined_grads=[numeric_grads]) + self.check_grad_with_place( + place, + ['Input'], + 'Output', + no_grad_set=set(['Filter']), + user_defined_grads=[numeric_grads], + ) def test_check_grad_no_input(self): place = core.CUDAPlace(0) numeric_grads = self.get_numeric_grad(place, 'Filter') - self.check_grad_with_place(place, ['Filter'], - 'Output', - no_grad_set=set(['Input']), - user_defined_grads=[numeric_grads]) + self.check_grad_with_place( + place, + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + user_defined_grads=[numeric_grads], + ) cls_name = "{0}_{1}".format(parent.__name__, "CUDNNBF16") TestConv2DCUDNNBF16.__name__ = cls_name @@ -222,9 +253,7 @@ def test_check_grad_no_input(self): def create_test_channel_last_class(parent): - class TestChannelLastCase(parent): - def init_data_format(self): self.data_format = "NHWC" @@ -238,15 +267,15 @@ def init_test_case_2(self): def create_test_cudnn_channel_last_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCudnnChannelLastCase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) def init_data_format(self): self.data_format = "NHWC" @@ -261,11 +290,10 @@ def init_test_case_2(self): def create_test_cudnn_channel_last_fp16_class(parent, grad_check=True): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCudnnChannelLastFp16(parent): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -279,16 +307,16 @@ def test_check_output(self): def test_check_grad_no_filter(self): place = core.CUDAPlace(0) if core.is_float16_supported(place) and grad_check: - self.check_grad_with_place(place, ['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + place, ['Input'], 'Output', no_grad_set=set(['Filter']) + ) def test_check_grad_no_input(self): place = core.CUDAPlace(0) if core.is_float16_supported(place) and grad_check: - self.check_grad_with_place(place, ['Filter'], - 'Output', - no_grad_set=set(['Input'])) + self.check_grad_with_place( + place, ['Filter'], 'Output', no_grad_set=set(['Input']) + ) def init_data_format(self): self.data_format = "NHWC" @@ -303,9 +331,7 @@ def init_test_case_2(self): def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" @@ -316,9 +342,7 @@ def init_paddings(self): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" @@ -329,15 +353,15 @@ def init_paddings(self): def create_test_cudnn_padding_SAME_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingSMAECase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) def init_paddings(self): self.pad = [1, 1] @@ -349,15 +373,15 @@ def init_paddings(self): def create_test_cudnn_padding_VALID_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingVALIDCase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) def init_paddings(self): self.pad = [1, 1] @@ -369,7 +393,6 @@ def init_paddings(self): class TestConv2DOp(OpTest): - def setUp(self): self.op_type = "conv2d" self.use_cudnn = False @@ -387,17 +410,19 @@ def setUp(self): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } if self.is_bfloat16_op(): input = np.random.random(self.input_size).astype(np.float32) - filter = np.random.uniform(-1, 1, - self.filter_size).astype(np.float32) + filter = np.random.uniform(-1, 1, self.filter_size).astype( + np.float32 + ) else: input = np.random.random(self.input_size).astype(self.dtype) - filter = np.random.uniform(-1, 1, - self.filter_size).astype(self.dtype) + filter = np.random.uniform(-1, 1, self.filter_size).astype( + self.dtype + ) if not self.has_cuda(): self.fuse_relu_before_depthwise_conv = False @@ -409,24 +434,25 @@ def setUp(self): else: input2 = input - output, _, _, _, _ = conv2d_forward_naive(input2, filter, self.groups, - conv2d_param) + output, _, _, _, _ = conv2d_forward_naive( + input2, filter, self.groups, conv2d_param + ) if self.is_bfloat16_op(): output = output.astype(np.float32) self.inputs = { 'Input': convert_float_to_uint16(input), - 'Filter': convert_float_to_uint16(filter) + 'Filter': convert_float_to_uint16(filter), } self.inputs_fp32 = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } else: output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { @@ -437,56 +463,71 @@ def setUp(self): 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, 'data_format': self.data_format, - 'fuse_relu_before_depthwise_conv': - self.fuse_relu_before_depthwise_conv, - 'exhaustive_search': self.exhaustive_search + 'fuse_relu_before_depthwise_conv': self.fuse_relu_before_depthwise_conv, + 'exhaustive_search': self.exhaustive_search, } self.outputs = {'Output': output} def has_cuda(self): - return core.is_compiled_with_cuda() and (self.use_cudnn - or self.use_cuda) + return core.is_compiled_with_cuda() and ( + self.use_cudnn or self.use_cuda + ) def test_check_output(self): place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output_with_place(place, - atol=1e-5, - check_dygraph=(self.use_mkldnn == False)) + self.check_output_with_place( + place, atol=1e-5, check_dygraph=(self.use_mkldnn == False) + ) def test_check_grad(self): - if self.dtype == np.float16 or (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if self.dtype == np.float16 or ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad_with_place(place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.02, - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.02, + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_no_filter(self): - if self.dtype == np.float16 or (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if self.dtype == np.float16 or ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad_with_place(place, ['Input'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Filter']), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + ['Input'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Filter']), + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_no_input(self): - if self.dtype == np.float16 or (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if self.dtype == np.float16 or ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad_with_place(place, ['Filter'], - 'Output', - no_grad_set=set(['Input']), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + check_dygraph=(self.use_mkldnn == False), + ) def init_test_case(self): self.pad = [0, 0] @@ -510,7 +551,6 @@ def init_kernel_type(self): class TestWithPad(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -521,7 +561,6 @@ def init_test_case(self): class TestWithStride(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -532,7 +571,6 @@ def init_test_case(self): class TestWithGroup(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -544,7 +582,6 @@ def init_test_case(self): class TestWith1x1(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -558,7 +595,6 @@ def init_group(self): class TestWithDepthWise3x3(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -575,7 +611,6 @@ def init_group(self): class TestWithDepthWise5x5(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -589,7 +624,6 @@ def init_group(self): class TestWithDepthWise7x7(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -603,7 +637,6 @@ def init_group(self): class TestWithDilation(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -620,7 +653,6 @@ def init_group(self): class TestWithInput1x1Filter1x1(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -642,7 +674,7 @@ def init_group(self): create_test_cudnn_class(TestWith1x1) create_test_cudnn_class(TestWithInput1x1Filter1x1) -#----------------Conv2DCUDNN fp16---------------- +# ----------------Conv2DCUDNN fp16---------------- create_test_cudnn_fp16_class(TestConv2DOp, grad_check=False) create_test_cudnn_fp16_class(TestWithPad, grad_check=False) @@ -651,7 +683,7 @@ def init_group(self): create_test_cudnn_fp16_class(TestWith1x1, grad_check=False) create_test_cudnn_fp16_class(TestWithInput1x1Filter1x1, grad_check=False) -#----------------Conv2DCUDNN bf16---------------- +# ----------------Conv2DCUDNN bf16---------------- create_test_cudnn_bf16_class(TestConv2DOp) create_test_cudnn_bf16_class(TestWithPad) @@ -662,7 +694,6 @@ def init_group(self): class TestCUDNNExhaustiveSearch(TestConv2DOp): - def init_kernel_type(self): self.use_cudnn = True self.exhaustive_search = True @@ -670,14 +701,14 @@ def init_kernel_type(self): class TestConv2DOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of conv2d must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) fluid.layers.conv2d(x1, 1, 1) self.assertRaises(TypeError, test_Variable) @@ -685,9 +716,9 @@ def test_Variable(): def test_dtype(): # the input dtype of conv2d must be float16 or float32 or float64 # float16 only can be set on GPU place - x2 = fluid.layers.data(name='x2', - shape=[3, 4, 5, 6], - dtype="int32") + x2 = fluid.layers.data( + name='x2', shape=[3, 4, 5, 6], dtype="int32" + ) fluid.layers.conv2d(x2, 1, 1) self.assertRaises(TypeError, test_dtype) @@ -703,7 +734,6 @@ def test_dtype(): class TestConv2DOp_v2(OpTest): - def setUp(self): self.op_type = "conv2d" self.use_cudnn = False @@ -723,7 +753,7 @@ def setUp(self): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) @@ -737,15 +767,19 @@ def setUp(self): else: input2 = input filter = np.random.uniform(-1, 1, self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input2, filter, self.groups, - conv2d_param, - self.padding_algorithm, - self.data_format) + output, _, _, _, _ = conv2d_forward_naive( + input2, + filter, + self.groups, + conv2d_param, + self.padding_algorithm, + self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -756,53 +790,62 @@ def setUp(self): 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, 'data_format': self.data_format, - 'fuse_relu_before_depthwise_conv': - self.fuse_relu_before_depthwise_conv, - 'exhaustive_search': self.exhaustive_search + 'fuse_relu_before_depthwise_conv': self.fuse_relu_before_depthwise_conv, + 'exhaustive_search': self.exhaustive_search, } self.outputs = {'Output': output} def has_cuda(self): - return core.is_compiled_with_cuda() and (self.use_cudnn - or self.use_cuda) + return core.is_compiled_with_cuda() and ( + self.use_cudnn or self.use_cuda + ) def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() - self.check_output_with_place(place, - atol=1e-5, - check_dygraph=(self.use_mkldnn == False)) + self.check_output_with_place( + place, atol=1e-5, check_dygraph=(self.use_mkldnn == False) + ) def test_check_grad(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() - self.check_grad_with_place(place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.02, - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.02, + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_no_filter(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() - self.check_grad_with_place(place, ['Input'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Filter']), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + ['Input'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Filter']), + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_no_input(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() - self.check_grad_with_place(place, ['Filter'], - 'Output', - no_grad_set=set(['Input']), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + ['Filter'], + 'Output', + no_grad_set=set(['Input']), + check_dygraph=(self.use_mkldnn == False), + ) def init_test_case(self): self.pad = [0, 0] @@ -833,14 +876,12 @@ def init_test_case_2(self): class TestConv2DOp_AsyPadding(TestConv2DOp_v2): - def init_paddings(self): self.pad = [0, 0, 1, 2] self.padding_algorithm = "EXPLICIT" class TestWithPad_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -854,7 +895,6 @@ def init_paddings(self): class TestWithStride_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 3, 6, 6] # NCHW @@ -868,7 +908,6 @@ def init_paddings(self): class TestWithGroup_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 2] @@ -880,7 +919,6 @@ def init_test_case(self): class TestWith1x1_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -897,7 +935,6 @@ def init_paddings(self): class TestWithDepthWise3x3_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [3, 4, 10, 10] # NCHW @@ -917,7 +954,6 @@ def init_paddings(self): class TestWithDepthWise5x5_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 4, 10, 10] # NCHW @@ -934,7 +970,6 @@ def init_paddings(self): class TestWithDepthWise7x7_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 8, 10, 10] # NCHW @@ -951,7 +986,6 @@ def init_paddings(self): class TestWithDilation_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 10, 10] # NCHW @@ -971,7 +1005,6 @@ def init_paddings(self): class TestWithInput1x1Filter1x1_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [40, 3, 1, 1] # NCHW @@ -994,7 +1027,7 @@ def init_paddings(self): create_test_cudnn_class(TestWith1x1_AsyPadding) create_test_cudnn_class(TestWithInput1x1Filter1x1_AsyPadding) -#---------- test SAME VALID ----------- +# ---------- test SAME VALID ----------- create_test_padding_SAME_class(TestConv2DOp_AsyPadding) create_test_padding_SAME_class(TestWithPad_AsyPadding) create_test_padding_SAME_class(TestWithStride_AsyPadding) @@ -1032,16 +1065,21 @@ def init_paddings(self): create_test_cudnn_channel_last_class(TestWithGroup_AsyPadding) create_test_cudnn_channel_last_class(TestWithDilation_AsyPadding) -create_test_cudnn_channel_last_fp16_class(TestConv2DOp_AsyPadding, - grad_check=False) -create_test_cudnn_channel_last_fp16_class(TestWithPad_AsyPadding, - grad_check=False) -create_test_cudnn_channel_last_fp16_class(TestWithStride_AsyPadding, - grad_check=False) -create_test_cudnn_channel_last_fp16_class(TestWithGroup_AsyPadding, - grad_check=False) -create_test_cudnn_channel_last_fp16_class(TestWithDilation_AsyPadding, - grad_check=False) +create_test_cudnn_channel_last_fp16_class( + TestConv2DOp_AsyPadding, grad_check=False +) +create_test_cudnn_channel_last_fp16_class( + TestWithPad_AsyPadding, grad_check=False +) +create_test_cudnn_channel_last_fp16_class( + TestWithStride_AsyPadding, grad_check=False +) +create_test_cudnn_channel_last_fp16_class( + TestWithGroup_AsyPadding, grad_check=False +) +create_test_cudnn_channel_last_fp16_class( + TestWithDilation_AsyPadding, grad_check=False +) if __name__ == '__main__': paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_op_depthwise_conv.py b/python/paddle/fluid/tests/unittests/test_conv2d_op_depthwise_conv.py index 73f9b9e6ced53..d6abdc0bcf41f 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_op_depthwise_conv.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_op_depthwise_conv.py @@ -19,13 +19,20 @@ paddle.enable_static() import paddle.fluid.core as core -from test_conv2d_op import TestConv2DOp, TestConv2DOp_v2, create_test_padding_SAME_class, create_test_padding_VALID_class, create_test_channel_last_class, create_test_cudnn_padding_SAME_class, create_test_cudnn_channel_last_class +from test_conv2d_op import ( + TestConv2DOp, + TestConv2DOp_v2, + create_test_padding_SAME_class, + create_test_padding_VALID_class, + create_test_channel_last_class, + create_test_cudnn_padding_SAME_class, + create_test_cudnn_channel_last_class, +) -#----------------TestDepthwiseConv ----- +# ----------------TestDepthwiseConv ----- class TestDepthwiseConv(TestConv2DOp): - def init_test_case(self): self.use_cuda = True self.pad = [1, 1] @@ -39,7 +46,6 @@ def init_test_case(self): class TestDepthwiseConv2(TestConv2DOp): - def init_test_case(self): self.use_cuda = True self.pad = [1, 1] @@ -53,7 +59,6 @@ def init_test_case(self): class TestDepthwiseConv3(TestConv2DOp): - def init_test_case(self): self.use_cuda = True self.pad = [1, 1] @@ -67,7 +72,6 @@ def init_test_case(self): class TestDepthwiseConvWithDilation(TestConv2DOp): - def init_test_case(self): self.use_cuda = True self.pad = [1, 1] @@ -82,7 +86,6 @@ def init_test_case(self): class TestDepthwiseConvWithDilation2(TestConv2DOp): - def init_test_case(self): self.use_cuda = True self.pad = [1, 1] @@ -97,7 +100,6 @@ def init_test_case(self): class TestDepthwiseConvandFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -112,7 +114,6 @@ def init_test_case(self): class TestDepthwiseConv2andFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -127,7 +128,6 @@ def init_test_case(self): class TestDepthwiseConv3andFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -142,7 +142,6 @@ def init_test_case(self): class TestDepthwiseConvWithDilationandFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -158,7 +157,6 @@ def init_test_case(self): class TestDepthwiseConvWithDilation2andFuse(TestConv2DOp): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -174,7 +172,6 @@ def init_test_case(self): class TestDepthwiseConv_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = True self.stride = [2, 2] @@ -191,7 +188,6 @@ def init_paddings(self): class TestDepthwiseConv2_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = True self.stride = [1, 1] @@ -208,7 +204,6 @@ def init_paddings(self): class TestDepthwiseConv3_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = True self.stride = [1, 1] @@ -225,7 +220,6 @@ def init_paddings(self): class TestDepthwiseConvWithDilation_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = True self.pad = [1, 1] @@ -244,7 +238,6 @@ def init_paddings(self): class TestDepthwiseConvWithDilation2_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = True self.pad = [1, 1] @@ -263,7 +256,6 @@ def init_paddings(self): class TestDepthwiseConvandFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -282,7 +274,6 @@ def init_paddings(self): class TestDepthwiseConv2andFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -301,7 +292,6 @@ def init_paddings(self): class TestDepthwiseConv3andFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -320,7 +310,6 @@ def init_paddings(self): class TestDepthwiseConvWithDilationandFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -340,7 +329,6 @@ def init_paddings(self): class TestDepthwiseConvWithDilation2andFuse_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.fuse_relu_before_depthwise_conv = True self.use_cuda = True @@ -382,12 +370,14 @@ def init_paddings(self): if core.is_compiled_with_rocm(): create_test_cudnn_padding_SAME_class(TestDepthwiseConv_AsyPadding) create_test_cudnn_padding_SAME_class( - TestDepthwiseConvWithDilation_AsyPadding) + TestDepthwiseConvWithDilation_AsyPadding + ) create_test_padding_VALID_class(TestDepthwiseConv_AsyPadding) create_test_padding_VALID_class(TestDepthwiseConvWithDilation_AsyPadding) create_test_cudnn_channel_last_class(TestDepthwiseConv_AsyPadding) create_test_cudnn_channel_last_class( - TestDepthwiseConvWithDilation2_AsyPadding) + TestDepthwiseConvWithDilation2_AsyPadding + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_layer.py b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_layer.py index 74d50c545c658..8e6bdefc5adeb 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_layer.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_layer.py @@ -21,23 +21,24 @@ class Conv2DTransposeTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - batch_size=4, - spartial_shape=(16, 16), - num_channels=6, - num_filters=8, - filter_size=3, - output_size=None, - output_padding=0, - padding=0, - stride=1, - dilation=1, - groups=1, - no_bias=False, - data_format="NCHW", - dtype="float32"): + def __init__( + self, + methodName='runTest', + batch_size=4, + spartial_shape=(16, 16), + num_channels=6, + num_filters=8, + filter_size=3, + output_size=None, + output_padding=0, + padding=0, + stride=1, + dilation=1, + groups=1, + no_bias=False, + data_format="NCHW", + dtype="float32", + ): super(Conv2DTransposeTestCase, self).__init__(methodName) self.batch_size = batch_size self.num_channels = num_channels @@ -58,24 +59,31 @@ def __init__(self, def setUp(self): self.channel_last = self.data_format == "NHWC" if self.channel_last: - input_shape = (self.batch_size, ) + self.spartial_shape + ( - self.num_channels, ) + input_shape = ( + (self.batch_size,) + self.spartial_shape + (self.num_channels,) + ) else: - input_shape = (self.batch_size, - self.num_channels) + self.spartial_shape + input_shape = ( + self.batch_size, + self.num_channels, + ) + self.spartial_shape self.input = np.random.randn(*input_shape).astype(self.dtype) if isinstance(self.filter_size, int): filter_size = [self.filter_size] * 2 else: filter_size = self.filter_size - self.weight_shape = weight_shape = (self.num_channels, self.num_filters - // self.groups) + tuple(filter_size) - self.weight = np.random.uniform(-1, 1, - size=weight_shape).astype(self.dtype) + self.weight_shape = weight_shape = ( + self.num_channels, + self.num_filters // self.groups, + ) + tuple(filter_size) + self.weight = np.random.uniform(-1, 1, size=weight_shape).astype( + self.dtype + ) if not self.no_bias: self.bias = np.random.uniform( - -1, 1, size=(self.num_filters, )).astype(self.dtype) + -1, 1, size=(self.num_filters,) + ).astype(self.dtype) else: self.bias = None @@ -84,8 +92,11 @@ def fluid_layer(self, place): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1,self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1) + input_shape = ( + (-1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) weight_attr = I.NumpyArrayInitializer(self.weight) if self.bias is None: @@ -104,11 +115,12 @@ def fluid_layer(self, place): groups=self.groups, param_attr=weight_attr, bias_attr=bias_attr, - data_format=self.data_format) + data_format=self.data_format, + ) feed_dict = {"input": self.input} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def functional(self, place): @@ -116,36 +128,42 @@ def functional(self, place): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1,self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1) + input_shape = ( + (-1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) - w_var = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) - b_var = fluid.data("bias", (self.num_filters, ), - dtype=self.dtype) + w_var = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) + b_var = fluid.data( + "bias", (self.num_filters,), dtype=self.dtype + ) if self.output_padding != 0: output_size = None else: output_size = self.output_size - y_var = F.conv2d_transpose(x_var, - w_var, - None if self.no_bias else b_var, - output_size=output_size, - padding=self.padding, - output_padding=self.output_padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y_var = F.conv2d_transpose( + x_var, + w_var, + None if self.no_bias else b_var, + output_size=output_size, + padding=self.padding, + output_padding=self.output_padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) feed_dict = {"input": self.input, "weight": self.weight} if self.bias is not None: feed_dict["bias"] = self.bias exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_nn_layer(self): @@ -156,15 +174,17 @@ def paddle_nn_layer(self): else: output_size = self.output_size - conv = nn.Conv2DTranspose(self.num_channels, - self.num_filters, - self.filter_size, - padding=self.padding, - output_padding=self.output_padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + conv = nn.Conv2DTranspose( + self.num_channels, + self.num_filters, + self.filter_size, + padding=self.padding, + output_padding=self.output_padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) conv.weight.set_value(self.weight) if not self.no_bias: conv.bias.set_value(self.bias) @@ -194,7 +214,6 @@ def runTest(self): class Conv2DTransposeErrorTestCase(Conv2DTransposeTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -205,46 +224,61 @@ def runTest(self): def add_cases(suite): suite.addTest(Conv2DTransposeTestCase(methodName='runTest')) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', - stride=[1, 2], - no_bias=True, - dilation=2)) + Conv2DTransposeTestCase( + methodName='runTest', stride=[1, 2], no_bias=True, dilation=2 + ) + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', - filter_size=(3, 3), - output_size=[20, 36], - stride=[1, 2], - dilation=2)) + Conv2DTransposeTestCase( + methodName='runTest', + filter_size=(3, 3), + output_size=[20, 36], + stride=[1, 2], + dilation=2, + ) + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', stride=2, - dilation=(2, 1))) - suite.addTest(Conv2DTransposeTestCase(methodName='runTest', - padding="valid")) + Conv2DTransposeTestCase(methodName='runTest', stride=2, dilation=(2, 1)) + ) + suite.addTest( + Conv2DTransposeTestCase(methodName='runTest', padding="valid") + ) suite.addTest(Conv2DTransposeTestCase(methodName='runTest', padding="same")) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', - filter_size=1, - padding=(2, 3))) + Conv2DTransposeTestCase( + methodName='runTest', filter_size=1, padding=(2, 3) + ) + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', padding=[1, 2, 2, 1])) + Conv2DTransposeTestCase(methodName='runTest', padding=[1, 2, 2, 1]) + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', - padding=[[0, 0], [0, 0], [1, 2], [2, 1]])) + Conv2DTransposeTestCase( + methodName='runTest', padding=[[0, 0], [0, 0], [1, 2], [2, 1]] + ) + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', data_format="NHWC")) + Conv2DTransposeTestCase(methodName='runTest', data_format="NHWC") + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', - data_format="NHWC", - padding=[[0, 0], [1, 1], [2, 2], [0, 0]])) + Conv2DTransposeTestCase( + methodName='runTest', + data_format="NHWC", + padding=[[0, 0], [1, 1], [2, 2], [0, 0]], + ) + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', groups=2, - padding="valid")) + Conv2DTransposeTestCase(methodName='runTest', groups=2, padding="valid") + ) suite.addTest( - Conv2DTransposeTestCase(methodName='runTest', - num_filters=6, - num_channels=3, - groups=3, - padding="valid")) + Conv2DTransposeTestCase( + methodName='runTest', + num_filters=6, + num_channels=3, + groups=3, + padding="valid", + ) + ) suite.addTest( Conv2DTransposeTestCase( methodName='runTest', @@ -257,17 +291,21 @@ def add_cases(suite): stride=2, output_size=[14, 14], output_padding=[1, 1], - )) + ) + ) def add_error_cases(suite): suite.addTest( - Conv2DTransposeErrorTestCase(methodName='runTest', - num_channels=5, - groups=2)) + Conv2DTransposeErrorTestCase( + methodName='runTest', num_channels=5, groups=2 + ) + ) suite.addTest( - Conv2DTransposeErrorTestCase(methodName='runTest', - output_size="not_valid")) + Conv2DTransposeErrorTestCase( + methodName='runTest', output_size="not_valid" + ) + ) def load_tests(loader, standard_tests, pattern): diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py index da822c3b124d5..29ffbd80d3623 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py @@ -30,9 +30,10 @@ def conv2dtranspose_forward_naive(input_, filter_, attrs): padding_algorithm = attrs['padding_algorithm'] if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if attrs['data_format'] == 'NHWC': input_ = np.transpose(input_, [0, 3, 1, 2]) @@ -43,18 +44,22 @@ def conv2dtranspose_forward_naive(input_, filter_, attrs): out_c = f_out_c * groups sub_in_c = in_c // groups - stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ - 'dilations'] + stride, pad, dilations = ( + attrs['strides'], + attrs['paddings'], + attrs['dilations'], + ) # update pad and dilation def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, - kernel_size, - kernel_stride): + for input_size, filter_size, stride_size in zip( + input_shape, kernel_size, kernel_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -88,37 +93,49 @@ def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride): if 'output_padding' in attrs: out_pad_h = attrs['output_padding'][0] out_pad_w = attrs['output_padding'][1] - out = np.zeros((in_n, out_c, out_h + out_pad_h, out_w + out_pad_w), - dtype=input_.dtype) + out = np.zeros( + (in_n, out_c, out_h + out_pad_h, out_w + out_pad_w), dtype=input_.dtype + ) for n in range(in_n): for i in range(in_h): for j in range(in_w): for g in range(groups): - input_masked = input_[n, g * sub_in_c:(g + 1) * sub_in_c, i, - j] # (c) + input_masked = input_[ + n, g * sub_in_c : (g + 1) * sub_in_c, i, j + ] # (c) input_masked = np.reshape(input_masked, (sub_in_c, 1, 1)) input_masked = np.tile(input_masked, (1, f_h, f_w)) for k in range(f_out_c): tmp_out = np.sum( - input_masked * - filter_[g * sub_in_c:(g + 1) * sub_in_c, k, :, :], - axis=0) + input_masked + * filter_[ + g * sub_in_c : (g + 1) * sub_in_c, k, :, : + ], + axis=0, + ) i1, i2 = i * stride[0], i * stride[0] + d_bolck_h j1, j2 = j * stride[1], j * stride[1] + d_bolck_w - out[n, g * f_out_c + k, i1:i2:dilations[0], - j1:j2:dilations[1]] += tmp_out - - out = out[:, :, pad_h_0:out_h - pad_h_1 + out_pad_h, - pad_w_0:out_w - pad_w_1 + out_pad_w] + out[ + n, + g * f_out_c + k, + i1 : i2 : dilations[0], + j1 : j2 : dilations[1], + ] += tmp_out + + out = out[ + :, + :, + pad_h_0 : out_h - pad_h_1 + out_pad_h, + pad_w_0 : out_w - pad_w_1 + out_pad_w, + ] if attrs['data_format'] == 'NHWC': out = np.transpose(out, [0, 2, 3, 1]) return out class TestConv2DTransposeOp(OpTest): - def setUp(self): # init as conv transpose self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 @@ -147,7 +164,7 @@ def setUp(self): 'use_cudnn': self.use_cudnn, 'is_test': self.is_test, 'use_mkldnn': self.use_mkldnn, - 'data_format': self.data_format + 'data_format': self.data_format, } if self.output_size is not None: self.attrs['output_size'] = self.output_size @@ -155,8 +172,9 @@ def setUp(self): if len(self.output_padding) > 0: self.attrs['output_padding'] = self.output_padding - output = conv2dtranspose_forward_naive(input_, filter_, - self.attrs).astype(self.dtype) + output = conv2dtranspose_forward_naive( + input_, filter_, self.attrs + ).astype(self.dtype) self.outputs = {'Output': output} @@ -165,7 +183,8 @@ def test_check_output(self): if self.use_cudnn: place = core.CUDAPlace(0) self.check_output_with_place( - place, atol=1e-5, check_dygraph=(self.use_mkldnn == False)) + place, atol=1e-5, check_dygraph=(self.use_mkldnn == False) + ) else: self.check_output(check_dygraph=(self.use_mkldnn == False)) @@ -173,39 +192,44 @@ def test_check_grad_no_input(self): if self.need_check_grad: if self.use_cudnn: place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['Filter'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Input'])) + self.check_grad_with_place( + place, + ['Filter'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Input']), + ) else: - self.check_grad(['Filter'], - 'Output', - no_grad_set=set(['Input'])) + self.check_grad( + ['Filter'], 'Output', no_grad_set=set(['Input']) + ) def test_check_grad_no_filter(self): if self.need_check_grad: if self.use_cudnn: place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + place, ['Input'], 'Output', no_grad_set=set(['Filter']) + ) else: - self.check_grad(['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad( + ['Input'], 'Output', no_grad_set=set(['Filter']) + ) def test_check_grad(self): if self.need_check_grad: if self.use_cudnn: place = core.CUDAPlace(0) - self.check_grad_with_place(place, - set(['Input', 'Filter']), - 'Output', - max_relative_error=0.02) + self.check_grad_with_place( + place, + set(['Input', 'Filter']), + 'Output', + max_relative_error=0.02, + ) else: - self.check_grad(set(['Input', 'Filter']), - 'Output', - max_relative_error=0.02) + self.check_grad( + set(['Input', 'Filter']), 'Output', max_relative_error=0.02 + ) def init_test_case(self): self.pad = [0, 0] @@ -221,7 +245,6 @@ def init_op_type(self): class TestWithSymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -233,7 +256,6 @@ def init_test_case(self): class TestWithAsymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -245,7 +267,6 @@ def init_test_case(self): class TestWithSAMEPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [2, 1] self.dilations = [1, 2] @@ -257,7 +278,6 @@ def init_test_case(self): class TestWithVALIDPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [1, 1] self.dilations = [1, 1] @@ -269,7 +289,6 @@ def init_test_case(self): class TestWithGroups(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -281,7 +300,6 @@ def init_test_case(self): class TestWithStride(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -293,7 +311,6 @@ def init_test_case(self): class TestWithDilation(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -305,7 +322,6 @@ def init_test_case(self): class TestWithEvenUpsample(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -318,7 +334,6 @@ def init_test_case(self): class TestWithEvenUpsampleOutputPadding(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -331,7 +346,6 @@ def init_test_case(self): class Test_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -344,7 +358,6 @@ def init_test_case(self): class TestWithSymmetricPad_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -357,7 +370,6 @@ def init_test_case(self): class TestWithAsymmetricPad_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -370,7 +382,6 @@ def init_test_case(self): class TestWithGroups_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -383,7 +394,6 @@ def init_test_case(self): class TestWithStride_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -396,7 +406,6 @@ def init_test_case(self): class TestWithDilation_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -409,7 +418,6 @@ def init_test_case(self): class TestWithEvenUpsample_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -423,7 +431,6 @@ def init_test_case(self): class TestWithEvenUpsample_NHWC_output_padding(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -437,19 +444,19 @@ def init_test_case(self): # ------------ test_cudnn ------------ -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNN(TestConv2DTransposeOp): - def init_op_type(self): self.use_cudnn = True self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithSymmetricPad(TestWithSymmetricPad): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -464,10 +471,10 @@ def init_op_type(self): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithAsymmetricPad(TestWithAsymmetricPad): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -482,10 +489,10 @@ def init_op_type(self): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithSAMEPad(TestWithSAMEPad): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 2] @@ -500,10 +507,10 @@ def init_op_type(self): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithVALIDPad(TestWithVALIDPad): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -518,10 +525,10 @@ def init_op_type(self): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithStride(TestWithStride): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -536,10 +543,10 @@ def init_op_type(self): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithGroups(TestWithGroups): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -555,10 +562,10 @@ def init_op_type(self): # ------------ test_cudnn ------------ -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithEvenUpsample(TestWithEvenUpsample): - def init_op_type(self): self.use_cudnn = True self.op_type = "conv2d_transpose" @@ -579,10 +586,10 @@ def init_op_type(self): # self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNN_NHWC(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -598,10 +605,10 @@ def init_op_type(self): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithSymmetricPad_NHWC(TestWithSymmetricPad): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -617,10 +624,10 @@ def init_op_type(self): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithAsymmetricPad_NHWC(TestWithSymmetricPad): - def init_test_case(self): self.pad = [1, 0, 2, 3] self.stride = [2, 2] @@ -636,10 +643,10 @@ def init_op_type(self): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithStride_NHWC(TestWithStride): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -655,10 +662,10 @@ def init_op_type(self): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithGroups_NHWC(TestWithGroups): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -674,10 +681,10 @@ def init_op_type(self): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithEvenUpsample_NHWC(TestWithEvenUpsample): - def init_test_case(self): self.pad = [2, 2] self.stride = [2, 2] @@ -694,10 +701,10 @@ def init_op_type(self): self.op_type = "conv2d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNN_FP16(TestConv2DTransposeOp): - def init_test_case(self): self.dtype = np.float16 self.pad = [1, 1] @@ -717,15 +724,16 @@ def test_check_output(self): if self.use_cudnn: place = core.CUDAPlace(0) self.check_output_with_place( - place, atol=0.02, check_dygraph=(self.use_mkldnn == False)) + place, atol=0.02, check_dygraph=(self.use_mkldnn == False) + ) else: self.check_output(check_dygraph=(self.use_mkldnn == False)) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNN_NHWC_FP16(TestCUDNN_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [0, 0] @@ -738,10 +746,10 @@ def init_test_case(self): self.data_format = 'NHWC' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithSymmetricPad_NHWC_FP16(TestCUDNN_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [1, 1] @@ -754,10 +762,10 @@ def init_test_case(self): self.data_format = 'NHWC' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithAsymmetricPad_NHWC_FP16(TestCUDNN_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [1, 0, 2, 3] @@ -770,10 +778,10 @@ def init_test_case(self): self.data_format = 'NHWC' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithStride_NHWC_FP16(TestCUDNN_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [1, 1] @@ -786,10 +794,10 @@ def init_test_case(self): self.data_format = 'NHWC' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithGroups_NHWC_FP16(TestCUDNN_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [1, 1] @@ -802,10 +810,10 @@ def init_test_case(self): self.data_format = 'NHWC' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithEvenUpsample_NHWC_FP16(TestCUDNN_FP16): - def init_test_case(self): self.dtype = np.float16 self.pad = [2, 2] @@ -820,56 +828,67 @@ def init_test_case(self): class TestConv2DTransposeAPI(unittest.TestCase): - def test_case1(self): - data1 = fluid.layers.data(name='data1', - shape=[3, 5, 5], - dtype='float32') - data2 = fluid.layers.data(name='data2', - shape=[5, 5, 3], - dtype='float32') - out1 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - data_format='NCHW') - out2 = fluid.layers.conv2d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - data_format='NHWC') - out3 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - padding=[[0, 0], [1, 1], [1, 1], - [0, 0]], - data_format='NHWC') - out4 = fluid.layers.conv2d_transpose(input=data1, - groups=3, - num_filters=6, - filter_size=3, - padding=[[0, 0], [0, 0], [2, 1], - [0, 0]], - data_format='NCHW') - out5 = fluid.layers.conv2d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - padding='SAME', - data_format='NCHW') - out6 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - padding='VALID', - data_format='NHWC') - out7 = fluid.layers.conv2d_transpose(input=data1, - groups=1, - num_filters=6, - output_size=[7, 7], - padding=[0, 0], - data_format='NHWC') + data1 = fluid.layers.data( + name='data1', shape=[3, 5, 5], dtype='float32' + ) + data2 = fluid.layers.data( + name='data2', shape=[5, 5, 3], dtype='float32' + ) + out1 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + data_format='NCHW', + ) + out2 = fluid.layers.conv2d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + data_format='NHWC', + ) + out3 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + padding=[[0, 0], [1, 1], [1, 1], [0, 0]], + data_format='NHWC', + ) + out4 = fluid.layers.conv2d_transpose( + input=data1, + groups=3, + num_filters=6, + filter_size=3, + padding=[[0, 0], [0, 0], [2, 1], [0, 0]], + data_format='NCHW', + ) + out5 = fluid.layers.conv2d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + padding='SAME', + data_format='NCHW', + ) + out6 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + padding='VALID', + data_format='NHWC', + ) + out7 = fluid.layers.conv2d_transpose( + input=data1, + groups=1, + num_filters=6, + output_size=[7, 7], + padding=[0, 0], + data_format='NHWC', + ) data1_np = np.random.random((2, 3, 5, 5)).astype("float32") data2_np = np.random.random((2, 5, 5, 3)).astype("float32") @@ -880,13 +899,12 @@ def test_case1(self): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "data1": data1_np, - "data2": data2_np - }, - fetch_list=[out1, out2, out3, out4, out5, out6, out7], - return_numpy=True) + results = exe.run( + fluid.default_main_program(), + feed={"data1": data1_np, "data2": data2_np}, + fetch_list=[out1, out2, out3, out4, out5, out6, out7], + return_numpy=True, + ) self.assertIsNotNone(results[0]) self.assertIsNotNone(results[1]) self.assertIsNotNone(results[2]) @@ -897,76 +915,81 @@ def test_case1(self): class TestConv2DTransposeOpException(unittest.TestCase): - def test_exception(self): data = fluid.layers.data(name='data', shape=[3, 5, 5], dtype="float32") def attr_data_format(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - data_format="NCDHW") + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + data_format="NCDHW", + ) self.assertRaises(ValueError, attr_data_format) def attr_padding_str(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding='Vald') + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding='Vald', + ) self.assertRaises(ValueError, attr_padding_str) def attr_padding_list(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding=[[1, 1], [1, 1], [0, 0], - [0, 0]]) + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding=[[1, 1], [1, 1], [0, 0], [0, 0]], + ) self.assertRaises(ValueError, attr_padding_list) def attr_padding_with_data_format(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding=[[1, 1], [0, 0], [0, 0], - [1, 1]], - data_format='NHWC') + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding=[[1, 1], [0, 0], [0, 0], [1, 1]], + data_format='NHWC', + ) self.assertRaises(ValueError, attr_padding_with_data_format) - error_input = fluid.layers.data(name='error_data', - shape=[1], - dtype="float32") + error_input = fluid.layers.data( + name='error_data', shape=[1], dtype="float32" + ) def error_input_size(): - out = fluid.layers.conv2d_transpose(input=error_input, - groups=1, - num_filters=6, - filter_size=3) + out = fluid.layers.conv2d_transpose( + input=error_input, groups=1, num_filters=6, filter_size=3 + ) self.assertRaises(ValueError, error_input_size) def error_groups(): - out = fluid.layers.conv2d_transpose(input=data, - groups=0, - num_filters=6, - filter_size=3, - data_format='NHWC') + out = fluid.layers.conv2d_transpose( + input=data, + groups=0, + num_filters=6, + filter_size=3, + data_format='NHWC', + ) self.assertRaises(ValueError, error_groups) class TestConv2DTransposeRepr(unittest.TestCase): - def test_case(self): paddle.disable_static() - x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.) + x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1.0, max=1.0) conv = nn.Conv2DTranspose(4, 6, (3, 3), output_padding=1, stride=2) print(conv) y_var = conv(x_var) @@ -976,7 +999,6 @@ def test_case(self): class TestTensorOutputSize1(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 8, 8]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -991,7 +1013,8 @@ def call_func(self, x): w_var = paddle.randn((3, 6, 3, 3), dtype='float32') output_size = paddle.assign([17]) out = paddle.paddle.nn.functional.conv2d_transpose( - x, w_var, stride=2, output_size=output_size) + x, w_var, stride=2, output_size=output_size + ) return out def test_static(self): @@ -1013,15 +1036,15 @@ def test_static(self): res = exe.run(fetch_list=[feat, out]) np.testing.assert_allclose(res[1].shape, (2, 6, 17, 17)) - paddle.static.save_inference_model(self.save_path, [x], [feat, out], - exe) + paddle.static.save_inference_model( + self.save_path, [x], [feat, out], exe + ) # Test for Inference Predictor infer_outs = self.infer_prog() np.testing.assert_allclose(infer_outs[1].shape, (2, 6, 17, 17)) class TestTensorOutputSize2(TestTensorOutputSize1): - def path_prefix(self): return 'conv2d_transpose_tensor_output_size2' @@ -1029,43 +1052,37 @@ def call_func(self, x): w_var = paddle.randn((3, 6, 3, 3), dtype='float32') output_size = [17, paddle.assign([17])] out = paddle.paddle.nn.functional.conv2d_transpose( - x, w_var, stride=2, output_size=output_size) + x, w_var, stride=2, output_size=output_size + ) return out class TestTensorOutputSize3(TestTensorOutputSize1): - def path_prefix(self): return 'conv2d_transpose_tensor_output_size3' def call_func(self, x): w_var = paddle.randn((3, 6, 3, 3), dtype='float32') output_size = paddle.assign([17]) - out = paddle.fluid.layers.conv2d_transpose(x, - num_filters=6, - output_size=output_size, - filter_size=3, - stride=2) + out = paddle.fluid.layers.conv2d_transpose( + x, num_filters=6, output_size=output_size, filter_size=3, stride=2 + ) return out class TestTensorOutputSize4(TestTensorOutputSize1): - def path_prefix(self): return 'conv2d_transpose_tensor_output_size4' def call_func(self, x): output_size = [17, paddle.assign([17])] - out = paddle.fluid.layers.conv2d_transpose(x, - num_filters=6, - output_size=output_size, - filter_size=3, - stride=2) + out = paddle.fluid.layers.conv2d_transpose( + x, num_filters=6, output_size=output_size, filter_size=3, stride=2 + ) return out class TestTensorOutputSize5(TestTensorOutputSize1): - def path_prefix(self): return 'conv2d_transpose_tensor_output_size5' @@ -1077,13 +1094,13 @@ def call_func(self, x): num_filters=6, filter_size=3, output_size=output_size, - stride=2) + stride=2, + ) out = conv2d_trans(x) return out class TestTensorOutputSize6(TestTensorOutputSize1): - def path_prefix(self): return 'conv2d_transpose_tensor_output_size6' @@ -1098,13 +1115,13 @@ def call_func(self, x): num_filters=6, filter_size=3, output_size=output_size, - stride=2) + stride=2, + ) out = conv2d_trans(x) return out class TestTensorOutputSize7(TestTensorOutputSize1): - def path_prefix(self): return 'conv2d_transpose_tensor_output_size7' @@ -1119,13 +1136,13 @@ def call_func(self, x): num_filters=6, filter_size=3, output_size=output_size, - stride=2) + stride=2, + ) out = conv2d_trans(x) return out class TestTensorOutputSize8(TestTensorOutputSize1): - def path_prefix(self): return 'conv2d_transpose_tensor_output_size8' @@ -1140,7 +1157,8 @@ def call_func(self, x): num_filters=6, filter_size=3, output_size=output_size, - stride=2) + stride=2, + ) out = conv2d_trans(x) return out diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op_depthwise_conv.py b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op_depthwise_conv.py index b837efaa783f4..a50b0e2cb9673 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op_depthwise_conv.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op_depthwise_conv.py @@ -22,7 +22,6 @@ class TestDepthwiseConvTranspose(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -36,7 +35,6 @@ def init_test_case(self): class TestDepthwiseConvTransposeAsymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1, 1, 2] self.stride = [1, 1] @@ -51,7 +49,6 @@ def init_test_case(self): class TestDepthwiseConvTransposeSAMEPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [1, 1] self.dilations = [1, 1] @@ -65,7 +62,6 @@ def init_test_case(self): class TestDepthwiseConvTransposeVALIDPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [1, 1] self.dilations = [1, 1] @@ -79,7 +75,6 @@ def init_test_case(self): class TestDepthwiseConvTranspose_NHWC_3x3kernel(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_layer.py b/python/paddle/fluid/tests/unittests/test_conv3d_layer.py index 5672fe0dd88e4..053bbb641665e 100644 --- a/python/paddle/fluid/tests/unittests/test_conv3d_layer.py +++ b/python/paddle/fluid/tests/unittests/test_conv3d_layer.py @@ -23,21 +23,22 @@ class Conv3DTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - batch_size=4, - spartial_shape=(8, 8, 8), - num_channels=6, - num_filters=8, - filter_size=3, - padding=0, - stride=1, - dilation=1, - groups=1, - no_bias=False, - data_format="NCDHW", - dtype="float32"): + def __init__( + self, + methodName='runTest', + batch_size=4, + spartial_shape=(8, 8, 8), + num_channels=6, + num_filters=8, + filter_size=3, + padding=0, + stride=1, + dilation=1, + groups=1, + no_bias=False, + data_format="NCDHW", + dtype="float32", + ): super(Conv3DTestCase, self).__init__(methodName) self.batch_size = batch_size self.num_channels = num_channels @@ -56,24 +57,31 @@ def __init__(self, def setUp(self): self.channel_last = self.data_format == "NDHWC" if self.channel_last: - input_shape = (self.batch_size, ) + self.spartial_shape + ( - self.num_channels, ) + input_shape = ( + (self.batch_size,) + self.spartial_shape + (self.num_channels,) + ) else: - input_shape = (self.batch_size, - self.num_channels) + self.spartial_shape + input_shape = ( + self.batch_size, + self.num_channels, + ) + self.spartial_shape self.input = np.random.randn(*input_shape).astype(self.dtype) if isinstance(self.filter_size, int): filter_size = [self.filter_size] * 3 else: filter_size = self.filter_size - self.weight_shape = weight_shape = (self.num_filters, self.num_channels - // self.groups) + tuple(filter_size) - self.weight = np.random.uniform(-1, 1, - size=weight_shape).astype(self.dtype) + self.weight_shape = weight_shape = ( + self.num_filters, + self.num_channels // self.groups, + ) + tuple(filter_size) + self.weight = np.random.uniform(-1, 1, size=weight_shape).astype( + self.dtype + ) if not self.no_bias: self.bias = np.random.uniform( - -1, 1, size=(self.num_filters, )).astype(self.dtype) + -1, 1, size=(self.num_filters,) + ).astype(self.dtype) else: self.bias = None @@ -82,28 +90,33 @@ def fluid_layer(self, place): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1, -1, self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1, -1) + input_shape = ( + (-1, -1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) weight_attr = I.NumpyArrayInitializer(self.weight) if self.bias is None: bias_attr = False else: bias_attr = I.NumpyArrayInitializer(self.bias) - y_var = fluid.layers.conv3d(x_var, - self.num_filters, - self.filter_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - param_attr=weight_attr, - bias_attr=bias_attr, - data_format=self.data_format) + y_var = fluid.layers.conv3d( + x_var, + self.num_filters, + self.filter_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + param_attr=weight_attr, + bias_attr=bias_attr, + data_format=self.data_format, + ) feed_dict = {"input": self.input} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def functional(self, place): @@ -111,41 +124,49 @@ def functional(self, place): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1, -1, self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1, -1) + input_shape = ( + (-1, -1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) - w_var = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) - b_var = fluid.data("bias", (self.num_filters, ), - dtype=self.dtype) - y_var = F.conv3d(x_var, - w_var, - None if self.no_bias else b_var, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + w_var = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) + b_var = fluid.data( + "bias", (self.num_filters,), dtype=self.dtype + ) + y_var = F.conv3d( + x_var, + w_var, + None if self.no_bias else b_var, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) feed_dict = {"input": self.input, "weight": self.weight} if self.bias is not None: feed_dict["bias"] = self.bias exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_nn_layer(self): x_var = paddle.to_tensor(self.input) x_var.stop_gradient = False - conv = nn.Conv3D(self.num_channels, - self.num_filters, - self.filter_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + conv = nn.Conv3D( + self.num_channels, + self.num_filters, + self.filter_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) conv.weight.set_value(self.weight) if not self.no_bias: conv.bias.set_value(self.bias) @@ -178,7 +199,6 @@ def runTest(self): class Conv3DErrorTestCase(Conv3DTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -189,44 +209,60 @@ def runTest(self): def add_cases(suite): suite.addTest(Conv3DTestCase(methodName='runTest')) suite.addTest( - Conv3DTestCase(methodName='runTest', stride=[1, 2, 1], dilation=2)) + Conv3DTestCase(methodName='runTest', stride=[1, 2, 1], dilation=2) + ) suite.addTest( - Conv3DTestCase(methodName='runTest', stride=2, dilation=(2, 1, 2))) + Conv3DTestCase(methodName='runTest', stride=2, dilation=(2, 1, 2)) + ) suite.addTest( - Conv3DTestCase(methodName='runTest', padding="same", no_bias=True)) + Conv3DTestCase(methodName='runTest', padding="same", no_bias=True) + ) suite.addTest( - Conv3DTestCase(methodName='runTest', - filter_size=(3, 2, 3), - padding='valid')) + Conv3DTestCase( + methodName='runTest', filter_size=(3, 2, 3), padding='valid' + ) + ) suite.addTest(Conv3DTestCase(methodName='runTest', padding=(2, 3, 1))) suite.addTest( - Conv3DTestCase(methodName='runTest', padding=[1, 2, 2, 1, 2, 3])) + Conv3DTestCase(methodName='runTest', padding=[1, 2, 2, 1, 2, 3]) + ) suite.addTest( - Conv3DTestCase(methodName='runTest', - padding=[[0, 0], [0, 0], [1, 2], [2, 1], [2, 2]])) + Conv3DTestCase( + methodName='runTest', + padding=[[0, 0], [0, 0], [1, 2], [2, 1], [2, 2]], + ) + ) suite.addTest(Conv3DTestCase(methodName='runTest', data_format="NDHWC")) suite.addTest( - Conv3DTestCase(methodName='runTest', - data_format="NDHWC", - padding=[[0, 0], [1, 1], [3, 3], [2, 2], [0, 0]])) + Conv3DTestCase( + methodName='runTest', + data_format="NDHWC", + padding=[[0, 0], [1, 1], [3, 3], [2, 2], [0, 0]], + ) + ) suite.addTest( - Conv3DTestCase(methodName='runTest', groups=2, padding="valid")) + Conv3DTestCase(methodName='runTest', groups=2, padding="valid") + ) suite.addTest( - Conv3DTestCase(methodName='runTest', - num_filters=6, - num_channels=3, - groups=3, - padding="valid")) + Conv3DTestCase( + methodName='runTest', + num_filters=6, + num_channels=3, + groups=3, + padding="valid", + ) + ) def add_error_cases(suite): suite.addTest( - Conv3DErrorTestCase(methodName='runTest', num_channels=5, groups=2)) + Conv3DErrorTestCase(methodName='runTest', num_channels=5, groups=2) + ) suite.addTest( - Conv3DErrorTestCase(methodName='runTest', - num_channels=5, - groups=2, - padding=[-1, 1, 3])) + Conv3DErrorTestCase( + methodName='runTest', num_channels=5, groups=2, padding=[-1, 1, 3] + ) + ) def load_tests(loader, standard_tests, pattern): diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_op.py b/python/paddle/fluid/tests/unittests/test_conv3d_op.py index a65d1c7310411..eaa6ba04c64e6 100644 --- a/python/paddle/fluid/tests/unittests/test_conv3d_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv3d_op.py @@ -21,24 +21,28 @@ import paddle -def conv3d_forward_naive(input, - filter, - group, - conv_param, - padding_algorithm='EXPLICIT', - data_format="NCDHW"): +def conv3d_forward_naive( + input, + filter, + group, + conv_param, + padding_algorithm='EXPLICIT', + data_format="NCDHW", +): if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if data_format not in ["NCDHW", "NDHWC"]: - raise ValueError("Unknown Attr(data_format): '%s' ." - "It can only be 'NCDHW' or 'NDHWC'." % - str(data_format)) + raise ValueError( + "Unknown Attr(data_format): '%s' ." + "It can only be 'NCDHW' or 'NDHWC'." % str(data_format) + ) - channel_last = (data_format == "NDHWC") + channel_last = data_format == "NDHWC" if channel_last: input = np.transpose(input, [0, 4, 1, 2, 3]) @@ -52,17 +56,22 @@ def conv3d_forward_naive(input, sub_out_c = out_c // group sub_f_n = f_n // group - stride, pad, dilation = conv_param['stride'], conv_param['pad'], conv_param[ - 'dilations'] + stride, pad, dilation = ( + conv_param['stride'], + conv_param['pad'], + conv_param['dilations'], + ) # update pad and dilation def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -85,59 +94,85 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): pad_h_0, pad_h_1 = pad[2], pad[3] pad_w_0, pad_w_1 = pad[4], pad[5] - out_d = 1 + (in_d + pad_d_0 + pad_d_1 - (dilation[0] * - (f_d - 1) + 1)) // stride[0] - out_h = 1 + (in_h + pad_h_0 + pad_h_1 - (dilation[1] * - (f_h - 1) + 1)) // stride[1] - out_w = 1 + (in_w + pad_w_0 + pad_w_1 - (dilation[2] * - (f_w - 1) + 1)) // stride[2] + out_d = ( + 1 + + (in_d + pad_d_0 + pad_d_1 - (dilation[0] * (f_d - 1) + 1)) + // stride[0] + ) + out_h = ( + 1 + + (in_h + pad_h_0 + pad_h_1 - (dilation[1] * (f_h - 1) + 1)) + // stride[1] + ) + out_w = ( + 1 + + (in_w + pad_w_0 + pad_w_1 - (dilation[2] * (f_w - 1) + 1)) + // stride[2] + ) out = np.zeros((in_n, out_c, out_d, out_h, out_w)) - d_bolck_d = (dilation[0] * (f_d - 1) + 1) - d_bolck_h = (dilation[1] * (f_h - 1) + 1) - d_bolck_w = (dilation[2] * (f_w - 1) + 1) - - input_pad = np.pad(input, ((0, 0), (0, 0), (pad_d_0, pad_d_1), - (pad_h_0, pad_h_1), (pad_w_0, pad_w_1)), - mode='constant', - constant_values=0) + d_bolck_d = dilation[0] * (f_d - 1) + 1 + d_bolck_h = dilation[1] * (f_h - 1) + 1 + d_bolck_w = dilation[2] * (f_w - 1) + 1 + + input_pad = np.pad( + input, + ( + (0, 0), + (0, 0), + (pad_d_0, pad_d_1), + (pad_h_0, pad_h_1), + (pad_w_0, pad_w_1), + ), + mode='constant', + constant_values=0, + ) filter_dilation = np.zeros((f_n, f_c, d_bolck_d, d_bolck_h, d_bolck_w)) - filter_dilation[:, :, 0:d_bolck_d:dilation[0], 0:d_bolck_h:dilation[1], - 0:d_bolck_w:dilation[2]] = filter + filter_dilation[ + :, + :, + 0 : d_bolck_d : dilation[0], + 0 : d_bolck_h : dilation[1], + 0 : d_bolck_w : dilation[2], + ] = filter for d in range(out_d): for i in range(out_h): for j in range(out_w): for g in range(group): - input_pad_masked = \ - input_pad[:, g * f_c:(g + 1) * f_c, - d * stride[0]:d * stride[0] + d_bolck_d, - i * stride[1]:i * stride[1] + d_bolck_h, - j * stride[2]:j * stride[2] + d_bolck_w] - - f_sub = filter_dilation[g * sub_f_n:(g + 1) * - sub_f_n, :, :, :, :] + input_pad_masked = input_pad[ + :, + g * f_c : (g + 1) * f_c, + d * stride[0] : d * stride[0] + d_bolck_d, + i * stride[1] : i * stride[1] + d_bolck_h, + j * stride[2] : j * stride[2] + d_bolck_w, + ] + + f_sub = filter_dilation[ + g * sub_f_n : (g + 1) * sub_f_n, :, :, :, : + ] for k in range(sub_out_c): - out[:, g * sub_out_c + k, d, i, j] = \ - np.sum(input_pad_masked * f_sub[k, :, :, :, :], - axis=(1, 2, 3, 4)) + out[:, g * sub_out_c + k, d, i, j] = np.sum( + input_pad_masked * f_sub[k, :, :, :, :], + axis=(1, 2, 3, 4), + ) if channel_last: out = np.transpose(out, [0, 2, 3, 4, 1]) return out def create_test_cudnn_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNCase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) cls_name = "{0}_{1}".format(parent.__name__, "CUDNN") TestCUDNNCase.__name__ = cls_name @@ -145,9 +180,7 @@ def init_kernel_type(self): def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.pad = [0, 0, 0] self.padding_algorithm = "SAME" @@ -158,9 +191,7 @@ def init_paddings(self): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1, 1] self.padding_algorithm = "VALID" @@ -171,15 +202,15 @@ def init_paddings(self): def create_test_cudnn_padding_SAME_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingSMAECase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) def init_paddings(self): self.pad = [1, 1, 1] @@ -191,15 +222,15 @@ def init_paddings(self): def create_test_cudnn_padding_VALID_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingVALIDCase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) def init_paddings(self): self.pad = [1, 1, 1] @@ -211,9 +242,7 @@ def init_paddings(self): def create_test_channel_last_class(parent): - class TestChannelLastCase(parent): - def init_data_format(self): self.data_format = "NDHWC" @@ -227,15 +256,15 @@ def init_test_case_2(self): def create_test_cudnn_channel_last_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCudnnChannelLastCase(parent): - def init_kernel_type(self): self.use_cudnn = True - self.dtype = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.dtype = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) def init_data_format(self): self.data_format = "NDHWC" @@ -250,7 +279,6 @@ def init_test_case_2(self): class TestConv3DOp(OpTest): - def setUp(self): self.op_type = "conv3d" self.use_cudnn = False @@ -265,7 +293,7 @@ def setUp(self): conv3d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilations': self.dilations + 'dilations': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) @@ -279,7 +307,7 @@ def setUp(self): self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -288,7 +316,7 @@ def setUp(self): 'dilations': self.dilations, 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, - 'data_format': self.data_format + 'data_format': self.data_format, } self.outputs = {'Output': output} @@ -298,41 +326,50 @@ def has_cudnn(self): def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace() - self.check_output_with_place(place, - atol=1e-5, - check_dygraph=(self.use_mkldnn == False)) + self.check_output_with_place( + place, atol=1e-5, check_dygraph=(self.use_mkldnn == False) + ) def test_check_grad(self): if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace() # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad_with_place(place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03, - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + {'Input', 'Filter'}, + 'Output', + max_relative_error=0.03, + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace() # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad_with_place(place, ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter']), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace() # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad_with_place(place, ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input']), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + check_dygraph=(self.use_mkldnn == False), + ) def init_test_case(self): self.pad = [0, 0, 0] @@ -356,7 +393,6 @@ def init_kernel_type(self): class TestCase1(TestConv3DOp): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -367,19 +403,16 @@ def init_test_case(self): class TestWithGroup1(TestConv3DOp): - def init_group(self): self.groups = 3 class TestWithGroup2(TestCase1): - def init_group(self): self.groups = 3 class TestWith1x1(TestConv3DOp): - def init_test_case(self): self.pad = [0, 0, 0] self.stride = [1, 1, 1] @@ -396,7 +429,6 @@ def init_group(self): class TestWithInput1x1Filter1x1(TestConv3DOp): - def init_test_case(self): self.pad = [0, 0, 0] self.stride = [1, 1, 1] @@ -413,7 +445,6 @@ def init_group(self): class TestWithDilation(TestConv3DOp): - def init_test_case(self): self.pad = [0, 0, 0] self.stride = [1, 1, 1] @@ -429,22 +460,22 @@ def init_group(self): self.groups = 3 -#---------------- Conv3DCUDNN ---------------- +# ---------------- Conv3DCUDNN ---------------- -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNN(TestConv3DOp): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16CUDNN(TestConv3DOp): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -456,19 +487,19 @@ def test_check_output(self): self.check_output_with_place(place, atol=2e-2) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestWithGroup1CUDNN(TestWithGroup1): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16WithGroup1CUDNN(TestWithGroup1): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -480,19 +511,19 @@ def test_check_output(self): self.check_output_with_place(place, atol=2e-2) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestWithGroup2CUDNN(TestWithGroup2): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16WithGroup2CUDNN(TestWithGroup2): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -504,19 +535,19 @@ def test_check_output(self): self.check_output_with_place(place, atol=2e-2) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestWith1x1CUDNN(TestWith1x1): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16With1x1CUDNN(TestWith1x1): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -528,19 +559,19 @@ def test_check_output(self): self.check_output_with_place(place, atol=2e-2) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestWithInput1x1Filter1x1CUDNN(TestWithInput1x1Filter1x1): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16WithInput1x1Filter1x1CUDNN(TestWithInput1x1Filter1x1): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -553,7 +584,6 @@ def test_check_output(self): class TestCUDNNExhaustiveSearch(TestCUDNN): - def init_kernel_type(self): self.use_cudnn = True self.exhaustive_search = True @@ -564,7 +594,6 @@ def init_kernel_type(self): class TestConv3DOp_2(OpTest): - def setUp(self): self.op_type = "conv3d" self.use_cudnn = False @@ -583,18 +612,23 @@ def setUp(self): conv3d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilations': self.dilations + 'dilations': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) filter = np.random.random(self.filter_size).astype(self.dtype) - output = conv3d_forward_naive(input, filter, self.groups, conv3d_param, - self.padding_algorithm, - self.data_format).astype(self.dtype) + output = conv3d_forward_naive( + input, + filter, + self.groups, + conv3d_param, + self.padding_algorithm, + self.data_format, + ).astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -604,7 +638,7 @@ def setUp(self): 'dilations': self.dilations, 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, - 'data_format': self.data_format + 'data_format': self.data_format, } self.outputs = {'Output': output} @@ -619,27 +653,33 @@ def test_check_grad(self): if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace() - self.check_grad_with_place(place, {'Input', 'Filter'}, - 'Output', - max_relative_error=0.03) + self.check_grad_with_place( + place, {'Input', 'Filter'}, 'Output', max_relative_error=0.03 + ) def test_check_grad_no_filter(self): if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace() - self.check_grad_with_place(place, ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + place, + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + ) def test_check_grad_no_input(self): if self.dtype == np.float16: return place = core.CUDAPlace(0) if self.has_cudnn() else core.CPUPlace() - self.check_grad_with_place(place, ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input'])) + self.check_grad_with_place( + place, + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + ) def init_test_case(self): self.stride = [1, 1, 1] @@ -669,7 +709,6 @@ def init_data_format(self): class TestConv3DOp_AsyPadding(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 2] self.input_size = [2, 3, 4, 4, 4] # NCDHW @@ -683,7 +722,6 @@ def init_paddings(self): class TestConv3DOp_DiffDataInDiffDim(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 2] self.input_size = [2, 3, 4, 5, 5] # NCDHW @@ -702,7 +740,6 @@ def init_paddings(self): class TestCase1_AsyPadding(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 1] self.input_size = [2, 3, 4, 4, 4] # NCDHW @@ -716,7 +753,6 @@ def init_paddings(self): class TestWithGroup1_AsyPadding(TestConv3DOp_2): - def init_group(self): self.groups = 3 @@ -726,7 +762,6 @@ def init_paddings(self): class TestWithGroup2_AsyPadding(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 1] self.input_size = [2, 3, 4, 4, 4] # NCDHW @@ -743,7 +778,6 @@ def init_paddings(self): class TestWith1x1_AsyPadding(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 1] self.input_size = [2, 3, 4, 4, 4] @@ -763,7 +797,6 @@ def init_paddings(self): class TestWithDilation_AsyPadding(TestConv3DOp_2): - def init_test_case(self): self.stride = [1, 1, 1] self.input_size = [2, 3, 6, 6, 6] @@ -829,196 +862,228 @@ def init_paddings(self): # --------- test python API --------------- class TestConv3DAPI(unittest.TestCase): - def test_api(self): - input_NDHWC = fluid.layers.data(name="input_NDHWC", - shape=[2, 5, 5, 5, 3], - append_batch_size=False, - dtype="float32") - - input_NCDHW = fluid.layers.data(name="input_NCDHW", - shape=[2, 3, 5, 5, 3], - append_batch_size=False, - dtype="float32") - - fluid.layers.conv3d(input=input_NDHWC, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=0, - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=[1, 2, 1, 0, 1, 0], - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=[[0, 0], [0, 0], [1, 1], [1, 1], [1, 1]], - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NDHWC, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=[[0, 0], [1, 1], [1, 1], [1, 1], [0, 0]], - dilation=[1, 1, 1], - groups=1, - data_format="NDHWC") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding="SAME", - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") - - fluid.layers.conv3d(input=input_NCDHW, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding="VALID", - dilation=[1, 1, 1], - groups=1, - data_format="NCDHW") + input_NDHWC = fluid.layers.data( + name="input_NDHWC", + shape=[2, 5, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + input_NCDHW = fluid.layers.data( + name="input_NCDHW", + shape=[2, 3, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + fluid.layers.conv3d( + input=input_NDHWC, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=0, + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=[1, 2, 1, 0, 1, 0], + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=[[0, 0], [0, 0], [1, 1], [1, 1], [1, 1]], + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NDHWC, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=[[0, 0], [1, 1], [1, 1], [1, 1], [0, 0]], + dilation=[1, 1, 1], + groups=1, + data_format="NDHWC", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding="SAME", + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) + + fluid.layers.conv3d( + input=input_NCDHW, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding="VALID", + dilation=[1, 1, 1], + groups=1, + data_format="NCDHW", + ) class TestConv3DAPI_Error(unittest.TestCase): - def test_api(self): - input = fluid.layers.data(name="input", - shape=[2, 5, 5, 5, 4], - append_batch_size=False, - dtype="float32") + input = fluid.layers.data( + name="input", + shape=[2, 5, 5, 5, 4], + append_batch_size=False, + dtype="float32", + ) # ValueError: cudnn def run_1(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding=0, - dilation=1, - groups=1, - use_cudnn=[0], - data_format="NCDHW") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding=0, + dilation=1, + groups=1, + use_cudnn=[0], + data_format="NCDHW", + ) self.assertRaises(ValueError, run_1) # ValueError: data_format def run_2(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=[3, 3, 3], - stride=[1, 1, 1], - padding=0, - dilation=[1, 1, 1], - groups=1, - use_cudnn=False, - data_format="NCHWC") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=[3, 3, 3], + stride=[1, 1, 1], + padding=0, + dilation=[1, 1, 1], + groups=1, + use_cudnn=False, + data_format="NCHWC", + ) self.assertRaises(ValueError, run_2) # ValueError: padding def run_3(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding="SAMEE", - dilation=1, - groups=1, - use_cudnn=False, - data_format="NCDHW") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding="SAMEE", + dilation=1, + groups=1, + use_cudnn=False, + data_format="NCDHW", + ) self.assertRaises(ValueError, run_3) def run_4(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding=[[0, 1], [0, 0], [0, 1], [0, 1], [0, - 1]], - dilation=1, - groups=1, - use_cudnn=False, - data_format="NCDHW") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding=[[0, 1], [0, 0], [0, 1], [0, 1], [0, 1]], + dilation=1, + groups=1, + use_cudnn=False, + data_format="NCDHW", + ) self.assertRaises(ValueError, run_4) def run_5(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=0, - stride=0, - padding=[[0, 1], [0, 1], [0, 1], [0, 1], [0, - 1]], - dilation=1, - groups=1, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=0, + stride=0, + padding=[[0, 1], [0, 1], [0, 1], [0, 1], [0, 1]], + dilation=1, + groups=1, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_5) # ValueError: channel dimmention - x = fluid.layers.data(name="x", - shape=[2, 5, 5, 5, -1], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name="x", + shape=[2, 5, 5, 5, -1], + append_batch_size=False, + dtype="float32", + ) def run_6(): - fluid.layers.conv3d(input=x, - num_filters=3, - filter_size=3, - stride=1, - padding=0, - dilation=1, - groups=1, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=x, + num_filters=3, + filter_size=3, + stride=1, + padding=0, + dilation=1, + groups=1, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_6) # ValueError: groups def run_7(): - fluid.layers.conv3d(input=input, - num_filters=3, - filter_size=3, - stride=1, - padding=0, - dilation=1, - groups=3, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=input, + num_filters=3, + filter_size=3, + stride=1, + padding=0, + dilation=1, + groups=3, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_7) # ValueError: filter num def run_8(): - fluid.layers.conv3d(input=input, - num_filters=0, - filter_size=0, - stride=0, - padding=0, - dilation=0, - groups=1, - use_cudnn=False, - data_format="NDHWC") + fluid.layers.conv3d( + input=input, + num_filters=0, + filter_size=0, + stride=0, + padding=0, + dilation=0, + groups=1, + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_8) diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_layer.py b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_layer.py index 9ad3eaaccfcf2..fa2138956b949 100644 --- a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_layer.py +++ b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_layer.py @@ -21,22 +21,23 @@ class Conv3DTransposeTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - batch_size=2, - spartial_shape=(8, 8, 8), - num_channels=6, - num_filters=8, - filter_size=3, - output_size=None, - padding=0, - stride=1, - dilation=1, - groups=1, - no_bias=False, - data_format="NCDHW", - dtype="float32"): + def __init__( + self, + methodName='runTest', + batch_size=2, + spartial_shape=(8, 8, 8), + num_channels=6, + num_filters=8, + filter_size=3, + output_size=None, + padding=0, + stride=1, + dilation=1, + groups=1, + no_bias=False, + data_format="NCDHW", + dtype="float32", + ): super(Conv3DTransposeTestCase, self).__init__(methodName) self.batch_size = batch_size self.num_channels = num_channels @@ -56,34 +57,44 @@ def __init__(self, def setUp(self): self.channel_last = self.data_format == "NDHWC" if self.channel_last: - input_shape = (self.batch_size, ) + self.spartial_shape + ( - self.num_channels, ) + input_shape = ( + (self.batch_size,) + self.spartial_shape + (self.num_channels,) + ) else: - input_shape = (self.batch_size, - self.num_channels) + self.spartial_shape + input_shape = ( + self.batch_size, + self.num_channels, + ) + self.spartial_shape self.input = np.random.randn(*input_shape).astype(self.dtype) if isinstance(self.filter_size, int): filter_size = [self.filter_size] * 3 else: filter_size = self.filter_size - self.weight_shape = weight_shape = (self.num_channels, self.num_filters - // self.groups) + tuple(filter_size) - self.weight = np.random.uniform(-1, 1, - size=weight_shape).astype(self.dtype) + self.weight_shape = weight_shape = ( + self.num_channels, + self.num_filters // self.groups, + ) + tuple(filter_size) + self.weight = np.random.uniform(-1, 1, size=weight_shape).astype( + self.dtype + ) if self.no_bias: self.bias = None else: self.bias = np.random.uniform( - -1, 1, size=(self.num_filters, )).astype(self.dtype) + -1, 1, size=(self.num_filters,) + ).astype(self.dtype) def fluid_layer(self, place): main = fluid.Program() start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1, -1, self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1, -1) + input_shape = ( + (-1, -1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) weight_attr = I.NumpyArrayInitializer(self.weight) if self.bias is None: @@ -101,11 +112,12 @@ def fluid_layer(self, place): groups=self.groups, param_attr=weight_attr, bias_attr=bias_attr, - data_format=self.data_format) + data_format=self.data_format, + ) feed_dict = {"input": self.input} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def functional(self, place): @@ -113,41 +125,49 @@ def functional(self, place): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - input_shape = (-1, -1, -1, -1, self.num_channels) \ - if self.channel_last else (-1, self.num_channels, -1, -1, -1) + input_shape = ( + (-1, -1, -1, -1, self.num_channels) + if self.channel_last + else (-1, self.num_channels, -1, -1, -1) + ) x_var = fluid.data("input", input_shape, dtype=self.dtype) - w_var = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) - b_var = fluid.data("bias", (self.num_filters, ), - dtype=self.dtype) - y_var = F.conv3d_transpose(x_var, - w_var, - None if self.no_bias else b_var, - output_size=self.output_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + w_var = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) + b_var = fluid.data( + "bias", (self.num_filters,), dtype=self.dtype + ) + y_var = F.conv3d_transpose( + x_var, + w_var, + None if self.no_bias else b_var, + output_size=self.output_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) feed_dict = {"input": self.input, "weight": self.weight} if self.bias is not None: feed_dict["bias"] = self.bias exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_nn_layer(self): x_var = dg.to_variable(self.input) - conv = nn.Conv3DTranspose(self.num_channels, - self.num_filters, - self.filter_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + conv = nn.Conv3DTranspose( + self.num_channels, + self.num_filters, + self.filter_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) conv.weight.set_value(self.weight) if not self.no_bias: conv.bias.set_value(self.bias) @@ -174,7 +194,6 @@ def runTest(self): class Conv3DTransposeErrorTestCase(Conv3DTransposeTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -185,65 +204,85 @@ def runTest(self): def add_cases(suite): suite.addTest(Conv3DTransposeTestCase(methodName='runTest')) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - stride=[1, 2, 1], - dilation=2, - no_bias=True)) + Conv3DTransposeTestCase( + methodName='runTest', stride=[1, 2, 1], dilation=2, no_bias=True + ) + ) + suite.addTest( + Conv3DTransposeTestCase( + methodName='runTest', + output_size=[12, 19, 12], + stride=[1, 2, 1], + dilation=2, + ) + ) + suite.addTest( + Conv3DTransposeTestCase( + methodName='runTest', stride=2, dilation=(2, 1, 2) + ) + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - output_size=[12, 19, 12], - stride=[1, 2, 1], - dilation=2)) + Conv3DTransposeTestCase(methodName='runTest', padding="valid") + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - stride=2, - dilation=(2, 1, 2))) - suite.addTest(Conv3DTransposeTestCase(methodName='runTest', - padding="valid")) - suite.addTest(Conv3DTransposeTestCase(methodName='runTest', - padding='valid')) + Conv3DTransposeTestCase(methodName='runTest', padding='valid') + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - filter_size=1, - padding=(2, 3, 1))) + Conv3DTransposeTestCase( + methodName='runTest', filter_size=1, padding=(2, 3, 1) + ) + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - padding=[1, 2, 2, 3, 2, 1])) + Conv3DTransposeTestCase( + methodName='runTest', padding=[1, 2, 2, 3, 2, 1] + ) + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - padding=[[0, 0], [0, 0], [2, 3], [1, 2], [2, - 1]])) + Conv3DTransposeTestCase( + methodName='runTest', + padding=[[0, 0], [0, 0], [2, 3], [1, 2], [2, 1]], + ) + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', data_format="NDHWC")) + Conv3DTransposeTestCase(methodName='runTest', data_format="NDHWC") + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - data_format="NDHWC", - padding=[[0, 0], [1, 1], [2, 2], [3, 3], [0, - 0]])) + Conv3DTransposeTestCase( + methodName='runTest', + data_format="NDHWC", + padding=[[0, 0], [1, 1], [2, 2], [3, 3], [0, 0]], + ) + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', groups=2, - padding="valid")) + Conv3DTransposeTestCase(methodName='runTest', groups=2, padding="valid") + ) suite.addTest( - Conv3DTransposeTestCase(methodName='runTest', - num_filters=6, - num_channels=3, - groups=3, - padding="valid")) + Conv3DTransposeTestCase( + methodName='runTest', + num_filters=6, + num_channels=3, + groups=3, + padding="valid", + ) + ) def add_error_cases(suite): suite.addTest( - Conv3DTransposeErrorTestCase(methodName='runTest', - num_channels=5, - groups=2)) + Conv3DTransposeErrorTestCase( + methodName='runTest', num_channels=5, groups=2 + ) + ) suite.addTest( - Conv3DTransposeErrorTestCase(methodName='runTest', - output_size="not_valid")) + Conv3DTransposeErrorTestCase( + methodName='runTest', output_size="not_valid" + ) + ) suite.addTest( - Conv3DTransposeErrorTestCase(methodName='runTest', - num_channels=5, - groups=2, - padding=[-1, 1, 3])) + Conv3DTransposeErrorTestCase( + methodName='runTest', num_channels=5, groups=2, padding=[-1, 1, 3] + ) + ) def load_tests(loader, standard_tests, pattern): diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py index ccda74b498ab3..869d1d5f29a13 100644 --- a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py @@ -25,9 +25,10 @@ def conv3dtranspose_forward_naive(input_, filter_, attrs): padding_algorithm = attrs['padding_algorithm'] if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if attrs['data_format'] == 'NHWC': input_ = np.transpose(input_, [0, 4, 1, 2, 3]) @@ -38,17 +39,21 @@ def conv3dtranspose_forward_naive(input_, filter_, attrs): out_c = f_out_c * groups sub_in_c = in_c // groups - stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ - 'dilations'] + stride, pad, dilations = ( + attrs['strides'], + attrs['paddings'], + attrs['dilations'], + ) def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, - kernel_size, - kernel_stride): + for input_size, filter_size, stride_size in zip( + input_shape, kernel_size, kernel_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -84,34 +89,50 @@ def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride): for i in range(in_h): for j in range(in_w): for g in range(groups): - input_masked = input_[n, - g * sub_in_c:(g + 1) * sub_in_c, - d, i, j] # (c) - input_masked = np.reshape(input_masked, - (sub_in_c, 1, 1, 1)) + input_masked = input_[ + n, g * sub_in_c : (g + 1) * sub_in_c, d, i, j + ] # (c) + input_masked = np.reshape( + input_masked, (sub_in_c, 1, 1, 1) + ) input_masked = np.tile(input_masked, (1, f_d, f_h, f_w)) for k in range(f_out_c): - tmp_out = np.sum(input_masked * - filter_[g * sub_in_c:(g + 1) * - sub_in_c, k, :, :, :], - axis=0) + tmp_out = np.sum( + input_masked + * filter_[ + g * sub_in_c : (g + 1) * sub_in_c, + k, + :, + :, + :, + ], + axis=0, + ) d1, d2 = d * stride[0], d * stride[0] + d_bolck_d i1, i2 = i * stride[1], i * stride[1] + d_bolck_h j1, j2 = j * stride[2], j * stride[2] + d_bolck_w - out[n, g * f_out_c + k, d1:d2:dilations[0], - i1:i2:dilations[1], - j1:j2:dilations[2]] += tmp_out - - out = out[:, :, pad_d_0:out_d - pad_d_1, pad_h_0:out_h - pad_h_1, - pad_w_0:out_w - pad_w_1] + out[ + n, + g * f_out_c + k, + d1 : d2 : dilations[0], + i1 : i2 : dilations[1], + j1 : j2 : dilations[2], + ] += tmp_out + + out = out[ + :, + :, + pad_d_0 : out_d - pad_d_1, + pad_h_0 : out_h - pad_h_1, + pad_w_0 : out_w - pad_w_1, + ] if attrs['data_format'] == 'NHWC': out = np.transpose(out, [0, 2, 3, 4, 1]) return out class TestConv3DTransposeOp(OpTest): - def setUp(self): # init as conv transpose self.use_cudnn = False @@ -134,11 +155,12 @@ def setUp(self): 'dilations': self.dilations, 'groups': self.groups, 'use_cudnn': self.use_cudnn, - 'data_format': self.data_format + 'data_format': self.data_format, } - output = conv3dtranspose_forward_naive(input_, filter_, - self.attrs).astype("float32") + output = conv3dtranspose_forward_naive( + input_, filter_, self.attrs + ).astype("float32") self.outputs = {'Output': output} @@ -152,40 +174,52 @@ def test_check_output(self): def test_check_grad(self): if self.use_cudnn: place = core.CUDAPlace(0) - self.check_grad_with_place(place, - set(['Input', 'Filter']), - 'Output', - max_relative_error=0.03) + self.check_grad_with_place( + place, + set(['Input', 'Filter']), + 'Output', + max_relative_error=0.03, + ) else: - self.check_grad(set(['Input', 'Filter']), - 'Output', - max_relative_error=0.03) + self.check_grad( + set(['Input', 'Filter']), 'Output', max_relative_error=0.03 + ) def test_check_grad_no_filter(self): if self.use_cudnn: place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + place, + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + ) elif self.check_no_filter: - self.check_grad(['Input'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Filter'])) + self.check_grad( + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter']), + ) def test_check_grad_no_input(self): if self.use_cudnn: place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input'])) + self.check_grad_with_place( + place, + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + ) elif self.check_no_input: - self.check_grad(['Filter'], - 'Output', - max_relative_error=0.03, - no_grad_set=set(['Input'])) + self.check_grad( + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input']), + ) def init_test_case(self): self.pad = [0, 0, 0] @@ -201,7 +235,6 @@ def init_op_type(self): class TestWithSymmetricPad(TestConv3DTransposeOp): - def init_test_case(self): self.check_no_input = True self.pad = [1, 1, 1] @@ -214,7 +247,6 @@ def init_test_case(self): class TestWithAsymmetricPad(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 0, 1, 2] self.stride = [1, 1, 1] @@ -226,7 +258,6 @@ def init_test_case(self): class TestWithSAMEPad(TestConv3DTransposeOp): - def init_test_case(self): self.stride = [1, 1, 2] self.dilations = [1, 2, 1] @@ -238,7 +269,6 @@ def init_test_case(self): class TestWithVALIDPad(TestConv3DTransposeOp): - def init_test_case(self): self.stride = [2, 1, 1] self.dilations = [1, 1, 1] @@ -250,7 +280,6 @@ def init_test_case(self): class TestWithStride(TestConv3DTransposeOp): - def init_test_case(self): self.check_no_filter = True self.pad = [1, 1, 1] @@ -263,7 +292,6 @@ def init_test_case(self): class TestWithGroups(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -275,7 +303,6 @@ def init_test_case(self): class TestWithDilation(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -287,7 +314,6 @@ def init_test_case(self): class Test_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [0, 0, 0] self.stride = [1, 1, 1] @@ -300,19 +326,19 @@ def init_test_case(self): # ------------ test_cudnn ------------ -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNN(TestConv3DTransposeOp): - def init_op_type(self): self.use_cudnn = True self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithSymmetricPad(TestWithSymmetricPad): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -327,10 +353,10 @@ def init_op_type(self): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithAsymmetricPad(TestWithAsymmetricPad): - def init_test_case(self): self.pad = [1, 1, 1, 0, 0, 2] self.stride = [1, 1, 1] @@ -345,10 +371,10 @@ def init_op_type(self): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithSAMEPad(TestWithSAMEPad): - def init_test_case(self): self.stride = [1, 1, 2] self.dilations = [1, 2, 1] @@ -363,10 +389,10 @@ def init_op_type(self): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithVALIDPad(TestWithVALIDPad): - def init_test_case(self): self.stride = [1, 1, 1] self.dilations = [1, 1, 1] @@ -381,10 +407,10 @@ def init_op_type(self): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithStride(TestWithStride): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [2, 2, 2] @@ -399,10 +425,10 @@ def init_op_type(self): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithGroups(TestWithGroups): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -432,10 +458,10 @@ def init_op_type(self): # self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNN_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [0, 0, 0] self.stride = [1, 1, 1] @@ -451,10 +477,10 @@ def init_op_type(self): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithSymmetricPad_NHWC(TestWithSymmetricPad): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -470,10 +496,10 @@ def init_op_type(self): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithAsymmetricPad_NHWC(TestWithAsymmetricPad): - def init_test_case(self): self.pad = [1, 0, 1, 0, 0, 2] self.stride = [1, 1, 1] @@ -489,10 +515,10 @@ def init_op_type(self): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithStride_NHWC(TestWithStride): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [2, 2, 2] @@ -508,10 +534,10 @@ def init_op_type(self): self.op_type = "conv3d_transpose" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNWithGroups_NHWC(TestWithGroups): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_part2_op.py b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_part2_op.py index 12d2137821599..0aed6c1c515be 100644 --- a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_part2_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_part2_op.py @@ -21,7 +21,6 @@ class TestWithSymmetricPad_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [1, 1, 1] @@ -34,7 +33,6 @@ def init_test_case(self): class TestWithAsymmetricPad_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 0, 1, 2] self.stride = [1, 1, 1] @@ -47,7 +45,6 @@ def init_test_case(self): class TestWithGroups_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.check_no_filter = True self.pad = [1, 1, 1] @@ -61,7 +58,6 @@ def init_test_case(self): class TestWithStride_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.pad = [1, 1, 1] self.stride = [2, 2, 2] @@ -74,7 +70,6 @@ def init_test_case(self): class TestWithDilation_NHWC(TestConv3DTransposeOp): - def init_test_case(self): self.check_no_input = True self.pad = [1, 1, 1] @@ -88,57 +83,68 @@ def init_test_case(self): class TestConv3DTransposeAPI(unittest.TestCase): - def test_case1(self): - data1 = fluid.layers.data(name='data1', - shape=[3, 5, 5, 5], - dtype='float32') - data2 = fluid.layers.data(name='data2', - shape=[5, 5, 5, 3], - dtype='float32') - - out1 = fluid.layers.conv3d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - data_format='NCDHW') - out2 = fluid.layers.conv3d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - data_format='NDHWC') - out3 = fluid.layers.conv3d_transpose(input=data1, - groups=1, - num_filters=6, - filter_size=3, - padding=[[0, 0], [0, 0], [1, 1], - [0, 0], [1, 1]], - data_format='NCDHW') - out4 = fluid.layers.conv3d_transpose(input=data2, - groups=3, - num_filters=6, - filter_size=3, - padding=[[0, 0], [0, 0], [1, 1], - [1, 2], [0, 0]], - data_format='NDHWC') - out5 = fluid.layers.conv3d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - padding='SAME', - data_format='NCDHW') - out6 = fluid.layers.conv3d_transpose(input=data2, - groups=1, - num_filters=6, - filter_size=3, - padding='VALID', - data_format='NDHWC') - out7 = fluid.layers.conv3d_transpose(input=data2, - groups=1, - num_filters=6, - output_size=[7, 7, 7], - padding=[0, 0, 0], - data_format='NDHWC') + data1 = fluid.layers.data( + name='data1', shape=[3, 5, 5, 5], dtype='float32' + ) + data2 = fluid.layers.data( + name='data2', shape=[5, 5, 5, 3], dtype='float32' + ) + + out1 = fluid.layers.conv3d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + data_format='NCDHW', + ) + out2 = fluid.layers.conv3d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + data_format='NDHWC', + ) + out3 = fluid.layers.conv3d_transpose( + input=data1, + groups=1, + num_filters=6, + filter_size=3, + padding=[[0, 0], [0, 0], [1, 1], [0, 0], [1, 1]], + data_format='NCDHW', + ) + out4 = fluid.layers.conv3d_transpose( + input=data2, + groups=3, + num_filters=6, + filter_size=3, + padding=[[0, 0], [0, 0], [1, 1], [1, 2], [0, 0]], + data_format='NDHWC', + ) + out5 = fluid.layers.conv3d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + padding='SAME', + data_format='NCDHW', + ) + out6 = fluid.layers.conv3d_transpose( + input=data2, + groups=1, + num_filters=6, + filter_size=3, + padding='VALID', + data_format='NDHWC', + ) + out7 = fluid.layers.conv3d_transpose( + input=data2, + groups=1, + num_filters=6, + output_size=[7, 7, 7], + padding=[0, 0, 0], + data_format='NDHWC', + ) data1_np = np.random.random((2, 3, 5, 5, 5)).astype("float32") data2_np = np.random.random((2, 5, 5, 5, 3)).astype("float32") @@ -149,13 +155,12 @@ def test_case1(self): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "data1": data1_np, - "data2": data2_np - }, - fetch_list=[out1, out2, out3, out4, out5, out6, out7], - return_numpy=True) + results = exe.run( + fluid.default_main_program(), + feed={"data1": data1_np, "data2": data2_np}, + fetch_list=[out1, out2, out3, out4, out5, out6, out7], + return_numpy=True, + ) self.assertIsNotNone(results[0]) self.assertIsNotNone(results[1]) self.assertIsNotNone(results[2]) @@ -166,48 +171,53 @@ def test_case1(self): class TestConv3DTransposeOpException(unittest.TestCase): - def test_exception(self): - data = fluid.layers.data(name='data', - shape=[3, 5, 5, 5], - dtype="float32") + data = fluid.layers.data( + name='data', shape=[3, 5, 5, 5], dtype="float32" + ) def attr_data_format(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - data_format="NCDW") + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + data_format="NCDW", + ) self.assertRaises(ValueError, attr_data_format) def attr_padding_str(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding='Vald') + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding='Vald', + ) self.assertRaises(ValueError, attr_padding_str) def attr_padding_list(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding=[[1, 1], [1, 1], [0, 0], - [0, 0], [1, 1]]) + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding=[[1, 1], [1, 1], [0, 0], [0, 0], [1, 1]], + ) self.assertRaises(ValueError, attr_padding_list) def attr_padding_with_data_format(): - out = fluid.layers.conv2d_transpose(input=data, - groups=1, - num_filters=6, - filter_size=3, - padding=[[1, 1], [0, 0], [0, 0], - [1, 0], [1, 1]], - data_format='NDHWC') + out = fluid.layers.conv2d_transpose( + input=data, + groups=1, + num_filters=6, + filter_size=3, + padding=[[1, 1], [0, 0], [0, 0], [1, 0], [1, 1]], + data_format='NDHWC', + ) self.assertRaises(ValueError, attr_padding_with_data_format) diff --git a/python/paddle/fluid/tests/unittests/test_conv_nn_grad.py b/python/paddle/fluid/tests/unittests/test_conv_nn_grad.py index 99d457c073e37..b09a86f5bfa0b 100644 --- a/python/paddle/fluid/tests/unittests/test_conv_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_conv_nn_grad.py @@ -25,7 +25,6 @@ class TestConvDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 4, 3, 3] @@ -39,11 +38,9 @@ def func(self, place): w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -55,7 +52,6 @@ def test_grad(self): class TestConvDoubleGradCheckTest0(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 4, 3, 3] @@ -69,11 +65,9 @@ def func(self, place): w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -84,7 +78,6 @@ def test_grad(self): class TestConvDoubleGradCheckTest1(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 3, 3, 3] @@ -98,11 +91,9 @@ def func(self, place): w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -113,7 +104,6 @@ def test_grad(self): class TestConv3DDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 4, 3, 4, 2] @@ -127,14 +117,12 @@ def func(self, place): w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): - #places = [fluid.CPUPlace()] + # places = [fluid.CPUPlace()] places = [] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -143,7 +131,6 @@ def test_grad(self): class TestConv3DDoubleGradCheckTest1(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 4, 5, 3, 2] @@ -157,11 +144,9 @@ def func(self, place): w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -172,30 +157,29 @@ def test_grad(self): class TestConv2DoubleGradCheck_AsyPadding(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 3, 3] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv2d(input=x, - num_filters=2, - filter_size=1, - padding=[1, 0, 0, 1], - bias_attr=False, - use_cudnn=True) + y = layers.conv2d( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 0, 0, 1], + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -206,30 +190,29 @@ def test_grad(self): class TestConv2DoubleGradCheck_PaddingSAME(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 3, 3] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv2d(input=x, - num_filters=2, - filter_size=1, - padding="SAME", - bias_attr=False, - use_cudnn=True) + y = layers.conv2d( + input=x, + num_filters=2, + filter_size=1, + padding="SAME", + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -240,30 +223,29 @@ def test_grad(self): class TestConv2DoubleGradCheck_PaddingVALID(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 3, 3] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv2d(input=x, - num_filters=2, - filter_size=1, - padding="VALID", - bias_attr=False, - use_cudnn=True) + y = layers.conv2d( + input=x, + num_filters=2, + filter_size=1, + padding="VALID", + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -274,32 +256,31 @@ def test_grad(self): class TestConv2DoubleGradCheck_ChannelLast(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 3, 3] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv2d(input=x, - num_filters=2, - filter_size=1, - padding=[1, 1], - bias_attr=False, - use_cudnn=True, - groups=1, - data_format="NHWC") + y = layers.conv2d( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 1], + bias_attr=False, + use_cudnn=True, + groups=1, + data_format="NHWC", + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -310,32 +291,31 @@ def test_grad(self): class TestConv2DoubleGradCheck_ChannelLast_AsyPadding(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 3, 3] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv2d(input=x, - num_filters=2, - filter_size=1, - padding=[1, 0, 1, 0], - bias_attr=False, - use_cudnn=True, - groups=1, - data_format="NHWC") + y = layers.conv2d( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 0, 1, 0], + bias_attr=False, + use_cudnn=True, + groups=1, + data_format="NHWC", + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -346,30 +326,29 @@ def test_grad(self): class TestConv3DDoubleGradCheck_AsyPadding(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 2, 2, 2] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv3d(input=x, - num_filters=2, - filter_size=1, - padding=[1, 0, 0, 1, 1, 2], - bias_attr=False, - use_cudnn=True) + y = layers.conv3d( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 0, 0, 1, 1, 2], + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -380,31 +359,30 @@ def test_grad(self): class TestConv3DoubleGradCheck_PaddingSAME(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 2, 2, 2] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv3d(input=x, - num_filters=2, - filter_size=1, - padding="SAME", - groups=1, - bias_attr=False, - use_cudnn=True) + y = layers.conv3d( + input=x, + num_filters=2, + filter_size=1, + padding="SAME", + groups=1, + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -415,30 +393,29 @@ def test_grad(self): class TestConv3DoubleGradCheck_PaddingVALID(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 3, 3, 2] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv3d(input=x, - num_filters=2, - filter_size=1, - padding="VALID", - bias_attr=False, - use_cudnn=True) + y = layers.conv3d( + input=x, + num_filters=2, + filter_size=1, + padding="VALID", + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -449,32 +426,31 @@ def test_grad(self): class TestConv3DDoubleGradCheck_ChannelLast(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 2, 2, 3] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv3d(input=x, - num_filters=2, - filter_size=1, - padding=[1, 1, 1], - bias_attr=False, - use_cudnn=True, - groups=1, - data_format="NDHWC") + y = layers.conv3d( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 1, 1], + bias_attr=False, + use_cudnn=True, + groups=1, + data_format="NDHWC", + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -485,32 +461,31 @@ def test_grad(self): class TestConv3DDoubleGradCheck_ChannelLast_AsyPadding(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 2, 2, 2, 3] eps = 0.005 dtype = np.float32 if fluid.core.is_compiled_with_rocm() else np.float64 x = layers.data('x', shape, False, dtype) - y = layers.conv3d(input=x, - num_filters=2, - filter_size=1, - padding=[1, 0, 1, 0, 1, 0], - bias_attr=False, - use_cudnn=True, - groups=1, - data_format="NDHWC") + y = layers.conv3d( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 0, 1, 0, 1, 0], + bias_attr=False, + use_cudnn=True, + groups=1, + data_format="NDHWC", + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -521,7 +496,6 @@ def test_grad(self): class TestDepthWiseConvDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): shape = [2, 4, 3, 3] @@ -533,23 +507,18 @@ def func(self, place): # use_cudnn == False # groups == filters # num_filters % num_channels == 0 - y = layers.conv2d(x, - shape[1], - 1, - groups=shape[1], - bias_attr=False, - use_cudnn=False) + y = layers.conv2d( + x, shape[1], 1, groups=shape[1], bias_attr=False, use_cudnn=False + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() w_arr = [] for p in w: w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) def test_grad(self): places = [] @@ -560,7 +529,6 @@ def test_grad(self): class TestDepthWiseConvDoubleGradCheckCase1(unittest.TestCase): - def depthwise_conv2d_wrapper(self, x): return paddle.nn.functional.conv2d(x[0], x[1], groups=4) @@ -582,16 +550,16 @@ def func(self, place): x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) w_arr = np.random.uniform(-1, 1, w_shape).astype(dtype) - gradient_checker.double_grad_check([x, w], - y, - x_init=[x_arr, w_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, w], y, x_init=[x_arr, w_arr], place=place, eps=eps + ) gradient_checker.double_grad_check_for_dygraph( - self.depthwise_conv2d_wrapper, [x, w], + self.depthwise_conv2d_wrapper, + [x, w], y, x_init=[x_arr, w_arr], - place=place) + place=place, + ) def test_grad(self): places = [] @@ -602,7 +570,6 @@ def test_grad(self): class TestConv3DDoubleGradCheck_NN(unittest.TestCase): - def conv3d_wrapper(self, x): return paddle.nn.functional.conv3d(x[0], x[1]) @@ -620,16 +587,12 @@ def func(self, place): x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) w_arr = np.random.uniform(-1, 1, w_shape).astype(dtype) - gradient_checker.double_grad_check([x, w], - y, - x_init=[x_arr, w_arr], - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.conv3d_wrapper, - [x, w], - y, - x_init=[x_arr, w_arr], - place=place) + gradient_checker.double_grad_check( + [x, w], y, x_init=[x_arr, w_arr], place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.conv3d_wrapper, [x, w], y, x_init=[x_arr, w_arr], place=place + ) def test_grad(self): places = [] diff --git a/python/paddle/fluid/tests/unittests/test_conv_shift_op.py b/python/paddle/fluid/tests/unittests/test_conv_shift_op.py index aa9d612c5db09..57a2038d3f59b 100644 --- a/python/paddle/fluid/tests/unittests/test_conv_shift_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv_shift_op.py @@ -29,7 +29,6 @@ def conv_shift_forward(x, y): class TestConvShiftOp(OpTest): - def setUp(self): self.op_type = "conv_shift" diff --git a/python/paddle/fluid/tests/unittests/test_conv_transpose_nn_grad.py b/python/paddle/fluid/tests/unittests/test_conv_transpose_nn_grad.py index 235f4a64f858e..f4c139a5463ee 100644 --- a/python/paddle/fluid/tests/unittests/test_conv_transpose_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_conv_transpose_nn_grad.py @@ -25,7 +25,6 @@ class TestConvTransposeDoubleGradCheck(unittest.TestCase): - def conv_transpose_wrapper(self, x): return paddle.nn.functional.conv2d_transpose(x[0], x[1], groups=1) @@ -37,11 +36,9 @@ def func(self, place): if core.is_compiled_with_rocm(): dtype = np.float32 x = layers.data('x', shape, False, dtype) - y = layers.conv2d_transpose(x, - 2, - filter_size=1, - groups=1, - bias_attr=False) + y = layers.conv2d_transpose( + x, 2, filter_size=1, groups=1, bias_attr=False + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() @@ -50,23 +47,25 @@ def func(self, place): w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) if core.is_compiled_with_rocm(): # HIP will sometimes fail if no atol - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps, - atol=1e-4) + gradient_checker.double_grad_check( + [x] + w, + y, + x_init=[x_arr] + w_arr, + place=place, + eps=eps, + atol=1e-4, + ) else: - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) gradient_checker.double_grad_check_for_dygraph( - self.conv_transpose_wrapper, [x] + w, + self.conv_transpose_wrapper, + [x] + w, y, x_init=[x_arr] + w_arr, - place=place) + place=place, + ) def test_grad(self): places = [] @@ -78,13 +77,12 @@ def test_grad(self): class TestConvTranspose2DoubleGradCheck_AsyPadding( - TestConvTransposeDoubleGradCheck): - + TestConvTransposeDoubleGradCheck +): def conv_transpose_wrapper(self, x): - return paddle.nn.functional.conv2d_transpose(x[0], - x[1], - groups=1, - padding=[1, 0, 0, 1]) + return paddle.nn.functional.conv2d_transpose( + x[0], x[1], groups=1, padding=[1, 0, 0, 1] + ) @prog_scope() def func(self, place): @@ -94,12 +92,14 @@ def func(self, place): if core.is_compiled_with_rocm(): dtype = np.float32 x = layers.data('x', shape, False, dtype) - y = layers.conv2d_transpose(input=x, - num_filters=2, - filter_size=1, - padding=[1, 0, 0, 1], - bias_attr=False, - use_cudnn=True) + y = layers.conv2d_transpose( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 0, 0, 1], + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() @@ -108,33 +108,34 @@ def func(self, place): w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) if core.is_compiled_with_rocm(): # HIP will sometimes fail if no atol - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps, - atol=1e-4) + gradient_checker.double_grad_check( + [x] + w, + y, + x_init=[x_arr] + w_arr, + place=place, + eps=eps, + atol=1e-4, + ) else: - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) gradient_checker.double_grad_check_for_dygraph( - self.conv_transpose_wrapper, [x] + w, + self.conv_transpose_wrapper, + [x] + w, y, x_init=[x_arr] + w_arr, - place=place) + place=place, + ) class TestConvTranspose2DoubleGradCheck_PaddingSAME( - TestConvTransposeDoubleGradCheck): - + TestConvTransposeDoubleGradCheck +): def conv_transpose_wrapper(self, x): - return paddle.nn.functional.conv2d_transpose(x[0], - x[1], - groups=1, - padding="SAME") + return paddle.nn.functional.conv2d_transpose( + x[0], x[1], groups=1, padding="SAME" + ) @prog_scope() def func(self, place): @@ -144,12 +145,14 @@ def func(self, place): if core.is_compiled_with_rocm(): dtype = np.float32 x = layers.data('x', shape, False, dtype) - y = layers.conv2d_transpose(input=x, - num_filters=2, - filter_size=1, - padding="SAME", - bias_attr=False, - use_cudnn=True) + y = layers.conv2d_transpose( + input=x, + num_filters=2, + filter_size=1, + padding="SAME", + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() @@ -158,33 +161,34 @@ def func(self, place): w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) if core.is_compiled_with_rocm(): # HIP will sometimes fail if no atol - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps, - atol=1e-4) + gradient_checker.double_grad_check( + [x] + w, + y, + x_init=[x_arr] + w_arr, + place=place, + eps=eps, + atol=1e-4, + ) else: - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) gradient_checker.double_grad_check_for_dygraph( - self.conv_transpose_wrapper, [x] + w, + self.conv_transpose_wrapper, + [x] + w, y, x_init=[x_arr] + w_arr, - place=place) + place=place, + ) class TestConvTranspose2DoubleGradCheck_PaddingVALID( - TestConvTransposeDoubleGradCheck): - + TestConvTransposeDoubleGradCheck +): def conv_transpose_wrapper(self, x): - return paddle.nn.functional.conv2d_transpose(x[0], - x[1], - groups=1, - padding="VALID") + return paddle.nn.functional.conv2d_transpose( + x[0], x[1], groups=1, padding="VALID" + ) @prog_scope() def func(self, place): @@ -194,12 +198,14 @@ def func(self, place): if core.is_compiled_with_rocm(): dtype = np.float32 x = layers.data('x', shape, False, dtype) - y = layers.conv2d_transpose(input=x, - num_filters=2, - filter_size=1, - padding="VALID", - bias_attr=False, - use_cudnn=True) + y = layers.conv2d_transpose( + input=x, + num_filters=2, + filter_size=1, + padding="VALID", + bias_attr=False, + use_cudnn=True, + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() @@ -208,34 +214,34 @@ def func(self, place): w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) if core.is_compiled_with_rocm(): # HIP will sometimes fail if no atol - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps, - atol=1e-4) + gradient_checker.double_grad_check( + [x] + w, + y, + x_init=[x_arr] + w_arr, + place=place, + eps=eps, + atol=1e-4, + ) else: - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) gradient_checker.double_grad_check_for_dygraph( - self.conv_transpose_wrapper, [x] + w, + self.conv_transpose_wrapper, + [x] + w, y, x_init=[x_arr] + w_arr, - place=place) + place=place, + ) class TestConvTranspose2DoubleGradCheck_ChannelLast( - TestConvTransposeDoubleGradCheck): - + TestConvTransposeDoubleGradCheck +): def conv_transpose_wrapper(self, x): - return paddle.nn.functional.conv2d_transpose(x[0], - x[1], - groups=1, - padding=[1, 1], - data_format="NHWC") + return paddle.nn.functional.conv2d_transpose( + x[0], x[1], groups=1, padding=[1, 1], data_format="NHWC" + ) @prog_scope() def func(self, place): @@ -245,14 +251,16 @@ def func(self, place): if core.is_compiled_with_rocm(): dtype = np.float32 x = layers.data('x', shape, False, dtype) - y = layers.conv2d_transpose(input=x, - num_filters=2, - filter_size=1, - padding=[1, 1], - bias_attr=False, - use_cudnn=True, - groups=1, - data_format="NHWC") + y = layers.conv2d_transpose( + input=x, + num_filters=2, + filter_size=1, + padding=[1, 1], + bias_attr=False, + use_cudnn=True, + groups=1, + data_format="NHWC", + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) w = fluid.default_main_program().global_block().all_parameters() @@ -261,23 +269,25 @@ def func(self, place): w_arr.append(np.random.uniform(-1, 1, p.shape).astype(dtype)) if core.is_compiled_with_rocm(): # HIP will sometimes fail if no atol - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps, - atol=1e-4) + gradient_checker.double_grad_check( + [x] + w, + y, + x_init=[x_arr] + w_arr, + place=place, + eps=eps, + atol=1e-4, + ) else: - gradient_checker.double_grad_check([x] + w, - y, - x_init=[x_arr] + w_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x] + w, y, x_init=[x_arr] + w_arr, place=place, eps=eps + ) gradient_checker.double_grad_check_for_dygraph( - self.conv_transpose_wrapper, [x] + w, + self.conv_transpose_wrapper, + [x] + w, y, x_init=[x_arr] + w_arr, - place=place) + place=place, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_corr.py b/python/paddle/fluid/tests/unittests/test_corr.py index 252948d8eed48..82f28d3cd0046 100644 --- a/python/paddle/fluid/tests/unittests/test_corr.py +++ b/python/paddle/fluid/tests/unittests/test_corr.py @@ -28,7 +28,6 @@ def numpy_corr(np_arr, rowvar=True, dtype='float64'): class Corr_Test(unittest.TestCase): - def setUp(self): self.shape = [4, 5] @@ -49,14 +48,13 @@ def test_tensor_corr_default(self): corr = paddle.linalg.corrcoef(tensor) np_corr = numpy_corr(np_arr, rowvar=True, dtype=dtype) if dtype == 'float32': - np.testing.assert_allclose(np_corr, - corr.numpy(), - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + np_corr, corr.numpy(), rtol=1e-05, atol=1e-05 + ) else: - np.testing.assert_allclose(np_corr, - corr.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + np_corr, corr.numpy(), rtol=1e-05 + ) def test_tensor_corr_rowvar(self): typelist = ['float64', 'float32'] @@ -76,37 +74,32 @@ def test_tensor_corr_rowvar(self): corr = paddle.linalg.corrcoef(tensor, rowvar=False) np_corr = numpy_corr(np_arr, rowvar=False, dtype=dtype) if dtype == 'float32': - np.testing.assert_allclose(np_corr, - corr.numpy(), - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + np_corr, corr.numpy(), rtol=1e-05, atol=1e-05 + ) else: - np.testing.assert_allclose(np_corr, - corr.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + np_corr, corr.numpy(), rtol=1e-05 + ) # Input(x) only support N-D (1<=N<=2) tensor class Corr_Test2(Corr_Test): - def setUp(self): self.shape = [10] class Corr_Test3(Corr_Test): - def setUp(self): self.shape = [4, 5] # Input(x) only support N-D (1<=N<=2) tensor class Corr_Test4(unittest.TestCase): - def setUp(self): self.shape = [2, 5, 2] def test_errors(self): - def test_err(): np_arr = np.random.rand(*self.shape).astype('float64') tensor = paddle.to_tensor(np_arr) @@ -117,7 +110,6 @@ def test_err(): # test unsupported complex input class Corr_Comeplex_Test(unittest.TestCase): - def setUp(self): self.dtype = 'complex128' @@ -129,7 +121,6 @@ def test_errors(self): class Corr_Test5(Corr_Comeplex_Test): - def setUp(self): self.dtype = 'complex64' diff --git a/python/paddle/fluid/tests/unittests/test_cos_sim_op.py b/python/paddle/fluid/tests/unittests/test_cos_sim_op.py index 5943214e9a324..86d1e0e1a8ce7 100644 --- a/python/paddle/fluid/tests/unittests/test_cos_sim_op.py +++ b/python/paddle/fluid/tests/unittests/test_cos_sim_op.py @@ -20,21 +20,23 @@ class TestCosSimOp(OpTest): - def setUp(self): self.op_type = "cos_sim" self.inputs = { 'X': np.random.random((6, 20)).astype("float32"), - 'Y': np.random.random((6, 20)).astype("float32") + 'Y': np.random.random((6, 20)).astype("float32"), } expect_x_norm = np.linalg.norm(self.inputs['X'], axis=1) expect_y_norm = np.linalg.norm(self.inputs['Y'], axis=1) - expect_out = (self.inputs['X'] * self.inputs['Y']).sum(axis=1) / \ - expect_x_norm / expect_y_norm + expect_out = ( + (self.inputs['X'] * self.inputs['Y']).sum(axis=1) + / expect_x_norm + / expect_y_norm + ) self.outputs = { 'XNorm': np.expand_dims(expect_x_norm, 1), 'YNorm': np.expand_dims(expect_y_norm, 1), - 'Out': np.expand_dims(expect_out, 1) + 'Out': np.expand_dims(expect_out, 1), } def test_check_output(self): @@ -44,84 +46,89 @@ def test_check_grad_normal(self): self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.06) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.06, - no_grad_set=set("X")) + self.check_grad( + ['Y'], 'Out', max_relative_error=0.06, no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.06, - no_grad_set=set('Y')) + self.check_grad( + ['X'], 'Out', max_relative_error=0.06, no_grad_set=set('Y') + ) class TestCosSimOp2(TestCosSimOp): - def setUp(self): self.op_type = "cos_sim" self.inputs = { 'X': np.random.random((6, 100)).astype("float32"), - 'Y': np.random.random((1, 100)).astype("float32") + 'Y': np.random.random((1, 100)).astype("float32"), } expect_x_norm = np.linalg.norm(self.inputs['X'], axis=1) expect_y_norm = np.linalg.norm(self.inputs['Y'], axis=1) - expect_out = (self.inputs['X'] * self.inputs['Y']).sum(axis=1) / \ - expect_x_norm / expect_y_norm + expect_out = ( + (self.inputs['X'] * self.inputs['Y']).sum(axis=1) + / expect_x_norm + / expect_y_norm + ) self.outputs = { 'XNorm': np.expand_dims(expect_x_norm, 1), 'YNorm': np.expand_dims(expect_y_norm, 1), - 'Out': np.expand_dims(expect_out, 1) + 'Out': np.expand_dims(expect_out, 1), } class TestCosSimOp3(TestCosSimOp): - def setUp(self): self.op_type = "cos_sim" self.inputs = { 'X': np.random.random((6, 5, 4)).astype("float32"), - 'Y': np.random.random((6, 5, 4)).astype("float32") + 'Y': np.random.random((6, 5, 4)).astype("float32"), } expect_x_norm = np.linalg.norm(self.inputs['X'], axis=(1, 2)) expect_y_norm = np.linalg.norm(self.inputs['Y'], axis=(1, 2)) - expect_out = (self.inputs['X'] * self.inputs['Y']).sum(axis=(1, 2)) / \ - expect_x_norm / expect_y_norm + expect_out = ( + (self.inputs['X'] * self.inputs['Y']).sum(axis=(1, 2)) + / expect_x_norm + / expect_y_norm + ) self.outputs = { 'XNorm': np.expand_dims(expect_x_norm, 1), 'YNorm': np.expand_dims(expect_y_norm, 1), - 'Out': np.expand_dims(expect_out, 1) + 'Out': np.expand_dims(expect_out, 1), } class TestCosSimOp4(TestCosSimOp): - def setUp(self): self.op_type = "cos_sim" self.inputs = { 'X': np.random.random((6, 5, 20)).astype("float32"), - 'Y': np.random.random((1, 5, 20)).astype("float32") + 'Y': np.random.random((1, 5, 20)).astype("float32"), } expect_x_norm = np.linalg.norm(self.inputs['X'], axis=(1, 2)) expect_y_norm = np.linalg.norm(self.inputs['Y'], axis=(1, 2)) - expect_out = (self.inputs['X'] * self.inputs['Y']).sum(axis=(1, 2)) / \ - expect_x_norm / expect_y_norm + expect_out = ( + (self.inputs['X'] * self.inputs['Y']).sum(axis=(1, 2)) + / expect_x_norm + / expect_y_norm + ) self.outputs = { 'XNorm': np.expand_dims(expect_x_norm, 1), 'YNorm': np.expand_dims(expect_y_norm, 1), - 'Out': np.expand_dims(expect_out, 1) + 'Out': np.expand_dims(expect_out, 1), } class TestCosSimOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of batch_norm must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) - x2 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + x2 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.cos_sim, x1, x2) # the input dtype of batch_norm must be float32 diff --git a/python/paddle/fluid/tests/unittests/test_cosine_embedding_loss.py b/python/paddle/fluid/tests/unittests/test_cosine_embedding_loss.py index 765ed8a67ac72..784e583b4f5f8 100644 --- a/python/paddle/fluid/tests/unittests/test_cosine_embedding_loss.py +++ b/python/paddle/fluid/tests/unittests/test_cosine_embedding_loss.py @@ -39,7 +39,6 @@ def cosine_embedding_loss(input1, input2, label, margin=0.5, reduction='mean'): class TestFunctionCosineEmbeddingLoss(unittest.TestCase): - def setUp(self): self.input1_np = np.random.random(size=(5, 3)).astype(np.float64) self.input2_np = np.random.random(size=(5, 3)).astype(np.float64) @@ -52,43 +51,43 @@ def run_dynamic(self): input1 = paddle.to_tensor(self.input1_np) input2 = paddle.to_tensor(self.input2_np) label = paddle.to_tensor(self.label_np) - dy_result = paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') - expected1 = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='mean') + dy_result = paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) + expected1 = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='mean', + ) np.testing.assert_allclose(dy_result.numpy(), expected1, rtol=1e-05) self.assertTrue(dy_result.shape, [1]) - dy_result = paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='sum') - expected2 = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='sum') + dy_result = paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='sum' + ) + expected2 = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='sum', + ) np.testing.assert_allclose(dy_result.numpy(), expected2, rtol=1e-05) self.assertTrue(dy_result.shape, [1]) - dy_result = paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='none') - expected3 = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='none') + dy_result = paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='none' + ) + expected3 = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='none', + ) np.testing.assert_allclose(dy_result.numpy(), expected3, rtol=1e-05) self.assertTrue(dy_result.shape, [5]) @@ -97,50 +96,52 @@ def run_static(self, use_gpu=False): input1 = static.data(name='input1', shape=[5, 3], dtype='float64') input2 = static.data(name='input2', shape=[5, 3], dtype='float64') label = static.data(name='label', shape=[5], dtype='int32') - result0 = paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='none') - result1 = paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='sum') - result2 = paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') + result0 = paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='none' + ) + result1 = paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='sum' + ) + result2 = paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) place = paddle.CUDAPlace(0) if use_gpu else paddle.CPUPlace() exe = static.Executor(place) exe.run(static.default_startup_program()) - static_result = exe.run(feed={ - "input1": self.input1_np, - "input2": self.input2_np, - "label": self.label_np - }, - fetch_list=[result0, result1, result2]) - expected = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='none') + static_result = exe.run( + feed={ + "input1": self.input1_np, + "input2": self.input2_np, + "label": self.label_np, + }, + fetch_list=[result0, result1, result2], + ) + expected = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='none', + ) np.testing.assert_allclose(static_result[0], expected, rtol=1e-05) - expected = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='sum') + expected = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='sum', + ) np.testing.assert_allclose(static_result[1], expected, rtol=1e-05) - expected = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='mean') + expected = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='mean', + ) np.testing.assert_allclose(static_result[2], expected, rtol=1e-05) @@ -171,72 +172,62 @@ def test_errors(self): def test_label_shape_error(): label = paddle.to_tensor( - np.random.randint(low=0, high=2, size=(2, 3))) - paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') + np.random.randint(low=0, high=2, size=(2, 3)) + ) + paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) self.assertRaises(ValueError, test_label_shape_error) def test_input_different_shape_error(): input1 = paddle.to_tensor(self.input1_np[0]) label = paddle.to_tensor(np.ndarray([1])) - paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') + paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) self.assertRaises(ValueError, test_input_different_shape_error) def test_input_shape2D_error(): input1 = paddle.to_tensor( - np.random.random(size=(2, 3, 4)).astype(np.float64)) + np.random.random(size=(2, 3, 4)).astype(np.float64) + ) input2 = paddle.to_tensor( - np.random.random(size=(2, 3, 4)).astype(np.float64)) - paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') + np.random.random(size=(2, 3, 4)).astype(np.float64) + ) + paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) self.assertRaises(ValueError, test_input_shape2D_error) def test_label_value_error(): label = paddle.to_tensor(np.ndarray([-1, -2])) - paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') + paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) self.assertRaises(ValueError, test_label_value_error) def test_input_type_error(): input1 = paddle.to_tensor(self.input1_np.astype(np.int64)) - paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') + paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) self.assertRaises(ValueError, test_input_type_error) def test_label_type_error(): label = paddle.to_tensor(self.label_np.astype(np.int16)) - paddle.nn.functional.cosine_embedding_loss(input1, - input2, - label, - margin=0.5, - reduction='mean') + paddle.nn.functional.cosine_embedding_loss( + input1, input2, label, margin=0.5, reduction='mean' + ) self.assertRaises(ValueError, test_label_type_error) class TestClassCosineEmbeddingLoss(unittest.TestCase): - def setUp(self): self.input1_np = np.random.random(size=(10, 3)).astype(np.float32) self.input2_np = np.random.random(size=(10, 3)).astype(np.float32) @@ -252,14 +243,17 @@ def run_dynamic(self): input1 = paddle.to_tensor(self.input1_np) input2 = paddle.to_tensor(self.input2_np) label = paddle.to_tensor(self.label_np) - CosineEmbeddingLoss = paddle.nn.CosineEmbeddingLoss(margin=0.5, - reduction='mean') + CosineEmbeddingLoss = paddle.nn.CosineEmbeddingLoss( + margin=0.5, reduction='mean' + ) dy_result = CosineEmbeddingLoss(input1, input2, label) - expected1 = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='mean') + expected1 = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='mean', + ) np.testing.assert_allclose(dy_result.numpy(), expected1, rtol=1e-05) self.assertTrue(dy_result.shape, [1]) @@ -267,35 +261,42 @@ def run_dynamic(self): input2_1D = paddle.to_tensor(self.input2_np_1D) label_1D = paddle.to_tensor(self.label_np_1D) dy_result = CosineEmbeddingLoss(input1_1D, input2_1D, label_1D) - expected2 = cosine_embedding_loss(self.input1_np_1D, - self.input2_np_1D, - self.label_np_1D, - margin=0.5, - reduction='mean') + expected2 = cosine_embedding_loss( + self.input1_np_1D, + self.input2_np_1D, + self.label_np_1D, + margin=0.5, + reduction='mean', + ) np.testing.assert_allclose(dy_result.numpy(), expected2, rtol=1e-05) def run_static(self): input1 = static.data(name='input1', shape=[10, 3], dtype='float32') input2 = static.data(name='input2', shape=[10, 3], dtype='float32') label = static.data(name='label', shape=[10], dtype='int64') - CosineEmbeddingLoss = paddle.nn.CosineEmbeddingLoss(margin=0.5, - reduction='mean') + CosineEmbeddingLoss = paddle.nn.CosineEmbeddingLoss( + margin=0.5, reduction='mean' + ) result = CosineEmbeddingLoss(input1, input2, label) place = paddle.CPUPlace() exe = static.Executor(place) exe.run(static.default_startup_program()) - static_result = exe.run(feed={ - "input1": self.input1_np, - "input2": self.input2_np, - "label": self.label_np - }, - fetch_list=[result]) - expected = cosine_embedding_loss(self.input1_np, - self.input2_np, - self.label_np, - margin=0.5, - reduction='mean') + static_result = exe.run( + feed={ + "input1": self.input1_np, + "input2": self.input2_np, + "label": self.label_np, + }, + fetch_list=[result], + ) + expected = cosine_embedding_loss( + self.input1_np, + self.input2_np, + self.label_np, + margin=0.5, + reduction='mean', + ) np.testing.assert_allclose(static_result[0], expected, rtol=1e-05) @@ -308,16 +309,17 @@ def test_cpu(self): self.run_static() def test_errors(self): - def test_margin_error(): CosineEmbeddingLoss = paddle.nn.CosineEmbeddingLoss( - margin=2, reduction='mean') + margin=2, reduction='mean' + ) self.assertRaises(ValueError, test_margin_error) def test_reduction_error(): CosineEmbeddingLoss = paddle.nn.CosineEmbeddingLoss( - margin=2, reduction='reduce_mean') + margin=2, reduction='reduce_mean' + ) self.assertRaises(ValueError, test_reduction_error) diff --git a/python/paddle/fluid/tests/unittests/test_cosine_similarity_api.py b/python/paddle/fluid/tests/unittests/test_cosine_similarity_api.py index db0bb0147cdeb..1c658d0ecc458 100644 --- a/python/paddle/fluid/tests/unittests/test_cosine_similarity_api.py +++ b/python/paddle/fluid/tests/unittests/test_cosine_similarity_api.py @@ -23,7 +23,6 @@ class TestCosineSimilarityAPI(unittest.TestCase): - def setUp(self): self.places = [paddle.CPUPlace()] if core.is_compiled_with_cuda(): @@ -52,12 +51,11 @@ def check_static_result(self, place): x2 = paddle.fluid.data(name="x2", shape=shape) result = F.cosine_similarity(x1, x2, axis=axis, eps=eps) exe = Executor(place) - fetches = exe.run(default_main_program(), - feed={ - "x1": np_x1, - "x2": np_x2 - }, - fetch_list=[result]) + fetches = exe.run( + default_main_program(), + feed={"x1": np_x1, "x2": np_x2}, + fetch_list=[result], + ) np_out = self._get_numpy_out(np_x1, np_x2, axis=axis, eps=eps) np.testing.assert_allclose(fetches[0], np_out, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_cost_model.py b/python/paddle/fluid/tests/unittests/test_cost_model.py index ba9b4c3572147..92fa062ad7a8a 100644 --- a/python/paddle/fluid/tests/unittests/test_cost_model.py +++ b/python/paddle/fluid/tests/unittests/test_cost_model.py @@ -24,13 +24,13 @@ class TestCostModel(unittest.TestCase): - def test_profiler_measure_empty_program(self): cost_model = core.CostModel() empty_program = paddle.static.Program() startup_program = paddle.static.Program() - cost_data = cost_model.profile_measure(empty_program, startup_program, - device, ["time"]) + cost_data = cost_model.profile_measure( + empty_program, startup_program, device, ["time"] + ) self.assertEqual(cost_data.get_whole_time_ms(), 0) def test_profiler_measure_program(self): @@ -42,14 +42,16 @@ def test_profiler_measure_program(self): hidden = paddle.static.nn.fc(data, 10) loss = paddle.mean(hidden) cost_model = core.CostModel() - cost_data = cost_model.profile_measure(main_program, startup_program, - device, ["time"]) + cost_data = cost_model.profile_measure( + main_program, startup_program, device, ["time"] + ) fc_op_time = cost_data.get_op_time_ms(0) mean_op_time = cost_data.get_op_time_ms(1) self.assertGreater(fc_op_time, 0) self.assertGreater(mean_op_time, 0) - self.assertGreaterEqual(cost_data.get_whole_time_ms(), - fc_op_time + mean_op_time) + self.assertGreaterEqual( + cost_data.get_whole_time_ms(), fc_op_time + mean_op_time + ) def test_static_op_benchmark_cost_model(self): op_name = "abs" @@ -70,16 +72,18 @@ def test_static_op_benchmark_cost_model(self): print("conv2d_op_time:", conv2d_op_time) print("conv2d_op_config:", conv2d_op_config) - conv2d_backward_op_cost = cost_model.get_static_op_time("conv2d", - forward=False) + conv2d_backward_op_cost = cost_model.get_static_op_time( + "conv2d", forward=False + ) conv2d_backward_op_time = conv2d_backward_op_cost["op_time"] conv2d_backward_op_config = conv2d_backward_op_cost["config"] self.assertGreater(float(conv2d_backward_op_time), 0) print("conv2d_backward_op_time:", conv2d_backward_op_time) print("conv2d_backward_op_config:", conv2d_backward_op_config) - conv2d_fp16_op_cost = cost_model.get_static_op_time("conv2d", - dtype="float16") + conv2d_fp16_op_cost = cost_model.get_static_op_time( + "conv2d", dtype="float16" + ) conv2d_fp16_op_time = conv2d_fp16_op_cost["op_time"] conv2d_fp16_op_config = conv2d_fp16_op_cost["config"] self.assertGreater(float(conv2d_fp16_op_time), 0) diff --git a/python/paddle/fluid/tests/unittests/test_count_nonzero_api.py b/python/paddle/fluid/tests/unittests/test_count_nonzero_api.py index 48a0f08fd4aa4..55f34dd261218 100644 --- a/python/paddle/fluid/tests/unittests/test_count_nonzero_api.py +++ b/python/paddle/fluid/tests/unittests/test_count_nonzero_api.py @@ -26,8 +26,11 @@ class TestCountNonzeroAPI(unittest.TestCase): def setUp(self): self.x_shape = [2, 3, 4, 5] self.x = np.random.uniform(-1, 1, self.x_shape).astype(np.float32) - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_api_static(self): paddle.enable_static() @@ -40,8 +43,9 @@ def test_api_static(self): out4 = paddle.count_nonzero(x, axis) out5 = paddle.count_nonzero(x, tuple(axis)) exe = paddle.static.Executor(self.place) - res = exe.run(feed={'X': self.x}, - fetch_list=[out1, out2, out3, out4, out5]) + res = exe.run( + feed={'X': self.x}, fetch_list=[out1, out2, out3, out4, out5] + ) out_ref = np.count_nonzero(self.x) for out in res: np.testing.assert_allclose(out, out_ref, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_cov.py b/python/paddle/fluid/tests/unittests/test_cov.py index d7fec9028ab03..b00fc5b3fe211 100644 --- a/python/paddle/fluid/tests/unittests/test_cov.py +++ b/python/paddle/fluid/tests/unittests/test_cov.py @@ -20,15 +20,16 @@ def numpy_cov(np_arr, rowvar=True, ddof=1, fweights=None, aweights=None): - return np.cov(np_arr, - rowvar=rowvar, - ddof=int(ddof), - fweights=fweights, - aweights=aweights) + return np.cov( + np_arr, + rowvar=rowvar, + ddof=int(ddof), + fweights=fweights, + aweights=aweights, + ) class Cov_Test(unittest.TestCase): - def setUp(self): self.shape = [20, 10] self.weightshape = [10] @@ -48,16 +49,12 @@ def func_test_tensor_cov_default(self): for dtype in typelist: np_arr = np.random.rand(*self.shape).astype(dtype) tensor = paddle.to_tensor(np_arr, place=p) - cov = paddle.linalg.cov(tensor, - rowvar=True, - ddof=True, - fweights=None, - aweights=None) - np_cov = numpy_cov(np_arr, - rowvar=True, - ddof=1, - fweights=None, - aweights=None) + cov = paddle.linalg.cov( + tensor, rowvar=True, ddof=True, fweights=None, aweights=None + ) + np_cov = numpy_cov( + np_arr, rowvar=True, ddof=1, fweights=None, aweights=None + ) np.testing.assert_allclose(np_cov, cov.numpy(), rtol=1e-05) def test_tensor_cov_default(self): @@ -80,16 +77,16 @@ def func_test_tensor_cov_rowvar(self): for dtype in typelist: np_arr = np.random.rand(*self.shape).astype(dtype) tensor = paddle.to_tensor(np_arr, place=p) - cov = paddle.linalg.cov(tensor, - rowvar=False, - ddof=True, - fweights=None, - aweights=None) - np_cov = numpy_cov(np_arr, - rowvar=False, - ddof=1, - fweights=None, - aweights=None) + cov = paddle.linalg.cov( + tensor, + rowvar=False, + ddof=True, + fweights=None, + aweights=None, + ) + np_cov = numpy_cov( + np_arr, rowvar=False, ddof=1, fweights=None, aweights=None + ) np.testing.assert_allclose(np_cov, cov.numpy(), rtol=1e-05) def test_tensor_cov_rowvar(self): @@ -112,16 +109,16 @@ def func_test_tensor_cov_ddof(self): for dtype in typelist: np_arr = np.random.rand(*self.shape).astype(dtype) tensor = paddle.to_tensor(np_arr, place=p) - cov = paddle.linalg.cov(tensor, - rowvar=True, - ddof=False, - fweights=None, - aweights=None) - np_cov = numpy_cov(np_arr, - rowvar=True, - ddof=0, - fweights=None, - aweights=None) + cov = paddle.linalg.cov( + tensor, + rowvar=True, + ddof=False, + fweights=None, + aweights=None, + ) + np_cov = numpy_cov( + np_arr, rowvar=True, ddof=0, fweights=None, aweights=None + ) np.testing.assert_allclose(np_cov, cov.numpy(), rtol=1e-05) def test_tensor_cov_ddof(self): @@ -143,20 +140,21 @@ def func_test_tensor_cov_fweights(self): for dtype in typelist: np_arr = np.random.rand(*self.shape).astype(dtype) - np_fw = np.random.randint(10, - size=self.weightshape).astype('int32') + np_fw = np.random.randint(10, size=self.weightshape).astype( + 'int32' + ) tensor = paddle.to_tensor(np_arr, place=p) fweights = paddle.to_tensor(np_fw, place=p) - cov = paddle.linalg.cov(tensor, - rowvar=True, - ddof=True, - fweights=fweights, - aweights=None) - np_cov = numpy_cov(np_arr, - rowvar=True, - ddof=1, - fweights=np_fw, - aweights=None) + cov = paddle.linalg.cov( + tensor, + rowvar=True, + ddof=True, + fweights=fweights, + aweights=None, + ) + np_cov = numpy_cov( + np_arr, rowvar=True, ddof=1, fweights=np_fw, aweights=None + ) np.testing.assert_allclose(np_cov, cov.numpy(), rtol=1e-05) def test_tensor_cov_fweights(self): @@ -178,20 +176,21 @@ def func_test_tensor_cov_aweights(self): for dtype in typelist: np_arr = np.random.rand(*self.shape).astype(dtype) - np_aw = np.random.randint(10, - size=self.weightshape).astype('int32') + np_aw = np.random.randint(10, size=self.weightshape).astype( + 'int32' + ) tensor = paddle.to_tensor(np_arr, place=p) aweights = paddle.to_tensor(np_aw, place=p) - cov = paddle.linalg.cov(tensor, - rowvar=True, - ddof=True, - fweights=None, - aweights=aweights) - np_cov = numpy_cov(np_arr, - rowvar=True, - ddof=1, - fweights=None, - aweights=np_aw) + cov = paddle.linalg.cov( + tensor, + rowvar=True, + ddof=True, + fweights=None, + aweights=aweights, + ) + np_cov = numpy_cov( + np_arr, rowvar=True, ddof=1, fweights=None, aweights=np_aw + ) np.testing.assert_allclose(np_cov, cov.numpy(), rtol=1e-05) def test_tensor_cov_aweights(self): @@ -213,22 +212,23 @@ def func_test_tensor_cov_weights(self): for dtype in typelist: np_arr = np.random.rand(*self.shape).astype(dtype) - np_fw = np.random.randint(10, - size=self.weightshape).astype('int64') + np_fw = np.random.randint(10, size=self.weightshape).astype( + 'int64' + ) np_aw = np.random.rand(*self.weightshape).astype('float64') tensor = paddle.to_tensor(np_arr, place=p) fweights = paddle.to_tensor(np_fw, place=p) aweights = paddle.to_tensor(np_aw, place=p) - cov = paddle.linalg.cov(tensor, - rowvar=True, - ddof=True, - fweights=fweights, - aweights=aweights) - np_cov = numpy_cov(np_arr, - rowvar=True, - ddof=1, - fweights=np_fw, - aweights=np_aw) + cov = paddle.linalg.cov( + tensor, + rowvar=True, + ddof=True, + fweights=fweights, + aweights=aweights, + ) + np_cov = numpy_cov( + np_arr, rowvar=True, ddof=1, fweights=np_fw, aweights=np_aw + ) np.testing.assert_allclose(np_cov, cov.numpy(), rtol=1e-05) def test_tensor_cov_weights(self): @@ -238,7 +238,6 @@ def test_tensor_cov_weights(self): class Cov_Test2(Cov_Test): - def setUp(self): self.shape = [10] self.weightshape = [10] @@ -246,30 +245,32 @@ def setUp(self): # Input(x) only support N-D (1<=N<=2) tensor class Cov_Test3(unittest.TestCase): - def setUp(self): self.shape = [2, 5, 10] self.fweightshape = [10] self.aweightshape = [10] - self.fw_s = 1. - self.aw_s = 1. + self.fw_s = 1.0 + self.aw_s = 1.0 def func_test_errors(self): - def test_err(): np_arr = np.random.rand(*self.shape).astype('float64') - np_fw = self.fw_s * np.random.rand( - *self.fweightshape).astype('int32') - np_aw = self.aw_s * np.random.rand( - *self.aweightshape).astype('float64') + np_fw = self.fw_s * np.random.rand(*self.fweightshape).astype( + 'int32' + ) + np_aw = self.aw_s * np.random.rand(*self.aweightshape).astype( + 'float64' + ) tensor = paddle.to_tensor(np_arr) fweights = paddle.to_tensor(np_fw) aweights = paddle.to_tensor(np_aw) - cov = paddle.linalg.cov(tensor, - rowvar=True, - ddof=True, - fweights=fweights, - aweights=aweights) + cov = paddle.linalg.cov( + tensor, + rowvar=True, + ddof=True, + fweights=fweights, + aweights=aweights, + ) self.assertRaises(ValueError, test_err) @@ -279,70 +280,64 @@ def test_errors(self): self.func_test_errors() -#Input(fweights) only support N-D (N<=1) tensor +# Input(fweights) only support N-D (N<=1) tensor class Cov_Test4(Cov_Test3): - def setUp(self): self.shape = [5, 10] self.fweightshape = [2, 10] self.aweightshape = [10] - self.fw_s = 1. - self.aw_s = 1. + self.fw_s = 1.0 + self.aw_s = 1.0 -#The number of Input(fweights) should equal to x's dim[1] +# The number of Input(fweights) should equal to x's dim[1] class Cov_Test5(Cov_Test3): - def setUp(self): self.shape = [5, 10] self.fweightshape = [5] self.aweightshape = [10] - self.fw_s = 1. - self.aw_s = 1. + self.fw_s = 1.0 + self.aw_s = 1.0 -#The value of Input(fweights) cannot be negtive +# The value of Input(fweights) cannot be negtive class Cov_Test6(Cov_Test3): - def setUp(self): self.shape = [5, 10] self.fweightshape = [10] self.aweightshape = [10] - self.fw_s = -1. - self.aw_s = 1. + self.fw_s = -1.0 + self.aw_s = 1.0 -#Input(aweights) only support N-D (N<=1) tensor +# Input(aweights) only support N-D (N<=1) tensor class Cov_Test7(Cov_Test3): - def setUp(self): self.shape = [5, 10] self.fweightshape = [10] self.aweightshape = [2, 10] - self.fw_s = 1. - self.aw_s = 1. + self.fw_s = 1.0 + self.aw_s = 1.0 -#The number of Input(aweights) should equal to x's dim[1] +# The number of Input(aweights) should equal to x's dim[1] class Cov_Test8(Cov_Test3): - def setUp(self): self.shape = [5, 10] self.fweightshape = [10] self.aweightshape = [5] - self.fw_s = 1. - self.aw_s = 1. + self.fw_s = 1.0 + self.aw_s = 1.0 -#The value of Input(aweights) cannot be negtive +# The value of Input(aweights) cannot be negtive class Cov_Test9(Cov_Test3): - def setUp(self): self.shape = [5, 10] self.fweightshape = [10] self.aweightshape = [10] - self.fw_s = 1. - self.aw_s = -1. + self.fw_s = 1.0 + self.aw_s = -1.0 if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_cpuonly_spawn.py b/python/paddle/fluid/tests/unittests/test_cpuonly_spawn.py index 3f5fadff3af7c..c0e6d173f5771 100644 --- a/python/paddle/fluid/tests/unittests/test_cpuonly_spawn.py +++ b/python/paddle/fluid/tests/unittests/test_cpuonly_spawn.py @@ -21,7 +21,6 @@ class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear1 = nn.Linear(10, 10) @@ -58,7 +57,6 @@ def train(print_result=False): class TestSpawn(unittest.TestCase): - def test_spawn(self): dist.spawn(train, backend='gloo', nprocs=4) diff --git a/python/paddle/fluid/tests/unittests/test_create_global_var.py b/python/paddle/fluid/tests/unittests/test_create_global_var.py index 6a9f188b36319..45dc1e5202285 100644 --- a/python/paddle/fluid/tests/unittests/test_create_global_var.py +++ b/python/paddle/fluid/tests/unittests/test_create_global_var.py @@ -19,7 +19,6 @@ class TestCreateGlobalVarError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): diff --git a/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py b/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py index af9b162e90939..07c89eefc32fa 100644 --- a/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py +++ b/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py @@ -17,7 +17,6 @@ class TestDocString(unittest.TestCase): - def test_layer_doc_string(self): print(layers.dropout.__doc__) diff --git a/python/paddle/fluid/tests/unittests/test_create_parameter.py b/python/paddle/fluid/tests/unittests/test_create_parameter.py index 53260374c5d20..d70e06f184356 100644 --- a/python/paddle/fluid/tests/unittests/test_create_parameter.py +++ b/python/paddle/fluid/tests/unittests/test_create_parameter.py @@ -20,7 +20,6 @@ class TestCreateParameterError(unittest.TestCase): - def func_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -36,18 +35,18 @@ def test_shape_item(): self.assertRaises(TypeError, test_shape_item) def test_attr(): - fluid.layers.create_parameter([1, 2, 3], - np.float32, - attr=np.array( - [i for i in range(6)])) + fluid.layers.create_parameter( + [1, 2, 3], np.float32, attr=np.array([i for i in range(6)]) + ) self.assertRaises(TypeError, test_attr) def test_default_initializer(): - fluid.layers.create_parameter([1, 2, 3], - np.float32, - default_initializer=np.array( - [i for i in range(6)])) + fluid.layers.create_parameter( + [1, 2, 3], + np.float32, + default_initializer=np.array([i for i in range(6)]), + ) self.assertRaises(TypeError, test_default_initializer) diff --git a/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py b/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py index 2dc0853f7190c..8336bad6de1bf 100644 --- a/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py +++ b/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py @@ -20,10 +20,10 @@ class CRFDecoding(object): - - def __init__(self, emission_weights, transition_weights, - seq_start_positions): - assert (emission_weights.shape[0] == sum(seq_start_positions)) + def __init__( + self, emission_weights, transition_weights, seq_start_positions + ): + assert emission_weights.shape[0] == sum(seq_start_positions) self.tag_num = emission_weights.shape[1] self.seq_num = len(seq_start_positions) @@ -34,10 +34,12 @@ def __init__(self, emission_weights, transition_weights, self.b = transition_weights[1, :] self.w = transition_weights[2:, :] - self.track = np.zeros((sum(seq_start_positions), self.tag_num), - dtype="int64") - self.decoded_path = np.zeros((sum(seq_start_positions), 1), - dtype="int64") + self.track = np.zeros( + (sum(seq_start_positions), self.tag_num), dtype="int64" + ) + self.decoded_path = np.zeros( + (sum(seq_start_positions), 1), dtype="int64" + ) def _decode_one_sequence(self, decoded_path, x): seq_len, tag_num = x.shape @@ -77,8 +79,9 @@ def decode(self): start = cur_pos cur_pos += self.seq_start_positions[i] end = cur_pos - self._decode_one_sequence(self.decoded_path[start:end, :], - self.x[start:end, :]) + self._decode_one_sequence( + self.decoded_path[start:end, :], self.x[start:end, :] + ) return self.decoded_path @@ -98,10 +101,12 @@ def set_test_data(self): for i in range(SEQ_NUM): lod[-1].append(random.randint(1, MAX_SEQ_LEN)) total_len += lod[-1][-1] - emission = np.random.uniform(-1, 1, - [total_len, TAG_NUM]).astype("float64") - transition = np.random.uniform(-0.5, 0.5, - [TAG_NUM + 2, TAG_NUM]).astype("float64") + emission = np.random.uniform(-1, 1, [total_len, TAG_NUM]).astype( + "float64" + ) + transition = np.random.uniform( + -0.5, 0.5, [TAG_NUM + 2, TAG_NUM] + ).astype("float64") self.inputs = { "Emission": (emission, lod), @@ -136,27 +141,29 @@ def setUp(self): self.init_lod() total_len = sum(self.lod[-1]) - transition = np.repeat(np.arange(TAG_NUM, - dtype="float64").reshape(1, TAG_NUM), - TAG_NUM + 2, - axis=0) - emission = np.repeat(np.arange(TAG_NUM, - dtype="float64").reshape(1, TAG_NUM), - total_len, - axis=0) - - labels = np.random.randint(low=0, - high=TAG_NUM, - size=(total_len, 1), - dtype="int64") - predicted_labels = np.ones( - (total_len, 1), dtype="int64") * (TAG_NUM - 1) + transition = np.repeat( + np.arange(TAG_NUM, dtype="float64").reshape(1, TAG_NUM), + TAG_NUM + 2, + axis=0, + ) + emission = np.repeat( + np.arange(TAG_NUM, dtype="float64").reshape(1, TAG_NUM), + total_len, + axis=0, + ) + + labels = np.random.randint( + low=0, high=TAG_NUM, size=(total_len, 1), dtype="int64" + ) + predicted_labels = np.ones((total_len, 1), dtype="int64") * ( + TAG_NUM - 1 + ) expected_output = (labels == predicted_labels).astype("int64") self.inputs = { "Emission": (emission, self.lod), "Transition": transition, - "Label": (labels, self.lod) + "Label": (labels, self.lod), } self.outputs = {"ViterbiPath": expected_output} @@ -166,13 +173,11 @@ def test_check_output(self): class TestCRFDecodingOp3(TestCRFDecodingOp2): - def init_lod(self): self.lod = [[1, 0, 0, 4]] class TestCRFDecodingOp4(TestCRFDecodingOp2): - def init_lod(self): self.lod = [[0, 2, 3, 0]] @@ -183,7 +188,7 @@ def seq_pad(data, length): padded = np.zeros(shape).astype(data.dtype) offset = 0 for i, l in enumerate(length): - padded[i, 0:l] = data[offset:offset + l] + padded[i, 0:l] = data[offset : offset + l] offset += l return np.squeeze(padded) @@ -204,10 +209,12 @@ def set_test_data(self): for i in range(SEQ_NUM): lod[-1].append(random.randint(1, MAX_SEQ_LEN)) total_len += lod[-1][-1] - emission = np.random.uniform(-1, 1, - [total_len, TAG_NUM]).astype("float64") - transition = np.random.uniform(-0.5, 0.5, - [TAG_NUM + 2, TAG_NUM]).astype("float64") + emission = np.random.uniform(-1, 1, [total_len, TAG_NUM]).astype( + "float64" + ) + transition = np.random.uniform( + -0.5, 0.5, [TAG_NUM + 2, TAG_NUM] + ).astype("float64") self.inputs = { "Emission": seq_pad(emission, lod[0]), @@ -229,7 +236,6 @@ def test_check_output(self): class TestCRFDecodingOp6(OpTest): - def init_lod(self): self.lod = [[1, 2, 3, 4]] @@ -239,21 +245,23 @@ def setUp(self): self.init_lod() total_len = sum(self.lod[-1]) - transition = np.repeat(np.arange(TAG_NUM, - dtype="float64").reshape(1, TAG_NUM), - TAG_NUM + 2, - axis=0) - emission = np.repeat(np.arange(TAG_NUM, - dtype="float64").reshape(1, TAG_NUM), - total_len, - axis=0) - - labels = np.random.randint(low=0, - high=TAG_NUM, - size=(total_len, 1), - dtype="int64") - predicted_labels = np.ones( - (total_len, 1), dtype="int64") * (TAG_NUM - 1) + transition = np.repeat( + np.arange(TAG_NUM, dtype="float64").reshape(1, TAG_NUM), + TAG_NUM + 2, + axis=0, + ) + emission = np.repeat( + np.arange(TAG_NUM, dtype="float64").reshape(1, TAG_NUM), + total_len, + axis=0, + ) + + labels = np.random.randint( + low=0, high=TAG_NUM, size=(total_len, 1), dtype="int64" + ) + predicted_labels = np.ones((total_len, 1), dtype="int64") * ( + TAG_NUM - 1 + ) expected_output = (labels == predicted_labels).astype("int64") self.inputs = { diff --git a/python/paddle/fluid/tests/unittests/test_crop_op.py b/python/paddle/fluid/tests/unittests/test_crop_op.py index 32e31e5366fdf..f495be8cbefae 100644 --- a/python/paddle/fluid/tests/unittests/test_crop_op.py +++ b/python/paddle/fluid/tests/unittests/test_crop_op.py @@ -20,7 +20,6 @@ def crop(data, offsets, crop_shape): - def indexOf(shape, index): result = [] for dim in reversed(shape): @@ -34,15 +33,17 @@ def indexOf(shape, index): selected = True if len(index) == len(offsets): for j, offset in enumerate(offsets): - selected = selected and index[j] >= offset and index[ - j] < crop_shape[j] + offset + selected = ( + selected + and index[j] >= offset + and index[j] < crop_shape[j] + offset + ) if selected: result.append(value) return np.array(result).reshape(crop_shape) class TestCropOp(OpTest): - def setUp(self): self.op_type = "crop" self.crop_by_input = False @@ -52,7 +53,7 @@ def setUp(self): if self.crop_by_input: self.inputs = { 'X': np.random.random(self.x_shape).astype("float64"), - 'Y': np.random.random(self.crop_shape).astype("float64") + 'Y': np.random.random(self.crop_shape).astype("float64"), } else: self.attrs['shape'] = self.crop_shape @@ -85,7 +86,6 @@ def test_check_grad_normal(self): class TestCase1(TestCropOp): - def initTestCase(self): self.x_shape = (16, 8, 32) self.crop_shape = [2, 2, 3] @@ -93,7 +93,6 @@ def initTestCase(self): class TestCase2(TestCropOp): - def initTestCase(self): self.x_shape = (15, 8) self.crop_shape = [15, 8] @@ -101,7 +100,6 @@ def initTestCase(self): class TestCase3(TestCropOp): - def initTestCase(self): self.x_shape = (4, 8, 16) self.crop_shape = [2, 2, 3] @@ -110,7 +108,6 @@ def initTestCase(self): class TestCase4(TestCropOp): - def initTestCase(self): self.x_shape = (10, 10) self.crop_shape = [10, 10] @@ -119,7 +116,6 @@ def initTestCase(self): class TestCase5(TestCropOp): - def initTestCase(self): self.x_shape = (3, 4, 10) self.crop_shape = [2, 2, 3] @@ -128,7 +124,6 @@ def initTestCase(self): class TestCase6(TestCropOp): - def initTestCase(self): self.x_shape = (10, 9, 14) self.crop_shape = [3, 3, 5] @@ -138,7 +133,6 @@ def initTestCase(self): class TestCropNoneOffset(unittest.TestCase): - def test_crop_none_offset(self): x = fluid.data(name="input1", shape=[3, 6, 6], dtype="float32") crop_shape = [2, 2, 2] @@ -147,7 +141,6 @@ def test_crop_none_offset(self): class TestCropNoneShape(unittest.TestCase): - def test_crop_none_shape(self): x = fluid.data(name="input1", shape=[3, 6, 6], dtype="float32") crop = paddle.crop(x) @@ -156,5 +149,6 @@ def test_crop_none_shape(self): if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_crop_tensor_op.py b/python/paddle/fluid/tests/unittests/test_crop_tensor_op.py index 8af23c6be5ee6..aeb0cfb6414a8 100644 --- a/python/paddle/fluid/tests/unittests/test_crop_tensor_op.py +++ b/python/paddle/fluid/tests/unittests/test_crop_tensor_op.py @@ -20,7 +20,6 @@ def crop(data, offsets, crop_shape): - def indexOf(shape, index): result = [] for dim in reversed(shape): @@ -34,15 +33,17 @@ def indexOf(shape, index): selected = True if len(index) == len(offsets): for j, offset in enumerate(offsets): - selected = selected and index[j] >= offset and index[ - j] < crop_shape[j] + offset + selected = ( + selected + and index[j] >= offset + and index[j] < crop_shape[j] + offset + ) if selected: result.append(value) return np.array(result).reshape(crop_shape) class TestCropTensorOp(OpTest): - def setUp(self): self.op_type = "crop_tensor" self.shape_by_input = False @@ -55,7 +56,7 @@ def setUp(self): if self.shape_by_input: self.inputs = { 'X': np.random.random(self.x_shape).astype("float64"), - 'Shape': np.array(self.crop_shape).astype("int32") + 'Shape': np.array(self.crop_shape).astype("int32"), } else: self.attrs['shape'] = self.crop_shape @@ -86,15 +87,13 @@ def test_check_grad_normal(self): class TestCase1(TestCropTensorOp): - def initTestCase(self): - self.x_shape = (100) + self.x_shape = 100 self.crop_shape = [64] self.offsets = [13] class TestCase2(TestCropTensorOp): - def initTestCase(self): self.x_shape = (12, 24) self.crop_shape = [-1, 8] @@ -102,7 +101,6 @@ def initTestCase(self): class TestCase3(TestCropTensorOp): - def initTestCase(self): self.x_shape = (4, 8, 16) self.crop_shape = [2, 2, 3] @@ -111,7 +109,6 @@ def initTestCase(self): class TestCase4(TestCropTensorOp): - def initTestCase(self): self.x_shape = (8, 3, 6, 6) self.crop_shape = [-1, 3, -1, 4] @@ -120,7 +117,6 @@ def initTestCase(self): class TestCase5(TestCropTensorOp): - def initTestCase(self): self.x_shape = (2, 4, 5, 8, 8) self.crop_shape = [1, 1, 2, 4, 4] @@ -129,7 +125,6 @@ def initTestCase(self): class TestCase6(TestCropTensorOp): - def initTestCase(self): self.x_shape = (2, 2, 4, 4, 4, 2) self.crop_shape = [1, 1, 4, 2, 2, 2] @@ -139,7 +134,6 @@ def initTestCase(self): class TestCropTensorOpTensorAttr(OpTest): - def setUp(self): self.op_type = "crop_tensor" self.OffsetsTensor = False @@ -151,22 +145,24 @@ def setUp(self): if self.ShapeTensor: shape_tensor = [] for index, ele in enumerate(self.crop_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.x_shape).astype("float64"), - 'ShapeTensor': shape_tensor + 'ShapeTensor': shape_tensor, } self.attrs['shape'] = self.shape_attr if self.OffsetsTensor: offsets_tensor = [] for index, ele in enumerate(self.offsets): - offsets_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + offsets_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.x_shape).astype("float64"), - 'OffsetsTensor': offsets_tensor + 'OffsetsTensor': offsets_tensor, } self.attrs['offsets'] = self.offsets_attr @@ -192,7 +188,6 @@ def test_check_grad_normal(self): class TestCropTensorOpTensorAttrCase1(TestCropTensorOpTensorAttr): - def initTestCase(self): self.x_shape = (16, 8, 32) self.crop_shape = [-1, -1, 3] @@ -201,7 +196,6 @@ def initTestCase(self): class TestCropTensorOpTensorAttrCase2(TestCropTensorOpTensorAttr): - def initTestCase(self): self.x_shape = (4, 8, 16, 8) self.crop_shape = [2, 2, 3, 4] @@ -210,7 +204,6 @@ def initTestCase(self): class TestCropTensorOpTensorAttrCase3(TestCropTensorOpTensorAttr): - def initTestCase(self): self.x_shape = (16, 8, 32) self.crop_shape = [2, 2, 3] @@ -221,7 +214,6 @@ def initTestCase(self): class TestCropTensorOpTensorAttrCase4(TestCropTensorOpTensorAttr): - def initTestCase(self): self.x_shape = (16, 8, 32) self.crop_shape = [2, 2, 3] @@ -232,7 +224,6 @@ def initTestCase(self): class TestCropTensorException(unittest.TestCase): - def test_exception(self): input1 = fluid.data(name="input1", shape=[2, 3, 6, 6], dtype="float32") input2 = fluid.data(name="input2", shape=[2, 3, 6, 6], dtype="float16") @@ -255,14 +246,14 @@ def attr_offsets_type(): out = paddle.crop(input1, shape=[2, 2, 3, 3], offsets=0) def attr_offsets_dtype(): - out = paddle.crop(input1, - shape=[2, 2, 3, 3], - offsets=[0, 1.0, 0, 0]) + out = paddle.crop( + input1, shape=[2, 2, 3, 3], offsets=[0, 1.0, 0, 0] + ) def attr_offsets_value(): - out = paddle.crop(input1, - shape=[2, 2, 3, 3], - offsets=[0, -1, offset, 0]) + out = paddle.crop( + input1, shape=[2, 2, 3, 3], offsets=[0, -1, offset, 0] + ) def input_dtype(): out = paddle.crop(input2, shape=[2, 2, 3, 3]) @@ -279,5 +270,6 @@ def input_dtype(): if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_cross_entropy2_op.py b/python/paddle/fluid/tests/unittests/test_cross_entropy2_op.py index 13fbd933899b8..9b8acb884ab45 100644 --- a/python/paddle/fluid/tests/unittests/test_cross_entropy2_op.py +++ b/python/paddle/fluid/tests/unittests/test_cross_entropy2_op.py @@ -18,7 +18,6 @@ class CrossEntropy2OpTestBase(OpTest): - def initParameters(self): return [32, 64], 'float64', -100, False @@ -33,26 +32,33 @@ def calc_output(self, logits, label, ignore_index): return ret, match_x def setUp(self): - self.shape, self.dtype, self.ignore_index, self.drop_last_dim = self.initParameters( - ) + ( + self.shape, + self.dtype, + self.ignore_index, + self.drop_last_dim, + ) = self.initParameters() self.op_type = 'cross_entropy2' feature_size = int(self.shape[-1]) batch_size = int(np.prod(self.shape) / feature_size) logits = (np.random.random(size=self.shape) + 1).astype(self.dtype) - label_shape = self.shape[ - 0:-1] if self.drop_last_dim else self.shape[0:-1] + [1] - label = np.random.random_integers(low=0, - high=feature_size - 1, - size=label_shape).astype('int64') + label_shape = ( + self.shape[0:-1] if self.drop_last_dim else self.shape[0:-1] + [1] + ) + label = np.random.random_integers( + low=0, high=feature_size - 1, size=label_shape + ).astype('int64') outputs, match_x = self.calc_output( np.reshape(logits, [batch_size, feature_size]), - np.reshape(label, [batch_size, 1]), self.ignore_index) + np.reshape(label, [batch_size, 1]), + self.ignore_index, + ) self.inputs = {'X': logits, 'Label': label} out_shape = label_shape self.outputs = { 'Y': np.reshape(outputs, out_shape), 'MatchX': np.reshape(match_x, self.shape[:-1] + [1]), - 'XShape': np.zeros(shape=logits.shape, dtype=logits.dtype) + 'XShape': np.zeros(shape=logits.shape, dtype=logits.dtype), } self.attrs = {'ignore_index': self.ignore_index} @@ -60,37 +66,34 @@ def test_check_output(self): self.check_output(no_check_set=['XShape']) def test_check_grad(self): - self.check_grad(inputs_to_check=['X'], - output_names=['Y'], - no_grad_set=['XShape', 'MatchX', 'Label']) + self.check_grad( + inputs_to_check=['X'], + output_names=['Y'], + no_grad_set=['XShape', 'MatchX', 'Label'], + ) class CrossEntropy2OpTest2(CrossEntropy2OpTestBase): - def initParameters(self): return [32, 64], 'float64', 3, False class CrossEntropy2OpTest2RemoveLastDim(CrossEntropy2OpTestBase): - def initParameters(self): return [32, 64], 'float64', 3, True class CrossEntropy2OpTest3(CrossEntropy2OpTestBase): - def initParameters(self): return [4, 8, 16, 32], 'float64', -100, False class CrossEntropy2OpTest3RemoveLastDim(CrossEntropy2OpTestBase): - def initParameters(self): return [4, 8, 16, 32], 'float64', -100, True class CrossEntropy2OpTest4(CrossEntropy2OpTestBase): - def initParameters(self): return [4, 8, 16, 32], 'float64', 3, False diff --git a/python/paddle/fluid/tests/unittests/test_cross_entropy_loss.py b/python/paddle/fluid/tests/unittests/test_cross_entropy_loss.py index 4ee9b8037fec7..f3e66f9bb8dbf 100644 --- a/python/paddle/fluid/tests/unittests/test_cross_entropy_loss.py +++ b/python/paddle/fluid/tests/unittests/test_cross_entropy_loss.py @@ -27,11 +27,9 @@ def log_softmax(x, axis=-1): return np.log(softmax_out) -def cross_entropy_loss_1d(input, - label, - weight=None, - reduction='mean', - ignore_index=-100): +def cross_entropy_loss_1d( + input, label, weight=None, reduction='mean', ignore_index=-100 +): log_softmax_out = log_softmax(input) input_shape = log_softmax_out.shape N = input_shape[0] @@ -59,11 +57,9 @@ def cross_entropy_loss_1d(input, return out -def cross_entropy_loss_2d(input, - label, - weight=None, - reduction='mean', - ignore_index=-100): +def cross_entropy_loss_2d( + input, label, weight=None, reduction='mean', ignore_index=-100 +): log_softmax_out = log_softmax(input) input_shape = log_softmax_out.shape N = input_shape[0] @@ -81,8 +77,9 @@ def cross_entropy_loss_2d(input, continue cur_weight = weight[cur_target] if weight is not None else 1 total_weight += cur_weight - out[i][h][ - w] = -log_softmax_out[i][h][w][cur_target] * cur_weight + out[i][h][w] = ( + -log_softmax_out[i][h][w][cur_target] * cur_weight + ) if reduction == 'sum': return np.sum(out), np.array([total_weight]).astype('float64') elif reduction == 'mean': @@ -92,27 +89,20 @@ def cross_entropy_loss_2d(input, return out -def cross_entropy_soft(softmax, - label, - axis, - N, - weight=None, - reduction='mean', - ignore_index=-100): - #1.loss +def cross_entropy_soft( + softmax, label, axis, N, weight=None, reduction='mean', ignore_index=-100 +): + # 1.loss loss = cross_entropy( - softmax, - label, - True, #soft_label, - axis, - ignore_index) + softmax, label, True, axis, ignore_index # soft_label, + ) if weight is None and reduction == 'none': return loss - #2.weight + # 2.weight weighted_loss = loss - total_weight = N #for weight is None + total_weight = N # for weight is None if weight is not None: weighted_loss = np.zeros_like(loss).astype(np.float64) total_weight = 0 @@ -122,7 +112,7 @@ def cross_entropy_soft(softmax, total_weight += cur_weight weighted_loss[i] = loss[i] * cur_weight - #3.reduce + # 3.reduce if reduction == 'none': return weighted_loss @@ -136,29 +126,28 @@ def cross_entropy_soft(softmax, return weighted_loss_sum -def cross_entropy_soft_2d(softmax, - label, - axis, - N, - H, - W, - weight=None, - reduction='mean', - ignore_index=-100): - #1.loss +def cross_entropy_soft_2d( + softmax, + label, + axis, + N, + H, + W, + weight=None, + reduction='mean', + ignore_index=-100, +): + # 1.loss loss = cross_entropy( - softmax, - label, - True, #soft_label, - axis, - ignore_index) + softmax, label, True, axis, ignore_index # soft_label, + ) if weight is None and reduction == 'none': return loss - #2.weight + # 2.weight weighted_loss = loss - total_weight = N #for weight is None + total_weight = N # for weight is None if weight is not None: weighted_loss = np.zeros_like(loss).astype(np.float64) total_weight = 0 @@ -170,7 +159,7 @@ def cross_entropy_soft_2d(softmax, total_weight += cur_weight weighted_loss[i][h][w] = loss[i][h][w] * cur_weight - #3.reduce + # 3.reduce if reduction == 'none': return weighted_loss @@ -185,19 +174,20 @@ def cross_entropy_soft_2d(softmax, class CrossEntropyLoss(unittest.TestCase): - def setUp(self): - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) ###test for deprecated softmax_with_cross_entropy def test_softmax_with_cross_entropy(self): self.numeric_stable_mode = False self.soft_label = True - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.axis = -1 - self.ignore_index = -100 #should not be changed + self.ignore_index = -100 # should not be changed self.N = 4 self.C = 3 self.shape = [self.N, self.C] @@ -205,20 +195,24 @@ def test_softmax_with_cross_entropy(self): self.reduction = 'none' self.weight = None self.logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, self.logits) self.labels = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) self.labels /= np.sum(self.labels, axis=self.axis, keepdims=True) - expected = cross_entropy_soft(softmax, - self.labels, - self.axis, - self.N, - weight=self.weight, - reduction=self.reduction, - ignore_index=self.ignore_index) + expected = cross_entropy_soft( + softmax, + self.labels, + self.axis, + self.N, + weight=self.weight, + reduction=self.reduction, + ignore_index=self.ignore_index, + ) paddle.set_device("cpu") @@ -227,7 +221,8 @@ def test_softmax_with_cross_entropy(self): fluid.dygraph.to_variable(self.logits), fluid.dygraph.to_variable(self.labels), soft_label=True, - axis=self.axis) + axis=self.axis, + ) paddle_loss_ce = paddle.nn.functional.cross_entropy( fluid.dygraph.to_variable(self.logits), @@ -235,12 +230,14 @@ def test_softmax_with_cross_entropy(self): soft_label=True, axis=self.axis, weight=fluid.dygraph.to_variable(self.weight) - if self.weight is not None else None, - reduction=self.reduction) - - np.testing.assert_allclose(paddle_loss_swce.numpy(), - expected, - rtol=1e-05) + if self.weight is not None + else None, + reduction=self.reduction, + ) + + np.testing.assert_allclose( + paddle_loss_swce.numpy(), expected, rtol=1e-05 + ) np.testing.assert_allclose(paddle_loss_ce.numpy(), expected, rtol=1e-05) ###soft_label test start @@ -248,10 +245,11 @@ def test_softmax_with_cross_entropy(self): def test_cross_entropy_loss_soft_1d(self): self.numeric_stable_mode = False self.soft_label = True - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.axis = -1 - self.ignore_index = -100 #should not be changed + self.ignore_index = -100 # should not be changed self.N = 4 self.C = 3 self.shape = [self.N, self.C] @@ -259,24 +257,28 @@ def test_cross_entropy_loss_soft_1d(self): self.reduction = 'none' self.weight = None self.logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, self.logits) self.labels = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) self.labels /= np.sum(self.labels, axis=self.axis, keepdims=True) - expected = cross_entropy_soft(softmax, - self.labels, - self.axis, - self.N, - weight=self.weight, - reduction=self.reduction, - ignore_index=self.ignore_index) + expected = cross_entropy_soft( + softmax, + self.labels, + self.axis, + self.N, + weight=self.weight, + reduction=self.reduction, + ignore_index=self.ignore_index, + ) paddle.set_device("cpu") - #2. dygraph + # 2. dygraph paddle.disable_static() paddle_loss_none_weight = paddle.nn.functional.cross_entropy( fluid.dygraph.to_variable(self.logits), @@ -284,35 +286,43 @@ def test_cross_entropy_loss_soft_1d(self): soft_label=True, axis=self.axis, weight=fluid.dygraph.to_variable(self.weight) - if self.weight is not None else None, - reduction=self.reduction) + if self.weight is not None + else None, + reduction=self.reduction, + ) dy_ret_value = paddle_loss_none_weight.numpy() - #3. static + # 3. static paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[self.N, self.C], - dtype=self.dtype) - label = fluid.data(name='label', - shape=[self.N, self.C], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[self.N, self.C], dtype=self.dtype + ) + label = fluid.data( + name='label', shape=[self.N, self.C], dtype=self.dtype + ) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction=self.reduction, soft_label=True) + reduction=self.reduction, soft_label=True + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': self.logits, - 'label': self.labels, - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': self.logits, + 'label': self.labels, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) paddle.disable_static() @@ -323,10 +333,11 @@ def test_cross_entropy_loss_soft_1d(self): def test_cross_entropy_loss_soft_1d_weight(self): self.numeric_stable_mode = False self.soft_label = True - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.axis = -1 - self.ignore_index = -100 #should not be changed + self.ignore_index = -100 # should not be changed self.N = 4 self.C = 3 self.shape = [self.N, self.C] @@ -334,34 +345,38 @@ def test_cross_entropy_loss_soft_1d_weight(self): self.reduction = 'none' self.weight = np.random.uniform(0.1, 1.0, self.C).astype(self.dtype) self.logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, self.logits) if self.soft_label: - self.labels = np.random.uniform(0.1, 1.0, - self.shape).astype(self.dtype) + self.labels = np.random.uniform(0.1, 1.0, self.shape).astype( + self.dtype + ) self.labels /= np.sum(self.labels, axis=self.axis, keepdims=True) else: axis_dim = self.shape[self.axis] self.shape[self.axis] = 1 - self.labels = np.random.randint(0, - axis_dim, - self.shape, - dtype="int64") - - #1. numpy - expected = cross_entropy_soft(softmax, - self.labels, - self.axis, - self.N, - weight=self.weight, - reduction=self.reduction, - ignore_index=self.ignore_index) + self.labels = np.random.randint( + 0, axis_dim, self.shape, dtype="int64" + ) + + # 1. numpy + expected = cross_entropy_soft( + softmax, + self.labels, + self.axis, + self.N, + weight=self.weight, + reduction=self.reduction, + ignore_index=self.ignore_index, + ) paddle.set_device("cpu") - #2. dygraph + # 2. dygraph paddle.disable_static() paddle_loss_none_weight = paddle.nn.functional.cross_entropy( fluid.dygraph.to_variable(self.logits), @@ -369,36 +384,43 @@ def test_cross_entropy_loss_soft_1d_weight(self): soft_label=True, axis=self.axis, weight=fluid.dygraph.to_variable(self.weight), - reduction=self.reduction) + reduction=self.reduction, + ) dy_ret_value = paddle_loss_none_weight.numpy() # 3.static paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[self.N, self.C], - dtype=self.dtype) - label = fluid.data(name='label', - shape=[self.N, self.C], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[self.N, self.C], dtype=self.dtype + ) + label = fluid.data( + name='label', shape=[self.N, self.C], dtype=self.dtype + ) weight = fluid.data(name='weight', shape=[self.C], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction=self.reduction, soft_label=True) + weight=weight, reduction=self.reduction, soft_label=True + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': self.logits, - 'label': self.labels, - "weight": self.weight - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': self.logits, + 'label': self.labels, + "weight": self.weight, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) paddle.disable_static() @@ -409,10 +431,11 @@ def test_cross_entropy_loss_soft_1d_weight(self): def test_cross_entropy_loss_soft_1d_mean(self): self.numeric_stable_mode = False self.soft_label = True - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.axis = -1 - self.ignore_index = -100 #should not be changed + self.ignore_index = -100 # should not be changed self.N = 4 self.C = 3 self.shape = [self.N, self.C] @@ -420,25 +443,29 @@ def test_cross_entropy_loss_soft_1d_mean(self): self.reduction = 'mean' self.weight = None self.logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, self.logits) self.labels = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) self.labels /= np.sum(self.labels, axis=self.axis, keepdims=True) - #1. numpy - expected = cross_entropy_soft(softmax, - self.labels, - self.axis, - self.N, - weight=self.weight, - reduction=self.reduction, - ignore_index=self.ignore_index) + # 1. numpy + expected = cross_entropy_soft( + softmax, + self.labels, + self.axis, + self.N, + weight=self.weight, + reduction=self.reduction, + ignore_index=self.ignore_index, + ) paddle.set_device("cpu") - #2 dygraph + # 2 dygraph paddle.disable_static() paddle_loss_mean = paddle.nn.functional.cross_entropy( fluid.dygraph.to_variable(self.logits), @@ -446,34 +473,38 @@ def test_cross_entropy_loss_soft_1d_mean(self): soft_label=True, axis=self.axis, weight=self.weight, - reduction=self.reduction) + reduction=self.reduction, + ) dy_ret_value = paddle_loss_mean.numpy() - #3. static + # 3. static paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[self.N, self.C], - dtype=self.dtype) - label = fluid.data(name='label', - shape=[self.N, self.C], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[self.N, self.C], dtype=self.dtype + ) + label = fluid.data( + name='label', shape=[self.N, self.C], dtype=self.dtype + ) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction=self.reduction, soft_label=True) + reduction=self.reduction, soft_label=True + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': self.logits, - 'label': self.labels - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={'input': self.logits, 'label': self.labels}, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) paddle.disable_static() @@ -484,10 +515,11 @@ def test_cross_entropy_loss_soft_1d_mean(self): def test_cross_entropy_loss_soft_1d_weight_mean(self): self.numeric_stable_mode = False self.soft_label = True - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.axis = -1 - self.ignore_index = -100 #should not be changed + self.ignore_index = -100 # should not be changed self.N = 4 self.C = 3 self.shape = [self.N, self.C] @@ -495,61 +527,72 @@ def test_cross_entropy_loss_soft_1d_weight_mean(self): self.reduction = 'mean' self.weight = np.random.uniform(0.1, 1.0, self.C).astype(self.dtype) self.logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, self.logits) self.labels = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) self.labels /= np.sum(self.labels, axis=self.axis, keepdims=True) - #1. numpy - expected = cross_entropy_soft(softmax, - self.labels, - self.axis, - self.N, - weight=self.weight, - reduction=self.reduction, - ignore_index=self.ignore_index) + # 1. numpy + expected = cross_entropy_soft( + softmax, + self.labels, + self.axis, + self.N, + weight=self.weight, + reduction=self.reduction, + ignore_index=self.ignore_index, + ) paddle.set_device("cpu") paddle.disable_static() - #2. dygraph + # 2. dygraph paddle_loss_none_weight = paddle.nn.functional.cross_entropy( fluid.dygraph.to_variable(self.logits), fluid.dygraph.to_variable(self.labels), soft_label=True, axis=self.axis, weight=fluid.dygraph.to_variable(self.weight), - reduction=self.reduction) + reduction=self.reduction, + ) dy_ret_value = paddle_loss_none_weight.numpy() - #3. static + # 3. static paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[self.N, self.C], - dtype=self.dtype) - label = fluid.data(name='label', - shape=[self.N, self.C], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[self.N, self.C], dtype=self.dtype + ) + label = fluid.data( + name='label', shape=[self.N, self.C], dtype=self.dtype + ) weight = fluid.data(name='weight', shape=[self.C], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction=self.reduction, soft_label=True) + weight=weight, reduction=self.reduction, soft_label=True + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': self.logits, - 'label': self.labels, - "weight": self.weight - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': self.logits, + 'label': self.labels, + "weight": self.weight, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) paddle.disable_static() @@ -558,14 +601,14 @@ def test_cross_entropy_loss_soft_1d_weight_mean(self): ###soft_label test 5 def test_cross_entropy_loss_soft_2d(self): - def inner_cross_entropy_loss_soft_2d(soft_label): self.numeric_stable_mode = False self.soft_label = soft_label - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.axis = -1 - self.ignore_index = -100 #should not be changed + self.ignore_index = -100 # should not be changed self.N = 3 self.H = 2 self.W = 2 @@ -575,64 +618,82 @@ def inner_cross_entropy_loss_soft_2d(soft_label): self.reduction = 'none' self.weight = None self.logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) - softmax = np.apply_along_axis(stable_softmax, self.axis, - self.logits) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) + softmax = np.apply_along_axis( + stable_softmax, self.axis, self.logits + ) - self.labels = np.random.uniform(0.1, 1.0, - self.shape).astype(self.dtype) + self.labels = np.random.uniform(0.1, 1.0, self.shape).astype( + self.dtype + ) self.labels /= np.sum(self.labels, axis=self.axis, keepdims=True) - #1. numpy - expected = cross_entropy_soft_2d(softmax, - self.labels, - self.axis, - self.N, - self.H, - self.W, - weight=self.weight, - reduction=self.reduction, - ignore_index=self.ignore_index) + # 1. numpy + expected = cross_entropy_soft_2d( + softmax, + self.labels, + self.axis, + self.N, + self.H, + self.W, + weight=self.weight, + reduction=self.reduction, + ignore_index=self.ignore_index, + ) paddle.set_device("cpu") paddle.disable_static() - #2. dygraph + # 2. dygraph paddle_loss_none_weight = paddle.nn.functional.cross_entropy( fluid.dygraph.to_variable(self.logits), fluid.dygraph.to_variable(self.labels), soft_label=True, axis=self.axis, weight=fluid.dygraph.to_variable(self.weight) - if self.weight is not None else None, - reduction=self.reduction) + if self.weight is not None + else None, + reduction=self.reduction, + ) dy_ret_value = paddle_loss_none_weight.numpy() - #3. static + # 3. static paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[self.N, self.H, self.W, self.C], - dtype=self.dtype) - label = fluid.data(name='label', - shape=[self.N, self.H, self.W, self.C], - dtype=self.dtype) + input = fluid.data( + name='input', + shape=[self.N, self.H, self.W, self.C], + dtype=self.dtype, + ) + label = fluid.data( + name='label', + shape=[self.N, self.H, self.W, self.C], + dtype=self.dtype, + ) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction=self.reduction, soft_label=True) + reduction=self.reduction, soft_label=True + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': self.logits, - 'label': self.labels, - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': self.logits, + 'label': self.labels, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) paddle.disable_static() @@ -647,10 +708,11 @@ def inner_cross_entropy_loss_soft_2d(soft_label): def test_cross_entropy_loss_soft_2d_weight_mean(self): self.numeric_stable_mode = False self.soft_label = True - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.axis = -1 - self.ignore_index = -100 #should not be changed + self.ignore_index = -100 # should not be changed self.N = 3 self.H = 2 self.W = 2 @@ -660,63 +722,78 @@ def test_cross_entropy_loss_soft_2d_weight_mean(self): self.reduction = 'mean' self.weight = np.random.uniform(0.1, 1.0, self.C).astype(self.dtype) self.logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, self.logits) self.labels = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) self.labels /= np.sum(self.labels, axis=self.axis, keepdims=True) - #1. numpy - expected = cross_entropy_soft_2d(softmax, - self.labels, - self.axis, - self.N, - self.H, - self.W, - weight=self.weight, - reduction=self.reduction, - ignore_index=self.ignore_index) + # 1. numpy + expected = cross_entropy_soft_2d( + softmax, + self.labels, + self.axis, + self.N, + self.H, + self.W, + weight=self.weight, + reduction=self.reduction, + ignore_index=self.ignore_index, + ) paddle.set_device("cpu") paddle.disable_static() - #2. dygraph + # 2. dygraph paddle_loss_none_weight = paddle.nn.functional.cross_entropy( fluid.dygraph.to_variable(self.logits), fluid.dygraph.to_variable(self.labels), soft_label=True, axis=self.axis, weight=fluid.dygraph.to_variable(self.weight), - reduction=self.reduction) + reduction=self.reduction, + ) dy_ret_value = paddle_loss_none_weight.numpy() - #3. static + # 3. static paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[self.N, self.H, self.W, self.C], - dtype=self.dtype) - label = fluid.data(name='label', - shape=[self.N, self.H, self.W, self.C], - dtype=self.dtype) + input = fluid.data( + name='input', + shape=[self.N, self.H, self.W, self.C], + dtype=self.dtype, + ) + label = fluid.data( + name='label', + shape=[self.N, self.H, self.W, self.C], + dtype=self.dtype, + ) weight = fluid.data(name='weight', shape=[self.C], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction=self.reduction, soft_label=True) + weight=weight, reduction=self.reduction, soft_label=True + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': self.logits, - 'label': self.labels, - "weight": self.weight - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': self.logits, + 'label': self.labels, + "weight": self.weight, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) paddle.disable_static() @@ -732,8 +809,11 @@ def test_cross_entropy_loss_1d_with_mean_ignore(self): paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[2, 4], dtype=self.dtype) label = fluid.data(name='label', shape=[2], dtype='int64') @@ -741,20 +821,25 @@ def test_cross_entropy_loss_1d_with_mean_ignore(self): ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) expected = cross_entropy_loss_1d(input_np, label_np)[0] with fluid.dygraph.guard(): - cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss(axis=1, - ignore_index=0) - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( + axis=1, ignore_index=0 + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = cross_entropy_loss_1d(input_np, label_np, ignore_index=0)[0] @@ -770,28 +855,37 @@ def test_cross_entropy_loss_1d_with_mean_ignore_negative(self): paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[N, C], dtype=self.dtype) label = fluid.data(name='label', shape=[N], dtype='int64') cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - ignore_index=-1) + ignore_index=-1 + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - axis=1, ignore_index=-1) - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + axis=1, ignore_index=-1 + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = cross_entropy_loss_1d(input_np, label_np, ignore_index=-1)[0] @@ -809,40 +903,49 @@ def test_cross_entropy_loss_1d_with_weight_mean_ignore(self): paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[N, C], dtype=self.dtype) label = fluid.data(name='label', shape=[N], dtype='int64') - weight = fluid.data(name='weight', shape=[C], - dtype=self.dtype) #weight for each class - cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss(weight=weight, - ignore_index=0) + weight = fluid.data( + name='weight', shape=[C], dtype=self.dtype + ) # weight for each class + cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( + weight=weight, ignore_index=0 + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( weight=fluid.dygraph.to_variable(weight_np), axis=1, - ignore_index=0) - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + ignore_index=0, + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_1d(input_np, - label_np, - weight=weight_np, - ignore_index=0)[0] + expected = cross_entropy_loss_1d( + input_np, label_np, weight=weight_np, ignore_index=0 + )[0] np.testing.assert_allclose(static_ret[0], dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret[0], expected, rtol=1e-05) @@ -858,172 +961,205 @@ def test_cross_entropy_loss_1d_with_weight_mean_ignore_exceedlabel(self): with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), ignore_index=255) - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), ignore_index=255 + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_1d(input_np, - label_np, - weight=weight_np, - ignore_index=255)[0] + expected = cross_entropy_loss_1d( + input_np, label_np, weight=weight_np, ignore_index=255 + )[0] np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_1d_with_weight_mean(self): input_np = np.random.random([2, 4]).astype(self.dtype) label_np = np.random.randint(0, 4, size=(2)).astype(np.int64) - weight_np = np.random.random([4]).astype(self.dtype) #shape:C + weight_np = np.random.random([4]).astype(self.dtype) # shape:C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[2, 4], dtype=self.dtype) label = fluid.data(name='label', shape=[2], dtype='int64') - weight = fluid.data(name='weight', shape=[4], - dtype=self.dtype) #weight for each class + weight = fluid.data( + name='weight', shape=[4], dtype=self.dtype + ) # weight for each class cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss(weight=weight) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) - expected = cross_entropy_loss_1d(input_np, label_np, - weight=weight_np)[0] + expected = cross_entropy_loss_1d(input_np, label_np, weight=weight_np)[ + 0 + ] with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), axis=1) - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), axis=1 + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_1d(input_np, label_np, - weight=weight_np)[0] + expected = cross_entropy_loss_1d(input_np, label_np, weight=weight_np)[ + 0 + ] np.testing.assert_allclose(static_ret[0], dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret[0], expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_1d_with_weight_sum(self): - input_np = np.random.random([100, 200]).astype(self.dtype) #N,C - label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) #N,1 - weight_np = np.random.random([200]).astype(self.dtype) #C + input_np = np.random.random([100, 200]).astype(self.dtype) # N,C + label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) # N,1 + weight_np = np.random.random([200]).astype(self.dtype) # C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype=self.dtype) label = fluid.data(name='label', shape=[100], dtype='int64') weight = fluid.data(name='weight', shape=[200], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction='sum') + weight=weight, reduction='sum' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), reduction='sum') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), reduction='sum' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_1d(input_np, - label_np, - weight=weight_np, - reduction='sum')[0] + expected = cross_entropy_loss_1d( + input_np, label_np, weight=weight_np, reduction='sum' + )[0] np.testing.assert_allclose(static_ret[0], dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret[0], expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_1d_with_weight_none(self): - input_np = np.random.random([100, 200]).astype(self.dtype) #N,C - label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) #N,1 - weight_np = np.random.random([200]).astype(self.dtype) #C + input_np = np.random.random([100, 200]).astype(self.dtype) # N,C + label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) # N,1 + weight_np = np.random.random([200]).astype(self.dtype) # C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype=self.dtype) label = fluid.data(name='label', shape=[100], dtype='int64') weight = fluid.data(name='weight', shape=[200], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction='none') + weight=weight, reduction='none' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) static_ret = np.squeeze(static_ret) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), reduction='none') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), reduction='none' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() dy_ret_value = np.squeeze(dy_ret_value) self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_1d(input_np, - label_np, - weight=weight_np, - reduction='none') + expected = cross_entropy_loss_1d( + input_np, label_np, weight=weight_np, reduction='none' + ) np.testing.assert_allclose(static_ret, dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret, expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_1d_with_weight_none_func(self): - input_np = np.random.random([100, 200]).astype(self.dtype) #N,C - label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) #N - weight_np = np.random.random([200]).astype(self.dtype) #C + input_np = np.random.random([100, 200]).astype(self.dtype) # N,C + label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) # N + weight_np = np.random.random([200]).astype(self.dtype) # C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype=self.dtype) label = fluid.data(name='label', shape=[100], dtype='int64') weight = fluid.data(name='weight', shape=[200], dtype=self.dtype) - ret = paddle.nn.functional.cross_entropy(input, - label, - weight=weight, - reduction='none') + ret = paddle.nn.functional.cross_entropy( + input, label, weight=weight, reduction='none' + ) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) static_ret = np.squeeze(static_ret) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): @@ -1031,43 +1167,47 @@ def test_cross_entropy_loss_1d_with_weight_none_func(self): fluid.dygraph.to_variable(input_np), fluid.dygraph.to_variable(label_np), weight=fluid.dygraph.to_variable(weight_np), - reduction='none') + reduction='none', + ) dy_ret_value = dy_ret.numpy() dy_ret_value = np.squeeze(dy_ret_value) self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_1d(input_np, - label_np, - weight=weight_np, - reduction='none') + expected = cross_entropy_loss_1d( + input_np, label_np, weight=weight_np, reduction='none' + ) np.testing.assert_allclose(static_ret, dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret, expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_1d_mean(self): - input_np = np.random.random([100, 200]).astype(self.dtype) #N,C - label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) #N,1 + input_np = np.random.random([100, 200]).astype(self.dtype) # N,C + label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) # N,1 paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype=self.dtype) label = fluid.data(name='label', shape=[100], dtype='int64') cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss() ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={'input': input_np, 'label': label_np}, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss() - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = cross_entropy_loss_1d(input_np, label_np)[0] @@ -1076,32 +1216,38 @@ def test_cross_entropy_loss_1d_mean(self): np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_1d_sum(self): - input_np = np.random.random([100, 200]).astype(self.dtype) #N,C - label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) #N,1 + input_np = np.random.random([100, 200]).astype(self.dtype) # N,C + label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) # N,1 paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype=self.dtype) label = fluid.data(name='label', shape=[100], dtype='int64') cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='sum') + reduction='sum' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={'input': input_np, 'label': label_np}, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='sum') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + reduction='sum' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = cross_entropy_loss_1d(input_np, label_np, reduction='sum')[0] @@ -1110,33 +1256,39 @@ def test_cross_entropy_loss_1d_sum(self): np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_1d_none(self): - input_np = np.random.random([100, 200]).astype(self.dtype) #N,C - label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) #N,1 + input_np = np.random.random([100, 200]).astype(self.dtype) # N,C + label_np = np.random.randint(0, 100, size=(100)).astype(np.int64) # N,1 paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype=self.dtype) label = fluid.data(name='label', shape=[100], dtype='int64') cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='none') + reduction='none' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={'input': input_np, 'label': label_np}, + fetch_list=[ret], + ) static_ret = np.squeeze(static_ret) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='none') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + reduction='none' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() dy_ret_value = np.squeeze(dy_ret_value) self.assertIsNotNone(dy_ret_value) @@ -1146,97 +1298,122 @@ def test_cross_entropy_loss_1d_none(self): np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_2d_with_weight_none(self): - input_np = np.random.random(size=(2, 2, 2, 3)).astype(self.dtype) #NHWC - label_np = np.random.randint(0, 3, - size=(2, 2, 2)).astype(np.int64) #NHW1 - weight_np = np.random.random(size=(3, )).astype(self.dtype) #C + input_np = np.random.random(size=(2, 2, 2, 3)).astype( + self.dtype + ) # NHWC + label_np = np.random.randint(0, 3, size=(2, 2, 2)).astype( + np.int64 + ) # NHW1 + weight_np = np.random.random(size=(3,)).astype(self.dtype) # C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 2, 2, 3], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[2, 2, 2, 3], dtype=self.dtype + ) label = fluid.data(name='label', shape=[2, 2, 2], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction='none') + weight=weight, reduction='none' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) static_ret = np.squeeze(static_ret) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), reduction='none') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), reduction='none' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() dy_ret_value = np.squeeze(dy_ret_value) self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_2d(input_np, - label_np, - weight=weight_np, - reduction='none') + expected = cross_entropy_loss_2d( + input_np, label_np, weight=weight_np, reduction='none' + ) np.testing.assert_allclose(static_ret, dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret, expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_2d_with_weight_axis_change_mean(self): - input_np = np.random.random(size=(2, 3, 2, 2)).astype(self.dtype) #NCHW - label_np = np.random.randint(0, 3, - size=(2, 2, 2)).astype(np.int64) #NHW - weight_np = np.random.random(size=(3, )).astype(self.dtype) #C + input_np = np.random.random(size=(2, 3, 2, 2)).astype( + self.dtype + ) # NCHW + label_np = np.random.randint(0, 3, size=(2, 2, 2)).astype( + np.int64 + ) # NHW + weight_np = np.random.random(size=(3,)).astype(self.dtype) # C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 3, 2, 2], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[2, 3, 2, 2], dtype=self.dtype + ) label = fluid.data(name='label', shape=[2, 2, 2], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction='mean', axis=1) + weight=weight, reduction='mean', axis=1 + ) # specify the class channels to axis 1 ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( weight=fluid.dygraph.to_variable(weight_np), reduction='mean', - axis=1) - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + axis=1, + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_2d(np.transpose(input_np, [0, 2, 3, 1]), - label_np, - weight=weight_np, - reduction='mean')[0] + expected = cross_entropy_loss_2d( + np.transpose(input_np, [0, 2, 3, 1]), + label_np, + weight=weight_np, + reduction='mean', + )[0] np.testing.assert_allclose(static_ret[0], dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret[0], expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) @@ -1252,137 +1429,173 @@ def test_cross_entropy_loss_2d_with_weight_mean_ignore_exceedlabel(self): weight_np = np.random.random([C]).astype(self.dtype) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), ignore_index=255) - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), ignore_index=255 + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_2d(input_np, - label_np, - weight=weight_np, - ignore_index=255)[0] + expected = cross_entropy_loss_2d( + input_np, label_np, weight=weight_np, ignore_index=255 + )[0] np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_2d_with_weight_mean(self): - input_np = np.random.random(size=(2, 2, 2, 3)).astype(self.dtype) #NHWC - label_np = np.random.randint(0, 3, - size=(2, 2, 2)).astype(np.int64) #NHW - weight_np = np.random.random(size=(3, )).astype(self.dtype) #C + input_np = np.random.random(size=(2, 2, 2, 3)).astype( + self.dtype + ) # NHWC + label_np = np.random.randint(0, 3, size=(2, 2, 2)).astype( + np.int64 + ) # NHW + weight_np = np.random.random(size=(3,)).astype(self.dtype) # C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 2, 2, 3], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[2, 2, 2, 3], dtype=self.dtype + ) label = fluid.data(name='label', shape=[2, 2, 2], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction='mean') + weight=weight, reduction='mean' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), reduction='mean') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), reduction='mean' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_2d(input_np, - label_np, - weight=weight_np, - reduction='mean')[0] + expected = cross_entropy_loss_2d( + input_np, label_np, weight=weight_np, reduction='mean' + )[0] np.testing.assert_allclose(static_ret[0], dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret[0], expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_2d_with_weight_sum(self): - input_np = np.random.random(size=(2, 2, 2, 3)).astype(self.dtype) #NHWC - label_np = np.random.randint(0, 3, - size=(2, 2, 2)).astype(np.int64) #NHW - weight_np = np.random.random(size=(3, )).astype(self.dtype) #C + input_np = np.random.random(size=(2, 2, 2, 3)).astype( + self.dtype + ) # NHWC + label_np = np.random.randint(0, 3, size=(2, 2, 2)).astype( + np.int64 + ) # NHW + weight_np = np.random.random(size=(3,)).astype(self.dtype) # C paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 2, 2, 3], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[2, 2, 2, 3], dtype=self.dtype + ) label = fluid.data(name='label', shape=[2, 2, 2], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype=self.dtype) cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight, reduction='sum') + weight=weight, reduction='sum' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=fluid.dygraph.to_variable(weight_np), reduction='sum') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + weight=fluid.dygraph.to_variable(weight_np), reduction='sum' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_2d(input_np, - label_np, - weight=weight_np, - reduction='sum')[0] + expected = cross_entropy_loss_2d( + input_np, label_np, weight=weight_np, reduction='sum' + )[0] np.testing.assert_allclose(static_ret[0], dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret[0], expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_2d_none(self): - input_np = np.random.random(size=(2, 2, 2, 3)).astype(self.dtype) #NHWC - label_np = np.random.randint(0, 3, - size=(2, 2, 2)).astype(np.int64) #NHW + input_np = np.random.random(size=(2, 2, 2, 3)).astype( + self.dtype + ) # NHWC + label_np = np.random.randint(0, 3, size=(2, 2, 2)).astype( + np.int64 + ) # NHW paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 2, 2, 3], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[2, 2, 2, 3], dtype=self.dtype + ) label = fluid.data(name='label', shape=[2, 2, 2], dtype='int64') cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='none') + reduction='none' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) static_ret = np.squeeze(static_ret) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='none') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + reduction='none' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() dy_ret_value = np.squeeze(dy_ret_value) self.assertIsNotNone(dy_ret_value) @@ -1392,75 +1605,100 @@ def test_cross_entropy_loss_2d_none(self): np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_2d_mean(self): - input_np = np.random.random(size=(2, 2, 2, 3)).astype(self.dtype) #NHWC - label_np = np.random.randint(0, 3, - size=(2, 2, 2)).astype(np.int64) #NHW + input_np = np.random.random(size=(2, 2, 2, 3)).astype( + self.dtype + ) # NHWC + label_np = np.random.randint(0, 3, size=(2, 2, 2)).astype( + np.int64 + ) # NHW paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 2, 2, 3], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[2, 2, 2, 3], dtype=self.dtype + ) label = fluid.data(name='label', shape=[2, 2, 2], dtype='int64') cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='mean') + reduction='mean' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='mean') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + reduction='mean' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = cross_entropy_loss_2d(input_np, label_np, - reduction='mean')[0] + expected = cross_entropy_loss_2d(input_np, label_np, reduction='mean')[ + 0 + ] np.testing.assert_allclose(static_ret[0], dy_ret_value, rtol=1e-05) np.testing.assert_allclose(static_ret[0], expected, rtol=1e-05) np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_cross_entropy_loss_2d_sum(self): - input_np = np.random.random(size=(2, 2, 2, 3)).astype(self.dtype) #NHWC - label_np = np.random.randint(0, 3, - size=(2, 2, 2)).astype(np.int64) #NHW + input_np = np.random.random(size=(2, 2, 2, 3)).astype( + self.dtype + ) # NHWC + label_np = np.random.randint(0, 3, size=(2, 2, 2)).astype( + np.int64 + ) # NHW paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 2, 2, 3], - dtype=self.dtype) + input = fluid.data( + name='input', shape=[2, 2, 2, 3], dtype=self.dtype + ) label = fluid.data(name='label', shape=[2, 2, 2], dtype='int64') cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='sum') + reduction='sum' + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - reduction='sum') - dy_ret = cross_entropy_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + reduction='sum' + ) + dy_ret = cross_entropy_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = cross_entropy_loss_2d(input_np, label_np, reduction='sum')[0] @@ -1486,8 +1724,7 @@ def test_other_dygraph_api(self): self.test_cross_entropy_loss_1d_with_mean_ignore() self.test_cross_entropy_loss_1d_with_mean_ignore_negative() self.test_cross_entropy_loss_1d_with_weight_mean_ignore() - self.test_cross_entropy_loss_1d_with_weight_mean_ignore_exceedlabel( - ) + self.test_cross_entropy_loss_1d_with_weight_mean_ignore_exceedlabel() self.test_cross_entropy_loss_1d_with_weight_mean() self.test_cross_entropy_loss_1d_with_weight_sum() self.test_cross_entropy_loss_1d_with_weight_none() @@ -1497,8 +1734,7 @@ def test_other_dygraph_api(self): self.test_cross_entropy_loss_1d_none() self.test_cross_entropy_loss_2d_with_weight_none() self.test_cross_entropy_loss_2d_with_weight_axis_change_mean() - self.test_cross_entropy_loss_2d_with_weight_mean_ignore_exceedlabel( - ) + self.test_cross_entropy_loss_2d_with_weight_mean_ignore_exceedlabel() self.test_cross_entropy_loss_2d_with_weight_mean() self.test_cross_entropy_loss_2d_with_weight_sum() self.test_cross_entropy_loss_2d_none() @@ -1507,51 +1743,53 @@ def test_other_dygraph_api(self): class TestCrossEntropyFAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_WeightLength_NotEqual(): input_data = paddle.rand(shape=[20, 100]) - label_data = paddle.randint(0, - 100, - shape=[20, 1], - dtype="int64") + label_data = paddle.randint( + 0, 100, shape=[20, 1], dtype="int64" + ) weight_data = paddle.rand([100 + 1]) - paddle.nn.functional.cross_entropy(input=input_data, - label=label_data, - weight=weight_data, - ignore_index=-100) + paddle.nn.functional.cross_entropy( + input=input_data, + label=label_data, + weight=weight_data, + ignore_index=-100, + ) self.assertRaises(ValueError, test_WeightLength_NotEqual) def test_LabelValue_ExceedMax(): input_data = paddle.rand(shape=[20, 100]) - label_data = paddle.randint(0, - 100, - shape=[20, 1], - dtype="int64") + label_data = paddle.randint( + 0, 100, shape=[20, 1], dtype="int64" + ) label_data[0] = 100 weight_data = paddle.rand([100]) - paddle.nn.functional.cross_entropy(input=input_data, - label=label_data, - weight=weight_data, - ignore_index=-100) + paddle.nn.functional.cross_entropy( + input=input_data, + label=label_data, + weight=weight_data, + ignore_index=-100, + ) self.assertRaises(ValueError, test_LabelValue_ExceedMax) def test_LabelValue_ExceedMin(): input_data = paddle.rand(shape=[20, 100]) - label_data = paddle.randint(0, - 100, - shape=[20, 1], - dtype="int64") + label_data = paddle.randint( + 0, 100, shape=[20, 1], dtype="int64" + ) label_data[0] = -1 weight_data = paddle.rand([100]) - paddle.nn.functional.cross_entropy(input=input_data, - label=label_data, - weight=weight_data, - ignore_index=-100) + paddle.nn.functional.cross_entropy( + input=input_data, + label=label_data, + weight=weight_data, + ignore_index=-100, + ) self.assertRaises(ValueError, test_LabelValue_ExceedMin) @@ -1562,28 +1800,34 @@ def static_test_WeightLength_NotEqual(): paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace(0) if fluid.core.is_compiled_with_cuda( - ) else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[2, 4], - dtype='float32') + input = fluid.data( + name='input', shape=[2, 4], dtype='float32' + ) label = fluid.data(name='label', shape=[2], dtype='int64') - weight = fluid.data(name='weight', - shape=[3], - dtype='float32') #weight for each class + weight = fluid.data( + name='weight', shape=[3], dtype='float32' + ) # weight for each class cross_entropy_loss = paddle.nn.loss.CrossEntropyLoss( - weight=weight) + weight=weight + ) ret = cross_entropy_loss(input, label) exe = fluid.Executor(place) - static_ret = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - "weight": weight_np - }, - fetch_list=[ret]) + static_ret = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + "weight": weight_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) self.assertRaises(ValueError, static_test_WeightLength_NotEqual) diff --git a/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py b/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py index a0f7430262887..c9fe88f318e88 100644 --- a/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py +++ b/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py @@ -21,8 +21,7 @@ class TestCrossEntropyOp(OpTest): - """Test cross-entropy with discrete one-hot labels. - """ + """Test cross-entropy with discrete one-hot labels.""" def setUp(self): self.op_type = "cross_entropy" @@ -43,24 +42,27 @@ def setUp(self): self.outputs = {"Y": self.cross_entropy} self.attrs = { "soft_label": self.soft_label, - "ignore_index": self.ignore_index + "ignore_index": self.ignore_index, } def init_x(self): - self.x = randomize_probability(self.batch_size, - self.class_num, - dtype=self.dtype) + self.x = randomize_probability( + self.batch_size, self.class_num, dtype=self.dtype + ) def init_label(self): - self.label = np.random.randint(0, - self.class_num, (self.batch_size, 1), - dtype="int64") + self.label = np.random.randint( + 0, self.class_num, (self.batch_size, 1), dtype="int64" + ) def get_cross_entropy(self): self.cross_entropy = np.asmatrix( - [[-np.log(self.x[i][self.label[i][0]])] - for i in range(self.x.shape[0])], - dtype="float64") + [ + [-np.log(self.x[i][self.label[i][0]])] + for i in range(self.x.shape[0]) + ], + dtype="float64", + ) def init_attr_type(self): pass @@ -79,32 +81,35 @@ def test_check_grad(self): class TestCrossEntropyOpRemoveLastDim(TestCrossEntropyOp): - """Test cross-entropy with discrete one-hot labels with shape [batch_size] - """ + """Test cross-entropy with discrete one-hot labels with shape [batch_size]""" def init_label(self): - self.label = np.random.randint(0, - self.class_num, (self.batch_size), - dtype="int64") + self.label = np.random.randint( + 0, self.class_num, (self.batch_size), dtype="int64" + ) def get_cross_entropy(self): self.cross_entropy = np.asmatrix( [-np.log(self.x[i][self.label[i]]) for i in range(self.x.shape[0])], - dtype="float64") + dtype="float64", + ) class TestCrossEntropyOp2(TestCrossEntropyOp): - """Test cross-entropy with vectorized soft labels. - """ + """Test cross-entropy with vectorized soft labels.""" def init_label(self): self.label = np.random.uniform( - 0.1, 1.0, [self.batch_size, self.class_num]).astype(self.dtype) + 0.1, 1.0, [self.batch_size, self.class_num] + ).astype(self.dtype) self.label /= self.label.sum(axis=1, keepdims=True) def get_cross_entropy(self): - self.cross_entropy = (-self.label * np.log(self.x)).sum( - axis=1, keepdims=True).astype(self.dtype) + self.cross_entropy = ( + (-self.label * np.log(self.x)) + .sum(axis=1, keepdims=True) + .astype(self.dtype) + ) def init_attr_type(self): self.soft_label = True @@ -117,26 +122,28 @@ def init_bs_class_num(self): self.class_num = 37 def test_check_grad(self): - self.check_grad(["X"], - "Y", - max_relative_error=0.05, - numeric_grad_delta=0.001) + self.check_grad( + ["X"], "Y", max_relative_error=0.05, numeric_grad_delta=0.001 + ) class TestCrossEntropyOp3(TestCrossEntropyOp): - """Test cross-entropy with vectorized one-hot representation of labels. - """ + """Test cross-entropy with vectorized one-hot representation of labels.""" def init_label(self): - self.label_index = np.random.randint(0, self.class_num, - (self.batch_size)) + self.label_index = np.random.randint( + 0, self.class_num, (self.batch_size) + ) self.label = np.zeros(self.x.shape).astype(self.dtype) self.label[np.arange(self.batch_size), self.label_index] = 1 def get_cross_entropy(self): self.cross_entropy = np.asmatrix( - [[-np.log(self.x[i][self.label_index[i]])] - for i in range(self.x.shape[0])]).astype(self.dtype) + [ + [-np.log(self.x[i][self.label_index[i]])] + for i in range(self.x.shape[0]) + ] + ).astype(self.dtype) def init_attr_type(self): self.soft_label = True @@ -149,35 +156,38 @@ def init_bs_class_num(self): self.class_num = 27 def test_check_grad(self): - self.check_grad(["X"], - "Y", - max_relative_error=0.05, - numeric_grad_delta=0.001) + self.check_grad( + ["X"], "Y", max_relative_error=0.05, numeric_grad_delta=0.001 + ) class TestCrossEntropyOp4(TestCrossEntropyOp): - """Test high rank tensor cross-entropy with discrete one-hot labels. - """ + """Test high rank tensor cross-entropy with discrete one-hot labels.""" def init_x(self): self.shape = [10, 2, 4] self.ins_num = np.prod(np.array(self.shape)) - self.X_2d = randomize_probability(self.ins_num, - self.class_num).astype(self.dtype) + self.X_2d = randomize_probability(self.ins_num, self.class_num).astype( + self.dtype + ) self.x = self.X_2d.reshape(self.shape + [self.class_num]) def init_label(self): - self.label_2d = np.random.randint(0, - self.class_num, (self.ins_num, 1), - dtype="int64") + self.label_2d = np.random.randint( + 0, self.class_num, (self.ins_num, 1), dtype="int64" + ) self.label = self.label_2d.reshape(self.shape + [1]) def get_cross_entropy(self): cross_entropy_2d = np.asmatrix( - [[-np.log(self.X_2d[i][self.label_2d[i][0]])] - for i in range(self.X_2d.shape[0])]).astype(self.dtype) - self.cross_entropy = np.array(cross_entropy_2d).reshape(self.shape + - [1]) + [ + [-np.log(self.X_2d[i][self.label_2d[i][0]])] + for i in range(self.X_2d.shape[0]) + ] + ).astype(self.dtype) + self.cross_entropy = np.array(cross_entropy_2d).reshape( + self.shape + [1] + ) def init_attr_type(self): self.soft_label = False @@ -190,44 +200,51 @@ def init_bs_class_num(self): class TestCrossEntropyOp4RemoveLastDim(TestCrossEntropyOp4): - """Test high rank tensor cross-entropy with discrete one-hot labels with shape [batch_size] - """ + """Test high rank tensor cross-entropy with discrete one-hot labels with shape [batch_size]""" def init_label(self): - self.label_2d = np.random.randint(0, - self.class_num, (self.ins_num, 1), - dtype="int64") + self.label_2d = np.random.randint( + 0, self.class_num, (self.ins_num, 1), dtype="int64" + ) self.label = self.label_2d.reshape(self.shape) def get_cross_entropy(self): cross_entropy_2d = np.asmatrix( - [[-np.log(self.X_2d[i][self.label_2d[i][0]])] - for i in range(self.X_2d.shape[0])]).astype(self.dtype) + [ + [-np.log(self.X_2d[i][self.label_2d[i][0]])] + for i in range(self.X_2d.shape[0]) + ] + ).astype(self.dtype) self.cross_entropy = np.array(cross_entropy_2d).reshape(self.shape) class TestCrossEntropyOp5(TestCrossEntropyOp): - """Test high rank tensor cross-entropy with vectorized soft labels. - """ + """Test high rank tensor cross-entropy with vectorized soft labels.""" def init_x(self): self.shape = [4, 3] self.ins_num = np.prod(np.array(self.shape)) - self.X_2d = randomize_probability(self.ins_num, - self.class_num).astype(self.dtype) + self.X_2d = randomize_probability(self.ins_num, self.class_num).astype( + self.dtype + ) self.x = self.X_2d.reshape(self.shape + [self.class_num]) def init_label(self): self.label_2d = np.random.uniform( - 0.1, 1.0, [self.ins_num, self.class_num]).astype(self.dtype) + 0.1, 1.0, [self.ins_num, self.class_num] + ).astype(self.dtype) self.label_2d /= self.label_2d.sum(axis=1, keepdims=True) self.label = self.label_2d.reshape(self.shape + [self.class_num]) def get_cross_entropy(self): - cross_entropy_2d = (-self.label_2d * np.log(self.X_2d)).sum( - axis=1, keepdims=True).astype(self.dtype) - self.cross_entropy = np.array(cross_entropy_2d).reshape(self.shape + - [1]) + cross_entropy_2d = ( + (-self.label_2d * np.log(self.X_2d)) + .sum(axis=1, keepdims=True) + .astype(self.dtype) + ) + self.cross_entropy = np.array(cross_entropy_2d).reshape( + self.shape + [1] + ) def init_attr_type(self): self.soft_label = True @@ -239,39 +256,44 @@ def init_bs_class_num(self): self.class_num = 37 def test_check_grad(self): - self.check_grad(["X"], - "Y", - max_relative_error=0.05, - numeric_grad_delta=0.001) + self.check_grad( + ["X"], "Y", max_relative_error=0.05, numeric_grad_delta=0.001 + ) class TestCrossEntropyOp6(TestCrossEntropyOp): - """Test high rank tensor cross-entropy with vectorized one-hot representation of labels. - """ + """Test high rank tensor cross-entropy with vectorized one-hot representation of labels.""" def init_x(self): self.shape = [4, 3, 2] self.ins_num = np.prod(np.array(self.shape)) - self.X_2d = randomize_probability(self.ins_num, - self.class_num).astype(self.dtype) + self.X_2d = randomize_probability(self.ins_num, self.class_num).astype( + self.dtype + ) self.x = self.X_2d.reshape(self.shape + [self.class_num]) def init_label(self): - self.label_index_2d = np.random.randint(0, - self.class_num, (self.ins_num), - dtype="int64") + self.label_index_2d = np.random.randint( + 0, self.class_num, (self.ins_num), dtype="int64" + ) label_2d = np.zeros(self.X_2d.shape) label_2d[np.arange(self.ins_num), self.label_index_2d] = 1 self.label = label_2d.reshape(self.shape + [self.class_num]).astype( - self.dtype) + self.dtype + ) def get_cross_entropy(self): cross_entropy_2d = np.asmatrix( - [[-np.log(self.X_2d[i][self.label_index_2d[i]])] - for i in range(self.X_2d.shape[0])]) - self.cross_entropy = np.array(cross_entropy_2d).reshape(self.shape + - [1]).astype( - self.dtype) + [ + [-np.log(self.X_2d[i][self.label_index_2d[i]])] + for i in range(self.X_2d.shape[0]) + ] + ) + self.cross_entropy = ( + np.array(cross_entropy_2d) + .reshape(self.shape + [1]) + .astype(self.dtype) + ) def init_attr_type(self): self.soft_label = True @@ -283,26 +305,28 @@ def init_bs_class_num(self): self.class_num = 17 def test_check_grad(self): - self.check_grad(["X"], - "Y", - max_relative_error=0.05, - numeric_grad_delta=0.001) + self.check_grad( + ["X"], "Y", max_relative_error=0.05, numeric_grad_delta=0.001 + ) class TestCrossEntropyOp7(TestCrossEntropyOp): - """Test cross-entropy with ignore index. - """ + """Test cross-entropy with ignore index.""" def init_label(self): - self.label = np.random.randint(0, - self.class_num, (self.batch_size, 1), - dtype="int64") + self.label = np.random.randint( + 0, self.class_num, (self.batch_size, 1), dtype="int64" + ) def get_cross_entropy(self): self.cross_entropy = np.asmatrix( - [[-np.log(self.x[i][self.label[i][0]])] - if self.label[i][0] != self.ignore_index else [0] - for i in range(self.x.shape[0])]).astype(self.dtype) + [ + [-np.log(self.x[i][self.label[i][0]])] + if self.label[i][0] != self.ignore_index + else [0] + for i in range(self.x.shape[0]) + ] + ).astype(self.dtype) def init_attr_type(self): self.soft_label = False @@ -317,30 +341,35 @@ def init_bs_class_num(self): class TestCrossEntropyOp7RemoveLastDim(TestCrossEntropyOp7): - """Test cross-entropy with ignore index with shape [batch_size]. - """ + """Test cross-entropy with ignore index with shape [batch_size].""" def init_label(self): - self.label = np.random.randint(0, - self.class_num, (self.batch_size), - dtype="int64") + self.label = np.random.randint( + 0, self.class_num, (self.batch_size), dtype="int64" + ) def get_cross_entropy(self): self.cross_entropy = np.asmatrix( - [[-np.log(self.x[i][self.label[i]])] - if self.label[i] != self.ignore_index else [0] - for i in range(self.x.shape[0])]).astype(self.dtype) - self.cross_entropy = np.array(self.cross_entropy).reshape( - [self.batch_size]).astype(self.dtype) + [ + [-np.log(self.x[i][self.label[i]])] + if self.label[i] != self.ignore_index + else [0] + for i in range(self.x.shape[0]) + ] + ).astype(self.dtype) + self.cross_entropy = ( + np.array(self.cross_entropy) + .reshape([self.batch_size]) + .astype(self.dtype) + ) # Add Fp16 test def create_test_class(parent, cls_name): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCrossEntropyFP16Op(parent): - def init_dtype_type(self): return np.float16 @@ -352,9 +381,9 @@ def test_check_output(self): def test_check_grad(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['X'], - 'Y', - max_relative_error=0.9) + self.check_grad_with_place( + place, ['X'], 'Y', max_relative_error=0.9 + ) cls_name = "{0}".format(cls_name) TestCrossEntropyFP16Op.__name__ = cls_name @@ -362,29 +391,32 @@ def test_check_grad(self): create_test_class(TestCrossEntropyOp, "TestCrossEntropyF16Op") -#create_test_class(TestCrossEntropyOp2, "TestCrossEntropyF16Op2") +# create_test_class(TestCrossEntropyOp2, "TestCrossEntropyF16Op2") create_test_class(TestCrossEntropyOp3, "TestCrossEntropyF16Op3") create_test_class(TestCrossEntropyOp4, "TestCrossEntropyF16Op4") -create_test_class(TestCrossEntropyOp4RemoveLastDim, - "TestCrossEntropyF16Op4RemoveLastDim") -#create_test_class(TestCrossEntropyOp5, "TestCrossEntropyF16Op5") +create_test_class( + TestCrossEntropyOp4RemoveLastDim, "TestCrossEntropyF16Op4RemoveLastDim" +) +# create_test_class(TestCrossEntropyOp5, "TestCrossEntropyF16Op5") create_test_class(TestCrossEntropyOp6, "TestCrossEntropyF16Op6") create_test_class(TestCrossEntropyOp7, "TestCrossEntropyF16Op7") -create_test_class(TestCrossEntropyOp7RemoveLastDim, - "TestCrossEntropyF16Op7RemoveLastDim") +create_test_class( + TestCrossEntropyOp7RemoveLastDim, "TestCrossEntropyF16Op7RemoveLastDim" +) class TestCrossEntropyOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of cross_entropy must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) - lab1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + lab1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) fluid.layers.cross_entropy(x1, lab1) self.assertRaises(TypeError, test_Variable) @@ -392,12 +424,12 @@ def test_Variable(): def test_dtype(): # the input dtype of cross_entropy must be float16 or float32 or float64 # float16 only can be set on GPU place - x2 = fluid.layers.data(name='x2', - shape=[3, 4, 5, 6], - dtype="int32") - lab2 = fluid.layers.data(name='lab2', - shape=[3, 4, 5, 6], - dtype="int32") + x2 = fluid.layers.data( + name='x2', shape=[3, 4, 5, 6], dtype="int32" + ) + lab2 = fluid.layers.data( + name='lab2', shape=[3, 4, 5, 6], dtype="int32" + ) fluid.layers.cross_entropy(x2, lab2) self.assertRaises(TypeError, test_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_cross_op.py b/python/paddle/fluid/tests/unittests/test_cross_op.py index 7ff724d2b5a53..73580d6fc8fcd 100644 --- a/python/paddle/fluid/tests/unittests/test_cross_op.py +++ b/python/paddle/fluid/tests/unittests/test_cross_op.py @@ -21,14 +21,13 @@ class TestCrossOp(OpTest): - def setUp(self): self.op_type = "cross" self.python_api = paddle.cross self.initTestCase() self.inputs = { 'X': np.random.random(self.shape).astype(self.dtype), - 'Y': np.random.random(self.shape).astype(self.dtype) + 'Y': np.random.random(self.shape).astype(self.dtype), } self.init_output() @@ -53,7 +52,6 @@ def test_check_grad_normal(self): class TestCrossOpCase1(TestCrossOp): - def initTestCase(self): self.shape = (2048, 3) self.dtype = np.float32 @@ -66,12 +64,13 @@ def init_output(self): class TestCrossAPI(unittest.TestCase): - def input_data(self): - self.data_x = np.array([[1.0, 1.0, 1.0], [2.0, 2.0, 2.0], - [3.0, 3.0, 3.0]]) - self.data_y = np.array([[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], - [1.0, 1.0, 1.0]]) + self.data_x = np.array( + [[1.0, 1.0, 1.0], [2.0, 2.0, 2.0], [3.0, 3.0, 3.0]] + ) + self.data_y = np.array( + [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]] + ) def test_cross_api(self): self.input_data() @@ -82,14 +81,14 @@ def test_cross_api(self): y = fluid.layers.data(name='y', shape=[-1, 3]) z = paddle.cross(x, y, axis=1) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={ - 'x': self.data_x, - 'y': self.data_y - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0]]) + (res,) = exe.run( + feed={'x': self.data_x, 'y': self.data_y}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) # case 2: @@ -98,14 +97,14 @@ def test_cross_api(self): y = fluid.layers.data(name='y', shape=[-1, 3]) z = paddle.cross(x, y) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={ - 'x': self.data_x, - 'y': self.data_y - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[-1.0, -1.0, -1.0], [2.0, 2.0, 2.0], - [-1.0, -1.0, -1.0]]) + (res,) = exe.run( + feed={'x': self.data_x, 'y': self.data_y}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[-1.0, -1.0, -1.0], [2.0, 2.0, 2.0], [-1.0, -1.0, -1.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) # case 3: @@ -134,8 +133,9 @@ def test_dygraph_api(self): y = fluid.dygraph.to_variable(self.data_y) z = paddle.cross(x, y, axis=1) np_z = z.numpy() - expect_out = np.array([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0]]) + expect_out = np.array( + [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]] + ) np.testing.assert_allclose(expect_out, np_z, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_crypto.py b/python/paddle/fluid/tests/unittests/test_crypto.py index 120a4588773a6..147615905fd85 100644 --- a/python/paddle/fluid/tests/unittests/test_crypto.py +++ b/python/paddle/fluid/tests/unittests/test_crypto.py @@ -19,7 +19,6 @@ class CipherUtilsTestCase(unittest.TestCase): - def test_gen_key(self): key1 = CipherUtils.gen_key(256) key2 = CipherUtils.gen_key_to_file(256, "paddle_aes_test.keyfile") @@ -31,7 +30,6 @@ def test_gen_key(self): class CipherTestCase(unittest.TestCase): - def test_aes_cipher(self): plaintext = "hello world" key = CipherUtils.gen_key(256) diff --git a/python/paddle/fluid/tests/unittests/test_ctc_align.py b/python/paddle/fluid/tests/unittests/test_ctc_align.py index 38e4a0721c6ef..b11dfe0254a25 100644 --- a/python/paddle/fluid/tests/unittests/test_ctc_align.py +++ b/python/paddle/fluid/tests/unittests/test_ctc_align.py @@ -28,8 +28,9 @@ def CTCAlign(input, lod, blank, merge_repeated, padding=0, input_length=None): prev_token = -1 for j in range(cur_offset, cur_offset + lod0[i]): token = input[j][0] - if (token != blank) and not (merge_repeated - and token == prev_token): + if (token != blank) and not ( + merge_repeated and token == prev_token + ): result.append(token) prev_token = token cur_offset += lod0[i] @@ -44,37 +45,44 @@ def CTCAlign(input, lod, blank, merge_repeated, padding=0, input_length=None): prev_token = -1 for j in range(input_length[i][0]): token = input[i][j] - if (token != blank) and not (merge_repeated - and token == prev_token): + if (token != blank) and not ( + merge_repeated and token == prev_token + ): result[i].append(token) prev_token = token start = len(result[i]) output_length.append([start]) for j in range(start, len(input[i])): result[i].append(padding) - result = np.array(result).reshape([len(input), - len(input[0])]).astype("int32") - output_length = np.array(output_length).reshape([len(input), - 1]).astype("int32") + result = ( + np.array(result) + .reshape([len(input), len(input[0])]) + .astype("int32") + ) + output_length = ( + np.array(output_length).reshape([len(input), 1]).astype("int32") + ) return result, output_length class TestCTCAlignOp(OpTest): - def config(self): self.op_type = "ctc_align" self.input_lod = [[11, 7]] self.blank = 0 self.merge_repeated = False - self.input = np.array( - [0, 1, 2, 2, 0, 4, 0, 4, 5, 0, 6, 6, 0, 0, 7, 7, 7, - 0]).reshape([18, 1]).astype("int32") + self.input = ( + np.array([0, 1, 2, 2, 0, 4, 0, 4, 5, 0, 6, 6, 0, 0, 7, 7, 7, 0]) + .reshape([18, 1]) + .astype("int32") + ) def setUp(self): self.config() - output = CTCAlign(self.input, self.input_lod, self.blank, - self.merge_repeated) + output = CTCAlign( + self.input, self.input_lod, self.blank, self.merge_repeated + ) self.inputs = { "Input": (self.input, self.input_lod), @@ -82,7 +90,7 @@ def setUp(self): self.outputs = {"Output": output} self.attrs = { "blank": self.blank, - "merge_repeated": self.merge_repeated + "merge_repeated": self.merge_repeated, } def test_check_output(self): @@ -90,19 +98,19 @@ def test_check_output(self): class TestCTCAlignOpCase1(TestCTCAlignOp): - def config(self): self.op_type = "ctc_align" self.input_lod = [[11, 8]] self.blank = 0 self.merge_repeated = True - self.input = np.array( - [0, 1, 2, 2, 0, 4, 0, 4, 5, 0, 6, 6, 0, 0, 7, 7, 7, 0, - 0]).reshape([19, 1]).astype("int32") + self.input = ( + np.array([0, 1, 2, 2, 0, 4, 0, 4, 5, 0, 6, 6, 0, 0, 7, 7, 7, 0, 0]) + .reshape([19, 1]) + .astype("int32") + ) class TestCTCAlignOpCase2(TestCTCAlignOp): - def config(self): self.op_type = "ctc_align" self.input_lod = [[4]] @@ -112,32 +120,43 @@ def config(self): class TestCTCAlignPaddingOp(OpTest): - def config(self): self.op_type = "ctc_align" self.input_lod = [] self.blank = 0 self.padding_value = 0 self.merge_repeated = True - self.input = np.array([[0, 2, 4, 4, 0, 6, 3, 6, 6, 0, 0], - [1, 1, 3, 0, 0, 4, 5, 6, 0, 0, - 0]]).reshape([2, 11]).astype("int32") + self.input = ( + np.array( + [ + [0, 2, 4, 4, 0, 6, 3, 6, 6, 0, 0], + [1, 1, 3, 0, 0, 4, 5, 6, 0, 0, 0], + ] + ) + .reshape([2, 11]) + .astype("int32") + ) self.input_length = np.array([[9], [8]]).reshape([2, 1]).astype("int32") def setUp(self): self.config() - output, output_length = CTCAlign(self.input, self.input_lod, self.blank, - self.merge_repeated, - self.padding_value, self.input_length) + output, output_length = CTCAlign( + self.input, + self.input_lod, + self.blank, + self.merge_repeated, + self.padding_value, + self.input_length, + ) self.inputs = { "Input": (self.input, self.input_lod), - "InputLength": self.input_length + "InputLength": self.input_length, } self.outputs = {"Output": output, "OutputLength": output_length} self.attrs = { "blank": self.blank, "merge_repeated": self.merge_repeated, - "padding_value": self.padding_value + "padding_value": self.padding_value, } def test_check_output(self): @@ -145,18 +164,22 @@ def test_check_output(self): class TestCTCAlignOpCase3(TestCTCAlignPaddingOp): - def config(self): self.op_type = "ctc_align" self.blank = 0 self.input_lod = [] self.merge_repeated = True self.padding_value = 0 - self.input = np.array([[0, 1, 2, 2, 0, 4], [0, 4, 5, 0, 6, 0], - [0, 7, 7, 7, 0, 0]]).reshape([3, - 6]).astype("int32") - self.input_length = np.array([[6], [5], - [4]]).reshape([3, 1]).astype("int32") + self.input = ( + np.array( + [[0, 1, 2, 2, 0, 4], [0, 4, 5, 0, 6, 0], [0, 7, 7, 7, 0, 0]] + ) + .reshape([3, 6]) + .astype("int32") + ) + self.input_length = ( + np.array([[6], [5], [4]]).reshape([3, 1]).astype("int32") + ) class TestCTCAlignOpCase4(TestCTCAlignPaddingOp): @@ -170,30 +193,38 @@ def config(self): self.input_lod = [] self.merge_repeated = False self.padding_value = 0 - self.input = np.array([[0, 1, 2, 2, 0, 4], [0, 4, 5, 0, 6, 0], - [0, 7, 7, 7, 0, 0]]).reshape([3, - 6]).astype("int32") - self.input_length = np.array([[6], [5], - [4]]).reshape([3, 1]).astype("int32") + self.input = ( + np.array( + [[0, 1, 2, 2, 0, 4], [0, 4, 5, 0, 6, 0], [0, 7, 7, 7, 0, 0]] + ) + .reshape([3, 6]) + .astype("int32") + ) + self.input_length = ( + np.array([[6], [5], [4]]).reshape([3, 1]).astype("int32") + ) class TestCTCAlignOpCase5(TestCTCAlignPaddingOp): - def config(self): self.op_type = "ctc_align" self.blank = 0 self.input_lod = [] self.merge_repeated = False self.padding_value = 1 - self.input = np.array([[0, 1, 2, 2, 0, 4], [0, 4, 5, 0, 6, 0], - [0, 7, 1, 7, 0, 0]]).reshape([3, - 6]).astype("int32") - self.input_length = np.array([[6], [5], - [4]]).reshape([3, 1]).astype("int32") + self.input = ( + np.array( + [[0, 1, 2, 2, 0, 4], [0, 4, 5, 0, 6, 0], [0, 7, 1, 7, 0, 0]] + ) + .reshape([3, 6]) + .astype("int32") + ) + self.input_length = ( + np.array([[6], [5], [4]]).reshape([3, 1]).astype("int32") + ) class TestCTCAlignOpApi(unittest.TestCase): - def test_api(self): x = fluid.layers.data('x', shape=[4], dtype='float32') y = fluid.layers.ctc_greedy_decoder(x, blank=0) @@ -201,11 +232,13 @@ def test_api(self): x_pad = fluid.layers.data('x_pad', shape=[4, 4], dtype='float32') x_pad_len = fluid.layers.data('x_pad_len', shape=[1], dtype='int64') y_pad, y_pad_len = fluid.layers.ctc_greedy_decoder( - x_pad, blank=0, input_length=x_pad_len) + x_pad, blank=0, input_length=x_pad_len + ) place = fluid.CPUPlace() x_tensor = fluid.create_lod_tensor( - np.random.rand(8, 4).astype("float32"), [[4, 4]], place) + np.random.rand(8, 4).astype("float32"), [[4, 4]], place + ) x_pad_tensor = np.random.rand(2, 4, 4).astype("float32") x_pad_len_tensor = np.array([[4], [4]]).reshape([2, 1]).astype("int64") @@ -213,17 +246,18 @@ def test_api(self): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'x': x_tensor, - 'x_pad': x_pad_tensor, - 'x_pad_len': x_pad_len_tensor - }, - fetch_list=[y, y_pad, y_pad_len], - return_numpy=False) + ret = exe.run( + feed={ + 'x': x_tensor, + 'x_pad': x_pad_tensor, + 'x_pad_len': x_pad_len_tensor, + }, + fetch_list=[y, y_pad, y_pad_len], + return_numpy=False, + ) class BadInputTestCTCAlignr(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): diff --git a/python/paddle/fluid/tests/unittests/test_cuda_cudnn_version.py b/python/paddle/fluid/tests/unittests/test_cuda_cudnn_version.py index 36637971f9e48..d8229247a817f 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_cudnn_version.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_cudnn_version.py @@ -17,7 +17,6 @@ class TestCPUVersion(unittest.TestCase): - def test_cuda_cudnn_version_in_cpu_package(self): if not paddle.is_compiled_with_cuda(): self.assertEqual(paddle.version.cuda(), 'False') diff --git a/python/paddle/fluid/tests/unittests/test_cuda_device_count.py b/python/paddle/fluid/tests/unittests/test_cuda_device_count.py index 482a3413caf46..7789bcd944ded 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_device_count.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_device_count.py @@ -17,7 +17,6 @@ class TestDeviceCount(unittest.TestCase): - def test_device_count(self): s = paddle.device.cuda.device_count() self.assertIsNotNone(s) diff --git a/python/paddle/fluid/tests/unittests/test_cuda_device_name_capability.py b/python/paddle/fluid/tests/unittests/test_cuda_device_name_capability.py index 0d749c5d17729..07ef914f0a311 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_device_name_capability.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_device_name_capability.py @@ -17,7 +17,6 @@ class TestDeviceName(unittest.TestCase): - def test_device_name_default(self): if paddle.is_compiled_with_cuda(): name = paddle.device.cuda.get_device_name() @@ -35,7 +34,6 @@ def test_device_name_CUDAPlace(self): class TestDeviceCapability(unittest.TestCase): - def test_device_capability_default(self): if paddle.is_compiled_with_cuda(): capability = paddle.device.cuda.get_device_capability() @@ -49,7 +47,8 @@ def test_device_capability_int(self): def test_device_capability_CUDAPlace(self): if paddle.is_compiled_with_cuda(): capability = paddle.device.cuda.get_device_capability( - paddle.CUDAPlace(0)) + paddle.CUDAPlace(0) + ) self.assertIsNotNone(capability) diff --git a/python/paddle/fluid/tests/unittests/test_cuda_empty_cache.py b/python/paddle/fluid/tests/unittests/test_cuda_empty_cache.py index 0ec066eb7cdce..4aefb234bbfc1 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_empty_cache.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_empty_cache.py @@ -17,7 +17,6 @@ class TestEmptyCache(unittest.TestCase): - def test_empty_cache(self): x = paddle.randn((2, 10, 12)).astype('float32') del x diff --git a/python/paddle/fluid/tests/unittests/test_cuda_graph.py b/python/paddle/fluid/tests/unittests/test_cuda_graph.py index 035c71b1dad56..ce446264f39d5 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_graph.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_graph.py @@ -28,19 +28,21 @@ def can_use_cuda_graph(): class TestCUDAGraph(unittest.TestCase): - def setUp(self): if can_use_cuda_graph(): - paddle.set_flags({ - 'FLAGS_allocator_strategy': 'auto_growth', - 'FLAGS_sync_nccl_allreduce': False, - 'FLAGS_cudnn_deterministic': True, - 'FLAGS_use_stream_safe_cuda_allocator': False, - }) + paddle.set_flags( + { + 'FLAGS_allocator_strategy': 'auto_growth', + 'FLAGS_sync_nccl_allreduce': False, + 'FLAGS_cudnn_deterministic': True, + 'FLAGS_use_stream_safe_cuda_allocator': False, + } + ) def random_tensor(self, shape): return paddle.to_tensor( - np.random.randint(low=0, high=10, size=shape).astype("float32")) + np.random.randint(low=0, high=10, size=shape).astype("float32") + ) @switch_to_static_graph def test_cuda_graph_static_graph(self): @@ -48,10 +50,12 @@ def test_cuda_graph_static_graph(self): return seed = 100 - loss_cuda_graph = self.cuda_graph_static_graph_main(seed, - use_cuda_graph=True) + loss_cuda_graph = self.cuda_graph_static_graph_main( + seed, use_cuda_graph=True + ) loss_no_cuda_graph = self.cuda_graph_static_graph_main( - seed, use_cuda_graph=False) + seed, use_cuda_graph=False + ) self.assertEqual(loss_cuda_graph, loss_no_cuda_graph) def cuda_graph_static_graph_main(self, seed, use_cuda_graph): @@ -65,18 +69,19 @@ def cuda_graph_static_graph_main(self, seed, use_cuda_graph): startup = paddle.static.Program() main = paddle.static.Program() with paddle.static.program_guard(main, startup): - image = paddle.static.data(name="image", - shape=image_shape, - dtype='float32') - label = paddle.static.data(name="label", - shape=label_shape, - dtype='int64') + image = paddle.static.data( + name="image", shape=image_shape, dtype='float32' + ) + label = paddle.static.data( + name="label", shape=label_shape, dtype='int64' + ) image.persistable = True label.persistable = True loss = simple_fc_net_with_inputs(image, label, class_num) loss.persistable = True lr = paddle.optimizer.lr.PiecewiseDecay( - boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04]) + boundaries=[2, 3, 4], values=[0.01, 0.02, 0.03, 0.04] + ) optimizer = paddle.optimizer.SGD(learning_rate=lr) optimizer.minimize(loss) place = paddle.CUDAPlace(0) @@ -89,9 +94,10 @@ def cuda_graph_static_graph_main(self, seed, use_cuda_graph): build_strategy.fix_op_run_order = True build_strategy.fuse_all_optimizer_ops = True compiled_program = paddle.static.CompiledProgram( - main).with_data_parallel(loss_name=loss.name, - build_strategy=build_strategy, - places=place) + main + ).with_data_parallel( + loss_name=loss.name, build_strategy=build_strategy, places=place + ) image_t = scope.var(image.name).get_tensor() label_t = scope.var(label.name).get_tensor() loss_t = scope.var(loss.name).get_tensor() @@ -101,12 +107,14 @@ def cuda_graph_static_graph_main(self, seed, use_cuda_graph): cuda_graph = None for batch_id in range(20): image_t.set( - np.random.rand(*image_shape).astype('float32'), place) + np.random.rand(*image_shape).astype('float32'), place + ) label_t.set( - np.random.randint(low=0, - high=class_num, - size=label_shape, - dtype='int64'), place) + np.random.randint( + low=0, high=class_num, size=label_shape, dtype='int64' + ), + place, + ) if batch_id == 1 and use_cuda_graph: cuda_graph = CUDAGraph(place, mode="global") @@ -195,7 +203,6 @@ def test_dataloader(self): return class AutoIncDataset(paddle.io.Dataset): - def __init__(self, n, dtype): self.n = n self.dtype = dtype @@ -209,10 +216,9 @@ def __getitem__(self, idx): n = 100 dtype = 'int64' dataset = AutoIncDataset(n, dtype) - data_loader = paddle.io.DataLoader(dataset, - batch_size=1, - num_workers=2, - use_buffer_reader=True) + data_loader = paddle.io.DataLoader( + dataset, batch_size=1, num_workers=2, use_buffer_reader=True + ) x = None y = None diff --git a/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph.py b/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph.py index ecd3f406e088d..7ae6a9f8e1998 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph.py @@ -20,7 +20,6 @@ class SimpleModel(nn.Layer): - def __init__(self, in_size, out_size): super(SimpleModel, self).__init__() self.linear = nn.Linear(in_size, out_size) @@ -39,7 +38,6 @@ def forward(self, x): class TestSimpleModel(unittest.TestCase): - def setUp(self): paddle.set_flags({'FLAGS_eager_delete_tensor_gb': 0.0}) diff --git a/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static.py b/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static.py index b45a862ffbf30..077caebc03f87 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static.py @@ -21,7 +21,6 @@ class SimpleModel(nn.Layer): - def __init__(self, in_size, out_size): super(SimpleModel, self).__init__() self.linear = nn.Linear(in_size, out_size) @@ -40,7 +39,6 @@ def forward(self, x): class TestCudaGraphAttrAll(unittest.TestCase): - def test_all_program(self): if not is_cuda_graph_supported(): return @@ -59,8 +57,10 @@ def test_all_program(self): if op._cuda_graph_attr is None: # the loss and opt are not wrapped assert op.type in [ - 'sgd', 'reduce_mean', 'fill_constant', - 'reduce_mean_grad' + 'sgd', + 'reduce_mean', + 'fill_constant', + 'reduce_mean_grad', ] else: assert op._cuda_graph_attr == 'thread_local;0;0' diff --git a/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static_run.py b/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static_run.py index 445211d35a1d9..0d2b737816bca 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static_run.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_graph_partial_graph_static_run.py @@ -16,13 +16,16 @@ import paddle.nn as nn import unittest import numpy as np -from paddle.device.cuda.graphs import wrap_cuda_graph, is_cuda_graph_supported, cuda_graph_transform +from paddle.device.cuda.graphs import ( + wrap_cuda_graph, + is_cuda_graph_supported, + cuda_graph_transform, +) paddle.enable_static() class SimpleModel(nn.Layer): - def __init__(self, in_size, out_size): super(SimpleModel, self).__init__() self.linear = nn.Linear(in_size, out_size) @@ -41,7 +44,6 @@ def forward(self, x): class TestCudaGraphAttrAll(unittest.TestCase): - def setUp(self): paddle.set_flags({'FLAGS_eager_delete_tensor_gb': 0.0}) diff --git a/python/paddle/fluid/tests/unittests/test_cuda_max_memory_allocated.py b/python/paddle/fluid/tests/unittests/test_cuda_max_memory_allocated.py index 5f047977ed354..909b9a44b11b7 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_max_memory_allocated.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_max_memory_allocated.py @@ -15,12 +15,15 @@ import paddle import unittest from paddle.fluid import core -from paddle.device.cuda import device_count, memory_allocated, max_memory_allocated +from paddle.device.cuda import ( + device_count, + memory_allocated, + max_memory_allocated, +) from paddle.fluid.framework import _test_eager_guard class TestMaxMemoryAllocated(unittest.TestCase): - def func_test_max_memory_allocated(self, device=None): if core.is_compiled_with_cuda(): alloc_time = 100 @@ -29,13 +32,15 @@ def func_test_max_memory_allocated(self, device=None): for i in range(alloc_time): shape = paddle.randint(max_alloc_size) tensor = paddle.zeros(shape) - peak_memory_allocated_size = max(peak_memory_allocated_size, - memory_allocated(device)) + peak_memory_allocated_size = max( + peak_memory_allocated_size, memory_allocated(device) + ) del shape del tensor - self.assertEqual(peak_memory_allocated_size, - max_memory_allocated(device)) + self.assertEqual( + peak_memory_allocated_size, max_memory_allocated(device) + ) def test_max_memory_allocated(self): with _test_eager_guard(): @@ -60,7 +65,11 @@ def func_test_max_memory_allocated_exception(self): if core.is_compiled_with_cuda(): wrong_device = [ core.CPUPlace(), - device_count() + 1, -2, 0.5, "gpu1", "npu" + device_count() + 1, + -2, + 0.5, + "gpu1", + "npu", ] for device in wrong_device: with self.assertRaises(BaseException): diff --git a/python/paddle/fluid/tests/unittests/test_cuda_max_memory_reserved.py b/python/paddle/fluid/tests/unittests/test_cuda_max_memory_reserved.py index 936a084abb704..e067d293074ab 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_max_memory_reserved.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_max_memory_reserved.py @@ -15,11 +15,14 @@ import paddle import unittest from paddle.fluid import core -from paddle.device.cuda import device_count, memory_reserved, max_memory_reserved +from paddle.device.cuda import ( + device_count, + memory_reserved, + max_memory_reserved, +) class TestMaxMemoryreserved(unittest.TestCase): - def test_max_memory_reserved(self, device=None): if core.is_compiled_with_cuda(): alloc_time = 100 @@ -28,13 +31,15 @@ def test_max_memory_reserved(self, device=None): for i in range(alloc_time): shape = paddle.randint(max_alloc_size) tensor = paddle.zeros(shape) - peak_memory_reserved_size = max(peak_memory_reserved_size, - memory_reserved(device)) + peak_memory_reserved_size = max( + peak_memory_reserved_size, memory_reserved(device) + ) del shape del tensor - self.assertEqual(peak_memory_reserved_size, - max_memory_reserved(device)) + self.assertEqual( + peak_memory_reserved_size, max_memory_reserved(device) + ) def test_max_memory_reserved_for_all_places(self): if core.is_compiled_with_cuda(): @@ -49,7 +54,11 @@ def test_max_memory_reserved_exception(self): if core.is_compiled_with_cuda(): wrong_device = [ core.CPUPlace(), - device_count() + 1, -2, 0.5, "gpu1", "npu" + device_count() + 1, + -2, + 0.5, + "gpu1", + "npu", ] for device in wrong_device: with self.assertRaises(BaseException): diff --git a/python/paddle/fluid/tests/unittests/test_cuda_memory_allocated.py b/python/paddle/fluid/tests/unittests/test_cuda_memory_allocated.py index 4e82ab7326cb5..5b21a9b07e88f 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_memory_allocated.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_memory_allocated.py @@ -19,7 +19,6 @@ class TestMemoryAllocated(unittest.TestCase): - def test_memory_allocated(self, device=None): if core.is_compiled_with_cuda(): tensor = paddle.zeros(shape=[256]) @@ -40,7 +39,11 @@ def test_memory_allocated_exception(self): if core.is_compiled_with_cuda(): wrong_device = [ core.CPUPlace(), - device_count() + 1, -2, 0.5, "gpu1", "npu" + device_count() + 1, + -2, + 0.5, + "gpu1", + "npu", ] for device in wrong_device: with self.assertRaises(BaseException): diff --git a/python/paddle/fluid/tests/unittests/test_cuda_memory_reserved.py b/python/paddle/fluid/tests/unittests/test_cuda_memory_reserved.py index f5db61a1e65de..6b2cdc45f1924 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_memory_reserved.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_memory_reserved.py @@ -20,7 +20,6 @@ class TestMemoryreserved(unittest.TestCase): - def func_test_memory_reserved(self, device=None): if core.is_compiled_with_cuda(): tensor = paddle.zeros(shape=[256]) @@ -51,7 +50,11 @@ def func_test_memory_reserved_exception(self): if core.is_compiled_with_cuda(): wrong_device = [ core.CPUPlace(), - device_count() + 1, -2, 0.5, "gpu1", "npu" + device_count() + 1, + -2, + 0.5, + "gpu1", + "npu", ] for device in wrong_device: with self.assertRaises(BaseException): diff --git a/python/paddle/fluid/tests/unittests/test_cuda_random_seed.py b/python/paddle/fluid/tests/unittests/test_cuda_random_seed.py index f12d3a5a4d64d..d9bef511c9c51 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_random_seed.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_random_seed.py @@ -24,8 +24,9 @@ import tempfile -@unittest.skipIf(not core.is_compiled_with_cuda(), - "Only test cuda Random Generator") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "Only test cuda Random Generator" +) class TestGeneratorSeed(unittest.TestCase): """ Test cases for cpu generator seed. @@ -39,18 +40,15 @@ def test_gen_dropout_dygraph(self): gen.manual_seed(111111111) st = paddle.get_cuda_rng_state() - x = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) - x_again = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) - x_third = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) + x = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) + x_again = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) + x_third = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) print("x: {}".format(x.numpy())) print("x_again: {}".format(x_again.numpy())) x = x + x_again + x_third @@ -58,18 +56,15 @@ def test_gen_dropout_dygraph(self): paddle.set_cuda_rng_state(st) - x1 = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) - x1_again = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) - x1_third = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) + x1 = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) + x1_again = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) + x1_third = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) x1 = x1 + x1_again + x1_third y1 = fluid.layers.dropout(x1, 0.5) y_np = y.numpy() @@ -136,26 +131,30 @@ def test_gen_TruncatedNormal_initializer(self): result_1 = fluid.layers.fc( input=x, size=10, - param_attr=fluid.initializer.TruncatedNormal(loc=0.0, - scale=2.0)) + param_attr=fluid.initializer.TruncatedNormal( + loc=0.0, scale=2.0 + ), + ) result_2 = fluid.layers.fc( input=x, size=10, - param_attr=fluid.initializer.TruncatedNormal(loc=0.0, - scale=2.0)) + param_attr=fluid.initializer.TruncatedNormal( + loc=0.0, scale=2.0 + ), + ) exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) paddle.seed(123123143) with fluid.program_guard(train_program, startup_program): exe.run(startup_program) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) diff --git a/python/paddle/fluid/tests/unittests/test_cuda_stream_event.py b/python/paddle/fluid/tests/unittests/test_cuda_stream_event.py index 099f19bd03ef0..600cd04d1970f 100644 --- a/python/paddle/fluid/tests/unittests/test_cuda_stream_event.py +++ b/python/paddle/fluid/tests/unittests/test_cuda_stream_event.py @@ -21,7 +21,6 @@ class TestCurrentStream(unittest.TestCase): - def test_current_stream(self): if paddle.is_compiled_with_cuda(): s = cuda.current_stream() @@ -39,7 +38,6 @@ def test_current_stream(self): class TestSynchronize(unittest.TestCase): - def test_synchronize(self): if paddle.is_compiled_with_cuda(): self.assertIsNone(cuda.synchronize()) @@ -50,7 +48,6 @@ def test_synchronize(self): class TestCUDAStream(unittest.TestCase): - def test_cuda_stream(self): if paddle.is_compiled_with_cuda(): s = paddle.device.cuda.Stream() @@ -88,7 +85,6 @@ def test_cuda_stream_wait_event_and_record_event(self): class TestCUDAEvent(unittest.TestCase): - def test_cuda_event(self): if paddle.is_compiled_with_cuda(): e = paddle.device.cuda.Event(True, False, False) @@ -158,14 +154,15 @@ def test_not_correct_stream_guard_input(): def test_set_current_stream_raise_error(self): if paddle.is_compiled_with_cuda(): - self.assertRaises(TypeError, paddle.device.cuda._set_current_stream, - np.zeros(5)) - self.assertRaises(TypeError, paddle.device.cuda._set_current_stream, - None) + self.assertRaises( + TypeError, paddle.device.cuda._set_current_stream, np.zeros(5) + ) + self.assertRaises( + TypeError, paddle.device.cuda._set_current_stream, None + ) class TestRawStream(unittest.TestCase): - def test_cuda_stream(self): if paddle.is_compiled_with_cuda(): cuda_stream = paddle.device.cuda.current_stream().cuda_stream diff --git a/python/paddle/fluid/tests/unittests/test_cudnn_grucell.py b/python/paddle/fluid/tests/unittests/test_cudnn_grucell.py index d82da93ba04d9..39c25c909fdc6 100644 --- a/python/paddle/fluid/tests/unittests/test_cudnn_grucell.py +++ b/python/paddle/fluid/tests/unittests/test_cudnn_grucell.py @@ -23,15 +23,16 @@ def sigmoid(x): - return 1. / (1. + np.exp(-x)) + return 1.0 / (1.0 + np.exp(-x)) def tanh(x): - return 2. * sigmoid(2. * x) - 1. + return 2.0 * sigmoid(2.0 * x) - 1.0 -def cudnn_step(step_input_np, pre_hidden_np, weight_ih, bias_ih, weight_hh, - bias_hh): +def cudnn_step( + step_input_np, pre_hidden_np, weight_ih, bias_ih, weight_hh, bias_hh +): igates = np.matmul(step_input_np, weight_ih.transpose(1, 0)) igates += bias_ih hgates = np.matmul(pre_hidden_np, weight_hh.transpose(1, 0)) @@ -55,8 +56,9 @@ def cudnn_step(step_input_np, pre_hidden_np, weight_ih, bias_ih, weight_hh, return new_hidden -def non_cudnn_step(step_in, pre_hidden, gate_w, gate_b, candidate_w, - candidate_b): +def non_cudnn_step( + step_in, pre_hidden, gate_w, gate_b, candidate_w, candidate_b +): concat_1 = np.concatenate([step_in, pre_hidden], 1) gate_input = np.matmul(concat_1, gate_w) @@ -77,7 +79,6 @@ def non_cudnn_step(step_in, pre_hidden, gate_w, gate_b, candidate_w, class TestCudnnGRU(unittest.TestCase): - def setUp(self): self.input_size = 100 self.hidden_size = 200 @@ -93,8 +94,9 @@ def test_run(self): with fluid.dygraph.guard(place): param_attr = fluid.ParamAttr(name="param_attr") bias_attr = fluid.ParamAttr(name="bias_attr") - named_cudnn_gru = GRUCell(self.hidden_size, self.input_size, - param_attr, bias_attr) + named_cudnn_gru = GRUCell( + self.hidden_size, self.input_size, param_attr, bias_attr + ) cudnn_gru = GRUCell(self.hidden_size, self.input_size) param_list = cudnn_gru.state_dict() @@ -109,51 +111,55 @@ def test_run(self): weight_ih = param_list[weight_ih_name].numpy() weight_ih = np.random.uniform( - -0.1, 0.1, size=weight_ih.shape).astype('float64') + -0.1, 0.1, size=weight_ih.shape + ).astype('float64') param_list[weight_ih_name].set_value(weight_ih) named_param_list[weight_ih_name].set_value(weight_ih) bias_ih = param_list[bias_ih_name].numpy() - bias_ih = np.random.uniform(-0.1, 0.1, - size=bias_ih.shape).astype('float64') + bias_ih = np.random.uniform(-0.1, 0.1, size=bias_ih.shape).astype( + 'float64' + ) param_list[bias_ih_name].set_value(bias_ih) named_param_list[bias_ih_name].set_value(bias_ih) weight_hh = param_list[weight_hh_name].numpy() weight_hh = np.random.uniform( - -0.1, 0.1, size=weight_hh.shape).astype('float64') + -0.1, 0.1, size=weight_hh.shape + ).astype('float64') param_list[weight_hh_name].set_value(weight_hh) named_param_list[weight_hh_name].set_value(weight_hh) bias_hh = param_list[bias_hh_name].numpy() - bias_hh = np.random.uniform(-0.1, 0.1, - size=bias_hh.shape).astype('float64') + bias_hh = np.random.uniform(-0.1, 0.1, size=bias_hh.shape).astype( + 'float64' + ) param_list[bias_hh_name].set_value(bias_hh) named_param_list[bias_hh_name].set_value(bias_hh) step_input_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.input_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.input_size) + ).astype('float64') pre_hidden_np = np.random.uniform( - -0.1, 0.1, - (self.batch_size, self.hidden_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float64') step_input_var = fluid.dygraph.to_variable(step_input_np) pre_hidden_var = fluid.dygraph.to_variable(pre_hidden_np) api_out = cudnn_gru(step_input_var, pre_hidden_var) named_api_out = named_cudnn_gru(step_input_var, pre_hidden_var) - np_out = cudnn_step(step_input_np, pre_hidden_np, weight_ih, bias_ih, - weight_hh, bias_hh) + np_out = cudnn_step( + step_input_np, pre_hidden_np, weight_ih, bias_ih, weight_hh, bias_hh + ) np.testing.assert_allclose(api_out.numpy(), np_out, rtol=1e-05, atol=0) - np.testing.assert_allclose(named_api_out.numpy(), - np_out, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + named_api_out.numpy(), np_out, rtol=1e-05, atol=0 + ) class TestNonCudnnGRU(unittest.TestCase): - def setUp(self): self.input_size = 100 self.hidden_size = 200 @@ -169,14 +175,16 @@ def test_run(self): with fluid.dygraph.guard(place): param_attr = fluid.ParamAttr(name="param_attr") bias_attr = fluid.ParamAttr(name="bias_attr") - named_non_cudnn_gru = GRUCell(self.hidden_size, - self.input_size, - param_attr, - bias_attr, - use_cudnn_impl=False) - non_cudnn_gru = GRUCell(self.hidden_size, - self.input_size, - use_cudnn_impl=False) + named_non_cudnn_gru = GRUCell( + self.hidden_size, + self.input_size, + param_attr, + bias_attr, + use_cudnn_impl=False, + ) + non_cudnn_gru = GRUCell( + self.hidden_size, self.input_size, use_cudnn_impl=False + ) param_list = non_cudnn_gru.state_dict() named_param_list = named_non_cudnn_gru.state_dict() @@ -189,48 +197,58 @@ def test_run(self): candidate_b_name = "_candidate_bias" gate_w = param_list[gate_w_name].numpy() - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float64') + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float64' + ) param_list[gate_w_name].set_value(gate_w) named_param_list[gate_w_name].set_value(gate_w) gate_b = param_list[gate_b_name].numpy() - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float64') + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float64' + ) param_list[gate_b_name].set_value(gate_b) named_param_list[gate_b_name].set_value(gate_b) candidate_w = param_list[candidate_w_name].numpy() candidate_w = np.random.uniform( - -0.1, 0.1, size=candidate_w.shape).astype('float64') + -0.1, 0.1, size=candidate_w.shape + ).astype('float64') param_list[candidate_w_name].set_value(candidate_w) named_param_list[candidate_w_name].set_value(candidate_w) candidate_b = param_list[candidate_b_name].numpy() candidate_b = np.random.uniform( - -0.1, 0.1, size=candidate_b.shape).astype('float64') + -0.1, 0.1, size=candidate_b.shape + ).astype('float64') param_list[candidate_b_name].set_value(candidate_b) named_param_list[candidate_b_name].set_value(candidate_b) step_input_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.input_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.input_size) + ).astype('float64') pre_hidden_np = np.random.uniform( - -0.1, 0.1, - (self.batch_size, self.hidden_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float64') step_input_var = fluid.dygraph.to_variable(step_input_np) pre_hidden_var = fluid.dygraph.to_variable(pre_hidden_np) api_out = non_cudnn_gru(step_input_var, pre_hidden_var) named_api_out = named_non_cudnn_gru(step_input_var, pre_hidden_var) - np_out = non_cudnn_step(step_input_np, pre_hidden_np, gate_w, gate_b, - candidate_w, candidate_b) + np_out = non_cudnn_step( + step_input_np, + pre_hidden_np, + gate_w, + gate_b, + candidate_w, + candidate_b, + ) np.testing.assert_allclose(api_out.numpy(), np_out, rtol=1e-05, atol=0) - np.testing.assert_allclose(named_api_out.numpy(), - np_out, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + named_api_out.numpy(), np_out, rtol=1e-05, atol=0 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_cudnn_lstmcell.py b/python/paddle/fluid/tests/unittests/test_cudnn_lstmcell.py index d21ec3033ed68..4eacff27d1432 100644 --- a/python/paddle/fluid/tests/unittests/test_cudnn_lstmcell.py +++ b/python/paddle/fluid/tests/unittests/test_cudnn_lstmcell.py @@ -23,19 +23,16 @@ def sigmoid(x): - return 1. / (1. + np.exp(-x)) + return 1.0 / (1.0 + np.exp(-x)) def tanh(x): - return 2. * sigmoid(2. * x) - 1. + return 2.0 * sigmoid(2.0 * x) - 1.0 -def non_cudnn_step(step_in, - pre_hidden, - pre_cell, - gate_w, - gate_b, - forget_bias=1.0): +def non_cudnn_step( + step_in, pre_hidden, pre_cell, gate_w, gate_b, forget_bias=1.0 +): concat_1 = np.concatenate([step_in, pre_hidden], 1) gate_input = np.matmul(concat_1, gate_w) @@ -48,8 +45,15 @@ def non_cudnn_step(step_in, return new_hidden, new_cell -def cudnn_step(step_input_np, pre_hidden_np, pre_cell_np, weight_ih, bias_ih, - weight_hh, bias_hh): +def cudnn_step( + step_input_np, + pre_hidden_np, + pre_cell_np, + weight_ih, + bias_ih, + weight_hh, + bias_hh, +): igates = np.matmul(step_input_np, weight_ih.transpose(1, 0)) igates = igates + bias_ih @@ -78,7 +82,6 @@ def cudnn_step(step_input_np, pre_hidden_np, pre_cell_np, weight_ih, bias_ih, class TestCudnnLSTM(unittest.TestCase): - def setUp(self): self.input_size = 100 self.hidden_size = 200 @@ -93,8 +96,9 @@ def test_run(self): with fluid.dygraph.guard(place): param_attr = fluid.ParamAttr(name="param_attr") bias_attr = fluid.ParamAttr(name="bias_attr") - named_cudnn_lstm = LSTMCell(self.hidden_size, self.input_size, - param_attr, bias_attr) + named_cudnn_lstm = LSTMCell( + self.hidden_size, self.input_size, param_attr, bias_attr + ) cudnn_lstm = LSTMCell(self.hidden_size, self.input_size) param_list = cudnn_lstm.state_dict() @@ -108,73 +112,79 @@ def test_run(self): bias_hh_name = "_bias_hh" weight_ih = param_list[weight_ih_name].numpy() weight_ih = np.random.uniform( - -0.1, 0.1, size=weight_ih.shape).astype('float64') + -0.1, 0.1, size=weight_ih.shape + ).astype('float64') param_list[weight_ih_name].set_value(weight_ih) named_param_list[weight_ih_name].set_value(weight_ih) bias_ih = param_list[bias_ih_name].numpy() - bias_ih = np.random.uniform(-0.1, 0.1, - size=bias_ih.shape).astype('float64') + bias_ih = np.random.uniform(-0.1, 0.1, size=bias_ih.shape).astype( + 'float64' + ) param_list[bias_ih_name].set_value(bias_ih) named_param_list[bias_ih_name].set_value(bias_ih) weight_hh = param_list[weight_hh_name].numpy() weight_hh = np.random.uniform( - -0.1, 0.1, size=weight_hh.shape).astype('float64') + -0.1, 0.1, size=weight_hh.shape + ).astype('float64') param_list[weight_hh_name].set_value(weight_hh) named_param_list[weight_hh_name].set_value(weight_hh) bias_hh = param_list[bias_hh_name].numpy() - bias_hh = np.random.uniform(-0.1, 0.1, - size=bias_hh.shape).astype('float64') + bias_hh = np.random.uniform(-0.1, 0.1, size=bias_hh.shape).astype( + 'float64' + ) param_list[bias_hh_name].set_value(bias_hh) named_param_list[bias_hh_name].set_value(bias_hh) step_input_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.input_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.input_size) + ).astype('float64') pre_hidden_np = np.random.uniform( - -0.1, 0.1, - (self.batch_size, self.hidden_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float64') pre_cell_np = np.random.uniform( - -0.1, 0.1, - (self.batch_size, self.hidden_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float64') step_input_var = fluid.dygraph.to_variable(step_input_np) pre_hidden_var = fluid.dygraph.to_variable(pre_hidden_np) pre_cell_var = fluid.dygraph.to_variable(pre_cell_np) api_out = cudnn_lstm(step_input_var, pre_hidden_var, pre_cell_var) - named_api_out = named_cudnn_lstm(step_input_var, pre_hidden_var, - pre_cell_var) + named_api_out = named_cudnn_lstm( + step_input_var, pre_hidden_var, pre_cell_var + ) api_hidden_out = api_out[0] api_cell_out = api_out[1] named_api_hidden_out = named_api_out[0] named_api_cell_out = named_api_out[1] - np_hidden_out, np_cell_out = cudnn_step(step_input_np, - pre_hidden_np, pre_cell_np, - weight_ih, bias_ih, - weight_hh, bias_hh) - np.testing.assert_allclose(api_hidden_out.numpy(), - np_hidden_out, - rtol=1e-05, - atol=0) - np.testing.assert_allclose(api_cell_out.numpy(), - np_cell_out, - rtol=1e-05, - atol=0) - np.testing.assert_allclose(named_api_hidden_out.numpy(), - np_hidden_out, - rtol=1e-05, - atol=0) - np.testing.assert_allclose(named_api_cell_out.numpy(), - np_cell_out, - rtol=1e-05, - atol=0) + np_hidden_out, np_cell_out = cudnn_step( + step_input_np, + pre_hidden_np, + pre_cell_np, + weight_ih, + bias_ih, + weight_hh, + bias_hh, + ) + np.testing.assert_allclose( + api_hidden_out.numpy(), np_hidden_out, rtol=1e-05, atol=0 + ) + np.testing.assert_allclose( + api_cell_out.numpy(), np_cell_out, rtol=1e-05, atol=0 + ) + np.testing.assert_allclose( + named_api_hidden_out.numpy(), np_hidden_out, rtol=1e-05, atol=0 + ) + np.testing.assert_allclose( + named_api_cell_out.numpy(), np_cell_out, rtol=1e-05, atol=0 + ) class TestNonCudnnLSTM(unittest.TestCase): - def setUp(self): self.input_size = 100 self.hidden_size = 200 @@ -189,14 +199,16 @@ def test_run(self): with fluid.dygraph.guard(place): param_attr = fluid.ParamAttr(name="param_attr") bias_attr = fluid.ParamAttr(name="bias_attr") - named_cudnn_lstm = LSTMCell(self.hidden_size, - self.input_size, - param_attr, - bias_attr, - use_cudnn_impl=False) - cudnn_lstm = LSTMCell(self.hidden_size, - self.input_size, - use_cudnn_impl=False) + named_cudnn_lstm = LSTMCell( + self.hidden_size, + self.input_size, + param_attr, + bias_attr, + use_cudnn_impl=False, + ) + cudnn_lstm = LSTMCell( + self.hidden_size, self.input_size, use_cudnn_impl=False + ) param_list = cudnn_lstm.state_dict() named_param_list = named_cudnn_lstm.state_dict() @@ -207,59 +219,58 @@ def test_run(self): gate_b_name = "_bias" gate_w = param_list[gate_w_name].numpy() - gate_w = np.random.uniform(-0.1, 0.1, - size=gate_w.shape).astype('float64') + gate_w = np.random.uniform(-0.1, 0.1, size=gate_w.shape).astype( + 'float64' + ) param_list[gate_w_name].set_value(gate_w) named_param_list[gate_w_name].set_value(gate_w) gate_b = param_list[gate_b_name].numpy() - gate_b = np.random.uniform(-0.1, 0.1, - size=gate_b.shape).astype('float64') + gate_b = np.random.uniform(-0.1, 0.1, size=gate_b.shape).astype( + 'float64' + ) param_list[gate_b_name].set_value(gate_b) named_param_list[gate_b_name].set_value(gate_b) step_input_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.input_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.input_size) + ).astype('float64') pre_hidden_np = np.random.uniform( - -0.1, 0.1, - (self.batch_size, self.hidden_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float64') pre_cell_np = np.random.uniform( - -0.1, 0.1, - (self.batch_size, self.hidden_size)).astype('float64') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float64') step_input_var = fluid.dygraph.to_variable(step_input_np) pre_hidden_var = fluid.dygraph.to_variable(pre_hidden_np) pre_cell_var = fluid.dygraph.to_variable(pre_cell_np) api_out = cudnn_lstm(step_input_var, pre_hidden_var, pre_cell_var) - named_api_out = named_cudnn_lstm(step_input_var, pre_hidden_var, - pre_cell_var) + named_api_out = named_cudnn_lstm( + step_input_var, pre_hidden_var, pre_cell_var + ) api_hidden_out = api_out[0] api_cell_out = api_out[1] named_api_hidden_out = named_api_out[0] named_api_cell_out = named_api_out[1] - np_hidden_out, np_cell_out = non_cudnn_step(step_input_np, - pre_hidden_np, - pre_cell_np, gate_w, - gate_b) - - np.testing.assert_allclose(api_hidden_out.numpy(), - np_hidden_out, - rtol=1e-05, - atol=0) - np.testing.assert_allclose(api_cell_out.numpy(), - np_cell_out, - rtol=1e-05, - atol=0) - np.testing.assert_allclose(named_api_hidden_out.numpy(), - np_hidden_out, - rtol=1e-05, - atol=0) - np.testing.assert_allclose(named_api_cell_out.numpy(), - np_cell_out, - rtol=1e-05, - atol=0) + np_hidden_out, np_cell_out = non_cudnn_step( + step_input_np, pre_hidden_np, pre_cell_np, gate_w, gate_b + ) + + np.testing.assert_allclose( + api_hidden_out.numpy(), np_hidden_out, rtol=1e-05, atol=0 + ) + np.testing.assert_allclose( + api_cell_out.numpy(), np_cell_out, rtol=1e-05, atol=0 + ) + np.testing.assert_allclose( + named_api_hidden_out.numpy(), np_hidden_out, rtol=1e-05, atol=0 + ) + np.testing.assert_allclose( + named_api_cell_out.numpy(), np_cell_out, rtol=1e-05, atol=0 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_cumprod_op.py b/python/paddle/fluid/tests/unittests/test_cumprod_op.py index c4c6724468f22..a6aea5e33fd33 100644 --- a/python/paddle/fluid/tests/unittests/test_cumprod_op.py +++ b/python/paddle/fluid/tests/unittests/test_cumprod_op.py @@ -47,8 +47,9 @@ def cumprod_grad(x, y, dy, dx, shape, dim): else: elem = dy[pos] * y[index - inner_dim] if pos > index: - for m in range(index + inner_dim, pos + inner_dim, - inner_dim): + for m in range( + index + inner_dim, pos + inner_dim, inner_dim + ): elem *= x[m] elif pos < index: elem = 0 @@ -57,7 +58,6 @@ def cumprod_grad(x, y, dy, dx, shape, dim): # test function. class TestCumprod(OpTest): - def init_params(self): self.shape = (2, 3, 4, 5) self.zero_nums = [0, 10, 20, 30, int(np.prod(self.shape))] @@ -98,8 +98,9 @@ def init_grad_input_output(self, dim): if self.dtype == np.complex128 or self.dtype == np.complex64: reshape_x = np.conj(reshape_x) out_data = np.conj(out_data) - cumprod_grad(reshape_x, out_data, self.grad_out, self.grad_x, - self.shape, dim) + cumprod_grad( + reshape_x, out_data, self.grad_out, self.grad_x, self.shape, dim + ) self.grad_x = self.grad_x.reshape(self.shape) self.grad_out = self.grad_out.reshape(self.shape) @@ -119,37 +120,35 @@ def test_check_grad(self): if self.dtype == np.float64: self.check_grad(['X'], 'Out', check_eager=True) else: - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) # test float32 case. class TestCumprod_float32(TestCumprod): - def init_dtype(self): self.dtype = np.float32 # test complex64 case. class TestCumprod_complex64(TestCumprod): - def init_dtype(self): self.dtype = np.complex64 # test complex128 case. class TestCumprod_complex128(TestCumprod): - def init_dtype(self): self.dtype = np.complex128 # test api. class TestCumprodAPI(unittest.TestCase): - def init_dtype(self): self.dtype = 'float64' self.shape = [2, 3, 10, 10] @@ -182,7 +181,6 @@ def run(place): # test dynamic graph api. def test_dygraph_api(self): - def run(place): paddle.disable_static(place) x = paddle.to_tensor(self.x) diff --git a/python/paddle/fluid/tests/unittests/test_cumsum_op.py b/python/paddle/fluid/tests/unittests/test_cumsum_op.py index e63252c2c0897..b28c2863d1872 100644 --- a/python/paddle/fluid/tests/unittests/test_cumsum_op.py +++ b/python/paddle/fluid/tests/unittests/test_cumsum_op.py @@ -27,7 +27,6 @@ class TestCumsumOp(unittest.TestCase): - def run_cases(self): data_np = np.arange(12).reshape(3, 4) data = paddle.to_tensor(data_np) @@ -68,11 +67,17 @@ def run_static(self, use_gpu=False): place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - out = exe.run(feed={'X': data_np}, - fetch_list=[ - y.name, y2.name, y3.name, y4.name, y5.name, - y6.name - ]) + out = exe.run( + feed={'X': data_np}, + fetch_list=[ + y.name, + y2.name, + y3.name, + y4.name, + y5.name, + y6.name, + ], + ) z = np.cumsum(data_np) np.testing.assert_allclose(z, out[0], rtol=1e-05) @@ -109,7 +114,6 @@ def test_name(self): class TestSumOp1(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2} @@ -124,14 +128,14 @@ def test_check_grad(self): class TestSumOp2(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': -1, 'reverse': True} self.inputs = {'X': np.random.random((5, 6, 10)).astype("float64")} self.outputs = { - 'Out': np.flip(np.flip(self.inputs['X'], axis=2).cumsum(axis=2), - axis=2) + 'Out': np.flip( + np.flip(self.inputs['X'], axis=2).cumsum(axis=2), axis=2 + ) } def test_check_output(self): @@ -142,7 +146,6 @@ def test_check_grad(self): class TestSumOp3(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 1} @@ -157,7 +160,6 @@ def test_check_grad(self): class TestSumOp4(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 0} @@ -172,7 +174,6 @@ def test_check_grad(self): class TestSumOp5(OpTest): - def setUp(self): self.op_type = "cumsum" self.inputs = {'X': np.random.random((5, 20)).astype("float64")} @@ -186,7 +187,6 @@ def test_check_grad(self): class TestSumOp7(OpTest): - def setUp(self): self.op_type = "cumsum" self.inputs = {'X': np.random.random((100)).astype("float64")} @@ -200,7 +200,6 @@ def test_check_grad(self): class TestCumsumFP16(unittest.TestCase): - def check_main(self, x_np, dtype): paddle.disable_static() x = paddle.to_tensor(x_np.astype(dtype)) @@ -226,17 +225,19 @@ def test_main(self): class TestSumOpExclusive1(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 65)).astype("float64") self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=np.float64), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=np.float64), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } def test_check_output(self): @@ -244,17 +245,19 @@ def test_check_output(self): class TestSumOpExclusive2(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((1, 1, 888)).astype("float64") self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (1, 1, 1), dtype=np.float64), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((1, 1, 1), dtype=np.float64), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } def test_check_output(self): @@ -262,17 +265,19 @@ def test_check_output(self): class TestSumOpExclusive3(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 888)).astype("float32") self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=np.float64), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=np.float64), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } def test_check_output(self): @@ -280,17 +285,19 @@ def test_check_output(self): class TestSumOpExclusive4(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((1, 1, 3049)).astype("float64") self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (1, 1, 1), dtype=np.float64), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((1, 1, 1), dtype=np.float64), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } def test_check_output(self): @@ -298,17 +305,19 @@ def test_check_output(self): class TestSumOpExclusive5(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2, "exclusive": True} a = np.random.random((4, 5, 3096)).astype("float64") self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=np.float64), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=np.float64), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } def test_check_output(self): @@ -316,17 +325,19 @@ def test_check_output(self): class TestSumOpExclusiveFP16(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2, "exclusive": True, "dtype": "float16"} a = np.random.random((4, 5, 3096)).astype("float64") self.inputs = {'X': a} self.outputs = { - 'Out': - np.concatenate((np.zeros( - (4, 5, 1), dtype=np.float64), a[:, :, :-1].cumsum(axis=2)), - axis=2) + 'Out': np.concatenate( + ( + np.zeros((4, 5, 1), dtype=np.float64), + a[:, :, :-1].cumsum(axis=2), + ), + axis=2, + ) } def test_check_output(self): @@ -334,7 +345,6 @@ def test_check_output(self): class TestSumOpReverseExclusive(OpTest): - def setUp(self): self.op_type = "cumsum" self.attrs = {'axis': 2, 'reverse': True, "exclusive": True} @@ -342,11 +352,13 @@ def setUp(self): self.inputs = {'X': a} a = np.flip(a, axis=2) self.outputs = { - 'Out': - np.concatenate( - (np.flip(a[:, :, :-1].cumsum(axis=2), - axis=2), np.zeros((4, 5, 1), dtype=np.float64)), - axis=2) + 'Out': np.concatenate( + ( + np.flip(a[:, :, :-1].cumsum(axis=2), axis=2), + np.zeros((4, 5, 1), dtype=np.float64), + ), + axis=2, + ) } def test_check_output(self): @@ -354,7 +366,6 @@ def test_check_output(self): class BadInputTest(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): @@ -366,21 +377,24 @@ def test_bad_x(): class TestTensorAxis(unittest.TestCase): - def setUp(self): paddle.seed(2022) self.temp_dir = tempfile.TemporaryDirectory() self.save_path = os.path.join(self.temp_dir.name, 'tensor_axis_cumsum') - self.place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) def test_dygraph(self): paddle.disable_static() x = np.random.randn(5, 6) axis = 1 np_out = np.cumsum(x, axis) - pd_out = paddle.cumsum(paddle.to_tensor(x), - axis=paddle.to_tensor([axis], dtype='int32')) + pd_out = paddle.cumsum( + paddle.to_tensor(x), axis=paddle.to_tensor([axis], dtype='int32') + ) np.testing.assert_allclose(np_out, pd_out.numpy()) def test_static_and_infer(self): @@ -398,7 +412,7 @@ def test_static_and_infer(self): axis = paddle.full([1], 2, dtype='int64') out = paddle.cumsum(relu_out, axis=axis) loss = paddle.mean(out) - sgd = paddle.optimizer.SGD(learning_rate=0.) + sgd = paddle.optimizer.SGD(learning_rate=0.0) sgd.minimize(paddle.mean(out)) exe = paddle.static.Executor(self.place) @@ -407,8 +421,9 @@ def test_static_and_infer(self): # run infer paddle.static.save_inference_model(self.save_path, [x], [out], exe) - config = paddle_infer.Config(self.save_path + '.pdmodel', - self.save_path + '.pdiparams') + config = paddle_infer.Config( + self.save_path + '.pdmodel', self.save_path + '.pdiparams' + ) if paddle.is_compiled_with_cuda(): config.enable_use_gpu(100, 0) else: @@ -428,7 +443,6 @@ def test_static_and_infer(self): class TestCumsumDoubleGradCheck(unittest.TestCase): - def cumsum_wrapper(self, x): return paddle.cumsum(x[0], 0) @@ -443,17 +457,13 @@ def func(self, place): out = paddle.cumsum(data, 0) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.cumsum_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.cumsum_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -465,7 +475,6 @@ def test_grad(self): class TestCumsumTripleGradCheck(unittest.TestCase): - def cumsum_wrapper(self, x): return paddle.cumsum(x[0], 0) @@ -480,17 +489,13 @@ def func(self, place): out = paddle.cumsum(data, 0) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.cumsum_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.cumsum_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_custom_grad_input.py b/python/paddle/fluid/tests/unittests/test_custom_grad_input.py index 562290c41cfb6..74f63f03f451f 100644 --- a/python/paddle/fluid/tests/unittests/test_custom_grad_input.py +++ b/python/paddle/fluid/tests/unittests/test_custom_grad_input.py @@ -21,7 +21,6 @@ class TestTensorBackward(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -45,9 +44,9 @@ def func_tensor_backward(self): x_grad = np.matmul(grad, y.T) - np.testing.assert_allclose(x_grad, - x_tensor.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_grad, x_tensor.grad.numpy(), rtol=1e-05 + ) def test_tensor_backward(self): with _test_eager_guard(): @@ -56,7 +55,6 @@ def test_tensor_backward(self): class TestBackwardAPI(unittest.TestCase): - def setUp(self): self._dtypes = ["float32", "float64"] self._places = [paddle.CPUPlace()] @@ -77,14 +75,15 @@ def func_backward_api(self): z_tensor2 = paddle.matmul(x_tensor, y_tensor) grad_tensor = paddle.to_tensor(grad) - paddle.autograd.backward([z_tensor1, z_tensor2], - [grad_tensor, grad_tensor], True) + paddle.autograd.backward( + [z_tensor1, z_tensor2], [grad_tensor, grad_tensor], True + ) x_grad = np.matmul(grad, y.T) - np.testing.assert_allclose(x_grad * 2, - x_tensor.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_grad * 2, x_tensor.grad.numpy(), rtol=1e-05 + ) def test_backward_api(self): with _test_eager_guard(): @@ -108,9 +107,9 @@ def func_backward_single_tensor(self): x_grad = np.matmul(grad, y.T) - np.testing.assert_allclose(x_grad, - x_tensor.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_grad, x_tensor.grad.numpy(), rtol=1e-05 + ) def test_backward_single_tensor(self): with _test_eager_guard(): @@ -133,9 +132,9 @@ def func_backward_none_grad_tensor(self): x_grad = np.matmul(grad, y.T) - np.testing.assert_allclose(x_grad, - x_tensor.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_grad, x_tensor.grad.numpy(), rtol=1e-05 + ) def test_backward_none_grad_tensor(self): with _test_eager_guard(): @@ -144,15 +143,21 @@ def test_backward_none_grad_tensor(self): def func_backward_accumulator_with_init_grad(self): for dtype in self._dtypes: - x = np.random.random([ - 10, - ]).astype(dtype) - y_grad = np.random.random([ - 10, - ]).astype(dtype) - z_grad = np.random.random([ - 10, - ]).astype(dtype) + x = np.random.random( + [ + 10, + ] + ).astype(dtype) + y_grad = np.random.random( + [ + 10, + ] + ).astype(dtype) + z_grad = np.random.random( + [ + 10, + ] + ).astype(dtype) self._places = [paddle.CPUPlace()] for place in self._places: with dg.guard(place): @@ -162,16 +167,17 @@ def func_backward_accumulator_with_init_grad(self): y_grad_tensor = paddle.to_tensor(y_grad) z_grad_tensor = paddle.to_tensor(z_grad) - paddle.autograd.backward([y_tensor, z_tensor], - [y_grad_tensor, z_grad_tensor]) + paddle.autograd.backward( + [y_tensor, z_tensor], [y_grad_tensor, z_grad_tensor] + ) y = x**2 z = x**3 x_grad = 2 * x * (y_grad + 3 * y * y * z_grad) - np.testing.assert_allclose(x_grad, - x_tensor.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + x_grad, x_tensor.grad.numpy(), rtol=1e-05 + ) def test_backward_accumulator_with_init_grad(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_cvm_op.py b/python/paddle/fluid/tests/unittests/test_cvm_op.py index b4f8af82b3c33..a6197e7d72611 100644 --- a/python/paddle/fluid/tests/unittests/test_cvm_op.py +++ b/python/paddle/fluid/tests/unittests/test_cvm_op.py @@ -52,7 +52,7 @@ def cvm_grad_compute(DY, CVM, item_width, use_cvm): class TestCVMOpWithLodTensor(OpTest): """ - Test cvm op with discrete one-hot labels. + Test cvm op with discrete one-hot labels. """ def setUp(self): @@ -64,11 +64,13 @@ def setUp(self): lod = [[1]] self.inputs = { - 'X': (np.random.uniform( - 0, 1, - [self.batch_size, self.item_width]).astype("float32"), lod), - 'CVM': - np.array([[0.6, 0.4]]).astype("float32"), + 'X': ( + np.random.uniform( + 0, 1, [self.batch_size, self.item_width] + ).astype("float32"), + lod, + ), + 'CVM': np.array([[0.6, 0.4]]).astype("float32"), } self.attrs = {'use_cvm': False} out = [] @@ -80,15 +82,16 @@ def test_check_output(self): self.check_output(check_dygraph=False) def test_check_grad(self): - user_grads = np.array( - [1.0 / (self.item_width - 2)] * self.item_width).reshape( - (self.batch_size, self.item_width)).astype("float32") + user_grads = ( + np.array([1.0 / (self.item_width - 2)] * self.item_width) + .reshape((self.batch_size, self.item_width)) + .astype("float32") + ) user_grads[:, :2] = self.inputs['CVM'].reshape(self.batch_size, 2) user_grads = [user_grads] - self.check_grad(['X'], - 'Y', - user_defined_grads=user_grads, - check_dygraph=False) + self.check_grad( + ['X'], 'Y', user_defined_grads=user_grads, check_dygraph=False + ) class TestCVMOpWithOutLodTensor1(OpTest): @@ -104,10 +107,14 @@ def setUp(self): self.item_width = 11 input = np.random.uniform( - 0, 1, (self.batch_size, self.item_width)).astype('float32') + 0, 1, (self.batch_size, self.item_width) + ).astype('float32') output = cvm_compute(input, self.item_width, self.use_cvm) - cvm = np.array([[0.6, 0.4] * self.batch_size]).reshape( - (self.batch_size, 2)).astype("float32") + cvm = ( + np.array([[0.6, 0.4] * self.batch_size]) + .reshape((self.batch_size, 2)) + .astype("float32") + ) self.inputs = {'X': input, 'CVM': cvm} self.attrs = {'use_cvm': self.use_cvm} @@ -118,14 +125,16 @@ def test_check_output(self): def test_check_grad(self): numel = self.batch_size * self.item_width - user_grads = np.array([1.0 / numel] * numel).reshape( - (self.batch_size, self.item_width)).astype("float32") + user_grads = ( + np.array([1.0 / numel] * numel) + .reshape((self.batch_size, self.item_width)) + .astype("float32") + ) user_grads[:, :2] = self.inputs['CVM'].reshape(self.batch_size, 2) user_grads = [user_grads] - self.check_grad(['X'], - 'Y', - user_defined_grads=user_grads, - check_dygraph=False) + self.check_grad( + ['X'], 'Y', user_defined_grads=user_grads, check_dygraph=False + ) class TestCVMOpWithOutLodTensor2(OpTest): @@ -141,10 +150,14 @@ def setUp(self): self.item_width = 11 input = np.random.uniform( - 0, 1, (self.batch_size, self.item_width)).astype('float32') + 0, 1, (self.batch_size, self.item_width) + ).astype('float32') output = cvm_compute(input, self.item_width, self.use_cvm) - cvm = np.array([[0.6, 0.4] * self.batch_size]).reshape( - (self.batch_size, 2)).astype("float32") + cvm = ( + np.array([[0.6, 0.4] * self.batch_size]) + .reshape((self.batch_size, 2)) + .astype("float32") + ) self.inputs = {'X': input, 'CVM': cvm} self.attrs = {'use_cvm': self.use_cvm} @@ -155,15 +168,16 @@ def test_check_output(self): def test_check_grad(self): numel = self.batch_size * self.item_width - user_grads = np.array( - [1.0 / (self.batch_size * (self.item_width - 2))] * numel).reshape( - (self.batch_size, self.item_width)).astype("float32") + user_grads = ( + np.array([1.0 / (self.batch_size * (self.item_width - 2))] * numel) + .reshape((self.batch_size, self.item_width)) + .astype("float32") + ) user_grads[:, :2] = self.inputs['CVM'].reshape(self.batch_size, 2) user_grads = [user_grads] - self.check_grad(['X'], - 'Y', - user_defined_grads=user_grads, - check_dygraph=False) + self.check_grad( + ['X'], 'Y', user_defined_grads=user_grads, check_dygraph=False + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_cyclic_cifar_dataset.py b/python/paddle/fluid/tests/unittests/test_cyclic_cifar_dataset.py index e014a25ab791b..01a588c4058a4 100644 --- a/python/paddle/fluid/tests/unittests/test_cyclic_cifar_dataset.py +++ b/python/paddle/fluid/tests/unittests/test_cyclic_cifar_dataset.py @@ -17,7 +17,6 @@ class TestCifar10(unittest.TestCase): - def test_main(self): reader = paddle.dataset.cifar.train10(cycle=False) sample_num = 0 diff --git a/python/paddle/fluid/tests/unittests/test_data.py b/python/paddle/fluid/tests/unittests/test_data.py index 9beb89205f56b..bcb956e25c38a 100644 --- a/python/paddle/fluid/tests/unittests/test_data.py +++ b/python/paddle/fluid/tests/unittests/test_data.py @@ -22,7 +22,6 @@ class TestApiDataError(unittest.TestCase): - def test_fluid_data(self): with program_guard(Program(), Program()): @@ -55,7 +54,6 @@ def test_shape_type(): class TestApiStaticDataError(unittest.TestCase): - def test_fluid_dtype(self): with program_guard(Program(), Program()): x1 = paddle.static.data(name="x1", shape=[2, 25]) @@ -100,19 +98,16 @@ def test_shape_type(): class TestApiErrorWithDynamicMode(unittest.TestCase): - def test_error(self): with program_guard(Program(), Program()): paddle.disable_static() self.assertRaises(AssertionError, fluid.data, 'a', [2, 25]) - self.assertRaises(AssertionError, - fluid.layers.data, - 'b', - shape=[2, 25]) - self.assertRaises(AssertionError, - paddle.static.data, - 'c', - shape=[2, 25]) + self.assertRaises( + AssertionError, fluid.layers.data, 'b', shape=[2, 25] + ) + self.assertRaises( + AssertionError, paddle.static.data, 'c', shape=[2, 25] + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_data_generator.py b/python/paddle/fluid/tests/unittests/test_data_generator.py index 6134a1dd86cf3..b804e6a96025e 100644 --- a/python/paddle/fluid/tests/unittests/test_data_generator.py +++ b/python/paddle/fluid/tests/unittests/test_data_generator.py @@ -15,9 +15,7 @@ class MyMultiSlotDataGenerator(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -28,9 +26,7 @@ def data_iter(): class MyMultiSlotStringDataGenerator(fleet.MultiSlotStringDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -41,9 +37,7 @@ def data_iter(): class MyMultiSlotDataGenerator_error(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -54,9 +48,7 @@ def data_iter(): class MyMultiSlotDataGenerator_error_2(fleet.MultiSlotStringDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -67,9 +59,7 @@ def data_iter(): class MyMultiSlotDataGenerator_error_3(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -80,9 +70,7 @@ def data_iter(): class MyMultiSlotDataGenerator_error_4(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -93,9 +81,7 @@ def data_iter(): class MyMultiSlotDataGenerator_error_5(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -106,9 +92,7 @@ def data_iter(): class MyMultiSlotStringDataGenerator_zip(fleet.MultiSlotStringDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -121,9 +105,7 @@ def data_iter(): class MyMultiSlotDataGenerator_zip(fleet.MultiSlotDataGenerator): - def generate_sample(self, line): - def data_iter(): for i in range(40): if i == 1: @@ -136,7 +118,6 @@ def data_iter(): class TestMultiSlotDataGenerator(unittest.TestCase): - def test_MultiSlotDataGenerator_basic(self): my_ms_dg = MyMultiSlotDataGenerator() my_ms_dg.set_batch(1) @@ -144,7 +125,6 @@ def test_MultiSlotDataGenerator_basic(self): class TestMultiSlotStringDataGenerator(unittest.TestCase): - def test_MyMultiSlotStringDataGenerator_basic(self): my_ms_dg = MyMultiSlotStringDataGenerator() my_ms_dg.set_batch(1) @@ -152,7 +132,6 @@ def test_MyMultiSlotStringDataGenerator_basic(self): class TestMultiSlotDataGenerator_error(unittest.TestCase): - def test_MultiSlotDataGenerator_error(self): with self.assertRaises(ValueError): my_ms_dg = MyMultiSlotDataGenerator_error() @@ -161,7 +140,6 @@ def test_MultiSlotDataGenerator_error(self): class TestMultiSlotDataGenerator_error_2(unittest.TestCase): - def test_MultiSlotDataGenerator_error(self): with self.assertRaises(ValueError): my_ms_dg = MyMultiSlotDataGenerator_error_2() @@ -170,7 +148,6 @@ def test_MultiSlotDataGenerator_error(self): class TestMultiSlotDataGenerator_error_3(unittest.TestCase): - def test_MultiSlotDataGenerator_error(self): with self.assertRaises(ValueError): my_ms_dg = MyMultiSlotDataGenerator_error_3() @@ -179,7 +156,6 @@ def test_MultiSlotDataGenerator_error(self): class TestMultiSlotDataGenerator_error_4(unittest.TestCase): - def test_MultiSlotDataGenerator_error(self): with self.assertRaises(ValueError): my_ms_dg = MyMultiSlotDataGenerator_error_4() @@ -188,7 +164,6 @@ def test_MultiSlotDataGenerator_error(self): class TestMultiSlotDataGenerator_error_5(unittest.TestCase): - def test_MultiSlotDataGenerator_error(self): with self.assertRaises(ValueError): my_ms_dg = MyMultiSlotDataGenerator_error_5() @@ -197,7 +172,6 @@ def test_MultiSlotDataGenerator_error(self): class TestMultiSlotStringDataGeneratorZip(unittest.TestCase): - def test_MultiSlotStringDataGenerator_zip(self): my_ms_dg = MyMultiSlotStringDataGenerator_zip() my_ms_dg.set_batch(1) @@ -205,7 +179,6 @@ def test_MultiSlotStringDataGenerator_zip(self): class TestMultiSlotDataGeneratorZip(unittest.TestCase): - def test_MultiSlotDataGenerator_zip(self): my_ms_dg = MyMultiSlotDataGenerator_zip() my_ms_dg.set_batch(1) diff --git a/python/paddle/fluid/tests/unittests/test_data_norm_op.py b/python/paddle/fluid/tests/unittests/test_data_norm_op.py index cf56d57173481..1f32feb35276e 100644 --- a/python/paddle/fluid/tests/unittests/test_data_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_data_norm_op.py @@ -38,8 +38,9 @@ def _reference_testing(x, batch_size, batch_sum, batch_square_sum, slot_dim=-1): for j in range(0, x_shape[1], slot_dim): if x[i][j] <= -min_precision or x[i][j] >= min_precision: for k in range(0, slot_dim): - y[i][j + k] = (x[i][j + k] - - means_arr[j + k]) * scales_arr[j + k] + y[i][j + k] = ( + x[i][j + k] - means_arr[j + k] + ) * scales_arr[j + k] return y @@ -66,19 +67,19 @@ def setUp(self): self.use_mkldnn = False def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=1e-05, - atol=atol, - err_msg=msg) - - def check_with_place(self, - place, - data_layout, - dtype, - shape, - slot_dim=-1, - enable_scale_and_shift=False): + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=1e-05, atol=atol, err_msg=msg + ) + + def check_with_place( + self, + place, + data_layout, + dtype, + shape, + slot_dim=-1, + enable_scale_and_shift=False, + ): """ do forward and check @@ -109,24 +110,31 @@ def check_with_place(self, batch_square_sum = np.ones(scale_shape).astype(np.float32) batch_square_sum *= 1e4 - y_out = _reference_testing(x_val, batch_size, batch_sum, - batch_square_sum, slot_dim).astype(dtype) + y_out = _reference_testing( + x_val, batch_size, batch_sum, batch_square_sum, slot_dim + ).astype(dtype) scope = core.Scope() # create input - x_tensor = create_or_get_tensor(scope, "x_val", - OpTest.np_dtype_to_fluid_dtype(x_val), - place) + x_tensor = create_or_get_tensor( + scope, "x_val", OpTest.np_dtype_to_fluid_dtype(x_val), place + ) batch_size_tensor = create_or_get_tensor( - scope, "batch_size", OpTest.np_dtype_to_fluid_dtype(batch_size), - place) + scope, + "batch_size", + OpTest.np_dtype_to_fluid_dtype(batch_size), + place, + ) batch_sum_tensor = create_or_get_tensor( - scope, "batch_sum", OpTest.np_dtype_to_fluid_dtype(batch_sum), - place) + scope, "batch_sum", OpTest.np_dtype_to_fluid_dtype(batch_sum), place + ) batch_square_sum_tensor = create_or_get_tensor( - scope, "batch_square_sum", - OpTest.np_dtype_to_fluid_dtype(batch_square_sum), place) + scope, + "batch_square_sum", + OpTest.np_dtype_to_fluid_dtype(batch_square_sum), + place, + ) # create output y_tensor = create_or_get_tensor(scope, "y_out", None, place) @@ -149,15 +157,17 @@ def check_with_place(self, epsilon=epsilon, use_mkldnn=self.use_mkldnn, slot_dim=slot_dim, - enable_scale_and_shift=False) + enable_scale_and_shift=False, + ) else: scale_w = np.ones(scale_shape).astype(np.float32) bias = np.zeros(scale_shape).astype(np.float32) scale_w_tensor = create_or_get_tensor( - scope, "scale_w", OpTest.np_dtype_to_fluid_dtype(scale_w), - place) + scope, "scale_w", OpTest.np_dtype_to_fluid_dtype(scale_w), place + ) bias_tensor = create_or_get_tensor( - scope, "bias", OpTest.np_dtype_to_fluid_dtype(bias), place) + scope, "bias", OpTest.np_dtype_to_fluid_dtype(bias), place + ) data_norm_op = Operator( "data_norm", # inputs @@ -175,17 +185,25 @@ def check_with_place(self, epsilon=epsilon, use_mkldnn=self.use_mkldnn, slot_dim=slot_dim, - enable_scale_and_shift=True) + enable_scale_and_shift=True, + ) data_norm_op.run(scope, place) # check inference result - self.__assert_close(y_tensor, - y_out, - "inference output are different at " + str(place) + - ", " + data_layout + ", " + str(np.dtype(dtype)) + - str(np.array(y_tensor)) + str(y_out), - atol=1e-3) + self.__assert_close( + y_tensor, + y_out, + "inference output are different at " + + str(place) + + ", " + + data_layout + + ", " + + str(np.dtype(dtype)) + + str(np.array(y_tensor)) + + str(y_out), + atol=1e-3, + ) def test_check_output(self): """ @@ -199,9 +217,11 @@ def test_check_output(self): self.check_with_place( place, data_format, - self.dtype, [2, 3], + self.dtype, + [2, 3], slot_dim=slot_dim, - enable_scale_and_shift=enable_scale_and_shift) + enable_scale_and_shift=enable_scale_and_shift, + ) class TestDataNormOp(OpTest): @@ -237,7 +257,7 @@ def setUp(self): "X": x_val, "BatchSize": batch_size, "BatchSum": batch_sum, - "BatchSquareSum": batch_square_sum + "BatchSquareSum": batch_square_sum, } self.outputs = {"Y": y, "Means": mean, "Scales": scale} self.attrs = {"epsilon": epsilon, "use_mkldnn": self.use_mkldnn} @@ -294,14 +314,14 @@ def setUp(self): "BatchSum": batch_sum, "BatchSquareSum": batch_square_sum, "scale_w": scale_w, - "bias": bias + "bias": bias, } self.outputs = {"Y": y, "Means": mean, "Scales": scale} self.attrs = { "epsilon": epsilon, "use_mkldnn": self.use_mkldnn, "slot_dim": slot_dim, - "enable_scale_and_shift": True + "enable_scale_and_shift": True, } def test_check_output(self): @@ -356,7 +376,7 @@ def setUp(self): "BatchSum": batch_sum, "BatchSquareSum": batch_square_sum, "scale_w": scale_w, - "bias": bias + "bias": bias, } self.outputs = {"Y": y, "Means": mean, "Scales": scale} self.attrs = {"epsilon": epsilon, "use_mkldnn": self.use_mkldnn} @@ -413,14 +433,14 @@ def setUp(self): "BatchSum": batch_sum, "BatchSquareSum": batch_square_sum, "scale_w": scale_w, - "bias": bias + "bias": bias, } self.outputs = {"Y": y, "Means": mean, "Scales": scale} self.attrs = { "epsilon": epsilon, "use_mkldnn": self.use_mkldnn, "slot_dim": slot_dim, - "enable_scale_and_shift": True + "enable_scale_and_shift": True, } def test_check_output(self): @@ -470,13 +490,13 @@ def setUp(self): "X": x_val, "BatchSize": batch_size, "BatchSum": batch_sum, - "BatchSquareSum": batch_square_sum + "BatchSquareSum": batch_square_sum, } self.outputs = {"Y": y, "Means": mean, "Scales": scale} self.attrs = { "epsilon": epsilon, "use_mkldnn": self.use_mkldnn, - "slot_dim": slot_dim + "slot_dim": slot_dim, } def test_check_output(self): @@ -493,14 +513,13 @@ def test_check_grad(self): class TestDataNormOpErrorr(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x2 = fluid.layers.data(name='x2', shape=[3, 4], dtype="int32") - #self.assertRaises(TypeError, fluid.data_norm, x2) - fluid.layers.data_norm(input=x2, - param_attr={}, - enable_scale_and_shift=True) + # self.assertRaises(TypeError, fluid.data_norm, x2) + fluid.layers.data_norm( + input=x2, param_attr={}, enable_scale_and_shift=True + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_dataloader_autotune.py b/python/paddle/fluid/tests/unittests/test_dataloader_autotune.py index 29169ca6d7654..00630a398223f 100755 --- a/python/paddle/fluid/tests/unittests/test_dataloader_autotune.py +++ b/python/paddle/fluid/tests/unittests/test_dataloader_autotune.py @@ -25,13 +25,12 @@ class RandomDataset(Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([10]).astype('float32') - label = np.random.randint(0, 10 - 1, (1, )).astype('int64') + label = np.random.randint(0, 10 - 1, (1,)).astype('int64') return image, label def __len__(self): @@ -39,7 +38,6 @@ def __len__(self): class SimpleNet(nn.Layer): - def __init__(self): super(SimpleNet, self).__init__() self.fc = nn.Linear(10, 10) @@ -49,20 +47,22 @@ def forward(self, image): class TestAutoTune(unittest.TestCase): - def setUp(self): self.batch_size = 1 self.dataset = RandomDataset(10) def test_dataloader_use_autotune(self): paddle.incubate.autotune.set_config( - config={"dataloader": { - "enable": True, - "tuning_steps": 1, - }}) - loader = DataLoader(self.dataset, - batch_size=self.batch_size, - num_workers=0) + config={ + "dataloader": { + "enable": True, + "tuning_steps": 1, + } + } + ) + loader = DataLoader( + self.dataset, batch_size=self.batch_size, num_workers=0 + ) def test_dataloader_disable_autotune(self): config = {"dataloader": {"enable": False, "tuning_steps": 1}} @@ -71,29 +71,32 @@ def test_dataloader_disable_autotune(self): tfile.close() paddle.incubate.autotune.set_config(tfile.name) os.remove(tfile.name) - loader = DataLoader(self.dataset, - batch_size=self.batch_size, - num_workers=2) - if (sys.platform == 'darwin' or sys.platform == 'win32'): + loader = DataLoader( + self.dataset, batch_size=self.batch_size, num_workers=2 + ) + if sys.platform == 'darwin' or sys.platform == 'win32': self.assertEqual(loader.num_workers, 0) else: self.assertEqual(loader.num_workers, 2) def test_distributer_batch_sampler_autotune(self): paddle.incubate.autotune.set_config( - config={"dataloader": { - "enable": True, - "tuning_steps": 1, - }}) + config={ + "dataloader": { + "enable": True, + "tuning_steps": 1, + } + } + ) batch_sampler = paddle.io.DistributedBatchSampler( - self.dataset, batch_size=self.batch_size) - loader = DataLoader(self.dataset, - batch_sampler=batch_sampler, - num_workers=2) + self.dataset, batch_size=self.batch_size + ) + loader = DataLoader( + self.dataset, batch_sampler=batch_sampler, num_workers=2 + ) class TestAutoTuneAPI(unittest.TestCase): - def test_set_config_warnings(self): with warnings.catch_warnings(record=True) as w: config = {"kernel": {"enable": 1, "tuning_range": True}} diff --git a/python/paddle/fluid/tests/unittests/test_dataloader_dataset.py b/python/paddle/fluid/tests/unittests/test_dataloader_dataset.py index 6b3ab060c3dbd..2c4bcfa444b0f 100644 --- a/python/paddle/fluid/tests/unittests/test_dataloader_dataset.py +++ b/python/paddle/fluid/tests/unittests/test_dataloader_dataset.py @@ -22,7 +22,6 @@ class TestDatasetAbstract(unittest.TestCase): - def func_test_main(self): dataset = Dataset() try: @@ -44,20 +43,21 @@ def test_main(self): class TestDatasetWithDiffOutputPlace(unittest.TestCase): - def get_dataloader(self, num_workers): dataset = paddle.vision.datasets.MNIST( mode='test', - transform=transforms.Compose([ - transforms.CenterCrop(20), - transforms.RandomResizedCrop(14), - transforms.Normalize(), - transforms.ToTensor() - ])) - loader = paddle.io.DataLoader(dataset, - batch_size=32, - num_workers=num_workers, - shuffle=True) + transform=transforms.Compose( + [ + transforms.CenterCrop(20), + transforms.RandomResizedCrop(14), + transforms.Normalize(), + transforms.ToTensor(), + ] + ), + ) + loader = paddle.io.DataLoader( + dataset, batch_size=32, num_workers=num_workers, shuffle=True + ) return loader def run_check_on_cpu(self): diff --git a/python/paddle/fluid/tests/unittests/test_dataloader_early_reset.py b/python/paddle/fluid/tests/unittests/test_dataloader_early_reset.py index 3c03a289728a2..fa0bac6c5bda9 100644 --- a/python/paddle/fluid/tests/unittests/test_dataloader_early_reset.py +++ b/python/paddle/fluid/tests/unittests/test_dataloader_early_reset.py @@ -25,7 +25,6 @@ def infinite_reader(): class TestDataLoaderEarlyReset(unittest.TestCase): - def setUp(self): self.stop_batch = 10 self.iterable = True @@ -45,9 +44,9 @@ def get_place(self): def create_data_loader(self): self.x = fluid.data(name='x', shape=[None, 32], dtype='float32') - return fluid.io.DataLoader.from_generator(feed_list=[self.x], - capacity=10, - iterable=self.iterable) + return fluid.io.DataLoader.from_generator( + feed_list=[self.x], capacity=10, iterable=self.iterable + ) def test_main(self): with fluid.program_guard(fluid.Program(), fluid.Program()): @@ -68,7 +67,7 @@ def run_network(self): batch_id = 0 if loader.iterable: for data in loader(): - x_val, = exe.run(prog, feed=data, fetch_list=[self.x]) + (x_val,) = exe.run(prog, feed=data, fetch_list=[self.x]) self.assertTrue(np.all(x_val == batch_id)) batch_id += 1 if batch_id >= self.stop_batch: @@ -89,7 +88,6 @@ def run_network(self): class TestDataLoaderEarlyReset2(TestDataLoaderEarlyReset): - def setUp(self): self.stop_batch = 20 self.iterable = False diff --git a/python/paddle/fluid/tests/unittests/test_dataloader_keep_order.py b/python/paddle/fluid/tests/unittests/test_dataloader_keep_order.py index 5d07586c7e62c..f2dd18c008082 100644 --- a/python/paddle/fluid/tests/unittests/test_dataloader_keep_order.py +++ b/python/paddle/fluid/tests/unittests/test_dataloader_keep_order.py @@ -19,7 +19,6 @@ def create_reader(shape, batch_number): - def __impl__(): idx = 0 for _ in range(batch_number): @@ -30,7 +29,6 @@ def __impl__(): class DataLoaderKeepOrderTestBase(unittest.TestCase): - def initParameters(self): self.iterable = False self.break_num = 100 @@ -43,15 +41,17 @@ def setUp(self): def build_network(self, places): input_data = fluid.data(shape=self.shape, dtype='float32', name="input") - loader = fluid.io.DataLoader.from_generator(capacity=16, - feed_list=[input_data], - iterable=self.iterable) + loader = fluid.io.DataLoader.from_generator( + capacity=16, feed_list=[input_data], iterable=self.iterable + ) fc = fluid.layers.fc(input_data, size=10) loss = fluid.layers.reduce_mean(fc) - loader.set_batch_generator(create_reader(self.shape, self.batch_num), - places=places if loader.iterable else None) + loader.set_batch_generator( + create_reader(self.shape, self.batch_num), + places=places if loader.iterable else None, + ) return input_data, loss, loader @@ -65,12 +65,15 @@ def assertInputData(self, batch_id, input_data, dev_cnt): self.assertTrue((input_tensor == start_val).all()) start_val += 1 else: - self.assertEqual(list(input_data.shape), - [self.shape[0] * dev_cnt] + self.shape[1:]) + self.assertEqual( + list(input_data.shape), + [self.shape[0] * dev_cnt] + self.shape[1:], + ) start_val = dev_cnt * batch_id for idx in range(dev_cnt): - data_part = input_data[idx * self.shape[0]:(idx + 1) * - self.shape[0], :] + data_part = input_data[ + idx * self.shape[0] : (idx + 1) * self.shape[0], : + ] self.assertTrue((data_part == start_val).all()) start_val += 1 @@ -81,8 +84,8 @@ def get_places(self): place_list.extend([fluid.cuda_places(0)]) else: place_list.extend( - [fluid.cuda_places(0), - fluid.cuda_places([0, 1])]) + [fluid.cuda_places(0), fluid.cuda_places([0, 1])] + ) return place_list def test_main(self): @@ -107,11 +110,12 @@ def run_main_with_place(self, places, use_compiled_program=True): main_program = fluid.default_main_program() if use_compiled_program: main_program = fluid.CompiledProgram( - main_program).with_data_parallel(loss_name=loss.name, - places=places) + main_program + ).with_data_parallel(loss_name=loss.name, places=places) - max_batch_num = min(self.break_num, - int(self.batch_num / dev_cnt)) + max_batch_num = min( + self.break_num, int(self.batch_num / dev_cnt) + ) if loader.iterable: early_break = False @@ -123,9 +127,11 @@ def run_main_with_place(self, places, use_compiled_program=True): early_break = True break self.assertInputData(batch_id, data, dev_cnt) - fetch_val, = exe.run(program=main_program, - feed=data, - fetch_list=fetch_list) + (fetch_val,) = exe.run( + program=main_program, + feed=data, + fetch_list=fetch_list, + ) self.assertInputData(batch_id, fetch_val, dev_cnt) batch_id += 1 @@ -142,10 +148,12 @@ def run_main_with_place(self, places, use_compiled_program=True): if batch_id >= self.break_num: loader.reset() break - fetch_val, = exe.run(program=main_program, - fetch_list=fetch_list) - self.assertInputData(batch_id, fetch_val, - dev_cnt) + (fetch_val,) = exe.run( + program=main_program, fetch_list=fetch_list + ) + self.assertInputData( + batch_id, fetch_val, dev_cnt + ) batch_id += 1 except fluid.core.EOFException: loader.reset() @@ -154,35 +162,30 @@ def run_main_with_place(self, places, use_compiled_program=True): class IterableDataLoaderKeepOrderTest2(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = True self.break_num = 100 class IterableDataLoaderKeepOrderTest3(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = False self.break_num = 2 class IterableDataLoaderKeepOrderTest4(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = True self.break_num = 2 class IterableDataLoaderKeepOrderTest5(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = False self.break_num = 0 class IterableDataLoaderKeepOrderTest6(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = True self.break_num = 0 diff --git a/python/paddle/fluid/tests/unittests/test_dataloader_unkeep_order.py b/python/paddle/fluid/tests/unittests/test_dataloader_unkeep_order.py index 236ffe19b6e2b..8e121ba401ee2 100644 --- a/python/paddle/fluid/tests/unittests/test_dataloader_unkeep_order.py +++ b/python/paddle/fluid/tests/unittests/test_dataloader_unkeep_order.py @@ -22,7 +22,6 @@ def create_reader(shape, batch_number): - def __impl__(): idx = 0 for _ in range(batch_number): @@ -33,7 +32,6 @@ def __impl__(): class DataLoaderKeepOrderTestBase(unittest.TestCase): - def initParameters(self): self.iterable = False self.break_num = 10000 @@ -49,23 +47,23 @@ def clear_visited(self): def build_network(self, places): input_data = fluid.data(shape=self.shape, dtype='float32', name="input") - loader = fluid.io.DataLoader.from_generator(capacity=16, - feed_list=[input_data], - iterable=self.iterable) + loader = fluid.io.DataLoader.from_generator( + capacity=16, feed_list=[input_data], iterable=self.iterable + ) fc = fluid.layers.fc(input_data, size=10) loss = fluid.layers.reduce_mean(fc) - loader.set_batch_generator(create_reader(self.shape, self.batch_num), - places=places if loader.iterable else None) + loader.set_batch_generator( + create_reader(self.shape, self.batch_num), + places=places if loader.iterable else None, + ) return input_data, loss, loader - def assertInputData(self, - batch_id, - input_data, - dev_cnt, - check_visited=True): + def assertInputData( + self, batch_id, input_data, dev_cnt, check_visited=True + ): if isinstance(input_data, list): self.assertTrue(len(input_data), dev_cnt) start_val = dev_cnt * batch_id @@ -82,12 +80,15 @@ def assertInputData(self, start_val += 1 else: - self.assertEqual(list(input_data.shape), - [self.shape[0] * dev_cnt] + self.shape[1:]) + self.assertEqual( + list(input_data.shape), + [self.shape[0] * dev_cnt] + self.shape[1:], + ) start_val = dev_cnt * batch_id for idx in range(dev_cnt): - data_part = input_data[idx * self.shape[0]:(idx + 1) * - self.shape[0], :] + data_part = input_data[ + idx * self.shape[0] : (idx + 1) * self.shape[0], : + ] num = data_part.flatten()[0] self.assertTrue((data_part == num).all()) if check_visited: @@ -103,8 +104,8 @@ def get_places(self): place_list.extend([fluid.cuda_places(0)]) else: place_list.extend( - [fluid.cuda_places(0), - fluid.cuda_places([0, 1])]) + [fluid.cuda_places(0), fluid.cuda_places([0, 1])] + ) return place_list def test_main(self): @@ -129,11 +130,12 @@ def run_main_with_place(self, places, use_compiled_program=True): main_program = fluid.default_main_program() if use_compiled_program: main_program = fluid.CompiledProgram( - main_program).with_data_parallel(loss_name=loss.name, - places=places) + main_program + ).with_data_parallel(loss_name=loss.name, places=places) - max_batch_num = min(self.break_num, - int(self.batch_num / dev_cnt)) + max_batch_num = min( + self.break_num, int(self.batch_num / dev_cnt) + ) if loader.iterable: early_break = False @@ -145,13 +147,14 @@ def run_main_with_place(self, places, use_compiled_program=True): if batch_id >= self.break_num: early_break = True break - self.assertInputData(batch_id, - data, - dev_cnt, - check_visited=False) - fetch_val, = exe.run(program=main_program, - feed=data, - fetch_list=fetch_list) + self.assertInputData( + batch_id, data, dev_cnt, check_visited=False + ) + (fetch_val,) = exe.run( + program=main_program, + feed=data, + fetch_list=fetch_list, + ) self.assertInputData(batch_id, fetch_val, dev_cnt) batch_id += 1 @@ -172,10 +175,12 @@ def run_main_with_place(self, places, use_compiled_program=True): if batch_id >= self.break_num: loader.reset() break - fetch_val, = exe.run(program=main_program, - fetch_list=fetch_list) - self.assertInputData(batch_id, fetch_val, - dev_cnt) + (fetch_val,) = exe.run( + program=main_program, fetch_list=fetch_list + ) + self.assertInputData( + batch_id, fetch_val, dev_cnt + ) batch_id += 1 except fluid.core.EOFException: loader.reset() @@ -187,35 +192,30 @@ def run_main_with_place(self, places, use_compiled_program=True): class IterableDataLoaderKeepOrderTest2(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = True self.break_num = 10000 class IterableDataLoaderKeepOrderTest3(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = False self.break_num = 2 class IterableDataLoaderKeepOrderTest4(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = True self.break_num = 2 class IterableDataLoaderKeepOrderTest5(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = False self.break_num = 0 class IterableDataLoaderKeepOrderTest6(DataLoaderKeepOrderTestBase): - def initParameters(self): self.iterable = True self.break_num = 0 diff --git a/python/paddle/fluid/tests/unittests/test_dataset.py b/python/paddle/fluid/tests/unittests/test_dataset.py index 8e09e8f75c90c..5c0dc9db6a244 100644 --- a/python/paddle/fluid/tests/unittests/test_dataset.py +++ b/python/paddle/fluid/tests/unittests/test_dataset.py @@ -25,7 +25,7 @@ class TestDataset(unittest.TestCase): - """ TestCases for Dataset. """ + """TestCases for Dataset.""" def setUp(self): self.use_data_loader = False @@ -33,7 +33,7 @@ def setUp(self): self.drop_last = False def test_dataset_create(self): - """ Testcase for dataset create. """ + """Testcase for dataset create.""" try: dataset = paddle.distributed.InMemoryDataset() except: @@ -99,22 +99,22 @@ def test_run_with_dump(self): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=slots_vars + ) dataset.update_settings(pipe_command="cat1") - dataset._init_distributed_settings(parse_ins_id=True, - parse_content=True, - fea_eval=True, - candidate_size=10000) + dataset._init_distributed_settings( + parse_ins_id=True, + parse_content=True, + fea_eval=True, + candidate_size=10000, + ) dataset.set_filelist([dump_a_path, dump_b_path]) dataset.load_into_memory() dataset.local_shuffle() @@ -136,7 +136,7 @@ def test_run_with_dump(self): temp_dir.cleanup() def test_dataset_config(self): - """ Testcase for dataset configuration. """ + """Testcase for dataset configuration.""" dataset = fluid.core.Dataset("MultiSlotDataset") dataset.set_thread_num(12) dataset.set_filelist(["a.txt", "b.txt", "c.txt"]) @@ -169,10 +169,12 @@ def test_set_download_cmd(self): Testcase for InMemoryDataset from create to run. """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "afs:test_in_memory_dataset_run_a.txt") - filename2 = os.path.join(temp_dir.name, - "afs:test_in_memory_dataset_run_b.txt") + filename1 = os.path.join( + temp_dir.name, "afs:test_in_memory_dataset_run_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "afs:test_in_memory_dataset_run_b.txt" + ) with open(filename1, "w") as f: data = "1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -189,18 +191,19 @@ def test_set_download_cmd(self): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - download_cmd="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, + thread_num=3, + pipe_command="cat", + download_cmd="cat", + use_var=slots_vars, + ) dataset.set_filelist([filename1, filename2]) dataset.load_into_memory() paddle.enable_static() @@ -212,7 +215,8 @@ def test_set_download_cmd(self): exe.run(startup_program) if self.use_data_loader: data_loader = fluid.io.DataLoader.from_dataset( - dataset, fluid.cpu_places(), self.drop_last) + dataset, fluid.cpu_places(), self.drop_last + ) for i in range(self.epoch_num): for data in data_loader(): exe.run(main_program, feed=data) @@ -230,10 +234,12 @@ def test_in_memory_dataset_run(self): Testcase for InMemoryDataset from create to run. """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "test_in_memory_dataset_run_a.txt") - filename2 = os.path.join(temp_dir.name, - "test_in_memory_dataset_run_b.txt") + filename1 = os.path.join( + temp_dir.name, "test_in_memory_dataset_run_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "test_in_memory_dataset_run_b.txt" + ) with open(filename1, "w") as f: data = "1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -250,17 +256,15 @@ def test_in_memory_dataset_run(self): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=slots_vars + ) dataset._init_distributed_settings(fea_eval=True, candidate_size=1) dataset.set_filelist([filename1, filename2]) dataset.load_into_memory() @@ -272,15 +276,17 @@ def test_in_memory_dataset_run(self): exe.run(fluid.default_startup_program()) if self.use_data_loader: data_loader = fluid.io.DataLoader.from_dataset( - dataset, fluid.cpu_places(), self.drop_last) + dataset, fluid.cpu_places(), self.drop_last + ) for i in range(self.epoch_num): for data in data_loader(): exe.run(fluid.default_main_program(), feed=data) else: for i in range(self.epoch_num): try: - exe.train_from_dataset(fluid.default_main_program(), - dataset) + exe.train_from_dataset( + fluid.default_main_program(), dataset + ) except Exception as e: self.assertTrue(False) @@ -291,10 +297,12 @@ def test_in_memory_dataset_masterpatch(self): Testcase for InMemoryDataset from create to run. """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "test_in_memory_dataset_masterpatch_a.txt") - filename2 = os.path.join(temp_dir.name, - "test_in_memory_dataset_masterpatch_b.txt") + filename1 = os.path.join( + temp_dir.name, "test_in_memory_dataset_masterpatch_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "test_in_memory_dataset_masterpatch_b.txt" + ) with open(filename1, "w") as f: data = "1 id1 1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -320,28 +328,27 @@ def test_in_memory_dataset_masterpatch(self): startup_program = fluid.Program() with fluid.program_guard(train_program, startup_program): for slot in slots[:2]: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) for slot in slots[2:]: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="float32", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="float32", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=1, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=1, pipe_command="cat", use_var=slots_vars + ) dataset._init_distributed_settings(parse_ins_id=True) - dataset.set_filelist([ - "test_in_memory_dataset_masterpatch_a.txt", - "test_in_memory_dataset_masterpatch_b.txt" - ]) + dataset.set_filelist( + [ + "test_in_memory_dataset_masterpatch_a.txt", + "test_in_memory_dataset_masterpatch_b.txt", + ] + ) dataset.load_into_memory() dataset.local_shuffle() @@ -356,7 +363,7 @@ def test_in_memory_dataset_masterpatch(self): except Exception as e: self.assertTrue(False) - #dataset._set_merge_by_lineid(2) + # dataset._set_merge_by_lineid(2) dataset.update_settings(merge_size=2) dataset.dataset.merge_by_lineid() @@ -367,10 +374,12 @@ def test_in_memory_dataset_masterpatch1(self): Testcase for InMemoryDataset from create to run. """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "test_in_memory_dataset_masterpatch1_a.txt") - filename2 = os.path.join(temp_dir.name, - "test_in_memory_dataset_masterpatch1_b.txt") + filename1 = os.path.join( + temp_dir.name, "test_in_memory_dataset_masterpatch1_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "test_in_memory_dataset_masterpatch1_b.txt" + ) with open(filename1, "w") as f: data = "1 id1 1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -394,34 +403,31 @@ def test_in_memory_dataset_masterpatch1(self): train_program = fluid.Program() startup_program = fluid.Program() with fluid.program_guard(train_program, startup_program): - var1 = fluid.layers.data(name="slot1", - shape=[1], - dtype="int64", - lod_level=0) - var2 = fluid.layers.data(name="slot2", - shape=[1], - dtype="int64", - lod_level=0) - var3 = fluid.layers.data(name="slot3", - shape=[1], - dtype="float32", - lod_level=0) - var4 = fluid.layers.data(name="slot4", - shape=[1], - dtype="float32", - lod_level=0) + var1 = fluid.layers.data( + name="slot1", shape=[1], dtype="int64", lod_level=0 + ) + var2 = fluid.layers.data( + name="slot2", shape=[1], dtype="int64", lod_level=0 + ) + var3 = fluid.layers.data( + name="slot3", shape=[1], dtype="float32", lod_level=0 + ) + var4 = fluid.layers.data( + name="slot4", shape=[1], dtype="float32", lod_level=0 + ) slots_vars = [var1, var2, var3, var4] dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=1, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=1, pipe_command="cat", use_var=slots_vars + ) dataset._init_distributed_settings(parse_ins_id=True) - dataset.set_filelist([ - "test_in_memory_dataset_masterpatch1_a.txt", - "test_in_memory_dataset_masterpatch1_b.txt" - ]) + dataset.set_filelist( + [ + "test_in_memory_dataset_masterpatch1_a.txt", + "test_in_memory_dataset_masterpatch1_b.txt", + ] + ) dataset.load_into_memory() dataset.local_shuffle() @@ -448,10 +454,12 @@ def test_in_memory_dataset_run_2(self): Use float type id """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "test_in_memory_dataset_run_a.txt") - filename2 = os.path.join(temp_dir.name, - "test_in_memory_dataset_run_b.txt") + filename1 = os.path.join( + temp_dir.name, "test_in_memory_dataset_run_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "test_in_memory_dataset_run_b.txt" + ) with open(filename1, "w") as f: data = "1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -468,43 +476,44 @@ def test_in_memory_dataset_run_2(self): slots = ["slot1_f", "slot2_f", "slot3_f", "slot4_f"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="float32", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="float32", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=slots_vars + ) dataset.set_filelist([filename1, filename2]) dataset.load_into_memory() dataset.local_shuffle() - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe.run(fluid.default_startup_program()) for i in range(2): try: exe.train_from_dataset(fluid.default_main_program(), dataset) - exe.train_from_dataset(fluid.default_main_program(), - dataset, - thread=1) - exe.train_from_dataset(fluid.default_main_program(), - dataset, - thread=2) - exe.train_from_dataset(fluid.default_main_program(), - dataset, - thread=2) - exe.train_from_dataset(fluid.default_main_program(), - dataset, - thread=3) - exe.train_from_dataset(fluid.default_main_program(), - dataset, - thread=4) + exe.train_from_dataset( + fluid.default_main_program(), dataset, thread=1 + ) + exe.train_from_dataset( + fluid.default_main_program(), dataset, thread=2 + ) + exe.train_from_dataset( + fluid.default_main_program(), dataset, thread=2 + ) + exe.train_from_dataset( + fluid.default_main_program(), dataset, thread=3 + ) + exe.train_from_dataset( + fluid.default_main_program(), dataset, thread=4 + ) except ImportError as e: pass except Exception as e: @@ -512,15 +521,17 @@ def test_in_memory_dataset_run_2(self): if self.use_data_loader: data_loader = fluid.io.DataLoader.from_dataset( - dataset, fluid.cpu_places(), self.drop_last) + dataset, fluid.cpu_places(), self.drop_last + ) for i in range(self.epoch_num): for data in data_loader(): exe.run(fluid.default_main_program(), feed=data) else: for i in range(self.epoch_num): try: - exe.train_from_dataset(fluid.default_main_program(), - dataset) + exe.train_from_dataset( + fluid.default_main_program(), dataset + ) except Exception as e: self.assertTrue(False) @@ -536,20 +547,22 @@ def test_in_memory_dataset_run_2(self): dataset._set_parse_ins_id(False) dataset.load_into_memory() dataset.dataset.merge_by_lineid() - dataset.update_settings(batch_size=1, - thread_num=2, - input_type=1, - pipe_command="cat", - use_var=[], - fs_name="", - fs_ugi="", - download_cmd="cat", - merge_size=-1, - parse_ins_id=False, - parse_content=False, - fleet_send_batch_size=2, - fleet_send_sleep_seconds=2, - fea_eval=True) + dataset.update_settings( + batch_size=1, + thread_num=2, + input_type=1, + pipe_command="cat", + use_var=[], + fs_name="", + fs_ugi="", + download_cmd="cat", + merge_size=-1, + parse_ins_id=False, + parse_content=False, + fleet_send_batch_size=2, + fleet_send_sleep_seconds=2, + fea_eval=True, + ) fleet_ptr = fluid.core.Fleet() fleet_ptr.set_client2client_config(1, 1, 1) fleet_ptr.get_cache_threshold(0) @@ -579,40 +592,39 @@ def test_queue_dataset_run(self): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.QueueDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=slots_vars + ) dataset.set_filelist([filename1, filename2]) exe = fluid.Executor(fluid.CPUPlace()) exe.run(fluid.default_startup_program()) if self.use_data_loader: data_loader = fluid.io.DataLoader.from_dataset( - dataset, fluid.cpu_places(), self.drop_last) + dataset, fluid.cpu_places(), self.drop_last + ) for i in range(self.epoch_num): for data in data_loader(): exe.run(fluid.default_main_program(), feed=data) else: for i in range(self.epoch_num): try: - exe.train_from_dataset(fluid.default_main_program(), - dataset) + exe.train_from_dataset( + fluid.default_main_program(), dataset + ) except Exception as e: self.assertTrue(False) dataset2 = paddle.distributed.QueueDataset() - dataset2.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset2.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=slots_vars + ) dataset.set_filelist([]) try: exe.train_from_dataset(fluid.default_main_program(), dataset2) @@ -648,33 +660,36 @@ def test_queue_dataset_run_2(self): slots = ["slot1_f", "slot2_f", "slot3_f", "slot4_f"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="float32", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="float32", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.QueueDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=slots_vars + ) dataset.set_filelist([filename1, filename2]) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe.run(fluid.default_startup_program()) if self.use_data_loader: data_loader = fluid.io.DataLoader.from_dataset( - dataset, fluid.cpu_places(), self.drop_last) + dataset, fluid.cpu_places(), self.drop_last + ) for i in range(self.epoch_num): for data in data_loader(): exe.run(fluid.default_main_program(), feed=data) else: for i in range(self.epoch_num): try: - exe.train_from_dataset(fluid.default_main_program(), - dataset) + exe.train_from_dataset( + fluid.default_main_program(), dataset + ) except Exception as e: self.assertTrue(False) @@ -706,35 +721,41 @@ def test_queue_dataset_run_3(self): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.data(name=slot, - shape=[None, 1], - dtype="int64", - lod_level=1) + var = fluid.data( + name=slot, shape=[None, 1], dtype="int64", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=1, - thread_num=2, - input_type=1, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=1, + thread_num=2, + input_type=1, + pipe_command="cat", + use_var=slots_vars, + ) dataset.set_filelist([filename1, filename2]) dataset.load_into_memory() - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe.run(fluid.default_startup_program()) if self.use_data_loader: data_loader = fluid.io.DataLoader.from_dataset( - dataset, fluid.cpu_places(), self.drop_last) + dataset, fluid.cpu_places(), self.drop_last + ) for i in range(self.epoch_num): for data in data_loader(): exe.run(fluid.default_main_program(), feed=data) else: for i in range(self.epoch_num): try: - exe.train_from_dataset(fluid.default_main_program(), - dataset) + exe.train_from_dataset( + fluid.default_main_program(), dataset + ) except Exception as e: self.assertTrue(False) @@ -764,22 +785,25 @@ def test_run_with_inmemory_dataset_train_debug_mode(self): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=1, - pipe_command="cat", - data_feed_type="SlotRecordInMemoryDataFeed", - use_var=slots_vars) - dataset._init_distributed_settings(parse_ins_id=True, - parse_content=True, - fea_eval=True, - candidate_size=10000) + dataset.init( + batch_size=32, + thread_num=1, + pipe_command="cat", + data_feed_type="SlotRecordInMemoryDataFeed", + use_var=slots_vars, + ) + dataset._init_distributed_settings( + parse_ins_id=True, + parse_content=True, + fea_eval=True, + candidate_size=10000, + ) dataset.set_filelist([dump_a_path, dump_b_path]) dataset.load_into_memory() @@ -827,10 +851,9 @@ def net(self): slots_vars = [] poolings = [] for slot in slots: - data = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) var = fluid.layers.cast(x=data, dtype='float32') pool = fluid.layers.sequence_pool(input=var, pool_type='AVERAGE') @@ -850,10 +873,9 @@ def get_dataset(self, inputs, files): files(list): files of get_dataset """ dataset = paddle.distributed.QueueDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=inputs) + dataset.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=inputs + ) dataset.set_filelist(files) return dataset @@ -862,10 +884,12 @@ def setUp(self): Test Dataset With Fetch Handler. TestCases. """ self.temp_dir = tempfile.TemporaryDirectory() - self.filename1 = os.path.join(self.temp_dir.name, - "test_queue_dataset_run_a.txt") - self.filename2 = os.path.join(self.temp_dir.name, - "test_queue_dataset_run_b.txt") + self.filename1 = os.path.join( + self.temp_dir.name, "test_queue_dataset_run_a.txt" + ) + self.filename2 = os.path.join( + self.temp_dir.name, "test_queue_dataset_run_b.txt" + ) with open(self.filename1, "w") as f: data = "1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -940,9 +964,11 @@ def test_fetch_handler(self): fh.help() try: - exe.train_from_dataset(program=fluid.default_main_program(), - dataset=dataset, - fetch_handler=fh) + exe.train_from_dataset( + program=fluid.default_main_program(), + dataset=dataset, + fetch_handler=fh, + ) except ImportError as e: print("warning: we skip trainer_desc_pb2 import problem in windows") except RuntimeError as e: @@ -953,10 +979,10 @@ def test_fetch_handler(self): class TestDataset2(unittest.TestCase): - """ TestCases for Dataset. """ + """TestCases for Dataset.""" def setUp(self): - """ TestCases for Dataset. """ + """TestCases for Dataset.""" self.use_data_loader = False self.epoch_num = 10 self.drop_last = False @@ -966,10 +992,12 @@ def test_dataset_fleet(self): Testcase for InMemoryDataset from create to run. """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "test_in_memory_dataset2_run_a.txt") - filename2 = os.path.join(temp_dir.name, - "test_in_memory_dataset2_run_b.txt") + filename1 = os.path.join( + temp_dir.name, "test_in_memory_dataset2_run_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "test_in_memory_dataset2_run_b.txt" + ) self.skipTest("parameter server will add pslib UT later") @@ -988,16 +1016,21 @@ def test_dataset_fleet(self): train_program = fluid.Program() startup_program = fluid.Program() scope = fluid.Scope() - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, + ) + with fluid.program_guard(train_program, startup_program): slots = ["slot1_ff", "slot2_ff", "slot3_ff", "slot4_ff"] slots_vars = [] for slot in slots: - var = fluid.layers.data(\ - name=slot, shape=[1], dtype="float32", lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="float32", lod_level=1 + ) slots_vars.append(var) - fake_cost = \ - fluid.layers.elementwise_sub(slots_vars[0], slots_vars[-1]) + fake_cost = fluid.layers.elementwise_sub( + slots_vars[0], slots_vars[-1] + ) fake_cost = paddle.mean(fake_cost) with fluid.scope_guard(scope): place = fluid.CPUPlace() @@ -1017,10 +1050,12 @@ def test_dataset_fleet(self): exe.run(startup_program) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, + thread_num=3, + pipe_command="cat", + use_var=slots_vars, + ) dataset.set_filelist([filename1, filename2]) dataset.load_into_memory() fleet._opt_info = None @@ -1033,10 +1068,12 @@ def test_dataset_fleet2(self): Testcase for InMemoryDataset from create to run. """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "test_in_memory_dataset2_run2_a.txt") - filename2 = os.path.join(temp_dir.name, - "test_in_memory_dataset2_run2_b.txt") + filename1 = os.path.join( + temp_dir.name, "test_in_memory_dataset2_run2_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "test_in_memory_dataset2_run2_b.txt" + ) with open(filename1, "w") as f: data = "1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -1054,15 +1091,18 @@ def test_dataset_fleet2(self): startup_program = fluid.Program() scope = fluid.Scope() from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet + with fluid.program_guard(train_program, startup_program): slots = ["slot1_ff", "slot2_ff", "slot3_ff", "slot4_ff"] slots_vars = [] for slot in slots: - var = fluid.layers.data(\ - name=slot, shape=[1], dtype="float32", lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="float32", lod_level=1 + ) slots_vars.append(var) - fake_cost = \ - fluid.layers.elementwise_sub(slots_vars[0], slots_vars[-1]) + fake_cost = fluid.layers.elementwise_sub( + slots_vars[0], slots_vars[-1] + ) fake_cost = paddle.mean(fake_cost) with fluid.scope_guard(scope): place = fluid.CPUPlace() @@ -1073,17 +1113,15 @@ def test_dataset_fleet2(self): print("warning: no mpi4py") adam = fluid.optimizer.Adam(learning_rate=0.000005) try: - adam = fleet.distributed_optimizer(adam, - strategy={ - "fs_uri": - "fs_uri_xxx", - "fs_user": - "fs_user_xxx", - "fs_passwd": - "fs_passwd_xxx", - "fs_hadoop_bin": - "fs_hadoop_bin_xxx" - }) + adam = fleet.distributed_optimizer( + adam, + strategy={ + "fs_uri": "fs_uri_xxx", + "fs_user": "fs_user_xxx", + "fs_passwd": "fs_passwd_xxx", + "fs_hadoop_bin": "fs_hadoop_bin_xxx", + }, + ) adam.minimize([fake_cost], [scope]) except AttributeError as e: print("warning: no mpi") @@ -1091,10 +1129,12 @@ def test_dataset_fleet2(self): print("warning: no mpi4py") exe.run(startup_program) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, + thread_num=3, + pipe_command="cat", + use_var=slots_vars, + ) dataset.set_filelist([filename1, filename2]) dataset.load_into_memory() try: @@ -1132,7 +1172,7 @@ def test_dataset_fleet2(self): dataset.global_shuffle() except: print("warning: catch expected error") - #dataset.get_pv_data_size() + # dataset.get_pv_data_size() dataset.get_memory_data_size() dataset.get_shuffle_data_size() dataset = paddle.distributed.QueueDataset() @@ -1161,10 +1201,12 @@ def test_bosps_dataset_fleet2(self): Testcase for InMemoryDataset from create to run. """ temp_dir = tempfile.TemporaryDirectory() - filename1 = os.path.join(temp_dir.name, - "test_in_memory_dataset2_run2_a.txt") - filename2 = os.path.join(temp_dir.name, - "test_in_memory_dataset2_run2_b.txt") + filename1 = os.path.join( + temp_dir.name, "test_in_memory_dataset2_run2_a.txt" + ) + filename2 = os.path.join( + temp_dir.name, "test_in_memory_dataset2_run2_b.txt" + ) with open(filename1, "w") as f: data = "1 1 2 3 3 4 5 5 5 5 1 1\n" @@ -1182,15 +1224,18 @@ def test_bosps_dataset_fleet2(self): startup_program = fluid.Program() scope = fluid.Scope() from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet + with fluid.program_guard(train_program, startup_program): slots = ["slot1_ff", "slot2_ff", "slot3_ff", "slot4_ff"] slots_vars = [] for slot in slots: - var = fluid.layers.data(\ - name=slot, shape=[1], dtype="float32", lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="float32", lod_level=1 + ) slots_vars.append(var) - fake_cost = \ - fluid.layers.elementwise_sub(slots_vars[0], slots_vars[-1]) + fake_cost = fluid.layers.elementwise_sub( + slots_vars[0], slots_vars[-1] + ) fake_cost = paddle.mean(fake_cost) with fluid.scope_guard(scope): place = fluid.CPUPlace() @@ -1201,17 +1246,15 @@ def test_bosps_dataset_fleet2(self): print("warning: no mpi4py") adam = fluid.optimizer.Adam(learning_rate=0.000005) try: - adam = fleet.distributed_optimizer(adam, - strategy={ - "fs_uri": - "fs_uri_xxx", - "fs_user": - "fs_user_xxx", - "fs_passwd": - "fs_passwd_xxx", - "fs_hadoop_bin": - "fs_hadoop_bin_xxx" - }) + adam = fleet.distributed_optimizer( + adam, + strategy={ + "fs_uri": "fs_uri_xxx", + "fs_user": "fs_user_xxx", + "fs_passwd": "fs_passwd_xxx", + "fs_hadoop_bin": "fs_hadoop_bin_xxx", + }, + ) adam.minimize([fake_cost], [scope]) except AttributeError as e: print("warning: no mpi") @@ -1219,10 +1262,12 @@ def test_bosps_dataset_fleet2(self): print("warning: no mpi4py") exe.run(startup_program) dataset = paddle.distributed.fleet.BoxPSDataset() - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) + dataset.init( + batch_size=32, + thread_num=3, + pipe_command="cat", + use_var=slots_vars, + ) dataset.set_filelist([filename1, filename2]) dataset.load_into_memory() try: @@ -1232,14 +1277,16 @@ def test_bosps_dataset_fleet2(self): fleet._opt_info = None fleet._fleet_ptr = None dataset = paddle.distributed.fleet.BoxPSDataset() - dataset.init(rank_offset="", - pv_batch_size=1, - fs_name="", - fs_ugi="", - data_feed_type="MultiSlotInMemoryDataFeed", - parse_logkey=True, - merge_by_sid=True, - enable_pv_merge=True) + dataset.init( + rank_offset="", + pv_batch_size=1, + fs_name="", + fs_ugi="", + data_feed_type="MultiSlotInMemoryDataFeed", + parse_logkey=True, + merge_by_sid=True, + enable_pv_merge=True, + ) d = paddle.distributed.fleet.DatasetBase() try: dataset._set_feed_type("MultiSlotInMemoryDataFeed") @@ -1270,7 +1317,7 @@ def test_bosps_dataset_fleet2(self): dataset.global_shuffle() except: print("warning: catch expected error") - #dataset.get_pv_data_size() + # dataset.get_pv_data_size() dataset.get_memory_data_size() dataset.get_shuffle_data_size() temp_dir.cleanup() diff --git a/python/paddle/fluid/tests/unittests/test_dataset_consistency_inspection.py b/python/paddle/fluid/tests/unittests/test_dataset_consistency_inspection.py index 46ba155f01753..51463e7ff6287 100644 --- a/python/paddle/fluid/tests/unittests/test_dataset_consistency_inspection.py +++ b/python/paddle/fluid/tests/unittests/test_dataset_consistency_inspection.py @@ -23,46 +23,57 @@ import unittest import paddle.fluid.incubate.data_generator as dg -#paddle.enable_static() +# paddle.enable_static() # fluid.disable_dygraph() fluid.disable_dygraph() url_schema_len = 5 query_schema = [ - 'Q_query_basic', 'Q_query_phrase', 'Q_quq', 'Q_timelevel', - 'Q_context_title_basic1', 'Q_context_title_basic2', - 'Q_context_title_basic3', 'Q_context_title_basic4', - 'Q_context_title_basic5', 'Q_context_title_phrase1', - 'Q_context_title_phrase2', 'Q_context_title_phrase3', - 'Q_context_title_phrase4', 'Q_context_title_phrase5', 'Q_context_site1', - 'Q_context_site2', 'Q_context_site3', 'Q_context_site4', 'Q_context_site5' + 'Q_query_basic', + 'Q_query_phrase', + 'Q_quq', + 'Q_timelevel', + 'Q_context_title_basic1', + 'Q_context_title_basic2', + 'Q_context_title_basic3', + 'Q_context_title_basic4', + 'Q_context_title_basic5', + 'Q_context_title_phrase1', + 'Q_context_title_phrase2', + 'Q_context_title_phrase3', + 'Q_context_title_phrase4', + 'Q_context_title_phrase5', + 'Q_context_site1', + 'Q_context_site2', + 'Q_context_site3', + 'Q_context_site4', + 'Q_context_site5', ] class CTRDataset(dg.MultiSlotDataGenerator): - def __init__(self, mode): self.test = mode def generate_sample(self, line): - def reader(): ins = line.strip().split(';') label_pos_num = int(ins[1].split(' ')[0]) label_neg_num = int(ins[1].split(' ')[1]) - #query fea parse + # query fea parse bias = 2 query_len = 0 sparse_query_feature = [] for index in range(len(query_schema)): pos = index + bias sparse_query_feature.append( - [int(x) for x in ins[pos].split(' ')]) + [int(x) for x in ins[pos].split(' ')] + ) if index == 0: query_len = len(ins[pos].split(' ')) query_len = 1.0 / (1 + pow(2.7182818, 3 - 1.0 * query_len)) - #positive url fea parse + # positive url fea parse bias = 2 + len(query_schema) pos_url_feas = [] pos_click_feas = [] @@ -73,14 +84,24 @@ def reader(): for index in range(url_schema_len - 1): pos = bias + k * (url_schema_len) + index pos_url_fea.append([int(x) for x in ins[pos].split(' ')]) - #click info - if (ins[pos + 1] == ''): + # click info + if ins[pos + 1] == '': continue item = ins[pos + 1].split(' ') if len(item) != 17: continue - stat_fea = [[max(float(item[i]), 0.0)] for i in range(len(item)) \ - if not (i == 5 or i == 9 or i == 13 or i == 14 or i ==15 or i ==16)] + stat_fea = [ + [max(float(item[i]), 0.0)] + for i in range(len(item)) + if not ( + i == 5 + or i == 9 + or i == 13 + or i == 14 + or i == 15 + or i == 16 + ) + ] pos_url_feas.append(pos_url_fea) pos_click_feas.append(stat_fea) @@ -90,7 +111,7 @@ def reader(): pos_context_fea = [[query_serach], [query_len]] pos_context_feas.append(pos_context_fea) - #negative url fea parse + # negative url fea parse bias = 2 + len(query_schema) + label_pos_num * (url_schema_len) neg_url_feas = [] neg_click_feas = [] @@ -101,15 +122,25 @@ def reader(): for index in range(url_schema_len - 1): pos = bias + k * (url_schema_len) + index neg_url_fea.append([int(x) for x in ins[pos].split(' ')]) - if (ins[pos + 1] == ''): + if ins[pos + 1] == '': continue item = ins[pos + 1].split(' ') - #zdf_tmp + # zdf_tmp if len(item) != 17: continue - #print ins[pos + 1] - stat_fea = [[max(float(item[i]), 0.0)] for i in range(len(item)) \ - if not (i == 5 or i == 9 or i == 13 or i == 14 or i == 15 or i == 16)] + # print ins[pos + 1] + stat_fea = [ + [max(float(item[i]), 0.0)] + for i in range(len(item)) + if not ( + i == 5 + or i == 9 + or i == 13 + or i == 14 + or i == 15 + or i == 16 + ) + ] neg_click_feas.append(stat_fea) neg_url_feas.append(neg_url_fea) @@ -119,7 +150,7 @@ def reader(): neg_context_fea = [[query_serach], [query_len]] neg_context_feas.append(neg_context_fea) - #make train data + # make train data if self.test == 1: for p in range(len(pos_url_feas)): # feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] @@ -129,9 +160,17 @@ def reader(): pos_url_fea = pos_url_feas[p] pos_click_fea = pos_click_feas[p] pos_context_fea = pos_context_feas[p] - yield zip(feature_name, [[1]] + sparse_query_feature + - pos_url_fea + pos_click_fea + pos_context_fea + - pos_url_fea + pos_click_fea + pos_context_fea) + yield zip( + feature_name, + [[1]] + + sparse_query_feature + + pos_url_fea + + pos_click_fea + + pos_context_fea + + pos_url_fea + + pos_click_fea + + pos_context_fea, + ) for n in range(len(neg_url_feas)): feature_name = ["click"] for i in range(1, 54): @@ -139,18 +178,26 @@ def reader(): neg_url_fea = neg_url_feas[n] neg_click_fea = neg_click_feas[n] neg_context_fea = neg_context_feas[n] - yield zip(feature_name, [[0]] + sparse_query_feature + - neg_url_fea + neg_click_fea + neg_context_fea + - neg_url_fea + neg_click_fea + neg_context_fea) + yield zip( + feature_name, + [[0]] + + sparse_query_feature + + neg_url_fea + + neg_click_fea + + neg_context_fea + + neg_url_fea + + neg_click_fea + + neg_context_fea, + ) elif self.test == 0: for p in range(len(pos_url_feas)): - #feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] + # feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] feature_name = ["click"] for i in range(1, 54): feature_name.append(str(i)) - #print("#######") - #print(feature_name) - #print("#######") + # print("#######") + # print(feature_name) + # print("#######") pos_url_fea = pos_url_feas[p] pos_click_fea = pos_click_feas[p] pos_context_fea = pos_context_feas[p] @@ -160,21 +207,32 @@ def reader(): neg_url_fea = neg_url_feas[n] neg_click_fea = neg_click_feas[n] neg_context_fea = neg_context_feas[n] - #print("q:", query_feas) - #print("pos:", pos_url_fea) - #print("neg:", neg_url_fea) + # print("q:", query_feas) + # print("pos:", pos_url_fea) + # print("neg:", neg_url_fea) # yield zip(feature_name[:3], sparse_query_feature[:3]) - yield list(zip(feature_name, [[1]] + sparse_query_feature + pos_url_fea + pos_click_fea + pos_context_fea + \ - neg_url_fea + neg_click_fea + neg_context_fea)) + yield list( + zip( + feature_name, + [[1]] + + sparse_query_feature + + pos_url_fea + + pos_click_fea + + pos_context_fea + + neg_url_fea + + neg_click_fea + + neg_context_fea, + ) + ) elif self.test == 2: for p in range(len(pos_url_feas)): - #feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] + # feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] feature_name = ["click"] for i in range(1, 54): feature_name.append(str(i)) - #print("#######") - #print(feature_name) - #print("#######") + # print("#######") + # print(feature_name) + # print("#######") pos_url_fea = pos_url_feas[p] pos_click_fea = pos_click_feas[p] pos_context_fea = pos_context_feas[p] @@ -184,21 +242,32 @@ def reader(): neg_url_fea = neg_url_feas[n] neg_click_fea = neg_click_feas[n] neg_context_fea = neg_context_feas[n] - #print("q:", query_feas) - #print("pos:", pos_url_fea) - #print("neg:", neg_url_fea) + # print("q:", query_feas) + # print("pos:", pos_url_fea) + # print("neg:", neg_url_fea) # yield zip(feature_name[:3], sparse_query_feature[:3]) - yield list(zip(feature_name, [[1], [2]] + sparse_query_feature + pos_url_fea + pos_click_fea + pos_context_fea + \ - neg_url_fea + neg_click_fea + neg_context_fea)) + yield list( + zip( + feature_name, + [[1], [2]] + + sparse_query_feature + + pos_url_fea + + pos_click_fea + + pos_context_fea + + neg_url_fea + + neg_click_fea + + neg_context_fea, + ) + ) elif self.test == 3: for p in range(len(pos_url_feas)): - #feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] + # feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] feature_name = ["click"] for i in range(1, 54): feature_name.append(str(i)) - #print("#######") - #print(feature_name) - #print("#######") + # print("#######") + # print(feature_name) + # print("#######") pos_url_fea = pos_url_feas[p] pos_click_fea = pos_click_feas[p] pos_context_fea = pos_context_feas[p] @@ -208,21 +277,32 @@ def reader(): neg_url_fea = neg_url_feas[n] neg_click_fea = neg_click_feas[n] neg_context_fea = neg_context_feas[n] - #print("q:", query_feas) - #print("pos:", pos_url_fea) - #print("neg:", neg_url_fea) + # print("q:", query_feas) + # print("pos:", pos_url_fea) + # print("neg:", neg_url_fea) # yield zip(feature_name[:3], sparse_query_feature[:3]) - yield list(zip(feature_name, [[1], [2.0]] + sparse_query_feature + pos_url_fea + pos_click_fea + pos_context_fea + \ - neg_url_fea + neg_click_fea + neg_context_fea)) + yield list( + zip( + feature_name, + [[1], [2.0]] + + sparse_query_feature + + pos_url_fea + + pos_click_fea + + pos_context_fea + + neg_url_fea + + neg_click_fea + + neg_context_fea, + ) + ) elif self.test == 4: for p in range(len(pos_url_feas)): - #feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] + # feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] feature_name = ["click"] for i in range(1, 54): feature_name.append(str(i)) - #print("#######") - #print(feature_name) - #print("#######") + # print("#######") + # print(feature_name) + # print("#######") pos_url_fea = pos_url_feas[p] pos_click_fea = pos_click_feas[p] pos_context_fea = pos_context_feas[p] @@ -232,21 +312,32 @@ def reader(): neg_url_fea = neg_url_feas[n] neg_click_fea = neg_click_feas[n] neg_context_fea = neg_context_feas[n] - #print("q:", query_feas) - #print("pos:", pos_url_fea) - #print("neg:", neg_url_fea) + # print("q:", query_feas) + # print("pos:", pos_url_fea) + # print("neg:", neg_url_fea) # yield zip(feature_name[:3], sparse_query_feature[:3]) - yield list(zip(feature_name, [[], [2.0]] + sparse_query_feature + pos_url_fea + pos_click_fea + pos_context_fea + \ - neg_url_fea + neg_click_fea + neg_context_fea)) + yield list( + zip( + feature_name, + [[], [2.0]] + + sparse_query_feature + + pos_url_fea + + pos_click_fea + + pos_context_fea + + neg_url_fea + + neg_click_fea + + neg_context_fea, + ) + ) elif self.test == 5: for p in range(len(pos_url_feas)): - #feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] + # feature_name = ["click"] + query_schema + url_schema[:4] + click_info_schema[:11] + context_schema[:2] + url_schema[4:] + click_info_schema[11:] + context_schema[2:] feature_name = ["click"] for i in range(1, 54): feature_name.append(str(i)) - #print("#######") - #print(feature_name) - #print("#######") + # print("#######") + # print(feature_name) + # print("#######") pos_url_fea = pos_url_feas[p] pos_click_fea = pos_click_feas[p] pos_context_fea = pos_context_feas[p] @@ -256,18 +347,28 @@ def reader(): neg_url_fea = neg_url_feas[n] neg_click_fea = neg_click_feas[n] neg_context_fea = neg_context_feas[n] - #print("q:", query_feas) - #print("pos:", pos_url_fea) - #print("neg:", neg_url_fea) + # print("q:", query_feas) + # print("pos:", pos_url_fea) + # print("neg:", neg_url_fea) # yield zip(feature_name[:3], sparse_query_feature[:3]) - yield list(zip(feature_name, sparse_query_feature + pos_url_fea + pos_click_fea + pos_context_fea + \ - neg_url_fea + neg_click_fea + neg_context_fea)) + yield list( + zip( + feature_name, + sparse_query_feature + + pos_url_fea + + pos_click_fea + + pos_context_fea + + neg_url_fea + + neg_click_fea + + neg_context_fea, + ) + ) return reader class TestDataset(unittest.TestCase): - """ TestCases for Dataset. """ + """TestCases for Dataset.""" def setUp(self): pass @@ -292,74 +393,80 @@ def test_var_consistency_insepection(self): f.write(data) slot_data = [] - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="int64", - lod_level=0, - append_batch_size=False) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=0, + append_batch_size=False, + ) slot_data.append(label) # sprase_query_feat_names len_sparse_query = 19 for feat_name in range(1, len_sparse_query + 1): slot_data.append( - fluid.layers.data(name=str(feat_name), - shape=[1], - dtype='int64', - lod_level=1)) + fluid.layers.data( + name=str(feat_name), shape=[1], dtype='int64', lod_level=1 + ) + ) # sparse_url_feat_names for feat_name in range(len_sparse_query + 1, len_sparse_query + 5): slot_data.append( - fluid.layers.data(name=str(feat_name), - shape=[1], - dtype='int64', - lod_level=1)) + fluid.layers.data( + name=str(feat_name), shape=[1], dtype='int64', lod_level=1 + ) + ) # dense_feat_names for feat_name in range(len_sparse_query + 5, len_sparse_query + 16): slot_data.append( - fluid.layers.data(name=str(feat_name), - shape=[1], - dtype='float32')) + fluid.layers.data( + name=str(feat_name), shape=[1], dtype='float32' + ) + ) # context_feat_namess for feat_name in range(len_sparse_query + 16, len_sparse_query + 18): slot_data.append( - fluid.layers.data(name=str(feat_name), - shape=[1], - dtype='float32')) + fluid.layers.data( + name=str(feat_name), shape=[1], dtype='float32' + ) + ) # neg sparse_url_feat_names for feat_name in range(len_sparse_query + 18, len_sparse_query + 22): slot_data.append( - fluid.layers.data(name=str(feat_name), - shape=[1], - dtype='int64', - lod_level=1)) + fluid.layers.data( + name=str(feat_name), shape=[1], dtype='int64', lod_level=1 + ) + ) # neg dense_feat_names for feat_name in range(len_sparse_query + 22, len_sparse_query + 33): slot_data.append( - fluid.layers.data(name=str(feat_name), - shape=[1], - dtype='float32')) + fluid.layers.data( + name=str(feat_name), shape=[1], dtype='float32' + ) + ) # neg context_feat_namess for feat_name in range(len_sparse_query + 33, len_sparse_query + 35): slot_data.append( - fluid.layers.data(name=str(feat_name), - shape=[1], - dtype='float32')) + fluid.layers.data( + name=str(feat_name), shape=[1], dtype='float32' + ) + ) dataset = paddle.distributed.InMemoryDataset() print("========================================") generator_class = CTRDataset(mode=0) try: - dataset._check_use_var_with_data_generator(slot_data, - generator_class, - dump_a_path) + dataset._check_use_var_with_data_generator( + slot_data, generator_class, dump_a_path + ) print("case 1: check passed!") except Exception as e: print("warning: catch expected error") @@ -370,9 +477,9 @@ def test_var_consistency_insepection(self): print("========================================") generator_class = CTRDataset(mode=2) try: - dataset._check_use_var_with_data_generator(slot_data, - generator_class, - dump_a_path) + dataset._check_use_var_with_data_generator( + slot_data, generator_class, dump_a_path + ) except Exception as e: print("warning: case 2 catch expected error") print(e) @@ -382,9 +489,9 @@ def test_var_consistency_insepection(self): print("========================================") generator_class = CTRDataset(mode=3) try: - dataset._check_use_var_with_data_generator(slot_data, - generator_class, - dump_a_path) + dataset._check_use_var_with_data_generator( + slot_data, generator_class, dump_a_path + ) except Exception as e: print("warning: case 3 catch expected error") print(e) @@ -394,9 +501,9 @@ def test_var_consistency_insepection(self): print("========================================") generator_class = CTRDataset(mode=4) try: - dataset._check_use_var_with_data_generator(slot_data, - generator_class, - dump_a_path) + dataset._check_use_var_with_data_generator( + slot_data, generator_class, dump_a_path + ) except Exception as e: print("warning: case 4 catch expected error") print(e) @@ -406,9 +513,9 @@ def test_var_consistency_insepection(self): print("========================================") generator_class = CTRDataset(mode=5) try: - dataset._check_use_var_with_data_generator(slot_data, - generator_class, - dump_a_path) + dataset._check_use_var_with_data_generator( + slot_data, generator_class, dump_a_path + ) except Exception as e: print("warning: case 5 catch expected error") print(e) diff --git a/python/paddle/fluid/tests/unittests/test_dataset_dataloader.py b/python/paddle/fluid/tests/unittests/test_dataset_dataloader.py index ec61ee3a83812..5c7c97991d63a 100644 --- a/python/paddle/fluid/tests/unittests/test_dataset_dataloader.py +++ b/python/paddle/fluid/tests/unittests/test_dataset_dataloader.py @@ -44,9 +44,12 @@ def write_reader_data_to_file(filename, reader): with open(filename, 'w') as fid: for instance_list in reader(): for i, instance in enumerate(instance_list): - instance = np.reshape(instance, [ - instance.size, - ]) + instance = np.reshape( + instance, + [ + instance.size, + ], + ) fid.write(str(instance.size) + ' ') fid.write(' '.join(map(str, instance))) fid.write(' ') @@ -55,21 +58,20 @@ def write_reader_data_to_file(filename, reader): def fake_reader(batch_size=BATCH_SIZE, batch_num=BATCH_NUM): - def __reader__(): iteration = BATCH_SIZE * BATCH_NUM iteration = int(iteration + BATCH_SIZE / 2) for _ in range(iteration): image = np.random.random(size=IMAGE_SHAPE).astype('float32') - label = np.random.random_integers(size=LABEL_SHAPE, low=0, - high=9).astype('int64') + label = np.random.random_integers( + size=LABEL_SHAPE, low=0, high=9 + ).astype('int64') yield image, label return __reader__ class DatasetLoaderTestBase(unittest.TestCase): - def setUp(self): self.dataset_name = "QueueDataset" self.drop_last = False @@ -82,12 +84,12 @@ def build_network(self): main_prog = fluid.Program() startup_prog = fluid.Program() with fluid.program_guard(main_prog, startup_prog): - image = fluid.layers.data(name='image', - shape=IMAGE_SHAPE, - dtype='float32') - label = fluid.layers.data(name='label', - shape=LABEL_SHAPE, - dtype='int64') + image = fluid.layers.data( + name='image', shape=IMAGE_SHAPE, dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=LABEL_SHAPE, dtype='int64' + ) simple_fc_net_with_inputs(image, label) @@ -114,19 +116,23 @@ def check_batch_number(self, place, randomize_batch_num=False): filelist = [] if file_num > 1 and randomize_batch_num: random_delta_batch_size = np.random.random_integers( - low=-BATCH_NUM / 2, high=BATCH_NUM / 2, size=[file_num]) + low=-BATCH_NUM / 2, high=BATCH_NUM / 2, size=[file_num] + ) random_delta_batch_size[-1] = -int( - np.sum(random_delta_batch_size[0:-1])) + np.sum(random_delta_batch_size[0:-1]) + ) else: random_delta_batch_size = np.zeros(shape=[file_num]) for i in range(file_num): - filename = os.path.join(self.temp_dir.name, - 'dataset_test_{}.txt'.format(i)) + filename = os.path.join( + self.temp_dir.name, 'dataset_test_{}.txt'.format(i) + ) filelist.append(filename) write_reader_data_to_file( filename, - fake_reader(batch_num=BATCH_NUM + random_delta_batch_size[i])) + fake_reader(batch_num=BATCH_NUM + random_delta_batch_size[i]), + ) dataset.set_filelist(filelist) dataset._set_use_var(feeds) @@ -134,9 +140,9 @@ def check_batch_number(self, place, randomize_batch_num=False): if self.dataset_name == 'InMemoryDataset': dataset.load_into_memory() - dataloader = fluid.io.DataLoader.from_dataset(dataset=dataset, - places=places, - drop_last=self.drop_last) + dataloader = fluid.io.DataLoader.from_dataset( + dataset=dataset, places=places, drop_last=self.drop_last + ) prog = fluid.CompiledProgram(main_prog).with_data_parallel() exe = fluid.Executor(place) @@ -159,22 +165,29 @@ def check_batch_number(self, place, randomize_batch_num=False): batch_size = BATCH_SIZE self.assertEquals(image.shape()[1:], IMAGE_SHAPE) - self.assertTrue(image._place()._equals(places[idx]), - msg=get_place_string(image._place()) + - ' vs ' + get_place_string(places[idx])) + self.assertTrue( + image._place()._equals(places[idx]), + msg=get_place_string(image._place()) + + ' vs ' + + get_place_string(places[idx]), + ) if self.drop_last: self.assertEquals(image.shape()[0], BATCH_SIZE) else: - self.assertTrue(image.shape()[0] == BATCH_SIZE - or image.shape()[0] == BATCH_SIZE / 2) + self.assertTrue( + image.shape()[0] == BATCH_SIZE + or image.shape()[0] == BATCH_SIZE / 2 + ) self.assertEquals(label.shape()[1:], LABEL_SHAPE) self.assertTrue(label._place()._equals(places[idx])) if self.drop_last: self.assertEquals(label.shape()[0], BATCH_SIZE) else: - self.assertTrue(label.shape()[0] == BATCH_SIZE - or label.shape()[0] == BATCH_SIZE / 2) + self.assertTrue( + label.shape()[0] == BATCH_SIZE + or label.shape()[0] == BATCH_SIZE / 2 + ) self.assertEquals(image.shape()[0], label.shape()[0]) @@ -203,7 +216,6 @@ def test_batch_number_with_different_length_files(self): class QueueDatasetTestWithoutDropLast(DatasetLoaderTestBase): - def setUp(self): self.dataset_name = "QueueDataset" self.drop_last = True @@ -211,7 +223,6 @@ def setUp(self): class InMemoryDatasetTestWithoutDropLast(DatasetLoaderTestBase): - def setUp(self): self.dataset_name = "InMemoryDataset" self.drop_last = False @@ -219,7 +230,6 @@ def setUp(self): class InMemoryDatasetTestWithDropLast(DatasetLoaderTestBase): - def setUp(self): self.dataset_name = "InMemoryDataset" self.drop_last = True diff --git a/python/paddle/fluid/tests/unittests/test_dataset_download.py b/python/paddle/fluid/tests/unittests/test_dataset_download.py index 06f015edf9561..f1fba215b931f 100644 --- a/python/paddle/fluid/tests/unittests/test_dataset_download.py +++ b/python/paddle/fluid/tests/unittests/test_dataset_download.py @@ -18,7 +18,6 @@ class TestDataSetDownload(unittest.TestCase): - def setUp(self): flower_path = DATA_HOME + "/flowers/imagelabels.mat" diff --git a/python/paddle/fluid/tests/unittests/test_debugger.py b/python/paddle/fluid/tests/unittests/test_debugger.py index 7e2e40e4ed04e..23beb7f1a638b 100644 --- a/python/paddle/fluid/tests/unittests/test_debugger.py +++ b/python/paddle/fluid/tests/unittests/test_debugger.py @@ -19,42 +19,41 @@ class TestDebugger(unittest.TestCase): - def test_debug_str(self): p = Program() b = p.current_block() - #selected_rows - b.create_var(name='selected_rows', - dtype="float32", - shape=[5, 10], - type=core.VarDesc.VarType.SELECTED_ROWS) + # selected_rows + b.create_var( + name='selected_rows', + dtype="float32", + shape=[5, 10], + type=core.VarDesc.VarType.SELECTED_ROWS, + ) - #tensor array - b.create_var(name='tensor_array', - shape=[5, 10], - type=core.VarDesc.VarType.LOD_TENSOR_ARRAY) + # tensor array + b.create_var( + name='tensor_array', + shape=[5, 10], + type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, + ) - #operator - mul_x = b.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x") - mul_y = b.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = b.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - b.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) + # operator + mul_x = b.create_parameter( + dtype="float32", shape=[5, 10], lod_level=0, name="mul.x" + ) + mul_y = b.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = b.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + b.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) print(debugger.pprint_program_codes(p)) diff --git a/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py b/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py index 5025186349bdd..dae5fd7b7b787 100644 --- a/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py +++ b/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py @@ -18,8 +18,7 @@ class TestDecayedAdagradOp1(OpTest): - ''' Test DecayedAdagrad operator with explicit attributes - ''' + '''Test DecayedAdagrad operator with explicit attributes''' def setUp(self): self.op_type = "decayed_adagrad" @@ -35,7 +34,7 @@ def setUp(self): 'Param': param, 'Grad': grad, 'Moment': moment, - 'LearningRate': np.array([lr]).astype("float32") + 'LearningRate': np.array([lr]).astype("float32"), } self.attrs = {'decay': decay, 'epsilon': epsilon} @@ -50,8 +49,7 @@ def test_check_output(self): class TestDecayedAdagradOp2(OpTest): - ''' Test DecayedAdagrad operator with default attributes - ''' + '''Test DecayedAdagrad operator with default attributes''' def setUp(self): self.op_type = "decayed_adagrad" @@ -67,7 +65,7 @@ def setUp(self): 'Param': param, 'Grad': grad, 'Moment': moment, - 'LearningRate': np.array([lr]).astype("float32") + 'LearningRate': np.array([lr]).astype("float32"), } self.attrs = {'decay': decay, 'epsilon': epsilon} @@ -83,5 +81,6 @@ def test_check_output(self): if __name__ == "__main__": import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py index 9b5ed11d9b9bf..d7a5d3714317c 100644 --- a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py +++ b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py @@ -40,14 +40,16 @@ def simple_fc_net(places, use_legacy_py_reader, use_double_buffer): with fluid.unique_name.guard(): with fluid.program_guard(main_prog, startup_prog): - image = fluid.layers.data(name='image', - shape=[784], - dtype='float32') + image = fluid.layers.data( + name='image', shape=[784], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') - py_reader = fluid.io.PyReader(feed_list=[image, label], - capacity=4, - iterable=not use_legacy_py_reader, - use_double_buffer=use_double_buffer) + py_reader = fluid.io.PyReader( + feed_list=[image, label], + capacity=4, + iterable=not use_legacy_py_reader, + use_double_buffer=use_double_buffer, + ) hidden = image for hidden_size in [10, 20, 30]: hidden = fluid.layers.fc( @@ -55,13 +57,16 @@ def simple_fc_net(places, use_legacy_py_reader, use_double_buffer): size=hidden_size, act='tanh', bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0))) + initializer=fluid.initializer.Constant(value=1.0) + ), + ) - predict_label = fluid.layers.fc(hidden, - size=CLASS_NUM, - act='softmax') + predict_label = fluid.layers.fc( + hidden, size=CLASS_NUM, act='softmax' + ) loss = paddle.mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) @@ -69,28 +74,35 @@ def simple_fc_net(places, use_legacy_py_reader, use_double_buffer): class TestBase(unittest.TestCase): - - def run_main(self, use_legacy_py_reader, with_data_parallel, places, - use_double_buffer): + def run_main( + self, + use_legacy_py_reader, + with_data_parallel, + places, + use_double_buffer, + ): scope = fluid.Scope() with fluid.scope_guard(scope): startup_prog, main_prog, py_reader, loss = simple_fc_net( - places, use_legacy_py_reader, use_double_buffer) + places, use_legacy_py_reader, use_double_buffer + ) reader = paddle.batch(random_reader, batch_size=BATCH_SIZE) ps = places if use_double_buffer else fluid.cpu_places(len(places)) py_reader.decorate_sample_list_generator( - reader, places=ps if py_reader.iterable else None) + reader, places=ps if py_reader.iterable else None + ) exe = fluid.Executor(place=places[0]) exe.run(startup_prog) prog = fluid.CompiledProgram(main_prog) if with_data_parallel: - prog = prog.with_data_parallel(loss_name=loss.name, - places=places) + prog = prog.with_data_parallel( + loss_name=loss.name, places=places + ) step = 0 step_list = [] @@ -102,9 +114,11 @@ def run_main(self, use_legacy_py_reader, with_data_parallel, places, py_reader.start() while True: try: - L, = exe.run(program=prog, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 except fluid.core.EOFException: @@ -123,10 +137,12 @@ def run_main(self, use_legacy_py_reader, with_data_parallel, places, assert label.shape() == [BATCH_SIZE, 1] assert image._place()._equals(ps[i]) assert label._place()._equals(ps[i]) - L, = exe.run(program=prog, - feed=d, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + feed=d, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 step_list.append(step) @@ -134,7 +150,7 @@ def run_main(self, use_legacy_py_reader, with_data_parallel, places, ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } return ret @@ -163,11 +179,13 @@ def test_main(self): use_legacy_py_reader=use_legacy_py_reader, with_data_parallel=with_data_parallel, places=p, - use_double_buffer=use_double_buffer) + use_double_buffer=use_double_buffer, + ) results.append(ret) if not use_double_buffer: diff = np.max( - np.abs(results[0]['loss'] - results[1]['loss'])) + np.abs(results[0]['loss'] - results[1]['loss']) + ) self.assertLess(diff, 1e-3) diff --git a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py index 0d27198127683..107792a892de3 100644 --- a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py +++ b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py @@ -19,7 +19,6 @@ class TestClass(unittest.TestCase): - def setUp(self): self.use_double_buffer = True self.use_py_reader = True @@ -34,7 +33,8 @@ def fake_reader(): for _ in range(batch_size * batch_num): img = np.random.random(size=img_shape).astype('float32') label = np.random.random_integers( - low=0, high=9, size=label_shape).astype('int64') + low=0, high=9, size=label_shape + ).astype('int64') yield img, label reader = paddle.reader.cache(fake_reader) @@ -48,18 +48,20 @@ def fake_reader(): main_prog = fluid.Program() startup_prog = fluid.Program() with fluid.program_guard(main_prog, startup_prog): - img = fluid.layers.data(shape=img_shape, - dtype='float32', - name='image') - label = fluid.layers.data(shape=label_shape, - dtype='int64', - name='label') + img = fluid.layers.data( + shape=img_shape, dtype='float32', name='image' + ) + label = fluid.layers.data( + shape=label_shape, dtype='int64', name='label' + ) feeder = fluid.DataFeeder(feed_list=[img, label], place=p) use_double_buffer = self.use_double_buffer - if p._type() != fluid.CPUPlace()._type( - ) and not use_double_buffer: + if ( + p._type() != fluid.CPUPlace()._type() + and not use_double_buffer + ): use_double_buffer = True if self.use_py_reader: @@ -67,15 +69,17 @@ def fake_reader(): feed_list=[img, label], capacity=4, iterable=True, - use_double_buffer=use_double_buffer) - py_reader.decorate_sample_list_generator(batch_reader, - places=p) + use_double_buffer=use_double_buffer, + ) + py_reader.decorate_sample_list_generator( + batch_reader, places=p + ) else: py_reader = fluid.io.DataLoader.from_generator( feed_list=[img, label], capacity=4, iterable=True, - use_double_buffer=use_double_buffer + use_double_buffer=use_double_buffer, ).set_sample_list_generator(batch_reader, places=p) for break_beforehand in [True, False]: @@ -97,7 +101,8 @@ def fake_reader(): batch_id += 1 if break_beforehand and batch_id >= int( - batch_num / 2): + batch_num / 2 + ): break if break_beforehand: @@ -107,21 +112,18 @@ def fake_reader(): class TestClass2(TestClass): - def setUp(self): self.use_double_buffer = False self.use_py_reader = True class TestClass3(TestClass): - def setUp(self): self.use_double_buffer = True self.use_py_reader = False class TestClass4(TestClass): - def setUp(self): self.use_double_buffer = False self.use_py_reader = False diff --git a/python/paddle/fluid/tests/unittests/test_default_dtype.py b/python/paddle/fluid/tests/unittests/test_default_dtype.py index 70c23d62073fc..ca95c820a718e 100644 --- a/python/paddle/fluid/tests/unittests/test_default_dtype.py +++ b/python/paddle/fluid/tests/unittests/test_default_dtype.py @@ -18,7 +18,6 @@ class TestDefaultType(unittest.TestCase): - def check_default(self): self.assertEqual("float32", get_default_dtype()) @@ -45,7 +44,6 @@ def test_api(self): class TestRaiseError(unittest.TestCase): - def test_error(self): self.assertRaises(TypeError, set_default_dtype, "int32") self.assertRaises(TypeError, set_default_dtype, np.int32) diff --git a/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py b/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py index beae3124ced82..88686585853f5 100644 --- a/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py +++ b/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py @@ -12,12 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle.fluid.default_scope_funcs import enter_local_scope, find_var, get_cur_scope, leave_local_scope, scoped_function, var +from paddle.fluid.default_scope_funcs import ( + enter_local_scope, + find_var, + get_cur_scope, + leave_local_scope, + scoped_function, + var, +) import unittest class TestDefaultScopeFuncs(unittest.TestCase): - def test_cur_scope(self): self.assertIsNotNone(get_cur_scope()) @@ -33,7 +39,6 @@ def test_create_var_get_var(self): leave_local_scope() def test_var_get_int(self): - def __new_scope__(): i = var("var_i") self.assertFalse(i.is_int()) diff --git a/python/paddle/fluid/tests/unittests/test_deform_conv2d.py b/python/paddle/fluid/tests/unittests/test_deform_conv2d.py index 53eea9ac4ab27..80bf06fa6db39 100644 --- a/python/paddle/fluid/tests/unittests/test_deform_conv2d.py +++ b/python/paddle/fluid/tests/unittests/test_deform_conv2d.py @@ -40,45 +40,70 @@ def prepare(self): np.random.seed(1) paddle.seed(1) if isinstance(self.kernel_size, int): - filter_shape = (self.kernel_size, ) * 2 + filter_shape = (self.kernel_size,) * 2 else: filter_shape = tuple(self.kernel_size) self.filter_shape = filter_shape self.weight = np.random.uniform( - -1, 1, (self.out_channels, self.in_channels // self.groups) + - filter_shape).astype(self.dtype) + -1, + 1, + (self.out_channels, self.in_channels // self.groups) + filter_shape, + ).astype(self.dtype) if not self.no_bias: - self.bias = np.random.uniform(-1, 1, (self.out_channels, )).astype( - self.dtype) + self.bias = np.random.uniform(-1, 1, (self.out_channels,)).astype( + self.dtype + ) - def out_size(in_size, pad_size, dilation_size, kernel_size, - stride_size): - return (in_size + 2 * pad_size - - (dilation_size * (kernel_size - 1) + 1)) / stride_size + 1 + def out_size( + in_size, pad_size, dilation_size, kernel_size, stride_size + ): + return ( + in_size + 2 * pad_size - (dilation_size * (kernel_size - 1) + 1) + ) / stride_size + 1 out_h = int( - out_size(self.spatial_shape[0], self.padding[0], self.dilation[0], - self.kernel_size[0], self.stride[0])) + out_size( + self.spatial_shape[0], + self.padding[0], + self.dilation[0], + self.kernel_size[0], + self.stride[0], + ) + ) out_w = int( - out_size(self.spatial_shape[1], self.padding[1], self.dilation[1], - self.kernel_size[1], self.stride[1])) + out_size( + self.spatial_shape[1], + self.padding[1], + self.dilation[1], + self.kernel_size[1], + self.stride[1], + ) + ) out_shape = (out_h, out_w) - self.input_shape = (self.batch_size, - self.in_channels) + self.spatial_shape + self.input_shape = ( + self.batch_size, + self.in_channels, + ) + self.spatial_shape - self.offset_shape = (self.batch_size, self.deformable_groups * 2 * - filter_shape[0] * filter_shape[1]) + out_shape + self.offset_shape = ( + self.batch_size, + self.deformable_groups * 2 * filter_shape[0] * filter_shape[1], + ) + out_shape - self.mask_shape = (self.batch_size, self.deformable_groups * - filter_shape[0] * filter_shape[1]) + out_shape + self.mask_shape = ( + self.batch_size, + self.deformable_groups * filter_shape[0] * filter_shape[1], + ) + out_shape - self.input = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.input = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) - self.offset = np.random.uniform(-1, 1, - self.offset_shape).astype(self.dtype) + self.offset = np.random.uniform(-1, 1, self.offset_shape).astype( + self.dtype + ) self.mask = np.random.uniform(-1, 1, self.mask_shape).astype(self.dtype) @@ -87,16 +112,34 @@ def static_graph_case_dcn(self): start = paddle.static.Program() paddle.enable_static() with paddle.static.program_guard(main, start): - x = paddle.static.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) + x = paddle.static.data( + "input", (-1, self.in_channels, -1, -1), dtype=self.dtype + ) offset = paddle.static.data( - "offset", (-1, self.deformable_groups * 2 * - self.filter_shape[0] * self.filter_shape[1], -1, -1), - dtype=self.dtype) + "offset", + ( + -1, + self.deformable_groups + * 2 + * self.filter_shape[0] + * self.filter_shape[1], + -1, + -1, + ), + dtype=self.dtype, + ) mask = paddle.static.data( - "mask", (-1, self.deformable_groups * self.filter_shape[0] * - self.filter_shape[1], -1, -1), - dtype=self.dtype) + "mask", + ( + -1, + self.deformable_groups + * self.filter_shape[0] + * self.filter_shape[1], + -1, + -1, + ), + dtype=self.dtype, + ) y_v1 = paddle.fluid.layers.deformable_conv( input=x, @@ -112,7 +155,8 @@ def static_graph_case_dcn(self): im2col_step=1, param_attr=I.Assign(self.weight), bias_attr=False if self.no_bias else I.Assign(self.bias), - modulated=False) + modulated=False, + ) y_v2 = paddle.fluid.layers.deformable_conv( input=x, @@ -127,17 +171,20 @@ def static_graph_case_dcn(self): deformable_groups=self.deformable_groups, im2col_step=1, param_attr=I.Assign(self.weight), - bias_attr=False if self.no_bias else I.Assign(self.bias)) + bias_attr=False if self.no_bias else I.Assign(self.bias), + ) exe = paddle.static.Executor(self.place) exe.run(start) - out_v1, out_v2 = exe.run(main, - feed={ - "input": self.input, - "offset": self.offset, - "mask": self.mask - }, - fetch_list=[y_v1, y_v2]) + out_v1, out_v2 = exe.run( + main, + feed={ + "input": self.input, + "offset": self.offset, + "mask": self.mask, + }, + fetch_list=[y_v1, y_v2], + ) return out_v1, out_v2 def dygraph_case_dcn(self): @@ -158,7 +205,8 @@ def dygraph_case_dcn(self): deformable_groups=self.deformable_groups, groups=self.groups, weight_attr=I.Assign(self.weight), - bias_attr=False if self.no_bias else I.Assign(self.bias)) + bias_attr=False if self.no_bias else I.Assign(self.bias), + ) y_v1 = deform_conv2d(x, offset) y_v2 = deform_conv2d(x, offset, mask) @@ -208,45 +256,70 @@ def prepare(self): np.random.seed(1) paddle.seed(1) if isinstance(self.kernel_size, int): - filter_shape = (self.kernel_size, ) * 2 + filter_shape = (self.kernel_size,) * 2 else: filter_shape = tuple(self.kernel_size) self.filter_shape = filter_shape self.weight = np.random.uniform( - -1, 1, (self.out_channels, self.in_channels // self.groups) + - filter_shape).astype(self.dtype) + -1, + 1, + (self.out_channels, self.in_channels // self.groups) + filter_shape, + ).astype(self.dtype) if not self.no_bias: - self.bias = np.random.uniform(-1, 1, (self.out_channels, )).astype( - self.dtype) + self.bias = np.random.uniform(-1, 1, (self.out_channels,)).astype( + self.dtype + ) - def out_size(in_size, pad_size, dilation_size, kernel_size, - stride_size): - return (in_size + 2 * pad_size - - (dilation_size * (kernel_size - 1) + 1)) / stride_size + 1 + def out_size( + in_size, pad_size, dilation_size, kernel_size, stride_size + ): + return ( + in_size + 2 * pad_size - (dilation_size * (kernel_size - 1) + 1) + ) / stride_size + 1 out_h = int( - out_size(self.spatial_shape[0], self.padding[0], self.dilation[0], - self.kernel_size[0], self.stride[0])) + out_size( + self.spatial_shape[0], + self.padding[0], + self.dilation[0], + self.kernel_size[0], + self.stride[0], + ) + ) out_w = int( - out_size(self.spatial_shape[1], self.padding[1], self.dilation[1], - self.kernel_size[1], self.stride[1])) + out_size( + self.spatial_shape[1], + self.padding[1], + self.dilation[1], + self.kernel_size[1], + self.stride[1], + ) + ) out_shape = (out_h, out_w) - self.input_shape = (self.batch_size, - self.in_channels) + self.spatial_shape + self.input_shape = ( + self.batch_size, + self.in_channels, + ) + self.spatial_shape - self.offset_shape = (self.batch_size, self.deformable_groups * 2 * - filter_shape[0] * filter_shape[1]) + out_shape + self.offset_shape = ( + self.batch_size, + self.deformable_groups * 2 * filter_shape[0] * filter_shape[1], + ) + out_shape - self.mask_shape = (self.batch_size, self.deformable_groups * - filter_shape[0] * filter_shape[1]) + out_shape + self.mask_shape = ( + self.batch_size, + self.deformable_groups * filter_shape[0] * filter_shape[1], + ) + out_shape - self.input = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.input = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) - self.offset = np.random.uniform(-1, 1, - self.offset_shape).astype(self.dtype) + self.offset = np.random.uniform(-1, 1, self.offset_shape).astype( + self.dtype + ) self.mask = np.random.uniform(-1, 1, self.mask_shape).astype(self.dtype) @@ -255,16 +328,34 @@ def static_graph_case_dcn(self): start = paddle.static.Program() paddle.enable_static() with paddle.static.program_guard(main, start): - x = paddle.static.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) + x = paddle.static.data( + "input", (-1, self.in_channels, -1, -1), dtype=self.dtype + ) offset = paddle.static.data( - "offset", (-1, self.deformable_groups * 2 * - self.filter_shape[0] * self.filter_shape[1], -1, -1), - dtype=self.dtype) + "offset", + ( + -1, + self.deformable_groups + * 2 + * self.filter_shape[0] + * self.filter_shape[1], + -1, + -1, + ), + dtype=self.dtype, + ) mask = paddle.static.data( - "mask", (-1, self.deformable_groups * self.filter_shape[0] * - self.filter_shape[1], -1, -1), - dtype=self.dtype) + "mask", + ( + -1, + self.deformable_groups + * self.filter_shape[0] + * self.filter_shape[1], + -1, + -1, + ), + dtype=self.dtype, + ) y_v1 = paddle.fluid.layers.deformable_conv( input=x, @@ -280,7 +371,8 @@ def static_graph_case_dcn(self): im2col_step=1, param_attr=I.Assign(self.weight), bias_attr=False if self.no_bias else I.Assign(self.bias), - modulated=False) + modulated=False, + ) y_v2 = paddle.fluid.layers.deformable_conv( input=x, @@ -295,17 +387,20 @@ def static_graph_case_dcn(self): deformable_groups=self.deformable_groups, im2col_step=1, param_attr=I.Assign(self.weight), - bias_attr=False if self.no_bias else I.Assign(self.bias)) + bias_attr=False if self.no_bias else I.Assign(self.bias), + ) exe = paddle.static.Executor(self.place) exe.run(start) - out_v1, out_v2 = exe.run(main, - feed={ - "input": self.input, - "offset": self.offset, - "mask": self.mask - }, - fetch_list=[y_v1, y_v2]) + out_v1, out_v2 = exe.run( + main, + feed={ + "input": self.input, + "offset": self.offset, + "mask": self.mask, + }, + fetch_list=[y_v1, y_v2], + ) return out_v1, out_v2 def dygraph_case_dcn(self): @@ -351,20 +446,38 @@ def new_api_static_graph_case_dcn(self): start = paddle.static.Program() paddle.enable_static() with paddle.static.program_guard(main, start): - x = paddle.static.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) + x = paddle.static.data( + "input", (-1, self.in_channels, -1, -1), dtype=self.dtype + ) offset = paddle.static.data( - "offset", (-1, self.deformable_groups * 2 * - self.filter_shape[0] * self.filter_shape[1], -1, -1), - dtype=self.dtype) + "offset", + ( + -1, + self.deformable_groups + * 2 + * self.filter_shape[0] + * self.filter_shape[1], + -1, + -1, + ), + dtype=self.dtype, + ) mask = paddle.static.data( - "mask", (-1, self.deformable_groups * self.filter_shape[0] * - self.filter_shape[1], -1, -1), - dtype=self.dtype) + "mask", + ( + -1, + self.deformable_groups + * self.filter_shape[0] + * self.filter_shape[1], + -1, + -1, + ), + dtype=self.dtype, + ) - weight = paddle.static.data("weight", - list(self.weight.shape), - dtype=self.dtype) + weight = paddle.static.data( + "weight", list(self.weight.shape), dtype=self.dtype + ) if not self.no_bias: bias = paddle.static.data("bias", [-1], dtype=self.dtype) @@ -400,7 +513,7 @@ def new_api_static_graph_case_dcn(self): "input": self.input, "offset": self.offset, "mask": self.mask, - "weight": self.weight + "weight": self.weight, } if not self.no_bias: feed_dict["bias"] = self.bias @@ -412,8 +525,10 @@ def _test_identity(self): self.prepare() static_dcn_v1, static_dcn_v2 = self.static_graph_case_dcn() dy_dcn_v1, dy_dcn_v2 = self.dygraph_case_dcn() - new_static_dcn_v1, new_static_dcn_v2 = self.new_api_static_graph_case_dcn( - ) + ( + new_static_dcn_v1, + new_static_dcn_v2, + ) = self.new_api_static_graph_case_dcn() np.testing.assert_array_almost_equal(static_dcn_v1, dy_dcn_v1) np.testing.assert_array_almost_equal(static_dcn_v2, dy_dcn_v2) np.testing.assert_array_almost_equal(static_dcn_v1, new_static_dcn_v1) @@ -434,7 +549,6 @@ def test_identity_with_eager_guard(self): # testcases for DeformConv2D class TestDeformConv2DWithPadding(TestDeformConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -448,7 +562,6 @@ def setUp(self): class TestDeformConv2DWithBias(TestDeformConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -462,7 +575,6 @@ def setUp(self): class TestDeformConv2DWithAsynPadding(TestDeformConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -476,7 +588,6 @@ def setUp(self): class TestDeformConv2DWithDilation(TestDeformConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -490,7 +601,6 @@ def setUp(self): class TestDeformConv2DWithStride(TestDeformConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -504,7 +614,6 @@ def setUp(self): class TestDeformConv2DWithDeformable_Groups(TestDeformConv2D): - def setUp(self): self.in_channels = 5 self.out_channels = 5 @@ -518,7 +627,6 @@ def setUp(self): class TestDeformConv2DWithGroups(TestDeformConv2D): - def setUp(self): self.in_channels = 5 self.out_channels = 5 @@ -533,7 +641,6 @@ def setUp(self): # testcases for deform_conv2d class TestDeformConv2DFunctionalWithPadding(TestDeformConv2DFunctional): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -547,7 +654,6 @@ def setUp(self): class TestDeformConv2DFunctionalWithBias(TestDeformConv2DFunctional): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -561,7 +667,6 @@ def setUp(self): class TestDeformConv2DFunctionalWithAsynPadding(TestDeformConv2DFunctional): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -575,7 +680,6 @@ def setUp(self): class TestDeformConv2DFunctionalWithDilation(TestDeformConv2DFunctional): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -589,7 +693,6 @@ def setUp(self): class TestDeformConv2DFunctionalWithStride(TestDeformConv2DFunctional): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -602,9 +705,9 @@ def setUp(self): self.no_bias = False -class TestDeformConv2DFunctionalWithDeformable_Groups(TestDeformConv2DFunctional - ): - +class TestDeformConv2DFunctionalWithDeformable_Groups( + TestDeformConv2DFunctional +): def setUp(self): self.in_channels = 5 self.out_channels = 5 @@ -618,7 +721,6 @@ def setUp(self): class TestDeformConv2DFunctionalWithGroups(TestDeformConv2DFunctional): - def setUp(self): self.in_channels = 5 self.out_channels = 5 diff --git a/python/paddle/fluid/tests/unittests/test_deformable_conv_op.py b/python/paddle/fluid/tests/unittests/test_deformable_conv_op.py index 6c0d7360e58db..f89ef98e95056 100644 --- a/python/paddle/fluid/tests/unittests/test_deformable_conv_op.py +++ b/python/paddle/fluid/tests/unittests/test_deformable_conv_op.py @@ -61,8 +61,11 @@ def dconv_im2col_gemm(input, offset, mask, filter, group, conv_param): assert f_c * group == in_c assert np.mod(out_c, group) == 0 - stride, pad, dilation = conv_param['stride'], conv_param['pad'],\ - conv_param['dilation'] + stride, pad, dilation = ( + conv_param['stride'], + conv_param['pad'], + conv_param['dilation'], + ) out_h = 1 + (in_h + 2 * pad[0] - (dilation[0] * (f_h - 1) + 1)) // stride[0] out_w = 1 + (in_w + 2 * pad[1] - (dilation[1] * (f_w - 1) + 1)) // stride[1] assert out_h == in_h @@ -75,31 +78,47 @@ def dconv_im2col_gemm(input, offset, mask, filter, group, conv_param): for w in range(out_w): for kh in range(f_h): for kw in range(f_w): - offset_h_table = \ - offset[n, ::2, h, w].reshape(f_h, f_w) - offset_w_table = \ - offset[n, 1::2, h, w].reshape(f_h, f_w) - mask_table = \ - mask[n, :, h, w].reshape(f_h, f_w) + offset_h_table = offset[n, ::2, h, w].reshape( + f_h, f_w + ) + offset_w_table = offset[n, 1::2, h, w].reshape( + f_h, f_w + ) + mask_table = mask[n, :, h, w].reshape(f_h, f_w) offset_h = offset_h_table[kh, kw] offset_w = offset_w_table[kh, kw] val = 0 - im_h = h * stride[0] + kh * dilation[0] \ - + offset_h - pad[0] - im_w = w * stride[0] + kw * dilation[0] \ - + offset_w - pad[1] - if im_h > -1 and im_w > -1 and \ - im_h < in_h and im_w < in_h: - val = dmc_bilinear(input[n, c], in_h, in_w, - im_h, im_w) + im_h = ( + h * stride[0] + + kh * dilation[0] + + offset_h + - pad[0] + ) + im_w = ( + w * stride[0] + + kw * dilation[0] + + offset_w + - pad[1] + ) + if ( + im_h > -1 + and im_w > -1 + and im_h < in_h + and im_w < in_h + ): + val = dmc_bilinear( + input[n, c], in_h, in_w, im_h, im_w + ) val_out = val * mask_table[kh, kw] - col_buffer[n, c * f_h * f_w + kh * f_w + kw, - h * in_w + w] = val_out + col_buffer[ + n, c * f_h * f_w + kh * f_w + kw, h * in_w + w + ] = val_out out = np.zeros((in_n, group, int(out_c // group), out_h * out_w)) weight = filter.reshape(group, int(out_c // group), f_c * f_h * f_w) col_buffer = col_buffer.reshape( - (in_n, group, int(in_c // group * f_h * f_w), in_h * in_w)) + (in_n, group, int(in_c // group * f_h * f_w), in_h * in_w) + ) for n in range(in_n): for g in range(group): out[n, g] = np.matmul(weight[g], col_buffer[n, g]) @@ -107,23 +126,33 @@ def dconv_im2col_gemm(input, offset, mask, filter, group, conv_param): return out -def deform_conv2d_wrapper(x, - offset, - weight, - mask=None, - stride=1, - padding=0, - dilation=1, - deformable_groups=1, - groups=1, - im2col_step=1): - return paddle.vision.ops.deform_conv2d(x, offset, weight, None, stride, - padding, dilation, deformable_groups, - groups, mask) +def deform_conv2d_wrapper( + x, + offset, + weight, + mask=None, + stride=1, + padding=0, + dilation=1, + deformable_groups=1, + groups=1, + im2col_step=1, +): + return paddle.vision.ops.deform_conv2d( + x, + offset, + weight, + None, + stride, + padding, + dilation, + deformable_groups, + groups, + mask, + ) class TestModulatedDeformableConvOp(OpTest): - def setUp(self): self.python_api = deform_conv2d_wrapper self.op_type = "deformable_conv" @@ -135,7 +164,7 @@ def setUp(self): conv_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) @@ -143,15 +172,16 @@ def setUp(self): mask = 10 * np.random.random(self.mask_size).astype(self.dtype) filter = np.random.random(self.filter_size).astype(self.dtype) - output = dconv_im2col_gemm(input, offset, mask, filter, self.groups, - conv_param) + output = dconv_im2col_gemm( + input, offset, mask, filter, self.groups, conv_param + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), 'Offset': OpTest.np_dtype_to_fluid_dtype(offset), 'Mask': OpTest.np_dtype_to_fluid_dtype(mask), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -167,10 +197,12 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad({'Input', 'Offset', 'Mask', 'Filter'}, - 'Output', - max_relative_error=0.05, - check_eager=True) + self.check_grad( + {'Input', 'Offset', 'Mask', 'Filter'}, + 'Output', + max_relative_error=0.05, + check_eager=True, + ) def init_test_case(self): self.pad = [1, 1] @@ -182,15 +214,26 @@ def init_test_case(self): self.filter_size = [4, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): @@ -204,7 +247,6 @@ def init_type(self): class TestWithStride(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [3, 3] self.stride = [2, 2] @@ -214,20 +256,30 @@ def init_test_case(self): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] class TestWithDilation(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [1, 1] @@ -237,15 +289,26 @@ def init_test_case(self): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): @@ -253,7 +316,6 @@ def init_dilation(self): class TestWith3x3(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -263,26 +325,35 @@ def init_test_case(self): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] class TestWithGroup(TestModulatedDeformableConvOp): - def init_group(self): self.groups = 2 class TestWithDouble(TestModulatedDeformableConvOp): - def init_type(self): self.dtype = np.float64 @@ -296,74 +367,77 @@ def init_test_case(self): self.filter_size = [4, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups * self.filter_size[2] * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] class TestModulatedDeformableConvInvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_input(): paddle.enable_static() input = [1, 3, 32, 32] - offset = fluid.data(name='offset', - shape=[None, 3, 32, 32], - dtype='float32') - mask = fluid.data(name='mask', - shape=[None, 3, 32, 32], - dtype='float32') - loss = fluid.layers.deformable_conv(input, - offset, - mask, - num_filters=4, - filter_size=1) + offset = fluid.data( + name='offset', shape=[None, 3, 32, 32], dtype='float32' + ) + mask = fluid.data( + name='mask', shape=[None, 3, 32, 32], dtype='float32' + ) + loss = fluid.layers.deformable_conv( + input, offset, mask, num_filters=4, filter_size=1 + ) self.assertRaises(TypeError, test_invalid_input) def test_invalid_offset(): paddle.enable_static() - input = fluid.data(name='input', - shape=[None, 3, 32, 32], - dtype='int32') - offset = fluid.data(name='offset', - shape=[None, 3, 32, 32], - dtype='float32') - mask = fluid.data(name='mask', - shape=[None, 3, 32, 32], - dtype='float32') - loss = fluid.layers.deformable_conv(input, - offset, - mask, - num_filters=4, - filter_size=1) + input = fluid.data( + name='input', shape=[None, 3, 32, 32], dtype='int32' + ) + offset = fluid.data( + name='offset', shape=[None, 3, 32, 32], dtype='float32' + ) + mask = fluid.data( + name='mask', shape=[None, 3, 32, 32], dtype='float32' + ) + loss = fluid.layers.deformable_conv( + input, offset, mask, num_filters=4, filter_size=1 + ) self.assertRaises(TypeError, test_invalid_offset) def test_invalid_filter(): paddle.enable_static() - input = fluid.data(name='input_filter', - shape=[None, 3, 32, 32], - dtype='float32') - offset = fluid.data(name='offset_filter', - shape=[None, 3, 32, 32], - dtype='float32') - mask = fluid.data(name='mask_filter', - shape=[None, 3, 32, 32], - dtype='float32') - loss = fluid.layers.deformable_conv(input, - offset, - mask, - num_filters=4, - filter_size=0) + input = fluid.data( + name='input_filter', shape=[None, 3, 32, 32], dtype='float32' + ) + offset = fluid.data( + name='offset_filter', shape=[None, 3, 32, 32], dtype='float32' + ) + mask = fluid.data( + name='mask_filter', shape=[None, 3, 32, 32], dtype='float32' + ) + loss = fluid.layers.deformable_conv( + input, offset, mask, num_filters=4, filter_size=0 + ) self.assertRaises(ValueError, test_invalid_filter) @@ -373,45 +447,39 @@ def test_error_with_eager_guard(self): class TestDeformConv2DAPI(unittest.TestCase): - def test_api(self): - def test_deform_conv2d_v1(): paddle.enable_static() - input = paddle.static.data(name='input_v1', - shape=[None, 3, 32, 32], - dtype='float32') - offset = paddle.static.data(name='offset_v1', - shape=[None, 4, 32, 32], - dtype='float32') - out = paddle.static.nn.deform_conv2d(input, - offset, - None, - num_filters=4, - filter_size=1) - - assert (out.shape == (-1, 4, 32, 32)) + input = paddle.static.data( + name='input_v1', shape=[None, 3, 32, 32], dtype='float32' + ) + offset = paddle.static.data( + name='offset_v1', shape=[None, 4, 32, 32], dtype='float32' + ) + out = paddle.static.nn.deform_conv2d( + input, offset, None, num_filters=4, filter_size=1 + ) + + assert out.shape == (-1, 4, 32, 32) test_deform_conv2d_v1() def test_deform_conv2d_v2(): paddle.enable_static() - input = paddle.static.data(name='input_v2', - shape=[None, 3, 32, 32], - dtype='float32') - offset = paddle.static.data(name='offset_v2', - shape=[None, 4, 32, 32], - dtype='float32') - mask = paddle.static.data(name='mask_v2', - shape=[None, 2, 32, 32], - dtype='float32') - out = paddle.static.nn.deform_conv2d(input, - offset, - mask, - num_filters=4, - filter_size=1) - - assert (out.shape == (-1, 4, 32, 32)) + input = paddle.static.data( + name='input_v2', shape=[None, 3, 32, 32], dtype='float32' + ) + offset = paddle.static.data( + name='offset_v2', shape=[None, 4, 32, 32], dtype='float32' + ) + mask = paddle.static.data( + name='mask_v2', shape=[None, 2, 32, 32], dtype='float32' + ) + out = paddle.static.nn.deform_conv2d( + input, offset, mask, num_filters=4, filter_size=1 + ) + + assert out.shape == (-1, 4, 32, 32) test_deform_conv2d_v2() diff --git a/python/paddle/fluid/tests/unittests/test_deformable_conv_v1_op.py b/python/paddle/fluid/tests/unittests/test_deformable_conv_v1_op.py index eacf6dba27312..19a81e21a6a11 100644 --- a/python/paddle/fluid/tests/unittests/test_deformable_conv_v1_op.py +++ b/python/paddle/fluid/tests/unittests/test_deformable_conv_v1_op.py @@ -58,8 +58,11 @@ def dconv_im2col_gemm(input, offset, filter, group, conv_param): assert f_c * group == in_c assert np.mod(out_c, group) == 0 - stride, pad, dilation = conv_param['stride'], conv_param['pad'],\ - conv_param['dilation'] + stride, pad, dilation = ( + conv_param['stride'], + conv_param['pad'], + conv_param['dilation'], + ) out_h = 1 + (in_h + 2 * pad[0] - (dilation[0] * (f_h - 1) + 1)) // stride[0] out_w = 1 + (in_w + 2 * pad[1] - (dilation[1] * (f_w - 1) + 1)) // stride[1] assert out_h == in_h @@ -72,30 +75,47 @@ def dconv_im2col_gemm(input, offset, filter, group, conv_param): for w in range(out_w): for kh in range(f_h): for kw in range(f_w): - offset_h_table = \ - offset[n, ::2, h, w].reshape(f_h, f_w) - offset_w_table = \ - offset[n, 1::2, h, w].reshape(f_h, f_w) + offset_h_table = offset[n, ::2, h, w].reshape( + f_h, f_w + ) + offset_w_table = offset[n, 1::2, h, w].reshape( + f_h, f_w + ) offset_h = offset_h_table[kh, kw] offset_w = offset_w_table[kh, kw] val = 0 - im_h = h * stride[0] + kh * dilation[0] \ - + offset_h - pad[0] - im_w = w * stride[0] + kw * dilation[0] \ - + offset_w - pad[1] - if im_h > -1 and im_w > -1 and \ - im_h < in_h and im_w < in_h: - val = dmc_bilinear(input[n, c], in_h, in_w, - im_h, im_w) + im_h = ( + h * stride[0] + + kh * dilation[0] + + offset_h + - pad[0] + ) + im_w = ( + w * stride[0] + + kw * dilation[0] + + offset_w + - pad[1] + ) + if ( + im_h > -1 + and im_w > -1 + and im_h < in_h + and im_w < in_h + ): + val = dmc_bilinear( + input[n, c], in_h, in_w, im_h, im_w + ) val_out = val - col_buffer[n, c * f_h * f_w + kh * f_w + kw, - h * in_w + w] = val_out + col_buffer[ + n, c * f_h * f_w + kh * f_w + kw, h * in_w + w + ] = val_out out = np.zeros((in_n, group, int(out_c // group), out_h * out_w)) weight = filter.reshape(group, int(out_c // group), f_c * f_h * f_w) col_buffer = col_buffer.reshape( - (in_n, group, int(in_c // group * f_h * f_w), in_h * in_w)) + (in_n, group, int(in_c // group * f_h * f_w), in_h * in_w) + ) for n in range(in_n): for g in range(group): out[n, g] = np.matmul(weight[g], col_buffer[n, g]) @@ -103,23 +123,33 @@ def dconv_im2col_gemm(input, offset, filter, group, conv_param): return out -def deform_conv2d_wrapper(x, - offset, - weight, - mask=None, - stride=1, - padding=0, - dilation=1, - deformable_groups=1, - groups=1, - im2col_step=1): - return paddle.vision.ops.deform_conv2d(x, offset, weight, None, stride, - padding, dilation, deformable_groups, - groups, mask) +def deform_conv2d_wrapper( + x, + offset, + weight, + mask=None, + stride=1, + padding=0, + dilation=1, + deformable_groups=1, + groups=1, + im2col_step=1, +): + return paddle.vision.ops.deform_conv2d( + x, + offset, + weight, + None, + stride, + padding, + dilation, + deformable_groups, + groups, + mask, + ) class TestModulatedDeformableConvOp(OpTest): - def setUp(self): self.python_api = deform_conv2d_wrapper self.op_type = "deformable_conv_v1" @@ -131,20 +161,21 @@ def setUp(self): conv_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) offset = 10 * np.random.random(self.offset_size).astype(self.dtype) filter = np.random.random(self.filter_size).astype(self.dtype) - output = dconv_im2col_gemm(input, offset, filter, self.groups, - conv_param) + output = dconv_im2col_gemm( + input, offset, filter, self.groups, conv_param + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), 'Offset': OpTest.np_dtype_to_fluid_dtype(offset), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -160,17 +191,21 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['Input', 'Offset', 'Filter'], - 'Output', - max_relative_error=0.05, - check_eager=True) + self.check_grad( + ['Input', 'Offset', 'Filter'], + 'Output', + max_relative_error=0.05, + check_eager=True, + ) def test_check_grad_no_filter(self): - self.check_grad(['Input', 'Offset'], - 'Output', - max_relative_error=0.1, - no_grad_set=set(['Filter']), - check_eager=True) + self.check_grad( + ['Input', 'Offset'], + 'Output', + max_relative_error=0.1, + no_grad_set=set(['Filter']), + check_eager=True, + ) def init_test_case(self): self.pad = [1, 1] @@ -182,10 +217,17 @@ def init_test_case(self): self.filter_size = [4, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): @@ -199,7 +241,6 @@ def init_type(self): class TestWithStride(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [3, 3] self.stride = [2, 2] @@ -209,15 +250,21 @@ def init_test_case(self): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] class TestWithDilation(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [1, 1] @@ -227,10 +274,17 @@ def init_test_case(self): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): @@ -238,7 +292,6 @@ def init_dilation(self): class TestWith1x1(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -248,15 +301,21 @@ def init_test_case(self): self.filter_size = [40, f_c, 1, 1] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] class TestWithGroup(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -267,10 +326,17 @@ def init_test_case(self): self.filter_size = [4, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] def init_group(self): @@ -278,42 +344,43 @@ def init_group(self): class TestWithDouble(TestModulatedDeformableConvOp): - def init_type(self): self.dtype = np.float64 class TestModulatedDeformableConvV1InvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_input(): input = [1, 3, 32, 32] - offset = fluid.data(name='offset', - shape=[None, 3, 32, 32], - dtype='float32') - loss = fluid.layers.deformable_conv(input, - offset, - mask=None, - num_filters=4, - filter_size=1, - modulated=False) + offset = fluid.data( + name='offset', shape=[None, 3, 32, 32], dtype='float32' + ) + loss = fluid.layers.deformable_conv( + input, + offset, + mask=None, + num_filters=4, + filter_size=1, + modulated=False, + ) self.assertRaises(TypeError, test_invalid_input) def test_invalid_offset(): - input = fluid.data(name='input', - shape=[None, 3, 32, 32], - dtype='int32') - offset = fluid.data(name='offset', - shape=[None, 3, 32, 32], - dtype='float32') - loss = fluid.layers.deformable_conv(input, - offset, - mask=None, - num_filters=4, - filter_size=1, - modulated=False) + input = fluid.data( + name='input', shape=[None, 3, 32, 32], dtype='int32' + ) + offset = fluid.data( + name='offset', shape=[None, 3, 32, 32], dtype='float32' + ) + loss = fluid.layers.deformable_conv( + input, + offset, + mask=None, + num_filters=4, + filter_size=1, + modulated=False, + ) self.assertRaises(TypeError, test_invalid_offset) diff --git a/python/paddle/fluid/tests/unittests/test_deformable_psroi_pooling.py b/python/paddle/fluid/tests/unittests/test_deformable_psroi_pooling.py index 4bf8a99137b41..8dbe2dfa32252 100644 --- a/python/paddle/fluid/tests/unittests/test_deformable_psroi_pooling.py +++ b/python/paddle/fluid/tests/unittests/test_deformable_psroi_pooling.py @@ -24,9 +24,17 @@ def set_input(input, rois, trans): return inputs -def set_attrs(no_trans, spatial_scale, output_channels, group_size, - pooled_height, pooled_width, part_size, sample_per_part, - trans_std): +def set_attrs( + no_trans, + spatial_scale, + output_channels, + group_size, + pooled_height, + pooled_width, + part_size, + sample_per_part, + trans_std, +): attrs = { 'no_trans': no_trans, 'spatial_scale': spatial_scale, @@ -36,7 +44,7 @@ def set_attrs(no_trans, spatial_scale, output_channels, group_size, 'pooled_width': pooled_width, 'part_size': part_size, 'sample_per_part': sample_per_part, - 'trans_std': trans_std + 'trans_std': trans_std, } return attrs @@ -44,13 +52,12 @@ def set_attrs(no_trans, spatial_scale, output_channels, group_size, def set_outputs(output, top_count): outputs = { 'Output': output.astype('float32'), - 'TopCount': top_count.astype('float32') + 'TopCount': top_count.astype('float32'), } return outputs class TestDeformablePSROIPoolOp(OpTest): - def set_data(self): self.start_test1() self.start_test2() @@ -77,9 +84,17 @@ def start_test1(self): sample_per_part = self.sample_per_part trans_std = self.trans_std - self.attrs = set_attrs(no_trans, spatial_scale, output_channels, - group_size, pooled_height, pooled_width, - part_size, sample_per_part, trans_std) + self.attrs = set_attrs( + no_trans, + spatial_scale, + output_channels, + group_size, + pooled_height, + pooled_width, + part_size, + sample_per_part, + trans_std, + ) output = self.out.astype('float32') top_count = self.top_count.astype('float32') @@ -105,9 +120,17 @@ def start_test2(self): sample_per_part = self.sample_per_part trans_std = self.trans_std - self.attrs = set_attrs(no_trans, spatial_scale, output_channels, - group_size, pooled_height, pooled_width, - part_size, sample_per_part, trans_std) + self.attrs = set_attrs( + no_trans, + spatial_scale, + output_channels, + group_size, + pooled_height, + pooled_width, + part_size, + sample_per_part, + trans_std, + ) output = self.out.astype('float32') top_count = self.top_count.astype('float32') @@ -133,9 +156,17 @@ def start_test3(self): sample_per_part = self.sample_per_part trans_std = self.trans_std - self.attrs = set_attrs(no_trans, spatial_scale, output_channels, - group_size, pooled_height, pooled_width, - part_size, sample_per_part, trans_std) + self.attrs = set_attrs( + no_trans, + spatial_scale, + output_channels, + group_size, + pooled_height, + pooled_width, + part_size, + sample_per_part, + trans_std, + ) output = self.out.astype('float32') top_count = self.top_count.astype('float32') @@ -161,9 +192,17 @@ def start_test4(self): sample_per_part = self.sample_per_part trans_std = self.trans_std - self.attrs = set_attrs(no_trans, spatial_scale, output_channels, - group_size, pooled_height, pooled_width, - part_size, sample_per_part, trans_std) + self.attrs = set_attrs( + no_trans, + spatial_scale, + output_channels, + group_size, + pooled_height, + pooled_width, + part_size, + sample_per_part, + trans_std, + ) output = self.out.astype('float32') top_count = self.top_count.astype('float32') @@ -175,7 +214,10 @@ def init_test_case1(self): self.height = 12 self.width = 12 self.input_dim = [ - self.batch_size, self.channels, self.height, self.width + self.batch_size, + self.channels, + self.height, + self.width, ] self.no_trans = False self.spatial_scale = 1.0 / 4.0 @@ -194,7 +236,10 @@ def init_test_case2(self): self.height = 12 self.width = 12 self.input_dim = [ - self.batch_size, self.channels, self.height, self.width + self.batch_size, + self.channels, + self.height, + self.width, ] self.no_trans = True self.spatial_scale = 1.0 / 2.0 @@ -213,7 +258,10 @@ def init_test_case3(self): self.height = 12 self.width = 12 self.input_dim = [ - self.batch_size, self.channels, self.height, self.width + self.batch_size, + self.channels, + self.height, + self.width, ] self.no_trans = False self.spatial_scale = 1.0 / 4.0 @@ -232,7 +280,10 @@ def init_test_case4(self): self.height = 12 self.width = 12 self.input_dim = [ - self.batch_size, self.channels, self.height, self.width + self.batch_size, + self.channels, + self.height, + self.width, ] self.no_trans = True self.spatial_scale = 1.0 / 2.0 @@ -252,13 +303,17 @@ def make_rois(self): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x_1 = np.random.randint( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y_1 = np.random.randint( - 0, self.height // self.spatial_scale - self.pooled_height) - x_2 = np.random.randint(x_1 + self.pooled_width, - self.width // self.spatial_scale) - y_2 = np.random.randint(y_1 + self.pooled_height, - self.height // self.spatial_scale) + 0, self.height // self.spatial_scale - self.pooled_height + ) + x_2 = np.random.randint( + x_1 + self.pooled_width, self.width // self.spatial_scale + ) + y_2 = np.random.randint( + y_1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x_1, y_1, x_2, y_2] rois.append(roi) self.rois_num = len(rois) @@ -290,20 +345,38 @@ def dmc_bilinear(self, data_im, p_h, p_w): return val def calc_deformable_psroi_pooling(self): - output_shape = (self.rois_num, self.output_channels, self.pooled_height, - self.pooled_width) + output_shape = ( + self.rois_num, + self.output_channels, + self.pooled_height, + self.pooled_width, + ) self.out = np.zeros(output_shape) - self.trans = np.random.rand(self.rois_num, 2, self.part_size[0], - self.part_size[1]).astype('float32') + self.trans = np.random.rand( + self.rois_num, 2, self.part_size[0], self.part_size[1] + ).astype('float32') self.top_count = np.random.random((output_shape)).astype('float32') - count = self.rois_num * self.output_channels * self.pooled_height * self.pooled_width + count = ( + self.rois_num + * self.output_channels + * self.pooled_height + * self.pooled_width + ) for index in range(count): p_w = int(index % self.pooled_width) p_h = int(index / self.pooled_width % self.pooled_height) - ctop = int(index / self.pooled_width / self.pooled_height % - self.output_channels) - n_out = int(index / self.pooled_width / self.pooled_height / - self.output_channels) + ctop = int( + index + / self.pooled_width + / self.pooled_height + % self.output_channels + ) + n_out = int( + index + / self.pooled_width + / self.pooled_height + / self.output_channels + ) roi = self.rois[n_out] roi_batch_id = int(roi[0]) roi_start_w = int(np.round(roi[1])) * self.spatial_scale - 0.5 @@ -339,15 +412,22 @@ def calc_deformable_psroi_pooling(self): for i_h in range(self.sample_per_part): w_sample = wstart + i_w * sub_bin_size_w h_sample = hstart + i_h * sub_bin_size_h - if w_sample < -0.5 or w_sample > self.width - 0.5 or \ - h_sample < -0.5 or h_sample > self.height - 0.5: + if ( + w_sample < -0.5 + or w_sample > self.width - 0.5 + or h_sample < -0.5 + or h_sample > self.height - 0.5 + ): continue - w_sample = min(max(w_sample, 0.), self.width - 1.) - h_sample = min(max(h_sample, 0.), self.height - 1.) - c_sample = int((ctop * self.group_size[0] + g_h) * - self.group_size[1] + g_w) - val = self.dmc_bilinear(input_i[c_sample], h_sample, - w_sample) + w_sample = min(max(w_sample, 0.0), self.width - 1.0) + h_sample = min(max(h_sample, 0.0), self.height - 1.0) + c_sample = int( + (ctop * self.group_size[0] + g_h) * self.group_size[1] + + g_w + ) + val = self.dmc_bilinear( + input_i[c_sample], h_sample, w_sample + ) sum = sum + val num_sample = num_sample + 1 if num_sample == 0: @@ -368,131 +448,144 @@ def test_check_grad(self): class TestDeformablePSROIPoolOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - input1 = fluid.data(name="input1", - shape=[2, 192, 64, 64], - dtype='float32') - rois1 = fluid.data(name="rois1", - shape=[-1, 4], - dtype='float32', - lod_level=1) - trans1 = fluid.data(name="trans1", - shape=[2, 384, 64, 64], - dtype='float32') + input1 = fluid.data( + name="input1", shape=[2, 192, 64, 64], dtype='float32' + ) + rois1 = fluid.data( + name="rois1", shape=[-1, 4], dtype='float32', lod_level=1 + ) + trans1 = fluid.data( + name="trans1", shape=[2, 384, 64, 64], dtype='float32' + ) # The `input` must be Variable and the data type of `input` Tensor must be one of float32 and float64. def test_input_type(): - fluid.layers.deformable_roi_pooling(input=[3, 4], - rois=rois1, - trans=trans1, - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - position_sensitive=True) + fluid.layers.deformable_roi_pooling( + input=[3, 4], + rois=rois1, + trans=trans1, + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_input_type) def test_input_tensor_dtype(): - input2 = fluid.data(name="input2", - shape=[2, 192, 64, 64], - dtype='int32') - fluid.layers.deformable_roi_pooling(input=input2, - rois=rois1, - trans=trans1, - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - position_sensitive=True) + input2 = fluid.data( + name="input2", shape=[2, 192, 64, 64], dtype='int32' + ) + fluid.layers.deformable_roi_pooling( + input=input2, + rois=rois1, + trans=trans1, + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_input_tensor_dtype) # The `rois` must be Variable and the data type of `rois` Tensor must be one of float32 and float64. def test_rois_type(): - fluid.layers.deformable_roi_pooling(input=input1, - rois=2, - trans=trans1, - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - position_sensitive=True) + fluid.layers.deformable_roi_pooling( + input=input1, + rois=2, + trans=trans1, + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_rois_type) def test_rois_tensor_dtype(): - rois2 = fluid.data(name="rois2", - shape=[-1, 4], - dtype='int32', - lod_level=1) - fluid.layers.deformable_roi_pooling(input=input1, - rois=rois2, - trans=trans1, - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - position_sensitive=True) + rois2 = fluid.data( + name="rois2", shape=[-1, 4], dtype='int32', lod_level=1 + ) + fluid.layers.deformable_roi_pooling( + input=input1, + rois=rois2, + trans=trans1, + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_rois_tensor_dtype) # The `trans` must be Variable and the data type of `trans` Tensor must be one of float32 and float64. def test_trans_type(): - fluid.layers.deformable_roi_pooling(input=input1, - rois=rois1, - trans=[2], - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - position_sensitive=True) + fluid.layers.deformable_roi_pooling( + input=input1, + rois=rois1, + trans=[2], + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_trans_type) def test_trans_tensor_dtype(): - trans2 = fluid.data(name="trans2", - shape=[2, 384, 64, 64], - dtype='int32') - fluid.layers.deformable_roi_pooling(input=input1, - rois=rois1, - trans=trans2, - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - position_sensitive=True) + trans2 = fluid.data( + name="trans2", shape=[2, 384, 64, 64], dtype='int32' + ) + fluid.layers.deformable_roi_pooling( + input=input1, + rois=rois1, + trans=trans2, + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_trans_tensor_dtype) # The `group_size` must be one of list and tuple. # Each element must be int. def test_group_size_type(): - fluid.layers.deformable_roi_pooling(input=input1, - rois=rois1, - trans=trans1, - group_size=1, - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - position_sensitive=True) + fluid.layers.deformable_roi_pooling( + input=input1, + rois=rois1, + trans=trans1, + group_size=1, + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_group_size_type) # The `part_size` must be one of list, tuple and None. # Each element must be int. def test_part_size_type(): - fluid.layers.deformable_roi_pooling(input=input1, - rois=rois1, - trans=trans1, - pooled_height=8, - pooled_width=8, - part_size=8, - sample_per_part=4, - position_sensitive=True) + fluid.layers.deformable_roi_pooling( + input=input1, + rois=rois1, + trans=trans1, + pooled_height=8, + pooled_width=8, + part_size=8, + sample_per_part=4, + position_sensitive=True, + ) self.assertRaises(TypeError, test_part_size_type) diff --git a/python/paddle/fluid/tests/unittests/test_deg2rad.py b/python/paddle/fluid/tests/unittests/test_deg2rad.py index 4c9ec1becc4a2..79e4541b635e1 100644 --- a/python/paddle/fluid/tests/unittests/test_deg2rad.py +++ b/python/paddle/fluid/tests/unittests/test_deg2rad.py @@ -22,11 +22,11 @@ class TestDeg2radAPI(unittest.TestCase): - def setUp(self): self.x_dtype = 'float64' - self.x_np = np.array([180.0, -180.0, 360.0, -360.0, 90.0, - -90.0]).astype(np.float64) + self.x_np = np.array( + [180.0, -180.0, 360.0, -360.0, 90.0, -90.0] + ).astype(np.float64) self.x_shape = [6] self.out_np = np.deg2rad(self.x_np) @@ -37,12 +37,17 @@ def test_static_graph(self): x = fluid.data(name='input', dtype=self.x_dtype, shape=self.x_shape) out = paddle.deg2rad(x) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(fluid.default_main_program(), - feed={'input': self.x_np}, - fetch_list=[out]) + res = exe.run( + fluid.default_main_program(), + feed={'input': self.x_np}, + fetch_list=[out], + ) self.assertTrue((np.array(out[0]) == self.out_np).all()) def test_dygraph(self): diff --git a/python/paddle/fluid/tests/unittests/test_density_prior_box_op.py b/python/paddle/fluid/tests/unittests/test_density_prior_box_op.py index ec747d5aef2e7..7254f9cc2f8bc 100644 --- a/python/paddle/fluid/tests/unittests/test_density_prior_box_op.py +++ b/python/paddle/fluid/tests/unittests/test_density_prior_box_op.py @@ -19,7 +19,6 @@ class TestDensityPriorBoxOp(OpTest): - def set_data(self): self.init_test_params() self.init_test_input() @@ -35,7 +34,7 @@ def set_data(self): 'densities': self.densities, 'fixed_sizes': self.fixed_sizes, 'fixed_ratios': self.fixed_ratios, - 'flatten_to_2d': self.flatten_to_2d + 'flatten_to_2d': self.flatten_to_2d, } self.outputs = {'Boxes': self.out_boxes, 'Variances': self.out_var} @@ -74,18 +73,19 @@ def init_test_params(self): if len(self.fixed_sizes) > 0 and len(self.densities) > 0: for density in self.densities: if len(self.fixed_ratios) > 0: - self.num_priors += len(self.fixed_ratios) * (pow( - density, 2)) + self.num_priors += len(self.fixed_ratios) * ( + pow(density, 2) + ) self.offset = 0.5 def init_test_input(self): self.image = np.random.random( - (self.batch_size, self.image_channels, self.image_w, - self.image_h)).astype('float32') + (self.batch_size, self.image_channels, self.image_w, self.image_h) + ).astype('float32') self.input = np.random.random( - (self.batch_size, self.input_channels, self.layer_w, - self.layer_h)).astype('float32') + (self.batch_size, self.input_channels, self.layer_w, self.layer_h) + ).astype('float32') def init_test_output(self): out_dim = (self.layer_h, self.layer_w, self.num_priors, 4) @@ -99,32 +99,56 @@ def init_test_output(self): c_x = (w + self.offset) * self.step_w c_y = (h + self.offset) * self.step_h # Generate density prior boxes with fixed size - for density, fixed_size in zip(self.densities, - self.fixed_sizes): - if (len(self.fixed_ratios) > 0): + for density, fixed_size in zip( + self.densities, self.fixed_sizes + ): + if len(self.fixed_ratios) > 0: for ar in self.fixed_ratios: shift = int(step_average / density) box_width_ratio = fixed_size * math.sqrt(ar) box_height_ratio = fixed_size / math.sqrt(ar) for di in range(density): for dj in range(density): - c_x_temp = c_x - step_average / 2.0 + shift / 2.0 + dj * shift - c_y_temp = c_y - step_average / 2.0 + shift / 2.0 + di * shift + c_x_temp = ( + c_x + - step_average / 2.0 + + shift / 2.0 + + dj * shift + ) + c_y_temp = ( + c_y + - step_average / 2.0 + + shift / 2.0 + + di * shift + ) out_boxes[h, w, idx, :] = [ - max((c_x_temp - box_width_ratio / 2.0) / - self.image_w, 0), - max((c_y_temp - box_height_ratio / 2.0) - / self.image_h, 0), - min((c_x_temp + box_width_ratio / 2.0) / - self.image_w, 1), - min((c_y_temp + box_height_ratio / 2.0) - / self.image_h, 1) + max( + (c_x_temp - box_width_ratio / 2.0) + / self.image_w, + 0, + ), + max( + (c_y_temp - box_height_ratio / 2.0) + / self.image_h, + 0, + ), + min( + (c_x_temp + box_width_ratio / 2.0) + / self.image_w, + 1, + ), + min( + (c_y_temp + box_height_ratio / 2.0) + / self.image_h, + 1, + ), ] idx += 1 if self.clip: out_boxes = np.clip(out_boxes, 0.0, 1.0) - out_var = np.tile(self.variances, - (self.layer_h, self.layer_w, self.num_priors, 1)) + out_var = np.tile( + self.variances, (self.layer_h, self.layer_w, self.num_priors, 1) + ) self.out_boxes = out_boxes.astype('float32') self.out_var = out_var.astype('float32') if self.flatten_to_2d: @@ -133,7 +157,6 @@ def init_test_output(self): class TestDensityPriorBox(TestDensityPriorBoxOp): - def set_density(self): self.densities = [3, 4] self.fixed_sizes = [1.0, 2.0] diff --git a/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py b/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py index 9933a2243fb04..ae24334fea572 100755 --- a/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py +++ b/python/paddle/fluid/tests/unittests/test_deprecated_decorator.py @@ -51,9 +51,11 @@ def get_warning_index(api): doc_lst = api.__doc__.splitlines() for idx, val in enumerate(doc_lst): - if val.startswith("Warning: ") and val.endswith( - " instead." - ) and "and will be removed in future versions." in val: + if ( + val.startswith("Warning: ") + and val.endswith(" instead.") + and "and will be removed in future versions." in val + ): return idx return ERROR_WARNING_POSTION @@ -152,7 +154,7 @@ def test_ops_elementwise_mul(self): def test_tensor_gradient(self): paddle.__version__ = '2.1.0' - x = paddle.to_tensor(5., stop_gradient=False) + x = paddle.to_tensor(5.0, stop_gradient=False) y = paddle.pow(x, 4.0) y.backward() @@ -160,7 +162,8 @@ def test_tensor_gradient(self): grad = x.gradient() assert ( 'API "paddle.fluid.dygraph.varbase_patch_methods.gradient" is ' - 'deprecated since 2.1.0') in str(w[-1].message) + 'deprecated since 2.1.0' + ) in str(w[-1].message) def test_softmax_with_cross_entropy(self): paddle.__version__ = '2.0.0' @@ -173,11 +176,13 @@ def test_softmax_with_cross_entropy(self): x = linear(data) with warnings.catch_warnings(record=True) as w: - out = paddle.nn.functional.softmax_with_cross_entropy(logits=x, - label=label) + out = paddle.nn.functional.softmax_with_cross_entropy( + logits=x, label=label + ) assert ( 'API "paddle.nn.functional.loss.softmax_with_cross_entropy" is ' - 'deprecated since 2.0.0') in str(w[-1].message) + 'deprecated since 2.0.0' + ) in str(w[-1].message) def test_deprecated_error(self): paddle.__version__ = '2.1.0' diff --git a/python/paddle/fluid/tests/unittests/test_deprecated_memory_optimize_interfaces.py b/python/paddle/fluid/tests/unittests/test_deprecated_memory_optimize_interfaces.py index bd91e14e34d38..c3a21ba0bcbb6 100644 --- a/python/paddle/fluid/tests/unittests/test_deprecated_memory_optimize_interfaces.py +++ b/python/paddle/fluid/tests/unittests/test_deprecated_memory_optimize_interfaces.py @@ -18,7 +18,6 @@ class DeprecatedMemoryOptimizationInterfaceTest(unittest.TestCase): - def setUp(self): self.method = fluid.memory_optimize @@ -61,7 +60,6 @@ def test_main(self): class ReleaseMemoryTest(DeprecatedMemoryOptimizationInterfaceTest): - def setUp(self): self.method = fluid.release_memory diff --git a/python/paddle/fluid/tests/unittests/test_dequantize_abs_max_op.py b/python/paddle/fluid/tests/unittests/test_dequantize_abs_max_op.py index 039686777bec6..c4806866eff6e 100644 --- a/python/paddle/fluid/tests/unittests/test_dequantize_abs_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_dequantize_abs_max_op.py @@ -30,7 +30,6 @@ def dequantize_max_abs(x, scale, max_range): class TestDequantizeMaxAbsOp(OpTest): - def set_args(self): self.num_bits = 8 self.max_range = math.pow(2, self.num_bits - 1) - 1 @@ -45,7 +44,7 @@ def setUp(self): self.inputs = { 'X': np.array(yq).astype(self.data_type), - 'Scale': np.array(scale).astype('float32') + 'Scale': np.array(scale).astype('float32'), } self.attrs = {'max_range': self.max_range} self.outputs = {'Out': ydq} @@ -55,7 +54,6 @@ def test_check_output(self): class TestDequantizeMaxAbsOp5Bits(TestDequantizeMaxAbsOp): - def set_args(self): self.num_bits = 5 self.max_range = math.pow(2, self.num_bits - 1) - 1 @@ -63,7 +61,6 @@ def set_args(self): class TestDequantizeMaxAbsOpInt16(TestDequantizeMaxAbsOp): - def set_args(self): self.num_bits = 16 self.max_range = math.pow(2, self.num_bits - 1) - 1 diff --git a/python/paddle/fluid/tests/unittests/test_dequantize_log_op.py b/python/paddle/fluid/tests/unittests/test_dequantize_log_op.py index 939d5d6afcfb6..5fb9d07d77a42 100644 --- a/python/paddle/fluid/tests/unittests/test_dequantize_log_op.py +++ b/python/paddle/fluid/tests/unittests/test_dequantize_log_op.py @@ -30,7 +30,6 @@ def dequantize_log(x, dict_data): class TestDequantizeLogOp(OpTest): - def setUp(self): self.op_type = "dequantize_log" x = np.random.randint(low=-128, high=127, size=(20, 10)).astype('int8') @@ -39,7 +38,7 @@ def setUp(self): self.inputs = { 'X': np.array(x).astype('int8'), - 'Dict': np.array(dict_data).astype('float32') + 'Dict': np.array(dict_data).astype('float32'), } self.outputs = {'Out': xdq} diff --git a/python/paddle/fluid/tests/unittests/test_desc_clone.py b/python/paddle/fluid/tests/unittests/test_desc_clone.py index 2744487b5883b..ad0e95d4bbcde 100644 --- a/python/paddle/fluid/tests/unittests/test_desc_clone.py +++ b/python/paddle/fluid/tests/unittests/test_desc_clone.py @@ -27,33 +27,41 @@ # random seed must set before configuring the network. # fluid.default_startup_program().random_seed = SEED def cnn_model(data): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=data, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=data, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) # TODO(dzhwinter) : refine the initializer and random seed settting SIZE = 10 input_shape = conv_pool_2.shape - param_shape = [functools.reduce(lambda a, b: a * b, input_shape[1:], 1) - ] + [SIZE] - scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5 + param_shape = [functools.reduce(lambda a, b: a * b, input_shape[1:], 1)] + [ + SIZE + ] + scale = (2.0 / (param_shape[0] ** 2 * SIZE)) ** 0.5 predict = fluid.layers.fc( input=conv_pool_2, size=SIZE, act="softmax", param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.NormalInitializer(loc=0.0, - scale=scale))) + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=scale + ) + ), + ) return predict @@ -69,23 +77,32 @@ def get_model(batch_size): # Evaluator batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size_tensor) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor + ) inference_program = fluid.default_main_program().clone() # Optimization - opt = fluid.optimizer.AdamOptimizer(learning_rate=0.001, - beta1=0.9, - beta2=0.999) + opt = fluid.optimizer.AdamOptimizer( + learning_rate=0.001, beta1=0.9, beta2=0.999 + ) # Reader - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size) - test_reader = paddle.batch(paddle.dataset.mnist.test(), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size + ) + test_reader = paddle.batch( + paddle.dataset.mnist.test(), batch_size=batch_size + ) opt.minimize(avg_cost) - return inference_program, avg_cost, train_reader, test_reader, batch_acc, predict + return ( + inference_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) def operator_equal(a, b): @@ -93,8 +110,9 @@ def operator_equal(a, b): raise ValueError("In operator_equal not equal\n") for k, v in a.__dict__.items(): - if isinstance(v, fluid.framework.Program) or \ - isinstance(v, fluid.framework.Block): + if isinstance(v, fluid.framework.Program) or isinstance( + v, fluid.framework.Block + ): continue elif isinstance(v, core.OpDesc): @@ -107,7 +125,7 @@ def operator_equal(a, b): if v0 != v1: raise ValueError("In operator_equal not equal:{0}\n".format(k)) - elif (v != b.__dict__[k]): + elif v != b.__dict__[k]: raise ValueError("In operator_equal not equal:{0}\n".format(k)) return True @@ -115,12 +133,15 @@ def operator_equal(a, b): def block_equal(a, b): for k, v in a.__dict__.items(): - if isinstance(v, core.ProgramDesc) or isinstance( - v, fluid.framework.Program) or isinstance(v, core.BlockDesc): + if ( + isinstance(v, core.ProgramDesc) + or isinstance(v, fluid.framework.Program) + or isinstance(v, core.BlockDesc) + ): continue elif k == "ops": - assert (len(a.ops) == len(b.ops)) + assert len(a.ops) == len(b.ops) for i in range(0, len(a.ops)): if not operator_equal(a.ops[i], b.ops[i]): raise ValueError("In block_equal not equal:{0}\n".format(k)) @@ -130,7 +151,7 @@ def block_equal(a, b): if str(value) != str(b.__dict__[k][key]): raise ValueError("In block_equal not equal:{0}\n".format(k)) - elif (v != b.__dict__[k]): + elif v != b.__dict__[k]: raise ValueError("In block_equal not equal:{0}\n".format(k)) return True @@ -145,19 +166,19 @@ def program_equal(a, b): for i in range(0, len(a.blocks)): if not block_equal(a.blocks[i], b.blocks[i]): raise ValueError( - "In operator_equal not equal:{0}\n".format(k)) + "In operator_equal not equal:{0}\n".format(k) + ) return False - assert (len(a.blocks) == len(b.blocks)) + assert len(a.blocks) == len(b.blocks) elif k == '_auto_checkpoint_name': continue - elif (v != b.__dict__[k]): + elif v != b.__dict__[k]: raise ValueError("In program_equal not equal:{0}\n".format(k)) return True class TestCloneWithStopGradient(unittest.TestCase): - def test_clone_with_stop_gradient(self): train_program = fluid.Program() startup_program = fluid.Program() @@ -168,18 +189,20 @@ def test_clone_with_stop_gradient(self): hidden2 = fluid.layers.dropout(hidden1, dropout_prob=0.5) loss = fluid.layers.cross_entropy( input=fluid.layers.fc(hidden2, size=10, act='softmax'), - label=fluid.layers.data(name='label', shape=[1], dtype='int64')) + label=fluid.layers.data(name='label', shape=[1], dtype='int64'), + ) avg_loss = paddle.mean(loss) test_program = train_program.clone(for_test=False) self.assertEqual( - test_program.block(0).var(hidden1.name).stop_gradient, True) + test_program.block(0).var(hidden1.name).stop_gradient, True + ) self.assertEqual( - test_program.block(0).var(hidden2.name).stop_gradient, False) + test_program.block(0).var(hidden2.name).stop_gradient, False + ) class TestCloneWithStopGradientInSubBlock(unittest.TestCase): - def test_clone_with_stop_gradient(self): train_program = fluid.Program() startup_program = fluid.Program() @@ -204,12 +227,14 @@ def false_fn(): loss = fluid.layers.cross_entropy( input=fluid.layers.fc(hidden2, size=10, act='softmax'), - label=fluid.layers.data(name='label', shape=[1], dtype='int64')) + label=fluid.layers.data(name='label', shape=[1], dtype='int64'), + ) avg_loss = paddle.mean(loss) test_program = train_program.clone(for_test=False) self.assertEqual( - test_program.block(0).var(hidden1.name).stop_gradient, True) + test_program.block(0).var(hidden1.name).stop_gradient, True + ) for var in test_program.block(1).vars.values(): var2 = train_program.block(1).var(var.name) self.assertEqual(var.stop_gradient, var2.stop_gradient) @@ -219,7 +244,6 @@ def false_fn(): class TestCloneWithRaise(unittest.TestCase): - def test_clone_with_stop_gradient(self): train_program = fluid.Program() startup_program = fluid.Program() @@ -243,14 +267,19 @@ def false_fn(): hidden2 = fluid.layers.cond(cond, true_fn, false_fn) loss = fluid.layers.cross_entropy( input=fluid.layers.fc(hidden2, size=10, act='softmax'), - label=fluid.layers.data(name='label', shape=[1], dtype='int64')) + label=fluid.layers.data(name='label', shape=[1], dtype='int64'), + ) avg_loss = paddle.mean(loss) test_program = train_program.clone(for_test=False) - self.assertRaises(ValueError, train_program._copy_data_info_from, - startup_program) - self.assertRaises(TypeError, train_program._copy_data_info_from, - startup_program.block(0)) + self.assertRaises( + ValueError, train_program._copy_data_info_from, startup_program + ) + self.assertRaises( + TypeError, + train_program._copy_data_info_from, + startup_program.block(0), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_detach.py b/python/paddle/fluid/tests/unittests/test_detach.py index 901f35a756bca..48abc41e3c11c 100644 --- a/python/paddle/fluid/tests/unittests/test_detach.py +++ b/python/paddle/fluid/tests/unittests/test_detach.py @@ -23,39 +23,51 @@ class Test_Detach(unittest.TestCase): - def generate_Data(self): - data = np.array([[1, 8, 3, 9], [7, 20, 9, 6], [4, 6, 8, - 10]]).astype('float32') + data = np.array([[1, 8, 3, 9], [7, 20, 9, 6], [4, 6, 8, 10]]).astype( + 'float32' + ) return data def no_detach_multi(self): data = self.generate_Data() with fluid.dygraph.guard(): linear_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(5.0)) + initializer=fluid.initializer.Constant(5.0) + ) linear_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(6.0)) - linear = Linear(4, - 10, - param_attr=linear_w_param_attrs, - bias_attr=linear_b_param_attrs) + initializer=fluid.initializer.Constant(6.0) + ) + linear = Linear( + 4, + 10, + param_attr=linear_w_param_attrs, + bias_attr=linear_b_param_attrs, + ) linear1_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(7.0)) + initializer=fluid.initializer.Constant(7.0) + ) linear1_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(8.0)) - linear1 = Linear(10, - 1, - param_attr=linear1_w_param_attrs, - bias_attr=linear1_b_param_attrs) + initializer=fluid.initializer.Constant(8.0) + ) + linear1 = Linear( + 10, + 1, + param_attr=linear1_w_param_attrs, + bias_attr=linear1_b_param_attrs, + ) linear2_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(9.0)) + initializer=fluid.initializer.Constant(9.0) + ) linear2_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(10.0)) - linear2 = Linear(10, - 1, - param_attr=linear2_w_param_attrs, - bias_attr=linear2_b_param_attrs) + initializer=fluid.initializer.Constant(10.0) + ) + linear2 = Linear( + 10, + 1, + param_attr=linear2_w_param_attrs, + bias_attr=linear2_b_param_attrs, + ) data = to_variable(data) x = linear(data) x1 = linear1(x) @@ -69,21 +81,29 @@ def no_detach_single(self): data = self.generate_Data() with fluid.dygraph.guard(): linear_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(5.0)) + initializer=fluid.initializer.Constant(5.0) + ) linear_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(6.0)) - linear = Linear(4, - 10, - param_attr=linear_w_param_attrs, - bias_attr=linear_b_param_attrs) + initializer=fluid.initializer.Constant(6.0) + ) + linear = Linear( + 4, + 10, + param_attr=linear_w_param_attrs, + bias_attr=linear_b_param_attrs, + ) linear1_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(7.0)) + initializer=fluid.initializer.Constant(7.0) + ) linear1_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(8.0)) - linear1 = Linear(10, - 1, - param_attr=linear1_w_param_attrs, - bias_attr=linear1_b_param_attrs) + initializer=fluid.initializer.Constant(8.0) + ) + linear1 = Linear( + 10, + 1, + param_attr=linear1_w_param_attrs, + bias_attr=linear1_b_param_attrs, + ) data = to_variable(data) x = linear(data) x1 = linear1(x) @@ -96,29 +116,41 @@ def detach_multi(self): data = self.generate_Data() with fluid.dygraph.guard(): linear_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(5.0)) + initializer=fluid.initializer.Constant(5.0) + ) linear_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(6.0)) - linear = Linear(4, - 10, - param_attr=linear_w_param_attrs, - bias_attr=linear_b_param_attrs) + initializer=fluid.initializer.Constant(6.0) + ) + linear = Linear( + 4, + 10, + param_attr=linear_w_param_attrs, + bias_attr=linear_b_param_attrs, + ) linear1_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(7.0)) + initializer=fluid.initializer.Constant(7.0) + ) linear1_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(8.0)) - linear1 = Linear(10, - 1, - param_attr=linear1_w_param_attrs, - bias_attr=linear1_b_param_attrs) + initializer=fluid.initializer.Constant(8.0) + ) + linear1 = Linear( + 10, + 1, + param_attr=linear1_w_param_attrs, + bias_attr=linear1_b_param_attrs, + ) linear2_w_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(9.0)) + initializer=fluid.initializer.Constant(9.0) + ) linear2_b_param_attrs = fluid.ParamAttr( - initializer=fluid.initializer.Constant(10.0)) - linear2 = Linear(10, - 1, - param_attr=linear2_w_param_attrs, - bias_attr=linear2_b_param_attrs) + initializer=fluid.initializer.Constant(10.0) + ) + linear2 = Linear( + 10, + 1, + param_attr=linear2_w_param_attrs, + bias_attr=linear2_b_param_attrs, + ) data = to_variable(data) x = linear(data) x_detach = x.detach() @@ -144,7 +176,6 @@ def test_NoDetachSingle_DetachMulti(self): class TestInplace(unittest.TestCase): - def test_forward_version(self): with paddle.fluid.dygraph.guard(): var = paddle.to_tensor(np.ones((4, 2, 3)).astype(np.float32)) @@ -180,9 +211,11 @@ def test_backward_error(self): loss = paddle.nn.functional.relu(var_c + var_d) with self.assertRaisesRegexp( - RuntimeError, - "received tensor_version:{} != wrapper_version_snapshot:{}". - format(1, 0)): + RuntimeError, + "received tensor_version:{} != wrapper_version_snapshot:{}".format( + 1, 0 + ), + ): loss.backward() diff --git a/python/paddle/fluid/tests/unittests/test_detection_map_op.py b/python/paddle/fluid/tests/unittests/test_detection_map_op.py index 68afa3feed908..00b2c0700ca80 100644 --- a/python/paddle/fluid/tests/unittests/test_detection_map_op.py +++ b/python/paddle/fluid/tests/unittests/test_detection_map_op.py @@ -20,7 +20,6 @@ class TestDetectionMAPOp(OpTest): - def set_data(self): self.class_num = 4 self.init_test_case() @@ -30,8 +29,9 @@ def set_data(self): self.mAP = np.array(self.mAP).astype('float32') if len(self.class_pos_count) > 0: - self.class_pos_count = np.array( - self.class_pos_count).astype('int32') + self.class_pos_count = np.array(self.class_pos_count).astype( + 'int32' + ) self.true_pos = np.array(self.true_pos).astype('float32') self.false_pos = np.array(self.false_pos).astype('float32') self.has_state = np.array([1]).astype('int32') @@ -42,7 +42,7 @@ def set_data(self): 'HasState': self.has_state, 'PosCount': self.class_pos_count, 'TruePos': (self.true_pos, self.true_pos_lod), - 'FalsePos': (self.false_pos, self.false_pos_lod) + 'FalsePos': (self.false_pos, self.false_pos_lod), } else: self.inputs = { @@ -54,11 +54,12 @@ def set_data(self): 'overlap_threshold': self.overlap_threshold, 'evaluate_difficult': self.evaluate_difficult, 'ap_type': self.ap_type, - 'class_num': self.class_num + 'class_num': self.class_num, } - self.out_class_pos_count = np.array( - self.out_class_pos_count).astype('int') + self.out_class_pos_count = np.array(self.out_class_pos_count).astype( + 'int' + ) self.out_true_pos = np.array(self.out_true_pos).astype('float32') self.out_false_pos = np.array(self.out_false_pos).astype('float32') @@ -66,7 +67,7 @@ def set_data(self): 'MAP': self.mAP, 'AccumPosCount': self.out_class_pos_count, 'AccumTruePos': (self.out_true_pos, self.out_true_pos_lod), - 'AccumFalsePos': (self.out_false_pos, self.out_false_pos_lod) + 'AccumFalsePos': (self.out_false_pos, self.out_false_pos_lod), } def init_test_case(self): @@ -76,24 +77,36 @@ def init_test_case(self): self.label_lod = [[2, 2]] # label difficult xmin ymin xmax ymax - self.label = [[1, 0, 0.1, 0.1, 0.3, 0.3], [1, 1, 0.6, 0.6, 0.8, 0.8], - [2, 0, 0.3, 0.3, 0.6, 0.5], [1, 0, 0.7, 0.1, 0.9, 0.3]] + self.label = [ + [1, 0, 0.1, 0.1, 0.3, 0.3], + [1, 1, 0.6, 0.6, 0.8, 0.8], + [2, 0, 0.3, 0.3, 0.6, 0.5], + [1, 0, 0.7, 0.1, 0.9, 0.3], + ] # label score xmin ymin xmax ymax difficult self.detect_lod = [[3, 4]] - self.detect = [[1, 0.3, 0.1, 0.0, 0.4, - 0.3], [1, 0.7, 0.0, 0.1, 0.2, 0.3], - [1, 0.9, 0.7, 0.6, 0.8, - 0.8], [2, 0.8, 0.2, 0.1, 0.4, 0.4], - [2, 0.1, 0.4, 0.3, 0.7, - 0.5], [1, 0.2, 0.8, 0.1, 1.0, 0.3], - [3, 0.2, 0.8, 0.1, 1.0, 0.3]] + self.detect = [ + [1, 0.3, 0.1, 0.0, 0.4, 0.3], + [1, 0.7, 0.0, 0.1, 0.2, 0.3], + [1, 0.9, 0.7, 0.6, 0.8, 0.8], + [2, 0.8, 0.2, 0.1, 0.4, 0.4], + [2, 0.1, 0.4, 0.3, 0.7, 0.5], + [1, 0.2, 0.8, 0.1, 1.0, 0.3], + [3, 0.2, 0.8, 0.1, 1.0, 0.3], + ] # label score true_pos false_pos self.tf_pos_lod = [[3, 4]] - self.tf_pos = [[1, 0.9, 1, 0], [1, 0.7, 1, 0], [1, 0.3, 0, 1], - [1, 0.2, 1, 0], [2, 0.8, 0, 1], [2, 0.1, 1, 0], - [3, 0.2, 0, 1]] + self.tf_pos = [ + [1, 0.9, 1, 0], + [1, 0.7, 1, 0], + [1, 0.3, 0, 1], + [1, 0.2, 1, 0], + [2, 0.8, 0, 1], + [2, 0.1, 1, 0], + [3, 0.2, 0, 1], + ] self.class_pos_count = [] self.true_pos_lod = [[]] @@ -105,8 +118,9 @@ def calc_map(self, tf_pos, tf_pos_lod): mAP = 0.0 count = 0 - def get_input_pos(class_pos_count, true_pos, true_pos_lod, false_pos, - false_pos_lod): + def get_input_pos( + class_pos_count, true_pos, true_pos_lod, false_pos, false_pos_lod + ): class_pos_count_dict = collections.Counter() true_pos_dict = collections.defaultdict(list) false_pos_dict = collections.defaultdict(list) @@ -149,9 +163,13 @@ def get_output_pos(label_count, true_pos, false_pos): out_false_pos += false_pos_list out_false_pos_lod.append(len(false_pos_list)) - return out_class_pos_count, out_true_pos, [ - out_true_pos_lod - ], out_false_pos, [out_false_pos_lod] + return ( + out_class_pos_count, + out_true_pos, + [out_true_pos_lod], + out_false_pos, + [out_false_pos_lod], + ) def get_accumulation(pos_list): sorted_list = sorted(pos_list, key=lambda pos: pos[0], reverse=True) @@ -163,8 +181,12 @@ def get_accumulation(pos_list): return accu_list label_count, true_pos, false_pos = get_input_pos( - self.class_pos_count, self.true_pos, self.true_pos_lod, - self.false_pos, self.false_pos_lod) + self.class_pos_count, + self.true_pos, + self.true_pos_lod, + self.false_pos, + self.false_pos_lod, + ) for v in self.label: label = v[0] difficult = False if len(v) == 5 else v[1] @@ -178,7 +200,8 @@ def get_accumulation(pos_list): false_pos[label].append([score, fp]) for (label, label_pos_num) in label_count.items(): - if label_pos_num == 0: continue + if label_pos_num == 0: + continue if label not in true_pos: count += 1 continue @@ -193,8 +216,9 @@ def get_accumulation(pos_list): for i in range(len(accu_tp_sum)): precision.append( - float(accu_tp_sum[i]) / - float(accu_tp_sum[i] + accu_fp_sum[i])) + float(accu_tp_sum[i]) + / float(accu_tp_sum[i] + accu_fp_sum[i]) + ) recall.append(float(accu_tp_sum[i]) / label_pos_num) if self.ap_type == "11point": @@ -218,14 +242,16 @@ def get_accumulation(pos_list): prev_recall = 0.0 for i in range(len(accu_tp_sum)): if math.fabs(recall[i] - prev_recall) > 1e-6: - average_precisions += precision[i] * \ - math.fabs(recall[i] - prev_recall) + average_precisions += precision[i] * math.fabs( + recall[i] - prev_recall + ) prev_recall = recall[i] mAP += average_precisions count += 1 - pcnt, tp, tp_lod, fp, fp_lod = get_output_pos(label_count, true_pos, - false_pos) + pcnt, tp, tp_lod, fp, fp_lod = get_output_pos( + label_count, true_pos, false_pos + ) self.out_class_pos_count = pcnt self.out_true_pos = tp self.out_true_pos_lod = tp_lod @@ -244,7 +270,6 @@ def test_check_output(self): class TestDetectionMAPOpSkipDiff(TestDetectionMAPOp): - def init_test_case(self): super(TestDetectionMAPOpSkipDiff, self).init_test_case() @@ -252,22 +277,30 @@ def init_test_case(self): self.tf_pos_lod = [[2, 4]] # label score true_pos false_pos - self.tf_pos = [[1, 0.7, 1, 0], [1, 0.3, 0, 1], [1, 0.2, 1, 0], - [2, 0.8, 0, 1], [2, 0.1, 1, 0], [3, 0.2, 0, 1]] + self.tf_pos = [ + [1, 0.7, 1, 0], + [1, 0.3, 0, 1], + [1, 0.2, 1, 0], + [2, 0.8, 0, 1], + [2, 0.1, 1, 0], + [3, 0.2, 0, 1], + ] class TestDetectionMAPOpWithoutDiff(TestDetectionMAPOp): - def init_test_case(self): super(TestDetectionMAPOpWithoutDiff, self).init_test_case() # label xmin ymin xmax ymax - self.label = [[1, 0.1, 0.1, 0.3, 0.3], [1, 0.6, 0.6, 0.8, 0.8], - [2, 0.3, 0.3, 0.6, 0.5], [1, 0.7, 0.1, 0.9, 0.3]] + self.label = [ + [1, 0.1, 0.1, 0.3, 0.3], + [1, 0.6, 0.6, 0.8, 0.8], + [2, 0.3, 0.3, 0.6, 0.5], + [1, 0.7, 0.1, 0.9, 0.3], + ] class TestDetectionMAPOp11Point(TestDetectionMAPOp): - def init_test_case(self): super(TestDetectionMAPOp11Point, self).init_test_case() @@ -275,18 +308,28 @@ def init_test_case(self): class TestDetectionMAPOpMultiBatch(TestDetectionMAPOp): - def init_test_case(self): super(TestDetectionMAPOpMultiBatch, self).init_test_case() self.class_pos_count = [0, 2, 1, 0] self.true_pos_lod = [[0, 3, 2]] - self.true_pos = [[0.7, 1.], [0.3, 0.], [0.2, 1.], [0.8, 0.], [0.1, 1.]] + self.true_pos = [ + [0.7, 1.0], + [0.3, 0.0], + [0.2, 1.0], + [0.8, 0.0], + [0.1, 1.0], + ] self.false_pos_lod = [[0, 3, 2]] - self.false_pos = [[0.7, 0.], [0.3, 1.], [0.2, 0.], [0.8, 1.], [0.1, 0.]] + self.false_pos = [ + [0.7, 0.0], + [0.3, 1.0], + [0.2, 0.0], + [0.8, 1.0], + [0.1, 0.0], + ] class TestDetectionMAPOp11PointWithClassNoTP(TestDetectionMAPOp): - def init_test_case(self): self.overlap_threshold = 0.3 self.evaluate_difficult = True diff --git a/python/paddle/fluid/tests/unittests/test_determinant_op.py b/python/paddle/fluid/tests/unittests/test_determinant_op.py index abd44669b25e6..730864eb23097 100644 --- a/python/paddle/fluid/tests/unittests/test_determinant_op.py +++ b/python/paddle/fluid/tests/unittests/test_determinant_op.py @@ -22,7 +22,6 @@ class TestDeterminantOp(OpTest): - def setUp(self): self.python_api = paddle.linalg.det self.init_data() @@ -43,7 +42,6 @@ def init_data(self): class TestDeterminantOpCase1(TestDeterminantOp): - def init_data(self): np.random.seed(0) self.case = np.random.rand(10, 10).astype('float32') @@ -52,7 +50,6 @@ def init_data(self): class TestDeterminantOpCase2(TestDeterminantOp): - def init_data(self): np.random.seed(0) # not invertible matrix @@ -62,7 +59,6 @@ def init_data(self): class TestDeterminantAPI(unittest.TestCase): - def setUp(self): np.random.seed(0) self.shape = [3, 3, 5, 5] @@ -95,7 +91,6 @@ def test_eager(self): class TestSlogDeterminantOp(OpTest): - def setUp(self): self.op_type = "slogdeterminant" self.python_api = paddle.linalg.slogdet @@ -107,9 +102,9 @@ def test_check_output(self): def test_check_grad(self): # the slog det's grad value is always huge - self.check_grad(['Input'], ['Out'], - max_relative_error=0.1, - check_eager=True) + self.check_grad( + ['Input'], ['Out'], max_relative_error=0.1, check_eager=True + ) def init_data(self): np.random.seed(0) @@ -119,7 +114,6 @@ def init_data(self): class TestSlogDeterminantOpCase1(TestSlogDeterminantOp): - def init_data(self): np.random.seed(0) self.case = np.random.rand(2, 2, 5, 5).astype(np.float32) @@ -128,7 +122,6 @@ def init_data(self): class TestSlogDeterminantAPI(unittest.TestCase): - def setUp(self): np.random.seed(0) self.shape = [3, 3, 5, 5] diff --git a/python/paddle/fluid/tests/unittests/test_device.py b/python/paddle/fluid/tests/unittests/test_device.py index 3d1653f123ffd..cf95ea96f7bad 100644 --- a/python/paddle/fluid/tests/unittests/test_device.py +++ b/python/paddle/fluid/tests/unittests/test_device.py @@ -21,7 +21,6 @@ class TestStaticDeviceManage(unittest.TestCase): - def _test_device(self, device_name, device_class): paddle.set_device(device_name) @@ -54,7 +53,6 @@ def test_npu_device(self): class TestImperativeDeviceManage(unittest.TestCase): - def test_cpu(self): with fluid.dygraph.guard(): paddle.set_device('cpu') @@ -64,7 +62,8 @@ def test_cpu(self): device = paddle.get_device() self.assertEqual( isinstance(framework._current_expected_place(), core.CPUPlace), - True) + True, + ) self.assertEqual(device, "cpu") def test_gpu(self): @@ -76,8 +75,11 @@ def test_gpu(self): out3 = paddle.concat(x=[out1, out2], axis=0) device = paddle.get_device() self.assertEqual( - isinstance(framework._current_expected_place(), - core.CUDAPlace), True) + isinstance( + framework._current_expected_place(), core.CUDAPlace + ), + True, + ) self.assertEqual(device, "gpu:0") def test_xpu(self): @@ -86,8 +88,11 @@ def test_xpu(self): out = paddle.to_tensor([1, 2]) device = paddle.get_device() self.assertEqual( - isinstance(framework._current_expected_place(), - core.XPUPlace), True) + isinstance( + framework._current_expected_place(), core.XPUPlace + ), + True, + ) self.assertTrue(out.place.is_xpu_place()) self.assertEqual(device, "xpu:0") @@ -100,8 +105,11 @@ def test_npu(self): out3 = paddle.concat(x=[out1, out2], axis=0) device = paddle.get_device() self.assertEqual( - isinstance(framework._current_expected_place(), - core.NPUPlace), True) + isinstance( + framework._current_expected_place(), core.NPUPlace + ), + True, + ) self.assertTrue(out1.place.is_npu_place()) self.assertTrue(out2.place.is_npu_place()) self.assertTrue(out3.place.is_npu_place()) diff --git a/python/paddle/fluid/tests/unittests/test_device_guard.py b/python/paddle/fluid/tests/unittests/test_device_guard.py index f3ac33a4f854c..b43fbb6e87f37 100644 --- a/python/paddle/fluid/tests/unittests/test_device_guard.py +++ b/python/paddle/fluid/tests/unittests/test_device_guard.py @@ -39,17 +39,16 @@ def get_vaild_warning_num(warning, w): class TestDeviceGuard(unittest.TestCase): - def test_device_guard(self): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - data1 = paddle.full(shape=[1, 3, 8, 8], - fill_value=0.5, - dtype='float32') - data2 = paddle.full(shape=[1, 3, 5, 5], - fill_value=0.5, - dtype='float32') + data1 = paddle.full( + shape=[1, 3, 8, 8], fill_value=0.5, dtype='float32' + ) + data2 = paddle.full( + shape=[1, 3, 5, 5], fill_value=0.5, dtype='float32' + ) shape = paddle.shape(data2) with paddle.static.device_guard("cpu"): shape = paddle.slice(shape, axes=[0], starts=[0], ends=[4]) @@ -70,12 +69,12 @@ def test_device_guard_with_id(self): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - data1 = paddle.full(shape=[1, 3, 8, 8], - fill_value=0.5, - dtype='float32') - data2 = paddle.full(shape=[1, 3, 5, 5], - fill_value=0.5, - dtype='float32') + data1 = paddle.full( + shape=[1, 3, 8, 8], fill_value=0.5, dtype='float32' + ) + data2 = paddle.full( + shape=[1, 3, 5, 5], fill_value=0.5, dtype='float32' + ) shape = paddle.shape(data2) with paddle.static.device_guard("cpu"): shape = paddle.slice(shape, axes=[0], starts=[0], ends=[4]) @@ -96,32 +95,50 @@ def test_cpu_only_op(self): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - x = paddle.full(shape=[2, 255, 13, 13], - fill_value=0.3, - dtype='float32') - gt_box = paddle.full(shape=[2, 6, 4], - fill_value=0.5, - dtype='float32') + x = paddle.full( + shape=[2, 255, 13, 13], fill_value=0.3, dtype='float32' + ) + gt_box = paddle.full( + shape=[2, 6, 4], fill_value=0.5, dtype='float32' + ) gt_label = paddle.full(shape=[2, 6], fill_value=1.0, dtype='int32') - gt_score = paddle.full(shape=[2, 6], - fill_value=0.5, - dtype='float32') + gt_score = paddle.full( + shape=[2, 6], fill_value=0.5, dtype='float32' + ) anchors = [ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, - 198, 373, 326 + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, ] anchor_mask = [0, 1, 2] with paddle.static.device_guard("gpu"): # yolov3_loss only has cpu kernel, so its cpu kernel will be executed - loss = fluid.layers.yolov3_loss(x=x, - gt_box=gt_box, - gt_label=gt_label, - gt_score=gt_score, - anchors=anchors, - anchor_mask=anchor_mask, - class_num=80, - ignore_thresh=0.7, - downsample_ratio=32) + loss = fluid.layers.yolov3_loss( + x=x, + gt_box=gt_box, + gt_label=gt_label, + gt_score=gt_score, + anchors=anchors, + anchor_mask=anchor_mask, + class_num=80, + ignore_thresh=0.7, + downsample_ratio=32, + ) execute(main_program, startup_program) @@ -154,7 +171,6 @@ def test_without_kernel_op(self): execute(main_program, startup_program) def test_error(self): - def device_attr(): with paddle.static.device_guard("cpu1"): out = paddle.full(shape=[1], fill_value=0.2, dtype='float32') @@ -171,17 +187,18 @@ def test_op_descs_device_attr(self): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - data1 = paddle.static.data(name="data_1", - shape=[4, 2], - dtype="float32") - label = paddle.static.data(name="label", - shape=[4, 1], - dtype="int64") + data1 = paddle.static.data( + name="data_1", shape=[4, 2], dtype="float32" + ) + label = paddle.static.data( + name="label", shape=[4, 1], dtype="int64" + ) fc1 = paddle.static.nn.fc(x=data1, size=10) fc2 = paddle.static.nn.fc(x=fc1, size=10) with paddle.static.device_guard("gpu"): out = paddle.nn.functional.softmax_with_cross_entropy( - logits=fc1 + fc2, label=label) + logits=fc1 + fc2, label=label + ) loss = paddle.mean(out) opt = paddle.optimizer.SGD(0.1) opt.minimize(loss) diff --git a/python/paddle/fluid/tests/unittests/test_diag.py b/python/paddle/fluid/tests/unittests/test_diag.py index 5148efe1b08f2..4135db34218ba 100644 --- a/python/paddle/fluid/tests/unittests/test_diag.py +++ b/python/paddle/fluid/tests/unittests/test_diag.py @@ -21,7 +21,6 @@ class TestDiagOp(OpTest): - def setUp(self): self.op_type = "diag" self.init_config() @@ -38,13 +37,11 @@ def init_config(self): class TestDiagOpCase1(TestDiagOp): - def init_config(self): self.case = np.array([3], dtype='int32') class TestDiagError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): diff --git a/python/paddle/fluid/tests/unittests/test_diag_embed.py b/python/paddle/fluid/tests/unittests/test_diag_embed.py index 8eeb7849da22d..c91f5d9b86c9c 100644 --- a/python/paddle/fluid/tests/unittests/test_diag_embed.py +++ b/python/paddle/fluid/tests/unittests/test_diag_embed.py @@ -21,7 +21,6 @@ class TestDiagEmbedOp(OpTest): - def setUp(self): self.op_type = "diag_embed" self.python_api = F.diag_embed @@ -39,17 +38,16 @@ def init_config(self): class TestDiagEmbedOpCase1(TestDiagEmbedOp): - def init_config(self): self.case = np.random.randn(2, 3).astype('float32') self.inputs = {'Input': self.case} self.attrs = {'offset': -1, 'dim1': 0, 'dim2': 2} - self.target = np.stack([np.diag(r, -1) for r in self.inputs['Input']], - 1) + self.target = np.stack( + [np.diag(r, -1) for r in self.inputs['Input']], 1 + ) class TestDiagEmbedAPICase(unittest.TestCase): - def test_case1(self): diag_embed = np.random.randn(2, 3, 4).astype('float32') data1 = fluid.data(name='data1', shape=[2, 3, 4], dtype='float32') @@ -58,14 +56,18 @@ def test_case1(self): place = core.CPUPlace() exe = fluid.Executor(place) - results = exe.run(fluid.default_main_program(), - feed={"data1": diag_embed}, - fetch_list=[out1, out2], - return_numpy=True) + results = exe.run( + fluid.default_main_program(), + feed={"data1": diag_embed}, + fetch_list=[out1, out2], + return_numpy=True, + ) target1 = np.stack( - [np.stack([np.diag(s, 0) for s in r], 0) for r in diag_embed], 0) + [np.stack([np.diag(s, 0) for s in r], 0) for r in diag_embed], 0 + ) target2 = np.stack( - [np.stack([np.diag(s, 1) for s in r], 0) for r in diag_embed], 0) + [np.stack([np.diag(s, 1) for s in r], 0) for r in diag_embed], 0 + ) np.testing.assert_allclose(results[0], target1, rtol=1e-05) np.testing.assert_allclose(results[1], target2, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_diag_v2.py b/python/paddle/fluid/tests/unittests/test_diag_v2.py index 5fb93f3b19a10..f1ff2b77d4041 100644 --- a/python/paddle/fluid/tests/unittests/test_diag_v2.py +++ b/python/paddle/fluid/tests/unittests/test_diag_v2.py @@ -22,7 +22,6 @@ class TestDiagV2Op(OpTest): - def setUp(self): self.op_type = "diag_v2" self.python_api = paddle.diag @@ -35,7 +34,7 @@ def setUp(self): self.inputs = {'X': self.x} self.attrs = { 'offset': self.offset, - 'padding_value': self.padding_value + 'padding_value': self.padding_value, } self.outputs = {'Out': self.out} @@ -52,38 +51,36 @@ def init_config(self): class TestDiagV2OpCase1(TestDiagV2Op): - def init_config(self): self.offset = 1 self.out = np.diag(self.x, self.offset) class TestDiagV2OpCase2(TestDiagV2Op): - def init_config(self): self.offset = -1 self.out = np.diag(self.x, self.offset) class TestDiagV2OpCase3(TestDiagV2Op): - def init_config(self): self.x = np.random.randint(-10, 10, size=(10, 10)).astype("float64") self.out = np.diag(self.x, self.offset) class TestDiagV2OpCase4(TestDiagV2Op): - def init_config(self): self.x = np.random.rand(100) self.padding_value = 2 n = self.x.size - self.out = self.padding_value * np.ones((n, n)) + np.diag( - self.x, self.offset) - np.diag(self.padding_value * np.ones(n)) + self.out = ( + self.padding_value * np.ones((n, n)) + + np.diag(self.x, self.offset) + - np.diag(self.padding_value * np.ones(n)) + ) class TestDiagV2Error(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -104,7 +101,6 @@ def test_diag_v2_type(): class TestDiagV2API(unittest.TestCase): - def setUp(self): self.input_np = np.random.random(size=(10, 10)).astype(np.float32) self.expected0 = np.diag(self.input_np) @@ -115,21 +111,27 @@ def setUp(self): self.offset = 0 self.padding_value = 8 n = self.input_np2.size - self.expected3 = self.padding_value * np.ones( - (n, n)) + np.diag(self.input_np2, self.offset) - np.diag( - self.padding_value * np.ones(n)) + self.expected3 = ( + self.padding_value * np.ones((n, n)) + + np.diag(self.input_np2, self.offset) + - np.diag(self.padding_value * np.ones(n)) + ) self.input_np3 = np.random.randint(-10, 10, size=(100)).astype(np.int64) self.padding_value = 8.0 n = self.input_np3.size - self.expected4 = self.padding_value * np.ones( - (n, n)) + np.diag(self.input_np3, self.offset) - np.diag( - self.padding_value * np.ones(n)) + self.expected4 = ( + self.padding_value * np.ones((n, n)) + + np.diag(self.input_np3, self.offset) + - np.diag(self.padding_value * np.ones(n)) + ) self.padding_value = -8 - self.expected5 = self.padding_value * np.ones( - (n, n)) + np.diag(self.input_np3, self.offset) - np.diag( - self.padding_value * np.ones(n)) + self.expected5 = ( + self.padding_value * np.ones((n, n)) + + np.diag(self.input_np3, self.offset) + - np.diag(self.padding_value * np.ones(n)) + ) self.input_np4 = np.random.random(size=(2000, 2000)).astype(np.float32) self.expected6 = np.diag(self.input_np4) @@ -194,13 +196,13 @@ def run_static(self, use_gpu=False): x = paddle.static.data(name='input', shape=[10, 10], dtype='float32') x2 = paddle.static.data(name='input2', shape=[100], dtype='float64') x3 = paddle.static.data(name='input3', shape=[100], dtype='int64') - x4 = paddle.static.data(name='input4', - shape=[2000, 2000], - dtype='float32') + x4 = paddle.static.data( + name='input4', shape=[2000, 2000], dtype='float32' + ) x5 = paddle.static.data(name='input5', shape=[2000], dtype='float32') - x6 = paddle.static.data(name='input6', - shape=[2000, 1500], - dtype='float32') + x6 = paddle.static.data( + name='input6', shape=[2000, 1500], dtype='float32' + ) result0 = paddle.diag(x) result1 = paddle.diag(x, offset=1) result2 = paddle.diag(x, offset=-1) @@ -219,19 +221,45 @@ def run_static(self, use_gpu=False): place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - res0, res1, res2, res4, res5, res6, res7, res8, res9, res10, res11, res12, res13 = exe.run( + ( + res0, + res1, + res2, + res4, + res5, + res6, + res7, + res8, + res9, + res10, + res11, + res12, + res13, + ) = exe.run( feed={ "input": self.input_np, "input2": self.input_np2, 'input3': self.input_np3, 'input4': self.input_np4, 'input5': self.input_np5, - 'input6': self.input_np6 + 'input6': self.input_np6, }, fetch_list=[ - result0, result1, result2, result4, result5, result6, result7, - result8, result9, result10, result11, result12, result13 - ]) + result0, + result1, + result2, + result4, + result5, + result6, + result7, + result8, + result9, + result10, + result11, + result12, + result13, + ], + ) np.testing.assert_allclose(res0, self.expected0, rtol=1e-05) np.testing.assert_allclose(res1, self.expected1, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_diagflat.py b/python/paddle/fluid/tests/unittests/test_diagflat.py index ec6bf98299188..f23c5051ddbbc 100644 --- a/python/paddle/fluid/tests/unittests/test_diagflat.py +++ b/python/paddle/fluid/tests/unittests/test_diagflat.py @@ -19,7 +19,6 @@ class TestDiagFlatError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -35,7 +34,6 @@ def test_diagflat_type(): class TestDiagFlatAPI(unittest.TestCase): - def setUp(self): self.input_np = np.random.random(size=(10, 10)).astype(np.float64) self.expected0 = np.diagflat(self.input_np) @@ -77,11 +75,10 @@ def run_static(self, use_gpu=False): place = paddle.CUDAPlace(0) if use_gpu else paddle.CPUPlace() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) - res0, res3 = exe.run(feed={ - "input": self.input_np, - 'input2': self.input_np2 - }, - fetch_list=[result0, result3]) + res0, res3 = exe.run( + feed={"input": self.input_np, 'input2': self.input_np2}, + fetch_list=[result0, result3], + ) np.testing.assert_allclose(res0, self.expected0, rtol=1e-05) np.testing.assert_allclose(res3, self.expected3, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_diagonal_op.py b/python/paddle/fluid/tests/unittests/test_diagonal_op.py index c7976856bd667..bdc64827fd2a8 100644 --- a/python/paddle/fluid/tests/unittests/test_diagonal_op.py +++ b/python/paddle/fluid/tests/unittests/test_diagonal_op.py @@ -22,7 +22,6 @@ class TestDiagonalOp(OpTest): - def setUp(self): self.op_type = "diagonal" self.python_api = paddle.diagonal @@ -39,62 +38,68 @@ def init_config(self): self.case = np.random.randn(10, 5, 2).astype('float64') self.inputs = {'Input': self.case} self.attrs = {'offset': 0, 'axis1': 0, 'axis2': 1} - self.target = np.diagonal(self.inputs['Input'], - offset=self.attrs['offset'], - axis1=self.attrs['axis1'], - axis2=self.attrs['axis2']) + self.target = np.diagonal( + self.inputs['Input'], + offset=self.attrs['offset'], + axis1=self.attrs['axis1'], + axis2=self.attrs['axis2'], + ) class TestDiagonalOpCase1(TestDiagonalOp): - def init_config(self): self.case = np.random.randn(4, 2, 4, 4).astype('float32') self.inputs = {'Input': self.case} self.attrs = {'offset': -2, 'axis1': 3, 'axis2': 0} - self.target = np.diagonal(self.inputs['Input'], - offset=self.attrs['offset'], - axis1=self.attrs['axis1'], - axis2=self.attrs['axis2']) + self.target = np.diagonal( + self.inputs['Input'], + offset=self.attrs['offset'], + axis1=self.attrs['axis1'], + axis2=self.attrs['axis2'], + ) class TestDiagonalOpCase2(TestDiagonalOp): - def init_config(self): self.case = np.random.randn(100, 100).astype('int64') self.inputs = {'Input': self.case} self.attrs = {'offset': 0, 'axis1': 0, 'axis2': 1} - self.target = np.diagonal(self.inputs['Input'], - offset=self.attrs['offset'], - axis1=self.attrs['axis1'], - axis2=self.attrs['axis2']) + self.target = np.diagonal( + self.inputs['Input'], + offset=self.attrs['offset'], + axis1=self.attrs['axis1'], + axis2=self.attrs['axis2'], + ) self.grad_x = np.eye(100).astype('int64') self.grad_out = np.ones(100).astype('int64') def test_check_grad(self): - self.check_grad(['Input'], - 'Out', - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['Input'], + 'Out', + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) class TestDiagonalOpCase3(TestDiagonalOp): - def init_config(self): self.case = np.random.randint(0, 2, (4, 2, 4, 4)).astype('bool') self.inputs = {'Input': self.case} self.attrs = {'offset': -2, 'axis1': 3, 'axis2': 0} - self.target = np.diagonal(self.inputs['Input'], - offset=self.attrs['offset'], - axis1=self.attrs['axis1'], - axis2=self.attrs['axis2']) + self.target = np.diagonal( + self.inputs['Input'], + offset=self.attrs['offset'], + axis1=self.attrs['axis1'], + axis2=self.attrs['axis2'], + ) def test_check_grad(self): pass class TestDiagonalAPI(unittest.TestCase): - def setUp(self): self.shape = [10, 3, 4] self.x = np.random.random((10, 3, 4)).astype(np.float32) diff --git a/python/paddle/fluid/tests/unittests/test_diff_op.py b/python/paddle/fluid/tests/unittests/test_diff_op.py index 8111d1b57b3b0..4b3981075a18d 100644 --- a/python/paddle/fluid/tests/unittests/test_diff_op.py +++ b/python/paddle/fluid/tests/unittests/test_diff_op.py @@ -21,7 +21,6 @@ class TestDiffOp(unittest.TestCase): - def set_args(self): self.input = np.array([1, 4, 5, 2]).astype('float32') self.n = 1 @@ -31,21 +30,21 @@ def set_args(self): def get_output(self): if self.prepend is not None and self.append is not None: - self.output = np.diff(self.input, - n=self.n, - axis=self.axis, - prepend=self.prepend, - append=self.append) + self.output = np.diff( + self.input, + n=self.n, + axis=self.axis, + prepend=self.prepend, + append=self.append, + ) elif self.prepend is not None: - self.output = np.diff(self.input, - n=self.n, - axis=self.axis, - prepend=self.prepend) + self.output = np.diff( + self.input, n=self.n, axis=self.axis, prepend=self.prepend + ) elif self.append is not None: - self.output = np.diff(self.input, - n=self.n, - axis=self.axis, - append=self.append) + self.output = np.diff( + self.input, n=self.n, axis=self.axis, append=self.append + ) else: self.output = np.diff(self.input, n=self.n, axis=self.axis) @@ -64,11 +63,13 @@ def func_dygraph(self): self.prepend = paddle.to_tensor(self.prepend, place=place) if self.append is not None: self.append = paddle.to_tensor(self.append, place=place) - out = paddle.diff(x, - n=self.n, - axis=self.axis, - prepend=self.prepend, - append=self.append) + out = paddle.diff( + x, + n=self.n, + axis=self.axis, + prepend=self.prepend, + append=self.append, + ) self.assertTrue((out.numpy() == self.output).all(), True) def test_dygraph(self): @@ -85,36 +86,40 @@ def test_static(self): places.append(fluid.CUDAPlace(0)) for place in places: with fluid.program_guard(fluid.Program(), fluid.Program()): - x = paddle.fluid.data(name="input", - shape=self.input.shape, - dtype=self.input.dtype) + x = paddle.fluid.data( + name="input", shape=self.input.shape, dtype=self.input.dtype + ) has_pend = False prepend = None append = None if self.prepend is not None: has_pend = True - prepend = paddle.fluid.data(name="prepend", - shape=self.prepend.shape, - dtype=self.prepend.dtype) + prepend = paddle.fluid.data( + name="prepend", + shape=self.prepend.shape, + dtype=self.prepend.dtype, + ) if self.append is not None: has_pend = True - append = paddle.fluid.data(name="append", - shape=self.append.shape, - dtype=self.append.dtype) + append = paddle.fluid.data( + name="append", + shape=self.append.shape, + dtype=self.append.dtype, + ) exe = fluid.Executor(place) - out = paddle.diff(x, - n=self.n, - axis=self.axis, - prepend=prepend, - append=append) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input": self.input, - "prepend": self.prepend, - "append": self.append - }, - fetch_list=[out]) + out = paddle.diff( + x, n=self.n, axis=self.axis, prepend=prepend, append=append + ) + fetches = exe.run( + fluid.default_main_program(), + feed={ + "input": self.input, + "prepend": self.prepend, + "append": self.append, + }, + fetch_list=[out], + ) self.assertTrue((fetches[0] == self.output).all(), True) def func_grad(self): @@ -124,11 +129,13 @@ def func_grad(self): self.prepend = paddle.to_tensor(self.prepend, place=place) if self.append is not None: self.append = paddle.to_tensor(self.append, place=place) - out = paddle.diff(x, - n=self.n, - axis=self.axis, - prepend=self.prepend, - append=self.append) + out = paddle.diff( + x, + n=self.n, + axis=self.axis, + prepend=self.prepend, + append=self.append, + ) try: out.backward() x_grad = x.grad @@ -144,7 +151,6 @@ def test_grad(self): class TestDiffOpAxis(TestDiffOp): - def set_args(self): self.input = np.array([[1, 4, 5, 2], [1, 5, 4, 2]]).astype('float32') self.n = 1 @@ -154,7 +160,6 @@ def set_args(self): class TestDiffOpNDim(TestDiffOp): - def set_args(self): self.input = np.random.rand(10, 10).astype('float32') self.n = 1 @@ -164,7 +169,6 @@ def set_args(self): class TestDiffOpBool(TestDiffOp): - def set_args(self): self.input = np.array([0, 1, 1, 0, 1, 0]).astype('bool') self.n = 1 @@ -174,7 +178,6 @@ def set_args(self): class TestDiffOpPrepend(TestDiffOp): - def set_args(self): self.input = np.array([[1, 4, 5, 2], [1, 5, 4, 2]]).astype('float32') self.n = 1 @@ -184,18 +187,17 @@ def set_args(self): class TestDiffOpPrependAxis(TestDiffOp): - def set_args(self): self.input = np.array([[1, 4, 5, 2], [1, 5, 4, 2]]).astype('float32') self.n = 1 self.axis = 0 - self.prepend = np.array([[0, 2, 3, 4], [1, 3, 5, 7], - [2, 5, 8, 0]]).astype('float32') + self.prepend = np.array( + [[0, 2, 3, 4], [1, 3, 5, 7], [2, 5, 8, 0]] + ).astype('float32') self.append = None class TestDiffOpAppend(TestDiffOp): - def set_args(self): self.input = np.array([[1, 4, 5, 2], [1, 5, 4, 2]]).astype('float32') self.n = 1 @@ -205,7 +207,6 @@ def set_args(self): class TestDiffOpAppendAxis(TestDiffOp): - def set_args(self): self.input = np.array([[1, 4, 5, 2], [1, 5, 4, 2]]).astype('float32') self.n = 1 @@ -215,7 +216,6 @@ def set_args(self): class TestDiffOpPreAppend(TestDiffOp): - def set_args(self): self.input = np.array([[1, 4, 5, 2], [1, 5, 4, 2]]).astype('float32') self.n = 1 @@ -225,7 +225,6 @@ def set_args(self): class TestDiffOpPreAppendAxis(TestDiffOp): - def set_args(self): self.input = np.array([[1, 4, 5, 2], [1, 5, 4, 2]]).astype('float32') self.n = 1 diff --git a/python/paddle/fluid/tests/unittests/test_digamma_op.py b/python/paddle/fluid/tests/unittests/test_digamma_op.py index 546744485b084..ff9e2b182759a 100644 --- a/python/paddle/fluid/tests/unittests/test_digamma_op.py +++ b/python/paddle/fluid/tests/unittests/test_digamma_op.py @@ -23,7 +23,6 @@ class TestDigammaOp(OpTest): - def setUp(self): # switch to static paddle.enable_static() @@ -49,7 +48,6 @@ def test_check_grad_normal(self): class TestDigammaOpFp32(TestDigammaOp): - def init_dtype_type(self): self.dtype = np.float32 @@ -58,7 +56,6 @@ def test_check_grad_normal(self): class TestDigammaAPI(unittest.TestCase): - def setUp(self): # switch to static paddle.enable_static() @@ -70,7 +67,6 @@ def setUp(self): self._shape = [8, 3, 32, 32] def test_in_static_mode(self): - def init_input_output(dtype): input = np.random.random(self._shape).astype(dtype) return {'x': input}, psi(input) diff --git a/python/paddle/fluid/tests/unittests/test_directory_migration.py b/python/paddle/fluid/tests/unittests/test_directory_migration.py index 46a2c53a73112..408ad42379e62 100644 --- a/python/paddle/fluid/tests/unittests/test_directory_migration.py +++ b/python/paddle/fluid/tests/unittests/test_directory_migration.py @@ -20,7 +20,6 @@ class TestDirectory(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -38,14 +37,25 @@ def get_import_command(self, module): def test_new_directory(self): new_directory = [ - 'paddle.enable_static', 'paddle.disable_static', - 'paddle.in_dynamic_mode', 'paddle.to_tensor', 'paddle.grad', - 'paddle.no_grad', 'paddle.static.save', 'paddle.static.load', - 'paddle.distributed.ParallelEnv', 'paddle.DataParallel', - 'paddle.jit', 'paddle.jit.TracedLayer', 'paddle.jit.to_static', - 'paddle.jit.ProgramTranslator', 'paddle.jit.TranslatedLayer', - 'paddle.jit.save', 'paddle.jit.load', - 'paddle.optimizer.lr.LRScheduler', 'paddle.optimizer.lr.NoamDecay', + 'paddle.enable_static', + 'paddle.disable_static', + 'paddle.in_dynamic_mode', + 'paddle.to_tensor', + 'paddle.grad', + 'paddle.no_grad', + 'paddle.static.save', + 'paddle.static.load', + 'paddle.distributed.ParallelEnv', + 'paddle.DataParallel', + 'paddle.jit', + 'paddle.jit.TracedLayer', + 'paddle.jit.to_static', + 'paddle.jit.ProgramTranslator', + 'paddle.jit.TranslatedLayer', + 'paddle.jit.save', + 'paddle.jit.load', + 'paddle.optimizer.lr.LRScheduler', + 'paddle.optimizer.lr.NoamDecay', 'paddle.optimizer.lr.PiecewiseDecay', 'paddle.optimizer.lr.NaturalExpDecay', 'paddle.optimizer.lr.ExponentialDecay', @@ -53,30 +63,47 @@ def test_new_directory(self): 'paddle.optimizer.lr.PolynomialDecay', 'paddle.optimizer.lr.CosineAnnealingDecay', 'paddle.optimizer.lr.MultiStepDecay', - 'paddle.optimizer.lr.StepDecay', 'paddle.optimizer.lr.LambdaDecay', + 'paddle.optimizer.lr.StepDecay', + 'paddle.optimizer.lr.LambdaDecay', 'paddle.optimizer.lr.ReduceOnPlateau', - 'paddle.optimizer.lr.LinearWarmup', 'paddle.static.Executor', - 'paddle.static.global_scope', 'paddle.static.scope_guard', - 'paddle.static.append_backward', 'paddle.static.gradients', - 'paddle.static.BuildStrategy', 'paddle.static.CompiledProgram', + 'paddle.optimizer.lr.LinearWarmup', + 'paddle.static.Executor', + 'paddle.static.global_scope', + 'paddle.static.scope_guard', + 'paddle.static.append_backward', + 'paddle.static.gradients', + 'paddle.static.BuildStrategy', + 'paddle.static.CompiledProgram', 'paddle.static.ExecutionStrategy', 'paddle.static.default_main_program', - 'paddle.static.default_startup_program', 'paddle.static.Program', - 'paddle.static.name_scope', 'paddle.static.program_guard', - 'paddle.static.Print', 'paddle.static.py_func', + 'paddle.static.default_startup_program', + 'paddle.static.Program', + 'paddle.static.name_scope', + 'paddle.static.program_guard', + 'paddle.static.Print', + 'paddle.static.py_func', 'paddle.static.ParallelExecutor', - 'paddle.static.WeightNormParamAttr', 'paddle.static.nn.fc', + 'paddle.static.WeightNormParamAttr', + 'paddle.static.nn.fc', 'paddle.static.nn.batch_norm', 'paddle.static.nn.bilinear_tensor_product', - 'paddle.static.nn.conv2d', 'paddle.static.nn.conv2d_transpose', - 'paddle.static.nn.conv3d', 'paddle.static.nn.conv3d_transpose', + 'paddle.static.nn.conv2d', + 'paddle.static.nn.conv2d_transpose', + 'paddle.static.nn.conv3d', + 'paddle.static.nn.conv3d_transpose', 'paddle.static.nn.create_parameter', - 'paddle.static.nn.crf_decoding', 'paddle.static.nn.data_norm', - 'paddle.static.nn.deform_conv2d', 'paddle.static.nn.group_norm', - 'paddle.static.nn.instance_norm', 'paddle.static.nn.layer_norm', - 'paddle.static.nn.multi_box_head', 'paddle.static.nn.nce', - 'paddle.static.nn.prelu', 'paddle.static.nn.row_conv', - 'paddle.static.nn.spectral_norm', 'paddle.static.nn.embedding' + 'paddle.static.nn.crf_decoding', + 'paddle.static.nn.data_norm', + 'paddle.static.nn.deform_conv2d', + 'paddle.static.nn.group_norm', + 'paddle.static.nn.instance_norm', + 'paddle.static.nn.layer_norm', + 'paddle.static.nn.multi_box_head', + 'paddle.static.nn.nce', + 'paddle.static.nn.prelu', + 'paddle.static.nn.row_conv', + 'paddle.static.nn.spectral_norm', + 'paddle.static.nn.embedding', ] import_file = os.path.join(self.temp_dir.name, 'run_import_modules.py') @@ -89,58 +116,89 @@ def test_new_directory(self): _python = sys.executable ps_cmd = "{} {}".format(_python, import_file) - ps_proc = subprocess.Popen(ps_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ps_proc = subprocess.Popen( + ps_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) stdout, stderr = ps_proc.communicate() - self.assertFalse("Error" in str(stderr), - "ErrorMessage:\n{}".format(bytes.decode(stderr))) + self.assertFalse( + "Error" in str(stderr), + "ErrorMessage:\n{}".format(bytes.decode(stderr)), + ) def test_old_directory(self): old_directory = [ - 'paddle.enable_imperative', 'paddle.disable_imperative', - 'paddle.in_imperative_mode', 'paddle.imperative.to_variable', - 'paddle.imperative.enable', 'paddle.imperative.guard', - 'paddle.imperative.grad', 'paddle.imperative.no_grad', - 'paddle.imperative.save', 'paddle.imperative.load', + 'paddle.enable_imperative', + 'paddle.disable_imperative', + 'paddle.in_imperative_mode', + 'paddle.imperative.to_variable', + 'paddle.imperative.enable', + 'paddle.imperative.guard', + 'paddle.imperative.grad', + 'paddle.imperative.no_grad', + 'paddle.imperative.save', + 'paddle.imperative.load', 'paddle.imperative.ParallelEnv', 'paddle.imperative.prepare_context', - 'paddle.imperative.DataParalell', 'paddle.imperative.jit', - 'paddle.imperative.TracedLayer', 'paddle.imperative.declarative', + 'paddle.imperative.DataParalell', + 'paddle.imperative.jit', + 'paddle.imperative.TracedLayer', + 'paddle.imperative.declarative', 'paddle.imperative.ProgramTranslator', - 'paddle.imperative.TranslatedLayer', 'paddle.imperative.jit.save', - 'paddle.imperative.jit.load', 'paddle.imperative.NoamDecay' - 'paddle.imperative.PiecewiseDecay', + 'paddle.imperative.TranslatedLayer', + 'paddle.imperative.jit.save', + 'paddle.imperative.jit.load', + 'paddle.imperative.NoamDecay' 'paddle.imperative.PiecewiseDecay', 'paddle.imperative.NaturalExpDecay', 'paddle.imperative.ExponentialDecay', 'paddle.imperative.InverseTimeDecay', 'paddle.imperative.PolynomialDecay', - 'paddle.imperative.CosineDecay', 'paddle.Executor', - 'paddle.global_scope', 'paddle.scope_guard', - 'paddle.append_backward', 'paddle.gradients', - 'paddle.BuildStrategy', 'paddle.CompiledProgram', - 'paddle.ExecutionStrategy', 'paddle.name_scope', - 'paddle.program_guard', 'paddle.Print', 'paddle.py_func', - 'paddle.ParallelExecutor', 'paddle.default_main_program', - 'paddle.default_startup_program', 'paddle.Program', - 'paddle.WeightNormParamAttr', 'paddle.declarative.fc', + 'paddle.imperative.CosineDecay', + 'paddle.Executor', + 'paddle.global_scope', + 'paddle.scope_guard', + 'paddle.append_backward', + 'paddle.gradients', + 'paddle.BuildStrategy', + 'paddle.CompiledProgram', + 'paddle.ExecutionStrategy', + 'paddle.name_scope', + 'paddle.program_guard', + 'paddle.Print', + 'paddle.py_func', + 'paddle.ParallelExecutor', + 'paddle.default_main_program', + 'paddle.default_startup_program', + 'paddle.Program', + 'paddle.WeightNormParamAttr', + 'paddle.declarative.fc', 'paddle.declarative.batch_norm', 'paddle.declarative.bilinear_tensor_product', - 'paddle.declarative.conv2d', 'paddle.declarative.conv2d_transpose', - 'paddle.declarative.conv3d', 'paddle.declarative.conv3d_transpose', + 'paddle.declarative.conv2d', + 'paddle.declarative.conv2d_transpose', + 'paddle.declarative.conv3d', + 'paddle.declarative.conv3d_transpose', 'paddle.declarative.create_parameter', - 'paddle.declarative.crf_decoding', 'paddle.declarative.data_norm', + 'paddle.declarative.crf_decoding', + 'paddle.declarative.data_norm', 'paddle.declarative.deformable_conv', - 'paddle.declarative.group_norm', 'paddle.declarative.hsigmoid', - 'paddle.declarative.instance_norm', 'paddle.declarative.layer_norm', - 'paddle.declarative.multi_box_head', 'paddle.declarative.nce', - 'paddle.declarative.prelu', 'paddle.declarative.row_conv', - 'paddle.declarative.spectral_norm', 'paddle.declarative.embedding' + 'paddle.declarative.group_norm', + 'paddle.declarative.hsigmoid', + 'paddle.declarative.instance_norm', + 'paddle.declarative.layer_norm', + 'paddle.declarative.multi_box_head', + 'paddle.declarative.nce', + 'paddle.declarative.prelu', + 'paddle.declarative.row_conv', + 'paddle.declarative.spectral_norm', + 'paddle.declarative.embedding', ] - import_file = os.path.join(self.temp_dir.name, - 'run_old_import_modules.py') + import_file = os.path.join( + self.temp_dir.name, 'run_old_import_modules.py' + ) with open(import_file, "w") as wb: cmd_context_count = """ @@ -159,22 +217,26 @@ def test_old_directory(self): err_module = "{module}" """ cmd_context_loop = cmd_context_loop_template.format( - run_cmd=run_cmd, module=module) + run_cmd=run_cmd, module=module + ) wb.write(cmd_context_loop) cmd_context_print_template = """ if count != {len_old_directory}: print("Error: Module " + err_module + " should not be imported") """ cmd_context_print = cmd_context_print_template.format( - len_old_directory=str(len(old_directory))) + len_old_directory=str(len(old_directory)) + ) wb.write(cmd_context_print) _python = sys.executable ps_cmd = "{} {}".format(_python, import_file) - ps_proc = subprocess.Popen(ps_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ps_proc = subprocess.Popen( + ps_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) stdout, stderr = ps_proc.communicate() self.assertFalse("Error" in str(stdout), bytes.decode(stdout)) diff --git a/python/paddle/fluid/tests/unittests/test_disable_signal_handler.py b/python/paddle/fluid/tests/unittests/test_disable_signal_handler.py index 50867ad8b0e03..f0de021eaadf0 100644 --- a/python/paddle/fluid/tests/unittests/test_disable_signal_handler.py +++ b/python/paddle/fluid/tests/unittests/test_disable_signal_handler.py @@ -17,21 +17,27 @@ import subprocess SignalsToTest = { - signal.SIGTERM, signal.SIGBUS, signal.SIGABRT, signal.SIGSEGV, - signal.SIGILL, signal.SIGFPE + signal.SIGTERM, + signal.SIGBUS, + signal.SIGABRT, + signal.SIGSEGV, + signal.SIGILL, + signal.SIGFPE, } class TestSignOpError(unittest.TestCase): - def test_errors(self): try: for sig in SignalsToTest: - output = subprocess.check_output([ - "python", "-c", - f"import paddle; import signal,os; paddle.disable_signal_handler(); os.kill(os.getpid(), {sig})" - ], - stderr=subprocess.STDOUT) + output = subprocess.check_output( + [ + "python", + "-c", + f"import paddle; import signal,os; paddle.disable_signal_handler(); os.kill(os.getpid(), {sig})", + ], + stderr=subprocess.STDOUT, + ) except Exception as e: # If paddle signal handler is enabled # One would expect "paddle::framework::SignalHandle" in STDERR diff --git a/python/paddle/fluid/tests/unittests/test_dist_allreduce_op.py b/python/paddle/fluid/tests/unittests/test_dist_allreduce_op.py index af2c8f1aee710..c2469ab92b7e5 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_allreduce_op.py +++ b/python/paddle/fluid/tests/unittests/test_dist_allreduce_op.py @@ -20,7 +20,6 @@ class TestDistMnistNCCL2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -30,6 +29,7 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_allreduce_op.py", delta=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_dist_base.py b/python/paddle/fluid/tests/unittests/test_dist_base.py index d528727076954..4c109feaef235 100755 --- a/python/paddle/fluid/tests/unittests/test_dist_base.py +++ b/python/paddle/fluid/tests/unittests/test_dist_base.py @@ -53,26 +53,30 @@ def eprint(*args, **kwargs): class TestDistRunnerBase(object): - - def get_model(self, - batch_size=DEFAULT_BATCH_SIZE, - lr=0.1, - single_device=False, - use_dgc=False, - dist_strategy=None): + def get_model( + self, + batch_size=DEFAULT_BATCH_SIZE, + lr=0.1, + single_device=False, + use_dgc=False, + dist_strategy=None, + ): raise NotImplementedError( - "get_model should be implemented by child classes.") + "get_model should be implemented by child classes." + ) @staticmethod - def get_transpiler(trainer_id, - main_program, - pserver_endpoints, - trainers, - sync_mode, - dc_asgd=False, - current_endpoint=None, - nccl_comm_num=1, - hogwild_mode=False): + def get_transpiler( + trainer_id, + main_program, + pserver_endpoints, + trainers, + sync_mode, + dc_asgd=False, + current_endpoint=None, + nccl_comm_num=1, + hogwild_mode=False, + ): # NOTE: import fluid until runtime, or else forking processes will cause error. config = fluid.DistributeTranspilerConfig() config.enable_dc_asgd = dc_asgd @@ -83,12 +87,14 @@ def get_transpiler(trainer_id, config.nccl_comm_num = nccl_comm_num # config.runtime_split_send_recv = True t = fluid.DistributeTranspiler(config=config) - t.transpile(trainer_id=trainer_id, - program=main_program, - pservers=pserver_endpoints, - trainers=trainers, - sync_mode=sync_mode, - current_endpoint=current_endpoint) + t.transpile( + trainer_id=trainer_id, + program=main_program, + pservers=pserver_endpoints, + trainers=trainers, + sync_mode=sync_mode, + current_endpoint=current_endpoint, + ) return t @staticmethod @@ -96,6 +102,7 @@ def get_lr_scheduler(program): lr_sheduler = None if hasattr(program, 'lr_sheduler'): from paddle.optimizer.lr import LRScheduler + lr_sheduler = program.lr_sheduler assert isinstance(lr_sheduler, LRScheduler), "must be LRScheduler" return lr_sheduler @@ -105,16 +112,19 @@ def run_pserver(self, args): self.get_model(batch_size=args.batch_size) # NOTE: pserver should not call memory optimize - t = self.get_transpiler(trainer_id=args.trainer_id, - main_program=fluid.default_main_program(), - pserver_endpoints=args.endpoints, - trainers=args.trainers, - sync_mode=args.sync_mode, - dc_asgd=args.dc_asgd, - hogwild_mode=args.hogwild) + t = self.get_transpiler( + trainer_id=args.trainer_id, + main_program=fluid.default_main_program(), + pserver_endpoints=args.endpoints, + trainers=args.trainers, + sync_mode=args.sync_mode, + dc_asgd=args.dc_asgd, + hogwild_mode=args.hogwild, + ) pserver_prog = t.get_pserver_program(args.current_endpoint) - startup_prog = t.get_startup_program(args.current_endpoint, - pserver_prog) + startup_prog = t.get_startup_program( + args.current_endpoint, pserver_prog + ) place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -127,8 +137,17 @@ def run_pipeline_trainer(self, args): self.lr = args.lr dist_strategy = DistributedStrategy() - test_program, avg_cost, train_reader, test_reader, batch_acc, predict, data_loader = \ - self.get_model(batch_size=args.batch_size, dist_strategy=dist_strategy) + ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + data_loader, + ) = self.get_model( + batch_size=args.batch_size, dist_strategy=dist_strategy + ) device_id = int(os.getenv("FLAGS_selected_gpus", "0")) eprint(type(self).__name__, "device_id: %d." % device_id) @@ -169,8 +188,14 @@ def run_use_fleet_api_20_trainer(self, args): self.lr = args.lr print_to_err("use_fleet 2.0", "fleet.node_num:") - test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \ - self.get_model(batch_size=args.batch_size) + ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) = self.get_model(batch_size=args.batch_size) if fluid.core.is_compiled_with_cuda(): device_id = int(os.getenv("FLAGS_selected_gpus", "0")) @@ -188,8 +213,8 @@ def run_use_fleet_api_20_trainer(self, args): eprint(type(self).__name__, "run worker startup program done.") feed_var_list = [ - var for var in - fluid.default_main_program().global_block().vars.values() + var + for var in fluid.default_main_program().global_block().vars.values() if var.is_data ] @@ -203,8 +228,10 @@ def run_use_fleet_api_20_trainer(self, args): def get_data(): origin_batch = next(reader_generator) - if paddle.distributed.get_world_size( - ) == 1 and args.update_method == 'gloo': # Gloo single mode + if ( + paddle.distributed.get_world_size() == 1 + and args.update_method == 'gloo' + ): # Gloo single mode return origin_batch elif args.update_method != "local" and args.use_reader_alloc: @@ -219,9 +246,11 @@ def get_data(): print_to_err(type(self).__name__, "begin to train on trainer") out_losses = [] for i in range(RUN_STEP): - loss, = exe.run(fluid.default_main_program(), - fetch_list=[avg_cost.name], - feed=feeder.feed(get_data())) + (loss,) = exe.run( + fluid.default_main_program(), + fetch_list=[avg_cost.name], + feed=feeder.feed(get_data()), + ) out_losses.append(loss[0]) print_to_err(type(self).__name__, "run step %d finished" % i) print_to_err(type(self).__name__, "trainer run finished") @@ -254,8 +283,16 @@ def run_use_fleet_api_trainer(self, args): # "fleet.node_id:", fleet.node_id(), # "fleet.trainer_num:", fleet.worker_num()) - test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \ - self.get_model(batch_size=args.batch_size, dist_strategy=dist_strategy) + ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) = self.get_model( + batch_size=args.batch_size, dist_strategy=dist_strategy + ) trainer_prog = fleet._origin_program dist_prog = fleet.main_program @@ -276,7 +313,8 @@ def run_use_fleet_api_trainer(self, args): eprint(type(self).__name__, "run worker startup program done.") feed_var_list = [ - var for var in trainer_prog.global_block().vars.values() + var + for var in trainer_prog.global_block().vars.values() if var.is_data ] @@ -304,9 +342,11 @@ def get_data(): print_to_err(type(self).__name__, "begin to train on trainer") out_losses = [] for i in range(RUN_STEP): - loss, = exe.run(dist_prog, - fetch_list=[avg_cost.name], - feed=feeder.feed(get_data())) + (loss,) = exe.run( + dist_prog, + fetch_list=[avg_cost.name], + feed=feeder.feed(get_data()), + ) out_losses.append(loss[0]) print_to_err(type(self).__name__, "run step %d finished" % i) print_to_err(type(self).__name__, "trainer run finished") @@ -316,86 +356,131 @@ def get_data(): if args.save_model: model_save_dir = "/tmp" if fleet.worker_index() == 0: - model_save_dir_fluid = os.path.join(model_save_dir, - "fluid_persistables") - model_save_dir_fleet = os.path.join(model_save_dir, - "fleet_persistables") - infer_save_dir_fluid = os.path.join(model_save_dir, - "fluid_infer") - infer_save_dir_fleet = os.path.join(model_save_dir, - "fleet_infer") + model_save_dir_fluid = os.path.join( + model_save_dir, "fluid_persistables" + ) + model_save_dir_fleet = os.path.join( + model_save_dir, "fleet_persistables" + ) + infer_save_dir_fluid = os.path.join( + model_save_dir, "fluid_infer" + ) + infer_save_dir_fleet = os.path.join( + model_save_dir, "fleet_infer" + ) else: - model_save_dir_fluid = os.path.join(model_save_dir, - "fluid_persistables_2") - model_save_dir_fleet = os.path.join(model_save_dir, - "fleet_persistables_2") - infer_save_dir_fluid = os.path.join(model_save_dir, - "fluid_infer_2") - infer_save_dir_fleet = os.path.join(model_save_dir, - "fleet_infer_2") - fluid.io.save_persistables(exe, model_save_dir_fluid, - fleet._origin_program) + model_save_dir_fluid = os.path.join( + model_save_dir, "fluid_persistables_2" + ) + model_save_dir_fleet = os.path.join( + model_save_dir, "fleet_persistables_2" + ) + infer_save_dir_fluid = os.path.join( + model_save_dir, "fluid_infer_2" + ) + infer_save_dir_fleet = os.path.join( + model_save_dir, "fleet_infer_2" + ) + fluid.io.save_persistables( + exe, model_save_dir_fluid, fleet._origin_program + ) fleet.save_persistables(executor=exe, dirname=model_save_dir_fleet) feeded_var_names = [var.name for var in feed_var_list] - fluid.io.save_inference_model(infer_save_dir_fluid, - feeded_var_names, [avg_cost], exe, - fleet._origin_program) - fleet.save_inference_model(exe, infer_save_dir_fleet, - feeded_var_names, [avg_cost]) + fluid.io.save_inference_model( + infer_save_dir_fluid, + feeded_var_names, + [avg_cost], + exe, + fleet._origin_program, + ) + fleet.save_inference_model( + exe, infer_save_dir_fleet, feeded_var_names, [avg_cost] + ) def run_trainer(self, args): self.lr = args.lr if args.nccl2_reduce_layer_local_run: - test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \ - self.get_model(batch_size=args.batch_size, single_device=True) + ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) = self.get_model(batch_size=args.batch_size, single_device=True) elif args.use_dgc: - test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \ - self.get_model(batch_size=args.batch_size, use_dgc=args.use_dgc) + ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) = self.get_model(batch_size=args.batch_size, use_dgc=args.use_dgc) else: - test_program, avg_cost, train_reader, test_reader, batch_acc, predict = \ - self.get_model(batch_size=args.batch_size) + ( + test_program, + avg_cost, + train_reader, + test_reader, + batch_acc, + predict, + ) = self.get_model(batch_size=args.batch_size) if args.update_method == "pserver": print_to_err( type(self).__name__, - "begin to run transpile on trainer with pserver mode") - t = self.get_transpiler(trainer_id=args.trainer_id, - main_program=fluid.default_main_program(), - pserver_endpoints=args.endpoints, - trainers=args.trainers, - sync_mode=args.sync_mode, - dc_asgd=args.dc_asgd, - hogwild_mode=args.hogwild) + "begin to run transpile on trainer with pserver mode", + ) + t = self.get_transpiler( + trainer_id=args.trainer_id, + main_program=fluid.default_main_program(), + pserver_endpoints=args.endpoints, + trainers=args.trainers, + sync_mode=args.sync_mode, + dc_asgd=args.dc_asgd, + hogwild_mode=args.hogwild, + ) trainer_prog = t.get_trainer_program() print_to_err( type(self).__name__, - "get trainer program done with pserver mode.") - elif args.update_method == "nccl2" or args.update_method == "nccl2_reduce_layer": + "get trainer program done with pserver mode.", + ) + elif ( + args.update_method == "nccl2" + or args.update_method == "nccl2_reduce_layer" + ): # transpile for nccl2 config = fluid.DistributeTranspilerConfig() config.mode = "nccl2" config.nccl_comm_num = args.nccl_comm_num if args.use_hallreduce: config.use_hierarchical_allreduce = True - config.hierarchical_allreduce_inter_nranks = args.hallreduce_inter_nranks + config.hierarchical_allreduce_inter_nranks = ( + args.hallreduce_inter_nranks + ) print_to_err( type(self).__name__, - "begin to run transpile on trainer with nccl2 mode") + "begin to run transpile on trainer with nccl2 mode", + ) nccl2_t = fluid.DistributeTranspiler(config=config) - nccl2_t.transpile(args.trainer_id, - program=fluid.default_main_program(), - startup_program=fluid.default_startup_program(), - trainers=args.endpoints, - current_endpoint=args.current_endpoint) + nccl2_t.transpile( + args.trainer_id, + program=fluid.default_main_program(), + startup_program=fluid.default_startup_program(), + trainers=args.endpoints, + current_endpoint=args.current_endpoint, + ) print_to_err( - type(self).__name__, - "get trainer program done. with nccl2 mode") + type(self).__name__, "get trainer program done. with nccl2 mode" + ) trainer_prog = fluid.default_main_program() else: print_to_err( type(self).__name__, - "do nothing about main program, just use it") + "do nothing about main program, just use it", + ) trainer_prog = fluid.default_main_program() print_to_err(type(self).__name__, "use main program done.") @@ -431,9 +516,13 @@ def run_trainer(self, args): build_stra.enable_backward_optimizer_op_deps = True if args.use_reduce: - build_stra.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce + build_stra.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + ) else: - build_stra.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.AllReduce + build_stra.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.AllReduce + ) pass_builder = None if args.batch_merge_repeat > 1: @@ -441,7 +530,10 @@ def run_trainer(self, args): mypass = pass_builder.insert_pass(0, "multi_batch_merge_pass") mypass.set("num_repeats", args.batch_merge_repeat) - if args.update_method == "nccl2" or args.update_method == "nccl2_reduce_layer": + if ( + args.update_method == "nccl2" + or args.update_method == "nccl2_reduce_layer" + ): build_stra.num_trainers = len(args.endpoints.split(",")) build_stra.trainer_id = args.trainer_id else: @@ -453,11 +545,13 @@ def run_trainer(self, args): binary = compiler.CompiledProgram(trainer_prog).with_data_parallel( loss_name=avg_cost.name, build_strategy=build_stra, - exec_strategy=exec_strategy) + exec_strategy=exec_strategy, + ) print_to_err(type(self).__name__, "program compiled with data parallel") feed_var_list = [ - var for var in trainer_prog.global_block().vars.values() + var + for var in trainer_prog.global_block().vars.values() if var.is_data ] @@ -479,9 +573,9 @@ def get_data(): print_to_err(type(self).__name__, "begin to train on trainer") out_losses = [] for i in range(RUN_STEP): - loss, = exe.run(binary, - fetch_list=[avg_cost.name], - feed=feeder.feed(get_data())) + (loss,) = exe.run( + binary, fetch_list=[avg_cost.name], feed=feeder.feed(get_data()) + ) out_losses.append(loss[0]) print_to_err(type(self).__name__, "run step %d finished" % i) if lr_scheduler is not None: @@ -493,18 +587,21 @@ def get_data(): class TestParallelDyGraphRunnerBase(object): - def get_model(self): raise NotImplementedError( - "get_model should be implemented by child classes.") + "get_model should be implemented by child classes." + ) def run_one_loop(self, model, opt, data): raise NotImplementedError( - "train_one_loop should be implemented by the child classes.") + "train_one_loop should be implemented by the child classes." + ) def _get_data(self, batch, args): - if paddle.distributed.get_world_size( - ) == 1 and args.update_method == 'gloo': # Gloo single mode + if ( + paddle.distributed.get_world_size() == 1 + and args.update_method == 'gloo' + ): # Gloo single mode return batch elif args.update_method != "local": new_batch = [] @@ -514,8 +611,9 @@ def _get_data(self, batch, args): # the second rank will get [3,4,5]. # this function is for test sparse_embedding_differ_length if hasattr(args, "diff_batch") and args.diff_batch: - assert len( - batch) > 2, "in differ_batch mode, len(batch) must > 2." + assert ( + len(batch) > 2 + ), "in differ_batch mode, len(batch) must > 2." if paddle.distributed.get_rank() == 0: new_batch.append(batch[0]) elif paddle.distributed.get_rank() == 1: @@ -550,19 +648,25 @@ def run_trainer(self, args): device_id = int(os.getenv("FLAGS_selected_mlus", "0")) place = fluid.MLUPlace(device_id) else: - assert ("Only support CUDAPlace or XPUPlace or CPU(Gloo) for now.") + assert "Only support CUDAPlace or XPUPlace or CPU(Gloo) for now." with fluid.dygraph.guard(place): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed np.random.seed(seed) import random + random.seed(seed) model, train_reader, opt = self.get_model() nranks = len(args.endpoints.split(",")) if args.endpoints else 1 - #if args.update_method == "nccl2": - if args.update_method == "nccl2" or args.update_method == "bkcl" or args.update_method == "hccl" or args.update_method == "cncl": + # if args.update_method == "nccl2": + if ( + args.update_method == "nccl2" + or args.update_method == "bkcl" + or args.update_method == "hccl" + or args.update_method == "cncl" + ): strategy = dygraph.parallel.ParallelStrategy() strategy.nranks = nranks strategy.local_rank = args.trainer_id @@ -571,24 +675,29 @@ def run_trainer(self, args): paddle.distributed.init_parallel_env() print_to_err( type(self).__name__, - "begin to prepare context in dygraph with nccl2") + "begin to prepare context in dygraph with nccl2", + ) dygraph.parallel.prepare_context(strategy) if not args.find_unused_parameters: model = dygraph.parallel.DataParallel( - model, strategy, find_unused_parameters=False) + model, strategy, find_unused_parameters=False + ) else: model = dygraph.parallel.DataParallel( - model, strategy, find_unused_parameters=True) + model, strategy, find_unused_parameters=True + ) print_to_err(type(self).__name__, "model built in dygraph") elif args.update_method == "gloo": paddle.distributed.init_parallel_env() if not args.find_unused_parameters: model = dygraph.parallel.DataParallel( - model, find_unused_parameters=False) + model, find_unused_parameters=False + ) else: model = dygraph.parallel.DataParallel( - model, find_unused_parameters=True) + model, find_unused_parameters=True + ) out_losses = [] print_to_err(type(self).__name__, "begin to run dygraph training") @@ -600,7 +709,8 @@ def run_trainer(self, args): if step_id % 10 == 0: print_to_err( type(self).__name__, - "loss at step %d: %f" % (step_id, loss.numpy())) + "loss at step %d: %f" % (step_id, loss.numpy()), + ) out_losses.append(loss.numpy()) loss.backward() @@ -632,7 +742,8 @@ def run_trainer_with_spawn(self, args): model, train_reader, opt = self.get_model() if args.update_method in ["nccl2", "gloo"]: model = paddle.DataParallel( - model, find_unused_parameters=args.find_unused_parameters) + model, find_unused_parameters=args.find_unused_parameters + ) out_losses = [] for step_id, data in enumerate(train_reader()): @@ -650,6 +761,7 @@ def run_trainer_with_spawn(self, args): def run_use_fleet_api_trainer(self, args): import paddle.distributed.fleet as fleet + # 1. enable dygraph paddle.disable_static() @@ -696,18 +808,25 @@ def run_use_fleet_api_trainer(self, args): def runtime_main(test_class): parser = argparse.ArgumentParser(description='Run dist test.') - parser.add_argument('--role', - type=str, - required=True, - choices=['pserver', 'trainer']) + parser.add_argument( + '--role', type=str, required=True, choices=['pserver', 'trainer'] + ) parser.add_argument('--endpoints', type=str, required=False, default="") - parser.add_argument('--update_method', - type=str, - default="local", - choices=[ - "pserver", "nccl2", "bkcl", "local", - "nccl2_reduce_layer", "gloo", "hccl", "cncl" - ]) + parser.add_argument( + '--update_method', + type=str, + default="local", + choices=[ + "pserver", + "nccl2", + "bkcl", + "local", + "nccl2_reduce_layer", + "gloo", + "hccl", + "cncl", + ], + ) parser.add_argument('--trainer_id', type=int, required=False, default=0) parser.add_argument('--trainers', type=int, required=False, default=1) parser.add_argument('--nccl_comm_num', type=int, required=False, default=1) @@ -719,14 +838,12 @@ def runtime_main(test_class): parser.add_argument('--use_local_sgd', action='store_true') parser.add_argument('--diff_batch', action='store_true') parser.add_argument('--ut4grad_allreduce', action='store_true') - parser.add_argument('--hallreduce_inter_nranks', - type=int, - required=False, - default=2) - parser.add_argument('--current_endpoint', - type=str, - required=False, - default="") + parser.add_argument( + '--hallreduce_inter_nranks', type=int, required=False, default=2 + ) + parser.add_argument( + '--current_endpoint', type=str, required=False, default="" + ) parser.add_argument('--sync_mode', action='store_true') parser.add_argument('--use_cuda', action='store_true') parser.add_argument('--use_cpu', action='store_true') @@ -740,24 +857,24 @@ def runtime_main(test_class): parser.add_argument('--dc_asgd', action='store_true') parser.add_argument('--hogwild', action='store_true') parser.add_argument('--save_model', action='store_true') - parser.add_argument('--use_reader_alloc', - action='store_true', - required=False) + parser.add_argument( + '--use_reader_alloc', action='store_true', required=False + ) parser.add_argument('--batch_size', required=False, type=int, default=2) parser.add_argument('--lr', required=False, type=float, default=0.001) - parser.add_argument('--batch_merge_repeat', - required=False, - type=int, - default=1) - parser.add_argument('--nccl2_reduce_layer_local_run', - required=False, - type=bool, - default=False) + parser.add_argument( + '--batch_merge_repeat', required=False, type=int, default=1 + ) + parser.add_argument( + '--nccl2_reduce_layer_local_run', + required=False, + type=bool, + default=False, + ) parser.add_argument('--sync_batch_norm', action='store_true') - parser.add_argument('--fuse_all_reduce', - required=False, - type=ast.literal_eval, - default=None) + parser.add_argument( + '--fuse_all_reduce', required=False, type=ast.literal_eval, default=None + ) args = parser.parse_args() @@ -782,7 +899,6 @@ def runtime_main(test_class): class TestDistBase(unittest.TestCase): - def _setup_config(self): raise NotImplementedError("tests should have _setup_config implemented") @@ -872,10 +988,14 @@ def setUp(self): if DIST_UT_PORT == 0: self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) else: self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - DIST_UT_PORT, DIST_UT_PORT + 1) + DIST_UT_PORT, + DIST_UT_PORT + 1, + ) DIST_UT_PORT += 2 self._dist_port = DIST_UT_PORT @@ -887,13 +1007,14 @@ def tearDown(self): self.temp_dir.cleanup() def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) print_to_err( - type(self).__name__, "socket name: %s" % s.getsockname()[1]) + type(self).__name__, "socket name: %s" % s.getsockname()[1] + ) return s.getsockname()[1] while True: @@ -902,11 +1023,9 @@ def __free_port(): self._port_set.add(port) return port - def start_pserver(self, - model_file, - check_error_log, - required_envs, - log_name=""): + def start_pserver( + self, model_file, check_error_log, required_envs, log_name="" + ): ps0_ep, ps1_ep = self._ps_endpoints.split(",") ps_cmd = "%s" @@ -916,12 +1035,20 @@ def start_pserver(self, ps_cmd += " %s --role pserver --endpoints %s --trainer_id 0 --current_endpoint %s --trainers %d --update_method pserver" - ps0_cmd = ps_cmd % \ - (self._python_interp, model_file, self._ps_endpoints, ps0_ep, - self._trainers) - ps1_cmd = ps_cmd % \ - (self._python_interp, model_file, self._ps_endpoints, ps1_ep, - self._trainers) + ps0_cmd = ps_cmd % ( + self._python_interp, + model_file, + self._ps_endpoints, + ps0_ep, + self._trainers, + ) + ps1_cmd = ps_cmd % ( + self._python_interp, + model_file, + self._ps_endpoints, + ps1_ep, + self._trainers, + ) if self._sync_mode: ps0_cmd += " --sync_mode" @@ -935,26 +1062,32 @@ def start_pserver(self, ps1_pipe = open(path1, "wb") print_to_err(type(self).__name__, "going to start pserver process 0") - ps0_proc = subprocess.Popen(ps0_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=ps0_pipe, - env=required_envs) + ps0_proc = subprocess.Popen( + ps0_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=ps0_pipe, + env=required_envs, + ) print_to_err(type(self).__name__, "going to start pserver process 1") - ps1_proc = subprocess.Popen(ps1_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=ps1_pipe, - env=required_envs) + ps1_proc = subprocess.Popen( + ps1_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=ps1_pipe, + env=required_envs, + ) return ps0_proc, ps1_proc, ps0_pipe, ps1_pipe - def _run_local(self, - model, - envs, - check_error_log=False, - batch_size=DEFAULT_BATCH_SIZE, - batch_merge_repeat=1, - log_name="", - devices="1"): + def _run_local( + self, + model, + envs, + check_error_log=False, + batch_size=DEFAULT_BATCH_SIZE, + batch_merge_repeat=1, + log_name="", + devices="1", + ): cmd = self._python_interp @@ -962,8 +1095,10 @@ def _run_local(self, envs['COVERAGE_FILE'] = os.getenv('COVERAGE_FILE', '') cmd += " -m coverage run --branch -p" - cmd += " %s --role trainer --update_method local --lr %f" % (model, - self._lr) + cmd += " %s --role trainer --update_method local --lr %f" % ( + model, + self._lr, + ) if batch_size != DEFAULT_BATCH_SIZE: cmd += " --batch_size %d" % batch_size @@ -977,21 +1112,21 @@ def _run_local(self, env_local = { "CUDA_VISIBLE_DEVICES": devices, "PADDLE_TRAINERS_NUM": "1", - "PADDLE_TRAINER_ID": "0" + "PADDLE_TRAINER_ID": "0", } elif self.__use_xpu: cmd += " --use_xpu" env_local = { "FLAGS_selected_xpus": devices, "PADDLE_TRAINERS_NUM": "1", - "PADDLE_TRAINER_ID": "0" + "PADDLE_TRAINER_ID": "0", } elif self.__use_npu: cmd += " --use_npu" env_local = { "FLAGS_selected_npus": devices, "PADDLE_TRAINERS_NUM": "1", - "PADDLE_TRAINER_ID": "0" + "PADDLE_TRAINER_ID": "0", } else: env_local = {'CPU_NUM': '1'} @@ -1012,15 +1147,19 @@ def _run_local(self, if check_error_log: path = os.path.join(self.temp_dir.name, log_name + "_local.log") err_log = open(path, "wb") - local_proc = subprocess.Popen(cmd.split(" "), - stdout=subprocess.PIPE, - stderr=err_log, - env=env_local) + local_proc = subprocess.Popen( + cmd.split(" "), + stdout=subprocess.PIPE, + stderr=err_log, + env=env_local, + ) else: - local_proc = subprocess.Popen(cmd.split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env_local) + local_proc = subprocess.Popen( + cmd.split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env_local, + ) local_out, local_err = local_proc.communicate() @@ -1032,27 +1171,29 @@ def _run_local(self, return pickle.loads(local_out) - def _run_local_gloo(self, - model, - envs, - check_error_log=False, - batch_size=DEFAULT_BATCH_SIZE, - batch_merge_repeat=1, - log_name="", - devices="0"): + def _run_local_gloo( + self, + model, + envs, + check_error_log=False, + batch_size=DEFAULT_BATCH_SIZE, + batch_merge_repeat=1, + log_name="", + devices="0", + ): saved_endpoints = self._ps_endpoints self._ps_endpoints = self._ps_endpoints.split(',')[0] - result = self._run_cluster_gloo(model, envs, 'gloo', check_error_log, - log_name) + result = self._run_cluster_gloo( + model, envs, 'gloo', check_error_log, log_name + ) self._ps_endpoints = saved_endpoints return result def _run_cluster(self, model, envs, check_error_log, log_name): # Run dist train to compare with local results - ps0, ps1, ps0_pipe, ps1_pipe = self.start_pserver(model, - check_error_log, - envs, - log_name=log_name) + ps0, ps1, ps0_pipe, ps1_pipe = self.start_pserver( + model, check_error_log, envs, log_name=log_name + ) ps0_ep, ps1_ep = self._ps_endpoints.split(",") @@ -1064,12 +1205,24 @@ def _run_cluster(self, model, envs, check_error_log, log_name): tr_cmd += " %s --role trainer --endpoints %s --trainer_id %d --current_endpoint %s --trainers %d --update_method pserver --lr %f" - tr0_cmd = tr_cmd % \ - (self._python_interp, model, self._ps_endpoints, - 0, ps0_ep, self._trainers, self._lr) - tr1_cmd = tr_cmd % \ - (self._python_interp, model, self._ps_endpoints, - 1, ps1_ep, self._trainers, self._lr) + tr0_cmd = tr_cmd % ( + self._python_interp, + model, + self._ps_endpoints, + 0, + ps0_ep, + self._trainers, + self._lr, + ) + tr1_cmd = tr_cmd % ( + self._python_interp, + model, + self._ps_endpoints, + 1, + ps1_ep, + self._trainers, + self._lr, + ) if self._sync_mode: tr0_cmd += " --sync_mode" @@ -1104,15 +1257,19 @@ def _run_cluster(self, model, envs, check_error_log, log_name): tr1_pipe = open(path1, "wb") print_to_err(type(self).__name__, "going to start trainer process 0") - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=env0) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=env0, + ) print_to_err(type(self).__name__, "going to start trainer process 1") - tr1_proc = subprocess.Popen(tr1_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=env1) + tr1_proc = subprocess.Popen( + tr1_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=env1, + ) # Wait until trainer process terminate while True: @@ -1140,8 +1297,9 @@ def _run_cluster(self, model, envs, check_error_log, log_name): return pickle.loads(tr0_out), pickle.loads(tr1_out) - def _get_gloo_trainer_cmd(self, model, ep, update_method, trainer_id, - trainer_num): + def _get_gloo_trainer_cmd( + self, model, ep, update_method, trainer_id, trainer_num + ): env = {} tr_cmd = "%s -u" @@ -1150,16 +1308,22 @@ def _get_gloo_trainer_cmd(self, model, ep, update_method, trainer_id, tr_cmd += " %s --role trainer --endpoints %s --trainer_id %d --current_endpoint %s --update_method %s --lr %f" - tr_cmd = tr_cmd % \ - (self._python_interp, model, self._ps_endpoints, - trainer_id, ep, update_method, self._lr) + tr_cmd = tr_cmd % ( + self._python_interp, + model, + self._ps_endpoints, + trainer_id, + ep, + update_method, + self._lr, + ) if self._use_reduce: tr_cmd += " --use_reduce" if self._use_reader_alloc: tr_cmd += " --use_reader_alloc" - #assert self._use_reduce == False, "gloo not support _use_reduce" - #assert self._use_reader_alloc == False, "gloo not support _use_reduce" + # assert self._use_reduce == False, "gloo not support _use_reduce" + # assert self._use_reader_alloc == False, "gloo not support _use_reduce" if self._save_model: tr_cmd += " --save_model" if self._diff_batch: @@ -1169,15 +1333,17 @@ def _get_gloo_trainer_cmd(self, model, ep, update_method, trainer_id, assert self.__use_cuda == False, "gloo not support use cuda" assert self.__use_xpu == False, "gloo not support use xpu" tr_cmd += " --use_cpu" - env.update({ - "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), - "PADDLE_TRAINER_ID": "{}".format(trainer_id), - "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": ep, - "PADDLE_CURRENT_ENDPOINT": ep, - "PADDLE_DISTRI_BACKEND": "gloo", - "GLOG_v": "2", - }) + env.update( + { + "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), + "PADDLE_TRAINER_ID": "{}".format(trainer_id), + "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, + "PADDLE_CURRENT_ENDPOINT": ep, + "PADDLE_CURRENT_ENDPOINT": ep, + "PADDLE_DISTRI_BACKEND": "gloo", + "GLOG_v": "2", + } + ) assert self._use_dgc == False, "gloo not support use dgc" @@ -1199,8 +1365,9 @@ def _get_gloo_trainer_cmd(self, model, ep, update_method, trainer_id, assert self._use_fleet_api_20 == False, "gloo not support use fleet api" return tr_cmd, env - def _get_nccl2_trainer_cmd(self, model, ep, update_method, trainer_id, - trainer_num): + def _get_nccl2_trainer_cmd( + self, model, ep, update_method, trainer_id, trainer_num + ): env = {} tr_cmd = "%s -u" @@ -1209,9 +1376,15 @@ def _get_nccl2_trainer_cmd(self, model, ep, update_method, trainer_id, tr_cmd += " %s --role trainer --endpoints %s --trainer_id %d --current_endpoint %s --update_method %s --lr %f" - tr_cmd = tr_cmd % \ - (self._python_interp, model, self._ps_endpoints, - trainer_id, ep, update_method, self._lr) + tr_cmd = tr_cmd % ( + self._python_interp, + model, + self._ps_endpoints, + trainer_id, + ep, + update_method, + self._lr, + ) if self._use_reduce: tr_cmd += " --use_reduce" @@ -1221,47 +1394,55 @@ def _get_nccl2_trainer_cmd(self, model, ep, update_method, trainer_id, tr_cmd += " --save_model" if self.__use_cuda: tr_cmd += " --use_cuda" - env.update({ - "FLAGS_selected_gpus": "{}".format(0), - "CUDA_VISIBLE_DEVICES": "{}".format(trainer_id), - "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), - "PADDLE_TRAINER_ID": "{}".format(trainer_id), - "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": ep, - }) + env.update( + { + "FLAGS_selected_gpus": "{}".format(0), + "CUDA_VISIBLE_DEVICES": "{}".format(trainer_id), + "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), + "PADDLE_TRAINER_ID": "{}".format(trainer_id), + "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, + "PADDLE_CURRENT_ENDPOINT": ep, + } + ) # TODO(liuyuhui):XPU_VISIBLE_DEVICES is not working right now, # will update it after Badiu Kunlun partners' support. elif self.__use_xpu: tr_cmd += " --use_xpu" - env.update({ - "FLAGS_selected_xpus": "{}".format(trainer_id), - #"XPU_VISIBLE_DEVICES": "{}".format(trainer_id + 1), - "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), - "PADDLE_TRAINER_ID": "{}".format(trainer_id), - "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": ep, - "GLOG_v": "2", - }) + env.update( + { + "FLAGS_selected_xpus": "{}".format(trainer_id), + # "XPU_VISIBLE_DEVICES": "{}".format(trainer_id + 1), + "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), + "PADDLE_TRAINER_ID": "{}".format(trainer_id), + "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, + "PADDLE_CURRENT_ENDPOINT": ep, + "GLOG_v": "2", + } + ) elif self.__use_npu: tr_cmd += " --use_npu" - env.update({ - "FLAGS_selected_npus": "{}".format(trainer_id), - "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), - "PADDLE_TRAINER_ID": "{}".format(trainer_id), - "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": ep, - "GLOG_v": "2", - }) + env.update( + { + "FLAGS_selected_npus": "{}".format(trainer_id), + "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), + "PADDLE_TRAINER_ID": "{}".format(trainer_id), + "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, + "PADDLE_CURRENT_ENDPOINT": ep, + "GLOG_v": "2", + } + ) elif self._use_mlu: tr_cmd += " --use_mlu" - env.update({ - "FLAGS_selected_mlus": "{}".format(trainer_id), - "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), - "PADDLE_TRAINER_ID": "{}".format(trainer_id), - "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": ep, - "GLOG_v": "4", - }) + env.update( + { + "FLAGS_selected_mlus": "{}".format(trainer_id), + "PADDLE_TRAINERS_NUM": "{}".format(trainer_num), + "PADDLE_TRAINER_ID": "{}".format(trainer_id), + "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, + "PADDLE_CURRENT_ENDPOINT": ep, + "GLOG_v": "4", + } + ) else: env.update({'CPU_NUM': '1'}) @@ -1292,7 +1473,11 @@ def _get_nccl2_trainer_cmd(self, model, ep, update_method, trainer_id, tr_cmd += " --fuse_all_reduce {}".format(self._fuse_all_reduce) if self._use_fleet_api: - tr_cmd += " --use_fleet_api_20" if self._use_fleet_api_20 else " --use_fleet_api" + tr_cmd += ( + " --use_fleet_api_20" + if self._use_fleet_api_20 + else " --use_fleet_api" + ) if self._use_local_sgd: tr_cmd += " --use_local_sgd" if self._ut4grad_allreduce: @@ -1305,10 +1490,16 @@ def _get_nccl2_trainer_cmd(self, model, ep, update_method, trainer_id, return tr_cmd, env - def _run_cluster_gloo(self, model, envs, update_method, check_error_log, - log_name): - assert update_method == "gloo", "_run_cluster_gloo must have update_method: gloo, but get %s" % update_method - assert not self._use_hallreduce, "_run_cluster_gloo must have _use_hallreduce = false" + def _run_cluster_gloo( + self, model, envs, update_method, check_error_log, log_name + ): + assert update_method == "gloo", ( + "_run_cluster_gloo must have update_method: gloo, but get %s" + % update_method + ) + assert ( + not self._use_hallreduce + ), "_run_cluster_gloo must have _use_hallreduce = false" worker_endpoints = self._ps_endpoints.split(",") @@ -1317,27 +1508,33 @@ def _run_cluster_gloo(self, model, envs, update_method, check_error_log, procs = [] pipes = [] for i in range(0, trainer_num): - tr_cmd, tr_env = self._get_gloo_trainer_cmd(model, - worker_endpoints[i], - update_method, i, - trainer_num) + tr_cmd, tr_env = self._get_gloo_trainer_cmd( + model, worker_endpoints[i], update_method, i, trainer_num + ) tr_env.update(envs) tr_env["GLOG_vmodule"] = 'gloo_context=4' tr_env["GLOG_v"] = '3' - print("use_hallreduce:{} tr_cmd:{}, env: {}".format( - self._use_hallreduce, tr_cmd, tr_env)) + print( + "use_hallreduce:{} tr_cmd:{}, env: {}".format( + self._use_hallreduce, tr_cmd, tr_env + ) + ) - path = os.path.join(self.temp_dir.name, - log_name + "_tr{}_err.log".format(i)) + path = os.path.join( + self.temp_dir.name, log_name + "_tr{}_err.log".format(i) + ) tr_pipe = open(path, "wb") print_to_err( type(self).__name__, - "going to start process {} with nccl2".format(i)) - tr_proc = subprocess.Popen(tr_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr_pipe, - env=tr_env) + "going to start process {} with nccl2".format(i), + ) + tr_proc = subprocess.Popen( + tr_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr_pipe, + env=tr_env, + ) procs.append(tr_proc) pipes.append(tr_pipe) @@ -1350,7 +1547,8 @@ def _run_cluster_gloo(self, model, envs, update_method, check_error_log, sys.stderr.write('trainer {} stderr: {}\n'.format(i, tr_err)) if trainer_num == 1: - if check_error_log: print("outs[0]:", outs[0]) + if check_error_log: + print("outs[0]:", outs[0]) return pickle.loads(outs[0]) else: @@ -1359,8 +1557,9 @@ def _run_cluster_gloo(self, model, envs, update_method, check_error_log, print("outs[1]:", outs[1]) return pickle.loads(outs[0]), pickle.loads(outs[1]) - def _run_cluster_nccl2(self, model, envs, update_method, check_error_log, - log_name): + def _run_cluster_nccl2( + self, model, envs, update_method, check_error_log, log_name + ): if self._use_hallreduce: self._ps_endpoints = "" @@ -1369,7 +1568,8 @@ def _run_cluster_nccl2(self, model, envs, update_method, check_error_log, # NOTE(wangxi). hallreduce test must use 4cards after nccl>=2.7 for i in range(0, 4): self._ps_endpoints += "127.0.0.1:%s," % ( - self._find_free_port()) + self._find_free_port() + ) else: for i in range(0, 4): self._ps_endpoints += "127.0.0.1:%s," % (DIST_UT_PORT + i) @@ -1385,22 +1585,30 @@ def _run_cluster_nccl2(self, model, envs, update_method, check_error_log, pipes = [] for i in range(0, trainer_num): tr_cmd, tr_env = self._get_nccl2_trainer_cmd( - model, worker_endpoints[i], update_method, i, trainer_num) + model, worker_endpoints[i], update_method, i, trainer_num + ) tr_env.update(envs) - print("use_hallreduce:{} tr_cmd:{}, env: {}".format( - self._use_hallreduce, tr_cmd, tr_env)) + print( + "use_hallreduce:{} tr_cmd:{}, env: {}".format( + self._use_hallreduce, tr_cmd, tr_env + ) + ) - path = os.path.join(self.temp_dir.name, - log_name + "_tr{}_err.log".format(i)) + path = os.path.join( + self.temp_dir.name, log_name + "_tr{}_err.log".format(i) + ) tr_pipe = open(path, "wb") print_to_err( type(self).__name__, - "going to start process {} with nccl2".format(i)) - tr_proc = subprocess.Popen(tr_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr_pipe, - env=tr_env) + "going to start process {} with nccl2".format(i), + ) + tr_proc = subprocess.Popen( + tr_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr_pipe, + env=tr_env, + ) procs.append(tr_proc) pipes.append(tr_pipe) @@ -1429,7 +1637,8 @@ def _run_pipeline(self, model, envs, check_error_log, log_name): pipes = [] for i in range(0, trainer_num): tr_cmd, tr_env = self._get_nccl2_trainer_cmd( - model, worker_endpoints[i], update_method, i, trainer_num) + model, worker_endpoints[i], update_method, i, trainer_num + ) tr_env.update(envs) tr_env['CUDA_VISIBLE_DEVICES'] = "0,1" tr_env['NCCL_SHM_DISABLE'] = '1' @@ -1442,11 +1651,14 @@ def _run_pipeline(self, model, envs, check_error_log, log_name): print_to_err( type(self).__name__, - "going to start process {} with nccl2".format(i)) - tr_proc = subprocess.Popen(tr_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr_pipe, - env=tr_env) + "going to start process {} with nccl2".format(i), + ) + tr_proc = subprocess.Popen( + tr_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr_pipe, + env=tr_env, + ) procs.append(tr_proc) pipes.append(tr_pipe) @@ -1477,67 +1689,79 @@ def _get_required_envs(self, check_error_log=False, need_envs={}): "http_proxy": "", "NCCL_P2P_DISABLE": "1", "NCCL_SHM_DISABLE": "1", - "FLAGS_CONVERT_GRAPH_TO_PROGRAM": "1" + "FLAGS_CONVERT_GRAPH_TO_PROGRAM": "1", } if check_error_log: - required_envs["GLOG_vmodule"] = \ - "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10," \ - "alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10,executor=10,operator=10," \ - "sparse_all_reduce_op_handle=10,gen_nccl_id_op=10,gen_nccl_id_op_help=10,nccl_helper=10,grpc_client=10," \ + required_envs["GLOG_vmodule"] = ( + "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10," + "alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10,executor=10,operator=10," + "sparse_all_reduce_op_handle=10,gen_nccl_id_op=10,gen_nccl_id_op_help=10,nccl_helper=10,grpc_client=10," "grpc_server=10,request_handler_impl=10,section_worker=10" + ) required_envs["GLOG_logtostderr"] = "1" if os.getenv('NVIDIA_TF32_OVERRIDE', '') is not None: required_envs['NVIDIA_TF32_OVERRIDE'] = os.getenv( - 'NVIDIA_TF32_OVERRIDE', '') + 'NVIDIA_TF32_OVERRIDE', '' + ) required_envs.update(need_envs) return required_envs - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}, - log_name=""): + def check_with_place( + self, + model_file, + delta=1e-3, + check_error_log=False, + need_envs={}, + log_name="", + ): if self._dygraph and (self._gloo_mode or self._nccl2_mode): need_envs.update({"FLAGS_enable_eager_mode": "1"}) with _test_eager_guard(): - self.check_with_place_func(model_file=model_file, - delta=delta, - check_error_log=check_error_log, - need_envs=need_envs, - log_name=log_name) + self.check_with_place_func( + model_file=model_file, + delta=delta, + check_error_log=check_error_log, + need_envs=need_envs, + log_name=log_name, + ) need_envs.update({"FLAGS_enable_eager_mode": "0"}) - self.check_with_place_func(model_file=model_file, - delta=delta, - check_error_log=check_error_log, - need_envs=need_envs, - log_name=log_name) + self.check_with_place_func( + model_file=model_file, + delta=delta, + check_error_log=check_error_log, + need_envs=need_envs, + log_name=log_name, + ) else: - self.check_with_place_func(model_file=model_file, - delta=delta, - check_error_log=check_error_log, - need_envs=need_envs, - log_name=log_name) - - def check_with_place_func(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}, - log_name=""): + self.check_with_place_func( + model_file=model_file, + delta=delta, + check_error_log=check_error_log, + need_envs=need_envs, + log_name=log_name, + ) + + def check_with_place_func( + self, + model_file, + delta=1e-3, + check_error_log=False, + need_envs={}, + log_name="", + ): required_envs = self._get_required_envs(check_error_log, need_envs) if self._gloo_mode: - local_losses \ - = self._run_local_gloo(model_file, required_envs, - check_error_log, log_name=log_name) + local_losses = self._run_local_gloo( + model_file, required_envs, check_error_log, log_name=log_name + ) else: - local_losses \ - = self._run_local(model_file, required_envs, - check_error_log, log_name=log_name) + local_losses = self._run_local( + model_file, required_envs, check_error_log, log_name=log_name + ) if self._nccl2_mode: if self._nccl2_reduce_layer: @@ -1546,21 +1770,24 @@ def check_with_place_func(self, required_envs, update_method="nccl2_reduce_layer", check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) else: tr0_losses, tr1_losses = self._run_cluster_nccl2( model_file, required_envs, update_method='nccl2', check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) elif self._bkcl_mode: tr0_losses, tr1_losses = self._run_cluster_nccl2( model_file, required_envs, update_method='bkcl', check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) elif self._gloo_mode: # gloo mode, cpu only parallel train @xiongkun03 tr0_losses, tr1_losses = self._run_cluster_gloo( @@ -1568,31 +1795,32 @@ def check_with_place_func(self, required_envs, update_method='gloo', check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) elif self._hccl_mode: tr0_losses, tr1_losses = self._run_cluster_nccl2( model_file, required_envs, update_method='hccl', check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) elif self._cncl_mode: tr0_losses, tr1_losses = self._run_cluster_nccl2( model_file, required_envs, update_method='cncl', check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) elif self._pipeline_mode: - tr0_losses, tr1_losses = self._run_pipeline(model_file, - required_envs, - check_error_log, - log_name=log_name) + tr0_losses, tr1_losses = self._run_pipeline( + model_file, required_envs, check_error_log, log_name=log_name + ) else: - tr0_losses, tr1_losses = self._run_cluster(model_file, - required_envs, - check_error_log, - log_name=log_name) + tr0_losses, tr1_losses = self._run_cluster( + model_file, required_envs, check_error_log, log_name=log_name + ) for step_id in range(RUN_STEP): local_loss = local_losses[step_id] @@ -1605,12 +1833,14 @@ def check_with_place_func(self, print("=======", local_loss, ":", dist_loss[0], "=======") self.assertAlmostEqual(local_loss, dist_loss[0], delta=delta) - def check_with_place_multi_cards(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}, - log_name=""): + def check_with_place_multi_cards( + self, + model_file, + delta=1e-3, + check_error_log=False, + need_envs={}, + log_name="", + ): # need open p2p or shm otherwise multi cards mode will hang need_envs.update({"NCCL_P2P_DISABLE": "0", "NCCL_SHM_DISABLE": "0"}) @@ -1618,19 +1848,22 @@ def check_with_place_multi_cards(self, required_envs = self._get_required_envs(check_error_log, need_envs) if self._use_dgc: - multi_cards_losses = self._run_local(model_file, - required_envs, - check_error_log, - log_name=log_name + - "_dgc_2cards", - devices="0,1") + multi_cards_losses = self._run_local( + model_file, + required_envs, + check_error_log, + log_name=log_name + "_dgc_2cards", + devices="0,1", + ) self._use_dgc = False - base_losses = self._run_local(model_file, - required_envs, - check_error_log, - log_name=log_name + "_base_2cards", - devices="0,1") + base_losses = self._run_local( + model_file, + required_envs, + check_error_log, + log_name=log_name + "_base_2cards", + devices="0,1", + ) self._use_dgc = True diff --git a/python/paddle/fluid/tests/unittests/test_dist_dygraph_apis.py b/python/paddle/fluid/tests/unittests/test_dist_dygraph_apis.py index a99d8221fe19e..aa5e1e93064fb 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_dygraph_apis.py +++ b/python/paddle/fluid/tests/unittests/test_dist_dygraph_apis.py @@ -17,7 +17,6 @@ class TestDygraphFleetApi(TestMultipleGpus): - def test_dygraph_fleet_api(self): self.run_mnist_2gpu('dygraph_fleet_api.py') diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_async.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_async.py index b269e61ec7842..4aaf596d57090 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_async.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_async.py @@ -22,7 +22,6 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_PSERVER_NUMS"] = "2" os.environ["PADDLE_TRAINERS_NUM"] = "2" @@ -30,8 +29,9 @@ def setUp(self): os.environ["PADDLE_PORT"] = "36001" os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_a_sync_optimizer_trainer(self): os.environ["TRAINING_ROLE"] = "TRAINER" diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto.py index f0cf60296cace..523170367fb3c 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto.py @@ -21,7 +21,6 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_PSERVER_NUMS"] = "2" os.environ["PADDLE_TRAINERS_NUM"] = "2" @@ -29,8 +28,9 @@ def setUp(self): os.environ["PADDLE_PORT"] = "36001" os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_a_sync_optimizer1(self): os.environ["TRAINING_ROLE"] = "TRAINER" @@ -43,16 +43,17 @@ def test_a_sync_optimizer1(self): paddle.fluid.framework.switch_startup_program(startup_program) fleet.init(role_maker.PaddleCloudRoleMaker()) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) os.environ["FLAGS_LAUNCH_BARRIER"] = "0" diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_async.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_async.py index dded4e7c7ba20..a51b01209b1d7 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_async.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_async.py @@ -23,7 +23,6 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_PSERVER_NUMS"] = "2" os.environ["PADDLE_TRAINERS_NUM"] = "2" @@ -31,8 +30,9 @@ def setUp(self): os.environ["PADDLE_PORT"] = "36001" os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_a_sync_optimizer3(self): os.environ["TRAINING_ROLE"] = "TRAINER" @@ -45,26 +45,31 @@ def test_a_sync_optimizer3(self): paddle.fluid.framework.switch_startup_program(startup_program) fleet.init(role_maker.PaddleCloudRoleMaker()) - input_x = paddle.fluid.layers.data(name="x", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) + input_x = paddle.fluid.layers.data( + name="x", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) x_embedding = paddle.fluid.layers.embedding( is_distributed=False, input=input_x, size=[1000000000, 100000], param_attr=paddle.fluid.ParamAttr( name="embedding", - initializer=paddle.fluid.initializer.Constant(value=0.01)), - is_sparse=True) + initializer=paddle.fluid.initializer.Constant(value=0.01), + ), + is_sparse=True, + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=x_embedding, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) os.environ["FLAGS_LAUNCH_BARRIER"] = "0" diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_geo.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_geo.py index 035506607e311..e6f618b63a658 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_geo.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_geo.py @@ -22,7 +22,6 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_PSERVER_NUMS"] = "2" os.environ["PADDLE_TRAINERS_NUM"] = "2" @@ -30,8 +29,9 @@ def setUp(self): os.environ["PADDLE_PORT"] = "36001" os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_a_sync_optimizer2(self): os.environ["TRAINING_ROLE"] = "TRAINER" @@ -48,15 +48,16 @@ def test_a_sync_optimizer2(self): input_x = paddle.fluid.layers.data(name="x", shape=[1], dtype='int64') input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') - emb = paddle.fluid.layers.embedding(input=input_x, - size=[100, 10], - is_sparse=True) + emb = paddle.fluid.layers.embedding( + input=input_x, size=[100, 10], is_sparse=True + ) fc_1 = paddle.fluid.layers.fc(input=emb, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) os.environ["FLAGS_LAUNCH_BARRIER"] = "0" strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_geo.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_geo.py index 45d3b315bcde5..05abc40a1d041 100755 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_geo.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_geo.py @@ -21,7 +21,6 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_PSERVER_NUMS"] = "2" os.environ["PADDLE_TRAINERS_NUM"] = "2" @@ -29,8 +28,9 @@ def setUp(self): os.environ["PADDLE_PORT"] = "36001" os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_a_sync_optimizer_trainer(self): os.environ["TRAINING_ROLE"] = "TRAINER" @@ -43,16 +43,17 @@ def test_a_sync_optimizer_trainer(self): paddle.fluid.framework.switch_startup_program(startup_program) fleet.init(role_maker.PaddleCloudRoleMaker()) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() @@ -75,16 +76,17 @@ def test_a_sync_optimizer_pserver(self): paddle.fluid.framework.switch_startup_program(startup_program) fleet.init(role_maker.PaddleCloudRoleMaker()) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_sync.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_sync.py index 753635e690437..837ea65d54f48 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_sync.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_sync.py @@ -22,7 +22,6 @@ class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_PSERVER_NUMS"] = "2" os.environ["PADDLE_TRAINERS_NUM"] = "2" @@ -31,8 +30,9 @@ def setUp(self): os.environ["TRAINING_ROLE"] = "TRAINER" os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_gradient_merge_optimizer(self): fleet.init(role_maker.PaddleCloudRoleMaker()) diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_base.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_base.py index c8ae35d3d55ae..6686fb03a01fe 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_base.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_base.py @@ -17,6 +17,7 @@ import paddle.distributed.fleet.base.role_maker as role_maker import paddle.fluid as fluid import paddle + """ high level unit test for distribute fleet. """ @@ -46,9 +47,9 @@ class FleetDistRunnerBase(object): """ - run_pserver,run_trainer : after init role, using transpiler split program - net : implment by child class, the network of model - do training : exe run program + run_pserver,run_trainer : after init role, using transpiler split program + net : implment by child class, the network of model + do training : exe run program """ def __init__(self): @@ -64,7 +65,8 @@ def build_role(self, args): current_id=args.current_id, role=role_maker.Role.SERVER, worker_endpoints=args.trainer_endpoints.split(","), - server_endpoints=args.endpoints.split(",")) + server_endpoints=args.endpoints.split(","), + ) else: role = role_maker.UserDefinedRoleMaker( is_collective=False, @@ -73,7 +75,8 @@ def build_role(self, args): current_id=args.current_id, role=role_maker.Role.WORKER, worker_endpoints=args.trainer_endpoints.split(","), - server_endpoints=args.endpoints.split(",")) + server_endpoints=args.endpoints.split(","), + ) self.role = role return role @@ -100,14 +103,13 @@ def build_strategy(self, args): debug = int(os.getenv("Debug", "0")) # TODO(update strategy to support dump params) if False: # debug: - self.strategy.set_debug_opt({ - "dump_param": - self.dump_param, - "dump_fields": - self.dump_fields, - "dump_fields_path": - self.dump_fields_path - }) + self.strategy.set_debug_opt( + { + "dump_param": self.dump_param, + "dump_fields": self.dump_fields, + "dump_fields_path": self.dump_fields_path, + } + ) return self.strategy @@ -126,7 +128,8 @@ def build_optimizer(self, avg_cost, strategy): use_decay = int(os.getenv("USE_DECAY", "0")) if use_decay: scheduler = paddle.optimizer.lr.ExponentialDecay( - learning_rate=LEARNING_RATE, gamma=0.999, verbose=True) + learning_rate=LEARNING_RATE, gamma=0.999, verbose=True + ) optimizer = fluid.optimizer.SGD(scheduler, grad_clip=grad_clip) """ # learning rate decay method before 2.0 @@ -154,7 +157,8 @@ def run_pyreader_trainer(self, args): def net(self, args, batch_size=4, lr=0.01): raise NotImplementedError( - "get_model should be implemented by child classes.") + "get_model should be implemented by child classes." + ) def get_executor(self): if self._exe is None: @@ -168,21 +172,24 @@ def get_executor(self): def do_dataset_training(self, fleet): raise NotImplementedError( - "do_dataset_training should be implemented by child classes.") + "do_dataset_training should be implemented by child classes." + ) def do_pyreader_training(self, fleet): raise NotImplementedError( - "do_pyreader_training should be implemented by child classes.") + "do_pyreader_training should be implemented by child classes." + ) def do_distributed_testing(self, fleet): raise NotImplementedError( - "do_distributed_testing should be implemented by child classes.") + "do_distributed_testing should be implemented by child classes." + ) class TestFleetBase(unittest.TestCase): """ - start_pserver,start_trainer : add start cmd to test - run_cluster : using multi process to test distribute program + start_pserver,start_trainer : add start cmd to test + run_cluster : using multi process to test distribute program """ def _setup_config(self): @@ -210,15 +217,23 @@ def setUp(self): if DIST_UT_PORT: print("set begin_port:", DIST_UT_PORT) self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - DIST_UT_PORT, DIST_UT_PORT + 1) + DIST_UT_PORT, + DIST_UT_PORT + 1, + ) self._tr_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - DIST_UT_PORT + 2, DIST_UT_PORT + 3) + DIST_UT_PORT + 2, + DIST_UT_PORT + 3, + ) DIST_UT_PORT += 4 else: self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._tr_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable self._geo_sgd_need_push_nums = 5 @@ -226,10 +241,10 @@ def setUp(self): self._setup_config() def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -259,18 +274,24 @@ def _start_pserver(self, cmd, required_envs): ps0_out = open(ps0_out_log, "wb+") ps1_out = open(ps1_out_log, "wb+") - ps0_proc = subprocess.Popen(ps0_cmd.strip().split(" "), - stdout=ps0_out, - stderr=ps0_err, - env=required_envs) - - ps1_proc = subprocess.Popen(ps1_cmd.strip().split(" "), - stdout=ps1_out, - stderr=ps1_err, - env=required_envs) - - return ((ps0_proc, ps0_out, ps0_err, ps0_out_log, ps0_err_log), - (ps1_proc, ps1_out, ps1_err, ps1_out_log, ps1_err_log)) + ps0_proc = subprocess.Popen( + ps0_cmd.strip().split(" "), + stdout=ps0_out, + stderr=ps0_err, + env=required_envs, + ) + + ps1_proc = subprocess.Popen( + ps1_cmd.strip().split(" "), + stdout=ps1_out, + stderr=ps1_err, + env=required_envs, + ) + + return ( + (ps0_proc, ps0_out, ps0_err, ps0_out_log, ps0_err_log), + (ps1_proc, ps1_out, ps1_err, ps1_out_log, ps1_err_log), + ) def _start_trainer(self, cmd, required_envs): tr0_cmd, tr1_cmd = cmd.format(0), cmd.format(1) @@ -292,18 +313,24 @@ def _start_trainer(self, cmd, required_envs): tr0_out = open(tr0_out_log, "wb+") tr1_out = open(tr1_out_log, "wb+") - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(" "), - stdout=tr0_out, - stderr=tr0_err, - env=required_envs) - - tr1_proc = subprocess.Popen(tr1_cmd.strip().split(" "), - stdout=tr1_out, - stderr=tr1_err, - env=required_envs) - - return ((tr0_proc, tr0_out, tr0_err, tr0_out_log, tr0_err_log), - (tr1_proc, tr1_out, tr1_err, tr1_out_log, tr1_err_log)) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(" "), + stdout=tr0_out, + stderr=tr0_err, + env=required_envs, + ) + + tr1_proc = subprocess.Popen( + tr1_cmd.strip().split(" "), + stdout=tr1_out, + stderr=tr1_err, + env=required_envs, + ) + + return ( + (tr0_proc, tr0_out, tr0_err, tr0_out_log, tr0_err_log), + (tr1_proc, tr1_out, tr1_err, tr1_out_log, tr1_err_log), + ) def _run_cluster(self, model, envs): env = {'GRAD_CLIP': str(self._grad_clip_mode), 'WITH_DISTRIBUTE': 'ON'} @@ -316,14 +343,30 @@ def _run_cluster(self, model, envs): env.update(envs) tr_cmd = "{0} {1} --role trainer --endpoints {2} --trainer_endpoints {3} --current_id {{}} --trainers {4} --mode {5} --geo_sgd_need_push_nums {6} --reader {7} --gloo_path {8} --test {9}".format( - python_path, model, self._ps_endpoints, self._tr_endpoints, - self._trainers, self._mode, self._geo_sgd_need_push_nums, - self._reader, gloo_path, self._need_test) + python_path, + model, + self._ps_endpoints, + self._tr_endpoints, + self._trainers, + self._mode, + self._geo_sgd_need_push_nums, + self._reader, + gloo_path, + self._need_test, + ) ps_cmd = "{0} {1} --role pserver --endpoints {2} --trainer_endpoints {3} --current_id {{}} --trainers {4} --mode {5} --geo_sgd_need_push_nums {6} --reader {7} --gloo_path {8} --test {9}".format( - python_path, model, self._ps_endpoints, self._tr_endpoints, - self._trainers, self._mode, self._geo_sgd_need_push_nums, - self._reader, gloo_path, self._need_test) + python_path, + model, + self._ps_endpoints, + self._tr_endpoints, + self._trainers, + self._mode, + self._geo_sgd_need_push_nums, + self._reader, + gloo_path, + self._need_test, + ) if self._model_dir: tr_cmd += " --model_dir {}".format(self._model_dir) @@ -340,7 +383,7 @@ def _run_cluster(self, model, envs): tr1_proc, tr1_out, tr1_err, tr1_out_log, tr1_err_log = tr1 # Wait until trainer process terminate - #time_out = 120 + # time_out = 120 time_out = 60 cur_time = 0 @@ -383,27 +426,41 @@ def is_listen_failed(logx): def catlog(logx): basename = os.path.basename(logx) - print("\n================== Error {} begin =====================". - format(basename)) + print( + "\n================== Error {} begin =====================".format( + basename + ) + ) os.system("cat {}".format(logx)) - print("================== Error {} end =====================\n". - format(basename)) + print( + "================== Error {} end =====================\n".format( + basename + ) + ) if tr0_ret != 0 or tr1_ret != 0: if is_listen_failed(ps0_err) or is_listen_failed(ps1_err): print("find parameter server port bind failed, skip the error") tr0_ret, tr1_ret = 0, 0 else: - for out, err in [(ps0_out_log, ps0_err_log), - (ps1_out_log, ps1_err_log), - (tr0_out_log, tr0_err_log), - (tr1_out_log, tr1_err_log)]: + for out, err in [ + (ps0_out_log, ps0_err_log), + (ps1_out_log, ps1_err_log), + (tr0_out_log, tr0_err_log), + (tr1_out_log, tr1_err_log), + ]: catlog(out) catlog(err) for pipe in [ - tr0_err, tr0_out, tr1_err, tr1_out, ps0_err, ps0_out, ps1_err, - ps1_out + tr0_err, + tr0_out, + tr1_err, + tr1_out, + ps0_err, + ps0_out, + ps1_err, + ps1_out, ]: pipe.close() @@ -414,17 +471,15 @@ def catlog(logx): return 0, 0 - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), "FLAGS_rpc_deadline": "5000", # 5sec to fail fast - "http_proxy": "" + "http_proxy": "", } required_envs.update(need_envs) @@ -438,23 +493,20 @@ def check_with_place(self, def runtime_main(test_class): parser = argparse.ArgumentParser(description='Run Fleet test.') - parser.add_argument('--role', - type=str, - required=True, - choices=['pserver', 'trainer']) + parser.add_argument( + '--role', type=str, required=True, choices=['pserver', 'trainer'] + ) parser.add_argument('--endpoints', type=str, required=False, default="") - parser.add_argument('--trainer_endpoints', - type=str, - required=False, - default="") + parser.add_argument( + '--trainer_endpoints', type=str, required=False, default="" + ) parser.add_argument('--gloo_path', type=str, required=False, default="") parser.add_argument('--current_id', type=int, required=False, default=0) parser.add_argument('--trainers', type=int, required=False, default=1) parser.add_argument('--mode', type=str, required=False, default='geo') - parser.add_argument('--geo_sgd_need_push_nums', - type=int, - required=False, - default=2) + parser.add_argument( + '--geo_sgd_need_push_nums', type=int, required=False, default=2 + ) parser.add_argument('--reader', type=str, required=False, default='dataset') parser.add_argument('--test', type=int, required=False, default=0) parser.add_argument('--model_dir', type=str, required=False, default="") @@ -467,14 +519,17 @@ def runtime_main(test_class): if args.test and args.model_dir != "": avg_cost = model.net(args, is_train=False) dist_infer = DistributedInfer() - dist_infer.init_distributed_infer_env(exe=model.get_executor(), - loss=model.avg_cost, - role_maker=role, - dirname=args.model_dir) + dist_infer.init_distributed_infer_env( + exe=model.get_executor(), + loss=model.avg_cost, + role_maker=role, + dirname=args.model_dir, + ) if fleet.is_worker(): with paddle.static.program_guard( - main_program=dist_infer.get_dist_infer_program()): + main_program=dist_infer.get_dist_infer_program() + ): model.do_distributed_testing(fleet) fleet.stop_worker() return @@ -499,13 +554,17 @@ def runtime_main(test_class): test_origin_program = paddle.static.Program() test_startup_program = paddle.static.Program() with paddle.static.program_guard( - main_program=test_origin_program, - startup_program=test_startup_program): + main_program=test_origin_program, + startup_program=test_startup_program, + ): with paddle.utils.unique_name.guard(): avg_cost = model.net(args, is_train=False) - dist_infer = DistributedInfer(main_program=test_origin_program, - startup_program=test_startup_program) + dist_infer = DistributedInfer( + main_program=test_origin_program, + startup_program=test_startup_program, + ) with paddle.static.program_guard( - main_program=dist_infer.get_dist_infer_program()): + main_program=dist_infer.get_dist_infer_program() + ): model.do_distributed_testing(fleet) fleet.stop_worker() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py index 06d57c86bd70a..33006b9d9e82c 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py @@ -18,17 +18,14 @@ class TestDistMnistAsyncInMemoryDataset2x2(TestFleetBase): - def _setup_config(self): self._mode = "async" - #self._reader = "pyreader" + # self._reader = "pyreader" self._reader = "dataset" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -38,14 +35,10 @@ def check_with_place(self, "CPU_NUM": "2", "LOG_DIRNAME": "/tmp", "SAVE_DIRNAME": "/tmp/TestDistMnistAsyncInMemoryDataset2x2/model", - "SAVE_CACHE_DIRNAME": - "/tmp/TestDistMnistAsyncInMemoryDataset2x2/cache_model", - "SAVE_DENSE_PARAM_DIRNAME": - "/tmp/TestDistMnistAsyncInMemoryDataset2x2/dense_param", - "SAVE_ONE_TABLE_DIRNAME": - "/tmp/TestDistMnistAsyncInMemoryDataset2x2/table_0", - "SAVE_PATCH_DIRNAME": - "/tmp/TestDistMnistAsyncInMemoryDataset2x2/patch_model", + "SAVE_CACHE_DIRNAME": "/tmp/TestDistMnistAsyncInMemoryDataset2x2/cache_model", + "SAVE_DENSE_PARAM_DIRNAME": "/tmp/TestDistMnistAsyncInMemoryDataset2x2/dense_param", + "SAVE_ONE_TABLE_DIRNAME": "/tmp/TestDistMnistAsyncInMemoryDataset2x2/table_0", + "SAVE_PATCH_DIRNAME": "/tmp/TestDistMnistAsyncInMemoryDataset2x2/patch_model", "LOG_PREFIX": self.__class__.__name__, } @@ -58,22 +51,19 @@ def check_with_place(self, tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) class TestDistMnistAsync2x2(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -94,22 +84,19 @@ def check_with_place(self, tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) class TestDistCtrHalfAsync2x2(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -133,9 +120,9 @@ def check_with_place(self, tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr2.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr2.py index b86fabde6209e..d9c725c3a89c4 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr2.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr2.py @@ -20,17 +20,14 @@ @unittest.skip(reason="Skip unstable ut, need paddle sync mode fix") class TestDistMnistSync2x2(TestFleetBase): - def _setup_config(self): self._mode = "sync" self._reader = "pyreader" self._need_test = 1 - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -51,23 +48,20 @@ def check_with_place(self, tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) # @unittest.skip(reason="Skip unstable ut, reader need to be rewrite") class TestDistMnistAsyncDataset2x2(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "dataset" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -92,9 +86,9 @@ def check_with_place(self, tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_decay.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_decay.py index a5e2e7073d0a9..9d5ac645b6136 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_decay.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_decay.py @@ -32,18 +32,17 @@ class TestNoamDecay(unittest.TestCase): - def net(self): - input_data = paddle.static.data(name="sparse_input", - shape=[None, 1], - dtype="int64") - input_label = paddle.static.data(name="label", - shape=[None, 1], - dtype="int64") + input_data = paddle.static.data( + name="sparse_input", shape=[None, 1], dtype="int64" + ) + input_label = paddle.static.data( + name="label", shape=[None, 1], dtype="int64" + ) label = paddle.cast(input_label, dtype="float32") - embedding = paddle.static.nn.embedding(input_data, - is_sparse=True, - size=[1000, 128]) + embedding = paddle.static.nn.embedding( + input_data, is_sparse=True, size=[1000, 128] + ) fc1 = paddle.static.nn.fc(embedding, size=1024, activation="relu") fc2 = paddle.static.nn.fc(fc1, size=512, activation="relu") @@ -56,20 +55,24 @@ def net(self): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.WORKER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.WORKER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss = self.net() - scheduler = paddle.optimizer.lr.NoamDecay(d_model=0.01, - warmup_steps=100, - verbose=True) + scheduler = paddle.optimizer.lr.NoamDecay( + d_model=0.01, warmup_steps=100, verbose=True + ) optimizer = fluid.optimizer.Adam(scheduler) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_geo.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_geo.py index 37ed07534d96d..2e63a28cd2478 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_geo.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_geo.py @@ -28,17 +28,14 @@ class TestDistGeoCtr_2x2(TestFleetBase): - def _setup_config(self): self._mode = "geo" self._reader = "pyreader" self._geo_sgd_need_push_nums = 5 - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -58,19 +55,19 @@ def check_with_place(self, tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) class TestGeoSgdTranspiler(unittest.TestCase): - def test_pserver(self): role = role_maker.UserDefinedRoleMaker( current_id=0, role=role_maker.Role.SERVER, worker_num=2, - server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"]) + server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"], + ) fleet.init(role) diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_gloo.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_gloo.py index 26590f5a75e53..dd31dc8ef0db8 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_gloo.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_gloo.py @@ -18,25 +18,25 @@ import unittest import subprocess import time -#import paddle.fluid.incubate.fleet.base.role_maker as role_maker + +# import paddle.fluid.incubate.fleet.base.role_maker as role_maker from test_dist_fleet_base import TestFleetBase -#from dist_simnet_bow import train_network +# from dist_simnet_bow import train_network class TestDistGloo_2x2(TestFleetBase): - def _setup_config(self): self._mode = "sync" self._reader = "pyreader" self._path = "./tmp4" - if (os.path.exists(self._path)): + if os.path.exists(self._path): shutil.rmtree(self._path) # if not os.path.exists(self._path): # os.mkdir(self._path) def _start_pserver(self, cmd, required_envs): - #env.update(required_envs) + # env.update(required_envs) ps0_cmd = cmd ps1_cmd = cmd @@ -46,24 +46,28 @@ def _start_pserver(self, cmd, required_envs): required_envs["POD_IP"] = "127.0.0.1" required_envs["PADDLE_PSERVER_ID"] = "0" required_envs["PADDLE_PORT"] = "36011" - ps0_proc = subprocess.Popen(ps0_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=ps0_pipe, - env=required_envs) + ps0_proc = subprocess.Popen( + ps0_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=ps0_pipe, + env=required_envs, + ) print("PADDLE_PSERVER_ID=0:") print(required_envs) required_envs["PADDLE_PSERVER_ID"] = "1" required_envs["PADDLE_PORT"] = "36012" - ps1_proc = subprocess.Popen(ps1_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=ps1_pipe, - env=required_envs) + ps1_proc = subprocess.Popen( + ps1_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=ps1_pipe, + env=required_envs, + ) print("PADDLE_PSERVER_ID=1:") print(required_envs) return ps0_proc, ps1_proc, ps0_pipe, ps1_pipe def _start_trainer(self, cmd, required_envs): - #env.update(required_envs) + # env.update(required_envs) tr0_cmd = cmd tr1_cmd = cmd @@ -71,17 +75,21 @@ def _start_trainer(self, cmd, required_envs): tr0_pipe = open(tempfile.gettempdir() + "/tr0_err.log", "wb+") tr1_pipe = open(tempfile.gettempdir() + "/tr1_err.log", "wb+") required_envs["PADDLE_TRAINER_ID"] = "0" - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr0_pipe, - env=required_envs) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr0_pipe, + env=required_envs, + ) print("PADDLE_TRAINER_ID=0:") print(required_envs) required_envs["PADDLE_TRAINER_ID"] = "1" - tr1_proc = subprocess.Popen(tr1_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=tr1_pipe, - env=required_envs) + tr1_proc = subprocess.Popen( + tr1_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=tr1_pipe, + env=required_envs, + ) print("PADDLE_TRAINER_ID=1:") print(required_envs) return tr0_proc, tr1_proc, tr0_pipe, tr1_pipe @@ -138,11 +146,9 @@ def _run_cluster(self, model, envs): return 0, 0 - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -150,16 +156,16 @@ def check_with_place(self, "FLAGS_rpc_deadline": "5000", # 5sec to fail fast "http_proxy": "", "CPU_NUM": "2", - #PSERVER + # PSERVER "PADDLE_PSERVERS_IP_PORT_LIST": "127.0.0.1:36011,127.0.0.1:36012", - #"PADDLE_PSERVER_PORT_ARRAY":"(36011 36012)", + # "PADDLE_PSERVER_PORT_ARRAY":"(36011 36012)", "PADDLE_PSERVER_NUMS": "2", "PADDLE_TRAINER_ID": "0", - #TRAINER + # TRAINER "PADDLE_TRAINER_ENDPOINTS": "127.0.0.1:36013,127.0.0.1:36014", "PADDLE_TRAINERS_NUM": "2", "PADDLE_PSERVER_ID": "0", - #GLOO FLAG + # GLOO FLAG "PADDLE_WITH_GLOO": "1", } @@ -173,9 +179,9 @@ def check_with_place(self, def test_dist_train(self): print("path is not delete", os.path.exists("./tmp4")) - self.check_with_place("dist_fleet_debug_gloo.py", - delta=1e-5, - check_error_log=True) + self.check_with_place( + "dist_fleet_debug_gloo.py", delta=1e-5, check_error_log=True + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_base.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_base.py index 13178ad233e2e..18c20ef675473 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_base.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_base.py @@ -41,9 +41,9 @@ class FleetDistHeterRunnerBase(object): """ - run_pserver,run_trainer : after init role, using transpiler split program - net : implment by child class, the network of model - do training : exe run program + run_pserver,run_trainer : after init role, using transpiler split program + net : implment by child class, the network of model + do training : exe run program """ def build_role(self, args): @@ -54,33 +54,45 @@ def build_role(self, args): environs["PADDLE_PSERVERS_IP_PORT_LIST"] = args.endpoints environs["PADDLE_TRAINER_ENDPOINTS"] = args.trainer_endpoints environs[ - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST"] = all_heter_trainer_endpoints - environs["POD_IP"] = args.endpoints.split(",")[int( - args.current_id)].split(":")[0] - environs["PADDLE_PORT"] = args.endpoints.split(",")[int( - args.current_id)].split(":")[1] + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST" + ] = all_heter_trainer_endpoints + environs["POD_IP"] = args.endpoints.split(",")[ + int(args.current_id) + ].split(":")[0] + environs["PADDLE_PORT"] = args.endpoints.split(",")[ + int(args.current_id) + ].split(":")[1] environs["TRAINING_ROLE"] = args.role.upper() environs["PADDLE_TRAINERS_NUM"] = args.trainers elif args.role.upper() == "HETER_TRAINER": - previous_endpoints = args.trainer_endpoints if args.stage_id == 2 else heter_trainer_endpoints[ - 0] - next_endpoints = heter_trainer_endpoints[ - 1] if args.stage_id == 2 else "" - heter_device = args.heter_trainer_device.split(";")[args.stage_id - - 2] + previous_endpoints = ( + args.trainer_endpoints + if args.stage_id == 2 + else heter_trainer_endpoints[0] + ) + next_endpoints = ( + heter_trainer_endpoints[1] if args.stage_id == 2 else "" + ) + heter_device = args.heter_trainer_device.split(";")[ + args.stage_id - 2 + ] environs["PADDLE_PSERVERS_IP_PORT_LIST"] = args.endpoints environs["PADDLE_TRAINER_ENDPOINTS"] = args.trainer_endpoints environs["PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST"] = next_endpoints environs[ - "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST"] = previous_endpoints + "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST" + ] = previous_endpoints environs[ - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST"] = all_heter_trainer_endpoints + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST" + ] = all_heter_trainer_endpoints environs["HETER_DEVICE_TYPE"] = heter_device environs["TRAINING_ROLE"] = args.role.upper() - environs["POD_IP"] = all_heter_trainer_endpoints.split(",")[int( - args.current_id)].split(":")[0] + environs["POD_IP"] = all_heter_trainer_endpoints.split(",")[ + int(args.current_id) + ].split(":")[0] environs["PADDLE_PORT"] = all_heter_trainer_endpoints.split(",")[ - int(args.current_id)].split(":")[1] + int(args.current_id) + ].split(":")[1] environs["PADDLE_TRAINERS_NUM"] = args.trainers environs["PADDLE_STAGE_TRAINERS_NUM"] = [2, 2, 2] environs["FLAGS_selected_gpus"] = 0 @@ -93,18 +105,21 @@ def build_role(self, args): environs["PADDLE_PSERVERS_IP_PORT_LIST"] = args.endpoints environs["PADDLE_TRAINER_ENDPOINTS"] = args.trainer_endpoints environs[ - "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST"] = heter_trainer_endpoints[ - 0] + "PADDLE_NEXT_HETER_TRAINER_IP_PORT_LIST" + ] = heter_trainer_endpoints[0] environs["PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST"] = "" environs[ - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST"] = all_heter_trainer_endpoints + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST" + ] = all_heter_trainer_endpoints environs["HETER_DEVICE_TYPE"] = "cpu" environs["TRAINING_ROLE"] = args.role.upper() environs["PADDLE_TRAINER_ID"] = args.current_id - environs["POD_IP"] = args.trainer_endpoints.split(",")[int( - args.current_id)].split(":")[0] - environs["PADDLE_PORT"] = args.trainer_endpoints.split(",")[int( - args.current_id)].split(":")[1] + environs["POD_IP"] = args.trainer_endpoints.split(",")[ + int(args.current_id) + ].split(":")[0] + environs["PADDLE_PORT"] = args.trainer_endpoints.split(",")[ + int(args.current_id) + ].split(":")[1] environs["PADDLE_TRAINERS_NUM"] = args.trainers environs["PADDLE_STAGE_TRAINERS_NUM"] = [2, 2, 2] environs["FLAGS_selected_gpus"] = 0 @@ -126,12 +141,12 @@ def build_strategy(self, args): self.strategy.a_sync = True self.strategy.a_sync_configs = { "launch_barrier": True, - "heter_worker_device_guard": 'gpu' + "heter_worker_device_guard": 'gpu', } self.strategy.pipeline = True self.strategy.pipeline_configs = { "accumulate_steps": 1, - "micro_batch_size": 2048 + "micro_batch_size": 2048, } return self.strategy @@ -152,21 +167,24 @@ def run_dataset_trainer(self, args): def net(self, args, batch_size=4, lr=0.01): raise NotImplementedError( - "get_model should be implemented by child classes.") + "get_model should be implemented by child classes." + ) def do_dataset_training(self, fleet): raise NotImplementedError( - "do_dataset_training should be implemented by child classes.") + "do_dataset_training should be implemented by child classes." + ) def do_dataset_heter_training(self, fleet): raise NotImplementedError( - "do_dataset_heter_training should be implemented by child classes.") + "do_dataset_heter_training should be implemented by child classes." + ) class TestFleetHeterBase(unittest.TestCase): """ - start_pserver,start_trainer : add start cmd to test - run_cluster : using multi process to test distribute program + start_pserver,start_trainer : add start cmd to test + run_cluster : using multi process to test distribute program """ def _setup_config(self): @@ -194,23 +212,39 @@ def setUp(self): if DIST_UT_PORT: print("set begin_port:", DIST_UT_PORT) self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - DIST_UT_PORT, DIST_UT_PORT + 1) + DIST_UT_PORT, + DIST_UT_PORT + 1, + ) self._tr_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - DIST_UT_PORT + 2, DIST_UT_PORT + 3) + DIST_UT_PORT + 2, + DIST_UT_PORT + 3, + ) self._heter_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - DIST_UT_PORT + 4, DIST_UT_PORT + 5) + DIST_UT_PORT + 4, + DIST_UT_PORT + 5, + ) self._heter_endpoints_2 = "127.0.0.1:%s,127.0.0.1:%s" % ( - DIST_UT_PORT + 6, DIST_UT_PORT + 7) + DIST_UT_PORT + 6, + DIST_UT_PORT + 7, + ) DIST_UT_PORT += 8 else: self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._tr_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._heter_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._heter_endpoints_2 = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable self._geo_sgd_need_push_nums = 5 @@ -218,10 +252,10 @@ def setUp(self): self._setup_config() def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -237,14 +271,18 @@ def _start_pserver(self, cmd, required_envs): ps0_pipe = open(tempfile.gettempdir() + "/ps0_err.log", "wb+") ps1_pipe = open(tempfile.gettempdir() + "/ps1_err.log", "wb+") - ps0_proc = subprocess.Popen(ps0_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=ps0_pipe, - env=required_envs) - ps1_proc = subprocess.Popen(ps1_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=ps1_pipe, - env=required_envs) + ps0_proc = subprocess.Popen( + ps0_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=ps0_pipe, + env=required_envs, + ) + ps1_proc = subprocess.Popen( + ps1_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=ps1_pipe, + env=required_envs, + ) return ps0_proc, ps1_proc, ps0_pipe, ps1_pipe def _start_trainer(self, cmd, required_envs): @@ -256,20 +294,28 @@ def _start_trainer(self, cmd, required_envs): tr0_out = open(tempfile.gettempdir() + "/tr0_out.log", "wb+") tr1_out = open(tempfile.gettempdir() + "/tr1_out.log", "wb+") - tr0_proc = subprocess.Popen(tr0_cmd.strip().split(" "), - stdout=tr0_out, - stderr=tr0_pipe, - env=required_envs) - tr1_proc = subprocess.Popen(tr1_cmd.strip().split(" "), - stdout=tr1_out, - stderr=tr1_pipe, - env=required_envs) + tr0_proc = subprocess.Popen( + tr0_cmd.strip().split(" "), + stdout=tr0_out, + stderr=tr0_pipe, + env=required_envs, + ) + tr1_proc = subprocess.Popen( + tr1_cmd.strip().split(" "), + stdout=tr1_out, + stderr=tr1_pipe, + env=required_envs, + ) return tr0_proc, tr1_proc, tr0_pipe, tr1_pipe def _start_heter_trainer(self, cmd, required_envs): - heter0_cmd, heter1_cmd, heter2_cmd, heter3_cmd = cmd.format( - 0, 2), cmd.format(1, 2), cmd.format(2, 3), cmd.format(3, 3) + heter0_cmd, heter1_cmd, heter2_cmd, heter3_cmd = ( + cmd.format(0, 2), + cmd.format(1, 2), + cmd.format(2, 3), + cmd.format(3, 3), + ) heter0_pipe = open(tempfile.gettempdir() + "/heter0_err.log", "wb+") heter1_pipe = open(tempfile.gettempdir() + "/heter1_err.log", "wb+") @@ -280,29 +326,46 @@ def _start_heter_trainer(self, cmd, required_envs): heter2_out = open(tempfile.gettempdir() + "/heter2_out.log", "wb+") heter3_out = open(tempfile.gettempdir() + "/heter3_out.log", "wb+") - heter0_proc = subprocess.Popen(heter0_cmd.strip().split(" "), - stdout=heter0_out, - stderr=heter0_pipe, - env=required_envs) - heter1_proc = subprocess.Popen(heter1_cmd.strip().split(" "), - stdout=heter1_out, - stderr=heter1_pipe, - env=required_envs) - heter2_proc = subprocess.Popen(heter2_cmd.strip().split(" "), - stdout=heter2_out, - stderr=heter2_pipe, - env=required_envs) - heter3_proc = subprocess.Popen(heter3_cmd.strip().split(" "), - stdout=heter3_out, - stderr=heter3_pipe, - env=required_envs) - - return heter0_proc, heter1_proc, heter2_proc, heter3_proc, heter0_pipe, heter1_pipe, heter2_pipe, heter3_pipe + heter0_proc = subprocess.Popen( + heter0_cmd.strip().split(" "), + stdout=heter0_out, + stderr=heter0_pipe, + env=required_envs, + ) + heter1_proc = subprocess.Popen( + heter1_cmd.strip().split(" "), + stdout=heter1_out, + stderr=heter1_pipe, + env=required_envs, + ) + heter2_proc = subprocess.Popen( + heter2_cmd.strip().split(" "), + stdout=heter2_out, + stderr=heter2_pipe, + env=required_envs, + ) + heter3_proc = subprocess.Popen( + heter3_cmd.strip().split(" "), + stdout=heter3_out, + stderr=heter3_pipe, + env=required_envs, + ) + + return ( + heter0_proc, + heter1_proc, + heter2_proc, + heter3_proc, + heter0_pipe, + heter1_pipe, + heter2_pipe, + heter3_pipe, + ) def _run_cluster(self, model, envs): env = { 'GRAD_CLIP': str(self._grad_clip_mode), - 'FLAGS_eager_delete_tensor_gb': str(-1) + 'FLAGS_eager_delete_tensor_gb': str(-1), } python_path = self._python_interp gloo_path = tempfile.mkdtemp() @@ -312,31 +375,64 @@ def _run_cluster(self, model, envs): python_path += " -m coverage run --branch -p" env.update(envs) self._all_heter_endpoints = ";".join( - (self._heter_endpoints, self._heter_endpoints_2)) + (self._heter_endpoints, self._heter_endpoints_2) + ) tr_cmd = "{0} {1} --role trainer --endpoints {2} --trainer_endpoints {3} --current_id {{}} --trainers {4} --mode {5} --geo_sgd_need_push_nums {6} --reader {7} --gloo_path {8} --heter_trainer_endpoints {9} --heter_trainer_device {10}".format( - python_path, model, self._ps_endpoints, self._tr_endpoints, - self._trainers, self._mode, self._geo_sgd_need_push_nums, - self._reader, gloo_path, self._all_heter_endpoints, - self._heter_device) + python_path, + model, + self._ps_endpoints, + self._tr_endpoints, + self._trainers, + self._mode, + self._geo_sgd_need_push_nums, + self._reader, + gloo_path, + self._all_heter_endpoints, + self._heter_device, + ) ps_cmd = "{0} {1} --role pserver --endpoints {2} --trainer_endpoints {3} --current_id {{}} --trainers {4} --mode {5} --geo_sgd_need_push_nums {6} --reader {7} --gloo_path {8} --heter_trainer_endpoints {9} --heter_trainer_device {10}".format( - python_path, model, self._ps_endpoints, self._tr_endpoints, - self._trainers, self._mode, self._geo_sgd_need_push_nums, - self._reader, gloo_path, self._all_heter_endpoints, - self._heter_device) + python_path, + model, + self._ps_endpoints, + self._tr_endpoints, + self._trainers, + self._mode, + self._geo_sgd_need_push_nums, + self._reader, + gloo_path, + self._all_heter_endpoints, + self._heter_device, + ) heter_cmd = "{0} {1} --role heter_trainer --endpoints {2} --trainer_endpoints {3} --current_id {{}} --stage_id {{}} --trainers {4} --mode {5} --geo_sgd_need_push_nums {6} --reader {7} --gloo_path {8} --heter_trainer_endpoints {9} --heter_trainer_device {10}".format( - python_path, model, self._ps_endpoints, self._tr_endpoints, - self._trainers, self._mode, self._geo_sgd_need_push_nums, - self._reader, gloo_path, self._all_heter_endpoints, - self._heter_device) + python_path, + model, + self._ps_endpoints, + self._tr_endpoints, + self._trainers, + self._mode, + self._geo_sgd_need_push_nums, + self._reader, + gloo_path, + self._all_heter_endpoints, + self._heter_device, + ) # Run dist train to compare with local results ps0, ps1, ps0_pipe, ps1_pipe = self._start_pserver(ps_cmd, env) tr0, tr1, tr0_pipe, tr1_pipe = self._start_trainer(tr_cmd, env) - heter0, heter1, heter2, heter3, heter0_pipe, heter1_pipe, heter2_pipe, heter3_pipe = self._start_heter_trainer( - heter_cmd, env) + ( + heter0, + heter1, + heter2, + heter3, + heter0_pipe, + heter1_pipe, + heter2_pipe, + heter3_pipe, + ) = self._start_heter_trainer(heter_cmd, env) # Wait until trainer process terminate while True: @@ -379,17 +475,15 @@ def _run_cluster(self, model, envs): shutil.rmtree(gloo_path) return 0, 0 - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), "FLAGS_rpc_deadline": "5000", # 5sec to fail fast - "http_proxy": "" + "http_proxy": "", } required_envs.update(need_envs) @@ -403,32 +497,30 @@ def check_with_place(self, def runtime_main(test_class): parser = argparse.ArgumentParser(description='Run Fleet test.') - parser.add_argument('--role', - type=str, - required=True, - choices=['pserver', 'trainer', 'heter_trainer']) + parser.add_argument( + '--role', + type=str, + required=True, + choices=['pserver', 'trainer', 'heter_trainer'], + ) parser.add_argument('--endpoints', type=str, required=False, default="") - parser.add_argument('--trainer_endpoints', - type=str, - required=False, - default="") - parser.add_argument('--heter_trainer_endpoints', - type=str, - required=False, - default="") - parser.add_argument('--heter_trainer_device', - type=str, - required=False, - default="gpu") + parser.add_argument( + '--trainer_endpoints', type=str, required=False, default="" + ) + parser.add_argument( + '--heter_trainer_endpoints', type=str, required=False, default="" + ) + parser.add_argument( + '--heter_trainer_device', type=str, required=False, default="gpu" + ) parser.add_argument('--gloo_path', type=str, required=False, default="") parser.add_argument('--current_id', type=int, required=False, default=0) parser.add_argument('--trainers', type=int, required=False, default=1) parser.add_argument('--stage_id', type=int, required=False, default=1) parser.add_argument('--mode', type=str, required=False, default='async') - parser.add_argument('--geo_sgd_need_push_nums', - type=int, - required=False, - default=2) + parser.add_argument( + '--geo_sgd_need_push_nums', type=int, required=False, default=2 + ) parser.add_argument('--reader', type=str, required=False, default='dataset') args = parser.parse_args() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_program.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_program.py index bf7ced9ce6013..251a2bf25d9db 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_program.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_heter_program.py @@ -25,16 +25,18 @@ class TestDistFleetHeterProgram(unittest.TestCase): - def build_role(self): environs = {} environs[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36012,127.0.0.1:36013" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36012,127.0.0.1:36013" environs["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36014,127.0.0.1:36015" environs[ - "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST"] = "127.0.0.1:36016,127.0.0.1:36017" + "PADDLE_ALL_HETER_TRAINER_IP_PORT_LIST" + ] = "127.0.0.1:36016,127.0.0.1:36017" environs[ - "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST"] = "127.0.0.1:36014,127.0.0.1:36015" + "PADDLE_PREVIOUS_HETER_TRAINER_IP_PORT_LIST" + ] = "127.0.0.1:36014,127.0.0.1:36015" environs["PADDLE_HETER_TRAINER_DEVICE"] = "gpu" environs["TRAINING_ROLE"] = "HETER_TRAINER" environs["STAGE_ID"] = 2 @@ -58,20 +60,20 @@ def build_strategy(self): self.strategy.a_sync = True self.strategy.a_sync_configs = { "launch_barrier": False, - "heter_worker_device_guard": "gpu" + "heter_worker_device_guard": "gpu", } return self.strategy def build_input(self): - dense_input = fluid.layers.data(name="dense_input", - shape=[10], - dtype="float32") + dense_input = fluid.layers.data( + name="dense_input", shape=[10], dtype="float32" + ) sparse_input_ids = [ - fluid.layers.data(name="C" + str(i), - shape=[1], - lod_level=1, - dtype="int64") for i in range(1, 27) + fluid.layers.data( + name="C" + str(i), shape=[1], lod_level=1, dtype="int64" + ) + for i in range(1, 27) ] label = fluid.layers.data(name="label", shape=[1], dtype="float32") @@ -80,7 +82,6 @@ def build_input(self): return inputs def build_net(self, inputs): - def embedding_layer(input): return fluid.layers.embedding( input=input, @@ -88,7 +89,8 @@ def embedding_layer(input): size=[100001, 10], param_attr=fluid.ParamAttr( name="SparseFeatFactors", - initializer=fluid.initializer.Uniform()), + initializer=fluid.initializer.Uniform(), + ), ) sparse_embed_seq = list(map(embedding_layer, inputs[1:-1])) @@ -100,35 +102,50 @@ def embedding_layer(input): input=concated, size=400, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Normal( - scale=1 / math.sqrt(concated.shape[1]))), - name="fc1") + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Normal( + scale=1 / math.sqrt(concated.shape[1]) + ) + ), + name="fc1", + ) with fluid.device_guard("cpu"): fc2 = fluid.layers.fc( input=fc1, size=400, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Normal( - scale=1 / math.sqrt(fc1.shape[1]))), - name="fc2") + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Normal( + scale=1 / math.sqrt(fc1.shape[1]) + ) + ), + name="fc2", + ) with fluid.device_guard("gpu"): fc3 = fluid.layers.fc( input=fc2, size=400, act="relu", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Normal( - scale=1 / math.sqrt(fc2.shape[1]))), - name="fc3") + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Normal( + scale=1 / math.sqrt(fc2.shape[1]) + ) + ), + name="fc3", + ) with fluid.device_guard("cpu"): predict = fluid.layers.fc( input=fc3, size=2, act="softmax", - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Normal( - scale=1 / math.sqrt(fc3.shape[1]))), + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Normal( + scale=1 / math.sqrt(fc3.shape[1]) + ) + ), ) with fluid.device_guard("gpu"): diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_infer.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_infer.py index 362474e27678f..c0c7070cf803b 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_infer.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_infer.py @@ -20,16 +20,13 @@ class TestDistCtrInfer(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -54,35 +51,34 @@ def check_with_place(self, def test_dist_infer(self): model_dirname = tempfile.mkdtemp() - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False, - need_envs={ - "SAVE_DIRNAME": model_dirname, - }) + self.check_with_place( + "dist_fleet_ctr.py", + delta=1e-5, + check_error_log=False, + need_envs={ + "SAVE_DIRNAME": model_dirname, + }, + ) self._need_test = 1 self._model_dir = model_dirname - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) shutil.rmtree(model_dirname) class TestDistCtrTrainInfer(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" self._need_test = 1 - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), @@ -106,9 +102,9 @@ def check_with_place(self, tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train_infer(self): - self.check_with_place("dist_fleet_ctr.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps.py index 137beef5f2dff..8d5ac58d62aaa 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps.py @@ -34,35 +34,34 @@ class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -70,10 +69,9 @@ def get_loss(cos_q_pt, cos_q_nt): is_sparse = True # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding q_emb = fluid.layers.embedding( input=q, @@ -82,8 +80,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -95,14 +95,15 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding pt_emb = fluid.layers.embedding( input=pt, @@ -111,8 +112,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -124,13 +127,14 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding nt_emb = fluid.layers.embedding( input=nt, @@ -139,8 +143,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -152,8 +158,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -164,14 +172,18 @@ def get_loss(cos_q_pt, cos_q_nt): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = fleet.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = fleet.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss, acc, _ = self.net() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps10.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps10.py index 332f314106b7e..db553b990343b 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps10.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps10.py @@ -35,18 +35,17 @@ class TestExponentialDecay(unittest.TestCase): - def net(self): - input_data = paddle.static.data(name="sparse_input", - shape=[None, 1], - dtype="int64") - input_label = paddle.static.data(name="label", - shape=[None, 1], - dtype="int64") + input_data = paddle.static.data( + name="sparse_input", shape=[None, 1], dtype="int64" + ) + input_label = paddle.static.data( + name="label", shape=[None, 1], dtype="int64" + ) label = paddle.cast(input_label, dtype="float32") - embedding = paddle.static.nn.embedding(input_data, - is_sparse=True, - size=[1000, 128]) + embedding = paddle.static.nn.embedding( + input_data, is_sparse=True, size=[1000, 128] + ) fc1 = paddle.static.nn.fc(embedding, size=1024, activation="relu") fc2 = paddle.static.nn.fc(fc1, size=512, activation="relu") @@ -59,20 +58,24 @@ def net(self): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss = self.net() - scheduler = paddle.optimizer.lr.InverseTimeDecay(learning_rate=base_lr, - gamma=0.999, - verbose=True) + scheduler = paddle.optimizer.lr.InverseTimeDecay( + learning_rate=base_lr, gamma=0.999, verbose=True + ) optimizer = fluid.optimizer.Adam(scheduler) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps11.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps11.py index 38fccfa5d8865..b0d8df316a8ab 100755 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps11.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps11.py @@ -34,35 +34,34 @@ class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -78,7 +77,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -90,7 +91,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt @@ -102,7 +105,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -114,8 +119,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt nt = fluid.layers.data(name="3", shape=[1], dtype="int64", lod_level=1) # embedding @@ -125,7 +132,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -137,8 +146,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -155,9 +166,11 @@ def test(self): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36001,127.0.0.2:36001" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36002,127.0.0.2:36002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36002,127.0.0.2:36002" os.environ["TRAINING_ROLE"] = "TRAINER" os.environ["FLAGS_selected_gpus"] = "0" role = role_maker.PaddleCloudRoleMaker() @@ -191,22 +204,22 @@ def test_gpups_dataset(self): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() dataset._set_use_ps_gpu(True) - dataset.init(batch_size=32, - thread_num=3, - pipe_command="cat", - use_var=slots_vars) - dataset.set_filelist([ - "test_in_memory_dataset_run_a.txt", - "test_in_memory_dataset_run_b.txt" - ]) + dataset.init( + batch_size=32, thread_num=3, pipe_command="cat", use_var=slots_vars + ) + dataset.set_filelist( + [ + "test_in_memory_dataset_run_a.txt", + "test_in_memory_dataset_run_b.txt", + ] + ) os.remove("./test_in_memory_dataset_run_a.txt") os.remove("./test_in_memory_dataset_run_b.txt") diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps12.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps12.py index 814e49247dcaf..b4c10116a55c3 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps12.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps12.py @@ -37,35 +37,34 @@ class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -81,7 +80,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -93,7 +94,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt @@ -105,7 +108,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -117,8 +122,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt nt = fluid.layers.data(name="3", shape=[1], dtype="int64", lod_level=1) # embedding @@ -128,7 +135,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -140,8 +149,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -158,7 +169,8 @@ def test(self): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" os.environ["TRAINING_ROLE"] = "PSERVER" role = role_maker.PaddleCloudRoleMaker() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps13.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps13.py index 99dd076e82913..47cbaefd68d7b 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps13.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps13.py @@ -38,35 +38,34 @@ # this unittest is tested for SparseSharedAdamSGDRule class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = fluid.layers.mean(loss_op3) return avg_cost @@ -74,10 +73,9 @@ def get_loss(cos_q_pt, cos_q_nt): is_sparse = True # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding q_emb = fluid.contrib.layers.sparse_embedding( input=q, @@ -85,7 +83,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -97,14 +97,15 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding pt_emb = fluid.contrib.layers.sparse_embedding( input=pt, @@ -112,7 +113,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -124,13 +127,14 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding nt_emb = fluid.contrib.layers.sparse_embedding( input=nt, @@ -138,7 +142,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -150,8 +156,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -168,7 +176,8 @@ def test(self): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" os.environ["TRAINING_ROLE"] = "PSERVER" role = role_maker.PaddleCloudRoleMaker() @@ -180,10 +189,8 @@ def test(self): configs = {} configs['__emb__'] = { - "table_parameters.__emb__.accessor.embed_sgd_param.name": - "SparseSharedAdamSGDRule", - "table_parameters.__emb__.accessor.embedx_sgd_param.name": - "SparseSharedAdamSGDRule", + "table_parameters.__emb__.accessor.embed_sgd_param.name": "SparseSharedAdamSGDRule", + "table_parameters.__emb__.accessor.embedx_sgd_param.name": "SparseSharedAdamSGDRule", } strategy.sparse_table_configs = configs optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps2.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps2.py index 0ab59f7ca88d5..a4cdcb32bd412 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps2.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps2.py @@ -37,35 +37,34 @@ class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -73,10 +72,9 @@ def get_loss(cos_q_pt, cos_q_nt): is_sparse = True # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding q_emb = fluid.contrib.layers.sparse_embedding( input=q, @@ -84,7 +82,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -97,14 +97,15 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding pt_emb = fluid.contrib.layers.sparse_embedding( input=pt, @@ -112,7 +113,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -124,13 +127,14 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding nt_emb = fluid.contrib.layers.sparse_embedding( input=nt, @@ -138,7 +142,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -150,8 +156,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -168,7 +176,8 @@ def test(self): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" os.environ["TRAINING_ROLE"] = "PSERVER" role = role_maker.PaddleCloudRoleMaker() @@ -180,14 +189,10 @@ def test(self): configs = {} configs['__emb__'] = { - "table_parameters.__emb__.enable_sparse_table_cache": - True, - "table_parameters.__emb__.shard_merge_rate": - 1, - "table_parameters.__emb__.accessor.embed_sgd_param.name": - "SparseNaiveSGDRule", - "table_parameters.__emb__.accessor.embedx_sgd_param.name": - "SparseAdamSGDRule", + "table_parameters.__emb__.enable_sparse_table_cache": True, + "table_parameters.__emb__.shard_merge_rate": 1, + "table_parameters.__emb__.accessor.embed_sgd_param.name": "SparseNaiveSGDRule", + "table_parameters.__emb__.accessor.embedx_sgd_param.name": "SparseAdamSGDRule", } strategy.sparse_table_configs = configs optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps3.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps3.py index 8a7556a268559..2eb6277018441 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps3.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps3.py @@ -34,35 +34,34 @@ class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -70,10 +69,9 @@ def get_loss(cos_q_pt, cos_q_nt): is_sparse = False # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding q_emb = fluid.layers.embedding( input=q, @@ -82,8 +80,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -95,14 +95,15 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding pt_emb = fluid.layers.embedding( input=pt, @@ -111,8 +112,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -124,13 +127,14 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding nt_emb = fluid.layers.embedding( input=nt, @@ -139,8 +143,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -152,8 +158,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -164,14 +172,18 @@ def get_loss(cos_q_pt, cos_q_nt): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = fleet.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.WORKER, - worker_num=2, - server_endpoints=endpoints) + role = fleet.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.WORKER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss, acc, _ = self.net() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps4.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps4.py index fae0cd926187a..0cb4ee6e3af2f 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps4.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps4.py @@ -34,35 +34,34 @@ class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -70,10 +69,9 @@ def get_loss(cos_q_pt, cos_q_nt): is_sparse = True # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding q_emb = fluid.contrib.layers.sparse_embedding( input=q, @@ -81,7 +79,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -93,14 +93,15 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding pt_emb = fluid.contrib.layers.sparse_embedding( input=pt, @@ -108,7 +109,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -120,13 +123,14 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding nt_emb = fluid.contrib.layers.sparse_embedding( input=nt, @@ -134,7 +138,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -146,8 +152,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -158,14 +166,18 @@ def get_loss(cos_q_pt, cos_q_nt): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss, acc, _ = self.net() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps5.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps5.py index c91d86d89ca6b..fbb640fc8bbc8 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps5.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps5.py @@ -34,35 +34,34 @@ class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -70,10 +69,9 @@ def get_loss(cos_q_pt, cos_q_nt): is_sparse = True # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding q_emb = fluid.layers.embedding( input=q, @@ -82,8 +80,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -95,14 +95,15 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding pt_emb = fluid.layers.embedding( input=pt, @@ -111,8 +112,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr), - is_sparse=is_sparse) + learning_rate=emb_lr, + ), + is_sparse=is_sparse, + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -124,13 +127,14 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding nt_emb = fluid.layers.embedding( input=nt, @@ -139,8 +143,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__tmp_", - learning_rate=emb_lr), - is_sparse=False) + learning_rate=emb_lr, + ), + is_sparse=False, + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -152,8 +158,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -164,23 +172,30 @@ def get_loss(cos_q_pt, cos_q_nt): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss, acc, _ = self.net() optimizer = fluid.optimizer.Adam( - learning_rate=fluid.layers.exponential_decay(learning_rate=base_lr, - decay_steps=500, - decay_rate=0.969, - staircase=True)) + learning_rate=fluid.layers.exponential_decay( + learning_rate=base_lr, + decay_steps=500, + decay_rate=0.969, + staircase=True, + ) + ) strategy = paddle.distributed.fleet.DistributedStrategy() strategy.a_sync = True diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps6.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps6.py index 21dfcee7e8ed5..e0b73b4344c68 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps6.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps6.py @@ -34,35 +34,34 @@ class TestPSPassWithBow(unittest.TestCase): - def net(self): - def get_acc(cos_q_nt, cos_q_pt, batch_size): cond = fluid.layers.less_than(cos_q_nt, cos_q_pt) cond = fluid.layers.cast(cond, dtype='float64') cond_3 = fluid.layers.reduce_sum(cond) - acc = fluid.layers.elementwise_div(cond_3, - fluid.layers.fill_constant( - shape=[1], - value=batch_size * 1.0, - dtype='float64'), - name="simnet_acc") + acc = fluid.layers.elementwise_div( + cond_3, + fluid.layers.fill_constant( + shape=[1], value=batch_size * 1.0, dtype='float64' + ), + name="simnet_acc", + ) return acc def get_loss(cos_q_pt, cos_q_nt): loss_op1 = fluid.layers.elementwise_sub( - fluid.layers.fill_constant_batch_size_like(input=cos_q_pt, - shape=[-1, 1], - value=margin, - dtype='float32'), - cos_q_pt) + fluid.layers.fill_constant_batch_size_like( + input=cos_q_pt, shape=[-1, 1], value=margin, dtype='float32' + ), + cos_q_pt, + ) loss_op2 = fluid.layers.elementwise_add(loss_op1, cos_q_nt) loss_op3 = fluid.layers.elementwise_max( - fluid.layers.fill_constant_batch_size_like(input=loss_op2, - shape=[-1, 1], - value=0.0, - dtype='float32'), - loss_op2) + fluid.layers.fill_constant_batch_size_like( + input=loss_op2, shape=[-1, 1], value=0.0, dtype='float32' + ), + loss_op2, + ) avg_cost = paddle.mean(loss_op3) return avg_cost @@ -70,10 +69,9 @@ def get_loss(cos_q_pt, cos_q_nt): is_sparse = True # query - q = fluid.layers.data(name="query_ids", - shape=[1], - dtype="int64", - lod_level=1) + q = fluid.layers.data( + name="query_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding q_emb = fluid.contrib.layers.sparse_embedding( input=q, @@ -81,7 +79,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim]) # vsum q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum') @@ -93,14 +93,15 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__q_fc__", - learning_rate=base_lr)) + learning_rate=base_lr, + ), + ) # label data label = fluid.layers.data(name="label", shape=[1], dtype="int64") # pt - pt = fluid.layers.data(name="pos_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + pt = fluid.layers.data( + name="pos_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding pt_emb = fluid.contrib.layers.sparse_embedding( input=pt, @@ -108,7 +109,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim]) # vsum pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum') @@ -120,13 +123,14 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) # nt - nt = fluid.layers.data(name="neg_title_ids", - shape=[1], - dtype="int64", - lod_level=1) + nt = fluid.layers.data( + name="neg_title_ids", shape=[1], dtype="int64", lod_level=1 + ) # embedding nt_emb = fluid.contrib.layers.sparse_embedding( input=nt, @@ -134,7 +138,9 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__emb__", - learning_rate=emb_lr)) + learning_rate=emb_lr, + ), + ) nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim]) # vsum nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum') @@ -146,8 +152,10 @@ def get_loss(cos_q_pt, cos_q_nt): param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.01), name="__fc__", - learning_rate=base_lr), - bias_attr=fluid.ParamAttr(name="__fc_b__")) + learning_rate=base_lr, + ), + bias_attr=fluid.ParamAttr(name="__fc_b__"), + ) cos_q_pt = fluid.layers.cos_sim(q_fc, pt_fc) cos_q_nt = fluid.layers.cos_sim(q_fc, nt_fc) # loss @@ -158,14 +166,18 @@ def get_loss(cos_q_pt, cos_q_nt): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss, acc, _ = self.net() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps7.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps7.py index bdba0d90bb0f8..9351791260b2a 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps7.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps7.py @@ -36,18 +36,17 @@ class TestNaturalExpDecay(unittest.TestCase): - def net(self): - input_data = paddle.static.data(name="sparse_input", - shape=[None, 1], - dtype="int64") - input_label = paddle.static.data(name="label", - shape=[None, 1], - dtype="int64") + input_data = paddle.static.data( + name="sparse_input", shape=[None, 1], dtype="int64" + ) + input_label = paddle.static.data( + name="label", shape=[None, 1], dtype="int64" + ) label = paddle.cast(input_label, dtype="float32") - embedding = paddle.static.nn.embedding(input_data, - is_sparse=True, - size=[1000, 128]) + embedding = paddle.static.nn.embedding( + input_data, is_sparse=True, size=[1000, 128] + ) fc1 = paddle.static.nn.fc(embedding, size=1024, activation="relu") fc2 = paddle.static.nn.fc(fc1, size=512, activation="relu") @@ -60,20 +59,24 @@ def net(self): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss = self.net() - scheduler = paddle.optimizer.lr.NaturalExpDecay(learning_rate=base_lr, - gamma=0.999, - verbose=True) + scheduler = paddle.optimizer.lr.NaturalExpDecay( + learning_rate=base_lr, gamma=0.999, verbose=True + ) optimizer = fluid.optimizer.Adam(scheduler) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps8.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps8.py index 3dcb16f5444bc..cbf12f19714ff 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps8.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps8.py @@ -35,18 +35,17 @@ class TestNoamDecay(unittest.TestCase): - def net(self): - input_data = paddle.static.data(name="sparse_input", - shape=[None, 1], - dtype="int64") - input_label = paddle.static.data(name="label", - shape=[None, 1], - dtype="int64") + input_data = paddle.static.data( + name="sparse_input", shape=[None, 1], dtype="int64" + ) + input_label = paddle.static.data( + name="label", shape=[None, 1], dtype="int64" + ) label = paddle.cast(input_label, dtype="float32") - embedding = paddle.static.nn.embedding(input_data, - is_sparse=True, - size=[1000, 128]) + embedding = paddle.static.nn.embedding( + input_data, is_sparse=True, size=[1000, 128] + ) fc1 = paddle.static.nn.fc(embedding, size=1024, activation="relu") fc2 = paddle.static.nn.fc(fc1, size=512, activation="relu") @@ -59,20 +58,24 @@ def net(self): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss = self.net() - scheduler = paddle.optimizer.lr.NoamDecay(d_model=0.01, - warmup_steps=100, - verbose=True) + scheduler = paddle.optimizer.lr.NoamDecay( + d_model=0.01, warmup_steps=100, verbose=True + ) optimizer = fluid.optimizer.Adam(scheduler) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps9.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps9.py index 958ad4c6052f0..bebc6bbb96536 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ps9.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ps9.py @@ -35,18 +35,17 @@ class TestExponentialDecay(unittest.TestCase): - def net(self): - input_data = paddle.static.data(name="sparse_input", - shape=[None, 1], - dtype="int64") - input_label = paddle.static.data(name="label", - shape=[None, 1], - dtype="int64") + input_data = paddle.static.data( + name="sparse_input", shape=[None, 1], dtype="int64" + ) + input_label = paddle.static.data( + name="label", shape=[None, 1], dtype="int64" + ) label = paddle.cast(input_label, dtype="float32") - embedding = paddle.static.nn.embedding(input_data, - is_sparse=True, - size=[1000, 128]) + embedding = paddle.static.nn.embedding( + input_data, is_sparse=True, size=[1000, 128] + ) fc1 = paddle.static.nn.fc(embedding, size=1024, activation="relu") fc2 = paddle.static.nn.fc(fc1, size=512, activation="relu") @@ -59,20 +58,24 @@ def net(self): def test(self): endpoints = [ - "127.0.0.1:36004", "127.0.0.1:36005", "127.0.0.1:36006", - "127.0.0.1:36007" + "127.0.0.1:36004", + "127.0.0.1:36005", + "127.0.0.1:36006", + "127.0.0.1:36007", ] - role = role_maker.UserDefinedRoleMaker(current_id=0, - role=role_maker.Role.SERVER, - worker_num=2, - server_endpoints=endpoints) + role = role_maker.UserDefinedRoleMaker( + current_id=0, + role=role_maker.Role.SERVER, + worker_num=2, + server_endpoints=endpoints, + ) fleet.init(role) loss = self.net() - scheduler = paddle.optimizer.lr.ExponentialDecay(learning_rate=base_lr, - gamma=0.999, - verbose=True) + scheduler = paddle.optimizer.lr.ExponentialDecay( + learning_rate=base_lr, gamma=0.999, verbose=True + ) optimizer = fluid.optimizer.Adam(scheduler) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py index b49a2599b76a5..ca3c5badc0922 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer.py @@ -22,7 +22,6 @@ class TestFleetMetaOptimizerPrecision(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -34,11 +33,14 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_fleet_raw_program_optimizer.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_fleet_raw_program_optimizer.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer_fuse_allreduce.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer_fuse_allreduce.py index be85ea71040ab..a6b994dde6d78 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer_fuse_allreduce.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_raw_program_optimizer_fuse_allreduce.py @@ -22,7 +22,6 @@ class TestFleetMetaOptimizerAllReduceFusePrecision(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -34,12 +33,14 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place( "dist_fleet_raw_program_optimizer_fuse_allreduce.py", delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_simnet.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_simnet.py index b2bf73511ce0c..27c76bee6ed9c 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_simnet.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_simnet.py @@ -21,23 +21,20 @@ class TestDistSimnetASync2x2(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), "FLAGS_rpc_deadline": "5000", # 5sec to fail fast "http_proxy": "", - "CPU_NUM": "2" + "CPU_NUM": "2", } required_envs.update(need_envs) @@ -49,9 +46,9 @@ def check_with_place(self, tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_simnet_bow.py", - delta=1e-5, - check_error_log=True) + self.check_with_place( + "dist_fleet_simnet_bow.py", delta=1e-5, check_error_log=True + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_sparse_embedding_ctr.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_sparse_embedding_ctr.py index 7fc0cc60aecfd..13eaa703ddc0c 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_sparse_embedding_ctr.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_sparse_embedding_ctr.py @@ -28,16 +28,13 @@ @unittest.skip(reason="Skip unstable ut, need paddle sync mode fix") class TestDistMnistSync2x2(TestFleetBase): - def _setup_config(self): self._mode = "sync" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -58,22 +55,21 @@ def check_with_place(self, tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_sparse_embedding_ctr.py", - delta=1e-5, - check_error_log=True) + self.check_with_place( + "dist_fleet_sparse_embedding_ctr.py", + delta=1e-5, + check_error_log=True, + ) class TestDistMnistAsync2x2(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -94,22 +90,21 @@ def check_with_place(self, tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_sparse_embedding_ctr.py", - delta=1e-5, - check_error_log=True) + self.check_with_place( + "dist_fleet_sparse_embedding_ctr.py", + delta=1e-5, + check_error_log=True, + ) class TestDistMnistAsync2x2WithDecay(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -131,22 +126,21 @@ def check_with_place(self, tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_sparse_embedding_ctr.py", - delta=1e-5, - check_error_log=True) + self.check_with_place( + "dist_fleet_sparse_embedding_ctr.py", + delta=1e-5, + check_error_log=True, + ) class TestDistMnistAsync2x2WithUnifrom(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), @@ -168,20 +162,20 @@ def check_with_place(self, tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) def test_dist_train(self): - self.check_with_place("dist_fleet_sparse_embedding_ctr.py", - delta=1e-5, - check_error_log=True) + self.check_with_place( + "dist_fleet_sparse_embedding_ctr.py", + delta=1e-5, + check_error_log=True, + ) @unittest.skip(reason="Skip unstable ut, need tensor table to enhance") class TestDistMnistAsync2x2WithGauss(TestFleetBase): - def _setup_config(self): self._mode = "async" self._reader = "pyreader" def _run_local_infer(self, model_file): - def net(): """ network definition @@ -194,21 +188,27 @@ def net(): """ dnn_input_dim, lr_input_dim = 10, 10 - dnn_data = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - lr_data = fluid.layers.data(name="lr_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) - label = fluid.layers.data(name="click", - shape=[-1, 1], - dtype="int64", - lod_level=0, - append_batch_size=False) + dnn_data = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + lr_data = fluid.layers.data( + name="lr_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=0, + append_batch_size=False, + ) datas = [dnn_data, lr_data, label] @@ -220,10 +220,13 @@ def net(): input=dnn_data, size=[dnn_input_dim, dnn_layer_dims[0]], is_test=inference, - param_attr=fluid.ParamAttr(name="deep_embedding", - initializer=init)) - dnn_pool = fluid.layers.sequence_pool(input=dnn_embedding, - pool_type="sum") + param_attr=fluid.ParamAttr( + name="deep_embedding", initializer=init + ), + ) + dnn_pool = fluid.layers.sequence_pool( + input=dnn_embedding, pool_type="sum" + ) dnn_out = dnn_pool for i, dim in enumerate(dnn_layer_dims[1:]): fc = fluid.layers.fc( @@ -231,8 +234,10 @@ def net(): size=dim, act="relu", param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.01)), - name='dnn-fc-%d' % i) + initializer=fluid.initializer.Constant(value=0.01) + ), + name='dnn-fc-%d' % i, + ) dnn_out = fc # build lr model @@ -242,10 +247,13 @@ def net(): is_test=inference, param_attr=fluid.ParamAttr( name="wide_embedding", - initializer=fluid.initializer.Constant(value=0.01))) + initializer=fluid.initializer.Constant(value=0.01), + ), + ) - lr_pool = fluid.layers.sequence_pool(input=lr_embbding, - pool_type="sum") + lr_pool = fluid.layers.sequence_pool( + input=lr_embbding, pool_type="sum" + ) merge_layer = fluid.layers.concat(input=[dnn_out, lr_pool], axis=1) predict = fluid.layers.fc(input=merge_layer, size=2, act='softmax') return datas, predict @@ -259,15 +267,15 @@ def net(): fluid.io.load_persistables(exe, model_file) for batch_id, data in enumerate(reader()): - score = exe.run(fluid.default_main_program(), - feed=feeder.feed(data), - fetch_list=[predict]) - - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + score = exe.run( + fluid.default_main_program(), + feed=feeder.feed(data), + fetch_list=[predict], + ) + + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): model_dir = tempfile.mkdtemp() required_envs = { @@ -293,9 +301,11 @@ def check_with_place(self, shutil.rmtree(model_dir) def test_dist_train(self): - self.check_with_place("dist_fleet_sparse_embedding_ctr.py", - delta=1e-5, - check_error_log=True) + self.check_with_place( + "dist_fleet_sparse_embedding_ctr.py", + delta=1e-5, + check_error_log=True, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_trainer_desc_config.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_trainer_desc_config.py index 0090d6bef1597..e64b7d8010ef5 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_trainer_desc_config.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_trainer_desc_config.py @@ -23,15 +23,15 @@ class TestDistStrategyTrainerDescConfig(unittest.TestCase): - def setUp(self): os.environ["PADDLE_PSERVER_NUMS"] = "2" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36001" os.environ["PADDLE_TRAINER_ID"] = "0" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_trainer_desc_config(self): os.environ["TRAINING_ROLE"] = "TRAINER" @@ -50,7 +50,7 @@ def test_trainer_desc_config(self): config = { "dump_fields_path": "dump_data", "dump_fields": ["xxx", "yyy"], - "dump_param": ['zzz'] + "dump_param": ['zzz'], } strategy.trainer_desc_configs = config @@ -62,8 +62,10 @@ def test_trainer_desc_config(self): self.assertEqual(program._fleet_opt["dump_fields_path"], "dump_data") self.assertEqual(len(program._fleet_opt["dump_fields"]), 2) self.assertEqual(len(program._fleet_opt["dump_param"]), 1) - self.assertEqual(program._fleet_opt["mpi_size"], - int(os.environ["PADDLE_TRAINERS_NUM"])) + self.assertEqual( + program._fleet_opt["mpi_size"], + int(os.environ["PADDLE_TRAINERS_NUM"]), + ) optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) @@ -73,8 +75,10 @@ def test_trainer_desc_config(self): self.assertEqual(program._fleet_opt["dump_fields_path"], "dump_data") self.assertEqual(len(program._fleet_opt["dump_fields"]), 2) self.assertEqual(len(program._fleet_opt["dump_param"]), 1) - self.assertEqual(program._fleet_opt["mpi_size"], - int(os.environ["PADDLE_TRAINERS_NUM"])) + self.assertEqual( + program._fleet_opt["mpi_size"], + int(os.environ["PADDLE_TRAINERS_NUM"]), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_lookup_sparse_table_fuse_ops.py b/python/paddle/fluid/tests/unittests/test_dist_lookup_sparse_table_fuse_ops.py index 8a753e52de7c1..f64a7e6882e12 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_lookup_sparse_table_fuse_ops.py +++ b/python/paddle/fluid/tests/unittests/test_dist_lookup_sparse_table_fuse_ops.py @@ -25,7 +25,6 @@ @unittest.skip("do not need currently") class TestLookupTableFuseOp(unittest.TestCase): - def test_fuse(self): places = [core.CPUPlace()] # currently only support CPU @@ -44,20 +43,23 @@ def check_with_place(self, place): persistable=True, type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[1], - dtype="float32") + dtype="float32", + ) ids = init_program.global_block().create_var( name="Ids", persistable=True, type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[100], - dtype="int64") + dtype="int64", + ) output = init_program.global_block().create_var( name="output", type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[100, 8], - dtype="float32") + dtype="float32", + ) metas = [] metas.append( @@ -71,27 +73,30 @@ def check_with_place(self, place): type="lookup_sparse_table_init", inputs=None, outputs=None, - attrs={"large_scale_metas": metas}) - - init_program.global_block().append_op(type="lookup_sparse_table_read", - inputs={"Ids": ids}, - outputs={"Out": output}, - attrs={ - "tablename": - "embedding_1.block0", - "init": True, - "value_names": ["Param"], - }) - - init_program.global_block().append_op(type="lookup_sparse_table_read", - inputs={"Ids": ids}, - outputs={"Out": output}, - attrs={ - "tablename": - "embedding_2.block0", - "init": True, - "value_names": ["Param"], - }) + attrs={"large_scale_metas": metas}, + ) + + init_program.global_block().append_op( + type="lookup_sparse_table_read", + inputs={"Ids": ids}, + outputs={"Out": output}, + attrs={ + "tablename": "embedding_1.block0", + "init": True, + "value_names": ["Param"], + }, + ) + + init_program.global_block().append_op( + type="lookup_sparse_table_read", + inputs={"Ids": ids}, + outputs={"Out": output}, + attrs={ + "tablename": "embedding_2.block0", + "init": True, + "value_names": ["Param"], + }, + ) executor = fluid.Executor(place) executor.run(init_program) @@ -99,9 +104,11 @@ def check_with_place(self, place): training_program = fluid.Program() scope.var('Beta1Pow').get_tensor().set( - np.array([0]).astype("float32"), place) + np.array([0]).astype("float32"), place + ) scope.var('Beta2Pow').get_tensor().set( - np.array([0]).astype("float32"), place) + np.array([0]).astype("float32"), place + ) rows = [0, 1, 2, 3, 4, 5, 6] row_numel = 8 @@ -119,28 +126,32 @@ def check_with_place(self, place): persistable=True, type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[1], - dtype="float32") + dtype="float32", + ) grads = training_program.global_block().create_var( name="Grad", persistable=True, type=fluid.core.VarDesc.VarType.SELECTED_ROWS, shape=[100, 8], - dtype="float32") + dtype="float32", + ) beta1 = training_program.global_block().create_var( name="Beta1Pow", persistable=True, type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[1], - dtype="float32") + dtype="float32", + ) beta2 = training_program.global_block().create_var( name="Beta2Pow", persistable=True, type=fluid.core.VarDesc.VarType.LOD_TENSOR, shape=[1], - dtype="float32") + dtype="float32", + ) training_program.global_block().append_op( type="lookup_sparse_table_fuse_adam", @@ -150,27 +161,23 @@ def check_with_place(self, place): "Beta1Pow": beta1, "Beta2Pow": beta2, }, - outputs={ - "Beta1PowOut": beta1, - "Beta2PowOut": beta2 - }, + outputs={"Beta1PowOut": beta1, "Beta2PowOut": beta2}, attrs={ "is_entry": False, "tablename": "embedding_1.block0", "value_names": ["Param", "Moment1", "Moment2"], - }) + }, + ) training_program.global_block().append_op( type="lookup_sparse_table_fuse_sgd", - inputs={ - "Grad": grads, - "LearningRate": lr - }, + inputs={"Grad": grads, "LearningRate": lr}, attrs={ "is_entry": False, "tablename": "embedding_2.block0", "value_names": ["Param"], - }) + }, + ) executor.run(training_program) diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_backward_deps.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_backward_deps.py index 425e8c21aae3b..ba1037b05730c 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_backward_deps.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_backward_deps.py @@ -20,7 +20,6 @@ class TestDistMnistNCCL2BackWardDeps(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -30,6 +29,7 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_mnist.py", delta=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_batch_merge.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_batch_merge.py index b21e7b208b296..a480b98731755 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_batch_merge.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_batch_merge.py @@ -23,7 +23,6 @@ class TestDistMnist2x2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -31,11 +30,9 @@ def _setup_config(self): def test_dist_train(self): self.check_with_place("dist_mnist_batch_merge.py", delta=1e-5) - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}): + def check_with_place( + self, model_file, delta=1e-3, check_error_log=False, need_envs={} + ): # TODO(typhoonzero): should auto adapt GPU count on the machine. required_envs = { "PATH": os.getenv("PATH", ""), @@ -48,22 +45,27 @@ def check_with_place(self, required_envs.update(need_envs) if check_error_log: - required_envs["GLOG_vmodule"] = \ - "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10,alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10" + required_envs[ + "GLOG_vmodule" + ] = "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10,alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10" required_envs["GLOG_logtostderr"] = "1" - no_merge_losses = self._run_local(model_file, - required_envs, - check_error_log=check_error_log, - batch_size=4, - log_name=flag_name) + no_merge_losses = self._run_local( + model_file, + required_envs, + check_error_log=check_error_log, + batch_size=4, + log_name=flag_name, + ) - batch_merge_losses = self._run_local(model_file, - required_envs, - check_error_log=check_error_log, - batch_size=2, - batch_merge_repeat=2, - log_name=flag_name) + batch_merge_losses = self._run_local( + model_file, + required_envs, + check_error_log=check_error_log, + batch_size=2, + batch_merge_repeat=2, + log_name=flag_name, + ) # Ensure both result have values. self.assertGreater(len(no_merge_losses), 1) self.assertEqual(len(no_merge_losses), len(batch_merge_losses)) diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_fleet_save.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_fleet_save.py index a83a3e83eca06..9a1afd250d8ef 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_fleet_save.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_fleet_save.py @@ -22,7 +22,6 @@ class TestDistMnistFleetSave(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -78,12 +77,14 @@ def _test_saved_files(self, dirname): self._rm_temp_files(dirname) return True - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}, - log_name=""): + def check_with_place( + self, + model_file, + delta=1e-3, + check_error_log=False, + need_envs={}, + log_name="", + ): required_envs = self._get_required_envs(check_error_log, need_envs) tr0_losses, tr1_losses = self._run_cluster_nccl2( @@ -91,13 +92,15 @@ def check_with_place(self, required_envs, update_method='nccl2', check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) dirname = '/tmp' self._test_saved_files(dirname) def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_mnist.py", delta=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_fleetapi.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_fleetapi.py index 6766b1c1baea2..fbfdccfeff41c 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_fleetapi.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_fleetapi.py @@ -20,7 +20,6 @@ class TestDistMnistNCCL2FleetApi(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -31,20 +30,24 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place( "dist_mnist.py", delta=1e-5, check_error_log=True, - need_envs={'FLAGS_allreduce_record_one_event': '1'}) + need_envs={'FLAGS_allreduce_record_one_event': '1'}, + ) class FleetCollectiveTest(unittest.TestCase): - def test_open_sync_batch_norm(self): import paddle.fluid as fluid import paddle.fluid.incubate.fleet.base.role_maker as role_maker - from paddle.fluid.incubate.fleet.collective import fleet, DistributedStrategy + from paddle.fluid.incubate.fleet.collective import ( + fleet, + DistributedStrategy, + ) if not fluid.core.is_compiled_with_cuda(): # Operator "gen_nccl_id" has not been registered @@ -62,8 +65,9 @@ def test_open_sync_batch_norm(self): dist_strategy = DistributedStrategy() dist_strategy.sync_batch_norm = True - dist_optimizer = fleet.distributed_optimizer(optimizer, - strategy=dist_strategy) + dist_optimizer = fleet.distributed_optimizer( + optimizer, strategy=dist_strategy + ) dist_optimizer.minimize(loss) self.assertEqual(dist_strategy.exec_strategy.num_threads, 1) diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_fp16_allreduce.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_fp16_allreduce.py index cff295d5802c5..ce74d92a7b81a 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_fp16_allreduce.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_fp16_allreduce.py @@ -17,7 +17,6 @@ class TestDistMnist2x2FP16AllReduce(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -25,6 +24,7 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_mnist_fp16_allreduce.py", delta=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_hallreduce.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_hallreduce.py index 93640b06ee6f1..ca1b7299b8c63 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_hallreduce.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_hallreduce.py @@ -23,7 +23,6 @@ class TestDistMnistNCCL2HAllreduce(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -34,11 +33,14 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_mnist.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_mnist.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_lars.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_lars.py index 03a4da1f45e38..81b96e7c8991c 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_lars.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_lars.py @@ -17,7 +17,6 @@ class TestDistMnist2x2Lars(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_multi_comm.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_multi_comm.py index 57d89b2d23d31..82f09bcfae1d0 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_multi_comm.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_multi_comm.py @@ -23,7 +23,6 @@ class TestDistMnistNCCL2MultiNCCLComm(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -33,11 +32,14 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_mnist.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_mnist.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_pg.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_pg.py index b813e96971eec..e517e38978e48 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_pg.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_pg.py @@ -20,7 +20,6 @@ class TestDistMnistNCCL2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -29,13 +28,16 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_mnist.py", - delta=1, - need_envs={ - "FLAGS_enable_parallel_graph": "1", - "FLAGS_sync_nccl_allreduce": "1" - }) + self.check_with_place( + "dist_mnist.py", + delta=1, + need_envs={ + "FLAGS_enable_parallel_graph": "1", + "FLAGS_sync_nccl_allreduce": "1", + }, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_ring_allreduce.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_ring_allreduce.py index c455dbe270a36..40ae6686ed98c 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_ring_allreduce.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_ring_allreduce.py @@ -20,7 +20,6 @@ class TestDistMnistNCCL2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -29,6 +28,7 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_mnist.py", delta=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_train.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_train.py index af2392b68528b..f6dfa73984660 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_train.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_train.py @@ -21,55 +21,53 @@ class TestDistMnist2x2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False def test_dist_train(self): - self.check_with_place("dist_mnist.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_mnist.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestDistMnist2x2WithMemopt(TestDistBase): - def _setup_config(self): self._sync_mode = True self._mem_opt = True def test_dist_train(self): - self.check_with_place("dist_mnist.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_mnist.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestDistMnistAsync(TestDistBase): - def _setup_config(self): self._sync_mode = False self._use_reduce = False def test_dist_train(self): - self.check_with_place("dist_mnist.py", - delta=200, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_mnist.py", delta=200, check_error_log=True, log_name=flag_name + ) class TestDistMnistDcAsgd(TestDistBase): - def _setup_config(self): self._sync_mode = False self._dc_asgd = True def test_se_resnext(self): - self.check_with_place("dist_mnist.py", - delta=200, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_mnist.py", delta=200, check_error_log=True, log_name=flag_name + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist_with_program.py b/python/paddle/fluid/tests/unittests/test_dist_mnist_with_program.py index 4475b52ee79e4..83c9a340e32cb 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist_with_program.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist_with_program.py @@ -20,7 +20,6 @@ class TestDistMnistLocalSGDFleetApi(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -31,12 +30,12 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_mnist.py", delta=1e-5) class TestDistMnistGradAllReduceFleetApi(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -47,6 +46,7 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_mnist.py", delta=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_dist_oneps.py b/python/paddle/fluid/tests/unittests/test_dist_oneps.py index 7704a4c715efc..c6b25263936ac 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_oneps.py +++ b/python/paddle/fluid/tests/unittests/test_dist_oneps.py @@ -22,7 +22,6 @@ class TestTable(unittest.TestCase): - def test_table_tensor(self): table = Table() table.id = 1001 diff --git a/python/paddle/fluid/tests/unittests/test_dist_op.py b/python/paddle/fluid/tests/unittests/test_dist_op.py index 37835d54176f3..e627567b1afd9 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_op.py +++ b/python/paddle/fluid/tests/unittests/test_dist_op.py @@ -23,7 +23,7 @@ def dist(x, y, p): - if p == 0.: + if p == 0.0: out = np.count_nonzero(x - y) elif p == float("inf"): out = np.max(np.abs(x - y)) @@ -35,7 +35,6 @@ def dist(x, y, p): class TestDistOp(OpTest): - def setUp(self): self.op_type = 'dist' self.python_api = paddle.dist @@ -44,7 +43,7 @@ def setUp(self): self.init_data_type() self.inputs = { "X": np.random.random(self.x_shape).astype(self.data_type), - "Y": np.random.random(self.y_shape).astype(self.data_type) + "Y": np.random.random(self.y_shape).astype(self.data_type), } self.attrs["p"] = self.p @@ -54,13 +53,14 @@ def setUp(self): self.gradient = self.calc_gradient() def init_case(self): - self.x_shape = (120) - self.y_shape = (120) - self.p = 0. + self.x_shape = 120 + self.y_shape = 120 + self.p = 0.0 def init_data_type(self): - self.data_type = np.float32 if core.is_compiled_with_rocm( - ) else np.float64 + self.data_type = ( + np.float32 if core.is_compiled_with_rocm() else np.float64 + ) def calc_gradient(self): x = self.inputs["X"] @@ -75,8 +75,11 @@ def calc_gradient(self): grad[x_minux_y_abs != norm] = 0 else: norm = dist(x, y, p) - grad = np.power(norm, 1 - p) * np.power(np.abs(x - y), - p - 1) * np.sign(x - y) + grad = ( + np.power(norm, 1 - p) + * np.power(np.abs(x - y), p - 1) + * np.sign(x - y) + ) def get_reduce_dims(x, y): x_reduce_dims = [] @@ -111,30 +114,29 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(["X", "Y"], - "Out", - user_defined_grads=self.gradient, - check_eager=True) + self.check_grad( + ["X", "Y"], + "Out", + user_defined_grads=self.gradient, + check_eager=True, + ) class TestDistOpCase1(TestDistOp): - def init_case(self): self.x_shape = (3, 5, 5, 6) self.y_shape = (5, 5, 6) - self.p = 1. + self.p = 1.0 class TestDistOpCase2(TestDistOp): - def init_case(self): self.x_shape = (10, 10) self.y_shape = (4, 10, 10) - self.p = 2. + self.p = 2.0 class TestDistOpCase3(TestDistOp): - def init_case(self): self.x_shape = (15, 10) self.y_shape = (15, 10) @@ -142,7 +144,6 @@ def init_case(self): class TestDistOpCase4(TestDistOp): - def init_case(self): self.x_shape = (2, 3, 4, 5, 8) self.y_shape = (3, 1, 5, 8) @@ -150,7 +151,6 @@ def init_case(self): class TestDistOpCase5(TestDistOp): - def init_case(self): self.x_shape = (4, 1, 4, 8) self.y_shape = (2, 2, 1, 4, 4, 8) @@ -158,10 +158,10 @@ def init_case(self): class TestDistAPI(unittest.TestCase): - def init_data_type(self): - self.data_type = 'float32' if core.is_compiled_with_rocm( - ) else 'float64' + self.data_type = ( + 'float32' if core.is_compiled_with_rocm() else 'float64' + ) def test_api(self): self.init_data_type() @@ -174,15 +174,17 @@ def test_api(self): x_i = np.random.random((2, 3, 4, 5)).astype(self.data_type) y_i = np.random.random((3, 1, 5)).astype(self.data_type) result = paddle.dist(x, y, p) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - out = exe.run(fluid.default_main_program(), - feed={ - 'x': x_i, - 'y': y_i - }, - fetch_list=[result]) + out = exe.run( + fluid.default_main_program(), + feed={'x': x_i, 'y': y_i}, + fetch_list=[result], + ) np.testing.assert_allclose(dist(x_i, y_i, p), out[0], rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_dist_save_load.py b/python/paddle/fluid/tests/unittests/test_dist_save_load.py index bee267b941a24..d4de87866227f 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_dist_save_load.py @@ -27,29 +27,31 @@ class TestDistSaveLoadDense2x2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._enforce_place = "CPU" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}, - log_name=""): + def check_with_place( + self, + model_file, + delta=1e-3, + check_error_log=False, + need_envs={}, + log_name="", + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), - "http_proxy": "" + "http_proxy": "", } required_envs.update(need_envs) if check_error_log: - required_envs["GLOG_vmodule"] = \ - "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10,alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10" + required_envs[ + "GLOG_vmodule" + ] = "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10,alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10" required_envs["GLOG_logtostderr"] = "1" model_dir = tempfile.mkdtemp() @@ -65,10 +67,9 @@ def check_with_place(self, cluster_env.update(required_envs) local_var = self._run_local(model_file, local_env, check_error_log) - tr0_var, tr1_var = self._run_cluster(model_file, - cluster_env, - check_error_log, - log_name=flag_name) + tr0_var, tr1_var = self._run_cluster( + model_file, cluster_env, check_error_log, log_name=flag_name + ) shutil.rmtree(model_dir) @@ -87,36 +88,40 @@ def test_dist(self): 'IS_SELF_CONTAINED_LR': '1', 'SAVE_MODE': 'LOCAL', } - self.check_with_place("dist_save_load.py", - delta=0, - check_error_log=False, - need_envs=need_envs) + self.check_with_place( + "dist_save_load.py", + delta=0, + check_error_log=False, + need_envs=need_envs, + ) class TestDistSaveLoadWithPServerStateDense2x2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._enforce_place = "CPU" - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=False, - need_envs={}, - log_name=""): + def check_with_place( + self, + model_file, + delta=1e-3, + check_error_log=False, + need_envs={}, + log_name="", + ): required_envs = { "PATH": os.getenv("PATH", ""), "PYTHONPATH": os.getenv("PYTHONPATH", ""), "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), - "http_proxy": "" + "http_proxy": "", } required_envs.update(need_envs) if check_error_log: - required_envs["GLOG_vmodule"] = \ - "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10,alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10" + required_envs[ + "GLOG_vmodule" + ] = "fused_all_reduce_op_handle=10,all_reduce_op_handle=10,alloc_continuous_space_op=10,fuse_all_reduce_op_pass=10,alloc_continuous_space_for_grad_pass=10,fast_threaded_ssa_graph_executor=10" required_envs["GLOG_logtostderr"] = "1" model_dir = tempfile.mkdtemp() @@ -127,19 +132,17 @@ def check_with_place(self, save_env["MODEL_DIR"] = model_dir save_env.update(required_envs) - tr0_var_1, tr1_var_1 = self._run_cluster(model_file, - save_env, - check_error_log, - log_name=flag_name) + tr0_var_1, tr1_var_1 = self._run_cluster( + model_file, save_env, check_error_log, log_name=flag_name + ) load_env = {} load_env["LOAD"] = "1" load_env["MODEL_DIR"] = model_dir load_env.update(required_envs) - tr0_var_2, tr1_var_2 = self._run_cluster(model_file, - load_env, - check_error_log, - log_name=flag_name) + tr0_var_2, tr1_var_2 = self._run_cluster( + model_file, load_env, check_error_log, log_name=flag_name + ) shutil.rmtree(model_dir) @@ -158,13 +161,15 @@ def test_dist(self): 'IS_SELF_CONTAINED_LR': '1', 'SAVE_MODE': 'DIST', 'OPTIMIZER': 'ADAM', - 'SKIP_STEPS': str(np.random.randint(2, 6)) + 'SKIP_STEPS': str(np.random.randint(2, 6)), } - self.check_with_place("dist_save_load.py", - delta=0, - check_error_log=True, - need_envs=need_envs, - log_name=flag_name) + self.check_with_place( + "dist_save_load.py", + delta=0, + check_error_log=True, + need_envs=need_envs, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_se_resnext_nccl.py b/python/paddle/fluid/tests/unittests/test_dist_se_resnext_nccl.py index aeacf2c00a6d7..35ffbbb80ca04 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_se_resnext_nccl.py +++ b/python/paddle/fluid/tests/unittests/test_dist_se_resnext_nccl.py @@ -24,7 +24,6 @@ class TestDistSeResneXtNCCL(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reader_alloc = False @@ -32,15 +31,17 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_se_resnext.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_se_resnext.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestDistSeResneXtNCCLMP(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reader_alloc = False @@ -49,12 +50,15 @@ def _setup_config(self): def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("dist_se_resnext.py", - delta=1e-5, - check_error_log=True, - need_envs={"NCCL_P2P_DISABLE": "1"}, - log_name=flag_name) + self.check_with_place( + "dist_se_resnext.py", + delta=1e-5, + check_error_log=True, + need_envs={"NCCL_P2P_DISABLE": "1"}, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_se_resnext_sync.py b/python/paddle/fluid/tests/unittests/test_dist_se_resnext_sync.py index 2d288df2ae4a3..cc5905506e6d0 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_se_resnext_sync.py +++ b/python/paddle/fluid/tests/unittests/test_dist_se_resnext_sync.py @@ -22,17 +22,18 @@ class TestDistSeResneXt2x2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reader_alloc = False @unittest.skip(reason="Skip unstable ci") def test_dist_train(self): - self.check_with_place("dist_se_resnext.py", - delta=1e-7, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_se_resnext.py", + delta=1e-7, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_sharding_save.py b/python/paddle/fluid/tests/unittests/test_dist_sharding_save.py index 43c849d8a8c38..29442a9fb6838 100755 --- a/python/paddle/fluid/tests/unittests/test_dist_sharding_save.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sharding_save.py @@ -22,7 +22,6 @@ class TestDistMnistFleetSave(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -40,11 +39,19 @@ def _test_saved_files(self, dirname): sharding_save_files = sorted(os.listdir(dirname)) check_files = [ - 'fc_0.b_0', 'fc_0.b_0_velocity_0', 'fc_0.w_0', - 'fc_0.w_0_velocity_0', 'fc_1.b_0', 'fc_1.b_0_velocity_0', - 'fc_1.w_0', 'fc_1.w_0_velocity_0', 'fc_2.b_0', - 'fc_2.b_0_velocity_0', 'fc_2.w_0', 'fc_2.w_0_velocity_0', - 'learning_rate_0' + 'fc_0.b_0', + 'fc_0.b_0_velocity_0', + 'fc_0.w_0', + 'fc_0.w_0_velocity_0', + 'fc_1.b_0', + 'fc_1.b_0_velocity_0', + 'fc_1.w_0', + 'fc_1.w_0_velocity_0', + 'fc_2.b_0', + 'fc_2.b_0_velocity_0', + 'fc_2.w_0', + 'fc_2.w_0_velocity_0', + 'learning_rate_0', ] if sharding_save_files != check_files: @@ -54,12 +61,14 @@ def _test_saved_files(self, dirname): return True - def check_with_place(self, - model_file, - delta=1e-3, - check_error_log=True, - need_envs={}, - log_name=""): + def check_with_place( + self, + model_file, + delta=1e-3, + check_error_log=True, + need_envs={}, + log_name="", + ): required_envs = self._get_required_envs(check_error_log, need_envs) tr0_losses, tr1_losses = self._run_cluster_nccl2( @@ -67,13 +76,15 @@ def check_with_place(self, required_envs, update_method='nccl2', check_error_log=check_error_log, - log_name=log_name) + log_name=log_name, + ) dirname = './ut_sharding_save_model' self._test_saved_files(dirname) def test_dist_train(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place("dist_sharding_save.py", delta=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps0.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps0.py index b4aa24bc9e3f2..bff5754df1fe8 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps0.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps0.py @@ -24,8 +24,7 @@ class SparseLoadOp(unittest.TestCase): - """ Test load operator. - """ + """Test load operator.""" def net(self, emb_array, fc_array): with fluid.unique_name.guard(): @@ -38,7 +37,9 @@ def net(self, emb_array, fc_array): param_attr=fluid.ParamAttr( name="embedding", initializer=fluid.initializer.NumpyArrayInitializer( - emb_array)), + emb_array + ), + ), ) fc1 = fluid.layers.fc( @@ -48,7 +49,10 @@ def net(self, emb_array, fc_array): param_attr=fluid.ParamAttr( name='fc', initializer=fluid.initializer.NumpyArrayInitializer( - fc_array))) + fc_array + ), + ), + ) loss = fluid.layers.reduce_mean(fc1) return loss @@ -70,7 +74,6 @@ def save_origin_model(self, emb_array, fc_array): @unittest.skip(reason="Skip unstable ut, need rewrite with new implement") class TestSparseLoadOpCase1(SparseLoadOp): - def test_2ps_0_load(self): # init No.0 server env env = {} @@ -110,7 +113,8 @@ def test_2ps_0_load(self): fc_w = np.array(fluid.global_scope().find_var("fc").get_tensor()) emb = np.array( - fluid.global_scope().find_var("embedding.block0").get_tensor()) + fluid.global_scope().find_var("embedding.block0").get_tensor() + ) assert fc_w.all() == fc_array.all() assert emb.all() == emb_array[::2].all() diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps1.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps1.py index bbbddd5d7c1ba..1152ff3e82b20 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps1.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_load_ps1.py @@ -25,7 +25,6 @@ @unittest.skip(reason="Skip unstable ut, need rewrite with new implement") class TestSparseLoadOpCase2(SparseLoadOp): - def test_2ps_0_load(self): # init No.1 server env env = {} @@ -64,7 +63,8 @@ def test_2ps_0_load(self): optimizer.minimize(loss) fleet.init_server(model_path) emb = np.array( - fluid.global_scope().find_var("embedding.block1").get_tensor()) + fluid.global_scope().find_var("embedding.block1").get_tensor() + ) assert emb.all() == emb_array[1::2].all() shutil.rmtree(model_path) diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adagrad.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adagrad.py index 32e5a0ac1f3e0..5fca7c3afa7b7 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adagrad.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adagrad.py @@ -29,8 +29,9 @@ def test_server_init(self): with fluid.scope_guard(scope): with fluid.program_guard(train_program, startup_program): optimizer = fluid.optimizer.Adam(1e-3) - optimizer = fleet.distributed_optimizer(optimizer, - self.strategy) + optimizer = fleet.distributed_optimizer( + optimizer, self.strategy + ) optimizer.minimize(loss) fleet.init_server() diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adam.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adam.py index 292e6e64fb5c3..8305ade7b3e98 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adam.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_adam.py @@ -29,8 +29,9 @@ def test_server_init(self): with fluid.scope_guard(scope): with fluid.program_guard(train_program, startup_program): optimizer = fluid.optimizer.Adam(1e-3) - optimizer = fleet.distributed_optimizer(optimizer, - self.strategy) + optimizer = fleet.distributed_optimizer( + optimizer, self.strategy + ) optimizer.minimize(loss) fleet.init_server() diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_ftrl.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_ftrl.py index 5bee7b73ef58a..8193e20037a38 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_ftrl.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_ftrl.py @@ -29,8 +29,9 @@ def test_server_init(self): with fluid.scope_guard(scope): with fluid.program_guard(train_program, startup_program): optimizer = fluid.optimizer.SGD(1e-3) - optimizer = fleet.distributed_optimizer(optimizer, - self.strategy) + optimizer = fleet.distributed_optimizer( + optimizer, self.strategy + ) optimizer.minimize(loss) fleet.init_server() diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_momentum.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_momentum.py index e3b01ee0b5002..83fc785bddd9f 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_momentum.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_momentum.py @@ -29,8 +29,9 @@ def test_server_init(self): with fluid.scope_guard(scope): with fluid.program_guard(train_program, startup_program): optimizer = fluid.optimizer.SGD(1e-3) - optimizer = fleet.distributed_optimizer(optimizer, - self.strategy) + optimizer = fleet.distributed_optimizer( + optimizer, self.strategy + ) optimizer.minimize(loss) fleet.init_server() diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_rmsprop.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_rmsprop.py index b3d9cde8b0a9d..049a9043cf1a2 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_rmsprop.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_rmsprop.py @@ -29,8 +29,9 @@ def test_server_init(self): with fluid.scope_guard(scope): with fluid.program_guard(train_program, startup_program): optimizer = fluid.optimizer.SGD(1e-3) - optimizer = fleet.distributed_optimizer(optimizer, - self.strategy) + optimizer = fleet.distributed_optimizer( + optimizer, self.strategy + ) optimizer.minimize(loss) fleet.init_server() diff --git a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_sgd.py b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_sgd.py index b3b20df47daec..42737d3d69b4f 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_sgd.py +++ b/python/paddle/fluid/tests/unittests/test_dist_sparse_tensor_load_sgd.py @@ -27,7 +27,8 @@ class TestSparseLoadProgram(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" os.environ["PADDLE_TRAINERS_NUM"] = str(2) os.environ["TRAINING_ROLE"] = "PSERVER" os.environ["PADDLE_PORT"] = "4001" @@ -45,9 +46,9 @@ def net(self): with fluid.program_guard(train_program, startup_program): with fluid.unique_name.guard(): inputs = fluid.data('input', shape=[None, 1], dtype="int64") - emb = fluid.layers.embedding(inputs, - is_sparse=True, - size=[10000, 128]) + emb = fluid.layers.embedding( + inputs, is_sparse=True, size=[10000, 128] + ) fc1 = fluid.layers.fc(input=emb, size=128, act="relu") fc2 = fluid.layers.fc(input=fc1, size=64, act="relu") loss = fluid.layers.reduce_mean(fc2) @@ -55,14 +56,14 @@ def net(self): class TestSparseLoadProgramSGD(TestSparseLoadProgram): - def test_server_init(self): scope, train_program, startup_program, loss = self.net() with fluid.scope_guard(scope): with fluid.program_guard(train_program, startup_program): optimizer = fluid.optimizer.SGD(1e-3) - optimizer = fleet.distributed_optimizer(optimizer, - self.strategy) + optimizer = fleet.distributed_optimizer( + optimizer, self.strategy + ) optimizer.minimize(loss) fleet.init_server() diff --git a/python/paddle/fluid/tests/unittests/test_dist_text_classification.py b/python/paddle/fluid/tests/unittests/test_dist_text_classification.py index 7b16aa3f20040..36215f4e26c48 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_text_classification.py +++ b/python/paddle/fluid/tests/unittests/test_dist_text_classification.py @@ -22,29 +22,31 @@ class TestDistTextClassification2x2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._enforce_place = "CPU" def test_text_classification(self): - self.check_with_place("dist_text_classification.py", - delta=1e-6, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_text_classification.py", + delta=1e-6, + check_error_log=True, + log_name=flag_name, + ) class TestDistTextClassification2x2Async(TestDistBase): - def _setup_config(self): self._sync_mode = False self._enforce_place = "CPU" def test_se_resnext(self): - self.check_with_place("dist_text_classification.py", - delta=100, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_text_classification.py", + delta=100, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_train.py b/python/paddle/fluid/tests/unittests/test_dist_train.py index 7299854241907..65753803282fa 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_train.py +++ b/python/paddle/fluid/tests/unittests/test_dist_train.py @@ -30,19 +30,19 @@ from paddle.fluid import core -RPC_OP_ROLE_ATTR_NAME = op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName( -) +RPC_OP_ROLE_ATTR_NAME = ( + op_role_attr_name +) = core.op_proto_and_checker_maker.kOpRoleAttrName() RPC_OP_ROLE_ATTR_VALUE = core.op_proto_and_checker_maker.OpRole.RPC class TestSendOp(unittest.TestCase): - def test_send(self): remove_ps_flag(os.getpid()) # Run init_serv in a thread place = fluid.CPUPlace() # NOTE: python thread will not work here due to GIL. - p = Process(target=self.init_serv, args=(place, )) + p = Process(target=self.init_serv, args=(place,)) p.daemon = True p.start() @@ -79,14 +79,18 @@ def init_serv(self, place): with fluid.program_guard(main): serv = ListenAndServ("127.0.0.1:0", ["X"], optimizer_mode=False) with serv.do(): - out_var = main.global_block().create_var(name="scale_0.tmp_0", - psersistable=True, - dtype="float32", - shape=[32, 32]) - x = layers.data(shape=[32, 32], - dtype='float32', - name="X", - append_batch_size=False) + out_var = main.global_block().create_var( + name="scale_0.tmp_0", + psersistable=True, + dtype="float32", + shape=[32, 32], + ) + x = layers.data( + shape=[32, 32], + dtype='float32', + name="X", + append_batch_size=False, + ) fluid.initializer.Constant(value=1.0)(x, main.global_block()) ops._scale(x=x, scale=10.0, out=out_var) @@ -96,20 +100,22 @@ def init_serv(self, place): def init_client(self, place, port): main = fluid.Program() with fluid.program_guard(main): - main.global_block().append_op(type="fetch_barrier", - inputs={}, - outputs={"Out": []}, - attrs={ - "endpoints": - ["127.0.0.1:{0}".format(port)], - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) - - x = layers.data(shape=[32, 32], - dtype='float32', - name='X', - append_batch_size=False) + main.global_block().append_op( + type="fetch_barrier", + inputs={}, + outputs={"Out": []}, + attrs={ + "endpoints": ["127.0.0.1:{0}".format(port)], + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) + + x = layers.data( + shape=[32, 32], + dtype='float32', + name='X', + append_batch_size=False, + ) x.persistable = True fluid.initializer.Constant(value=2.3)(x, main.global_block()) @@ -117,7 +123,8 @@ def init_client(self, place, port): name="scale_0.tmp_0", # server side var dtype="float32", persistable=False, - shape=[32, 32]) + shape=[32, 32], + ) fluid.initializer.Constant(value=2.3)(get_var, main.global_block()) # NOTE(zjl): `Send` is async send, which means that the sent @@ -136,10 +143,12 @@ def init_client(self, place, port): def run_local(self, place): main = fluid.Program() with fluid.program_guard(main): - x = layers.data(shape=[32, 32], - dtype='float32', - name='X', - append_batch_size=False) + x = layers.data( + shape=[32, 32], + dtype='float32', + name='X', + append_batch_size=False, + ) fluid.initializer.Constant(value=2.3)(x, main.global_block()) o = layers.scale(x=x, scale=10.0) exe = fluid.Executor(place) diff --git a/python/paddle/fluid/tests/unittests/test_dist_transformer.py b/python/paddle/fluid/tests/unittests/test_dist_transformer.py index a4d5a5a2cbfae..470468f7fe3a1 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_transformer.py +++ b/python/paddle/fluid/tests/unittests/test_dist_transformer.py @@ -22,59 +22,65 @@ def download_files(): url_prefix = 'http://paddle-unittest-data.bj.bcebos.com/dist_transformer/' vocab_url = url_prefix + 'vocab.bpe.32000' vocab_md5 = 'a86d345ca6e27f6591d0dccb1b9be853' - paddle.dataset.common.download(vocab_url, 'test_dist_transformer', - vocab_md5) + paddle.dataset.common.download( + vocab_url, 'test_dist_transformer', vocab_md5 + ) local_train_url = url_prefix + 'train.tok.clean.bpe.32000.en-de' local_train_md5 = '033eb02b9449e6dd823f050782ac8914' - paddle.dataset.common.download(local_train_url, 'test_dist_transformer', - local_train_md5) + paddle.dataset.common.download( + local_train_url, 'test_dist_transformer', local_train_md5 + ) train0_url = url_prefix + 'train.tok.clean.bpe.32000.en-de.train_0' train0_md5 = 'ddce7f602f352a0405267285379a38b1' - paddle.dataset.common.download(train0_url, 'test_dist_transformer', - train0_md5) + paddle.dataset.common.download( + train0_url, 'test_dist_transformer', train0_md5 + ) train1_url = url_prefix + 'train.tok.clean.bpe.32000.en-de.train_1' train1_md5 = '8757798200180285b1a619cd7f408747' - paddle.dataset.common.download(train1_url, 'test_dist_transformer', - train1_md5) + paddle.dataset.common.download( + train1_url, 'test_dist_transformer', train1_md5 + ) test_url = url_prefix + 'newstest2013.tok.bpe.32000.en-de' test_md5 = '9dd74a266dbdb25314183899f269b4a2' paddle.dataset.common.download(test_url, 'test_dist_transformer', test_md5) # cut test data for faster CI - orig_path = os.path.join(paddle.dataset.common.DATA_HOME, - "test_dist_transformer", - "newstest2013.tok.bpe.32000.en-de") - head_path = os.path.join(paddle.dataset.common.DATA_HOME, - "test_dist_transformer", - "newstest2013.tok.bpe.32000.en-de.cut") + orig_path = os.path.join( + paddle.dataset.common.DATA_HOME, + "test_dist_transformer", + "newstest2013.tok.bpe.32000.en-de", + ) + head_path = os.path.join( + paddle.dataset.common.DATA_HOME, + "test_dist_transformer", + "newstest2013.tok.bpe.32000.en-de.cut", + ) os.system("head -n10 %s > %s" % (orig_path, head_path)) class TestDistTransformer2x2Sync(TestDistBase): - def _setup_config(self): self._sync_mode = True def test_dist_train(self): download_files() - self.check_with_place("dist_transformer.py", - delta=1e-5, - check_error_log=False) + self.check_with_place( + "dist_transformer.py", delta=1e-5, check_error_log=False + ) class TestDistTransformer2x2Async(TestDistBase): - def _setup_config(self): self._sync_mode = False def test_dist_train(self): download_files() - self.check_with_place("dist_transformer.py", - delta=1.0, - check_error_log=False) + self.check_with_place( + "dist_transformer.py", delta=1.0, check_error_log=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_transpiler.py b/python/paddle/fluid/tests/unittests/test_dist_transpiler.py index 8fc636af2de05..b5419eca108e9 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_transpiler.py +++ b/python/paddle/fluid/tests/unittests/test_dist_transpiler.py @@ -27,7 +27,6 @@ class TranspilerTest(unittest.TestCase): - def setUp(self): self.trainer_id = 0 self.trainers = 2 @@ -41,11 +40,13 @@ def setUp(self): def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -68,8 +69,8 @@ def get_trainer(self, config=None, sync_mode=True): trainer_main = t.get_trainer_program(wait_port=False) trainer_startup = fluid.default_startup_program() - assert (src.num_blocks == 1) - assert (trainer_startup.num_blocks == src.num_blocks) + assert src.num_blocks == 1 + assert trainer_startup.num_blocks == src.num_blocks return trainer_main, trainer_startup @@ -83,11 +84,13 @@ def _transpiler_instance(self, config=None, sync_mode=True): if not self.transpiler: main = self.get_main_program() self.transpiler = fluid.DistributeTranspiler(config=config) - self.transpiler.transpile(self.trainer_id, - program=main, - pservers=self.pserver_eps, - trainers=self.trainers, - sync_mode=sync_mode) + self.transpiler.transpile( + self.trainer_id, + program=main, + pservers=self.pserver_eps, + trainers=self.trainers, + sync_mode=sync_mode, + ) return self.transpiler @@ -109,7 +112,6 @@ def test_transpiler(self): class TestBasicModel(TranspilerTest): - def transpiler_test_impl(self): pserver, startup = self.get_pserver(self.pserver1_ep) pserver2, startup2 = self.get_pserver(self.pserver2_ep) @@ -125,28 +127,57 @@ def transpiler_test_impl(self): self.assertTrue("fc_b@GRAD" not in trainer_startup.global_block().vars) src = [op.type for op in trainer_startup.global_block().ops] - dst = ['fill_constant', 'fill_constant', 'uniform_random', 'recv', 'recv', \ - 'fetch_barrier', 'concat'] + dst = [ + 'fill_constant', + 'fill_constant', + 'uniform_random', + 'recv', + 'recv', + 'fetch_barrier', + 'concat', + ] self.assertEqual(src, dst) - self.assertEqual([op.type for op in trainer.global_block().ops], [ - 'mul', 'elementwise_add', 'elementwise_sub', 'square', 'mean', - 'fill_constant', 'mean_grad', 'square_grad', 'elementwise_sub_grad', - 'elementwise_add_grad', 'send', 'mul_grad', 'split_byref', 'send', - 'send_barrier', 'recv', 'recv', 'fetch_barrier', 'concat' - ]) + self.assertEqual( + [op.type for op in trainer.global_block().ops], + [ + 'mul', + 'elementwise_add', + 'elementwise_sub', + 'square', + 'mean', + 'fill_constant', + 'mean_grad', + 'square_grad', + 'elementwise_sub_grad', + 'elementwise_add_grad', + 'send', + 'mul_grad', + 'split_byref', + 'send', + 'send_barrier', + 'recv', + 'recv', + 'fetch_barrier', + 'concat', + ], + ) self.assertEqual(len(pserver.blocks), 3) # block0: listen_and_serv - self.assertEqual([op.type for op in pserver.blocks[0].ops], - ["listen_and_serv"]) + self.assertEqual( + [op.type for op in pserver.blocks[0].ops], ["listen_and_serv"] + ) # block1~2: optimize pass - self.assertEqual([op.type for op in pserver.blocks[1].ops], - ["sum", "scale", "sgd"]) + self.assertEqual( + [op.type for op in pserver.blocks[1].ops], ["sum", "scale", "sgd"] + ) # confirm startup program - self.assertEqual([op.type for op in startup.global_block().ops], - ["fill_constant", "fill_constant", "uniform_random"]) + self.assertEqual( + [op.type for op in startup.global_block().ops], + ["fill_constant", "fill_constant", "uniform_random"], + ) # the variable #fc_w will be split into two blocks fc_w_var = startup.global_block().var("fc_w.block1") self.assertEqual(fc_w_var.shape, (500, 1000)) @@ -172,7 +203,6 @@ def transpiler_test_impl(self): class TestBasicModelWithLargeBlockSize(TranspilerTest): - def transpiler_test_impl(self): config = fluid.DistributeTranspilerConfig() config.min_block_size = 1048576 @@ -182,23 +212,43 @@ def transpiler_test_impl(self): trainer, _ = self.get_trainer(config) - self.assertEqual([op.type for op in trainer.global_block().ops], [ - 'mul', 'elementwise_add', 'elementwise_sub', 'square', 'mean', - 'fill_constant', 'mean_grad', 'square_grad', 'elementwise_sub_grad', - 'elementwise_add_grad', 'send', 'mul_grad', 'send', 'send_barrier', - 'recv', 'recv', 'fetch_barrier' - ]) + self.assertEqual( + [op.type for op in trainer.global_block().ops], + [ + 'mul', + 'elementwise_add', + 'elementwise_sub', + 'square', + 'mean', + 'fill_constant', + 'mean_grad', + 'square_grad', + 'elementwise_sub_grad', + 'elementwise_add_grad', + 'send', + 'mul_grad', + 'send', + 'send_barrier', + 'recv', + 'recv', + 'fetch_barrier', + ], + ) self.assertEqual(len(pserver.blocks), 2) # block0: listen_and_serv - self.assertEqual([op.type for op in pserver.blocks[0].ops], - ["listen_and_serv"]) + self.assertEqual( + [op.type for op in pserver.blocks[0].ops], ["listen_and_serv"] + ) # block1~2: optimize pass - self.assertEqual([op.type for op in pserver.blocks[1].ops], - ["sum", "scale", "sgd"]) + self.assertEqual( + [op.type for op in pserver.blocks[1].ops], ["sum", "scale", "sgd"] + ) # confirm startup program - self.assertEqual([op.type for op in startup.global_block().ops], - ["fill_constant", "fill_constant"]) + self.assertEqual( + [op.type for op in startup.global_block().ops], + ["fill_constant", "fill_constant"], + ) # the variable #fc_w will be split into two blocks fc_w_var = startup2.global_block().var("fc_w") self.assertEqual(fc_w_var.shape, (1000, 1000)) @@ -224,7 +274,6 @@ def transpiler_test_impl(self): class TestNoSliceVar(TranspilerTest): - def setUp(self): super(TestNoSliceVar, self).setUp() @@ -244,22 +293,26 @@ def transpiler_test_impl(self): class TestLRDecay(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.exponential_decay(learning_rate=1.0, - decay_steps=2100, - decay_rate=0.1, - staircase=True)) + learning_rate=fluid.layers.exponential_decay( + learning_rate=1.0, + decay_steps=2100, + decay_rate=0.1, + staircase=True, + ) + ) sgd_optimizer.minimize(avg_cost) def transpiler_test_impl(self): @@ -268,30 +321,35 @@ def transpiler_test_impl(self): self.assertEqual(len(pserver.blocks), 4) lr_decay_ops = [op.type for op in pserver.blocks[1].ops] - self.assertEqual(lr_decay_ops, [ - "increment", "cast", "fill_constant", "elementwise_div", "floor", - "fill_constant", "elementwise_pow", "fill_constant", - "elementwise_mul" - ]) + self.assertEqual( + lr_decay_ops, + [ + "increment", + "cast", + "fill_constant", + "elementwise_div", + "floor", + "fill_constant", + "elementwise_pow", + "fill_constant", + "elementwise_mul", + ], + ) class TestFakeInit(TranspilerTest): - def net_conf(self): dict_size, embedding_size, neg_num = 10000, 8, 5 - input_word = fluid.layers.data(name="input_word", - shape=[1], - dtype='int64', - lod_level=1) - true_word = fluid.layers.data(name='true_label', - shape=[1], - dtype='int64', - lod_level=1) - neg_word = fluid.layers.data(name="neg_label", - shape=[1], - dtype='int64', - lod_level=1) + input_word = fluid.layers.data( + name="input_word", shape=[1], dtype='int64', lod_level=1 + ) + true_word = fluid.layers.data( + name='true_label', shape=[1], dtype='int64', lod_level=1 + ) + neg_word = fluid.layers.data( + name="neg_label", shape=[1], dtype='int64', lod_level=1 + ) inputs = [input_word, true_word, neg_word] init_width = 0.5 / embedding_size @@ -299,82 +357,99 @@ def net_conf(self): input=inputs[0], is_sparse=True, size=[dict_size, embedding_size], - param_attr=fluid.ParamAttr(name='emb', - initializer=fluid.initializer.Uniform( - -init_width, init_width))) + param_attr=fluid.ParamAttr( + name='emb', + initializer=fluid.initializer.Uniform(-init_width, init_width), + ), + ) true_emb_w = fluid.layers.embedding( input=inputs[1], is_sparse=True, size=[dict_size, embedding_size], param_attr=fluid.ParamAttr( - name='emb_w', - initializer=fluid.initializer.Constant(value=0.0))) + name='emb_w', initializer=fluid.initializer.Constant(value=0.0) + ), + ) true_emb_b = fluid.layers.embedding( input=inputs[1], is_sparse=True, size=[dict_size, 1], param_attr=fluid.ParamAttr( - name='emb_b', - initializer=fluid.initializer.Constant(value=0.0))) + name='emb_b', initializer=fluid.initializer.Constant(value=0.0) + ), + ) neg_word_reshape = fluid.layers.reshape(inputs[2], shape=[-1, 1]) neg_word_reshape.stop_gradient = True - neg_emb_w = fluid.layers.embedding(input=neg_word_reshape, - is_sparse=True, - size=[dict_size, embedding_size], - param_attr=fluid.ParamAttr( - name='emb_w', learning_rate=1.0)) + neg_emb_w = fluid.layers.embedding( + input=neg_word_reshape, + is_sparse=True, + size=[dict_size, embedding_size], + param_attr=fluid.ParamAttr(name='emb_w', learning_rate=1.0), + ) - neg_emb_w_re = fluid.layers.reshape(neg_emb_w, - shape=[-1, neg_num, embedding_size]) + neg_emb_w_re = fluid.layers.reshape( + neg_emb_w, shape=[-1, neg_num, embedding_size] + ) - neg_emb_b = fluid.layers.embedding(input=neg_word_reshape, - is_sparse=True, - size=[dict_size, 1], - param_attr=fluid.ParamAttr( - name='emb_b', learning_rate=1.0)) + neg_emb_b = fluid.layers.embedding( + input=neg_word_reshape, + is_sparse=True, + size=[dict_size, 1], + param_attr=fluid.ParamAttr(name='emb_b', learning_rate=1.0), + ) neg_emb_b_vec = fluid.layers.reshape(neg_emb_b, shape=[-1, neg_num]) true_logits = fluid.layers.elementwise_add( - fluid.layers.reduce_sum(fluid.layers.elementwise_mul( - input_emb, true_emb_w), - dim=1, - keep_dim=True), true_emb_b) - - input_emb_re = fluid.layers.reshape(input_emb, - shape=[-1, 1, embedding_size]) - - neg_matmul = fluid.layers.matmul(input_emb_re, - neg_emb_w_re, - transpose_y=True) + fluid.layers.reduce_sum( + fluid.layers.elementwise_mul(input_emb, true_emb_w), + dim=1, + keep_dim=True, + ), + true_emb_b, + ) + + input_emb_re = fluid.layers.reshape( + input_emb, shape=[-1, 1, embedding_size] + ) + + neg_matmul = fluid.layers.matmul( + input_emb_re, neg_emb_w_re, transpose_y=True + ) neg_matmul_re = fluid.layers.reshape(neg_matmul, shape=[-1, neg_num]) neg_logits = fluid.layers.elementwise_add(neg_matmul_re, neg_emb_b_vec) # nce loss - label_ones = fluid.layers.fill_constant_batch_size_like(true_logits, - shape=[-1, 1], - value=1.0, - dtype='float32') + label_ones = fluid.layers.fill_constant_batch_size_like( + true_logits, shape=[-1, 1], value=1.0, dtype='float32' + ) label_zeros = fluid.layers.fill_constant_batch_size_like( - true_logits, shape=[-1, neg_num], value=0.0, dtype='float32') + true_logits, shape=[-1, neg_num], value=0.0, dtype='float32' + ) true_xent = fluid.layers.sigmoid_cross_entropy_with_logits( - true_logits, label_ones) + true_logits, label_ones + ) neg_xent = fluid.layers.sigmoid_cross_entropy_with_logits( - neg_logits, label_zeros) + neg_logits, label_zeros + ) cost = fluid.layers.elementwise_add( fluid.layers.reduce_sum(true_xent, dim=1), - fluid.layers.reduce_sum(neg_xent, dim=1)) + fluid.layers.reduce_sum(neg_xent, dim=1), + ) avg_cost = fluid.layers.reduce_mean(cost) sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.exponential_decay(learning_rate=1.0, - decay_steps=2100, - decay_rate=0.1, - staircase=True)) + learning_rate=fluid.layers.exponential_decay( + learning_rate=1.0, + decay_steps=2100, + decay_rate=0.1, + staircase=True, + ) + ) sgd_optimizer.minimize(avg_cost) def transpiler_test_impl(self): @@ -389,14 +464,15 @@ def transpiler_test_impl(self): class TestDecayedAdagrad(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -409,14 +485,15 @@ def transpiler_test_impl(self): class TestFtrl(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -429,20 +506,23 @@ def transpiler_test_impl(self): class TestLRDecayConditional(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.piecewise_decay([10000, 20000], - [1.0, 0.5, 1.0])) + learning_rate=fluid.layers.piecewise_decay( + [10000, 20000], [1.0, 0.5, 1.0] + ) + ) sgd_optimizer.minimize(avg_cost) def transpiler_test_impl(self): @@ -460,13 +540,27 @@ def transpiler_test_impl(self): self.assertEqual(len(pserver.blocks), 7) lr_decay_ops = [op.type for op in pserver.blocks[1].ops] - self.assertEqual(lr_decay_ops, [ - "increment", "cast", "fill_constant", "fill_constant", "less_than", - "logical_not", "conditional_block", "fill_constant", - "fill_constant", "less_than", "logical_not", "logical_and", - "logical_and", "conditional_block", "fill_constant", - "conditional_block" - ]) + self.assertEqual( + lr_decay_ops, + [ + "increment", + "cast", + "fill_constant", + "fill_constant", + "less_than", + "logical_not", + "conditional_block", + "fill_constant", + "fill_constant", + "less_than", + "logical_not", + "logical_and", + "logical_and", + "conditional_block", + "fill_constant", + "conditional_block", + ], + ) # test the condition blocks for b in sub_blocks: if b == 0: @@ -476,16 +570,17 @@ def transpiler_test_impl(self): class TestL2Decay(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') y_predict = fluid.layers.fc( input=x, size=1000, act=None, - param_attr=fluid.ParamAttr(name='fc_w', - regularizer=fluid.regularizer.L2Decay()), - bias_attr=fluid.ParamAttr(name='fc_b')) + param_attr=fluid.ParamAttr( + name='fc_w', regularizer=fluid.regularizer.L2Decay() + ), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -502,22 +597,27 @@ def transpiler_test_impl(self): trainer, _ = self.get_trainer() self.assertEqual(len(pserver.blocks), 3) - self.assertEqual([op.type for op in pserver.blocks[1].ops], - ["sum", "scale", "clip", "sgd"]) - self.assertEqual([op.type for op in pserver.blocks[2].ops], - ["sum", "scale", "clip", "scale", "sum", "sgd"]) + self.assertEqual( + [op.type for op in pserver.blocks[1].ops], + ["sum", "scale", "clip", "sgd"], + ) + self.assertEqual( + [op.type for op in pserver.blocks[2].ops], + ["sum", "scale", "clip", "scale", "sum", "sgd"], + ) # TODO(typhoonzero): test clipping and L2Decay ops are removed from trainer class TestL2DecayWithPiecewise(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -525,10 +625,12 @@ def net_conf(self): bd = [1, 10, 20, 30] lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] sgd_optimizer = fluid.optimizer.Momentum( - learning_rate=fluid.layers.piecewise_decay(boundaries=bd, - values=lr), + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr + ), momentum=0.9, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) sgd_optimizer.minimize(avg_cost) def transpiler_test_impl(self): @@ -536,33 +638,62 @@ def transpiler_test_impl(self): trainer, _ = self.get_trainer() self.assertEqual(len(pserver.blocks), 9) - self.assertEqual([op.type for op in pserver.blocks[1].ops], [ - "increment", "cast", "fill_constant", "fill_constant", "less_than", - "logical_not", "conditional_block", "fill_constant", - "fill_constant", "less_than", "logical_not", "logical_and", - "logical_and", "conditional_block", "fill_constant", - "fill_constant", "less_than", "logical_not", "logical_and", - "logical_and", "conditional_block", "fill_constant", - "fill_constant", "less_than", "logical_not", "logical_and", - "logical_and", "conditional_block", "fill_constant", - "conditional_block" - ]) - self.assertEqual([op.type for op in pserver.blocks[7].ops], - ["sum", "scale", "scale", "sum", "momentum"]) - self.assertEqual([op.type for op in pserver.blocks[8].ops], - ["sum", "scale", "scale", "sum", "momentum"]) + self.assertEqual( + [op.type for op in pserver.blocks[1].ops], + [ + "increment", + "cast", + "fill_constant", + "fill_constant", + "less_than", + "logical_not", + "conditional_block", + "fill_constant", + "fill_constant", + "less_than", + "logical_not", + "logical_and", + "logical_and", + "conditional_block", + "fill_constant", + "fill_constant", + "less_than", + "logical_not", + "logical_and", + "logical_and", + "conditional_block", + "fill_constant", + "fill_constant", + "less_than", + "logical_not", + "logical_and", + "logical_and", + "conditional_block", + "fill_constant", + "conditional_block", + ], + ) + self.assertEqual( + [op.type for op in pserver.blocks[7].ops], + ["sum", "scale", "scale", "sum", "momentum"], + ) + self.assertEqual( + [op.type for op in pserver.blocks[8].ops], + ["sum", "scale", "scale", "sum", "momentum"], + ) class TestEmptyPserverOptimizeBlocks(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') # only one parameter - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=False) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=False, + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -580,44 +711,45 @@ def transpiler_test_impl(self): class TestDistLookupTableBase(TranspilerTest): - def network_with_table(self, is_sparse, is_distributed): self.table_size = 1000 self.emb_size = 64 self.lookup_table_name = 'shared_w' def emb_pool(ids, table_name, is_distributed): - emb = fluid.layers.embedding(input=ids, - size=[self.table_size, self.emb_size], - dtype='float32', - param_attr=table_name, - is_sparse=is_sparse, - is_distributed=is_distributed) + emb = fluid.layers.embedding( + input=ids, + size=[self.table_size, self.emb_size], + dtype='float32', + param_attr=table_name, + is_sparse=is_sparse, + is_distributed=is_distributed, + ) pool = fluid.layers.sequence_pool(input=emb, pool_type='average') return pool - title_ids = fluid.layers.data(name='title_ids', - shape=[1], - dtype='int64', - lod_level=1) - brand_ids = fluid.layers.data(name='brand_ids', - shape=[1], - dtype='int64', - lod_level=1) - profile_ids = fluid.layers.data(name='brand_ids', - shape=[1], - dtype='int64', - lod_level=1) + title_ids = fluid.layers.data( + name='title_ids', shape=[1], dtype='int64', lod_level=1 + ) + brand_ids = fluid.layers.data( + name='brand_ids', shape=[1], dtype='int64', lod_level=1 + ) + profile_ids = fluid.layers.data( + name='brand_ids', shape=[1], dtype='int64', lod_level=1 + ) title_emb = emb_pool(title_ids, self.lookup_table_name, is_distributed) brand_emb = emb_pool(brand_ids, self.lookup_table_name, is_distributed) profile_emb = emb_pool(profile_ids, "profile_emb", False) - fc0 = fluid.layers.concat(input=[title_emb, brand_emb, profile_emb], - axis=1) - predict = fluid.layers.fc(input=fc0, - size=2, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + fc0 = fluid.layers.concat( + input=[title_emb, brand_emb, profile_emb], axis=1 + ) + predict = fluid.layers.fc( + input=fc0, + size=2, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') cost = fluid.layers.cross_entropy(input=predict, label=label) @@ -627,7 +759,6 @@ def emb_pool(ids, table_name, is_distributed): class TestLocalLookupTable(TestDistLookupTableBase): - def net_conf(self): self.network_with_table(is_sparse=True, is_distributed=False) @@ -637,36 +768,66 @@ def transpiler_test_impl(self): self.assertEqual(len(pserver1.blocks), 4) # 0 listen_and_serv # 1 optimize for fc_w or fc_b adam - self.assertEqual([op.type for op in pserver1.blocks[1].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[1].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) # 2 optimize for table adam # NOTE: if param is not selected rows, the grad will scaled to grad / trainer_num - self.assertEqual([op.type for op in pserver1.blocks[2].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[2].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) # 3 optimize for table 2 adam # NOTE: if param is not selected rows, the grad will scaled to grad / trainer_num - self.assertEqual([op.type for op in pserver1.blocks[3].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[3].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) trainer, _ = self.get_trainer() self.assertEqual(len(trainer.blocks), 1) ops = [ - 'lookup_table', 'sequence_pool', 'lookup_table', 'sequence_pool', - 'lookup_table', 'sequence_pool', 'concat', 'mul', 'elementwise_add', - 'cross_entropy2', 'mean', 'fill_constant', 'mean_grad', - 'cross_entropy_grad2', 'elementwise_add_grad', 'send', 'mul_grad', - 'send', 'concat_grad', 'sequence_pool_grad', 'lookup_table_grad', - 'split_selected_rows', 'send', 'sequence_pool_grad', - 'lookup_table_grad', 'sequence_pool_grad', 'lookup_table_grad', - 'sum', 'split_selected_rows', 'send', 'send_barrier', 'recv', - 'recv', 'fetch_barrier' + 'lookup_table', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'concat', + 'mul', + 'elementwise_add', + 'cross_entropy2', + 'mean', + 'fill_constant', + 'mean_grad', + 'cross_entropy_grad2', + 'elementwise_add_grad', + 'send', + 'mul_grad', + 'send', + 'concat_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'split_selected_rows', + 'send', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sum', + 'split_selected_rows', + 'send', + 'send_barrier', + 'recv', + 'recv', + 'fetch_barrier', ] self.assertEqual([op.type for op in trainer.blocks[0].ops], ops) class TestDistLookupTable(TestDistLookupTableBase): - def net_conf(self): self.network_with_table(is_sparse=True, is_distributed=True) @@ -676,48 +837,97 @@ def transpiler_test_impl(self): self.assertEqual(len(pserver1.blocks), 6) # 0 listen_and_serv # 1 optimize for fc_w or fc_b adam - self.assertEqual([op.type for op in pserver1.blocks[1].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[1].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) # 4 prefetch -> lookup_sparse_table_read for data0 - self.assertEqual([op.type for op in pserver1.blocks[2].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[2].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) # 2 optimize for table sgd - self.assertEqual([op.type for op in pserver1.blocks[3].ops], - ["sum", "sgd"]) + self.assertEqual( + [op.type for op in pserver1.blocks[3].ops], ["sum", "sgd"] + ) # 3 prefetch -> lookup_sparse_table_read for data0 - self.assertEqual([op.type for op in pserver1.blocks[4].ops], - ["lookup_sparse_table_read"]) + self.assertEqual( + [op.type for op in pserver1.blocks[4].ops], + ["lookup_sparse_table_read"], + ) # 5 save table self.assertEqual([op.type for op in pserver1.blocks[5].ops], ["save"]) trainer, trainer_startup = self.get_trainer() self.assertEqual(len(trainer.blocks), 1) ops = [ - 'split_ids', 'prefetch', 'merge_ids', 'sequence_pool', - 'sequence_pool', 'lookup_table', 'sequence_pool', 'concat', 'mul', - 'elementwise_add', 'cross_entropy2', 'mean', 'fill_constant', - 'mean_grad', 'cross_entropy_grad2', 'elementwise_add_grad', 'send', - 'mul_grad', 'send', 'concat_grad', 'sequence_pool_grad', - 'lookup_table_grad', 'split_selected_rows', 'send', - 'sequence_pool_grad', 'lookup_table_grad', 'sequence_pool_grad', - 'lookup_table_grad', 'sum', 'split_ids', 'send', 'send_barrier', - 'recv', 'recv', 'fetch_barrier' + 'split_ids', + 'prefetch', + 'merge_ids', + 'sequence_pool', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'concat', + 'mul', + 'elementwise_add', + 'cross_entropy2', + 'mean', + 'fill_constant', + 'mean_grad', + 'cross_entropy_grad2', + 'elementwise_add_grad', + 'send', + 'mul_grad', + 'send', + 'concat_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'split_selected_rows', + 'send', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sum', + 'split_ids', + 'send', + 'send_barrier', + 'recv', + 'recv', + 'fetch_barrier', ] self.assertEqual([op.type for op in trainer.blocks[0].ops], ops) startup_ops = [ - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'uniform_random', - 'uniform_random', 'recv', 'recv', 'recv', 'fetch_barrier', 'concat', - 'fake_init' + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'uniform_random', + 'uniform_random', + 'recv', + 'recv', + 'recv', + 'fetch_barrier', + 'concat', + 'fake_init', ] - self.assertEqual([op.type for op in trainer_startup.blocks[0].ops], - startup_ops) + self.assertEqual( + [op.type for op in trainer_startup.blocks[0].ops], startup_ops + ) class TestAsyncLocalLookupTable(TestDistLookupTableBase): - def net_conf(self): self.network_with_table(is_sparse=True, is_distributed=False) @@ -728,34 +938,63 @@ def transpiler_test_impl(self): self.assertEqual(len(pserver1.blocks), 4) # 0 listen_and_serv # 1 optimize for fc_w or fc_b adam - self.assertEqual([op.type for op in pserver1.blocks[1].ops], - ["adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[1].ops], + ["adam", "scale", "scale"], + ) # 2 optimize for table adam # NOTE: if param is not selected rows, the grad will scaled to grad / trainer_num - self.assertEqual([op.type for op in pserver1.blocks[2].ops], - ["adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[2].ops], + ["adam", "scale", "scale"], + ) # 3 optimize for table adam # NOTE: if param is not selected rows, the grad will scaled to grad / trainer_num - self.assertEqual([op.type for op in pserver1.blocks[3].ops], - ["adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[3].ops], + ["adam", "scale", "scale"], + ) trainer, _ = self.get_trainer(config) self.assertEqual(len(trainer.blocks), 1) ops = [ - 'lookup_table', 'sequence_pool', 'lookup_table', 'sequence_pool', - 'lookup_table', 'sequence_pool', 'concat', 'mul', 'elementwise_add', - 'cross_entropy2', 'mean', 'fill_constant', 'mean_grad', - 'cross_entropy_grad2', 'elementwise_add_grad', 'send', 'mul_grad', - 'send', 'concat_grad', 'sequence_pool_grad', 'lookup_table_grad', - 'split_selected_rows', 'send', 'sequence_pool_grad', - 'lookup_table_grad', 'sequence_pool_grad', 'lookup_table_grad', - 'sum', 'split_selected_rows', 'send', 'recv', 'recv' + 'lookup_table', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'concat', + 'mul', + 'elementwise_add', + 'cross_entropy2', + 'mean', + 'fill_constant', + 'mean_grad', + 'cross_entropy_grad2', + 'elementwise_add_grad', + 'send', + 'mul_grad', + 'send', + 'concat_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'split_selected_rows', + 'send', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sum', + 'split_selected_rows', + 'send', + 'recv', + 'recv', ] self.assertEqual([op.type for op in trainer.blocks[0].ops], ops) class TestAsyncDistLookupTable(TestDistLookupTableBase): - def net_conf(self): self.network_with_table(is_sparse=True, is_distributed=True) @@ -767,46 +1006,93 @@ def transpiler_test_impl(self): self.assertEqual(len(pserver1.blocks), 6) # 0 listen_and_serv # 1 optimize for fc_w or fc_b adam - self.assertEqual([op.type for op in pserver1.blocks[1].ops], - ["adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[1].ops], + ["adam", "scale", "scale"], + ) # 2 optimize for table adam - self.assertEqual([op.type for op in pserver1.blocks[2].ops], - ["adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[2].ops], + ["adam", "scale", "scale"], + ) # 3 optimize for table sgd self.assertEqual([op.type for op in pserver1.blocks[3].ops], ["sgd"]) # 4 prefetch -> lookup_sparse_table_read for data0 - self.assertEqual([op.type for op in pserver1.blocks[4].ops], - ["lookup_sparse_table_read"]) + self.assertEqual( + [op.type for op in pserver1.blocks[4].ops], + ["lookup_sparse_table_read"], + ) # 5 save table self.assertEqual([op.type for op in pserver1.blocks[5].ops], ["save"]) trainer, trainer_startup = self.get_trainer(config) self.assertEqual(len(trainer.blocks), 1) ops = [ - 'split_ids', 'prefetch', 'merge_ids', 'sequence_pool', - 'sequence_pool', 'lookup_table', 'sequence_pool', 'concat', 'mul', - 'elementwise_add', 'cross_entropy2', 'mean', 'fill_constant', - 'mean_grad', 'cross_entropy_grad2', 'elementwise_add_grad', 'send', - 'mul_grad', 'send', 'concat_grad', 'sequence_pool_grad', - 'lookup_table_grad', 'split_selected_rows', 'send', - 'sequence_pool_grad', 'lookup_table_grad', 'sequence_pool_grad', - 'lookup_table_grad', 'sum', 'split_ids', 'send', 'recv', 'recv' + 'split_ids', + 'prefetch', + 'merge_ids', + 'sequence_pool', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'concat', + 'mul', + 'elementwise_add', + 'cross_entropy2', + 'mean', + 'fill_constant', + 'mean_grad', + 'cross_entropy_grad2', + 'elementwise_add_grad', + 'send', + 'mul_grad', + 'send', + 'concat_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'split_selected_rows', + 'send', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sum', + 'split_ids', + 'send', + 'recv', + 'recv', ] self.assertEqual([op.type for op in trainer.blocks[0].ops], ops) startup_ops = [ - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'fill_constant', 'fill_constant', - 'fill_constant', 'fill_constant', 'uniform_random', - 'uniform_random', 'recv', 'recv', 'recv', 'fetch_barrier', 'concat', - 'fake_init' + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'fill_constant', + 'uniform_random', + 'uniform_random', + 'recv', + 'recv', + 'recv', + 'fetch_barrier', + 'concat', + 'fake_init', ] - self.assertEqual([op.type for op in trainer_startup.blocks[0].ops], - startup_ops) + self.assertEqual( + [op.type for op in trainer_startup.blocks[0].ops], startup_ops + ) class TestDistLookupTableSliceSize(TestDistLookupTableBase): - def net_conf(self): self.network_with_table(is_sparse=True, is_distributed=True) @@ -816,14 +1102,14 @@ def transpiler_test_impl(self): self.assertTrue(self.transpiler.has_distributed_lookup_table) lookup_table_var = pserver1.global_block().vars[ - self.transpiler.table_name] + self.transpiler.table_name + ] row_size = lookup_table_var.shape[0] calc_row_size = int(math.ceil(self.table_size / self.pservers)) self.assertEqual(row_size, calc_row_size) class TestDistArgsInProgram(TestDistLookupTableBase): - def net_conf(self): self.network_with_table(is_sparse=True, is_distributed=True) @@ -832,21 +1118,24 @@ def transpiler_test_impl(self): self.assertTrue(trainer._is_distributed) self.assertTrue(trainer._is_chief) - self.assertEqual(trainer._distributed_lookup_table, - self.lookup_table_name) - self.assertEqual(trainer._endpoints, - [self.pserver1_ep, self.pserver2_ep]) + self.assertEqual( + trainer._distributed_lookup_table, self.lookup_table_name + ) + self.assertEqual( + trainer._endpoints, [self.pserver1_ep, self.pserver2_ep] + ) class TestRMSPropOptimizer(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -859,8 +1148,10 @@ def transpiler_test_impl(self): self.assertEqual(len(pserver.blocks), 3) # block1~2: optimize pass - self.assertEqual([op.type for op in pserver.blocks[1].ops], - ["sum", "scale", "rmsprop"]) + self.assertEqual( + [op.type for op in pserver.blocks[1].ops], + ["sum", "scale", "rmsprop"], + ) # the variable #fc_w will be split into two blocks fc_w_var = startup.global_block().var("fc_w.block1") self.assertEqual(fc_w_var.shape, (500, 1000)) @@ -869,14 +1160,15 @@ def transpiler_test_impl(self): class TestLoadSliceVar(TranspilerTest): - def net_conf(self): x = fluid.layers.data(name='x', shape=[1000], dtype='float32') - y_predict = fluid.layers.fc(input=x, - size=1000, - act=None, - param_attr=fluid.ParamAttr(name='fc_w'), - bias_attr=fluid.ParamAttr(name='fc_b')) + y_predict = fluid.layers.fc( + input=x, + size=1000, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b'), + ) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) @@ -888,9 +1180,11 @@ def transpiler_test_impl(self): pserver2, _ = self.get_pserver(self.pserver2_ep) vars_ps1 = pserver._parameters_on_pservers.get_distributed_vars_by_ep( - self.pserver1_ep) + self.pserver1_ep + ) vars_ps2 = pserver._parameters_on_pservers.get_distributed_vars_by_ep( - self.pserver2_ep) + self.pserver2_ep + ) self.assertTrue(vars_ps1) self.assertTrue(vars_ps2) @@ -902,10 +1196,12 @@ def transpiler_test_impl(self): ps1_var = vars_ps1[idx] if not ps1_var.is_slice: - total_numel = functools.reduce(lambda x, y: x * y, - vars_ps1[idx].origin.shape) - ps1_numel = functools.reduce(lambda x, y: x * y, - vars_ps1[idx].slice.shape) + total_numel = functools.reduce( + lambda x, y: x * y, vars_ps1[idx].origin.shape + ) + ps1_numel = functools.reduce( + lambda x, y: x * y, vars_ps1[idx].slice.shape + ) else: ps2_var = None for var in vars_ps2: @@ -913,18 +1209,20 @@ def transpiler_test_impl(self): ps2_var = var break - total_numel = functools.reduce(lambda x, y: x * y, - ps1_var.origin.shape) - ps1_numel = functools.reduce(lambda x, y: x * y, - ps1_var.slice.shape) - ps2_numel = functools.reduce(lambda x, y: x * y, - ps2_var.slice.shape) + total_numel = functools.reduce( + lambda x, y: x * y, ps1_var.origin.shape + ) + ps1_numel = functools.reduce( + lambda x, y: x * y, ps1_var.slice.shape + ) + ps2_numel = functools.reduce( + lambda x, y: x * y, ps2_var.slice.shape + ) self.assertEqual(total_numel, ps1_numel + ps2_numel) class TestNCCL2Transpile(TranspilerTest): - def test_nccl2_transpile(self): if fluid.core.is_compiled_with_cuda(): # test nccl2 only with cuda main = fluid.Program() @@ -936,10 +1234,12 @@ def test_nccl2_transpile(self): config.mode = "nccl2" config.wait_port = False t = fluid.DistributeTranspiler(config=config) - t.transpile(0, - trainers="127.0.0.1:6174,127.0.0.1:6175", - current_endpoint="127.0.0.1:6174", - startup_program=startup) + t.transpile( + 0, + trainers="127.0.0.1:6174,127.0.0.1:6175", + current_endpoint="127.0.0.1:6174", + startup_program=startup, + ) print([op.type for op in startup.global_block().ops]) self.assertEqual(startup.global_block().ops[-1].type, "gen_nccl_id") self.assertIsNotNone(startup.global_block().vars.get("NCCLID")) @@ -950,9 +1250,9 @@ def test_nccl2_transpile(self): # test for remote prefetch class TestRemoteLookupTable(TestDistLookupTableBase): - def net_conf(self): import os + os.environ['PADDLE_ENABLE_REMOTE_PREFETCH'] = "1" self.network_with_table(is_sparse=True, is_distributed=False) @@ -962,37 +1262,67 @@ def transpiler_test_impl(self): self.assertEqual(len(pserver1.blocks), 4) # 0 listen_and_serv # 1 optimize for fc_w or fc_b adam - self.assertEqual([op.type for op in pserver1.blocks[1].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[1].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) # 2 optimize for table adam # NOTE: if param is not selected rows, the grad will scaled to grad / trainer_num - self.assertEqual([op.type for op in pserver1.blocks[2].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[2].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) # 3 optimize for table 2 adam # NOTE: if param is not selected rows, the grad will scaled to grad / trainer_num - self.assertEqual([op.type for op in pserver1.blocks[3].ops], - ["sum", "scale", "adam", "scale", "scale"]) + self.assertEqual( + [op.type for op in pserver1.blocks[3].ops], + ["sum", "scale", "adam", "scale", "scale"], + ) trainer, _ = self.get_trainer() self.assertEqual(len(trainer.blocks), 1) ops = [ - 'lookup_table', 'sequence_pool', 'lookup_table', 'sequence_pool', - 'lookup_table', 'sequence_pool', 'concat', 'mul', 'elementwise_add', - 'cross_entropy2', 'mean', 'fill_constant', 'mean_grad', - 'cross_entropy_grad2', 'elementwise_add_grad', 'send', 'mul_grad', - 'send', 'concat_grad', 'sequence_pool_grad', 'lookup_table_grad', - 'split_selected_rows', 'send', 'sequence_pool_grad', - 'lookup_table_grad', 'sequence_pool_grad', 'lookup_table_grad', - 'sum', 'split_selected_rows', 'send', 'send_barrier', 'recv', - 'recv', 'fetch_barrier' + 'lookup_table', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'lookup_table', + 'sequence_pool', + 'concat', + 'mul', + 'elementwise_add', + 'cross_entropy2', + 'mean', + 'fill_constant', + 'mean_grad', + 'cross_entropy_grad2', + 'elementwise_add_grad', + 'send', + 'mul_grad', + 'send', + 'concat_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'split_selected_rows', + 'send', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sequence_pool_grad', + 'lookup_table_grad', + 'sum', + 'split_selected_rows', + 'send', + 'send_barrier', + 'recv', + 'recv', + 'fetch_barrier', ] self.assertEqual([op.type for op in trainer.blocks[0].ops], ops) # test for remote prefetch class TestRemoteNce(TestDistLookupTableBase): - def network_with_table(self, is_sparse, is_distributed): num_total_classes = 20 @@ -1002,28 +1332,40 @@ def network_with_table(self, is_sparse, is_distributed): input = fluid.layers.data(name="input", shape=[10], dtype="float32") label = fluid.layers.data(name="label", shape=[1], dtype="int64") - w_param = fluid.default_main_program().global_block().create_parameter( - shape=[num_total_classes, 10], - dtype='float32', - name='nce_w', - initializer=fluid.initializer.ConstantInitializer()) - b_param = fluid.default_main_program().global_block().create_parameter( - shape=[num_total_classes, 1], - dtype='float32', - name='nce_b', - initializer=fluid.initializer.ConstantInitializer()) - - cost = fluid.layers.nce(input=input, - label=label, - num_total_classes=num_total_classes, - sampler=sampler, - custom_dist=nid_freq_arr.tolist(), - sample_weight=None, - param_attr='nce_w', - bias_attr='nce_b', - seed=1, - num_neg_samples=5, - is_sparse=is_sparse) + w_param = ( + fluid.default_main_program() + .global_block() + .create_parameter( + shape=[num_total_classes, 10], + dtype='float32', + name='nce_w', + initializer=fluid.initializer.ConstantInitializer(), + ) + ) + b_param = ( + fluid.default_main_program() + .global_block() + .create_parameter( + shape=[num_total_classes, 1], + dtype='float32', + name='nce_b', + initializer=fluid.initializer.ConstantInitializer(), + ) + ) + + cost = fluid.layers.nce( + input=input, + label=label, + num_total_classes=num_total_classes, + sampler=sampler, + custom_dist=nid_freq_arr.tolist(), + sample_weight=None, + param_attr='nce_w', + bias_attr='nce_b', + seed=1, + num_neg_samples=5, + is_sparse=is_sparse, + ) avg_cost = paddle.mean(cost) # optimizer optimizer = fluid.optimizer.Adam(learning_rate=0.003) @@ -1031,6 +1373,7 @@ def network_with_table(self, is_sparse, is_distributed): def net_conf(self): import os + os.environ['PADDLE_ENABLE_REMOTE_PREFETCH'] = "1" self.network_with_table(is_sparse=True, is_distributed=False) @@ -1055,44 +1398,59 @@ def transpiler_test_impl(self): # test for remote prefetch class TestRemoteHsigmoid(TestDistLookupTableBase): - def network_with_table(self, is_sparse, is_distributed): num_total_classes = 3 input = fluid.layers.data(name="input", shape=[1], dtype="float32") label = fluid.layers.data(name="label", shape=[1], dtype="int64") - path_table = fluid.layers.data(name='path_table', - shape=[3], - dtype='int64') - path_code = fluid.layers.data(name='path_code', - shape=[3], - dtype='int64') - w_param = fluid.default_main_program().global_block().create_parameter( - shape=[num_total_classes, 10], - dtype='float32', - name='hs_w', - initializer=fluid.initializer.ConstantInitializer()) - b_param = fluid.default_main_program().global_block().create_parameter( - shape=[3, 1], - dtype='float32', - name='hs_b', - initializer=fluid.initializer.ConstantInitializer()) + path_table = fluid.layers.data( + name='path_table', shape=[3], dtype='int64' + ) + path_code = fluid.layers.data( + name='path_code', shape=[3], dtype='int64' + ) + w_param = ( + fluid.default_main_program() + .global_block() + .create_parameter( + shape=[num_total_classes, 10], + dtype='float32', + name='hs_w', + initializer=fluid.initializer.ConstantInitializer(), + ) + ) + b_param = ( + fluid.default_main_program() + .global_block() + .create_parameter( + shape=[3, 1], + dtype='float32', + name='hs_b', + initializer=fluid.initializer.ConstantInitializer(), + ) + ) emb = fluid.layers.embedding( input=input, is_sparse=is_sparse, size=[3, 3], - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Normal( - scale=1 / math.sqrt(num_total_classes)))) - - cost = fluid.layers.hsigmoid(input=emb, - label=label, - num_classes=num_total_classes, - path_table=path_table, - path_code=path_code, - is_custom=True, - is_sparse=is_sparse) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Normal( + scale=1 / math.sqrt(num_total_classes) + ) + ), + ) + + cost = fluid.layers.hsigmoid( + input=emb, + label=label, + num_classes=num_total_classes, + path_table=path_table, + path_code=path_code, + is_custom=True, + is_sparse=is_sparse, + ) avg_cost = paddle.mean(cost) # optimizer optimizer = fluid.optimizer.SGD(learning_rate=0.003) @@ -1100,6 +1458,7 @@ def network_with_table(self, is_sparse, is_distributed): def net_conf(self): import os + os.environ['PADDLE_ENABLE_REMOTE_PREFETCH'] = "1" self.network_with_table(is_sparse=True, is_distributed=False) diff --git a/python/paddle/fluid/tests/unittests/test_dist_tree_index.py b/python/paddle/fluid/tests/unittests/test_dist_tree_index.py index 0fae4fe09773e..d3d3d249c4125 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_tree_index.py +++ b/python/paddle/fluid/tests/unittests/test_dist_tree_index.py @@ -24,35 +24,32 @@ def create_feeds(): - user_input = fluid.layers.data(name="item_id", - shape=[1], - dtype="int64", - lod_level=1) - - item = fluid.layers.data(name="unit_id", - shape=[1], - dtype="int64", - lod_level=1) - - label = fluid.layers.data(name="label", - shape=[1], - dtype="int64", - lod_level=1) - labels = fluid.layers.data(name="labels", - shape=[1], - dtype="int64", - lod_level=1) + user_input = fluid.layers.data( + name="item_id", shape=[1], dtype="int64", lod_level=1 + ) + + item = fluid.layers.data( + name="unit_id", shape=[1], dtype="int64", lod_level=1 + ) + + label = fluid.layers.data( + name="label", shape=[1], dtype="int64", lod_level=1 + ) + labels = fluid.layers.data( + name="labels", shape=[1], dtype="int64", lod_level=1 + ) feed_list = [user_input, item, label, labels] return feed_list class TestTreeIndex(unittest.TestCase): - def test_tree_index(self): path = download( "https://paddlerec.bj.bcebos.com/tree-based/data/mini_tree.pb", - "tree_index_unittest", "e2ba4561c2e9432b532df40546390efa") + "tree_index_unittest", + "e2ba4561c2e9432b532df40546390efa", + ) ''' path = download( "https://paddlerec.bj.bcebos.com/tree-based/data/mini_tree.pb", @@ -72,7 +69,8 @@ def test_tree_index(self): for i in range(tree.height()): layer_node_codes.append(tree.get_layer_codes(i)) layer_node_ids.append( - [node.id() for node in tree.get_nodes(layer_node_codes[-1])]) + [node.id() for node in tree.get_nodes(layer_node_codes[-1])] + ) all_leaf_ids = [node.id() for node in tree.get_all_leafs()] self.assertEqual(sum(all_leaf_ids), sum(layer_node_ids[-1])) @@ -97,8 +95,9 @@ def test_tree_index(self): self.assertEqual(pi_relation[all_leaf_ids[0]], ancestor_codes[0]) # get_travel_path - travel_path_codes = tree.get_travel_path(travel_codes[0], - travel_codes[-1]) + travel_path_codes = tree.get_travel_path( + travel_codes[0], travel_codes[-1] + ) travel_path_ids = [ node.id() for node in tree.get_nodes(travel_path_codes) ] @@ -113,7 +112,6 @@ def test_tree_index(self): class TestIndexSampler(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -123,14 +121,17 @@ def tearDown(self): def test_layerwise_sampler(self): path = download( "https://paddlerec.bj.bcebos.com/tree-based/data/mini_tree.pb", - "tree_index_unittest", "e2ba4561c2e9432b532df40546390efa") + "tree_index_unittest", + "e2ba4561c2e9432b532df40546390efa", + ) tdm_layer_counts = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - #tree = TreeIndex("demo", path) - file_name = os.path.join(self.temp_dir.name, - "test_in_memory_dataset_tdm_sample_run.txt") + # tree = TreeIndex("demo", path) + file_name = os.path.join( + self.temp_dir.name, "test_in_memory_dataset_tdm_sample_run.txt" + ) with open(file_name, "w") as f: - #data = "29 d 29 d 29 29 29 29 29 29 29 29 29 29 29 29\n" + # data = "29 d 29 d 29 29 29 29 29 29 29 29 29 29 29 29\n" data = "1 1 1 15 15 15\n" data += "1 1 1 15 15 15\n" f.write(data) @@ -142,26 +143,30 @@ def test_layerwise_sampler(self): slots_vars.append(var) dataset = paddle.distributed.InMemoryDataset() - dataset.init(batch_size=1, - pipe_command="cat", - download_cmd="cat", - use_var=slots_vars) + dataset.init( + batch_size=1, + pipe_command="cat", + download_cmd="cat", + use_var=slots_vars, + ) dataset.set_filelist([file_name]) - #dataset.update_settings(pipe_command="cat") - #dataset._init_distributed_settings( + # dataset.update_settings(pipe_command="cat") + # dataset._init_distributed_settings( # parse_ins_id=True, # parse_content=True, # fea_eval=True, # candidate_size=10000) dataset.load_into_memory() - dataset.tdm_sample('demo', - tree_path=path, - tdm_layer_counts=tdm_layer_counts, - start_sample_layer=1, - with_hierachy=False, - seed=0, - id_slot=2) + dataset.tdm_sample( + 'demo', + tree_path=path, + tdm_layer_counts=tdm_layer_counts, + start_sample_layer=1, + with_hierachy=False, + seed=0, + id_slot=2, + ) self.assertTrue(dataset.get_shuffle_data_size() == 8) diff --git a/python/paddle/fluid/tests/unittests/test_dist_word2vec.py b/python/paddle/fluid/tests/unittests/test_dist_word2vec.py index 6d9727f6f3ac5..be100d7699f0f 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_word2vec.py +++ b/python/paddle/fluid/tests/unittests/test_dist_word2vec.py @@ -21,43 +21,46 @@ class TestDistW2V2x2(TestDistBase): - def _setup_config(self): self._sync_mode = True self._enforce_place = "CPU" def test_dist_train(self): - self.check_with_place("dist_word2vec.py", - delta=1e-4, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_word2vec.py", + delta=1e-4, + check_error_log=True, + log_name=flag_name, + ) class TestDistW2V2x2WithMemOpt(TestDistBase): - def _setup_config(self): self._sync_mode = True self._mem_opt = True self._enforce_place = "CPU" def test_dist_train(self): - self.check_with_place("dist_word2vec.py", - delta=1e-4, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_word2vec.py", + delta=1e-4, + check_error_log=True, + log_name=flag_name, + ) class TestDistW2V2x2Async(TestDistBase): - def _setup_config(self): self._sync_mode = False self._enforce_place = "CPU" def test_dist_train(self): - self.check_with_place("dist_word2vec.py", - delta=100, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "dist_word2vec.py", + delta=100, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_distribute_fpn_proposals_op.py b/python/paddle/fluid/tests/unittests/test_distribute_fpn_proposals_op.py index 29bd02a578e18..354a630bc3847 100644 --- a/python/paddle/fluid/tests/unittests/test_distribute_fpn_proposals_op.py +++ b/python/paddle/fluid/tests/unittests/test_distribute_fpn_proposals_op.py @@ -19,18 +19,26 @@ from op_test import OpTest -def distribute_fpn_proposals_wrapper(fpn_rois, rois_num, min_level, max_level, - refer_level, refer_scale, pixel_offset): - return paddle.vision.ops.distribute_fpn_proposals(fpn_rois=fpn_rois, - min_level=min_level, - max_level=max_level, - refer_level=refer_level, - refer_scale=refer_scale, - rois_num=rois_num) +def distribute_fpn_proposals_wrapper( + fpn_rois, + rois_num, + min_level, + max_level, + refer_level, + refer_scale, + pixel_offset, +): + return paddle.vision.ops.distribute_fpn_proposals( + fpn_rois=fpn_rois, + min_level=min_level, + max_level=max_level, + refer_level=refer_level, + refer_scale=refer_scale, + rois_num=rois_num, + ) class TestDistributeFPNProposalsOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() @@ -43,8 +51,9 @@ def set_data(self): 'refer_level': self.canonical_level, 'pixel_offset': self.pixel_offset, } - output = [('out%d' % i, self.rois_fpn[i]) - for i in range(len(self.rois_fpn))] + output = [ + ('out%d' % i, self.rois_fpn[i]) for i in range(len(self.rois_fpn)) + ] self.outputs = { 'MultiFpnRois': output, @@ -63,8 +72,8 @@ def init_test_case(self): def boxes_area(self, boxes): offset = 1 if self.pixel_offset else 0 - w = (boxes[:, 2] - boxes[:, 0] + offset) - h = (boxes[:, 3] - boxes[:, 1] + offset) + w = boxes[:, 2] - boxes[:, 0] + offset + h = boxes[:, 3] - boxes[:, 1] + offset areas = w * h assert np.all(areas >= 0), 'Negative areas founds' return areas @@ -85,7 +94,7 @@ def get_sub_lod(self, sub_lvl): return sub_lod def add_multilevel_roi(self, rois, target_lvls, lvl_min, lvl_max): - rois_idx_order = np.empty((0, )) + rois_idx_order = np.empty((0,)) rois_fpn = [] for lvl in range(lvl_min, lvl_max + 1): idx_lvl = np.where(target_lvls == lvl)[0] @@ -95,17 +104,20 @@ def add_multilevel_roi(self, rois, target_lvls, lvl_min, lvl_max): sub_lod = self.get_sub_lod(rois[idx_lvl, 0]) rois_fpn.append((rois[idx_lvl, 1:], [sub_lod])) rois_idx_order = np.concatenate((rois_idx_order, idx_lvl)) - rois_idx_restore = np.argsort(rois_idx_order).astype(np.int32, - copy=False) + rois_idx_restore = np.argsort(rois_idx_order).astype( + np.int32, copy=False + ) return rois_fpn, rois_idx_restore def calc_rois_distribute(self): lvl_min = self.roi_min_level lvl_max = self.roi_max_level - target_lvls = self.map_rois_to_fpn_levels(self.rois[:, 1:5], lvl_min, - lvl_max) + target_lvls = self.map_rois_to_fpn_levels( + self.rois[:, 1:5], lvl_min, lvl_max + ) rois_fpn, rois_idx_restore = self.add_multilevel_roi( - self.rois, target_lvls, lvl_min, lvl_max) + self.rois, target_lvls, lvl_min, lvl_max + ) return rois_fpn, rois_idx_restore def make_rois(self): @@ -133,14 +145,13 @@ def test_check_output(self): class TestDistributeFPNProposalsOpWithRoisNum(TestDistributeFPNProposalsOp): - def set_data(self): self.init_test_case() self.make_rois() self.rois_fpn, self.rois_idx_restore = self.calc_rois_distribute() self.inputs = { 'FpnRois': (self.rois[:, 1:5], self.rois_lod), - 'RoisNum': np.array(self.rois_lod[0]).astype('int32') + 'RoisNum': np.array(self.rois_lod[0]).astype('int32'), } self.attrs = { 'max_level': self.roi_max_level, @@ -149,26 +160,30 @@ def set_data(self): 'refer_level': self.canonical_level, 'pixel_offset': self.pixel_offset, } - output = [('out%d' % i, self.rois_fpn[i]) - for i in range(len(self.rois_fpn))] - rois_num_per_level = [('rois_num%d' % i, - np.array(self.rois_fpn[i][1][0]).astype('int32')) - for i in range(len(self.rois_fpn))] + output = [ + ('out%d' % i, self.rois_fpn[i]) for i in range(len(self.rois_fpn)) + ] + rois_num_per_level = [ + ('rois_num%d' % i, np.array(self.rois_fpn[i][1][0]).astype('int32')) + for i in range(len(self.rois_fpn)) + ] self.outputs = { 'MultiFpnRois': output, 'RestoreIndex': self.rois_idx_restore.reshape(-1, 1), - 'MultiLevelRoIsNum': rois_num_per_level + 'MultiLevelRoIsNum': rois_num_per_level, } self.python_api = distribute_fpn_proposals_wrapper self.python_out_sig = [ - 'MultiFpnRois', 'MultiLevelRoIsNum', 'RestoreIndex' + 'MultiFpnRois', + 'MultiLevelRoIsNum', + 'RestoreIndex', ] class TestDistributeFPNProposalsOpNoOffset( - TestDistributeFPNProposalsOpWithRoisNum): - + TestDistributeFPNProposalsOpWithRoisNum +): def init_test_case(self): self.roi_max_level = 5 self.roi_min_level = 2 @@ -179,7 +194,6 @@ def init_test_case(self): class TestDistributeFpnProposalsAPI(unittest.TestCase): - def setUp(self): np.random.seed(678) self.rois_np = np.random.rand(10, 4).astype('float32') @@ -188,26 +202,30 @@ def setUp(self): def test_dygraph_with_static(self): paddle.enable_static() rois = paddle.static.data(name='rois', shape=[10, 4], dtype='float32') - rois_num = paddle.static.data(name='rois_num', - shape=[None], - dtype='int32') - multi_rois, restore_ind, rois_num_per_level = paddle.vision.ops.distribute_fpn_proposals( + rois_num = paddle.static.data( + name='rois_num', shape=[None], dtype='int32' + ) + ( + multi_rois, + restore_ind, + rois_num_per_level, + ) = paddle.vision.ops.distribute_fpn_proposals( fpn_rois=rois, min_level=2, max_level=5, refer_level=4, refer_scale=224, - rois_num=rois_num) + rois_num=rois_num, + ) fetch_list = multi_rois + [restore_ind] + rois_num_per_level exe = paddle.static.Executor() - output_stat = exe.run(paddle.static.default_main_program(), - feed={ - 'rois': self.rois_np, - 'rois_num': self.rois_num_np - }, - fetch_list=fetch_list, - return_numpy=False) + output_stat = exe.run( + paddle.static.default_main_program(), + feed={'rois': self.rois_np, 'rois_num': self.rois_num_np}, + fetch_list=fetch_list, + return_numpy=False, + ) output_stat_np = [] for output in output_stat: output_np = np.array(output) @@ -217,13 +235,18 @@ def test_dygraph_with_static(self): paddle.disable_static() rois_dy = paddle.to_tensor(self.rois_np) rois_num_dy = paddle.to_tensor(self.rois_num_np) - multi_rois_dy, restore_ind_dy, rois_num_per_level_dy = paddle.vision.ops.distribute_fpn_proposals( + ( + multi_rois_dy, + restore_ind_dy, + rois_num_per_level_dy, + ) = paddle.vision.ops.distribute_fpn_proposals( fpn_rois=rois_dy, min_level=2, max_level=5, refer_level=4, refer_scale=224, - rois_num=rois_num_dy) + rois_num=rois_num_dy, + ) output_dy = multi_rois_dy + [restore_ind_dy] + rois_num_per_level_dy output_dy_np = [] for output in output_dy: diff --git a/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_clip.py b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_clip.py index fac4e79ce139c..7638f7cf2436f 100644 --- a/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_clip.py +++ b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_clip.py @@ -35,10 +35,12 @@ def remove_file_if_exists(file_name): shutil.rmtree(file_name) -def run_test(clip_after_allreduce=True, - max_global_norm=-1.0, - gradient_merge_steps=1, - use_master_acc_grad=True): +def run_test( + clip_after_allreduce=True, + max_global_norm=-1.0, + gradient_merge_steps=1, + use_master_acc_grad=True, +): temp_dir = tempfile.TemporaryDirectory() if not paddle.is_compiled_with_cuda(): return @@ -66,17 +68,18 @@ def run_test(clip_after_allreduce=True, touch_file_env = 'SUCCESS_TOUCH_FILE' touch_file_name = os.path.join( temp_dir.name, - 'distributed_fused_lamb_touch_file_{}'.format(os.getpid())) + 'distributed_fused_lamb_touch_file_{}'.format(os.getpid()), + ) os.environ[touch_file_env] = touch_file_name try: assert os.system(cmd) == 0 and os.path.exists( - touch_file_name), 'Test failed when {}'.format(args) + touch_file_name + ), 'Test failed when {}'.format(args) finally: temp_dir.cleanup() class TestDistributedFusedLambWithClip(unittest.TestCase): - def test_1(self): run_test(clip_after_allreduce=True, max_global_norm=0.01) diff --git a/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_gradient_merge.py b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_gradient_merge.py index 01ca09916a1e6..37b315e61a808 100644 --- a/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_gradient_merge.py +++ b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_with_gradient_merge.py @@ -17,17 +17,20 @@ class TestDistributedFusedLambGradientMerge(unittest.TestCase): - def test_gm(self): - run_test(clip_after_allreduce=True, - max_global_norm=-1.0, - gradient_merge_steps=2) + run_test( + clip_after_allreduce=True, + max_global_norm=-1.0, + gradient_merge_steps=2, + ) def test_gm_with_fp16_acc_grad(self): - run_test(clip_after_allreduce=True, - max_global_norm=-1.0, - gradient_merge_steps=2, - use_master_acc_grad=False) + run_test( + clip_after_allreduce=True, + max_global_norm=-1.0, + gradient_merge_steps=2, + use_master_acc_grad=False, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_without_clip.py b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_without_clip.py index 8d4dfa84d2f1b..6e9d8ddea1728 100644 --- a/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_without_clip.py +++ b/python/paddle/fluid/tests/unittests/test_distributed_fused_lamb_op_without_clip.py @@ -17,7 +17,6 @@ class TestDistributedFusedLambWithoutClip(unittest.TestCase): - def test_1(self): run_test(clip_after_allreduce=True, max_global_norm=-1.0) diff --git a/python/paddle/fluid/tests/unittests/test_distributions.py b/python/paddle/fluid/tests/unittests/test_distributions.py index d79e8784e2b22..a032b74f796bd 100644 --- a/python/paddle/fluid/tests/unittests/test_distributions.py +++ b/python/paddle/fluid/tests/unittests/test_distributions.py @@ -16,13 +16,18 @@ import unittest from paddle import fluid from paddle.fluid import layers -from paddle.fluid.layers.distributions import Categorical, MultivariateNormalDiag, Normal, Uniform +from paddle.fluid.layers.distributions import ( + Categorical, + MultivariateNormalDiag, + Normal, + Uniform, +) import math -class DistributionNumpy(): +class DistributionNumpy: """ - Distribution is the abstract base class for probability distributions. + Distribution is the abstract base class for probability distributions. """ def sample(self): @@ -43,15 +48,15 @@ def log_prob(self, value): class UniformNumpy(DistributionNumpy): - def __init__(self, low, high): self.low = np.array(low).astype('float32') self.high = np.array(high).astype('float32') def sample(self, shape): shape = tuple(shape) + (self.low + self.high).shape - return self.low + (np.random.uniform(size=shape) * - (self.high - self.low)) + return self.low + ( + np.random.uniform(size=shape) * (self.high - self.low) + ) def log_prob(self, value): lb = np.less(self.low, value).astype('float32') @@ -63,7 +68,6 @@ def entropy(self): class NormalNumpy(DistributionNumpy): - def __init__(self, loc, scale): self.loc = np.array(loc).astype('float32') self.scale = np.array(scale).astype('float32') @@ -75,24 +79,28 @@ def sample(self, shape): def log_prob(self, value): var = self.scale * self.scale log_scale = np.log(self.scale) - return -((value - self.loc) * - (value - self.loc)) / (2. * var) - log_scale - math.log( - math.sqrt(2. * math.pi)) + return ( + -((value - self.loc) * (value - self.loc)) / (2.0 * var) + - log_scale + - math.log(math.sqrt(2.0 * math.pi)) + ) def entropy(self): - return 0.5 + 0.5 * np.log(np.array( - 2. * math.pi).astype('float32')) + np.log(self.scale) + return ( + 0.5 + + 0.5 * np.log(np.array(2.0 * math.pi).astype('float32')) + + np.log(self.scale) + ) def kl_divergence(self, other): - var_ratio = (self.scale / other.scale) + var_ratio = self.scale / other.scale var_ratio = var_ratio * var_ratio - t1 = ((self.loc - other.loc) / other.scale) - t1 = (t1 * t1) + t1 = (self.loc - other.loc) / other.scale + t1 = t1 * t1 return 0.5 * (var_ratio + t1 - 1 - np.log(var_ratio)) class CategoricalNumpy(DistributionNumpy): - def __init__(self, logits): self.logits = np.array(logits).astype('float32') @@ -101,23 +109,28 @@ def entropy(self): e_logits = np.exp(logits) z = np.sum(e_logits, axis=-1, keepdims=True) prob = e_logits / z - return -1. * np.sum(prob * (logits - np.log(z)), axis=-1, keepdims=True) + return -1.0 * np.sum( + prob * (logits - np.log(z)), axis=-1, keepdims=True + ) def kl_divergence(self, other): logits = self.logits - np.max(self.logits, axis=-1, keepdims=True) other_logits = other.logits - np.max( - other.logits, axis=-1, keepdims=True) + other.logits, axis=-1, keepdims=True + ) e_logits = np.exp(logits) other_e_logits = np.exp(other_logits) z = np.sum(e_logits, axis=-1, keepdims=True) other_z = np.sum(other_e_logits, axis=-1, keepdims=True) prob = e_logits / z - return np.sum(prob * (logits - np.log(z) - other_logits \ - + np.log(other_z)), axis=-1, keepdims=True) + return np.sum( + prob * (logits - np.log(z) - other_logits + np.log(other_z)), + axis=-1, + keepdims=True, + ) class MultivariateNormalDiagNumpy(DistributionNumpy): - def __init__(self, loc, scale): self.loc = np.array(loc).astype('float32') self.scale = np.array(scale).astype('float32') @@ -139,14 +152,17 @@ def _inv(self, value): return inv_diag def entropy(self): - return 0.5 * (self.scale.shape[0] * - (1.0 + np.log(np.array(2 * math.pi).astype('float32'))) + - np.log(self._det(self.scale))) + return 0.5 * ( + self.scale.shape[0] + * (1.0 + np.log(np.array(2 * math.pi).astype('float32'))) + + np.log(self._det(self.scale)) + ) def kl_divergence(self, other): tr_cov_matmul = np.sum(self._inv(other.scale) * self.scale) - loc_matmul_cov = np.matmul((other.loc - self.loc), - self._inv(other.scale)) + loc_matmul_cov = np.matmul( + (other.loc - self.loc), self._inv(other.scale) + ) tri_matmul = np.matmul(loc_matmul_cov, (other.loc - self.loc)) k = list(self.scale.shape)[0] ln_cov = np.log(self._det(other.scale)) - np.log(self._det(self.scale)) @@ -156,7 +172,6 @@ def kl_divergence(self, other): class DistributionTest(unittest.TestCase): - def setUp(self, use_gpu=False): self.use_gpu = use_gpu if not use_gpu: @@ -167,20 +182,31 @@ def setUp(self, use_gpu=False): self.gpu_id = 0 self.executor = fluid.Executor(place) - def build_normal_program(self, test_program, batch_size, dims, loc_float, - scale_float, other_loc_float, other_scale_float, - scale_np, other_scale_np, loc_np, other_loc_np, - values_np): + def build_normal_program( + self, + test_program, + batch_size, + dims, + loc_float, + scale_float, + other_loc_float, + other_scale_float, + scale_np, + other_scale_np, + loc_np, + other_loc_np, + values_np, + ): with fluid.program_guard(test_program): loc = layers.data(name='loc', shape=[dims], dtype='float32') scale = layers.data(name='scale', shape=[dims], dtype='float32') - other_loc = layers.data(name='other_loc', - shape=[dims], - dtype='float32') - other_scale = layers.data(name='other_scale', - shape=[dims], - dtype='float32') + other_loc = layers.data( + name='other_loc', shape=[dims], dtype='float32' + ) + other_scale = layers.data( + name='other_scale', shape=[dims], dtype='float32' + ) values = layers.data(name='values', shape=[dims], dtype='float32') @@ -188,8 +214,9 @@ def build_normal_program(self, test_program, batch_size, dims, loc_float, other_normal_float = Normal(other_loc_float, other_scale_float) normal_float_np_broadcast = Normal(loc_float, scale_np) - other_normal_float_np_broadcast = Normal(other_loc_float, - other_scale_np) + other_normal_float_np_broadcast = Normal( + other_loc_float, other_scale_np + ) normal_np = Normal(loc_np, scale_np) other_normal_np = Normal(other_loc_np, other_scale_np) @@ -199,7 +226,8 @@ def build_normal_program(self, test_program, batch_size, dims, loc_float, sample_float = normal_float.sample([batch_size, dims]) sample_float_np_broadcast = normal_float_np_broadcast.sample( - [batch_size, dims]) + [batch_size, dims] + ) sample_np = normal_np.sample([batch_size, dims]) sample_variable = normal_variable.sample([batch_size, dims]) @@ -214,22 +242,34 @@ def build_normal_program(self, test_program, batch_size, dims, loc_float, kl_float = normal_float.kl_divergence(other_normal_float) kl_float_np_broadcast = normal_float_np_broadcast.kl_divergence( - other_normal_float_np_broadcast) + other_normal_float_np_broadcast + ) kl_np = normal_np.kl_divergence(other_normal_np) kl_variable = normal_variable.kl_divergence(other_normal_variable) fetch_list = [ - sample_float, sample_float_np_broadcast, sample_np, sample_variable, - entropy_float, entropy_float_np_broadcast, entropy_np, - entropy_variable, lp_float_np_broadcast, lp_np, lp_variable, - kl_float, kl_float_np_broadcast, kl_np, kl_variable + sample_float, + sample_float_np_broadcast, + sample_np, + sample_variable, + entropy_float, + entropy_float_np_broadcast, + entropy_np, + entropy_variable, + lp_float_np_broadcast, + lp_np, + lp_variable, + kl_float, + kl_float_np_broadcast, + kl_np, + kl_variable, ] feed_vars = { 'loc': loc_np, 'scale': scale_np, 'other_loc': other_loc_np, 'other_scale': other_scale_np, - 'values': values_np + 'values': values_np, } return feed_vars, fetch_list @@ -255,117 +295,176 @@ def get_normal_random_input(self, batch_size, dims): scale_np = np.random.randn(batch_size, dims).astype('float32') while not np.all(other_scale_np > 0): other_scale_np = np.random.randn(batch_size, dims).astype('float32') - return loc_np, other_loc_np, loc_float, scale_float, other_loc_float, \ - other_scale_float, scale_np, other_scale_np, values_np + return ( + loc_np, + other_loc_np, + loc_float, + scale_float, + other_loc_float, + other_scale_float, + scale_np, + other_scale_np, + values_np, + ) def test_normal_distribution(self, batch_size=2, dims=3, tolerance=1e-6): test_program = fluid.Program() - loc_np, other_loc_np, loc_float, scale_float, other_loc_float, other_scale_float, scale_np, other_scale_np, values_np = self.get_normal_random_input( - batch_size, dims) + ( + loc_np, + other_loc_np, + loc_float, + scale_float, + other_loc_float, + other_scale_float, + scale_np, + other_scale_np, + values_np, + ) = self.get_normal_random_input(batch_size, dims) feed_vars, fetch_list = self.build_normal_program( - test_program, batch_size, dims, loc_float, scale_float, - other_loc_float, other_scale_float, scale_np, other_scale_np, - loc_np, other_loc_np, values_np) + test_program, + batch_size, + dims, + loc_float, + scale_float, + other_loc_float, + other_scale_float, + scale_np, + other_scale_np, + loc_np, + other_loc_np, + values_np, + ) self.executor.run(fluid.default_startup_program()) np_normal_float = NormalNumpy(loc_float, scale_float) np_other_normal_float = NormalNumpy(other_loc_float, other_scale_float) np_normal_float_np_broadcast = NormalNumpy(loc_float, scale_np) np_other_normal_float_np_broadcast = NormalNumpy( - other_loc_float, other_scale_np) + other_loc_float, other_scale_np + ) np_normal = NormalNumpy(loc_np, scale_np) np_other_normal = NormalNumpy(other_loc_np, other_scale_np) gt_sample_float = np_normal_float.sample([batch_size, dims]) gt_sample_float_np_broadcast = np_normal_float_np_broadcast.sample( - [batch_size, dims]) + [batch_size, dims] + ) gt_sample_np = np_normal.sample([batch_size, dims]) gt_entropy_float = np_normal_float.entropy() gt_entropy_float_np_broadcast = np_normal_float_np_broadcast.entropy() gt_entropy = np_normal.entropy() gt_lp_float_np_broadcast = np_normal_float_np_broadcast.log_prob( - values_np) + values_np + ) gt_lp = np_normal.log_prob(values_np) gt_kl_float = np_normal_float.kl_divergence(np_other_normal_float) gt_kl_float_np_broadcast = np_normal_float_np_broadcast.kl_divergence( - np_other_normal_float_np_broadcast) + np_other_normal_float_np_broadcast + ) gt_kl = np_normal.kl_divergence(np_other_normal) [ - output_sample_float, output_sample_float_np_broadcast, - output_sample_np, output_sample_variable, output_entropy_float, - output_entropy_float_np_broadcast, output_entropy_np, - output_entropy_variable, output_lp_float_np_broadcast, output_lp_np, - output_lp_variable, output_kl_float, output_kl_float_np_broadcast, - output_kl_np, output_kl_variable - ] = self.executor.run(program=test_program, - feed=feed_vars, - fetch_list=fetch_list) - - np.testing.assert_allclose(output_sample_float.shape, - gt_sample_float.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_sample_float_np_broadcast.shape, - gt_sample_float_np_broadcast.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_sample_np.shape, - gt_sample_np.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_sample_variable.shape, - gt_sample_np.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_float, - gt_entropy_float, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_float_np_broadcast, - gt_entropy_float_np_broadcast, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_np, - gt_entropy, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_variable, - gt_entropy, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_lp_float_np_broadcast, - gt_lp_float_np_broadcast, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_lp_np, - gt_lp, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_lp_variable, - gt_lp, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_kl_float, - gt_kl_float, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_kl_float_np_broadcast, - gt_kl_float_np_broadcast, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_kl_np, - gt_kl, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_kl_variable, - gt_kl, - rtol=tolerance, - atol=tolerance) - - def build_uniform_program(self, test_program, batch_size, dims, low_float, - high_float, high_np, low_np, values_np): + output_sample_float, + output_sample_float_np_broadcast, + output_sample_np, + output_sample_variable, + output_entropy_float, + output_entropy_float_np_broadcast, + output_entropy_np, + output_entropy_variable, + output_lp_float_np_broadcast, + output_lp_np, + output_lp_variable, + output_kl_float, + output_kl_float_np_broadcast, + output_kl_np, + output_kl_variable, + ] = self.executor.run( + program=test_program, feed=feed_vars, fetch_list=fetch_list + ) + + np.testing.assert_allclose( + output_sample_float.shape, + gt_sample_float.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_sample_float_np_broadcast.shape, + gt_sample_float_np_broadcast.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_sample_np.shape, + gt_sample_np.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_sample_variable.shape, + gt_sample_np.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_entropy_float, + gt_entropy_float, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_entropy_float_np_broadcast, + gt_entropy_float_np_broadcast, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_entropy_np, gt_entropy, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_entropy_variable, gt_entropy, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_lp_float_np_broadcast, + gt_lp_float_np_broadcast, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_lp_np, gt_lp, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_lp_variable, gt_lp, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_kl_float, gt_kl_float, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_kl_float_np_broadcast, + gt_kl_float_np_broadcast, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_kl_np, gt_kl, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_kl_variable, gt_kl, rtol=tolerance, atol=tolerance + ) + + def build_uniform_program( + self, + test_program, + batch_size, + dims, + low_float, + high_float, + high_np, + low_np, + values_np, + ): with fluid.program_guard(test_program): low = layers.data(name='low', shape=[dims], dtype='float32') high = layers.data(name='high', shape=[dims], dtype='float32') @@ -379,7 +478,8 @@ def build_uniform_program(self, test_program, batch_size, dims, low_float, sample_float = uniform_float.sample([batch_size, dims]) sample_float_np_broadcast = uniform_float_np_broadcast.sample( - [batch_size, dims]) + [batch_size, dims] + ) sample_np = uniform_np.sample([batch_size, dims]) sample_variable = uniform_variable.sample([batch_size, dims]) @@ -393,9 +493,17 @@ def build_uniform_program(self, test_program, batch_size, dims, low_float, lp_variable = uniform_variable.log_prob(values) fetch_list = [ - sample_float, sample_float_np_broadcast, sample_np, sample_variable, - entropy_float, entropy_float_np_broadcast, entropy_np, - entropy_variable, lp_float_np_broadcast, lp_np, lp_variable + sample_float, + sample_float_np_broadcast, + sample_np, + sample_variable, + entropy_float, + entropy_float_np_broadcast, + entropy_np, + entropy_variable, + lp_float_np_broadcast, + lp_np, + lp_variable, ] feed_vars = {'low': low_np, 'high': high_np, 'values': values_np} return feed_vars, fetch_list @@ -406,13 +514,21 @@ def test_uniform_distribution(self, batch_size=2, dims=3, tolerance=1e-6): low_np = np.random.randn(batch_size, dims).astype('float32') low_float = np.random.uniform(-2, 1) high_float = np.random.uniform(1, 3) - high_np = np.random.uniform(-5.0, 5.0, - (batch_size, dims)).astype('float32') + high_np = np.random.uniform(-5.0, 5.0, (batch_size, dims)).astype( + 'float32' + ) values_np = np.random.randn(batch_size, dims).astype('float32') feed_vars, fetch_list = self.build_uniform_program( - test_program, batch_size, dims, low_float, high_float, high_np, - low_np, values_np) + test_program, + batch_size, + dims, + low_float, + high_float, + high_np, + low_np, + values_np, + ) self.executor.run(fluid.default_startup_program()) @@ -422,75 +538,92 @@ def test_uniform_distribution(self, batch_size=2, dims=3, tolerance=1e-6): gt_sample_float = np_uniform_float.sample([batch_size, dims]) gt_sample_float_np_broadcast = np_uniform_float_np_broadcast.sample( - [batch_size, dims]) + [batch_size, dims] + ) gt_sample_np = np_uniform.sample([batch_size, dims]) gt_entropy_float = np_uniform_float.entropy() gt_entropy_float_np_broadcast = np_uniform_float_np_broadcast.entropy() gt_entropy = np_uniform.entropy() gt_lp_float_np_broadcast = np_uniform_float_np_broadcast.log_prob( - values_np) + values_np + ) gt_lp = np_uniform.log_prob(values_np) # result calculated by paddle [ - output_sample_float, output_sample_float_np_broadcast, - output_sample_np, output_sample_variable, output_entropy_float, - output_entropy_float_np_broadcast, output_entropy_np, - output_entropy_variable, output_lp_float_np_broadcast, output_lp_np, - output_lp_variable - ] = self.executor.run(program=test_program, - feed=feed_vars, - fetch_list=fetch_list) - - np.testing.assert_allclose(output_sample_float.shape, - gt_sample_float.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_sample_float_np_broadcast.shape, - gt_sample_float_np_broadcast.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_sample_np.shape, - gt_sample_np.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_sample_variable.shape, - gt_sample_np.shape, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_float, - gt_entropy_float, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_float_np_broadcast, - gt_entropy_float_np_broadcast, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_np, - gt_entropy, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_entropy_variable, - gt_entropy, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_lp_float_np_broadcast, - gt_lp_float_np_broadcast, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_lp_np, - gt_lp, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_lp_variable, - gt_lp, - rtol=tolerance, - atol=tolerance) - - def test_categorical_distribution(self, - batch_size=2, - dims=3, - tolerance=1e-6): + output_sample_float, + output_sample_float_np_broadcast, + output_sample_np, + output_sample_variable, + output_entropy_float, + output_entropy_float_np_broadcast, + output_entropy_np, + output_entropy_variable, + output_lp_float_np_broadcast, + output_lp_np, + output_lp_variable, + ] = self.executor.run( + program=test_program, feed=feed_vars, fetch_list=fetch_list + ) + + np.testing.assert_allclose( + output_sample_float.shape, + gt_sample_float.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_sample_float_np_broadcast.shape, + gt_sample_float_np_broadcast.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_sample_np.shape, + gt_sample_np.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_sample_variable.shape, + gt_sample_np.shape, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_entropy_float, + gt_entropy_float, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_entropy_float_np_broadcast, + gt_entropy_float_np_broadcast, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_entropy_np, gt_entropy, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_entropy_variable, gt_entropy, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_lp_float_np_broadcast, + gt_lp_float_np_broadcast, + rtol=tolerance, + atol=tolerance, + ) + np.testing.assert_allclose( + output_lp_np, gt_lp, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_lp_variable, gt_lp, rtol=tolerance, atol=tolerance + ) + + def test_categorical_distribution( + self, batch_size=2, dims=3, tolerance=1e-6 + ): test_program = fluid.Program() logits_np = np.random.randn(batch_size, dims).astype('float32') @@ -498,9 +631,9 @@ def test_categorical_distribution(self, with fluid.program_guard(test_program): logits = layers.data(name='logits', shape=[dims], dtype='float32') - other_logits = layers.data(name='other_logits', - shape=[dims], - dtype='float32') + other_logits = layers.data( + name='other_logits', shape=[dims], dtype='float32' + ) categorical_np = Categorical(logits_np) other_categorical_np = Categorical(other_logits_np) @@ -516,55 +649,74 @@ def test_categorical_distribution(self, gt_kl_np = np_categorical.kl_divergence(np_other_categorical) # result calculated by paddle - [output_entropy_np, - output_kl_np] = self.executor.run(program=test_program, - feed={'logits': logits_np}, - fetch_list=[entropy_np, kl_np]) - np.testing.assert_allclose(output_entropy_np, - gt_entropy_np, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_kl_np, - gt_kl_np, - rtol=tolerance, - atol=tolerance) - - def test_multivariateNormalDiag_distribution(self, - batch_size=2, - tolerance=1e-6): + [output_entropy_np, output_kl_np] = self.executor.run( + program=test_program, + feed={'logits': logits_np}, + fetch_list=[entropy_np, kl_np], + ) + np.testing.assert_allclose( + output_entropy_np, gt_entropy_np, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_kl_np, gt_kl_np, rtol=tolerance, atol=tolerance + ) + + def test_multivariateNormalDiag_distribution( + self, batch_size=2, tolerance=1e-6 + ): test_program = fluid.Program() - loc_np = np.random.random(batch_size, ).astype('float32') - scale_np = np.diag(np.random.random(batch_size, )).astype('float32') - other_loc_np = np.random.random(batch_size, ).astype('float32') - other_scale_np = np.diag(np.random.random( - batch_size, )).astype('float32') + loc_np = np.random.random( + batch_size, + ).astype('float32') + scale_np = np.diag( + np.random.random( + batch_size, + ) + ).astype('float32') + other_loc_np = np.random.random( + batch_size, + ).astype('float32') + other_scale_np = np.diag( + np.random.random( + batch_size, + ) + ).astype('float32') with fluid.program_guard(test_program): - loc = layers.data(name='loc', - shape=[ - batch_size, - ], - dtype='float32', - append_batch_size=False) - scale = layers.data(name='scale', - shape=[batch_size, batch_size], - dtype='float32', - append_batch_size=False) - other_loc = layers.data(name='other_loc', - shape=[ - batch_size, - ], - dtype='float32', - append_batch_size=False) - other_scale = layers.data(name='other_scale', - shape=[batch_size, batch_size], - dtype='float32', - append_batch_size=False) + loc = layers.data( + name='loc', + shape=[ + batch_size, + ], + dtype='float32', + append_batch_size=False, + ) + scale = layers.data( + name='scale', + shape=[batch_size, batch_size], + dtype='float32', + append_batch_size=False, + ) + other_loc = layers.data( + name='other_loc', + shape=[ + batch_size, + ], + dtype='float32', + append_batch_size=False, + ) + other_scale = layers.data( + name='other_scale', + shape=[batch_size, batch_size], + dtype='float32', + append_batch_size=False, + ) multivariate_np = MultivariateNormalDiag(loc, scale) other_multivariate_np = MultivariateNormalDiag( - other_loc, other_scale) + other_loc, other_scale + ) entropy_np = multivariate_np.entropy() other_entropy_np = other_multivariate_np.entropy() @@ -574,32 +726,31 @@ def test_multivariateNormalDiag_distribution(self, np_multivariate = MultivariateNormalDiagNumpy(loc_np, scale_np) np_other_multivariate = MultivariateNormalDiagNumpy( - other_loc_np, other_scale_np) + other_loc_np, other_scale_np + ) gt_entropy_np = np_multivariate.entropy() gt_kl_np = np_multivariate.kl_divergence(np_other_multivariate) # result calculated by paddle - [output_entropy_np, - output_kl_np] = self.executor.run(program=test_program, - feed={ - 'loc': loc_np, - 'scale': scale_np, - 'other_loc': other_loc_np, - 'other_scale': other_scale_np - }, - fetch_list=[entropy_np, kl_np]) - np.testing.assert_allclose(output_entropy_np, - gt_entropy_np, - rtol=tolerance, - atol=tolerance) - np.testing.assert_allclose(output_kl_np, - gt_kl_np, - rtol=tolerance, - atol=tolerance) + [output_entropy_np, output_kl_np] = self.executor.run( + program=test_program, + feed={ + 'loc': loc_np, + 'scale': scale_np, + 'other_loc': other_loc_np, + 'other_scale': other_scale_np, + }, + fetch_list=[entropy_np, kl_np], + ) + np.testing.assert_allclose( + output_entropy_np, gt_entropy_np, rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose( + output_kl_np, gt_kl_np, rtol=tolerance, atol=tolerance + ) class DistributionTestError(unittest.TestCase): - def test_normal_error(self): loc = int(1) scale = int(1) @@ -658,8 +809,9 @@ def test_categorical_error(self): categorical_other = Normal(1.0, 2.0) # type of other must be an instance of Normal - self.assertRaises(TypeError, categorical.kl_divergence, - categorical_other) + self.assertRaises( + TypeError, categorical.kl_divergence, categorical_other + ) def test_multivariate_normal_diag_error(self): loc = 1.0 diff --git a/python/paddle/fluid/tests/unittests/test_dot_op.py b/python/paddle/fluid/tests/unittests/test_dot_op.py index d9850a88d0dc4..006c91d60e64b 100644 --- a/python/paddle/fluid/tests/unittests/test_dot_op.py +++ b/python/paddle/fluid/tests/unittests/test_dot_op.py @@ -22,7 +22,6 @@ class DotOp(OpTest): - def setUp(self): self.op_type = "dot" self.python_api = paddle.dot @@ -31,7 +30,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {} @@ -45,35 +44,38 @@ def test_check_grad_normal(self): ['X', 'Y'], 'Out', user_defined_grads=[self.inputs['Y'], self.inputs['X']], - check_eager=True) + check_eager=True, + ) else: self.check_grad(['X', 'Y'], 'Out', check_eager=True) def test_check_grad_ingore_x(self): if core.is_compiled_with_rocm(): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.inputs['X']], - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.inputs['X']], + check_eager=True, + ) else: - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - check_eager=True) + self.check_grad( + ['Y'], 'Out', no_grad_set=set("X"), check_eager=True + ) def test_check_grad_ingore_y(self): if core.is_compiled_with_rocm(): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.inputs['Y']], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.inputs['Y']], + check_eager=True, + ) else: - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - check_eager=True) + self.check_grad( + ['X'], 'Out', no_grad_set=set('Y'), check_eager=True + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [121]).astype(self.dtype) @@ -85,12 +87,15 @@ def init_dtype(self): class DotOpBatch(DotOp): - def init_input_output(self): - self.x = np.random.uniform(0.1, 1, - [132]).astype(self.dtype).reshape([11, 12]) - self.y = np.random.uniform(1, 3, - [132]).astype(self.dtype).reshape([11, 12]) + self.x = ( + np.random.uniform(0.1, 1, [132]) + .astype(self.dtype) + .reshape([11, 12]) + ) + self.y = ( + np.random.uniform(1, 3, [132]).astype(self.dtype).reshape([11, 12]) + ) self.out = np.sum(self.x * self.y, axis=1).reshape([11, 1]) def test_check_grad_normal(self): @@ -104,7 +109,6 @@ def test_check_grad_ingore_y(self): class TestDotOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -124,25 +128,26 @@ def test_errors(self): class TestDygraph(unittest.TestCase): - def test_dygraph(self): with fluid.dygraph.guard(): x1 = fluid.dygraph.to_variable(np.array([1, 3]).astype(np.float32)) y1 = fluid.dygraph.to_variable(np.array([2, 5]).astype(np.float32)) - np.testing.assert_allclose(paddle.dot(x1, y1).numpy(), - np.array([17]), - rtol=1e-05) + np.testing.assert_allclose( + paddle.dot(x1, y1).numpy(), np.array([17]), rtol=1e-05 + ) x1 = fluid.dygraph.to_variable( - np.array([[1, 3], [3, 5]]).astype(np.float32)) + np.array([[1, 3], [3, 5]]).astype(np.float32) + ) y1 = fluid.dygraph.to_variable( - np.array([[2, 5], [6, 8]]).astype(np.float32)) + np.array([[2, 5], [6, 8]]).astype(np.float32) + ) np.testing.assert_array_equal( - paddle.dot(x1, y1).numpy(), np.array([[17], [58]])) + paddle.dot(x1, y1).numpy(), np.array([[17], [58]]) + ) class TestComplexDotOp(OpTest): - def setUp(self): self.op_type = "dot" self.python_api = paddle.dot @@ -152,7 +157,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} @@ -161,13 +166,15 @@ def init_base_dtype(self): def init_input_output(self): self.x = np.random.random(100).astype( - self.dtype) + 1J * np.random.random(100).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(100).astype(self.dtype) self.y = np.random.random(100).astype( - self.dtype) + 1J * np.random.random(100).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(100).astype(self.dtype) self.out = np.dot(self.x, self.y) def init_grad_input_output(self): - self.grad_out = np.ones(1, self.dtype) + 1J * np.ones(1, self.dtype) + self.grad_out = np.ones(1, self.dtype) + 1j * np.ones(1, self.dtype) self.grad_x = self.grad_out * np.conj(self.y) self.grad_y = self.grad_out * np.conj(self.x) @@ -175,31 +182,36 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) class TestComplexDotOp2D(OpTest): - def setUp(self): self.op_type = "dot" self.init_base_dtype() @@ -208,7 +220,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} @@ -216,17 +228,18 @@ def init_base_dtype(self): self.dtype = np.float64 def init_input_output(self): - self.x = np.random.random( - (2, 100)).astype(self.dtype) + 1J * np.random.random( - (2, 100)).astype(self.dtype) - self.y = np.random.random( - (2, 100)).astype(self.dtype) + 1J * np.random.random( - (2, 100)).astype(self.dtype) + self.x = np.random.random((2, 100)).astype( + self.dtype + ) + 1j * np.random.random((2, 100)).astype(self.dtype) + self.y = np.random.random((2, 100)).astype( + self.dtype + ) + 1j * np.random.random((2, 100)).astype(self.dtype) self.out = np.diag(np.dot(self.x, self.y.T)).reshape(-1, 1) def init_grad_input_output(self): - self.grad_out = np.ones((2, 1), self.dtype) + 1J * np.ones( - (2, 1), self.dtype) + self.grad_out = np.ones((2, 1), self.dtype) + 1j * np.ones( + (2, 1), self.dtype + ) self.grad_x = self._get_grad(self.grad_out, self.y) self.grad_y = self._get_grad(self.grad_out, self.x) @@ -240,24 +253,30 @@ def test_check_output(self): self.check_output() def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_downpoursgd.py b/python/paddle/fluid/tests/unittests/test_downpoursgd.py index f241a854c89c3..6626f0ebfa047 100644 --- a/python/paddle/fluid/tests/unittests/test_downpoursgd.py +++ b/python/paddle/fluid/tests/unittests/test_downpoursgd.py @@ -18,7 +18,10 @@ import os import unittest import sys -from paddle.fluid.incubate.fleet.parameter_server.pslib.node import DownpourWorker, DownpourServer +from paddle.fluid.incubate.fleet.parameter_server.pslib.node import ( + DownpourWorker, + DownpourServer, +) from google.protobuf import text_format import paddle.fluid.incubate.fleet.parameter_server.pslib.ps_pb2 as pslib from paddle.fluid.trainer_factory import TrainerFactory @@ -40,15 +43,17 @@ def test_device_work_use_cvm(self): pass else: print(sys.platform) - if not os.path.exists('{}/{}'.format(cache_path, - 'fleet_desc.prototxt')): + if not os.path.exists( + '{}/{}'.format(cache_path, 'fleet_desc.prototxt') + ): cmd = "wget --no-check-certificate https://pslib.bj.bcebos.com/fleet_desc.prototxt -P {}/".format( - cache_path) + cache_path + ) os.system(cmd) x = fluid.layers.data(name='x', shape=[1], dtype='int64') - x_emb = fluid.layers.embedding(input=x, - size=[1, 2], - is_distributed=True) + x_emb = fluid.layers.embedding( + input=x, size=[1, 2], is_distributed=True + ) y_predict = fluid.layers.fc(input=x_emb, size=1, act=None) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) @@ -67,7 +72,7 @@ def test_device_work_use_cvm(self): program_configs = {} program_configs[program_id] = { "pull_sparse": [0], - "push_sparse": [0] + "push_sparse": [0], } program_configs[program_id]["pull_dense"] = [1] program_configs[program_id]["push_dense"] = [1] @@ -100,15 +105,17 @@ def test_device_work(self): pass else: print(sys.platform) - if not os.path.exists('{}/{}'.format(cache_path, - 'fleet_desc.prototxt')): + if not os.path.exists( + '{}/{}'.format(cache_path, 'fleet_desc.prototxt') + ): cmd = "wget --no-check-certificate https://pslib.bj.bcebos.com/fleet_desc.prototxt -P {}/".format( - cache_path) + cache_path + ) os.system(cmd) x = fluid.layers.data(name='x', shape=[1], dtype='int64') - x_emb = fluid.layers.embedding(input=x, - size=[1, 2], - is_distributed=True) + x_emb = fluid.layers.embedding( + input=x, size=[1, 2], is_distributed=True + ) y_predict = fluid.layers.fc(input=x_emb, size=1, act=None) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) @@ -127,7 +134,7 @@ def test_device_work(self): program_configs = {} program_configs[program_id] = { "pull_sparse": [0], - "push_sparse": [0] + "push_sparse": [0], } program_configs[program_id]["pull_dense"] = [1] program_configs[program_id]["push_dense"] = [1] @@ -158,15 +165,17 @@ def test_downpour_opt_work(self): pass else: print(sys.platform) - if not os.path.exists('{}/{}'.format(cache_path, - 'fleet_desc.prototxt')): + if not os.path.exists( + '{}/{}'.format(cache_path, 'fleet_desc.prototxt') + ): cmd = "wget --no-check-certificate https://pslib.bj.bcebos.com/fleet_desc.prototxt -P {}/".format( - cache_path) + cache_path + ) os.system(cmd) x = fluid.layers.data(name='x', shape=[1], dtype='int64') - x_emb = fluid.layers.embedding(input=x, - size=[1, 2], - is_distributed=True) + x_emb = fluid.layers.embedding( + input=x, size=[1, 2], is_distributed=True + ) y_predict = fluid.layers.fc(input=x_emb, size=1, act=None) y = fluid.layers.data(name='y', shape=[1], dtype='float32') cost = fluid.layers.square_error_cost(input=y_predict, label=y) @@ -185,7 +194,7 @@ def test_downpour_opt_work(self): program_configs = {} program_configs[program_id] = { "pull_sparse": [0], - "push_sparse": [0] + "push_sparse": [0], } program_configs[program_id]["pull_dense"] = [1] program_configs[program_id]["push_dense"] = [1] diff --git a/python/paddle/fluid/tests/unittests/test_dpsgd_op.py b/python/paddle/fluid/tests/unittests/test_dpsgd_op.py index 1b529d1760128..2e505c05a891f 100644 --- a/python/paddle/fluid/tests/unittests/test_dpsgd_op.py +++ b/python/paddle/fluid/tests/unittests/test_dpsgd_op.py @@ -18,10 +18,8 @@ class TestDpsgdOp(OpTest): - def setUp(self): - '''Test Dpsgd Operator with supplied attributes - ''' + '''Test Dpsgd Operator with supplied attributes''' self.op_type = "dpsgd" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -34,7 +32,7 @@ def setUp(self): self.inputs = { 'Param': param, 'Grad': grad, - 'LearningRate': np.array([learning_rate]).astype("float32") + 'LearningRate': np.array([learning_rate]).astype("float32"), } self.attrs = {'clip': clip, 'batch_size': batch_size, 'sigma': sigma} @@ -70,5 +68,6 @@ def dpsgd_step(inputs, attributes): if __name__ == "__main__": import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_dropout_nd_op.py b/python/paddle/fluid/tests/unittests/test_dropout_nd_op.py index d558032dd3727..a21ec9625f927 100644 --- a/python/paddle/fluid/tests/unittests/test_dropout_nd_op.py +++ b/python/paddle/fluid/tests/unittests/test_dropout_nd_op.py @@ -24,34 +24,44 @@ from paddle.static import default_main_program -def dropout_nd(x, - p=0.5, - axis=None, - training=True, - mode="upscale_in_train", - name=None): +def dropout_nd( + x, p=0.5, axis=None, training=True, mode="upscale_in_train", name=None +): drop_axes = [axis] if isinstance(axis, int) else list(axis) seed = None - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer if _non_static_mode(): if default_main_program().random_seed != 0: seed = default_main_program().random_seed - out, mask = _legacy_C_ops.dropout_nd(x, 'dropout_prob', p, 'is_test', - not training, 'fix_seed', seed - is not None, 'seed', - seed if seed is not None else 0, - 'dropout_implementation', mode, - 'axis', drop_axes) + out, mask = _legacy_C_ops.dropout_nd( + x, + 'dropout_prob', + p, + 'is_test', + not training, + 'fix_seed', + seed is not None, + 'seed', + seed if seed is not None else 0, + 'dropout_implementation', + mode, + 'axis', + drop_axes, + ) return out helper = LayerHelper('dropout_nd', **locals()) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'dropout') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'dropout' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) mask = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) def get_attrs(prog, dropout_prob, is_test, seed): if (seed is None or seed == 0) and prog.random_seed != 0: @@ -62,19 +72,18 @@ def get_attrs(prog, dropout_prob, is_test, seed): 'fix_seed': seed is not None, 'seed': seed if seed is not None else 0, 'dropout_implementation': mode, - 'axis': drop_axes + 'axis': drop_axes, } return attrs attrs = get_attrs(helper.main_program, p, not training, seed) - helper.append_op(type='dropout_nd', - inputs={'X': [x]}, - outputs={ - 'Out': [out], - 'Mask': [mask] - }, - attrs=attrs) + helper.append_op( + type='dropout_nd', + inputs={'X': [x]}, + outputs={'Out': [out], 'Mask': [mask]}, + attrs=attrs, + ) return out @@ -82,7 +91,6 @@ def get_attrs(prog, dropout_prob, is_test, seed): class TestDropoutNdOp(OpTest): - def setUp(self): self.op_type = "dropout_nd" self.inputs = {'X': np.random.random((4, 32, 16)).astype("float64")} @@ -90,11 +98,11 @@ def setUp(self): 'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False, - 'axis': [1] + 'axis': [1], } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((1, 32, 1)).astype('uint8') + 'Mask': np.ones((1, 32, 1)).astype('uint8'), } def test_check_output(self): @@ -105,7 +113,6 @@ def test_check_grad_normal(self): class TestDropoutNdAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -118,7 +125,7 @@ def test_dygraph(self): with fluid.dygraph.guard(place): in_np = np.random.random([4, 32, 16]).astype("float32") input = paddle.to_tensor(in_np) - res1 = dropout_nd(x=input, p=0., axis=[0, 1]) + res1 = dropout_nd(x=input, p=0.0, axis=[0, 1]) res2 = dropout_nd(x=input, p=0.5, axis=[0, 1]) np.testing.assert_allclose(res1.numpy(), in_np, rtol=1e-05) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_dropout_op.py b/python/paddle/fluid/tests/unittests/test_dropout_op.py index b7da098182b9f..862ca0c1f9135 100644 --- a/python/paddle/fluid/tests/unittests/test_dropout_op.py +++ b/python/paddle/fluid/tests/unittests/test_dropout_op.py @@ -26,14 +26,13 @@ class TestDropoutOp(OpTest): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64)).astype("float32")} self.attrs = {'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False} self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64)).astype('uint8') + 'Mask': np.ones((32, 64)).astype('uint8'), } def test_check_output(self): @@ -44,14 +43,13 @@ def test_check_grad_normal(self): class TestDropoutOpInput1d(OpTest): - def setUp(self): self.op_type = "dropout" - self.inputs = {'X': np.random.random((2000, )).astype("float32")} + self.inputs = {'X': np.random.random((2000,)).astype("float32")} self.attrs = {'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False} self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((2000)).astype('uint8') + 'Mask': np.ones((2000)).astype('uint8'), } def test_check_output(self): @@ -62,32 +60,29 @@ def test_check_grad_normal(self): class TestDropoutOp2(TestDropoutOp): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64)).astype("float32")} self.attrs = {'dropout_prob': 1.0, 'fix_seed': True, 'is_test': False} self.outputs = { 'Out': np.zeros((32, 64)).astype('float32'), - 'Mask': np.zeros((32, 64)).astype('uint8') + 'Mask': np.zeros((32, 64)).astype('uint8'), } class TestDropoutOp3(TestDropoutOp): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64, 2)).astype("float32")} self.attrs = {'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False} self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64, 2)).astype('uint8') + 'Mask': np.ones((32, 64, 2)).astype('uint8'), } @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestDropoutOp4(OpTest): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64)).astype("float32")} @@ -102,7 +97,6 @@ def test_check_output(self): @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestDropoutOp5(OpTest): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64, 3)).astype("float32")} @@ -116,7 +110,6 @@ def test_check_output(self): class TestDropoutOp6(TestDropoutOp): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64)).astype("float32")} @@ -124,16 +117,15 @@ def setUp(self): 'dropout_prob': 1.0, 'fix_seed': True, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': np.zeros((32, 64)).astype('float32'), - 'Mask': np.zeros((32, 64)).astype('uint8') + 'Mask': np.zeros((32, 64)).astype('uint8'), } class TestDropoutOp7(TestDropoutOp): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64, 2)).astype("float32")} @@ -141,17 +133,16 @@ def setUp(self): 'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64, 2)).astype('uint8') + 'Mask': np.ones((32, 64, 2)).astype('uint8'), } @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestDropoutOp8(OpTest): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64)).astype("float32")} @@ -159,7 +150,7 @@ def setUp(self): 'dropout_prob': 0.35, 'fix_seed': True, 'is_test': True, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = {'Out': self.inputs['X']} @@ -169,14 +160,13 @@ def test_check_output(self): @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestDropoutOp9(OpTest): - def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64, 3)).astype("float32")} self.attrs = { 'dropout_prob': 0.75, 'is_test': True, - 'dropout_implementation': 'upscale_in_train' + 'dropout_implementation': 'upscale_in_train', } self.outputs = {'Out': self.inputs['X']} @@ -185,19 +175,18 @@ def test_check_output(self): class TestDropoutOpWithSeed(OpTest): - def setUp(self): self.op_type = "dropout" self.inputs = { "X": np.random.random((32, 64)).astype("float32"), - "Seed": np.asarray([125], dtype="int32") + "Seed": np.asarray([125], dtype="int32"), } self.attrs = { 'dropout_prob': 0.0, } self.outputs = { 'Out': self.inputs['X'], - 'Mask': np.ones((32, 64)).astype('uint8') + 'Mask': np.ones((32, 64)).astype('uint8'), } def test_check_output(self): @@ -207,13 +196,12 @@ def test_check_grad_normal(self): self.check_grad(['X'], 'Out', max_relative_error=0.05) -@unittest.skipIf(not core.is_compiled_with_cuda() - or not core.op_support_gpu("dropout"), - "core is not compiled with CUDA or core is not support dropout" - ) +@unittest.skipIf( + not core.is_compiled_with_cuda() or not core.op_support_gpu("dropout"), + "core is not compiled with CUDA or core is not support dropout", +) @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestFP16DropoutOp(OpTest): - def setUp(self): self.op_type = "dropout" self.init_test_case() @@ -224,7 +212,7 @@ def setUp(self): self.attrs = { 'dropout_prob': self.prob, 'fix_seed': self.fix_seed, - 'is_test': True + 'is_test': True, } self.outputs = {'Out': out} @@ -237,13 +225,12 @@ def test_check_output(self): self.check_output_with_place(core.CUDAPlace(0), atol=1e-3) -@unittest.skipIf(not core.is_compiled_with_cuda() - or not core.op_support_gpu("dropout"), - "core is not compiled with CUDA or core is not support dropout" - ) +@unittest.skipIf( + not core.is_compiled_with_cuda() or not core.op_support_gpu("dropout"), + "core is not compiled with CUDA or core is not support dropout", +) @skip_check_grad_ci(reason="For inference, check_grad is not required.") class TestFP16DropoutOp2(TestFP16DropoutOp): - def init_test_case(self): self.input_size = [32, 64, 3] self.prob = 0.75 @@ -251,7 +238,6 @@ def init_test_case(self): class TestBF16DropoutOp(OpTest): - def setUp(self): self.op_type = "dropout" self.dtype = np.uint16 @@ -260,9 +246,10 @@ def setUp(self): self.inputs = {'X': convert_float_to_uint16(x)} self.attrs = {'dropout_prob': 1.0, 'fix_seed': True, 'is_test': False} self.outputs = { - 'Out': - convert_float_to_uint16(np.zeros((32, 64)).astype('float32')), - 'Mask': np.zeros((32, 64)).astype('uint8') + 'Out': convert_float_to_uint16( + np.zeros((32, 64)).astype('float32') + ), + 'Mask': np.zeros((32, 64)).astype('uint8'), } def test_check_output(self): @@ -273,7 +260,6 @@ def test_check_grad_normal(self): class TestDropoutOpWithSeedOnCPUPlace(unittest.TestCase): - def test_seed_cpu_place(self): paddle.enable_static() main_program = Program() @@ -288,51 +274,52 @@ def test_seed_cpu_place(self): shape=[1], dtype='int32', persistable=False, - stop_gradient=True) + stop_gradient=True, + ) x_out_var = main_program.global_block().create_var( name=x_out_var, shape=[40, 40], dtype='float32', persistable=False, - stop_gradient=True) - x_var = main_program.global_block().create_var(name=x_var_name, - shape=[40, 40], - dtype='float32', - persistable=False, - stop_gradient=True) + stop_gradient=True, + ) + x_var = main_program.global_block().create_var( + name=x_var_name, + shape=[40, 40], + dtype='float32', + persistable=False, + stop_gradient=True, + ) mask_var = main_program.global_block().create_var( name=mask_var_name, shape=[1], dtype='int', persistable=False, - stop_gradient=True) - - main_program.global_block().append_op(type="fill_constant", - outputs={"Out": x_var_name}, - attrs={ - "shape": [40, 40], - "dtype": x_var.dtype, - "value": 1.0, - "place_type": 0 - }) + stop_gradient=True, + ) + + main_program.global_block().append_op( + type="fill_constant", + outputs={"Out": x_var_name}, + attrs={ + "shape": [40, 40], + "dtype": x_var.dtype, + "value": 1.0, + "place_type": 0, + }, + ) main_program.global_block().append_op( type='seed', inputs={}, outputs={'Out': seed_input_var}, - attrs={ - 'seed': 1, - 'force_cpu': True - }) - main_program.global_block().append_op(type='dropout', - inputs={ - 'X': x_var, - 'Seed': seed_input_var - }, - attrs={'dropout_prob': 0.}, - outputs={ - 'Out': x_out_var, - 'Mask': mask_var - }) + attrs={'seed': 1, 'force_cpu': True}, + ) + main_program.global_block().append_op( + type='dropout', + inputs={'X': x_var, 'Seed': seed_input_var}, + attrs={'dropout_prob': 0.0}, + outputs={'Out': x_out_var, 'Mask': mask_var}, + ) place = fluid.CPUPlace() if core.is_compiled_with_cuda(): place = fluid.CUDAPlace(0) @@ -340,20 +327,21 @@ def test_seed_cpu_place(self): x_out, mask_out = exe.run( main_program, feed={}, - fetch_list=[x_out_var.name, mask_var.name]) + fetch_list=[x_out_var.name, mask_var.name], + ) x_in_np = np.ones([40, 40]).astype("float32") np.testing.assert_allclose(x_out, x_in_np, rtol=1e-05) class TestDropoutOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of dropout must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) fluid.layers.dropout(x1, dropout_prob=0.5) self.assertRaises(TypeError, test_Variable) @@ -361,16 +349,15 @@ def test_Variable(): def test_dtype(): # the input dtype of dropout must be float16 or float32 or float64 # float16 only can be set on GPU place - x2 = fluid.layers.data(name='x2', - shape=[3, 4, 5, 6], - dtype="int32") + x2 = fluid.layers.data( + name='x2', shape=[3, 4, 5, 6], dtype="int32" + ) fluid.layers.dropout(x2, dropout_prob=0.5) self.assertRaises(TypeError, test_dtype) class TestDropoutFAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -380,60 +367,64 @@ def setUp(self): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): input = fluid.data(name="input", shape=[-1, -1], dtype="float32") - res1 = paddle.nn.functional.dropout(x=input, p=0., training=False) - res2 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=True, - mode='upscale_in_train') - res3 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=True, - mode='downscale_in_infer') - res4 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=False, - mode='upscale_in_train') - res5 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=False, - mode='downscale_in_infer') - res6 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=True, - mode='upscale_in_train') - res7 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=True, - mode='downscale_in_infer') - res8 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=False, - mode='upscale_in_train') - res9 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=False, - mode='downscale_in_infer') - res10 = paddle.nn.functional.dropout(x=input, p=1., training=True) - res11 = paddle.fluid.layers.dropout(x=input, dropout_prob=0.) - res12 = paddle.nn.functional.dropout(x=input, - p=0., - axis=(0, 1), - training=False, - mode='upscale_in_train') - - res13 = paddle.nn.functional.dropout(x=input, - p=0.7, - axis=1, - training=True, - mode='upscale_in_train') + res1 = paddle.nn.functional.dropout(x=input, p=0.0, training=False) + res2 = paddle.nn.functional.dropout( + x=input, p=0.0, axis=0, training=True, mode='upscale_in_train' + ) + res3 = paddle.nn.functional.dropout( + x=input, p=0.0, axis=0, training=True, mode='downscale_in_infer' + ) + res4 = paddle.nn.functional.dropout( + x=input, p=0.0, axis=0, training=False, mode='upscale_in_train' + ) + res5 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=0, + training=False, + mode='downscale_in_infer', + ) + res6 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=True, + mode='upscale_in_train', + ) + res7 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=True, + mode='downscale_in_infer', + ) + res8 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=False, + mode='upscale_in_train', + ) + res9 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=False, + mode='downscale_in_infer', + ) + res10 = paddle.nn.functional.dropout(x=input, p=1.0, training=True) + res11 = paddle.fluid.layers.dropout(x=input, dropout_prob=0.0) + res12 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=(0, 1), + training=False, + mode='upscale_in_train', + ) + + res13 = paddle.nn.functional.dropout( + x=input, p=0.7, axis=1, training=True, mode='upscale_in_train' + ) in_np = np.ones([40, 40]).astype("float32") res_np = in_np @@ -441,21 +432,36 @@ def check_static_result(self, place): exe = fluid.Executor(place) res_list = [ - res1, res2, res3, res4, res5, res6, res7, res8, res9, res11, - res12 + res1, + res2, + res3, + res4, + res5, + res6, + res7, + res8, + res9, + res11, + res12, ] for res in res_list: - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res], + ) np.testing.assert_allclose(fetches[0], res_np, rtol=1e-05) - fetches2 = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res10]) + fetches2 = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res10], + ) np.testing.assert_allclose(fetches2[0], res_np2, rtol=1e-05) - fetches3 = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res13]) + fetches3 = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res13], + ) def test_static(self): for place in self.places: @@ -469,68 +475,99 @@ def test_dygraph(self): res_np2 = np.zeros_like(in_np) input = fluid.dygraph.to_variable(in_np) - res1 = paddle.nn.functional.dropout(x=input, - p=0., - training=False) - res2 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=True, - mode='upscale_in_train') - res3 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=True, - mode='downscale_in_infer') - res4 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=False, - mode='upscale_in_train') - res5 = paddle.nn.functional.dropout(x=input, - p=0., - axis=0, - training=False, - mode='downscale_in_infer') - res6 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=True, - mode='upscale_in_train') - res7 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=True, - mode='downscale_in_infer') - res8 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=False, - mode='upscale_in_train') - res9 = paddle.nn.functional.dropout(x=input, - p=0., - axis=[0, 1], - training=False, - mode='downscale_in_infer') - res10 = paddle.nn.functional.dropout(x=input, - p=1., - training=True) - dropout = paddle.fluid.dygraph.Dropout(p=0, ) + res1 = paddle.nn.functional.dropout( + x=input, p=0.0, training=False + ) + res2 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=0, + training=True, + mode='upscale_in_train', + ) + res3 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=0, + training=True, + mode='downscale_in_infer', + ) + res4 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=0, + training=False, + mode='upscale_in_train', + ) + res5 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=0, + training=False, + mode='downscale_in_infer', + ) + res6 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=True, + mode='upscale_in_train', + ) + res7 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=True, + mode='downscale_in_infer', + ) + res8 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=False, + mode='upscale_in_train', + ) + res9 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=[0, 1], + training=False, + mode='downscale_in_infer', + ) + res10 = paddle.nn.functional.dropout( + x=input, p=1.0, training=True + ) + dropout = paddle.fluid.dygraph.Dropout( + p=0, + ) res11 = dropout(input) - res12 = paddle.nn.functional.dropout(x=input, - p=0., - axis=(0, 1), - training=False, - mode='upscale_in_train') - res13 = paddle.nn.functional.dropout(x=input, - p=0.5, - axis=1, - training=True, - mode='upscale_in_train') + res12 = paddle.nn.functional.dropout( + x=input, + p=0.0, + axis=(0, 1), + training=False, + mode='upscale_in_train', + ) + res13 = paddle.nn.functional.dropout( + x=input, + p=0.5, + axis=1, + training=True, + mode='upscale_in_train', + ) res_list = [ - res1, res2, res3, res4, res5, res6, res7, res8, res9, res11, - res12 + res1, + res2, + res3, + res4, + res5, + res6, + res7, + res8, + res9, + res11, + res12, ] for res in res_list: np.testing.assert_allclose(res.numpy(), res_np, rtol=1e-05) @@ -538,22 +575,23 @@ def test_dygraph(self): class TestDropoutFAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of dropout must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) paddle.nn.functional.dropout(x1, p=0.5) self.assertRaises(TypeError, test_Variable) def test_Variable2(): # the input of dropout must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) paddle.nn.functional.dropout(x1, p=0.5, axis=0) self.assertRaises(TypeError, test_Variable2) @@ -617,7 +655,6 @@ def test_axis_len(): class TestDropoutCAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -630,16 +667,15 @@ def test_dygraph(self): input_np = np.random.random([40, 40]).astype("float32") result_np = input_np input = fluid.dygraph.to_variable(input_np) - m = paddle.nn.Dropout(p=0.) + m = paddle.nn.Dropout(p=0.0) m.eval() result = m(input) - np.testing.assert_allclose(result.numpy(), - result_np, - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), result_np, rtol=1e-05 + ) class TestDropout2DFAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -648,17 +684,15 @@ def setUp(self): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.data(name="input", - shape=[2, 3, 4, 5], - dtype="float32") - res1 = paddle.nn.functional.dropout2d(x=input, - p=0., - training=False, - data_format='NCHW') - res2 = paddle.nn.functional.dropout2d(x=input, - p=0., - training=False, - data_format='NHWC') + input = fluid.data( + name="input", shape=[2, 3, 4, 5], dtype="float32" + ) + res1 = paddle.nn.functional.dropout2d( + x=input, p=0.0, training=False, data_format='NCHW' + ) + res2 = paddle.nn.functional.dropout2d( + x=input, p=0.0, training=False, data_format='NHWC' + ) in_np = np.random.random([2, 3, 4, 5]).astype("float32") res_np = in_np @@ -666,9 +700,11 @@ def check_static_result(self, place): exe = fluid.Executor(place) res_list = [res1, res2] for res in res_list: - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res], + ) np.testing.assert_allclose(fetches[0], res_np, rtol=1e-05) def test_static(self): @@ -682,14 +718,12 @@ def test_dygraph(self): res_np = in_np input = fluid.dygraph.to_variable(in_np) - res1 = paddle.nn.functional.dropout2d(x=input, - p=0., - training=False, - data_format='NCHW') - res2 = paddle.nn.functional.dropout2d(x=input, - p=0., - training=False, - data_format='NHWC') + res1 = paddle.nn.functional.dropout2d( + x=input, p=0.0, training=False, data_format='NCHW' + ) + res2 = paddle.nn.functional.dropout2d( + x=input, p=0.0, training=False, data_format='NHWC' + ) res_list = [res1, res2] for res in res_list: @@ -697,7 +731,6 @@ def test_dygraph(self): class TestDropout2DFAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -717,7 +750,6 @@ def test_dataformat(): class TestDropout2DCAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -730,16 +762,15 @@ def test_dygraph(self): input_np = np.random.random([2, 3, 4, 5]).astype("float32") result_np = input_np input = fluid.dygraph.to_variable(input_np) - m = paddle.nn.Dropout2D(p=0.) + m = paddle.nn.Dropout2D(p=0.0) m.eval() result = m(input) - np.testing.assert_allclose(result.numpy(), - result_np, - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), result_np, rtol=1e-05 + ) class TestDropout3DFAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -748,17 +779,15 @@ def setUp(self): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.data(name="input", - shape=[2, 3, 4, 5, 6], - dtype="float32") - res1 = paddle.nn.functional.dropout3d(x=input, - p=0., - training=False, - data_format='NCDHW') - res2 = paddle.nn.functional.dropout3d(x=input, - p=0., - training=False, - data_format='NDHWC') + input = fluid.data( + name="input", shape=[2, 3, 4, 5, 6], dtype="float32" + ) + res1 = paddle.nn.functional.dropout3d( + x=input, p=0.0, training=False, data_format='NCDHW' + ) + res2 = paddle.nn.functional.dropout3d( + x=input, p=0.0, training=False, data_format='NDHWC' + ) in_np = np.random.random([2, 3, 4, 5, 6]).astype("float32") res_np = in_np @@ -766,9 +795,11 @@ def check_static_result(self, place): exe = fluid.Executor(place) res_list = [res1, res2] for res in res_list: - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res], + ) np.testing.assert_allclose(fetches[0], res_np, rtol=1e-05) def test_static(self): @@ -782,14 +813,12 @@ def test_dygraph(self): res_np = in_np input = fluid.dygraph.to_variable(in_np) - res1 = paddle.nn.functional.dropout3d(x=input, - p=0., - training=False, - data_format='NCDHW') - res2 = paddle.nn.functional.dropout3d(x=input, - p=0., - training=False, - data_format='NDHWC') + res1 = paddle.nn.functional.dropout3d( + x=input, p=0.0, training=False, data_format='NCDHW' + ) + res2 = paddle.nn.functional.dropout3d( + x=input, p=0.0, training=False, data_format='NDHWC' + ) res_list = [res1, res2] for res in res_list: @@ -797,7 +826,6 @@ def test_dygraph(self): class TestDropout3DFAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -817,7 +845,6 @@ def test_dataformat(): class TestDropout3DCAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -830,16 +857,15 @@ def test_dygraph(self): input_np = np.random.random([2, 3, 4, 5, 6]).astype("float32") result_np = input_np input = fluid.dygraph.to_variable(input_np) - m = paddle.nn.Dropout3D(p=0.) + m = paddle.nn.Dropout3D(p=0.0) m.eval() result = m(input) - np.testing.assert_allclose(result.numpy(), - result_np, - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), result_np, rtol=1e-05 + ) class TestAlphaDropoutFAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -849,11 +875,11 @@ def setUp(self): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): input = fluid.data(name="input", shape=[40, 40], dtype="float32") - res1 = paddle.nn.functional.alpha_dropout(x=input, p=0.) - res2 = paddle.nn.functional.alpha_dropout(x=input, - p=0., - training=False) - res3 = paddle.nn.functional.alpha_dropout(x=input, p=1.) + res1 = paddle.nn.functional.alpha_dropout(x=input, p=0.0) + res2 = paddle.nn.functional.alpha_dropout( + x=input, p=0.0, training=False + ) + res3 = paddle.nn.functional.alpha_dropout(x=input, p=1.0) in_np = np.random.random([40, 40]).astype("float32") res_np = in_np @@ -862,13 +888,17 @@ def check_static_result(self, place): exe = fluid.Executor(place) res_list = [res1, res2] for res in res_list: - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res], + ) np.testing.assert_allclose(fetches[0], res_np, rtol=1e-05) - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res3]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res3], + ) np.testing.assert_allclose(fetches[0], res_np3, rtol=1e-05) def test_static(self): @@ -883,11 +913,11 @@ def test_dygraph(self): res_np3 = np.zeros_like(in_np) input = fluid.dygraph.to_variable(in_np) - res1 = paddle.nn.functional.alpha_dropout(x=input, p=0.) - res2 = paddle.nn.functional.alpha_dropout(x=input, - p=0., - training=False) - res3 = paddle.nn.functional.alpha_dropout(x=input, p=1.) + res1 = paddle.nn.functional.alpha_dropout(x=input, p=0.0) + res2 = paddle.nn.functional.alpha_dropout( + x=input, p=0.0, training=False + ) + res3 = paddle.nn.functional.alpha_dropout(x=input, p=1.0) res_list = [res1, res2] for res in res_list: @@ -896,14 +926,14 @@ def test_dygraph(self): class TestAlphaDropoutFAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of dropout must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) paddle.nn.functional.alpha_dropout(x1, p=0.5) self.assertRaises(TypeError, test_Variable) @@ -931,7 +961,6 @@ def test_pvalue(): class TestAlphaDropoutCAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -944,16 +973,15 @@ def test_dygraph(self): input_np = np.random.random([40, 40]).astype("float32") result_np = input_np input = fluid.dygraph.to_variable(input_np) - m = paddle.nn.AlphaDropout(p=0.) + m = paddle.nn.AlphaDropout(p=0.0) m.eval() result = m(input) - np.testing.assert_allclose(result.numpy(), - result_np, - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), result_np, rtol=1e-05 + ) class TestDropoutWithDeterminateSeedGenerator(unittest.TestCase): - def setUp(self): paddle.framework.random.set_random_seed_generator('seed0', 123) paddle.framework.random.set_random_seed_generator('seed1', 123) @@ -964,19 +992,26 @@ def setUp(self): self.places.append(paddle.CUDAPlace(0)) def check_static_result(self, place): - from paddle.distributed.fleet.meta_parallel.parallel_layers.random import dropout + from paddle.distributed.fleet.meta_parallel.parallel_layers.random import ( + dropout, + ) + with static.program_guard(static.Program(), static.Program()): input = static.data(name="input", shape=[40, 40], dtype="float32") - res1 = dropout(input, - p=0.3, - training=True, - mode='upscale_in_train', - rng_name='seed0') - res2 = dropout(input, - p=0.3, - training=True, - mode='upscale_in_train', - rng_name='seed1') + res1 = dropout( + input, + p=0.3, + training=True, + mode='upscale_in_train', + rng_name='seed0', + ) + res2 = dropout( + input, + p=0.3, + training=True, + mode='upscale_in_train', + rng_name='seed1', + ) res3 = dropout(input, p=0.3) in_np = np.random.random([40, 40]).astype("float32") @@ -984,9 +1019,11 @@ def check_static_result(self, place): exe = static.Executor(place) res_list = [res1, res2] for i in range(2): - out1, out2 = exe.run(static.default_main_program(), - feed={"input": in_np}, - fetch_list=res_list) + out1, out2 = exe.run( + static.default_main_program(), + feed={"input": in_np}, + fetch_list=res_list, + ) np.testing.assert_allclose(out1, out2, rtol=1e-05) def test_static(self): @@ -995,7 +1032,6 @@ def test_static(self): class TestDropoutBackward(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -1020,7 +1056,8 @@ def test_backward_downscale_in_infer(self): np.testing.assert_array_equal( input.gradient(), - self.cal_grad_downscale_in_infer(mask.numpy())) + self.cal_grad_downscale_in_infer(mask.numpy()), + ) def test_backward_downscale_in_infer_eager(self): for place in self.places: @@ -1028,12 +1065,14 @@ def test_backward_downscale_in_infer_eager(self): with _test_eager_guard(): input = paddle.uniform([40, 40], dtype="float32") input.stop_gradient = False - out, mask = _C_ops.dropout(input, None, 0.5, False, - "downgrade_in_infer", 0, False) + out, mask = _C_ops.dropout( + input, None, 0.5, False, "downgrade_in_infer", 0, False + ) out.backward() np.testing.assert_array_equal( input.gradient(), - self.cal_grad_downscale_in_infer(mask.numpy())) + self.cal_grad_downscale_in_infer(mask.numpy()), + ) def test_backward_upscale_train(self): _enable_legacy_dygraph() @@ -1043,15 +1082,20 @@ def test_backward_upscale_train(self): prob = 0.5 input = paddle.uniform([40, 40], dtype="float32") input.stop_gradient = False - out, mask = core.ops.dropout(input, 'dropout_prob', prob, - "dropout_implementation", - "upscale_in_train") + out, mask = core.ops.dropout( + input, + 'dropout_prob', + prob, + "dropout_implementation", + "upscale_in_train", + ) out.backward() - np.testing.assert_allclose(input.gradient(), - self.cal_grad_upscale_train( - mask.numpy(), prob), - rtol=1e-05) + np.testing.assert_allclose( + input.gradient(), + self.cal_grad_upscale_train(mask.numpy(), prob), + rtol=1e-05, + ) def test_backward_upscale_train_eager(self): for place in self.places: @@ -1060,14 +1104,16 @@ def test_backward_upscale_train_eager(self): prob = 0.5 input = paddle.uniform([40, 40], dtype="float32") input.stop_gradient = False - out, mask = _C_ops.dropout(input, None, 0.5, False, - "upscale_in_train", 0, False) + out, mask = _C_ops.dropout( + input, None, 0.5, False, "upscale_in_train", 0, False + ) out.backward() - np.testing.assert_allclose(input.gradient(), - self.cal_grad_upscale_train( - mask.numpy(), prob), - rtol=1e-05) + np.testing.assert_allclose( + input.gradient(), + self.cal_grad_upscale_train(mask.numpy(), prob), + rtol=1e-05, + ) def test_backward_upscale_train_2(self): _enable_legacy_dygraph() @@ -1077,15 +1123,20 @@ def test_backward_upscale_train_2(self): prob = 0.3 input = paddle.uniform([40, 40], dtype="float32") input.stop_gradient = False - out, mask = core.ops.dropout(input, 'dropout_prob', prob, - "dropout_implementation", - "upscale_in_train") + out, mask = core.ops.dropout( + input, + 'dropout_prob', + prob, + "dropout_implementation", + "upscale_in_train", + ) out.backward() - np.testing.assert_allclose(input.gradient(), - self.cal_grad_upscale_train( - mask.numpy(), prob), - rtol=1e-05) + np.testing.assert_allclose( + input.gradient(), + self.cal_grad_upscale_train(mask.numpy(), prob), + rtol=1e-05, + ) def test_backward_upscale_train_2_eager(self): for place in self.places: @@ -1095,24 +1146,28 @@ def test_backward_upscale_train_2_eager(self): prob = 0.3 input = paddle.uniform([40, 40], dtype="float32") input.stop_gradient = False - out, mask = _C_ops.dropout(input, None, 0.3, False, - "upscale_in_train", 0, False) + out, mask = _C_ops.dropout( + input, None, 0.3, False, "upscale_in_train", 0, False + ) out.backward() - np.testing.assert_allclose(input.gradient(), - self.cal_grad_upscale_train( - mask.numpy(), prob), - rtol=1e-05) + np.testing.assert_allclose( + input.gradient(), + self.cal_grad_upscale_train(mask.numpy(), prob), + rtol=1e-05, + ) class TestDropOutWithProbTensor(unittest.TestCase): - def setUp(self): self.init_info() self.input = np.random.random(self.shape).astype("float32") - self.place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) def init_info(self): self.shape = [10, 10] @@ -1151,21 +1206,18 @@ def test_p_tensor(self): class TestDropOut2DWithProbTensor(TestDropOutWithProbTensor): - def init_info(self): self.shape = [2, 3, 10, 10] self.api = paddle.nn.functional.dropout2d class TestDropOut3DWithProbTensor(TestDropOutWithProbTensor): - def init_info(self): self.shape = [2, 3, 8, 8, 8] self.api = paddle.nn.functional.dropout3d class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -1188,8 +1240,16 @@ def test_fixed_random_number(self): self.assertEqual(np.sum(index2), 12872777397) self.assertEqual(np.sum(out), 16778744.0) expect = [ - 0.6914956, 0.5294584, 0.19032137, 0.6996228, 0.3338527, 0.8442094, - 0.96965003, 1.1726775, 0., 0.28037727 + 0.6914956, + 0.5294584, + 0.19032137, + 0.6996228, + 0.3338527, + 0.8442094, + 0.96965003, + 1.1726775, + 0.0, + 0.28037727, ] np.testing.assert_allclose(out[10, 100, 500:510], expect, rtol=1e-05) @@ -1201,8 +1261,16 @@ def test_fixed_random_number(self): self.assertEqual(np.sum(index2), 8582219962) self.assertEqual(np.sum(out), 16778396.563660286) expect = [ - 1.28587354, 0.15563703, 0., 0.28799703, 0., 0., 0., 0.54964, - 0.51355682, 0.33818988 + 1.28587354, + 0.15563703, + 0.0, + 0.28799703, + 0.0, + 0.0, + 0.0, + 0.54964, + 0.51355682, + 0.33818988, ] np.testing.assert_allclose(out[20, 100, 500:510], expect, rtol=1e-05) @@ -1212,7 +1280,7 @@ def test_fixed_random_number(self): self.assertEqual(np.sum(index0), 130086900) self.assertEqual(np.sum(index1), 4291190105) self.assertEqual(np.sum(index2), 4292243807) - expect = [0., 0., 0., 0., 0., 0., 0., 0., 4., 4.] + expect = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 4.0, 4.0] np.testing.assert_allclose(out[0, 100, 500:510], expect, rtol=1e-05) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py b/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py index 30109d430be9b..0d435691560d4 100644 --- a/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py +++ b/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py @@ -22,46 +22,51 @@ class SimpleImgConvPool(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - pool_size, - pool_stride, - pool_padding=0, - pool_type='max', - global_pooling=False, - conv_stride=1, - conv_padding=0, - conv_dilation=1, - conv_groups=1, - act=None, - use_cudnn=False, - dtype='float32', - param_attr=None, - bias_attr=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + pool_size, + pool_stride, + pool_padding=0, + pool_type='max', + global_pooling=False, + conv_stride=1, + conv_padding=0, + conv_dilation=1, + conv_groups=1, + act=None, + use_cudnn=False, + dtype='float32', + param_attr=None, + bias_attr=None, + ): super(SimpleImgConvPool, self).__init__() - self._conv2d = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=conv_stride, - padding=conv_padding, - dilation=conv_dilation, - groups=conv_groups, - param_attr=param_attr, - bias_attr=bias_attr, - use_cudnn=use_cudnn, - dtype=dtype, - act=act) - - self._pool2d = Pool2D(pool_size=pool_size, - pool_type=pool_type, - pool_stride=pool_stride, - pool_padding=pool_padding, - global_pooling=global_pooling, - use_cudnn=use_cudnn) + self._conv2d = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=conv_stride, + padding=conv_padding, + dilation=conv_dilation, + groups=conv_groups, + param_attr=param_attr, + bias_attr=bias_attr, + use_cudnn=use_cudnn, + dtype=dtype, + act=act, + ) + + self._pool2d = Pool2D( + pool_size=pool_size, + pool_type=pool_type, + pool_stride=pool_stride, + pool_padding=pool_padding, + global_pooling=global_pooling, + use_cudnn=use_cudnn, + ) def forward(self, inputs): x = self._conv2d(inputs) @@ -70,39 +75,45 @@ def forward(self, inputs): class MNIST(fluid.dygraph.Layer): - def __init__(self, dtype="float32"): super(MNIST, self).__init__() - self._simple_img_conv_pool_1 = SimpleImgConvPool(num_channels=3, - num_filters=20, - filter_size=5, - pool_size=2, - pool_stride=2, - act="relu", - dtype=dtype, - use_cudnn=True) - - self._simple_img_conv_pool_2 = SimpleImgConvPool(num_channels=20, - num_filters=50, - filter_size=5, - pool_size=2, - pool_stride=2, - act="relu", - dtype=dtype, - use_cudnn=True) + self._simple_img_conv_pool_1 = SimpleImgConvPool( + num_channels=3, + num_filters=20, + filter_size=5, + pool_size=2, + pool_stride=2, + act="relu", + dtype=dtype, + use_cudnn=True, + ) + + self._simple_img_conv_pool_2 = SimpleImgConvPool( + num_channels=20, + num_filters=50, + filter_size=5, + pool_size=2, + pool_stride=2, + act="relu", + dtype=dtype, + use_cudnn=True, + ) self.pool_2_shape = 50 * 53 * 53 SIZE = 10 - scale = (2.0 / (self.pool_2_shape**2 * SIZE))**0.5 + scale = (2.0 / (self.pool_2_shape**2 * SIZE)) ** 0.5 self._linear = Linear( self.pool_2_shape, 10, param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.NormalInitializer(loc=0.0, - scale=scale)), + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=scale + ) + ), act="softmax", - dtype=dtype) + dtype=dtype, + ) def forward(self, inputs, label): x = self._simple_img_conv_pool_1(inputs) @@ -115,7 +126,6 @@ def forward(self, inputs, label): class TestMnist(unittest.TestCase): - def func_mnist_fp16(self): if not fluid.is_compiled_with_cuda(): return diff --git a/python/paddle/fluid/tests/unittests/test_dygraph_mode_of_unittest.py b/python/paddle/fluid/tests/unittests/test_dygraph_mode_of_unittest.py index ce8bb122f637f..8da813c00ef44 100644 --- a/python/paddle/fluid/tests/unittests/test_dygraph_mode_of_unittest.py +++ b/python/paddle/fluid/tests/unittests/test_dygraph_mode_of_unittest.py @@ -17,11 +17,10 @@ class TestDygraphModeOfUnittest(unittest.TestCase): - def test_dygraph_mode(self): self.assertTrue( paddle.in_dynamic_mode(), - 'Default Mode of Unittest should be dygraph mode, but get static mode.' + 'Default Mode of Unittest should be dygraph mode, but get static mode.', ) diff --git a/python/paddle/fluid/tests/unittests/test_dygraph_multi_forward.py b/python/paddle/fluid/tests/unittests/test_dygraph_multi_forward.py index 559424c7fb440..4486da7eb3039 100644 --- a/python/paddle/fluid/tests/unittests/test_dygraph_multi_forward.py +++ b/python/paddle/fluid/tests/unittests/test_dygraph_multi_forward.py @@ -27,43 +27,48 @@ class SimpleImgConvPool(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - pool_size, - pool_stride, - pool_padding=0, - pool_type='max', - global_pooling=False, - conv_stride=1, - conv_padding=0, - conv_dilation=1, - conv_groups=1, - act=None, - use_cudnn=False, - param_attr=None, - bias_attr=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + pool_size, + pool_stride, + pool_padding=0, + pool_type='max', + global_pooling=False, + conv_stride=1, + conv_padding=0, + conv_dilation=1, + conv_groups=1, + act=None, + use_cudnn=False, + param_attr=None, + bias_attr=None, + ): super(SimpleImgConvPool, self).__init__() - self._conv2d = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=conv_stride, - padding=conv_padding, - dilation=conv_dilation, - groups=conv_groups, - param_attr=None, - bias_attr=None, - use_cudnn=use_cudnn) - - self._pool2d = Pool2D(pool_size=pool_size, - pool_type=pool_type, - pool_stride=pool_stride, - pool_padding=pool_padding, - global_pooling=global_pooling, - use_cudnn=use_cudnn) + self._conv2d = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=conv_stride, + padding=conv_padding, + dilation=conv_dilation, + groups=conv_groups, + param_attr=None, + bias_attr=None, + use_cudnn=use_cudnn, + ) + + self._pool2d = Pool2D( + pool_size=pool_size, + pool_type=pool_type, + pool_stride=pool_stride, + pool_padding=pool_padding, + global_pooling=global_pooling, + use_cudnn=use_cudnn, + ) def forward(self, inputs): x = self._conv2d(inputs) @@ -72,33 +77,30 @@ def forward(self, inputs): class MNIST(fluid.dygraph.Layer): - def __init__(self): super(MNIST, self).__init__() - self._simple_img_conv_pool_1 = SimpleImgConvPool(1, - 20, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_1 = SimpleImgConvPool( + 1, 20, 5, 2, 2, act="relu" + ) - self._simple_img_conv_pool_2 = SimpleImgConvPool(20, - 50, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_2 = SimpleImgConvPool( + 20, 50, 5, 2, 2, act="relu" + ) self.pool_2_shape = 50 * 4 * 4 - SIZE = 100 #10 - scale = (2.0 / (self.pool_2_shape**2 * SIZE))**0.5 - self._fc = Linear(self.pool_2_shape, - SIZE, - param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.NormalInitializer( - loc=0.0, scale=scale)), - act="softmax") + SIZE = 100 # 10 + scale = (2.0 / (self.pool_2_shape**2 * SIZE)) ** 0.5 + self._fc = Linear( + self.pool_2_shape, + SIZE, + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=scale + ) + ), + act="softmax", + ) def forward(self, inputs): x = self._simple_img_conv_pool_1(inputs) @@ -109,7 +111,6 @@ def forward(self, inputs): class TestDygraphMultiForward(unittest.TestCase): - def test_mnist_forward_float32(self): epoch_num = 1 @@ -117,21 +118,25 @@ def test_mnist_forward_float32(self): paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) mnist = MNIST() - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=mnist.parameters()) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=128, - drop_last=True) + sgd = SGDOptimizer( + learning_rate=1e-3, parameter_list=mnist.parameters() + ) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=128, drop_last=True + ) dy_param_init_value = {} mnist.eval() for epoch in range(epoch_num): for batch_id, data in enumerate(train_reader()): - dy_x_data = np.array([ - x[0].reshape(1, 28, 28) for x in data - ]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(128, 1) + dy_x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(128, 1) + ) img = to_variable(dy_x_data) label = to_variable(y_data) @@ -150,18 +155,21 @@ def test_mnist_forward_float32(self): with new_program_scope(): paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) mnist = MNIST() sgd = SGDOptimizer(learning_rate=1e-3) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=128, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=128, drop_last=True + ) - img = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype='float32') + img = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') cost = mnist(img) loss = fluid.layers.cross_entropy(cost, label) @@ -173,38 +181,42 @@ def test_mnist_forward_float32(self): for param in mnist.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] for epoch in range(epoch_num): for batch_id, data in enumerate(train_reader()): - static_x_data = np.array([ - x[0].reshape(1, 28, 28) for x in data - ]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape([128, 1]) + static_x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([128, 1]) + ) fetch_list = [avg_loss.name] - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) static_out = out[0] - np.testing.assert_allclose(dy_x_data.all(), - static_x_data.all(), - rtol=1e-05) + np.testing.assert_allclose( + dy_x_data.all(), static_x_data.all(), rtol=1e-05 + ) for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value[key], rtol=1e-05 + ) np.testing.assert_allclose(static_out, dy_out, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_dygraph_spectral_norm.py b/python/paddle/fluid/tests/unittests/test_dygraph_spectral_norm.py index 9c3e6e1529e43..b865a1b179933 100644 --- a/python/paddle/fluid/tests/unittests/test_dygraph_spectral_norm.py +++ b/python/paddle/fluid/tests/unittests/test_dygraph_spectral_norm.py @@ -21,14 +21,13 @@ class TestDygraphSpectralNorm(unittest.TestCase): - def setUp(self): self.init_test_case() self.set_data() def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 12, 12]], ) + self.data_desc = (['x', [2, 12, 12]],) self.n_power_iterations = 1 self.eps = 1e-12 self.dim = None @@ -38,8 +37,9 @@ def set_data(self): for desc in self.data_desc: data_name = desc[0] data_shape = desc[1] - data_value = np.random.random(size=[self.batch_size] + - data_shape).astype('float32') + data_value = np.random.random( + size=[self.batch_size] + data_shape + ).astype('float32') self.data[data_name] = data_value def spectral_normalize(self, weight, u, v, dim, power_iters, eps): @@ -68,18 +68,27 @@ def test_check_output(self): linear = paddle.nn.Conv2D(2, 1, 3) before_weight = linear.weight.numpy().copy() if self.dim == None: - if isinstance(linear, (nn.Conv1DTranspose, nn.Conv2DTranspose, - nn.Conv3DTranspose, nn.Linear)): + if isinstance( + linear, + ( + nn.Conv1DTranspose, + nn.Conv2DTranspose, + nn.Conv3DTranspose, + nn.Linear, + ), + ): self.dim = 1 else: self.dim = 0 else: self.dim = (self.dim + len(before_weight)) % len(before_weight) - sn = spectral_norm(linear, - n_power_iterations=self.n_power_iterations, - eps=self.eps, - dim=self.dim) + sn = spectral_norm( + linear, + n_power_iterations=self.n_power_iterations, + eps=self.eps, + dim=self.dim, + ) u = sn.weight_u.numpy().copy() v = sn.weight_v.numpy().copy() outputs = [] @@ -88,52 +97,47 @@ def test_check_output(self): outputs.append(output.numpy()) self.actual_outputs = linear.weight.numpy() - expect_output = self.spectral_normalize(before_weight, u, v, self.dim, - self.n_power_iterations, - self.eps) + expect_output = self.spectral_normalize( + before_weight, u, v, self.dim, self.n_power_iterations, self.eps + ) for expect, actual in zip(expect_output, self.actual_outputs): - np.testing.assert_allclose(np.array(actual), - np.array(expect), - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + np.array(actual), np.array(expect), rtol=1e-05, atol=0.001 + ) class TestDygraphWeightNormCase(TestDygraphSpectralNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.n_power_iterations = 1 self.eps = 1e-12 self.dim = None class TestDygraphWeightNormWithIterations(TestDygraphSpectralNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.n_power_iterations = 2 self.eps = 1e-12 self.dim = None class TestDygraphWeightNormWithDim(TestDygraphSpectralNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.n_power_iterations = 1 self.eps = 1e-12 self.dim = 1 class TestDygraphWeightNormWithEps(TestDygraphSpectralNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.n_power_iterations = 1 self.eps = 1e-10 self.dim = None diff --git a/python/paddle/fluid/tests/unittests/test_dygraph_weight_norm.py b/python/paddle/fluid/tests/unittests/test_dygraph_weight_norm.py index 270a1afe04b4a..ebae140f88c25 100644 --- a/python/paddle/fluid/tests/unittests/test_dygraph_weight_norm.py +++ b/python/paddle/fluid/tests/unittests/test_dygraph_weight_norm.py @@ -22,14 +22,13 @@ class TestDygraphWeightNorm(unittest.TestCase): - def setUp(self): self.init_test_case() self.set_data() def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.dim = None def set_data(self): @@ -37,8 +36,9 @@ def set_data(self): for desc in self.data_desc: data_name = desc[0] data_shape = desc[1] - data_value = np.random.random(size=[self.batch_size] + - data_shape).astype('float32') + data_value = np.random.random( + size=[self.batch_size] + data_shape + ).astype('float32') self.data[data_name] = data_value def norm_except_dim(self, w, dim=None): @@ -88,14 +88,22 @@ def weight_normalize(self, w, dim=None): perm[dim] = 0 p_transposed = np.transpose(v, perm) transposed_shape = p_transposed.shape - transposed_shape_numel = reduce(lambda x, y: x * y, - transposed_shape) + transposed_shape_numel = reduce( + lambda x, y: x * y, transposed_shape + ) p_matrix = np.reshape( - p_transposed, (p_transposed.shape[0], - transposed_shape_numel // p_transposed.shape[0])) - v_norm = v / np.expand_dims(np.expand_dims( - np.linalg.norm(p_matrix, axis=1, keepdims=True), axis=0), - axis=(ndims - 1)) + p_transposed, + ( + p_transposed.shape[0], + transposed_shape_numel // p_transposed.shape[0], + ), + ) + v_norm = v / np.expand_dims( + np.expand_dims( + np.linalg.norm(p_matrix, axis=1, keepdims=True), axis=0 + ), + axis=(ndims - 1), + ) v_norm = np.reshape(v_norm, transposed_shape) v_norm = np.transpose(v_norm, perm) g = np.squeeze(g, axis=1) @@ -103,9 +111,10 @@ def weight_normalize(self, w, dim=None): eaxis = 2 elif dim == 2: eaxis = 1 - g_mul = np.expand_dims(np.expand_dims(np.expand_dims(g, axis=0), - axis=eaxis), - axis=(ndims - 1)) + g_mul = np.expand_dims( + np.expand_dims(np.expand_dims(g, axis=0), axis=eaxis), + axis=(ndims - 1), + ) w = g_mul * v_norm return g, v @@ -129,52 +138,46 @@ def test_check_output(self): expect_output = self.weight_normalize(before_weight, self.dim) for expect, actual in zip(expect_output, self.actual_outputs): - np.testing.assert_allclose(np.array(actual), - expect, - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + np.array(actual), expect, rtol=1e-05, atol=0.001 + ) class TestDygraphWeightNormCase1(TestDygraphWeightNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.dim = 0 class TestDygraphWeightNormCase2(TestDygraphWeightNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.dim = 1 class TestDygraphWeightNormCase3(TestDygraphWeightNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.dim = 3 class TestDygraphWeightNormCase4(TestDygraphWeightNorm): - def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.dim = -3 class TestDygraphRemoveWeightNorm(unittest.TestCase): - def setUp(self): self.init_test_case() def init_test_case(self): self.batch_size = 3 - self.data_desc = (['x', [2, 3, 3]], ) + self.data_desc = (['x', [2, 3, 3]],) self.dim = None def test_check_output(self): @@ -184,10 +187,9 @@ def test_check_output(self): wn = weight_norm(linear, dim=self.dim) rwn = remove_weight_norm(linear) after_weight = linear.weight - np.testing.assert_allclose(before_weight.numpy(), - after_weight.numpy(), - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + before_weight.numpy(), after_weight.numpy(), rtol=1e-05, atol=0.001 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_dyn_rnn.py b/python/paddle/fluid/tests/unittests/test_dyn_rnn.py index de03749769d8d..e4f0a3a173a3f 100644 --- a/python/paddle/fluid/tests/unittests/test_dyn_rnn.py +++ b/python/paddle/fluid/tests/unittests/test_dyn_rnn.py @@ -29,20 +29,21 @@ class TestDynamicRNN(unittest.TestCase): - def setUp(self): self.word_dict_len = 5147 self.BATCH_SIZE = 2 reader = fake_imdb_reader(self.word_dict_len, self.BATCH_SIZE * 100) self.train_data = paddle.batch(reader, batch_size=self.BATCH_SIZE) - def _train(self, - main_program, - startup_program, - feed_list, - fetch_list, - is_nested=False, - max_iters=1): + def _train( + self, + main_program, + startup_program, + feed_list, + fetch_list, + is_nested=False, + max_iters=1, + ): place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup_program) @@ -50,10 +51,12 @@ def _train(self, data = next(self.train_data()) for iter_id in range(max_iters): - fetch_outs = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=fetch_list, - return_numpy=False) + fetch_outs = exe.run( + main_program, + feed=feeder.feed(data), + fetch_list=fetch_list, + return_numpy=False, + ) if len(fetch_list) == 3: rnn_in_seq = fetch_outs[0] rnn_out_seq = fetch_outs[1] @@ -65,9 +68,9 @@ def _train(self, loss_i = numpy.array(fetch_outs[2]) elif len(fetch_list) == 1: loss_i = numpy.array(fetch_outs[0]) - #print(loss_i) + # print(loss_i) - self.assertEqual((1, ), loss_i.shape) + self.assertEqual((1,), loss_i.shape) self.assertFalse(numpy.isnan(loss_i)) if iter_id == 0: loss_0 = loss_i @@ -81,13 +84,12 @@ def test_plain_while_op(self): startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): - sentence = fluid.layers.data(name='word', - shape=[1], - dtype='int64', - lod_level=1) - sent_emb = fluid.layers.embedding(input=sentence, - size=[self.word_dict_len, 32], - dtype='float32') + sentence = fluid.layers.data( + name='word', shape=[1], dtype='int64', lod_level=1 + ) + sent_emb = fluid.layers.embedding( + input=sentence, size=[self.word_dict_len, 32], dtype='float32' + ) rank_table = lod_rank_table(x=sent_emb) sent_emb_array = lod_tensor_to_array(x=sent_emb, table=rank_table) @@ -100,7 +102,8 @@ def test_plain_while_op(self): input=fluid.layers.array_read(array=sent_emb_array, i=i), value=0, shape=[-1, 100], - dtype='float32') + dtype='float32', + ) boot_mem.stop_gradient = False mem_array = fluid.layers.array_write(x=boot_mem, i=i) @@ -127,8 +130,9 @@ def test_plain_while_op(self): logits = fluid.layers.fc(input=last, size=1, act=None) label = fluid.layers.data(name='label', shape=[1], dtype='float32') - loss = fluid.layers.sigmoid_cross_entropy_with_logits(x=logits, - label=label) + loss = fluid.layers.sigmoid_cross_entropy_with_logits( + x=logits, label=label + ) loss = paddle.mean(loss) sgd = fluid.optimizer.SGD(1e-4) sgd.minimize(loss=loss) @@ -136,12 +140,14 @@ def test_plain_while_op(self): # Check for lod_level set in compile-time. self.assertEqual(sent_emb.lod_level, result_all_timesteps.lod_level) - self._train(main_program=main_program, - startup_program=startup_program, - feed_list=[sentence, label], - fetch_list=[sent_emb, result_all_timesteps, loss], - is_nested=False, - max_iters=1) + self._train( + main_program=main_program, + startup_program=startup_program, + feed_list=[sentence, label], + fetch_list=[sent_emb, result_all_timesteps, loss], + is_nested=False, + max_iters=1, + ) def test_train_dynamic_rnn(self): main_program = fluid.Program() @@ -149,13 +155,12 @@ def test_train_dynamic_rnn(self): main_program.random_seed = 10 startup_program.random_seed = 10 with fluid.program_guard(main_program, startup_program): - sentence = fluid.layers.data(name='word', - shape=[1], - dtype='int64', - lod_level=1) - sent_emb = fluid.layers.embedding(input=sentence, - size=[self.word_dict_len, 32], - dtype='float32') + sentence = fluid.layers.data( + name='word', shape=[1], dtype='int64', lod_level=1 + ) + sent_emb = fluid.layers.embedding( + input=sentence, size=[self.word_dict_len, 32], dtype='float32' + ) drnn = fluid.layers.DynamicRNN() with drnn.block(): @@ -170,8 +175,9 @@ def test_train_dynamic_rnn(self): logits = fluid.layers.fc(input=last, size=1, act=None) label = fluid.layers.data(name='label', shape=[1], dtype='float32') - loss = fluid.layers.sigmoid_cross_entropy_with_logits(x=logits, - label=label) + loss = fluid.layers.sigmoid_cross_entropy_with_logits( + x=logits, label=label + ) loss = paddle.mean(loss) sgd = fluid.optimizer.Adam(1e-3) sgd.minimize(loss=loss) @@ -179,12 +185,14 @@ def test_train_dynamic_rnn(self): # Check for lod_level set in compile-time. self.assertEqual(sent_emb.lod_level, drnn_result.lod_level) - self._train(main_program=main_program, - startup_program=startup_program, - feed_list=[sentence, label], - fetch_list=[sent_emb, drnn_result, loss], - is_nested=False, - max_iters=100) + self._train( + main_program=main_program, + startup_program=startup_program, + feed_list=[sentence, label], + fetch_list=[sent_emb, drnn_result, loss], + is_nested=False, + max_iters=100, + ) def _fake_reader(self): seq_len, label = [[2, 2]], [0, 1] @@ -205,30 +213,30 @@ def test_train_nested_dynamic_rnn(self): main_program.random_seed = 10 startup_program.random_seed = 10 with fluid.program_guard(main_program, startup_program): - sentence = fluid.layers.data(name='word', - shape=[1], - dtype='int64', - lod_level=2) - label = fluid.layers.data(name='label', - shape=[1], - dtype='float32', - lod_level=1) + sentence = fluid.layers.data( + name='word', shape=[1], dtype='int64', lod_level=2 + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='float32', lod_level=1 + ) drnn0 = fluid.layers.DynamicRNN() with drnn0.block(): in_0 = drnn0.step_input(sentence) assert in_0.lod_level == 1, "the lod level of in_ should be 1" - sentence_emb = fluid.layers.embedding(input=in_0, - size=[len(word_dict), 32], - dtype='float32') - out_0 = fluid.layers.fc(input=sentence_emb, - size=100, - act='tanh') + sentence_emb = fluid.layers.embedding( + input=in_0, size=[len(word_dict), 32], dtype='float32' + ) + out_0 = fluid.layers.fc( + input=sentence_emb, size=100, act='tanh' + ) drnn1 = fluid.layers.DynamicRNN() with drnn1.block(): in_1 = drnn1.step_input(out_0) - assert in_1.lod_level == 0, "the lod level of in_1 should be 0" + assert ( + in_1.lod_level == 0 + ), "the lod level of in_1 should be 0" out_1 = fluid.layers.fc(input=[in_1], size=100, act='tanh') drnn1.output(out_1) @@ -238,20 +246,23 @@ def test_train_nested_dynamic_rnn(self): last = drnn0() logits = fluid.layers.fc(input=last, size=1, act=None) - loss = fluid.layers.sigmoid_cross_entropy_with_logits(x=logits, - label=label) + loss = fluid.layers.sigmoid_cross_entropy_with_logits( + x=logits, label=label + ) loss = paddle.mean(loss) sgd = fluid.optimizer.SGD(1e-3) sgd.minimize(loss=loss) train_data_orig = self.train_data self.train_data = paddle.batch(self._fake_reader, batch_size=2) - self._train(main_program=main_program, - startup_program=startup_program, - feed_list=[sentence, label], - fetch_list=[loss], - is_nested=True, - max_iters=100) + self._train( + main_program=main_program, + startup_program=startup_program, + feed_list=[sentence, label], + fetch_list=[loss], + is_nested=True, + max_iters=100, + ) self.train_data = train_data_orig # this unit test is just used to the two layer nested dyn_rnn. @@ -264,14 +275,12 @@ def test_train_nested_dynamic_rnn2(self): main_program.random_seed = 10 startup_program.random_seed = 10 with fluid.program_guard(main_program, startup_program): - sentence = fluid.layers.data(name='word', - shape=[1], - dtype='int64', - lod_level=2) - label = fluid.layers.data(name='label', - shape=[1], - dtype='float32', - lod_level=1) + sentence = fluid.layers.data( + name='word', shape=[1], dtype='int64', lod_level=2 + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='float32', lod_level=1 + ) drnn0 = fluid.layers.DynamicRNN() with drnn0.block(): @@ -279,14 +288,19 @@ def test_train_nested_dynamic_rnn2(self): sentence_emb = fluid.layers.embedding( input=in_0, size=[len(word_dict), hidden_size], - dtype='float32') - input_forward_proj = fluid.layers.fc(input=sentence_emb, - size=hidden_size * 4, - act=None, - bias_attr=False) - forward, _ = fluid.layers.dynamic_lstm(input=input_forward_proj, - size=hidden_size * 4, - use_peepholes=False) + dtype='float32', + ) + input_forward_proj = fluid.layers.fc( + input=sentence_emb, + size=hidden_size * 4, + act=None, + bias_attr=False, + ) + forward, _ = fluid.layers.dynamic_lstm( + input=input_forward_proj, + size=hidden_size * 4, + use_peepholes=False, + ) drnn1 = fluid.layers.DynamicRNN() with drnn1.block(): @@ -299,33 +313,34 @@ def test_train_nested_dynamic_rnn2(self): last = drnn0() logits = fluid.layers.fc(input=last, size=1, act=None) - loss = fluid.layers.sigmoid_cross_entropy_with_logits(x=logits, - label=label) + loss = fluid.layers.sigmoid_cross_entropy_with_logits( + x=logits, label=label + ) loss = paddle.mean(loss) sgd = fluid.optimizer.SGD(1e-3) sgd.minimize(loss=loss) train_data_orig = self.train_data self.train_data = paddle.batch(self._fake_reader, batch_size=2) - self._train(main_program=main_program, - startup_program=startup_program, - feed_list=[sentence, label], - fetch_list=[loss], - is_nested=True, - max_iters=100) + self._train( + main_program=main_program, + startup_program=startup_program, + feed_list=[sentence, label], + fetch_list=[loss], + is_nested=True, + max_iters=100, + ) self.train_data = train_data_orig class TestDynamicRNNErrors(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): init = fluid.layers.zeros(shape=[1], dtype='float32') shape = 'shape' - sentence = fluid.data(name='sentence', - shape=[None, 32], - dtype='float32', - lod_level=1) + sentence = fluid.data( + name='sentence', shape=[None, 32], dtype='float32', lod_level=1 + ) # The type of Input(shape) in API(memory) must be list or tuple def input_shape_type_of_memory(): @@ -341,9 +356,9 @@ def outputs_type_of_output(): with drnn.block(): word = drnn.step_input(sentence) memory = drnn.memory(shape=[10], dtype='float32', value=0) - hidden = fluid.layers.fc(input=[word, memory], - size=10, - act='tanh') + hidden = fluid.layers.fc( + input=[word, memory], size=10, act='tanh' + ) out = numpy.ones(1).astype('float32') drnn.update_memory(ex_mem=memory, new_mem=hidden) drnn.output(hidden, out) diff --git a/python/paddle/fluid/tests/unittests/test_dynamic_rnn_stop_gradient.py b/python/paddle/fluid/tests/unittests/test_dynamic_rnn_stop_gradient.py index 0a186b0c8afba..3922f6a8c229d 100644 --- a/python/paddle/fluid/tests/unittests/test_dynamic_rnn_stop_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_dynamic_rnn_stop_gradient.py @@ -24,16 +24,15 @@ def build_and_run_program(place, batch_size, beam_size, stop_gradient=False): np.random.seed(2) x = layers.assign( - np.random.rand(batch_size, beam_size, 32).astype("float32")) + np.random.rand(batch_size, beam_size, 32).astype("float32") + ) indices = fluid.data(shape=[None, beam_size], dtype="int64", name="indices") - step_idx = layers.fill_constant(shape=[1], - dtype="int64", - value=0, - force_cpu=True) - max_len = layers.fill_constant(shape=[1], - dtype="int64", - value=10, - force_cpu=True) + step_idx = layers.fill_constant( + shape=[1], dtype="int64", value=0, force_cpu=True + ) + max_len = layers.fill_constant( + shape=[1], dtype="int64", value=10, force_cpu=True + ) cond = layers.less_than(x=step_idx, y=max_len) while_op = layers.While(cond) scores = layers.array_write(x, step_idx) @@ -44,7 +43,8 @@ def build_and_run_program(place, batch_size, beam_size, stop_gradient=False): bs.stop_gradient = stop_gradient batch_pos = layers.expand( layers.unsqueeze(layers.range(0, bs, 1, dtype=bs.dtype), [1]), - [1, beam_size]) + [1, beam_size], + ) topk_coordinates = layers.stack([batch_pos, indices], axis=2) topk_coordinates.stop_gradient = stop_gradient score = layers.gather_nd(x, topk_coordinates) @@ -58,17 +58,15 @@ def build_and_run_program(place, batch_size, beam_size, stop_gradient=False): opt = fluid.optimizer.Adam(0.01) opt.minimize(loss) exe = fluid.Executor(place) - data = np.random.random_integers(low=0, - high=beam_size - 1, - size=(batch_size, - beam_size)).astype("int64") - loss_val, = exe.run(feed={"indices": data}, fetch_list=[loss]) + data = np.random.random_integers( + low=0, high=beam_size - 1, size=(batch_size, beam_size) + ).astype("int64") + (loss_val,) = exe.run(feed={"indices": data}, fetch_list=[loss]) return loss_val class TestDynRNNStopGradient(unittest.TestCase): - def setUp(self): self.batch_size = 20 self.beam_size = 64 @@ -76,10 +74,12 @@ def setUp(self): def run_main(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): with fluid.scope_guard(fluid.Scope()): - value1 = build_and_run_program(place, self.batch_size, - self.beam_size, False) - value2 = build_and_run_program(place, self.batch_size, - self.beam_size, True) + value1 = build_and_run_program( + place, self.batch_size, self.beam_size, False + ) + value2 = build_and_run_program( + place, self.batch_size, self.beam_size, True + ) np.testing.assert_array_equal(value1, value2) diff --git a/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py b/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py index 7b7921d91c22a..01cfedf063ece 100644 --- a/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py +++ b/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py @@ -22,7 +22,6 @@ class Memory(object): - def __init__(self, shape, dtype='float32'): self.ex = np.zeros(shape=shape, dtype=dtype) self.cur = None @@ -45,7 +44,6 @@ def reset(self): class Output(object): - def __init__(self): self.outs = [] @@ -60,7 +58,6 @@ def last(self): class BaseRNN(object): - def __init__(self, ins, mems, params, outs, num_seq=5, max_seq_len=15): self.num_seq = num_seq self.inputs = collections.defaultdict(list) @@ -159,8 +156,10 @@ def to_feed(self, place): def get_numeric_gradient_of_param(self, param_name, delta=0.001): p = self.params[param_name] if len(p.shape) != 2: - raise ValueError("Not support get numeric gradient of an parameter," - " which is not matrix") + raise ValueError( + "Not support get numeric gradient of an parameter," + " which is not matrix" + ) g = np.zeros(shape=p.shape, dtype=p.dtype) for i in range(p.shape[0]): @@ -174,10 +173,9 @@ def get_numeric_gradient_of_param(self, param_name, delta=0.001): g[i][j] = (pos - neg) / (delta * 2) return g - def get_numeric_gradient_of_input(self, - input_name, - delta=0.001, - return_one_tensor=True): + def get_numeric_gradient_of_input( + self, input_name, delta=0.001, return_one_tensor=True + ): ipt = self.inputs[input_name] grad = [] @@ -213,7 +211,6 @@ def _exe_mean_out_(self): class SeedFixedTestCase(unittest.TestCase): - @classmethod def setUpClass(cls): """Fix random seeds to remove randomness from tests""" @@ -238,17 +235,18 @@ class TestSimpleMul(SeedFixedTestCase): OUT_NAME = 'Out' class SimpleMul(BaseRNN): - def __init__(self): base = TestSimpleMul - super(base.SimpleMul, - self).__init__({base.DATA_NAME: { - 'shape': [base.DATA_WIDTH] - }}, {}, { - base.PARAM_NAME: { - 'shape': [base.DATA_WIDTH, base.HIDDEN_WIDTH] - } - }, [base.OUT_NAME]) + super(base.SimpleMul, self).__init__( + {base.DATA_NAME: {'shape': [base.DATA_WIDTH]}}, + {}, + { + base.PARAM_NAME: { + 'shape': [base.DATA_WIDTH, base.HIDDEN_WIDTH] + } + }, + [base.OUT_NAME], + ) def step(self, X, W, Out): Out.out(np.matmul(X, W)) @@ -258,19 +256,21 @@ def step(self, X, W, Out): @prog_scope() def test_forward_backward(self): py_rnn = TestSimpleMul.SimpleMul() - dat = fluid.layers.data(name=self.DATA_NAME, - shape=[self.DATA_WIDTH], - lod_level=1) + dat = fluid.layers.data( + name=self.DATA_NAME, shape=[self.DATA_WIDTH], lod_level=1 + ) dat.stop_gradient = False rnn = fluid.layers.DynamicRNN() with rnn.block(): d = rnn.step_input(dat) - o = fluid.layers.fc(input=d, - param_attr=self.PARAM_NAME, - bias_attr=False, - size=self.HIDDEN_WIDTH, - act=None) + o = fluid.layers.fc( + input=d, + param_attr=self.PARAM_NAME, + bias_attr=False, + size=self.HIDDEN_WIDTH, + act=None, + ) rnn.output(o) out = rnn() @@ -283,18 +283,24 @@ def test_forward_backward(self): out, w_g, i_g = list( map( np.array, - exe.run(feed=py_rnn.to_feed(cpu), - fetch_list=[ - out, self.PARAM_NAME + "@GRAD", - self.DATA_NAME + "@GRAD" - ], - return_numpy=False))) + exe.run( + feed=py_rnn.to_feed(cpu), + fetch_list=[ + out, + self.PARAM_NAME + "@GRAD", + self.DATA_NAME + "@GRAD", + ], + return_numpy=False, + ), + ) + ) out_by_python = py_rnn.exe()[self.OUT_NAME] np.testing.assert_allclose(out, out_by_python, rtol=1e-05) w_g_num = py_rnn.get_numeric_gradient_of_param(self.PARAM_NAME) np.testing.assert_allclose(w_g_num, w_g, rtol=0.05) i_g_num = py_rnn.get_numeric_gradient_of_input( - input_name=self.DATA_NAME) + input_name=self.DATA_NAME + ) i_g_num = i_g_num.reshape(i_g.shape) np.testing.assert_allclose(i_g_num, i_g, rtol=0.05) @@ -306,23 +312,24 @@ class TestSimpleMulWithMemory(SeedFixedTestCase): PARAM_NAME = 'W' class SimpleMulWithMemory(BaseRNN): - def __init__(self): super(TestSimpleMulWithMemory.SimpleMulWithMemory, self).__init__( { TestSimpleMulWithMemory.DATA_NAME: { 'shape': [TestSimpleMulWithMemory.DATA_WIDTH] } - }, {'Mem': { - 'shape': [TestSimpleMulWithMemory.HIDDEN_WIDTH] - }}, { + }, + {'Mem': {'shape': [TestSimpleMulWithMemory.HIDDEN_WIDTH]}}, + { TestSimpleMulWithMemory.PARAM_NAME: { 'shape': [ TestSimpleMulWithMemory.DATA_WIDTH, - TestSimpleMulWithMemory.HIDDEN_WIDTH + TestSimpleMulWithMemory.HIDDEN_WIDTH, ] } - }, ['Out']) + }, + ['Out'], + ) def step(self, X, Mem, W, Out): o = np.matmul(X, W) @@ -337,19 +344,21 @@ def step(self, X, Mem, W, Out): @prog_scope() def test_forward_backward(self): py_rnn = TestSimpleMulWithMemory.SimpleMulWithMemory() - data = fluid.layers.data(name=self.DATA_NAME, - shape=[self.DATA_WIDTH], - lod_level=1) + data = fluid.layers.data( + name=self.DATA_NAME, shape=[self.DATA_WIDTH], lod_level=1 + ) data.stop_gradient = False rnn = fluid.layers.DynamicRNN() with rnn.block(): d = rnn.step_input(data) mem = rnn.memory(value=0.0, shape=[self.HIDDEN_WIDTH]) - hidden = fluid.layers.fc(input=d, - size=self.HIDDEN_WIDTH, - param_attr=self.PARAM_NAME, - bias_attr=False, - act=None) + hidden = fluid.layers.fc( + input=d, + size=self.HIDDEN_WIDTH, + param_attr=self.PARAM_NAME, + bias_attr=False, + act=None, + ) o = fluid.layers.elementwise_add(x=hidden, y=mem) rnn.update_memory(mem, o) rnn.output(o) @@ -365,13 +374,18 @@ def test_forward_backward(self): last_np, w_g, i_g = list( map( np.array, - exe.run(feed=feed, - fetch_list=[ - last, self.PARAM_NAME + "@GRAD", - self.DATA_NAME + "@GRAD" - ], - return_numpy=False))) - last_by_py, = list(py_rnn.exe().values()) + exe.run( + feed=feed, + fetch_list=[ + last, + self.PARAM_NAME + "@GRAD", + self.DATA_NAME + "@GRAD", + ], + return_numpy=False, + ), + ) + ) + (last_by_py,) = list(py_rnn.exe().values()) w_g_num = py_rnn.get_numeric_gradient_of_param(self.PARAM_NAME) np.testing.assert_allclose(last_np, last_by_py, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py b/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py index 3bbf9ff3266b2..2f0a99247564c 100644 --- a/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py +++ b/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py @@ -27,7 +27,6 @@ class TestDyRnnStaticInput(unittest.TestCase): - def setUp(self): self._delta = 0.005 self._max_sequence_len = 3 @@ -58,14 +57,14 @@ def prepare_static_input_tensor(self): self.static_input_tensor.set(self.static_input_data, self.place) def fetch_value(self, var): - fetch_outs = self.exe.run(feed={ - 'x_tensor': - self.x_tensor, - 'static_input_tensor': - self.static_input_tensor - }, - fetch_list=[var], - return_numpy=False) + fetch_outs = self.exe.run( + feed={ + 'x_tensor': self.x_tensor, + 'static_input_tensor': self.static_input_tensor, + }, + fetch_list=[var], + return_numpy=False, + ) return self._lodtensor_to_ndarray(fetch_outs[0]) def _lodtensor_to_ndarray(self, lod_tensor): @@ -76,24 +75,28 @@ def _lodtensor_to_ndarray(self, lod_tensor): return ndarray, lod_tensor.recursive_sequence_lengths() def build_graph(self, only_forward=False): - x_tensor = fluid.layers.data(name='x_tensor', - shape=[self.x_tensor_dim], - dtype='float32', - lod_level=1) + x_tensor = fluid.layers.data( + name='x_tensor', + shape=[self.x_tensor_dim], + dtype='float32', + lod_level=1, + ) x_tensor.stop_gradient = False static_input_tensor = fluid.layers.data( name='static_input_tensor', shape=[self.static_input_tensor_dim], dtype='float32', - lod_level=1) + lod_level=1, + ) static_input_tensor.stop_gradient = False if only_forward: static_input_out_array = self._program.global_block().create_var( name='static_input_out_array', type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype='float32') + dtype='float32', + ) static_input_out_array.stop_gradient = True rnn = fluid.layers.DynamicRNN() @@ -101,25 +104,30 @@ def build_graph(self, only_forward=False): step_x = rnn.step_input(x_tensor) step_static_input = rnn.static_input(static_input_tensor) if only_forward: - fluid.layers.array_write(x=step_static_input, - i=rnn.step_idx, - array=static_input_out_array) - last = fluid.layers.sequence_pool(input=step_static_input, - pool_type='last') - projected = fluid.layers.fc(input=[step_x, last], - size=self.output_dim) + fluid.layers.array_write( + x=step_static_input, + i=rnn.step_idx, + array=static_input_out_array, + ) + last = fluid.layers.sequence_pool( + input=step_static_input, pool_type='last' + ) + projected = fluid.layers.fc( + input=[step_x, last], size=self.output_dim + ) rnn.output(projected) if only_forward: static_input_step_outs = [] - step_idx = fluid.layers.fill_constant(shape=[1], - dtype='int64', - value=0) + step_idx = fluid.layers.fill_constant( + shape=[1], dtype='int64', value=0 + ) step_idx.stop_gradient = True for i in range(self._max_sequence_len): - step_out = fluid.layers.array_read(static_input_out_array, - step_idx) + step_out = fluid.layers.array_read( + static_input_out_array, step_idx + ) step_out.stop_gradient = True static_input_step_outs.append(step_out) fluid.layers.increment(x=step_idx, value=1.0, in_place=True) @@ -131,7 +139,8 @@ def build_graph(self, only_forward=False): loss = paddle.mean(last) append_backward(loss) static_input_grad = self._program.global_block().var( - framework.grad_var_name('static_input_tensor')) + framework.grad_var_name('static_input_tensor') + ) return static_input_grad, loss def get_expected_static_step_outs(self): @@ -145,8 +154,10 @@ def get_expected_static_step_outs(self): cur_offset = 0 for i in range(len(static_lod[0])): static_sliced.append( - self.static_input_data[cur_offset:(cur_offset + - static_lod[0][i])]) + self.static_input_data[ + cur_offset : (cur_offset + static_lod[0][i]) + ] + ) cur_offset += static_lod[0][i] static_seq_len = static_lod[0] static_reordered = [] @@ -170,7 +181,8 @@ def get_expected_static_step_outs(self): static_step_lods.append([lod]) end = total_len static_step_outs.append( - np.array(static_reordered[:end]).astype('float32')) + np.array(static_reordered[:end]).astype('float32') + ) return static_step_outs, static_step_lods @@ -203,13 +215,14 @@ def test_network_gradient(self): y_neg = self.fetch_value(loss)[0][0] self.static_input_tensor._set_float_element(i, origin) numeric_gradients.ravel()[i] = (y_pos - y_neg) / self._delta / 2 - np.testing.assert_allclose(actual_gradients, - numeric_gradients, - rtol=0.001) + np.testing.assert_allclose( + actual_gradients, numeric_gradients, rtol=0.001 + ) np.testing.assert_allclose( actual_lod, self.static_input_tensor.recursive_sequence_lengths(), - rtol=1e-05) + rtol=1e-05, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_delete_vars.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_delete_vars.py index b6052d33c95cb..eebc321b1a8f6 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_delete_vars.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_delete_vars.py @@ -39,8 +39,10 @@ def simple_fc_net(): hidden, size=200, act='tanh', - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) prediction = fluid.layers.fc(hidden, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) loss = paddle.mean(loss) @@ -65,7 +67,6 @@ def get_persistables_and_non_persistables(prog, fetch_list): class TestExecutor(unittest.TestCase): - def test_executor_main(self): places = [fluid.CPUPlace()] if fluid.core.is_compiled_with_cuda(): @@ -87,12 +88,13 @@ def test_executor_main(self): def prepare_feed(self, image, label, dev_cnt=1): batch_size = 32 * dev_cnt - image_shape = (batch_size, ) + tuple(image.shape[1:]) - label_shape = (batch_size, ) + tuple(label.shape[1:]) + image_shape = (batch_size,) + tuple(image.shape[1:]) + label_shape = (batch_size,) + tuple(label.shape[1:]) image_np = np.random.random(size=image_shape).astype('float32') - label_np = np.random.random_integers(low=0, high=9, - size=label_shape).astype('int64') + label_np = np.random.random_integers( + low=0, high=9, size=label_shape + ).astype('int64') return image_np, label_np @@ -113,10 +115,16 @@ def assertScopeVar(self, scope, persitables, non_persistables): if t._is_initialized(): outline_np_vars.append(name) - print('Non-alive persistable vars {} in {}'.format( - outline_p_vars, persitables)) - print('Alive non-persistable vars {} in {}'.format( - outline_np_vars, non_persistables)) + print( + 'Non-alive persistable vars {} in {}'.format( + outline_p_vars, persitables + ) + ) + print( + 'Alive non-persistable vars {} in {}'.format( + outline_np_vars, non_persistables + ) + ) self.assertEqual(len(outline_p_vars), 0) self.assertEqual(len(outline_np_vars), 0) @@ -130,12 +138,14 @@ def executor_main(self): image, label, loss = simple_fc_net() loss.persistable = False persistables, non_persistables = get_persistables_and_non_persistables( - fluid.default_main_program(), [loss.name]) + fluid.default_main_program(), [loss.name] + ) print('Non-persistable var number {}'.format(len(non_persistables))) print(non_persistables) - self.assert_gc_vars(fluid.default_main_program(), [loss.name], - non_persistables) + self.assert_gc_vars( + fluid.default_main_program(), [loss.name], non_persistables + ) exe = fluid.Executor(self.place) exe.run(fluid.default_startup_program()) @@ -147,23 +157,34 @@ def executor_main(self): for _ in range(10): image_np, label_np = self.prepare_feed(image, label) fluid.global_scope().var(image.name).get_tensor().set( - image_np, self.place) + image_np, self.place + ) fluid.global_scope().var(label.name).get_tensor().set( - label_np, self.place) + label_np, self.place + ) # exe.run would not create local scope # so that we can detect whether gc clears temporary variables - exe.run(fluid.default_main_program().desc, fluid.global_scope(), 0, - False, True, [loss.name]) - self.assertScopeVar(fluid.global_scope(), persistables, - non_persistables) + exe.run( + fluid.default_main_program().desc, + fluid.global_scope(), + 0, + False, + True, + [loss.name], + ) + self.assertScopeVar( + fluid.global_scope(), persistables, non_persistables + ) def pe_main(self): image, label, loss = simple_fc_net() loss.persistable = False persistables, non_persistables = get_persistables_and_non_persistables( - fluid.default_main_program(), [loss.name]) - self.assert_gc_vars(fluid.default_main_program(), [loss.name], - non_persistables) + fluid.default_main_program(), [loss.name] + ) + self.assert_gc_vars( + fluid.default_main_program(), [loss.name], non_persistables + ) exe = fluid.Executor(self.place) exe.run(fluid.default_startup_program()) @@ -176,11 +197,14 @@ def pe_main(self): build_strategy.enable_inplace = False prog = fluid.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - loss_name=loss.name, exec_strategy=exec_strategy) + fluid.default_main_program() + ).with_data_parallel(loss_name=loss.name, exec_strategy=exec_strategy) - dev_cnt = fluid.core.get_cuda_device_count() if isinstance(self.place, fluid.CUDAPlace) \ + dev_cnt = ( + fluid.core.get_cuda_device_count() + if isinstance(self.place, fluid.CUDAPlace) else int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + ) for idx in range(10): image_np, label_np = self.prepare_feed(image, label, dev_cnt) diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_dynamic_rnn_base.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_dynamic_rnn_base.py index 6e1ed17a3b679..86952d74e397b 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_dynamic_rnn_base.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_dynamic_rnn_base.py @@ -40,10 +40,9 @@ def train(network, use_cuda, use_parallel_executor, batch_size=32, pass_num=2): reader = fake_imdb_reader(word_dict_size, batch_size * 40) train_reader = paddle.batch(reader, batch_size=batch_size) - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") @@ -54,8 +53,9 @@ def train(network, use_cuda, use_parallel_executor, batch_size=32, pass_num=2): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() feeder = fluid.DataFeeder(feed_list=[data, label], place=place) - reader = feeder.decorate_reader(train_reader, - multi_devices=use_parallel_executor) + reader = feeder.decorate_reader( + train_reader, multi_devices=use_parallel_executor + ) exe = fluid.Executor(place) fluid.default_startup_program().random_seed = 1 @@ -65,8 +65,8 @@ def train(network, use_cuda, use_parallel_executor, batch_size=32, pass_num=2): train_cp = fluid.default_main_program() if use_parallel_executor: train_cp = compiler.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - loss_name=cost.name) + fluid.default_main_program() + ).with_data_parallel(loss_name=cost.name) fetch_list = [cost.name] else: fetch_list = [cost] @@ -74,16 +74,17 @@ def train(network, use_cuda, use_parallel_executor, batch_size=32, pass_num=2): for pass_id in range(pass_num): batch_id = 0 for data in reader(): - exe.run(train_cp, - feed=data, - fetch_list=fetch_list if batch_id % 4 == 0 else []) + exe.run( + train_cp, + feed=data, + fetch_list=fetch_list if batch_id % 4 == 0 else [], + ) batch_id += 1 if batch_id > 16: break class TestBase(unittest.TestCase): - def setUp(self): self.net = None @@ -93,9 +94,11 @@ def test_network(self): for use_cuda in [True, False]: for use_parallel_executor in [False, True]: - print('network: {}, use_cuda: {}, use_parallel_executor: {}'. - format(self.net.__name__, use_cuda, - use_parallel_executor)) + print( + 'network: {}, use_cuda: {}, use_parallel_executor: {}'.format( + self.net.__name__, use_cuda, use_parallel_executor + ) + ) with fluid.program_guard(fluid.Program(), fluid.Program()): with fluid.scope_guard(core.Scope()): train(self.net, use_cuda, use_parallel_executor) diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_gru_net.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_gru_net.py index ac501a43ca75e..f785936dcb99f 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_gru_net.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_gru_net.py @@ -20,18 +20,21 @@ fluid.core._set_eager_deletion_mode(0.0, 1.0, True) -def gru_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - emb_lr=400.0): +def gru_net( + data, + label, + dict_dim, + emb_dim=128, + hid_dim=128, + hid_dim2=96, + class_dim=2, + emb_lr=400.0, +): emb = fluid.layers.embedding( input=data, size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) + param_attr=fluid.ParamAttr(learning_rate=emb_lr), + ) fc0 = fluid.layers.fc(input=emb, size=hid_dim * 3) gru_h = fluid.layers.dynamic_gru(input=fc0, size=hid_dim, is_reverse=False) gru_max = fluid.layers.sequence_pool(input=gru_h, pool_type='max') @@ -44,7 +47,6 @@ def gru_net(data, class GRUTest(TestBase): - def setUp(self): self.net = gru_net diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_lstm_net.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_lstm_net.py index bb6f608201573..7709460111ac7 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_lstm_net.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_lstm_net.py @@ -20,22 +20,25 @@ fluid.core._set_eager_deletion_mode(0.0, 1.0, True) -def lstm_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - emb_lr=30.0): +def lstm_net( + data, + label, + dict_dim, + emb_dim=128, + hid_dim=128, + hid_dim2=96, + class_dim=2, + emb_lr=30.0, +): emb = fluid.layers.embedding( input=data, size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) + param_attr=fluid.ParamAttr(learning_rate=emb_lr), + ) fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4) - lstm_h, c = fluid.layers.dynamic_lstm(input=fc0, - size=hid_dim * 4, - is_reverse=False) + lstm_h, c = fluid.layers.dynamic_lstm( + input=fc0, size=hid_dim * 4, is_reverse=False + ) lstm_max = fluid.layers.sequence_pool(input=lstm_h, pool_type='max') lstm_max_tanh = fluid.layers.tanh(lstm_max) fc1 = fluid.layers.fc(input=lstm_max_tanh, size=hid_dim2, act='tanh') @@ -46,7 +49,6 @@ def lstm_net(data, class LSTMTest(TestBase): - def setUp(self): self.net = lstm_net diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_padding_rnn.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_padding_rnn.py index e31595a4b016d..e48a8056d030f 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_padding_rnn.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_padding_rnn.py @@ -28,7 +28,6 @@ class RNNConfig(object): - def __init__(self, model_type, rnn_model): self.model_type = model_type self.rnn_model = rnn_model @@ -95,26 +94,30 @@ def __init__(self, model_type, rnn_model): # Fake data reader for test class Reader(object): - def get_data_iter(self, rnn_config): for i in range(rnn_config.max_epoch): - x = np.zeros(shape=(rnn_config.batch_size, rnn_config.num_steps), - dtype='int64') - y = np.ones(shape=(rnn_config.batch_size, rnn_config.num_steps), - dtype='int64') + x = np.zeros( + shape=(rnn_config.batch_size, rnn_config.num_steps), + dtype='int64', + ) + y = np.ones( + shape=(rnn_config.batch_size, rnn_config.num_steps), + dtype='int64', + ) yield (x, y) # Model from PaddleNLP/models/language_model/lm_model.py in Paddle Models repo -def lm_model(hidden_size, - vocab_size, - batch_size, - num_layers=2, - num_steps=20, - init_scale=0.1, - dropout=None, - rnn_model='static'): - +def lm_model( + hidden_size, + vocab_size, + batch_size, + num_layers=2, + num_steps=20, + init_scale=0.1, + dropout=None, + rnn_model='static', +): def padding_rnn(input_embedding, len=3, init_hidden=None, init_cell=None): weight_1_arr = [] weight_2_arr = [] @@ -128,23 +131,24 @@ def padding_rnn(input_embedding, len=3, init_hidden=None, init_cell=None): dtype="float32", name="fc_weight1_" + str(i), default_initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale)) + low=-init_scale, high=init_scale + ), + ) weight_1_arr.append(weight_1) bias_1 = layers.create_parameter( [hidden_size * 4], dtype="float32", name="fc_bias1_" + str(i), - default_initializer=fluid.initializer.Constant(0.0)) + default_initializer=fluid.initializer.Constant(0.0), + ) bias_arr.append(bias_1) - pre_hidden = layers.slice(init_hidden, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_cell = layers.slice(init_cell, - axes=[0], - starts=[i], - ends=[i + 1]) + pre_hidden = layers.slice( + init_hidden, axes=[0], starts=[i], ends=[i + 1] + ) + pre_cell = layers.slice( + init_cell, axes=[0], starts=[i], ends=[i + 1] + ) pre_hidden = layers.reshape(pre_hidden, shape=[-1, hidden_size]) pre_cell = layers.reshape(pre_cell, shape=[-1, hidden_size]) hidden_array.append(pre_hidden) @@ -165,25 +169,31 @@ def padding_rnn(input_embedding, len=3, init_hidden=None, init_cell=None): gate_input = layers.matmul(x=nn, y=weight_1) gate_input = layers.elementwise_add(gate_input, bias) - i = layers.slice(gate_input, - axes=[1], - starts=[0], - ends=[hidden_size]) - j = layers.slice(gate_input, - axes=[1], - starts=[hidden_size], - ends=[hidden_size * 2]) - f = layers.slice(gate_input, - axes=[1], - starts=[hidden_size * 2], - ends=[hidden_size * 3]) - o = layers.slice(gate_input, - axes=[1], - starts=[hidden_size * 3], - ends=[hidden_size * 4]) + i = layers.slice( + gate_input, axes=[1], starts=[0], ends=[hidden_size] + ) + j = layers.slice( + gate_input, + axes=[1], + starts=[hidden_size], + ends=[hidden_size * 2], + ) + f = layers.slice( + gate_input, + axes=[1], + starts=[hidden_size * 2], + ends=[hidden_size * 3], + ) + o = layers.slice( + gate_input, + axes=[1], + starts=[hidden_size * 3], + ends=[hidden_size * 4], + ) c = pre_cell * layers.sigmoid(f) + layers.sigmoid( - i) * layers.tanh(j) + i + ) * layers.tanh(j) m = layers.tanh(c) * layers.sigmoid(o) rnn.update_memory(pre_hidden, m) @@ -198,7 +208,8 @@ def padding_rnn(input_embedding, len=3, init_hidden=None, init_cell=None): input = layers.dropout( input, dropout_prob=dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn.step_output(input) rnnout = rnn() @@ -211,15 +222,13 @@ def padding_rnn(input_embedding, len=3, init_hidden=None, init_cell=None): c = rnnout[i * 2 + 1] m.stop_gradient = True c.stop_gradient = True - last_h = layers.slice(m, - axes=[0], - starts=[num_steps - 1], - ends=[num_steps]) + last_h = layers.slice( + m, axes=[0], starts=[num_steps - 1], ends=[num_steps] + ) last_hidden_array.append(last_h) - last_c = layers.slice(c, - axes=[0], - starts=[num_steps - 1], - ends=[num_steps]) + last_c = layers.slice( + c, axes=[0], starts=[num_steps - 1], ends=[num_steps] + ) last_cell_array.append(last_c) real_res = layers.transpose(x=real_res, perm=[1, 0, 2]) last_hidden = layers.concat(last_hidden_array, 0) @@ -227,10 +236,9 @@ def padding_rnn(input_embedding, len=3, init_hidden=None, init_cell=None): return real_res, last_hidden, last_cell - def encoder_static(input_embedding, - len=3, - init_hidden=None, - init_cell=None): + def encoder_static( + input_embedding, len=3, init_hidden=None, init_cell=None + ): weight_1_arr = [] weight_2_arr = [] @@ -244,36 +252,37 @@ def encoder_static(input_embedding, dtype="float32", name="fc_weight1_" + str(i), default_initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale)) + low=-init_scale, high=init_scale + ), + ) weight_1_arr.append(weight_1) bias_1 = layers.create_parameter( [hidden_size * 4], dtype="float32", name="fc_bias1_" + str(i), - default_initializer=fluid.initializer.Constant(0.0)) + default_initializer=fluid.initializer.Constant(0.0), + ) bias_arr.append(bias_1) - pre_hidden = layers.slice(init_hidden, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_cell = layers.slice(init_cell, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_hidden = layers.reshape(pre_hidden, - shape=[-1, hidden_size], - inplace=True) - pre_cell = layers.reshape(pre_cell, - shape=[-1, hidden_size], - inplace=True) + pre_hidden = layers.slice( + init_hidden, axes=[0], starts=[i], ends=[i + 1] + ) + pre_cell = layers.slice( + init_cell, axes=[0], starts=[i], ends=[i + 1] + ) + pre_hidden = layers.reshape( + pre_hidden, shape=[-1, hidden_size], inplace=True + ) + pre_cell = layers.reshape( + pre_cell, shape=[-1, hidden_size], inplace=True + ) hidden_array.append(pre_hidden) cell_array.append(pre_cell) res = [] - sliced_inputs = layers.split(input_embedding, - num_or_sections=len, - dim=1) + sliced_inputs = layers.split( + input_embedding, num_or_sections=len, dim=1 + ) for index in range(len): input = sliced_inputs[index] @@ -291,7 +300,8 @@ def encoder_static(input_embedding, i, j, f, o = layers.split(gate_input, num_or_sections=4, dim=-1) c = pre_cell * layers.sigmoid(f) + layers.sigmoid( - i) * layers.tanh(j) + i + ) * layers.tanh(j) m = layers.tanh(c) * layers.sigmoid(o) hidden_array[k] = m @@ -302,55 +312,67 @@ def encoder_static(input_embedding, input = layers.dropout( input, dropout_prob=dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) res.append(input) last_hidden = layers.concat(hidden_array, 1) - last_hidden = layers.reshape(last_hidden, - shape=[-1, num_layers, hidden_size], - inplace=True) + last_hidden = layers.reshape( + last_hidden, shape=[-1, num_layers, hidden_size], inplace=True + ) last_hidden = layers.transpose(x=last_hidden, perm=[1, 0, 2]) last_cell = layers.concat(cell_array, 1) - last_cell = layers.reshape(last_cell, - shape=[-1, num_layers, hidden_size]) + last_cell = layers.reshape( + last_cell, shape=[-1, num_layers, hidden_size] + ) last_cell = layers.transpose(x=last_cell, perm=[1, 0, 2]) real_res = layers.concat(res, 0) - real_res = layers.reshape(real_res, - shape=[len, -1, hidden_size], - inplace=True) + real_res = layers.reshape( + real_res, shape=[len, -1, hidden_size], inplace=True + ) real_res = layers.transpose(x=real_res, perm=[1, 0, 2]) return real_res, last_hidden, last_cell batch_size_each = batch_size - x = layers.data(name="x", - shape=[batch_size_each, num_steps, 1], - dtype='int64', - append_batch_size=False) - y = layers.data(name="y", - shape=[batch_size_each * num_steps, 1], - dtype='int64', - append_batch_size=False) - - init_hidden = layers.data(name="init_hidden", - shape=[num_layers, batch_size_each, hidden_size], - dtype='float32', - append_batch_size=False) - init_cell = layers.data(name="init_cell", - shape=[num_layers, batch_size_each, hidden_size], - dtype='float32', - append_batch_size=False) + x = layers.data( + name="x", + shape=[batch_size_each, num_steps, 1], + dtype='int64', + append_batch_size=False, + ) + y = layers.data( + name="y", + shape=[batch_size_each * num_steps, 1], + dtype='int64', + append_batch_size=False, + ) + + init_hidden = layers.data( + name="init_hidden", + shape=[num_layers, batch_size_each, hidden_size], + dtype='float32', + append_batch_size=False, + ) + init_cell = layers.data( + name="init_cell", + shape=[num_layers, batch_size_each, hidden_size], + dtype='float32', + append_batch_size=False, + ) init_cell.persistable = True init_hidden.persistable = True - init_hidden_reshape = layers.reshape(init_hidden, - shape=[num_layers, -1, hidden_size]) - init_cell_reshape = layers.reshape(init_cell, - shape=[num_layers, -1, hidden_size]) + init_hidden_reshape = layers.reshape( + init_hidden, shape=[num_layers, -1, hidden_size] + ) + init_cell_reshape = layers.reshape( + init_cell, shape=[num_layers, -1, hidden_size] + ) x_emb = layers.embedding( input=x, @@ -359,29 +381,36 @@ def encoder_static(input_embedding, is_sparse=False, param_attr=fluid.ParamAttr( name='embedding_para', - initializer=fluid.initializer.UniformInitializer(low=-init_scale, - high=init_scale))) - - x_emb = layers.reshape(x_emb, - shape=[-1, num_steps, hidden_size], - inplace=True) + initializer=fluid.initializer.UniformInitializer( + low=-init_scale, high=init_scale + ), + ), + ) + + x_emb = layers.reshape( + x_emb, shape=[-1, num_steps, hidden_size], inplace=True + ) if dropout != None and dropout > 0.0: - x_emb = layers.dropout(x_emb, - dropout_prob=dropout, - dropout_implementation='upscale_in_train') + x_emb = layers.dropout( + x_emb, + dropout_prob=dropout, + dropout_implementation='upscale_in_train', + ) if rnn_model == "padding": rnn_out, last_hidden, last_cell = padding_rnn( x_emb, len=num_steps, init_hidden=init_hidden_reshape, - init_cell=init_cell_reshape) + init_cell=init_cell_reshape, + ) elif rnn_model == "static": rnn_out, last_hidden, last_cell = encoder_static( x_emb, len=num_steps, init_hidden=init_hidden_reshape, - init_cell=init_cell_reshape) + init_cell=init_cell_reshape, + ) elif rnn_model == "cudnn": x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) rnn_out, last_hidden, last_cell = layers.lstm( @@ -393,44 +422,61 @@ def encoder_static(input_embedding, num_layers, is_bidirec=False, default_initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale)) + low=-init_scale, high=init_scale + ), + ) rnn_out = layers.transpose(rnn_out, perm=[1, 0, 2]) elif rnn_model == "basic_lstm": - rnn_out, last_hidden, last_cell = basic_lstm( x_emb, init_hidden, init_cell, hidden_size, \ - num_layers=num_layers, batch_first=True, dropout_prob=dropout, \ - param_attr = ParamAttr( initializer=fluid.initializer.UniformInitializer(low=-init_scale, high=init_scale) ), \ - bias_attr = ParamAttr( initializer = fluid.initializer.Constant(0.0) ), \ - forget_bias = 0.0) + rnn_out, last_hidden, last_cell = basic_lstm( + x_emb, + init_hidden, + init_cell, + hidden_size, + num_layers=num_layers, + batch_first=True, + dropout_prob=dropout, + param_attr=ParamAttr( + initializer=fluid.initializer.UniformInitializer( + low=-init_scale, high=init_scale + ) + ), + bias_attr=ParamAttr(initializer=fluid.initializer.Constant(0.0)), + forget_bias=0.0, + ) else: print("type not support") return - rnn_out = layers.reshape(rnn_out, - shape=[-1, num_steps, hidden_size], - inplace=True) + rnn_out = layers.reshape( + rnn_out, shape=[-1, num_steps, hidden_size], inplace=True + ) softmax_weight = layers.create_parameter( [hidden_size, vocab_size], dtype="float32", name="softmax_weight", default_initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale)) + low=-init_scale, high=init_scale + ), + ) softmax_bias = layers.create_parameter( [vocab_size], dtype="float32", name='softmax_bias', default_initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale)) + low=-init_scale, high=init_scale + ), + ) projection = layers.matmul(rnn_out, softmax_weight) projection = layers.elementwise_add(projection, softmax_bias) - projection = layers.reshape(projection, - shape=[-1, vocab_size], - inplace=True) + projection = layers.reshape( + projection, shape=[-1, vocab_size], inplace=True + ) - loss = layers.softmax_with_cross_entropy(logits=projection, - label=y, - soft_label=False) + loss = layers.softmax_with_cross_entropy( + logits=projection, label=y, soft_label=False + ) loss = layers.reshape(loss, shape=[-1, num_steps], inplace=True) loss = layers.reduce_mean(loss, dim=[0]) @@ -452,7 +498,6 @@ def encoder_static(input_embedding, class PaddingRNNTestBase(unittest.TestCase): - def setUp(self): self.reader = Reader() self.device_count = 1 @@ -485,64 +530,91 @@ def _prepare_program(self, config, parallel=True): self.startup_program = fluid.Program() with fluid.program_guard(self.main_program, self.startup_program): with fluid.unique_name.guard(): - res_vars = lm_model(config.hidden_size, - config.vocab_size, - config.batch_size, - num_layers=config.num_layers, - num_steps=config.num_steps, - init_scale=config.init_scale, - dropout=config.dropout, - rnn_model=config.rnn_model) - self.loss, self.last_hidden, self.last_cell, self.feed_order = res_vars + res_vars = lm_model( + config.hidden_size, + config.vocab_size, + config.batch_size, + num_layers=config.num_layers, + num_steps=config.num_steps, + init_scale=config.init_scale, + dropout=config.dropout, + rnn_model=config.rnn_model, + ) + ( + self.loss, + self.last_hidden, + self.last_cell, + self.feed_order, + ) = res_vars fluid.clip.set_gradient_clip( clip=fluid.clip.GradientClipByGlobalNorm( - clip_norm=config.max_grad_norm)) + clip_norm=config.max_grad_norm + ) + ) self.learning_rate = fluid.layers.create_global_var( name="learning_rate", shape=[1], value=1.0, dtype='float32', - persistable=True) + persistable=True, + ) optimizer = fluid.optimizer.SGD( - learning_rate=self.learning_rate) + learning_rate=self.learning_rate + ) optimizer.minimize(self.loss) self.exe.run(self.startup_program) if parallel: self.train_program = fluid.compiler.CompiledProgram( - self.main_program).with_data_parallel( - loss_name=self.loss.name, - build_strategy=self.build_strategy, - exec_strategy=self.exec_strategy) + self.main_program + ).with_data_parallel( + loss_name=self.loss.name, + build_strategy=self.build_strategy, + exec_strategy=self.exec_strategy, + ) else: self.train_program = self.main_program def _generate_init_data(self): - init_hidden = np.zeros((self.config.num_layers, self.config.batch_size, - self.config.hidden_size), - dtype='float32') - init_cell = np.zeros((self.config.num_layers, self.config.batch_size, - self.config.hidden_size), - dtype='float32') + init_hidden = np.zeros( + ( + self.config.num_layers, + self.config.batch_size, + self.config.hidden_size, + ), + dtype='float32', + ) + init_cell = np.zeros( + ( + self.config.num_layers, + self.config.batch_size, + self.config.hidden_size, + ), + dtype='float32', + ) return init_hidden, init_cell def _generate_new_lr(self, epoch_id=0, device_count=1): - new_lr = self.config.base_learning_rate * (self.config.lr_decay**max( - epoch_id + 1 - self.config.epoch_start_decay, 0.0)) + new_lr = self.config.base_learning_rate * ( + self.config.lr_decay + ** max(epoch_id + 1 - self.config.epoch_start_decay, 0.0) + ) lr = np.ones((self.device_count), dtype='float32') * new_lr return lr - def _prepare_input(self, - batch, - init_hidden=None, - init_cell=None, - epoch_id=0, - with_lr=True, - device_count=1): + def _prepare_input( + self, + batch, + init_hidden=None, + init_cell=None, + epoch_id=0, + with_lr=True, + device_count=1, + ): x, y = batch x = x.reshape((-1, self.config.num_steps, 1)) y = y.reshape((-1, 1)) @@ -573,16 +645,20 @@ def _train_an_epoch(self, epoch_id, use_program_cache=True): init_cell=init_cell, epoch_id=epoch_id, with_lr=True, - device_count=self.device_count) - - fetch_outs = self.exe.run(self.train_program, - feed=input_data_feed, - fetch_list=[ - self.loss.name, "learning_rate", - self.last_hidden.name, - self.last_cell.name - ], - use_program_cache=use_program_cache) + device_count=self.device_count, + ) + + fetch_outs = self.exe.run( + self.train_program, + feed=input_data_feed, + fetch_list=[ + self.loss.name, + "learning_rate", + self.last_hidden.name, + self.last_cell.name, + ], + use_program_cache=use_program_cache, + ) cost_train = np.array(fetch_outs[0]) lr = np.array(fetch_outs[1]) @@ -607,9 +683,9 @@ def train(self, config, parallel=True, use_program_cache=True): ppl = np.append(ppl, train_ppl) return ppl - def compare_padding_static_mode(self, - parallel=True, - use_program_cache=True): + def compare_padding_static_mode( + self, parallel=True, use_program_cache=True + ): ''' Test that train ppl of padding mode is same to that of static mode ''' @@ -623,7 +699,6 @@ def compare_padding_static_mode(self, class EagerDeletionPaddingRNNTest(PaddingRNNTestBase): - def test_padding_mode_no_eager_deletion(self): ''' Test that train ppl of padding mode is same to that of static mode without eager deletion diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_recurrent_op.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_recurrent_op.py index 018ab42c50a55..b67a72a5da2fd 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_recurrent_op.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_recurrent_op.py @@ -33,7 +33,6 @@ class PyRNNBase(object): - def __init__(self, input_shape, output_shape): self.x = np.ones(shape=input_shape).astype("float32") self.y = np.zeros(shape=output_shape).astype("float32") @@ -51,13 +50,13 @@ def segment_inputs(self): class PySimpleRNN1(PyRNNBase): - def __init__(self, input_shape, output_shape): super(PySimpleRNN1, self).__init__(input_shape, output_shape) seq_len, batch_size, input_dim = input_shape - self.h_boot = np.random.normal(size=(batch_size, - input_dim)).astype("float32") + self.h_boot = np.random.normal(size=(batch_size, input_dim)).astype( + "float32" + ) self.scale = 1.0 / 2.0 men_dim = (seq_len, batch_size, input_dim) @@ -73,7 +72,6 @@ def step(self, step_id, x): class PySimpleRNN2(PyRNNBase): - def __init__(self, input_shape, output_shape): super(PySimpleRNN2, self).__init__(input_shape, output_shape) @@ -94,7 +92,7 @@ def step(self, step_id, x): hU = np.matmul(pre_mem, self.U).astype("float32") def py_sigmoid(x): - return 1. / (1. + np.exp(-x)) + return 1.0 / (1.0 + np.exp(-x)) self.mems[step_id] = py_sigmoid(xW + hU) self.y[step_id] = self.mems[step_id] @@ -140,14 +138,16 @@ def setUp(self): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False - h_boot = layers.data(shape=[self.input_dim], - dtype='float32', - name='h_boot') + h_boot = layers.data( + shape=[self.input_dim], dtype='float32', name='h_boot' + ) h_boot.stop_gradient = False rnn = layers.StaticRNN() @@ -155,8 +155,10 @@ def create_rnn_op(self): h_pre = rnn.memory(init=h_boot) x_t = rnn.step_input(x) - h = layers.scale(x=layers.elementwise_add(x=h_pre, y=x_t), - scale=self.py_rnn.scale) + h = layers.scale( + x=layers.elementwise_add(x=h_pre, y=x_t), + scale=self.py_rnn.scale, + ) rnn.update_memory(h_pre, h) rnn.output(h) @@ -164,17 +166,18 @@ def create_rnn_op(self): return rnn() def forward(self): - gc_vars = core._get_eager_deletion_vars(self.main_program.desc, - [self.output.name]) + gc_vars = core._get_eager_deletion_vars( + self.main_program.desc, [self.output.name] + ) self.assertEqual(len(gc_vars), self.main_program.num_blocks) self.feed_map = { x: create_tensor(getattr(self.py_rnn, x), self.place) for x in self.data_field } exe = Executor(self.place) - out = exe.run(self.main_program, - feed=self.feed_map, - fetch_list=[self.output]) + out = exe.run( + self.main_program, feed=self.feed_map, fetch_list=[self.output] + ) return out[0] @@ -189,14 +192,17 @@ def backward(self): ] gc_vars = core._get_eager_deletion_vars( - self.main_program.desc, [var.name for var in fetch_list]) + self.main_program.desc, [var.name for var in fetch_list] + ) self.assertEqual(len(gc_vars), self.main_program.num_blocks) exe = Executor(self.place) - return exe.run(self.main_program, - feed=self.feed_map, - fetch_list=fetch_list, - return_numpy=False) + return exe.run( + self.main_program, + feed=self.feed_map, + fetch_list=fetch_list, + return_numpy=False, + ) def test_backward(self, rtol=0.01): self.check_forward() @@ -213,10 +219,18 @@ def test_backward(self, rtol=0.01): num_grad[idx], ana_grad[idx], rtol=rtol, - err_msg='num_grad (' + name + ') has diff at ' + - str(self.place) + '\nExpect ' + str(num_grad[idx]) + '\n' + - 'But Got' + str(ana_grad[idx]) + ' in class ' + - self.__class__.__name__) + err_msg='num_grad (' + + name + + ') has diff at ' + + str(self.place) + + '\nExpect ' + + str(num_grad[idx]) + + '\n' + + 'But Got' + + str(ana_grad[idx]) + + ' in class ' + + self.__class__.__name__, + ) def check_forward(self): pd_output = self.forward() @@ -277,14 +291,16 @@ def setUp(self): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False - h_boot = layers.data(shape=[self.input_dim], - dtype='float32', - name='h_boot') + h_boot = layers.data( + shape=[self.input_dim], dtype='float32', name='h_boot' + ) h_boot.stop_gradient = False rnn = layers.StaticRNN() @@ -297,15 +313,19 @@ def create_rnn_op(self): size=self.input_dim, param_attr=ParamAttr( name='W', - initializer=fluid.initializer.ConstantInitializer(1.0)), - bias_attr=False) + initializer=fluid.initializer.ConstantInitializer(1.0), + ), + bias_attr=False, + ) temp_r = layers.fc( input=h_pre, size=self.input_dim, param_attr=ParamAttr( name='U', - initializer=fluid.initializer.ConstantInitializer(0.0)), - bias_attr=False) + initializer=fluid.initializer.ConstantInitializer(0.0), + ), + bias_attr=False, + ) h = layers.sigmoid(x=layers.elementwise_add(x=temp_l, y=temp_r)) @@ -334,16 +354,18 @@ class EagerDeletionRecurrentOpMultipleMemoryTest(EagerDeletionRecurrentOpTest1): ''' class PySimpleRNN3(PyRNNBase): - def __init__(self, input_shape, output_shape): - super(EagerDeletionRecurrentOpMultipleMemoryTest.PySimpleRNN3, - self).__init__(input_shape, output_shape) + super( + EagerDeletionRecurrentOpMultipleMemoryTest.PySimpleRNN3, self + ).__init__(input_shape, output_shape) seq_len, batch_size, input_dim = input_shape - self.h_boot1 = np.random.normal(size=(batch_size, - input_dim)).astype("float32") - self.h_boot2 = np.random.normal(size=(batch_size, - input_dim)).astype("float32") + self.h_boot1 = np.random.normal( + size=(batch_size, input_dim) + ).astype("float32") + self.h_boot2 = np.random.normal( + size=(batch_size, input_dim) + ).astype("float32") men_dim = (seq_len, batch_size, input_dim) self.mems1 = np.zeros(shape=men_dim).astype("float32") @@ -372,26 +394,33 @@ def setUp(self): self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = EagerDeletionRecurrentOpMultipleMemoryTest.PySimpleRNN3( - self.input_shape, self.output_shape) + self.input_shape, self.output_shape + ) with fluid.program_guard(self.main_program, self.startup_program): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False - h_boot1 = layers.data(shape=[self.batch_size, self.input_dim], - dtype='float32', - name='h_boot1', - append_batch_size=False) + h_boot1 = layers.data( + shape=[self.batch_size, self.input_dim], + dtype='float32', + name='h_boot1', + append_batch_size=False, + ) h_boot1.stop_gradient = False - h_boot2 = layers.data(shape=[self.batch_size, self.input_dim], - dtype='float32', - name='h_boot2', - append_batch_size=False) + h_boot2 = layers.data( + shape=[self.batch_size, self.input_dim], + dtype='float32', + name='h_boot2', + append_batch_size=False, + ) h_boot2.stop_gradient = False rnn = layers.StaticRNN() @@ -426,10 +455,10 @@ class EagerDeletionRecurrentOpNoMemBootTest(EagerDeletionRecurrentOpTest1): ''' class PySimpleRNN4(PyRNNBase): - def __init__(self, input_shape, output_shape): - super(EagerDeletionRecurrentOpNoMemBootTest.PySimpleRNN4, - self).__init__(input_shape, output_shape) + super( + EagerDeletionRecurrentOpNoMemBootTest.PySimpleRNN4, self + ).__init__(input_shape, output_shape) men_dim = input_shape self.mems = np.zeros(shape=men_dim).astype("float32") @@ -453,16 +482,19 @@ def setUp(self): self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = EagerDeletionRecurrentOpNoMemBootTest.PySimpleRNN4( - self.input_shape, self.output_shape) + self.input_shape, self.output_shape + ) with fluid.program_guard(self.main_program, self.startup_program): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False rnn = layers.StaticRNN() @@ -496,10 +528,10 @@ class EagerDeletionTwoRecurrentOpsTest(EagerDeletionRecurrentOpTest1): ''' class PySimpleRNN5(PyRNNBase): - def __init__(self, input_shape, output_shape): - super(EagerDeletionTwoRecurrentOpsTest.PySimpleRNN5, - self).__init__(input_shape, output_shape) + super(EagerDeletionTwoRecurrentOpsTest.PySimpleRNN5, self).__init__( + input_shape, output_shape + ) self.mem_0 = np.zeros(shape=input_shape).astype("float32") self.mem_1 = np.zeros(shape=input_shape).astype("float32") self.rnn_0_output = np.zeros(shape=input_shape).astype("float32") @@ -508,13 +540,15 @@ def step(self, step_id, x): # First Rnn for step in range(self.x.shape[0]): x_t = self.x[step] - pre_mem = np.zeros_like(x_t) if step == 0 else self.mem_0[step - - 1] + pre_mem = ( + np.zeros_like(x_t) if step == 0 else self.mem_0[step - 1] + ) self.mem_0[step] = x_t + pre_mem self.rnn_0_output[step] = self.mem_0[step] # Second RNN - pre_mem = np.zeros_like(x) if step_id == 0 else self.mem_1[step_id - - 1] + pre_mem = ( + np.zeros_like(x) if step_id == 0 else self.mem_1[step_id - 1] + ) self.mem_1[step_id] = x + np.sum(self.rnn_0_output) self.y[step_id] = self.mem_1[step_id] + pre_mem @@ -530,16 +564,19 @@ def setUp(self): self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = EagerDeletionTwoRecurrentOpsTest.PySimpleRNN5( - self.input_shape, self.output_shape) + self.input_shape, self.output_shape + ) with fluid.program_guard(self.main_program, self.startup_program): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False rnn_0 = layers.StaticRNN() @@ -563,8 +600,9 @@ def create_rnn_op(self): return rnn_1() -class EagerDeletionRecurrentOpParallelExecutorTest(EagerDeletionRecurrentOpTest1 - ): +class EagerDeletionRecurrentOpParallelExecutorTest( + EagerDeletionRecurrentOpTest1 +): ''' Test RNNOp with ParallelExecutor equation: @@ -586,10 +624,12 @@ def forward(self): build_strategy = fluid.BuildStrategy() build_strategy.enable_inplace = True exec_strategy = fluid.ExecutionStrategy() - parallel_exe = fluid.ParallelExecutor(use_cuda=False, - main_program=self.main_program, - build_strategy=build_strategy, - exec_strategy=exec_strategy) + parallel_exe = fluid.ParallelExecutor( + use_cuda=False, + main_program=self.main_program, + build_strategy=build_strategy, + exec_strategy=exec_strategy, + ) out = parallel_exe.run(feed=self.feed_map, fetch_list=[self.output]) return out[0] @@ -606,20 +646,23 @@ def backward(self): build_strategy = fluid.BuildStrategy() build_strategy.enable_inplace = True exec_strategy = fluid.ExecutionStrategy() - parallel_exe = fluid.ParallelExecutor(use_cuda=False, - loss_name=self.output.name, - main_program=self.main_program, - build_strategy=build_strategy, - exec_strategy=exec_strategy) - return parallel_exe.run(feed=self.feed_map, - fetch_list=fetch_list, - return_numpy=False) + parallel_exe = fluid.ParallelExecutor( + use_cuda=False, + loss_name=self.output.name, + main_program=self.main_program, + build_strategy=build_strategy, + exec_strategy=exec_strategy, + ) + return parallel_exe.run( + feed=self.feed_map, fetch_list=fetch_list, return_numpy=False + ) class EagerDeletionFarwardOnlyRnnAndBackwardRnnTest( - EagerDeletionRecurrentOpTest1): + EagerDeletionRecurrentOpTest1 +): ''' - Test one forward only RNN and one backward RNN in one program + Test one forward only RNN and one backward RNN in one program ''' def setUp(self): @@ -635,11 +678,12 @@ def setUp(self): shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', name='x', - append_batch_size=False) + append_batch_size=False, + ) x.stop_gradient = False - h_boot = layers.data(shape=[self.input_dim], - dtype='float32', - name='h_boot') + h_boot = layers.data( + shape=[self.input_dim], dtype='float32', name='h_boot' + ) h_boot.stop_gradient = False forward_only_rnn = layers.StaticRNN() @@ -647,8 +691,10 @@ def setUp(self): h_pre = forward_only_rnn.memory(init=h_boot) x_t = forward_only_rnn.step_input(x) - h = layers.scale(x=layers.elementwise_add(x=h_pre, y=x_t), - scale=self.py_rnn.scale) + h = layers.scale( + x=layers.elementwise_add(x=h_pre, y=x_t), + scale=self.py_rnn.scale, + ) forward_only_rnn.update_memory(h_pre, h) forward_only_rnn.output(h) @@ -661,8 +707,10 @@ def setUp(self): h_pre = rnn.memory(init=h_boot) x_t = rnn.step_input(x) - h = layers.scale(x=layers.elementwise_add(x=h_pre, y=x_t), - scale=self.py_rnn.scale) + h = layers.scale( + x=layers.elementwise_add(x=h_pre, y=x_t), + scale=self.py_rnn.scale, + ) rnn.update_memory(h_pre, h) rnn.output(h) @@ -675,9 +723,11 @@ def forward_two_rnn(self): for x in self.data_field } exe = Executor(self.place) - out = exe.run(self.main_program, - feed=self.feed_map, - fetch_list=[self.forward_only_output, self.output]) + out = exe.run( + self.main_program, + feed=self.feed_map, + fetch_list=[self.forward_only_output, self.output], + ) return out[0], out[1] @@ -691,7 +741,6 @@ def check_forward(self): class RecurrentNet(paddle.nn.Layer): - def __init__(self): super(RecurrentNet, self).__init__() self.cell = paddle.nn.SimpleRNNCell(16, 32) @@ -703,7 +752,6 @@ def forward(self, inputs, prev_h): class TestDy2StRecurrentOpBackward(unittest.TestCase): - def setUp(self): paddle.disable_static() paddle.seed(100) diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_while_op.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_while_op.py index 5aa5ff4eeccd3..1793d69f48f18 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_while_op.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_while_op.py @@ -32,7 +32,6 @@ class TestEagerDeletionWhileOpBase(unittest.TestCase): - def test_main(self): places = [ core.CPUPlace(), @@ -51,29 +50,30 @@ def run_main(self, place, with_data_parallel): self.with_data_parallel = with_data_parallel if not core.is_compiled_with_cuda() and isinstance( - self.place, core.CUDAPlace): + self.place, core.CUDAPlace + ): return if isinstance(self.place, core.CUDAPlace): - device_cnt = core.get_cuda_device_count( - ) if self.with_data_parallel else 1 + device_cnt = ( + core.get_cuda_device_count() if self.with_data_parallel else 1 + ) else: - device_cnt = int( - os.environ.get('CPU_NUM', multiprocessing.cpu_count()) - ) if self.with_data_parallel else 1 - - d0 = layers.data("d0", - shape=[10], - append_batch_size=False, - dtype='float32') - d1 = layers.data("d1", - shape=[10], - append_batch_size=False, - dtype='float32') - d2 = layers.data("d2", - shape=[10], - append_batch_size=False, - dtype='float32') + device_cnt = ( + int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + if self.with_data_parallel + else 1 + ) + + d0 = layers.data( + "d0", shape=[10], append_batch_size=False, dtype='float32' + ) + d1 = layers.data( + "d1", shape=[10], append_batch_size=False, dtype='float32' + ) + d2 = layers.data( + "d2", shape=[10], append_batch_size=False, dtype='float32' + ) i = layers.zeros(shape=[1], dtype='int64') i.stop_gradient = True @@ -136,7 +136,8 @@ def run_main(self, place, with_data_parallel): optim.minimize(loss) gc_vars = core._get_eager_deletion_vars( - fluid.default_main_program().desc, [loss.name]) + fluid.default_main_program().desc, [loss.name] + ) self.assertEqual(len(gc_vars), 5) exe = Executor(self.place) @@ -145,8 +146,8 @@ def run_main(self, place, with_data_parallel): prog = fluid.default_main_program() if self.with_data_parallel: prog = compiler.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - loss_name=loss.name) + fluid.default_main_program() + ).with_data_parallel(loss_name=loss.name) for _ in range(5): d = [] @@ -157,13 +158,11 @@ def run_main(self, place, with_data_parallel): else: d.append(numpy.array([tmp] * device_cnt)) - outs = exe.run(program=prog, - feed={ - 'd0': d[0], - 'd1': d[1], - 'd2': d[2] - }, - fetch_list=[sum_result]) + outs = exe.run( + program=prog, + feed={'d0': d[0], 'd1': d[1], 'd2': d[2]}, + fetch_list=[sum_result], + ) self.assertAlmostEqual(numpy.sum(d), numpy.sum(outs[0]), delta=0.01) diff --git a/python/paddle/fluid/tests/unittests/test_eager_run_program.py b/python/paddle/fluid/tests/unittests/test_eager_run_program.py index df151f27ec21a..847bdc3d1a8fc 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_run_program.py +++ b/python/paddle/fluid/tests/unittests/test_eager_run_program.py @@ -15,11 +15,18 @@ import paddle import numpy as np from paddle import _legacy_C_ops -from paddle.fluid.framework import _test_eager_guard, Variable, _in_legacy_dygraph +from paddle.fluid.framework import ( + _test_eager_guard, + Variable, + _in_legacy_dygraph, +) from paddle.fluid import core from paddle.fluid.layers.utils import _hash_with_id from paddle.fluid.dygraph.base import switch_to_static_graph -from paddle.fluid.executor import _is_enable_standalone_executor, _is_dy2st_enable_standalone_executor +from paddle.fluid.executor import ( + _is_enable_standalone_executor, + _is_dy2st_enable_standalone_executor, +) import unittest @@ -60,11 +67,21 @@ def _create_out(var): var_desc = var.desc varbase = None if _in_legacy_dygraph(): - var_base = core.VarBase(var_desc.dtype(), var_desc.shape(), - var_desc.name(), var_desc.type(), False) + var_base = core.VarBase( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) else: - var_base = core.eager.Tensor(var_desc.dtype(), var_desc.shape(), - var_desc.name(), var_desc.type(), False) + var_base = core.eager.Tensor( + var_desc.dtype(), + var_desc.shape(), + var_desc.name(), + var_desc.type(), + False, + ) return var_base @@ -72,16 +89,17 @@ def _create_out(var): def _add_build_strategy_for(input_program, start_op_index, end_op_index): compiled_program = paddle.static.CompiledProgram( core.Graph(input_program.desc, start_op_index, end_op_index), - build_strategy=paddle.static.BuildStrategy()) - compiled_program._compile(core.Scope(), - paddle.framework._current_expected_place()) + build_strategy=paddle.static.BuildStrategy(), + ) + compiled_program._compile( + core.Scope(), paddle.framework._current_expected_place() + ) ir_graph = paddle.fluid.framework.IrGraph(compiled_program._graph) builded_program = ir_graph.to_program() return builded_program class TestRunProgram(unittest.TestCase): - def test_eager(self): paddle.set_device('cpu') paddle.enable_static() @@ -95,12 +113,13 @@ def test_eager(self): main_program = paddle.static.default_main_program() program = _append_backward_desc(main_program, [out]) forward_program = _add_build_strategy_for( - program, 0, - main_program.desc.block(0).op_size()) + program, 0, main_program.desc.block(0).op_size() + ) backward_program = _add_build_strategy_for( program, main_program.desc.block(0).op_size() + 2, - program.desc.block(0).op_size()) + program.desc.block(0).op_size(), + ) paddle.disable_static('cpu') # step 2: call run_program in eager mode @@ -120,31 +139,52 @@ def test_eager(self): scope = core.Scope() attrs = [ 'global_block', - program.desc.block(0), 'start_op_index', 0, 'end_op_index', - main_program.desc.block(0).op_size(), 'is_test', False, + program.desc.block(0), + 'start_op_index', + 0, + 'end_op_index', + main_program.desc.block(0).op_size(), + 'is_test', + False, 'program_id', - _hash_with_id(program) + _hash_with_id(program), ] - use_interpretorcore = _is_enable_standalone_executor( - ) and _is_dy2st_enable_standalone_executor() + use_interpretorcore = ( + _is_enable_standalone_executor() + and _is_dy2st_enable_standalone_executor() + ) attrs.extend(('use_interpretorcore', use_interpretorcore)) if use_interpretorcore: attrs.extend( - ('forward_global_block', forward_program.desc.block(0), - 'backward_global_block', backward_program.desc.block(0))) - - _legacy_C_ops.run_program([x_t, y_t], [fake_var], [out_t], [scope], - [fake_var], None, *attrs) + ( + 'forward_global_block', + forward_program.desc.block(0), + 'backward_global_block', + backward_program.desc.block(0), + ) + ) + + _legacy_C_ops.run_program( + [x_t, y_t], + [fake_var], + [out_t], + [scope], + [fake_var], + None, + *attrs + ) loss = paddle.mean(out_t) loss.backward() np.testing.assert_array_equal(np.ones([2, 2]) * 4, out_t.numpy()) np.testing.assert_array_equal( - np.ones([2, 4]) * 0.5, x_t.grad.numpy()) + np.ones([2, 4]) * 0.5, x_t.grad.numpy() + ) np.testing.assert_array_equal( - np.ones([4, 2]) * 0.5, y_t.grad.numpy()) + np.ones([4, 2]) * 0.5, y_t.grad.numpy() + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_eager_trace_op.py b/python/paddle/fluid/tests/unittests/test_eager_trace_op.py index 771395cd2239f..6f93c4bd19dc1 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_trace_op.py +++ b/python/paddle/fluid/tests/unittests/test_eager_trace_op.py @@ -19,26 +19,25 @@ class TestEagerTraceOp(unittest.TestCase): - def test_branches(self): with _test_eager_guard(): data = np.random.random([1, 1]).astype(np.float32) x = paddle.to_tensor(data) paddle.fluid.framework._dygraph_tracer().trace_op( - 'broadcast_tensors', { - 'X': [x, x], - 'Out': [x, x] - }, {'Out': [x, x]}, {}) + 'broadcast_tensors', + {'X': [x, x], 'Out': [x, x]}, + {'Out': [x, x]}, + {}, + ) paddle.fluid.framework._dygraph_tracer().trace_op( - 'scale', {'X': x}, {'Out': x}, {'scale': 0.5}) + 'scale', {'X': x}, {'Out': x}, {'scale': 0.5} + ) scale = paddle.to_tensor(np.random.random([1]).astype(np.float32)) paddle.fluid.framework._dygraph_tracer().trace_op( - 'instance_norm', { - 'Scale': [scale], - 'X': [x] - }, {'Y': [x]}, {}) + 'instance_norm', {'Scale': [scale], 'X': [x]}, {'Y': [x]}, {} + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_edit_distance_op.py b/python/paddle/fluid/tests/unittests/test_edit_distance_op.py index 0a245d851cf03..c737ed2e41ca3 100644 --- a/python/paddle/fluid/tests/unittests/test_edit_distance_op.py +++ b/python/paddle/fluid/tests/unittests/test_edit_distance_op.py @@ -18,23 +18,26 @@ import paddle -def python_edit_distance(input, - label, - input_length=None, - label_length=None, - normalized=True, - ignored_tokens=None): +def python_edit_distance( + input, + label, + input_length=None, + label_length=None, + normalized=True, + ignored_tokens=None, +): return paddle.nn.functional.loss.edit_distance( input, label, normalized=normalized, ignored_tokens=ignored_tokens, input_length=input_length, - label_length=label_length) + label_length=label_length, + ) def Levenshtein(hyp, ref): - """ Compute the Levenshtein distance between two strings. + """Compute the Levenshtein distance between two strings. :param hyp: hypothesis string in index :type hyp: list @@ -65,7 +68,6 @@ def Levenshtein(hyp, ref): class TestEditDistanceOp(OpTest): - def setUp(self): self.op_type = "edit_distance" self.python_api = python_edit_distance @@ -85,8 +87,9 @@ def setUp(self): x2_offset = 0 for i in range(0, num_strs): distance[i] = Levenshtein( - hyp=x1[x1_offset:(x1_offset + self.x1_lod[i])], - ref=x2[x2_offset:(x2_offset + self.x2_lod[i])]) + hyp=x1[x1_offset : (x1_offset + self.x1_lod[i])], + ref=x2[x2_offset : (x2_offset + self.x2_lod[i])], + ) x1_offset += self.x1_lod[i] x2_offset += self.x2_lod[i] if normalized is True: @@ -102,7 +105,6 @@ def test_check_output(self): class TestEditDistanceOpNormalizedCase0(OpTest): - def reset_config(self): pass @@ -130,8 +132,9 @@ def setUp(self): x2_offset = 0 for i in range(0, num_strs): distance[i] = Levenshtein( - hyp=self.x1[x1_offset:(x1_offset + self.x1_lod[i])], - ref=self.x2[x2_offset:(x2_offset + self.x2_lod[i])]) + hyp=self.x1[x1_offset : (x1_offset + self.x1_lod[i])], + ref=self.x2[x2_offset : (x2_offset + self.x2_lod[i])], + ) x1_offset += self.x1_lod[i] x2_offset += self.x2_lod[i] if normalized is True: @@ -141,7 +144,7 @@ def setUp(self): self.attrs = {'normalized': normalized} self.inputs = { 'Hyps': (self.x1, [self.x1_lod]), - 'Refs': (self.x2, [self.x2_lod]) + 'Refs': (self.x2, [self.x2_lod]), } self.outputs = {'Out': distance, 'SequenceNum': sequence_num} @@ -152,21 +155,18 @@ def test_check_output(self): class TestEditDistanceOpNormalizedCase1(TestEditDistanceOpNormalizedCase0): - def reset_config(self): self.x1_lod = [0, 6, 0] self.x2_lod = [2, 1, 2] class TestEditDistanceOpNormalizedCase2(TestEditDistanceOpNormalizedCase0): - def reset_config(self): self.x1_lod = [0, 0, 6] self.x2_lod = [2, 2, 1] class TestEditDistanceOpNormalizedTensor(OpTest): - def reset_config(self): self.x1 = np.array([[10, 3, 0, 0], [6, 5, 8, 2]], dtype=np.int64) self.x2 = np.array([[10, 4, 0], [6, 7, 8]], dtype=np.int64) @@ -185,8 +185,10 @@ def setUp(self): sequence_num = np.array(num_strs).astype("int64") for i in range(0, num_strs): - distance[i] = Levenshtein(hyp=self.x1[i][0:self.x1_lod[i]], - ref=self.x2[i][0:self.x2_lod[i]]) + distance[i] = Levenshtein( + hyp=self.x1[i][0 : self.x1_lod[i]], + ref=self.x2[i][0 : self.x2_lod[i]], + ) if normalized is True: len_ref = self.x2_lod[i] distance[i] = distance[i] / len_ref @@ -196,7 +198,7 @@ def setUp(self): 'Hyps': self.x1, 'Refs': self.x2, 'HypsLength': self.x1_lod, - 'RefsLength': self.x2_lod + 'RefsLength': self.x2_lod, } self.outputs = {'Out': distance, 'SequenceNum': sequence_num} diff --git a/python/paddle/fluid/tests/unittests/test_egr_code_generate_api.py b/python/paddle/fluid/tests/unittests/test_egr_code_generate_api.py index 3ed4962cff66c..aea56537654da 100644 --- a/python/paddle/fluid/tests/unittests/test_egr_code_generate_api.py +++ b/python/paddle/fluid/tests/unittests/test_egr_code_generate_api.py @@ -19,7 +19,6 @@ class EagerOpAPIGenerateTestCase(unittest.TestCase): - def test_elementwise_add(self): with _test_eager_guard(): paddle.set_device("cpu") @@ -35,8 +34,9 @@ def test_elementwise_add(self): def test_sum(self): with _test_eager_guard(): - x_data = np.array([[0.2, 0.3, 0.5, 0.9], [0.1, 0.2, 0.6, - 0.7]]).astype('float32') + x_data = np.array( + [[0.2, 0.3, 0.5, 0.9], [0.1, 0.2, 0.6, 0.7]] + ).astype('float32') x = paddle.to_tensor(x_data, 'float32') out = paddle.sum(x, axis=0) out_arr = out.numpy() @@ -61,8 +61,8 @@ def test_sigmoid(self): out = paddle.nn.functional.sigmoid(x) out_arr = out.numpy() out_arr_expected = np.array( - [0.40131234, 0.450166, 0.52497919, - 0.57444252]).astype('float32') + [0.40131234, 0.450166, 0.52497919, 0.57444252] + ).astype('float32') np.testing.assert_allclose(out_arr, out_arr_expected, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_egr_python_api.py b/python/paddle/fluid/tests/unittests/test_egr_python_api.py index 82881ee0da619..eee9e8eac4b0e 100644 --- a/python/paddle/fluid/tests/unittests/test_egr_python_api.py +++ b/python/paddle/fluid/tests/unittests/test_egr_python_api.py @@ -15,13 +15,18 @@ import paddle.fluid.core as core import paddle import numpy as np -from paddle.fluid.framework import EagerParamBase, _current_expected_place, _disable_legacy_dygraph, _test_eager_guard, in_dygraph_mode +from paddle.fluid.framework import ( + EagerParamBase, + _current_expected_place, + _disable_legacy_dygraph, + _test_eager_guard, + in_dygraph_mode, +) import unittest import copy class EagerScaleTestCase(unittest.TestCase): - def test_scale_base(self): with _test_eager_guard(): paddle.set_device("cpu") @@ -40,8 +45,9 @@ def test_retain_grad_and_run_backward(self): paddle.set_device("cpu") input_data = np.ones([4, 16, 16, 32]).astype('float32') - data_eager = paddle.to_tensor(input_data, 'float32', - core.CPUPlace(), False) + data_eager = paddle.to_tensor( + input_data, 'float32', core.CPUPlace(), False + ) grad_data = np.ones([4, 16, 16, 32]).astype('float32') grad_eager = paddle.to_tensor(grad_data, 'float32', core.CPUPlace()) @@ -59,32 +65,34 @@ def test_retain_grad_and_run_backward_raises(self): paddle.set_device("cpu") input_data = np.ones([4, 16, 16, 32]).astype('float32') - data_eager = paddle.to_tensor(input_data, 'float32', - core.CPUPlace(), False) + data_eager = paddle.to_tensor( + input_data, 'float32', core.CPUPlace(), False + ) grad_data = np.ones([4, 16, 16, 32]).astype('float32') grad_data2 = np.ones([4, 16]).astype('float32') grad_eager = paddle.to_tensor(grad_data, 'float32', core.CPUPlace()) - grad_eager2 = paddle.to_tensor(grad_data2, 'float32', - core.CPUPlace()) + grad_eager2 = paddle.to_tensor( + grad_data2, 'float32', core.CPUPlace() + ) data_eager.retain_grads() out_eager = core.eager.scale(data_eager, 1.0, 0.9, True, True) self.assertIsNone(data_eager.grad) with self.assertRaisesRegexp( - AssertionError, - "The type of grad_tensor must be paddle.Tensor"): + AssertionError, "The type of grad_tensor must be paddle.Tensor" + ): out_eager.backward(grad_data, False) with self.assertRaisesRegexp( - AssertionError, - "Tensor shape not match, Tensor of grad_tensor /*"): + AssertionError, + "Tensor shape not match, Tensor of grad_tensor /*", + ): out_eager.backward(grad_eager2, False) class EagerDtypeTestCase(unittest.TestCase): - def check_to_tesnsor_and_numpy(self, dtype, proto_dtype): with _test_eager_guard(): arr = np.random.random([4, 16, 16, 32]).astype(dtype) @@ -103,14 +111,15 @@ def test_dtype_base(self): self.check_to_tesnsor_and_numpy('float16', core.VarDesc.VarType.FP16) self.check_to_tesnsor_and_numpy('float32', core.VarDesc.VarType.FP32) self.check_to_tesnsor_and_numpy('float64', core.VarDesc.VarType.FP64) - self.check_to_tesnsor_and_numpy('complex64', - core.VarDesc.VarType.COMPLEX64) - self.check_to_tesnsor_and_numpy('complex128', - core.VarDesc.VarType.COMPLEX128) + self.check_to_tesnsor_and_numpy( + 'complex64', core.VarDesc.VarType.COMPLEX64 + ) + self.check_to_tesnsor_and_numpy( + 'complex128', core.VarDesc.VarType.COMPLEX128 + ) class EagerVariablePropertiesAndMethodsTestCase(unittest.TestCase): - def constructor(self, place): egr_tensor = core.eager.Tensor() self.assertEqual(egr_tensor.persistable, False) @@ -119,17 +128,22 @@ def constructor(self, place): self.assertEqual(egr_tensor.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor.stop_gradient, True) - egr_tensor0 = core.eager.Tensor(core.VarDesc.VarType.FP32, - [4, 16, 16, 32], "test_eager_tensor", - core.VarDesc.VarType.LOD_TENSOR, True) + egr_tensor0 = core.eager.Tensor( + core.VarDesc.VarType.FP32, + [4, 16, 16, 32], + "test_eager_tensor", + core.VarDesc.VarType.LOD_TENSOR, + True, + ) self.assertEqual(egr_tensor0.persistable, True) self.assertEqual(egr_tensor0.name, "test_eager_tensor") self.assertEqual(egr_tensor0.shape, [4, 16, 16, 32]) self.assertEqual(egr_tensor0.dtype, core.VarDesc.VarType.FP32) arr0 = np.random.rand(4, 16, 16, 32).astype('float32') - egr_tensor1 = core.eager.Tensor(arr0, place, True, False, - "numpy_tensor1", False) + egr_tensor1 = core.eager.Tensor( + arr0, place, True, False, "numpy_tensor1", False + ) self.assertEqual(egr_tensor1.persistable, True) self.assertEqual(egr_tensor1.name, "numpy_tensor1") self.assertEqual(egr_tensor1.shape, [4, 16, 16, 32]) @@ -139,8 +153,9 @@ def constructor(self, place): np.testing.assert_array_equal(egr_tensor1.numpy(), arr0) arr1 = np.random.randint(100, size=(4, 16, 16, 32), dtype=np.int64) - egr_tensor2 = core.eager.Tensor(arr1, place, False, True, - "numpy_tensor2", True) + egr_tensor2 = core.eager.Tensor( + arr1, place, False, True, "numpy_tensor2", True + ) self.assertEqual(egr_tensor2.persistable, False) self.assertEqual(egr_tensor2.name, "numpy_tensor2") self.assertEqual(egr_tensor2.shape, [4, 16, 16, 32]) @@ -158,7 +173,9 @@ def constructor(self, place): self.assertEqual(egr_tensor3.stop_gradient, True) self.assertTrue( egr_tensor3.place._equals( - paddle.fluid.framework._current_expected_place())) + paddle.fluid.framework._current_expected_place() + ) + ) np.testing.assert_array_equal(egr_tensor3.numpy(), arr2) egr_tensor3.stop_gradient = False @@ -170,7 +187,9 @@ def constructor(self, place): self.assertEqual(egr_tensor4.stop_gradient, True) self.assertTrue( egr_tensor4.place._equals( - paddle.fluid.framework._current_expected_place())) + paddle.fluid.framework._current_expected_place() + ) + ) np.testing.assert_array_equal(egr_tensor4.numpy(), egr_tensor3.numpy()) arr4 = np.random.rand(4, 16, 16, 32).astype('float32') @@ -253,16 +272,18 @@ def constructor(self, place): self.assertEqual(zero_dim_param.shape, []) with self.assertRaisesRegexp( - ValueError, "The shape of Parameter should not be None"): + ValueError, "The shape of Parameter should not be None" + ): eager_param = EagerParamBase(shape=None, dtype="float32") with self.assertRaisesRegexp( - ValueError, "The dtype of Parameter should not be None"): + ValueError, "The dtype of Parameter should not be None" + ): eager_param = EagerParamBase(shape=[1, 1], dtype=None) with self.assertRaisesRegexp( - ValueError, - "Each dimension of shape for Parameter must be greater than 0, but received /*" + ValueError, + "Each dimension of shape for Parameter must be greater than 0, but received /*", ): eager_param = EagerParamBase(shape=[-1], dtype="float32") @@ -271,19 +292,19 @@ def constructor(self, place): eager_param.trainable = False self.assertFalse(eager_param.trainable) with self.assertRaisesRegexp( - ValueError, - "The type of trainable MUST be bool, but the type is /*"): + ValueError, "The type of trainable MUST be bool, but the type is /*" + ): eager_param.trainable = "False" - eager_param_2 = EagerParamBase(shape=paddle.shape( - paddle.to_tensor([1, 2, 3, 4])), - dtype="float32") + eager_param_2 = EagerParamBase( + shape=paddle.shape(paddle.to_tensor([1, 2, 3, 4])), dtype="float32" + ) self.assertTrue(eager_param_2.trainable) eager_param_2.trainable = False self.assertFalse(eager_param_2.trainable) with self.assertRaisesRegexp( - ValueError, - "The type of trainable MUST be bool, but the type is /*"): + ValueError, "The type of trainable MUST be bool, but the type is /*" + ): eager_param_2.trainable = "False" def test_constructor(self): @@ -306,7 +327,9 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor0.shape, [4, 16, 16, 32]) self.assertTrue( egr_tensor0.place._equals( - paddle.fluid.framework._current_expected_place())) + paddle.fluid.framework._current_expected_place() + ) + ) self.assertEqual(egr_tensor0.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor0.stop_gradient, True) @@ -326,9 +349,9 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor2.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor2.stop_gradient, True) - egr_tensor3 = core.eager.Tensor(arr, - place=place, - name="new_eager_tensor") + egr_tensor3 = core.eager.Tensor( + arr, place=place, name="new_eager_tensor" + ) self.assertEqual(egr_tensor3.persistable, False) self.assertTrue("new_eager_tensor" in egr_tensor3.name) self.assertEqual(egr_tensor3.shape, [4, 16, 16, 32]) @@ -336,10 +359,9 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor3.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor3.stop_gradient, True) - egr_tensor4 = core.eager.Tensor(arr, - place=place, - persistable=True, - name="new_eager_tensor") + egr_tensor4 = core.eager.Tensor( + arr, place=place, persistable=True, name="new_eager_tensor" + ) self.assertEqual(egr_tensor4.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor4.name) self.assertEqual(egr_tensor4.shape, [4, 16, 16, 32]) @@ -347,11 +369,13 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor4.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor4.stop_gradient, True) - egr_tensor5 = core.eager.Tensor(arr, - core.CPUPlace(), - persistable=True, - name="new_eager_tensor", - zero_copy=True) + egr_tensor5 = core.eager.Tensor( + arr, + core.CPUPlace(), + persistable=True, + name="new_eager_tensor", + zero_copy=True, + ) self.assertEqual(egr_tensor5.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor5.name) self.assertEqual(egr_tensor5.shape, [4, 16, 16, 32]) @@ -359,11 +383,13 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor5.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor5.stop_gradient, True) - egr_tensor6 = core.eager.Tensor(arr, - place=core.CPUPlace(), - persistable=True, - name="new_eager_tensor", - zero_copy=True) + egr_tensor6 = core.eager.Tensor( + arr, + place=core.CPUPlace(), + persistable=True, + name="new_eager_tensor", + zero_copy=True, + ) self.assertEqual(egr_tensor6.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor6.name) self.assertEqual(egr_tensor6.shape, [4, 16, 16, 32]) @@ -371,11 +397,13 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor6.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor6.stop_gradient, True) - egr_tensor7 = core.eager.Tensor(arr, - place=place, - persistable=True, - name="new_eager_tensor", - zero_copy=True) + egr_tensor7 = core.eager.Tensor( + arr, + place=place, + persistable=True, + name="new_eager_tensor", + zero_copy=True, + ) self.assertEqual(egr_tensor7.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor7.name) self.assertEqual(egr_tensor7.shape, [4, 16, 16, 32]) @@ -383,12 +411,14 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor7.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor7.stop_gradient, True) - egr_tensor8 = core.eager.Tensor(arr, - place=place, - persistable=True, - name="new_eager_tensor", - zero_copy=True, - stop_gradient=False) + egr_tensor8 = core.eager.Tensor( + arr, + place=place, + persistable=True, + name="new_eager_tensor", + zero_copy=True, + stop_gradient=False, + ) self.assertEqual(egr_tensor8.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor8.name) self.assertEqual(egr_tensor8.shape, [4, 16, 16, 32]) @@ -396,12 +426,9 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor8.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor8.stop_gradient, False) - egr_tensor9 = core.eager.Tensor(arr, - place, - True, - True, - "new_eager_tensor", - stop_gradient=False) + egr_tensor9 = core.eager.Tensor( + arr, place, True, True, "new_eager_tensor", stop_gradient=False + ) self.assertEqual(egr_tensor9.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor9.name) self.assertEqual(egr_tensor9.shape, [4, 16, 16, 32]) @@ -409,12 +436,9 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor9.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor9.stop_gradient, False) - egr_tensor10 = core.eager.Tensor(arr, - place, - True, - True, - name="new_eager_tensor", - stop_gradient=False) + egr_tensor10 = core.eager.Tensor( + arr, place, True, True, name="new_eager_tensor", stop_gradient=False + ) self.assertEqual(egr_tensor10.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor10.name) self.assertEqual(egr_tensor10.shape, [4, 16, 16, 32]) @@ -422,12 +446,14 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor10.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor10.stop_gradient, False) - egr_tensor11 = core.eager.Tensor(arr, - place, - True, - zero_copy=True, - name="new_eager_tensor", - stop_gradient=False) + egr_tensor11 = core.eager.Tensor( + arr, + place, + True, + zero_copy=True, + name="new_eager_tensor", + stop_gradient=False, + ) self.assertEqual(egr_tensor11.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor11.name) self.assertEqual(egr_tensor11.shape, [4, 16, 16, 32]) @@ -435,12 +461,14 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor11.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor11.stop_gradient, False) - egr_tensor12 = core.eager.Tensor(arr, - place, - persistable=True, - zero_copy=True, - name="new_eager_tensor", - stop_gradient=False) + egr_tensor12 = core.eager.Tensor( + arr, + place, + persistable=True, + zero_copy=True, + name="new_eager_tensor", + stop_gradient=False, + ) self.assertEqual(egr_tensor12.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor12.name) self.assertEqual(egr_tensor12.shape, [4, 16, 16, 32]) @@ -448,12 +476,14 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor12.dtype, core.VarDesc.VarType.FP32) self.assertEqual(egr_tensor12.stop_gradient, False) - egr_tensor13 = core.eager.Tensor(value=arr, - place=place, - persistable=True, - zero_copy=True, - name="new_eager_tensor", - stop_gradient=False) + egr_tensor13 = core.eager.Tensor( + value=arr, + place=place, + persistable=True, + zero_copy=True, + name="new_eager_tensor", + stop_gradient=False, + ) self.assertEqual(egr_tensor13.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor13.name) self.assertEqual(egr_tensor13.shape, [4, 16, 16, 32]) @@ -462,11 +492,13 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor13.stop_gradient, False) # special case - egr_tensor14 = core.eager.Tensor(dtype=core.VarDesc.VarType.FP32, - dims=[4, 16, 16, 32], - name="special_eager_tensor", - type=core.VarDesc.VarType.LOD_TENSOR, - persistable=True) + egr_tensor14 = core.eager.Tensor( + dtype=core.VarDesc.VarType.FP32, + dims=[4, 16, 16, 32], + name="special_eager_tensor", + type=core.VarDesc.VarType.LOD_TENSOR, + persistable=True, + ) self.assertEqual(egr_tensor14.persistable, True) self.assertEqual(egr_tensor14.name, "special_eager_tensor") self.assertEqual(egr_tensor14.shape, [4, 16, 16, 32]) @@ -481,11 +513,14 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor15.stop_gradient, True) self.assertTrue( egr_tensor15.place._equals( - paddle.fluid.framework._current_expected_place())) + paddle.fluid.framework._current_expected_place() + ) + ) np.testing.assert_array_equal(egr_tensor15.numpy(), egr_tensor4.numpy()) - egr_tensor16 = core.eager.Tensor(value=egr_tensor4, - name="new_eager_tensor") + egr_tensor16 = core.eager.Tensor( + value=egr_tensor4, name="new_eager_tensor" + ) self.assertEqual(egr_tensor16.persistable, True) self.assertTrue("new_eager_tensor" in egr_tensor16.name) self.assertEqual(egr_tensor16.shape, egr_tensor4.shape) @@ -493,7 +528,9 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor16.stop_gradient, True) self.assertTrue( egr_tensor16.place._equals( - paddle.fluid.framework._current_expected_place())) + paddle.fluid.framework._current_expected_place() + ) + ) np.testing.assert_array_equal(egr_tensor16.numpy(), egr_tensor4.numpy()) egr_tensor17 = core.eager.Tensor( @@ -547,7 +584,9 @@ def constructor_with_kwargs(self, place): self.assertEqual(egr_tensor20.stop_gradient, True) self.assertTrue( egr_tensor20.place._equals( - paddle.fluid.framework._current_expected_place())) + paddle.fluid.framework._current_expected_place() + ) + ) np.testing.assert_array_equal(egr_tensor20.numpy(), x) egr_tensor21 = core.eager.Tensor(value=t, place=place) @@ -577,9 +616,9 @@ def constructor_with_kwargs(self, place): self.assertTrue(egr_tensor23.place._equals(place)) np.testing.assert_array_equal(egr_tensor23.numpy(), x) - egr_tensor24 = core.eager.Tensor(value=t, - place=place, - name="from_framework_tensor") + egr_tensor24 = core.eager.Tensor( + value=t, place=place, name="from_framework_tensor" + ) self.assertEqual(egr_tensor24.persistable, False) self.assertTrue("from_framework_tensor" in egr_tensor24.name) self.assertEqual(egr_tensor24.shape, [3, 3]) @@ -609,15 +648,18 @@ def test_copy_and_copy_to(self): arr = np.ones([4, 16, 16, 32]).astype('float32') arr1 = np.zeros([4, 16]).astype('float32') arr2 = np.ones([4, 16, 16, 32]).astype('float32') + np.ones( - [4, 16, 16, 32]).astype('float32') - tensor = paddle.to_tensor(arr, core.VarDesc.VarType.FP32, - core.CPUPlace()) + [4, 16, 16, 32] + ).astype('float32') + tensor = paddle.to_tensor( + arr, core.VarDesc.VarType.FP32, core.CPUPlace() + ) self.assertEqual(tensor.stop_gradient, True) tensor.stop_gradient = False print("Set persistable") tensor.persistable = False - tensor1 = paddle.to_tensor(arr1, core.VarDesc.VarType.FP32, - core.CPUPlace()) + tensor1 = paddle.to_tensor( + arr1, core.VarDesc.VarType.FP32, core.CPUPlace() + ) tensor1.persistable = True self.assertEqual(tensor1.stop_gradient, True) np.testing.assert_array_equal(tensor.numpy(), arr) @@ -629,8 +671,9 @@ def test_copy_and_copy_to(self): np.testing.assert_array_equal(tensor.numpy(), arr1) print("Test _copy_to") - tensor2 = paddle.to_tensor(arr2, core.VarDesc.VarType.FP32, - core.CPUPlace()) + tensor2 = paddle.to_tensor( + arr2, core.VarDesc.VarType.FP32, core.CPUPlace() + ) np.testing.assert_array_equal(tensor2.numpy(), arr2) self.assertTrue(tensor2.place.is_cpu_place()) tensor2.persistable = True @@ -656,8 +699,9 @@ def test_copy_and_copy_to(self): tensor10 = paddle.to_tensor([1, 2, 3], place='gpu_pinned') tensor11 = tensor10._copy_to(core.CUDAPlace(0), True) - np.testing.assert_array_equal(tensor10.numpy(), - tensor11.numpy()) + np.testing.assert_array_equal( + tensor10.numpy(), tensor11.numpy() + ) else: tensor3 = tensor2._copy_to(core.CPUPlace(), True) np.testing.assert_array_equal(tensor3.numpy(), arr2) @@ -676,18 +720,22 @@ def test_share_buffer_to(self): arr = np.ones([4, 16, 16, 32]).astype('float32') arr1 = np.zeros([4, 16]).astype('float32') arr2 = np.ones([4, 16, 16, 32]).astype('float32') + np.ones( - [4, 16, 16, 32]).astype('float32') + [4, 16, 16, 32] + ).astype('float32') tensor = None tensor2 = None - tensor = paddle.to_tensor(arr, core.VarDesc.VarType.FP32, - core.CPUPlace()) + tensor = paddle.to_tensor( + arr, core.VarDesc.VarType.FP32, core.CPUPlace() + ) tensor3 = core.eager.Tensor(value=tensor, place=core.CPUPlace()) if core.is_compiled_with_cuda(): - tensor2 = paddle.to_tensor(arr2, core.VarDesc.VarType.FP32, - core.CUDAPlace(0)) + tensor2 = paddle.to_tensor( + arr2, core.VarDesc.VarType.FP32, core.CUDAPlace(0) + ) else: - tensor2 = paddle.to_tensor(arr2, core.VarDesc.VarType.FP32, - core.CPUPlace()) + tensor2 = paddle.to_tensor( + arr2, core.VarDesc.VarType.FP32, core.CPUPlace() + ) np.testing.assert_array_equal(tensor.numpy(), arr) np.testing.assert_array_equal(tensor2.numpy(), arr2) tensor2._share_buffer_to(tensor) @@ -704,18 +752,22 @@ def test_share_underline_tensor_to(self): arr = np.ones([4, 16, 16, 32]).astype('float32') arr1 = np.zeros([4, 16]).astype('float32') arr2 = np.ones([4, 16, 16, 32]).astype('float32') + np.ones( - [4, 16, 16, 32]).astype('float32') + [4, 16, 16, 32] + ).astype('float32') tensor = None tensor2 = None - tensor = paddle.to_tensor(arr, core.VarDesc.VarType.FP32, - core.CPUPlace()) + tensor = paddle.to_tensor( + arr, core.VarDesc.VarType.FP32, core.CPUPlace() + ) tensor3 = core.eager.Tensor() if core.is_compiled_with_cuda(): - tensor2 = paddle.to_tensor(arr2, core.VarDesc.VarType.FP32, - core.CUDAPlace(0)) + tensor2 = paddle.to_tensor( + arr2, core.VarDesc.VarType.FP32, core.CUDAPlace(0) + ) else: - tensor2 = paddle.to_tensor(arr2, core.VarDesc.VarType.FP32, - core.CPUPlace()) + tensor2 = paddle.to_tensor( + arr2, core.VarDesc.VarType.FP32, core.CPUPlace() + ) np.testing.assert_array_equal(tensor.numpy(), arr) np.testing.assert_array_equal(tensor2.numpy(), arr2) tensor2._share_underline_tensor_to(tensor) @@ -732,8 +784,9 @@ def test_properties(self): with _test_eager_guard(): paddle.set_device("cpu") arr = np.ones([4, 16, 16, 32]).astype('float32') - tensor = paddle.to_tensor(arr, core.VarDesc.VarType.FP32, - core.CPUPlace()) + tensor = paddle.to_tensor( + arr, core.VarDesc.VarType.FP32, core.CPUPlace() + ) self.assertEqual(tensor.shape, [4, 16, 16, 32]) tensor.name = 'tensor_name_test' self.assertEqual(tensor.name, 'tensor_name_test') @@ -764,14 +817,14 @@ def test_place_guard(self): paddle.set_device("gpu:0") with paddle.fluid.framework._dygraph_place_guard(core.CPUPlace()): self.assertTrue( - isinstance(_current_expected_place(), - type(core.CPUPlace()))) + isinstance(_current_expected_place(), type(core.CPUPlace())) + ) else: paddle.set_device("cpu") with paddle.fluid.framework._dygraph_place_guard(core.CPUPlace()): self.assertTrue( - isinstance(_current_expected_place(), - type(core.CPUPlace()))) + isinstance(_current_expected_place(), type(core.CPUPlace())) + ) def test_value(self): with _test_eager_guard(): @@ -783,13 +836,19 @@ def test_value(self): self.assertEqual(egr_tensor0.shape, [4, 16, 16, 32]) self.assertTrue( egr_tensor0.place._equals( - paddle.fluid.framework._current_expected_place())) + paddle.fluid.framework._current_expected_place() + ) + ) self.assertEqual(egr_tensor0.dtype, core.VarDesc.VarType.FP64) self.assertEqual(egr_tensor0.stop_gradient, True) - self.assertTrue(egr_tensor0.value().get_tensor()._dtype(), - core.VarDesc.VarType.FP64) - self.assertTrue(egr_tensor0.value().get_tensor()._place(), - paddle.fluid.framework._current_expected_place()) + self.assertTrue( + egr_tensor0.value().get_tensor()._dtype(), + core.VarDesc.VarType.FP64, + ) + self.assertTrue( + egr_tensor0.value().get_tensor()._place(), + paddle.fluid.framework._current_expected_place(), + ) self.assertTrue(egr_tensor0.value().get_tensor()._is_initialized()) def test_set_value(self): @@ -813,8 +872,9 @@ def test_set_value(self): def test_sharding_related_api(self): with _test_eager_guard(): arr0 = np.random.rand(4, 16, 16, 32).astype('float32') - egr_tensor1 = core.eager.Tensor(arr0, core.CPUPlace(), True, False, - "numpy_tensor1", False) + egr_tensor1 = core.eager.Tensor( + arr0, core.CPUPlace(), True, False, "numpy_tensor1", False + ) self.assertEqual(egr_tensor1._numel(), 32768) self.assertEqual(egr_tensor1._slice(0, 2)._numel(), 16384) @@ -839,7 +899,6 @@ def test_clear(self): class EagerParamBaseUsageTestCase(unittest.TestCase): - def test_print(self): with _test_eager_guard(): linear = paddle.nn.Linear(3, 3, bias_attr=False) @@ -850,10 +909,12 @@ def test_copy(self): linear = paddle.nn.Linear(1, 3) linear_copy = copy.deepcopy(linear) linear_copy2 = linear.weight._copy_to(core.CPUPlace(), True) - np.testing.assert_array_equal(linear.weight.numpy(), - linear_copy.weight.numpy()) - np.testing.assert_array_equal(linear.weight.numpy(), - linear_copy2.numpy()) + np.testing.assert_array_equal( + linear.weight.numpy(), linear_copy.weight.numpy() + ) + np.testing.assert_array_equal( + linear.weight.numpy(), linear_copy2.numpy() + ) def func_fp16_initilaizer(self): paddle.set_default_dtype("float16") @@ -862,22 +923,25 @@ def func_fp16_initilaizer(self): 1, 3, bias_attr=False, - weight_attr=paddle.fluid.initializer.Uniform()) + weight_attr=paddle.fluid.initializer.Uniform(), + ) linear3 = paddle.nn.Linear( 1, 3, bias_attr=False, - weight_attr=paddle.fluid.initializer.TruncatedNormalInitializer()) + weight_attr=paddle.fluid.initializer.TruncatedNormalInitializer(), + ) linear4 = paddle.nn.Linear( 1, 3, bias_attr=False, - weight_attr=paddle.fluid.initializer.MSRAInitializer()) + weight_attr=paddle.fluid.initializer.MSRAInitializer(), + ) res = [ linear1.weight.numpy(), linear2.weight.numpy(), linear3.weight.numpy(), - linear4.weight.numpy() + linear4.weight.numpy(), ] paddle.set_default_dtype("float32") return res @@ -896,7 +960,8 @@ def test_fp16_initializer(self): def func_layer_helper_base(self, value): base = paddle.fluid.layer_helper_base.LayerHelperBase( - "test_layer", "test_layer") + "test_layer", "test_layer" + ) return base.to_variable(value).numpy() def func_base_to_variable(self, value): @@ -945,7 +1010,6 @@ def test_set_value(self): class EagerGuardTestCase(unittest.TestCase): - def test__test_eager_guard(self): tracer = paddle.fluid.dygraph.tracer.Tracer() with _test_eager_guard(tracer): diff --git a/python/paddle/fluid/tests/unittests/test_egr_string_tensor_api.py b/python/paddle/fluid/tests/unittests/test_egr_string_tensor_api.py index e0a1058e61894..b25e40149bac7 100644 --- a/python/paddle/fluid/tests/unittests/test_egr_string_tensor_api.py +++ b/python/paddle/fluid/tests/unittests/test_egr_string_tensor_api.py @@ -19,13 +19,15 @@ class EagerStringTensorTestCase(unittest.TestCase): - def setUp(self): - self.str_arr = np.array([ - ["15.4寸笔记本的键盘确实爽，基本跟台式机差不多了，蛮喜欢数字小键盘，输数字特方便，样子也很美观，做工也相当不错" - ], # From ChnSentiCorp - ["One of the very best Three Stooges shorts ever."] - ]) # From IMDB + self.str_arr = np.array( + [ + [ + "15.4寸笔记本的键盘确实爽，基本跟台式机差不多了，蛮喜欢数字小键盘，输数字特方便，样子也很美观，做工也相当不错" + ], # From ChnSentiCorp + ["One of the very best Three Stooges shorts ever."], + ] + ) # From IMDB def test_constructor_with_args(self): with _test_eager_guard(): @@ -38,8 +40,9 @@ def test_constructor_with_args(self): ST2 = core.eager.StringTensor(shape, "ST2") # constructor 2 self.assertEqual(ST2.name, "ST2") self.assertEqual(ST2.shape, shape) - np.testing.assert_array_equal(ST2.numpy(), - np.empty(shape, dtype=np.unicode_)) + np.testing.assert_array_equal( + ST2.numpy(), np.empty(shape, dtype=np.unicode_) + ) ST3 = core.eager.StringTensor(self.str_arr, "ST3") # constructor 3 self.assertEqual(ST3.name, "ST3") @@ -68,15 +71,18 @@ def test_constructor_with_args(self): def test_constructor_with_kwargs(self): with _test_eager_guard(): shape = [2, 3] - ST1 = core.eager.StringTensor(dims=shape, - name="ST1") # constructor 2 + ST1 = core.eager.StringTensor( + dims=shape, name="ST1" + ) # constructor 2 self.assertEqual(ST1.name, "ST1") self.assertEqual(ST1.shape, shape) - np.testing.assert_array_equal(ST1.numpy(), - np.empty(shape, dtype=np.unicode_)) + np.testing.assert_array_equal( + ST1.numpy(), np.empty(shape, dtype=np.unicode_) + ) - ST2 = core.eager.StringTensor(self.str_arr, - name="ST2") # constructor 3 + ST2 = core.eager.StringTensor( + self.str_arr, name="ST2" + ) # constructor 3 self.assertEqual(ST2.name, "ST2") self.assertEqual(ST2.shape, list(self.str_arr.shape)) np.testing.assert_array_equal(ST2.numpy(), self.str_arr) @@ -86,8 +92,9 @@ def test_constructor_with_kwargs(self): self.assertEqual(ST3.shape, list(self.str_arr.shape)) np.testing.assert_array_equal(ST3.numpy(), self.str_arr) - ST4 = core.eager.StringTensor(value=ST2, - name="ST4") # constructor 6 + ST4 = core.eager.StringTensor( + value=ST2, name="ST4" + ) # constructor 6 self.assertEqual(ST4.name, "ST4") self.assertEqual(ST4.shape, list(self.str_arr.shape)) np.testing.assert_array_equal(ST4.numpy(), self.str_arr) diff --git a/python/paddle/fluid/tests/unittests/test_eig_op.py b/python/paddle/fluid/tests/unittests/test_eig_op.py index d8af7048c2a64..338eb3512e57a 100644 --- a/python/paddle/fluid/tests/unittests/test_eig_op.py +++ b/python/paddle/fluid/tests/unittests/test_eig_op.py @@ -22,9 +22,9 @@ # cast output to complex for numpy.linalg.eig def cast_to_complex(input, output): - if (input.dtype == np.float32): + if input.dtype == np.float32: output = output.astype(np.complex64) - elif (input.dtype == np.float64): + elif input.dtype == np.float64: output = output.astype(np.complex128) return output @@ -57,7 +57,6 @@ def eig_backward(w, v, grad_w, grad_v): class TestEigOp(OpTest): - def setUp(self): paddle.enable_static() paddle.device.set_device("cpu") @@ -72,8 +71,10 @@ def init_input(self): self.set_dims() self.x = np.random.random(self.shape).astype(self.dtype) self.out = np.linalg.eig(self.x) - self.out = (cast_to_complex(self.x, self.out[0]), - cast_to_complex(self.x, self.out[1])) + self.out = ( + cast_to_complex(self.x, self.out[0]), + cast_to_complex(self.x, self.out[1]), + ) # for the real input, a customized checker is needed def checker(self, outs): @@ -84,13 +85,17 @@ def checker(self, outs): length_w = len(expect_out_w) act_w_real = np.sort( - np.array([np.abs(actual_out_w[i].real) for i in range(length_w)])) + np.array([np.abs(actual_out_w[i].real) for i in range(length_w)]) + ) act_w_imag = np.sort( - np.array([np.abs(actual_out_w[i].imag) for i in range(length_w)])) + np.array([np.abs(actual_out_w[i].imag) for i in range(length_w)]) + ) exp_w_real = np.sort( - np.array([np.abs(expect_out_w[i].real) for i in range(length_w)])) + np.array([np.abs(expect_out_w[i].real) for i in range(length_w)]) + ) exp_w_imag = np.sort( - np.array([np.abs(expect_out_w[i].imag) for i in range(length_w)])) + np.array([np.abs(expect_out_w[i].imag) for i in range(length_w)]) + ) for i in range(length_w): np.testing.assert_allclose( @@ -98,25 +103,37 @@ def checker(self, outs): exp_w_real[i], rtol=1e-06, atol=1e-05, - err_msg='The eigenvalues real part have diff: \nExpected ' + - str(act_w_real[i]) + '\n' + 'But got: ' + str(exp_w_real[i])) + err_msg='The eigenvalues real part have diff: \nExpected ' + + str(act_w_real[i]) + + '\n' + + 'But got: ' + + str(exp_w_real[i]), + ) np.testing.assert_allclose( act_w_imag[i], exp_w_imag[i], rtol=1e-06, atol=1e-05, - err_msg='The eigenvalues image part have diff: \nExpected ' + - str(act_w_imag[i]) + '\n' + 'But got: ' + str(exp_w_imag[i])) + err_msg='The eigenvalues image part have diff: \nExpected ' + + str(act_w_imag[i]) + + '\n' + + 'But got: ' + + str(exp_w_imag[i]), + ) length_v = len(expect_out_v) act_v_real = np.sort( - np.array([np.abs(actual_out_v[i].real) for i in range(length_v)])) + np.array([np.abs(actual_out_v[i].real) for i in range(length_v)]) + ) act_v_imag = np.sort( - np.array([np.abs(actual_out_v[i].imag) for i in range(length_v)])) + np.array([np.abs(actual_out_v[i].imag) for i in range(length_v)]) + ) exp_v_real = np.sort( - np.array([np.abs(expect_out_v[i].real) for i in range(length_v)])) + np.array([np.abs(expect_out_v[i].real) for i in range(length_v)]) + ) exp_v_imag = np.sort( - np.array([np.abs(expect_out_v[i].imag) for i in range(length_v)])) + np.array([np.abs(expect_out_v[i].imag) for i in range(length_v)]) + ) for i in range(length_v): np.testing.assert_allclose( @@ -124,15 +141,23 @@ def checker(self, outs): exp_v_real[i], rtol=1e-06, atol=1e-05, - err_msg='The eigenvectors real part have diff: \nExpected ' + - str(act_v_real[i]) + '\n' + 'But got: ' + str(exp_v_real[i])) + err_msg='The eigenvectors real part have diff: \nExpected ' + + str(act_v_real[i]) + + '\n' + + 'But got: ' + + str(exp_v_real[i]), + ) np.testing.assert_allclose( act_v_imag[i], exp_v_imag[i], rtol=1e-06, atol=1e-05, - err_msg='The eigenvectors image part have diff: \nExpected ' + - str(act_v_imag[i]) + '\n' + 'But got: ' + str(exp_v_imag[i])) + err_msg='The eigenvectors image part have diff: \nExpected ' + + str(act_v_imag[i]) + + '\n' + + 'But got: ' + + str(exp_v_imag[i]), + ) def set_dtype(self): self.dtype = np.complex64 @@ -149,32 +174,34 @@ def init_grad(self): gtype = np.complex128 self.grad_w = np.ones(self.out[0].shape, gtype) self.grad_v = np.ones(self.out[1].shape, gtype) - self.grad_x = eig_backward(self.out[0], self.out[1], self.grad_w, - self.grad_v) + self.grad_x = eig_backward( + self.out[0], self.out[1], self.grad_w, self.grad_v + ) def test_check_output(self): - self.check_output_with_place_customized(checker=self.checker, - place=core.CPUPlace()) + self.check_output_with_place_customized( + checker=self.checker, place=core.CPUPlace() + ) def test_check_grad(self): self.init_grad() - self.check_grad(['X'], ['Eigenvalues', 'Eigenvectors'], - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_w, self.grad_v]) + self.check_grad( + ['X'], + ['Eigenvalues', 'Eigenvectors'], + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_w, self.grad_v], + ) class TestComplex128(TestEigOp): - def set_dtype(self): self.dtype = np.complex128 @skip_check_grad_ci( - reason= - "For float dtype, numpy.linalg.eig forward outputs real or complex when input is real, therefore the grad computation may be not the same with paddle.linalg.eig" + reason="For float dtype, numpy.linalg.eig forward outputs real or complex when input is real, therefore the grad computation may be not the same with paddle.linalg.eig" ) class TestDouble(TestEigOp): - def set_dtype(self): self.dtype = np.float64 @@ -183,11 +210,9 @@ def test_check_grad(self): @skip_check_grad_ci( - reason= - "For float dtype, numpy.linalg.eig forward outputs real or complex when input is real, therefore the grad computation may be not the same with paddle.linalg.eig" + reason="For float dtype, numpy.linalg.eig forward outputs real or complex when input is real, therefore the grad computation may be not the same with paddle.linalg.eig" ) class TestEigBatchMarices(TestEigOp): - def set_dtype(self): self.dtype = np.float64 @@ -199,11 +224,9 @@ def test_check_grad(self): @skip_check_grad_ci( - reason= - "For float dtype, numpy.linalg.eig forward outputs real or complex when input is real, therefore the grad computation may be not the same with paddle.linalg.eig" + reason="For float dtype, numpy.linalg.eig forward outputs real or complex when input is real, therefore the grad computation may be not the same with paddle.linalg.eig" ) class TestFloat(TestEigOp): - def set_dtype(self): self.dtype = np.float32 @@ -212,7 +235,6 @@ def test_check_grad(self): class TestEigStatic(TestEigOp): - def test_check_output_with_place(self): paddle.enable_static() place = core.CPUPlace() @@ -223,28 +245,36 @@ def test_check_output_with_place(self): act_val, act_vec = paddle.linalg.eig(input) exe = fluid.Executor(place) - fetch_val, fetch_vec = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[act_val, act_vec]) + fetch_val, fetch_vec = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[act_val, act_vec], + ) np.testing.assert_allclose( expect_val, fetch_val, rtol=1e-06, atol=1e-06, - err_msg='The eigen values have diff: \nExpected ' + - str(expect_val) + '\n' + 'But got: ' + str(fetch_val)) + err_msg='The eigen values have diff: \nExpected ' + + str(expect_val) + + '\n' + + 'But got: ' + + str(fetch_val), + ) np.testing.assert_allclose( np.abs(expect_vec), np.abs(fetch_vec), rtol=1e-06, atol=1e-06, - err_msg='The eigen vectors have diff: \nExpected ' + - str(np.abs(expect_vec)) + '\n' + 'But got: ' + - str(np.abs(fetch_vec))) + err_msg='The eigen vectors have diff: \nExpected ' + + str(np.abs(expect_vec)) + + '\n' + + 'But got: ' + + str(np.abs(fetch_vec)), + ) class TestEigDyGraph(unittest.TestCase): - def test_check_output_with_place(self): input_np = np.random.random([3, 3]).astype('complex') expect_val, expect_vec = np.linalg.eig(input_np) @@ -260,16 +290,23 @@ def test_check_output_with_place(self): fetch_val.numpy(), rtol=1e-06, atol=1e-06, - err_msg='The eigen values have diff: \nExpected ' + - str(expect_val) + '\n' + 'But got: ' + str(fetch_val)) + err_msg='The eigen values have diff: \nExpected ' + + str(expect_val) + + '\n' + + 'But got: ' + + str(fetch_val), + ) np.testing.assert_allclose( np.abs(expect_vec), np.abs(fetch_vec.numpy()), rtol=1e-06, atol=1e-06, - err_msg='The eigen vectors have diff: \nExpected ' + - str(np.abs(expect_vec)) + '\n' + 'But got: ' + - str(np.abs(fetch_vec.numpy()))) + err_msg='The eigen vectors have diff: \nExpected ' + + str(np.abs(expect_vec)) + + '\n' + + 'But got: ' + + str(np.abs(fetch_vec.numpy())), + ) def test_check_grad(self): test_shape = [3, 3] @@ -289,17 +326,20 @@ def test_check_grad(self): w, v = paddle.linalg.eig(x) (w.sum() + v.sum()).backward() - np.testing.assert_allclose(np.abs(x.grad.numpy()), - np.abs(grad_x), - rtol=1e-05, - atol=1e-05, - err_msg='The grad x have diff: \nExpected ' + - str(np.abs(grad_x)) + '\n' + 'But got: ' + - str(np.abs(x.grad.numpy()))) + np.testing.assert_allclose( + np.abs(x.grad.numpy()), + np.abs(grad_x), + rtol=1e-05, + atol=1e-05, + err_msg='The grad x have diff: \nExpected ' + + str(np.abs(grad_x)) + + '\n' + + 'But got: ' + + str(np.abs(x.grad.numpy())), + ) class TestEigWrongDimsError(unittest.TestCase): - def test_error(self): paddle.device.set_device("cpu") paddle.disable_static() @@ -309,7 +349,6 @@ def test_error(self): class TestEigNotSquareError(unittest.TestCase): - def test_error(self): paddle.device.set_device("cpu") paddle.disable_static() @@ -319,7 +358,6 @@ def test_error(self): class TestEigUnsupportedDtypeError(unittest.TestCase): - def test_error(self): paddle.device.set_device("cpu") paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_eigh_op.py b/python/paddle/fluid/tests/unittests/test_eigh_op.py index 6118b761283a8..928234a138f13 100644 --- a/python/paddle/fluid/tests/unittests/test_eigh_op.py +++ b/python/paddle/fluid/tests/unittests/test_eigh_op.py @@ -57,8 +57,8 @@ def valid_single_eigh_result(A, eigh_value, eigh_vector, uplo): # ||A - Q*T*Q'|| / (N*||A||) < rtol np.testing.assert_array_less( - np.linalg.norm(residual, np.inf) / (N * np.linalg.norm(A, np.inf)), - rtol) + np.linalg.norm(residual, np.inf) / (N * np.linalg.norm(A, np.inf)), rtol + ) # ||I - Q*Q'|| / M < rtol residual = np.eye(M) - eigh_vector @ np.linalg.inv(eigh_vector) @@ -66,7 +66,6 @@ def valid_single_eigh_result(A, eigh_value, eigh_vector, uplo): class TestEighOp(OpTest): - def setUp(self): paddle.enable_static() self.op_type = "eigh" @@ -94,13 +93,11 @@ def test_grad(self): class TestEighUPLOCase(TestEighOp): - def init_config(self): self.UPLO = 'U' class TestEighGPUCase(unittest.TestCase): - def setUp(self): self.x_shape = [32, 32] self.dtype = "float32" @@ -113,19 +110,22 @@ def test_check_output_gpu(self): paddle.disable_static(place=paddle.CUDAPlace(0)) input_real_data = paddle.to_tensor(self.x_np) actual_w, actual_v = paddle.linalg.eigh(input_real_data, self.UPLO) - valid_eigh_result(self.x_np, actual_w.numpy(), actual_v.numpy(), - self.UPLO) + valid_eigh_result( + self.x_np, actual_w.numpy(), actual_v.numpy(), self.UPLO + ) class TestEighAPI(unittest.TestCase): - def setUp(self): self.init_input_data() self.UPLO = 'L' self.rtol = 1e-5 # for test_eigh_grad self.atol = 1e-5 # for test_eigh_grad - self.place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) np.random.seed(123) def init_input_shape(self): @@ -136,26 +136,31 @@ def init_input_data(self): self.dtype = "float32" self.real_data = np.random.random(self.x_shape).astype(self.dtype) complex_data = np.random.random(self.x_shape).astype( - self.dtype) + 1J * np.random.random(self.x_shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.x_shape).astype(self.dtype) self.trans_dims = list(range(len(self.x_shape) - 2)) + [ - len(self.x_shape) - 1, len(self.x_shape) - 2 + len(self.x_shape) - 1, + len(self.x_shape) - 2, ] - #build a random conjugate matrix + # build a random conjugate matrix self.complex_symm = np.divide( - complex_data + np.conj(complex_data.transpose(self.trans_dims)), 2) + complex_data + np.conj(complex_data.transpose(self.trans_dims)), 2 + ) def check_static_float_result(self): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, startup_prog): - input_x = paddle.static.data('input_x', - shape=self.x_shape, - dtype=self.dtype) + input_x = paddle.static.data( + 'input_x', shape=self.x_shape, dtype=self.dtype + ) output_w, output_v = paddle.linalg.eigh(input_x) exe = paddle.static.Executor(self.place) - actual_w, actual_v = exe.run(main_prog, - feed={"input_x": self.real_data}, - fetch_list=[output_w, output_v]) + actual_w, actual_v = exe.run( + main_prog, + feed={"input_x": self.real_data}, + fetch_list=[output_w, output_v], + ) valid_eigh_result(self.real_data, actual_w, actual_v, self.UPLO) def check_static_complex_result(self): @@ -163,14 +168,16 @@ def check_static_complex_result(self): startup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, startup_prog): x_dtype = np.complex64 if self.dtype == "float32" else np.complex128 - input_x = paddle.static.data('input_x', - shape=self.x_shape, - dtype=x_dtype) + input_x = paddle.static.data( + 'input_x', shape=self.x_shape, dtype=x_dtype + ) output_w, output_v = paddle.linalg.eigh(input_x) exe = paddle.static.Executor(self.place) - actual_w, actual_v = exe.run(main_prog, - feed={"input_x": self.complex_symm}, - fetch_list=[output_w, output_v]) + actual_w, actual_v = exe.run( + main_prog, + feed={"input_x": self.complex_symm}, + fetch_list=[output_w, output_v], + ) valid_eigh_result(self.complex_symm, actual_w, actual_v, self.UPLO) def test_in_static_mode(self): @@ -182,61 +189,62 @@ def test_in_dynamic_mode(self): paddle.disable_static() input_real_data = paddle.to_tensor(self.real_data) actual_w, actual_v = paddle.linalg.eigh(input_real_data) - valid_eigh_result(self.real_data, actual_w.numpy(), actual_v.numpy(), - self.UPLO) + valid_eigh_result( + self.real_data, actual_w.numpy(), actual_v.numpy(), self.UPLO + ) input_complex_data = paddle.to_tensor(self.complex_symm) actual_w, actual_v = paddle.linalg.eigh(input_complex_data) - valid_eigh_result(self.complex_symm, actual_w.numpy(), actual_v.numpy(), - self.UPLO) + valid_eigh_result( + self.complex_symm, actual_w.numpy(), actual_v.numpy(), self.UPLO + ) def test_eigh_grad(self): paddle.disable_static() x = paddle.to_tensor(self.complex_symm, stop_gradient=False) w, v = paddle.linalg.eigh(x) (w.sum() + paddle.abs(v).sum()).backward() - np.testing.assert_allclose(abs(x.grad.numpy()), - abs(x.grad.numpy().conj().transpose( - self.trans_dims)), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + abs(x.grad.numpy()), + abs(x.grad.numpy().conj().transpose(self.trans_dims)), + rtol=self.rtol, + atol=self.atol, + ) class TestEighBatchAPI(TestEighAPI): - def init_input_shape(self): self.x_shape = [2, 5, 5] class TestEighAPIError(unittest.TestCase): - def test_error(self): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, startup_prog): - #input maxtrix must greater than 2 dimensions - input_x = paddle.static.data(name='x_1', - shape=[12], - dtype='float32') + # input maxtrix must greater than 2 dimensions + input_x = paddle.static.data( + name='x_1', shape=[12], dtype='float32' + ) self.assertRaises(ValueError, paddle.linalg.eigh, input_x) - #input matrix must be square matrix - input_x = paddle.static.data(name='x_2', - shape=[12, 32], - dtype='float32') + # input matrix must be square matrix + input_x = paddle.static.data( + name='x_2', shape=[12, 32], dtype='float32' + ) self.assertRaises(ValueError, paddle.linalg.eigh, input_x) - #uplo must be in 'L' or 'U' - input_x = paddle.static.data(name='x_3', - shape=[4, 4], - dtype="float32") + # uplo must be in 'L' or 'U' + input_x = paddle.static.data( + name='x_3', shape=[4, 4], dtype="float32" + ) uplo = 'R' self.assertRaises(ValueError, paddle.linalg.eigh, input_x, uplo) - #x_data cannot be integer - input_x = paddle.static.data(name='x_4', - shape=[4, 4], - dtype="int32") + # x_data cannot be integer + input_x = paddle.static.data( + name='x_4', shape=[4, 4], dtype="int32" + ) self.assertRaises(TypeError, paddle.linalg.eigh, input_x) diff --git a/python/paddle/fluid/tests/unittests/test_eigvals_op.py b/python/paddle/fluid/tests/unittests/test_eigvals_op.py index a361ad2402459..df8d6b001cffb 100644 --- a/python/paddle/fluid/tests/unittests/test_eigvals_op.py +++ b/python/paddle/fluid/tests/unittests/test_eigvals_op.py @@ -23,7 +23,7 @@ def np_eigvals(a): res = np.linalg.eigvals(a) - if (a.dtype == np.float32 or a.dtype == np.complex64): + if a.dtype == np.float32 or a.dtype == np.complex64: res = res.astype(np.complex64) else: res = res.astype(np.complex128) @@ -32,7 +32,6 @@ def np_eigvals(a): class TestEigvalsOp(OpTest): - def setUp(self): np.random.seed(0) paddle.enable_static() @@ -53,111 +52,121 @@ def set_input_dims(self): self.input_dims = (5, 5) def set_input_data(self): - if (self.dtype == np.float32 or self.dtype == np.float64): + if self.dtype == np.float32 or self.dtype == np.float64: self.input_data = np.random.random(self.input_dims).astype( - self.dtype) + self.dtype + ) else: - self.input_data = (np.random.random(self.input_dims) + - np.random.random(self.input_dims) * 1j).astype( - self.dtype) + self.input_data = ( + np.random.random(self.input_dims) + + np.random.random(self.input_dims) * 1j + ).astype(self.dtype) def test_check_output(self): self.__class__.no_need_check_grad = True - self.check_output_with_place_customized(checker=self.verify_output, - place=core.CPUPlace()) + self.check_output_with_place_customized( + checker=self.verify_output, place=core.CPUPlace() + ) def verify_output(self, outs): actual_outs = np.sort(np.array(outs[0])) expect_outs = np.sort(np.array(self.outputs['Out'])) self.assertTrue( - actual_outs.shape == expect_outs.shape, "Output shape has diff.\n" - "Expect shape " + str(expect_outs.shape) + "\n" + "But Got" + - str(actual_outs.shape) + " in class " + self.__class__.__name__) + actual_outs.shape == expect_outs.shape, + "Output shape has diff.\n" + "Expect shape " + + str(expect_outs.shape) + + "\n" + + "But Got" + + str(actual_outs.shape) + + " in class " + + self.__class__.__name__, + ) n_dim = actual_outs.shape[-1] - for actual_row, expect_row in zip(actual_outs.reshape((-1, n_dim)), - expect_outs.reshape((-1, n_dim))): - is_mapped_index = np.zeros((n_dim, )) + for actual_row, expect_row in zip( + actual_outs.reshape((-1, n_dim)), expect_outs.reshape((-1, n_dim)) + ): + is_mapped_index = np.zeros((n_dim,)) for i in range(n_dim): is_mapped = False for j in range(n_dim): if is_mapped_index[j] == 0 and np.isclose( - np.array(actual_row[i]), - np.array(expect_row[j]), - atol=1e-5): + np.array(actual_row[i]), + np.array(expect_row[j]), + atol=1e-5, + ): is_mapped_index[j] = True is_mapped = True break self.assertTrue( is_mapped, - "Output has diff in class " + self.__class__.__name__ + - "\nExpect " + str(expect_outs) + "\n" + "But Got" + - str(actual_outs) + "\nThe data " + str(actual_row[i]) + - " in " + str(actual_row) + " mismatch.") + "Output has diff in class " + + self.__class__.__name__ + + "\nExpect " + + str(expect_outs) + + "\n" + + "But Got" + + str(actual_outs) + + "\nThe data " + + str(actual_row[i]) + + " in " + + str(actual_row) + + " mismatch.", + ) class TestEigvalsOpFloat64(TestEigvalsOp): - def set_dtype(self): self.dtype = np.float64 class TestEigvalsOpComplex64(TestEigvalsOp): - def set_dtype(self): self.dtype = np.complex64 class TestEigvalsOpComplex128(TestEigvalsOp): - def set_dtype(self): self.dtype = np.complex128 class TestEigvalsOpLargeScare(TestEigvalsOp): - def set_input_dims(self): self.input_dims = (128, 128) class TestEigvalsOpLargeScareFloat64(TestEigvalsOpLargeScare): - def set_dtype(self): self.dtype = np.float64 class TestEigvalsOpLargeScareComplex64(TestEigvalsOpLargeScare): - def set_dtype(self): self.dtype = np.complex64 class TestEigvalsOpLargeScareComplex128(TestEigvalsOpLargeScare): - def set_dtype(self): self.dtype = np.complex128 class TestEigvalsOpBatch1(TestEigvalsOp): - def set_input_dims(self): self.input_dims = (1, 2, 3, 4, 4) class TestEigvalsOpBatch2(TestEigvalsOp): - def set_input_dims(self): self.input_dims = (3, 1, 4, 5, 5) class TestEigvalsOpBatch3(TestEigvalsOp): - def set_input_dims(self): self.input_dims = (6, 2, 9, 6, 6) class TestEigvalsAPI(unittest.TestCase): - def setUp(self): np.random.seed(0) @@ -183,42 +192,62 @@ def set_dtype(self): self.dtype = np.float32 def set_input_data(self): - if (self.dtype == np.float32 or self.dtype == np.float64): + if self.dtype == np.float32 or self.dtype == np.float64: self.input_data = np.random.random(self.input_dims).astype( - self.dtype) + self.dtype + ) else: - self.input_data = (np.random.random(self.input_dims) + - np.random.random(self.input_dims) * 1j).astype( - self.dtype) + self.input_data = ( + np.random.random(self.input_dims) + + np.random.random(self.input_dims) * 1j + ).astype(self.dtype) def verify_output(self, actural_outs, expect_outs): actual_outs = np.array(actural_outs) expect_outs = np.array(expect_outs) self.assertTrue( - actual_outs.shape == expect_outs.shape, "Output shape has diff." - "\nExpect shape " + str(expect_outs.shape) + "\n" + "But Got" + - str(actual_outs.shape) + " in class " + self.__class__.__name__) + actual_outs.shape == expect_outs.shape, + "Output shape has diff." + "\nExpect shape " + + str(expect_outs.shape) + + "\n" + + "But Got" + + str(actual_outs.shape) + + " in class " + + self.__class__.__name__, + ) n_dim = actual_outs.shape[-1] - for actual_row, expect_row in zip(actual_outs.reshape((-1, n_dim)), - expect_outs.reshape((-1, n_dim))): - is_mapped_index = np.zeros((n_dim, )) + for actual_row, expect_row in zip( + actual_outs.reshape((-1, n_dim)), expect_outs.reshape((-1, n_dim)) + ): + is_mapped_index = np.zeros((n_dim,)) for i in range(n_dim): is_mapped = False for j in range(n_dim): if is_mapped_index[j] == 0 and np.isclose( - np.array(actual_row[i]), - np.array(expect_row[j]), - atol=1e-5): + np.array(actual_row[i]), + np.array(expect_row[j]), + atol=1e-5, + ): is_mapped_index[j] = True is_mapped = True break self.assertTrue( is_mapped, - "Output has diff in class " + self.__class__.__name__ + - "\nExpect " + str(expect_outs) + "\n" + "But Got" + - str(actual_outs) + "\nThe data " + str(actual_row[i]) + - " in " + str(actual_row) + " mismatch.") + "Output has diff in class " + + self.__class__.__name__ + + "\nExpect " + + str(expect_outs) + + "\n" + + "But Got" + + str(actual_outs) + + "\nThe data " + + str(actual_row[i]) + + " in " + + str(actual_row) + + " mismatch.", + ) def run_dygraph(self, place): paddle.disable_static() @@ -241,24 +270,28 @@ def run_dygraph(self, place): def run_static(self, place): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - small_input_tensor = paddle.static.data(name='small_x', - shape=self.small_dims, - dtype=self.dtype) - large_input_tensor = paddle.static.data(name='large_x', - shape=self.large_dims, - dtype=self.dtype) - batch_input_tensor = paddle.static.data(name='batch_x', - shape=self.batch_dims, - dtype=self.dtype) - - small_outs = paddle.linalg.eigvals(small_input_tensor, - name='small_x') - large_outs = paddle.linalg.eigvals(large_input_tensor, - name='large_x') - batch_outs = paddle.linalg.eigvals(batch_input_tensor, - name='batch_x') + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + small_input_tensor = paddle.static.data( + name='small_x', shape=self.small_dims, dtype=self.dtype + ) + large_input_tensor = paddle.static.data( + name='large_x', shape=self.large_dims, dtype=self.dtype + ) + batch_input_tensor = paddle.static.data( + name='batch_x', shape=self.batch_dims, dtype=self.dtype + ) + + small_outs = paddle.linalg.eigvals( + small_input_tensor, name='small_x' + ) + large_outs = paddle.linalg.eigvals( + large_input_tensor, name='large_x' + ) + batch_outs = paddle.linalg.eigvals( + batch_input_tensor, name='batch_x' + ) exe = paddle.static.Executor(place) @@ -266,9 +299,10 @@ def run_static(self, place): feed={ "small_x": self.small_input, "large_x": self.large_input, - "batch_x": self.batch_input + "batch_x": self.batch_input, }, - fetch_list=[small_outs, large_outs, batch_outs]) + fetch_list=[small_outs, large_outs, batch_outs], + ) np_outs = np_eigvals(self.small_input) self.verify_output(paddle_outs[0], np_outs) @@ -281,7 +315,7 @@ def run_static(self, place): def test_cases(self): places = [core.CPUPlace()] - #if core.is_compiled_with_cuda(): + # if core.is_compiled_with_cuda(): # places.append(core.CUDAPlace(0)) for place in places: self.run_dygraph(place) @@ -301,19 +335,16 @@ def test_error(self): class TestEigvalsAPIFloat64(TestEigvalsAPI): - def set_dtype(self): self.dtype = np.float64 class TestEigvalsAPIComplex64(TestEigvalsAPI): - def set_dtype(self): self.dtype = np.complex64 class TestEigvalsAPIComplex128(TestEigvalsAPI): - def set_dtype(self): self.dtype = np.complex128 diff --git a/python/paddle/fluid/tests/unittests/test_eigvalsh_op.py b/python/paddle/fluid/tests/unittests/test_eigvalsh_op.py index 08d3da2103cfe..2dc378a0810dc 100644 --- a/python/paddle/fluid/tests/unittests/test_eigvalsh_op.py +++ b/python/paddle/fluid/tests/unittests/test_eigvalsh_op.py @@ -34,7 +34,11 @@ def valid_eigenvalues(actual, expected): FP32_MAX_RELATIVE_ERR = 5e-5 FP64_MAX_RELATIVE_ERR = 1e-14 - rtol = FP32_MAX_RELATIVE_ERR if actual.dtype == np.single else FP64_MAX_RELATIVE_ERR + rtol = ( + FP32_MAX_RELATIVE_ERR + if actual.dtype == np.single + else FP64_MAX_RELATIVE_ERR + ) diff = np.abs(expected - actual) max_diff = np.max(diff) @@ -44,7 +48,6 @@ def valid_eigenvalues(actual, expected): class TestEigvalshOp(OpTest): - def setUp(self): paddle.enable_static() self.op_type = "eigvalsh" @@ -75,13 +78,11 @@ def test_grad(self): class TestEigvalshUPLOCase(TestEigvalshOp): - def init_config(self): self.UPLO = 'U' class TestEigvalshGPUCase(unittest.TestCase): - def setUp(self): self.x_shape = [32, 32] self.dtype = "float32" @@ -98,14 +99,16 @@ def test_check_output_gpu(self): class TestEigvalshAPI(unittest.TestCase): - def setUp(self): self.dtype = "float32" self.UPLO = 'L' self.rtol = 1e-5 # test_eigvalsh_grad self.atol = 1e-5 # test_eigvalsh_grad - self.place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) np.random.seed(123) self.init_input_shape() self.init_input_data() @@ -116,25 +119,30 @@ def init_input_shape(self): def init_input_data(self): self.real_data = np.random.random(self.x_shape).astype(self.dtype) complex_data = np.random.random(self.x_shape).astype( - self.dtype) + 1J * np.random.random(self.x_shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.x_shape).astype(self.dtype) self.trans_dims = list(range(len(self.x_shape) - 2)) + [ - len(self.x_shape) - 1, len(self.x_shape) - 2 + len(self.x_shape) - 1, + len(self.x_shape) - 2, ] self.complex_symm = np.divide( - complex_data + np.conj(complex_data.transpose(self.trans_dims)), 2) + complex_data + np.conj(complex_data.transpose(self.trans_dims)), 2 + ) def check_static_float_result(self): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, startup_prog): - input_x = paddle.static.data('input_x', - shape=self.x_shape, - dtype=self.dtype) + input_x = paddle.static.data( + 'input_x', shape=self.x_shape, dtype=self.dtype + ) output_w = paddle.linalg.eigvalsh(input_x) exe = paddle.static.Executor(self.place) - actual_w = exe.run(main_prog, - feed={"input_x": self.real_data}, - fetch_list=[output_w]) + actual_w = exe.run( + main_prog, + feed={"input_x": self.real_data}, + fetch_list=[output_w], + ) expected_w = np.linalg.eigvalsh(self.real_data) compare_result(actual_w[0], expected_w) @@ -144,14 +152,16 @@ def check_static_complex_result(self): startup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, startup_prog): x_dtype = np.complex64 if self.dtype == "float32" else np.complex128 - input_x = paddle.static.data('input_x', - shape=self.x_shape, - dtype=x_dtype) + input_x = paddle.static.data( + 'input_x', shape=self.x_shape, dtype=x_dtype + ) output_w = paddle.linalg.eigvalsh(input_x) exe = paddle.static.Executor(self.place) - actual_w = exe.run(main_prog, - feed={"input_x": self.complex_symm}, - fetch_list=[output_w]) + actual_w = exe.run( + main_prog, + feed={"input_x": self.complex_symm}, + fetch_list=[output_w], + ) expected_w = np.linalg.eigvalsh(self.complex_symm) compare_result(actual_w[0], expected_w) @@ -177,48 +187,47 @@ def test_eigvalsh_grad(self): x = paddle.to_tensor(self.complex_symm, stop_gradient=False) w = paddle.linalg.eigvalsh(x) (w.sum()).backward() - np.testing.assert_allclose(abs(x.grad.numpy()), - abs(x.grad.numpy().conj().transpose( - self.trans_dims)), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + abs(x.grad.numpy()), + abs(x.grad.numpy().conj().transpose(self.trans_dims)), + rtol=self.rtol, + atol=self.atol, + ) class TestEigvalshBatchAPI(TestEigvalshAPI): - def init_input_shape(self): self.x_shape = [2, 5, 5] class TestEigvalshAPIError(unittest.TestCase): - def test_error(self): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, startup_prog): - #input maxtrix must greater than 2 dimensions - input_x = paddle.static.data(name='x_1', - shape=[12], - dtype='float32') + # input maxtrix must greater than 2 dimensions + input_x = paddle.static.data( + name='x_1', shape=[12], dtype='float32' + ) self.assertRaises(ValueError, paddle.linalg.eigvalsh, input_x) - #input matrix must be square matrix - input_x = paddle.static.data(name='x_2', - shape=[12, 32], - dtype='float32') + # input matrix must be square matrix + input_x = paddle.static.data( + name='x_2', shape=[12, 32], dtype='float32' + ) self.assertRaises(ValueError, paddle.linalg.eigvalsh, input_x) - #uplo must be in 'L' or 'U' - input_x = paddle.static.data(name='x_3', - shape=[4, 4], - dtype="float32") + # uplo must be in 'L' or 'U' + input_x = paddle.static.data( + name='x_3', shape=[4, 4], dtype="float32" + ) uplo = 'R' self.assertRaises(ValueError, paddle.linalg.eigvalsh, input_x, uplo) - #x_data cannot be integer - input_x = paddle.static.data(name='x_4', - shape=[4, 4], - dtype="int32") + # x_data cannot be integer + input_x = paddle.static.data( + name='x_4', shape=[4, 4], dtype="int32" + ) self.assertRaises(TypeError, paddle.linalg.eigvalsh, input_x) diff --git a/python/paddle/fluid/tests/unittests/test_einsum.py b/python/paddle/fluid/tests/unittests/test_einsum.py index 40583c01f34c5..2e8a0a6c7cb5e 100644 --- a/python/paddle/fluid/tests/unittests/test_einsum.py +++ b/python/paddle/fluid/tests/unittests/test_einsum.py @@ -23,85 +23,109 @@ class TestErrors(unittest.TestCase): - def setUp(self): pass def test_diagonalize_errors(self): a = np.arange(4 * 3 * 4 * 4).reshape(4, 3, 4, 4).astype('float') a = paddle.to_tensor(a) - with self.assertRaisesRegex(AssertionError, - ('Duplicate labels are not supported.')): + with self.assertRaisesRegex( + AssertionError, ('Duplicate labels are not supported.') + ): paddle.einsum('...ii->...i', a) - with self.assertRaisesRegex(AssertionError, - ('Duplicate labels are not supported.')): + with self.assertRaisesRegex( + AssertionError, ('Duplicate labels are not supported.') + ): paddle.einsum('i...i', a) - with self.assertRaisesRegex(AssertionError, - ('Duplicate labels are not supported.')): + with self.assertRaisesRegex( + AssertionError, ('Duplicate labels are not supported.') + ): paddle.einsum('i...i->i...', a) def test_param_errors(self): a = np.arange(4 * 3 * 4 * 4).reshape(4, 3, 4, 4).astype('float') a = paddle.to_tensor(a) - with self.assertRaisesRegex(AssertionError, - ('At least one operand is expected.')): + with self.assertRaisesRegex( + AssertionError, ('At least one operand is expected.') + ): paddle.einsum('ijk') with self.assertRaisesRegex( - AssertionError, - ('Invalid equation: multiple `->` were found.')): + AssertionError, ('Invalid equation: multiple `->` were found.') + ): paddle.einsum('i -> j -> k', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the number of operands is 2, " - "but found 3 segments in the label equation.")): + AssertionError, + ( + "Invalid equation: the number of operands is 2, " + "but found 3 segments in the label equation." + ), + ): paddle.einsum('i,j,k', a, a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the number of operands is 2, " - "but found 1 segments in the label equation.")): + AssertionError, + ( + "Invalid equation: the number of operands is 2, " + "but found 1 segments in the label equation." + ), + ): paddle.einsum('ij -> k', a, a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the number of operands is 1, " - "but found 2 segments in the label equation.")): + AssertionError, + ( + "Invalid equation: the number of operands is 1, " + "but found 2 segments in the label equation." + ), + ): paddle.einsum('i, -> k', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the label string '' misses dimensions.")): + AssertionError, + ("Invalid equation: the label string '' misses dimensions."), + ): paddle.einsum('->', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the label string 'i' misses dimensions.")): + AssertionError, + ("Invalid equation: the label string 'i' misses dimensions."), + ): paddle.einsum('i', a) with self.assertRaisesRegex( - AssertionError, ("Invalid equation: _ is not a valid label, " - "which should be letters.")): + AssertionError, + ( + "Invalid equation: _ is not a valid label, " + "which should be letters." + ), + ): paddle.einsum('i_', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: `.` is found outside of an ellipsis.")): + AssertionError, + ("Invalid equation: `.` is found outside of an ellipsis."), + ): paddle.einsum('i..j', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: `.` is found outside of an ellipsis.")): + AssertionError, + ("Invalid equation: `.` is found outside of an ellipsis."), + ): paddle.einsum('...k...', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: missing ellipsis in output labels.")): + AssertionError, + ("Invalid equation: missing ellipsis in output labels."), + ): paddle.einsum('i...->i', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: duplicate output labels are found.")): + AssertionError, + ("Invalid equation: duplicate output labels are found."), + ): paddle.einsum('i...->i...i', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid operands: label i " - "corresponds to non-broadcastable dimensions.")): + AssertionError, + ( + "Invalid operands: label i " + "corresponds to non-broadcastable dimensions." + ), + ): paddle.einsum('ij...,ji...', a, a) class TestEinsum(unittest.TestCase): - @classmethod def setUpClass(cls): np.random.seed(12345) @@ -132,15 +156,17 @@ def _get_place(self, force_to_use_cpu=False): return core.CUDAPlace(0) return core.CPUPlace() - def check_output_equal(self, actual, expect, rtol=1.e-5, atol=1.e-8): + def check_output_equal(self, actual, expect, rtol=1.0e-5, atol=1.0e-8): error_msg = 'Output has diff at place:{}. \nExpect: {} \nBut Got: {} in class {}' - np.testing.assert_allclose(actual, - expect, - rtol=rtol, - atol=atol, - err_msg=error_msg.format( - paddle.get_device(), expect, actual, - self.__class__.__name__)) + np.testing.assert_allclose( + actual, + expect, + rtol=rtol, + atol=atol, + err_msg=error_msg.format( + paddle.get_device(), expect, actual, self.__class__.__name__ + ), + ) def setUp(self): self.sample = {"paradigm": "i->", "data": ["x"]} @@ -153,7 +179,8 @@ def test_forward(self): equation = self.sample["paradigm"] with paddle.fluid.dygraph.guard( - self._get_place(force_to_use_cpu=False)): + self._get_place(force_to_use_cpu=False) + ): pd_operands = [paddle.to_tensor(operand) for operand in operands] result = paddle.einsum(equation, *pd_operands) self.check_output_equal(result.numpy(), expected_result) @@ -165,163 +192,136 @@ def test_forward(self): class TestEinsumVectorDot(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i,i->", "data": ["x", "x"]} class TestEinsumVectorMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i,i->i", "data": ["x", "x"]} class TestEinsumVectorOuter(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i,j->ij", "data": ["x", "y"]} class TestEinsumMatrixTranspose(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij->ji", "data": ["A"]} class TestEinsumMatrixRowSum(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij->j", "data": ["A"]} class TestEinsumMatrixColSum(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij->i", "data": ["A"]} class TestEinsumMatrixEleMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,ij->ij", "data": ["A", "A"]} class TestEinsumDegenerateMatrixVecMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,j", "data": ["a", "b"]} class TestEinsumMatrixVecMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,j->i", "data": ["A", "x"]} class TestEinsumMatrixMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,kj->ik", "data": ["A", "B"]} class TestEinsumMatrixOuter(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,kl->ijkl", "data": ["A", "C"]} class TestEinsumTensorBMM(TestEinsum): - def setUp(self): self.sample = {"paradigm": "bij,bjk->bik", "data": ["D", "E"]} class TestEinsumTensorContract1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,jk->i", "data": ["D", "A"]} class TestEinsumTensorContract2(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,lk->ijl", "data": ["D", "B"]} class TestEinsumTensorContract3(TestEinsum): - def setUp(self): self.sample = {"paradigm": "abcd,dfg->abcfg", "data": ["F", "D"]} class TestEinsumTensorContract4(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,jk->ik", "data": ["D", "A"]} class TestEinsumTensorContract5(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,jk->ij", "data": ["D", "A"]} class TestEinsumTensorContract6(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ik, ijk->j", "data": ["A", "G"]} class TestEinsumTensorContract7(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk, ik->jk", "data": ["G", "A"]} class TestEinsumEllipsis1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i...->...", "data": ["G"]} class TestEinsumEllipsis2(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,...i->j...", "data": ["A", "H"]} class TestEinsumEllipsis3(TestEinsum): - def setUp(self): self.sample = {"paradigm": "k...,jk", "data": ["F", "I"]} class TestEinsumTestEinsumBilinear(TestEinsum): - def setUp(self): self.sample = {"paradigm": "bn,anm,bm->ba", "data": ["B", "E", "I"]} class TestEinsumTestEinsumOthers1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijkl, lmn->kmn", "data": ["F", "H"]} class TestEinsumTestEinsumOthers2(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijkl, lmn->ijn", "data": ["F", "H"]} class TestEinsumBatch1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "blq,bhlk->bhlqk", "data": ["J", "K"]} class TestNumpyTests(unittest.TestCase): - def setUp(self): pass @@ -333,20 +333,23 @@ def _get_place(self, force_to_use_cpu=False): return core.CUDAPlace(0) return core.CPUPlace() - def check_output_equal(self, actual, expect, rtol=1.e-5, atol=1.e-8): + def check_output_equal(self, actual, expect, rtol=1.0e-5, atol=1.0e-8): error_msg = 'Output has diff at place:{}. \nExpect: {} \nBut Got: {} in class {}' - np.testing.assert_allclose(actual, - expect, - rtol=rtol, - atol=atol, - err_msg=error_msg.format( - paddle.get_device(), expect, actual, - self.__class__.__name__)) + np.testing.assert_allclose( + actual, + expect, + rtol=rtol, + atol=atol, + err_msg=error_msg.format( + paddle.get_device(), expect, actual, self.__class__.__name__ + ), + ) def check_output(self, eqn, *ops): expect = np.einsum(eqn, *ops) with paddle.fluid.dygraph.guard( - self._get_place(force_to_use_cpu=False)): + self._get_place(force_to_use_cpu=False) + ): pd_operands = [paddle.to_tensor(op) for op in ops] actual = paddle.einsum(eqn, *pd_operands) self.check_output_equal(actual.numpy(), expect) @@ -421,8 +424,8 @@ def test_sums(self): q = np.ones((1, 2)).astype('float') self.check_output('ij,ij->j', p, q) - x = np.array([2., 3.]).astype('float') - y = np.array([4.]).astype('float') + x = np.array([2.0, 3.0]).astype('float') + y = np.array([4.0]).astype('float') self.check_output("i, i", x, y) p = np.ones((1, 5)) / 2 @@ -453,21 +456,21 @@ def test_static_graph(self): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): - a = paddle.static.data(name='a', - shape=[3, None, None, None], - dtype='float') - b = paddle.static.data(name='b', - shape=[2, None, None, None], - dtype='float') - c = paddle.static.data(name='c', - shape=[None, None, 2, None], - dtype='float') - d = paddle.static.data(name='d', - shape=[None, None, 5], - dtype='float') - e = paddle.static.data(name='e', - shape=[None, 2, None], - dtype='float') + a = paddle.static.data( + name='a', shape=[3, None, None, None], dtype='float' + ) + b = paddle.static.data( + name='b', shape=[2, None, None, None], dtype='float' + ) + c = paddle.static.data( + name='c', shape=[None, None, 2, None], dtype='float' + ) + d = paddle.static.data( + name='d', shape=[None, None, 5], dtype='float' + ) + e = paddle.static.data( + name='e', shape=[None, 2, None], dtype='float' + ) outs = [] outs.append(paddle.einsum("ibnd,jbnd->bnij", a, b)) diff --git a/python/paddle/fluid/tests/unittests/test_einsum_op.py b/python/paddle/fluid/tests/unittests/test_einsum_op.py index f3f7b9093664c..9db367a23357f 100644 --- a/python/paddle/fluid/tests/unittests/test_einsum_op.py +++ b/python/paddle/fluid/tests/unittests/test_einsum_op.py @@ -19,7 +19,6 @@ class TestEinsumBinary(OpTest): - def setUp(self): paddle.enable_static() self.op_type = "einsum" @@ -34,12 +33,15 @@ def setUp(self): self.inputs = {"Operands": self.operands} self.attrs = {"equation": self.equation} self.outputs = { - 'Out': - out, - "InnerCache": [('cache_' + str(i), np.array([1.0])) - for i in range(len(self.operands))], - "XShape": [('xshape_' + str(i), np.array([1.0])) - for i in range(len(self.operands))], + 'Out': out, + "InnerCache": [ + ('cache_' + str(i), np.array([1.0])) + for i in range(len(self.operands)) + ], + "XShape": [ + ('xshape_' + str(i), np.array([1.0])) + for i in range(len(self.operands)) + ], } def init_input(self): @@ -62,7 +64,6 @@ def test_grad(self): class TestEinsum1(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(20, 3, 3), (20, 3, 3)] self.types = [np.float64, np.float64] @@ -70,7 +71,6 @@ def set_mandatory(self): class TestEinsum2(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(20, 3, 3), (20, 3, 3)] self.types = [np.float64, np.float64] @@ -78,7 +78,6 @@ def set_mandatory(self): class TestEinsum3(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(10, 10), (10, 10)] self.types = [np.float64, np.float64] @@ -86,7 +85,6 @@ def set_mandatory(self): class TestEinsumWithReduction(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(10, 3, 5), (5, 30)] self.types = [np.float64, np.float64] @@ -94,7 +92,6 @@ def set_mandatory(self): class TestEinsumWithReduction1(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(10, 3, 3, 5), (10, 5, 10, 10)] self.types = [np.float64, np.float64] @@ -102,7 +99,6 @@ def set_mandatory(self): class TestEinsumWithUnary(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(10, 10, 3, 5)] self.types = [np.float64] @@ -110,7 +106,6 @@ def set_mandatory(self): class TestEinsumWithUnary1(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(5, 10, 3, 3), (3, 6, 3, 10)] self.types = [np.float64, np.float64] @@ -118,7 +113,6 @@ def set_mandatory(self): class TestEinsumWithBroadcast1(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(5, 10, 3, 3)] self.types = [np.float64] @@ -126,7 +120,6 @@ def set_mandatory(self): class TestEinsumWithBroadcast2(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(10, 11), (3, 4, 5, 10)] self.types = [np.float64, np.float64] @@ -134,7 +127,6 @@ def set_mandatory(self): class TestEinsumWithBroadcast3(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(10, 3, 2, 3, 4), (12, 10)] self.types = [np.float64, np.float64] @@ -142,7 +134,6 @@ def set_mandatory(self): class TestEinsumWithBroadcast4(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(10, 3, 2, 3, 4), (12, 10)] self.types = [np.float64, np.float64] @@ -150,7 +141,6 @@ def set_mandatory(self): class TestEinsumWithBroadcast5(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(3, 2, 2, 10), (10, 3, 2, 2)] self.types = [np.float64, np.float64] @@ -158,7 +148,6 @@ def set_mandatory(self): class TestEinsumWithBroadcast6(TestEinsumBinary): - def set_mandatory(self): self.shapes = [(100), (100)] self.types = [np.float64, np.float64] diff --git a/python/paddle/fluid/tests/unittests/test_einsum_v2.py b/python/paddle/fluid/tests/unittests/test_einsum_v2.py index 0cfe6bdece91e..e7b041124c257 100644 --- a/python/paddle/fluid/tests/unittests/test_einsum_v2.py +++ b/python/paddle/fluid/tests/unittests/test_einsum_v2.py @@ -31,91 +31,117 @@ def error_trans(func, *args, **kargs): out = func(*args, **kargs) except ValueError as e: if "Same label have different shapes" in str(e): - raise AssertionError("Invalid operands: label i " - "corresponds to non-broadcastable dimensions.") + raise AssertionError( + "Invalid operands: label i " + "corresponds to non-broadcastable dimensions." + ) class TestErrors(unittest.TestCase): - def setUp(self): pass def test_diagonalize_errors(self): a = np.arange(4 * 3 * 4 * 4).reshape(4, 3, 4, 4).astype('float') a = paddle.to_tensor(a) - with self.assertRaisesRegex(AssertionError, - ('Duplicate labels are not supported.')): + with self.assertRaisesRegex( + AssertionError, ('Duplicate labels are not supported.') + ): paddle.einsum('...ii->...i', a) - with self.assertRaisesRegex(AssertionError, - ('Duplicate labels are not supported.')): + with self.assertRaisesRegex( + AssertionError, ('Duplicate labels are not supported.') + ): paddle.einsum('i...i', a) - with self.assertRaisesRegex(AssertionError, - ('Duplicate labels are not supported.')): + with self.assertRaisesRegex( + AssertionError, ('Duplicate labels are not supported.') + ): paddle.einsum('i...i->i...', a) def test_param_errors(self): a = np.arange(4 * 3 * 4 * 4).reshape(4, 3, 4, 4).astype('float') a = paddle.to_tensor(a) with self.assertRaisesRegex( - AssertionError, - ("Required at least one operand in Einsum API, but received 0 ")): + AssertionError, + ("Required at least one operand in Einsum API, but received 0 "), + ): paddle.einsum('ijk') with self.assertRaisesRegex( - AssertionError, - ('Invalid equation: multiple `->` were found.')): + AssertionError, ('Invalid equation: multiple `->` were found.') + ): paddle.einsum('i -> j -> k', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the number of operands is 2, " - "but found 3 segments in the label equation.")): + AssertionError, + ( + "Invalid equation: the number of operands is 2, " + "but found 3 segments in the label equation." + ), + ): paddle.einsum('i,j,k', a, a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the number of operands is 2, " - "but found 1 segments in the label equation.")): + AssertionError, + ( + "Invalid equation: the number of operands is 2, " + "but found 1 segments in the label equation." + ), + ): paddle.einsum('ij -> k', a, a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the number of operands is 1, " - "but found 2 segments in the label equation.")): + AssertionError, + ( + "Invalid equation: the number of operands is 1, " + "but found 2 segments in the label equation." + ), + ): paddle.einsum('i, -> k', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the label string '' misses dimensions.")): + AssertionError, + ("Invalid equation: the label string '' misses dimensions."), + ): paddle.einsum('->', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: the label string 'i' misses dimensions.")): + AssertionError, + ("Invalid equation: the label string 'i' misses dimensions."), + ): paddle.einsum('i', a) with self.assertRaisesRegex( - AssertionError, ("Invalid equation: _ is not a valid label, " - "which should be letters.")): + AssertionError, + ( + "Invalid equation: _ is not a valid label, " + "which should be letters." + ), + ): paddle.einsum('i_', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: `.` is found outside of an ellipsis.")): + AssertionError, + ("Invalid equation: `.` is found outside of an ellipsis."), + ): paddle.einsum('i..j', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: `.` is found outside of an ellipsis.")): + AssertionError, + ("Invalid equation: `.` is found outside of an ellipsis."), + ): paddle.einsum('...k...', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: missing ellipsis in output labels.")): + AssertionError, + ("Invalid equation: missing ellipsis in output labels."), + ): paddle.einsum('i...->i', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid equation: duplicate output labels are found.")): + AssertionError, + ("Invalid equation: duplicate output labels are found."), + ): paddle.einsum('i...->i...i', a) with self.assertRaisesRegex( - AssertionError, - ("Invalid operands: label i " - "corresponds to non-broadcastable dimensions.")): + AssertionError, + ( + "Invalid operands: label i " + "corresponds to non-broadcastable dimensions." + ), + ): error_trans(paddle.einsum, 'ij...,ji...', a, a) class TestEinsum(unittest.TestCase): - @classmethod def setUpClass(cls): np.random.seed(12345) @@ -146,15 +172,17 @@ def _get_place(self, force_to_use_cpu=False): return core.CUDAPlace(0) return core.CPUPlace() - def check_output_equal(self, actual, expect, rtol=1.e-5, atol=1.e-8): + def check_output_equal(self, actual, expect, rtol=1.0e-5, atol=1.0e-8): error_msg = 'Output has diff at place:{}. \nExpect: {} \nBut Got: {} in class {}' - np.testing.assert_allclose(actual, - expect, - rtol=rtol, - atol=atol, - err_msg=error_msg.format( - paddle.get_device(), expect, actual, - self.__class__.__name__)) + np.testing.assert_allclose( + actual, + expect, + rtol=rtol, + atol=atol, + err_msg=error_msg.format( + paddle.get_device(), expect, actual, self.__class__.__name__ + ), + ) def setUp(self): self.sample = {"paradigm": "i->", "data": ["x"]} @@ -167,7 +195,8 @@ def test_forward(self): equation = self.sample["paradigm"] with paddle.fluid.dygraph.guard( - self._get_place(force_to_use_cpu=False)): + self._get_place(force_to_use_cpu=False) + ): pd_operands = [paddle.to_tensor(operand) for operand in operands] result = paddle.einsum(equation, *pd_operands) self.check_output_equal(result.numpy(), expected_result) @@ -179,163 +208,136 @@ def test_forward(self): class TestEinsumVectorDot(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i,i->", "data": ["x", "x"]} class TestEinsumVectorMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i,i->i", "data": ["x", "x"]} class TestEinsumVectorOuter(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i,j->ij", "data": ["x", "y"]} class TestEinsumMatrixTranspose(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij->ji", "data": ["A"]} class TestEinsumMatrixRowSum(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij->j", "data": ["A"]} class TestEinsumMatrixColSum(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij->i", "data": ["A"]} class TestEinsumMatrixEleMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,ij->ij", "data": ["A", "A"]} class TestEinsumDegenerateMatrixVecMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,j", "data": ["a", "b"]} class TestEinsumMatrixVecMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,j->i", "data": ["A", "x"]} class TestEinsumMatrixMul(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,kj->ik", "data": ["A", "B"]} class TestEinsumMatrixOuter(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,kl->ijkl", "data": ["A", "C"]} class TestEinsumTensorBMM(TestEinsum): - def setUp(self): self.sample = {"paradigm": "bij,bjk->bik", "data": ["D", "E"]} class TestEinsumTensorContract1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,jk->i", "data": ["D", "A"]} class TestEinsumTensorContract2(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,lk->ijl", "data": ["D", "B"]} class TestEinsumTensorContract3(TestEinsum): - def setUp(self): self.sample = {"paradigm": "abcd,dfg->abcfg", "data": ["F", "D"]} class TestEinsumTensorContract4(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,jk->ik", "data": ["D", "A"]} class TestEinsumTensorContract5(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk,jk->ij", "data": ["D", "A"]} class TestEinsumTensorContract6(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ik, ijk->j", "data": ["A", "G"]} class TestEinsumTensorContract7(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijk, ik->jk", "data": ["G", "A"]} class TestEinsumEllipsis1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "i...->...", "data": ["G"]} class TestEinsumEllipsis2(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ij,...i->j...", "data": ["A", "H"]} class TestEinsumEllipsis3(TestEinsum): - def setUp(self): self.sample = {"paradigm": "k...,jk", "data": ["F", "I"]} class TestEinsumTestEinsumBilinear(TestEinsum): - def setUp(self): self.sample = {"paradigm": "bn,anm,bm->ba", "data": ["B", "E", "I"]} class TestEinsumTestEinsumOthers1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijkl, lmn->kmn", "data": ["F", "H"]} class TestEinsumTestEinsumOthers2(TestEinsum): - def setUp(self): self.sample = {"paradigm": "ijkl, lmn->ijn", "data": ["F", "H"]} class TestEinsumBatch1(TestEinsum): - def setUp(self): self.sample = {"paradigm": "blq,bhlk->bhlqk", "data": ["J", "K"]} class TestNumpyTests(unittest.TestCase): - def setUp(self): pass @@ -347,20 +349,23 @@ def _get_place(self, force_to_use_cpu=False): return core.CUDAPlace(0) return core.CPUPlace() - def check_output_equal(self, actual, expect, rtol=1.e-5, atol=1.e-8): + def check_output_equal(self, actual, expect, rtol=1.0e-5, atol=1.0e-8): error_msg = 'Output has diff at place:{}. \nExpect: {} \nBut Got: {} in class {}' - np.testing.assert_allclose(actual, - expect, - rtol=rtol, - atol=atol, - err_msg=error_msg.format( - paddle.get_device(), expect, actual, - self.__class__.__name__)) + np.testing.assert_allclose( + actual, + expect, + rtol=rtol, + atol=atol, + err_msg=error_msg.format( + paddle.get_device(), expect, actual, self.__class__.__name__ + ), + ) def check_output(self, eqn, *ops): expect = np.einsum(eqn, *ops) with paddle.fluid.dygraph.guard( - self._get_place(force_to_use_cpu=False)): + self._get_place(force_to_use_cpu=False) + ): pd_operands = [paddle.to_tensor(op) for op in ops] actual = paddle.einsum(eqn, *pd_operands) self.check_output_equal(actual.numpy(), expect) @@ -432,20 +437,20 @@ def test_sums(self): self.check_output("i,i", a, a) # TODO(@xiongkun): explict broadcast in EinsumOp is not supported, it's not recommend to use einsum like this. - #p = np.ones((10, 2)).astype('float') - #q = np.ones((1, 2)).astype('float') - #self.check_output('ij,ij->j', p, q) + # p = np.ones((10, 2)).astype('float') + # q = np.ones((1, 2)).astype('float') + # self.check_output('ij,ij->j', p, q) # TODO(@xiongkun): explict-label-broadcast in EinsumOp is not supported, it's not recommend to use einsum like this. - #x = np.array([2., 3.]).astype('float') - #y = np.array([4.]).astype('float') - #self.check_output("i, i", x, y) + # x = np.array([2., 3.]).astype('float') + # y = np.array([4.]).astype('float') + # self.check_output("i, i", x, y) # TODO(@xiongkun): explict-label-broadcast in EinsumOp is not supported, it's not recommend to use einsum like this. - #p = np.ones((1, 5)) / 2 - #q = np.ones((5, 5)) / 2 - #self.check_output("...ij,...jk->...ik", p, p) - #self.check_output("...ij,...jk->...ik", p, q) + # p = np.ones((1, 5)) / 2 + # q = np.ones((5, 5)) / 2 + # self.check_output("...ij,...jk->...ik", p, p) + # self.check_output("...ij,...jk->...ik", p, q) x = np.eye(2).astype('float') y = np.ones(2).astype('float') @@ -456,11 +461,11 @@ def test_sums(self): def test_large_nops(self): pass # TODO(@xiongkun): explict broadcast in EinsumOp is not supported, it's not recommend to use einsum like this. - #a = np.arange(4 * 3 * 1 * 4).reshape(4, 3, 1, 4).astype('float') - #self.check_output('a...b,b...c,c...d', a, a, a) - #self.check_output('a...b,b...c,c...a', a, a, a) - #self.check_output('a...b,b...c,c...a', a, a, a) - #self.check_output('...ab,...ba,...ab,...ab', a, a, a, a) + # a = np.arange(4 * 3 * 1 * 4).reshape(4, 3, 1, 4).astype('float') + # self.check_output('a...b,b...c,c...d', a, a, a) + # self.check_output('a...b,b...c,c...a', a, a, a) + # self.check_output('a...b,b...c,c...a', a, a, a) + # self.check_output('...ab,...ba,...ab,...ab', a, a, a, a) def test_static_graph(self): paddle.enable_static() @@ -472,21 +477,21 @@ def test_static_graph(self): main = fluid.Program() startup = fluid.Program() with fluid.program_guard(main, startup): - a = paddle.static.data(name='a', - shape=[3, None, None, None], - dtype='float') - b = paddle.static.data(name='b', - shape=[2, None, None, None], - dtype='float') - c = paddle.static.data(name='c', - shape=[None, None, 2, None], - dtype='float') - d = paddle.static.data(name='d', - shape=[None, None, 5], - dtype='float') - e = paddle.static.data(name='e', - shape=[None, 2, None], - dtype='float') + a = paddle.static.data( + name='a', shape=[3, None, None, None], dtype='float' + ) + b = paddle.static.data( + name='b', shape=[2, None, None, None], dtype='float' + ) + c = paddle.static.data( + name='c', shape=[None, None, 2, None], dtype='float' + ) + d = paddle.static.data( + name='d', shape=[None, None, 5], dtype='float' + ) + e = paddle.static.data( + name='e', shape=[None, 2, None], dtype='float' + ) outs = [] outs.append(paddle.einsum("ibnd,jbnd->bnij", a, b)) @@ -514,7 +519,6 @@ def test_static_graph(self): class TestStaticGraphShape(unittest.TestCase): - def setUp(self): paddle.enable_static() @@ -528,9 +532,11 @@ def test_shape(self): self.assertEqual(C.shape, (-1, 384)) -@unittest.skipIf(not core.is_compiled_with_cuda() - or not core.is_bfloat16_supported(core.CUDAPlace(0)), - "core is not compiled with CUDA or not support the bfloat16") +@unittest.skipIf( + not core.is_compiled_with_cuda() + or not core.is_bfloat16_supported(core.CUDAPlace(0)), + "core is not compiled with CUDA or not support the bfloat16", +) class TestBF16(unittest.TestCase): """ EinsumOp support bfloat16 type, add unittest here for the correctness. @@ -539,8 +545,7 @@ class TestBF16(unittest.TestCase): def test_shape(self): cuda_major = paddle.version.cuda().split('.')[0].strip() if int(cuda_major) >= 11: - """ MatmulKernel support bfloat16 only if cuda_major > 11.0. - """ + """MatmulKernel support bfloat16 only if cuda_major > 11.0.""" A = paddle.to_tensor(np.array([1.0, 2.0])).astype(paddle.bfloat16) A = A.cuda() B = paddle.to_tensor(np.array([2.0, 3.0])).astype(paddle.bfloat16) diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py index f99f8ea1c8e35..ae516bc44cad3 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py @@ -16,14 +16,17 @@ import numpy as np import paddle import paddle.fluid.core as core -from paddle.fluid.tests.unittests.op_test import OpTest, skip_check_grad_ci, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + skip_check_grad_ci, + convert_float_to_uint16, +) import paddle.fluid as fluid from paddle.fluid import Program, program_guard from paddle.fluid.framework import _test_eager_guard class TestElementwiseAddOp(OpTest): - def init_kernel_type(self): self.use_mkldnn = False @@ -37,47 +40,55 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} def check_eager(self): - return (self.use_mkldnn == False and self.axis == -1) + return self.use_mkldnn == False and self.axis == -1 def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_output(check_dygraph=(self.use_mkldnn == False), - check_eager=self.check_eager()) + self.check_output( + check_dygraph=(self.use_mkldnn == False), + check_eager=self.check_eager(), + ) def test_check_grad_normal(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode if self.dtype == np.float16: return - self.check_grad(['X', 'Y'], - 'Out', - check_dygraph=(self.use_mkldnn == False), - check_eager=self.check_eager()) + self.check_grad( + ['X', 'Y'], + 'Out', + check_dygraph=(self.use_mkldnn == False), + check_eager=self.check_eager(), + ) def test_check_grad_ingore_x(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode if self.dtype == np.float16: return - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - check_dygraph=(self.use_mkldnn == False), - check_eager=self.check_eager()) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + check_dygraph=(self.use_mkldnn == False), + check_eager=self.check_eager(), + ) def test_check_grad_ingore_y(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode if self.dtype == np.float16: return - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - check_dygraph=(self.use_mkldnn == False), - check_eager=self.check_eager()) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + check_dygraph=(self.use_mkldnn == False), + check_eager=self.check_eager(), + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -91,10 +102,10 @@ def init_axis(self): self.axis = -1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16ElementwiseAddOp(TestElementwiseAddOp): - def init_dtype(self): self.dtype = np.float16 @@ -104,16 +115,17 @@ def test_check_output(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): self.check_output_with_place( - place, atol=1e-3, check_dygraph=(self.use_mkldnn == False)) + place, atol=1e-3, check_dygraph=(self.use_mkldnn == False) + ) @unittest.skipIf( - not core.is_compiled_with_cuda() or core.cudnn_version() < 8100 + not core.is_compiled_with_cuda() + or core.cudnn_version() < 8100 or paddle.device.cuda.get_device_capability()[0] < 8, - "only support compiled with CUDA and cudnn version need larger than 8.1.0 and device's compute capability is at least 8.0" + "only support compiled with CUDA and cudnn version need larger than 8.1.0 and device's compute capability is at least 8.0", ) class TestBF16ElementwiseAddOp(OpTest): - def setUp(self): self.op_type = "elementwise_add" self.dtype = np.uint16 @@ -125,9 +137,12 @@ def setUp(self): self.axis = -1 self.inputs = { - 'X': - OpTest.np_dtype_to_fluid_dtype(convert_float_to_uint16(self.x)), - 'Y': OpTest.np_dtype_to_fluid_dtype(convert_float_to_uint16(self.y)) + 'X': OpTest.np_dtype_to_fluid_dtype( + convert_float_to_uint16(self.x) + ), + 'Y': OpTest.np_dtype_to_fluid_dtype( + convert_float_to_uint16(self.y) + ), } self.attrs = {'axis': self.axis, 'use_mkldnn': False} self.outputs = {'Out': convert_float_to_uint16(self.out)} @@ -142,23 +157,21 @@ def test_check_grad_normal(self): def test_check_grad_ingore_x(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['Y'], - 'Out', - no_grad_set=set("X"), - check_eager=False) + self.check_grad_with_place( + place, ['Y'], 'Out', no_grad_set=set("X"), check_eager=False + ) def test_check_grad_ingore_y(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - no_grad_set=set('Y'), - check_eager=False) + self.check_grad_with_place( + place, ['X'], 'Out', no_grad_set=set('Y'), check_eager=False + ) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_scalar(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -166,9 +179,9 @@ def init_input_output(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestFP16ElementwiseAddOp_scalar(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -176,9 +189,9 @@ def init_input_output(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." +) class TestElementwiseAddOp_scalar2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) @@ -186,9 +199,9 @@ def init_input_output(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." +) class TestFP16ElementwiseAddOp_scalar2(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) @@ -196,23 +209,20 @@ def init_input_output(self): class TestElementwiseAddOp_Vector(TestElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestFP16ElementwiseAddOp_Vector(TestFP16ElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestElementwiseAddOp_broadcast_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -223,7 +233,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_broadcast_0(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -234,7 +243,6 @@ def init_axis(self): class TestElementwiseAddOp_broadcast_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -245,7 +253,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_broadcast_1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -256,7 +263,6 @@ def init_axis(self): class TestElementwiseAddOp_broadcast_2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -264,7 +270,6 @@ def init_input_output(self): class TestFP16ElementwiseAddOp_broadcast_2(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -272,7 +277,6 @@ def init_input_output(self): class TestElementwiseAddOp_broadcast_3(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 1).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -283,7 +287,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_broadcast_3(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -294,7 +297,6 @@ def init_axis(self): class TestElementwiseAddOp_broadcast_4(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 1, 2).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -305,7 +307,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_broadcast_4(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 1, 2).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -316,7 +317,6 @@ def init_axis(self): class TestElementwiseAddOp_broadcast_5(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) @@ -324,7 +324,6 @@ def init_input_output(self): class TestFP16ElementwiseAddOp_broadcast_5(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) @@ -332,7 +331,6 @@ def init_input_output(self): class TestElementwiseAddOp_broadcast_6(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) @@ -340,7 +338,6 @@ def init_input_output(self): class TestElementwiseAddOp_broadcast_7(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(1, 1, 20, 5).astype(self.dtype) self.y = np.random.rand(20, 5, 1, 1).astype(self.dtype) @@ -348,7 +345,6 @@ def init_input_output(self): class TestFP16ElementwiseAddOp_broadcast_6(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) @@ -356,7 +352,6 @@ def init_input_output(self): class TestElementwiseAddOp_rowwise_add_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -367,7 +362,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_rowwise_add_0(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -378,9 +372,9 @@ def init_axis(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_rowwise_add_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -391,9 +385,9 @@ def init_axis(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestFP16ElementwiseAddOp_rowwise_add_1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -404,7 +398,6 @@ def init_axis(self): class TestElementwiseAddOp_channelwise_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -415,7 +408,6 @@ def init_axis(self): class TestFP16ElementwiseAddOp_channelwise_add(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -426,7 +418,6 @@ def init_axis(self): class TestElementwiseAddOp_commonuse_add1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -437,7 +428,6 @@ def init_axis(self): class TestElementwiseFP16AddOp_commonuse_add1(TestFP16ElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -448,7 +438,6 @@ def init_axis(self): class TestElementwiseAddOp_commonuse_add2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 1, 4).astype(self.dtype) self.y = np.random.rand(10, 1, 12, 1).astype(self.dtype) @@ -459,7 +448,6 @@ def init_axis(self): class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 12).astype(self.dtype) self.y = np.random.rand(2, 2, 10, 12).astype(self.dtype) @@ -470,7 +458,6 @@ def init_axis(self): class TestElementwiseAddOp_same_shape_ysize_large(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 1, 12).astype(self.dtype) self.y = np.random.rand(10, 2, 12).astype(self.dtype) @@ -481,14 +468,15 @@ def init_axis(self): class TestElementwiseAddOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_add must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.elementwise_add, x1, y1) # the input dtype of elementwise_add must be float16 or float32 or float64 or int32 or int64 @@ -499,7 +487,6 @@ def test_errors(self): class TestAddApi(unittest.TestCase): - def _executed_api(self, x, y, name=None): return paddle.add(x, y, name) @@ -517,7 +504,7 @@ def test_declarative(self): def gen_data(): return { "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') + "y": np.array([1, 5, 2]).astype('float32'), } x = fluid.data(name="x", shape=[3], dtype='float32') @@ -527,7 +514,7 @@ def gen_data(): place = fluid.CPUPlace() exe = fluid.Executor(place) z_value = exe.run(feed=gen_data(), fetch_list=[z.name]) - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((z_value == z_expected).all(), True) def test_dygraph(self): @@ -538,18 +525,16 @@ def test_dygraph(self): y = fluid.dygraph.to_variable(np_y) z = self._executed_api(x, y) np_z = z.numpy() - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((np_z == z_expected).all(), True) class TestAddInplaceApi(TestAddApi): - def _executed_api(self, x, y, name=None): return x.add_(y, name) class TestAddInplaceBroadcastSuccess(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 4).astype('float') self.y_numpy = np.random.rand(3, 4).astype('float') @@ -566,21 +551,18 @@ def test_broadcast_success(self): class TestAddInplaceBroadcastSuccess2(TestAddInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(1, 2, 3, 1).astype('float') self.y_numpy = np.random.rand(3, 1).astype('float') class TestAddInplaceBroadcastSuccess3(TestAddInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 1, 5).astype('float') self.y_numpy = np.random.rand(1, 3, 1, 5).astype('float') class TestAddInplaceBroadcastError(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(3, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') @@ -599,21 +581,18 @@ def broadcast_shape_error(): class TestAddInplaceBroadcastError2(TestAddInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(2, 1, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') class TestAddInplaceBroadcastError3(TestAddInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(5, 2, 1, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') class TestComplexElementwiseAddOp(OpTest): - def setUp(self): self.op_type = "elementwise_add" self.dtype = np.float64 @@ -623,7 +602,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': -1, 'use_mkldnn': False} self.outputs = {'Out': self.out} @@ -633,14 +612,17 @@ def init_base_dtype(self): def init_input_output(self): self.x = np.random.random(self.shape).astype( - self.dtype) + 1J * np.random.random(self.shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.shape).astype(self.dtype) self.y = np.random.random(self.shape).astype( - self.dtype) + 1J * np.random.random(self.shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.shape).astype(self.dtype) self.out = self.x + self.y def init_grad_input_output(self): - self.grad_out = np.ones( - self.shape, self.dtype) + 1J * np.ones(self.shape, self.dtype) + self.grad_out = np.ones(self.shape, self.dtype) + 1j * np.ones( + self.shape, self.dtype + ) self.grad_x = self.grad_out self.grad_y = self.grad_out @@ -648,43 +630,49 @@ def test_check_output(self): self.check_output(check_eager=False) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + ) class TestRealComplexElementwiseAddOp(TestComplexElementwiseAddOp): - def init_input_output(self): self.x = np.random.random(self.shape).astype(self.dtype) self.y = np.random.random(self.shape).astype( - self.dtype) + 1J * np.random.random(self.shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.shape).astype(self.dtype) self.out = self.x + self.y def init_grad_input_output(self): - self.grad_out = np.ones( - self.shape, self.dtype) + 1J * np.ones(self.shape, self.dtype) + self.grad_out = np.ones(self.shape, self.dtype) + 1j * np.ones( + self.shape, self.dtype + ) self.grad_x = np.real(self.grad_out) self.grad_y = self.grad_out class TestBoolAddFloatElementwiseAddop(unittest.TestCase): - def test_static_add(self): paddle.enable_static() a = 1.5 diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py index 3cea2fdf5cd0d..16bf0df5af38b 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py @@ -21,7 +21,6 @@ class ElementwiseDivOp(OpTest): - def setUp(self): self.op_type = "elementwise_div" self.python_api = paddle.divide @@ -34,8 +33,9 @@ def setUp(self): out = self.compute_output(x, y).astype(self.val_dtype) grad_out = np.ones(out.shape).astype(self.val_dtype) grad_x = self.compute_gradient_x(grad_out, y).astype(self.val_dtype) - grad_y = self.compute_gradient_y(grad_out, out, - y).astype(self.val_dtype) + grad_y = self.compute_gradient_y(grad_out, out, y).astype( + self.val_dtype + ) # Convert np.float32 data to np.uint16 for bfloat16 Paddle OP if self.dtype == np.uint16: @@ -84,28 +84,26 @@ def test_check_output(self): def test_check_gradient(self): check_list = [] - check_list.append({ - 'grad': ['X', 'Y'], - 'no_grad': None, - 'val_grad': [self.grad_x, self.grad_y] - }) - check_list.append({ - 'grad': ['Y'], - 'no_grad': set('X'), - 'val_grad': [self.grad_y] - }) - check_list.append({ - 'grad': ['X'], - 'no_grad': set('Y'), - 'val_grad': [self.grad_x] - }) + check_list.append( + { + 'grad': ['X', 'Y'], + 'no_grad': None, + 'val_grad': [self.grad_x, self.grad_y], + } + ) + check_list.append( + {'grad': ['Y'], 'no_grad': set('X'), 'val_grad': [self.grad_y]} + ) + check_list.append( + {'grad': ['X'], 'no_grad': set('Y'), 'val_grad': [self.grad_x]} + ) for check_option in check_list: check_args = [check_option['grad'], 'Out'] check_kwargs = { 'no_grad_set': check_option['no_grad'], 'user_defined_grads': check_option['val_grad'], 'user_defined_grad_outputs': [self.grad_out], - 'check_dygraph': self.check_dygraph + 'check_dygraph': self.check_dygraph, } if self.place is None: self.check_grad(*check_args, **check_kwargs) @@ -114,11 +112,12 @@ def test_check_gradient(self): self.check_grad_with_place(*check_args, **check_kwargs) -@unittest.skipIf(not core.is_compiled_with_cuda() - or not core.is_bfloat16_supported(core.CUDAPlace(0)), - "core is not compiled with CUDA or not support the bfloat16") +@unittest.skipIf( + not core.is_compiled_with_cuda() + or not core.is_bfloat16_supported(core.CUDAPlace(0)), + "core is not compiled with CUDA or not support the bfloat16", +) class TestElementwiseDivOpBF16(ElementwiseDivOp): - def init_args(self): # In due to output data type inconsistence of bfloat16 paddle op, we disable the dygraph check. self.check_dygraph = False @@ -134,9 +133,9 @@ def init_shape(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseDivOpScalar(ElementwiseDivOp): - def init_shape(self): self.x_shape = [20, 3, 4] self.y_shape = [1] @@ -146,14 +145,12 @@ def compute_gradient_y(self, grad_out, out, y): class TestElementwiseDivOpVector(ElementwiseDivOp): - def init_shape(self): self.x_shape = [100] self.y_shape = [100] class TestElementwiseDivOpBroadcast0(ElementwiseDivOp): - def init_shape(self): self.x_shape = [100, 3, 4] self.y_shape = [100] @@ -170,7 +167,6 @@ def compute_gradient_y(self, grad_out, out, y): class TestElementwiseDivOpBroadcast1(ElementwiseDivOp): - def init_shape(self): self.x_shape = [2, 100, 4] self.y_shape = [100] @@ -187,7 +183,6 @@ def compute_gradient_y(self, grad_out, out, y): class TestElementwiseDivOpBroadcast2(ElementwiseDivOp): - def init_shape(self): self.x_shape = [2, 3, 100] self.y_shape = [100] @@ -203,7 +198,6 @@ def compute_gradient_y(self, grad_out, out, y): class TestElementwiseDivOpBroadcast3(ElementwiseDivOp): - def init_shape(self): self.x_shape = [2, 10, 12, 5] self.y_shape = [10, 12] @@ -216,12 +210,12 @@ def compute_gradient_x(self, grad_out, y): return grad_out / y.reshape(1, 10, 12, 1) def compute_gradient_y(self, grad_out, out, y): - return np.sum(-1 * grad_out * out / y.reshape(1, 10, 12, 1), - axis=(0, 3)) + return np.sum( + -1 * grad_out * out / y.reshape(1, 10, 12, 1), axis=(0, 3) + ) class TestElementwiseDivOpBroadcast4(ElementwiseDivOp): - def init_shape(self): self.x_shape = [2, 3, 50] self.y_shape = [2, 1, 50] @@ -231,7 +225,6 @@ def compute_gradient_y(self, grad_out, out, y): class TestElementwiseDivOpBroadcast5(ElementwiseDivOp): - def init_shape(self): self.x_shape = [2, 3, 4, 20] self.y_shape = [2, 3, 1, 20] @@ -241,7 +234,6 @@ def compute_gradient_y(self, grad_out, out, y): class TestElementwiseDivOpCommonuse1(ElementwiseDivOp): - def init_shape(self): self.x_shape = [2, 3, 100] self.y_shape = [1, 1, 100] @@ -251,7 +243,6 @@ def compute_gradient_y(self, grad_out, out, y): class TestElementwiseDivOpCommonuse2(ElementwiseDivOp): - def init_shape(self): self.x_shape = [30, 3, 1, 5] self.y_shape = [30, 1, 4, 1] @@ -264,7 +255,6 @@ def compute_gradient_y(self, grad_out, out, y): class TestElementwiseDivOpXsizeLessThanYsize(ElementwiseDivOp): - def init_shape(self): self.x_shape = [10, 12] self.y_shape = [2, 3, 10, 12] @@ -275,7 +265,6 @@ def compute_gradient_x(self, grad_out, y): class TestElementwiseDivOpInt(ElementwiseDivOp): - def init_dtype(self): self.dtype = np.int32 self.val_dtype = np.int32 @@ -287,32 +276,30 @@ def compute_output(self, x, y): return x // y -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestElementwiseDivOpFp16(ElementwiseDivOp): - def init_dtype(self): self.dtype = np.float16 self.val_dtype = np.float16 class TestElementwiseDivBroadcast(unittest.TestCase): - def test_shape_with_batch_sizes(self): with fluid.program_guard(fluid.Program()): - x_var = fluid.data(name='x', - dtype='float32', - shape=[None, 3, None, None]) - one = 2. + x_var = fluid.data( + name='x', dtype='float32', shape=[None, 3, None, None] + ) + one = 2.0 out = one / x_var exe = fluid.Executor(fluid.CPUPlace()) x = np.random.uniform(0.1, 0.6, (1, 3, 32, 32)).astype("float32") - out_result, = exe.run(feed={'x': x}, fetch_list=[out]) + (out_result,) = exe.run(feed={'x': x}, fetch_list=[out]) self.assertEqual((out_result == (2 / x)).all(), True) class TestDivideOp(unittest.TestCase): - def test_name(self): with fluid.program_guard(fluid.Program()): x = fluid.data(name="x", shape=[2, 3], dtype="float32") @@ -329,12 +316,11 @@ def test_dygraph(self): y = paddle.to_tensor(np_y) z = paddle.divide(x, y) np_z = z.numpy() - z_expected = np.array([2., 0.6, 2.]) + z_expected = np.array([2.0, 0.6, 2.0]) self.assertEqual((np_z == z_expected).all(), True) class TestComplexElementwiseDivOp(OpTest): - def setUp(self): self.op_type = "elementwise_div" self.python_api = paddle.divide @@ -344,7 +330,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': -1, 'use_mkldnn': False} self.outputs = {'Out': self.out} @@ -353,17 +339,18 @@ def init_base_dtype(self): self.dtype = np.float64 def init_input_output(self): - self.x = np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(self.dtype) - self.y = np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + self.x = np.random.random((2, 3, 4, 5)).astype( + self.dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(self.dtype) + self.y = np.random.random((2, 3, 4, 5)).astype( + self.dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(self.dtype) self.out = self.x / self.y def init_grad_input_output(self): - self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1J * np.ones( - (2, 3, 4, 5), self.dtype) + self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1j * np.ones( + (2, 3, 4, 5), self.dtype + ) self.grad_x = self.grad_out / np.conj(self.y) self.grad_y = -self.grad_out * np.conj(self.x / self.y / self.y) @@ -371,38 +358,44 @@ def test_check_output(self): self.check_output(check_eager=False) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + ) class TestRealComplexElementwiseDivOp(TestComplexElementwiseDivOp): - def init_input_output(self): self.x = np.random.random((2, 3, 4, 5)).astype(self.dtype) - self.y = np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + self.y = np.random.random((2, 3, 4, 5)).astype( + self.dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(self.dtype) self.out = self.x / self.y def init_grad_input_output(self): - self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1J * np.ones( - (2, 3, 4, 5), self.dtype) + self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1j * np.ones( + (2, 3, 4, 5), self.dtype + ) self.grad_x = np.real(self.grad_out / np.conj(self.y)) self.grad_y = -self.grad_out * np.conj(self.x / self.y / self.y) diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_floordiv_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_floordiv_op.py index 746bb88866951..de058ed2b3b09 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_floordiv_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_floordiv_op.py @@ -22,7 +22,6 @@ class TestElementwiseModOp(OpTest): - def init_kernel_type(self): self.use_mkldnn = False @@ -38,7 +37,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -59,7 +58,6 @@ def init_axis(self): class TestElementwiseModOp_scalar(TestElementwiseModOp): - def init_input_output(self): scale_x = random.randint(0, 100000000) scale_y = random.randint(1, 100000000) @@ -69,7 +67,6 @@ def init_input_output(self): class TestElementwiseModOpInverse(TestElementwiseModOp): - def init_input_output(self): self.x = np.random.uniform(0, 10000, [10]).astype(self.dtype) self.y = np.random.uniform(0, 1000, [10, 10]).astype(self.dtype) @@ -77,7 +74,6 @@ def init_input_output(self): class TestFloorDivideOp(unittest.TestCase): - def test_name(self): with fluid.program_guard(fluid.Program()): x = fluid.data(name="x", shape=[2, 3], dtype="int64") diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py index 823714a9a59e9..91401ff574f6a 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py @@ -20,16 +20,12 @@ class TestElementWiseAddOp(unittest.TestCase): - def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=1e-05, - atol=atol, - err_msg=msg) + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=1e-05, atol=atol, err_msg=msg + ) def check_forward_backward(self): - def test_with_place(place): out_grad = np.random.random_sample(self.x.shape).astype(np.float32) x_grad = out_grad @@ -52,25 +48,29 @@ def test_with_place(place): with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) - elementwise_add_op = block.append_op(type="elementwise_add", - inputs={ - "X": block.var('x'), - "Y": block.var('y'), - }, - outputs={ - "Out": - block.var('out'), - }, - attrs={ - "axis": self.axis, - }) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) + elementwise_add_op = block.append_op( + type="elementwise_add", + inputs={ + "X": block.var('x'), + "Y": block.var('y'), + }, + outputs={ + "Out": block.var('out'), + }, + attrs={ + "axis": self.axis, + }, + ) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - elementwise_add_op.desc, set(), []) + elementwise_add_op.desc, set(), [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -83,18 +83,20 @@ def test_with_place(place): grad_var.set_dtype(core.VarDesc.VarType.FP32) exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in ['x', 'y', 'out@GRAD'] - }, - fetch_list=['x@GRAD', 'y@GRAD']) + out = exe.run( + program, + feed={ + name: var_dict[name] for name in ['x', 'y', 'out@GRAD'] + }, + fetch_list=['x@GRAD', 'y@GRAD'], + ) self.__assert_close(x_grad, out[0], "x@GRAD") self.__assert_close(y_grad, out[1], "y@GRAD", atol=1.4) places = [core.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu( - "elementwise_add"): + "elementwise_add" + ): places.append(core.CUDAPlace(0)) for place in places: diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_heaviside_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_heaviside_op.py index 7789f872d4ccb..aebf1c3f4fe82 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_heaviside_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_heaviside_op.py @@ -26,7 +26,6 @@ def Heaviside_grad(x, y, dout): class TestElementwiseOp(OpTest): - def setUp(self): self.op_type = "elementwise_heaviside" x = np.random.random((13, 17)).astype("float64") @@ -48,7 +47,6 @@ def test_check_grad_ingore_y(self): class TestHeavisideBroadcast(unittest.TestCase): - def setUp(self): self.input_1 = np.random.rand(2, 100, 13, 17).astype("float32") self.input_2 = np.random.rand(100, 13, 17).astype("float32") @@ -87,7 +85,6 @@ def test_broadcast(self): class TestHeavisideAPI_float64(unittest.TestCase): - def setUp(self): self.x_np = np.random.random((13, 17)).astype("float64") self.y_np = np.random.random((13, 17)).astype("float64") @@ -95,45 +92,49 @@ def setUp(self): self.dtype = "float64" def test_static(self): - for use_cuda in ([False, True] - if paddle.device.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if paddle.device.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() prog = paddle.static.Program() with paddle.static.program_guard(prog): - x = paddle.static.data(name=f"x_{self.dtype}", - shape=[13, 17], - dtype=self.dtype) - y = paddle.static.data(name=f"y_{self.dtype}", - shape=[13, 17], - dtype=self.dtype) + x = paddle.static.data( + name=f"x_{self.dtype}", shape=[13, 17], dtype=self.dtype + ) + y = paddle.static.data( + name=f"y_{self.dtype}", shape=[13, 17], dtype=self.dtype + ) out = paddle.heaviside(x, y) exe = paddle.static.Executor(place=place) - res, = exe.run(prog, - feed={ - f"x_{self.dtype}": self.x_np, - f"y_{self.dtype}": self.y_np - }, - fetch_list=out, - use_prune=True) + (res,) = exe.run( + prog, + feed={ + f"x_{self.dtype}": self.x_np, + f"y_{self.dtype}": self.y_np, + }, + fetch_list=out, + use_prune=True, + ) np.testing.assert_allclose(res, self.out_np, rtol=1e-05) def test_dygraph(self): - for use_cuda in ([False, True] - if paddle.device.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if paddle.device.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) - result = paddle.heaviside(paddle.to_tensor(self.x_np), - paddle.to_tensor(self.y_np)) + result = paddle.heaviside( + paddle.to_tensor(self.x_np), paddle.to_tensor(self.y_np) + ) np.testing.assert_allclose(result.numpy(), self.out_np, rtol=1e-05) class TestHeavisideAPI_float32(TestHeavisideAPI_float64): - def setUp(self): self.x_np = np.random.random((13, 17)).astype("float32") self.y_np = np.random.random((13, 17)).astype("float32") @@ -142,7 +143,6 @@ def setUp(self): class TestHeavisideAPI_int64(TestHeavisideAPI_float64): - def setUp(self): self.x_np = np.random.random((13, 17)).astype("int64") self.y_np = np.random.random((13, 17)).astype("int64") @@ -151,7 +151,6 @@ def setUp(self): class TestHeavisideAPI_int32(TestHeavisideAPI_float64): - def setUp(self): self.x_np = np.random.random((13, 17)).astype("int32") self.y_np = np.random.random((13, 17)).astype("int32") @@ -160,14 +159,13 @@ def setUp(self): class TestHeavisideAPI_float16(OpTest): - def setUp(self): self.dtype = np.float16 self.op_type = "elementwise_heaviside" self.python_api = paddle.heaviside self.inputs = { 'X': np.random.uniform(1, 2, [20, 5]).astype("float16"), - 'Y': np.random.uniform(1, 2, [20, 5]).astype("float16") + 'Y': np.random.uniform(1, 2, [20, 5]).astype("float16"), } self.outputs = {'Out': np.heaviside(self.inputs['X'], self.inputs['Y'])} @@ -175,16 +173,17 @@ def test_check_output(self): self.check_output() def test_check_grad(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=Heaviside_grad( - self.inputs['X'], self.inputs['Y'], - 1 / self.inputs['X'].size), - check_eager=True) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=Heaviside_grad( + self.inputs['X'], self.inputs['Y'], 1 / self.inputs['X'].size + ), + check_eager=True, + ) class TestHeavisideError(unittest.TestCase): - def test_input(self): paddle.disable_static() @@ -199,8 +198,9 @@ def test_input_y(): self.assertRaises(ValueError, test_input_y) def test_input_xy(): - paddle.heaviside(paddle.randn([100], 'float32'), - paddle.randn([100], 'float64')) + paddle.heaviside( + paddle.randn([100], 'float32'), paddle.randn([100], 'float64') + ) self.assertRaises(ValueError, test_input_xy) diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py index d44eac529b4be..018a44c2be964 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py @@ -20,7 +20,6 @@ class TestElementwiseOp(OpTest): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum @@ -46,25 +45,25 @@ def test_check_grad_normal(self): self.check_grad(['X', 'Y'], 'Out', check_eager=True) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad( + ['Y'], 'Out', max_relative_error=0.005, no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y')) - - -@unittest.skipIf(core.is_compiled_with_cuda() and ( - core.cudnn_version() < 8100 - or paddle.device.cuda.get_device_capability()[0] < 8 -), "run test when gpu is availble and the minimum cudnn version is 8.1.0 and gpu's compute capability is at least 8.0." - ) + self.check_grad( + ['X'], 'Out', max_relative_error=0.005, no_grad_set=set('Y') + ) + + +@unittest.skipIf( + core.is_compiled_with_cuda() + and ( + core.cudnn_version() < 8100 + or paddle.device.cuda.get_device_capability()[0] < 8 + ), + "run test when gpu is availble and the minimum cudnn version is 8.1.0 and gpu's compute capability is at least 8.0.", +) class TestElementwiseBF16Op(OpTest): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum @@ -77,7 +76,7 @@ def setUp(self): y = x + sgn * np.random.uniform(0.1, 1, [13, 17]).astype(np.float32) self.inputs = { 'X': convert_float_to_uint16(x), - 'Y': convert_float_to_uint16(y) + 'Y': convert_float_to_uint16(y), } self.outputs = {'Out': convert_float_to_uint16(np.maximum(x, y))} @@ -101,9 +100,9 @@ def test_check_grad_ingore_y(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMaxOp_scalar(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum @@ -114,97 +113,98 @@ def setUp(self): class TestElementwiseMaxOp_Vector(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum - x = np.random.random((100, )).astype("float64") - sgn = np.random.choice([-1, 1], (100, )).astype("float64") - y = x + sgn * np.random.uniform(0.1, 1, (100, )).astype("float64") + x = np.random.random((100,)).astype("float64") + sgn = np.random.choice([-1, 1], (100,)).astype("float64") + y = x + sgn * np.random.uniform(0.1, 1, (100,)).astype("float64") self.inputs = {'X': x, 'Y': y} self.outputs = {'Out': np.maximum(self.inputs['X'], self.inputs['Y'])} class TestElementwiseMaxOp_broadcast_0(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum x = np.random.uniform(0.5, 1, (100, 5, 2)).astype(np.float64) - sgn = np.random.choice([-1, 1], (100, )).astype(np.float64) - y = x[:, 0, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(np.float64) + sgn = np.random.choice([-1, 1], (100,)).astype(np.float64) + y = x[:, 0, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 0} self.outputs = { - 'Out': np.maximum(self.inputs['X'], - self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestElementwiseMaxOp_broadcast_1(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum x = np.random.uniform(0.5, 1, (2, 100, 3)).astype(np.float64) - sgn = np.random.choice([-1, 1], (100, )).astype(np.float64) - y = x[0, :, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(np.float64) + sgn = np.random.choice([-1, 1], (100,)).astype(np.float64) + y = x[0, :, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 1} self.outputs = { - 'Out': np.maximum(self.inputs['X'], - self.inputs['Y'].reshape(1, 100, 1)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1) + ) } class TestElementwiseMaxOp_broadcast_2(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum x = np.random.uniform(0.5, 1, (1, 3, 100)).astype(np.float64) - sgn = np.random.choice([-1, 1], (100, )).astype(np.float64) - y = x[0, 0, :] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(np.float64) + sgn = np.random.choice([-1, 1], (100,)).astype(np.float64) + y = x[0, 0, :] + sgn * np.random.uniform(1, 2, (100,)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.outputs = { - 'Out': np.maximum(self.inputs['X'], - self.inputs['Y'].reshape(1, 1, 100)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100) + ) } class TestElementwiseMaxOp_broadcast_3(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum x = np.random.uniform(0.5, 1, (2, 50, 2, 1)).astype(np.float64) sgn = np.random.choice([-1, 1], (50, 2)).astype(np.float64) - y = x[0, :, :, 0] + sgn * \ - np.random.uniform(1, 2, (50, 2)).astype(np.float64) + y = x[0, :, :, 0] + sgn * np.random.uniform(1, 2, (50, 2)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.maximum(self.inputs['X'], self.inputs['Y'].reshape(1, 50, 2, 1)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(1, 50, 2, 1) + ) } class TestElementwiseMaxOp_broadcast_4(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_max" self.python_api = paddle.maximum x = np.random.uniform(0.5, 1, (2, 3, 4, 5)).astype(np.float64) sgn = np.random.choice([-1, 1], (2, 3, 1, 5)).astype(np.float64) - y = x + sgn * \ - np.random.uniform(1, 2, (2, 3, 1, 5)).astype(np.float64) + y = x + sgn * np.random.uniform(1, 2, (2, 3, 1, 5)).astype(np.float64) self.inputs = {'X': x, 'Y': y} self.outputs = {'Out': np.maximum(self.inputs['X'], self.inputs['Y'])} diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py index e4c08742b01c3..5a2cdc691faeb 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py @@ -23,7 +23,6 @@ class TestElementwiseOp(OpTest): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum @@ -49,22 +48,20 @@ def test_check_grad_normal(self): self.check_grad(['X', 'Y'], 'Out', check_eager=True) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad( + ['Y'], 'Out', max_relative_error=0.005, no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y')) + self.check_grad( + ['X'], 'Out', max_relative_error=0.005, no_grad_set=set('Y') + ) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMinOp_scalar(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum @@ -75,104 +72,104 @@ def setUp(self): class TestElementwiseMinOp_Vector(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum - x = np.random.random((100, )).astype("float64") - sgn = np.random.choice([-1, 1], (100, )).astype("float64") - y = x + sgn * np.random.uniform(0.1, 1, (100, )).astype("float64") + x = np.random.random((100,)).astype("float64") + sgn = np.random.choice([-1, 1], (100,)).astype("float64") + y = x + sgn * np.random.uniform(0.1, 1, (100,)).astype("float64") self.inputs = {'X': x, 'Y': y} self.outputs = {'Out': np.minimum(self.inputs['X'], self.inputs['Y'])} class TestElementwiseMinOp_broadcast_0(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum x = np.random.uniform(0.5, 1, (100, 3, 2)).astype(np.float64) - sgn = np.random.choice([-1, 1], (100, )).astype(np.float64) - y = x[:, 0, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(np.float64) + sgn = np.random.choice([-1, 1], (100,)).astype(np.float64) + y = x[:, 0, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 0} self.outputs = { - 'Out': np.minimum(self.inputs['X'], - self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestElementwiseMinOp_broadcast_1(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum x = np.random.uniform(0.5, 1, (2, 100, 3)).astype(np.float64) - sgn = np.random.choice([-1, 1], (100, )).astype(np.float64) - y = x[0, :, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(np.float64) + sgn = np.random.choice([-1, 1], (100,)).astype(np.float64) + y = x[0, :, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 1} self.outputs = { - 'Out': np.minimum(self.inputs['X'], - self.inputs['Y'].reshape(1, 100, 1)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1) + ) } class TestElementwiseMinOp_broadcast_2(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum x = np.random.uniform(0.5, 1, (2, 3, 100)).astype(np.float64) - sgn = np.random.choice([-1, 1], (100, )).astype(np.float64) - y = x[0, 0, :] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(np.float64) + sgn = np.random.choice([-1, 1], (100,)).astype(np.float64) + y = x[0, 0, :] + sgn * np.random.uniform(1, 2, (100,)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.outputs = { - 'Out': np.minimum(self.inputs['X'], - self.inputs['Y'].reshape(1, 1, 100)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100) + ) } class TestElementwiseMinOp_broadcast_3(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum x = np.random.uniform(0.5, 1, (2, 25, 4, 1)).astype(np.float64) sgn = np.random.choice([-1, 1], (25, 4)).astype(np.float64) - y = x[0, :, :, 0] + sgn * \ - np.random.uniform(1, 2, (25, 4)).astype(np.float64) + y = x[0, :, :, 0] + sgn * np.random.uniform(1, 2, (25, 4)).astype( + np.float64 + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.minimum(self.inputs['X'], self.inputs['Y'].reshape(1, 25, 4, 1)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(1, 25, 4, 1) + ) } class TestElementwiseMinOp_broadcast_4(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_min" self.python_api = paddle.minimum x = np.random.uniform(0.5, 1, (2, 10, 2, 5)).astype(np.float64) sgn = np.random.choice([-1, 1], (2, 10, 1, 5)).astype(np.float64) - y = x + sgn * \ - np.random.uniform(1, 2, (2, 10, 1, 5)).astype(np.float64) + y = x + sgn * np.random.uniform(1, 2, (2, 10, 1, 5)).astype(np.float64) self.inputs = {'X': x, 'Y': y} self.outputs = {'Out': np.minimum(self.inputs['X'], self.inputs['Y'])} class TestElementwiseMinOpFP16(unittest.TestCase): - def get_out_and_grad(self, x_np, y_np, axis, place, use_fp32=False): assert x_np.dtype == np.float16 assert y_np.dtype == np.float16 @@ -188,8 +185,11 @@ def get_out_and_grad(self, x_np, y_np, axis, place, use_fp32=False): y.stop_gradient = False z = fluid.layers.elementwise_min(x, y, axis) x_g, y_g = paddle.grad([z], [x, y]) - return z.numpy().astype(dtype), x_g.numpy().astype( - dtype), y_g.numpy().astype(dtype) + return ( + z.numpy().astype(dtype), + x_g.numpy().astype(dtype), + y_g.numpy().astype(dtype), + ) def check_main(self, x_shape, y_shape, axis=-1): if not paddle.is_compiled_with_cuda(): @@ -201,8 +201,9 @@ def check_main(self, x_shape, y_shape, axis=-1): x_np = np.random.random(size=x_shape).astype(np.float16) y_np = np.random.random(size=y_shape).astype(np.float16) - z_1, x_g_1, y_g_1 = self.get_out_and_grad(x_np, y_np, axis, place, - False) + z_1, x_g_1, y_g_1 = self.get_out_and_grad( + x_np, y_np, axis, place, False + ) z_2, x_g_2, y_g_2 = self.get_out_and_grad(x_np, y_np, axis, place, True) np.testing.assert_array_equal(z_1, z_2) np.testing.assert_array_equal(x_g_1, x_g_2) @@ -210,11 +211,11 @@ def check_main(self, x_shape, y_shape, axis=-1): def test_main(self): self.check_main((13, 17), (13, 17)) - self.check_main((10, 3, 4), (1, )) - self.check_main((100, ), (100, )) - self.check_main((100, 3, 2), (100, ), 0) - self.check_main((2, 100, 3), (100, ), 1) - self.check_main((2, 3, 100), (100, )) + self.check_main((10, 3, 4), (1,)) + self.check_main((100,), (100,)) + self.check_main((100, 3, 2), (100,), 0) + self.check_main((2, 100, 3), (100,), 1) + self.check_main((2, 3, 100), (100,)) self.check_main((2, 25, 4, 1), (25, 4), 1) self.check_main((2, 10, 2, 5), (2, 10, 1, 5)) diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py index 8c4ba799171fd..8969d76ce5165 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py @@ -22,7 +22,6 @@ class TestElementwiseModOp(OpTest): - def init_kernel_type(self): self.use_mkldnn = False @@ -37,7 +36,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -61,7 +60,6 @@ def init_axis(self): class TestElementwiseModOp_scalar(TestElementwiseModOp): - def init_input_output(self): scale_x = random.randint(0, 100000000) scale_y = random.randint(1, 100000000) @@ -71,7 +69,6 @@ def init_input_output(self): class TestElementwiseModOpFloat(TestElementwiseModOp): - def init_dtype(self): self.dtype = np.float32 @@ -88,7 +85,6 @@ def test_check_output(self): class TestElementwiseModOpFp16(TestElementwiseModOp): - def init_dtype(self): self.dtype = np.float16 @@ -105,13 +101,11 @@ def test_check_output(self): class TestElementwiseModOpDouble(TestElementwiseModOpFloat): - def init_dtype(self): self.dtype = np.float64 class TestRemainderOp(unittest.TestCase): - def _executed_api(self, x, y, name=None): return paddle.remainder(x, y, name) @@ -135,7 +129,7 @@ def test_dygraph(self): self.assertEqual((np_z == z_expected).all(), True) np_x = np.array([-3.3, 11.5, -2, 3.5]) - np_y = np.array([-1.2, 2., 3.3, -2.3]) + np_y = np.array([-1.2, 2.0, 3.3, -2.3]) x = paddle.to_tensor(np_x) y = paddle.to_tensor(np_y) z = x % y @@ -152,13 +146,11 @@ def test_dygraph(self): class TestRemainderInplaceOp(TestRemainderOp): - def _executed_api(self, x, y, name=None): return x.remainder_(y, name) class TestRemainderInplaceBroadcastSuccess(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 4).astype('float') self.y_numpy = np.random.rand(3, 4).astype('float') @@ -174,17 +166,17 @@ def test_broadcast_success(self): paddle.enable_static() -class TestRemainderInplaceBroadcastSuccess2(TestRemainderInplaceBroadcastSuccess - ): - +class TestRemainderInplaceBroadcastSuccess2( + TestRemainderInplaceBroadcastSuccess +): def init_data(self): self.x_numpy = np.random.rand(1, 2, 3, 1).astype('float') self.y_numpy = np.random.rand(3, 1).astype('float') -class TestRemainderInplaceBroadcastSuccess3(TestRemainderInplaceBroadcastSuccess - ): - +class TestRemainderInplaceBroadcastSuccess3( + TestRemainderInplaceBroadcastSuccess +): def init_data(self): self.x_numpy = np.random.rand(2, 3, 1, 5).astype('float') self.y_numpy = np.random.rand(1, 3, 1, 5).astype('float') diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py index 05ed9cc9ded03..cc3cd9be8236c 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py @@ -20,11 +20,14 @@ import paddle.fluid.core as core from paddle.fluid import Program, program_guard -from paddle.fluid.tests.unittests.op_test import OpTest, skip_check_grad_ci, convert_float_to_uint16 +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + skip_check_grad_ci, + convert_float_to_uint16, +) class ElementwiseMulOp(OpTest): - def init_kernel_type(self): self.use_mkldnn = False @@ -39,7 +42,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} @@ -50,23 +53,27 @@ def test_check_output(self): def test_check_grad_normal(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad(['X', 'Y'], - 'Out', - check_dygraph=(self.use_mkldnn == False)) + self.check_grad( + ['X', 'Y'], 'Out', check_dygraph=(self.use_mkldnn == False) + ) def test_check_grad_ingore_x(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_ingore_y(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - check_dygraph=(self.use_mkldnn == False)) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + check_dygraph=(self.use_mkldnn == False), + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -81,7 +88,6 @@ def init_axis(self): class TestBF16ElementwiseMulOp(OpTest): - def setUp(self): self.op_type = "elementwise_mul" self.dtype = np.uint16 @@ -93,9 +99,12 @@ def setUp(self): self.axis = -1 self.inputs = { - 'X': - OpTest.np_dtype_to_fluid_dtype(convert_float_to_uint16(self.x)), - 'Y': OpTest.np_dtype_to_fluid_dtype(convert_float_to_uint16(self.y)) + 'X': OpTest.np_dtype_to_fluid_dtype( + convert_float_to_uint16(self.x) + ), + 'Y': OpTest.np_dtype_to_fluid_dtype( + convert_float_to_uint16(self.y) + ), } self.outputs = {'Out': convert_float_to_uint16(self.out)} self.attrs = {'axis': self.axis, 'use_mkldnn': False} @@ -114,33 +123,31 @@ def test_check_grad_ingore_y(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseMulOp_scalar(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 3, 4).astype(np.float64), - 'Y': np.random.rand(1).astype(np.float64) + 'Y': np.random.rand(1).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_Vector(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { - 'X': np.random.random((100, )).astype("float64"), - 'Y': np.random.random((100, )).astype("float64") + 'X': np.random.random((100,)).astype("float64"), + 'Y': np.random.random((100,)).astype("float64"), } self.outputs = {'Out': np.multiply(self.inputs['X'], self.inputs['Y'])} self.init_kernel_type() class TestElementwiseMulOp_broadcast_0(ElementwiseMulOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -151,12 +158,11 @@ def init_axis(self): class TestElementwiseMulOp_broadcast_1(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 100, 3).astype(np.float64), - 'Y': np.random.rand(100).astype(np.float64) + 'Y': np.random.rand(100).astype(np.float64), } self.attrs = {'axis': 1} @@ -167,12 +173,11 @@ def setUp(self): class TestElementwiseMulOp_broadcast_2(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float64), - 'Y': np.random.rand(100).astype(np.float64) + 'Y': np.random.rand(100).astype(np.float64), } self.outputs = { @@ -182,12 +187,11 @@ def setUp(self): class TestElementwiseMulOp_broadcast_3(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 10, 12, 3).astype(np.float64), - 'Y': np.random.rand(10, 12).astype(np.float64) + 'Y': np.random.rand(10, 12).astype(np.float64), } self.attrs = {'axis': 1} @@ -198,68 +202,63 @@ def setUp(self): class TestElementwiseMulOp_broadcast_4(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 2, 11).astype(np.float64), - 'Y': np.random.rand(10, 1, 11).astype(np.float64) + 'Y': np.random.rand(10, 1, 11).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_broadcast_5(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 4, 2, 3).astype(np.float64), - 'Y': np.random.rand(10, 4, 1, 3).astype(np.float64) + 'Y': np.random.rand(10, 4, 1, 3).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestElementwiseMulOpFp16(ElementwiseMulOp): - def init_dtype(self): self.dtype = np.float16 class TestElementwiseMulOp_commonuse_1(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float64), - 'Y': np.random.rand(1, 1, 100).astype(np.float64) + 'Y': np.random.rand(1, 1, 100).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_commonuse_2(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(30, 3, 1, 5).astype(np.float64), - 'Y': np.random.rand(30, 1, 4, 1).astype(np.float64) + 'Y': np.random.rand(30, 1, 4, 1).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} self.init_kernel_type() class TestElementwiseMulOp_xsize_lessthan_ysize(ElementwiseMulOp): - def setUp(self): self.op_type = "elementwise_mul" self.inputs = { 'X': np.random.rand(10, 10).astype(np.float64), - 'Y': np.random.rand(2, 2, 10, 10).astype(np.float64) + 'Y': np.random.rand(2, 2, 10, 10).astype(np.float64), } self.attrs = {'axis': 2} @@ -271,14 +270,15 @@ def setUp(self): class TestElementwiseMulOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_mul must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.elementwise_mul, x1, y1) # the input dtype of elementwise_mul must be float16 or float32 or float64 or int32 or int64 @@ -289,7 +289,6 @@ def test_errors(self): class TestComplexElementwiseMulOp(OpTest): - def setUp(self): self.op_type = "elementwise_mul" self.init_base_dtype() @@ -298,7 +297,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': -1, 'use_mkldnn': False} self.outputs = {'Out': self.out} @@ -307,17 +306,18 @@ def init_base_dtype(self): self.dtype = np.float64 def init_input_output(self): - self.x = np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(self.dtype) - self.y = np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + self.x = np.random.random((2, 3, 4, 5)).astype( + self.dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(self.dtype) + self.y = np.random.random((2, 3, 4, 5)).astype( + self.dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(self.dtype) self.out = self.x * self.y def init_grad_input_output(self): - self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1J * np.ones( - (2, 3, 4, 5), self.dtype) + self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1j * np.ones( + (2, 3, 4, 5), self.dtype + ) self.grad_x = self.grad_out * np.conj(self.y) self.grad_y = self.grad_out * np.conj(self.x) @@ -325,38 +325,44 @@ def test_check_output(self): self.check_output() def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + ) class TestRealComplexElementwiseMulOp(TestComplexElementwiseMulOp): - def init_input_output(self): self.x = np.random.random((2, 3, 4, 5)).astype(self.dtype) - self.y = np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + 1J * np.random.random( - (2, 3, 4, 5)).astype(self.dtype) + self.y = np.random.random((2, 3, 4, 5)).astype( + self.dtype + ) + 1j * np.random.random((2, 3, 4, 5)).astype(self.dtype) self.out = self.x * self.y def init_grad_input_output(self): - self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1J * np.ones( - (2, 3, 4, 5), self.dtype) + self.grad_out = np.ones((2, 3, 4, 5), self.dtype) + 1j * np.ones( + (2, 3, 4, 5), self.dtype + ) self.grad_x = np.real(self.grad_out * np.conj(self.y)) self.grad_y = self.grad_out * np.conj(self.x) diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py b/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py index 875da9df79bfe..3f816b6165013 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py @@ -25,7 +25,6 @@ class TestElementwiseMulDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -41,11 +40,9 @@ def func(self, place): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -57,7 +54,6 @@ def test_grad(self): class TestElementwiseMulBroadcastDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -73,11 +69,9 @@ def func(self, place): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape[:-1]).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -89,7 +83,6 @@ def test_grad(self): class TestElementwiseAddDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -105,11 +98,9 @@ def func(self, place): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -121,7 +112,6 @@ def test_grad(self): class TestElementwiseAddBroadcastDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -137,11 +127,9 @@ def func(self, place): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape[:-1]).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -153,7 +141,6 @@ def test_grad(self): class TestElementwiseSubDoubleGradCheck(unittest.TestCase): - def subtract_wrapper(self, x): return paddle.subtract(x[0], x[1]) @@ -172,16 +159,16 @@ def func(self, place): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.subtract_wrapper, - [x, y], - out, - x_init=[x_arr, y_arr], - place=place) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.subtract_wrapper, + [x, y], + out, + x_init=[x_arr, y_arr], + place=place, + ) def test_grad(self): paddle.enable_static() @@ -193,7 +180,6 @@ def test_grad(self): class TestElementwiseSubBroadcastDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -209,11 +195,9 @@ def func(self, place): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape[:-1]).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -225,7 +209,6 @@ def test_grad(self): class TestElementwiseDivDoubleGradCheck(unittest.TestCase): - def divide_wrapper(self, x): return paddle.divide(x[0], x[1]) @@ -245,18 +228,17 @@ def func(self, place): y_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr[np.abs(y_arr) < 0.005] = 0.02 - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps, - atol=1e-3) - gradient_checker.double_grad_check_for_dygraph(self.divide_wrapper, - [x, y], - out, - x_init=[x_arr, y_arr], - place=place, - atol=1e-3) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps, atol=1e-3 + ) + gradient_checker.double_grad_check_for_dygraph( + self.divide_wrapper, + [x, y], + out, + x_init=[x_arr, y_arr], + place=place, + atol=1e-3, + ) def test_grad(self): paddle.enable_static() @@ -268,7 +250,6 @@ def test_grad(self): class TestElementwiseDivBroadcastDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -285,12 +266,9 @@ def func(self, place): y_arr = np.random.uniform(-1, 1, shape[1:-1]).astype(dtype) y_arr[np.abs(y_arr) < 0.005] = 0.02 - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps, - atol=1e-3) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps, atol=1e-3 + ) def test_grad(self): paddle.enable_static() @@ -302,7 +280,6 @@ def test_grad(self): class TestElementwiseAddTripleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -318,11 +295,9 @@ def func(self, place): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -334,7 +309,6 @@ def test_grad(self): class TestElementwiseAddBroadcastTripleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -350,11 +324,9 @@ def func(self, place): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape[:-1]).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -366,7 +338,6 @@ def test_grad(self): class TestElementwiseMulTripleGradCheck(unittest.TestCase): - def multiply_wrapper(self, x): return paddle.multiply(x[0], x[1]) @@ -385,17 +356,17 @@ def func(self, place): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.multiply_wrapper, - [x, y], - out, - x_init=[x_arr, y_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.multiply_wrapper, + [x, y], + out, + x_init=[x_arr, y_arr], + place=place, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_grad(self): @@ -408,7 +379,6 @@ def test_grad(self): class TestElementwiseMulBroadcastTripleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -424,11 +394,9 @@ def func(self, place): x_arr = np.random.uniform(-1, 1, shape).astype(dtype) y_arr = np.random.uniform(-1, 1, shape[:-1]).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py index 4050e4a4c607d..53cb18f8aa33b 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py @@ -26,13 +26,12 @@ def pow_grad(x, y, dout): class TestElementwisePowOp(OpTest): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(1, 2, [20, 5]).astype("float64"), - 'Y': np.random.uniform(1, 2, [20, 5]).astype("float64") + 'Y': np.random.uniform(1, 2, [20, 5]).astype("float64"), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} @@ -50,75 +49,70 @@ def test_check_grad_normal(self): class TestElementwisePowOp_big_shape_1(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(1, 2, [10, 10]).astype("float64"), - 'Y': np.random.uniform(0.1, 1, [10, 10]).astype("float64") + 'Y': np.random.uniform(0.1, 1, [10, 10]).astype("float64"), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_big_shape_2(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(1, 2, [10, 10]).astype("float64"), - 'Y': np.random.uniform(0.2, 2, [10, 10]).astype("float64") + 'Y': np.random.uniform(0.2, 2, [10, 10]).astype("float64"), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwisePowOp_scalar(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(0.1, 1, [3, 3, 4]).astype(np.float64), - 'Y': np.random.uniform(0.1, 1, [1]).astype(np.float64) + 'Y': np.random.uniform(0.1, 1, [1]).astype(np.float64), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_tensor(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(0.1, 1, [100]).astype("float64"), - 'Y': np.random.uniform(1, 3, [100]).astype("float64") + 'Y': np.random.uniform(1, 3, [100]).astype("float64"), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_broadcast_0(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 1, 100]).astype("float64"), - 'Y': np.random.uniform(0.1, 1, [100]).astype("float64") + 'Y': np.random.uniform(0.1, 1, [100]).astype("float64"), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_broadcast_1(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 100, 1]).astype("float64"), - 'Y': np.random.uniform(0.1, 1, [100]).astype("float64") + 'Y': np.random.uniform(0.1, 1, [100]).astype("float64"), } self.attrs = {'axis': 1} self.outputs = { @@ -127,51 +121,49 @@ def setUp(self): class TestElementwisePowOp_broadcast_2(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(0.1, 1, [100, 3, 1]).astype("float64"), - 'Y': np.random.uniform(0.1, 1, [100]).astype("float64") + 'Y': np.random.uniform(0.1, 1, [100]).astype("float64"), } self.attrs = {'axis': 0} self.outputs = { - 'Out': np.power(self.inputs['X'], - self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.power( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestElementwisePowOp_broadcast_3(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 20, 5, 1]).astype("float64"), - 'Y': np.random.uniform(0.1, 1, [20, 5]).astype("float64") + 'Y': np.random.uniform(0.1, 1, [20, 5]).astype("float64"), } self.attrs = {'axis': 1} self.outputs = { - 'Out': np.power(self.inputs['X'], - self.inputs['Y'].reshape(1, 20, 5, 1)) + 'Out': np.power( + self.inputs['X'], self.inputs['Y'].reshape(1, 20, 5, 1) + ) } class TestElementwisePowOp_broadcast_4(TestElementwisePowOp): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 10, 3, 5]).astype("float64"), - 'Y': np.random.uniform(0.1, 1, [2, 10, 1, 5]).astype("float64") + 'Y': np.random.uniform(0.1, 1, [2, 10, 1, 5]).astype("float64"), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOpInt(OpTest): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow @@ -186,7 +178,6 @@ def test_check_output(self): class TestElementwisePowGradOpInt(unittest.TestCase): - def setUp(self): self.x = np.asarray([1, 3, 6]) self.y = np.asarray([1, 1, 1]) @@ -194,11 +185,13 @@ def setUp(self): # dout = 1 self.grad_res = np.asarray([1, 1, 1]) # dx = dout * y * pow(x, y-1) - self.grad_x = self.grad_res * self.y * (self.x - **(self.y - 1)).astype("int") + self.grad_x = ( + self.grad_res * self.y * (self.x ** (self.y - 1)).astype("int") + ) # dy = dout * log(x) * pow(x, y) - self.grad_y = (self.grad_res * np.log(self.x) * - (self.x**self.y)).astype("int") + self.grad_y = ( + self.grad_res * np.log(self.x) * (self.x**self.y) + ).astype("int") def test_grad(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -220,13 +213,12 @@ def test_grad(self): class TestElementwisePowOpFP16(OpTest): - def setUp(self): self.op_type = "elementwise_pow" self.python_api = paddle.pow self.inputs = { 'X': np.random.uniform(1, 2, [20, 5]).astype("float16"), - 'Y': np.random.uniform(1, 2, [20, 5]).astype("float16") + 'Y': np.random.uniform(1, 2, [20, 5]).astype("float16"), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} @@ -237,12 +229,14 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=pow_grad(self.inputs['X'], - self.inputs['Y'], - 1 / self.inputs['X'].size), - check_eager=True) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=pow_grad( + self.inputs['X'], self.inputs['Y'], 1 / self.inputs['X'].size + ), + check_eager=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py index 9c8a0002bd03a..f8f050d6f6b08 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py @@ -21,12 +21,11 @@ class TestElementwiseOp(OpTest): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype("float64"), - 'Y': np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype("float64") + 'Y': np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype("float64"), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} @@ -37,20 +36,17 @@ def test_check_grad_normal(self): self.check_grad(['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad( + ['Y'], 'Out', max_relative_error=0.005, no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y')) + self.check_grad( + ['X'], 'Out', max_relative_error=0.005, no_grad_set=set('Y') + ) class TestBF16ElementwiseOp(OpTest): - def setUp(self): self.op_type = "elementwise_sub" self.dtype = np.uint16 @@ -60,7 +56,7 @@ def setUp(self): self.inputs = { 'X': convert_float_to_uint16(x), - 'Y': convert_float_to_uint16(y) + 'Y': convert_float_to_uint16(y), } self.outputs = {'Out': convert_float_to_uint16(out)} @@ -78,36 +74,34 @@ def test_check_grad_ingore_y(self): @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseSubOp_scalar(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(10, 3, 4).astype(np.float64), - 'Y': np.random.rand(1).astype(np.float64) + 'Y': np.random.rand(1).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_Vector(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { - 'X': np.random.random((100, )).astype("float64"), - 'Y': np.random.random((100, )).astype("float64") + 'X': np.random.random((100,)).astype("float64"), + 'Y': np.random.random((100,)).astype("float64"), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_broadcast_0(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(100, 3, 2).astype(np.float64), - 'Y': np.random.rand(100).astype(np.float64) + 'Y': np.random.rand(100).astype(np.float64), } self.attrs = {'axis': 0} @@ -117,12 +111,11 @@ def setUp(self): class TestElementwiseSubOp_broadcast_1(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 100, 3).astype(np.float64), - 'Y': np.random.rand(100).astype(np.float64) + 'Y': np.random.rand(100).astype(np.float64), } self.attrs = {'axis': 1} @@ -132,12 +125,11 @@ def setUp(self): class TestElementwiseSubOp_broadcast_2(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float64), - 'Y': np.random.rand(100).astype(np.float64) + 'Y': np.random.rand(100).astype(np.float64), } self.outputs = { @@ -146,12 +138,11 @@ def setUp(self): class TestElementwiseSubOp_broadcast_3(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 10, 12, 3).astype(np.float64), - 'Y': np.random.rand(10, 12).astype(np.float64) + 'Y': np.random.rand(10, 12).astype(np.float64), } self.attrs = {'axis': 1} @@ -161,45 +152,41 @@ def setUp(self): class TestElementwiseSubOp_broadcast_4(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 5, 3, 12).astype(np.float64), - 'Y': np.random.rand(2, 5, 1, 12).astype(np.float64) + 'Y': np.random.rand(2, 5, 1, 12).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_commonuse_1(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(2, 3, 100).astype(np.float64), - 'Y': np.random.rand(1, 1, 100).astype(np.float64) + 'Y': np.random.rand(1, 1, 100).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_commonuse_2(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(10, 3, 1, 4).astype(np.float64), - 'Y': np.random.rand(10, 1, 12, 1).astype(np.float64) + 'Y': np.random.rand(10, 1, 12, 1).astype(np.float64), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_xsize_lessthan_ysize(TestElementwiseOp): - def setUp(self): self.op_type = "elementwise_sub" self.inputs = { 'X': np.random.rand(10, 12).astype(np.float64), - 'Y': np.random.rand(2, 3, 10, 12).astype(np.float64) + 'Y': np.random.rand(2, 3, 10, 12).astype(np.float64), } self.attrs = {'axis': 2} @@ -210,7 +197,6 @@ def setUp(self): class TestComplexElementwiseSubOp(OpTest): - def setUp(self): self.op_type = "elementwise_sub" self.dtype = np.float64 @@ -220,7 +206,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': -1, 'use_mkldnn': False} self.outputs = {'Out': self.out} @@ -230,14 +216,17 @@ def init_base_dtype(self): def init_input_output(self): self.x = np.random.random(self.shape).astype( - self.dtype) + 1J * np.random.random(self.shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.shape).astype(self.dtype) self.y = np.random.random(self.shape).astype( - self.dtype) + 1J * np.random.random(self.shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.shape).astype(self.dtype) self.out = self.x - self.y def init_grad_input_output(self): - self.grad_out = np.ones( - self.shape, self.dtype) + 1J * np.ones(self.shape, self.dtype) + self.grad_out = np.ones(self.shape, self.dtype) + 1j * np.ones( + self.shape, self.dtype + ) self.grad_x = self.grad_out self.grad_y = -self.grad_out @@ -245,43 +234,49 @@ def test_check_output(self): self.check_output() def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out]) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + ) class TestRealComplexElementwiseSubOp(TestComplexElementwiseSubOp): - def init_input_output(self): self.x = np.random.random(self.shape).astype(self.dtype) self.y = np.random.random(self.shape).astype( - self.dtype) + 1J * np.random.random(self.shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.shape).astype(self.dtype) self.out = self.x - self.y def init_grad_input_output(self): - self.grad_out = np.ones( - self.shape, self.dtype) + 1J * np.ones(self.shape, self.dtype) + self.grad_out = np.ones(self.shape, self.dtype) + 1j * np.ones( + self.shape, self.dtype + ) self.grad_x = np.real(self.grad_out) self.grad_y = -self.grad_out class TestSubtractApi(unittest.TestCase): - def _executed_api(self, x, y, name=None): return paddle.subtract(x, y, name) @@ -299,7 +294,7 @@ def test_declarative(self): def gen_data(): return { "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') + "y": np.array([1, 5, 2]).astype('float32'), } x = fluid.data(name="x", shape=[3], dtype='float32') @@ -308,7 +303,7 @@ def gen_data(): place = fluid.CPUPlace() exe = fluid.Executor(place) z_value = exe.run(feed=gen_data(), fetch_list=[z.name]) - z_expected = np.array([1., -2., 2.]) + z_expected = np.array([1.0, -2.0, 2.0]) self.assertEqual((z_value == z_expected).all(), True) def test_dygraph(self): @@ -319,18 +314,16 @@ def test_dygraph(self): y = fluid.dygraph.to_variable(np_y) z = self._executed_api(x, y) np_z = z.numpy() - z_expected = np.array([1., -2., 2.]) + z_expected = np.array([1.0, -2.0, 2.0]) self.assertEqual((np_z == z_expected).all(), True) class TestSubtractInplaceApi(TestSubtractApi): - def _executed_api(self, x, y, name=None): return x.subtract_(y, name) class TestSubtractInplaceBroadcastSuccess(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 4).astype('float') self.y_numpy = np.random.rand(3, 4).astype('float') @@ -347,21 +340,18 @@ def test_broadcast_success(self): class TestSubtractInplaceBroadcastSuccess2(TestSubtractInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(1, 2, 3, 1).astype('float') self.y_numpy = np.random.rand(3, 1).astype('float') class TestSubtractInplaceBroadcastSuccess3(TestSubtractInplaceBroadcastSuccess): - def init_data(self): self.x_numpy = np.random.rand(2, 3, 1, 5).astype('float') self.y_numpy = np.random.rand(1, 3, 1, 5).astype('float') class TestSubtractInplaceBroadcastError(unittest.TestCase): - def init_data(self): self.x_numpy = np.random.rand(3, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') @@ -380,21 +370,18 @@ def broadcast_shape_error(): class TestSubtractInplaceBroadcastError2(TestSubtractInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(2, 1, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') class TestSubtractInplaceBroadcastError3(TestSubtractInplaceBroadcastError): - def init_data(self): self.x_numpy = np.random.rand(5, 2, 1, 4).astype('float') self.y_numpy = np.random.rand(2, 3, 4).astype('float') class TestFloatElementwiseSubop(unittest.TestCase): - def func_dygraph_sub(self): paddle.disable_static() @@ -407,26 +394,23 @@ def func_dygraph_sub(self): # normal case: tensor - tensor expect_out = np_a - np_b actual_out = tensor_a - tensor_b - np.testing.assert_allclose(actual_out, - expect_out, - rtol=1e-07, - atol=1e-07) + np.testing.assert_allclose( + actual_out, expect_out, rtol=1e-07, atol=1e-07 + ) # normal case: tensor - scalar expect_out = np_a - 1 actual_out = tensor_a - 1 - np.testing.assert_allclose(actual_out, - expect_out, - rtol=1e-07, - atol=1e-07) + np.testing.assert_allclose( + actual_out, expect_out, rtol=1e-07, atol=1e-07 + ) # normal case: scalar - tenor expect_out = 1 - np_a actual_out = 1 - tensor_a - np.testing.assert_allclose(actual_out, - expect_out, - rtol=1e-07, - atol=1e-07) + np.testing.assert_allclose( + actual_out, expect_out, rtol=1e-07, atol=1e-07 + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_ema.py b/python/paddle/fluid/tests/unittests/test_ema.py index 1e2434a6a5256..fe4178c825980 100644 --- a/python/paddle/fluid/tests/unittests/test_ema.py +++ b/python/paddle/fluid/tests/unittests/test_ema.py @@ -19,7 +19,6 @@ class TestExponentialMovingAverage(unittest.TestCase): - def setUp(self): self._places = [fluid.CPUPlace()] if fluid.core.is_compiled_with_cuda(): @@ -32,19 +31,21 @@ def setUp(self): with fluid.program_guard(self._train_program, self._startup_prog): with fluid.unique_name.guard(): data = fluid.data(name='x', shape=[-1, 5], dtype='float32') - hidden = fluid.layers.fc(input=data, - size=10, - param_attr=self._param_name) + hidden = fluid.layers.fc( + input=data, size=10, param_attr=self._param_name + ) cost = paddle.mean(hidden) self._test_program = fluid.default_main_program().clone( - for_test=True) + for_test=True + ) optimizer = fluid.optimizer.Adam(learning_rate=0.001) optimizer.minimize(cost) self._ema = fluid.optimizer.ExponentialMovingAverage( - self._ema_decay) + self._ema_decay + ) self._ema.update() def train(self, place): @@ -55,16 +56,19 @@ def train(self, place): for pass_id in range(2): for batch_id in range(3): data = np.random.random(size=(10, 5)).astype('float32') - tmp_param = np.array(fluid.global_scope().find_var( - self._param_name).get_tensor()) + tmp_param = np.array( + fluid.global_scope().find_var(self._param_name).get_tensor() + ) exe.run(program=self._train_program, feed={'x': data}) - tmp_param = np.array(fluid.global_scope().find_var( - self._param_name).get_tensor()) + tmp_param = np.array( + fluid.global_scope().find_var(self._param_name).get_tensor() + ) params.append(tmp_param) with self._ema.apply(exe): - final_ema = np.array(fluid.global_scope().find_var( - self._param_name).get_tensor()) + final_ema = np.array( + fluid.global_scope().find_var(self._param_name).get_tensor() + ) data = np.random.random(size=(10, 5)).astype('float32') exe.run(program=self._test_program, feed={'x': data}) return params, final_ema @@ -75,9 +79,11 @@ def test_check_ema(self): manu_ema = np.zeros_like(final_ema) if len(params) > 0: for param in params: - manu_ema = self._ema_decay * manu_ema + ( - 1 - self._ema_decay) * param - manu_ema = manu_ema / (1.0 - self._ema_decay**len(params)) + manu_ema = ( + self._ema_decay * manu_ema + + (1 - self._ema_decay) * param + ) + manu_ema = manu_ema / (1.0 - self._ema_decay ** len(params)) np.testing.assert_allclose(manu_ema, final_ema, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_ema_fleet.py b/python/paddle/fluid/tests/unittests/test_ema_fleet.py index 206e3cf736277..8a3a6993d050d 100644 --- a/python/paddle/fluid/tests/unittests/test_ema_fleet.py +++ b/python/paddle/fluid/tests/unittests/test_ema_fleet.py @@ -24,7 +24,6 @@ def gen_data(): class TestFleetStaticEMA(unittest.TestCase): - def setUp(self): self._places = [paddle.CPUPlace()] if paddle.device.is_compiled_with_cuda(): @@ -41,17 +40,19 @@ def setUp(self): with static.program_guard(self._train_program, self._startup_prog): with utils.unique_name.guard(): data = static.data(name='x', shape=[-1, 5], dtype='float32') - hidden = static.nn.fc(x=data, - size=10, - weight_attr=self._param_name) + hidden = static.nn.fc( + x=data, size=10, weight_attr=self._param_name + ) cost = paddle.mean(hidden) self._test_program = static.default_main_program().clone( - for_test=True) + for_test=True + ) optimizer = paddle.optimizer.Adam(learning_rate=0.001) optimizer = paddle.distributed.fleet.distributed_optimizer( - optimizer, strategy) + optimizer, strategy + ) optimizer.minimize(cost) self._ema = static.ExponentialMovingAverage(self._ema_decay) @@ -65,13 +66,19 @@ def train(self, place, restore): for pass_id in range(2): for batch_id in range(3): exe.run(program=self._train_program, feed={'x': gen_data()}) - tmp_param = np.array(static.global_scope().find_var( - self._param_name).get_tensor()) + tmp_param = np.array( + static.global_scope() + .find_var(self._param_name) + .get_tensor() + ) params.append(tmp_param) with self._ema.apply(exe, restore): - final_ema = np.array(static.global_scope().find_var( - self._param_name).get_tensor()) + final_ema = np.array( + static.global_scope() + .find_var(self._param_name) + .get_tensor() + ) exe.run(program=self._test_program, feed={'x': gen_data()}) if not restore: self._ema.restore(exe) @@ -85,9 +92,11 @@ def test_check_ema(self): manu_ema = np.zeros_like(final_ema) if len(params) > 0: for param in params: - manu_ema = self._ema_decay * manu_ema + ( - 1 - self._ema_decay) * param - manu_ema = manu_ema / (1.0 - self._ema_decay**len(params)) + manu_ema = ( + self._ema_decay * manu_ema + + (1 - self._ema_decay) * param + ) + manu_ema = manu_ema / (1.0 - self._ema_decay ** len(params)) np.testing.assert_allclose(manu_ema, final_ema, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_embedding_id_stop_gradient.py b/python/paddle/fluid/tests/unittests/test_embedding_id_stop_gradient.py index db3af060d2645..68f9696ffe226 100644 --- a/python/paddle/fluid/tests/unittests/test_embedding_id_stop_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_embedding_id_stop_gradient.py @@ -19,7 +19,6 @@ class TestEmbeddingIdStopGradientBase(unittest.TestCase): - def setUp(self): self.reshape_times = 1 self.iteration = 10 @@ -70,17 +69,15 @@ def run_program(self, place, stop_gradient=False): fetch_val = None for _ in range(self.iteration): - fetch_val = exe.run(feed={ - x_1.name: x1_data, - x_2.name: x2_data - }, - fetch_list=[emb])[0] + fetch_val = exe.run( + feed={x_1.name: x1_data, x_2.name: x2_data}, + fetch_list=[emb], + )[0] return fetch_val class TestEmbeddingIdStopGradient2(TestEmbeddingIdStopGradientBase): - def setUp(self): self.reshape_times = 100 self.iteration = 10 diff --git a/python/paddle/fluid/tests/unittests/test_empty_like_op.py b/python/paddle/fluid/tests/unittests/test_empty_like_op.py index 6c1d179fa4efc..4ce4ab6a6d527 100644 --- a/python/paddle/fluid/tests/unittests/test_empty_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_empty_like_op.py @@ -21,37 +21,43 @@ class TestEmptyLikeAPICommon(unittest.TestCase): - def __check_out__(self, out): data_type = convert_dtype(out.dtype) self.assertEqual( - data_type, self.dst_dtype, - 'dtype should be %s, but get %s' % (self.dst_dtype, data_type)) + data_type, + self.dst_dtype, + 'dtype should be %s, but get %s' % (self.dst_dtype, data_type), + ) shape = out.shape self.assertTupleEqual( - shape, self.dst_shape, - 'shape should be %s, but get %s' % (self.dst_shape, shape)) + shape, + self.dst_shape, + 'shape should be %s, but get %s' % (self.dst_shape, shape), + ) if data_type in ['float32', 'float64', 'int32', 'int64']: max_value = np.nanmax(out) min_value = np.nanmin(out) always_non_full_zero = max_value >= min_value always_full_zero = max_value == 0.0 and min_value == 0.0 - self.assertTrue(always_full_zero or always_non_full_zero, - 'always_full_zero or always_non_full_zero.') + self.assertTrue( + always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.', + ) elif data_type in ['bool']: total_num = out.size true_num = np.sum(out == True) false_num = np.sum(out == False) - self.assertTrue(total_num == true_num + false_num, - 'The value should always be True or False.') + self.assertTrue( + total_num == true_num + false_num, + 'The value should always be True or False.', + ) else: self.assertTrue(False, 'invalid data type') class TestEmptyLikeAPI(TestEmptyLikeAPICommon): - def setUp(self): self.init_config() @@ -69,7 +75,6 @@ def init_config(self): class TestEmptyLikeAPI2(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("float64") self.dtype = self.x.dtype @@ -78,7 +83,6 @@ def init_config(self): class TestEmptyLikeAPI3(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("int") self.dtype = self.x.dtype @@ -87,7 +91,6 @@ def init_config(self): class TestEmptyLikeAPI4(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("int64") self.dtype = self.x.dtype @@ -96,7 +99,6 @@ def init_config(self): class TestEmptyLikeAPI5(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("bool") self.dtype = self.x.dtype @@ -105,7 +107,6 @@ def init_config(self): class TestEmptyLikeAPI6(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("float64") self.dtype = "float32" @@ -114,7 +115,6 @@ def init_config(self): class TestEmptyLikeAPI7(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("int") self.dtype = "float32" @@ -123,7 +123,6 @@ def init_config(self): class TestEmptyLikeAPI8(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("int64") self.dtype = "float32" @@ -132,7 +131,6 @@ def init_config(self): class TestEmptyLikeAPI9(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("bool") self.dtype = "float32" @@ -141,7 +139,6 @@ def init_config(self): class TestEmptyLikeAPI10(TestEmptyLikeAPI): - def init_config(self): self.x = np.random.random((200, 3)).astype("float32") self.dtype = "bool" @@ -150,7 +147,6 @@ def init_config(self): class TestEmptyLikeAPI_Static(TestEmptyLikeAPICommon): - def setUp(self): self.init_config() @@ -164,14 +160,17 @@ def test_static_graph(self): with program_guard(train_program, startup_program): x = np.random.random(self.x_shape).astype(dtype) - data_x = paddle.static.data('x', - shape=self.data_x_shape, - dtype=dtype) + data_x = paddle.static.data( + 'x', shape=self.data_x_shape, dtype=dtype + ) out = paddle.empty_like(data_x) - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) res = exe.run(train_program, feed={'x': x}, fetch_list=[out]) @@ -187,16 +186,13 @@ def init_config(self): class TestEmptyLikeAPI_Static2(TestEmptyLikeAPI_Static): - def init_config(self): self.x_shape = (3, 200, 3) self.data_x_shape = [-1, 200, 3] class TestEmptyError(unittest.TestCase): - def test_attr(self): - def test_dtype(): x = np.random.random((200, 3)).astype("float64") dtype = 'uint8' diff --git a/python/paddle/fluid/tests/unittests/test_empty_op.py b/python/paddle/fluid/tests/unittests/test_empty_op.py index d9994110f1d27..11b66325c1f5b 100644 --- a/python/paddle/fluid/tests/unittests/test_empty_op.py +++ b/python/paddle/fluid/tests/unittests/test_empty_op.py @@ -1,4 +1,4 @@ -#Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,7 +22,6 @@ # Situation 1: Attr(shape) is a list(without tensor) class TestEmptyOp(OpTest): - def setUp(self): self.op_type = "empty" self.init_config() @@ -38,14 +37,18 @@ def verify_output(self, outs): always_full_zero = max_value == 0.0 and min_value == 0.0 always_non_full_zero = max_value >= min_value - self.assertTrue(always_full_zero or always_non_full_zero, - 'always_full_zero or always_non_full_zero.') + self.assertTrue( + always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.', + ) elif data_type in ['bool']: total_num = outs[0].size true_num = np.sum(outs[0] == True) false_num = np.sum(outs[0] == False) - self.assertTrue(total_num == true_num + false_num, - 'The value should always be True or False.') + self.assertTrue( + total_num == true_num + false_num, + 'The value should always be True or False.', + ) else: self.assertTrue(False, 'invalid data type') @@ -59,7 +62,6 @@ def init_config(self): class TestEmptyOp2(TestEmptyOp): - def init_config(self): shape = [500, 3] dtype = 'float64' @@ -70,7 +72,6 @@ def init_config(self): class TestEmptyOp3(TestEmptyOp): - def init_config(self): shape = [500, 3] dtype = 'int32' @@ -81,7 +82,6 @@ def init_config(self): class TestEmptyOp4(TestEmptyOp): - def init_config(self): shape = [500, 3] dtype = 'int64' @@ -92,7 +92,6 @@ def init_config(self): class TestEmptyOp5(TestEmptyOp): - def init_config(self): shape = [500, 3] dtype = 'bool' @@ -104,7 +103,6 @@ def init_config(self): # Situation 2: shape is a tensor class TestEmptyOp_ShapeTensor(OpTest): - def setUp(self): self.op_type = "empty" self.init_config() @@ -128,21 +126,24 @@ def verify_output(self, outs): always_full_zero = max_value == 0.0 and min_value == 0.0 always_non_full_zero = max_value >= min_value - self.assertTrue(always_full_zero or always_non_full_zero, - 'always_full_zero or always_non_full_zero.') + self.assertTrue( + always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.', + ) elif data_type in ['bool']: total_num = outs[0].size true_num = np.sum(outs[0] == True) false_num = np.sum(outs[0] == False) - self.assertTrue(total_num == true_num + false_num, - 'The value should always be True or False.') + self.assertTrue( + total_num == true_num + false_num, + 'The value should always be True or False.', + ) else: self.assertTrue(False, 'invalid data type') # Situation 3: Attr(shape) is a list(with tensor) class TestEmptyOp_ShapeTensorList(OpTest): - def setUp(self): self.op_type = "empty" self.init_config() @@ -156,8 +157,9 @@ def init_config(self): shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} self.attrs = {'shape': self.infer_shape, 'dtype': dtype_inner} @@ -174,27 +176,32 @@ def verify_output(self, outs): always_full_zero = max_value == 0.0 and min_value == 0.0 always_non_full_zero = max_value >= min_value - self.assertTrue(always_full_zero or always_non_full_zero, - 'always_full_zero or always_non_full_zero.') + self.assertTrue( + always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.', + ) elif data_type in ['bool']: total_num = outs[0].size true_num = np.sum(outs[0] == True) false_num = np.sum(outs[0] == False) - self.assertTrue(total_num == true_num + false_num, - 'The value should always be True or False.') + self.assertTrue( + total_num == true_num + false_num, + 'The value should always be True or False.', + ) else: self.assertTrue(False, 'invalid data type') class TestEmptyAPI(unittest.TestCase): - def __check_out__(self, out, dtype='float32'): max_value = np.nanmax(np.array(out)) min_value = np.nanmin(np.array(out)) always_non_full_zero = max_value >= min_value always_full_zero = max_value == 0.0 and min_value == 0.0 - self.assertTrue(always_full_zero or always_non_full_zero, - 'always_full_zero or always_non_full_zero.') + self.assertTrue( + always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.', + ) def test_dygraph_api_out(self): paddle.disable_static() @@ -233,15 +240,15 @@ def test_static_graph(self): positive_2_int32 = fluid.layers.fill_constant([1], "int32", 3) positive_2_int64 = fluid.layers.fill_constant([1], "int64", 3) - shape_tensor_int32 = fluid.data(name="shape_tensor_int32", - shape=[2], - dtype="int32") - shape_tensor_int64 = fluid.data(name="shape_tensor_int64", - shape=[2], - dtype="int64") - shape_tensor_unknown = fluid.data(name="shape_tensor_unknown", - shape=[-1], - dtype="int64") + shape_tensor_int32 = fluid.data( + name="shape_tensor_int32", shape=[2], dtype="int32" + ) + shape_tensor_int64 = fluid.data( + name="shape_tensor_int64", shape=[2], dtype="int64" + ) + shape_tensor_unknown = fluid.data( + name="shape_tensor_unknown", shape=[-1], dtype="int64" + ) out_1 = paddle.empty(shape=[200, 3], dtype=dtype) out_2 = paddle.empty(shape=shape_tensor_int32, dtype=dtype) @@ -259,7 +266,8 @@ def test_static_graph(self): "shape_tensor_int64": np.array([200, 3]).astype("int64"), "shape_tensor_unknown": np.array([200, 3]).astype("int64"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6], + ) self.__check_out__(res_1, dtype) self.__check_out__(res_2, dtype) @@ -270,9 +278,7 @@ def test_static_graph(self): class TestEmptyError(unittest.TestCase): - def test_attr(self): - def test_dtype(): shape = [200, 3] dtype = 'uint8' diff --git a/python/paddle/fluid/tests/unittests/test_entry_attr.py b/python/paddle/fluid/tests/unittests/test_entry_attr.py index eb5bd30d1c211..cfbff2113c304 100644 --- a/python/paddle/fluid/tests/unittests/test_entry_attr.py +++ b/python/paddle/fluid/tests/unittests/test_entry_attr.py @@ -18,14 +18,18 @@ import unittest import paddle.fluid as fluid -from paddle.distributed import ProbabilityEntry, CountFilterEntry, ShowClickEntry +from paddle.distributed import ( + ProbabilityEntry, + CountFilterEntry, + ShowClickEntry, +) class EntryAttrChecks(unittest.TestCase): - def base(self): with self.assertRaises(NotImplementedError): from paddle.distributed.entry_attr import EntryAttr + base = EntryAttr() base._to_attr() @@ -62,18 +66,21 @@ def spaese_layer(self): with fluid.scope_guard(scope): with fluid.program_guard(prog): - input = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) + input = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) prob = ProbabilityEntry(0.5) emb = paddle.static.nn.sparse_embedding( input=input, size=[100, 10], is_test=False, entry=prob, - param_attr=fluid.ParamAttr(name="deep_embedding")) + param_attr=fluid.ParamAttr(name="deep_embedding"), + ) pool = fluid.layers.sequence_pool(input=emb, pool_type="sum") predict = fluid.layers.fc(input=pool, size=2, act='softmax') @@ -92,7 +99,6 @@ def spaese_layer(self): class TestEntryAttrs(EntryAttrChecks): - def test_base(self): self.base() diff --git a/python/paddle/fluid/tests/unittests/test_entry_attr2.py b/python/paddle/fluid/tests/unittests/test_entry_attr2.py index b65fd12eba7d6..f4f098798ac01 100644 --- a/python/paddle/fluid/tests/unittests/test_entry_attr2.py +++ b/python/paddle/fluid/tests/unittests/test_entry_attr2.py @@ -21,24 +21,26 @@ class EntryAttrChecks(unittest.TestCase): - def embedding_layer(self): prog = fluid.Program() scope = fluid.core.Scope() with fluid.scope_guard(scope): with fluid.program_guard(prog): - input = fluid.layers.data(name="dnn_data", - shape=[-1, 1], - dtype="int64", - lod_level=1, - append_batch_size=False) + input = fluid.layers.data( + name="dnn_data", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) emb = fluid.layers.embedding( input=input, size=[100, 10], is_sparse=True, is_distributed=True, - param_attr=fluid.ParamAttr(name="deep_embedding")) + param_attr=fluid.ParamAttr(name="deep_embedding"), + ) pool = fluid.layers.sequence_pool(input=emb, pool_type="sum") predict = fluid.layers.fc(input=pool, size=2, act='softmax') @@ -53,7 +55,6 @@ def embedding_layer(self): class TestEntryAttrs(EntryAttrChecks): - def test_embedding_layer(self): self.embedding_layer() diff --git a/python/paddle/fluid/tests/unittests/test_erf_op.py b/python/paddle/fluid/tests/unittests/test_erf_op.py index 338ca508db666..089fdc0a0b4bc 100644 --- a/python/paddle/fluid/tests/unittests/test_erf_op.py +++ b/python/paddle/fluid/tests/unittests/test_erf_op.py @@ -23,7 +23,6 @@ class TestErfOp(OpTest): - def setUp(self): self.op_type = "erf" self.dtype = self._init_dtype() @@ -44,7 +43,6 @@ def test_check_grad(self): class TestErfLayer(unittest.TestCase): - def _test_case(self, place): x = np.random.uniform(-1, 1, size=(11, 17)).astype(np.float64) y_ref = erf(x) diff --git a/python/paddle/fluid/tests/unittests/test_erfinv_op.py b/python/paddle/fluid/tests/unittests/test_erfinv_op.py index 8822b3e07650d..e605d8e0e62a4 100644 --- a/python/paddle/fluid/tests/unittests/test_erfinv_op.py +++ b/python/paddle/fluid/tests/unittests/test_erfinv_op.py @@ -24,7 +24,6 @@ class TestErfinv(OpTest): - def setUp(self): self.op_type = "erfinv" self.python_api = paddle.erfinv @@ -33,8 +32,9 @@ def setUp(self): self.x = np.random.uniform(-1, 1, size=self.shape).astype(self.dtype) self.res_ref = erfinv(self.x).astype(self.dtype) self.grad_out = np.ones(self.shape, self.dtype) - self.gradient = np.sqrt(np.pi) / 2 * np.exp(np.square( - self.res_ref)) * self.grad_out + self.gradient = ( + np.sqrt(np.pi) / 2 * np.exp(np.square(self.res_ref)) * self.grad_out + ) self.inputs = {'X': self.x} self.outputs = {'Out': self.res_ref} @@ -45,20 +45,20 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.gradient], - user_defined_grad_outputs=self.grad_out) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[self.gradient], + user_defined_grad_outputs=self.grad_out, + ) class TestErfinvFP32(TestErfinv): - def init_dtype(self): self.dtype = np.float32 class TestErfinvAPI(unittest.TestCase): - def init_dtype(self): self.dtype = 'float32' @@ -86,7 +86,6 @@ def run(place): run(place) def test_dygraph_api(self): - def run(place): paddle.disable_static(place) x = paddle.to_tensor(self.x) @@ -98,7 +97,6 @@ def run(place): run(place) def test_inplace_api(self): - def run(place): paddle.disable_static(place) x = paddle.to_tensor(self.x) diff --git a/python/paddle/fluid/tests/unittests/test_exception.py b/python/paddle/fluid/tests/unittests/test_exception.py index 6d39f9f5f41c5..bb53ae950287e 100644 --- a/python/paddle/fluid/tests/unittests/test_exception.py +++ b/python/paddle/fluid/tests/unittests/test_exception.py @@ -21,7 +21,6 @@ class TestException(unittest.TestCase): - def test_exception(self): exception = None try: @@ -34,7 +33,6 @@ def test_exception(self): class TestExceptionNoCStack(unittest.TestCase): - def setUp(self): paddle.enable_static() # test no C++ stack format @@ -57,12 +55,11 @@ def test_exception_in_static_mode(self): y = numpy.random.random(size=(8, 1)).astype('float32') with self.assertRaises(ValueError): - exe.run(fluid.default_main_program(), - feed={ - 'X': x, - 'Y': y - }, - fetch_list=[avg_loss.name]) + exe.run( + fluid.default_main_program(), + feed={'X': x, 'Y': y}, + fetch_list=[avg_loss.name], + ) def test_exception_in_dynamic_mode(self): place = fluid.CPUPlace() diff --git a/python/paddle/fluid/tests/unittests/test_executor_and_mul.py b/python/paddle/fluid/tests/unittests/test_executor_and_mul.py index a9cd5c4f587ae..f85f7b97ec857 100644 --- a/python/paddle/fluid/tests/unittests/test_executor_and_mul.py +++ b/python/paddle/fluid/tests/unittests/test_executor_and_mul.py @@ -20,17 +20,15 @@ class TestExecutor(unittest.TestCase): - def test_mul(self): i = zeros(shape=[1], dtype='int64') a = data(name='a', shape=[784], dtype='float32') array = array_write(x=a, i=i) i = increment(i) - b = data(name='b', - shape=[784, 100], - dtype='float32', - append_batch_size=False) + b = data( + name='b', shape=[784, 100], dtype='float32', append_batch_size=False + ) array_write(x=b, i=i, array=array) i = increment(i) @@ -41,11 +39,9 @@ def test_mul(self): b_np = np.random.random((784, 100)).astype('float32') exe = Executor() - res, res_array = exe.run(feed={ - 'a': a_np, - 'b': b_np - }, - fetch_list=[out, array]) + res, res_array = exe.run( + feed={'a': a_np, 'b': b_np}, fetch_list=[out, array] + ) self.assertEqual((100, 100), res.shape) np.testing.assert_allclose(res, np.dot(a_np, b_np), rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_executor_and_use_program_cache.py b/python/paddle/fluid/tests/unittests/test_executor_and_use_program_cache.py index 34f9674f27d89..2412794929c05 100644 --- a/python/paddle/fluid/tests/unittests/test_executor_and_use_program_cache.py +++ b/python/paddle/fluid/tests/unittests/test_executor_and_use_program_cache.py @@ -21,16 +21,17 @@ class TestExecutor(unittest.TestCase): - def test_mul(self): main_program = fluid.Program() startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): a = fluid.layers.data(name='a', shape=[784], dtype='float32') - b = fluid.layers.data(name='b', - shape=[784, 100], - dtype='float32', - append_batch_size=False) + b = fluid.layers.data( + name='b', + shape=[784, 100], + dtype='float32', + append_batch_size=False, + ) output = fluid.layers.mul(x=a, y=b) # Compute with numpy @@ -47,13 +48,12 @@ def _train(use_program_cache, max_iters=1): run_time = 0.0 for i in range(max_iters): begin = time.time() - outs = exe.run(program=main_program, - feed={ - 'a': a_np, - 'b': b_np - }, - fetch_list=[output.name], - use_program_cache=use_program_cache) + outs = exe.run( + program=main_program, + feed={'a': a_np, 'b': b_np}, + fetch_list=[output.name], + use_program_cache=use_program_cache, + ) end = time.time() run_time += end - begin out = outs[0] @@ -62,29 +62,31 @@ def _train(use_program_cache, max_iters=1): return run_time max_iters = 3 - run_time_with_cache = _train(use_program_cache=True, - max_iters=max_iters) + run_time_with_cache = _train( + use_program_cache=True, max_iters=max_iters + ) print("run time with program cache: %f" % run_time_with_cache) - run_time_without_cache = _train(use_program_cache=False, - max_iters=max_iters) + run_time_without_cache = _train( + use_program_cache=False, max_iters=max_iters + ) print("run time without program cache: %f" % run_time_without_cache) - run_time_with_cache = _train(use_program_cache=True, - max_iters=max_iters) + run_time_with_cache = _train( + use_program_cache=True, max_iters=max_iters + ) print("run time with program cache: %f" % run_time_with_cache) - run_time_with_cache = _train(use_program_cache=True, - max_iters=max_iters) + run_time_with_cache = _train( + use_program_cache=True, max_iters=max_iters + ) print("run time with program cache: %f" % run_time_with_cache) class ExecutorPaddingRNNTest(PaddingRNNTestBase): - - def train_and_save_inference_program(self, - rnn_model="static", - parallel=True, - use_program_cache=True): + def train_and_save_inference_program( + self, rnn_model="static", parallel=True, use_program_cache=True + ): config = RNNConfig("test", rnn_model) with fluid.scope_guard(fluid.Scope()): self.train(config, parallel, use_program_cache) @@ -94,55 +96,75 @@ def train_and_save_inference_program(self, target_vars=[self.loss, self.last_hidden, self.last_cell], executor=self.exe, dirname="padding_rnn." + rnn_model + ".inference_model", - params_filename="__params__") + params_filename="__params__", + ) def test_inference_output(self): for rnn_model in ["static", "padding"]: # Set parallel to False to use the default executor. - self.train_and_save_inference_program(rnn_model=rnn_model, - parallel=True, - use_program_cache=True) + self.train_and_save_inference_program( + rnn_model=rnn_model, parallel=True, use_program_cache=True + ) - x_np = np.random.random((self.config.batch_size, - self.config.num_steps, 1)).astype("int64") + x_np = np.random.random( + (self.config.batch_size, self.config.num_steps, 1) + ).astype("int64") y_np = np.random.random( - (self.config.batch_size * self.config.num_steps, - 1)).astype("int64") + (self.config.batch_size * self.config.num_steps, 1) + ).astype("int64") init_hidden_np = np.random.random( - (self.config.num_layers, self.config.batch_size, - self.config.hidden_size)).astype("float32") + ( + self.config.num_layers, + self.config.batch_size, + self.config.hidden_size, + ) + ).astype("float32") init_cell_np = np.random.random( - (self.config.num_layers, self.config.batch_size, - self.config.hidden_size)).astype("float32") + ( + self.config.num_layers, + self.config.batch_size, + self.config.hidden_size, + ) + ).astype("float32") for use_program_cache in [False, True]: with fluid.scope_guard(fluid.Scope()): - save_dirname = "padding_rnn." + rnn_model + ".inference_model" - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model( - save_dirname, self.exe, params_filename="__params__") - - results = self.exe.run(program=inference_program, - feed={ - "x": x_np, - "y": y_np, - "init_hidden": init_hidden_np, - "init_cell": init_cell_np - }, - fetch_list=fetch_targets, - use_program_cache=use_program_cache) + save_dirname = ( + "padding_rnn." + rnn_model + ".inference_model" + ) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + save_dirname, self.exe, params_filename="__params__" + ) + + results = self.exe.run( + program=inference_program, + feed={ + "x": x_np, + "y": y_np, + "init_hidden": init_hidden_np, + "init_cell": init_cell_np, + }, + fetch_list=fetch_targets, + use_program_cache=use_program_cache, + ) if use_program_cache is True: results_with_cache = results else: results_without_cache = results - self.assertEqual(len(results_with_cache), - len(results_without_cache)) + self.assertEqual( + len(results_with_cache), len(results_without_cache) + ) for i in range(len(results_with_cache)): - self.assertEqual(results_with_cache[i].shape, - results_without_cache[i].shape) - np.testing.assert_allclose(results_with_cache[i], - results_without_cache[i], - rtol=1e-05) + self.assertEqual( + results_with_cache[i].shape, results_without_cache[i].shape + ) + np.testing.assert_allclose( + results_with_cache[i], results_without_cache[i], rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_executor_check_feed.py b/python/paddle/fluid/tests/unittests/test_executor_check_feed.py index cfcd81a0a3580..77122f46bdcf5 100644 --- a/python/paddle/fluid/tests/unittests/test_executor_check_feed.py +++ b/python/paddle/fluid/tests/unittests/test_executor_check_feed.py @@ -19,7 +19,6 @@ class TestExecutor(unittest.TestCase): - def net(self): lr = fluid.data(name="lr", shape=[1], dtype='float32') x = fluid.data(name="x", shape=[None, 1], dtype='float32') @@ -48,13 +47,12 @@ def test_program_check_feed(self): y_true = [[2.0], [4.0], [6.0], [8.0]] a = 0 with self.assertRaises(ValueError): - exe.run(feed={ - 'x': train_data, - 'lr': a - }, - fetch_list=[lr, cost], - return_numpy=False, - use_prune=True) + exe.run( + feed={'x': train_data, 'lr': a}, + fetch_list=[lr, cost], + return_numpy=False, + use_prune=True, + ) def test_compiled_program_check_feed(self): main_program = fluid.Program() @@ -67,19 +65,19 @@ def test_compiled_program_check_feed(self): lr, cost = self.net() exe.run(startup_program) compiled_prog = fluid.CompiledProgram( - main_program).with_data_parallel(loss_name=cost.name) + main_program + ).with_data_parallel(loss_name=cost.name) train_data = [[1.0], [2.0], [3.0], [4.0]] y_true = [[2.0], [4.0], [6.0], [8.0]] a = 0 with self.assertRaises(ValueError): - exe.run(compiled_prog, - feed={ - 'x': train_data, - 'lr': a - }, - fetch_list=[lr, cost], - return_numpy=False, - use_prune=True) + exe.run( + compiled_prog, + feed={'x': train_data, 'lr': a}, + fetch_list=[lr, cost], + return_numpy=False, + use_prune=True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_executor_check_fetch_list.py b/python/paddle/fluid/tests/unittests/test_executor_check_fetch_list.py index e91bdf976d311..d6711a612d693 100644 --- a/python/paddle/fluid/tests/unittests/test_executor_check_fetch_list.py +++ b/python/paddle/fluid/tests/unittests/test_executor_check_fetch_list.py @@ -20,7 +20,6 @@ class TestCheckFetchList(unittest.TestCase): - def setUp(self): paddle.enable_static() self.feed = {"x": np.array([[0], [0], [1], [0]], dtype='float32')} @@ -32,10 +31,9 @@ def build_program(self): main_program = paddle.static.Program() with paddle.static.program_guard(main_program): x = paddle.static.data(name='x', shape=[4, 1], dtype='float32') - output = paddle.unique_consecutive(x, - return_inverse=True, - return_counts=True, - axis=0) + output = paddle.unique_consecutive( + x, return_inverse=True, return_counts=True, axis=0 + ) self.main_program = main_program self.fetch_list = output @@ -46,22 +44,23 @@ def test_with_tuple(self): self.main_program, feed=self.feed, fetch_list=[self.fetch_list], # support single list/tuple - return_numpy=True) + return_numpy=True, + ) np.testing.assert_array_equal(res[0], self.expected) def test_with_error(self): with self.assertRaises(TypeError): fetch_list = [23] - res = self.exe.run(self.main_program, - feed=self.feed, - fetch_list=fetch_list) + res = self.exe.run( + self.main_program, feed=self.feed, fetch_list=fetch_list + ) with self.assertRaises(TypeError): fetch_list = [(self.fetch_list[0], 32)] - res = self.exe.run(self.main_program, - feed=self.feed, - fetch_list=fetch_list) + res = self.exe.run( + self.main_program, feed=self.feed, fetch_list=fetch_list + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_executor_feed_non_tensor.py b/python/paddle/fluid/tests/unittests/test_executor_feed_non_tensor.py index 6441ba66e5787..41b4211a8bcc6 100644 --- a/python/paddle/fluid/tests/unittests/test_executor_feed_non_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_executor_feed_non_tensor.py @@ -20,7 +20,6 @@ class TestExecutor(unittest.TestCase): - def net(self): lr = fluid.data(name="lr", shape=[1], dtype='float32') x = fluid.data(name="x", shape=[None, 1], dtype='float32') @@ -45,18 +44,18 @@ def test_program_feed_float(self): exe = fluid.Executor(cpu) lr, cost = self.net() exe.run(startup_program) - train_data = numpy.array([[1.0], [2.0], [3.0], - [4.0]]).astype('float32') - y_true = numpy.array([[2.0], [4.0], [6.0], - [8.0]]).astype('float32') + train_data = numpy.array([[1.0], [2.0], [3.0], [4.0]]).astype( + 'float32' + ) + y_true = numpy.array([[2.0], [4.0], [6.0], [8.0]]).astype( + 'float32' + ) a = 0.01 - _lr, _ = exe.run(feed={ - 'x': train_data, - 'y': y_true, - 'lr': a - }, - fetch_list=[lr, cost], - return_numpy=False) + _lr, _ = exe.run( + feed={'x': train_data, 'y': y_true, 'lr': a}, + fetch_list=[lr, cost], + return_numpy=False, + ) self.assertEqual(_lr._dtype(), lr.dtype) self.assertEqual(_lr._dtype(), fluid.core.VarDesc.VarType.FP32) self.assertEqual(type(a), float) @@ -71,18 +70,18 @@ def test_program_feed_int(self): exe = fluid.Executor(cpu) lr, cost = self.net() exe.run(startup_program) - train_data = numpy.array([[1.0], [2.0], [3.0], - [4.0]]).astype('float32') - y_true = numpy.array([[2.0], [4.0], [6.0], - [8.0]]).astype('float32') + train_data = numpy.array([[1.0], [2.0], [3.0], [4.0]]).astype( + 'float32' + ) + y_true = numpy.array([[2.0], [4.0], [6.0], [8.0]]).astype( + 'float32' + ) a = 0 - _lr, _ = exe.run(feed={ - 'x': train_data, - 'y': y_true, - 'lr': a - }, - fetch_list=[lr, cost], - return_numpy=False) + _lr, _ = exe.run( + feed={'x': train_data, 'y': y_true, 'lr': a}, + fetch_list=[lr, cost], + return_numpy=False, + ) self.assertEqual(_lr._dtype(), lr.dtype) self.assertEqual(_lr._dtype(), fluid.core.VarDesc.VarType.FP32) self.assertEqual(type(a), int) @@ -100,13 +99,11 @@ def test_program_feed_list(self): train_data = [[1.0], [2.0], [3.0], [4.0]] y_true = [[2.0], [4.0], [6.0], [8.0]] a = 0 - _lr, _ = exe.run(feed={ - 'x': train_data, - 'y': y_true, - 'lr': a - }, - fetch_list=[lr, cost], - return_numpy=False) + _lr, _ = exe.run( + feed={'x': train_data, 'y': y_true, 'lr': a}, + fetch_list=[lr, cost], + return_numpy=False, + ) self.assertEqual(_lr._dtype(), lr.dtype) self.assertEqual(_lr._dtype(), fluid.core.VarDesc.VarType.FP32) self.assertEqual(type(y_true), list) @@ -122,37 +119,39 @@ def test_compiled_program_feed_scalar(self): exe = fluid.Executor(cpu) exe.run(startup_program) compiled_prog = fluid.CompiledProgram( - main_program).with_data_parallel(loss_name=cost.name) - train_data = numpy.array([[1.0], [2.0], [3.0], - [4.0]]).astype('float32') - y_true = numpy.array([[2.0], [4.0], [6.0], - [8.0]]).astype('float32') + main_program + ).with_data_parallel(loss_name=cost.name) + train_data = numpy.array([[1.0], [2.0], [3.0], [4.0]]).astype( + 'float32' + ) + y_true = numpy.array([[2.0], [4.0], [6.0], [8.0]]).astype( + 'float32' + ) a = 0.01 - _lr, _ = exe.run(compiled_prog, - feed={ - 'x': train_data, - 'y': y_true, - 'lr': a - }, - fetch_list=[lr, cost], - return_numpy=False) + _lr, _ = exe.run( + compiled_prog, + feed={'x': train_data, 'y': y_true, 'lr': a}, + fetch_list=[lr, cost], + return_numpy=False, + ) self.assertEqual(_lr._dtype(), lr.dtype) self.assertEqual(_lr._dtype(), fluid.core.VarDesc.VarType.FP32) self.assertEqual(type(a), float) class TestAsLodTensor(unittest.TestCase): - def test_as_lodtensor_int32(self): cpu = fluid.CPUPlace() - tensor = fluid.executor._as_lodtensor(1.0, cpu, - fluid.core.VarDesc.VarType.INT32) + tensor = fluid.executor._as_lodtensor( + 1.0, cpu, fluid.core.VarDesc.VarType.INT32 + ) self.assertEqual(tensor._dtype(), fluid.core.VarDesc.VarType.INT32) def test_as_lodtensor_fp64(self): cpu = fluid.CPUPlace() - tensor = fluid.executor._as_lodtensor(1, cpu, - fluid.core.VarDesc.VarType.FP64) + tensor = fluid.executor._as_lodtensor( + 1, cpu, fluid.core.VarDesc.VarType.FP64 + ) self.assertEqual(tensor._dtype(), fluid.core.VarDesc.VarType.FP64) def test_as_lodtensor_assertion_error(self): @@ -161,25 +160,37 @@ def test_as_lodtensor_assertion_error(self): def test_as_lodtensor_type_error(self): cpu = fluid.CPUPlace() - self.assertRaises(TypeError, fluid.executor._as_lodtensor, {"a": 1}, - cpu, fluid.core.VarDesc.VarType.INT32) + self.assertRaises( + TypeError, + fluid.executor._as_lodtensor, + {"a": 1}, + cpu, + fluid.core.VarDesc.VarType.INT32, + ) def test_as_lodtensor_list(self): cpu = fluid.CPUPlace() - tensor = fluid.executor._as_lodtensor([1, 2], cpu, - fluid.core.VarDesc.VarType.FP64) + tensor = fluid.executor._as_lodtensor( + [1, 2], cpu, fluid.core.VarDesc.VarType.FP64 + ) self.assertEqual(tensor._dtype(), fluid.core.VarDesc.VarType.FP64) def test_as_lodtensor_tuple(self): cpu = fluid.CPUPlace() - tensor = fluid.executor._as_lodtensor((1, 2), cpu, - fluid.core.VarDesc.VarType.FP64) + tensor = fluid.executor._as_lodtensor( + (1, 2), cpu, fluid.core.VarDesc.VarType.FP64 + ) self.assertEqual(tensor._dtype(), fluid.core.VarDesc.VarType.FP64) def test_as_lodtensor_nested_list(self): cpu = fluid.CPUPlace() - self.assertRaises(TypeError, fluid.executor._as_lodtensor, - [[1], [1, 2]], cpu, fluid.core.VarDesc.VarType.INT32) + self.assertRaises( + TypeError, + fluid.executor._as_lodtensor, + [[1], [1, 2]], + cpu, + fluid.core.VarDesc.VarType.INT32, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_executor_return_tensor_not_overwriting.py b/python/paddle/fluid/tests/unittests/test_executor_return_tensor_not_overwriting.py index fbb45e9c90625..7a3063025d5dc 100644 --- a/python/paddle/fluid/tests/unittests/test_executor_return_tensor_not_overwriting.py +++ b/python/paddle/fluid/tests/unittests/test_executor_return_tensor_not_overwriting.py @@ -20,7 +20,6 @@ @skip_check_grad_ci(reason="Not op test but call the method of class OpTest.") class TestExecutorReturnTensorNotOverwritingWithOptest(OpTest): - def setUp(self): pass @@ -30,7 +29,7 @@ def calc_add_out(self, place=None, parallel=None): self.out = np.add(self.x, self.y) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.op_type = "elementwise_add" @@ -44,7 +43,7 @@ def calc_mul_out(self, place=None, parallel=None): self.out = np.dot(self.x, self.y) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.op_type = "mul" @@ -67,7 +66,6 @@ def test_executor_run_twice(self): class TestExecutorReturnTensorNotOverOverwritingWithLayers(unittest.TestCase): - def setUp(self): pass @@ -78,7 +76,8 @@ def calc_add_out(self, place=None, parallel=None): program = fluid.default_main_program() if parallel: program = fluid.CompiledProgram(program).with_data_parallel( - places=place) + places=place + ) exe = fluid.Executor(place) out = exe.run(program, fetch_list=[out], return_numpy=False) return out @@ -90,7 +89,8 @@ def calc_sub_out(self, place=None, parallel=None): program = fluid.default_main_program() if parallel: program = fluid.CompiledProgram(program).with_data_parallel( - places=place) + places=place + ) exe = fluid.Executor(place) out = exe.run(program, fetch_list=[out], return_numpy=False) return out diff --git a/python/paddle/fluid/tests/unittests/test_expand_as_op.py b/python/paddle/fluid/tests/unittests/test_expand_as_op.py index 9815f587ddb3a..827f7a73a75de 100755 --- a/python/paddle/fluid/tests/unittests/test_expand_as_op.py +++ b/python/paddle/fluid/tests/unittests/test_expand_as_op.py @@ -29,7 +29,6 @@ def bcast(x, target_tensor): class TestExpandAsOpRank1(OpTest): - def setUp(self): self.op_type = "expand_as" x = np.random.rand(100).astype("float64") @@ -48,7 +47,6 @@ def test_check_grad(self): class TestExpandAsOpRank2(OpTest): - def setUp(self): self.op_type = "expand_as" x = np.random.rand(10, 12).astype("float64") @@ -67,7 +65,6 @@ def test_check_grad(self): class TestExpandAsOpRank3(OpTest): - def setUp(self): self.op_type = "expand_as" x = np.random.rand(2, 3, 20).astype("float64") @@ -86,7 +83,6 @@ def test_check_grad(self): class TestExpandAsOpRank4(OpTest): - def setUp(self): self.op_type = "expand_as" x = np.random.rand(1, 1, 7, 16).astype("float64") @@ -106,9 +102,9 @@ def test_check_grad(self): # Test dygraph API class TestExpandAsDygraphAPI(unittest.TestCase): - def test_api(self): import paddle + paddle.disable_static() np_data_x = np.array([1, 2, 3]).astype('int32') np_data_y = np.array([1, 2, 3, 1, 2, 3]).astype('int32') @@ -122,29 +118,28 @@ def test_api(self): # Test python API class TestExpandAsAPI(unittest.TestCase): - def test_api(self): input1 = np.random.random([12, 14]).astype("float32") input2 = np.random.random([48, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) - y = fluid.layers.data(name='target_tensor', - shape=[48, 14], - append_batch_size=False, - dtype="float32") + y = fluid.layers.data( + name='target_tensor', + shape=[48, 14], + append_batch_size=False, + dtype="float32", + ) out_1 = fluid.layers.expand_as(x, target_tensor=y) exe = fluid.Executor(place=fluid.CPUPlace()) - res_1 = exe.run(fluid.default_main_program(), - feed={ - "x": input1, - "target_tensor": input2 - }, - fetch_list=[out_1]) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": input1, "target_tensor": input2}, + fetch_list=[out_1], + ) assert np.array_equal(res_1[0], np.tile(input1, (4, 1))) diff --git a/python/paddle/fluid/tests/unittests/test_expand_as_v2_op.py b/python/paddle/fluid/tests/unittests/test_expand_as_v2_op.py index e0506f8eb5253..5f4c04470ff93 100755 --- a/python/paddle/fluid/tests/unittests/test_expand_as_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_expand_as_v2_op.py @@ -20,7 +20,6 @@ class TestExpandAsBasic(OpTest): - def setUp(self): self.op_type = "expand_as_v2" self.python_api = paddle.expand_as @@ -40,7 +39,6 @@ def test_check_grad(self): class TestExpandAsOpRank2(TestExpandAsBasic): - def setUp(self): self.op_type = "expand_as_v2" self.python_api = paddle.expand_as @@ -54,7 +52,6 @@ def setUp(self): class TestExpandAsOpRank3(TestExpandAsBasic): - def setUp(self): self.op_type = "expand_as_v2" self.python_api = paddle.expand_as @@ -68,7 +65,6 @@ def setUp(self): class TestExpandAsOpRank4(TestExpandAsBasic): - def setUp(self): self.op_type = "expand_as_v2" self.python_api = paddle.expand_as @@ -100,7 +96,6 @@ def test_check_grad(self): class TestExpandAsV2Error(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): x1 = fluid.layers.data(name='x1', shape=[4], dtype="uint8") @@ -113,29 +108,28 @@ def test_errors(self): # Test python API class TestExpandAsV2API(unittest.TestCase): - def test_api(self): input1 = np.random.random([12, 14]).astype("float32") input2 = np.random.random([2, 12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) - y = fluid.layers.data(name='target_tensor', - shape=[2, 12, 14], - append_batch_size=False, - dtype="float32") + y = fluid.layers.data( + name='target_tensor', + shape=[2, 12, 14], + append_batch_size=False, + dtype="float32", + ) out_1 = paddle.expand_as(x, y=y) exe = fluid.Executor(place=fluid.CPUPlace()) - res_1 = exe.run(fluid.default_main_program(), - feed={ - "x": input1, - "target_tensor": input2 - }, - fetch_list=[out_1]) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": input1, "target_tensor": input2}, + fetch_list=[out_1], + ) assert np.array_equal(res_1[0], np.tile(input1, (2, 1, 1))) diff --git a/python/paddle/fluid/tests/unittests/test_expand_op.py b/python/paddle/fluid/tests/unittests/test_expand_op.py index 36be4ab1a9beb..fd3dac2472e7c 100644 --- a/python/paddle/fluid/tests/unittests/test_expand_op.py +++ b/python/paddle/fluid/tests/unittests/test_expand_op.py @@ -22,12 +22,12 @@ # Situation 1: expand_times is a list(without tensor) class TestExpandOpRank1(OpTest): - def setUp(self): self.op_type = "expand" self.init_data() - self.dtype = "float32" if fluid.core.is_compiled_with_rocm( - ) else "float64" + self.dtype = ( + "float32" if fluid.core.is_compiled_with_rocm() else "float64" + ) self.inputs = {'X': np.random.random(self.ori_shape).astype(self.dtype)} self.attrs = {'expand_times': self.expand_times} @@ -46,35 +46,30 @@ def test_check_grad(self): class TestExpandOpRank2_Corner(TestExpandOpRank1): - def init_data(self): self.ori_shape = [120] self.expand_times = [2] class TestExpandOpRank2(TestExpandOpRank1): - def init_data(self): self.ori_shape = [12, 14] self.expand_times = [2, 3] class TestExpandOpRank3_Corner(TestExpandOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.expand_times = (1, 1, 1) class TestExpandOpRank3(TestExpandOpRank1): - def init_data(self): self.ori_shape = (2, 4, 15) self.expand_times = (2, 1, 4) class TestExpandOpRank4(TestExpandOpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.expand_times = (3, 2, 1, 2) @@ -82,17 +77,18 @@ def init_data(self): # Situation 2: expand_times is a list(with tensor) class TestExpandOpRank1_tensor_attr(OpTest): - def setUp(self): self.op_type = "expand" self.init_data() - self.dtype = "float32" if fluid.core.is_compiled_with_rocm( - ) else "float64" + self.dtype = ( + "float32" if fluid.core.is_compiled_with_rocm() else "float64" + ) expand_times_tensor = [] for index, ele in enumerate(self.expand_times): - expand_times_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + expand_times_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype(self.dtype), @@ -115,7 +111,6 @@ def test_check_grad(self): class TestExpandOpRank2_Corner_tensor_attr(TestExpandOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.expand_times = [1, 1] @@ -123,7 +118,6 @@ def init_data(self): class TestExpandOpRank2_attr_tensor(TestExpandOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.expand_times = [2, 3] @@ -132,12 +126,12 @@ def init_data(self): # Situation 3: expand_times is a tensor class TestExpandOpRank1_tensor(OpTest): - def setUp(self): self.op_type = "expand" self.init_data() - self.dtype = "float32" if fluid.core.is_compiled_with_rocm( - ) else "float64" + self.dtype = ( + "float32" if fluid.core.is_compiled_with_rocm() else "float64" + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype(self.dtype), @@ -159,7 +153,6 @@ def test_check_grad(self): class TestExpandOpRank2_tensor(TestExpandOpRank1_tensor): - def init_data(self): self.ori_shape = [12, 14] self.expand_times = [2, 3] @@ -167,7 +160,6 @@ def init_data(self): # Situation 4: input x is Integer class TestExpandOpInteger(OpTest): - def setUp(self): self.op_type = "expand" self.inputs = { @@ -183,7 +175,6 @@ def test_check_output(self): # Situation 5: input x is Bool class TestExpandOpBoolean(OpTest): - def setUp(self): self.op_type = "expand" self.inputs = {'X': np.random.randint(2, size=(2, 4, 5)).astype("bool")} @@ -197,7 +188,6 @@ def test_check_output(self): # Situation 56: input x is Integer class TestExpandOpInt64_t(OpTest): - def setUp(self): self.op_type = "expand" self.inputs = { @@ -212,11 +202,11 @@ def test_check_output(self): class TestExpandError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) expand_times = [2, 2] self.assertRaises(TypeError, fluid.layers.expand, x1, expand_times) x2 = fluid.layers.data(name='x2', shape=[4], dtype="uint8") @@ -228,18 +218,16 @@ def test_errors(self): # Test python API class TestExpandAPI(unittest.TestCase): - def test_api(self): input = np.random.random([12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) positive_2 = fluid.layers.fill_constant([1], "int32", 2) - expand_times = fluid.layers.data(name="expand_times", - shape=[2], - append_batch_size=False) + expand_times = fluid.layers.data( + name="expand_times", shape=[2], append_batch_size=False + ) out_1 = fluid.layers.expand(x, expand_times=[2, 3]) out_2 = fluid.layers.expand(x, expand_times=[positive_2, 3]) @@ -248,28 +236,24 @@ def test_api(self): g0 = fluid.backward.calc_gradient(out_2, x) exe = fluid.Executor(place=fluid.CPUPlace()) - res_1, res_2, res_3 = exe.run(fluid.default_main_program(), - feed={ - "x": - input, - "expand_times": - np.array([1, 3]).astype("int32") - }, - fetch_list=[out_1, out_2, out_3]) + res_1, res_2, res_3 = exe.run( + fluid.default_main_program(), + feed={"x": input, "expand_times": np.array([1, 3]).astype("int32")}, + fetch_list=[out_1, out_2, out_3], + ) assert np.array_equal(res_1, np.tile(input, (2, 3))) assert np.array_equal(res_2, np.tile(input, (2, 3))) assert np.array_equal(res_3, np.tile(input, (1, 3))) class TestExpandDygraphAPI(unittest.TestCase): - def test_expand_times_is_tensor(self): with paddle.fluid.dygraph.guard(): a = paddle.rand([2, 5]) b = paddle.fluid.layers.expand(a, expand_times=[2, 3]) - c = paddle.fluid.layers.expand(a, - expand_times=paddle.to_tensor( - [2, 3], dtype='int32')) + c = paddle.fluid.layers.expand( + a, expand_times=paddle.to_tensor([2, 3], dtype='int32') + ) np.testing.assert_array_equal(b.numpy(), np.tile(a.numpy(), [2, 3])) np.testing.assert_array_equal(c.numpy(), np.tile(a.numpy(), [2, 3])) diff --git a/python/paddle/fluid/tests/unittests/test_expand_v2_op.py b/python/paddle/fluid/tests/unittests/test_expand_v2_op.py index d89332f393aa1..da27fb397cc59 100644 --- a/python/paddle/fluid/tests/unittests/test_expand_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_expand_v2_op.py @@ -26,7 +26,6 @@ # Situation 1: shape is a list(without tensor) class TestExpandV2OpRank1(OpTest): - def setUp(self): self.op_type = "expand_v2" self.init_data() @@ -50,7 +49,6 @@ def test_check_grad(self): class TestExpandV2OpRank2_DimExpanding(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = [120] self.shape = [2, 120] @@ -58,7 +56,6 @@ def init_data(self): class TestExpandV2OpRank2(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = [1, 140] self.shape = [12, 140] @@ -66,7 +63,6 @@ def init_data(self): class TestExpandV2OpRank3_Corner(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.shape = (2, 10, 5) @@ -74,7 +70,6 @@ def init_data(self): class TestExpandV2OpRank4(TestExpandV2OpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.shape = (-1, -1, -1, -1) @@ -83,14 +78,14 @@ def init_data(self): # Situation 2: shape is a list(with tensor) class TestExpandV2OpRank1_tensor_attr(OpTest): - def setUp(self): self.op_type = "expand_v2" self.init_data() expand_shapes_tensor = [] for index, ele in enumerate(self.expand_shape): - expand_shapes_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + expand_shapes_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype("float64"), @@ -114,7 +109,6 @@ def test_check_grad(self): class TestExpandV2OpRank2_Corner_tensor_attr(TestExpandV2OpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.expand_times = [1, 1] @@ -124,7 +118,6 @@ def init_data(self): # Situation 3: shape is a tensor class TestExpandV2OpRank1_tensor(OpTest): - def setUp(self): self.op_type = "expand_v2" self.init_data() @@ -151,7 +144,6 @@ def test_check_grad(self): # Situation 4: input x is Integer class TestExpandV2OpInteger(OpTest): - def setUp(self): self.op_type = "expand_v2" self.inputs = { @@ -167,7 +159,6 @@ def test_check_output(self): # Situation 5: input x is Bool class TestExpandV2OpBoolean(OpTest): - def setUp(self): self.op_type = "expand_v2" self.inputs = {'X': np.random.randint(2, size=(2, 4, 5)).astype("bool")} @@ -181,7 +172,6 @@ def test_check_output(self): # Situation 56: input x is Integer class TestExpandV2OpInt64_t(OpTest): - def setUp(self): self.op_type = "expand_v2" self.inputs = { @@ -196,11 +186,11 @@ def test_check_output(self): class TestExpandV2Error(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) shape = [2, 2] self.assertRaises(TypeError, paddle.tensor.expand, x1, shape) x2 = fluid.layers.data(name='x2', shape=[4], dtype="uint8") @@ -212,19 +202,19 @@ def test_errors(self): # Test python API class TestExpandV2API(unittest.TestCase): - def test_api(self): input = np.random.random([12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) positive_2 = fluid.layers.fill_constant([1], "int32", 12) - expand_shape = fluid.layers.data(name="expand_shape", - shape=[2], - append_batch_size=False, - dtype="int32") + expand_shape = fluid.layers.data( + name="expand_shape", + shape=[2], + append_batch_size=False, + dtype="int32", + ) out_1 = paddle.expand(x, shape=[12, 14]) out_2 = paddle.expand(x, shape=[positive_2, 14]) @@ -233,28 +223,28 @@ def test_api(self): g0 = fluid.backward.calc_gradient(out_2, x) exe = fluid.Executor(place=fluid.CPUPlace()) - res_1, res_2, res_3 = exe.run(fluid.default_main_program(), - feed={ - "x": - input, - "expand_shape": - np.array([12, 14]).astype("int32") - }, - fetch_list=[out_1, out_2, out_3]) + res_1, res_2, res_3 = exe.run( + fluid.default_main_program(), + feed={ + "x": input, + "expand_shape": np.array([12, 14]).astype("int32"), + }, + fetch_list=[out_1, out_2, out_3], + ) assert np.array_equal(res_1, np.tile(input, (1, 1))) assert np.array_equal(res_2, np.tile(input, (1, 1))) assert np.array_equal(res_3, np.tile(input, (1, 1))) class TestExpandInferShape(unittest.TestCase): - def test_shape_with_var(self): with program_guard(Program(), Program()): x = paddle.static.data(shape=[-1, 1, 3], name='x') fake_var = paddle.randn([2, 3]) target_shape = [ - -1, paddle.shape(fake_var)[0], - paddle.shape(fake_var)[1] + -1, + paddle.shape(fake_var)[0], + paddle.shape(fake_var)[1], ] out = paddle.expand(x, shape=target_shape) self.assertListEqual(list(out.shape), [-1, -1, -1]) @@ -262,7 +252,6 @@ def test_shape_with_var(self): # Test python Dygraph API class TestExpandV2DygraphAPI(unittest.TestCase): - def test_expand_times_is_tensor(self): with paddle.fluid.dygraph.guard(): with _test_eager_guard(): @@ -278,15 +267,16 @@ def test_expand_times_is_tensor(self): np_array = np.array([2, 5]) expand_2 = paddle.expand(a, shape=np_array) - np.testing.assert_array_equal(egr_expand_1.numpy(), - egr_expand_2.numpy()) + np.testing.assert_array_equal( + egr_expand_1.numpy(), egr_expand_2.numpy() + ) np.testing.assert_array_equal(expand_1.numpy(), expand_2.numpy()) - np.testing.assert_array_equal(expand_1.numpy(), - egr_expand_1.numpy()) + np.testing.assert_array_equal( + expand_1.numpy(), egr_expand_1.numpy() + ) class TestExpandDoubleGradCheck(unittest.TestCase): - def expand_wrapper(self, x): return paddle.expand(x[0], [2, 3]) @@ -301,17 +291,13 @@ def func(self, place): out = paddle.expand(data, [2, 3]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.expand_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.expand_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -323,7 +309,6 @@ def test_grad(self): class TestExpandTripleGradCheck(unittest.TestCase): - def expand_wrapper(self, x): return paddle.expand(x[0], [2, 3]) @@ -338,17 +323,13 @@ def func(self, place): out = paddle.expand(data, [2, 3]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.expand_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.expand_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_exponential_op.py b/python/paddle/fluid/tests/unittests/test_exponential_op.py index 54a6befc8edcc..458be9f3635b2 100644 --- a/python/paddle/fluid/tests/unittests/test_exponential_op.py +++ b/python/paddle/fluid/tests/unittests/test_exponential_op.py @@ -21,7 +21,6 @@ class TestExponentialOp1(OpTest): - def setUp(self): paddle.enable_static() self.op_type = "exponential" @@ -43,7 +42,7 @@ def verify_output(self, outs): hist1 = hist1.astype("float32") hist1 = hist1 / float(outs[0].size) - data_np = np.random.exponential(1. / self.lam, [1024, 1024]) + data_np = np.random.exponential(1.0 / self.lam, [1024, 1024]) hist2, _ = np.histogram(data_np, range=(0, 5)) hist2 = hist2.astype("float32") hist2 = hist2 / float(data_np.size) @@ -57,34 +56,36 @@ def test_check_grad_normal(self): user_defined_grads=[np.zeros([1024, 1024], dtype=self.dtype)], user_defined_grad_outputs=[ np.random.rand(1024, 1024).astype(self.dtype) - ]) + ], + ) class TestExponentialOp2(TestExponentialOp1): - def config(self): self.lam = 0.25 self.dtype = "float32" class TestExponentialAPI(unittest.TestCase): - def test_static(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - x_np = np.full([10, 10], -1.) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + x_np = np.full([10, 10], -1.0) x = paddle.static.data(name="X", shape=[10, 10], dtype='float64') x.exponential_(1.0) exe = paddle.static.Executor() - out = exe.run(paddle.static.default_main_program(), - feed={"X": x_np}, - fetch_list=[x]) + out = exe.run( + paddle.static.default_main_program(), + feed={"X": x_np}, + fetch_list=[x], + ) self.assertTrue(np.min(out) >= 0) def test_dygraph(self): paddle.disable_static() - x = paddle.full([10, 10], -1., dtype='float32') + x = paddle.full([10, 10], -1.0, dtype='float32') x.stop_gradient = False y = 2 * x y.exponential_(0.5) @@ -113,47 +114,94 @@ def test_fixed_random_number(self): x.exponential_(1.0) x_np = x.numpy() expect = [ - 0.80073667, 0.2249291, 0.07734892, 1.25392, 0.14013891, 0.45736602, - 1.9735607, 0.30490234, 0.57100505, 0.8115938 + 0.80073667, + 0.2249291, + 0.07734892, + 1.25392, + 0.14013891, + 0.45736602, + 1.9735607, + 0.30490234, + 0.57100505, + 0.8115938, ] np.testing.assert_allclose(x_np[0, 0, 0, 0:10], expect, rtol=1e-05) expect = [ - 1.4296371e+00, 9.5411777e-01, 5.2575850e-01, 2.4805880e-01, - 1.2322118e-04, 8.4604341e-01, 2.1111444e-01, 1.4143821e+00, - 2.8194717e-01, 1.1360573e+00 + 1.4296371e00, + 9.5411777e-01, + 5.2575850e-01, + 2.4805880e-01, + 1.2322118e-04, + 8.4604341e-01, + 2.1111444e-01, + 1.4143821e00, + 2.8194717e-01, + 1.1360573e00, ] - np.testing.assert_allclose(x_np[16, 1, 300, 200:210], - expect, - rtol=1e-05) + np.testing.assert_allclose( + x_np[16, 1, 300, 200:210], expect, rtol=1e-05 + ) expect = [ - 1.3448033, 0.35146526, 1.7380928, 0.32012638, 0.10396296, - 0.51344526, 0.15308502, 0.18712929, 0.03888268, 0.20771872 + 1.3448033, + 0.35146526, + 1.7380928, + 0.32012638, + 0.10396296, + 0.51344526, + 0.15308502, + 0.18712929, + 0.03888268, + 0.20771872, ] - np.testing.assert_allclose(x_np[32, 1, 600, 500:510], - expect, - rtol=1e-05) + np.testing.assert_allclose( + x_np[32, 1, 600, 500:510], expect, rtol=1e-05 + ) expect = [ - 0.5107464, 0.20970327, 2.1986802, 1.580056, 0.31036147, 0.43966478, - 0.9056133, 0.30119267, 1.4797124, 1.4319834 + 0.5107464, + 0.20970327, + 2.1986802, + 1.580056, + 0.31036147, + 0.43966478, + 0.9056133, + 0.30119267, + 1.4797124, + 1.4319834, ] - np.testing.assert_allclose(x_np[48, 2, 900, 800:810], - expect, - rtol=1e-05) + np.testing.assert_allclose( + x_np[48, 2, 900, 800:810], expect, rtol=1e-05 + ) expect = [ - 3.4640615, 1.1019983, 0.41195083, 0.22681557, 0.291846, 0.53617656, - 1.5791925, 2.4645927, 0.04094889, 0.9057725 + 3.4640615, + 1.1019983, + 0.41195083, + 0.22681557, + 0.291846, + 0.53617656, + 1.5791925, + 2.4645927, + 0.04094889, + 0.9057725, ] - np.testing.assert_allclose(x_np[63, 2, 1023, 1000:1010], - expect, - rtol=1e-05) + np.testing.assert_allclose( + x_np[63, 2, 1023, 1000:1010], expect, rtol=1e-05 + ) x = paddle.empty([10, 10], dtype="float32") x.exponential_(3.0) x_np = x.numpy() expect = [ - 0.02831675, 0.1691551, 0.6798956, 0.69347525, 0.0243443, 0.22180498, - 0.30574575, 0.9839696, 0.2834912, 0.59420055 + 0.02831675, + 0.1691551, + 0.6798956, + 0.69347525, + 0.0243443, + 0.22180498, + 0.30574575, + 0.9839696, + 0.2834912, + 0.59420055, ] np.testing.assert_allclose(x_np[5, 0:10], expect, rtol=1e-05) @@ -161,51 +209,115 @@ def test_fixed_random_number(self): x.exponential_(0.25) x_np = x.numpy() expect = [ - 10.0541229, 12.67860643, 1.09850734, 7.35289643, 2.65471225, - 3.86217432, 2.97902086, 2.92744479, 2.67927152, 0.19667352 + 10.0541229, + 12.67860643, + 1.09850734, + 7.35289643, + 2.65471225, + 3.86217432, + 2.97902086, + 2.92744479, + 2.67927152, + 0.19667352, ] np.testing.assert_allclose(x_np[0, 0, 0, 100:110], expect, rtol=1e-05) expect = [ - 0.68328125, 3.1454553, 0.92158376, 1.95842188, 1.05296941, - 12.93242051, 5.20255978, 3.3588624, 1.57377174, 5.73194183 + 0.68328125, + 3.1454553, + 0.92158376, + 1.95842188, + 1.05296941, + 12.93242051, + 5.20255978, + 3.3588624, + 1.57377174, + 5.73194183, ] np.testing.assert_allclose(x_np[4, 0, 300, 190:200], expect, rtol=1e-05) expect = [ - 1.37973974, 3.45036798, 7.94625406, 1.62610973, 0.31032122, - 4.13596493, 1.98494535, 1.13207041, 8.30592769, 2.81460147 + 1.37973974, + 3.45036798, + 7.94625406, + 1.62610973, + 0.31032122, + 4.13596493, + 1.98494535, + 1.13207041, + 8.30592769, + 2.81460147, ] np.testing.assert_allclose(x_np[8, 1, 600, 300:310], expect, rtol=1e-05) expect = [ - 2.27710811, 12.25003028, 2.96409124, 4.72405788, 0.67917249, - 4.35856718, 0.46870976, 2.31120149, 9.61595826, 4.64446271 + 2.27710811, + 12.25003028, + 2.96409124, + 4.72405788, + 0.67917249, + 4.35856718, + 0.46870976, + 2.31120149, + 9.61595826, + 4.64446271, ] - np.testing.assert_allclose(x_np[12, 1, 900, 500:510], - expect, - rtol=1e-05) + np.testing.assert_allclose( + x_np[12, 1, 900, 500:510], expect, rtol=1e-05 + ) expect = [ - 0.95883744, 1.57316361, 15.22524512, 20.49559882, 13.70008548, - 3.29430143, 3.90390424, 0.9146657, 0.80972249, 0.33376219 + 0.95883744, + 1.57316361, + 15.22524512, + 20.49559882, + 13.70008548, + 3.29430143, + 3.90390424, + 0.9146657, + 0.80972249, + 0.33376219, ] - np.testing.assert_allclose(x_np[15, 1, 1023, 750:760], - expect, - rtol=1e-05) + np.testing.assert_allclose( + x_np[15, 1, 1023, 750:760], expect, rtol=1e-05 + ) x = paddle.empty([512, 768], dtype="float64") x.exponential_(0.3) x_np = x.numpy() expect = [ - 8.79266704, 4.79596009, 2.75480243, 6.04670011, 0.35379556, - 0.76864868, 3.17428251, 0.26556859, 12.22485885, 10.51690383 + 8.79266704, + 4.79596009, + 2.75480243, + 6.04670011, + 0.35379556, + 0.76864868, + 3.17428251, + 0.26556859, + 12.22485885, + 10.51690383, ] np.testing.assert_allclose(x_np[0, 200:210], expect, rtol=1e-05) expect = [ - 5.6341126, 0.52243418, 5.36410796, 6.83672002, 11.9243311, - 5.85985566, 5.75169548, 0.13877972, 6.1348385, 3.82436519 + 5.6341126, + 0.52243418, + 5.36410796, + 6.83672002, + 11.9243311, + 5.85985566, + 5.75169548, + 0.13877972, + 6.1348385, + 3.82436519, ] np.testing.assert_allclose(x_np[300, 400:410], expect, rtol=1e-05) expect = [ - 4.94883581, 0.56345306, 0.85841585, 1.92287801, 6.10036656, - 1.19524847, 3.64735434, 5.19618716, 2.57467974, 3.49152791 + 4.94883581, + 0.56345306, + 0.85841585, + 1.92287801, + 6.10036656, + 1.19524847, + 3.64735434, + 5.19618716, + 2.57467974, + 3.49152791, ] np.testing.assert_allclose(x_np[500, 700:710], expect, rtol=1e-05) @@ -213,8 +325,16 @@ def test_fixed_random_number(self): x.exponential_(4.0) x_np = x.numpy() expect = [ - 0.15713826, 0.56395964, 0.0680941, 0.00316643, 0.27046853, - 0.19852724, 0.12776634, 0.09642974, 0.51977551, 1.33739699 + 0.15713826, + 0.56395964, + 0.0680941, + 0.00316643, + 0.27046853, + 0.19852724, + 0.12776634, + 0.09642974, + 0.51977551, + 1.33739699, ] np.testing.assert_allclose(x_np[5, 0:10], expect, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_eye_op.py b/python/paddle/fluid/tests/unittests/test_eye_op.py index c67388c133959..e61037ec1afbd 100644 --- a/python/paddle/fluid/tests/unittests/test_eye_op.py +++ b/python/paddle/fluid/tests/unittests/test_eye_op.py @@ -26,7 +26,6 @@ class TestEyeOp(OpTest): - def setUp(self): ''' Test eye op with specified shape @@ -38,7 +37,7 @@ def setUp(self): self.attrs = { 'num_rows': 219, 'num_columns': 319, - 'dtype': framework.convert_np_dtype_to_dtype_(np.int32) + 'dtype': framework.convert_np_dtype_to_dtype_(np.int32), } self.outputs = {'Out': np.eye(219, 319, dtype=np.int32)} @@ -47,7 +46,6 @@ def test_check_output(self): class TestEyeOp1(OpTest): - def setUp(self): ''' Test eye op with default parameters @@ -64,7 +62,6 @@ def test_check_output(self): class TestEyeOp2(OpTest): - def setUp(self): ''' Test eye op with specified shape @@ -81,13 +78,12 @@ def test_check_output(self): class API_TestTensorEye(unittest.TestCase): - def test_out(self): with paddle.static.program_guard(paddle.static.Program()): data = paddle.eye(10) place = fluid.CPUPlace() exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[data]) + (result,) = exe.run(fetch_list=[data]) expected_result = np.eye(10, dtype="float32") self.assertEqual((result == expected_result).all(), True) @@ -95,7 +91,7 @@ def test_out(self): data = paddle.eye(10, num_columns=7, dtype="float64") place = paddle.CPUPlace() exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[data]) + (result,) = exe.run(fetch_list=[data]) expected_result = np.eye(10, 7, dtype="float64") self.assertEqual((result == expected_result).all(), True) @@ -103,7 +99,7 @@ def test_out(self): data = paddle.eye(10, dtype="int64") place = paddle.CPUPlace() exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[data]) + (result,) = exe.run(fetch_list=[data]) expected_result = np.eye(10, dtype="int64") self.assertEqual((result == expected_result).all(), True) @@ -125,8 +121,9 @@ def test_out(self): result = tmp_result expected_result = np.stack(result, axis=0) paddle.enable_static() - self.assertEqual(out.numpy().shape == np.array(expected_result).shape, - True) + self.assertEqual( + out.numpy().shape == np.array(expected_result).shape, True + ) self.assertEqual((out.numpy() == expected_result).all(), True) paddle.disable_static() @@ -141,8 +138,9 @@ def test_out(self): result = tmp_result expected_result = np.stack(result, axis=0) paddle.enable_static() - self.assertEqual(out.numpy().shape == np.array(expected_result).shape, - True) + self.assertEqual( + out.numpy().shape == np.array(expected_result).shape, True + ) self.assertEqual((out.numpy() == expected_result).all(), True) def test_errors(self): @@ -165,7 +163,6 @@ def test_num_columns_type_check1(): class TestEyeRowsCol(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -191,8 +188,9 @@ def test_static(self): res = exe.run(fetch_list=[tmp, out]) gt = np.eye(3, 10) np.testing.assert_allclose(res[0], gt) - paddle.static.save_inference_model(self.save_path, [x], [tmp, out], - exe) + paddle.static.save_inference_model( + self.save_path, [x], [tmp, out], exe + ) # Test for Inference Predictor infer_outs = self.infer_prog() np.testing.assert_allclose(infer_outs[0], gt) @@ -215,7 +213,6 @@ def test_error(self): class TestEyeRowsCol2(TestEyeRowsCol): - def call_func(self, x): rows = paddle.assign(3) cols = paddle.assign(10) diff --git a/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py b/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py index 239f5a93b1f9f..b56ff6edbfe97 100644 --- a/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py +++ b/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py @@ -47,11 +47,9 @@ def channel_wise_quantize_max_abs(x, quant_bit=8, quant_axis=0): return y, scales -def channel_wise_dequantize_max_abs(x, - scales, - quant_bits, - quant_axis, - activation_scale=None): +def channel_wise_dequantize_max_abs( + x, scales, quant_bits, quant_axis, activation_scale=None +): assert quant_axis in [0, 1], "The quant_axis should be 0 or 1." if isinstance(quant_bits, list): @@ -72,7 +70,6 @@ def channel_wise_dequantize_max_abs(x, class TestFakeChannelWiseDequantizeMaxAbsOpTwoScales(OpTest): - def set_args(self): self.quant_bits = [8, 8] self.activation_scale = 0.7861 @@ -86,15 +83,19 @@ def setUp(self): self.op_type = "fake_channel_wise_dequantize_max_abs" x = np.random.randn(4, 3, 64, 64).astype(self.dtype) yq, scales = channel_wise_quantize_max_abs(x, self.quant_bits[0], 1) - ydq = channel_wise_dequantize_max_abs(yq, scales, self.quant_bits, 1, - self.activation_scale) + ydq = channel_wise_dequantize_max_abs( + yq, scales, self.quant_bits, 1, self.activation_scale + ) self.inputs = { - 'X': - yq, - 'Scales': - [("scales0", np.array(scales).astype(self.dtype)), - ("scales1", np.array([self.activation_scale]).astype(self.dtype))] + 'X': yq, + 'Scales': [ + ("scales0", np.array(scales).astype(self.dtype)), + ( + "scales1", + np.array([self.activation_scale]).astype(self.dtype), + ), + ], } self.attrs = {'quant_bits': self.quant_bits} self.outputs = {'Out': ydq} @@ -104,8 +105,8 @@ def test_check_output(self): class TestFakeChannelWiseDequantizeMaxAbsOpTwoScalesFloat16( - TestFakeChannelWiseDequantizeMaxAbsOpTwoScales): - + TestFakeChannelWiseDequantizeMaxAbsOpTwoScales +): def set_dtype(self): self.dtype = np.float16 @@ -114,7 +115,6 @@ def test_check_output(self): class TestFakeChannelWiseDequantizeMaxAbsOpOneScale(OpTest): - def set_args(self): self.quant_bits = [8] self.quant_axis = 0 @@ -127,18 +127,20 @@ def setUp(self): self.set_dtype() self.op_type = "fake_channel_wise_dequantize_max_abs" x = np.random.randn(4, 3, 64, 64).astype(self.dtype) - yq, scales = channel_wise_quantize_max_abs(x, self.quant_bits[0], - self.quant_axis) - ydq = channel_wise_dequantize_max_abs(yq, scales, self.quant_bits, - self.quant_axis) + yq, scales = channel_wise_quantize_max_abs( + x, self.quant_bits[0], self.quant_axis + ) + ydq = channel_wise_dequantize_max_abs( + yq, scales, self.quant_bits, self.quant_axis + ) self.inputs = { 'X': yq, - 'Scales': [("scales0", np.array(scales).astype(self.dtype))] + 'Scales': [("scales0", np.array(scales).astype(self.dtype))], } self.attrs = { 'quant_bits': self.quant_bits, - 'quant_axis': self.quant_axis + 'quant_axis': self.quant_axis, } self.outputs = {'Out': ydq} @@ -147,16 +149,16 @@ def test_check_output(self): class TestFakeChannelWiseDequantizeMaxAbsOpOneScale1( - TestFakeChannelWiseDequantizeMaxAbsOpOneScale): - + TestFakeChannelWiseDequantizeMaxAbsOpOneScale +): def set_args(self): self.quant_bits = [8] self.quant_axis = 1 class TestFakeChannelWiseDequantizeMaxAbsOpOneScaleFloat16( - TestFakeChannelWiseDequantizeMaxAbsOpOneScale): - + TestFakeChannelWiseDequantizeMaxAbsOpOneScale +): def set_dtype(self): self.dtype = np.float16 @@ -165,8 +167,8 @@ def test_check_output(self): class TestFakeChannelWiseDequantizeMaxAbsOpOneScale1Float16( - TestFakeChannelWiseDequantizeMaxAbsOpOneScale1): - + TestFakeChannelWiseDequantizeMaxAbsOpOneScale1 +): def set_dtype(self): self.dtype = np.float16 @@ -175,7 +177,6 @@ def test_check_output(self): class TestFakeDequantizeMaxAbsOp(OpTest): - def set_args(self): self.num_bits = 8 self.max_range = math.pow(2, self.num_bits - 1) - 1 @@ -200,20 +201,17 @@ def test_check_output(self): class TestFakeDequantizeMaxAbsOpDouble(TestFakeDequantizeMaxAbsOp): - def set_dtype(self): self.dtype = np.float64 class TestFakeDequantizeMaxAbsOp5Bits(TestFakeDequantizeMaxAbsOp): - def set_args(self): self.num_bits = 5 self.max_range = math.pow(2, self.num_bits - 1) - 1 class TestFakeDequantizeMaxAbsOpFloat16(TestFakeDequantizeMaxAbsOp): - def set_dtype(self): self.dtype = np.float16 @@ -222,7 +220,6 @@ def test_check_output(self): class TestChannelWiseDequantizeOp(OpTest): - def set_args(self): self.bit_length = 8 self.data_type = "float32" @@ -232,17 +229,19 @@ def setUp(self): self.set_args() self.op_type = "dequantize_linear" x = np.random.randn(4, 3, 64, 64).astype(self.data_type) - yq, scale = channel_wise_quantize_max_abs(x, self.bit_length, - self.quant_axis) - ydq = channel_wise_dequantize_max_abs(yq, scale, self.bit_length, - self.quant_axis) + yq, scale = channel_wise_quantize_max_abs( + x, self.bit_length, self.quant_axis + ) + ydq = channel_wise_dequantize_max_abs( + yq, scale, self.bit_length, self.quant_axis + ) scale = np.array(scale).astype(self.data_type) zero_point = np.zeros(scale.shape, dtype="int32") print('TestChannelWiseDequantizeOp:') self.inputs = {'X': yq, 'Scale': scale, 'ZeroPoint': zero_point} self.attrs = { 'bit_length': self.bit_length, - 'quant_axis': self.quant_axis + 'quant_axis': self.quant_axis, } self.outputs = {'Y': ydq} @@ -251,7 +250,6 @@ def test_check_output(self): class TestChannelWiseDequantizeOp1(TestChannelWiseDequantizeOp): - def set_args(self): self.bit_length = 8 self.data_type = "float32" @@ -259,7 +257,6 @@ def set_args(self): class TestDequantizeOp(OpTest): - def set_args(self): self.bit_length = 8 self.quant_axis = -1 @@ -278,7 +275,7 @@ def setUp(self): self.inputs = {'X': yq, 'Scale': scale, 'ZeroPoint': zero_point} self.attrs = { 'bit_length': self.bit_length, - 'quant_axis': self.quant_axis + 'quant_axis': self.quant_axis, } self.outputs = {'Y': ydq} @@ -287,7 +284,6 @@ def test_check_output(self): class TestDequantizeOpDouble(TestDequantizeOp): - def set_args(self): self.bit_length = 8 self.max_range = math.pow(2, self.bit_length - 1) - 1 @@ -296,7 +292,6 @@ def set_args(self): class TestDequantizeOp5Bits(TestDequantizeOp): - def set_args(self): self.bit_length = 5 self.max_range = math.pow(2, self.bit_length - 1) - 1 diff --git a/python/paddle/fluid/tests/unittests/test_fake_init_op.py b/python/paddle/fluid/tests/unittests/test_fake_init_op.py index 6ff7abe0c228f..4a18c09b389ea 100644 --- a/python/paddle/fluid/tests/unittests/test_fake_init_op.py +++ b/python/paddle/fluid/tests/unittests/test_fake_init_op.py @@ -19,14 +19,14 @@ class TestFakeInitOpSelectedRows(unittest.TestCase): - def check_with_place(self, place, is_selected_rows): scope = core.Scope() out_var_name = 'Out' if is_selected_rows: - out_tensor = scope.var( - out_var_name).get_selected_rows().get_tensor() + out_tensor = ( + scope.var(out_var_name).get_selected_rows().get_tensor() + ) else: out_tensor = scope.var(out_var_name).get_tensor() diff --git a/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py b/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py index 49d6329373be9..5bcb0eec733f5 100644 --- a/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py +++ b/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py @@ -38,16 +38,13 @@ def get_compute_type(dtype): class TestFakeQuantizeAbsMaxOp(OpTest): - def setUp(self): self.op_type = 'fake_quantize_abs_max' self.attrs = {'bit_length': 8} - def _fake_quantize_abs_max(self, - dtype, - input_shape, - distribution, - round_type='TiesAwayFromZero'): + def _fake_quantize_abs_max( + self, dtype, input_shape, distribution, round_type='TiesAwayFromZero' + ): input_data = distribution(input_shape).astype(dtype) compute_type = get_compute_type(dtype) scale = np.max(np.abs(input_data)) @@ -55,12 +52,14 @@ def _fake_quantize_abs_max(self, inv_scale = 1.0 / (scale + 1e-6) if scale < 1e-30 else 1.0 / scale if round_type == 'TiesToEven': round_out = np.round( - input_data.astype(compute_type) * inv_scale * bnt) + input_data.astype(compute_type) * inv_scale * bnt + ) output_data = np.clip(round_out, -bnt - 1, bnt) self.attrs['round_type'] = 0 else: output_data = round_c( - input_data.astype(compute_type) * inv_scale * bnt) + input_data.astype(compute_type) * inv_scale * bnt + ) self.attrs['round_type'] = 1 self.inputs = {'X': input_data} self.outputs = {'Out': output_data, 'OutScale': scale} @@ -71,9 +70,9 @@ def test_fake_quantize_abs_max(self): self._fake_quantize_abs_max(np.float32, (124, 240), np.random.random) def test_fake_quantize_abs_max_round1(self): - self._fake_quantize_abs_max(np.float32, (124, 240), - np.random.random, - round_type='TiesToEven') + self._fake_quantize_abs_max( + np.float32, (124, 240), np.random.random, round_type='TiesToEven' + ) def test_fake_quantize_abs_max_float16(self): self._fake_quantize_abs_max(np.float16, (124, 240), np.random.random) @@ -82,41 +81,45 @@ def test_fake_quantize_abs_max_underflow(self): self._fake_quantize_abs_max(np.float32, (10, 10), np.zeros) def test_fake_quantize_abs_max_underflow2(self): - self._fake_quantize_abs_max(np.float32, (10, 10), - lambda shape: np.full(shape, 1e-40)) + self._fake_quantize_abs_max( + np.float32, (10, 10), lambda shape: np.full(shape, 1e-40) + ) class TestFakeChannelWiseQuantizeAbsMaxOp(OpTest): - def setUp(self): self.op_type = 'fake_channel_wise_quantize_abs_max' self.attrs = {'bit_length': 8} - def _fake_channel_wise_quantize_abs_max(self, - dtype, - input_shape, - quant_axis, - distribution, - round_type='TiesToEven'): + def _fake_channel_wise_quantize_abs_max( + self, + dtype, + input_shape, + quant_axis, + distribution, + round_type='TiesToEven', + ): assert quant_axis in [0, 1], 'quant_axis should be 0 or 1.' input_data = distribution(input_shape).astype(dtype) compute_type = get_compute_type(dtype) bnt = (1 << (self.attrs['bit_length'] - 1)) - 1 - compute_axis = tuple(i for i in range(len(input_shape)) - if i != quant_axis) + compute_axis = tuple( + i for i in range(len(input_shape)) if i != quant_axis + ) scale_broadcast = np.amax(input_data, axis=compute_axis, keepdims=True) if round_type == 'TiesToEven': round_out = np.round( - input_data.astype(compute_type) / scale_broadcast * bnt) + input_data.astype(compute_type) / scale_broadcast * bnt + ) output_data = np.clip(round_out, -bnt - 1, bnt) self.attrs['round_type'] = 0 else: - output_data = round_c(bnt * input_data.astype(compute_type) / - scale_broadcast) + output_data = round_c( + bnt * input_data.astype(compute_type) / scale_broadcast + ) self.attrs['round_type'] = 1 if quant_axis == 1: - scale_broadcast = np.transpose(scale_broadcast, - (1, ) + compute_axis) + scale_broadcast = np.transpose(scale_broadcast, (1,) + compute_axis) scale = scale_broadcast.reshape(input_shape[quant_axis], -1)[:, 0] self.inputs = {'X': input_data} self.outputs = {'Out': output_data, 'OutScale': scale} @@ -126,35 +129,41 @@ def _fake_channel_wise_quantize_abs_max(self, def test_fake_channel_wise_quantize_abs_max(self): dtype_options = [np.float32, np.float16] - input_shape_quant_axis_options = [[(20, 15, 6, 6), 0], - [(20, 15, 6, 6), 1], [(30, 30), 0], - [(30, 30), 1]] + input_shape_quant_axis_options = [ + [(20, 15, 6, 6), 0], + [(20, 15, 6, 6), 1], + [(30, 30), 0], + [(30, 30), 1], + ] round_type_options = ['TiesToEven', 'TiesAwayFromZero'] for dtype, input_shape_quant_axis, round_type in itertools.product( - dtype_options, input_shape_quant_axis_options, - round_type_options): + dtype_options, input_shape_quant_axis_options, round_type_options + ): input_shape, quant_axis = input_shape_quant_axis - with self.subTest(dtype=dtype, - input_shape=input_shape, - quant_axis=quant_axis, - round_type=round_type): + with self.subTest( + dtype=dtype, + input_shape=input_shape, + quant_axis=quant_axis, + round_type=round_type, + ): self._fake_channel_wise_quantize_abs_max( - dtype, input_shape, quant_axis, np.random.random, - round_type) + dtype, input_shape, quant_axis, np.random.random, round_type + ) class TestFakeQuantizeRangeAbsMaxOp(OpTest): - def setUp(self): self.op_type = 'fake_quantize_range_abs_max' self.attrs = {'bit_length': 5, 'window_size': 1} - def _fake_quantize_range_abs_max(self, - dtype, - input_shape, - distribution, - is_test=False, - round_type='TiesToEven'): + def _fake_quantize_range_abs_max( + self, + dtype, + input_shape, + distribution, + is_test=False, + round_type='TiesToEven', + ): input_data = distribution(input_shape).astype(dtype) compute_type = get_compute_type(dtype) bnt = (1 << (self.attrs['bit_length'] - 1)) - 1 @@ -165,7 +174,8 @@ def _fake_quantize_range_abs_max(self, out_scale[0] = in_scale[0] = out_scale[0] - 1.0 if round_type == 'TiesToEven': round_out = np.round( - input_data.astype(compute_type) / out_scale[0] * bnt) + input_data.astype(compute_type) / out_scale[0] * bnt + ) self.attrs['round_type'] = 0 output_data = np.clip(round_out, -bnt - 1, bnt) else: @@ -174,17 +184,18 @@ def _fake_quantize_range_abs_max(self, else: clip_data = input_data output_data = round_c( - clip_data.astype(compute_type) / out_scale[0] * bnt) + clip_data.astype(compute_type) / out_scale[0] * bnt + ) self.attrs['round_type'] = 1 self.inputs = { 'X': input_data, 'Iter': np.zeros(1).astype(np.int64), - 'InScale': in_scale + 'InScale': in_scale, } self.outputs = { 'Out': output_data, 'OutScale': out_scale[0], - 'OutScales': out_scale + 'OutScales': out_scale, } self.dtype = dtype self.attrs['is_test'] = is_test @@ -195,20 +206,22 @@ def test_fake_quantize_range_abs_max(self): is_test_options = [False, True] round_type_options = ['TiesToEven', 'TiesAwayFromZero'] for dtype, is_test, round_type in itertools.product( - dtype_options, is_test_options, round_type_options): + dtype_options, is_test_options, round_type_options + ): self.attrs['bit_length'] = 8 if is_test else 5 - with self.subTest(dtype=dtype, - is_test=is_test, - round_type=round_type): + with self.subTest( + dtype=dtype, is_test=is_test, round_type=round_type + ): self._fake_quantize_range_abs_max( - dtype, (8, 16, 6, 6), + dtype, + (8, 16, 6, 6), lambda shape: (np.random.random(shape) - 0.4) * 10, is_test=is_test, - round_type=round_type) + round_type=round_type, + ) class TestMovingAverageAbsMaxScaleOp(OpTest): - def setUp(self): self.op_type = 'moving_average_abs_max_scale' self.attrs = {'moving_rate': float(0.9), 'is_test': False} @@ -218,41 +231,44 @@ def _moving_average_abs_max_scale(self, dtype, input_shape, distribution): in_accum = np.ones(1).astype(dtype) in_state = np.ones(1).astype(dtype) out_accum = self.attrs['moving_rate'] * in_accum[0] + np.max( - np.abs(input_data)) + np.abs(input_data) + ) out_state = self.attrs['moving_rate'] * in_state[0] + 1.0 out_scale = out_accum / out_state self.inputs = { 'X': input_data, 'InAccum': in_accum, - 'InState': in_state + 'InState': in_state, } self.outputs = { 'Out': input_data, 'OutAccum': out_accum, 'OutState': out_state, - 'OutScale': out_scale + 'OutScale': out_scale, } self.dtype = dtype self.check_output() def test_moving_average_abs_max(self): - self._moving_average_abs_max_scale(np.float32, (8, 16, 7, 7), - np.random.random) + self._moving_average_abs_max_scale( + np.float32, (8, 16, 7, 7), np.random.random + ) class TestFakeQuantizeMovingAverageAbsMaxOp(OpTest): - def setUp(self): self.op_type = 'fake_quantize_moving_average_abs_max' self.attrs = {'bit_length': 5, 'moving_rate': 0.9, 'is_test': False} - def _fake_quantize_moving_average_abs_max(self, - dtype, - input_shape, - distribution, - dequantize=False, - with_gradient=False, - round_type='TiesAwayFromZero'): + def _fake_quantize_moving_average_abs_max( + self, + dtype, + input_shape, + distribution, + dequantize=False, + with_gradient=False, + round_type='TiesAwayFromZero', + ): input_data = distribution(input_shape).astype(dtype) compute_type = get_compute_type(dtype) bnt = (1 << (self.attrs['bit_length'] - 1)) - 1 @@ -263,17 +279,20 @@ def _fake_quantize_moving_average_abs_max(self, out_state = np.zeros(1).astype(dtype) out_scale = np.zeros(1).astype(dtype) out_accum[0] = self.attrs['moving_rate'] * in_accum[0] + np.max( - np.abs(input_data)) + np.abs(input_data) + ) out_state[0] = self.attrs['moving_rate'] * in_state[0] + 1.0 out_scale = out_accum / out_state if round_type == 'TiesToEven': round_out = np.round( - input_data.astype(compute_type) / out_scale * bnt) + input_data.astype(compute_type) / out_scale * bnt + ) quant_data = np.clip(round_out, -bnt - 1, bnt) self.attrs['round_type'] = 0 else: quant_data = round_c( - input_data.astype(compute_type) / out_scale * bnt) + input_data.astype(compute_type) / out_scale * bnt + ) self.attrs['round_type'] = 1 if dequantize: output_data = (quant_data * out_scale / bnt).astype(dtype) @@ -284,13 +303,13 @@ def _fake_quantize_moving_average_abs_max(self, 'X': input_data, 'InScale': in_scale, 'InAccum': in_accum, - 'InState': in_state + 'InState': in_state, } self.outputs = { 'Out': output_data, 'OutAccum': out_accum, 'OutState': out_state, - 'OutScale': out_scale + 'OutScale': out_scale, } self.dtype = dtype self.check_output() @@ -301,36 +320,38 @@ def _fake_quantize_moving_average_abs_max(self, self.check_grad(['X'], 'Out', user_defined_grads=gradient) def test_fake_quantize_moving_average_abs_max(self): - self._fake_quantize_moving_average_abs_max(np.float32, (8, 16, 7, 7), - np.random.random) + self._fake_quantize_moving_average_abs_max( + np.float32, (8, 16, 7, 7), np.random.random + ) def test_fake_quantize_moving_average_abs_max_float16(self): - self._fake_quantize_moving_average_abs_max(np.float16, (8, 16, 7, 7), - np.random.random) + self._fake_quantize_moving_average_abs_max( + np.float16, (8, 16, 7, 7), np.random.random + ) def test_fake_quantize_moving_average_abs_max_round1(self): - self._fake_quantize_moving_average_abs_max(np.float32, (8, 16, 7, 7), - np.random.random, - round_type='TiesToEven') + self._fake_quantize_moving_average_abs_max( + np.float32, (8, 16, 7, 7), np.random.random, round_type='TiesToEven' + ) def test_fake_quantize_dequantize_moving_average_abs_max(self): - self._fake_quantize_moving_average_abs_max(np.float32, (8, 16, 7, 7), - np.random.random, - dequantize=True, - with_gradient=True) + self._fake_quantize_moving_average_abs_max( + np.float32, + (8, 16, 7, 7), + np.random.random, + dequantize=True, + with_gradient=True, + ) class TestFakeQuantizeDequantizeAbsMaxOp(OpTest): - def setUp(self): self.op_type = 'fake_quantize_dequantize_abs_max' self.attrs = {'bit_length': 8} - def _fake_quantize_dequantize_abs_max(self, - dtype, - input_shape, - distribution, - round_type='TiesAwayFromZero'): + def _fake_quantize_dequantize_abs_max( + self, dtype, input_shape, distribution, round_type='TiesAwayFromZero' + ): input_data = distribution(input_shape).astype(dtype) scale = np.max(np.abs(input_data)).astype(dtype) bnt = (1 << (self.attrs['bit_length'] - 1)) - 1 @@ -344,7 +365,7 @@ def _fake_quantize_dequantize_abs_max(self, self.inputs = {'X': input_data} self.outputs = { 'Out': output_data, - 'OutScale': np.array(scale).astype(dtype) + 'OutScale': np.array(scale).astype(dtype), } self.dtype = dtype self.check_output() @@ -352,47 +373,53 @@ def _fake_quantize_dequantize_abs_max(self, self.check_grad(['X'], 'Out', user_defined_grads=gradient) def test_fake_quantize_dequantize_abs_max(self): - self._fake_quantize_dequantize_abs_max(np.float32, (124, 240), - np.random.random) + self._fake_quantize_dequantize_abs_max( + np.float32, (124, 240), np.random.random + ) def test_fake_quantize_dequantize_abs_max_round1(self): - self._fake_quantize_dequantize_abs_max(np.float32, (124, 240), - np.random.random, - round_type='TiesToEven') + self._fake_quantize_dequantize_abs_max( + np.float32, (124, 240), np.random.random, round_type='TiesToEven' + ) class TestChannelWiseFakeQuantizeDequantizeAbsMaxOp(OpTest): - def setUp(self): self.op_type = 'fake_channel_wise_quantize_dequantize_abs_max' self.attrs = {'bit_length': 8} - def _fake_channel_wise_quantize_dequantize_abs_max(self, - dtype, - input_shape, - quant_axis, - distribution, - round_type='TiesToEven'): + def _fake_channel_wise_quantize_dequantize_abs_max( + self, + dtype, + input_shape, + quant_axis, + distribution, + round_type='TiesToEven', + ): assert quant_axis in [0, 1], 'quant_axis should be 0 or 1.' input_data = distribution(input_shape).astype(dtype) compute_type = get_compute_type(dtype) bnt = (1 << (self.attrs['bit_length'] - 1)) - 1 output_data = input_data.copy().astype(compute_type) - compute_axis = tuple(i for i in range(len(input_shape)) - if i != quant_axis) + compute_axis = tuple( + i for i in range(len(input_shape)) if i != quant_axis + ) scale_broadcast = np.amax(input_data, axis=compute_axis, keepdims=True) if round_type == 'TiesToEven': round_out = np.round(bnt * output_data / scale_broadcast) - output_data = np.clip(round_out, -bnt - 1, - bnt) * scale_broadcast / bnt + output_data = ( + np.clip(round_out, -bnt - 1, bnt) * scale_broadcast / bnt + ) self.attrs['round_type'] = 0 else: - output_data = round_c( - bnt * output_data / scale_broadcast) * scale_broadcast / bnt + output_data = ( + round_c(bnt * output_data / scale_broadcast) + * scale_broadcast + / bnt + ) self.attrs['round_type'] = 1 if quant_axis == 1: - scale_broadcast = np.transpose(scale_broadcast, - (1, ) + compute_axis) + scale_broadcast = np.transpose(scale_broadcast, (1,) + compute_axis) scale = scale_broadcast.reshape(input_shape[quant_axis], -1)[:, 0] self.inputs = {'X': input_data} self.outputs = {'Out': output_data, 'OutScale': scale} @@ -403,22 +430,29 @@ def _fake_channel_wise_quantize_dequantize_abs_max(self, self.check_grad(['X'], 'Out', user_defined_grads=gradient) def test_channel_wise_fake_quant_dequant_abs_max(self): - input_shape_quant_axis_options = [[(3, 4, 64, 64), 0], - [(15, 20, 5, 5), 1], [(30, 15), 0], - [(30, 15), 1]] + input_shape_quant_axis_options = [ + [(3, 4, 64, 64), 0], + [(15, 20, 5, 5), 1], + [(30, 15), 0], + [(30, 15), 1], + ] round_type_options = ['TiesToEven', 'TiesAwayFromZero'] for input_shape_quant_axis, round_type in itertools.product( - input_shape_quant_axis_options, round_type_options): + input_shape_quant_axis_options, round_type_options + ): input_shape, quant_axis = input_shape_quant_axis - with self.subTest(input_shape=input_shape, - quant_axis=quant_axis, - round_type=round_type): + with self.subTest( + input_shape=input_shape, + quant_axis=quant_axis, + round_type=round_type, + ): self._fake_channel_wise_quantize_dequantize_abs_max( np.float32, input_shape, quant_axis, np.random.random, - round_type=round_type) + round_type=round_type, + ) def quantize_max_abs(x, max_range): @@ -446,7 +480,6 @@ def channel_wise_quantize_max_abs(x, quant_bit=8, quant_axis=0): class TestChannelWiseQuantizeOp(OpTest): - def set_args(self): self.bit_length = 8 self.data_type = "float32" @@ -456,15 +489,16 @@ def setUp(self): self.set_args() self.op_type = "quantize_linear" x = np.random.randn(4, 3, 64, 64).astype(self.data_type) - yq, scale = channel_wise_quantize_max_abs(x, self.bit_length, - self.quant_axis) + yq, scale = channel_wise_quantize_max_abs( + x, self.bit_length, self.quant_axis + ) scale = np.array(scale).astype(self.data_type) zero_point = np.zeros(scale.shape, dtype="int32") self.inputs = {'X': x, 'Scale': scale, 'ZeroPoint': zero_point} self.attrs = { 'bit_length': self.bit_length, - 'quant_axis': self.quant_axis + 'quant_axis': self.quant_axis, } self.outputs = {'Y': yq} @@ -473,7 +507,6 @@ def test_check_output(self): class TestChannelWiseQuantizeOp1(TestChannelWiseQuantizeOp): - def set_args(self): self.bit_length = 8 self.data_type = "float32" @@ -481,7 +514,6 @@ def set_args(self): class TestChannelWiseQuantizeOpTrain(OpTest): - def set_args(self): self.bit_length = 8 self.data_type = "float32" @@ -492,8 +524,9 @@ def setUp(self): self.set_args() self.op_type = "quantize_linear" x = np.random.randn(4, 3, 64, 64).astype(self.data_type) - yq, scale = channel_wise_quantize_max_abs(x, self.bit_length, - self.quant_axis) + yq, scale = channel_wise_quantize_max_abs( + x, self.bit_length, self.quant_axis + ) scale = np.array(scale).astype(self.data_type) zero_point = np.zeros(scale.shape, dtype="int32") @@ -501,7 +534,7 @@ def setUp(self): self.attrs = { 'bit_length': self.bit_length, 'quant_axis': self.quant_axis, - 'is_test': self.is_test + 'is_test': self.is_test, } self.outputs = {'Y': yq, 'OutScale': scale} @@ -510,7 +543,6 @@ def test_check_output(self): class TestquantizeOp(OpTest): - def set_args(self): self.bit_length = 8 self.quant_axis = -1 @@ -537,7 +569,6 @@ def test_check_output(self): class TestquantizeOpTrain(TestquantizeOp): - def set_args(self): self.bit_length = 8 self.quant_axis = -1 @@ -552,7 +583,7 @@ def setUp(self): 'bit_length': self.bit_length, 'quant_axis': self.quant_axis, 'moving_rate': 0.9, - 'is_test': self.is_test + 'is_test': self.is_test, } x = np.random.randn(31, 65).astype(self.data_type) @@ -563,7 +594,8 @@ def setUp(self): out_accum = np.zeros(1).astype(self.data_type) out_state = np.zeros(1).astype(self.data_type) out_accum[0] = self.attrs['moving_rate'] * in_accum[0] + np.max( - np.abs(x)) + np.abs(x) + ) out_state[0] = self.attrs['moving_rate'] * in_state[0] + 1.0 out_scale = out_accum / out_state diff --git a/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py b/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py index 44aa0225059ee..b21f25b75a364 100755 --- a/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py +++ b/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py @@ -38,8 +38,13 @@ def to_string_tensor(string_values, name): string_values(list[string]): The value will be setted to the tensor. name(string): The name of the tensor. """ - tensor = paddle.Tensor(core.VarDesc.VarType.STRING, [], name, - core.VarDesc.VarType.STRINGS, False) + tensor = paddle.Tensor( + core.VarDesc.VarType.STRING, + [], + name, + core.VarDesc.VarType.STRINGS, + False, + ) tensor.value().set_string_list(string_values) return tensor @@ -54,31 +59,42 @@ def to_map_tensor(string_dict, name): string_dict(dict): The value will be setted to the tensor. name(string): The name of the tensor. """ - tensor = paddle.Tensor(core.VarDesc.VarType.RAW, [], name, - core.VarDesc.VarType.VOCAB, True) + tensor = paddle.Tensor( + core.VarDesc.VarType.RAW, [], name, core.VarDesc.VarType.VOCAB, True + ) tensor.value().set_vocab(string_dict) return tensor class FasterTokenizer(nn.Layer): - def __init__(self, vocab_dict): super(FasterTokenizer, self).__init__() vocab_tensor = to_map_tensor(vocab_dict, "vocab") self.register_buffer("vocab", vocab_tensor, persistable=True) - def forward(self, - text, - text_pair=None, - do_lower_case=True, - max_seq_len=-1, - is_split_into_words=False, - pad_to_max_seq_len=False): + def forward( + self, + text, + text_pair=None, + do_lower_case=True, + max_seq_len=-1, + is_split_into_words=False, + pad_to_max_seq_len=False, + ): if _non_static_mode(): input_ids, seg_ids = _legacy_C_ops.faster_tokenizer( - self.vocab, text, text_pair, "do_lower_case", do_lower_case, - "max_seq_len", max_seq_len, "pad_to_max_seq_len", - pad_to_max_seq_len, "is_split_into_words", is_split_into_words) + self.vocab, + text, + text_pair, + "do_lower_case", + do_lower_case, + "max_seq_len", + max_seq_len, + "pad_to_max_seq_len", + pad_to_max_seq_len, + "is_split_into_words", + is_split_into_words, + ) return input_ids, seg_ids attrs = { @@ -91,33 +107,27 @@ def forward(self, input_ids = helper.create_variable_for_type_inference(dtype="int64") seg_ids = helper.create_variable_for_type_inference(dtype="int64") if text_pair is None: - helper.append_op(type='faster_tokenizer', - inputs={ - 'Vocab': self.vocab, - 'Text': text - }, - outputs={ - 'InputIds': input_ids, - 'SegmentIds': seg_ids - }, - attrs=attrs) + helper.append_op( + type='faster_tokenizer', + inputs={'Vocab': self.vocab, 'Text': text}, + outputs={'InputIds': input_ids, 'SegmentIds': seg_ids}, + attrs=attrs, + ) else: - helper.append_op(type='faster_tokenizer', - inputs={ - 'Vocab': self.vocab, - 'Text': text, - 'TextPair': text_pair - }, - outputs={ - 'InputIds': input_ids, - 'SegmentIds': seg_ids - }, - attrs=attrs) + helper.append_op( + type='faster_tokenizer', + inputs={ + 'Vocab': self.vocab, + 'Text': text, + 'TextPair': text_pair, + }, + outputs={'InputIds': input_ids, 'SegmentIds': seg_ids}, + attrs=attrs, + ) return input_ids, seg_ids class Predictor(object): - def __init__(self, model_dir): model_file = os.path.join(model_dir, "inference.pdmodel") params_file = os.path.join(model_dir, "inference.pdiparams") @@ -152,7 +162,6 @@ def predict(self, data): class TestBertTokenizerOp(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.bert_tokenizer = BertTokenizer.from_pretrained("bert-base-chinese") @@ -177,11 +186,12 @@ def init_data(self): '很好的地理位置，一蹋糊涂的服务，萧条的酒店。', ' 选择珠江花园的原因就是方便，有电动扶梯直接到达海边，周围餐馆、食廊、商场、超市、摊位一应俱全。酒店装修一般，' '但还算整洁。 泳池在大堂的屋顶，因此很小，不过女儿倒是喜欢。 包的早餐是西式的，还算丰富。 服务吗，一般', - 'Test bert tokenizer. The first text.' + 'Test bert tokenizer. The first text.', ] self.text_pairs = [ - '非常不错，服务很好，位于市中心区，交通方便，不过价格也高！', '房间太小。其他的都一般。。。。。。。。。', - 'Test bert tokenizer. The second text.' + '非常不错，服务很好，位于市中心区，交通方便，不过价格也高！', + '房间太小。其他的都一般。。。。。。。。。', + 'Test bert tokenizer. The second text.', ] self.texts_tensor = to_string_tensor(self.texts, "texts") self.text_pairs_tensor = to_string_tensor(self.text_pairs, "text_pairs") @@ -198,7 +208,8 @@ def run_padding(self): do_lower_case=self.bert_tokenizer.do_lower_case, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) input_ids = input_ids.numpy() token_type_ids = token_type_ids.numpy() @@ -206,15 +217,16 @@ def run_padding(self): text=self.text, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) py_input_ids = np.array(encoded_inputs[0]["input_ids"]).reshape([1, -1]) py_token_type_ids = np.array( - encoded_inputs[0]["token_type_ids"]).reshape([1, -1]) + encoded_inputs[0]["token_type_ids"] + ).reshape([1, -1]) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) # case 2: only one text and one text_pair (batch_size = 1) input_ids, token_type_ids = self.faster_tokenizer( @@ -223,7 +235,8 @@ def run_padding(self): do_lower_case=self.bert_tokenizer.do_lower_case, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) input_ids = input_ids.numpy() token_type_ids = token_type_ids.numpy() @@ -232,15 +245,16 @@ def run_padding(self): text_pair=self.text_pair, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) py_input_ids = np.array(encoded_inputs[0]["input_ids"]).reshape([1, -1]) py_token_type_ids = np.array( - encoded_inputs[0]["token_type_ids"]).reshape([1, -1]) + encoded_inputs[0]["token_type_ids"] + ).reshape([1, -1]) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) # case 3: only texts (batch_size = 3) input_ids, token_type_ids = self.faster_tokenizer( @@ -248,7 +262,8 @@ def run_padding(self): do_lower_case=self.bert_tokenizer.do_lower_case, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) input_ids = input_ids.numpy() token_type_ids = token_type_ids.numpy() @@ -256,16 +271,16 @@ def run_padding(self): self.texts, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) py_input_ids = [i["input_ids"] for i in encoded_inputs] py_token_type_ids = [i["token_type_ids"] for i in encoded_inputs] py_input_ids = np.array(py_input_ids).reshape([3, -1]) py_token_type_ids = np.array(py_token_type_ids).reshape([3, -1]) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) # case 4: texts and text pairs (batch_size = 3) input_ids, token_type_ids = self.faster_tokenizer( @@ -274,7 +289,8 @@ def run_padding(self): do_lower_case=self.bert_tokenizer.do_lower_case, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) input_ids = input_ids.numpy() token_type_ids = token_type_ids.numpy() @@ -283,16 +299,16 @@ def run_padding(self): self.text_pairs, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) py_input_ids = [i["input_ids"] for i in encoded_inputs] py_token_type_ids = [i["token_type_ids"] for i in encoded_inputs] py_input_ids = np.array(py_input_ids).reshape([3, -1]) py_token_type_ids = np.array(py_token_type_ids).reshape([3, -1]) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) def test_padding(self): with _test_eager_guard(): @@ -311,7 +327,8 @@ def run_no_padding(self): do_lower_case=self.bert_tokenizer.do_lower_case, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) input_ids = input_ids.numpy() token_type_ids = token_type_ids.numpy() @@ -319,15 +336,16 @@ def run_no_padding(self): self.text, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) py_input_ids = np.array(encoded_inputs[0]["input_ids"]).reshape([1, -1]) py_token_type_ids = np.array( - encoded_inputs[0]["token_type_ids"]).reshape([1, -1]) + encoded_inputs[0]["token_type_ids"] + ).reshape([1, -1]) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) # case 2: only one text and one text_pair (batch_size = 1) input_ids, token_type_ids = self.faster_tokenizer( @@ -336,7 +354,8 @@ def run_no_padding(self): do_lower_case=self.bert_tokenizer.do_lower_case, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) input_ids = input_ids.numpy() token_type_ids = token_type_ids.numpy() @@ -345,15 +364,16 @@ def run_no_padding(self): self.text_pair, max_seq_len=self.max_seq_len, pad_to_max_seq_len=self.pad_to_max_seq_len, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) py_input_ids = np.array(encoded_inputs[0]["input_ids"]).reshape([1, -1]) py_token_type_ids = np.array( - encoded_inputs[0]["token_type_ids"]).reshape([1, -1]) + encoded_inputs[0]["token_type_ids"] + ).reshape([1, -1]) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) def test_no_padding(self): with _test_eager_guard(): @@ -367,19 +387,21 @@ def run_is_split_into_words(self): input_ids, token_type_ids = self.faster_tokenizer( self.text_tensor, do_lower_case=self.bert_tokenizer.do_lower_case, - is_split_into_words=self.is_split_into_words) + is_split_into_words=self.is_split_into_words, + ) input_ids = input_ids.numpy() token_type_ids = token_type_ids.numpy() encoded_inputs = self.bert_tokenizer( - list(self.text[0]), is_split_into_words=self.is_split_into_words) + list(self.text[0]), is_split_into_words=self.is_split_into_words + ) py_input_ids = np.array(encoded_inputs["input_ids"]).reshape([1, -1]) py_token_type_ids = np.array(encoded_inputs["token_type_ids"]).reshape( - [1, -1]) + [1, -1] + ) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) def test_is_split_into_words(self): with _test_eager_guard(): @@ -398,8 +420,10 @@ def test_inference(self): self.faster_tokenizer, input_spec=[ paddle.static.InputSpec( - shape=[None], dtype=core.VarDesc.VarType.STRINGS), # texts - ]) + shape=[None], dtype=core.VarDesc.VarType.STRINGS + ), # texts + ], + ) # Save in static graph model. paddle.jit.save(static_model, self.inference_path) predictor = Predictor(self.save_path) @@ -408,19 +432,19 @@ def test_inference(self): encoded_inputs = self.bert_tokenizer(self.text) py_input_ids = np.array(encoded_inputs[0]["input_ids"]).reshape([1, -1]) py_token_type_ids = np.array( - encoded_inputs[0]["token_type_ids"]).reshape([1, -1]) + encoded_inputs[0]["token_type_ids"] + ).reshape([1, -1]) np.testing.assert_allclose(input_ids, py_input_ids, rtol=0, atol=0.01) - np.testing.assert_allclose(token_type_ids, - py_token_type_ids, - rtol=0, - atol=0.01) + np.testing.assert_allclose( + token_type_ids, py_token_type_ids, rtol=0, atol=0.01 + ) def test_feed_string_var(self): self.init_data() paddle.enable_static() - x = paddle.static.data(name="x", - shape=[-1], - dtype=core.VarDesc.VarType.STRINGS) + x = paddle.static.data( + name="x", shape=[-1], dtype=core.VarDesc.VarType.STRINGS + ) exe = paddle.static.Executor(paddle.framework.CPUPlace()) exe.run(paddle.static.default_main_program(), feed={'x': self.text}) paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_fc_op.py b/python/paddle/fluid/tests/unittests/test_fc_op.py index ac69b8e8c6ff3..8003c0f99e84b 100644 --- a/python/paddle/fluid/tests/unittests/test_fc_op.py +++ b/python/paddle/fluid/tests/unittests/test_fc_op.py @@ -43,7 +43,6 @@ def fc_refer(matrix, with_bias, with_relu=False): class MatrixGenerate: - def __init__(self, mb, ic, oc, h, w, bias_dims=2): self.input = np.random.random((mb, ic, h, w)).astype("float32") self.weights = np.random.random((ic * h * w, oc)).astype("float32") @@ -54,7 +53,6 @@ def __init__(self, mb, ic, oc, h, w, bias_dims=2): class TestFCOp(OpTest): - def config(self): self.with_bias = True self.with_relu = True @@ -68,7 +66,7 @@ def setUp(self): self.inputs = { 'Input': self.matrix.input, 'W': self.matrix.weights, - 'Bias': self.matrix.bias + 'Bias': self.matrix.bias, } else: self.inputs = {'Input': self.matrix.input, 'W': self.matrix.weights} @@ -88,7 +86,6 @@ def test_check_output(self): class TestFCOpNoBias1(TestFCOp): - def config(self): self.with_bias = False self.with_relu = False @@ -96,7 +93,6 @@ def config(self): class TestFCOpNoBias2(TestFCOp): - def config(self): self.with_bias = False self.with_relu = False @@ -104,7 +100,6 @@ def config(self): class TestFCOpNoBias4(TestFCOp): - def config(self): self.with_bias = False self.with_relu = False @@ -112,7 +107,6 @@ def config(self): class TestFCOpWithBias1(TestFCOp): - def config(self): self.with_bias = True self.with_relu = False @@ -120,7 +114,6 @@ def config(self): class TestFCOpWithBias2(TestFCOp): - def config(self): self.with_bias = True self.with_relu = True @@ -128,7 +121,6 @@ def config(self): class TestFCOpWithBias3(TestFCOp): - def config(self): self.with_bias = True self.with_relu = True @@ -136,7 +128,6 @@ def config(self): class TestFCOpWithPadding(TestFCOp): - def config(self): self.with_bias = True self.with_relu = True @@ -144,9 +135,7 @@ def config(self): class TestFcOp_NumFlattenDims_NegOne(unittest.TestCase): - def test_api(self): - def run_program(num_flatten_dims): paddle.seed(SEED) np.random.seed(SEED) @@ -155,17 +144,22 @@ def run_program(num_flatten_dims): with program_guard(main_program, startup_program): input = np.random.random([2, 2, 25]).astype("float32") - x = fluid.layers.data(name="x", - shape=[2, 2, 25], - append_batch_size=False, - dtype="float32") - - out = paddle.static.nn.fc(x=x, - size=1, - num_flatten_dims=num_flatten_dims) - - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + x = fluid.layers.data( + name="x", + shape=[2, 2, 25], + append_batch_size=False, + dtype="float32", + ) + + out = paddle.static.nn.fc( + x=x, size=1, num_flatten_dims=num_flatten_dims + ) + + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe = fluid.Executor(place=place) exe.run(startup_program) out = exe.run(main_program, feed={"x": input}, fetch_list=[out]) @@ -177,7 +171,6 @@ def run_program(num_flatten_dims): class TestFCOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.random((2, 4)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_feed_data_check_shape_type.py b/python/paddle/fluid/tests/unittests/test_feed_data_check_shape_type.py index be58e03f661af..3b885f26683cd 100644 --- a/python/paddle/fluid/tests/unittests/test_feed_data_check_shape_type.py +++ b/python/paddle/fluid/tests/unittests/test_feed_data_check_shape_type.py @@ -27,8 +27,8 @@ class TestFeedData(unittest.TestCase): ''' - Test paddle.fluid.data feeds with different shape and types. - Note: paddle.fluid.data is not paddle.fluid.layers.data. + Test paddle.fluid.data feeds with different shape and types. + Note: paddle.fluid.data is not paddle.fluid.layers.data. ''' def setUp(self): @@ -38,16 +38,22 @@ def setUp(self): self.iterations = 5 def _get_device_count(self, use_cuda): - return core.get_cuda_device_count() if use_cuda else int( - os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + return ( + core.get_cuda_device_count() + if use_cuda + else int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + ) def _get_feed_batch_size(self, use_cuda, use_parallel_executor): """ Returns actual fed data size. We should multiple the number of devices when it is using ParallelExecutor """ - return self.data_batch_size * self._get_device_count( - use_cuda) if use_parallel_executor else self.data_batch_size + return ( + self.data_batch_size * self._get_device_count(use_cuda) + if use_parallel_executor + else self.data_batch_size + ) def _simple_fc_net(self, in_size, label_size, class_num, hidden_sizes): in_data = fluid.data(name="data", dtype='float32', shape=in_size) @@ -59,26 +65,32 @@ def _simple_fc_net(self, in_size, label_size, class_num, hidden_sizes): predict_label = fluid.layers.fc(hidden, size=class_num, act='softmax') loss = paddle.mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) return in_data, label, loss def test(self): - for use_cuda in [True, False - ] if core.is_compiled_with_cuda() else [False]: + for use_cuda in ( + [True, False] if core.is_compiled_with_cuda() else [False] + ): for use_parallel_executor in [False, True]: print('Test Parameters:'), - print({ - 'use_cuda': use_cuda, - 'use_parallel_executor': use_parallel_executor, - }) + print( + { + 'use_cuda': use_cuda, + 'use_parallel_executor': use_parallel_executor, + } + ) # Test feeding without error - self._test_feed_data_match_shape_type(use_cuda, - use_parallel_executor) - self._test_feed_data_contains_neg_one(use_cuda, - use_parallel_executor) + self._test_feed_data_match_shape_type( + use_cuda, use_parallel_executor + ) + self._test_feed_data_contains_neg_one( + use_cuda, use_parallel_executor + ) self._test_feed_lod_tensor(use_cuda, use_parallel_executor) # Test exception message when feeding with error @@ -86,74 +98,110 @@ def test(self): error_shape_list = [self.data_batch_size, 3, 4, 5] with self.assertRaises(ValueError) as shape_mismatch_err: - self._test_feed_data_shape_mismatch(use_cuda, - use_parallel_executor) + self._test_feed_data_shape_mismatch( + use_cuda, use_parallel_executor + ) self.assertEqual( str(shape_mismatch_err.exception), "The fed Variable %r should have dimensions = %r, " - "shape = %r, but received fed shape %r on each device" % - (u'data', len(in_shape_tuple), in_shape_tuple, - error_shape_list)) + "shape = %r, but received fed shape %r on each device" + % ( + u'data', + len(in_shape_tuple), + in_shape_tuple, + error_shape_list, + ), + ) with self.assertRaises(ValueError) as dtype_mismatch_err: - self._test_feed_data_dtype_mismatch(use_cuda, - use_parallel_executor) + self._test_feed_data_dtype_mismatch( + use_cuda, use_parallel_executor + ) self.assertEqual( str(dtype_mismatch_err.exception), "The data type of fed Variable %r must be 'int64', but " - "received 'float64'" % (u'label')) + "received 'float64'" % (u'label'), + ) def _test_feed_data_dtype_mismatch(self, use_cuda, use_parallel_executor): - feed_batch_size = self._get_feed_batch_size(use_cuda, - use_parallel_executor) + feed_batch_size = self._get_feed_batch_size( + use_cuda, use_parallel_executor + ) in_size = [self.data_batch_size, 3, 4, 5] feed_in_data = np.random.uniform( - size=[feed_batch_size, 3, 4, 5]).astype(np.float32) + size=[feed_batch_size, 3, 4, 5] + ).astype(np.float32) label_size = [self.data_batch_size, 1] - feed_label = np.random.randint(low=0, - high=self.class_num, - size=[feed_batch_size, - 1]).astype(np.float64) - self._feed_data_in_executor(in_size, label_size, feed_in_data, - feed_label, use_cuda, use_parallel_executor) + feed_label = np.random.randint( + low=0, high=self.class_num, size=[feed_batch_size, 1] + ).astype(np.float64) + self._feed_data_in_executor( + in_size, + label_size, + feed_in_data, + feed_label, + use_cuda, + use_parallel_executor, + ) def _test_feed_data_shape_mismatch(self, use_cuda, use_parallel_executor): batch_size = self._get_feed_batch_size(use_cuda, use_parallel_executor) in_size = [None, 3, 4, 8] feed_in_data = np.random.uniform(size=[batch_size, 3, 4, 5]).astype( - np.float32) + np.float32 + ) label_size = [-1, 1] - feed_label = np.random.randint(low=0, - high=self.class_num, - size=[batch_size, 1]).astype(np.int64) - self._feed_data_in_executor(in_size, label_size, feed_in_data, - feed_label, use_cuda, use_parallel_executor) + feed_label = np.random.randint( + low=0, high=self.class_num, size=[batch_size, 1] + ).astype(np.int64) + self._feed_data_in_executor( + in_size, + label_size, + feed_in_data, + feed_label, + use_cuda, + use_parallel_executor, + ) def _test_feed_data_contains_neg_one(self, use_cuda, use_parallel_executor): batch_size = self._get_feed_batch_size(use_cuda, use_parallel_executor) in_size = [-1, 3, 4, 5] feed_in_data = np.random.uniform(size=[batch_size, 3, 4, 5]).astype( - np.float32) + np.float32 + ) label_size = (None, 1) - feed_label = np.random.randint(low=0, - high=self.class_num, - size=[batch_size, 1]).astype(np.int64) - self._feed_data_in_executor(in_size, label_size, feed_in_data, - feed_label, use_cuda, use_parallel_executor) + feed_label = np.random.randint( + low=0, high=self.class_num, size=[batch_size, 1] + ).astype(np.int64) + self._feed_data_in_executor( + in_size, + label_size, + feed_in_data, + feed_label, + use_cuda, + use_parallel_executor, + ) def _test_feed_data_match_shape_type(self, use_cuda, use_parallel_executor): - feed_batch_size = self._get_feed_batch_size(use_cuda, - use_parallel_executor) + feed_batch_size = self._get_feed_batch_size( + use_cuda, use_parallel_executor + ) in_size = [self.data_batch_size, 3, 4, 5] feed_in_data = np.random.uniform( - size=[feed_batch_size, 3, 4, 5]).astype(np.float32) + size=[feed_batch_size, 3, 4, 5] + ).astype(np.float32) label_size = [self.data_batch_size, 1] - feed_label = np.random.randint(low=0, - high=self.class_num, - size=[feed_batch_size, - 1]).astype(np.int64) - self._feed_data_in_executor(in_size, label_size, feed_in_data, - feed_label, use_cuda, use_parallel_executor) + feed_label = np.random.randint( + low=0, high=self.class_num, size=[feed_batch_size, 1] + ).astype(np.int64) + self._feed_data_in_executor( + in_size, + label_size, + feed_in_data, + feed_label, + use_cuda, + use_parallel_executor, + ) def _test_feed_lod_tensor(self, use_cuda, use_parallel_executor): device_count = self._get_device_count(use_cuda) @@ -164,33 +212,46 @@ def _test_feed_lod_tensor(self, use_cuda, use_parallel_executor): sum_length = int((device_count + 1) * device_count / 2) feed_in_data = np.random.uniform(size=[sum_length, 3, 4, 5]).astype( - np.float32) + np.float32 + ) feed_data_tensor = fluid.LoDTensor() feed_data_tensor.set(feed_in_data, fluid.CPUPlace()) feed_data_tensor.set_recursive_sequence_lengths(sequence_lengths) label_size = [device_count, 1] feed_label_tensor = fluid.LoDTensor() - feed_label = np.random.randint(low=0, - high=self.class_num, - size=[sum_length, 1]).astype(np.int64) + feed_label = np.random.randint( + low=0, high=self.class_num, size=[sum_length, 1] + ).astype(np.int64) feed_label_tensor.set(feed_label, fluid.CPUPlace()) feed_label_tensor.set_recursive_sequence_lengths(sequence_lengths) - self._feed_data_in_executor(in_size, label_size, feed_data_tensor, - feed_label_tensor, use_cuda, - use_parallel_executor) - - def _feed_data_in_executor(self, in_size, label_size, feed_in_data, - feed_label, use_cuda, use_parallel_executor): + self._feed_data_in_executor( + in_size, + label_size, + feed_data_tensor, + feed_label_tensor, + use_cuda, + use_parallel_executor, + ) + + def _feed_data_in_executor( + self, + in_size, + label_size, + feed_in_data, + feed_label, + use_cuda, + use_parallel_executor, + ): startup_program = fluid.Program() main_program = fluid.Program() with fluid.program_guard(main_program, startup_program): - in_data, label, loss = self._simple_fc_net(in_size, label_size, - self.class_num, - self.hidden_sizes) + in_data, label, loss = self._simple_fc_net( + in_size, label_size, self.class_num, self.hidden_sizes + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() @@ -200,15 +261,15 @@ def _feed_data_in_executor(self, in_size, label_size, feed_in_data, train_program = main_program if use_parallel_executor: train_program = compiler.CompiledProgram( - main_program).with_data_parallel(loss_name=loss.name) + main_program + ).with_data_parallel(loss_name=loss.name) for i in range(self.iterations): - fetches = exe.run(train_program, - feed={ - in_data.name: feed_in_data, - label.name: feed_label - }, - fetch_list=[loss.name]) + fetches = exe.run( + train_program, + feed={in_data.name: feed_in_data, label.name: feed_label}, + fetch_list=[loss.name], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py b/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py index 1a6d780fb61fe..4054af8fb5192 100644 --- a/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py +++ b/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py @@ -18,7 +18,6 @@ class TestFeedFetch(unittest.TestCase): - def test_feed_fetch(self): scope = core.Scope() place = core.CPUPlace() diff --git a/python/paddle/fluid/tests/unittests/test_fetch_handler.py b/python/paddle/fluid/tests/unittests/test_fetch_handler.py index 562c79cc6ebed..6a8b1f689fd79 100644 --- a/python/paddle/fluid/tests/unittests/test_fetch_handler.py +++ b/python/paddle/fluid/tests/unittests/test_fetch_handler.py @@ -22,7 +22,6 @@ class TestFetchHandler(unittest.TestCase): - @unittest.skip(reason="Skip unstable ci") def test_fetch_handler(self): place = core.CPUPlace() @@ -36,7 +35,6 @@ def test_fetch_handler(self): var_emb3 = block.create_var(name='emb3', type=core.VarDesc.VarType.FP32) class FH(fluid.executor.FetchHandler): - def handler(self, fetch_dict): assert len(fetch_dict) == 1 @@ -49,14 +47,13 @@ def handler(self, fetch_dict): time.sleep(3) fm.stop() - default_fh = fluid.executor.FetchHandler(var_dict={ - 'emb': var_emb, - 'emb2': None, - 'emb3': var_emb3 - }, - period_secs=1) + default_fh = fluid.executor.FetchHandler( + var_dict={'emb': var_emb, 'emb2': None, 'emb3': var_emb3}, + period_secs=1, + ) default_fm = fluid.trainer_factory.FetchHandlerMonitor( - scope, default_fh) + scope, default_fh + ) default_fm.start() time.sleep(5) default_fm.stop() diff --git a/python/paddle/fluid/tests/unittests/test_fetch_lod_tensor_array.py b/python/paddle/fluid/tests/unittests/test_fetch_lod_tensor_array.py index 2b59c88948dde..c096a4c8a6e02 100644 --- a/python/paddle/fluid/tests/unittests/test_fetch_lod_tensor_array.py +++ b/python/paddle/fluid/tests/unittests/test_fetch_lod_tensor_array.py @@ -21,7 +21,6 @@ class TestFetchLoDTensorArray(unittest.TestCase): - def build_program(self, main_program, startup_program): with fluid.unique_name.guard(): with fluid.program_guard(main_program, startup_program): @@ -59,27 +58,34 @@ def check_network(self, use_cuda=True): build_strategy = fluid.BuildStrategy() binary = fluid.CompiledProgram(main_program).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) device_num = fluid.core.get_cuda_device_count() if use_cuda else 2 for _ in range(3): - loss_v, array_v = exe.run(binary, - feed=feed_dict, - fetch_list=[loss, array], - return_merged=False) + loss_v, array_v = exe.run( + binary, + feed=feed_dict, + fetch_list=[loss, array], + return_merged=False, + ) self.assertEqual(np.array(loss_v).shape, (device_num, 1)) self.assertEqual( - np.array(array_v[0][0]).shape, (batch_size / device_num, 784)) + np.array(array_v[0][0]).shape, (batch_size / device_num, 784) + ) self.assertEqual( - np.array(array_v[0][1]).shape, (batch_size / device_num, 1)) - self.assertEqual(np.array(array_v[0][2]).shape, (1, )) + np.array(array_v[0][1]).shape, (batch_size / device_num, 1) + ) + self.assertEqual(np.array(array_v[0][2]).shape, (1,)) for _ in range(3): - loss_v, array_v = exe.run(binary, - feed=feed_dict, - fetch_list=[loss, array], - return_merged=True) - self.assertEqual(np.array(loss_v).shape, (device_num, )) + loss_v, array_v = exe.run( + binary, + feed=feed_dict, + fetch_list=[loss, array], + return_merged=True, + ) + self.assertEqual(np.array(loss_v).shape, (device_num,)) self.assertEqual(np.array(array_v[0]).shape, (batch_size, 784)) self.assertEqual(np.array(array_v[1]).shape, (batch_size, 1)) np.testing.assert_allclose(loss_v, array_v[2], rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_fetch_unmerged.py b/python/paddle/fluid/tests/unittests/test_fetch_unmerged.py index e9ce5294c37e7..0da628db92c11 100644 --- a/python/paddle/fluid/tests/unittests/test_fetch_unmerged.py +++ b/python/paddle/fluid/tests/unittests/test_fetch_unmerged.py @@ -22,23 +22,26 @@ class TestFetchUnmerged(unittest.TestCase): - def conv_net(self, img, label): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=8, - pool_size=2, - pool_stride=2, - pool_type='max', - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=8, + pool_size=2, + pool_stride=2, + pool_type='max', + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=16, - pool_size=2, - pool_stride=2, - pool_type='avg', - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=16, + pool_size=2, + pool_stride=2, + pool_type='avg', + act="relu", + ) hidden = fluid.layers.fc(input=conv_pool_2, size=32, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) @@ -48,12 +51,12 @@ def conv_net(self, img, label): def build_program(self, main, startup, is_test): with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - img = fluid.layers.data(name='image', - shape=[1, 28, 28], - dtype='float32') - label = fluid.layers.data(name='label', - shape=[1], - dtype='int64') + img = fluid.layers.data( + name='image', shape=[1, 28, 28], dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=[1], dtype='int64' + ) loss, prediction = self.conv_net(img, label) if not is_test: opt = fluid.optimizer.Adam(learning_rate=0.001) @@ -63,8 +66,9 @@ def build_program(self, main, startup, is_test): def fetch_unmerged(self, use_cuda=True): main_program = fluid.Program() startup_program = fluid.Program() - feeds, loss, prediction = self.build_program(main_program, - startup_program, False) + feeds, loss, prediction = self.build_program( + main_program, startup_program, False + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) @@ -72,34 +76,41 @@ def fetch_unmerged(self, use_cuda=True): build_strategy = fluid.BuildStrategy() binary = fluid.CompiledProgram(main_program).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) iters = 2 batch_size = 16 - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=500), - batch_size=batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=500), + batch_size=batch_size, + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) device_num = fluid.core.get_cuda_device_count() if use_cuda else 2 for _ in range(iters): data = next(train_reader()) - loss_v, prediction_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss, prediction], - return_merged=False) + loss_v, prediction_v = exe.run( + binary, + feed=feeder.feed(data), + fetch_list=[loss, prediction], + return_merged=False, + ) self.assertEqual(np.array(loss_v).shape, (device_num, 1)) self.assertEqual( np.array(prediction_v).shape, - (device_num, batch_size / device_num, 10)) + (device_num, batch_size / device_num, 10), + ) for _ in range(iters): data = next(train_reader()) - loss_v, prediction_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss, prediction], - return_merged=True) - self.assertEqual(np.array(loss_v).shape, (device_num, )) + loss_v, prediction_v = exe.run( + binary, + feed=feeder.feed(data), + fetch_list=[loss, prediction], + return_merged=True, + ) + self.assertEqual(np.array(loss_v).shape, (device_num,)) self.assertEqual(np.array(prediction_v).shape, (batch_size, 10)) def test_fetch_unmerged(self): diff --git a/python/paddle/fluid/tests/unittests/test_fetch_var.py b/python/paddle/fluid/tests/unittests/test_fetch_var.py index 1ce597ef98ad1..98b06b31368d9 100644 --- a/python/paddle/fluid/tests/unittests/test_fetch_var.py +++ b/python/paddle/fluid/tests/unittests/test_fetch_var.py @@ -19,7 +19,6 @@ class TestFetchVar(unittest.TestCase): - def set_input(self): self.val = np.array([1, 3, 5]).astype(np.int32) @@ -35,7 +34,6 @@ def test_fetch_var(self): class TestFetchNullVar(TestFetchVar): - def set_input(self): self.val = np.array([]).astype(np.int32) diff --git a/python/paddle/fluid/tests/unittests/test_fill_any_like_op.py b/python/paddle/fluid/tests/unittests/test_fill_any_like_op.py index fae7b3c2b0a0a..1739361080d3b 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_any_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_any_like_op.py @@ -20,7 +20,6 @@ class TestFillAnyLikeOp(OpTest): - def setUp(self): self.op_type = "fill_any_like" self.dtype = np.int32 @@ -38,16 +37,15 @@ def test_check_output(self): class TestFillAnyLikeOpFloat32(TestFillAnyLikeOp): - def init(self): self.dtype = np.float32 self.value = 0.0 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFillAnyLikeOpBfloat16(OpTest): - def setUp(self): self.op_type = "fill_any_like" self.dtype = np.uint16 @@ -55,8 +53,9 @@ def setUp(self): self.inputs = {'X': np.random.random((219, 232)).astype(np.float32)} self.attrs = {'value': self.value, 'dtype': core.VarDesc.VarType.BF16} self.outputs = { - 'Out': - convert_float_to_uint16(self.value * np.ones_like(self.inputs["X"])) + 'Out': convert_float_to_uint16( + self.value * np.ones_like(self.inputs["X"]) + ) } def test_check_output(self): @@ -65,25 +64,21 @@ def test_check_output(self): class TestFillAnyLikeOpValue1(TestFillAnyLikeOp): - def init(self): self.value = 1.0 class TestFillAnyLikeOpValue2(TestFillAnyLikeOp): - def init(self): self.value = 1e-10 class TestFillAnyLikeOpValue3(TestFillAnyLikeOp): - def init(self): self.value = 1e-100 class TestFillAnyLikeOpType(TestFillAnyLikeOp): - def setUp(self): self.op_type = "fill_any_like" self.dtype = np.int32 @@ -92,16 +87,15 @@ def setUp(self): self.inputs = {'X': np.random.random((219, 232)).astype(self.dtype)} self.attrs = { 'value': self.value, - 'dtype': int(core.VarDesc.VarType.FP32) + 'dtype': int(core.VarDesc.VarType.FP32), } self.outputs = { - 'Out': - self.value * np.ones_like(self.inputs["X"]).astype(np.float32) + 'Out': self.value + * np.ones_like(self.inputs["X"]).astype(np.float32) } class TestFillAnyLikeOpFloat16(TestFillAnyLikeOp): - def init(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/test_fill_any_op.py b/python/paddle/fluid/tests/unittests/test_fill_any_op.py index 7dbdfe8d58a6c..76daed5a5d84a 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_any_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_any_op.py @@ -19,7 +19,6 @@ class TestFillAnyOp(OpTest): - def setUp(self): self.op_type = "fill_any" self.dtype = 'float64' @@ -28,11 +27,11 @@ def setUp(self): self.inputs = {'X': np.random.random((20, 30)).astype(self.dtype)} self.attrs = { 'value_float': float(self.value), - 'value_int': int(self.value) + 'value_int': int(self.value), } self.outputs = { - 'Out': - self.value * np.ones_like(self.inputs["X"]).astype(self.dtype) + 'Out': self.value + * np.ones_like(self.inputs["X"]).astype(self.dtype) } def init(self): @@ -46,34 +45,29 @@ def test_check_grad(self): class TestFillAnyOpFloat32(TestFillAnyOp): - def init(self): self.dtype = np.float32 self.value = 0.0 class TestFillAnyOpFloat16(TestFillAnyOp): - def init(self): self.dtype = np.float16 class TestFillAnyOpvalue1(TestFillAnyOp): - def init(self): self.dtype = np.float32 self.value = 111111555 class TestFillAnyOpvalue2(TestFillAnyOp): - def init(self): self.dtype = np.float32 self.value = 11111.1111 class TestFillAnyInplace(unittest.TestCase): - def test_fill_any_version(self): with paddle.fluid.dygraph.guard(): var = paddle.to_tensor(np.ones((4, 2, 3)).astype(np.float32)) @@ -91,7 +85,8 @@ def test_fill_any_version(self): def test_fill_any_eqaul(self): with paddle.fluid.dygraph.guard(): tensor = paddle.to_tensor( - np.random.random((20, 30)).astype(np.float32)) + np.random.random((20, 30)).astype(np.float32) + ) target = tensor.numpy() target[...] = 1 @@ -100,7 +95,7 @@ def test_fill_any_eqaul(self): def test_backward(self): with paddle.fluid.dygraph.guard(): - x = paddle.full([10, 10], -1., dtype='float32') + x = paddle.full([10, 10], -1.0, dtype='float32') x.stop_gradient = False y = 2 * x y.fill_(1) diff --git a/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like.py b/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like.py index ebe0ed179d277..339fafbd0b4e2 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like.py +++ b/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like.py @@ -21,16 +21,18 @@ paddle.enable_static() -def fill_constant_batch_size_like(input, - shape, - value, - data_type, - input_dim_idx=0, - output_dim_idx=0, - force_cpu=False): +def fill_constant_batch_size_like( + input, + shape, + value, + data_type, + input_dim_idx=0, + output_dim_idx=0, + force_cpu=False, +): return paddle.fluid.layers.fill_constant_batch_size_like( - input, shape, data_type, value, input_dim_idx, output_dim_idx, - force_cpu) + input, shape, data_type, value, input_dim_idx, output_dim_idx, force_cpu + ) class TestFillConstatnBatchSizeLike1(OpTest): @@ -51,7 +53,7 @@ def setUp(self): 'value': self.value, 'input_dim_idx': self.input_dim_idx, 'output_dim_idx': self.output_dim_idx, - 'force_cpu': self.force_cpu + 'force_cpu': self.force_cpu, } def init_data(self): diff --git a/python/paddle/fluid/tests/unittests/test_fill_constant_op.py b/python/paddle/fluid/tests/unittests/test_fill_constant_op.py index a031385aa0a09..838efff490ab8 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_constant_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_constant_op.py @@ -26,10 +26,8 @@ # Situation 1: Attr(shape) is a list(without tensor) class TestFillConstantOp1(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.inputs = {} @@ -41,10 +39,8 @@ def test_check_output(self): class TestFillConstantOp2(OpTest): - def setUp(self): - '''Test fill_constant op with default value - ''' + '''Test fill_constant op with default value''' self.op_type = "fill_constant" self.inputs = {} @@ -56,10 +52,8 @@ def test_check_output(self): class TestFillConstantOp3(OpTest): - def setUp(self): - '''Test fill_constant op with specified int64 value - ''' + '''Test fill_constant op with specified int64 value''' self.op_type = "fill_constant" self.inputs = {} @@ -71,10 +65,8 @@ def test_check_output(self): class TestFillConstantOp4(OpTest): - def setUp(self): - '''Test fill_constant op with specified int value - ''' + '''Test fill_constant op with specified int value''' self.op_type = "fill_constant" self.inputs = {} @@ -85,20 +77,19 @@ def test_check_output(self): self.check_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFillConstantBF16Op(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.dtype = np.uint16 self.inputs = {} self.attrs = { 'shape': [123, 92], 'value': 3.8, - 'dtype': core.VarDesc.VarType.BF16 + 'dtype': core.VarDesc.VarType.BF16, } self.outputs = {'Out': convert_float_to_uint16(np.full((123, 92), 3.8))} @@ -108,17 +99,15 @@ def test_check_output(self): class TestFillConstantOpWithSelectedRows(unittest.TestCase): - def check_with_place(self, place): scope = core.Scope() # create Out Variable out = scope.var('Out').get_selected_rows() # create and run fill_constant_op operator - fill_constant_op = Operator("fill_constant", - shape=[123, 92], - value=3.8, - Out='Out') + fill_constant_op = Operator( + "fill_constant", shape=[123, 92], value=3.8, Out='Out' + ) fill_constant_op.run(scope, place) # get result from Out @@ -138,16 +127,15 @@ def test_fill_constant_with_selected_rows(self): # Situation 2: Attr(shape) is a list(with tensor) class TestFillConstantOp1_ShapeTensorList(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} self.attrs = {'shape': self.infer_shape, 'value': self.value} @@ -163,16 +151,15 @@ def test_check_output(self): class TestFillConstantOp2_ShapeTensorList(OpTest): - def setUp(self): - '''Test fill_constant op with default value - ''' + '''Test fill_constant op with default value''' self.op_type = "fill_constant" self.init_data() shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} self.attrs = {'shape': self.infer_shape} @@ -187,7 +174,6 @@ def test_check_output(self): class TestFillConstantOp3_ShapeTensorList(TestFillConstantOp1_ShapeTensorList): - def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -195,7 +181,6 @@ def init_data(self): class TestFillConstantOp4_ShapeTensorList(TestFillConstantOp1_ShapeTensorList): - def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -204,10 +189,8 @@ def init_data(self): # Situation 3: shape is a tensor class TestFillConstantOp1_ShapeTensor(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() @@ -225,16 +208,14 @@ def test_check_output(self): # Situation 4: value is a tensor class TestFillConstantOp1_ValueTensor(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() self.inputs = { "ShapeTensor": np.array(self.shape).astype("int32"), - 'ValueTensor': np.array([self.value]).astype("float32") + 'ValueTensor': np.array([self.value]).astype("float32"), } self.attrs = {'value': self.value + 1.0} self.outputs = {'Out': np.full(self.shape, self.value)} @@ -250,16 +231,14 @@ def test_check_output(self): # Situation 5: value is a tensor class TestFillConstantOp2_ValueTensor(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() self.inputs = { "ShapeTensor": np.array(self.shape).astype("int32"), - 'ValueTensor': np.array([self.value]).astype("int32") + 'ValueTensor': np.array([self.value]).astype("int32"), } self.attrs = {'value': self.value, 'dtype': 2} self.outputs = {'Out': np.full(self.shape, self.value)} @@ -275,56 +254,55 @@ def test_check_output(self): # Test python API class TestFillConstantAPI(unittest.TestCase): - def test_api(self): positive_2_int32 = fluid.layers.fill_constant([1], "int32", 2) positive_2_int64 = fluid.layers.fill_constant([1], "int64", 2) - shape_tensor_int32 = fluid.data(name="shape_tensor_int32", - shape=[2], - dtype="int32") - shape_tensor_int64 = fluid.data(name="shape_tensor_int64", - shape=[2], - dtype="int64") - - out_1 = fluid.layers.fill_constant(shape=[1, 2], - dtype="float32", - value=1.1) - - out_2 = fluid.layers.fill_constant(shape=[1, positive_2_int32], - dtype="float32", - value=1.1) - - out_3 = fluid.layers.fill_constant(shape=[1, positive_2_int64], - dtype="float32", - value=1.1) - - out_4 = fluid.layers.fill_constant(shape=shape_tensor_int32, - dtype="float32", - value=1.1) - - out_5 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype="float32", - value=1.1) - - out_6 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype=np.float32, - value=1.1) - - val1 = fluid.layers.fill_constant(shape=[1], - dtype=np.float32, - value=1.1) - val2 = fluid.layers.fill_constant(shape=[1], - dtype=np.float64, - value=1.1) - out_7 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype=np.float32, - value=val1) - - out_8 = fluid.layers.fill_constant(shape=shape_tensor_int64, - dtype=np.float32, - value=val2) + shape_tensor_int32 = fluid.data( + name="shape_tensor_int32", shape=[2], dtype="int32" + ) + shape_tensor_int64 = fluid.data( + name="shape_tensor_int64", shape=[2], dtype="int64" + ) + + out_1 = fluid.layers.fill_constant( + shape=[1, 2], dtype="float32", value=1.1 + ) + + out_2 = fluid.layers.fill_constant( + shape=[1, positive_2_int32], dtype="float32", value=1.1 + ) + + out_3 = fluid.layers.fill_constant( + shape=[1, positive_2_int64], dtype="float32", value=1.1 + ) + + out_4 = fluid.layers.fill_constant( + shape=shape_tensor_int32, dtype="float32", value=1.1 + ) + + out_5 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype="float32", value=1.1 + ) + + out_6 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype=np.float32, value=1.1 + ) + + val1 = fluid.layers.fill_constant( + shape=[1], dtype=np.float32, value=1.1 + ) + val2 = fluid.layers.fill_constant( + shape=[1], dtype=np.float64, value=1.1 + ) + out_7 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype=np.float32, value=val1 + ) + + out_8 = fluid.layers.fill_constant( + shape=shape_tensor_int64, dtype=np.float32, value=val2 + ) exe = fluid.Executor(place=fluid.CPUPlace()) res_1, res_2, res_3, res_4, res_5, res_6, res_7, res_8 = exe.run( @@ -333,7 +311,8 @@ def test_api(self): "shape_tensor_int32": np.array([1, 2]).astype("int32"), "shape_tensor_int64": np.array([1, 2]).astype("int64"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8], + ) assert np.array_equal(res_1, np.full([1, 2], 1.1, dtype="float32")) assert np.array_equal(res_2, np.full([1, 2], 1.1, dtype="float32")) @@ -346,7 +325,6 @@ def test_api(self): class TestFillConstantImperative(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): data1 = np.array([1, 2]).astype('int32') @@ -355,26 +333,30 @@ def test_api(self): shape = fluid.dygraph.to_variable(data1) val = fluid.dygraph.to_variable(data2) value = fluid.dygraph.to_variable(data3) - res1 = fluid.layers.fill_constant(shape=[1, 2], - dtype='float32', - value=1.1) - res2 = fluid.layers.fill_constant(shape=shape, - dtype='float32', - value=1.1) - res3 = fluid.layers.fill_constant(shape=shape, - dtype='float32', - value=val) - res4 = fluid.layers.fill_constant(shape=shape, - dtype='int32', - value=value) - assert np.array_equal(res1.numpy(), - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(res2.numpy(), - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(res3.numpy(), - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(res4.numpy(), - np.full([1, 2], 88, dtype="int32")) + res1 = fluid.layers.fill_constant( + shape=[1, 2], dtype='float32', value=1.1 + ) + res2 = fluid.layers.fill_constant( + shape=shape, dtype='float32', value=1.1 + ) + res3 = fluid.layers.fill_constant( + shape=shape, dtype='float32', value=val + ) + res4 = fluid.layers.fill_constant( + shape=shape, dtype='int32', value=value + ) + assert np.array_equal( + res1.numpy(), np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + res2.numpy(), np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + res3.numpy(), np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + res4.numpy(), np.full([1, 2], 88, dtype="int32") + ) def test_nan(self): with fluid.dygraph.guard(): @@ -394,42 +376,49 @@ def test_ninf(self): class TestFillConstantOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - #for ci coverage + # for ci coverage x1 = fluid.layers.data(name='x1', shape=[1], dtype="int16") - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[1], - value=5, - dtype='uint4') - - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[1.1], - value=5, - dtype='float32', - out=x1) + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[1], + value=5, + dtype='uint4', + ) + + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[1.1], + value=5, + dtype='float32', + out=x1, + ) # The argument dtype of fill_constant_op must be one of bool, float16, - #float32, float64, uint8, int16, int32 or int64 + # float32, float64, uint8, int16, int32 or int64 x2 = fluid.layers.data(name='x2', shape=[1], dtype="int32") - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[1], - value=5, - dtype='float64', - out=x2) + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[1], + value=5, + dtype='float64', + out=x2, + ) x3 = np.random.randn(100, 100).astype('int32') - self.assertRaises(TypeError, - fluid.layers.fill_constant, - shape=[100, 100], - value=5, - dtype='float64', - out=x3) + self.assertRaises( + TypeError, + fluid.layers.fill_constant, + shape=[100, 100], + value=5, + dtype='float64', + out=x3, + ) # The argument shape's type of fill_constant_op must be list, tuple or Variable. def test_shape_type(): @@ -439,39 +428,37 @@ def test_shape_type(): # The shape dtype of fill_constant_op must be int32 or int64. def test_shape_tensor_dtype(): - shape = fluid.data(name="shape_tensor", - shape=[2], - dtype="float32") - fluid.layers.fill_constant(shape=shape, - dtype="float32", - value=1) + shape = fluid.data( + name="shape_tensor", shape=[2], dtype="float32" + ) + fluid.layers.fill_constant( + shape=shape, dtype="float32", value=1 + ) self.assertRaises(TypeError, test_shape_tensor_dtype) def test_shape_tensor_list_dtype(): - shape = fluid.data(name="shape_tensor_list", - shape=[1], - dtype="bool") - fluid.layers.fill_constant(shape=[shape, 2], - dtype="float32", - value=1) + shape = fluid.data( + name="shape_tensor_list", shape=[1], dtype="bool" + ) + fluid.layers.fill_constant( + shape=[shape, 2], dtype="float32", value=1 + ) self.assertRaises(TypeError, test_shape_tensor_list_dtype) class TestFillConstantOp_ValueTensorBf16(OpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.op_type = "fill_constant" self.init_data() self.inputs = { - "ShapeTensor": - np.array(self.shape).astype("int32"), - 'ValueTensor': - convert_float_to_uint16(np.array([self.value]).astype("float32")) + "ShapeTensor": np.array(self.shape).astype("int32"), + 'ValueTensor': convert_float_to_uint16( + np.array([self.value]).astype("float32") + ), } self.attrs = {'value': self.value, 'dtype': core.VarDesc.VarType.BF16} self.outputs = {'Out': np.full(self.shape, self.value)} diff --git a/python/paddle/fluid/tests/unittests/test_fill_diagonal_tensor_op.py b/python/paddle/fluid/tests/unittests/test_fill_diagonal_tensor_op.py index 9823f5e258519..f412a161eb63c 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_diagonal_tensor_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_diagonal_tensor_op.py @@ -34,13 +34,15 @@ def fill_diagonal_ndarray(x, value, offset=0, dim1=0, dim2=1): diagonal = np.lib.stride_tricks.as_strided( x[:, offset:] if dim_sum == 1 else x[:, :, offset:], shape=(shape[dim3], diagdim), - strides=(strides[dim3], strides[dim1] + strides[dim2])) + strides=(strides[dim3], strides[dim1] + strides[dim2]), + ) else: diagdim = min(shape[dim2], shape[dim1] + offset) diagonal = np.lib.stride_tricks.as_strided( x[-offset:, :] if dim_sum in [1, 2] else x[:, -offset:], shape=(shape[dim3], diagdim), - strides=(strides[dim3], strides[dim1] + strides[dim2])) + strides=(strides[dim3], strides[dim1] + strides[dim2]), + ) diagonal[...] = value return x @@ -79,13 +81,12 @@ def fill_gt(x, y, offset, dim1, dim2): class TensorFillDiagTensor_Test(OpTest): - def setUp(self): self.op_type = "fill_diagonal_tensor" self.python_api = paddle.tensor.manipulation.fill_diagonal_tensor self.init_kernel_type() x = np.random.random((10, 10)).astype(self.dtype) - y = np.random.random((10, )).astype(self.dtype) + y = np.random.random((10,)).astype(self.dtype) dim1 = 0 dim2 = 1 offset = 0 @@ -106,7 +107,6 @@ def test_check_grad(self): class TensorFillDiagTensor_Test2(TensorFillDiagTensor_Test): - def setUp(self): self.op_type = "fill_diagonal_tensor" self.python_api = paddle.tensor.manipulation.fill_diagonal_tensor @@ -127,7 +127,6 @@ def init_kernel_type(self): class TensorFillDiagTensor_Test3(TensorFillDiagTensor_Test): - def setUp(self): self.op_type = "fill_diagonal_tensor" self.python_api = paddle.tensor.manipulation.fill_diagonal_tensor diff --git a/python/paddle/fluid/tests/unittests/test_fill_op.py b/python/paddle/fluid/tests/unittests/test_fill_op.py index 0cfa7ecbed024..95a432136caca 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_op.py @@ -20,7 +20,6 @@ class TestFillOp1(OpTest): - def setUp(self): self.op_type = "fill" val = np.random.random(size=[100, 200]) @@ -29,7 +28,7 @@ def setUp(self): 'value': val.flatten().tolist(), 'shape': [100, 200], 'dtype': int(core.VarDesc.VarType.FP64), - 'force_cpu': False + 'force_cpu': False, } self.outputs = {'Out': val.astype('float64')} @@ -38,7 +37,6 @@ def test_check_output(self): class TestFillOp2(OpTest): - def setUp(self): self.op_type = "fill" val = np.random.random(size=[100, 200]) @@ -47,7 +45,7 @@ def setUp(self): 'value': val.flatten().tolist(), 'shape': [100, 200], 'dtype': int(core.VarDesc.VarType.FP64), - 'force_cpu': True + 'force_cpu': True, } self.outputs = {'Out': val.astype('float64')} @@ -56,7 +54,6 @@ def test_check_output(self): class TestFillOp3(unittest.TestCase): - def check_with_place(self, place, f_cpu): scope = core.Scope() # create Out Variable @@ -64,12 +61,14 @@ def check_with_place(self, place, f_cpu): # create and run fill_op operator val = np.random.random(size=[300, 200]) - fill_op = Operator("fill", - value=val.flatten(), - shape=[300, 200], - dtype=int(core.VarDesc.VarType.FP32), - force_cpu=f_cpu, - Out='Out') + fill_op = Operator( + "fill", + value=val.flatten(), + shape=[300, 200], + dtype=int(core.VarDesc.VarType.FP32), + force_cpu=f_cpu, + Out='Out', + ) fill_op.run(scope, place) # get result from Out diff --git a/python/paddle/fluid/tests/unittests/test_fill_zeros_like2_op.py b/python/paddle/fluid/tests/unittests/test_fill_zeros_like2_op.py index a298bc5c58bc3..f1a1039ea016c 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_zeros_like2_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_zeros_like2_op.py @@ -20,7 +20,6 @@ class TestFillZerosLike2Op(OpTest): - def setUp(self): self.op_type = "fill_zeros_like2" self.dtype = np.float32 @@ -37,21 +36,17 @@ def test_check_output(self): class TestFillZerosLike2OpFp16(TestFillZerosLike2Op): - def init_dtype(self): self.dtype = np.float16 class TestFillZerosLike2OpFp64(TestFillZerosLike2Op): - def init_dtype(self): self.dtype = np.float64 class TestZerosError(unittest.TestCase): - def test_errors(self): - def test_zeros_like_type_error(): with fluid.program_guard(fluid.Program(), fluid.Program()): fluid.layers.zeros_like([10], dtype="float") diff --git a/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py b/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py index e6515e426e40c..7317973e774ab 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py @@ -18,7 +18,6 @@ class TestFillZerosLikeOp(OpTest): - def setUp(self): self.op_type = "fill_zeros_like" self.dtype = np.float32 @@ -34,7 +33,6 @@ def test_check_output(self): class TestFillZerosLikeOpFp16(TestFillZerosLikeOp): - def init_dtype(self): self.dtype = np.float16 diff --git a/python/paddle/fluid/tests/unittests/test_filter_by_instag_op.py b/python/paddle/fluid/tests/unittests/test_filter_by_instag_op.py index b60dcbe9c899b..e1136d677a13d 100644 --- a/python/paddle/fluid/tests/unittests/test_filter_by_instag_op.py +++ b/python/paddle/fluid/tests/unittests/test_filter_by_instag_op.py @@ -16,11 +16,11 @@ import unittest import numpy as np from op_test import OpTest + """This is Test Case 1""" class TestFilterByInstagOp(OpTest): - def setUp(self): self.op_type = 'filter_by_instag' x1 = np.zeros((36, 4), dtype=np.float64) @@ -48,8 +48,9 @@ def setUp(self): out[ln, k] = cur ln += 1 - mmap = np.array([[0, 1, 2], [2, 6, 4], [6, 15, 6], [12, 28, - 8]]).astype('int64') + mmap = np.array([[0, 1, 2], [2, 6, 4], [6, 15, 6], [12, 28, 8]]).astype( + 'int64' + ) mmap_lod = [[1, 1, 1, 1]] loss_weight = np.array([[1], [1], [1], [1]]).astype('double') @@ -62,7 +63,7 @@ def setUp(self): self.outputs = { 'Out': (out, out_lod), 'LossWeight': (loss_weight, mmap_lod), - 'IndexMap': (mmap, mmap_lod) + 'IndexMap': (mmap, mmap_lod), } self.attrs = {'is_lod': True, 'out_val_if_empty': 0} @@ -71,16 +72,15 @@ def test_check_output(self): self.check_output() def test_check_grad(self): - self.check_grad(['Ins'], - 'Out', - no_grad_set=set(['Ins_tag', 'Filter_tag'])) + self.check_grad( + ['Ins'], 'Out', no_grad_set=set(['Ins_tag', 'Filter_tag']) + ) """This is Test Case 2""" class TestFilterByInstagOp2(OpTest): - def setUp(self): self.op_type = 'filter_by_instag' @@ -110,7 +110,7 @@ def setUp(self): self.outputs = { 'Out': (out, out_lod), 'LossWeight': (loss_weight, mmap_lod), - 'IndexMap': (mmap, mmap_lod) + 'IndexMap': (mmap, mmap_lod), } self.attrs = {'is_lod': True, 'out_val_if_empty': 0} @@ -118,16 +118,15 @@ def test_check_output(self): self.check_output() def test_check_grad(self): - self.check_grad(['Ins'], - 'Out', - no_grad_set=set(['Ins_tag', 'Filter_tag'])) + self.check_grad( + ['Ins'], 'Out', no_grad_set=set(['Ins_tag', 'Filter_tag']) + ) """This is Test Case 3""" class TestFilterByInstagOp3(OpTest): - def setUp(self): self.op_type = 'filter_by_instag' @@ -154,7 +153,7 @@ def setUp(self): self.outputs = { 'Out': (out, out_lod), 'LossWeight': (loss_weight, mmap_lod), - 'IndexMap': (mmap, mmap_lod) + 'IndexMap': (mmap, mmap_lod), } self.attrs = {'is_lod': True, 'out_val_if_empty': 0} @@ -162,16 +161,15 @@ def test_check_output(self): self.check_output() def test_check_grad(self): - self.check_grad(['Ins'], - 'Out', - no_grad_set=set(['Ins_tag', 'Filter_tag'])) + self.check_grad( + ['Ins'], 'Out', no_grad_set=set(['Ins_tag', 'Filter_tag']) + ) """This is Test Case 4""" class TestFilterByInstagOp4(OpTest): - def setUp(self): self.op_type = 'filter_by_instag' @@ -197,7 +195,7 @@ def setUp(self): self.outputs = { 'Out': (out, out_lod), 'LossWeight': (loss_weight, mmap_lod), - 'IndexMap': (mmap, mmap_lod) + 'IndexMap': (mmap, mmap_lod), } self.attrs = {'is_lod': False, 'out_val_if_empty': 0} @@ -205,13 +203,12 @@ def test_check_output(self): self.check_output() def test_check_grad(self): - self.check_grad(['Ins'], - 'Out', - no_grad_set=set(['Ins_tag', 'Filter_tag'])) + self.check_grad( + ['Ins'], 'Out', no_grad_set=set(['Ins_tag', 'Filter_tag']) + ) class TestFilterByInstagOp6(OpTest): - def setUp(self): self.op_type = 'filter_by_instag' @@ -237,7 +234,7 @@ def setUp(self): self.outputs = { 'Out': (out, out_lod), 'LossWeight': (loss_weight, mmap_lod), - 'IndexMap': (mmap, mmap_lod) + 'IndexMap': (mmap, mmap_lod), } self.attrs = {'is_lod': False, 'out_val_if_empty': 0} @@ -249,7 +246,6 @@ def test_check_grad(self): class TestFilterByInstagOp7(OpTest): - def setUp(self): self.op_type = 'filter_by_instag' @@ -275,7 +271,7 @@ def setUp(self): self.outputs = { 'Out': (out, out_lod), 'LossWeight': (loss_weight, mmap_lod), - 'IndexMap': (mmap, mmap_lod) + 'IndexMap': (mmap, mmap_lod), } self.attrs = {'is_lod': False, 'out_val_if_empty': 0} diff --git a/python/paddle/fluid/tests/unittests/test_flatten2_op.py b/python/paddle/fluid/tests/unittests/test_flatten2_op.py index cbfceb720ad1c..fe5aad118af46 100644 --- a/python/paddle/fluid/tests/unittests/test_flatten2_op.py +++ b/python/paddle/fluid/tests/unittests/test_flatten2_op.py @@ -20,7 +20,6 @@ class TestFlattenOp(OpTest): - def setUp(self): self.op_type = "flatten2" self.init_test_case() @@ -28,7 +27,7 @@ def setUp(self): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype("float32") + "XShape": np.random.random(self.in_shape).astype("float32"), } def test_check_output(self): @@ -47,7 +46,6 @@ def init_attrs(self): class TestFlattenOp1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.axis = 0 @@ -55,7 +53,6 @@ def init_test_case(self): class TestFlattenOpWithDefaultAxis(TestFlattenOp): - def init_test_case(self): self.in_shape = (10, 2, 2, 3) self.new_shape = (10, 12) @@ -65,7 +62,6 @@ def init_attrs(self): class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 @@ -73,7 +69,6 @@ def init_test_case(self): class TestStaticFlattenInferShapePythonAPI(unittest.TestCase): - def execute_api(self, x, axis=1): return fluid.layers.flatten(x, axis=axis) @@ -81,15 +76,14 @@ def test_static_api(self): paddle.enable_static() main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, paddle.static.Program()): - x = paddle.static.data(name="x", - shape=[-1, 3, -1, -1], - dtype='float32') + x = paddle.static.data( + name="x", shape=[-1, 3, -1, -1], dtype='float32' + ) out = self.execute_api(x, axis=2) self.assertTrue((-1, -1) == out.shape) class TestFlatten2OpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): input_data = np.random.random((3, 2, 4, 5)).astype("float64") @@ -102,9 +96,9 @@ def test_Variable(): def test_type(): # dtype must be float32, float64, int8, int32, int64, uint8. - x2 = fluid.layers.data(name='x2', - shape=[3, 2, 4, 5], - dtype='float16') + x2 = fluid.layers.data( + name='x2', shape=[3, 2, 4, 5], dtype='float16' + ) fluid.layers.flatten(x2, axis=1) self.assertRaises(TypeError, test_type) diff --git a/python/paddle/fluid/tests/unittests/test_flatten_contiguous_range_op.py b/python/paddle/fluid/tests/unittests/test_flatten_contiguous_range_op.py index b938adfbf1426..8d1bcc6254d01 100644 --- a/python/paddle/fluid/tests/unittests/test_flatten_contiguous_range_op.py +++ b/python/paddle/fluid/tests/unittests/test_flatten_contiguous_range_op.py @@ -19,7 +19,6 @@ class TestFlattenOp(OpTest): - def setUp(self): self.python_api = paddle.flatten self.python_out_sig = ["Out"] @@ -31,7 +30,7 @@ def setUp(self): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype("float32") + "XShape": np.random.random(self.in_shape).astype("float32"), } def test_check_output(self): @@ -44,17 +43,16 @@ def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 self.stop_axis = -1 - self.new_shape = (120) + self.new_shape = 120 def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 1 @@ -64,12 +62,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_2(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -79,12 +76,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_3(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -94,12 +90,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_4(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = -2 @@ -109,12 +104,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_5(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 2 @@ -124,12 +118,11 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.start_axis = 3 @@ -139,50 +132,63 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlatten2OpError(unittest.TestCase): - def test_errors(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_ValueError1(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) out = paddle.flatten(x_var, start_axis=2, stop_axis=1) self.assertRaises(ValueError, test_ValueError1) def test_ValueError2(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=10, stop_axis=1) self.assertRaises(ValueError, test_ValueError2) def test_ValueError3(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=2, stop_axis=10) self.assertRaises(ValueError, test_ValueError3) def test_type(): # dtype must be float32, float64, int8, int32, int64, uint8. - x2 = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x2 = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x2 = x2.astype('float16') - x2_var = paddle.fluid.data(name='x2', - shape=[3, 2, 4, 5], - dtype='float16') + x2_var = paddle.fluid.data( + name='x2', shape=[3, 2, 4, 5], dtype='float16' + ) paddle.flatten(x2_var) self.assertRaises(TypeError, test_type) @@ -194,7 +200,6 @@ def test_InputError(): class TestStaticFlattenPythonAPI(unittest.TestCase): - def execute_api(self, x, start_axis=0, stop_axis=-1): return paddle.flatten(x, start_axis, stop_axis) @@ -204,9 +209,9 @@ def test_static_api(self): main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, paddle.static.Program()): - x = paddle.static.data(name="x", - shape=[2, 3, 4, 4], - dtype='float32') + x = paddle.static.data( + name="x", shape=[2, 3, 4, 4], dtype='float32' + ) out = self.execute_api(x, start_axis=-2, stop_axis=-1) exe = paddle.static.Executor(place=paddle.CPUPlace()) @@ -215,7 +220,6 @@ def test_static_api(self): class TestStaticFlattenInferShapePythonAPI(unittest.TestCase): - def execute_api(self, x, start_axis=0, stop_axis=-1): return paddle.flatten(x, start_axis, stop_axis) @@ -223,25 +227,30 @@ def test_static_api(self): paddle.enable_static() main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, paddle.static.Program()): - x = paddle.static.data(name="x", - shape=[-1, 3, -1, -1], - dtype='float32') + x = paddle.static.data( + name="x", shape=[-1, 3, -1, -1], dtype='float32' + ) out = self.execute_api(x, start_axis=2, stop_axis=3) self.assertTrue((-1, 3, -1) == out.shape) class TestStaticInplaceFlattenPythonAPI(TestStaticFlattenPythonAPI): - def execute_api(self, x, start_axis=0, stop_axis=-1): return x.flatten_(start_axis, stop_axis) class TestFlattenPython(unittest.TestCase): - def test_python_api(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_InputError(): @@ -260,11 +269,17 @@ def test_Negative(): class TestDygraphInplaceFlattenPython(unittest.TestCase): - def test_python_api(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_Negative(): diff --git a/python/paddle/fluid/tests/unittests/test_flatten_op.py b/python/paddle/fluid/tests/unittests/test_flatten_op.py index 9a371465f33e8..a1108d3392ca5 100644 --- a/python/paddle/fluid/tests/unittests/test_flatten_op.py +++ b/python/paddle/fluid/tests/unittests/test_flatten_op.py @@ -19,7 +19,6 @@ class TestFlattenOp(OpTest): - def setUp(self): self.op_type = "flatten" self.init_test_case() @@ -43,7 +42,6 @@ def init_attrs(self): class TestFlattenOp1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 2, 10) self.axis = 0 @@ -51,7 +49,6 @@ def init_test_case(self): class TestFlattenOpWithDefaultAxis(TestFlattenOp): - def init_test_case(self): self.in_shape = (10, 2, 2, 3) self.new_shape = (10, 12) @@ -61,7 +58,6 @@ def init_attrs(self): class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 diff --git a/python/paddle/fluid/tests/unittests/test_fleet.py b/python/paddle/fluid/tests/unittests/test_fleet.py index a06f264750978..a9a75868ee396 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet.py +++ b/python/paddle/fluid/tests/unittests/test_fleet.py @@ -27,7 +27,8 @@ def setUp(self): """Set up, set envs.""" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_pslib_1(self): """Test cases for pslib.""" @@ -42,24 +43,38 @@ def test_pslib_1(self): os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36002" os.environ["PADDLE_TRAINER_ID"] = "0" role_maker = GeneralRoleMaker() - #role_maker.generate_role() + # role_maker.generate_role() place = fluid.CPUPlace() exe = fluid.Executor(place) - #fleet.init(role_maker) + # fleet.init(role_maker) train_program = fluid.Program() startup_program = fluid.Program() scope = fluid.Scope() with fluid.program_guard(train_program, startup_program): - show = fluid.layers.data(name="show", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) - emb = fluid.layers.embedding(input=show, size=[1, 1], \ - is_sparse=True, is_distributed=True, \ - param_attr=fluid.ParamAttr(name="embedding")) + show = fluid.layers.data( + name="show", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + emb = fluid.layers.embedding( + input=show, + size=[1, 1], + is_sparse=True, + is_distributed=True, + param_attr=fluid.ParamAttr(name="embedding"), + ) bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') bow = fluid.layers.data_norm(input=bow, epsilon=1e-4, name="norm") fc = fluid.layers.fc(input=bow, size=1, act=None) - label = fluid.layers.data(name="click", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) label_cast = fluid.layers.cast(label, dtype='float32') cost = fluid.layers.log_loss(fc, label_cast) try: @@ -70,7 +85,8 @@ def test_pslib_1(self): "embedding": { "sparse_accessor_class": "DownpourSparseValueAccessor" } - }) + }, + ) adam.minimize([cost], [scope]) fleet.run_server() except: diff --git a/python/paddle/fluid/tests/unittests/test_fleet_api_input.py b/python/paddle/fluid/tests/unittests/test_fleet_api_input.py index 56b06e3b8f2e9..ac77fc42efdfb 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_api_input.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_api_input.py @@ -15,19 +15,24 @@ import unittest import paddle import paddle.fluid as fluid -from paddle.fluid.transpiler.distribute_transpiler import DistributeTranspilerConfig +from paddle.fluid.transpiler.distribute_transpiler import ( + DistributeTranspilerConfig, +) from paddle.fluid.incubate.fleet.base.role_maker import UserDefinedRoleMaker -from paddle.fluid.incubate.fleet.base.role_maker import UserDefinedCollectiveRoleMaker +from paddle.fluid.incubate.fleet.base.role_maker import ( + UserDefinedCollectiveRoleMaker, +) from paddle.fluid.incubate.fleet.base.role_maker import Role import paddle.fluid.incubate.fleet.base.role_maker as role_maker -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, +) from paddle.fluid.incubate.fleet.parameter_server import TranspilerOptimizer from paddle.fluid.incubate.fleet.collective import CollectiveOptimizer from dist_simnet_bow import train_network class DistributeTranspilerConfigTest(unittest.TestCase): - def set_runtime_split_send_recv(self, config, value): config.runtime_split_send_recv = value @@ -37,10 +42,12 @@ def set_sync_mode(self, config, value): def testConfig(self): config = DistributeTranspilerConfig() self.assertRaises(Exception, self.set_sync_mode, config, None) - self.assertRaises(Exception, self.set_runtime_split_send_recv, config, - None) - self.assertRaises(Exception, self.set_runtime_split_send_recv, config, - True) + self.assertRaises( + Exception, self.set_runtime_split_send_recv, config, None + ) + self.assertRaises( + Exception, self.set_runtime_split_send_recv, config, True + ) self.set_sync_mode(config, False) self.assertFalse(config.sync_mode) self.set_runtime_split_send_recv(config, True) @@ -48,7 +55,6 @@ def testConfig(self): class FleetTest(unittest.TestCase): - def testInvalidInputs(self): self.assertRaises(Exception, fleet.split_files, "files") self.assertRaises(Exception, fleet.init, "pserver") @@ -61,40 +67,50 @@ def testInvalidInputs(self): place = fluid.CPUPlace() exe = fluid.Executor(place) pe = fluid.ParallelExecutor(use_cuda=False, loss_name=loss.name) - self.assertRaises(Exception, - fleet.save_inference_model, - dirname='/tmp/', - feeded_var_names=['X'], - target_vars=[loss], - executor=pe) - self.assertRaises(Exception, - fleet.save_inference_model, - dirname='/tmp/', - feeded_var_names=['X'], - target_vars=[loss], - executor="executor") + self.assertRaises( + Exception, + fleet.save_inference_model, + dirname='/tmp/', + feeded_var_names=['X'], + target_vars=[loss], + executor=pe, + ) + self.assertRaises( + Exception, + fleet.save_inference_model, + dirname='/tmp/', + feeded_var_names=['X'], + target_vars=[loss], + executor="executor", + ) compiled_prog = fluid.compiler.CompiledProgram( - fluid.default_main_program()) - self.assertRaises(Exception, - fleet.save_inference_model, - dirname='/tmp/', - feeded_var_names=['X'], - target_vars=[loss], - executor=exe, - main_program=compiled_prog) - self.assertRaises(Exception, - fleet.save_persistables, - executor=pe, - dirname='/tmp/') - self.assertRaises(Exception, - fleet.save_persistables, - executor="executor", - dirname='/tmp/') - self.assertRaises(Exception, - fleet.save_persistables, - executor=exe, - dirname='/tmp/', - main_program=compiled_prog) + fluid.default_main_program() + ) + self.assertRaises( + Exception, + fleet.save_inference_model, + dirname='/tmp/', + feeded_var_names=['X'], + target_vars=[loss], + executor=exe, + main_program=compiled_prog, + ) + self.assertRaises( + Exception, fleet.save_persistables, executor=pe, dirname='/tmp/' + ) + self.assertRaises( + Exception, + fleet.save_persistables, + executor="executor", + dirname='/tmp/', + ) + self.assertRaises( + Exception, + fleet.save_persistables, + executor=exe, + dirname='/tmp/', + main_program=compiled_prog, + ) self.assertRaises(Exception, fleet._transpile, "config") def set_program(self, avg_cost, strategy): @@ -108,7 +124,8 @@ def test_init_role(self): current_id=0, role=role_maker.Role.SERVER, worker_num=2, - server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"]) + server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"], + ) # for test optimizer without init(role) # fleet.init(role) batch_size = 128 @@ -127,7 +144,8 @@ def test_transpile(self): current_id=0, role=role_maker.Role.SERVER, worker_num=2, - server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"]) + server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"], + ) # for test optimizer without init(role) fleet.init(role) batch_size = 128 @@ -145,119 +163,120 @@ def test_transpile(self): class TranspilerOptimizerTest(unittest.TestCase): - def testInvalidInputs(self): self.assertRaises(Exception, TranspilerOptimizer, "Adam", None) - self.assertRaises(Exception, TranspilerOptimizer, - fluid.optimizer.Adam(0.001), "strategy") + self.assertRaises( + Exception, + TranspilerOptimizer, + fluid.optimizer.Adam(0.001), + "strategy", + ) transpiler = TranspilerOptimizer(fluid.optimizer.Adam(0.001)) self.assertRaises(Exception, transpiler.minimize, loss=[]) data = fluid.layers.data(name='X', shape=[1], dtype='float32') hidden = fluid.layers.fc(input=data, size=10) loss = paddle.mean(hidden) - self.assertRaises(Exception, - transpiler.minimize, - loss=loss.name, - startup_program=[]) + self.assertRaises( + Exception, transpiler.minimize, loss=loss.name, startup_program=[] + ) class UserDefinedRoleMakerTest(unittest.TestCase): - - def createRoleMaker(self, - current_id=0, - role=Role.WORKER, - worker_num=1, - server_endpoints=["127.0.0.1:8080"]): - role = UserDefinedRoleMaker(current_id, role, worker_num, - server_endpoints) + def createRoleMaker( + self, + current_id=0, + role=Role.WORKER, + worker_num=1, + server_endpoints=["127.0.0.1:8080"], + ): + role = UserDefinedRoleMaker( + current_id, role, worker_num, server_endpoints + ) def testRoleMaker(self): self.createRoleMaker() # test all invalid server_endpoints self.assertRaises( - Exception, self.createRoleMaker, - server_endpoints=None) # server_endpoints must be as list + Exception, self.createRoleMaker, server_endpoints=None + ) # server_endpoints must be as list + self.assertRaises( + Exception, self.createRoleMaker, server_endpoints=[] + ) # server_endpoints can't be empty + self.assertRaises( + Exception, self.createRoleMaker, server_endpoints=[3, []] + ) # element in server_endpoints must be as string self.assertRaises( - Exception, self.createRoleMaker, - server_endpoints=[]) # server_endpoints can't be empty - self.assertRaises(Exception, - self.createRoleMaker, - server_endpoints=[ - 3, [] - ]) # element in server_endpoints must be as string - self.assertRaises(Exception, - self.createRoleMaker, - server_endpoints=[ - "127.0.0.1:8080", "127.0.0.1:8080" - ]) # element in server_endpoints can't be duplicate + Exception, + self.createRoleMaker, + server_endpoints=["127.0.0.1:8080", "127.0.0.1:8080"], + ) # element in server_endpoints can't be duplicate # test all invalid current_id - self.assertRaises(Exception, self.createRoleMaker, - current_id="0") # current_id must be as int self.assertRaises( - Exception, self.createRoleMaker, - current_id=-1) # current_id must be greater than or equal to 0 + Exception, self.createRoleMaker, current_id="0" + ) # current_id must be as int + self.assertRaises( + Exception, self.createRoleMaker, current_id=-1 + ) # current_id must be greater than or equal to 0 self.assertRaises( Exception, self.createRoleMaker, current_id=1, role=Role.SERVER, - server_endpoints=["127.0.0.1:8080"] + server_endpoints=["127.0.0.1:8080"], ) # if role is server, current_id must be less than len(server_endpoints) # test all invalid worker_num - self.assertRaises(Exception, self.createRoleMaker, - worker_num="1") # worker_num must be as int - self.assertRaises(Exception, self.createRoleMaker, - worker_num=0) # worker_num must be greater than 0 + self.assertRaises( + Exception, self.createRoleMaker, worker_num="1" + ) # worker_num must be as int + self.assertRaises( + Exception, self.createRoleMaker, worker_num=0 + ) # worker_num must be greater than 0 # test all invalid role self.assertRaises( - Exception, self.createRoleMaker, - role=3) # role must be as Role(Role.WORKER=1, Role.SERVER=2) + Exception, self.createRoleMaker, role=3 + ) # role must be as Role(Role.WORKER=1, Role.SERVER=2) class UserDefinedCollectiveRoleMakerTest(unittest.TestCase): - - def createRoleMaker(self, - current_id=0, - worker_endpoints=["127.0.0.1:8080"]): + def createRoleMaker( + self, current_id=0, worker_endpoints=["127.0.0.1:8080"] + ): role = UserDefinedCollectiveRoleMaker(current_id, worker_endpoints) def testRoleMaker(self): self.createRoleMaker() # test all invalid worker_endpoints self.assertRaises( - Exception, self.createRoleMaker, - worker_endpoints=None) # worker_endpoints must be as list + Exception, self.createRoleMaker, worker_endpoints=None + ) # worker_endpoints must be as list + self.assertRaises( + Exception, self.createRoleMaker, worker_endpoints=[] + ) # worker_endpoints can't be empty + self.assertRaises( + Exception, self.createRoleMaker, worker_endpoints=[3, []] + ) # element worker_endpoints must be as string self.assertRaises( - Exception, self.createRoleMaker, - worker_endpoints=[]) # worker_endpoints can't be empty - self.assertRaises(Exception, - self.createRoleMaker, - worker_endpoints=[ - 3, [] - ]) # element worker_endpoints must be as string - self.assertRaises(Exception, - self.createRoleMaker, - worker_endpoints=[ - "127.0.0.1:8080", "127.0.0.1:8080" - ]) # element in worker_endpoints can't be duplicate + Exception, + self.createRoleMaker, + worker_endpoints=["127.0.0.1:8080", "127.0.0.1:8080"], + ) # element in worker_endpoints can't be duplicate # test all invalid current_id - self.assertRaises(Exception, self.createRoleMaker, - current_id="0") # current_id must be as int self.assertRaises( - Exception, self.createRoleMaker, - current_id=-1) # current_id must be greater than or equal to 0 + Exception, self.createRoleMaker, current_id="0" + ) # current_id must be as int + self.assertRaises( + Exception, self.createRoleMaker, current_id=-1 + ) # current_id must be greater than or equal to 0 self.assertRaises( Exception, self.createRoleMaker, current_id=1, - worker_endpoints=[ - "127.0.0.1:8080" - ]) # current_id must be less than len(worker_endpoints) + worker_endpoints=["127.0.0.1:8080"], + ) # current_id must be less than len(worker_endpoints) class CollectiveOptimizerTest(unittest.TestCase): - def test_ds_as_None(self): optimizer = fluid.optimizer.AdamOptimizer() dist_optimizer = CollectiveOptimizer(optimizer, strategy=None) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_ascend_utils.py b/python/paddle/fluid/tests/unittests/test_fleet_ascend_utils.py index 7ee6ee83f22c9..03260616cda7d 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_ascend_utils.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_ascend_utils.py @@ -18,30 +18,22 @@ import paddle.distributed.fleet.ascend_utils as ascend_utils RANK_TABLE_JSON = { - "status": - "completed", - "version": - "1.0", - "server_count": - "1", - "server_list": [{ - "server_id": - "127.0.0.1", - "device": [{ - "device_id": "0", - "device_ip": "192.1.184.23", - "rank_id": "0" - }, { - "device_id": "1", - "device_ip": "192.2.21.93", - "rank_id": "1" - }] - }] + "status": "completed", + "version": "1.0", + "server_count": "1", + "server_list": [ + { + "server_id": "127.0.0.1", + "device": [ + {"device_id": "0", "device_ip": "192.1.184.23", "rank_id": "0"}, + {"device_id": "1", "device_ip": "192.2.21.93", "rank_id": "1"}, + ], + } + ], } class TestAscendUtil(unittest.TestCase): - def test_get_cloud_cluster(self): cluster, pod = ascend_utils.get_cloud_cluster() self.assertTrue(cluster) @@ -51,7 +43,8 @@ def test_get_cloud_cluster(self): json.dump(RANK_TABLE_JSON, f) rank_table_file = "./rank_table_file.json" cluster, pod = ascend_utils.get_cloud_cluster( - rank_table_file=rank_table_file) + rank_table_file=rank_table_file + ) self.assertTrue(cluster) self.assertTrue(pod) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_auto.py b/python/paddle/fluid/tests/unittests/test_fleet_auto.py index 29710dc588568..608b948d35f06 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_auto.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_auto.py @@ -21,26 +21,27 @@ class TestDistributedStrategyAuto(unittest.TestCase): - def setUp(self): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_distributed_strategy_auto(self): fleet.init(is_collective=True) - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) strategy = paddle.distributed.fleet.DistributedStrategy() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_base.py b/python/paddle/fluid/tests/unittests/test_fleet_base.py index aba343b1f2952..49dd5d6928bfd 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_base.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_base.py @@ -22,13 +22,13 @@ class TestFleetBase(unittest.TestCase): - def setUp(self): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36000" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36002" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36002" def test_init(self): role = role_maker.PaddleCloudRoleMaker(is_collective=True) @@ -59,8 +59,9 @@ def test_is_worker(self): def test_worker_endpoints(self): role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) - self.assertEqual("127.0.0.1:36000", - fleet.worker_endpoints(to_string=True)) + self.assertEqual( + "127.0.0.1:36000", fleet.worker_endpoints(to_string=True) + ) self.assertEqual(["127.0.0.1:36000"], fleet.worker_endpoints()) def test_server_num(self): @@ -90,10 +91,13 @@ def test_server_endpoints(self): role = role_maker.PaddleCloudRoleMaker() fleet.init(role) if fleet.is_server(): - self.assertEqual("127.0.0.1:36001,127.0.0.2:36002", - fleet.server_endpoints(to_string=True)) - self.assertEqual(["127.0.0.1:36001", "127.0.0.2:36002"], - fleet.server_endpoints()) + self.assertEqual( + "127.0.0.1:36001,127.0.0.2:36002", + fleet.server_endpoints(to_string=True), + ) + self.assertEqual( + ["127.0.0.1:36001", "127.0.0.2:36002"], fleet.server_endpoints() + ) def test_is_server(self): os.environ["TRAINING_ROLE"] = "PSERVER" @@ -139,14 +143,15 @@ def test_distributed_optimizer(self): def test_exception(self): import paddle.distributed.fleet as fleet + self.assertRaises(Exception, fleet.init_worker) class TestFleetDygraph(unittest.TestCase): - def setUp(self): os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213,127.0.0.1:36214" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:36213,127.0.0.1:36214" os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ["PADDLE_TRAINER_ID"] = "0" @@ -156,8 +161,9 @@ def test_dygraph_method(self): value = np.arange(26).reshape(2, 13).astype("float32") a = fluid.dygraph.to_variable(value) layer = paddle.nn.Linear(13, 5) - adam = paddle.optimizer.Adam(learning_rate=0.01, - parameters=layer.parameters()) + adam = paddle.optimizer.Adam( + learning_rate=0.01, parameters=layer.parameters() + ) # remove init cause this UT cannot launch distributed task adam = fleet.distributed_optimizer(adam) try: @@ -169,7 +175,7 @@ def test_dygraph_method(self): lr = 0.001 adam.set_lr(lr) cur_lr = adam.get_lr() - assert (lr == cur_lr) + assert lr == cur_lr state_dict = adam.state_dict() adam.set_state_dict(state_dict) @@ -177,22 +183,20 @@ def test_dygraph_method(self): class TestFleetBaseSingleError(unittest.TestCase): - def setUp(self): os.environ.pop("PADDLE_TRAINER_ENDPOINTS") def gen_data(self): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } def test_single_run_collective_minimize(self): - def test_single_error(): - input_x = paddle.static.data(name="x", - shape=[-1, 32], - dtype='float32') + input_x = paddle.static.data( + name="x", shape=[-1, 32], dtype='float32' + ) input_y = paddle.static.data(name="y", shape=[-1, 1], dtype='int64') fc_1 = fluid.layers.fc(input=input_x, size=64, act='tanh') @@ -202,8 +206,10 @@ def test_single_error(): fleet.init(is_collective=True) # in non_distributed mode(use `python` to launch), raise error if has multi cards - if fluid.core.is_compiled_with_cuda( - ) and fluid.core.get_cuda_device_count() > 1: + if ( + fluid.core.is_compiled_with_cuda() + and fluid.core.get_cuda_device_count() > 1 + ): self.assertRaises(ValueError, test_single_error) else: test_single_error() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_base_2.py b/python/paddle/fluid/tests/unittests/test_fleet_base_2.py index ed914d2866510..084718c5407a3 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_base_2.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_base_2.py @@ -23,12 +23,11 @@ class TestFleetBase(unittest.TestCase): - def setUp(self): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_PORT"] = "36000" os.environ["PADDLE_TRAINERS_NUM"] = "1" - #os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ + # os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ # "127.0.0.1:36001,127.0.0.2:36001" def test_ps_minimize(self): @@ -38,23 +37,24 @@ def test_ps_minimize(self): os.environ["TRAINING_ROLE"] = "TRAINER" os.environ["PADDLE_TRAINER_ID"] = "1" - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') - input_slot = paddle.fluid.layers.data(name="slot", - shape=[1], - dtype='int64') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) + input_slot = paddle.fluid.layers.data( + name="slot", shape=[1], dtype='int64' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') - emb = paddle.fluid.layers.embedding(input=input_slot, - size=[10, 9], - is_sparse=True) + emb = paddle.fluid.layers.embedding( + input=input_slot, size=[10, 9], is_sparse=True + ) input_x = paddle.concat(x=[input_x, emb], axis=1) fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) role = fleet.PaddleCloudRoleMaker(is_collective=False) @@ -73,16 +73,17 @@ def test_ps_minimize(self): exe.run(paddle.static.default_startup_program()) pe = fluid.ParallelExecutor(use_cuda=False, loss_name=avg_cost.name) compiled_prog = fluid.compiler.CompiledProgram( - fluid.default_main_program()) + fluid.default_main_program() + ) temp_dir = tempfile.TemporaryDirectory() fleet.init_worker() - fleet.fleet.save(dirname=temp_dir.name, - feed=['x', 'y'], - fetch=[avg_cost]) - fleet.fleet.save(dirname=temp_dir.name, - feed=[input_x, input_y], - fetch=[avg_cost]) + fleet.fleet.save( + dirname=temp_dir.name, feed=['x', 'y'], fetch=[avg_cost] + ) + fleet.fleet.save( + dirname=temp_dir.name, feed=[input_x, input_y], fetch=[avg_cost] + ) fleet.fleet.save(dirname=temp_dir.name) fleet.load_model(path=temp_dir.name, mode=0) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_base_3.py b/python/paddle/fluid/tests/unittests/test_fleet_base_3.py index 40ee3836dd27d..d9014b2c6f858 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_base_3.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_base_3.py @@ -22,25 +22,26 @@ class TestFleetBase_1(unittest.TestCase): - def setUp(self): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_collective_minimize(self): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) role = role_maker.PaddleCloudRoleMaker(is_collective=True) @@ -52,25 +53,26 @@ def test_collective_minimize(self): class TestFleetBase(unittest.TestCase): - def setUp(self): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_fleet_get_applied_optimizer(self): - input_x = paddle.fluid.layers.data(name="x", - shape=[32], - dtype='float32') + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32' + ) input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') - cost = paddle.fluid.layers.cross_entropy(input=prediction, - label=input_y) + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) fleet.init(is_collective=True) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_base_4.py b/python/paddle/fluid/tests/unittests/test_fleet_base_4.py index 5d70193212cf9..986a48162b28c 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_base_4.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_base_4.py @@ -21,13 +21,13 @@ class TestFleetBase(unittest.TestCase): - def setUp(self): os.environ["POD_IP"] = "127.0.0.1" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ[ + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_fleet_init(self): diff --git a/python/paddle/fluid/tests/unittests/test_fleet_base_single.py b/python/paddle/fluid/tests/unittests/test_fleet_base_single.py index 776a286d97de4..13318d2d3df5c 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_base_single.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_base_single.py @@ -28,7 +28,6 @@ class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear1 = nn.Linear(10, 10) @@ -39,7 +38,6 @@ def forward(self, x): class TestFleetDygraphSingle(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36213" os.environ["PADDLE_CURRENT_ENDPOINTS"] = "127.0.0.1:36213" @@ -52,8 +50,9 @@ def test_dygraph_single(self): layer = LinearNet() loss_fn = nn.MSELoss() - adam = paddle.optimizer.Adam(learning_rate=0.001, - parameters=layer.parameters()) + adam = paddle.optimizer.Adam( + learning_rate=0.001, parameters=layer.parameters() + ) adam = fleet.distributed_optimizer(adam) dp_layer = fleet.distributed_model(layer) @@ -68,14 +67,13 @@ def test_dygraph_single(self): class TestFleetBaseSingleRunCollective(unittest.TestCase): - def setUp(self): pass def gen_data(self): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } def test_single_run_collective_minimize(self): @@ -93,8 +91,11 @@ def test_single_run_collective_minimize(self): optimizer = fleet.distributed_optimizer(optimizer) optimizer.minimize(avg_cost) - place = fluid.CUDAPlace( - 0) if paddle.fluid.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if paddle.fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(paddle.static.default_startup_program()) @@ -105,14 +106,13 @@ def test_single_run_collective_minimize(self): class TestFleetBaseSingleRunPS(unittest.TestCase): - def setUp(self): pass def gen_data(self): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } def test_single_run_ps_minimize(self): @@ -139,11 +139,15 @@ def test_single_run_ps_minimize(self): exe.run(paddle.static.default_startup_program()) step = 10 for i in range(step): - cost_val = exe.run(program=fluid.default_main_program(), - feed=self.gen_data(), - fetch_list=[avg_cost.name]) - print("worker_index: %d, step%d cost = %f" % - (fleet.worker_index(), i, cost_val[0])) + cost_val = exe.run( + program=fluid.default_main_program(), + feed=self.gen_data(), + fetch_list=[avg_cost.name], + ) + print( + "worker_index: %d, step%d cost = %f" + % (fleet.worker_index(), i, cost_val[0]) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_fleet_elastic_collective.py b/python/paddle/fluid/tests/unittests/test_fleet_elastic_collective.py index 8d31fc4efb50c..2d9a9268ee3fa 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_elastic_collective.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_elastic_collective.py @@ -25,11 +25,11 @@ class TestCollectiveLauncher(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() - self.code_path = os.path.join(self.temp_dir.name, - "fake_python_for_elastic.py") + self.code_path = os.path.join( + self.temp_dir.name, "fake_python_for_elastic.py" + ) with open(self.code_path, "w") as f: f.write(fake_python_code) @@ -37,7 +37,6 @@ def tearDown(self): self.temp_dir.cleanup() def test_launch(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -76,7 +75,6 @@ class Argument: pass def test_stop(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_elastic_init.py b/python/paddle/fluid/tests/unittests/test_fleet_elastic_init.py index e32f74ffb1459..5bd3859013cf9 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_elastic_init.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_elastic_init.py @@ -19,9 +19,7 @@ class TestElasticInit(unittest.TestCase): - def setUp(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" diff --git a/python/paddle/fluid/tests/unittests/test_fleet_elastic_manager.py b/python/paddle/fluid/tests/unittests/test_fleet_elastic_manager.py index 3c07edc78037e..a4ad3e1a7c4ea 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_elastic_manager.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_elastic_manager.py @@ -17,17 +17,17 @@ from paddle.distributed.fleet.elastic.manager import ElasticManager from paddle.distributed.fleet.elastic.manager import LauncherInterface -from paddle.distributed.fleet.elastic.manager import ELASTIC_AUTO_PARALLEL_EXIT_CODE +from paddle.distributed.fleet.elastic.manager import ( + ELASTIC_AUTO_PARALLEL_EXIT_CODE, +) -class MockLease(): - +class MockLease: def refresh(self): pass class MockEtcdClient: - def __init__(self, lease=None): self._lease = lease @@ -66,12 +66,10 @@ def lease(self, ttl): class TestElasticManager(unittest.TestCase): - def setUp(self): self.etcd_client = MockEtcdClient() def test_elastic_manager_init(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -87,8 +85,7 @@ class Argument: args = Argument() - class _MockLease(): - + class _MockLease: def refresh(self): raise ValueError("valid error, this only for unittest") @@ -96,7 +93,6 @@ def refresh(self): elastic = ElasticManager(args, etcd_client=etcd_client) def test_match_faulttolerance(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -117,7 +113,8 @@ class Argument: hosts = ["10.10.10.1:6001", "10.10.10.2:6001"] os.environ[ - 'PADDLE_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001" + 'PADDLE_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001" self.assertEqual(elastic._match(hosts), True) @@ -127,7 +124,6 @@ class Argument: self.assertEqual(elastic._match(hosts), False) def test_match_elastic(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -146,16 +142,20 @@ class Argument: args.ips = "10.10.10.1,10.10.10.2,10.10.10.3,10.10.10.4" os.environ['FLAGS_START_PORT'] = "6001" os.environ[ - 'DISTRIBUTED_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001,10.10.10.4:6001" + 'DISTRIBUTED_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001,10.10.10.4:6001" os.environ[ - 'PADDLE_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001,10.10.10.4:6001" + 'PADDLE_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001,10.10.10.4:6001" elastic = ElasticManager(args, self.etcd_client) hosts = ["10.10.10.1:6001", "10.10.10.2:6001"] self.assertEqual(elastic._match(hosts), False) hosts = [ - "10.10.10.1:6001", "10.10.10.2:6001", "10.10.10.3:6001", - "10.10.10.4:6001" + "10.10.10.1:6001", + "10.10.10.2:6001", + "10.10.10.3:6001", + "10.10.10.4:6001", ] self.assertEqual(elastic._match(hosts), True) @@ -167,19 +167,20 @@ class Argument: args.ips = "10.10.10.1,10.10.10.2" os.environ[ - 'DISTRIBUTED_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001" + 'DISTRIBUTED_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001" os.environ[ - 'PADDLE_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001" + 'PADDLE_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001" elastic = ElasticManager(args, self.etcd_client) hosts = ["10.10.10.1:6001", "10.10.10.2:6001"] self.assertEqual(elastic._match(hosts), True) # TODO test timeout - #time.sleep(60) - #self.assertEqual(elastic._match(hosts), True) + # time.sleep(60) + # self.assertEqual(elastic._match(hosts), True) def test_update_hosts_for_faulttolerance(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -198,9 +199,11 @@ class Argument: os.environ['PADDLE_ELASTIC_NP'] = "2" os.environ['PADDLE_TRAINERS'] = "10.10.10.1,10.10.10.2" os.environ[ - 'DISTRIBUTED_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001" + 'DISTRIBUTED_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001" os.environ[ - 'PADDLE_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001" + 'PADDLE_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001" elastic = ElasticManager(args, self.etcd_client) # add 10.10.10.3:6001 os.environ['PADDLE_TRAINER_ID'] = "0" @@ -244,30 +247,38 @@ class Argument: os.environ['FLAGS_START_PORT'] = "6001" os.environ['PADDLE_TRAINERS'] = "10.10.10.1,10.10.10.2" os.environ[ - 'DISTRIBUTED_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001" + 'DISTRIBUTED_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001" os.environ[ - 'PADDLE_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.2:6001" + 'PADDLE_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.2:6001" elastic = ElasticManager(args, self.etcd_client) # add 10.10.10.3:6001 elastic.curr_host = "10.10.10.1:6001" elastic.hosts = [ - "10.10.10.1:6001", "10.10.10.2:6001", "10.10.10.3:6001" + "10.10.10.1:6001", + "10.10.10.2:6001", + "10.10.10.3:6001", ] elastic._update_hosts() - #self.assertEqual(elastic.all_host_endpoints, + # self.assertEqual(elastic.all_host_endpoints, # ["10.10.10.1:6001", "10.10.10.2:6001", "10.10.10.3:6001"]) - self.assertEqual(os.getenv('PADDLE_TRAINERS'), - "10.10.10.1,10.10.10.2,10.10.10.3") + self.assertEqual( + os.getenv('PADDLE_TRAINERS'), "10.10.10.1,10.10.10.2,10.10.10.3" + ) ####################### # elastic, scale in # ####################### os.environ[ - 'PADDLE_TRAINERS'] = "10.10.10.0,10.10.10.1,10.10.10.2,10.10.10.3" + 'PADDLE_TRAINERS' + ] = "10.10.10.0,10.10.10.1,10.10.10.2,10.10.10.3" os.environ[ - 'DISTRIBUTED_TRAINER_ENDPOINTS'] = "10.10.10.0:6000,10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001" + 'DISTRIBUTED_TRAINER_ENDPOINTS' + ] = "10.10.10.0:6000,10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001" os.environ[ - 'PADDLE_TRAINER_ENDPOINTS'] = "10.10.10.0:6000,10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001" + 'PADDLE_TRAINER_ENDPOINTS' + ] = "10.10.10.0:6000,10.10.10.1:6001,10.10.10.2:6001,10.10.10.3:6001" os.environ['POD_IP'] = "10.10.10.1" os.environ['TRAINER_PORTS_NUM'] = "4" os.environ['PADDLE_TRAINER_ID'] = "1" @@ -277,23 +288,31 @@ class Argument: # remove 10.10.10.1:6001 elastic.curr_host = "10.10.10.1:6001" elastic.hosts = [ - "10.10.10.1:6001", "10.10.10.2:6001", "10.10.10.3:6001" + "10.10.10.1:6001", + "10.10.10.2:6001", + "10.10.10.3:6001", ] elastic._update_hosts() - #self.assertEqual(elastic.all_host_endpoints, + # self.assertEqual(elastic.all_host_endpoints, # ["10.10.10.3:6001", "10.10.10.1:6001", "10.10.10.2:6001"]) - self.assertEqual(os.getenv('PADDLE_TRAINERS'), - "10.10.10.3,10.10.10.1,10.10.10.2") - self.assertEqual(os.getenv('DISTRIBUTED_TRAINER_ENDPOINTS'), - "10.10.10.3:6001,10.10.10.1:6001,10.10.10.2:6001") + self.assertEqual( + os.getenv('PADDLE_TRAINERS'), "10.10.10.3,10.10.10.1,10.10.10.2" + ) + self.assertEqual( + os.getenv('DISTRIBUTED_TRAINER_ENDPOINTS'), + "10.10.10.3:6001,10.10.10.1:6001,10.10.10.2:6001", + ) ############ os.environ[ - 'PADDLE_TRAINERS'] = "10.10.10.1,10.10.10.1,10.10.10.1,10.10.10.1" + 'PADDLE_TRAINERS' + ] = "10.10.10.1,10.10.10.1,10.10.10.1,10.10.10.1" os.environ[ - 'DISTRIBUTED_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.1:6002,10.10.10.1:6003,10.10.10.1:6004" + 'DISTRIBUTED_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.1:6002,10.10.10.1:6003,10.10.10.1:6004" os.environ[ - 'PADDLE_TRAINER_ENDPOINTS'] = "10.10.10.1:6001,10.10.10.1:6002,10.10.10.1:6003,10.10.10.1:6004" + 'PADDLE_TRAINER_ENDPOINTS' + ] = "10.10.10.1:6001,10.10.10.1:6002,10.10.10.1:6003,10.10.10.1:6004" os.environ['POD_IP'] = "10.10.10.1" os.environ['TRAINER_PORTS_NUM'] = "4" os.environ['PADDLE_PORT'] = "6001" @@ -304,14 +323,15 @@ class Argument: os.environ['PADDLE_TRAINER_ID'] = "-1" elastic.hosts = ["10.10.10.1:6001", "10.10.10.1:6003"] elastic._update_hosts() - #self.assertEqual(elastic.all_host_endpoints, + # self.assertEqual(elastic.all_host_endpoints, # ["10.10.10.1:6001", "10.10.10.1:6001"]) self.assertEqual(os.getenv('PADDLE_TRAINERS'), "10.10.10.1,10.10.10.1") - self.assertEqual(os.getenv('DISTRIBUTED_TRAINER_ENDPOINTS'), - "10.10.10.1:6001,10.10.10.1:6003") + self.assertEqual( + os.getenv('DISTRIBUTED_TRAINER_ENDPOINTS'), + "10.10.10.1:6001,10.10.10.1:6003", + ) def test_exit(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -330,7 +350,6 @@ class Argument: elastic.exit() def test_pre_hook(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -353,7 +372,6 @@ class Argument: elastic.pre_hook() def test_watch(self): - class Argument: elastic_server = "127.0.0.1:2379" job_id = "test_job_id_123" @@ -369,7 +387,6 @@ class Argument: elastic_pre_hook = None class ElasticLauncher: - def watch(self): return ELASTIC_AUTO_PARALLEL_EXIT_CODE @@ -383,14 +400,11 @@ def stop(self): elastic.watch() def test_launcher_interface_check_procs(self): - class Proc: - def poll(self): return ELASTIC_AUTO_PARALLEL_EXIT_CODE class ProcList: - def __init__(self): self.proc = Proc() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_run.py b/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_run.py index 9bde09ab7243b..4e37988f7673e 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_run.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_run.py @@ -23,7 +23,6 @@ class TestDistModelRun(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -34,8 +33,9 @@ def tearDown(self): def test_dist_model_run(self): # step 0: declare folder to save the model and params - path_prefix = os.path.join(self.temp_dir.name, - "dist_model_run_test/inf") + path_prefix = os.path.join( + self.temp_dir.name, "dist_model_run_test/inf" + ) # step 1: saving the inference model and params x = paddle.static.data(name='x', shape=[28, 28], dtype='float32') @@ -47,12 +47,11 @@ def test_dist_model_run(self): exe.run(paddle.static.default_startup_program()) x_data = np.random.randn(28, 28).astype('float32') y_data = np.random.randint(0, 9, size=[28, 1]).astype('int64') - exe.run(paddle.static.default_main_program(), - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[avg_loss]) + exe.run( + paddle.static.default_main_program(), + feed={'x': x_data, 'y': y_data}, + fetch_list=[avg_loss], + ) paddle.static.save_inference_model(path_prefix, [x, y], [avg_loss], exe) print('save model to', path_prefix) @@ -74,21 +73,23 @@ def test_dist_model_run(self): print("dist model rst:", dist_model_rst) # step 4: use framework's api to inference with fake data - [inference_program, feed_target_names, - fetch_targets] = (paddle.static.load_inference_model(path_prefix, exe)) - results = exe.run(inference_program, - feed={ - 'x': x_tensor, - 'y': y_tensor - }, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(path_prefix, exe) + results = exe.run( + inference_program, + feed={'x': x_tensor, 'y': y_tensor}, + fetch_list=fetch_targets, + ) load_inference_model_rst = results[0] print("load inference model api rst:", load_inference_model_rst) # step 5: compare two results - np.testing.assert_allclose(dist_model_rst, - load_inference_model_rst, - rtol=1e-05) + np.testing.assert_allclose( + dist_model_rst, load_inference_model_rst, rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_tensor.py b/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_tensor.py index 98affdfa54071..85da732ab5bd6 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_exe_dist_model_tensor.py @@ -22,53 +22,67 @@ class TestDistModelTensor(unittest.TestCase): - def test_dist_model_tensor(self): tensor_32 = np.random.randint(10, 20, size=[20, 2]).astype('int32') dist_tensor32 = DistModelTensor(tensor_32, '32_tensor') self.assertEqual(dist_tensor32.dtype, DistModelDataType.INT32) - self.assertEqual(dist_tensor32.data.tolist('int32'), - tensor_32.ravel().tolist()) + self.assertEqual( + dist_tensor32.data.tolist('int32'), tensor_32.ravel().tolist() + ) # the length is how many byte the data contains self.assertEqual(dist_tensor32.data.length(), 40 * 4) self.assertEqual(dist_tensor32.name, '32_tensor') dist_tensor32.data.reset(tensor_32) - self.assertEqual(dist_tensor32.as_ndarray().ravel().tolist(), - tensor_32.ravel().tolist()) + self.assertEqual( + dist_tensor32.as_ndarray().ravel().tolist(), + tensor_32.ravel().tolist(), + ) tensor_64 = np.random.randint(10, 20, size=[20, 2]).astype('int64') dist_tensor64 = DistModelTensor(tensor_64, '64_tensor') self.assertEqual(dist_tensor64.dtype, DistModelDataType.INT64) - self.assertEqual(dist_tensor64.data.tolist('int64'), - tensor_64.ravel().tolist()) + self.assertEqual( + dist_tensor64.data.tolist('int64'), tensor_64.ravel().tolist() + ) self.assertEqual(dist_tensor64.data.length(), 40 * 8) self.assertEqual(dist_tensor64.name, '64_tensor') dist_tensor64.data.reset(tensor_64) - self.assertEqual(dist_tensor64.as_ndarray().ravel().tolist(), - tensor_64.ravel().tolist()) + self.assertEqual( + dist_tensor64.as_ndarray().ravel().tolist(), + tensor_64.ravel().tolist(), + ) tensor_float = np.random.randn(20, 2).astype('float32') dist_tensor_float = DistModelTensor(tensor_float, 'float_tensor') self.assertEqual(dist_tensor_float.dtype, DistModelDataType.FLOAT32) - self.assertEqual(dist_tensor_float.data.tolist('float32'), - tensor_float.ravel().tolist()) + self.assertEqual( + dist_tensor_float.data.tolist('float32'), + tensor_float.ravel().tolist(), + ) self.assertEqual(dist_tensor_float.data.length(), 40 * 4) self.assertEqual(dist_tensor_float.name, 'float_tensor') dist_tensor_float.data.reset(tensor_float) - self.assertEqual(dist_tensor_float.as_ndarray().ravel().tolist(), - tensor_float.ravel().tolist()) + self.assertEqual( + dist_tensor_float.as_ndarray().ravel().tolist(), + tensor_float.ravel().tolist(), + ) tensor_float_16 = np.random.randn(20, 2).astype('float16') - dist_tensor_float_16 = DistModelTensor(tensor_float_16, - 'float_tensor_16') + dist_tensor_float_16 = DistModelTensor( + tensor_float_16, 'float_tensor_16' + ) self.assertEqual(dist_tensor_float_16.dtype, DistModelDataType.FLOAT16) - self.assertEqual(dist_tensor_float_16.data.tolist('float16'), - tensor_float_16.ravel().tolist()) + self.assertEqual( + dist_tensor_float_16.data.tolist('float16'), + tensor_float_16.ravel().tolist(), + ) self.assertEqual(dist_tensor_float_16.data.length(), 40 * 2) self.assertEqual(dist_tensor_float_16.name, 'float_tensor_16') dist_tensor_float_16.data.reset(tensor_float_16) - self.assertEqual(dist_tensor_float_16.as_ndarray().ravel().tolist(), - tensor_float_16.ravel().tolist()) + self.assertEqual( + dist_tensor_float_16.as_ndarray().ravel().tolist(), + tensor_float_16.ravel().tolist(), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_fleet_executor.py b/python/paddle/fluid/tests/unittests/test_fleet_executor.py index f8469c8ed26be..502f03784cbcc 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_executor.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_executor.py @@ -21,21 +21,21 @@ class TestFleetExecutor(unittest.TestCase): - def fake_fleet_opt(self): # TODO: Fake for coverage will be removed in the future import paddle.distributed.fleet as fleet + strategy = fleet.DistributedStrategy() strategy.sharding_configs = { "dp_degree": 1, "mp_degree": 1, - "pp_degree": 1 + "pp_degree": 1, } strategy.pipeline_configs = {"accumulate_steps": 1} fleet_opt = { "dist_strategy": strategy.sharding_configs, "num_micro_batches": strategy.pipeline_configs["accumulate_steps"], - "scheduler": "1F1B" + "scheduler": "1F1B", } return fleet_opt @@ -43,12 +43,12 @@ def run_fleet_executor(self, place, x_data, y_data): exe = paddle.static.Executor(place) empty_program = paddle.static.Program() with fluid.program_guard(empty_program, empty_program): - x = fluid.layers.data(name='x', - shape=x_data.shape, - dtype=x_data.dtype) - y = fluid.layers.data(name='y', - shape=y_data.shape, - dtype=y_data.dtype) + x = fluid.layers.data( + name='x', shape=x_data.shape, dtype=x_data.dtype + ) + y = fluid.layers.data( + name='y', shape=y_data.shape, dtype=y_data.dtype + ) z = x + y a = 2 * x + 3 * y loss = paddle.mean(a) @@ -57,23 +57,24 @@ def run_fleet_executor(self, place, x_data, y_data): steps_per_pass = 10 bd = [steps_per_pass * p for p in passes] lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] - lr_val = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr) + lr_val = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr + ) opt = paddle.optimizer.AdamW( learning_rate=lr_val, - grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0)) + grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0), + ) opt.minimize(loss) # TODO: section_program will be removed in the future empty_program._pipeline_opt = { "fleet_opt": self.fake_fleet_opt(), - "section_program": empty_program + "section_program": empty_program, } - res = exe.run(empty_program, - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[z.name, a.name]) + res = exe.run( + empty_program, + feed={'x': x_data, 'y': y_data}, + fetch_list=[z.name, a.name], + ) return res def test_executor_on_single_device(self): diff --git a/python/paddle/fluid/tests/unittests/test_fleet_executor_multi_devices.py b/python/paddle/fluid/tests/unittests/test_fleet_executor_multi_devices.py index c21549c3ce334..5c635e4ce4807 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_executor_multi_devices.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_executor_multi_devices.py @@ -22,7 +22,6 @@ class TestFleetExecutor(unittest.TestCase): - def run_fleet_executor(self, place, fleet_opt=dict()): exe = paddle.static.Executor(place) empty_program = paddle.static.Program() @@ -30,24 +29,25 @@ def run_fleet_executor(self, place, fleet_opt=dict()): x = fluid.layers.data(name='x', shape=[1], dtype=paddle.float32) empty_program._pipeline_opt = { "fleet_opt": fleet_opt, - "section_program": empty_program + "section_program": empty_program, } exe.run(empty_program, feed={'x': [1]}) def test_dist_executor_on_multi_devices(self): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ[ - "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:7000,127.0.0.1:7001,127.0.0.1:7002,127.0.0.1:7003,127.0.0.1:7004,127.0.0.1:7005,127.0.0.1:7006,127.0.0.1:7007" + "PADDLE_TRAINER_ENDPOINTS" + ] = "127.0.0.1:7000,127.0.0.1:7001,127.0.0.1:7002,127.0.0.1:7003,127.0.0.1:7004,127.0.0.1:7005,127.0.0.1:7006,127.0.0.1:7007" strategy = fleet.DistributedStrategy() strategy.sharding_configs = { "dp_degree": 2, "mp_degree": 2, - "pp_degree": 2 + "pp_degree": 2, } strategy.pipeline_configs = {"accumulate_steps": 8} fleet_opt = { "dist_strategy": strategy.sharding_configs, - "num_micro_batches": strategy.pipeline_configs["accumulate_steps"] + "num_micro_batches": strategy.pipeline_configs["accumulate_steps"], } if fluid.is_compiled_with_cuda(): # TODO: Distribute test case is not supported for executor can not stop diff --git a/python/paddle/fluid/tests/unittests/test_fleet_executor_origin_scheduler.py b/python/paddle/fluid/tests/unittests/test_fleet_executor_origin_scheduler.py index 9a71937ddb25f..cda6be8f0a6be 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_executor_origin_scheduler.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_executor_origin_scheduler.py @@ -21,21 +21,21 @@ class TestFleetExecutor(unittest.TestCase): - def fake_fleet_opt(self): # TODO: Fake for coverage will be removed in the future import paddle.distributed.fleet as fleet + strategy = fleet.DistributedStrategy() strategy.sharding_configs = { "dp_degree": 1, "mp_degree": 1, - "pp_degree": 1 + "pp_degree": 1, } strategy.pipeline_configs = {"accumulate_steps": 1} fleet_opt = { "dist_strategy": strategy.sharding_configs, "num_micro_batches": strategy.pipeline_configs["accumulate_steps"], - "scheduler": "Origin" + "scheduler": "Origin", } return fleet_opt @@ -43,12 +43,12 @@ def run_fleet_executor(self, place, x_data, y_data): exe = paddle.static.Executor(place) empty_program = paddle.static.Program() with fluid.program_guard(empty_program, empty_program): - x = fluid.layers.data(name='x', - shape=x_data.shape, - dtype=x_data.dtype) - y = fluid.layers.data(name='y', - shape=y_data.shape, - dtype=y_data.dtype) + x = fluid.layers.data( + name='x', shape=x_data.shape, dtype=x_data.dtype + ) + y = fluid.layers.data( + name='y', shape=y_data.shape, dtype=y_data.dtype + ) z = x + y a = 2 * x + 3 * y loss = paddle.mean(a) @@ -57,23 +57,24 @@ def run_fleet_executor(self, place, x_data, y_data): steps_per_pass = 10 bd = [steps_per_pass * p for p in passes] lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] - lr_val = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr) + lr_val = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr + ) opt = paddle.optimizer.AdamW( learning_rate=lr_val, - grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0)) + grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0), + ) opt.minimize(loss) # TODO: section_program will be removed in the future empty_program._pipeline_opt = { "fleet_opt": self.fake_fleet_opt(), - "section_program": empty_program + "section_program": empty_program, } - res = exe.run(empty_program, - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[z.name, a.name]) + res = exe.run( + empty_program, + feed={'x': x_data, 'y': y_data}, + fetch_list=[z.name, a.name], + ) return res def test_executor_on_single_device(self): diff --git a/python/paddle/fluid/tests/unittests/test_fleet_executor_task_node.py b/python/paddle/fluid/tests/unittests/test_fleet_executor_task_node.py index 07ecf85c3db2e..03d6be10111f3 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_executor_task_node.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_executor_task_node.py @@ -21,7 +21,6 @@ class TestFleetExecutorTaskNode(unittest.TestCase): - def test_task_node(self): program = paddle.static.Program() task_node_0 = core.TaskNode(program.desc, 0, 1, 1) @@ -31,16 +30,19 @@ def test_task_node(self): self.assertEqual(task_node_1.task_id(), 1) self.assertEqual(task_node_2.task_id(), 2) self.assertTrue( - task_node_0.add_downstream_task(task_node_1.task_id(), 1)) + task_node_0.add_downstream_task(task_node_1.task_id(), 1) + ) self.assertTrue(task_node_1.add_upstream_task(task_node_0.task_id(), 1)) def test_lazy_task_node(self): program = paddle.static.Program() - task = TaskNode(program=program, - rank=0, - max_run_times=1, - max_slot_times=1, - lazy_initialize=True) + task = TaskNode( + program=program, + rank=0, + max_run_times=1, + max_slot_times=1, + lazy_initialize=True, + ) task_node = task.task_node() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_executor_utils.py b/python/paddle/fluid/tests/unittests/test_fleet_executor_utils.py index cbc54e8c70f5c..fb003f450f6e1 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_executor_utils.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_executor_utils.py @@ -20,25 +20,27 @@ class TestFleetExecutorUtils(unittest.TestCase): - def test_construct_program(self): # TODO(liyurui): These functions are not ready now. strategy = paddle.distributed.fleet.DistributedStrategy() strategy.sharding_configs = { "dp_degree": 2, "mp_degree": 2, - "pp_degree": 2 + "pp_degree": 2, } fleet_executor_utils = FleetExecutorUtils( dist_strategy=strategy.sharding_configs, rank=0, nrank=1, - max_run_times=1) + max_run_times=1, + ) op_list = {"lr": [], "fwd": [], "bwd": [], "opt": []} program_map = fleet_executor_utils.convert_op_list_to_program( - op_list, paddle.static.Program()) + op_list, paddle.static.Program() + ) task_node_map = fleet_executor_utils.construct_task_nodes_1f1b( - program_map) + program_map + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_fleet_executor_with_task_nodes.py b/python/paddle/fluid/tests/unittests/test_fleet_executor_with_task_nodes.py index f80f998c047dc..802ce0cbfe3c9 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_executor_with_task_nodes.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_executor_with_task_nodes.py @@ -22,17 +22,16 @@ class TestFleetExecutor(unittest.TestCase): - def run_fleet_executor(self, place, x_data, y_data): exe = paddle.static.Executor(place) empty_program = paddle.static.Program() with fluid.program_guard(empty_program, empty_program): - x = fluid.layers.data(name='x', - shape=x_data.shape, - dtype=x_data.dtype) - y = fluid.layers.data(name='y', - shape=y_data.shape, - dtype=y_data.dtype) + x = fluid.layers.data( + name='x', shape=x_data.shape, dtype=x_data.dtype + ) + y = fluid.layers.data( + name='y', shape=y_data.shape, dtype=y_data.dtype + ) z = x + y a = 2 * x + 3 * y loss = paddle.mean(a) @@ -41,11 +40,13 @@ def run_fleet_executor(self, place, x_data, y_data): steps_per_pass = 10 bd = [steps_per_pass * p for p in passes] lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] - lr_val = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr) + lr_val = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr + ) opt = paddle.optimizer.AdamW( learning_rate=lr_val, - grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0)) + grad_clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0), + ) opt.minimize(loss) # TODO: section_program will be removed in the future task_node = TaskNode( @@ -55,22 +56,20 @@ def run_fleet_executor(self, place, x_data, y_data): node_type="Compute", max_run_times=1, max_slot_times=1, - lazy_initialize=True) + lazy_initialize=True, + ) empty_program._pipeline_opt = { "fleet_opt": { 'tasks': [task_node], - 'task_id_to_rank': { - task_node.task_id(): 0 - } + 'task_id_to_rank': {task_node.task_id(): 0}, }, - "section_program": empty_program + "section_program": empty_program, } - res = exe.run(empty_program, - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[z.name, a.name]) + res = exe.run( + empty_program, + feed={'x': x_data, 'y': y_data}, + fetch_list=[z.name, a.name], + ) return res def test_executor_on_single_device(self): diff --git a/python/paddle/fluid/tests/unittests/test_fleet_gradient_scale.py b/python/paddle/fluid/tests/unittests/test_fleet_gradient_scale.py index 6aefb2926e40c..36e823a2bc92a 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_gradient_scale.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_gradient_scale.py @@ -22,7 +22,6 @@ class TestGradientScale(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" @@ -30,18 +29,19 @@ def setUp(self): def mlp(self, input_x, input_y, hid_dim=128, label_dim=2): fc_1 = paddle.static.nn.fc(x=input_x, size=hid_dim, activation='tanh') fc_2 = paddle.static.nn.fc(x=fc_1, size=hid_dim, activation='tanh') - prediction = paddle.static.nn.fc(x=[fc_2], - size=label_dim, - activation='softmax') - cost = paddle.nn.functional.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.static.nn.fc( + x=[fc_2], size=label_dim, activation='softmax' + ) + cost = paddle.nn.functional.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) return avg_cost def gen_data(self): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } def test_single_gpu(self): @@ -53,16 +53,17 @@ def test_single_gpu(self): strategy.gradient_scale_configs = {'scale_strategy': 'sum'} with fluid.program_guard(main_program, startup_program): with fluid.unique_name.guard(): - input_x = paddle.static.data(name="x", - shape=[None, 32], - dtype='float32') - input_y = paddle.static.data(name="y", - shape=[None, 1], - dtype='int64') + input_x = paddle.static.data( + name="x", shape=[None, 32], dtype='float32' + ) + input_y = paddle.static.data( + name="y", shape=[None, 1], dtype='int64' + ) cost = self.mlp(input_x=input_x, input_y=input_y) output_name = cost.name - optimizer = fleet.distributed_optimizer(fluid.optimizer.Adam(), - strategy) + optimizer = fleet.distributed_optimizer( + fluid.optimizer.Adam(), strategy + ) optimizer.minimize(cost) final_strategy = fleet._final_strategy() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_metric.py b/python/paddle/fluid/tests/unittests/test_fleet_metric.py index f4af98e649db2..c8fd7e0d2e529 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_metric.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_metric.py @@ -32,7 +32,6 @@ def setUp(self): """Set up, set envs.""" class FakeUtil(UtilBase): - def __init__(self, fake_fleet): super(FakeUtil, self).__init__() self.fleet = fake_fleet @@ -79,16 +78,20 @@ def test_metric_1(self): train = fluid.Program() startup = fluid.Program() with fluid.program_guard(train, startup): - t = fluid.layers.create_global_var(shape=[1, 1], - value=1, - dtype='int64', - persistable=True, - force_cpu=True) - t1 = fluid.layers.create_global_var(shape=[1, 1], - value=1, - dtype='int64', - persistable=True, - force_cpu=True) + t = fluid.layers.create_global_var( + shape=[1, 1], + value=1, + dtype='int64', + persistable=True, + force_cpu=True, + ) + t1 = fluid.layers.create_global_var( + shape=[1, 1], + value=1, + dtype='int64', + persistable=True, + force_cpu=True, + ) place = fluid.CPUPlace() exe = fluid.Executor(place) scope = fluid.Scope() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py b/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py index 81daea3af9310..8c92efe323650 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py @@ -26,7 +26,8 @@ def setUp(self): """Set up, set envs.""" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_pslib_1(self): """Test cases for pslib.""" @@ -41,33 +42,48 @@ def test_pslib_1(self): os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36002" os.environ["PADDLE_TRAINER_ID"] = "0" role_maker = GeneralRoleMaker() - #role_maker.generate_role() + # role_maker.generate_role() place = fluid.CPUPlace() exe = fluid.Executor(place) - #fleet.init(role_maker) + # fleet.init(role_maker) train_program = fluid.Program() startup_program = fluid.Program() scope = fluid.Scope() with fluid.program_guard(train_program, startup_program): - show = fluid.layers.data(name="show", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) - emb = fluid.layers.embedding(input=show, size=[1, 1], \ - is_sparse=True, is_distributed=True, \ - param_attr=fluid.ParamAttr(name="embedding")) + show = fluid.layers.data( + name="show", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + emb = fluid.layers.embedding( + input=show, + size=[1, 1], + is_sparse=True, + is_distributed=True, + param_attr=fluid.ParamAttr(name="embedding"), + ) fc = fluid.layers.fc(input=emb, size=1, act=None) - label = fluid.layers.data(name="click", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) label_cast = fluid.layers.cast(label, dtype='float32') cost = fluid.layers.log_loss(fc, label_cast) try: adam = fluid.optimizer.Adam(learning_rate=0.000005) - adam = fleet.distributed_optimizer(adam, - strategy={ - "embedding": { - "sparse_accessor_class": - "DownpourCtrAccessor" - } - }) + adam = fleet.distributed_optimizer( + adam, + strategy={ + "embedding": { + "sparse_accessor_class": "DownpourCtrAccessor" + } + }, + ) adam.minimize([cost], [scope]) fleet.run_server() except: diff --git a/python/paddle/fluid/tests/unittests/test_fleet_ps.py b/python/paddle/fluid/tests/unittests/test_fleet_ps.py index 687c9a6eaf59e..f5f7f81311cfc 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_ps.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_ps.py @@ -14,15 +14,17 @@ import unittest from paddle.fluid.framework import default_main_program -from paddle.fluid.incubate.fleet.parameter_server.ir.pserver_pass import _get_optimizer_input_shape +from paddle.fluid.incubate.fleet.parameter_server.ir.pserver_pass import ( + _get_optimizer_input_shape, +) main_program = default_main_program() class TestFleetPS(unittest.TestCase): - def test_version(self): from paddle.fluid.incubate.fleet.parameter_server import version + transpiler = version.is_transpiler() self.assertEqual(transpiler, True) @@ -43,8 +45,9 @@ def test_optimizer_shape(self): for attrs in optimizers: op_type, varkey, orig_shape, param_shape = attrs - new_shape = _get_optimizer_input_shape(op_type, varkey, orig_shape, - param_shape) + new_shape = _get_optimizer_input_shape( + op_type, varkey, orig_shape, param_shape + ) self.assertListEqual(new_shape, param_shape) optimizers = [] @@ -52,8 +55,9 @@ def test_optimizer_shape(self): for attrs in optimizers: op_type, varkey, orig_shape, param_shape = attrs - new_shape = _get_optimizer_input_shape(op_type, varkey, orig_shape, - param_shape) + new_shape = _get_optimizer_input_shape( + op_type, varkey, orig_shape, param_shape + ) self.assertListEqual(new_shape, orig_shape) with self.assertRaises(ValueError): @@ -62,8 +66,9 @@ def test_optimizer_shape(self): for attrs in optimizers: op_type, varkey, orig_shape, param_shape = attrs - _get_optimizer_input_shape(op_type, varkey, orig_shape, - param_shape) + _get_optimizer_input_shape( + op_type, varkey, orig_shape, param_shape + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_fleet_pyramid_hash.py b/python/paddle/fluid/tests/unittests/test_fleet_pyramid_hash.py index d22fc3a1b8c43..fb4ed901d814b 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_pyramid_hash.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_pyramid_hash.py @@ -15,12 +15,15 @@ import unittest import paddle.fluid as fluid import paddle.fluid.incubate.fleet.base.role_maker as role_maker -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, +) +from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( + StrategyFactory, +) class TestPyramidHashOpApi(unittest.TestCase): - def test_dist_geo_server_transpiler(self): num_voc = 128 embed_dim = 64 @@ -49,7 +52,8 @@ def test_dist_geo_server_transpiler(self): ), param_attr_bl=None, distribute_update_vars=["PyramidHash_emb_0"], - name=None) + name=None, + ) cost = fluid.layers.reduce_sum(hash_embd) @@ -57,7 +61,8 @@ def test_dist_geo_server_transpiler(self): current_id=0, role=role_maker.Role.SERVER, worker_num=2, - server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"]) + server_endpoints=["127.0.0.1:36011", "127.0.0.1:36012"], + ) fleet.init(role) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py index 9824b59028fad..05732d3cc8619 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker.py @@ -27,7 +27,8 @@ def setUp(self): """Set up, set envs.""" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_tr_rolemaker(self): """Test tr rolenamer.""" @@ -70,20 +71,30 @@ def test_pslib_1(self): os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36002" os.environ["PADDLE_TRAINER_ID"] = "0" role_maker = GeneralRoleMaker() - #print("init rolemaker") - #role_maker.generate_role() + # print("init rolemaker") + # role_maker.generate_role() place = fluid.CPUPlace() exe = fluid.Executor(place) - #fleet.init(role_maker) + # fleet.init(role_maker) train_program = fluid.Program() startup_program = fluid.Program() scope = fluid.Scope() with fluid.program_guard(train_program, startup_program): - show = fluid.layers.data(name="show", shape=[-1, 1], \ - dtype="float32", lod_level=1, append_batch_size=False) + show = fluid.layers.data( + name="show", + shape=[-1, 1], + dtype="float32", + lod_level=1, + append_batch_size=False, + ) fc = fluid.layers.fc(input=show, size=1, act=None) - label = fluid.layers.data(name="click", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) label_cast = fluid.layers.cast(label, dtype='float32') cost = fluid.layers.log_loss(fc, label_cast) try: @@ -95,8 +106,10 @@ def test_pslib_1(self): print("do not support pslib test, skip") return fleet.clear_one_table(0) - from paddle.fluid.incubate.fleet.base.role_maker import \ - MPISymetricRoleMaker + from paddle.fluid.incubate.fleet.base.role_maker import ( + MPISymetricRoleMaker, + ) + try: role = MPISymetricRoleMaker() role._all_reduce([1], [2]) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py index b60c43a209a14..7d03b00fbdd8b 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_2.py @@ -34,7 +34,9 @@ def tearDown(self): def test_pslib_2(self): """Test cases for pslib.""" import paddle.fluid as fluid - from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet + from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import ( + fleet, + ) from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker from paddle.fluid.incubate.fleet.base.role_maker import RoleMakerBase @@ -58,11 +60,21 @@ def test_pslib_2(self): startup_program = fluid.Program() scope = fluid.Scope() with fluid.program_guard(train_program, startup_program): - show = fluid.layers.data(name="show", shape=[-1, 1], \ - dtype="float32", lod_level=1, append_batch_size=False) + show = fluid.layers.data( + name="show", + shape=[-1, 1], + dtype="float32", + lod_level=1, + append_batch_size=False, + ) fc = fluid.layers.fc(input=show, size=1, act=None) - label = fluid.layers.data(name="click", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) label_cast = fluid.layers.cast(label, dtype='float32') cost = fluid.layers.log_loss(fc, label_cast) try: @@ -159,8 +171,9 @@ def test_pslib_2(self): role23._get_size() role23._get_size() - path = os.path.join(self.temp_dir.name, - "test_fleet_gloo_role_maker_1.txt") + path = os.path.join( + self.temp_dir.name, "test_fleet_gloo_role_maker_1.txt" + ) with open(path, "w") as f: data = "1 1 1 1\n" f.write(data) @@ -172,7 +185,7 @@ def test_pslib_2(self): dataset.get_memory_data_size(fleet) dataset.get_shuffle_data_size(fleet) - class TmpClass(): + class TmpClass: """ dummy tmp class """ @@ -262,13 +275,16 @@ def save_persistables(self): tmp.all_reduce_worker([], []) tmp.barrier_worker() from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker + tmp = RoleMakerBase() tmp.all_gather(1) tmp.all_reduce_worker([], []) tmp.barrier_worker() tmp.barrier_all() - from paddle.fluid.incubate.fleet.base.role_maker import \ - MPISymetricRoleMaker + from paddle.fluid.incubate.fleet.base.role_maker import ( + MPISymetricRoleMaker, + ) + tmp1 = MPISymetricRoleMaker() tmp1.all_gather(1) tmp1.all_gather(1) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py index 2e639a8167984..0ce1b0743aa92 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_3.py @@ -26,7 +26,8 @@ def setUp(self): """Set up, set envs.""" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_pslib_1(self): """Test cases for pslib.""" @@ -40,22 +41,34 @@ def test_pslib_1(self): os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36002" os.environ["PADDLE_TRAINER_ID"] = "0" - role_maker = GeneralRoleMaker(init_timeout_seconds=100, - run_timeout_seconds=100, - http_ip_port="127.0.0.1:36003") - #role_maker.generate_role() + role_maker = GeneralRoleMaker( + init_timeout_seconds=100, + run_timeout_seconds=100, + http_ip_port="127.0.0.1:36003", + ) + # role_maker.generate_role() place = fluid.CPUPlace() exe = fluid.Executor(place) - #fleet.init(role_maker) + # fleet.init(role_maker) train_program = fluid.Program() startup_program = fluid.Program() scope = fluid.Scope() with fluid.program_guard(train_program, startup_program): - show = fluid.layers.data(name="show", shape=[-1, 1], \ - dtype="float32", lod_level=1, append_batch_size=False) + show = fluid.layers.data( + name="show", + shape=[-1, 1], + dtype="float32", + lod_level=1, + append_batch_size=False, + ) fc = fluid.layers.fc(input=show, size=1, act=None) - label = fluid.layers.data(name="click", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) label_cast = fluid.layers.cast(label, dtype='float32') cost = fluid.layers.log_loss(fc, label_cast) try: @@ -72,6 +85,7 @@ def test_pslib_1(self): return from paddle.fluid.incubate.fleet.base.role_maker import MockBarrier + mb = MockBarrier() mb.barrier() mb.barrier_all() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py index 14d5277446bb0..2825a1232fc05 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_4.py @@ -26,11 +26,13 @@ def setUp(self): """Set up, set envs.""" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_pslib_1(self): """Test cases for pslib.""" import threading + try: from paddle.distributed.fleet.utils.http_server import KVHandler from paddle.distributed.fleet.utils.http_server import KVServer @@ -39,7 +41,7 @@ def test_pslib_1(self): print("warning: no fleet, skip test_pslib_4") return - class FakeStream(): + class FakeStream: """ it is a fake stream only for test. """ @@ -113,6 +115,7 @@ def end_headers(self): fake end header, it will do nothing. """ pass + except: print("warning: no KVHandler, skip test_pslib_4") return @@ -130,6 +133,7 @@ def __init__(self): self.delete_kv = {} self.kv_lock = threading.Lock() self.kv = {} + except: print("warning: no KVHTTPServer, skip test_pslib_4") return @@ -147,6 +151,7 @@ def __init__(self): self.listen_thread = None self.size = {} self.size["a"] = 999 + except: print("warning: no KVServer, skip test_pslib_4") return diff --git a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_init.py b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_init.py index dfc88afaf1f9d..f7f2b11acdcb2 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_init.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_rolemaker_init.py @@ -25,7 +25,8 @@ class TestPSCloudRoleMakerCase1(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" def test_paddle_trainers_num(self): # PADDLE_TRAINERS_NUM @@ -40,7 +41,8 @@ class TestPSCloudRoleMakerCase2(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" os.environ["PADDLE_TRAINERS_NUM"] = str(2) def test_training_role(self): @@ -56,7 +58,8 @@ class TestPSCloudRoleMakerCase3(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" os.environ["PADDLE_TRAINERS_NUM"] = str(2) os.environ["TRAINING_ROLE"] = 'TRAINER' @@ -73,7 +76,8 @@ class TestPSCloudRoleMakerCase4(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" os.environ["PADDLE_TRAINERS_NUM"] = str(2) os.environ["TRAINING_ROLE"] = 'PSERVER' @@ -90,7 +94,8 @@ class TestPSCloudRoleMakerCase5(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" os.environ["PADDLE_TRAINERS_NUM"] = str(2) os.environ["TRAINING_ROLE"] = 'PSERVER' os.environ["PADDLE_PORT"] = str(4001) @@ -108,9 +113,11 @@ class TestPSCloudRoleMakerCase6(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" os.environ[ - "PADDLE_HETER_TRAINER_IP_PORT_LIST"] = "127.0.0.1:4003,127.0.0.1:4004" + "PADDLE_HETER_TRAINER_IP_PORT_LIST" + ] = "127.0.0.1:4003,127.0.0.1:4004" os.environ["PADDLE_TRAINERS_NUM"] = str(2) os.environ["TRAINING_ROLE"] = 'HETER_TRAINER' @@ -127,9 +134,11 @@ class TestPSCloudRoleMakerCase7(unittest.TestCase): def setUp(self): os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:4001,127.0.0.1:4002" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:4001,127.0.0.1:4002" os.environ[ - "PADDLE_HETER_TRAINER_IP_PORT_LIST"] = "127.0.0.1:4003,127.0.0.1:4004" + "PADDLE_HETER_TRAINER_IP_PORT_LIST" + ] = "127.0.0.1:4003,127.0.0.1:4004" os.environ["PADDLE_TRAINERS_NUM"] = str(2) os.environ["TRAINING_ROLE"] = 'HETER_TRAINER' os.environ["PADDLE_PORT"] = str(4003) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_runtime.py b/python/paddle/fluid/tests/unittests/test_fleet_runtime.py index 21a0afab0126a..4a415f92ca5a1 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_runtime.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_runtime.py @@ -17,9 +17,9 @@ class TestFleetRuntime(unittest.TestCase): - def test_fleet_runtime_base(self): import paddle.distributed.fleet.runtime + base = paddle.distributed.fleet.runtime.runtime_base.RuntimeBase() base._run_worker() base._init_server() @@ -30,7 +30,9 @@ def test_fleet_runtime_base(self): def test_fleet_collective_runtime(self): import paddle.distributed.fleet.runtime - collective_runtime = paddle.distributed.fleet.runtime.CollectiveRuntime( + + collective_runtime = ( + paddle.distributed.fleet.runtime.CollectiveRuntime() ) collective_runtime._init_worker() collective_runtime._run_worker() @@ -42,20 +44,26 @@ def test_fleet_collective_runtime(self): def test_fleet_ps_runtime(self): ps_runtime = paddle.distributed.fleet.runtime.ParameterServerRuntime() - self.assertRaises(Exception, ps_runtime._get_optimizer_status, - "test_op", None) + self.assertRaises( + Exception, ps_runtime._get_optimizer_status, "test_op", None + ) reshaped_names, origin_names = ps_runtime._get_optimizer_status( - "adam", "param") + "adam", "param" + ) self.assertTrue( - len(reshaped_names) == 2 and reshaped_names[0] == 'param_moment1_0' - and reshaped_names[1] == 'param_moment2_0') + len(reshaped_names) == 2 + and reshaped_names[0] == 'param_moment1_0' + and reshaped_names[1] == 'param_moment2_0' + ) self.assertTrue( len(origin_names) == 2 and origin_names[0] == 'param_beta1_pow_acc_0' - and origin_names[1] == 'param_beta2_pow_acc_0') + and origin_names[1] == 'param_beta2_pow_acc_0' + ) reshaped_names, origin_names = ps_runtime._get_optimizer_status( - "sgd", "param") + "sgd", "param" + ) self.assertTrue(len(reshaped_names) == 0 and len(origin_names) == 0) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py b/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py index 09cc08dc327c2..d1f4df6532800 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_unitaccessor.py @@ -26,7 +26,8 @@ def setUp(self): """Set up, set envs.""" os.environ["PADDLE_TRAINERS_NUM"] = "2" os.environ[ - "PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36001,127.0.0.2:36001" + "PADDLE_PSERVERS_IP_PORT_LIST" + ] = "127.0.0.1:36001,127.0.0.2:36001" def test_pslib_1(self): """Test cases for pslib.""" @@ -41,22 +42,36 @@ def test_pslib_1(self): os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = "127.0.0.1:36002" os.environ["PADDLE_TRAINER_ID"] = "0" role_maker = GeneralRoleMaker() - #role_maker.generate_role() + # role_maker.generate_role() place = fluid.CPUPlace() exe = fluid.Executor(place) - #fleet.init(role_maker) + # fleet.init(role_maker) train_program = fluid.Program() startup_program = fluid.Program() scope = fluid.Scope() with fluid.program_guard(train_program, startup_program): - show = fluid.layers.data(name="show", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) - emb = fluid.layers.embedding(input=show, size=[1, 1], \ - is_sparse=True, is_distributed=True, \ - param_attr=fluid.ParamAttr(name="embedding")) + show = fluid.layers.data( + name="show", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) + emb = fluid.layers.embedding( + input=show, + size=[1, 1], + is_sparse=True, + is_distributed=True, + param_attr=fluid.ParamAttr(name="embedding"), + ) fc = fluid.layers.fc(input=emb, size=1, act=None) - label = fluid.layers.data(name="click", shape=[-1, 1], \ - dtype="int64", lod_level=1, append_batch_size=False) + label = fluid.layers.data( + name="click", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False, + ) label_cast = fluid.layers.cast(label, dtype='float32') cost = fluid.layers.log_loss(fc, label_cast) diff --git a/python/paddle/fluid/tests/unittests/test_fleet_util.py b/python/paddle/fluid/tests/unittests/test_fleet_util.py index 1b6f2cf855277..ff12ef64b5f61 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_util.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_util.py @@ -31,6 +31,7 @@ class TestFleetUtil(unittest.TestCase): def test_util_base(self): import paddle.distributed.fleet as fleet + util = fleet.UtilBase() strategy = fleet.DistributedStrategy() util._set_strategy(strategy) @@ -39,6 +40,7 @@ def test_util_base(self): def test_util_factory(self): import paddle.distributed.fleet as fleet + factory = fleet.base.util_factory.UtilFactory() strategy = fleet.DistributedStrategy() role_maker = None # should be fleet.PaddleCloudRoleMaker() @@ -53,6 +55,7 @@ def test_util_factory(self): def test_get_util(self): import paddle.distributed.fleet as fleet import paddle.distributed.fleet.base.role_maker as role_maker + role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) self.assertNotEqual(fleet.util, None) @@ -61,7 +64,6 @@ def test_set_user_defined_util(self): import paddle.distributed.fleet as fleet class UserDefinedUtil(fleet.UtilBase): - def __init__(self): super(UserDefinedUtil, self).__init__() @@ -69,6 +71,7 @@ def get_user_id(self): return 10 import paddle.distributed.fleet.base.role_maker as role_maker + role = role_maker.PaddleCloudRoleMaker(is_collective=True) fleet.init(role) my_util = UserDefinedUtil() @@ -87,8 +90,9 @@ def test_fs(self): fleet.util._set_file_system(fs) def download_files(self): - path = download(self.proto_data_url, self.module_name, - self.proto_data_md5) + path = download( + self.proto_data_url, self.module_name, self.proto_data_md5 + ) print('data is downloaded at ' + path) tar = tarfile.open(path) unzip_folder = tempfile.mkdtemp() @@ -97,6 +101,7 @@ def download_files(self): def test_get_file_shard(self): import paddle.distributed.fleet as fleet + self.assertRaises(Exception, fleet.util.get_file_shard, "files") role = role_maker.UserDefinedRoleMaker( @@ -105,7 +110,8 @@ def test_get_file_shard(self): current_id=0, role=role_maker.Role.WORKER, worker_endpoints=["127.0.0.1:6003", "127.0.0.1:6004"], - server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"]) + server_endpoints=["127.0.0.1:6001", "127.0.0.1:6002"], + ) fleet.init(role) files = fleet.util.get_file_shard(["1", "2", "3"]) @@ -113,21 +119,27 @@ def test_get_file_shard(self): def test_program_type_trans(self): import paddle.distributed.fleet as fleet + data_dir = self.download_files() program_dir = os.path.join(data_dir, self.pruned_dir) text_program = "pruned_main_program.pbtxt" binary_program = "pruned_main_program.bin" - text_to_binary = fleet.util._program_type_trans(program_dir, - text_program, True) - binary_to_text = fleet.util._program_type_trans(program_dir, - binary_program, False) + text_to_binary = fleet.util._program_type_trans( + program_dir, text_program, True + ) + binary_to_text = fleet.util._program_type_trans( + program_dir, binary_program, False + ) self.assertTrue( - os.path.exists(os.path.join(program_dir, text_to_binary))) + os.path.exists(os.path.join(program_dir, text_to_binary)) + ) self.assertTrue( - os.path.exists(os.path.join(program_dir, binary_to_text))) + os.path.exists(os.path.join(program_dir, binary_to_text)) + ) def test_prams_check(self): import paddle.distributed.fleet as fleet + data_dir = self.download_files() class config: @@ -139,7 +151,7 @@ class config: feed_config.feeded_vars_types = [np.float32, np.float32] feed_config.feeded_vars_filelist = [ os.path.join(data_dir, os.path.join(self.pruned_dir, "concat_1")), - os.path.join(data_dir, os.path.join(self.pruned_dir, "concat_2")) + os.path.join(data_dir, os.path.join(self.pruned_dir, "concat_2")), ] fetch_config = config() @@ -155,7 +167,9 @@ class config: conf.save_params_filename = None # test saved var's shape - conf.dump_program_filename = "pruned_main_program.save_var_shape_not_match" + conf.dump_program_filename = ( + "pruned_main_program.save_var_shape_not_match" + ) self.assertRaises(Exception, fleet.util._params_check) @@ -164,10 +178,13 @@ class config: results = fleet.util._params_check(conf) self.assertTrue(len(results) == 1) np.testing.assert_array_almost_equal( - results[0], np.array([[3.0590223e-07]], dtype=np.float32)) + results[0], np.array([[3.0590223e-07]], dtype=np.float32) + ) # test feed_var's shape - conf.dump_program_filename = "pruned_main_program.feed_var_shape_not_match" + conf.dump_program_filename = ( + "pruned_main_program.feed_var_shape_not_match" + ) self.assertRaises(Exception, fleet.util._params_check) # test correct case with feed_vars_filelist @@ -175,7 +192,8 @@ class config: results = fleet.util._params_check(conf) self.assertTrue(len(results) == 1) np.testing.assert_array_almost_equal( - results[0], np.array([[3.0590223e-07]], dtype=np.float32)) + results[0], np.array([[3.0590223e-07]], dtype=np.float32) + ) # test correct case without feed_vars_filelist conf.feed_config.feeded_vars_filelist = None @@ -189,6 +207,7 @@ class config: def test_proto_check(self): import paddle.distributed.fleet as fleet + data_dir = self.download_files() class config: @@ -196,14 +215,17 @@ class config: conf = config() conf.train_prog_path = os.path.join( - data_dir, os.path.join(self.train_dir, "join_main_program.pbtxt")) + data_dir, os.path.join(self.train_dir, "join_main_program.pbtxt") + ) conf.is_text_train_program = True # test not match conf.pruned_prog_path = os.path.join( data_dir, - os.path.join(self.pruned_dir, - "pruned_main_program.save_var_shape_not_match")) + os.path.join( + self.pruned_dir, "pruned_main_program.save_var_shape_not_match" + ), + ) conf.is_text_pruned_program = True conf.draw = False res = fleet.util._proto_check(conf) @@ -211,8 +233,8 @@ class config: # test match conf.pruned_prog_path = os.path.join( - data_dir, os.path.join(self.pruned_dir, - "pruned_main_program.pbtxt")) + data_dir, os.path.join(self.pruned_dir, "pruned_main_program.pbtxt") + ) if sys.platform == 'win32' or sys.platform == 'sys.platform': conf.draw = False else: @@ -223,13 +245,15 @@ class config: def test_visualize(self): import paddle.distributed.fleet as fleet + if sys.platform == 'win32' or sys.platform == 'sys.platform': pass else: data_dir = self.download_files() program_path = os.path.join( - data_dir, os.path.join(self.train_dir, - "join_main_program.pbtxt")) + data_dir, + os.path.join(self.train_dir, "join_main_program.pbtxt"), + ) is_text = True program = fleet.util._load_program(program_path, is_text) output_dir = os.path.join(data_dir, self.train_dir) @@ -237,10 +261,14 @@ def test_visualize(self): fleet.util._visualize_graphviz(program, output_dir, output_filename) self.assertTrue( os.path.exists( - os.path.join(output_dir, output_filename + ".dot"))) + os.path.join(output_dir, output_filename + ".dot") + ) + ) self.assertTrue( os.path.exists( - os.path.join(output_dir, output_filename + ".pdf"))) + os.path.join(output_dir, output_filename + ".pdf") + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_flip.py b/python/paddle/fluid/tests/unittests/test_flip.py index ddf3534ad336c..f5943db87e0c7 100644 --- a/python/paddle/fluid/tests/unittests/test_flip.py +++ b/python/paddle/fluid/tests/unittests/test_flip.py @@ -41,13 +41,15 @@ def test_static_graph(self): exe = fluid.Executor(place) exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[3, 2, 1], [6, 5, 4]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='flip output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='flip output is wrong, out =' + str(out_np), + ) def test_dygraph(self): img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) @@ -60,11 +62,11 @@ def test_dygraph(self): self.assertTrue( (ret.numpy() == out_ref).all(), - msg='flip output is wrong, out =' + str(ret.numpy())) + msg='flip output is wrong, out =' + str(ret.numpy()), + ) class TestFlipOp(OpTest): - def setUp(self): self.op_type = 'flip' self.python_api = paddle.tensor.flip @@ -96,49 +98,42 @@ def calc_ref_res(self): class TestFlipOpAxis1(TestFlipOp): - def init_test_case(self): self.in_shape = (2, 4, 4) self.axis = [0] class TestFlipOpAxis2(TestFlipOp): - def init_test_case(self): self.in_shape = (4, 4, 6, 3) self.axis = [0, 2] class TestFlipOpAxis3(TestFlipOp): - def init_test_case(self): self.in_shape = (4, 3, 1) self.axis = [0, 1, 2] class TestFlipOpAxis4(TestFlipOp): - def init_test_case(self): self.in_shape = (6, 4, 2, 2) self.axis = [0, 1, 2, 3] class TestFlipOpEmptyAxis(TestFlipOp): - def init_test_case(self): self.in_shape = (6, 4, 2, 2) self.axis = [] class TestFlipOpNegAxis(TestFlipOp): - def init_test_case(self): self.in_shape = (6, 4, 2, 2) self.axis = [-1] class TestFlipDoubleGradCheck(unittest.TestCase): - def flip_wrapper(self, x): return paddle.flip(x[0], [0, 1]) @@ -153,17 +148,13 @@ def func(self, place): out = paddle.flip(data, [0, 1]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.flip_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.flip_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -175,7 +166,6 @@ def test_grad(self): class TestFlipTripleGradCheck(unittest.TestCase): - def flip_wrapper(self, x): return paddle.flip(x[0], [0, 1]) @@ -190,17 +180,13 @@ def func(self, place): out = paddle.flip(data, [0, 1]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.flip_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.flip_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_fmax_op.py b/python/paddle/fluid/tests/unittests/test_fmax_op.py index 16e462754790e..986417fede626 100644 --- a/python/paddle/fluid/tests/unittests/test_fmax_op.py +++ b/python/paddle/fluid/tests/unittests/test_fmax_op.py @@ -44,56 +44,56 @@ def setUp(self): def test_static_api(self): """test_static_api""" paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_y = paddle.static.data("y", shape=[10, 15], dtype="float32") result_fmax = paddle.fmax(data_x, data_y) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "y": self.input_y - }, - fetch_list=[result_fmax]) + (res,) = exe.run( + feed={"x": self.input_x, "y": self.input_y}, + fetch_list=[result_fmax], + ) np.testing.assert_allclose(res, self.np_expected1, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_z = paddle.static.data("z", shape=[15], dtype="float32") result_fmax = paddle.fmax(data_x, data_z) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "z": self.input_z - }, - fetch_list=[result_fmax]) + (res,) = exe.run( + feed={"x": self.input_x, "z": self.input_z}, + fetch_list=[result_fmax], + ) np.testing.assert_allclose(res, self.np_expected2, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_a = paddle.static.data("a", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_fmax = paddle.fmax(data_a, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "a": self.input_a, - "c": self.input_c - }, - fetch_list=[result_fmax]) + (res,) = exe.run( + feed={"a": self.input_a, "c": self.input_c}, + fetch_list=[result_fmax], + ) np.testing.assert_allclose(res, self.np_expected3, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_b = paddle.static.data("b", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_fmax = paddle.fmax(data_b, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "b": self.input_b, - "c": self.input_c - }, - fetch_list=[result_fmax]) + (res,) = exe.run( + feed={"b": self.input_b, "c": self.input_c}, + fetch_list=[result_fmax], + ) np.testing.assert_allclose(res, self.np_expected4, rtol=1e-05) def test_dynamic_api(self): @@ -151,19 +151,23 @@ def test_check_grad_normal(self): def test_check_grad_ingore_x(self): """test_check_grad_ingore_x""" - self.check_grad(['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X"), - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + max_relative_error=0.005, + no_grad_set=set("X"), + check_eager=True, + ) def test_check_grad_ingore_y(self): """test_check_grad_ingore_y""" - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y'), - check_eager=True) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.005, + no_grad_set=set('Y'), + check_eager=True, + ) class TestElementwiseFmax2Op(OpTest): @@ -194,19 +198,23 @@ def test_check_grad_normal(self): def test_check_grad_ingore_x(self): """test_check_grad_ingore_x""" - self.check_grad(['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X"), - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + max_relative_error=0.005, + no_grad_set=set("X"), + check_eager=True, + ) def test_check_grad_ingore_y(self): """test_check_grad_ingore_y""" - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y'), - check_eager=True) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.005, + no_grad_set=set('Y'), + check_eager=True, + ) class TestElementwiseFmax3Op(OpTest): diff --git a/python/paddle/fluid/tests/unittests/test_fmin_op.py b/python/paddle/fluid/tests/unittests/test_fmin_op.py index daad19a40593d..e1a9cf7a912d7 100644 --- a/python/paddle/fluid/tests/unittests/test_fmin_op.py +++ b/python/paddle/fluid/tests/unittests/test_fmin_op.py @@ -46,56 +46,56 @@ def setUp(self): def test_static_api(self): """test_static_api""" paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_y = paddle.static.data("y", shape=[10, 15], dtype="float32") result_fmin = paddle.fmin(data_x, data_y) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "y": self.input_y - }, - fetch_list=[result_fmin]) + (res,) = exe.run( + feed={"x": self.input_x, "y": self.input_y}, + fetch_list=[result_fmin], + ) np.testing.assert_allclose(res, self.np_expected1, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_z = paddle.static.data("z", shape=[15], dtype="float32") result_fmin = paddle.fmin(data_x, data_z) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "z": self.input_z - }, - fetch_list=[result_fmin]) + (res,) = exe.run( + feed={"x": self.input_x, "z": self.input_z}, + fetch_list=[result_fmin], + ) np.testing.assert_allclose(res, self.np_expected2, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_a = paddle.static.data("a", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_fmin = paddle.fmin(data_a, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "a": self.input_a, - "c": self.input_c - }, - fetch_list=[result_fmin]) + (res,) = exe.run( + feed={"a": self.input_a, "c": self.input_c}, + fetch_list=[result_fmin], + ) np.testing.assert_allclose(res, self.np_expected3, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_b = paddle.static.data("b", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_fmin = paddle.fmin(data_b, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "b": self.input_b, - "c": self.input_c - }, - fetch_list=[result_fmin]) + (res,) = exe.run( + feed={"b": self.input_b, "c": self.input_c}, + fetch_list=[result_fmin], + ) np.testing.assert_allclose(res, self.np_expected4, rtol=1e-05) def test_dynamic_api(self): @@ -153,19 +153,23 @@ def test_check_grad_normal(self): def test_check_grad_ingore_x(self): """test_check_grad_ingore_x""" - self.check_grad(['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X"), - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + max_relative_error=0.005, + no_grad_set=set("X"), + check_eager=True, + ) def test_check_grad_ingore_y(self): """test_check_grad_ingore_y""" - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y'), - check_eager=True) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.005, + no_grad_set=set('Y'), + check_eager=True, + ) class TestElementwiseFmin2Op(OpTest): @@ -196,19 +200,23 @@ def test_check_grad_normal(self): def test_check_grad_ingore_x(self): """test_check_grad_ingore_x""" - self.check_grad(['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X"), - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + max_relative_error=0.005, + no_grad_set=set("X"), + check_eager=True, + ) def test_check_grad_ingore_y(self): """test_check_grad_ingore_y""" - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y'), - check_eager=True) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.005, + no_grad_set=set('Y'), + check_eager=True, + ) class TestElementwiseFmin3Op(OpTest): diff --git a/python/paddle/fluid/tests/unittests/test_fold_op.py b/python/paddle/fluid/tests/unittests/test_fold_op.py index 62833173909e3..b0c9a757585e6 100644 --- a/python/paddle/fluid/tests/unittests/test_fold_op.py +++ b/python/paddle/fluid/tests/unittests/test_fold_op.py @@ -42,34 +42,64 @@ def init_data(self): def calc_fold(self): output_shape = [0] * 4 output_shape[0] = self.batch_size - output_shape[1] = int(self.input_channels / - (self.kernel_sizes[0] * self.kernel_sizes[1])) + output_shape[1] = int( + self.input_channels / (self.kernel_sizes[0] * self.kernel_sizes[1]) + ) output_shape[2] = self.output_sizes[0] output_shape[3] = self.output_sizes[1] dkernel_h = self.dilations[0] * (self.kernel_sizes[0] - 1) + 1 dkernel_w = self.dilations[1] * (self.kernel_sizes[1] - 1) + 1 - col_height = int((self.output_sizes[0] + self.paddings[0] + - self.paddings[2] - dkernel_h) / self.strides[0]) + 1 - col_width = int((self.output_sizes[1] + self.paddings[1] + - self.paddings[3] - dkernel_w) / self.strides[1]) + 1 + col_height = ( + int( + ( + self.output_sizes[0] + + self.paddings[0] + + self.paddings[2] + - dkernel_h + ) + / self.strides[0] + ) + + 1 + ) + col_width = ( + int( + ( + self.output_sizes[1] + + self.paddings[1] + + self.paddings[3] + - dkernel_w + ) + / self.strides[1] + ) + + 1 + ) output = np.zeros(output_shape).astype(np.float64) ############ calculate output ############## for b in range(output_shape[0]): for c in range(self.input_channels): w_offset = int(c % self.kernel_sizes[1]) h_offset = int( - (c / self.kernel_sizes[1]) % self.kernel_sizes[0]) + (c / self.kernel_sizes[1]) % self.kernel_sizes[0] + ) c_out = int(c / self.kernel_sizes[0] / self.kernel_sizes[1]) for h in range(col_height): - h_out = int(h * self.strides[0] - self.paddings[0] + - h_offset * self.dilations[0]) + h_out = int( + h * self.strides[0] + - self.paddings[0] + + h_offset * self.dilations[0] + ) for w in range(col_width): - w_out = int(w * self.strides[1] - self.paddings[1] + - w_offset * self.dilations[1]) + w_out = int( + w * self.strides[1] + - self.paddings[1] + + w_offset * self.dilations[1] + ) if (h_out >= 0 and h_out < self.output_sizes[0]) and ( - w_out >= 0 and w_out < self.output_sizes[1]): - output[b, c_out, h_out, - w_out] += self.x[b, c, w + col_width * h] + w_out >= 0 and w_out < self.output_sizes[1] + ): + output[b, c_out, h_out, w_out] += self.x[ + b, c, w + col_width * h + ] self.outputs = output @@ -82,7 +112,7 @@ def set_data(self): 'paddings': self.paddings, 'dilations': self.dilations, 'strides': self.strides, - 'output_sizes': self.output_sizes + 'output_sizes': self.output_sizes, } self.outputs = {'Y': self.outputs} @@ -100,7 +130,7 @@ def test_check_grad(self): class TestFoldAPI(TestFoldOp): - #This is for test on paddle.nn.Fold + # This is for test on paddle.nn.Fold def setUp(self): self.op_type = 'fold' @@ -117,19 +147,19 @@ def test_api(self): m = paddle.nn.Fold(**self.attrs) m.eval() result = m(input) - np.testing.assert_allclose(result.numpy(), - self.outputs['Y'], - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), self.outputs['Y'], rtol=1e-05 + ) def test_info(self): str(paddle.nn.Fold(**self.attrs)) class TestFoldOpError(unittest.TestCase): - def test_errors(self): from paddle.nn.functional import fold from paddle.fluid.framework import Program, program_guard + with program_guard(Program(), Program()): def test_input_shape(): @@ -145,59 +175,67 @@ def test_kernel_shape(): def test_padding_shape(): # padding_size must be 2 or 4 x = paddle.randn(shape=[2, 6, 6], dtype="float32") - out = fold(x, - output_sizes=[2, 3], - kernel_sizes=[2, 2], - paddings=[2, 2, 3]) + out = fold( + x, + output_sizes=[2, 3], + kernel_sizes=[2, 2], + paddings=[2, 2, 3], + ) def test_dilations_shape(): # dialtions_size must be 2 x = paddle.randn(shape=[2, 6, 6], dtype="float32") - out = fold(x, - output_sizes=[2, 3], - kernel_sizes=[2, 2], - dilations=[2, 2, 3]) + out = fold( + x, + output_sizes=[2, 3], + kernel_sizes=[2, 2], + dilations=[2, 2, 3], + ) def test_strides_shape(): # strids_size must be 2 x = paddle.randn(shape=[2, 6, 6], dtype="float32") - out = fold(x, - output_sizes=[2, 3], - kernel_sizes=[2, 2], - strides=[2, 2, 3]) + out = fold( + x, + output_sizes=[2, 3], + kernel_sizes=[2, 2], + strides=[2, 2, 3], + ) def test_output_size(): # im_h * im_w must be L x = paddle.randn(shape=[2, 6, 6], dtype="float32") - out = fold(x, - output_sizes=[6, 6], - kernel_sizes=[2, 2], - strides=[1, 1]) + out = fold( + x, output_sizes=[6, 6], kernel_sizes=[2, 2], strides=[1, 1] + ) def test_output_size_2(): # out_size must GT 1 x = paddle.randn(shape=[2, 6, 6], dtype="float32") - out = fold(x, - output_sizes=[0.1, 0.2], - kernel_sizes=[2, 2], - strides=[1, 1]) + out = fold( + x, + output_sizes=[0.1, 0.2], + kernel_sizes=[2, 2], + strides=[1, 1], + ) def test_block_h_w(): # test_block_h_w GT 0 x = paddle.randn(shape=[2, 1, 1], dtype="float32") - out = fold(x, - output_sizes=[1, 1], - kernel_sizes=[2, 2], - strides=1) + out = fold( + x, output_sizes=[1, 1], kernel_sizes=[2, 2], strides=1 + ) def test_GT_0(): x = paddle.randn(shape=[2, 1, 1], dtype="float32") - out = fold(x, - output_sizes=[0, 0], - kernel_sizes=[0, 0], - dilations=0, - paddings=[0, 0], - strides=0) + out = fold( + x, + output_sizes=[0, 0], + kernel_sizes=[0, 0], + dilations=0, + paddings=[0, 0], + strides=0, + ) self.assertRaises(AssertionError, test_input_shape) self.assertRaises(AssertionError, test_kernel_shape) diff --git a/python/paddle/fluid/tests/unittests/test_frac_api.py b/python/paddle/fluid/tests/unittests/test_frac_api.py index c3f826bed0e99..b455dbbdb4ddc 100644 --- a/python/paddle/fluid/tests/unittests/test_frac_api.py +++ b/python/paddle/fluid/tests/unittests/test_frac_api.py @@ -34,8 +34,11 @@ def set_dtype(self): def setUp(self): self.set_dtype() self.x_np = np.random.uniform(-3, 3, [2, 3]).astype(self.dtype) - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_api_static(self): paddle.enable_static() @@ -46,7 +49,7 @@ def test_api_static(self): if fluid.core.is_compiled_with_cuda(): place = fluid.CUDAPlace(0) exe = fluid.Executor(place) - res, = exe.run(feed={'X': self.x_np}, fetch_list=[out]) + (res,) = exe.run(feed={'X': self.x_np}, fetch_list=[out]) out_ref = ref_frac(self.x_np) np.testing.assert_allclose(out_ref, res, rtol=1e-05) @@ -97,8 +100,11 @@ class TestFracError(unittest.TestCase): def setUp(self): self.x_np = np.random.uniform(-3, 3, [2, 3]).astype('int16') - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_error(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_frame_op.py b/python/paddle/fluid/tests/unittests/test_frame_op.py index 7f38b952665cb..fc0c092e91e63 100644 --- a/python/paddle/fluid/tests/unittests/test_frame_op.py +++ b/python/paddle/fluid/tests/unittests/test_frame_op.py @@ -44,7 +44,6 @@ def frame_from_librosa(x, frame_length, hop_length, axis=-1): class TestFrameOp(OpTest): - def setUp(self): self.op_type = "frame" self.python_api = paddle.signal.frame @@ -57,7 +56,7 @@ def setUp(self): } def initTestCase(self): - input_shape = (150, ) + input_shape = (150,) input_type = 'float64' attrs = { 'frame_length': 50, @@ -78,9 +77,8 @@ def test_check_grad_normal(self): class TestCase1(TestFrameOp): - def initTestCase(self): - input_shape = (150, ) + input_shape = (150,) input_type = 'float64' attrs = { 'frame_length': 50, @@ -91,7 +89,6 @@ def initTestCase(self): class TestCase2(TestFrameOp): - def initTestCase(self): input_shape = (8, 150) input_type = 'float64' @@ -104,7 +101,6 @@ def initTestCase(self): class TestCase3(TestFrameOp): - def initTestCase(self): input_shape = (150, 8) input_type = 'float64' @@ -117,7 +113,6 @@ def initTestCase(self): class TestCase4(TestFrameOp): - def initTestCase(self): input_shape = (4, 2, 150) input_type = 'float64' @@ -130,7 +125,6 @@ def initTestCase(self): class TestCase5(TestFrameOp): - def initTestCase(self): input_shape = (150, 4, 2) input_type = 'float64' diff --git a/python/paddle/fluid/tests/unittests/test_framework_debug_str.py b/python/paddle/fluid/tests/unittests/test_framework_debug_str.py index d398b66f8fe55..6a1e72c072ac2 100644 --- a/python/paddle/fluid/tests/unittests/test_framework_debug_str.py +++ b/python/paddle/fluid/tests/unittests/test_framework_debug_str.py @@ -17,7 +17,6 @@ class TestDebugStringFramework(unittest.TestCase): - def test_debug_str(self): p = Program() p.current_block().create_var(name='t', shape=[0, 1]) diff --git a/python/paddle/fluid/tests/unittests/test_frexp_api.py b/python/paddle/fluid/tests/unittests/test_frexp_api.py index a7c4fa6e2f7d8..9604080ffd41e 100644 --- a/python/paddle/fluid/tests/unittests/test_frexp_api.py +++ b/python/paddle/fluid/tests/unittests/test_frexp_api.py @@ -18,13 +18,15 @@ class TestFrexpAPI(unittest.TestCase): - def setUp(self): np.random.seed(1024) self.rtol = 1e-5 self.atol = 1e-8 - self.place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) self.set_input() def set_input(self): @@ -35,8 +37,9 @@ def test_static_api(self): # 开启静态图模式 paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - input_data = paddle.fluid.data('X', self.x_np.shape, - self.x_np.dtype) + input_data = paddle.fluid.data( + 'X', self.x_np.shape, self.x_np.dtype + ) out = paddle.frexp(input_data) # 计算静态图结果 exe = paddle.static.Executor(self.place) @@ -65,7 +68,7 @@ def test_dygraph_api(self): class TestSplitsFloat32Case1(TestFrexpAPI): """ - Test num_or_sections which is an integer and data type is float32. + Test num_or_sections which is an integer and data type is float32. """ def set_input(self): @@ -74,7 +77,7 @@ def set_input(self): class TestSplitsFloat64Case1(TestFrexpAPI): """ - Test num_or_sections which is an integer and data type is float64. + Test num_or_sections which is an integer and data type is float64. """ def set_input(self): @@ -83,7 +86,7 @@ def set_input(self): class TestSplitsFloat64Case2(TestFrexpAPI): """ - Test num_or_sections which is an integer and data type is float64. + Test num_or_sections which is an integer and data type is float64. """ def set_input(self): diff --git a/python/paddle/fluid/tests/unittests/test_fs_interface.py b/python/paddle/fluid/tests/unittests/test_fs_interface.py index e4682e5f731fe..4705e44eba62b 100644 --- a/python/paddle/fluid/tests/unittests/test_fs_interface.py +++ b/python/paddle/fluid/tests/unittests/test_fs_interface.py @@ -19,7 +19,6 @@ class FSTest(unittest.TestCase): - def _test_method(self, func): args = inspect.getfullargspec(func).args diff --git a/python/paddle/fluid/tests/unittests/test_fsp_op.py b/python/paddle/fluid/tests/unittests/test_fsp_op.py index bac40a98d4b17..3a1de0833a6ff 100644 --- a/python/paddle/fluid/tests/unittests/test_fsp_op.py +++ b/python/paddle/fluid/tests/unittests/test_fsp_op.py @@ -27,16 +27,18 @@ def fsp_matrix(a, b): a_t = a.transpose([0, 2, 3, 1]) a_t = a_t.reshape([batch, h * w, a_channel]) b_t = b.transpose([0, 2, 3, 1]).reshape([batch, h * w, b_channel]) - a_r = a_t.repeat(b_channel, - axis=1).reshape([batch, h * w, b_channel, - a_channel]).transpose([0, 1, 3, 2]) - b_r = b_t.repeat(a_channel, - axis=1).reshape([batch, h * w, a_channel, b_channel]) + a_r = ( + a_t.repeat(b_channel, axis=1) + .reshape([batch, h * w, b_channel, a_channel]) + .transpose([0, 1, 3, 2]) + ) + b_r = b_t.repeat(a_channel, axis=1).reshape( + [batch, h * w, a_channel, b_channel] + ) return np.mean(a_r * b_r, axis=1) class TestFSPOp(OpTest): - def setUp(self): self.op_type = "fsp" self.initTestCase() @@ -59,25 +61,24 @@ def test_check_grad_normal(self): class BadInputTest(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): def test_bad_x(): data = fluid.layers.data(name='data', shape=[3, 32, 32]) feature_map_0 = [1, 2, 3] - feature_map_1 = fluid.layers.conv2d(data, - num_filters=2, - filter_size=3) + feature_map_1 = fluid.layers.conv2d( + data, num_filters=2, filter_size=3 + ) loss = fluid.layers.fsp_matrix(feature_map_0, feature_map_1) self.assertRaises(TypeError, test_bad_x) def test_bad_y(): data = fluid.layers.data(name='data', shape=[3, 32, 32]) - feature_map_0 = fluid.layers.conv2d(data, - num_filters=2, - filter_size=3) + feature_map_0 = fluid.layers.conv2d( + data, num_filters=2, filter_size=3 + ) feature_map_1 = [1, 2, 3] loss = fluid.layers.fsp_matrix(feature_map_0, feature_map_1) diff --git a/python/paddle/fluid/tests/unittests/test_ftrl_op.py b/python/paddle/fluid/tests/unittests/test_ftrl_op.py index 7feabc4be600d..e346b4f2d118e 100644 --- a/python/paddle/fluid/tests/unittests/test_ftrl_op.py +++ b/python/paddle/fluid/tests/unittests/test_ftrl_op.py @@ -29,24 +29,38 @@ def ftrl_step(param, grad, rows, sq_accum, lin_accum, lr, l1, l2, lr_power): new_accum = sq_accum_hit + grad * grad if lr_power == -0.5: - lin_accum_updated = lin_accum_hit + grad - ( - (np.sqrt(new_accum) - np.sqrt(sq_accum_hit)) / lr) * param_hit + lin_accum_updated = ( + lin_accum_hit + + grad + - ((np.sqrt(new_accum) - np.sqrt(sq_accum_hit)) / lr) * param_hit + ) else: - lin_accum_updated = lin_accum_hit + grad - ( - (np.power(new_accum, -lr_power) - np.power(sq_accum_hit, -lr_power)) - / lr) * param_hit + lin_accum_updated = ( + lin_accum_hit + + grad + - ( + ( + np.power(new_accum, -lr_power) + - np.power(sq_accum_hit, -lr_power) + ) + / lr + ) + * param_hit + ) x = l1 * np.sign(lin_accum_updated) - lin_accum_updated if lr_power == -0.5: y = (np.sqrt(new_accum) / lr) + (2 * l2) pre_shrink = x / y param_updated = np.where( - np.abs(lin_accum_updated) > l1, pre_shrink, 0.0) + np.abs(lin_accum_updated) > l1, pre_shrink, 0.0 + ) else: y = (np.power(new_accum, -lr_power) / lr) + (2 * l2) pre_shrink = x / y param_updated = np.where( - np.abs(lin_accum_updated) > l1, pre_shrink, 0.0) + np.abs(lin_accum_updated) > l1, pre_shrink, 0.0 + ) sq_accum_updated = sq_accum_hit + grad * grad @@ -63,7 +77,6 @@ def ftrl_step(param, grad, rows, sq_accum, lin_accum, lr, l1, l2, lr_power): class TestFTRLOp(OpTest): - def setUp(self): self.op_type = "ftrl" rows = 102 @@ -81,22 +94,23 @@ def setUp(self): 'SquaredAccumulator': sq_accum, 'LinearAccumulator': linear_accum, 'Grad': g, - 'LearningRate': lr + 'LearningRate': lr, } self.attrs = { 'l1': l1, 'l2': l2, 'lr_power': lr_power, - 'learning_rate': lr + 'learning_rate': lr, } param_out, sq_accum_out, lin_accum_out = ftrl_step( - w, g, range(rows), sq_accum, linear_accum, lr, l1, l2, lr_power) + w, g, range(rows), sq_accum, linear_accum, lr, l1, l2, lr_power + ) self.outputs = { 'ParamOut': param_out, 'SquaredAccumOut': sq_accum_out, - 'LinearAccumOut': lin_accum_out + 'LinearAccumOut': lin_accum_out, } def test_check_output(self): @@ -104,7 +118,6 @@ def test_check_output(self): class TestSparseFTRLOp(unittest.TestCase): - def setUp(self): self.lr_power = -0.5 @@ -150,22 +163,32 @@ def check_with_place(self, place): # calculate ground-truth answer param_out, sq_accum_out, lin_accum_out = ftrl_step( - param_array, grad_array, rows, sq_accum_array, lin_accum_array, lr, - l1, l2, lr_power) + param_array, + grad_array, + rows, + sq_accum_array, + lin_accum_array, + lr, + l1, + l2, + lr_power, + ) # create and run operator - op = Operator("ftrl", - Param='Param', - Grad='Grad', - ParamOut='Param', - SquaredAccumulator='SquaredAccumulator', - SquaredAccumOut='SquaredAccumulator', - LinearAccumulator='LinearAccumulator', - LinearAccumOut='LinearAccumulator', - LearningRate='LearningRate', - l1=l1, - l2=l2, - lr_power=lr_power) + op = Operator( + "ftrl", + Param='Param', + Grad='Grad', + ParamOut='Param', + SquaredAccumulator='SquaredAccumulator', + SquaredAccumOut='SquaredAccumulator', + LinearAccumulator='LinearAccumulator', + LinearAccumOut='LinearAccumulator', + LearningRate='LearningRate', + l1=l1, + l2=l2, + lr_power=lr_power, + ) op.run(scope, place) @@ -176,15 +199,15 @@ def check_with_place(self, place): for i in range(height): for j in range(row_numel): - self.assertAlmostEqual(param_out[i][j], - param_array[i][j], - places=4) - self.assertAlmostEqual(sq_accum_out[i][j], - sq_accum_array[i][j], - places=4) - self.assertAlmostEqual(lin_accum_out[i][j], - lin_accum_array[i][j], - places=4) + self.assertAlmostEqual( + param_out[i][j], param_array[i][j], places=4 + ) + self.assertAlmostEqual( + sq_accum_out[i][j], sq_accum_array[i][j], places=4 + ) + self.assertAlmostEqual( + lin_accum_out[i][j], lin_accum_array[i][j], places=4 + ) def init_kernel(self): pass @@ -198,12 +221,12 @@ def test_sparse_ftrl(self): class TestSparseFTRLOp2(TestSparseFTRLOp): - def init_kernel(self): self.lr_power = -0.6 if __name__ == "__main__": import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_full_like_op.py b/python/paddle/fluid/tests/unittests/test_full_like_op.py index fe565140e70c4..c24ee6791b14a 100644 --- a/python/paddle/fluid/tests/unittests/test_full_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_full_like_op.py @@ -23,16 +23,16 @@ class TestFullOp(unittest.TestCase): - """ Test fill_any_like op(whose API is full_like) for attr out. """ + """Test fill_any_like op(whose API is full_like) for attr out.""" def test_attr_tensor_API(self): startup_program = Program() train_program = Program() with program_guard(train_program, startup_program): fill_value = 2.0 - input = paddle.fluid.data(name='input', - dtype='float32', - shape=[2, 3]) + input = paddle.fluid.data( + name='input', dtype='float32', shape=[2, 3] + ) output = paddle.full_like(input, fill_value) output_dtype = paddle.full_like(input, fill_value, dtype='float32') @@ -44,14 +44,15 @@ def test_attr_tensor_API(self): img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) - self.assertTrue(not (out_np - np.full_like(img, fill_value)).any(), - msg="full_like output is wrong, out = " + - str(out_np)) + self.assertTrue( + not (out_np - np.full_like(img, fill_value)).any(), + msg="full_like output is wrong, out = " + str(out_np), + ) def test_full_like_imperative(self): paddle.disable_static() @@ -73,24 +74,25 @@ def test_full_like_fill_inf(self): class TestFullOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - #for ci coverage + # for ci coverage - input_data = paddle.fluid.data(name='input', - dtype='float32', - shape=[2, 3]) + input_data = paddle.fluid.data( + name='input', dtype='float32', shape=[2, 3] + ) output = paddle.full_like(input_data, 2.0) def test_input_dtype(): paddle.full_like - self.assertRaises(TypeError, - paddle.full_like, - x=input_data, - fill_value=2, - dtype='uint4') + self.assertRaises( + TypeError, + paddle.full_like, + x=input_data, + fill_value=2, + dtype='uint4', + ) class TestFullLikeOp1(OpTest): @@ -107,7 +109,7 @@ def setUp(self): self.outputs = {'Out': out} self.attrs = { 'value': self.fill_value, - 'dtype': convert_np_dtype_to_dtype_(self.dtype) + 'dtype': convert_np_dtype_to_dtype_(self.dtype), } def init_data(self): @@ -120,7 +122,6 @@ def test_check_output(self): class TestFullLikeOp2(TestFullLikeOp1): - def init_data(self): self.fill_value = 1000 self.shape = [1024, 1024] @@ -128,25 +129,26 @@ def init_data(self): class TestFullLikeOp3(TestFullLikeOp1): - def init_data(self): self.fill_value = 8888 self.shape = [5000, 5000] self.dtype = np.int64 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFullLikeOp4(unittest.TestCase): - def test_skip_data_transform(self): paddle.disable_static() with _test_eager_guard(): - x = paddle.to_tensor([1., 2., 3., 4.], - place=paddle.CUDAPinnedPlace()) - out = paddle.full_like(x, 1.) + x = paddle.to_tensor( + [1.0, 2.0, 3.0, 4.0], place=paddle.CUDAPinnedPlace() + ) + out = paddle.full_like(x, 1.0) self.assertTrue( - (out.numpy() == np.ones([4]).astype(np.float32)).all(), True) + (out.numpy() == np.ones([4]).astype(np.float32)).all(), True + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_full_op.py b/python/paddle/fluid/tests/unittests/test_full_op.py index 1f20cfd6c726c..a8c54188e48bb 100644 --- a/python/paddle/fluid/tests/unittests/test_full_op.py +++ b/python/paddle/fluid/tests/unittests/test_full_op.py @@ -23,45 +23,44 @@ # Test python API class TestFullAPI(unittest.TestCase): - def test_api(self): positive_2_int32 = fluid.layers.fill_constant([1], "int32", 2) positive_2_int64 = fluid.layers.fill_constant([1], "int64", 2) - shape_tensor_int32 = fluid.data(name="shape_tensor_int32", - shape=[2], - dtype="int32") + shape_tensor_int32 = fluid.data( + name="shape_tensor_int32", shape=[2], dtype="int32" + ) - shape_tensor_int64 = fluid.data(name="shape_tensor_int64", - shape=[2], - dtype="int64") + shape_tensor_int64 = fluid.data( + name="shape_tensor_int64", shape=[2], dtype="int64" + ) out_1 = paddle.full(shape=[1, 2], dtype="float32", fill_value=1.1) - out_2 = paddle.full(shape=[1, positive_2_int32], - dtype="float32", - fill_value=1.1) + out_2 = paddle.full( + shape=[1, positive_2_int32], dtype="float32", fill_value=1.1 + ) - out_3 = paddle.full(shape=[1, positive_2_int64], - dtype="float32", - fill_value=1.1) + out_3 = paddle.full( + shape=[1, positive_2_int64], dtype="float32", fill_value=1.1 + ) - out_4 = paddle.full(shape=shape_tensor_int32, - dtype="float32", - fill_value=1.2) + out_4 = paddle.full( + shape=shape_tensor_int32, dtype="float32", fill_value=1.2 + ) - out_5 = paddle.full(shape=shape_tensor_int64, - dtype="float32", - fill_value=1.1) + out_5 = paddle.full( + shape=shape_tensor_int64, dtype="float32", fill_value=1.1 + ) - out_6 = paddle.full(shape=shape_tensor_int64, - dtype=np.float32, - fill_value=1.1) + out_6 = paddle.full( + shape=shape_tensor_int64, dtype=np.float32, fill_value=1.1 + ) val = fluid.layers.fill_constant(shape=[1], dtype=np.float32, value=1.1) - out_7 = paddle.full(shape=shape_tensor_int64, - dtype=np.float32, - fill_value=val) + out_7 = paddle.full( + shape=shape_tensor_int64, dtype=np.float32, fill_value=val + ) exe = fluid.Executor(place=fluid.CPUPlace()) res_1, res_2, res_3, res_4, res_5, res_6, res_7 = exe.run( @@ -70,7 +69,8 @@ def test_api(self): "shape_tensor_int32": np.array([1, 2]).astype("int32"), "shape_tensor_int64": np.array([1, 2]).astype("int64"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7], + ) assert np.array_equal(res_1, np.full([1, 2], 1.1, dtype="float32")) assert np.array_equal(res_2, np.full([1, 2], 1.1, dtype="float32")) @@ -85,89 +85,104 @@ def test_api_eager(self): with _test_eager_guard(): positive_2_int32 = fluid.layers.fill_constant([1], "int32", 2) positive_2_int64 = fluid.layers.fill_constant([1], "int64", 2) - positive_4_int64 = fluid.layers.fill_constant([1], "int64", 4, - True) - - out_1 = paddle.full(shape=[1, 2], - dtype="float32", - fill_value=1.1) - - out_2 = paddle.full(shape=[1, positive_2_int32.item()], - dtype="float32", - fill_value=1.1) - - out_3 = paddle.full(shape=[1, positive_2_int64.item()], - dtype="float32", - fill_value=1.1) - - out_4 = paddle.full(shape=[1, 2], - dtype="float32", - fill_value=1.2) - - out_5 = paddle.full(shape=[1, 2], - dtype="float32", - fill_value=1.1) - - out_6 = paddle.full(shape=[1, 2], - dtype=np.float32, - fill_value=1.1) - - val = fluid.layers.fill_constant(shape=[1], - dtype=np.float32, - value=1.1) - out_7 = paddle.full(shape=[1, 2], - dtype=np.float32, - fill_value=val) - - out_8 = paddle.full(shape=positive_2_int32, - dtype="float32", - fill_value=1.1) - - out_9 = paddle.full(shape=[ - positive_2_int32, positive_2_int64, positive_4_int64 - ], - dtype="float32", - fill_value=1.1) + positive_4_int64 = fluid.layers.fill_constant( + [1], "int64", 4, True + ) + + out_1 = paddle.full( + shape=[1, 2], dtype="float32", fill_value=1.1 + ) + + out_2 = paddle.full( + shape=[1, positive_2_int32.item()], + dtype="float32", + fill_value=1.1, + ) + + out_3 = paddle.full( + shape=[1, positive_2_int64.item()], + dtype="float32", + fill_value=1.1, + ) + + out_4 = paddle.full( + shape=[1, 2], dtype="float32", fill_value=1.2 + ) + + out_5 = paddle.full( + shape=[1, 2], dtype="float32", fill_value=1.1 + ) + + out_6 = paddle.full( + shape=[1, 2], dtype=np.float32, fill_value=1.1 + ) + + val = fluid.layers.fill_constant( + shape=[1], dtype=np.float32, value=1.1 + ) + out_7 = paddle.full( + shape=[1, 2], dtype=np.float32, fill_value=val + ) + + out_8 = paddle.full( + shape=positive_2_int32, dtype="float32", fill_value=1.1 + ) + + out_9 = paddle.full( + shape=[ + positive_2_int32, + positive_2_int64, + positive_4_int64, + ], + dtype="float32", + fill_value=1.1, + ) # test for numpy.float64 as fill_value - out_10 = paddle.full_like(out_7, - dtype=np.float32, - fill_value=np.abs(1.1)) - - assert np.array_equal(out_1, - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(out_2, - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(out_3, - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(out_4, - np.full([1, 2], 1.2, dtype="float32")) - assert np.array_equal(out_5, - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(out_6, - np.full([1, 2], 1.1, dtype="float32")) - assert np.array_equal(out_7, - np.full([1, 2], 1.1, dtype="float32")) + out_10 = paddle.full_like( + out_7, dtype=np.float32, fill_value=np.abs(1.1) + ) + + assert np.array_equal( + out_1, np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + out_2, np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + out_3, np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + out_4, np.full([1, 2], 1.2, dtype="float32") + ) + assert np.array_equal( + out_5, np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + out_6, np.full([1, 2], 1.1, dtype="float32") + ) + assert np.array_equal( + out_7, np.full([1, 2], 1.1, dtype="float32") + ) assert np.array_equal(out_8, np.full([2], 1.1, dtype="float32")) - assert np.array_equal(out_9, - np.full([2, 2, 4], 1.1, dtype="float32")) - assert np.array_equal(out_10, - np.full([1, 2], 1.1, dtype="float32")) + assert np.array_equal( + out_9, np.full([2, 2, 4], 1.1, dtype="float32") + ) + assert np.array_equal( + out_10, np.full([1, 2], 1.1, dtype="float32") + ) class TestFullOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - #for ci coverage - self.assertRaises(TypeError, - paddle.full, - shape=[1], - fill_value=5, - dtype='uint4') + # for ci coverage + self.assertRaises( + TypeError, paddle.full, shape=[1], fill_value=5, dtype='uint4' + ) # The argument dtype of full must be one of bool, float16, - #float32, float64, uint8, int16, int32 or int64 + # float32, float64, uint8, int16, int32 or int64 # The argument shape's type of full_op must be list, tuple or Variable. def test_shape_type(): @@ -177,17 +192,17 @@ def test_shape_type(): # The shape dtype of full op must be int32 or int64. def test_shape_tensor_dtype(): - shape = fluid.data(name="shape_tensor", - shape=[2], - dtype="float32") + shape = fluid.data( + name="shape_tensor", shape=[2], dtype="float32" + ) paddle.full(shape=shape, dtype="float32", fill_value=1) self.assertRaises(TypeError, test_shape_tensor_dtype) def test_shape_tensor_list_dtype(): - shape = fluid.data(name="shape_tensor_list", - shape=[1], - dtype="bool") + shape = fluid.data( + name="shape_tensor_list", shape=[1], dtype="bool" + ) paddle.full(shape=[shape, 2], dtype="float32", fill_value=1) self.assertRaises(TypeError, test_shape_tensor_list_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_function_hook.py b/python/paddle/fluid/tests/unittests/test_function_hook.py index b884d62e7d32e..7ba230233a415 100644 --- a/python/paddle/fluid/tests/unittests/test_function_hook.py +++ b/python/paddle/fluid/tests/unittests/test_function_hook.py @@ -21,7 +21,6 @@ class TestCapture: - def __init__(self): self.list = [] @@ -40,7 +39,6 @@ def grad_hook(grad): class TestBakcwardFunctionHookError(unittest.TestCase): - def func_hook(self): input_data = np.ones([4, 4]).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_functional_conv1d.py b/python/paddle/fluid/tests/unittests/test_functional_conv1d.py index 251ac52451d7e..6869c511b9fbe 100644 --- a/python/paddle/fluid/tests/unittests/test_functional_conv1d.py +++ b/python/paddle/fluid/tests/unittests/test_functional_conv1d.py @@ -21,7 +21,6 @@ class TestFunctionalConv1DError(TestCase): - def setUp(self): self.input = [] self.filter = [] @@ -36,16 +35,21 @@ def dygraph_case(self): with dg.guard(): x = dg.to_variable(self.input, dtype=paddle.float32) w = dg.to_variable(self.filter, dtype=paddle.float32) - b = None if self.bias is None else dg.to_variable( - self.bias, dtype=paddle.float32) - y = F.conv1d(x, - w, - b, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + b = ( + None + if self.bias is None + else dg.to_variable(self.bias, dtype=paddle.float32) + ) + y = F.conv1d( + x, + w, + b, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) def test_exception(self): with self.assertRaises(ValueError): @@ -53,7 +57,6 @@ def test_exception(self): class TestFunctionalConv1DErrorCase1(TestFunctionalConv1DError): - def setUp(self): self.input = np.random.randn(1, 3, 3) self.filter = np.random.randn(3, 3, 1) diff --git a/python/paddle/fluid/tests/unittests/test_functional_conv1d_transpose.py b/python/paddle/fluid/tests/unittests/test_functional_conv1d_transpose.py index 98daff216a760..431c940f0b4a5 100644 --- a/python/paddle/fluid/tests/unittests/test_functional_conv1d_transpose.py +++ b/python/paddle/fluid/tests/unittests/test_functional_conv1d_transpose.py @@ -21,7 +21,6 @@ class TestFunctionalConv1DError(TestCase): - def setUp(self): self.input = [] self.filter = [] @@ -36,16 +35,21 @@ def dygraph_case(self): with dg.guard(): x = dg.to_variable(self.input, dtype=paddle.float32) w = dg.to_variable(self.filter, dtype=paddle.float32) - b = None if self.bias is None else dg.to_variable( - self.bias, dtype=paddle.float32) - y = F.conv1d_transpose(x, - w, - b, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + b = ( + None + if self.bias is None + else dg.to_variable(self.bias, dtype=paddle.float32) + ) + y = F.conv1d_transpose( + x, + w, + b, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) def test_exception(self): with self.assertRaises(ValueError): @@ -53,7 +57,6 @@ def test_exception(self): class TestFunctionalConv1DErrorCase1(TestFunctionalConv1DError): - def setUp(self): self.input = np.random.randn(1, 3, 3) self.filter = np.random.randn(3, 3, 1) diff --git a/python/paddle/fluid/tests/unittests/test_functional_conv2d.py b/python/paddle/fluid/tests/unittests/test_functional_conv2d.py index 6c0f526f236cb..1a92a9babb727 100644 --- a/python/paddle/fluid/tests/unittests/test_functional_conv2d.py +++ b/python/paddle/fluid/tests/unittests/test_functional_conv2d.py @@ -41,27 +41,34 @@ def setUp(self): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 2 + filter_shape = (self.filter_shape,) * 2 else: filter_shape = tuple(self.filter_shape) self.weight = np.random.uniform( - -1, 1, (self.out_channels, self.in_channels // self.groups) + - filter_shape).astype(self.dtype) + -1, + 1, + (self.out_channels, self.in_channels // self.groups) + filter_shape, + ).astype(self.dtype) if not self.no_bias: - self.bias = np.random.uniform(-1, 1, (self.out_channels, )).astype( - self.dtype) + self.bias = np.random.uniform(-1, 1, (self.out_channels,)).astype( + self.dtype + ) - self.channel_last = (self.data_format == "NHWC") + self.channel_last = self.data_format == "NHWC" if self.channel_last: - self.input_shape = (self.batch_size, ) + self.spatial_shape + ( - self.in_channels, ) + self.input_shape = ( + (self.batch_size,) + self.spatial_shape + (self.in_channels,) + ) else: - self.input_shape = (self.batch_size, - self.in_channels) + self.spatial_shape + self.input_shape = ( + self.batch_size, + self.in_channels, + ) + self.spatial_shape - self.input = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.input = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) def static_graph_case_1(self): main = fluid.Program() @@ -69,11 +76,17 @@ def static_graph_case_1(self): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = fluid.data("input", (-1, -1, -1, self.in_channels), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1), + dtype=self.dtype, + ) y = fluid.layers.conv2d( x, self.out_channels, @@ -84,12 +97,14 @@ def static_graph_case_1(self): groups=self.groups, param_attr=I.NumpyArrayInitializer(self.weight), bias_attr=False - if self.no_bias else I.NumpyArrayInitializer(self.bias), + if self.no_bias + else I.NumpyArrayInitializer(self.bias), act=self.act, - data_format=self.data_format) + data_format=self.data_format, + ) exe = fluid.Executor(self.place) exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def static_graph_case_2(self): @@ -98,24 +113,32 @@ def static_graph_case_2(self): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = x = fluid.data("input", (-1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight.shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight.shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias.shape, dtype=self.dtype) - y = F.conv2d(x, - weight, - None if self.no_bias else bias, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv2d( + x, + weight, + None if self.no_bias else bias, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) @@ -125,7 +148,7 @@ def static_graph_case_2(self): feed_dict = {"input": self.input, "weight": self.weight} if not self.no_bias: feed_dict["bias"] = self.bias - out, = exe.run(main, feed=feed_dict, fetch_list=[y]) + (out,) = exe.run(main, feed=feed_dict, fetch_list=[y]) return out def dygraph_case(self): @@ -133,14 +156,16 @@ def dygraph_case(self): x = dg.to_variable(self.input) weight = dg.to_variable(self.weight) bias = None if self.no_bias else dg.to_variable(self.bias) - y = F.conv2d(x, - weight, - bias, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv2d( + x, + weight, + bias, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) @@ -160,8 +185,9 @@ def test_identity_cpu(self): self.place = fluid.CPUPlace() self._test_identity() - @unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def test_identity_gpu(self): self.place = fluid.CUDAPlace(0) self._test_identity() @@ -191,12 +217,14 @@ def test_exception(self): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 2 + filter_shape = (self.filter_shape,) * 2 else: filter_shape = tuple(self.filter_shape) - self.weight_shape = (self.out_channels, - self.in_channels // self.groups) + filter_shape - self.bias_shape = (self.out_channels, ) + self.weight_shape = ( + self.out_channels, + self.in_channels // self.groups, + ) + filter_shape + self.bias_shape = (self.out_channels,) def static_graph_case(self): main = fluid.Program() @@ -205,28 +233,35 @@ def static_graph_case(self): with fluid.program_guard(main, start): self.channel_last = self.data_format == "NHWC" if self.channel_last: - x = x = fluid.data("input", (-1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias_shape, dtype=self.dtype) - y = F.conv2d(x, - weight, - None if self.no_bias else bias, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv2d( + x, + weight, + None if self.no_bias else bias, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) class TestFunctionalConv2DCase2(TestFunctionalConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -242,7 +277,6 @@ def setUp(self): class TestFunctionalConv2DCase3(TestFunctionalConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -258,7 +292,6 @@ def setUp(self): class TestFunctionalConv2DCase4(TestFunctionalConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -274,7 +307,6 @@ def setUp(self): class TestFunctionalConv2DCase5(TestFunctionalConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -290,7 +322,6 @@ def setUp(self): class TestFunctionalConv2DCase6(TestFunctionalConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -306,7 +337,6 @@ def setUp(self): class TestFunctionalConv2DCase7(TestFunctionalConv2D): - def setUp(self): self.in_channels = 6 self.out_channels = 8 @@ -322,7 +352,6 @@ def setUp(self): class TestFunctionalConv2DCase8(TestFunctionalConv2D): - def setUp(self): self.in_channels = 6 self.out_channels = 12 @@ -338,7 +367,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase2(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -354,7 +382,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase3(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 4 @@ -370,7 +397,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase4(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 4 self.out_channels = 3 @@ -386,7 +412,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase7(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -402,7 +427,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase8(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -418,7 +442,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase9(TestFunctionalConv2DError): - def setUp(self): self.in_channels = -5 self.out_channels = 5 @@ -434,7 +457,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase10(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 4 @@ -450,7 +472,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase11(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -466,7 +487,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase12(TestCase): - def setUp(self): self.input = np.array([]) self.filter = np.array([]) @@ -485,38 +505,45 @@ def static_graph_case(self): with fluid.unique_name.guard(): with fluid.program_guard(main, start): x = fluid.data("input", self.input.shape, dtype=paddle.float32) - y = fluid.layers.conv2d(x, - self.num_filters, - self.filter_size, - stride=self.stride, - padding=self.padding, - dilation=self.dilation, - groups=self.groups, - param_attr=I.NumpyArrayInitializer( - self.filter), - bias_attr=False if self.bias is None - else I.NumpyArrayInitializer(self.bias), - act=None, - data_format=self.data_format) + y = fluid.layers.conv2d( + x, + self.num_filters, + self.filter_size, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + groups=self.groups, + param_attr=I.NumpyArrayInitializer(self.filter), + bias_attr=False + if self.bias is None + else I.NumpyArrayInitializer(self.bias), + act=None, + data_format=self.data_format, + ) exe = fluid.Executor() exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def dygraph_case(self): with dg.guard(): x = dg.to_variable(self.input, dtype=paddle.float32) w = dg.to_variable(self.filter, dtype=paddle.float32) - b = None if self.bias is None else dg.to_variable( - self.bias, dtype=paddle.float32) - y = F.conv2d(x, - w, - b, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + b = ( + None + if self.bias is None + else dg.to_variable(self.bias, dtype=paddle.float32) + ) + y = F.conv2d( + x, + w, + b, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) def test_dygraph_exception(self): with self.assertRaises(ValueError): @@ -528,7 +555,6 @@ def test_static_exception(self): class TestFunctionalConv2DErrorCase13(TestFunctionalConv2DErrorCase12): - def setUp(self): self.input = np.random.randn(1, 3, 3, 3) self.filter = np.random.randn(3, 3, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/test_functional_conv2d_transpose.py b/python/paddle/fluid/tests/unittests/test_functional_conv2d_transpose.py index dce6a37c6bbb8..d377be24ddf87 100644 --- a/python/paddle/fluid/tests/unittests/test_functional_conv2d_transpose.py +++ b/python/paddle/fluid/tests/unittests/test_functional_conv2d_transpose.py @@ -43,27 +43,34 @@ def setUp(self): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 2 + filter_shape = (self.filter_shape,) * 2 else: filter_shape = tuple(self.filter_shape) self.weight = np.random.uniform( - -1, 1, (self.in_channels, self.out_channels // self.groups) + - filter_shape).astype(self.dtype) + -1, + 1, + (self.in_channels, self.out_channels // self.groups) + filter_shape, + ).astype(self.dtype) if not self.no_bias: - self.bias = np.random.uniform(-1, 1, (self.out_channels, )).astype( - self.dtype) + self.bias = np.random.uniform(-1, 1, (self.out_channels,)).astype( + self.dtype + ) - self.channel_last = (self.data_format == "NHWC") + self.channel_last = self.data_format == "NHWC" if self.channel_last: - self.input_shape = (self.batch_size, ) + self.spatial_shape + ( - self.in_channels, ) + self.input_shape = ( + (self.batch_size,) + self.spatial_shape + (self.in_channels,) + ) else: - self.input_shape = (self.batch_size, - self.in_channels) + self.spatial_shape + self.input_shape = ( + self.batch_size, + self.in_channels, + ) + self.spatial_shape - self.input = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.input = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) def static_graph_case_1(self): main = fluid.Program() @@ -71,11 +78,17 @@ def static_graph_case_1(self): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = fluid.data("input", (-1, -1, -1, self.in_channels), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1), + dtype=self.dtype, + ) y = fluid.layers.conv2d_transpose( x, self.out_channels, @@ -87,11 +100,13 @@ def static_graph_case_1(self): groups=self.groups, param_attr=I.NumpyArrayInitializer(self.weight), bias_attr=False - if self.no_bias else I.NumpyArrayInitializer(self.bias), - data_format=self.data_format) + if self.no_bias + else I.NumpyArrayInitializer(self.bias), + data_format=self.data_format, + ) exe = fluid.Executor(self.place) exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def static_graph_case_2(self): @@ -100,31 +115,39 @@ def static_graph_case_2(self): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = x = fluid.data("input", (-1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight.shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight.shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias.shape, dtype=self.dtype) - y = F.conv2d_transpose(x, - weight, - None if self.no_bias else bias, - output_size=self.output_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv2d_transpose( + x, + weight, + None if self.no_bias else bias, + output_size=self.output_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) exe = fluid.Executor(self.place) exe.run(start) feed_dict = {"input": self.input, "weight": self.weight} if not self.no_bias: feed_dict["bias"] = self.bias - out, = exe.run(main, feed=feed_dict, fetch_list=[y]) + (out,) = exe.run(main, feed=feed_dict, fetch_list=[y]) return out def dygraph_case(self): @@ -132,15 +155,17 @@ def dygraph_case(self): x = dg.to_variable(self.input) weight = dg.to_variable(self.weight) bias = None if self.no_bias else dg.to_variable(self.bias) - y = F.conv2d_transpose(x, - weight, - bias, - output_size=self.output_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv2d_transpose( + x, + weight, + bias, + output_size=self.output_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) out = y.numpy() return out @@ -160,14 +185,16 @@ def test_identity_cpu_check_eager(self): with _test_eager_guard(): self.test_identity_cpu() - @unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def test_identity_gpu(self): self.place = fluid.CUDAPlace(0) self._test_identity() - @unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def test_identity_gpu_check_eager(self): with _test_eager_guard(): self.test_identity_gpu() @@ -198,12 +225,14 @@ def test_exception(self): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 2 + filter_shape = (self.filter_shape,) * 2 else: filter_shape = tuple(self.filter_shape) - self.weight_shape = (self.in_channels, - self.out_channels // self.groups) + filter_shape - self.bias_shape = (self.out_channels, ) + self.weight_shape = ( + self.in_channels, + self.out_channels // self.groups, + ) + filter_shape + self.bias_shape = (self.out_channels,) def static_graph_case(self): main = fluid.Program() @@ -212,29 +241,36 @@ def static_graph_case(self): with fluid.program_guard(main, start): self.channel_last = self.data_format == "NHWC" if self.channel_last: - x = x = fluid.data("input", (-1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias_shape, dtype=self.dtype) - y = F.conv2d_transpose(x, - weight, - None if self.no_bias else bias, - output_size=self.output_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv2d_transpose( + x, + weight, + None if self.no_bias else bias, + output_size=self.output_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) class TestFunctionalConv2DCase2(TestFunctionalConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -248,7 +284,6 @@ def setUp(self): class TestFunctionalConv2DCase3(TestFunctionalConv2D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -262,7 +297,6 @@ def setUp(self): class TestFunctionalConv2DCase4(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -276,7 +310,6 @@ def setUp(self): class TestFunctionalConv2DCase5(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -290,7 +323,6 @@ def setUp(self): class TestFunctionalConv2DCase6(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -304,7 +336,6 @@ def setUp(self): class TestFunctionalConv2DCase7(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 4 @@ -318,7 +349,6 @@ def setUp(self): class TestFunctionalConv2DCase8(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 4 @@ -333,7 +363,6 @@ def setUp(self): class TestFunctionalConv2DCase9(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -347,7 +376,6 @@ def setUp(self): class TestFunctionalConv2DCase10(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -361,7 +389,6 @@ def setUp(self): class TestFunctionalConv2DCase11(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -375,7 +402,6 @@ def setUp(self): class TestFunctionalConv2DCase12(TestFunctionalConv2D): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -389,7 +415,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase2(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -403,7 +428,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase3(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -417,7 +441,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase4(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -431,7 +454,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase5(TestFunctionalConv2DError): - def setUp(self): self.in_channels = -2 self.out_channels = 5 @@ -445,7 +467,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase7(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 4 self.out_channels = 5 @@ -460,7 +481,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase8(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 4 self.out_channels = 5 @@ -474,7 +494,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase9(TestFunctionalConv2DError): - def setUp(self): self.in_channels = 3 self.out_channels = 4 @@ -488,7 +507,6 @@ def setUp(self): class TestFunctionalConv2DErrorCase10(TestCase): - def setUp(self): self.input = np.array([]) self.filter = np.array([]) @@ -507,38 +525,45 @@ def static_graph_case(self): with fluid.unique_name.guard(): with fluid.program_guard(main, start): x = fluid.data("input", self.input.shape, dtype=paddle.float32) - y = fluid.layers.conv2d(x, - self.num_filters, - self.filter_size, - stride=self.stride, - padding=self.padding, - dilation=self.dilation, - groups=self.groups, - param_attr=I.NumpyArrayInitializer( - self.filter), - bias_attr=False if self.bias is None - else I.NumpyArrayInitializer(self.bias), - act=None, - data_format=self.data_format) + y = fluid.layers.conv2d( + x, + self.num_filters, + self.filter_size, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + groups=self.groups, + param_attr=I.NumpyArrayInitializer(self.filter), + bias_attr=False + if self.bias is None + else I.NumpyArrayInitializer(self.bias), + act=None, + data_format=self.data_format, + ) exe = fluid.Executor() exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def dygraph_case(self): with dg.guard(): x = dg.to_variable(self.input, dtype=paddle.float32) w = dg.to_variable(self.filter, dtype=paddle.float32) - b = None if self.bias is None else dg.to_variable( - self.bias, dtype=paddle.float32) - y = F.conv2d_transpose(x, - w, - b, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + b = ( + None + if self.bias is None + else dg.to_variable(self.bias, dtype=paddle.float32) + ) + y = F.conv2d_transpose( + x, + w, + b, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) def test_dygraph_exception(self): with self.assertRaises(ValueError): @@ -554,7 +579,6 @@ def test_static_exception(self): class TestFunctionalConv2DErrorCase11(TestFunctionalConv2DErrorCase10): - def setUp(self): self.input = np.random.randn(1, 3, 3, 3) self.filter = np.random.randn(3, 3, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/test_functional_conv3d.py b/python/paddle/fluid/tests/unittests/test_functional_conv3d.py index 9ecbf2bf46c5d..d2d7074a4b1a3 100644 --- a/python/paddle/fluid/tests/unittests/test_functional_conv3d.py +++ b/python/paddle/fluid/tests/unittests/test_functional_conv3d.py @@ -41,27 +41,34 @@ def setUp(self): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 3 + filter_shape = (self.filter_shape,) * 3 else: filter_shape = tuple(self.filter_shape) self.weight = np.random.uniform( - -1, 1, (self.out_channels, self.in_channels // self.groups) + - filter_shape).astype(self.dtype) + -1, + 1, + (self.out_channels, self.in_channels // self.groups) + filter_shape, + ).astype(self.dtype) if not self.no_bias: - self.bias = np.random.uniform(-1, 1, (self.out_channels, )).astype( - self.dtype) + self.bias = np.random.uniform(-1, 1, (self.out_channels,)).astype( + self.dtype + ) - self.channel_last = (self.data_format == "NDHWC") + self.channel_last = self.data_format == "NDHWC" if self.channel_last: - self.input_shape = (self.batch_size, ) + self.spatial_shape + ( - self.in_channels, ) + self.input_shape = ( + (self.batch_size,) + self.spatial_shape + (self.in_channels,) + ) else: - self.input_shape = (self.batch_size, - self.in_channels) + self.spatial_shape + self.input_shape = ( + self.batch_size, + self.in_channels, + ) + self.spatial_shape - self.input = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.input = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) def static_graph_case_1(self): main = fluid.Program() @@ -69,11 +76,17 @@ def static_graph_case_1(self): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = fluid.data("input", (-1, -1, -1, -1, self.in_channels), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, -1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1, -1), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1, -1), + dtype=self.dtype, + ) y = fluid.layers.conv3d( x, self.out_channels, @@ -84,12 +97,14 @@ def static_graph_case_1(self): groups=self.groups, param_attr=I.NumpyArrayInitializer(self.weight), bias_attr=False - if self.no_bias else I.NumpyArrayInitializer(self.bias), + if self.no_bias + else I.NumpyArrayInitializer(self.bias), act=self.act, - data_format=self.data_format) + data_format=self.data_format, + ) exe = fluid.Executor(self.place) exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def static_graph_case_2(self): @@ -98,25 +113,32 @@ def static_graph_case_2(self): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = x = fluid.data("input", - (-1, -1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight.shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight.shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias.shape, dtype=self.dtype) - y = F.conv3d(x, - weight, - None if self.no_bias else bias, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv3d( + x, + weight, + None if self.no_bias else bias, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) @@ -126,7 +148,7 @@ def static_graph_case_2(self): feed_dict = {"input": self.input, "weight": self.weight} if not self.no_bias: feed_dict["bias"] = self.bias - out, = exe.run(main, feed=feed_dict, fetch_list=[y]) + (out,) = exe.run(main, feed=feed_dict, fetch_list=[y]) return out def dygraph_case(self): @@ -134,14 +156,16 @@ def dygraph_case(self): x = dg.to_variable(self.input) weight = dg.to_variable(self.weight) bias = None if self.no_bias else dg.to_variable(self.bias) - y = F.conv3d(x, - weight, - bias, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv3d( + x, + weight, + bias, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) @@ -161,8 +185,9 @@ def test_identity_cpu(self): self.place = fluid.CPUPlace() self._test_identity() - @unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def test_identity_gpu(self): self.place = fluid.CUDAPlace(0) self._test_identity() @@ -192,12 +217,14 @@ def test_exception(self): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 3 + filter_shape = (self.filter_shape,) * 3 else: filter_shape = tuple(self.filter_shape) - self.weight_shape = (self.out_channels, - self.in_channels // self.groups) + filter_shape - self.bias_shape = (self.out_channels, ) + self.weight_shape = ( + self.out_channels, + self.in_channels // self.groups, + ) + filter_shape + self.bias_shape = (self.out_channels,) def static_graph_case(self): main = fluid.Program() @@ -206,32 +233,38 @@ def static_graph_case(self): with fluid.program_guard(main, start): self.channel_last = self.data_format == "NDHWC" if self.channel_last: - x = x = fluid.data("input", - (-1, -1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias_shape, dtype=self.dtype) - y = F.conv3d(x, - weight, - None if self.no_bias else bias, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv3d( + x, + weight, + None if self.no_bias else bias, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) class TestFunctionalConv3DCase2(TestFunctionalConv3D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -246,7 +279,6 @@ def setUp(self): class TestFunctionalConv3DCase3(TestFunctionalConv3D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -261,7 +293,6 @@ def setUp(self): class TestFunctionalConv3DCase4(TestFunctionalConv3D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -276,7 +307,6 @@ def setUp(self): class TestFunctionalConv3DCase5(TestFunctionalConv3D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -291,7 +321,6 @@ def setUp(self): class TestFunctionalConv3DCase6(TestFunctionalConv3D): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -306,7 +335,6 @@ def setUp(self): class TestFunctionalConv3DCase7(TestFunctionalConv3D): - def setUp(self): self.in_channels = 6 self.out_channels = 8 @@ -321,7 +349,6 @@ def setUp(self): class TestFunctionalConv3DCase8(TestFunctionalConv3D): - def setUp(self): self.in_channels = 6 self.out_channels = 12 @@ -337,7 +364,6 @@ def setUp(self): class TestFunctionalConv3DErrorCase2(TestFunctionalConv3DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -352,7 +378,6 @@ def setUp(self): class TestFunctionalConv3DErrorCase3(TestFunctionalConv3DError): - def setUp(self): self.in_channels = 3 self.out_channels = 4 @@ -367,7 +392,6 @@ def setUp(self): class TestFunctionalConv3DErrorCase4(TestFunctionalConv3DError): - def setUp(self): self.in_channels = 4 self.out_channels = 3 @@ -382,7 +406,6 @@ def setUp(self): class TestFunctionalConv3DErrorCase7(TestFunctionalConv3DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -397,7 +420,6 @@ def setUp(self): class TestFunctionalConv3DErrorCase8(TestFunctionalConv3DError): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -412,7 +434,6 @@ def setUp(self): class TestFunctionalConv3DErrorCase9(TestFunctionalConv3DError): - def setUp(self): self.in_channels = -5 self.out_channels = 5 @@ -427,7 +448,6 @@ def setUp(self): class TestFunctionalConv3DErrorCase10(TestFunctionalConv3DError): - def setUp(self): self.in_channels = 3 self.out_channels = 4 @@ -442,7 +462,6 @@ def setUp(self): class TestFunctionalConv3DErrorCase11(TestCase): - def setUp(self): self.input = np.array([]) self.filter = np.array([]) @@ -461,38 +480,45 @@ def static_graph_case(self): with fluid.unique_name.guard(): with fluid.program_guard(main, start): x = fluid.data("input", self.input.shape, dtype=paddle.float32) - y = fluid.layers.conv3d(x, - self.num_filters, - self.filter_size, - stride=self.stride, - padding=self.padding, - dilation=self.dilation, - groups=self.groups, - param_attr=I.NumpyArrayInitializer( - self.filter), - bias_attr=False if self.bias is None - else I.NumpyArrayInitializer(self.bias), - act=None, - data_format=self.data_format) + y = fluid.layers.conv3d( + x, + self.num_filters, + self.filter_size, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + groups=self.groups, + param_attr=I.NumpyArrayInitializer(self.filter), + bias_attr=False + if self.bias is None + else I.NumpyArrayInitializer(self.bias), + act=None, + data_format=self.data_format, + ) exe = fluid.Executor() exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def dygraph_case(self): with dg.guard(): x = dg.to_variable(self.input, dtype=paddle.float32) w = dg.to_variable(self.filter, dtype=paddle.float32) - b = None if self.bias is None else dg.to_variable( - self.bias, dtype=paddle.float32) - y = F.conv3d(x, - w, - b, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + b = ( + None + if self.bias is None + else dg.to_variable(self.bias, dtype=paddle.float32) + ) + y = F.conv3d( + x, + w, + b, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) def test_dygraph_exception(self): with self.assertRaises(ValueError): @@ -504,7 +530,6 @@ def test_static_exception(self): class TestFunctionalConv3DErrorCase12(TestFunctionalConv3DErrorCase11): - def setUp(self): self.input = np.random.randn(1, 3, 3, 3, 3) self.filter = np.random.randn(3, 3, 1, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/test_functional_conv3d_transpose.py b/python/paddle/fluid/tests/unittests/test_functional_conv3d_transpose.py index 0190779a021c9..d9ee04953592b 100644 --- a/python/paddle/fluid/tests/unittests/test_functional_conv3d_transpose.py +++ b/python/paddle/fluid/tests/unittests/test_functional_conv3d_transpose.py @@ -43,27 +43,34 @@ def setUp(self): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 3 + filter_shape = (self.filter_shape,) * 3 else: filter_shape = tuple(self.filter_shape) self.weight = np.random.uniform( - -1, 1, (self.in_channels, self.out_channels // self.groups) + - filter_shape).astype(self.dtype) + -1, + 1, + (self.in_channels, self.out_channels // self.groups) + filter_shape, + ).astype(self.dtype) if not self.no_bias: - self.bias = np.random.uniform(-1, 1, (self.out_channels, )).astype( - self.dtype) + self.bias = np.random.uniform(-1, 1, (self.out_channels,)).astype( + self.dtype + ) - self.channel_last = (self.data_format == "NDHWC") + self.channel_last = self.data_format == "NDHWC" if self.channel_last: - self.input_shape = (self.batch_size, ) + self.spatial_shape + ( - self.in_channels, ) + self.input_shape = ( + (self.batch_size,) + self.spatial_shape + (self.in_channels,) + ) else: - self.input_shape = (self.batch_size, - self.in_channels) + self.spatial_shape + self.input_shape = ( + self.batch_size, + self.in_channels, + ) + self.spatial_shape - self.input = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.input = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) def static_graph_case_1(self): main = fluid.Program() @@ -71,11 +78,17 @@ def static_graph_case_1(self): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = fluid.data("input", (-1, -1, -1, -1, self.in_channels), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, -1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1, -1), - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1, -1), + dtype=self.dtype, + ) y = fluid.layers.conv3d_transpose( x, self.out_channels, @@ -87,12 +100,14 @@ def static_graph_case_1(self): groups=self.groups, param_attr=I.NumpyArrayInitializer(self.weight), bias_attr=False - if self.no_bias else I.NumpyArrayInitializer(self.bias), + if self.no_bias + else I.NumpyArrayInitializer(self.bias), act=self.act, - data_format=self.data_format) + data_format=self.data_format, + ) exe = fluid.Executor(self.place) exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def static_graph_case_2(self): @@ -101,26 +116,33 @@ def static_graph_case_2(self): with fluid.unique_name.guard(): with fluid.program_guard(main, start): if self.channel_last: - x = x = fluid.data("input", - (-1, -1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight.shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight.shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias.shape, dtype=self.dtype) - y = F.conv3d_transpose(x, - weight, - None if self.no_bias else bias, - output_size=self.output_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv3d_transpose( + x, + weight, + None if self.no_bias else bias, + output_size=self.output_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) exe = fluid.Executor(self.place) @@ -128,7 +150,7 @@ def static_graph_case_2(self): feed_dict = {"input": self.input, "weight": self.weight} if not self.no_bias: feed_dict["bias"] = self.bias - out, = exe.run(main, feed=feed_dict, fetch_list=[y]) + (out,) = exe.run(main, feed=feed_dict, fetch_list=[y]) return out def dygraph_case(self): @@ -136,15 +158,17 @@ def dygraph_case(self): x = dg.to_variable(self.input) weight = dg.to_variable(self.weight) bias = None if self.no_bias else dg.to_variable(self.bias) - y = F.conv3d_transpose(x, - weight, - bias, - output_size=self.output_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv3d_transpose( + x, + weight, + bias, + output_size=self.output_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) out = y.numpy() @@ -166,14 +190,16 @@ def test_identity_cpu_check_eager(self): with _test_eager_guard(): self.test_identity_cpu() - @unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def test_identity_gpu(self): self.place = fluid.CUDAPlace(0) self._test_identity() - @unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def test_identity_gpu_check_eager(self): with _test_eager_guard(): self.test_identity_gpu() @@ -204,12 +230,14 @@ def test_exception(self): def prepare(self): if isinstance(self.filter_shape, int): - filter_shape = (self.filter_shape, ) * 3 + filter_shape = (self.filter_shape,) * 3 else: filter_shape = tuple(self.filter_shape) - self.weight_shape = (self.in_channels, - self.out_channels // self.groups) + filter_shape - self.bias_shape = (self.out_channels, ) + self.weight_shape = ( + self.in_channels, + self.out_channels // self.groups, + ) + filter_shape + self.bias_shape = (self.out_channels,) def static_graph_case(self): main = fluid.Program() @@ -218,32 +246,38 @@ def static_graph_case(self): with fluid.program_guard(main, start): self.channel_last = self.data_format == "NDHWC" if self.channel_last: - x = x = fluid.data("input", - (-1, -1, -1, -1, self.in_channels), - dtype=self.dtype) + x = x = fluid.data( + "input", + (-1, -1, -1, -1, self.in_channels), + dtype=self.dtype, + ) else: - x = fluid.data("input", (-1, self.in_channels, -1, -1, -1), - dtype=self.dtype) - weight = fluid.data("weight", - self.weight_shape, - dtype=self.dtype) + x = fluid.data( + "input", + (-1, self.in_channels, -1, -1, -1), + dtype=self.dtype, + ) + weight = fluid.data( + "weight", self.weight_shape, dtype=self.dtype + ) if not self.no_bias: bias = fluid.data("bias", self.bias_shape, dtype=self.dtype) - y = F.conv3d_transpose(x, - weight, - None if self.no_bias else bias, - output_size=self.output_size, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + y = F.conv3d_transpose( + x, + weight, + None if self.no_bias else bias, + output_size=self.output_size, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) if self.act == 'sigmoid': y = F.sigmoid(y) class TestFunctionalConv3DTransposeCase2(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -258,7 +292,6 @@ def setUp(self): class TestFunctionalConv3DTransposeCase3(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -273,7 +306,6 @@ def setUp(self): class TestFunctionalConv3DTransposeCase4(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -288,7 +320,6 @@ def setUp(self): class TestFunctionalConv3DTransposeCase5(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -303,7 +334,6 @@ def setUp(self): class TestFunctionalConv3DTransposeCase6(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 4 @@ -318,7 +348,6 @@ def setUp(self): class TestFunctionalConv3DTransposeCase7(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 4 @@ -334,7 +363,6 @@ def setUp(self): class TestFunctionalConv3DTransposeCase8(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -349,7 +377,6 @@ def setUp(self): class TestFunctionalConv3DTransposeCase9(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -364,7 +391,6 @@ def setUp(self): class TestFunctionalConv3DTransposeCase10(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -379,7 +405,6 @@ def setUp(self): class TestFunctionalConv3DTransposeCase11(TestFunctionalConv3DTranspose): - def setUp(self): self.in_channels = 4 self.out_channels = 6 @@ -393,9 +418,9 @@ def setUp(self): self.data_format = "NCDHW" -class TestFunctionalConv3DTransposeErrorCase2(TestFunctionalConv3DTransposeError - ): - +class TestFunctionalConv3DTransposeErrorCase2( + TestFunctionalConv3DTransposeError +): def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -409,9 +434,9 @@ def setUp(self): self.data_format = "NDHWC" -class TestFunctionalConv3DTransposeErrorCase3(TestFunctionalConv3DTransposeError - ): - +class TestFunctionalConv3DTransposeErrorCase3( + TestFunctionalConv3DTransposeError +): def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -425,9 +450,9 @@ def setUp(self): self.data_format = "NDHWC" -class TestFunctionalConv3DTransposeErrorCase4(TestFunctionalConv3DTransposeError - ): - +class TestFunctionalConv3DTransposeErrorCase4( + TestFunctionalConv3DTransposeError +): def setUp(self): self.in_channels = 3 self.out_channels = 5 @@ -441,9 +466,9 @@ def setUp(self): self.data_format = "NCDHW" -class TestFunctionalConv3DTransposeErrorCase5(TestFunctionalConv3DTransposeError - ): - +class TestFunctionalConv3DTransposeErrorCase5( + TestFunctionalConv3DTransposeError +): def setUp(self): self.in_channels = -2 self.out_channels = 5 @@ -457,9 +482,9 @@ def setUp(self): self.data_format = "NCDHW" -class TestFunctionalConv3DTransposeErrorCase7(TestFunctionalConv3DTransposeError - ): - +class TestFunctionalConv3DTransposeErrorCase7( + TestFunctionalConv3DTransposeError +): def setUp(self): self.in_channels = 4 self.out_channels = 5 @@ -474,9 +499,9 @@ def setUp(self): self.data_format = "NCDHW" -class TestFunctionalConv3DTransposeErrorCase8(TestFunctionalConv3DTransposeError - ): - +class TestFunctionalConv3DTransposeErrorCase8( + TestFunctionalConv3DTransposeError +): def setUp(self): self.in_channels = 4 self.out_channels = 5 @@ -490,9 +515,9 @@ def setUp(self): self.data_format = "not_valid" -class TestFunctionalConv3DTransposeErrorCase9(TestFunctionalConv3DTransposeError - ): - +class TestFunctionalConv3DTransposeErrorCase9( + TestFunctionalConv3DTransposeError +): def setUp(self): self.in_channels = 3 self.out_channels = 4 @@ -507,7 +532,6 @@ def setUp(self): class TestFunctionalConv3DTransposeErrorCase10(TestCase): - def setUp(self): self.input = np.array([]) self.filter = np.array([]) @@ -535,29 +559,36 @@ def static_graph_case(self): dilation=self.dilation, groups=self.groups, param_attr=I.NumpyArrayInitializer(self.filter), - bias_attr=False if self.bias is None else - I.NumpyArrayInitializer(self.bias), + bias_attr=False + if self.bias is None + else I.NumpyArrayInitializer(self.bias), act=None, - data_format=self.data_format) + data_format=self.data_format, + ) exe = fluid.Executor() exe.run(start) - out, = exe.run(main, feed={"input": self.input}, fetch_list=[y]) + (out,) = exe.run(main, feed={"input": self.input}, fetch_list=[y]) return out def dygraph_case(self): with dg.guard(): x = dg.to_variable(self.input, dtype=paddle.float32) w = dg.to_variable(self.filter, dtype=paddle.float32) - b = None if self.bias is None else dg.to_variable( - self.bias, dtype=paddle.float32) - y = F.conv3d_transpose(x, - w, - b, - padding=self.padding, - stride=self.stride, - dilation=self.dilation, - groups=self.groups, - data_format=self.data_format) + b = ( + None + if self.bias is None + else dg.to_variable(self.bias, dtype=paddle.float32) + ) + y = F.conv3d_transpose( + x, + w, + b, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.groups, + data_format=self.data_format, + ) def test_dygraph_exception(self): with self.assertRaises(ValueError): @@ -573,8 +604,8 @@ def test_static_exception(self): class TestFunctionalConv3DTransposeErrorCase11( - TestFunctionalConv3DTransposeErrorCase10): - + TestFunctionalConv3DTransposeErrorCase10 +): def setUp(self): self.input = np.random.randn(1, 3, 3, 3, 3) self.filter = np.random.randn(3, 3, 1, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py index 67729d6633d0e..9431bcac4e34a 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py @@ -26,18 +26,19 @@ class TestFuseAllReduceOpsBase(TestParallelExecutorBase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) - def compare_fuse_all_reduce_ops(self, - model, - use_device, - init_feed_dict=None, - get_data_from_feeder=None, - optimizer=None, - fuse_all_optimizer_ops=False): + def compare_fuse_all_reduce_ops( + self, + model, + use_device, + init_feed_dict=None, + get_data_from_feeder=None, + optimizer=None, + fuse_all_optimizer_ops=False, + ): if use_device == DeviceType.CUDA and not core.is_compiled_with_cuda(): return if use_device == DeviceType.XPU and not core.is_compiled_with_xpu(): @@ -48,22 +49,32 @@ def compare_fuse_all_reduce_ops(self, img, label = init_feed_dict() feed_dict_data = {"image": img, "label": label} - not_fuse_op_first_loss, not_fuse_op_last_loss, _ = self.check_network_convergence( + ( + not_fuse_op_first_loss, + not_fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, feed_dict=feed_dict_data, get_data_from_feeder=get_data_from_feeder, use_device=use_device, fuse_all_reduce_ops=False, fuse_all_optimizer_ops=fuse_all_optimizer_ops, - optimizer=optimizer) - fuse_op_first_loss, fuse_op_last_loss, _ = self.check_network_convergence( + optimizer=optimizer, + ) + ( + fuse_op_first_loss, + fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, feed_dict=feed_dict_data, get_data_from_feeder=get_data_from_feeder, use_device=use_device, fuse_all_reduce_ops=True, fuse_all_optimizer_ops=fuse_all_optimizer_ops, - optimizer=optimizer) + optimizer=optimizer, + ) for loss in zip(not_fuse_op_first_loss, fuse_op_first_loss): self.assertAlmostEquals(loss[0], loss[1], delta=1e-6) @@ -73,18 +84,20 @@ def compare_fuse_all_reduce_ops(self, def optimizer(self, learning_rate=1e-3): optimizer = fluid.optimizer.SGD( learning_rate=learning_rate, - regularization=fluid.regularizer.L2Decay(1e-3)) + regularization=fluid.regularizer.L2Decay(1e-3), + ) return optimizer class TestFuseAllReduceOps(TestFuseAllReduceOpsBase): - def _decorate_compare_fused_all_reduce(self, model, use_device): - self.compare_fuse_all_reduce_ops(model, - use_device, - init_feed_dict=init_data, - optimizer=self.optimizer, - fuse_all_optimizer_ops=True) + self.compare_fuse_all_reduce_ops( + model, + use_device, + init_feed_dict=init_data, + optimizer=self.optimizer, + fuse_all_optimizer_ops=True, + ) def test_simple_fc_with_fuse_all_reduce(self): self._decorate_compare_fused_all_reduce(simple_fc_net, DeviceType.CUDA) @@ -92,27 +105,29 @@ def test_simple_fc_with_fuse_all_reduce(self): self._decorate_compare_fused_all_reduce(simple_fc_net, DeviceType.CPU) def test_batchnorm_fc_with_fuse_all_reduce(self): - self._decorate_compare_fused_all_reduce(fc_with_batchnorm, - DeviceType.CUDA) + self._decorate_compare_fused_all_reduce( + fc_with_batchnorm, DeviceType.CUDA + ) # TODO(wangxi): xpu batch_norm op only support dim = 4 # self._decorate_compare_fused_all_reduce(fc_with_batchnorm, # DeviceType.XPU) - self._decorate_compare_fused_all_reduce(fc_with_batchnorm, - DeviceType.CPU) + self._decorate_compare_fused_all_reduce( + fc_with_batchnorm, DeviceType.CPU + ) class TestFuseAllReduceOpsAndOptiOps(TestFuseAllReduceOps): - def _decorate_compare_fused_all_reduce(self, model, use_device): - self.compare_fuse_all_reduce_ops(model, - use_device, - init_feed_dict=init_data, - optimizer=self.optimizer, - fuse_all_optimizer_ops=True) + self.compare_fuse_all_reduce_ops( + model, + use_device, + init_feed_dict=init_data, + optimizer=self.optimizer, + fuse_all_optimizer_ops=True, + ) class TestFuseAllReduceOpsWithSparseGrad(TestFuseAllReduceOpsBase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) @@ -132,7 +147,8 @@ def _decorate_compare_fused_all_reduce(self, model, use_device): model, use_device, get_data_from_feeder=self.get_data_from_feeder, - optimizer=self.optimizer) + optimizer=self.optimizer, + ) def test_simple_bow_net_with_fuse_all_reduce(self): model = partial(bow_net, dict_dim=self.word_dict_len, is_sparse=True) diff --git a/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py index d3a18ad28ca54..5018b7bd5e43b 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py @@ -18,41 +18,47 @@ class TestFuseBatchNormActPass(unittest.TestCase): - def build_program(self, main_program, startup_program, use_cuda, seed=1): with fluid.program_guard(main_program, startup_program): x = fluid.layers.data(name='x', shape=[1, 28, 28], dtype='float32') y = fluid.layers.data(name="y", shape=[1], dtype='int64') - hidden1 = fluid.layers.conv2d(input=x, - filter_size=3, - num_filters=16, - stride=1, - padding=1, - act=None, - bias_attr=False, - data_format='NHWC') + hidden1 = fluid.layers.conv2d( + input=x, + filter_size=3, + num_filters=16, + stride=1, + padding=1, + act=None, + bias_attr=False, + data_format='NHWC', + ) param_attr = fluid.ParamAttr( name='batch_norm_w', - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0), + ) bias_attr = fluid.ParamAttr( name='batch_norm_b', - initializer=fluid.initializer.Constant(value=0.0)) - hidden2 = fluid.layers.batch_norm(input=hidden1, - param_attr=param_attr, - bias_attr=bias_attr, - act='relu', - data_layout='NHWC') + initializer=fluid.initializer.Constant(value=0.0), + ) + hidden2 = fluid.layers.batch_norm( + input=hidden1, + param_attr=param_attr, + bias_attr=bias_attr, + act='relu', + data_layout='NHWC', + ) hidden3 = fluid.layers.fc(input=hidden2, size=32, act='relu') - hidden4 = fluid.layers.batch_norm(input=hidden3, - act='relu', - data_layout='NHWC') + hidden4 = fluid.layers.batch_norm( + input=hidden3, act='relu', data_layout='NHWC' + ) prediction = fluid.layers.fc(input=hidden4, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=y) loss = paddle.mean(loss) sgd = fluid.optimizer.SGD(learning_rate=0.001) if use_cuda: sgd = fluid.contrib.mixed_precision.decorate( - sgd, use_dynamic_loss_scaling=True, init_loss_scaling=128.0) + sgd, use_dynamic_loss_scaling=True, init_loss_scaling=128.0 + ) sgd.minimize(loss) return x, y, loss @@ -71,36 +77,40 @@ def check(self, place, use_cuda): build_strategy = fluid.BuildStrategy() build_strategy.fuse_bn_act_ops = False binary = fluid.CompiledProgram(main_program).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size) + loss_name=loss.name, build_strategy=build_strategy + ) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size + ) loss_vals = [] scope = fluid.Scope() with fluid.scope_guard(scope): exe.run(startup_program) for _ in range(iters): data = next(train_reader()) - loss_v = exe.run(binary, - feed=feeder.feed(data), - fetch_list=[loss]) + loss_v = exe.run( + binary, feed=feeder.feed(data), fetch_list=[loss] + ) loss_vals.append(loss_v[0][0]) # open fused_bn_act_ops build_strategy_fused = fluid.BuildStrategy() build_strategy_fused.fuse_bn_act_ops = True binary_fused = fluid.CompiledProgram(main_program).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy_fused) - train_reader_fused = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size) + loss_name=loss.name, build_strategy=build_strategy_fused + ) + train_reader_fused = paddle.batch( + paddle.dataset.mnist.train(), batch_size=batch_size + ) loss_vals_fused = [] scope_fused = fluid.Scope() with fluid.scope_guard(scope_fused): exe.run(startup_program) for _ in range(iters): data = next(train_reader_fused()) - loss_v = exe.run(binary_fused, - feed=feeder.feed(data), - fetch_list=[loss]) + loss_v = exe.run( + binary_fused, feed=feeder.feed(data), fetch_list=[loss] + ) loss_vals_fused.append(loss_v[0][0]) # check loss diff --git a/python/paddle/fluid/tests/unittests/test_fuse_bn_add_act_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_bn_add_act_pass.py index 67c4879a05ada..47c2a1a5f16cd 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_bn_add_act_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_bn_add_act_pass.py @@ -21,130 +21,151 @@ paddle.enable_static() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "Paddle core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "Paddle core is not compiled with CUDA" +) class TestFusedBnAddActAPI(unittest.TestCase): - def setUp(self): self.conv_param_attr1 = fluid.ParamAttr( name='conv2d_1.weight', initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) + learning_rate=0.001, + ) self.conv_param_attr2 = fluid.ParamAttr( name='conv2d_2.weight', initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) + learning_rate=0.001, + ) self.bn_param_attr1 = fluid.ParamAttr( name='batch_norm_w_1', - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0), + ) self.bn_bias_attr1 = fluid.ParamAttr( name='batch_norm_b_1', - initializer=fluid.initializer.Constant(value=0.0)) + initializer=fluid.initializer.Constant(value=0.0), + ) self.bn_param_attr2 = fluid.ParamAttr( name='batch_norm_w_2', - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0), + ) self.bn_bias_attr2 = fluid.ParamAttr( name='batch_norm_b_2', - initializer=fluid.initializer.Constant(value=0.0)) + initializer=fluid.initializer.Constant(value=0.0), + ) self.fc_param_attr = fluid.ParamAttr( name='fc.weight', - initializer=fluid.initializer.Xavier(uniform=False)) + initializer=fluid.initializer.Xavier(uniform=False), + ) - def build_fused_program(self, - main_program, - startup_program, - use_cuda, - seed=1): + def build_fused_program( + self, main_program, startup_program, use_cuda, seed=1 + ): with fluid.program_guard(main_program, startup_program): x = fluid.layers.data(name='x', shape=[1, 28, 28], dtype='float32') y = fluid.layers.data(name="y", shape=[1], dtype='int64') - conv1_1 = fluid.layers.conv2d(input=x, - filter_size=3, - num_filters=32, - stride=1, - padding=1, - act=None, - param_attr=self.conv_param_attr1, - bias_attr=False, - data_format='NHWC') - conv1_2 = fluid.layers.conv2d(input=x, - filter_size=3, - num_filters=32, - stride=1, - padding=1, - act=None, - param_attr=self.conv_param_attr2, - bias_attr=False, - data_format='NHWC') - bn = fluid.layers.batch_norm(input=conv1_1, - param_attr=self.bn_param_attr1, - bias_attr=self.bn_bias_attr1, - act=None, - data_layout='NHWC') + conv1_1 = fluid.layers.conv2d( + input=x, + filter_size=3, + num_filters=32, + stride=1, + padding=1, + act=None, + param_attr=self.conv_param_attr1, + bias_attr=False, + data_format='NHWC', + ) + conv1_2 = fluid.layers.conv2d( + input=x, + filter_size=3, + num_filters=32, + stride=1, + padding=1, + act=None, + param_attr=self.conv_param_attr2, + bias_attr=False, + data_format='NHWC', + ) + bn = fluid.layers.batch_norm( + input=conv1_1, + param_attr=self.bn_param_attr1, + bias_attr=self.bn_bias_attr1, + act=None, + data_layout='NHWC', + ) fused_bn_add_act = fluid.contrib.layers.fused_bn_add_act( conv1_2, bn, param_attr=self.bn_param_attr2, - bias_attr=self.bn_bias_attr2) - prediction = fluid.layers.fc(input=fused_bn_add_act, - size=10, - act='softmax', - param_attr=self.fc_param_attr) + bias_attr=self.bn_bias_attr2, + ) + prediction = fluid.layers.fc( + input=fused_bn_add_act, + size=10, + act='softmax', + param_attr=self.fc_param_attr, + ) loss = fluid.layers.cross_entropy(input=prediction, label=y) loss = paddle.mean(loss) sgd = fluid.optimizer.SGD(learning_rate=0.001) sgd = fluid.contrib.mixed_precision.decorate( - sgd, use_dynamic_loss_scaling=True, init_loss_scaling=128.0) + sgd, use_dynamic_loss_scaling=True, init_loss_scaling=128.0 + ) sgd.minimize(loss) return x, y, loss - def build_origin_program(self, - main_program, - startup_program, - use_cuda, - seed=1): + def build_origin_program( + self, main_program, startup_program, use_cuda, seed=1 + ): with fluid.program_guard(main_program, startup_program): x = fluid.layers.data(name='x', shape=[1, 28, 28], dtype='float32') y = fluid.layers.data(name="y", shape=[1], dtype='int64') - conv1_1 = fluid.layers.conv2d(input=x, - filter_size=3, - num_filters=32, - stride=1, - padding=1, - act=None, - param_attr=self.conv_param_attr1, - bias_attr=False, - data_format='NHWC') - bn1 = fluid.layers.batch_norm(input=conv1_1, - param_attr=self.bn_param_attr1, - bias_attr=self.bn_bias_attr1, - act=None, - data_layout='NHWC') - conv1_2 = fluid.layers.conv2d(input=conv1_1, - filter_size=1, - num_filters=32, - stride=1, - act=None, - param_attr=self.conv_param_attr2, - bias_attr=False, - data_format='NHWC') - bn2 = fluid.layers.batch_norm(input=conv1_1, - param_attr=self.bn_param_attr2, - bias_attr=self.bn_bias_attr2, - act=None, - data_layout='NHWC') + conv1_1 = fluid.layers.conv2d( + input=x, + filter_size=3, + num_filters=32, + stride=1, + padding=1, + act=None, + param_attr=self.conv_param_attr1, + bias_attr=False, + data_format='NHWC', + ) + bn1 = fluid.layers.batch_norm( + input=conv1_1, + param_attr=self.bn_param_attr1, + bias_attr=self.bn_bias_attr1, + act=None, + data_layout='NHWC', + ) + conv1_2 = fluid.layers.conv2d( + input=conv1_1, + filter_size=1, + num_filters=32, + stride=1, + act=None, + param_attr=self.conv_param_attr2, + bias_attr=False, + data_format='NHWC', + ) + bn2 = fluid.layers.batch_norm( + input=conv1_1, + param_attr=self.bn_param_attr2, + bias_attr=self.bn_bias_attr2, + act=None, + data_layout='NHWC', + ) out = bn1 + bn2 out = fluid.layers.relu(out) - prediction = fluid.layers.fc(input=out, - size=10, - act='softmax', - param_attr=self.fc_param_attr) + prediction = fluid.layers.fc( + input=out, size=10, act='softmax', param_attr=self.fc_param_attr + ) loss = fluid.layers.cross_entropy(input=prediction, label=y) loss = paddle.mean(loss) sgd = fluid.optimizer.SGD(learning_rate=0.001) sgd = fluid.contrib.mixed_precision.decorate( - sgd, use_dynamic_loss_scaling=True, init_loss_scaling=128.0) + sgd, use_dynamic_loss_scaling=True, init_loss_scaling=128.0 + ) sgd.minimize(loss) return loss @@ -158,12 +179,14 @@ def check(self, place, use_cuda): # build_fused_program: turn on fuse_bn_add_act_ops main_program = fluid.Program() startup_program = fluid.Program() - loss = self.build_origin_program(main_program, startup_program, - use_cuda) + loss = self.build_origin_program( + main_program, startup_program, use_cuda + ) build_strategy_fused = fluid.BuildStrategy() build_strategy_fused.fuse_bn_add_act_ops = True binary_fused = fluid.CompiledProgram(main_program).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy_fused) + loss_name=loss.name, build_strategy=build_strategy_fused + ) exe = fluid.Executor(place) loss_vals_fused = [] x_data = [] @@ -176,30 +199,27 @@ def check(self, place, use_cuda): y = np.random.random((batch_size, 1)).astype("int64") x_data.append(x) y_data.append(y) - loss_v = exe.run(binary_fused, - feed={ - "x": x, - "y": y - }, - fetch_list=[loss]) + loss_v = exe.run( + binary_fused, feed={"x": x, "y": y}, fetch_list=[loss] + ) loss_vals_fused.append(loss_v[0][0]) # build_origin_program: turn off fused_bn_act_ops build_strategy = fluid.BuildStrategy() build_strategy.fuse_bn_add_act_ops = False binary = fluid.CompiledProgram(main_program).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy_fused) + loss_name=loss.name, build_strategy=build_strategy_fused + ) loss_vals = [] scope = fluid.Scope() with fluid.scope_guard(scope): exe.run(startup_program) for i in range(iters): - loss_v = exe.run(binary, - feed={ - "x": x_data[i], - "y": y_data[i] - }, - fetch_list=[loss]) + loss_v = exe.run( + binary, + feed={"x": x_data[i], "y": y_data[i]}, + fetch_list=[loss], + ) loss_vals.append(loss_v[0][0]) # check loss @@ -215,9 +235,9 @@ def test_fuse_bn_add_act_API(self): main_program = fluid.Program() startup_program = fluid.Program() place = fluid.CUDAPlace(0) - x, y, loss = self.build_fused_program(main_program, - startup_program, - use_cuda=True) + x, y, loss = self.build_fused_program( + main_program, startup_program, use_cuda=True + ) exe = fluid.Executor(place) scope = fluid.Scope() with fluid.scope_guard(scope): @@ -225,12 +245,9 @@ def test_fuse_bn_add_act_API(self): for _ in range(5): x = np.random.random((4, 1, 28, 28)).astype("float32") y = np.random.random((4, 1)).astype("int64") - loss_v = exe.run(main_program, - feed={ - "x": x, - "y": y - }, - fetch_list=[loss]) + loss_v = exe.run( + main_program, feed={"x": x, "y": y}, fetch_list=[loss] + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py index 15b79bf0a7fd8..bc052ec1cf722 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py @@ -21,7 +21,6 @@ class TestMNIST(TestParallelExecutorBase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) @@ -34,7 +33,8 @@ def _compare_fuse_elewise_add_act_ops(self, model, use_device): def _optimizer(learning_rate=1e-6): optimizer = fluid.optimizer.SGD( learning_rate=learning_rate, - regularization=fluid.regularizer.L2Decay(1e-6)) + regularization=fluid.regularizer.L2Decay(1e-6), + ) return optimizer # NOTE(dzh): @@ -42,28 +42,32 @@ def _optimizer(learning_rate=1e-6): # FIXME (liuwei12) # the new memory optimize strategy will crash this unittest # add enable_inplace=False here to force pass the unittest - not_fuse_op_first_loss, not_fuse_op_last_loss, _ = self.check_network_convergence( + ( + not_fuse_op_first_loss, + not_fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, fuse_elewise_add_act_ops=False, use_ir_memory_optimize=False, enable_inplace=False, - optimizer=_optimizer) - fuse_op_first_loss, fuse_op_last_loss, _ = self.check_network_convergence( + optimizer=_optimizer, + ) + ( + fuse_op_first_loss, + fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, fuse_elewise_add_act_ops=True, use_ir_memory_optimize=False, enable_inplace=False, - optimizer=_optimizer) + optimizer=_optimizer, + ) for loss in zip(not_fuse_op_first_loss, fuse_op_first_loss): self.assertAlmostEquals(loss[0], loss[1], delta=1e-6) @@ -75,13 +79,16 @@ def test_simple_fc_with_fuse_op(self): self._compare_fuse_elewise_add_act_ops(simple_fc_net, DeviceType.CPU) def test_batchnorm_fc_with_fuse_op(self): - self._compare_fuse_elewise_add_act_ops(fc_with_batchnorm, - DeviceType.CUDA) - self._compare_fuse_elewise_add_act_ops(fc_with_batchnorm, - DeviceType.CPU) + self._compare_fuse_elewise_add_act_ops( + fc_with_batchnorm, DeviceType.CUDA + ) + self._compare_fuse_elewise_add_act_ops( + fc_with_batchnorm, DeviceType.CPU + ) if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py index 1d7dc0e70d4aa..3d95b77bae6ce 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py @@ -45,7 +45,6 @@ def verify_node_count(graph, node_name, target_count): class MultiFCLayer(paddle.nn.Layer): - def __init__(self, hidden, Activation): super(MultiFCLayer, self).__init__() self.linear1 = paddle.nn.Linear(hidden, 4 * hidden) @@ -72,10 +71,10 @@ def forward(self, x, matmul_y, ele_y): return output -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueFWDBase(unittest.TestCase): - def setUp(self): self.batch = 64 self.seqlen = 128 @@ -87,29 +86,42 @@ def setUp(self): self.startup_prog = paddle.static.Program() with paddle.static.program_guard(self.main_prog, self.startup_prog): - data = paddle.static.data(name="_data", - shape=[-1, self.seqlen, self.hidden], - dtype='float32') - matmul_y = paddle.static.data(name="_matmul_y", - shape=[1, self.hidden, self.hidden], - dtype='float32') - ele_y = paddle.static.data(name="_ele_y", - shape=[ - self.hidden, - ], - dtype='float32') + data = paddle.static.data( + name="_data", + shape=[-1, self.seqlen, self.hidden], + dtype='float32', + ) + matmul_y = paddle.static.data( + name="_matmul_y", + shape=[1, self.hidden, self.hidden], + dtype='float32', + ) + ele_y = paddle.static.data( + name="_ele_y", + shape=[ + self.hidden, + ], + dtype='float32', + ) multi_layer = MultiFCLayer(self.hidden, self._get_act_type()[0]) with paddle.static.amp.fp16_guard(): out = multi_layer(data, matmul_y, ele_y) self.loss = paddle.mean(out) - self.data_arr = np.random.random( - (self.batch, self.seqlen, self.hidden)).astype("float32") - 0.5 - self.matmul_y_arr = np.random.random( - (1, self.hidden, self.hidden)).astype("float32") - 0.5 - self.ele_y_arr = np.random.random( - (self.hidden, )).astype("float32") - 0.5 + self.data_arr = ( + np.random.random((self.batch, self.seqlen, self.hidden)).astype( + "float32" + ) + - 0.5 + ) + self.matmul_y_arr = ( + np.random.random((1, self.hidden, self.hidden)).astype("float32") + - 0.5 + ) + self.ele_y_arr = ( + np.random.random((self.hidden,)).astype("float32") - 0.5 + ) self.place = paddle.CUDAPlace(0) self.exe = paddle.static.Executor(self.place) @@ -120,37 +132,45 @@ def setUp(self): self.feed = { "_data": self.data_arr, "_matmul_y": self.matmul_y_arr, - "_ele_y": self.ele_y_arr + "_ele_y": self.ele_y_arr, } - self.reference = self.exe.run(self.main_prog, - feed=self.feed, - fetch_list=[self.loss.name]) + self.reference = self.exe.run( + self.main_prog, feed=self.feed, fetch_list=[self.loss.name] + ) - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def _test_output(self): build_strategy = paddle.static.BuildStrategy() build_strategy.fuse_gemm_epilogue = True program = paddle.static.CompiledProgram(self.main_prog) - program = program.with_data_parallel(loss_name=self.loss.name, - build_strategy=build_strategy, - places=paddle.static.cuda_places()) - - result = self.exe.run(program, - feed=self.feed, - fetch_list=[self.loss.name]) + program = program.with_data_parallel( + loss_name=self.loss.name, + build_strategy=build_strategy, + places=paddle.static.cuda_places(), + ) + + result = self.exe.run( + program, feed=self.feed, fetch_list=[self.loss.name] + ) self.assertTrue( compare(self.reference, result, self.atol, self.rtol), - "[{}] outputs are miss-matched.".format(type(self).__name__)) + "[{}] outputs are miss-matched.".format(type(self).__name__), + ) self.assertTrue( verify_node_count(program._graph, "fused_gemm_epilogue", 3), - "[{}] The number of fused_gemm_epilogue is miss-matched in the computing graph." - .format(type(self).__name__)) + "[{}] The number of fused_gemm_epilogue is miss-matched in the computing graph.".format( + type(self).__name__ + ), + ) act_fwd_name = self._get_act_type()[1] self.assertTrue( verify_node_count(program._graph, act_fwd_name, 1), - "[{}] The number of {} is miss-matched in the computing graph.". - format(type(self).__name__, act_fwd_name)) + "[{}] The number of {} is miss-matched in the computing graph.".format( + type(self).__name__, act_fwd_name + ), + ) def _pre_test_hooks(self): self.atol = 1e-4 @@ -160,10 +180,10 @@ def _get_act_type(self): return paddle.nn.ReLU, "relu" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueReluFWDFP32(TestFuseGemmEpilogueFWDBase): - def _pre_test_hooks(self): self.atol = 1e-3 self.rtol = 1e-2 @@ -175,27 +195,28 @@ def test_output(self): self._test_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueReluFWDFP16(TestFuseGemmEpilogueReluFWDFP32): - def _pre_test_hooks(self): self.atol = 1e-3 self.rtol = 1e-2 fp16_var_list = paddle.static.amp.cast_model_to_fp16(self.main_prog) paddle.static.amp.cast_parameters_to_fp16( - self.place, self.main_prog, to_fp16_var_names=fp16_var_list) + self.place, self.main_prog, to_fp16_var_names=fp16_var_list + ) self.data_arr = self.data_arr.astype("float16") self.matmul_y_arr = self.matmul_y_arr.astype("float16") self.ele_y_arr = self.ele_y_arr.astype("float16") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGeluFWDFP32(TestFuseGemmEpilogueFWDBase): - def _pre_test_hooks(self): self.atol = 1e-4 self.rtol = 1e-3 @@ -207,27 +228,28 @@ def test_output(self): self._test_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGeluFWDFP16(TestFuseGemmEpilogueGeluFWDFP32): - def _pre_test_hooks(self): self.atol = 1e-3 self.rtol = 1e-2 fp16_var_list = paddle.static.amp.cast_model_to_fp16(self.main_prog) paddle.static.amp.cast_parameters_to_fp16( - self.place, self.main_prog, to_fp16_var_names=fp16_var_list) + self.place, self.main_prog, to_fp16_var_names=fp16_var_list + ) self.data_arr = self.data_arr.astype("float16") self.matmul_y_arr = self.matmul_y_arr.astype("float16") self.ele_y_arr = self.ele_y_arr.astype("float16") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueBWDBase(unittest.TestCase): - def setUp(self): self.batch = 64 self.seqlen = 128 @@ -239,17 +261,23 @@ def setUp(self): self.startup_prog = paddle.static.Program() with paddle.static.program_guard(self.main_prog, self.startup_prog): - data = paddle.static.data(name="_data", - shape=[-1, self.seqlen, self.hidden], - dtype='float32') - matmul_y = paddle.static.data(name="_matmul_y", - shape=[1, self.hidden, self.hidden], - dtype='float32') - ele_y = paddle.static.data(name="_ele_y", - shape=[ - self.hidden, - ], - dtype='float32') + data = paddle.static.data( + name="_data", + shape=[-1, self.seqlen, self.hidden], + dtype='float32', + ) + matmul_y = paddle.static.data( + name="_matmul_y", + shape=[1, self.hidden, self.hidden], + dtype='float32', + ) + ele_y = paddle.static.data( + name="_ele_y", + shape=[ + self.hidden, + ], + dtype='float32', + ) multi_layer = MultiFCLayer(self.hidden, self._get_act_type()[0]) with paddle.static.amp.fp16_guard(): @@ -257,12 +285,19 @@ def setUp(self): self.loss = paddle.mean(out) paddle.static.append_backward(loss=self.loss) - self.data_arr = np.random.random( - (self.batch, self.seqlen, self.hidden)).astype("float32") - 0.5 - self.matmul_y_arr = np.random.random( - (1, self.hidden, self.hidden)).astype("float32") - 0.5 - self.ele_y_arr = np.random.random( - (self.hidden, )).astype("float32") - 0.5 + self.data_arr = ( + np.random.random((self.batch, self.seqlen, self.hidden)).astype( + "float32" + ) + - 0.5 + ) + self.matmul_y_arr = ( + np.random.random((1, self.hidden, self.hidden)).astype("float32") + - 0.5 + ) + self.ele_y_arr = ( + np.random.random((self.hidden,)).astype("float32") - 0.5 + ) self.place = paddle.CUDAPlace(0) self.exe = paddle.static.Executor(self.place) @@ -273,7 +308,7 @@ def setUp(self): self.feed = { "_data": self.data_arr, "_matmul_y": self.matmul_y_arr, - "_ele_y": self.ele_y_arr + "_ele_y": self.ele_y_arr, } self.fetch = [ @@ -283,46 +318,58 @@ def setUp(self): '{}.w_0@GRAD'.format(multi_layer.linear2.full_name()), '{}.b_0@GRAD'.format(multi_layer.linear2.full_name()), '{}.w_0@GRAD'.format(multi_layer.linear3.full_name()), - '{}.b_0@GRAD'.format(multi_layer.linear3.full_name()) + '{}.b_0@GRAD'.format(multi_layer.linear3.full_name()), ] - self.outs_ref = self.exe.run(self.main_prog, - feed=self.feed, - fetch_list=self.fetch) + self.outs_ref = self.exe.run( + self.main_prog, feed=self.feed, fetch_list=self.fetch + ) - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) def _test_output(self): build_strategy = paddle.static.BuildStrategy() build_strategy.fuse_gemm_epilogue = True program = paddle.static.CompiledProgram(self.main_prog) - program = program.with_data_parallel(loss_name=self.loss.name, - build_strategy=build_strategy, - places=paddle.static.cuda_places()) + program = program.with_data_parallel( + loss_name=self.loss.name, + build_strategy=build_strategy, + places=paddle.static.cuda_places(), + ) outs_res = self.exe.run(program, feed=self.feed, fetch_list=self.fetch) for ref, res in zip(self.outs_ref, outs_res): self.assertTrue( compare(ref, res, self.atol, self.rtol), - "[{}] output is miss-matched.".format(type(self).__name__)) + "[{}] output is miss-matched.".format(type(self).__name__), + ) self.assertTrue( verify_node_count(program._graph, "fused_gemm_epilogue", 3), - "[{}] The number of fused_gemm_epilogue is miss-matched in the computing graph." - .format(type(self).__name__)) + "[{}] The number of fused_gemm_epilogue is miss-matched in the computing graph.".format( + type(self).__name__ + ), + ) self.assertTrue( verify_node_count(program._graph, "fused_gemm_epilogue_grad", 3), - "[{}] The number of fused_gemm_epilogue_grad is miss-matched in the computing graph." - .format(type(self).__name__)) + "[{}] The number of fused_gemm_epilogue_grad is miss-matched in the computing graph.".format( + type(self).__name__ + ), + ) _, act_fwd_name, act_bwd_name = self._get_act_type() self.assertTrue( verify_node_count(program._graph, act_fwd_name, 1), - "[{}] The number of {} is miss-matched in the computing graph.". - format(type(self).__name__, act_fwd_name)) + "[{}] The number of {} is miss-matched in the computing graph.".format( + type(self).__name__, act_fwd_name + ), + ) self.assertTrue( verify_node_count(program._graph, act_bwd_name, 2), - "[{}] The number of {} is miss-matched in the computing graph.". - format(type(self).__name__, act_bwd_name)) + "[{}] The number of {} is miss-matched in the computing graph.".format( + type(self).__name__, act_bwd_name + ), + ) def _pre_test_hooks(self): self.atol = 1e-4 @@ -332,10 +379,10 @@ def _get_act_type(self): return paddle.nn.ReLU, "relu", "relu_grad" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueReLUBWDFP32(TestFuseGemmEpilogueBWDBase): - def _pre_test_hooks(self): self.atol = 1e-4 self.rtol = 1e-3 @@ -347,27 +394,28 @@ def test_output(self): self._test_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueReLUBWDFP16(TestFuseGemmEpilogueReLUBWDFP32): - def _pre_test_hooks(self): self.atol = 1e-3 self.rtol = 1e-2 fp16_var_list = paddle.static.amp.cast_model_to_fp16(self.main_prog) paddle.static.amp.cast_parameters_to_fp16( - self.place, self.main_prog, to_fp16_var_names=fp16_var_list) + self.place, self.main_prog, to_fp16_var_names=fp16_var_list + ) self.data_arr = self.data_arr.astype("float16") self.matmul_y_arr = self.matmul_y_arr.astype("float16") self.ele_y_arr = self.ele_y_arr.astype("float16") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGeLUBWDFP32(TestFuseGemmEpilogueBWDBase): - def _pre_test_hooks(self): self.atol = 5e-4 self.rtol = 1e-3 @@ -379,17 +427,18 @@ def test_output(self): self._test_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGeLUBWDFP16(TestFuseGemmEpilogueGeLUBWDFP32): - def _pre_test_hooks(self): self.atol = 1e-3 self.rtol = 1e-2 fp16_var_list = paddle.static.amp.cast_model_to_fp16(self.main_prog) paddle.static.amp.cast_parameters_to_fp16( - self.place, self.main_prog, to_fp16_var_names=fp16_var_list) + self.place, self.main_prog, to_fp16_var_names=fp16_var_list + ) self.data_arr = self.data_arr.astype("float16") self.matmul_y_arr = self.matmul_y_arr.astype("float16") diff --git a/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py index 5021870e9b62d..9fd6fcc276ecf 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py @@ -24,7 +24,6 @@ class TestFuseOptimizationOps(TestParallelExecutorBase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) @@ -33,72 +32,84 @@ def _get_feed_dict(self): img, label = init_data() return {"image": img, "label": label} - def _compare_fused_optimizer_ops(self, - model, - use_device, - feed_dict=None, - get_data_from_feeder=None, - optimizer=fluid.optimizer.Adam): + def _compare_fused_optimizer_ops( + self, + model, + use_device, + feed_dict=None, + get_data_from_feeder=None, + optimizer=fluid.optimizer.Adam, + ): if use_device == DeviceType.CUDA and not core.is_compiled_with_cuda(): return - not_fuse_op_first_loss, not_fuse_op_last_loss, _ = self.check_network_convergence( + ( + not_fuse_op_first_loss, + not_fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, feed_dict=feed_dict, get_data_from_feeder=get_data_from_feeder, use_device=use_device, fuse_all_optimizer_ops=False, - optimizer=optimizer) - fuse_op_first_loss, fuse_op_last_loss, _ = self.check_network_convergence( + optimizer=optimizer, + ) + ( + fuse_op_first_loss, + fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, feed_dict=feed_dict, get_data_from_feeder=get_data_from_feeder, use_device=use_device, fuse_all_optimizer_ops=True, - optimizer=optimizer) + optimizer=optimizer, + ) for loss in zip(not_fuse_op_first_loss, fuse_op_first_loss): self.assertAlmostEquals(loss[0], loss[1], delta=1e-6) for loss in zip(not_fuse_op_last_loss, fuse_op_last_loss): self.assertAlmostEquals(loss[0], loss[1], delta=1e-6) - def _decorate_compare_fused_optimizer_ops(self, model, use_device, - optimizer): - self._compare_fused_optimizer_ops(model, - use_device, - feed_dict=self._get_feed_dict(), - optimizer=optimizer) + def _decorate_compare_fused_optimizer_ops( + self, model, use_device, optimizer + ): + self._compare_fused_optimizer_ops( + model, + use_device, + feed_dict=self._get_feed_dict(), + optimizer=optimizer, + ) class TestFuseAdamOps(TestFuseOptimizationOps): - def optimizer(self, learning_rate=1e-4): return fluid.optimizer.Adam(learning_rate=learning_rate) def test_batchnorm_fc_with_fuse_op(self): - self._decorate_compare_fused_optimizer_ops(fc_with_batchnorm, - DeviceType.CUDA, - optimizer=self.optimizer) - self._decorate_compare_fused_optimizer_ops(fc_with_batchnorm, - DeviceType.CPU, - optimizer=self.optimizer) + self._decorate_compare_fused_optimizer_ops( + fc_with_batchnorm, DeviceType.CUDA, optimizer=self.optimizer + ) + self._decorate_compare_fused_optimizer_ops( + fc_with_batchnorm, DeviceType.CPU, optimizer=self.optimizer + ) class TestFuseSGDOps(TestFuseAdamOps): - def optimizer(self, learning_rate=1e-3): return fluid.optimizer.SGD(learning_rate=learning_rate) class TestFuseMomentumOps(TestFuseAdamOps): - def optimizer(self, learning_rate=1e-3): - return fluid.optimizer.Momentum(learning_rate=learning_rate, - momentum=0.1) + return fluid.optimizer.Momentum( + learning_rate=learning_rate, momentum=0.1 + ) class TestSpareFuseAdamOps(TestFuseOptimizationOps): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) @@ -113,85 +124,87 @@ def _get_data_from_feeder(self): feeder = fluid.DataFeeder(feed_list=["words", "label"], place=place) return feeder.feed(self.train_data) - def _decorate_compare_fused_optimizer_ops(self, model, use_device, - optimizer): + def _decorate_compare_fused_optimizer_ops( + self, model, use_device, optimizer + ): self._compare_fused_optimizer_ops( model, use_device, get_data_from_feeder=self._get_data_from_feeder, - optimizer=optimizer) + optimizer=optimizer, + ) def optimizer(self, learning_rate=1e-4): return fluid.optimizer.Adam(learning_rate=learning_rate) def test_simple_bow_net_with_fuse_op(self): model = partial(bow_net, dict_dim=self.word_dict_len, is_sparse=True) - self._decorate_compare_fused_optimizer_ops(model, - DeviceType.CUDA, - optimizer=self.optimizer) - self._decorate_compare_fused_optimizer_ops(model, - DeviceType.CPU, - optimizer=self.optimizer) + self._decorate_compare_fused_optimizer_ops( + model, DeviceType.CUDA, optimizer=self.optimizer + ) + self._decorate_compare_fused_optimizer_ops( + model, DeviceType.CPU, optimizer=self.optimizer + ) class TestSpareFuseSGDOps(TestSpareFuseAdamOps): - def optimizer(self, learning_rate=1e-3): return fluid.optimizer.SGD(learning_rate=learning_rate) class TestSpareFuseMomentumOps(TestSpareFuseAdamOps): - def optimizer(self, learning_rate=1e-3): - return fluid.optimizer.Momentum(learning_rate=learning_rate, - momentum=0.1) + return fluid.optimizer.Momentum( + learning_rate=learning_rate, momentum=0.1 + ) class TestPassConflictBase(TestFuseAdamOps): - - def _compare_fused_optimizer_ops(self, - model, - use_device, - feed_dict=None, - get_data_from_feeder=None, - optimizer=fluid.optimizer.Adam): + def _compare_fused_optimizer_ops( + self, + model, + use_device, + feed_dict=None, + get_data_from_feeder=None, + optimizer=fluid.optimizer.Adam, + ): if use_device == DeviceType.CUDA and not core.is_compiled_with_cuda(): return - self.check_pass_conflict(model, - feed_dict=feed_dict, - get_data_from_feeder=get_data_from_feeder, - use_device=use_device, - fuse_all_optimizer_ops=True, - optimizer=optimizer, - enable_sequential_execution=True) + self.check_pass_conflict( + model, + feed_dict=feed_dict, + get_data_from_feeder=get_data_from_feeder, + use_device=use_device, + fuse_all_optimizer_ops=True, + optimizer=optimizer, + enable_sequential_execution=True, + ) class TestFuseAdamOpsPassConflict(TestPassConflictBase): - def optimizer(self, learning_rate=1e-4): return fluid.optimizer.Adam(learning_rate=learning_rate) def test_batchnorm_fc_with_fuse_op(self): - self._decorate_compare_fused_optimizer_ops(fc_with_batchnorm, - DeviceType.CPU, - optimizer=self.optimizer) - self._decorate_compare_fused_optimizer_ops(fc_with_batchnorm, - DeviceType.CUDA, - optimizer=self.optimizer) + self._decorate_compare_fused_optimizer_ops( + fc_with_batchnorm, DeviceType.CPU, optimizer=self.optimizer + ) + self._decorate_compare_fused_optimizer_ops( + fc_with_batchnorm, DeviceType.CUDA, optimizer=self.optimizer + ) class TestFuseSGDOpsPassConflict(TestFuseAdamOpsPassConflict): - def optimizer(self, learning_rate=1e-3): return fluid.optimizer.SGD(learning_rate=learning_rate) class TestFuseMomentumOpsPassConflict(TestFuseAdamOpsPassConflict): - def optimizer(self, learning_rate=1e-3): - return fluid.optimizer.Momentum(learning_rate=learning_rate, - momentum=0.1) + return fluid.optimizer.Momentum( + learning_rate=learning_rate, momentum=0.1 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py index a93c1380c0b31..686a5c1e41088 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_relu_depthwise_conv_pass.py @@ -26,27 +26,27 @@ def norm(*args, **kargs): def sep_conv(input, channel, stride, filter, dilation=1, act=None): # with scope('depthwise'): - input = fluid.layers.conv2d(input, - input.shape[1], - filter, - stride, - groups=input.shape[1], - padding=(filter // 2) * dilation, - dilation=dilation, - use_cudnn=False, - bias_attr=False) + input = fluid.layers.conv2d( + input, + input.shape[1], + filter, + stride, + groups=input.shape[1], + padding=(filter // 2) * dilation, + dilation=dilation, + use_cudnn=False, + bias_attr=False, + ) input = norm(input) - if act: input = act(input) + if act: + input = act(input) # with scope('pointwise'): - input = fluid.layers.conv2d(input, - channel, - 1, - 1, - groups=1, - padding=0, - bias_attr=False) + input = fluid.layers.conv2d( + input, channel, 1, 1, groups=1, padding=0, bias_attr=False + ) input = norm(input) - if act: input = act(input) + if act: + input = act(input) return input @@ -65,7 +65,6 @@ def simple_depthwise_net(use_feed): class TestMNIST(TestParallelExecutorBase): - def _init_data(self, random=True): np.random.seed(5) if random: @@ -83,31 +82,36 @@ def _compare(self, model, use_device, random_data=True, only_forward=False): def _optimizer(learning_rate=1e-6): optimizer = fluid.optimizer.SGD( learning_rate=learning_rate, - regularization=fluid.regularizer.L2Decay(1e-6)) + regularization=fluid.regularizer.L2Decay(1e-6), + ) return optimizer if only_forward: _optimizer = None - fuse_op_first_loss, fuse_op_last_loss, _ = self.check_network_convergence( + ( + fuse_op_first_loss, + fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, fuse_relu_depthwise_conv=True, use_ir_memory_optimize=True, - optimizer=_optimizer) - not_fuse_op_first_loss, not_fuse_op_last_loss, _ = self.check_network_convergence( + optimizer=_optimizer, + ) + ( + not_fuse_op_first_loss, + not_fuse_op_last_loss, + _, + ) = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, fuse_relu_depthwise_conv=False, - optimizer=_optimizer) + optimizer=_optimizer, + ) for loss in zip(not_fuse_op_first_loss, fuse_op_first_loss): self.assertAlmostEquals(loss[0], loss[1], delta=1e-6) diff --git a/python/paddle/fluid/tests/unittests/test_fused_attention_op.py b/python/paddle/fluid/tests/unittests/test_fused_attention_op.py index 1e27005499a09..6e3c001442267 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_attention_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_attention_op.py @@ -30,7 +30,6 @@ class TestFusedAttentionOp(OpTest): - def setUp(self): self.config() self.generate_input_data() @@ -50,22 +49,30 @@ def setUp(self): self.__class__.op_type = "fused_attention" # use autograd to check grad in this unittest. self.__class__.no_need_check_grad = True - self.q_proj = Linear(self.embed_dim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) - self.k_proj = Linear(self.kdim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) - self.v_proj = Linear(self.vdim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) - self.out_proj = Linear(self.embed_dim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) + self.q_proj = Linear( + self.embed_dim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) + self.k_proj = Linear( + self.kdim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) + self.v_proj = Linear( + self.vdim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) + self.out_proj = Linear( + self.embed_dim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) paddle.set_default_dtype(np.float32) self.norm1 = LayerNorm(self.embed_dim) self.norm2 = LayerNorm(self.embed_dim) @@ -92,40 +99,57 @@ def config(self): self.weight_attr = None self.bias_attr = None self.kdim, self.vdim = self.embed_dim, self.embed_dim - self.key_length, self.value_length = self.query_length, self.query_length + self.key_length, self.value_length = ( + self.query_length, + self.query_length, + ) def generate_input_data(self): - self.query = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) + self.query = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) out_seq_len = self.key_length if self.has_cache_kv: assert self.training is False, ValueError( - 'cache_kv can only used in inference') - self.cache_kv = np.random.rand(2, self.batch_size, self.num_heads, - self.cache_length, - self.head_dim).astype(self.x_type) + 'cache_kv can only used in inference' + ) + self.cache_kv = np.random.rand( + 2, + self.batch_size, + self.num_heads, + self.cache_length, + self.head_dim, + ).astype(self.x_type) out_seq_len += self.cache_length else: self.cache_kv = None if self.has_attn_mask: # [B, n_head, seq_len, out_seq_len] - self.attn_mask = np.ones((self.batch_size, self.num_heads, - self.query_length, out_seq_len), - dtype=self.attn_mask_type) + self.attn_mask = np.ones( + ( + self.batch_size, + self.num_heads, + self.query_length, + out_seq_len, + ), + dtype=self.attn_mask_type, + ) if self.attn_mask_type == np.int64: self.attn_mask = np.tril(self.attn_mask) elif self.attn_mask_type == np.float64: self.attn_mask = (np.tril(self.attn_mask) - 1.0) * 1e9 else: raise ValueError( - "'attn_mask_type' should be 'int64' or 'float64'.") + "'attn_mask_type' should be 'int64' or 'float64'." + ) else: self.attn_mask = None self.key, self.value = self.query, self.query - self.dout = np.random.random((self.batch_size, self.query_length, - self.embed_dim)).astype(self.x_type) + self.dout = np.random.random( + (self.batch_size, self.query_length, self.embed_dim) + ).astype(self.x_type) def GetBaselineOut(self): paddle.disable_static(place=paddle.CUDAPlace(0)) @@ -167,10 +191,9 @@ def GetBaselineOut(self): # [B, n_head, seq_len, head_dim] * [B, n_head, out_seq_len, head_dim] # --> [B, n_head, seq_len, out_seq_len] - qk_out = layers.matmul(x=q_out, - y=k_out, - transpose_y=True, - alpha=self.head_dim**-0.5) + qk_out = layers.matmul( + x=q_out, y=k_out, transpose_y=True, alpha=self.head_dim**-0.5 + ) if attn_mask is not None: attn_mask = _convert_attention_mask(attn_mask, qk_out.dtype) @@ -180,10 +203,12 @@ def GetBaselineOut(self): softmax_out = F.softmax(qk_out) if self.dropout_prob: - dropout_out = F.dropout(softmax_out, - self.dropout_prob, - training=self.training, - mode="upscale_in_train") + dropout_out = F.dropout( + softmax_out, + self.dropout_prob, + training=self.training, + mode="upscale_in_train", + ) # [B, n_head, seq_len, out_seq_len] * [B, n_head, out_seq_len, head_dim] # --> [B, n_head, seq_len, head_dim] qktv_out = tensor.matmul(dropout_out, v_out) @@ -192,7 +217,8 @@ def GetBaselineOut(self): fmha_out = tensor.transpose(qktv_out, perm=[0, 2, 1, 3]) out_linear_in = tensor.reshape( - x=fmha_out, shape=[0, 0, fmha_out.shape[2] * fmha_out.shape[3]]) + x=fmha_out, shape=[0, 0, fmha_out.shape[2] * fmha_out.shape[3]] + ) out = self.out_proj(out_linear_in) residual_out = residual + self.dropout(out) @@ -204,37 +230,47 @@ def GetBaselineOut(self): if self.has_cache_kv: return final_out - paddle.autograd.backward([final_out], [paddle.to_tensor(self.dout)], - retain_graph=True) + paddle.autograd.backward( + [final_out], [paddle.to_tensor(self.dout)], retain_graph=True + ) return final_out, tensor_query.grad def GetFusedAttentionOut(self): paddle.disable_static(place=paddle.CUDAPlace(0)) - q_proj_weight = paddle.to_tensor(self.q_proj.weight, - stop_gradient=False) - k_proj_weight = paddle.to_tensor(self.k_proj.weight, - stop_gradient=False) - v_proj_weight = paddle.to_tensor(self.v_proj.weight, - stop_gradient=False) - out_linear_weight = paddle.to_tensor(self.out_proj.weight, - stop_gradient=False) + q_proj_weight = paddle.to_tensor( + self.q_proj.weight, stop_gradient=False + ) + k_proj_weight = paddle.to_tensor( + self.k_proj.weight, stop_gradient=False + ) + v_proj_weight = paddle.to_tensor( + self.v_proj.weight, stop_gradient=False + ) + out_linear_weight = paddle.to_tensor( + self.out_proj.weight, stop_gradient=False + ) if self.bias_attr is False: qkv_bias_tensor = None out_linear_bias = None else: - q_proj_bias = paddle.to_tensor(self.q_proj.bias, - stop_gradient=False) - k_proj_bias = paddle.to_tensor(self.k_proj.bias, - stop_gradient=False) - v_proj_bias = paddle.to_tensor(self.v_proj.bias, - stop_gradient=False) + q_proj_bias = paddle.to_tensor( + self.q_proj.bias, stop_gradient=False + ) + k_proj_bias = paddle.to_tensor( + self.k_proj.bias, stop_gradient=False + ) + v_proj_bias = paddle.to_tensor( + self.v_proj.bias, stop_gradient=False + ) qkv_bias = np.concatenate( - (q_proj_bias.numpy(), k_proj_bias.numpy(), v_proj_bias.numpy())) + (q_proj_bias.numpy(), k_proj_bias.numpy(), v_proj_bias.numpy()) + ) qkv_bias = qkv_bias.reshape((3, self.num_heads, self.head_dim)) qkv_bias_tensor = paddle.to_tensor(qkv_bias, stop_gradient=False) - out_linear_bias = paddle.to_tensor(self.out_proj.bias, - stop_gradient=False) + out_linear_bias = paddle.to_tensor( + self.out_proj.bias, stop_gradient=False + ) ln1_scale = paddle.to_tensor(self.norm1.weight, stop_gradient=False) ln1_bias = paddle.to_tensor(self.norm1.bias, stop_gradient=False) @@ -245,9 +281,11 @@ def GetFusedAttentionOut(self): k_proj_weight = k_proj_weight.numpy().transpose((1, 0)) v_proj_weight = v_proj_weight.numpy().transpose((1, 0)) qkv_weight = np.concatenate( - (q_proj_weight, k_proj_weight, v_proj_weight)) + (q_proj_weight, k_proj_weight, v_proj_weight) + ) qkv_weight = qkv_weight.reshape( - (3, self.num_heads, self.head_dim, self.embed_dim)) + (3, self.num_heads, self.head_dim, self.embed_dim) + ) x = paddle.to_tensor(self.query, stop_gradient=False) cache_kv = None @@ -264,47 +302,56 @@ def GetFusedAttentionOut(self): if attn_mask is not None: attn_mask = _convert_attention_mask(attn_mask, x.dtype) final_out = incubate_f.fused_multi_head_attention( - x, qkv_weight_tensor, out_linear_weight, self.pre_layer_norm, - ln1_scale, ln1_bias, ln2_scale, ln2_bias, epsilon, qkv_bias_tensor, - out_linear_bias, cache_kv, attn_mask, self.dropout_prob, - self.attn_dropout_prob, ln2_epsilon) + x, + qkv_weight_tensor, + out_linear_weight, + self.pre_layer_norm, + ln1_scale, + ln1_bias, + ln2_scale, + ln2_bias, + epsilon, + qkv_bias_tensor, + out_linear_bias, + cache_kv, + attn_mask, + self.dropout_prob, + self.attn_dropout_prob, + ln2_epsilon, + ) if self.has_cache_kv: return final_out[0], final_out[1] - paddle.autograd.backward([final_out], [paddle.to_tensor(self.dout)], - retain_graph=True) + paddle.autograd.backward( + [final_out], [paddle.to_tensor(self.dout)], retain_graph=True + ) return final_out, x.grad def test_fused_attention_op(self): final_out_ref, x_grad_ref = self.GetBaselineOut() final_out, x_grad = self.GetFusedAttentionOut() - np.testing.assert_allclose(final_out_ref, - final_out.numpy(), - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(x_grad_ref, - x_grad.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + final_out_ref, final_out.numpy(), rtol=self.rtol, atol=self.atol + ) + np.testing.assert_allclose( + x_grad_ref, x_grad.numpy(), rtol=self.rtol, atol=self.atol + ) class TestFusedAttentionOpBiasIsNone(TestFusedAttentionOp): - def config(self): super().config() self.bias_attr = False class TestFusedAttentionOpPreLn(TestFusedAttentionOp): - def config(self): super().config() self.pre_layer_norm = True class TestFusedAttentionOpNoneAttnMask(TestFusedAttentionOp): - def config(self): super().config() self.pre_layer_norm = True @@ -312,7 +359,6 @@ def config(self): class TestFusedAttentionOpFp16(TestFusedAttentionOp): - def config(self): super().config() self.x_type = np.float16 @@ -320,18 +366,15 @@ def config(self): def test_fused_attention_op(self): final_out_ref, x_grad_ref = self.GetBaselineOut() final_out, x_grad = self.GetFusedAttentionOut() - np.testing.assert_allclose(final_out_ref, - final_out.numpy(), - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(x_grad_ref, - x_grad.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + final_out_ref, final_out.numpy(), rtol=self.rtol, atol=self.atol + ) + np.testing.assert_allclose( + x_grad_ref, x_grad.numpy(), rtol=self.rtol, atol=self.atol + ) class TestFusedAttentionOpCacheKV(TestFusedAttentionOp): - def config(self): super().config() self.has_cache_kv = True @@ -343,10 +386,9 @@ def test_fused_attention_op(self): with paddle.no_grad(): final_out_ref = self.GetBaselineOut() final_out, cache_kv_out = self.GetFusedAttentionOut() - np.testing.assert_allclose(final_out_ref, - final_out.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + final_out_ref, final_out.numpy(), rtol=self.rtol, atol=self.atol + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_fused_attention_op_api.py b/python/paddle/fluid/tests/unittests/test_fused_attention_op_api.py index ee8efdc81d0b3..3a3872bb6726f 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_attention_op_api.py +++ b/python/paddle/fluid/tests/unittests/test_fused_attention_op_api.py @@ -39,8 +39,9 @@ def softmax(x): def batch_matmul(x, y): assert x.shape[0] == y.shape[0] assert x.shape[1] == y.shape[1] - retval = np.zeros((x.shape[0], x.shape[1], x.shape[2], y.shape[3]), - dtype=np.float64) + retval = np.zeros( + (x.shape[0], x.shape[1], x.shape[2], y.shape[3]), dtype=np.float64 + ) for i in range(x.shape[0]): for j in range(x.shape[1]): retval[i, j, :, :] = np.matmul(x[i, j, :, :], y[i, j, :, :]) @@ -52,24 +53,34 @@ def layer_norm(x, has_scale, has_bias, weight, bias, epsilon=1e-05): x = x.reshape((batch_size * src_len, d_model)) mu = np.mean(x, axis=1, keepdims=True) sigma_squar = np.sum(np.square(x - mu), axis=1) / d_model - x1_up = (x - mu) + x1_up = x - mu x1_down_1 = sigma_squar + epsilon x1_down = np.sqrt(x1_down_1) x1_down = x1_down.reshape((x1_down.shape[0], 1)) x1 = x1_up / x1_down x_scaled = x1 - if (has_scale): + if has_scale: x_scaled = weight * x1 x_scaled_bias = x_scaled - if (has_bias): + if has_bias: x_scaled_bias = x_scaled + bias x_scaled_bias = x_scaled_bias.reshape((batch_size, src_len, d_model)) return x_scaled_bias -def compute_reference(pre_layer_norm, query, attn_mask, ln_scale, ln_bias, - ln_2_scale, ln_2_bias, qkv_weight, qkv_bias, - out_linear_weight, out_linear_bias): +def compute_reference( + pre_layer_norm, + query, + attn_mask, + ln_scale, + ln_bias, + ln_2_scale, + ln_2_bias, + qkv_weight, + qkv_bias, + out_linear_weight, + out_linear_bias, +): batch_size = query.shape[0] seq_len = query.shape[1] embed_dim = query.shape[2] @@ -87,11 +98,13 @@ def compute_reference(pre_layer_norm, query, attn_mask, ln_scale, ln_bias, qkv_weight = qkv_weight.transpose((3, 0, 1, 2)) qkv_weight = qkv_weight.reshape( qkv_weight.shape[0], - qkv_weight.shape[1] * qkv_weight.shape[2] * qkv_weight.shape[3]) + qkv_weight.shape[1] * qkv_weight.shape[2] * qkv_weight.shape[3], + ) if qkv_bias is not None: - qkv_bias = qkv_bias.reshape(qkv_bias.shape[0] * qkv_bias.shape[1] * - qkv_bias.shape[2]) + qkv_bias = qkv_bias.reshape( + qkv_bias.shape[0] * qkv_bias.shape[1] * qkv_bias.shape[2] + ) if pre_layer_norm: ln_out = ln_out.reshape(batch_size * seq_len, embed_dim) qkv = fc(ln_out, qkv_weight) @@ -109,22 +122,25 @@ def compute_reference(pre_layer_norm, query, attn_mask, ln_scale, ln_bias, qkv_bias_out = qkv query = query.reshape(batch_size, seq_len, embed_dim) - qkv_bias_out = qkv_bias_out.reshape(batch_size, seq_len, 3, num_head, - head_dim) + qkv_bias_out = qkv_bias_out.reshape( + batch_size, seq_len, 3, num_head, head_dim + ) # q*k^t qkv_bias_out = qkv_bias_out.transpose( - (2, 0, 1, 3, 4)) # 3, batch_size, seq_len, num_head, head_dim + (2, 0, 1, 3, 4) + ) # 3, batch_size, seq_len, num_head, head_dim qkv_bias_out = qkv_bias_out.transpose( - (0, 1, 3, 2, 4)) # 3, batch_size, num_head, seq_len, head_dim + (0, 1, 3, 2, 4) + ) # 3, batch_size, num_head, seq_len, head_dim q = qkv_bias_out[0:1, ::] q = q.reshape(batch_size, num_head, seq_len, head_dim) - k = qkv_bias_out[1:2, ::] #[1, batch_size, num_head, seq_len, head_dim] + k = qkv_bias_out[1:2, ::] # [1, batch_size, num_head, seq_len, head_dim] k = k.reshape(batch_size, num_head, seq_len, head_dim) v = qkv_bias_out[2::] v = v.reshape(batch_size, num_head, seq_len, head_dim) - k = k.transpose([0, 1, 3, 2]) #[batch_size, num_head, head_dim, seq_len] + k = k.transpose([0, 1, 3, 2]) # [batch_size, num_head, head_dim, seq_len] qkt = batch_matmul(q, k / np.sqrt(head_dim, dtype=np.float64)) if attn_mask is not None: @@ -139,11 +155,13 @@ def compute_reference(pre_layer_norm, query, attn_mask, ln_scale, ln_bias, attn_heads = batch_matmul(softmax_out, v) attn_heads = attn_heads.transpose( - (0, 2, 1, 3)) # [batch_size, seq_len, num_head, head_dim] + (0, 2, 1, 3) + ) # [batch_size, seq_len, num_head, head_dim] # out_linear - out_linear_input = attn_heads.reshape(batch_size, seq_len, - num_head * head_dim) + out_linear_input = attn_heads.reshape( + batch_size, seq_len, num_head * head_dim + ) out_linear_out = fc(out_linear_input, out_linear_weight) # bias add, dropout, residual add, layer_norm. @@ -155,13 +173,16 @@ def compute_reference(pre_layer_norm, query, attn_mask, ln_scale, ln_bias, out_linear_bias_dropout_residual_out = query + out_linear_bias_dropout_out if not pre_layer_norm: out_linear_bias_dropout_residual_out = layer_norm( - out_linear_bias_dropout_residual_out, True, has_bias, ln_2_scale, - ln_2_bias) + out_linear_bias_dropout_residual_out, + True, + has_bias, + ln_2_scale, + ln_2_bias, + ) return out_linear_bias_dropout_residual_out class TestFusedAttentionAPI(unittest.TestCase): - def setUp(self): self.setXType() self.setPreLn() @@ -209,22 +230,33 @@ def config(self): self.weight_attr = None self.kdim, self.vdim = self.embed_dim, self.embed_dim - self.key_length, self.value_length = self.query_length, self.query_length + self.key_length, self.value_length = ( + self.query_length, + self.query_length, + ) def generate_input_data(self): - self.query = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) + self.query = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) if self.has_attn_mask: - self.attn_mask = np.ones((self.batch_size, self.num_heads, - self.query_length, self.key_length), - dtype=self.attn_mask_type) + self.attn_mask = np.ones( + ( + self.batch_size, + self.num_heads, + self.query_length, + self.key_length, + ), + dtype=self.attn_mask_type, + ) if self.attn_mask_type == np.int64: self.attn_mask = np.tril(self.attn_mask) elif self.attn_mask_type == np.float64: self.attn_mask = (np.tril(self.attn_mask) - 1.0) * 1e9 else: raise ValueError( - "'attn_mask_type' should be 'int64' or 'float64'.") + "'attn_mask_type' should be 'int64' or 'float64'." + ) else: self.attn_mask = None self.key, self.value = self.query, self.query @@ -235,18 +267,34 @@ def run_imperative(self): else: attn_mask_tensor = None fused_attn = FusedMultiHeadAttention( - self.embed_dim, self.num_heads, self.dropout_prob, - self.attn_dropout_prob, self.kdim, self.vdim, self.pre_layer_norm, - self.need_weight, self.weight_attr, self.bias_attr, - self.weight_attr, self.bias_attr, self.weight_attr, self.bias_attr, - self.weight_attr, self.bias_attr) + self.embed_dim, + self.num_heads, + self.dropout_prob, + self.attn_dropout_prob, + self.kdim, + self.vdim, + self.pre_layer_norm, + self.need_weight, + self.weight_attr, + self.bias_attr, + self.weight_attr, + self.bias_attr, + self.weight_attr, + self.bias_attr, + self.weight_attr, + self.bias_attr, + ) if self.bias_attr is not False: - qkv_bias = np.random.random( - fused_attn.qkv_bias.shape).astype('float32') + qkv_bias = np.random.random(fused_attn.qkv_bias.shape).astype( + 'float32' + ) fused_attn.qkv_bias.set_value(paddle.to_tensor(qkv_bias)) - out = fused_attn(paddle.to_tensor(self.query), - paddle.to_tensor(self.query), - paddle.to_tensor(self.query), attn_mask_tensor) + out = fused_attn( + paddle.to_tensor(self.query), + paddle.to_tensor(self.query), + paddle.to_tensor(self.query), + attn_mask_tensor, + ) fused_attn_qkv_bias = None fused_attn_linear_bias = None @@ -263,38 +311,58 @@ def run_imperative(self): fused_attn_ln_bias = fused_attn.ln_bias.numpy() ref_out = compute_reference( - self.pre_layer_norm, self.query, self.attn_mask, + self.pre_layer_norm, + self.query, + self.attn_mask, fused_attn.pre_ln_scale.numpy() if self.pre_layer_norm else None, fused_attn_pre_ln_bias, fused_attn.ln_scale.numpy() if not self.pre_layer_norm else None, fused_attn_ln_bias, - fused_attn.qkv_weight.numpy(), fused_attn_qkv_bias, - fused_attn.linear_weight.numpy(), fused_attn_linear_bias) - np.testing.assert_allclose(ref_out, - out.numpy(), - rtol=self.rtol, - atol=self.atol) + fused_attn.qkv_weight.numpy(), + fused_attn_qkv_bias, + fused_attn.linear_weight.numpy(), + fused_attn_linear_bias, + ) + np.testing.assert_allclose( + ref_out, out.numpy(), rtol=self.rtol, atol=self.atol + ) def run_static(self): fused_attn = FusedMultiHeadAttention( - self.embed_dim, self.num_heads, self.dropout_prob, - self.attn_dropout_prob, self.kdim, self.vdim, self.pre_layer_norm, - self.need_weight, self.weight_attr, self.bias_attr, - self.weight_attr, self.bias_attr, self.weight_attr, self.bias_attr, - self.weight_attr, self.bias_attr) + self.embed_dim, + self.num_heads, + self.dropout_prob, + self.attn_dropout_prob, + self.kdim, + self.vdim, + self.pre_layer_norm, + self.need_weight, + self.weight_attr, + self.bias_attr, + self.weight_attr, + self.bias_attr, + self.weight_attr, + self.bias_attr, + self.weight_attr, + self.bias_attr, + ) x = paddle.static.data( name='X', shape=[self.batch_size, self.query_length, self.embed_dim], - dtype=self.x_type) + dtype=self.x_type, + ) if self.has_attn_mask: - attn_mask = paddle.static.data(name='SrcMask', - shape=[ - self.batch_size, self.num_heads, - self.query_length, - self.key_length - ], - dtype=self.attn_mask_type) + attn_mask = paddle.static.data( + name='SrcMask', + shape=[ + self.batch_size, + self.num_heads, + self.query_length, + self.key_length, + ], + dtype=self.attn_mask_type, + ) final_out = fused_attn(x, x, x, attn_mask) else: final_out = fused_attn(x, x, x) @@ -314,35 +382,38 @@ def run_static(self): if self.pre_layer_norm: out, qkv_weight, out_linear_weight, ln_scale = exe.run( paddle.static.default_main_program(), - feed={ - "X": self.query, - "SrcMask": self.attn_mask - }, + feed={"X": self.query, "SrcMask": self.attn_mask}, fetch_list=[ final_out, fused_attn.qkv_weight, fused_attn.linear_weight, fused_attn.pre_ln_scale, - ]) + ], + ) else: out, qkv_weight, out_linear_weight, ln_2_scale = exe.run( paddle.static.default_main_program(), - feed={ - "X": self.query, - "SrcMask": self.attn_mask - }, + feed={"X": self.query, "SrcMask": self.attn_mask}, fetch_list=[ - final_out, fused_attn.qkv_weight, - fused_attn.linear_weight, fused_attn.ln_scale - ]) + final_out, + fused_attn.qkv_weight, + fused_attn.linear_weight, + fused_attn.ln_scale, + ], + ) else: if self.pre_layer_norm: - out, qkv_weight, qkv_bias, out_linear_weight, linear_bias, ln_scale, ln_bias = exe.run( + ( + out, + qkv_weight, + qkv_bias, + out_linear_weight, + linear_bias, + ln_scale, + ln_bias, + ) = exe.run( paddle.static.default_main_program(), - feed={ - "X": self.query, - "SrcMask": self.attn_mask - }, + feed={"X": self.query, "SrcMask": self.attn_mask}, fetch_list=[ final_out, fused_attn.qkv_weight, @@ -351,20 +422,30 @@ def run_static(self): fused_attn.linear_bias, fused_attn.pre_ln_scale, fused_attn.pre_ln_bias, - ]) + ], + ) else: - out, qkv_weight, qkv_bias, out_linear_weight, linear_bias, ln_2_scale, ln_2_bias = exe.run( + ( + out, + qkv_weight, + qkv_bias, + out_linear_weight, + linear_bias, + ln_2_scale, + ln_2_bias, + ) = exe.run( paddle.static.default_main_program(), - feed={ - "X": self.query, - "SrcMask": self.attn_mask - }, + feed={"X": self.query, "SrcMask": self.attn_mask}, fetch_list=[ - final_out, fused_attn.qkv_weight, - fused_attn.qkv_bias, fused_attn.linear_weight, - fused_attn.linear_bias, fused_attn.ln_scale, - fused_attn.ln_bias - ]) + final_out, + fused_attn.qkv_weight, + fused_attn.qkv_bias, + fused_attn.linear_weight, + fused_attn.linear_bias, + fused_attn.ln_scale, + fused_attn.ln_bias, + ], + ) else: if self.bias_attr is False: if self.pre_layer_norm: @@ -378,7 +459,8 @@ def run_static(self): fused_attn.qkv_weight, fused_attn.linear_weight, fused_attn.pre_ln_scale, - ]) + ], + ) else: out, qkv_weight, out_linear_weight, ln_2_scale = exe.run( paddle.static.default_main_program(), @@ -386,12 +468,23 @@ def run_static(self): "X": self.query, }, fetch_list=[ - final_out, fused_attn.qkv_weight, - fused_attn.linear_weight, fused_attn.ln_scale - ]) + final_out, + fused_attn.qkv_weight, + fused_attn.linear_weight, + fused_attn.ln_scale, + ], + ) else: if self.pre_layer_norm: - out, qkv_weight, qkv_bias, out_linear_weight, linear_bias, ln_scale, ln_bias = exe.run( + ( + out, + qkv_weight, + qkv_bias, + out_linear_weight, + linear_bias, + ln_scale, + ln_bias, + ) = exe.run( paddle.static.default_main_program(), feed={ "X": self.query, @@ -404,30 +497,71 @@ def run_static(self): fused_attn.linear_bias, fused_attn.pre_ln_scale, fused_attn.pre_ln_bias, - ]) + ], + ) else: - out, qkv_weight, qkv_bias, out_linear_weight, linear_bias, ln_2_scale, ln_2_bias = exe.run( + ( + out, + qkv_weight, + qkv_bias, + out_linear_weight, + linear_bias, + ln_2_scale, + ln_2_bias, + ) = exe.run( paddle.static.default_main_program(), feed={ "X": self.query, }, fetch_list=[ - final_out, fused_attn.qkv_weight, - fused_attn.qkv_bias, fused_attn.linear_weight, - fused_attn.linear_bias, fused_attn.ln_scale, - fused_attn.ln_bias - ]) - return out, qkv_weight, qkv_bias, out_linear_weight, linear_bias, ln_scale, ln_bias, ln_2_scale, ln_2_bias + final_out, + fused_attn.qkv_weight, + fused_attn.qkv_bias, + fused_attn.linear_weight, + fused_attn.linear_bias, + fused_attn.ln_scale, + fused_attn.ln_bias, + ], + ) + return ( + out, + qkv_weight, + qkv_bias, + out_linear_weight, + linear_bias, + ln_scale, + ln_bias, + ln_2_scale, + ln_2_bias, + ) def test_static_api(self): paddle.enable_static() with paddle.static.program_guard(Program()): - out, qkv_weight, qkv_bias, linear_weight, linear_bias, ln_scale, ln_bias, ln_2_scale, ln_2_bias = self.run_static( - ) - ref_out = compute_reference(self.pre_layer_norm, self.query, - self.attn_mask, ln_scale, ln_bias, - ln_2_scale, ln_2_bias, qkv_weight, qkv_bias, - linear_weight, linear_bias) + ( + out, + qkv_weight, + qkv_bias, + linear_weight, + linear_bias, + ln_scale, + ln_bias, + ln_2_scale, + ln_2_bias, + ) = self.run_static() + ref_out = compute_reference( + self.pre_layer_norm, + self.query, + self.attn_mask, + ln_scale, + ln_bias, + ln_2_scale, + ln_2_bias, + qkv_weight, + qkv_bias, + linear_weight, + linear_bias, + ) np.testing.assert_allclose(ref_out, out, rtol=self.rtol, atol=self.atol) def test_dynamic_api(self): @@ -436,7 +570,6 @@ def test_dynamic_api(self): class TestFusedAttentionAPINoneAttnMask(TestFusedAttentionAPI): - def setAttnMask(self): self.has_attn_mask = False @@ -445,7 +578,6 @@ def setPreLn(self): class TestFusedAttentionAPIBiasIsNone(TestFusedAttentionAPI): - def setBiasAttr(self): self.bias_attr = False diff --git a/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op.py b/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op.py index 7572a1dfe14de..8ae92fe194f35 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op.py @@ -28,7 +28,6 @@ class TestFusedBiasDropoutResidualLayerNormOp(OpTest): - def setUp(self): self.config() self.generate_input_data() @@ -53,23 +52,28 @@ def config(self): self.bias_attr = None def generate_input_data(self): - self.x = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) - self.residual = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) + self.x = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) + self.residual = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) self.linear_bias = np.random.rand(self.embed_dim).astype(self.x_type) - self.dout = np.random.random((self.batch_size, self.query_length, - self.embed_dim)).astype(self.x_type) + self.dout = np.random.random( + (self.batch_size, self.query_length, self.embed_dim) + ).astype(self.x_type) if self.bias_attr is False: self.tensor_linear_bias = None else: - self.tensor_linear_bias = paddle.to_tensor(self.linear_bias, - stop_gradient=False) + self.tensor_linear_bias = paddle.to_tensor( + self.linear_bias, stop_gradient=False + ) self.tensor_x = paddle.to_tensor(self.x, stop_gradient=False) - self.tensor_residual = paddle.to_tensor(self.residual, - stop_gradient=False) + self.tensor_residual = paddle.to_tensor( + self.residual, stop_gradient=False + ) def GetBaselineOut(self): paddle.disable_static(place=paddle.CUDAPlace(0)) @@ -82,14 +86,20 @@ def GetBaselineOut(self): residual_out = self.tensor_residual + self.dropout(out) final_out = self.norm1(residual_out) - paddle.autograd.backward([final_out], [paddle.to_tensor(self.dout)], - retain_graph=True) + paddle.autograd.backward( + [final_out], [paddle.to_tensor(self.dout)], retain_graph=True + ) if self.tensor_linear_bias is not None: tensor_linear_bias_grad = self.tensor_linear_bias.grad else: tensor_linear_bias_grad = None - return final_out, self.tensor_x.grad, self.tensor_residual.grad, tensor_linear_bias_grad + return ( + final_out, + self.tensor_x.grad, + self.tensor_residual.grad, + tensor_linear_bias_grad, + ) def GetFusedBiasDropoutResidualLayerNormOut(self): paddle.disable_static(place=paddle.CUDAPlace(0)) @@ -99,52 +109,71 @@ def GetFusedBiasDropoutResidualLayerNormOut(self): epsilon = 1e-05 final_out = incubate_f.fused_bias_dropout_residual_layer_norm( - self.tensor_x, self.tensor_residual, self.tensor_linear_bias, - ln_scale, ln_bias, self.dropout_prob, epsilon) + self.tensor_x, + self.tensor_residual, + self.tensor_linear_bias, + ln_scale, + ln_bias, + self.dropout_prob, + epsilon, + ) - paddle.autograd.backward([final_out], [paddle.to_tensor(self.dout)], - retain_graph=True) + paddle.autograd.backward( + [final_out], [paddle.to_tensor(self.dout)], retain_graph=True + ) if self.tensor_linear_bias is not None: tensor_linear_bias_grad = self.tensor_linear_bias.grad else: tensor_linear_bias_grad = None - return final_out, self.tensor_x.grad, self.tensor_residual.grad, tensor_linear_bias_grad + return ( + final_out, + self.tensor_x.grad, + self.tensor_residual.grad, + tensor_linear_bias_grad, + ) def test_fused_op(self): - out_ref, x_grad_ref, residual_grad_ref, linear_bias_grad_ref = self.GetBaselineOut( + ( + out_ref, + x_grad_ref, + residual_grad_ref, + linear_bias_grad_ref, + ) = self.GetBaselineOut() + ( + out, + x_grad, + residual_grad, + linear_bias_grad, + ) = self.GetFusedBiasDropoutResidualLayerNormOut() + np.testing.assert_allclose( + out_ref, out.numpy(), rtol=1e-5, atol=self.atol + ) + np.testing.assert_allclose( + x_grad_ref, x_grad.numpy(), rtol=1e-5, atol=self.atol ) - out, x_grad, residual_grad, linear_bias_grad = self.GetFusedBiasDropoutResidualLayerNormOut( + np.testing.assert_allclose( + residual_grad_ref, residual_grad.numpy(), rtol=1e-5, atol=self.atol ) - np.testing.assert_allclose(out_ref, - out.numpy(), - rtol=1e-5, - atol=self.atol) - np.testing.assert_allclose(x_grad_ref, - x_grad.numpy(), - rtol=1e-5, - atol=self.atol) - np.testing.assert_allclose(residual_grad_ref, - residual_grad.numpy(), - rtol=1e-5, - atol=self.atol) if linear_bias_grad_ref is not None: - np.testing.assert_allclose(linear_bias_grad_ref, - linear_bias_grad.numpy(), - rtol=1e-5, - atol=self.atol) + np.testing.assert_allclose( + linear_bias_grad_ref, + linear_bias_grad.numpy(), + rtol=1e-5, + atol=self.atol, + ) class TestFusedBiasDropoutResidualLayerNormOpBiasIsNone( - TestFusedBiasDropoutResidualLayerNormOp): - + TestFusedBiasDropoutResidualLayerNormOp +): def config(self): super().config() self.bias_attr = False class TestFusedBiasDropoutResidualLayerNormOpFp16( - TestFusedBiasDropoutResidualLayerNormOp): - + TestFusedBiasDropoutResidualLayerNormOp +): def config(self): super().config() self.x_type = np.float16 diff --git a/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op_api.py b/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op_api.py index aa54860e0996b..261a2686b271c 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op_api.py +++ b/python/paddle/fluid/tests/unittests/test_fused_bias_dropout_residual_layer_norm_op_api.py @@ -15,7 +15,9 @@ import numpy as np import paddle -from paddle.incubate.nn.layer.fused_transformer import FusedBiasDropoutResidualLayerNorm +from paddle.incubate.nn.layer.fused_transformer import ( + FusedBiasDropoutResidualLayerNorm, +) from paddle.static import Program import unittest @@ -25,16 +27,16 @@ def layer_norm(x, has_scale, has_bias, weight, bias, epsilon=1e-05): x = x.reshape((batch_size * src_len, d_model)) mu = np.mean(x, axis=1, keepdims=True) sigma_squar = np.sum(np.square(x - mu), axis=1) / d_model - x1_up = (x - mu) + x1_up = x - mu x1_down_1 = sigma_squar + epsilon x1_down = np.sqrt(x1_down_1) x1_down = x1_down.reshape((x1_down.shape[0], 1)) x1 = x1_up / x1_down x_scaled = x1 - if (has_scale): + if has_scale: x_scaled = weight * x1 x_scaled_bias = x_scaled - if (has_bias): + if has_bias: x_scaled_bias = x_scaled + bias x_scaled_bias = x_scaled_bias.reshape((batch_size, src_len, d_model)) return x_scaled_bias @@ -56,12 +58,12 @@ def compute_reference(x, residual, ln_scale, ln_bias, linear_bias): linear_bias_dropout_out = linear_bias_out linear_bias_dropout_residual_out = residual + linear_bias_dropout_out linear_bias_dropout_residual_ln_out = layer_norm( - linear_bias_dropout_residual_out, True, has_bias, ln_scale, ln_bias) + linear_bias_dropout_residual_out, True, has_bias, ln_scale, ln_bias + ) return linear_bias_dropout_residual_ln_out class TestFusedBiasDropoutResidualLayerNormAPI(unittest.TestCase): - def setUp(self): self.setXType() self.setBiasAttr() @@ -84,50 +86,56 @@ def config(self): self.weight_attr = None def generate_input_data(self): - self.x = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) - self.residual = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) + self.x = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) + self.residual = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) def run_imperative(self): fused_bias_dropout_residual_ln = FusedBiasDropoutResidualLayerNorm( - self.embed_dim, self.dropout_prob, self.weight_attr, self.bias_attr) + self.embed_dim, self.dropout_prob, self.weight_attr, self.bias_attr + ) linear_bias = None if self.bias_attr is not False: linear_bias = np.random.random( - fused_bias_dropout_residual_ln.linear_bias.shape).astype( - 'float32') + fused_bias_dropout_residual_ln.linear_bias.shape + ).astype('float32') fused_bias_dropout_residual_ln.linear_bias.set_value( - paddle.to_tensor(linear_bias)) - out = fused_bias_dropout_residual_ln(paddle.to_tensor(self.x), - paddle.to_tensor(self.residual)) + paddle.to_tensor(linear_bias) + ) + out = fused_bias_dropout_residual_ln( + paddle.to_tensor(self.x), paddle.to_tensor(self.residual) + ) ln_bias = None if self.bias_attr is not False: ln_bias = fused_bias_dropout_residual_ln.ln_bias.numpy() - ln_scale = fused_bias_dropout_residual_ln.ln_scale.numpy(), - ref_out = compute_reference(self.x, self.residual, ln_scale, ln_bias, - linear_bias) - np.testing.assert_allclose(ref_out, - out.numpy(), - rtol=1e-5, - atol=self.atol) + ln_scale = (fused_bias_dropout_residual_ln.ln_scale.numpy(),) + ref_out = compute_reference( + self.x, self.residual, ln_scale, ln_bias, linear_bias + ) + np.testing.assert_allclose( + ref_out, out.numpy(), rtol=1e-5, atol=self.atol + ) def run_static(self): - fused_op = FusedBiasDropoutResidualLayerNorm(self.embed_dim, - self.dropout_prob, - self.weight_attr, - self.bias_attr) + fused_op = FusedBiasDropoutResidualLayerNorm( + self.embed_dim, self.dropout_prob, self.weight_attr, self.bias_attr + ) x = paddle.static.data( name='X', shape=[self.batch_size, self.query_length, self.embed_dim], - dtype=self.x_type) + dtype=self.x_type, + ) residual = paddle.static.data( name='Residual', shape=[self.batch_size, self.query_length, self.embed_dim], - dtype=self.x_type) + dtype=self.x_type, + ) final_out = fused_op(x, residual) place = paddle.CUDAPlace(0) @@ -137,31 +145,31 @@ def run_static(self): linear_bias = None ln_bias = None if self.bias_attr is False: - out, ln_scale = exe.run(paddle.static.default_main_program(), - feed={ - "X": self.x, - "Residual": self.residual - }, - fetch_list=[final_out, fused_op.ln_scale]) + out, ln_scale = exe.run( + paddle.static.default_main_program(), + feed={"X": self.x, "Residual": self.residual}, + fetch_list=[final_out, fused_op.ln_scale], + ) else: out, linear_bias, ln_scale, ln_bias = exe.run( paddle.static.default_main_program(), - feed={ - "X": self.x, - "Residual": self.residual - }, + feed={"X": self.x, "Residual": self.residual}, fetch_list=[ - final_out, fused_op.linear_bias, fused_op.ln_scale, - fused_op.ln_bias - ]) + final_out, + fused_op.linear_bias, + fused_op.ln_scale, + fused_op.ln_bias, + ], + ) return out, linear_bias, ln_scale, ln_bias def test_static_api(self): paddle.enable_static() with paddle.static.program_guard(Program()): out, linear_bias, ln_scale, ln_bias = self.run_static() - ref_out = compute_reference(self.x, self.residual, ln_scale, ln_bias, - linear_bias) + ref_out = compute_reference( + self.x, self.residual, ln_scale, ln_bias, linear_bias + ) np.testing.assert_allclose(ref_out, out, rtol=1e-5, atol=self.atol) def test_dynamic_api(self): @@ -170,8 +178,8 @@ def test_dynamic_api(self): class TestFusedBiasDropoutResidualLayerNormAPIBiasIsNone( - TestFusedBiasDropoutResidualLayerNormAPI): - + TestFusedBiasDropoutResidualLayerNormAPI +): def setBiasAttr(self): self.bias_attr = False diff --git a/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py b/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py index d44c6720d8bb5..cc672f5a16f02 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py @@ -32,14 +32,10 @@ # TestFusedElementwiseActivationOp_channelwise_add -def create_test_class(test_case, - callback, - attrs, - dtype=np.float32, - grad_chek=True): - +def create_test_class( + test_case, callback, attrs, dtype=np.float32, grad_chek=True +): class TestFusedElementwiseActivationOp_base(OpTest): - def setUp(self): self.op_type = "fused_elemwise_activation" self.dtype = dtype @@ -54,12 +50,12 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } if self.attrs["save_intermediate_out"]: self.outputs = { 'Out': self.out, - "IntermediateOut": self.intermediate_out + "IntermediateOut": self.intermediate_out, } else: self.outputs = {'Out': self.out} @@ -70,8 +66,9 @@ def init_input(self): self.axis = -1 def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y + ) def init_attr(self): self.attrs = { @@ -101,133 +98,152 @@ def test_check_grad_ingore_x(self): if not grad_chek: return if self.attrs["save_intermediate_out"]: - self.check_grad(['Y'], ['Out'], - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad( + ['Y'], + ['Out'], + max_relative_error=0.005, + no_grad_set=set("X"), + ) else: - self.check_grad(['Y'], ['Out'], - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad( + ['Y'], + ['Out'], + max_relative_error=0.005, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): if not grad_chek: return if self.attrs["save_intermediate_out"]: - self.check_grad(['X'], ['Out'], - max_relative_error=0.005, - no_grad_set=set("Y")) + self.check_grad( + ['X'], + ['Out'], + max_relative_error=0.005, + no_grad_set=set("Y"), + ) else: - self.check_grad(['X'], ['Out'], - max_relative_error=0.005, - no_grad_set=set("Y")) + self.check_grad( + ['X'], + ['Out'], + max_relative_error=0.005, + no_grad_set=set("Y"), + ) class TestFusedElementwiseActivationOp_scalar( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) class TestFusedElementwiseActivationOp_scalar2( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) class TestFusedElementwiseActivationOp_Vector( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): - self.x = np.random.random((32, )).astype(self.dtype) - self.y = np.random.random((32, )).astype(self.dtype) + self.x = np.random.random((32,)).astype(self.dtype) + self.y = np.random.random((32,)).astype(self.dtype) class TestFusedElementwiseActivationOp_broadcast_0( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(2).astype(self.dtype) self.axis = 0 def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y.reshape(2, 1, 1)) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y.reshape(2, 1, 1) + ) class TestFusedElementwiseActivationOp_broadcast_1( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(3).astype(self.dtype) self.axis = 1 def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y.reshape(1, 3, 1)) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y.reshape(1, 3, 1) + ) class TestFusedElementwiseActivationOp_broadcast_2( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(4).astype(self.dtype) def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y.reshape(1, 1, 4)) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y.reshape(1, 1, 4) + ) class TestFusedElementwiseActivationOp_broadcast_3( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4, 5).astype(self.dtype) self.y = np.random.rand(3, 4).astype(self.dtype) self.axis = 1 def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y.reshape(1, 3, 4, 1)) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y.reshape(1, 3, 4, 1) + ) class TestFusedElementwiseActivationOp_broadcast_4( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4, 5).astype(self.dtype) self.y = np.random.rand(2, 1).astype(self.dtype) self.axis = 0 def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y.reshape(2, 1, 1, 1)) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y.reshape(2, 1, 1, 1) + ) class TestFusedElementwiseActivationOp_rowwise_add_0( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(3, 4).astype(self.dtype) self.axis = 1 def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y.reshape(1, 3, 4)) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y.reshape(1, 3, 4) + ) class TestFusedElementwiseActivationOp_rowwise_add_1( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(2, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) self.axis = 1 def init_output(self): - self.x, self.y, self.intermediate_out, self.out = \ - callback(self.x, self.y, self.x, self.y.reshape(1, 1)) + self.x, self.y, self.intermediate_out, self.out = callback( + self.x, self.y, self.x, self.y.reshape(1, 1) + ) class TestFusedElementwiseActivationOp_channelwise_add( - TestFusedElementwiseActivationOp_base): - + TestFusedElementwiseActivationOp_base + ): def init_input(self): self.x = np.random.rand(3, 20, 20).astype(self.dtype) self.y = np.random.rand(3, 1, 1).astype(self.dtype) @@ -236,36 +252,59 @@ def init_input(self): TestFusedElementwiseActivationOp_scalar.__name__ = test_case + "_scalar" TestFusedElementwiseActivationOp_scalar2.__name__ = test_case + "_scalar2" TestFusedElementwiseActivationOp_Vector.__name__ = test_case + "_Vector" - TestFusedElementwiseActivationOp_broadcast_0.__name__ = test_case + "_broadcast_0" - TestFusedElementwiseActivationOp_broadcast_1.__name__ = test_case + "_broadcast_1" - TestFusedElementwiseActivationOp_broadcast_2.__name__ = test_case + "_broadcast_2" - TestFusedElementwiseActivationOp_broadcast_3.__name__ = test_case + "_broadcast_3" - TestFusedElementwiseActivationOp_broadcast_4.__name__ = test_case + "_broadcast_4" - TestFusedElementwiseActivationOp_rowwise_add_0.__name__ = test_case + "_rowwise_add_0" - TestFusedElementwiseActivationOp_rowwise_add_1.__name__ = test_case + "_rowwise_add_1" - TestFusedElementwiseActivationOp_channelwise_add.__name__ = test_case + "_channelwise_add" + TestFusedElementwiseActivationOp_broadcast_0.__name__ = ( + test_case + "_broadcast_0" + ) + TestFusedElementwiseActivationOp_broadcast_1.__name__ = ( + test_case + "_broadcast_1" + ) + TestFusedElementwiseActivationOp_broadcast_2.__name__ = ( + test_case + "_broadcast_2" + ) + TestFusedElementwiseActivationOp_broadcast_3.__name__ = ( + test_case + "_broadcast_3" + ) + TestFusedElementwiseActivationOp_broadcast_4.__name__ = ( + test_case + "_broadcast_4" + ) + TestFusedElementwiseActivationOp_rowwise_add_0.__name__ = ( + test_case + "_rowwise_add_0" + ) + TestFusedElementwiseActivationOp_rowwise_add_1.__name__ = ( + test_case + "_rowwise_add_1" + ) + TestFusedElementwiseActivationOp_channelwise_add.__name__ = ( + test_case + "_channelwise_add" + ) globals()[test_case + "_base"] = TestFusedElementwiseActivationOp_base globals()[test_case + "_scalar"] = TestFusedElementwiseActivationOp_scalar globals()[test_case + "_scalar2"] = TestFusedElementwiseActivationOp_scalar2 globals()[test_case + "_Vector"] = TestFusedElementwiseActivationOp_Vector - globals()[test_case + - "_broadcast_0"] = TestFusedElementwiseActivationOp_broadcast_0 - globals()[test_case + - "_broadcast_1"] = TestFusedElementwiseActivationOp_broadcast_1 - globals()[test_case + - "_broadcast_2"] = TestFusedElementwiseActivationOp_broadcast_2 - globals()[test_case + - "_broadcast_3"] = TestFusedElementwiseActivationOp_broadcast_3 - globals()[test_case + - "_broadcast_4"] = TestFusedElementwiseActivationOp_broadcast_4 - globals()[test_case + - "_rowwise_add_0"] = TestFusedElementwiseActivationOp_rowwise_add_0 - globals()[test_case + - "_rowwise_add_1"] = TestFusedElementwiseActivationOp_rowwise_add_1 globals()[ - test_case + - "_channelwise_add"] = TestFusedElementwiseActivationOp_channelwise_add + test_case + "_broadcast_0" + ] = TestFusedElementwiseActivationOp_broadcast_0 + globals()[ + test_case + "_broadcast_1" + ] = TestFusedElementwiseActivationOp_broadcast_1 + globals()[ + test_case + "_broadcast_2" + ] = TestFusedElementwiseActivationOp_broadcast_2 + globals()[ + test_case + "_broadcast_3" + ] = TestFusedElementwiseActivationOp_broadcast_3 + globals()[ + test_case + "_broadcast_4" + ] = TestFusedElementwiseActivationOp_broadcast_4 + globals()[ + test_case + "_rowwise_add_0" + ] = TestFusedElementwiseActivationOp_rowwise_add_0 + globals()[ + test_case + "_rowwise_add_1" + ] = TestFusedElementwiseActivationOp_rowwise_add_1 + globals()[ + test_case + "_channelwise_add" + ] = TestFusedElementwiseActivationOp_channelwise_add def scale_add_func(x, y, x_bcast, y_bcast, scale, mode=0): @@ -338,97 +377,129 @@ def gelu_add_func(x, y, x_bcast, y_bcast, mode=0): gelu_add_func = partial(gelu_add_func, mode=mode) for save_intermediate_out in {True, False}: - suffix = ("_save_intermediate_out" if save_intermediate_out else "") \ - + ("_mode_"+ str(mode)) + suffix = ("_save_intermediate_out" if save_intermediate_out else "") + ( + "_mode_" + str(mode) + ) create_test_class( - 'scale_add' + suffix, scale_add_func, { + 'scale_add' + suffix, + scale_add_func, + { 'scale': scale, 'functor_list': ["scale", "elementwise_add"], 'save_intermediate_out': save_intermediate_out, - }) + }, + ) create_test_class( - 'add_scale' + suffix, add_scale_func, { + 'add_scale' + suffix, + add_scale_func, + { 'scale': scale, 'functor_list': ["elementwise_add", "scale"], 'save_intermediate_out': save_intermediate_out, - }) + }, + ) create_test_class( - 'add_relu' + suffix, add_relu_func, { + 'add_relu' + suffix, + add_relu_func, + { 'functor_list': ["elementwise_add", "relu"], 'save_intermediate_out': save_intermediate_out, - }) + }, + ) create_test_class( - 'relu_add' + suffix, relu_add_func, { + 'relu_add' + suffix, + relu_add_func, + { 'functor_list': ["relu", "elementwise_add"], 'save_intermediate_out': save_intermediate_out, - }) + }, + ) create_test_class( - 'mul_scale' + suffix, mul_scale_func, { + 'mul_scale' + suffix, + mul_scale_func, + { 'scale': scale, 'functor_list': ["elementwise_mul", "scale"], 'save_intermediate_out': save_intermediate_out, - }) + }, + ) create_test_class( - 'gelu_add' + suffix, gelu_add_func, { + 'gelu_add' + suffix, + gelu_add_func, + { 'functor_list': ["gelu", "elementwise_add"], 'save_intermediate_out': save_intermediate_out, - }) + }, + ) if core.is_compiled_with_cuda(): create_test_class( 'scale_add_fp16' + suffix, - scale_add_func, { + scale_add_func, + { 'scale': scale, 'functor_list': ["scale", "elementwise_add"], 'save_intermediate_out': save_intermediate_out, }, dtype=np.float16, - grad_chek=False) + grad_chek=False, + ) create_test_class( 'add_scale_fp16' + suffix, - add_scale_func, { + add_scale_func, + { 'scale': scale, 'functor_list': ["elementwise_add", "scale"], 'save_intermediate_out': save_intermediate_out, }, dtype=np.float16, - grad_chek=False) + grad_chek=False, + ) create_test_class( 'add_relu_fp16' + suffix, - add_relu_func, { + add_relu_func, + { 'functor_list': ["elementwise_add", "relu"], 'save_intermediate_out': save_intermediate_out, }, dtype=np.float16, - grad_chek=False) + grad_chek=False, + ) create_test_class( 'relu_add_fp16' + suffix, - relu_add_func, { + relu_add_func, + { 'functor_list': ["relu", "elementwise_add"], 'save_intermediate_out': save_intermediate_out, }, dtype=np.float16, - grad_chek=False) + grad_chek=False, + ) create_test_class( 'mul_scale_fp16' + suffix, - mul_scale_func, { + mul_scale_func, + { 'scale': scale, 'functor_list': ["elementwise_mul", "scale"], 'save_intermediate_out': save_intermediate_out, }, dtype=np.float16, - grad_chek=False) + grad_chek=False, + ) create_test_class( 'gelu_add_fp16' + suffix, - gelu_add_func, { + gelu_add_func, + { 'functor_list': ["gelu", "elementwise_add"], 'save_intermediate_out': save_intermediate_out, }, dtype=np.float16, - grad_chek=False) + grad_chek=False, + ) if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_fused_emb_seq_pool_op.py b/python/paddle/fluid/tests/unittests/test_fused_emb_seq_pool_op.py index 5963ef07e92a5..392c04f730a3f 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_emb_seq_pool_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_emb_seq_pool_op.py @@ -19,27 +19,34 @@ import paddle.version as ver -@skip_check_grad_ci(reason="check_grad is called when ver.mkl() == ON" - "and 'Linux' in platform.platform().") +@skip_check_grad_ci( + reason="check_grad is called when ver.mkl() == ON" + "and 'Linux' in platform.platform()." +) class TestFusedEmbeddingSeqPoolOp(OpTest): - def setUp(self): self.op_type = "fused_embedding_seq_pool" self.emb_size = 6 self.table = np.random.random((17, self.emb_size)).astype("float64") - self.ids = np.array([[[4], [3]], [[4], [3]], [[2], [1]], - [[16], [1]]]).astype("int64") + self.ids = np.array( + [[[4], [3]], [[4], [3]], [[2], [1]], [[16], [1]]] + ).astype("int64") ids_expand = np.expand_dims(self.ids, axis=1) self.lod = [[3, 1]] self.attrs = {'is_sparse': True} self.inputs = {'W': self.table, 'Ids': (ids_expand, self.lod)} self.outputs = { - 'Out': - np.reshape( - np.array([ - self.table[[4, 3]] + self.table[[4, 3]] + - self.table[[2, 1]], self.table[[16, 1]] - ]), [len(self.lod[0]), 2 * self.emb_size]) + 'Out': np.reshape( + np.array( + [ + self.table[[4, 3]] + + self.table[[4, 3]] + + self.table[[2, 1]], + self.table[[16, 1]], + ] + ), + [len(self.lod[0]), 2 * self.emb_size], + ) } def test_check_output(self): @@ -50,14 +57,12 @@ def test_check_grad(self): # TODO(wangzhongpu): support lod in dygraph mode if ver.mkl() == "ON" and 'Linux' in platform.platform(): self.attrs = {'is_sparse': False} - self.check_grad(['W'], - 'Out', - no_grad_set=['Ids'], - check_dygraph=False) + self.check_grad( + ['W'], 'Out', no_grad_set=['Ids'], check_dygraph=False + ) class TestLookupTableOpWithPadding(TestFusedEmbeddingSeqPoolOp): - def test_check_output(self): if ver.mkl() == "ON" and 'Linux' in platform.platform(): ids = np.squeeze(self.ids, axis=2) @@ -65,18 +70,20 @@ def test_check_output(self): output = list() index = 0 for count in self.lod[0]: - arr = ids[index:count + index] - out = np.reshape(self.table[arr.flatten()], - [arr.shape[0], arr.shape[1], self.emb_size]) + arr = ids[index : count + index] + out = np.reshape( + self.table[arr.flatten()], + [arr.shape[0], arr.shape[1], self.emb_size], + ) idx = np.argwhere(arr == padding_idx) for item in idx: out[item[0], item[1], :] = np.zeros(self.emb_size) output.append(np.sum(out, 0)) index += count self.outputs = { - 'Out': - np.reshape(np.array(output), - [len(self.lod[0]), 2 * self.emb_size]) + 'Out': np.reshape( + np.array(output), [len(self.lod[0]), 2 * self.emb_size] + ) } self.attrs = {'padding_idx': int(padding_idx)} # TODO(wangzhongpu): support lod in dygraph mode @@ -88,30 +95,28 @@ def test_check_grad(self): padding_idx = np.random.choice(ids.flatten(), 1)[0] self.attrs = {'padding_idx': int(padding_idx), 'is_sparse': False} # TODO(wangzhongpu): support lod in dygraph mode - self.check_grad(['W'], - 'Out', - no_grad_set=['Ids'], - check_dygraph=False) + self.check_grad( + ['W'], 'Out', no_grad_set=['Ids'], check_dygraph=False + ) class TestFusedEmbeddingSeqPoolApi(unittest.TestCase): - def test_api(self): if ver.mkl() == "ON" and 'Linux' in platform.platform(): import paddle.fluid as fluid dict_size = 20 - data_t = fluid.layers.data(name='word', - shape=[1], - dtype='int64', - lod_level=1) + data_t = fluid.layers.data( + name='word', shape=[1], dtype='int64', lod_level=1 + ) padding_idx = np.random.randint(1, 10) out = fluid.contrib.fused_embedding_seq_pool( input=data_t, size=[dict_size, 32], param_attr='w', padding_idx=padding_idx, - is_sparse=False) + is_sparse=False, + ) place = fluid.CPUPlace() exe = fluid.Executor(place) diff --git a/python/paddle/fluid/tests/unittests/test_fused_embedding_fc_lstm_op.py b/python/paddle/fluid/tests/unittests/test_fused_embedding_fc_lstm_op.py index 4d1b38e54eee3..54fc4d77b7921 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_embedding_fc_lstm_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_embedding_fc_lstm_op.py @@ -23,30 +23,41 @@ def fc(x, w, b): def fused_embedded_fc_lstm( - ids, # T x 1 - lod, # 1 x N - embeddings=None, # Dict_size x M - wx=None, # M x 4D - bx=None, # 1 x 4D - h0=None, # N x D - c0=None, # N x D - w_h=None, # D x 4D - w_b=None, # 1 x 4D - w_c=None, # 1 x 3D - is_reverse=False, - act_gate=None, - act_cell=None, - act_cand=None): + ids, # T x 1 + lod, # 1 x N + embeddings=None, # Dict_size x M + wx=None, # M x 4D + bx=None, # 1 x 4D + h0=None, # N x D + c0=None, # N x D + w_h=None, # D x 4D + w_b=None, # 1 x 4D + w_c=None, # 1 x 3D + is_reverse=False, + act_gate=None, + act_cell=None, + act_cand=None, +): # Make a lookup for embeddings and pass result into lstm reference T = ids.shape[0] M = embeddings.shape[1] x = embeddings[ids].reshape([T, M]) - return lstm(fc(x, wx, bx), lod, h0, c0, w_h, w_b, w_c, is_reverse, act_gate, - act_cell, act_cand) + return lstm( + fc(x, wx, bx), + lod, + h0, + c0, + w_h, + w_b, + w_c, + is_reverse, + act_gate, + act_cell, + act_cand, + ) class TestFusionLSTMOp(OpTest): - def set_conf(self): pass @@ -76,22 +87,24 @@ def setUp(self): b = np.random.normal(size=(1, 7 * self.D)).astype('float32') else: b = np.random.normal(size=(1, 4 * self.D)).astype('float32') - w_b = np.copy(b[:, 0:4 * self.D]) - w_c = b[:, 4 * self.D:] if self.use_peepholes else None + w_b = np.copy(b[:, 0 : 4 * self.D]) + w_c = b[:, 4 * self.D :] if self.use_peepholes else None # low is 0 , high is voc_size - 1 - ids = np.random.randint(low=0, high=self.dict_size - 1, - size=(T, 1)).astype("int64") + ids = np.random.randint( + low=0, high=self.dict_size - 1, size=(T, 1) + ).astype("int64") # embeddings as they were trained , so each entry is of M size - embeddings = np.random.random( - (self.dict_size, self.M)).astype("float32") + embeddings = np.random.random((self.dict_size, self.M)).astype( + "float32" + ) # multiply embeddings via Weights fc_embeddings = np.dot(embeddings, wx) # bias should be manually added into the bias of this fused embedding fc LSTM - b[0, 0:4 * self.D] += bx[0, :] - combined_biases = b[:, 0:4 * self.D] + b[0, 0 : 4 * self.D] += bx[0, :] + combined_biases = b[:, 0 : 4 * self.D] # So let broadcast it , so they can be added ones = np.ones([self.dict_size, 1]) broadcasted_biases = np.dot(ones, combined_biases) @@ -107,17 +120,28 @@ def setUp(self): wh = np.random.normal(size=(self.D, 4 * self.D)).astype('float32') - h, c = fused_embedded_fc_lstm(ids, self.lod, embeddings, wx, bx, h0, c0, - wh, w_b, w_c, self.is_reverse, - ACTIVATION[self.act_gate], - ACTIVATION[self.act_cell], - ACTIVATION[self.act_cand]) + h, c = fused_embedded_fc_lstm( + ids, + self.lod, + embeddings, + wx, + bx, + h0, + c0, + wh, + w_b, + w_c, + self.is_reverse, + ACTIVATION[self.act_gate], + ACTIVATION[self.act_cell], + ACTIVATION[self.act_cand], + ) self.inputs = { 'Ids': (ids, self.lod), 'Embeddings': fc_embeddings, 'WeightH': wh, - 'Bias': b + 'Bias': b, } if self.has_initial_state: @@ -133,7 +157,7 @@ def setUp(self): 'is_reverse': self.is_reverse, 'gate_activation': self.act_gate, 'cell_activation': self.act_cell, - 'candidate_activation': self.act_cand + 'candidate_activation': self.act_cand, } def test_check_output(self): @@ -143,74 +167,63 @@ def test_check_output(self): class TestFusionLSTMOpInit(TestFusionLSTMOp): - def set_conf(self): self.has_initial_state = True class TestFusionLSTMOpReverse(TestFusionLSTMOp): - def set_conf(self): self.is_reverse = True class TestFusionLSTMOpInitReverse(TestFusionLSTMOp): - def set_conf(self): self.has_initial_state = True self.is_reverse = True class TestFusionLSTMOpMD1(TestFusionLSTMOp): - def set_conf(self): self.M = 36 self.D = 8 class TestFusionLSTMOpMD2(TestFusionLSTMOp): - def set_conf(self): self.M = 8 self.D = 8 class TestFusionLSTMOpMD3(TestFusionLSTMOp): - def set_conf(self): self.M = 15 self.D = 3 class TestFusionLSTMOpBS1(TestFusionLSTMOp): - def set_conf(self): self.lod = [[3]] self.D = 16 class TestFusionLSTMOpPeepholes(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True class TestFusionLSTMOpPeepholesInit(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.has_initial_state = True class TestFusionLSTMOpPeepholesReverse(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.is_reverse = True class TestFusionLSTMOpPeepholesInitReverse(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.has_initial_state = True @@ -218,7 +231,6 @@ def set_conf(self): class TestFusionLSTMOpPeepholesBS1(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.lod = [[2]] diff --git a/python/paddle/fluid/tests/unittests/test_fused_fc_elementwise_layernorm_op.py b/python/paddle/fluid/tests/unittests/test_fused_fc_elementwise_layernorm_op.py index fa9948128dee2..09797b0384919 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_fc_elementwise_layernorm_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_fc_elementwise_layernorm_op.py @@ -22,10 +22,10 @@ np.random.random(123) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "Paddle core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "Paddle core is not compiled with CUDA" +) class TestFusedFCElementwiseLayerNormOp(OpTest): - def config(self): self.matrix = MatrixGenerate(1, 10, 15, 3, 3, 2) self.y_shape = [1, 15] @@ -44,11 +44,12 @@ def setUp(self): y = np.random.random_sample(self.y_shape).astype(np.float32) add_out = fc_out + y # layer_norm - scale_shape = [np.prod(self.y_shape[self.begin_norm_axis:])] + scale_shape = [np.prod(self.y_shape[self.begin_norm_axis :])] scale = np.random.random_sample(scale_shape).astype(np.float32) bias_1 = np.random.random_sample(scale_shape).astype(np.float32) out, mean, variance = _reference_layer_norm_naive( - add_out, scale, bias_1, epsilon, self.begin_norm_axis) + add_out, scale, bias_1, epsilon, self.begin_norm_axis + ) self.inputs = { "X": self.matrix.input, @@ -56,12 +57,12 @@ def setUp(self): "Bias0": self.matrix.bias, "Y": y, "Scale": scale, - "Bias1": bias_1 + "Bias1": bias_1, } self.attrs = { "activation_type": "relu", "epsilon": epsilon, - "begin_norm_axis": self.begin_norm_axis + "begin_norm_axis": self.begin_norm_axis, } self.outputs = {"Out": out, "Mean": mean, "Variance": variance} @@ -71,7 +72,6 @@ def test_check_output(self): class TestFusedFCElementwiseLayerNormOp2(TestFusedFCElementwiseLayerNormOp): - def config(self): self.matrix = MatrixGenerate(4, 5, 6, 2, 2, 1) self.y_shape = [4, 6] diff --git a/python/paddle/fluid/tests/unittests/test_fused_feedforward_op.py b/python/paddle/fluid/tests/unittests/test_fused_feedforward_op.py index 85b7ee9285d86..71bf2bad2cc47 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_feedforward_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_feedforward_op.py @@ -25,7 +25,6 @@ class TestFusedFFNOp(OpTest): - def getDtype(self): self.dtype = "float32" self.layer_norm_dtype = "float32" @@ -54,7 +53,7 @@ def getNormalizeBefore(self): def setUp(self): paddle.disable_static() self.__class__.op_type = "fused_feedforward" - #check grad in test_out_and_grad() + # check grad in test_out_and_grad() self.__class__.no_need_check_grad = True self.getDtype() self.getShape() @@ -66,17 +65,23 @@ def setUp(self): self.bias_attr = None self.weight_attrs = transformer._convert_param_attr_to_list( - self.weight_attr, 2) + self.weight_attr, 2 + ) self.bias_attrs = transformer._convert_param_attr_to_list( - self.bias_attr, 2) - self.linear1 = Linear(self.d_model, - self.dim_feedforward, - self.weight_attrs[1], - bias_attr=self.bias_attrs[1]) - self.linear2 = Linear(self.dim_feedforward, - self.d_model, - self.weight_attrs[1], - bias_attr=self.bias_attrs[1]) + self.bias_attr, 2 + ) + self.linear1 = Linear( + self.d_model, + self.dim_feedforward, + self.weight_attrs[1], + bias_attr=self.bias_attrs[1], + ) + self.linear2 = Linear( + self.dim_feedforward, + self.d_model, + self.weight_attrs[1], + bias_attr=self.bias_attrs[1], + ) paddle.set_default_dtype(self.layer_norm_dtype) self.norm1 = LayerNorm(self.d_model) @@ -86,10 +91,12 @@ def setUp(self): self.dropout2 = Dropout(0.0, mode="upscale_in_train") self.activation = getattr(F, self.act_method) - self.src = np.random.random((self.batch_size, self.query_length, - self.d_model)).astype(self.dtype) - self.dout = np.random.random((self.batch_size, self.query_length, - self.d_model)).astype(self.dtype) + self.src = np.random.random( + (self.batch_size, self.query_length, self.d_model) + ).astype(self.dtype) + self.dout = np.random.random( + (self.batch_size, self.query_length, self.d_model) + ).astype(self.dtype) def Base(self): paddle.disable_static() @@ -98,46 +105,54 @@ def Base(self): if self.pre_layer_norm: ln1_out = self.norm1(tensor_src) linear2_out = self.linear2( - self.dropout(self.activation(self.linear1(ln1_out)))) + self.dropout(self.activation(self.linear1(ln1_out))) + ) dropout2_out = residual + self.dropout2(linear2_out) - paddle.autograd.backward([dropout2_out], - [paddle.to_tensor(self.dout)], True) + paddle.autograd.backward( + [dropout2_out], [paddle.to_tensor(self.dout)], True + ) return dropout2_out, tensor_src.grad else: linear2_out = self.linear2( - self.dropout(self.activation(self.linear1(tensor_src)))) + self.dropout(self.activation(self.linear1(tensor_src))) + ) dropout2_out = residual + self.dropout2(linear2_out) dropout2_out = self.norm2(dropout2_out) - paddle.autograd.backward([dropout2_out], - [paddle.to_tensor(self.dout)], True) + paddle.autograd.backward( + [dropout2_out], [paddle.to_tensor(self.dout)], True + ) return dropout2_out, tensor_src.grad def FusedFFN(self): paddle.disable_static() - linear1_weight = paddle.to_tensor(self.linear1.weight, - stop_gradient=False) + linear1_weight = paddle.to_tensor( + self.linear1.weight, stop_gradient=False + ) linear1_bias = paddle.to_tensor(self.linear1.bias, stop_gradient=False) - linear2_weight = paddle.to_tensor(self.linear2.weight, - stop_gradient=False) + linear2_weight = paddle.to_tensor( + self.linear2.weight, stop_gradient=False + ) linear2_bias = paddle.to_tensor(self.linear2.bias, stop_gradient=False) ln1_scale = paddle.to_tensor(self.norm1.weight, stop_gradient=False) ln1_bias = paddle.to_tensor(self.norm1.bias, stop_gradient=False) ln2_scale = paddle.to_tensor(self.norm2.weight, stop_gradient=False) ln2_bias = paddle.to_tensor(self.norm2.bias, stop_gradient=False) x = paddle.to_tensor(self.src, stop_gradient=False) - out = incubate_f.fused_feedforward(x, - linear1_weight, - linear2_weight, - linear1_bias, - linear2_bias, - ln1_scale, - ln1_bias, - ln2_scale, - ln2_bias, - 0.0, - 0.0, - activation=self.act_method, - pre_layer_norm=self.pre_layer_norm) + out = incubate_f.fused_feedforward( + x, + linear1_weight, + linear2_weight, + linear1_bias, + linear2_bias, + ln1_scale, + ln1_bias, + ln2_scale, + ln2_bias, + 0.0, + 0.0, + activation=self.act_method, + pre_layer_norm=self.pre_layer_norm, + ) paddle.autograd.backward([out], [paddle.to_tensor(self.dout)]) return out, x.grad @@ -145,18 +160,18 @@ def test_out_and_grad(self): default_main_program().random_seed = 42 base_out, base_grad = self.Base() fused_out, fused_grad = self.FusedFFN() - np.testing.assert_allclose(base_out.numpy(), - fused_out.numpy(), - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(base_grad.numpy(), - fused_grad.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + base_out.numpy(), fused_out.numpy(), rtol=self.rtol, atol=self.atol + ) + np.testing.assert_allclose( + base_grad.numpy(), + fused_grad.numpy(), + rtol=self.rtol, + atol=self.atol, + ) class TestFusedFFNOpFp16(TestFusedFFNOp): - def getDtype(self): self.dtype = "float16" self.layer_norm_dtype = "float32" @@ -173,20 +188,17 @@ def getShape(self): class TestFusedFFNOpFp64(TestFusedFFNOp): - def getDtype(self): self.dtype = "float64" self.layer_norm_dtype = "float64" class TestFusedFFNOpActivation(TestFusedFFNOp): - def getActivation(self): self.act_method = "relu" class TestFusedFFNOpNormalizeBefore(TestFusedFFNOp): - def getNormalizeBefore(self): self.pre_layer_norm = True @@ -198,7 +210,6 @@ def getShape(self): class APITestStaticFusedFFN(unittest.TestCase): - def test_static(self): paddle.enable_static() default_main_program().random_seed = 42 @@ -208,36 +219,39 @@ def test_static(self): d_model = 8 dim_feedforward = 8 - x = paddle.static.data(name='x', - shape=[batch_size, d_model, dim_feedforward], - dtype=dtype) - linear1_weight = paddle.static.data(name='linear1_weight', - shape=[d_model, dim_feedforward], - dtype=dtype) - linear1_bias = paddle.static.data(name='linear1_bias', - shape=[dim_feedforward]) - linear2_weight = paddle.static.data(name='linear2_weight', - shape=[dim_feedforward, d_model], - dtype=dtype) + x = paddle.static.data( + name='x', shape=[batch_size, d_model, dim_feedforward], dtype=dtype + ) + linear1_weight = paddle.static.data( + name='linear1_weight', shape=[d_model, dim_feedforward], dtype=dtype + ) + linear1_bias = paddle.static.data( + name='linear1_bias', shape=[dim_feedforward] + ) + linear2_weight = paddle.static.data( + name='linear2_weight', shape=[dim_feedforward, d_model], dtype=dtype + ) linear2_bias = paddle.static.data(name='linear2_bias', shape=[d_model]) ln1_scale = paddle.static.data(name='ln1_scale', shape=[d_model]) ln1_bias = paddle.static.data(name='ln1_scale', shape=[d_model]) ln2_scale = paddle.static.data(name='ln2_scale', shape=[d_model]) ln2_bias = paddle.static.data(name='ln2_scale', shape=[d_model]) - fused_out = incubate_f.fused_feedforward(x, - linear1_weight, - linear2_weight, - linear1_bias, - linear2_bias, - ln1_scale, - ln1_bias, - ln2_scale, - ln2_bias, - 0.0, - 0.0, - activation="relu", - pre_layer_norm=False) + fused_out = incubate_f.fused_feedforward( + x, + linear1_weight, + linear2_weight, + linear1_bias, + linear2_bias, + ln1_scale, + ln1_bias, + ln2_scale, + ln2_bias, + 0.0, + 0.0, + activation="relu", + pre_layer_norm=False, + ) ######base ffn###### linear1_out = F.linear(x, linear1_weight, linear1_bias) @@ -245,21 +259,26 @@ def test_static(self): dropout1_out = F.dropout(x=act_out, p=0.0, training=False) linear2_out = F.linear(dropout1_out, linear2_weight, linear2_bias) dropout2_out = x + F.dropout(x=linear2_out, p=0.0, training=False) - ln_out = F.layer_norm(dropout2_out, - normalized_shape=list([d_model]), - weight=ln2_scale, - bias=ln2_bias) + ln_out = F.layer_norm( + dropout2_out, + normalized_shape=list([d_model]), + weight=ln2_scale, + bias=ln2_bias, + ) ######base ffn###### exe = paddle.static.Executor(paddle.CUDAPlace(0)) x_data = np.random.random( - (batch_size, d_model, dim_feedforward)).astype(dtype) + (batch_size, d_model, dim_feedforward) + ).astype(dtype) linear1_weight_data = np.random.random( - (d_model, dim_feedforward)).astype(dtype) + (d_model, dim_feedforward) + ).astype(dtype) linear1_bias_data = np.zeros((dim_feedforward)).astype(dtype) linear2_weight_data = np.random.random( - (dim_feedforward, d_model)).astype(dtype) + (dim_feedforward, d_model) + ).astype(dtype) linear2_bias_data = np.zeros((d_model)).astype(dtype) ln1_scale_data = np.ones((d_model)).astype(layer_norm_dtype) @@ -271,94 +290,92 @@ def test_static(self): real_res = [] for res in res_list: - fetch = exe.run(feed={ - 'x': x_data, - 'linear1_weight': linear1_weight_data, - 'linear1_bias': linear1_bias_data, - 'linear2_weight': linear2_weight_data, - 'linear2_bias': linear2_bias_data, - 'ln1_scale': ln1_scale_data, - 'ln1_bias': ln1_bias_data, - 'ln2_scale': ln2_scale_data, - 'ln2_bias': ln2_bias_data - }, - fetch_list=[res]) + fetch = exe.run( + feed={ + 'x': x_data, + 'linear1_weight': linear1_weight_data, + 'linear1_bias': linear1_bias_data, + 'linear2_weight': linear2_weight_data, + 'linear2_bias': linear2_bias_data, + 'ln1_scale': ln1_scale_data, + 'ln1_bias': ln1_bias_data, + 'ln2_scale': ln2_scale_data, + 'ln2_bias': ln2_bias_data, + }, + fetch_list=[res], + ) real_res.append(fetch) - np.testing.assert_allclose(real_res[0], - real_res[1], - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + real_res[0], real_res[1], rtol=1e-05, atol=0.001 + ) class TestFusedFFNOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): def test_dtype(): - x = paddle.static.data(name='x', - shape=[1, 10, 10], - dtype="int32") - linear1_weight = paddle.static.data(name='linear1_weight', - shape=[1, 10, 10], - dtype="float32") - linear2_weight = paddle.static.data(name='linear2_weight', - shape=[1, 10, 10], - dtype="float32") + x = paddle.static.data( + name='x', shape=[1, 10, 10], dtype="int32" + ) + linear1_weight = paddle.static.data( + name='linear1_weight', shape=[1, 10, 10], dtype="float32" + ) + linear2_weight = paddle.static.data( + name='linear2_weight', shape=[1, 10, 10], dtype="float32" + ) incubate_f.fused_feedforward(x, linear1_weight, linear2_weight) self.assertRaises(TypeError, test_dtype) def test_dropout_rate_type(): - x = paddle.static.data(name='x1', - shape=[1, 10, 10], - dtype="float32") - linear1_weight = paddle.static.data(name='linear1_weight1', - shape=[10, 10], - dtype="float32") - linear2_weight = paddle.static.data(name='linear2_weight1', - shape=[10, 10], - dtype="float32") - incubate_f.fused_feedforward(x, - linear1_weight, - linear2_weight, - dropout1_rate="a") + x = paddle.static.data( + name='x1', shape=[1, 10, 10], dtype="float32" + ) + linear1_weight = paddle.static.data( + name='linear1_weight1', shape=[10, 10], dtype="float32" + ) + linear2_weight = paddle.static.data( + name='linear2_weight1', shape=[10, 10], dtype="float32" + ) + incubate_f.fused_feedforward( + x, linear1_weight, linear2_weight, dropout1_rate="a" + ) self.assertRaises(TypeError, test_dropout_rate_type) def test_dropout_rate_value(): - x = paddle.static.data(name='x2', - shape=[1, 10, 10], - dtype="float32") - linear1_weight = paddle.static.data(name='linear1_weight2', - shape=[10, 10], - dtype="float32") - linear2_weight = paddle.static.data(name='linear2_weight2', - shape=[10, 10], - dtype="float32") - incubate_f.fused_feedforward(x, - linear1_weight, - linear2_weight, - dropout2_rate=-1) + x = paddle.static.data( + name='x2', shape=[1, 10, 10], dtype="float32" + ) + linear1_weight = paddle.static.data( + name='linear1_weight2', shape=[10, 10], dtype="float32" + ) + linear2_weight = paddle.static.data( + name='linear2_weight2', shape=[10, 10], dtype="float32" + ) + incubate_f.fused_feedforward( + x, linear1_weight, linear2_weight, dropout2_rate=-1 + ) self.assertRaises(ValueError, test_dropout_rate_value) def test_dropout_mode(): - x = paddle.static.data(name='x3', - shape=[1, 10, 10], - dtype="float32") - linear1_weight = paddle.static.data(name='linear1_weight3', - shape=[10, 10], - dtype="float32") - linear2_weight = paddle.static.data(name='linear2_weight3', - shape=[10, 10], - dtype="float32") - incubate_f.fused_feedforward(x, - linear1_weight, - linear2_weight, - mode='test') + x = paddle.static.data( + name='x3', shape=[1, 10, 10], dtype="float32" + ) + linear1_weight = paddle.static.data( + name='linear1_weight3', shape=[10, 10], dtype="float32" + ) + linear2_weight = paddle.static.data( + name='linear2_weight3', shape=[10, 10], dtype="float32" + ) + incubate_f.fused_feedforward( + x, linear1_weight, linear2_weight, mode='test' + ) self.assertRaises(ValueError, test_dropout_mode) diff --git a/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py b/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py index 2baab279462d0..1eed59690d88d 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_gate_attention_op.py @@ -28,10 +28,10 @@ from paddle.fluid import core -@unittest.skipIf(not core.is_compiled_with_cuda(), - "Paddle is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "Paddle is not compiled with CUDA" +) class TestFusedGateAttentionOp(OpTest): - def setUp(self): self.__class__.op_type = "fused_gate_attention" # use autograd to check grad in this unittest. @@ -55,7 +55,6 @@ def config(self): self.bias_attr = True def generate_input_data(self): - def _random(shape): if self.dtype == "bfloat16": data = np.random.random(shape).astype("float32") @@ -65,7 +64,8 @@ def _random(shape): np.random.seed(123) self.query = _random( - (self.batch_size, self.msa_len, self.res_len, self.q_dim)) + (self.batch_size, self.msa_len, self.res_len, self.q_dim) + ) self.q_weight = _random((self.q_dim, self.num_heads, self.head_dim)) self.k_weight = _random((self.kv_dim, self.num_heads, self.head_dim)) self.v_weight = _random((self.kv_dim, self.num_heads, self.head_dim)) @@ -78,15 +78,18 @@ def _random(shape): self.qkv_weight = np.stack([q_weight_t, k_weight_t, v_weight_t]) else: self.key = _random( - (self.batch_size, self.msa_len, self.m_size, self.kv_dim)) + (self.batch_size, self.msa_len, self.m_size, self.kv_dim) + ) self.qkv_weight = None self.attn_mask = _random( - (self.batch_size, self.msa_len, 1, 1, self.m_size)) + (self.batch_size, self.msa_len, 1, 1, self.m_size) + ) if self.bias_attr: self.nonbatched_bias = _random( - (self.batch_size, 1, self.num_heads, self.res_len, self.m_size)) + (self.batch_size, 1, self.num_heads, self.res_len, self.m_size) + ) if self.has_gating: self.gating_w = _random((self.q_dim, self.num_heads, self.head_dim)) @@ -96,12 +99,17 @@ def _random(shape): self.output_b = _random((self.out_dim)) self.dout = _random( - (self.batch_size, self.msa_len, self.res_len, self.q_dim)) + (self.batch_size, self.msa_len, self.res_len, self.q_dim) + ) def collect_outputs(self, query, key, softmax_out, fmha_out, gate_out, out): outputs = [ - softmax_out, fmha_out, gate_out if self.has_gating else None, out, - query.grad, None if self.merge_qkv else key.grad + softmax_out, + fmha_out, + gate_out if self.has_gating else None, + out, + query.grad, + None if self.merge_qkv else key.grad, ] return outputs @@ -109,14 +117,17 @@ def get_reference_out(self): paddle.disable_static(place=paddle.CUDAPlace(0)) query = paddle.to_tensor(self.query, stop_gradient=False) - key = query if self.merge_qkv else paddle.to_tensor(self.key, - stop_gradient=False) + key = ( + query + if self.merge_qkv + else paddle.to_tensor(self.key, stop_gradient=False) + ) q_weight = paddle.to_tensor(self.q_weight, stop_gradient=False) k_weight = paddle.to_tensor(self.k_weight, stop_gradient=False) v_weight = paddle.to_tensor(self.v_weight, stop_gradient=False) src_mask = paddle.to_tensor(self.attn_mask, stop_gradient=True) - c = self.head_dim**(-0.5) + c = self.head_dim ** (-0.5) # [batch_size, msa_len, res_len, q_dim], [q_dim, num_heads, head_dim] # -> [batch_size, msa_len, res_len, num_heads, head_dim] q = paddle.einsum('nbqa,ahc->nbqhc', query, q_weight) * c @@ -134,8 +145,9 @@ def get_reference_out(self): # -> [batch_size, msa_len, num_heads, res_len, m_size] logits = logits + src_mask if self.bias_attr: - nonbatched_bias = paddle.to_tensor(self.nonbatched_bias, - stop_gradient=False) + nonbatched_bias = paddle.to_tensor( + self.nonbatched_bias, stop_gradient=False + ) # [batch_size, msa_len, num_heads, res_len, m_size], [batch_size, 1, num_heads, res_len, m_size] # -> [batch_size, msa_len, num_heads, res_len, m_size] logits = logits + nonbatched_bias @@ -157,14 +169,22 @@ def get_reference_out(self): # gate_values = paddle.einsum('nbqc,chv->nbqhv', query, # gating_w) + gating_b gating_w_2d = paddle.reshape( - gating_w, shape=[self.q_dim, self.num_heads * self.head_dim]) + gating_w, shape=[self.q_dim, self.num_heads * self.head_dim] + ) gate_values_4d = paddle.matmul(query, gating_w_2d) - gate_values = paddle.reshape( - gate_values_4d, - shape=[ - self.batch_size, self.msa_len, self.res_len, self.num_heads, - self.head_dim - ]) + gating_b + gate_values = ( + paddle.reshape( + gate_values_4d, + shape=[ + self.batch_size, + self.msa_len, + self.res_len, + self.num_heads, + self.head_dim, + ], + ) + + gating_b + ) gate_values = nn.functional.sigmoid(gate_values) gate_out = fmha_out * gate_values else: @@ -181,20 +201,32 @@ def get_reference_out(self): gate_out, shape=[ self.batch_size * self.msa_len * self.res_len, - self.num_heads * self.head_dim - ]) + self.num_heads * self.head_dim, + ], + ) output_w_2d = paddle.reshape( - output_w, shape=[self.num_heads * self.head_dim, self.out_dim]) + output_w, shape=[self.num_heads * self.head_dim, self.out_dim] + ) out_2d = paddle.matmul(gate_out_2d, output_w_2d) - out = paddle.reshape( - out_2d, - shape=[self.batch_size, self.msa_len, self.res_len, self.out_dim - ]) + output_b - - paddle.autograd.backward([out], [paddle.to_tensor(self.dout)], - retain_graph=True) - return self.collect_outputs(query, key, softmax_out, fmha_out, gate_out, - out) + out = ( + paddle.reshape( + out_2d, + shape=[ + self.batch_size, + self.msa_len, + self.res_len, + self.out_dim, + ], + ) + + output_b + ) + + paddle.autograd.backward( + [out], [paddle.to_tensor(self.dout)], retain_graph=True + ) + return self.collect_outputs( + query, key, softmax_out, fmha_out, gate_out, out + ) def get_fused_gate_attention_out(self): paddle.disable_static(place=paddle.CUDAPlace(0)) @@ -216,8 +248,9 @@ def get_fused_gate_attention_out(self): src_mask = paddle.to_tensor(self.attn_mask, stop_gradient=True) if self.bias_attr: - nonbatched_bias = paddle.to_tensor(self.nonbatched_bias, - stop_gradient=False) + nonbatched_bias = paddle.to_tensor( + self.nonbatched_bias, stop_gradient=False + ) else: nonbatched_bias = None if self.has_gating: @@ -230,18 +263,42 @@ def get_fused_gate_attention_out(self): output_w = paddle.to_tensor(self.output_w, stop_gradient=False) output_b = paddle.to_tensor(self.output_b, stop_gradient=False) - _, _, _, _, softmax_out, fmha_out, gate_out, out = _legacy_C_ops.fused_gate_attention( - query, key, q_weight, k_weight, v_weight, qkv_weight, - nonbatched_bias, src_mask, gating_w, gating_b, output_w, output_b, - 'has_gating', self.has_gating, 'merge_qkv', self.merge_qkv) - - paddle.autograd.backward([out], [paddle.to_tensor(self.dout)], - retain_graph=True) - return self.collect_outputs(query, key, softmax_out, fmha_out, gate_out, - out) + ( + _, + _, + _, + _, + softmax_out, + fmha_out, + gate_out, + out, + ) = _legacy_C_ops.fused_gate_attention( + query, + key, + q_weight, + k_weight, + v_weight, + qkv_weight, + nonbatched_bias, + src_mask, + gating_w, + gating_b, + output_w, + output_b, + 'has_gating', + self.has_gating, + 'merge_qkv', + self.merge_qkv, + ) + + paddle.autograd.backward( + [out], [paddle.to_tensor(self.dout)], retain_graph=True + ) + return self.collect_outputs( + query, key, softmax_out, fmha_out, gate_out, out + ) def check(self, ref, out, atol, rtol, check_equal, name): - def _convert(value): if self.dtype == "bfloat16": return convert_uint16_to_float(value) @@ -250,19 +307,25 @@ def _convert(value): if check_equal: self.assertTrue( np.equal(_convert(ref), _convert(out)).all(), - "Checking < {} > failed!".format(name)) + "Checking < {} > failed!".format(name), + ) else: np.testing.assert_allclose( _convert(ref), _convert(out), atol=atol, rtol=rtol, - err_msg="Checking < {} > failed!".format(name)) + err_msg="Checking < {} > failed!".format(name), + ) def check_output_and_grad(self, atol, rtol): output_names = [ - "softmax_out", "fmha_out", "gate_out", "out", "query_grad", - "key_grad" + "softmax_out", + "fmha_out", + "gate_out", + "out", + "query_grad", + "key_grad", ] outputs_ref = self.get_reference_out() outputs_fused = self.get_fused_gate_attention_out() @@ -278,22 +341,26 @@ def check_output_and_grad(self, atol, rtol): # that in fused ops, check_equal is set to False and we use allclose # to check the correctness. check_equal = False - self.check(ref_res.numpy(), fused_res.numpy(), atol, rtol, - check_equal, output_names[i]) + self.check( + ref_res.numpy(), + fused_res.numpy(), + atol, + rtol, + check_equal, + output_names[i], + ) def test_output_and_grad(self): self.check_output_and_grad(atol=1e-5, rtol=1e-6) class TestMergeQKVLargeBatchSizeCase(TestFusedGateAttentionOp): - def config(self): super().config() self.batch_size = 2 class TestSeparatedQKVCase(TestFusedGateAttentionOp): - def config(self): self.dtype = "float32" self.has_gating = False @@ -310,7 +377,6 @@ def config(self): class TestMergeQKVNoBiasGatingCase(TestFusedGateAttentionOp): - def config(self): super().config() self.has_gating = False @@ -318,7 +384,6 @@ def config(self): class TestMergeQKVFp16Case(TestFusedGateAttentionOp): - def config(self): super().config() self.dtype = "float16" @@ -330,7 +395,6 @@ def test_output_and_grad(self): class TestMergeQKVLargeBatchSizeFp16Case(TestMergeQKVFp16Case): - def config(self): super().config() self.batch_size = 2 @@ -338,10 +402,9 @@ def config(self): @unittest.skipIf( not core.is_compiled_with_cuda() or get_cuda_version() < 11000, - "core is not compiled with CUDA and cuda version need larger than or equal to 11.3" + "core is not compiled with CUDA and cuda version need larger than or equal to 11.3", ) class TestMergeQKVBF16Case(TestFusedGateAttentionOp): - def config(self): super().config() self.dtype = "bfloat16" @@ -351,7 +414,6 @@ def test_output_and_grad(self): class TestMergeQKVLargeBatchSizeBF16Case(TestMergeQKVBF16Case): - def config(self): super().config() self.batch_size = 2 diff --git a/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_grad_op.py b/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_grad_op.py index ceefa72631c2e..322e63dd4fd45 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_grad_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_grad_op.py @@ -29,10 +29,10 @@ def get_outputs(DOut, X, Y): @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDXYBiasFP16(OpTest): - def setUp(self): self.op_type = "fused_gemm_epilogue_grad" self.place = core.CUDAPlace(0) @@ -41,13 +41,14 @@ def setUp(self): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - DX, DY, DBias = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + DX, DY, DBias = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DX': DX, 'DY': DY, 'DBias': DBias} def init_dtype_type(self): @@ -56,38 +57,41 @@ def init_dtype_type(self): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDXYBiasFP32( - TestFuseGemmEpilogueGradOpDXYBiasFP16): - + TestFuseGemmEpilogueGradOpDXYBiasFP16 +): def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDXYBiasFP64( - TestFuseGemmEpilogueGradOpDXYBiasFP16): - + TestFuseGemmEpilogueGradOpDXYBiasFP16 +): def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDYBiasFP16(OpTest): - def setUp(self): self.op_type = "fused_gemm_epilogue_grad" self.place = core.CUDAPlace(0) @@ -96,13 +100,14 @@ def setUp(self): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - _, DY, DBias = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + _, DY, DBias = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DY': DY, 'DBias': DBias} def init_dtype_type(self): @@ -111,38 +116,41 @@ def init_dtype_type(self): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") -class TestFuseGemmEpilogueGradOpDYBiasFP32(TestFuseGemmEpilogueGradOpDYBiasFP16 - ): - +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) +class TestFuseGemmEpilogueGradOpDYBiasFP32( + TestFuseGemmEpilogueGradOpDYBiasFP16 +): def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") -class TestFuseGemmEpilogueGradOpDYBiasFP64(TestFuseGemmEpilogueGradOpDYBiasFP16 - ): - +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) +class TestFuseGemmEpilogueGradOpDYBiasFP64( + TestFuseGemmEpilogueGradOpDYBiasFP16 +): def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDYFP16(OpTest): - def setUp(self): self.op_type = "fused_gemm_epilogue_grad" self.place = core.CUDAPlace(0) @@ -151,13 +159,14 @@ def setUp(self): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - _, DY, _ = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + _, DY, _ = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DY': DY} def init_dtype_type(self): @@ -166,36 +175,37 @@ def init_dtype_type(self): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDYFP32(TestFuseGemmEpilogueGradOpDYFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDYFP64(TestFuseGemmEpilogueGradOpDYFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDXYFP16(OpTest): - def setUp(self): self.op_type = "fused_gemm_epilogue_grad" self.place = core.CUDAPlace(0) @@ -204,13 +214,14 @@ def setUp(self): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - DX, DY, _ = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + DX, DY, _ = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DX': DX, 'DY': DY} def init_dtype_type(self): @@ -219,26 +230,27 @@ def init_dtype_type(self): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDXYFP32(TestFuseGemmEpilogueGradOpDXYFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueGradOpDXYFP64(TestFuseGemmEpilogueGradOpDXYFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 diff --git a/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_op.py b/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_op.py index 66ea9d7fa833c..a3af9cc194f73 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_gemm_epilogue_op.py @@ -28,8 +28,11 @@ def is_fused_gemm_epilogue_supported(): def gelu(x): - y_ref = 0.5 * x * ( - 1.0 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)))) + y_ref = ( + 0.5 + * x + * (1.0 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)))) + ) return y_ref.astype(x.dtype) @@ -54,10 +57,10 @@ class TestFuseGemmBase(OpTest): @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMFP16(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -66,12 +69,12 @@ def setUp(self): self.inputs = { 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.outputs = { - 'Out': - get_output(self.inputs['X'], self.inputs['Y'], self.inputs['Bias'], - 'relu') + 'Out': get_output( + self.inputs['X'], self.inputs['Y'], self.inputs['Bias'], 'relu' + ) } self.attrs = {"activation": 'relu'} @@ -81,36 +84,37 @@ def init_dtype_type(self): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMFP32(TestFuseGemmEpilogueOpReluMMFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMFP64(TestFuseGemmEpilogueOpReluMMFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMFP16(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -119,12 +123,15 @@ def setUp(self): self.inputs = { 'X': np.random.random((4, 8)).astype(self.dtype) - 0.5, 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.outputs = { - 'Out': - get_output(self.inputs['X'].T, self.inputs['Y'], - self.inputs['Bias'], 'relu') + 'Out': get_output( + self.inputs['X'].T, + self.inputs['Y'], + self.inputs['Bias'], + 'relu', + ) } self.attrs = {'trans_x': True, "activation": 'relu'} @@ -134,36 +141,37 @@ def init_dtype_type(self): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMFP32(TestFuseGemmEpilogueOpReluMTMFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMFP64(TestFuseGemmEpilogueOpReluMTMFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMTFP16(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -172,12 +180,15 @@ def setUp(self): self.inputs = { 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, 'Y': np.random.random((128, 4)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.outputs = { - 'Out': - get_output(self.inputs['X'], self.inputs['Y'].T, - self.inputs['Bias'], 'relu') + 'Out': get_output( + self.inputs['X'], + self.inputs['Y'].T, + self.inputs['Bias'], + 'relu', + ) } self.attrs = {'trans_y': True, "activation": 'relu'} @@ -187,36 +198,37 @@ def init_dtype_type(self): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMTFP32(TestFuseGemmEpilogueOpReluMMTFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMTFP64(TestFuseGemmEpilogueOpReluMMTFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMTFP16(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -225,12 +237,15 @@ def setUp(self): self.inputs = { 'X': np.random.random((4, 8)).astype(self.dtype) - 0.5, 'Y': np.random.random((128, 4)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.outputs = { - 'Out': - get_output(self.inputs['X'].T, self.inputs['Y'].T, - self.inputs['Bias'], 'relu') + 'Out': get_output( + self.inputs['X'].T, + self.inputs['Y'].T, + self.inputs['Bias'], + 'relu', + ) } self.attrs = {'trans_x': True, 'trans_y': True, "activation": 'relu'} @@ -240,36 +255,37 @@ def init_dtype_type(self): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMTFP32(TestFuseGemmEpilogueOpReluMTMTFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMTFP64(TestFuseGemmEpilogueOpReluMTMTFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMFP16MultiDimX(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -278,12 +294,15 @@ def setUp(self): self.inputs = { 'X': np.random.random((2, 2, 8, 4)).astype(self.dtype) - 0.5, 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.outputs = { - 'Out': - get_output(self.inputs['X'].reshape((-1, 4)), self.inputs['Y'], - self.inputs['Bias'], 'relu').reshape((2, 2, 8, 128)) + 'Out': get_output( + self.inputs['X'].reshape((-1, 4)), + self.inputs['Y'], + self.inputs['Bias'], + 'relu', + ).reshape((2, 2, 8, 128)) } self.attrs = {"activation": 'relu'} @@ -293,38 +312,41 @@ def init_dtype_type(self): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMFP32MultiDimX( - TestFuseGemmEpilogueOpReluMMFP16MultiDimX): - + TestFuseGemmEpilogueOpReluMMFP16MultiDimX +): def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMMFP64MultiDimX( - TestFuseGemmEpilogueOpReluMMFP16MultiDimX): - + TestFuseGemmEpilogueOpReluMMFP16MultiDimX +): def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMFP16MultiDimX(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -333,12 +355,15 @@ def setUp(self): self.inputs = { 'X': np.random.random((4, 2, 2, 8)).astype(self.dtype) - 0.5, 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.outputs = { - 'Out': - get_output(self.inputs['X'].reshape((4, -1)).T, self.inputs['Y'], - self.inputs['Bias'], 'relu').reshape((2, 2, 8, 128)) + 'Out': get_output( + self.inputs['X'].reshape((4, -1)).T, + self.inputs['Y'], + self.inputs['Bias'], + 'relu', + ).reshape((2, 2, 8, 128)) } self.attrs = {'trans_x': True, "activation": 'relu'} @@ -348,38 +373,41 @@ def init_dtype_type(self): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMFP32MultiDimX( - TestFuseGemmEpilogueOpReluMTMFP16MultiDimX): - + TestFuseGemmEpilogueOpReluMTMFP16MultiDimX +): def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpReluMTMFP64MultiDimX( - TestFuseGemmEpilogueOpReluMTMFP16MultiDimX): - + TestFuseGemmEpilogueOpReluMTMFP16MultiDimX +): def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpGeluMMFP16(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -388,15 +416,15 @@ def setUp(self): self.inputs = { 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'gelu'} self.outputs = { - 'Out': - get_output(self.inputs['X'], self.inputs['Y'], self.inputs['Bias'], - 'gelu') + 'Out': get_output( + self.inputs['X'], self.inputs['Y'], self.inputs['Bias'], 'gelu' + ) } def init_dtype_type(self): @@ -405,36 +433,37 @@ def init_dtype_type(self): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpGeluMMFP32(TestFuseGemmEpilogueOpGeluMMFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpGeluMMFP64(TestFuseGemmEpilogueOpGeluMMFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpNoneMMFP16(TestFuseGemmBase): - def setUp(self): self.op_type = "fused_gemm_epilogue" self.place = core.CUDAPlace(0) @@ -443,15 +472,15 @@ def setUp(self): self.inputs = { 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, - 'Bias': np.random.random((128, )).astype(self.dtype) - 0.5 + 'Bias': np.random.random((128,)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} self.outputs = { - 'Out': - get_output(self.inputs['X'], self.inputs['Y'], self.inputs['Bias'], - 'none') + 'Out': get_output( + self.inputs['X'], self.inputs['Y'], self.inputs['Bias'], 'none' + ) } def init_dtype_type(self): @@ -460,26 +489,27 @@ def init_dtype_type(self): def test_check_output(self): if self.dtype == np.float16 and not core.is_float16_supported( - self.place): + self.place + ): return self.check_output_with_place(self.place, atol=self.atol) @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpNoneMMFP32(TestFuseGemmEpilogueOpNoneMMFP16): - def init_dtype_type(self): self.dtype = np.single self.atol = 1e-6 @skip_check_grad_ci(reason="no grap op") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFuseGemmEpilogueOpNoneMMFP64(TestFuseGemmEpilogueOpNoneMMFP16): - def init_dtype_type(self): self.dtype = np.double self.atol = 1e-6 @@ -522,9 +552,9 @@ def matmul_grad(x, y, bias, dz, trans_x, trans_y): @unittest.skipIf( not is_fused_gemm_epilogue_supported(), - "fused_gemm_epilogue is only supported when CUDA version >= 11.6") + "fused_gemm_epilogue is only supported when CUDA version >= 11.6", +) class TestEagerFusedGemmEpilogue(unittest.TestCase): - def setUp(self): paddle.set_device('gpu') @@ -532,22 +562,22 @@ def test_case_act(self): paddle.disable_static() x_np = np.random.random((8, 4)).astype(np.float64) - 0.5 y_np = np.random.random((4, 128)).astype(np.float64) - 0.5 - bias_np = np.random.random((128, )).astype(np.float64) - 0.5 + bias_np = np.random.random((128,)).astype(np.float64) - 0.5 x = paddle.to_tensor(x_np) y = paddle.to_tensor(y_np) bias = paddle.to_tensor(bias_np) x.stop_gradient = False y.stop_gradient = False - out1 = core.eager.ops.fused_gemm_epilogue(x, y, bias, 'trans_x', False, - 'trans_y', False, - 'activation', 'none') - out2 = core.eager.ops.fused_gemm_epilogue(x, y, bias, 'trans_x', False, - 'trans_y', False, - 'activation', 'relu') - out3 = core.eager.ops.fused_gemm_epilogue(x, y, bias, 'trans_x', False, - 'trans_y', False, - 'activation', 'gelu') + out1 = core.eager.ops.fused_gemm_epilogue( + x, y, bias, 'trans_x', False, 'trans_y', False, 'activation', 'none' + ) + out2 = core.eager.ops.fused_gemm_epilogue( + x, y, bias, 'trans_x', False, 'trans_y', False, 'activation', 'relu' + ) + out3 = core.eager.ops.fused_gemm_epilogue( + x, y, bias, 'trans_x', False, 'trans_y', False, 'activation', 'gelu' + ) out_np1 = get_output(x_np, y_np, bias_np, 'none') out_np2 = get_output(x_np, y_np, bias_np, 'relu') @@ -557,13 +587,16 @@ def test_case_act(self): np.testing.assert_allclose(out2, out_np2, rtol=1e-05) np.testing.assert_allclose(out3, out_np3, rtol=1e-05) - out_grad_np1 = np.random.randint(low=-20, high=20, - size=out_np1.shape).astype(np.float64) - paddle.autograd.backward(out1, - grad_tensors=[paddle.to_tensor(out_grad_np1)]) + out_grad_np1 = np.random.randint( + low=-20, high=20, size=out_np1.shape + ).astype(np.float64) + paddle.autograd.backward( + out1, grad_tensors=[paddle.to_tensor(out_grad_np1)] + ) x_grad_np, y_grad_np, bias_grad_np = matmul_grad( - x_np, y_np, bias_np, out_grad_np1, False, False) + x_np, y_np, bias_np, out_grad_np1, False, False + ) np.testing.assert_allclose(x.grad.numpy(), x_grad_np, rtol=1e-05) self.assertEqual(y_grad_np.shape, y_np.shape) np.testing.assert_allclose(y.grad.numpy(), y_grad_np, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_fused_matmul_bias.py b/python/paddle/fluid/tests/unittests/test_fused_matmul_bias.py index b26b5e2f04634..a4d152c75b997 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_matmul_bias.py +++ b/python/paddle/fluid/tests/unittests/test_fused_matmul_bias.py @@ -64,9 +64,9 @@ def matmul_grad(x, y, bias, dz, trans_x, trans_y): @unittest.skipIf( not is_fused_matmul_bias_supported(), - "fused_gemm_epilogue is only supported when CUDA version >= 11.6") + "fused_gemm_epilogue is only supported when CUDA version >= 11.6", +) class TestFusedMatmulBias(unittest.TestCase): - def setUp(self): paddle.set_device('gpu') @@ -103,7 +103,8 @@ def rand_test_base(self, m, n, k, trans_x, trans_y, need_bias, dtype, seed): paddle.autograd.backward(z, grad_tensors=[paddle.to_tensor(z_grad_np)]) x_grad_np, y_grad_np, bias_grad_np = matmul_grad( - x_np, y_np, bias_np, z_grad_np, trans_x, trans_y) + x_np, y_np, bias_np, z_grad_np, trans_x, trans_y + ) np.testing.assert_array_equal(x.grad.numpy(), x_grad_np) self.assertEqual(y_grad_np.shape, y_np.shape) np.testing.assert_array_equal(y.grad.numpy(), y_grad_np) @@ -118,8 +119,9 @@ def rand_test(self, m, n, k, dtype): for trans_x in [False, True]: for trans_y in [False, True]: for need_bias in [False, True]: - self.rand_test_base(m, n, k, trans_x, trans_y, need_bias, - dtype, seed) + self.rand_test_base( + m, n, k, trans_x, trans_y, need_bias, dtype, seed + ) def test_fp32(self): self.rand_test(30, 40, 50, np.float32) @@ -130,9 +132,9 @@ def test_fp16(self): @unittest.skipIf( not is_fused_matmul_bias_supported(), - "fused_gemm_epilogue is only supported when CUDA version >= 11.6") + "fused_gemm_epilogue is only supported when CUDA version >= 11.6", +) class TestFusedLinear(unittest.TestCase): - def check_fused_linear(self, transpose): x = paddle.randn([30, 40]) linear = FusedLinear(40, 50, transpose_weight=transpose) @@ -149,9 +151,9 @@ def test_transpose(self): @unittest.skipIf( not is_fused_matmul_bias_supported(), - "fused_gemm_epilogue is only supported when CUDA version >= 11.6") + "fused_gemm_epilogue is only supported when CUDA version >= 11.6", +) class TestStaticGraph(unittest.TestCase): - def test_static_graph(self): paddle.enable_static() x = paddle.static.data(name='x', dtype='float32', shape=[-1, 100]) diff --git a/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_int8_op.py b/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_int8_op.py index 3384e86668eae..e3fb41412fd39 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_int8_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_int8_op.py @@ -68,27 +68,69 @@ def fused_multi_transformer_int8( ffn1_in_scale=[], ffn2_in_scale=[], ): - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer cache_kv_out, final_out = _legacy_C_ops.fused_multi_transformer_int8( - x, ln_scales, ln_biases, qkv_weights, qkv_biases, cache_kvs, time_step, - attn_mask, linear_weights, linear_biases, ffn_ln_scales, ffn_ln_biases, - ffn1_weights, ffn1_biases, ffn2_weights, ffn2_biases, qkv_out_scales, - out_linear_out_scales, ffn1_out_scales, ffn2_out_scales, cache_kvs, - 'num_head', num_head, 'dim_head', dim_head, 'dim_ffn', dim_ffn, - 'qkv_in_scale', qkv_in_scale, 'out_linear_in_scale', - out_linear_in_scale, 'ffn1_in_scale', ffn1_in_scale, 'ffn2_in_scale', - ffn2_in_scale, 'pre_layer_norm', pre_layer_norm, 'epsilon', epsilon, - 'dropout_rate', dropout_rate, 'is_test', not training, - 'dropout_implementation', mode, 'act_method', activation, 'trans_qkvw', - trans_qkvw, 'ring_id', ring_id) + x, + ln_scales, + ln_biases, + qkv_weights, + qkv_biases, + cache_kvs, + time_step, + attn_mask, + linear_weights, + linear_biases, + ffn_ln_scales, + ffn_ln_biases, + ffn1_weights, + ffn1_biases, + ffn2_weights, + ffn2_biases, + qkv_out_scales, + out_linear_out_scales, + ffn1_out_scales, + ffn2_out_scales, + cache_kvs, + 'num_head', + num_head, + 'dim_head', + dim_head, + 'dim_ffn', + dim_ffn, + 'qkv_in_scale', + qkv_in_scale, + 'out_linear_in_scale', + out_linear_in_scale, + 'ffn1_in_scale', + ffn1_in_scale, + 'ffn2_in_scale', + ffn2_in_scale, + 'pre_layer_norm', + pre_layer_norm, + 'epsilon', + epsilon, + 'dropout_rate', + dropout_rate, + 'is_test', + not training, + 'dropout_implementation', + mode, + 'act_method', + activation, + 'trans_qkvw', + trans_qkvw, + 'ring_id', + ring_id, + ) if cache_kvs is not None: return final_out, cache_kv_out return final_out class TestFusedMultiTransformerInt8Op(unittest.TestCase): - def setUp(self): self.config() self.generate_input_data() @@ -110,12 +152,12 @@ def setUp(self): self.__class__.no_need_check_grad = True paddle.set_default_dtype(np.float32) - self.norm = LayerNorm(self.embed_dim, - weight_attr=False, - bias_attr=False) - self.ffn_norm = LayerNorm(self.embed_dim, - weight_attr=False, - bias_attr=False) + self.norm = LayerNorm( + self.embed_dim, weight_attr=False, bias_attr=False + ) + self.ffn_norm = LayerNorm( + self.embed_dim, weight_attr=False, bias_attr=False + ) paddle.set_default_dtype(self.x_type) self.dropout = Dropout(self.dropout_prob, mode="upscale_in_train") @@ -127,7 +169,7 @@ def config(self): self.x_type = np.float32 self.attn_mask_type = np.float64 - #self.attn_mask_type = np.bool + # self.attn_mask_type = np.bool self.pre_layer_norm = True self.has_attn_mask = True @@ -154,31 +196,38 @@ def config(self): self.weight_attr = None self.bias_attr = None self.kdim, self.vdim = self.embed_dim, self.embed_dim - self.key_length, self.value_length = self.query_length, self.query_length + self.key_length, self.value_length = ( + self.query_length, + self.query_length, + ) def generate_input_data(self): - self.query = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) - q_weight = np.random.randint(-64, 64, [self.embed_dim, self.embed_dim], - np.int32).astype('float64') - k_weight = np.random.randint(-64, 64, [self.kdim, self.embed_dim], - np.int32).astype('float64') - v_weight = np.random.randint(-64, 64, [self.vdim, self.embed_dim], - np.int32).astype('float64') + self.query = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) + q_weight = np.random.randint( + -64, 64, [self.embed_dim, self.embed_dim], np.int32 + ).astype('float64') + k_weight = np.random.randint( + -64, 64, [self.kdim, self.embed_dim], np.int32 + ).astype('float64') + v_weight = np.random.randint( + -64, 64, [self.vdim, self.embed_dim], np.int32 + ).astype('float64') self.q_weight_tensor = paddle.to_tensor(q_weight) self.k_weight_tensor = paddle.to_tensor(k_weight) self.v_weight_tensor = paddle.to_tensor(v_weight) - out_weight = np.random.randint(-64, 64, - [self.embed_dim, self.embed_dim], - np.int32).astype('float64') - ffn1_weight = np.random.randint(-64, 64, - [self.embed_dim, 4 * self.embed_dim], - np.int32).astype('float64') - ffn2_weight = np.random.randint(-64, 64, - [4 * self.embed_dim, self.embed_dim], - np.int32).astype('float64') + out_weight = np.random.randint( + -64, 64, [self.embed_dim, self.embed_dim], np.int32 + ).astype('float64') + ffn1_weight = np.random.randint( + -64, 64, [self.embed_dim, 4 * self.embed_dim], np.int32 + ).astype('float64') + ffn2_weight = np.random.randint( + -64, 64, [4 * self.embed_dim, self.embed_dim], np.int32 + ).astype('float64') self.out_weight_tensor = paddle.to_tensor(out_weight) self.ffn1_weight_tensor = paddle.to_tensor(ffn1_weight) @@ -193,12 +242,14 @@ def generate_input_data(self): self.v_proj_bias_tensor = paddle.to_tensor(v_proj_bias) out_linear_proj_bias = np.random.rand(self.embed_dim).astype( - self.x_type) + self.x_type + ) ffn1_proj_bias = np.random.rand(4 * self.embed_dim).astype(self.x_type) ffn2_proj_bias = np.random.rand(self.embed_dim).astype(self.x_type) self.out_linear_proj_bias_tensor = paddle.to_tensor( - out_linear_proj_bias) + out_linear_proj_bias + ) self.ffn1_proj_bias_tensor = paddle.to_tensor(ffn1_proj_bias) self.ffn2_proj_bias_tensor = paddle.to_tensor(ffn2_proj_bias) @@ -214,9 +265,13 @@ def generate_input_data(self): self.ffn2_out_scales = [] if self.has_cache_kv: - self.cache_kv = np.random.rand(2, self.batch_size, self.num_heads, - self.cache_length, - self.head_dim).astype(self.x_type) + self.cache_kv = np.random.rand( + 2, + self.batch_size, + self.num_heads, + self.cache_length, + self.head_dim, + ).astype(self.x_type) if self.gen_cache_kv: self.cache_kv[:] = 0 @@ -229,7 +284,8 @@ def generate_input_data(self): # [B, n_head, seq_len, out_seq_len] self.attn_mask = np.ones( (self.batch_size, 1, self.query_length, out_seq_len), - dtype=self.attn_mask_type) + dtype=self.attn_mask_type, + ) if self.attn_mask_type == np.int64: self.attn_mask = np.tril(self.attn_mask) elif self.attn_mask_type == np.float64: @@ -246,7 +302,8 @@ def generate_input_data(self): self.attn_mask = np.tril(self.attn_mask) else: raise ValueError( - "'attn_mask_type' should be 'int64' or 'float64'.") + "'attn_mask_type' should be 'int64' or 'float64'." + ) else: self.attn_mask = None @@ -289,8 +346,11 @@ def GetBaselineOut(self): q = paddle.nn.functional.linear(ln1_out, self.q_weight_tensor) # de quant q = paddle.cast( - paddle.cast(q, 'float32') * self.qkv_in_scales[i] / - self.qkv_out_scales[i], self.x_type) + paddle.cast(q, 'float32') + * self.qkv_in_scales[i] + / self.qkv_out_scales[i], + self.x_type, + ) q = q + self.q_proj_bias_tensor q = tensor.reshape(x=q, shape=[0, 0, self.num_heads, self.head_dim]) @@ -298,13 +358,19 @@ def GetBaselineOut(self): k = paddle.nn.functional.linear(ln1_out, self.k_weight_tensor) k = paddle.cast( - paddle.cast(k, 'float32') * self.qkv_in_scales[i] / - self.qkv_out_scales[i], self.x_type) + paddle.cast(k, 'float32') + * self.qkv_in_scales[i] + / self.qkv_out_scales[i], + self.x_type, + ) k = k + self.k_proj_bias_tensor v = paddle.nn.functional.linear(ln1_out, self.v_weight_tensor) v = paddle.cast( - paddle.cast(v, 'float32') * self.qkv_in_scales[i] / - self.qkv_out_scales[i], self.x_type) + paddle.cast(v, 'float32') + * self.qkv_in_scales[i] + / self.qkv_out_scales[i], + self.x_type, + ) v = v + self.v_proj_bias_tensor k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim]) @@ -332,10 +398,9 @@ def GetBaselineOut(self): # [B, n_head, seq_len, head_dim] * [B, n_head, out_seq_len, head_dim] # --> [B, n_head, seq_len, out_seq_len] - qk_out = layers.matmul(x=q_out, - y=k_out, - transpose_y=True, - alpha=self.head_dim**-0.5) + qk_out = layers.matmul( + x=q_out, y=k_out, transpose_y=True, alpha=self.head_dim**-0.5 + ) if self.debug: print('qk out is') @@ -355,10 +420,12 @@ def GetBaselineOut(self): print('softmax out is') print(softmax_out[0][0][0]) if self.dropout_prob: - dropout_out = F.dropout(softmax_out, - self.dropout_prob, - training=self.training, - mode="upscale_in_train") + dropout_out = F.dropout( + softmax_out, + self.dropout_prob, + training=self.training, + mode="upscale_in_train", + ) # [B, n_head, seq_len, out_seq_len] * [B, n_head, out_seq_len, head_dim] # --> [B, n_head, seq_len, head_dim] qktv_out = tensor.matmul(dropout_out, v_out) @@ -370,23 +437,30 @@ def GetBaselineOut(self): print('fmha out is') print(fmha_out[0][0][0]) out_linear_in = tensor.reshape( - x=fmha_out, shape=[0, 0, fmha_out.shape[2] * fmha_out.shape[3]]) + x=fmha_out, shape=[0, 0, fmha_out.shape[2] * fmha_out.shape[3]] + ) - max_v = paddle.max(paddle.abs(paddle.cast(out_linear_in, - 'float32')))[0] + max_v = paddle.max( + paddle.abs(paddle.cast(out_linear_in, 'float32')) + )[0] # self.out_linear_in_scales.append(127.0 / max_v) self.out_linear_in_scales.append(max_v) self.out_linear_out_scales.append((127.0 * 127.0)) - out_linear_in = self.fake_quant(out_linear_in, - self.out_linear_in_scales[i]) + out_linear_in = self.fake_quant( + out_linear_in, self.out_linear_in_scales[i] + ) - out = paddle.nn.functional.linear(out_linear_in, - self.out_weight_tensor) + out = paddle.nn.functional.linear( + out_linear_in, self.out_weight_tensor + ) out = paddle.cast( - paddle.cast(out, 'float32') * self.out_linear_in_scales[i] / - self.out_linear_out_scales[i], self.x_type) + paddle.cast(out, 'float32') + * self.out_linear_in_scales[i] + / self.out_linear_out_scales[i], + self.x_type, + ) out = out + self.out_linear_proj_bias_tensor @@ -400,18 +474,23 @@ def GetBaselineOut(self): if self.pre_layer_norm: ffn_ln_out = self.ffn_norm(attn_out) - max_v = paddle.max(paddle.abs(paddle.cast(ffn_ln_out, - 'float32')))[0] + max_v = paddle.max(paddle.abs(paddle.cast(ffn_ln_out, 'float32')))[ + 0 + ] self.ffn1_in_scales.append(max_v) self.ffn1_out_scales.append((127.0 * 127.0)) ffn_ln_out = self.fake_quant(ffn_ln_out, self.ffn1_in_scales[i]) - ffn1_out = paddle.nn.functional.linear(ffn_ln_out, - self.ffn1_weight_tensor) + ffn1_out = paddle.nn.functional.linear( + ffn_ln_out, self.ffn1_weight_tensor + ) ffn1_out = paddle.cast( - paddle.cast(ffn1_out, 'float32') * self.ffn1_in_scales[i] / - self.ffn1_out_scales[i], self.x_type) + paddle.cast(ffn1_out, 'float32') + * self.ffn1_in_scales[i] + / self.ffn1_out_scales[i], + self.x_type, + ) ffn1_out = ffn1_out + self.ffn1_proj_bias_tensor ffn1_out = self.dropout(self.activation(ffn1_out)) @@ -423,12 +502,16 @@ def GetBaselineOut(self): # print('ffn2_in_scales ', i, self.ffn2_in_scales[i]) ffn1_out = self.fake_quant(ffn1_out, self.ffn2_in_scales[i]) - ffn2_out = paddle.nn.functional.linear(ffn1_out, - self.ffn2_weight_tensor) + ffn2_out = paddle.nn.functional.linear( + ffn1_out, self.ffn2_weight_tensor + ) ffn2_out = paddle.cast( - paddle.cast(ffn2_out, 'float32') * self.ffn2_in_scales[i] / - self.ffn2_out_scales[i], self.x_type) + paddle.cast(ffn2_out, 'float32') + * self.ffn2_in_scales[i] + / self.ffn2_out_scales[i], + self.x_type, + ) ffn2_out = ffn2_out + self.ffn2_proj_bias_tensor residual_out = attn_out + self.dropout(ffn2_out) @@ -456,26 +539,35 @@ def GetFusedMultiTransformerOut(self): k_proj_weight = self.k_weight_tensor.numpy().transpose((1, 0)) v_proj_weight = self.v_weight_tensor.numpy().transpose((1, 0)) qkv_weight = np.concatenate( - (q_proj_weight, k_proj_weight, v_proj_weight)) + (q_proj_weight, k_proj_weight, v_proj_weight) + ) qkv_weight = qkv_weight.reshape( - (3, self.num_heads, self.head_dim, self.embed_dim)) + (3, self.num_heads, self.head_dim, self.embed_dim) + ) qkv_weight_tensor = paddle.to_tensor(qkv_weight) qkv_weight_tensor = paddle.cast(qkv_weight_tensor, 'int8') out_weight_tensor = paddle.cast( paddle.to_tensor(self.out_weight_tensor.numpy().transpose((1, 0))), - 'int8') + 'int8', + ) ffn1_weight_tensor = paddle.cast( paddle.to_tensor(self.ffn1_weight_tensor.numpy().transpose((1, 0))), - 'int8') + 'int8', + ) ffn2_weight_tensor = paddle.cast( paddle.to_tensor(self.ffn2_weight_tensor.numpy().transpose((1, 0))), - 'int8') + 'int8', + ) qkv_bias = np.concatenate( - (self.q_proj_bias_tensor.numpy(), self.k_proj_bias_tensor.numpy(), - self.v_proj_bias_tensor.numpy())) + ( + self.q_proj_bias_tensor.numpy(), + self.k_proj_bias_tensor.numpy(), + self.v_proj_bias_tensor.numpy(), + ) + ) qkv_bias = qkv_bias.reshape((3, self.num_heads, self.head_dim)) qkv_bias_tensor = paddle.to_tensor(qkv_bias) @@ -486,11 +578,16 @@ def GetFusedMultiTransformerOut(self): cache_kvs = [] max_seq_length = (self.cache_length + 128) // 128 * 128 - cache_kv = np.zeros([ - 2, self.batch_size, self.num_heads, max_seq_length, - self.head_dim - ], - dtype=self.x_type) + cache_kv = np.zeros( + [ + 2, + self.batch_size, + self.num_heads, + max_seq_length, + self.head_dim, + ], + dtype=self.x_type, + ) elems = 4 if self.x_type is np.float16: @@ -502,25 +599,36 @@ def GetFusedMultiTransformerOut(self): # [B, num_head, 128, head_dim] # cache_k_tmp = self.cache_kv[0, :] # [B, num_head, 128, head_dim / 4, 4] - cache_k_tmp = self.cache_kv[0].reshape([ - self.batch_size, self.num_heads, self.cache_length, v_elems, - elems - ]) + cache_k_tmp = self.cache_kv[0].reshape( + [ + self.batch_size, + self.num_heads, + self.cache_length, + v_elems, + elems, + ] + ) # [B, num_head, head_dim / 4, 128, 4] cache_k_tmp = cache_k_tmp.transpose([0, 1, 3, 2, 4]) - cache_kv[0, :].reshape([ - self.batch_size, self.num_heads, v_elems, max_seq_length, elems - ])[:, :, :, :self.cache_length, :] = cache_k_tmp - - cache_kv[1, :, :, :self.cache_length, :] = self.cache_kv[1] + cache_kv[0, :].reshape( + [ + self.batch_size, + self.num_heads, + v_elems, + max_seq_length, + elems, + ] + )[:, :, :, : self.cache_length, :] = cache_k_tmp + + cache_kv[1, :, :, : self.cache_length, :] = self.cache_kv[1] if self.gen_cache_kv: assert self.query_length == self.cache_length cache_kv[:] = 0 else: - time_step = paddle.to_tensor([self.cache_length], - dtype='int32', - place=paddle.CPUPlace()) + time_step = paddle.to_tensor( + [self.cache_length], dtype='int32', place=paddle.CPUPlace() + ) if self.has_attn_mask: attn_mask = paddle.to_tensor(self.attn_mask, stop_gradient=True) else: @@ -542,14 +650,18 @@ def GetFusedMultiTransformerOut(self): ffn1_in_scale = [] ffn2_in_scale = [] - qkv_out_scales_tensor = paddle.ones([self.layers, 3 * self.embed_dim], - 'float32') + qkv_out_scales_tensor = paddle.ones( + [self.layers, 3 * self.embed_dim], 'float32' + ) out_linear_out_scales_tensor = paddle.ones( - [self.layers, self.embed_dim], 'float32') - ffn1_out_scales_tensor = paddle.ones([self.layers, 4 * self.embed_dim], - 'float32') - ffn2_out_scales_tensor = paddle.ones([self.layers, self.embed_dim], - 'float32') + [self.layers, self.embed_dim], 'float32' + ) + ffn1_out_scales_tensor = paddle.ones( + [self.layers, 4 * self.embed_dim], 'float32' + ) + ffn2_out_scales_tensor = paddle.ones( + [self.layers, self.embed_dim], 'float32' + ) for i in range(self.layers): qkv_weights.append(qkv_weight_tensor) @@ -612,7 +724,8 @@ def GetFusedMultiTransformerOut(self): qkv_in_scale=qkv_in_scale, out_linear_in_scale=out_linear_in_scale, ffn1_in_scale=ffn1_in_scale, - ffn2_in_scale=ffn2_in_scale) + ffn2_in_scale=ffn2_in_scale, + ) if self.has_cache_kv: return final_out[0], final_out[1] @@ -634,9 +747,11 @@ def test_fused_multi_transformer_op(self): if self.debug: print("cache_k out timestep=128") - print(cache_kv_out[0].reshape( - [2, bsz, num_head, v_elems, max_seq_len, - elems])[0, 0, 0, :, self.cache_length, :]) + print( + cache_kv_out[0].reshape( + [2, bsz, num_head, v_elems, max_seq_len, elems] + )[0, 0, 0, :, self.cache_length, :] + ) print("cache_v out timestep=128") print(cache_kv_out[0][1, 0, 0, self.cache_length, :]) @@ -649,33 +764,31 @@ def test_fused_multi_transformer_op(self): cache_k = cache_kv_out[i][0, :] cache_k = cache_k.reshape( - [bsz, num_head, v_elems, max_seq_len, elems]) - cache_k = cache_k[:, :, :, :self.cache_length, :] + [bsz, num_head, v_elems, max_seq_len, elems] + ) + cache_k = cache_k[:, :, :, : self.cache_length, :] cache_k = cache_k.transpose([0, 1, 3, 2, 4]) cache_k = cache_k.reshape( - [bsz, num_head, self.cache_length, head_dim]) - - cache_v = cache_kv_out[i][1, :, :, :self.cache_length, :] - - np.testing.assert_allclose(cache_k_ref, - cache_k, - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(cache_v_ref, - cache_v, - rtol=self.rtol, - atol=self.atol) + [bsz, num_head, self.cache_length, head_dim] + ) + + cache_v = cache_kv_out[i][1, :, :, : self.cache_length, :] + + np.testing.assert_allclose( + cache_k_ref, cache_k, rtol=self.rtol, atol=self.atol + ) + np.testing.assert_allclose( + cache_v_ref, cache_v, rtol=self.rtol, atol=self.atol + ) if i == 0: break - np.testing.assert_allclose(final_out_ref, - final_out, - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + final_out_ref, final_out, rtol=self.rtol, atol=self.atol + ) class TestFusedMultiTransformerInt8OpFp16(TestFusedMultiTransformerInt8Op): - def config(self): super().config() self.x_type = np.float16 @@ -683,7 +796,6 @@ def config(self): class TestFusedMultiTransformerInt8OpCacheKV(TestFusedMultiTransformerInt8Op): - def config(self): super().config() super().generate_input_data() @@ -693,9 +805,9 @@ def config(self): self.layers = 3 # odd layers -class TestFusedMultiTransformerInt8OpCacheKVFp16(TestFusedMultiTransformerInt8Op - ): - +class TestFusedMultiTransformerInt8OpCacheKVFp16( + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.has_cache_kv = True @@ -704,9 +816,9 @@ def config(self): self.x_type = np.float16 -class TestFusedMultiTransformerInt8OpGenCacheKV(TestFusedMultiTransformerInt8Op - ): - +class TestFusedMultiTransformerInt8OpGenCacheKV( + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.has_cache_kv = True @@ -714,8 +826,8 @@ def config(self): class TestFusedMultiTransformerInt8OpGenCacheKVFp16( - TestFusedMultiTransformerInt8Op): - + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.has_cache_kv = True @@ -725,8 +837,8 @@ def config(self): class TestFusedMultiTransformerInt8OpPostLayerNormFp16( - TestFusedMultiTransformerInt8Op): - + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.x_type = np.float16 @@ -735,8 +847,8 @@ def config(self): class TestFusedMultiTransformerInt8OpCacheKVPostLayerNorm( - TestFusedMultiTransformerInt8Op): - + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.has_cache_kv = True @@ -747,8 +859,8 @@ def config(self): class TestFusedMultiTransformerInt8OpCacheKVPostLayerNormFp16( - TestFusedMultiTransformerInt8Op): - + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.has_cache_kv = True @@ -759,8 +871,8 @@ def config(self): class TestFusedMultiTransformerInt8OpGenCacheKVPostLayerNorm( - TestFusedMultiTransformerInt8Op): - + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.has_cache_kv = True @@ -769,8 +881,8 @@ def config(self): class TestFusedMultiTransformerInt8OpGenCacheKVPostLayerNormFp16( - TestFusedMultiTransformerInt8Op): - + TestFusedMultiTransformerInt8Op +): def config(self): super().config() self.has_cache_kv = True diff --git a/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_op.py b/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_op.py index dcfb72940403c..0daf8342294f4 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_multi_transformer_op.py @@ -32,7 +32,6 @@ class TestFusedMultiTransformerOp(OpTest): - def setUp(self): self.config() self.generate_input_data() @@ -54,34 +53,47 @@ def setUp(self): self.__class__.no_need_check_grad = False bias_attr = paddle.fluid.ParamAttr( - initializer=paddle.fluid.initializer.Constant(value=0.0005)) - self.q_proj = Linear(self.embed_dim, - self.embed_dim, - self.weight_attr, - bias_attr=bias_attr) - #bias_attr=self.bias_attr) - - self.k_proj = Linear(self.kdim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) - self.v_proj = Linear(self.vdim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) - self.out_proj = Linear(self.embed_dim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) - - self.ffn1_proj = Linear(self.embed_dim, - 4 * self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) - self.ffn2_proj = Linear(4 * self.embed_dim, - self.embed_dim, - self.weight_attr, - bias_attr=self.bias_attr) + initializer=paddle.fluid.initializer.Constant(value=0.0005) + ) + self.q_proj = Linear( + self.embed_dim, + self.embed_dim, + self.weight_attr, + bias_attr=bias_attr, + ) + # bias_attr=self.bias_attr) + + self.k_proj = Linear( + self.kdim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) + self.v_proj = Linear( + self.vdim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) + self.out_proj = Linear( + self.embed_dim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) + + self.ffn1_proj = Linear( + self.embed_dim, + 4 * self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) + self.ffn2_proj = Linear( + 4 * self.embed_dim, + self.embed_dim, + self.weight_attr, + bias_attr=self.bias_attr, + ) paddle.set_default_dtype(np.float32) self.norm = LayerNorm(self.embed_dim) @@ -97,7 +109,7 @@ def config(self): self.x_type = np.float32 self.attn_mask_type = np.float64 - #self.attn_mask_type = np.bool + # self.attn_mask_type = np.bool self.pre_layer_norm = True self.has_attn_mask = True @@ -126,18 +138,27 @@ def config(self): self.weight_attr = None self.bias_attr = None self.kdim, self.vdim = self.embed_dim, self.embed_dim - self.key_length, self.value_length = self.query_length, self.query_length + self.key_length, self.value_length = ( + self.query_length, + self.query_length, + ) def generate_input_data(self): - self.query = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.x_type) + self.query = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.x_type) out_seq_len = self.key_length if self.has_cache_kv: assert self.training is False, ValueError( - 'cache_kv can only used in inference') - self.cache_kv = np.random.rand(2, self.batch_size, self.num_heads, - self.cache_length, - self.head_dim).astype(self.x_type) + 'cache_kv can only used in inference' + ) + self.cache_kv = np.random.rand( + 2, + self.batch_size, + self.num_heads, + self.cache_length, + self.head_dim, + ).astype(self.x_type) if self.gen_cache_kv: self.cache_kv[:] = 0 else: @@ -148,14 +169,19 @@ def generate_input_data(self): if self.has_pre_cache: out_seq_len += self.pre_cache_num self.pre_cache_kv = np.random.rand( - 2, self.batch_size, self.num_heads, self.pre_cache_num, - self.head_dim).astype(self.x_type) + 2, + self.batch_size, + self.num_heads, + self.pre_cache_num, + self.head_dim, + ).astype(self.x_type) if self.has_attn_mask: # [B, n_head, seq_len, out_seq_len] self.attn_mask = np.ones( (self.batch_size, 1, self.query_length, out_seq_len), - dtype=self.attn_mask_type) + dtype=self.attn_mask_type, + ) if self.attn_mask_type == np.int64: self.attn_mask = np.tril(self.attn_mask) elif self.attn_mask_type == np.float64: @@ -172,13 +198,15 @@ def generate_input_data(self): self.attn_mask = np.tril(self.attn_mask) else: raise ValueError( - "'attn_mask_type' should be 'int64' or 'float64'.") + "'attn_mask_type' should be 'int64' or 'float64'." + ) else: self.attn_mask = None self.key, self.value = self.query, self.query - self.dout = np.random.random((self.batch_size, self.query_length, - self.embed_dim)).astype(self.x_type) + self.dout = np.random.random( + (self.batch_size, self.query_length, self.embed_dim) + ).astype(self.x_type) def GetBaselineOut(self): paddle.disable_static(place=paddle.CUDAPlace(0)) @@ -190,8 +218,9 @@ def GetBaselineOut(self): cache_kv = paddle.to_tensor(self.cache_kv, stop_gradient=False) if self.has_pre_cache: - pre_cache_kv = paddle.to_tensor(self.pre_cache_kv, - stop_gradient=False) + pre_cache_kv = paddle.to_tensor( + self.pre_cache_kv, stop_gradient=False + ) if self.has_attn_mask: attn_mask = paddle.to_tensor(self.attn_mask, stop_gradient=False) @@ -241,10 +270,9 @@ def GetBaselineOut(self): # [B, n_head, seq_len, head_dim] * [B, n_head, out_seq_len, head_dim] # --> [B, n_head, seq_len, out_seq_len] - qk_out = layers.matmul(x=q_out, - y=k_out, - transpose_y=True, - alpha=self.head_dim**-0.5) + qk_out = layers.matmul( + x=q_out, y=k_out, transpose_y=True, alpha=self.head_dim**-0.5 + ) if self.debug: print('qk out is') @@ -264,10 +292,12 @@ def GetBaselineOut(self): print('softmax out is') print(softmax_out[0][0][0]) if self.dropout_prob: - dropout_out = F.dropout(softmax_out, - self.dropout_prob, - training=self.training, - mode="upscale_in_train") + dropout_out = F.dropout( + softmax_out, + self.dropout_prob, + training=self.training, + mode="upscale_in_train", + ) # [B, n_head, seq_len, out_seq_len] * [B, n_head, out_seq_len, head_dim] # --> [B, n_head, seq_len, head_dim] qktv_out = tensor.matmul(dropout_out, v_out) @@ -279,7 +309,8 @@ def GetBaselineOut(self): print('fmha out is') print(fmha_out[0][0][0]) out_linear_in = tensor.reshape( - x=fmha_out, shape=[0, 0, fmha_out.shape[2] * fmha_out.shape[3]]) + x=fmha_out, shape=[0, 0, fmha_out.shape[2] * fmha_out.shape[3]] + ) out = self.out_proj(out_linear_in) residual_out = residual + self.dropout(out) @@ -309,53 +340,69 @@ def GetBaselineOut(self): def GetFusedMultiTransformerOut(self): paddle.disable_static(place=paddle.CUDAPlace(0)) - q_proj_weight = paddle.to_tensor(self.q_proj.weight, - stop_gradient=False) - k_proj_weight = paddle.to_tensor(self.k_proj.weight, - stop_gradient=False) - v_proj_weight = paddle.to_tensor(self.v_proj.weight, - stop_gradient=False) - out_linear_weight = paddle.to_tensor(self.out_proj.weight, - stop_gradient=False) - ffn1_weight = paddle.to_tensor(self.ffn1_proj.weight, - stop_gradient=False) - ffn2_weight = paddle.to_tensor(self.ffn2_proj.weight, - stop_gradient=False) + q_proj_weight = paddle.to_tensor( + self.q_proj.weight, stop_gradient=False + ) + k_proj_weight = paddle.to_tensor( + self.k_proj.weight, stop_gradient=False + ) + v_proj_weight = paddle.to_tensor( + self.v_proj.weight, stop_gradient=False + ) + out_linear_weight = paddle.to_tensor( + self.out_proj.weight, stop_gradient=False + ) + ffn1_weight = paddle.to_tensor( + self.ffn1_proj.weight, stop_gradient=False + ) + ffn2_weight = paddle.to_tensor( + self.ffn2_proj.weight, stop_gradient=False + ) if self.bias_attr is False: qkv_bias_tensor = None out_linear_bias = None else: - q_proj_bias = paddle.to_tensor(self.q_proj.bias, - stop_gradient=False) - k_proj_bias = paddle.to_tensor(self.k_proj.bias, - stop_gradient=False) - v_proj_bias = paddle.to_tensor(self.v_proj.bias, - stop_gradient=False) + q_proj_bias = paddle.to_tensor( + self.q_proj.bias, stop_gradient=False + ) + k_proj_bias = paddle.to_tensor( + self.k_proj.bias, stop_gradient=False + ) + v_proj_bias = paddle.to_tensor( + self.v_proj.bias, stop_gradient=False + ) qkv_bias = np.concatenate( - (q_proj_bias.numpy(), k_proj_bias.numpy(), v_proj_bias.numpy())) + (q_proj_bias.numpy(), k_proj_bias.numpy(), v_proj_bias.numpy()) + ) qkv_bias = qkv_bias.reshape((3, self.num_heads, self.head_dim)) qkv_bias_tensor = paddle.to_tensor(qkv_bias, stop_gradient=False) - out_linear_bias = paddle.to_tensor(self.out_proj.bias, - stop_gradient=False) - ffn1_bias = paddle.to_tensor(self.ffn1_proj.bias, - stop_gradient=False) - ffn2_bias = paddle.to_tensor(self.ffn2_proj.bias, - stop_gradient=False) + out_linear_bias = paddle.to_tensor( + self.out_proj.bias, stop_gradient=False + ) + ffn1_bias = paddle.to_tensor( + self.ffn1_proj.bias, stop_gradient=False + ) + ffn2_bias = paddle.to_tensor( + self.ffn2_proj.bias, stop_gradient=False + ) ln_scale = paddle.to_tensor(self.norm.weight, stop_gradient=False) ln_bias = paddle.to_tensor(self.norm.bias, stop_gradient=False) - ffn_ln_scale = paddle.to_tensor(self.ffn_norm.weight, - stop_gradient=False) + ffn_ln_scale = paddle.to_tensor( + self.ffn_norm.weight, stop_gradient=False + ) ffn_ln_bias = paddle.to_tensor(self.ffn_norm.bias, stop_gradient=False) q_proj_weight = q_proj_weight.numpy().transpose((1, 0)) k_proj_weight = k_proj_weight.numpy().transpose((1, 0)) v_proj_weight = v_proj_weight.numpy().transpose((1, 0)) qkv_weight = np.concatenate( - (q_proj_weight, k_proj_weight, v_proj_weight)) + (q_proj_weight, k_proj_weight, v_proj_weight) + ) qkv_weight = qkv_weight.reshape( - (3, self.num_heads, self.head_dim, self.embed_dim)) + (3, self.num_heads, self.head_dim, self.embed_dim) + ) x = paddle.to_tensor(self.query, stop_gradient=False) cache_kvs, cache_kv = None, None @@ -365,11 +412,16 @@ def GetFusedMultiTransformerOut(self): cache_kvs = [] max_seq_length = (self.cache_length + 128) // 128 * 128 - cache_kv = np.zeros([ - 2, self.batch_size, self.num_heads, max_seq_length, - self.head_dim - ], - dtype=self.x_type) + cache_kv = np.zeros( + [ + 2, + self.batch_size, + self.num_heads, + max_seq_length, + self.head_dim, + ], + dtype=self.x_type, + ) elems = 4 if self.x_type is np.float16: @@ -381,35 +433,52 @@ def GetFusedMultiTransformerOut(self): # [B, num_head, 128, head_dim] # cache_k_tmp = self.cache_kv[0, :] # [B, num_head, 128, head_dim / 4, 4] - cache_k_tmp = self.cache_kv[0].reshape([ - self.batch_size, self.num_heads, self.cache_length, v_elems, - elems - ]) + cache_k_tmp = self.cache_kv[0].reshape( + [ + self.batch_size, + self.num_heads, + self.cache_length, + v_elems, + elems, + ] + ) # [B, num_head, head_dim / 4, 128, 4] cache_k_tmp = cache_k_tmp.transpose([0, 1, 3, 2, 4]) - cache_kv[0, :].reshape([ - self.batch_size, self.num_heads, v_elems, max_seq_length, elems - ])[:, :, :, :self.cache_length, :] = cache_k_tmp - - cache_kv[1, :, :, :self.cache_length, :] = self.cache_kv[1] + cache_kv[0, :].reshape( + [ + self.batch_size, + self.num_heads, + v_elems, + max_seq_length, + elems, + ] + )[:, :, :, : self.cache_length, :] = cache_k_tmp + + cache_kv[1, :, :, : self.cache_length, :] = self.cache_kv[1] if self.gen_cache_kv: assert self.query_length == self.cache_length cache_kv[:] = 0 else: - time_step = paddle.to_tensor([self.cache_length], - dtype='int32', - place=paddle.CPUPlace()) + time_step = paddle.to_tensor( + [self.cache_length], dtype='int32', place=paddle.CPUPlace() + ) if self.has_pre_cache: cache_kvs = [] - max_seq_length = (self.cache_length + - 128) // 128 * 128 + self.pre_cache_num - cache_kv = np.zeros([ - 2, self.batch_size, self.num_heads, max_seq_length, - self.head_dim - ], - dtype=self.x_type) + max_seq_length = ( + self.cache_length + 128 + ) // 128 * 128 + self.pre_cache_num + cache_kv = np.zeros( + [ + 2, + self.batch_size, + self.num_heads, + max_seq_length, + self.head_dim, + ], + dtype=self.x_type, + ) pre_caches = [] if self.has_attn_mask: @@ -443,35 +512,40 @@ def GetFusedMultiTransformerOut(self): ffn_ln_scales.append(ffn_ln_scale) ffn_ln_biases.append(ffn_ln_bias) if self.has_cache_kv: - cache_kvs.append(paddle.to_tensor(cache_kv, - stop_gradient=False)) + cache_kvs.append( + paddle.to_tensor(cache_kv, stop_gradient=False) + ) if self.has_pre_cache: - cache_kvs.append(paddle.to_tensor(cache_kv, - stop_gradient=False)) + cache_kvs.append( + paddle.to_tensor(cache_kv, stop_gradient=False) + ) pre_caches.append( - paddle.to_tensor(self.pre_cache_kv, stop_gradient=False)) - - final_out = fused_multi_transformer(x, - ln_scales, - ln_biases, - qkv_weights, - qkv_biases, - out_weights, - out_biases, - ffn_ln_scales, - ffn_ln_biases, - ffn1_weights, - ffn1_biases, - ffn2_weights, - ffn2_biases, - pre_layer_norm=self.pre_layer_norm, - epsilon=epsilon, - cache_kvs=cache_kvs, - pre_caches=pre_caches, - time_step=time_step, - attn_mask=attn_mask, - dropout_rate=self.dropout_prob, - training=self.training) + paddle.to_tensor(self.pre_cache_kv, stop_gradient=False) + ) + + final_out = fused_multi_transformer( + x, + ln_scales, + ln_biases, + qkv_weights, + qkv_biases, + out_weights, + out_biases, + ffn_ln_scales, + ffn_ln_biases, + ffn1_weights, + ffn1_biases, + ffn2_weights, + ffn2_biases, + pre_layer_norm=self.pre_layer_norm, + epsilon=epsilon, + cache_kvs=cache_kvs, + pre_caches=pre_caches, + time_step=time_step, + attn_mask=attn_mask, + dropout_rate=self.dropout_prob, + training=self.training, + ) if self.has_cache_kv: return final_out[0], final_out[1] @@ -492,11 +566,16 @@ def GetFusedMultiTransformerOutStatic(self): cache_kvs = [] max_seq_length = (self.cache_length + 128) // 128 * 128 - cache_kv = np.zeros([ - 2, self.batch_size, self.num_heads, max_seq_length, - self.head_dim - ], - dtype=self.x_type) + cache_kv = np.zeros( + [ + 2, + self.batch_size, + self.num_heads, + max_seq_length, + self.head_dim, + ], + dtype=self.x_type, + ) elems = 4 if self.x_type is np.float16: @@ -504,37 +583,53 @@ def GetFusedMultiTransformerOutStatic(self): assert self.head_dim % elems == 0 v_elems = self.head_dim // elems - cache_k_tmp = self.cache_kv[0].reshape([ - self.batch_size, self.num_heads, self.cache_length, v_elems, - elems - ]) + cache_k_tmp = self.cache_kv[0].reshape( + [ + self.batch_size, + self.num_heads, + self.cache_length, + v_elems, + elems, + ] + ) # [B, num_head, head_dim / 4, 128, 4] cache_k_tmp = cache_k_tmp.transpose([0, 1, 3, 2, 4]) - cache_kv[0, :].reshape([ - self.batch_size, self.num_heads, v_elems, max_seq_length, elems - ])[:, :, :, :self.cache_length, :] = cache_k_tmp - - cache_kv[1, :, :, :self.cache_length, :] = self.cache_kv[1] + cache_kv[0, :].reshape( + [ + self.batch_size, + self.num_heads, + v_elems, + max_seq_length, + elems, + ] + )[:, :, :, : self.cache_length, :] = cache_k_tmp + + cache_kv[1, :, :, : self.cache_length, :] = self.cache_kv[1] if self.gen_cache_kv: assert self.query_length == self.cache_length cache_kv[:] = 0 else: - time_step = layers.fill_constant(shape=[1], - dtype="int32", - value=0, - force_cpu=True) + time_step = layers.fill_constant( + shape=[1], dtype="int32", value=0, force_cpu=True + ) time_step_feed = self.cache_length if self.has_pre_cache: cache_kvs = [] - max_seq_length = (self.cache_length + - 128) // 128 * 128 + self.pre_cache_num - cache_kv = np.zeros([ - 2, self.batch_size, self.num_heads, max_seq_length, - self.head_dim - ], - dtype=self.x_type) + max_seq_length = ( + self.cache_length + 128 + ) // 128 * 128 + self.pre_cache_num + cache_kv = np.zeros( + [ + 2, + self.batch_size, + self.num_heads, + max_seq_length, + self.head_dim, + ], + dtype=self.x_type, + ) pre_caches = [] attn_mask = None @@ -585,35 +680,43 @@ def GetFusedMultiTransformerOutStatic(self): ffn1_weight_attrs=ffn1_weights_attr, ffn1_bias_attrs=ffn1_biases_attr, ffn2_weight_attrs=ffn2_weights_attr, - ffn2_bias_attrs=ffn2_biases_attr) + ffn2_bias_attrs=ffn2_biases_attr, + ) transformer.eval() for i in range(self.layers): if self.has_cache_kv: cache_kvs.append( - layers.fill_constant(shape=cache_kv.shape, - dtype=cache_kv.dtype, - value=0)) + layers.fill_constant( + shape=cache_kv.shape, dtype=cache_kv.dtype, value=0 + ) + ) cache_kvs_feed.append(cache_kv) if self.has_pre_cache: cache_kvs.append( - layers.fill_constant(shape=cache_kv.shape, - dtype=cache_kv.dtype, - value=0)) + layers.fill_constant( + shape=cache_kv.shape, dtype=cache_kv.dtype, value=0 + ) + ) cache_kvs_feed.append(cache_kv) pre_caches.append( - layers.fill_constant(shape=self.pre_cache_kv.shape, - dtype=self.pre_cache_kv.dtype, - value=0)) + layers.fill_constant( + shape=self.pre_cache_kv.shape, + dtype=self.pre_cache_kv.dtype, + value=0, + ) + ) pre_caches_feed.append(self.pre_cache_kv) - final_out = transformer(x, - attn_mask=attn_mask, - caches=cache_kvs, - pre_caches=pre_caches, - time_step=time_step)[0] + final_out = transformer( + x, + attn_mask=attn_mask, + caches=cache_kvs, + pre_caches=pre_caches, + time_step=time_step, + )[0] exe = paddle.static.Executor(place=paddle.CUDAPlace(0)) exe.run(paddle.static.default_startup_program()) feed_data = { @@ -621,11 +724,13 @@ def GetFusedMultiTransformerOutStatic(self): 'cache_kvs': cache_kvs_feed, 'pre_caches': pre_caches_feed, 'time_step': time_step_feed, - 'attn_mask': attn_mask + 'attn_mask': attn_mask, } - out = exe.run(paddle.fluid.default_main_program(), - feed=feed_data, - fetch_list=[final_out]) + out = exe.run( + paddle.fluid.default_main_program(), + feed=feed_data, + fetch_list=[final_out], + ) paddle.disable_static() return out[0] @@ -644,9 +749,11 @@ def test_fused_multi_transformer_op(self): if self.debug: print("cache_k out timestep=128") - print(cache_kv_out[0].reshape( - [2, bsz, num_head, v_elems, max_seq_len, - elems])[0, 0, 0, :, self.cache_length, :]) + print( + cache_kv_out[0].reshape( + [2, bsz, num_head, v_elems, max_seq_len, elems] + )[0, 0, 0, :, self.cache_length, :] + ) print("cache_v out timestep=128") print(cache_kv_out[0][1, 0, 0, self.cache_length, :]) @@ -659,33 +766,31 @@ def test_fused_multi_transformer_op(self): cache_k = cache_kv_out[i][0, :] cache_k = cache_k.reshape( - [bsz, num_head, v_elems, max_seq_len, elems]) - cache_k = cache_k[:, :, :, :self.cache_length, :] + [bsz, num_head, v_elems, max_seq_len, elems] + ) + cache_k = cache_k[:, :, :, : self.cache_length, :] cache_k = cache_k.transpose([0, 1, 3, 2, 4]) cache_k = cache_k.reshape( - [bsz, num_head, self.cache_length, head_dim]) - - cache_v = cache_kv_out[i][1, :, :, :self.cache_length, :] - - np.testing.assert_allclose(cache_k_ref, - cache_k, - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(cache_v_ref, - cache_v, - rtol=self.rtol, - atol=self.atol) + [bsz, num_head, self.cache_length, head_dim] + ) + + cache_v = cache_kv_out[i][1, :, :, : self.cache_length, :] + + np.testing.assert_allclose( + cache_k_ref, cache_k, rtol=self.rtol, atol=self.atol + ) + np.testing.assert_allclose( + cache_v_ref, cache_v, rtol=self.rtol, atol=self.atol + ) if i == 0: break - np.testing.assert_allclose(final_out_ref, - final_out, - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + final_out_ref, final_out, rtol=self.rtol, atol=self.atol + ) class TestFusedMultiTransformerOpFp16(TestFusedMultiTransformerOp): - def config(self): super().config() self.x_type = np.float16 @@ -693,7 +798,6 @@ def config(self): class TestFusedMultiTransformerOpCacheKV(TestFusedMultiTransformerOp): - def config(self): super().config() self.has_cache_kv = True @@ -703,7 +807,6 @@ def config(self): class TestFusedMultiTransformerOpCacheKVFp16(TestFusedMultiTransformerOp): - def config(self): super().config() self.has_cache_kv = True @@ -713,7 +816,6 @@ def config(self): class TestFusedMultiTransformerOpGenCacheKV(TestFusedMultiTransformerOp): - def config(self): super().config() self.has_cache_kv = True @@ -721,7 +823,6 @@ def config(self): class TestFusedMultiTransformerOpGenCacheKVFp16(TestFusedMultiTransformerOp): - def config(self): super().config() self.has_cache_kv = True @@ -731,7 +832,6 @@ def config(self): class TestFusedMultiTransformerOpPostLayerNormFp16(TestFusedMultiTransformerOp): - def config(self): super().config() self.x_type = np.float16 @@ -740,8 +840,8 @@ def config(self): class TestFusedMultiTransformerOpCacheKVPostLayerNorm( - TestFusedMultiTransformerOp): - + TestFusedMultiTransformerOp +): def config(self): super().config() self.has_cache_kv = True @@ -752,8 +852,8 @@ def config(self): class TestFusedMultiTransformerOpCacheKVPostLayerNormFp16( - TestFusedMultiTransformerOp): - + TestFusedMultiTransformerOp +): def config(self): super().config() self.has_cache_kv = True @@ -764,8 +864,8 @@ def config(self): class TestFusedMultiTransformerOpGenCacheKVPostLayerNorm( - TestFusedMultiTransformerOp): - + TestFusedMultiTransformerOp +): def config(self): super().config() self.has_cache_kv = True @@ -774,8 +874,8 @@ def config(self): class TestFusedMultiTransformerOpGenCacheKVPostLayerNormFp16( - TestFusedMultiTransformerOp): - + TestFusedMultiTransformerOp +): def config(self): super().config() self.has_cache_kv = True @@ -786,7 +886,6 @@ def config(self): class TestFusedMultiTransformerOpPreCache(TestFusedMultiTransformerOp): - def config(self): super().config() self.has_pre_cache = True @@ -794,29 +893,31 @@ def config(self): class TestFusedMultiTransformerOpPreCacheStatic(TestFusedMultiTransformerOp): - def config(self): super().config() self.has_pre_cache = True self.has_attn_mask = False self.x_type = np.float32 self.weight_attr = paddle.ParamAttr( - initializer=paddle.fluid.initializer.Constant(0.)) + initializer=paddle.fluid.initializer.Constant(0.0) + ) self.bias_attr = paddle.ParamAttr( - initializer=paddle.fluid.initializer.Constant(0.0005)) + initializer=paddle.fluid.initializer.Constant(0.0005) + ) self.ln_w_attr = paddle.ParamAttr( - initializer=paddle.fluid.initializer.Constant(1.)) + initializer=paddle.fluid.initializer.Constant(1.0) + ) self.ln_b_attr = paddle.ParamAttr( - initializer=paddle.fluid.initializer.Constant(0.)) + initializer=paddle.fluid.initializer.Constant(0.0) + ) def test_fused_multi_transformer_op(self): final_out_ref = self.GetBaselineOut() final_out = self.GetFusedMultiTransformerOutStatic() - np.testing.assert_allclose(final_out_ref, - final_out, - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + final_out_ref, final_out, rtol=self.rtol, atol=self.atol + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_fused_multihead_matmul_op.py b/python/paddle/fluid/tests/unittests/test_fused_multihead_matmul_op.py index 717a98abd50bb..e2b53903b6d72 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_multihead_matmul_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_multihead_matmul_op.py @@ -24,15 +24,15 @@ def stable_softmax(x): """Compute the softmax of vector x in a numerically stable way.""" # clip to shiftx, otherwise, when calc loss with # log(exp(shiftx)), may get log(0)=INF - shiftx = (x - np.max(x)).clip(-64.) + shiftx = (x - np.max(x)).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "Paddle core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "Paddle core is not compiled with CUDA" +) class TestFusedMultiheadMatmulOp(OpTest): - def config(self): self.seq_len = 128 self.size_per_head = 64 @@ -45,13 +45,15 @@ def setUp(self): self.config() h = self.seq_len w = self.head_number * self.size_per_head - self.Input = np.random.random( - (self.batch_size, h, w)).astype("float32") - 0.5 + self.Input = ( + np.random.random((self.batch_size, h, w)).astype("float32") - 0.5 + ) self.WQ = np.random.random((w, w)).astype("float32") self.KQ = np.random.random((w, w)).astype("float32") self.VQ = np.random.random((w, w)).astype("float32") self.CombinedW = np.hstack((self.WQ, self.KQ, self.VQ)).reshape( - (w, 3, w)) + (w, 3, w) + ) self.Q = np.dot(self.Input, self.WQ) self.K = np.dot(self.Input, self.KQ) self.V = np.dot(self.Input, self.VQ) @@ -61,18 +63,32 @@ def setUp(self): self.BiasV = np.random.random((1, w)).astype("float32") self.CombinedB = np.vstack((self.BiasQ, self.BiasK, self.BiasV)) self.BiasQK = np.random.random( - (self.batch_size, self.head_number, self.seq_len, - self.seq_len)).astype("float32") + (self.batch_size, self.head_number, self.seq_len, self.seq_len) + ).astype("float32") # Compute Q path fc_q = self.Q + self.BiasQ - reshape_q = np.reshape(fc_q, (self.batch_size, self.seq_len, - self.head_number, self.size_per_head)) + reshape_q = np.reshape( + fc_q, + ( + self.batch_size, + self.seq_len, + self.head_number, + self.size_per_head, + ), + ) transpose_q = np.transpose(reshape_q, (0, 2, 1, 3)) scale_q = self.scale * transpose_q # Compute K path fc_k = self.K + self.BiasK - reshape_k = np.reshape(fc_k, (self.batch_size, self.seq_len, - self.head_number, self.size_per_head)) + reshape_k = np.reshape( + fc_k, + ( + self.batch_size, + self.seq_len, + self.head_number, + self.size_per_head, + ), + ) transpose_k = np.transpose(reshape_k, (0, 2, 3, 1)) # Compute Q*K @@ -81,8 +97,15 @@ def setUp(self): softmax_qk = np.apply_along_axis(stable_softmax, 3, eltadd_qk) # Compute V path fc_v = self.V + self.BiasV - reshape_v = np.reshape(fc_v, (self.batch_size, self.seq_len, - self.head_number, self.size_per_head)) + reshape_v = np.reshape( + fc_v, + ( + self.batch_size, + self.seq_len, + self.head_number, + self.size_per_head, + ), + ) transpose_v = np.transpose(reshape_v, (0, 2, 1, 3)) # Compute QK*V @@ -94,14 +117,14 @@ def setUp(self): "Input": self.Input, "W": self.CombinedW, "Bias": self.CombinedB, - "BiasQK": self.BiasQK + "BiasQK": self.BiasQK, } self.attrs = { "transpose_Q": False, "transpose_K": True, "transpose_V": False, "head_number": self.head_number, - "alpha": self.scale + "alpha": self.scale, } self.outputs = {"Out": reshape_qkv} @@ -111,7 +134,6 @@ def test_check_output(self): class TestFusedMultiHeadMatmulOp2(TestFusedMultiheadMatmulOp): - def config(self): self.seq_len = 256 self.size_per_head = 32 diff --git a/python/paddle/fluid/tests/unittests/test_fused_token_prune_op.py b/python/paddle/fluid/tests/unittests/test_fused_token_prune_op.py index adfb9e0917776..22d0fe4ba761d 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_token_prune_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_token_prune_op.py @@ -18,10 +18,10 @@ from paddle.framework import core -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFusedTokenPruneOp(OpTest): - def setDtype(self): self.dtype = np.float32 @@ -30,19 +30,21 @@ def setInouts(self): attn = np.array(attn, dtype=self.dtype) attn = np.expand_dims(attn, axis=0) self.attn = np.expand_dims( - attn, axis=0) # [1,1,2,2] bsz = 1, nd_head=1, max_seq_len=2 + attn, axis=0 + ) # [1,1,2,2] bsz = 1, nd_head=1, max_seq_len=2 mask = [[1, 1], [-1, -1]] mask = np.array(mask, dtype=self.dtype) mask = np.expand_dims(mask, axis=0) self.mask = np.expand_dims(mask, axis=0) # same as attn x = [[1, 2, 3], [4, 5, 6]] x = np.array(x, dtype=self.dtype) - self.x = np.expand_dims(x, - axis=0) # [1, 2, 3] bsz = 1, max_seq_len=2, c=3 + self.x = np.expand_dims( + x, axis=0 + ) # [1, 2, 3] bsz = 1, max_seq_len=2, c=3 new_mask = [[1]] new_mask = np.array(new_mask, dtype=self.dtype) new_mask = np.expand_dims(new_mask, axis=0) - self.new_mask = np.expand_dims(new_mask, axis=0) #[1, 1, 1, 1] + self.new_mask = np.expand_dims(new_mask, axis=0) # [1, 1, 1, 1] out_slimmedx_py = [[[1, 2, 3]]] self.out_slimmedx_py = np.array(out_slimmedx_py, dtype=self.dtype) @@ -58,49 +60,68 @@ def setUp(self): 'Attn': self.attn, 'Mask': self.mask, 'X': self.x, - 'NewMask': self.new_mask + 'NewMask': self.new_mask, } self.outputs = { 'SlimmedX': self.out_slimmedx_py, - 'CLSInds': self.out_cls_inds_py + 'CLSInds': self.out_cls_inds_py, } def test_check_output(self): self.check_output_with_place(core.CUDAPlace(0)) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFusedTokenPruneOpFloat64(TestFusedTokenPruneOp): - def setDtype(self): self.dtype = np.float64 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFusedTokenPruneOp2(TestFusedTokenPruneOp): - def setInouts(self): - attn = [[[[1, 2, 3, 4], [4, 3, 2, 1], [5, 9, 5, 4], [9, 6, 5, 4]], - [[8, 5, 2, 0], [1, 0, 2, 3], [2, 2, 3, 2], [7, 4, 1, 8]]]] + attn = [ + [ + [[1, 2, 3, 4], [4, 3, 2, 1], [5, 9, 5, 4], [9, 6, 5, 4]], + [[8, 5, 2, 0], [1, 0, 2, 3], [2, 2, 3, 2], [7, 4, 1, 8]], + ] + ] self.attn = np.array( - attn, - dtype=self.dtype) # [1,2,4,4] bsz = 1, nd_head=2, max_seq_len=4 - mask = [[[[-1, -1, -1, 1], [-1, -1, 1, 1], [-1, -1, 1, 1], - [-1, -1, 1, 1]], - [[-1, -1, 1, 1], [-1, -1, 1, 1], [-1, -1, 1, 1], - [-1, -1, 1, 1]]]] + attn, dtype=self.dtype + ) # [1,2,4,4] bsz = 1, nd_head=2, max_seq_len=4 + mask = [ + [ + [ + [-1, -1, -1, 1], + [-1, -1, 1, 1], + [-1, -1, 1, 1], + [-1, -1, 1, 1], + ], + [ + [-1, -1, 1, 1], + [-1, -1, 1, 1], + [-1, -1, 1, 1], + [-1, -1, 1, 1], + ], + ] + ] self.mask = np.array(mask, dtype=self.dtype) # same as attn - x = [[[1.1, 1.1, 1.1], [2.2, 2.2, 2.2], [3.3, 3.3, 3.3], - [4.4, 4.4, 4.4]]] + x = [ + [[1.1, 1.1, 1.1], [2.2, 2.2, 2.2], [3.3, 3.3, 3.3], [4.4, 4.4, 4.4]] + ] self.x = np.array( - x, dtype=self.dtype) # [1, 4, 3] bsz = 1, max_seq_len=4, c=3 - self.new_mask = np.random.rand(1, 2, 2, - 2).astype(self.dtype) #[1, 2, 2, 2] + x, dtype=self.dtype + ) # [1, 4, 3] bsz = 1, max_seq_len=4, c=3 + self.new_mask = np.random.rand(1, 2, 2, 2).astype( + self.dtype + ) # [1, 2, 2, 2] - out_slimmedx_py = [[[1.1, 1.1, 1.1], [4.4, 4.4, 4.4]]] #[1, 2, 3] + out_slimmedx_py = [[[1.1, 1.1, 1.1], [4.4, 4.4, 4.4]]] # [1, 2, 3] self.out_slimmedx_py = np.array(out_slimmedx_py, dtype=self.dtype) out_cls_inds_py = [[0, 3]] diff --git a/python/paddle/fluid/tests/unittests/test_fused_transformer_encoder_layer.py b/python/paddle/fluid/tests/unittests/test_fused_transformer_encoder_layer.py index 50632b3f76f8f..a8a349265b46c 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_transformer_encoder_layer.py +++ b/python/paddle/fluid/tests/unittests/test_fused_transformer_encoder_layer.py @@ -21,7 +21,6 @@ class TestFusedTransformerEncoderLayer(unittest.TestCase): - def setActivation(self): self.activation = 'gelu' @@ -61,8 +60,8 @@ def setUp(self): def fused_weight(self, weight, num_head): a = paddle.transpose(weight, perm=[1, 0]) return paddle.reshape( - a, shape=[1, num_head, - int(a.shape[0] / num_head), a.shape[1]]) + a, shape=[1, num_head, int(a.shape[0] / num_head), a.shape[1]] + ) def fused_qkv(self, q, k, v, num_head): fq = self.fused_weight(q, num_head) @@ -75,16 +74,29 @@ def test_out(self): return default_main_program().random_seed = 42 base_encoder = TransformerEncoderLayer( - self.d_model, self.nhead, self.dim_feedforward, self.dropout_rate, - self.activation, self.attn_dropout_rate, self.act_dropout_rate, - self.pre_layer_norm) - src = np.random.rand(self.batch_size, self.query_length, - self.embed_dim).astype(self.dtype) + self.d_model, + self.nhead, + self.dim_feedforward, + self.dropout_rate, + self.activation, + self.attn_dropout_rate, + self.act_dropout_rate, + self.pre_layer_norm, + ) + src = np.random.rand( + self.batch_size, self.query_length, self.embed_dim + ).astype(self.dtype) if self.has_attn_mask: - attn_mask = np.ones((self.batch_size, self.num_heads, - self.query_length, self.key_length), - dtype=self.attn_mask_type) + attn_mask = np.ones( + ( + self.batch_size, + self.num_heads, + self.query_length, + self.key_length, + ), + dtype=self.attn_mask_type, + ) attn_mask_tensor = paddle.to_tensor(attn_mask) else: attn_mask = None @@ -92,14 +104,21 @@ def test_out(self): dout = np.random.random(src.shape).astype(self.dtype) - base_out = base_encoder(paddle.to_tensor(src, stop_gradient=False), - attn_mask_tensor) + base_out = base_encoder( + paddle.to_tensor(src, stop_gradient=False), attn_mask_tensor + ) paddle.autograd.backward([base_out], [paddle.to_tensor(dout)], True) fused_encoder = FusedTransformerEncoderLayer( - self.d_model, self.nhead, self.dim_feedforward, self.dropout_rate, - self.activation, self.attn_dropout_rate, self.act_dropout_rate, - self.pre_layer_norm) + self.d_model, + self.nhead, + self.dim_feedforward, + self.dropout_rate, + self.activation, + self.attn_dropout_rate, + self.act_dropout_rate, + self.pre_layer_norm, + ) fused_encoder.ffn._linear1_weight.set_value(base_encoder.linear1.weight) fused_encoder.ffn._linear1_bias.set_value(base_encoder.linear1.bias) @@ -113,17 +132,22 @@ def test_out(self): fused_encoder.ffn._ln2_bias.set_value(base_encoder.norm2.bias) fused_encoder.fused_attn.linear_weight.set_value( - base_encoder.self_attn.out_proj.weight) + base_encoder.self_attn.out_proj.weight + ) fused_encoder.fused_attn.linear_bias.set_value( - base_encoder.self_attn.out_proj.bias) + base_encoder.self_attn.out_proj.bias + ) if self.pre_layer_norm: fused_encoder.fused_attn.pre_ln_scale.set_value( - base_encoder.norm1.weight) + base_encoder.norm1.weight + ) fused_encoder.fused_attn.pre_ln_bias.set_value( - base_encoder.norm1.bias) + base_encoder.norm1.bias + ) else: fused_encoder.fused_attn.ln_scale.set_value( - base_encoder.norm1.weight) + base_encoder.norm1.weight + ) fused_encoder.fused_attn.ln_bias.set_value(base_encoder.norm1.bias) q = base_encoder.self_attn.q_proj.weight @@ -138,59 +162,74 @@ def test_out(self): tmp = paddle.concat(x=[q_bias, k_bias, v_bias], axis=0) qkv_bias = paddle.reshape( tmp, - shape=[3, self.num_heads, - int(tmp.shape[0] / 3 / self.num_heads)]) + shape=[3, self.num_heads, int(tmp.shape[0] / 3 / self.num_heads)], + ) fused_encoder.fused_attn.qkv_bias.set_value(qkv_bias) - fused_out = fused_encoder(paddle.to_tensor(src, stop_gradient=False), - attn_mask_tensor) + fused_out = fused_encoder( + paddle.to_tensor(src, stop_gradient=False), attn_mask_tensor + ) paddle.autograd.backward([fused_out], [paddle.to_tensor(dout)], True) correct_ffn_str = 'd_model={}, dim_feedforward={}, dropout_rate={}, epsilon={}, activation={}, act_dropout_rate={}, normalize_before={}, dtype={}'.format( - self.d_model, self.dim_feedforward, self.dropout_rate, - fused_encoder.ffn._epsilon, self.activation, self.dropout_rate, - self.pre_layer_norm, self.dtype) + self.d_model, + self.dim_feedforward, + self.dropout_rate, + fused_encoder.ffn._epsilon, + self.activation, + self.dropout_rate, + self.pre_layer_norm, + self.dtype, + ) self.assertTrue(fused_encoder.ffn.extra_repr(), correct_ffn_str) correct_attn_str = 'embed_dim={}, num_heads={}, dropout_rate={}, attn_dropout_rate={}, epsilon={}, kdim={}, vdim={}, normalize_before={}, need_weights={}, dtype={}'.format( - self.embed_dim, self.num_heads, self.dropout_rate, - self.dropout_rate, fused_encoder.fused_attn._epsilon, None, None, - self.pre_layer_norm, False, self.dtype) + self.embed_dim, + self.num_heads, + self.dropout_rate, + self.dropout_rate, + fused_encoder.fused_attn._epsilon, + None, + None, + self.pre_layer_norm, + False, + self.dtype, + ) self.assertTrue(fused_encoder.fused_attn.extra_repr(), correct_attn_str) - np.testing.assert_allclose(fused_out.numpy(), - base_out.numpy(), - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(fused_out.grad.numpy(), - base_out.grad.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + fused_out.numpy(), base_out.numpy(), rtol=self.rtol, atol=self.atol + ) + np.testing.assert_allclose( + fused_out.grad.numpy(), + base_out.grad.numpy(), + rtol=self.rtol, + atol=self.atol, + ) class TestFusedTransformerEncoderLayerAct(TestFusedTransformerEncoderLayer): - def setActivation(self): self.activation = 'relu' class TestFusedTransformerEncoderLayerPreLayerNorm( - TestFusedTransformerEncoderLayer): - + TestFusedTransformerEncoderLayer +): def setPreLayerNorm(self): self.pre_layer_norm = True class TestFusedTransformerEncoderLayerAttnMaskIsNone( - TestFusedTransformerEncoderLayer): - + TestFusedTransformerEncoderLayer +): def setAttnMask(self): self.has_attn_mask = False class TestFusedTransformerEncoderLayerPreLnTrueAttnMaskIsNone( - TestFusedTransformerEncoderLayer): - + TestFusedTransformerEncoderLayer +): def setPreLayerNorm(self): self.pre_layer_norm = True diff --git a/python/paddle/fluid/tests/unittests/test_fused_transformer_with_amp_decorator.py b/python/paddle/fluid/tests/unittests/test_fused_transformer_with_amp_decorator.py index f0173d9632fff..212db2406aaa8 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_transformer_with_amp_decorator.py +++ b/python/paddle/fluid/tests/unittests/test_fused_transformer_with_amp_decorator.py @@ -14,12 +14,14 @@ import paddle import paddle.nn as nn -from paddle.incubate.nn.layer.fused_transformer import FusedMultiHeadAttention, FusedFeedForward +from paddle.incubate.nn.layer.fused_transformer import ( + FusedMultiHeadAttention, + FusedFeedForward, +) import unittest class PreModel(nn.Layer): - def __init__(self): super(PreModel, self).__init__() self.attn = FusedMultiHeadAttention( @@ -27,9 +29,9 @@ def __init__(self): num_heads=16, normalize_before=False, ) - self.ffn = FusedFeedForward(d_model=1024, - dim_feedforward=4096, - normalize_before=False) + self.ffn = FusedFeedForward( + d_model=1024, dim_feedforward=4096, normalize_before=False + ) def forward(self, x): x = self.attn(x) @@ -37,7 +39,6 @@ def forward(self, x): class PostModel(nn.Layer): - def __init__(self): super(PostModel, self).__init__() self.attn = FusedMultiHeadAttention( @@ -45,9 +46,9 @@ def __init__(self): num_heads=16, normalize_before=True, ) - self.ffn = FusedFeedForward(d_model=1024, - dim_feedforward=4096, - normalize_before=True) + self.ffn = FusedFeedForward( + d_model=1024, dim_feedforward=4096, normalize_before=True + ) def forward(self, x): x = self.attn(x) @@ -55,19 +56,18 @@ def forward(self, x): class TestFusedTransformerWithAmpDecorator(unittest.TestCase): - def get_model(self): self.pre_model = PreModel() self.post_model = PostModel() def test_run(self): self.get_model() - pre_model = paddle.amp.decorate(models=self.pre_model, - level='O2', - save_dtype='float32') - post_model = paddle.amp.decorate(models=self.post_model, - level='O2', - save_dtype='float32') + pre_model = paddle.amp.decorate( + models=self.pre_model, level='O2', save_dtype='float32' + ) + post_model = paddle.amp.decorate( + models=self.post_model, level='O2', save_dtype='float32' + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_fusion_gru_op.py b/python/paddle/fluid/tests/unittests/test_fusion_gru_op.py index e2aa2fc901e06..68777ca256757 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_gru_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_gru_op.py @@ -20,29 +20,31 @@ def fusion_gru( - x, # T x M - lod, # 1 x N - h0, # N x D - wx, # M x 3D - wh, # D x 3D - bias, # 1 x 3D + x, # T x M + lod, # 1 x N + h0, # N x D + wx, # M x 3D + wh, # D x 3D + bias, # 1 x 3D + is_reverse, + origin_mode, + act_state, + act_gate, +): + return gru( + fc(x, wx, bias), + lod, + h0, + wh, + np.zeros((1, wh.shape[1]), dtype='float32'), is_reverse, - origin_mode, act_state, - act_gate): - return gru(fc(x, wx, bias), - lod, - h0, - wh, - np.zeros((1, wh.shape[1]), dtype='float32'), - is_reverse, - act_state, - act_gate, - origin_mode=origin_mode) + act_gate, + origin_mode=origin_mode, + ) class TestFusionGRUOp(OpTest): - def set_confs(self): pass @@ -66,17 +68,29 @@ def setUp(self): x = np.random.rand(T, self.M).astype('float32') wx = np.random.rand(self.M, 3 * self.D).astype('float32') wh = np.random.rand(self.D, 3 * self.D).astype('float32') - bias = np.random.rand( - 1, 3 * self.D).astype('float32') if self.with_bias else np.zeros( - (1, 3 * self.D), dtype='float32') - h0 = np.random.rand( - N, self.D).astype('float32') if self.with_h0 else np.zeros( - (N, self.D), dtype='float32') - - _, _, _, hidden = fusion_gru(x, self.lod, h0, wx, wh, bias, - self.is_reverse, self.origin_mode, - ACTIVATION[self.act_state], - ACTIVATION[self.act_gate]) + bias = ( + np.random.rand(1, 3 * self.D).astype('float32') + if self.with_bias + else np.zeros((1, 3 * self.D), dtype='float32') + ) + h0 = ( + np.random.rand(N, self.D).astype('float32') + if self.with_h0 + else np.zeros((N, self.D), dtype='float32') + ) + + _, _, _, hidden = fusion_gru( + x, + self.lod, + h0, + wx, + wh, + bias, + self.is_reverse, + self.origin_mode, + ACTIVATION[self.act_state], + ACTIVATION[self.act_gate], + ) self.inputs = {'X': (x, self.lod), 'WeightX': wx, 'WeightH': wh} @@ -93,7 +107,7 @@ def setUp(self): 'gate_activation': self.act_gate, 'is_reverse': self.is_reverse, 'origin_mode': self.origin_mode, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } def test_check_output(self): @@ -103,46 +117,39 @@ def test_check_output(self): class TestFusionGRUOpNoInitial(TestFusionGRUOp): - def set_confs(self): self.with_h0 = False class TestFusionGRUOpNoBias(TestFusionGRUOp): - def set_confs(self): self.with_bias = False class TestFusionGRUOpReverse(TestFusionGRUOp): - def set_confs(self): self.is_reverse = True class TestFusionGRUOpMD1(TestFusionGRUOp): - def set_confs(self): self.M = 36 self.D = 8 class TestFusionGRUOpMD2(TestFusionGRUOp): - def set_confs(self): self.M = 8 self.D = 8 class TestFusionGRUOpMD3(TestFusionGRUOp): - def set_confs(self): self.M = 17 self.D = 15 class TestFusionGRUOpBS1(TestFusionGRUOp): - def set_confs(self): self.lod = [[3]] self.D = 16 @@ -150,5 +157,6 @@ def set_confs(self): if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py b/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py index 9b1d9dfffc6aa..fcde3d2f5db66 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_lstm_op.py @@ -23,25 +23,36 @@ def fc(x, w, b): def fusion_lstm( - x, # T x M - lod, # 1 x N - wx=None, # M x 4D - bx=None, # 1 x 4D - h0=None, # N x D - c0=None, # N x D - w_h=None, # D x 4D - w_b=None, # 1 x 4D - w_c=None, # 1 x 3D - is_reverse=False, - act_gate=None, - act_cell=None, - act_cand=None): - return lstm(fc(x, wx, bx), lod, h0, c0, w_h, w_b, w_c, is_reverse, act_gate, - act_cell, act_cand) + x, # T x M + lod, # 1 x N + wx=None, # M x 4D + bx=None, # 1 x 4D + h0=None, # N x D + c0=None, # N x D + w_h=None, # D x 4D + w_b=None, # 1 x 4D + w_c=None, # 1 x 3D + is_reverse=False, + act_gate=None, + act_cell=None, + act_cand=None, +): + return lstm( + fc(x, wx, bx), + lod, + h0, + c0, + w_h, + w_b, + w_c, + is_reverse, + act_gate, + act_cell, + act_cand, + ) class TestFusionLSTMOp(OpTest): - def set_conf(self): pass @@ -76,24 +87,36 @@ def setUp(self): b = np.random.normal(size=(1, 7 * self.D)).astype('float32') else: b = np.random.normal(size=(1, 4 * self.D)).astype('float32') - w_b = np.copy(b[:, 0:4 * self.D]) - w_c = b[:, 4 * self.D:] if self.use_peepholes else None + w_b = np.copy(b[:, 0 : 4 * self.D]) + w_c = b[:, 4 * self.D :] if self.use_peepholes else None # this is the weight of fc wx = np.random.normal(size=(self.M, 4 * self.D)).astype('float32') # this is the bias of fc # and it should be manually added into the bias of this fusion LSTM bx = np.random.normal(size=(1, 4 * self.D)).astype('float32') - b[0, 0:4 * self.D] += bx[0, :] - h, c = fusion_lstm(x, self.lod, wx, bx, h0, c0, wh, w_b, w_c, - self.is_reverse, ACTIVATION[self.act_gate], - ACTIVATION[self.act_cell], ACTIVATION[self.act_cand]) + b[0, 0 : 4 * self.D] += bx[0, :] + h, c = fusion_lstm( + x, + self.lod, + wx, + bx, + h0, + c0, + wh, + w_b, + w_c, + self.is_reverse, + ACTIVATION[self.act_gate], + ACTIVATION[self.act_cell], + ACTIVATION[self.act_cand], + ) self.inputs = { 'X': (x, self.lod), 'WeightX': wx, 'WeightH': wh, - 'Bias': b + 'Bias': b, } if self.has_initial_state: @@ -110,7 +133,7 @@ def setUp(self): 'gate_activation': self.act_gate, 'cell_activation': self.act_cell, 'candidate_activation': self.act_cand, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } def test_check_output(self): @@ -120,74 +143,63 @@ def test_check_output(self): class TestFusionLSTMOpInit(TestFusionLSTMOp): - def set_conf(self): self.has_initial_state = True class TestFusionLSTMOpReverse(TestFusionLSTMOp): - def set_conf(self): self.is_reverse = True class TestFusionLSTMOpInitReverse(TestFusionLSTMOp): - def set_conf(self): self.has_initial_state = True self.is_reverse = True class TestFusionLSTMOpMD1(TestFusionLSTMOp): - def set_conf(self): self.M = 36 self.D = 8 class TestFusionLSTMOpMD2(TestFusionLSTMOp): - def set_conf(self): self.M = 8 self.D = 8 class TestFusionLSTMOpMD3(TestFusionLSTMOp): - def set_conf(self): self.M = 15 self.D = 3 class TestFusionLSTMOpBS1(TestFusionLSTMOp): - def set_conf(self): self.lod = [[3]] self.D = 16 class TestFusionLSTMOpPeepholes(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True class TestFusionLSTMOpPeepholesInit(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.has_initial_state = True class TestFusionLSTMOpPeepholesReverse(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.is_reverse = True class TestFusionLSTMOpPeepholesInitReverse(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.has_initial_state = True @@ -195,7 +207,6 @@ def set_conf(self): class TestFusionLSTMOpPeepholesBS1(TestFusionLSTMOp): - def set_conf(self): self.use_peepholes = True self.lod = [[2]] @@ -204,5 +215,6 @@ def set_conf(self): if __name__ == '__main__': from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_fusion_repeated_fc_relu_op.py b/python/paddle/fluid/tests/unittests/test_fusion_repeated_fc_relu_op.py index c1b117dfe57d4..c58570d9de992 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_repeated_fc_relu_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_repeated_fc_relu_op.py @@ -19,7 +19,6 @@ class TestFusionRepeatedFCReluOp(OpTest): - def setUp(self): self.bs = 3 self.ic = 9 @@ -28,7 +27,7 @@ def setUp(self): self.set_conf() self.op_type = 'fusion_repeated_fc_relu' sz = len(self.oc) - ics = [self.ic] + self.oc[0:sz - 1] + ics = [self.ic] + self.oc[0 : sz - 1] assert len(ics) == len(self.oc) weights = [] biases = [] @@ -38,23 +37,32 @@ def setUp(self): matrix = MatrixGenerate(self.bs, ics[i], self.oc[i], 1, 1) inp = np.reshape(matrix.input, [self.bs, ics[i]]) weights.append( - ('W_{0}'.format(i), np.reshape(matrix.weights, - [ics[i], self.oc[i]]))) + ( + 'W_{0}'.format(i), + np.reshape(matrix.weights, [ics[i], self.oc[i]]), + ) + ) biases.append(('B_{0}'.format(i), matrix.bias)) outs.append( - np.reshape(np.maximum(fc_refer(matrix, True), 0), - [self.bs, self.oc[i]])) + np.reshape( + np.maximum(fc_refer(matrix, True), 0), [self.bs, self.oc[i]] + ) + ) for i in range(sz - 1): matrix = MatrixGenerate(self.bs, ics[i + 1], self.oc[i + 1], 1, 1) matrix.input = np.reshape(outs[i], [self.bs, ics[i + 1], 1, 1]) out = fc_refer(matrix, True) - weights.append(('W_{0}'.format(i + 1), - np.reshape(matrix.weights, - [ics[i + 1], self.oc[i + 1]]))) + weights.append( + ( + 'W_{0}'.format(i + 1), + np.reshape(matrix.weights, [ics[i + 1], self.oc[i + 1]]), + ) + ) biases.append(('B_{0}'.format(i + 1), matrix.bias)) outs.append( - np.reshape(np.maximum(out, 0), [self.bs, self.oc[i + 1]])) + np.reshape(np.maximum(out, 0), [self.bs, self.oc[i + 1]]) + ) relu_outs = [] for i in range(sz - 1): @@ -76,7 +84,6 @@ def set_conf(self): class TestFusionRepeatedFCReluOpBS1(TestFusionRepeatedFCReluOp): - def set_conf(self): self.bs = 1 self.oc = [4, 2, 7, 5, 512, 1024] diff --git a/python/paddle/fluid/tests/unittests/test_fusion_seqconv_eltadd_relu_op.py b/python/paddle/fluid/tests/unittests/test_fusion_seqconv_eltadd_relu_op.py index cceced75adc8e..91ed3f564429e 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_seqconv_eltadd_relu_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_seqconv_eltadd_relu_op.py @@ -19,7 +19,6 @@ class TestSeqConvEltAddRelu(OpTest): - def set_conf(self): pass @@ -38,8 +37,8 @@ def setUp(self): T = sum(self.lod[0]) x = np.random.uniform(-1, 1, [T, self.in_fea_size]).astype('float32') w = np.random.uniform( - -1, 1, [self.in_fea_size * self.context_length, self.out_fea_size - ]).astype('float32') + -1, 1, [self.in_fea_size * self.context_length, self.out_fea_size] + ).astype('float32') b = np.random.uniform(-2, 1, [1, self.out_fea_size]).astype('float32') out = seqconv(x, self.lod, w, self.context_length, self.context_start) out = np.maximum(out + b, 0) @@ -48,7 +47,7 @@ def setUp(self): self.attrs = { 'contextStart': self.context_start, 'contextLength': self.context_length, - 'contextStride': self.context_stride + 'contextStride': self.context_stride, } self.outputs = {'Out': out} @@ -57,19 +56,16 @@ def test_check_output(self): class TestSeqConvEltAddReluBS1(TestSeqConvEltAddRelu): - def set_conf(self): self.lod = [[10]] class TestSeqConvEltAddReluBS1Case2(TestSeqConvEltAddRelu): - def set_conf(self): self.lod = [[2]] class TestSeqConvEltAddReluCase1(TestSeqConvEltAddRelu): - def set_conf(self): self.lod = [[3, 5, 1, 6]] self.context_length = 3 @@ -77,7 +73,6 @@ def set_conf(self): class TestSeqConvEltAddReluCase2(TestSeqConvEltAddRelu): - def set_conf(self): self.lod = [[10, 1, 2, 4, 1, 5, 6]] self.in_fea_size = 2 @@ -86,7 +81,6 @@ def set_conf(self): class TestSeqConvEltAddReluCase3(TestSeqConvEltAddRelu): - def set_conf(self): self.lod = [[10, 1, 2, 4, 1, 5, 6]] self.context_length = 5 diff --git a/python/paddle/fluid/tests/unittests/test_fusion_seqexpand_concat_fc_op.py b/python/paddle/fluid/tests/unittests/test_fusion_seqexpand_concat_fc_op.py index be3065ec86a23..10a8623ec9c1d 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_seqexpand_concat_fc_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_seqexpand_concat_fc_op.py @@ -45,7 +45,6 @@ def fusion_seqexpand_concat_fc(xs, lod, w, b, fc_act): class TestFusionSeqExpandConcatFCOp(OpTest): - def set_conf(self): pass @@ -65,19 +64,24 @@ def setUp(self): x0 = np.random.normal(size=(T, self.inputs_M[0])).astype('float32') xs = [x0] for i in range(num_inputs - 1): - xi = np.random.normal(size=(bs, - self.inputs_M[i + 1])).astype('float32') + xi = np.random.normal(size=(bs, self.inputs_M[i + 1])).astype( + 'float32' + ) xs.append(xi) # fc weight and bias - w = np.random.normal(size=(sum(self.inputs_M), - self.D)).astype('float32') - b = np.random.normal( - size=(1, self.D)).astype('float32') if self.with_bias else np.zeros( - (1, self.D)).astype('float32') - - out = fusion_seqexpand_concat_fc(xs, self.lod, w, b, - ACTIVATION[self.fc_act]) + w = np.random.normal(size=(sum(self.inputs_M), self.D)).astype( + 'float32' + ) + b = ( + np.random.normal(size=(1, self.D)).astype('float32') + if self.with_bias + else np.zeros((1, self.D)).astype('float32') + ) + + out = fusion_seqexpand_concat_fc( + xs, self.lod, w, b, ACTIVATION[self.fc_act] + ) self.inputs = {'X': [('x0', (x0, self.lod))], 'FCWeight': w} normal_lod = [[1] * bs] @@ -95,47 +99,40 @@ def test_check_output(self): class TestFusionSECFCOpNonBias(TestFusionSeqExpandConcatFCOp): - def set_conf(self): self.with_bias = False class TestFusionSECFCOpNonAct(TestFusionSeqExpandConcatFCOp): - def set_conf(self): self.fc_act = 'identity' class TestFusionSECFCOpMD1(TestFusionSeqExpandConcatFCOp): - def set_conf(self): self.inputs_M = [3, 4, 2, 1, 5] self.D = 8 class TestFusionSECFCOpMD2(TestFusionSeqExpandConcatFCOp): - def set_conf(self): self.lod = [[5, 6]] self.inputs_M = [1, 1] class TestFusionSECFCOpBS1_1(TestFusionSeqExpandConcatFCOp): - def set_conf(self): self.lod = [[1]] self.inputs_M = [3, 4, 2] class TestFusionSECFCOpBS1_2(TestFusionSeqExpandConcatFCOp): - def set_conf(self): self.lod = [[1]] self.inputs_M = [3, 4] class TestFusionSECFCOpBS1_3(TestFusionSeqExpandConcatFCOp): - def set_conf(self): self.lod = [[5]] self.inputs_M = [6, 3] diff --git a/python/paddle/fluid/tests/unittests/test_fusion_seqpool_concat_op.py b/python/paddle/fluid/tests/unittests/test_fusion_seqpool_concat_op.py index c634d8d2b58f6..ceb69451c9bf8 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_seqpool_concat_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_seqpool_concat_op.py @@ -16,11 +16,14 @@ import numpy as np from op_test import OpTest from test_reorder_lod_tensor import convert_to_offset -from sequence.test_sequence_pool import compute_seqpool_sum, compute_seqpool_avg, compute_seqpool_sqrt +from sequence.test_sequence_pool import ( + compute_seqpool_sum, + compute_seqpool_avg, + compute_seqpool_sqrt, +) class TestFusionSeqPoolConcatOp(OpTest): - def setUp(self): self.w = 11 self.lods = [[[2, 3, 5]], [[1, 5, 2]]] @@ -34,8 +37,9 @@ def setUp(self): i = 0 for lod in self.lods: assert bs == len(lod[0]), 'All lod size should be equal' - x = np.random.uniform(0.1, 1, - [sum(lod[0]), self.w]).astype('float32') + x = np.random.uniform(0.1, 1, [sum(lod[0]), self.w]).astype( + 'float32' + ) offset = convert_to_offset(lod) out = np.zeros((bs, self.w)).astype('float32') if self.pooltype == "SUM": @@ -68,26 +72,22 @@ def test_check_output(self): class TestFusionSeqPoolConcatOpCase1(TestFusionSeqPoolConcatOp): - def set_conf(self): self.lods = [[[1]]] class TestFusionSeqPoolConcatOpCase2(TestFusionSeqPoolConcatOp): - def set_conf(self): self.lods = [[[1]], [[1]], [[1]]] class TestFusionSeqPoolConcatOpCase3(TestFusionSeqPoolConcatOp): - def set_conf(self): self.lods = [[[1, 3, 4, 6]]] self.w = 10 class TestFusionSeqPoolConcatOpCase4(TestFusionSeqPoolConcatOp): - def set_conf(self): self.lods = [[[2, 13, 4]], [[1, 1, 1]], [[5, 3, 1]], [[9, 10, 3]]] self.w = 3 @@ -95,14 +95,11 @@ def set_conf(self): ## test avg pool and sqrt def create_test_avg_sqrt_class(parent): - class TestSeqPoolAvgCase(parent): - def set_pooltype(self): self.pooltype = "AVERAGE" class TestSeqPoolSqrtCase(parent): - def set_pooltype(self): self.pooltype = "SQRT" diff --git a/python/paddle/fluid/tests/unittests/test_fusion_seqpool_cvm_concat_op.py b/python/paddle/fluid/tests/unittests/test_fusion_seqpool_cvm_concat_op.py index be298c155357a..9d9324cefe1c4 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_seqpool_cvm_concat_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_seqpool_cvm_concat_op.py @@ -16,12 +16,15 @@ import numpy as np from op_test import OpTest from test_reorder_lod_tensor import convert_to_offset -from sequence.test_sequence_pool import compute_seqpool_sum, compute_seqpool_avg, compute_seqpool_sqrt +from sequence.test_sequence_pool import ( + compute_seqpool_sum, + compute_seqpool_avg, + compute_seqpool_sqrt, +) from test_cvm_op import cvm_compute class TestFusionSeqPoolCVMConcatOp(OpTest): - def setUp(self): self.w = 11 self.use_cvm = True @@ -38,8 +41,9 @@ def setUp(self): i = 0 for lod in self.lods: assert bs == len(lod[0]), 'All lod size should be equal' - x = np.random.uniform(0.1, 1, - [sum(lod[0]), self.w]).astype('float32') + x = np.random.uniform(0.1, 1, [sum(lod[0]), self.w]).astype( + 'float32' + ) offset = convert_to_offset(lod) out = np.zeros((bs, self.w)).astype('float32') if self.pooltype == "SUM": @@ -75,26 +79,22 @@ def test_check_output(self): class TestFusionSeqPoolCVMConcatOpCase1(TestFusionSeqPoolCVMConcatOp): - def set_conf(self): self.lods = [[[1]]] class TestFusionSeqPoolCVMConcatOpCase2(TestFusionSeqPoolCVMConcatOp): - def set_conf(self): self.lods = [[[1]], [[1]], [[1]]] class TestFusionSeqPoolCVMConcatOpCase3(TestFusionSeqPoolCVMConcatOp): - def set_conf(self): self.lods = [[[1, 3, 4, 6]]] self.w = 10 class TestFusionSeqPoolCVMConcatOpCase4(TestFusionSeqPoolCVMConcatOp): - def set_conf(self): self.lods = [[[2, 13, 4]], [[1, 1, 1]], [[5, 3, 1]], [[9, 10, 3]]] self.w = 3 @@ -102,14 +102,11 @@ def set_conf(self): ## test avg pool and sqrt def create_test_avg_sqrt_class(parent): - class TestSeqPoolAvgCase(parent): - def set_pooltype(self): self.pooltype = "AVERAGE" class TestSeqPoolSqrtCase(parent): - def set_pooltype(self): self.pooltype = "SQRT" diff --git a/python/paddle/fluid/tests/unittests/test_fusion_squared_mat_sub_op.py b/python/paddle/fluid/tests/unittests/test_fusion_squared_mat_sub_op.py index a5dd503e43afd..2a6098a8a6a06 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_squared_mat_sub_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_squared_mat_sub_op.py @@ -18,7 +18,6 @@ class TestFusionSquaredMatSubOp(OpTest): - def setUp(self): self.op_type = 'fusion_squared_mat_sub' self.m = 11 @@ -31,8 +30,8 @@ def setUp(self): self.inputs = {'X': matx, 'Y': maty} self.outputs = { - 'Out': - (np.dot(matx, maty)**2 - np.dot(matx**2, maty**2)) * self.scalar + 'Out': (np.dot(matx, maty) ** 2 - np.dot(matx**2, maty**2)) + * self.scalar } self.attrs = { 'scalar': self.scalar, @@ -46,7 +45,6 @@ def test_check_output(self): class TestFusionSquaredMatSubOpCase1(TestFusionSquaredMatSubOp): - def set_conf(self): self.scalar = -0.3 diff --git a/python/paddle/fluid/tests/unittests/test_fusion_transpose_flatten_concat_op.py b/python/paddle/fluid/tests/unittests/test_fusion_transpose_flatten_concat_op.py index f60357fdfa91b..768bbec552dbf 100644 --- a/python/paddle/fluid/tests/unittests/test_fusion_transpose_flatten_concat_op.py +++ b/python/paddle/fluid/tests/unittests/test_fusion_transpose_flatten_concat_op.py @@ -18,10 +18,10 @@ import paddle.fluid.core as core -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFusionTransposeFlattenConcationOp(OpTest): - def setUp(self): self.init_test_case() self.op_type = "fusion_transpose_flatten_concat" @@ -34,8 +34,10 @@ def setUp(self): ins.append(("x%d" % i, a)) b = a.transpose(self.trans_axis) - flat_shape = (np.prod(b.shape[:self.flatten_axis]), - np.prod(b.shape[self.flatten_axis:])) + flat_shape = ( + np.prod(b.shape[: self.flatten_axis]), + np.prod(b.shape[self.flatten_axis :]), + ) c = b.reshape(flat_shape) flats.append(c) out = np.concatenate(flats, axis=self.concat_axis) @@ -44,7 +46,7 @@ def setUp(self): self.attrs = { 'trans_axis': list(self.trans_axis), 'flatten_axis': self.flatten_axis, - 'concat_axis': self.concat_axis + 'concat_axis': self.concat_axis, } self.outputs = {'Out': out} @@ -59,10 +61,10 @@ def init_test_case(self): self.concat_axis = 1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCase1(TestFusionTransposeFlattenConcationOp): - def init_test_case(self): self.shapes = [(3, 4, 18, 17), (3, 8, 18, 7), (6, 12, 9, 5)] self.trans_axis = (0, 2, 3, 1) @@ -70,10 +72,10 @@ def init_test_case(self): self.concat_axis = 1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCase2(TestFusionTransposeFlattenConcationOp): - def init_test_case(self): self.shapes = [(3, 8, 20, 17), (3, 8, 19, 17), (3, 8, 40, 17)] self.trans_axis = (0, 2, 3, 1) @@ -81,10 +83,10 @@ def init_test_case(self): self.concat_axis = 0 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCase3(TestFusionTransposeFlattenConcationOp): - def init_test_case(self): self.shapes = [(3, 8, 20, 17), (3, 8, 19, 17), (3, 8, 40, 17)] self.trans_axis = (0, 3, 2, 1) @@ -92,10 +94,10 @@ def init_test_case(self): self.concat_axis = 1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCase4(TestFusionTransposeFlattenConcationOp): - def init_test_case(self): self.shapes = [(3, 8, 9, 17), (8, 3, 9, 17), (4, 6, 9, 17)] self.trans_axis = (0, 2, 1, 3) @@ -103,10 +105,10 @@ def init_test_case(self): self.concat_axis = 1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCase5(TestFusionTransposeFlattenConcationOp): - def init_test_case(self): self.shapes = [(3, 8, 9, 17, 2), (3, 8, 2, 17, 9), (3, 17, 9, 8, 2)] self.trans_axis = (0, 2, 1, 4, 3) diff --git a/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py b/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py index 17d5fb11ea94f..b098dcf4e1648 100644 --- a/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py +++ b/python/paddle/fluid/tests/unittests/test_gast_with_compatibility.py @@ -20,7 +20,6 @@ class GastNodeTransformer(gast.NodeTransformer): - def __init__(self, root): self.root = root @@ -122,7 +121,6 @@ def code_ast(source): class TestPythonCompatibility(unittest.TestCase): - def _check_compatibility(self, source, target): source_dump = code_gast_ast(source) target_dump = code_ast(target) diff --git a/python/paddle/fluid/tests/unittests/test_gather_nd_op.py b/python/paddle/fluid/tests/unittests/test_gather_nd_op.py index 5154d7feb7758..176012e96a226 100644 --- a/python/paddle/fluid/tests/unittests/test_gather_nd_op.py +++ b/python/paddle/fluid/tests/unittests/test_gather_nd_op.py @@ -39,7 +39,6 @@ def test_check_grad(self): class TestGatherNdOpWithIndex1(OpTest): - def setUp(self): self.op_type = "gather_nd" self.python_api = paddle.gather_nd @@ -55,7 +54,7 @@ def test_check_grad(self): class TestGatherNdOpWithLowIndex(OpTest): - #Index has low rank, X has high rank + # Index has low rank, X has high rank def setUp(self): self.op_type = "gather_nd" @@ -65,7 +64,9 @@ def setUp(self): self.inputs = {'X': xnp, 'Index': index} - self.outputs = {'Out': xnp[tuple(index.T)]} #[[14, 25, 1], [76, 22, 3]] + self.outputs = { + 'Out': xnp[tuple(index.T)] + } # [[14, 25, 1], [76, 22, 3]] def test_check_output(self): self.check_output(check_eager=False) @@ -75,7 +76,7 @@ def test_check_grad(self): class TestGatherNdOpIndex1(OpTest): - #Index has low rank, X has high rank + # Index has low rank, X has high rank def setUp(self): self.op_type = "gather_nd" @@ -95,7 +96,7 @@ def test_check_grad(self): class TestGatherNdOpWithSameIndexAsX(OpTest): - #Index has same rank as X's rank + # Index has same rank as X's rank def setUp(self): self.op_type = "gather_nd" @@ -104,7 +105,7 @@ def setUp(self): index = np.array([[1, 1], [2, 1]]).astype("int64") self.inputs = {'X': xnp, 'Index': index} - self.outputs = {'Out': xnp[tuple(index.T)]} #[25, 22] + self.outputs = {'Out': xnp[tuple(index.T)]} # [25, 22] def test_check_output(self): self.check_output(check_eager=False) @@ -114,7 +115,7 @@ def test_check_grad(self): class TestGatherNdOpWithHighRankSame(OpTest): - #Both Index and X have high rank, and Rank(Index) = Rank(X) + # Both Index and X have high rank, and Rank(Index) = Rank(X) def setUp(self): self.op_type = "gather_nd" @@ -134,7 +135,7 @@ def test_check_grad(self): class TestGatherNdOpWithHighRankDiff(OpTest): - #Both Index and X have high rank, and Rank(Index) < Rank(X) + # Both Index and X have high rank, and Rank(Index) < Rank(X) def setUp(self): self.op_type = "gather_nd" @@ -154,13 +155,12 @@ def test_check_grad(self): self.check_grad(['X'], 'Out', check_eager=False) -#Test Python API +# Test Python API class TestGatherNdOpAPI(unittest.TestCase): - def test_case1(self): - x1 = fluid.layers.data(name='x1', - shape=[30, 40, 50, 60], - dtype='float32') + x1 = fluid.layers.data( + name='x1', shape=[30, 40, 50, 60], dtype='float32' + ) index1 = fluid.layers.data(name='index1', shape=[2, 4], dtype='int32') output1 = fluid.layers.gather_nd(x1, index1) @@ -175,23 +175,20 @@ def test_case3(self): output3 = fluid.layers.gather_nd(x3, index3, name="gather_nd_layer") -#Test Raise Index Error +# Test Raise Index Error class TestGatherNdOpRaise(unittest.TestCase): - def test_check_raise(self): - def check_raise_is_test(): try: - x = fluid.layers.data(name='x', - shape=[3, 4, 5], - dtype='float32') - index = fluid.layers.data(name='index', - shape=[2, 10], - dtype='int32') + x = fluid.layers.data( + name='x', shape=[3, 4, 5], dtype='float32' + ) + index = fluid.layers.data( + name='index', shape=[2, 10], dtype='int32' + ) output = fluid.layers.gather_nd(x, index) except Exception as e: - t = \ - "Input(Index).shape[-1] should be no greater than Input(X).rank" + t = "Input(Index).shape[-1] should be no greater than Input(X).rank" if t in str(e): raise IndexError @@ -199,17 +196,17 @@ def check_raise_is_test(): class TestGatherNdError(unittest.TestCase): - def test_error(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): shape = [8, 9, 6] x = paddle.fluid.data(shape=shape, dtype='float32', name='x') index = paddle.fluid.data(shape=shape, dtype='bool', name='index') - index_float = paddle.fluid.data(shape=shape, - dtype='float32', - name='index_float') + index_float = paddle.fluid.data( + shape=shape, dtype='float32', name='index_float' + ) np_x = np.random.random(shape).astype('float32') np_index = np.array(np.random.randint(2, size=shape, dtype=bool)) @@ -230,7 +227,6 @@ def test_index_dtype(): class TestGatherNdAPI2(unittest.TestCase): - def test_static(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[-1, 2], dtype='float64') @@ -240,11 +236,9 @@ def test_static(self): exe = fluid.Executor(place) input = np.array([[1, 2], [3, 4], [5, 6]]) index_1 = np.array([[1]]) - result, = exe.run(feed={ - "data1": input, - "index": index_1 - }, - fetch_list=[out]) + (result,) = exe.run( + feed={"data1": input, "index": index_1}, fetch_list=[out] + ) expected_output = np.array([[3, 4]]) np.testing.assert_allclose(result, expected_output, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_gather_op.py b/python/paddle/fluid/tests/unittests/test_gather_op.py index 9e2dab2608173..c6301d3e472cd 100644 --- a/python/paddle/fluid/tests/unittests/test_gather_op.py +++ b/python/paddle/fluid/tests/unittests/test_gather_op.py @@ -29,7 +29,6 @@ def gather_numpy(x, index, axis): class TestGatherOp(OpTest): - def setUp(self): self.op_type = "gather" self.python_api = paddle.gather @@ -37,7 +36,7 @@ def setUp(self): xnp = np.random.random(self.x_shape).astype(self.x_type) self.inputs = { 'X': xnp, - 'Index': np.array(self.index).astype(self.index_type) + 'Index': np.array(self.index).astype(self.index_type), } self.outputs = {'Out': self.inputs["X"][self.inputs["Index"]]} @@ -58,31 +57,28 @@ def config(self): class TestCase1(TestGatherOp): - def config(self): """ For one dimension input """ - self.x_shape = (100) + self.x_shape = 100 self.x_type = "float64" self.index = [1, 3, 5] self.index_type = "int32" class TestCase2(TestGatherOp): - def config(self): """ For int64_t index type """ - self.x_shape = (100) + self.x_shape = 100 self.x_type = "float64" self.index = [1, 3, 5] self.index_type = "int64" class TestCase3(TestGatherOp): - def config(self): """ For other input type @@ -94,7 +90,6 @@ def config(self): class TestCase4(TestGatherOp): - def config(self): self.x_shape = (10, 20) self.attrs = {'overwrite': False} @@ -104,7 +99,6 @@ def config(self): class TestCase5(TestGatherOp): - def config(self): self.x_shape = (10, 20) self.attrs = {'overwrite': False} @@ -114,7 +108,6 @@ def config(self): class TestCase6(TestGatherOp): - def config(self): self.x_shape = (10, 20) self.attrs = {'overwrite': True} @@ -124,7 +117,6 @@ def config(self): class TestGatherBF16Op(OpTest): - def setUp(self): self.op_type = "gather" self.python_api = paddle.gather @@ -136,7 +128,7 @@ def setUp(self): self.inputs = { 'X': convert_float_to_uint16(xnp), 'Index': index_np, - 'Axis': axis_np + 'Axis': axis_np, } out = gather_numpy(self.inputs['X'], index_np, axis_np[0]) self.outputs = {'Out': out} @@ -159,7 +151,6 @@ def config(self): class TestGatherOp1(OpTest): - def setUp(self): self.op_type = "gather" self.python_api = paddle.gather @@ -190,7 +181,6 @@ def config(self): class TestGatherOp2(TestGatherOp1): - def config(self): """ For multi-dimension input @@ -204,7 +194,6 @@ def config(self): class TestGatherOp3(TestGatherOp1): - def config(self): """ For multi-dimension input @@ -218,7 +207,6 @@ def config(self): class TestGatherOp4(TestGatherOp1): - def config(self): """ For multi-dimension input @@ -233,7 +221,6 @@ def config(self): class API_TestGather(unittest.TestCase): - def test_out1(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[-1, 2], dtype='float64') @@ -243,17 +230,16 @@ def test_out1(self): exe = fluid.Executor(place) input = np.array([[1, 2], [3, 4], [5, 6]]) index_1 = np.array([1, 2]) - result, = exe.run(feed={ - "data1": input, - "index": index_1 - }, - fetch_list=[out]) + (result,) = exe.run( + feed={"data1": input, "index": index_1}, fetch_list=[out] + ) expected_output = np.array([[3, 4], [5, 6]]) np.testing.assert_allclose(result, expected_output, rtol=1e-05) def test_out2(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data('x', shape=[-1, 2], dtype='float64') index = paddle.fluid.data('index', shape=[-1, 1], dtype='int32') axis = paddle.fluid.data('axis', shape=[1], dtype='int32') @@ -263,18 +249,15 @@ def test_out2(self): x_np = np.array([[1, 2], [3, 4], [5, 6]]).astype('float64') index_np = np.array([1, 1]).astype('int32') axis_np = np.array([1]).astype('int32') - result, = exe.run(feed={ - "x": x_np, - "index": index_np, - 'axis': axis_np - }, - fetch_list=[out]) + (result,) = exe.run( + feed={"x": x_np, "index": index_np, 'axis': axis_np}, + fetch_list=[out], + ) expected_output = gather_numpy(x_np, index_np, axis_np[0]) np.testing.assert_allclose(result, expected_output, rtol=1e-05) class API_TestDygraphGather(unittest.TestCase): - def test_out1(self): paddle.disable_static() input_1 = np.array([[1, 2], [3, 4], [5, 6]]) @@ -319,18 +302,20 @@ def test_large_data(self): def test_dygraph(): with fluid.dygraph.guard(): - gpu_out = paddle.gather(paddle.to_tensor(x), - paddle.to_tensor(index)) + gpu_out = paddle.gather( + paddle.to_tensor(x), paddle.to_tensor(index) + ) return gpu_out.numpy() @switch_to_static_graph def test_static_graph(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x_t = paddle.static.data(name="x", dtype=x.dtype, shape=x.shape) - index_t = paddle.static.data(name="index", - dtype=index.dtype, - shape=index.shape) + index_t = paddle.static.data( + name="index", dtype=index.dtype, shape=index.shape + ) out_t = paddle.gather(x_t, index_t) feed = {x_t.name: x, index_t.name: index} fetch = [out_t] @@ -343,18 +328,18 @@ def test_static_graph(): class TestGathertError(unittest.TestCase): - def test_error1(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): shape = [8, 9, 6] x = paddle.fluid.data(shape=shape, dtype='int8', name='x') axis = paddle.fluid.data(shape=[1], dtype='float32', name='axis') index = paddle.fluid.data(shape=shape, dtype='int32', name='index') - index_float = paddle.fluid.data(shape=shape, - dtype='float32', - name='index_float') + index_float = paddle.fluid.data( + shape=shape, dtype='float32', name='index_float' + ) def test_x_type(): paddle.gather(x, index) @@ -382,9 +367,9 @@ def test_error2(self): shape = [8, 9, 6] x = fluid.data(shape=shape, dtype='int8', name='x') index = fluid.data(shape=shape, dtype='int32', name='mask') - index_float = fluid.data(shape=shape, - dtype='float32', - name='index_float') + index_float = fluid.data( + shape=shape, dtype='float32', name='index_float' + ) def test_x_type(): paddle.fluid.layers.gather(x, index) @@ -398,7 +383,6 @@ def test_index_type(): class TestCheckOutType(unittest.TestCase): - def test_out_type(self): data = paddle.static.data(shape=[16, 10], dtype='int64', name='x') index = paddle.static.data(shape=[4], dtype='int64', name='index') diff --git a/python/paddle/fluid/tests/unittests/test_gather_tree_op.py b/python/paddle/fluid/tests/unittests/test_gather_tree_op.py index 9d0d193e496e3..18fb61c69f1eb 100644 --- a/python/paddle/fluid/tests/unittests/test_gather_tree_op.py +++ b/python/paddle/fluid/tests/unittests/test_gather_tree_op.py @@ -21,17 +21,16 @@ class TestGatherTreeOp(OpTest): - def setUp(self): self.op_type = "gather_tree" self.python_api = paddle.nn.functional.gather_tree max_length, batch_size, beam_size = 5, 2, 2 - ids = np.random.randint(0, - high=10, - size=(max_length, batch_size, beam_size)) - parents = np.random.randint(0, - high=beam_size, - size=(max_length, batch_size, beam_size)) + ids = np.random.randint( + 0, high=10, size=(max_length, batch_size, beam_size) + ) + parents = np.random.randint( + 0, high=beam_size, size=(max_length, batch_size, beam_size) + ) self.inputs = {"Ids": ids, "Parents": parents} self.outputs = {'Out': self.backtrace(ids, parents)} @@ -44,8 +43,9 @@ def backtrace(ids, parents): (max_length, batch_size, beam_size) = ids.shape for batch in range(batch_size): for beam in range(beam_size): - out[max_length - 1, batch, beam] = ids[max_length - 1, batch, - beam] + out[max_length - 1, batch, beam] = ids[ + max_length - 1, batch, beam + ] parent = parents[max_length - 1, batch, beam] for step in range(max_length - 2, -1, -1): out[step, batch, beam] = ids[step, batch, parent] @@ -54,41 +54,46 @@ def backtrace(ids, parents): class TestGatherTreeOpAPI(unittest.TestCase): - def test_case(self): paddle.enable_static() - ids = fluid.layers.data(name='ids', - shape=[5, 2, 2], - dtype='int64', - append_batch_size=False) - parents = fluid.layers.data(name='parents', - shape=[5, 2, 2], - dtype='int64', - append_batch_size=False) + ids = fluid.layers.data( + name='ids', shape=[5, 2, 2], dtype='int64', append_batch_size=False + ) + parents = fluid.layers.data( + name='parents', + shape=[5, 2, 2], + dtype='int64', + append_batch_size=False, + ) final_sequences = fluid.layers.gather_tree(ids, parents) paddle.disable_static() def test_case2(self): - ids = paddle.to_tensor([[[2, 2], [6, 1]], [[3, 9], [6, 1]], - [[0, 1], [9, 0]]]) - parents = paddle.to_tensor([[[0, 0], [1, 1]], [[1, 0], [1, 0]], - [[0, 0], [0, 1]]]) + ids = paddle.to_tensor( + [[[2, 2], [6, 1]], [[3, 9], [6, 1]], [[0, 1], [9, 0]]] + ) + parents = paddle.to_tensor( + [[[0, 0], [1, 1]], [[1, 0], [1, 0]], [[0, 0], [0, 1]]] + ) final_sequences = paddle.nn.functional.gather_tree(ids, parents) class TestGatherTreeOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): - ids = fluid.layers.data(name='ids', - shape=[5, 2, 2], - dtype='int64', - append_batch_size=False) - parents = fluid.layers.data(name='parents', - shape=[5, 2, 2], - dtype='int64', - append_batch_size=False) + ids = fluid.layers.data( + name='ids', + shape=[5, 2, 2], + dtype='int64', + append_batch_size=False, + ) + parents = fluid.layers.data( + name='parents', + shape=[5, 2, 2], + dtype='int64', + append_batch_size=False, + ) def test_Variable_ids(): # the input type must be Variable @@ -106,38 +111,46 @@ def test_Variable_parents(): def test_type_ids(): # dtype must be int32 or int64 - bad_ids = fluid.layers.data(name='bad_ids', - shape=[5, 2, 2], - dtype='float32', - append_batch_size=False) + bad_ids = fluid.layers.data( + name='bad_ids', + shape=[5, 2, 2], + dtype='float32', + append_batch_size=False, + ) fluid.layers.gather_tree(bad_ids, parents) self.assertRaises(TypeError, test_type_ids) def test_type_parents(): # dtype must be int32 or int64 - bad_parents = fluid.layers.data(name='bad_parents', - shape=[5, 2, 2], - dtype='float32', - append_batch_size=False) + bad_parents = fluid.layers.data( + name='bad_parents', + shape=[5, 2, 2], + dtype='float32', + append_batch_size=False, + ) fluid.layers.gather_tree(ids, bad_parents) self.assertRaises(TypeError, test_type_parents) def test_ids_ndim(): - bad_ids = fluid.layers.data(name='bad_test_ids', - shape=[5, 2], - dtype='int64', - append_batch_size=False) + bad_ids = fluid.layers.data( + name='bad_test_ids', + shape=[5, 2], + dtype='int64', + append_batch_size=False, + ) paddle.nn.functional.gather_tree(bad_ids, parents) self.assertRaises(ValueError, test_ids_ndim) def test_parents_ndim(): - bad_parents = fluid.layers.data(name='bad_test_parents', - shape=[5, 2], - dtype='int64', - append_batch_size=False) + bad_parents = fluid.layers.data( + name='bad_test_parents', + shape=[5, 2], + dtype='int64', + append_batch_size=False, + ) paddle.nn.functional.gather_tree(ids, bad_parents) self.assertRaises(ValueError, test_parents_ndim) diff --git a/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py b/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py index 16e34dc67898d..c3ecf7303c175 100644 --- a/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py +++ b/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py @@ -23,7 +23,6 @@ class TestGaussianRandomOp(OpTest): - def setUp(self): self.op_type = "gaussian_random" self.python_api = paddle.normal @@ -35,7 +34,7 @@ def setUp(self): "mean": self.mean, "std": self.std, "seed": 10, - "use_mkldnn": self.use_mkldnn + "use_mkldnn": self.use_mkldnn, } paddle.seed(10) @@ -43,7 +42,7 @@ def setUp(self): def set_attrs(self): self.mean = 1.0 - self.std = 2. + self.std = 2.0 def test_check_output(self): self.check_output_customized(self.verify_output) @@ -64,10 +63,10 @@ def verify_output(self, outs): np.testing.assert_allclose(hist, hist2, rtol=0, atol=0.01) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestGaussianRandomBF16Op(OpTest): - def setUp(self): self.op_type = "gaussian_random" self.python_api = paddle.normal @@ -80,7 +79,7 @@ def setUp(self): "std": self.std, "seed": 10, "dtype": paddle.fluid.core.VarDesc.VarType.BF16, - "use_mkldnn": self.use_mkldnn + "use_mkldnn": self.use_mkldnn, } paddle.seed(10) @@ -88,11 +87,12 @@ def setUp(self): def set_attrs(self): self.mean = 1.0 - self.std = 2. + self.std = 2.0 def test_check_output(self): - self.check_output_with_place_customized(self.verify_output, - place=core.CUDAPlace(0)) + self.check_output_with_place_customized( + self.verify_output, place=core.CUDAPlace(0) + ) def test_eager(self): with _test_eager_guard(): @@ -112,7 +112,6 @@ def verify_output(self, outs): class TestMeanStdAreInt(TestGaussianRandomOp): - def set_attrs(self): self.mean = 1 self.std = 2 @@ -120,23 +119,22 @@ def set_attrs(self): # Situation 2: Attr(shape) is a list(with tensor) class TestGaussianRandomOp_ShapeTensorList(TestGaussianRandomOp): - def setUp(self): - '''Test gaussian_random op with specified value - ''' + '''Test gaussian_random op with specified value''' self.op_type = "gaussian_random" self.init_data() shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.attrs = { 'shape': self.infer_shape, 'mean': self.mean, 'std': self.std, 'seed': self.seed, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.inputs = {"ShapeTensorList": shape_tensor_list} @@ -154,9 +152,9 @@ def test_check_output(self): self.check_output_customized(self.verify_output) -class TestGaussianRandomOp2_ShapeTensorList(TestGaussianRandomOp_ShapeTensorList - ): - +class TestGaussianRandomOp2_ShapeTensorList( + TestGaussianRandomOp_ShapeTensorList +): def init_data(self): self.shape = [123, 92] self.infer_shape = [-1, -1] @@ -166,9 +164,9 @@ def init_data(self): self.seed = 10 -class TestGaussianRandomOp3_ShapeTensorList(TestGaussianRandomOp_ShapeTensorList - ): - +class TestGaussianRandomOp3_ShapeTensorList( + TestGaussianRandomOp_ShapeTensorList +): def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -178,9 +176,9 @@ def init_data(self): self.seed = 10 -class TestGaussianRandomOp4_ShapeTensorList(TestGaussianRandomOp_ShapeTensorList - ): - +class TestGaussianRandomOp4_ShapeTensorList( + TestGaussianRandomOp_ShapeTensorList +): def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -192,10 +190,8 @@ def init_data(self): # Situation 3: shape is a tensor class TestGaussianRandomOp1_ShapeTensor(TestGaussianRandomOp): - def setUp(self): - '''Test gaussian_random op with specified value - ''' + '''Test gaussian_random op with specified value''' self.op_type = "gaussian_random" self.init_data() self.use_mkldnn = False @@ -205,7 +201,7 @@ def setUp(self): 'mean': self.mean, 'std': self.std, 'seed': self.seed, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.outputs = {'Out': np.zeros((123, 92), dtype='float32')} @@ -219,54 +215,61 @@ def init_data(self): # Test python API class TestGaussianRandomAPI(unittest.TestCase): - def test_api(self): positive_2_int32 = fluid.layers.fill_constant([1], "int32", 2000) positive_2_int64 = fluid.layers.fill_constant([1], "int64", 500) - shape_tensor_int32 = fluid.data(name="shape_tensor_int32", - shape=[2], - dtype="int32") - - shape_tensor_int64 = fluid.data(name="shape_tensor_int64", - shape=[2], - dtype="int64") - - out_1 = fluid.layers.gaussian_random(shape=[2000, 500], - dtype="float32", - mean=0.0, - std=1.0, - seed=10) - - out_2 = fluid.layers.gaussian_random(shape=[2000, positive_2_int32], - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_3 = fluid.layers.gaussian_random(shape=[2000, positive_2_int64], - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_4 = fluid.layers.gaussian_random(shape=shape_tensor_int32, - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_5 = fluid.layers.gaussian_random(shape=shape_tensor_int64, - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_6 = fluid.layers.gaussian_random(shape=shape_tensor_int64, - dtype=np.float32, - mean=0., - std=1.0, - seed=10) + shape_tensor_int32 = fluid.data( + name="shape_tensor_int32", shape=[2], dtype="int32" + ) + + shape_tensor_int64 = fluid.data( + name="shape_tensor_int64", shape=[2], dtype="int64" + ) + + out_1 = fluid.layers.gaussian_random( + shape=[2000, 500], dtype="float32", mean=0.0, std=1.0, seed=10 + ) + + out_2 = fluid.layers.gaussian_random( + shape=[2000, positive_2_int32], + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_3 = fluid.layers.gaussian_random( + shape=[2000, positive_2_int64], + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_4 = fluid.layers.gaussian_random( + shape=shape_tensor_int32, + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_5 = fluid.layers.gaussian_random( + shape=shape_tensor_int64, + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_6 = fluid.layers.gaussian_random( + shape=shape_tensor_int64, + dtype=np.float32, + mean=0.0, + std=1.0, + seed=10, + ) exe = fluid.Executor(place=fluid.CPUPlace()) res_1, res_2, res_3, res_4, res_5, res_6 = exe.run( @@ -275,20 +278,21 @@ def test_api(self): "shape_tensor_int32": np.array([2000, 500]).astype("int32"), "shape_tensor_int64": np.array([2000, 500]).astype("int64"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6], + ) self.assertAlmostEqual(np.mean(res_1), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_1), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_1), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_2), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_2), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_2), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_3), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_3), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_3), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_4), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_5), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_5), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_5), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_5), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_5), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_6), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_6), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_6), 1.0, delta=0.1) def test_default_dtype(self): paddle.disable_static() @@ -316,7 +320,6 @@ def test_default_fp64(): class TestStandardNormalDtype(unittest.TestCase): - def test_default_dtype(self): paddle.disable_static() @@ -343,7 +346,6 @@ def test_default_fp64(): class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -356,9 +358,9 @@ def test_fixed_random_number(self): def _check_random_value(dtype, expect, expect_mean, expect_std): x = paddle.randn([32, 3, 1024, 1024], dtype=dtype) actual = x.numpy() - np.testing.assert_allclose(actual[2, 1, 512, 1000:1010], - expect, - rtol=1e-05) + np.testing.assert_allclose( + actual[2, 1, 512, 1000:1010], expect, rtol=1e-05 + ) self.assertTrue(np.mean(actual), expect_mean) self.assertTrue(np.std(actual), expect_std) @@ -367,22 +369,42 @@ def _check_random_value(dtype, expect, expect_mean, expect_std): paddle.set_device('gpu') paddle.seed(2021) expect = [ - -0.79037829, -0.54411126, -0.32266671, 0.35791815, 1.44169267, - -0.87785644, -1.23909874, -2.18194139, 0.49489656, 0.40703062 + -0.79037829, + -0.54411126, + -0.32266671, + 0.35791815, + 1.44169267, + -0.87785644, + -1.23909874, + -2.18194139, + 0.49489656, + 0.40703062, ] - expect_mean = -0.0000053026194133403266873214888799115129813799285329878330230713 + expect_mean = ( + -0.0000053026194133403266873214888799115129813799285329878330230713 + ) expect_std = 0.99999191058126390974081232343451119959354400634765625 - _check_random_value(core.VarDesc.VarType.FP64, expect, expect_mean, - expect_std) + _check_random_value( + core.VarDesc.VarType.FP64, expect, expect_mean, expect_std + ) expect = [ - -0.7988942, 1.8644791, 0.02782744, 1.3692524, 0.6419724, 0.12436751, - 0.12058455, -1.9984808, 1.5635862, 0.18506318 + -0.7988942, + 1.8644791, + 0.02782744, + 1.3692524, + 0.6419724, + 0.12436751, + 0.12058455, + -1.9984808, + 1.5635862, + 0.18506318, ] expect_mean = -0.00004762359094456769526004791259765625 expect_std = 0.999975681304931640625 - _check_random_value(core.VarDesc.VarType.FP32, expect, expect_mean, - expect_std) + _check_random_value( + core.VarDesc.VarType.FP32, expect, expect_mean, expect_std + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_gcd.py b/python/paddle/fluid/tests/unittests/test_gcd.py index d5ef16cea423c..7a2d02dcb8112 100644 --- a/python/paddle/fluid/tests/unittests/test_gcd.py +++ b/python/paddle/fluid/tests/unittests/test_gcd.py @@ -22,7 +22,6 @@ class TestGcdAPI(unittest.TestCase): - def setUp(self): self.x_np = 12 self.y_np = 20 @@ -37,32 +36,34 @@ def test_static_graph(self): y = fluid.data(name='input2', dtype='int32', shape=self.y_shape) out = paddle.gcd(x, y) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(fluid.default_main_program(), - feed={ - 'input1': self.x_np, - 'input2': self.y_np - }, - fetch_list=[out]) - self.assertTrue((np.array(res[0]) == np.gcd(self.x_np, - self.y_np)).all()) + res = exe.run( + fluid.default_main_program(), + feed={'input1': self.x_np, 'input2': self.y_np}, + fetch_list=[out], + ) + self.assertTrue( + (np.array(res[0]) == np.gcd(self.x_np, self.y_np)).all() + ) def test_dygraph(self): paddle.disable_static() x = paddle.to_tensor(self.x_np) y = paddle.to_tensor(self.y_np) result = paddle.gcd(x, y) - np.testing.assert_allclose(np.gcd(self.x_np, self.y_np), - result.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + np.gcd(self.x_np, self.y_np), result.numpy(), rtol=1e-05 + ) paddle.enable_static() class TestGcdAPI2(TestGcdAPI): - def setUp(self): self.x_np = np.arange(6).astype(np.int32) self.y_np = np.array([20]).astype(np.int32) @@ -71,7 +72,6 @@ def setUp(self): class TestGcdAPI3(TestGcdAPI): - def setUp(self): self.x_np = 0 self.y_np = 20 @@ -80,7 +80,6 @@ def setUp(self): class TestGcdAPI4(TestGcdAPI): - def setUp(self): self.x_np = 0 self.y_np = 0 @@ -89,7 +88,6 @@ def setUp(self): class TestGcdAPI5(TestGcdAPI): - def setUp(self): self.x_np = 12 self.y_np = -20 diff --git a/python/paddle/fluid/tests/unittests/test_gelu_op.py b/python/paddle/fluid/tests/unittests/test_gelu_op.py index 6af627be73b34..203e2517cd425 100644 --- a/python/paddle/fluid/tests/unittests/test_gelu_op.py +++ b/python/paddle/fluid/tests/unittests/test_gelu_op.py @@ -24,15 +24,20 @@ def gelu(x, approximate): if approximate: - y_ref = 0.5 * x * ( - 1.0 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)))) + y_ref = ( + 0.5 + * x + * ( + 1.0 + + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3))) + ) + ) else: y_ref = 0.5 * x * (1 + erf(x / np.sqrt(2))) return y_ref.astype(x.dtype) class TestGeluOp(unittest.TestCase): - def _test_case1_cpu(self, approximate): x = np.random.uniform(-1, 1, size=(11, 17)).astype(np.float32) y_ref = gelu(x, approximate) @@ -87,10 +92,9 @@ def run_gelu_op(approximate): y_ref, x_g_ref = run_gelu_op(True) np.testing.assert_allclose(y_ref, y_fast_math, rtol=1e-05, atol=0.0005) - np.testing.assert_allclose(x_g_ref, - x_g_fast_math, - rtol=1e-05, - atol=0.0005) + np.testing.assert_allclose( + x_g_ref, x_g_fast_math, rtol=1e-05, atol=0.0005 + ) def test_fast_math_eager(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_generate_mask_labels_op.py b/python/paddle/fluid/tests/unittests/test_generate_mask_labels_op.py index 18a13197044ac..a2121ee3ef50b 100644 --- a/python/paddle/fluid/tests/unittests/test_generate_mask_labels_op.py +++ b/python/paddle/fluid/tests/unittests/test_generate_mask_labels_op.py @@ -16,6 +16,7 @@ import numpy as np import math from op_test import OpTest + ''' # Equivalent code rles = mask_util.frPyObjects([segm], im_h, im_w) @@ -34,15 +35,17 @@ def decode(cnts, m): def poly2mask(xy, k, h, w): - scale = 5. + scale = 5.0 x = [int(scale * p + 0.5) for p in xy[::2]] x = x + [x[0]] y = [int(scale * p + 0.5) for p in xy[1::2]] y = y + [y[0]] - m = sum([ - int(max(abs(x[j] - x[j + 1]), abs(y[j] - y[j + 1]))) + int(1) - for j in range(k) - ]) + m = sum( + [ + int(max(abs(x[j] - x[j + 1]), abs(y[j] - y[j + 1]))) + int(1) + for j in range(k) + ] + ) u, v = [], [] for j in range(k): @@ -58,20 +61,22 @@ def poly2mask(xy, k, h, w): ys, ye = ye, ys if dx >= dy: - if (dx == 0): assert ye - ys == 0 + if dx == 0: + assert ye - ys == 0 s = 0 if dx == 0 else float(ye - ys) / dx else: - if (dy == 0): assert xe - xs == 0 + if dy == 0: + assert xe - xs == 0 s = 0 if dy == 0 else float(xe - xs) / dy if dx >= dy: ts = [dx - d if flip else d for d in range(dx + 1)] u.extend([xs + t for t in ts]) - v.extend([int(ys + s * t + .5) for t in ts]) + v.extend([int(ys + s * t + 0.5) for t in ts]) else: ts = [dy - d if flip else d for d in range(dy + 1)] v.extend([t + ys for t in ts]) - u.extend([int(xs + s * t + .5) for t in ts]) + u.extend([int(xs + s * t + 0.5) for t in ts]) k = len(u) x = np.zeros((k), np.int_) @@ -80,11 +85,11 @@ def poly2mask(xy, k, h, w): for j in range(1, k): if u[j] != u[j - 1]: xd = float(u[j] if (u[j] < u[j - 1]) else (u[j] - 1)) - xd = (xd + .5) / scale - .5 - if (math.floor(xd) != xd or xd < 0 or xd > (w - 1)): + xd = (xd + 0.5) / scale - 0.5 + if math.floor(xd) != xd or xd < 0 or xd > (w - 1): continue yd = float(v[j] if v[j] < v[j - 1] else v[j - 1]) - yd = (yd + .5) / scale - .5 + yd = (yd + 0.5) / scale - 0.5 yd = math.ceil(0 if yd < 0 else (h if yd > h else yd)) x[m] = int(xd) y[m] = int(yd) @@ -93,21 +98,21 @@ def poly2mask(xy, k, h, w): a = [int(x[i] * h + y[i]) for i in range(k)] a.append(h * w) a.sort() - b = [0] + a[:len(a) - 1] + b = [0] + a[: len(a) - 1] a = [c - d for (c, d) in zip(a, b)] k += 1 b = [0 for i in range(k)] b[0] = a[0] m, j = 1, 1 - while (j < k): + while j < k: if a[j] > 0: b[m] = a[j] m += 1 j += 1 else: j += 1 - if (j < k): + if j < k: b[m - 1] += a[j] j += 1 mask = decode(b, m) @@ -134,19 +139,28 @@ def bbox_overlaps(boxes, query_boxes): K = query_boxes.shape[0] overlaps = np.zeros((N, K), dtype=boxes.dtype) for k in range(K): - box_area = (query_boxes[k, 2] - query_boxes[k, 0] + 1) *\ - (query_boxes[k, 3] - query_boxes[k, 1] + 1) + box_area = (query_boxes[k, 2] - query_boxes[k, 0] + 1) * ( + query_boxes[k, 3] - query_boxes[k, 1] + 1 + ) for n in range(N): - iw = min(boxes[n, 2], query_boxes[k, 2]) -\ - max(boxes[n, 0], query_boxes[k, 0]) + 1 + iw = ( + min(boxes[n, 2], query_boxes[k, 2]) + - max(boxes[n, 0], query_boxes[k, 0]) + + 1 + ) if iw > 0: - ih = min(boxes[n, 3], query_boxes[k, 3]) -\ - max(boxes[n, 1], query_boxes[k, 1]) + 1 + ih = ( + min(boxes[n, 3], query_boxes[k, 3]) + - max(boxes[n, 1], query_boxes[k, 1]) + + 1 + ) if ih > 0: ua = float( - (boxes[n, 2] - boxes[n, 0] + 1) *\ - (boxes[n, 3] - boxes[n, 1] + 1) +\ - box_area - iw * ih) + (boxes[n, 2] - boxes[n, 0] + 1) + * (boxes[n, 3] - boxes[n, 1] + 1) + + box_area + - iw * ih + ) overlaps[n, k] = iw * ih / ua return overlaps @@ -191,7 +205,8 @@ def expand_mask_targets(masks, mask_class_labels, resolution, num_classes): # Target values of -1 are "don't care" / ignore labels mask_targets = -np.ones( - (masks.shape[0], num_classes * resolution**2), dtype=np.int32) + (masks.shape[0], num_classes * resolution**2), dtype=np.int32 + ) for i in range(masks.shape[0]): cls = int(mask_class_labels[i]) start = resolution**2 * cls @@ -203,9 +218,18 @@ def expand_mask_targets(masks, mask_class_labels, resolution, num_classes): return mask_targets -def generate_mask_labels(num_classes, im_info, gt_classes, is_crowd, - label_int32, gt_polys, resolution, rois, roi_lod, - gt_lod): +def generate_mask_labels( + num_classes, + im_info, + gt_classes, + is_crowd, + label_int32, + gt_polys, + resolution, + rois, + roi_lod, + gt_lod, +): mask_rois = [] roi_has_mask_int32 = [] mask_int32 = [] @@ -215,9 +239,16 @@ def generate_mask_labels(num_classes, im_info, gt_classes, is_crowd, roi_e = roi_lod[i + 1] gt_s = gt_lod[i] gt_e = gt_lod[i + 1] - mask_blob = _sample_mask(num_classes, im_info[i], gt_classes[gt_s:gt_e], - is_crowd[gt_s:gt_e], label_int32[roi_s:roi_e], - gt_polys[i], resolution, rois[roi_s:roi_e]) + mask_blob = _sample_mask( + num_classes, + im_info[i], + gt_classes[gt_s:gt_e], + is_crowd[gt_s:gt_e], + label_int32[roi_s:roi_e], + gt_polys[i], + resolution, + rois[roi_s:roi_e], + ) new_lod.append(mask_blob['mask_rois'].shape[0]) mask_rois.append(mask_blob['mask_rois']) roi_has_mask_int32.append(mask_blob['roi_has_mask_int32']) @@ -226,14 +257,15 @@ def generate_mask_labels(num_classes, im_info, gt_classes, is_crowd, def _sample_mask( - num_classes, - im_info, - gt_classes, - is_crowd, - label_int32, - gt_polys, # [[[], []], []] - resolution, - rois): + num_classes, + im_info, + gt_classes, + is_crowd, + label_int32, + gt_polys, # [[[], []], []] + resolution, + rois, +): mask_blob = {} im_scale = im_info[2] sample_boxes = rois @@ -248,7 +280,8 @@ def _sample_mask( masks = np.zeros((fg_inds.shape[0], resolution**2), dtype=np.int32) rois_fg = sample_boxes[fg_inds] overlaps_bbfg_bbpolys = bbox_overlaps( - rois_fg.astype(np.float32), boxes_from_polys.astype(np.float32)) + rois_fg.astype(np.float32), boxes_from_polys.astype(np.float32) + ) fg_polys_inds = np.argmax(overlaps_bbfg_bbpolys, axis=1) for i in range(rois_fg.shape[0]): fg_polys_ind = fg_polys_inds[i] @@ -261,10 +294,11 @@ def _sample_mask( bg_inds = np.where(label_int32 == 0)[0] rois_fg = sample_boxes[bg_inds[0]].reshape((1, -1)) masks = -np.ones((1, resolution**2), dtype=np.int32) - mask_class_labels = np.zeros((1, )) + mask_class_labels = np.zeros((1,)) roi_has_mask = np.append(roi_has_mask, 0) - masks = expand_mask_targets(masks, mask_class_labels, resolution, - num_classes) + masks = expand_mask_targets( + masks, mask_class_labels, resolution, num_classes + ) rois_fg *= im_scale mask_blob['mask_rois'] = rois_fg mask_blob['roi_has_mask_int32'] = roi_has_mask @@ -280,7 +314,6 @@ def trans_lod(lod): class TestGenerateMaskLabels(OpTest): - def set_data(self): self.init_test_case() self.make_generate_proposal_labels_out() @@ -293,16 +326,16 @@ def set_data(self): 'IsCrowd': (self.is_crowd.astype(np.int32), self.gt_lod), 'LabelsInt32': (self.label_int32.astype(np.int32), self.rois_lod), 'GtSegms': (self.gt_polys.astype(np.float32), self.masks_lod), - 'Rois': (self.rois.astype(np.float32), self.rois_lod) + 'Rois': (self.rois.astype(np.float32), self.rois_lod), } self.attrs = { 'num_classes': self.num_classes, - 'resolution': self.resolution + 'resolution': self.resolution, } self.outputs = { 'MaskRois': (self.mask_rois, [self.new_lod]), 'RoiHasMaskInt32': (self.roi_has_mask_int32, [self.new_lod]), - 'MaskInt32': (self.mask_int32, [self.new_lod]) + 'MaskInt32': (self.mask_int32, [self.new_lod]), } def init_test_case(self): @@ -358,9 +391,9 @@ def generate_gt_polys(self): lod1.append(poly_num) pts = [] for j in range(poly_num): - poly_size = np.random.randint(min_poly_size, - max_poly_size, - size=1)[0] + poly_size = np.random.randint( + min_poly_size, max_poly_size, size=1 + )[0] x = np.random.rand(poly_size, 1) * w y = np.random.rand(poly_size, 1) * h xy = np.concatenate((x, y), axis=1) @@ -392,18 +425,27 @@ def generate_groundtruth(self): def init_test_output(self): roi_lod = trans_lod(self.rois_lod[0]) gt_lod = trans_lod(self.gt_lod[0]) - outs = generate_mask_labels(self.num_classes, self.im_info, - self.gt_classes, self.is_crowd, - self.label_int32, self.gt_polys_list, - self.resolution, self.rois, roi_lod, gt_lod) + outs = generate_mask_labels( + self.num_classes, + self.im_info, + self.gt_classes, + self.is_crowd, + self.label_int32, + self.gt_polys_list, + self.resolution, + self.rois, + roi_lod, + gt_lod, + ) self.mask_rois = outs[0] self.roi_has_mask_int32 = outs[1] self.mask_int32 = outs[2] self.new_lod = outs[3] self.mask_rois = np.vstack(self.mask_rois) - self.roi_has_mask_int32 = np.hstack(self.roi_has_mask_int32)[:, - np.newaxis] + self.roi_has_mask_int32 = np.hstack(self.roi_has_mask_int32)[ + :, np.newaxis + ] self.mask_int32 = np.vstack(self.mask_int32) def setUp(self): diff --git a/python/paddle/fluid/tests/unittests/test_generate_proposal_labels_op.py b/python/paddle/fluid/tests/unittests/test_generate_proposal_labels_op.py index 4b5a5ebc2ba99..8027234ce5bbc 100644 --- a/python/paddle/fluid/tests/unittests/test_generate_proposal_labels_op.py +++ b/python/paddle/fluid/tests/unittests/test_generate_proposal_labels_op.py @@ -17,22 +17,24 @@ from op_test import OpTest -def generate_proposal_labels_in_python(rpn_rois, - gt_classes, - is_crowd, - gt_boxes, - im_info, - batch_size_per_im, - fg_fraction, - fg_thresh, - bg_thresh_hi, - bg_thresh_lo, - bbox_reg_weights, - class_nums, - use_random, - is_cls_agnostic, - is_cascade_rcnn, - max_overlaps=None): +def generate_proposal_labels_in_python( + rpn_rois, + gt_classes, + is_crowd, + gt_boxes, + im_info, + batch_size_per_im, + fg_fraction, + fg_thresh, + bg_thresh_hi, + bg_thresh_lo, + bbox_reg_weights, + class_nums, + use_random, + is_cls_agnostic, + is_cascade_rcnn, + max_overlaps=None, +): rois = [] labels_int32 = [] bbox_targets = [] @@ -41,16 +43,29 @@ def generate_proposal_labels_in_python(rpn_rois, max_overlap_with_gt = [] lod = [] assert len(rpn_rois) == len( - im_info), 'batch size of rpn_rois and ground_truth is not matched' + im_info + ), 'batch size of rpn_rois and ground_truth is not matched' for im_i in range(len(im_info)): max_overlap = max_overlaps[im_i] if is_cascade_rcnn else None - frcn_blobs = _sample_rois(rpn_rois[im_i], gt_classes[im_i], - is_crowd[im_i], gt_boxes[im_i], im_info[im_i], - batch_size_per_im, fg_fraction, fg_thresh, - bg_thresh_hi, bg_thresh_lo, bbox_reg_weights, - class_nums, use_random, is_cls_agnostic, - is_cascade_rcnn, max_overlap) + frcn_blobs = _sample_rois( + rpn_rois[im_i], + gt_classes[im_i], + is_crowd[im_i], + gt_boxes[im_i], + im_info[im_i], + batch_size_per_im, + fg_fraction, + fg_thresh, + bg_thresh_hi, + bg_thresh_lo, + bbox_reg_weights, + class_nums, + use_random, + is_cls_agnostic, + is_cascade_rcnn, + max_overlap, + ) lod.append(frcn_blobs['rois'].shape[0]) rois.append(frcn_blobs['rois']) labels_int32.append(frcn_blobs['labels_int32']) @@ -59,7 +74,15 @@ def generate_proposal_labels_in_python(rpn_rois, bbox_outside_weights.append(frcn_blobs['bbox_outside_weights']) max_overlap_with_gt.append(frcn_blobs['max_overlap']) - return rois, labels_int32, bbox_targets, bbox_inside_weights, bbox_outside_weights, max_overlap_with_gt, lod + return ( + rois, + labels_int32, + bbox_targets, + bbox_inside_weights, + bbox_outside_weights, + max_overlap_with_gt, + lod, + ) def filter_roi(rois, max_overlap): @@ -71,16 +94,30 @@ def filter_roi(rois, max_overlap): return np.zeros((1, 4)).astype('float32') -def _sample_rois(rpn_rois, gt_classes, is_crowd, gt_boxes, im_info, - batch_size_per_im, fg_fraction, fg_thresh, bg_thresh_hi, - bg_thresh_lo, bbox_reg_weights, class_nums, use_random, - is_cls_agnostic, is_cascade_rcnn, max_overlap): +def _sample_rois( + rpn_rois, + gt_classes, + is_crowd, + gt_boxes, + im_info, + batch_size_per_im, + fg_fraction, + fg_thresh, + bg_thresh_hi, + bg_thresh_lo, + bbox_reg_weights, + class_nums, + use_random, + is_cls_agnostic, + is_cascade_rcnn, + max_overlap, +): rois_per_image = int(batch_size_per_im) fg_rois_per_im = int(np.round(fg_fraction * rois_per_image)) # Roidb im_scale = im_info[2] - inv_im_scale = 1. / im_scale + inv_im_scale = 1.0 / im_scale rpn_rois = rpn_rois * inv_im_scale if is_cascade_rcnn: @@ -97,12 +134,14 @@ def _sample_rois(rpn_rois, gt_classes, is_crowd, gt_boxes, im_info, # Boxes which with non-zero overlap with gt boxes overlapped_boxes_ind = np.where(overlaps_max > 0)[0] overlapped_boxes_gt_classes = gt_classes[ - overlaps_argmax[overlapped_boxes_ind]] + overlaps_argmax[overlapped_boxes_ind] + ] gt_overlaps[ - overlapped_boxes_ind, - overlapped_boxes_gt_classes] = overlaps_max[overlapped_boxes_ind] + overlapped_boxes_ind, overlapped_boxes_gt_classes + ] = overlaps_max[overlapped_boxes_ind] box_to_gt_ind_map[overlapped_boxes_ind] = overlaps_argmax[ - overlapped_boxes_ind] + overlapped_boxes_ind + ] crowd_ind = np.where(is_crowd)[0] gt_overlaps[crowd_ind] = -1.0 @@ -112,8 +151,9 @@ def _sample_rois(rpn_rois, gt_classes, is_crowd, gt_boxes, im_info, if is_cascade_rcnn: # Cascade RCNN Decode Filter fg_inds = np.where(max_overlaps >= fg_thresh)[0] - bg_inds = np.where((max_overlaps < bg_thresh_hi) - & (max_overlaps >= bg_thresh_lo))[0] + bg_inds = np.where( + (max_overlaps < bg_thresh_hi) & (max_overlaps >= bg_thresh_lo) + )[0] fg_rois_per_this_image = fg_inds.shape[0] bg_rois_per_this_image = bg_inds.shape[0] else: @@ -122,21 +162,23 @@ def _sample_rois(rpn_rois, gt_classes, is_crowd, gt_boxes, im_info, fg_rois_per_this_image = np.minimum(fg_rois_per_im, fg_inds.shape[0]) # Sample foreground if there are too many if (fg_inds.shape[0] > fg_rois_per_this_image) and use_random: - fg_inds = np.random.choice(fg_inds, - size=fg_rois_per_this_image, - replace=False) + fg_inds = np.random.choice( + fg_inds, size=fg_rois_per_this_image, replace=False + ) fg_inds = fg_inds[:fg_rois_per_this_image] # Background - bg_inds = np.where((max_overlaps < bg_thresh_hi) - & (max_overlaps >= bg_thresh_lo))[0] + bg_inds = np.where( + (max_overlaps < bg_thresh_hi) & (max_overlaps >= bg_thresh_lo) + )[0] bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image - bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, - bg_inds.shape[0]) + bg_rois_per_this_image = np.minimum( + bg_rois_per_this_image, bg_inds.shape[0] + ) # Sample background if there are too many if (bg_inds.shape[0] > bg_rois_per_this_image) and use_random: - bg_inds = np.random.choice(bg_inds, - size=bg_rois_per_this_image, - replace=False) + bg_inds = np.random.choice( + bg_inds, size=bg_rois_per_this_image, replace=False + ) bg_inds = bg_inds[:bg_rois_per_this_image] keep_inds = np.append(fg_inds, bg_inds) @@ -146,22 +188,27 @@ def _sample_rois(rpn_rois, gt_classes, is_crowd, gt_boxes, im_info, sampled_max_overlap = max_overlaps[keep_inds] sampled_gts = gt_boxes[box_to_gt_ind_map[keep_inds]] sampled_gts[fg_rois_per_this_image:, :] = gt_boxes[0] - bbox_label_targets = _compute_targets(sampled_boxes, sampled_gts, - sampled_labels, bbox_reg_weights) + bbox_label_targets = _compute_targets( + sampled_boxes, sampled_gts, sampled_labels, bbox_reg_weights + ) bbox_targets, bbox_inside_weights = _expand_bbox_targets( - bbox_label_targets, class_nums, is_cls_agnostic) - bbox_outside_weights = np.array(bbox_inside_weights > 0, - dtype=bbox_inside_weights.dtype) + bbox_label_targets, class_nums, is_cls_agnostic + ) + bbox_outside_weights = np.array( + bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype + ) # Scale rois sampled_rois = sampled_boxes * im_scale # Faster RCNN blobs - frcn_blobs = dict(rois=sampled_rois, - labels_int32=sampled_labels, - bbox_targets=bbox_targets, - bbox_inside_weights=bbox_inside_weights, - bbox_outside_weights=bbox_outside_weights, - max_overlap=sampled_max_overlap) + frcn_blobs = dict( + rois=sampled_rois, + labels_int32=sampled_labels, + bbox_targets=bbox_targets, + bbox_inside_weights=bbox_inside_weights, + bbox_outside_weights=bbox_outside_weights, + max_overlap=sampled_max_overlap, + ) return frcn_blobs @@ -195,12 +242,13 @@ def _compute_targets(roi_boxes, gt_boxes, labels, bbox_reg_weights): targets = np.zeros(roi_boxes.shape) bbox_reg_weights = np.asarray(bbox_reg_weights) - targets = _box_to_delta(ex_boxes=roi_boxes, - gt_boxes=gt_boxes, - weights=bbox_reg_weights) + targets = _box_to_delta( + ex_boxes=roi_boxes, gt_boxes=gt_boxes, weights=bbox_reg_weights + ) - return np.hstack([labels[:, np.newaxis], targets]).astype(np.float32, - copy=False) + return np.hstack([labels[:, np.newaxis], targets]).astype( + np.float32, copy=False + ) def _box_to_delta(ex_boxes, gt_boxes, weights): @@ -230,8 +278,12 @@ def _expand_bbox_targets(bbox_targets_input, class_nums, is_cls_agnostic): # class_labels = [1 if ll > 0 else 0 for ll in class_labels] # class_labels = np.array(class_labels, dtype=np.int32) # class_nums = 2 - bbox_targets = np.zeros((class_labels.shape[0], - 4 * class_nums if not is_cls_agnostic else 4 * 2)) + bbox_targets = np.zeros( + ( + class_labels.shape[0], + 4 * class_nums if not is_cls_agnostic else 4 * 2, + ) + ) bbox_inside_weights = np.zeros(bbox_targets.shape) for ind in fg_inds: class_label = int(class_labels[ind]) if not is_cls_agnostic else 1 @@ -243,9 +295,8 @@ def _expand_bbox_targets(bbox_targets_input, class_nums, is_cls_agnostic): class TestGenerateProposalLabelsOp(OpTest): - def set_data(self): - #self.use_random = False + # self.use_random = False self.init_use_random() self.init_test_params() self.init_test_input() @@ -260,8 +311,10 @@ def set_data(self): 'ImInfo': self.im_info, } if self.max_overlaps is not None: - self.inputs['MaxOverlap'] = (self.max_overlaps[0], - self.rpn_rois_lod) + self.inputs['MaxOverlap'] = ( + self.max_overlaps[0], + self.rpn_rois_lod, + ) self.attrs = { 'batch_size_per_im': self.batch_size_per_im, @@ -273,7 +326,7 @@ def set_data(self): 'class_nums': self.class_nums, 'use_random': self.use_random, 'is_cls_agnostic': self.is_cls_agnostic, - 'is_cascade_rcnn': self.is_cascade_rcnn + 'is_cascade_rcnn': self.is_cascade_rcnn, } self.outputs = { 'Rois': (self.rois, [self.lod]), @@ -291,7 +344,9 @@ def setUp(self): self.op_type = 'generate_proposal_labels' self.set_data() - def init_test_cascade(self, ): + def init_test_cascade( + self, + ): self.is_cascade_rcnn = False self.max_overlaps = None @@ -317,28 +372,46 @@ def init_test_input(self): for i in range(len(images_shape)): self.im_info[i, 0] = images_shape[i][0] self.im_info[i, 1] = images_shape[i][1] - self.im_info[i, 2] = 0.8 #scale + self.im_info[i, 2] = 0.8 # scale self.rpn_rois, self.rpn_rois_lod = _generate_proposals( - images_shape, proposal_nums) + images_shape, proposal_nums + ) ground_truth, self.gts_lod = _generate_groundtruth( - images_shape, self.class_nums, gt_nums) + images_shape, self.class_nums, gt_nums + ) self.gt_classes = [gt['gt_classes'] for gt in ground_truth] self.gt_boxes = [gt['boxes'] for gt in ground_truth] self.is_crowd = [gt['is_crowd'] for gt in ground_truth] def init_test_output(self): - self.rois, self.labels_int32, self.bbox_targets, \ - self.bbox_inside_weights, self.bbox_outside_weights, \ - self.max_overlap_with_gt, \ - self.lod = generate_proposal_labels_in_python( - self.rpn_rois, self.gt_classes, self.is_crowd, self.gt_boxes, self.im_info, - self.batch_size_per_im, self.fg_fraction, - self.fg_thresh, self.bg_thresh_hi, self.bg_thresh_lo, - self.bbox_reg_weights, self.class_nums, self.use_random, - self.is_cls_agnostic, self.is_cascade_rcnn, self.max_overlaps - ) + ( + self.rois, + self.labels_int32, + self.bbox_targets, + self.bbox_inside_weights, + self.bbox_outside_weights, + self.max_overlap_with_gt, + self.lod, + ) = generate_proposal_labels_in_python( + self.rpn_rois, + self.gt_classes, + self.is_crowd, + self.gt_boxes, + self.im_info, + self.batch_size_per_im, + self.fg_fraction, + self.fg_thresh, + self.bg_thresh_hi, + self.bg_thresh_lo, + self.bbox_reg_weights, + self.class_nums, + self.use_random, + self.is_cls_agnostic, + self.is_cascade_rcnn, + self.max_overlaps, + ) self.rois = np.vstack(self.rois) self.labels_int32 = np.hstack(self.labels_int32) self.labels_int32 = self.labels_int32[:, np.newaxis] @@ -349,19 +422,17 @@ def init_test_output(self): class TestCascade(TestGenerateProposalLabelsOp): - def init_test_cascade(self): self.is_cascade_rcnn = True roi_num = len(self.rpn_rois[0]) self.max_overlaps = [] max_overlap = np.random.rand(roi_num).astype('float32') # Make GT samples with overlap = 1 - max_overlap[max_overlap > 0.9] = 1. + max_overlap[max_overlap > 0.9] = 1.0 self.max_overlaps.append(max_overlap) class TestUseRandom(TestGenerateProposalLabelsOp): - def init_use_random(self): self.use_random = True self.is_cascade_rcnn = False @@ -384,7 +455,6 @@ def init_test_params(self): class TestClsAgnostic(TestCascade): - def init_test_params(self): self.batch_size_per_im = 512 self.fg_fraction = 0.25 @@ -397,7 +467,6 @@ def init_test_params(self): class TestOnlyGT(TestCascade): - def init_test_input(self): np.random.seed(0) gt_nums = 6 # Keep same with batch_size_per_im for unittest @@ -407,10 +476,11 @@ def init_test_input(self): for i in range(len(images_shape)): self.im_info[i, 0] = images_shape[i][0] self.im_info[i, 1] = images_shape[i][1] - self.im_info[i, 2] = 0.8 #scale + self.im_info[i, 2] = 0.8 # scale ground_truth, self.gts_lod = _generate_groundtruth( - images_shape, self.class_nums, gt_nums) + images_shape, self.class_nums, gt_nums + ) self.gt_classes = [gt['gt_classes'] for gt in ground_truth] self.gt_boxes = [gt['boxes'] for gt in ground_truth] @@ -420,7 +490,6 @@ def init_test_input(self): class TestOnlyGT2(TestCascade): - def init_test_cascade(self): self.is_cascade_rcnn = True roi_num = len(self.rpn_rois[0]) @@ -447,13 +516,15 @@ def _generate_groundtruth(images_shape, class_nums, gt_nums): num_gts = 0 for i, image_shape in enumerate(images_shape): # Avoid background - gt_classes = np.random.randint(low=1, high=class_nums, - size=gt_nums).astype(np.int32) + gt_classes = np.random.randint( + low=1, high=class_nums, size=gt_nums + ).astype(np.int32) gt_boxes = _generate_boxes(image_shape, gt_nums) is_crowd = np.zeros((gt_nums), dtype=np.int32) is_crowd[0] = 1 ground_truth.append( - dict(gt_classes=gt_classes, boxes=gt_boxes, is_crowd=is_crowd)) + dict(gt_classes=gt_classes, boxes=gt_boxes, is_crowd=is_crowd) + ) num_gts += len(gt_classes) gts_lod.append(num_gts) return ground_truth, [gts_lod] @@ -467,8 +538,12 @@ def _generate_boxes(image_size, box_nums): wh = xywh[:, [2, 3]] * (image_size - xy1) xy2 = xy1 + wh boxes = np.hstack([xy1, xy2]) - boxes[:, [0, 2]] = np.minimum(width - 1., np.maximum(0., boxes[:, [0, 2]])) - boxes[:, [1, 3]] = np.minimum(height - 1., np.maximum(0., boxes[:, [1, 3]])) + boxes[:, [0, 2]] = np.minimum( + width - 1.0, np.maximum(0.0, boxes[:, [0, 2]]) + ) + boxes[:, [1, 3]] = np.minimum( + height - 1.0, np.maximum(0.0, boxes[:, [1, 3]]) + ) return boxes.astype(np.float32) diff --git a/python/paddle/fluid/tests/unittests/test_generate_proposals_op.py b/python/paddle/fluid/tests/unittests/test_generate_proposals_op.py index d5fbca5be9ecb..8a23973ede38a 100644 --- a/python/paddle/fluid/tests/unittests/test_generate_proposals_op.py +++ b/python/paddle/fluid/tests/unittests/test_generate_proposals_op.py @@ -21,9 +21,18 @@ import copy -def generate_proposals_in_python(scores, bbox_deltas, im_info, anchors, - variances, pre_nms_topN, post_nms_topN, - nms_thresh, min_size, eta): +def generate_proposals_in_python( + scores, + bbox_deltas, + im_info, + anchors, + variances, + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, +): all_anchors = anchors.reshape(-1, 4) rois = np.empty((0, 5), dtype=np.float32) roi_probs = np.empty((0, 1), dtype=np.float32) @@ -34,9 +43,17 @@ def generate_proposals_in_python(scores, bbox_deltas, im_info, anchors, num_images = scores.shape[0] for img_idx in range(num_images): img_i_boxes, img_i_probs = proposal_for_one_image( - im_info[img_idx, :], all_anchors, variances, - bbox_deltas[img_idx, :, :, :], scores[img_idx, :, :, :], - pre_nms_topN, post_nms_topN, nms_thresh, min_size, eta) + im_info[img_idx, :], + all_anchors, + variances, + bbox_deltas[img_idx, :, :, :], + scores[img_idx, :, :, :], + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, + ) rois_num.append(img_i_probs.shape[0]) rpn_rois.append(img_i_boxes) rpn_roi_probs.append(img_i_probs) @@ -44,9 +61,18 @@ def generate_proposals_in_python(scores, bbox_deltas, im_info, anchors, return rpn_rois, rpn_roi_probs, rois_num -def proposal_for_one_image(im_info, all_anchors, variances, bbox_deltas, scores, - pre_nms_topN, post_nms_topN, nms_thresh, min_size, - eta): +def proposal_for_one_image( + im_info, + all_anchors, + variances, + bbox_deltas, + scores, + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, +): # Transpose and reshape predicted bbox transformations to get them # into the same order as the anchors: # - bbox deltas will be (4 * A, H, W) format from conv output @@ -94,10 +120,9 @@ def proposal_for_one_image(im_info, all_anchors, variances, bbox_deltas, scores, # take post_nms_topN (e.g. 1000) # return the top proposals if nms_thresh > 0: - keep = nms(boxes=proposals, - scores=scores, - nms_threshold=nms_thresh, - eta=eta) + keep = nms( + boxes=proposals, scores=scores, nms_threshold=nms_thresh, eta=eta + ) if post_nms_topN > 0 and post_nms_topN < len(keep): keep = keep[:post_nms_topN] proposals = proposals[keep, :] @@ -111,10 +136,10 @@ def box_coder(all_anchors, bbox_deltas, variances, pixel_offset=True): Decode proposals by anchors and bbox_deltas from RPN """ offset = 1 if pixel_offset else 0 - #proposals: xmin, ymin, xmax, ymax + # proposals: xmin, ymin, xmax, ymax proposals = np.zeros_like(bbox_deltas, dtype=np.float32) - #anchor_loc: width, height, center_x, center_y + # anchor_loc: width, height, center_x, center_y anchor_loc = np.zeros_like(bbox_deltas, dtype=np.float32) anchor_loc[:, 0] = all_anchors[:, 2] - all_anchors[:, 0] + offset @@ -122,34 +147,52 @@ def box_coder(all_anchors, bbox_deltas, variances, pixel_offset=True): anchor_loc[:, 2] = all_anchors[:, 0] + 0.5 * anchor_loc[:, 0] anchor_loc[:, 3] = all_anchors[:, 1] + 0.5 * anchor_loc[:, 1] - #predicted bbox: bbox_center_x, bbox_center_y, bbox_width, bbox_height + # predicted bbox: bbox_center_x, bbox_center_y, bbox_width, bbox_height pred_bbox = np.zeros_like(bbox_deltas, dtype=np.float32) if variances is not None: for i in range(bbox_deltas.shape[0]): - pred_bbox[i, 0] = variances[i, 0] * bbox_deltas[i, 0] * anchor_loc[ - i, 0] + anchor_loc[i, 2] - pred_bbox[i, 1] = variances[i, 1] * bbox_deltas[i, 1] * anchor_loc[ - i, 1] + anchor_loc[i, 3] - pred_bbox[i, 2] = math.exp( - min(variances[i, 2] * bbox_deltas[i, 2], math.log( - 1000 / 16.0))) * anchor_loc[i, 0] - pred_bbox[i, 3] = math.exp( - min(variances[i, 3] * bbox_deltas[i, 3], math.log( - 1000 / 16.0))) * anchor_loc[i, 1] + pred_bbox[i, 0] = ( + variances[i, 0] * bbox_deltas[i, 0] * anchor_loc[i, 0] + + anchor_loc[i, 2] + ) + pred_bbox[i, 1] = ( + variances[i, 1] * bbox_deltas[i, 1] * anchor_loc[i, 1] + + anchor_loc[i, 3] + ) + pred_bbox[i, 2] = ( + math.exp( + min( + variances[i, 2] * bbox_deltas[i, 2], + math.log(1000 / 16.0), + ) + ) + * anchor_loc[i, 0] + ) + pred_bbox[i, 3] = ( + math.exp( + min( + variances[i, 3] * bbox_deltas[i, 3], + math.log(1000 / 16.0), + ) + ) + * anchor_loc[i, 1] + ) else: for i in range(bbox_deltas.shape[0]): - pred_bbox[i, - 0] = bbox_deltas[i, 0] * anchor_loc[i, 0] + anchor_loc[i, - 2] - pred_bbox[i, - 1] = bbox_deltas[i, 1] * anchor_loc[i, 1] + anchor_loc[i, - 3] - pred_bbox[i, 2] = math.exp( - min(bbox_deltas[i, 2], math.log(1000 / 16.0))) * anchor_loc[i, - 0] - pred_bbox[i, 3] = math.exp( - min(bbox_deltas[i, 3], math.log(1000 / 16.0))) * anchor_loc[i, - 1] + pred_bbox[i, 0] = ( + bbox_deltas[i, 0] * anchor_loc[i, 0] + anchor_loc[i, 2] + ) + pred_bbox[i, 1] = ( + bbox_deltas[i, 1] * anchor_loc[i, 1] + anchor_loc[i, 3] + ) + pred_bbox[i, 2] = ( + math.exp(min(bbox_deltas[i, 2], math.log(1000 / 16.0))) + * anchor_loc[i, 0] + ) + pred_bbox[i, 3] = ( + math.exp(min(bbox_deltas[i, 3], math.log(1000 / 16.0))) + * anchor_loc[i, 1] + ) proposals[:, 0] = pred_bbox[:, 0] - pred_bbox[:, 2] / 2 proposals[:, 1] = pred_bbox[:, 1] - pred_bbox[:, 3] / 2 proposals[:, 2] = pred_bbox[:, 0] + pred_bbox[:, 2] / 2 - offset @@ -161,33 +204,33 @@ def box_coder(all_anchors, bbox_deltas, variances, pixel_offset=True): def clip_tiled_boxes(boxes, im_shape, pixel_offset=True): """Clip boxes to image boundaries. im_shape is [height, width] and boxes has shape (N, 4 * num_tiled_boxes).""" - assert boxes.shape[1] % 4 == 0, \ - 'boxes.shape[1] is {:d}, but must be divisible by 4.'.format( + assert ( + boxes.shape[1] % 4 == 0 + ), 'boxes.shape[1] is {:d}, but must be divisible by 4.'.format( boxes.shape[1] ) offset = 1 if pixel_offset else 0 # x1 >= 0 - boxes[:, - 0::4] = np.maximum(np.minimum(boxes[:, 0::4], im_shape[1] - offset), - 0) + boxes[:, 0::4] = np.maximum( + np.minimum(boxes[:, 0::4], im_shape[1] - offset), 0 + ) # y1 >= 0 - boxes[:, - 1::4] = np.maximum(np.minimum(boxes[:, 1::4], im_shape[0] - offset), - 0) + boxes[:, 1::4] = np.maximum( + np.minimum(boxes[:, 1::4], im_shape[0] - offset), 0 + ) # x2 < im_shape[1] - boxes[:, - 2::4] = np.maximum(np.minimum(boxes[:, 2::4], im_shape[1] - offset), - 0) + boxes[:, 2::4] = np.maximum( + np.minimum(boxes[:, 2::4], im_shape[1] - offset), 0 + ) # y2 < im_shape[0] - boxes[:, - 3::4] = np.maximum(np.minimum(boxes[:, 3::4], im_shape[0] - offset), - 0) + boxes[:, 3::4] = np.maximum( + np.minimum(boxes[:, 3::4], im_shape[0] - offset), 0 + ) return boxes def filter_boxes(boxes, min_size, im_info, pixel_offset=True): - """Only keep boxes with both sides >= min_size and center within the image. - """ + """Only keep boxes with both sides >= min_size and center within the image.""" # Scale min_size to match image scale im_scale = im_info[2] min_size = max(min_size, 1.0) @@ -197,11 +240,14 @@ def filter_boxes(boxes, min_size, im_info, pixel_offset=True): if pixel_offset: ws_orig_scale = (boxes[:, 2] - boxes[:, 0]) / im_scale + 1 hs_orig_scale = (boxes[:, 3] - boxes[:, 1]) / im_scale + 1 - x_ctr = boxes[:, 0] + ws / 2. - y_ctr = boxes[:, 1] + hs / 2. - keep = np.where((ws_orig_scale >= min_size) - & (hs_orig_scale >= min_size) & (x_ctr < im_info[1]) - & (y_ctr < im_info[0]))[0] + x_ctr = boxes[:, 0] + ws / 2.0 + y_ctr = boxes[:, 1] + hs / 2.0 + keep = np.where( + (ws_orig_scale >= min_size) + & (hs_orig_scale >= min_size) + & (x_ctr < im_info[1]) + & (y_ctr < im_info[0]) + )[0] else: keep = np.where((ws >= min_size) & (hs >= min_size))[0] return keep @@ -263,9 +309,9 @@ def nms(boxes, scores, nms_threshold, eta=1.0, pixel_offset=True): for k in range(len(selected_indices)): if keep: kept_idx = selected_indices[k] - overlap = iou(boxes[idx], - boxes[kept_idx], - pixel_offset=pixel_offset) + overlap = iou( + boxes[idx], boxes[kept_idx], pixel_offset=pixel_offset + ) keep = True if overlap <= adaptive_threshold else False else: break @@ -277,7 +323,6 @@ def nms(boxes, scores, nms_threshold, eta=1.0, pixel_offset=True): class TestGenerateProposalsOp(OpTest): - def set_data(self): self.init_test_params() self.init_test_input() @@ -287,7 +332,7 @@ def set_data(self): 'BboxDeltas': self.bbox_deltas, 'ImInfo': self.im_info.astype(np.float32), 'Anchors': self.anchors, - 'Variances': self.variances + 'Variances': self.variances, } self.attrs = { @@ -295,7 +340,7 @@ def set_data(self): 'post_nms_topN': self.post_nms_topN, 'nms_thresh': self.nms_thresh, 'min_size': self.min_size, - 'eta': self.eta + 'eta': self.eta, } self.outputs = { @@ -315,7 +360,7 @@ def init_test_params(self): self.post_nms_topN = 5000 # train 6000, test 1000 self.nms_thresh = 0.7 self.min_size = 3.0 - self.eta = 1. + self.eta = 1.0 def init_test_input(self): batch_size = 1 @@ -323,30 +368,47 @@ def init_test_input(self): layer_h = 16 layer_w = 16 input_feat = np.random.random( - (batch_size, input_channels, layer_h, layer_w)).astype('float32') + (batch_size, input_channels, layer_h, layer_w) + ).astype('float32') self.anchors, self.variances = anchor_generator_in_python( input_feat=input_feat, - anchor_sizes=[16., 32.], + anchor_sizes=[16.0, 32.0], aspect_ratios=[0.5, 1.0], variances=[1.0, 1.0, 1.0, 1.0], stride=[16.0, 16.0], - offset=0.5) - self.im_info = np.array([[64., 64., 8.]]) #im_height, im_width, scale + offset=0.5, + ) + self.im_info = np.array( + [[64.0, 64.0, 8.0]] + ) # im_height, im_width, scale num_anchors = self.anchors.shape[2] self.scores = np.random.random( - (batch_size, num_anchors, layer_h, layer_w)).astype('float32') + (batch_size, num_anchors, layer_h, layer_w) + ).astype('float32') self.bbox_deltas = np.random.random( - (batch_size, num_anchors * 4, layer_h, layer_w)).astype('float32') + (batch_size, num_anchors * 4, layer_h, layer_w) + ).astype('float32') def init_test_output(self): - self.rpn_rois, self.rpn_roi_probs, self.rois_num = generate_proposals_in_python( - self.scores, self.bbox_deltas, self.im_info, self.anchors, - self.variances, self.pre_nms_topN, self.post_nms_topN, - self.nms_thresh, self.min_size, self.eta) + ( + self.rpn_rois, + self.rpn_roi_probs, + self.rois_num, + ) = generate_proposals_in_python( + self.scores, + self.bbox_deltas, + self.im_info, + self.anchors, + self.variances, + self.pre_nms_topN, + self.post_nms_topN, + self.nms_thresh, + self.min_size, + self.eta, + ) class TestGenerateProposalsOutLodOp(TestGenerateProposalsOp): - def set_data(self): self.init_test_params() self.init_test_input() @@ -356,7 +418,7 @@ def set_data(self): 'BboxDeltas': self.bbox_deltas, 'ImInfo': self.im_info.astype(np.float32), 'Anchors': self.anchors, - 'Variances': self.variances + 'Variances': self.variances, } self.attrs = { @@ -365,24 +427,23 @@ def set_data(self): 'nms_thresh': self.nms_thresh, 'min_size': self.min_size, 'eta': self.eta, - 'return_rois_num': True + 'return_rois_num': True, } self.outputs = { 'RpnRois': (self.rpn_rois[0], [self.rois_num]), 'RpnRoiProbs': (self.rpn_roi_probs[0], [self.rois_num]), - 'RpnRoisNum': (np.asarray(self.rois_num, dtype=np.int32)) + 'RpnRoisNum': (np.asarray(self.rois_num, dtype=np.int32)), } class TestGenerateProposalsOpNoBoxLeft(TestGenerateProposalsOp): - def init_test_params(self): self.pre_nms_topN = 12000 # train 12000, test 2000 self.post_nms_topN = 5000 # train 6000, test 1000 self.nms_thresh = 0.7 self.min_size = 1000.0 - self.eta = 1. + self.eta = 1.0 if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_generate_proposals_v2_op.py b/python/paddle/fluid/tests/unittests/test_generate_proposals_v2_op.py index 06db10c5bd512..f0b5d56114167 100644 --- a/python/paddle/fluid/tests/unittests/test_generate_proposals_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_generate_proposals_v2_op.py @@ -34,7 +34,11 @@ def python_generate_proposals_v2( pixel_offset=False, return_rois_num=True, ): - rpn_rois, rpn_roi_probs, rpn_rois_num = paddle.vision.ops.generate_proposals( + ( + rpn_rois, + rpn_roi_probs, + rpn_rois_num, + ) = paddle.vision.ops.generate_proposals( scores, bbox_deltas, img_size, @@ -46,13 +50,24 @@ def python_generate_proposals_v2( min_size=min_size, eta=eta, pixel_offset=pixel_offset, - return_rois_num=return_rois_num) + return_rois_num=return_rois_num, + ) return rpn_rois, rpn_roi_probs -def generate_proposals_v2_in_python(scores, bbox_deltas, im_shape, anchors, - variances, pre_nms_topN, post_nms_topN, - nms_thresh, min_size, eta, pixel_offset): +def generate_proposals_v2_in_python( + scores, + bbox_deltas, + im_shape, + anchors, + variances, + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, + pixel_offset, +): all_anchors = anchors.reshape(-1, 4) rois = np.empty((0, 5), dtype=np.float32) roi_probs = np.empty((0, 1), dtype=np.float32) @@ -63,10 +78,18 @@ def generate_proposals_v2_in_python(scores, bbox_deltas, im_shape, anchors, num_images = scores.shape[0] for img_idx in range(num_images): img_i_boxes, img_i_probs = proposal_for_one_image( - im_shape[img_idx, :], all_anchors, variances, - bbox_deltas[img_idx, :, :, :], scores[img_idx, :, :, :], - pre_nms_topN, post_nms_topN, nms_thresh, min_size, eta, - pixel_offset) + im_shape[img_idx, :], + all_anchors, + variances, + bbox_deltas[img_idx, :, :, :], + scores[img_idx, :, :, :], + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, + pixel_offset, + ) rois_num.append(img_i_probs.shape[0]) rpn_rois.append(img_i_boxes) rpn_roi_probs.append(img_i_probs) @@ -74,9 +97,19 @@ def generate_proposals_v2_in_python(scores, bbox_deltas, im_shape, anchors, return rpn_rois, rpn_roi_probs, rois_num -def proposal_for_one_image(im_shape, all_anchors, variances, bbox_deltas, - scores, pre_nms_topN, post_nms_topN, nms_thresh, - min_size, eta, pixel_offset): +def proposal_for_one_image( + im_shape, + all_anchors, + variances, + bbox_deltas, + scores, + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, + pixel_offset, +): # Transpose and reshape predicted bbox transformations to get them # into the same order as the anchors: # - bbox deltas will be (4 * A, H, W) format from conv output @@ -124,11 +157,13 @@ def proposal_for_one_image(im_shape, all_anchors, variances, bbox_deltas, # take post_nms_topN (e.g. 1000) # return the top proposals if nms_thresh > 0: - keep = nms(boxes=proposals, - scores=scores, - nms_threshold=nms_thresh, - eta=eta, - pixel_offset=pixel_offset) + keep = nms( + boxes=proposals, + scores=scores, + nms_threshold=nms_thresh, + eta=eta, + pixel_offset=pixel_offset, + ) if post_nms_topN > 0 and post_nms_topN < len(keep): keep = keep[:post_nms_topN] proposals = proposals[keep, :] @@ -138,25 +173,27 @@ def proposal_for_one_image(im_shape, all_anchors, variances, bbox_deltas, def filter_boxes(boxes, min_size, im_shape, pixel_offset=True): - """Only keep boxes with both sides >= min_size and center within the image. - """ + """Only keep boxes with both sides >= min_size and center within the image.""" # Scale min_size to match image scale min_size = max(min_size, 1.0) offset = 1 if pixel_offset else 0 ws = boxes[:, 2] - boxes[:, 0] + offset hs = boxes[:, 3] - boxes[:, 1] + offset if pixel_offset: - x_ctr = boxes[:, 0] + ws / 2. - y_ctr = boxes[:, 1] + hs / 2. - keep = np.where((ws >= min_size) & (hs >= min_size) - & (x_ctr < im_shape[1]) & (y_ctr < im_shape[0]))[0] + x_ctr = boxes[:, 0] + ws / 2.0 + y_ctr = boxes[:, 1] + hs / 2.0 + keep = np.where( + (ws >= min_size) + & (hs >= min_size) + & (x_ctr < im_shape[1]) + & (y_ctr < im_shape[0]) + )[0] else: keep = np.where((ws >= min_size) & (hs >= min_size))[0] return keep class TestGenerateProposalsV2Op(OpTest): - def set_data(self): self.init_test_params() self.init_test_input() @@ -166,7 +203,7 @@ def set_data(self): 'BboxDeltas': self.bbox_deltas, 'ImShape': self.im_shape.astype(np.float32), 'Anchors': self.anchors, - 'Variances': self.variances + 'Variances': self.variances, } self.attrs = { @@ -196,7 +233,7 @@ def init_test_params(self): self.post_nms_topN = 5000 # train 6000, test 1000 self.nms_thresh = 0.7 self.min_size = 3.0 - self.eta = 1. + self.eta = 1.0 self.pixel_offset = True def init_test_input(self): @@ -205,26 +242,43 @@ def init_test_input(self): layer_h = 16 layer_w = 16 input_feat = np.random.random( - (batch_size, input_channels, layer_h, layer_w)).astype('float32') + (batch_size, input_channels, layer_h, layer_w) + ).astype('float32') self.anchors, self.variances = anchor_generator_in_python( input_feat=input_feat, - anchor_sizes=[16., 32.], + anchor_sizes=[16.0, 32.0], aspect_ratios=[0.5, 1.0], variances=[1.0, 1.0, 1.0, 1.0], stride=[16.0, 16.0], - offset=0.5) + offset=0.5, + ) self.im_shape = np.array([[64, 64]]).astype('float32') num_anchors = self.anchors.shape[2] self.scores = np.random.random( - (batch_size, num_anchors, layer_h, layer_w)).astype('float32') + (batch_size, num_anchors, layer_h, layer_w) + ).astype('float32') self.bbox_deltas = np.random.random( - (batch_size, num_anchors * 4, layer_h, layer_w)).astype('float32') + (batch_size, num_anchors * 4, layer_h, layer_w) + ).astype('float32') def init_test_output(self): - self.rpn_rois, self.rpn_roi_probs, self.rois_num = generate_proposals_v2_in_python( - self.scores, self.bbox_deltas, self.im_shape, self.anchors, - self.variances, self.pre_nms_topN, self.post_nms_topN, - self.nms_thresh, self.min_size, self.eta, self.pixel_offset) + ( + self.rpn_rois, + self.rpn_roi_probs, + self.rois_num, + ) = generate_proposals_v2_in_python( + self.scores, + self.bbox_deltas, + self.im_shape, + self.anchors, + self.variances, + self.pre_nms_topN, + self.post_nms_topN, + self.nms_thresh, + self.min_size, + self.eta, + self.pixel_offset, + ) # class TestGenerateProposalsV2OpNoBoxLeft(TestGenerateProposalsV2Op): diff --git a/python/paddle/fluid/tests/unittests/test_generator.py b/python/paddle/fluid/tests/unittests/test_generator.py index 445c7b050d026..5adef46fc5de2 100644 --- a/python/paddle/fluid/tests/unittests/test_generator.py +++ b/python/paddle/fluid/tests/unittests/test_generator.py @@ -33,9 +33,9 @@ def test_basic_generator(self): def test_basic_generator_error(self): if paddle.fluid.core.is_compiled_with_cuda(): - self.assertRaises(ValueError, - generator.Generator, - place=paddle.CUDAPlace(0)) + self.assertRaises( + ValueError, generator.Generator, place=paddle.CUDAPlace(0) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_generator_dataloader.py b/python/paddle/fluid/tests/unittests/test_generator_dataloader.py index dbc8169bd1131..67b2c37accb70 100644 --- a/python/paddle/fluid/tests/unittests/test_generator_dataloader.py +++ b/python/paddle/fluid/tests/unittests/test_generator_dataloader.py @@ -41,15 +41,16 @@ def simple_fc_net(places, use_legacy_py_reader, use_double_buffer): with fluid.unique_name.guard(): with fluid.program_guard(main_prog, startup_prog): - image = fluid.layers.data(name='image', - shape=[784], - dtype='float32') + image = fluid.layers.data( + name='image', shape=[784], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') py_reader = fluid.io.DataLoader.from_generator( feed_list=[image, label], capacity=4, iterable=not use_legacy_py_reader, - use_double_buffer=use_double_buffer) + use_double_buffer=use_double_buffer, + ) hidden = image for hidden_size in [10, 20, 30]: hidden = fluid.layers.fc( @@ -57,13 +58,16 @@ def simple_fc_net(places, use_legacy_py_reader, use_double_buffer): size=hidden_size, act='tanh', bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0))) + initializer=fluid.initializer.Constant(value=1.0) + ), + ) - predict_label = fluid.layers.fc(hidden, - size=CLASS_NUM, - act='softmax') + predict_label = fluid.layers.fc( + hidden, size=CLASS_NUM, act='softmax' + ) loss = paddle.mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) @@ -71,28 +75,35 @@ def simple_fc_net(places, use_legacy_py_reader, use_double_buffer): class TestBase(unittest.TestCase): - - def run_main(self, use_legacy_py_reader, with_data_parallel, places, - use_double_buffer): + def run_main( + self, + use_legacy_py_reader, + with_data_parallel, + places, + use_double_buffer, + ): scope = fluid.Scope() with fluid.scope_guard(scope): startup_prog, main_prog, py_reader, loss = simple_fc_net( - places, use_legacy_py_reader, use_double_buffer) + places, use_legacy_py_reader, use_double_buffer + ) reader = paddle.batch(random_reader, batch_size=BATCH_SIZE) ps = places if use_double_buffer else fluid.cpu_places(len(places)) py_reader.set_sample_list_generator( - reader, places=ps if py_reader.iterable else None) + reader, places=ps if py_reader.iterable else None + ) exe = fluid.Executor(place=places[0]) exe.run(startup_prog) prog = fluid.CompiledProgram(main_prog) if with_data_parallel: - prog = prog.with_data_parallel(loss_name=loss.name, - places=places) + prog = prog.with_data_parallel( + loss_name=loss.name, places=places + ) step = 0 step_list = [] @@ -104,9 +115,11 @@ def run_main(self, use_legacy_py_reader, with_data_parallel, places, py_reader.start() while True: try: - L, = exe.run(program=prog, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 except fluid.core.EOFException: @@ -118,7 +131,8 @@ def run_main(self, use_legacy_py_reader, with_data_parallel, places, step = 0 for d in py_reader(): assert len(d) == len(places), "{} != {}".format( - len(d), len(places)) + len(d), len(places) + ) for i, item in enumerate(d): image = item['image'] label = item['label'] @@ -126,10 +140,12 @@ def run_main(self, use_legacy_py_reader, with_data_parallel, places, assert label.shape() == [BATCH_SIZE, 1] assert image._place()._equals(ps[i]) assert label._place()._equals(ps[i]) - L, = exe.run(program=prog, - feed=d, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + feed=d, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 step_list.append(step) @@ -137,7 +153,7 @@ def run_main(self, use_legacy_py_reader, with_data_parallel, places, ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } return ret @@ -167,16 +183,17 @@ def test_main(self): use_legacy_py_reader=use_legacy_py_reader, with_data_parallel=with_data_parallel, places=p, - use_double_buffer=use_double_buffer) + use_double_buffer=use_double_buffer, + ) results.append(ret) if not use_double_buffer: diff = np.max( - np.abs(results[0]['loss'] - results[1]['loss'])) + np.abs(results[0]['loss'] - results[1]['loss']) + ) self.assertLess(diff, 1e-3) class TestDataLoaderBaseAbstract(unittest.TestCase): - def test_main(self): loader = DataLoaderBase() try: diff --git a/python/paddle/fluid/tests/unittests/test_get_all_op_or_kernel_names.py b/python/paddle/fluid/tests/unittests/test_get_all_op_or_kernel_names.py index 999912494ad30..47af7355cb7e2 100644 --- a/python/paddle/fluid/tests/unittests/test_get_all_op_or_kernel_names.py +++ b/python/paddle/fluid/tests/unittests/test_get_all_op_or_kernel_names.py @@ -38,15 +38,15 @@ def test_all_kernels(self): class TestGetAllOpNames(unittest.TestCase): - def test_get_all_op_names(self): all_op_names = core.get_all_op_names() all_op_with_phi_kernels = core.get_all_op_names("phi") all_op_with_fluid_kernels = core.get_all_op_names("fluid") self.assertTrue( - len(all_op_names) > len( - set(all_op_with_phi_kernels) | set(all_op_with_fluid_kernels))) + len(all_op_names) + > len(set(all_op_with_phi_kernels) | set(all_op_with_fluid_kernels)) + ) self.assertTrue("scale" in all_op_with_phi_kernels) self.assertTrue("scale" in all_op_with_phi_kernels) diff --git a/python/paddle/fluid/tests/unittests/test_get_device_properties.py b/python/paddle/fluid/tests/unittests/test_get_device_properties.py index e29b565ff459c..3fea41dbadd67 100644 --- a/python/paddle/fluid/tests/unittests/test_get_device_properties.py +++ b/python/paddle/fluid/tests/unittests/test_get_device_properties.py @@ -18,7 +18,6 @@ class TestGetDeviceProperties(unittest.TestCase): - def test_get_device_properties_default(self): if core.is_compiled_with_cuda(): props = get_device_properties() @@ -44,7 +43,6 @@ def test_get_device_properties_CUDAPlace(self): class TestGetDevicePropertiesError(unittest.TestCase): - def test_error_api(self): if core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_get_inputs_outputs_in_block.py b/python/paddle/fluid/tests/unittests/test_get_inputs_outputs_in_block.py index d19ccea8ea038..b419679403207 100644 --- a/python/paddle/fluid/tests/unittests/test_get_inputs_outputs_in_block.py +++ b/python/paddle/fluid/tests/unittests/test_get_inputs_outputs_in_block.py @@ -21,7 +21,6 @@ class TestGetInputsOutputsInBlock(unittest.TestCase): - def test_ordered(self): # Program variable names may be different when test order is different # This helper makes the test ordered. @@ -49,7 +48,8 @@ def while_body(i): sub_block = main_program.block(1) inner_inputs, inner_outputs = utils.get_inputs_outputs_in_block( - sub_block) + sub_block + ) # 'assign_0.tmp_0', 'assign_1.tmp_0' are name of i and ten in program self.assertTrue(inner_inputs == {'assign_0.tmp_0', 'assign_1.tmp_0'}) # 'tmp_0', 'assign_0.tmp_0' are name of i < ten and i in program @@ -66,7 +66,8 @@ def _test_cond(self): sub_block = main_program.block(1) inner_inputs, inner_outputs = utils.get_inputs_outputs_in_block( - sub_block) + sub_block + ) #'fill_constant_1.tmp_0', 'tmp_3' are names of a, c self.assertTrue(inner_inputs == {'fill_constant_1.tmp_0', 'tmp_3'}) #'_generated_var_1', is name of a + c diff --git a/python/paddle/fluid/tests/unittests/test_get_places_op.py b/python/paddle/fluid/tests/unittests/test_get_places_op.py index 93b03d029cc06..d51e36f2514cd 100644 --- a/python/paddle/fluid/tests/unittests/test_get_places_op.py +++ b/python/paddle/fluid/tests/unittests/test_get_places_op.py @@ -20,7 +20,6 @@ class TestGetPlaces(unittest.TestCase): - @prog_scope() def check_get_cpu_places(self): places = get_places() diff --git a/python/paddle/fluid/tests/unittests/test_get_set_flags.py b/python/paddle/fluid/tests/unittests/test_get_set_flags.py index 80300eb7dfcb4..dad58ae08ba96 100644 --- a/python/paddle/fluid/tests/unittests/test_get_set_flags.py +++ b/python/paddle/fluid/tests/unittests/test_get_set_flags.py @@ -17,11 +17,10 @@ class TestGetAndSetFlags(unittest.TestCase): - def test_api(self): flags = { 'FLAGS_eager_delete_tensor_gb': 1.0, - 'FLAGS_check_nan_inf': True + 'FLAGS_check_nan_inf': True, } fluid.set_flags(flags) @@ -38,7 +37,6 @@ def test_api(self): class TestGetAndSetFlagsErrors(unittest.TestCase): - def test_errors(self): flags_list = ['FLAGS_eager_delete_tensor_gb', 'FLAGS_check_nan_inf'] flag = 1 diff --git a/python/paddle/fluid/tests/unittests/test_get_tensor_from_selected_rows_op.py b/python/paddle/fluid/tests/unittests/test_get_tensor_from_selected_rows_op.py index 425cca109f024..17bf702b88607 100644 --- a/python/paddle/fluid/tests/unittests/test_get_tensor_from_selected_rows_op.py +++ b/python/paddle/fluid/tests/unittests/test_get_tensor_from_selected_rows_op.py @@ -40,7 +40,6 @@ def test_SELECTED_ROWS(): class TestGetTensorFromSelectedRows(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_global_var_getter_setter.py b/python/paddle/fluid/tests/unittests/test_global_var_getter_setter.py index 90fe9988ac2d8..3394a08de8b19 100644 --- a/python/paddle/fluid/tests/unittests/test_global_var_getter_setter.py +++ b/python/paddle/fluid/tests/unittests/test_global_var_getter_setter.py @@ -17,7 +17,6 @@ class VarInfo(object): - def __init__(self, var_name, var_type, writable): self.name = var_name self.type = var_type @@ -25,7 +24,6 @@ def __init__(self, var_name, var_type, writable): class TestGlobalVarGetterSetter(unittest.TestCase): - def test_main(self): var_infos = [ VarInfo("FLAGS_free_idle_chunk", bool, False), diff --git a/python/paddle/fluid/tests/unittests/test_glu.py b/python/paddle/fluid/tests/unittests/test_glu.py index c8f0098456cbd..25f1975db0c52 100644 --- a/python/paddle/fluid/tests/unittests/test_glu.py +++ b/python/paddle/fluid/tests/unittests/test_glu.py @@ -32,7 +32,6 @@ def glu(x, dim=-1): class TestGLUCase(unittest.TestCase): - def setUp(self): self.x = np.random.randn(5, 20) self.dim = -1 @@ -53,7 +52,6 @@ def test_case(self): class TestGLUV2(unittest.TestCase): - def setUp(self): self.x = np.random.randn(5, 20) self.dim = -1 diff --git a/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py b/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py index fc130426aef84..ac786dea529f4 100644 --- a/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py +++ b/python/paddle/fluid/tests/unittests/test_gpu_package_without_gpu_device.py @@ -21,7 +21,6 @@ class TestGPUPackagePaddle(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -34,8 +33,9 @@ def test_import_paddle(self): os.environ['HIP_VISIBLE_DEVICES'] = '' else: os.environ['CUDA_VISIBLE_DEVICES'] = '' - test_file = os.path.join(self.temp_dir.name, - 'test_no_gpu_run_rand.py') + test_file = os.path.join( + self.temp_dir.name, 'test_no_gpu_run_rand.py' + ) with open(test_file, 'w') as wb: cmd_test = """ import paddle @@ -48,10 +48,12 @@ def test_import_paddle(self): _python = sys.executable ps_cmd = '{} {}'.format(_python, test_file) - ps_proc = subprocess.Popen(ps_cmd.strip().split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=os.environ) + ps_proc = subprocess.Popen( + ps_cmd.strip().split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=os.environ, + ) stdout, stderr = ps_proc.communicate() assert 'CPU device will be used by default' in str( diff --git a/python/paddle/fluid/tests/unittests/test_grad_clip_minimize.py b/python/paddle/fluid/tests/unittests/test_grad_clip_minimize.py index fc8c1c946bbec..ad1fc3ad18369 100644 --- a/python/paddle/fluid/tests/unittests/test_grad_clip_minimize.py +++ b/python/paddle/fluid/tests/unittests/test_grad_clip_minimize.py @@ -19,11 +19,14 @@ from paddle.fluid.dygraph.base import to_variable -from paddle.fluid.clip import GradientClipByValue, GradientClipByNorm, GradientClipByGlobalNorm +from paddle.fluid.clip import ( + GradientClipByValue, + GradientClipByNorm, + GradientClipByGlobalNorm, +) class TestGradClipByGlobalNorm(unittest.TestCase): - def init_value(self): self.max_global_norm = 5.0 self.init_scale = 1.0 @@ -35,10 +38,15 @@ def generate_p_g(self): self.para_and_grad = [] for i in range(10): self.para_and_grad.append( - (np.random.uniform(-self.init_scale, self.init_scale, - self.shape).astype('float32'), - np.random.uniform(-self.init_scale, self.init_scale, - self.shape).astype('float32'))) + ( + np.random.uniform( + -self.init_scale, self.init_scale, self.shape + ).astype('float32'), + np.random.uniform( + -self.init_scale, self.init_scale, self.shape + ).astype('float32'), + ) + ) def get_numpy_global_norm_result(self): gloabl_norm = 0.0 @@ -98,7 +106,6 @@ def test_clip_by_global_norm_2(self): class TestGradClipByNorm(unittest.TestCase): - def init_value(self): self.max_norm = 5.0 self.init_scale = 1.0 @@ -110,10 +117,15 @@ def generate_p_g(self): self.para_and_grad = [] for i in range(10): self.para_and_grad.append( - (np.random.uniform(-self.init_scale, self.init_scale, - self.shape).astype('float32'), - np.random.uniform(-self.init_scale, self.init_scale, - self.shape).astype('float32'))) + ( + np.random.uniform( + -self.init_scale, self.init_scale, self.shape + ).astype('float32'), + np.random.uniform( + -self.init_scale, self.init_scale, self.shape + ).astype('float32'), + ) + ) def get_numpy_norm_result(self): @@ -169,7 +181,6 @@ def test_clip_by_norm_2(self): class TestGradClipByValue(unittest.TestCase): - def init_value(self): self.max_value = 0.8 self.min_value = -0.1 @@ -182,10 +193,15 @@ def generate_p_g(self): self.para_and_grad = [] for i in range(10): self.para_and_grad.append( - (np.random.uniform(-self.init_scale, self.init_scale, - self.shape).astype('float32'), - np.random.uniform(-self.init_scale, self.init_scale, - self.shape).astype('float32'))) + ( + np.random.uniform( + -self.init_scale, self.init_scale, self.shape + ).astype('float32'), + np.random.uniform( + -self.init_scale, self.init_scale, self.shape + ).astype('float32'), + ) + ) def get_numpy_clip_result(self): @@ -197,8 +213,9 @@ def get_numpy_clip_result(self): def get_dygrap_clip_result(self): with fluid.dygraph.guard(): - value_clip = GradientClipByValue(max=self.max_value, - min=self.min_value) + value_clip = GradientClipByValue( + max=self.max_value, min=self.min_value + ) p_g_var = [] for p, g in self.para_and_grad: new_p = to_variable(p) diff --git a/python/paddle/fluid/tests/unittests/test_gradient_clip.py b/python/paddle/fluid/tests/unittests/test_gradient_clip.py index 825e84f9f6d9a..1687d7e59839f 100644 --- a/python/paddle/fluid/tests/unittests/test_gradient_clip.py +++ b/python/paddle/fluid/tests/unittests/test_gradient_clip.py @@ -23,21 +23,17 @@ paddle.enable_static() -def bow_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2): +def bow_net( + data, label, dict_dim, emb_dim=128, hid_dim=128, hid_dim2=96, class_dim=2 +): """ BOW net This model is from https://github.com/PaddlePaddle/models: fluid/PaddleNLP/text_classification/nets.py """ - emb = fluid.layers.embedding(input=data, - is_sparse=True, - size=[dict_dim, emb_dim]) + emb = fluid.layers.embedding( + input=data, is_sparse=True, size=[dict_dim, emb_dim] + ) bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') bow_tanh = fluid.layers.tanh(bow) fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh") @@ -50,7 +46,6 @@ def bow_net(data, class TestGradientClip(unittest.TestCase): - def setUp(self): self.word_dict_len = 5147 self.BATCH_SIZE = 2 @@ -74,8 +69,9 @@ def check_clip_result(self, out, out_clip): def check_gradient_clip(self, place, dtype='float32'): prog = fluid.Program() startup_program = fluid.Program() - with fluid.program_guard(main_program=prog, - startup_program=startup_program): + with fluid.program_guard( + main_program=prog, startup_program=startup_program + ): image = fluid.data(name="a", shape=[-1, 784], dtype='float32') label = fluid.data(name="b", shape=[-1, 1], dtype='int64') if dtype != 'float32': @@ -96,8 +92,9 @@ def check_gradient_clip(self, place, dtype='float32'): p_g = sorted(p_g, key=lambda x: x[0].name) p_g_clip = sorted(p_g_clip, key=lambda x: x[0].name) - with fluid.program_guard(main_program=prog_clip, - startup_program=startup_program): + with fluid.program_guard( + main_program=prog_clip, startup_program=startup_program + ): p_g_clip = self.clip_gradient(p_g_clip) grad_list = [elem[1] for elem in p_g] @@ -110,20 +107,20 @@ def check_gradient_clip(self, place, dtype='float32'): data = next(train_reader()) out = exe.run(prog, feed=feeder.feed(data), fetch_list=grad_list) - out_clip = exe.run(prog_clip, - feed=feeder.feed(data), - fetch_list=grad_clip_list) + out_clip = exe.run( + prog_clip, feed=feeder.feed(data), fetch_list=grad_clip_list + ) self.check_clip_result(out, out_clip) def check_sparse_gradient_clip(self, place): prog = fluid.Program() startup_program = fluid.Program() - with fluid.program_guard(main_program=prog, - startup_program=startup_program): - data = fluid.data(name="words", - shape=[-1, 1], - dtype="int64", - lod_level=1) + with fluid.program_guard( + main_program=prog, startup_program=startup_program + ): + data = fluid.data( + name="words", shape=[-1, 1], dtype="int64", lod_level=1 + ) label = fluid.data(name="label", shape=[-1, 1], dtype="int64") cost = bow_net(data, label, self.word_dict_len) @@ -135,7 +132,7 @@ def check_sparse_gradient_clip(self, place): data = next(self.train_data()) val = exe.run(prog, feed=feeder.feed(data), fetch_list=[cost])[0] - self.assertEqual((1, ), val.shape) + self.assertEqual((1,), val.shape) self.assertFalse(np.isnan(val)) def backward_and_optimize(self, cost): @@ -143,7 +140,6 @@ def backward_and_optimize(self, cost): class TestGradientClipByGlobalNorm(TestGradientClip): - def init(self): self.clip_norm = 0.2 @@ -163,13 +159,13 @@ def check_clip_result(self, out, out_clip): v, rtol=1e-05, atol=1e-08, - err_msg= - 'gradient clip by global norm has wrong results!, \nu={}\nv={}\ndiff={}' - .format(u, v, u - v)) + err_msg='gradient clip by global norm has wrong results!, \nu={}\nv={}\ndiff={}'.format( + u, v, u - v + ), + ) # test whether the output is right when use 'set_gradient_clip' def test_old_gradient_clip(self): - def func(params_grads): clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=self.clip_norm) fluid.clip.set_gradient_clip(clip) @@ -180,7 +176,6 @@ def func(params_grads): # test whether the output is right when use grad_clip def test_new_gradient_clip(self): - def func(params_grads): clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=self.clip_norm) return clip(params_grads) @@ -190,7 +185,6 @@ def func(params_grads): # test whether the output is right when use grad_clip under float64 def test_new_gradient_clip_fp64(self): - def func(params_grads): clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=self.clip_norm) return clip(params_grads) @@ -200,12 +194,12 @@ def func(params_grads): # invoke 'set_gradient_clip' in a wrong order def test_wrong_API_order(self): - def backward_func(cost): clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=5.0) fluid.clip.set_gradient_clip(clip) - sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.01, - grad_clip=clip) + sgd_optimizer = fluid.optimizer.SGD( + learning_rate=0.01, grad_clip=clip + ) # if 'set_gradient_clip' and 'optimize(grad_clip)' together, 'set_gradient_clip' will be ineffective sgd_optimizer.minimize(cost) # 'set_gradient_clip' must before 'minimize', otherwise, 'set_gradient_clip' will be ineffective @@ -219,43 +213,74 @@ def backward_func(cost): def test_tpyeError(self): # the type of optimizer(grad_clip=) must be an instance of GradientClipBase's derived class with self.assertRaises(TypeError): - sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.1, - grad_clip="test") + sgd_optimizer = fluid.optimizer.SGD( + learning_rate=0.1, grad_clip="test" + ) # if grad is None or not need clip def test_none_grad_fp32(self): ops = self._test_none_grad_helper("float32") - self.assertListEqual(ops, [ - 'squared_l2_norm', 'squared_l2_norm', 'sum', 'sqrt', - 'fill_constant', 'elementwise_max', 'elementwise_div', - 'elementwise_mul', 'elementwise_mul' - ]) + self.assertListEqual( + ops, + [ + 'squared_l2_norm', + 'squared_l2_norm', + 'sum', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'elementwise_mul', + 'elementwise_mul', + ], + ) def test_none_grad_fp16(self): ops = self._test_none_grad_helper("float16") - self.assertListEqual(ops, [ - 'square', 'reduce_sum', 'square', 'reduce_sum', 'sum', 'cast', - 'sqrt', 'fill_constant', 'elementwise_max', 'elementwise_div', - 'cast', 'elementwise_mul', 'cast', 'elementwise_mul' - ]) + self.assertListEqual( + ops, + [ + 'square', + 'reduce_sum', + 'square', + 'reduce_sum', + 'sum', + 'cast', + 'sqrt', + 'fill_constant', + 'elementwise_max', + 'elementwise_div', + 'cast', + 'elementwise_mul', + 'cast', + 'elementwise_mul', + ], + ) def _test_none_grad_helper(self, dtype): prog = fluid.Program() startup_program = fluid.Program() - with fluid.program_guard(main_program=prog, - startup_program=startup_program): + with fluid.program_guard( + main_program=prog, startup_program=startup_program + ): clip = fluid.clip.GradientClipByGlobalNorm(self.clip_norm) - x = fluid.default_main_program().global_block().create_parameter( - name="x", shape=[2, 3], dtype=dtype) - y = fluid.default_main_program().global_block().create_parameter( - name="y", shape=[2, 3], dtype=dtype) + x = ( + fluid.default_main_program() + .global_block() + .create_parameter(name="x", shape=[2, 3], dtype=dtype) + ) + y = ( + fluid.default_main_program() + .global_block() + .create_parameter(name="y", shape=[2, 3], dtype=dtype) + ) # (x, None) should not be returned params_grads = [(x, None), (x, y), (y, x)] params_grads = clip(params_grads) self.assertTrue( len(params_grads) == 2, - "ClipByGlobalNorm: when grad is None, it shouldn't be returned by gradient clip!" + "ClipByGlobalNorm: when grad is None, it shouldn't be returned by gradient clip!", ) ops = [op.type for op in x.block.ops] @@ -263,7 +288,6 @@ def _test_none_grad_helper(self, dtype): class TestGradientClipByNorm(TestGradientClip): - def init(self): self.clip_norm = 0.2 @@ -277,11 +301,11 @@ def check_clip_result(self, out, out_clip): v, rtol=1e-05, atol=1e-08, - err_msg='gradient clip by norm has wrong results!') + err_msg='gradient clip by norm has wrong results!', + ) # test whether the output is right when use grad_clip def test_gradient_clip(self): - def func(params_grads): clip = fluid.clip.GradientClipByNorm(clip_norm=self.clip_norm) return clip(params_grads) @@ -292,25 +316,35 @@ def func(params_grads): # if grad is None or not need clip def test_none_grad(self): clip = fluid.clip.GradientClipByNorm(self.clip_norm) - x = fluid.default_main_program().global_block().create_parameter( - name="x", shape=[2, 3], dtype="float32", need_clip=False) - y = fluid.default_main_program().global_block().create_parameter( - name="y", shape=[2, 3], dtype="float32", need_clip=False) + x = ( + fluid.default_main_program() + .global_block() + .create_parameter( + name="x", shape=[2, 3], dtype="float32", need_clip=False + ) + ) + y = ( + fluid.default_main_program() + .global_block() + .create_parameter( + name="y", shape=[2, 3], dtype="float32", need_clip=False + ) + ) # (x, None) should not be returned params_grads = [(x, None), (x, y)] params_grads = clip(params_grads) self.assertTrue( len(clip(params_grads)) == 1, - "ClipGradByNorm: when grad is None, it shouldn't be returned by gradient clip!" + "ClipGradByNorm: when grad is None, it shouldn't be returned by gradient clip!", ) self.assertTrue( params_grads[0][1].name == 'y', - "ClipGradByNorm: grad should not be clipped when filtered out!") + "ClipGradByNorm: grad should not be clipped when filtered out!", + ) class TestGradientClipByValue(TestGradientClip): - def init(self): self.max = 0.2 self.min = 0.1 @@ -325,11 +359,11 @@ def check_clip_result(self, out, out_clip): v, rtol=1e-06, atol=1e-08, - err_msg='gradient clip by value has wrong results!') + err_msg='gradient clip by value has wrong results!', + ) # test whether the output is right when use grad_clip def test_gradient_clip(self): - def func(params_grads): clip = fluid.clip.GradientClipByValue(max=self.max, min=self.min) return clip(params_grads) @@ -340,37 +374,49 @@ def func(params_grads): # if grad is None or not need clip def test_none_grad(self): clip = fluid.clip.GradientClipByValue(self.max, self.min) - x = fluid.default_main_program().global_block().create_parameter( - name="x", shape=[2, 3], dtype="float32", need_clip=False) - y = fluid.default_main_program().global_block().create_parameter( - name="y", shape=[2, 3], dtype="float32", need_clip=False) + x = ( + fluid.default_main_program() + .global_block() + .create_parameter( + name="x", shape=[2, 3], dtype="float32", need_clip=False + ) + ) + y = ( + fluid.default_main_program() + .global_block() + .create_parameter( + name="y", shape=[2, 3], dtype="float32", need_clip=False + ) + ) # (x, None) should not be returned params_grads = [(x, None), (x, y)] params_grads = clip(params_grads) self.assertTrue( len(clip(params_grads)) == 1, - "ClipGradByValue: when grad is None, it shouldn't be returned by gradient clip!" + "ClipGradByValue: when grad is None, it shouldn't be returned by gradient clip!", ) self.assertTrue( params_grads[0][1].name == 'y', - "ClipGradByValue: grad should not be clipped when filtered out!") + "ClipGradByValue: grad should not be clipped when filtered out!", + ) class TestDygraphGradientClip(unittest.TestCase): - def test_gradient_clip(self): with fluid.dygraph.guard(): linear = fluid.dygraph.Linear(5, 5) - inputs = fluid.layers.uniform_random([16, 5], min=-10, - max=10).astype('float32') + inputs = fluid.layers.uniform_random( + [16, 5], min=-10, max=10 + ).astype('float32') out = linear(fluid.dygraph.to_variable(inputs)) loss = fluid.layers.reduce_mean(out) loss.backward() sgd_optimizer = fluid.optimizer.SGD( learning_rate=0.0, parameter_list=linear.parameters(), - grad_clip=fluid.clip.GradientClipByGlobalNorm(0.1)) + grad_clip=fluid.clip.GradientClipByGlobalNorm(0.1), + ) self.check_clip_result(loss, sgd_optimizer) def check_clip_result(self, loss, optimizer): @@ -378,20 +424,23 @@ def check_clip_result(self, loss, optimizer): class TestDygraphGradientClipByGlobalNorm(TestDygraphGradientClip): - def setUp(self): self.clip_norm = 0.8 self.clip1 = fluid.clip.GradientClipByGlobalNorm( - clip_norm=self.clip_norm) + clip_norm=self.clip_norm + ) self.clip2 = fluid.clip.GradientClipByGlobalNorm( - clip_norm=self.clip_norm) + clip_norm=self.clip_norm + ) def check_clip_result(self, loss, optimizer): # if grad is None - x = fluid.dygraph.to_variable(np.array([2, 3]).astype("float32"), - name="x") - y = fluid.dygraph.to_variable(np.array([3, 4]).astype("float32"), - name="y") + x = fluid.dygraph.to_variable( + np.array([2, 3]).astype("float32"), name="x" + ) + y = fluid.dygraph.to_variable( + np.array([3, 4]).astype("float32"), name="y" + ) assert len(self.clip1([(x, x), (x, y), (x, None)])) == 2 # get params and grads from network opt, params_grads = optimizer.minimize(loss) @@ -416,11 +465,11 @@ def check_clip_result(self, loss, optimizer): self.assertTrue( np.isclose(a=a, b=b, rtol=1e-6, atol=1e-8), "gradient clip by global norm has wrong results, expetcd:%f, but received:%f" - % (a, b)) + % (a, b), + ) class TestDygraphGradientClipByNorm(TestDygraphGradientClip): - def setUp(self): self.clip_norm = 0.8 self.clip = fluid.clip.GradientClipByNorm(clip_norm=self.clip_norm) @@ -445,11 +494,11 @@ def check_clip_result(self, loss, optimizer): self.assertTrue( np.isclose(a=a, b=b, rtol=1e-6, atol=1e-8), "gradient clip by norm has wrong results, expetcd:%f, but received:%f" - % (a, b)) + % (a, b), + ) class TestDygraphGradientClipByValue(TestDygraphGradientClip): - def setUp(self): self.max = 0.2 self.min = 0.1 @@ -472,11 +521,11 @@ def check_clip_result(self, loss, optimizer): v, rtol=1e-06, atol=1e-08, - err_msg='gradient clip by value has wrong results!') + err_msg='gradient clip by value has wrong results!', + ) class SimpleNet(paddle.nn.Layer): - def __init__(self): super(SimpleNet, self).__init__() self.linear = paddle.nn.Linear(5, 5) @@ -489,19 +538,21 @@ def forward(self, x): class TestDygraphGradientClipFP16(unittest.TestCase): - def test_gradient_clip(self): if fluid.core.is_compiled_with_cuda(): with fluid.dygraph.guard(): paddle.seed(10) model = SimpleNet() sgd_optimizer = paddle.optimizer.SGD( - learning_rate=0.0, parameters=model.parameters()) + learning_rate=0.0, parameters=model.parameters() + ) model, sgd_optimizer = paddle.amp.decorate( - models=model, optimizers=sgd_optimizer, level='O2') + models=model, optimizers=sgd_optimizer, level='O2' + ) scaler = paddle.amp.GradScaler(init_loss_scaling=1024) - inputs = fluid.layers.uniform_random([1, 5], min=-10, - max=10).astype('float32') + inputs = fluid.layers.uniform_random( + [1, 5], min=-10, max=10 + ).astype('float32') with paddle.amp.auto_cast(level='O2'): out = model(fluid.dygraph.to_variable(inputs)) loss = fluid.layers.reduce_mean(out) @@ -540,15 +591,16 @@ def test_gradient_clip(self): self.assertTrue( np.isclose(a=a, b=b, rtol=1e-3, atol=1e-8), "gradient clip by global norm has wrong results, expetcd:%f, but received:%f" - % (a, b)) + % (a, b), + ) class TestDygraphGradientClipFP64(unittest.TestCase): - def test_gradient_clip(self): with fluid.dygraph.guard(): - inputs = fluid.layers.uniform_random([16, 5], min=-10, - max=10).astype('float64') + inputs = fluid.layers.uniform_random( + [16, 5], min=-10, max=10 + ).astype('float64') linear = fluid.dygraph.Linear(5, 5, dtype="float64") out = linear(fluid.dygraph.to_variable(inputs)) loss = fluid.layers.reduce_mean(out) @@ -586,11 +638,11 @@ def test_gradient_clip(self): self.assertTrue( np.isclose(a=a, b=b, rtol=1e-6, atol=1e-8), "gradient clip by global norm has wrong results, expetcd:%f, but received:%f" - % (a, b)) + % (a, b), + ) class TestPureFP16ClipGradByGlobalNorm(unittest.TestCase): - def check_main(self, expected_has_cast_op): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -601,12 +653,12 @@ def check_main(self, expected_has_cast_op): param_and_grads = [] main_block = main_prog.global_block() for name, shape in zip(names, shapes): - p = main_block.create_parameter(name=name, - shape=shape, - dtype='float16') - g = main_block.create_parameter(name=p.name + '@GRAD', - shape=p.shape, - dtype=p.dtype) + p = main_block.create_parameter( + name=name, shape=shape, dtype='float16' + ) + g = main_block.create_parameter( + name=p.name + '@GRAD', shape=p.shape, dtype=p.dtype + ) param_and_grads.append((p, g)) clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) diff --git a/python/paddle/fluid/tests/unittests/test_graph_khop_sampler.py b/python/paddle/fluid/tests/unittests/test_graph_khop_sampler.py index 57b8209d8d69b..b2f57af35de90 100644 --- a/python/paddle/fluid/tests/unittests/test_graph_khop_sampler.py +++ b/python/paddle/fluid/tests/unittests/test_graph_khop_sampler.py @@ -19,7 +19,6 @@ class TestGraphKhopSampler(unittest.TestCase): - def setUp(self): num_nodes = 20 edges = np.random.randint(num_nodes, size=(100, 2)) @@ -42,8 +41,9 @@ def setUp(self): self.row = sorted_edges[:, 0].astype("int64") self.colptr = colptr.astype("int64") self.sorted_eid = sorted_eid.astype("int64") - self.nodes = np.unique(np.random.randint(num_nodes, - size=5)).astype("int64") + self.nodes = np.unique(np.random.randint(num_nodes, size=5)).astype( + "int64" + ) self.sample_sizes = [5, 5] self.dst_src_dict = dst_src_dict @@ -53,10 +53,14 @@ def func_sample_result(self): colptr = paddle.to_tensor(self.colptr) nodes = paddle.to_tensor(self.nodes) - edge_src, edge_dst, sample_index, reindex_nodes = \ - paddle.incubate.graph_khop_sampler(row, colptr, - nodes, self.sample_sizes, - return_eids=False) + ( + edge_src, + edge_dst, + sample_index, + reindex_nodes, + ) = paddle.incubate.graph_khop_sampler( + row, colptr, nodes, self.sample_sizes, return_eids=False + ) # Reindex edge_src and edge_dst to original index. edge_src = edge_src.reshape([-1]) edge_dst = edge_dst.reshape([-1]) @@ -72,10 +76,13 @@ def func_sample_result(self): continue # Ensure no repetitive sample neighbors. self.assertTrue( - edge_src_n.shape[0] == paddle.unique(edge_src_n).shape[0]) + edge_src_n.shape[0] == paddle.unique(edge_src_n).shape[0] + ) # Ensure the correct sample size. - self.assertTrue(edge_src_n.shape[0] == self.sample_sizes[0] - or edge_src_n.shape[0] == len(self.dst_src_dict[n])) + self.assertTrue( + edge_src_n.shape[0] == self.sample_sizes[0] + or edge_src_n.shape[0] == len(self.dst_src_dict[n]) + ) in_neighbors = np.isin(edge_src_n.numpy(), self.dst_src_dict[n]) # Ensure the correct sample neighbors. self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) @@ -91,22 +98,35 @@ def func_uva_sample_result(self): row = None if fluid.framework.in_dygraph_mode(): row = paddle.fluid.core.eager.to_uva_tensor( - self.row.astype(self.row.dtype), 0) + self.row.astype(self.row.dtype), 0 + ) sorted_eid = paddle.fluid.core.eager.to_uva_tensor( - self.sorted_eid.astype(self.sorted_eid.dtype), 0) + self.sorted_eid.astype(self.sorted_eid.dtype), 0 + ) else: row = paddle.fluid.core.to_uva_tensor( - self.row.astype(self.row.dtype)) + self.row.astype(self.row.dtype) + ) sorted_eid = paddle.fluid.core.to_uva_tensor( - self.sorted_eid.astype(self.sorted_eid.dtype)) + self.sorted_eid.astype(self.sorted_eid.dtype) + ) colptr = paddle.to_tensor(self.colptr) nodes = paddle.to_tensor(self.nodes) - edge_src, edge_dst, sample_index, reindex_nodes, edge_eids = \ - paddle.incubate.graph_khop_sampler(row, colptr, - nodes, self.sample_sizes, - sorted_eids=sorted_eid, - return_eids=True) + ( + edge_src, + edge_dst, + sample_index, + reindex_nodes, + edge_eids, + ) = paddle.incubate.graph_khop_sampler( + row, + colptr, + nodes, + self.sample_sizes, + sorted_eids=sorted_eid, + return_eids=True, + ) edge_src = edge_src.reshape([-1]) edge_dst = edge_dst.reshape([-1]) sample_index = sample_index.reshape([-1]) @@ -120,10 +140,12 @@ def func_uva_sample_result(self): if edge_src_n.shape[0] == 0: continue self.assertTrue( - edge_src_n.shape[0] == paddle.unique(edge_src_n).shape[0]) + edge_src_n.shape[0] == paddle.unique(edge_src_n).shape[0] + ) self.assertTrue( edge_src_n.shape[0] == self.sample_sizes[0] - or edge_src_n.shape[0] == len(self.dst_src_dict[n])) + or edge_src_n.shape[0] == len(self.dst_src_dict[n]) + ) in_neighbors = np.isin(edge_src_n.numpy(), self.dst_src_dict[n]) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) @@ -135,31 +157,40 @@ def test_uva_sample_result(self): def test_sample_result_static_with_eids(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - row = paddle.static.data(name="row", - shape=self.row.shape, - dtype=self.row.dtype) - sorted_eids = paddle.static.data(name="eids", - shape=self.sorted_eid.shape, - dtype=self.sorted_eid.dtype) - colptr = paddle.static.data(name="colptr", - shape=self.colptr.shape, - dtype=self.colptr.dtype) - nodes = paddle.static.data(name="nodes", - shape=self.nodes.shape, - dtype=self.nodes.dtype) - - edge_src, edge_dst, sample_index, reindex_nodes, edge_eids = \ - paddle.incubate.graph_khop_sampler(row, colptr, - nodes, self.sample_sizes, - sorted_eids, True) + row = paddle.static.data( + name="row", shape=self.row.shape, dtype=self.row.dtype + ) + sorted_eids = paddle.static.data( + name="eids", + shape=self.sorted_eid.shape, + dtype=self.sorted_eid.dtype, + ) + colptr = paddle.static.data( + name="colptr", shape=self.colptr.shape, dtype=self.colptr.dtype + ) + nodes = paddle.static.data( + name="nodes", shape=self.nodes.shape, dtype=self.nodes.dtype + ) + + ( + edge_src, + edge_dst, + sample_index, + reindex_nodes, + edge_eids, + ) = paddle.incubate.graph_khop_sampler( + row, colptr, nodes, self.sample_sizes, sorted_eids, True + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'row': self.row, - 'eids': self.sorted_eid, - 'colptr': self.colptr, - 'nodes': self.nodes - }, - fetch_list=[edge_src, edge_dst, sample_index]) + ret = exe.run( + feed={ + 'row': self.row, + 'eids': self.sorted_eid, + 'colptr': self.colptr, + 'nodes': self.nodes, + }, + fetch_list=[edge_src, edge_dst, sample_index], + ) edge_src, edge_dst, sample_index = ret edge_src = edge_src.reshape([-1]) @@ -175,35 +206,44 @@ def test_sample_result_static_with_eids(self): if edge_src_n.shape[0] == 0: continue self.assertTrue( - edge_src_n.shape[0] == np.unique(edge_src_n).shape[0]) + edge_src_n.shape[0] == np.unique(edge_src_n).shape[0] + ) self.assertTrue( edge_src_n.shape[0] == self.sample_sizes[0] - or edge_src_n.shape[0] == len(self.dst_src_dict[n])) + or edge_src_n.shape[0] == len(self.dst_src_dict[n]) + ) in_neighbors = np.isin(edge_src_n, self.dst_src_dict[n]) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) def test_sample_result_static_without_eids(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - row = paddle.static.data(name="row", - shape=self.row.shape, - dtype=self.row.dtype) - colptr = paddle.static.data(name="colptr", - shape=self.colptr.shape, - dtype=self.colptr.dtype) - nodes = paddle.static.data(name="nodes", - shape=self.nodes.shape, - dtype=self.nodes.dtype) - edge_src, edge_dst, sample_index, reindex_nodes = \ - paddle.incubate.graph_khop_sampler(row, colptr, - nodes, self.sample_sizes) + row = paddle.static.data( + name="row", shape=self.row.shape, dtype=self.row.dtype + ) + colptr = paddle.static.data( + name="colptr", shape=self.colptr.shape, dtype=self.colptr.dtype + ) + nodes = paddle.static.data( + name="nodes", shape=self.nodes.shape, dtype=self.nodes.dtype + ) + ( + edge_src, + edge_dst, + sample_index, + reindex_nodes, + ) = paddle.incubate.graph_khop_sampler( + row, colptr, nodes, self.sample_sizes + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'row': self.row, - 'colptr': self.colptr, - 'nodes': self.nodes - }, - fetch_list=[edge_src, edge_dst, sample_index]) + ret = exe.run( + feed={ + 'row': self.row, + 'colptr': self.colptr, + 'nodes': self.nodes, + }, + fetch_list=[edge_src, edge_dst, sample_index], + ) edge_src, edge_dst, sample_index = ret edge_src = edge_src.reshape([-1]) edge_dst = edge_dst.reshape([-1]) @@ -218,10 +258,12 @@ def test_sample_result_static_without_eids(self): if edge_src_n.shape[0] == 0: continue self.assertTrue( - edge_src_n.shape[0] == np.unique(edge_src_n).shape[0]) + edge_src_n.shape[0] == np.unique(edge_src_n).shape[0] + ) self.assertTrue( edge_src_n.shape[0] == self.sample_sizes[0] - or edge_src_n.shape[0] == len(self.dst_src_dict[n])) + or edge_src_n.shape[0] == len(self.dst_src_dict[n]) + ) in_neighbors = np.isin(edge_src_n, self.dst_src_dict[n]) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) diff --git a/python/paddle/fluid/tests/unittests/test_graph_reindex.py b/python/paddle/fluid/tests/unittests/test_graph_reindex.py index db1d7d3cff99e..318deda61f2a7 100644 --- a/python/paddle/fluid/tests/unittests/test_graph_reindex.py +++ b/python/paddle/fluid/tests/unittests/test_graph_reindex.py @@ -18,7 +18,6 @@ class TestGraphReindex(unittest.TestCase): - def setUp(self): self.x = np.arange(5).astype("int64") self.neighbors = np.random.randint(100, size=20).astype("int64") @@ -32,7 +31,8 @@ def setUp(self): self.out_nodes = np.array(out_nodes, dtype="int64") reindex_dict = {node: ind for ind, node in enumerate(self.out_nodes)} self.reindex_src = np.array( - [reindex_dict[node] for node in self.neighbors]) + [reindex_dict[node] for node in self.neighbors] + ) reindex_dst = [] for node, c in zip(self.x, self.count): for i in range(c): @@ -48,16 +48,21 @@ def test_reindex_result(self): value_buffer = paddle.full([self.num_nodes], -1, dtype="int32") index_buffer = paddle.full([self.num_nodes], -1, dtype="int32") - reindex_src, reindex_dst, out_nodes = \ - paddle.incubate.graph_reindex(x, neighbors, count) + reindex_src, reindex_dst, out_nodes = paddle.incubate.graph_reindex( + x, neighbors, count + ) np.testing.assert_allclose(self.reindex_src, reindex_src, rtol=1e-05) np.testing.assert_allclose(self.reindex_dst, reindex_dst, rtol=1e-05) np.testing.assert_allclose(self.out_nodes, out_nodes, rtol=1e-05) - reindex_src, reindex_dst, out_nodes = \ - paddle.incubate.graph_reindex(x, neighbors, count, - value_buffer, index_buffer, - flag_buffer_hashtable=True) + reindex_src, reindex_dst, out_nodes = paddle.incubate.graph_reindex( + x, + neighbors, + count, + value_buffer, + index_buffer, + flag_buffer_hashtable=True, + ) np.testing.assert_allclose(self.reindex_src, reindex_src, rtol=1e-05) np.testing.assert_allclose(self.reindex_dst, reindex_dst, rtol=1e-05) np.testing.assert_allclose(self.out_nodes, out_nodes, rtol=1e-05) @@ -70,20 +75,21 @@ def test_heter_reindex_result(self): count = paddle.to_tensor(self.count) count = paddle.concat([count, count]) - reindex_src, reindex_dst, out_nodes = \ - paddle.incubate.graph_reindex(x, neighbors, count) - np.testing.assert_allclose(self.reindex_src, - reindex_src[:self.neighbors.shape[0]], - rtol=1e-05) - np.testing.assert_allclose(self.reindex_src, - reindex_src[self.neighbors.shape[0]:], - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst[:self.neighbors.shape[0]], - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst[self.neighbors.shape[0]:], - rtol=1e-05) + reindex_src, reindex_dst, out_nodes = paddle.incubate.graph_reindex( + x, neighbors, count + ) + np.testing.assert_allclose( + self.reindex_src, reindex_src[: self.neighbors.shape[0]], rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_src, reindex_src[self.neighbors.shape[0] :], rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst[: self.neighbors.shape[0]], rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst[self.neighbors.shape[0] :], rtol=1e-05 + ) np.testing.assert_allclose(self.out_nodes, out_nodes, rtol=1e-05) def test_heter_reindex_result_v2(self): @@ -111,10 +117,11 @@ def test_heter_reindex_result_v2(self): reindex_dst.append(reindex_dict[node]) reindex_dst = np.array(reindex_dst, dtype="int64") - reindex_src_, reindex_dst_, out_nodes_ = \ - paddle.incubate.graph_reindex(paddle.to_tensor(x), - paddle.to_tensor(neighbors), - paddle.to_tensor(counts)) + reindex_src_, reindex_dst_, out_nodes_ = paddle.incubate.graph_reindex( + paddle.to_tensor(x), + paddle.to_tensor(neighbors), + paddle.to_tensor(counts), + ) np.testing.assert_allclose(reindex_src, reindex_src_, rtol=1e-05) np.testing.assert_allclose(reindex_dst, reindex_dst_, rtol=1e-05) np.testing.assert_allclose(out_nodes, out_nodes_, rtol=1e-05) @@ -122,66 +129,89 @@ def test_heter_reindex_result_v2(self): def test_reindex_result_static(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - x = paddle.static.data(name="x", - shape=self.x.shape, - dtype=self.x.dtype) - neighbors = paddle.static.data(name="neighbors", - shape=self.neighbors.shape, - dtype=self.neighbors.dtype) - count = paddle.static.data(name="count", - shape=self.count.shape, - dtype=self.count.dtype) - value_buffer = paddle.static.data(name="value_buffer", - shape=[self.num_nodes], - dtype="int32") - index_buffer = paddle.static.data(name="index_buffer", - shape=[self.num_nodes], - dtype="int32") - - reindex_src_1, reindex_dst_1, out_nodes_1 = \ - paddle.incubate.graph_reindex(x, neighbors, count) - reindex_src_2, reindex_dst_2, out_nodes_2 = \ - paddle.incubate.graph_reindex(x, neighbors, count, - value_buffer, index_buffer, - flag_buffer_hashtable=True) + x = paddle.static.data( + name="x", shape=self.x.shape, dtype=self.x.dtype + ) + neighbors = paddle.static.data( + name="neighbors", + shape=self.neighbors.shape, + dtype=self.neighbors.dtype, + ) + count = paddle.static.data( + name="count", shape=self.count.shape, dtype=self.count.dtype + ) + value_buffer = paddle.static.data( + name="value_buffer", shape=[self.num_nodes], dtype="int32" + ) + index_buffer = paddle.static.data( + name="index_buffer", shape=[self.num_nodes], dtype="int32" + ) + + ( + reindex_src_1, + reindex_dst_1, + out_nodes_1, + ) = paddle.incubate.graph_reindex(x, neighbors, count) + ( + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ) = paddle.incubate.graph_reindex( + x, + neighbors, + count, + value_buffer, + index_buffer, + flag_buffer_hashtable=True, + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'x': - self.x, - 'neighbors': - self.neighbors, - 'count': - self.count, - 'value_buffer': - np.full([self.num_nodes], -1, dtype="int32"), - 'index_buffer': - np.full([self.num_nodes], -1, dtype="int32") - }, - fetch_list=[ - reindex_src_1, reindex_dst_1, out_nodes_1, - reindex_src_2, reindex_dst_2, out_nodes_2 - ]) - reindex_src_1, reindex_dst_1, out_nodes_1, reindex_src_2, \ - reindex_dst_2, out_nodes_2 = ret - np.testing.assert_allclose(self.reindex_src, - reindex_src_1, - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst_1, - rtol=1e-05) + ret = exe.run( + feed={ + 'x': self.x, + 'neighbors': self.neighbors, + 'count': self.count, + 'value_buffer': np.full( + [self.num_nodes], -1, dtype="int32" + ), + 'index_buffer': np.full( + [self.num_nodes], -1, dtype="int32" + ), + }, + fetch_list=[ + reindex_src_1, + reindex_dst_1, + out_nodes_1, + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ], + ) + ( + reindex_src_1, + reindex_dst_1, + out_nodes_1, + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ) = ret + np.testing.assert_allclose( + self.reindex_src, reindex_src_1, rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst_1, rtol=1e-05 + ) np.testing.assert_allclose(self.out_nodes, out_nodes_1, rtol=1e-05) - np.testing.assert_allclose(self.reindex_src, - reindex_src_2, - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst_2, - rtol=1e-05) + np.testing.assert_allclose( + self.reindex_src, reindex_src_2, rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst_2, rtol=1e-05 + ) np.testing.assert_allclose(self.out_nodes, out_nodes_2, rtol=1e-05) class TestGeometricGraphReindex(unittest.TestCase): - def setUp(self): self.x = np.arange(5).astype("int64") self.neighbors = np.random.randint(100, size=20).astype("int64") @@ -195,7 +225,8 @@ def setUp(self): self.out_nodes = np.array(out_nodes, dtype="int64") reindex_dict = {node: ind for ind, node in enumerate(self.out_nodes)} self.reindex_src = np.array( - [reindex_dict[node] for node in self.neighbors]) + [reindex_dict[node] for node in self.neighbors] + ) reindex_dst = [] for node, c in zip(self.x, self.count): for i in range(c): @@ -211,15 +242,16 @@ def test_reindex_result(self): value_buffer = paddle.full([self.num_nodes], -1, dtype="int32") index_buffer = paddle.full([self.num_nodes], -1, dtype="int32") - reindex_src, reindex_dst, out_nodes = \ - paddle.geometric.reindex_graph(x, neighbors, count) + reindex_src, reindex_dst, out_nodes = paddle.geometric.reindex_graph( + x, neighbors, count + ) np.testing.assert_allclose(self.reindex_src, reindex_src, rtol=1e-05) np.testing.assert_allclose(self.reindex_dst, reindex_dst, rtol=1e-05) np.testing.assert_allclose(self.out_nodes, out_nodes, rtol=1e-05) - reindex_src, reindex_dst, out_nodes = \ - paddle.geometric.reindex_graph(x, neighbors, count, - value_buffer, index_buffer) + reindex_src, reindex_dst, out_nodes = paddle.geometric.reindex_graph( + x, neighbors, count, value_buffer, index_buffer + ) np.testing.assert_allclose(self.reindex_src, reindex_src, rtol=1e-05) np.testing.assert_allclose(self.reindex_dst, reindex_dst, rtol=1e-05) np.testing.assert_allclose(self.out_nodes, out_nodes, rtol=1e-05) @@ -232,20 +264,21 @@ def test_heter_reindex_result(self): count = paddle.to_tensor(self.count) count = paddle.concat([count, count]) - reindex_src, reindex_dst, out_nodes = \ - paddle.geometric.reindex_graph(x, neighbors, count) - np.testing.assert_allclose(self.reindex_src, - reindex_src[:self.neighbors.shape[0]], - rtol=1e-05) - np.testing.assert_allclose(self.reindex_src, - reindex_src[self.neighbors.shape[0]:], - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst[:self.neighbors.shape[0]], - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst[self.neighbors.shape[0]:], - rtol=1e-05) + reindex_src, reindex_dst, out_nodes = paddle.geometric.reindex_graph( + x, neighbors, count + ) + np.testing.assert_allclose( + self.reindex_src, reindex_src[: self.neighbors.shape[0]], rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_src, reindex_src[self.neighbors.shape[0] :], rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst[: self.neighbors.shape[0]], rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst[self.neighbors.shape[0] :], rtol=1e-05 + ) np.testing.assert_allclose(self.out_nodes, out_nodes, rtol=1e-05) def test_heter_reindex_result_v2(self): @@ -273,10 +306,11 @@ def test_heter_reindex_result_v2(self): reindex_dst.append(reindex_dict[node]) reindex_dst = np.array(reindex_dst, dtype="int64") - reindex_src_, reindex_dst_, out_nodes_ = \ - paddle.geometric.reindex_graph(paddle.to_tensor(x), - paddle.to_tensor(neighbors), - paddle.to_tensor(counts)) + reindex_src_, reindex_dst_, out_nodes_ = paddle.geometric.reindex_graph( + paddle.to_tensor(x), + paddle.to_tensor(neighbors), + paddle.to_tensor(counts), + ) np.testing.assert_allclose(reindex_src, reindex_src_, rtol=1e-05) np.testing.assert_allclose(reindex_dst, reindex_dst_, rtol=1e-05) np.testing.assert_allclose(out_nodes, out_nodes_, rtol=1e-05) @@ -308,9 +342,13 @@ def test_heter_reindex_result_v3(self): neighbors = [paddle.to_tensor(neighbors1), paddle.to_tensor(neighbors2)] count = [paddle.to_tensor(count1), paddle.to_tensor(count2)] - reindex_src_, reindex_dst_, out_nodes_ = \ - paddle.geometric.reindex_heter_graph(paddle.to_tensor(x), - neighbors, count) + ( + reindex_src_, + reindex_dst_, + out_nodes_, + ) = paddle.geometric.reindex_heter_graph( + paddle.to_tensor(x), neighbors, count + ) np.testing.assert_allclose(reindex_src, reindex_src_, rtol=1e-05) np.testing.assert_allclose(reindex_dst, reindex_dst_, rtol=1e-05) np.testing.assert_allclose(out_nodes, out_nodes_, rtol=1e-05) @@ -318,60 +356,80 @@ def test_heter_reindex_result_v3(self): def test_reindex_result_static(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - x = paddle.static.data(name="x", - shape=self.x.shape, - dtype=self.x.dtype) - neighbors = paddle.static.data(name="neighbors", - shape=self.neighbors.shape, - dtype=self.neighbors.dtype) - count = paddle.static.data(name="count", - shape=self.count.shape, - dtype=self.count.dtype) - value_buffer = paddle.static.data(name="value_buffer", - shape=[self.num_nodes], - dtype="int32") - index_buffer = paddle.static.data(name="index_buffer", - shape=[self.num_nodes], - dtype="int32") - - reindex_src_1, reindex_dst_1, out_nodes_1 = \ - paddle.geometric.reindex_graph(x, neighbors, count) - reindex_src_2, reindex_dst_2, out_nodes_2 = \ - paddle.geometric.reindex_graph(x, neighbors, count, - value_buffer, index_buffer) + x = paddle.static.data( + name="x", shape=self.x.shape, dtype=self.x.dtype + ) + neighbors = paddle.static.data( + name="neighbors", + shape=self.neighbors.shape, + dtype=self.neighbors.dtype, + ) + count = paddle.static.data( + name="count", shape=self.count.shape, dtype=self.count.dtype + ) + value_buffer = paddle.static.data( + name="value_buffer", shape=[self.num_nodes], dtype="int32" + ) + index_buffer = paddle.static.data( + name="index_buffer", shape=[self.num_nodes], dtype="int32" + ) + + ( + reindex_src_1, + reindex_dst_1, + out_nodes_1, + ) = paddle.geometric.reindex_graph(x, neighbors, count) + ( + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ) = paddle.geometric.reindex_graph( + x, neighbors, count, value_buffer, index_buffer + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'x': - self.x, - 'neighbors': - self.neighbors, - 'count': - self.count, - 'value_buffer': - np.full([self.num_nodes], -1, dtype="int32"), - 'index_buffer': - np.full([self.num_nodes], -1, dtype="int32") - }, - fetch_list=[ - reindex_src_1, reindex_dst_1, out_nodes_1, - reindex_src_2, reindex_dst_2, out_nodes_2 - ]) - reindex_src_1, reindex_dst_1, out_nodes_1, reindex_src_2, \ - reindex_dst_2, out_nodes_2 = ret - np.testing.assert_allclose(self.reindex_src, - reindex_src_1, - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst_1, - rtol=1e-05) + ret = exe.run( + feed={ + 'x': self.x, + 'neighbors': self.neighbors, + 'count': self.count, + 'value_buffer': np.full( + [self.num_nodes], -1, dtype="int32" + ), + 'index_buffer': np.full( + [self.num_nodes], -1, dtype="int32" + ), + }, + fetch_list=[ + reindex_src_1, + reindex_dst_1, + out_nodes_1, + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ], + ) + ( + reindex_src_1, + reindex_dst_1, + out_nodes_1, + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ) = ret + np.testing.assert_allclose( + self.reindex_src, reindex_src_1, rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst_1, rtol=1e-05 + ) np.testing.assert_allclose(self.out_nodes, out_nodes_1, rtol=1e-05) - np.testing.assert_allclose(self.reindex_src, - reindex_src_2, - rtol=1e-05) - np.testing.assert_allclose(self.reindex_dst, - reindex_dst_2, - rtol=1e-05) + np.testing.assert_allclose( + self.reindex_src, reindex_src_2, rtol=1e-05 + ) + np.testing.assert_allclose( + self.reindex_dst, reindex_dst_2, rtol=1e-05 + ) np.testing.assert_allclose(self.out_nodes, out_nodes_2, rtol=1e-05) def test_heter_reindex_result_static(self): @@ -401,48 +459,69 @@ def test_heter_reindex_result_static(self): with paddle.static.program_guard(paddle.static.Program()): x = paddle.static.data(name="x", shape=[5], dtype="int64") - neighbors1 = paddle.static.data(name="neighbors1", - shape=[20], - dtype="int64") + neighbors1 = paddle.static.data( + name="neighbors1", shape=[20], dtype="int64" + ) count1 = paddle.static.data(name="count1", shape=[5], dtype="int32") - neighbors2 = paddle.static.data(name="neighbors2", - shape=[20], - dtype="int64") + neighbors2 = paddle.static.data( + name="neighbors2", shape=[20], dtype="int64" + ) count2 = paddle.static.data(name="count2", shape=[5], dtype="int32") - value_buffer = paddle.static.data(name="value_buffer", - shape=[5], - dtype="int32") - index_buffer = paddle.static.data(name="index_buffer", - shape=[5], - dtype="int32") - - reindex_src_1, reindex_dst_1, out_nodes_1 = \ - paddle.geometric.reindex_heter_graph(x, - [neighbors1, neighbors2], - [count1, count2]) - reindex_src_2, reindex_dst_2, out_nodes_2 = \ - paddle.geometric.reindex_heter_graph(x, - [neighbors1, neighbors2], - [count1, count2], - value_buffer, index_buffer) + value_buffer = paddle.static.data( + name="value_buffer", shape=[5], dtype="int32" + ) + index_buffer = paddle.static.data( + name="index_buffer", shape=[5], dtype="int32" + ) + + ( + reindex_src_1, + reindex_dst_1, + out_nodes_1, + ) = paddle.geometric.reindex_heter_graph( + x, [neighbors1, neighbors2], [count1, count2] + ) + ( + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ) = paddle.geometric.reindex_heter_graph( + x, + [neighbors1, neighbors2], + [count1, count2], + value_buffer, + index_buffer, + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'x': np_x, - 'neighbors1': np_neighbors1, - 'count1': np_count1, - 'neighbors2': np_neighbors2, - 'count2': np_count2, - 'value_buffer': np.full([5], -1, dtype="int32"), - 'index_buffer': np.full([5], -1, dtype="int32") - }, - fetch_list=[ - reindex_src_1, reindex_dst_1, out_nodes_1, - reindex_src_2, reindex_dst_2, out_nodes_2 - ]) - - reindex_src_1, reindex_dst_1, out_nodes_1, reindex_src_2, \ - reindex_dst_2, out_nodes_2 = ret + ret = exe.run( + feed={ + 'x': np_x, + 'neighbors1': np_neighbors1, + 'count1': np_count1, + 'neighbors2': np_neighbors2, + 'count2': np_count2, + 'value_buffer': np.full([5], -1, dtype="int32"), + 'index_buffer': np.full([5], -1, dtype="int32"), + }, + fetch_list=[ + reindex_src_1, + reindex_dst_1, + out_nodes_1, + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ], + ) + + ( + reindex_src_1, + reindex_dst_1, + out_nodes_1, + reindex_src_2, + reindex_dst_2, + out_nodes_2, + ) = ret np.testing.assert_allclose(reindex_src, reindex_src_1, rtol=1e-05) np.testing.assert_allclose(reindex_dst, reindex_dst_1, rtol=1e-05) np.testing.assert_allclose(out_nodes, out_nodes_1, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_graph_sample_neighbors.py b/python/paddle/fluid/tests/unittests/test_graph_sample_neighbors.py index 557d41993b878..20366b80e2ad7 100644 --- a/python/paddle/fluid/tests/unittests/test_graph_sample_neighbors.py +++ b/python/paddle/fluid/tests/unittests/test_graph_sample_neighbors.py @@ -19,7 +19,6 @@ class TestGraphSampleNeighbors(unittest.TestCase): - def setUp(self): num_nodes = 20 edges = np.random.randint(num_nodes, size=(100, 2)) @@ -40,8 +39,9 @@ def setUp(self): self.row = sorted_edges[:, 0].astype("int64") self.colptr = colptr.astype("int64") - self.nodes = np.unique(np.random.randint(num_nodes, - size=5)).astype("int64") + self.nodes = np.unique(np.random.randint(num_nodes, size=5)).astype( + "int64" + ) self.sample_size = 5 self.dst_src_dict = dst_src_dict @@ -52,24 +52,29 @@ def test_sample_result(self): nodes = paddle.to_tensor(self.nodes) out_neighbors, out_count = paddle.incubate.graph_sample_neighbors( - row, colptr, nodes, sample_size=self.sample_size) + row, colptr, nodes, sample_size=self.sample_size + ) out_count_cumsum = paddle.cumsum(out_count) for i in range(len(out_count)): if i == 0: - neighbors = out_neighbors[0:out_count_cumsum[i]] + neighbors = out_neighbors[0 : out_count_cumsum[i]] else: neighbors = out_neighbors[ - out_count_cumsum[i - 1]:out_count_cumsum[i]] + out_count_cumsum[i - 1] : out_count_cumsum[i] + ] # Ensure the correct sample size. self.assertTrue( out_count[i] == self.sample_size - or out_count[i] == len(self.dst_src_dict[self.nodes[i]])) + or out_count[i] == len(self.dst_src_dict[self.nodes[i]]) + ) # Ensure no repetitive sample neighbors. self.assertTrue( - neighbors.shape[0] == paddle.unique(neighbors).shape[0]) + neighbors.shape[0] == paddle.unique(neighbors).shape[0] + ) # Ensure the correct sample neighbors. - in_neighbors = np.isin(neighbors.numpy(), - self.dst_src_dict[self.nodes[i]]) + in_neighbors = np.isin( + neighbors.numpy(), self.dst_src_dict[self.nodes[i]] + ) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) def test_sample_result_fisher_yates_sampling(self): @@ -86,57 +91,69 @@ def test_sample_result_fisher_yates_sampling(self): nodes, perm_buffer=perm_buffer, sample_size=self.sample_size, - flag_perm_buffer=True) + flag_perm_buffer=True, + ) out_count_cumsum = paddle.cumsum(out_count) for i in range(len(out_count)): if i == 0: - neighbors = out_neighbors[0:out_count_cumsum[i]] + neighbors = out_neighbors[0 : out_count_cumsum[i]] else: neighbors = out_neighbors[ - out_count_cumsum[i - 1]:out_count_cumsum[i]] + out_count_cumsum[i - 1] : out_count_cumsum[i] + ] # Ensure the correct sample size. self.assertTrue( out_count[i] == self.sample_size - or out_count[i] == len(self.dst_src_dict[self.nodes[i]])) + or out_count[i] == len(self.dst_src_dict[self.nodes[i]]) + ) # Ensure no repetitive sample neighbors. self.assertTrue( - neighbors.shape[0] == paddle.unique(neighbors).shape[0]) + neighbors.shape[0] == paddle.unique(neighbors).shape[0] + ) # Ensure the correct sample neighbors. - in_neighbors = np.isin(neighbors.numpy(), - self.dst_src_dict[self.nodes[i]]) + in_neighbors = np.isin( + neighbors.numpy(), self.dst_src_dict[self.nodes[i]] + ) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) def test_sample_result_static(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - row = paddle.static.data(name="row", - shape=self.row.shape, - dtype=self.row.dtype) - colptr = paddle.static.data(name="colptr", - shape=self.colptr.shape, - dtype=self.colptr.dtype) - nodes = paddle.static.data(name="nodes", - shape=self.nodes.shape, - dtype=self.nodes.dtype) + row = paddle.static.data( + name="row", shape=self.row.shape, dtype=self.row.dtype + ) + colptr = paddle.static.data( + name="colptr", shape=self.colptr.shape, dtype=self.colptr.dtype + ) + nodes = paddle.static.data( + name="nodes", shape=self.nodes.shape, dtype=self.nodes.dtype + ) out_neighbors, out_count = paddle.incubate.graph_sample_neighbors( - row, colptr, nodes, sample_size=self.sample_size) + row, colptr, nodes, sample_size=self.sample_size + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'row': self.row, - 'colptr': self.colptr, - 'nodes': self.nodes - }, - fetch_list=[out_neighbors, out_count]) + ret = exe.run( + feed={ + 'row': self.row, + 'colptr': self.colptr, + 'nodes': self.nodes, + }, + fetch_list=[out_neighbors, out_count], + ) out_neighbors, out_count = ret out_count_cumsum = np.cumsum(out_count) out_neighbors = np.split(out_neighbors, out_count_cumsum)[:-1] - for neighbors, node, count in zip(out_neighbors, self.nodes, - out_count): - self.assertTrue(count == self.sample_size - or count == len(self.dst_src_dict[node])) + for neighbors, node, count in zip( + out_neighbors, self.nodes, out_count + ): + self.assertTrue( + count == self.sample_size + or count == len(self.dst_src_dict[node]) + ) self.assertTrue( - neighbors.shape[0] == np.unique(neighbors).shape[0]) + neighbors.shape[0] == np.unique(neighbors).shape[0] + ) in_neighbors = np.isin(neighbors, self.dst_src_dict[node]) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) @@ -147,18 +164,22 @@ def test_raise_errors(self): nodes = paddle.to_tensor(self.nodes) def check_eid_error(): - paddle.incubate.graph_sample_neighbors(row, - colptr, - nodes, - sample_size=self.sample_size, - return_eids=True) + paddle.incubate.graph_sample_neighbors( + row, + colptr, + nodes, + sample_size=self.sample_size, + return_eids=True, + ) def check_perm_buffer_error(): - paddle.incubate.graph_sample_neighbors(row, - colptr, - nodes, - sample_size=self.sample_size, - flag_perm_buffer=True) + paddle.incubate.graph_sample_neighbors( + row, + colptr, + nodes, + sample_size=self.sample_size, + flag_perm_buffer=True, + ) self.assertRaises(ValueError, check_eid_error) self.assertRaises(ValueError, check_perm_buffer_error) @@ -171,15 +192,24 @@ def test_sample_result_with_eids(self): eids = paddle.to_tensor(self.edges_id) perm_buffer = paddle.to_tensor(self.edges_id) - out_neighbors, out_count, out_eids = paddle.incubate.graph_sample_neighbors( + ( + out_neighbors, + out_count, + out_eids, + ) = paddle.incubate.graph_sample_neighbors( row, colptr, nodes, eids=eids, sample_size=self.sample_size, - return_eids=True) + return_eids=True, + ) - out_neighbors, out_count, out_eids = paddle.incubate.graph_sample_neighbors( + ( + out_neighbors, + out_count, + out_eids, + ) = paddle.incubate.graph_sample_neighbors( row, colptr, nodes, @@ -187,42 +217,49 @@ def test_sample_result_with_eids(self): perm_buffer=perm_buffer, sample_size=self.sample_size, return_eids=True, - flag_perm_buffer=True) + flag_perm_buffer=True, + ) paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - row = paddle.static.data(name="row", - shape=self.row.shape, - dtype=self.row.dtype) - colptr = paddle.static.data(name="colptr", - shape=self.colptr.shape, - dtype=self.colptr.dtype) - nodes = paddle.static.data(name="nodes", - shape=self.nodes.shape, - dtype=self.nodes.dtype) - eids = paddle.static.data(name="eids", - shape=self.edges_id.shape, - dtype=self.nodes.dtype) - - out_neighbors, out_count, out_eids = paddle.incubate.graph_sample_neighbors( + row = paddle.static.data( + name="row", shape=self.row.shape, dtype=self.row.dtype + ) + colptr = paddle.static.data( + name="colptr", shape=self.colptr.shape, dtype=self.colptr.dtype + ) + nodes = paddle.static.data( + name="nodes", shape=self.nodes.shape, dtype=self.nodes.dtype + ) + eids = paddle.static.data( + name="eids", shape=self.edges_id.shape, dtype=self.nodes.dtype + ) + + ( + out_neighbors, + out_count, + out_eids, + ) = paddle.incubate.graph_sample_neighbors( row, colptr, nodes, eids, sample_size=self.sample_size, - return_eids=True) + return_eids=True, + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'row': self.row, - 'colptr': self.colptr, - 'nodes': self.nodes, - 'eids': self.edges_id - }, - fetch_list=[out_neighbors, out_count, out_eids]) + ret = exe.run( + feed={ + 'row': self.row, + 'colptr': self.colptr, + 'nodes': self.nodes, + 'eids': self.edges_id, + }, + fetch_list=[out_neighbors, out_count, out_eids], + ) class TestGeometricGraphSampleNeighbors(unittest.TestCase): - def setUp(self): num_nodes = 20 edges = np.random.randint(num_nodes, size=(100, 2)) @@ -243,8 +280,9 @@ def setUp(self): self.row = sorted_edges[:, 0].astype("int64") self.colptr = colptr.astype("int64") - self.nodes = np.unique(np.random.randint(num_nodes, - size=5)).astype("int64") + self.nodes = np.unique(np.random.randint(num_nodes, size=5)).astype( + "int64" + ) self.sample_size = 5 self.dst_src_dict = dst_src_dict @@ -255,24 +293,29 @@ def test_sample_result(self): nodes = paddle.to_tensor(self.nodes) out_neighbors, out_count = paddle.geometric.sample_neighbors( - row, colptr, nodes, sample_size=self.sample_size) + row, colptr, nodes, sample_size=self.sample_size + ) out_count_cumsum = paddle.cumsum(out_count) for i in range(len(out_count)): if i == 0: - neighbors = out_neighbors[0:out_count_cumsum[i]] + neighbors = out_neighbors[0 : out_count_cumsum[i]] else: neighbors = out_neighbors[ - out_count_cumsum[i - 1]:out_count_cumsum[i]] + out_count_cumsum[i - 1] : out_count_cumsum[i] + ] # Ensure the correct sample size. self.assertTrue( out_count[i] == self.sample_size - or out_count[i] == len(self.dst_src_dict[self.nodes[i]])) + or out_count[i] == len(self.dst_src_dict[self.nodes[i]]) + ) # Ensure no repetitive sample neighbors. self.assertTrue( - neighbors.shape[0] == paddle.unique(neighbors).shape[0]) + neighbors.shape[0] == paddle.unique(neighbors).shape[0] + ) # Ensure the correct sample neighbors. - in_neighbors = np.isin(neighbors.numpy(), - self.dst_src_dict[self.nodes[i]]) + in_neighbors = np.isin( + neighbors.numpy(), self.dst_src_dict[self.nodes[i]] + ) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) def test_sample_result_fisher_yates_sampling(self): @@ -288,57 +331,69 @@ def test_sample_result_fisher_yates_sampling(self): colptr, nodes, perm_buffer=perm_buffer, - sample_size=self.sample_size) + sample_size=self.sample_size, + ) out_count_cumsum = paddle.cumsum(out_count) for i in range(len(out_count)): if i == 0: - neighbors = out_neighbors[0:out_count_cumsum[i]] + neighbors = out_neighbors[0 : out_count_cumsum[i]] else: neighbors = out_neighbors[ - out_count_cumsum[i - 1]:out_count_cumsum[i]] + out_count_cumsum[i - 1] : out_count_cumsum[i] + ] # Ensure the correct sample size. self.assertTrue( out_count[i] == self.sample_size - or out_count[i] == len(self.dst_src_dict[self.nodes[i]])) + or out_count[i] == len(self.dst_src_dict[self.nodes[i]]) + ) # Ensure no repetitive sample neighbors. self.assertTrue( - neighbors.shape[0] == paddle.unique(neighbors).shape[0]) + neighbors.shape[0] == paddle.unique(neighbors).shape[0] + ) # Ensure the correct sample neighbors. - in_neighbors = np.isin(neighbors.numpy(), - self.dst_src_dict[self.nodes[i]]) + in_neighbors = np.isin( + neighbors.numpy(), self.dst_src_dict[self.nodes[i]] + ) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) def test_sample_result_static(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - row = paddle.static.data(name="row", - shape=self.row.shape, - dtype=self.row.dtype) - colptr = paddle.static.data(name="colptr", - shape=self.colptr.shape, - dtype=self.colptr.dtype) - nodes = paddle.static.data(name="nodes", - shape=self.nodes.shape, - dtype=self.nodes.dtype) + row = paddle.static.data( + name="row", shape=self.row.shape, dtype=self.row.dtype + ) + colptr = paddle.static.data( + name="colptr", shape=self.colptr.shape, dtype=self.colptr.dtype + ) + nodes = paddle.static.data( + name="nodes", shape=self.nodes.shape, dtype=self.nodes.dtype + ) out_neighbors, out_count = paddle.geometric.sample_neighbors( - row, colptr, nodes, sample_size=self.sample_size) + row, colptr, nodes, sample_size=self.sample_size + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'row': self.row, - 'colptr': self.colptr, - 'nodes': self.nodes - }, - fetch_list=[out_neighbors, out_count]) + ret = exe.run( + feed={ + 'row': self.row, + 'colptr': self.colptr, + 'nodes': self.nodes, + }, + fetch_list=[out_neighbors, out_count], + ) out_neighbors, out_count = ret out_count_cumsum = np.cumsum(out_count) out_neighbors = np.split(out_neighbors, out_count_cumsum)[:-1] - for neighbors, node, count in zip(out_neighbors, self.nodes, - out_count): - self.assertTrue(count == self.sample_size - or count == len(self.dst_src_dict[node])) + for neighbors, node, count in zip( + out_neighbors, self.nodes, out_count + ): self.assertTrue( - neighbors.shape[0] == np.unique(neighbors).shape[0]) + count == self.sample_size + or count == len(self.dst_src_dict[node]) + ) + self.assertTrue( + neighbors.shape[0] == np.unique(neighbors).shape[0] + ) in_neighbors = np.isin(neighbors, self.dst_src_dict[node]) self.assertTrue(np.sum(in_neighbors) == in_neighbors.shape[0]) @@ -349,11 +404,13 @@ def test_raise_errors(self): nodes = paddle.to_tensor(self.nodes) def check_eid_error(): - paddle.geometric.sample_neighbors(row, - colptr, - nodes, - sample_size=self.sample_size, - return_eids=True) + paddle.geometric.sample_neighbors( + row, + colptr, + nodes, + sample_size=self.sample_size, + return_eids=True, + ) self.assertRaises(ValueError, check_eid_error) @@ -371,7 +428,8 @@ def test_sample_result_with_eids(self): nodes, eids=eids, sample_size=self.sample_size, - return_eids=True) + return_eids=True, + ) out_neighbors, out_count, out_eids = paddle.geometric.sample_neighbors( row, @@ -380,38 +438,46 @@ def test_sample_result_with_eids(self): eids=eids, perm_buffer=perm_buffer, sample_size=self.sample_size, - return_eids=True) + return_eids=True, + ) paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - row = paddle.static.data(name="row", - shape=self.row.shape, - dtype=self.row.dtype) - colptr = paddle.static.data(name="colptr", - shape=self.colptr.shape, - dtype=self.colptr.dtype) - nodes = paddle.static.data(name="nodes", - shape=self.nodes.shape, - dtype=self.nodes.dtype) - eids = paddle.static.data(name="eids", - shape=self.edges_id.shape, - dtype=self.nodes.dtype) - - out_neighbors, out_count, out_eids = paddle.geometric.sample_neighbors( + row = paddle.static.data( + name="row", shape=self.row.shape, dtype=self.row.dtype + ) + colptr = paddle.static.data( + name="colptr", shape=self.colptr.shape, dtype=self.colptr.dtype + ) + nodes = paddle.static.data( + name="nodes", shape=self.nodes.shape, dtype=self.nodes.dtype + ) + eids = paddle.static.data( + name="eids", shape=self.edges_id.shape, dtype=self.nodes.dtype + ) + + ( + out_neighbors, + out_count, + out_eids, + ) = paddle.geometric.sample_neighbors( row, colptr, nodes, sample_size=self.sample_size, eids=eids, - return_eids=True) + return_eids=True, + ) exe = paddle.static.Executor(paddle.CPUPlace()) - ret = exe.run(feed={ - 'row': self.row, - 'colptr': self.colptr, - 'nodes': self.nodes, - 'eids': self.edges_id - }, - fetch_list=[out_neighbors, out_count, out_eids]) + ret = exe.run( + feed={ + 'row': self.row, + 'colptr': self.colptr, + 'nodes': self.nodes, + 'eids': self.edges_id, + }, + fetch_list=[out_neighbors, out_count, out_eids], + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_graph_send_recv_op.py b/python/paddle/fluid/tests/unittests/test_graph_send_recv_op.py index 40d1673438fa7..a5344a96a319e 100644 --- a/python/paddle/fluid/tests/unittests/test_graph_send_recv_op.py +++ b/python/paddle/fluid/tests/unittests/test_graph_send_recv_op.py @@ -21,18 +21,15 @@ from op_test import OpTest -def graph_send_recv_wrapper(x, - src_index, - dst_index, - reduce_op="sum", - out_size=None, - name=None): - return paddle.geometric.send_u_recv(x, src_index, dst_index, - reduce_op.lower(), out_size, name) +def graph_send_recv_wrapper( + x, src_index, dst_index, reduce_op="sum", out_size=None, name=None +): + return paddle.geometric.send_u_recv( + x, src_index, dst_index, reduce_op.lower(), out_size, name + ) class TestGraphSendRecvMaxOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_recv_wrapper @@ -48,21 +45,20 @@ def setUp(self): self.attrs = {'reduce_op': 'MAX'} out, self.gradient = compute_graph_send_recv_for_min_max( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = {'Out': out} def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.gradient], - check_eager=True) + self.check_grad( + ['X'], 'Out', user_defined_grads=[self.gradient], check_eager=True + ) class TestGraphSendRecvMinOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_recv_wrapper @@ -78,7 +74,8 @@ def setUp(self): self.attrs = {'reduce_op': 'MIN'} out, self.gradient = compute_graph_send_recv_for_min_max( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = {'Out': out} @@ -86,14 +83,12 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.gradient], - check_eager=True) + self.check_grad( + ['X'], 'Out', user_defined_grads=[self.gradient], check_eager=True + ) class TestGraphSendRecvSumOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_recv_wrapper @@ -120,7 +115,6 @@ def test_check_grad(self): class TestGraphSendRecvMeanOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_recv_wrapper @@ -136,7 +130,8 @@ def setUp(self): self.attrs = {'reduce_op': 'MEAN'} out, dst_count = compute_graph_send_recv_for_sum_mean( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = {'Out': out, 'Dst_count': dst_count} @@ -197,8 +192,9 @@ def compute_graph_send_recv_for_min_max(inputs, attributes): results[s_id, :] += gather_x[index, :] first_set.add(s_id) else: - results[s_id, :] = np.maximum(results[s_id, :], - gather_x[index, :]) + results[s_id, :] = np.maximum( + results[s_id, :], gather_x[index, :] + ) elif reduce_op == "MIN": first_set = set() for index, s_id in enumerate(dst_index): @@ -206,8 +202,9 @@ def compute_graph_send_recv_for_min_max(inputs, attributes): results[s_id, :] += gather_x[index, :] first_set.add(s_id) else: - results[s_id, :] = np.minimum(results[s_id, :], - gather_x[index, :]) + results[s_id, :] = np.minimum( + results[s_id, :], gather_x[index, :] + ) else: raise ValueError("Invalid reduce_op, only MAX, MIN supported!") @@ -216,14 +213,14 @@ def compute_graph_send_recv_for_min_max(inputs, attributes): for i in range(index_size): forward_src_idx = src_index[i] forward_dst_idx = dst_index[i] - gradient[forward_src_idx] += 1 * (x[forward_src_idx] - == results[forward_dst_idx]) + gradient[forward_src_idx] += 1 * ( + x[forward_src_idx] == results[forward_dst_idx] + ) return results, gradient / results.size class API_GraphSendRecvOpTest(unittest.TestCase): - def test_static(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): @@ -231,53 +228,64 @@ def test_static(self): src_index = paddle.static.data(name="src", shape=[4], dtype="int32") dst_index = paddle.static.data(name="dst", shape=[4], dtype="int32") - res_sum = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "sum") - res_mean = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "mean") - res_max = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "max") - res_min = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "min") + res_sum = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "sum" + ) + res_mean = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "mean" + ) + res_max = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "max" + ) + res_min = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "min" + ) exe = paddle.static.Executor(paddle.CPUPlace()) data1 = np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]], dtype='float32') data2 = np.array([0, 1, 2, 0], dtype="int32") data3 = np.array([1, 2, 1, 0], dtype="int32") - np_sum = np.array([[0, 2, 3], [2, 8, 10], [1, 4, 5]], - dtype="float32") - np_mean = np.array([[0, 2, 3], [1, 4, 5], [1, 4, 5]], - dtype="float32") - np_max = np.array([[0, 2, 3], [2, 6, 7], [1, 4, 5]], - dtype="float32") - np_min = np.array([[0, 2, 3], [0, 2, 3], [1, 4, 5]], - dtype="float32") - - ret = exe.run(feed={ - 'x': data1, - 'src': data2, - 'dst': data3 - }, - fetch_list=[res_sum, res_mean, res_max, res_min]) + np_sum = np.array( + [[0, 2, 3], [2, 8, 10], [1, 4, 5]], dtype="float32" + ) + np_mean = np.array( + [[0, 2, 3], [1, 4, 5], [1, 4, 5]], dtype="float32" + ) + np_max = np.array( + [[0, 2, 3], [2, 6, 7], [1, 4, 5]], dtype="float32" + ) + np_min = np.array( + [[0, 2, 3], [0, 2, 3], [1, 4, 5]], dtype="float32" + ) + + ret = exe.run( + feed={'x': data1, 'src': data2, 'dst': data3}, + fetch_list=[res_sum, res_mean, res_max, res_min], + ) for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): np.testing.assert_allclose(np_res, ret_res, rtol=1e-05, atol=1e-06) def test_dygraph(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float32" + ) src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") - res_sum = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "sum") - res_mean = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "mean") - res_max = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "max") - res_min = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "min") + res_sum = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "sum" + ) + res_mean = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "mean" + ) + res_max = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "max" + ) + res_min = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "min" + ) np_sum = np.array([[0, 2, 3], [2, 8, 10], [1, 4, 5]], dtype="float32") np_mean = np.array([[0, 2, 3], [1, 4, 5], [1, 4, 5]], dtype="float32") @@ -291,18 +299,23 @@ def test_dygraph(self): def test_int32_input(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), - dtype="int32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), dtype="int32" + ) src_index = paddle.to_tensor(np.array([0, 1, 2, 0, 1]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0, 1]), dtype="int32") - res_sum = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "sum") - res_mean = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "mean") - res_max = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "max") - res_min = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "min") + res_sum = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "sum" + ) + res_mean = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "mean" + ) + res_max = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "max" + ) + res_min = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "min" + ) np_sum = np.array([[0, 2, 3], [3, 12, 14], [1, 4, 5]], dtype="int32") np_mean = np.array([[0, 2, 3], [1, 4, 4], [1, 4, 5]], dtype="int32") @@ -316,23 +329,24 @@ def test_int32_input(self): def test_set_outsize_gpu(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), dtype="float32" + ) src_index = paddle.to_tensor(np.array([0, 0, 1]), dtype="int32") dst_index = paddle.to_tensor(np.array([0, 1, 1]), dtype="int32") res = paddle.incubate.graph_send_recv(x, src_index, dst_index, "sum") out_size = paddle.max(dst_index) + 1 res_set_outsize = paddle.incubate.graph_send_recv( - x, src_index, dst_index, "sum", out_size) + x, src_index, dst_index, "sum", out_size + ) np_res = np.array([[0, 2, 3], [1, 6, 8], [0, 0, 0]], dtype="float32") np_res_set_outsize = np.array([[0, 2, 3], [1, 6, 8]], dtype="float32") np.testing.assert_allclose(np_res, res, rtol=1e-05, atol=1e-06) - np.testing.assert_allclose(np_res_set_outsize, - res_set_outsize, - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np_res_set_outsize, res_set_outsize, rtol=1e-05, atol=1e-06 + ) def test_out_size_tensor_static(self): paddle.enable_static() @@ -340,12 +354,13 @@ def test_out_size_tensor_static(self): x = paddle.static.data(name="x", shape=[3, 3], dtype="float32") src_index = paddle.static.data(name="src", shape=[3], dtype="int32") dst_index = paddle.static.data(name="dst", shape=[3], dtype="int32") - out_size = paddle.static.data(name="out_size", - shape=[1], - dtype="int32") + out_size = paddle.static.data( + name="out_size", shape=[1], dtype="int32" + ) - res_sum = paddle.incubate.graph_send_recv(x, src_index, dst_index, - "sum", out_size) + res_sum = paddle.incubate.graph_send_recv( + x, src_index, dst_index, "sum", out_size + ) exe = paddle.static.Executor(paddle.CPUPlace()) data1 = np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]], dtype='float32') @@ -355,13 +370,15 @@ def test_out_size_tensor_static(self): np_sum = np.array([[0, 2, 3], [1, 6, 8]], dtype="float32") - ret = exe.run(feed={ - 'x': data1, - 'src': data2, - 'dst': data3, - 'out_size': data4, - }, - fetch_list=[res_sum]) + ret = exe.run( + feed={ + 'x': data1, + 'src': data2, + 'dst': data3, + 'out_size': data4, + }, + fetch_list=[res_sum], + ) np.testing.assert_allclose(np_sum, ret[0], rtol=1e-05, atol=1e-06) def test_api_eager_dygraph(self): @@ -372,7 +389,6 @@ def test_api_eager_dygraph(self): class API_GeometricSendURecvTest(unittest.TestCase): - def test_static(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): @@ -380,43 +396,50 @@ def test_static(self): src_index = paddle.static.data(name="src", shape=[4], dtype="int32") dst_index = paddle.static.data(name="dst", shape=[4], dtype="int32") - res_sum = paddle.geometric.send_u_recv(x, src_index, dst_index, - "sum") - res_mean = paddle.geometric.send_u_recv(x, src_index, dst_index, - "mean") - res_max = paddle.geometric.send_u_recv(x, src_index, dst_index, - "max") - res_min = paddle.geometric.send_u_recv(x, src_index, dst_index, - "min") + res_sum = paddle.geometric.send_u_recv( + x, src_index, dst_index, "sum" + ) + res_mean = paddle.geometric.send_u_recv( + x, src_index, dst_index, "mean" + ) + res_max = paddle.geometric.send_u_recv( + x, src_index, dst_index, "max" + ) + res_min = paddle.geometric.send_u_recv( + x, src_index, dst_index, "min" + ) exe = paddle.static.Executor(paddle.CPUPlace()) data1 = np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]], dtype='float32') data2 = np.array([0, 1, 2, 0], dtype="int32") data3 = np.array([1, 2, 1, 0], dtype="int32") - np_sum = np.array([[0, 2, 3], [2, 8, 10], [1, 4, 5]], - dtype="float32") - np_mean = np.array([[0, 2, 3], [1, 4, 5], [1, 4, 5]], - dtype="float32") - np_max = np.array([[0, 2, 3], [2, 6, 7], [1, 4, 5]], - dtype="float32") - np_min = np.array([[0, 2, 3], [0, 2, 3], [1, 4, 5]], - dtype="float32") - - ret = exe.run(feed={ - 'x': data1, - 'src': data2, - 'dst': data3 - }, - fetch_list=[res_sum, res_mean, res_max, res_min]) + np_sum = np.array( + [[0, 2, 3], [2, 8, 10], [1, 4, 5]], dtype="float32" + ) + np_mean = np.array( + [[0, 2, 3], [1, 4, 5], [1, 4, 5]], dtype="float32" + ) + np_max = np.array( + [[0, 2, 3], [2, 6, 7], [1, 4, 5]], dtype="float32" + ) + np_min = np.array( + [[0, 2, 3], [0, 2, 3], [1, 4, 5]], dtype="float32" + ) + + ret = exe.run( + feed={'x': data1, 'src': data2, 'dst': data3}, + fetch_list=[res_sum, res_mean, res_max, res_min], + ) for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): np.testing.assert_allclose(np_res, ret_res, rtol=1e-05, atol=1e-06) def test_dygraph(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float32" + ) src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") res_sum = paddle.geometric.send_u_recv(x, src_index, dst_index, "sum") @@ -436,8 +459,9 @@ def test_dygraph(self): def test_int32_input(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), - dtype="int32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), dtype="int32" + ) src_index = paddle.to_tensor(np.array([0, 1, 2, 0, 1]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0, 1]), dtype="int32") res_sum = paddle.geometric.send_u_recv(x, src_index, dst_index, "sum") @@ -457,23 +481,24 @@ def test_int32_input(self): def test_set_outsize_gpu(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]]), dtype="float32" + ) src_index = paddle.to_tensor(np.array([0, 0, 1]), dtype="int32") dst_index = paddle.to_tensor(np.array([0, 1, 1]), dtype="int32") res = paddle.geometric.send_u_recv(x, src_index, dst_index, "sum") out_size = paddle.max(dst_index) + 1 - res_set_outsize = paddle.geometric.send_u_recv(x, src_index, dst_index, - "sum", out_size) + res_set_outsize = paddle.geometric.send_u_recv( + x, src_index, dst_index, "sum", out_size + ) np_res = np.array([[0, 2, 3], [1, 6, 8], [0, 0, 0]], dtype="float32") np_res_set_outsize = np.array([[0, 2, 3], [1, 6, 8]], dtype="float32") np.testing.assert_allclose(np_res, res, rtol=1e-05, atol=1e-06) - np.testing.assert_allclose(np_res_set_outsize, - res_set_outsize, - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np_res_set_outsize, res_set_outsize, rtol=1e-05, atol=1e-06 + ) def test_out_size_tensor_static(self): paddle.enable_static() @@ -481,12 +506,13 @@ def test_out_size_tensor_static(self): x = paddle.static.data(name="x", shape=[3, 3], dtype="float32") src_index = paddle.static.data(name="src", shape=[3], dtype="int32") dst_index = paddle.static.data(name="dst", shape=[3], dtype="int32") - out_size = paddle.static.data(name="out_size", - shape=[1], - dtype="int32") + out_size = paddle.static.data( + name="out_size", shape=[1], dtype="int32" + ) - res_sum = paddle.geometric.send_u_recv(x, src_index, dst_index, - "sum", out_size) + res_sum = paddle.geometric.send_u_recv( + x, src_index, dst_index, "sum", out_size + ) exe = paddle.static.Executor(paddle.CPUPlace()) data1 = np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]], dtype='float32') @@ -496,13 +522,15 @@ def test_out_size_tensor_static(self): np_sum = np.array([[0, 2, 3], [1, 6, 8]], dtype="float32") - ret = exe.run(feed={ - 'x': data1, - 'src': data2, - 'dst': data3, - 'out_size': data4, - }, - fetch_list=[res_sum]) + ret = exe.run( + feed={ + 'x': data1, + 'src': data2, + 'dst': data3, + 'out_size': data4, + }, + fetch_list=[res_sum], + ) np.testing.assert_allclose(np_sum, ret[0], rtol=1e-05, atol=1e-06) def test_api_eager_dygraph(self): diff --git a/python/paddle/fluid/tests/unittests/test_graph_send_ue_recv_op.py b/python/paddle/fluid/tests/unittests/test_graph_send_ue_recv_op.py index bf74885ba5a49..c3bb651a3de39 100644 --- a/python/paddle/fluid/tests/unittests/test_graph_send_ue_recv_op.py +++ b/python/paddle/fluid/tests/unittests/test_graph_send_ue_recv_op.py @@ -40,7 +40,6 @@ def get_broadcast_shape(shp1, shp2): class BroadCastInfo(object): - def __init__(self, x_shape, y_shape): self.x_shape = x_shape self.y_shape = y_shape @@ -71,16 +70,24 @@ def calculate_bcastinfo(self): lhs_offset = [0] rhs_offset = [0] for j in range(0, max_ndim): - dl = 1 if (len(self.x_shape) - 1 - j) < 1 \ - else self.x_shape[len(self.x_shape) - 1 - j] - dr = 1 if (len(self.y_shape) - 1 - j) < 1 \ - else self.y_shape[len(self.y_shape) - 1 - j] + dl = ( + 1 + if (len(self.x_shape) - 1 - j) < 1 + else self.x_shape[len(self.x_shape) - 1 - j] + ) + dr = ( + 1 + if (len(self.y_shape) - 1 - j) < 1 + else self.y_shape[len(self.y_shape) - 1 - j] + ) for i in range(1, max(dl, dr)): for k in range(0, out_len): - lhs_offset.append(lhs_offset[k] + i * - (i < dl) * stride_l) - rhs_offset.append(rhs_offset[k] + i * - (i < dr) * stride_r) + lhs_offset.append( + lhs_offset[k] + i * (i < dl) * stride_l + ) + rhs_offset.append( + rhs_offset[k] + i * (i < dr) * stride_r + ) out_len *= max(dl, dr) stride_l *= dl @@ -176,16 +183,18 @@ def compute_graph_send_ue_recv_for_max_min(inputs, attributes): results[s_id, :] += x_compute_y[index, :] first_set.add(s_id) else: - results[s_id, :] = np.maximum(results[s_id, :], - x_compute_y[index, :]) + results[s_id, :] = np.maximum( + results[s_id, :], x_compute_y[index, :] + ) elif reduce_op == 'MIN': for index, s_id in enumerate(dst_index): if s_id not in first_set: results[s_id, :] += x_compute_y[index, :] first_set.add(s_id) else: - results[s_id, :] = np.minimum(results[s_id, :], - x_compute_y[index, :]) + results[s_id, :] = np.minimum( + results[s_id, :], x_compute_y[index, :] + ) else: raise ValueError("Invalid reduce_op, only MAX, MIN supported!") @@ -220,9 +229,11 @@ def compute_graph_send_ue_recv_for_max_min(inputs, attributes): out_add_1 = int(j % out_off.shape[1]) val = x_off[x_add_0][x_add_1] + y_off[y_add_0][y_add_1] x_grad_off[x_add_0][x_add_1] += 1 * ( - val == out_off[out_add_0][out_add_1]) + val == out_off[out_add_0][out_add_1] + ) y_grad_off[y_add_0][y_add_1] += 1 * ( - val == out_off[out_add_0][out_add_1]) + val == out_off[out_add_0][out_add_1] + ) elif message_op == 'MUL': if len(x_off.shape) == 1 and len(y_off.shape) == 1: val = x_off[x_add] * y_off[y_add] @@ -237,33 +248,45 @@ def compute_graph_send_ue_recv_for_max_min(inputs, attributes): out_add_0 = int(j / out_off.shape[1]) out_add_1 = int(j % out_off.shape[1]) val = x_off[x_add_0][x_add_1] * y_off[y_add_0][y_add_1] - x_grad_off[x_add_0][x_add_1] += 1 * ( - val == out_off[out_add_0][out_add_1] - ) * y_off[y_add_0][y_add_1] - y_grad_off[y_add_0][y_add_1] += 1 * ( - val == out_off[out_add_0][out_add_1] - ) * x_off[x_add_0][x_add_1] + x_grad_off[x_add_0][x_add_1] += ( + 1 + * (val == out_off[out_add_0][out_add_1]) + * y_off[y_add_0][y_add_1] + ) + y_grad_off[y_add_0][y_add_1] += ( + 1 + * (val == out_off[out_add_0][out_add_1]) + * x_off[x_add_0][x_add_1] + ) gradients = [x_gradient / results.size, y_gradient / results.size] return results, gradients -def graph_send_ue_recv_wrapper(x, - y, - src_index, - dst_index, - message_op="add", - reduce_op="sum", - out_size=None, - name=None): - return paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - message_op.lower(), reduce_op.lower(), - out_size, name) +def graph_send_ue_recv_wrapper( + x, + y, + src_index, + dst_index, + message_op="add", + reduce_op="sum", + out_size=None, + name=None, +): + return paddle.geometric.send_ue_recv( + x, + y, + src_index, + dst_index, + message_op.lower(), + reduce_op.lower(), + out_size, + name, + ) class TestGraphSendUERecvSumOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_ue_recv_wrapper @@ -274,7 +297,7 @@ def setUp(self): 'X': self.x, 'Y': self.y, 'Src_index': self.src_index, - 'Dst_index': self.dst_index + 'Dst_index': self.dst_index, } self.attrs = {'message_op': self.message_op, 'reduce_op': 'SUM'} @@ -298,7 +321,6 @@ def test_check_grad(self): class TestSumCase1(TestGraphSendUERecvSumOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -309,7 +331,6 @@ def set_config(self): class TestSumCase2(TestGraphSendUERecvSumOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -320,7 +341,6 @@ def set_config(self): class TestSumCase3(TestGraphSendUERecvSumOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -331,7 +351,6 @@ def set_config(self): class TestSumCase4(TestGraphSendUERecvSumOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -342,7 +361,6 @@ def set_config(self): class TestSumCase5(TestGraphSendUERecvSumOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -353,7 +371,6 @@ def set_config(self): class TestSumCase6(TestGraphSendUERecvSumOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -364,7 +381,6 @@ def set_config(self): class TestSumCase7(TestGraphSendUERecvSumOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -375,7 +391,6 @@ def set_config(self): class TestGraphSendUERecvMeanOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_ue_recv_wrapper @@ -386,12 +401,13 @@ def setUp(self): 'X': self.x, 'Y': self.y, 'Src_index': self.src_index, - 'Dst_index': self.dst_index + 'Dst_index': self.dst_index, } self.attrs = {'message_op': self.message_op, 'reduce_op': 'MEAN'} out, dst_count = compute_graph_send_ue_recv_for_mean( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = {'Out': out, 'Dst_count': dst_count} @@ -411,7 +427,6 @@ def test_check_grad(self): class TestMeanCase1(TestGraphSendUERecvMeanOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -422,7 +437,6 @@ def set_config(self): class TestMeanCase2(TestGraphSendUERecvMeanOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -433,7 +447,6 @@ def set_config(self): class TestMeanCase3(TestGraphSendUERecvMeanOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -444,7 +457,6 @@ def set_config(self): class TestMeanCase4(TestGraphSendUERecvMeanOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -455,7 +467,6 @@ def set_config(self): class TestMeanCase5(TestGraphSendUERecvMeanOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -466,7 +477,6 @@ def set_config(self): class TestMeanCase6(TestGraphSendUERecvMeanOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -477,7 +487,6 @@ def set_config(self): class TestMeanCase7(TestGraphSendUERecvMeanOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -488,7 +497,6 @@ def set_config(self): class TestGraphSendUERecvMaxOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_ue_recv_wrapper @@ -499,12 +507,13 @@ def setUp(self): 'X': self.x, 'Y': self.y, 'Src_index': self.src_index, - 'Dst_index': self.dst_index + 'Dst_index': self.dst_index, } self.attrs = {'message_op': self.message_op, 'reduce_op': 'MAX'} out, self.gradients = compute_graph_send_ue_recv_for_max_min( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = {'Out': out} @@ -520,14 +529,15 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=self.gradients, - check_eager=True) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=self.gradients, + check_eager=True, + ) class TestMaxCase1(TestGraphSendUERecvMaxOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -538,7 +548,6 @@ def set_config(self): class TestMaxCase2(TestGraphSendUERecvMaxOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -549,7 +558,6 @@ def set_config(self): class TestMaxCase3(TestGraphSendUERecvMaxOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -560,7 +568,6 @@ def set_config(self): class TestMaxCase4(TestGraphSendUERecvMaxOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -571,7 +578,6 @@ def set_config(self): class TestMaxCase5(TestGraphSendUERecvMaxOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -582,7 +588,6 @@ def set_config(self): class TestMaxCase6(TestGraphSendUERecvMaxOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -593,7 +598,6 @@ def set_config(self): class TestMaxCase7(TestGraphSendUERecvMaxOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -604,7 +608,6 @@ def set_config(self): class TestGraphSendUERecvMinOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_ue_recv_wrapper @@ -615,12 +618,13 @@ def setUp(self): 'X': self.x, 'Y': self.y, 'Src_index': self.src_index, - 'Dst_index': self.dst_index + 'Dst_index': self.dst_index, } self.attrs = {'message_op': self.message_op, 'reduce_op': 'MIN'} out, self.gradients = compute_graph_send_ue_recv_for_max_min( - self.inputs, self.attrs) + self.inputs, self.attrs + ) self.outputs = {'Out': out} @@ -636,14 +640,15 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=self.gradients, - check_eager=True) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=self.gradients, + check_eager=True, + ) class TestMinCase1(TestGraphSendUERecvMinOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -654,7 +659,6 @@ def set_config(self): class TestMinCase2(TestGraphSendUERecvMinOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -665,7 +669,6 @@ def set_config(self): class TestMinCase3(TestGraphSendUERecvMinOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((150, 1)).astype("float64") @@ -676,7 +679,6 @@ def set_config(self): class TestMinCase4(TestGraphSendUERecvMinOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -687,7 +689,6 @@ def set_config(self): class TestMinCase5(TestGraphSendUERecvMinOp): - def set_config(self): self.x = np.random.random((10, 8, 5)).astype("float64") self.y = np.random.random((15, 8, 1)).astype("float64") @@ -698,7 +699,6 @@ def set_config(self): class TestMinCase6(TestGraphSendUERecvMinOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -709,7 +709,6 @@ def set_config(self): class TestMinCase7(TestGraphSendUERecvMinOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((15, 20)).astype("float64") @@ -720,23 +719,27 @@ def set_config(self): class API_GeometricSendUERecvTest(unittest.TestCase): - def test_compute_all_with_sum(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float32" + ) y = paddle.ones(shape=[4, 1], dtype="float32") src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") - res_add = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "add", "sum") - res_sub = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "sub", "sum") - res_mul = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "mul", "sum") - res_div = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "div", "sum") + res_add = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "add", "sum" + ) + res_sub = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "sub", "sum" + ) + res_mul = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "mul", "sum" + ) + res_div = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "div", "sum" + ) res = [res_add, res_sub, res_mul, res_div] np_add = np.array([[1, 3, 4], [4, 10, 12], [2, 5, 6]], dtype="float32") @@ -750,25 +753,32 @@ def test_compute_all_with_sum(self): paddle_res, rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!'. - format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_compute_all_with_mean(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float32" + ) y = paddle.ones(shape=[4, 1], dtype="float32") src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") - res_add = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "add", "mean") - res_sub = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "sub", "mean") - res_mul = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "mul", "mean") - res_div = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "div", "mean") + res_add = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "add", "mean" + ) + res_sub = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "sub", "mean" + ) + res_mul = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "mul", "mean" + ) + res_div = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "div", "mean" + ) res = [res_add, res_sub, res_mul, res_div] np_add = np.array([[1, 3, 4], [2, 5, 6], [2, 5, 6]], dtype="float32") @@ -782,25 +792,32 @@ def test_compute_all_with_mean(self): paddle_res, rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!'. - format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_compute_all_with_max(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float32" + ) y = paddle.ones(shape=[4, 1], dtype="float32") src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") - res_add = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "add", "max") - res_sub = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "sub", "max") - res_mul = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "mul", "max") - res_div = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "div", "max") + res_add = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "add", "max" + ) + res_sub = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "sub", "max" + ) + res_mul = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "mul", "max" + ) + res_div = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "div", "max" + ) res = [res_add, res_sub, res_mul, res_div] np_add = np.array([[1, 3, 4], [3, 7, 8], [2, 5, 6]], dtype="float32") @@ -815,73 +832,91 @@ def test_compute_all_with_max(self): paddle_res, rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!'. - format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_compute_all_with_max_fp16(self): paddle.disable_static() if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) if core.is_float16_supported(place): - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, - 7]]), - dtype="float16") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float16" + ) y = paddle.ones(shape=[4, 1], dtype="float16") - src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), - dtype="int32") - dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), - dtype="int32") - - res_add = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "add", "max") - res_sub = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "sub", "max") - res_mul = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "mul", "max") - res_div = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "div", "max") + src_index = paddle.to_tensor( + np.array([0, 1, 2, 0]), dtype="int32" + ) + dst_index = paddle.to_tensor( + np.array([1, 2, 1, 0]), dtype="int32" + ) + + res_add = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "add", "max" + ) + res_sub = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "sub", "max" + ) + res_mul = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "mul", "max" + ) + res_div = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "div", "max" + ) res = [res_add, res_sub, res_mul, res_div] - np_add = np.array([[1, 3, 4], [3, 7, 8], [2, 5, 6]], - dtype="float16") - np_sub = np.array([[-1, 1, 2], [1, 5, 6], [0, 3, 4]], - dtype="float16") - np_mul = np.array([[0, 2, 3], [2, 6, 7], [1, 4, 5]], - dtype="float16") - np_div = np.array([[0, 2, 3], [2, 6, 7], [1, 4, 5]], - dtype="float16") - - np.testing.assert_allclose(np_sub, - res_sub, - rtol=1e-05, - atol=1e-06) - for np_res, paddle_res in zip([np_add, np_sub, np_mul, np_div], - res): + np_add = np.array( + [[1, 3, 4], [3, 7, 8], [2, 5, 6]], dtype="float16" + ) + np_sub = np.array( + [[-1, 1, 2], [1, 5, 6], [0, 3, 4]], dtype="float16" + ) + np_mul = np.array( + [[0, 2, 3], [2, 6, 7], [1, 4, 5]], dtype="float16" + ) + np_div = np.array( + [[0, 2, 3], [2, 6, 7], [1, 4, 5]], dtype="float16" + ) + + np.testing.assert_allclose( + np_sub, res_sub, rtol=1e-05, atol=1e-06 + ) + for np_res, paddle_res in zip( + [np_add, np_sub, np_mul, np_div], res + ): np.testing.assert_allclose( np_res, paddle_res, rtol=1e-05, atol=1e-06, - err_msg= - 'two value is {}\n{}, check diff!' - .format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_compute_all_with_min(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float32" + ) y = paddle.ones(shape=[4, 1], dtype="float32") src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") - res_add = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "add", "min") - res_sub = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "sub", "min") - res_mul = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "mul", "min") - res_div = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "div", "min") + res_add = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "add", "min" + ) + res_sub = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "sub", "min" + ) + res_mul = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "mul", "min" + ) + res_div = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "div", "min" + ) res = [res_add, res_sub, res_mul, res_div] np_add = np.array([[1, 3, 4], [1, 3, 4], [2, 5, 6]], dtype="float32") @@ -895,71 +930,90 @@ def test_compute_all_with_min(self): paddle_res, rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!'. - format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_compute_all_with_min_fp16(self): paddle.disable_static() if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) if core.is_float16_supported(place): - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, - 7]]), - dtype="float16") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float16" + ) y = paddle.ones(shape=[4, 1], dtype="float16") - src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), - dtype="int32") - dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), - dtype="int32") - res_add = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "add", "min") - res_sub = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "sub", "min") - res_mul = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "mul", "min") - res_div = paddle.geometric.send_ue_recv(x, y, src_index, - dst_index, "div", "min") + src_index = paddle.to_tensor( + np.array([0, 1, 2, 0]), dtype="int32" + ) + dst_index = paddle.to_tensor( + np.array([1, 2, 1, 0]), dtype="int32" + ) + res_add = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "add", "min" + ) + res_sub = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "sub", "min" + ) + res_mul = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "mul", "min" + ) + res_div = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "div", "min" + ) res = [res_add, res_sub, res_mul, res_div] - np_add = np.array([[1, 3, 4], [1, 3, 4], [2, 5, 6]], - dtype="float16") - np_sub = np.array([[-1, 1, 2], [-1, 1, 2], [0, 3, 4]], - dtype="float16") - np_mul = np.array([[0, 2, 3], [0, 2, 3], [1, 4, 5]], - dtype="float16") - np_div = np.array([[0, 2, 3], [0, 2, 3], [1, 4, 5]], - dtype="float16") - - for np_res, paddle_res in zip([np_add, np_sub, np_mul, np_div], - res): + np_add = np.array( + [[1, 3, 4], [1, 3, 4], [2, 5, 6]], dtype="float16" + ) + np_sub = np.array( + [[-1, 1, 2], [-1, 1, 2], [0, 3, 4]], dtype="float16" + ) + np_mul = np.array( + [[0, 2, 3], [0, 2, 3], [1, 4, 5]], dtype="float16" + ) + np_div = np.array( + [[0, 2, 3], [0, 2, 3], [1, 4, 5]], dtype="float16" + ) + + for np_res, paddle_res in zip( + [np_add, np_sub, np_mul, np_div], res + ): np.testing.assert_allclose( np_res, paddle_res, rtol=1e-05, atol=1e-06, - err_msg= - 'two value is {}\n{}, check diff!' - .format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_reshape_lhs_rhs(self): paddle.disable_static() - x = paddle.to_tensor(np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), - dtype="float32") + x = paddle.to_tensor( + np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]]), dtype="float32" + ) x = x.reshape(shape=[3, 3, 1]) y = paddle.ones([4, 1], dtype="float32") src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") - res_add = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "add", "min") - np_add = np.array([[1, 3, 4], [1, 3, 4], [2, 5, 6]], - dtype="float16").reshape([3, 3, 1]) + res_add = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "add", "min" + ) + np_add = np.array( + [[1, 3, 4], [1, 3, 4], [2, 5, 6]], dtype="float16" + ).reshape([3, 3, 1]) np.testing.assert_allclose( np_add, res_add, rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!'. - format(np_add, res_add)) + err_msg='two value is {}\n{}, check diff!'.format( + np_add, res_add + ), + ) def test_out_size_tensor_static(self): paddle.enable_static() @@ -968,12 +1022,13 @@ def test_out_size_tensor_static(self): y = paddle.static.data(name="y", shape=[3], dtype="float32") src_index = paddle.static.data(name="src", shape=[3], dtype="int32") dst_index = paddle.static.data(name="dst", shape=[3], dtype="int32") - out_size = paddle.static.data(name="out_size", - shape=[1], - dtype="int32") + out_size = paddle.static.data( + name="out_size", shape=[1], dtype="int32" + ) - res_sum = paddle.geometric.send_ue_recv(x, y, src_index, dst_index, - "mul", "sum", out_size) + res_sum = paddle.geometric.send_ue_recv( + x, y, src_index, dst_index, "mul", "sum", out_size + ) exe = paddle.static.Executor(paddle.CPUPlace()) data1 = np.array([[0, 2, 3], [1, 4, 5], [2, 6, 6]], dtype="float32") @@ -984,21 +1039,25 @@ def test_out_size_tensor_static(self): np_sum = np.array([[0, 2, 3], [3, 16, 21]], dtype="float32") - ret = exe.run(feed={ - 'x': data1, - 'y': data2, - 'src': data3, - 'dst': data4, - 'out_size': data5, - }, - fetch_list=[res_sum]) + ret = exe.run( + feed={ + 'x': data1, + 'y': data2, + 'src': data3, + 'dst': data4, + 'out_size': data5, + }, + fetch_list=[res_sum], + ) np.testing.assert_allclose( np_sum, ret[0], rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!'. - format(np_sum, ret[0])) + err_msg='two value is {}\n{}, check diff!'.format( + np_sum, ret[0] + ), + ) def test_api_eager_dygraph(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_graph_send_uv_op.py b/python/paddle/fluid/tests/unittests/test_graph_send_uv_op.py index df22555d357ba..9dd252b17c531 100644 --- a/python/paddle/fluid/tests/unittests/test_graph_send_uv_op.py +++ b/python/paddle/fluid/tests/unittests/test_graph_send_uv_op.py @@ -40,12 +40,12 @@ def compute_graph_send_uv(inputs, attributes): def graph_send_uv_wrapper(x, y, src_index, dst_index, message_op="add"): - return paddle.geometric.send_uv(x, y, src_index, dst_index, - message_op.lower()) + return paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op.lower() + ) class TestGraphSendUVOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = graph_send_uv_wrapper @@ -56,7 +56,7 @@ def setUp(self): 'x': self.x, 'y': self.y, 'src_index': self.src_index, - 'dst_index': self.dst_index + 'dst_index': self.dst_index, } self.attrs = {'message_op': self.message_op} out = compute_graph_send_uv(self.inputs, self.attrs) @@ -78,7 +78,6 @@ def set_config(self): class TestCase1(TestGraphSendUVOp): - def set_config(self): self.x = np.random.random((10, 20)).astype("float64") self.y = np.random.random((10, 20)).astype("float64") @@ -89,7 +88,6 @@ def set_config(self): class TestCase2(TestGraphSendUVOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((100, 20)).astype("float64") @@ -100,7 +98,6 @@ def set_config(self): class TestCase3(TestGraphSendUVOp): - def set_config(self): self.x = np.random.random((100, 20)).astype("float64") self.y = np.random.random((100, 1)).astype("float64") @@ -111,7 +108,6 @@ def set_config(self): class TestCase4(TestGraphSendUVOp): - def set_config(self): self.x = np.random.random((100, 1)).astype("float64") self.y = np.random.random((100, 20)).astype("float64") @@ -122,7 +118,6 @@ def set_config(self): class TestCase5(TestGraphSendUVOp): - def set_config(self): self.x = np.random.random((100, 20)).astype("float64") self.y = np.random.random((100, 1)).astype("float64") @@ -133,7 +128,6 @@ def set_config(self): class TestCase6(TestGraphSendUVOp): - def set_config(self): self.x = np.random.random((10, 10, 1)).astype("float64") self.y = np.random.random((10, 10, 10)) @@ -144,7 +138,6 @@ def set_config(self): class TestCase7(TestGraphSendUVOp): - def set_config(self): self.x = np.random.random((10, 10, 1)).astype("float64") self.y = np.random.random((10, 10, 10)) @@ -155,7 +148,6 @@ def set_config(self): class API_GeometricSendUVTest(unittest.TestCase): - def test_compute_all_dygraph(self): paddle.disable_static() x = paddle.to_tensor([[0, 2, 3], [1, 4, 5], [2, 6, 7]], dtype="float32") @@ -163,37 +155,33 @@ def test_compute_all_dygraph(self): src_index = paddle.to_tensor(np.array([0, 1, 2, 0]), dtype="int32") dst_index = paddle.to_tensor(np.array([1, 2, 1, 0]), dtype="int32") - res_add = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="add") - res_sub = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="sub") - res_mul = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="mul") - res_div = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="div") + res_add = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="add" + ) + res_sub = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="sub" + ) + res_mul = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="mul" + ) + res_div = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="div" + ) res = [res_add, res_sub, res_mul, res_div] - np_add = np.array([[2, 5, 7], [5, 9, 11], [4, 9, 11], [1, 3, 5]], - dtype="float32") - np_sub = np.array([[-2, -1, -1], [-3, -1, -1], [0, 3, 3], [-1, 1, 1]], - dtype="float32") - np_mul = np.array([[0, 6, 12], [4, 20, 30], [4, 18, 28], [0, 2, 6]], - dtype="float32") + np_add = np.array( + [[2, 5, 7], [5, 9, 11], [4, 9, 11], [1, 3, 5]], dtype="float32" + ) + np_sub = np.array( + [[-2, -1, -1], [-3, -1, -1], [0, 3, 3], [-1, 1, 1]], dtype="float32" + ) + np_mul = np.array( + [[0, 6, 12], [4, 20, 30], [4, 18, 28], [0, 2, 6]], dtype="float32" + ) np_div = np.array( [[0, 2 / 3, 0.75], [0.25, 0.8, 5 / 6], [1, 2, 7 / 4], [0, 2, 1.5]], - dtype="float32") + dtype="float32", + ) for np_res, paddle_res in zip([np_add, np_sub, np_mul, np_div], res): np.testing.assert_allclose( @@ -201,8 +189,10 @@ def test_compute_all_dygraph(self): paddle_res, rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!'. - format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_compute_all_static(self): paddle.enable_static() @@ -211,26 +201,18 @@ def test_compute_all_static(self): y = paddle.static.data(name="y", shape=[3, 3], dtype="float32") src_index = paddle.static.data(name="src", shape=[4], dtype="int32") dst_index = paddle.static.data(name="dst", shape=[4], dtype="int32") - res_add = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="add") - res_sub = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="sub") - res_mul = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="mul") - res_div = paddle.geometric.send_uv(x, - y, - src_index, - dst_index, - message_op="div") + res_add = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="add" + ) + res_sub = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="sub" + ) + res_mul = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="mul" + ) + res_div = paddle.geometric.send_uv( + x, y, src_index, dst_index, message_op="div" + ) exe = paddle.static.Executor(paddle.CPUPlace()) data1 = np.array([[0, 2, 3], [1, 4, 5], [2, 6, 7]], dtype="float32") @@ -238,33 +220,48 @@ def test_compute_all_static(self): data3 = np.array([0, 1, 2, 0], dtype="int32") data4 = np.array([1, 2, 1, 0], dtype="int32") - np_add = np.array([[2, 5, 7], [5, 9, 11], [4, 9, 11], [1, 3, 5]], - dtype="float32") + np_add = np.array( + [[2, 5, 7], [5, 9, 11], [4, 9, 11], [1, 3, 5]], dtype="float32" + ) np_sub = np.array( [[-2, -1, -1], [-3, -1, -1], [0, 3, 3], [-1, 1, 1]], - dtype="float32") - np_mul = np.array([[0, 6, 12], [4, 20, 30], [4, 18, 28], [0, 2, 6]], - dtype="float32") - np_div = np.array([[0, 2 / 3, 0.75], [0.25, 0.8, 5 / 6], - [1, 2, 7 / 4], [0, 2, 1.5]], - dtype="float32") - - ret = exe.run(feed={ - 'x': data1, - 'y': data2, - 'src': data3, - 'dst': data4, - }, - fetch_list=[res_add, res_sub, res_mul, res_div]) - for np_res, paddle_res in zip([np_add, np_sub, np_mul, np_div], - ret): + dtype="float32", + ) + np_mul = np.array( + [[0, 6, 12], [4, 20, 30], [4, 18, 28], [0, 2, 6]], + dtype="float32", + ) + np_div = np.array( + [ + [0, 2 / 3, 0.75], + [0.25, 0.8, 5 / 6], + [1, 2, 7 / 4], + [0, 2, 1.5], + ], + dtype="float32", + ) + + ret = exe.run( + feed={ + 'x': data1, + 'y': data2, + 'src': data3, + 'dst': data4, + }, + fetch_list=[res_add, res_sub, res_mul, res_div], + ) + for np_res, paddle_res in zip( + [np_add, np_sub, np_mul, np_div], ret + ): np.testing.assert_allclose( np_res, paddle_res, rtol=1e-05, atol=1e-06, - err_msg='two value is {}\n{}, check diff!' - .format(np_res, paddle_res)) + err_msg='two value is {}\n{}, check diff!'.format( + np_res, paddle_res + ), + ) def test_api_eager_dygraph(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_grid_sample_function.py b/python/paddle/fluid/tests/unittests/test_grid_sample_function.py index 7255f82bc0757..7500109185532 100644 --- a/python/paddle/fluid/tests/unittests/test_grid_sample_function.py +++ b/python/paddle/fluid/tests/unittests/test_grid_sample_function.py @@ -21,14 +21,15 @@ class GridSampleTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - x_shape=[2, 2, 3, 3], - grid_shape=[2, 3, 3, 2], - mode="bilinear", - padding_mode="zeros", - align_corners=False): + def __init__( + self, + methodName='runTest', + x_shape=[2, 2, 3, 3], + grid_shape=[2, 3, 3, 2], + mode="bilinear", + padding_mode="zeros", + align_corners=False, + ): super(GridSampleTestCase, self).__init__(methodName) self.padding_mode = padding_mode self.x_shape = x_shape @@ -49,25 +50,29 @@ def static_functional(self, place): with fluid.program_guard(main, start): x = fluid.data("x", self.x_shape, dtype=self.dtype) grid = fluid.data("grid", self.grid_shape, dtype=self.dtype) - y_var = F.grid_sample(x, - grid, - mode=self.mode, - padding_mode=self.padding_mode, - align_corners=self.align_corners) + y_var = F.grid_sample( + x, + grid, + mode=self.mode, + padding_mode=self.padding_mode, + align_corners=self.align_corners, + ) feed_dict = {"x": self.x, "grid": self.grid} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def dynamic_functional(self): x_t = paddle.to_tensor(self.x) grid_t = paddle.to_tensor(self.grid) - y_t = F.grid_sample(x_t, - grid_t, - mode=self.mode, - padding_mode=self.padding_mode, - align_corners=self.align_corners) + y_t = F.grid_sample( + x_t, + grid_t, + mode=self.mode, + padding_mode=self.padding_mode, + align_corners=self.align_corners, + ) y_np = y_t.numpy() return y_np @@ -87,7 +92,6 @@ def runTest(self): class GridSampleErrorTestCase(GridSampleTestCase): - def runTest(self): place = fluid.CPUPlace() with self.assertRaises(ValueError): @@ -97,22 +101,30 @@ def runTest(self): def add_cases(suite): suite.addTest(GridSampleTestCase(methodName='runTest')) suite.addTest( - GridSampleTestCase(methodName='runTest', - mode='bilinear', - padding_mode='reflection', - align_corners=True)) + GridSampleTestCase( + methodName='runTest', + mode='bilinear', + padding_mode='reflection', + align_corners=True, + ) + ) suite.addTest( - GridSampleTestCase(methodName='runTest', - mode='bilinear', - padding_mode='zeros', - align_corners=True)) + GridSampleTestCase( + methodName='runTest', + mode='bilinear', + padding_mode='zeros', + align_corners=True, + ) + ) def add_error_cases(suite): suite.addTest( - GridSampleErrorTestCase(methodName='runTest', padding_mode="VALID")) + GridSampleErrorTestCase(methodName='runTest', padding_mode="VALID") + ) suite.addTest( - GridSampleErrorTestCase(methodName='runTest', align_corners="VALID")) + GridSampleErrorTestCase(methodName='runTest', align_corners="VALID") + ) suite.addTest(GridSampleErrorTestCase(methodName='runTest', mode="VALID")) @@ -124,7 +136,6 @@ def load_tests(loader, standard_tests, pattern): class TestGridSampleAPI(unittest.TestCase): - def test_errors(self): with self.assertRaises(ValueError): x = paddle.randn([1, 1, 3, 3]) diff --git a/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py b/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py index c162da6152800..0245383045b85 100644 --- a/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py +++ b/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py @@ -25,12 +25,15 @@ def AffineGrid(theta, grid_shape): n = grid_shape[0] h = grid_shape[1] w = grid_shape[2] - h_idx = np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, - axis=0).T[:, :, np.newaxis] - w_idx = np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, - axis=0)[:, :, np.newaxis] - grid = np.concatenate([w_idx, h_idx, np.ones([h, w, 1])], - axis=2) # h * w * 3 + h_idx = np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, axis=0).T[ + :, :, np.newaxis + ] + w_idx = np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, axis=0)[ + :, :, np.newaxis + ] + grid = np.concatenate( + [w_idx, h_idx, np.ones([h, w, 1])], axis=2 + ) # h * w * 3 grid = np.repeat(grid[np.newaxis, :], n, axis=0) # n * h * w *3 ret = np.zeros([n, h * w, 2]) @@ -50,13 +53,17 @@ def getGridPointValue(data, x, y): out_H = x.shape[1] out_W = x.shape[2] - #out = np.zeros(data_shape, dtype='float64') + # out = np.zeros(data_shape, dtype='float64') out = np.zeros([N, C, out_H, out_W], dtype='float64') for i in range(N): for j in range(out_H): for k in range(out_W): - if y[i, j, k] < 0 or y[i, j, k] > in_H - 1 or x[ - i, j, k] < 0 or x[i, j, k] > in_W - 1: + if ( + y[i, j, k] < 0 + or y[i, j, k] > in_H - 1 + or x[i, j, k] < 0 + or x[i, j, k] > in_W - 1 + ): out[i, :, j, k] = 0 else: out[i, :, j, k] = data[i, :, y[i, j, k], x[i, j, k]] @@ -69,26 +76,24 @@ def AffineGrid3D(theta, grid_shape): d = grid_shape[1] h = grid_shape[2] w = grid_shape[3] - d_idx = np.repeat(np.repeat(np.linspace(-1, 1, d)[:, np.newaxis, - np.newaxis], - h, - axis=1), - w, - axis=2)[:, :, :, np.newaxis] - h_idx = np.repeat(np.repeat(np.linspace(-1, 1, h)[np.newaxis, :, - np.newaxis], - w, - axis=2), - d, - axis=0)[:, :, :, np.newaxis] - w_idx = np.repeat(np.repeat(np.linspace(-1, 1, w)[np.newaxis, - np.newaxis, :], - h, - axis=1), - d, - axis=0)[:, :, :, np.newaxis] + d_idx = np.repeat( + np.repeat(np.linspace(-1, 1, d)[:, np.newaxis, np.newaxis], h, axis=1), + w, + axis=2, + )[:, :, :, np.newaxis] + h_idx = np.repeat( + np.repeat(np.linspace(-1, 1, h)[np.newaxis, :, np.newaxis], w, axis=2), + d, + axis=0, + )[:, :, :, np.newaxis] + w_idx = np.repeat( + np.repeat(np.linspace(-1, 1, w)[np.newaxis, np.newaxis, :], h, axis=1), + d, + axis=0, + )[:, :, :, np.newaxis] grid = np.concatenate( - [w_idx, h_idx, d_idx, np.ones([d, h, w, 1])], axis=3) # d * h * w * 4 + [w_idx, h_idx, d_idx, np.ones([d, h, w, 1])], axis=3 + ) # d * h * w * 4 grid = np.repeat(grid[np.newaxis, :], n, axis=0) # n * d * h * w *4 ret = np.zeros([n, d * h * w, 3]) theta = theta.transpose([0, 2, 1]) @@ -114,13 +119,19 @@ def getGridPointValue3D(data, x, y, z): for j in range(out_D): for k in range(out_H): for l in range(out_W): - if y[i, j, k, l] < 0 or y[i, j, k, l] > in_H - 1 or x[ - i, j, k, l] < 0 or x[i, j, k, l] > in_W - 1 or z[ - i, j, k, l] < 0 or z[i, j, k, l] > in_D - 1: + if ( + y[i, j, k, l] < 0 + or y[i, j, k, l] > in_H - 1 + or x[i, j, k, l] < 0 + or x[i, j, k, l] > in_W - 1 + or z[i, j, k, l] < 0 + or z[i, j, k, l] > in_D - 1 + ): out[i, :, j, k, l] = 0 else: - out[i, :, j, k, l] = data[i, :, z[i, j, k, l], - y[i, j, k, l], x[i, j, k, l]] + out[i, :, j, k, l] = data[ + i, :, z[i, j, k, l], y[i, j, k, l], x[i, j, k, l] + ] return out @@ -133,27 +144,28 @@ def unnormalizeAndClip(grid_slice, max_val, align_corners, padding_mode): if align_corners: grid_slice = 0.5 * ((grid_slice.astype('float64') + 1.0) * max_val) else: - grid_slice = 0.5 * ((grid_slice.astype('float64') + 1.0) * - (max_val + 1)) - 0.5 + grid_slice = ( + 0.5 * ((grid_slice.astype('float64') + 1.0) * (max_val + 1)) - 0.5 + ) if padding_mode == "border": grid_slice = clip(grid_slice, 0, max_val) elif padding_mode == "reflection": double_range = 2 * max_val if align_corners else (max_val + 1) * 2 - grid_abs = np.abs(grid_slice) if align_corners else np.abs(grid_slice + - 0.5) + grid_abs = ( + np.abs(grid_slice) if align_corners else np.abs(grid_slice + 0.5) + ) extra = grid_abs - np.floor(grid_abs / double_range) * double_range grid_slice = np.minimum(extra, double_range - extra) - grid_slice = grid_slice if align_corners else clip( - grid_slice - 0.5, 0, max_val) + grid_slice = ( + grid_slice if align_corners else clip(grid_slice - 0.5, 0, max_val) + ) return grid_slice -def GridSampler(data, - grid, - align_corners=True, - mode="bilinear", - padding_mode="zeros"): +def GridSampler( + data, grid, align_corners=True, mode="bilinear", padding_mode="zeros" +): dims = data.shape N = dims[0] in_C = dims[1] @@ -177,14 +189,18 @@ def GridSampler(data, y0 = np.floor(y).astype('int32') y1 = y0 + 1 - wa = np.tile(((x1 - x) * (y1 - y)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wb = np.tile(((x1 - x) * (y - y0)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wc = np.tile(((x - x0) * (y1 - y)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wd = np.tile(((x - x0) * (y - y0)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) + wa = np.tile( + ((x1 - x) * (y1 - y)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wb = np.tile( + ((x1 - x) * (y - y0)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wc = np.tile( + ((x - x0) * (y1 - y)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wd = np.tile( + ((x - x0) * (y - y0)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) va = getGridPointValue(data, x0, y0) vb = getGridPointValue(data, x0, y1) @@ -199,11 +215,9 @@ def GridSampler(data, return out -def GridSampler3D(data, - grid, - align_corners=True, - mode="bilinear", - padding_mode="zeros"): +def GridSampler3D( + data, grid, align_corners=True, mode="bilinear", padding_mode="zeros" +): dims = data.shape N = dims[0] in_C = dims[1] @@ -235,22 +249,54 @@ def GridSampler3D(data, z0 = np.floor(z).astype('int32') z1 = z0 + 1 - w_tnw = np.tile(((x1 - x) * (y1 - y) * (z1 - z)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) - w_tne = np.tile(((x - x0) * (y1 - y) * (z1 - z)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) - w_tsw = np.tile(((x1 - x) * (y - y0) * (z1 - z)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) - w_tse = np.tile(((x - x0) * (y - y0) * (z1 - z)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) - w_bnw = np.tile(((x1 - x) * (y1 - y) * (z - z0)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) - w_bne = np.tile(((x - x0) * (y1 - y) * (z - z0)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) - w_bsw = np.tile(((x1 - x) * (y - y0) * (z - z0)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) - w_bse = np.tile(((x - x0) * (y - y0) * (z - z0)).reshape( - (N, 1, out_D, out_H, out_W)), (1, in_C, 1, 1, 1)) + w_tnw = np.tile( + ((x1 - x) * (y1 - y) * (z1 - z)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) + w_tne = np.tile( + ((x - x0) * (y1 - y) * (z1 - z)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) + w_tsw = np.tile( + ((x1 - x) * (y - y0) * (z1 - z)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) + w_tse = np.tile( + ((x - x0) * (y - y0) * (z1 - z)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) + w_bnw = np.tile( + ((x1 - x) * (y1 - y) * (z - z0)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) + w_bne = np.tile( + ((x - x0) * (y1 - y) * (z - z0)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) + w_bsw = np.tile( + ((x1 - x) * (y - y0) * (z - z0)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) + w_bse = np.tile( + ((x - x0) * (y - y0) * (z - z0)).reshape( + (N, 1, out_D, out_H, out_W) + ), + (1, in_C, 1, 1, 1), + ) v_tnw = getGridPointValue3D(data, x0, y0, z0) v_tne = getGridPointValue3D(data, x1, y0, z0) @@ -261,9 +307,16 @@ def GridSampler3D(data, v_bsw = getGridPointValue3D(data, x0, y1, z1) v_bse = getGridPointValue3D(data, x1, y1, z1) - out = (w_tnw * v_tnw + w_tne * v_tne + w_tsw * v_tsw + w_tse * v_tse + - w_bnw * v_bnw + w_bne * v_bne + w_bsw * v_bsw + - w_bse * v_bse).astype('float64') + out = ( + w_tnw * v_tnw + + w_tne * v_tne + + w_tsw * v_tsw + + w_tse * v_tse + + w_bnw * v_bnw + + w_bne * v_bne + + w_bsw * v_bsw + + w_bse * v_bse + ).astype('float64') elif mode == "nearest": x = np.round(x).astype('int32') @@ -274,7 +327,6 @@ def GridSampler3D(data, class TestGridSamplerOp(OpTest): - def setUp(self): self.use_cudnn = False self.numeric_grad_delta = 0.0001 @@ -298,12 +350,12 @@ def setUp(self): 'use_cudnn': self.use_cudnn, "align_corners": self.align_corners, "padding_mode": self.padding_mode, - "mode": self.mode + "mode": self.mode, } self.outputs = { - 'Output': - GridSampler(x, grid, self.align_corners, self.mode, - self.padding_mode) + 'Output': GridSampler( + x, grid, self.align_corners, self.mode, self.padding_mode + ) } else: for i in range(self.theta_shape[0]): @@ -316,23 +368,25 @@ def setUp(self): 'use_cudnn': self.use_cudnn, "align_corners": self.align_corners, "padding_mode": self.padding_mode, - "mode": self.mode + "mode": self.mode, } self.outputs = { - 'Output': - GridSampler3D(x, grid, self.align_corners, self.mode, - self.padding_mode) + 'Output': GridSampler3D( + x, grid, self.align_corners, self.mode, self.padding_mode + ) } def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Grid'], - 'Output', - max_relative_error=0.01, - numeric_grad_delta=self.numeric_grad_delta, - check_eager=True) + self.check_grad( + ['X', 'Grid'], + 'Output', + max_relative_error=0.01, + numeric_grad_delta=self.numeric_grad_delta, + check_eager=True, + ) def initTestCase(self): self.x_shape = (2, 3, 8, 8) @@ -345,7 +399,6 @@ def initTestCase(self): class Case1(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -356,7 +409,6 @@ def initTestCase(self): class Case1_(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -367,7 +419,6 @@ def initTestCase(self): class Case2(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -378,7 +429,6 @@ def initTestCase(self): class Case3(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -389,7 +439,6 @@ def initTestCase(self): class Case4(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -400,10 +449,11 @@ def initTestCase(self): self.numeric_grad_delta = 0.0001 -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class LargeInputCase(TestGridSamplerOp): - def get_places(self): places = [] if core.is_compiled_with_cuda(): @@ -423,10 +473,11 @@ def test_check_grad_normal(self): pass -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class Case5(LargeInputCase): - def initTestCase(self): self.no_need_check_grad = True self.x_shape = (2, 3, 128, 128) @@ -439,7 +490,6 @@ def initTestCase(self): class Case6(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6, 7) self.grid_shape = (2, 8, 9, 10, 3) @@ -451,7 +501,6 @@ def initTestCase(self): class Case6_(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 4, 5, 6) self.grid_shape = (2, 7, 8, 9, 3) @@ -463,7 +512,6 @@ def initTestCase(self): class Case7(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 4, 5, 6) self.grid_shape = (2, 7, 8, 9, 3) @@ -475,7 +523,6 @@ def initTestCase(self): class Case8(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 4, 5, 6) self.grid_shape = (2, 7, 8, 9, 3) @@ -487,7 +534,6 @@ def initTestCase(self): class Case9(TestGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 4, 5, 6) self.grid_shape = (2, 7, 8, 9, 3) @@ -498,10 +544,11 @@ def initTestCase(self): self.numeric_grad_delta = 0.000001 -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class LargeInput3DCase(TestGridSamplerOp): - def get_places(self): places = [] if core.is_compiled_with_cuda(): @@ -523,10 +570,11 @@ def test_check_grad_normal(self): pass -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class Case10(LargeInput3DCase): - def initTestCase(self): self.no_need_check_grad = True self.x_shape = (2, 3, 24, 24, 12) diff --git a/python/paddle/fluid/tests/unittests/test_group_norm_op.py b/python/paddle/fluid/tests/unittests/test_group_norm_op.py index 3ae2bf0a391f7..480efd5686e7c 100644 --- a/python/paddle/fluid/tests/unittests/test_group_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_group_norm_op.py @@ -32,14 +32,14 @@ def group_norm_naive(x, scale, bias, epsilon, groups, data_layout): var = np.var(x, axis=1, keepdims=True) output = (x - mean) / np.sqrt(var + epsilon) output = output.reshape((N, C, H, W)) * scale.reshape( - (-1, 1, 1)) + bias.reshape((-1, 1, 1)) + (-1, 1, 1) + ) + bias.reshape((-1, 1, 1)) if data_layout == "NHWC": output = np.transpose(output, (0, 2, 3, 1)) # NCHW => NHWC return output, mean.reshape((N, G)), var.reshape((N, G)) class TestGroupNormOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): @@ -51,9 +51,9 @@ def test_x_type(): self.assertRaises(TypeError, test_x_type) def test_x_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[2, 100, 3, 5], - dtype='int32') + x2 = fluid.layers.data( + name='x2', shape=[2, 100, 3, 5], dtype='int32' + ) groups = 2 fluid.layers.group_norm(x2, groups) @@ -61,7 +61,6 @@ def test_x_dtype(): class TestGroupNormOp(OpTest): - def setUp(self): self.op_type = "group_norm" self.data_format = "NCHW" @@ -76,15 +75,19 @@ def setUp(self): input = np.transpose(input, (0, 2, 3, 1)) scale = np.random.random([self.shape[1]]).astype(self.dtype) bias = np.random.random([self.shape[1]]).astype(self.dtype) - output, mean, var = group_norm_naive(input, scale, bias, - self.attrs['epsilon'], - self.attrs['groups'], - self.data_format) + output, mean, var = group_norm_naive( + input, + scale, + bias, + self.attrs['epsilon'], + self.attrs['groups'], + self.data_format, + ) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(input), 'Scale': OpTest.np_dtype_to_fluid_dtype(scale), - 'Bias': OpTest.np_dtype_to_fluid_dtype(bias) + 'Bias': OpTest.np_dtype_to_fluid_dtype(bias), } self.outputs = {'Y': output, 'Mean': mean, 'Variance': var} self.attrs['data_layout'] = self.data_format @@ -106,28 +109,37 @@ def test_check_output(self): # Set to inplace_atol to 0, which means the absolute error is 0, and the # relative error is 1e-05 in numpy.allclose by default. # Reference: https://docs.scipy.org/doc/numpy/reference/generated/numpy.allclose.html - self.check_output_with_place(place, - atol=atol, - inplace_atol=inplace_atol) + self.check_output_with_place( + place, atol=atol, inplace_atol=inplace_atol + ) def do_compare_between_place(self): - if not core.is_compiled_with_cuda(): return + if not core.is_compiled_with_cuda(): + return place = core.CPUPlace() place2 = core.CUDAPlace(0) self.scope = core.Scope() op_inputs = self.inputs if hasattr(self, "inputs") else dict() op_outputs = self.outputs if hasattr(self, "outputs") else dict() op_attrs = self.attrs if hasattr(self, "attrs") else dict() - self.op = create_op(self.scope, self.op_type, op_inputs, op_outputs, - op_attrs) + self.op = create_op( + self.scope, self.op_type, op_inputs, op_outputs, op_attrs + ) inputs_to_check = set(['X', 'Scale', 'Bias']) output_names = 'Y' - cpu_grads = self._get_gradient(inputs_to_check, place, output_names, - None) - gpu_grads = self._get_gradient(inputs_to_check, place2, output_names, - None) - self._assert_is_close(cpu_grads, gpu_grads, inputs_to_check, 0.005, - "Gradient Check On %s" % str(place)) + cpu_grads = self._get_gradient( + inputs_to_check, place, output_names, None + ) + gpu_grads = self._get_gradient( + inputs_to_check, place2, output_names, None + ) + self._assert_is_close( + cpu_grads, + gpu_grads, + inputs_to_check, + 0.005, + "Gradient Check On %s" % str(place), + ) def test_check_grad(self): if self.compare_between_place: @@ -149,44 +161,37 @@ def init_test_case(self): class TestGroupNormOp1(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 class TestGroupNormOp2(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 class TestGroupNormOpBigEps1(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 self.attrs['epsilon'] = 0.5 class TestGroupNormOpBigEps2(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 self.attrs['epsilon'] = 0.5 class TestGroupNormOpBigEps3(TestGroupNormOp): - def init_test_case(self): self.attrs['epsilon'] = 0.5 @skip_check_grad_ci( - reason= - '''This test case is used to ensure whether the gradient checking results between CPU and GPU + reason='''This test case is used to ensure whether the gradient checking results between CPU and GPU are consistent when using the same inputs, thus, it doesn't need to call check_grad.''' ) class TestGroupNormOpLargeData(TestGroupNormOp): - def init_test_case(self): self.shape = (2, 32, 64, 64) self.attrs['groups'] = 8 @@ -194,21 +199,18 @@ def init_test_case(self): class TestGroupNormOp1_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 self.data_format = "NHWC" class TestGroupNormOp2_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 self.data_format = "NHWC" class TestGroupNormOpBigEps1_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 1 self.attrs['epsilon'] = 0.5 @@ -216,7 +218,6 @@ def init_test_case(self): class TestGroupNormOpBigEps2_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['groups'] = 4 self.attrs['epsilon'] = 0.5 @@ -224,19 +225,16 @@ def init_test_case(self): class TestGroupNormOpBigEps3_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.attrs['epsilon'] = 0.5 self.data_format = "NHWC" @skip_check_grad_ci( - reason= - '''This test case is used to ensure whether the gradient checking results between CPU and GPU + reason='''This test case is used to ensure whether the gradient checking results between CPU and GPU are consistent when using the same inputs, thus, it doesn't need to call check_grad.''' ) class TestGroupNormOpLargeData_With_NHWC(TestGroupNormOp): - def init_test_case(self): self.shape = (2, 64, 32, 32) # NCHW self.attrs['groups'] = 8 @@ -245,16 +243,15 @@ def init_test_case(self): class TestGroupNormAPI_With_NHWC(unittest.TestCase): - def test_case1(self): data1 = fluid.data(name='data1', shape=[None, 3, 3, 4], dtype='float64') - out1 = fluid.layers.group_norm(input=data1, - groups=2, - data_layout="NHWC") + out1 = fluid.layers.group_norm( + input=data1, groups=2, data_layout="NHWC" + ) data2 = fluid.data(name='data2', shape=[None, 4, 3, 3], dtype='float64') - out2 = fluid.layers.group_norm(input=data2, - groups=2, - data_layout="NCHW") + out2 = fluid.layers.group_norm( + input=data2, groups=2, data_layout="NCHW" + ) data1_np = np.random.random((2, 3, 3, 4)).astype("float64") data2_np = np.random.random((2, 4, 3, 3)).astype("float64") @@ -263,25 +260,18 @@ def test_case1(self): place = core.CPUPlace() exe = fluid.Executor(place) - results = exe.run(fluid.default_main_program(), - feed={ - "data1": data1_np, - "data2": data2_np - }, - fetch_list=[out1, out2], - return_numpy=True) - expect_res1 = group_norm_naive(data1_np, - scale, - bias, - epsilon=1e-5, - groups=2, - data_layout="NHWC") - expect_res2 = group_norm_naive(data2_np, - scale, - bias, - epsilon=1e-5, - groups=2, - data_layout="NCHW") + results = exe.run( + fluid.default_main_program(), + feed={"data1": data1_np, "data2": data2_np}, + fetch_list=[out1, out2], + return_numpy=True, + ) + expect_res1 = group_norm_naive( + data1_np, scale, bias, epsilon=1e-5, groups=2, data_layout="NHWC" + ) + expect_res2 = group_norm_naive( + data2_np, scale, bias, epsilon=1e-5, groups=2, data_layout="NCHW" + ) np.testing.assert_allclose(results[0], expect_res1[0], rtol=1e-05) np.testing.assert_allclose(results[1], expect_res2[0], rtol=1e-05) @@ -292,15 +282,14 @@ def test_exception(self): data = fluid.data(name='data', shape=[None, 3, 3, 4], dtype="float64") def attr_data_format(): - out = fluid.layers.group_norm(input=data, - groups=2, - data_layout="NDHW") + out = fluid.layers.group_norm( + input=data, groups=2, data_layout="NDHW" + ) self.assertRaises(ValueError, attr_data_format) class TestGroupNormEager(unittest.TestCase): - def test_dygraph_api(self): self.dtype = np.float64 self.shape = (8, 32, 32) @@ -315,13 +304,17 @@ def test_dygraph_api(self): with _test_eager_guard(): tensor_eager_1 = fluid.dygraph.to_variable(input) tensor_eager_1.stop_gradient = False - groupNorm_eager = fluid.dygraph.nn.GroupNorm(channels=32, - groups=4) + groupNorm_eager = fluid.dygraph.nn.GroupNorm( + channels=32, groups=4 + ) ret2 = groupNorm_eager(tensor_eager_1) ret2.backward() - self.assertEqual(( - tensor_1.grad.numpy() == tensor_eager_1.grad.numpy()).all(), - True) + self.assertEqual( + ( + tensor_1.grad.numpy() == tensor_eager_1.grad.numpy() + ).all(), + True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_group_norm_op_v2.py b/python/paddle/fluid/tests/unittests/test_group_norm_op_v2.py index 52889f757df72..63b69e6107b78 100644 --- a/python/paddle/fluid/tests/unittests/test_group_norm_op_v2.py +++ b/python/paddle/fluid/tests/unittests/test_group_norm_op_v2.py @@ -33,18 +33,23 @@ def group_norm_naive_for_general_dimension(x, scale, bias, epsilon, groups): var = np.var(x, axis=1, keepdims=True) output = (x - mean) / np.sqrt(var + epsilon) output = output.reshape(input_shape) * scale.reshape( - (-1, 1, 1)) + bias.reshape((-1, 1, 1)) + (-1, 1, 1) + ) + bias.reshape((-1, 1, 1)) return output class TestDygraphGroupNormv2(unittest.TestCase): - def test_dygraph(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu("group_norm"): places.append(fluid.CUDAPlace(0)) - shapes = [[2, 2, 2, 2], [2, 2, 4], [4, 2], [4, 2, 6, 6, 2], - [2, 2, 2, 2, 2, 2]] + shapes = [ + [2, 2, 2, 2], + [2, 2, 4], + [4, 2], + [4, 2, 6, 6, 2], + [2, 2, 2, 2, 2, 2], + ] for p in places: def compute_v1(x): @@ -61,18 +66,20 @@ def compute_v2(x): def test_weight_bias_false(): with fluid.dygraph.guard(p): - gn = paddle.nn.GroupNorm(num_channels=2, - num_groups=2, - weight_attr=False, - bias_attr=False) + gn = paddle.nn.GroupNorm( + num_channels=2, + num_groups=2, + weight_attr=False, + bias_attr=False, + ) def test_nn_exception(): with fluid.dygraph.guard(p): def attr_data_format(): - out = paddle.nn.GroupNorm(num_groups=2, - num_channels=2, - data_format="NHWC") + out = paddle.nn.GroupNorm( + num_groups=2, num_channels=2, data_format="NHWC" + ) self.assertRaises(ValueError, attr_data_format) @@ -92,8 +99,13 @@ def test_static(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu("group_norm"): places.append(fluid.CUDAPlace(0)) - shapes = [[2, 6, 2, 2], [2, 6, 4], [4, 6], [4, 6, 6, 6, 2], - [4, 6, 2, 2, 2, 2]] + shapes = [ + [2, 6, 2, 2], + [2, 6, 4], + [4, 6], + [4, 6, 6, 6, 2], + [4, 6, 2, 2, 2, 2], + ] for p in places: exe = fluid.Executor(p) @@ -127,11 +139,15 @@ def test_eager_api(self): class TestGroupNormAPIV2_With_General_Dimensions(unittest.TestCase): - def test_numerical_accuracy(self): paddle.disable_static() - shapes = [(2, 6), (2, 6, 4), (2, 6, 4, 4), (2, 6, 6, 6, 2), - (2, 6, 6, 6, 2, 3)] + shapes = [ + (2, 6), + (2, 6, 4), + (2, 6, 4, 4), + (2, 6, 6, 6, 2), + (2, 6, 6, 6, 2, 3), + ] np.random.seed(10) places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu("group_norm"): @@ -143,9 +159,11 @@ def test_numerical_accuracy(self): bias = np.array([0]).astype("float32") data = np.random.random(shape).astype("float32") expect_res1 = group_norm_naive_for_general_dimension( - data, scale, bias, epsilon=1e-5, groups=6) + data, scale, bias, epsilon=1e-5, groups=6 + ) expect_res2 = group_norm_naive_for_general_dimension( - data, scale, bias, epsilon=1e-5, groups=2) + data, scale, bias, epsilon=1e-5, groups=2 + ) gn1 = paddle.nn.GroupNorm(num_channels=6, num_groups=6) gn2 = paddle.nn.GroupNorm(num_channels=6, num_groups=2) @@ -161,9 +179,7 @@ def test_eager_api(self): class TestGroupNormDimException(unittest.TestCase): - def test_exception(self): - def test_empty_input_static_API(): x = paddle.to_tensor([], dtype='float32') paddle.static.nn.group_norm(x, 3) @@ -171,7 +187,7 @@ def test_empty_input_static_API(): self.assertRaises(ValueError, test_empty_input_static_API) def test_one_dim_input_static_API(): - x = paddle.randn((3, ), dtype='float32') + x = paddle.randn((3,), dtype='float32') paddle.static.nn.group_norm(x, 3) self.assertRaises(ValueError, test_one_dim_input_static_API) diff --git a/python/paddle/fluid/tests/unittests/test_gru_op.py b/python/paddle/fluid/tests/unittests/test_gru_op.py index 4e51c1da0e8b2..d2753dfd205f8 100644 --- a/python/paddle/fluid/tests/unittests/test_gru_op.py +++ b/python/paddle/fluid/tests/unittests/test_gru_op.py @@ -22,17 +22,17 @@ def gru( - input, # T x 3D - lod, # 1 x N - h0, # N x D - weight, # D x 3D - bias, # 1 x 3D - is_reverse, - act_state, - act_gate, - dtype='float32', - origin_mode=False): - + input, # T x 3D + lod, # 1 x N + h0, # N x D + weight, # D x 3D + bias, # 1 x 3D + is_reverse, + act_state, + act_gate, + dtype='float32', + origin_mode=False, +): def _seq_to_batch(lod, is_reverse): idx_in_seq_list = [] seq_lens = lod[0] @@ -41,16 +41,19 @@ def _seq_to_batch(lod, is_reverse): seq_starts.append(seq_starts[-1] + seq_lens[i]) sorted_seqs = sorted( list(range(len(seq_lens))), - key=functools.cmp_to_key(lambda x, y: seq_lens[y] - seq_lens[x])) + key=functools.cmp_to_key(lambda x, y: seq_lens[y] - seq_lens[x]), + ) num_batch = seq_lens[sorted_seqs[0]] for batch_idx in range(num_batch): idx_in_seq = [] for i in range(len(seq_lens)): if seq_lens[sorted_seqs[i]] <= batch_idx: break - idx = (seq_starts[sorted_seqs[i] + 1] - 1 - - batch_idx) if is_reverse else ( - seq_starts[sorted_seqs[i]] + batch_idx) + idx = ( + (seq_starts[sorted_seqs[i] + 1] - 1 - batch_idx) + if is_reverse + else (seq_starts[sorted_seqs[i]] + batch_idx) + ) idx_in_seq.append(idx) idx_in_seq_list.append(idx_in_seq) return idx_in_seq_list, sorted_seqs @@ -59,13 +62,13 @@ def _step(x, h_p, w, b, act_state, act_gate): T = x.shape[0] D = w.shape[0] g = x + np.tile(b, (T, 1)) - w_u_r = w.flatten()[:D * D * 2].reshape((D, D * 2)) - u_r = act_gate(np.dot(h_p, w_u_r) + g[:, :D * 2]) + w_u_r = w.flatten()[: D * D * 2].reshape((D, D * 2)) + u_r = act_gate(np.dot(h_p, w_u_r) + g[:, : D * 2]) u = u_r[:, :D] - r = u_r[:, D:D * 2] + r = u_r[:, D : D * 2] r_h_p = r * h_p - w_c = w.flatten()[D * D * 2:].reshape((D, D)) - c = act_state(np.dot(r_h_p, w_c) + g[:, D * 2:]) + w_c = w.flatten()[D * D * 2 :].reshape((D, D)) + c = act_state(np.dot(r_h_p, w_c) + g[:, D * 2 :]) g = np.hstack((u_r, c)) if origin_mode: h = (1 - u) * c + u * h_p @@ -90,7 +93,7 @@ def _step(x, h_p, w, b, act_state, act_gate): x = input[idx_in_seq_list[batch_idx]] g, r_h_p, h = _step(x, h_p, weight, bias, act_state, act_gate) if batch_idx < (max_seq_len - 1): - h_p = h[:len(idx_in_seq_list[batch_idx + 1])] + h_p = h[: len(idx_in_seq_list[batch_idx + 1])] start_idx = end_idx end_idx = start_idx + len(idx_in_seq_list[batch_idx]) batch_gate[start_idx:end_idx] = g @@ -101,7 +104,6 @@ def _step(x, h_p, w, b, act_state, act_gate): class TestGRUOp(OpTest): - def set_confs(self): pass @@ -126,17 +128,29 @@ def setUp(self): N = len(self.lod[0]) input = np.random.rand(T, 3 * self.D).astype(self.dtype) weight = np.random.rand(self.D, 3 * self.D).astype(self.dtype) - bias = np.random.rand(1, 3 * self.D).astype( - self.dtype) if self.with_bias else np.zeros( - (1, 3 * self.D), dtype=self.dtype) - h0 = np.random.rand(N, self.D).astype( - self.dtype) if self.with_h0 else np.zeros( - (N, self.D), dtype=self.dtype) + bias = ( + np.random.rand(1, 3 * self.D).astype(self.dtype) + if self.with_bias + else np.zeros((1, 3 * self.D), dtype=self.dtype) + ) + h0 = ( + np.random.rand(N, self.D).astype(self.dtype) + if self.with_h0 + else np.zeros((N, self.D), dtype=self.dtype) + ) batch_gate, batch_reset_hidden_prev, batch_hidden, hidden = gru( - input, self.lod, h0, weight, bias, self.is_reverse, - ACTIVATION[self.act_state], ACTIVATION[self.act_gate], self.dtype, - self.origin_mode) + input, + self.lod, + h0, + weight, + bias, + self.is_reverse, + ACTIVATION[self.act_state], + ACTIVATION[self.act_gate], + self.dtype, + self.origin_mode, + ) self.inputs = {'Input': (input, self.lod), 'Weight': weight} if self.with_bias: @@ -157,45 +171,41 @@ def setUp(self): 'gate_activation': self.act_gate, 'is_reverse': self.is_reverse, 'origin_mode': self.origin_mode, - 'is_test': self.is_test + 'is_test': self.is_test, } def test_check_output(self): self.check_output(atol=1e-8, check_dygraph=False) def test_check_grad(self): - self.check_grad(['Input', 'H0', 'Weight', 'Bias'], ['Hidden'], - check_dygraph=False) + self.check_grad( + ['Input', 'H0', 'Weight', 'Bias'], ['Hidden'], check_dygraph=False + ) class TestGRUOriginMode(TestGRUOp): - def set_confs(self): self.origin_mode = True class TestGRUOp2(TestGRUOp): - def set_confs(self): self.dtype = 'float64' class TestGRUOp2Len0(TestGRUOp): - def set_confs(self): self.lod = [[2, 0, 4]] self.dtype = 'float64' class TestGRUOp2OriginMode(TestGRUOp): - def set_confs(self): self.dtype = 'float64' self.origin_mode = True class TestGRUOp2OriginModeLen0(TestGRUOp): - def set_confs(self): self.lod = [[0, 3, 4]] self.dtype = 'float64' @@ -203,7 +213,6 @@ def set_confs(self): class TestGRUOp2OriginModeLastLen0(TestGRUOp): - def set_confs(self): self.lod = [[0, 3, 0]] self.dtype = 'float64' @@ -211,40 +220,37 @@ def set_confs(self): class TestGRUOpNoInitial(TestGRUOp): - def set_confs(self): self.with_h0 = False def test_check_grad(self): - self.check_grad(['Input', 'Weight', 'Bias'], ['Hidden'], - check_dygraph=False) + self.check_grad( + ['Input', 'Weight', 'Bias'], ['Hidden'], check_dygraph=False + ) class TestGRUOpNoBias(TestGRUOp): - def set_confs(self): self.with_bias = False def test_check_grad(self): - self.check_grad(['Input', 'H0', 'Weight'], ['Hidden'], - check_dygraph=False) + self.check_grad( + ['Input', 'H0', 'Weight'], ['Hidden'], check_dygraph=False + ) class TestGRUOpReverse(TestGRUOp): - def set_confs(self): self.is_reverse = True class TestGRUOpReverseOriginMode(TestGRUOp): - def set_confs(self): self.is_reverse = True self.origin_mode = True class TestGRUOpInference(TestGRUOp): - def set_is_test(self): self.is_test = True @@ -260,7 +266,6 @@ def test_check_grad(self): class TestGruOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -271,9 +276,9 @@ def test_Variable(): self.assertRaises(TypeError, test_Variable) def test_h_0(): - in_data = fluid.data(name="input", - shape=[None, 1536], - dtype="float32") + in_data = fluid.data( + name="input", shape=[None, 1536], dtype="float32" + ) h = fluid.data(name="h", shape=[None, 512], dtype="int32") fluid.layers.dynamic_gru(input=in_data, size=512, h_0=h) diff --git a/python/paddle/fluid/tests/unittests/test_gru_rnn_op.py b/python/paddle/fluid/tests/unittests/test_gru_rnn_op.py index 0a10eb74f5511..89379ac87f349 100644 --- a/python/paddle/fluid/tests/unittests/test_gru_rnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_gru_rnn_op.py @@ -31,7 +31,6 @@ class TestGRUOp(OpTest): - def get_weight_names(self): weight_names = [] for i in range(self.num_layers): @@ -45,13 +44,16 @@ def get_weight_names(self): def setUp(self): self.op_type = "rnn" self.dtype = "float32" if core.is_compiled_with_rocm() else "float64" - self.sequence_length = None if core.is_compiled_with_rocm( - ) else np.array([12, 11, 10, 9, 8, 7, 6, 5], dtype=np.int32) + self.sequence_length = ( + None + if core.is_compiled_with_rocm() + else np.array([12, 11, 10, 9, 8, 7, 6, 5], dtype=np.int32) + ) self.num_layers = 1 self.is_bidirec = False self.is_test = False self.mode = "GRU" - self.dropout = 0. + self.dropout = 0.0 seq_length = 12 batch_size = 8 input_size = 4 @@ -61,10 +63,9 @@ def setUp(self): self.direction_num = 2 if self.is_bidirec else 1 direction = "bidirectional" if self.is_bidirec else "forward" - input = np.random.uniform(low=-0.1, - high=0.1, - size=(seq_length, batch_size, - input_size)).astype(self.dtype) + input = np.random.uniform( + low=-0.1, high=0.1, size=(seq_length, batch_size, input_size) + ).astype(self.dtype) if self.sequence_length is not None: input[3][1:][:] = 0 @@ -72,13 +73,15 @@ def setUp(self): input[2][3:][:] = 0 input[1][4:][:] = 0 - rnn1 = GRU(input_size, - self.hidden_size, - num_layers=self.num_layers, - time_major=True, - direction=direction, - dropout=self.dropout, - dtype=self.dtype) + rnn1 = GRU( + input_size, + self.hidden_size, + num_layers=self.num_layers, + time_major=True, + direction=direction, + dropout=self.dropout, + dtype=self.dtype, + ) flat_w = get_params_for_net(rnn1) @@ -92,8 +95,9 @@ def rocm_rnn_get_place(): self._get_places = rocm_rnn_get_place - init_h = np.zeros((self.num_layers * self.direction_num, batch_size, - self.hidden_size)).astype(self.dtype) + init_h = np.zeros( + (self.num_layers * self.direction_num, batch_size, self.hidden_size) + ).astype(self.dtype) state_out = np.ndarray((300)).astype("uint8") @@ -101,7 +105,7 @@ def rocm_rnn_get_place(): 'Input': input, 'WeightList': flat_w, 'PreState': [('init_h', init_h)], - 'SequenceLength': self.sequence_length + 'SequenceLength': self.sequence_length, } if self.sequence_length is None: self.inputs = { @@ -116,13 +120,13 @@ def rocm_rnn_get_place(): 'hidden_size': self.hidden_size, 'num_layers': self.num_layers, 'is_test': self.is_test, - 'mode': self.mode + 'mode': self.mode, } self.outputs = { 'Out': output, 'State': [('last_hidden', last_hidden)], 'Reserve': np.ndarray((400)).astype("uint8"), - 'DropoutState': state_out + 'DropoutState': state_out, } def set_attrs(self): @@ -140,27 +144,23 @@ def test_grad(self): class TestGRUOp1(TestGRUOp): - def set_attrs(self): self.sequence_length = None class TestGRUOp2(TestGRUOp): - def set_attrs(self): self.sequence_length = None self.is_bidirec = True class TestGRUOp3(TestGRUOp): - def set_attrs(self): self.sequence_length = None self.is_test = True class TestGRUOp4(TestGRUOp): - def set_attrs(self): self.sequence_length = None self.is_bidirec = True @@ -168,7 +168,6 @@ def set_attrs(self): class TestGRUOpAvx(TestGRUOp): - def set_attrs(self): self.dtype = "float32" self.hidden_size = 8 diff --git a/python/paddle/fluid/tests/unittests/test_gru_unit_op.py b/python/paddle/fluid/tests/unittests/test_gru_unit_op.py index 96c277416107a..7111e4f8b9cd3 100644 --- a/python/paddle/fluid/tests/unittests/test_gru_unit_op.py +++ b/python/paddle/fluid/tests/unittests/test_gru_unit_op.py @@ -23,14 +23,14 @@ class TestGRUUnitAPIError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): D = 5 layer = fluid.dygraph.nn.GRUUnit(size=D * 3) # the input must be Variable. - x0 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x0 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, layer, x0) # the input dtype must be float32 or float64 x = fluid.data(name='x', shape=[-1, D * 3], dtype='float16') @@ -50,11 +50,11 @@ def identity(x): def sigmoid(x): - return 1. / (1. + np.exp(-x)) + return 1.0 / (1.0 + np.exp(-x)) def tanh(x): - return 2. * sigmoid(2. * x) - 1. + return 2.0 * sigmoid(2.0 * x) - 1.0 def relu(x): @@ -62,21 +62,22 @@ def relu(x): class TestGRUUnitOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_size = 5 hidden_dim = 40 - input = fluid.data(name='input', - shape=[None, hidden_dim * 3], - dtype='float32') - pre_hidden = fluid.data(name='pre_hidden', - shape=[None, hidden_dim], - dtype='float32') + input = fluid.data( + name='input', shape=[None, hidden_dim * 3], dtype='float32' + ) + pre_hidden = fluid.data( + name='pre_hidden', shape=[None, hidden_dim], dtype='float32' + ) np_input = np.random.uniform( - -0.1, 0.1, (batch_size, hidden_dim * 3)).astype('float64') + -0.1, 0.1, (batch_size, hidden_dim * 3) + ).astype('float64') np_pre_hidden = np.random.uniform( - -0.1, 0.1, (batch_size, hidden_dim)).astype('float64') + -0.1, 0.1, (batch_size, hidden_dim) + ).astype('float64') def test_input_Variable(): gru_unit(np_input, pre_hidden, hidden_dim * 3) @@ -89,17 +90,21 @@ def test_pre_hidden_Variable(): self.assertRaises(TypeError, test_pre_hidden_Variable) def test_input_type(): - error_input = fluid.data(name='error_input', - shape=[None, hidden_dim * 3], - dtype='int32') + error_input = fluid.data( + name='error_input', + shape=[None, hidden_dim * 3], + dtype='int32', + ) gru_unit(error_input, pre_hidden, hidden_dim * 3) self.assertRaises(TypeError, test_input_type) def test_pre_hidden_type(): - error_pre_hidden = fluid.data(name='error_pre_hidden', - shape=[None, hidden_dim], - dtype='int32') + error_pre_hidden = fluid.data( + name='error_pre_hidden', + shape=[None, hidden_dim], + dtype='int32', + ) gru_unit(input, error_pre_hidden, hidden_dim * 3) self.assertRaises(TypeError, test_pre_hidden_type) @@ -120,21 +125,22 @@ def set_inputs(self, origin_mode=False): frame_size = self.frame_size self.op_type = 'gru_unit' self.inputs = { - 'Input': - np.random.uniform(-0.1, 0.1, - (batch_size, frame_size * 3)).astype(self.dtype), - 'HiddenPrev': - np.random.uniform(-0.1, 0.1, - (batch_size, frame_size)).astype(self.dtype), - 'Weight': - np.random.uniform(-1. / math.sqrt(frame_size), - 1. / math.sqrt(frame_size), - (frame_size, frame_size * 3)).astype(self.dtype), + 'Input': np.random.uniform( + -0.1, 0.1, (batch_size, frame_size * 3) + ).astype(self.dtype), + 'HiddenPrev': np.random.uniform( + -0.1, 0.1, (batch_size, frame_size) + ).astype(self.dtype), + 'Weight': np.random.uniform( + -1.0 / math.sqrt(frame_size), + 1.0 / math.sqrt(frame_size), + (frame_size, frame_size * 3), + ).astype(self.dtype), } self.attrs = { 'activation': GRUActivationType.tanh, 'gate_activation': GRUActivationType.sigmoid, - 'origin_mode': origin_mode + 'origin_mode': origin_mode, } def set_outputs(self, origin_mode=False): @@ -144,21 +150,27 @@ def set_outputs(self, origin_mode=False): x = self.inputs['Input'] h_p = self.inputs['HiddenPrev'] w = self.inputs['Weight'] - b = self.inputs['Bias'] if 'Bias' in self.inputs else np.zeros( - (1, frame_size * 3)) + b = ( + self.inputs['Bias'] + if 'Bias' in self.inputs + else np.zeros((1, frame_size * 3)) + ) g = x + np.tile(b, (batch_size, 1)) - w_u_r = w.flatten()[:frame_size * frame_size * 2].reshape( - (frame_size, frame_size * 2)) - u_r = self.activate[self.attrs['gate_activation']](np.dot(h_p, w_u_r) + - g[:, :frame_size * - 2]) + w_u_r = w.flatten()[: frame_size * frame_size * 2].reshape( + (frame_size, frame_size * 2) + ) + u_r = self.activate[self.attrs['gate_activation']]( + np.dot(h_p, w_u_r) + g[:, : frame_size * 2] + ) u = u_r[:, :frame_size] - r = u_r[:, frame_size:frame_size * 2] + r = u_r[:, frame_size : frame_size * 2] r_h_p = r * h_p - w_c = w.flatten()[frame_size * frame_size * 2:].reshape( - (frame_size, frame_size)) - c = self.activate[self.attrs['activation']](np.dot(r_h_p, w_c) + - g[:, frame_size * 2:]) + w_c = w.flatten()[frame_size * frame_size * 2 :].reshape( + (frame_size, frame_size) + ) + c = self.activate[self.attrs['activation']]( + np.dot(r_h_p, w_c) + g[:, frame_size * 2 :] + ) g = np.hstack((u_r, c)) if origin_mode: h = (1 - u) * c + u * h_p @@ -167,12 +179,13 @@ def set_outputs(self, origin_mode=False): self.outputs = { 'Gate': g.astype(self.dtype), 'ResetHiddenPrev': r_h_p.astype(self.dtype), - 'Hidden': h.astype(self.dtype) + 'Hidden': h.astype(self.dtype), } def setUp(self): - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.set_inputs() self.set_outputs() @@ -184,41 +197,44 @@ def test_check_grad(self): class TestGRUUnitOpOriginMode(TestGRUUnitOp): - def setUp(self): - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.set_inputs(origin_mode=True) self.set_outputs(origin_mode=True) class TestGRUUnitOpWithBias(TestGRUUnitOp): - def set_inputs(self, origin_mode=False): batch_size = self.batch_size frame_size = self.frame_size super(TestGRUUnitOpWithBias, self).set_inputs() self.inputs['Bias'] = np.random.uniform( - -0.1, 0.1, (1, frame_size * 3)).astype(self.dtype) + -0.1, 0.1, (1, frame_size * 3) + ).astype(self.dtype) self.attrs = { 'activation': GRUActivationType.identity, 'gate_activation': GRUActivationType.sigmoid, - 'origin_mode': origin_mode + 'origin_mode': origin_mode, } def test_check_grad(self): self.check_grad(['Input', 'HiddenPrev', 'Weight', 'Bias'], ['Hidden']) def test_check_grad_ingore_input(self): - self.check_grad(['HiddenPrev', 'Weight', 'Bias'], ['Hidden'], - no_grad_set=set('Input')) + self.check_grad( + ['HiddenPrev', 'Weight', 'Bias'], + ['Hidden'], + no_grad_set=set('Input'), + ) class TestGRUUnitOpWithBiasOriginMode(TestGRUUnitOpWithBias): - def setUp(self): - self.dtype = 'float32' if fluid.core.is_compiled_with_rocm( - ) else 'float64' + self.dtype = ( + 'float32' if fluid.core.is_compiled_with_rocm() else 'float64' + ) self.set_inputs(origin_mode=True) self.set_outputs(origin_mode=True) diff --git a/python/paddle/fluid/tests/unittests/test_gumbel_softmax_op.py b/python/paddle/fluid/tests/unittests/test_gumbel_softmax_op.py index 90de83fce248f..de7f5d4635236 100644 --- a/python/paddle/fluid/tests/unittests/test_gumbel_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/test_gumbel_softmax_op.py @@ -21,7 +21,6 @@ class TestGumbelSoftmaxOp(OpTest): - def init_attrs(self): self.shape = [20, 10] self.attrs = {"hard": True, "axis": -1} @@ -51,7 +50,6 @@ def test_check_grad(self): class TestGumbelSoftmaxOp2(TestGumbelSoftmaxOp): - def init_attrs(self): self.shape = [20, 10] self.attrs = {"hard": True, "axis": 0} @@ -60,7 +58,6 @@ def init_attrs(self): class TestGumbelSoftmaxOp3(TestGumbelSoftmaxOp): - def init_attrs(self): self.shape = [100] self.attrs = {"hard": True, "axis": -1} @@ -69,7 +66,6 @@ def init_attrs(self): class TestGumbelSoftmaxOp4(TestGumbelSoftmaxOp): - def init_attrs(self): self.shape = [20, 10, 5] self.attrs = {"hard": True, "axis": -1} @@ -78,7 +74,6 @@ def init_attrs(self): class TestGumbelSoftmaxOp5(TestGumbelSoftmaxOp): - def init_attrs(self): self.shape = [20, 10, 5] self.attrs = {"hard": True, "axis": 1} @@ -87,7 +82,6 @@ def init_attrs(self): class TestGumbelSoftmaxOpSampleDistribution(OpTest): - def softmax(self, x): x_row_max = x.max(axis=-1) x_row_max = x_row_max.reshape(list(x.shape)[:-1] + [1]) @@ -139,7 +133,6 @@ def test_check_grad(self): class TestGumbelSoftmaxOpGrad(unittest.TestCase): - def init_attrs(self): self.shape = [20, 10] self.dtype = "float64" @@ -159,22 +152,22 @@ def test_dygraph_check(self): out_hard.sum().backward() out_soft.sum().backward() - np.testing.assert_allclose(x_hard.grad.numpy(), - x_soft.grad.numpy(), - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + x_hard.grad.numpy(), x_soft.grad.numpy(), rtol=1e-5, atol=1e-8 + ) paddle.enable_static() class TestGumbelSoftmaxAPI(unittest.TestCase): - def setUp(self): self.x_shape = [2, 3, 4, 5] - self.x = np.random.uniform(-1., 1., self.x_shape).astype(np.float32) + self.x = np.random.uniform(-1.0, 1.0, self.x_shape).astype(np.float32) self.count_expected = 24 - self.place = paddle.CUDAPlace(0) \ - if paddle.fluid.core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_check_api(self): # test static api @@ -201,13 +194,13 @@ def test_check_api(self): class TestGumbelSoftmaxOpError(unittest.TestCase): - def test_errors(self): paddle.disable_static() def test_Variable(): - x1 = fluid.create_lod_tensor(np.zeros((100, 784)), - [[10, 10, 10, 70]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.zeros((100, 784)), [[10, 10, 10, 70]], fluid.CPUPlace() + ) paddle.nn.functional.gumbel_softmax(x1) self.assertRaises(ValueError, test_Variable) @@ -234,9 +227,9 @@ def test_argument2(): def test_dtype(): with paddle.static.program_guard(paddle.static.Program()): - x_int32 = paddle.fluid.data(name='x_int32', - shape=[2, 3], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[2, 3], dtype='int32' + ) paddle.nn.functional.gumbel_softmax(x_int32) self.assertRaises(TypeError, test_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_hash_op.py b/python/paddle/fluid/tests/unittests/test_hash_op.py index fe0762909488a..0070a3fadc75a 100644 --- a/python/paddle/fluid/tests/unittests/test_hash_op.py +++ b/python/paddle/fluid/tests/unittests/test_hash_op.py @@ -19,7 +19,6 @@ class TestHashOp(OpTest): - def setUp(self): self.op_type = "hash" self.init_test_case() @@ -31,9 +30,16 @@ def init_test_case(self): np.random.seed(1) self.in_seq = np.random.randint(0, 10, (8, 1)).astype("int32") self.lod = [[2, 6]] - self.out_seq = [[[3481], [7475]], [[1719], [5986]], [[8473], [694]], - [[3481], [7475]], [[4372], [9456]], [[4372], [9456]], - [[6897], [3218]], [[9038], [7951]]] + self.out_seq = [ + [[3481], [7475]], + [[1719], [5986]], + [[8473], [694]], + [[3481], [7475]], + [[4372], [9456]], + [[4372], [9456]], + [[6897], [3218]], + [[9038], [7951]], + ] self.out_seq = np.array(self.out_seq) def test_check_output(self): @@ -41,7 +47,6 @@ def test_check_output(self): class TestHashNotLoDOp(TestHashOp): - def setUp(self): self.op_type = "hash" self.init_test_case() @@ -52,9 +57,16 @@ def setUp(self): def init_test_case(self): np.random.seed(1) self.in_seq = np.random.randint(0, 10, (8, 1)).astype("int32") - self.out_seq = [[[3481], [7475]], [[1719], [5986]], [[8473], [694]], - [[3481], [7475]], [[4372], [9456]], [[4372], [9456]], - [[6897], [3218]], [[9038], [7951]]] + self.out_seq = [ + [[3481], [7475]], + [[1719], [5986]], + [[8473], [694]], + [[3481], [7475]], + [[4372], [9456]], + [[4372], [9456]], + [[6897], [3218]], + [[9038], [7951]], + ] self.out_seq = np.array(self.out_seq) def test_check_output(self): @@ -98,15 +110,15 @@ def setUp(self): def init_test_case(self): self.in_seq = np.array([10, 5]).reshape((2, 1)).astype("int64") - self.out_seq = np.array([1204014882, 393011615, 3586283837, - 2814821595]).reshape((2, 2, 1)) + self.out_seq = np.array( + [1204014882, 393011615, 3586283837, 2814821595] + ).reshape((2, 2, 1)) def test_check_output(self): self.check_output() class TestHashOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): input_data = np.random.randint(0, 10, (8, 1)).astype("int32") @@ -119,30 +131,27 @@ def test_Variable(): def test_type(): # dtype must be int32, int64. - x2 = fluid.layers.data(name='x2', - shape=[1], - dtype="float32", - lod_level=1) + x2 = fluid.layers.data( + name='x2', shape=[1], dtype="float32", lod_level=1 + ) fluid.layers.hash(input=x2, hash_size=2**32) self.assertRaises(TypeError, test_type) def test_hash_size_type(): # hash_size dtype must be int32, int64. - x3 = fluid.layers.data(name='x3', - shape=[1], - dtype="int32", - lod_level=1) + x3 = fluid.layers.data( + name='x3', shape=[1], dtype="int32", lod_level=1 + ) fluid.layers.hash(input=x3, hash_size=1024.5) self.assertRaises(TypeError, test_hash_size_type) def test_num_hash_type(): # num_hash dtype must be int32, int64. - x4 = fluid.layers.data(name='x4', - shape=[1], - dtype="int32", - lod_level=1) + x4 = fluid.layers.data( + name='x4', shape=[1], dtype="int32", lod_level=1 + ) fluid.layers.hash(input=x4, hash_size=2**32, num_hash=2.5) self.assertRaises(TypeError, test_num_hash_type) diff --git a/python/paddle/fluid/tests/unittests/test_hinge_embedding_loss.py b/python/paddle/fluid/tests/unittests/test_hinge_embedding_loss.py index 8088ca7c363c2..68e7b4210a692 100644 --- a/python/paddle/fluid/tests/unittests/test_hinge_embedding_loss.py +++ b/python/paddle/fluid/tests/unittests/test_hinge_embedding_loss.py @@ -21,8 +21,9 @@ def calc_hinge_embedding_loss(input, label, margin=1.0, reduction='mean'): - result = np.where(label == -1., np.maximum(0., margin - input), 0.) + \ - np.where(label == 1., input, 0.) + result = np.where( + label == -1.0, np.maximum(0.0, margin - input), 0.0 + ) + np.where(label == 1.0, input, 0.0) if reduction == 'none': return result elif reduction == 'sum': @@ -32,13 +33,12 @@ def calc_hinge_embedding_loss(input, label, margin=1.0, reduction='mean'): class TestFunctionalHingeEmbeddingLoss(unittest.TestCase): - def setUp(self): self.margin = 1.0 self.shape = (10, 10, 5) self.input_np = np.random.random(size=self.shape).astype(np.float64) # get label elem in {1., -1.} - self.label_np = 2 * np.random.randint(0, 2, size=self.shape) - 1. + self.label_np = 2 * np.random.randint(0, 2, size=self.shape) - 1.0 def run_dynamic_check(self, place=paddle.CPUPlace()): paddle.disable_static(place=place) @@ -50,45 +50,45 @@ def run_dynamic_check(self, place=paddle.CPUPlace()): np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05) self.assertTrue(dy_result.shape, [1]) - dy_result = paddle.nn.functional.hinge_embedding_loss(input, - label, - reduction='sum') - expected = calc_hinge_embedding_loss(self.input_np, - self.label_np, - reduction='sum') + dy_result = paddle.nn.functional.hinge_embedding_loss( + input, label, reduction='sum' + ) + expected = calc_hinge_embedding_loss( + self.input_np, self.label_np, reduction='sum' + ) np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05) self.assertTrue(dy_result.shape, [1]) - dy_result = paddle.nn.functional.hinge_embedding_loss(input, - label, - reduction='none') - expected = calc_hinge_embedding_loss(self.input_np, - self.label_np, - reduction='none') + dy_result = paddle.nn.functional.hinge_embedding_loss( + input, label, reduction='none' + ) + expected = calc_hinge_embedding_loss( + self.input_np, self.label_np, reduction='none' + ) np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05) self.assertTrue(dy_result.shape, self.shape) def run_static_check(self, place=paddle.CPUPlace): paddle.enable_static() for reduction in ['none', 'mean', 'sum']: - expected = calc_hinge_embedding_loss(self.input_np, - self.label_np, - reduction=reduction) + expected = calc_hinge_embedding_loss( + self.input_np, self.label_np, reduction=reduction + ) with program_guard(Program(), Program()): - input = paddle.static.data(name="input", - shape=self.shape, - dtype=paddle.float64) - label = paddle.static.data(name="label", - shape=self.shape, - dtype=paddle.float64) + input = paddle.static.data( + name="input", shape=self.shape, dtype=paddle.float64 + ) + label = paddle.static.data( + name="label", shape=self.shape, dtype=paddle.float64 + ) st_result = paddle.nn.functional.hinge_embedding_loss( - input, label, reduction=reduction) + input, label, reduction=reduction + ) exe = paddle.static.Executor(place) - result_numpy, = exe.run(feed={ - "input": self.input_np, - "label": self.label_np - }, - fetch_list=[st_result]) + (result_numpy,) = exe.run( + feed={"input": self.input_np, "label": self.label_np}, + fetch_list=[st_result], + ) np.testing.assert_allclose(result_numpy, expected, rtol=1e-05) def test_cpu(self): @@ -103,22 +103,21 @@ def test_gpu(self): # test case the raise message def test_reduce_errors(self): - def test_value_error(): loss = paddle.nn.functional.hinge_embedding_loss( - self.input_np, self.label_np, reduction='reduce_mean') + self.input_np, self.label_np, reduction='reduce_mean' + ) self.assertRaises(ValueError, test_value_error) class TestClassHingeEmbeddingLoss(unittest.TestCase): - def setUp(self): self.margin = 1.0 self.shape = (10, 10, 5) self.input_np = np.random.random(size=self.shape).astype(np.float64) # get label elem in {1., -1.} - self.label_np = 2 * np.random.randint(0, 2, size=self.shape) - 1. + self.label_np = 2 * np.random.randint(0, 2, size=self.shape) - 1.0 def run_dynamic_check(self, place=paddle.CPUPlace()): paddle.disable_static(place=place) @@ -131,45 +130,47 @@ def run_dynamic_check(self, place=paddle.CPUPlace()): self.assertTrue(dy_result.shape, [1]) hinge_embedding_loss = paddle.nn.loss.HingeEmbeddingLoss( - reduction='sum') + reduction='sum' + ) dy_result = hinge_embedding_loss(input, label) - expected = calc_hinge_embedding_loss(self.input_np, - self.label_np, - reduction='sum') + expected = calc_hinge_embedding_loss( + self.input_np, self.label_np, reduction='sum' + ) np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05) self.assertTrue(dy_result.shape, [1]) hinge_embedding_loss = paddle.nn.loss.HingeEmbeddingLoss( - reduction='none') + reduction='none' + ) dy_result = hinge_embedding_loss(input, label) - expected = calc_hinge_embedding_loss(self.input_np, - self.label_np, - reduction='none') + expected = calc_hinge_embedding_loss( + self.input_np, self.label_np, reduction='none' + ) np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05) self.assertTrue(dy_result.shape, self.shape) def run_static_check(self, place=paddle.CPUPlace): paddle.enable_static() for reduction in ['none', 'mean', 'sum']: - expected = calc_hinge_embedding_loss(self.input_np, - self.label_np, - reduction=reduction) + expected = calc_hinge_embedding_loss( + self.input_np, self.label_np, reduction=reduction + ) with program_guard(Program(), Program()): - input = paddle.static.data(name="input", - shape=self.shape, - dtype=paddle.float64) - label = paddle.static.data(name="label", - shape=self.shape, - dtype=paddle.float64) + input = paddle.static.data( + name="input", shape=self.shape, dtype=paddle.float64 + ) + label = paddle.static.data( + name="label", shape=self.shape, dtype=paddle.float64 + ) hinge_embedding_loss = paddle.nn.loss.HingeEmbeddingLoss( - reduction=reduction) + reduction=reduction + ) st_result = hinge_embedding_loss(input, label) exe = paddle.static.Executor(place) - result_numpy, = exe.run(feed={ - "input": self.input_np, - "label": self.label_np - }, - fetch_list=[st_result]) + (result_numpy,) = exe.run( + feed={"input": self.input_np, "label": self.label_np}, + fetch_list=[st_result], + ) np.testing.assert_allclose(result_numpy, expected, rtol=1e-05) def test_cpu(self): @@ -184,10 +185,10 @@ def test_gpu(self): # test case the raise message def test_reduce_errors(self): - def test_value_error(): hinge_embedding_loss = paddle.nn.loss.HingeEmbeddingLoss( - reduction='reduce_mean') + reduction='reduce_mean' + ) loss = hinge_embedding_loss(self.input_np, self.label_np) self.assertRaises(ValueError, test_value_error) diff --git a/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py b/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py index c2fbd5d915009..1b320f864491a 100644 --- a/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py @@ -18,7 +18,6 @@ class TestHingeLossOp(OpTest): - def setUp(self): self.op_type = 'hinge_loss' samples_num = 100 diff --git a/python/paddle/fluid/tests/unittests/test_histogram_op.py b/python/paddle/fluid/tests/unittests/test_histogram_op.py index 725b6c0236201..516bc53a34aca 100644 --- a/python/paddle/fluid/tests/unittests/test_histogram_op.py +++ b/python/paddle/fluid/tests/unittests/test_histogram_op.py @@ -36,14 +36,15 @@ def test_static_graph(self): exe = fluid.Executor(place) exe.run(startup_program) img = np.array([[2, 4, 2], [2, 5, 4]]).astype(np.int64) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) actual = np.array(res[0]) expected = np.array([0, 3, 0, 2, 1]).astype(np.int64) self.assertTrue( (actual == expected).all(), - msg='histogram output is wrong, out =' + str(actual)) + msg='histogram output is wrong, out =' + str(actual), + ) def test_dygraph(self): with fluid.dygraph.guard(): @@ -53,15 +54,18 @@ def test_dygraph(self): expected = np.array([0, 3, 0, 2, 1]).astype(np.int64) self.assertTrue( (actual.numpy() == expected).all(), - msg='histogram output is wrong, out =' + str(actual.numpy())) + msg='histogram output is wrong, out =' + str(actual.numpy()), + ) with _test_eager_guard(): inputs_np = np.array([[2, 4, 2], [2, 5, 4]]).astype(np.int64) inputs = paddle.to_tensor(inputs_np) actual = paddle.histogram(inputs, bins=5, min=1, max=5) - self.assertTrue((actual.numpy() == expected).all(), - msg='histogram output is wrong, out =' + - str(actual.numpy())) + self.assertTrue( + (actual.numpy() == expected).all(), + msg='histogram output is wrong, out =' + + str(actual.numpy()), + ) class TestHistogramOpError(unittest.TestCase): @@ -79,9 +83,9 @@ def test_bins_error(self): """Test bins should be greater than or equal to 1.""" def net_func(): - input_value = paddle.fluid.layers.fill_constant(shape=[3, 4], - dtype='float32', - value=3.0) + input_value = paddle.fluid.layers.fill_constant( + shape=[3, 4], dtype='float32', value=3.0 + ) paddle.histogram(input=input_value, bins=-1, min=1, max=5) with self.assertRaises(IndexError): @@ -91,9 +95,9 @@ def test_min_max_error(self): """Test max must be larger or equal to min.""" def net_func(): - input_value = paddle.fluid.layers.fill_constant(shape=[3, 4], - dtype='float32', - value=3.0) + input_value = paddle.fluid.layers.fill_constant( + shape=[3, 4], dtype='float32', value=3.0 + ) paddle.histogram(input=input_value, bins=1, min=5, max=1) with self.assertRaises(ValueError): @@ -103,9 +107,9 @@ def test_min_max_range_error(self): """Test range of min, max is not finite""" def net_func(): - input_value = paddle.fluid.layers.fill_constant(shape=[3, 4], - dtype='float32', - value=3.0) + input_value = paddle.fluid.layers.fill_constant( + shape=[3, 4], dtype='float32', value=3.0 + ) paddle.histogram(input=input_value, bins=1, min=-np.inf, max=5) with self.assertRaises(TypeError): @@ -114,24 +118,17 @@ def net_func(): def test_type_errors(self): with program_guard(Program()): # The input type must be Variable. - self.assertRaises(TypeError, - paddle.histogram, - 1, - bins=5, - min=1, - max=5) + self.assertRaises( + TypeError, paddle.histogram, 1, bins=5, min=1, max=5 + ) # The input type must be 'int32', 'int64', 'float32', 'float64' x_bool = fluid.data(name='x_bool', shape=[4, 3], dtype='bool') - self.assertRaises(TypeError, - paddle.histogram, - x_bool, - bins=5, - min=1, - max=5) + self.assertRaises( + TypeError, paddle.histogram, x_bool, bins=5, min=1, max=5 + ) class TestHistogramOp(OpTest): - def setUp(self): self.op_type = "histogram" self.init_test_case() @@ -139,9 +136,9 @@ def setUp(self): self.python_api = paddle.histogram self.inputs = {"X": np_input} self.init_attrs() - Out, _ = np.histogram(np_input, - bins=self.bins, - range=(self.min, self.max)) + Out, _ = np.histogram( + np_input, bins=self.bins, range=(self.min, self.max) + ) self.outputs = {"Out": Out.astype(np.int64)} def init_test_case(self): diff --git a/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py b/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py index 66cb6ac427a54..a905b9c2ae024 100644 --- a/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py +++ b/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py @@ -31,7 +31,6 @@ def find_latest_set(num): class CodeTable(object): - def __init__(self, num_classes, code): self.c = num_classes + code @@ -46,7 +45,6 @@ def cal_bit(self, bit): class CodeTableWithCustomTree(object): - def __init__(self, path_table, path_code, index): self.ptable_ = path_table self.pcode_ = path_code @@ -129,8 +127,9 @@ def hsigmoid_grad(x, w, label, bias, num_classes): return [dx, dw, db] -def hsigmoidWithCustomTree(x, w, path_table, path_code, label, bias, - num_classes): +def hsigmoidWithCustomTree( + x, w, path_table, path_code, label, bias, num_classes +): batch_size = x.shape[0] code_length = len(path_table[0]) code_table = [0 for _ in range(code_length)] @@ -169,26 +168,36 @@ def hsigmoidWithCustomTree(x, w, path_table, path_code, label, bias, return pre_output, out -def python_api(input, - weight, - label, - path_table=None, - path_code=None, - bias=None, - num_classes=-1, - is_sparse=False, - remote_prefetch=False): - assert is_sparse == remote_prefetch, "is_sparse is equal to remote_prefetch in dygraph." - return paddle.nn.functional.hsigmoid_loss(input, label, num_classes, weight, - bias, path_table, path_code, - is_sparse) +def python_api( + input, + weight, + label, + path_table=None, + path_code=None, + bias=None, + num_classes=-1, + is_sparse=False, + remote_prefetch=False, +): + assert ( + is_sparse == remote_prefetch + ), "is_sparse is equal to remote_prefetch in dygraph." + return paddle.nn.functional.hsigmoid_loss( + input, + label, + num_classes, + weight, + bias, + path_table, + path_code, + is_sparse, + ) python_out_sig = ["Out"] class TestHSigmoidOp(OpTest): - def setUp(self): self.op_type = "hierarchical_sigmoid" self.python_api = python_api @@ -196,12 +205,15 @@ def setUp(self): num_classes = 101 feature_size = 5 batch_size = 20 - x = np.random.uniform(-1, 1, - (batch_size, feature_size)).astype('float64') - w = np.random.uniform(-1, 1, - (num_classes - 1, feature_size)).astype('float64') - label = np.random.randint(0, num_classes, - (batch_size, 1)).astype('int64') + x = np.random.uniform(-1, 1, (batch_size, feature_size)).astype( + 'float64' + ) + w = np.random.uniform(-1, 1, (num_classes - 1, feature_size)).astype( + 'float64' + ) + label = np.random.randint(0, num_classes, (batch_size, 1)).astype( + 'int64' + ) bias = np.random.uniform(-1, 1, (num_classes - 1, 1)).astype('float64') self.attrs = {'num_classes': num_classes, 'is_sparse': False} self.inputs = {'X': x, 'W': w, 'Label': label, 'Bias': bias} @@ -213,35 +225,48 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X', 'W', 'Bias'], ['Out'], - user_defined_grads=self.user_grads, - check_eager=True) + self.check_grad( + ['X', 'W', 'Bias'], + ['Out'], + user_defined_grads=self.user_grads, + check_eager=True, + ) @skip_check_grad_ci( - reason= - "For 'TestHSigmoidOpSparse', check_grad is separately calculated by 'TestHSigmoidOpWithSparseGrad'." + reason="For 'TestHSigmoidOpSparse', check_grad is separately calculated by 'TestHSigmoidOpWithSparseGrad'." ) class TestHSigmoidOpSparse(OpTest): - def setUp(self): self.op_type = "hierarchical_sigmoid" self.python_api = python_api self.python_out_sig = python_out_sig - num_classes = 6 #using 1,2,3,4,5,6 to build a huffman tree and select 1,2,5,6 as sample + num_classes = 6 # using 1,2,3,4,5,6 to build a huffman tree and select 1,2,5,6 as sample feature_size = 8 batch_size = 4 x = np.random.random((batch_size, feature_size)) w = np.random.random((num_classes - 1, feature_size)) label = np.array([0, 1, 4, 5]).astype('int64') - path_table = np.array([ - (0, 2, -1, -1, -1), (0, 1, 3, -1, -1), (0, 1, 4, -1, -1), - (0, 2, -1, -1, -1) - ]).astype( - 'int64') #np.array to store 1,2,5,6s' non-leaf path(root -> leaf) - path_code = np.array([(0, 0, -1, -1, -1), (1, 1, 1, -1, -1), - (1, 0, 0, -1, -1), (0, 1, -1, -1, -1) - ]).astype('int64') #np.array to store + path_table = np.array( + [ + (0, 2, -1, -1, -1), + (0, 1, 3, -1, -1), + (0, 1, 4, -1, -1), + (0, 2, -1, -1, -1), + ] + ).astype( + 'int64' + ) # np.array to store 1,2,5,6s' non-leaf path(root -> leaf) + path_code = np.array( + [ + (0, 0, -1, -1, -1), + (1, 1, 1, -1, -1), + (1, 0, 0, -1, -1), + (0, 1, -1, -1, -1), + ] + ).astype( + 'int64' + ) # np.array to store bias = np.random.random((num_classes - 1, 1)) self.attrs = {'num_classes': num_classes, 'is_sparse': True} self.inputs = { @@ -250,10 +275,11 @@ def setUp(self): 'PathTable': path_table, 'PathCode': path_code, 'Label': label, - 'Bias': bias + 'Bias': bias, } - pre_output, out = hsigmoidWithCustomTree(x, w, path_table, path_code, - label, bias, num_classes) + pre_output, out = hsigmoidWithCustomTree( + x, w, path_table, path_code, label, bias, num_classes + ) self.outputs = {'PreOut': pre_output, 'Out': out} def test_check_output(self): @@ -261,15 +287,14 @@ def test_check_output(self): class TestHSigmoidOpWithSparseGrad(unittest.TestCase): - def hs_net_conf(self, is_sparse): input_word = fluid.layers.data(name="x", shape=[1], dtype='int64') - path_table = fluid.layers.data(name='path_table', - shape=[3], - dtype='int64') - path_code = fluid.layers.data(name='path_code', - shape=[3], - dtype='int64') + path_table = fluid.layers.data( + name='path_table', shape=[3], dtype='int64' + ) + path_code = fluid.layers.data( + name='path_code', shape=[3], dtype='int64' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') data_list = [input_word, path_table, path_code, label] @@ -278,17 +303,21 @@ def hs_net_conf(self, is_sparse): input=input_word, is_sparse=is_sparse, size=[3, 3], - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Normal( - scale=1 / math.sqrt(3)))) - - cost = fluid.layers.hsigmoid(input=emb, - label=label, - bias_attr=True, - num_classes=3, - path_table=path_table, - path_code=path_code, - is_custom=True, - is_sparse=is_sparse) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Normal(scale=1 / math.sqrt(3)) + ), + ) + + cost = fluid.layers.hsigmoid( + input=emb, + label=label, + bias_attr=True, + num_classes=3, + path_table=path_table, + path_code=path_code, + is_custom=True, + is_sparse=is_sparse, + ) avg_cost = fluid.layers.reduce_mean(cost) @@ -315,45 +344,61 @@ def training_test(self, is_sparse): exe.run(start_up) result = list() for i in range(10): - data = [([[x[i % 2]]], [list(path_table[i % 2])], - [list(path_code[i % 2])], [label[i % 2]])] - - loss_val = exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[loss]) + data = [ + ( + [[x[i % 2]]], + [list(path_table[i % 2])], + [list(path_code[i % 2])], + [label[i % 2]], + ) + ] + + loss_val = exe.run( + main_program, feed=feeder.feed(data), fetch_list=[loss] + ) result.append(loss_val) return result def test_hs_grad_with_sparse(self): dense_result = self.training_test(is_sparse=False) sparse_result = self.training_test(is_sparse=True) - assert (dense_result == sparse_result) + assert dense_result == sparse_result @skip_check_grad_ci( - reason= - "[skip shape check] The huffman tree is structed separately. It will be complicated if use large shape." + reason="[skip shape check] The huffman tree is structed separately. It will be complicated if use large shape." ) class TestHSigmoidOpWithCostumTree(OpTest): - def setUp(self): self.op_type = "hierarchical_sigmoid" self.python_api = python_api self.python_out_sig = python_out_sig - num_classes = 6 #using 1,2,3,4,5,6 to build a huffman tree and select 1,2,5,6 as sample + num_classes = 6 # using 1,2,3,4,5,6 to build a huffman tree and select 1,2,5,6 as sample feature_size = 8 batch_size = 4 x = np.random.uniform(-1, 1, (batch_size, feature_size)) w = np.random.uniform(-1, 1, (num_classes - 1, feature_size)) label = np.array([0, 1, 4, 5]).astype('int64') - path_table = np.array([ - (0, 2, -1, -1, -1), (0, 1, 3, -1, -1), (0, 1, 4, -1, -1), - (0, 2, -1, -1, -1) - ]).astype( - 'int64') #np.array to store 1,2,5,6s' non-leaf path(root -> leaf) - path_code = np.array([(0, 0, -1, -1, -1), (1, 1, 1, -1, -1), - (1, 0, 0, -1, -1), (0, 1, -1, -1, -1) - ]).astype('int64') #np.array to store + path_table = np.array( + [ + (0, 2, -1, -1, -1), + (0, 1, 3, -1, -1), + (0, 1, 4, -1, -1), + (0, 2, -1, -1, -1), + ] + ).astype( + 'int64' + ) # np.array to store 1,2,5,6s' non-leaf path(root -> leaf) + path_code = np.array( + [ + (0, 0, -1, -1, -1), + (1, 1, 1, -1, -1), + (1, 0, 0, -1, -1), + (0, 1, -1, -1, -1), + ] + ).astype( + 'int64' + ) # np.array to store bias = np.random.random((num_classes - 1, 1)) self.attrs = {'num_classes': num_classes, 'is_sparse': False} self.inputs = { @@ -362,45 +407,59 @@ def setUp(self): 'PathTable': path_table, 'PathCode': path_code, 'Label': label, - 'Bias': bias + 'Bias': bias, } - pre_output, out = hsigmoidWithCustomTree(x, w, path_table, path_code, - label, bias, num_classes) + pre_output, out = hsigmoidWithCustomTree( + x, w, path_table, path_code, label, bias, num_classes + ) self.outputs = {'PreOut': pre_output, 'Out': out} def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['Bias', 'X', 'W'], ['Out'], - no_grad_set=set('Label'), - check_eager=True) + self.check_grad( + ['Bias', 'X', 'W'], + ['Out'], + no_grad_set=set('Label'), + check_eager=True, + ) @skip_check_grad_ci( - reason= - "[skip shape check] The huffman tree is structed separately. It will be complicated if use large shape." + reason="[skip shape check] The huffman tree is structed separately. It will be complicated if use large shape." ) class TestHSigmoidOpWithCostumTreeWithoutBias(OpTest): - def setUp(self): self.op_type = "hierarchical_sigmoid" self.python_api = python_api self.python_out_sig = python_out_sig - num_classes = 6 #using 1,2,3,4,5,6 to build a huffman tree and select 1,2,5,6 as sample + num_classes = 6 # using 1,2,3,4,5,6 to build a huffman tree and select 1,2,5,6 as sample feature_size = 8 batch_size = 4 x = np.random.uniform(-1, 1, (batch_size, feature_size)) w = np.random.uniform(-1, 1, (num_classes - 1, feature_size)) label = np.array([0, 1, 4, 5]).astype('int64') - path_table = np.array([ - (0, 2, -1, -1, -1), (0, 1, 3, -1, -1), (0, 1, 4, -1, -1), - (0, 2, -1, -1, -1) - ]).astype( - 'int64') #np.array to store 1,2,5,6s' non-leaf path(root -> leaf) - path_code = np.array([(0, 0, -1, -1, -1), (1, 1, 1, -1, -1), - (1, 0, 0, -1, -1), (0, 1, -1, -1, -1) - ]).astype('int64') #np.array to store + path_table = np.array( + [ + (0, 2, -1, -1, -1), + (0, 1, 3, -1, -1), + (0, 1, 4, -1, -1), + (0, 2, -1, -1, -1), + ] + ).astype( + 'int64' + ) # np.array to store 1,2,5,6s' non-leaf path(root -> leaf) + path_code = np.array( + [ + (0, 0, -1, -1, -1), + (1, 1, 1, -1, -1), + (1, 0, 0, -1, -1), + (0, 1, -1, -1, -1), + ] + ).astype( + 'int64' + ) # np.array to store # bias = np.random.random((num_classes - 1, 1)).astype("float32") self.attrs = {'num_classes': num_classes, 'is_sparse': False} self.inputs = { @@ -410,22 +469,24 @@ def setUp(self): 'PathCode': path_code, 'Label': label, } - pre_output, out = hsigmoidWithCustomTree(x=x, - w=w, - path_table=path_table, - path_code=path_code, - label=label, - bias=None, - num_classes=num_classes) + pre_output, out = hsigmoidWithCustomTree( + x=x, + w=w, + path_table=path_table, + path_code=path_code, + label=label, + bias=None, + num_classes=num_classes, + ) self.outputs = {'PreOut': pre_output, 'Out': out} def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X', 'W'], ['Out'], - no_grad_set=set('Label'), - check_eager=True) + self.check_grad( + ['X', 'W'], ['Out'], no_grad_set=set('Label'), check_eager=True + ) class TestHSigmoidLossAPI(unittest.TestCase): @@ -441,27 +502,38 @@ def setUp(self): paddle.set_default_dtype(self.dtype) self.x_np = np.random.uniform( - -1, 1, [self.batch_size, self.feature_size]).astype(self.dtype) - self.labels_np = np.random.randint(self.num_classes, - size=(self.batch_size, 1), - dtype='int64') + -1, 1, [self.batch_size, self.feature_size] + ).astype(self.dtype) + self.labels_np = np.random.randint( + self.num_classes, size=(self.batch_size, 1), dtype='int64' + ) self.weight_np = np.random.uniform( - -1, 1, [self.num_classes - 1, self.feature_size]).astype(self.dtype) - self.bias_np = np.random.uniform( - -1, 1, (self.num_classes - 1, )).astype(self.dtype) + -1, 1, [self.num_classes - 1, self.feature_size] + ).astype(self.dtype) + self.bias_np = np.random.uniform(-1, 1, (self.num_classes - 1,)).astype( + self.dtype + ) self.path_table_np = None self.path_code_np = None - _, self.out_np = hsigmoid(self.x_np, self.weight_np, self.labels_np, - self.bias_np, self.num_classes) + _, self.out_np = hsigmoid( + self.x_np, + self.weight_np, + self.labels_np, + self.bias_np, + self.num_classes, + ) self.set_attrs() if self.is_custom: - _, self.out_np = hsigmoidWithCustomTree(self.x_np, self.weight_np, - self.path_table_np, - self.path_code_np, - self.labels_np, - self.bias_np.reshape(-1, 1), - self.num_classes) + _, self.out_np = hsigmoidWithCustomTree( + self.x_np, + self.weight_np, + self.path_table_np, + self.path_code_np, + self.labels_np, + self.bias_np.reshape(-1, 1), + self.num_classes, + ) def set_attrs(self): pass @@ -477,13 +549,19 @@ def test_dygraph_api(self): if self.is_custom: path_table = paddle.to_tensor(self.path_table_np) path_code = paddle.to_tensor(self.path_code_np) - out1 = F.hsigmoid_loss(x, labels, self.num_classes, weight, bias, - path_table, path_code) + out1 = F.hsigmoid_loss( + x, labels, self.num_classes, weight, bias, path_table, path_code + ) weight_attr = I.NumpyArrayInitializer(self.weight_np) bias_attr = I.NumpyArrayInitializer(self.bias_np) - m = paddle.nn.HSigmoidLoss(self.feature_size, self.num_classes, - weight_attr, bias_attr, self.is_custom) + m = paddle.nn.HSigmoidLoss( + self.feature_size, + self.num_classes, + weight_attr, + bias_attr, + self.is_custom, + ) out2 = m(x, labels, path_table, path_code) for out in [out1, out2]: @@ -497,23 +575,34 @@ def test_static_api(self): x = paddle.static.data('x', [-1, self.feature_size]) labels = paddle.static.data('labels', [-1, 1], 'int64') weight = paddle.static.data('weight', [-1, self.feature_size]) - bias = paddle.static.data('bias', [ - -1, - ]) + bias = paddle.static.data( + 'bias', + [ + -1, + ], + ) path_table = None path_code = None if self.is_custom: path_table = paddle.static.data('path_table', [-1, -1], 'int64') path_code = paddle.static.data('path_code', [-1, -1], 'int64') - out1 = F.hsigmoid_loss(x, labels, self.num_classes, weight, bias, - path_table, path_code) + out1 = F.hsigmoid_loss( + x, labels, self.num_classes, weight, bias, path_table, path_code + ) weight_attr = paddle.framework.ParamAttr( - initializer=I.NumpyArrayInitializer(self.weight_np)) + initializer=I.NumpyArrayInitializer(self.weight_np) + ) bias_attr = paddle.framework.ParamAttr( - initializer=I.NumpyArrayInitializer(self.bias_np)) - m = paddle.nn.HSigmoidLoss(self.feature_size, self.num_classes, - weight_attr, bias_attr, self.is_custom) + initializer=I.NumpyArrayInitializer(self.bias_np) + ) + m = paddle.nn.HSigmoidLoss( + self.feature_size, + self.num_classes, + weight_attr, + bias_attr, + self.is_custom, + ) out2 = m(x, labels, path_table, path_code) exe = paddle.static.Executor(self.place) @@ -522,14 +611,14 @@ def test_static_api(self): 'x': self.x_np, 'labels': self.labels_np, 'weight': self.weight_np, - 'bias': self.bias_np + 'bias': self.bias_np, } if self.is_custom: feed_dict["path_code"] = self.path_code_np feed_dict["path_table"] = self.path_table_np - ret1, ret2 = exe.run(train_program, - feed=feed_dict, - fetch_list=[out1, out2]) + ret1, ret2 = exe.run( + train_program, feed=feed_dict, fetch_list=[out1, out2] + ) for ret in [ret1, ret2]: np.testing.assert_allclose(self.out_np, ret, rtol=1e-05) @@ -547,9 +636,17 @@ def test_fluid_api(self): path_code = fluid.data('path_code', [-1, -1], 'int64') weight_attr = I.NumpyArrayInitializer(self.weight_np) bias_attr = I.NumpyArrayInitializer(self.bias_np) - out = fluid.layers.hsigmoid(x, labels, self.num_classes, - weight_attr, bias_attr, 'out', - path_table, path_code, self.is_custom) + out = fluid.layers.hsigmoid( + x, + labels, + self.num_classes, + weight_attr, + bias_attr, + 'out', + path_table, + path_code, + self.is_custom, + ) exe = fluid.Executor(self.place) exe.run(startup_program) @@ -557,13 +654,14 @@ def test_fluid_api(self): if self.is_custom: feed_dict["path_code"] = self.path_code_np feed_dict["path_table"] = self.path_table_np - ret, = exe.run(train_program, feed=feed_dict, fetch_list=[out]) + (ret,) = exe.run(train_program, feed=feed_dict, fetch_list=[out]) np.testing.assert_allclose(ret, self.out_np, rtol=1e-05) def test_errors(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): # test paddle.nn.HSigmoidLoss self.assertRaises(ValueError, paddle.nn.HSigmoidLoss, 6, 1) @@ -574,46 +672,52 @@ def test_errors(self): bias = paddle.static.data('bias', [7]) x_int32 = paddle.static.data('x_int32', [4, 6], 'int32') - self.assertRaises(TypeError, F.hsigmoid_loss, x_int32, label, 8, - weight) + self.assertRaises( + TypeError, F.hsigmoid_loss, x_int32, label, 8, weight + ) - label_float32 = paddle.static.data('label_float32', [4, 1], - 'float32') - self.assertRaises(TypeError, F.hsigmoid_loss, x, label_float32, 8, - weight) + label_float32 = paddle.static.data( + 'label_float32', [4, 1], 'float32' + ) + self.assertRaises( + TypeError, F.hsigmoid_loss, x, label_float32, 8, weight + ) weight_int32 = paddle.static.data('weight_int32', [7, 6], 'int32') - self.assertRaises(TypeError, F.hsigmoid_loss, x, label, 8, - weight_int32) + self.assertRaises( + TypeError, F.hsigmoid_loss, x, label, 8, weight_int32 + ) bias_int32 = paddle.static.data('bias_int32', [7], 'int32') - self.assertRaises(TypeError, - F.hsigmoid_loss, - x, - label, - 8, - weight, - bias=bias_int32) - - path_table_int32 = paddle.static.data('path_table_int32', [7], - 'int32') - self.assertRaises(TypeError, - F.hsigmoid_loss, - x, - label, - 8, - weight, - path_table=path_table_int32) - - path_code_int32 = paddle.static.data('path_code_int32', [7], - 'int32') - self.assertRaises(TypeError, - F.hsigmoid_loss, - x, - label, - 8, - weight, - path_code=path_code_int32) + self.assertRaises( + TypeError, F.hsigmoid_loss, x, label, 8, weight, bias=bias_int32 + ) + + path_table_int32 = paddle.static.data( + 'path_table_int32', [7], 'int32' + ) + self.assertRaises( + TypeError, + F.hsigmoid_loss, + x, + label, + 8, + weight, + path_table=path_table_int32, + ) + + path_code_int32 = paddle.static.data( + 'path_code_int32', [7], 'int32' + ) + self.assertRaises( + TypeError, + F.hsigmoid_loss, + x, + label, + 8, + weight, + path_code=path_code_int32, + ) # test paddle.nn.HSigmoidLoss paddle.disable_static(self.place) @@ -636,8 +740,9 @@ def test_errors(self): self.assertRaises(TypeError, fluid.layers.hsigmoid, 1, label, 2) # The input dtype must be float16, float32, float64. x_int32 = fluid.data(name='x_int32', shape=[4, 3], dtype='int32') - self.assertRaises(TypeError, fluid.layers.hsigmoid, x_int32, label, - 2) + self.assertRaises( + TypeError, fluid.layers.hsigmoid, x_int32, label, 2 + ) # support the input dtype is float32 x_fp32 = fluid.data(name='x_fp32', shape=[4, 3], dtype='float32') fluid.layers.hsigmoid(x_fp32, label, 2) @@ -646,20 +751,30 @@ def test_errors(self): self.assertRaises(TypeError, fluid.layers.hsigmoid, x_fp32, 1, 2) # The label dtype must be int64. label_int32 = fluid.data('label_int32', [4, 1], 'int32') - self.assertRaises(TypeError, fluid.layers.hsigmoid, x_fp32, - label_int32, 2) + self.assertRaises( + TypeError, fluid.layers.hsigmoid, x_fp32, label_int32, 2 + ) class TestHSigmoidLossAPICustom(TestHSigmoidLossAPI): - def set_attrs(self): self.is_custom = True - self.path_table_np = np.array([(0, 2, -1, -1, -1), (0, 1, 3, -1, -1), - (0, 1, 4, -1, -1), - (0, 2, -1, -1, -1)]).astype(np.int64) - self.path_code_np = np.array([(0, 0, -1, -1, -1), (1, 1, 1, -1, -1), - (1, 0, 0, -1, -1), - (0, 1, -1, -1, -1)]).astype(np.int64) + self.path_table_np = np.array( + [ + (0, 2, -1, -1, -1), + (0, 1, 3, -1, -1), + (0, 1, 4, -1, -1), + (0, 2, -1, -1, -1), + ] + ).astype(np.int64) + self.path_code_np = np.array( + [ + (0, 0, -1, -1, -1), + (1, 1, 1, -1, -1), + (1, 0, 0, -1, -1), + (0, 1, -1, -1, -1), + ] + ).astype(np.int64) def test_errors(self): pass diff --git a/python/paddle/fluid/tests/unittests/test_huber_loss_op.py b/python/paddle/fluid/tests/unittests/test_huber_loss_op.py index d679d2b719f43..778fedfd4aa08 100644 --- a/python/paddle/fluid/tests/unittests/test_huber_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_huber_loss_op.py @@ -29,7 +29,6 @@ def huber_loss_forward(val, delta): class TestHuberLossOp(OpTest): - def setUp(self): self.op_type = 'huber_loss' self.python_api = paddle.fluid.layers.huber_loss @@ -38,16 +37,17 @@ def setUp(self): self.init_input() shape = self.set_shape() residual = self.inputs['Y'] - self.inputs['X'] - loss = np.vectorize(huber_loss_forward)(residual, - self.delta).astype('float32') + loss = np.vectorize(huber_loss_forward)(residual, self.delta).astype( + 'float32' + ) self.attrs = {'delta': self.delta} self.outputs = {'Residual': residual, 'Out': loss.reshape(shape)} def init_input(self): shape = self.set_shape() self.inputs = { - 'X': np.random.uniform(0, 1., shape).astype('float32'), - 'Y': np.random.uniform(0, 1., shape).astype('float32'), + 'X': np.random.uniform(0, 1.0, shape).astype('float32'), + 'Y': np.random.uniform(0, 1.0, shape).astype('float32'), } def set_shape(self): @@ -60,38 +60,32 @@ def test_check_grad_normal(self): self.check_grad(['X', 'Y'], 'Out', check_eager=True) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.008, - no_grad_set=set("residual")) + self.check_grad( + ['Y'], 'Out', max_relative_error=0.008, no_grad_set=set("residual") + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.008, - no_grad_set=set('residual')) + self.check_grad( + ['X'], 'Out', max_relative_error=0.008, no_grad_set=set('residual') + ) def TestHuberLossOp1(TestHuberLossOp): - def set_shape(self): - return (64) + return 64 def TestHuberLossOp2(TestHuberLossOp): - def set_shape(self): return (6, 6) def TestHuberLossOp3(TestHuberLossOp): - def set_shape(self): return (6, 6, 1) class TestHuberLossOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input and label must be Variable @@ -106,10 +100,12 @@ def test_errors(self): # the dtype of input and label must be float32 or float64 xw2 = fluid.data(name='xw2', shape=[None, 6], dtype="int32") lw2 = fluid.data(name='lw2', shape=[None, 6], dtype="int32") - self.assertRaises(TypeError, fluid.layers.huber_loss, xw2, lr, - delta) - self.assertRaises(TypeError, fluid.layers.huber_loss, xr, lw2, - delta) + self.assertRaises( + TypeError, fluid.layers.huber_loss, xw2, lr, delta + ) + self.assertRaises( + TypeError, fluid.layers.huber_loss, xr, lw2, delta + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_hybrid_parallel_topology.py b/python/paddle/fluid/tests/unittests/test_hybrid_parallel_topology.py index 00b1f10ede850..d9210c128dc5c 100644 --- a/python/paddle/fluid/tests/unittests/test_hybrid_parallel_topology.py +++ b/python/paddle/fluid/tests/unittests/test_hybrid_parallel_topology.py @@ -18,7 +18,6 @@ class TestCommunicateTopology(unittest.TestCase): - def test_topology(self): topo = fleet.CommunicateTopology(["dp", "mp", "pp"], [2, 2, 2]) @@ -33,8 +32,9 @@ def test_topology(self): # test get_hybrid_group_names parallel_names = ["dp", "mp", "pp"] - np.testing.assert_array_equal(parallel_names, - topo.get_hybrid_group_names()) + np.testing.assert_array_equal( + parallel_names, topo.get_hybrid_group_names() + ) # test get_dims np.testing.assert_array_equal(2, topo.get_dim("dp")) @@ -78,29 +78,64 @@ def test_topology(self): self.assertEqual(topo.get_dim_size("pp"), 2) def test_topology_4D(self): - topo = fleet.CommunicateTopology(["dp", "pp", "sharding", "mp"], - [2, 2, 2, 2]) + topo = fleet.CommunicateTopology( + ["dp", "pp", "sharding", "mp"], [2, 2, 2, 2] + ) # test get_comm_list - dp_comm_list = [[0, 8], [1, 9], [2, 10], [3, 11], [4, 12], [5, 13], - [6, 14], [7, 15]] - mp_comm_list = [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], - [12, 13], [14, 15]] - pp_comm_list = [[0, 4], [1, 5], [2, 6], [3, 7], [8, 12], [9, 13], - [10, 14], [11, 15]] - sharding_comm_list = [[0, 2], [1, 3], [4, 6], [5, 7], [8, 10], [9, 11], - [12, 14], [13, 15]] + dp_comm_list = [ + [0, 8], + [1, 9], + [2, 10], + [3, 11], + [4, 12], + [5, 13], + [6, 14], + [7, 15], + ] + mp_comm_list = [ + [0, 1], + [2, 3], + [4, 5], + [6, 7], + [8, 9], + [10, 11], + [12, 13], + [14, 15], + ] + pp_comm_list = [ + [0, 4], + [1, 5], + [2, 6], + [3, 7], + [8, 12], + [9, 13], + [10, 14], + [11, 15], + ] + sharding_comm_list = [ + [0, 2], + [1, 3], + [4, 6], + [5, 7], + [8, 10], + [9, 11], + [12, 14], + [13, 15], + ] np.testing.assert_array_equal(dp_comm_list, topo.get_comm_list("dp")) np.testing.assert_array_equal(mp_comm_list, topo.get_comm_list("mp")) np.testing.assert_array_equal(pp_comm_list, topo.get_comm_list("pp")) - np.testing.assert_array_equal(sharding_comm_list, - topo.get_comm_list("sharding")) + np.testing.assert_array_equal( + sharding_comm_list, topo.get_comm_list("sharding") + ) # test get_hybrid_group_names parallel_names = ["dp", "pp", "sharding", "mp"] - np.testing.assert_array_equal(parallel_names, - topo.get_hybrid_group_names()) + np.testing.assert_array_equal( + parallel_names, topo.get_hybrid_group_names() + ) # test get_dims np.testing.assert_array_equal(2, topo.get_dim("dp")) @@ -149,20 +184,27 @@ def test_topology_4D(self): # test get_axis_list self.assertEqual(topo.get_axis_list("dp", 0), [0, 1, 2, 3, 4, 5, 6, 7]) - self.assertEqual(topo.get_axis_list("dp", 1), - [8, 9, 10, 11, 12, 13, 14, 15]) - self.assertEqual(topo.get_axis_list("mp", 0), - [0, 2, 4, 6, 8, 10, 12, 14]) - self.assertEqual(topo.get_axis_list("mp", 1), - [1, 3, 5, 7, 9, 11, 13, 15]) - self.assertEqual(topo.get_axis_list("pp", 0), - [0, 1, 2, 3, 8, 9, 10, 11]) - self.assertEqual(topo.get_axis_list("pp", 1), - [4, 5, 6, 7, 12, 13, 14, 15]) - self.assertEqual(topo.get_axis_list("sharding", 0), - [0, 1, 4, 5, 8, 9, 12, 13]) - self.assertEqual(topo.get_axis_list("sharding", 1), - [2, 3, 6, 7, 10, 11, 14, 15]) + self.assertEqual( + topo.get_axis_list("dp", 1), [8, 9, 10, 11, 12, 13, 14, 15] + ) + self.assertEqual( + topo.get_axis_list("mp", 0), [0, 2, 4, 6, 8, 10, 12, 14] + ) + self.assertEqual( + topo.get_axis_list("mp", 1), [1, 3, 5, 7, 9, 11, 13, 15] + ) + self.assertEqual( + topo.get_axis_list("pp", 0), [0, 1, 2, 3, 8, 9, 10, 11] + ) + self.assertEqual( + topo.get_axis_list("pp", 1), [4, 5, 6, 7, 12, 13, 14, 15] + ) + self.assertEqual( + topo.get_axis_list("sharding", 0), [0, 1, 4, 5, 8, 9, 12, 13] + ) + self.assertEqual( + topo.get_axis_list("sharding", 1), [2, 3, 6, 7, 10, 11, 14, 15] + ) # test get_dim_size self.assertEqual(topo.get_dim_size("dp"), 2) diff --git a/python/paddle/fluid/tests/unittests/test_identity_loss_op.py b/python/paddle/fluid/tests/unittests/test_identity_loss_op.py index de81c16da7dfa..0d808e7be3440 100644 --- a/python/paddle/fluid/tests/unittests/test_identity_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_identity_loss_op.py @@ -21,7 +21,6 @@ class TestIdentityLossOp(OpTest): - def setUp(self): self.max_relative_error = 0.006 self.python_api = paddle.incubate.identity_loss @@ -61,34 +60,29 @@ def initTestCase(self): class TestCase1(TestIdentityLossOp): - def initTestCase(self): self.shape = (8, 16, 8) self.reduction = 0 class TestCase2(TestIdentityLossOp): - def initTestCase(self): self.shape = (8, 16) self.reduction = 1 class TestCase3(TestIdentityLossOp): - def initTestCase(self): self.shape = (4, 8, 16) self.reduction = 2 class TestIdentityLossFloat32(TestIdentityLossOp): - def set_attrs(self): self.dtype = 'float32' class TestIdentityLossOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -100,8 +94,9 @@ def test_int(): self.assertRaises(Exception, test_int) def test_string(): - paddle.incubate.identity_loss(x=input_data, - reduction="wrongkey") + paddle.incubate.identity_loss( + x=input_data, reduction="wrongkey" + ) self.assertRaises(Exception, test_string) @@ -114,7 +109,6 @@ def test_dtype(): class TestIdentityLossAPI(unittest.TestCase): - def setUp(self): self.x_shape = [2, 3, 4, 5] self.x = np.random.uniform(-1, 1, self.x_shape).astype(np.float32) @@ -138,13 +132,14 @@ def test_api_static(self): out4 = paddle.incubate.identity_loss(x, reduction=2) exe = paddle.static.Executor(self.place) - res = exe.run(feed={'X': self.x}, - fetch_list=[out1, out2, out3, out4]) + res = exe.run( + feed={'X': self.x}, fetch_list=[out1, out2, out3, out4] + ) ref = [ self.identity_loss_ref(self.x, 2), self.identity_loss_ref(self.x, 0), self.identity_loss_ref(self.x, 1), - self.identity_loss_ref(self.x, 2) + self.identity_loss_ref(self.x, 2), ] for out, out_ref in zip(res, ref): np.testing.assert_allclose(out, out_ref, rtol=0.0001) @@ -172,8 +167,9 @@ def test_errors(self): x = paddle.to_tensor(x) self.assertRaises(Exception, paddle.incubate.identity_loss, x, -1) self.assertRaises(Exception, paddle.incubate.identity_loss, x, 3) - self.assertRaises(Exception, paddle.incubate.identity_loss, x, - "wrongkey") + self.assertRaises( + Exception, paddle.incubate.identity_loss, x, "wrongkey" + ) paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): x = paddle.fluid.data('X', [10, 12], 'int32') diff --git a/python/paddle/fluid/tests/unittests/test_identity_op.py b/python/paddle/fluid/tests/unittests/test_identity_op.py index f257db187eba6..f798b421cfb02 100644 --- a/python/paddle/fluid/tests/unittests/test_identity_op.py +++ b/python/paddle/fluid/tests/unittests/test_identity_op.py @@ -18,7 +18,6 @@ class TestIdentityAPI(unittest.TestCase): - def setUp(self): self.shape = [4, 4] self.x = np.random.random((4, 4)).astype(np.float32) diff --git a/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py b/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py index 9debbccdb3d7e..bacb13a0981b7 100644 --- a/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py +++ b/python/paddle/fluid/tests/unittests/test_iinfo_and_finfo.py @@ -18,21 +18,27 @@ class TestIInfoAndFInfoAPI(unittest.TestCase): - def test_invalid_input(self): for dtype in [ - paddle.float16, paddle.float32, paddle.float64, paddle.bfloat16, - paddle.complex64, paddle.complex128, paddle.bool + paddle.float16, + paddle.float32, + paddle.float64, + paddle.bfloat16, + paddle.complex64, + paddle.complex128, + paddle.bool, ]: with self.assertRaises(ValueError): _ = paddle.iinfo(dtype) def test_iinfo(self): - for paddle_dtype, np_dtype in [(paddle.int64, np.int64), - (paddle.int32, np.int32), - (paddle.int16, np.int16), - (paddle.int8, np.int8), - (paddle.uint8, np.uint8)]: + for paddle_dtype, np_dtype in [ + (paddle.int64, np.int64), + (paddle.int32, np.int32), + (paddle.int16, np.int16), + (paddle.int8, np.int8), + (paddle.uint8, np.uint8), + ]: xinfo = paddle.iinfo(paddle_dtype) xninfo = np.iinfo(np_dtype) self.assertEqual(xinfo.bits, xninfo.bits) diff --git a/python/paddle/fluid/tests/unittests/test_im2sequence_op.py b/python/paddle/fluid/tests/unittests/test_im2sequence_op.py index 5a90f79dd7ece..24a5e696a10c8 100644 --- a/python/paddle/fluid/tests/unittests/test_im2sequence_op.py +++ b/python/paddle/fluid/tests/unittests/test_im2sequence_op.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np @@ -39,26 +39,58 @@ def get_output_shape(attrs, in_shape, img_real_size): imgreal_w = img_real_size[index, 1] / out_stride[1] else: imgreal_w = img_real_size[index, 0] / out_stride[1] + 1 - output_height[0,index] = \ - 1 + \ - (imgreal_h + paddings[0] + paddings[2] - kernels[0] + strides[0] - 1) / \ - strides[0] - - output_width[0,index] = \ - 1 + \ - (imgreal_w + paddings[1] + paddings[3] - kernels[1] + strides[1] - 1) / \ - strides[1] + output_height[0, index] = ( + 1 + + ( + imgreal_h + + paddings[0] + + paddings[2] + - kernels[0] + + strides[0] + - 1 + ) + / strides[0] + ) + + output_width[0, index] = ( + 1 + + ( + imgreal_w + + paddings[1] + + paddings[3] + - kernels[1] + + strides[1] + - 1 + ) + / strides[1] + ) else: for index in range(batchsize): - output_height[0,index] = \ - 1 + \ - (img_height + paddings[0] + paddings[2] - kernels[0] + strides[0] - 1) / \ - strides[0] - - output_width[0,index] = \ - 1 + \ - (img_width + paddings[1] + paddings[3] - kernels[1] + strides[1] - 1) / \ - strides[1] + output_height[0, index] = ( + 1 + + ( + img_height + + paddings[0] + + paddings[2] + - kernels[0] + + strides[0] + - 1 + ) + / strides[0] + ) + + output_width[0, index] = ( + 1 + + ( + img_width + + paddings[1] + + paddings[3] + - kernels[1] + + strides[1] + - 1 + ) + / strides[1] + ) return output_height, output_width @@ -80,50 +112,70 @@ def im2col(attrs, im, col): for channel in range(0, input_channels): for filter_row_idx in range(0, filter_height): for filter_col_idx in range(0, filter_width): - im_row_offset = col_row_idx * stride_height \ - + filter_row_idx - padding_height - - im_col_offset = col_col_idx * stride_width \ - + filter_col_idx - padding_width - - if (im_row_offset < 0 or im_row_offset >= input_height - or im_col_offset < 0 - or im_col_offset >= input_width): - col[col_row_idx][col_col_idx][channel][\ - filter_row_idx][filter_col_idx] = 0.0 + im_row_offset = ( + col_row_idx * stride_height + + filter_row_idx + - padding_height + ) + + im_col_offset = ( + col_col_idx * stride_width + + filter_col_idx + - padding_width + ) + + if ( + im_row_offset < 0 + or im_row_offset >= input_height + or im_col_offset < 0 + or im_col_offset >= input_width + ): + col[col_row_idx][col_col_idx][channel][ + filter_row_idx + ][filter_col_idx] = 0.0 else: - im_offset = (channel * input_height + im_row_offset \ - ) * input_width + im_col_offset + im_offset = ( + channel * input_height + im_row_offset + ) * input_width + im_col_offset - col[col_row_idx][col_col_idx][channel][\ - filter_row_idx][filter_col_idx] = im[channel][ \ - im_row_offset][im_col_offset] + col[col_row_idx][col_col_idx][channel][ + filter_row_idx + ][filter_col_idx] = im[channel][im_row_offset][ + im_col_offset + ] def Im2Sequence(inputs, img_real_size, attrs): - output_height, output_width = get_output_shape(attrs, inputs.shape, - img_real_size) + output_height, output_width = get_output_shape( + attrs, inputs.shape, img_real_size + ) img_channels = inputs.shape[1] batch_size = inputs.shape[0] out = [] for index in range(batch_size): - tmp = np.zeros([ - output_height[0, index], output_width[0, index], img_channels, - attrs['kernels'][0], attrs['kernels'][1] - ]).astype("float32") + tmp = np.zeros( + [ + output_height[0, index], + output_width[0, index], + img_channels, + attrs['kernels'][0], + attrs['kernels'][1], + ] + ).astype("float32") out.append(tmp) for index in range(len(inputs)): im2col(attrs, inputs[index], out[index]) - out[index] = out[index].reshape([ - output_height[0, index] * output_width[0, index], - img_channels * attrs['kernels'][0] * attrs['kernels'][1] - ]) + out[index] = out[index].reshape( + [ + output_height[0, index] * output_width[0, index], + img_channels * attrs['kernels'][0] * attrs['kernels'][1], + ] + ) out = np.concatenate(out, axis=0) return out class TestBlockExpandOp(OpTest): - def config(self): self.batch_size = 1 self.img_channels = 3 @@ -138,9 +190,16 @@ def config(self): def setUp(self): self.config() self.op_type = "im2sequence" - x = np.random.uniform(0.1, 1, [ - self.batch_size, self.img_channels, self.img_height, self.img_width - ]).astype("float32") + x = np.random.uniform( + 0.1, + 1, + [ + self.batch_size, + self.img_channels, + self.img_height, + self.img_width, + ], + ).astype("float32") real_size = np.array([]).astype("float32") out = Im2Sequence(x, real_size, self.attrs) self.inputs = {'X': x} @@ -154,7 +213,6 @@ def test_check_grad_normal(self): class TestBlockExpandOpCase2(TestBlockExpandOp): - def config(self): self.batch_size = 2 self.img_channels = 3 @@ -168,7 +226,6 @@ def config(self): class TestBlockExpandOpCase3(TestBlockExpandOp): - def config(self): self.batch_size = 6 self.img_channels = 1 @@ -182,7 +239,6 @@ def config(self): class TestBlockExpandOpCase4(TestBlockExpandOp): - def config(self): self.batch_size = 6 self.img_channels = 2 @@ -196,11 +252,9 @@ def config(self): @skip_check_grad_ci( - reason= - "Since 'real_size' is used just in forward computation, we don't test the gradient here." + reason="Since 'real_size' is used just in forward computation, we don't test the gradient here." ) class TestBlockExpandOpCase5(OpTest): - def config(self): self.batch_size = 1 self.img_channels = 3 @@ -217,9 +271,16 @@ def config(self): def setUp(self): self.config() self.op_type = "im2sequence" - x = np.random.uniform(0.1, 1, [ - self.batch_size, self.img_channels, self.img_height, self.img_width - ]).astype("float32") + x = np.random.uniform( + 0.1, + 1, + [ + self.batch_size, + self.img_channels, + self.img_height, + self.img_width, + ], + ).astype("float32") out = np.array(Im2Sequence(x, self.real_size, self.attrs)) self.inputs = {'X': x, 'Y': self.real_size} self.outputs = {'Out': out} @@ -229,7 +290,6 @@ def test_check_output(self): class TestBlockExpandOpCase6(TestBlockExpandOpCase5): - def config(self): self.batch_size = 3 self.img_channels = 1 @@ -245,7 +305,6 @@ def config(self): class TestBlockExpandOpCase7(TestBlockExpandOpCase6): - def config(self): self.batch_size = 2 self.img_channels = 2 diff --git a/python/paddle/fluid/tests/unittests/test_image_classification_layer.py b/python/paddle/fluid/tests/unittests/test_image_classification_layer.py index fc3fee1ec65b6..fbcf634634b34 100644 --- a/python/paddle/fluid/tests/unittests/test_image_classification_layer.py +++ b/python/paddle/fluid/tests/unittests/test_image_classification_layer.py @@ -20,26 +20,27 @@ def conv_block(input, num_filter, groups, dropouts): - return nets.img_conv_group(input=input, - pool_size=2, - pool_stride=2, - conv_num_filter=[num_filter] * groups, - conv_filter_size=3, - conv_act='relu', - conv_with_batchnorm=True, - conv_batchnorm_drop_rate=dropouts, - pool_type='max') + return nets.img_conv_group( + input=input, + pool_size=2, + pool_stride=2, + conv_num_filter=[num_filter] * groups, + conv_filter_size=3, + conv_act='relu', + conv_with_batchnorm=True, + conv_batchnorm_drop_rate=dropouts, + pool_type='max', + ) class TestLayer(unittest.TestCase): - def test_batch_norm_layer(self): main_program = Program() startup_program = Program() with fluid.program_guard(main_program, startup_program): - images = fluid.layers.data(name='pixel', - shape=[3, 48, 48], - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=[3, 48, 48], dtype='float32' + ) hidden1 = fluid.layers.batch_norm(input=images) hidden2 = fluid.layers.fc(input=hidden1, size=128, act='relu') fluid.layers.batch_norm(input=hidden2) @@ -50,9 +51,9 @@ def test_dropout_layer(self): main_program = Program() startup_program = Program() with fluid.program_guard(main_program, startup_program): - images = fluid.layers.data(name='pixel', - shape=[3, 48, 48], - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=[3, 48, 48], dtype='float32' + ) fluid.layers.dropout(x=images, dropout_prob=0.5) print(str(main_program)) @@ -62,9 +63,9 @@ def test_img_conv_group(self): startup_program = Program() with fluid.program_guard(main_program, startup_program): - images = fluid.layers.data(name='pixel', - shape=[3, 48, 48], - dtype='float32') + images = fluid.layers.data( + name='pixel', shape=[3, 48, 48], dtype='float32' + ) conv1 = conv_block(images, 64, 2, [0.3, 0]) conv_block(conv1, 256, 3, [0.4, 0.4, 0]) @@ -74,12 +75,12 @@ def test_elementwise_add_with_act(self): main_program = Program() startup_program = Program() with fluid.program_guard(main_program, startup_program): - image1 = fluid.layers.data(name='pixel1', - shape=[3, 48, 48], - dtype='float32') - image2 = fluid.layers.data(name='pixel2', - shape=[3, 48, 48], - dtype='float32') + image1 = fluid.layers.data( + name='pixel1', shape=[3, 48, 48], dtype='float32' + ) + image2 = fluid.layers.data( + name='pixel2', shape=[3, 48, 48], dtype='float32' + ) fluid.layers.elementwise_add(x=image1, y=image2, act='relu') print(main_program) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_auto_prune.py b/python/paddle/fluid/tests/unittests/test_imperative_auto_prune.py index 0cd97cbf32887..3cda7d5d216fd 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_auto_prune.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_auto_prune.py @@ -20,19 +20,20 @@ class AutoPruneLayer0(fluid.Layer): - def __init__(self, input_size): super(AutoPruneLayer0, self).__init__() self.linear1 = fluid.dygraph.Linear( input_size, 5, param_attr=fluid.initializer.ConstantInitializer(value=2), - bias_attr=False) + bias_attr=False, + ) self.linear2 = fluid.dygraph.Linear( 5, 5, param_attr=fluid.initializer.ConstantInitializer(value=2), - bias_attr=False) + bias_attr=False, + ) def forward(self, x, y): a = self.linear1(x) @@ -43,19 +44,20 @@ def forward(self, x, y): class AutoPruneLayer1(fluid.Layer): - def __init__(self, input_size): super(AutoPruneLayer1, self).__init__() self.linear1 = fluid.dygraph.Linear( input_size, 5, param_attr=fluid.initializer.ConstantInitializer(value=2), - bias_attr=False) + bias_attr=False, + ) self.linear2 = fluid.dygraph.Linear( 5, 5, param_attr=fluid.initializer.ConstantInitializer(value=2), - bias_attr=False) + bias_attr=False, + ) def forward(self, x, y): a = self.linear1(x) @@ -67,7 +69,6 @@ def forward(self, x, y): class AutoPruneLayer2(fluid.Layer): - def __init__(self, input_size): super(AutoPruneLayer2, self).__init__() self.linear = fluid.dygraph.Linear(input_size, 10, act=None) @@ -85,16 +86,15 @@ def forward(self, x, label): class AutoPruneLayer3(fluid.Layer): - def __init__(self, input_size): super(AutoPruneLayer3, self).__init__() self.linear = fluid.dygraph.Linear(input_size, 20, act=None) def forward(self, x, label, test_num): feature = self.linear(x) - part1, part2 = fluid.layers.split(feature, - num_or_sections=[10, 10], - dim=1) + part1, part2 = fluid.layers.split( + feature, num_or_sections=[10, 10], dim=1 + ) # Note that: part2 is not used. loss = fluid.layers.cross_entropy(input=part1, label=label) loss = paddle.mean(loss) @@ -105,7 +105,6 @@ def forward(self, x, label, test_num): class MyLayer(fluid.Layer): - def __init__(self, input_size, vocab_size, size, dtype="float32"): super(MyLayer, self).__init__(dtype=dtype) self.embed0 = fluid.Embedding(size=(vocab_size, size)) @@ -128,7 +127,6 @@ def embed_linear0(self, x): class MyLayer2(fluid.Layer): - def __init__(self, input_size, vocab_size, size, dtype="float32"): super(MyLayer2, self).__init__(dtype=dtype) self.embed0 = fluid.Embedding(size=(vocab_size, size)) @@ -140,8 +138,9 @@ def forward(self, indices): # mind the difference with MyLayer # In this example, the forward method involes all params loss = fluid.layers.reduce_mean( - self.linear_0(self.embed0(indices)) + - self.linear_1(self.embed1(indices))) + self.linear_0(self.embed0(indices)) + + self.linear_1(self.embed1(indices)) + ) return loss def linear0(self, x): @@ -154,7 +153,6 @@ def embed_linear0(self, x): class TestImperativeAutoPrune(unittest.TestCase): - def func_auto_prune(self): with fluid.dygraph.guard(): case1 = AutoPruneLayer0(input_size=5) @@ -312,12 +310,15 @@ def func_auto_prune8(self): out2.backward() optimizer = fluid.optimizer.SGD( learning_rate=0.003, - parameter_list=(linear.parameters() + linear2.parameters())) + parameter_list=(linear.parameters() + linear2.parameters()), + ) optimizer.minimize(out2) - np.testing.assert_array_equal(linear2_origin, - linear2.weight.numpy()) + np.testing.assert_array_equal( + linear2_origin, linear2.weight.numpy() + ) self.assertFalse( - np.array_equal(linear_origin, linear.weight.numpy())) + np.array_equal(linear_origin, linear.weight.numpy()) + ) def test_auto_prune8(self): with _test_eager_guard(): @@ -342,10 +343,12 @@ def func_auto_prune9(self): out2.backward() optimizer = fluid.optimizer.SGD( learning_rate=0.003, - parameter_list=(linear.parameters() + linear2.parameters())) + parameter_list=(linear.parameters() + linear2.parameters()), + ) optimizer.minimize(out2) - np.testing.assert_array_equal(linear2_origin, - linear2.weight.numpy()) + np.testing.assert_array_equal( + linear2_origin, linear2.weight.numpy() + ) np.testing.assert_array_equal(linear_origin, linear.weight.numpy()) try: linear2.weight.gradient() @@ -371,7 +374,7 @@ def func_auto_prune10(self): out2 = linear2(b) out1.stop_gradient = True out = fluid.layers.concat(input=[out1, out2, c], axis=1) - #TODO(jiabin): In Eager Mode we don't actually need sort_sum_gradient, this test should be removed when we don't support fluid anymore. + # TODO(jiabin): In Eager Mode we don't actually need sort_sum_gradient, this test should be removed when we don't support fluid anymore. fluid.set_flags({'FLAGS_sort_sum_gradient': True}) out.backward() self.assertTrue(linear.weight.gradient() is None) @@ -387,8 +390,9 @@ def func_auto_prune_with_optimizer(self): size = 20 batch_size = 16 - indices = np.random.randint(low=0, high=100, - size=(batch_size, 1)).astype("int64") + indices = np.random.randint( + low=0, high=100, size=(batch_size, 1) + ).astype("int64") embed = np.random.randn(batch_size, size).astype("float32") place = fluid.CPUPlace() @@ -396,7 +400,8 @@ def func_auto_prune_with_optimizer(self): model = MyLayer(size, vocab_size, size) grad_clip = fluid.clip.GradientClipByGlobalNorm(0.001) optimizer = fluid.optimizer.AdamOptimizer( - 0.001, parameter_list=model.parameters(), grad_clip=grad_clip) + 0.001, parameter_list=model.parameters(), grad_clip=grad_clip + ) indices = fluid.dygraph.to_variable(indices) embed = fluid.dygraph.to_variable(embed) dummy_loss = model(embed) @@ -414,7 +419,8 @@ def func_auto_prune_with_optimizer(self): model = MyLayer2(size, vocab_size, size) grad_clip = fluid.clip.GradientClipByGlobalNorm(0.001) optimizer = fluid.optimizer.AdamOptimizer( - 0.001, parameter_list=model.parameters(), grad_clip=grad_clip) + 0.001, parameter_list=model.parameters(), grad_clip=grad_clip + ) indices = fluid.dygraph.to_variable(indices) emebd = fluid.dygraph.to_variable(embed) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_basic.py b/python/paddle/fluid/tests/unittests/test_imperative_basic.py index 3c8088a105557..62afffb59abca 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_basic.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_basic.py @@ -27,7 +27,6 @@ class MyLayer(fluid.Layer): - def __init__(self): super(MyLayer, self).__init__() @@ -40,23 +39,28 @@ def forward(self, inputs): class MLP(fluid.Layer): - def __init__(self, input_size): super(MLP, self).__init__() self._linear1 = Linear( input_size, 3, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + ) self._linear2 = Linear( 3, 4, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + ) def forward(self, inputs): x = self._linear1(inputs) @@ -66,7 +70,6 @@ def forward(self, inputs): class SimpleRNNCell(fluid.Layer): - def __init__(self, step_input_size, hidden_size, output_size, param_attr): super(SimpleRNNCell, self).__init__() self.step_input_size = step_input_size @@ -79,18 +82,24 @@ def __init__(self, step_input_size, hidden_size, output_size, param_attr): h2h_param_shape = [self.hidden_size, self.hidden_size] h2o_param_shape = [self.output_size, self.hidden_size] self._i2h_w = None - self._i2h_w = self.create_parameter(attr=self.param_attr, - shape=i2h_param_shape, - dtype=self._dtype, - is_bias=False) - self._h2h_w = self.create_parameter(attr=self.param_attr, - shape=h2h_param_shape, - dtype=self._dtype, - is_bias=False) - self._h2o_w = self.create_parameter(attr=self.param_attr, - shape=h2o_param_shape, - dtype=self._dtype, - is_bias=False) + self._i2h_w = self.create_parameter( + attr=self.param_attr, + shape=i2h_param_shape, + dtype=self._dtype, + is_bias=False, + ) + self._h2h_w = self.create_parameter( + attr=self.param_attr, + shape=h2h_param_shape, + dtype=self._dtype, + is_bias=False, + ) + self._h2o_w = self.create_parameter( + attr=self.param_attr, + shape=h2o_param_shape, + dtype=self._dtype, + is_bias=False, + ) def forward(self, input, pre_hidden): tmp_i2h = paddle.fluid.layers.nn.mul(input, self._i2h_w) @@ -104,29 +113,33 @@ def forward(self, input, pre_hidden): class SimpleRNN(fluid.Layer): - def __init__(self): super(SimpleRNN, self).__init__() self.seq_len = 4 self._cell = SimpleRNNCell( - 3, 3, 3, - fluid.ParamAttr(initializer=fluid.initializer.Constant(value=0.1))) + 3, + 3, + 3, + fluid.ParamAttr(initializer=fluid.initializer.Constant(value=0.1)), + ) def forward(self, inputs): outs = list() pre_hiddens = list() - init_hidden = self.create_parameter(attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.1)), - shape=[1, 3], - dtype='float32', - is_bias=False) + init_hidden = self.create_parameter( + attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + shape=[1, 3], + dtype='float32', + is_bias=False, + ) pre_hidden = init_hidden for i in range(self.seq_len): - input = fluid.layers.slice(inputs, - axes=[1], - starts=[i], - ends=[i + 1]) + input = fluid.layers.slice( + inputs, axes=[1], starts=[i], ends=[i + 1] + ) input = fluid.layers.reshape(input, shape=[1, 3]) out_softmax, pre_hidden = self._cell(input, pre_hidden) outs.append(out_softmax) @@ -135,7 +148,6 @@ def forward(self, inputs): class TestImperative(unittest.TestCase): - def functional_dygraph_context(self): self.assertFalse(fluid.dygraph.enabled()) fluid.enable_dygraph() @@ -220,8 +232,9 @@ def func_create_varbase(self): t = fluid.Tensor() t.set(x, fluid.CPUPlace()) if not _in_legacy_dygraph(): - egr_tmp = fluid.core.eager.Tensor(value=x, - place=fluid.core.CPUPlace()) + egr_tmp = fluid.core.eager.Tensor( + value=x, place=fluid.core.CPUPlace() + ) egr_tmp2 = fluid.core.eager.Tensor(y, fluid.core.CPUPlace()) egr_tmp3 = paddle.to_tensor(x) egr_tmp4 = fluid.core.eager.Tensor(y) @@ -359,13 +372,13 @@ def func_empty_var(self): cur_block = cur_program.current_block() # Normally, we don't allow tensor with -1 shape being created in dygraph mode, this test is not good. if _in_legacy_dygraph(): - new_variable = cur_block.create_var(name="X", - shape=[-1, 23, 48], - dtype='float32') + new_variable = cur_block.create_var( + name="X", shape=[-1, 23, 48], dtype='float32' + ) else: - new_variable = cur_block.create_var(name="X", - shape=[1, 23, 48], - dtype='float32') + new_variable = cur_block.create_var( + name="X", shape=[1, 23, 48], dtype='float32' + ) try: new_variable.numpy() except Exception as e: @@ -400,13 +413,13 @@ def func_empty_grad(self): cur_block = cur_program.current_block() # Normally, we don't allow tensor with -1 shape being created in dygraph mode, this test is not good. if _in_legacy_dygraph(): - new_variable = cur_block.create_var(name="X", - shape=[-1, 23, 48], - dtype='float32') + new_variable = cur_block.create_var( + name="X", shape=[-1, 23, 48], dtype='float32' + ) else: - new_variable = cur_block.create_var(name="X", - shape=[1, 23, 48], - dtype='float32') + new_variable = cur_block.create_var( + name="X", shape=[1, 23, 48], dtype='float32' + ) try: new_variable.gradient() except Exception as e: @@ -464,19 +477,24 @@ def func_layer_in_out(self): dy_grad2 = l2._x_for_debug.gradient() with new_program_scope(): - inp = fluid.layers.data(name="inp", - shape=[3], - append_batch_size=False) + inp = fluid.layers.data( + name="inp", shape=[3], append_batch_size=False + ) l = MyLayer() x = l(inp)[0] param_grads = fluid.backward.append_backward( - x, parameter_list=[l._x_for_debug.name])[0] - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + x, parameter_list=[l._x_for_debug.name] + )[0] + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) static_out, static_grad = exe.run( feed={inp.name: np_inp}, - fetch_list=[x.name, param_grads[1].name]) + fetch_list=[x.name, param_grads[1].name], + ) np.testing.assert_array_equal(dy_out, static_out) np.testing.assert_array_equal(dy_grad, static_grad) @@ -510,20 +528,25 @@ def func_mlp(self): dy_grad2 = mlp2._linear1.weight.gradient() with new_program_scope(): - inp = fluid.layers.data(name="inp", - shape=[2, 2], - append_batch_size=False) + inp = fluid.layers.data( + name="inp", shape=[2, 2], append_batch_size=False + ) mlp = MLP(input_size=2) out = mlp(inp) param_grads = fluid.backward.append_backward( - out, parameter_list=[mlp._linear1.weight.name])[0] - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + out, parameter_list=[mlp._linear1.weight.name] + )[0] + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe.run(fluid.default_startup_program()) static_out, static_grad = exe.run( feed={inp.name: np_inp}, - fetch_list=[out.name, param_grads[1].name]) + fetch_list=[out.name, param_grads[1].name], + ) np.testing.assert_allclose(dy_out, static_out, rtol=1e-05) np.testing.assert_allclose(dy_grad, static_grad, rtol=1e-05) @@ -548,28 +571,27 @@ def test_mlp(self): self.func_mlp() def test_gradient_accumulation(self): - def test_single_api(sort_sum_gradient): fluid.set_flags({'FLAGS_sort_sum_gradient': sort_sum_gradient}) - x = paddle.to_tensor(5., stop_gradient=False) + x = paddle.to_tensor(5.0, stop_gradient=False) for i in range(10): y = paddle.pow(x, 4.0) y.backward() self.assertEqual(x.grad.numpy(), (i + 1) * 500) x.clear_gradient() - self.assertEqual(x.grad.numpy(), 0.) + self.assertEqual(x.grad.numpy(), 0.0) for i in range(10): y = paddle.pow(x, 4.0) y.backward() self.assertEqual(x.grad.numpy(), (i + 1) * 500) x.clear_grad() - self.assertEqual(x.grad.numpy(), 0.) + self.assertEqual(x.grad.numpy(), 0.0) def test_simple_net(sort_sum_gradient): fluid.set_flags({'FLAGS_sort_sum_gradient': sort_sum_gradient}) - x = paddle.to_tensor(5., stop_gradient=False) - y = paddle.to_tensor(2., stop_gradient=False) - z = paddle.to_tensor(3., stop_gradient=False) + x = paddle.to_tensor(5.0, stop_gradient=False) + y = paddle.to_tensor(2.0, stop_gradient=False) + z = paddle.to_tensor(3.0, stop_gradient=False) def fun(x, y, z): loss1 = x * x * y @@ -618,59 +640,72 @@ def test_mlp(sort_sum_gradient): # generate the gradient of each step mlp2 = MLP(input_size=input_size) - expected_weight1_grad = 0. - expected_bias1_grad = 0. - expected_weight2_grad = 0. - expected_bias2_grad = 0. + expected_weight1_grad = 0.0 + expected_bias1_grad = 0.0 + expected_weight2_grad = 0.0 + expected_bias2_grad = 0.0 for batch_id in range(100): x = paddle.uniform([10, input_size]) detach_x = x.detach() clear_loss = mlp2(detach_x) clear_loss.backward() - expected_weight1_grad = (expected_weight1_grad + - mlp2._linear1.weight.grad.numpy()) - expected_bias1_grad = (expected_bias1_grad + - mlp2._linear1.bias.grad.numpy()) - expected_weight2_grad = (expected_weight2_grad + - mlp2._linear2.weight.grad.numpy()) - expected_bias2_grad = (expected_bias2_grad + - mlp2._linear2.bias.grad.numpy()) + expected_weight1_grad = ( + expected_weight1_grad + mlp2._linear1.weight.grad.numpy() + ) + expected_bias1_grad = ( + expected_bias1_grad + mlp2._linear1.bias.grad.numpy() + ) + expected_weight2_grad = ( + expected_weight2_grad + mlp2._linear2.weight.grad.numpy() + ) + expected_bias2_grad = ( + expected_bias2_grad + mlp2._linear2.bias.grad.numpy() + ) loss = mlp1(x) loss.backward() np.testing.assert_array_equal(loss.grad.numpy(), [1]) - np.testing.assert_allclose(mlp1._linear1.weight.grad.numpy(), - expected_weight1_grad, - rtol=1e-05) - np.testing.assert_allclose(mlp1._linear1.bias.grad.numpy(), - expected_bias1_grad, - rtol=1e-05) - np.testing.assert_allclose(mlp1._linear2.weight.grad.numpy(), - expected_weight2_grad, - rtol=1e-05) - np.testing.assert_allclose(mlp1._linear2.bias.grad.numpy(), - expected_bias2_grad, - rtol=1e-05) + np.testing.assert_allclose( + mlp1._linear1.weight.grad.numpy(), + expected_weight1_grad, + rtol=1e-05, + ) + np.testing.assert_allclose( + mlp1._linear1.bias.grad.numpy(), + expected_bias1_grad, + rtol=1e-05, + ) + np.testing.assert_allclose( + mlp1._linear2.weight.grad.numpy(), + expected_weight2_grad, + rtol=1e-05, + ) + np.testing.assert_allclose( + mlp1._linear2.bias.grad.numpy(), + expected_bias2_grad, + rtol=1e-05, + ) mlp2.clear_gradients() np.testing.assert_array_equal(clear_loss.grad.numpy(), [1]) if ((batch_id + 1) % 10) % 2 == 0: mlp1.clear_gradients() - expected_weight1_grad = 0. - expected_bias1_grad = 0. - expected_weight2_grad = 0. - expected_bias2_grad = 0. + expected_weight1_grad = 0.0 + expected_bias1_grad = 0.0 + expected_weight2_grad = 0.0 + expected_bias2_grad = 0.0 elif ((batch_id + 1) % 10) % 2 == 1: mlp1.clear_gradients() mlp1._linear1.weight._set_grad_ivar( - paddle.ones([input_size, 3])) + paddle.ones([input_size, 3]) + ) mlp1._linear2.weight._set_grad_ivar(paddle.ones([3, 4])) - expected_weight1_grad = 1. - expected_bias1_grad = 0. - expected_weight2_grad = 1. - expected_bias2_grad = 0. + expected_weight1_grad = 1.0 + expected_bias1_grad = 0.0 + expected_weight2_grad = 1.0 + expected_bias2_grad = 0.0 with fluid.dygraph.guard(): test_single_api(False) @@ -696,19 +731,25 @@ def func_dygraph_vs_static(self): # static graph with new_program_scope(): - inp_data1 = fluid.layers.data(name='inp1', - shape=[3, 3], - dtype=np.float32) - inp_data2 = fluid.layers.data(name='inp2', - shape=[3, 3], - dtype=np.float32) + inp_data1 = fluid.layers.data( + name='inp1', shape=[3, 3], dtype=np.float32 + ) + inp_data2 = fluid.layers.data( + name='inp2', shape=[3, 3], dtype=np.float32 + ) a = fluid.layers.expand( - fluid.layers.reshape(fluid.layers.reduce_sum(inp_data1), - [1, 1]), [4, 1]) + fluid.layers.reshape( + fluid.layers.reduce_sum(inp_data1), [1, 1] + ), + [4, 1], + ) b = fluid.layers.expand( - fluid.layers.reshape(fluid.layers.reduce_sum(inp_data2), - [1, 1]), [4, 1]) + fluid.layers.reshape( + fluid.layers.reduce_sum(inp_data2), [1, 1] + ), + [4, 1], + ) cond = fluid.layers.less_than(x=a, y=b) ie = fluid.layers.IfElse(cond) @@ -725,14 +766,16 @@ def func_dygraph_vs_static(self): ie.output(d3) out = ie() - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) - static_result = exe.run(fluid.default_main_program(), - feed={ - 'inp1': np_inp1, - 'inp2': np_inp2 - }, - fetch_list=out)[0] + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + static_result = exe.run( + fluid.default_main_program(), + feed={'inp1': np_inp1, 'inp2': np_inp2}, + fetch_list=out, + )[0] np.testing.assert_allclose(dygraph_result, static_result, rtol=1e-05) def test_dygraph_vs_static(self): @@ -741,8 +784,14 @@ def test_dygraph_vs_static(self): self.func_dygraph_vs_static() def func_rnn(self): - np_inp = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0], - [10.0, 11.0, 12.0]]) + np_inp = np.array( + [ + [1.0, 2.0, 3.0], + [4.0, 5.0, 6.0], + [7.0, 8.0, 9.0], + [10.0, 11.0, 12.0], + ] + ) np_inp = np_inp.reshape((1, 4, 3)) np_inp = np_inp.astype(np.float32) with fluid.dygraph.guard(): @@ -769,20 +818,28 @@ def func_rnn(self): dy_grad_i2h2 = simple_rnn2._cell._i2h_w.gradient() with new_program_scope(): - inp = fluid.layers.data(name="inp", - shape=[1, 4, 3], - append_batch_size=False) + inp = fluid.layers.data( + name="inp", shape=[1, 4, 3], append_batch_size=False + ) simple_rnn = SimpleRNN() outs, pre_hiddens = simple_rnn(inp) param_grads = fluid.backward.append_backward(outs[3]) exe = fluid.Executor(fluid.CPUPlace()) exe.run(fluid.default_startup_program()) - static_out, static_grad_h2o, static_grad_h2h, static_grad_i2h = exe.run( + ( + static_out, + static_grad_h2o, + static_grad_h2h, + static_grad_i2h, + ) = exe.run( feed={inp.name: np_inp}, fetch_list=[ - outs[3].name, param_grads[0][1].name, - param_grads[1][1].name, param_grads[2][1].name - ]) + outs[3].name, + param_grads[0][1].name, + param_grads[1][1].name, + param_grads[2][1].name, + ], + ) np.testing.assert_array_equal(dy_out, static_out) np.testing.assert_array_equal(dy_grad_h2o, static_grad_h2o) @@ -825,7 +882,6 @@ def test_layer_attrs(self): class TestDygraphUtils(unittest.TestCase): - def func_append_activation_in_dygraph_exception(self): with new_program_scope(): np_inp = np.random.random(size=(10, 20, 30)).astype(np.float32) @@ -883,9 +939,9 @@ def test_append_activation_in_dygraph3(self): def func_append_activation_in_dygraph_use_mkldnn(self): a_np = np.random.uniform(-2, 2, (10, 20, 30)).astype(np.float32) - helper = LayerHelper(fluid.unique_name.generate("test"), - act="relu", - use_mkldnn=True) + helper = LayerHelper( + fluid.unique_name.generate("test"), act="relu", use_mkldnn=True + ) func = helper.append_activation with fluid.dygraph.guard(): a = paddle.to_tensor(a_np) @@ -945,12 +1001,12 @@ def test_append_bias_in_dygraph(self): class TestDygraphGuardWithError(unittest.TestCase): - def func_without_guard(self): with fluid.dygraph.guard(): x = paddle.to_tensor(np.zeros([10, 10])) - with self.assertRaisesRegexp(TypeError, - "Please use `with fluid.dygraph.guard()"): + with self.assertRaisesRegexp( + TypeError, "Please use `with fluid.dygraph.guard()" + ): y = fluid.layers.matmul(x, x) def test_without_guard(self): @@ -960,16 +1016,17 @@ def test_without_guard(self): class TestMetaclass(unittest.TestCase): - def func_metaclass(self): self.assertEqual(type(MyLayer).__name__, 'type') self.assertNotEqual(type(MyLayer).__name__, 'pybind11_type') if not _in_legacy_dygraph(): self.assertEqual( - type(paddle.fluid.core.eager.Tensor).__name__, 'type') + type(paddle.fluid.core.eager.Tensor).__name__, 'type' + ) else: self.assertEqual( - type(paddle.fluid.core.VarBase).__name__, 'pybind11_type') + type(paddle.fluid.core.VarBase).__name__, 'pybind11_type' + ) def test_metaclass(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_container_layerdict.py b/python/paddle/fluid/tests/unittests/test_imperative_container_layerdict.py index 2e7bd103ec501..d81cef00f7bca 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_container_layerdict.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_container_layerdict.py @@ -19,12 +19,13 @@ class TestLayerDict(unittest.TestCase): - def func_layer_dict(self): - layers = OrderedDict([ - ('conv1d', paddle.nn.Conv1D(3, 2, 3)), - ('conv2d', paddle.nn.Conv2D(3, 2, 3)), - ]) + layers = OrderedDict( + [ + ('conv1d', paddle.nn.Conv1D(3, 2, 3)), + ('conv2d', paddle.nn.Conv2D(3, 2, 3)), + ] + ) layers_dicts = paddle.nn.LayerDict(sublayers=layers) @@ -58,10 +59,12 @@ def check_layer_dict(): layers_dicts['linear'] = layers['linear'] check_layer_dict() - sublayer = OrderedDict([ - ('sigmod', paddle.nn.Sigmoid()), - ('relu', paddle.nn.ReLU()), - ]) + sublayer = OrderedDict( + [ + ('sigmod', paddle.nn.Sigmoid()), + ('relu', paddle.nn.ReLU()), + ] + ) layers.update(sublayer) layers_dicts.update(sublayer) check_layer_dict() diff --git a/python/paddle/fluid/tests/unittests/test_imperative_container_layerlist.py b/python/paddle/fluid/tests/unittests/test_imperative_container_layerlist.py index d4d2bd0e5197a..89d92b3820554 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_container_layerlist.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_container_layerlist.py @@ -20,7 +20,6 @@ class MyLayer(fluid.Layer): - def __init__(self, layerlist): super(MyLayer, self).__init__() self.layerlist = layerlist @@ -32,53 +31,58 @@ def forward(self, x): class TestImperativeContainer(unittest.TestCase): - def fluid_dygraph_list(self): return fluid.dygraph.LayerList( - [fluid.dygraph.Linear(2**i, 2**(i + 1)) for i in range(6)]) + [fluid.dygraph.Linear(2**i, 2 ** (i + 1)) for i in range(6)] + ) def paddle_imperative_list(self): return paddle.nn.LayerList( - [fluid.dygraph.Linear(2**i, 2**(i + 1)) for i in range(6)]) + [fluid.dygraph.Linear(2**i, 2 ** (i + 1)) for i in range(6)] + ) def layer_list(self, use_fluid_api): data_np = np.random.uniform(-1, 1, [5, 1]).astype('float32') with fluid.dygraph.guard(): x = fluid.dygraph.to_variable(data_np) - layerlist = self.fluid_dygraph_list( - ) if use_fluid_api else self.paddle_imperative_list() + layerlist = ( + self.fluid_dygraph_list() + if use_fluid_api + else self.paddle_imperative_list() + ) size = len(layerlist) model = MyLayer(layerlist) res1 = model(x) self.assertListEqual(res1.shape, [5, 2**size]) - model.layerlist[size - 1] = fluid.dygraph.Linear(2**(size - 1), 5) + model.layerlist[size - 1] = fluid.dygraph.Linear(2 ** (size - 1), 5) res2 = model(x) self.assertListEqual(res2.shape, [5, 5]) del model.layerlist[size - 1] res3 = model(x) - self.assertListEqual(res3.shape, [5, 2**(size - 1)]) - model.layerlist.append(fluid.dygraph.Linear(2**(size - 1), 3)) + self.assertListEqual(res3.shape, [5, 2 ** (size - 1)]) + model.layerlist.append(fluid.dygraph.Linear(2 ** (size - 1), 3)) res4 = model(x) self.assertListEqual(res4.shape, [5, 3]) res4.backward() model2 = MyLayer(layerlist[:-1]) res5 = model2(x) - self.assertListEqual(res5.shape, [5, 2**(size - 1)]) + self.assertListEqual(res5.shape, [5, 2 ** (size - 1)]) del model2.layerlist[1:] res6 = model2(x) - self.assertListEqual(res6.shape, [5, 2**(0 + 1)]) + self.assertListEqual(res6.shape, [5, 2 ** (0 + 1)]) res6.backward() model3 = MyLayer(layerlist[:-2]) model3.layerlist.append(fluid.dygraph.Linear(3, 1)) - model3.layerlist.insert(size - 2, - fluid.dygraph.Linear(2**(size - 2), 3)) + model3.layerlist.insert( + size - 2, fluid.dygraph.Linear(2 ** (size - 2), 3) + ) res7 = model3(x) self.assertListEqual(res7.shape, [5, 1]) to_be_extended = [ - fluid.dygraph.Linear(3**i, 3**(i + 1)) for i in range(3) + fluid.dygraph.Linear(3**i, 3 ** (i + 1)) for i in range(3) ] model3.layerlist.extend(to_be_extended) res8 = model3(x) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_container_parameterlist.py b/python/paddle/fluid/tests/unittests/test_imperative_container_parameterlist.py index 2101e84858ab8..55e666c10aa88 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_container_parameterlist.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_container_parameterlist.py @@ -21,24 +21,26 @@ class MyLayer(fluid.Layer): - def __init__(self, num_stacked_param, use_fluid_api): super(MyLayer, self).__init__() # create ParameterList with iterable Parameters - self.params = self.fluid_dygraph_ParameterList( - num_stacked_param - ) if use_fluid_api else self.paddle_imperative_ParameterList( - num_stacked_param) + self.params = ( + self.fluid_dygraph_ParameterList(num_stacked_param) + if use_fluid_api + else self.paddle_imperative_ParameterList(num_stacked_param) + ) def fluid_dygraph_ParameterList(self, num_stacked_param): return fluid.dygraph.ParameterList( - [fluid.layers.create_parameter(shape=[2, 2], dtype='float32')] * - num_stacked_param) + [fluid.layers.create_parameter(shape=[2, 2], dtype='float32')] + * num_stacked_param + ) def paddle_imperative_ParameterList(self, num_stacked_param): return paddle.nn.ParameterList( - [fluid.layers.create_parameter(shape=[2, 2], dtype='float32')] * - num_stacked_param) + [fluid.layers.create_parameter(shape=[2, 2], dtype='float32')] + * num_stacked_param + ) def forward(self, x): for i, p in enumerate(self.params): @@ -47,7 +49,6 @@ def forward(self, x): class TestImperativeContainerParameterList(unittest.TestCase): - def paramter_list(self, use_fluid_api): data_np = np.random.uniform(-1, 1, [5, 2]).astype('float32') with fluid.dygraph.guard(): @@ -61,11 +62,13 @@ def paramter_list(self, use_fluid_api): loss.backward() model.params[num_stacked_param - 1] = fluid.layers.create_parameter( - shape=[2, 3], dtype='float32') + shape=[2, 3], dtype='float32' + ) res = model(x) self.assertListEqual(res.shape, [5, 3]) model.params.append( - fluid.layers.create_parameter(shape=[3, 4], dtype='float32')) + fluid.layers.create_parameter(shape=[3, 4], dtype='float32') + ) self.assertEqual(len(model.params), num_stacked_param + 1) res = model(x) self.assertListEqual(res.shape, [5, 4]) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_container_sequential.py b/python/paddle/fluid/tests/unittests/test_imperative_container_sequential.py index c0b4ed44e5b87..c1b0a42675f6a 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_container_sequential.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_container_sequential.py @@ -19,13 +19,13 @@ class TestImperativeContainerSequential(unittest.TestCase): - def func_sequential(self): data = np.random.uniform(-1, 1, [5, 10]).astype('float32') with fluid.dygraph.guard(): data = fluid.dygraph.to_variable(data) - model1 = fluid.dygraph.Sequential(fluid.Linear(10, 1), - fluid.Linear(1, 2)) + model1 = fluid.dygraph.Sequential( + fluid.Linear(10, 1), fluid.Linear(1, 2) + ) res1 = model1(data) self.assertListEqual(res1.shape, [5, 2]) model1[1] = fluid.Linear(1, 3) @@ -64,8 +64,9 @@ def func_sequential_list_params(self): data = np.random.uniform(-1, 1, [5, 10]).astype('float32') with fluid.dygraph.guard(): data = fluid.dygraph.to_variable(data) - model1 = fluid.dygraph.Sequential(fluid.Linear(10, 1), - fluid.Linear(1, 2)) + model1 = fluid.dygraph.Sequential( + fluid.Linear(10, 1), fluid.Linear(1, 2) + ) res1 = model1(data) self.assertListEqual(res1.shape, [5, 2]) model1[1] = fluid.Linear(1, 3) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_base.py b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_base.py index 5ae1b3f00144c..da15dcf29b570 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_base.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_base.py @@ -26,7 +26,6 @@ def get_random_images_and_labels(image_shape, label_shape): def sample_generator_creator(batch_size, batch_num): - def __reader__(): for _ in range(batch_num * batch_size): image, label = get_random_images_and_labels([784], [1]) @@ -36,7 +35,6 @@ def __reader__(): class TestDygraphDataLoader(unittest.TestCase): - def setUp(self): self.batch_size = 8 self.batch_num = 4 @@ -53,13 +51,14 @@ def iter_loader_data(self, loader): def func_test_single_process_loader(self): with fluid.dygraph.guard(): - loader = fluid.io.DataLoader.from_generator(capacity=self.capacity, - iterable=False, - use_multiprocess=False) - loader.set_sample_generator(sample_generator_creator( - self.batch_size, self.batch_num), - batch_size=self.batch_size, - places=fluid.CPUPlace()) + loader = fluid.io.DataLoader.from_generator( + capacity=self.capacity, iterable=False, use_multiprocess=False + ) + loader.set_sample_generator( + sample_generator_creator(self.batch_size, self.batch_num), + batch_size=self.batch_size, + places=fluid.CPUPlace(), + ) self.iter_loader_data(loader) def test_single_process_loader(self): @@ -69,12 +68,14 @@ def test_single_process_loader(self): def func_test_multi_process_loader(self): with fluid.dygraph.guard(): - loader = fluid.io.DataLoader.from_generator(capacity=self.capacity, - use_multiprocess=True) - loader.set_sample_generator(sample_generator_creator( - self.batch_size, self.batch_num), - batch_size=self.batch_size, - places=fluid.CPUPlace()) + loader = fluid.io.DataLoader.from_generator( + capacity=self.capacity, use_multiprocess=True + ) + loader.set_sample_generator( + sample_generator_creator(self.batch_size, self.batch_num), + batch_size=self.batch_size, + places=fluid.CPUPlace(), + ) self.iter_loader_data(loader) def test_multi_process_loader(self): @@ -85,9 +86,10 @@ def test_multi_process_loader(self): def func_test_generator_no_places(self): with fluid.dygraph.guard(): loader = fluid.io.DataLoader.from_generator(capacity=self.capacity) - loader.set_sample_generator(sample_generator_creator( - self.batch_size, self.batch_num), - batch_size=self.batch_size) + loader.set_sample_generator( + sample_generator_creator(self.batch_size, self.batch_num), + batch_size=self.batch_size, + ) self.iter_loader_data(loader) def test_generator_no_places(self): @@ -98,13 +100,14 @@ def test_generator_no_places(self): def func_test_set_pin_memory(self): with fluid.dygraph.guard(): use_pinned_memory(False) - loader = fluid.io.DataLoader.from_generator(capacity=self.capacity, - iterable=False, - use_multiprocess=False) - loader.set_sample_generator(sample_generator_creator( - self.batch_size, self.batch_num), - batch_size=self.batch_size, - places=fluid.CPUPlace()) + loader = fluid.io.DataLoader.from_generator( + capacity=self.capacity, iterable=False, use_multiprocess=False + ) + loader.set_sample_generator( + sample_generator_creator(self.batch_size, self.batch_num), + batch_size=self.batch_size, + places=fluid.CPUPlace(), + ) self.iter_loader_data(loader) use_pinned_memory(True) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exception.py b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exception.py index 1389b93c38e78..eac20ea03c788 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exception.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exception.py @@ -27,7 +27,6 @@ def get_random_images_and_labels(image_shape, label_shape): class TestDygraphDataLoaderWithException(unittest.TestCase): - def setUp(self): self.batch_size = 8 self.batch_num = 4 @@ -36,8 +35,9 @@ def setUp(self): def func_test_not_capacity(self): with fluid.dygraph.guard(): - with self.assertRaisesRegexp(ValueError, - "Please give value to capacity."): + with self.assertRaisesRegexp( + ValueError, "Please give value to capacity." + ): fluid.io.DataLoader.from_generator() def test_not_capacity(self): @@ -46,9 +46,7 @@ def test_not_capacity(self): self.func_test_not_capacity() def func_test_single_process_with_thread_expection(self): - def error_sample_genarator(batch_num): - def __reader__(): for _ in range(batch_num): yield [[[1, 2], [1]]] @@ -56,11 +54,12 @@ def __reader__(): return __reader__ with fluid.dygraph.guard(): - loader = fluid.io.DataLoader.from_generator(capacity=self.capacity, - iterable=False, - use_multiprocess=False) - loader.set_batch_generator(error_sample_genarator(self.batch_num), - places=fluid.CPUPlace()) + loader = fluid.io.DataLoader.from_generator( + capacity=self.capacity, iterable=False, use_multiprocess=False + ) + loader.set_batch_generator( + error_sample_genarator(self.batch_num), places=fluid.CPUPlace() + ) exception = None try: for _ in loader(): @@ -76,9 +75,7 @@ def test_single_process_with_thread_expection(self): self.func_test_single_process_with_thread_expection() def func_test_multi_process_with_process_expection(self): - def error_sample_genarator(batch_num): - def __reader__(): for _ in range(batch_num): yield [[[1, 2], [1]]] @@ -86,10 +83,12 @@ def __reader__(): return __reader__ with fluid.dygraph.guard(): - loader = fluid.io.DataLoader.from_generator(capacity=self.capacity, - use_multiprocess=True) - loader.set_batch_generator(error_sample_genarator(self.batch_num), - places=fluid.CPUPlace()) + loader = fluid.io.DataLoader.from_generator( + capacity=self.capacity, use_multiprocess=True + ) + loader.set_batch_generator( + error_sample_genarator(self.batch_num), places=fluid.CPUPlace() + ) exception = None try: for _ in loader(): @@ -104,24 +103,25 @@ def test_multi_process_with_process_expection(self): self.func_test_multi_process_with_process_expection() def func_test_multi_process_with_get_timeout(self): - def slow_batch_generator_creator(batch_size, batch_num): - def __reader__(): for _ in range(batch_num): time.sleep(80) batch_image, batch_label = get_random_images_and_labels( - [batch_size, 784], [batch_size, 1]) + [batch_size, 784], [batch_size, 1] + ) yield batch_image, batch_label return __reader__ with fluid.dygraph.guard(): - loader = fluid.io.DataLoader.from_generator(capacity=self.capacity, - use_multiprocess=True) - loader.set_batch_generator(slow_batch_generator_creator( - self.batch_size, self.batch_num), - places=fluid.CPUPlace()) + loader = fluid.io.DataLoader.from_generator( + capacity=self.capacity, use_multiprocess=True + ) + loader.set_batch_generator( + slow_batch_generator_creator(self.batch_size, self.batch_num), + places=fluid.CPUPlace(), + ) exception = None try: for _ in range(self.epoch_num): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exit_func.py b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exit_func.py index ec59d473c13df..ff9ae7343b38b 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exit_func.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_exit_func.py @@ -21,14 +21,17 @@ import queue -from paddle.fluid.reader import multiprocess_queue_set, _cleanup, CleanupFuncRegistrar +from paddle.fluid.reader import ( + multiprocess_queue_set, + _cleanup, + CleanupFuncRegistrar, +) # NOTE: These special functions cannot be detected by the existing coverage mechanism, # so the following unittests are added for these internal functions. class TestDygraphDataLoaderCleanUpFunc(unittest.TestCase): - def setUp(self): self.capacity = 10 @@ -65,8 +68,9 @@ def test_not_callable_func(self): self.func_test_not_callable_func() def func_test_old_handler_for_sigint(self): - CleanupFuncRegistrar.register(function=self.none_func, - signals=[signal.SIGINT]) + CleanupFuncRegistrar.register( + function=self.none_func, signals=[signal.SIGINT] + ) def test_old_handler_for_sigint(self): with _test_eager_guard(): @@ -78,8 +82,9 @@ def func_test_signal_wrapper_by_sigchld(self): def __test_process__(): pass - CleanupFuncRegistrar.register(function=self.none_func, - signals=[signal.SIGCHLD]) + CleanupFuncRegistrar.register( + function=self.none_func, signals=[signal.SIGCHLD] + ) exception = None try: diff --git a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_fds_clear.py b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_fds_clear.py index f0bdc8dacba9a..355f35ce5e9e6 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_fds_clear.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_fds_clear.py @@ -26,25 +26,24 @@ def get_random_images_and_labels(image_shape, label_shape): def batch_generator_creator(batch_size, batch_num): - def __reader__(): for _ in range(batch_num): batch_image, batch_label = get_random_images_and_labels( - [batch_size, 784], [batch_size, 1]) + [batch_size, 784], [batch_size, 1] + ) yield batch_image, batch_label return __reader__ class RandomDataset(Dataset): - def __init__(self, sample_num): self.sample_num = sample_num def __getitem__(self, idx): np.random.seed(idx) image = np.random.random([784]).astype('float32') - label = np.random.randint(0, 9, (1, )).astype('int64') + label = np.random.randint(0, 9, (1,)).astype('int64') return image, label def __len__(self): @@ -52,7 +51,6 @@ def __len__(self): class TestDygraphDataLoaderMmapFdsClear(unittest.TestCase): - def setUp(self): self.batch_size = 8 self.batch_num = 100 @@ -60,11 +58,13 @@ def setUp(self): self.capacity = 50 def prepare_data_loader(self): - loader = fluid.io.DataLoader.from_generator(capacity=self.capacity, - use_multiprocess=True) - loader.set_batch_generator(batch_generator_creator( - self.batch_size, self.batch_num), - places=fluid.CPUPlace()) + loader = fluid.io.DataLoader.from_generator( + capacity=self.capacity, use_multiprocess=True + ) + loader.set_batch_generator( + batch_generator_creator(self.batch_size, self.batch_num), + places=fluid.CPUPlace(), + ) return loader def run_one_epoch_with_break(self, loader): @@ -102,16 +102,17 @@ def test_data_loader_continue_break(self): class TestMultiProcessDataLoaderMmapFdsClear(TestDygraphDataLoaderMmapFdsClear): - def prepare_data_loader(self): place = fluid.CPUPlace() with fluid.dygraph.guard(place): dataset = RandomDataset(self.batch_size * self.batch_num) - loader = DataLoader(dataset, - places=place, - batch_size=self.batch_size, - drop_last=True, - num_workers=2) + loader = DataLoader( + dataset, + places=place, + batch_size=self.batch_size, + drop_last=True, + num_workers=2, + ) return loader diff --git a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_process.py b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_process.py index 94322a6795959..c03b9b3bbf16b 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_process.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_process.py @@ -29,11 +29,11 @@ def get_random_images_and_labels(image_shape, label_shape): def batch_generator_creator(batch_size, batch_num): - def __reader__(): for _ in range(batch_num): batch_image, batch_label = get_random_images_and_labels( - [batch_size, 784], [batch_size, 1]) + [batch_size, 784], [batch_size, 1] + ) yield batch_image, batch_label return __reader__ @@ -42,7 +42,6 @@ def __reader__(): # NOTE: coverage CI can't cover child process code, so need these test. # Here test child process loop function in main process class TestDygraphDataLoaderProcess(unittest.TestCase): - def setUp(self): self.batch_size = 8 self.batch_num = 4 @@ -60,10 +59,12 @@ def __clear_process__(util_queue): with fluid.dygraph.guard(): loader = fluid.io.DataLoader.from_generator( - capacity=self.batch_num + 1, use_multiprocess=True) - loader.set_batch_generator(batch_generator_creator( - self.batch_size, self.batch_num), - places=fluid.CPUPlace()) + capacity=self.batch_num + 1, use_multiprocess=True + ) + loader.set_batch_generator( + batch_generator_creator(self.batch_size, self.batch_num), + places=fluid.CPUPlace(), + ) loader._data_queue = queue.Queue(self.batch_num + 1) _reader_process_loop(loader._batch_reader, loader._data_queue) # For clean memory mapped files @@ -73,8 +74,9 @@ def __clear_process__(util_queue): util_queue.put(data) # Clean up memory mapped files - clear_process = multiprocessing.Process(target=__clear_process__, - args=(util_queue, )) + clear_process = multiprocessing.Process( + target=__clear_process__, args=(util_queue,) + ) clear_process.start() def test_reader_process_loop(self): @@ -83,9 +85,7 @@ def test_reader_process_loop(self): self.func_test_reader_process_loop() def func_test_reader_process_loop_simple_none(self): - def none_sample_genarator(batch_num): - def __reader__(): for _ in range(batch_num): yield None @@ -94,9 +94,11 @@ def __reader__(): with fluid.dygraph.guard(): loader = fluid.io.DataLoader.from_generator( - capacity=self.batch_num + 1, use_multiprocess=True) - loader.set_batch_generator(none_sample_genarator(self.batch_num), - places=fluid.CPUPlace()) + capacity=self.batch_num + 1, use_multiprocess=True + ) + loader.set_batch_generator( + none_sample_genarator(self.batch_num), places=fluid.CPUPlace() + ) loader._data_queue = queue.Queue(self.batch_num + 1) exception = None try: diff --git a/python/paddle/fluid/tests/unittests/test_imperative_data_parallel.py b/python/paddle/fluid/tests/unittests/test_imperative_data_parallel.py index 4d61d2a67fd87..9118788617d88 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_data_parallel.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_data_parallel.py @@ -23,7 +23,6 @@ class MLP(fluid.Layer): - def __init__(self, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -37,7 +36,6 @@ def forward(self, inputs): class TestDataParallelStateDict(unittest.TestCase): - def test_data_parallel_state_dict(self): with fluid.dygraph.guard(): strategy = dygraph.parallel.prepare_context() @@ -48,13 +46,17 @@ def test_data_parallel_state_dict(self): parallel_state = parallel_mlp.state_dict() base_para = {} - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) for k, v in single_state.items(): self.assertTrue(k in parallel_state) - np.testing.assert_array_equal(v.numpy(), - parallel_state[k].numpy()) + np.testing.assert_array_equal( + v.numpy(), parallel_state[k].numpy() + ) base_para[k] = v.numpy() diff --git a/python/paddle/fluid/tests/unittests/test_imperative_decorator.py b/python/paddle/fluid/tests/unittests/test_imperative_decorator.py index f783b18b2b2e9..42407c0b36f3f 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_decorator.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_decorator.py @@ -23,7 +23,6 @@ class TestTracerMode(unittest.TestCase): - def setUp(self): self.init_mode = True @@ -60,8 +59,9 @@ def need_no_grad_func(a, b=1): decorated_func = fluid.dygraph.no_grad(need_no_grad_func) self.assertTrue( - str(inspect.getfullargspec(decorated_func)) == str( - inspect.getfullargspec(need_no_grad_func))) + str(inspect.getfullargspec(decorated_func)) + == str(inspect.getfullargspec(need_no_grad_func)) + ) self.assertEqual(self.tracer._train_mode, self.init_mode) @@ -79,13 +79,11 @@ def test_main(self): class TestTracerMode2(TestTracerMode): - def setUp(self): self.init_mode = False class TestNoGradClass(unittest.TestCase): - @paddle.no_grad() def no_grad_func(self, a): self.assertEqual(self.tracer._train_mode, True) @@ -105,8 +103,10 @@ def need_no_grad_func(a, b=1): return a + b decorated_func = paddle.no_grad()(need_no_grad_func) - self.assertEqual(str(inspect.getfullargspec(decorated_func)), - str(inspect.getfullargspec(need_no_grad_func))) + self.assertEqual( + str(inspect.getfullargspec(decorated_func)), + str(inspect.getfullargspec(need_no_grad_func)), + ) def test_gen(): for i in range(3): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py b/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py index 3da576045c587..361792f5077f8 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py @@ -28,7 +28,6 @@ class DMF(fluid.Layer): - def __init__(self): super(DMF, self).__init__() self._user_latent = Linear(1000, 256) @@ -41,15 +40,23 @@ def __init__(self): self._user_layers.append( self.add_sublayer( 'user_layer_%d' % i, - Linear(256 if i == 0 else self._hid_sizes[i - 1], - self._hid_sizes[i], - act='relu'))) + Linear( + 256 if i == 0 else self._hid_sizes[i - 1], + self._hid_sizes[i], + act='relu', + ), + ) + ) self._item_layers.append( self.add_sublayer( 'item_layer_%d' % i, - Linear(256 if i == 0 else self._hid_sizes[i - 1], - self._hid_sizes[i], - act='relu'))) + Linear( + 256 if i == 0 else self._hid_sizes[i - 1], + self._hid_sizes[i], + act='relu', + ), + ) + ) def forward(self, users, items): users = self._user_latent(users) @@ -62,7 +69,6 @@ def forward(self, users, items): class MLP(fluid.Layer): - def __init__(self): super(MLP, self).__init__() self._user_latent = Linear(1000, 256) @@ -73,22 +79,26 @@ def __init__(self): self._match_layers.append( self.add_sublayer( 'match_layer_%d' % i, - Linear(256 * 2 if i == 0 else self._hid_sizes[i - 1], - self._hid_sizes[i], - act='relu'))) + Linear( + 256 * 2 if i == 0 else self._hid_sizes[i - 1], + self._hid_sizes[i], + act='relu', + ), + ) + ) def forward(self, users, items): users = self._user_latent(users) items = self._item_latent(items) - match_vec = fluid.layers.concat([users, items], - axis=len(users.shape) - 1) + match_vec = fluid.layers.concat( + [users, items], axis=len(users.shape) - 1 + ) for l in self._match_layers: match_vec = l(match_vec) return match_vec class DeepCF(fluid.Layer): - def __init__(self, num_users, num_items, matrix): super(DeepCF, self).__init__() self._num_users = num_users @@ -98,7 +108,8 @@ def __init__(self, num_users, num_items, matrix): shape=matrix.shape, dtype=matrix.dtype, is_bias=False, - default_initializer=fluid.initializer.NumpyArrayInitializer(matrix)) + default_initializer=fluid.initializer.NumpyArrayInitializer(matrix), + ) self._rating_matrix.stop_gradient = True self._mlp = MLP() @@ -110,20 +121,21 @@ def forward(self, users, items): # items_emb = self._item_emb(items) users_emb = fluid.layers.gather(self._rating_matrix, users) items_emb = fluid.layers.gather( - fluid.layers.transpose(self._rating_matrix, [1, 0]), items) + fluid.layers.transpose(self._rating_matrix, [1, 0]), items + ) users_emb.stop_gradient = True items_emb.stop_gradient = True mlp_predictive = self._mlp(users_emb, items_emb) dmf_predictive = self._dmf(users_emb, items_emb) - predictive = fluid.layers.concat([mlp_predictive, dmf_predictive], - axis=len(mlp_predictive.shape) - 1) + predictive = fluid.layers.concat( + [mlp_predictive, dmf_predictive], axis=len(mlp_predictive.shape) - 1 + ) prediction = self._match_fc(predictive) return prediction class TestDygraphDeepCF(unittest.TestCase): - def setUp(self): # Can use Amusic dataset as the DeepCF describes. self.data_path = os.environ.get('DATA_PATH', '') @@ -152,9 +164,14 @@ def get_data(self): users_np = np.array(user_ids, dtype=np.int32)[indices] items_np = np.array(item_ids, dtype=np.int32)[indices] labels_np = np.array(labels, dtype=np.float32)[indices] - return np.expand_dims(users_np, -1), \ - np.expand_dims(items_np, -1), \ - np.expand_dims(labels_np, -1), NUM_USERS, NUM_ITEMS, matrix + return ( + np.expand_dims(users_np, -1), + np.expand_dims(items_np, -1), + np.expand_dims(labels_np, -1), + NUM_USERS, + NUM_ITEMS, + matrix, + ) def load_data(self): sys.stderr.write('loading from %s\n' % self.data_path) @@ -194,18 +211,35 @@ def load_data(self): users_np = np.array(user_ids, dtype=np.int32)[indices] items_np = np.array(item_ids, dtype=np.int32)[indices] labels_np = np.array(labels, dtype=np.float32)[indices] - return np.expand_dims(users_np, -1), \ - np.expand_dims(items_np, -1), \ - np.expand_dims(labels_np, -1), num_users, num_items, matrix + return ( + np.expand_dims(users_np, -1), + np.expand_dims(items_np, -1), + np.expand_dims(labels_np, -1), + num_users, + num_items, + matrix, + ) def test_deefcf(self): seed = 90 if self.data_path: - (users_np, items_np, labels_np, num_users, num_items, - matrix) = self.load_data() + ( + users_np, + items_np, + labels_np, + num_users, + num_items, + matrix, + ) = self.load_data() else: - (users_np, items_np, labels_np, num_users, num_items, - matrix) = self.get_data() + ( + users_np, + items_np, + labels_np, + num_users, + num_items, + matrix, + ) = self.get_data() paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) startup = fluid.Program() @@ -220,28 +254,39 @@ def test_deefcf(self): deepcf = DeepCF(num_users, num_items, matrix) prediction = deepcf(users, items) loss = fluid.layers.reduce_sum( - fluid.layers.log_loss(prediction, labels)) + fluid.layers.log_loss(prediction, labels) + ) adam = fluid.optimizer.AdamOptimizer(0.01) adam.minimize(loss) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe.run(startup) for e in range(self.num_epoches): sys.stderr.write('epoch %d\n' % e) - for slice in range(0, self.batch_size * self.num_batches, - self.batch_size): + for slice in range( + 0, self.batch_size * self.num_batches, self.batch_size + ): if slice + self.batch_size >= users_np.shape[0]: break static_loss = exe.run( main, feed={ - users.name: users_np[slice:slice + self.batch_size], - items.name: items_np[slice:slice + self.batch_size], - labels.name: - labels_np[slice:slice + self.batch_size] + users.name: users_np[ + slice : slice + self.batch_size + ], + items.name: items_np[ + slice : slice + self.batch_size + ], + labels.name: labels_np[ + slice : slice + self.batch_size + ], }, - fetch_list=[loss])[0] + fetch_list=[loss], + )[0] sys.stderr.write('static loss %s\n' % static_loss) with fluid.dygraph.guard(): @@ -250,21 +295,27 @@ def test_deefcf(self): deepcf = DeepCF(num_users, num_items, matrix) adam = fluid.optimizer.AdamOptimizer( - 0.01, parameter_list=deepcf.parameters()) + 0.01, parameter_list=deepcf.parameters() + ) for e in range(self.num_epoches): sys.stderr.write('epoch %d\n' % e) - for slice in range(0, self.batch_size * self.num_batches, - self.batch_size): + for slice in range( + 0, self.batch_size * self.num_batches, self.batch_size + ): if slice + self.batch_size >= users_np.shape[0]: break prediction = deepcf( - to_variable(users_np[slice:slice + self.batch_size]), - to_variable(items_np[slice:slice + self.batch_size])) + to_variable(users_np[slice : slice + self.batch_size]), + to_variable(items_np[slice : slice + self.batch_size]), + ) loss = fluid.layers.reduce_sum( fluid.layers.log_loss( prediction, - to_variable(labels_np[slice:slice + - self.batch_size]))) + to_variable( + labels_np[slice : slice + self.batch_size] + ), + ) + ) loss.backward() adam.minimize(loss) deepcf.clear_gradients() @@ -277,28 +328,35 @@ def test_deefcf(self): deepcf2 = DeepCF(num_users, num_items, matrix) adam2 = fluid.optimizer.AdamOptimizer( - 0.01, parameter_list=deepcf2.parameters()) + 0.01, parameter_list=deepcf2.parameters() + ) fluid.set_flags({'FLAGS_sort_sum_gradient': True}) for e in range(self.num_epoches): sys.stderr.write('epoch %d\n' % e) - for slice in range(0, self.batch_size * self.num_batches, - self.batch_size): + for slice in range( + 0, self.batch_size * self.num_batches, self.batch_size + ): if slice + self.batch_size >= users_np.shape[0]: break prediction2 = deepcf2( - to_variable(users_np[slice:slice + self.batch_size]), - to_variable(items_np[slice:slice + self.batch_size])) + to_variable(users_np[slice : slice + self.batch_size]), + to_variable(items_np[slice : slice + self.batch_size]), + ) loss2 = fluid.layers.reduce_sum( fluid.layers.log_loss( prediction2, - to_variable(labels_np[slice:slice + - self.batch_size]))) + to_variable( + labels_np[slice : slice + self.batch_size] + ), + ) + ) loss2.backward() adam2.minimize(loss2) deepcf2.clear_gradients() dy_loss2 = loss2.numpy() - sys.stderr.write('dynamic loss: %s %s\n' % - (slice, dy_loss2)) + sys.stderr.write( + 'dynamic loss: %s %s\n' % (slice, dy_loss2) + ) with fluid.dygraph.guard(): with _test_eager_guard(): @@ -309,30 +367,39 @@ def test_deefcf(self): deepcf = DeepCF(num_users, num_items, matrix) adam = fluid.optimizer.AdamOptimizer( - 0.01, parameter_list=deepcf.parameters()) + 0.01, parameter_list=deepcf.parameters() + ) for e in range(self.num_epoches): sys.stderr.write('epoch %d\n' % e) - for slice in range(0, self.batch_size * self.num_batches, - self.batch_size): + for slice in range( + 0, self.batch_size * self.num_batches, self.batch_size + ): if slice + self.batch_size >= users_np.shape[0]: break prediction = deepcf( - to_variable(users_np[slice:slice + - self.batch_size]), - to_variable(items_np[slice:slice + - self.batch_size])) + to_variable( + users_np[slice : slice + self.batch_size] + ), + to_variable( + items_np[slice : slice + self.batch_size] + ), + ) loss = fluid.layers.reduce_sum( fluid.layers.log_loss( prediction, - to_variable(labels_np[slice:slice + - self.batch_size]))) + to_variable( + labels_np[slice : slice + self.batch_size] + ), + ) + ) loss.backward() adam.minimize(loss) deepcf.clear_gradients() eager_loss = loss.numpy() - sys.stderr.write('eager loss: %s %s\n' % - (slice, eager_loss)) + sys.stderr.write( + 'eager loss: %s %s\n' % (slice, eager_loss) + ) self.assertEqual(static_loss, dy_loss) self.assertEqual(static_loss, dy_loss2) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_double_grad.py b/python/paddle/fluid/tests/unittests/test_imperative_double_grad.py index 1d77c5132399b..140f1191c9644 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_double_grad.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_double_grad.py @@ -23,7 +23,6 @@ def _dygraph_guard_(func): - def __impl__(*args, **kwargs): if fluid._non_static_mode(): return func(*args, **kwargs) @@ -43,7 +42,6 @@ def random_var(size, low=-1, high=1, dtype='float32'): class TestEagerGrad(TestCase): - def func_simple_example_eager_grad(self): np.random.seed(2021) paddle.set_device('cpu') @@ -184,8 +182,9 @@ def record_hook(grad): out4 = paddle.mean(out3) egr_dout2, egr_dout3 = paddle.grad([out4], [out2, out3]) - np.testing.assert_array_equal(dout2_record_by_hook[0].numpy(), - np.array([1.0, 2.0])) + np.testing.assert_array_equal( + dout2_record_by_hook[0].numpy(), np.array([1.0, 2.0]) + ) x1 = paddle.to_tensor([1.0, 2.0]) x1.stop_gradient = False @@ -205,27 +204,30 @@ def record_hook(grad): class TestDygraphDoubleGrad(TestCase): - def setUp(self): self.sort_sum_gradient = False self.shape = [5, 10] - def grad(self, - outputs, - inputs, - grad_outputs=None, - no_grad_vars=None, - retain_graph=None, - create_graph=False, - allow_unused=False): + def grad( + self, + outputs, + inputs, + grad_outputs=None, + no_grad_vars=None, + retain_graph=None, + create_graph=False, + allow_unused=False, + ): fluid.set_flags({'FLAGS_sort_sum_gradient': self.sort_sum_gradient}) - return fluid.dygraph.grad(outputs=outputs, - inputs=inputs, - grad_outputs=grad_outputs, - no_grad_vars=no_grad_vars, - retain_graph=retain_graph, - create_graph=create_graph, - allow_unused=allow_unused) + return fluid.dygraph.grad( + outputs=outputs, + inputs=inputs, + grad_outputs=grad_outputs, + no_grad_vars=no_grad_vars, + retain_graph=retain_graph, + create_graph=create_graph, + allow_unused=allow_unused, + ) @dygraph_guard def func_exception(self): @@ -247,12 +249,16 @@ def func_exception(self): self.grad([random_var(shape)], [1]) with self.assertRaises(AssertionError): - self.grad([random_var(shape), random_var(shape)], - [random_var(shape)], [random_var(shape)]) + self.grad( + [random_var(shape), random_var(shape)], + [random_var(shape)], + [random_var(shape)], + ) with self.assertRaises(AssertionError): - self.grad([random_var(shape)], [random_var(shape)], - no_grad_vars=[1]) + self.grad( + [random_var(shape)], [random_var(shape)], no_grad_vars=[1] + ) with self.assertRaises(AssertionError): self.grad([random_var(shape)], [random_var(shape)], no_grad_vars=1) @@ -269,31 +275,33 @@ def func_simple_example(self): y = x + 1 for create_graph in [False, True]: - dx, = self.grad([x], [x], - create_graph=create_graph, - retain_graph=True) + (dx,) = self.grad( + [x], [x], create_graph=create_graph, retain_graph=True + ) self.assertEqual(dx.shape, x.shape) self.assertTrue(np.all(dx.numpy() == 1)) self.assertNotEqual(dx.stop_gradient, create_graph) - dx_mul_2, = self.grad([y, x], [x], - create_graph=create_graph, - retain_graph=True) + (dx_mul_2,) = self.grad( + [y, x], [x], create_graph=create_graph, retain_graph=True + ) self.assertEqual(dx_mul_2.shape, x.shape) self.assertTrue(np.all(dx_mul_2.numpy() == 2)) self.assertNotEqual(dx_mul_2.stop_gradient, create_graph) - none_grad, = self.grad([x], [y], - create_graph=create_graph, - allow_unused=True) + (none_grad,) = self.grad( + [x], [y], create_graph=create_graph, allow_unused=True + ) self.assertTrue(none_grad is None) - grad_with_none_and_not_none, = self.grad([x, y], [y], - create_graph=create_graph) + (grad_with_none_and_not_none,) = self.grad( + [x, y], [y], create_graph=create_graph + ) self.assertTrue(grad_with_none_and_not_none.shape, x.shape) self.assertTrue(np.all(grad_with_none_and_not_none.numpy() == 1)) - self.assertNotEqual(grad_with_none_and_not_none.stop_gradient, - create_graph) + self.assertNotEqual( + grad_with_none_and_not_none.stop_gradient, create_graph + ) def test_simple_example(self): with _test_eager_guard(): @@ -315,15 +323,20 @@ def func_example_no_grad_vars(self): w_mean = fluid.layers.reduce_mean(w) del y1, z, w - dx_actual, = self.grad([w_mean], [x], - create_graph=True, - no_grad_vars=[y2]) + (dx_actual,) = self.grad( + [w_mean], [x], create_graph=True, no_grad_vars=[y2] + ) self.assertFalse(y2.stop_gradient) self.assertFalse(dx_actual.stop_gradient) - dx_expected = (1.0 / float(numel) * (np.maximum(x_np, 0) + y2.numpy()) * - (x_np > 0) * 2).astype('float32') + dx_expected = ( + 1.0 + / float(numel) + * (np.maximum(x_np, 0) + y2.numpy()) + * (x_np > 0) + * 2 + ).astype('float32') np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) @@ -340,11 +353,12 @@ def func_none_one_initial_gradient(self): half_numel = int(numel / 2) half_x_positive = np.random.uniform(low=1, high=2, size=[half_numel]) - half_x_negative = np.random.uniform(low=-2, - high=-1, - size=[numel - half_numel]) - x_np = np.array(list(half_x_positive) + - list(half_x_negative)).astype('float32') + half_x_negative = np.random.uniform( + low=-2, high=-1, size=[numel - half_numel] + ) + x_np = np.array(list(half_x_positive) + list(half_x_negative)).astype( + 'float32' + ) np.random.shuffle(x_np) x = fluid.dygraph.to_variable(x_np) @@ -359,8 +373,9 @@ def func_none_one_initial_gradient(self): relu_x_np = np.maximum(x_np, alpha * x_np).astype('float32') relu_x_grad_np = ((x_np > 0) + (x_np < 0) * alpha).astype('float32') dy_expected = (relu_x_np * relu_x_grad_np * 2).astype('float32') - dz_expected = (np.power(relu_x_np, 3) * relu_x_grad_np * - 4).astype('float32') + dz_expected = (np.power(relu_x_np, 3) * relu_x_grad_np * 4).astype( + 'float32' + ) random_grad_y = random_var(y.shape, low=1, high=2) random_grad_z = random_var(z.shape, low=1, high=2) @@ -373,31 +388,39 @@ def func_none_one_initial_gradient(self): for grad_y in [random_grad_y]: for grad_z in [random_grad_z]: for create_graph in [False, True]: - dx_actual, = self.grad(outputs=[y, z], - inputs=[x], - grad_outputs=[grad_y, grad_z], - create_graph=create_graph, - retain_graph=True) + (dx_actual,) = self.grad( + outputs=[y, z], + inputs=[x], + grad_outputs=[grad_y, grad_z], + create_graph=create_graph, + retain_graph=True, + ) - grad_y_np = ones_grad_y if grad_y is None else grad_y.numpy( + grad_y_np = ( + ones_grad_y if grad_y is None else grad_y.numpy() ) - grad_z_np = ones_grad_z if grad_z is None else grad_z.numpy( + grad_z_np = ( + ones_grad_z if grad_z is None else grad_z.numpy() ) - dx_expected = dy_expected * grad_y_np + dz_expected * grad_z_np - np.testing.assert_allclose(dx_actual.numpy(), - dx_expected, - rtol=1e-05) + dx_expected = ( + dy_expected * grad_y_np + dz_expected * grad_z_np + ) + np.testing.assert_allclose( + dx_actual.numpy(), dx_expected, rtol=1e-05 + ) if grad_y is not None: self.assertTrue(grad_y.stop_gradient) - np.testing.assert_array_equal(grad_y.numpy(), - original_random_grad_y) + np.testing.assert_array_equal( + grad_y.numpy(), original_random_grad_y + ) if grad_z is not None: self.assertTrue(grad_z.stop_gradient) - np.testing.assert_array_equal(grad_z.numpy(), - original_random_grad_z) + np.testing.assert_array_equal( + grad_z.numpy(), original_random_grad_z + ) def test_none_one_initial_gradient(self): with _test_eager_guard(): @@ -418,35 +441,39 @@ def func_example_with_gradient_accumulation_and_create_graph(self): w_mean = fluid.layers.reduce_mean(w) del y, z, w - dx_actual, = self.grad([w_mean], [x], create_graph=True) + (dx_actual,) = self.grad([w_mean], [x], create_graph=True) del w_mean self.assertFalse(dx_actual.stop_gradient) # Theoritical result based on math calculation - dx_expected = (1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * - (x_np > 0) * 2).astype('float32') + dx_expected = ( + 1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * (x_np > 0) * 2 + ).astype('float32') np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) loss = fluid.layers.reduce_mean(dx_actual * dx_actual + x * x) loss.backward(retain_graph=True) x_grad_actual = x.gradient() - x_grad_expected = (2.0 / float(numel) * - (x_np + dx_expected * - (x_np > 0) * 2 / float(numel))).astype('float32') + x_grad_expected = ( + 2.0 + / float(numel) + * (x_np + dx_expected * (x_np > 0) * 2 / float(numel)) + ).astype('float32') np.testing.assert_allclose(x_grad_actual, x_grad_expected, rtol=1e-05) for i in range(5): loss.backward(retain_graph=True) x_grad_actual = x.gradient() - x_grad_expected = ( - i + 2) * (2.0 / float(numel) * - (x_np + dx_expected * - (x_np > 0) * 2 / float(numel))).astype('float32') - np.testing.assert_allclose(x_grad_actual, - x_grad_expected, - rtol=1e-05) + x_grad_expected = (i + 2) * ( + 2.0 + / float(numel) + * (x_np + dx_expected * (x_np > 0) * 2 / float(numel)) + ).astype('float32') + np.testing.assert_allclose( + x_grad_actual, x_grad_expected, rtol=1e-05 + ) def test_example_with_gradient_accumulation_and_create_graph(self): with _test_eager_guard(): @@ -468,25 +495,35 @@ def func_example_with_gradient_accumulation_and_no_grad_vars(self): w_mean = fluid.layers.reduce_mean(w) del y1, z, w - dx_actual, = self.grad([w_mean], [x], - retain_graph=True, - create_graph=True, - no_grad_vars=[y2]) + (dx_actual,) = self.grad( + [w_mean], + [x], + retain_graph=True, + create_graph=True, + no_grad_vars=[y2], + ) self.assertFalse(y2.stop_gradient) self.assertFalse(dx_actual.stop_gradient) - dx_expected = (1.0 / float(numel) * (np.maximum(x_np, 0) + y2.numpy()) * - (x_np > 0) * 2).astype('float32') + dx_expected = ( + 1.0 + / float(numel) + * (np.maximum(x_np, 0) + y2.numpy()) + * (x_np > 0) + * 2 + ).astype('float32') np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) loss = fluid.layers.reduce_mean(dx_actual * dx_actual + x * x) loss.backward() x_grad_actual = x.gradient() - x_grad_expected = (2.0 / float(numel) * - (x_np + dx_expected * - (x_np > 0) * 4 / float(numel))).astype('float32') + x_grad_expected = ( + 2.0 + / float(numel) + * (x_np + dx_expected * (x_np > 0) * 4 / float(numel)) + ).astype('float32') np.testing.assert_allclose(x_grad_actual, x_grad_expected, rtol=1e-05) def test_example_with_gradient_accumulation_and_no_grad_vars(self): @@ -508,13 +545,14 @@ def func_example_with_gradient_accumulation_and_not_create_graph(self): w_mean = fluid.layers.reduce_mean(w) del y, z, w - dx_actual, = self.grad([w_mean], [x], create_graph=False) + (dx_actual,) = self.grad([w_mean], [x], create_graph=False) del w_mean self.assertTrue(dx_actual.stop_gradient) - dx_expected = (1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * - (x_np > 0) * 2).astype('float32') + dx_expected = ( + 1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * (x_np > 0) * 2 + ).astype('float32') np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) @@ -532,17 +570,18 @@ def test_example_with_gradient_accumulation_and_not_create_graph(self): class TestDygraphDoubleGradSortGradient(TestDygraphDoubleGrad): - def setUp(self): self.sort_sum_gradient = True self.shape = [5, 10] class TestDygraphDoubleGradVisitedUniq(TestCase): - def func_compare(self): - value = np.random.uniform(-0.5, 0.5, 100).reshape(10, 2, - 5).astype("float32") + value = ( + np.random.uniform(-0.5, 0.5, 100) + .reshape(10, 2, 5) + .astype("float32") + ) def model_f(input): linear = fluid.dygraph.Linear(5, 3, bias_attr=False) @@ -563,11 +602,13 @@ def model_f(input): out = model_f(a) - dx = fluid.dygraph.grad(outputs=[out], - inputs=[a], - create_graph=False, - only_inputs=True, - allow_unused=False) + dx = fluid.dygraph.grad( + outputs=[out], + inputs=[a], + create_graph=False, + only_inputs=True, + allow_unused=False, + ) grad_1 = dx[0].numpy() @@ -591,17 +632,15 @@ def test_compare(self): class TestRaiseNoDoubleGradOp(TestCase): - def raise_no_grad_op(self): with fluid.dygraph.guard(): x = fluid.layers.ones(shape=[2, 3, 2, 2], dtype='float32') x.stop_gradient = False y = paddle.fluid.layers.group_norm(x, groups=1) - dx = fluid.dygraph.grad(outputs=[y], - inputs=[x], - create_graph=True, - retain_graph=True)[0] + dx = fluid.dygraph.grad( + outputs=[y], inputs=[x], create_graph=True, retain_graph=True + )[0] loss = fluid.layers.reduce_mean(dx) loss.backward() @@ -611,7 +650,6 @@ def test_raise(self): class TestDoubleGradResNet(TestCase): - def setUp(self): paddle.seed(123) paddle.framework.random._manual_program_seed(123) @@ -626,7 +664,8 @@ def test_resnet_resnet50(self): egr_out = model(egr_data) egr_preds = paddle.argmax(egr_out, axis=1) egr_label_onehot = paddle.nn.functional.one_hot( - paddle.to_tensor(egr_preds), num_classes=egr_out.shape[1]) + paddle.to_tensor(egr_preds), num_classes=egr_out.shape[1] + ) egr_target = paddle.sum(egr_out * egr_label_onehot, axis=1) egr_g = paddle.grad(outputs=egr_target, inputs=egr_out)[0] @@ -638,8 +677,9 @@ def test_resnet_resnet50(self): data.stop_gradient = False out = model(data) preds = paddle.argmax(out, axis=1) - label_onehot = paddle.nn.functional.one_hot(paddle.to_tensor(preds), - num_classes=out.shape[1]) + label_onehot = paddle.nn.functional.one_hot( + paddle.to_tensor(preds), num_classes=out.shape[1] + ) target = paddle.sum(out * label_onehot, axis=1) g = paddle.grad(outputs=target, inputs=out)[0] @@ -658,7 +698,8 @@ def test_resnet_resnet101(self): egr_out = model(egr_data) egr_preds = paddle.argmax(egr_out, axis=1) egr_label_onehot = paddle.nn.functional.one_hot( - paddle.to_tensor(egr_preds), num_classes=egr_out.shape[1]) + paddle.to_tensor(egr_preds), num_classes=egr_out.shape[1] + ) egr_target = paddle.sum(egr_out * egr_label_onehot, axis=1) egr_g = paddle.grad(outputs=egr_target, inputs=egr_out)[0] @@ -670,8 +711,9 @@ def test_resnet_resnet101(self): data.stop_gradient = False out = model(data) preds = paddle.argmax(out, axis=1) - label_onehot = paddle.nn.functional.one_hot(paddle.to_tensor(preds), - num_classes=out.shape[1]) + label_onehot = paddle.nn.functional.one_hot( + paddle.to_tensor(preds), num_classes=out.shape[1] + ) target = paddle.sum(out * label_onehot, axis=1) g = paddle.grad(outputs=target, inputs=out)[0] @@ -683,24 +725,23 @@ def test_resnet_resnet101(self): class TestDoubleGradBasics(TestCase): - def test_matmul(self): input_numpy = np.ones([3, 3]) * 2 with _test_eager_guard(): - x = paddle.to_tensor(input_numpy, - stop_gradient=False, - dtype='float32') - y = paddle.to_tensor(input_numpy, - stop_gradient=False, - dtype='float32') - grad_out = paddle.to_tensor(np.ones([3, 3]), - stop_gradient=False, - dtype='float32') + x = paddle.to_tensor( + input_numpy, stop_gradient=False, dtype='float32' + ) + y = paddle.to_tensor( + input_numpy, stop_gradient=False, dtype='float32' + ) + grad_out = paddle.to_tensor( + np.ones([3, 3]), stop_gradient=False, dtype='float32' + ) out = paddle.matmul(x, y, False, False) - new_x_g, new_y_g = paddle.grad([out], [x, y], [grad_out], - retain_graph=True, - create_graph=True) + new_x_g, new_y_g = paddle.grad( + [out], [x, y], [grad_out], retain_graph=True, create_graph=True + ) new_x_g.backward() out_ref = np.ones([3, 3]) * 12.0 @@ -718,8 +759,9 @@ def test_matmul(self): np.testing.assert_array_equal(y.grad.numpy(), y_grad_ref) grad_out_grad_ref = np.ones([3, 3]) * 6.0 - np.testing.assert_array_equal(grad_out.grad.numpy(), - grad_out_grad_ref) + np.testing.assert_array_equal( + grad_out.grad.numpy(), grad_out_grad_ref + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_framework.py b/python/paddle/fluid/tests/unittests/test_imperative_framework.py index 3a1e7c9cd1270..dc23f8b0a97b0 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_framework.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_framework.py @@ -20,23 +20,28 @@ class MLP(fluid.Layer): - def __init__(self, input_size): super(MLP, self).__init__() self._linear1 = fluid.dygraph.Linear( input_size, 3, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + ) self._linear2 = fluid.dygraph.Linear( 3, 4, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.1))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.1) + ), + ) def forward(self, inputs): x = self._linear1(inputs) @@ -46,19 +51,18 @@ def forward(self, inputs): class TestDygraphFramework(unittest.TestCase): - def func_test_dygraph_backward(self): with new_program_scope(): mlp = MLP(input_size=2) - var_inp = fluid.layers.data("input", - shape=[2, 2], - dtype="float32", - append_batch_size=False) + var_inp = fluid.layers.data( + "input", shape=[2, 2], dtype="float32", append_batch_size=False + ) out = mlp(var_inp) try: out.backward() raise AssertionError( - "backward should not be usable in static graph mode") + "backward should not be usable in static graph mode" + ) except AssertionError as e: self.assertTrue((e is not None)) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_gan.py b/python/paddle/fluid/tests/unittests/test_imperative_gan.py index 9edecc8998956..8cfe9befb629d 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_gan.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_gan.py @@ -26,7 +26,6 @@ class Discriminator(fluid.Layer): - def __init__(self): super(Discriminator, self).__init__() self._fc1 = Linear(1, 32, act='elu') @@ -39,7 +38,6 @@ def forward(self, inputs): class Generator(fluid.Layer): - def __init__(self): super(Generator, self).__init__() self._fc1 = Linear(2, 64, act='elu') @@ -54,7 +52,6 @@ def forward(self, inputs): class TestDygraphGAN(unittest.TestCase): - def func_test_gan_float32(self): seed = 90 paddle.seed(1) @@ -64,34 +61,38 @@ def func_test_gan_float32(self): generate_p = fluid.Program() scope = fluid.core.Scope() - with new_program_scope(main=discriminate_p, - startup=startup, - scope=scope): + with new_program_scope( + main=discriminate_p, startup=startup, scope=scope + ): discriminator = Discriminator() generator = Generator() - img = fluid.layers.data(name="img", - shape=[2, 1], - append_batch_size=False) - noise = fluid.layers.data(name="noise", - shape=[2, 2], - append_batch_size=False) + img = fluid.layers.data( + name="img", shape=[2, 1], append_batch_size=False + ) + noise = fluid.layers.data( + name="noise", shape=[2, 2], append_batch_size=False + ) d_real = discriminator(img) d_loss_real = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( x=d_real, - label=fluid.layers.fill_constant(shape=[2, 1], - dtype='float32', - value=1.0))) + label=fluid.layers.fill_constant( + shape=[2, 1], dtype='float32', value=1.0 + ), + ) + ) d_fake = discriminator(generator(noise)) d_loss_fake = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( x=d_fake, - label=fluid.layers.fill_constant(shape=[2, 1], - dtype='float32', - value=0.0))) + label=fluid.layers.fill_constant( + shape=[2, 1], dtype='float32', value=0.0 + ), + ) + ) d_loss = d_loss_real + d_loss_fake @@ -102,42 +103,47 @@ def func_test_gan_float32(self): discriminator = Discriminator() generator = Generator() - noise = fluid.layers.data(name="noise", - shape=[2, 2], - append_batch_size=False) + noise = fluid.layers.data( + name="noise", shape=[2, 2], append_batch_size=False + ) d_fake = discriminator(generator(noise)) g_loss = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( x=d_fake, - label=fluid.layers.fill_constant(shape=[2, 1], - dtype='float32', - value=1.0))) + label=fluid.layers.fill_constant( + shape=[2, 1], dtype='float32', value=1.0 + ), + ) + ) sgd = SGDOptimizer(learning_rate=1e-3) sgd.minimize(g_loss) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) static_params = dict() with fluid.scope_guard(scope): img = np.ones([2, 1], np.float32) noise = np.ones([2, 2], np.float32) exe.run(startup) - static_d_loss = exe.run(discriminate_p, - feed={ - 'img': img, - 'noise': noise - }, - fetch_list=[d_loss])[0] - static_g_loss = exe.run(generate_p, - feed={'noise': noise}, - fetch_list=[g_loss])[0] + static_d_loss = exe.run( + discriminate_p, + feed={'img': img, 'noise': noise}, + fetch_list=[d_loss], + )[0] + static_g_loss = exe.run( + generate_p, feed={'noise': noise}, fetch_list=[g_loss] + )[0] # generate_p contains all parameters needed. for param in generate_p.global_block().all_parameters(): static_params[param.name] = np.array( - scope.find_var(param.name).get_tensor()) + scope.find_var(param.name).get_tensor() + ) dy_params = dict() with fluid.dygraph.guard(): @@ -146,20 +152,28 @@ def func_test_gan_float32(self): discriminator = Discriminator() generator = Generator() - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=(discriminator.parameters() + - generator.parameters())) + sgd = SGDOptimizer( + learning_rate=1e-3, + parameter_list=( + discriminator.parameters() + generator.parameters() + ), + ) d_real = discriminator(to_variable(np.ones([2, 1], np.float32))) d_loss_real = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( - x=d_real, label=to_variable(np.ones([2, 1], np.float32)))) + x=d_real, label=to_variable(np.ones([2, 1], np.float32)) + ) + ) d_fake = discriminator( - generator(to_variable(np.ones([2, 2], np.float32)))) + generator(to_variable(np.ones([2, 2], np.float32))) + ) d_loss_fake = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( - x=d_fake, label=to_variable(np.zeros([2, 1], np.float32)))) + x=d_fake, label=to_variable(np.zeros([2, 1], np.float32)) + ) + ) d_loss = d_loss_real + d_loss_fake d_loss.backward() @@ -168,10 +182,13 @@ def func_test_gan_float32(self): generator.clear_gradients() d_fake = discriminator( - generator(to_variable(np.ones([2, 2], np.float32)))) + generator(to_variable(np.ones([2, 2], np.float32))) + ) g_loss = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( - x=d_fake, label=to_variable(np.ones([2, 1], np.float32)))) + x=d_fake, label=to_variable(np.ones([2, 1], np.float32)) + ) + ) g_loss.backward() sgd.minimize(g_loss) for p in discriminator.parameters(): @@ -189,20 +206,28 @@ def func_test_gan_float32(self): paddle.framework.random._manual_program_seed(1) discriminator2 = Discriminator() generator2 = Generator() - sgd2 = SGDOptimizer(learning_rate=1e-3, - parameter_list=(discriminator2.parameters() + - generator2.parameters())) + sgd2 = SGDOptimizer( + learning_rate=1e-3, + parameter_list=( + discriminator2.parameters() + generator2.parameters() + ), + ) d_real2 = discriminator2(to_variable(np.ones([2, 1], np.float32))) d_loss_real2 = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( - x=d_real2, label=to_variable(np.ones([2, 1], np.float32)))) + x=d_real2, label=to_variable(np.ones([2, 1], np.float32)) + ) + ) d_fake2 = discriminator2( - generator2(to_variable(np.ones([2, 2], np.float32)))) + generator2(to_variable(np.ones([2, 2], np.float32))) + ) d_loss_fake2 = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( - x=d_fake2, label=to_variable(np.zeros([2, 1], np.float32)))) + x=d_fake2, label=to_variable(np.zeros([2, 1], np.float32)) + ) + ) d_loss2 = d_loss_real2 + d_loss_fake2 d_loss2.backward() @@ -211,10 +236,13 @@ def func_test_gan_float32(self): generator2.clear_gradients() d_fake2 = discriminator2( - generator2(to_variable(np.ones([2, 2], np.float32)))) + generator2(to_variable(np.ones([2, 2], np.float32))) + ) g_loss2 = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( - x=d_fake2, label=to_variable(np.ones([2, 1], np.float32)))) + x=d_fake2, label=to_variable(np.ones([2, 1], np.float32)) + ) + ) g_loss2.backward() sgd2.minimize(g_loss2) for p in discriminator2.parameters(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_gnn.py b/python/paddle/fluid/tests/unittests/test_imperative_gnn.py index f001e7c4e0c6c..1d2e61b0863ac 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_gnn.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_gnn.py @@ -30,7 +30,6 @@ def gen_data(): class GraphConv(fluid.Layer): - def __init__(self, name_scope, in_features, out_features): super(GraphConv, self).__init__(name_scope) @@ -39,10 +38,11 @@ def __init__(self, name_scope, in_features, out_features): self.weight = self.create_parameter( attr=None, dtype='float32', - shape=[self._in_features, self._out_features]) - self.bias = self.create_parameter(attr=None, - dtype='float32', - shape=[self._out_features]) + shape=[self._in_features, self._out_features], + ) + self.bias = self.create_parameter( + attr=None, dtype='float32', shape=[self._out_features] + ) def forward(self, features, adj): support = fluid.layers.matmul(features, self.weight) @@ -51,7 +51,6 @@ def forward(self, features, adj): class GCN(fluid.Layer): - def __init__(self, name_scope, num_hidden): super(GCN, self).__init__(name_scope) self.gc = GraphConv(self.full_name(), num_hidden, 32) @@ -63,7 +62,6 @@ def forward(self, x, adj): class TestDygraphGNN(unittest.TestCase): - def func_gnn_float32(self): paddle.seed(90) paddle.framework.random._manual_program_seed(90) @@ -72,19 +70,25 @@ def func_gnn_float32(self): scope = fluid.core.Scope() with new_program_scope(main=main, startup=startup, scope=scope): - features = fluid.layers.data(name='features', - shape=[1, 100, 50], - dtype='float32', - append_batch_size=False) + features = fluid.layers.data( + name='features', + shape=[1, 100, 50], + dtype='float32', + append_batch_size=False, + ) # Use selected rows when it's supported. - adj = fluid.layers.data(name='adj', - shape=[1, 100, 100], - dtype='float32', - append_batch_size=False) - labels = fluid.layers.data(name='labels', - shape=[100, 1], - dtype='int64', - append_batch_size=False) + adj = fluid.layers.data( + name='adj', + shape=[1, 100, 100], + dtype='float32', + append_batch_size=False, + ) + labels = fluid.layers.data( + name='labels', + shape=[100, 1], + dtype='int64', + append_batch_size=False, + ) model = GCN('test_gcn', 50) logits = model(features, adj) @@ -96,21 +100,24 @@ def func_gnn_float32(self): adam = AdamOptimizer(learning_rate=1e-3) adam.minimize(loss) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe.run(startup) - static_loss = exe.run(feed={ - 'features': - np.ones([1, 100, 50], dtype=np.float32), - 'adj': - np.ones([1, 100, 100], dtype=np.float32), - 'labels': - np.ones([100, 1], dtype=np.int64) - }, - fetch_list=[loss])[0] + static_loss = exe.run( + feed={ + 'features': np.ones([1, 100, 50], dtype=np.float32), + 'adj': np.ones([1, 100, 100], dtype=np.float32), + 'labels': np.ones([100, 1], dtype=np.int64), + }, + fetch_list=[loss], + )[0] static_weight = np.array( - scope.find_var(model.gc.weight.name).get_tensor()) + scope.find_var(model.gc.weight.name).get_tensor() + ) with fluid.dygraph.guard(): paddle.seed(90) @@ -127,11 +134,13 @@ def func_gnn_float32(self): # In other example, it's nll with log_softmax. However, paddle's # log_loss only supports binary classification now. loss = fluid.layers.softmax_with_cross_entropy( - logits, to_variable(labels)) + logits, to_variable(labels) + ) loss = fluid.layers.reduce_sum(loss) loss.backward() - adam = AdamOptimizer(learning_rate=1e-3, - parameter_list=model.parameters()) + adam = AdamOptimizer( + learning_rate=1e-3, parameter_list=model.parameters() + ) adam.minimize(loss) model.clear_gradients() @@ -153,24 +162,26 @@ def func_gnn_float32(self): # In other example, it's nll with log_softmax. However, paddle's # log_loss only supports binary classification now. loss2 = fluid.layers.softmax_with_cross_entropy( - logits2, to_variable(labels2)) + logits2, to_variable(labels2) + ) loss2 = fluid.layers.reduce_sum(loss2) loss2.backward() - adam2 = AdamOptimizer(learning_rate=1e-3, - parameter_list=model2.parameters()) + adam2 = AdamOptimizer( + learning_rate=1e-3, parameter_list=model2.parameters() + ) adam2.minimize(loss2) model2.clear_gradients() loss2_value = loss2.numpy() model2_gc_weight_value = model2.gc.weight.numpy() self.assertEqual(static_loss, loss_value) - np.testing.assert_allclose(static_weight, - model_gc_weight_value, - rtol=1e-05) + np.testing.assert_allclose( + static_weight, model_gc_weight_value, rtol=1e-05 + ) self.assertEqual(static_loss, loss2_value) - np.testing.assert_allclose(static_weight, - model2_gc_weight_value, - rtol=1e-05) + np.testing.assert_allclose( + static_weight, model2_gc_weight_value, rtol=1e-05 + ) sys.stderr.write('%s %s\n' % (static_loss, loss_value)) def test_gnn_float32(self): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_group.py b/python/paddle/fluid/tests/unittests/test_imperative_group.py index f67592312c84a..672dee430f889 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_group.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_group.py @@ -17,11 +17,14 @@ import paddle import paddle.fluid.core as core -from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph, in_dygraph_mode +from paddle.fluid.framework import ( + _test_eager_guard, + _in_legacy_dygraph, + in_dygraph_mode, +) class TestDataParallelGroup(unittest.TestCase): - def create_varbase(self, dtype, shape): return paddle.rand(shape=shape, dtype=dtype) @@ -38,8 +41,9 @@ def test_construct_group0(self): var_list.append(self.create_varbase("float32", [2, 100])) var_list.append(self.create_varbase("float32", [2, 50])) var_list.append(self.create_varbase("float32", [2, 25])) - res = self.assign_group_by_size(var_list, [False, False, False, False], - [400]) + res = self.assign_group_by_size( + var_list, [False, False, False, False], [400] + ) self.assertEqual([[0], [1], [2], [3]], res) def test_construct_group1(self): @@ -52,7 +56,8 @@ def test_construct_group1(self): var_list.append(self.create_varbase("float32", [1, 50])) var_list.append(self.create_varbase("float64", [1, 25])) res = self.assign_group_by_size( - var_list, [False, False, False, False, False, False], [400]) + var_list, [False, False, False, False, False, False], [400] + ) self.assertEqual([[0, 2], [1, 3], [4], [5]], res) def test_construct_group2(self): @@ -62,8 +67,9 @@ def test_construct_group2(self): var_list.append(self.create_varbase("float32", [2, 50])) var_list.append(self.create_varbase("float32", [2, 50])) var_list.append(self.create_varbase("float32", [2, 50])) - res = self.assign_group_by_size(var_list, [False, False, False, False], - [400, 800]) + res = self.assign_group_by_size( + var_list, [False, False, False, False], [400, 800] + ) self.assertEqual([[0], [1, 2], [3]], res) def test_construct_group3(self): @@ -76,7 +82,8 @@ def test_construct_group3(self): var_list.append(self.create_varbase("float32", [1, 50])) var_list.append(self.create_varbase("float64", [1, 25])) res = self.assign_group_by_size( - var_list, [False, False, False, False, False, False], [200, 400]) + var_list, [False, False, False, False, False, False], [200, 400] + ) self.assertEqual([[0], [1], [2, 4], [3, 5]], res) def test_construct_group4(self): @@ -89,7 +96,8 @@ def test_construct_group4(self): var_list.append(self.create_varbase("float32", [1, 50])) var_list.append(self.create_varbase("float64", [1, 25])) res = self.assign_group_by_size( - var_list, [False, False, False, False, False, False], [0]) + var_list, [False, False, False, False, False, False], [0] + ) self.assertEqual([[0], [1], [2], [3], [4], [5]], res) def test_construct_group5(self): @@ -102,23 +110,27 @@ def test_construct_group5(self): var_list.append(self.create_varbase("float32", [1, 50])) var_list.append(self.create_varbase("float64", [1, 25])) res = self.assign_group_by_size( - var_list, [False, False, False, False, False, False], [10000]) + var_list, [False, False, False, False, False, False], [10000] + ) self.assertEqual([[0, 2, 4], [1, 3, 5]], res) def test_construct_group6(self): # multi dtype & limit capability & multi tensor type var_list = [] - var_list.append(self.create_varbase( - "float32", - [1, 50], - )) + var_list.append( + self.create_varbase( + "float32", + [1, 50], + ) + ) var_list.append(self.create_varbase("float64", [1, 25])) var_list.append(self.create_varbase("float32", [1, 50])) var_list.append(self.create_varbase("float64", [1, 25])) var_list.append(self.create_varbase("float32", [1, 50])) var_list.append(self.create_varbase("float64", [1, 25])) res = self.assign_group_by_size( - var_list, [True, False, False, False, False, True], [400]) + var_list, [True, False, False, False, False, True], [400] + ) self.assertEqual([[0], [1, 3], [2, 4], [5]], res) def test_construct_group7(self): @@ -131,7 +143,8 @@ def test_construct_group7(self): var_list.append(self.create_varbase("float32", [1, 50])) var_list.append(self.create_varbase("float64", [1, 25])) res = self.assign_group_by_size( - var_list, [True, False, False, False, False, True], [200, 400]) + var_list, [True, False, False, False, False, True], [200, 400] + ) self.assertEqual([[0], [1], [2], [3], [4], [5]], res) def test_construct_group8(self): @@ -141,8 +154,9 @@ def test_construct_group8(self): var_list.append(self.create_varbase("float32", [2, 100])) var_list.append(self.create_varbase("float32", [2, 50])) var_list.append(self.create_varbase("float32", [2, 25])) - res = self.assign_group_by_size(var_list, [False, False, False, False], - [400], [3, 0, 1, 2]) + res = self.assign_group_by_size( + var_list, [False, False, False, False], [400], [3, 0, 1, 2] + ) self.assertEqual([[3, 0], [1], [2]], res) def test_construct_group9(self): @@ -152,8 +166,9 @@ def test_construct_group9(self): var_list.append(self.create_varbase("float32", [2, 25])) var_list.append(self.create_varbase("float32", [2, 25])) var_list.append(self.create_varbase("float32", [2, 1000])) - res = self.assign_group_by_size(var_list, [False, False, False, True], - [300], [1, 0, 2, 3]) + res = self.assign_group_by_size( + var_list, [False, False, False, True], [300], [1, 0, 2, 3] + ) self.assertEqual([[1, 0], [3], [2]], res) def test_construct_group_in_legacy_mode(self): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_hook_for_layer.py b/python/paddle/fluid/tests/unittests/test_imperative_hook_for_layer.py index 1f60bf20e09c8..462f614defaf2 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_hook_for_layer.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_hook_for_layer.py @@ -60,27 +60,37 @@ def func_forward_hook_return_value(self): fluid.default_main_program().random_seed = seed fluid.set_flags({'FLAGS_sort_sum_gradient': True}) - input_word = np.array( - [0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, - 8]).reshape(6, 3).astype('int64') + input_word = ( + np.array( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8] + ) + .reshape(6, 3) + .astype('int64') + ) input_word1 = input_word * 2 input_word = input_word.reshape((-1, 3, 1)) input_word1 = input_word1.reshape((-1, 3, 1)) - y_data = np.array( - [1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, - 9]).reshape(6, 3).astype('int64') + y_data = ( + np.array( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9] + ) + .reshape(6, 3) + .astype('int64') + ) y_data = y_data.reshape((-1, 1)) input = base.to_variable(input_word) input1 = base.to_variable(input_word1) y = base.to_variable(y_data) - simplenet = SimpleNet(hidden_size=20, - vocab_size=32, - num_steps=3, - init_scale=0.1, - is_sparse=False, - dtype="float32") + simplenet = SimpleNet( + hidden_size=20, + vocab_size=32, + num_steps=3, + init_scale=0.1, + is_sparse=False, + dtype="float32", + ) # origin, don't register any hook outs_origin = simplenet(input, y) @@ -88,29 +98,35 @@ def func_forward_hook_return_value(self): # register forward_pre_hook forward_pre_hook_handle1 = simplenet.register_forward_pre_hook( - forward_pre_hook1) + forward_pre_hook1 + ) outs_pre_hook = simplenet(input, y) - np.testing.assert_array_equal(outs_pre_hook.numpy(), - outs_origin1.numpy()) + np.testing.assert_array_equal( + outs_pre_hook.numpy(), outs_origin1.numpy() + ) # remove forward_pre_hook forward_pre_hook_handle1.remove() outs_pre_hook = simplenet(input, y) - np.testing.assert_array_equal(outs_pre_hook.numpy(), - outs_origin.numpy()) + np.testing.assert_array_equal( + outs_pre_hook.numpy(), outs_origin.numpy() + ) # register forward_posst_hook - forward_post_hook_handle1 = simplenet.register_forward_post_hook( - forward_post_hook1) + forward_post_hook_handle1 = ( + simplenet.register_forward_post_hook(forward_post_hook1) + ) outs_forward_hook = simplenet(input, y) - np.testing.assert_array_equal(outs_forward_hook.numpy(), - outs_origin.numpy() * 2) + np.testing.assert_array_equal( + outs_forward_hook.numpy(), outs_origin.numpy() * 2 + ) # remove forward_post_hook forward_post_hook_handle1.remove() outs_forward_hook = simplenet(input, y) - np.testing.assert_array_equal(outs_forward_hook.numpy(), - outs_origin.numpy()) + np.testing.assert_array_equal( + outs_forward_hook.numpy(), outs_origin.numpy() + ) # test forward_pre_hook and forward_post_hook that don't have return value def func_forward_hook(self): @@ -129,24 +145,34 @@ def func_forward_hook(self): global call_forward_post_hook global call_forward_pre_hook - input_word = np.array( - [0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, - 8]).reshape(6, 3).astype('int64') + input_word = ( + np.array( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8] + ) + .reshape(6, 3) + .astype('int64') + ) input_word = input_word.reshape((-1, 3, 1)) - y_data = np.array( - [1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, - 9]).reshape(6, 3).astype('int64') + y_data = ( + np.array( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9] + ) + .reshape(6, 3) + .astype('int64') + ) y_data = y_data.reshape((-1, 1)) input = base.to_variable(input_word) y = base.to_variable(y_data) - simplenet = SimpleNet(hidden_size=20, - vocab_size=32, - num_steps=3, - init_scale=0.1, - is_sparse=False, - dtype="float32") + simplenet = SimpleNet( + hidden_size=20, + vocab_size=32, + num_steps=3, + init_scale=0.1, + is_sparse=False, + dtype="float32", + ) # origin, don't register any hook outs_origin = simplenet(input, y) @@ -155,9 +181,11 @@ def func_forward_hook(self): # register forward_post_hook and forward_pre_hook forward_post_hook_handle = simplenet.register_forward_post_hook( - forward_post_hook) + forward_post_hook + ) forward_pre_hook_handle = simplenet.register_forward_pre_hook( - forward_pre_hook) + forward_pre_hook + ) outs_hook = simplenet(input, y) self.assertTrue(call_forward_post_hook) self.assertTrue(call_forward_pre_hook) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_layer_apply.py b/python/paddle/fluid/tests/unittests/test_imperative_layer_apply.py index 58e56fc6a6d8c..40e24abeccdbd 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_layer_apply.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_layer_apply.py @@ -23,21 +23,25 @@ class LeNetDygraph(fluid.dygraph.Layer): - def __init__(self, num_classes=10, classifier_activation='softmax'): super(LeNetDygraph, self).__init__() self.num_classes = num_classes - self.features = nn.Sequential(nn.Conv2D(1, 6, 3, stride=1, padding=1), - nn.ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2), - nn.Conv2D(6, 16, 5, stride=1, padding=0), - nn.ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2)) + self.features = nn.Sequential( + nn.Conv2D(1, 6, 3, stride=1, padding=1), + nn.ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + nn.Conv2D(6, 16, 5, stride=1, padding=0), + nn.ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + ) if num_classes > 0: - self.fc = nn.Sequential(nn.Linear(400, 120), nn.Linear(120, 84), - nn.Linear(84, 10), - nn.Softmax()) #Todo: accept any activation + self.fc = nn.Sequential( + nn.Linear(400, 120), + nn.Linear(120, 84), + nn.Linear(84, 10), + nn.Softmax(), + ) # Todo: accept any activation def forward(self, inputs): x = self.features(inputs) @@ -50,27 +54,26 @@ def forward(self, inputs): def init_weights(layer): if type(layer) == nn.Linear: - new_weight = paddle.fluid.layers.fill_constant(layer.weight.shape, - layer.weight.dtype, - value=0.9) + new_weight = paddle.fluid.layers.fill_constant( + layer.weight.shape, layer.weight.dtype, value=0.9 + ) layer.weight.set_value(new_weight) - new_bias = paddle.fluid.layers.fill_constant(layer.bias.shape, - layer.bias.dtype, - value=-0.1) + new_bias = paddle.fluid.layers.fill_constant( + layer.bias.shape, layer.bias.dtype, value=-0.1 + ) layer.bias.set_value(new_bias) elif type(layer) == nn.Conv2D: - new_weight = paddle.fluid.layers.fill_constant(layer.weight.shape, - layer.weight.dtype, - value=0.7) + new_weight = paddle.fluid.layers.fill_constant( + layer.weight.shape, layer.weight.dtype, value=0.7 + ) layer.weight.set_value(new_weight) - new_bias = paddle.fluid.layers.fill_constant(layer.bias.shape, - layer.bias.dtype, - value=-0.2) + new_bias = paddle.fluid.layers.fill_constant( + layer.bias.shape, layer.bias.dtype, value=-0.2 + ) layer.bias.set_value(new_bias) class TestLayerApply(unittest.TestCase): - def func_apply_init_weight(self): with fluid.dygraph.guard(): net = LeNetDygraph() diff --git a/python/paddle/fluid/tests/unittests/test_imperative_layer_children.py b/python/paddle/fluid/tests/unittests/test_imperative_layer_children.py index ded833bd9a836..4eeaf8da85784 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_layer_children.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_layer_children.py @@ -23,15 +23,16 @@ class LeNetDygraph(fluid.dygraph.Layer): - def __init__(self): super(LeNetDygraph, self).__init__() - self.features = nn.Sequential(nn.Conv2D(1, 6, 3, stride=1, padding=1), - nn.ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2), - nn.Conv2D(6, 16, 5, stride=1, padding=0), - nn.ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2)) + self.features = nn.Sequential( + nn.Conv2D(1, 6, 3, stride=1, padding=1), + nn.ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + nn.Conv2D(6, 16, 5, stride=1, padding=0), + nn.ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + ) def forward(self, inputs): x = self.features(inputs) @@ -39,7 +40,6 @@ def forward(self, inputs): class TestLayerChildren(unittest.TestCase): - def func_apply_init_weight(self): with fluid.dygraph.guard(): net = LeNetDygraph() diff --git a/python/paddle/fluid/tests/unittests/test_imperative_layer_trainable.py b/python/paddle/fluid/tests/unittests/test_imperative_layer_trainable.py index c359d99c81946..b0dcfd653fb75 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_layer_trainable.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_layer_trainable.py @@ -21,7 +21,6 @@ class TestImperativeLayerTrainable(unittest.TestCase): - def func_set_trainable(self): with fluid.dygraph.guard(): label = np.random.uniform(-1, 1, [10, 10]).astype(np.float32) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_layers.py b/python/paddle/fluid/tests/unittests/test_imperative_layers.py index e2b9855af41c1..e0cd3d4f88f64 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_layers.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_layers.py @@ -18,7 +18,6 @@ class TestLayerPrint(unittest.TestCase): - def func_test_layer_str(self): module = nn.ELU(0.2) self.assertEqual(str(module), 'ELU(alpha=0.2)') @@ -39,13 +38,14 @@ def func_test_layer_str(self): self.assertEqual(str(module), 'Tanh(name=Tanh)') module = nn.Hardtanh(name="Hardtanh") - self.assertEqual(str(module), - 'Hardtanh(min=-1.0, max=1.0, name=Hardtanh)') + self.assertEqual( + str(module), 'Hardtanh(min=-1.0, max=1.0, name=Hardtanh)' + ) module = nn.PReLU(1, 0.25, name="PReLU", data_format="NCHW") self.assertEqual( str(module), - 'PReLU(num_parameters=1, data_format=NCHW, init=0.25, dtype=float32, name=PReLU)' + 'PReLU(num_parameters=1, data_format=NCHW, init=0.25, dtype=float32, name=PReLU)', ) module = nn.ReLU() @@ -57,7 +57,8 @@ def func_test_layer_str(self): module = nn.SELU() self.assertEqual( str(module), - 'SELU(scale=1.0507009873554805, alpha=1.6732632423543772)') + 'SELU(scale=1.0507009873554805, alpha=1.6732632423543772)', + ) module = nn.LeakyReLU() self.assertEqual(str(module), 'LeakyReLU(negative_slope=0.01)') @@ -101,32 +102,35 @@ def func_test_layer_str(self): module = nn.Linear(2, 4, name='linear') self.assertEqual( str(module), - 'Linear(in_features=2, out_features=4, dtype=float32, name=linear)') + 'Linear(in_features=2, out_features=4, dtype=float32, name=linear)', + ) module = nn.Upsample(size=[12, 12]) self.assertEqual( str(module), - 'Upsample(size=[12, 12], mode=nearest, align_corners=False, align_mode=0, data_format=NCHW)' + 'Upsample(size=[12, 12], mode=nearest, align_corners=False, align_mode=0, data_format=NCHW)', ) module = nn.UpsamplingNearest2D(size=[12, 12]) self.assertEqual( - str(module), 'UpsamplingNearest2D(size=[12, 12], data_format=NCHW)') + str(module), 'UpsamplingNearest2D(size=[12, 12], data_format=NCHW)' + ) module = nn.UpsamplingBilinear2D(size=[12, 12]) self.assertEqual( - str(module), - 'UpsamplingBilinear2D(size=[12, 12], data_format=NCHW)') + str(module), 'UpsamplingBilinear2D(size=[12, 12], data_format=NCHW)' + ) module = nn.Bilinear(in1_features=5, in2_features=4, out_features=1000) self.assertEqual( str(module), - 'Bilinear(in1_features=5, in2_features=4, out_features=1000, dtype=float32)' + 'Bilinear(in1_features=5, in2_features=4, out_features=1000, dtype=float32)', ) module = nn.Dropout(p=0.5) - self.assertEqual(str(module), - 'Dropout(p=0.5, axis=None, mode=upscale_in_train)') + self.assertEqual( + str(module), 'Dropout(p=0.5, axis=None, mode=upscale_in_train)' + ) module = nn.Dropout2D(p=0.5) self.assertEqual(str(module), 'Dropout2D(p=0.5, data_format=NCHW)') @@ -140,22 +144,24 @@ def func_test_layer_str(self): module = nn.Pad1D(padding=[1, 2], mode='constant') self.assertEqual( str(module), - 'Pad1D(padding=[1, 2], mode=constant, value=0.0, data_format=NCL)') + 'Pad1D(padding=[1, 2], mode=constant, value=0.0, data_format=NCL)', + ) module = nn.Pad2D(padding=[1, 0, 1, 2], mode='constant') self.assertEqual( str(module), - 'Pad2D(padding=[1, 0, 1, 2], mode=constant, value=0.0, data_format=NCHW)' + 'Pad2D(padding=[1, 0, 1, 2], mode=constant, value=0.0, data_format=NCHW)', ) module = nn.ZeroPad2D(padding=[1, 0, 1, 2]) - self.assertEqual(str(module), - 'ZeroPad2D(padding=[1, 0, 1, 2], data_format=NCHW)') + self.assertEqual( + str(module), 'ZeroPad2D(padding=[1, 0, 1, 2], data_format=NCHW)' + ) module = nn.Pad3D(padding=[1, 0, 1, 2, 0, 0], mode='constant') self.assertEqual( str(module), - 'Pad3D(padding=[1, 0, 1, 2, 0, 0], mode=constant, value=0.0, data_format=NCDHW)' + 'Pad3D(padding=[1, 0, 1, 2, 0, 0], mode=constant, value=0.0, data_format=NCDHW)', ) module = nn.CosineSimilarity(axis=0) @@ -165,107 +171,127 @@ def func_test_layer_str(self): self.assertEqual(str(module), 'Embedding(10, 3, sparse=True)') module = nn.Conv1D(3, 2, 3) - self.assertEqual(str(module), - 'Conv1D(3, 2, kernel_size=[3], data_format=NCL)') + self.assertEqual( + str(module), 'Conv1D(3, 2, kernel_size=[3], data_format=NCL)' + ) module = nn.Conv1DTranspose(2, 1, 2) self.assertEqual( str(module), - 'Conv1DTranspose(2, 1, kernel_size=[2], data_format=NCL)') + 'Conv1DTranspose(2, 1, kernel_size=[2], data_format=NCL)', + ) module = nn.Conv2D(4, 6, (3, 3)) - self.assertEqual(str(module), - 'Conv2D(4, 6, kernel_size=[3, 3], data_format=NCHW)') + self.assertEqual( + str(module), 'Conv2D(4, 6, kernel_size=[3, 3], data_format=NCHW)' + ) module = nn.Conv2DTranspose(4, 6, (3, 3)) self.assertEqual( str(module), - 'Conv2DTranspose(4, 6, kernel_size=[3, 3], data_format=NCHW)') + 'Conv2DTranspose(4, 6, kernel_size=[3, 3], data_format=NCHW)', + ) module = nn.Conv3D(4, 6, (3, 3, 3)) self.assertEqual( str(module), - 'Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)') + 'Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)', + ) module = nn.Conv3DTranspose(4, 6, (3, 3, 3)) self.assertEqual( str(module), - 'Conv3DTranspose(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)') + 'Conv3DTranspose(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)', + ) module = nn.PairwiseDistance() self.assertEqual(str(module), 'PairwiseDistance(p=2.0)') module = nn.InstanceNorm1D(2) - self.assertEqual(str(module), - 'InstanceNorm1D(num_features=2, epsilon=1e-05)') + self.assertEqual( + str(module), 'InstanceNorm1D(num_features=2, epsilon=1e-05)' + ) module = nn.InstanceNorm2D(2) - self.assertEqual(str(module), - 'InstanceNorm2D(num_features=2, epsilon=1e-05)') + self.assertEqual( + str(module), 'InstanceNorm2D(num_features=2, epsilon=1e-05)' + ) module = nn.InstanceNorm3D(2) - self.assertEqual(str(module), - 'InstanceNorm3D(num_features=2, epsilon=1e-05)') + self.assertEqual( + str(module), 'InstanceNorm3D(num_features=2, epsilon=1e-05)' + ) module = nn.GroupNorm(num_channels=6, num_groups=6) self.assertEqual( str(module), - 'GroupNorm(num_groups=6, num_channels=6, epsilon=1e-05)') + 'GroupNorm(num_groups=6, num_channels=6, epsilon=1e-05)', + ) module = nn.LayerNorm([2, 2, 3]) self.assertEqual( - str(module), 'LayerNorm(normalized_shape=[2, 2, 3], epsilon=1e-05)') + str(module), 'LayerNorm(normalized_shape=[2, 2, 3], epsilon=1e-05)' + ) module = nn.BatchNorm1D(1) self.assertEqual( str(module), - 'BatchNorm1D(num_features=1, momentum=0.9, epsilon=1e-05, data_format=NCL)' + 'BatchNorm1D(num_features=1, momentum=0.9, epsilon=1e-05, data_format=NCL)', ) module = nn.BatchNorm2D(1) self.assertEqual( str(module), - 'BatchNorm2D(num_features=1, momentum=0.9, epsilon=1e-05)') + 'BatchNorm2D(num_features=1, momentum=0.9, epsilon=1e-05)', + ) module = nn.BatchNorm3D(1) self.assertEqual( str(module), - 'BatchNorm3D(num_features=1, momentum=0.9, epsilon=1e-05, data_format=NCDHW)' + 'BatchNorm3D(num_features=1, momentum=0.9, epsilon=1e-05, data_format=NCDHW)', ) module = nn.SyncBatchNorm(2) self.assertEqual( str(module), - 'SyncBatchNorm(num_features=2, momentum=0.9, epsilon=1e-05)') + 'SyncBatchNorm(num_features=2, momentum=0.9, epsilon=1e-05)', + ) module = nn.LocalResponseNorm(size=5) self.assertEqual( str(module), - 'LocalResponseNorm(size=5, alpha=0.0001, beta=0.75, k=1.0)') + 'LocalResponseNorm(size=5, alpha=0.0001, beta=0.75, k=1.0)', + ) module = nn.AvgPool1D(kernel_size=2, stride=2, padding=0) - self.assertEqual(str(module), - 'AvgPool1D(kernel_size=2, stride=2, padding=0)') + self.assertEqual( + str(module), 'AvgPool1D(kernel_size=2, stride=2, padding=0)' + ) module = nn.AvgPool2D(kernel_size=2, stride=2, padding=0) - self.assertEqual(str(module), - 'AvgPool2D(kernel_size=2, stride=2, padding=0)') + self.assertEqual( + str(module), 'AvgPool2D(kernel_size=2, stride=2, padding=0)' + ) module = nn.AvgPool3D(kernel_size=2, stride=2, padding=0) - self.assertEqual(str(module), - 'AvgPool3D(kernel_size=2, stride=2, padding=0)') + self.assertEqual( + str(module), 'AvgPool3D(kernel_size=2, stride=2, padding=0)' + ) module = nn.MaxPool1D(kernel_size=2, stride=2, padding=0) - self.assertEqual(str(module), - 'MaxPool1D(kernel_size=2, stride=2, padding=0)') + self.assertEqual( + str(module), 'MaxPool1D(kernel_size=2, stride=2, padding=0)' + ) module = nn.MaxPool2D(kernel_size=2, stride=2, padding=0) - self.assertEqual(str(module), - 'MaxPool2D(kernel_size=2, stride=2, padding=0)') + self.assertEqual( + str(module), 'MaxPool2D(kernel_size=2, stride=2, padding=0)' + ) module = nn.MaxPool3D(kernel_size=2, stride=2, padding=0) - self.assertEqual(str(module), - 'MaxPool3D(kernel_size=2, stride=2, padding=0)') + self.assertEqual( + str(module), 'MaxPool3D(kernel_size=2, stride=2, padding=0)' + ) module = nn.AdaptiveAvgPool1D(output_size=16) self.assertEqual(str(module), 'AdaptiveAvgPool1D(output_size=16)') @@ -277,16 +303,19 @@ def func_test_layer_str(self): self.assertEqual(str(module), 'AdaptiveAvgPool3D(output_size=3)') module = nn.AdaptiveMaxPool1D(output_size=16, return_mask=True) - self.assertEqual(str(module), - 'AdaptiveMaxPool1D(output_size=16, return_mask=True)') + self.assertEqual( + str(module), 'AdaptiveMaxPool1D(output_size=16, return_mask=True)' + ) module = nn.AdaptiveMaxPool2D(output_size=3, return_mask=True) - self.assertEqual(str(module), - 'AdaptiveMaxPool2D(output_size=3, return_mask=True)') + self.assertEqual( + str(module), 'AdaptiveMaxPool2D(output_size=3, return_mask=True)' + ) module = nn.AdaptiveMaxPool3D(output_size=3, return_mask=True) - self.assertEqual(str(module), - 'AdaptiveMaxPool3D(output_size=3, return_mask=True)') + self.assertEqual( + str(module), 'AdaptiveMaxPool3D(output_size=3, return_mask=True)' + ) module = nn.SimpleRNNCell(16, 32) self.assertEqual(str(module), 'SimpleRNNCell(16, 32)') @@ -303,49 +332,56 @@ def func_test_layer_str(self): module = nn.SimpleRNN(16, 32, 2) self.assertEqual( str(module), - 'SimpleRNN(16, 32, num_layers=2\n (0): RNN(\n (cell): SimpleRNNCell(16, 32)\n )\n (1): RNN(\n (cell): SimpleRNNCell(32, 32)\n )\n)' + 'SimpleRNN(16, 32, num_layers=2\n (0): RNN(\n (cell): SimpleRNNCell(16, 32)\n )\n (1): RNN(\n (cell): SimpleRNNCell(32, 32)\n )\n)', ) module = nn.LSTM(16, 32, 2) self.assertEqual( str(module), - 'LSTM(16, 32, num_layers=2\n (0): RNN(\n (cell): LSTMCell(16, 32)\n )\n (1): RNN(\n (cell): LSTMCell(32, 32)\n )\n)' + 'LSTM(16, 32, num_layers=2\n (0): RNN(\n (cell): LSTMCell(16, 32)\n )\n (1): RNN(\n (cell): LSTMCell(32, 32)\n )\n)', ) module = nn.GRU(16, 32, 2) self.assertEqual( str(module), - 'GRU(16, 32, num_layers=2\n (0): RNN(\n (cell): GRUCell(16, 32)\n )\n (1): RNN(\n (cell): GRUCell(32, 32)\n )\n)' + 'GRU(16, 32, num_layers=2\n (0): RNN(\n (cell): GRUCell(16, 32)\n )\n (1): RNN(\n (cell): GRUCell(32, 32)\n )\n)', ) module1 = nn.Sequential( - ('conv1', nn.Conv2D(1, 20, 5)), ('relu1', nn.ReLU()), - ('conv2', nn.Conv2D(20, 64, 5)), ('relu2', nn.ReLU())) + ('conv1', nn.Conv2D(1, 20, 5)), + ('relu1', nn.ReLU()), + ('conv2', nn.Conv2D(20, 64, 5)), + ('relu2', nn.ReLU()), + ) self.assertEqual( str(module1), - 'Sequential(\n '\ - '(conv1): Conv2D(1, 20, kernel_size=[5, 5], data_format=NCHW)\n '\ - '(relu1): ReLU()\n '\ - '(conv2): Conv2D(20, 64, kernel_size=[5, 5], data_format=NCHW)\n '\ - '(relu2): ReLU()\n)' + 'Sequential(\n ' + '(conv1): Conv2D(1, 20, kernel_size=[5, 5], data_format=NCHW)\n ' + '(relu1): ReLU()\n ' + '(conv2): Conv2D(20, 64, kernel_size=[5, 5], data_format=NCHW)\n ' + '(relu2): ReLU()\n)', ) module2 = nn.Sequential( nn.Conv3DTranspose(4, 6, (3, 3, 3)), nn.AvgPool3D(kernel_size=2, stride=2, padding=0), - nn.Tanh(name="Tanh"), module1, nn.Conv3D(4, 6, (3, 3, 3)), - nn.MaxPool3D(kernel_size=2, stride=2, padding=0), nn.GELU(True)) + nn.Tanh(name="Tanh"), + module1, + nn.Conv3D(4, 6, (3, 3, 3)), + nn.MaxPool3D(kernel_size=2, stride=2, padding=0), + nn.GELU(True), + ) self.assertEqual( str(module2), - 'Sequential(\n '\ - '(0): Conv3DTranspose(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)\n '\ - '(1): AvgPool3D(kernel_size=2, stride=2, padding=0)\n '\ - '(2): Tanh(name=Tanh)\n '\ - '(3): Sequential(\n (conv1): Conv2D(1, 20, kernel_size=[5, 5], data_format=NCHW)\n (relu1): ReLU()\n'\ - ' (conv2): Conv2D(20, 64, kernel_size=[5, 5], data_format=NCHW)\n (relu2): ReLU()\n )\n '\ - '(4): Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)\n '\ - '(5): MaxPool3D(kernel_size=2, stride=2, padding=0)\n '\ - '(6): GELU(approximate=True)\n)' + 'Sequential(\n ' + '(0): Conv3DTranspose(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)\n ' + '(1): AvgPool3D(kernel_size=2, stride=2, padding=0)\n ' + '(2): Tanh(name=Tanh)\n ' + '(3): Sequential(\n (conv1): Conv2D(1, 20, kernel_size=[5, 5], data_format=NCHW)\n (relu1): ReLU()\n' + ' (conv2): Conv2D(20, 64, kernel_size=[5, 5], data_format=NCHW)\n (relu2): ReLU()\n )\n ' + '(4): Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)\n ' + '(5): MaxPool3D(kernel_size=2, stride=2, padding=0)\n ' + '(6): GELU(approximate=True)\n)', ) def test_layer_str(self): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py b/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py index e61671e39f169..27a27c3ce70f0 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_load_static_param.py @@ -15,14 +15,23 @@ import unittest import paddle.fluid as fluid import paddle.fluid.framework as framework -from paddle.fluid.dygraph.nn import BatchNorm, Conv2D, Conv3D, Embedding, GroupNorm, LayerNorm, Linear, NCE, PRelu +from paddle.fluid.dygraph.nn import ( + BatchNorm, + Conv2D, + Conv3D, + Embedding, + GroupNorm, + LayerNorm, + Linear, + NCE, + PRelu, +) import numpy as np import os import tempfile class TestDygraphLoadStatic(unittest.TestCase): - def testLoadStaticModel(self): # static mode temp_dir = tempfile.TemporaryDirectory() @@ -32,30 +41,26 @@ def testLoadStaticModel(self): fc_out1 = fluid.layers.fc(a, 10) fc_out2 = fluid.layers.fc(a, 20) - conv_out_1 = fluid.layers.conv2d(conv_in, - num_filters=10, - filter_size=5, - act="relu") - conv_out_2 = fluid.layers.conv2d(conv_in, - num_filters=10, - filter_size=5, - act="relu") - - conv3d_in = fluid.data(name='conv3d_in', - shape=[None, 3, 12, 32, 32], - dtype='float32') - conv3d_out_1 = fluid.layers.conv3d(input=conv3d_in, - num_filters=2, - filter_size=3, - act="relu") - conv3d_out_2 = fluid.layers.conv3d(input=conv3d_in, - num_filters=2, - filter_size=3, - act="relu") - - batchnorm_in = fluid.data(name="batchnorm_in", - shape=[None, 10], - dtype='float32') + conv_out_1 = fluid.layers.conv2d( + conv_in, num_filters=10, filter_size=5, act="relu" + ) + conv_out_2 = fluid.layers.conv2d( + conv_in, num_filters=10, filter_size=5, act="relu" + ) + + conv3d_in = fluid.data( + name='conv3d_in', shape=[None, 3, 12, 32, 32], dtype='float32' + ) + conv3d_out_1 = fluid.layers.conv3d( + input=conv3d_in, num_filters=2, filter_size=3, act="relu" + ) + conv3d_out_2 = fluid.layers.conv3d( + input=conv3d_in, num_filters=2, filter_size=3, act="relu" + ) + + batchnorm_in = fluid.data( + name="batchnorm_in", shape=[None, 10], dtype='float32' + ) batchnorm_out_1 = fluid.layers.batch_norm(batchnorm_in) batchnorm_out_2 = fluid.layers.batch_norm(batchnorm_in) @@ -68,53 +73,56 @@ def testLoadStaticModel(self): layernorm_2 = fluid.layers.layer_norm(layernorm) nce_in = fluid.data(name="nce_in", shape=[None, 100], dtype='float32') - nce_label = fluid.data(name="nce_label", - shape=[None, 10], - dtype='int64') + nce_label = fluid.data( + name="nce_label", shape=[None, 10], dtype='int64' + ) nce_out_1 = fluid.layers.nce(nce_in, nce_label, 10000) nce_out_2 = fluid.layers.nce(nce_in, nce_label, 10000) - prelu_in = fluid.data(name="prelu_in", - shape=[None, 5, 10, 10], - dtype='float32') + prelu_in = fluid.data( + name="prelu_in", shape=[None, 5, 10, 10], dtype='float32' + ) prelu_out_1 = fluid.layers.prelu(prelu_in, "channel") prelu_out_2 = fluid.layers.prelu(prelu_in, "channel") - bilinear_tensor_pro_x = fluid.data("t1", - shape=[None, 5], - dtype="float32") - bilinear_tensor_pro_y = fluid.data("t2", - shape=[None, 4], - dtype="float32") + bilinear_tensor_pro_x = fluid.data( + "t1", shape=[None, 5], dtype="float32" + ) + bilinear_tensor_pro_y = fluid.data( + "t2", shape=[None, 4], dtype="float32" + ) bilinear_tensor_pro_out_1 = fluid.layers.bilinear_tensor_product( - x=bilinear_tensor_pro_x, y=bilinear_tensor_pro_y, size=1000) + x=bilinear_tensor_pro_x, y=bilinear_tensor_pro_y, size=1000 + ) bilinear_tensor_pro_out_2 = fluid.layers.bilinear_tensor_product( - x=bilinear_tensor_pro_x, y=bilinear_tensor_pro_y, size=1000) - - conv2d_trans_in = fluid.data(name="conv2d_trans_in", - shape=[None, 10, 10, 10]) - - conv2d_trans_out_1 = fluid.layers.conv2d_transpose(conv2d_trans_in, - num_filters=10, - filter_size=5, - act="relu") - conv2d_trans_out_2 = fluid.layers.conv2d_transpose(conv2d_trans_in, - num_filters=10, - filter_size=5, - act="relu") - - conv3d_trans_in = fluid.data(name='conv3d_trans_in', - shape=[None, 3, 12, 32, 32], - dtype='float32') + x=bilinear_tensor_pro_x, y=bilinear_tensor_pro_y, size=1000 + ) + + conv2d_trans_in = fluid.data( + name="conv2d_trans_in", shape=[None, 10, 10, 10] + ) + + conv2d_trans_out_1 = fluid.layers.conv2d_transpose( + conv2d_trans_in, num_filters=10, filter_size=5, act="relu" + ) + conv2d_trans_out_2 = fluid.layers.conv2d_transpose( + conv2d_trans_in, num_filters=10, filter_size=5, act="relu" + ) + + conv3d_trans_in = fluid.data( + name='conv3d_trans_in', shape=[None, 3, 12, 32, 32], dtype='float32' + ) conv3d_trans_out_1 = fluid.layers.conv3d_transpose( - input=conv3d_trans_in, num_filters=2, filter_size=3, act="relu") + input=conv3d_trans_in, num_filters=2, filter_size=3, act="relu" + ) conv3d_trans_out_2 = fluid.layers.conv3d_transpose( - input=conv3d_trans_in, num_filters=2, filter_size=3, act="relu") + input=conv3d_trans_in, num_filters=2, filter_size=3, act="relu" + ) - groupnorm_in = fluid.data(name='groupnorm_in', - shape=[None, 8, 32, 32], - dtype='float32') + groupnorm_in = fluid.data( + name='groupnorm_in', shape=[None, 8, 32, 32], dtype='float32' + ) groupnorm_out1 = fluid.layers.group_norm(input=groupnorm_in, groups=4) groupnorm_out2 = fluid.layers.group_norm(input=groupnorm_in, groups=4) ''' @@ -123,39 +131,47 @@ def testLoadStaticModel(self): spe_norm_out_2 = fluid.layers.spectral_norm(weight=spec_norm, dim=1, power_iters=2) ''' - nodes_vector = fluid.data(name='vectors', - shape=[None, 10, 5], - dtype='float32') - edge_set = fluid.data(name='edge_set', - shape=[None, 10, 2], - dtype='float32') - tree_conv_out1 = fluid.contrib.layers.tree_conv(nodes_vector, edge_set, - 6, 1, 2) - tree_conv_out2 = fluid.contrib.layers.tree_conv(nodes_vector, edge_set, - 6, 1, 2) - - para1 = fluid.layers.create_parameter([100, 100], - 'float32', - name="weight_test_1") - para2 = fluid.layers.create_parameter([20, 200], - 'float32', - name="weight_test_2") + nodes_vector = fluid.data( + name='vectors', shape=[None, 10, 5], dtype='float32' + ) + edge_set = fluid.data( + name='edge_set', shape=[None, 10, 2], dtype='float32' + ) + tree_conv_out1 = fluid.contrib.layers.tree_conv( + nodes_vector, edge_set, 6, 1, 2 + ) + tree_conv_out2 = fluid.contrib.layers.tree_conv( + nodes_vector, edge_set, 6, 1, 2 + ) + + para1 = fluid.layers.create_parameter( + [100, 100], 'float32', name="weight_test_1" + ) + para2 = fluid.layers.create_parameter( + [20, 200], 'float32', name="weight_test_2" + ) para_list = fluid.default_main_program().list_vars() - exe = fluid.Executor(fluid.CPUPlace( - ) if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not fluid.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) out = exe.run(framework.default_startup_program()) - fluid.save(framework.default_main_program(), - os.path.join(temp_dir.name, "test_1")) + fluid.save( + framework.default_main_program(), + os.path.join(temp_dir.name, "test_1"), + ) para_dict = fluid.load_program_state( - os.path.join(temp_dir.name, "test_1")) + os.path.join(temp_dir.name, "test_1") + ) new_dict = {} for k, v in para_dict.items(): - #print( k, v.shape ) + # print( k, v.shape ) if k.startswith("fc"): name = k.replace("fc", "linear", 1) new_dict[name] = v @@ -165,30 +181,31 @@ def testLoadStaticModel(self): with fluid.dygraph.guard(): class MyTest(fluid.dygraph.Layer): - def __init__(self): super(MyTest, self).__init__() self.linear1 = Linear(10, 10) self.lienar2 = Linear(10, 20) - self.conv2d_1 = Conv2D(num_channels=10, - num_filters=10, - filter_size=5, - act="relu") - self.conv2d_2 = Conv2D(num_channels=10, - num_filters=10, - filter_size=5, - act="relu") - - self.conv3d_1 = Conv3D(num_channels=3, - num_filters=2, - filter_size=3, - act="relu") - self.conv3d_2 = Conv3D(num_channels=3, - num_filters=2, - filter_size=3, - act="relu") + self.conv2d_1 = Conv2D( + num_channels=10, + num_filters=10, + filter_size=5, + act="relu", + ) + self.conv2d_2 = Conv2D( + num_channels=10, + num_filters=10, + filter_size=5, + act="relu", + ) + + self.conv3d_1 = Conv3D( + num_channels=3, num_filters=2, filter_size=3, act="relu" + ) + self.conv3d_2 = Conv3D( + num_channels=3, num_filters=2, filter_size=3, act="relu" + ) self.batch_norm_1 = BatchNorm(10) self.batch_norm_2 = BatchNorm(10) @@ -208,12 +225,12 @@ def __init__(self): self.group_norm1 = GroupNorm(8, 4) self.gourp_norm2 = GroupNorm(8, 4) - self.w_1 = self.create_parameter([100, 100], - dtype='float32', - attr="weight_test_1") - self.w_2 = self.create_parameter([20, 200], - dtype='float32', - attr="weight_test_2") + self.w_1 = self.create_parameter( + [100, 100], dtype='float32', attr="weight_test_1" + ) + self.w_2 = self.create_parameter( + [20, 200], dtype='float32', attr="weight_test_2" + ) my_test = MyTest() my_test.set_dict(new_dict, use_structured_name=False) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_lod_tensor_to_selected_rows.py b/python/paddle/fluid/tests/unittests/test_imperative_lod_tensor_to_selected_rows.py index a359d43d9c6aa..3e74ba69d5221 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_lod_tensor_to_selected_rows.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_lod_tensor_to_selected_rows.py @@ -26,14 +26,15 @@ class SimpleNet(fluid.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_steps=20, - init_scale=0.1, - is_sparse=False, - dtype='float32'): + def __init__( + self, + hidden_size, + vocab_size, + num_steps=20, + init_scale=0.1, + is_sparse=False, + dtype='float32', + ): super(SimpleNet, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -46,24 +47,31 @@ def __init__(self, param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype=dtype, default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label): x_emb = self.embedding(input) projection = fluid.layers.matmul( - x_emb, fluid.layers.transpose(self.embedding.weight, perm=[1, 0])) + x_emb, fluid.layers.transpose(self.embedding.weight, perm=[1, 0]) + ) projection = fluid.layers.elementwise_add(projection, self.softmax_bias) - projection = fluid.layers.reshape(projection, - shape=[-1, self.vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + projection = fluid.layers.reshape( + projection, shape=[-1, self.vocab_size] + ) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -72,7 +80,6 @@ def forward(self, input, label): class TestDygraphSimpleNet(unittest.TestCase): - def func_simple_net(self): for is_sparse in [True, False]: dtype_list = ["float32"] @@ -105,22 +112,27 @@ def simple_net_float32(self, is_sparse, dtype): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - simple_net = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=is_sparse, - dtype=dtype) - - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=simple_net.parameters()) + simple_net = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=is_sparse, + dtype=dtype, + ) + + sgd = SGDOptimizer( + learning_rate=1e-3, + parameter_list=simple_net.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None helper = DyGraphProgramDescTracerTestHelper(self) fluid.set_flags( - {'FLAGS_sort_sum_gradient': is_sort_sum_gradient}) + {'FLAGS_sort_sum_gradient': is_sort_sum_gradient} + ) for i in range(batch_num): x_data = np.arange(12).reshape(4, 3).astype('int64') @@ -147,17 +159,19 @@ def simple_net_float32(self, is_sparse, dtype): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - simple_net = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - is_sparse=is_sparse, - dtype=dtype) + simple_net = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + is_sparse=is_sparse, + dtype=dtype, + ) exe = fluid.Executor(place) sgd = SGDOptimizer(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype=dtype) static_loss = simple_net(x, y) @@ -168,8 +182,10 @@ def simple_net_float32(self, is_sparse, dtype): for param in simple_net.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init[static_param_name_list[i]] = out[i] static_loss_value = None @@ -180,22 +196,22 @@ def simple_net_float32(self, is_sparse, dtype): y_data = y_data.reshape((-1, 1)) fetch_list = [static_loss] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"x": x_data, "y": y_data}, + fetch_list=fetch_list, + ) static_loss_value = out[0] if i == batch_num - 1: for k in range(3, len(out)): - static_param_updated[static_param_name_list[ - k - 1]] = out[k] + static_param_updated[ + static_param_name_list[k - 1] + ] = out[k] - np.testing.assert_allclose(static_loss_value, - dy_loss_value, - rtol=0.001) + np.testing.assert_allclose( + static_loss_value, dy_loss_value, rtol=0.001 + ) for key, value in static_param_init.items(): np.testing.assert_array_equal(value, dy_param_init[key]) for key, value in static_param_updated.items(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_mnist.py b/python/paddle/fluid/tests/unittests/test_imperative_mnist.py index 22954d11e283b..41b0b365e7afd 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_mnist.py @@ -26,43 +26,48 @@ class SimpleImgConvPool(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - pool_size, - pool_stride, - pool_padding=0, - pool_type='max', - global_pooling=False, - conv_stride=1, - conv_padding=0, - conv_dilation=1, - conv_groups=1, - act=None, - use_cudnn=False, - param_attr=None, - bias_attr=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + pool_size, + pool_stride, + pool_padding=0, + pool_type='max', + global_pooling=False, + conv_stride=1, + conv_padding=0, + conv_dilation=1, + conv_groups=1, + act=None, + use_cudnn=False, + param_attr=None, + bias_attr=None, + ): super(SimpleImgConvPool, self).__init__() - self._conv2d = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=conv_stride, - padding=conv_padding, - dilation=conv_dilation, - groups=conv_groups, - param_attr=None, - bias_attr=None, - use_cudnn=use_cudnn) - - self._pool2d = Pool2D(pool_size=pool_size, - pool_type=pool_type, - pool_stride=pool_stride, - pool_padding=pool_padding, - global_pooling=global_pooling, - use_cudnn=use_cudnn) + self._conv2d = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=conv_stride, + padding=conv_padding, + dilation=conv_dilation, + groups=conv_groups, + param_attr=None, + bias_attr=None, + use_cudnn=use_cudnn, + ) + + self._pool2d = Pool2D( + pool_size=pool_size, + pool_type=pool_type, + pool_stride=pool_stride, + pool_padding=pool_padding, + global_pooling=global_pooling, + use_cudnn=use_cudnn, + ) def forward(self, inputs): x = self._conv2d(inputs) @@ -71,33 +76,30 @@ def forward(self, inputs): class MNIST(fluid.dygraph.Layer): - def __init__(self): super(MNIST, self).__init__() - self._simple_img_conv_pool_1 = SimpleImgConvPool(1, - 20, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_1 = SimpleImgConvPool( + 1, 20, 5, 2, 2, act="relu" + ) - self._simple_img_conv_pool_2 = SimpleImgConvPool(20, - 50, - 5, - 2, - 2, - act="relu") + self._simple_img_conv_pool_2 = SimpleImgConvPool( + 20, 50, 5, 2, 2, act="relu" + ) self.pool_2_shape = 50 * 4 * 4 SIZE = 10 - scale = (2.0 / (self.pool_2_shape**2 * SIZE))**0.5 - self._fc = Linear(self.pool_2_shape, - 10, - param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.NormalInitializer( - loc=0.0, scale=scale)), - act="softmax") + scale = (2.0 / (self.pool_2_shape**2 * SIZE)) ** 0.5 + self._fc = Linear( + self.pool_2_shape, + 10, + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.NormalInitializer( + loc=0.0, scale=scale + ) + ), + act="softmax", + ) def forward(self, inputs): x = self._simple_img_conv_pool_1(inputs) @@ -108,9 +110,7 @@ def forward(self, inputs): class TestImperativeMnist(unittest.TestCase): - def reader_decorator(self, reader): - def _reader_imple(): for item in reader(): image = np.array(item[0]).reshape(1, 28, 28) @@ -132,16 +132,19 @@ def func_test_mnist_float32(self): fluid.default_main_program().random_seed = seed mnist = MNIST() - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=mnist.parameters()) + sgd = SGDOptimizer( + learning_rate=1e-3, parameter_list=mnist.parameters() + ) batch_py_reader = fluid.io.PyReader(capacity=1) batch_py_reader.decorate_sample_list_generator( - paddle.batch(self.reader_decorator( - paddle.dataset.mnist.train()), - batch_size=batch_size, - drop_last=True), - places=fluid.CPUPlace()) + paddle.batch( + self.reader_decorator(paddle.dataset.mnist.train()), + batch_size=batch_size, + drop_last=True, + ), + places=fluid.CPUPlace(), + ) mnist.train() dy_param_init_value = {} @@ -159,12 +162,14 @@ def func_test_mnist_float32(self): if batch_id % 10 == 0 and _in_legacy_dygraph(): cost, traced_layer = paddle.jit.TracedLayer.trace( - mnist, inputs=img) + mnist, inputs=img + ) if program is not None: self.assertTrue(program, traced_layer.program) program = traced_layer.program traced_layer.save_inference_model( - './infer_imperative_mnist') + './infer_imperative_mnist' + ) else: cost = mnist(img) @@ -193,18 +198,23 @@ def func_test_mnist_float32(self): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) mnist = MNIST() sgd = SGDOptimizer(learning_rate=1e-3) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=batch_size, - drop_last=True) - - img = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype='float32') + train_reader = paddle.batch( + paddle.dataset.mnist.train(), + batch_size=batch_size, + drop_last=True, + ) + + img = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') cost = mnist(img) loss = fluid.layers.cross_entropy(cost, label) @@ -217,8 +227,10 @@ def func_test_mnist_float32(self): for param in mnist.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -227,12 +239,14 @@ def func_test_mnist_float32(self): for batch_id, data in enumerate(train_reader()): if batch_id >= batch_num: break - static_x_data = np.array([ - x[0].reshape(1, 28, 28) for x in data - ]).astype('float32') - y_data = np.array([x[1] - for x in data]).astype('int64').reshape( - [batch_size, 1]) + static_x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([batch_size, 1]) + ) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) @@ -240,35 +254,34 @@ def func_test_mnist_float32(self): if traced_layer is not None: traced_layer([static_x_data]) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) static_param_value = {} static_out = out[0] for i in range(1, len(out)): - static_param_value[static_param_name_list[i - - 1]] = out[i] + static_param_value[static_param_name_list[i - 1]] = out[ + i + ] - np.testing.assert_allclose(dy_x_data.all(), - static_x_data.all(), - rtol=1e-05) + np.testing.assert_allclose( + dy_x_data.all(), static_x_data.all(), rtol=1e-05 + ) for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value[key], rtol=1e-05 + ) np.testing.assert_allclose(static_out, dy_out, rtol=1e-05) for key, value in static_param_value.items(): - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05, atol=1e-05 + ) def test_mnist_float32(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_mnist_sorted_gradient.py b/python/paddle/fluid/tests/unittests/test_imperative_mnist_sorted_gradient.py index 214214f871020..1a077311e0541 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_mnist_sorted_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_mnist_sorted_gradient.py @@ -26,7 +26,6 @@ class TestImperativeMnistSortGradient(unittest.TestCase): - def func_test_mnist_sort_gradient_float32(self): seed = 90 epoch_num = 1 @@ -37,21 +36,25 @@ def func_test_mnist_sort_gradient_float32(self): fluid.set_flags({'FLAGS_sort_sum_gradient': True}) mnist2 = MNIST() - sgd2 = SGDOptimizer(learning_rate=1e-3, - parameter_list=mnist2.parameters()) - train_reader2 = paddle.batch(paddle.dataset.mnist.train(), - batch_size=128, - drop_last=True) + sgd2 = SGDOptimizer( + learning_rate=1e-3, parameter_list=mnist2.parameters() + ) + train_reader2 = paddle.batch( + paddle.dataset.mnist.train(), batch_size=128, drop_last=True + ) mnist2.train() dy_param_init_value2 = {} for epoch in range(epoch_num): for batch_id, data in enumerate(train_reader2()): - dy_x_data2 = np.array([ - x[0].reshape(1, 28, 28) for x in data - ]).astype('float32') - y_data2 = np.array([x[1] for x in data - ]).astype('int64').reshape(128, 1) + dy_x_data2 = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data2 = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(128, 1) + ) img2 = to_variable(dy_x_data2) label2 = to_variable(y_data2) @@ -81,18 +84,21 @@ def func_test_mnist_sort_gradient_float32(self): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) mnist = MNIST() sgd = SGDOptimizer(learning_rate=1e-3) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=128, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=128, drop_last=True + ) - img = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype='float32') + img = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') cost = mnist(img) loss = fluid.layers.cross_entropy(cost, label) @@ -105,53 +111,57 @@ def func_test_mnist_sort_gradient_float32(self): for param in mnist.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] for epoch in range(epoch_num): for batch_id, data in enumerate(train_reader()): - static_x_data = np.array([ - x[0].reshape(1, 28, 28) for x in data - ]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape([128, 1]) + static_x_data = np.array( + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([128, 1]) + ) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) static_param_value = {} static_out = out[0] for i in range(1, len(out)): - static_param_value[static_param_name_list[i - - 1]] = out[i] + static_param_value[static_param_name_list[i - 1]] = out[ + i + ] if batch_id == 20: break - np.testing.assert_allclose(dy_x_data2.all(), - static_x_data.all(), - rtol=1e-05) + np.testing.assert_allclose( + dy_x_data2.all(), static_x_data.all(), rtol=1e-05 + ) for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value2[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value2[key], rtol=1e-05 + ) np.testing.assert_allclose(static_out, dy_out2, rtol=1e-05) for key, value in static_param_value.items(): - np.testing.assert_allclose(value, - dy_param_value2[key], - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + value, dy_param_value2[key], rtol=1e-05, atol=1e-05 + ) def test_mnist_sort_gradient_float32(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_named_members.py b/python/paddle/fluid/tests/unittests/test_imperative_named_members.py index 57fad812bf8ec..7043eb0c2e778 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_named_members.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_named_members.py @@ -20,7 +20,6 @@ class MyLayer(fluid.Layer): - def __init__(self, num_channel, dim, num_filter=5): super(MyLayer, self).__init__() self.fc = fluid.dygraph.Linear(dim, dim) @@ -33,7 +32,6 @@ def forward(self, x): class TestImperativeNamedSubLayers(unittest.TestCase): - def func_test_named_sublayers(self): with fluid.dygraph.guard(): fc1 = fluid.Linear(10, 3) @@ -45,21 +43,22 @@ def func_test_named_sublayers(self): expected_sublayers = [fc1, fc2, custom, custom.fc, custom.conv] self.assertEqual(len(list_named_sublayers), len(expected_sublayers)) - for (name, - sublayer), expected_sublayer in zip(list_named_sublayers, - expected_sublayers): + for (name, sublayer), expected_sublayer in zip( + list_named_sublayers, expected_sublayers + ): self.assertEqual(sublayer, expected_sublayer) list_sublayers = list(model.sublayers()) self.assertEqual(len(list_named_sublayers), len(list_sublayers)) - for (name, - sublayer), expected_sublayer in zip(list_named_sublayers, - list_sublayers): + for (name, sublayer), expected_sublayer in zip( + list_named_sublayers, list_sublayers + ): self.assertEqual(sublayer, expected_sublayer) self.assertListEqual( [l for _, l in list(model.named_sublayers(include_self=True))], - [model] + expected_sublayers) + [model] + expected_sublayers, + ) def test_named_sublayers(self): with _test_eager_guard(): @@ -68,7 +67,6 @@ def test_named_sublayers(self): class TestImperativeNamedParameters(unittest.TestCase): - def func_test_named_parameters(self): with fluid.dygraph.guard(): fc1 = fluid.Linear(10, 3) @@ -80,7 +78,8 @@ def func_test_named_parameters(self): expected_named_parameters = list() for prefix, layer in model.named_sublayers(): for name, param in layer.named_parameters( - include_sublayers=False): + include_sublayers=False + ): full_name = prefix + ('.' if prefix else '') + name expected_named_parameters.append((full_name, param)) @@ -95,7 +94,6 @@ def func_test_dir_layer(self): with fluid.dygraph.guard(): class Mymodel(fluid.dygraph.Layer): - def __init__(self): super(Mymodel, self).__init__() self.linear1 = fluid.dygraph.Linear(10, 10) @@ -103,28 +101,42 @@ def __init__(self): self.conv2d = fluid.dygraph.Conv2D(3, 2, 3) self.embedding = fluid.dygraph.Embedding(size=[128, 16]) self.h_0 = fluid.dygraph.to_variable( - np.zeros([10, 10]).astype('float32')) - self.weight = self.create_parameter(shape=[2, 3], - attr=fluid.ParamAttr(), - dtype="float32", - is_bias=False) + np.zeros([10, 10]).astype('float32') + ) + self.weight = self.create_parameter( + shape=[2, 3], + attr=fluid.ParamAttr(), + dtype="float32", + is_bias=False, + ) model = Mymodel() expected_members = dir(model) - self.assertTrue("linear1" in expected_members, - "model should contain Layer: linear1") - self.assertTrue("linear2" in expected_members, - "model should contain Layer: linear2") - self.assertTrue("conv2d" in expected_members, - "model should contain Layer: conv2d") - self.assertTrue("embedding" in expected_members, - "model should contain Layer: embedding") - self.assertTrue("h_0" in expected_members, - "model should contain buffer: h_0") - self.assertTrue("weight" in expected_members, - "model should contain parameter: weight") + self.assertTrue( + "linear1" in expected_members, + "model should contain Layer: linear1", + ) + self.assertTrue( + "linear2" in expected_members, + "model should contain Layer: linear2", + ) + self.assertTrue( + "conv2d" in expected_members, + "model should contain Layer: conv2d", + ) + self.assertTrue( + "embedding" in expected_members, + "model should contain Layer: embedding", + ) + self.assertTrue( + "h_0" in expected_members, "model should contain buffer: h_0" + ) + self.assertTrue( + "weight" in expected_members, + "model should contain parameter: weight", + ) def test_dir_layer(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_numpy_bridge.py b/python/paddle/fluid/tests/unittests/test_imperative_numpy_bridge.py index a93471a09c9c3..4e333f7ca4e0f 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_numpy_bridge.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_numpy_bridge.py @@ -21,15 +21,16 @@ class TestImperativeNumpyBridge(unittest.TestCase): - def func_tensor_from_numpy(self): data_np = np.array([[2, 3, 1]]).astype('float32') with fluid.dygraph.guard(fluid.CPUPlace()): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") var = fluid.dygraph.to_variable(data_np, zero_copy=True) - assert "Currently, zero_copy is not supported, and it will be discarded." in str( - w[-1].message) + assert ( + "Currently, zero_copy is not supported, and it will be discarded." + in str(w[-1].message) + ) # Temporally diable zero_copy # var = fluid.dygraph.to_variable(data_np, zero_copy=True) # np.testing.assert_array_equal(var.numpy(), data_np) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_ocr_attention_model.py b/python/paddle/fluid/tests/unittests/test_imperative_ocr_attention_model.py index eb582310df46b..7971bdd34cd4c 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_ocr_attention_model.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_ocr_attention_model.py @@ -17,7 +17,14 @@ import paddle import paddle.fluid as fluid from paddle.fluid import core -from paddle.fluid.dygraph.nn import Conv2D, Pool2D, Linear, BatchNorm, Embedding, GRUUnit +from paddle.fluid.dygraph.nn import ( + Conv2D, + Pool2D, + Linear, + BatchNorm, + Embedding, + GRUUnit, +) from paddle.fluid.dygraph.base import to_variable from test_imperative_base import new_program_scope from paddle.fluid.framework import _test_eager_guard @@ -27,6 +34,7 @@ class Config(object): ''' config for training ''' + # encoder rnn hidden_size encoder_size = 8 # decoder size for decoder stage @@ -58,53 +66,62 @@ class Config(object): class ConvBNPool(fluid.dygraph.Layer): - - def __init__(self, - group, - out_ch, - channels, - act="relu", - is_test=False, - pool=True, - use_cudnn=True): + def __init__( + self, + group, + out_ch, + channels, + act="relu", + is_test=False, + pool=True, + use_cudnn=True, + ): super(ConvBNPool, self).__init__() self.group = group self.pool = pool filter_size = 3 - conv_std_0 = (2.0 / (filter_size**2 * channels[0]))**0.5 + conv_std_0 = (2.0 / (filter_size**2 * channels[0])) ** 0.5 conv_param_0 = fluid.ParamAttr( - initializer=fluid.initializer.Normal(0.0, conv_std_0)) + initializer=fluid.initializer.Normal(0.0, conv_std_0) + ) - conv_std_1 = (2.0 / (filter_size**2 * channels[1]))**0.5 + conv_std_1 = (2.0 / (filter_size**2 * channels[1])) ** 0.5 conv_param_1 = fluid.ParamAttr( - initializer=fluid.initializer.Normal(0.0, conv_std_1)) - - self.conv_0_layer = Conv2D(channels[0], - out_ch[0], - 3, - padding=1, - param_attr=conv_param_0, - bias_attr=False, - act=None, - use_cudnn=use_cudnn) + initializer=fluid.initializer.Normal(0.0, conv_std_1) + ) + + self.conv_0_layer = Conv2D( + channels[0], + out_ch[0], + 3, + padding=1, + param_attr=conv_param_0, + bias_attr=False, + act=None, + use_cudnn=use_cudnn, + ) self.bn_0_layer = BatchNorm(out_ch[0], act=act, is_test=is_test) - self.conv_1_layer = Conv2D(out_ch[0], - num_filters=out_ch[1], - filter_size=3, - padding=1, - param_attr=conv_param_1, - bias_attr=False, - act=None, - use_cudnn=use_cudnn) + self.conv_1_layer = Conv2D( + out_ch[0], + num_filters=out_ch[1], + filter_size=3, + padding=1, + param_attr=conv_param_1, + bias_attr=False, + act=None, + use_cudnn=use_cudnn, + ) self.bn_1_layer = BatchNorm(out_ch[1], act=act, is_test=is_test) if self.pool: - self.pool_layer = Pool2D(pool_size=2, - pool_type='max', - pool_stride=2, - use_cudnn=use_cudnn, - ceil_mode=True) + self.pool_layer = Pool2D( + pool_size=2, + pool_type='max', + pool_stride=2, + use_cudnn=use_cudnn, + ceil_mode=True, + ) def forward(self, inputs): conv_0 = self.conv_0_layer(inputs) @@ -118,22 +135,25 @@ def forward(self, inputs): class OCRConv(fluid.dygraph.Layer): - def __init__(self, is_test=False, use_cudnn=True): super(OCRConv, self).__init__() - self.conv_bn_pool_1 = ConvBNPool(2, [8, 8], [1, 8], - is_test=is_test, - use_cudnn=use_cudnn) - self.conv_bn_pool_2 = ConvBNPool(2, [8, 8], [8, 8], - is_test=is_test, - use_cudnn=use_cudnn) - self.conv_bn_pool_3 = ConvBNPool(2, [8, 8], [8, 8], - is_test=is_test, - use_cudnn=use_cudnn) - self.conv_bn_pool_4 = ConvBNPool(2, [16, 16], [8, 16], - is_test=is_test, - pool=False, - use_cudnn=use_cudnn) + self.conv_bn_pool_1 = ConvBNPool( + 2, [8, 8], [1, 8], is_test=is_test, use_cudnn=use_cudnn + ) + self.conv_bn_pool_2 = ConvBNPool( + 2, [8, 8], [8, 8], is_test=is_test, use_cudnn=use_cudnn + ) + self.conv_bn_pool_3 = ConvBNPool( + 2, [8, 8], [8, 8], is_test=is_test, use_cudnn=use_cudnn + ) + self.conv_bn_pool_4 = ConvBNPool( + 2, + [16, 16], + [8, 16], + is_test=is_test, + pool=False, + use_cudnn=use_cudnn, + ) def forward(self, inputs): inputs_1 = self.conv_bn_pool_1(inputs) @@ -145,24 +165,27 @@ def forward(self, inputs): class DynamicGRU(fluid.dygraph.Layer): - - def __init__(self, - size, - param_attr=None, - bias_attr=None, - is_reverse=False, - gate_activation='sigmoid', - candidate_activation='tanh', - h_0=None, - origin_mode=False): + def __init__( + self, + size, + param_attr=None, + bias_attr=None, + is_reverse=False, + gate_activation='sigmoid', + candidate_activation='tanh', + h_0=None, + origin_mode=False, + ): super(DynamicGRU, self).__init__() - self.gru_unit = GRUUnit(size * 3, - param_attr=param_attr, - bias_attr=bias_attr, - activation=candidate_activation, - gate_activation=gate_activation, - origin_mode=origin_mode) + self.gru_unit = GRUUnit( + size * 3, + param_attr=param_attr, + bias_attr=bias_attr, + activation=candidate_activation, + gate_activation=gate_activation, + origin_mode=origin_mode, + ) self.h_0 = h_0 self.is_reverse = is_reverse @@ -174,15 +197,16 @@ def forward(self, inputs): for i in range(inputs.shape[1]): if self.is_reverse: i = inputs.shape[1] - 1 - i - input_ = fluid.layers.slice(inputs, - axes=[1], - starts=[i], - ends=[i + 1]) - input_ = fluid.layers.reshape(input_, [-1, input_.shape[2]], - inplace=False) + input_ = fluid.layers.slice( + inputs, axes=[1], starts=[i], ends=[i + 1] + ) + input_ = fluid.layers.reshape( + input_, [-1, input_.shape[2]], inplace=False + ) hidden, reset, gate = self.gru_unit(input_, hidden) - hidden_ = fluid.layers.reshape(hidden, [-1, 1, hidden.shape[1]], - inplace=False) + hidden_ = fluid.layers.reshape( + hidden, [-1, 1, hidden.shape[1]], inplace=False + ) if self.is_reverse: res = [hidden_] + res else: @@ -192,75 +216,85 @@ def forward(self, inputs): class EncoderNet(fluid.dygraph.Layer): - - def __init__(self, - rnn_hidden_size=Config.encoder_size, - is_test=False, - use_cudnn=True): + def __init__( + self, rnn_hidden_size=Config.encoder_size, is_test=False, use_cudnn=True + ): super(EncoderNet, self).__init__() self.rnn_hidden_size = rnn_hidden_size para_attr = fluid.ParamAttr( - initializer=fluid.initializer.Normal(0.0, 0.02)) - bias_attr = fluid.ParamAttr(initializer=fluid.initializer.Normal( - 0.0, 0.02), - learning_rate=2.0) + initializer=fluid.initializer.Normal(0.0, 0.02) + ) + bias_attr = fluid.ParamAttr( + initializer=fluid.initializer.Normal(0.0, 0.02), learning_rate=2.0 + ) if fluid.framework._non_static_mode(): - h_0 = np.zeros((Config.batch_size, rnn_hidden_size), - dtype="float32") + h_0 = np.zeros( + (Config.batch_size, rnn_hidden_size), dtype="float32" + ) h_0 = to_variable(h_0) else: h_0 = fluid.layers.fill_constant( shape=[Config.batch_size, rnn_hidden_size], dtype='float32', - value=0) + value=0, + ) self.ocr_convs = OCRConv(is_test=is_test, use_cudnn=use_cudnn) - self.fc_1_layer = Linear(32, - rnn_hidden_size * 3, - param_attr=para_attr, - bias_attr=False) - self.fc_2_layer = Linear(32, - rnn_hidden_size * 3, - param_attr=para_attr, - bias_attr=False) - self.gru_forward_layer = DynamicGRU(size=rnn_hidden_size, - h_0=h_0, - param_attr=para_attr, - bias_attr=bias_attr, - candidate_activation='relu') - self.gru_backward_layer = DynamicGRU(size=rnn_hidden_size, - h_0=h_0, - param_attr=para_attr, - bias_attr=bias_attr, - candidate_activation='relu', - is_reverse=True) - - self.encoded_proj_fc = Linear(rnn_hidden_size * 2, - Config.decoder_size, - bias_attr=False) + self.fc_1_layer = Linear( + 32, rnn_hidden_size * 3, param_attr=para_attr, bias_attr=False + ) + self.fc_2_layer = Linear( + 32, rnn_hidden_size * 3, param_attr=para_attr, bias_attr=False + ) + self.gru_forward_layer = DynamicGRU( + size=rnn_hidden_size, + h_0=h_0, + param_attr=para_attr, + bias_attr=bias_attr, + candidate_activation='relu', + ) + self.gru_backward_layer = DynamicGRU( + size=rnn_hidden_size, + h_0=h_0, + param_attr=para_attr, + bias_attr=bias_attr, + candidate_activation='relu', + is_reverse=True, + ) + + self.encoded_proj_fc = Linear( + rnn_hidden_size * 2, Config.decoder_size, bias_attr=False + ) def forward(self, inputs): conv_features = self.ocr_convs(inputs) - #sliced_feature = fluid.layers.im2sequence( + # sliced_feature = fluid.layers.im2sequence( # input=conv_features, # stride=[1, 1], # filter_size=[conv_features.shape[2], 1]) - transpose_conv_features = fluid.layers.transpose(conv_features, - perm=[0, 3, 1, 2]) - sliced_feature = fluid.layers.reshape(transpose_conv_features, [ - -1, 8, - transpose_conv_features.shape[2] * transpose_conv_features.shape[3] - ], - inplace=False) + transpose_conv_features = fluid.layers.transpose( + conv_features, perm=[0, 3, 1, 2] + ) + sliced_feature = fluid.layers.reshape( + transpose_conv_features, + [ + -1, + 8, + transpose_conv_features.shape[2] + * transpose_conv_features.shape[3], + ], + inplace=False, + ) fc_1 = self.fc_1_layer(sliced_feature) fc_2 = self.fc_2_layer(sliced_feature) gru_forward = self.gru_forward_layer(fc_1) gru_backward = self.gru_backward_layer(fc_2) - encoded_vector = fluid.layers.concat(input=[gru_forward, gru_backward], - axis=2) + encoded_vector = fluid.layers.concat( + input=[gru_forward, gru_backward], axis=2 + ) encoded_proj = self.encoded_proj_fc(encoded_vector) @@ -268,79 +302,80 @@ def forward(self, inputs): class SimpleAttention(fluid.dygraph.Layer): - def __init__(self, decoder_size): super(SimpleAttention, self).__init__() - self.fc_1 = Linear(decoder_size, - decoder_size, - act=None, - bias_attr=False) + self.fc_1 = Linear( + decoder_size, decoder_size, act=None, bias_attr=False + ) self.fc_2 = Linear(decoder_size, 1, act=None, bias_attr=False) def forward(self, encoder_vec, encoder_proj, decoder_state): decoder_state_fc = self.fc_1(decoder_state) decoder_state_proj_reshape = fluid.layers.reshape( - decoder_state_fc, [-1, 1, decoder_state_fc.shape[1]], inplace=False) + decoder_state_fc, [-1, 1, decoder_state_fc.shape[1]], inplace=False + ) decoder_state_expand = fluid.layers.expand( - decoder_state_proj_reshape, [1, encoder_proj.shape[1], 1]) - concated = fluid.layers.elementwise_add(encoder_proj, - decoder_state_expand) + decoder_state_proj_reshape, [1, encoder_proj.shape[1], 1] + ) + concated = fluid.layers.elementwise_add( + encoder_proj, decoder_state_expand + ) concated = fluid.layers.tanh(x=concated) attention_weight = self.fc_2(concated) weights_reshape = fluid.layers.reshape( x=attention_weight, shape=[attention_weight.shape[0], attention_weight.shape[1]], - inplace=False) + inplace=False, + ) weights_reshape = fluid.layers.softmax(weights_reshape) - scaled = fluid.layers.elementwise_mul(x=encoder_vec, - y=weights_reshape, - axis=0) + scaled = fluid.layers.elementwise_mul( + x=encoder_vec, y=weights_reshape, axis=0 + ) context = fluid.layers.reduce_sum(scaled, dim=1) return context class GRUDecoderWithAttention(fluid.dygraph.Layer): - def __init__(self, decoder_size, num_classes): super(GRUDecoderWithAttention, self).__init__() self.simple_attention = SimpleAttention(decoder_size) - self.fc_1_layer = Linear(Config.encoder_size * 2, - decoder_size * 3, - bias_attr=False) - self.fc_2_layer = Linear(decoder_size, - decoder_size * 3, - bias_attr=False) - self.gru_unit = GRUUnit(size=decoder_size * 3, - param_attr=None, - bias_attr=None) - self.out_layer = Linear(decoder_size, - num_classes + 2, - bias_attr=None, - act='softmax') + self.fc_1_layer = Linear( + Config.encoder_size * 2, decoder_size * 3, bias_attr=False + ) + self.fc_2_layer = Linear( + decoder_size, decoder_size * 3, bias_attr=False + ) + self.gru_unit = GRUUnit( + size=decoder_size * 3, param_attr=None, bias_attr=None + ) + self.out_layer = Linear( + decoder_size, num_classes + 2, bias_attr=None, act='softmax' + ) self.decoder_size = decoder_size - def forward(self, target_embedding, encoder_vec, encoder_proj, - decoder_boot): + def forward( + self, target_embedding, encoder_vec, encoder_proj, decoder_boot + ): res = [] hidden_mem = decoder_boot for i in range(target_embedding.shape[1]): - current_word = fluid.layers.slice(target_embedding, - axes=[1], - starts=[i], - ends=[i + 1]) - current_word = fluid.layers.reshape(current_word, - [-1, current_word.shape[2]], - inplace=False) - - context = self.simple_attention(encoder_vec, encoder_proj, - hidden_mem) + current_word = fluid.layers.slice( + target_embedding, axes=[1], starts=[i], ends=[i + 1] + ) + current_word = fluid.layers.reshape( + current_word, [-1, current_word.shape[2]], inplace=False + ) + + context = self.simple_attention( + encoder_vec, encoder_proj, hidden_mem + ) fc_1 = self.fc_1_layer(context) fc_2 = self.fc_2_layer(current_word) decoder_inputs = fluid.layers.elementwise_add(x=fc_1, y=fc_2) @@ -356,45 +391,48 @@ def forward(self, target_embedding, encoder_vec, encoder_proj, class OCRAttention(fluid.dygraph.Layer): - def __init__(self): super(OCRAttention, self).__init__() self.encoder_net = EncoderNet() - self.fc = Linear(Config.encoder_size, - Config.decoder_size, - bias_attr=False, - act='relu') + self.fc = Linear( + Config.encoder_size, + Config.decoder_size, + bias_attr=False, + act='relu', + ) self.embedding = Embedding( - [Config.num_classes + 2, Config.word_vector_dim], dtype='float32') + [Config.num_classes + 2, Config.word_vector_dim], dtype='float32' + ) self.gru_decoder_with_attention = GRUDecoderWithAttention( - Config.decoder_size, Config.num_classes) + Config.decoder_size, Config.num_classes + ) def forward(self, inputs, label_in): gru_backward, encoded_vector, encoded_proj = self.encoder_net(inputs) - backward_first = fluid.layers.slice(gru_backward, - axes=[1], - starts=[0], - ends=[1]) - backward_first = fluid.layers.reshape(backward_first, - [-1, backward_first.shape[2]], - inplace=False) + backward_first = fluid.layers.slice( + gru_backward, axes=[1], starts=[0], ends=[1] + ) + backward_first = fluid.layers.reshape( + backward_first, [-1, backward_first.shape[2]], inplace=False + ) decoder_boot = self.fc(backward_first) label_in = fluid.layers.reshape(label_in, [-1], inplace=False) trg_embedding = self.embedding(label_in) trg_embedding = fluid.layers.reshape( - trg_embedding, [-1, Config.max_length, trg_embedding.shape[1]], - inplace=False) + trg_embedding, + [-1, Config.max_length, trg_embedding.shape[1]], + inplace=False, + ) - prediction = self.gru_decoder_with_attention(trg_embedding, - encoded_vector, - encoded_proj, decoder_boot) + prediction = self.gru_decoder_with_attention( + trg_embedding, encoded_vector, encoded_proj, decoder_boot + ) return prediction class TestDygraphOCRAttention(unittest.TestCase): - def test_ocr_test(self): seed = 90 epoch_num = 1 @@ -403,26 +441,41 @@ def test_ocr_test(self): else: batch_num = 2 np.random.seed = seed - image_np = np.random.randn(Config.batch_size, Config.DATA_SHAPE[0], - Config.DATA_SHAPE[1], - Config.DATA_SHAPE[2]).astype('float32') - label_in_np = np.arange(0, Config.max_length, - dtype='int64').reshape([1, Config.max_length]) + image_np = np.random.randn( + Config.batch_size, + Config.DATA_SHAPE[0], + Config.DATA_SHAPE[1], + Config.DATA_SHAPE[2], + ).astype('float32') + label_in_np = np.arange(0, Config.max_length, dtype='int64').reshape( + [1, Config.max_length] + ) for i in range(2, Config.batch_size + 1): label_in_np = np.vstack( - (label_in_np, - np.arange((i - 1) * Config.max_length, - i * Config.max_length, - dtype='int64').reshape([1, Config.max_length]))) + ( + label_in_np, + np.arange( + (i - 1) * Config.max_length, + i * Config.max_length, + dtype='int64', + ).reshape([1, Config.max_length]), + ) + ) - label_out_np = np.arange(0, Config.max_length, - dtype='int64').reshape([1, Config.max_length]) + label_out_np = np.arange(0, Config.max_length, dtype='int64').reshape( + [1, Config.max_length] + ) for i in range(2, Config.batch_size + 1): label_out_np = np.vstack( - (label_out_np, - np.arange((i - 1) * Config.max_length, - i * Config.max_length, - dtype='int64').reshape([1, Config.max_length]))) + ( + label_out_np, + np.arange( + (i - 1) * Config.max_length, + i * Config.max_length, + dtype='int64', + ).reshape([1, Config.max_length]), + ) + ) def run_dygraph(): fluid.set_flags({'FLAGS_sort_sum_gradient': True}) @@ -432,11 +485,13 @@ def run_dygraph(): if Config.learning_rate_decay == "piecewise_decay": learning_rate = fluid.layers.piecewise_decay( - [50000], [Config.LR, Config.LR * 0.01]) + [50000], [Config.LR, Config.LR * 0.01] + ) else: learning_rate = Config.LR optimizer = fluid.optimizer.SGD( - learning_rate=0.001, parameter_list=ocr_attention.parameters()) + learning_rate=0.001, parameter_list=ocr_attention.parameters() + ) dy_param_init_value = {} for param in ocr_attention.parameters(): dy_param_init_value[param.name] = param.numpy() @@ -447,12 +502,15 @@ def run_dygraph(): label_out.stop_gradient = True img = to_variable(image_np) dy_prediction = ocr_attention(img, label_in) - label_out = fluid.layers.reshape(label_out, [-1, 1], - inplace=False) + label_out = fluid.layers.reshape( + label_out, [-1, 1], inplace=False + ) dy_prediction = fluid.layers.reshape( - dy_prediction, [label_out.shape[0], -1], inplace=False) - loss = fluid.layers.cross_entropy(input=dy_prediction, - label=label_out) + dy_prediction, [label_out.shape[0], -1], inplace=False + ) + loss = fluid.layers.cross_entropy( + input=dy_prediction, label=label_out + ) avg_loss = fluid.layers.reduce_sum(loss) dy_out = avg_loss.numpy() @@ -466,9 +524,11 @@ def run_dygraph(): for param in ocr_attention.parameters(): if param.trainable: np_array = np.array( - param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - core.grad_var_suffix()] = np_array + param._grad_ivar().value().get_tensor() + ) + dy_grad_value[ + param.name + core.grad_var_suffix() + ] = np_array optimizer.minimize(avg_loss) ocr_attention.clear_gradients() @@ -483,45 +543,52 @@ def run_dygraph(): with fluid.dygraph.guard(): with _test_eager_guard(): - eager_out, eager_param_init_value, eager_param_value = run_dygraph( - ) + ( + eager_out, + eager_param_init_value, + eager_param_value, + ) = run_dygraph() with new_program_scope(): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) ocr_attention = OCRAttention() if Config.learning_rate_decay == "piecewise_decay": learning_rate = fluid.layers.piecewise_decay( - [50000], [Config.LR, Config.LR * 0.01]) + [50000], [Config.LR, Config.LR * 0.01] + ) else: learning_rate = Config.LR optimizer = fluid.optimizer.SGD(learning_rate=0.001) - images = fluid.layers.data(name='pixel', - shape=Config.DATA_SHAPE, - dtype='float32') - static_label_in = fluid.layers.data(name='label_in', - shape=[1], - dtype='int64', - lod_level=0) - static_label_out = fluid.layers.data(name='label_out', - shape=[1], - dtype='int64', - lod_level=0) + images = fluid.layers.data( + name='pixel', shape=Config.DATA_SHAPE, dtype='float32' + ) + static_label_in = fluid.layers.data( + name='label_in', shape=[1], dtype='int64', lod_level=0 + ) + static_label_out = fluid.layers.data( + name='label_out', shape=[1], dtype='int64', lod_level=0 + ) static_label_out.stop_gradient = True static_label_out.trainable = False static_prediction = ocr_attention(images, static_label_in) static_prediction = fluid.layers.reshape( - static_prediction, shape=[-1, Config.num_classes + 2]) + static_prediction, shape=[-1, Config.num_classes + 2] + ) - cost = fluid.layers.cross_entropy(input=static_prediction, - label=static_label_out) + cost = fluid.layers.cross_entropy( + input=static_prediction, label=static_label_out + ) static_avg_loss = fluid.layers.reduce_sum(cost) # param_grad_list = fluid.backward.append_backward(static_avg_loss) optimizer.minimize(static_avg_loss) @@ -532,11 +599,14 @@ def run_dygraph(): for param in ocr_attention.parameters(): static_param_name_list.append(param.name) if param.trainable: - static_grad_name_list.append(param.name + - core.grad_var_suffix()) + static_grad_name_list.append( + param.name + core.grad_var_suffix() + ) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -549,24 +619,30 @@ def run_dygraph(): static_label_in = label_in_np static_label_out = label_out_np static_label_out = static_label_out.reshape((-1, 1)) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": image_np, - "label_in": static_label_in, - "label_out": static_label_out - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "pixel": image_np, + "label_in": static_label_in, + "label_out": static_label_out, + }, + fetch_list=fetch_list, + ) static_param_value = {} static_grad_value = {} static_out = out[0] for i in range(1, len(static_param_name_list) + 1): - static_param_value[static_param_name_list[i - - 1]] = out[i] + static_param_value[static_param_name_list[i - 1]] = out[ + i + ] grad_start_pos = len(static_param_name_list) + 1 - for i in range(grad_start_pos, - len(static_grad_name_list) + grad_start_pos): - static_grad_value[static_grad_name_list[ - i - grad_start_pos]] = out[i] + for i in range( + grad_start_pos, + len(static_grad_name_list) + grad_start_pos, + ): + static_grad_value[ + static_grad_name_list[i - grad_start_pos] + ] = out[i] np.testing.assert_allclose(static_out, dy_out, rtol=1e-05, atol=1e-8) @@ -574,10 +650,9 @@ def run_dygraph(): np.testing.assert_array_equal(value, dy_param_init_value[key]) for key, value in static_param_value.items(): - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05, - atol=1e-8) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05, atol=1e-8 + ) # check eager here np.testing.assert_allclose(static_out, eager_out, rtol=1e-05, atol=1e-8) @@ -586,10 +661,9 @@ def run_dygraph(): np.testing.assert_array_equal(value, eager_param_init_value[key]) for key, value in static_param_value.items(): - np.testing.assert_allclose(value, - eager_param_value[key], - rtol=1e-05, - atol=1e-8) + np.testing.assert_allclose( + value, eager_param_value[key], rtol=1e-05, atol=1e-8 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_optimizer.py b/python/paddle/fluid/tests/unittests/test_imperative_optimizer.py index ba1f73bcbcb42..0219cc9947975 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_optimizer.py @@ -19,8 +19,28 @@ import paddle import paddle.fluid as fluid from paddle.fluid import core -from paddle.fluid.optimizer import SGDOptimizer, Adam, MomentumOptimizer, LarsMomentumOptimizer, AdagradOptimizer, AdamaxOptimizer, DpsgdOptimizer, DecayedAdagradOptimizer, AdadeltaOptimizer, RMSPropOptimizer, FtrlOptimizer, LambOptimizer -from paddle.fluid.optimizer import ModelAverage, DGCMomentumOptimizer, ExponentialMovingAverage, PipelineOptimizer, LookaheadOptimizer, RecomputeOptimizer +from paddle.fluid.optimizer import ( + SGDOptimizer, + Adam, + MomentumOptimizer, + LarsMomentumOptimizer, + AdagradOptimizer, + AdamaxOptimizer, + DpsgdOptimizer, + DecayedAdagradOptimizer, + AdadeltaOptimizer, + RMSPropOptimizer, + FtrlOptimizer, + LambOptimizer, +) +from paddle.fluid.optimizer import ( + ModelAverage, + DGCMomentumOptimizer, + ExponentialMovingAverage, + PipelineOptimizer, + LookaheadOptimizer, + RecomputeOptimizer, +) from paddle.fluid.dygraph import Linear from test_imperative_base import new_program_scope from paddle.fluid.framework import _test_eager_guard @@ -30,7 +50,6 @@ class MLP(fluid.Layer): - def __init__(self, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -44,7 +63,6 @@ def forward(self, inputs): class TestImperativeOptimizerBase(unittest.TestCase): - def setUp(self): self.batch_num = 20 @@ -55,7 +73,6 @@ def get_optimizer(self): raise NotImplementedError() def reader_decorator(self, reader): - def _reader_imple(): for item in reader(): image = np.array(item[0]).reshape(1, 784) @@ -68,8 +85,11 @@ def _check_exception(self, exception_message, place=None): seed = 90 batch_size = 128 if place == None: - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.dygraph.guard(place): try: @@ -77,7 +97,8 @@ def _check_exception(self, exception_message, place=None): paddle.framework.random._manual_program_seed(seed) mlp = MLP() optimizer = self.get_optimizer_dygraph( - parameter_list=mlp.parameters()) + parameter_list=mlp.parameters() + ) except Exception as e: assert str(e) == exception_message @@ -86,8 +107,11 @@ def _check_mlp(self, place=None): batch_size = 128 if place == None: - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) with fluid.dygraph.guard(place): paddle.seed(seed) @@ -95,15 +119,18 @@ def _check_mlp(self, place=None): mlp = MLP() optimizer = self.get_optimizer_dygraph( - parameter_list=mlp.parameters()) + parameter_list=mlp.parameters() + ) batch_py_reader = fluid.io.PyReader(capacity=1) batch_py_reader.decorate_sample_list_generator( - paddle.batch(self.reader_decorator( - paddle.dataset.mnist.train()), - batch_size=batch_size, - drop_last=True), - places=fluid.CPUPlace()) + paddle.batch( + self.reader_decorator(paddle.dataset.mnist.train()), + batch_size=batch_size, + drop_last=True, + ), + places=fluid.CPUPlace(), + ) dy_param_init_value = {} for batch_id, data in enumerate(batch_py_reader()): @@ -135,20 +162,23 @@ def _check_mlp(self, place=None): paddle.framework.random._manual_program_seed(seed) if place == None: - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe = fluid.Executor(place) mlp = MLP() optimizer = self.get_optimizer() - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=128, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=128, drop_last=True + ) - img = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype='float32') + img = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') img = fluid.layers.reshape(img, shape=[batch_size, 784]) cost = mlp(img) @@ -161,8 +191,10 @@ def _check_mlp(self, place=None): for param in mlp.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -172,18 +204,21 @@ def _check_mlp(self, place=None): break static_x_data = np.array( - [x[0].reshape(1, 28, 28) for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape([128, 1]) + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([128, 1]) + ) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) static_param_value = {} static_out = out[0] @@ -191,43 +226,48 @@ def _check_mlp(self, place=None): static_param_value[static_param_name_list[i - 1]] = out[i] for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value[key], rtol=1e-05 + ) if core.is_compiled_with_rocm(): - np.testing.assert_allclose(static_out, - dy_out, - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + static_out, dy_out, rtol=1e-05, atol=0.001 + ) else: np.testing.assert_allclose(static_out, dy_out, rtol=1e-05) for key, value in static_param_value.items(): if core.is_compiled_with_rocm(): - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05, atol=0.001 + ) else: - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05 + ) class TestImperativeOptimizerPiecewiseDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): bd = [3, 6, 9] - optimizer = SGDOptimizer(learning_rate=fluid.layers.piecewise_decay( - boundaries=bd, values=[0.1 * (0.1**i) for i in range(len(bd) + 1)]), - parameter_list=parameter_list) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, + values=[0.1 * (0.1**i) for i in range(len(bd) + 1)], + ), + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): bd = [3, 6, 9] - optimizer = SGDOptimizer(learning_rate=fluid.layers.piecewise_decay( - boundaries=bd, values=[0.1 * (0.1**i) for i in range(len(bd) + 1)])) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, + values=[0.1 * (0.1**i) for i in range(len(bd) + 1)], + ) + ) return optimizer def func_test_sgd(self): @@ -240,22 +280,27 @@ def test_sgd(self): class TestImperativeOptimizerNaturalExpDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = SGDOptimizer(learning_rate=fluid.layers.natural_exp_decay( - learning_rate=0.1, - decay_steps=10000, - decay_rate=0.5, - staircase=True), - parameter_list=parameter_list) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.natural_exp_decay( + learning_rate=0.1, + decay_steps=10000, + decay_rate=0.5, + staircase=True, + ), + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = SGDOptimizer( - learning_rate=fluid.layers.natural_exp_decay(learning_rate=0.1, - decay_steps=10000, - decay_rate=0.5, - staircase=True)) + learning_rate=fluid.layers.natural_exp_decay( + learning_rate=0.1, + decay_steps=10000, + decay_rate=0.5, + staircase=True, + ) + ) return optimizer def func_test_sgd(self): @@ -268,22 +313,27 @@ def test_sgd(self): class TestImperativeOptimizerExponentialDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = SGDOptimizer(learning_rate=fluid.layers.exponential_decay( - learning_rate=0.1, - decay_steps=10000, - decay_rate=0.5, - staircase=True), - parameter_list=parameter_list) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.exponential_decay( + learning_rate=0.1, + decay_steps=10000, + decay_rate=0.5, + staircase=True, + ), + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = SGDOptimizer( - learning_rate=fluid.layers.exponential_decay(learning_rate=0.1, - decay_steps=10000, - decay_rate=0.5, - staircase=True)) + learning_rate=fluid.layers.exponential_decay( + learning_rate=0.1, + decay_steps=10000, + decay_rate=0.5, + staircase=True, + ) + ) return optimizer def func_test_sgd(self): @@ -296,22 +346,27 @@ def test_sgd(self): class TestImperativeOptimizerInverseTimeDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = Adam(learning_rate=fluid.layers.inverse_time_decay( - learning_rate=0.1, - decay_steps=10000, - decay_rate=0.5, - staircase=True), - parameter_list=parameter_list) + optimizer = Adam( + learning_rate=fluid.layers.inverse_time_decay( + learning_rate=0.1, + decay_steps=10000, + decay_rate=0.5, + staircase=True, + ), + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = Adam( - learning_rate=fluid.layers.inverse_time_decay(learning_rate=0.1, - decay_steps=10000, - decay_rate=0.5, - staircase=True)) + learning_rate=fluid.layers.inverse_time_decay( + learning_rate=0.1, + decay_steps=10000, + decay_rate=0.5, + staircase=True, + ) + ) return optimizer def func_test_adam(self): @@ -324,16 +379,21 @@ def test_adam(self): class TestImperativeOptimizerPolynomialDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = SGDOptimizer(learning_rate=fluid.layers.polynomial_decay( - learning_rate=0.1, decay_steps=5, cycle=self.cycle), - parameter_list=parameter_list) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.polynomial_decay( + learning_rate=0.1, decay_steps=5, cycle=self.cycle + ), + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): - optimizer = SGDOptimizer(learning_rate=fluid.layers.polynomial_decay( - learning_rate=0.1, decay_steps=5, cycle=self.cycle)) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.polynomial_decay( + learning_rate=0.1, decay_steps=5, cycle=self.cycle + ) + ) return optimizer def func_test_sgd_cycle(self): @@ -356,16 +416,21 @@ def test_sgd(self): class TestImperativeOptimizerCosineDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = SGDOptimizer(learning_rate=fluid.layers.cosine_decay( - learning_rate=0.1, step_each_epoch=10000, epochs=120), - parameter_list=parameter_list) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.cosine_decay( + learning_rate=0.1, step_each_epoch=10000, epochs=120 + ), + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): - optimizer = SGDOptimizer(learning_rate=fluid.layers.cosine_decay( - learning_rate=0.1, step_each_epoch=10000, epochs=120)) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.cosine_decay( + learning_rate=0.1, step_each_epoch=10000, epochs=120 + ) + ) return optimizer def func_test_sgd(self): @@ -378,16 +443,21 @@ def test_sgd(self): class TestImperativeOptimizerNoamDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = SGDOptimizer(learning_rate=fluid.layers.noam_decay( - d_model=512, warmup_steps=8000), - parameter_list=parameter_list) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.noam_decay( + d_model=512, warmup_steps=8000 + ), + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): - optimizer = SGDOptimizer(learning_rate=fluid.layers.noam_decay( - d_model=512, warmup_steps=8000)) + optimizer = SGDOptimizer( + learning_rate=fluid.layers.noam_decay( + d_model=512, warmup_steps=8000 + ) + ) return optimizer def func_test_sgd(self): @@ -400,7 +470,6 @@ def test_sgd(self): class TestOptimizerLearningRate(unittest.TestCase): - def func_test_constant_lr(self): with fluid.dygraph.guard(): a = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32") @@ -413,13 +482,13 @@ def func_test_constant_lr(self): loss = fluid.layers.reduce_mean(b) - adam = fluid.optimizer.Adam(0.001, - parameter_list=linear.parameters()) + adam = fluid.optimizer.Adam( + 0.001, parameter_list=linear.parameters() + ) - np.testing.assert_allclose(adam.current_step_lr(), - 0.001, - rtol=1e-06, - atol=0.0) + np.testing.assert_allclose( + adam.current_step_lr(), 0.001, rtol=1e-06, atol=0.0 + ) for i in range(10): adam.minimize(loss) @@ -447,14 +516,14 @@ def func_test_lr_decay(self): bd = [2, 4, 6, 8] value = [0.2, 0.4, 0.6, 0.8, 1.0] - adam = fluid.optimizer.Adam(fluid.dygraph.PiecewiseDecay( - bd, value, 0), - parameter_list=linear.parameters()) + adam = fluid.optimizer.Adam( + fluid.dygraph.PiecewiseDecay(bd, value, 0), + parameter_list=linear.parameters(), + ) - np.testing.assert_allclose(adam.current_step_lr(), - 0.2, - rtol=1e-06, - atol=0.0) + np.testing.assert_allclose( + adam.current_step_lr(), 0.2, rtol=1e-06, atol=0.0 + ) ret = [0.2, 0.2, 0.4, 0.4, 0.6, 0.6, 0.8, 0.8, 1.0, 1.0, 1.0, 1.0] for i in range(12): @@ -481,17 +550,19 @@ def func_test_lr_decay_natural_exp(self): loss = fluid.layers.reduce_mean(b) base_lr = 1.0 - adam = fluid.optimizer.Adam(fluid.dygraph.NaturalExpDecay( - learning_rate=base_lr, - decay_steps=3, - decay_rate=0.5, - staircase=True), - parameter_list=linear.parameters()) + adam = fluid.optimizer.Adam( + fluid.dygraph.NaturalExpDecay( + learning_rate=base_lr, + decay_steps=3, + decay_rate=0.5, + staircase=True, + ), + parameter_list=linear.parameters(), + ) - np.testing.assert_allclose(adam.current_step_lr(), - 1.0, - rtol=1e-06, - atol=0.0) + np.testing.assert_allclose( + adam.current_step_lr(), 1.0, rtol=1e-06, atol=0.0 + ) ret = [1.0, 1.0, 1.0, np.exp(-0.5), np.exp(-0.5)] for i in range(5): @@ -526,21 +597,24 @@ def func_test_set_lr(self): lr = adam.current_step_lr() np.testing.assert_allclose(lr, lr_list[i], rtol=1e-06, atol=0.0) - lr_var = fluid.layers.create_global_var(shape=[1], - value=0.7, - dtype='float32') + lr_var = fluid.layers.create_global_var( + shape=[1], value=0.7, dtype='float32' + ) adam.set_lr(lr_var) adam.minimize(loss) lr = adam.current_step_lr() np.testing.assert_allclose(lr, 0.7, rtol=1e-06, atol=0.0) with self.assertRaises(RuntimeError): - adam = fluid.optimizer.Adam(fluid.dygraph.NaturalExpDecay( - learning_rate=0.1, - decay_steps=3, - decay_rate=0.5, - staircase=True), - parameter_list=linear.parameters()) + adam = fluid.optimizer.Adam( + fluid.dygraph.NaturalExpDecay( + learning_rate=0.1, + decay_steps=3, + decay_rate=0.5, + staircase=True, + ), + parameter_list=linear.parameters(), + ) adam.set_lr(0.01) def test_set_lr(self): @@ -550,11 +624,10 @@ def test_set_lr(self): class TestImperativeMomentumOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = MomentumOptimizer(learning_rate=0.001, - momentum=0.9, - parameter_list=parameter_list) + optimizer = MomentumOptimizer( + learning_rate=0.001, momentum=0.9, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -571,11 +644,10 @@ def test_momentum(self): class TestImperativeLarsMomentumOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = LarsMomentumOptimizer(learning_rate=0.001, - momentum=0.9, - parameter_list=parameter_list) + optimizer = LarsMomentumOptimizer( + learning_rate=0.001, momentum=0.9, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -592,10 +664,10 @@ def test_larsmomentum(self): class TestImperativeAdagradOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = AdagradOptimizer(learning_rate=0.2, - parameter_list=parameter_list) + optimizer = AdagradOptimizer( + learning_rate=0.2, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -612,10 +684,10 @@ def test_adagrad(self): class TestImperativeAdamaxOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = AdamaxOptimizer(learning_rate=0.2, - parameter_list=parameter_list) + optimizer = AdamaxOptimizer( + learning_rate=0.2, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -632,21 +704,21 @@ def test_adamax(self): class TestImperativeDpsgdOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = DpsgdOptimizer(learning_rate=0.01, - clip=10.0, - batch_size=16.0, - sigma=1.0, - parameter_list=parameter_list) + optimizer = DpsgdOptimizer( + learning_rate=0.01, + clip=10.0, + batch_size=16.0, + sigma=1.0, + parameter_list=parameter_list, + ) optimizer._seed = 100 return optimizer def get_optimizer(self): - optimizer = DpsgdOptimizer(learning_rate=0.01, - clip=10.0, - batch_size=16.0, - sigma=1.0) + optimizer = DpsgdOptimizer( + learning_rate=0.01, clip=10.0, batch_size=16.0, sigma=1.0 + ) optimizer._seed = 100 return optimizer @@ -660,10 +732,10 @@ def test_dpsgd(self): class TestImperativeDecayedAdagradOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = DecayedAdagradOptimizer(learning_rate=0.2, - parameter_list=parameter_list) + optimizer = DecayedAdagradOptimizer( + learning_rate=0.2, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -680,18 +752,19 @@ def test_decayadagrad(self): class TestImperativeAdadeltaOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = AdadeltaOptimizer(learning_rate=0.0003, - epsilon=1.0e-6, - rho=0.95, - parameter_list=parameter_list) + optimizer = AdadeltaOptimizer( + learning_rate=0.0003, + epsilon=1.0e-6, + rho=0.95, + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): - optimizer = AdadeltaOptimizer(learning_rate=0.0003, - epsilon=1.0e-6, - rho=0.95) + optimizer = AdadeltaOptimizer( + learning_rate=0.0003, epsilon=1.0e-6, rho=0.95 + ) return optimizer def func_test_adadelta(self): @@ -704,10 +777,10 @@ def test_adadelta(self): class TestImperativeRMSPropOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = RMSPropOptimizer(learning_rate=0.1, - parameter_list=parameter_list) + optimizer = RMSPropOptimizer( + learning_rate=0.1, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -724,10 +797,10 @@ def test_rmsprop(self): class TestImperativeFtrlOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = FtrlOptimizer(learning_rate=0.1, - parameter_list=parameter_list) + optimizer = FtrlOptimizer( + learning_rate=0.1, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -748,16 +821,18 @@ def exclude_fn(param): class TestImperativeLambOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = LambOptimizer(learning_rate=0.002, - exclude_from_weight_decay_fn=exclude_fn, - parameter_list=parameter_list) + optimizer = LambOptimizer( + learning_rate=0.002, + exclude_from_weight_decay_fn=exclude_fn, + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): - optimizer = LambOptimizer(learning_rate=0.002, - exclude_from_weight_decay_fn=exclude_fn) + optimizer = LambOptimizer( + learning_rate=0.002, exclude_from_weight_decay_fn=exclude_fn + ) return optimizer # should fix: may fail in CI-windows @@ -766,11 +841,10 @@ def _test_lamb(self): class TestImperativeModelAverage(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = ModelAverage(0.15, - min_average_window=10000, - max_average_window=12500) + optimizer = ModelAverage( + 0.15, min_average_window=10000, max_average_window=12500 + ) return optimizer def func_test_modelaverage(self): @@ -784,13 +858,14 @@ def test_modelaverage(self): class TestImperativeDGCMomentumOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = DGCMomentumOptimizer(learning_rate=0.0001, - momentum=0.9, - rampup_step=1000, - rampup_begin_step=1252, - sparsity=[0.999, 0.999]) + optimizer = DGCMomentumOptimizer( + learning_rate=0.0001, + momentum=0.9, + rampup_step=1000, + rampup_begin_step=1252, + sparsity=[0.999, 0.999], + ) return optimizer def func_test_dgcmomentum(self): @@ -804,13 +879,14 @@ def test_dgcmomentum(self): class TestImperativeExponentialMovingAverage(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = ExponentialMovingAverage(0.999) return optimizer def func_test_exponentialmoving(self): - exception_message = "In dygraph, don't support ExponentialMovingAverage." + exception_message = ( + "In dygraph, don't support ExponentialMovingAverage." + ) self._check_exception(exception_message) def test_exponentialmoving(self): @@ -820,10 +896,10 @@ def test_exponentialmoving(self): class TestImperativePipelineOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = fluid.optimizer.SGD(learning_rate=0.5, - parameter_list=parameter_list) + optimizer = fluid.optimizer.SGD( + learning_rate=0.5, parameter_list=parameter_list + ) optimizer = PipelineOptimizer(optimizer) return optimizer @@ -838,10 +914,10 @@ def test_pipline(self): class TestImperativeLookaheadOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = fluid.optimizer.SGD(learning_rate=0.5, - parameter_list=parameter_list) + optimizer = fluid.optimizer.SGD( + learning_rate=0.5, parameter_list=parameter_list + ) optimizer = LookaheadOptimizer(optimizer, alpha=0.5, k=5) return optimizer @@ -856,10 +932,10 @@ def test_lookahead(self): class TestImperativeRecomputeOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = fluid.optimizer.SGD(learning_rate=0.5, - parameter_list=parameter_list) + optimizer = fluid.optimizer.SGD( + learning_rate=0.5, parameter_list=parameter_list + ) optimizer = RecomputeOptimizer(optimizer) return optimizer @@ -874,16 +950,17 @@ def test_recompute(self): class TestImperativeOptimizerList(unittest.TestCase): - def func_test_parameter_list(self): with fluid.dygraph.guard(): linear_1 = Linear(10, 10) linear_2 = Linear(10, 10) - sgd = SGDOptimizer(1.0, - parameter_list=itertools.chain( - linear_1.parameters(), - linear_2.parameters())) + sgd = SGDOptimizer( + 1.0, + parameter_list=itertools.chain( + linear_1.parameters(), linear_2.parameters() + ), + ) in_np = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32") in_data = fluid.dygraph.to_variable(in_np) @@ -895,8 +972,9 @@ def func_test_parameter_list(self): sgd.minimize(loss) self.assertTrue( - len(sgd._parameter_list) == len(linear_1.parameters() + - linear_2.parameters())) + len(sgd._parameter_list) + == len(linear_1.parameters() + linear_2.parameters()) + ) def test_parameter_list(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_optimizer_v2.py b/python/paddle/fluid/tests/unittests/test_imperative_optimizer_v2.py index 3b0ba3572cc54..251ac5b6b9338 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_optimizer_v2.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_optimizer_v2.py @@ -19,8 +19,25 @@ import paddle import paddle.fluid as fluid from paddle.fluid import core -from paddle.fluid.optimizer import MomentumOptimizer, LarsMomentumOptimizer, AdagradOptimizer, AdamaxOptimizer, DpsgdOptimizer, DecayedAdagradOptimizer, AdadeltaOptimizer, RMSPropOptimizer, FtrlOptimizer -from paddle.fluid.optimizer import ModelAverage, DGCMomentumOptimizer, ExponentialMovingAverage, PipelineOptimizer, LookaheadOptimizer, RecomputeOptimizer +from paddle.fluid.optimizer import ( + MomentumOptimizer, + LarsMomentumOptimizer, + AdagradOptimizer, + AdamaxOptimizer, + DpsgdOptimizer, + DecayedAdagradOptimizer, + AdadeltaOptimizer, + RMSPropOptimizer, + FtrlOptimizer, +) +from paddle.fluid.optimizer import ( + ModelAverage, + DGCMomentumOptimizer, + ExponentialMovingAverage, + PipelineOptimizer, + LookaheadOptimizer, + RecomputeOptimizer, +) from paddle.fluid.dygraph import Linear from test_imperative_base import new_program_scope from paddle.fluid.framework import _test_eager_guard @@ -30,7 +47,6 @@ class MLP(fluid.Layer): - def __init__(self, param_attr=None, bias_attr=None): super(MLP, self).__init__() @@ -44,7 +60,6 @@ def forward(self, inputs): class TestImperativeOptimizerBase(unittest.TestCase): - def setUp(self): self.batch_num = 20 @@ -55,7 +70,6 @@ def get_optimizer(self): raise NotImplementedError() def reader_decorator(self, reader): - def _reader_imple(): for item in reader(): image = np.array(item[0]).reshape(1, 784) @@ -68,8 +82,11 @@ def _check_exception(self, exception_message, place=None): seed = 90 batch_size = 128 if place == None: - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) try: paddle.disable_static() @@ -77,7 +94,8 @@ def _check_exception(self, exception_message, place=None): paddle.framework.random._manual_program_seed(seed) mlp = MLP() optimizer = self.get_optimizer_dygraph( - parameter_list=mlp.parameters()) + parameter_list=mlp.parameters() + ) except Exception as e: assert str(e) == exception_message finally: @@ -88,8 +106,11 @@ def _check_mlp(self, place=None): batch_size = 128 if place == None: - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) paddle.disable_static(place) paddle.seed(seed) @@ -99,11 +120,14 @@ def _check_mlp(self, place=None): optimizer = self.get_optimizer_dygraph(parameter_list=mlp.parameters()) batch_py_reader = fluid.io.PyReader(capacity=1) - batch_py_reader.decorate_sample_list_generator(paddle.batch( - self.reader_decorator(paddle.dataset.mnist.train()), - batch_size=batch_size, - drop_last=True), - places=fluid.CPUPlace()) + batch_py_reader.decorate_sample_list_generator( + paddle.batch( + self.reader_decorator(paddle.dataset.mnist.train()), + batch_size=batch_size, + drop_last=True, + ), + places=fluid.CPUPlace(), + ) dy_param_init_value = {} for batch_id, data in enumerate(batch_py_reader()): @@ -126,10 +150,13 @@ def _check_mlp(self, place=None): avg_loss.backward() optimizer.minimize(avg_loss) - if isinstance(optimizer._learning_rate, - paddle.optimizer.lr.LRScheduler): - if isinstance(optimizer._learning_rate, - paddle.optimizer.lr.ReduceOnPlateau): + if isinstance( + optimizer._learning_rate, paddle.optimizer.lr.LRScheduler + ): + if isinstance( + optimizer._learning_rate, + paddle.optimizer.lr.ReduceOnPlateau, + ): optimizer._learning_rate.step(avg_loss) else: optimizer._learning_rate.step() @@ -144,20 +171,23 @@ def _check_mlp(self, place=None): paddle.framework.random._manual_program_seed(seed) if place == None: - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe = fluid.Executor(place) mlp = MLP() optimizer = self.get_optimizer() - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=128, - drop_last=True) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=128, drop_last=True + ) - img = fluid.layers.data(name='pixel', - shape=[1, 28, 28], - dtype='float32') + img = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') img = fluid.layers.reshape(img, shape=[batch_size, 784]) cost = mlp(img) @@ -170,8 +200,10 @@ def _check_mlp(self, place=None): for param in mlp.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -181,22 +213,28 @@ def _check_mlp(self, place=None): break static_x_data = np.array( - [x[0].reshape(1, 28, 28) for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape([128, 1]) + [x[0].reshape(1, 28, 28) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([128, 1]) + ) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) - if isinstance(optimizer._learning_rate, - paddle.optimizer.lr.LRScheduler): - if isinstance(optimizer._learning_rate, - paddle.optimizer.lr.ReduceOnPlateau): + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) + if isinstance( + optimizer._learning_rate, paddle.optimizer.lr.LRScheduler + ): + if isinstance( + optimizer._learning_rate, + paddle.optimizer.lr.ReduceOnPlateau, + ): optimizer._learning_rate.step(out[0]) else: optimizer._learning_rate.step() @@ -207,39 +245,38 @@ def _check_mlp(self, place=None): static_param_value[static_param_name_list[i - 1]] = out[i] for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value[key], rtol=1e-05 + ) if core.is_compiled_with_rocm(): - np.testing.assert_allclose(static_out, - dy_out, - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + static_out, dy_out, rtol=1e-05, atol=0.001 + ) else: np.testing.assert_allclose(static_out, dy_out, rtol=1e-05) for key, value in static_param_value.items(): if core.is_compiled_with_rocm(): - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05, atol=0.001 + ) else: - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05 + ) class TestImperativeOptimizerPiecewiseDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): bd = [3, 6, 9] optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.PiecewiseDecay( boundaries=bd, - values=[0.1 * (0.1**i) for i in range(len(bd) + 1)]), - parameters=parameter_list) + values=[0.1 * (0.1**i) for i in range(len(bd) + 1)], + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): @@ -247,7 +284,9 @@ def get_optimizer(self): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.PiecewiseDecay( boundaries=bd, - values=[0.1 * (0.1**i) for i in range(len(bd) + 1)])) + values=[0.1 * (0.1**i) for i in range(len(bd) + 1)], + ) + ) return optimizer def func_test_sgd(self): @@ -260,18 +299,21 @@ def test_sgd(self): class TestImperativeOptimizerNaturalExpDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.NaturalExpDecay(learning_rate=0.5, - gamma=0.9), - parameters=parameter_list) + learning_rate=paddle.optimizer.lr.NaturalExpDecay( + learning_rate=0.5, gamma=0.9 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.NaturalExpDecay(learning_rate=0.5, - gamma=0.9)) + learning_rate=paddle.optimizer.lr.NaturalExpDecay( + learning_rate=0.5, gamma=0.9 + ) + ) return optimizer def func_test_sgd(self): @@ -284,18 +326,21 @@ def test_sgd(self): class TestImperativeOptimizerExponentialDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.ExponentialDecay( - learning_rate=0.5, gamma=0.9), - parameters=parameter_list) + learning_rate=0.5, gamma=0.9 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.ExponentialDecay( - learning_rate=0.5, gamma=0.9)) + learning_rate=0.5, gamma=0.9 + ) + ) return optimizer def func_test_sgd(self): @@ -308,18 +353,21 @@ def test_sgd(self): class TestImperativeOptimizerInverseTimeDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.Adam( learning_rate=paddle.optimizer.lr.InverseTimeDecay( - learning_rate=0.5, gamma=0.9), - parameters=parameter_list) + learning_rate=0.5, gamma=0.9 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.Adam( learning_rate=paddle.optimizer.lr.InverseTimeDecay( - learning_rate=0.5, gamma=0.9)) + learning_rate=0.5, gamma=0.9 + ) + ) return optimizer def func_test_adam(self): @@ -332,19 +380,21 @@ def test_adam(self): class TestImperativeOptimizerPolynomialDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.PolynomialDecay(learning_rate=0.5, - decay_steps=5, - cycle=self.cycle), - parameters=parameter_list) + learning_rate=paddle.optimizer.lr.PolynomialDecay( + learning_rate=0.5, decay_steps=5, cycle=self.cycle + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.PolynomialDecay( - learning_rate=0.5, decay_steps=5, cycle=self.cycle)) + learning_rate=0.5, decay_steps=5, cycle=self.cycle + ) + ) return optimizer def func_test_sgd_cycle(self): @@ -367,18 +417,21 @@ def test_sgd(self): class TestImperativeOptimizerCosineAnnealingDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.CosineAnnealingDecay( - learning_rate=0.5, T_max=5), - parameters=parameter_list) + learning_rate=0.5, T_max=5 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.CosineAnnealingDecay( - learning_rate=0.5, T_max=5)) + learning_rate=0.5, T_max=5 + ) + ) return optimizer def func_test_sgd(self): @@ -391,19 +444,21 @@ def test_sgd(self): class TestImperativeOptimizerNoamDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.NoamDecay(d_model=0.01, - warmup_steps=100, - verbose=True), - parameters=parameter_list) + learning_rate=paddle.optimizer.lr.NoamDecay( + d_model=0.01, warmup_steps=100, verbose=True + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.NoamDecay(d_model=0.01, - warmup_steps=100)) + learning_rate=paddle.optimizer.lr.NoamDecay( + d_model=0.01, warmup_steps=100 + ) + ) return optimizer def func_test_sgd(self): @@ -416,18 +471,21 @@ def test_sgd(self): class TestImperativeOptimizerLambdaDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.LambdaDecay( - learning_rate=0.5, lr_lambda=lambda epoch: 0.9**epoch), - parameters=parameter_list) + learning_rate=0.5, lr_lambda=lambda epoch: 0.9**epoch + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.LambdaDecay( - learning_rate=0.5, lr_lambda=lambda epoch: 0.9**epoch)) + learning_rate=0.5, lr_lambda=lambda epoch: 0.9**epoch + ) + ) return optimizer def func_test_sgd(self): @@ -440,23 +498,25 @@ def test_sgd(self): class TestImperativeOptimizerLinearWarmup(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.LinearWarmup(learning_rate=0.5, - warmup_steps=20, - start_lr=0, - end_lr=0.5), - parameters=parameter_list) + learning_rate=paddle.optimizer.lr.LinearWarmup( + learning_rate=0.5, warmup_steps=20, start_lr=0, end_lr=0.5 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.LinearWarmup(learning_rate=0.5, - warmup_steps=20, - start_lr=0, - end_lr=0.5, - verbose=True)) + learning_rate=paddle.optimizer.lr.LinearWarmup( + learning_rate=0.5, + warmup_steps=20, + start_lr=0, + end_lr=0.5, + verbose=True, + ) + ) return optimizer def func_test_sgd(self): @@ -469,18 +529,21 @@ def test_sgd(self): class TestImperativeOptimizerMultiStepDecay(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.MultiStepDecay( - learning_rate=0.5, milestones=[2, 4, 6], gamma=0.8), - parameters=parameter_list) + learning_rate=0.5, milestones=[2, 4, 6], gamma=0.8 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.MultiStepDecay( - learning_rate=0.5, milestones=[2, 4, 6], gamma=0.8)) + learning_rate=0.5, milestones=[2, 4, 6], gamma=0.8 + ) + ) return optimizer def func_test_sgd(self): @@ -493,19 +556,21 @@ def test_sgd(self): class TestImperativeOptimizerStepLR(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.StepDecay(learning_rate=0.5, - step_size=5, - gamma=0.8), - parameters=parameter_list) + learning_rate=paddle.optimizer.lr.StepDecay( + learning_rate=0.5, step_size=5, gamma=0.8 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.StepDecay( - learning_rate=0.5, step_size=5, gamma=0.8)) + learning_rate=0.5, step_size=5, gamma=0.8 + ) + ) return optimizer def func_test_sgd(self): @@ -518,18 +583,19 @@ def test_sgd(self): class TestImperativeOptimizerReduceOnPlateau(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.SGD( learning_rate=paddle.optimizer.lr.ReduceOnPlateau( - learning_rate=0.5), - parameters=parameter_list) + learning_rate=0.5 + ), + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.SGD( - learning_rate=paddle.optimizer.lr.ReduceOnPlateau( - learning_rate=0.5)) + learning_rate=paddle.optimizer.lr.ReduceOnPlateau(learning_rate=0.5) + ) return optimizer def func_test_sgd(self): @@ -542,7 +608,6 @@ def test_sgd(self): class TestOptimizerLearningRate(unittest.TestCase): - def func_test_constant_lr(self): with fluid.dygraph.guard(): a = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32") @@ -557,10 +622,9 @@ def func_test_constant_lr(self): adam = paddle.optimizer.Adam(0.001, parameters=linear.parameters()) - np.testing.assert_allclose(adam.get_lr(), - 0.001, - rtol=1e-06, - atol=0.0) + np.testing.assert_allclose( + adam.get_lr(), 0.001, rtol=1e-06, atol=0.0 + ) for i in range(10): adam.minimize(loss) @@ -589,8 +653,9 @@ def func_test_lr_decay(self): value = [0.2, 0.4, 0.6, 0.8, 1.0] scheduler = paddle.optimizer.lr.PiecewiseDecay(bd, value) - adam = paddle.optimizer.Adam(scheduler, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + scheduler, parameters=linear.parameters() + ) np.testing.assert_allclose(adam.get_lr(), 0.2, rtol=1e-06, atol=0.0) @@ -618,8 +683,9 @@ def func_test_lr_scheduler_natural_exp(self): base_lr = 1.0 scheduler = paddle.optimizer.lr.NaturalExpDecay(1.0, gamma=0.5) - adam = paddle.optimizer.Adam(scheduler, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + scheduler, parameters=linear.parameters() + ) np.testing.assert_allclose(adam.get_lr(), 1.0, rtol=1e-06, atol=0.0) @@ -657,16 +723,18 @@ def func_test_set_lr(self): np.testing.assert_allclose(lr, lr_list[i], rtol=1e-06, atol=0.0) with self.assertRaises(TypeError): - lr_var = fluid.layers.create_global_var(shape=[1], - value=0.7, - dtype='float32') + lr_var = fluid.layers.create_global_var( + shape=[1], value=0.7, dtype='float32' + ) adam.set_lr(lr_var) with self.assertRaises(RuntimeError): adam = paddle.optimizer.Adam( - paddle.optimizer.lr.NaturalExpDecay(learning_rate=0.1, - gamma=0.5), - parameters=linear.parameters()) + paddle.optimizer.lr.NaturalExpDecay( + learning_rate=0.1, gamma=0.5 + ), + parameters=linear.parameters(), + ) adam.set_lr(0.01) def test_set_lr(self): @@ -676,11 +744,10 @@ def test_set_lr(self): class TestImperativeMomentumOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = MomentumOptimizer(learning_rate=0.001, - momentum=0.9, - parameter_list=parameter_list) + optimizer = MomentumOptimizer( + learning_rate=0.001, momentum=0.9, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -697,11 +764,10 @@ def test_momentum(self): class TestImperativeLarsMomentumOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = LarsMomentumOptimizer(learning_rate=0.001, - momentum=0.9, - parameter_list=parameter_list) + optimizer = LarsMomentumOptimizer( + learning_rate=0.001, momentum=0.9, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -718,10 +784,10 @@ def test_larsmomentum(self): class TestImperativeAdagradOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = AdagradOptimizer(learning_rate=0.2, - parameter_list=parameter_list) + optimizer = AdagradOptimizer( + learning_rate=0.2, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -738,10 +804,10 @@ def test_adagrad(self): class TestImperativeAdamaxOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = AdamaxOptimizer(learning_rate=0.2, - parameter_list=parameter_list) + optimizer = AdamaxOptimizer( + learning_rate=0.2, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -758,21 +824,21 @@ def test_adamax(self): class TestImperativeDpsgdOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = DpsgdOptimizer(learning_rate=0.01, - clip=10.0, - batch_size=16.0, - sigma=1.0, - parameter_list=parameter_list) + optimizer = DpsgdOptimizer( + learning_rate=0.01, + clip=10.0, + batch_size=16.0, + sigma=1.0, + parameter_list=parameter_list, + ) optimizer._seed = 100 return optimizer def get_optimizer(self): - optimizer = DpsgdOptimizer(learning_rate=0.01, - clip=10.0, - batch_size=16.0, - sigma=1.0) + optimizer = DpsgdOptimizer( + learning_rate=0.01, clip=10.0, batch_size=16.0, sigma=1.0 + ) optimizer._seed = 100 return optimizer @@ -786,10 +852,10 @@ def test_dpsgd(self): class TestImperativeDecayedAdagradOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = DecayedAdagradOptimizer(learning_rate=0.2, - parameter_list=parameter_list) + optimizer = DecayedAdagradOptimizer( + learning_rate=0.2, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -806,18 +872,19 @@ def test_decayadagrad(self): class TestImperativeAdadeltaOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = AdadeltaOptimizer(learning_rate=0.0003, - epsilon=1.0e-6, - rho=0.95, - parameter_list=parameter_list) + optimizer = AdadeltaOptimizer( + learning_rate=0.0003, + epsilon=1.0e-6, + rho=0.95, + parameter_list=parameter_list, + ) return optimizer def get_optimizer(self): - optimizer = AdadeltaOptimizer(learning_rate=0.0003, - epsilon=1.0e-6, - rho=0.95) + optimizer = AdadeltaOptimizer( + learning_rate=0.0003, epsilon=1.0e-6, rho=0.95 + ) return optimizer def func_test_adadelta(self): @@ -830,10 +897,10 @@ def test_adadelta(self): class TestImperativeRMSPropOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = RMSPropOptimizer(learning_rate=0.1, - parameter_list=parameter_list) + optimizer = RMSPropOptimizer( + learning_rate=0.1, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -850,10 +917,10 @@ def test_rmsprop(self): class TestImperativeFtrlOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = FtrlOptimizer(learning_rate=0.1, - parameter_list=parameter_list) + optimizer = FtrlOptimizer( + learning_rate=0.1, parameter_list=parameter_list + ) return optimizer def get_optimizer(self): @@ -874,17 +941,18 @@ def exclude_fn(param): class TestImperativeLambOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = paddle.optimizer.Lamb( learning_rate=0.002, exclude_from_weight_decay_fn=exclude_fn, - parameters=parameter_list) + parameters=parameter_list, + ) return optimizer def get_optimizer(self): optimizer = paddle.optimizer.Lamb( - learning_rate=0.002, exclude_from_weight_decay_fn=exclude_fn) + learning_rate=0.002, exclude_from_weight_decay_fn=exclude_fn + ) return optimizer # should fix: may fail in CI-windows @@ -893,11 +961,10 @@ def _test_lamb(self): class TestImperativeModelAverage(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = ModelAverage(0.15, - min_average_window=10000, - max_average_window=12500) + optimizer = ModelAverage( + 0.15, min_average_window=10000, max_average_window=12500 + ) return optimizer def func_test_modelaverage(self): @@ -911,13 +978,14 @@ def test_modelaverage(self): class TestImperativeDGCMomentumOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = DGCMomentumOptimizer(learning_rate=0.0001, - momentum=0.9, - rampup_step=1000, - rampup_begin_step=1252, - sparsity=[0.999, 0.999]) + optimizer = DGCMomentumOptimizer( + learning_rate=0.0001, + momentum=0.9, + rampup_step=1000, + rampup_begin_step=1252, + sparsity=[0.999, 0.999], + ) return optimizer def func_test_dgcmomentum(self): @@ -931,13 +999,14 @@ def test_dgcmomentum(self): class TestImperativeExponentialMovingAverage(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): optimizer = ExponentialMovingAverage(0.999) return optimizer def func_test_exponentialmoving(self): - exception_message = "In dygraph, don't support ExponentialMovingAverage." + exception_message = ( + "In dygraph, don't support ExponentialMovingAverage." + ) self._check_exception(exception_message) def test_exponentialmoving(self): @@ -947,10 +1016,10 @@ def test_exponentialmoving(self): class TestImperativePipelineOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = paddle.optimizer.SGD(learning_rate=0.5, - parameters=parameter_list) + optimizer = paddle.optimizer.SGD( + learning_rate=0.5, parameters=parameter_list + ) optimizer = PipelineOptimizer(optimizer) return optimizer @@ -965,10 +1034,10 @@ def test_pipline(self): class TestImperativeLookaheadOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = paddle.optimizer.SGD(learning_rate=0.5, - parameters=parameter_list) + optimizer = paddle.optimizer.SGD( + learning_rate=0.5, parameters=parameter_list + ) optimizer = LookaheadOptimizer(optimizer, alpha=0.5, k=5) return optimizer @@ -983,10 +1052,10 @@ def test_lookahead(self): class TestImperativeRecomputeOptimizer(TestImperativeOptimizerBase): - def get_optimizer_dygraph(self, parameter_list): - optimizer = paddle.optimizer.SGD(learning_rate=0.5, - parameters=parameter_list) + optimizer = paddle.optimizer.SGD( + learning_rate=0.5, parameters=parameter_list + ) optimizer = RecomputeOptimizer(optimizer) return optimizer @@ -1001,16 +1070,17 @@ def test_recompute(self): class TestImperativeOptimizerList(unittest.TestCase): - def func_test_parameter_list(self): with fluid.dygraph.guard(): linear_1 = Linear(10, 10) linear_2 = Linear(10, 10) - sgd = paddle.optimizer.SGD(1.0, - parameters=itertools.chain( - linear_1.parameters(), - linear_2.parameters())) + sgd = paddle.optimizer.SGD( + 1.0, + parameters=itertools.chain( + linear_1.parameters(), linear_2.parameters() + ), + ) in_np = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32") in_data = fluid.dygraph.to_variable(in_np) @@ -1022,8 +1092,9 @@ def func_test_parameter_list(self): sgd.minimize(loss) self.assertTrue( - len(sgd._parameter_list) == len(linear_1.parameters() + - linear_2.parameters())) + len(sgd._parameter_list) + == len(linear_1.parameters() + linear_2.parameters()) + ) def test_parameter_list(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_parallel_coalesce_split.py b/python/paddle/fluid/tests/unittests/test_imperative_parallel_coalesce_split.py index 7c6d13399c0e0..300a195f6dce3 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_parallel_coalesce_split.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_parallel_coalesce_split.py @@ -20,11 +20,14 @@ from paddle.fluid import core from paddle.fluid.dygraph.parallel import DataParallel from paddle.fluid.dygraph.base import to_variable -from paddle.fluid.dygraph.parallel import _coalesce_tensors, _split_tensors, _reshape_inplace +from paddle.fluid.dygraph.parallel import ( + _coalesce_tensors, + _split_tensors, + _reshape_inplace, +) class MyLayer(fluid.Layer): - def __init__(self, name_scope): super(MyLayer, self).__init__(name_scope) @@ -36,7 +39,6 @@ def forward(self, inputs): class TestImperativeParallelCoalesceSplit(unittest.TestCase): - def test_coalesce_split(self): with fluid.dygraph.guard(): test_layer = MyLayer("test_layer") @@ -47,8 +49,9 @@ def test_coalesce_split(self): vars = [] vars.append(to_variable(np.random.random([2, 3]).astype("float32"))) vars.append(to_variable(np.random.random([4, 9]).astype("float32"))) - vars.append(to_variable( - np.random.random([10, 1]).astype("float32"))) + vars.append( + to_variable(np.random.random([10, 1]).astype("float32")) + ) var_groups = OrderedDict() var_groups.setdefault(0, vars) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_partitial_backward.py b/python/paddle/fluid/tests/unittests/test_imperative_partitial_backward.py index f966b2ba9a8e3..7f723a5d665d5 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_partitial_backward.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_partitial_backward.py @@ -19,7 +19,6 @@ class TestImperativePartitialBackward(unittest.TestCase): - def func_partitial_backward(self): with fluid.dygraph.guard(): x = np.random.randn(2, 4, 5).astype("float32") @@ -39,11 +38,14 @@ def func_partitial_backward(self): self.assertIsNone(param._grad_ivar()) optimizer = fluid.optimizer.AdamOptimizer( - parameter_list=(linear1.parameters() + linear2.parameters())) + parameter_list=(linear1.parameters() + linear2.parameters()) + ) _, params_grads = optimizer.minimize(loss) - self.assertListEqual(sorted([p.name for p in linear1.parameters()]), - sorted([p_g[0].name for p_g in params_grads])) + self.assertListEqual( + sorted([p.name for p in linear1.parameters()]), + sorted([p_g[0].name for p_g in params_grads]), + ) linear1.clear_gradients() linear2.clear_gradients() diff --git a/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn.py b/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn.py index 0588016ce2765..ecf81ed54336c 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn.py @@ -28,13 +28,9 @@ class SimpleLSTMRNN(fluid.Layer): - - def __init__(self, - hidden_size, - num_steps, - num_layers=2, - init_scale=0.1, - dropout=None): + def __init__( + self, hidden_size, num_steps, num_layers=2, init_scale=0.1, dropout=None + ): super(SimpleLSTMRNN, self).__init__() self._hidden_size = hidden_size self._num_layers = num_layers @@ -56,19 +52,26 @@ def _create_parameter(self): weight_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 2, self._hidden_size * 4], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)) + low=-self._init_scale, high=self._init_scale + ), + ) self.weight_1_arr.append(self.add_parameter('w_%d' % i, weight_1)) bias_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 4], dtype="float32", - default_initializer=fluid.initializer.Constant(0.0)) + default_initializer=fluid.initializer.Constant(0.0), + ) self.bias_arr.append(self.add_parameter('b_%d' % i, bias_1)) def forward(self, input_embedding, init_hidden=None, init_cell=None): @@ -76,29 +79,29 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): self.hidden_array = [] for i in range(self._num_layers): - pre_hidden = fluid.layers.slice(init_hidden, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_cell = fluid.layers.slice(init_cell, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_hidden = fluid.layers.reshape(pre_hidden, - shape=[-1, self._hidden_size]) - pre_cell = fluid.layers.reshape(pre_cell, - shape=[-1, self._hidden_size]) + pre_hidden = fluid.layers.slice( + init_hidden, axes=[0], starts=[i], ends=[i + 1] + ) + pre_cell = fluid.layers.slice( + init_cell, axes=[0], starts=[i], ends=[i + 1] + ) + pre_hidden = fluid.layers.reshape( + pre_hidden, shape=[-1, self._hidden_size] + ) + pre_cell = fluid.layers.reshape( + pre_cell, shape=[-1, self._hidden_size] + ) self.hidden_array.append(pre_hidden) self.cell_array.append(pre_cell) res = [] for index in range(self._num_steps): - self._input = fluid.layers.slice(input_embedding, - axes=[1], - starts=[index], - ends=[index + 1]) - self._input = fluid.layers.reshape(self._input, - shape=[-1, self._hidden_size]) + self._input = fluid.layers.slice( + input_embedding, axes=[1], starts=[index], ends=[index + 1] + ) + self._input = fluid.layers.reshape( + self._input, shape=[-1, self._hidden_size] + ) for k in range(self._num_layers): pre_hidden = self.hidden_array[k] pre_cell = self.cell_array[k] @@ -109,11 +112,12 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): gate_input = fluid.layers.matmul(x=nn, y=weight_1) gate_input = fluid.layers.elementwise_add(gate_input, bias) - i, j, f, o = fluid.layers.split(gate_input, - num_or_sections=4, - dim=-1) + i, j, f, o = fluid.layers.split( + gate_input, num_or_sections=4, dim=-1 + ) c = pre_cell * fluid.layers.sigmoid(f) + fluid.layers.sigmoid( - i) * fluid.layers.tanh(j) + i + ) * fluid.layers.tanh(j) m = fluid.layers.tanh(c) * fluid.layers.sigmoid(o) self.hidden_array[k] = m self.cell_array[k] = c @@ -123,33 +127,39 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): self._input = fluid.layers.dropout( self._input, dropout_prob=self._dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) res.append( - fluid.layers.reshape(self._input, - shape=[1, -1, self._hidden_size])) + fluid.layers.reshape( + self._input, shape=[1, -1, self._hidden_size] + ) + ) real_res = fluid.layers.concat(res, 0) real_res = fluid.layers.transpose(x=real_res, perm=[1, 0, 2]) last_hidden = fluid.layers.concat(self.hidden_array, 1) last_hidden = fluid.layers.reshape( - last_hidden, shape=[-1, self._num_layers, self._hidden_size]) + last_hidden, shape=[-1, self._num_layers, self._hidden_size] + ) last_hidden = fluid.layers.transpose(x=last_hidden, perm=[1, 0, 2]) last_cell = fluid.layers.concat(self.cell_array, 1) last_cell = fluid.layers.reshape( - last_cell, shape=[-1, self._num_layers, self._hidden_size]) + last_cell, shape=[-1, self._num_layers, self._hidden_size] + ) last_cell = fluid.layers.transpose(x=last_cell, perm=[1, 0, 2]) return real_res, last_hidden, last_cell class PtbModel(fluid.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_layers=2, - num_steps=20, - init_scale=0.1, - is_sparse=False, - dropout=None): + def __init__( + self, + hidden_size, + vocab_size, + num_layers=2, + num_steps=20, + init_scale=0.1, + is_sparse=False, + dropout=None, + ): super(PtbModel, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -157,11 +167,13 @@ def __init__(self, self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout - self.simple_lstm_rnn = SimpleLSTMRNN(hidden_size, - num_steps, - num_layers=num_layers, - init_scale=init_scale, - dropout=dropout) + self.simple_lstm_rnn = SimpleLSTMRNN( + hidden_size, + num_steps, + num_layers=num_layers, + init_scale=init_scale, + dropout=dropout, + ) self.embedding = Embedding( size=[vocab_size, hidden_size], dtype='float32', @@ -169,46 +181,60 @@ def __init__(self, param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label, init_hidden, init_cell): init_h = fluid.layers.reshape( - init_hidden, shape=[self.num_layers, -1, self.hidden_size]) + init_hidden, shape=[self.num_layers, -1, self.hidden_size] + ) init_c = fluid.layers.reshape( - init_cell, shape=[self.num_layers, -1, self.hidden_size]) + init_cell, shape=[self.num_layers, -1, self.hidden_size] + ) x_emb = self.embedding(input) x_emb = fluid.layers.reshape( - x_emb, shape=[-1, self.num_steps, self.hidden_size]) + x_emb, shape=[-1, self.num_steps, self.hidden_size] + ) if self.dropout is not None and self.dropout > 0.0: x_emb = fluid.layers.dropout( x_emb, dropout_prob=self.drop_out, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn_out, last_hidden, last_cell = self.simple_lstm_rnn( - x_emb, init_h, init_c) + x_emb, init_h, init_c + ) rnn_out = fluid.layers.reshape( - rnn_out, shape=[-1, self.num_steps, self.hidden_size]) + rnn_out, shape=[-1, self.num_steps, self.hidden_size] + ) projection = fluid.layers.matmul(rnn_out, self.softmax_weight) projection = fluid.layers.elementwise_add(projection, self.softmax_bias) - projection = fluid.layers.reshape(projection, - shape=[-1, self.vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + projection = fluid.layers.reshape( + projection, shape=[-1, self.vocab_size] + ) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -217,7 +243,6 @@ def forward(self, input, label, init_hidden, init_cell): class TestDygraphPtbRnn(unittest.TestCase): - def func_test_ptb_rnn(self): for is_sparse in [True, False]: self.ptb_rnn_cpu_float32(is_sparse) @@ -242,15 +267,18 @@ def ptb_rnn_cpu_float32(self, is_sparse): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=is_sparse) - - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=ptb_model.parameters()) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=is_sparse, + ) + + sgd = SGDOptimizer( + learning_rate=1e-3, parameter_list=ptb_model.parameters() + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -265,27 +293,32 @@ def ptb_rnn_cpu_float32(self, is_sparse): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) if i % 5 == 0 and _in_legacy_dygraph(): outs, traced_layer = TracedLayer.trace( - ptb_model, [x, y, init_hidden, init_cell]) + ptb_model, [x, y, init_hidden, init_cell] + ) outs_static = traced_layer([x, y, init_hidden, init_cell]) helper.assertEachVar(outs, outs_static) if program is not None: self.assertTrue( - is_equal_program(traced_layer.program, program)) + is_equal_program(traced_layer.program, program) + ) program = traced_layer.program traced_layer.save_inference_model( - './infe_imperative_ptb_rnn', feed=list(range(4))) + './infe_imperative_ptb_rnn', feed=list(range(4)) + ) else: outs = ptb_model(x, y, init_hidden, init_cell) @@ -308,29 +341,35 @@ def ptb_rnn_cpu_float32(self, is_sparse): with new_program_scope(): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=is_sparse) - - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=is_sparse, + ) + + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) sgd = SGDOptimizer(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) sgd.minimize(static_loss) static_param_updated = dict() static_param_init = dict() @@ -338,8 +377,10 @@ def ptb_rnn_cpu_float32(self, is_sparse): for param in ptb_model.parameters(): static_param_name_list.append(param.name) - out = exe.run(framework.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + framework.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init[static_param_name_list[i]] = out[i] static_loss_value = None @@ -351,33 +392,40 @@ def ptb_rnn_cpu_float32(self, is_sparse): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] if i == batch_num - 1: for k in range(3, len(out)): - static_param_updated[static_param_name_list[k - - 3]] = out[k] + static_param_updated[ + static_param_name_list[k - 3] + ] = out[k] np.testing.assert_array_equal(static_loss_value, dy_loss_value) - np.testing.assert_array_equal(static_last_cell_value, - dy_last_cell_value) - np.testing.assert_array_equal(static_last_hidden_value, - dy_last_hidden_value) + np.testing.assert_array_equal( + static_last_cell_value, dy_last_cell_value + ) + np.testing.assert_array_equal( + static_last_hidden_value, dy_last_hidden_value + ) for key, value in static_param_init.items(): np.testing.assert_array_equal(value, dy_param_init[key]) for key, value in static_param_updated.items(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn_sorted_gradient.py b/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn_sorted_gradient.py index 286049500c296..38c8b9ca73dd4 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn_sorted_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn_sorted_gradient.py @@ -26,7 +26,6 @@ class TestDygraphPtbRnnSortGradient(unittest.TestCase): - def func_ptb_rnn_sort_gradient(self): for is_sparse in [True, False]: self.ptb_rnn_sort_gradient_cpu_float32(is_sparse) @@ -47,15 +46,18 @@ def ptb_rnn_sort_gradient_cpu_float32(self, is_sparse): paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=is_sparse) - - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=ptb_model.parameters()) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=is_sparse, + ) + + sgd = SGDOptimizer( + learning_rate=1e-3, parameter_list=ptb_model.parameters() + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -68,15 +70,18 @@ def ptb_rnn_sort_gradient_cpu_float32(self, is_sparse): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -95,29 +100,35 @@ def ptb_rnn_sort_gradient_cpu_float32(self, is_sparse): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=is_sparse) - - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=is_sparse, + ) + + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) sgd = SGDOptimizer(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps, 1], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps, 1], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) sgd.minimize(static_loss) static_param_updated = dict() static_param_init = dict() @@ -125,8 +136,10 @@ def ptb_rnn_sort_gradient_cpu_float32(self, is_sparse): for param in ptb_model.parameters(): static_param_name_list.append(param.name) - out = exe.run(framework.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + framework.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init[static_param_name_list[i]] = out[i] static_loss_value = None @@ -138,33 +151,40 @@ def ptb_rnn_sort_gradient_cpu_float32(self, is_sparse): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] if i == batch_num - 1: for k in range(3, len(out)): - static_param_updated[static_param_name_list[k - - 3]] = out[k] + static_param_updated[ + static_param_name_list[k - 3] + ] = out[k] np.testing.assert_array_equal(static_loss_value, dy_loss_value) - np.testing.assert_array_equal(static_last_cell_value, - dy_last_cell_value) - np.testing.assert_array_equal(static_last_hidden_value, - dy_last_hidden_value) + np.testing.assert_array_equal( + static_last_cell_value, dy_last_cell_value + ) + np.testing.assert_array_equal( + static_last_hidden_value, dy_last_hidden_value + ) for key, value in static_param_init.items(): np.testing.assert_array_equal(value, dy_param_init[key]) for key, value in static_param_updated.items(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_recurrent_usage.py b/python/paddle/fluid/tests/unittests/test_imperative_recurrent_usage.py index 622d174eefcc0..123803731da1b 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_recurrent_usage.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_recurrent_usage.py @@ -23,7 +23,6 @@ class RecurrentTest(fluid.Layer): - def __init__(self, name_scope): super(RecurrentTest, self).__init__(name_scope) @@ -34,7 +33,6 @@ def forward(self, in1, in2): class TestRecurrentFeed(unittest.TestCase): - def test_recurrent_feed(self): seed = 90 @@ -84,28 +82,33 @@ def test_recurrent_feed(self): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - in1 = fluid.layers.data(name="inp1", - shape=[2, 2], - append_batch_size=False) - in2 = fluid.layers.data(name="inp2", - shape=[2, 2], - append_batch_size=False) + in1 = fluid.layers.data( + name="inp1", shape=[2, 2], append_batch_size=False + ) + in2 = fluid.layers.data( + name="inp2", shape=[2, 2], append_batch_size=False + ) rt1 = RecurrentTest("RecurrentTest") static_sum_out, static_out = rt1(in1, in2) fluid.backward.append_backward(static_sum_out) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) - - static_dout = fluid.default_main_program().block( - 0)._find_var_recursive(static_out.name + "@GRAD") + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + + static_dout = ( + fluid.default_main_program() + .block(0) + ._find_var_recursive(static_out.name + "@GRAD") + ) fetch_list = [static_sum_out, static_out, static_dout] for i in range(3): - out = exe.run(fluid.default_main_program(), - feed={ - "inp1": original_np1, - "inp2": original_np2 - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"inp1": original_np1, "inp2": original_np2}, + fetch_list=fetch_list, + ) static_out_value = out[1] static_sum_out = out[0] static_dout = out[2] diff --git a/python/paddle/fluid/tests/unittests/test_imperative_reinforcement.py b/python/paddle/fluid/tests/unittests/test_imperative_reinforcement.py index a831737d75242..5e52815bbb30d 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_reinforcement.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_reinforcement.py @@ -25,7 +25,6 @@ class Policy(fluid.dygraph.Layer): - def __init__(self, input_size): super(Policy, self).__init__() @@ -46,7 +45,6 @@ def forward(self, inputs): class TestImperativeMnist(unittest.TestCase): - def test_mnist_float32(self): seed = 90 epoch_num = 1 @@ -83,8 +81,9 @@ def run_dygraph(): loss_probs = fluid.layers.elementwise_mul(dy_reward, loss_probs) loss = fluid.layers.reduce_sum(loss_probs) - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=policy.parameters()) + sgd = SGDOptimizer( + learning_rate=1e-3, parameter_list=policy.parameters() + ) dy_param_init_value = {} @@ -108,29 +107,35 @@ def run_dygraph(): with fluid.dygraph.guard(): with _test_eager_guard(): - eager_out, eager_param_init_value, eager_param_value = run_dygraph( - ) + ( + eager_out, + eager_param_init_value, + eager_param_value, + ) = run_dygraph() with new_program_scope(): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) policy = Policy(input_size=4) st_sgd = SGDOptimizer(learning_rate=1e-3) - st_state = fluid.layers.data(name='st_state', - shape=[4], - dtype='float32') - st_reward = fluid.layers.data(name='st_reward', - shape=[1], - dtype='float32') - st_mask = fluid.layers.data(name='st_mask', - shape=[2], - dtype='float32') + st_state = fluid.layers.data( + name='st_state', shape=[4], dtype='float32' + ) + st_reward = fluid.layers.data( + name='st_reward', shape=[1], dtype='float32' + ) + st_mask = fluid.layers.data( + name='st_mask', shape=[2], dtype='float32' + ) st_loss_probs = policy(st_state) @@ -139,7 +144,8 @@ def run_dygraph(): st_loss_probs = fluid.layers.reduce_sum(st_loss_probs, dim=-1) st_loss_probs = fluid.layers.elementwise_mul( - st_reward, st_loss_probs) + st_reward, st_loss_probs + ) st_loss = fluid.layers.reduce_sum(st_loss_probs) st_sgd.minimize(st_loss) @@ -150,8 +156,10 @@ def run_dygraph(): for param in policy.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -159,13 +167,11 @@ def run_dygraph(): fetch_list = [st_loss.name] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "st_state": state, - "st_reward": reward, - "st_mask": mask - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"st_state": state, "st_reward": reward, "st_mask": mask}, + fetch_list=fetch_list, + ) static_param_value = {} static_out = out[0] diff --git a/python/paddle/fluid/tests/unittests/test_imperative_resnet.py b/python/paddle/fluid/tests/unittests/test_imperative_resnet.py index 55a51347283a8..657246795f7b5 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_resnet.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_resnet.py @@ -26,7 +26,7 @@ from paddle.fluid.dygraph import TracedLayer from paddle.fluid.framework import _test_eager_guard, _in_legacy_dygraph -#NOTE(zhiqiu): run with FLAGS_cudnn_deterministic=1 +# NOTE(zhiqiu): run with FLAGS_cudnn_deterministic=1 batch_size = 8 train_parameters = { @@ -37,7 +37,7 @@ "name": "piecewise_decay", "batch_size": batch_size, "epochs": [30, 60, 90], - "steps": [0.1, 0.01, 0.001, 0.0001] + "steps": [0.1, 0.01, 0.001, 0.0001], }, "batch_size": batch_size, "lr": 0.1, @@ -60,8 +60,9 @@ def optimizer_setting(params, parameter_list=None): lr = [] lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] if fluid._non_static_mode(): - optimizer = fluid.optimizer.SGD(learning_rate=0.01, - parameter_list=parameter_list) + optimizer = fluid.optimizer.SGD( + learning_rate=0.01, parameter_list=parameter_list + ) else: optimizer = fluid.optimizer.SGD(learning_rate=0.01) # TODO(minqiyang): Add learning rate scheduler support to dygraph mode @@ -76,26 +77,29 @@ def optimizer_setting(params, parameter_list=None): class ConvBNLayer(fluid.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None, - use_cudnn=False): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + use_cudnn=False, + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - bias_attr=False, - use_cudnn=use_cudnn) + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + bias_attr=False, + use_cudnn=use_cudnn, + ) self._batch_norm = BatchNorm(num_filters, act=act) @@ -107,38 +111,42 @@ def forward(self, inputs): class BottleneckBlock(fluid.Layer): - - def __init__(self, - num_channels, - num_filters, - stride, - shortcut=True, - use_cudnn=False): + def __init__( + self, num_channels, num_filters, stride, shortcut=True, use_cudnn=False + ): super(BottleneckBlock, self).__init__() - self.conv0 = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=1, - act='relu', - use_cudnn=use_cudnn) - self.conv1 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=stride, - act='relu', - use_cudnn=use_cudnn) - self.conv2 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters * 4, - filter_size=1, - act=None, - use_cudnn=use_cudnn) + self.conv0 = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=1, + act='relu', + use_cudnn=use_cudnn, + ) + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + act='relu', + use_cudnn=use_cudnn, + ) + self.conv2 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters * 4, + filter_size=1, + act=None, + use_cudnn=use_cudnn, + ) if not shortcut: - self.short = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters * 4, - filter_size=1, - stride=stride, - use_cudnn=use_cudnn) + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters * 4, + filter_size=1, + stride=stride, + use_cudnn=use_cudnn, + ) self.shortcut = shortcut @@ -159,14 +167,16 @@ def forward(self, inputs): class ResNet(fluid.Layer): - def __init__(self, layers=50, class_dim=102, use_cudnn=True): super(ResNet, self).__init__() self.layers = layers supported_layers = [50, 101, 152] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format(supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) if layers == 50: depth = [3, 4, 6, 3] @@ -177,16 +187,17 @@ def __init__(self, layers=50, class_dim=102, use_cudnn=True): num_channels = [64, 256, 512, 1024] num_filters = [64, 128, 256, 512] - self.conv = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu', - use_cudnn=use_cudnn) - self.pool2d_max = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + use_cudnn=use_cudnn, + ) + self.pool2d_max = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) self.bottleneck_block_list = [] for block in range(len(depth)): @@ -194,22 +205,27 @@ def __init__(self, layers=50, class_dim=102, use_cudnn=True): for i in range(depth[block]): bottleneck_block = self.add_sublayer( 'bb_%d_%d' % (block, i), - BottleneckBlock(num_channels=num_channels[block] - if i == 0 else num_filters[block] * 4, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - shortcut=shortcut, - use_cudnn=use_cudnn)) + BottleneckBlock( + num_channels=num_channels[block] + if i == 0 + else num_filters[block] * 4, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + shortcut=shortcut, + use_cudnn=use_cudnn, + ), + ) self.bottleneck_block_list.append(bottleneck_block) shortcut = True - self.pool2d_avg = Pool2D(pool_size=7, - pool_type='avg', - global_pooling=True) + self.pool2d_avg = Pool2D( + pool_size=7, pool_type='avg', global_pooling=True + ) self.pool2d_avg_output = num_filters[-1] * 4 * 1 * 1 import math + stdv = 1.0 / math.sqrt(2048 * 1.0) self.out = Linear( @@ -217,7 +233,9 @@ def __init__(self, layers=50, class_dim=102, use_cudnn=True): class_dim, act='softmax', param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv))) + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + ) def forward(self, inputs): y = self.conv(inputs) @@ -231,9 +249,7 @@ def forward(self, inputs): class TestDygraphResnet(unittest.TestCase): - def reader_decorator(self, reader): - def _reader_imple(): for item in reader(): doc = np.array(item[0]).reshape(3, 224, 224) @@ -255,13 +271,15 @@ def func_test_resnet_float32(self): paddle.framework.random._manual_program_seed(seed) resnet = ResNet() - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) np.random.seed(seed) train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), - batch_size=batch_size) + batch_size=batch_size, + ) dy_param_init_value = {} for param in resnet.parameters(): @@ -274,10 +292,14 @@ def func_test_resnet_float32(self): if batch_id >= batch_num: break - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(batch_size, 1) + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(batch_size, 1) + ) img = to_variable(dy_x_data) label = to_variable(y_data) @@ -288,10 +310,12 @@ def func_test_resnet_float32(self): out, traced_layer = TracedLayer.trace(resnet, img) if program is not None: self.assertTrue( - is_equal_program(program, traced_layer.program)) + is_equal_program(program, traced_layer.program) + ) traced_layer.save_inference_model( - './infer_imperative_resnet') + './infer_imperative_resnet' + ) program = traced_layer.program else: @@ -322,9 +346,11 @@ def func_test_resnet_float32(self): for param in resnet.parameters(): if param.trainable: np_array = np.array( - param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - core.grad_var_suffix()] = np_array + param._grad_ivar().value().get_tensor() + ) + dy_grad_value[ + param.name + core.grad_var_suffix() + ] = np_array optimizer.minimize(avg_loss) resnet.clear_gradients() @@ -337,8 +363,11 @@ def func_test_resnet_float32(self): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) resnet = ResNet() optimizer = optimizer_setting(train_parameters) @@ -346,11 +375,12 @@ def func_test_resnet_float32(self): np.random.seed(seed) train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), - batch_size=batch_size) + batch_size=batch_size, + ) - img = fluid.layers.data(name='pixel', - shape=[3, 224, 224], - dtype='float32') + img = fluid.layers.data( + name='pixel', shape=[3, 224, 224], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') out = resnet(img) loss = fluid.layers.cross_entropy(input=out, label=label) @@ -365,11 +395,14 @@ def func_test_resnet_float32(self): static_param_name_list.append(param.name) for param in resnet.parameters(): if param.trainable: - static_grad_name_list.append(param.name + - core.grad_var_suffix()) + static_grad_name_list.append( + param.name + core.grad_var_suffix() + ) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -379,9 +412,13 @@ def func_test_resnet_float32(self): break static_x_data = np.array( - [x[0].reshape(3, 224, 224) for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape([batch_size, 1]) + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([batch_size, 1]) + ) if traced_layer is not None: traced_layer([static_x_data]) @@ -389,26 +426,30 @@ def func_test_resnet_float32(self): fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) fetch_list.extend(static_grad_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) static_param_value = {} static_grad_value = {} static_out = out[0] param_start_pos = 1 grad_start_pos = len(static_param_name_list) + param_start_pos - for i in range(param_start_pos, - len(static_param_name_list) + param_start_pos): - static_param_value[static_param_name_list[ - i - param_start_pos]] = out[i] - for i in range(grad_start_pos, - len(static_grad_name_list) + grad_start_pos): - static_grad_value[static_grad_name_list[ - i - grad_start_pos]] = out[i] + for i in range( + param_start_pos, + len(static_param_name_list) + param_start_pos, + ): + static_param_value[ + static_param_name_list[i - param_start_pos] + ] = out[i] + for i in range( + grad_start_pos, len(static_grad_name_list) + grad_start_pos + ): + static_grad_value[ + static_grad_name_list[i - grad_start_pos] + ] = out[i] print("static", static_out) print("dygraph", dy_out) @@ -417,9 +458,9 @@ def func_test_resnet_float32(self): self.assertEqual(len(dy_param_init_value), len(static_param_init_value)) for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value[key], rtol=1e-05 + ) self.assertTrue(np.isfinite(value.all())) self.assertFalse(np.isnan(value.any())) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_resnet_sorted_gradient.py b/python/paddle/fluid/tests/unittests/test_imperative_resnet_sorted_gradient.py index e28f2c1408620..9e203092dc77b 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_resnet_sorted_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_resnet_sorted_gradient.py @@ -32,7 +32,7 @@ "name": "piecewise_decay", "batch_size": batch_size, "epochs": [30, 60, 90], - "steps": [0.1, 0.01, 0.001, 0.0001] + "steps": [0.1, 0.01, 0.001, 0.0001], }, "batch_size": batch_size, "lr": 0.1, @@ -55,8 +55,9 @@ def optimizer_setting(params, parameter_list=None): lr = [] lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] if fluid._non_static_mode(): - optimizer = fluid.optimizer.SGD(learning_rate=0.01, - parameter_list=parameter_list) + optimizer = fluid.optimizer.SGD( + learning_rate=0.01, parameter_list=parameter_list + ) else: optimizer = fluid.optimizer.SGD(learning_rate=0.01) # TODO(minqiyang): Add learning rate scheduler support to dygraph mode @@ -71,7 +72,6 @@ def optimizer_setting(params, parameter_list=None): class TestDygraphResnetSortGradient(unittest.TestCase): - def func_test_resnet_sort_gradient_float32(self): seed = 90 @@ -83,14 +83,17 @@ def func_test_resnet_sort_gradient_float32(self): paddle.framework.random._manual_program_seed(seed) resnet = ResNet() - optimizer = optimizer_setting(train_parameters, - parameter_list=resnet.parameters()) + optimizer = optimizer_setting( + train_parameters, parameter_list=resnet.parameters() + ) np.random.seed(seed) import random + random.seed = seed train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), - batch_size=batch_size) + batch_size=batch_size, + ) dy_param_init_value = {} for param in resnet.parameters(): @@ -100,10 +103,14 @@ def func_test_resnet_sort_gradient_float32(self): if batch_id >= batch_num: break - dy_x_data = np.array([x[0].reshape(3, 224, 224) - for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape(batch_size, 1) + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape(batch_size, 1) + ) img = to_variable(dy_x_data) label = to_variable(y_data) @@ -126,9 +133,11 @@ def func_test_resnet_sort_gradient_float32(self): for param in resnet.parameters(): if param.trainable: np_array = np.array( - param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - core.grad_var_suffix()] = np_array + param._grad_ivar().value().get_tensor() + ) + dy_grad_value[ + param.name + core.grad_var_suffix() + ] = np_array optimizer.minimize(avg_loss) resnet.clear_gradients() @@ -141,22 +150,27 @@ def func_test_resnet_sort_gradient_float32(self): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) resnet = ResNet() optimizer = optimizer_setting(train_parameters) np.random.seed(seed) import random + random.seed = seed train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), - batch_size=batch_size) + batch_size=batch_size, + ) - img = fluid.layers.data(name='pixel', - shape=[3, 224, 224], - dtype='float32') + img = fluid.layers.data( + name='pixel', shape=[3, 224, 224], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') out = resnet(img) loss = fluid.layers.cross_entropy(input=out, label=label) @@ -171,11 +185,14 @@ def func_test_resnet_sort_gradient_float32(self): static_param_name_list.append(param.name) for param in resnet.parameters(): if param.trainable: - static_grad_name_list.append(param.name + - core.grad_var_suffix()) + static_grad_name_list.append( + param.name + core.grad_var_suffix() + ) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -185,42 +202,50 @@ def func_test_resnet_sort_gradient_float32(self): break static_x_data = np.array( - [x[0].reshape(3, 224, 224) for x in data]).astype('float32') - y_data = np.array([x[1] for x in data - ]).astype('int64').reshape([batch_size, 1]) + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([batch_size, 1]) + ) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) fetch_list.extend(static_grad_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) static_param_value = {} static_grad_value = {} static_out = out[0] param_start_pos = 1 grad_start_pos = len(static_param_name_list) + param_start_pos - for i in range(param_start_pos, - len(static_param_name_list) + param_start_pos): - static_param_value[static_param_name_list[ - i - param_start_pos]] = out[i] - for i in range(grad_start_pos, - len(static_grad_name_list) + grad_start_pos): - static_grad_value[static_grad_name_list[ - i - grad_start_pos]] = out[i] + for i in range( + param_start_pos, + len(static_param_name_list) + param_start_pos, + ): + static_param_value[ + static_param_name_list[i - param_start_pos] + ] = out[i] + for i in range( + grad_start_pos, len(static_grad_name_list) + grad_start_pos + ): + static_grad_value[ + static_grad_name_list[i - grad_start_pos] + ] = out[i] np.testing.assert_allclose(static_out, dy_out, rtol=1e-05) self.assertEqual(len(dy_param_init_value), len(static_param_init_value)) for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value[key], rtol=1e-05 + ) self.assertTrue(np.isfinite(value.all())) self.assertFalse(np.isnan(value.any())) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_save_load.py b/python/paddle/fluid/tests/unittests/test_imperative_save_load.py index 7933d4f15456e..55cc00b12ab91 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_save_load.py @@ -26,13 +26,9 @@ class SimpleLSTMRNN(fluid.Layer): - - def __init__(self, - hidden_size, - num_steps, - num_layers=2, - init_scale=0.1, - dropout=None): + def __init__( + self, hidden_size, num_steps, num_layers=2, init_scale=0.1, dropout=None + ): super(SimpleLSTMRNN, self).__init__() self._hidden_size = hidden_size self._num_layers = num_layers @@ -51,19 +47,26 @@ def __init__(self, weight_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 2, self._hidden_size * 4], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)) + low=-self._init_scale, high=self._init_scale + ), + ) self.weight_1_arr.append(self.add_parameter('w_%d' % i, weight_1)) bias_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 4], dtype="float32", - default_initializer=fluid.initializer.Constant(0.0)) + default_initializer=fluid.initializer.Constant(0.0), + ) self.bias_arr.append(self.add_parameter('b_%d' % i, bias_1)) def forward(self, input_embedding, init_hidden=None, init_cell=None): @@ -71,29 +74,29 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): self.hidden_array = [] for i in range(self._num_layers): - pre_hidden = fluid.layers.slice(init_hidden, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_cell = fluid.layers.slice(init_cell, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_hidden = fluid.layers.reshape(pre_hidden, - shape=[-1, self._hidden_size]) - pre_cell = fluid.layers.reshape(pre_cell, - shape=[-1, self._hidden_size]) + pre_hidden = fluid.layers.slice( + init_hidden, axes=[0], starts=[i], ends=[i + 1] + ) + pre_cell = fluid.layers.slice( + init_cell, axes=[0], starts=[i], ends=[i + 1] + ) + pre_hidden = fluid.layers.reshape( + pre_hidden, shape=[-1, self._hidden_size] + ) + pre_cell = fluid.layers.reshape( + pre_cell, shape=[-1, self._hidden_size] + ) self.hidden_array.append(pre_hidden) self.cell_array.append(pre_cell) res = [] for index in range(self._num_steps): - self._input = fluid.layers.slice(input_embedding, - axes=[1], - starts=[index], - ends=[index + 1]) - self._input = fluid.layers.reshape(self._input, - shape=[-1, self._hidden_size]) + self._input = fluid.layers.slice( + input_embedding, axes=[1], starts=[index], ends=[index + 1] + ) + self._input = fluid.layers.reshape( + self._input, shape=[-1, self._hidden_size] + ) for k in range(self._num_layers): pre_hidden = self.hidden_array[k] pre_cell = self.cell_array[k] @@ -104,11 +107,12 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): gate_input = fluid.layers.matmul(x=nn, y=weight_1) gate_input = fluid.layers.elementwise_add(gate_input, bias) - i, j, f, o = fluid.layers.split(gate_input, - num_or_sections=4, - dim=-1) + i, j, f, o = fluid.layers.split( + gate_input, num_or_sections=4, dim=-1 + ) c = pre_cell * fluid.layers.sigmoid(f) + fluid.layers.sigmoid( - i) * fluid.layers.tanh(j) + i + ) * fluid.layers.tanh(j) m = fluid.layers.tanh(c) * fluid.layers.sigmoid(o) self.hidden_array[k] = m self.cell_array[k] = c @@ -118,32 +122,38 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): self._input = fluid.layers.dropout( self._input, dropout_prob=self._dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) res.append( - fluid.layers.reshape(self._input, - shape=[1, -1, self._hidden_size])) + fluid.layers.reshape( + self._input, shape=[1, -1, self._hidden_size] + ) + ) real_res = fluid.layers.concat(res, 0) real_res = fluid.layers.transpose(x=real_res, perm=[1, 0, 2]) last_hidden = fluid.layers.concat(self.hidden_array, 1) last_hidden = fluid.layers.reshape( - last_hidden, shape=[-1, self._num_layers, self._hidden_size]) + last_hidden, shape=[-1, self._num_layers, self._hidden_size] + ) last_hidden = fluid.layers.transpose(x=last_hidden, perm=[1, 0, 2]) last_cell = fluid.layers.concat(self.cell_array, 1) last_cell = fluid.layers.reshape( - last_cell, shape=[-1, self._num_layers, self._hidden_size]) + last_cell, shape=[-1, self._num_layers, self._hidden_size] + ) last_cell = fluid.layers.transpose(x=last_cell, perm=[1, 0, 2]) return real_res, last_hidden, last_cell class PtbModel(fluid.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_layers=2, - num_steps=20, - init_scale=0.1, - dropout=None): + def __init__( + self, + hidden_size, + vocab_size, + num_layers=2, + num_steps=20, + init_scale=0.1, + dropout=None, + ): super(PtbModel, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -151,11 +161,13 @@ def __init__(self, self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout - self.simple_lstm_rnn = SimpleLSTMRNN(hidden_size, - num_steps, - num_layers=num_layers, - init_scale=init_scale, - dropout=dropout) + self.simple_lstm_rnn = SimpleLSTMRNN( + hidden_size, + num_steps, + num_layers=num_layers, + init_scale=init_scale, + dropout=dropout, + ) self.embedding = Embedding( size=[vocab_size, hidden_size], dtype='float32', @@ -163,48 +175,62 @@ def __init__(self, param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label, init_hidden, init_cell): init_h = fluid.layers.reshape( - init_hidden, shape=[self.num_layers, -1, self.hidden_size]) + init_hidden, shape=[self.num_layers, -1, self.hidden_size] + ) init_c = fluid.layers.reshape( - init_cell, shape=[self.num_layers, -1, self.hidden_size]) + init_cell, shape=[self.num_layers, -1, self.hidden_size] + ) x_emb = self.embedding(input) x_emb = fluid.layers.reshape( - x_emb, shape=[-1, self.num_steps, self.hidden_size]) + x_emb, shape=[-1, self.num_steps, self.hidden_size] + ) if self.dropout is not None and self.dropout > 0.0: x_emb = fluid.layers.dropout( x_emb, dropout_prob=self.drop_out, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn_out, last_hidden, last_cell = self.simple_lstm_rnn( - x_emb, init_h, init_c) + x_emb, init_h, init_c + ) rnn_out = fluid.layers.reshape( - rnn_out, shape=[-1, self.num_steps, self.hidden_size]) + rnn_out, shape=[-1, self.num_steps, self.hidden_size] + ) projection = fluid.layers.matmul(rnn_out, self.softmax_weight) projection = fluid.layers.elementwise_add(projection, self.softmax_bias) - projection = fluid.layers.reshape(projection, - shape=[-1, self.vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + projection = fluid.layers.reshape( + projection, shape=[-1, self.vocab_size] + ) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -213,7 +239,6 @@ def forward(self, input, label, init_hidden, init_cell): class TestDygraphPtbRnn(unittest.TestCase): - def func_setUp(self): seed = 90 hidden_size = 10 @@ -228,11 +253,13 @@ def func_setUp(self): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -242,11 +269,17 @@ def func_setUp(self): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=fluid.layers.piecewise_decay( - boundaries=bd, values=lr_arr), - parameter_list=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr_arr + ), + parameter_list=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -258,15 +291,18 @@ def func_setUp(self): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -312,11 +348,13 @@ def func_testLoadAndSetVarBase(self): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -326,11 +364,17 @@ def func_testLoadAndSetVarBase(self): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=fluid.layers.piecewise_decay( - boundaries=bd, values=lr_arr), - parameter_list=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr_arr + ), + parameter_list=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -342,15 +386,18 @@ def func_testLoadAndSetVarBase(self): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -382,8 +429,9 @@ def func_testLoadAndSetVarBase(self): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if isinstance(v, (core.VarBase, core.eager.Tensor)): - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name]) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + ) else: self.assertEqual(v, self.base_opti[k]) @@ -420,11 +468,13 @@ def func_testSetVariable(self): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -434,11 +484,17 @@ def func_testSetVariable(self): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=fluid.layers.piecewise_decay( - boundaries=bd, values=lr_arr), - parameter_list=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr_arr + ), + parameter_list=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -450,15 +506,18 @@ def func_testSetVariable(self): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -487,8 +546,9 @@ def func_testSetVariable(self): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if isinstance(v, (core.VarBase, core.eager.Tensor)): - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name]) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + ) else: self.assertEqual(v, self.base_opti[k]) @@ -525,11 +585,13 @@ def func_testSetNumpy(self): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -539,11 +601,17 @@ def func_testSetNumpy(self): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=fluid.layers.piecewise_decay( - boundaries=bd, values=lr_arr), - parameter_list=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr_arr + ), + parameter_list=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -555,15 +623,18 @@ def func_testSetNumpy(self): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -596,8 +667,9 @@ def func_testSetNumpy(self): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if isinstance(v, (core.VarBase, core.eager.Tensor)): - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name]) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + ) else: self.assertEqual(v, self.base_opti[k]) @@ -634,18 +706,25 @@ def func_testSetVariableBeforeTrain(self): with fluid.dygraph.guard(): # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) - - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=0.0, - beta1=0.8, - beta2=0.6, - parameter_list=ptb_model.parameters()) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) + + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=0.0, + beta1=0.8, + beta2=0.6, + parameter_list=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -660,15 +739,18 @@ def func_testSetVariableBeforeTrain(self): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) dy_loss.backward() adam.minimize(dy_loss) @@ -677,15 +759,18 @@ def func_testSetVariableBeforeTrain(self): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if k == "global_step": - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name] + 1) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + 1 + ) if k.find("beta1_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta1) + v.numpy(), self.base_opti[v.name] * adam._beta1 + ) if k.find("beta2_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta2) + v.numpy(), self.base_opti[v.name] * adam._beta2 + ) state_dict = ptb_model.state_dict() @@ -709,11 +794,13 @@ def func_testLoadAndSetVarBaseBeforeTrain(self): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [0.0] @@ -724,12 +811,17 @@ def func_testLoadAndSetVarBaseBeforeTrain(self): new_lr = 0.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=0.0, - beta1=0.8, - beta2=0.6, - parameter_list=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=0.0, + beta1=0.8, + beta2=0.6, + parameter_list=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -745,15 +837,18 @@ def func_testLoadAndSetVarBaseBeforeTrain(self): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) dy_loss.backward() adam.minimize(dy_loss) @@ -762,15 +857,18 @@ def func_testLoadAndSetVarBaseBeforeTrain(self): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if k == "global_step": - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name] + 1) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + 1 + ) if k.find("beta1_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta1) + v.numpy(), self.base_opti[v.name] * adam._beta1 + ) if k.find("beta2_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta2) + v.numpy(), self.base_opti[v.name] * adam._beta2 + ) # check parameter @@ -797,11 +895,13 @@ def func_testSetNumpyBeforeTrain(self): paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [0.0] @@ -812,13 +912,19 @@ def func_testSetNumpyBeforeTrain(self): new_lr = 0.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=fluid.layers.piecewise_decay( - boundaries=bd, values=lr_arr), - beta1=0.8, - beta2=0.6, - parameter_list=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr_arr + ), + beta1=0.8, + beta2=0.6, + parameter_list=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -844,15 +950,18 @@ def func_testSetNumpyBeforeTrain(self): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) dy_loss.backward() adam.minimize(dy_loss) @@ -861,15 +970,18 @@ def func_testSetNumpyBeforeTrain(self): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if k == "global_step": - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name] + 1) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + 1 + ) if k.find("beta1_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta1) + v.numpy(), self.base_opti[v.name] * adam._beta1 + ) if k.find("beta2_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta2) + v.numpy(), self.base_opti[v.name] * adam._beta2 + ) # check parameter @@ -888,15 +1000,18 @@ def func_testOnlyLoadParams(self): fluid.save_dygraph(state_dict, os.path.join('saved_dy', 'emb_dy')) para_state_dict, opti_state_dict = fluid.load_dygraph( - os.path.join('saved_dy', 'emb_dy')) + os.path.join('saved_dy', 'emb_dy') + ) self.assertTrue(opti_state_dict == None) para_state_dict, opti_state_dict = fluid.load_dygraph( - os.path.join('saved_dy', 'emb_dy.pdparams')) + os.path.join('saved_dy', 'emb_dy.pdparams') + ) para_state_dict, opti_state_dict = fluid.load_dygraph( - os.path.join('saved_dy', 'emb_dy.pdopt')) + os.path.join('saved_dy', 'emb_dy.pdopt') + ) def func_test_load_compatible_with_keep_name_table(self): with fluid.dygraph.guard(): @@ -905,7 +1020,8 @@ def func_test_load_compatible_with_keep_name_table(self): fluid.save_dygraph(state_dict, os.path.join('saved_dy', 'emb_dy')) para_state_dict, opti_state_dict = fluid.load_dygraph( - os.path.join('saved_dy', 'emb_dy'), keep_name_table=True) + os.path.join('saved_dy', 'emb_dy'), keep_name_table=True + ) self.assertTrue(para_state_dict != None) self.assertTrue(opti_state_dict == None) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py b/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py index 0e051381adf71..795870e3d0187 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_save_load_v2.py @@ -28,13 +28,9 @@ class SimpleLSTMRNN(fluid.Layer): - - def __init__(self, - hidden_size, - num_steps, - num_layers=2, - init_scale=0.1, - dropout=None): + def __init__( + self, hidden_size, num_steps, num_layers=2, init_scale=0.1, dropout=None + ): super(SimpleLSTMRNN, self).__init__() self._hidden_size = hidden_size self._num_layers = num_layers @@ -53,19 +49,26 @@ def __init__(self, weight_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 2, self._hidden_size * 4], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)) + low=-self._init_scale, high=self._init_scale + ), + ) self.weight_1_arr.append(self.add_parameter('w_%d' % i, weight_1)) bias_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 4], dtype="float32", - default_initializer=fluid.initializer.Constant(0.0)) + default_initializer=fluid.initializer.Constant(0.0), + ) self.bias_arr.append(self.add_parameter('b_%d' % i, bias_1)) def forward(self, input_embedding, init_hidden=None, init_cell=None): @@ -73,29 +76,29 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): self.hidden_array = [] for i in range(self._num_layers): - pre_hidden = fluid.layers.slice(init_hidden, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_cell = fluid.layers.slice(init_cell, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_hidden = fluid.layers.reshape(pre_hidden, - shape=[-1, self._hidden_size]) - pre_cell = fluid.layers.reshape(pre_cell, - shape=[-1, self._hidden_size]) + pre_hidden = fluid.layers.slice( + init_hidden, axes=[0], starts=[i], ends=[i + 1] + ) + pre_cell = fluid.layers.slice( + init_cell, axes=[0], starts=[i], ends=[i + 1] + ) + pre_hidden = fluid.layers.reshape( + pre_hidden, shape=[-1, self._hidden_size] + ) + pre_cell = fluid.layers.reshape( + pre_cell, shape=[-1, self._hidden_size] + ) self.hidden_array.append(pre_hidden) self.cell_array.append(pre_cell) res = [] for index in range(self._num_steps): - self._input = fluid.layers.slice(input_embedding, - axes=[1], - starts=[index], - ends=[index + 1]) - self._input = fluid.layers.reshape(self._input, - shape=[-1, self._hidden_size]) + self._input = fluid.layers.slice( + input_embedding, axes=[1], starts=[index], ends=[index + 1] + ) + self._input = fluid.layers.reshape( + self._input, shape=[-1, self._hidden_size] + ) for k in range(self._num_layers): pre_hidden = self.hidden_array[k] pre_cell = self.cell_array[k] @@ -106,11 +109,12 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): gate_input = fluid.layers.matmul(x=nn, y=weight_1) gate_input = fluid.layers.elementwise_add(gate_input, bias) - i, j, f, o = fluid.layers.split(gate_input, - num_or_sections=4, - dim=-1) + i, j, f, o = fluid.layers.split( + gate_input, num_or_sections=4, dim=-1 + ) c = pre_cell * fluid.layers.sigmoid(f) + fluid.layers.sigmoid( - i) * fluid.layers.tanh(j) + i + ) * fluid.layers.tanh(j) m = fluid.layers.tanh(c) * fluid.layers.sigmoid(o) self.hidden_array[k] = m self.cell_array[k] = c @@ -120,32 +124,38 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): self._input = fluid.layers.dropout( self._input, dropout_prob=self._dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) res.append( - fluid.layers.reshape(self._input, - shape=[1, -1, self._hidden_size])) + fluid.layers.reshape( + self._input, shape=[1, -1, self._hidden_size] + ) + ) real_res = fluid.layers.concat(res, 0) real_res = fluid.layers.transpose(x=real_res, perm=[1, 0, 2]) last_hidden = fluid.layers.concat(self.hidden_array, 1) last_hidden = fluid.layers.reshape( - last_hidden, shape=[-1, self._num_layers, self._hidden_size]) + last_hidden, shape=[-1, self._num_layers, self._hidden_size] + ) last_hidden = fluid.layers.transpose(x=last_hidden, perm=[1, 0, 2]) last_cell = fluid.layers.concat(self.cell_array, 1) last_cell = fluid.layers.reshape( - last_cell, shape=[-1, self._num_layers, self._hidden_size]) + last_cell, shape=[-1, self._num_layers, self._hidden_size] + ) last_cell = fluid.layers.transpose(x=last_cell, perm=[1, 0, 2]) return real_res, last_hidden, last_cell class PtbModel(fluid.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_layers=2, - num_steps=20, - init_scale=0.1, - dropout=None): + def __init__( + self, + hidden_size, + vocab_size, + num_layers=2, + num_steps=20, + init_scale=0.1, + dropout=None, + ): super(PtbModel, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -153,11 +163,13 @@ def __init__(self, self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout - self.simple_lstm_rnn = SimpleLSTMRNN(hidden_size, - num_steps, - num_layers=num_layers, - init_scale=init_scale, - dropout=dropout) + self.simple_lstm_rnn = SimpleLSTMRNN( + hidden_size, + num_steps, + num_layers=num_layers, + init_scale=init_scale, + dropout=dropout, + ) self.embedding = Embedding( size=[vocab_size, hidden_size], dtype='float32', @@ -165,48 +177,62 @@ def __init__(self, param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label, init_hidden, init_cell): init_h = fluid.layers.reshape( - init_hidden, shape=[self.num_layers, -1, self.hidden_size]) + init_hidden, shape=[self.num_layers, -1, self.hidden_size] + ) init_c = fluid.layers.reshape( - init_cell, shape=[self.num_layers, -1, self.hidden_size]) + init_cell, shape=[self.num_layers, -1, self.hidden_size] + ) x_emb = self.embedding(input) x_emb = fluid.layers.reshape( - x_emb, shape=[-1, self.num_steps, self.hidden_size]) + x_emb, shape=[-1, self.num_steps, self.hidden_size] + ) if self.dropout is not None and self.dropout > 0.0: x_emb = fluid.layers.dropout( x_emb, dropout_prob=self.drop_out, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn_out, last_hidden, last_cell = self.simple_lstm_rnn( - x_emb, init_h, init_c) + x_emb, init_h, init_c + ) rnn_out = fluid.layers.reshape( - rnn_out, shape=[-1, self.num_steps, self.hidden_size]) + rnn_out, shape=[-1, self.num_steps, self.hidden_size] + ) projection = fluid.layers.matmul(rnn_out, self.softmax_weight) projection = fluid.layers.elementwise_add(projection, self.softmax_bias) - projection = fluid.layers.reshape(projection, - shape=[-1, self.vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + projection = fluid.layers.reshape( + projection, shape=[-1, self.vocab_size] + ) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -215,7 +241,6 @@ def forward(self, input, label, init_hidden, init_cell): class TestDygraphPtbRnn(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -236,11 +261,13 @@ def func_setUp(self): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -250,12 +277,17 @@ def func_setUp(self): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr_arr) - adam = Adam(learning_rate=scheduler, - parameters=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr_arr + ) + adam = Adam( + learning_rate=scheduler, parameters=ptb_model.parameters() + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -267,15 +299,18 @@ def func_setUp(self): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -298,8 +333,10 @@ def func_setUp(self): else: self.base_opti[k] = v - paddle.save(self.opti_dict, - os.path.join(self.temp_dir.name, "test_dy_v2.pdopt")) + paddle.save( + self.opti_dict, + os.path.join(self.temp_dir.name, "test_dy_v2.pdopt"), + ) self.state_dict = ptb_model.state_dict() @@ -308,8 +345,10 @@ def func_setUp(self): np_t = v.numpy() self.model_base[k] = np_t - paddle.save(self.state_dict, - os.path.join(self.temp_dir.name, "test_dy_v2.pdparams")) + paddle.save( + self.state_dict, + os.path.join(self.temp_dir.name, "test_dy_v2.pdparams"), + ) def func_testLoadAndSetVarBase(self): seed = 90 @@ -325,11 +364,13 @@ def func_testLoadAndSetVarBase(self): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -339,12 +380,17 @@ def func_testLoadAndSetVarBase(self): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr_arr) - adam = Adam(learning_rate=scheduler, - parameters=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr_arr + ) + adam = Adam( + learning_rate=scheduler, parameters=ptb_model.parameters() + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -356,15 +402,18 @@ def func_testLoadAndSetVarBase(self): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -388,16 +437,19 @@ def func_testLoadAndSetVarBase(self): self.assertTrue(np.sum(np.abs(v.numpy())) == 0) para_state_dict = paddle.load( - os.path.join(self.temp_dir.name, "test_dy_v2.pdparams")) + os.path.join(self.temp_dir.name, "test_dy_v2.pdparams") + ) opti_state_dict = paddle.load( - os.path.join(self.temp_dir.name, "test_dy_v2.pdopt")) + os.path.join(self.temp_dir.name, "test_dy_v2.pdopt") + ) adam.set_state_dict(opti_state_dict) opti_dict = adam.state_dict() for k, v in opti_dict.items(): if isinstance(v, (core.VarBase, core.eager.Tensor)): - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name]) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + ) else: self.assertEqual(v, self.base_opti[k]) @@ -434,11 +486,13 @@ def func_testSetVariable(self): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -448,12 +502,17 @@ def func_testSetVariable(self): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr_arr) - adam = Adam(learning_rate=scheduler, - parameters=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr_arr + ) + adam = Adam( + learning_rate=scheduler, parameters=ptb_model.parameters() + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -465,15 +524,18 @@ def func_testSetVariable(self): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -503,8 +565,9 @@ def func_testSetVariable(self): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if isinstance(v, (core.VarBase, core.eager.Tensor)): - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name]) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + ) else: self.assertEqual(v, self.base_opti[k]) @@ -541,11 +604,13 @@ def func_testSetNumpy(self): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [1.0] @@ -555,12 +620,17 @@ def func_testSetNumpy(self): new_lr = 1.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr_arr) - adam = Adam(learning_rate=scheduler, - parameters=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr_arr + ) + adam = Adam( + learning_rate=scheduler, parameters=ptb_model.parameters() + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -572,15 +642,18 @@ def func_testSetNumpy(self): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) if i == 0: for param in ptb_model.parameters(): dy_param_init[param.name] = param.numpy() @@ -614,8 +687,9 @@ def func_testSetNumpy(self): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if isinstance(v, (core.VarBase, core.eager.Tensor)): - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name]) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + ) else: self.assertEqual(v, self.base_opti[k]) @@ -654,18 +728,25 @@ def func_testSetVariableBeforeTrain(self): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) - - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=0.0, - beta1=0.8, - beta2=0.6, - parameters=ptb_model.parameters()) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) + + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=0.0, + beta1=0.8, + beta2=0.6, + parameters=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -680,15 +761,18 @@ def func_testSetVariableBeforeTrain(self): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) dy_loss.backward() adam.minimize(dy_loss) @@ -697,15 +781,18 @@ def func_testSetVariableBeforeTrain(self): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if k == "global_step": - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name] + 1) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + 1 + ) if k.find("beta1_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta1) + v.numpy(), self.base_opti[v.name] * adam._beta1 + ) if k.find("beta2_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta2) + v.numpy(), self.base_opti[v.name] * adam._beta2 + ) state_dict = ptb_model.state_dict() @@ -729,11 +816,13 @@ def func_testLoadAndSetVarBaseBeforeTrain(self): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [0.0] @@ -744,12 +833,17 @@ def func_testLoadAndSetVarBaseBeforeTrain(self): new_lr = 0.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - adam = Adam(learning_rate=0.0, - beta1=0.8, - beta2=0.6, - parameters=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + adam = Adam( + learning_rate=0.0, + beta1=0.8, + beta2=0.6, + parameters=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -757,7 +851,8 @@ def func_testLoadAndSetVarBaseBeforeTrain(self): last_cell = None state_dict, opti_dict = fluid.load_dygraph( - os.path.join(self.temp_dir.name, "test_dy_v2")) + os.path.join(self.temp_dir.name, "test_dy_v2") + ) adam.set_state_dict(opti_dict) ptb_model.set_dict(state_dict) @@ -766,15 +861,18 @@ def func_testLoadAndSetVarBaseBeforeTrain(self): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) dy_loss.backward() adam.minimize(dy_loss) @@ -783,15 +881,18 @@ def func_testLoadAndSetVarBaseBeforeTrain(self): opti_dict = adam.state_dict() for k, v in opti_dict.items(): if k == "global_step": - np.testing.assert_array_equal(v.numpy(), - self.base_opti[v.name] + 1) + np.testing.assert_array_equal( + v.numpy(), self.base_opti[v.name] + 1 + ) if k.find("beta1_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta1) + v.numpy(), self.base_opti[v.name] * adam._beta1 + ) if k.find("beta2_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta2) + v.numpy(), self.base_opti[v.name] * adam._beta2 + ) # check parameter @@ -817,11 +918,13 @@ def func_testSetNumpyBeforeTrain(self): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) # TODO: marsyang1993 Change seed to - ptb_model = PtbModel(hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) bd = [] lr_arr = [0.0] @@ -832,14 +935,20 @@ def func_testSetNumpyBeforeTrain(self): new_lr = 0.0 lr_arr.append(new_lr) - place = fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) - scheduler = paddle.optimizer.lr.PiecewiseDecay(boundaries=bd, - values=lr_arr) - adam = Adam(learning_rate=scheduler, - beta1=0.8, - beta2=0.6, - parameters=ptb_model.parameters()) + place = ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) + scheduler = paddle.optimizer.lr.PiecewiseDecay( + boundaries=bd, values=lr_arr + ) + adam = Adam( + learning_rate=scheduler, + beta1=0.8, + beta2=0.6, + parameters=ptb_model.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None @@ -865,15 +974,18 @@ def func_testSetNumpyBeforeTrain(self): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) x = to_variable(x_data) y = to_variable(y_data) init_hidden = to_variable(init_hidden_data) init_cell = to_variable(init_cell_data) dy_loss, last_hidden, last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) dy_loss.backward() scheduler.step() @@ -884,14 +996,17 @@ def func_testSetNumpyBeforeTrain(self): for k, v in opti_dict.items(): if k == "LR_Scheduler": np.testing.assert_array_equal( - v['last_epoch'], self.base_opti[k]['last_epoch'] + 1) + v['last_epoch'], self.base_opti[k]['last_epoch'] + 1 + ) if k.find("beta1_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta1) + v.numpy(), self.base_opti[v.name] * adam._beta1 + ) if k.find("beta2_pow_acc_0") > 0: np.testing.assert_array_equal( - v.numpy(), self.base_opti[v.name] * adam._beta2) + v.numpy(), self.base_opti[v.name] * adam._beta2 + ) # check parameter @@ -909,10 +1024,12 @@ def func_testOnlyLoadParams(self): state_dict = emb.state_dict() paddle.save( state_dict, - os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'), + ) para_state_dict = paddle.load( - os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams') + ) def func_test_no_state_in_input_dict(self): with fluid.dygraph.guard(): @@ -920,10 +1037,12 @@ def func_test_no_state_in_input_dict(self): state_dict = emb.state_dict() paddle.save( state_dict, - os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'), + ) para_state_dict = paddle.load( - os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams') + ) para_state_dict.pop('weight') emb.set_state_dict(para_state_dict) @@ -934,14 +1053,16 @@ def func_test_state_shape_mismatch(self): state_dict = emb.state_dict() paddle.save( state_dict, - os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams')) + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'), + ) - para_state_dict = paddle.load(os.path.join(self.temp_dir.name, - 'saved_dy', - 'emb_dy.pdparams'), - return_numpy=True) + para_state_dict = paddle.load( + os.path.join(self.temp_dir.name, 'saved_dy', 'emb_dy.pdparams'), + return_numpy=True, + ) para_state_dict['weight'] = np.expand_dims( - para_state_dict['weight'], axis=-1) + para_state_dict['weight'], axis=-1 + ) emb.set_state_dict(para_state_dict) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_se_resnext.py b/python/paddle/fluid/tests/unittests/test_imperative_se_resnext.py index c04f3adcadd53..1f995817c0f77 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_se_resnext.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_se_resnext.py @@ -35,7 +35,7 @@ "name": "piecewise_decay", "batch_size": batch_size, "epochs": [30, 60, 90], - "steps": [0.1, 0.01, 0.001, 0.0001] + "steps": [0.1, 0.01, 0.001, 0.0001], }, "batch_size": batch_size, "lr": 0.1, @@ -51,15 +51,16 @@ def optimizer_setting(params, parameter_list=None): else: total_images = params["total_images"] # TODO(Yancey1989): using lr decay if it is ready. - #batch_size = ls["batch_size"] - #step = int(total_images / batch_size + 1) + # batch_size = ls["batch_size"] + # step = int(total_images / batch_size + 1) - #bd = [step * e for e in ls["epochs"]] - #base_lr = params["lr"] - #lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] + # bd = [step * e for e in ls["epochs"]] + # base_lr = params["lr"] + # lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] if fluid._non_static_mode(): - optimizer = fluid.optimizer.SGD(learning_rate=0.01, - parameter_list=parameter_list) + optimizer = fluid.optimizer.SGD( + learning_rate=0.01, parameter_list=parameter_list + ) else: optimizer = fluid.optimizer.SGD(learning_rate=0.01) @@ -67,24 +68,27 @@ def optimizer_setting(params, parameter_list=None): class ConvBNLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - act=None): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + act=None, + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - act=None, - bias_attr=None) + self._conv = Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + act=None, + bias_attr=None, + ) self._batch_norm = BatchNorm(num_filters, act=act) @@ -96,7 +100,6 @@ def forward(self, inputs): class SqueezeExcitation(fluid.dygraph.Layer): - def __init__(self, num_channels, reduction_ratio): super(SqueezeExcitation, self).__init__() @@ -105,15 +108,19 @@ def __init__(self, num_channels, reduction_ratio): self._squeeze = Linear( num_channels, num_channels // reduction_ratio, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.05)), - act='relu') + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.05) + ), + act='relu', + ) self._excitation = Linear( num_channels // reduction_ratio, num_channels, - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.05)), - act='sigmoid') + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.05) + ), + act='sigmoid', + ) def forward(self, input): y = self._pool(input) @@ -125,37 +132,45 @@ def forward(self, input): class BottleneckBlock(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters, - stride, - cardinality, - reduction_ratio, - shortcut=True): + def __init__( + self, + num_channels, + num_filters, + stride, + cardinality, + reduction_ratio, + shortcut=True, + ): super(BottleneckBlock, self).__init__() - self.conv0 = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=1) - self.conv1 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=stride, - groups=cardinality) - self.conv2 = ConvBNLayer(num_channels=num_filters, - num_filters=num_filters * 4, - filter_size=1, - act='relu') - - self.scale = SqueezeExcitation(num_channels=num_filters * 4, - reduction_ratio=reduction_ratio) + self.conv0 = ConvBNLayer( + num_channels=num_channels, num_filters=num_filters, filter_size=1 + ) + self.conv1 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=stride, + groups=cardinality, + ) + self.conv2 = ConvBNLayer( + num_channels=num_filters, + num_filters=num_filters * 4, + filter_size=1, + act='relu', + ) + + self.scale = SqueezeExcitation( + num_channels=num_filters * 4, reduction_ratio=reduction_ratio + ) if not shortcut: - self.short = ConvBNLayer(num_channels=num_channels, - num_filters=num_filters * 4, - filter_size=1, - stride=stride) + self.short = ConvBNLayer( + num_channels=num_channels, + num_filters=num_filters * 4, + filter_size=1, + stride=stride, + ) self.shortcut = shortcut @@ -180,67 +195,76 @@ def forward(self, inputs): class SeResNeXt(fluid.dygraph.Layer): - def __init__(self, layers=50, class_dim=102): super(SeResNeXt, self).__init__() self.layers = layers supported_layers = [50, 101, 152] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format(supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) if layers == 50: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 6, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) elif layers == 101: cardinality = 32 reduction_ratio = 16 depth = [3, 4, 23, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=7, - stride=2, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=7, + stride=2, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) elif layers == 152: cardinality = 64 reduction_ratio = 16 depth = [3, 8, 36, 3] num_filters = [128, 256, 512, 1024] - self.conv0 = ConvBNLayer(num_channels=3, - num_filters=64, - filter_size=3, - stride=2, - act='relu') - self.conv1 = ConvBNLayer(num_channels=64, - num_filters=64, - filter_size=3, - stride=2, - act='relu') - self.conv2 = ConvBNLayer(num_channels=64, - num_filters=128, - filter_size=3, - stride=1, - act='relu') - self.pool = Pool2D(pool_size=3, - pool_stride=2, - pool_padding=1, - pool_type='max') + self.conv0 = ConvBNLayer( + num_channels=3, + num_filters=64, + filter_size=3, + stride=2, + act='relu', + ) + self.conv1 = ConvBNLayer( + num_channels=64, + num_filters=64, + filter_size=3, + stride=2, + act='relu', + ) + self.conv2 = ConvBNLayer( + num_channels=64, + num_filters=128, + filter_size=3, + stride=1, + act='relu', + ) + self.pool = Pool2D( + pool_size=3, pool_stride=2, pool_padding=1, pool_type='max' + ) self.bottleneck_block_list = [] num_channels = 64 @@ -251,20 +275,24 @@ def __init__(self, layers=50, class_dim=102): for i in range(depth[block]): bottleneck_block = self.add_sublayer( 'bb_%d_%d' % (block, i), - BottleneckBlock(num_channels=num_channels, - num_filters=num_filters[block], - stride=2 if i == 0 and block != 0 else 1, - cardinality=cardinality, - reduction_ratio=reduction_ratio, - shortcut=shortcut)) + BottleneckBlock( + num_channels=num_channels, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + cardinality=cardinality, + reduction_ratio=reduction_ratio, + shortcut=shortcut, + ), + ) num_channels = bottleneck_block._num_channels_out self.bottleneck_block_list.append(bottleneck_block) shortcut = True - self.pool2d_avg = Pool2D(pool_size=7, - pool_type='avg', - global_pooling=True) + self.pool2d_avg = Pool2D( + pool_size=7, pool_type='avg', global_pooling=True + ) import math + stdv = 1.0 / math.sqrt(2048 * 1.0) self.pool2d_avg_output = num_filters[-1] * 4 * 1 * 1 @@ -274,7 +302,9 @@ def __init__(self, layers=50, class_dim=102): class_dim, act='softmax', param_attr=fluid.param_attr.ParamAttr( - initializer=fluid.initializer.Uniform(-stdv, stdv))) + initializer=fluid.initializer.Uniform(-stdv, stdv) + ), + ) def forward(self, inputs): if self.layers == 50 or self.layers == 101: @@ -295,9 +325,7 @@ def forward(self, inputs): class TestImperativeResneXt(unittest.TestCase): - def reader_decorator(self, reader): - def _reader_imple(): for item in reader(): doc = np.array(item[0]).reshape(3, 224, 224) @@ -319,16 +347,21 @@ def run_dygraph(): se_resnext = SeResNeXt() optimizer = optimizer_setting( - train_parameters, parameter_list=se_resnext.parameters()) + train_parameters, parameter_list=se_resnext.parameters() + ) np.random.seed(seed) batch_py_reader = fluid.io.PyReader(capacity=1) batch_py_reader.decorate_sample_list_generator( - paddle.batch(self.reader_decorator( - paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True), - places=fluid.CPUPlace()) + paddle.batch( + self.reader_decorator( + paddle.dataset.flowers.train(use_xmap=False) + ), + batch_size=batch_size, + drop_last=True, + ), + places=fluid.CPUPlace(), + ) dy_param_init_value = {} for param in se_resnext.parameters(): @@ -346,8 +379,9 @@ def run_dygraph(): out = se_resnext(img) softmax_out = fluid.layers.softmax(out, use_cudnn=False) - loss = fluid.layers.cross_entropy(input=softmax_out, - label=label) + loss = fluid.layers.cross_entropy( + input=softmax_out, label=label + ) avg_loss = paddle.mean(x=loss) dy_out = avg_loss.numpy() @@ -362,9 +396,11 @@ def run_dygraph(): for param in se_resnext.parameters(): if param.trainable: np_array = np.array( - param._grad_ivar().value().get_tensor()) - dy_grad_value[param.name + - core.grad_var_suffix()] = np_array + param._grad_ivar().value().get_tensor() + ) + dy_grad_value[ + param.name + core.grad_var_suffix() + ] = np_array optimizer.minimize(avg_loss) se_resnext.clear_gradients() @@ -373,23 +409,39 @@ def run_dygraph(): for param in se_resnext.parameters(): dy_param_value[param.name] = param.numpy() - return dy_out, dy_param_init_value, dy_param_value, dy_grad_value + return ( + dy_out, + dy_param_init_value, + dy_param_value, + dy_grad_value, + ) with fluid.dygraph.guard(): - dy_out, dy_param_init_value, dy_param_value, dy_grad_value = run_dygraph( - ) + ( + dy_out, + dy_param_init_value, + dy_param_value, + dy_grad_value, + ) = run_dygraph() with fluid.dygraph.guard(): with _test_eager_guard(): - eager_out, eager_param_init_value, eager_param_value, eager_grad_value = run_dygraph( - ) + ( + eager_out, + eager_param_init_value, + eager_param_value, + eager_grad_value, + ) = run_dygraph() with new_program_scope(): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) se_resnext = SeResNeXt() optimizer = optimizer_setting(train_parameters) @@ -398,11 +450,12 @@ def run_dygraph(): train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), batch_size=batch_size, - drop_last=True) + drop_last=True, + ) - img = fluid.layers.data(name='pixel', - shape=[3, 224, 224], - dtype='float32') + img = fluid.layers.data( + name='pixel', shape=[3, 224, 224], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') out = se_resnext(img) softmax_out = fluid.layers.softmax(out, use_cudnn=False) @@ -418,11 +471,14 @@ def run_dygraph(): static_param_name_list.append(param.name) for param in se_resnext.parameters(): if param.trainable: - static_grad_name_list.append(param.name + - core.grad_var_suffix()) + static_grad_name_list.append( + param.name + core.grad_var_suffix() + ) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] @@ -431,47 +487,54 @@ def run_dygraph(): if batch_id >= batch_num and batch_num != -1: break - static_x_data = np.array([ - x[0].reshape(3, 224, 224) for x in data - ]).astype('float32') - y_data = np.array([x[1] - for x in data]).astype('int64').reshape( - [batch_size, 1]) + static_x_data = np.array( + [x[0].reshape(3, 224, 224) for x in data] + ).astype('float32') + y_data = ( + np.array([x[1] for x in data]) + .astype('int64') + .reshape([batch_size, 1]) + ) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) fetch_list.extend(static_grad_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "pixel": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) static_param_value = {} static_grad_value = {} static_out = out[0] param_start_pos = 1 - grad_start_pos = len( - static_param_name_list) + param_start_pos + grad_start_pos = ( + len(static_param_name_list) + param_start_pos + ) + for i in range( + param_start_pos, + len(static_param_name_list) + param_start_pos, + ): + static_param_value[ + static_param_name_list[i - param_start_pos] + ] = out[i] for i in range( - param_start_pos, - len(static_param_name_list) + param_start_pos): - static_param_value[static_param_name_list[ - i - param_start_pos]] = out[i] - for i in range(grad_start_pos, - len(static_grad_name_list) + grad_start_pos): - static_grad_value[static_grad_name_list[ - i - grad_start_pos]] = out[i] + grad_start_pos, + len(static_grad_name_list) + grad_start_pos, + ): + static_grad_value[ + static_grad_name_list[i - grad_start_pos] + ] = out[i] np.testing.assert_allclose(static_out, dy_out, rtol=1e-05) self.assertEqual(len(dy_param_init_value), len(static_param_init_value)) for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - dy_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, dy_param_init_value[key], rtol=1e-05 + ) self.assertTrue(np.isfinite(value.all())) self.assertFalse(np.isnan(value.any())) @@ -491,13 +554,14 @@ def run_dygraph(): # check eager np.testing.assert_allclose(static_out, eager_out, rtol=1e-05) - self.assertEqual(len(eager_param_init_value), - len(static_param_init_value)) + self.assertEqual( + len(eager_param_init_value), len(static_param_init_value) + ) for key, value in static_param_init_value.items(): - np.testing.assert_allclose(value, - eager_param_init_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, eager_param_init_value[key], rtol=1e-05 + ) self.assertEqual(len(eager_grad_value), len(static_grad_value)) @@ -506,9 +570,9 @@ def run_dygraph(): self.assertEqual(len(eager_param_value), len(static_param_value)) for key, value in static_param_value.items(): - np.testing.assert_allclose(value, - eager_param_value[key], - rtol=1e-05) + np.testing.assert_allclose( + value, eager_param_value[key], rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_selected_rows.py b/python/paddle/fluid/tests/unittests/test_imperative_selected_rows.py index 602d2e07ce55e..93a04a36115c6 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_selected_rows.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_selected_rows.py @@ -23,13 +23,14 @@ class SimpleNet(paddle.nn.Layer): - def __init__(self, vocab_size, hidden_size, dtype): super(SimpleNet, self).__init__() - self.emb = fluid.dygraph.Embedding(size=[vocab_size, hidden_size], - dtype=dtype, - param_attr='emb.w', - is_sparse=True) + self.emb = fluid.dygraph.Embedding( + size=[vocab_size, hidden_size], + dtype=dtype, + param_attr='emb.w', + is_sparse=True, + ) def forward(self, input): input_emb = self.emb(input) @@ -37,7 +38,6 @@ def forward(self, input): class TestSimpleNet(unittest.TestCase): - def func_selectedrows_gradient1(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -48,16 +48,18 @@ def func_selectedrows_gradient1(self): for sort_sum_gradient in [True, False]: paddle.disable_static(place) fluid.set_flags( - {'FLAGS_sort_sum_gradient': sort_sum_gradient}) + {'FLAGS_sort_sum_gradient': sort_sum_gradient} + ) # grad_clip = fluid.clip.GradientClipByGlobalNorm(5.0) input_word = np.array([[1, 2], [2, 1]]).astype('int64') input = paddle.to_tensor(input_word) simplenet = SimpleNet(20, 32, dtype) - adam = SGDOptimizer(learning_rate=0.001, - parameter_list=simplenet.parameters() - ) # grad_clip=grad_clip + adam = SGDOptimizer( + learning_rate=0.001, + parameter_list=simplenet.parameters(), + ) # grad_clip=grad_clip input_emb, emb = simplenet(input) self.assertTrue(emb.weight.gradient() is None) @@ -90,16 +92,19 @@ def func_selectedrows_gradient2(self): for sort_sum_gradient in [True, False]: with fluid.dygraph.guard(place): fluid.set_flags( - {'FLAGS_sort_sum_gradient': sort_sum_gradient}) + {'FLAGS_sort_sum_gradient': sort_sum_gradient} + ) grad_clip = fluid.clip.GradientClipByGlobalNorm(5.0) input_word = np.array([[1, 2], [2, 1]]).astype('int64') input = to_variable(input_word) simplenet = SimpleNet(20, 32, "float32") - adam = SGDOptimizer(learning_rate=0.001, - parameter_list=simplenet.parameters(), - grad_clip=grad_clip) + adam = SGDOptimizer( + learning_rate=0.001, + parameter_list=simplenet.parameters(), + grad_clip=grad_clip, + ) input_emb, emb = simplenet(input) self.assertTrue(emb.weight.gradient() is None) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_selected_rows_to_lod_tensor.py b/python/paddle/fluid/tests/unittests/test_imperative_selected_rows_to_lod_tensor.py index 35d863b774b02..023753322d1df 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_selected_rows_to_lod_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_selected_rows_to_lod_tensor.py @@ -26,14 +26,15 @@ class SimpleNet(fluid.Layer): - - def __init__(self, - hidden_size, - vocab_size, - num_steps=20, - init_scale=0.1, - is_sparse=False, - dtype='float32'): + def __init__( + self, + hidden_size, + vocab_size, + num_steps=20, + init_scale=0.1, + is_sparse=False, + dtype='float32', + ): super(SimpleNet, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -46,31 +47,40 @@ def __init__(self, param_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.hidden_size], dtype=dtype, default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size], dtype=dtype, default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label): x_emb = self.embedding(input) fc = fluid.layers.matmul(x_emb, self.softmax_weight) fc = fluid.layers.elementwise_add(fc, self.softmax_bias) projection = fluid.layers.matmul( - fc, fluid.layers.transpose(self.embedding.weight, perm=[1, 0])) - projection = fluid.layers.reshape(projection, - shape=[-1, self.vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + fc, fluid.layers.transpose(self.embedding.weight, perm=[1, 0]) + ) + projection = fluid.layers.reshape( + projection, shape=[-1, self.vocab_size] + ) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -79,7 +89,6 @@ def forward(self, input, label): class TestDygraphSimpleNet(unittest.TestCase): - def func_simple_net(self): for is_sparse in [True, False]: dtype_list = ["float32"] @@ -113,21 +122,26 @@ def simple_net_float(self, is_sparse, dtype): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - simple_net = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - init_scale=init_scale, - is_sparse=is_sparse, - dtype=dtype) - - sgd = SGDOptimizer(learning_rate=1e-3, - parameter_list=simple_net.parameters()) + simple_net = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + init_scale=init_scale, + is_sparse=is_sparse, + dtype=dtype, + ) + + sgd = SGDOptimizer( + learning_rate=1e-3, + parameter_list=simple_net.parameters(), + ) dy_param_updated = dict() dy_param_init = dict() dy_loss = None fluid.set_flags( - {'FLAGS_sort_sum_gradient': is_sort_sum_gradient}) + {'FLAGS_sort_sum_gradient': is_sort_sum_gradient} + ) for i in range(batch_num): x_data = np.arange(12).reshape(4, 3).astype('int64') @@ -154,17 +168,19 @@ def simple_net_float(self, is_sparse, dtype): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - simple_net = SimpleNet(hidden_size=hidden_size, - vocab_size=vocab_size, - num_steps=num_steps, - is_sparse=is_sparse, - dtype=dtype) + simple_net = SimpleNet( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_steps=num_steps, + is_sparse=is_sparse, + dtype=dtype, + ) exe = fluid.Executor(place) sgd = SGDOptimizer(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype=dtype) static_loss = simple_net(x, y) @@ -175,8 +191,10 @@ def simple_net_float(self, is_sparse, dtype): for param in simple_net.parameters(): static_param_name_list.append(param.name) - out = exe.run(framework.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + framework.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init[static_param_name_list[i]] = out[i] static_loss_value = None @@ -187,18 +205,18 @@ def simple_net_float(self, is_sparse, dtype): y_data = y_data.reshape((-1, 1)) fetch_list = [static_loss] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"x": x_data, "y": y_data}, + fetch_list=fetch_list, + ) static_loss_value = out[0] if i == batch_num - 1: for k in range(3, len(out)): - static_param_updated[static_param_name_list[ - k - 1]] = out[k] + static_param_updated[ + static_param_name_list[k - 1] + ] = out[k] np.testing.assert_array_equal(static_loss_value, dy_loss_value) for key, value in static_param_init.items(): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_signal_handler.py b/python/paddle/fluid/tests/unittests/test_imperative_signal_handler.py index 977cb9cc2576d..488500cb3802a 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_signal_handler.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_signal_handler.py @@ -38,9 +38,7 @@ def __handler__(signum, frame): class DygraphDataLoaderSingalHandler(unittest.TestCase): - def func_child_process_exit_with_error(self): - def __test_process__(): core._set_process_signal_handler() sys.exit(1) @@ -73,7 +71,6 @@ def test_child_process_exit_with_error(self): self.func_child_process_exit_with_error() def func_child_process_killed_by_sigsegv(self): - def __test_process__(): core._set_process_signal_handler() os.kill(os.getpid(), signal.SIGSEGV) @@ -106,7 +103,6 @@ def test_child_process_killed_by_sigsegv(self): self.func_child_process_killed_by_sigsegv() def func_child_process_killed_by_sigbus(self): - def __test_process__(): core._set_process_signal_handler() os.kill(os.getpid(), signal.SIGBUS) @@ -139,7 +135,6 @@ def test_child_process_killed_by_sigbus(self): self.func_child_process_killed_by_sigbus() def func_child_process_killed_by_sigterm(self): - def __test_process__(): core._set_process_signal_handler() time.sleep(10) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py b/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py index 4b394ba87fdd1..3a43709997ed5 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py @@ -24,7 +24,6 @@ class Config(object): - def __init__(self, place, sort_sum_gradient=True): self.place = place @@ -60,7 +59,6 @@ def __init__(self, place, sort_sum_gradient=True): def create_mnist_dataset(cfg): - def create_target_label(label): return label # return (label + 1) % cfg.c_dim # fake label target @@ -107,7 +105,6 @@ def __impl__(): class InstanceNorm(fluid.dygraph.Layer): - def __init__(self, num_channels, epsilon=1e-5): super(InstanceNorm, self).__init__() self.epsilon = epsilon @@ -117,36 +114,40 @@ def __init__(self, num_channels, epsilon=1e-5): def forward(self, input): if fluid._non_static_mode(): - out, _, _ = _legacy_C_ops.instance_norm(input, self.scale, - self.bias, 'epsilon', - self.epsilon) + out, _, _ = _legacy_C_ops.instance_norm( + input, self.scale, self.bias, 'epsilon', self.epsilon + ) return out else: return fluid.layers.instance_norm( input, epsilon=self.epsilon, param_attr=fluid.ParamAttr(self.scale.name), - bias_attr=fluid.ParamAttr(self.bias.name)) + bias_attr=fluid.ParamAttr(self.bias.name), + ) class Conv2DLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters=64, - filter_size=7, - stride=1, - padding=0, - norm=None, - use_bias=False, - relufactor=None): + def __init__( + self, + num_channels, + num_filters=64, + filter_size=7, + stride=1, + padding=0, + norm=None, + use_bias=False, + relufactor=None, + ): super(Conv2DLayer, self).__init__() - self._conv = fluid.dygraph.Conv2D(num_channels=num_channels, - num_filters=num_filters, - filter_size=filter_size, - stride=stride, - padding=padding, - bias_attr=None if use_bias else False) + self._conv = fluid.dygraph.Conv2D( + num_channels=num_channels, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=padding, + bias_attr=None if use_bias else False, + ) if norm is not None: self._norm = InstanceNorm(num_filters) @@ -168,16 +169,17 @@ def forward(self, input): class Deconv2DLayer(fluid.dygraph.Layer): - - def __init__(self, - num_channels, - num_filters=64, - filter_size=7, - stride=1, - padding=0, - norm=None, - use_bias=False, - relufactor=None): + def __init__( + self, + num_channels, + num_filters=64, + filter_size=7, + stride=1, + padding=0, + norm=None, + use_bias=False, + relufactor=None, + ): super(Deconv2DLayer, self).__init__() self._deconv = fluid.dygraph.Conv2DTranspose( @@ -186,7 +188,8 @@ def __init__(self, filter_size=filter_size, stride=stride, padding=padding, - bias_attr=None if use_bias else False) + bias_attr=None if use_bias else False, + ) if norm is not None: self._norm = InstanceNorm(num_filters) @@ -208,24 +211,27 @@ def forward(self, input): class ResidualBlock(fluid.dygraph.Layer): - def __init__(self, num_channels, num_filters): super(ResidualBlock, self).__init__() - self._conv0 = Conv2DLayer(num_channels=num_channels, - num_filters=num_filters, - filter_size=3, - stride=1, - padding=1, - norm=True, - relufactor=0) - - self._conv1 = Conv2DLayer(num_channels=num_filters, - num_filters=num_filters, - filter_size=3, - stride=1, - padding=1, - norm=True, - relufactor=None) + self._conv0 = Conv2DLayer( + num_channels=num_channels, + num_filters=num_filters, + filter_size=3, + stride=1, + padding=1, + norm=True, + relufactor=0, + ) + + self._conv1 = Conv2DLayer( + num_channels=num_filters, + num_filters=num_filters, + filter_size=3, + stride=1, + padding=1, + norm=True, + relufactor=None, + ) def forward(self, input): conv0 = self._conv0(input) @@ -234,27 +240,30 @@ def forward(self, input): class Generator(fluid.dygraph.Layer): - def __init__(self, cfg, num_channels=3): super(Generator, self).__init__() - conv_base = Conv2DLayer(num_channels=cfg.c_dim + num_channels, - num_filters=cfg.g_base_dims, - filter_size=7, - stride=1, - padding=3, - norm=True, - relufactor=0) + conv_base = Conv2DLayer( + num_channels=cfg.c_dim + num_channels, + num_filters=cfg.g_base_dims, + filter_size=7, + stride=1, + padding=3, + norm=True, + relufactor=0, + ) sub_layers = [conv_base] cur_channels = cfg.g_base_dims for i in range(2): - sub_layer = Conv2DLayer(num_channels=cur_channels, - num_filters=cur_channels * 2, - filter_size=4, - stride=2, - padding=1, - norm=True, - relufactor=0) + sub_layer = Conv2DLayer( + num_channels=cur_channels, + num_filters=cur_channels * 2, + filter_size=4, + stride=2, + padding=1, + norm=True, + relufactor=0, + ) cur_channels *= 2 sub_layers.append(sub_layer) @@ -264,8 +273,9 @@ def __init__(self, cfg, num_channels=3): repeat_num = cfg.g_repeat_num sub_layers = [] for i in range(repeat_num): - res_block = ResidualBlock(num_channels=cur_channels, - num_filters=cfg.g_base_dims * 4) + res_block = ResidualBlock( + num_channels=cur_channels, num_filters=cfg.g_base_dims * 4 + ) sub_layers.append(res_block) self._res_block = fluid.dygraph.Sequential(*sub_layers) @@ -273,32 +283,38 @@ def __init__(self, cfg, num_channels=3): cur_channels = cfg.g_base_dims * 4 sub_layers = [] for i in range(2): - rate = 2**(1 - i) - deconv = Deconv2DLayer(num_channels=cur_channels, - num_filters=cfg.g_base_dims * rate, - filter_size=4, - stride=2, - padding=1, - relufactor=0, - norm=True) + rate = 2 ** (1 - i) + deconv = Deconv2DLayer( + num_channels=cur_channels, + num_filters=cfg.g_base_dims * rate, + filter_size=4, + stride=2, + padding=1, + relufactor=0, + norm=True, + ) cur_channels = cfg.g_base_dims * rate sub_layers.append(deconv) self._deconv = fluid.dygraph.Sequential(*sub_layers) - self._conv1 = Conv2DLayer(num_channels=cur_channels, - num_filters=3, - filter_size=7, - stride=1, - padding=3, - relufactor=None) + self._conv1 = Conv2DLayer( + num_channels=cur_channels, + num_filters=3, + filter_size=7, + stride=1, + padding=3, + relufactor=None, + ) def forward(self, input, label_trg): shape = input.shape - label_trg_e = fluid.layers.reshape(label_trg, - [-1, label_trg.shape[1], 1, 1]) + label_trg_e = fluid.layers.reshape( + label_trg, [-1, label_trg.shape[1], 1, 1] + ) label_trg_e = fluid.layers.expand( - x=label_trg_e, expand_times=[1, 1, shape[2], shape[3]]) + x=label_trg_e, expand_times=[1, 1, shape[2], shape[3]] + ) input1 = fluid.layers.concat([input, label_trg_e], 1) @@ -311,28 +327,31 @@ def forward(self, input, label_trg): class Discriminator(fluid.dygraph.Layer): - def __init__(self, cfg, num_channels=3): super(Discriminator, self).__init__() cur_dim = cfg.d_base_dims - conv_base = Conv2DLayer(num_channels=num_channels, - num_filters=cur_dim, - filter_size=4, - stride=2, - padding=1, - relufactor=0.2) + conv_base = Conv2DLayer( + num_channels=num_channels, + num_filters=cur_dim, + filter_size=4, + stride=2, + padding=1, + relufactor=0.2, + ) repeat_num = cfg.d_repeat_num sub_layers = [conv_base] for i in range(1, repeat_num): - sub_layer = Conv2DLayer(num_channels=cur_dim, - num_filters=cur_dim * 2, - filter_size=4, - stride=2, - padding=1, - relufactor=0.2) + sub_layer = Conv2DLayer( + num_channels=cur_dim, + num_filters=cur_dim * 2, + filter_size=4, + stride=2, + padding=1, + relufactor=0.2, + ) cur_dim *= 2 sub_layers.append(sub_layer) @@ -340,15 +359,17 @@ def __init__(self, cfg, num_channels=3): kernel_size = int(cfg.image_size / np.power(2, repeat_num)) - self._conv1 = Conv2DLayer(num_channels=cur_dim, - num_filters=1, - filter_size=3, - stride=1, - padding=1) + self._conv1 = Conv2DLayer( + num_channels=cur_dim, + num_filters=1, + filter_size=3, + stride=1, + padding=1, + ) - self._conv2 = Conv2DLayer(num_channels=cur_dim, - num_filters=cfg.c_dim, - filter_size=kernel_size) + self._conv2 = Conv2DLayer( + num_channels=cur_dim, num_filters=cfg.c_dim, filter_size=kernel_size + ) def forward(self, input): conv = self._conv0(input) @@ -359,38 +380,41 @@ def forward(self, input): def loss_cls(cls, label, cfg): cls_shape = cls.shape - cls = fluid.layers.reshape(cls, - [-1, cls_shape[1] * cls_shape[2] * cls_shape[3]]) - return fluid.layers.reduce_sum( - fluid.layers.sigmoid_cross_entropy_with_logits(cls, - label)) / cfg.batch_size + cls = fluid.layers.reshape( + cls, [-1, cls_shape[1] * cls_shape[2] * cls_shape[3]] + ) + return ( + fluid.layers.reduce_sum( + fluid.layers.sigmoid_cross_entropy_with_logits(cls, label) + ) + / cfg.batch_size + ) def calc_gradients(outputs, inputs, no_grad_set): if fluid._non_static_mode(): - return fluid.dygraph.grad(outputs=outputs, - inputs=inputs, - no_grad_vars=no_grad_set, - create_graph=True) + return fluid.dygraph.grad( + outputs=outputs, + inputs=inputs, + no_grad_vars=no_grad_set, + create_graph=True, + ) else: - return fluid.gradients(targets=outputs, - inputs=inputs, - no_grad_set=no_grad_set) + return fluid.gradients( + targets=outputs, inputs=inputs, no_grad_set=no_grad_set + ) def gradient_penalty(f, real, fake, no_grad_set, cfg): - def _interpolate(a, b): shape = [a.shape[0]] - alpha = fluid.layers.uniform_random_batch_size_like(input=a, - shape=shape, - min=0.1, - max=1.0, - seed=cfg.seed) + alpha = fluid.layers.uniform_random_batch_size_like( + input=a, shape=shape, min=0.1, max=1.0, seed=cfg.seed + ) inner = fluid.layers.elementwise_mul( - b, 1.0 - alpha, axis=0) + fluid.layers.elementwise_mul( - a, alpha, axis=0) + b, 1.0 - alpha, axis=0 + ) + fluid.layers.elementwise_mul(a, alpha, axis=0) return inner x = _interpolate(real, fake) @@ -398,9 +422,9 @@ def _interpolate(a, b): if isinstance(pred, tuple): pred = pred[0] - gradient = calc_gradients(outputs=[pred], - inputs=[x], - no_grad_set=no_grad_set) + gradient = calc_gradients( + outputs=[pred], inputs=[x], no_grad_set=no_grad_set + ) if gradient is None: return None @@ -409,22 +433,26 @@ def _interpolate(a, b): grad_shape = gradient.shape gradient = fluid.layers.reshape( - gradient, [-1, grad_shape[1] * grad_shape[2] * grad_shape[3]]) + gradient, [-1, grad_shape[1] * grad_shape[2] * grad_shape[3]] + ) epsilon = 1e-16 norm = fluid.layers.sqrt( - fluid.layers.reduce_sum(fluid.layers.square(gradient), dim=1) + epsilon) + fluid.layers.reduce_sum(fluid.layers.square(gradient), dim=1) + epsilon + ) gp = fluid.layers.reduce_mean(fluid.layers.square(norm - 1.0)) return gp -def get_generator_loss(image_real, label_org, label_trg, generator, - discriminator, cfg): +def get_generator_loss( + image_real, label_org, label_trg, generator, discriminator, cfg +): fake_img = generator(image_real, label_trg) rec_img = generator(fake_img, label_org) g_loss_rec = fluid.layers.reduce_mean( - fluid.layers.abs(fluid.layers.elementwise_sub(image_real, rec_img))) + fluid.layers.abs(fluid.layers.elementwise_sub(image_real, rec_img)) + ) pred_fake, cls_fake = discriminator(fake_img) @@ -434,8 +462,9 @@ def get_generator_loss(image_real, label_org, label_trg, generator, return g_loss -def get_discriminator_loss(image_real, label_org, label_trg, generator, - discriminator, cfg): +def get_discriminator_loss( + image_real, label_org, label_trg, generator, discriminator, cfg +): fake_img = generator(image_real, label_trg) pred_real, cls_real = discriminator(image_real) pred_fake, _ = discriminator(fake_img) @@ -444,8 +473,13 @@ def get_discriminator_loss(image_real, label_org, label_trg, generator, d_loss_real = -paddle.mean(pred_real) d_loss = d_loss_real + d_loss_fake + d_loss_cls - d_loss_gp = gradient_penalty(discriminator, image_real, fake_img, - set(discriminator.parameters()), cfg) + d_loss_gp = gradient_penalty( + discriminator, + image_real, + fake_img, + set(discriminator.parameters()), + cfg, + ) if d_loss_gp is not None: d_loss += cfg.lambda_gp * d_loss_gp @@ -457,21 +491,22 @@ def build_optimizer(layer, cfg, loss=None): beta1 = 0.5 beta2 = 0.999 if fluid._non_static_mode(): - return fluid.optimizer.Adam(learning_rate=learning_rate, - beta1=beta1, - beta2=beta2, - parameter_list=layer.parameters()) + return fluid.optimizer.Adam( + learning_rate=learning_rate, + beta1=beta1, + beta2=beta2, + parameter_list=layer.parameters(), + ) else: - optimizer = fluid.optimizer.Adam(learning_rate=learning_rate, - beta1=beta1, - beta2=beta2) + optimizer = fluid.optimizer.Adam( + learning_rate=learning_rate, beta1=beta1, beta2=beta2 + ) optimizer.minimize(loss, parameter_list=layer.parameters()) return optimizer class DyGraphTrainModel(object): - def __init__(self, cfg): paddle.seed(1) paddle.framework.random._manual_program_seed(1) @@ -498,18 +533,28 @@ def run(self, image_real, label_org, label_trg): label_org = fluid.dygraph.to_variable(label_org) label_trg = fluid.dygraph.to_variable(label_trg) - g_loss = get_generator_loss(image_real, label_org, label_trg, - self.generator, self.discriminator, - self.cfg) + g_loss = get_generator_loss( + image_real, + label_org, + label_trg, + self.generator, + self.discriminator, + self.cfg, + ) g_loss.backward() if self.g_optimizer: self.g_optimizer.minimize(g_loss) self.clear_gradients() - d_loss = get_discriminator_loss(image_real, label_org, label_trg, - self.generator, self.discriminator, - self.cfg) + d_loss = get_discriminator_loss( + image_real, + label_org, + label_trg, + self.generator, + self.discriminator, + self.cfg, + ) d_loss.backward() if self.d_optimizer: self.d_optimizer.minimize(d_loss) @@ -520,7 +565,6 @@ def run(self, image_real, label_org, label_trg): class StaticGraphTrainModel(object): - def __init__(self, cfg): self.cfg = cfg @@ -528,13 +572,14 @@ def create_data_layer(): image_real = fluid.data( shape=[None, 3, cfg.image_size, cfg.image_size], dtype='float32', - name='image_real') - label_org = fluid.data(shape=[None, cfg.c_dim], - dtype='float32', - name='label_org') - label_trg = fluid.data(shape=[None, cfg.c_dim], - dtype='float32', - name='label_trg') + name='image_real', + ) + label_org = fluid.data( + shape=[None, cfg.c_dim], dtype='float32', name='label_org' + ) + label_trg = fluid.data( + shape=[None, cfg.c_dim], dtype='float32', name='label_trg' + ) return image_real, label_org, label_trg paddle.seed(cfg.seed) @@ -547,8 +592,14 @@ def create_data_layer(): image_real, label_org, label_trg = create_data_layer() generator = Generator(cfg) discriminator = Discriminator(cfg) - g_loss = get_generator_loss(image_real, label_org, label_trg, - generator, discriminator, cfg) + g_loss = get_generator_loss( + image_real, + label_org, + label_trg, + generator, + discriminator, + cfg, + ) build_optimizer(generator, cfg, loss=g_loss) self.dis_program = fluid.Program() @@ -558,9 +609,14 @@ def create_data_layer(): image_real, label_org, label_trg = create_data_layer() generator = Generator(cfg) discriminator = Discriminator(cfg) - d_loss = get_discriminator_loss(image_real, label_org, - label_trg, generator, - discriminator, cfg) + d_loss = get_discriminator_loss( + image_real, + label_org, + label_trg, + generator, + discriminator, + cfg, + ) build_optimizer(discriminator, cfg, loss=d_loss) self.executor = fluid.Executor(cfg.place) @@ -577,20 +633,19 @@ def run(self, image_real, label_org, label_trg): feed = { 'image_real': image_real, 'label_org': label_org, - 'label_trg': label_trg + 'label_trg': label_trg, } with fluid.scope_guard(self.scope): - g_loss_val = self.executor.run(self.gen_program, - feed=feed, - fetch_list=[self.g_loss])[0] - d_loss_val = self.executor.run(self.dis_program, - feed=feed, - fetch_list=[self.d_loss])[0] + g_loss_val = self.executor.run( + self.gen_program, feed=feed, fetch_list=[self.g_loss] + )[0] + d_loss_val = self.executor.run( + self.dis_program, feed=feed, fetch_list=[self.d_loss] + )[0] return g_loss_val[0], d_loss_val[0] class TestStarGANWithGradientPenalty(unittest.TestCase): - def func_main(self): self.place_test(fluid.CPUPlace()) @@ -606,8 +661,9 @@ def place_test(self, place): fluid_dygraph_loss = [] with fluid.dygraph.guard(cfg.place): fluid_dygraph_model = DyGraphTrainModel(cfg) - for batch_id, (image_real, label_org, - label_trg) in enumerate(dataset()): + for batch_id, (image_real, label_org, label_trg) in enumerate( + dataset() + ): loss = fluid_dygraph_model.run(image_real, label_org, label_trg) fluid_dygraph_loss.append(loss) @@ -615,15 +671,17 @@ def place_test(self, place): with _test_eager_guard(): with fluid.dygraph.guard(cfg.place): eager_dygraph_model = DyGraphTrainModel(cfg) - for batch_id, (image_real, label_org, - label_trg) in enumerate(dataset()): - loss = eager_dygraph_model.run(image_real, label_org, - label_trg) + for batch_id, (image_real, label_org, label_trg) in enumerate( + dataset() + ): + loss = eager_dygraph_model.run( + image_real, label_org, label_trg + ) eager_dygraph_loss.append(loss) - for (g_loss_f, d_loss_f), (g_loss_e, - d_loss_e) in zip(fluid_dygraph_loss, - eager_dygraph_loss): + for (g_loss_f, d_loss_f), (g_loss_e, d_loss_e) in zip( + fluid_dygraph_loss, eager_dygraph_loss + ): self.assertEqual(g_loss_f, g_loss_e) self.assertEqual(d_loss_f, d_loss_e) @@ -632,7 +690,6 @@ def test_all_cases(self): class TestStarGANWithGradientPenaltyLegacy(unittest.TestCase): - def func_main(self): self.place_test(fluid.CPUPlace()) @@ -647,21 +704,24 @@ def place_test(self, place): static_graph_model = StaticGraphTrainModel(cfg) static_loss = [] - for batch_id, (image_real, label_org, - label_trg) in enumerate(dataset()): + for batch_id, (image_real, label_org, label_trg) in enumerate( + dataset() + ): loss = static_graph_model.run(image_real, label_org, label_trg) static_loss.append(loss) dygraph_loss = [] with fluid.dygraph.guard(cfg.place): dygraph_model = DyGraphTrainModel(cfg) - for batch_id, (image_real, label_org, - label_trg) in enumerate(dataset()): + for batch_id, (image_real, label_org, label_trg) in enumerate( + dataset() + ): loss = dygraph_model.run(image_real, label_org, label_trg) dygraph_loss.append(loss) - for (g_loss_s, d_loss_s), (g_loss_d, - d_loss_d) in zip(static_loss, dygraph_loss): + for (g_loss_s, d_loss_s), (g_loss_d, d_loss_d) in zip( + static_loss, dygraph_loss + ): self.assertEqual(g_loss_s, g_loss_d) self.assertEqual(d_loss_s, d_loss_d) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_static_runner_mnist.py b/python/paddle/fluid/tests/unittests/test_imperative_static_runner_mnist.py index cc204e6b3d42c..825144cd16ade 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_static_runner_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_static_runner_mnist.py @@ -25,19 +25,23 @@ def convolutional_neural_network(img): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) prediction = fluid.layers.fc(input=conv_pool_2, size=10, act='softmax') return prediction @@ -55,7 +59,6 @@ def static_train_net(img, label): class TestImperativeStaticModelRunnerMnist(unittest.TestCase): - def setUp(self): self.seed = 90 self.epoch_num = 1 @@ -63,7 +66,6 @@ def setUp(self): self.batch_num = 50 def reader_decorator(self, reader): - def _reader_impl(): for item in reader(): image = np.array(item[0]).reshape(1, 28, 28) @@ -77,44 +79,58 @@ def train_and_save_model(self): startup_program = fluid.default_startup_program() main_program = fluid.default_main_program() - img = fluid.data(name='img', - shape=[None, 1, 28, 28], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 1, 28, 28], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') prediction, avg_loss = static_train_net(img, label) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[img, label], place=place) exe.run(startup_program) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=100), - batch_size=self.batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.mnist.train(), buf_size=100 + ), + batch_size=self.batch_size, + ) for _ in range(0, self.epoch_num): for batch_id, data in enumerate(train_reader()): - exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[avg_loss]) + exe.run( + main_program, + feed=feeder.feed(data), + fetch_list=[avg_loss], + ) if batch_id > self.batch_num: break - fluid.io.save_inference_model(self.save_dirname, ["img"], - [prediction], - exe, - model_filename=self.model_filename, - params_filename=self.params_filename, - clip_extra=False) + fluid.io.save_inference_model( + self.save_dirname, + ["img"], + [prediction], + exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + clip_extra=False, + ) def load_and_train_dygraph(self): - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.dygraph.guard(place): fluid.default_startup_program().random_seed = self.seed fluid.default_main_program().random_seed = self.seed @@ -123,22 +139,26 @@ def load_and_train_dygraph(self): mnist = fluid.dygraph.static_runner.StaticModelRunner( model_dir=self.save_dirname, model_filename=self.model_filename, - params_filename=self.params_filename) + params_filename=self.params_filename, + ) suffix_varname_dict = mnist._program_holder_dict[ - 'forward']._suffix_varname_dict + 'forward' + ]._suffix_varname_dict dict_old_new = {v: k for k, v in suffix_varname_dict.items()} dy_param_init_value = {} for param in mnist.parameters(): dy_param_init_value[param.name] = param.numpy() - sgd = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=mnist.parameters()) + sgd = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=mnist.parameters() + ) - train_reader = paddle.batch(self.reader_decorator( - paddle.dataset.mnist.train()), - batch_size=self.batch_size, - drop_last=True) + train_reader = paddle.batch( + self.reader_decorator(paddle.dataset.mnist.train()), + batch_size=self.batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator(capacity=10) train_loader.set_sample_list_generator(train_reader, places=place) @@ -169,58 +189,71 @@ def load_and_train_dygraph(self): for param in mnist.parameters(): dy_param_value[param.name] = param.numpy() - return dy_x_data, dy_out, dy_param_init_value, dy_param_value, dict_old_new + return ( + dy_x_data, + dy_out, + dy_param_init_value, + dy_param_value, + dict_old_new, + ) def load_and_train_static(self): with new_program_scope(): fluid.default_startup_program().random_seed = self.seed fluid.default_main_program().random_seed = self.seed - img = fluid.data(name='img', - shape=[None, 1, 28, 28], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 1, 28, 28], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') prediction, avg_loss = static_train_net(img, label) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - fluid.io.load_params(exe, - self.save_dirname, - main_program=fluid.default_main_program(), - filename=self.params_filename) + fluid.io.load_params( + exe, + self.save_dirname, + main_program=fluid.default_main_program(), + filename=self.params_filename, + ) static_param_init_value = {} static_param_name_list = [] for param in fluid.default_main_program().all_parameters(): static_param_name_list.append(param.name) static_param_init_value[param.name] = fluid.executor._fetch_var( - param.name) + param.name + ) - train_reader = paddle.batch(self.reader_decorator( - paddle.dataset.mnist.train()), - batch_size=self.batch_size, - drop_last=True) + train_reader = paddle.batch( + self.reader_decorator(paddle.dataset.mnist.train()), + batch_size=self.batch_size, + drop_last=True, + ) for epoch in range(self.epoch_num): for batch_id, data in enumerate(train_reader()): static_x_data = np.array([x[0] for x in data]) - y_data = np.array([x[1] for x in data - ]).reshape([self.batch_size, 1]) + y_data = np.array([x[1] for x in data]).reshape( + [self.batch_size, 1] + ) fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed={ - "img": static_x_data, - "label": y_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={"img": static_x_data, "label": y_data}, + fetch_list=fetch_list, + ) if batch_id >= self.batch_num: break @@ -230,21 +263,31 @@ def load_and_train_static(self): for i in range(1, len(out)): static_param_value[static_param_name_list[i - 1]] = out[i] - return static_x_data, static_out, static_param_init_value, static_param_value + return ( + static_x_data, + static_out, + static_param_init_value, + static_param_value, + ) def load_and_infer_dygraph(self): - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.dygraph.guard(place): fluid.default_main_program().random_seed = self.seed mnist = fluid.dygraph.static_runner.StaticModelRunner( - model_dir=self.save_dirname, model_filename=self.model_filename) - - train_reader = paddle.batch(self.reader_decorator( - paddle.dataset.mnist.test()), - batch_size=self.batch_size, - drop_last=True) + model_dir=self.save_dirname, model_filename=self.model_filename + ) + + train_reader = paddle.batch( + self.reader_decorator(paddle.dataset.mnist.test()), + batch_size=self.batch_size, + drop_last=True, + ) train_loader = fluid.io.DataLoader.from_generator(capacity=10) train_loader.set_sample_list_generator(train_reader, places=place) @@ -264,24 +307,33 @@ def load_and_infer_dygraph(self): def load_and_infer_static(self): with new_program_scope(): - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - [infer_program, feed_target_names, fetch_targets - ] = fluid.io.load_inference_model(self.save_dirname, exe) + [ + infer_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model(self.save_dirname, exe) infer_program.random_seed = self.seed - train_reader = paddle.batch(self.reader_decorator( - paddle.dataset.mnist.test()), - batch_size=self.batch_size, - drop_last=True) + train_reader = paddle.batch( + self.reader_decorator(paddle.dataset.mnist.test()), + batch_size=self.batch_size, + drop_last=True, + ) for batch_id, data in enumerate(train_reader()): static_x_data = np.array([x[0] for x in data]) - out = exe.run(infer_program, - feed={feed_target_names[0]: static_x_data}, - fetch_list=fetch_targets) + out = exe.run( + infer_program, + feed={feed_target_names[0]: static_x_data}, + fetch_list=fetch_targets, + ) if batch_id >= 1: break @@ -300,11 +352,20 @@ def test_mnist_train_no_params_filename(self): # Phase 2. load model & train dygraph - dy_x_data, dy_out, dy_param_init_value, dy_param_value, dict_old_new_init= \ - self.load_and_train_dygraph() - - static_x_data, static_out, static_param_init_value, static_param_value = \ - self.load_and_train_static() + ( + dy_x_data, + dy_out, + dy_param_init_value, + dy_param_value, + dict_old_new_init, + ) = self.load_and_train_dygraph() + + ( + static_x_data, + static_out, + static_param_init_value, + static_param_value, + ) = self.load_and_train_static() # Phase 3. compare np.testing.assert_array_equal(static_x_data, dy_x_data) @@ -318,10 +379,9 @@ def test_mnist_train_no_params_filename(self): for key, value in static_param_value.items(): key = dict_old_new_init[key] - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05, - atol=1e-4) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05, atol=1e-4 + ) def test_mnist_train_with_params_filename(self): self.save_dirname = "mnist.inference.model" @@ -331,11 +391,20 @@ def test_mnist_train_with_params_filename(self): self.train_and_save_model() # Phase 2. load model & train dygraph - dy_x_data, dy_out, dy_param_init_value, dy_param_value, dict_old_new_init= \ - self.load_and_train_dygraph() - - static_x_data, static_out, static_param_init_value, static_param_value = \ - self.load_and_train_static() + ( + dy_x_data, + dy_out, + dy_param_init_value, + dy_param_value, + dict_old_new_init, + ) = self.load_and_train_dygraph() + + ( + static_x_data, + static_out, + static_param_init_value, + static_param_value, + ) = self.load_and_train_static() # Phase 3. compare np.testing.assert_array_equal(static_x_data, dy_x_data) @@ -348,10 +417,9 @@ def test_mnist_train_with_params_filename(self): for key, value in static_param_value.items(): key = dict_old_new_init[key] - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05, - atol=1e-4) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05, atol=1e-4 + ) def test_mnist_infer_no_params_filename(self): self.save_dirname = "mnist.inference.model.noname" @@ -361,11 +429,9 @@ def test_mnist_infer_no_params_filename(self): self.train_and_save_model() # Phase 2. load model & train dygraph - dy_x_data, dy_out = \ - self.load_and_infer_dygraph() + dy_x_data, dy_out = self.load_and_infer_dygraph() - static_x_data, static_out = \ - self.load_and_infer_static() + static_x_data, static_out = self.load_and_infer_static() # Phase 3. compare np.testing.assert_array_equal(static_x_data, dy_x_data) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_static_runner_while.py b/python/paddle/fluid/tests/unittests/test_imperative_static_runner_while.py index b12d8e6a3db48..d163350f136b7 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_static_runner_while.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_static_runner_while.py @@ -27,7 +27,6 @@ def while_softmax_regression(img): - def cond(i, times, pred): return i < times @@ -39,14 +38,13 @@ def body(i, times, pred): i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=0) times = fluid.layers.fill_constant(shape=[1], dtype='int64', value=5) pred = fluid.layers.fc(input=img, size=10, act='softmax') - i, times, pred = fluid.layers.while_loop(cond=cond, - body=body, - loop_vars=[i, times, pred]) + i, times, pred = fluid.layers.while_loop( + cond=cond, body=body, loop_vars=[i, times, pred] + ) return pred class TestImperativeStaticModelRunnerWhile(unittest.TestCase): - def setUp(self): self.seed = 90 self.batch_size = 32 @@ -56,7 +54,6 @@ def setUp(self): self.params_filename = None def _random_batch_reader(self): - def _get_random_images_and_labels(image_shape, label_shape): image = np.random.random(size=image_shape).astype('float32') label = np.random.random(size=label_shape).astype('int64') @@ -65,7 +62,8 @@ def _get_random_images_and_labels(image_shape, label_shape): def __reader__(): for _ in range(self.batch_num): batch_image, batch_label = _get_random_images_and_labels( - [self.batch_size, 784], [self.batch_size, 1]) + [self.batch_size, 784], [self.batch_size, 1] + ) yield batch_image, batch_label return __reader__ @@ -85,29 +83,39 @@ def train_and_save_model(self): optimizer = fluid.optimizer.SGD(learning_rate=0.001) optimizer.minimize(avg_loss) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(startup_program) - loader = fluid.io.DataLoader.from_generator(feed_list=[img, label], - capacity=5, - iterable=True) + loader = fluid.io.DataLoader.from_generator( + feed_list=[img, label], capacity=5, iterable=True + ) loader.set_batch_generator(self._random_batch_reader(), places=place) for data in loader(): exe.run(main_program, feed=data, fetch_list=[avg_loss]) - fluid.io.save_inference_model(self.save_dirname, ["img"], [pred], - exe, - model_filename=self.model_filename, - params_filename=self.params_filename, - clip_extra=False) + fluid.io.save_inference_model( + self.save_dirname, + ["img"], + [pred], + exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + clip_extra=False, + ) def load_and_train_dygraph(self): - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.dygraph.guard(place): fluid.default_startup_program().random_seed = self.seed fluid.default_main_program().random_seed = self.seed @@ -115,18 +123,21 @@ def load_and_train_dygraph(self): fluid.set_flags({'FLAGS_sort_sum_gradient': True}) while_net = fluid.dygraph.static_runner.StaticModelRunner( - self.save_dirname) + self.save_dirname + ) dy_param_init_value = {} for param in while_net.parameters(): dy_param_init_value[param.name] = param.numpy() - sgd = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=while_net.parameters()) + sgd = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=while_net.parameters() + ) train_loader = fluid.io.DataLoader.from_generator(capacity=10) - train_loader.set_batch_generator(self._random_batch_reader(), - places=place) + train_loader.set_batch_generator( + self._random_batch_reader(), places=place + ) while_net.train() @@ -168,37 +179,46 @@ def load_and_train_static(self): optimizer = fluid.optimizer.SGD(learning_rate=0.001) optimizer.minimize(avg_loss) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - fluid.io.load_params(exe, - self.save_dirname, - main_program=fluid.default_main_program(), - filename=self.params_filename) + fluid.io.load_params( + exe, + self.save_dirname, + main_program=fluid.default_main_program(), + filename=self.params_filename, + ) static_param_init_value = {} static_param_name_list = [] for param in fluid.default_main_program().all_parameters(): static_param_name_list.append(param.name) static_param_init_value[param.name] = fluid.executor._fetch_var( - param.name) + param.name + ) - loader = fluid.io.DataLoader.from_generator(feed_list=[img, label], - capacity=5, - iterable=True) - loader.set_batch_generator(self._random_batch_reader(), - places=place) + loader = fluid.io.DataLoader.from_generator( + feed_list=[img, label], capacity=5, iterable=True + ) + loader.set_batch_generator( + self._random_batch_reader(), places=place + ) for data in loader(): fetch_list = [avg_loss.name] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed=data, - fetch_list=[avg_loss]) + out = exe.run( + fluid.default_main_program(), + feed=data, + fetch_list=[avg_loss], + ) static_param_value = {} static_out = out[0] @@ -213,17 +233,24 @@ def test_while_no_params_filename(self): # # Phase 2. load model & train dygraph with unique_name.guard(): - dy_out, dy_param_init_value, dy_param_value = \ - self.load_and_train_dygraph() + ( + dy_out, + dy_param_init_value, + dy_param_value, + ) = self.load_and_train_dygraph() with unique_name.guard(): - static_out, static_param_init_value, static_param_value = \ - self.load_and_train_static() + ( + static_out, + static_param_init_value, + static_param_value, + ) = self.load_and_train_static() # Phase 3. compare with unique_name.guard(): dict_old_new_init = rename_var_with_generator( - static_param_init_value.keys()) + static_param_init_value.keys() + ) for key, value in static_param_init_value.items(): key = dict_old_new_init[key] np.testing.assert_array_equal(value, dy_param_init_value[key]) @@ -232,10 +259,9 @@ def test_while_no_params_filename(self): for key, value in static_param_value.items(): key += LOADED_VAR_SUFFIX - np.testing.assert_allclose(value, - dy_param_value[key], - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + value, dy_param_value[key], rtol=1e-05, atol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_tensor_clear_gradient.py b/python/paddle/fluid/tests/unittests/test_imperative_tensor_clear_gradient.py index 55879293734ad..0445e09232f2c 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_tensor_clear_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_tensor_clear_gradient.py @@ -21,7 +21,6 @@ def _dygraph_guard_(func): - def __impl__(*args, **kwargs): if fluid._non_static_mode(): return func(*args, **kwargs) @@ -36,7 +35,6 @@ def __impl__(*args, **kwargs): class TestDygraphClearGradient(TestCase): - def setUp(self): self.input_shape = [10, 2] diff --git a/python/paddle/fluid/tests/unittests/test_imperative_thread_local_has_grad.py b/python/paddle/fluid/tests/unittests/test_imperative_thread_local_has_grad.py index 5e8a54ca39014..7acbc2d339c1c 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_thread_local_has_grad.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_thread_local_has_grad.py @@ -22,7 +22,6 @@ class SimpleNet(nn.Layer): - def __init__(self, in_dim, out_dim): super(SimpleNet, self).__init__() self.fc = nn.Linear(in_dim, out_dim) @@ -32,7 +31,6 @@ def forward(self, x): class TestCases(unittest.TestCase): - @paddle.no_grad() def thread_1_main(self): time.sleep(8) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_trace_non_persistable_inputs.py b/python/paddle/fluid/tests/unittests/test_imperative_trace_non_persistable_inputs.py index a346bb2298588..66a1e7f84119c 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_trace_non_persistable_inputs.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_trace_non_persistable_inputs.py @@ -19,12 +19,12 @@ class SimpleFCLayer(fluid.dygraph.Layer): - def __init__(self, feature_size, batch_size, fc_size): super(SimpleFCLayer, self).__init__() self._linear = fluid.dygraph.Linear(feature_size, fc_size) self._offset = fluid.dygraph.to_variable( - np.random.random((batch_size, fc_size)).astype('float32')) + np.random.random((batch_size, fc_size)).astype('float32') + ) def forward(self, x): fc = self._linear(x) @@ -32,7 +32,6 @@ def forward(self, x): class TestTracedLayerRecordNonPersistableInput(unittest.TestCase): - def test_main(self): if fluid.framework.in_dygraph_mode(): return @@ -42,21 +41,28 @@ def test_main(self): batch_size = 4 fc_size = 2 layer = SimpleFCLayer(feature_size, batch_size, fc_size) - optimizer = fluid.optimizer.SGD(learning_rate=1e-3, - parameter_list=layer.parameters()) - - expected_persistable_vars = set([ - layer._linear.weight.name, layer._linear.bias.name, - layer._offset.name - ]) + optimizer = fluid.optimizer.SGD( + learning_rate=1e-3, parameter_list=layer.parameters() + ) + + expected_persistable_vars = set( + [ + layer._linear.weight.name, + layer._linear.bias.name, + layer._offset.name, + ] + ) for _ in range(10): in_x = fluid.dygraph.to_variable( - np.random.random( - (batch_size, feature_size)).astype('float32')) + np.random.random((batch_size, feature_size)).astype( + 'float32' + ) + ) if traced_layer is None: dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - layer, [in_x]) + layer, [in_x] + ) else: dygraph_out = layer(in_x) dygraph_out_numpy = dygraph_out.numpy() @@ -79,11 +85,15 @@ def test_main(self): self.assertEqual(actual_persistable_vars, expected_persistable_vars) traced_layer.save_inference_model( - path='./traced_layer_test_non_persistable_vars') - self.assertTrue('traced_layer_test_non_persistable_vars.pdmodel' in - os.listdir('./')) - self.assertTrue('traced_layer_test_non_persistable_vars.pdiparams' in - os.listdir('./')) + path='./traced_layer_test_non_persistable_vars' + ) + self.assertTrue( + 'traced_layer_test_non_persistable_vars.pdmodel' in os.listdir('./') + ) + self.assertTrue( + 'traced_layer_test_non_persistable_vars.pdiparams' + in os.listdir('./') + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_transformer_sorted_gradient.py b/python/paddle/fluid/tests/unittests/test_imperative_transformer_sorted_gradient.py index 3ba02fe8dafcd..22189d88d3102 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_transformer_sorted_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_transformer_sorted_gradient.py @@ -149,12 +149,15 @@ def position_encoding_init(n_position, d_pos_vec): channels = d_pos_vec position = np.arange(n_position) num_timescales = channels // 2 - log_timescale_increment = (np.log(float(1e4) / float(1)) / - (num_timescales - 1)) - inv_timescales = np.exp( - np.arange(num_timescales)) * -log_timescale_increment + log_timescale_increment = np.log(float(1e4) / float(1)) / ( + num_timescales - 1 + ) + inv_timescales = ( + np.exp(np.arange(num_timescales)) * -log_timescale_increment + ) scaled_time = np.expand_dims(position, 1) * np.expand_dims( - inv_timescales, 0) + inv_timescales, 0 + ) signal = np.concatenate([np.sin(scaled_time), np.cos(scaled_time)], axis=1) signal = np.pad(signal, [[0, 0], [0, np.mod(channels, 2)]], 'constant') position_enc = signal @@ -164,21 +167,27 @@ def position_encoding_init(n_position, d_pos_vec): def create_data(is_static=False): if is_static: return [ - src_word_np, src_pos_np, src_slf_attn_bias_np, trg_word_np, - trg_pos_np, trg_slf_attn_bias_np, trg_src_attn_bias_np, lbl_word_np, - lbl_weight_np + src_word_np, + src_pos_np, + src_slf_attn_bias_np, + trg_word_np, + trg_pos_np, + trg_slf_attn_bias_np, + trg_src_attn_bias_np, + lbl_word_np, + lbl_weight_np, ] else: enc_inputs = [ to_variable(src_word_np, name='src_word'), to_variable(src_pos_np, name='src_pos'), - to_variable(src_slf_attn_bias_np, name='src_slf_attn_bias') + to_variable(src_slf_attn_bias_np, name='src_slf_attn_bias'), ] dec_inputs = [ to_variable(trg_word_np, name='trg_word'), to_variable(trg_pos_np, name='trg_pos'), to_variable(trg_slf_attn_bias_np, name='trg_slf_attn_bias'), - to_variable(trg_src_attn_bias_np, name='trg_src_attn_bias') + to_variable(trg_src_attn_bias_np, name='trg_src_attn_bias'), ] label = to_variable(lbl_word_np, name='lbl_word') weight = to_variable(lbl_weight_np, name='lbl_weight') @@ -187,11 +196,18 @@ def create_data(is_static=False): def create_feed_dict_list(data, init=False): if init: - data_input_names = encoder_data_input_fields + \ - decoder_data_input_fields[:-1] + label_data_input_fields + pos_enc_param_names + data_input_names = ( + encoder_data_input_fields + + decoder_data_input_fields[:-1] + + label_data_input_fields + + pos_enc_param_names + ) else: - data_input_names = encoder_data_input_fields + \ - decoder_data_input_fields[:-1] + label_data_input_fields + data_input_names = ( + encoder_data_input_fields + + decoder_data_input_fields[:-1] + + label_data_input_fields + ) feed_dict_list = dict() for i in range(len(data_input_names)): feed_dict_list[data_input_names[i]] = data[i] @@ -204,12 +220,15 @@ def make_all_inputs(input_fields): """ inputs = [] for input_field in input_fields: - input_var = fluid.layers.data(name=input_field, - shape=input_descs[input_field][0], - dtype=input_descs[input_field][1], - lod_level=input_descs[input_field][2] if - len(input_descs[input_field]) == 3 else 0, - append_batch_size=False) + input_var = fluid.layers.data( + name=input_field, + shape=input_descs[input_field][0], + dtype=input_descs[input_field][1], + lod_level=input_descs[input_field][2] + if len(input_descs[input_field]) == 3 + else 0, + append_batch_size=False, + ) inputs.append(input_var) return inputs @@ -234,12 +253,17 @@ def make_all_inputs(input_fields): # encoder. # The actual data shape of src_slf_attn_bias is: # [batch_size, n_head, max_src_len_in_batch, max_src_len_in_batch] - "src_slf_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "src_slf_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # The actual data shape of trg_word is: # [batch_size, max_trg_len_in_batch] - "trg_word": [(batch_size, seq_len), "int64", - 2], # lod_level is only used in fast decoder. + "trg_word": [ + (batch_size, seq_len), + "int64", + 2, + ], # lod_level is only used in fast decoder. # The actual data shape of trg_pos is: # [batch_size, max_trg_len_in_batch] "trg_pos": [(batch_size, seq_len), "int64"], @@ -247,14 +271,18 @@ def make_all_inputs(input_fields): # subsequent words in the decoder. # The actual data shape of trg_slf_attn_bias is: # [batch_size, n_head, max_trg_len_in_batch, max_trg_len_in_batch] - "trg_slf_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "trg_slf_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # This input is used to remove attention weights on paddings of the source # input in the encoder-decoder attention. # The actual data shape of trg_src_attn_bias is: # [batch_size, n_head, max_trg_len_in_batch, max_src_len_in_batch] - "trg_src_attn_bias": - [(batch_size, ModelHyperParams.n_head, seq_len, seq_len), "float32"], + "trg_src_attn_bias": [ + (batch_size, ModelHyperParams.n_head, seq_len, seq_len), + "float32", + ], # This input is used in independent decoder program for inference. # The actual data shape of enc_output is: # [batch_size, max_src_len_in_batch, d_model] @@ -270,7 +298,7 @@ def make_all_inputs(input_fields): "init_score": [(batch_size, 1), "float32", 2], # This input is used in beam-search decoder for the first gather # (cell states updation) - "init_idx": [(batch_size, ), "int32"], + "init_idx": [(batch_size,), "int32"], } # Names of word embedding table which might be reused for weight sharing. @@ -318,44 +346,52 @@ def make_all_inputs(input_fields): batch_num = 5 np.random.seed(90) -src_word_np = np.arange(1, TrainTaskConfig.batch_size * seq_len + 1).reshape( - [TrainTaskConfig.batch_size, seq_len]).astype('int64') -src_pos_np = np.random.randint(1, - seq_len, - size=(TrainTaskConfig.batch_size, seq_len), - dtype='int64') -src_slf_attn_bias_np = np.random.randn(TrainTaskConfig.batch_size, - ModelHyperParams.n_head, seq_len, - seq_len).astype('float32') - -trg_word_np = np.arange(1, TrainTaskConfig.batch_size * seq_len + 1).reshape( - [TrainTaskConfig.batch_size, seq_len]).astype('int64') -trg_pos_np = np.random.randint(1, - seq_len, - size=(TrainTaskConfig.batch_size, seq_len), - dtype='int64') -trg_slf_attn_bias_np = np.random.randn(TrainTaskConfig.batch_size, - ModelHyperParams.n_head, seq_len, - seq_len).astype('float32') -trg_src_attn_bias_np = np.random.randn(TrainTaskConfig.batch_size, - ModelHyperParams.n_head, seq_len, - seq_len).astype('float32') - -lbl_word_np = np.random.randint(1, - ModelHyperParams.src_vocab_size - 1, - size=(TrainTaskConfig.batch_size * seq_len, 1), - dtype='int64') -lbl_weight_np = np.random.randn(TrainTaskConfig.batch_size * seq_len, - 1).astype('float32') - -pos_inp1 = position_encoding_init(ModelHyperParams.max_length, - ModelHyperParams.d_model) -pos_inp2 = position_encoding_init(ModelHyperParams.max_length, - ModelHyperParams.d_model) +src_word_np = ( + np.arange(1, TrainTaskConfig.batch_size * seq_len + 1) + .reshape([TrainTaskConfig.batch_size, seq_len]) + .astype('int64') +) +src_pos_np = np.random.randint( + 1, seq_len, size=(TrainTaskConfig.batch_size, seq_len), dtype='int64' +) +src_slf_attn_bias_np = np.random.randn( + TrainTaskConfig.batch_size, ModelHyperParams.n_head, seq_len, seq_len +).astype('float32') + +trg_word_np = ( + np.arange(1, TrainTaskConfig.batch_size * seq_len + 1) + .reshape([TrainTaskConfig.batch_size, seq_len]) + .astype('int64') +) +trg_pos_np = np.random.randint( + 1, seq_len, size=(TrainTaskConfig.batch_size, seq_len), dtype='int64' +) +trg_slf_attn_bias_np = np.random.randn( + TrainTaskConfig.batch_size, ModelHyperParams.n_head, seq_len, seq_len +).astype('float32') +trg_src_attn_bias_np = np.random.randn( + TrainTaskConfig.batch_size, ModelHyperParams.n_head, seq_len, seq_len +).astype('float32') + +lbl_word_np = np.random.randint( + 1, + ModelHyperParams.src_vocab_size - 1, + size=(TrainTaskConfig.batch_size * seq_len, 1), + dtype='int64', +) +lbl_weight_np = np.random.randn(TrainTaskConfig.batch_size * seq_len, 1).astype( + 'float32' +) +pos_inp1 = position_encoding_init( + ModelHyperParams.max_length, ModelHyperParams.d_model +) +pos_inp2 = position_encoding_init( + ModelHyperParams.max_length, ModelHyperParams.d_model +) -class PrePostProcessLayer(Layer): +class PrePostProcessLayer(Layer): def __init__(self, d_model, process_cmd, shape_len=None): super(PrePostProcessLayer, self).__init__() for cmd in process_cmd: @@ -363,11 +399,14 @@ def __init__(self, d_model, process_cmd, shape_len=None): self._layer_norm = LayerNorm( normalized_shape=d_model, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(1.)), + initializer=fluid.initializer.Constant(1.0) + ), bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.))) + initializer=fluid.initializer.Constant(0.0) + ), + ) - def forward(self, prev_out, out, process_cmd, dropout_rate=0.): + def forward(self, prev_out, out, process_cmd, dropout_rate=0.0): for cmd in process_cmd: if cmd == "a": # add residual connection out = out + prev_out if prev_out is not None else out @@ -379,12 +418,12 @@ def forward(self, prev_out, out, process_cmd, dropout_rate=0.): out, dropout_prob=dropout_rate, seed=ModelHyperParams.dropout_seed, - is_test=False) + is_test=False, + ) return out class PositionwiseFeedForwardLayer(Layer): - def __init__(self, d_inner_hid, d_hid, dropout_rate): super(PositionwiseFeedForwardLayer, self).__init__() self._i2h = Linear(d_hid, d_inner_hid, act="relu") @@ -394,25 +433,28 @@ def __init__(self, d_inner_hid, d_hid, dropout_rate): def forward(self, x): hidden = self._i2h(x) if self._dropout_rate: - hidden = fluid.layers.dropout(hidden, - dropout_prob=self._dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) + hidden = fluid.layers.dropout( + hidden, + dropout_prob=self._dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False, + ) out = self._h2o(hidden) return out class MultiHeadAttentionLayer(Layer): - - def __init__(self, - d_key, - d_value, - d_model, - n_head=1, - dropout_rate=0., - cache=None, - gather_idx=None, - static_kv=False): + def __init__( + self, + d_key, + d_value, + d_model, + n_head=1, + dropout_rate=0.0, + cache=None, + gather_idx=None, + static_kv=False, + ): super(MultiHeadAttentionLayer, self).__init__() self._n_head = n_head self._d_key = d_key @@ -435,20 +477,25 @@ def forward(self, queries, keys, values, attn_bias): # split head reshaped_q = fluid.layers.reshape( - x=q, shape=[0, 0, self._n_head, self._d_key], inplace=False) + x=q, shape=[0, 0, self._n_head, self._d_key], inplace=False + ) transpose_q = fluid.layers.transpose(x=reshaped_q, perm=[0, 2, 1, 3]) reshaped_k = fluid.layers.reshape( - x=k, shape=[0, 0, self._n_head, self._d_key], inplace=False) + x=k, shape=[0, 0, self._n_head, self._d_key], inplace=False + ) transpose_k = fluid.layers.transpose(x=reshaped_k, perm=[0, 2, 1, 3]) reshaped_v = fluid.layers.reshape( - x=v, shape=[0, 0, self._n_head, self._d_value], inplace=False) + x=v, shape=[0, 0, self._n_head, self._d_value], inplace=False + ) transpose_v = fluid.layers.transpose(x=reshaped_v, perm=[0, 2, 1, 3]) # scale dot product attention - product = fluid.layers.matmul(x=transpose_q, - y=transpose_k, - transpose_y=True, - alpha=self._d_model**-0.5) + product = fluid.layers.matmul( + x=transpose_q, + y=transpose_k, + transpose_y=True, + alpha=self._d_model**-0.5, + ) if attn_bias is not None: product += attn_bias weights = fluid.layers.softmax(product) @@ -457,7 +504,8 @@ def forward(self, queries, keys, values, attn_bias): weights, dropout_prob=self._dropout_rate, seed=ModelHyperParams.dropout_seed, - is_test=False) + is_test=False, + ) out = fluid.layers.matmul(weights_droped, transpose_v) else: out = fluid.layers.matmul(weights, transpose_v) @@ -469,7 +517,8 @@ def forward(self, queries, keys, values, attn_bias): final_out = fluid.layers.reshape( x=trans_x, shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]], - inplace=False) + inplace=False, + ) # fc to output proj_out = self._proj_fc(final_out) @@ -477,118 +526,150 @@ def forward(self, queries, keys, values, attn_bias): class EncoderSubLayer(Layer): - - def __init__(self, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): + def __init__( + self, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + ): super(EncoderSubLayer, self).__init__() self._preprocess_cmd = preprocess_cmd self._postprocess_cmd = postprocess_cmd self._prepostprocess_dropout = prepostprocess_dropout - self._preprocess_layer = PrePostProcessLayer(d_model, - self._preprocess_cmd, 3) + self._preprocess_layer = PrePostProcessLayer( + d_model, self._preprocess_cmd, 3 + ) self._multihead_attention_layer = MultiHeadAttentionLayer( - d_key, d_value, d_model, n_head, attention_dropout) - self._postprocess_layer = PrePostProcessLayer(d_model, - self._postprocess_cmd, - None) - self._preprocess_layer2 = PrePostProcessLayer(d_model, - self._preprocess_cmd, 3) + d_key, d_value, d_model, n_head, attention_dropout + ) + self._postprocess_layer = PrePostProcessLayer( + d_model, self._postprocess_cmd, None + ) + self._preprocess_layer2 = PrePostProcessLayer( + d_model, self._preprocess_cmd, 3 + ) self._positionwise_feed_forward = PositionwiseFeedForwardLayer( - d_inner_hid, d_model, relu_dropout) - self._postprocess_layer2 = PrePostProcessLayer(d_model, - self._postprocess_cmd, - None) + d_inner_hid, d_model, relu_dropout + ) + self._postprocess_layer2 = PrePostProcessLayer( + d_model, self._postprocess_cmd, None + ) def forward(self, enc_input, attn_bias): pre_process_multihead = self._preprocess_layer( - None, enc_input, self._preprocess_cmd, self._prepostprocess_dropout) - attn_output = self._multihead_attention_layer(pre_process_multihead, - None, None, attn_bias) - attn_output = self._postprocess_layer(enc_input, attn_output, - self._postprocess_cmd, - self._prepostprocess_dropout) + None, enc_input, self._preprocess_cmd, self._prepostprocess_dropout + ) + attn_output = self._multihead_attention_layer( + pre_process_multihead, None, None, attn_bias + ) + attn_output = self._postprocess_layer( + enc_input, + attn_output, + self._postprocess_cmd, + self._prepostprocess_dropout, + ) pre_process2_output = self._preprocess_layer2( - None, attn_output, self._preprocess_cmd, - self._prepostprocess_dropout) + None, + attn_output, + self._preprocess_cmd, + self._prepostprocess_dropout, + ) ffd_output = self._positionwise_feed_forward(pre_process2_output) - return self._postprocess_layer2(attn_output, ffd_output, - self._postprocess_cmd, - self._prepostprocess_dropout) + return self._postprocess_layer2( + attn_output, + ffd_output, + self._postprocess_cmd, + self._prepostprocess_dropout, + ) class EncoderLayer(Layer): - - def __init__(self, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd="n", - postprocess_cmd="da"): + def __init__( + self, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd="n", + postprocess_cmd="da", + ): super(EncoderLayer, self).__init__() self._preprocess_cmd = preprocess_cmd self._encoder_sublayers = list() self._prepostprocess_dropout = prepostprocess_dropout self._n_layer = n_layer - self._preprocess_layer = PrePostProcessLayer(d_model, - self._preprocess_cmd, 3) + self._preprocess_layer = PrePostProcessLayer( + d_model, self._preprocess_cmd, 3 + ) for i in range(n_layer): self._encoder_sublayers.append( self.add_sublayer( 'esl_%d' % i, - EncoderSubLayer(n_head, d_key, d_value, d_model, - d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, - preprocess_cmd, postprocess_cmd))) + EncoderSubLayer( + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ), + ) + ) def forward(self, enc_input, attn_bias): for i in range(self._n_layer): enc_output = self._encoder_sublayers[i](enc_input, attn_bias) enc_input = enc_output - return self._preprocess_layer(None, enc_output, self._preprocess_cmd, - self._prepostprocess_dropout) + return self._preprocess_layer( + None, enc_output, self._preprocess_cmd, self._prepostprocess_dropout + ) class PrepareEncoderDecoderLayer(Layer): - - def __init__(self, - src_vocab_size, - src_emb_dim, - src_max_len, - dropout_rate, - is_sparse=False, - word_emb_param_name=None, - pos_enc_param_name=None): + def __init__( + self, + src_vocab_size, + src_emb_dim, + src_max_len, + dropout_rate, + is_sparse=False, + word_emb_param_name=None, + pos_enc_param_name=None, + ): super(PrepareEncoderDecoderLayer, self).__init__() self._src_max_len = src_max_len self._src_emb_dim = src_emb_dim self._src_vocab_size = src_vocab_size self._dropout_rate = dropout_rate - self._input_emb = Embedding(size=[src_vocab_size, src_emb_dim], - is_sparse=is_sparse, - padding_idx=0, - param_attr=fluid.ParamAttr( - name=word_emb_param_name, - initializer=fluid.initializer.Normal( - 0., src_emb_dim**-0.5))) + self._input_emb = Embedding( + size=[src_vocab_size, src_emb_dim], + is_sparse=is_sparse, + padding_idx=0, + param_attr=fluid.ParamAttr( + name=word_emb_param_name, + initializer=fluid.initializer.Normal(0.0, src_emb_dim**-0.5), + ), + ) if pos_enc_param_name is pos_enc_param_names[0]: pos_inp = pos_inp1 @@ -600,7 +681,9 @@ def __init__(self, param_attr=fluid.ParamAttr( name=pos_enc_param_name, initializer=fluid.initializer.NumpyArrayInitializer(pos_inp), - trainable=False)) + trainable=False, + ), + ) # use in dygraph_mode to fit different length batch # self._pos_emb._w = to_variable( @@ -608,37 +691,44 @@ def __init__(self, def forward(self, src_word, src_pos): src_word_emb = self._input_emb(src_word) - src_word_emb = fluid.layers.scale(x=src_word_emb, - scale=self._src_emb_dim**0.5) + src_word_emb = fluid.layers.scale( + x=src_word_emb, scale=self._src_emb_dim**0.5 + ) # # TODO change this to fit dynamic length input src_pos_emb = self._pos_emb(src_pos) src_pos_emb.stop_gradient = True enc_input = src_word_emb + src_pos_emb - return fluid.layers.dropout( - enc_input, - dropout_prob=self._dropout_rate, - seed=ModelHyperParams.dropout_seed, - is_test=False) if self._dropout_rate else enc_input + return ( + fluid.layers.dropout( + enc_input, + dropout_prob=self._dropout_rate, + seed=ModelHyperParams.dropout_seed, + is_test=False, + ) + if self._dropout_rate + else enc_input + ) class WrapEncoderLayer(Layer): - - def __init__(self, - src_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - is_sparse=False): + def __init__( + self, + src_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + is_sparse=False, + ): """ The wrapper assembles together all needed layers for the encoder. """ @@ -651,11 +741,21 @@ def __init__(self, prepostprocess_dropout, is_sparse=is_sparse, word_emb_param_name=word_emb_param_names[0], - pos_enc_param_name=pos_enc_param_names[0]) - self._encoder = EncoderLayer(n_layer, n_head, d_key, d_value, d_model, - d_inner_hid, prepostprocess_dropout, - attention_dropout, relu_dropout, - preprocess_cmd, postprocess_cmd) + pos_enc_param_name=pos_enc_param_names[0], + ) + self._encoder = EncoderLayer( + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + ) def forward(self, enc_inputs): src_word, src_pos, src_slf_attn_bias = enc_inputs @@ -665,26 +765,28 @@ def forward(self, enc_inputs): class DecoderSubLayer(Layer): - - def __init__(self, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - cache=None, - gather_idx=None): + def __init__( + self, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + cache=None, + gather_idx=None, + ): super(DecoderSubLayer, self).__init__() self._postprocess_cmd = postprocess_cmd self._preprocess_cmd = preprocess_cmd self._prepostprcess_dropout = prepostprocess_dropout - self._pre_process_layer = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + self._pre_process_layer = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._multihead_attention_layer = MultiHeadAttentionLayer( d_key, d_value, @@ -692,11 +794,14 @@ def __init__(self, n_head, attention_dropout, cache=cache, - gather_idx=gather_idx) - self._post_process_layer = PrePostProcessLayer(d_model, postprocess_cmd, - None) - self._pre_process_layer2 = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + gather_idx=gather_idx, + ) + self._post_process_layer = PrePostProcessLayer( + d_model, postprocess_cmd, None + ) + self._pre_process_layer2 = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._multihead_attention_layer2 = MultiHeadAttentionLayer( d_key, d_value, @@ -705,63 +810,86 @@ def __init__(self, attention_dropout, cache=cache, gather_idx=gather_idx, - static_kv=True) - self._post_process_layer2 = PrePostProcessLayer(d_model, - postprocess_cmd, None) - self._pre_process_layer3 = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + static_kv=True, + ) + self._post_process_layer2 = PrePostProcessLayer( + d_model, postprocess_cmd, None + ) + self._pre_process_layer3 = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._positionwise_feed_forward_layer = PositionwiseFeedForwardLayer( - d_inner_hid, d_model, relu_dropout) - self._post_process_layer3 = PrePostProcessLayer(d_model, - postprocess_cmd, None) + d_inner_hid, d_model, relu_dropout + ) + self._post_process_layer3 = PrePostProcessLayer( + d_model, postprocess_cmd, None + ) def forward(self, dec_input, enc_output, slf_attn_bias, dec_enc_attn_bias): - pre_process_rlt = self._pre_process_layer(None, dec_input, - self._preprocess_cmd, - self._prepostprcess_dropout) + pre_process_rlt = self._pre_process_layer( + None, dec_input, self._preprocess_cmd, self._prepostprcess_dropout + ) slf_attn_output = self._multihead_attention_layer( - pre_process_rlt, None, None, slf_attn_bias) + pre_process_rlt, None, None, slf_attn_bias + ) slf_attn_output_pp = self._post_process_layer( - dec_input, slf_attn_output, self._postprocess_cmd, - self._prepostprcess_dropout) - pre_process_rlt2 = self._pre_process_layer2(None, slf_attn_output_pp, - self._preprocess_cmd, - self._prepostprcess_dropout) + dec_input, + slf_attn_output, + self._postprocess_cmd, + self._prepostprcess_dropout, + ) + pre_process_rlt2 = self._pre_process_layer2( + None, + slf_attn_output_pp, + self._preprocess_cmd, + self._prepostprcess_dropout, + ) enc_attn_output_pp = self._multihead_attention_layer2( - pre_process_rlt2, enc_output, enc_output, dec_enc_attn_bias) - enc_attn_output = self._post_process_layer2(slf_attn_output_pp, - enc_attn_output_pp, - self._postprocess_cmd, - self._prepostprcess_dropout) - pre_process_rlt3 = self._pre_process_layer3(None, enc_attn_output, - self._preprocess_cmd, - self._prepostprcess_dropout) + pre_process_rlt2, enc_output, enc_output, dec_enc_attn_bias + ) + enc_attn_output = self._post_process_layer2( + slf_attn_output_pp, + enc_attn_output_pp, + self._postprocess_cmd, + self._prepostprcess_dropout, + ) + pre_process_rlt3 = self._pre_process_layer3( + None, + enc_attn_output, + self._preprocess_cmd, + self._prepostprcess_dropout, + ) ffd_output = self._positionwise_feed_forward_layer(pre_process_rlt3) - dec_output = self._post_process_layer3(enc_attn_output, ffd_output, - self._postprocess_cmd, - self._prepostprcess_dropout) + dec_output = self._post_process_layer3( + enc_attn_output, + ffd_output, + self._postprocess_cmd, + self._prepostprcess_dropout, + ) return dec_output class DecoderLayer(Layer): - - def __init__(self, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - caches=None, - gather_idx=None): + def __init__( + self, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + caches=None, + gather_idx=None, + ): super(DecoderLayer, self).__init__() - self._pre_process_layer = PrePostProcessLayer(d_model, preprocess_cmd, - 3) + self._pre_process_layer = PrePostProcessLayer( + d_model, preprocess_cmd, 3 + ) self._decoder_sub_layers = list() self._n_layer = n_layer self._preprocess_cmd = preprocess_cmd @@ -770,53 +898,62 @@ def __init__(self, self._decoder_sub_layers.append( self.add_sublayer( 'dsl_%d' % i, - DecoderSubLayer(n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - cache=None if caches is None else caches[i], - gather_idx=gather_idx))) - - def forward(self, dec_input, enc_output, dec_slf_attn_bias, - dec_enc_attn_bias): + DecoderSubLayer( + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + cache=None if caches is None else caches[i], + gather_idx=gather_idx, + ), + ) + ) + + def forward( + self, dec_input, enc_output, dec_slf_attn_bias, dec_enc_attn_bias + ): for i in range(self._n_layer): - tmp_dec_output = self._decoder_sub_layers[i](dec_input, enc_output, - dec_slf_attn_bias, - dec_enc_attn_bias) + tmp_dec_output = self._decoder_sub_layers[i]( + dec_input, enc_output, dec_slf_attn_bias, dec_enc_attn_bias + ) dec_input = tmp_dec_output - dec_output = self._pre_process_layer(None, tmp_dec_output, - self._preprocess_cmd, - self._prepostprocess_dropout) + dec_output = self._pre_process_layer( + None, + tmp_dec_output, + self._preprocess_cmd, + self._prepostprocess_dropout, + ) return dec_output class WrapDecoderLayer(Layer): - - def __init__(self, - trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - caches=None, - gather_idx=None, - is_sparse=False): + def __init__( + self, + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + caches=None, + gather_idx=None, + is_sparse=False, + ): """ The wrapper assembles together all needed layers for the encoder. """ @@ -829,20 +966,23 @@ def __init__(self, prepostprocess_dropout, is_sparse=is_sparse, word_emb_param_name=word_emb_param_names[1], - pos_enc_param_name=pos_enc_param_names[1]) - self._decoder_layer = DecoderLayer(n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - caches=caches, - gather_idx=gather_idx) + pos_enc_param_name=pos_enc_param_names[1], + ) + self._decoder_layer = DecoderLayer( + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + caches=caches, + gather_idx=gather_idx, + ) self._weight_sharing = weight_sharing if not weight_sharing: self._fc = Linear(d_model, trg_vocab_size, bias_attr=False) @@ -850,17 +990,20 @@ def __init__(self, def forward(self, dec_inputs=None, enc_output=None): trg_word, trg_pos, trg_slf_attn_bias, trg_src_attn_bias = dec_inputs dec_input = self._prepare_decoder_layer(trg_word, trg_pos) - dec_output = self._decoder_layer(dec_input, enc_output, - trg_slf_attn_bias, trg_src_attn_bias) + dec_output = self._decoder_layer( + dec_input, enc_output, trg_slf_attn_bias, trg_src_attn_bias + ) dec_output_reshape = fluid.layers.reshape( - dec_output, shape=[-1, dec_output.shape[-1]], inplace=False) + dec_output, shape=[-1, dec_output.shape[-1]], inplace=False + ) if self._weight_sharing: predict = fluid.layers.matmul( x=dec_output_reshape, y=self._prepare_decoder_layer._input_emb.weight, - transpose_y=True) + transpose_y=True, + ) else: predict = self._fc(dec_output_reshape) @@ -872,81 +1015,91 @@ def forward(self, dec_inputs=None, enc_output=None): class TransFormer(Layer): - - def __init__(self, - src_vocab_size, - trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - label_smooth_eps, - use_py_reader=False, - is_test=False, - is_sparse=False): + def __init__( + self, + src_vocab_size, + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + label_smooth_eps, + use_py_reader=False, + is_test=False, + is_sparse=False, + ): super(TransFormer, self).__init__() self._label_smooth_eps = label_smooth_eps self._trg_vocab_size = trg_vocab_size if weight_sharing: - assert src_vocab_size == trg_vocab_size, ( - "Vocabularies in source and target should be same for weight sharing." - ) - self._wrap_encoder_layer = WrapEncoderLayer(src_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - is_sparse=is_sparse) - self._wrap_decoder_layer = WrapDecoderLayer(trg_vocab_size, - max_length, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - prepostprocess_dropout, - attention_dropout, - relu_dropout, - preprocess_cmd, - postprocess_cmd, - weight_sharing, - is_sparse=is_sparse) + assert ( + src_vocab_size == trg_vocab_size + ), "Vocabularies in source and target should be same for weight sharing." + self._wrap_encoder_layer = WrapEncoderLayer( + src_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + is_sparse=is_sparse, + ) + self._wrap_decoder_layer = WrapDecoderLayer( + trg_vocab_size, + max_length, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + prepostprocess_dropout, + attention_dropout, + relu_dropout, + preprocess_cmd, + postprocess_cmd, + weight_sharing, + is_sparse=is_sparse, + ) if weight_sharing: - self._wrap_decoder_layer._prepare_decoder_layer._input_emb.weight = self._wrap_encoder_layer._prepare_encoder_layer._input_emb.weight + self._wrap_decoder_layer._prepare_decoder_layer._input_emb.weight = ( + self._wrap_encoder_layer._prepare_encoder_layer._input_emb.weight + ) def forward(self, enc_inputs, dec_inputs, label, weights): enc_output = self._wrap_encoder_layer(enc_inputs) predict = self._wrap_decoder_layer(dec_inputs, enc_output) if self._label_smooth_eps: label_out = fluid.layers.label_smooth( - label=fluid.layers.one_hot(input=label, - depth=self._trg_vocab_size), - epsilon=self._label_smooth_eps) + label=fluid.layers.one_hot( + input=label, depth=self._trg_vocab_size + ), + epsilon=self._label_smooth_eps, + ) cost = fluid.layers.softmax_with_cross_entropy( logits=predict, label=label_out, - soft_label=True if self._label_smooth_eps else False) + soft_label=True if self._label_smooth_eps else False, + ) weighted_cost = cost * weights sum_cost = fluid.layers.reduce_sum(weighted_cost) token_num = fluid.layers.reduce_sum(weights) @@ -956,7 +1109,6 @@ def forward(self, enc_inputs, dec_inputs, label, weights): class TestDygraphTransformerSortGradient(unittest.TestCase): - def test_transformer_sort_gradient(self): for is_sparse in [True, False]: self.transformer_sort_gradient_float32(is_sparse) @@ -969,28 +1121,31 @@ def run_dygraph(): fluid.set_flags({'FLAGS_new_executor_use_inplace': False}) paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - transformer = TransFormer(ModelHyperParams.src_vocab_size, - ModelHyperParams.trg_vocab_size, - ModelHyperParams.max_length + 1, - ModelHyperParams.n_layer, - ModelHyperParams.n_head, - ModelHyperParams.d_key, - ModelHyperParams.d_value, - ModelHyperParams.d_model, - ModelHyperParams.d_inner_hid, - ModelHyperParams.prepostprocess_dropout, - ModelHyperParams.attention_dropout, - ModelHyperParams.relu_dropout, - ModelHyperParams.preprocess_cmd, - ModelHyperParams.postprocess_cmd, - ModelHyperParams.weight_sharing, - TrainTaskConfig.label_smooth_eps, - use_py_reader=use_py_reader, - is_test=False, - is_sparse=is_sparse) + transformer = TransFormer( + ModelHyperParams.src_vocab_size, + ModelHyperParams.trg_vocab_size, + ModelHyperParams.max_length + 1, + ModelHyperParams.n_layer, + ModelHyperParams.n_head, + ModelHyperParams.d_key, + ModelHyperParams.d_value, + ModelHyperParams.d_model, + ModelHyperParams.d_inner_hid, + ModelHyperParams.prepostprocess_dropout, + ModelHyperParams.attention_dropout, + ModelHyperParams.relu_dropout, + ModelHyperParams.preprocess_cmd, + ModelHyperParams.postprocess_cmd, + ModelHyperParams.weight_sharing, + TrainTaskConfig.label_smooth_eps, + use_py_reader=use_py_reader, + is_test=False, + is_sparse=is_sparse, + ) if sync: lr_decay = fluid.layers.learning_rate_scheduler.noam_decay( - ModelHyperParams.d_model, TrainTaskConfig.warmup_steps) + ModelHyperParams.d_model, TrainTaskConfig.warmup_steps + ) with fluid.default_main_program()._lr_schedule_guard(): learning_rate = lr_decay * TrainTaskConfig.learning_rate optimizer = fluid.optimizer.Adam( @@ -998,11 +1153,12 @@ def run_dygraph(): beta1=TrainTaskConfig.beta1, beta2=TrainTaskConfig.beta2, epsilon=TrainTaskConfig.eps, - parameter_list=transformer.parameters()) + parameter_list=transformer.parameters(), + ) else: optimizer = fluid.optimizer.SGD( - learning_rate=0.003, - parameter_list=transformer.parameters()) + learning_rate=0.003, parameter_list=transformer.parameters() + ) dy_param_init = dict() dy_param_updated = dict() @@ -1013,20 +1169,23 @@ def run_dygraph(): enc_inputs, dec_inputs, label, weights = create_data() if False: outs, traced_layer = TracedLayer.trace( - transformer, [enc_inputs, dec_inputs, label, weights]) + transformer, [enc_inputs, dec_inputs, label, weights] + ) ins_static = enc_inputs + dec_inputs + [label, weights] outs_static = traced_layer(ins_static) helper.assertEachVar(outs, outs_static) if program is not None: self.assertTrue( - is_equal_program(program, traced_layer.program)) + is_equal_program(program, traced_layer.program) + ) program = traced_layer.program traced_layer.save_inference_model( './infer_imperative_transformer', feed=list(range(len(ins_static))), - fetch=list(range(len(outs_static)))) + fetch=list(range(len(outs_static))), + ) else: outs = transformer(enc_inputs, dec_inputs, label, weights) @@ -1049,61 +1208,88 @@ def run_dygraph(): dy_predict_value = dy_predict.numpy() dy_token_num_value = dy_token_num.numpy() - return dy_avg_cost_value, dy_sum_cost_value, dy_predict_value, dy_token_num_value, \ - dy_param_init, dy_param_updated + return ( + dy_avg_cost_value, + dy_sum_cost_value, + dy_predict_value, + dy_token_num_value, + dy_param_init, + dy_param_updated, + ) with guard(): fluid.set_flags({'FLAGS_sort_sum_gradient': True}) if _in_legacy_dygraph(): - dy_avg_cost_value, dy_sum_cost_value, dy_predict_value, dy_token_num_value, \ - dy_param_init, dy_param_updated = run_dygraph() + ( + dy_avg_cost_value, + dy_sum_cost_value, + dy_predict_value, + dy_token_num_value, + dy_param_init, + dy_param_updated, + ) = run_dygraph() with new_program_scope(): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) - transformer = TransFormer(ModelHyperParams.src_vocab_size, - ModelHyperParams.trg_vocab_size, - ModelHyperParams.max_length + 1, - ModelHyperParams.n_layer, - ModelHyperParams.n_head, - ModelHyperParams.d_key, - ModelHyperParams.d_value, - ModelHyperParams.d_model, - ModelHyperParams.d_inner_hid, - ModelHyperParams.prepostprocess_dropout, - ModelHyperParams.attention_dropout, - ModelHyperParams.relu_dropout, - ModelHyperParams.preprocess_cmd, - ModelHyperParams.postprocess_cmd, - ModelHyperParams.weight_sharing, - TrainTaskConfig.label_smooth_eps, - use_py_reader=use_py_reader, - is_test=False, - is_sparse=is_sparse) - exe = fluid.Executor(fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + transformer = TransFormer( + ModelHyperParams.src_vocab_size, + ModelHyperParams.trg_vocab_size, + ModelHyperParams.max_length + 1, + ModelHyperParams.n_layer, + ModelHyperParams.n_head, + ModelHyperParams.d_key, + ModelHyperParams.d_value, + ModelHyperParams.d_model, + ModelHyperParams.d_inner_hid, + ModelHyperParams.prepostprocess_dropout, + ModelHyperParams.attention_dropout, + ModelHyperParams.relu_dropout, + ModelHyperParams.preprocess_cmd, + ModelHyperParams.postprocess_cmd, + ModelHyperParams.weight_sharing, + TrainTaskConfig.label_smooth_eps, + use_py_reader=use_py_reader, + is_test=False, + is_sparse=is_sparse, + ) + exe = fluid.Executor( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) optimizer = fluid.optimizer.SGD(learning_rate=0.003) - data_input_names = encoder_data_input_fields + decoder_data_input_fields[: - -1] + label_data_input_fields + data_input_names = ( + encoder_data_input_fields + + decoder_data_input_fields[:-1] + + label_data_input_fields + ) all_inputs = make_all_inputs(data_input_names) enc_inputs_len = len(encoder_data_input_fields) dec_inputs_len = len(decoder_data_input_fields[:-1]) enc_inputs = all_inputs[0:enc_inputs_len] - dec_inputs = all_inputs[enc_inputs_len:enc_inputs_len + - dec_inputs_len] + dec_inputs = all_inputs[ + enc_inputs_len : enc_inputs_len + dec_inputs_len + ] label = all_inputs[-2] weights = all_inputs[-1] static_param_updated = dict() static_param_init = dict() static_param_name_list = list() - static_sum_cost, static_avg_cost, static_predict, static_token_num = transformer( - enc_inputs, dec_inputs, label, weights) + ( + static_sum_cost, + static_avg_cost, + static_predict, + static_token_num, + ) = transformer(enc_inputs, dec_inputs, label, weights) optimizer.minimize(static_avg_cost) for param in transformer.parameters(): static_param_name_list.append(param.name) - out = exe.run(fluid.default_startup_program(), - fetch_list=static_param_name_list) + out = exe.run( + fluid.default_startup_program(), + fetch_list=static_param_name_list, + ) for i in range(len(static_param_name_list)): static_param_init[static_param_name_list[i]] = out[i] static_sum_cost_value = None @@ -1113,31 +1299,40 @@ def run_dygraph(): for i in range(batch_num): feed_dict = create_feed_dict_list(create_data(True)) fetch_list = [ - static_sum_cost, static_avg_cost, static_predict, - static_token_num + static_sum_cost, + static_avg_cost, + static_predict, + static_token_num, ] fetch_list.extend(static_param_name_list) - out = exe.run(fluid.default_main_program(), - feed=feed_dict, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed=feed_dict, + fetch_list=fetch_list, + ) static_sum_cost_value = out[0] static_avg_cost_value = out[1] static_predict_value = out[2] static_token_num_value = out[3] if i == batch_num - 1: for k in range(4, len(out)): - static_param_updated[static_param_name_list[k - - 4]] = out[k] + static_param_updated[ + static_param_name_list[k - 4] + ] = out[k] if _in_legacy_dygraph(): - np.testing.assert_array_equal(static_avg_cost_value, - dy_avg_cost_value) - np.testing.assert_array_equal(static_sum_cost_value, - dy_sum_cost_value) - np.testing.assert_array_equal(static_predict_value, - dy_predict_value) - np.testing.assert_array_equal(static_token_num_value, - dy_token_num_value) + np.testing.assert_array_equal( + static_avg_cost_value, dy_avg_cost_value + ) + np.testing.assert_array_equal( + static_sum_cost_value, dy_sum_cost_value + ) + np.testing.assert_array_equal( + static_predict_value, dy_predict_value + ) + np.testing.assert_array_equal( + static_token_num_value, dy_token_num_value + ) for key, value in static_param_init.items(): np.testing.assert_array_equal(value, dy_param_init[key]) @@ -1147,33 +1342,45 @@ def run_dygraph(): # compare eager result with imperative result with guard(): fluid.set_flags({'FLAGS_sort_sum_gradient': False}) - dy_avg_cost_value, dy_sum_cost_value, dy_predict_value, dy_token_num_value, \ - dy_param_init, dy_param_updated = run_dygraph() + ( + dy_avg_cost_value, + dy_sum_cost_value, + dy_predict_value, + dy_token_num_value, + dy_param_init, + dy_param_updated, + ) = run_dygraph() with guard(): with _test_eager_guard(): - eager_avg_cost_value, eager_sum_cost_value, eager_predict_value, eager_token_num_value, \ - eager_param_init, eager_param_updated = run_dygraph() - np.testing.assert_allclose(dy_avg_cost_value, - eager_avg_cost_value, - rtol=1e-05) - np.testing.assert_allclose(dy_sum_cost_value, - eager_sum_cost_value, - rtol=1e-05) - - np.testing.assert_allclose(dy_predict_value, - eager_predict_value, - rtol=1e-05) - np.testing.assert_allclose(dy_token_num_value, - eager_token_num_value, - rtol=1e-05) + ( + eager_avg_cost_value, + eager_sum_cost_value, + eager_predict_value, + eager_token_num_value, + eager_param_init, + eager_param_updated, + ) = run_dygraph() + np.testing.assert_allclose( + dy_avg_cost_value, eager_avg_cost_value, rtol=1e-05 + ) + np.testing.assert_allclose( + dy_sum_cost_value, eager_sum_cost_value, rtol=1e-05 + ) + + np.testing.assert_allclose( + dy_predict_value, eager_predict_value, rtol=1e-05 + ) + np.testing.assert_allclose( + dy_token_num_value, eager_token_num_value, rtol=1e-05 + ) for key, value in static_param_init.items(): np.testing.assert_array_equal(value, eager_param_init[key]) for key, value in dy_param_updated.items(): - np.testing.assert_allclose(value, - eager_param_updated[key], - rtol=1e-05) + np.testing.assert_allclose( + value, eager_param_updated[key], rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_triple_grad.py b/python/paddle/fluid/tests/unittests/test_imperative_triple_grad.py index 7f0ea807271be..ebebe754b2176 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_triple_grad.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_triple_grad.py @@ -22,7 +22,6 @@ def _dygraph_guard_(func): - def __impl__(*args, **kwargs): if fluid._non_static_mode(): return func(*args, **kwargs) @@ -43,36 +42,37 @@ def random_var(size, low=-1, high=1, dtype='float32'): class TestDygraphTripleGradMatmul(TestCase): - def test_matmul_triple_grad(self): input_numpy = np.ones([3, 3]) * 2 with _test_eager_guard(): - x = paddle.to_tensor(input_numpy, - stop_gradient=False, - dtype='float32') - y = paddle.to_tensor(input_numpy, - stop_gradient=False, - dtype='float32') + x = paddle.to_tensor( + input_numpy, stop_gradient=False, dtype='float32' + ) + y = paddle.to_tensor( + input_numpy, stop_gradient=False, dtype='float32' + ) out = paddle.matmul(x, y, False, False) - new_out_g = paddle.to_tensor(np.ones([3, 3]), - stop_gradient=False, - dtype='float32') - new_x_g, new_y_g = paddle.grad([out], [x, y], [new_out_g], - retain_graph=True, - create_graph=True) - - new_x_g_g = paddle.to_tensor(np.ones([3, 3]), - stop_gradient=False, - dtype='float32') - new_y_g_g = paddle.to_tensor(np.ones([3, 3]), - stop_gradient=False, - dtype='float32') - new_a, new_b, new_c = paddle.grad([new_x_g, new_y_g], - [x, y, new_out_g], - [new_x_g_g, new_y_g_g], - retain_graph=True, - create_graph=True) + new_out_g = paddle.to_tensor( + np.ones([3, 3]), stop_gradient=False, dtype='float32' + ) + new_x_g, new_y_g = paddle.grad( + [out], [x, y], [new_out_g], retain_graph=True, create_graph=True + ) + + new_x_g_g = paddle.to_tensor( + np.ones([3, 3]), stop_gradient=False, dtype='float32' + ) + new_y_g_g = paddle.to_tensor( + np.ones([3, 3]), stop_gradient=False, dtype='float32' + ) + new_a, new_b, new_c = paddle.grad( + [new_x_g, new_y_g], + [x, y, new_out_g], + [new_x_g_g, new_y_g_g], + retain_graph=True, + create_graph=True, + ) new_a.backward() @@ -108,27 +108,30 @@ def test_matmul_triple_grad(self): class TestDygraphTripleGrad(TestCase): - def setUp(self): self.sort_sum_gradient = False self.shape = [5, 5] - def grad(self, - outputs, - inputs, - grad_outputs=None, - no_grad_vars=None, - retain_graph=None, - create_graph=False, - allow_unused=False): + def grad( + self, + outputs, + inputs, + grad_outputs=None, + no_grad_vars=None, + retain_graph=None, + create_graph=False, + allow_unused=False, + ): fluid.set_flags({'FLAGS_sort_sum_gradient': self.sort_sum_gradient}) - return fluid.dygraph.grad(outputs=outputs, - inputs=inputs, - grad_outputs=grad_outputs, - no_grad_vars=no_grad_vars, - retain_graph=retain_graph, - create_graph=create_graph, - allow_unused=allow_unused) + return fluid.dygraph.grad( + outputs=outputs, + inputs=inputs, + grad_outputs=grad_outputs, + no_grad_vars=no_grad_vars, + retain_graph=retain_graph, + create_graph=create_graph, + allow_unused=allow_unused, + ) @dygraph_guard def func_exception(self): @@ -150,12 +153,16 @@ def func_exception(self): self.grad([random_var(shape)], [1]) with self.assertRaises(AssertionError): - self.grad([random_var(shape), random_var(shape)], - [random_var(shape)], [random_var(shape)]) + self.grad( + [random_var(shape), random_var(shape)], + [random_var(shape)], + [random_var(shape)], + ) with self.assertRaises(AssertionError): - self.grad([random_var(shape)], [random_var(shape)], - no_grad_vars=[1]) + self.grad( + [random_var(shape)], [random_var(shape)], no_grad_vars=[1] + ) with self.assertRaises(AssertionError): self.grad([random_var(shape)], [random_var(shape)], no_grad_vars=1) @@ -178,34 +185,41 @@ def func_example_with_gradient_and_create_graph(self): out = fluid.layers.sigmoid(paddle.matmul(x, y) + z) out_np = out.numpy() - dx_actual, = self.grad([out], [x], create_graph=True) + (dx_actual,) = self.grad([out], [x], create_graph=True) # Theoritical result based on math calculation dout = np.ones(self.shape).astype('float32') - dx_expected = np.matmul(dout * out_np * (1 - out_np), - np.transpose(y_np)) + dx_expected = np.matmul( + dout * out_np * (1 - out_np), np.transpose(y_np) + ) np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) - ddx_actual, = self.grad([dx_actual], [x], create_graph=True) + (ddx_actual,) = self.grad([dx_actual], [x], create_graph=True) # Theoritical result based on math calculation DDY = np.zeros(self.shape).astype('float32') DDX = np.ones(self.shape).astype('float32') - double_grad_tmp1 = np.matmul(dout * out_np * (1 - out_np), - np.transpose(DDY)) + double_grad_tmp1 = np.matmul( + dout * out_np * (1 - out_np), np.transpose(DDY) + ) double_grad_tmp2 = np.matmul(DDX, y_np) + np.matmul(x_np, DDY) double_grad_tmp3 = ( - 1 - 2 * out_np) * dout * double_grad_tmp2 * out_np * (1 - out_np) - ddx_expected = double_grad_tmp1 + np.matmul(double_grad_tmp3, - np.transpose(y_np)) + (1 - 2 * out_np) * dout * double_grad_tmp2 * out_np * (1 - out_np) + ) + ddx_expected = double_grad_tmp1 + np.matmul( + double_grad_tmp3, np.transpose(y_np) + ) np.testing.assert_allclose(ddx_actual.numpy(), ddx_expected, rtol=1e-05) # Theoritical result based on math calculation d_ddout = np.zeros(self.shape).astype('float32') tmp0 = np.matmul(DDX, y_np) + np.matmul(x_np, DDY) tmp1 = (1 - 2 * out_np) * ((1 - 2 * out_np) * dout * tmp0 * tmp0) - tmp2 = tmp0 * (1 - 2 * out_np) * d_ddout - 2 * dout * ( - 1 - out_np) * out_np * tmp0 * tmp0 - dddx_expected = np.matmul(((tmp1 + tmp2) * out_np * (1 - out_np)), - np.transpose(y_np)) + tmp2 = ( + tmp0 * (1 - 2 * out_np) * d_ddout + - 2 * dout * (1 - out_np) * out_np * tmp0 * tmp0 + ) + dddx_expected = np.matmul( + ((tmp1 + tmp2) * out_np * (1 - out_np)), np.transpose(y_np) + ) ddx_actual.backward() dddx_grad_actual = x.gradient() @@ -222,29 +236,32 @@ def test_all_cases(self): class TestDygraphTripleGradBradcastCase(TestCase): - def setUp(self): self.sort_sum_gradient = False self.x_shape = [3, 2, 2] self.y_shape = [1, 2, 2] self.z_shape = [2, 2] - def grad(self, - outputs, - inputs, - grad_outputs=None, - no_grad_vars=None, - retain_graph=None, - create_graph=False, - allow_unused=False): + def grad( + self, + outputs, + inputs, + grad_outputs=None, + no_grad_vars=None, + retain_graph=None, + create_graph=False, + allow_unused=False, + ): fluid.set_flags({'FLAGS_sort_sum_gradient': self.sort_sum_gradient}) - return fluid.dygraph.grad(outputs=outputs, - inputs=inputs, - grad_outputs=grad_outputs, - no_grad_vars=no_grad_vars, - retain_graph=retain_graph, - create_graph=create_graph, - allow_unused=allow_unused) + return fluid.dygraph.grad( + outputs=outputs, + inputs=inputs, + grad_outputs=grad_outputs, + no_grad_vars=no_grad_vars, + retain_graph=retain_graph, + create_graph=create_graph, + allow_unused=allow_unused, + ) @dygraph_guard def func_example_with_gradient_and_create_graph(self): @@ -264,34 +281,42 @@ def func_example_with_gradient_and_create_graph(self): out = fluid.layers.sigmoid(paddle.matmul(x, y) + z) out_np = out.numpy() - dx_actual, = self.grad([out], [x], create_graph=True) + (dx_actual,) = self.grad([out], [x], create_graph=True) # Theoritical result based on math calculation dout = np.ones(self.x_shape).astype('float32') - dx_expected = np.matmul(dout * out_np * (1 - out_np), - np.transpose(y_np, axes=(0, 2, 1))) + dx_expected = np.matmul( + dout * out_np * (1 - out_np), np.transpose(y_np, axes=(0, 2, 1)) + ) np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) - ddx_actual, = self.grad([dx_actual], [x], create_graph=True) + (ddx_actual,) = self.grad([dx_actual], [x], create_graph=True) # Theoritical result based on math calculation DDY = np.zeros(self.y_shape).astype('float32') DDX = np.ones(self.x_shape).astype('float32') - double_grad_tmp1 = np.matmul(dout * out_np * (1 - out_np), - np.transpose(DDY, axes=(0, 2, 1))) + double_grad_tmp1 = np.matmul( + dout * out_np * (1 - out_np), np.transpose(DDY, axes=(0, 2, 1)) + ) double_grad_tmp2 = np.matmul(DDX, y_np) + np.matmul(x_np, DDY) double_grad_tmp3 = ( - 1 - 2 * out_np) * dout * double_grad_tmp2 * out_np * (1 - out_np) + (1 - 2 * out_np) * dout * double_grad_tmp2 * out_np * (1 - out_np) + ) ddx_expected = double_grad_tmp1 + np.matmul( - double_grad_tmp3, np.transpose(y_np, axes=(0, 2, 1))) + double_grad_tmp3, np.transpose(y_np, axes=(0, 2, 1)) + ) np.testing.assert_allclose(ddx_actual.numpy(), ddx_expected, rtol=1e-05) # Theoritical result based on math calculation d_ddout = np.zeros(self.x_shape).astype('float32') tmp0 = np.matmul(DDX, y_np) + np.matmul(x_np, DDY) tmp1 = (1 - 2 * out_np) * ((1 - 2 * out_np) * dout * tmp0 * tmp0) - tmp2 = tmp0 * (1 - 2 * out_np) * d_ddout - 2 * dout * ( - 1 - out_np) * out_np * tmp0 * tmp0 - dddx_expected = np.matmul(((tmp1 + tmp2) * out_np * (1 - out_np)), - np.transpose(y_np, axes=(0, 2, 1))) + tmp2 = ( + tmp0 * (1 - 2 * out_np) * d_ddout + - 2 * dout * (1 - out_np) * out_np * tmp0 * tmp0 + ) + dddx_expected = np.matmul( + ((tmp1 + tmp2) * out_np * (1 - out_np)), + np.transpose(y_np, axes=(0, 2, 1)), + ) ddx_actual.backward() dddx_grad_actual = x.gradient() diff --git a/python/paddle/fluid/tests/unittests/test_imperative_using_non_zero_gpu.py b/python/paddle/fluid/tests/unittests/test_imperative_using_non_zero_gpu.py index 2cc157ae050cf..9d7ece633604d 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_using_non_zero_gpu.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_using_non_zero_gpu.py @@ -21,7 +21,6 @@ class TestImperativeUsingNonZeroGpu(unittest.TestCase): - def run_main(self, np_arr, place): with guard(place): var = to_variable(np_arr) diff --git a/python/paddle/fluid/tests/unittests/test_increment.py b/python/paddle/fluid/tests/unittests/test_increment.py index 34c7af4ac081e..4e435e55794e6 100755 --- a/python/paddle/fluid/tests/unittests/test_increment.py +++ b/python/paddle/fluid/tests/unittests/test_increment.py @@ -20,12 +20,11 @@ class TestIncrement(unittest.TestCase): - def test_api(self): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.layers.fill_constant(shape=[1], - dtype='int64', - value=5) + input = fluid.layers.fill_constant( + shape=[1], dtype='int64', value=5 + ) expected_result = np.array([8], dtype='int64') output = paddle.tensor.math.increment(input, value=3) @@ -41,7 +40,6 @@ def test_api(self): class TestInplaceApiWithDataTransform(unittest.TestCase): - def test_increment(self): if fluid.core.is_compiled_with_cuda(): paddle.enable_static() @@ -50,7 +48,7 @@ def test_increment(self): with paddle.fluid.device_guard("cpu"): x = paddle.increment(x) exe = paddle.static.Executor(paddle.CUDAPlace(0)) - a, = exe.run(paddle.static.default_main_program(), fetch_list=[x]) + (a,) = exe.run(paddle.static.default_main_program(), fetch_list=[x]) paddle.disable_static() self.assertEqual(a[0], 1) diff --git a/python/paddle/fluid/tests/unittests/test_index_add_op.py b/python/paddle/fluid/tests/unittests/test_index_add_op.py index f3b799f0d3501..c54479b352968 100644 --- a/python/paddle/fluid/tests/unittests/test_index_add_op.py +++ b/python/paddle/fluid/tests/unittests/test_index_add_op.py @@ -19,8 +19,9 @@ from paddle.fluid import Program -def compute_index_add_ref(axis, x_shape, x_np, add_value_shape, add_value_np, - index_size, index_np): +def compute_index_add_ref( + axis, x_shape, x_np, add_value_shape, add_value_np, index_size, index_np +): if axis < 0: axis = axis + len(x_shape) if axis != 0: @@ -28,11 +29,13 @@ def compute_index_add_ref(axis, x_shape, x_np, add_value_shape, add_value_np, x_reshape = [outer_loop] + list(x_shape[axis:]) x_np_reshape = np.reshape(x_np, tuple(x_reshape)) - add_value_reshape = [np.prod(add_value_shape[:axis]).astype(int) - ] + list(add_value_shape[axis:]) + add_value_reshape = [ + np.prod(add_value_shape[:axis]).astype(int) + ] + list(add_value_shape[axis:]) - add_value_np_reshape = np.reshape(add_value_np, - tuple(add_value_reshape)) + add_value_np_reshape = np.reshape( + add_value_np, tuple(add_value_reshape) + ) else: x_np_reshape = x_np add_value_np_reshape = add_value_np @@ -54,23 +57,29 @@ def raw_index_add(x, index, value, axis): class TestIndexAddOp(OpTest): - def setUp(self): self.python_api = raw_index_add self.op_type = "index_add" self.init_dtype_type() - index_np = np.random.randint(low=0, - high=self.x_shape[self.axis], - size=self.index_size) + index_np = np.random.randint( + low=0, high=self.x_shape[self.axis], size=self.index_size + ) x_np = np.random.random(self.x_shape).astype(self.x_type) add_value_np = np.random.random(self.add_value_shape).astype( - self.x_type) + self.x_type + ) self.inputs = {'X': x_np, 'Index': index_np, 'AddValue': add_value_np} self.attrs = {'axis': self.axis} - out = compute_index_add_ref(self.axis, self.x_shape, x_np, - self.add_value_shape, add_value_np, - self.index_size, index_np) + out = compute_index_add_ref( + self.axis, + self.x_shape, + x_np, + self.add_value_shape, + add_value_np, + self.index_size, + index_np, + ) self.outputs = {'Out': out} def init_dtype_type(self): @@ -89,7 +98,6 @@ def test_check_grad_normal(self): class TestIndexAddAPI(unittest.TestCase): - def setUp(self): self.setType() self.setPlace() @@ -126,11 +134,11 @@ def generate_input_data(self): self.x_np = np.random.random(self.x_shape).astype(self.x_type) self.add_value_np = np.random.random(self.add_value_shape).astype( - self.x_type) - self.index_np = np.random.randint(low=0, - high=self.x_shape[axis], - size=self.index_size).astype( - self.index_type) + self.x_type + ) + self.index_np = np.random.randint( + low=0, high=self.x_shape[axis], size=self.index_size + ).astype(self.index_type) if self.check_backward: self.dout_np = np.random.random(self.x_shape).astype(self.x_type) @@ -154,36 +162,47 @@ def run_imperative(self, device): add_value = paddle.to_tensor(self.add_value_np, stop_gradient=False) out = paddle.index_add(input_tensor, index, self.axis, add_value) - ref_out = compute_index_add_ref(self.axis, self.x_shape, self.x_np, - self.add_value_shape, self.add_value_np, - self.index_size, self.index_np) - np.testing.assert_allclose(ref_out, - out.numpy(), - rtol=self.rtol, - atol=self.atol) + ref_out = compute_index_add_ref( + self.axis, + self.x_shape, + self.x_np, + self.add_value_shape, + self.add_value_np, + self.index_size, + self.index_np, + ) + np.testing.assert_allclose( + ref_out, out.numpy(), rtol=self.rtol, atol=self.atol + ) if self.check_backward: dout_tensor = paddle.to_tensor(self.dout_np) paddle.autograd.backward([out], [dout_tensor], retain_graph=True) - ref_x_grad, ref_add_value_grad = self.compute_index_add_backward_ref( + ( + ref_x_grad, + ref_add_value_grad, + ) = self.compute_index_add_backward_ref() + np.testing.assert_allclose( + ref_x_grad, + input_tensor.grad.numpy(), + rtol=self.rtol, + atol=self.atol, + ) + np.testing.assert_allclose( + ref_add_value_grad, + add_value.grad.numpy(), + rtol=self.rtol, + atol=self.atol, ) - np.testing.assert_allclose(ref_x_grad, - input_tensor.grad.numpy(), - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(ref_add_value_grad, - add_value.grad.numpy(), - rtol=self.rtol, - atol=self.atol) def run_static(self, device): x = paddle.static.data(name='X', shape=self.x_shape, dtype=self.x_type) - index = paddle.static.data(name='Index', - shape=self.index_shape, - dtype=self.index_type) - add_value = paddle.static.data(name='AddValue', - shape=self.add_value_shape, - dtype=self.x_type) + index = paddle.static.data( + name='Index', shape=self.index_shape, dtype=self.index_type + ) + add_value = paddle.static.data( + name='AddValue', shape=self.add_value_shape, dtype=self.x_type + ) out = paddle.index_add(x, index, self.axis, add_value) @@ -193,19 +212,22 @@ def run_static(self, device): place = paddle.CUDAPlace(0) else: raise TypeError( - "paddle.index_add api only support cpu and gpu device now.") + "paddle.index_add api only support cpu and gpu device now." + ) exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) - res = exe.run(paddle.static.default_main_program(), - feed={ - "X": self.x_np, - "Index": self.index_np, - "AddValue": self.add_value_np, - }, - fetch_list=[out.name], - return_numpy=False) + res = exe.run( + paddle.static.default_main_program(), + feed={ + "X": self.x_np, + "Index": self.index_np, + "AddValue": self.add_value_np, + }, + fetch_list=[out.name], + return_numpy=False, + ) return res def test_static(self): @@ -213,14 +235,18 @@ def test_static(self): for device in self.place: with paddle.static.program_guard(Program()): out = self.run_static(device) - ref_out = compute_index_add_ref(self.axis, self.x_shape, self.x_np, - self.add_value_shape, - self.add_value_np, self.index_size, - self.index_np) - np.testing.assert_allclose(ref_out, - np.array(out[0]), - rtol=self.rtol, - atol=self.atol) + ref_out = compute_index_add_ref( + self.axis, + self.x_shape, + self.x_np, + self.add_value_shape, + self.add_value_np, + self.index_size, + self.index_np, + ) + np.testing.assert_allclose( + ref_out, np.array(out[0]), rtol=self.rtol, atol=self.atol + ) def test_dynamic(self): paddle.disable_static() @@ -229,14 +255,12 @@ def test_dynamic(self): class TestIndexAddAPIMoreType(TestIndexAddAPI): - def setType(self): self.x_type = np.float64 self.index_type = np.int64 class TestIndexAddAPICase2(TestIndexAddAPI): - def config(self): self.axis = 1 self.x_shape = (100, 100, 5) @@ -245,7 +269,6 @@ def config(self): class TestIndexAddAPICase3(TestIndexAddAPI): - def config(self): self.axis = 2 self.x_shape = (100, 100, 25) @@ -254,16 +277,14 @@ def config(self): class TestIndexAddAPICase4(TestIndexAddAPI): - def config(self): self.axis = 0 - self.x_shape = (10, ) + self.x_shape = (10,) self.index_size = 4 - self.add_value_shape = (4, ) + self.add_value_shape = (4,) class TestIndexAddAPICase5(TestIndexAddAPI): - def config(self): self.axis = -1 self.x_shape = (10, 10) diff --git a/python/paddle/fluid/tests/unittests/test_index_sample_op.py b/python/paddle/fluid/tests/unittests/test_index_sample_op.py index 7c4be22305b8c..550ddfe344f3e 100644 --- a/python/paddle/fluid/tests/unittests/test_index_sample_op.py +++ b/python/paddle/fluid/tests/unittests/test_index_sample_op.py @@ -20,16 +20,14 @@ class TestIndexSampleOp(OpTest): - def setUp(self): self.op_type = "index_sample" self.python_api = paddle.index_sample self.config() xnp = np.random.random(self.x_shape).astype(self.x_type) - indexnp = np.random.randint(low=0, - high=self.x_shape[1], - size=self.index_shape).astype( - self.index_type) + indexnp = np.random.randint( + low=0, high=self.x_shape[1], size=self.index_shape + ).astype(self.index_type) self.inputs = {'X': xnp, 'Index': indexnp} index_array = [] for i in range(self.index_shape[0]): @@ -56,7 +54,6 @@ def config(self): class TestCase1(TestIndexSampleOp): - def config(self): """ For one dimension input @@ -68,7 +65,6 @@ def config(self): class TestCase2(TestIndexSampleOp): - def config(self): """ For int64_t index type @@ -80,7 +76,6 @@ def config(self): class TestCase3(TestIndexSampleOp): - def config(self): """ For int index type @@ -92,7 +87,6 @@ def config(self): class TestCase4(TestIndexSampleOp): - def config(self): """ For int64 index type @@ -104,7 +98,6 @@ def config(self): class TestIndexSampleShape(unittest.TestCase): - def test_shape(self): paddle.enable_static() # create x value @@ -115,8 +108,9 @@ def test_shape(self): # create index value index_shape = (2, 3) index_type = "int32" - index_np = np.random.randint(low=0, high=x_shape[1], - size=index_shape).astype(index_type) + index_np = np.random.randint( + low=0, high=x_shape[1], size=index_shape + ).astype(index_type) x = fluid.data(name='x', shape=[-1, 5], dtype='float64') index = fluid.data(name='index', shape=[-1, 3], dtype='int32') @@ -131,18 +125,24 @@ def test_shape(self): class TestIndexSampleDynamic(unittest.TestCase): - def test_result(self): with fluid.dygraph.guard(): - x = paddle.to_tensor([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [9.0, 10.0, 11.0, 12.0]], - dtype='float32') - index = paddle.to_tensor([[0, 1, 2], [1, 2, 3], [0, 0, 0]], - dtype='int32') + x = paddle.to_tensor( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ], + dtype='float32', + ) + index = paddle.to_tensor( + [[0, 1, 2], [1, 2, 3], [0, 0, 0]], dtype='int32' + ) out_z1 = paddle.index_sample(x, index) - except_output = np.array([[1.0, 2.0, 3.0], [6.0, 7.0, 8.0], - [9.0, 9.0, 9.0]]) + except_output = np.array( + [[1.0, 2.0, 3.0], [6.0, 7.0, 8.0], [9.0, 9.0, 9.0]] + ) assert out_z1.numpy().all() == except_output.all() diff --git a/python/paddle/fluid/tests/unittests/test_index_select_op.py b/python/paddle/fluid/tests/unittests/test_index_select_op.py index 2d73a29e2c936..edb2dc711822a 100644 --- a/python/paddle/fluid/tests/unittests/test_index_select_op.py +++ b/python/paddle/fluid/tests/unittests/test_index_select_op.py @@ -21,19 +21,18 @@ class TestIndexSelectOp(OpTest): - def setUp(self): self.python_api = paddle.index_select self.op_type = "index_select" self.init_dtype_type() - index_np = np.random.randint(low=0, - high=self.x_shape[self.dim], - size=self.index_size) + index_np = np.random.randint( + low=0, high=self.x_shape[self.dim], size=self.index_size + ) x_np = np.random.random(self.x_shape).astype(self.x_type) self.inputs = {'X': x_np, 'Index': index_np} self.attrs = {'dim': self.dim} - outer_loop = np.prod(self.x_shape[:self.dim]) - x_reshape = [outer_loop] + list(self.x_shape[self.dim:]) + outer_loop = np.prod(self.x_shape[: self.dim]) + x_reshape = [outer_loop] + list(self.x_shape[self.dim :]) x_np_reshape = np.reshape(x_np, tuple(x_reshape)) out_list = [] for i in range(outer_loop): @@ -61,7 +60,6 @@ def test_check_grad_normal(self): class TestIndexSelectOpCase2(TestIndexSelectOp): - def init_dtype_type(self): self.x_type = np.float32 self.index_type = np.int32 @@ -71,7 +69,6 @@ def init_dtype_type(self): class TestIndexSelectOpCaseSingleThread(TestIndexSelectOp): - def init_dtype_type(self): if fluid.is_compiled_with_cuda(): fluid.set_flags({'FLAGS_cudnn_deterministic': True}) @@ -83,10 +80,14 @@ def init_dtype_type(self): class TestIndexSelectAPI(unittest.TestCase): - def input_data(self): - self.data_x = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [9.0, 10.0, 11.0, 12.0]]) + self.data_x = np.array( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ] + ) self.data_index = np.array([0, 1, 1]).astype('int32') def test_index_select_api(self): @@ -95,39 +96,37 @@ def test_index_select_api(self): # case 1: with program_guard(Program(), Program()): x = fluid.layers.data(name='x', shape=[-1, 4]) - index = fluid.layers.data(name='index', - shape=[3], - dtype='int32', - append_batch_size=False) + index = fluid.layers.data( + name='index', shape=[3], dtype='int32', append_batch_size=False + ) z = paddle.index_select(x, index, axis=1) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={ - 'x': self.data_x, - 'index': self.data_index - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], - [9.0, 10.0, 10.0]]) + (res,) = exe.run( + feed={'x': self.data_x, 'index': self.data_index}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], [9.0, 10.0, 10.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) # case 2: with program_guard(Program(), Program()): x = fluid.layers.data(name='x', shape=[-1, 4]) - index = fluid.layers.data(name='index', - shape=[3], - dtype='int32', - append_batch_size=False) + index = fluid.layers.data( + name='index', shape=[3], dtype='int32', append_batch_size=False + ) z = paddle.index_select(x, index) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={ - 'x': self.data_x, - 'index': self.data_index - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [5.0, 6.0, 7.0, 8.0]]) + (res,) = exe.run( + feed={'x': self.data_x, 'index': self.data_index}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [5.0, 6.0, 7.0, 8.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) def test_dygraph_api(self): @@ -138,8 +137,9 @@ def test_dygraph_api(self): index = fluid.dygraph.to_variable(self.data_index) z = paddle.index_select(x, index) np_z = z.numpy() - expect_out = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [5.0, 6.0, 7.0, 8.0]]) + expect_out = np.array( + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [5.0, 6.0, 7.0, 8.0]] + ) np.testing.assert_allclose(expect_out, np_z, rtol=1e-05) # case 2: @@ -148,8 +148,9 @@ def test_dygraph_api(self): index = fluid.dygraph.to_variable(self.data_index) z = paddle.index_select(x, index, axis=1) np_z = z.numpy() - expect_out = np.array([[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], - [9.0, 10.0, 10.0]]) + expect_out = np.array( + [[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], [9.0, 10.0, 10.0]] + ) np.testing.assert_allclose(expect_out, np_z, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_infer_no_need_buffer_slots.py b/python/paddle/fluid/tests/unittests/test_infer_no_need_buffer_slots.py index 51f6cd53ca376..659aa36c334d3 100644 --- a/python/paddle/fluid/tests/unittests/test_infer_no_need_buffer_slots.py +++ b/python/paddle/fluid/tests/unittests/test_infer_no_need_buffer_slots.py @@ -20,12 +20,17 @@ class TestInferNoNeedBufferSlots(unittest.TestCase): - def net(self): - x1 = fluid.default_main_program().global_block().create_var( - dtype="float32", shape=[1], lod_level=0, name="x1") - x2 = fluid.default_main_program().global_block().create_var( - dtype="float32", shape=[1], lod_level=0, name="x2") + x1 = ( + fluid.default_main_program() + .global_block() + .create_var(dtype="float32", shape=[1], lod_level=0, name="x1") + ) + x2 = ( + fluid.default_main_program() + .global_block() + .create_var(dtype="float32", shape=[1], lod_level=0, name="x2") + ) x = fluid.layers.elementwise_add(x1, x2) return x @@ -52,18 +57,27 @@ def test_infer_no_need_buffer_slots(self): if idx == 0: # elementwise_add op self.assertEqual( - core.infer_no_need_buffer_slots(op.type, inputs, outputs, - attrs), set([])) + core.infer_no_need_buffer_slots( + op.type, inputs, outputs, attrs + ), + set([]), + ) elif idx == 1: # fill constant op self.assertEqual( - core.infer_no_need_buffer_slots(op.type, inputs, outputs, - attrs), set([])) + core.infer_no_need_buffer_slots( + op.type, inputs, outputs, attrs + ), + set([]), + ) else: # elementwise_add_grad op self.assertEqual( - core.infer_no_need_buffer_slots(op.type, inputs, outputs, - attrs), set(['Y', 'X'])) + core.infer_no_need_buffer_slots( + op.type, inputs, outputs, attrs + ), + set(['Y', 'X']), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_infer_shape.py b/python/paddle/fluid/tests/unittests/test_infer_shape.py index f4c718bf80ad6..df2ba25afbca4 100644 --- a/python/paddle/fluid/tests/unittests/test_infer_shape.py +++ b/python/paddle/fluid/tests/unittests/test_infer_shape.py @@ -18,7 +18,6 @@ class TestInferShape(unittest.TestCase): - def test_sum_op(self): prog = core.ProgramDesc() self.assertIsNotNone(prog) diff --git a/python/paddle/fluid/tests/unittests/test_inference_api.py b/python/paddle/fluid/tests/unittests/test_inference_api.py index 27ddf815aa0d0..e62b258fefefd 100644 --- a/python/paddle/fluid/tests/unittests/test_inference_api.py +++ b/python/paddle/fluid/tests/unittests/test_inference_api.py @@ -25,37 +25,46 @@ class TestInferenceApi(unittest.TestCase): - def test_inference_api(self): tensor32 = np.random.randint(10, 20, size=[20, 2]).astype('int32') paddletensor32 = PaddleTensor(tensor32) dtype32 = paddletensor32.dtype self.assertEqual(dtype32, PaddleDType.INT32) - self.assertEqual(paddletensor32.data.tolist('int32'), - tensor32.ravel().tolist()) + self.assertEqual( + paddletensor32.data.tolist('int32'), tensor32.ravel().tolist() + ) paddletensor32.data.reset(tensor32) - self.assertEqual(paddletensor32.as_ndarray().ravel().tolist(), - tensor32.ravel().tolist()) + self.assertEqual( + paddletensor32.as_ndarray().ravel().tolist(), + tensor32.ravel().tolist(), + ) tensor64 = np.random.randint(10, 20, size=[20, 2]).astype('int64') paddletensor64 = PaddleTensor(tensor64) dtype64 = paddletensor64.dtype self.assertEqual(dtype64, PaddleDType.INT64) - self.assertEqual(paddletensor64.data.tolist('int64'), - tensor64.ravel().tolist()) + self.assertEqual( + paddletensor64.data.tolist('int64'), tensor64.ravel().tolist() + ) paddletensor64.data.reset(tensor64) - self.assertEqual(paddletensor64.as_ndarray().ravel().tolist(), - tensor64.ravel().tolist()) + self.assertEqual( + paddletensor64.as_ndarray().ravel().tolist(), + tensor64.ravel().tolist(), + ) tensor_float = np.random.randn(20, 2).astype('float32') paddletensor_float = PaddleTensor(tensor_float) dtype_float = paddletensor_float.dtype self.assertEqual(dtype_float, PaddleDType.FLOAT32) - self.assertEqual(paddletensor_float.data.tolist('float32'), - tensor_float.ravel().tolist()) + self.assertEqual( + paddletensor_float.data.tolist('float32'), + tensor_float.ravel().tolist(), + ) paddletensor_float.data.reset(tensor_float) - self.assertEqual(paddletensor_float.as_ndarray().ravel().tolist(), - tensor_float.ravel().tolist()) + self.assertEqual( + paddletensor_float.as_ndarray().ravel().tolist(), + tensor_float.ravel().tolist(), + ) def get_sample_model(): @@ -66,24 +75,26 @@ def get_sample_model(): startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): data = fluid.data(name="data", shape=[-1, 6, 64, 64], dtype="float32") - conv_out = fluid.layers.conv2d(input=data, - num_filters=3, - filter_size=3, - groups=1, - padding=0, - bias_attr=False, - act=None) + conv_out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + groups=1, + padding=0, + bias_attr=False, + act=None, + ) exe.run(startup_program) - serialized_program = paddle.static.serialize_program(data, - conv_out, - program=main_program) + serialized_program = paddle.static.serialize_program( + data, conv_out, program=main_program + ) serialized_params = paddle.static.serialize_persistables( - data, conv_out, executor=exe, program=main_program) + data, conv_out, executor=exe, program=main_program + ) return serialized_program, serialized_params class TestInferenceBaseAPI(unittest.TestCase): - def get_config(self, model, params): config = Config() config.set_model_buffer(model, len(model), params, len(params)) diff --git a/python/paddle/fluid/tests/unittests/test_inference_model_io.py b/python/paddle/fluid/tests/unittests/test_inference_model_io.py index f63368dffc890..bf9ac53f99c53 100644 --- a/python/paddle/fluid/tests/unittests/test_inference_model_io.py +++ b/python/paddle/fluid/tests/unittests/test_inference_model_io.py @@ -28,13 +28,16 @@ import paddle.fluid.optimizer as optimizer from paddle.fluid.compiler import CompiledProgram from paddle.fluid.framework import Program, program_guard -from paddle.fluid.io import save_inference_model, load_inference_model, save_persistables +from paddle.fluid.io import ( + save_inference_model, + load_inference_model, + save_persistables, +) paddle.enable_static() class InferModel(object): - def __init__(self, list): self.program = list[0] self.feed_var_names = list[1] @@ -42,7 +45,6 @@ def __init__(self, list): class TestBook(unittest.TestCase): - def test_fit_line_inference_model(self): root_path = tempfile.TemporaryDirectory() MODEL_DIR = os.path.join(root_path.name, "inference_model") @@ -69,31 +71,36 @@ def test_fit_line_inference_model(self): exe.run(init_program, feed={}, fetch_list=[]) for i in range(100): - tensor_x = np.array([[1, 1], [1, 2], [3, 4], [5, - 2]]).astype("float32") + tensor_x = np.array([[1, 1], [1, 2], [3, 4], [5, 2]]).astype( + "float32" + ) tensor_y = np.array([[-2], [-3], [-7], [-7]]).astype("float32") - exe.run(program, - feed={ - 'x': tensor_x, - 'y': tensor_y - }, - fetch_list=[avg_cost]) + exe.run( + program, + feed={'x': tensor_x, 'y': tensor_y}, + fetch_list=[avg_cost], + ) # Separated model and unified model save_inference_model(MODEL_DIR, ["x", "y"], [avg_cost], exe, program) - save_inference_model(UNI_MODEL_DIR, ["x", "y"], [avg_cost], exe, - program, 'model', 'params') + save_inference_model( + UNI_MODEL_DIR, + ["x", "y"], + [avg_cost], + exe, + program, + 'model', + 'params', + ) main_program = program.clone()._prune_with_input( - feeded_var_names=["x", "y"], targets=[avg_cost]) + feeded_var_names=["x", "y"], targets=[avg_cost] + ) params_str = save_persistables(exe, None, main_program, None) - expected = exe.run(program, - feed={ - 'x': tensor_x, - 'y': tensor_y - }, - fetch_list=[avg_cost])[0] + expected = exe.run( + program, feed={'x': tensor_x, 'y': tensor_y}, fetch_list=[avg_cost] + )[0] importlib.reload(executor) # reload to build a new scope @@ -101,15 +108,18 @@ def test_fit_line_inference_model(self): with open(os.path.join(UNI_MODEL_DIR, 'model'), "rb") as f: model_str = f.read() model_1 = InferModel( - load_inference_model(None, exe, model_str, params_str)) + load_inference_model(None, exe, model_str, params_str) + ) for model in [model_0, model_1]: - outs = exe.run(model.program, - feed={ - model.feed_var_names[0]: tensor_x, - model.feed_var_names[1]: tensor_y - }, - fetch_list=model.fetch_vars) + outs = exe.run( + model.program, + feed={ + model.feed_var_names[0]: tensor_x, + model.feed_var_names[1]: tensor_y, + }, + fetch_list=model.fetch_vars, + ) actual = outs[0] self.assertEqual(model.feed_var_names, ["x", "y"]) @@ -119,12 +129,17 @@ def test_fit_line_inference_model(self): root_path.cleanup() - self.assertRaises(ValueError, fluid.io.load_inference_model, None, exe, - model_str, None) + self.assertRaises( + ValueError, + fluid.io.load_inference_model, + None, + exe, + model_str, + None, + ) class TestSaveInferenceModel(unittest.TestCase): - def test_save_inference_model(self): root_path = tempfile.TemporaryDirectory() MODEL_DIR = os.path.join(root_path.name, "inference_model2") @@ -160,8 +175,9 @@ def test_save_inference_model_with_auc(self): y = layers.data(name='y', shape=[1], dtype='int32') predict = fluid.layers.fc(input=x, size=2, act='softmax') acc = fluid.layers.accuracy(input=predict, label=y) - auc_var, batch_auc_var, auc_states = fluid.layers.auc(input=predict, - label=y) + auc_var, batch_auc_var, auc_states = fluid.layers.auc( + input=predict, label=y + ) cost = fluid.layers.cross_entropy(input=predict, label=y) avg_cost = paddle.mean(x=cost) @@ -170,8 +186,9 @@ def test_save_inference_model_with_auc(self): exe.run(init_program, feed={}, fetch_list=[]) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") - save_inference_model(MODEL_DIR, ["x", "y"], [avg_cost], exe, - program) + save_inference_model( + MODEL_DIR, ["x", "y"], [avg_cost], exe, program + ) root_path.cleanup() expected_warn = "please ensure that you have set the auc states to zeros before saving inference model" self.assertTrue(len(w) > 0) @@ -179,7 +196,6 @@ def test_save_inference_model_with_auc(self): class TestInstance(unittest.TestCase): - def test_save_inference_model(self): root_path = tempfile.TemporaryDirectory() MODEL_DIR = os.path.join(root_path.name, "inference_model3") @@ -203,16 +219,19 @@ def test_save_inference_model(self): # will print warning message cp_prog = CompiledProgram(program).with_data_parallel( - loss_name=avg_cost.name) + loss_name=avg_cost.name + ) save_inference_model(MODEL_DIR, ["x", "y"], [avg_cost], exe, cp_prog) - self.assertRaises(TypeError, save_inference_model, - [MODEL_DIR, ["x", "y"], [avg_cost], [], cp_prog]) + self.assertRaises( + TypeError, + save_inference_model, + [MODEL_DIR, ["x", "y"], [avg_cost], [], cp_prog], + ) root_path.cleanup() class TestSaveInferenceModelNew(unittest.TestCase): - def test_save_and_load_inference_model(self): root_path = tempfile.TemporaryDirectory() MODEL_DIR = os.path.join(root_path.name, "inference_model5") @@ -239,86 +258,143 @@ def test_save_and_load_inference_model(self): tensor_x = np.array([[1, 1], [1, 2], [5, 2]]).astype("float32") tensor_y = np.array([[-2], [-3], [-7]]).astype("float32") for i in range(3): - exe.run(program, - feed={ - 'x': tensor_x, - 'y': tensor_y - }, - fetch_list=[avg_cost]) - - self.assertRaises(ValueError, paddle.static.save_inference_model, None, - ['x', 'y'], [avg_cost], exe) - self.assertRaises(ValueError, paddle.static.save_inference_model, - MODEL_DIR + "/", [x, y], [avg_cost], exe) - self.assertRaises(ValueError, paddle.static.save_inference_model, - MODEL_DIR, ['x', 'y'], [avg_cost], exe) - self.assertRaises(ValueError, paddle.static.save_inference_model, - MODEL_DIR, 'x', [avg_cost], exe) - self.assertRaises(ValueError, paddle.static.save_inference_model, - MODEL_DIR, [x, y], ['avg_cost'], exe) - self.assertRaises(ValueError, paddle.static.save_inference_model, - MODEL_DIR, [x, y], 'avg_cost', exe) + exe.run( + program, + feed={'x': tensor_x, 'y': tensor_y}, + fetch_list=[avg_cost], + ) + + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + None, + ['x', 'y'], + [avg_cost], + exe, + ) + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + MODEL_DIR + "/", + [x, y], + [avg_cost], + exe, + ) + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + MODEL_DIR, + ['x', 'y'], + [avg_cost], + exe, + ) + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + MODEL_DIR, + 'x', + [avg_cost], + exe, + ) + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + MODEL_DIR, + [x, y], + ['avg_cost'], + exe, + ) + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + MODEL_DIR, + [x, y], + 'avg_cost', + exe, + ) model_path = MODEL_DIR + "_isdir.pdmodel" os.makedirs(model_path) - self.assertRaises(ValueError, paddle.static.save_inference_model, - MODEL_DIR + "_isdir", [x, y], [avg_cost], exe) + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + MODEL_DIR + "_isdir", + [x, y], + [avg_cost], + exe, + ) os.rmdir(model_path) params_path = MODEL_DIR + "_isdir.pdmodel" os.makedirs(params_path) - self.assertRaises(ValueError, paddle.static.save_inference_model, - MODEL_DIR + "_isdir", [x, y], [avg_cost], exe) + self.assertRaises( + ValueError, + paddle.static.save_inference_model, + MODEL_DIR + "_isdir", + [x, y], + [avg_cost], + exe, + ) os.rmdir(params_path) - paddle.static.io.save_inference_model(MODEL_DIR, [x, y], [avg_cost], - exe) + paddle.static.io.save_inference_model( + MODEL_DIR, [x, y], [avg_cost], exe + ) self.assertTrue(os.path.exists(MODEL_DIR + ".pdmodel")) self.assertTrue(os.path.exists(MODEL_DIR + ".pdiparams")) - expected = exe.run(program, - feed={ - 'x': tensor_x, - 'y': tensor_y - }, - fetch_list=[avg_cost])[0] + expected = exe.run( + program, feed={'x': tensor_x, 'y': tensor_y}, fetch_list=[avg_cost] + )[0] importlib.reload(executor) # reload to build a new scope - self.assertRaises(ValueError, paddle.static.load_inference_model, None, - exe) - self.assertRaises(ValueError, paddle.static.load_inference_model, - MODEL_DIR + "/", exe) - self.assertRaises(ValueError, paddle.static.load_inference_model, - [MODEL_DIR], exe) - self.assertRaises(ValueError, - paddle.static.load_inference_model, - MODEL_DIR, - exe, - pserver_endpoints=None) - self.assertRaises(ValueError, - paddle.static.load_inference_model, - MODEL_DIR, - exe, - unsupported_param=None) - self.assertRaises((TypeError, ValueError), - paddle.static.load_inference_model, - None, - exe, - model_filename="illegal", - params_filename="illegal") - - model = InferModel(paddle.static.io.load_inference_model( - MODEL_DIR, exe)) + self.assertRaises( + ValueError, paddle.static.load_inference_model, None, exe + ) + self.assertRaises( + ValueError, paddle.static.load_inference_model, MODEL_DIR + "/", exe + ) + self.assertRaises( + ValueError, paddle.static.load_inference_model, [MODEL_DIR], exe + ) + self.assertRaises( + ValueError, + paddle.static.load_inference_model, + MODEL_DIR, + exe, + pserver_endpoints=None, + ) + self.assertRaises( + ValueError, + paddle.static.load_inference_model, + MODEL_DIR, + exe, + unsupported_param=None, + ) + self.assertRaises( + (TypeError, ValueError), + paddle.static.load_inference_model, + None, + exe, + model_filename="illegal", + params_filename="illegal", + ) + + model = InferModel( + paddle.static.io.load_inference_model(MODEL_DIR, exe) + ) root_path.cleanup() - outs = exe.run(model.program, - feed={ - model.feed_var_names[0]: tensor_x, - model.feed_var_names[1]: tensor_y - }, - fetch_list=model.fetch_vars) + outs = exe.run( + model.program, + feed={ + model.feed_var_names[0]: tensor_x, + model.feed_var_names[1]: tensor_y, + }, + fetch_list=model.fetch_vars, + ) actual = outs[0] self.assertEqual(model.feed_var_names, ["x", "y"]) @@ -359,12 +435,11 @@ def test_serialize_program_and_persistables(self): tensor_x = np.array([[1, 1], [1, 2], [5, 2]]).astype("float32") tensor_y = np.array([[-2], [-3], [-7]]).astype("float32") for i in range(3): - exe.run(program, - feed={ - 'x': tensor_x, - 'y': tensor_y - }, - fetch_list=[avg_cost]) + exe.run( + program, + feed={'x': tensor_x, 'y': tensor_y}, + fetch_list=[avg_cost], + ) # test if return type of serialize_program is bytes res1 = paddle.static.io.serialize_program([x, y], [avg_cost]) @@ -375,8 +450,13 @@ def test_serialize_program_and_persistables(self): # test if variables in program is empty res = paddle.static.io._serialize_persistables(Program(), None) self.assertEqual(res, None) - self.assertRaises(TypeError, paddle.static.io.deserialize_persistables, - None, None, None) + self.assertRaises( + TypeError, + paddle.static.io.deserialize_persistables, + None, + None, + None, + ) def test_normalize_program(self): init_program = fluid.default_startup_program() @@ -402,34 +482,40 @@ def test_normalize_program(self): tensor_x = np.array([[1, 1], [1, 2], [5, 2]]).astype("float32") tensor_y = np.array([[-2], [-3], [-7]]).astype("float32") for i in range(3): - exe.run(program, - feed={ - 'x': tensor_x, - 'y': tensor_y - }, - fetch_list=[avg_cost]) + exe.run( + program, + feed={'x': tensor_x, 'y': tensor_y}, + fetch_list=[avg_cost], + ) # test if return type of serialize_program is bytes res = paddle.static.normalize_program(program, [x, y], [avg_cost]) self.assertTrue(isinstance(res, Program)) # test program type - self.assertRaises(TypeError, paddle.static.normalize_program, None, - [x, y], [avg_cost]) + self.assertRaises( + TypeError, paddle.static.normalize_program, None, [x, y], [avg_cost] + ) # test feed_vars type - self.assertRaises(TypeError, paddle.static.normalize_program, program, - 'x', [avg_cost]) + self.assertRaises( + TypeError, paddle.static.normalize_program, program, 'x', [avg_cost] + ) # test fetch_vars type - self.assertRaises(TypeError, paddle.static.normalize_program, program, - [x, y], 'avg_cost') + self.assertRaises( + TypeError, + paddle.static.normalize_program, + program, + [x, y], + 'avg_cost', + ) class TestLoadInferenceModelError(unittest.TestCase): - def test_load_model_not_exist(self): place = core.CPUPlace() exe = executor.Executor(place) - self.assertRaises(ValueError, load_inference_model, - './test_not_exist_dir', exe) + self.assertRaises( + ValueError, load_inference_model, './test_not_exist_dir', exe + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_initializer.py b/python/paddle/fluid/tests/unittests/test_initializer.py index 24a384b3aa0c7..4454d7a9bb8d5 100644 --- a/python/paddle/fluid/tests/unittests/test_initializer.py +++ b/python/paddle/fluid/tests/unittests/test_initializer.py @@ -27,9 +27,11 @@ def check_cast_op(op): - return op.type == 'cast' and \ - op.attr('in_dtype') == VarDesc.VarType.FP32 and \ - op.attr('out_dtype') in [VarDesc.VarType.FP16, VarDesc.VarType.BF16] + return ( + op.type == 'cast' + and op.attr('in_dtype') == VarDesc.VarType.FP32 + and op.attr('out_dtype') in [VarDesc.VarType.FP16, VarDesc.VarType.BF16] + ) def output_hist(out): @@ -41,21 +43,21 @@ def output_hist(out): class TestConstantInitializer(unittest.TestCase): - def test_calculate_gain(self): self.assertEqual(paddle.nn.initializer.calculate_gain('sigmoid'), 1) self.assertEqual(paddle.nn.initializer.calculate_gain('linear'), 1) self.assertEqual(paddle.nn.initializer.calculate_gain('conv2d'), 1) self.assertEqual(paddle.nn.initializer.calculate_gain('tanh'), 5.0 / 3) - self.assertEqual(paddle.nn.initializer.calculate_gain('relu'), - math.sqrt(2.0)) - self.assertEqual(paddle.nn.initializer.calculate_gain('leaky_relu', 1), - 1) + self.assertEqual( + paddle.nn.initializer.calculate_gain('relu'), math.sqrt(2.0) + ) + self.assertEqual( + paddle.nn.initializer.calculate_gain('leaky_relu', 1), 1 + ) self.assertEqual(paddle.nn.initializer.calculate_gain('selu'), 3.0 / 4) def test_constant_initializer_default_value(self, dtype="float32"): - """Test the constant initializer with default value - """ + """Test the constant initializer with default value""" program = framework.Program() block = program.global_block() for _ in range(2): @@ -64,7 +66,8 @@ def test_constant_initializer_default_value(self, dtype="float32"): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.ConstantInitializer()) + initializer=initializer.ConstantInitializer(), + ) num_ops = 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -73,8 +76,7 @@ def test_constant_initializer_default_value(self, dtype="float32"): return block def test_constant_initializer(self, dtype="float32"): - """Test constant initializer with supplied value - """ + """Test constant initializer with supplied value""" program = framework.Program() block = program.global_block() for _ in range(2): @@ -83,7 +85,8 @@ def test_constant_initializer(self, dtype="float32"): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.ConstantInitializer(2.3)) + initializer=initializer.ConstantInitializer(2.3), + ) num_ops = 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -92,32 +95,31 @@ def test_constant_initializer(self, dtype="float32"): return block def test_constant_initializer_fp16(self): - """Test constant initializer with float16 - """ + """Test constant initializer with float16""" self.test_constant_initializer_default_value("float16") self.test_constant_initializer("float16") def test_constant_initializer_bf16(self): """Test constant initializer with bfloat16 - No cast operator has been added here + No cast operator has been added here """ self.test_constant_initializer_default_value("uint16") self.test_constant_initializer("uint16") class TestUniformInitializer(unittest.TestCase): - def test_uniform_initializer_default_value(self, dtype="float32"): - """Test the uniform initializer with default value - """ + """Test the uniform initializer with default value""" program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.UniformInitializer()) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.UniformInitializer(), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -128,40 +130,42 @@ def test_uniform_initializer_default_value(self, dtype="float32"): return block def test_uniform_initializer_random_seed(self): - """Test the uniform initializer with manually setting seed - """ + """Test the uniform initializer with manually setting seed""" program = framework.Program() program.random_seed = 123 block = program.global_block() for _ in range(2): - block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="param1", - initializer=initializer.UniformInitializer()) + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param1", + initializer=initializer.UniformInitializer(), + ) block.create_parameter( dtype="float32", shape=[5, 10], lod_level=0, name="param2", - initializer=initializer.UniformInitializer(seed=456)) + initializer=initializer.UniformInitializer(seed=456), + ) init_op = block.ops[1] self.assertEqual(init_op.attr("seed"), 456) init_op1 = block.ops[0] self.assertEqual(init_op1.attr("seed"), 123) def test_uniform_initializer(self, dtype="float32"): - """Test uniform initializer with supplied attributes - """ + """Test uniform initializer with supplied attributes""" program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.UniformInitializer( - -4.2, 3.1, 123)) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.UniformInitializer(-4.2, 3.1, 123), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -172,17 +176,17 @@ def test_uniform_initializer(self, dtype="float32"): return block def test_uniform_initializer_two_op(self, dtype="float32"): - """Test uniform initializer with supplied attributes - """ + """Test uniform initializer with supplied attributes""" program = framework.Program() block = program.global_block() for i in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.UniformInitializer( - -4.2, float(i), 123)) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.UniformInitializer(-4.2, float(i), 123), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op0 = block.ops[0] @@ -193,8 +197,7 @@ def test_uniform_initializer_two_op(self, dtype="float32"): return block def test_uniform_initializer_fp16(self): - """Test uniform initializer with float16 - """ + """Test uniform initializer with float16""" block = self.test_uniform_initializer_default_value("float16") self.assertTrue(check_cast_op(block.ops[1])) block = self.test_uniform_initializer(dtype="float16") @@ -204,7 +207,7 @@ def test_uniform_initializer_fp16(self): def test_uniform_initializer_bf16(self): """Test uniform initializer with bfloat16 - No cast operator has been added here + No cast operator has been added here """ block = self.test_uniform_initializer_default_value("uint16") block = self.test_uniform_initializer(dtype="uint16") @@ -212,18 +215,18 @@ def test_uniform_initializer_bf16(self): class TestNormalInitializer(unittest.TestCase): - def test_normal_initializer_default_value(self): - """Test the normal initializer with default value - """ + """Test the normal initializer with default value""" program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.NormalInitializer()) + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.NormalInitializer(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -232,17 +235,17 @@ def test_normal_initializer_default_value(self): self.assertEqual(init_op.attr('seed'), 0) def test_normal_initializer(self, dtype="float32"): - """Test normal initializer with supplied attributes - """ + """Test normal initializer with supplied attributes""" program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.NormalInitializer( - 2.3, 1.9, 123)) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.NormalInitializer(2.3, 1.9, 123), + ) num_ops = 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -253,21 +256,18 @@ def test_normal_initializer(self, dtype="float32"): return block def test_normal_initializer_fp16(self): - """Test normal initializer with float16 - """ + """Test normal initializer with float16""" self.test_normal_initializer("float16") def test_normal_initializer_bf16(self): - """Test normal initializer with bfloat16 - """ + """Test normal initializer with bfloat16""" self.test_normal_initializer("uint16") class TestXavierInitializer(unittest.TestCase): - def test_uniform_xavier_initializer(self): """Test Xavier initializer with uniform distribution on - for matrix multiply. + for matrix multiply. """ program = framework.Program() block = program.global_block() @@ -277,7 +277,8 @@ def test_uniform_xavier_initializer(self): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.XavierInitializer()) + initializer=initializer.XavierInitializer(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -288,7 +289,7 @@ def test_uniform_xavier_initializer(self): def test_uniform_xavier_initializer_conv(self): """Test Xavier initializer with uniform distribution on - for convolutions. + for convolutions. """ program = framework.Program() block = program.global_block() @@ -298,20 +299,22 @@ def test_uniform_xavier_initializer_conv(self): shape=[5, 10, 15, 20], lod_level=0, name="param", - initializer=initializer.XavierInitializer()) + initializer=initializer.XavierInitializer(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') receptive_field_size = float(15 * 20) limit = np.sqrt( - 6.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size)) + 6.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size) + ) self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) self.assertEqual(init_op.attr('seed'), 0) def test_normal_xavier_initializer(self): """Test Xavier initializer with normal distribution on - for matrix multiply. + for matrix multiply. """ program = framework.Program() block = program.global_block() @@ -321,7 +324,8 @@ def test_normal_xavier_initializer(self): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.XavierInitializer(uniform=False)) + initializer=initializer.XavierInitializer(uniform=False), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -332,7 +336,7 @@ def test_normal_xavier_initializer(self): def test_normal_xavier_initializer_conv(self): """Test Xavier initializer with normal distribution on - for convolutions. + for convolutions. """ program = framework.Program() block = program.global_block() @@ -342,36 +346,40 @@ def test_normal_xavier_initializer_conv(self): shape=[5, 10, 15, 20], lod_level=0, name="param", - initializer=initializer.XavierInitializer(uniform=False)) + initializer=initializer.XavierInitializer(uniform=False), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') receptive_field_size = float(15 * 20) std = np.sqrt( - 2.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size)) + 2.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size) + ) self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA) self.assertEqual(init_op.attr('seed'), 0) - def test_xavier_initializer_supplied_arguments(self, - dtype="float32", - uniform=True): - """Test the Xavier initializer with supplied arguments - """ + def test_xavier_initializer_supplied_arguments( + self, dtype="float32", uniform=True + ): + """Test the Xavier initializer with supplied arguments""" program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.XavierInitializer( - uniform=uniform, - fan_in=12, - fan_out=23, - seed=134)) - num_ops = 2 if (dtype == "float16" or - (dtype == "uint16" and not uniform)) else 1 + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer( + uniform=uniform, fan_in=12, fan_out=23, seed=134 + ), + ) + num_ops = ( + 2 + if (dtype == "float16" or (dtype == "uint16" and not uniform)) + else 1 + ) self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] if uniform: @@ -385,25 +393,24 @@ def test_xavier_initializer_supplied_arguments(self, return block def test_xavier_initializer_fp16(self): - """Test the Xavier initializer with float16 - """ + """Test the Xavier initializer with float16""" block = self.test_xavier_initializer_supplied_arguments("float16") def test_xavier_initializer_bf16(self): - """Test the Xavier initializer with bfloat16 - """ + """Test the Xavier initializer with bfloat16""" block_uniform = self.test_xavier_initializer_supplied_arguments( - "uint16") + "uint16" + ) self.assertEqual(len(block_uniform.ops), 1) block_gaussian = self.test_xavier_initializer_supplied_arguments( - "uint16", False) + "uint16", False + ) class TestMSRAInitializer(unittest.TestCase): - def test_uniform_msra_initializer(self): """Test MSRA initializer with uniform distribution on - for matrix multiply. + for matrix multiply. """ program = framework.Program() block = program.global_block() @@ -413,7 +420,8 @@ def test_uniform_msra_initializer(self): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.MSRAInitializer()) + initializer=initializer.MSRAInitializer(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -424,7 +432,7 @@ def test_uniform_msra_initializer(self): def test_uniform_msra_initializer_conv(self): """Test MSRA initializer with uniform distribution on - for convolutions. + for convolutions. """ program = framework.Program() block = program.global_block() @@ -434,7 +442,8 @@ def test_uniform_msra_initializer_conv(self): shape=[5, 10, 15, 20], lod_level=0, name="param", - initializer=initializer.MSRAInitializer()) + initializer=initializer.MSRAInitializer(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -446,7 +455,7 @@ def test_uniform_msra_initializer_conv(self): def test_normal_msra_initializer(self): """Test MSRA initializer with normal distribution on - for matrix multiply. + for matrix multiply. """ program = framework.Program() block = program.global_block() @@ -456,7 +465,8 @@ def test_normal_msra_initializer(self): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.MSRAInitializer(uniform=False)) + initializer=initializer.MSRAInitializer(uniform=False), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -467,7 +477,7 @@ def test_normal_msra_initializer(self): def test_normal_msra_initializer_conv(self): """Test MSRA initializer with normal distribution on - for convolutions. + for convolutions. """ program = framework.Program() block = program.global_block() @@ -477,7 +487,8 @@ def test_normal_msra_initializer_conv(self): shape=[5, 10, 15, 20], lod_level=0, name="param", - initializer=initializer.MSRAInitializer(uniform=False)) + initializer=initializer.MSRAInitializer(uniform=False), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -488,17 +499,17 @@ def test_normal_msra_initializer_conv(self): self.assertEqual(init_op.attr('seed'), 0) def test_msra_initializer_supplied_arguments(self, dtype="float32"): - """Test the MSRA initializer with supplied arguments - """ + """Test the MSRA initializer with supplied arguments""" program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.MSRAInitializer( - fan_in=12, seed=134)) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.MSRAInitializer(fan_in=12, seed=134), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -510,22 +521,18 @@ def test_msra_initializer_supplied_arguments(self, dtype="float32"): return block def test_msra_initializer_fp16(self): - """Test the MSRA initializer with float16 - """ + """Test the MSRA initializer with float16""" block = self.test_msra_initializer_supplied_arguments("float16") self.assertTrue(check_cast_op(block.ops[1])) def test_msra_initializer_bf16(self): - """Test the MSRA initializer with bfloat16 - """ + """Test the MSRA initializer with bfloat16""" block = self.test_msra_initializer_supplied_arguments("uint16") class TestBilinearInitializer(unittest.TestCase): - def test_bilinear_initializer(self, dtype="float32"): - """Test the bilinear initializer with supplied arguments - """ + """Test the bilinear initializer with supplied arguments""" program = framework.Program() block = program.global_block() for _ in range(2): @@ -534,7 +541,8 @@ def test_bilinear_initializer(self, dtype="float32"): shape=[8, 1, 3, 3], lod_level=0, name="param", - initializer=initializer.BilinearInitializer()) + initializer=initializer.BilinearInitializer(), + ) num_ops = 2 if dtype in ["float16", "uint16", "float64"] else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -545,14 +553,12 @@ def test_bilinear_initializer_fp64(self): self.test_bilinear_initializer(dtype='float64') def test_bilinear_initializer_fp16(self): - """Test the bilinear initializer with supplied arguments - """ + """Test the bilinear initializer with supplied arguments""" block = self.test_bilinear_initializer("float16") self.assertTrue(check_cast_op(block.ops[1])) def test_bilinear_initializer_bf16(self): - """Test the bilinear initializer with supplied arguments - """ + """Test the bilinear initializer with supplied arguments""" block = self.test_bilinear_initializer("uint16") self.assertTrue(check_cast_op(block.ops[1])) @@ -561,33 +567,37 @@ def test_type_error(self): class TestBilinearInitializerDygraphAPI(unittest.TestCase): - def func_test_case(self): factor = 2 C = 2 B = 8 H = W = 32 - w_attr = paddle.ParamAttr(learning_rate=0., - regularizer=L2Decay(0.), - initializer=initializer.BilinearInitializer()) + w_attr = paddle.ParamAttr( + learning_rate=0.0, + regularizer=L2Decay(0.0), + initializer=initializer.BilinearInitializer(), + ) data = paddle.rand([B, 3, H, W], dtype='float32') - conv_up = paddle.nn.Conv2DTranspose(3, - out_channels=C, - kernel_size=2 * factor - factor % 2, - padding=int( - math.ceil((factor - 1) / 2.)), - stride=factor, - weight_attr=w_attr, - bias_attr=False) + conv_up = paddle.nn.Conv2DTranspose( + 3, + out_channels=C, + kernel_size=2 * factor - factor % 2, + padding=int(math.ceil((factor - 1) / 2.0)), + stride=factor, + weight_attr=w_attr, + bias_attr=False, + ) x = conv_up(data) return x def func_test_case_fp16(self): paddle.set_default_dtype("float16") paddle.seed(1234) - w_attr = paddle.ParamAttr(learning_rate=0., - regularizer=L2Decay(0.), - initializer=initializer.BilinearInitializer()) + w_attr = paddle.ParamAttr( + learning_rate=0.0, + regularizer=L2Decay(0.0), + initializer=initializer.BilinearInitializer(), + ) conv2d = paddle.nn.Conv2D(1, 2, 3, weight_attr=w_attr) paddle.set_default_dtype("float32") return conv2d.weight @@ -610,11 +620,10 @@ def test_bilinear_initializer_fp16(self): class TestNumpyArrayInitializer(unittest.TestCase): - def test_numpy_array_initializer(self, dtype="float32"): - """Test the numpy array initializer with supplied arguments - """ + """Test the numpy array initializer with supplied arguments""" import numpy + program = framework.Program() block = program.global_block() np_array = numpy.random.random((10000)).astype(dtype) @@ -624,7 +633,8 @@ def test_numpy_array_initializer(self, dtype="float32"): shape=np_array.shape, lod_level=0, name="param", - initializer=initializer.NumpyArrayInitializer(np_array)) + initializer=initializer.NumpyArrayInitializer(np_array), + ) num_ops = 2 if dtype in ["float16", "uint16"] else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -633,23 +643,19 @@ def test_numpy_array_initializer(self, dtype="float32"): return block def test_numpy_array_initializer_fp16(self): - """Test the numpy array initializer with float16 - """ + """Test the numpy array initializer with float16""" block = self.test_numpy_array_initializer("float16") self.assertTrue(block.ops[1]) def test_numpy_array_initializer_bf16(self): - """Test the numpy array initializer with bfloat16 - """ + """Test the numpy array initializer with bfloat16""" block = self.test_numpy_array_initializer("uint16") self.assertTrue(block.ops[1]) class TestSetGlobalInitializer(unittest.TestCase): - def test_set_global_weight_initilizer(self): - """Test Set Global Param initilizer with UniformInitializer - """ + """Test Set Global Param initilizer with UniformInitializer""" main_prog = framework.Program() startup_prog = framework.Program() fluid.set_global_initializer(initializer.Uniform(low=-0.5, high=0.5)) @@ -674,13 +680,13 @@ def test_set_global_weight_initilizer(self): fluid.set_global_initializer(None) def test_set_global_bias_initilizer(self): - """Test Set Global Bias initilizer with NormalInitializer - """ + """Test Set Global Bias initilizer with NormalInitializer""" main_prog = framework.Program() startup_prog = framework.Program() - fluid.set_global_initializer(initializer.Uniform(low=-0.5, high=0.5), - bias_init=initializer.Normal(loc=0.0, - scale=2.0)) + fluid.set_global_initializer( + initializer.Uniform(low=-0.5, high=0.5), + bias_init=initializer.Normal(loc=0.0, scale=2.0), + ) with fluid.program_guard(main_prog, startup_prog): x = fluid.data(name="x", shape=[1, 3, 32, 32]) # default initilizer of bias in layers.conv2d is ConstantInitializer @@ -705,7 +711,6 @@ def test_set_global_bias_initilizer(self): class TestUniformInitializerDygraph(unittest.TestCase): - def func_uniform_initializer(self, dtype="float32"): """ In dygraph mode, we can use initializer directly to initialize a tensor. @@ -714,15 +719,16 @@ def func_uniform_initializer(self, dtype="float32"): tensor = paddle.zeros([1024, 1024, 16]) tensor.stop_gradient = False - np.testing.assert_allclose(np.zeros((1024, 1024, 16)), - tensor.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + np.zeros((1024, 1024, 16)), tensor.numpy(), rtol=1e-05 + ) uniform_ = paddle.nn.initializer.Uniform() uniform_(tensor) - self.assertEqual(tensor.stop_gradient, - False) # stop_gradient is not changed + self.assertEqual( + tensor.stop_gradient, False + ) # stop_gradient is not changed hist, prob = output_hist(tensor.numpy()) @@ -737,7 +743,6 @@ def test_uniform_initializer(self, dtype="float32"): class TestXavierInitializerDygraph(unittest.TestCase): - def func_xvarier_initializer(self, dtype="float32"): """ In dygraph mode, we can use initializer directly to initialize a tensor. @@ -747,15 +752,16 @@ def func_xvarier_initializer(self, dtype="float32"): tensor = paddle.zeros([1024, 1024, 16]) tensor.stop_gradient = False - xavier_ = paddle.fluid.initializer.XavierInitializer(uniform=False, - fan_in=3, - fan_out=5) + xavier_ = paddle.fluid.initializer.XavierInitializer( + uniform=False, fan_in=3, fan_out=5 + ) xavier_(tensor) hist, _ = output_hist(tensor.numpy()) hist2, _ = output_hist( - np.random.normal(0, np.sqrt(2.0 / (3 + 5)), [1024, 1024, 16])) + np.random.normal(0, np.sqrt(2.0 / (3 + 5)), [1024, 1024, 16]) + ) np.testing.assert_allclose(hist, hist2, rtol=0, atol=0.01) paddle.enable_static() @@ -767,7 +773,6 @@ def test_xavier_initializer(self, dtype="float32"): class TestMSRAInitializerDygraph(unittest.TestCase): - def func_msra_initializer(self, dtype="float32"): """ In dygraph mode, we can use initializer directly to initialize a tensor. @@ -777,14 +782,16 @@ def func_msra_initializer(self, dtype="float32"): tensor = paddle.zeros([1024, 1024, 16]) tensor.stop_gradient = False - msra_ = paddle.fluid.initializer.MSRAInitializer(uniform=False, - fan_in=4) + msra_ = paddle.fluid.initializer.MSRAInitializer( + uniform=False, fan_in=4 + ) msra_(tensor) hist, _ = output_hist(tensor.numpy()) hist2, _ = output_hist( - np.random.normal(0, np.sqrt(2.0 / (4)), [1024, 1024, 16])) + np.random.normal(0, np.sqrt(2.0 / (4)), [1024, 1024, 16]) + ) np.testing.assert_allclose(hist, hist2, rtol=0, atol=0.01) paddle.enable_static() @@ -796,7 +803,6 @@ def test_msra_initializer(self, dtype="float32"): class TesetconsistencyOfDynamicAndStaticGraph(unittest.TestCase): - def func_order(self): paddle.set_device('cpu') SEED = 123 @@ -805,23 +811,26 @@ def func_order(self): learning_rate=1.0, trainable=False, regularizer=None, - initializer=paddle.nn.initializer.TruncatedNormal(mean=0.0, - std=2.0)) + initializer=paddle.nn.initializer.TruncatedNormal( + mean=0.0, std=2.0 + ), + ) bias_attr = paddle.framework.ParamAttr( name="linear_bias", learning_rate=1.0, trainable=False, regularizer=None, - initializer=paddle.nn.initializer.TruncatedNormal(mean=0.0, - std=2.0)) + initializer=paddle.nn.initializer.TruncatedNormal( + mean=0.0, std=2.0 + ), + ) def run_dynamic_graph(): paddle.disable_static() paddle.seed(SEED) - linear = paddle.nn.Linear(1, - 1, - weight_attr=weight_attr, - bias_attr=bias_attr) + linear = paddle.nn.Linear( + 1, 1, weight_attr=weight_attr, bias_attr=bias_attr + ) return linear.weight.numpy(), linear.bias.numpy() paddle.enable_static() @@ -829,12 +838,13 @@ def run_static_graph(): paddle.enable_static() exe = paddle.static.Executor(paddle.CPUPlace()) paddle.seed(SEED) - linear = paddle.nn.Linear(1, - 1, - weight_attr=weight_attr, - bias_attr=bias_attr) - res = exe.run(paddle.static.default_startup_program(), - fetch_list=['linear_weight', 'linear_bias']) + linear = paddle.nn.Linear( + 1, 1, weight_attr=weight_attr, bias_attr=bias_attr + ) + res = exe.run( + paddle.static.default_startup_program(), + fetch_list=['linear_weight', 'linear_bias'], + ) return res[0], res[1] dynamic_res = run_dynamic_graph() @@ -857,7 +867,8 @@ class TestOrthogonalInitializer1(unittest.TestCase): def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Orthogonal(gain=3.0)) + initializer=paddle.nn.initializer.Orthogonal(gain=3.0) + ) self.dtype = "float64" self.in_features = 10 self.out_features = 15 @@ -865,10 +876,9 @@ def config(self): def check_result(self, a, b): np.testing.assert_array_equal(a, b) - np.testing.assert_allclose(np.matmul(a, a.T), - 9 * np.eye(10), - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + np.matmul(a, a.T), 9 * np.eye(10), rtol=1e-5, atol=1e-8 + ) def func_orthogonal(self): self.config() @@ -876,9 +886,9 @@ def func_orthogonal(self): paddle.disable_static() paddle.seed(2021) - linear = paddle.nn.Linear(self.in_features, - self.out_features, - weight_attr=self.weight_attr) + linear = paddle.nn.Linear( + self.in_features, self.out_features, weight_attr=self.weight_attr + ) res_dygraph = linear.weight.numpy() paddle.enable_static() @@ -886,9 +896,11 @@ def func_orthogonal(self): start_prog = paddle.static.Program() main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, start_prog): - linear = paddle.nn.Linear(self.in_features, - self.out_features, - weight_attr=self.weight_attr) + linear = paddle.nn.Linear( + self.in_features, + self.out_features, + weight_attr=self.weight_attr, + ) block = start_prog.global_block() self.assertEqual(len(block.ops), self.num_ops) @@ -919,7 +931,8 @@ class TestOrthogonalInitializer2(TestOrthogonalInitializer1): def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Orthogonal(gain=2.0)) + initializer=paddle.nn.initializer.Orthogonal(gain=2.0) + ) self.dtype = "float64" self.in_features = 15 self.out_features = 10 @@ -927,10 +940,9 @@ def config(self): def check_result(self, a, b): np.testing.assert_array_equal(a, b) - np.testing.assert_allclose(np.matmul(a.T, a), - 4 * np.eye(10), - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + np.matmul(a.T, a), 4 * np.eye(10), rtol=1e-5, atol=1e-8 + ) # 2-D Parameter with shape: [10, 10] @@ -941,7 +953,8 @@ class TestOrthogonalInitializer3(TestOrthogonalInitializer1): def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Orthogonal()) + initializer=paddle.nn.initializer.Orthogonal() + ) self.dtype = "float32" self.in_features = 10 self.out_features = 10 @@ -949,14 +962,12 @@ def config(self): def check_result(self, a, b): np.testing.assert_array_equal(a, b) - np.testing.assert_allclose(np.matmul(a.T, a), - np.eye(10), - rtol=1e-05, - atol=1e-06) - np.testing.assert_allclose(np.matmul(a, a.T), - np.eye(10), - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np.matmul(a.T, a), np.eye(10), rtol=1e-05, atol=1e-06 + ) + np.testing.assert_allclose( + np.matmul(a, a.T), np.eye(10), rtol=1e-05, atol=1e-06 + ) def test_error(self): self.config() @@ -972,7 +983,8 @@ class TestOrthogonalInitializer4(unittest.TestCase): def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Orthogonal(gain=3.0)) + initializer=paddle.nn.initializer.Orthogonal(gain=3.0) + ) self.dtype = "float64" self.in_features = 4 self.out_features = 6 @@ -981,10 +993,9 @@ def config(self): def check_result(self, a, b): np.testing.assert_array_equal(a, b) a = a.reshape(6, -1) - np.testing.assert_allclose(np.matmul(a, a.T), - 9 * np.eye(6), - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + np.matmul(a, a.T), 9 * np.eye(6), rtol=1e-5, atol=1e-8 + ) def func_orthogonal(self): self.config() @@ -992,10 +1003,12 @@ def func_orthogonal(self): paddle.disable_static() paddle.seed(2021) - conv2d = paddle.nn.Conv2D(self.in_features, - self.out_features, - self.kernel_size, - weight_attr=self.weight_attr) + conv2d = paddle.nn.Conv2D( + self.in_features, + self.out_features, + self.kernel_size, + weight_attr=self.weight_attr, + ) res_dygraph = conv2d.weight.numpy() paddle.enable_static() @@ -1003,13 +1016,17 @@ def func_orthogonal(self): start_prog = paddle.static.Program() main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, start_prog): - conv2d = paddle.nn.Conv2D(self.in_features, - self.out_features, - self.kernel_size, - weight_attr=self.weight_attr) + conv2d = paddle.nn.Conv2D( + self.in_features, + self.out_features, + self.kernel_size, + weight_attr=self.weight_attr, + ) exe = paddle.static.Executor() - res_static = exe.run(paddle.static.default_startup_program(), - fetch_list=[conv2d.weight])[0] + res_static = exe.run( + paddle.static.default_startup_program(), + fetch_list=[conv2d.weight], + )[0] self.check_result(res_dygraph, res_static) def test_orthogonal(self): @@ -1026,7 +1043,8 @@ class TestOrthogonalInitializer5(TestOrthogonalInitializer4): def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Orthogonal(gain=2.0)) + initializer=paddle.nn.initializer.Orthogonal(gain=2.0) + ) self.dtype = "float64" self.in_features = 4 self.out_features = 50 @@ -1035,10 +1053,9 @@ def config(self): def check_result(self, a, b): np.testing.assert_array_equal(a, b) a = a.reshape(50, -1) - np.testing.assert_allclose(np.matmul(a.T, a), - 4 * np.eye(36), - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + np.matmul(a.T, a), 4 * np.eye(36), rtol=1e-5, atol=1e-8 + ) # 4-D Parameter with shape: [36, 4, 3, 3] @@ -1049,7 +1066,8 @@ class TestOrthogonalInitializer6(TestOrthogonalInitializer4): def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Orthogonal()) + initializer=paddle.nn.initializer.Orthogonal() + ) self.dtype = "float32" self.in_features = 4 self.out_features = 36 @@ -1058,29 +1076,29 @@ def config(self): def check_result(self, a, b): np.testing.assert_array_equal(a, b) a = a.reshape(36, -1) - np.testing.assert_allclose(np.matmul(a.T, a), - np.eye(36), - rtol=1e-05, - atol=1e-06) - np.testing.assert_allclose(np.matmul(a, a.T), - np.eye(36), - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np.matmul(a.T, a), np.eye(36), rtol=1e-05, atol=1e-06 + ) + np.testing.assert_allclose( + np.matmul(a, a.T), np.eye(36), rtol=1e-05, atol=1e-06 + ) # initialize Conv1D weight class TestDiracInitializer1(unittest.TestCase): - def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Dirac()) + initializer=paddle.nn.initializer.Dirac() + ) self.dtype = "float64" self.in_channels = 3 self.out_channels = 2 self.kernel_size = 3 self.input_shape = [8, self.in_channels, 10] self.conv_layer = paddle.nn.Conv1D - self.num_ops = 8 #fill_constant*2, reshape*2, assign_value*2, scatter, cast + self.num_ops = ( + 8 # fill_constant*2, reshape*2, assign_value*2, scatter, cast + ) def check_result(self, w_dygraph, w_static, conv_in, conv_out): np.testing.assert_array_equal(w_dygraph, w_static) @@ -1091,10 +1109,12 @@ def func_dirac(self): paddle.set_default_dtype(self.dtype) paddle.disable_static() - conv = self.conv_layer(self.in_channels, - self.out_channels, - self.kernel_size, - weight_attr=self.weight_attr) + conv = self.conv_layer( + self.in_channels, + self.out_channels, + self.kernel_size, + weight_attr=self.weight_attr, + ) weight_dygraph = conv.weight.numpy() paddle.enable_static() @@ -1102,10 +1122,12 @@ def func_dirac(self): main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, start_prog): inp = paddle.rand(self.input_shape) - conv = self.conv_layer(self.in_channels, - self.out_channels, - self.kernel_size, - weight_attr=self.weight_attr) + conv = self.conv_layer( + self.in_channels, + self.out_channels, + self.kernel_size, + weight_attr=self.weight_attr, + ) output = conv(inp) block = start_prog.global_block() @@ -1124,8 +1146,9 @@ def func_dirac(self): conv_output = fetch[1] weight_static = fetch[2] - self.check_result(weight_dygraph, weight_static, conv_input, - conv_output) + self.check_result( + weight_dygraph, weight_static, conv_input, conv_output + ) def test_dirac(self): with framework._test_eager_guard(): @@ -1135,10 +1158,10 @@ def test_dirac(self): # initialize Conv2D weight class TestDiracInitializer2(TestDiracInitializer1): - def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Dirac(groups=1)) + initializer=paddle.nn.initializer.Dirac(groups=1) + ) self.dtype = "float64" self.in_channels = 4 self.out_channels = 8 @@ -1149,18 +1172,20 @@ def config(self): def check_result(self, w_dygraph, w_static, conv_in, conv_out): np.testing.assert_array_equal(w_dygraph, w_static) - np.testing.assert_array_equal(conv_out[:, 0:4, :, :], conv_in[:, :, 1:9, - 1:9]) - np.testing.assert_array_equal(conv_out[:, 4:8, :, :], - np.zeros([8, 4, 8, 8])) + np.testing.assert_array_equal( + conv_out[:, 0:4, :, :], conv_in[:, :, 1:9, 1:9] + ) + np.testing.assert_array_equal( + conv_out[:, 4:8, :, :], np.zeros([8, 4, 8, 8]) + ) # initialize Conv3D weight class TestDiracInitializer3(TestDiracInitializer1): - def config(self): self.weight_attr = paddle.ParamAttr( - initializer=paddle.nn.initializer.Dirac(groups=2)) + initializer=paddle.nn.initializer.Dirac(groups=2) + ) self.dtype = "float32" self.in_channels = 5 self.out_channels = 10 @@ -1171,10 +1196,12 @@ def config(self): def check_result(self, w_dygraph, w_static, conv_in, conv_out): np.testing.assert_array_equal(w_dygraph, w_static) - np.testing.assert_array_equal(conv_out[:, 0:5, :, :, :], - conv_in[:, :, 1:9, 1:9, 1:9]) - np.testing.assert_array_equal(conv_out[:, 5:10, :, :, :], - conv_in[:, :, 1:9, 1:9, 1:9]) + np.testing.assert_array_equal( + conv_out[:, 0:5, :, :, :], conv_in[:, :, 1:9, 1:9, 1:9] + ) + np.testing.assert_array_equal( + conv_out[:, 5:10, :, :, :], conv_in[:, :, 1:9, 1:9, 1:9] + ) def test_error(self): self.config() diff --git a/python/paddle/fluid/tests/unittests/test_initializer_nn.py b/python/paddle/fluid/tests/unittests/test_initializer_nn.py index 54063f74a6e84..c9b81720ba94b 100644 --- a/python/paddle/fluid/tests/unittests/test_initializer_nn.py +++ b/python/paddle/fluid/tests/unittests/test_initializer_nn.py @@ -32,26 +32,28 @@ def get_uniform_min_and_max(weight): def check_cast_op(op): - return op.type == 'cast' and \ - op.attr('in_dtype') == VarDesc.VarType.FP32 and \ - op.attr('out_dtype') in [VarDesc.VarType.FP16, VarDesc.VarType.BF16] + return ( + op.type == 'cast' + and op.attr('in_dtype') == VarDesc.VarType.FP32 + and op.attr('out_dtype') in [VarDesc.VarType.FP16, VarDesc.VarType.BF16] + ) class TestConstantInitializer(unittest.TestCase): - - def static_test_constant_initializer_common(self, - init_inst, - dtype="float32", - value_target=0.0): + def static_test_constant_initializer_common( + self, init_inst, dtype="float32", value_target=0.0 + ): paddle.enable_static() program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=init_inst) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=init_inst, + ) num_ops = 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -61,46 +63,45 @@ def static_test_constant_initializer_common(self, return block def test_constant_initializer_default_value_static(self, dtype="float32"): - """Test the constant initializer with default value in static graph - """ + """Test the constant initializer with default value in static graph""" block = self.static_test_constant_initializer_common( - init_inst=initializer.Constant(), dtype=dtype, value_target=0.0) + init_inst=initializer.Constant(), dtype=dtype, value_target=0.0 + ) return block def test_constant_initializer_default_value_dygraph(self, dtype="float32"): - """Test constant initializer with supplied value in dygraph - """ + """Test constant initializer with supplied value in dygraph""" with fluid.dygraph.guard(): linear = nn.Linear(2, 4, weight_attr=nn.initializer.Constant()) mat_target = np.ones((2, 4), dtype=dtype) * 0.0 mat_linear = linear.weight.numpy() mismatch = np.sum( - (mat_target - mat_linear) * (mat_target - mat_linear)) + (mat_target - mat_linear) * (mat_target - mat_linear) + ) self.assertAlmostEqual(mismatch, 0.0, delta=DELTA) def test_constant_initializer_static(self, dtype="float32"): - """Test constant initializer with supplied value in static graph - """ + """Test constant initializer with supplied value in static graph""" block = self.static_test_constant_initializer_common( - init_inst=initializer.Constant(2.3), dtype=dtype, value_target=2.3) + init_inst=initializer.Constant(2.3), dtype=dtype, value_target=2.3 + ) return block def test_constant_initializer_dygraph(self, dtype="float32"): - """Test constant initializer with supplied value in dygraph - """ + """Test constant initializer with supplied value in dygraph""" with fluid.dygraph.guard(): - linear = nn.Linear(2, - 4, - weight_attr=nn.initializer.Constant(value=2.0)) + linear = nn.Linear( + 2, 4, weight_attr=nn.initializer.Constant(value=2.0) + ) mat_target = np.ones((2, 4), dtype=dtype) * 2.0 mat_linear = linear.weight.numpy() mismatch = np.sum( - (mat_target - mat_linear) * (mat_target - mat_linear)) + (mat_target - mat_linear) * (mat_target - mat_linear) + ) self.assertAlmostEqual(mismatch, 0.0, delta=DELTA) def test_constant_initializer_fp16(self): - """Test constant initializer with float16 - """ + """Test constant initializer with float16""" block = self.test_constant_initializer_default_value_static("float16") block = self.test_constant_initializer_static("float16") self.test_constant_initializer_default_value_dygraph("float16") @@ -108,29 +109,30 @@ def test_constant_initializer_fp16(self): def test_constant_initializer_bf16(self): """Test constant initializer with bfloat16 - No cast operator has been added here + No cast operator has been added here """ - self.test_constant_initializer_default_value_static("uint16") #bfloat16 - self.test_constant_initializer_static("uint16") #bfloat16 + self.test_constant_initializer_default_value_static( + "uint16" + ) # bfloat16 + self.test_constant_initializer_static("uint16") # bfloat16 class TestKaimingInitializer(unittest.TestCase): - - def static_test_kaiming_initializer_common(self, - init_inst, - dtype="float32", - uniform=False, - is_conv=False): + def static_test_kaiming_initializer_common( + self, init_inst, dtype="float32", uniform=False, is_conv=False + ): paddle.enable_static() program = framework.Program() block = program.global_block() shape_mat = [5, 10, 15, 20] if is_conv else [5, 10] for _ in range(2): - param = block.create_parameter(dtype="float32", - shape=shape_mat, - lod_level=0, - name="param", - initializer=init_inst) + param = block.create_parameter( + dtype="float32", + shape=shape_mat, + lod_level=0, + name="param", + initializer=init_inst, + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] if uniform: @@ -153,75 +155,76 @@ def static_test_kaiming_initializer_common(self, self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA) paddle.disable_static() - def dygraph_test_kaiming_initializer_common(self, - init_inst, - dtype="float32", - uniform=False): + def dygraph_test_kaiming_initializer_common( + self, init_inst, dtype="float32", uniform=False + ): linear = nn.Linear(40, 20, weight_attr=init_inst) def test_kaiming_dygraph(self): self.dygraph_test_kaiming_initializer_common( init_inst=initializer.KaimingUniform(), dtype="float32", - uniform=True) + uniform=True, + ) self.dygraph_test_kaiming_initializer_common( init_inst=initializer.KaimingNormal(), dtype="float32", - uniform=False) + uniform=False, + ) def test_kaiming_uniform_initializer_static(self): - """Test Kaiming unorm initializer for matrix multiply. - """ + """Test Kaiming unorm initializer for matrix multiply.""" self.static_test_kaiming_initializer_common( init_inst=initializer.KaimingUniform(), dtype="float32", uniform=True, - is_conv=False) + is_conv=False, + ) def test_kaiming_uniform_initializer_conv_static(self): - """Test Kaiming unorm initializer for convolutions. - """ + """Test Kaiming unorm initializer for convolutions.""" self.static_test_kaiming_initializer_common( init_inst=initializer.KaimingUniform(), dtype="float32", uniform=True, - is_conv=True) + is_conv=True, + ) def test_kaiming_normal_initializer_static(self): - """Test Kaiming normal initializer for matrix multiply. - """ + """Test Kaiming normal initializer for matrix multiply.""" self.static_test_kaiming_initializer_common( init_inst=initializer.KaimingNormal(), dtype="float32", uniform=False, - is_conv=False) + is_conv=False, + ) def test_kaiming_normal_initializer_conv_static(self): - """Test Kaiming normal initializer for convolutions. - """ + """Test Kaiming normal initializer for convolutions.""" self.static_test_kaiming_initializer_common( init_inst=initializer.KaimingNormal(), dtype="float32", uniform=False, - is_conv=True) + is_conv=True, + ) class TestUniform(unittest.TestCase): - def test_uniform_common(self, dtype="float32", seed=0): - """Test the uniform initializer with default value - """ + """Test the uniform initializer with default value""" paddle.enable_static() program = framework.Program() program.random_seed = seed block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.Uniform()) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Uniform(), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -234,24 +237,23 @@ def test_uniform_common(self, dtype="float32", seed=0): return block - def test_uniform_initializer_default_value(self, - dtype="float32", - seed=0, - min_value=-1.0, - max_vlaue=1.0): - """Test the uniform initializer with default value - """ + def test_uniform_initializer_default_value( + self, dtype="float32", seed=0, min_value=-1.0, max_vlaue=1.0 + ): + """Test the uniform initializer with default value""" paddle.enable_static() program = framework.Program() program.random_seed = seed block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.Uniform()) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Uniform(), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -264,25 +266,23 @@ def test_uniform_initializer_default_value(self, return block - def test_uniform_initializer(self, - dtype="float32", - seed=0, - min_value=-4.2, - max_vlaue=3.1): - """Test uniform initializer with supplied attributes - """ + def test_uniform_initializer( + self, dtype="float32", seed=0, min_value=-4.2, max_vlaue=3.1 + ): + """Test uniform initializer with supplied attributes""" paddle.enable_static() program = framework.Program() program.random_seed = seed block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.Uniform( - min_value, max_vlaue)) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Uniform(min_value, max_vlaue), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -294,25 +294,23 @@ def test_uniform_initializer(self, return block - def test_uniform_initializer_two_op(self, - dtype="float32", - seed=123, - min_value=-4.2, - max_vlaue=0.0): - """Test uniform initializer with supplied attributes - """ + def test_uniform_initializer_two_op( + self, dtype="float32", seed=123, min_value=-4.2, max_vlaue=0.0 + ): + """Test uniform initializer with supplied attributes""" paddle.enable_static() program = framework.Program() program.random_seed = seed block = program.global_block() for i in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.Uniform( - min_value, float(i))) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Uniform(min_value, float(i)), + ) num_ops = 2 if dtype == "float16" else 1 self.assertEqual(len(block.ops), num_ops) init_op0 = block.ops[0] @@ -326,8 +324,7 @@ def test_uniform_initializer_two_op(self, return block def test_uniform_initializer_fp16(self): - """Test uniform initializer with float16 - """ + """Test uniform initializer with float16""" block = self.test_uniform_initializer_default_value("float16") self.assertTrue(check_cast_op(block.ops[1])) block = self.test_uniform_initializer(dtype="float16") @@ -336,44 +333,47 @@ def test_uniform_initializer_fp16(self): self.assertTrue(check_cast_op(block.ops[1])) def test_uniform_initializer_bf16(self): - """Test uniform initializer with bfloat16 - """ - block = self.test_uniform_initializer_default_value("uint16") #bfloat16 - block = self.test_uniform_initializer(dtype="uint16") #bfloat16 - block = self.test_uniform_initializer_two_op("uint16") #bfloat16 + """Test uniform initializer with bfloat16""" + block = self.test_uniform_initializer_default_value( + "uint16" + ) # bfloat16 + block = self.test_uniform_initializer(dtype="uint16") # bfloat16 + block = self.test_uniform_initializer_two_op("uint16") # bfloat16 def test_uniform_initializer_dygraph(self): - """Test uniform initializer in dygraph model. - """ + """Test uniform initializer in dygraph model.""" paddle.disable_static() weight_attr = paddle.framework.ParamAttr( name="linear_weight", - initializer=paddle.nn.initializer.Uniform(low=-0.5, high=0.5)) + initializer=paddle.nn.initializer.Uniform(low=-0.5, high=0.5), + ) linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr) min_value, max_value = get_uniform_min_and_max(linear.weight.numpy()) - self.assertTrue(min_value >= -0.5, - 'min value {} should >= -0.5'.format(min_value)) - self.assertTrue(max_value <= 0.5, - 'max value {} should <= 0.5'.format(max_value)) + self.assertTrue( + min_value >= -0.5, 'min value {} should >= -0.5'.format(min_value) + ) + self.assertTrue( + max_value <= 0.5, 'max value {} should <= 0.5'.format(max_value) + ) class TestNormal(unittest.TestCase): - def test_normal_initializer_default_value(self): - """Test the normal initializer with default value - """ + """Test the normal initializer with default value""" paddle.enable_static() program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.Normal()) + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Normal(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -384,18 +384,19 @@ def test_normal_initializer_default_value(self): paddle.disable_static() def test_normal_initializer(self, dtype="float32"): - """Test normal initializer with supplied attributes - """ + """Test normal initializer with supplied attributes""" paddle.enable_static() program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.Normal(2.3, 1.9)) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Normal(2.3, 1.9), + ) num_ops = 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -408,41 +409,39 @@ def test_normal_initializer(self, dtype="float32"): return block def test_normal_initializer_fp16(self): - """Test normal initializer with float16 - """ + """Test normal initializer with float16""" block = self.test_normal_initializer("float16") def test_normal_initializer_bf16(self): - """Test normal initializer with bfloat16 - """ - block = self.test_normal_initializer("uint16") #bfloat16 + """Test normal initializer with bfloat16""" + block = self.test_normal_initializer("uint16") # bfloat16 def test_normal_initializer_dygraph(self): - """Test normal initializer in dygraph model. - """ + """Test normal initializer in dygraph model.""" paddle.disable_static() weight_attr = paddle.framework.ParamAttr( name="linear_weight", - initializer=paddle.nn.initializer.Normal(mean=0.0, std=2.0)) + initializer=paddle.nn.initializer.Normal(mean=0.0, std=2.0), + ) linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr) class TestTruncatedNormal(unittest.TestCase): - def test_truncated_normal_initializer_default_value(self): - """Test the truncated normal initializer with default value - """ + """Test the truncated normal initializer with default value""" paddle.enable_static() program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.TruncatedNormal()) + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.TruncatedNormal(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'truncated_gaussian_random') @@ -453,19 +452,19 @@ def test_truncated_normal_initializer_default_value(self): paddle.disable_static() def test_truncated_normal_initializer(self, dtype="float32"): - """Test truncated normal initializer with supplied attributes - """ + """Test truncated normal initializer with supplied attributes""" paddle.enable_static() program = framework.Program() block = program.global_block() for _ in range(2): - block.create_parameter(dtype=dtype, - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.TruncatedNormal( - 2.3, 1.9)) + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.TruncatedNormal(2.3, 1.9), + ) num_ops = 2 if dtype in ["float16", "uint16"] else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -478,38 +477,36 @@ def test_truncated_normal_initializer(self, dtype="float32"): return block def test_truncated_normal_initializer_fp16(self): - """Test truncated normal initializer with float16 - """ + """Test truncated normal initializer with float16""" paddle.enable_static() block = self.test_truncated_normal_initializer("float16") self.assertTrue(check_cast_op(block.ops[1])) def test_truncated_normal_initializer_bf16(self): - """Test truncated normal initializer with bfloat16 - """ + """Test truncated normal initializer with bfloat16""" paddle.enable_static() - block = self.test_truncated_normal_initializer("uint16") #bfloat16 + block = self.test_truncated_normal_initializer("uint16") # bfloat16 self.assertTrue(check_cast_op(block.ops[1])) def test_truncated_normal_initializer_dygraph(self): - """Test truncated normal initializer in dygraph model. - """ + """Test truncated normal initializer in dygraph model.""" paddle.disable_static() weight_attr = paddle.framework.ParamAttr( name="linear_weight", - initializer=paddle.nn.initializer.TruncatedNormal(mean=0.0, - std=2.0)) + initializer=paddle.nn.initializer.TruncatedNormal( + mean=0.0, std=2.0 + ), + ) linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr) class TestXavierUniform(unittest.TestCase): - def test_xavier_uniform_initializer(self): """Test Xavier initializer with uniform distribution on - for matrix multiply. + for matrix multiply. """ paddle.enable_static() @@ -521,7 +518,8 @@ def test_xavier_uniform_initializer(self): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.XavierUniform()) + initializer=initializer.XavierUniform(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -534,7 +532,7 @@ def test_xavier_uniform_initializer(self): def test_xavier_uniform_initializer_conv(self): """Test Xavier initializer with uniform distribution on - for convolutions. + for convolutions. """ paddle.enable_static() @@ -546,33 +544,34 @@ def test_xavier_uniform_initializer_conv(self): shape=[5, 10, 15, 20], lod_level=0, name="param", - initializer=initializer.XavierUniform()) + initializer=initializer.XavierUniform(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') receptive_field_size = float(15 * 20) limit = np.sqrt( - 6.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size)) + 6.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size) + ) self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) self.assertEqual(init_op.attr('seed'), 0) def test_xavier_uniform_initializer_dygraph(self): - """Test xavier uniform initializer in dygraph model. - """ + """Test xavier uniform initializer in dygraph model.""" paddle.disable_static() weight_attr = paddle.framework.ParamAttr( name="linear_weight", - initializer=paddle.nn.initializer.XavierUniform()) + initializer=paddle.nn.initializer.XavierUniform(), + ) linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr) class TestXavierNormal(unittest.TestCase): - def test_xavier_normal_initializer(self): """Test Xavier initializer with normal distribution on - for matrix multiply. + for matrix multiply. """ paddle.enable_static() @@ -584,7 +583,8 @@ def test_xavier_normal_initializer(self): shape=[5, 10], lod_level=0, name="param", - initializer=initializer.XavierNormal()) + initializer=initializer.XavierNormal(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -597,7 +597,7 @@ def test_xavier_normal_initializer(self): def test_xavier_normal_initializer_conv(self): """Test Xavier initializer with normal distribution on - for convolutions. + for convolutions. """ paddle.enable_static() @@ -609,13 +609,15 @@ def test_xavier_normal_initializer_conv(self): shape=[5, 10, 15, 20], lod_level=0, name="param", - initializer=initializer.XavierNormal()) + initializer=initializer.XavierNormal(), + ) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') receptive_field_size = float(15 * 20) std = np.sqrt( - 2.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size)) + 2.0 / ((param.shape[0] + param.shape[1]) * receptive_field_size) + ) self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA) self.assertEqual(init_op.attr('seed'), 0) @@ -623,33 +625,34 @@ def test_xavier_normal_initializer_conv(self): paddle.disable_static() def test_xavier_normal_initializer_dygraph(self): - """Test xavier normal initializer in dygraph model. - """ + """Test xavier normal initializer in dygraph model.""" paddle.disable_static() weight_attr = paddle.framework.ParamAttr( name="linear_weight", - initializer=paddle.nn.initializer.XavierNormal()) + initializer=paddle.nn.initializer.XavierNormal(), + ) linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr) class TestAssign(unittest.TestCase): - def test_assign_initializer(self, dtype="float32"): - """Test the numpy array initializer with supplied arguments - """ + """Test the numpy array initializer with supplied arguments""" paddle.enable_static() import numpy + program = framework.Program() block = program.global_block() np_array = numpy.random.random((10000)).astype(dtype) for _ in range(2): - block.create_parameter(dtype=np_array.dtype, - shape=np_array.shape, - lod_level=0, - name="param", - initializer=initializer.Assign(np_array)) + block.create_parameter( + dtype=np_array.dtype, + shape=np_array.shape, + lod_level=0, + name="param", + initializer=initializer.Assign(np_array), + ) num_ops = 2 if dtype in ["float16", "uint16"] else 1 self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] @@ -661,61 +664,59 @@ def test_assign_initializer(self, dtype="float32"): return block def test_assign_initializer_fp16(self): - """Test the numpy array initializer with float16 - """ + """Test the numpy array initializer with float16""" block = self.test_assign_initializer("float16") self.assertTrue(block.ops[1]) def test_assign_initializer_bf16(self): - """Test the numpy array initializer with bfloat16 - """ - block = self.test_assign_initializer("uint16") #bfloat16 + """Test the numpy array initializer with bfloat16""" + block = self.test_assign_initializer("uint16") # bfloat16 self.assertTrue(block.ops[1]) def test_assign_initializer_dygraph_1(self): - """Test assign initializer in dygraph model. - """ + """Test assign initializer in dygraph model.""" paddle.disable_static() weight_attr_1 = paddle.framework.ParamAttr( name="linear_weight_1", - initializer=paddle.nn.initializer.Assign(np.array([2, 2]))) + initializer=paddle.nn.initializer.Assign(np.array([2, 2])), + ) linear_1 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_1) self.assertTrue((linear_1.weight.numpy() == [2.0, 2.0]).all(), '') def test_assign_initializer_dygraph_2(self): - """Test assign initializer in dygraph model. - """ + """Test assign initializer in dygraph model.""" paddle.disable_static() weight_attr_2 = paddle.framework.ParamAttr( name="linear_weight_2", - initializer=paddle.nn.initializer.Assign([2, 2])) + initializer=paddle.nn.initializer.Assign([2, 2]), + ) linear_2 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_2) self.assertTrue((linear_2.weight.numpy() == [2.0, 2.0]).all(), '') def test_assign_initializer_dygraph_3(self): - """Test assign initializer in dygraph model. - """ + """Test assign initializer in dygraph model.""" paddle.disable_static() weight_attr_3 = paddle.framework.ParamAttr( name="linear_weight_3", - initializer=paddle.nn.initializer.Assign(paddle.full([2], 2))) + initializer=paddle.nn.initializer.Assign(paddle.full([2], 2)), + ) linear_3 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_3) self.assertTrue((linear_3.weight.numpy() == [2.0, 2.0]).all(), '') def test_assign_initializer_dygraph_4(self): - """Test assign initializer in dygraph model. - """ + """Test assign initializer in dygraph model.""" paddle.disable_static() weight_attr_4 = paddle.framework.ParamAttr( name="linear_weight_4", - initializer=paddle.nn.initializer.Assign((2, 2))) + initializer=paddle.nn.initializer.Assign((2, 2)), + ) linear_4 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_4) self.assertTrue((linear_4.weight.numpy() == [2.0, 2.0]).all(), '') diff --git a/python/paddle/fluid/tests/unittests/test_inner.py b/python/paddle/fluid/tests/unittests/test_inner.py index cf631101e8ee8..3e30e9fa2284b 100644 --- a/python/paddle/fluid/tests/unittests/test_inner.py +++ b/python/paddle/fluid/tests/unittests/test_inner.py @@ -22,27 +22,28 @@ class TestMultiplyApi(unittest.TestCase): - def _run_static_graph_case(self, x_data, y_data): with program_guard(Program(), Program()): paddle.enable_static() - x = paddle.static.data(name='x', - shape=x_data.shape, - dtype=x_data.dtype) - y = paddle.static.data(name='y', - shape=y_data.shape, - dtype=y_data.dtype) + x = paddle.static.data( + name='x', shape=x_data.shape, dtype=x_data.dtype + ) + y = paddle.static.data( + name='y', shape=y_data.shape, dtype=y_data.dtype + ) res = paddle.inner(x, y) - place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) - outs = exe.run(paddle.static.default_main_program(), - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[res]) + outs = exe.run( + paddle.static.default_main_program(), + feed={'x': x_data, 'y': y_data}, + fetch_list=[res], + ) res = outs[0] return res @@ -94,17 +95,21 @@ def func_test_multiply(self): # test dynamic computation graph: 2-d array Complex x_data = np.random.rand(20, 50).astype( - np.float64) + 1J * np.random.rand(20, 50).astype(np.float64) - y_data = np.random.rand(50).astype( - np.float64) + 1J * np.random.rand(50).astype(np.float64) + np.float64 + ) + 1j * np.random.rand(20, 50).astype(np.float64) + y_data = np.random.rand(50).astype(np.float64) + 1j * np.random.rand( + 50 + ).astype(np.float64) res = self._run_dynamic_graph_case(x_data, y_data) np.testing.assert_allclose(res, np.inner(x_data, y_data), rtol=1e-05) # test dynamic computation graph: 3-d array Complex x_data = np.random.rand(5, 10, 10).astype( - np.float64) + 1J * np.random.rand(5, 10, 10).astype(np.float64) - y_data = np.random.rand(2, 10).astype( - np.float64) + 1J * np.random.rand(2, 10).astype(np.float64) + np.float64 + ) + 1j * np.random.rand(5, 10, 10).astype(np.float64) + y_data = np.random.rand(2, 10).astype(np.float64) + 1j * np.random.rand( + 2, 10 + ).astype(np.float64) res = self._run_dynamic_graph_case(x_data, y_data) np.testing.assert_allclose(res, np.inner(x_data, y_data), rtol=1e-05) @@ -115,7 +120,6 @@ def test_multiply(self): class TestMultiplyError(unittest.TestCase): - def func_test_errors(self): # test static computation graph: dtype can not be int8 paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_inplace.py b/python/paddle/fluid/tests/unittests/test_inplace.py index 0d88a35ff2453..2496ee37a05e4 100644 --- a/python/paddle/fluid/tests/unittests/test_inplace.py +++ b/python/paddle/fluid/tests/unittests/test_inplace.py @@ -20,7 +20,6 @@ class TestInplace(unittest.TestCase): - def func_test_forward_version(self): with paddle.fluid.dygraph.guard(): var = paddle.to_tensor(np.ones((4, 2, 3)).astype(np.float32)) @@ -60,9 +59,11 @@ def func_test_backward_error(self): loss = paddle.nn.functional.relu(var_c + var_d) with self.assertRaisesRegexp( - RuntimeError, - "received tensor_version:{} != wrapper_version_snapshot:{}". - format(1, 0)): + RuntimeError, + "received tensor_version:{} != wrapper_version_snapshot:{}".format( + 1, 0 + ), + ): loss.backward() def test_backward_error(self): @@ -101,7 +102,9 @@ def func_test_backward_success_2(self): var_b[1:2] = 3 # var_b is modified inplace before using it - var_c = var_b + var_b # Here, the grad op of sum doesn't use the value of var_b + var_c = ( + var_b + var_b + ) # Here, the grad op of sum doesn't use the value of var_b loss = var_c.sum() var_b[1:2] = 3 # var_b is modified inplace after using it @@ -115,7 +118,6 @@ def test_backward_success_2(self): class TestDygraphInplace(unittest.TestCase): - def setUp(self): self.init_data() self.set_np_compare_func() @@ -138,7 +140,7 @@ def func_test_inplace_api(self): inplace_var = self.inplace_api_processing(var) self.assertTrue(id(var) == id(inplace_var)) - inplace_var[0] = 2. + inplace_var[0] = 2.0 np.testing.assert_array_equal(var.numpy(), inplace_var.numpy()) def test_inplace_api(self): @@ -154,7 +156,7 @@ def func_test_forward_version(self): inplace_var = self.inplace_api_processing(var) self.assertEqual(var.inplace_version, 1) - inplace_var[0] = 2. + inplace_var[0] = 2.0 self.assertEqual(var.inplace_version, 2) inplace_var = self.inplace_api_processing(inplace_var) @@ -195,9 +197,11 @@ def func_test_backward_error(self): loss = paddle.nn.functional.relu(var_c) with self.assertRaisesRegexp( - RuntimeError, - "received tensor_version:{} != wrapper_version_snapshot:{}". - format(1, 0)): + RuntimeError, + "received tensor_version:{} != wrapper_version_snapshot:{}".format( + 1, 0 + ), + ): loss.backward() def test_backward_error(self): @@ -215,7 +219,8 @@ def func_test_backward_success_1(self): var_b = var_a**2 var_c = self.inplace_api_processing( - var_b) # var_b is modified inplace before using it + var_b + ) # var_b is modified inplace before using it # Here, the gradient computation will use the value of var_b var_d = var_c**2 @@ -252,9 +257,12 @@ def func_test_backward_success_2(self): var_b = var_a**2 var_c = self.inplace_api_processing( - var_b) # var_b is modified inplace before using it + var_b + ) # var_b is modified inplace before using it - var_d = var_c + var_c # Here, the grad op of sum doesn't use the value of var_b + var_d = ( + var_c + var_c + ) # Here, the grad op of sum doesn't use the value of var_b loss = var_d.sum() loss.backward() @@ -268,7 +276,9 @@ def func_test_backward_success_2(self): var_c = self.non_inplace_api_processing(var_b) - var_d = var_c + var_c # Here, the grad op of sum doesn't use the value of var_b + var_d = ( + var_c + var_c + ) # Here, the grad op of sum doesn't use the value of var_b loss = var_d.sum() loss.backward() @@ -282,7 +292,6 @@ def test_backward_success_2(self): class TestDygraphInplaceUnsqueeze(TestDygraphInplace): - def non_inplace_api_processing(self, var): return paddle.unsqueeze(var, -1) @@ -291,7 +300,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceReshape(TestDygraphInplace): - def non_inplace_api_processing(self, var): return paddle.reshape(var, [-1]) @@ -300,7 +308,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceReshapeTensor(TestDygraphInplace): - def non_inplace_api_processing(self, var): shape = paddle.to_tensor(-1) return paddle.reshape(var, shape) @@ -311,7 +318,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceFlatten(TestDygraphInplace): - def non_inplace_api_processing(self, var): return var.flatten() @@ -320,28 +326,28 @@ def inplace_api_processing(self, var): class TestDygraphInplaceScatter(TestDygraphInplace): - def init_data(self): self.input_var_numpy = np.array([[1, 1], [2, 2], [3, 3]]) self.dtype = "float32" def non_inplace_api_processing(self, var): index = paddle.to_tensor([2, 1, 0, 1], dtype='int64') - updates = paddle.to_tensor([[1, 1], [2, 2], [3, 3], [4, 4]], - dtype='float32') + updates = paddle.to_tensor( + [[1, 1], [2, 2], [3, 3], [4, 4]], dtype='float32' + ) return paddle.scatter(var, index, updates, overwrite=False) def inplace_api_processing(self, var): index = paddle.to_tensor([2, 1, 0, 1], dtype='int64') - updates = paddle.to_tensor([[1, 1], [2, 2], [3, 3], [4, 4]], - dtype='float32') + updates = paddle.to_tensor( + [[1, 1], [2, 2], [3, 3], [4, 4]], dtype='float32' + ) return paddle.scatter_(var, index, updates, overwrite=False) class TestDygraphInplaceElu(TestDygraphInplace): - def non_inplace_api_processing(self, var): return paddle.nn.functional.elu(var) @@ -350,7 +356,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceRelu(TestDygraphInplace): - def non_inplace_api_processing(self, var): return paddle.nn.functional.relu(var) @@ -359,7 +364,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceSoftmax(TestDygraphInplace): - def non_inplace_api_processing(self, var): return paddle.nn.functional.softmax(var) @@ -368,7 +372,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceTanh(TestDygraphInplace): - def non_inplace_api_processing(self, var): return paddle.tanh(var) @@ -377,7 +380,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceCeil(TestDygraphInplace): - def non_inplace_api_processing(self, var): return var.ceil() @@ -386,7 +388,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceFloor(TestDygraphInplace): - def non_inplace_api_processing(self, var): return var.floor() @@ -395,7 +396,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceExp(TestDygraphInplace): - def set_np_compare_func(self): self.np_compare = np.allclose @@ -407,7 +407,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceReciprocal(TestDygraphInplace): - def non_inplace_api_processing(self, var): return var.reciprocal() @@ -416,7 +415,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceRound(TestDygraphInplace): - def non_inplace_api_processing(self, var): return var.round() @@ -425,7 +423,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceSqrt(TestDygraphInplace): - def init_data(self): self.input_var_numpy = np.random.uniform(0, 5, [10, 20, 1]) self.dtype = "float32" @@ -438,7 +435,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceRsqrt(TestDygraphInplaceSqrt): - def non_inplace_api_processing(self, var): return var.rsqrt() @@ -447,7 +443,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceClip(TestDygraphInplace): - def non_inplace_api_processing(self, var): return var.clip(0.6, 1.5) @@ -456,7 +451,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceScale(TestDygraphInplace): - def non_inplace_api_processing(self, var): return var.scale(scale=2.0, bias=3.0) @@ -465,7 +459,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceAdd(TestDygraphInplace): - def init_data(self): self.input_var_numpy = np.random.rand(2, 3, 4) self.dtype = "float32" @@ -481,7 +474,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceSubtract(TestDygraphInplaceAdd): - def non_inplace_api_processing(self, var): input_var_2 = paddle.to_tensor(self.input_var_numpy_2) return var.subtract(input_var_2) @@ -492,7 +484,6 @@ def inplace_api_processing(self, var): class TestDygraphInplaceRemainder(TestDygraphInplaceAdd): - def non_inplace_api_processing(self, var): input_var_2 = paddle.to_tensor(self.input_var_numpy_2) return var.remainder(input_var_2) @@ -515,7 +506,6 @@ def test_backward_success_2(self): class TestLossIsInplaceVar(unittest.TestCase): - def func_test_loss_is_inplace_var(self): with paddle.fluid.dygraph.guard(): var_a = paddle.ones((2, 2)) @@ -546,7 +536,6 @@ def test_loss_is_inplace_var(self): class TestContinuouslyInplace(unittest.TestCase): - def func_test_continuously_inplace(self): a = paddle.rand([2, 3]) a.stop_gradient = False @@ -565,7 +554,6 @@ def test_continuously_inplace(self): class TestGetitemBeforeInplace(unittest.TestCase): - def test_getitem_before_inplace(self): with _test_eager_guard(): a = paddle.ones(shape=[4, 2, 3], dtype="float32") diff --git a/python/paddle/fluid/tests/unittests/test_inplace_abn_op.py b/python/paddle/fluid/tests/unittests/test_inplace_abn_op.py index 998480a9193de..1048c6710d27c 100644 --- a/python/paddle/fluid/tests/unittests/test_inplace_abn_op.py +++ b/python/paddle/fluid/tests/unittests/test_inplace_abn_op.py @@ -22,7 +22,6 @@ class TestInplaceANBOpTraining(unittest.TestCase): - def setUp(self): self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 self.N = 4 @@ -31,26 +30,30 @@ def setUp(self): self.W = 9 self.dshape = [self.N, self.C, self.H, self.W] - def build_program(self, - place, - layout, - seed, - only_forward=False, - activation="identity", - alpha=1.0, - use_cuda=False, - inplace=False): + def build_program( + self, + place, + layout, + seed, + only_forward=False, + activation="identity", + alpha=1.0, + use_cuda=False, + inplace=False, + ): main = fluid.Program() startup = fluid.Program() main.random_seed = seed startup.random_seed = seed with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - data = fluid.layers.data(name='input', - shape=self.dshape, - dtype=self.dtype, - append_batch_size=False, - stop_gradient=False) + data = fluid.layers.data( + name='input', + shape=self.dshape, + dtype=self.dtype, + append_batch_size=False, + stop_gradient=False, + ) if inplace: bn = fluid.layers.inplace_abn( data, @@ -61,7 +64,8 @@ def build_program(self, moving_variance_name='bn_moving_variance', data_layout=layout, is_test=only_forward, - act_alpha=alpha) + act_alpha=alpha, + ) else: bn = fluid.layers.batch_norm( data, @@ -71,7 +75,8 @@ def build_program(self, moving_variance_name='bn_moving_variance', data_layout=layout, is_test=only_forward, - in_place=inplace) + in_place=inplace, + ) if activation == 'leaky_relu': bn = fluid.layers.leaky_relu(bn, alpha) if activation == 'elu': @@ -80,7 +85,7 @@ def build_program(self, # NOTE: in inplace mode input and output of bn # may have same name, multiply 1. to generate # a new Variable for fetch - bn = bn * 1. + bn = bn * 1.0 sigmoid = fluid.layers.sigmoid(bn) out = fluid.layers.reduce_sum(sigmoid) @@ -92,23 +97,28 @@ def build_program(self, def compare(self, place, layout, only_forward, activation, alpha, use_cuda): seed = 10 os.environ['FLAGS_cudnn_deterministic'] = "1" - data = np.random.random(size=self.dshape).astype(self.dtype) * 4. - 2 + data = np.random.random(size=self.dshape).astype(self.dtype) * 4.0 - 2 fetch_outs = [] fetch_names = [] for inplace in [False, True]: - main, startup, outs = self.build_program(place, - layout, - seed, - only_forward, - activation, - alpha, - inplace=inplace) + main, startup, outs = self.build_program( + place, + layout, + seed, + only_forward, + activation, + alpha, + inplace=inplace, + ) exe = fluid.Executor(place) exe.run(startup) fetch_name = [v.name for v in outs] + [ - 'bn_moving_mean', 'bn_moving_variance', 'bn_scale', 'bn_bias' + 'bn_moving_mean', + 'bn_moving_variance', + 'bn_scale', + 'bn_bias', ] if not only_forward: others = [ @@ -124,50 +134,69 @@ def compare(self, place, layout, only_forward, activation, alpha, use_cuda): fv.persistable = True build_strategy = fluid.BuildStrategy() - build_strategy.sync_batch_norm = use_cuda and \ - fluid.core.get_cuda_device_count() > 1 + build_strategy.sync_batch_norm = ( + use_cuda and fluid.core.get_cuda_device_count() > 1 + ) build_strategy.enable_inplace = inplace exec_strategy = fluid.ExecutionStrategy() exec_strategy.num_threads = 1 if os.name == 'nt' else 0 comp_prog1 = compiler.CompiledProgram(main).with_data_parallel( outs[0].name if not only_forward else None, build_strategy=build_strategy, - exec_strategy=exec_strategy) - bn_fetches = exe.run(program=main, - feed={'input': data}, - fetch_list=fetch_name) + exec_strategy=exec_strategy, + ) + bn_fetches = exe.run( + program=main, feed={'input': data}, fetch_list=fetch_name + ) fetch_outs.append(bn_fetches) fetch_names.append(fetch_name) - for bn_val, inplace_abn_val, name1, name2 in zip(*(fetch_outs + - fetch_names)): + for bn_val, inplace_abn_val, name1, name2 in zip( + *(fetch_outs + fetch_names) + ): np.testing.assert_allclose( bn_val, inplace_abn_val, rtol=1e-05, atol=0.01, - err_msg='Output (' + name1 + ':' + name2 + - ') has diff on {} with {} layout and {} activation. \n'.format( - place, layout, activation) + '\nBN ' + str(bn_val) + - '\n' + 'Inplace ABN ' + str(inplace_abn_val)) + err_msg='Output (' + + name1 + + ':' + + name2 + + ') has diff on {} with {} layout and {} activation. \n'.format( + place, layout, activation + ) + + '\nBN ' + + str(bn_val) + + '\n' + + 'Inplace ABN ' + + str(inplace_abn_val), + ) def test_op(self): use_cudas = [False, True] if core.is_compiled_with_cuda() else [False] - #use_cudas = [False] + # use_cudas = [False] for use_cuda in use_cudas: place = core.CUDAPlace(0) if use_cuda else core.CPUPlace() layouts = ["NCHW", "NHWC"] for layout in layouts: - for activation, alpha in zip([None, 'elu', 'leaky_relu'], - [0., 1., 0.02]): + for activation, alpha in zip( + [None, 'elu', 'leaky_relu'], [0.0, 1.0, 0.02] + ): for infer_only in [True, False]: - self.compare(place, layout, infer_only, activation, - alpha, use_cuda) + self.compare( + place, + layout, + infer_only, + activation, + alpha, + use_cuda, + ) def test_all_branches(self): seed = 10 os.environ['FLAGS_cudnn_deterministic'] = "1" - data = np.random.random(size=self.dshape).astype(self.dtype) * 4. - 2 + data = np.random.random(size=self.dshape).astype(self.dtype) * 4.0 - 2 use_cudas = [False, True] if core.is_compiled_with_cuda() else [False] alpha = 0.1 layouts = ["NCHW", "NHWC"] @@ -176,8 +205,15 @@ def test_all_branches(self): for layout in layouts: for activation in ['identity', 'leaky_relu']: main, startup, outs = self.build_program( - place, layout, seed, False, activation, alpha, use_cuda, - True) + place, + layout, + seed, + False, + activation, + alpha, + use_cuda, + True, + ) exe = fluid.Executor(place) exe.run(startup) exe.run(program=main, feed={'input': data}) diff --git a/python/paddle/fluid/tests/unittests/test_inplace_addto_strategy.py b/python/paddle/fluid/tests/unittests/test_inplace_addto_strategy.py index 825fcbd0620c3..4f0aebf6c3c07 100644 --- a/python/paddle/fluid/tests/unittests/test_inplace_addto_strategy.py +++ b/python/paddle/fluid/tests/unittests/test_inplace_addto_strategy.py @@ -20,27 +20,31 @@ class ConvBNLayer(fluid.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1, - data_format="NCHW"): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + groups=1, + data_format="NCHW", + ): super(ConvBNLayer, self).__init__() - self._conv = paddle.nn.Conv2D(in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=False, - data_format=data_format) - - self._batch_norm = paddle.nn.BatchNorm(num_filters, - data_layout=data_format) + self._conv = paddle.nn.Conv2D( + in_channels=num_channels, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + bias_attr=False, + data_format=data_format, + ) + + self._batch_norm = paddle.nn.BatchNorm( + num_filters, data_layout=data_format + ) def forward(self, inputs): y = self._conv(inputs) @@ -57,10 +61,12 @@ def create_program(data_format="NCHW"): if data_format == "NHWC": x = paddle.transpose(x, [0, 2, 3, 1]) x = fluid.layers.prelu(x, mode="channel") - conv = ConvBNLayer(num_channels=3, - num_filters=3, - filter_size=1, - data_format=data_format) + conv = ConvBNLayer( + num_channels=3, + num_filters=3, + filter_size=1, + data_format=data_format, + ) y = conv(x) + x loss = fluid.layers.reduce_sum(y) @@ -72,9 +78,7 @@ def create_program(data_format="NCHW"): class TestInplaceAddto(unittest.TestCase): - def check_result(self, data_format="NCHW"): - def run_program(enable_addto): np.random.seed(10) paddle.seed(10) @@ -83,22 +87,27 @@ def run_program(enable_addto): fluid.set_flags({"FLAGS_cudnn_deterministic": True}) fluid.set_flags({"FLAGS_max_inplace_grad_add": 2}) loss, main, startup, w = create_program(data_format=data_format) - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) strategy = fluid.BuildStrategy() strategy.enable_addto = enable_addto compiled = fluid.CompiledProgram(main).with_data_parallel( - loss_name=loss.name, build_strategy=strategy) + loss_name=loss.name, build_strategy=strategy + ) exe.run(startup) - img = np.random.uniform(-128, 128, - [8, 3, 224, 224]).astype(np.float32) + img = np.random.uniform(-128, 128, [8, 3, 224, 224]).astype( + np.float32 + ) for i in range(10): - res = exe.run(compiled, - feed={'img': img}, - fetch_list=[loss.name, w.name]) + res = exe.run( + compiled, feed={'img': img}, fetch_list=[loss.name, w.name] + ) return res res1, w1 = run_program(True) diff --git a/python/paddle/fluid/tests/unittests/test_inplace_and_clear_gradient.py b/python/paddle/fluid/tests/unittests/test_inplace_and_clear_gradient.py index 7f4bf6a5fefb7..aa7e837b6cd28 100644 --- a/python/paddle/fluid/tests/unittests/test_inplace_and_clear_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_inplace_and_clear_gradient.py @@ -21,7 +21,6 @@ def clear_grad(w, a): - @paddle.no_grad() def warp(*_): assert w.grad is not None @@ -32,7 +31,6 @@ def warp(*_): class TestInplaceAndClearGradient(unittest.TestCase): - def test(self): paddle.set_device('cpu') diff --git a/python/paddle/fluid/tests/unittests/test_inplace_auto_generated_apis.py b/python/paddle/fluid/tests/unittests/test_inplace_auto_generated_apis.py index 09e8c61123677..d8f8b5dbc72ce 100644 --- a/python/paddle/fluid/tests/unittests/test_inplace_auto_generated_apis.py +++ b/python/paddle/fluid/tests/unittests/test_inplace_auto_generated_apis.py @@ -21,7 +21,6 @@ # In static mode, inplace strategy will not be used in Inplace APIs. class TestStaticAutoGeneratedAPI(unittest.TestCase): - def setUp(self): paddle.enable_static() self.init_data() @@ -48,23 +47,22 @@ def test_api(self): out = self.executed_paddle_api(x) exe = paddle.static.Executor(place=paddle.CPUPlace()) - fetch_x, fetch_out = exe.run(main_prog, - feed={"x": self.np_x}, - fetch_list=[x, out]) + fetch_x, fetch_out = exe.run( + main_prog, feed={"x": self.np_x}, fetch_list=[x, out] + ) np.testing.assert_array_equal(fetch_x, self.np_x) self.assertTrue( - self.np_compare(fetch_out, self.executed_numpy_api(self.np_x))) + self.np_compare(fetch_out, self.executed_numpy_api(self.np_x)) + ) class TestStaticInplaceAutoGeneratedAPI(TestStaticAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.ceil_() class TestStaticFloorAPI(TestStaticAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.floor() @@ -73,13 +71,11 @@ def executed_numpy_api(self, x): class TestStaticInplaceFloorAPI(TestStaticFloorAPI): - def executed_paddle_api(self, x): return x.floor_() class TestStaticExpAPI(TestStaticAutoGeneratedAPI): - def set_np_compare_func(self): self.np_compare = np.allclose @@ -91,13 +87,11 @@ def executed_numpy_api(self, x): class TestStaticInplaceExpAPI(TestStaticExpAPI): - def executed_paddle_api(self, x): return x.exp_() class TestStaticReciprocalAPI(TestStaticAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.reciprocal() @@ -106,13 +100,11 @@ def executed_numpy_api(self, x): class TestStaticInplaceReciprocalAPI(TestStaticReciprocalAPI): - def executed_paddle_api(self, x): return x.reciprocal_() class TestStaticRoundAPI(TestStaticAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.round() @@ -121,13 +113,11 @@ def executed_numpy_api(self, x): class TestStaticInplaceRoundAPI(TestStaticRoundAPI): - def executed_paddle_api(self, x): return x.round_() class TestStaticSqrtAPI(TestStaticAutoGeneratedAPI): - def init_data(self): self.dtype = 'float32' self.shape = [10, 20] @@ -144,13 +134,11 @@ def executed_numpy_api(self, x): class TestStaticInplaceSqrtAPI(TestStaticSqrtAPI): - def executed_paddle_api(self, x): return x.sqrt_() class TestStaticRsqrtAPI(TestStaticSqrtAPI): - def executed_paddle_api(self, x): return x.rsqrt() @@ -159,14 +147,12 @@ def executed_numpy_api(self, x): class TestStaticInplaceRsqrtAPI(TestStaticRsqrtAPI): - def executed_paddle_api(self, x): return x.rsqrt_() # In dygraph mode, inplace strategy will be used in Inplace APIs. class TestDygraphAutoGeneratedAPI(unittest.TestCase): - def setUp(self): paddle.disable_static() self.init_data() @@ -191,17 +177,16 @@ def test_api(self): out = self.executed_paddle_api(x) self.assertTrue( - self.np_compare(out.numpy(), self.executed_numpy_api(self.np_x))) + self.np_compare(out.numpy(), self.executed_numpy_api(self.np_x)) + ) class TestDygraphInplaceAutoGeneratedAPI(TestDygraphAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.ceil_() class TestDygraphFloorAPI(TestDygraphAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.floor() @@ -210,13 +195,11 @@ def executed_numpy_api(self, x): class TestDygraphInplaceFloorAPI(TestDygraphFloorAPI): - def executed_paddle_api(self, x): return x.floor_() class TestDygraphExpAPI(TestDygraphAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.exp() @@ -228,13 +211,11 @@ def set_np_compare_func(self): class TestDygraphInplaceExpAPI(TestDygraphExpAPI): - def executed_paddle_api(self, x): return x.exp_() class TestDygraphReciprocalAPI(TestDygraphAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.reciprocal() @@ -243,13 +224,11 @@ def executed_numpy_api(self, x): class TestDygraphInplaceReciprocalAPI(TestDygraphReciprocalAPI): - def executed_paddle_api(self, x): return x.reciprocal_() class TestDygraphRoundAPI(TestDygraphAutoGeneratedAPI): - def executed_paddle_api(self, x): return x.round() @@ -258,13 +237,11 @@ def executed_numpy_api(self, x): class TestDygraphInplaceRoundAPI(TestDygraphRoundAPI): - def executed_paddle_api(self, x): return x.round_() class TestDygraphSqrtAPI(TestDygraphAutoGeneratedAPI): - def init_data(self): self.dtype = 'float32' self.shape = [10, 20] @@ -281,22 +258,19 @@ def executed_numpy_api(self, x): class TestDygraphInplaceSqrtAPI(TestDygraphSqrtAPI): - def executed_paddle_api(self, x): return x.sqrt_() class TestDygraphRsqrtAPI(TestDygraphSqrtAPI): - def executed_paddle_api(self, x): return x.rsqrt() def executed_numpy_api(self, x): - return 1. / np.sqrt(x) + return 1.0 / np.sqrt(x) class TestDygraphInplaceRsqrtAPI(TestDygraphRsqrtAPI): - def executed_paddle_api(self, x): return x.rsqrt_() diff --git a/python/paddle/fluid/tests/unittests/test_inplace_softmax_with_cross_entropy.py b/python/paddle/fluid/tests/unittests/test_inplace_softmax_with_cross_entropy.py index 759454a2887fc..8e9b154a41cce 100644 --- a/python/paddle/fluid/tests/unittests/test_inplace_softmax_with_cross_entropy.py +++ b/python/paddle/fluid/tests/unittests/test_inplace_softmax_with_cross_entropy.py @@ -18,40 +18,41 @@ class TestSoftmaxWithXe(unittest.TestCase): - def setUp(self): self.initParameter() - self.m, self.n = np.random.random_integers(low=100, high=2000, - size=[2]).astype('int64') + self.m, self.n = np.random.random_integers( + low=100, high=2000, size=[2] + ).astype('int64') def initParameter(self): self.dtype = 'float32' self.soft_label = False - def softmax_with_xe(self, - x, - y, - place, - inplace=True, - numeric_stable_mode=True): + def softmax_with_xe( + self, x, y, place, inplace=True, numeric_stable_mode=True + ): m, n = x.shape with fluid.program_guard(fluid.Program(), fluid.Program()): with fluid.scope_guard(fluid.Scope()): - x_d = fluid.layers.data(name='x', - shape=[m, n], - dtype=self.dtype, - append_batch_size=False) + x_d = fluid.layers.data( + name='x', + shape=[m, n], + dtype=self.dtype, + append_batch_size=False, + ) y_d = fluid.layers.data( name='y', shape=[m, 1] if not self.soft_label else [m, n], dtype='int64' if not self.soft_label else self.dtype, - append_batch_size=False) + append_batch_size=False, + ) z_d, s_d = fluid.layers.softmax_with_cross_entropy( x_d, y_d, soft_label=self.soft_label, return_softmax=True, - numeric_stable_mode=numeric_stable_mode) + numeric_stable_mode=numeric_stable_mode, + ) exe = fluid.Executor(place) @@ -60,19 +61,18 @@ def softmax_with_xe(self, build_strategy = fluid.BuildStrategy() build_strategy.enable_inplace = inplace prog = fluid.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - build_strategy=build_strategy, places=place) + fluid.default_main_program() + ).with_data_parallel( + build_strategy=build_strategy, places=place + ) fetch_list = [z_d.name, s_d.name] print('Inplace is {}'.format("ON" if inplace else "OFF")) - z, s = exe.run(prog, - feed={ - x_d.name: x, - y_d.name: y - }, - fetch_list=fetch_list) + z, s = exe.run( + prog, feed={x_d.name: x, y_d.name: y}, fetch_list=fetch_list + ) return z, s def main_with_place(self, place): @@ -82,39 +82,32 @@ def main_with_place(self, place): for a, b in x_range: x = ((b - a) * x + a).astype(self.dtype) if not self.soft_label: - y = np.random.random_integers(size=[self.m, 1], - low=0, - high=self.n - 1).astype('int64') + y = np.random.random_integers( + size=[self.m, 1], low=0, high=self.n - 1 + ).astype('int64') else: y = np.random.random(size=[self.m, self.n]).astype(self.dtype) - norm_y = np.broadcast_to(np.reshape(np.sum(y, axis=1), [-1, 1]), - y.shape) + norm_y = np.broadcast_to( + np.reshape(np.sum(y, axis=1), [-1, 1]), y.shape + ) y = y / norm_y - z1, s1 = self.softmax_with_xe(x, - y, - place, - inplace=False, - numeric_stable_mode=False) - z2, s2 = self.softmax_with_xe(x, - y, - place, - inplace=True, - numeric_stable_mode=False) + z1, s1 = self.softmax_with_xe( + x, y, place, inplace=False, numeric_stable_mode=False + ) + z2, s2 = self.softmax_with_xe( + x, y, place, inplace=True, numeric_stable_mode=False + ) self.assertTrue((z1 == z2).all()) self.assertTrue((s1 == s2).all()) - z1, s1 = self.softmax_with_xe(x, - y, - place, - inplace=False, - numeric_stable_mode=True) - z2, s2 = self.softmax_with_xe(x, - y, - place, - inplace=True, - numeric_stable_mode=True) + z1, s1 = self.softmax_with_xe( + x, y, place, inplace=False, numeric_stable_mode=True + ) + z2, s2 = self.softmax_with_xe( + x, y, place, inplace=True, numeric_stable_mode=True + ) self.assertTrue((z1 == z2).all()) self.assertTrue((s1 == s2).all()) @@ -125,21 +118,18 @@ def test_main(self): class TestSoftmaxWithXe1(TestSoftmaxWithXe): - def initParameter(self): self.dtype = 'float32' self.soft_label = True class TestSoftmaxWithXe2(TestSoftmaxWithXe): - def initParameter(self): self.dtype = 'float64' self.soft_label = False class TestSoftmaxWithXe3(TestSoftmaxWithXe): - def initParameter(self): self.dtype = 'float64' self.soft_label = True diff --git a/python/paddle/fluid/tests/unittests/test_input_spec.py b/python/paddle/fluid/tests/unittests/test_input_spec.py index bb31c4e7f51e6..9d805d64e93cc 100644 --- a/python/paddle/fluid/tests/unittests/test_input_spec.py +++ b/python/paddle/fluid/tests/unittests/test_input_spec.py @@ -21,15 +21,17 @@ import paddle.fluid as fluid from paddle.static import InputSpec from paddle.fluid.framework import convert_np_dtype_to_dtype_ -from paddle.fluid.dygraph.dygraph_to_static.utils import _compatible_non_tensor_spec +from paddle.fluid.dygraph.dygraph_to_static.utils import ( + _compatible_non_tensor_spec, +) class TestInputSpec(unittest.TestCase): - def test_default(self): tensor_spec = InputSpec([3, 4]) - self.assertEqual(tensor_spec.dtype, - convert_np_dtype_to_dtype_('float32')) + self.assertEqual( + tensor_spec.dtype, convert_np_dtype_to_dtype_('float32') + ) self.assertEqual(tensor_spec.name, None) def test_from_tensor(self): @@ -45,15 +47,17 @@ def test_from_tensor(self): def test_from_numpy(self): x_numpy = np.ones([10, 12]) x_np_spec = InputSpec.from_numpy(x_numpy) - self.assertEqual(x_np_spec.dtype, - convert_np_dtype_to_dtype_(x_numpy.dtype)) + self.assertEqual( + x_np_spec.dtype, convert_np_dtype_to_dtype_(x_numpy.dtype) + ) self.assertEqual(x_np_spec.shape, x_numpy.shape) self.assertEqual(x_np_spec.name, None) x_numpy2 = np.array([1, 2, 3, 4]).astype('int64') x_np_spec2 = InputSpec.from_numpy(x_numpy2, name='x_np_int64') - self.assertEqual(x_np_spec2.dtype, - convert_np_dtype_to_dtype_(x_numpy2.dtype)) + self.assertEqual( + x_np_spec2.dtype, convert_np_dtype_to_dtype_(x_numpy2.dtype) + ) self.assertEqual(x_np_spec2.shape, x_numpy2.shape) self.assertEqual(x_np_spec2.name, 'x_np_int64') @@ -84,7 +88,7 @@ def test_batch_and_unbatch(self): # unbatch unbatch_spec = batch_tensor_spec.unbatch() - self.assertEqual(unbatch_spec.shape, (10, )) + self.assertEqual(unbatch_spec.shape, (10,)) # 1. `unbatch` requires len(shape) > 1 with self.assertRaises(ValueError): @@ -116,7 +120,6 @@ def test_eq_and_hash(self): class NetWithNonTensorSpec(paddle.nn.Layer): - def __init__(self, in_num, out_num): super(NetWithNonTensorSpec, self).__init__() self.linear_1 = paddle.nn.Linear(in_num, out_num) @@ -157,7 +160,6 @@ def another_func(self, x, config=None): class TestNetWithNonTensorSpec(unittest.TestCase): - def setUp(self): self.in_num = 16 self.out_num = 16 @@ -243,7 +245,6 @@ def test_spec_compatible(self): class NetWithNonTensorSpecPrune(paddle.nn.Layer): - def __init__(self, in_num, out_num): super(NetWithNonTensorSpecPrune, self).__init__() self.linear_1 = paddle.nn.Linear(in_num, out_num) @@ -263,7 +264,6 @@ def forward(self, x, y, use_bn=False): class TestNetWithNonTensorSpecWithPrune(unittest.TestCase): - def setUp(self): self.in_num = 16 self.out_num = 16 @@ -311,7 +311,6 @@ def test_non_tensor_with_prune(self): class UnHashableObject: - def __init__(self, val): self.val = val @@ -320,7 +319,6 @@ def __hash__(self): class TestCompatibleNonTensorSpec(unittest.TestCase): - def test_case(self): self.assertTrue(_compatible_non_tensor_spec([1, 2, 3], [1, 2, 3])) self.assertFalse(_compatible_non_tensor_spec([1, 2, 3], [1, 2])) @@ -328,8 +326,10 @@ def test_case(self): # not supported unhashable object. self.assertTrue( - _compatible_non_tensor_spec(UnHashableObject(1), - UnHashableObject(1))) + _compatible_non_tensor_spec( + UnHashableObject(1), UnHashableObject(1) + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_install_check.py b/python/paddle/fluid/tests/unittests/test_install_check.py index e51079278db18..15f2b5f3b7eed 100644 --- a/python/paddle/fluid/tests/unittests/test_install_check.py +++ b/python/paddle/fluid/tests/unittests/test_install_check.py @@ -18,7 +18,6 @@ class TestInstallCheck(unittest.TestCase): - def test_paddle_fluid(self): paddle.fluid.install_check.run_check() diff --git a/python/paddle/fluid/tests/unittests/test_instance_norm_op.py b/python/paddle/fluid/tests/unittests/test_instance_norm_op.py index 88bed96b35203..50278505f98d6 100644 --- a/python/paddle/fluid/tests/unittests/test_instance_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_instance_norm_op.py @@ -62,10 +62,19 @@ def _reference_instance_norm_grad(x, d_y, scale, mean, var, epsilon): scale_tile = np.reshape(scale, (1, c, 1, 1)) scale_tile = np.tile(scale_tile, (n, 1, h, w)) - d_x = scale_tile * var_inv * ( - d_y - np.mean(d_y, axis=(2, 3), keepdims=True) - - (x - mean_tile) * var_inv * - np.mean(d_y * (x - mean_tile) * var_inv, axis=(2, 3), keepdims=True)) + d_x = ( + scale_tile + * var_inv + * ( + d_y + - np.mean(d_y, axis=(2, 3), keepdims=True) + - (x - mean_tile) + * var_inv + * np.mean( + d_y * (x - mean_tile) * var_inv, axis=(2, 3), keepdims=True + ) + ) + ) return d_x, d_scale, d_bias @@ -76,7 +85,6 @@ def _cal_mean_variance(x, epsilon, mean_shape): class TestInstanceNormOpTraining(unittest.TestCase): - def setUp(self): self.epsilon = 1e-5 self.init_test_case() @@ -85,23 +93,24 @@ def init_test_case(self): self.shape = [2, 3, 4, 5] self.no_grad_set = set() self.fetch_list = [ - 'y', 'saved_mean', 'saved_variance', 'x@GRAD', 'scale@GRAD', - 'bias@GRAD' + 'y', + 'saved_mean', + 'saved_variance', + 'x@GRAD', + 'scale@GRAD', + 'bias@GRAD', ] def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=1e-05, - atol=atol, - err_msg=msg) + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=1e-05, atol=atol, err_msg=msg + ) def set_global_mean_var(self, mean_shape, x): mean, variance = _cal_mean_variance(x, self.epsilon, mean_shape) return mean, variance def test_forward_backward(self): - def test_with_place(place, shape): epsilon = self.epsilon n, c, h, w = shape[0], shape[1], shape[2], shape[3] @@ -116,12 +125,14 @@ def test_with_place(place, shape): d_y = np.random.random_sample(shape).astype(np.float32) y, saved_mean, variance_tmp = _reference_instance_norm_naive( - x, scale, bias, epsilon, mean, variance) + x, scale, bias, epsilon, mean, variance + ) saved_variance = 1 / np.sqrt(variance_tmp + epsilon) d_x, d_scale, d_bias = _reference_instance_norm_grad( - x, d_y, scale, saved_mean, saved_variance, epsilon) + x, d_y, scale, saved_mean, saved_variance, epsilon + ) var_dict = locals() var_dict['y@GRAD'] = d_y @@ -130,7 +141,12 @@ def test_with_place(place, shape): var_dict['bias@GRAD'] = d_bias var_names = [ - 'x', 'scale', 'bias', 'y', 'saved_mean', 'saved_variance' + 'x', + 'scale', + 'bias', + 'y', + 'saved_mean', + 'saved_variance', ] ground_truth = {name: var_dict[name] for name in var_names} @@ -138,31 +154,33 @@ def test_with_place(place, shape): with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) - in_op = block.append_op(type="instance_norm", - inputs={ - "X": block.var("x"), - "Scale": block.var("scale"), - "Bias": block.var("bias"), - }, - outputs={ - "Y": - block.var("y"), - "SavedMean": - block.var("saved_mean"), - "SavedVariance": - block.var("saved_variance") - }, - attrs={ - "epsilon": epsilon, - }) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) + in_op = block.append_op( + type="instance_norm", + inputs={ + "X": block.var("x"), + "Scale": block.var("scale"), + "Bias": block.var("bias"), + }, + outputs={ + "Y": block.var("y"), + "SavedMean": block.var("saved_mean"), + "SavedVariance": block.var("saved_variance"), + }, + attrs={ + "epsilon": epsilon, + }, + ) block.create_var(name="y@GRAD", dtype='float32', shape=y.shape) grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - in_op.desc, self.no_grad_set, []) + in_op.desc, self.no_grad_set, [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -177,12 +195,14 @@ def test_with_place(place, shape): program._sync_with_cpp() exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in ['x', 'scale', 'bias', 'y@GRAD'] - }, - fetch_list=self.fetch_list) + out = exe.run( + program, + feed={ + name: var_dict[name] + for name in ['x', 'scale', 'bias', 'y@GRAD'] + }, + fetch_list=self.fetch_list, + ) for id, name in enumerate(self.fetch_list): self.__assert_close(var_dict[name], out[id], name) @@ -191,14 +211,14 @@ def test_with_place(place, shape): places = [core.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu( - "instance_norm"): + "instance_norm" + ): places.append(core.CUDAPlace(0)) for place in places: test_with_place(place, self.shape) class TestInstanceNormOpTrainingCase1(TestInstanceNormOpTraining): - def init_test_case(self): self.shape = [2, 3, 4, 5] self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -206,7 +226,6 @@ def init_test_case(self): class TestInstanceNormOpTrainingCase2(TestInstanceNormOpTraining): - def init_test_case(self): self.shape = [20, 50, 4, 5] self.no_grad_set = set(['scale@GRAD', 'bias@GRAD']) @@ -214,12 +233,12 @@ def init_test_case(self): class TestInstanceNormOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of instance_norm must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.instance_norm, x1) # the input dtype of instance_norm must be float32 or float64 @@ -228,24 +247,22 @@ def test_errors(self): class TestInstanceNormOpErrorCase1(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the first dimension of input for instance_norm must between [2d, 5d] - x = fluid.layers.data(name='x', - shape=[3], - dtype="float32", - append_batch_size=False) + x = fluid.layers.data( + name='x', shape=[3], dtype="float32", append_batch_size=False + ) self.assertRaises(ValueError, paddle.static.nn.instance_norm, x) class TestElasticNormOp(unittest.TestCase): - def init_test_case(self): self.epsilon = 1e-5 self.places = [core.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu( - "instance_norm"): + "instance_norm" + ): self.places.append(core.CUDAPlace(0)) def test_norm(self): @@ -258,20 +275,19 @@ def test_norm(self): scale = np.ones(scale_shape).astype(np.float32) bias = np.zeros(scale_shape).astype(np.float32) mean, variance = _cal_mean_variance(inputs, self.epsilon, mean_shape) - out_np, _, _ = _reference_instance_norm_naive(inputs, scale, bias, - self.epsilon, mean, - variance) + out_np, _, _ = _reference_instance_norm_naive( + inputs, scale, bias, self.epsilon, mean, variance + ) for place in self.places: with fluid.dygraph.guard(place): - instance_norm = fluid.dygraph.InstanceNorm(5, - param_attr=False, - bias_attr=False) + instance_norm = fluid.dygraph.InstanceNorm( + 5, param_attr=False, bias_attr=False + ) outputs = instance_norm(to_variable(inputs)) - np.testing.assert_allclose(outputs.numpy(), - out_np, - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + outputs.numpy(), out_np, rtol=1e-05, atol=1e-06 + ) def test_eager_api(self): with _test_eager_guard(): @@ -279,12 +295,12 @@ def test_eager_api(self): class TestElasticNormOpCase2(unittest.TestCase): - def init_test_case(self): self.epsilon = 1e-5 self.places = [core.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu( - "instance_norm"): + "instance_norm" + ): self.places.append(core.CUDAPlace(0)) def test_norm(self): @@ -297,20 +313,19 @@ def test_norm(self): scale = np.ones(scale_shape).astype(np.float32) bias = np.zeros(scale_shape).astype(np.float32) mean, variance = _cal_mean_variance(inputs, self.epsilon, mean_shape) - out_np, _, _ = _reference_instance_norm_naive(inputs, scale, bias, - self.epsilon, mean, - variance) + out_np, _, _ = _reference_instance_norm_naive( + inputs, scale, bias, self.epsilon, mean, variance + ) for place in self.places: with fluid.dygraph.guard(place): - instance_norm = fluid.dygraph.InstanceNorm(3, - param_attr=True, - bias_attr=True) + instance_norm = fluid.dygraph.InstanceNorm( + 3, param_attr=True, bias_attr=True + ) outputs = instance_norm(to_variable(inputs)) - np.testing.assert_allclose(outputs.numpy(), - out_np, - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + outputs.numpy(), out_np, rtol=1e-05, atol=1e-06 + ) def test_eager_api(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_instance_norm_op_v2.py b/python/paddle/fluid/tests/unittests/test_instance_norm_op_v2.py index cbb91795a5005..66bbc3a338714 100644 --- a/python/paddle/fluid/tests/unittests/test_instance_norm_op_v2.py +++ b/python/paddle/fluid/tests/unittests/test_instance_norm_op_v2.py @@ -23,11 +23,11 @@ class TestInstanceNorm(unittest.TestCase): - def test_error(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu( - "instance_norm"): + "instance_norm" + ): places.append(fluid.CUDAPlace(0)) for p in places: @@ -48,9 +48,9 @@ def error3d(): def weight_bias_false(): x_data_4 = np.random.random(size=(2, 1, 3, 3)).astype('float32') - instance_norm3d = paddle.nn.InstanceNorm3D(1, - weight_attr=False, - bias_attr=False) + instance_norm3d = paddle.nn.InstanceNorm3D( + 1, weight_attr=False, bias_attr=False + ) with fluid.dygraph.guard(p): weight_bias_false() @@ -61,7 +61,8 @@ def weight_bias_false(): def test_dygraph(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu( - "instance_norm"): + "instance_norm" + ): places.append(fluid.CUDAPlace(0)) for p in places: shape = [4, 10, 4, 4] @@ -86,7 +87,8 @@ def compute_v2(x): def test_static(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu( - "instance_norm"): + "instance_norm" + ): places.append(fluid.CUDAPlace(0)) for p in places: exe = fluid.Executor(p) diff --git a/python/paddle/fluid/tests/unittests/test_inverse_op.py b/python/paddle/fluid/tests/unittests/test_inverse_op.py index b155df0498d09..f06d32170ba9b 100644 --- a/python/paddle/fluid/tests/unittests/test_inverse_op.py +++ b/python/paddle/fluid/tests/unittests/test_inverse_op.py @@ -21,7 +21,6 @@ class TestInverseOp(OpTest): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -46,7 +45,6 @@ def test_grad(self): class TestInverseOpBatched(TestInverseOp): - def config(self): self.matrix_shape = [8, 4, 4] self.dtype = "float64" @@ -54,35 +52,30 @@ def config(self): class TestInverseOpLarge(TestInverseOp): - def config(self): self.matrix_shape = [32, 32] self.dtype = "float64" self.python_api = paddle.tensor.math.inverse def test_grad(self): - self.check_grad(['Input'], - 'Output', - max_relative_error=1e-6, - check_eager=True) + self.check_grad( + ['Input'], 'Output', max_relative_error=1e-6, check_eager=True + ) class TestInverseOpFP32(TestInverseOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float32" self.python_api = paddle.tensor.math.inverse def test_grad(self): - self.check_grad(['Input'], - 'Output', - max_relative_error=1e-2, - check_eager=True) + self.check_grad( + ['Input'], 'Output', max_relative_error=1e-2, check_eager=True + ) class TestInverseOpBatchedFP32(TestInverseOpFP32): - def config(self): self.matrix_shape = [8, 4, 4] self.dtype = "float32" @@ -90,7 +83,6 @@ def config(self): class TestInverseOpLargeFP32(TestInverseOpFP32): - def config(self): self.matrix_shape = [32, 32] self.dtype = "float32" @@ -98,7 +90,6 @@ def config(self): class TestInverseAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -113,12 +104,14 @@ def check_static_result(self, place): result_np = np.linalg.inv(input_np) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) - np.testing.assert_allclose(fetches[0], - np.linalg.inv(input_np), - rtol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) + np.testing.assert_allclose( + fetches[0], np.linalg.inv(input_np), rtol=1e-05 + ) def test_static(self): for place in self.places: @@ -130,13 +123,12 @@ def test_dygraph(self): input_np = np.random.random([4, 4]).astype("float64") input = fluid.dygraph.to_variable(input_np) result = paddle.inverse(input) - np.testing.assert_allclose(result.numpy(), - np.linalg.inv(input_np), - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), np.linalg.inv(input_np), rtol=1e-05 + ) class TestInverseAPIError(unittest.TestCase): - def test_errors(self): input_np = np.random.random([4, 4]).astype("float64") @@ -159,7 +151,6 @@ def test_errors(self): class TestInverseSingularAPI(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -174,9 +165,11 @@ def check_static_result(self, place): exe = fluid.Executor(place) try: - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) except RuntimeError as ex: print("The mat is singular") except ValueError as ex: diff --git a/python/paddle/fluid/tests/unittests/test_io_save_load.py b/python/paddle/fluid/tests/unittests/test_io_save_load.py index f5feb8b78d604..80c2ae1f92ff5 100644 --- a/python/paddle/fluid/tests/unittests/test_io_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_io_save_load.py @@ -22,7 +22,6 @@ class TestSaveLoadAPIError(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.save_dir = os.path.join(self.temp_dir.name, "fake_dir") @@ -51,16 +50,18 @@ def func_test_load_vars_error(self): exe = fluid.Executor(place) # case 1: main_program type error when vars None with self.assertRaises(TypeError): - fluid.io.load_vars(executor=exe, - dirname=self.save_dir, - main_program="program") + fluid.io.load_vars( + executor=exe, dirname=self.save_dir, main_program="program" + ) # case 2: main_program type error when vars not None with self.assertRaises(TypeError): - fluid.io.load_vars(executor=exe, - dirname=self.save_dir, - main_program="program", - vars="vars") + fluid.io.load_vars( + executor=exe, + dirname=self.save_dir, + main_program="program", + vars="vars", + ) def test_load_vars_error(self): with _test_eager_guard(): @@ -69,7 +70,6 @@ def test_load_vars_error(self): class TestSaveInferenceModelAPIError(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -87,13 +87,15 @@ def func_test_useless_feeded_var_names(self): exe = fluid.Executor(fluid.CPUPlace()) exe.run(start_prog) with self.assertRaisesRegexp( - ValueError, "not involved in the target_vars calculation"): - fluid.io.save_inference_model(dirname=os.path.join( - self.temp_dir.name, 'model'), - feeded_var_names=['x', 'y'], - target_vars=[z], - executor=exe, - main_program=main_prog) + ValueError, "not involved in the target_vars calculation" + ): + fluid.io.save_inference_model( + dirname=os.path.join(self.temp_dir.name, 'model'), + feeded_var_names=['x', 'y'], + target_vars=[z], + executor=exe, + main_program=main_prog, + ) def test_useless_feeded_var_names(self): with _test_eager_guard(): @@ -102,7 +104,6 @@ def test_useless_feeded_var_names(self): class TestWhenTrainWithNoGrad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() diff --git a/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py b/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py index 4d7c66993c0d9..4324280efcbb6 100644 --- a/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py +++ b/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py @@ -18,7 +18,6 @@ class TestIOUSimilarityOp(OpTest): - def test_check_output(self): self.check_output(check_dygraph=False) @@ -34,7 +33,9 @@ def setUp(self): self.attrs = {"box_normalized": self.box_normalized} self.outputs = {'Out': self.output} - def _compute_iou(self, ): + def _compute_iou( + self, + ): for row in range(self.boxes1.shape[0]): for col in range(self.boxes2.shape[0]): xmin1, ymin1, xmax1, ymax1 = self.boxes1[row] @@ -64,7 +65,6 @@ def _compute_iou(self, ): class TestIOUSimilarityOpWithLoD(TestIOUSimilarityOp): - def test_check_output(self): self.check_output(check_dygraph=False) @@ -81,7 +81,6 @@ def setUp(self): class TestIOUSimilarityOpWithBoxNormalized(TestIOUSimilarityOp): - def test_check_output(self): self.check_output(check_dygraph=False) diff --git a/python/paddle/fluid/tests/unittests/test_ir_graph.py b/python/paddle/fluid/tests/unittests/test_ir_graph.py index 1951db0395fa9..a8effa96920a8 100644 --- a/python/paddle/fluid/tests/unittests/test_ir_graph.py +++ b/python/paddle/fluid/tests/unittests/test_ir_graph.py @@ -24,8 +24,9 @@ class TestIRGraph(unittest.TestCase): def test_nodes(self): graph = build_graph() self.assertTrue( - {node.name() - for node in graph.nodes()} == {"x1", "x2", "out", "sum"}) + {node.name() for node in graph.nodes()} + == {"x1", "x2", "out", "sum"} + ) def test_has_set_get(self): graph = build_graph() @@ -85,8 +86,9 @@ def test_release_nodes(self): graph = build_graph() nodes = graph.release_nodes() self.assertTrue(len(graph.nodes()) == 0) - self.assertTrue({node.name() - for node in nodes} == {"x1", "x2", "out", "sum"}) + self.assertTrue( + {node.name() for node in nodes} == {"x1", "x2", "out", "sum"} + ) def test_remove_node(self): graph = build_graph() @@ -94,8 +96,9 @@ def test_remove_node(self): for node in nodes: if node.name() == "sum": break - self.assertTrue({node.name() - for node in nodes} == {"x1", "x2", "out", "sum"}) + self.assertTrue( + {node.name() for node in nodes} == {"x1", "x2", "out", "sum"} + ) nodes.remove(node) self.assertTrue({node.name() for node in nodes} == {"x1", "x2", "out"}) diff --git a/python/paddle/fluid/tests/unittests/test_ir_inplace_pass.py b/python/paddle/fluid/tests/unittests/test_ir_inplace_pass.py index 3fde50653b66e..62ea08307ce15 100644 --- a/python/paddle/fluid/tests/unittests/test_ir_inplace_pass.py +++ b/python/paddle/fluid/tests/unittests/test_ir_inplace_pass.py @@ -31,8 +31,10 @@ def fc_with_batchnorm(use_feed): hidden, size=200, act='tanh', - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) hidden = fluid.layers.batch_norm(input=hidden) prediction = fluid.layers.fc(hidden, size=10, act='softmax') @@ -42,7 +44,6 @@ def fc_with_batchnorm(use_feed): class TestIrInplace(TestParallelExecutorBase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) @@ -56,13 +57,11 @@ def _fc_with_batchnorm(self, ir_memory_optimize, enable_inplace): label = np.ones(shape=[32, 1], dtype='int64') self.check_network_convergence( fc_with_batchnorm, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=DeviceType.CUDA, use_ir_memory_optimize=ir_memory_optimize, - enable_inplace=enable_inplace) + enable_inplace=enable_inplace, + ) def test_fc_with_batchnorm(self, delta=1e-3): loss00 = self._fc_with_batchnorm(False, False) diff --git a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_ifelse_op.py b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_ifelse_op.py index 9cc78982aab57..657c267767dfa 100644 --- a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_ifelse_op.py +++ b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_ifelse_op.py @@ -29,11 +29,9 @@ class TestIrMemoryOptimizeIfElseOp(unittest.TestCase): - - def check_network_convergence(self, - use_cuda=True, - use_mem_opt=False, - iter_num=5): + def check_network_convergence( + self, use_cuda=True, use_mem_opt=False, iter_num=5 + ): paddle.seed(100) paddle.framework.random._manual_program_seed(100) prog = Program() @@ -65,14 +63,17 @@ def check_network_convergence(self, optimizer = MomentumOptimizer(learning_rate=0.001, momentum=0.9) optimizer.minimize(avg_loss, startup_prog) - train_reader = paddle.batch(paddle.dataset.mnist.train(), - batch_size=200) + train_reader = paddle.batch( + paddle.dataset.mnist.train(), batch_size=200 + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = Executor(place) exec_strategy = fluid.ExecutionStrategy() - exec_strategy._use_device = core.DeviceType.CUDA if use_cuda else core.DeviceType.CPU + exec_strategy._use_device = ( + core.DeviceType.CUDA if use_cuda else core.DeviceType.CPU + ) build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = use_mem_opt @@ -81,7 +82,8 @@ def check_network_convergence(self, train_cp = train_cp.with_data_parallel( loss_name=avg_loss.name, exec_strategy=exec_strategy, - build_strategy=build_strategy) + build_strategy=build_strategy, + ) fetch_list = [avg_loss.name] exe.run(startup_prog) @@ -94,12 +96,11 @@ def check_network_convergence(self, y_data = np.array([x[1] for x in data]).astype("int64") y_data = y_data.reshape((y_data.shape[0], 1)) - outs = exe.run(train_cp, - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[avg_loss]) + outs = exe.run( + train_cp, + feed={'x': x_data, 'y': y_data}, + fetch_list=[avg_loss], + ) loop += 1 ret.append(outs[0]) diff --git a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_nlp.py b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_nlp.py index ed4ab08a506d7..f661db6d15e50 100644 --- a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_nlp.py +++ b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_nlp.py @@ -20,23 +20,26 @@ from ir_memory_optimize_net_base import TestIrMemOptBase -def lstm_net(data, - label, - dict_dim, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2, - emb_lr=30.0): +def lstm_net( + data, + label, + dict_dim, + emb_dim=128, + hid_dim=128, + hid_dim2=96, + class_dim=2, + emb_lr=30.0, +): emb = fluid.layers.embedding( input=data, size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) + param_attr=fluid.ParamAttr(learning_rate=emb_lr), + ) fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4) - lstm_h, c = fluid.layers.dynamic_lstm(input=fc0, - size=hid_dim * 4, - is_reverse=False) + lstm_h, c = fluid.layers.dynamic_lstm( + input=fc0, size=hid_dim * 4, is_reverse=False + ) lstm_max = fluid.layers.sequence_pool(input=lstm_h, pool_type='max') lstm_max_tanh = fluid.layers.tanh(lstm_max) fc1 = fluid.layers.fc(input=lstm_max_tanh, size=hid_dim2, act='tanh') @@ -47,7 +50,6 @@ def lstm_net(data, class TestIrMemOptRNN(TestIrMemOptBase): - def setUp(self): self.network = lstm_net diff --git a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_pass.py b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_pass.py index d26a0a678940c..d21156b43e8cc 100644 --- a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_pass.py +++ b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_pass.py @@ -52,7 +52,6 @@ def fc_with_inplace_net(use_feed): class TestMNIST(TestParallelExecutorBase): - def _dummy_data(self): np.random.seed(5) img = np.random.random(size=[32, 784]).astype(np.float32) @@ -66,20 +65,16 @@ def _compare_ir_memory_optimize(self, model, use_device): img, label = self._dummy_data() first_loss0, last_loss0, _ = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_ir_memory_optimize=False) + use_ir_memory_optimize=False, + ) first_loss1, last_loss1, _ = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_ir_memory_optimize=True) + use_ir_memory_optimize=True, + ) for loss in zip(first_loss0, first_loss1): self.assertAlmostEqual(loss[0], loss[1], delta=1e-6) for loss in zip(last_loss0, last_loss1): diff --git a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_transformer.py b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_transformer.py index 8b7fe96805fff..8d1191fddd5b3 100644 --- a/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_transformer.py +++ b/python/paddle/fluid/tests/unittests/test_ir_memory_optimize_transformer.py @@ -25,7 +25,6 @@ # NOTE(dzhwinter): test diferent strategy colisions. # open the eager delete tensor strategy by default. class TestTransformerWithIR(TestParallelExecutorBase): - def test_main(self): if core.is_compiled_with_cuda(): # check python transpiler @@ -34,14 +33,16 @@ def test_main(self): use_device=DeviceType.CUDA, feed_data_reader=get_feed_data_reader(), use_ir_memory_optimize=False, - iter=2) + iter=2, + ) # check IR memory optimize self.check_network_convergence( transformer, use_device=DeviceType.CUDA, feed_data_reader=get_feed_data_reader(), use_ir_memory_optimize=True, - iter=2) + iter=2, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_is_complex.py b/python/paddle/fluid/tests/unittests/test_is_complex.py index a441bd8629670..fd2e58eff6399 100644 --- a/python/paddle/fluid/tests/unittests/test_is_complex.py +++ b/python/paddle/fluid/tests/unittests/test_is_complex.py @@ -18,7 +18,6 @@ class TestIsComplex(unittest.TestCase): - def test_for_integer(self): x = paddle.arange(10) self.assertFalse(paddle.is_complex(x)) diff --git a/python/paddle/fluid/tests/unittests/test_is_empty_op.py b/python/paddle/fluid/tests/unittests/test_is_empty_op.py index edbc60c68e507..d6235f7abc4bb 100644 --- a/python/paddle/fluid/tests/unittests/test_is_empty_op.py +++ b/python/paddle/fluid/tests/unittests/test_is_empty_op.py @@ -19,7 +19,6 @@ class TestEmpty(OpTest): - def setUp(self): self.op_type = "is_empty" self.inputs = {'X': np.array([1, 2, 3])} @@ -30,7 +29,6 @@ def test_check_output(self): class TestNotEmpty(TestEmpty): - def setUp(self): self.op_type = "is_empty" self.inputs = {'X': np.array([])} @@ -38,11 +36,11 @@ def setUp(self): class TestIsEmptyOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): input_data = np.random.random((3, 2)).astype("float64") def test_Variable(): @@ -53,25 +51,24 @@ def test_Variable(): def test_type(): # dtype must be float32, float64, int32, int64 - x3 = paddle.static.data(name="x3", - shape=[4, 32, 32], - dtype="bool") + x3 = paddle.static.data( + name="x3", shape=[4, 32, 32], dtype="bool" + ) res = paddle.is_empty(x=x3) self.assertRaises(TypeError, test_type) def test_name_type(): # name type must be string. - x4 = paddle.static.data(name="x4", - shape=[3, 2], - dtype="float32") + x4 = paddle.static.data( + name="x4", shape=[3, 2], dtype="float32" + ) res = paddle.is_empty(x=x4, name=1) self.assertRaises(TypeError, test_name_type) class TestIsEmptyOpDygraph(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() input = paddle.rand(shape=[4, 32, 32], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/test_is_integer.py b/python/paddle/fluid/tests/unittests/test_is_integer.py index a933e9fac6678..9e0eae9129296 100644 --- a/python/paddle/fluid/tests/unittests/test_is_integer.py +++ b/python/paddle/fluid/tests/unittests/test_is_integer.py @@ -18,7 +18,6 @@ class TestIsInteger(unittest.TestCase): - def test_for_integer(self): x = paddle.arange(10) self.assertTrue(paddle.is_integer(x)) diff --git a/python/paddle/fluid/tests/unittests/test_is_tensor.py b/python/paddle/fluid/tests/unittests/test_is_tensor.py index deef1225cd06a..b91b5aaf3e702 100644 --- a/python/paddle/fluid/tests/unittests/test_is_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_is_tensor.py @@ -19,24 +19,20 @@ class TestIsTensorApi(unittest.TestCase): - def test_is_tensor_real(self, dtype="float32"): - """Test is_tensor api with a real tensor - """ + """Test is_tensor api with a real tensor""" paddle.disable_static() x = paddle.rand([3, 2, 4], dtype=dtype) self.assertTrue(paddle.is_tensor(x)) def test_is_tensor_list(self, dtype="float32"): - """Test is_tensor api with a list - """ + """Test is_tensor api with a list""" paddle.disable_static() x = [1, 2, 3] self.assertFalse(paddle.is_tensor(x)) def test_is_tensor_number(self, dtype="float32"): - """Test is_tensor api with a number - """ + """Test is_tensor api with a number""" paddle.disable_static() x = 5 self.assertFalse(paddle.is_tensor(x)) diff --git a/python/paddle/fluid/tests/unittests/test_isclose_op.py b/python/paddle/fluid/tests/unittests/test_isclose_op.py index 04b7fbe54e7ee..5d53c337ce18c 100644 --- a/python/paddle/fluid/tests/unittests/test_isclose_op.py +++ b/python/paddle/fluid/tests/unittests/test_isclose_op.py @@ -19,9 +19,8 @@ class TestIscloseOp(OpTest): - def set_args(self): - self.input = np.array([10000., 1e-07]).astype("float32") + self.input = np.array([10000.0, 1e-07]).astype("float32") self.other = np.array([10000.1, 1e-08]).astype("float32") self.rtol = np.array([1e-05]).astype("float64") self.atol = np.array([1e-08]).astype("float64") @@ -36,18 +35,21 @@ def setUp(self): 'Input': self.input, 'Other': self.other, "Rtol": self.rtol, - "Atol": self.atol + "Atol": self.atol, } self.attrs = {'equal_nan': self.equal_nan} self.outputs = { - 'Out': - np.array([ - np.isclose(self.inputs['Input'], - self.inputs['Other'], - rtol=self.rtol, - atol=self.atol, - equal_nan=self.equal_nan) - ]) + 'Out': np.array( + [ + np.isclose( + self.inputs['Input'], + self.inputs['Other'], + rtol=self.rtol, + atol=self.atol, + equal_nan=self.equal_nan, + ) + ] + ) } def test_check_output(self): @@ -55,9 +57,7 @@ def test_check_output(self): class TestIscloseOpException(TestIscloseOp): - def test_check_output(self): - def test_rtol_num(): self.inputs['Rtol'] = np.array([1e-05, 1e-05]).astype("float64") self.inputs['Atol'] = np.array([1e-08]).astype("float64") @@ -88,9 +88,8 @@ def test_atol_type(): class TestIscloseOpSmallNum(TestIscloseOp): - def set_args(self): - self.input = np.array([10000., 1e-08]).astype("float32") + self.input = np.array([10000.0, 1e-08]).astype("float32") self.other = np.array([10000.1, 1e-09]).astype("float32") self.rtol = np.array([1e-05]).astype("float64") self.atol = np.array([1e-08]).astype("float64") @@ -98,7 +97,6 @@ def set_args(self): class TestIscloseOpNanFalse(TestIscloseOp): - def set_args(self): self.input = np.array([1.0, float('nan')]).astype("float32") self.other = np.array([1.0, float('nan')]).astype("float32") @@ -108,7 +106,6 @@ def set_args(self): class TestIscloseOpNanTrue(TestIscloseOp): - def set_args(self): self.input = np.array([1.0, float('nan')]).astype("float32") self.other = np.array([1.0, float('nan')]).astype("float32") @@ -118,7 +115,6 @@ def set_args(self): class TestIscloseStatic(unittest.TestCase): - def test_api_case(self): paddle.enable_static() x_data = np.random.rand(10, 10) @@ -127,24 +123,23 @@ def test_api_case(self): if paddle.fluid.core.is_compiled_with_cuda(): places.append(paddle.fluid.CUDAPlace(0)) for place in places: - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float64') y = paddle.fluid.data(name='y', shape=[10, 10], dtype='float64') result = paddle.isclose(x, y) exe = paddle.fluid.Executor(place) - fetches = exe.run(paddle.fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data - }, - fetch_list=[result]) + fetches = exe.run( + paddle.fluid.default_main_program(), + feed={"x": x_data, "y": y_data}, + fetch_list=[result], + ) expected_out = np.isclose(x_data, y_data) self.assertTrue((fetches[0] == expected_out).all(), True) class TestIscloseDygraph(unittest.TestCase): - def test_api_case(self): places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): @@ -162,13 +157,13 @@ def test_api_case(self): class TestIscloseError(unittest.TestCase): - def test_input_dtype(self): paddle.enable_static() def test_x_dtype(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float16') y = paddle.fluid.data(name='y', shape=[10, 10], dtype='float64') result = paddle.isclose(x, y) @@ -176,8 +171,9 @@ def test_x_dtype(): self.assertRaises(TypeError, test_x_dtype) def test_y_dtype(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float64') y = paddle.fluid.data(name='y', shape=[10, 10], dtype='int32') result = paddle.isclose(x, y) @@ -206,7 +202,6 @@ def test_equal_nan(): class TestIscloseOpFloat32(TestIscloseOp): - def set_args(self): self.input = np.array([10.1]).astype("float32") self.other = np.array([10]).astype("float32") @@ -216,7 +211,6 @@ def set_args(self): class TestIscloseOpFloat64(TestIscloseOp): - def set_args(self): self.input = np.array([10.1]).astype("float64") self.other = np.array([10]).astype("float64") @@ -229,7 +223,6 @@ def test_check_output(self): class TestIscloseOpLargeDimInput(TestIscloseOp): - def set_args(self): self.input = np.array(np.zeros([2048, 1024])).astype("float64") self.other = np.array(np.zeros([2048, 1024])).astype("float64") diff --git a/python/paddle/fluid/tests/unittests/test_isfinite_op.py b/python/paddle/fluid/tests/unittests/test_isfinite_op.py index f2fa252ff0a6d..e2fa9f67b5331 100644 --- a/python/paddle/fluid/tests/unittests/test_isfinite_op.py +++ b/python/paddle/fluid/tests/unittests/test_isfinite_op.py @@ -21,7 +21,6 @@ class TestInf(OpTest): - def setUp(self): self.op_type = "isinf" self.dtype = np.float32 @@ -42,9 +41,7 @@ def test_output(self): class TestRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.isfinite([10]) @@ -57,16 +54,15 @@ def test_dtype(): self.assertRaises(TypeError, test_dtype) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16Inf(TestInf): - def init_dtype(self): self.dtype = np.float16 class TestNAN(OpTest): - def setUp(self): self.op_type = "isnan" self.dtype = np.float32 @@ -86,16 +82,15 @@ def test_output(self): self.check_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16NAN(TestNAN): - def init_dtype(self): self.dtype = np.float16 class TestIsfinite(OpTest): - def setUp(self): self.op_type = "isfinite" self.dtype = np.float32 @@ -116,16 +111,15 @@ def test_output(self): self.check_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16Isfinite(TestIsfinite): - def init_dtype(self): self.dtype = np.float16 class BadInputTest(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): diff --git a/python/paddle/fluid/tests/unittests/test_isfinite_v2_op.py b/python/paddle/fluid/tests/unittests/test_isfinite_v2_op.py index 252e43b3423dc..e3d094890ac7c 100644 --- a/python/paddle/fluid/tests/unittests/test_isfinite_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_isfinite_v2_op.py @@ -31,9 +31,9 @@ def run_static(x_np, dtype, op_str, use_gpu=False): x = paddle.fluid.data(name='x', shape=x_np.shape, dtype=dtype) res = getattr(paddle.tensor, op_str)(x) exe.run(startup_program) - static_result = exe.run(main_program, - feed={'x': x_np}, - fetch_list=[res]) + static_result = exe.run( + main_program, feed={'x': x_np}, fetch_list=[res] + ) return static_result @@ -59,15 +59,16 @@ def run_eager(x_np, op_str, use_gpu=True): return dygraph_result -def np_data_generator(low, high, np_shape, type, sv_list, op_str, *args, - **kwargs): +def np_data_generator( + low, high, np_shape, type, sv_list, op_str, *args, **kwargs +): x_np = np.random.uniform(low, high, np_shape).astype(getattr(np, type)) # x_np.shape[0] >= len(sv_list) if type in ['float16', 'float32', 'float64']: for i, v in enumerate(sv_list): x_np[i] = v ori_shape = x_np.shape - x_np = x_np.reshape((np.product(ori_shape), )) + x_np = x_np.reshape((np.product(ori_shape),)) np.random.shuffle(x_np) x_np = x_np.reshape(ori_shape) result_np = getattr(np, op_str)(x_np) @@ -80,35 +81,35 @@ def np_data_generator(low, high, np_shape, type, sv_list, op_str, *args, 'high': 1, 'np_shape': [8, 17, 5, 6, 7], 'type': 'float16', - 'sv_list': [np.inf, np.nan] + 'sv_list': [np.inf, np.nan], }, { 'low': 0.1, 'high': 1, 'np_shape': [11, 17], 'type': 'float32', - 'sv_list': [np.inf, np.nan] + 'sv_list': [np.inf, np.nan], }, { 'low': 0.1, 'high': 1, 'np_shape': [2, 3, 4, 5], 'type': 'float64', - 'sv_list': [np.inf, np.nan] + 'sv_list': [np.inf, np.nan], }, { 'low': 0, 'high': 100, 'np_shape': [11, 17, 10], 'type': 'int32', - 'sv_list': [np.inf, np.nan] + 'sv_list': [np.inf, np.nan], }, { 'low': 0, 'high': 999, 'np_shape': [132], 'type': 'int64', - 'sv_list': [np.inf, np.nan] + 'sv_list': [np.inf, np.nan], }, ] @@ -127,7 +128,6 @@ def test(test_case, op_str, use_gpu=False): class TestCPUNormal(unittest.TestCase): - def test_inf(self): test(self, 'isinf') @@ -139,7 +139,6 @@ def test_finite(self): class TestCUDANormal(unittest.TestCase): - def test_inf(self): test(self, 'isinf', True) @@ -151,7 +150,6 @@ def test_finite(self): class TestError(unittest.TestCase): - def test_bad_input(self): paddle.enable_static() with fluid.program_guard(fluid.Program()): diff --git a/python/paddle/fluid/tests/unittests/test_jit_layer.py b/python/paddle/fluid/tests/unittests/test_jit_layer.py index 5a03e0ac3b80e..1713e61f9de61 100644 --- a/python/paddle/fluid/tests/unittests/test_jit_layer.py +++ b/python/paddle/fluid/tests/unittests/test_jit_layer.py @@ -20,13 +20,14 @@ from paddle.static import InputSpec from paddle.fluid.framework import _dygraph_place_guard from paddle.jit.layer import Layer -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramTranslator, +) paddle.seed(1) class Net(paddle.nn.Layer): - def __init__(self): super(Net, self).__init__() self.fc1 = paddle.nn.Linear(4, 4) @@ -50,7 +51,6 @@ def infer(self, input): class TestMultiLoad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -81,20 +81,19 @@ def test_multi_load(self): class SaveLinear(paddle.nn.Layer): - def __init__(self): super().__init__() self.linear = paddle.nn.Linear(80, 80) @paddle.jit.to_static( - input_spec=[InputSpec(shape=[None, 80], dtype='float32')]) + input_spec=[InputSpec(shape=[None, 80], dtype='float32')] + ) def forward(self, x): out = self.linear(x) return out class TestMKLOutput(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() diff --git a/python/paddle/fluid/tests/unittests/test_jit_pre_save_hooks.py b/python/paddle/fluid/tests/unittests/test_jit_pre_save_hooks.py index 1d3a35b617352..9b9ce8297ba4d 100644 --- a/python/paddle/fluid/tests/unittests/test_jit_pre_save_hooks.py +++ b/python/paddle/fluid/tests/unittests/test_jit_pre_save_hooks.py @@ -16,15 +16,17 @@ import unittest import paddle -from paddle.fluid.dygraph.jit import _run_save_pre_hooks, _clear_save_pre_hooks, _register_save_pre_hook +from paddle.fluid.dygraph.jit import ( + _run_save_pre_hooks, + _clear_save_pre_hooks, + _register_save_pre_hook, +) _counter = 0 class TestPreSaveHooks(unittest.TestCase): - def test_pre_save_hook_functions(self): - def fake_func(*args, **kwgs): global _counter _counter += 1 @@ -32,13 +34,15 @@ def fake_func(*args, **kwgs): remove_handler = _register_save_pre_hook(fake_func) self.assertEqual(len(paddle.fluid.dygraph.jit._save_pre_hooks), 1) self.assertTrue( - paddle.fluid.dygraph.jit._save_pre_hooks[0] is fake_func) + paddle.fluid.dygraph.jit._save_pre_hooks[0] is fake_func + ) # Test of avoiding redundancy hanging remove_handler = _register_save_pre_hook(fake_func) self.assertEqual(len(paddle.fluid.dygraph.jit._save_pre_hooks), 1) self.assertTrue( - paddle.fluid.dygraph.jit._save_pre_hooks[0] is fake_func) + paddle.fluid.dygraph.jit._save_pre_hooks[0] is fake_func + ) remove_handler.remove() self.assertEqual(len(paddle.fluid.dygraph.jit._save_pre_hooks), 0) diff --git a/python/paddle/fluid/tests/unittests/test_jit_save_load.py b/python/paddle/fluid/tests/unittests/test_jit_save_load.py index efb5a196f8183..2f6405e7aa6bf 100644 --- a/python/paddle/fluid/tests/unittests/test_jit_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_jit_save_load.py @@ -34,7 +34,6 @@ def random_batch_reader(input_size, label_size): - def _get_random_inputs_and_labels(input_size, label_size): np.random.seed(SEED) input = np.random.random(size=input_size).astype('float32') @@ -44,14 +43,14 @@ def _get_random_inputs_and_labels(input_size, label_size): def __reader__(): for _ in range(BATCH_NUM): batch_input, batch_label = _get_random_inputs_and_labels( - [BATCH_SIZE, input_size], [BATCH_SIZE, label_size]) + [BATCH_SIZE, input_size], [BATCH_SIZE, label_size] + ) yield batch_input, batch_label return __reader__ class LinearNet(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNet, self).__init__() self._linear = Linear(in_size, out_size) @@ -62,7 +61,6 @@ def forward(self, x): class LinearNetWithInputSpec(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetWithInputSpec, self).__init__() self._linear = Linear(in_size, out_size) @@ -73,7 +71,6 @@ def forward(self, x): class LinearNetNotDeclarative(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetNotDeclarative, self).__init__() self._linear = Linear(in_size, out_size) @@ -83,15 +80,16 @@ def forward(self, x): class LinerNetWithLabel(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(LinerNetWithLabel, self).__init__() self._linear = Linear(in_size, out_size) - @declarative(input_spec=[ - InputSpec(shape=[None, 784], dtype='float32', name="image"), - InputSpec(shape=[None, 1], dtype='int64', name="label") - ]) + @declarative( + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image"), + InputSpec(shape=[None, 1], dtype='int64', name="label"), + ] + ) def forward(self, x, label): out = self._linear(x) loss = fluid.layers.cross_entropy(out, label) @@ -100,15 +98,16 @@ def forward(self, x, label): class LinerNetWithPruneInput(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(LinerNetWithPruneInput, self).__init__() self._linear = Linear(in_size, out_size) - @declarative(input_spec=[ - InputSpec(shape=[None, 784], dtype='float32', name="image"), - InputSpec(shape=[None, 1], dtype='int64', name="label") - ]) + @declarative( + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image"), + InputSpec(shape=[None, 1], dtype='int64', name="label"), + ] + ) def forward(self, x, label): out = self._linear(x) loss = fluid.layers.cross_entropy(out, label) @@ -117,22 +116,22 @@ def forward(self, x, label): class LinerNetWithUselessInput(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(LinerNetWithUselessInput, self).__init__() self._linear = Linear(in_size, out_size) - @declarative(input_spec=[ - InputSpec(shape=[None, 784], dtype='float32', name="image"), - InputSpec(shape=[None, 1], dtype='int64', name="label") - ]) + @declarative( + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image"), + InputSpec(shape=[None, 1], dtype='int64', name="label"), + ] + ) def forward(self, x, label): out = self._linear(x) return out class LinearNetReturnLoss(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetReturnLoss, self).__init__() self._linear = Linear(in_size, out_size) @@ -146,16 +145,17 @@ def forward(self, x): class LinearNetMultiInput(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetMultiInput, self).__init__() self._linear1 = Linear(in_size, out_size) self._linear2 = Linear(in_size, out_size) - @declarative(input_spec=[ - InputSpec([None, 8], dtype='float32'), - InputSpec([None, 8], dtype='float32') - ]) + @declarative( + input_spec=[ + InputSpec([None, 8], dtype='float32'), + InputSpec([None, 8], dtype='float32'), + ] + ) def forward(self, x, y): x_out = self._linear1(x) y_out = self._linear2(y) @@ -164,14 +164,17 @@ def forward(self, x, y): class LinearNetMultiInput1(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetMultiInput1, self).__init__() self._linear1 = Linear(in_size, out_size) self._linear2 = Linear(in_size, out_size) - @declarative(input_spec=(InputSpec([None, 8], dtype='float32'), - InputSpec([None, 8], dtype='float32'))) + @declarative( + input_spec=( + InputSpec([None, 8], dtype='float32'), + InputSpec([None, 8], dtype='float32'), + ) + ) def forward(self, x, y): x_out = self._linear1(x) y_out = self._linear2(y) @@ -180,7 +183,6 @@ def forward(self, x, y): class MultiLoadingLinearNet(fluid.dygraph.Layer): - def __init__(self, size, model_path): super(MultiLoadingLinearNet, self).__init__() self._linear = Linear(size, size) @@ -197,7 +199,6 @@ def forward(self, x): class LinearNetReturnHidden(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetReturnHidden, self).__init__() self._linear_1 = Linear(in_size, out_size) @@ -212,7 +213,6 @@ def forward(self, x): class LinearNetWithNestOut(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetWithNestOut, self).__init__() self._linear_1 = Linear(in_size, out_size) @@ -228,18 +228,16 @@ def forward(self, x): class LinearNetWithDictInput(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(LinearNetWithDictInput, self).__init__() self._linear = Linear(in_size, out_size) - @paddle.jit.to_static(input_spec=[{ - 'img': - InputSpec(shape=[None, 8], dtype='float32', name='img') - }, { - 'label': - InputSpec(shape=[None, 1], dtype='int64', name='label') - }]) + @paddle.jit.to_static( + input_spec=[ + {'img': InputSpec(shape=[None, 8], dtype='float32', name='img')}, + {'label': InputSpec(shape=[None, 1], dtype='int64', name='label')}, + ] + ) def forward(self, img, label): out = self._linear(img['img']) # not return loss to avoid prune output @@ -248,7 +246,6 @@ def forward(self, img, label): class LinearNetWithDictInputNoPrune(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(LinearNetWithDictInputNoPrune, self).__init__() self._linear = Linear(in_size, out_size) @@ -259,7 +256,6 @@ def forward(self, img): class EmptyLayer(paddle.nn.Layer): - def __init__(self): super(EmptyLayer, self).__init__() @@ -269,7 +265,6 @@ def forward(self, x): class NoParamLayer(paddle.nn.Layer): - def __init__(self): super(NoParamLayer, self).__init__() @@ -279,7 +274,6 @@ def forward(self, x, y): class LinearNetWithMultiStaticFunc(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): super(LinearNetWithMultiStaticFunc, self).__init__() self._linear_0 = Linear(in_size, out_size) @@ -301,12 +295,14 @@ def forward_general(self, x): def train(layer, input_size=784, label_size=1): # create optimizer - sgd = fluid.optimizer.SGDOptimizer(learning_rate=0.01, - parameter_list=layer.parameters()) + sgd = fluid.optimizer.SGDOptimizer( + learning_rate=0.01, parameter_list=layer.parameters() + ) # create data loader train_loader = fluid.io.DataLoader.from_generator(capacity=5) - train_loader.set_batch_generator(random_batch_reader( - input_size, label_size)) + train_loader.set_batch_generator( + random_batch_reader(input_size, label_size) + ) # train for data in train_loader(): img, label = data @@ -325,12 +321,14 @@ def train(layer, input_size=784, label_size=1): def train_with_label(layer, input_size=784, label_size=1): # create optimizer - sgd = fluid.optimizer.SGDOptimizer(learning_rate=0.01, - parameter_list=layer.parameters()) + sgd = fluid.optimizer.SGDOptimizer( + learning_rate=0.01, parameter_list=layer.parameters() + ) # create data loader train_loader = fluid.io.DataLoader.from_generator(capacity=5) - train_loader.set_batch_generator(random_batch_reader( - input_size, label_size)) + train_loader.set_batch_generator( + random_batch_reader(input_size, label_size) + ) # train for data in train_loader(): img, label = data @@ -345,11 +343,11 @@ def train_with_label(layer, input_size=784, label_size=1): class TestJitSaveLoad(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() - self.model_path = os.path.join(self.temp_dir.name, - "test_jit_save_load/model") + self.model_path = os.path.join( + self.temp_dir.name, "test_jit_save_load/model" + ) # enable dygraph mode fluid.enable_dygraph() # config seed @@ -364,9 +362,9 @@ def train_and_save_model(self, model_path=None): example_inputs, layer, _ = train(layer) final_model_path = model_path if model_path else self.model_path orig_input_types = [type(x) for x in example_inputs] - paddle.jit.save(layer=layer, - path=final_model_path, - input_spec=example_inputs) + paddle.jit.save( + layer=layer, path=final_model_path, input_spec=example_inputs + ) new_input_types = [type(x) for x in example_inputs] self.assertEqual(orig_input_types, new_input_types) return layer @@ -385,10 +383,11 @@ def load_and_inference(self, train_layer, infer_layer): infer_layer.eval() # inference & compare x = fluid.dygraph.to_variable( - np.random.random((1, 784)).astype('float32')) + np.random.random((1, 784)).astype('float32') + ) np.testing.assert_array_equal( - train_layer(x).numpy(), - infer_layer(x).numpy()) + train_layer(x).numpy(), infer_layer(x).numpy() + ) def load_and_finetune(self, train_layer, load_train_layer): train_layer.train() @@ -396,8 +395,9 @@ def load_and_finetune(self, train_layer, load_train_layer): # train & compare img0, _, train_loss = train(train_layer) img1, _, load_train_loss = train(load_train_layer) - np.testing.assert_array_equal(train_loss.numpy(), - load_train_loss.numpy()) + np.testing.assert_array_equal( + train_loss.numpy(), load_train_loss.numpy() + ) def load_dygraph_state_dict(self, train_layer): train_layer.eval() @@ -411,26 +411,28 @@ def load_dygraph_state_dict(self, train_layer): new_layer.eval() # inference & compare x = fluid.dygraph.to_variable( - np.random.random((1, 784)).astype('float32')) + np.random.random((1, 784)).astype('float32') + ) np.testing.assert_array_equal( - train_layer(x).numpy(), - new_layer(x).numpy()) + train_layer(x).numpy(), new_layer(x).numpy() + ) def test_load_dygraph_no_path(self): - model_path = os.path.join(self.temp_dir.name, - "test_jit_save_load.no_path/model_path") + model_path = os.path.join( + self.temp_dir.name, "test_jit_save_load.no_path/model_path" + ) with self.assertRaises(ValueError): model_dict, _ = fluid.dygraph.load_dygraph(model_path) def test_jit_load_no_path(self): - path = os.path.join(self.temp_dir.name, - "test_jit_save_load.no_path/model_path") + path = os.path.join( + self.temp_dir.name, "test_jit_save_load.no_path/model_path" + ) with self.assertRaises(ValueError): loaded_layer = paddle.jit.load(path) class TestSaveLoadWithNestOut(unittest.TestCase): - def setUp(self): # enable dygraph mode fluid.enable_dygraph() @@ -441,7 +443,8 @@ def tearDown(self): def test_nest_output(self): x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) net = LinearNetWithNestOut(8, 8) dy_outs = flatten(net(x)) @@ -455,13 +458,12 @@ def test_nest_output(self): self.assertTrue(len(dy_outs) == 4) for dy_out, load_out in zip(dy_outs, load_outs): - np.testing.assert_allclose(dy_out.numpy(), - load_out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + dy_out.numpy(), load_out.numpy(), rtol=1e-05 + ) class TestSaveLoadWithDictInput(unittest.TestCase): - def test_dict_input(self): # NOTE: This net cannot be executed, it is just # a special case for exporting models in model validation @@ -473,17 +475,17 @@ def test_dict_input(self): # {'label': var label : fluid.VarType.LOD_TENSOR.shape(-1, 1).astype(VarType.INT64)}) self.assertEqual(len(net.forward.concrete_program.inputs), 3) temp_dir = tempfile.TemporaryDirectory() - path = os.path.join(temp_dir.name, - "test_jit_save_load_with_dict_input/model") + path = os.path.join( + temp_dir.name, "test_jit_save_load_with_dict_input/model" + ) # prune inputs - paddle.jit.save(layer=net, - path=path, - input_spec=[{ - 'img': - InputSpec(shape=[None, 8], - dtype='float32', - name='img') - }]) + paddle.jit.save( + layer=net, + path=path, + input_spec=[ + {'img': InputSpec(shape=[None, 8], dtype='float32', name='img')} + ], + ) img = paddle.randn(shape=[4, 8], dtype='float32') loaded_net = paddle.jit.load(path) @@ -496,25 +498,27 @@ def test_dict_input(self): class TestSaveLoadWithDictInputNoPrune(unittest.TestCase): - def test_dict_input(self): net = LinearNetWithDictInputNoPrune(8, 8) temp_dir = tempfile.TemporaryDirectory() path = os.path.join( - temp_dir.name, "test_jit_save_load_with_dict_input_no_prune/model") + temp_dir.name, "test_jit_save_load_with_dict_input_no_prune/model" + ) # prune inputs - paddle.jit.save(layer=net, - path=path, - input_spec=[{ - 'img': - InputSpec(shape=[None, 8], - dtype='float32', - name='img'), - 'img2': - InputSpec(shape=[None, 8], - dtype='float32', - name='img2') - }]) + paddle.jit.save( + layer=net, + path=path, + input_spec=[ + { + 'img': InputSpec( + shape=[None, 8], dtype='float32', name='img' + ), + 'img2': InputSpec( + shape=[None, 8], dtype='float32', name='img2' + ), + } + ], + ) img = paddle.randn(shape=[4, 8], dtype='float32') img2 = paddle.randn(shape=[4, 8], dtype='float32') @@ -526,7 +530,6 @@ def test_dict_input(self): class TestSaveLoadWithInputSpec(unittest.TestCase): - def setUp(self): # enable dygraph mode fluid.enable_dygraph() @@ -538,11 +541,13 @@ def tearDown(self): def test_with_input_spec(self): net = LinearNetReturnLoss(8, 8) # set x.shape = [None, 8] - net.forward = declarative(net.forward, - input_spec=[InputSpec([None, 8], name='x')]) + net.forward = declarative( + net.forward, input_spec=[InputSpec([None, 8], name='x')] + ) - model_path = os.path.join(self.temp_dir.name, - "input_spec.output_spec/model") + model_path = os.path.join( + self.temp_dir.name, "input_spec.output_spec/model" + ) # check inputs and outputs self.assertTrue(len(net.forward.inputs) == 1) input_x = net.forward.inputs[0] @@ -556,14 +561,16 @@ def test_with_input_spec(self): # 2. load to infer infer_layer = paddle.jit.load(model_path) x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) pred = infer_layer(x) def test_multi_in_out(self): net = LinearNetMultiInput(8, 8) - model_path = os.path.join(self.temp_dir.name, - "multi_inout.output_spec1/model") + model_path = os.path.join( + self.temp_dir.name, "multi_inout.output_spec1/model" + ) # 1. check inputs and outputs self.assertTrue(len(net.forward.inputs) == 2) input_x = net.forward.inputs[0] @@ -578,15 +585,18 @@ def test_multi_in_out(self): # 3. load to infer infer_layer = paddle.jit.load(model_path) x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) y = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) # 4. predict pred_x, pred_y = infer_layer(x, y) # 1. prune y and loss - model_path = os.path.join(self.temp_dir.name, - "multi_inout.output_spec2/model") + model_path = os.path.join( + self.temp_dir.name, "multi_inout.output_spec2/model" + ) output_spec = net.forward.outputs[:1] paddle.jit.save(net, model_path, [input_x], output_spec=output_spec) # 2. load again @@ -600,8 +610,9 @@ def test_multi_in_out(self): def test_multi_in_out1(self): net = LinearNetMultiInput1(8, 8) - model_path = os.path.join(self.temp_dir.name, - "multi_inout1.output_spec1/model") + model_path = os.path.join( + self.temp_dir.name, "multi_inout1.output_spec1/model" + ) # 1. check inputs and outputs self.assertTrue(len(net.forward.inputs) == 2) input_x = net.forward.inputs[0] @@ -616,17 +627,20 @@ def test_multi_in_out1(self): # 3. load to infer infer_layer = paddle.jit.load(model_path) x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) y = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) # 4. predict pred_x, pred_y = infer_layer(x, y) # 1. prune y and loss - model_path = os.path.join(self.temp_dir.name, - "multi_inout1.output_spec2/model") + model_path = os.path.join( + self.temp_dir.name, "multi_inout1.output_spec2/model" + ) output_spec = net.forward.outputs[:1] - paddle.jit.save(net, model_path, (input_x, ), output_spec=output_spec) + paddle.jit.save(net, model_path, (input_x,), output_spec=output_spec) # 2. load again infer_layer2 = paddle.jit.load(model_path) # 3. predict @@ -637,7 +651,6 @@ def test_multi_in_out1(self): class TestJitSaveLoadConfig(unittest.TestCase): - def setUp(self): # enable dygraph mode fluid.enable_dygraph() @@ -652,30 +665,36 @@ def tearDown(self): def test_output_spec(self): train_layer = LinearNetReturnLoss(8, 8) adam = fluid.optimizer.AdamOptimizer( - learning_rate=0.1, parameter_list=train_layer.parameters()) + learning_rate=0.1, parameter_list=train_layer.parameters() + ) x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) for i in range(10): out, loss = train_layer(x) loss.backward() adam.minimize(loss) train_layer.clear_gradients() - model_path = os.path.join(self.temp_dir.name, - "save_load_config.output_spec") + model_path = os.path.join( + self.temp_dir.name, "save_load_config.output_spec" + ) output_spec = [out] - paddle.jit.save(layer=train_layer, - path=model_path, - input_spec=[x], - output_spec=output_spec) + paddle.jit.save( + layer=train_layer, + path=model_path, + input_spec=[x], + output_spec=output_spec, + ) train_layer.eval() infer_layer = paddle.jit.load(model_path) x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) np.testing.assert_array_equal( - train_layer(x)[0].numpy(), - infer_layer(x).numpy()) + train_layer(x)[0].numpy(), infer_layer(x).numpy() + ) def test_save_no_support_config_error(self): layer = LinearNet(784, 1) @@ -700,12 +719,12 @@ def test_load_with_no_support_config(self): class TestJitMultipleLoading(unittest.TestCase): - def setUp(self): self.linear_size = 4 self.temp_dir = tempfile.TemporaryDirectory() - self.model_path = os.path.join(self.temp_dir.name, - "jit_multi_load/model") + self.model_path = os.path.join( + self.temp_dir.name, "jit_multi_load/model" + ) # enable dygraph mode fluid.enable_dygraph() # config seed @@ -720,13 +739,14 @@ def tearDown(self): def train_and_save_orig_model(self): layer = LinearNet(self.linear_size, self.linear_size) example_inputs, layer, _ = train(layer, self.linear_size, 1) - paddle.jit.save(layer=layer, - path=self.model_path, - input_spec=example_inputs) + paddle.jit.save( + layer=layer, path=self.model_path, input_spec=example_inputs + ) def test_load_model_retransform_inference(self): - multi_loaded_layer = MultiLoadingLinearNet(self.linear_size, - self.model_path) + multi_loaded_layer = MultiLoadingLinearNet( + self.linear_size, self.model_path + ) state_dict = multi_loaded_layer.state_dict() name_set = set() for _, var in state_dict.items(): @@ -735,12 +755,12 @@ def test_load_model_retransform_inference(self): class TestJitPruneModelAndLoad(unittest.TestCase): - def setUp(self): self.linear_size = 4 self.temp_dir = tempfile.TemporaryDirectory() - self.model_path = os.path.join(self.temp_dir.name, - "jit_prune_model_and_load/model") + self.model_path = os.path.join( + self.temp_dir.name, "jit_prune_model_and_load/model" + ) # enable dygraph mode fluid.enable_dygraph() # config seed @@ -753,9 +773,11 @@ def tearDown(self): def train_and_save(self): train_layer = LinearNetReturnHidden(8, 8) adam = fluid.optimizer.AdamOptimizer( - learning_rate=0.1, parameter_list=train_layer.parameters()) + learning_rate=0.1, parameter_list=train_layer.parameters() + ) x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) for i in range(10): hidden, loss = train_layer(x) loss.backward() @@ -763,10 +785,12 @@ def train_and_save(self): train_layer.clear_gradients() output_spec = [hidden] - paddle.jit.save(layer=train_layer, - path=self.model_path, - input_spec=[x], - output_spec=output_spec) + paddle.jit.save( + layer=train_layer, + path=self.model_path, + input_spec=[x], + output_spec=output_spec, + ) return train_layer @@ -777,10 +801,11 @@ def test_load_pruned_model(self): infer_layer = paddle.jit.load(self.model_path) x = fluid.dygraph.to_variable( - np.random.random((4, 8)).astype('float32')) + np.random.random((4, 8)).astype('float32') + ) np.testing.assert_array_equal( - train_layer(x)[0].numpy(), - infer_layer(x).numpy()) + train_layer(x)[0].numpy(), infer_layer(x).numpy() + ) def test_load_var_not_in_extra_var_info(self): self.train_and_save() @@ -798,7 +823,6 @@ def test_load_var_not_in_extra_var_info(self): class TestJitSaveMultiCases(unittest.TestCase): - def setUp(self): # enable dygraph mode fluid.enable_dygraph() @@ -810,11 +834,9 @@ def setUp(self): def tearDown(self): self.temp_dir.cleanup() - def verify_inference_correctness(self, - layer, - model_path, - with_label_and_loss=False, - with_label=False): + def verify_inference_correctness( + self, layer, model_path, with_label_and_loss=False, with_label=False + ): layer.eval() loaded_layer = paddle.jit.load(model_path) loaded_layer.eval() @@ -834,17 +856,19 @@ def verify_inference_correctness(self, np.testing.assert_array_equal( pred, loaded_pred, - err_msg= - 'Result diff when load and inference:\nlayer result:\n{}\nloaded layer result:\n{}' - .format(pred, loaded_pred)) + err_msg='Result diff when load and inference:\nlayer result:\n{}\nloaded layer result:\n{}'.format( + pred, loaded_pred + ), + ) def test_no_prune_to_static_after_train(self): layer = LinearNet(784, 1) train(layer) - model_path = os.path.join(self.temp_dir.name, - "test_no_prune_to_static_after_train/model") + model_path = os.path.join( + self.temp_dir.name, "test_no_prune_to_static_after_train/model" + ) paddle.jit.save(layer, model_path) self.verify_inference_correctness(layer, model_path) @@ -852,8 +876,9 @@ def test_no_prune_to_static_after_train(self): def test_no_prune_to_static_no_train(self): layer = LinearNetWithInputSpec(784, 1) - model_path = os.path.join(self.temp_dir.name, - "test_no_prune_to_static_no_train/model") + model_path = os.path.join( + self.temp_dir.name, "test_no_prune_to_static_no_train/model" + ) paddle.jit.save(layer, model_path) self.verify_inference_correctness(layer, model_path) @@ -864,11 +889,13 @@ def test_no_prune_no_to_static_after_train(self): train(layer) model_path = os.path.join( - self.temp_dir.name, "test_no_prune_no_to_static_after_train/model") + self.temp_dir.name, "test_no_prune_no_to_static_after_train/model" + ) paddle.jit.save( layer, model_path, - input_spec=[InputSpec(shape=[None, 784], dtype='float32')]) + input_spec=[InputSpec(shape=[None, 784], dtype='float32')], + ) self.verify_inference_correctness(layer, model_path) @@ -879,7 +906,8 @@ def test_no_prune_no_to_static_after_train_with_examples(self): model_path = os.path.join( self.temp_dir.name, - "test_no_prune_no_to_static_after_train_with_examples/model") + "test_no_prune_no_to_static_after_train_with_examples/model", + ) paddle.jit.save(layer=layer, path=model_path, input_spec=example_inputs) self.verify_inference_correctness(layer, model_path) @@ -887,12 +915,14 @@ def test_no_prune_no_to_static_after_train_with_examples(self): def test_no_prune_no_to_static_no_train(self): layer = LinearNetNotDeclarative(784, 1) - model_path = os.path.join(self.temp_dir.name, - "test_no_prune_no_to_static_no_train/model") + model_path = os.path.join( + self.temp_dir.name, "test_no_prune_no_to_static_no_train/model" + ) paddle.jit.save( layer, model_path, - input_spec=[InputSpec(shape=[None, 784], dtype='float32')]) + input_spec=[InputSpec(shape=[None, 784], dtype='float32')], + ) self.verify_inference_correctness(layer, model_path) @@ -901,54 +931,57 @@ def test_prune_to_static_after_train(self): out = train_with_label(layer) - model_path = os.path.join(self.temp_dir.name, - "test_prune_to_static_after_train/model") - paddle.jit.save(layer, - model_path, - input_spec=[ - InputSpec(shape=[None, 784], - dtype='float32', - name="image") - ], - output_spec=[out]) - - self.verify_inference_correctness(layer, - model_path, - with_label_and_loss=True) + model_path = os.path.join( + self.temp_dir.name, "test_prune_to_static_after_train/model" + ) + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image") + ], + output_spec=[out], + ) + + self.verify_inference_correctness( + layer, model_path, with_label_and_loss=True + ) def test_prune_to_static_no_train(self): layer = LinerNetWithLabel(784, 1) - model_path = os.path.join(self.temp_dir.name, - "test_prune_to_static_no_train/model") + model_path = os.path.join( + self.temp_dir.name, "test_prune_to_static_no_train/model" + ) # TODO: no train, cannot get output_spec var here # now only can use index output_spec = layer.forward.outputs[:1] - paddle.jit.save(layer, - model_path, - input_spec=[ - InputSpec(shape=[None, 784], - dtype='float32', - name="image") - ], - output_spec=output_spec) - - self.verify_inference_correctness(layer, - model_path, - with_label_and_loss=True) + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image") + ], + output_spec=output_spec, + ) + + self.verify_inference_correctness( + layer, model_path, with_label_and_loss=True + ) def test_prune_input_to_static_no_train(self): layer = LinerNetWithPruneInput(784, 1) - model_path = os.path.join(self.temp_dir.name, - "test_prune_input_to_static_no_train/model") - paddle.jit.save(layer, - model_path, - input_spec=[ - InputSpec(shape=[None, 784], - dtype='float32', - name="image") - ]) + model_path = os.path.join( + self.temp_dir.name, "test_prune_input_to_static_no_train/model" + ) + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image") + ], + ) self.verify_inference_correctness(layer, model_path, with_label=True) @@ -957,14 +990,15 @@ def test_prune_useless_input_to_static_no_train(self): model_path = os.path.join( self.temp_dir.name, - "test_prune_useless_input_to_static_no_train/model") - paddle.jit.save(layer, - model_path, - input_spec=[ - InputSpec(shape=[None, 784], - dtype='float32', - name="image") - ]) + "test_prune_useless_input_to_static_no_train/model", + ) + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image") + ], + ) self.verify_inference_correctness(layer, model_path, with_label=True) @@ -974,18 +1008,20 @@ def test_no_prune_input_spec_name_warning(self): train(layer) model_path = os.path.join( - self.temp_dir.name, "test_no_prune_input_spec_name_warning/model") + self.temp_dir.name, "test_no_prune_input_spec_name_warning/model" + ) paddle.jit.save( layer, model_path, - input_spec=[InputSpec(shape=[None, 784], dtype='float32')]) - paddle.jit.save(layer, - model_path, - input_spec=[ - InputSpec(shape=[None, 784], - dtype='float32', - name='feed_input') - ]) + input_spec=[InputSpec(shape=[None, 784], dtype='float32')], + ) + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name='feed_input') + ], + ) self.verify_inference_correctness(layer, model_path) @@ -995,7 +1031,8 @@ def test_not_prune_output_spec_name_warning(self): train(layer) model_path = os.path.join( - self.temp_dir.name, "test_not_prune_output_spec_name_warning/model") + self.temp_dir.name, "test_not_prune_output_spec_name_warning/model" + ) out = paddle.to_tensor(np.random.random((1, 1)).astype('float')) paddle.jit.save(layer, model_path, output_spec=[out]) @@ -1004,47 +1041,52 @@ def test_not_prune_output_spec_name_warning(self): def test_prune_input_spec_name_error(self): layer = LinerNetWithLabel(784, 1) - model_path = os.path.join(self.temp_dir.name, - "test_prune_input_spec_name_error/model") + model_path = os.path.join( + self.temp_dir.name, "test_prune_input_spec_name_error/model" + ) with self.assertRaises(ValueError): paddle.jit.save( layer, model_path, - input_spec=[InputSpec(shape=[None, 784], dtype='float32')]) + input_spec=[InputSpec(shape=[None, 784], dtype='float32')], + ) with self.assertRaises(ValueError): - paddle.jit.save(layer, - model_path, - input_spec=[ - InputSpec(shape=[None, 784], - dtype='float32', - name='feed_input') - ]) + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec( + shape=[None, 784], dtype='float32', name='feed_input' + ) + ], + ) def test_prune_output_spec_name_error(self): layer = LinerNetWithLabel(784, 1) train_with_label(layer) - model_path = os.path.join(self.temp_dir.name, - "test_prune_to_static_after_train/model") + model_path = os.path.join( + self.temp_dir.name, "test_prune_to_static_after_train/model" + ) out = paddle.to_tensor(np.random.random((1, 1)).astype('float')) with self.assertRaises(ValueError): - paddle.jit.save(layer, - model_path, - input_spec=[ - InputSpec(shape=[None, 784], - dtype='float32', - name="image") - ], - output_spec=[out]) + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec(shape=[None, 784], dtype='float32', name="image") + ], + output_spec=[out], + ) class TestJitSaveLoadEmptyLayer(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() - self.model_path = os.path.join(self.temp_dir.name, - "jit_save_load_empty_layer/model") + self.model_path = os.path.join( + self.temp_dir.name, "jit_save_load_empty_layer/model" + ) # enable dygraph mode paddle.disable_static() @@ -1062,11 +1104,11 @@ def test_save_load_empty_layer(self): class TestJitSaveLoadNoParamLayer(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() - self.model_path = os.path.join(self.temp_dir.name, - "jit_save_load_no_param_layer/model") + self.model_path = os.path.join( + self.temp_dir.name, "jit_save_load_no_param_layer/model" + ) # enable dygraph mode paddle.disable_static() @@ -1085,7 +1127,6 @@ def test_save_load_no_param_layer(self): class TestJitSaveLoadMultiMethods(unittest.TestCase): - def setUp(self): # enable dygraph mode paddle.disable_static() @@ -1096,7 +1137,8 @@ def tearDown(self): def test_jit_save_load_inference(self): model_path_inference = os.path.join( - self.temp_dir.name, "jit_save_load_multi_methods/model") + self.temp_dir.name, "jit_save_load_multi_methods/model" + ) IMAGE_SIZE = 224 layer = LinearNetWithMultiStaticFunc(IMAGE_SIZE, 10) inps = paddle.randn([1, IMAGE_SIZE]) @@ -1108,21 +1150,26 @@ def test_jit_save_load_inference(self): load_net = paddle.jit.load(model_path_inference) for func, result in result_origin.items(): self.assertTrue( - float((result - - getattr(load_net, func, None)(inps)).abs().max()) < 1e-5) + float( + (result - getattr(load_net, func, None)(inps)).abs().max() + ) + < 1e-5 + ) def test_jit_save_load_multi_methods_inputspec(self): - model_path = os.path.join(self.temp_dir.name, - 'jit_save_load_multi_methods/model') + model_path = os.path.join( + self.temp_dir.name, 'jit_save_load_multi_methods/model' + ) layer = LinearNetWithMultiStaticFunc(784, 1) with self.assertRaises(ValueError): - paddle.jit.save(layer, - model_path, - input_spec=[InputSpec(shape=[None, 784])]) + paddle.jit.save( + layer, model_path, input_spec=[InputSpec(shape=[None, 784])] + ) def test_parse_name(self): - model_path_inference = os.path.join(self.temp_dir.name, - "jit_save_load_parse_name/model") + model_path_inference = os.path.join( + self.temp_dir.name, "jit_save_load_parse_name/model" + ) IMAGE_SIZE = 224 layer = LinearNet(IMAGE_SIZE, 1) inps = paddle.randn([1, IMAGE_SIZE]) @@ -1135,7 +1182,6 @@ def test_parse_name(self): class LayerSaved(paddle.nn.Layer): - def __init__(self, in_size, out_size): super(LayerSaved, self).__init__() self.hidden = 100 @@ -1157,7 +1203,6 @@ def forward(self, x): class Net(paddle.nn.Layer): - def __init__(self): super().__init__() self.fc1 = paddle.nn.Linear(4, 4) @@ -1210,7 +1255,6 @@ def strs(self): class NetTensor(paddle.nn.Layer): - def __init__(self): super().__init__() self.fc1 = paddle.nn.Linear(4, 4) @@ -1231,7 +1275,6 @@ def fflag(self): class TestJitSaveCombineProperty(unittest.TestCase): - def setUp(self): # enable dygraph mode paddle.disable_static() @@ -1241,17 +1284,19 @@ def tearDown(self): self.temp_dir.cleanup() def test_jit_save_combine_property(self): - model_path = os.path.join(self.temp_dir.name, - "test_jit_save_combine/model") + model_path = os.path.join( + self.temp_dir.name, "test_jit_save_combine/model" + ) # Use new namespace with unique_name.guard(): net = Net() - #save + # save paddle.jit.save(net, model_path, combine_params=True) def test_jit_save_tensor_property(self): - model_path = os.path.join(self.temp_dir.name, - "test_jit_save_combine/model") + model_path = os.path.join( + self.temp_dir.name, "test_jit_save_combine/model" + ) # Use new namespace with unique_name.guard(): net = NetTensor() @@ -1260,7 +1305,6 @@ def test_jit_save_tensor_property(self): class LayerLoadFinetune(paddle.nn.Layer): - def __init__(self, in_size, out_size, load_path): super(LayerLoadFinetune, self).__init__() # Test duplicate name @@ -1294,7 +1338,6 @@ def forward(self, x): class TestJitSaveLoadSaveWithoutRunning(unittest.TestCase): - def setUp(self): # enable dygraph mode paddle.disable_static() @@ -1305,32 +1348,39 @@ def tearDown(self): def test_save_load_finetune_load(self): model_path = os.path.join( - self.temp_dir.name, "test_jit_save_load_save_without_running/model") + self.temp_dir.name, "test_jit_save_load_save_without_running/model" + ) IMAGE_SIZE = 224 inps0 = paddle.randn([1, IMAGE_SIZE]) inps1 = paddle.randn([2, IMAGE_SIZE]) # Use new namespace with unique_name.guard(): layer_save = LayerSaved(IMAGE_SIZE, IMAGE_SIZE) - #save - paddle.jit.save(layer_save, - model_path, - input_spec=[ - paddle.static.InputSpec(shape=[None, IMAGE_SIZE], - dtype='float32') - ]) + # save + paddle.jit.save( + layer_save, + model_path, + input_spec=[ + paddle.static.InputSpec( + shape=[None, IMAGE_SIZE], dtype='float32' + ) + ], + ) result_00 = layer_save(inps0) result_01 = layer_save(inps1) - #load and save without running + # load and save without running with unique_name.guard(): layer_load = paddle.jit.load(model_path) - paddle.jit.save(layer_load, - model_path, - input_spec=[ - paddle.static.InputSpec( - shape=[None, IMAGE_SIZE], dtype='float32') - ]) - #reload + paddle.jit.save( + layer_load, + model_path, + input_spec=[ + paddle.static.InputSpec( + shape=[None, IMAGE_SIZE], dtype='float32' + ) + ], + ) + # reload layer_reload = paddle.jit.load(model_path) result_10 = layer_reload(inps0) result_11 = layer_reload(inps1) @@ -1340,7 +1390,6 @@ def test_save_load_finetune_load(self): class TestJitSaveLoadFinetuneLoad(unittest.TestCase): - def setUp(self): # enable dygraph mode paddle.disable_static() @@ -1350,8 +1399,9 @@ def tearDown(self): self.temp_dir.cleanup() def test_save_load_finetune_load(self): - model_path = os.path.join(self.temp_dir.name, - "test_jit_save_load_finetune_load/model") + model_path = os.path.join( + self.temp_dir.name, "test_jit_save_load_finetune_load/model" + ) IMAGE_SIZE = 224 inps0 = paddle.randn([1, IMAGE_SIZE]) inps1 = paddle.randn([2, IMAGE_SIZE]) @@ -1359,18 +1409,18 @@ def test_save_load_finetune_load(self): with unique_name.guard(): layer_save = LayerSaved(IMAGE_SIZE, IMAGE_SIZE) layer_save(inps0) - #save + # save paddle.jit.save(layer_save, model_path) - #load + # load with unique_name.guard(): layer_load = LayerLoadFinetune(IMAGE_SIZE, IMAGE_SIZE, model_path) - #train + # train train(layer_load, input_size=IMAGE_SIZE) result_00 = layer_load(inps0) result_01 = layer_load(inps1) - #save + # save paddle.jit.save(layer_load, model_path) - #load + # load layer_finetune = paddle.jit.load(model_path) result_10 = layer_finetune(inps0) result_11 = layer_finetune(inps1) @@ -1385,7 +1435,6 @@ def test_save_load_finetune_load(self): # So divided into three TestCase: TestJitSaveLoadFunctionCase1, # TestJitSaveLoadFunctionCase2, TestJitSaveLoadFunctionCase3. class TestJitSaveLoadFunctionCase1(unittest.TestCase): - def setUp(self): paddle.disable_static() self.temp_dir = tempfile.TemporaryDirectory() @@ -1394,13 +1443,13 @@ def tearDown(self): self.temp_dir.cleanup() def test_jit_save_load_static_function(self): - @paddle.jit.to_static def fun(inputs): return paddle.tanh(inputs) - path = os.path.join(self.temp_dir.name, - 'test_jit_save_load_function_1/func') + path = os.path.join( + self.temp_dir.name, 'test_jit_save_load_function_1/func' + ) inps = paddle.rand([3, 6]) origin = fun(inps) @@ -1412,7 +1461,6 @@ def fun(inputs): class TestJitSaveLoadFunctionCase2(unittest.TestCase): - def setUp(self): paddle.disable_static() self.temp_dir = tempfile.TemporaryDirectory() @@ -1421,15 +1469,17 @@ def tearDown(self): self.temp_dir.cleanup() def test_jit_save_load_function_input_spec(self): - - @paddle.jit.to_static(input_spec=[ - InputSpec(shape=[None, 6], dtype='float32', name='x'), - ]) + @paddle.jit.to_static( + input_spec=[ + InputSpec(shape=[None, 6], dtype='float32', name='x'), + ] + ) def fun(inputs): return paddle.nn.functional.relu(inputs) - path = os.path.join(self.temp_dir.name, - 'test_jit_save_load_function_2/func') + path = os.path.join( + self.temp_dir.name, 'test_jit_save_load_function_2/func' + ) inps = paddle.rand([3, 6]) origin = fun(inps) @@ -1440,7 +1490,6 @@ def fun(inputs): class TestJitSaveLoadFunctionCase3(unittest.TestCase): - def setUp(self): paddle.disable_static() self.temp_dir = tempfile.TemporaryDirectory() @@ -1449,22 +1498,22 @@ def tearDown(self): self.temp_dir.cleanup() def test_jit_save_load_function_function(self): - def fun(inputs): return paddle.tanh(inputs) - path = os.path.join(self.temp_dir.name, - 'test_jit_save_load_function_3/func') + path = os.path.join( + self.temp_dir.name, 'test_jit_save_load_function_3/func' + ) inps = paddle.rand([3, 6]) origin = fun(inps) - paddle.jit.save(fun, - path, - input_spec=[ - InputSpec(shape=[None, 6], - dtype='float32', - name='x'), - ]) + paddle.jit.save( + fun, + path, + input_spec=[ + InputSpec(shape=[None, 6], dtype='float32', name='x'), + ], + ) load_func = paddle.jit.load(path) load_result = load_func(inps) @@ -1472,7 +1521,6 @@ def fun(inputs): class TestJitSaveLoadFunctionWithParamCase1(unittest.TestCase): - def setUp(self): paddle.disable_static() self.temp_dir = tempfile.TemporaryDirectory() @@ -1481,9 +1529,7 @@ def tearDown(self): self.temp_dir.cleanup() def test_jit_save_load_function(self): - class LinearNet(paddle.nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = paddle.nn.Linear(5, 6) @@ -1499,11 +1545,13 @@ def anothor_forward(self, x): inps = paddle.rand([3, 5]) origin = layer.anothor_forward(inps) - func = paddle.jit.to_static(layer.anothor_forward, - [paddle.static.InputSpec(shape=[-1, 5])]) + func = paddle.jit.to_static( + layer.anothor_forward, [paddle.static.InputSpec(shape=[-1, 5])] + ) path = os.path.join( self.temp_dir.name, - 'test_jit_save_load_function_with_params_case1/func') + 'test_jit_save_load_function_with_params_case1/func', + ) paddle.jit.save(func, path) load_func = paddle.jit.load(path) @@ -1512,7 +1560,6 @@ def anothor_forward(self, x): class TestJitSaveLoadFunctionWithParamCase2(unittest.TestCase): - def setUp(self): paddle.disable_static() self.temp_dir = tempfile.TemporaryDirectory() @@ -1521,9 +1568,7 @@ def tearDown(self): self.temp_dir.cleanup() def test_jit_save_load_function(self): - class LinearNet(paddle.nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = paddle.nn.Linear(5, 6) @@ -1541,19 +1586,20 @@ def anothor_forward(self, x): path = os.path.join( self.temp_dir.name, - 'test_jit_save_load_function_with_params_case2/func') + 'test_jit_save_load_function_with_params_case2/func', + ) paddle.jit.save(layer.anothor_forward, path) origin_result = layer.anothor_forward(inps) load_func = paddle.jit.load(path) load_result = load_func(inps) - np.testing.assert_array_equal(origin_result.numpy(), - load_result.numpy()) + np.testing.assert_array_equal( + origin_result.numpy(), load_result.numpy() + ) class TestJitSaveLoadFunctionWithParamCase3(unittest.TestCase): - def setUp(self): paddle.disable_static() self.temp_dir = tempfile.TemporaryDirectory() @@ -1562,9 +1608,7 @@ def tearDown(self): self.temp_dir.cleanup() def test_jit_save_load_function(self): - class LinearNet(paddle.nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = paddle.nn.Linear(5, 6) @@ -1583,7 +1627,8 @@ def anothor_forward(self, x): path = os.path.join( self.temp_dir.name, - 'test_jit_save_load_function_with_params_case3/func') + 'test_jit_save_load_function_with_params_case3/func', + ) paddle.jit.save(layer.anothor_forward, path) load_func = paddle.jit.load(path) @@ -1592,7 +1637,6 @@ def anothor_forward(self, x): class TestJitSaveLoadDataParallel(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -1610,18 +1654,20 @@ def verify_inference_correctness(self, layer, path): np.testing.assert_array_equal( pred, loaded_pred, - err_msg= - 'Result diff when load and inference:\nlayer result:\n{}\nloaded layer result:\n{}' - .format(pred, loaded_pred)) + err_msg='Result diff when load and inference:\nlayer result:\n{}\nloaded layer result:\n{}'.format( + pred, loaded_pred + ), + ) def test_jit_save_data_parallel_with_inputspec(self): layer = LinearNetNotDeclarative(784, 1) layer = paddle.DataParallel(layer) - path = os.path.join(self.temp_dir.name, - "jit_save_data_parallel_with_inputspec/model") - paddle.jit.save(layer=layer, - path=path, - input_spec=[InputSpec(shape=[None, 784])]) + path = os.path.join( + self.temp_dir.name, "jit_save_data_parallel_with_inputspec/model" + ) + paddle.jit.save( + layer=layer, path=path, input_spec=[InputSpec(shape=[None, 784])] + ) self.verify_inference_correctness(layer, path) @@ -1629,8 +1675,9 @@ def test_jit_save_data_parallel_with_to_static(self): layer = LinearNetWithInputSpec(784, 1) layer = paddle.DataParallel(layer) - path = os.path.join(self.temp_dir.name, - "jit_save_data_parallel_with_to_static/model") + path = os.path.join( + self.temp_dir.name, "jit_save_data_parallel_with_to_static/model" + ) paddle.jit.save(layer, path) self.verify_inference_correctness(layer, path) @@ -1641,16 +1688,17 @@ class InputSepcLayer(paddle.nn.Layer): A layer with InputSpec to test InputSpec compatibility ''' - @paddle.jit.to_static(input_spec=[ - InputSpec(shape=[None, 8], dtype='float32', name='x'), - InputSpec(shape=[None, 1], dtype='float64', name='y') - ]) + @paddle.jit.to_static( + input_spec=[ + InputSpec(shape=[None, 8], dtype='float32', name='x'), + InputSpec(shape=[None, 1], dtype='float64', name='y'), + ] + ) def forward(self, x, y): return x, y class TestInputSpecCompatibility(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -1662,15 +1710,18 @@ def _assert_input_spec_layer_return(self, expect_layer, test_layer): input_y = paddle.uniform([8, 1], dtype='float64') expected_result = expect_layer(input_x, input_y) test_result = test_layer(input_x, input_y) - np.testing.assert_allclose(expected_result[0].numpy(), - test_result[0].numpy()) - np.testing.assert_allclose(expected_result[1].numpy(), - test_result[1].numpy()) + np.testing.assert_allclose( + expected_result[0].numpy(), test_result[0].numpy() + ) + np.testing.assert_allclose( + expected_result[1].numpy(), test_result[1].numpy() + ) def test_jit_save_compatible_input_sepc(self): layer = InputSepcLayer() - save_dir = os.path.join(self.temp_dir.name, - "jit_save_compatible_input_spec") + save_dir = os.path.join( + self.temp_dir.name, "jit_save_compatible_input_spec" + ) path = save_dir + "/model" paddle.jit.save(layer=layer, path=path) @@ -1678,68 +1729,74 @@ def test_jit_save_compatible_input_sepc(self): self._assert_input_spec_layer_return(layer, no_input_spec_layer) shutil.rmtree(save_dir) - paddle.jit.save(layer=layer, - path=path, - input_spec=[ - InputSpec(shape=[None, 8], - dtype='float32', - name='x'), - InputSpec(shape=[None, 1], - dtype='float64', - name='y') - ]) + paddle.jit.save( + layer=layer, + path=path, + input_spec=[ + InputSpec(shape=[None, 8], dtype='float32', name='x'), + InputSpec(shape=[None, 1], dtype='float64', name='y'), + ], + ) same_input_spec_layer = paddle.jit.load(path) self._assert_input_spec_layer_return(layer, same_input_spec_layer) shutil.rmtree(save_dir) - paddle.jit.save(layer=layer, - path=path, - input_spec=[ - InputSpec(shape=[8, 8], dtype='float32'), - InputSpec(shape=[8, -1], dtype='float64') - ]) + paddle.jit.save( + layer=layer, + path=path, + input_spec=[ + InputSpec(shape=[8, 8], dtype='float32'), + InputSpec(shape=[8, -1], dtype='float64'), + ], + ) compatible_input_spec_layer = paddle.jit.load(path) self._assert_input_spec_layer_return(layer, compatible_input_spec_layer) shutil.rmtree(save_dir) def test_jit_save_incompatible_input_sepc(self): layer = InputSepcLayer() - save_dir = os.path.join(self.temp_dir.name, - "jit_save_compatible_input_spec") + save_dir = os.path.join( + self.temp_dir.name, "jit_save_compatible_input_spec" + ) path = save_dir + "/model" with self.assertRaises(ValueError): # type mismatch - paddle.jit.save(layer=layer, - path=path, - input_spec=[ - InputSpec(shape=[None, 8], dtype='float64'), - InputSpec(shape=[None, 1], dtype='float64') - ]) + paddle.jit.save( + layer=layer, + path=path, + input_spec=[ + InputSpec(shape=[None, 8], dtype='float64'), + InputSpec(shape=[None, 1], dtype='float64'), + ], + ) with self.assertRaises(ValueError): # shape len mismatch - paddle.jit.save(layer=layer, - path=path, - input_spec=[ - InputSpec(shape=[None, 8, 1], dtype='float32'), - InputSpec(shape=[None, 1], dtype='float64') - ]) + paddle.jit.save( + layer=layer, + path=path, + input_spec=[ + InputSpec(shape=[None, 8, 1], dtype='float32'), + InputSpec(shape=[None, 1], dtype='float64'), + ], + ) with self.assertRaises(ValueError): # shape mismatch - paddle.jit.save(layer=layer, - path=path, - input_spec=[ - InputSpec(shape=[None, 8], dtype='float32'), - InputSpec(shape=[None, 2], dtype='float64') - ]) + paddle.jit.save( + layer=layer, + path=path, + input_spec=[ + InputSpec(shape=[None, 8], dtype='float32'), + InputSpec(shape=[None, 2], dtype='float64'), + ], + ) if os.path.exists(save_dir): shutil.rmtree(save_dir) class NotJitForward(paddle.nn.Layer): - def __init__(self): super(NotJitForward, self).__init__() @@ -1748,7 +1805,6 @@ def forward(self, x, y): class TestNotJitForward(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() diff --git a/python/paddle/fluid/tests/unittests/test_kldiv_loss_op.py b/python/paddle/fluid/tests/unittests/test_kldiv_loss_op.py index 6358854255d91..9fd0868f1e4c7 100644 --- a/python/paddle/fluid/tests/unittests/test_kldiv_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_kldiv_loss_op.py @@ -36,7 +36,6 @@ def kldiv_loss(x, target, reduction): class TestKLDivLossOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'kldiv_loss' @@ -57,10 +56,9 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Loss', - no_grad_set=set(["Target"]), - check_eager=True) + self.check_grad( + ['X'], 'Loss', no_grad_set=set(["Target"]), check_eager=True + ) def initTestCase(self): self.x_shape = (4, 5, 5) @@ -68,28 +66,24 @@ def initTestCase(self): class TestKLDivLossOp2(TestKLDivLossOp): - def initTestCase(self): self.x_shape = (3, 2, 7, 7) self.reduction = 'none' class TestKLDivLossOp3(TestKLDivLossOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 7, 9) self.reduction = 'mean' class TestKLDivLossOp4(TestKLDivLossOp): - def initTestCase(self): self.x_shape = (5, 20) self.reduction = 'sum' class TestKLDivLossDygraph(unittest.TestCase): - def run_kl_loss(self, reduction, shape=(5, 20)): x = np.random.uniform(-10, 10, shape).astype('float64') target = np.random.uniform(-10, 10, shape).astype('float64') @@ -97,8 +91,9 @@ def run_kl_loss(self, reduction, shape=(5, 20)): with paddle.fluid.dygraph.guard(): kldiv_criterion = paddle.nn.KLDivLoss(reduction) - pred_loss = kldiv_criterion(paddle.to_tensor(x), - paddle.to_tensor(target)) + pred_loss = kldiv_criterion( + paddle.to_tensor(x), paddle.to_tensor(target) + ) np.testing.assert_allclose(pred_loss.numpy(), gt_loss, rtol=1e-05) def test_kl_loss_batchmean(self): @@ -126,7 +121,6 @@ def test_kl_loss_static_api(self): class TestKLDivLossTypePromotion(unittest.TestCase): - def test_kl_div_promotion(self): with paddle.fluid.dygraph.guard(): diff --git a/python/paddle/fluid/tests/unittests/test_kron_op.py b/python/paddle/fluid/tests/unittests/test_kron_op.py index b29df857b6286..a2f1657a06e66 100644 --- a/python/paddle/fluid/tests/unittests/test_kron_op.py +++ b/python/paddle/fluid/tests/unittests/test_kron_op.py @@ -23,7 +23,6 @@ class TestKronOp(OpTest): - def setUp(self): self.op_type = "kron" self.python_api = paddle.kron @@ -51,7 +50,6 @@ def test_check_grad_ignore_y(self): class TestKronOp2(TestKronOp): - def setUp(self): self.op_type = "kron" self.python_api = paddle.kron @@ -64,7 +62,6 @@ def setUp(self): class TestKronOp3(TestKronOp): - def setUp(self): self.op_type = "kron" self.python_api = paddle.kron @@ -77,7 +74,6 @@ def setUp(self): class TestKronLayer(unittest.TestCase): - def test_case(self): a = np.random.randn(10, 10).astype(np.float64) b = np.random.randn(10, 10).astype(np.float64) @@ -104,7 +100,7 @@ def test_case_with_output(self): place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(start) - c, = exe.run(main, feed={'a': a, 'b': b}, fetch_list=[out_var]) + (c,) = exe.run(main, feed={'a': a, 'b': b}, fetch_list=[out_var]) np.testing.assert_allclose(c, np.kron(a, b)) def test_api_eager_dygraph(self): @@ -114,7 +110,6 @@ def test_api_eager_dygraph(self): class TestComplexKronOp(OpTest): - def setUp(self): self.op_type = "kron" self.python_api = paddle.kron @@ -127,7 +122,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': -1, 'use_mkldnn': False} self.outputs = {'Out': self.out} @@ -137,14 +132,17 @@ def init_base_dtype(self): def init_input_output(self): self.x = np.random.random(self.x_shape).astype( - self.dtype) + 1J * np.random.random(self.x_shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.x_shape).astype(self.dtype) self.y = np.random.random(self.y_shape).astype( - self.dtype) + 1J * np.random.random(self.y_shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.y_shape).astype(self.dtype) self.out = np.kron(self.x, self.y) def init_grad_input_output(self): - self.grad_out = np.ones(self.out_shape, self.dtype) + 1J * np.ones( - self.out_shape, self.dtype) + self.grad_out = np.ones(self.out_shape, self.dtype) + 1j * np.ones( + self.out_shape, self.dtype + ) self.grad_x = self.get_grad_x_by_numpy() self.grad_y = self.get_grad_y_by_numpy() @@ -157,7 +155,8 @@ def get_grad_x_by_numpy(self): idx_i = x_i * self.y_shape[0] + i idx_j = x_j * self.y_shape[1] + j grad_x[x_i][x_j] += self.grad_out[idx_i][ - idx_j] * np.conj(self.y[i][j]) + idx_j + ] * np.conj(self.y[i][j]) return grad_x def get_grad_y_by_numpy(self): @@ -169,47 +168,55 @@ def get_grad_y_by_numpy(self): idx_i = x_i * self.y_shape[0] + y_i idx_j = x_j * self.y_shape[1] + y_j grad_y[y_i][y_j] += self.grad_out[idx_i][ - idx_j] * np.conj(self.x[x_i][x_j]) + idx_j + ] * np.conj(self.x[x_i][x_j]) return grad_y def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) class TestKronOpTypePromotion(TestComplexKronOp): - def init_input_output(self): self.x = np.random.random(self.x_shape).astype(self.dtype) self.y = np.random.random(self.y_shape).astype( - self.dtype) + 1J * np.random.random(self.y_shape).astype(self.dtype) + self.dtype + ) + 1j * np.random.random(self.y_shape).astype(self.dtype) self.out = np.kron(self.x, self.y) def init_grad_input_output(self): - self.grad_out = np.ones(self.out_shape, self.dtype) + 1J * np.ones( - self.out_shape, self.dtype) + self.grad_out = np.ones(self.out_shape, self.dtype) + 1j * np.ones( + self.out_shape, self.dtype + ) self.grad_x = self.get_grad_x_by_numpy().real self.grad_y = self.get_grad_y_by_numpy() diff --git a/python/paddle/fluid/tests/unittests/test_kthvalue_op.py b/python/paddle/fluid/tests/unittests/test_kthvalue_op.py index 823efffd3bab8..f3f5ccf18797d 100644 --- a/python/paddle/fluid/tests/unittests/test_kthvalue_op.py +++ b/python/paddle/fluid/tests/unittests/test_kthvalue_op.py @@ -33,7 +33,6 @@ def cal_kthvalue(x, k, axis, keepdim=False): class TestKthvalueOp(OpTest): - def init_args(self): self.k = 5 self.axis = -1 @@ -46,9 +45,9 @@ def setUp(self): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis} - output, indices = cal_kthvalue(self.input_data, - k=self.k, - axis=self.axis) + output, indices = cal_kthvalue( + self.input_data, k=self.k, axis=self.axis + ) self.outputs = {'Out': output, 'Indices': indices} def test_check_output(self): @@ -61,7 +60,6 @@ def test_check_grad(self): class TestKthvalueOpWithKeepdim(OpTest): - def init_args(self): self.k = 2 self.axis = 1 @@ -74,10 +72,9 @@ def setUp(self): self.input_data = np.random.random((1, 3, 2, 4, 10)) self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'keepdim': True} - output, indices = cal_kthvalue(self.input_data, - k=self.k, - axis=self.axis, - keepdim=True) + output, indices = cal_kthvalue( + self.input_data, k=self.k, axis=self.axis, keepdim=True + ) self.outputs = {'Out': output, 'Indices': indices} def test_check_output(self): @@ -90,7 +87,6 @@ def test_check_grad(self): class TestKthvalueOpKernels(unittest.TestCase): - def setUp(self): self.axises = [2, -1] @@ -107,9 +103,9 @@ def test_cpu_kernel(): value_expect, indice_expect = cal_kthvalue(inputs, k, axis) v, inds = paddle.kthvalue(tensor, k, axis) np.testing.assert_allclose(v.numpy(), value_expect, rtol=1e-05) - np.testing.assert_allclose(inds.numpy(), - indice_expect, - rtol=1e-05) + np.testing.assert_allclose( + inds.numpy(), indice_expect, rtol=1e-05 + ) def test_gpu_kernel(): shape = (2, 30, 250) @@ -121,9 +117,9 @@ def test_gpu_kernel(): value_expect, indice_expect = cal_kthvalue(inputs, k, axis) v, inds = paddle.kthvalue(tensor, k, axis) np.testing.assert_allclose(v.numpy(), value_expect, rtol=1e-05) - np.testing.assert_allclose(inds.numpy(), - indice_expect, - rtol=1e-05) + np.testing.assert_allclose( + inds.numpy(), indice_expect, rtol=1e-05 + ) test_cpu_kernel() if fluid.core.is_compiled_with_cuda(): @@ -131,13 +127,11 @@ def test_gpu_kernel(): class TestKthvalueOpWithNaN(unittest.TestCase): - def setUp(self): paddle.disable_static() self.x = paddle.uniform([2, 200, 10], dtype='float32') def test_errors(self): - def test_nan_in_cpu_kernel(): paddle.set_device('cpu') nan_position = 100 @@ -160,7 +154,6 @@ def test_nan_in_gpu_kernel(): class TestKthvalueOpErrors(unittest.TestCase): - def setUp(self): self.x = paddle.uniform([2, 10, 20, 25], dtype='float32') @@ -184,7 +177,6 @@ def test_dim_range_error(): class TestModeOpInStatic(unittest.TestCase): - def setUp(self): np.random.seed(666) self.input_data = np.random.random((2, 20, 1, 2, 80)).astype(np.float64) @@ -192,16 +184,18 @@ def setUp(self): def test_run_static(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - input_tensor = paddle.static.data(name="x", - shape=[2, 20, 1, 2, 80], - dtype="float64") + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + input_tensor = paddle.static.data( + name="x", shape=[2, 20, 1, 2, 80], dtype="float64" + ) result = paddle.kthvalue(input_tensor, self.k, axis=1) expect_value = cal_kthvalue(self.input_data, self.k, axis=1)[0] exe = paddle.static.Executor(paddle.CPUPlace()) - paddle_result = exe.run(feed={"x": self.input_data}, - fetch_list=[result])[0] + paddle_result = exe.run( + feed={"x": self.input_data}, fetch_list=[result] + )[0] np.testing.assert_allclose(paddle_result, expect_value, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_l1_loss.py b/python/paddle/fluid/tests/unittests/test_l1_loss.py index bd68a125e3a26..00b716ce87154 100644 --- a/python/paddle/fluid/tests/unittests/test_l1_loss.py +++ b/python/paddle/fluid/tests/unittests/test_l1_loss.py @@ -19,7 +19,6 @@ class TestFunctionalL1Loss(unittest.TestCase): - def setUp(self): self.input_np = np.random.random(size=(10, 10, 5)).astype(np.float32) self.label_np = np.random.random(size=(10, 10, 5)).astype(np.float32) @@ -43,12 +42,12 @@ def run_imperative(self): self.assertTrue(dy_result.shape, [10, 10, 5]) def run_static(self, use_gpu=False): - input = paddle.fluid.data(name='input', - shape=[10, 10, 5], - dtype='float32') - label = paddle.fluid.data(name='label', - shape=[10, 10, 5], - dtype='float32') + input = paddle.fluid.data( + name='input', shape=[10, 10, 5], dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=[10, 10, 5], dtype='float32' + ) result0 = paddle.nn.functional.l1_loss(input, label) result1 = paddle.nn.functional.l1_loss(input, label, reduction='sum') result2 = paddle.nn.functional.l1_loss(input, label, reduction='none') @@ -57,11 +56,10 @@ def run_static(self, use_gpu=False): place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - static_result = exe.run(feed={ - "input": self.input_np, - "label": self.label_np - }, - fetch_list=[result0, result1, result2]) + static_result = exe.run( + feed={"input": self.input_np, "label": self.label_np}, + fetch_list=[result0, result1, result2], + ) expected = np.mean(np.abs(self.input_np - self.label_np)) np.testing.assert_allclose(static_result[0], expected, rtol=1e-05) @@ -93,23 +91,21 @@ def test_gpu(self): # test case the raise message def test_errors(self): - def test_value_error(): - input = paddle.fluid.data(name='input', - shape=[10, 10, 5], - dtype='float32') - label = paddle.fluid.data(name='label', - shape=[10, 10, 5], - dtype='float32') - loss = paddle.nn.functional.l1_loss(input, - label, - reduction='reduce_mean') + input = paddle.fluid.data( + name='input', shape=[10, 10, 5], dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=[10, 10, 5], dtype='float32' + ) + loss = paddle.nn.functional.l1_loss( + input, label, reduction='reduce_mean' + ) self.assertRaises(ValueError, test_value_error) class TestClassL1Loss(unittest.TestCase): - def setUp(self): self.input_np = np.random.random(size=(10, 10, 5)).astype(np.float32) self.label_np = np.random.random(size=(10, 10, 5)).astype(np.float32) @@ -136,12 +132,12 @@ def run_imperative(self): self.assertTrue(dy_result.shape, [10, 10, 5]) def run_static(self, use_gpu=False): - input = paddle.fluid.data(name='input', - shape=[10, 10, 5], - dtype='float32') - label = paddle.fluid.data(name='label', - shape=[10, 10, 5], - dtype='float32') + input = paddle.fluid.data( + name='input', shape=[10, 10, 5], dtype='float32' + ) + label = paddle.fluid.data( + name='label', shape=[10, 10, 5], dtype='float32' + ) l1_loss = paddle.nn.loss.L1Loss() result0 = l1_loss(input, label) l1_loss = paddle.nn.loss.L1Loss(reduction='sum') @@ -154,11 +150,10 @@ def run_static(self, use_gpu=False): place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - static_result = exe.run(feed={ - "input": self.input_np, - "label": self.label_np - }, - fetch_list=[result0, result1, result2]) + static_result = exe.run( + feed={"input": self.input_np, "label": self.label_np}, + fetch_list=[result0, result1, result2], + ) expected = np.mean(np.abs(self.input_np - self.label_np)) np.testing.assert_allclose(static_result[0], expected, rtol=1e-05) @@ -189,7 +184,6 @@ def test_gpu(self): # test case the raise message def test_errors(self): - def test_value_error(): loss = paddle.nn.loss.L1Loss(reduction="reduce_mean") diff --git a/python/paddle/fluid/tests/unittests/test_l1_norm_op.py b/python/paddle/fluid/tests/unittests/test_l1_norm_op.py index bc568dd5414a0..03d69e2334750 100644 --- a/python/paddle/fluid/tests/unittests/test_l1_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_l1_norm_op.py @@ -18,8 +18,7 @@ class TestL1NormOp(OpTest): - """Test l1_norm - """ + """Test l1_norm""" def setUp(self): self.op_type = "l1_norm" diff --git a/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py b/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py index c5859b7311659..12b2f0a47148a 100644 --- a/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py +++ b/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py @@ -21,13 +21,14 @@ class LabelSmoothTestCase(unittest.TestCase): - - def __init__(self, - methodName='runTest', - label_shape=(20, 1), - prior_dist=None, - epsilon=0.1, - dtype="float32"): + def __init__( + self, + methodName='runTest', + label_shape=(20, 1), + prior_dist=None, + epsilon=0.1, + dtype="float32", + ): super(LabelSmoothTestCase, self).__init__(methodName) self.label_shape = label_shape @@ -44,17 +45,19 @@ def fluid_layer(self, place): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - label_var = fluid.data("input", - self.label_shape, - dtype=self.dtype) - y_var = fluid.layers.label_smooth(label_var, - prior_dist=self.prior_dist, - epsilon=self.epsilon, - dtype=self.dtype) + label_var = fluid.data( + "input", self.label_shape, dtype=self.dtype + ) + y_var = fluid.layers.label_smooth( + label_var, + prior_dist=self.prior_dist, + epsilon=self.epsilon, + dtype=self.dtype, + ) feed_dict = {"input": self.label} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def functional(self, place): @@ -63,24 +66,24 @@ def functional(self, place): start = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, start): - label_var = fluid.data("input", - self.label_shape, - dtype=self.dtype) - y_var = F.label_smooth(label_var, - prior_dist=self.prior_dist, - epsilon=self.epsilon) + label_var = fluid.data( + "input", self.label_shape, dtype=self.dtype + ) + y_var = F.label_smooth( + label_var, prior_dist=self.prior_dist, epsilon=self.epsilon + ) feed_dict = {"input": self.label} exe = fluid.Executor(place) exe.run(start) - y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var]) + (y_np,) = exe.run(main, feed=feed_dict, fetch_list=[y_var]) return y_np def paddle_dygraph_layer(self): paddle.disable_static() label_var = dg.to_variable(self.label) - y_var = F.label_smooth(label_var, - prior_dist=self.prior_dist, - epsilon=self.epsilon) + y_var = F.label_smooth( + label_var, prior_dist=self.prior_dist, epsilon=self.epsilon + ) y_np = y_var.numpy() return y_np @@ -101,7 +104,6 @@ def runTest(self): class LabelSmoothErrorTestCase(LabelSmoothTestCase): - def runTest(self): place = fluid.CPUPlace() with dg.guard(place): @@ -112,7 +114,8 @@ def runTest(self): def add_cases(suite): suite.addTest(LabelSmoothTestCase(methodName='runTest')) suite.addTest( - LabelSmoothTestCase(methodName='runTest', label_shape=[2, 3, 1])) + LabelSmoothTestCase(methodName='runTest', label_shape=[2, 3, 1]) + ) def add_error_cases(suite): diff --git a/python/paddle/fluid/tests/unittests/test_label_smooth_op.py b/python/paddle/fluid/tests/unittests/test_label_smooth_op.py index 0fe5862f2c9c6..9166d07c741ed 100644 --- a/python/paddle/fluid/tests/unittests/test_label_smooth_op.py +++ b/python/paddle/fluid/tests/unittests/test_label_smooth_op.py @@ -19,7 +19,6 @@ class TestLabelSmoothOp(OpTest): - def config(self): self.op_type = "label_smooth" self.python_api = paddle.nn.functional.label_smooth @@ -32,7 +31,8 @@ def config(self): def setUp(self): self.config() smoothed_label = ( - 1 - self.epsilon) * self.label + self.epsilon / self.label_dim + 1 - self.epsilon + ) * self.label + self.epsilon / self.label_dim self.inputs = {'X': self.label} self.attrs = {'epsilon': self.epsilon} self.outputs = {'Out': smoothed_label} @@ -45,7 +45,6 @@ def test_check_grad(self): class TestLabelSmoothOpWithPriorDist(TestLabelSmoothOp): - def setUp(self): self.config() dist = np.random.random((1, self.label_dim)) @@ -56,23 +55,25 @@ def setUp(self): class TestLabelSmoothOp3D(TestLabelSmoothOp): - def setUp(self): super(TestLabelSmoothOp3D, self).setUp() self.inputs['X'] = self.inputs['X'].reshape( - [2, -1, self.inputs['X'].shape[-1]]) + [2, -1, self.inputs['X'].shape[-1]] + ) self.outputs['Out'] = self.outputs['Out'].reshape( - self.inputs['X'].shape) + self.inputs['X'].shape + ) class TestLabelSmoothOpWithPriorDist3D(TestLabelSmoothOpWithPriorDist): - def setUp(self): super(TestLabelSmoothOpWithPriorDist3D, self).setUp() self.inputs['X'] = self.inputs['X'].reshape( - [2, -1, self.inputs['X'].shape[-1]]) + [2, -1, self.inputs['X'].shape[-1]] + ) self.outputs['Out'] = self.outputs['Out'].reshape( - self.inputs['X'].shape) + self.inputs['X'].shape + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_lamb_op.py b/python/paddle/fluid/tests/unittests/test_lamb_op.py index eaa8db7031308..9c3bbe6d830bc 100644 --- a/python/paddle/fluid/tests/unittests/test_lamb_op.py +++ b/python/paddle/fluid/tests/unittests/test_lamb_op.py @@ -23,18 +23,16 @@ class TestLambOp1(OpTest): - def set_attrs(self): self.attrs = { 'epsilon': 1e-4, 'beta1': 0.78, 'beta2': 0.836, - 'weight_decay': 0.01 + 'weight_decay': 0.01, } def setUp(self): - '''Test Lamb Op with supplied attributes - ''' + '''Test Lamb Op with supplied attributes''' self.op_type = "lamb" param = np.random.uniform(-1, 1, (102, 105)).astype("float32") grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") @@ -53,19 +51,23 @@ def setUp(self): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } - - param_out, moment1_out, moment2_out, \ - beta1_pow_out, beta2_pow_out = lamb_step(self.inputs, self.attrs) + ( + param_out, + moment1_out, + moment2_out, + beta1_pow_out, + beta2_pow_out, + ) = lamb_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out + 'Beta2PowOut': beta2_pow_out, } def test_check_output(self): @@ -73,38 +75,41 @@ def test_check_output(self): class TestLambOp2(TestLambOp1): - def set_attrs(self): self.attrs = { 'epsilon': 1e-8, 'beta1': 0.9, 'beta2': 0.999, - 'weight_decay': 0.01 + 'weight_decay': 0.01, } class TestLambOpMultipleSteps(TestLambOp1): - def set_attrs(self): self.attrs = { 'epsilon': 1e-8, 'beta1': 0.9, 'beta2': 0.999, - 'weight_decay': 0.01 + 'weight_decay': 0.01, } self.num_steps = 10 def test_check_output(self): for i in range(self.num_steps): - param_out, moment1_out, moment2_out, \ - beta1_pow_out, beta2_pow_out = lamb_step(self.inputs, self.attrs) + ( + param_out, + moment1_out, + moment2_out, + beta1_pow_out, + beta2_pow_out, + ) = lamb_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out + 'Beta2PowOut': beta2_pow_out, } # Verify output for this step @@ -120,8 +125,9 @@ def test_check_output(self): self.inputs['Beta2Pow'] = beta2_pow_out # Randomize gradient for next step - self.inputs['Grad'] = np.random.uniform( - -1, 1, (102, 105)).astype("float32") + self.inputs['Grad'] = np.random.uniform(-1, 1, (102, 105)).astype( + "float32" + ) def lamb_step(inputs, attributes): @@ -152,14 +158,16 @@ def lamb_step(inputs, attributes): moment2_unbiased = moment2_out / (1 - beta2_pow) r_1 = np.linalg.norm(param) - r_2 = np.linalg.norm(moment1_unbiased / - (np.sqrt(moment2_unbiased) + epsilon) + - weight_decay * param) + r_2 = np.linalg.norm( + moment1_unbiased / (np.sqrt(moment2_unbiased) + epsilon) + + weight_decay * param + ) lr_t = lr * r_1 / r_2 - param_out = param - lr_t * (moment1_unbiased / - (np.sqrt(moment2_unbiased) + epsilon) + - weight_decay * param) + param_out = param - lr_t * ( + moment1_unbiased / (np.sqrt(moment2_unbiased) + epsilon) + + weight_decay * param + ) beta1_pow_out = beta1_pow * beta1 beta2_pow_out = beta2_pow * beta2 @@ -195,25 +203,32 @@ def lamb_step_sparse(inputs, attributes, height, rows, row_numel, np_grad): moment2_unbiased = np.zeros(shape=[height, row_numel]) def update_mom(row_id, update_value): - moment1_out[row_id] = beta1 * moment1[row_id] + (1 - - beta1) * update_value - moment2_out[row_id] = beta2 * moment2[row_id] + ( - 1 - beta2) * np.square(update_value) - - moment1_out[row_id] = beta1 * moment1[row_id] + (1 - - beta1) * update_value - moment2_out[row_id] = beta2 * moment2[row_id] + ( - 1 - beta2) * np.square(update_value) + moment1_out[row_id] = ( + beta1 * moment1[row_id] + (1 - beta1) * update_value + ) + moment2_out[row_id] = beta2 * moment2[row_id] + (1 - beta2) * np.square( + update_value + ) + + moment1_out[row_id] = ( + beta1 * moment1[row_id] + (1 - beta1) * update_value + ) + moment2_out[row_id] = beta2 * moment2[row_id] + (1 - beta2) * np.square( + update_value + ) def update_param(): r_1 = np.linalg.norm(param) - r_2 = np.linalg.norm(moment1_out / (np.sqrt(moment2_out) + epsilon) + - weight_decay * param) + r_2 = np.linalg.norm( + moment1_out / (np.sqrt(moment2_out) + epsilon) + + weight_decay * param + ) lr_t = lr * r_1 / r_2 - param_out = param - lr_t * (moment1_out / - (np.sqrt(moment2_out) + epsilon) + - weight_decay * param) + param_out = param - lr_t * ( + moment1_out / (np.sqrt(moment2_out) + epsilon) + + weight_decay * param + ) for row_id in range(param_out.shape[0]): update_value = np.zeros(np_grad[0].shape).astype("float32") @@ -229,7 +244,6 @@ def update_param(): class TestSparseLambOp(unittest.TestCase): - def setup(self, scope, place): beta1 = 0.78 beta2 = 0.836 @@ -246,14 +260,14 @@ def setup(self, scope, place): "Moment2": np.full((height, row_numel), 5.0).astype("float32"), 'Beta1Pow': np.array([beta1]).astype("float32"), 'Beta2Pow': np.array([beta2]).astype("float32"), - "LearningRate": np.full((1), 2.0).astype("float32") + "LearningRate": np.full((1), 2.0).astype("float32"), } self.init_output = np.full((height, row_numel), 0.0).astype("float32") self.attrs = { 'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2, - 'weight_decay': 0.05 + 'weight_decay': 0.05, } grad_selected_rows = scope.var('Grad').get_selected_rows() @@ -269,13 +283,14 @@ def setup(self, scope, place): self.sparse_inputs = ["Grad"] param_out, mom1, mom2, beta1_pow_out, beta2_pow_out = lamb_step_sparse( - self.dense_inputs, self.attrs, height, rows, row_numel, np_array) + self.dense_inputs, self.attrs, height, rows, row_numel, np_array + ) self.outputs = { "ParamOut": param_out, "Moment1Out": mom1, "Moment2Out": mom2, 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out + 'Beta2PowOut': beta2_pow_out, } def check_with_place(self, place): diff --git a/python/paddle/fluid/tests/unittests/test_lambv2_op.py b/python/paddle/fluid/tests/unittests/test_lambv2_op.py index 1e01f3a4ce394..475866fb903e8 100644 --- a/python/paddle/fluid/tests/unittests/test_lambv2_op.py +++ b/python/paddle/fluid/tests/unittests/test_lambv2_op.py @@ -22,39 +22,40 @@ class LAMBOptimizer(paddle.optimizer.Lamb): - def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, fluid.framework.Block) block.program._use_lamb = True - m = moment1 = self._get_accumulator(self._moment1_acc_str, - param_and_grad[0]) + m = moment1 = self._get_accumulator( + self._moment1_acc_str, param_and_grad[0] + ) v = self._get_accumulator(self._moment2_acc_str, param_and_grad[0]) - beta_1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) - beta_2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str, - param_and_grad[0]) - - beta_1 = layers.fill_constant(dtype='float32', - shape=[1], - value=self._beta1, - name='lamb_beta_1') - beta_2 = layers.fill_constant(dtype='float32', - shape=[1], - value=self._beta2, - name='lamb_beta_2') - epsilon = layers.fill_constant(dtype='float32', - shape=[1], - value=self._epsilon, - name='epsilon') + beta_1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) + beta_2_pow_acc = self._get_accumulator( + self._beta2_pow_acc_str, param_and_grad[0] + ) + + beta_1 = layers.fill_constant( + dtype='float32', shape=[1], value=self._beta1, name='lamb_beta_1' + ) + beta_2 = layers.fill_constant( + dtype='float32', shape=[1], value=self._beta2, name='lamb_beta_2' + ) + epsilon = layers.fill_constant( + dtype='float32', shape=[1], value=self._epsilon, name='epsilon' + ) one = paddle.ones(shape=[1]).astype('float32') zero = paddle.zeros(shape=[1]).astype('float32') next_m = paddle.multiply(m, beta_1) + paddle.multiply( - param_and_grad[1], one - beta_1) + param_and_grad[1], one - beta_1 + ) next_v = paddle.multiply(v, beta_2) + paddle.multiply( - paddle.pow(param_and_grad[1], 2), one - beta_2) + paddle.pow(param_and_grad[1], 2), one - beta_2 + ) beta1_correction = one - beta_1_pow_acc beta2_correction = one - beta_2_pow_acc @@ -64,8 +65,10 @@ def _append_optimize_op(self, block, param_and_grad): update = next_m_unbiased / (paddle.sqrt(next_v_unbiased) + epsilon) - if self._exclude_from_weight_decay_fn is not None and self._exclude_from_weight_decay_fn( - param_and_grad[0]): + if ( + self._exclude_from_weight_decay_fn is not None + and self._exclude_from_weight_decay_fn(param_and_grad[0]) + ): self._lamb_weight_decay = 0.0 update += self._lamb_weight_decay * param_and_grad[0] @@ -76,8 +79,11 @@ def _append_optimize_op(self, block, param_and_grad): ratio = paddle.where( paddle.greater_than(w_norm, zero), - paddle.where(paddle.greater_than(g_norm, zero), (w_norm / g_norm), - one), one) + paddle.where( + paddle.greater_than(g_norm, zero), (w_norm / g_norm), one + ), + one, + ) update_with_lr = ratio * learning_rate * update next_param = param_and_grad[0] - update_with_lr @@ -92,16 +98,15 @@ def _append_optimize_op(self, block, param_and_grad): class TestLambOpV2(unittest.TestCase): - def test_lamb_op(self): shape = [2, 4, 8, 8] data = paddle.to_tensor(np.random.random(size=shape).astype("float32")) conv = paddle.nn.Conv2D(4, 6, (3, 3)) data = conv(data) loss = paddle.mean(data) - opt = paddle.optimizer.Lamb(learning_rate=1e-5, - epsilon=1e-8, - parameters=conv.parameters()) + opt = paddle.optimizer.Lamb( + learning_rate=1e-5, epsilon=1e-8, parameters=conv.parameters() + ) loss.backward() opt.minimize(loss) @@ -109,7 +114,6 @@ def test_lamb_op(self): class TestLambOpWithCombinedOp(unittest.TestCase): - def test_lamb_op_with_multi_steps(self): paddle.enable_static() @@ -140,12 +144,11 @@ def _build_static_model(main, startup, seed=100): executor = fluid.Executor(place) executor.run(startup_program) - output = executor.run(program=main_program, - feed={ - 'X': feed_x, - 'Y': feed_y - }, - fetch_list=[avg_loss.name]) + output = executor.run( + program=main_program, + feed={'X': feed_x, 'Y': feed_y}, + fetch_list=[avg_loss.name], + ) main = fluid.Program() startup = fluid.Program() @@ -156,18 +159,16 @@ def _build_static_model(main, startup, seed=100): exe = fluid.Executor(place) exe.run(startup) - out = exe.run(program=main, - feed={ - 'X': feed_x, - 'Y': feed_y - }, - fetch_list=[loss.name]) + out = exe.run( + program=main, + feed={'X': feed_x, 'Y': feed_y}, + fetch_list=[loss.name], + ) np.testing.assert_allclose(out, output, rtol=1e-05) class TestLambOpV2Group(TestLambOpV2): - def test_lamb_op(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") @@ -175,16 +176,19 @@ def test_lamb_op(self): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Lamb(learning_rate=0.01, - parameters=[{ - 'params': linear_1.parameters() - }, { - 'params': linear_2.parameters(), - 'lamb_weight_decay': 0.001, - 'beta1': 0.9, - 'beta2': 0.99 - }], - lamb_weight_decay=0.01) + adam = paddle.optimizer.Lamb( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'lamb_weight_decay': 0.001, + 'beta1': 0.9, + 'beta2': 0.99, + }, + ], + lamb_weight_decay=0.01, + ) out = linear_1(a) out = linear_2(out) out.backward() @@ -193,16 +197,15 @@ def test_lamb_op(self): class TestLambOpMultiPrecision(unittest.TestCase): - def check_main(self, x_np, place, multi_precision=False, seed=10, n=10): main_prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, startup_prog): paddle.seed(seed) with paddle.static.amp.fp16_guard(): - x = paddle.static.data(name='x', - shape=[None, 10], - dtype='float32') + x = paddle.static.data( + name='x', shape=[None, 10], dtype='float32' + ) linear = paddle.nn.Linear(10, 2) hidden = linear(x) loss = paddle.mean(hidden) @@ -210,9 +213,9 @@ def check_main(self, x_np, place, multi_precision=False, seed=10, n=10): original_optimizer = paddle.optimizer.Lamb(learning_rate=1e-3) original_optimizer._multi_precision = multi_precision if multi_precision: - optimizer = paddle.static.amp.decorate(original_optimizer, - use_pure_fp16=True, - use_fp16_guard=True) + optimizer = paddle.static.amp.decorate( + original_optimizer, use_pure_fp16=True, use_fp16_guard=True + ) else: optimizer = original_optimizer optimizer.minimize(loss) @@ -233,8 +236,9 @@ def get_parameter(var): if multi_precision: params[0] = np.array(params[0]) params[1] = np.array(params[1]) - np.testing.assert_array_equal(params[0], - params[1].astype(np.float16)) + np.testing.assert_array_equal( + params[0], params[1].astype(np.float16) + ) return params[0].astype(np.float32) else: self.assertTrue(params[0] is not None) @@ -250,9 +254,9 @@ def get_parameter(var): weight_np, bias_np = None, None for i in range(n): feed_dict = {x.name: x_np} - weight_np, bias_np = exe.run(main_prog, - feed=feed_dict, - fetch_list=[weight, bias]) + weight_np, bias_np = exe.run( + main_prog, feed=feed_dict, fetch_list=[weight, bias] + ) weight_np = weight_np.astype('float32') bias_np = bias_np.astype('float32') np.testing.assert_array_equal(weight_np, get_parameter(weight)) diff --git a/python/paddle/fluid/tests/unittests/test_launch_coverage.py b/python/paddle/fluid/tests/unittests/test_launch_coverage.py index 00db6590d6f3f..1f00ea9af5563 100644 --- a/python/paddle/fluid/tests/unittests/test_launch_coverage.py +++ b/python/paddle/fluid/tests/unittests/test_launch_coverage.py @@ -15,7 +15,11 @@ import unittest from argparse import ArgumentParser, REMAINDER -from paddle.distributed.utils.launch_utils import _print_arguments, get_gpus, get_cluster_from_args +from paddle.distributed.utils.launch_utils import ( + _print_arguments, + get_gpus, + get_cluster_from_args, +) from paddle.distributed.fleet.launch_utils import find_free_ports @@ -32,72 +36,75 @@ def _parse_args(): PADDLE_TRAINERS_NUM PADDLE_TRAINER_ENDPOINTS POD_IP (current node ip address, not needed for local training) -''') +''' + ) - #Optional arguments for the launch helper + # Optional arguments for the launch helper parser.add_argument( "--cluster_node_ips", type=str, default="127.0.0.1", - help="Paddle cluster nodes ips, such as 192.168.0.16,192.168.0.17..") - parser.add_argument("--node_ip", - type=str, - default="127.0.0.1", - help="The current node ip. ") + help="Paddle cluster nodes ips, such as 192.168.0.16,192.168.0.17..", + ) + parser.add_argument( + "--node_ip", type=str, default="127.0.0.1", help="The current node ip. " + ) parser.add_argument( "--use_paddlecloud", action='store_true', - help= - "wheter to use paddlecloud platform to run your multi-process job. If false, no need to set this argument." + help="wheter to use paddlecloud platform to run your multi-process job. If false, no need to set this argument.", + ) + parser.add_argument( + "--started_port", + type=int, + default=None, + help="The trainer's started port on a single node", ) - parser.add_argument("--started_port", - type=int, - default=None, - help="The trainer's started port on a single node") - parser.add_argument("--print_config", - type=bool, - default=True, - help="Print the config or not") + parser.add_argument( + "--print_config", + type=bool, + default=True, + help="Print the config or not", + ) parser.add_argument( "--selected_gpus", type=str, default=None, - help= - "It's for gpu training and the training process will run on the selected_gpus," - "each process is bound to a single GPU. And if it's not set, this module will use all the gpu cards for training." + help="It's for gpu training and the training process will run on the selected_gpus," + "each process is bound to a single GPU. And if it's not set, this module will use all the gpu cards for training.", ) parser.add_argument( "--log_level", type=int, - default= - 20, # logging.INFO, details are here:https://docs.python.org/3/library/logging.html#levels - help="Logging level, default is logging.INFO") + default=20, # logging.INFO, details are here:https://docs.python.org/3/library/logging.html#levels + help="Logging level, default is logging.INFO", + ) parser.add_argument( "--log_dir", type=str, - help= - "The path for each process's log.If it's not set, the log will printed to default pipe." + help="The path for each process's log.If it's not set, the log will printed to default pipe.", ) - #positional - parser.add_argument("training_script", - type=str, - help="The full path to the single GPU training " - "program/script to be launched in parallel, " - "followed by all the arguments for the " - "training script") + # positional + parser.add_argument( + "training_script", + type=str, + help="The full path to the single GPU training " + "program/script to be launched in parallel, " + "followed by all the arguments for the " + "training script", + ) - #rest from the training program + # rest from the training program parser.add_argument('training_script_args', nargs=REMAINDER) return parser.parse_args() class TestCoverage(unittest.TestCase): - def test_gpus(self): args = _parse_args() diff --git a/python/paddle/fluid/tests/unittests/test_layer_norm_op.py b/python/paddle/fluid/tests/unittests/test_layer_norm_op.py index e49cf0b24e775..17d2730b3e8f7 100644 --- a/python/paddle/fluid/tests/unittests/test_layer_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_layer_norm_op.py @@ -23,7 +23,9 @@ from functools import reduce from op_test import _set_use_system_allocator from paddle.fluid import Program, program_guard -from paddle.fluid.contrib.mixed_precision.fp16_utils import _keep_layer_norm_scale_bias_to_fp32 +from paddle.fluid.contrib.mixed_precision.fp16_utils import ( + _keep_layer_norm_scale_bias_to_fp32, +) paddle.enable_static() @@ -35,13 +37,14 @@ def _reference_layer_norm_naive(x, scale, beta, epsilon, begin_norm_axis=1): x_shape = x.shape N = reduce(mul, x_shape[0:begin_norm_axis], 1) - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) x.shape = [N, D] mean = np.mean(x, axis=1) var = np.var(x, axis=1) + epsilon - output = np.divide((x - mean.reshape([N, 1])), - (np.sqrt(var)).reshape([N, 1])) + output = np.divide( + (x - mean.reshape([N, 1])), (np.sqrt(var)).reshape([N, 1]) + ) if scale is not None: output = scale.reshape([1, D]) * output if beta is not None: @@ -51,16 +54,12 @@ def _reference_layer_norm_naive(x, scale, beta, epsilon, begin_norm_axis=1): return output, mean, var -def _reference_layer_norm_grad(x, - grad_y, - scale, - bias, - mean, - var, - begin_norm_axis=1): +def _reference_layer_norm_grad( + x, grad_y, scale, bias, mean, var, begin_norm_axis=1 +): x_shape = x.shape N = reduce(mul, x_shape[0:begin_norm_axis], 1) - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) if scale is not None: scale_shape = scale.shape @@ -75,38 +74,39 @@ def _reference_layer_norm_grad(x, d_bias = None # d_scale if scale is not None: - d_scale = np.sum(((x - mean) * np.sqrt(1 / var)) * grad_y, - axis=0).reshape([1, D]) + d_scale = np.sum( + ((x - mean) * np.sqrt(1 / var)) * grad_y, axis=0 + ).reshape([1, D]) else: d_scale = None # dx if scale is not None: dx_end = scale * np.sqrt(1.0 / var) * grad_y - d_mean_0 = np.sum(-np.sqrt(1.0 / var) * grad_y * scale, - axis=1).reshape([N, 1 - ]) # the second part equals to zero. + d_mean_0 = np.sum(-np.sqrt(1.0 / var) * grad_y * scale, axis=1).reshape( + [N, 1] + ) # the second part equals to zero. d_mean = 1.0 / D * d_mean_0 - d_std = np.sum(-(1.0 / var) * - (x - mean) * grad_y * scale, axis=1).reshape([N, 1]) * ( - 1.0 / D * np.sqrt(1.0 / var).reshape([N, 1]) * - (x - mean)) + d_std = np.sum( + -(1.0 / var) * (x - mean) * grad_y * scale, axis=1 + ).reshape([N, 1]) * ( + 1.0 / D * np.sqrt(1.0 / var).reshape([N, 1]) * (x - mean) + ) else: dx_end = 1.0 * np.sqrt(1.0 / var) * grad_y - d_mean_0 = np.sum(-np.sqrt(1.0 / var) * grad_y * 1.0, - axis=1).reshape([N, 1 - ]) # the second part equals to zero. + d_mean_0 = np.sum(-np.sqrt(1.0 / var) * grad_y * 1.0, axis=1).reshape( + [N, 1] + ) # the second part equals to zero. d_mean = 1.0 / D * d_mean_0 - d_std = np.sum(-(1.0 / var) * - (x - mean) * grad_y * 1.0, axis=1).reshape([N, 1]) * ( - 1.0 / D * np.sqrt(1.0 / var).reshape([N, 1]) * - (x - mean)) + d_std = np.sum( + -(1.0 / var) * (x - mean) * grad_y * 1.0, axis=1 + ).reshape([N, 1]) * ( + 1.0 / D * np.sqrt(1.0 / var).reshape([N, 1]) * (x - mean) + ) grad_x = dx_end + d_mean + d_std grad_x.shape, x.shape, grad_y.shape = x_shape, x_shape, x_shape - var.shape, mean.shape = [ - N, - ], [ + var.shape, mean.shape = [N,], [ N, ] @@ -116,45 +116,53 @@ def _reference_layer_norm_grad(x, class TestLayerNormOp(unittest.TestCase): - def setUp(self): self.use_cudnn = True def __assert_close(self, tensor, np_array, msg, atol=1e-4): self.assertTrue(np.allclose(np.array(tensor), np_array, atol=atol), msg) - def check_forward_backward(self, - shape, - begin_norm_axis, - has_scale=True, - has_bias=True, - y_grad_scale=1.0, - use_mkldnn=False): - - def test_with_place(place, - shape, - begin_norm_axis, - use_mkldnn=use_mkldnn): + def check_forward_backward( + self, + shape, + begin_norm_axis, + has_scale=True, + has_bias=True, + y_grad_scale=1.0, + use_mkldnn=False, + ): + def test_with_place( + place, shape, begin_norm_axis, use_mkldnn=use_mkldnn + ): # attr epsilon = 0.00001 x_shape = shape - D = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + D = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) scale_shape = [D] np.random.seed(123) x = np.random.random_sample(x_shape).astype(np.float32) - scale = np.random.random_sample(scale_shape).astype( - np.float32) if has_scale else None - bias = np.random.random_sample(scale_shape).astype( - np.float32) if has_bias else None + scale = ( + np.random.random_sample(scale_shape).astype(np.float32) + if has_scale + else None + ) + bias = ( + np.random.random_sample(scale_shape).astype(np.float32) + if has_bias + else None + ) y_grad = (np.random.random_sample(x_shape) * y_grad_scale).astype( - np.float32) + np.float32 + ) # reference forward & backward y, mean, variance = _reference_layer_norm_naive( - x, scale, bias, epsilon, begin_norm_axis) + x, scale, bias, epsilon, begin_norm_axis + ) x_grad, scale_grad, bias_grad = _reference_layer_norm_grad( - x, y_grad, scale, bias, mean, variance, begin_norm_axis) + x, y_grad, scale, bias, mean, variance, begin_norm_axis + ) var_dict = locals() var_dict['y@GRAD'] = y_grad @@ -169,9 +177,11 @@ def test_with_place(place, with fluid.program_guard(program): block = program.global_block() for name in ground_truth: - block.create_var(name=name, - dtype='float32', - shape=ground_truth[name].shape) + block.create_var( + name=name, + dtype='float32', + shape=ground_truth[name].shape, + ) inputs = {"X": block.var('x')} fetch_list = [ 'y', @@ -191,17 +201,20 @@ def test_with_place(place, outputs={ "Y": block.var('y'), "Mean": block.var('mean'), # share the same memory - "Variance": - block.var('variance'), # share the same memory + "Variance": block.var( + 'variance' + ), # share the same memory }, attrs={ "epsilon": epsilon, "begin_norm_axis": begin_norm_axis, - "use_mkldnn": use_mkldnn - }) + "use_mkldnn": use_mkldnn, + }, + ) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - layer_norm_op.desc, set(), []) + layer_norm_op.desc, set(), [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -215,12 +228,14 @@ def test_with_place(place, program._sync_with_cpp() exe = fluid.Executor(place) - out = exe.run(program, - feed={ - name: var_dict[name] - for name in ['x', 'scale', 'bias', 'y@GRAD'] - }, - fetch_list=fetch_list) + out = exe.run( + program, + feed={ + name: var_dict[name] + for name in ['x', 'scale', 'bias', 'y@GRAD'] + }, + fetch_list=fetch_list, + ) # print(y) # print(out[0]) self.__assert_close(y, out[0], "y") @@ -228,17 +243,25 @@ def test_with_place(place, self.__assert_close(variance, out[2], "variance", 1e-3) self.__assert_close(x_grad, out[3], "x_grad") if has_scale: - self.__assert_close(scale_grad, - out[fetch_list.index('scale@GRAD')], - "scale_grad", 1e-3) + self.__assert_close( + scale_grad, + out[fetch_list.index('scale@GRAD')], + "scale_grad", + 1e-3, + ) if has_bias: - self.__assert_close(bias_grad, - out[fetch_list.index('bias@GRAD')], - "bias_grad") + self.__assert_close( + bias_grad, + out[fetch_list.index('bias@GRAD')], + "bias_grad", + ) places = [core.CPUPlace()] - if core.is_compiled_with_cuda() and core.op_support_gpu( - "layer_norm") and self.use_cudnn: + if ( + core.is_compiled_with_cuda() + and core.op_support_gpu("layer_norm") + and self.use_cudnn + ): places.append(core.CUDAPlace(0)) for place in places: @@ -247,87 +270,108 @@ def test_with_place(place, def test_check_forward_backward_with_scale_and_bias(self): self.check_forward_backward(shape=[1, 3, 4, 5], begin_norm_axis=1) self.check_forward_backward(shape=[2, 3, 4, 5], begin_norm_axis=1) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=False, - has_bias=True) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=True, - has_bias=False) - self.check_forward_backward(shape=[2, 3, 4, 5], - begin_norm_axis=1, - has_scale=False, - has_bias=False) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=False, + has_bias=True, + ) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=True, + has_bias=False, + ) + self.check_forward_backward( + shape=[2, 3, 4, 5], + begin_norm_axis=1, + has_scale=False, + has_bias=False, + ) self.check_forward_backward(shape=[2, 3, 4, 5], begin_norm_axis=3) - self.check_forward_backward(shape=[92, 513, 129], - begin_norm_axis=2, - y_grad_scale=0.1) + self.check_forward_backward( + shape=[92, 513, 129], begin_norm_axis=2, y_grad_scale=0.1 + ) self.check_forward_backward(shape=[3, 34, 1134], begin_norm_axis=2) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - y_grad_scale=0.1) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - has_scale=False, - has_bias=True, - y_grad_scale=0.1) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - has_scale=True, - has_bias=False, - y_grad_scale=0.1) - self.check_forward_backward(shape=[92, 513, 1134], - begin_norm_axis=2, - has_scale=False, - has_bias=False, - y_grad_scale=0.1) - self.check_forward_backward(shape=[512, 1024], - begin_norm_axis=1, - has_scale=True, - has_bias=True) - self.check_forward_backward(shape=[1, 128, 256, 256], - begin_norm_axis=3, - has_scale=True, - has_bias=True) - self.check_forward_backward(shape=[1, 256, 384], - begin_norm_axis=2, - has_scale=True, - has_bias=True) + self.check_forward_backward( + shape=[92, 513, 1134], begin_norm_axis=2, y_grad_scale=0.1 + ) + self.check_forward_backward( + shape=[92, 513, 1134], + begin_norm_axis=2, + has_scale=False, + has_bias=True, + y_grad_scale=0.1, + ) + self.check_forward_backward( + shape=[92, 513, 1134], + begin_norm_axis=2, + has_scale=True, + has_bias=False, + y_grad_scale=0.1, + ) + self.check_forward_backward( + shape=[92, 513, 1134], + begin_norm_axis=2, + has_scale=False, + has_bias=False, + y_grad_scale=0.1, + ) + self.check_forward_backward( + shape=[512, 1024], begin_norm_axis=1, has_scale=True, has_bias=True + ) + self.check_forward_backward( + shape=[1, 128, 256, 256], + begin_norm_axis=3, + has_scale=True, + has_bias=True, + ) + self.check_forward_backward( + shape=[1, 256, 384], + begin_norm_axis=2, + has_scale=True, + has_bias=True, + ) class TestLayerNormAPI(unittest.TestCase): - def test_case(self): - x = fluid.layers.data(name='x', - shape=[64, 32, 256], - dtype='float32', - append_batch_size=False) - x = fluid.layers.layer_norm(x, - scale=True, - shift=True, - begin_norm_axis=1, - epsilon=1e-05, - param_attr=None, - bias_attr=None) - x = fluid.layers.layer_norm(x, - scale=False, - shift=False, - begin_norm_axis=1, - epsilon=1e-05, - param_attr=None, - bias_attr=None) - x = fluid.layers.layer_norm(x, - scale=False, - shift=False, - begin_norm_axis=1, - epsilon=1e-05, - param_attr="scale", - bias_attr="shift") + x = fluid.layers.data( + name='x', + shape=[64, 32, 256], + dtype='float32', + append_batch_size=False, + ) + x = fluid.layers.layer_norm( + x, + scale=True, + shift=True, + begin_norm_axis=1, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + ) + x = fluid.layers.layer_norm( + x, + scale=False, + shift=False, + begin_norm_axis=1, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + ) + x = fluid.layers.layer_norm( + x, + scale=False, + shift=False, + begin_norm_axis=1, + epsilon=1e-05, + param_attr="scale", + bias_attr="shift", + ) class TestDygraphLayerNormAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): paddle.enable_static() @@ -344,7 +388,6 @@ def test_errors(self): class TestFP16ScaleBiasLayerNorm(unittest.TestCase): - def check_main(self, x_np, weight_np, bias_np, dtype): paddle.disable_static() @@ -375,9 +418,11 @@ def test_main(self): bias_np = np.random.random([20]).astype('float16') y_np_1, x_g_np_1, w_g_np_1, b_g_np_1 = self.check_main( - x_np, weight_np, bias_np, 'float16') + x_np, weight_np, bias_np, 'float16' + ) y_np_2, x_g_np_2, w_g_np_2, b_g_np_2 = self.check_main( - x_np, weight_np, bias_np, 'float32') + x_np, weight_np, bias_np, 'float32' + ) def assert_equal(x, y): np.testing.assert_array_equal(x, y) @@ -389,7 +434,6 @@ def assert_equal(x, y): class TestBF16ScaleBiasLayerNorm(unittest.TestCase): - def check_main(self, x_np, weight_np, bias_np, dtype): paddle.disable_static() @@ -416,18 +460,22 @@ def check_main(self, x_np, weight_np, bias_np, dtype): return y_np, x_g_np, w_g_np, b_g_np def test_main(self): - if (not core.is_compiled_with_cuda()) or ( - core.cudnn_version() < - 8100) or (paddle.device.cuda.get_device_capability()[0] < 8): + if ( + (not core.is_compiled_with_cuda()) + or (core.cudnn_version() < 8100) + or (paddle.device.cuda.get_device_capability()[0] < 8) + ): return x_np = np.random.random([10, 20]).astype('float32') weight_np = np.random.random([20]).astype('float32') bias_np = np.random.random([20]).astype('float32') y_np_1, x_g_np_1, w_g_np_1, b_g_np_1 = self.check_main( - x_np, weight_np, bias_np, 'float32') + x_np, weight_np, bias_np, 'float32' + ) y_np_2, x_g_np_2, w_g_np_2, b_g_np_2 = self.check_main( - x_np, weight_np, bias_np, 'bfloat16') + x_np, weight_np, bias_np, 'bfloat16' + ) def assert_equal(x, y): np.testing.assert_allclose(x, y, rtol=1e-05, atol=0.1) @@ -439,7 +487,6 @@ def assert_equal(x, y): class TestGetSetKeepLayerNormScaleBiasFP32Flag(unittest.TestCase): - def test_main(self): self.assertTrue(_keep_layer_norm_scale_bias_to_fp32()) _keep_layer_norm_scale_bias_to_fp32(False) diff --git a/python/paddle/fluid/tests/unittests/test_layer_norm_op_v2.py b/python/paddle/fluid/tests/unittests/test_layer_norm_op_v2.py index 8d6dfab71990b..6e9e493146543 100644 --- a/python/paddle/fluid/tests/unittests/test_layer_norm_op_v2.py +++ b/python/paddle/fluid/tests/unittests/test_layer_norm_op_v2.py @@ -23,7 +23,6 @@ class TestDygraphLayerNormv2(unittest.TestCase): - def test_dygraph(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu("layer_norm"): @@ -114,7 +113,6 @@ def compute_v2(x_np): class TestLayerNormFunction(unittest.TestCase): - def test_dygraph(self): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu("layer_norm"): @@ -162,10 +160,12 @@ def compute_v4(x): y4 = compute_v4(x) np.testing.assert_allclose(y3, y4, rtol=1e-05) - self.assertRaises(ValueError, - paddle.nn.functional.layer_norm, - x=x, - normalized_shape=1.0) + self.assertRaises( + ValueError, + paddle.nn.functional.layer_norm, + x=x, + normalized_shape=1.0, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_layers.py b/python/paddle/fluid/tests/unittests/test_layers.py index 176a8385c3c97..14754e347f9b9 100644 --- a/python/paddle/fluid/tests/unittests/test_layers.py +++ b/python/paddle/fluid/tests/unittests/test_layers.py @@ -36,7 +36,6 @@ class LayerTest(unittest.TestCase): - @classmethod def setUpClass(cls): cls.seed = 111 @@ -61,38 +60,36 @@ def static_graph(self): paddle.framework.random._manual_program_seed(self.seed) yield - def get_static_graph_result(self, - feed, - fetch_list, - with_lod=False, - force_to_use_cpu=False): + def get_static_graph_result( + self, feed, fetch_list, with_lod=False, force_to_use_cpu=False + ): exe = fluid.Executor(self._get_place(force_to_use_cpu)) exe.run(fluid.default_startup_program()) - return exe.run(fluid.default_main_program(), - feed=feed, - fetch_list=fetch_list, - return_numpy=(not with_lod)) + return exe.run( + fluid.default_main_program(), + feed=feed, + fetch_list=fetch_list, + return_numpy=(not with_lod), + ) @contextlib.contextmanager def dynamic_graph(self, force_to_use_cpu=False): with fluid.dygraph.guard( - self._get_place(force_to_use_cpu=force_to_use_cpu)): + self._get_place(force_to_use_cpu=force_to_use_cpu) + ): paddle.seed(self.seed) paddle.framework.random._manual_program_seed(self.seed) yield class TestLayer(LayerTest): - def test_custom_layer_with_kwargs(self): - class CustomLayer(fluid.Layer): - def __init__(self, input_size, linear1_size=4): super(CustomLayer, self).__init__() - self.linear1 = nn.Linear(input_size, - linear1_size, - bias_attr=False) + self.linear1 = nn.Linear( + input_size, linear1_size, bias_attr=False + ) self.linear2 = nn.Linear(linear1_size, 1, bias_attr=False) def forward(self, x, do_linear2=False): @@ -121,37 +118,37 @@ def forward(self, x, do_linear2=False): def test_dropout(self): inp = np.ones([3, 32, 32], dtype='float32') with self.static_graph(): - t = layers.data(name='data', - shape=[3, 32, 32], - dtype='float32', - append_batch_size=False) + t = layers.data( + name='data', + shape=[3, 32, 32], + dtype='float32', + append_batch_size=False, + ) dropout = nn.Dropout(p=0.35, seed=1, is_test=False) ret = dropout(t) - ret2 = fluid.layers.dropout(t, - dropout_prob=0.35, - seed=1, - is_test=False) + ret2 = fluid.layers.dropout( + t, dropout_prob=0.35, seed=1, is_test=False + ) static_ret, static_ret2 = self.get_static_graph_result( - feed={'data': inp}, fetch_list=[ret, ret2]) + feed={'data': inp}, fetch_list=[ret, ret2] + ) with self.dynamic_graph(): with _test_eager_guard(): t = base.to_variable(inp) dropout = nn.Dropout(p=0.35, seed=1, is_test=False) dy_eager_ret = dropout(t) - dy_eager_ret2 = fluid.layers.dropout(t, - dropout_prob=0.35, - seed=1, - is_test=False) + dy_eager_ret2 = fluid.layers.dropout( + t, dropout_prob=0.35, seed=1, is_test=False + ) dy_eager_ret_value = dy_eager_ret.numpy() dy_eager_ret2_value = dy_eager_ret2.numpy() t = base.to_variable(inp) dropout = nn.Dropout(p=0.35, seed=1, is_test=False) dy_ret = dropout(t) - dy_ret2 = fluid.layers.dropout(t, - dropout_prob=0.35, - seed=1, - is_test=False) + dy_ret2 = fluid.layers.dropout( + t, dropout_prob=0.35, seed=1, is_test=False + ) dy_ret_value = dy_ret.numpy() dy_ret2_value = dy_ret2.numpy() @@ -165,28 +162,34 @@ def test_dropout(self): def test_linear(self): inp = np.ones([3, 32, 32], dtype='float32') with self.static_graph(): - t = layers.data(name='data', - shape=[3, 32, 32], - dtype='float32', - append_batch_size=False) + t = layers.data( + name='data', + shape=[3, 32, 32], + dtype='float32', + append_batch_size=False, + ) linear = nn.Linear( - 32, 4, bias_attr=fluid.initializer.ConstantInitializer(value=1)) + 32, 4, bias_attr=fluid.initializer.ConstantInitializer(value=1) + ) ret = linear(t) - static_ret = self.get_static_graph_result(feed={'data': inp}, - fetch_list=[ret])[0] + static_ret = self.get_static_graph_result( + feed={'data': inp}, fetch_list=[ret] + )[0] with self.dynamic_graph(): with _test_eager_guard(): t = base.to_variable(inp) linear = nn.Linear( 32, 4, - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) dy_eager_ret = linear(t) dy_eager_ret_value = dy_eager_ret.numpy() t = base.to_variable(inp) linear = nn.Linear( - 32, 4, bias_attr=fluid.initializer.ConstantInitializer(value=1)) + 32, 4, bias_attr=fluid.initializer.ConstantInitializer(value=1) + ) dy_ret = linear(t) dy_ret_value = dy_ret.numpy() @@ -201,7 +204,8 @@ def test_Variable(): linear = nn.Linear( 32, 4, - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) linear_ret1 = linear(inp) self.assertRaises(TypeError, test_Variable) @@ -213,7 +217,8 @@ def test_type(): linear = nn.Linear( 32, 4, - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) linear_ret2 = linear(inp) self.assertRaises(TypeError, test_type) @@ -221,14 +226,17 @@ def test_type(): def test_Flatten(self): inp = np.ones([3, 4, 4, 5], dtype='float32') with self.static_graph(): - t = layers.data(name='data', - shape=[3, 4, 4, 5], - dtype='float32', - append_batch_size=False) + t = layers.data( + name='data', + shape=[3, 4, 4, 5], + dtype='float32', + append_batch_size=False, + ) flatten = nn.Flatten() ret = flatten(t) - static_ret = self.get_static_graph_result(feed={'data': inp}, - fetch_list=[ret])[0] + static_ret = self.get_static_graph_result( + feed={'data': inp}, fetch_list=[ret] + )[0] with self.dynamic_graph(): with _test_eager_guard(): t = base.to_variable(inp) @@ -252,7 +260,8 @@ def test_Variable(): linear = nn.Linear( 32, 4, - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) linear_ret1 = linear(inp) self.assertRaises(TypeError, test_Variable) @@ -264,7 +273,8 @@ def test_type(): linear = nn.Linear( 32, 4, - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) linear_ret2 = linear(inp) self.assertRaises(TypeError, test_type) @@ -272,41 +282,51 @@ def test_type(): def test_layer_norm(self): inp = np.ones([3, 32, 32], dtype='float32') with self.static_graph(): - t = layers.data(name='data', - shape=[3, 32, 32], - dtype='float32', - append_batch_size=False) + t = layers.data( + name='data', + shape=[3, 32, 32], + dtype='float32', + append_batch_size=False, + ) ret = layers.layer_norm( t, bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') - static_ret = self.get_static_graph_result(feed={'data': inp}, - fetch_list=[ret])[0] + act='sigmoid', + ) + static_ret = self.get_static_graph_result( + feed={'data': inp}, fetch_list=[ret] + )[0] with self.static_graph(): - t = layers.data(name='data', - shape=[3, 32, 32], - dtype='float32', - append_batch_size=False) + t = layers.data( + name='data', + shape=[3, 32, 32], + dtype='float32', + append_batch_size=False, + ) lm = nn.LayerNorm( normalized_shape=[32, 32], bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) ret = lm(t) - static_ret2 = self.get_static_graph_result(feed={'data': inp}, - fetch_list=[ret])[0] + static_ret2 = self.get_static_graph_result( + feed={'data': inp}, fetch_list=[ret] + )[0] with self.dynamic_graph(): with _test_eager_guard(): lm = nn.LayerNorm( normalized_shape=[32, 32], bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) dy_eager_ret = lm(base.to_variable(inp)) dy_eager_ret_value = dy_eager_ret.numpy() lm = nn.LayerNorm( normalized_shape=[32, 32], bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) dy_ret = lm(base.to_variable(inp)) dy_ret_value = dy_ret.numpy() @@ -318,7 +338,8 @@ def test_layer_norm(self): scale=False, param_attr=fluid.initializer.ConstantInitializer(value=1), bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) lm(base.to_variable(inp)) self.assertFalse(hasattr(lm, "_scale_w")) @@ -330,7 +351,8 @@ def test_layer_norm(self): scale=False, param_attr=fluid.initializer.ConstantInitializer(value=1), bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) lm(base.to_variable(inp)) self.assertFalse(hasattr(lm, "_scale_w")) @@ -345,14 +367,16 @@ def test_layer_norm(self): lm = nn.LayerNorm( normalized_shape=[16, 32], bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) with self.assertRaises(ValueError): lm(base.to_variable(inp)) lm = nn.LayerNorm( normalized_shape=[16, 32], bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) with self.assertRaises(ValueError): lm(base.to_variable(inp)) @@ -364,7 +388,8 @@ def test_SyncBatchNorm(self): ret = my_sync_bn(t) static_ret = self.get_static_graph_result( feed={'t': np.ones([3, 3, 5, 5], dtype='float32')}, - fetch_list=[ret])[0] + fetch_list=[ret], + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -385,8 +410,8 @@ def test_relu(self): t = layers.data(name='t', shape=[3, 3], dtype='float32') ret = layers.relu(t) static_ret = self.get_static_graph_result( - feed={'t': np.ones([3, 3], - dtype='float32')}, fetch_list=[ret])[0] + feed={'t': np.ones([3, 3], dtype='float32')}, fetch_list=[ret] + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -406,20 +431,21 @@ def test_matmul(self): t = layers.data(name='t', shape=[3, 3], dtype='float32') t2 = layers.data(name='t2', shape=[3, 3], dtype='float32') ret = layers.matmul(t, t2) - static_ret = self.get_static_graph_result(feed={ - 't': - np.ones([3, 3], dtype='float32'), - 't2': - np.ones([3, 3], dtype='float32') - }, - fetch_list=[ret])[0] + static_ret = self.get_static_graph_result( + feed={ + 't': np.ones([3, 3], dtype='float32'), + 't2': np.ones([3, 3], dtype='float32'), + }, + fetch_list=[ret], + )[0] with self.dynamic_graph(): with _test_eager_guard(): t = np.ones([3, 3], dtype='float32') t2 = np.ones([3, 3], dtype='float32') - dy_eager_ret = layers.matmul(base.to_variable(t), - base.to_variable(t2)) + dy_eager_ret = layers.matmul( + base.to_variable(t), base.to_variable(t2) + ) dy_eager_ret_value = dy_eager_ret.numpy() t = np.ones([3, 3], dtype='float32') @@ -436,49 +462,55 @@ def test_conv2d(self): ret = layers.conv2d(input=images, num_filters=3, filter_size=[2, 2]) static_ret = self.get_static_graph_result( feed={'pixel': np.ones([2, 3, 5, 5], dtype='float32')}, - fetch_list=[ret])[0] + fetch_list=[ret], + )[0] with self.static_graph(): images = layers.data(name='pixel', shape=[3, 5, 5], dtype='float32') - conv2d = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2]) + conv2d = nn.Conv2D( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) ret = conv2d(images) static_ret2 = self.get_static_graph_result( feed={'pixel': np.ones([2, 3, 5, 5], dtype='float32')}, - fetch_list=[ret])[0] + fetch_list=[ret], + )[0] with self.dynamic_graph(): with _test_eager_guard(): images = np.ones([2, 3, 5, 5], dtype='float32') - conv2d = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2]) + conv2d = nn.Conv2D( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) dy_eager_ret = conv2d(base.to_variable(images)) dy_eager_ret_value = dy_eager_ret.numpy() images = np.ones([2, 3, 5, 5], dtype='float32') - conv2d = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2]) + conv2d = nn.Conv2D( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) dy_ret = conv2d(base.to_variable(images)) dy_ret_value = dy_ret.numpy() with self.dynamic_graph(): with _test_eager_guard(): images = np.ones([2, 3, 5, 5], dtype='float32') - conv2d = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2], - bias_attr=False) + conv2d = nn.Conv2D( + num_channels=3, + num_filters=3, + filter_size=[2, 2], + bias_attr=False, + ) dy_ret = conv2d(base.to_variable(images)) self.assertTrue(conv2d.bias is None) images = np.ones([2, 3, 5, 5], dtype='float32') - conv2d = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2], - bias_attr=False) + conv2d = nn.Conv2D( + num_channels=3, + num_filters=3, + filter_size=[2, 2], + bias_attr=False, + ) dy_ret = conv2d(base.to_variable(images)) self.assertTrue(conv2d.bias is None) @@ -486,9 +518,9 @@ def test_conv2d(self): # the input of Conv2D must be Variable. def test_Variable(): images = np.ones([2, 3, 5, 5], dtype='float32') - conv2d = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2]) + conv2d = nn.Conv2D( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) conv2d_ret1 = conv2d(images) self.assertRaises(TypeError, test_Variable) @@ -496,12 +528,12 @@ def test_Variable(): # the input dtype of Conv2D must be float16 or float32 or float64 # float16 only can be set on GPU place def test_type(): - images = layers.data(name='pixel', - shape=[3, 5, 5], - dtype='int32') - conv2d = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2]) + images = layers.data( + name='pixel', shape=[3, 5, 5], dtype='int32' + ) + conv2d = nn.Conv2D( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) conv2d_ret2 = conv2d(images) self.assertRaises(TypeError, test_type) @@ -516,26 +548,33 @@ def test_type(): custom_weight = np.random.randn(3, 3, 2, 2).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) - conv2d1 = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2]) - conv2d2 = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2], - param_attr=weight_attr) + custom_weight + ) + ) + conv2d1 = nn.Conv2D( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) + conv2d2 = nn.Conv2D( + num_channels=3, + num_filters=3, + filter_size=[2, 2], + param_attr=weight_attr, + ) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) self.assertFalse( - np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) + np.array_equal(dy_ret1.numpy(), dy_ret2.numpy()) + ) conv2d1_weight_np = conv2d1.weight.numpy() conv2d1_bias = conv2d1.bias self.assertFalse( - np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy())) + np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy()) + ) conv2d2.weight.set_value(conv2d1_weight_np) - np.testing.assert_array_equal(conv2d1_weight_np, - conv2d2.weight.numpy()) + np.testing.assert_array_equal( + conv2d1_weight_np, conv2d2.weight.numpy() + ) conv2d2.bias.set_value(conv2d1_bias) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) @@ -543,22 +582,29 @@ def test_type(): conv2d2.weight = conv2d1.weight conv2d2.bias = conv2d1.bias - np.testing.assert_array_equal(conv2d1.weight.numpy(), - conv2d2.weight.numpy()) - np.testing.assert_array_equal(conv2d1.bias.numpy(), - conv2d2.bias.numpy()) + np.testing.assert_array_equal( + conv2d1.weight.numpy(), conv2d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv2d1.bias.numpy(), conv2d2.bias.numpy() + ) images = np.ones([2, 3, 5, 5], dtype='float32') custom_weight = np.random.randn(3, 3, 2, 2).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) - conv2d1 = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2]) - conv2d2 = nn.Conv2D(num_channels=3, - num_filters=3, - filter_size=[2, 2], - param_attr=weight_attr) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) + conv2d1 = nn.Conv2D( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) + conv2d2 = nn.Conv2D( + num_channels=3, + num_filters=3, + filter_size=[2, 2], + param_attr=weight_attr, + ) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) self.assertFalse(np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) @@ -566,10 +612,12 @@ def test_type(): conv2d1_weight_np = conv2d1.weight.numpy() conv2d1_bias = conv2d1.bias self.assertFalse( - np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy())) + np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy()) + ) conv2d2.weight.set_value(conv2d1_weight_np) - np.testing.assert_array_equal(conv2d1_weight_np, - conv2d2.weight.numpy()) + np.testing.assert_array_equal( + conv2d1_weight_np, conv2d2.weight.numpy() + ) conv2d2.bias.set_value(conv2d1_bias) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) @@ -577,10 +625,12 @@ def test_type(): conv2d2.weight = conv2d1.weight conv2d2.bias = conv2d1.bias - np.testing.assert_array_equal(conv2d1.weight.numpy(), - conv2d2.weight.numpy()) - np.testing.assert_array_equal(conv2d1.bias.numpy(), - conv2d2.bias.numpy()) + np.testing.assert_array_equal( + conv2d1.weight.numpy(), conv2d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv2d1.bias.numpy(), conv2d2.bias.numpy() + ) def test_gru_unit(self): lod = [[2, 4, 3]] @@ -595,114 +645,132 @@ def test_gru_unit(self): x = layers.data(name='x', shape=[-1, D * 3], dtype='float32') hidden = layers.data(name='hidden', shape=[-1, D], dtype='float32') updated_hidden, reset_hidden_pre, gate = layers.gru_unit( - input=x, hidden=hidden, size=D * 3) + input=x, hidden=hidden, size=D * 3 + ) static_ret = self.get_static_graph_result( - feed={ - 'x': input, - 'hidden': hidden_input - }, - fetch_list=[updated_hidden, reset_hidden_pre, gate]) + feed={'x': input, 'hidden': hidden_input}, + fetch_list=[updated_hidden, reset_hidden_pre, gate], + ) with self.static_graph(): x = layers.data(name='x', shape=[-1, D * 3], dtype='float32') hidden = layers.data(name='hidden', shape=[-1, D], dtype='float32') updated_hidden, reset_hidden_pre, gate = layers.gru_unit( - input=x, hidden=hidden, size=D * 3) + input=x, hidden=hidden, size=D * 3 + ) gru = nn.GRUUnit(size=D * 3) updated_hidden, reset_hidden_pre, gate = gru(x, hidden) static_ret2 = self.get_static_graph_result( - feed={ - 'x': input, - 'hidden': hidden_input - }, - fetch_list=[updated_hidden, reset_hidden_pre, gate]) + feed={'x': input, 'hidden': hidden_input}, + fetch_list=[updated_hidden, reset_hidden_pre, gate], + ) with self.dynamic_graph(): with _test_eager_guard(): gru = nn.GRUUnit(size=D * 3) - dy_eager_ret = gru(base.to_variable(input), - base.to_variable(hidden_input)) + dy_eager_ret = gru( + base.to_variable(input), base.to_variable(hidden_input) + ) dy_eager_ret_value = [] for i in range(len(static_ret)): dy_eager_ret_value.append(dy_eager_ret[i].numpy()) gru = nn.GRUUnit(size=D * 3) - dy_ret = gru(base.to_variable(input), - base.to_variable(hidden_input)) + dy_ret = gru( + base.to_variable(input), base.to_variable(hidden_input) + ) dy_ret_value = [] for i in range(len(static_ret)): dy_ret_value.append(dy_ret[i].numpy()) for i in range(len(static_ret)): - np.testing.assert_allclose(static_ret[i], - static_ret2[i], - rtol=1e-05) - np.testing.assert_allclose(static_ret[i], - dy_ret_value[i], - rtol=1e-05) - np.testing.assert_allclose(static_ret[i], - dy_eager_ret_value[i], - rtol=1e-05) + np.testing.assert_allclose( + static_ret[i], static_ret2[i], rtol=1e-05 + ) + np.testing.assert_allclose( + static_ret[i], dy_ret_value[i], rtol=1e-05 + ) + np.testing.assert_allclose( + static_ret[i], dy_eager_ret_value[i], rtol=1e-05 + ) with self.dynamic_graph(): with _test_eager_guard(): custom_weight = np.random.randn(D, D * 3).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) + custom_weight + ) + ) gru1 = nn.GRUUnit(size=D * 3) gru2 = nn.GRUUnit(size=D * 3, param_attr=weight_attr) - dy_ret1 = gru1(base.to_variable(input), - base.to_variable(hidden_input)) - dy_ret2 = gru2(base.to_variable(input), - base.to_variable(hidden_input)) + dy_ret1 = gru1( + base.to_variable(input), base.to_variable(hidden_input) + ) + dy_ret2 = gru2( + base.to_variable(input), base.to_variable(hidden_input) + ) self.assertFalse( - np.array_equal(gru1.weight.numpy(), gru2.weight.numpy())) + np.array_equal(gru1.weight.numpy(), gru2.weight.numpy()) + ) for o1, o2 in zip(dy_ret1, dy_ret2): self.assertFalse(np.array_equal(o1.numpy(), o2.numpy())) gru2.weight.set_value(gru1.weight.numpy()) gru2.bias.set_value(gru1.bias) - dy_ret1 = gru1(base.to_variable(input), - base.to_variable(hidden_input)) - dy_ret2 = gru2(base.to_variable(input), - base.to_variable(hidden_input)) + dy_ret1 = gru1( + base.to_variable(input), base.to_variable(hidden_input) + ) + dy_ret2 = gru2( + base.to_variable(input), base.to_variable(hidden_input) + ) for o1, o2 in zip(dy_ret1, dy_ret2): np.testing.assert_array_equal(o1.numpy(), o2.numpy()) gru2.weight = gru1.weight gru2.bias = gru1.bias - np.testing.assert_array_equal(gru1.weight.numpy(), - gru2.weight.numpy()) - np.testing.assert_array_equal(gru1.bias.numpy(), - gru2.bias.numpy()) + np.testing.assert_array_equal( + gru1.weight.numpy(), gru2.weight.numpy() + ) + np.testing.assert_array_equal( + gru1.bias.numpy(), gru2.bias.numpy() + ) custom_weight = np.random.randn(D, D * 3).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) gru1 = nn.GRUUnit(size=D * 3) gru2 = nn.GRUUnit(size=D * 3, param_attr=weight_attr) - dy_ret1 = gru1(base.to_variable(input), - base.to_variable(hidden_input)) - dy_ret2 = gru2(base.to_variable(input), - base.to_variable(hidden_input)) + dy_ret1 = gru1( + base.to_variable(input), base.to_variable(hidden_input) + ) + dy_ret2 = gru2( + base.to_variable(input), base.to_variable(hidden_input) + ) self.assertFalse( - np.array_equal(gru1.weight.numpy(), gru2.weight.numpy())) + np.array_equal(gru1.weight.numpy(), gru2.weight.numpy()) + ) for o1, o2 in zip(dy_ret1, dy_ret2): self.assertFalse(np.array_equal(o1.numpy(), o2.numpy())) gru2.weight.set_value(gru1.weight.numpy()) gru2.bias.set_value(gru1.bias) - dy_ret1 = gru1(base.to_variable(input), - base.to_variable(hidden_input)) - dy_ret2 = gru2(base.to_variable(input), - base.to_variable(hidden_input)) + dy_ret1 = gru1( + base.to_variable(input), base.to_variable(hidden_input) + ) + dy_ret2 = gru2( + base.to_variable(input), base.to_variable(hidden_input) + ) for o1, o2 in zip(dy_ret1, dy_ret2): np.testing.assert_array_equal(o1.numpy(), o2.numpy()) gru2.weight = gru1.weight gru2.bias = gru1.bias - np.testing.assert_array_equal(gru1.weight.numpy(), - gru2.weight.numpy()) + np.testing.assert_array_equal( + gru1.weight.numpy(), gru2.weight.numpy() + ) np.testing.assert_array_equal(gru1.bias.numpy(), gru2.bias.numpy()) def test_elementwise_math(self): @@ -727,15 +795,10 @@ def test_elementwise_math(self): ret = layers.elementwise_sub(ret, t5) ret = layers.elementwise_mul(ret, t6) - static_ret = self.get_static_graph_result(feed={ - 't': n, - 't2': n2, - 't3': n3, - 't4': n4, - 't5': n5, - 't6': n6 - }, - fetch_list=[ret])[0] + static_ret = self.get_static_graph_result( + feed={'t': n, 't2': n2, 't3': n3, 't4': n4, 't5': n5, 't6': n6}, + fetch_list=[ret], + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -762,10 +825,12 @@ def test_elementwise_minmax(self): with self.dynamic_graph(): with _test_eager_guard(): - min_eager_ret = layers.elementwise_min(to_variable(n), - to_variable(n2)) - max_eager_ret = layers.elementwise_max(to_variable(n), - to_variable(n2)) + min_eager_ret = layers.elementwise_min( + to_variable(n), to_variable(n2) + ) + max_eager_ret = layers.elementwise_max( + to_variable(n), to_variable(n2) + ) min_eager_ret_value = min_eager_ret.numpy() max_eager_ret_value = max_eager_ret.numpy() @@ -786,39 +851,46 @@ def test_sequence_conv(self): else: place = core.CPUPlace() with self.static_graph(): - seq = layers.data(name='seq_in', - shape=[3, 4], - dtype='float32', - lod_level=1, - append_batch_size=False) + seq = layers.data( + name='seq_in', + shape=[3, 4], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) out = layers.sequence_conv(seq, 2, act='sigmoid') - static_rlt = self.get_static_graph_result(feed={ - "seq_in": - fluid.create_lod_tensor(data=inp_np, - recursive_seq_lens=[[1, 1, 1]], - place=place) - }, - fetch_list=[out], - with_lod=True)[0] + static_rlt = self.get_static_graph_result( + feed={ + "seq_in": fluid.create_lod_tensor( + data=inp_np, recursive_seq_lens=[[1, 1, 1]], place=place + ) + }, + fetch_list=[out], + with_lod=True, + )[0] with self.static_graph(): - seq = layers.data(name='seq_in', - shape=[3, 4], - dtype='float32', - lod_level=1, - append_batch_size=False) + seq = layers.data( + name='seq_in', + shape=[3, 4], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) seq_conv = nn.SequenceConv('seq_conv', num_filters=2, act='sigmoid') out = seq_conv(seq) - static_rlt2 = self.get_static_graph_result(feed={ - "seq_in": - fluid.create_lod_tensor(data=inp_np, - recursive_seq_lens=[[1, 1, 1]], - place=place) - }, - fetch_list=[out], - with_lod=True)[0] - np.testing.assert_array_equal(np.array(static_rlt), - np.array(static_rlt2)) + static_rlt2 = self.get_static_graph_result( + feed={ + "seq_in": fluid.create_lod_tensor( + data=inp_np, recursive_seq_lens=[[1, 1, 1]], place=place + ) + }, + fetch_list=[out], + with_lod=True, + )[0] + np.testing.assert_array_equal( + np.array(static_rlt), np.array(static_rlt2) + ) def test_conv2d_transpose(self): inp_np = np.arange(0, 24).reshape([2, 3, 2, 2]).astype('float32') @@ -829,9 +901,11 @@ def test_conv2d_transpose(self): num_filters=10, filter_size=27, act='sigmoid', - bias_attr=fluid.initializer.ConstantInitializer(value=1)) - static_rlt = self.get_static_graph_result(feed={'pixel': inp_np}, - fetch_list=[out])[0] + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) + static_rlt = self.get_static_graph_result( + feed={'pixel': inp_np}, fetch_list=[out] + )[0] with self.static_graph(): img = layers.data(name='pixel', shape=[3, 2, 2], dtype='float32') conv2d_transpose = nn.Conv2DTranspose( @@ -839,10 +913,12 @@ def test_conv2d_transpose(self): num_filters=10, filter_size=27, act='sigmoid', - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) out = conv2d_transpose(img) - static_rlt2 = self.get_static_graph_result(feed={'pixel': inp_np}, - fetch_list=[out])[0] + static_rlt2 = self.get_static_graph_result( + feed={'pixel': inp_np}, fetch_list=[out] + )[0] with self.dynamic_graph(): with _test_eager_guard(): conv2d_transpose = nn.Conv2DTranspose( @@ -850,7 +926,8 @@ def test_conv2d_transpose(self): num_filters=10, filter_size=27, act='sigmoid', - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) dy_eager_rlt = conv2d_transpose(base.to_variable(inp_np)) dy_eager_rlt_value = dy_eager_rlt.numpy() @@ -859,7 +936,8 @@ def test_conv2d_transpose(self): num_filters=10, filter_size=27, act='sigmoid', - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) dy_rlt = conv2d_transpose(base.to_variable(inp_np)) dy_rlt_value = dy_rlt.numpy() np.testing.assert_allclose(static_rlt2, static_rlt, rtol=1e-05) @@ -872,26 +950,33 @@ def test_conv2d_transpose(self): custom_weight = np.random.randn(3, 3, 2, 2).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) - conv2d1 = nn.Conv2DTranspose(num_channels=3, - num_filters=3, - filter_size=[2, 2]) - conv2d2 = nn.Conv2DTranspose(num_channels=3, - num_filters=3, - filter_size=[2, 2], - param_attr=weight_attr) + custom_weight + ) + ) + conv2d1 = nn.Conv2DTranspose( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) + conv2d2 = nn.Conv2DTranspose( + num_channels=3, + num_filters=3, + filter_size=[2, 2], + param_attr=weight_attr, + ) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) self.assertFalse( - np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) + np.array_equal(dy_ret1.numpy(), dy_ret2.numpy()) + ) conv2d1_weight_np = conv2d1.weight.numpy() conv2d1_bias = conv2d1.bias self.assertFalse( - np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy())) + np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy()) + ) conv2d2.weight.set_value(conv2d1_weight_np) - np.testing.assert_array_equal(conv2d1_weight_np, - conv2d2.weight.numpy()) + np.testing.assert_array_equal( + conv2d1_weight_np, conv2d2.weight.numpy() + ) conv2d2.bias.set_value(conv2d1_bias) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) @@ -899,22 +984,29 @@ def test_conv2d_transpose(self): conv2d2.weight = conv2d1.weight conv2d2.bias = conv2d1.bias - np.testing.assert_array_equal(conv2d1.weight.numpy(), - conv2d2.weight.numpy()) - np.testing.assert_array_equal(conv2d1.bias.numpy(), - conv2d2.bias.numpy()) + np.testing.assert_array_equal( + conv2d1.weight.numpy(), conv2d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv2d1.bias.numpy(), conv2d2.bias.numpy() + ) images = np.ones([2, 3, 5, 5], dtype='float32') custom_weight = np.random.randn(3, 3, 2, 2).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) - conv2d1 = nn.Conv2DTranspose(num_channels=3, - num_filters=3, - filter_size=[2, 2]) - conv2d2 = nn.Conv2DTranspose(num_channels=3, - num_filters=3, - filter_size=[2, 2], - param_attr=weight_attr) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) + conv2d1 = nn.Conv2DTranspose( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) + conv2d2 = nn.Conv2DTranspose( + num_channels=3, + num_filters=3, + filter_size=[2, 2], + param_attr=weight_attr, + ) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) self.assertFalse(np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) @@ -922,10 +1014,12 @@ def test_conv2d_transpose(self): conv2d1_weight_np = conv2d1.weight.numpy() conv2d1_bias = conv2d1.bias self.assertFalse( - np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy())) + np.array_equal(conv2d1_weight_np, conv2d2.weight.numpy()) + ) conv2d2.weight.set_value(conv2d1_weight_np) - np.testing.assert_array_equal(conv2d1_weight_np, - conv2d2.weight.numpy()) + np.testing.assert_array_equal( + conv2d1_weight_np, conv2d2.weight.numpy() + ) conv2d2.bias.set_value(conv2d1_bias) dy_ret1 = conv2d1(base.to_variable(images)) dy_ret2 = conv2d2(base.to_variable(images)) @@ -933,19 +1027,21 @@ def test_conv2d_transpose(self): conv2d2.weight = conv2d1.weight conv2d2.bias = conv2d1.bias - np.testing.assert_array_equal(conv2d1.weight.numpy(), - conv2d2.weight.numpy()) - np.testing.assert_array_equal(conv2d1.bias.numpy(), - conv2d2.bias.numpy()) + np.testing.assert_array_equal( + conv2d1.weight.numpy(), conv2d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv2d1.bias.numpy(), conv2d2.bias.numpy() + ) with self.static_graph(): # the input of Conv2DTranspose must be Variable. def test_Variable(): images = np.ones([2, 3, 5, 5], dtype='float32') - conv2d = nn.Conv2DTranspose(num_channels=3, - num_filters=3, - filter_size=[2, 2]) + conv2d = nn.Conv2DTranspose( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) conv2d_ret1 = conv2d(images) self.assertRaises(TypeError, test_Variable) @@ -953,12 +1049,12 @@ def test_Variable(): # the input dtype of Conv2DTranspose must be float16 or float32 or float64 # float16 only can be set on GPU place def test_type(): - images = layers.data(name='pixel', - shape=[3, 5, 5], - dtype='int32') - conv2d = nn.Conv2DTranspose(num_channels=3, - num_filters=3, - filter_size=[2, 2]) + images = layers.data( + name='pixel', shape=[3, 5, 5], dtype='int32' + ) + conv2d = nn.Conv2DTranspose( + num_channels=3, num_filters=3, filter_size=[2, 2] + ) conv2d_ret2 = conv2d(images) self.assertRaises(TypeError, test_type) @@ -968,48 +1064,42 @@ def test_bilinear_tensor_product(self): inp_np_y = np.array([[4, 5, 6]]).astype('float32') with self.static_graph(): - data_x = layers.data(name='x', - shape=[1, 3], - dtype="float32", - append_batch_size=False) - data_y = layers.data(name='y', - shape=[1, 3], - dtype="float32", - append_batch_size=False) + data_x = layers.data( + name='x', shape=[1, 3], dtype="float32", append_batch_size=False + ) + data_y = layers.data( + name='y', shape=[1, 3], dtype="float32", append_batch_size=False + ) out = layers.bilinear_tensor_product( data_x, data_y, 6, bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) - static_rlt = self.get_static_graph_result(feed={ - 'x': inp_np_x, - 'y': inp_np_y - }, - fetch_list=[out])[0] + static_rlt = self.get_static_graph_result( + feed={'x': inp_np_x, 'y': inp_np_y}, fetch_list=[out] + )[0] with self.static_graph(): - data_x = layers.data(name='x', - shape=[1, 3], - dtype="float32", - append_batch_size=False) - data_y = layers.data(name='y', - shape=[1, 3], - dtype="float32", - append_batch_size=False) + data_x = layers.data( + name='x', shape=[1, 3], dtype="float32", append_batch_size=False + ) + data_y = layers.data( + name='y', shape=[1, 3], dtype="float32", append_batch_size=False + ) btp = nn.BilinearTensorProduct( 3, 3, 6, bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) out = btp(data_x, data_y) - static_rlt2 = self.get_static_graph_result(feed={ - 'x': inp_np_x, - 'y': inp_np_y - }, - fetch_list=[out])[0] + static_rlt2 = self.get_static_graph_result( + feed={'x': inp_np_x, 'y': inp_np_y}, fetch_list=[out] + )[0] with self.dynamic_graph(): with _test_eager_guard(): btp = nn.BilinearTensorProduct( @@ -1017,9 +1107,11 @@ def test_bilinear_tensor_product(self): 3, 6, bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') - dy_eager_rlt = btp(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) + act='sigmoid', + ) + dy_eager_rlt = btp( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) dy_eager_rlt_value = dy_eager_rlt.numpy() btp = nn.BilinearTensorProduct( @@ -1027,41 +1119,39 @@ def test_bilinear_tensor_product(self): 3, 6, bias_attr=fluid.initializer.ConstantInitializer(value=1), - act='sigmoid') + act='sigmoid', + ) dy_rlt = btp(base.to_variable(inp_np_x), base.to_variable(inp_np_y)) dy_rlt_value = dy_rlt.numpy() with self.dynamic_graph(): with _test_eager_guard(): btp2 = nn.BilinearTensorProduct(3, 3, 6, act='sigmoid') - dy_eager_rlt2 = btp2(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) + dy_eager_rlt2 = btp2( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) dy_eager_rlt2_value = dy_eager_rlt2.numpy() btp2 = nn.BilinearTensorProduct(3, 3, 6, act='sigmoid') - dy_rlt2 = btp2(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) + dy_rlt2 = btp2( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) dy_rlt2_value = dy_rlt2.numpy() with self.static_graph(): - data_x2 = layers.data(name='x', - shape=[1, 3], - dtype="float32", - append_batch_size=False) - data_y2 = layers.data(name='y', - shape=[1, 3], - dtype="float32", - append_batch_size=False) - out2 = layers.bilinear_tensor_product(data_x2, - data_y2, - 6, - act='sigmoid') - - static_rlt3 = self.get_static_graph_result(feed={ - 'x': inp_np_x, - 'y': inp_np_y - }, - fetch_list=[out2])[0] + data_x2 = layers.data( + name='x', shape=[1, 3], dtype="float32", append_batch_size=False + ) + data_y2 = layers.data( + name='y', shape=[1, 3], dtype="float32", append_batch_size=False + ) + out2 = layers.bilinear_tensor_product( + data_x2, data_y2, 6, act='sigmoid' + ) + + static_rlt3 = self.get_static_graph_result( + feed={'x': inp_np_x, 'y': inp_np_y}, fetch_list=[out2] + )[0] np.testing.assert_array_equal(dy_rlt2_value, static_rlt3) np.testing.assert_array_equal(dy_eager_rlt2_value, static_rlt3) @@ -1074,87 +1164,108 @@ def test_bilinear_tensor_product(self): custom_weight = np.random.randn(6, 3, 3).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) + custom_weight + ) + ) btp1 = nn.BilinearTensorProduct(3, 3, 6, act='sigmoid') - btp2 = nn.BilinearTensorProduct(3, - 3, - 6, - act='sigmoid', - param_attr=weight_attr) - dy_rlt1 = btp1(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) - dy_rlt2 = btp2(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) + btp2 = nn.BilinearTensorProduct( + 3, 3, 6, act='sigmoid', param_attr=weight_attr + ) + dy_rlt1 = btp1( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) + dy_rlt2 = btp2( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) self.assertFalse( - np.array_equal(dy_rlt1.numpy(), dy_rlt2.numpy())) + np.array_equal(dy_rlt1.numpy(), dy_rlt2.numpy()) + ) btp2.weight.set_value(btp1.weight.numpy()) btp2.bias.set_value(btp1.bias) - dy_rlt1 = btp1(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) - dy_rlt2 = btp2(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) + dy_rlt1 = btp1( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) + dy_rlt2 = btp2( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) np.testing.assert_array_equal(dy_rlt1.numpy(), dy_rlt2.numpy()) btp2.weight = btp1.weight btp2.bias = btp1.bias - np.testing.assert_array_equal(btp1.weight.numpy(), - btp2.weight.numpy()) - np.testing.assert_array_equal(btp1.bias.numpy(), - btp2.bias.numpy()) + np.testing.assert_array_equal( + btp1.weight.numpy(), btp2.weight.numpy() + ) + np.testing.assert_array_equal( + btp1.bias.numpy(), btp2.bias.numpy() + ) custom_weight = np.random.randn(6, 3, 3).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) btp1 = nn.BilinearTensorProduct(3, 3, 6, act='sigmoid') - btp2 = nn.BilinearTensorProduct(3, - 3, - 6, - act='sigmoid', - param_attr=weight_attr) - dy_rlt1 = btp1(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) - dy_rlt2 = btp2(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) + btp2 = nn.BilinearTensorProduct( + 3, 3, 6, act='sigmoid', param_attr=weight_attr + ) + dy_rlt1 = btp1( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) + dy_rlt2 = btp2( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) self.assertFalse(np.array_equal(dy_rlt1.numpy(), dy_rlt2.numpy())) btp2.weight.set_value(btp1.weight.numpy()) btp2.bias.set_value(btp1.bias) - dy_rlt1 = btp1(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) - dy_rlt2 = btp2(base.to_variable(inp_np_x), - base.to_variable(inp_np_y)) + dy_rlt1 = btp1( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) + dy_rlt2 = btp2( + base.to_variable(inp_np_x), base.to_variable(inp_np_y) + ) np.testing.assert_array_equal(dy_rlt1.numpy(), dy_rlt2.numpy()) btp2.weight = btp1.weight btp2.bias = btp1.bias - np.testing.assert_array_equal(btp1.weight.numpy(), - btp2.weight.numpy()) + np.testing.assert_array_equal( + btp1.weight.numpy(), btp2.weight.numpy() + ) np.testing.assert_array_equal(btp1.bias.numpy(), btp2.bias.numpy()) def prelu_test(self, mode): inp_np = np.ones([5, 200, 100, 100]).astype('float32') with self.static_graph(): - data_t = layers.data(name="input", - shape=[5, 200, 100, 100], - dtype="float32", - append_batch_size=False) - out = layers.prelu(data_t, - mode, - param_attr=ParamAttr(initializer=Constant(1.0))) - static_rlt = self.get_static_graph_result(feed={"input": inp_np}, - fetch_list=[out])[0] + data_t = layers.data( + name="input", + shape=[5, 200, 100, 100], + dtype="float32", + append_batch_size=False, + ) + out = layers.prelu( + data_t, mode, param_attr=ParamAttr(initializer=Constant(1.0)) + ) + static_rlt = self.get_static_graph_result( + feed={"input": inp_np}, fetch_list=[out] + )[0] with self.static_graph(): - data_t = layers.data(name="input", - shape=[5, 200, 100, 100], - dtype="float32", - append_batch_size=False) - prelu = nn.PRelu(mode=mode, - channel=inp_np.shape[1], - input_shape=data_t.shape, - param_attr=ParamAttr(initializer=Constant(1.0))) + data_t = layers.data( + name="input", + shape=[5, 200, 100, 100], + dtype="float32", + append_batch_size=False, + ) + prelu = nn.PRelu( + mode=mode, + channel=inp_np.shape[1], + input_shape=data_t.shape, + param_attr=ParamAttr(initializer=Constant(1.0)), + ) out = prelu(data_t) - static_rlt2 = self.get_static_graph_result(feed={"input": inp_np}, - fetch_list=[out])[0] + static_rlt2 = self.get_static_graph_result( + feed={"input": inp_np}, fetch_list=[out] + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -1162,14 +1273,17 @@ def prelu_test(self, mode): mode=mode, channel=inp_np.shape[1], input_shape=inp_np.shape, - param_attr=ParamAttr(initializer=Constant(1.0))) + param_attr=ParamAttr(initializer=Constant(1.0)), + ) dy_eager_rlt = prelu(base.to_variable(inp_np)) dy_eager_rlt_value = dy_eager_rlt.numpy() - prelu = nn.PRelu(mode=mode, - channel=inp_np.shape[1], - input_shape=inp_np.shape, - param_attr=ParamAttr(initializer=Constant(1.0))) + prelu = nn.PRelu( + mode=mode, + channel=inp_np.shape[1], + input_shape=inp_np.shape, + param_attr=ParamAttr(initializer=Constant(1.0)), + ) dy_rlt = prelu(base.to_variable(inp_np)) dy_rlt_value = dy_rlt.numpy() @@ -1185,42 +1299,51 @@ def prelu_test(self, mode): mode=mode, channel=inp_np.shape[1], input_shape=inp_np.shape, - param_attr=ParamAttr(initializer=Constant(2.0))) + param_attr=ParamAttr(initializer=Constant(2.0)), + ) prelu2 = nn.PRelu( mode=mode, channel=inp_np.shape[1], input_shape=inp_np.shape, - param_attr=ParamAttr(initializer=Constant(1.0))) + param_attr=ParamAttr(initializer=Constant(1.0)), + ) dy_rlt1 = prelu1(inp) dy_rlt2 = prelu2(inp) self.assertFalse( - np.array_equal(prelu1.weight.numpy(), - prelu2.weight.numpy())) + np.array_equal(prelu1.weight.numpy(), prelu2.weight.numpy()) + ) self.assertFalse( - np.array_equal(dy_rlt1.numpy(), dy_rlt2.numpy())) + np.array_equal(dy_rlt1.numpy(), dy_rlt2.numpy()) + ) prelu2.weight.set_value(prelu1.weight.numpy()) dy_rlt1 = prelu1(inp) dy_rlt2 = prelu2(inp) np.testing.assert_array_equal(dy_rlt1.numpy(), dy_rlt2.numpy()) prelu2.weight = prelu1.weight - np.testing.assert_array_equal(prelu1.weight.numpy(), - prelu2.weight.numpy()) + np.testing.assert_array_equal( + prelu1.weight.numpy(), prelu2.weight.numpy() + ) inp_np = np.random.randn(5, 200, 100, 100).astype("float32") inp = base.to_variable(inp_np) - prelu1 = nn.PRelu(mode=mode, - channel=inp_np.shape[1], - input_shape=inp_np.shape, - param_attr=ParamAttr(initializer=Constant(2.0))) - prelu2 = nn.PRelu(mode=mode, - channel=inp_np.shape[1], - input_shape=inp_np.shape, - param_attr=ParamAttr(initializer=Constant(1.0))) + prelu1 = nn.PRelu( + mode=mode, + channel=inp_np.shape[1], + input_shape=inp_np.shape, + param_attr=ParamAttr(initializer=Constant(2.0)), + ) + prelu2 = nn.PRelu( + mode=mode, + channel=inp_np.shape[1], + input_shape=inp_np.shape, + param_attr=ParamAttr(initializer=Constant(1.0)), + ) dy_rlt1 = prelu1(inp) dy_rlt2 = prelu2(inp) self.assertFalse( - np.array_equal(prelu1.weight.numpy(), prelu2.weight.numpy())) + np.array_equal(prelu1.weight.numpy(), prelu2.weight.numpy()) + ) self.assertFalse(np.array_equal(dy_rlt1.numpy(), dy_rlt2.numpy())) prelu2.weight.set_value(prelu1.weight.numpy()) dy_rlt1 = prelu1(inp) @@ -1228,8 +1351,9 @@ def prelu_test(self, mode): np.testing.assert_array_equal(dy_rlt1.numpy(), dy_rlt2.numpy()) prelu2.weight = prelu1.weight - np.testing.assert_array_equal(prelu1.weight.numpy(), - prelu2.weight.numpy()) + np.testing.assert_array_equal( + prelu1.weight.numpy(), prelu2.weight.numpy() + ) def test_prelu(self): self.prelu_test("channel") @@ -1241,31 +1365,37 @@ def test_embeding(self): dict_size = 20 with self.static_graph(): data_t = layers.data(name='word', shape=[1], dtype='int64') - emb = layers.embedding(input=data_t, - size=[dict_size, 32], - param_attr='emb.w', - is_sparse=False) - static_rlt = self.get_static_graph_result(feed={'word': inp_word}, - fetch_list=[emb])[0] + emb = layers.embedding( + input=data_t, + size=[dict_size, 32], + param_attr='emb.w', + is_sparse=False, + ) + static_rlt = self.get_static_graph_result( + feed={'word': inp_word}, fetch_list=[emb] + )[0] with self.static_graph(): data_t = layers.data(name='word', shape=[1], dtype='int64') - emb2 = nn.Embedding(size=[dict_size, 32], - param_attr='emb.w', - is_sparse=False) + emb2 = nn.Embedding( + size=[dict_size, 32], param_attr='emb.w', is_sparse=False + ) emb_rlt = emb2(data_t) - static_rlt2 = self.get_static_graph_result(feed={'word': inp_word}, - fetch_list=[emb_rlt])[0] + static_rlt2 = self.get_static_graph_result( + feed={'word': inp_word}, fetch_list=[emb_rlt] + )[0] with self.dynamic_graph(): with _test_eager_guard(): - emb2 = nn.Embedding(size=[dict_size, 32], - param_attr='eager_emb.w', - is_sparse=False) + emb2 = nn.Embedding( + size=[dict_size, 32], + param_attr='eager_emb.w', + is_sparse=False, + ) dy_eager_rlt = emb2(base.to_variable(inp_word)) dy_eager_rlt_value = dy_eager_rlt.numpy() - emb2 = nn.Embedding(size=[dict_size, 32], - param_attr='emb.w', - is_sparse=False) + emb2 = nn.Embedding( + size=[dict_size, 32], param_attr='emb.w', is_sparse=False + ) dy_rlt = emb2(base.to_variable(inp_word)) dy_rlt_value = dy_rlt.numpy() @@ -1278,33 +1408,43 @@ def test_embeding(self): custom_weight = np.random.randn(dict_size, 32).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) + custom_weight + ) + ) emb1 = nn.Embedding(size=[dict_size, 32], is_sparse=False) - emb2 = nn.Embedding(size=[dict_size, 32], - param_attr=weight_attr, - is_sparse=False) + emb2 = nn.Embedding( + size=[dict_size, 32], + param_attr=weight_attr, + is_sparse=False, + ) rep1 = emb1(base.to_variable(inp_word)) rep2 = emb2(base.to_variable(inp_word)) self.assertFalse( - np.array_equal(emb1.weight.numpy(), custom_weight)) - np.testing.assert_array_equal(emb2.weight.numpy(), - custom_weight) + np.array_equal(emb1.weight.numpy(), custom_weight) + ) + np.testing.assert_array_equal( + emb2.weight.numpy(), custom_weight + ) self.assertFalse(np.array_equal(rep1.numpy(), rep2.numpy())) emb2.weight.set_value(emb1.weight.numpy()) rep2 = emb2(base.to_variable(inp_word)) np.testing.assert_array_equal(rep1.numpy(), rep2.numpy()) emb2.weight = emb1.weight - np.testing.assert_array_equal(emb1.weight.numpy(), - emb2.weight.numpy()) + np.testing.assert_array_equal( + emb1.weight.numpy(), emb2.weight.numpy() + ) custom_weight = np.random.randn(dict_size, 32).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) emb1 = nn.Embedding(size=[dict_size, 32], is_sparse=False) - emb2 = nn.Embedding(size=[dict_size, 32], - param_attr=weight_attr, - is_sparse=False) + emb2 = nn.Embedding( + size=[dict_size, 32], param_attr=weight_attr, is_sparse=False + ) rep1 = emb1(base.to_variable(inp_word)) rep2 = emb2(base.to_variable(inp_word)) self.assertFalse(np.array_equal(emb1.weight.numpy(), custom_weight)) @@ -1315,8 +1455,9 @@ def test_embeding(self): np.testing.assert_array_equal(rep1.numpy(), rep2.numpy()) emb2.weight = emb1.weight - np.testing.assert_array_equal(emb1.weight.numpy(), - emb2.weight.numpy()) + np.testing.assert_array_equal( + emb1.weight.numpy(), emb2.weight.numpy() + ) def test_nce(self): window_size = 5 @@ -1329,54 +1470,61 @@ def test_nce(self): words = [] for i in range(window_size): words.append( - layers.data(name='word_{0}'.format(i), - shape=[None], - dtype='int64')) - sample_weights = layers.fill_constant(shape=[5, 1], - dtype='float32', - value=1) + layers.data( + name='word_{0}'.format(i), shape=[None], dtype='int64' + ) + ) + sample_weights = layers.fill_constant( + shape=[5, 1], dtype='float32', value=1 + ) embs = [] for i in range(window_size): if i == label_word: continue - emb = fluid.embedding(input=words[i], - size=[dict_size, 32], - param_attr='emb.w', - is_sparse=False) + emb = fluid.embedding( + input=words[i], + size=[dict_size, 32], + param_attr='emb.w', + is_sparse=False, + ) embs.append(emb) embs = layers.concat(input=embs, axis=1) wl = fluid.layers.unsqueeze(words[label_word], axes=[0]) - nce_loss = layers.nce(input=embs, - label=wl, - num_total_classes=dict_size, - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr='nce.w', - bias_attr='nce.b', - sample_weight=sample_weights) + nce_loss = layers.nce( + input=embs, + label=wl, + num_total_classes=dict_size, + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr='nce.w', + bias_attr='nce.b', + sample_weight=sample_weights, + ) feed_dict = dict() for i in range(window_size): feed_dict['word_{0}'.format(i)] = inp_word[i] - static_rlt = self.get_static_graph_result(feed=feed_dict, - fetch_list=[nce_loss])[0] + static_rlt = self.get_static_graph_result( + feed=feed_dict, fetch_list=[nce_loss] + )[0] with self.static_graph(): words = [] for i in range(window_size): words.append( - layers.data(name='word_{0}'.format(i), - shape=[None], - dtype='int64')) - sample_weights = layers.fill_constant(shape=[5, 1], - dtype='float32', - value=1) - emb = nn.Embedding(size=[dict_size, 32], - param_attr='emb.w', - is_sparse=False) + layers.data( + name='word_{0}'.format(i), shape=[None], dtype='int64' + ) + ) + sample_weights = layers.fill_constant( + shape=[5, 1], dtype='float32', value=1 + ) + emb = nn.Embedding( + size=[dict_size, 32], param_attr='emb.w', is_sparse=False + ) embs2 = [] for i in range(window_size): @@ -1387,15 +1535,17 @@ def test_nce(self): embs2.append(emb_rlt) embs2 = layers.concat(input=embs2, axis=1) - nce = nn.NCE(num_total_classes=dict_size, - dim=embs2.shape[1], - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr='nce.w', - bias_attr='nce.b', - sample_weight=sample_weights) + nce = nn.NCE( + num_total_classes=dict_size, + dim=embs2.shape[1], + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr='nce.w', + bias_attr='nce.b', + sample_weight=sample_weights, + ) wl = fluid.layers.unsqueeze(words[label_word], axes=[0]) nce_loss2 = nce(embs2, wl) @@ -1403,21 +1553,23 @@ def test_nce(self): for i in range(len(words)): feed_dict['word_{0}'.format(i)] = inp_word[i] - static_rlt2 = self.get_static_graph_result(feed=feed_dict, - fetch_list=[nce_loss2 - ])[0] + static_rlt2 = self.get_static_graph_result( + feed=feed_dict, fetch_list=[nce_loss2] + )[0] with self.dynamic_graph(): with _test_eager_guard(): words = [] for i in range(window_size): words.append(base.to_variable(inp_word[i])) - sample_weights = layers.fill_constant(shape=[5, 1], - dtype='float32', - value=1) - emb = nn.Embedding(size=[dict_size, 32], - param_attr='eager_emb.w', - is_sparse=False) + sample_weights = layers.fill_constant( + shape=[5, 1], dtype='float32', value=1 + ) + emb = nn.Embedding( + size=[dict_size, 32], + param_attr='eager_emb.w', + is_sparse=False, + ) embs3 = [] for i in range(window_size): @@ -1427,18 +1579,20 @@ def test_nce(self): emb_rlt = emb(words[i]) embs3.append(emb_rlt) - embs3 = layers.concat(input=embs3, - axis=fluid.dygraph.to_variable( - np.array([1]))) - nce = nn.NCE(num_total_classes=dict_size, - dim=embs3.shape[1], - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr='eager_nce.w', - bias_attr='eager_nce.b', - sample_weight=sample_weights) + embs3 = layers.concat( + input=embs3, axis=fluid.dygraph.to_variable(np.array([1])) + ) + nce = nn.NCE( + num_total_classes=dict_size, + dim=embs3.shape[1], + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr='eager_nce.w', + bias_attr='eager_nce.b', + sample_weight=sample_weights, + ) wl = fluid.layers.unsqueeze(words[label_word], axes=[0]) dy_eager_rlt = nce(embs3, wl) @@ -1447,12 +1601,12 @@ def test_nce(self): words = [] for i in range(window_size): words.append(base.to_variable(inp_word[i])) - sample_weights = layers.fill_constant(shape=[5, 1], - dtype='float32', - value=1) - emb = nn.Embedding(size=[dict_size, 32], - param_attr='emb.w', - is_sparse=False) + sample_weights = layers.fill_constant( + shape=[5, 1], dtype='float32', value=1 + ) + emb = nn.Embedding( + size=[dict_size, 32], param_attr='emb.w', is_sparse=False + ) embs3 = [] for i in range(window_size): @@ -1462,17 +1616,20 @@ def test_nce(self): emb_rlt = emb(words[i]) embs3.append(emb_rlt) - embs3 = layers.concat(input=embs3, - axis=fluid.dygraph.to_variable(np.array([1]))) - nce = nn.NCE(num_total_classes=dict_size, - dim=embs3.shape[1], - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr='nce.w', - bias_attr='nce.b', - sample_weight=sample_weights) + embs3 = layers.concat( + input=embs3, axis=fluid.dygraph.to_variable(np.array([1])) + ) + nce = nn.NCE( + num_total_classes=dict_size, + dim=embs3.shape[1], + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr='nce.w', + bias_attr='nce.b', + sample_weight=sample_weights, + ) wl = fluid.layers.unsqueeze(words[label_word], axes=[0]) dy_rlt = nce(embs3, wl) @@ -1484,21 +1641,27 @@ def test_nce(self): with self.dynamic_graph(): with _test_eager_guard(): - custom_weight = np.random.randn(dict_size, - 128).astype("float32") + custom_weight = np.random.randn(dict_size, 128).astype( + "float32" + ) weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) + custom_weight + ) + ) words = [] for i in range(window_size): words.append(base.to_variable(inp_word[i])) sample_weights = layers.fill_constant( shape=fluid.dygraph.to_variable(np.array([5, 1])), dtype='float32', - value=1) - emb = nn.Embedding(size=[dict_size, 32], - param_attr='eager_emb.w', - is_sparse=False) + value=1, + ) + emb = nn.Embedding( + size=[dict_size, 32], + param_attr='eager_emb.w', + is_sparse=False, + ) embs3 = [] for i in range(window_size): @@ -1509,58 +1672,70 @@ def test_nce(self): embs3.append(emb_rlt) embs3 = layers.concat(input=embs3, axis=1) - nce1 = nn.NCE(num_total_classes=dict_size, - dim=embs3.shape[1], - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr='eager_nce1.w', - bias_attr='eager_nce1.b', - sample_weight=sample_weights) - - nce2 = nn.NCE(num_total_classes=dict_size, - dim=embs3.shape[1], - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr=weight_attr, - bias_attr='eager_nce2.b', - sample_weight=sample_weights) + nce1 = nn.NCE( + num_total_classes=dict_size, + dim=embs3.shape[1], + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr='eager_nce1.w', + bias_attr='eager_nce1.b', + sample_weight=sample_weights, + ) + + nce2 = nn.NCE( + num_total_classes=dict_size, + dim=embs3.shape[1], + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr=weight_attr, + bias_attr='eager_nce2.b', + sample_weight=sample_weights, + ) wl = fluid.layers.unsqueeze(words[label_word], axes=[0]) nce1_loss = nce1(embs3, wl) nce2_loss = nce2(embs3, wl) self.assertFalse( - np.array_equal(nce1_loss.numpy(), nce2_loss.numpy())) + np.array_equal(nce1_loss.numpy(), nce2_loss.numpy()) + ) nce2.weight.set_value(nce1.weight.numpy()) nce2.bias.set_value(nce1.bias) nce1_loss = nce1(embs3, wl) nce2_loss = nce2(embs3, wl) - np.testing.assert_array_equal(nce1_loss.numpy(), - nce2_loss.numpy()) + np.testing.assert_array_equal( + nce1_loss.numpy(), nce2_loss.numpy() + ) nce2.weight = nce1.weight nce2.bias = nce1.bias - np.testing.assert_array_equal(nce1.weight.numpy(), - nce2.weight.numpy()) - np.testing.assert_array_equal(nce1.bias.numpy(), - nce2.bias.numpy()) + np.testing.assert_array_equal( + nce1.weight.numpy(), nce2.weight.numpy() + ) + np.testing.assert_array_equal( + nce1.bias.numpy(), nce2.bias.numpy() + ) custom_weight = np.random.randn(dict_size, 128).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) words = [] for i in range(window_size): words.append(base.to_variable(inp_word[i])) sample_weights = layers.fill_constant( shape=fluid.dygraph.to_variable(np.array([5, 1])), dtype='float32', - value=1) - emb = nn.Embedding(size=[dict_size, 32], - param_attr='emb.w', - is_sparse=False) + value=1, + ) + emb = nn.Embedding( + size=[dict_size, 32], param_attr='emb.w', is_sparse=False + ) embs3 = [] for i in range(window_size): @@ -1571,31 +1746,36 @@ def test_nce(self): embs3.append(emb_rlt) embs3 = layers.concat(input=embs3, axis=1) - nce1 = nn.NCE(num_total_classes=dict_size, - dim=embs3.shape[1], - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr='nce1.w', - bias_attr='nce1.b', - sample_weight=sample_weights) - - nce2 = nn.NCE(num_total_classes=dict_size, - dim=embs3.shape[1], - num_neg_samples=2, - sampler="custom_dist", - custom_dist=nid_freq_arr.tolist(), - seed=seed, - param_attr=weight_attr, - bias_attr='nce2.b', - sample_weight=sample_weights) + nce1 = nn.NCE( + num_total_classes=dict_size, + dim=embs3.shape[1], + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr='nce1.w', + bias_attr='nce1.b', + sample_weight=sample_weights, + ) + + nce2 = nn.NCE( + num_total_classes=dict_size, + dim=embs3.shape[1], + num_neg_samples=2, + sampler="custom_dist", + custom_dist=nid_freq_arr.tolist(), + seed=seed, + param_attr=weight_attr, + bias_attr='nce2.b', + sample_weight=sample_weights, + ) wl = fluid.layers.unsqueeze(words[label_word], axes=[0]) nce1_loss = nce1(embs3, wl) nce2_loss = nce2(embs3, wl) self.assertFalse( - np.array_equal(nce1_loss.numpy(), nce2_loss.numpy())) + np.array_equal(nce1_loss.numpy(), nce2_loss.numpy()) + ) nce2.weight.set_value(nce1.weight.numpy()) nce2.bias.set_value(nce1.bias) nce1_loss = nce1(embs3, wl) @@ -1604,46 +1784,54 @@ def test_nce(self): nce2.weight = nce1.weight nce2.bias = nce1.bias - np.testing.assert_array_equal(nce1.weight.numpy(), - nce2.weight.numpy()) + np.testing.assert_array_equal( + nce1.weight.numpy(), nce2.weight.numpy() + ) np.testing.assert_array_equal(nce1.bias.numpy(), nce2.bias.numpy()) def test_one_hot(self): with self.dynamic_graph(): with _test_eager_guard(): - label = fluid.dygraph.to_variable(np.array([[1], [1], [3], - [0]])) + label = fluid.dygraph.to_variable( + np.array([[1], [1], [3], [0]]) + ) one_hot_label1 = fluid.layers.one_hot(input=label, depth=4) one_hot_label2 = fluid.layers.one_hot( - input=label, depth=fluid.dygraph.to_variable(np.array([4]))) - np.testing.assert_array_equal(one_hot_label1.numpy(), - one_hot_label2.numpy()) + input=label, depth=fluid.dygraph.to_variable(np.array([4])) + ) + np.testing.assert_array_equal( + one_hot_label1.numpy(), one_hot_label2.numpy() + ) label = fluid.dygraph.to_variable(np.array([[1], [1], [3], [0]])) one_hot_label1 = fluid.layers.one_hot(input=label, depth=4) one_hot_label2 = fluid.layers.one_hot( - input=label, depth=fluid.dygraph.to_variable(np.array([4]))) - np.testing.assert_array_equal(one_hot_label1.numpy(), - one_hot_label2.numpy()) + input=label, depth=fluid.dygraph.to_variable(np.array([4])) + ) + np.testing.assert_array_equal( + one_hot_label1.numpy(), one_hot_label2.numpy() + ) def test_split(self): with self.dynamic_graph(): with _test_eager_guard(): input = fluid.dygraph.to_variable(np.random.random((3, 8, 5))) x0, x1 = fluid.layers.split(input, num_or_sections=2, dim=1) - x00, x11 = fluid.layers.split(input, - num_or_sections=2, - dim=fluid.dygraph.to_variable( - np.array([1]))) + x00, x11 = fluid.layers.split( + input, + num_or_sections=2, + dim=fluid.dygraph.to_variable(np.array([1])), + ) np.testing.assert_array_equal(x0.numpy(), x00.numpy()) np.testing.assert_array_equal(x1.numpy(), x11.numpy()) input = fluid.dygraph.to_variable(np.random.random((3, 8, 5))) x0, x1 = fluid.layers.split(input, num_or_sections=2, dim=1) - x00, x11 = fluid.layers.split(input, - num_or_sections=2, - dim=fluid.dygraph.to_variable( - np.array([1]))) + x00, x11 = fluid.layers.split( + input, + num_or_sections=2, + dim=fluid.dygraph.to_variable(np.array([1])), + ) np.testing.assert_array_equal(x0.numpy(), x00.numpy()) np.testing.assert_array_equal(x1.numpy(), x11.numpy()) @@ -1653,40 +1841,48 @@ def test_topk(self): input = fluid.dygraph.to_variable(np.random.random((13, 11))) top5_values1, top5_indices1 = layers.topk(input, k=5) top5_values2, top5_indices2 = layers.topk( - input, k=fluid.dygraph.to_variable(np.array([5]))) - np.testing.assert_array_equal(top5_values1.numpy(), - top5_values2.numpy()) - np.testing.assert_array_equal(top5_indices1.numpy(), - top5_indices2.numpy()) + input, k=fluid.dygraph.to_variable(np.array([5])) + ) + np.testing.assert_array_equal( + top5_values1.numpy(), top5_values2.numpy() + ) + np.testing.assert_array_equal( + top5_indices1.numpy(), top5_indices2.numpy() + ) input = fluid.dygraph.to_variable(np.random.random((13, 11))) top5_values1, top5_indices1 = layers.topk(input, k=5) top5_values2, top5_indices2 = layers.topk( - input, k=fluid.dygraph.to_variable(np.array([5]))) - np.testing.assert_array_equal(top5_values1.numpy(), - top5_values2.numpy()) - np.testing.assert_array_equal(top5_indices1.numpy(), - top5_indices2.numpy()) + input, k=fluid.dygraph.to_variable(np.array([5])) + ) + np.testing.assert_array_equal( + top5_values1.numpy(), top5_values2.numpy() + ) + np.testing.assert_array_equal( + top5_indices1.numpy(), top5_indices2.numpy() + ) def test_conv3d(self): with self.static_graph(): - images = layers.data(name='pixel', - shape=[3, 6, 6, 6], - dtype='float32') + images = layers.data( + name='pixel', shape=[3, 6, 6, 6], dtype='float32' + ) ret = layers.conv3d(input=images, num_filters=3, filter_size=2) static_ret = self.get_static_graph_result( feed={'pixel': np.ones([2, 3, 6, 6, 6], dtype='float32')}, - fetch_list=[ret])[0] + fetch_list=[ret], + )[0] with self.static_graph(): - images = layers.data(name='pixel', - shape=[3, 6, 6, 6], - dtype='float32') + images = layers.data( + name='pixel', shape=[3, 6, 6, 6], dtype='float32' + ) conv3d = nn.Conv3D(num_channels=3, num_filters=3, filter_size=2) ret = conv3d(images) static_ret2 = self.get_static_graph_result( feed={'pixel': np.ones([2, 3, 6, 6, 6], dtype='float32')}, - fetch_list=[ret])[0] + fetch_list=[ret], + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -1710,26 +1906,33 @@ def test_conv3d(self): custom_weight = np.random.randn(3, 3, 2, 2, 2).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) - conv3d1 = nn.Conv3D(num_channels=3, - num_filters=3, - filter_size=2) - conv3d2 = nn.Conv3D(num_channels=3, - num_filters=3, - filter_size=2, - param_attr=weight_attr) + custom_weight + ) + ) + conv3d1 = nn.Conv3D( + num_channels=3, num_filters=3, filter_size=2 + ) + conv3d2 = nn.Conv3D( + num_channels=3, + num_filters=3, + filter_size=2, + param_attr=weight_attr, + ) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) self.assertFalse( - np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) + np.array_equal(dy_ret1.numpy(), dy_ret2.numpy()) + ) conv3d1_weight_np = conv3d1.weight.numpy() conv3d1_bias = conv3d1.bias self.assertFalse( - np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy())) + np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy()) + ) conv3d2.weight.set_value(conv3d1_weight_np) - np.testing.assert_array_equal(conv3d1_weight_np, - conv3d2.weight.numpy()) + np.testing.assert_array_equal( + conv3d1_weight_np, conv3d2.weight.numpy() + ) conv3d1.bias.set_value(conv3d1_bias) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) @@ -1737,20 +1940,27 @@ def test_conv3d(self): conv3d2.weight = conv3d1.weight conv3d2.bias = conv3d1.bias - np.testing.assert_array_equal(conv3d1.weight.numpy(), - conv3d2.weight.numpy()) - np.testing.assert_array_equal(conv3d1.bias.numpy(), - conv3d2.bias.numpy()) + np.testing.assert_array_equal( + conv3d1.weight.numpy(), conv3d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv3d1.bias.numpy(), conv3d2.bias.numpy() + ) images = np.ones([2, 3, 6, 6, 6], dtype='float32') custom_weight = np.random.randn(3, 3, 2, 2, 2).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) conv3d1 = nn.Conv3D(num_channels=3, num_filters=3, filter_size=2) - conv3d2 = nn.Conv3D(num_channels=3, - num_filters=3, - filter_size=2, - param_attr=weight_attr) + conv3d2 = nn.Conv3D( + num_channels=3, + num_filters=3, + filter_size=2, + param_attr=weight_attr, + ) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) self.assertFalse(np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) @@ -1758,10 +1968,12 @@ def test_conv3d(self): conv3d1_weight_np = conv3d1.weight.numpy() conv3d1_bias = conv3d1.bias self.assertFalse( - np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy())) + np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy()) + ) conv3d2.weight.set_value(conv3d1_weight_np) - np.testing.assert_array_equal(conv3d1_weight_np, - conv3d2.weight.numpy()) + np.testing.assert_array_equal( + conv3d1_weight_np, conv3d2.weight.numpy() + ) conv3d1.bias.set_value(conv3d1_bias) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) @@ -1769,10 +1981,12 @@ def test_conv3d(self): conv3d2.weight = conv3d1.weight conv3d2.bias = conv3d1.bias - np.testing.assert_array_equal(conv3d1.weight.numpy(), - conv3d2.weight.numpy()) - np.testing.assert_array_equal(conv3d1.bias.numpy(), - conv3d2.bias.numpy()) + np.testing.assert_array_equal( + conv3d1.weight.numpy(), conv3d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv3d1.bias.numpy(), conv3d2.bias.numpy() + ) def test_row_conv(self): input = np.arange(15).reshape([3, 5]).astype('float32') @@ -1782,37 +1996,43 @@ def test_row_conv(self): place = core.CPUPlace() with self.static_graph(): - x = layers.data(name='X', - shape=[3, 5], - dtype='float32', - lod_level=1, - append_batch_size=False) + x = layers.data( + name='X', + shape=[3, 5], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) ret = layers.row_conv(input=x, future_context_size=2) - static_ret = self.get_static_graph_result(feed={ - 'X': - fluid.create_lod_tensor(data=input, - recursive_seq_lens=[[1, 1, 1]], - place=place) - }, - fetch_list=[ret], - with_lod=True)[0] + static_ret = self.get_static_graph_result( + feed={ + 'X': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1, 1]], place=place + ) + }, + fetch_list=[ret], + with_lod=True, + )[0] with self.static_graph(): - x = layers.data(name='X', - shape=[3, 5], - dtype='float32', - lod_level=1, - append_batch_size=False) + x = layers.data( + name='X', + shape=[3, 5], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) rowConv = nn.RowConv('RowConv', future_context_size=2) ret = rowConv(x) - static_ret2 = self.get_static_graph_result(feed={ - 'X': - fluid.create_lod_tensor(data=input, - recursive_seq_lens=[[1, 1, 1]], - place=place) - }, - fetch_list=[ret], - with_lod=True)[0] + static_ret2 = self.get_static_graph_result( + feed={ + 'X': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1, 1]], place=place + ) + }, + fetch_list=[ret], + with_lod=True, + )[0] # TODO: dygraph can't support LODTensor @@ -1829,52 +2049,61 @@ def func_group_norm(self): input = np.random.random(shape).astype('float32') with self.static_graph(): - X = fluid.layers.data(name='X', - shape=shape, - dtype='float32', - lod_level=1, - append_batch_size=False) + X = fluid.layers.data( + name='X', + shape=shape, + dtype='float32', + lod_level=1, + append_batch_size=False, + ) ret = layers.group_norm( input=X, groups=2, param_attr=fluid.initializer.Uniform(low=-0.5, high=0.5), - bias_attr=fluid.initializer.ConstantInitializer(value=1)) - static_ret = self.get_static_graph_result(feed={ - 'X': - fluid.create_lod_tensor(data=input, - recursive_seq_lens=[[1, 1]], - place=place) - }, - fetch_list=[ret], - with_lod=True)[0] + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) + static_ret = self.get_static_graph_result( + feed={ + 'X': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1]], place=place + ) + }, + fetch_list=[ret], + with_lod=True, + )[0] with self.static_graph(): - X = fluid.layers.data(name='X', - shape=shape, - dtype='float32', - lod_level=1, - append_batch_size=False) + X = fluid.layers.data( + name='X', + shape=shape, + dtype='float32', + lod_level=1, + append_batch_size=False, + ) groupNorm = nn.GroupNorm( channels=shape[1], groups=2, param_attr=fluid.initializer.Uniform(low=-0.5, high=0.5), - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) ret = groupNorm(X) - static_ret2 = self.get_static_graph_result(feed={ - 'X': - fluid.create_lod_tensor(data=input, - recursive_seq_lens=[[1, 1]], - place=place) - }, - fetch_list=[ret], - with_lod=True)[0] + static_ret2 = self.get_static_graph_result( + feed={ + 'X': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1]], place=place + ) + }, + fetch_list=[ret], + with_lod=True, + )[0] with self.dynamic_graph(): groupNorm = nn.GroupNorm( channels=shape[1], groups=2, param_attr=fluid.initializer.Uniform(low=-0.5, high=0.5), - bias_attr=fluid.initializer.ConstantInitializer(value=1)) + bias_attr=fluid.initializer.ConstantInitializer(value=1), + ) dy_ret = groupNorm(base.to_variable(input)) dy_rlt_value = dy_ret.numpy() @@ -1897,23 +2126,23 @@ def test_instance_norm(self): input = np.random.random(shape).astype('float32') with self.static_graph(): - X = fluid.layers.data(name='X', - shape=shape, - dtype='float32', - append_batch_size=False) + X = fluid.layers.data( + name='X', shape=shape, dtype='float32', append_batch_size=False + ) ret = layers.instance_norm(input=X) - static_ret = self.get_static_graph_result(feed={'X': input}, - fetch_list=[ret])[0] + static_ret = self.get_static_graph_result( + feed={'X': input}, fetch_list=[ret] + )[0] with self.static_graph(): - X = fluid.layers.data(name='X', - shape=shape, - dtype='float32', - append_batch_size=False) + X = fluid.layers.data( + name='X', shape=shape, dtype='float32', append_batch_size=False + ) instanceNorm = nn.InstanceNorm(num_channels=shape[1]) ret = instanceNorm(X) - static_ret2 = self.get_static_graph_result(feed={'X': input}, - fetch_list=[ret])[0] + static_ret2 = self.get_static_graph_result( + feed={'X': input}, fetch_list=[ret] + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -1968,37 +2197,43 @@ def test_spectral_norm(self): input = np.random.random(shape).astype('float32') with self.static_graph(): - Weight = fluid.layers.data(name='Weight', - shape=shape, - dtype='float32', - lod_level=1, - append_batch_size=False) + Weight = fluid.layers.data( + name='Weight', + shape=shape, + dtype='float32', + lod_level=1, + append_batch_size=False, + ) ret = layers.spectral_norm(weight=Weight, dim=1, power_iters=2) - static_ret = self.get_static_graph_result(feed={ - 'Weight': - fluid.create_lod_tensor(data=input, - recursive_seq_lens=[[1, 1]], - place=place), - }, - fetch_list=[ret], - with_lod=True)[0] + static_ret = self.get_static_graph_result( + feed={ + 'Weight': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1]], place=place + ), + }, + fetch_list=[ret], + with_lod=True, + )[0] with self.static_graph(): - Weight = fluid.layers.data(name='Weight', - shape=shape, - dtype='float32', - lod_level=1, - append_batch_size=False) + Weight = fluid.layers.data( + name='Weight', + shape=shape, + dtype='float32', + lod_level=1, + append_batch_size=False, + ) spectralNorm = nn.SpectralNorm(shape, dim=1, power_iters=2) ret = spectralNorm(Weight) - static_ret2 = self.get_static_graph_result(feed={ - 'Weight': - fluid.create_lod_tensor(data=input, - recursive_seq_lens=[[1, 1]], - place=place) - }, - fetch_list=[ret], - with_lod=True)[0] + static_ret2 = self.get_static_graph_result( + feed={ + 'Weight': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1]], place=place + ) + }, + fetch_list=[ret], + with_lod=True, + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -2024,77 +2259,85 @@ def test_tree_conv(self): adj = np.tile(adj, (1, 1, 1)) vectors = np.random.random((1, 10, 5)).astype('float32') with self.static_graph(): - NodesVector = fluid.layers.data(name='NodesVector', - shape=(1, 10, 5), - dtype='float32', - lod_level=1, - append_batch_size=False) - EdgeSet = fluid.layers.data(name='EdgeSet', - shape=(1, 9, 2), - dtype='int32', - lod_level=1, - append_batch_size=False) - ret = fluid.contrib.layers.tree_conv(nodes_vector=NodesVector, - edge_set=EdgeSet, - output_size=6, - num_filters=1, - max_depth=2) - static_ret = self.get_static_graph_result(feed={ - 'NodesVector': - fluid.create_lod_tensor(data=vectors, - recursive_seq_lens=[[1]], - place=place), - 'EdgeSet': - fluid.create_lod_tensor(data=adj, - recursive_seq_lens=[[1]], - place=place) - }, - fetch_list=[ret], - with_lod=False)[0] + NodesVector = fluid.layers.data( + name='NodesVector', + shape=(1, 10, 5), + dtype='float32', + lod_level=1, + append_batch_size=False, + ) + EdgeSet = fluid.layers.data( + name='EdgeSet', + shape=(1, 9, 2), + dtype='int32', + lod_level=1, + append_batch_size=False, + ) + ret = fluid.contrib.layers.tree_conv( + nodes_vector=NodesVector, + edge_set=EdgeSet, + output_size=6, + num_filters=1, + max_depth=2, + ) + static_ret = self.get_static_graph_result( + feed={ + 'NodesVector': fluid.create_lod_tensor( + data=vectors, recursive_seq_lens=[[1]], place=place + ), + 'EdgeSet': fluid.create_lod_tensor( + data=adj, recursive_seq_lens=[[1]], place=place + ), + }, + fetch_list=[ret], + with_lod=False, + )[0] with self.static_graph(): - NodesVector = fluid.layers.data(name='NodesVector', - shape=(1, 10, 5), - dtype='float32', - lod_level=1, - append_batch_size=False) - EdgeSet = fluid.layers.data(name='EdgeSet', - shape=(1, 9, 2), - dtype='int32', - lod_level=1, - append_batch_size=False) - treeConv = nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2) + NodesVector = fluid.layers.data( + name='NodesVector', + shape=(1, 10, 5), + dtype='float32', + lod_level=1, + append_batch_size=False, + ) + EdgeSet = fluid.layers.data( + name='EdgeSet', + shape=(1, 9, 2), + dtype='int32', + lod_level=1, + append_batch_size=False, + ) + treeConv = nn.TreeConv( + feature_size=5, output_size=6, num_filters=1, max_depth=2 + ) ret = treeConv(NodesVector, EdgeSet) - static_ret2 = self.get_static_graph_result(feed={ - 'NodesVector': - fluid.create_lod_tensor(data=vectors, - recursive_seq_lens=[[1]], - place=place), - 'EdgeSet': - fluid.create_lod_tensor(data=adj, - recursive_seq_lens=[[1]], - place=place) - }, - fetch_list=[ret], - with_lod=False)[0] + static_ret2 = self.get_static_graph_result( + feed={ + 'NodesVector': fluid.create_lod_tensor( + data=vectors, recursive_seq_lens=[[1]], place=place + ), + 'EdgeSet': fluid.create_lod_tensor( + data=adj, recursive_seq_lens=[[1]], place=place + ), + }, + fetch_list=[ret], + with_lod=False, + )[0] with self.dynamic_graph(): with _test_eager_guard(): - treeConv = nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2) - dy_eager_ret = treeConv(base.to_variable(vectors), - base.to_variable(adj)) + treeConv = nn.TreeConv( + feature_size=5, output_size=6, num_filters=1, max_depth=2 + ) + dy_eager_ret = treeConv( + base.to_variable(vectors), base.to_variable(adj) + ) dy_eager_rlt_value = dy_eager_ret.numpy() - treeConv = nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2) + treeConv = nn.TreeConv( + feature_size=5, output_size=6, num_filters=1, max_depth=2 + ) dy_ret = treeConv(base.to_variable(vectors), base.to_variable(adj)) dy_rlt_value = dy_ret.numpy() @@ -2107,107 +2350,135 @@ def test_tree_conv(self): custom_weight = np.random.randn(5, 3, 6, 1).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) - treeConv1 = nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2, - bias_attr='eager_tc1_b') - treeConv2 = nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2, - param_attr=weight_attr, - bias_attr='eager_tc2_b') - dy_ret1 = treeConv1(base.to_variable(vectors), - base.to_variable(adj)) - dy_ret2 = treeConv2(base.to_variable(vectors), - base.to_variable(adj)) + custom_weight + ) + ) + treeConv1 = nn.TreeConv( + feature_size=5, + output_size=6, + num_filters=1, + max_depth=2, + bias_attr='eager_tc1_b', + ) + treeConv2 = nn.TreeConv( + feature_size=5, + output_size=6, + num_filters=1, + max_depth=2, + param_attr=weight_attr, + bias_attr='eager_tc2_b', + ) + dy_ret1 = treeConv1( + base.to_variable(vectors), base.to_variable(adj) + ) + dy_ret2 = treeConv2( + base.to_variable(vectors), base.to_variable(adj) + ) self.assertFalse( - np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) + np.array_equal(dy_ret1.numpy(), dy_ret2.numpy()) + ) treeConv2.weight.set_value(treeConv1.weight.numpy()) treeConv2.bias.set_value(treeConv1.bias) - dy_ret1 = treeConv1(base.to_variable(vectors), - base.to_variable(adj)) - dy_ret2 = treeConv2(base.to_variable(vectors), - base.to_variable(adj)) + dy_ret1 = treeConv1( + base.to_variable(vectors), base.to_variable(adj) + ) + dy_ret2 = treeConv2( + base.to_variable(vectors), base.to_variable(adj) + ) np.testing.assert_array_equal(dy_ret1.numpy(), dy_ret2.numpy()) treeConv2.weight = treeConv1.weight treeConv2.bias = treeConv1.bias - np.testing.assert_array_equal(treeConv1.weight.numpy(), - treeConv2.weight.numpy()) - np.testing.assert_array_equal(treeConv1.bias.numpy(), - treeConv2.bias.numpy()) + np.testing.assert_array_equal( + treeConv1.weight.numpy(), treeConv2.weight.numpy() + ) + np.testing.assert_array_equal( + treeConv1.bias.numpy(), treeConv2.bias.numpy() + ) custom_weight = np.random.randn(5, 3, 6, 1).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) - treeConv1 = nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2, - bias_attr='tc1_b') - treeConv2 = nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2, - param_attr=weight_attr, - bias_attr='tc2_b') - dy_ret1 = treeConv1(base.to_variable(vectors), - base.to_variable(adj)) - dy_ret2 = treeConv2(base.to_variable(vectors), - base.to_variable(adj)) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) + treeConv1 = nn.TreeConv( + feature_size=5, + output_size=6, + num_filters=1, + max_depth=2, + bias_attr='tc1_b', + ) + treeConv2 = nn.TreeConv( + feature_size=5, + output_size=6, + num_filters=1, + max_depth=2, + param_attr=weight_attr, + bias_attr='tc2_b', + ) + dy_ret1 = treeConv1( + base.to_variable(vectors), base.to_variable(adj) + ) + dy_ret2 = treeConv2( + base.to_variable(vectors), base.to_variable(adj) + ) self.assertFalse(np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) treeConv2.weight.set_value(treeConv1.weight.numpy()) treeConv2.bias.set_value(treeConv1.bias) - dy_ret1 = treeConv1(base.to_variable(vectors), - base.to_variable(adj)) - dy_ret2 = treeConv2(base.to_variable(vectors), - base.to_variable(adj)) + dy_ret1 = treeConv1( + base.to_variable(vectors), base.to_variable(adj) + ) + dy_ret2 = treeConv2( + base.to_variable(vectors), base.to_variable(adj) + ) np.testing.assert_array_equal(dy_ret1.numpy(), dy_ret2.numpy()) treeConv2.weight = treeConv1.weight treeConv2.bias = treeConv1.bias - np.testing.assert_array_equal(treeConv1.weight.numpy(), - treeConv2.weight.numpy()) - np.testing.assert_array_equal(treeConv1.bias.numpy(), - treeConv2.bias.numpy()) + np.testing.assert_array_equal( + treeConv1.weight.numpy(), treeConv2.weight.numpy() + ) + np.testing.assert_array_equal( + treeConv1.bias.numpy(), treeConv2.bias.numpy() + ) def test_conv3d_transpose(self): - input_array = np.arange(0, 48).reshape([2, 3, 2, 2, - 2]).astype('float32') + input_array = ( + np.arange(0, 48).reshape([2, 3, 2, 2, 2]).astype('float32') + ) with self.static_graph(): img = layers.data(name='pixel', shape=[3, 2, 2, 2], dtype='float32') - out = layers.conv3d_transpose(input=img, - num_filters=12, - filter_size=12, - use_cudnn=False) + out = layers.conv3d_transpose( + input=img, num_filters=12, filter_size=12, use_cudnn=False + ) static_rlt = self.get_static_graph_result( - feed={'pixel': input_array}, fetch_list=[out])[0] + feed={'pixel': input_array}, fetch_list=[out] + )[0] with self.static_graph(): img = layers.data(name='pixel', shape=[3, 2, 2, 2], dtype='float32') - conv3d_transpose = nn.Conv3DTranspose(num_channels=3, - num_filters=12, - filter_size=12, - use_cudnn=False) + conv3d_transpose = nn.Conv3DTranspose( + num_channels=3, num_filters=12, filter_size=12, use_cudnn=False + ) out = conv3d_transpose(img) static_rlt2 = self.get_static_graph_result( - feed={'pixel': input_array}, fetch_list=[out])[0] + feed={'pixel': input_array}, fetch_list=[out] + )[0] with self.dynamic_graph(): with _test_eager_guard(): - conv3d_transpose = nn.Conv3DTranspose(num_channels=3, - num_filters=12, - filter_size=12, - use_cudnn=False) + conv3d_transpose = nn.Conv3DTranspose( + num_channels=3, + num_filters=12, + filter_size=12, + use_cudnn=False, + ) dy_eager_rlt = conv3d_transpose(base.to_variable(input_array)) dy_eager_rlt_value = dy_eager_rlt.numpy() - conv3d_transpose = nn.Conv3DTranspose(num_channels=3, - num_filters=12, - filter_size=12, - use_cudnn=False) + conv3d_transpose = nn.Conv3DTranspose( + num_channels=3, num_filters=12, filter_size=12, use_cudnn=False + ) dy_rlt = conv3d_transpose(base.to_variable(input_array)) dy_rlt_value = dy_rlt.numpy() np.testing.assert_allclose(static_rlt2, static_rlt, rtol=1e-05) @@ -2220,30 +2491,39 @@ def test_conv3d_transpose(self): custom_weight = np.random.randn(3, 3, 2, 2, 2).astype("float32") weight_attr = fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - custom_weight)) - conv3d1 = nn.Conv3DTranspose(num_channels=3, - num_filters=3, - filter_size=2, - bias_attr='eager_conv3d1_b', - use_cudnn=False) - conv3d2 = nn.Conv3DTranspose(num_channels=3, - num_filters=3, - filter_size=2, - param_attr=weight_attr, - bias_attr='eager_conv3d2_b', - use_cudnn=False) + custom_weight + ) + ) + conv3d1 = nn.Conv3DTranspose( + num_channels=3, + num_filters=3, + filter_size=2, + bias_attr='eager_conv3d1_b', + use_cudnn=False, + ) + conv3d2 = nn.Conv3DTranspose( + num_channels=3, + num_filters=3, + filter_size=2, + param_attr=weight_attr, + bias_attr='eager_conv3d2_b', + use_cudnn=False, + ) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) self.assertFalse( - np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) + np.array_equal(dy_ret1.numpy(), dy_ret2.numpy()) + ) conv3d1_weight_np = conv3d1.weight.numpy() conv3d1_bias = conv3d1.bias self.assertFalse( - np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy())) + np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy()) + ) conv3d2.weight.set_value(conv3d1_weight_np) - np.testing.assert_array_equal(conv3d1_weight_np, - conv3d2.weight.numpy()) + np.testing.assert_array_equal( + conv3d1_weight_np, conv3d2.weight.numpy() + ) conv3d1.bias.set_value(conv3d1_bias) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) @@ -2251,26 +2531,35 @@ def test_conv3d_transpose(self): conv3d2.weight = conv3d1.weight conv3d2.bias = conv3d1.bias - np.testing.assert_array_equal(conv3d1.weight.numpy(), - conv3d2.weight.numpy()) - np.testing.assert_array_equal(conv3d1.bias.numpy(), - conv3d2.bias.numpy()) + np.testing.assert_array_equal( + conv3d1.weight.numpy(), conv3d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv3d1.bias.numpy(), conv3d2.bias.numpy() + ) images = np.ones([2, 3, 6, 6, 6], dtype='float32') custom_weight = np.random.randn(3, 3, 2, 2, 2).astype("float32") - weight_attr = fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(custom_weight)) - conv3d1 = nn.Conv3DTranspose(num_channels=3, - num_filters=3, - filter_size=2, - bias_attr='conv3d1_b', - use_cudnn=False) - conv3d2 = nn.Conv3DTranspose(num_channels=3, - num_filters=3, - filter_size=2, - param_attr=weight_attr, - bias_attr='conv3d2_b', - use_cudnn=False) + weight_attr = fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + custom_weight + ) + ) + conv3d1 = nn.Conv3DTranspose( + num_channels=3, + num_filters=3, + filter_size=2, + bias_attr='conv3d1_b', + use_cudnn=False, + ) + conv3d2 = nn.Conv3DTranspose( + num_channels=3, + num_filters=3, + filter_size=2, + param_attr=weight_attr, + bias_attr='conv3d2_b', + use_cudnn=False, + ) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) self.assertFalse(np.array_equal(dy_ret1.numpy(), dy_ret2.numpy())) @@ -2278,10 +2567,12 @@ def test_conv3d_transpose(self): conv3d1_weight_np = conv3d1.weight.numpy() conv3d1_bias = conv3d1.bias self.assertFalse( - np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy())) + np.array_equal(conv3d1_weight_np, conv3d2.weight.numpy()) + ) conv3d2.weight.set_value(conv3d1_weight_np) - np.testing.assert_array_equal(conv3d1_weight_np, - conv3d2.weight.numpy()) + np.testing.assert_array_equal( + conv3d1_weight_np, conv3d2.weight.numpy() + ) conv3d1.bias.set_value(conv3d1_bias) dy_ret1 = conv3d1(base.to_variable(images)) dy_ret2 = conv3d2(base.to_variable(images)) @@ -2289,10 +2580,12 @@ def test_conv3d_transpose(self): conv3d2.weight = conv3d1.weight conv3d2.bias = conv3d1.bias - np.testing.assert_array_equal(conv3d1.weight.numpy(), - conv3d2.weight.numpy()) - np.testing.assert_array_equal(conv3d1.bias.numpy(), - conv3d2.bias.numpy()) + np.testing.assert_array_equal( + conv3d1.weight.numpy(), conv3d2.weight.numpy() + ) + np.testing.assert_array_equal( + conv3d1.bias.numpy(), conv3d2.bias.numpy() + ) def test_eye_op(self): np_eye = np.eye(3, 2) @@ -2304,12 +2597,12 @@ def test_eye_op(self): with self.dynamic_graph(): with _test_eager_guard(): eager_eye_tensor = layers.eye(num_rows=3, num_columns=2) - eager_eye_tensor_rlt1 = layers.eye(num_rows=3, - num_columns=2, - batch_shape=[3]) - eager_eye_tensor_rlt2 = layers.eye(num_rows=3, - num_columns=2, - batch_shape=[4, 3]) + eager_eye_tensor_rlt1 = layers.eye( + num_rows=3, num_columns=2, batch_shape=[3] + ) + eager_eye_tensor_rlt2 = layers.eye( + num_rows=3, num_columns=2, batch_shape=[4, 3] + ) eager_diag_tensor = layers.eye(20) eager_eye_tensor_value = eager_eye_tensor.numpy() eager_eye_tensor_rlt1_value = eager_eye_tensor_rlt1.numpy() @@ -2317,12 +2610,12 @@ def test_eye_op(self): eager_diag_tensor_value = eager_diag_tensor.numpy() eye_tensor = layers.eye(num_rows=3, num_columns=2) - eye_tensor_rlt1 = layers.eye(num_rows=3, - num_columns=2, - batch_shape=[3]) - eye_tensor_rlt2 = layers.eye(num_rows=3, - num_columns=2, - batch_shape=[4, 3]) + eye_tensor_rlt1 = layers.eye( + num_rows=3, num_columns=2, batch_shape=[3] + ) + eye_tensor_rlt2 = layers.eye( + num_rows=3, num_columns=2, batch_shape=[4, 3] + ) diag_tensor = layers.eye(20) eye_tensor_value = eye_tensor.numpy() eye_tensor_rlt1_value = eye_tensor_rlt1.numpy() @@ -2330,23 +2623,23 @@ def test_eye_op(self): diag_tensor_value = diag_tensor.numpy() np.testing.assert_allclose(eager_eye_tensor_value, np_eye, rtol=1e-05) - np.testing.assert_allclose(eager_eye_tensor_rlt1_value, - stack_rlt1, - rtol=1e-05) - np.testing.assert_allclose(eager_eye_tensor_rlt2_value, - stack_rlt2, - rtol=1e-05) - np.testing.assert_allclose(eager_diag_tensor_value, - np.eye(20), - rtol=1e-05) + np.testing.assert_allclose( + eager_eye_tensor_rlt1_value, stack_rlt1, rtol=1e-05 + ) + np.testing.assert_allclose( + eager_eye_tensor_rlt2_value, stack_rlt2, rtol=1e-05 + ) + np.testing.assert_allclose( + eager_diag_tensor_value, np.eye(20), rtol=1e-05 + ) np.testing.assert_allclose(eye_tensor_value, np_eye, rtol=1e-05) - np.testing.assert_allclose(eye_tensor_rlt1_value, - stack_rlt1, - rtol=1e-05) - np.testing.assert_allclose(eye_tensor_rlt2_value, - stack_rlt2, - rtol=1e-05) + np.testing.assert_allclose( + eye_tensor_rlt1_value, stack_rlt1, rtol=1e-05 + ) + np.testing.assert_allclose( + eye_tensor_rlt2_value, stack_rlt2, rtol=1e-05 + ) np.testing.assert_allclose(diag_tensor_value, np.eye(20), rtol=1e-05) with self.assertRaises(TypeError): @@ -2406,11 +2699,9 @@ def test_compare(self): a = layers.data(name='a', shape=[1], dtype='int64') b = layers.data(name='b', shape=[1], dtype='int64') cond = layers.less_than(x=a, y=b) - static_ret = self.get_static_graph_result(feed={ - "a": value_a, - "b": value_b - }, - fetch_list=[cond])[0] + static_ret = self.get_static_graph_result( + feed={"a": value_a, "b": value_b}, fetch_list=[cond] + )[0] with self.dynamic_graph(): with _test_eager_guard(): da = base.to_variable(value_a) @@ -2432,11 +2723,9 @@ def test_compare(self): a1 = layers.data(name='a1', shape=[1], dtype='int64') b1 = layers.data(name='b1', shape=[1], dtype='int64') cond1 = layers.less_equal(x=a1, y=b1) - static_ret1 = self.get_static_graph_result(feed={ - "a1": value_a, - "b1": value_b - }, - fetch_list=[cond1])[0] + static_ret1 = self.get_static_graph_result( + feed={"a1": value_a, "b1": value_b}, fetch_list=[cond1] + )[0] with self.dynamic_graph(): with _test_eager_guard(): da1 = base.to_variable(value_a) @@ -2453,16 +2742,14 @@ def test_compare(self): for i in range(len(static_ret1)): self.assertTrue(dcond1.numpy()[i] == static_ret1[i]) - #greater than + # greater than with self.static_graph(): a2 = layers.data(name='a2', shape=[1], dtype='int64') b2 = layers.data(name='b2', shape=[1], dtype='int64') cond2 = layers.greater_than(x=a2, y=b2) - static_ret2 = self.get_static_graph_result(feed={ - "a2": value_a, - "b2": value_b - }, - fetch_list=[cond2])[0] + static_ret2 = self.get_static_graph_result( + feed={"a2": value_a, "b2": value_b}, fetch_list=[cond2] + )[0] with self.dynamic_graph(): with _test_eager_guard(): da2 = base.to_variable(value_a) @@ -2479,16 +2766,14 @@ def test_compare(self): for i in range(len(static_ret2)): self.assertTrue(dcond2.numpy()[i] == static_ret2[i]) - #greater equal + # greater equal with self.static_graph(): a3 = layers.data(name='a3', shape=[1], dtype='int64') b3 = layers.data(name='b3', shape=[1], dtype='int64') cond3 = layers.greater_equal(x=a3, y=b3) - static_ret3 = self.get_static_graph_result(feed={ - "a3": value_a, - "b3": value_b - }, - fetch_list=[cond3])[0] + static_ret3 = self.get_static_graph_result( + feed={"a3": value_a, "b3": value_b}, fetch_list=[cond3] + )[0] with self.dynamic_graph(): with _test_eager_guard(): da3 = base.to_variable(value_a) @@ -2510,11 +2795,9 @@ def test_compare(self): a4 = layers.data(name='a4', shape=[1], dtype='int64') b4 = layers.data(name='b4', shape=[1], dtype='int64') cond4 = layers.equal(x=a4, y=b4) - static_ret4 = self.get_static_graph_result(feed={ - "a4": value_a, - "b4": value_b - }, - fetch_list=[cond4])[0] + static_ret4 = self.get_static_graph_result( + feed={"a4": value_a, "b4": value_b}, fetch_list=[cond4] + )[0] with self.dynamic_graph(): with _test_eager_guard(): da4 = base.to_variable(value_a) @@ -2536,11 +2819,9 @@ def test_compare(self): a5 = layers.data(name='a5', shape=[1], dtype='int64') b5 = layers.data(name='b5', shape=[1], dtype='int64') cond5 = layers.equal(x=a5, y=b5) - static_ret5 = self.get_static_graph_result(feed={ - "a5": value_a, - "b5": value_b - }, - fetch_list=[cond5])[0] + static_ret5 = self.get_static_graph_result( + feed={"a5": value_a, "b5": value_b}, fetch_list=[cond5] + )[0] with self.dynamic_graph(): with _test_eager_guard(): da5 = base.to_variable(value_a) @@ -2558,7 +2839,6 @@ def test_compare(self): self.assertTrue(dcond5.numpy()[i] == static_ret5[i]) def test_cond(self): - def less_than_branch(a, b): return fluid.layers.elementwise_add(a, b) @@ -2566,16 +2846,22 @@ def greater_equal_branch(a, b): return fluid.layers.elementwise_sub(a, b) with self.static_graph(): - a = fluid.layers.fill_constant(shape=[1], - dtype='float32', - value=0.1) - b = fluid.layers.fill_constant(shape=[1], - dtype='float32', - value=0.23) - out = fluid.layers.cond(a >= b, lambda: greater_equal_branch(a, b), - lambda: less_than_branch(a, b)) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + a = fluid.layers.fill_constant( + shape=[1], dtype='float32', value=0.1 + ) + b = fluid.layers.fill_constant( + shape=[1], dtype='float32', value=0.23 + ) + out = fluid.layers.cond( + a >= b, + lambda: greater_equal_branch(a, b), + lambda: less_than_branch(a, b), + ) + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) ret = exe.run(fetch_list=[out]) static_res = ret[0] @@ -2584,15 +2870,23 @@ def greater_equal_branch(a, b): with _test_eager_guard(): a = fluid.dygraph.to_variable(np.array([0.1]).astype('float32')) b = fluid.dygraph.to_variable( - np.array([0.23]).astype('float32')) - out = layers.cond(a < b, lambda: less_than_branch(a, b), - lambda: greater_equal_branch(a, b)) - out2 = layers.cond(a >= b, lambda: greater_equal_branch(a, b), - lambda: less_than_branch(a, b)) + np.array([0.23]).astype('float32') + ) + out = layers.cond( + a < b, + lambda: less_than_branch(a, b), + lambda: greater_equal_branch(a, b), + ) + out2 = layers.cond( + a >= b, + lambda: greater_equal_branch(a, b), + lambda: less_than_branch(a, b), + ) eager_dynamic_res = out.numpy() eager_dynamic_res2 = out2.numpy() - np.testing.assert_array_equal(eager_dynamic_res, - eager_dynamic_res2) + np.testing.assert_array_equal( + eager_dynamic_res, eager_dynamic_res2 + ) with self.assertRaises(TypeError): layers.cond(a < b, 'str', 'str') with self.assertRaises(TypeError): @@ -2600,10 +2894,16 @@ def greater_equal_branch(a, b): a = fluid.dygraph.to_variable(np.array([0.1]).astype('float32')) b = fluid.dygraph.to_variable(np.array([0.23]).astype('float32')) - out = layers.cond(a < b, lambda: less_than_branch(a, b), - lambda: greater_equal_branch(a, b)) - out2 = layers.cond(a >= b, lambda: greater_equal_branch(a, b), - lambda: less_than_branch(a, b)) + out = layers.cond( + a < b, + lambda: less_than_branch(a, b), + lambda: greater_equal_branch(a, b), + ) + out2 = layers.cond( + a >= b, + lambda: greater_equal_branch(a, b), + lambda: less_than_branch(a, b), + ) dynamic_res = out.numpy() dynamic_res2 = out2.numpy() np.testing.assert_array_equal(dynamic_res, dynamic_res2) @@ -2616,7 +2916,6 @@ def greater_equal_branch(a, b): np.testing.assert_array_equal(static_res, eager_dynamic_res) def test_case(self): - def fn_1(): return layers.fill_constant(shape=[1, 2], dtype='float32', value=1) @@ -2635,12 +2934,16 @@ def fn_3(): pred_2 = layers.less_than(x, y) # false: 0.3 < 0.1 pred_3 = layers.equal(x, y) # false: 0.3 == 0.1 - out_1 = layers.case(pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], - default=fn_3) + out_1 = layers.case( + pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], default=fn_3 + ) out_2 = layers.case(pred_fn_pairs=[(pred_2, fn_2), (pred_3, fn_3)]) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) static_res1, static_res2 = exe.run(fetch_list=[out_1, out_2]) @@ -2654,11 +2957,12 @@ def fn_3(): pred_2 = layers.less_than(x, y) # false: 0.3 < 0.1 pred_3 = layers.equal(x, y) # false: 0.3 == 0.1 - out_1 = layers.case(pred_fn_pairs=[(pred_1, fn_1), - (pred_2, fn_2)], - default=fn_3) - out_2 = layers.case(pred_fn_pairs=[(pred_2, fn_2), (pred_3, - fn_3)]) + out_1 = layers.case( + pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], default=fn_3 + ) + out_2 = layers.case( + pred_fn_pairs=[(pred_2, fn_2), (pred_3, fn_3)] + ) eager_dynamic_res1 = out_1.numpy() eager_dynamic_res2 = out_2.numpy() @@ -2670,8 +2974,9 @@ def fn_3(): pred_2 = layers.less_than(x, y) # false: 0.3 < 0.1 pred_3 = layers.equal(x, y) # false: 0.3 == 0.1 - out_1 = layers.case(pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], - default=fn_3) + out_1 = layers.case( + pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], default=fn_3 + ) out_2 = layers.case(pred_fn_pairs=[(pred_2, fn_2), (pred_3, fn_3)]) dynamic_res1 = out_1.numpy() dynamic_res2 = out_2.numpy() @@ -2682,7 +2987,6 @@ def fn_3(): np.testing.assert_array_equal(static_res2, eager_dynamic_res2) def test_switch_case(self): - def fn_1(): return layers.fill_constant(shape=[1, 2], dtype='float32', value=1) @@ -2696,46 +3000,54 @@ def fn_3(): index_1 = layers.fill_constant(shape=[1], dtype='int32', value=1) index_2 = layers.fill_constant(shape=[1], dtype='int32', value=2) - out_1 = layers.switch_case(branch_index=index_1, - branch_fns={ - 1: fn_1, - 2: fn_2 - }, - default=fn_3) - out_2 = layers.switch_case(branch_index=index_2, - branch_fns=[(1, fn_1), (2, fn_2)], - default=fn_3) - out_3 = layers.switch_case(branch_index=index_2, - branch_fns=[(0, fn_1), (4, fn_2), - (7, fn_3)]) - - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + out_1 = layers.switch_case( + branch_index=index_1, + branch_fns={1: fn_1, 2: fn_2}, + default=fn_3, + ) + out_2 = layers.switch_case( + branch_index=index_2, + branch_fns=[(1, fn_1), (2, fn_2)], + default=fn_3, + ) + out_3 = layers.switch_case( + branch_index=index_2, + branch_fns=[(0, fn_1), (4, fn_2), (7, fn_3)], + ) + + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) static_res1, static_res2, static_res3 = exe.run( - fetch_list=[out_1, out_2, out_3]) + fetch_list=[out_1, out_2, out_3] + ) with self.dynamic_graph(): with _test_eager_guard(): - index_1 = layers.fill_constant(shape=[1], - dtype='int32', - value=1) - index_2 = layers.fill_constant(shape=[1], - dtype='int32', - value=2) - - out_1 = layers.switch_case(branch_index=index_1, - branch_fns={ - 1: fn_1, - 2: fn_2 - }, - default=fn_3) - out_2 = layers.switch_case(branch_index=index_2, - branch_fns=[(1, fn_1), (2, fn_2)], - default=fn_3) - out_3 = layers.switch_case(branch_index=index_2, - branch_fns=[(0, fn_1), (4, fn_2), - (7, fn_3)]) + index_1 = layers.fill_constant( + shape=[1], dtype='int32', value=1 + ) + index_2 = layers.fill_constant( + shape=[1], dtype='int32', value=2 + ) + + out_1 = layers.switch_case( + branch_index=index_1, + branch_fns={1: fn_1, 2: fn_2}, + default=fn_3, + ) + out_2 = layers.switch_case( + branch_index=index_2, + branch_fns=[(1, fn_1), (2, fn_2)], + default=fn_3, + ) + out_3 = layers.switch_case( + branch_index=index_2, + branch_fns=[(0, fn_1), (4, fn_2), (7, fn_3)], + ) eager_dynamic_res1 = out_1.numpy() eager_dynamic_res2 = out_2.numpy() @@ -2744,18 +3056,20 @@ def fn_3(): index_1 = layers.fill_constant(shape=[1], dtype='int32', value=1) index_2 = layers.fill_constant(shape=[1], dtype='int32', value=2) - out_1 = layers.switch_case(branch_index=index_1, - branch_fns={ - 1: fn_1, - 2: fn_2 - }, - default=fn_3) - out_2 = layers.switch_case(branch_index=index_2, - branch_fns=[(1, fn_1), (2, fn_2)], - default=fn_3) - out_3 = layers.switch_case(branch_index=index_2, - branch_fns=[(0, fn_1), (4, fn_2), - (7, fn_3)]) + out_1 = layers.switch_case( + branch_index=index_1, + branch_fns={1: fn_1, 2: fn_2}, + default=fn_3, + ) + out_2 = layers.switch_case( + branch_index=index_2, + branch_fns=[(1, fn_1), (2, fn_2)], + default=fn_3, + ) + out_3 = layers.switch_case( + branch_index=index_2, + branch_fns=[(0, fn_1), (4, fn_2), (7, fn_3)], + ) dynamic_res1 = out_1.numpy() dynamic_res2 = out_2.numpy() @@ -2772,32 +3086,32 @@ def test_crop_tensor(self): with self.static_graph(): x = fluid.layers.data(name="x1", shape=[6, 5, 8]) - dim1 = fluid.layers.data(name="dim1", - shape=[1], - append_batch_size=False) - dim2 = fluid.layers.data(name="dim2", - shape=[1], - append_batch_size=False) + dim1 = fluid.layers.data( + name="dim1", shape=[1], append_batch_size=False + ) + dim2 = fluid.layers.data( + name="dim2", shape=[1], append_batch_size=False + ) crop_shape1 = (1, 2, 4, 4) - crop_shape2 = fluid.layers.data(name="crop_shape", - shape=[4], - append_batch_size=False) + crop_shape2 = fluid.layers.data( + name="crop_shape", shape=[4], append_batch_size=False + ) crop_shape3 = [-1, dim1, dim2, 4] crop_offsets1 = [0, 0, 1, 0] - crop_offsets2 = fluid.layers.data(name="crop_offset", - shape=[4], - append_batch_size=False) + crop_offsets2 = fluid.layers.data( + name="crop_offset", shape=[4], append_batch_size=False + ) crop_offsets3 = [0, dim1, dim2, 0] - out1 = fluid.layers.crop_tensor(x, - shape=crop_shape1, - offsets=crop_offsets1) - out2 = fluid.layers.crop_tensor(x, - shape=crop_shape2, - offsets=crop_offsets2) - out3 = fluid.layers.crop_tensor(x, - shape=crop_shape3, - offsets=crop_offsets3) + out1 = fluid.layers.crop_tensor( + x, shape=crop_shape1, offsets=crop_offsets1 + ) + out2 = fluid.layers.crop_tensor( + x, shape=crop_shape2, offsets=crop_offsets2 + ) + out3 = fluid.layers.crop_tensor( + x, shape=crop_shape3, offsets=crop_offsets3 + ) self.assertIsNotNone(out1) self.assertIsNotNone(out2) @@ -2806,10 +3120,9 @@ def test_crop_tensor(self): def test_shard_index(self): with self.static_graph(): x = fluid.layers.data(name="label", shape=[4, 1], dtype='int64') - shard_label = fluid.layers.shard_index(input=x, - index_num=20, - nshards=2, - shard_id=0) + shard_label = fluid.layers.shard_index( + input=x, index_num=20, nshards=2, shard_id=0 + ) self.assertIsNotNone(shard_label) @@ -2828,11 +3141,9 @@ def test_accuracy(self): exe.run(fluid.default_startup_program()) # x = np.random.rand(3, 32, 32).astype("float32") # y = np.array([[1], [0], [1]]) - static_out = exe.run(feed={ - "input": x, - "label": y - }, - fetch_list=result[0]) + static_out = exe.run( + feed={"input": x, "label": y}, fetch_list=result[0] + ) with self.dynamic_graph(force_to_use_cpu=True): data = base.to_variable(x) @@ -2845,15 +3156,19 @@ def test_accuracy(self): class TestBook(LayerTest): - def setUp(self): self.only_static_set = set({"make_word_embedding"}) - self.not_compare_static_dygraph_set = set({ - "make_gaussian_random", "make_gaussian_random_batch_size_like", - "make_kldiv_loss", "make_prelu", - "make_sampled_softmax_with_cross_entropy", "make_sampling_id", - "make_uniform_random_batch_size_like" - }) + self.not_compare_static_dygraph_set = set( + { + "make_gaussian_random", + "make_gaussian_random_batch_size_like", + "make_kldiv_loss", + "make_prelu", + "make_sampled_softmax_with_cross_entropy", + "make_sampling_id", + "make_uniform_random_batch_size_like", + } + ) self.all_close_compare = set({"make_spectral_norm"}) def func_all_layers(self): @@ -2877,7 +3192,8 @@ def func_all_layers(self): static_result = self.get_static_graph_result( feed=self._feed_dict, fetch_list=fetch_list, - force_to_use_cpu=self._force_to_use_cpu) + force_to_use_cpu=self._force_to_use_cpu, + ) else: assert method.__name__ in ('make_get_places') @@ -2898,7 +3214,9 @@ def func_all_layers(self): rtol=1e-05, atol=0, err_msg='Result of function [{}] compare failed'.format( - method.__name__)) + method.__name__ + ), + ) continue if method.__name__ not in self.not_compare_static_dygraph_set: @@ -2906,7 +3224,9 @@ def func_all_layers(self): static_result[0], dy_result_value, err_msg='Result of function [{}] not equal'.format( - method.__name__)) + method.__name__ + ), + ) def test_all_layers(self): with _test_eager_guard(): @@ -2922,95 +3242,111 @@ def _get_np_data(self, shape, dtype, append_batch_size=True): elif dtype == 'float64': return np.random.random(shape).astype(dtype) elif dtype == 'int32': - return np.random.randint(self._low_data_bound, - self._high_data_bound, shape).astype(dtype) + return np.random.randint( + self._low_data_bound, self._high_data_bound, shape + ).astype(dtype) elif dtype == 'int64': - return np.random.randint(self._low_data_bound, - self._high_data_bound, shape).astype(dtype) - - def _get_data(self, - name, - shape, - dtype, - set_feed_dict=True, - append_batch_size=True): + return np.random.randint( + self._low_data_bound, self._high_data_bound, shape + ).astype(dtype) + + def _get_data( + self, name, shape, dtype, set_feed_dict=True, append_batch_size=True + ): if base.enabled(): - return base.to_variable(value=self._get_np_data( - shape, dtype, append_batch_size), - name=name, - zero_copy=False) + return base.to_variable( + value=self._get_np_data(shape, dtype, append_batch_size), + name=name, + zero_copy=False, + ) else: if set_feed_dict: self._feed_dict[name] = self._get_np_data( - shape, dtype, append_batch_size) - return layers.data(name=name, - shape=shape, - dtype=dtype, - append_batch_size=append_batch_size) + shape, dtype, append_batch_size + ) + return layers.data( + name=name, + shape=shape, + dtype=dtype, + append_batch_size=append_batch_size, + ) def make_sampled_softmax_with_cross_entropy(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): logits = self._get_data(name='Logits', shape=[256], dtype='float32') label = self._get_data(name='Label', shape=[1], dtype='int64') num_samples = 25 output = layers.sampled_softmax_with_cross_entropy( - logits, label, num_samples) - return (output) + logits, label, num_samples + ) + return output def make_fit_a_line(self): - with program_guard(fluid.default_main_program(), - startup_program=fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), + startup_program=fluid.default_startup_program(), + ): x = self._get_data(name='x', shape=[13], dtype='float32') y_predict = layers.fc(input=x, size=1, act=None) y = self._get_data(name='y', shape=[1], dtype='float32') cost = layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) - return (avg_cost) + return avg_cost def make_recognize_digits_mlp(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): # Change g_program, so the rest layers use `g_program` images = self._get_data(name='pixel', shape=[784], dtype='float32') label = self._get_data(name='label', shape=[1], dtype='int64') hidden1 = layers.fc(input=images, size=128, act='relu') hidden2 = layers.fc(input=hidden1, size=64, act='relu') - predict = layers.fc(input=[hidden2, hidden1], - size=10, - act='softmax', - param_attr=["sftmax.w1", "sftmax.w2"]) + predict = layers.fc( + input=[hidden2, hidden1], + size=10, + act='softmax', + param_attr=["sftmax.w1", "sftmax.w2"], + ) cost = layers.cross_entropy(input=predict, label=label) avg_cost = paddle.mean(cost) - return (avg_cost) + return avg_cost def make_conv2d_transpose(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): img = self._get_data(name='pixel', shape=[3, 2, 2], dtype='float32') - return layers.conv2d_transpose(input=img, - num_filters=10, - output_size=28) + return layers.conv2d_transpose( + input=img, num_filters=10, output_size=28 + ) def make_recognize_digits_conv(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - images = self._get_data(name='pixel', - shape=[1, 28, 28], - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + images = self._get_data( + name='pixel', shape=[1, 28, 28], dtype='float32' + ) label = self._get_data(name='label', shape=[1], dtype='int64') - conv_pool_1 = nets.simple_img_conv_pool(input=images, - filter_size=5, - num_filters=2, - pool_size=2, - pool_stride=2, - act="relu") - conv_pool_2 = nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=4, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = nets.simple_img_conv_pool( + input=images, + filter_size=5, + num_filters=2, + pool_size=2, + pool_stride=2, + act="relu", + ) + conv_pool_2 = nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=4, + pool_size=2, + pool_stride=2, + act="relu", + ) predict = layers.fc(input=conv_pool_2, size=10, act="softmax") cost = layers.cross_entropy(input=predict, label=label) @@ -3018,185 +3354,212 @@ def make_recognize_digits_conv(self): return avg_cost def make_word_embedding(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): dict_size = 10000 embed_size = 32 first_word = self._get_data(name='firstw', shape=[1], dtype='int64') - second_word = self._get_data(name='secondw', - shape=[1], - dtype='int64') + second_word = self._get_data( + name='secondw', shape=[1], dtype='int64' + ) third_word = self._get_data(name='thirdw', shape=[1], dtype='int64') forth_word = self._get_data(name='forthw', shape=[1], dtype='int64') next_word = self._get_data(name='nextw', shape=[1], dtype='int64') - embed_first = layers.embedding(input=first_word, - size=[dict_size, embed_size], - dtype='float32', - param_attr='shared_w') - embed_second = layers.embedding(input=second_word, - size=[dict_size, embed_size], - dtype='float32', - param_attr='shared_w') - - embed_third = layers.embedding(input=third_word, - size=[dict_size, embed_size], - dtype='float32', - param_attr='shared_w') - embed_forth = layers.embedding(input=forth_word, - size=[dict_size, embed_size], - dtype='float32', - param_attr='shared_w') + embed_first = layers.embedding( + input=first_word, + size=[dict_size, embed_size], + dtype='float32', + param_attr='shared_w', + ) + embed_second = layers.embedding( + input=second_word, + size=[dict_size, embed_size], + dtype='float32', + param_attr='shared_w', + ) + + embed_third = layers.embedding( + input=third_word, + size=[dict_size, embed_size], + dtype='float32', + param_attr='shared_w', + ) + embed_forth = layers.embedding( + input=forth_word, + size=[dict_size, embed_size], + dtype='float32', + param_attr='shared_w', + ) concat_embed = layers.concat( input=[embed_first, embed_second, embed_third, embed_forth], - axis=1) + axis=1, + ) hidden1 = layers.fc(input=concat_embed, size=256, act='sigmoid') - predict_word = layers.fc(input=hidden1, - size=dict_size, - act='softmax') + predict_word = layers.fc( + input=hidden1, size=dict_size, act='softmax' + ) cost = layers.cross_entropy(input=predict_word, label=next_word) avg_cost = paddle.mean(cost) - return (avg_cost) + return avg_cost def make_sigmoid_cross_entropy(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): dat = self._get_data(name='data', shape=[10], dtype='float32') lbl = self._get_data(name='label', shape=[10], dtype='float32') ignore_index = -1 - return (layers.sigmoid_cross_entropy_with_logits( - x=dat, label=lbl, ignore_index=ignore_index)) + return layers.sigmoid_cross_entropy_with_logits( + x=dat, label=lbl, ignore_index=ignore_index + ) def make_hsigmoid(self): self._force_to_use_cpu = True with fluid.framework._dygraph_place_guard(place=fluid.CPUPlace()): x = self._get_data(name='x', shape=[2], dtype='float32') y = self._get_data(name='y', shape=[2], dtype='int64') - return (layers.hsigmoid(input=x, label=y, num_classes=2)) + return layers.hsigmoid(input=x, label=y, num_classes=2) # test hsigmod with custom tree structure program2 = Program() with program_guard(program2): x2 = self._get_data(name='x2', shape=[4, 8], dtype='float32') y2 = self._get_data(name='y2', shape=[4], dtype='int64') - path_table = self._get_data(name='path_table', - shape=[4, 6], - dtype='int64') - path_code = self._get_data(name='path_code', - shape=[4, 6], - dtype='int64') - return (layers.hsigmoid(input=x2, - label=y2, - num_classes=6, - path_table=path_table, - path_code=path_code, - is_custom=True)) + path_table = self._get_data( + name='path_table', shape=[4, 6], dtype='int64' + ) + path_code = self._get_data( + name='path_code', shape=[4, 6], dtype='int64' + ) + return layers.hsigmoid( + input=x2, + label=y2, + num_classes=6, + path_table=path_table, + path_code=path_code, + is_custom=True, + ) def make_pool2d(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 224, 224], dtype='float32') - return (layers.pool2d(x, - pool_size=[5, 3], - pool_stride=[1, 2], - pool_padding=(2, 1))) + return layers.pool2d( + x, pool_size=[5, 3], pool_stride=[1, 2], pool_padding=(2, 1) + ) def make_pool2d_infershape(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): theta = self._get_data("theta", shape=[2, 3], dtype='float32') x = fluid.layers.affine_grid(theta, out_shape=[2, 3, 244, 244]) - return (layers.pool2d(x, - pool_size=[5, 3], - pool_stride=[1, 2], - pool_padding=(2, 1))) + return layers.pool2d( + x, pool_size=[5, 3], pool_stride=[1, 2], pool_padding=(2, 1) + ) def make_pool3d(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x = self._get_data(name='x', - shape=[3, 244, 244, 244], - dtype='float32') - return (layers.pool3d(x, - pool_size=[5, 3, 2], - pool_stride=[1, 2, 3], - pool_padding=(2, 1, 1))) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x = self._get_data( + name='x', shape=[3, 244, 244, 244], dtype='float32' + ) + return layers.pool3d( + x, + pool_size=[5, 3, 2], + pool_stride=[1, 2, 3], + pool_padding=(2, 1, 1), + ) def make_adaptive_pool2d(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 224, 224], dtype='float32') - return (layers.adaptive_pool2d(x, [3, 3], pool_type='avg')) + return layers.adaptive_pool2d(x, [3, 3], pool_type='avg') pool, mask = layers.adaptive_pool2d(x, [3, 3], require_index=True) - return (pool) - return (mask) - return (layers.adaptive_pool2d(x, 3, pool_type='avg')) + return pool + return mask + return layers.adaptive_pool2d(x, 3, pool_type='avg') pool, mask = layers.adaptive_pool2d(x, 3, require_index=True) - return (pool) - return (mask) + return pool + return mask def make_adaptive_pool3d(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x = self._get_data(name='x', - shape=[3, 244, 224, 224], - dtype='float32') - return (layers.adaptive_pool3d(x, [3, 3, 3], pool_type='avg')) - pool, mask = layers.adaptive_pool3d(x, [3, 3, 3], - require_index=True) - return (pool) - return (mask) - return (layers.adaptive_pool3d(x, 3, pool_type='avg')) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x = self._get_data( + name='x', shape=[3, 244, 224, 224], dtype='float32' + ) + return layers.adaptive_pool3d(x, [3, 3, 3], pool_type='avg') + pool, mask = layers.adaptive_pool3d( + x, [3, 3, 3], require_index=True + ) + return pool + return mask + return layers.adaptive_pool3d(x, 3, pool_type='avg') pool, mask = layers.adaptive_pool3d(x, 3, require_index=True) - return (pool) - return (mask) + return pool + return mask def make_lstm_unit(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x_t_data = self._get_data(name='x_t_data', - shape=[10, 10], - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x_t_data = self._get_data( + name='x_t_data', shape=[10, 10], dtype='float32' + ) x_t = layers.fc(input=x_t_data, size=10) - prev_hidden_data = self._get_data(name='prev_hidden_data', - shape=[10, 30], - dtype='float32') + prev_hidden_data = self._get_data( + name='prev_hidden_data', shape=[10, 30], dtype='float32' + ) prev_hidden = layers.fc(input=prev_hidden_data, size=30) - prev_cell_data = self._get_data(name='prev_cell', - shape=[10, 30], - dtype='float32') + prev_cell_data = self._get_data( + name='prev_cell', shape=[10, 30], dtype='float32' + ) prev_cell = layers.fc(input=prev_cell_data, size=30) - return (layers.lstm_unit(x_t=x_t, - hidden_t_prev=prev_hidden, - cell_t_prev=prev_cell)) + return layers.lstm_unit( + x_t=x_t, hidden_t_prev=prev_hidden, cell_t_prev=prev_cell + ) def make_softmax(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): data = self._get_data(name='data', shape=[10], dtype='float32') hid = layers.fc(input=data, size=20) - return (layers.softmax(hid, axis=1)) + return layers.softmax(hid, axis=1) def make_space_to_depth(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - data = self._get_data(name='data', - shape=[32, 9, 6, 6], - append_batch_size=False, - dtype='float32') - return (layers.space_to_depth(data, 3)) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + data = self._get_data( + name='data', + shape=[32, 9, 6, 6], + append_batch_size=False, + dtype='float32', + ) + return layers.space_to_depth(data, 3) def make_lrn(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): data = self._get_data(name='data', shape=[6, 2, 2], dtype='float32') - return (layers.lrn(data)) + return layers.lrn(data) def make_get_places(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): get_places(device_count=1) @prog_scope() @@ -3205,9 +3568,10 @@ def make_nce(self): words = [] for i in range(window_size): words.append( - self._get_data(name='word_{0}'.format(i), - shape=[1], - dtype='int64')) + self._get_data( + name='word_{0}'.format(i), shape=[1], dtype='int64' + ) + ) dict_size = 10000 label_word = int(window_size // 2) + 1 @@ -3217,38 +3581,45 @@ def make_nce(self): if i == label_word: continue - emb = layers.embedding(input=words[i], - size=[dict_size, 32], - param_attr='emb.w', - is_sparse=True) + emb = layers.embedding( + input=words[i], + size=[dict_size, 32], + param_attr='emb.w', + is_sparse=True, + ) embs.append(emb) embs = layers.concat(input=embs, axis=1) - loss = layers.nce(input=embs, - label=words[label_word], - num_total_classes=dict_size, - param_attr='nce.w', - bias_attr='nce.b') + loss = layers.nce( + input=embs, + label=words[label_word], + num_total_classes=dict_size, + param_attr='nce.w', + bias_attr='nce.b', + ) avg_loss = paddle.mean(loss) - return (avg_loss) + return avg_loss def make_multiplex(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x1 = self._get_data(name='x1', shape=[4], dtype='float32') x2 = self._get_data(name='x2', shape=[4], dtype='float32') index = self._get_data(name='index', shape=[1], dtype='int32') out = layers.multiplex(inputs=[x1, x2], index=index) - return (out) + return out def make_softmax_with_cross_entropy(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[16], dtype='float32') y = self._get_data(name='label', shape=[1], dtype='int64') loss, softmax = layers.softmax_with_cross_entropy( - x, y, return_softmax=True) + x, y, return_softmax=True + ) self.assertIsNotNone(loss) self.assertIsNotNone(softmax) @@ -3267,39 +3638,41 @@ def make_softmax_with_cross_entropy(self): self.assertIsNotNone(loss2) self.assertIsNotNone(loss3) self.assertIsNotNone(loss4) - return (loss4) + return loss4 def make_smooth_l1(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[4], dtype='float32') y = self._get_data(name='label', shape=[4], dtype='float32') loss = layers.smooth_l1(x, y) - return (loss) + return loss def make_scatter(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x = self._get_data(name='x', - shape=[3, 3], - append_batch_size=False, - dtype='float32') - idx = self._get_data(name='idx', - shape=[2], - append_batch_size=False, - dtype='int32') - updates = self._get_data(name='updates', - shape=[2, 3], - append_batch_size=False, - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x = self._get_data( + name='x', shape=[3, 3], append_batch_size=False, dtype='float32' + ) + idx = self._get_data( + name='idx', shape=[2], append_batch_size=False, dtype='int32' + ) + updates = self._get_data( + name='updates', + shape=[2, 3], + append_batch_size=False, + dtype='float32', + ) out = layers.scatter(input=x, index=idx, updates=updates) - return (out) + return out def make_one_hot(self): with fluid.framework._dygraph_place_guard(place=fluid.CPUPlace()): label = self._get_data(name="label", shape=[1], dtype="int32") one_hot_label = layers.one_hot(input=label, depth=10) - return (one_hot_label) + return one_hot_label def make_label_smooth(self): # TODO(minqiyang): support gpu ut @@ -3307,321 +3680,370 @@ def make_label_smooth(self): with fluid.framework._dygraph_place_guard(place=fluid.CPUPlace()): label = self._get_data(name="label", shape=[1], dtype="int32") one_hot_label = layers.one_hot(input=label, depth=10) - smooth_label = layers.label_smooth(label=one_hot_label, - epsilon=0.1, - dtype="int32") - return (smooth_label) + smooth_label = layers.label_smooth( + label=one_hot_label, epsilon=0.1, dtype="int32" + ) + return smooth_label def make_topk(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): data = self._get_data(name="label", shape=[200], dtype="float32") values, indices = layers.topk(data, k=5) - return (values) - return (indices) + return values + return indices def make_resize_bilinear(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 9, 6], dtype="float32") output = layers.resize_bilinear(x, out_shape=[12, 12]) - return (output) + return output def make_resize_bilinear_by_scale(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 9, 6], dtype="float32") output = layers.resize_bilinear(x, scale=1.5) - return (output) + return output def make_resize_nearest(self): try: - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x1', shape=[3, 9, 6], dtype="float32") output = layers.resize_nearest(x, out_shape=[12, 12]) except ValueError: pass try: - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x = self._get_data(name='x2', - shape=[3, 9, 6, 7], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x = self._get_data( + name='x2', shape=[3, 9, 6, 7], dtype="float32" + ) output = layers.resize_nearest(x, out_shape=[12, 12, 12]) except ValueError: pass - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 9, 6], dtype="float32") output = layers.resize_nearest(x, out_shape=[12, 12]) - return (output) + return output def make_resize_nearest_by_scale(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x1', shape=[3, 9, 6], dtype="float32") output = layers.resize_nearest(x, scale=1.8) - return (output) + return output def make_resize_trilinear(self): try: - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x2', shape=[3, 9, 6], dtype="float32") output = layers.resize_trilinear(x, out_shape=[12, 12, 12]) except ValueError: pass try: - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x = self._get_data(name='x', - shape=[3, 9, 6, 7], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x = self._get_data( + name='x', shape=[3, 9, 6, 7], dtype="float32" + ) output = layers.resize_trilinear(x, out_shape=[12, 12]) except ValueError: pass - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 9, 6, 7], dtype="float32") output = layers.resize_trilinear(x, out_shape=[12, 12, 12]) - return (output) + return output def make_resize_trilinear_by_scale(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 9, 6, 7], dtype="float32") output = layers.resize_trilinear(x, scale=2.1) - return (output) + return output def make_polygon_box_transform(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[8, 4, 4], dtype="float32") output = layers.polygon_box_transform(input=x) - return (output) + return output def make_l2_normalize(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[8, 7, 10], dtype="float32") output = layers.l2_normalize(x, axis=1) return output def make_crop(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 5], dtype="float32") y = self._get_data(name='y', shape=[2, 3], dtype="float32") output = layers.crop(x, shape=y) - return (output) + return output def make_mean_iou(self): with fluid.framework._dygraph_place_guard(place=fluid.CPUPlace()): x = self._get_data(name='x', shape=[16], dtype='int32') y = self._get_data(name='label', shape=[16], dtype='int32') iou = layers.mean_iou(x, y, self._high_data_bound) - return (iou) + return iou def make_argsort(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): data = self._get_data(name='x', shape=[2, 3, 3], dtype="float32") out, ids = layers.argsort(input=data, axis=1) - return (out) - return (ids) + return out + return ids def make_rank_loss(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - label = self._get_data(name='label', - append_batch_size=False, - shape=[16, 1], - dtype="float32") - left = self._get_data(name='left', - append_batch_size=False, - shape=[16, 1], - dtype="float32") - right = self._get_data(name='right', - append_batch_size=False, - shape=[16, 1], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + label = self._get_data( + name='label', + append_batch_size=False, + shape=[16, 1], + dtype="float32", + ) + left = self._get_data( + name='left', + append_batch_size=False, + shape=[16, 1], + dtype="float32", + ) + right = self._get_data( + name='right', + append_batch_size=False, + shape=[16, 1], + dtype="float32", + ) out = layers.rank_loss(label, left, right, name="rank_loss") - return (out) + return out def make_shape(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[3, 100, 100], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[3, 100, 100], dtype="float32" + ) out = layers.shape(input) - return (out) + return out def make_pad2d(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[3, 100, 100], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[3, 100, 100], dtype="float32" + ) paddings = layers.fill_constant(shape=[4], dtype='int32', value=1) - out = layers.pad2d(input, - paddings=[1, 2, 3, 4], - mode='reflect', - data_format='NCHW', - name="shape") - out_1 = layers.pad2d(input, - paddings=paddings, - mode='reflect', - data_format='NCHW', - name="shape") - return (out) - return (out_1) + out = layers.pad2d( + input, + paddings=[1, 2, 3, 4], + mode='reflect', + data_format='NCHW', + name="shape", + ) + out_1 = layers.pad2d( + input, + paddings=paddings, + mode='reflect', + data_format='NCHW', + name="shape", + ) + return out + return out_1 def make_prelu(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[5, 200, 100, 100], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[5, 200, 100, 100], dtype="float32" + ) mode = 'channel' - out = layers.prelu(input, - mode, - param_attr=ParamAttr(initializer=Constant(1.0)), - name='prelu') - return (out) + out = layers.prelu( + input, + mode, + param_attr=ParamAttr(initializer=Constant(1.0)), + name='prelu', + ) + return out def make_soft_relu(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.soft_relu(input, threshold=30.0, name='soft_relu') - return (out) + return out def make_sigmoid(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.sigmoid(input, name='sigmoid') - return (out) + return out def make_exp(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.exp(input, name='exp') - return (out) + return out def make_tanh(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.tanh(input, name='tanh') - return (out) + return out def make_tanh_shrink(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.tanh_shrink(input, name='tanh_shrink') - return (out) + return out def make_sqrt(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.sqrt(input, name='sqrt') - return (out) + return out def make_abs(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.abs(input, name='abs') - return (out) + return out def make_ceil(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.ceil(input, name='ceil') - return (out) + return out def make_floor(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.floor(input, name='floor') - return (out) + return out def make_cos(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.cos(input, name='cos') - return (out) + return out def make_sin(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.sin(input, name='sin') - return (out) + return out def make_round(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.round(input, name='round') - return (out) + return out def make_reciprocal(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.reciprocal(input, name='reciprocal') - return (out) + return out def make_square(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.square(input, name='square') - return (out) + return out def make_softplus(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.softplus(input, name='softplus') - return (out) + return out def make_softsign(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.softsign(input, name='softsign') - return (out) + return out def make_mish(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.mish(input, name='mish') - return (out) + return out def make_cross_entropy(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="x", shape=[30, 10], dtype="float32") label = self._get_data(name="label", shape=[30, 1], dtype="int64") mode = 'channel' out = layers.cross_entropy(x, label, False, 4) - return (out) + return out def make_bpr_loss(self): self._force_to_use_cpu = True @@ -3629,173 +4051,196 @@ def make_bpr_loss(self): x = self._get_data(name="x", shape=[30, 10], dtype="float32") label = self._get_data(name="label", shape=[30, 1], dtype="int64") out = layers.bpr_loss(x, label) - return (out) + return out def make_expand(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="input", shape=[10], dtype='int32') out = layers.expand(x, [1, 2]) return out def make_uniform_random_batch_size_like(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[13, 11], - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[13, 11], dtype='float32' + ) out = layers.uniform_random_batch_size_like(input, [-1, 11]) - return (out) + return out def make_gaussian_random(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): out = layers.gaussian_random(shape=[20, 30]) - return (out) + return out def make_sampling_id(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x = self._get_data(name="X", - shape=[13, 11], - dtype='float32', - append_batch_size=False) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x = self._get_data( + name="X", + shape=[13, 11], + dtype='float32', + append_batch_size=False, + ) out = layers.sampling_id(x) - return (out) + return out def make_gaussian_random_batch_size_like(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[13, 11], - dtype='float32') - - out = layers.gaussian_random_batch_size_like(input, - shape=[-1, 11], - mean=1.0, - std=2.0) - return (out) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[13, 11], dtype='float32' + ) + + out = layers.gaussian_random_batch_size_like( + input, shape=[-1, 11], mean=1.0, std=2.0 + ) + return out def make_sum(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[13, 11], - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[13, 11], dtype='float32' + ) out = layers.sum(input) - return (out) + return out def make_slice(self): starts = [1, 0, 2] ends = [3, 3, 4] axes = [0, 1, 2] - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[3, 4, 5, 6], - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[3, 4, 5, 6], dtype='float32' + ) out = layers.slice(input, axes=axes, starts=starts, ends=ends) return out def make_scale_variable(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", - shape=[3, 4, 5, 6], - dtype='float32') - scale_var = self._get_data(name="scale", - shape=[1], - dtype='float32', - append_batch_size=False) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = self._get_data( + name="input", shape=[3, 4, 5, 6], dtype='float32' + ) + scale_var = self._get_data( + name="scale", + shape=[1], + dtype='float32', + append_batch_size=False, + ) out = layers.scale(input, scale=scale_var) return out def make_softshrink(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.softshrink(input, alpha=0.3) - return (out) + return out def make_iou_similarity(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="x", shape=[4], dtype="float32") y = self._get_data(name="y", shape=[4], dtype="float32") out = layers.iou_similarity(x, y, name='iou_similarity') - return (out) + return out def make_grid_sampler(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name='x', shape=[3, 5, 7], dtype='float32') grid = self._get_data(name='grid', shape=[5, 7, 2], dtype='float32') out = layers.grid_sampler(x, grid) - return (out) + return out def make_bilinear_tensor_product_layer(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): data = self._get_data(name='data', shape=[4], dtype="float32") theta = self._get_data(name="theta", shape=[5], dtype="float32") out = layers.bilinear_tensor_product(data, theta, 6) - return (out) + return out def make_batch_norm(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - data = self._get_data(name='data', - shape=[32, 128, 128], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + data = self._get_data( + name='data', shape=[32, 128, 128], dtype="float32" + ) out = layers.batch_norm(data) - return (out) + return out def make_batch_norm_momentum_variable(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - data = self._get_data(name='data', - shape=[32, 128, 128], - dtype="float32") - momentum = self._get_data(name='momentum', - shape=[1], - dtype='float32', - append_batch_size=False) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + data = self._get_data( + name='data', shape=[32, 128, 128], dtype="float32" + ) + momentum = self._get_data( + name='momentum', + shape=[1], + dtype='float32', + append_batch_size=False, + ) out = layers.batch_norm(data, momentum=momentum) - return (out) + return out def make_inplace_abn(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - data = self._get_data(name='data', - shape=[32, 128, 128], - dtype="float32") + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + data = self._get_data( + name='data', shape=[32, 128, 128], dtype="float32" + ) out = layers.inplace_abn(data, act='leaky_relu', act_alpha=0.2) - return (out) + return out def make_inplace_abn_momentum_variable(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - data = self._get_data(name='data', - shape=[32, 128, 128], - dtype="float32") - momentum = self._get_data(name='momentum', - shape=[1], - dtype='float32', - append_batch_size=False) - out = layers.inplace_abn(data, - momentum=momentum, - act='elu', - act_alpha=2.0) - return (out) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + data = self._get_data( + name='data', shape=[32, 128, 128], dtype="float32" + ) + momentum = self._get_data( + name='momentum', + shape=[1], + dtype='float32', + append_batch_size=False, + ) + out = layers.inplace_abn( + data, momentum=momentum, act='elu', act_alpha=2.0 + ) + return out def make_range(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): layers.range(0, 10, 2, 'int32') layers.range(0.1, 10.0, 0.2, 'float32') layers.range(0.1, 10.0, 0.2, 'float64') @@ -3806,87 +4251,101 @@ def make_range(self): return y def make_spectral_norm(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - weight = self._get_data(name='weight', - shape=[2, 3, 32, 32], - dtype="float32", - append_batch_size=False) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + weight = self._get_data( + name='weight', + shape=[2, 3, 32, 32], + dtype="float32", + append_batch_size=False, + ) out = layers.spectral_norm(weight, dim=1, power_iters=1) - return (out) + return out def make_kldiv_loss(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - x = self._get_data(name='x', - shape=[32, 128, 128], - dtype="float32", - append_batch_size=False) - target = self._get_data(name='target', - shape=[32, 128, 128], - dtype="float32", - append_batch_size=False) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + x = self._get_data( + name='x', + shape=[32, 128, 128], + dtype="float32", + append_batch_size=False, + ) + target = self._get_data( + name='target', + shape=[32, 128, 128], + dtype="float32", + append_batch_size=False, + ) loss = layers.kldiv_loss(x=x, target=target, reduction='batchmean') - return (loss) + return loss def make_temporal_shift(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="X", shape=[16, 4, 4], dtype="float32") out = layers.temporal_shift(x, seg_num=2, shift_ratio=0.2) - return (out) + return out def make_shuffle_channel(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="X", shape=[16, 4, 4], dtype="float32") out = layers.shuffle_channel(x, group=4) - return (out) + return out def make_fsp_matrix(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="X", shape=[16, 4, 4], dtype="float32") y = self._get_data(name="Y", shape=[8, 4, 4], dtype="float32") out = layers.fsp_matrix(x, y) - return (out) + return out def make_pixel_shuffle(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="X", shape=[9, 4, 4], dtype="float32") out = layers.pixel_shuffle(x, upscale_factor=3) - return (out) + return out def make_mse_loss(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="X", shape=[1], dtype="float32") y = self._get_data(name="Y", shape=[1], dtype="float32") out = layers.mse_loss(input=x, label=y) - return (out) + return out def make_square_error_cost(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): x = self._get_data(name="X", shape=[1], dtype="float32") y = self._get_data(name="Y", shape=[1], dtype="float32") out = layers.square_error_cost(input=x, label=y) - return (out) + return out def test_dynamic_lstmp(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): hidden_dim, proj_dim = 16, 8 - seq_data = layers.data(name='seq_data', - shape=[10, 10], - dtype='float32', - lod_level=1) + seq_data = layers.data( + name='seq_data', shape=[10, 10], dtype='float32', lod_level=1 + ) fc_out = layers.fc(input=seq_data, size=4 * hidden_dim) self.assertIsNotNone( - layers.dynamic_lstmp(input=fc_out, - size=4 * hidden_dim, - proj_size=proj_dim)) + layers.dynamic_lstmp( + input=fc_out, size=4 * hidden_dim, proj_size=proj_dim + ) + ) def test_linear_chain_crf(self): with self.static_graph(): @@ -3894,63 +4353,71 @@ def test_linear_chain_crf(self): feature = layers.data(name='feature', shape=[784], dtype='float32') label = layers.data(name='label', shape=[1], dtype='int64') emission = layers.fc(input=feature, size=10) - crf = layers.linear_chain_crf(input=emission, - label=label, - param_attr=ParamAttr(name="crfw")) - crf_decode = layers.crf_decoding(input=emission, - param_attr=ParamAttr(name="crfw")) + crf = layers.linear_chain_crf( + input=emission, label=label, param_attr=ParamAttr(name="crfw") + ) + crf_decode = layers.crf_decoding( + input=emission, param_attr=ParamAttr(name="crfw") + ) self.assertFalse(crf is None) self.assertFalse(crf_decode is None) - return layers.chunk_eval(input=crf_decode, - label=label, - chunk_scheme="IOB", - num_chunk_types=(label_dict_len - 1) // 2) + return layers.chunk_eval( + input=crf_decode, + label=label, + chunk_scheme="IOB", + num_chunk_types=(label_dict_len - 1) // 2, + ) def test_linear_chain_crf_padding(self): with self.static_graph(): label_dict_len, max_len = 10, 20 - feature = layers.data(name='feature', - shape=[max_len, 784], - dtype='float32') + feature = layers.data( + name='feature', shape=[max_len, 784], dtype='float32' + ) label = layers.data(name='label', shape=[max_len], dtype='int64') length = layers.data(name='length', shape=[1], dtype='int64') emission = layers.fc(input=feature, size=10, num_flatten_dims=2) - crf = layers.linear_chain_crf(input=emission, - label=label, - length=length, - param_attr=ParamAttr(name="crfw")) - crf_decode = layers.crf_decoding(input=emission, - length=length, - param_attr=ParamAttr(name="crfw")) + crf = layers.linear_chain_crf( + input=emission, + label=label, + length=length, + param_attr=ParamAttr(name="crfw"), + ) + crf_decode = layers.crf_decoding( + input=emission, length=length, param_attr=ParamAttr(name="crfw") + ) self.assertFalse(crf is None) self.assertFalse(crf_decode is None) - return layers.chunk_eval(input=crf_decode, - label=label, - seq_length=length, - chunk_scheme="IOB", - num_chunk_types=(label_dict_len - 1) // 2) + return layers.chunk_eval( + input=crf_decode, + label=label, + seq_length=length, + chunk_scheme="IOB", + num_chunk_types=(label_dict_len - 1) // 2, + ) def test_im2sequence(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name='x', shape=[3, 128, 128], dtype='float32') y = layers.data(name='y', shape=[], dtype='float32') - output = layers.im2sequence(input=x, - input_image_size=y, - stride=[1, 1], - filter_size=[2, 2], - out_stride=[1, 1]) - return (output) + output = layers.im2sequence( + input=x, + input_image_size=y, + stride=[1, 1], + filter_size=[2, 2], + out_stride=[1, 1], + ) + return output def test_lod_reset(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): # case 1 x = layers.data(name='x', shape=[10], dtype='float32') - y = layers.data(name='y', - shape=[10, 20], - dtype='float32', - lod_level=2) + y = layers.data( + name='y', shape=[10, 20], dtype='float32', lod_level=2 + ) z = layers.lod_reset(x=x, y=y) self.assertTrue(z.lod_level == 2) # case 2 @@ -3982,153 +4449,152 @@ def test_stridedslice(self): strides = [1, 1, 1] with self.static_graph(): x = layers.data(name="x", shape=[245, 30, 30], dtype="float32") - out = layers.strided_slice(x, - axes=axes, - starts=starts, - ends=ends, - strides=strides) + out = layers.strided_slice( + x, axes=axes, starts=starts, ends=ends, strides=strides + ) return out def test_fill_constant_batch_size_like(self): with self.static_graph(): - like = fluid.layers.fill_constant(shape=[1, 200], - value=10, - dtype='int64') - out = layers.fill_constant_batch_size_like(input=like, - shape=[2, 3300], - value=1315454564656, - dtype='int64') + like = fluid.layers.fill_constant( + shape=[1, 200], value=10, dtype='int64' + ) + out = layers.fill_constant_batch_size_like( + input=like, shape=[2, 3300], value=1315454564656, dtype='int64' + ) return out def test_psroi_pool(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name="x", shape=[245, 30, 30], dtype="float32") - rois = layers.data(name="rois", - shape=[4], - dtype="float32", - lod_level=1) + rois = layers.data( + name="rois", shape=[4], dtype="float32", lod_level=1 + ) output = layers.psroi_pool(x, rois, 5, 0.25, 7, 7) - return (output) + return output def test_sequence_expand(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name='x', shape=[10], dtype='float32') - y = layers.data(name='y', - shape=[10, 20], - dtype='float32', - lod_level=2) - return (layers.sequence_expand(x=x, y=y, ref_level=1)) + y = layers.data( + name='y', shape=[10, 20], dtype='float32', lod_level=2 + ) + return layers.sequence_expand(x=x, y=y, ref_level=1) def test_sequence_reshape(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name='x', shape=[8], dtype='float32', lod_level=1) out = layers.sequence_reshape(input=x, new_dim=16) - return (out) + return out def test_sequence_unpad(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name='x', shape=[10, 5], dtype='float32') length = layers.data(name='length', shape=[], dtype='int64') - return (layers.sequence_unpad(x=x, length=length)) + return layers.sequence_unpad(x=x, length=length) def test_sequence_softmax(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): - seq_data = layers.data(name='seq_data', - shape=[10, 10], - dtype='float32', - lod_level=1) + seq_data = layers.data( + name='seq_data', shape=[10, 10], dtype='float32', lod_level=1 + ) seq = layers.fc(input=seq_data, size=20) - return (layers.sequence_softmax(seq)) + return layers.sequence_softmax(seq) def test_sequence_unsqueeze(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name='x', shape=[8, 2], dtype='float32') out = layers.unsqueeze(input=x, axes=[1]) - return (out) + return out def test_sequence_scatter(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): - x = layers.data(name='x', - shape=[3, 6], - append_batch_size=False, - dtype='float32') - idx = layers.data(name='idx', - shape=[12, 1], - append_batch_size=False, - dtype='int32', - lod_level=1) - updates = layers.data(name='updates', - shape=[12, 1], - append_batch_size=False, - dtype='float32', - lod_level=1) + x = layers.data( + name='x', shape=[3, 6], append_batch_size=False, dtype='float32' + ) + idx = layers.data( + name='idx', + shape=[12, 1], + append_batch_size=False, + dtype='int32', + lod_level=1, + ) + updates = layers.data( + name='updates', + shape=[12, 1], + append_batch_size=False, + dtype='float32', + lod_level=1, + ) out = layers.sequence_scatter(input=x, index=idx, updates=updates) - return (out) + return out def test_sequence_slice(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): import numpy as np - seqs = layers.data(name='x', - shape=[10, 5], - dtype='float32', - lod_level=1) + + seqs = layers.data( + name='x', shape=[10, 5], dtype='float32', lod_level=1 + ) offset = layers.assign(input=np.array([[0, 1]]).astype('int32')) length = layers.assign(input=np.array([[2, 1]]).astype('int32')) - out = layers.sequence_slice(input=seqs, - offset=offset, - length=length) - return (out) + out = layers.sequence_slice( + input=seqs, offset=offset, length=length + ) + return out def test_filter_by_instag(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): - x1 = layers.data(name='Ins', - shape=[32, 1], - dtype='float32', - lod_level=0) - x2 = layers.data(name='Ins_tag', - shape=[32, 1], - dtype='int64', - lod_level=0, - stop_gradient=True) - x3 = layers.create_global_var(shape=[1, 1], - value=20, - dtype='int64', - persistable=True, - force_cpu=True, - name='Filter_tag') + x1 = layers.data( + name='Ins', shape=[32, 1], dtype='float32', lod_level=0 + ) + x2 = layers.data( + name='Ins_tag', + shape=[32, 1], + dtype='int64', + lod_level=0, + stop_gradient=True, + ) + x3 = layers.create_global_var( + shape=[1, 1], + value=20, + dtype='int64', + persistable=True, + force_cpu=True, + name='Filter_tag', + ) out1, out2 = layers.filter_by_instag(x1, x2, x3, is_lod=True) def test_shuffle_batch(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): - x = layers.data(name='X', - shape=[4, 50], - dtype='float32', - lod_level=0) + x = layers.data( + name='X', shape=[4, 50], dtype='float32', lod_level=0 + ) out1 = fluid.contrib.layers.shuffle_batch(x) default_main_program().random_seed = 1000 out2 = fluid.contrib.layers.shuffle_batch(x) self.assertIsNotNone(out1) self.assertIsNotNone(out2) - return (out1) + return out1 def test_partial_sum(self): with self.static_graph(): x = fluid.data(name="x", shape=[None, 3], dtype="float32") y = fluid.data(name="y", shape=[None, 3], dtype="float32") - sum = fluid.contrib.layers.partial_sum([x, y], - start_index=0, - length=2) - return (sum) + sum = fluid.contrib.layers.partial_sum( + [x, y], start_index=0, length=2 + ) + return sum def test_batch_fc(self): with self.static_graph(): @@ -4139,21 +4605,24 @@ def test_batch_fc(self): param_attr=fluid.ParamAttr( learning_rate=1.0, name="w_0", - initializer=fluid.initializer.Xavier(uniform=False)), + initializer=fluid.initializer.Xavier(uniform=False), + ), bias_size=[16, 10], bias_attr=fluid.ParamAttr( learning_rate=1.0, name="b_0", - initializer=fluid.initializer.Xavier(uniform=False)), - act="relu") - return (out) + initializer=fluid.initializer.Xavier(uniform=False), + ), + act="relu", + ) + return out def test_rank_attention(self): with self.static_graph(): input = fluid.data(name="input", shape=[None, 2], dtype="float32") - rank_offset = fluid.data(name="rank_offset", - shape=[None, 7], - dtype="int32") + rank_offset = fluid.data( + name="rank_offset", shape=[None, 7], dtype="int32" + ) out = fluid.contrib.layers.rank_attention( input=input, rank_offset=rank_offset, @@ -4161,9 +4630,11 @@ def test_rank_attention(self): rank_param_attr=fluid.ParamAttr( learning_rate=1.0, name="ubm_rank_param.w_0", - initializer=fluid.initializer.Xavier(uniform=False)), - max_rank=3) - return (out) + initializer=fluid.initializer.Xavier(uniform=False), + ), + max_rank=3, + ) + return out def test_roi_pool(self): x_np = np.random.rand(2, 3, 8, 8).astype('float32') @@ -4175,35 +4646,27 @@ def test_roi_pool(self): rois = layers.data(name="rois", shape=[4], dtype="float32") rois_num = fluid.data(name="rois_num", shape=[None], dtype="int32") output = layers.roi_pool(x, rois, 4, 4, 0.5, rois_num=rois_num) - static_res = self.get_static_graph_result(feed={ - 'x': x_np, - 'rois': rois_np, - 'rois_num': rois_num_np - }, - fetch_list=[output])[0] + static_res = self.get_static_graph_result( + feed={'x': x_np, 'rois': rois_np, 'rois_num': rois_num_np}, + fetch_list=[output], + )[0] with self.dynamic_graph(): with _test_eager_guard(): x_dy = base.to_variable(x_np) rois_dy = base.to_variable(rois_np) rois_num_dy = base.to_variable(rois_num_np) - dy_eager_res = layers.roi_pool(x_dy, - rois_dy, - 4, - 4, - 0.5, - rois_num=rois_num_dy) + dy_eager_res = layers.roi_pool( + x_dy, rois_dy, 4, 4, 0.5, rois_num=rois_num_dy + ) dy_eager_res_value = dy_eager_res[0].numpy() x_dy = base.to_variable(x_np) rois_dy = base.to_variable(rois_np) rois_num_dy = base.to_variable(rois_num_np) - dy_res = layers.roi_pool(x_dy, - rois_dy, - 4, - 4, - 0.5, - rois_num=rois_num_dy) + dy_res = layers.roi_pool( + x_dy, rois_dy, 4, 4, 0.5, rois_num=rois_num_dy + ) dy_res_value = dy_res[0].numpy() np.testing.assert_array_equal(static_res, dy_res_value) np.testing.assert_array_equal(static_res, dy_eager_res_value) @@ -4224,37 +4687,27 @@ def test_roi_align(self): rois = layers.data(name="rois", shape=[4], dtype="float32") rois_num = fluid.data(name="rois_num", shape=[None], dtype="int32") output = layers.roi_align(x, rois, 4, 4, 0.5, 2, rois_num=rois_num) - static_res = self.get_static_graph_result(feed={ - 'x': x_np, - 'rois': rois_np, - 'rois_num': rois_num_np - }, - fetch_list=[output])[0] + static_res = self.get_static_graph_result( + feed={'x': x_np, 'rois': rois_np, 'rois_num': rois_num_np}, + fetch_list=[output], + )[0] with self.dynamic_graph(): with _test_eager_guard(): x_dy = base.to_variable(x_np) rois_dy = base.to_variable(rois_np) rois_num_dy = base.to_variable(rois_num_np) - dy_eager_res = layers.roi_align(x_dy, - rois_dy, - 4, - 4, - 0.5, - 2, - rois_num=rois_num_dy) + dy_eager_res = layers.roi_align( + x_dy, rois_dy, 4, 4, 0.5, 2, rois_num=rois_num_dy + ) dy_eager_res_value = dy_eager_res.numpy() x_dy = base.to_variable(x_np) rois_dy = base.to_variable(rois_np) rois_num_dy = base.to_variable(rois_num_np) - dy_res = layers.roi_align(x_dy, - rois_dy, - 4, - 4, - 0.5, - 2, - rois_num=rois_num_dy) + dy_res = layers.roi_align( + x_dy, rois_dy, 4, 4, 0.5, 2, rois_num=rois_num_dy + ) dy_res_value = dy_res.numpy() np.testing.assert_array_equal(static_res, dy_eager_res_value) np.testing.assert_array_equal(static_res, dy_res_value) @@ -4266,18 +4719,16 @@ def test_dice_loss(self): label_np = np.random.randint(0, num_classes, [2, 3, 1], dtype=np.int64) with self.static_graph(): - input_ = layers.data(name="input", - shape=[None, 3, num_classes], - dtype="float32") - label_ = layers.data(name="label", - shape=[None, 3, 1], - dtype="int64") + input_ = layers.data( + name="input", shape=[None, 3, num_classes], dtype="float32" + ) + label_ = layers.data( + name="label", shape=[None, 3, 1], dtype="int64" + ) output = layers.dice_loss(input_, label_, eps) - static_res = self.get_static_graph_result(feed={ - 'input': input_np, - 'label': label_np - }, - fetch_list=[output])[0] + static_res = self.get_static_graph_result( + feed={'input': input_np, 'label': label_np}, fetch_list=[output] + )[0] with self.dynamic_graph(): with _test_eager_guard(): @@ -4297,46 +4748,47 @@ def test_roi_perspective_transform(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name="x", shape=[256, 30, 30], dtype="float32") - rois = layers.data(name="rois", - shape=[8], - dtype="float32", - lod_level=1) + rois = layers.data( + name="rois", shape=[8], dtype="float32", lod_level=1 + ) output = layers.roi_perspective_transform(x, rois, 7, 7, 0.6) - return (output) + return output def test_row_conv(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): x = layers.data(name='x', shape=[16], dtype='float32', lod_level=1) out = layers.row_conv(input=x, future_context_size=2) - return (out) + return out def test_simple_conv2d(self): # TODO(minqiyang): dygraph do not support layers with param now with self.static_graph(): - images = layers.data(name='pixel', - shape=[3, 48, 48], - dtype='float32') - return layers.conv2d(input=images, - num_filters=3, - filter_size=[4, 4]) + images = layers.data( + name='pixel', shape=[3, 48, 48], dtype='float32' + ) + return layers.conv2d( + input=images, num_filters=3, filter_size=[4, 4] + ) def test_squeeze(self): # TODO(minqiyang): dygraph do not support layers with param now with self.static_graph(): x = layers.data(name='x', shape=[1, 1, 4], dtype='float32') out = layers.squeeze(input=x, axes=[2]) - return (out) + return out def test_flatten(self): # TODO(minqiyang): dygraph do not support op without kernel now with self.static_graph(): - x = layers.data(name='x', - append_batch_size=False, - shape=[4, 4, 3], - dtype="float32") + x = layers.data( + name='x', + append_batch_size=False, + shape=[4, 4, 3], + dtype="float32", + ) out = layers.flatten(x, axis=1, name="flatten") - return (out) + return out def test_linspace(self): program = Program() @@ -4347,211 +4799,270 @@ def test_linspace(self): def test_deformable_conv(self): with self.static_graph(): - input = layers.data(name='input', - append_batch_size=False, - shape=[2, 3, 32, 32], - dtype="float32") - offset = layers.data(name='offset', - append_batch_size=False, - shape=[2, 18, 32, 32], - dtype="float32") - mask = layers.data(name='mask', - append_batch_size=False, - shape=[2, 9, 32, 32], - dtype="float32") - out = layers.deformable_conv(input=input, - offset=offset, - mask=mask, - num_filters=2, - filter_size=3, - padding=1) - return (out) + input = layers.data( + name='input', + append_batch_size=False, + shape=[2, 3, 32, 32], + dtype="float32", + ) + offset = layers.data( + name='offset', + append_batch_size=False, + shape=[2, 18, 32, 32], + dtype="float32", + ) + mask = layers.data( + name='mask', + append_batch_size=False, + shape=[2, 9, 32, 32], + dtype="float32", + ) + out = layers.deformable_conv( + input=input, + offset=offset, + mask=mask, + num_filters=2, + filter_size=3, + padding=1, + ) + return out def test_deformable_conv2(self): with self.static_graph(): - input = fluid.data(name='input', - shape=[None, 3, None, None], - dtype="float32") - offset = fluid.data(name='offset', - shape=[None, 18, None, None], - dtype="float32") - mask = fluid.data(name='mask', - shape=[None, 9, None, None], - dtype="float32") - out = layers.deformable_conv(input=input, - offset=offset, - mask=mask, - num_filters=2, - filter_size=3, - padding=1) - return (out) + input = fluid.data( + name='input', shape=[None, 3, None, None], dtype="float32" + ) + offset = fluid.data( + name='offset', shape=[None, 18, None, None], dtype="float32" + ) + mask = fluid.data( + name='mask', shape=[None, 9, None, None], dtype="float32" + ) + out = layers.deformable_conv( + input=input, + offset=offset, + mask=mask, + num_filters=2, + filter_size=3, + padding=1, + ) + return out def test_unfold(self): with self.static_graph(): x = layers.data(name='x', shape=[3, 20, 20], dtype='float32') out = layers.unfold(x, [3, 3], 1, 1, 1) - return (out) + return out def test_partial_concat(self): with self.static_graph(): x = fluid.data(name="x", shape=[None, 3], dtype="float32") y = fluid.data(name="y", shape=[None, 3], dtype="float32") - concat1 = fluid.contrib.layers.partial_concat([x, y], - start_index=0, - length=2) - concat2 = fluid.contrib.layers.partial_concat(x, - start_index=0, - length=-1) + concat1 = fluid.contrib.layers.partial_concat( + [x, y], start_index=0, length=2 + ) + concat2 = fluid.contrib.layers.partial_concat( + x, start_index=0, length=-1 + ) return concat1, concat2 def test_deform_roi_pooling(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = layers.data(name='input', - shape=[2, 3, 32, 32], - dtype='float32', - append_batch_size=False) - rois = layers.data(name="rois", - shape=[4], - dtype='float32', - lod_level=1) - trans = layers.data(name="trans", - shape=[2, 3, 32, 32], - dtype='float32', - append_batch_size=False) - out = layers.deformable_roi_pooling(input=input, - rois=rois, - trans=trans, - no_trans=False, - spatial_scale=1.0, - group_size=(1, 1), - pooled_height=8, - pooled_width=8, - part_size=(8, 8), - sample_per_part=4, - trans_std=0.1) - return (out) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = layers.data( + name='input', + shape=[2, 3, 32, 32], + dtype='float32', + append_batch_size=False, + ) + rois = layers.data( + name="rois", shape=[4], dtype='float32', lod_level=1 + ) + trans = layers.data( + name="trans", + shape=[2, 3, 32, 32], + dtype='float32', + append_batch_size=False, + ) + out = layers.deformable_roi_pooling( + input=input, + rois=rois, + trans=trans, + no_trans=False, + spatial_scale=1.0, + group_size=(1, 1), + pooled_height=8, + pooled_width=8, + part_size=(8, 8), + sample_per_part=4, + trans_std=0.1, + ) + return out def test_deformable_conv_v1(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = layers.data(name='input', - append_batch_size=False, - shape=[2, 3, 32, 32], - dtype="float32") - offset = layers.data(name='offset', - append_batch_size=False, - shape=[2, 18, 32, 32], - dtype="float32") - out = layers.deformable_conv(input=input, - offset=offset, - mask=None, - num_filters=2, - filter_size=3, - padding=1, - modulated=False) - return (out) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = layers.data( + name='input', + append_batch_size=False, + shape=[2, 3, 32, 32], + dtype="float32", + ) + offset = layers.data( + name='offset', + append_batch_size=False, + shape=[2, 18, 32, 32], + dtype="float32", + ) + out = layers.deformable_conv( + input=input, + offset=offset, + mask=None, + num_filters=2, + filter_size=3, + padding=1, + modulated=False, + ) + return out def test_retinanet_target_assign(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - bbox_pred = layers.data(name='bbox_pred', - shape=[1, 100, 4], - append_batch_size=False, - dtype='float32') - cls_logits = layers.data(name='cls_logits', - shape=[1, 100, 10], - append_batch_size=False, - dtype='float32') - anchor_box = layers.data(name='anchor_box', - shape=[100, 4], - append_batch_size=False, - dtype='float32') - anchor_var = layers.data(name='anchor_var', - shape=[100, 4], - append_batch_size=False, - dtype='float32') - gt_boxes = layers.data(name='gt_boxes', - shape=[10, 4], - append_batch_size=False, - dtype='float32') - gt_labels = layers.data(name='gt_labels', - shape=[10, 1], - append_batch_size=False, - dtype='int32') - is_crowd = layers.data(name='is_crowd', - shape=[1], - append_batch_size=False, - dtype='int32') - im_info = layers.data(name='im_info', - shape=[1, 3], - append_batch_size=False, - dtype='float32') - return (layers.retinanet_target_assign(bbox_pred, cls_logits, - anchor_box, anchor_var, - gt_boxes, gt_labels, - is_crowd, im_info, 10)) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + bbox_pred = layers.data( + name='bbox_pred', + shape=[1, 100, 4], + append_batch_size=False, + dtype='float32', + ) + cls_logits = layers.data( + name='cls_logits', + shape=[1, 100, 10], + append_batch_size=False, + dtype='float32', + ) + anchor_box = layers.data( + name='anchor_box', + shape=[100, 4], + append_batch_size=False, + dtype='float32', + ) + anchor_var = layers.data( + name='anchor_var', + shape=[100, 4], + append_batch_size=False, + dtype='float32', + ) + gt_boxes = layers.data( + name='gt_boxes', + shape=[10, 4], + append_batch_size=False, + dtype='float32', + ) + gt_labels = layers.data( + name='gt_labels', + shape=[10, 1], + append_batch_size=False, + dtype='int32', + ) + is_crowd = layers.data( + name='is_crowd', + shape=[1], + append_batch_size=False, + dtype='int32', + ) + im_info = layers.data( + name='im_info', + shape=[1, 3], + append_batch_size=False, + dtype='float32', + ) + return layers.retinanet_target_assign( + bbox_pred, + cls_logits, + anchor_box, + anchor_var, + gt_boxes, + gt_labels, + is_crowd, + im_info, + 10, + ) def test_sigmoid_focal_loss(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = layers.data(name='data', - shape=[10, 80], - append_batch_size=False, - dtype='float32') - label = layers.data(name='label', - shape=[10, 1], - append_batch_size=False, - dtype='int32') - fg_num = layers.data(name='fg_num', - shape=[1], - append_batch_size=False, - dtype='int32') - out = fluid.layers.sigmoid_focal_loss(x=input, - label=label, - fg_num=fg_num, - gamma=2., - alpha=0.25) - return (out) + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = layers.data( + name='data', + shape=[10, 80], + append_batch_size=False, + dtype='float32', + ) + label = layers.data( + name='label', + shape=[10, 1], + append_batch_size=False, + dtype='int32', + ) + fg_num = layers.data( + name='fg_num', shape=[1], append_batch_size=False, dtype='int32' + ) + out = fluid.layers.sigmoid_focal_loss( + x=input, label=label, fg_num=fg_num, gamma=2.0, alpha=0.25 + ) + return out def test_addmm(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = layers.data(name='input_data', - shape=[3, 3], - append_batch_size=False, - dtype='float32') - x = layers.data(name='x', - shape=[3, 2], - append_batch_size=False, - dtype='float32') - y = layers.data(name='y', - shape=[2, 3], - append_batch_size=False, - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + input = layers.data( + name='input_data', + shape=[3, 3], + append_batch_size=False, + dtype='float32', + ) + x = layers.data( + name='x', shape=[3, 2], append_batch_size=False, dtype='float32' + ) + y = layers.data( + name='y', shape=[2, 3], append_batch_size=False, dtype='float32' + ) out = paddle.addmm(input=input, x=x, y=y) - return (out) + return out def test_retinanet_detection_output(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - bboxes = layers.data(name='bboxes', - shape=[1, 21, 4], - append_batch_size=False, - dtype='float32') - scores = layers.data(name='scores', - shape=[1, 21, 10], - append_batch_size=False, - dtype='float32') - anchors = layers.data(name='anchors', - shape=[21, 4], - append_batch_size=False, - dtype='float32') - im_info = layers.data(name="im_info", - shape=[1, 3], - append_batch_size=False, - dtype='float32') + with program_guard( + fluid.default_main_program(), fluid.default_startup_program() + ): + bboxes = layers.data( + name='bboxes', + shape=[1, 21, 4], + append_batch_size=False, + dtype='float32', + ) + scores = layers.data( + name='scores', + shape=[1, 21, 10], + append_batch_size=False, + dtype='float32', + ) + anchors = layers.data( + name='anchors', + shape=[21, 4], + append_batch_size=False, + dtype='float32', + ) + im_info = layers.data( + name="im_info", + shape=[1, 3], + append_batch_size=False, + dtype='float32', + ) nmsed_outs = layers.retinanet_detection_output( bboxes=[bboxes, bboxes], scores=[scores, scores], @@ -4561,38 +5072,39 @@ def test_retinanet_detection_output(self): nms_top_k=1000, keep_top_k=100, nms_threshold=0.3, - nms_eta=1.) - return (nmsed_outs) + nms_eta=1.0, + ) + return nmsed_outs def test_warpctc_with_padding(self): # TODO(minqiyang): dygraph do not support lod now with self.static_graph(): - input_length = layers.data(name='logits_length', - shape=[11], - dtype='int64') - label_length = layers.data(name='labels_length', - shape=[12], - dtype='int64') + input_length = layers.data( + name='logits_length', shape=[11], dtype='int64' + ) + label_length = layers.data( + name='labels_length', shape=[12], dtype='int64' + ) label = layers.data(name='label', shape=[12, 1], dtype='int32') - predict = layers.data(name='predict', - shape=[4, 4, 8], - dtype='float32') - output = layers.warpctc(input=predict, - label=label, - input_length=input_length, - label_length=label_length) - return (output) + predict = layers.data( + name='predict', shape=[4, 4, 8], dtype='float32' + ) + output = layers.warpctc( + input=predict, + label=label, + input_length=input_length, + label_length=label_length, + ) + return output def test_edit_distance(self): with self.static_graph(): - predict = layers.data(name='predict', - shape=[-1, 1], - dtype='int64', - lod_level=1) - label = layers.data(name='label', - shape=[-1, 1], - dtype='int64', - lod_level=1) + predict = layers.data( + name='predict', shape=[-1, 1], dtype='int64', lod_level=1 + ) + label = layers.data( + name='label', shape=[-1, 1], dtype='int64', lod_level=1 + ) evaluator = fluid.evaluator.EditDistance(predict, label) return evaluator.metrics @@ -4600,15 +5112,15 @@ def test_basic_gru(self): input_size = 128 hidden_size = 256 with self.static_graph(): - input = fluid.data(name="input", - shape=[None, None, input_size], - dtype='float32') - pre_hidden = fluid.data(name="pre_hidden", - shape=[None, hidden_size], - dtype='float32') - sequence_length = fluid.data(name="sequence_length", - shape=[None], - dtype='int32') + input = fluid.data( + name="input", shape=[None, None, input_size], dtype='float32' + ) + pre_hidden = fluid.data( + name="pre_hidden", shape=[None, hidden_size], dtype='float32' + ) + sequence_length = fluid.data( + name="sequence_length", shape=[None], dtype='int32' + ) for bidirectional in [True, False]: for batch_first in [True, False]: @@ -4620,30 +5132,35 @@ def test_basic_gru(self): sequence_length=sequence_length, dropout_prob=0.5, bidirectional=bidirectional, - batch_first=batch_first) + batch_first=batch_first, + ) class TestMetricsDetectionMap(unittest.TestCase): - def test_detection_map(self): program = fluid.Program() with program_guard(program): - detect_res = fluid.layers.data(name='detect_res', - shape=[10, 6], - append_batch_size=False, - dtype='float32') - label = fluid.layers.data(name='label', - shape=[10, 1], - append_batch_size=False, - dtype='float32') - box = fluid.layers.data(name='bbox', - shape=[10, 4], - append_batch_size=False, - dtype='float32') - map_eval = fluid.metrics.DetectionMAP(detect_res, - label, - box, - class_num=21) + detect_res = fluid.layers.data( + name='detect_res', + shape=[10, 6], + append_batch_size=False, + dtype='float32', + ) + label = fluid.layers.data( + name='label', + shape=[10, 1], + append_batch_size=False, + dtype='float32', + ) + box = fluid.layers.data( + name='bbox', + shape=[10, 4], + append_batch_size=False, + dtype='float32', + ) + map_eval = fluid.metrics.DetectionMAP( + detect_res, label, box, class_num=21 + ) cur_map, accm_map = map_eval.get_map_var() self.assertIsNotNone(cur_map) self.assertIsNotNone(accm_map) @@ -4651,11 +5168,11 @@ def test_detection_map(self): class ExampleNet(paddle.nn.Layer): - def __init__(self): super(ExampleNet, self).__init__() self.weight = self.create_parameter( - shape=[1, 1], attr=paddle.ParamAttr(trainable=False)) + shape=[1, 1], attr=paddle.ParamAttr(trainable=False) + ) def forward(self): # only for test parameter trainable attr @@ -4663,7 +5180,6 @@ def forward(self): class TestLayerParameterTrainableSet(unittest.TestCase): - def test_layer_parameter_set(self): with fluid.dygraph.guard(): net = ExampleNet() @@ -4671,7 +5187,6 @@ def test_layer_parameter_set(self): class TestLayerTrainingAttribute(unittest.TestCase): - def test_set_train_eval_in_dynamic_mode(self): with fluid.dygraph.guard(): net = paddle.nn.Dropout() @@ -4689,7 +5204,6 @@ def test_set_train_eval_in_static_mode(self): class MyLayer(paddle.nn.Layer): - def __init__(self): super(MyLayer, self).__init__() self._linear = paddle.nn.Linear(1, 1) @@ -4702,7 +5216,6 @@ def forward(self, input): class MySuperLayer(paddle.nn.Layer): - def __init__(self): super(MySuperLayer, self).__init__() self._mylayer = MyLayer() @@ -4713,7 +5226,6 @@ def forward(self, input): class TestSubLayerCount(unittest.TestCase): - def test_sublayer(self): with fluid.dygraph.guard(): mySuperlayer = MySuperLayer() diff --git a/python/paddle/fluid/tests/unittests/test_layout_autotune.py b/python/paddle/fluid/tests/unittests/test_layout_autotune.py index b7af0464ba9f9..95618edcc38fe 100644 --- a/python/paddle/fluid/tests/unittests/test_layout_autotune.py +++ b/python/paddle/fluid/tests/unittests/test_layout_autotune.py @@ -23,7 +23,6 @@ class SimpleNet(paddle.nn.Layer): - def __init__(self, data_format="NCHW", class_num=2): super(SimpleNet, self).__init__() self.conv = paddle.nn.Conv2D(3, 8, (3, 3)) @@ -44,7 +43,6 @@ def forward(self, image): class LayoutAutoTune(unittest.TestCase): - def test_config(self): paddle.fluid.core.enable_layout_autotune() if self.use_autoune(): @@ -59,9 +57,8 @@ def setUp(self): def use_autoune(self): if paddle.is_compiled_with_cuda(): paddle.incubate.autotune.set_config( - config={"layout": { - "enable": True - }}) + config={"layout": {"enable": True}} + ) return paddle.fluid.core.use_layout_autotune() else: config = {"layout": {"enable": False}} @@ -75,11 +72,12 @@ def use_autoune(self): def train(self, data_format): model = SimpleNet(data_format="NCHW", class_num=2) data = paddle.rand([1, 3, 16, 16]) - if (data_format == "NHWC"): + if data_format == "NHWC": data = paddle.rand([1, 16, 16, 3]) label_data = paddle.randint(0, 1, shape=[1, 1], dtype="int64") - optimizer = paddle.optimizer.SGD(learning_rate=0.0001, - parameters=model.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.0001, parameters=model.parameters() + ) scaler = paddle.amp.GradScaler() for i in range(2): with paddle.amp.auto_cast(level="O2"): @@ -101,8 +99,9 @@ def test_transpose_op_transposer(self): conv = paddle.nn.Conv2D(3, 8, (3, 3)) data = paddle.rand([1, 3, 16, 14]) label_data = paddle.randint(0, 1, shape=[1, 1], dtype="int64") - optimizer = paddle.optimizer.SGD(learning_rate=0.0001, - parameters=conv.parameters()) + optimizer = paddle.optimizer.SGD( + learning_rate=0.0001, parameters=conv.parameters() + ) scaler = paddle.amp.GradScaler() with paddle.amp.auto_cast(level="O2"): conv_out = conv(data) @@ -170,7 +169,6 @@ def test_concat_op_no_transposer(self): class TestAutoTuneAPI(unittest.TestCase): - def test_set_config_warnings(self): with warnings.catch_warnings(record=True) as w: config = {"layout": {"enable": 1}} diff --git a/python/paddle/fluid/tests/unittests/test_lazy_init.py b/python/paddle/fluid/tests/unittests/test_lazy_init.py index 79b3429b10e0b..914fc14126152 100644 --- a/python/paddle/fluid/tests/unittests/test_lazy_init.py +++ b/python/paddle/fluid/tests/unittests/test_lazy_init.py @@ -17,12 +17,18 @@ import numpy as np from paddle import LazyGuard from paddle.nn import Linear, Layer -from paddle.nn.initializer import Constant, Normal, TruncatedNormal, Uniform, XavierNormal, XavierUniform +from paddle.nn.initializer import ( + Constant, + Normal, + TruncatedNormal, + Uniform, + XavierNormal, + XavierUniform, +) from paddle.fluid import unique_name class TestInitializerBase(unittest.TestCase): - def setUp(self): self.set_initializer() self.set_param_attr() @@ -34,11 +40,13 @@ def set_initializer(self): self.b_initializer = Constant(0.3) def set_param_attr(self): - self.weight_attr = paddle.ParamAttr(name="weight", - initializer=self.w_initializer) + self.weight_attr = paddle.ParamAttr( + name="weight", initializer=self.w_initializer + ) - self.bias_attr = paddle.ParamAttr(name="bias", - initializer=self.b_initializer) + self.bias_attr = paddle.ParamAttr( + name="bias", initializer=self.b_initializer + ) def set_init_ops(self): self.init_ops = ['fill_constant', 'fill_constant'] @@ -48,29 +56,26 @@ def clear_nameset(self): def test_wrapper(self): with LazyGuard(): - fc = Linear(10, - 10, - weight_attr=self.weight_attr, - bias_attr=self.bias_attr) + fc = Linear( + 10, 10, weight_attr=self.weight_attr, bias_attr=self.bias_attr + ) program = fc._startup_program() print(program) self.check_program(program) def check_program(self, program): self.assertEqual(program.block(0).var("weight").shape, (10, 10)) - self.assertEqual(program.block(0).var("bias").shape, (10, )) + self.assertEqual(program.block(0).var("bias").shape, (10,)) ops = [op.type for op in program.block(0).ops] self.assertEqual(ops, self.init_ops) class TestDygraphLazy(TestInitializerBase): - def test_wrapper(self): with LazyGuard(): - fc = Linear(10, - 10, - weight_attr=self.weight_attr, - bias_attr=self.bias_attr) + fc = Linear( + 10, 10, weight_attr=self.weight_attr, bias_attr=self.bias_attr + ) self.check_data(fc) @@ -86,14 +91,15 @@ def check_data(self, model): out = model(x) self.assertEqual(out.shape, [2, 10]) - np.testing.assert_allclose(model.weight.numpy(), - np.ones([10, 10], dtype=np.float32) * 0.6) - np.testing.assert_allclose(model.bias.numpy(), - np.ones([10], dtype=np.float32) * 0.3) + np.testing.assert_allclose( + model.weight.numpy(), np.ones([10, 10], dtype=np.float32) * 0.6 + ) + np.testing.assert_allclose( + model.bias.numpy(), np.ones([10], dtype=np.float32) * 0.3 + ) class NestModel(Layer): - def __init__(self, base_model): super(NestModel, self).__init__() self.base_model = base_model @@ -106,13 +112,11 @@ def forward(self, x): class TestNestModelLazy(TestInitializerBase): - def test_wrapper(self): with LazyGuard(): - base_model = Linear(10, - 10, - weight_attr=self.weight_attr, - bias_attr=self.bias_attr) + base_model = Linear( + 10, 10, weight_attr=self.weight_attr, bias_attr=self.bias_attr + ) nest_model = NestModel(base_model) self.check_data(nest_model) @@ -130,16 +134,19 @@ def check_data(self, model): out = model(x) self.assertEqual(out.shape, [2, 10]) - np.testing.assert_allclose(model.base_model.weight.numpy(), - np.ones([10, 10], dtype=np.float32) * 0.6) - np.testing.assert_allclose(model.base_model.bias.numpy(), - np.ones([10], dtype=np.float32) * 0.3) + np.testing.assert_allclose( + model.base_model.weight.numpy(), + np.ones([10, 10], dtype=np.float32) * 0.6, + ) + np.testing.assert_allclose( + model.base_model.bias.numpy(), np.ones([10], dtype=np.float32) * 0.3 + ) def check_program(self, model): # verify nest_model startup_program whole_program = model._startup_program() self.assertEqual(whole_program.block(0).var("weight").shape, (10, 10)) - self.assertEqual(whole_program.block(0).var("bias").shape, (10, )) + self.assertEqual(whole_program.block(0).var("bias").shape, (10,)) ops = [op.type for op in whole_program.block(0).ops] init_ops = self.init_ops + ['uniform_random', 'fill_constant'] self.assertEqual(ops, init_ops) @@ -147,13 +154,12 @@ def check_program(self, model): # verify base_model startup_program sub_program = model.base_model._startup_program() self.assertEqual(sub_program.block(0).var("weight").shape, (10, 10)) - self.assertEqual(sub_program.block(0).var("bias").shape, (10, )) + self.assertEqual(sub_program.block(0).var("bias").shape, (10,)) ops = [op.type for op in sub_program.block(0).ops] self.assertEqual(ops, self.init_ops) class TestUniform(TestInitializerBase): - def set_initializer(self): self.w_initializer = Uniform() self.b_initializer = Uniform() @@ -163,7 +169,6 @@ def set_init_ops(self): class TestNormal(TestInitializerBase): - def set_initializer(self): self.w_initializer = Normal() self.b_initializer = Normal() @@ -173,26 +178,24 @@ def set_init_ops(self): class TestTruncatedNormal(TestInitializerBase): - def set_initializer(self): self.w_initializer = TruncatedNormal() self.b_initializer = TruncatedNormal() def set_init_ops(self): self.init_ops = [ - 'truncated_gaussian_random', 'truncated_gaussian_random' + 'truncated_gaussian_random', + 'truncated_gaussian_random', ] class TestXavierNormal(TestNormal): - def set_initializer(self): self.w_initializer = XavierNormal() self.b_initializer = XavierNormal() class TestXavierUniform(TestUniform): - def set_initializer(self): self.w_initializer = XavierUniform() self.b_initializer = XavierUniform() diff --git a/python/paddle/fluid/tests/unittests/test_lbfgs.py b/python/paddle/fluid/tests/unittests/test_lbfgs.py index 36b01d05e7865..21d96d17c2ad9 100644 --- a/python/paddle/fluid/tests/unittests/test_lbfgs.py +++ b/python/paddle/fluid/tests/unittests/test_lbfgs.py @@ -44,37 +44,35 @@ def test_static_graph_H0(func, x0, H0, dtype='float32'): startup = paddle.static.Program() with paddle.static.program_guard(main, startup): X = paddle.static.data(name='x', shape=[x0.shape[0]], dtype=dtype) - H = paddle.static.data(name='h', - shape=[H0.shape[0], H0.shape[1]], - dtype=dtype) - Y = minimize_lbfgs(func, - X, - initial_inverse_hessian_estimate=H, - dtype=dtype) + H = paddle.static.data( + name='h', shape=[H0.shape[0], H0.shape[1]], dtype=dtype + ) + Y = minimize_lbfgs( + func, X, initial_inverse_hessian_estimate=H, dtype=dtype + ) exe = paddle.static.Executor() exe.run(startup) return exe.run(main, feed={'x': x0, 'h': H0}, fetch_list=[Y]) -def test_dynamic_graph(func, - x0, - H0=None, - line_search_fn='strong_wolfe', - dtype='float32'): +def test_dynamic_graph( + func, x0, H0=None, line_search_fn='strong_wolfe', dtype='float32' +): paddle.disable_static() x0 = paddle.to_tensor(x0) if H0 is not None: H0 = paddle.to_tensor(H0) - return minimize_lbfgs(func, - x0, - initial_inverse_hessian_estimate=H0, - line_search_fn=line_search_fn, - dtype=dtype) + return minimize_lbfgs( + func, + x0, + initial_inverse_hessian_estimate=H0, + line_search_fn=line_search_fn, + dtype=dtype, + ) class TestLbfgs(unittest.TestCase): - def test_quadratic_nd(self): for dimension in [1, 10]: minimum = np.random.random(size=[dimension]).astype('float32') @@ -84,7 +82,8 @@ def func(x): minimum_ = paddle.assign(minimum) scale_ = paddle.assign(scale) return paddle.sum( - paddle.multiply(scale_, (F.square_error_cost(x, minimum_)))) + paddle.multiply(scale_, (F.square_error_cost(x, minimum_))) + ) x0 = np.random.random(size=[dimension]).astype('float32') results = test_static_graph(func, x0) @@ -99,16 +98,17 @@ def test_inf_minima(self): def func(x): # df = 3(x - 1.01)(x - 0.99) # f = x^3 - 3x^2 + 3*1.01*0.99x - return x * x * x / 3.0 - ( - extream_point[0] + extream_point[1] - ) * x * x / 2 + extream_point[0] * extream_point[1] * x + return ( + x * x * x / 3.0 + - (extream_point[0] + extream_point[1]) * x * x / 2 + + extream_point[0] * extream_point[1] * x + ) x0 = np.array([-1.7]).astype('float32') results = test_static_graph(func, x0) self.assertFalse(results[0][0]) def test_multi_minima(self): - def func(x): # df = 12(x + 1.1)(x - 0.2)(x - 0.8) # f = 3*x^4+0.4*x^3-5.46*x^2+2.112*x @@ -131,7 +131,7 @@ def func(position): # f(x, y) = (a - x)^2 + b (y - x^2)^2 # minimum = (a, a^2) x, y = position[0], position[1] - c = (a - x)**2 + b * (y - x**2)**2 + c = (a - x) ** 2 + b * (y - x**2) ** 2 # the return cant be np array[1], or in jacobin will cause flat error return c[0] @@ -141,7 +141,6 @@ def func(position): np.testing.assert_allclose(minimum, results[2], rtol=1e-05) def test_exception(self): - def func(x): return paddle.dot(x, x) @@ -150,11 +149,9 @@ def func(x): # test dtype is not float32 or float64 x1 = np.random.random(size=[2]).astype('int32') - self.assertRaises(ValueError, - test_static_graph, - func, - x1, - dtype='int32') + self.assertRaises( + ValueError, test_static_graph, func, x1, dtype='int32' + ) # test initial_inverse_hessian_estimate is good results = test_static_graph_H0(func, x0, H0, dtype='float32') @@ -164,12 +161,9 @@ def func(x): # test initial_inverse_hessian_estimate is bad and float64 x2 = np.random.random(size=[2]).astype('float64') H1 = np.array([[1.0, 2.0], [3.0, 1.0]]).astype('float64') - self.assertRaises(ValueError, - test_static_graph_H0, - func, - x2, - H0=H1, - dtype='float64') + self.assertRaises( + ValueError, test_static_graph_H0, func, x2, H0=H1, dtype='float64' + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_lcm.py b/python/paddle/fluid/tests/unittests/test_lcm.py index 6f113c13a01a7..83f3f82d7e7a2 100644 --- a/python/paddle/fluid/tests/unittests/test_lcm.py +++ b/python/paddle/fluid/tests/unittests/test_lcm.py @@ -22,7 +22,6 @@ class TestLcmAPI(unittest.TestCase): - def setUp(self): self.x_np = 12 self.y_np = 20 @@ -37,32 +36,34 @@ def test_static_graph(self): x2 = fluid.data(name='input2', dtype='int32', shape=self.y_shape) out = paddle.lcm(x1, x2) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(fluid.default_main_program(), - feed={ - 'input1': self.x_np, - 'input2': self.y_np - }, - fetch_list=[out]) - self.assertTrue((np.array(res[0]) == np.lcm(self.x_np, - self.y_np)).all()) + res = exe.run( + fluid.default_main_program(), + feed={'input1': self.x_np, 'input2': self.y_np}, + fetch_list=[out], + ) + self.assertTrue( + (np.array(res[0]) == np.lcm(self.x_np, self.y_np)).all() + ) def test_dygraph(self): paddle.disable_static() x1 = paddle.to_tensor(self.x_np) x2 = paddle.to_tensor(self.y_np) result = paddle.lcm(x1, x2) - np.testing.assert_allclose(np.lcm(self.x_np, self.y_np), - result.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + np.lcm(self.x_np, self.y_np), result.numpy(), rtol=1e-05 + ) paddle.enable_static() class TestLcmAPI2(TestLcmAPI): - def setUp(self): self.x_np = np.arange(6).astype(np.int32) self.y_np = np.array([20]).astype(np.int32) @@ -71,7 +72,6 @@ def setUp(self): class TestLcmAPI3(TestLcmAPI): - def setUp(self): self.x_np = 0 self.y_np = 20 @@ -80,7 +80,6 @@ def setUp(self): class TestLcmAPI4(TestLcmAPI): - def setUp(self): self.x_np = 0 self.y_np = 0 @@ -89,7 +88,6 @@ def setUp(self): class TestLcmAPI5(TestLcmAPI): - def setUp(self): self.x_np = 12 self.y_np = -20 diff --git a/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py b/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py index bd73f628a0298..ffe45f3ec312e 100644 --- a/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py +++ b/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py @@ -23,45 +23,41 @@ import paddle.fluid.core as core -def exponential_decay(learning_rate, - global_step, - decay_steps, - decay_rate, - staircase=False): +def exponential_decay( + learning_rate, global_step, decay_steps, decay_rate, staircase=False +): exponent = global_step / decay_steps if staircase: exponent = math.floor(exponent) return learning_rate * decay_rate**exponent -def natural_exp_decay(learning_rate, - global_step, - decay_steps, - decay_rate, - staircase=False): +def natural_exp_decay( + learning_rate, global_step, decay_steps, decay_rate, staircase=False +): exponent = float(global_step) / float(decay_steps) if staircase: exponent = math.floor(exponent) return learning_rate * math.exp(-1 * decay_rate * exponent) -def inverse_time_decay(learning_rate, - global_step, - decay_steps, - decay_rate, - staircase=False): +def inverse_time_decay( + learning_rate, global_step, decay_steps, decay_rate, staircase=False +): temp = float(global_step) / float(decay_steps) if staircase: temp = math.floor(temp) return learning_rate / (1 + decay_rate * temp) -def polynomial_decay(learning_rate, - global_step, - decay_steps, - end_learning_rate=0.0001, - power=1.0, - cycle=False): +def polynomial_decay( + learning_rate, + global_step, + decay_steps, + end_learning_rate=0.0001, + power=1.0, + cycle=False, +): if cycle: div = math.ceil(global_step / float(decay_steps)) if div == 0: @@ -69,8 +65,9 @@ def polynomial_decay(learning_rate, decay_steps = decay_steps * div else: global_step = min(global_step, decay_steps) - return (learning_rate - end_learning_rate) * \ - ((1 - float(global_step) / float(decay_steps)) ** power) + end_learning_rate + return (learning_rate - end_learning_rate) * ( + (1 - float(global_step) / float(decay_steps)) ** power + ) + end_learning_rate def piecewise_decay(global_step, boundaries, values): @@ -83,8 +80,9 @@ def piecewise_decay(global_step, boundaries, values): def cosine_decay(global_step, learning_rate, step_each_epoch, epochs): cur_epoch = math.floor(global_step / step_each_epoch) - decayed_lr = learning_rate * 0.5 * (math.cos(cur_epoch * math.pi / epochs) + - 1) + decayed_lr = ( + learning_rate * 0.5 * (math.cos(cur_epoch * math.pi / epochs) + 1) + ) return decayed_lr @@ -119,7 +117,6 @@ def lambda_decay(global_step, learning_rate, lr_lambda): class TestLearningRateDecayDygraph(unittest.TestCase): - def test_LR_state_dict(self): with fluid.dygraph.guard(): x = np.random.uniform(-1, 1, [3, 10]).astype("float32") @@ -130,17 +127,24 @@ def test_LR_state_dict(self): learning_rate=0.1, decay_steps=10000, decay_rate=0.5, - staircase=True) + staircase=True, + ) Step_scheduler = fluid.dygraph.StepDecay(0.5, step_size=3) Reducelr_scheduler = fluid.dygraph.ReduceLROnPlateau( - learning_rate=1.0, decay_rate=0.5, patience=5, cooldown=3) - - adam1 = fluid.optimizer.Adam(learning_rate=Exponential_scheduler, - parameter_list=linear.parameters()) - adam2 = fluid.optimizer.Adam(learning_rate=Step_scheduler, - parameter_list=linear.parameters()) - adam3 = fluid.optimizer.Adam(learning_rate=Reducelr_scheduler, - parameter_list=linear.parameters()) + learning_rate=1.0, decay_rate=0.5, patience=5, cooldown=3 + ) + + adam1 = fluid.optimizer.Adam( + learning_rate=Exponential_scheduler, + parameter_list=linear.parameters(), + ) + adam2 = fluid.optimizer.Adam( + learning_rate=Step_scheduler, parameter_list=linear.parameters() + ) + adam3 = fluid.optimizer.Adam( + learning_rate=Reducelr_scheduler, + parameter_list=linear.parameters(), + ) print(adam3.state_dict()) for epoch in range(10): @@ -161,59 +165,76 @@ def test_LR_state_dict(self): learning_rate=0.1, decay_steps=10000, decay_rate=0.5, - staircase=True) + staircase=True, + ) Step_scheduler_test = fluid.dygraph.StepDecay(0.5, step_size=3) Reducelr_scheduler_test = fluid.dygraph.ReduceLROnPlateau( - learning_rate=1.0, decay_rate=0.5, patience=5, cooldown=3) + learning_rate=1.0, decay_rate=0.5, patience=5, cooldown=3 + ) fluid.dygraph.save_dygraph(adam1.state_dict(), "save_path") _, opt_state = fluid.dygraph.load_dygraph("save_path") adam_test = fluid.optimizer.Adam( learning_rate=Exponential_scheduler_test, - parameter_list=linear.parameters()) + parameter_list=linear.parameters(), + ) adam_test.set_dict(opt_state) self.assertEqual( adam_test._learning_rate.step_num, adam1._learning_rate.step_num, - "epoch_num is different before and after set_dict") + "epoch_num is different before and after set_dict", + ) fluid.dygraph.save_dygraph(adam2.state_dict(), "save_path") _, opt_state = fluid.dygraph.load_dygraph("save_path") - adam_test = fluid.optimizer.Adam(learning_rate=Step_scheduler_test, - parameter_list=linear.parameters()) + adam_test = fluid.optimizer.Adam( + learning_rate=Step_scheduler_test, + parameter_list=linear.parameters(), + ) adam_test.set_dict(opt_state) self.assertEqual( adam_test._learning_rate.epoch_num, adam2._learning_rate.epoch_num, - "epoch_num is different before and after set_dict") + "epoch_num is different before and after set_dict", + ) self.assertEqual( - adam_test._learning_rate(), adam2._learning_rate(), - "current learning rate is different before and after set_dict") + adam_test._learning_rate(), + adam2._learning_rate(), + "current learning rate is different before and after set_dict", + ) fluid.dygraph.save_dygraph(adam3.state_dict(), "save_path") _, opt_state = fluid.dygraph.load_dygraph("save_path") adam_test = fluid.optimizer.Adam( learning_rate=Reducelr_scheduler_test, - parameter_list=linear.parameters()) + parameter_list=linear.parameters(), + ) adam_test.set_dict(opt_state) self.assertEqual( adam_test._learning_rate.best_loss, adam3._learning_rate.best_loss.numpy()[0], - "best_loss is different before and after set_dict") + "best_loss is different before and after set_dict", + ) self.assertEqual( adam_test._learning_rate.cooldown_counter, adam3._learning_rate.cooldown_counter, - "cooldown_counter is different before and after set_dict") + "cooldown_counter is different before and after set_dict", + ) self.assertEqual( adam_test._learning_rate.num_bad_epochs, adam3._learning_rate.num_bad_epochs, - "num_bad_epochs is different before and after set_dict") - self.assertEqual(adam_test._learning_rate.epoch_num, - adam3._learning_rate.epoch_num, - "epoch is different before and after set_dict") + "num_bad_epochs is different before and after set_dict", + ) + self.assertEqual( + adam_test._learning_rate.epoch_num, + adam3._learning_rate.epoch_num, + "epoch is different before and after set_dict", + ) self.assertEqual( - adam_test._learning_rate(), adam3._learning_rate(), - "current learning rate is different before and after set_dict") + adam_test._learning_rate(), + adam3._learning_rate(), + "current learning rate is different before and after set_dict", + ) def test_NoamDecay(self): with fluid.dygraph.guard(): @@ -223,42 +244,47 @@ def test_NoamDecay(self): lr = fluid.layers.noam_decay(d_model, warmup_steps, learning_rate) for step in range(5): step += 1 - right_result = noam_decay(step, d_model, warmup_steps, - learning_rate) + right_result = noam_decay( + step, d_model, warmup_steps, learning_rate + ) fluid_result = lr() self.assertAlmostEqual( right_result, fluid_result[0], - msg= - 'Failed lr scheduler in step {0}, Python result is {1}, Fluid result is {2}' - .format(step, right_result, fluid_result[0])) + msg='Failed lr scheduler in step {0}, Python result is {1}, Fluid result is {2}'.format( + step, right_result, fluid_result[0] + ), + ) def test_LinearLrWarmup(self): with fluid.dygraph.guard(): - lr = fluid.layers.polynomial_decay(learning_rate=1.0, - decay_steps=10, - end_learning_rate=0.0, - power=1.0) - lr = fluid.layers.linear_lr_warmup(learning_rate=lr, - warmup_steps=2, - start_lr=0.0, - end_lr=1.0) + lr = fluid.layers.polynomial_decay( + learning_rate=1.0, + decay_steps=10, + end_learning_rate=0.0, + power=1.0, + ) + lr = fluid.layers.linear_lr_warmup( + learning_rate=lr, warmup_steps=2, start_lr=0.0, end_lr=1.0 + ) right_result = [0.5, 0.9, 0.8, 0.7, 0.6] for i in range(5): t = lr() - np.testing.assert_allclose(t.numpy()[0].item(), - right_result[i], - rtol=1e-05) + np.testing.assert_allclose( + t.numpy()[0].item(), right_result[i], rtol=1e-05 + ) with self.assertRaises(TypeError): - lr = fluid.layers.linear_lr_warmup(learning_rate="fake_lr", - warmup_steps=2, - start_lr=0.0, - end_lr=1.0) + lr = fluid.layers.linear_lr_warmup( + learning_rate="fake_lr", + warmup_steps=2, + start_lr=0.0, + end_lr=1.0, + ) def test_MultiStepDecay(self): with fluid.dygraph.guard(): @@ -267,30 +293,36 @@ def test_MultiStepDecay(self): decay_rate = 0.2 linear = fluid.dygraph.Linear(10, 10) - scheduler = fluid.dygraph.MultiStepDecay(learning_rate, milestones, - decay_rate) + scheduler = fluid.dygraph.MultiStepDecay( + learning_rate, milestones, decay_rate + ) adam = fluid.optimizer.AdamOptimizer( - learning_rate=scheduler, parameter_list=linear.parameters()) + learning_rate=scheduler, parameter_list=linear.parameters() + ) for epoch in range(10): - right_result = multi_step_decay(epoch, learning_rate, - milestones, decay_rate) + right_result = multi_step_decay( + epoch, learning_rate, milestones, decay_rate + ) fluid_result = adam.current_step_lr() scheduler.epoch() self.assertAlmostEqual( right_result, fluid_result, - msg= - 'Failed lr scheduler in epoch {0}, Python result is {1}, Fluid result is {2}' - .format(epoch, right_result, fluid_result)) + msg='Failed lr scheduler in epoch {0}, Python result is {1}, Fluid result is {2}'.format( + epoch, right_result, fluid_result + ), + ) with self.assertRaises(ValueError): - lr = fluid.dygraph.MultiStepDecay(learning_rate, [30, 50, 20], - 0.1) + lr = fluid.dygraph.MultiStepDecay( + learning_rate, [30, 50, 20], 0.1 + ) with self.assertRaises(ValueError): - lr = fluid.dygraph.MultiStepDecay(learning_rate, [20, 30, 50], - 1) + lr = fluid.dygraph.MultiStepDecay( + learning_rate, [20, 30, 50], 1 + ) with self.assertRaises(TypeError): lr = fluid.dygraph.MultiStepDecay("test", [20, 30, 50]) @@ -303,19 +335,22 @@ def test_StepDecay(self): learning_rate = 0.5 step_size = 3 decay_rate = 0.2 - scheduler = fluid.dygraph.StepDecay(learning_rate, step_size, - decay_rate) + scheduler = fluid.dygraph.StepDecay( + learning_rate, step_size, decay_rate + ) for epoch in range(10): - right_result = step_decay(epoch, learning_rate, step_size, - decay_rate) + right_result = step_decay( + epoch, learning_rate, step_size, decay_rate + ) fluid_result = scheduler().numpy()[0] scheduler.epoch() self.assertAlmostEqual( right_result, fluid_result, - msg= - 'Failed lr scheduler in epoch {0}, Python result is {1}, Fluid result is {2}' - .format(epoch, right_result, fluid_result)) + msg='Failed lr scheduler in epoch {0}, Python result is {1}, Fluid result is {2}'.format( + epoch, right_result, fluid_result + ), + ) with self.assertRaises(TypeError): lr = fluid.dygraph.StepDecay(learning_rate, "test", 0.1) @@ -330,8 +365,9 @@ def test_LambdaDecay(self): scheduler = fluid.dygraph.LambdaDecay(learning_rate, lr_lambda) linear = fluid.dygraph.nn.Linear(10, 10) - adam = fluid.optimizer.Adam(scheduler, - parameter_list=linear.parameters()) + adam = fluid.optimizer.Adam( + scheduler, parameter_list=linear.parameters() + ) for epoch in range(30): right_result = lambda_decay(epoch, learning_rate, lr_lambda) @@ -340,26 +376,28 @@ def test_LambdaDecay(self): self.assertAlmostEqual( right_result, fluid_result, - msg= - 'Failed lr scheduler in epoch {0}, Python result is {1}, Fluid result is {2}' - .format(epoch, right_result, fluid_result)) + msg='Failed lr scheduler in epoch {0}, Python result is {1}, Fluid result is {2}'.format( + epoch, right_result, fluid_result + ), + ) with self.assertRaises(TypeError): lr = fluid.dygraph.LambdaDecay(learning_rate, "test") class TestLearningRateDecay(unittest.TestCase): - def check_decay(self, python_decay_fn, fluid_decay_fn, kwargs): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) for place in places: - self.check_decay_with_place(place, python_decay_fn, fluid_decay_fn, - kwargs) + self.check_decay_with_place( + place, python_decay_fn, fluid_decay_fn, kwargs + ) - def check_decay_with_place(self, place, python_decay_fn, fluid_decay_fn, - kwargs): + def check_decay_with_place( + self, place, python_decay_fn, fluid_decay_fn, kwargs + ): main_prog = fluid.Program() startup_prog = fluid.Program() @@ -375,23 +413,27 @@ def check_decay_with_place(self, place, python_decay_fn, fluid_decay_fn, # Step of NoamDecay starts from 1. if python_decay_fn.__name__ == 'noam_decay': step += 1 - lr_val, = exe.run(main_prog, feed={}, fetch_list=[decayed_lr]) - python_decayed_lr = python_decay_fn(global_step=float(step), - **kwargs) + (lr_val,) = exe.run(main_prog, feed={}, fetch_list=[decayed_lr]) + python_decayed_lr = python_decay_fn( + global_step=float(step), **kwargs + ) self.assertAlmostEqual( python_decayed_lr, lr_val[0], - msg= - 'Failed lr scheduler is {0}, step {1}, Python result is {2}, Fluid result is {3}' - .format(python_decay_fn.__name__, str(step), - str(python_decayed_lr), str(lr_val[0]))) + msg='Failed lr scheduler is {0}, step {1}, Python result is {2}, Fluid result is {3}'.format( + python_decay_fn.__name__, + str(step), + str(python_decayed_lr), + str(lr_val[0]), + ), + ) def test_decay(self): common_kwargs_true = { "learning_rate": 1.0, "decay_steps": 5, "decay_rate": 0.5, - "staircase": True + "staircase": True, } common_kwargs_false = copy.deepcopy(common_kwargs_true) common_kwargs_false["staircase"] = False @@ -402,37 +444,47 @@ def test_decay(self): (natural_exp_decay, layers.natural_exp_decay, common_kwargs_true), (natural_exp_decay, layers.natural_exp_decay, common_kwargs_false), (inverse_time_decay, layers.inverse_time_decay, common_kwargs_true), - (inverse_time_decay, layers.inverse_time_decay, - common_kwargs_false), - (polynomial_decay, layers.polynomial_decay, { - "learning_rate": 1.0, - "decay_steps": 5, - "cycle": True - }), - (polynomial_decay, layers.polynomial_decay, { - "learning_rate": 1.0, - "decay_steps": 5, - "cycle": False - }), - (piecewise_decay, layers.piecewise_decay, { - "boundaries": [3, 6, 9], - "values": [0.1, 0.2, 0.3, 0.4] - }), - (cosine_decay, layers.cosine_decay, { - "learning_rate": 0.1, - "step_each_epoch": 100, - "epochs": 120 - }), - (noam_decay, layers.noam_decay, { - "d_model": 0.01, - "warmup_steps": 200, - "learning_rate": 2.0 - }) + ( + inverse_time_decay, + layers.inverse_time_decay, + common_kwargs_false, + ), + ( + polynomial_decay, + layers.polynomial_decay, + {"learning_rate": 1.0, "decay_steps": 5, "cycle": True}, + ), + ( + polynomial_decay, + layers.polynomial_decay, + {"learning_rate": 1.0, "decay_steps": 5, "cycle": False}, + ), + ( + piecewise_decay, + layers.piecewise_decay, + {"boundaries": [3, 6, 9], "values": [0.1, 0.2, 0.3, 0.4]}, + ), + ( + cosine_decay, + layers.cosine_decay, + {"learning_rate": 0.1, "step_each_epoch": 100, "epochs": 120}, + ), + ( + noam_decay, + layers.noam_decay, + {"d_model": 0.01, "warmup_steps": 200, "learning_rate": 2.0}, + ), ] for py_decay_fn, fluid_decay_fn, kwargs in decay_fns: - print("class=" + self.__class__.__name__ + " decay_fn=" + - py_decay_fn.__name__ + " kwargs=" + str(kwargs)) + print( + "class=" + + self.__class__.__name__ + + " decay_fn=" + + py_decay_fn.__name__ + + " kwargs=" + + str(kwargs) + ) main_program = framework.Program() startup_program = framework.Program() with framework.program_guard(main_program, startup_program): @@ -440,19 +492,20 @@ def test_decay(self): class TestLinearWamrupLearningRateDecay(unittest.TestCase): - - def check_decay_with_place(self, place, python_decay_fn, fluid_decay_fn, - kwargs): + def check_decay_with_place( + self, place, python_decay_fn, fluid_decay_fn, kwargs + ): main_prog = fluid.Program() startup_prog = fluid.Program() warmup_steps = 10 - start_lr = 0.1 / 3. + start_lr = 0.1 / 3.0 end_lr = 0.1 with fluid.program_guard(main_prog, startup_prog): - decayed_lr = layers.linear_lr_warmup(fluid_decay_fn(**kwargs), - warmup_steps, start_lr, end_lr) + decayed_lr = layers.linear_lr_warmup( + fluid_decay_fn(**kwargs), warmup_steps, start_lr, end_lr + ) place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -462,24 +515,28 @@ def check_decay_with_place(self, place, python_decay_fn, fluid_decay_fn, # Step of NoamDecay starts from 1. if fluid_decay_fn.__name__ == 'noam_decay': step += 1 - lr_val, = exe.run(main_prog, feed={}, fetch_list=[decayed_lr]) + (lr_val,) = exe.run(main_prog, feed={}, fetch_list=[decayed_lr]) if step < warmup_steps: - python_decayed_lr = linear_lr_warmup(float(step), warmup_steps, - start_lr, end_lr) + python_decayed_lr = linear_lr_warmup( + float(step), warmup_steps, start_lr, end_lr + ) else: - python_decayed_lr = python_decay_fn(global_step=float(step), - **kwargs) + python_decayed_lr = python_decay_fn( + global_step=float(step), **kwargs + ) self.assertAlmostEqual( python_decayed_lr, lr_val[0], - msg= - 'Test {0} Failed, step {1}, Python result is {2}, Fluid result is {3}' - .format(python_decay_fn.__name__, str(step), - str(python_decayed_lr), str(lr_val[0]))) + msg='Test {0} Failed, step {1}, Python result is {2}, Fluid result is {3}'.format( + python_decay_fn.__name__, + str(step), + str(python_decayed_lr), + str(lr_val[0]), + ), + ) class TestLinearWamrupLearningRateDecayWithScalarInput(unittest.TestCase): - def run_scalar_lr(self, place, lr, start_lr, end_lr): main_prog = fluid.Program() startup_prog = fluid.Program() @@ -487,27 +544,30 @@ def run_scalar_lr(self, place, lr, start_lr, end_lr): warmup_steps = 10 with fluid.program_guard(main_prog, startup_prog): - decayed_lr = layers.linear_lr_warmup(lr, warmup_steps, start_lr, - end_lr) + decayed_lr = layers.linear_lr_warmup( + lr, warmup_steps, start_lr, end_lr + ) exe = fluid.Executor(place) exe.run(startup_prog) for step in range(20): - lr_val, = exe.run(main_prog, feed={}, fetch_list=[decayed_lr]) + (lr_val,) = exe.run(main_prog, feed={}, fetch_list=[decayed_lr]) if step < warmup_steps: - expected_lr = linear_lr_warmup(float(step), warmup_steps, - start_lr, end_lr) + expected_lr = linear_lr_warmup( + float(step), warmup_steps, start_lr, end_lr + ) else: expected_lr = lr self.assertAlmostEqual( expected_lr, lr_val[0], msg='Test failed, step {0}, expected {1}, but got {2}'.format( - step, expected_lr, lr_val[0])) + step, expected_lr, lr_val[0] + ), + ) def test_scalar_lr(self): - def run_places(lr, start_lr, end_lr): places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -517,13 +577,13 @@ def run_places(lr, start_lr, end_lr): # float lr = 0.2 - start_lr = 0.1 / 3. + start_lr = 0.1 / 3.0 end_lr = 0.2 run_places(lr, start_lr, end_lr) # int end_lr - lr = 2. - start_lr = 0.1 / 3. + lr = 2.0 + start_lr = 0.1 / 3.0 end_lr = 1 run_places(lr, start_lr, end_lr) diff --git a/python/paddle/fluid/tests/unittests/test_lerp_op.py b/python/paddle/fluid/tests/unittests/test_lerp_op.py index f7f096a8d5a2f..1cae3fbaba9ac 100644 --- a/python/paddle/fluid/tests/unittests/test_lerp_op.py +++ b/python/paddle/fluid/tests/unittests/test_lerp_op.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import unittest import numpy as np from op_test import OpTest @@ -25,14 +23,13 @@ class TestLerp(OpTest): - def setUp(self): self.op_type = "lerp" self.python_api = paddle.lerp self.init_dtype() self.init_shape() - x = np.arange(1., 101.).astype(self.dtype).reshape(self.shape) - y = np.full(100, 10.).astype(self.dtype).reshape(self.shape) + x = np.arange(1.0, 101.0).astype(self.dtype).reshape(self.shape) + y = np.full(100, 10.0).astype(self.dtype).reshape(self.shape) w = np.asarray([0.5]).astype(self.dtype) self.inputs = {'X': x, 'Y': y, 'Weight': w} self.outputs = {'Out': x + w * (y - x)} @@ -51,72 +48,64 @@ def test_check_grad(self): class TestLerpWithDim2(TestLerp): - def init_shape(self): self.shape = [2, 50] class TestLerpWithDim3(TestLerp): - def init_shape(self): self.shape = [2, 2, 25] class TestLerpWithDim4(TestLerp): - def init_shape(self): self.shape = [2, 2, 5, 5] class TestLerpWithDim5(TestLerp): - def init_shape(self): self.shape = [2, 1, 2, 5, 5] class TestLerpWithDim6(TestLerp): - def init_shape(self): self.shape = [2, 1, 2, 5, 1, 5] class TestLerpBroadXY(TestLerp): - def setUp(self): self.op_type = "lerp" self.python_api = paddle.lerp self.init_dtype() self.init_shape() - x = np.arange(1., 201.).astype(self.dtype).reshape([2, 1, 2, 50]) - y = np.full(200, 10.).astype(self.dtype).reshape([2, 2, 1, 50]) + x = np.arange(1.0, 201.0).astype(self.dtype).reshape([2, 1, 2, 50]) + y = np.full(200, 10.0).astype(self.dtype).reshape([2, 2, 1, 50]) w = np.asarray([0.5]).astype(self.dtype) self.inputs = {'X': x, 'Y': y, 'Weight': w} self.outputs = {'Out': x + w * (y - x)} class TestLerpBroadWToXY(TestLerp): - def setUp(self): self.op_type = "lerp" self.python_api = paddle.lerp self.init_dtype() self.init_shape() x = np.full(600, 2.5).astype(self.dtype).reshape([50, 2, 2, 3]) - y = np.full(600, 1.).astype(self.dtype).reshape([50, 2, 2, 3]) + y = np.full(600, 1.0).astype(self.dtype).reshape([50, 2, 2, 3]) w = np.random.random([3]).astype(self.dtype) self.inputs = {'X': x, 'Y': y, 'Weight': w} self.outputs = {'Out': x + w * (y - x)} class TestLerpAPI(unittest.TestCase): - def init_dtype(self): self.dtype = 'float32' def setUp(self): self.init_dtype() - self.x = np.arange(1., 5.).astype(self.dtype) - self.y = np.full(4, 10.).astype(self.dtype) + self.x = np.arange(1.0, 5.0).astype(self.dtype) + self.y = np.full(4, 10.0).astype(self.dtype) self.w = np.asarray([0.75]).astype(self.dtype) self.res_ref = self.x + self.w * (self.y - self.x) self.place = [paddle.CPUPlace()] @@ -132,10 +121,12 @@ def run(place): y = paddle.fluid.data('y', [1, 4], dtype=self.dtype) out = paddle.lerp(x, y, 0.5) exe = paddle.static.Executor(place) - res = exe.run(feed={ - 'x': self.x.reshape([1, 4]), - 'y': self.y.reshape([1, 4]), - }) + res = exe.run( + feed={ + 'x': self.x.reshape([1, 4]), + 'y': self.y.reshape([1, 4]), + } + ) for r in res: np.testing.assert_allclose(self.res_ref, r, rtol=1e-05) @@ -143,7 +134,6 @@ def run(place): run(place) def test_dygraph_api(self): - def run(place): paddle.disable_static(place) x = paddle.to_tensor(self.x) @@ -157,7 +147,6 @@ def run(place): run(place) def test_inplace_api(self): - def run(place): paddle.disable_static(place) x = paddle.to_tensor(self.x) @@ -170,7 +159,6 @@ def run(place): run(place) def test_inplace_api_exception(self): - def run(place): paddle.disable_static(place) x = paddle.to_tensor(self.x) @@ -185,8 +173,8 @@ def run(place): def test_x_broadcast_y(self): paddle.disable_static() - x = np.arange(1., 21.).astype(self.dtype).reshape([2, 2, 5]) - y = np.full(30, 10.).astype(self.dtype).reshape([3, 2, 1, 5]) + x = np.arange(1.0, 21.0).astype(self.dtype).reshape([2, 2, 5]) + y = np.full(30, 10.0).astype(self.dtype).reshape([3, 2, 1, 5]) out = paddle.lerp(paddle.to_tensor(x), paddle.to_tensor(y), 0.5) res_ref = x + 0.5 * (y - x) np.testing.assert_allclose(res_ref, out.numpy(), rtol=1e-05) @@ -194,11 +182,12 @@ def test_x_broadcast_y(self): def test_x_y_broadcast_w(self): paddle.disable_static() - x = np.arange(11., 21.).astype(self.dtype).reshape([2, 5]) + x = np.arange(11.0, 21.0).astype(self.dtype).reshape([2, 5]) y = np.full(20, 7.5).astype(self.dtype).reshape([2, 2, 5]) w = np.full(40, 0.225).astype(self.dtype).reshape([2, 2, 2, 5]) - out = paddle.lerp(paddle.to_tensor(x), paddle.to_tensor(y), - paddle.to_tensor(w)) + out = paddle.lerp( + paddle.to_tensor(x), paddle.to_tensor(y), paddle.to_tensor(w) + ) res_ref = x + w * (y - x) np.testing.assert_allclose(res_ref, out.numpy(), rtol=1e-05) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_lgamma_op.py b/python/paddle/fluid/tests/unittests/test_lgamma_op.py index 13e420abad348..54211aef14ffa 100644 --- a/python/paddle/fluid/tests/unittests/test_lgamma_op.py +++ b/python/paddle/fluid/tests/unittests/test_lgamma_op.py @@ -23,7 +23,6 @@ class TestLgammaOp(OpTest): - def setUp(self): self.op_type = 'lgamma' self.python_api = paddle.lgamma @@ -48,19 +47,16 @@ def test_check_grad_normal(self): class TestLgammaOpFp32(TestLgammaOp): - def init_dtype_type(self): self.dtype = np.float32 def test_check_grad_normal(self): - self.check_grad(['X'], - 'Out', - numeric_grad_delta=0.005, - check_eager=True) + self.check_grad( + ['X'], 'Out', numeric_grad_delta=0.005, check_eager=True + ) class TestLgammaOpApi(unittest.TestCase): - def test_lgamma(self): paddle.disable_static() self.dtype = "float32" diff --git a/python/paddle/fluid/tests/unittests/test_limit_by_capacity_op.py b/python/paddle/fluid/tests/unittests/test_limit_by_capacity_op.py index 3e604d25657ee..02c71ea3a4147 100644 --- a/python/paddle/fluid/tests/unittests/test_limit_by_capacity_op.py +++ b/python/paddle/fluid/tests/unittests/test_limit_by_capacity_op.py @@ -43,17 +43,17 @@ def all_close(exp, out, n_worker): return np.allclose(exp.sum(0), out.sum(0)) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestLimitByCapacityInt64API(unittest.TestCase): - def init_test_case(self): - self.expert_count = np.random.randint(0, - 1000, - size=(len(self.capacity) * - self.n_worker)) - self.out = limit_by_capacity(self.expert_count, self.capacity, - self.n_worker) + self.expert_count = np.random.randint( + 0, 1000, size=(len(self.capacity) * self.n_worker) + ) + self.out = limit_by_capacity( + self.expert_count, self.capacity, self.n_worker + ) self.expert_count = self.expert_count.astype("int64") self.capacity = self.capacity.astype("int64") self.place = paddle.CUDAPlace(0) @@ -66,19 +66,23 @@ def setUp(self): def test_static_api(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - capacity = paddle.static.data('capacity', - shape=self.capacity.shape, - dtype="int64") + capacity = paddle.static.data( + 'capacity', shape=self.capacity.shape, dtype="int64" + ) expert_count_tensor = paddle.static.data( - 'ExpertCount', shape=self.expert_count.shape, dtype="int64") - out = utils._limit_by_capacity(expert_count_tensor, capacity, - self.n_worker) + 'ExpertCount', shape=self.expert_count.shape, dtype="int64" + ) + out = utils._limit_by_capacity( + expert_count_tensor, capacity, self.n_worker + ) exe = paddle.static.Executor(self.place) - res = exe.run(feed={ - 'capacity': self.capacity, - 'ExpertCount': self.expert_count, - }, - fetch_list=out) + res = exe.run( + feed={ + 'capacity': self.capacity, + 'ExpertCount': self.expert_count, + }, + fetch_list=out, + ) assert all_close(self.out, res[0], self.n_worker) @@ -86,8 +90,9 @@ def func_dygraph_api(self): paddle.disable_static(self.place) capacity = paddle.to_tensor(self.capacity) expert_count_tensor = paddle.to_tensor(self.expert_count) - out = utils._limit_by_capacity(expert_count_tensor, capacity, - self.n_worker) + out = utils._limit_by_capacity( + expert_count_tensor, capacity, self.n_worker + ) assert all_close(self.out, out.numpy(), self.n_worker) def test_dygraph_api(self): @@ -96,10 +101,10 @@ def test_dygraph_api(self): self.func_dygraph_api() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestLimitByCapacityInt64API_SmallWorker(TestLimitByCapacityInt64API): - def setUp(self): self.capacity = np.array([100, 12000, 1200, 0, 4700, 1000, 57, 200]) self.n_worker = 1 diff --git a/python/paddle/fluid/tests/unittests/test_linalg_cond.py b/python/paddle/fluid/tests/unittests/test_linalg_cond.py index 175d44cb1cefe..4724bbc6d0802 100644 --- a/python/paddle/fluid/tests/unittests/test_linalg_cond.py +++ b/python/paddle/fluid/tests/unittests/test_linalg_cond.py @@ -31,9 +31,9 @@ def test_static_assert_true(self, x_list, p_list): exe = static.Executor() result = exe.run(feed={"X": x}, fetch_list=[output]) expected_output = np.linalg.cond(x, p) - np.testing.assert_allclose(result[0], - expected_output, - rtol=5e-5) + np.testing.assert_allclose( + result[0], expected_output, rtol=5e-5 + ) def test_dygraph_assert_true(self, x_list, p_list): @@ -42,9 +42,9 @@ def test_dygraph_assert_true(self, x_list, p_list): input_tensor = paddle.to_tensor(x) output = paddle.linalg.cond(input_tensor, p) expected_output = np.linalg.cond(x, p) - np.testing.assert_allclose(output.numpy(), - expected_output, - rtol=5e-5) + np.testing.assert_allclose( + output.numpy(), expected_output, rtol=5e-5 + ) def gen_input(): @@ -81,7 +81,6 @@ def gen_empty_input(): class API_TestStaticCond(unittest.TestCase): - def test_out(self): paddle.enable_static() # test calling results of 'cond' in static mode @@ -91,7 +90,6 @@ def test_out(self): class API_TestDygraphCond(unittest.TestCase): - def func_out(self): paddle.disable_static() # test calling results of 'cond' in dynamic mode @@ -106,14 +104,13 @@ def test_out(self): class TestCondAPIError(unittest.TestCase): - def func_dygraph_api_error(self): paddle.disable_static() # test raising errors when 'cond' is called in dygraph mode p_list_error = ('fro_', '_nuc', -0.7, 0, 1.5, 3) x_list_n_n, x_list_m_n = gen_input() for p in p_list_error: - for x in (x_list_n_n + x_list_m_n): + for x in x_list_n_n + x_list_m_n: x_tensor = paddle.to_tensor(x) self.assertRaises(ValueError, paddle.linalg.cond, x_tensor, p) @@ -133,7 +130,7 @@ def test_static_api_error(self): p_list_error = ('f ro', 'fre', 'NUC', -1.6, 0, 5) x_list_n_n, x_list_m_n = gen_input() for p in p_list_error: - for x in (x_list_n_n + x_list_m_n): + for x in x_list_n_n + x_list_m_n: with static.program_guard(static.Program(), static.Program()): x_data = static.data("X", shape=x.shape, dtype=x.dtype) self.assertRaises(ValueError, paddle.linalg.cond, x_data, p) @@ -149,13 +146,13 @@ def test_static_empty_input_error(self): paddle.enable_static() x_list_n_n, x_list_m_n = gen_empty_input() - for p in (p_list_n_n + p_list_m_n): + for p in p_list_n_n + p_list_m_n: for x in x_list_n_n: with static.program_guard(static.Program(), static.Program()): x_data = static.data("X", shape=x.shape, dtype=x.dtype) self.assertRaises(ValueError, paddle.linalg.cond, x_data, p) - for p in (p_list_n_n + p_list_m_n): + for p in p_list_n_n + p_list_m_n: for x in x_list_n_n: with static.program_guard(static.Program(), static.Program()): x_data = static.data("X", shape=x.shape, dtype=x.dtype) @@ -163,7 +160,6 @@ def test_static_empty_input_error(self): class TestCondEmptyTensorInput(unittest.TestCase): - def func_dygraph_empty_tensor_input(self): paddle.disable_static() # test calling results of 'cond' when input is an empty tensor in dynamic mode diff --git a/python/paddle/fluid/tests/unittests/test_linalg_lstsq_op.py b/python/paddle/fluid/tests/unittests/test_linalg_lstsq_op.py index f4180941be235..f627057ed242d 100644 --- a/python/paddle/fluid/tests/unittests/test_linalg_lstsq_op.py +++ b/python/paddle/fluid/tests/unittests/test_linalg_lstsq_op.py @@ -20,7 +20,6 @@ class LinalgLstsqTestCase(unittest.TestCase): - def setUp(self): self.devices = ["cpu"] self.init_config() @@ -39,15 +38,17 @@ def init_config(self): def generate_input(self): self._input_data_1 = np.random.random(self._input_shape_1).astype( - self.dtype) + self.dtype + ) self._input_data_2 = np.random.random(self._input_shape_2).astype( - self.dtype) + self.dtype + ) def generate_output(self): if len(self._input_shape_1) == 2: - out = np.linalg.lstsq(self._input_data_1, - self._input_data_2, - rcond=self.rcond) + out = np.linalg.lstsq( + self._input_data_1, self._input_data_2, rcond=self.rcond + ) self._output_solution = out[0] self._output_residuals = out[1] self._output_rank = out[2] @@ -58,9 +59,11 @@ def generate_output(self): self._output_rank = [] self._output_sg_values = [] for i in range(self._input_shape_1[0]): - out = np.linalg.lstsq(self._input_data_1[i], - self._input_data_2[i], - rcond=self.rcond) + out = np.linalg.lstsq( + self._input_data_1[i], + self._input_data_2[i], + rcond=self.rcond, + ) self._output_solution.append(out[0]) self._output_residuals.append(out[1]) self._output_rank.append(out[2]) @@ -72,16 +75,15 @@ def test_eager_dygraph(self): for dev in self.devices: paddle.set_device(dev) place = paddle.CPUPlace() if dev == "cpu" else paddle.CUDAPlace(0) - x = paddle.to_tensor(self._input_data_1, - place=place, - dtype=self.dtype) - y = paddle.to_tensor(self._input_data_2, - place=place, - dtype=self.dtype) - results = paddle.linalg.lstsq(x, - y, - rcond=self.rcond, - driver=self.driver) + x = paddle.to_tensor( + self._input_data_1, place=place, dtype=self.dtype + ) + y = paddle.to_tensor( + self._input_data_2, place=place, dtype=self.dtype + ) + results = paddle.linalg.lstsq( + x, y, rcond=self.rcond, driver=self.driver + ) self._result_solution = results[0].numpy() self._result_residuals = results[1].numpy() self._result_rank = results[2].numpy() @@ -94,16 +96,15 @@ def test_legacy_dygraph(self): for dev in self.devices: paddle.set_device(dev) place = paddle.CPUPlace() if dev == "cpu" else paddle.CUDAPlace(0) - x = paddle.to_tensor(self._input_data_1, - place=place, - dtype=self.dtype) - y = paddle.to_tensor(self._input_data_2, - place=place, - dtype=self.dtype) - results = paddle.linalg.lstsq(x, - y, - rcond=self.rcond, - driver=self.driver) + x = paddle.to_tensor( + self._input_data_1, place=place, dtype=self.dtype + ) + y = paddle.to_tensor( + self._input_data_2, place=place, dtype=self.dtype + ) + results = paddle.linalg.lstsq( + x, y, rcond=self.rcond, driver=self.driver + ) self._result_solution = results[0].numpy() self._result_residuals = results[1].numpy() self._result_rank = results[2].numpy() @@ -116,23 +117,25 @@ def test_static(self): paddle.set_device(dev) place = fluid.CPUPlace() if dev == "cpu" else fluid.CUDAPlace(0) with fluid.program_guard(fluid.Program(), fluid.Program()): - x = paddle.fluid.data(name="x", - shape=self._input_shape_1, - dtype=self._input_data_1.dtype) - y = paddle.fluid.data(name="y", - shape=self._input_shape_2, - dtype=self._input_data_2.dtype) - results = paddle.linalg.lstsq(x, - y, - rcond=self.rcond, - driver=self.driver) + x = paddle.fluid.data( + name="x", + shape=self._input_shape_1, + dtype=self._input_data_1.dtype, + ) + y = paddle.fluid.data( + name="y", + shape=self._input_shape_2, + dtype=self._input_data_2.dtype, + ) + results = paddle.linalg.lstsq( + x, y, rcond=self.rcond, driver=self.driver + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "x": self._input_data_1, - "y": self._input_data_2 - }, - fetch_list=[results]) + fetches = exe.run( + fluid.default_main_program(), + feed={"x": self._input_data_1, "y": self._input_data_2}, + fetch_list=[results], + ) self._result_solution = fetches[0] self._result_residuals = fetches[1] self._result_rank = fetches[2] @@ -141,44 +144,53 @@ def test_static(self): def assert_np_close(self): if len(self._input_shape_1) == 2: - np.testing.assert_allclose(self._result_solution, - self._output_solution, - rtol=1e-3) - if self._input_shape_1[-2] > self._input_shape_1[ - -1] and self._output_rank == self._input_shape_1[-1]: - np.testing.assert_allclose(self._result_residuals, - self._output_residuals, - rtol=1e-5) + np.testing.assert_allclose( + self._result_solution, self._output_solution, rtol=1e-3 + ) + if ( + self._input_shape_1[-2] > self._input_shape_1[-1] + and self._output_rank == self._input_shape_1[-1] + ): + np.testing.assert_allclose( + self._result_residuals, self._output_residuals, rtol=1e-5 + ) if self.driver in ("gelsy", "gelsd", "gelss"): - np.testing.assert_allclose(self._result_rank, - self._output_rank, - rtol=1e-5) + np.testing.assert_allclose( + self._result_rank, self._output_rank, rtol=1e-5 + ) if self.driver in ("gelsd", "gelss"): - np.testing.assert_allclose(self._result_sg_values, - self._output_sg_values, - rtol=1e-5) + np.testing.assert_allclose( + self._result_sg_values, self._output_sg_values, rtol=1e-5 + ) else: for i in range(len(self._output_solution)): - np.testing.assert_allclose(self._result_solution[i], - self._output_solution[i], - rtol=1e-3) - if self._input_shape_1[-2] > self._input_shape_1[ - -1] and self._output_rank[i] == self._input_shape_1[-1]: - np.testing.assert_allclose(self._result_residuals[i], - self._output_residuals[i], - rtol=1e-5) + np.testing.assert_allclose( + self._result_solution[i], + self._output_solution[i], + rtol=1e-3, + ) + if ( + self._input_shape_1[-2] > self._input_shape_1[-1] + and self._output_rank[i] == self._input_shape_1[-1] + ): + np.testing.assert_allclose( + self._result_residuals[i], + self._output_residuals[i], + rtol=1e-5, + ) if self.driver in ("gelsy", "gelsd", "gelss"): - np.testing.assert_allclose(self._result_rank[i], - self._output_rank[i], - rtol=1e-5) + np.testing.assert_allclose( + self._result_rank[i], self._output_rank[i], rtol=1e-5 + ) if self.driver in ("gelsd", "gelss"): - np.testing.assert_allclose(self._result_sg_values[i], - self._output_sg_values[i], - rtol=1e-5) + np.testing.assert_allclose( + self._result_sg_values[i], + self._output_sg_values[i], + rtol=1e-5, + ) class LinalgLstsqTestCase1(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float32' self.rcond = 1e-15 @@ -188,7 +200,6 @@ def init_config(self): class LinalgLstsqTestCase2(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float64' self.rcond = 1e-15 @@ -198,7 +209,6 @@ def init_config(self): class LinalgLstsqTestCase3(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float64' self.rcond = 1e-15 @@ -208,7 +218,6 @@ def init_config(self): class LinalgLstsqTestCaseRcond(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float64' self.rcond = 1e-7 @@ -218,7 +227,6 @@ def init_config(self): class LinalgLstsqTestCaseGelsFloat32(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float32' self.rcond = None @@ -228,7 +236,6 @@ def init_config(self): class LinalgLstsqTestCaseGelsFloat64(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float32' self.rcond = None @@ -238,7 +245,6 @@ def init_config(self): class LinalgLstsqTestCaseGelssFloat64(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float64' self.rcond = None @@ -248,7 +254,6 @@ def init_config(self): class LinalgLstsqTestCaseGelsyFloat32(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float32' self.rcond = 1e-15 @@ -258,7 +263,6 @@ def init_config(self): class LinalgLstsqTestCaseBatch1(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float32' self.rcond = 1e-15 @@ -268,7 +272,6 @@ def init_config(self): class LinalgLstsqTestCaseBatch2(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float64' self.rcond = 1e-15 @@ -278,7 +281,6 @@ def init_config(self): class LinalgLstsqTestCaseLarge1(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float64' self.rcond = 1e-15 @@ -288,7 +290,6 @@ def init_config(self): class LinalgLstsqTestCaseLarge2(LinalgLstsqTestCase): - def init_config(self): self.dtype = 'float64' self.rcond = 1e-15 diff --git a/python/paddle/fluid/tests/unittests/test_linalg_pinv_op.py b/python/paddle/fluid/tests/unittests/test_linalg_pinv_op.py index 03e671d883219..8c39cbb7c9517 100644 --- a/python/paddle/fluid/tests/unittests/test_linalg_pinv_op.py +++ b/python/paddle/fluid/tests/unittests/test_linalg_pinv_op.py @@ -20,7 +20,6 @@ class LinalgPinvTestCase(unittest.TestCase): - def setUp(self): self.init_config() self.generate_input() @@ -33,11 +32,13 @@ def generate_input(self): self._input_shape = (5, 5) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) def generate_output(self): - self._output_data = np.linalg.pinv(self._input_data, \ - rcond=self.rcond, hermitian=self.hermitian) + self._output_data = np.linalg.pinv( + self._input_data, rcond=self.rcond, hermitian=self.hermitian + ) def init_config(self): self.dtype = 'float64' @@ -48,9 +49,9 @@ def test_dygraph(self): for place in self.places: paddle.disable_static(place) x = paddle.to_tensor(self._input_data, place=place) - out = paddle.linalg.pinv(x, - rcond=self.rcond, - hermitian=self.hermitian).numpy() + out = paddle.linalg.pinv( + x, rcond=self.rcond, hermitian=self.hermitian + ).numpy() if (np.abs(out - self._output_data) < 1e-6).any(): pass else: @@ -65,16 +66,20 @@ def test_static(self): places.append(fluid.CUDAPlace(0)) for place in places: with fluid.program_guard(fluid.Program(), fluid.Program()): - x = paddle.fluid.data(name="input", - shape=self._input_shape, - dtype=self._input_data.dtype) - out = paddle.linalg.pinv(x, - rcond=self.rcond, - hermitian=self.hermitian) + x = paddle.fluid.data( + name="input", + shape=self._input_shape, + dtype=self._input_data.dtype, + ) + out = paddle.linalg.pinv( + x, rcond=self.rcond, hermitian=self.hermitian + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": self._input_data}, - fetch_list=[out]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": self._input_data}, + fetch_list=[out], + ) if (np.abs(fetches[0] - self._output_data) < 1e-6).any(): pass else: @@ -84,12 +89,12 @@ def test_static(self): def test_grad(self): for place in self.places: - x = paddle.to_tensor(self._input_data, - place=place, - stop_gradient=False) - out = paddle.linalg.pinv(x, - rcond=self.rcond, - hermitian=self.hermitian) + x = paddle.to_tensor( + self._input_data, place=place, stop_gradient=False + ) + out = paddle.linalg.pinv( + x, rcond=self.rcond, hermitian=self.hermitian + ) try: out.backward() x_grad = x.grad @@ -99,75 +104,75 @@ def test_grad(self): class LinalgPinvTestCase1(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (4, 5) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) class LinalgPinvTestCase2(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (5, 4) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) class LinalgPinvTestCaseBatch1(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 5) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) class LinalgPinvTestCaseBatch2(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 4, 5) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) class LinalgPinvTestCaseBatch3(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 4) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) class LinalgPinvTestCaseBatch4(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 6, 5, 4) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) class LinalgPinvTestCaseBatchBig(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (2, 200, 300) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) class LinalgPinvTestCaseFP32(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 5) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) def init_config(self): self.dtype = 'float32' @@ -176,12 +181,12 @@ def init_config(self): class LinalgPinvTestCaseRcond(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 5) np.random.seed(123) self._input_data = np.random.random(self._input_shape).astype( - self.dtype) + self.dtype + ) def init_config(self): self.dtype = 'float64' @@ -190,12 +195,12 @@ def init_config(self): class LinalgPinvTestCaseHermitian1(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (5, 5) np.random.seed(123) - x = np.random.random(self._input_shape).astype(self.dtype) + \ - 1J * np.random.random(self._input_shape).astype(self.dtype) + x = np.random.random(self._input_shape).astype( + self.dtype + ) + 1j * np.random.random(self._input_shape).astype(self.dtype) self._input_data = x + x.transpose().conj() def init_config(self): @@ -205,12 +210,12 @@ def init_config(self): class LinalgPinvTestCaseHermitian2(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 5) np.random.seed(123) - x = np.random.random(self._input_shape).astype(self.dtype) + \ - 1J * np.random.random(self._input_shape).astype(self.dtype) + x = np.random.random(self._input_shape).astype( + self.dtype + ) + 1j * np.random.random(self._input_shape).astype(self.dtype) self._input_data = x + x.transpose((0, 2, 1)).conj() def init_config(self): @@ -220,12 +225,12 @@ def init_config(self): class LinalgPinvTestCaseHermitian3(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 5) np.random.seed(123) - x = np.random.random(self._input_shape).astype(self.dtype) + \ - 1J * np.random.random(self._input_shape).astype(self.dtype) + x = np.random.random(self._input_shape).astype( + self.dtype + ) + 1j * np.random.random(self._input_shape).astype(self.dtype) self._input_data = x + x.transpose((0, 2, 1)).conj() def init_config(self): @@ -235,7 +240,6 @@ def init_config(self): class LinalgPinvTestCaseHermitian4(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (5, 5) np.random.seed(123) @@ -249,7 +253,6 @@ def init_config(self): class LinalgPinvTestCaseHermitian5(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 5) np.random.seed(123) @@ -263,7 +266,6 @@ def init_config(self): class LinalgPinvTestCaseHermitianFP32(LinalgPinvTestCase): - def generate_input(self): self._input_shape = (3, 5, 5) np.random.seed(123) diff --git a/python/paddle/fluid/tests/unittests/test_linear.py b/python/paddle/fluid/tests/unittests/test_linear.py index eaf2cb528cc2a..0cec046354294 100644 --- a/python/paddle/fluid/tests/unittests/test_linear.py +++ b/python/paddle/fluid/tests/unittests/test_linear.py @@ -21,14 +21,16 @@ class LinearTestCase(unittest.TestCase): - def setUp(self): self.dtype = 'float32' self.input = np.ones((3, 1, 2)).astype(self.dtype) self.weight = np.ones((2, 2)).astype(self.dtype) self.bias = np.ones((2)).astype(self.dtype) - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) def functional(self, place): paddle.disable_static(place) @@ -46,17 +48,18 @@ def paddle_nn_layer(self, place): learning_rate=1.0, trainable=False, regularizer=None, - initializer=paddle.fluid.initializer.ConstantInitializer(value=1.0)) + initializer=paddle.fluid.initializer.ConstantInitializer(value=1.0), + ) bias_attr = fluid.ParamAttr( name="linear_bias", learning_rate=1.0, trainable=False, regularizer=None, - initializer=paddle.fluid.initializer.ConstantInitializer(value=1.0)) - linear = paddle.nn.Linear(2, - 2, - weight_attr=weight_attr, - bias_attr=bias_attr) + initializer=paddle.fluid.initializer.ConstantInitializer(value=1.0), + ) + linear = paddle.nn.Linear( + 2, 2, weight_attr=weight_attr, bias_attr=bias_attr + ) y = linear(input) return y.numpy() @@ -75,18 +78,21 @@ def test_weight_init(self): if not paddle.is_compiled_with_cuda(): return paddle.seed(100) - linear = paddle.nn.Linear(2, - 3, - weight_attr=paddle.nn.initializer.Normal( - 0, 1.)) + linear = paddle.nn.Linear( + 2, 3, weight_attr=paddle.nn.initializer.Normal(0, 1.0) + ) paddle.nn.utils._stride_column(linear.weight) - expect = [[1.4349908, -0.8099171, -2.64788], - [-1.4981681, -1.1784115, -0.023253186]] + expect = [ + [1.4349908, -0.8099171, -2.64788], + [-1.4981681, -1.1784115, -0.023253186], + ] np.testing.assert_allclose(linear.weight.numpy(), expect, rtol=1e-05) linear = paddle.nn.Linear(2, 3) - expect = [[0.73261100, 0.43836895, 0.07908206], - [0.85075015, -1.04724526, 0.64371765]] + expect = [ + [0.73261100, 0.43836895, 0.07908206], + [0.85075015, -1.04724526, 0.64371765], + ] np.testing.assert_allclose(linear.weight.numpy(), expect, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py b/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py index 526601bbe6801..203a4ac62ca49 100755 --- a/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py +++ b/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py @@ -20,9 +20,16 @@ class LinearChainCrfForward(object): - - def __init__(self, seq_start_positions, emission_weights, emission_row_max, - emission_exps, transition_weights, transition_exps, labels): + def __init__( + self, + seq_start_positions, + emission_weights, + emission_row_max, + emission_exps, + transition_weights, + transition_exps, + labels, + ): self.tag_num = emission_weights.shape[1] self.seq_num = len(seq_start_positions) - 1 @@ -46,8 +53,9 @@ def __init__(self, seq_start_positions, emission_weights, emission_row_max, # The output of linear chain crf operator. # alpha is a memo table in dynamic programming to calculate # nomalization factor. - self.alpha = np.zeros((seq_start_positions[-1], self.tag_num), - dtype="float64") + self.alpha = np.zeros( + (seq_start_positions[-1], self.tag_num), dtype="float64" + ) self.log_likelihood = np.zeros((self.seq_num, 1)) def _l1_norm(self, x): @@ -57,7 +65,7 @@ def _l1_norm(self, x): def _forward_a_sequence(self, x, x_row_max, x_exps, label, alpha): seq_len = x_row_max.shape[0] - log_likelihood = 0. + log_likelihood = 0.0 for i in range(self.tag_num): alpha[0, i] = self.a_exps[i] * x_exps[0, i] @@ -66,22 +74,21 @@ def _forward_a_sequence(self, x, x_row_max, x_exps, label, alpha): # calculate the unnormalized logits of the normalization factor. for k in range(1, seq_len): for i in range(self.tag_num): - s = 0. + s = 0.0 for j in range(self.tag_num): s += alpha[k - 1, j] * self.w_exps[j, i] alpha[k, i] = x_exps[k, i] * s log_likelihood -= x_row_max[k] + np.log(self._l1_norm(alpha[k, :])) - s = 0. + s = 0.0 for i in range(self.tag_num): s += alpha[-1, i] * self.b_exps[i] log_likelihood -= np.log(s) # calculate the nominator part. - log_likelihood += (self.a[label[0]] + x[0, label[0]] + - self.b[label[-1]]) + log_likelihood += self.a[label[0]] + x[0, label[0]] + self.b[label[-1]] for k in range(1, seq_len): - log_likelihood += (x[k, label[k]] + self.w[label[k - 1], label[k]]) + log_likelihood += x[k, label[k]] + self.w[label[k - 1], label[k]] return -log_likelihood def crf_forward_compute(self): @@ -91,14 +98,16 @@ def crf_forward_compute(self): if start >= end: continue self.log_likelihood[i] = self._forward_a_sequence( - self.x[start:end, :], self.x_row_max[start:end, :], - self.x_exps[start:end, :], self.labels[start:end, :], - self.alpha[start:end, :]) + self.x[start:end, :], + self.x_row_max[start:end, :], + self.x_exps[start:end, :], + self.labels[start:end, :], + self.alpha[start:end, :], + ) return self.alpha, self.log_likelihood class TestLinearChainCrfOp(OpTest): - def set_test_data(self): # TODO(caoying) Fix the unittest by: add the boundary cases when # sequence lengths are 1, 2, and 3. @@ -114,34 +123,41 @@ def set_test_data(self): lod[-1].append(random.randint(1, MAX_SEQ_LEN)) seq_start_pos.append(seq_start_pos[-1] + lod[-1][-1]) emission = np.random.uniform( - -1, 1, [seq_start_pos[-1], TAG_NUM]).astype("float64") + -1, 1, [seq_start_pos[-1], TAG_NUM] + ).astype("float64") emission_row_max = np.amax(emission, axis=1, keepdims=True) emission_exps = np.exp(emission - emission_row_max) - transition = np.random.uniform(-0.5, 0.5, - [TAG_NUM + 2, TAG_NUM]).astype("float64") + transition = np.random.uniform( + -0.5, 0.5, [TAG_NUM + 2, TAG_NUM] + ).astype("float64") transition_exps = np.exp(transition) - labels = np.random.randint(low=0, - high=TAG_NUM, - size=(seq_start_pos[-1], 1), - dtype="int64") + labels = np.random.randint( + low=0, high=TAG_NUM, size=(seq_start_pos[-1], 1), dtype="int64" + ) self.inputs = { "Emission": (emission, lod), "Transition": transition, - "Label": (labels, lod) + "Label": (labels, lod), } - crf = LinearChainCrfForward(seq_start_pos, emission, emission_row_max, - emission_exps, transition, transition_exps, - labels) + crf = LinearChainCrfForward( + seq_start_pos, + emission, + emission_row_max, + emission_exps, + transition, + transition_exps, + labels, + ) alpha, log_likelihood = crf.crf_forward_compute() self.outputs = { "Alpha": alpha, "EmissionExps": emission_exps, "TransitionExps": transition_exps, - "LogLikelihood": log_likelihood + "LogLikelihood": log_likelihood, } def setUp(self): @@ -155,20 +171,19 @@ def test_check_grad(self): self.check_grad(["Emission", "Transition"], "LogLikelihood") def test_check_grad_ignore_transition(self): - self.check_grad(["Emission"], - "LogLikelihood", - no_grad_set=set("Transition")) + self.check_grad( + ["Emission"], "LogLikelihood", no_grad_set=set("Transition") + ) class TestLinearChainCrfPaddingTensor(OpTest): - def seq_pad(self, data, length): max_len = np.max(length) shape = [len(length), max_len] + list(data.shape[1:]) padded = np.zeros(shape).astype(data.dtype) offset = 0 for i, l in enumerate(length): - padded[i, 0:l] = data[offset:offset + l] + padded[i, 0:l] = data[offset : offset + l] offset += l return padded @@ -179,7 +194,7 @@ def seq_pad_exps(self, data, length): padded = np.ones(shape).astype(data.dtype) offset = 0 for i, l in enumerate(length): - padded[i, 0:l] = data[offset:offset + l] + padded[i, 0:l] = data[offset : offset + l] offset += l return padded @@ -196,32 +211,39 @@ def set_test_data_1(self): lod[-1].append(random.randint(1, MAX_SEQ_LEN)) seq_start_pos.append(seq_start_pos[-1] + lod[-1][-1]) emission = np.random.uniform( - -1, 1, [seq_start_pos[-1], TAG_NUM]).astype("float64") + -1, 1, [seq_start_pos[-1], TAG_NUM] + ).astype("float64") emission_row_max = np.amax(emission, axis=1, keepdims=True) emission_exps = np.exp(emission - emission_row_max) - transition = np.random.uniform(-0.5, 0.5, - [TAG_NUM + 2, TAG_NUM]).astype("float64") + transition = np.random.uniform( + -0.5, 0.5, [TAG_NUM + 2, TAG_NUM] + ).astype("float64") transition_exps = np.exp(transition) - labels = np.random.randint(low=0, - high=TAG_NUM, - size=(seq_start_pos[-1], 1), - dtype="int64") + labels = np.random.randint( + low=0, high=TAG_NUM, size=(seq_start_pos[-1], 1), dtype="int64" + ) self.inputs = { "Emission": self.seq_pad(emission, lod[0]), "Transition": transition, "Label": self.seq_pad(labels, lod[0]), - "Length": np.array(lod).astype("int64") + "Length": np.array(lod).astype("int64"), } - crf = LinearChainCrfForward(seq_start_pos, emission, emission_row_max, - emission_exps, transition, transition_exps, - labels) + crf = LinearChainCrfForward( + seq_start_pos, + emission, + emission_row_max, + emission_exps, + transition, + transition_exps, + labels, + ) alpha, log_likelihood = crf.crf_forward_compute() self.outputs = { "Alpha": self.seq_pad(alpha, lod[0]), "EmissionExps": self.seq_pad_exps(emission_exps, lod[0]), "TransitionExps": transition_exps, - "LogLikelihood": log_likelihood + "LogLikelihood": log_likelihood, } def setUp(self): @@ -235,9 +257,9 @@ def test_check_grad(self): self.check_grad(["Emission", "Transition"], "LogLikelihood") def test_check_grad_ignore_transition(self): - self.check_grad(["Emission"], - "LogLikelihood", - no_grad_set=set("Transition")) + self.check_grad( + ["Emission"], "LogLikelihood", no_grad_set=set("Transition") + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_linear_interp_op.py b/python/paddle/fluid/tests/unittests/test_linear_interp_op.py index cd5d799a5bd85..f52e597d66549 100755 --- a/python/paddle/fluid/tests/unittests/test_linear_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_linear_interp_op.py @@ -23,13 +23,15 @@ from paddle.nn.functional import interpolate -def linear_interp_np(input, - out_w, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCHW'): +def linear_interp_np( + input, + out_w, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCHW', +): if data_layout == "NHWC": input = np.transpose(input, (0, 2, 1)) # NHWC => NCHW if out_size is not None: @@ -40,7 +42,7 @@ def linear_interp_np(input, ratio_w = 0.0 if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: ratio_w = 1.0 * in_w / out_w @@ -48,22 +50,23 @@ def linear_interp_np(input, out = np.zeros((batch_size, channel, out_w)) for j in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (j + 0.5) - 0.5) else: w = int(ratio_w * j) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (j + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * j - w w2lambda = 1.0 - w1lambda - out[:, :, - j] = w2lambda * input[:, :, w] + w1lambda * input[:, :, w + wid] + out[:, :, j] = ( + w2lambda * input[:, :, w] + w1lambda * input[:, :, w + wid] + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 1)) # NCHW => NHWC @@ -72,7 +75,6 @@ def linear_interp_np(input, class TestLinearInterpOp(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -91,9 +93,15 @@ def setUp(self): else: out_w = self.out_w - output_np = linear_interp_np(input_np, out_w, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode, self.data_layout) + output_np = linear_interp_np( + input_np, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -106,7 +114,7 @@ def setUp(self): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } self.outputs = {'Out': output_np} @@ -123,59 +131,63 @@ def init_test_case(self): self.interp_method = 'linear' self.input_shape = [1, 3, 100] self.out_w = 50 - self.scale = 0. - self.out_size = np.array([ - 50, - ]).astype("int32") + self.scale = 0.0 + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 1 class TestLinearInterpOpDataLayout(TestLinearInterpOp): - def init_test_case(self): self.interp_method = 'linear' self.input_shape = [1, 3, 100] self.out_w = 50 - self.scale = 0. - self.out_size = np.array([ - 50, - ]).astype("int32") + self.scale = 0.0 + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 1 self.data_layout = 'NHWC' class TestLinearInterpOpAlignMode(TestLinearInterpOp): - def init_test_case(self): self.interp_method = 'linear' self.input_shape = [1, 3, 100] self.out_w = 50 - self.scale = 0. - self.out_size = np.array([ - 50, - ]).astype("int32") + self.scale = 0.0 + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 0 class TestLinearInterpOpScale(TestLinearInterpOp): - def init_test_case(self): self.interp_method = 'linear' self.input_shape = [1, 3, 100] self.out_w = 50 self.scale = 0.5 - self.out_size = np.array([ - 50, - ]).astype("int32") + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 0 class TestLinearInterpOpSizeTensor(TestLinearInterpOp): - def setUp(self): self.out_size = None self.actual_shape = None @@ -196,9 +208,15 @@ def setUp(self): else: out_w = self.out_w - output_np = linear_interp_np(input_np, out_w, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode, self.data_layout) + output_np = linear_interp_np( + input_np, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None and self.shape_by_1Dtensor: @@ -208,8 +226,9 @@ def setUp(self): else: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs = { @@ -218,78 +237,88 @@ def setUp(self): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } self.outputs = {'Out': output_np} class TestResizeLinearAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[1, 3, 64], dtype="float32") dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[1], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[1], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") - - out1 = fluid.layers.resize_linear(x, - out_shape=[ - 128, - ], - align_mode=1, - align_corners=False) - out2 = fluid.layers.resize_linear(x, - out_shape=[128], - align_mode=1, - align_corners=False) - out3 = fluid.layers.resize_linear(x, - out_shape=shape_tensor, - align_mode=1, - align_corners=False) - out4 = fluid.layers.resize_linear(x, - out_shape=[ - 128, - ], - actual_shape=actual_size, - align_mode=1, - align_corners=False) - out5 = fluid.layers.resize_linear(x, - scale=scale_tensor, - align_mode=1, - align_corners=False) - - out6 = interpolate(x, - scale_factor=scale_tensor, - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') - out7 = interpolate(x, - size=[ - 128, - ], - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') - out8 = interpolate(x, - size=shape_tensor, - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) + + out1 = fluid.layers.resize_linear( + x, + out_shape=[ + 128, + ], + align_mode=1, + align_corners=False, + ) + out2 = fluid.layers.resize_linear( + x, out_shape=[128], align_mode=1, align_corners=False + ) + out3 = fluid.layers.resize_linear( + x, out_shape=shape_tensor, align_mode=1, align_corners=False + ) + out4 = fluid.layers.resize_linear( + x, + out_shape=[ + 128, + ], + actual_shape=actual_size, + align_mode=1, + align_corners=False, + ) + out5 = fluid.layers.resize_linear( + x, scale=scale_tensor, align_mode=1, align_corners=False + ) + + out6 = interpolate( + x, + scale_factor=scale_tensor, + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) + out7 = interpolate( + x, + size=[ + 128, + ], + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) + out8 = interpolate( + x, + size=shape_tensor, + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) x_data = np.random.random((1, 3, 64)).astype("float32") dim_data = np.array([128]).astype("int32") - shape_data = np.array([ - 128, - ]).astype("int32") - actual_size_data = np.array([ - 128, - ]).astype("int32") + shape_data = np.array( + [ + 128, + ] + ).astype("int32") + actual_size_data = np.array( + [ + 128, + ] + ).astype("int32") scale_data = np.array([2.0]).astype("float32") if core.is_compiled_with_cuda(): @@ -305,46 +334,45 @@ def test_case(self): "dim": dim_data, "shape_tensor": shape_data, "actual_size": actual_size_data, - "scale_tensor": scale_data + "scale_tensor": scale_data, }, fetch_list=[out1, out2, out3, out4, out5, out6, out7, out8], - return_numpy=True) + return_numpy=True, + ) - expect_res = linear_interp_np(x_data, - out_w=128, - align_mode=1, - align_corners=False) + expect_res = linear_interp_np( + x_data, out_w=128, align_mode=1, align_corners=False + ) for res in results: np.testing.assert_allclose(res, expect_res, rtol=1e-05) class TestLinearInterpOpAPI2_0(unittest.TestCase): - def test_case(self): # dygraph x_data = np.random.random((1, 3, 128)).astype("float32") - us_1 = paddle.nn.Upsample(size=[ - 64, - ], - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') + us_1 = paddle.nn.Upsample( + size=[ + 64, + ], + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) with fluid.dygraph.guard(): x = fluid.dygraph.to_variable(x_data) interp = us_1(x) - expect = linear_interp_np(x_data, - out_w=64, - align_mode=1, - align_corners=False) + expect = linear_interp_np( + x_data, out_w=64, align_mode=1, align_corners=False + ) np.testing.assert_allclose(interp.numpy(), expect, rtol=1e-05) class TestResizeLinearOpUint8(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -357,9 +385,14 @@ def setUp(self): else: out_w = self.out_w - output_np = linear_interp_np(input_np, out_w, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode) + output_np = linear_interp_np( + input_np, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -369,7 +402,7 @@ def setUp(self): 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } self.outputs = {'Out': output_np} @@ -383,42 +416,48 @@ def init_test_case(self): self.interp_method = 'linear' self.input_shape = [2, 3, 100] self.out_w = 50 - self.scale = 0. - self.out_size = np.array([ - 50, - ]).astype("int32") + self.scale = 0.0 + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = True self.align_mode = 1 class TestLinearInterpOpException(unittest.TestCase): - def test_exception(self): - def input_shape_error(): x1 = fluid.data(name="x1", shape=[1], dtype="float32") - out = fluid.layers.resize_linear(x1, - out_shape=[ - 256, - ], - data_format='NCW') + out = fluid.layers.resize_linear( + x1, + out_shape=[ + 256, + ], + data_format='NCW', + ) def data_format_error(): x2 = fluid.data(name="x2", shape=[1, 3, 128], dtype="float32") - out = fluid.layers.resize_linear(x2, - out_shape=[ - 256, - ], - data_format='NHWCD') + out = fluid.layers.resize_linear( + x2, + out_shape=[ + 256, + ], + data_format='NHWCD', + ) def out_shape_error(): x3 = fluid.data(name="x3", shape=[1, 3, 128], dtype="float32") - out = fluid.layers.resize_linear(x3, - out_shape=[ - 256, - 256, - ], - data_format='NHWC') + out = fluid.layers.resize_linear( + x3, + out_shape=[ + 256, + 256, + ], + data_format='NHWC', + ) self.assertRaises(ValueError, input_shape_error) self.assertRaises(ValueError, data_format_error) @@ -426,36 +465,41 @@ def out_shape_error(): class TestLinearInterpOpError(unittest.TestCase): - def test_error(self): with program_guard(Program(), Program()): def input_shape_error(): x1 = fluid.data(name="x1", shape=[1], dtype="float32") - out1 = paddle.nn.Upsample(size=[ - 256, - ], - data_format='NCW', - mode='linear') + out1 = paddle.nn.Upsample( + size=[ + 256, + ], + data_format='NCW', + mode='linear', + ) out1_res = out1(x1) def data_format_error(): x2 = fluid.data(name="x2", shape=[1, 3, 128], dtype="float32") - out2 = paddle.nn.Upsample(size=[ - 256, - ], - data_format='NHWCD', - mode='linear') + out2 = paddle.nn.Upsample( + size=[ + 256, + ], + data_format='NHWCD', + mode='linear', + ) out2_res = out2(x2) def out_shape_error(): x3 = fluid.data(name="x3", shape=[1, 3, 128], dtype="float32") - out3 = paddle.nn.Upsample(size=[ - 256, - 256, - ], - data_format='NHWC', - mode='linear') + out3 = paddle.nn.Upsample( + size=[ + 256, + 256, + ], + data_format='NHWC', + mode='linear', + ) out3_res = out3(x3) self.assertRaises(ValueError, input_shape_error) diff --git a/python/paddle/fluid/tests/unittests/test_linear_interp_v2_op.py b/python/paddle/fluid/tests/unittests/test_linear_interp_v2_op.py index 9fbbc8ceaa0fc..30e7d2ed97b05 100755 --- a/python/paddle/fluid/tests/unittests/test_linear_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_linear_interp_v2_op.py @@ -23,18 +23,20 @@ from paddle.nn.functional import interpolate -def linear_interp_test(x, - OutSize=None, - SizeTensor=None, - Scale=None, - data_layout='NCHW', - out_d=-1, - out_h=-1, - out_w=-1, - scale=[], - interp_method='linear', - align_corners=True, - align_mode=0): +def linear_interp_test( + x, + OutSize=None, + SizeTensor=None, + Scale=None, + data_layout='NCHW', + out_d=-1, + out_h=-1, + out_w=-1, + scale=[], + interp_method='linear', + align_corners=True, + align_mode=0, +): if isinstance(scale, float) or isinstance(scale, int): scale_list = [] for _ in range(len(x.shape) - 2): @@ -44,21 +46,35 @@ def linear_interp_test(x, scale = list(map(float, scale)) if SizeTensor is not None: if not isinstance(SizeTensor, list) and not isinstance( - SizeTensor, tuple): + SizeTensor, tuple + ): SizeTensor = [SizeTensor] - return paddle._C_ops.linear_interp(x, OutSize, SizeTensor, Scale, - data_layout, out_d, out_h, out_w, scale, - interp_method, align_corners, align_mode) - - -def linear_interp_np(input, - out_w, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCHW'): + return paddle._C_ops.linear_interp( + x, + OutSize, + SizeTensor, + Scale, + data_layout, + out_d, + out_h, + out_w, + scale, + interp_method, + align_corners, + align_mode, + ) + + +def linear_interp_np( + input, + out_w, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCHW', +): if data_layout == "NHWC": input = np.transpose(input, (0, 2, 1)) # NHWC => NCHW if out_size is not None: @@ -69,7 +85,7 @@ def linear_interp_np(input, ratio_w = 0.0 if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -80,22 +96,23 @@ def linear_interp_np(input, out = np.zeros((batch_size, channel, out_w)) for j in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (j + 0.5) - 0.5) else: w = int(ratio_w * j) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (j + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * j - w w2lambda = 1.0 - w1lambda - out[:, :, - j] = w2lambda * input[:, :, w] + w1lambda * input[:, :, w + wid] + out[:, :, j] = ( + w2lambda * input[:, :, w] + w1lambda * input[:, :, w + wid] + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 1)) # NCHW => NHWC @@ -104,7 +121,6 @@ def linear_interp_np(input, class TestLinearInterpOp(OpTest): - def setUp(self): self.python_api = linear_interp_test self.out_size = None @@ -129,9 +145,16 @@ def setUp(self): else: out_w = self.out_w - output_np = linear_interp_np(input_np, out_w, self.scale, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode, self.data_layout) + output_np = linear_interp_np( + input_np, + out_w, + self.scale, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -143,7 +166,7 @@ def setUp(self): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale > 0: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -165,58 +188,62 @@ def init_test_case(self): self.input_shape = [1, 3, 100] self.out_w = 50 self.scale = 0.5 - self.out_size = np.array([ - 50, - ]).astype("int32") + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 1 class TestLinearInterpOpDataLayout(TestLinearInterpOp): - def init_test_case(self): self.interp_method = 'linear' self.input_shape = [1, 100, 3] self.out_w = 50 self.scale = 0.5 - self.out_size = np.array([ - 50, - ]).astype("int32") + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 1 self.data_layout = 'NHWC' class TestLinearInterpOpAlignMode(TestLinearInterpOp): - def init_test_case(self): self.interp_method = 'linear' self.input_shape = [1, 3, 100] self.out_w = 50 self.scale = 0.5 - self.out_size = np.array([ - 50, - ]).astype("int32") + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 0 class TestLinearInterpOpScale(TestLinearInterpOp): - def init_test_case(self): self.interp_method = 'linear' self.input_shape = [1, 3, 100] self.out_w = 50 self.scale = 0.8 - self.out_size = np.array([ - 50, - ]).astype("int32") + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = False self.align_mode = 0 class TestLinearInterpOpSizeTensor(TestLinearInterpOp): - def setUp(self): self.python_api = linear_interp_test self.out_size = None @@ -242,9 +269,16 @@ def setUp(self): else: out_w = self.out_w - output_np = linear_interp_np(input_np, out_w, 0, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode, self.data_layout) + output_np = linear_interp_np( + input_np, + out_w, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None and self.shape_by_1Dtensor: @@ -254,8 +288,9 @@ def setUp(self): else: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs = { @@ -263,7 +298,7 @@ def setUp(self): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale > 0: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -275,72 +310,82 @@ def setUp(self): class TestResizeLinearAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[1, 3, 64], dtype="float32") dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[1], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[1], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") - - out1 = fluid.layers.resize_linear(x, - out_shape=[ - 128, - ], - align_mode=1, - align_corners=False) - out2 = fluid.layers.resize_linear(x, - out_shape=[128], - align_mode=1, - align_corners=False) - out3 = fluid.layers.resize_linear(x, - out_shape=shape_tensor, - align_mode=1, - align_corners=False) - out4 = fluid.layers.resize_linear(x, - out_shape=[ - 128, - ], - actual_shape=actual_size, - align_mode=1, - align_corners=False) - out5 = fluid.layers.resize_linear(x, - scale=scale_tensor, - align_mode=1, - align_corners=False) - - out6 = interpolate(x, - scale_factor=scale_tensor, - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') - out7 = interpolate(x, - size=[ - 128, - ], - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') - out8 = interpolate(x, - size=shape_tensor, - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) + + out1 = fluid.layers.resize_linear( + x, + out_shape=[ + 128, + ], + align_mode=1, + align_corners=False, + ) + out2 = fluid.layers.resize_linear( + x, out_shape=[128], align_mode=1, align_corners=False + ) + out3 = fluid.layers.resize_linear( + x, out_shape=shape_tensor, align_mode=1, align_corners=False + ) + out4 = fluid.layers.resize_linear( + x, + out_shape=[ + 128, + ], + actual_shape=actual_size, + align_mode=1, + align_corners=False, + ) + out5 = fluid.layers.resize_linear( + x, scale=scale_tensor, align_mode=1, align_corners=False + ) + + out6 = interpolate( + x, + scale_factor=scale_tensor, + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) + out7 = interpolate( + x, + size=[ + 128, + ], + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) + out8 = interpolate( + x, + size=shape_tensor, + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) x_data = np.random.random((1, 3, 64)).astype("float32") dim_data = np.array([128]).astype("int32") - shape_data = np.array([ - 128, - ]).astype("int32") - actual_size_data = np.array([ - 128, - ]).astype("int32") + shape_data = np.array( + [ + 128, + ] + ).astype("int32") + actual_size_data = np.array( + [ + 128, + ] + ).astype("int32") scale_data = np.array([2.0]).astype("float32") if core.is_compiled_with_cuda(): @@ -356,44 +401,43 @@ def test_case(self): "dim": dim_data, "shape_tensor": shape_data, "actual_size": actual_size_data, - "scale_tensor": scale_data + "scale_tensor": scale_data, }, fetch_list=[out1, out2, out3, out4, out5, out6, out7, out8], - return_numpy=True) + return_numpy=True, + ) - expect_res = linear_interp_np(x_data, - out_w=128, - align_mode=1, - align_corners=False) + expect_res = linear_interp_np( + x_data, out_w=128, align_mode=1, align_corners=False + ) for res in results: np.testing.assert_allclose(res, expect_res, rtol=1e-05) class TestLinearInterpOpAPI2_0(unittest.TestCase): - def test_case(self): # dygraph x_data = np.random.random((1, 3, 128)).astype("float32") - us_1 = paddle.nn.Upsample(size=[64], - mode='linear', - align_mode=1, - align_corners=False, - data_format='NCW') + us_1 = paddle.nn.Upsample( + size=[64], + mode='linear', + align_mode=1, + align_corners=False, + data_format='NCW', + ) with fluid.dygraph.guard(): x = fluid.dygraph.to_variable(x_data) interp = us_1(x) - expect = linear_interp_np(x_data, - out_w=64, - align_mode=1, - align_corners=False) + expect = linear_interp_np( + x_data, out_w=64, align_mode=1, align_corners=False + ) np.testing.assert_allclose(interp.numpy(), expect, rtol=1e-05) class TestResizeLinearOpUint8(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -410,9 +454,15 @@ def setUp(self): else: out_w = self.out_w - output_np = linear_interp_np(input_np, out_w, 0, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode) + output_np = linear_interp_np( + input_np, + out_w, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -421,7 +471,7 @@ def setUp(self): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } if self.scale > 0: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -441,42 +491,48 @@ def init_test_case(self): self.interp_method = 'linear' self.input_shape = [2, 3, 100] self.out_w = 50 - self.scale = 0. - self.out_size = np.array([ - 50, - ]).astype("int32") + self.scale = 0.0 + self.out_size = np.array( + [ + 50, + ] + ).astype("int32") self.align_corners = True self.align_mode = 1 class TestLinearInterpOpException(unittest.TestCase): - def test_exception(self): - def input_shape_error(): x1 = fluid.data(name="x1", shape=[1], dtype="float32") - out = fluid.layers.resize_linear(x1, - out_shape=[ - 256, - ], - data_format='NCW') + out = fluid.layers.resize_linear( + x1, + out_shape=[ + 256, + ], + data_format='NCW', + ) def data_format_error(): x2 = fluid.data(name="x2", shape=[1, 3, 128], dtype="float32") - out = fluid.layers.resize_linear(x2, - out_shape=[ - 256, - ], - data_format='NHWCD') + out = fluid.layers.resize_linear( + x2, + out_shape=[ + 256, + ], + data_format='NHWCD', + ) def out_shape_error(): x3 = fluid.data(name="x3", shape=[1, 3, 128], dtype="float32") - out = fluid.layers.resize_linear(x3, - out_shape=[ - 256, - 256, - ], - data_format='NHWC') + out = fluid.layers.resize_linear( + x3, + out_shape=[ + 256, + 256, + ], + data_format='NHWC', + ) self.assertRaises(ValueError, input_shape_error) self.assertRaises(ValueError, data_format_error) @@ -484,29 +540,28 @@ def out_shape_error(): class TestLinearInterpOpError(unittest.TestCase): - def test_error(self): with program_guard(Program(), Program()): def input_shape_error(): x1 = fluid.data(name="x1", shape=[1], dtype="float32") - out1 = paddle.nn.Upsample(size=[256], - data_format='NCW', - mode='linear') + out1 = paddle.nn.Upsample( + size=[256], data_format='NCW', mode='linear' + ) out1_res = out1(x1) def data_format_error(): x2 = fluid.data(name="x2", shape=[1, 3, 128], dtype="float32") - out2 = paddle.nn.Upsample(size=[256], - data_format='NHWCD', - mode='linear') + out2 = paddle.nn.Upsample( + size=[256], data_format='NHWCD', mode='linear' + ) out2_res = out2(x2) def out_shape_error(): x3 = fluid.data(name="x3", shape=[1, 3, 128], dtype="float32") - out3 = paddle.nn.Upsample(size=[256, 256], - data_format='NHWC', - mode='linear') + out3 = paddle.nn.Upsample( + size=[256, 256], data_format='NHWC', mode='linear' + ) out3_res = out3(x3) self.assertRaises(ValueError, input_shape_error) @@ -514,10 +569,10 @@ def out_shape_error(): self.assertRaises(ValueError, out_shape_error) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestLinearInterpOpForFloat16(unittest.TestCase): - def init_test_case(self): self.interp_method = 'linear' self.input_shape = [1, 3, 64] @@ -531,12 +586,14 @@ def check_main(self, x_np, dtype): x_np = x_np.astype(dtype) x = paddle.to_tensor(x_np) x.stop_gradient = False - y = interpolate(x, - scale_factor=self.scale, - mode=self.interp_method, - align_mode=self.align_mode, - align_corners=self.align_corners, - data_format=self.data_layout) + y = interpolate( + x, + scale_factor=self.scale, + mode=self.interp_method, + align_mode=self.align_mode, + align_corners=self.align_corners, + data_format=self.data_layout, + ) x_g = paddle.grad(y, x) y_np = y[0].numpy().astype('float32') x_g_np = x_g[0].numpy().astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_linspace.py b/python/paddle/fluid/tests/unittests/test_linspace.py index c38d5b1fe591b..3549d46ec9eec 100644 --- a/python/paddle/fluid/tests/unittests/test_linspace.py +++ b/python/paddle/fluid/tests/unittests/test_linspace.py @@ -23,7 +23,6 @@ class TestLinspaceOpCommonCase(OpTest): - def setUp(self): self.op_type = "linspace" self.python_api = paddle.linspace @@ -31,7 +30,7 @@ def setUp(self): self.inputs = { 'Start': np.array([0]).astype(dtype), 'Stop': np.array([10]).astype(dtype), - 'Num': np.array([11]).astype('int32') + 'Num': np.array([11]).astype('int32'), } self.attrs = {'dtype': int(core.VarDesc.VarType.FP32)} @@ -42,7 +41,6 @@ def test_check_output(self): class TestLinspaceOpReverseCase(OpTest): - def setUp(self): self.op_type = "linspace" self.python_api = paddle.linspace @@ -50,7 +48,7 @@ def setUp(self): self.inputs = { 'Start': np.array([10]).astype(dtype), 'Stop': np.array([0]).astype(dtype), - 'Num': np.array([11]).astype('int32') + 'Num': np.array([11]).astype('int32'), } self.attrs = {'dtype': int(core.VarDesc.VarType.FP32)} @@ -61,7 +59,6 @@ def test_check_output(self): class TestLinspaceOpNumOneCase(OpTest): - def setUp(self): self.op_type = "linspace" self.python_api = paddle.linspace @@ -69,7 +66,7 @@ def setUp(self): self.inputs = { 'Start': np.array([10]).astype(dtype), 'Stop': np.array([0]).astype(dtype), - 'Num': np.array([1]).astype('int32') + 'Num': np.array([1]).astype('int32'), } self.attrs = {'dtype': int(core.VarDesc.VarType.FP32)} @@ -80,7 +77,6 @@ def test_check_output(self): class TestLinspaceAPI(unittest.TestCase): - def test_variable_input1(self): start = paddle.full(shape=[1], fill_value=0, dtype='float32') stop = paddle.full(shape=[1], fill_value=10, dtype='float32') @@ -106,17 +102,16 @@ def test_dtype(self): out_2 = paddle.linspace(0, 10, 5, dtype=np.float32) out_3 = paddle.linspace(0, 10, 5, dtype=core.VarDesc.VarType.FP32) exe = fluid.Executor(place=fluid.CPUPlace()) - res_1, res_2, res_3 = exe.run(fluid.default_main_program(), - fetch_list=[out_1, out_2, out_3]) + res_1, res_2, res_3 = exe.run( + fluid.default_main_program(), fetch_list=[out_1, out_2, out_3] + ) assert np.array_equal(res_1, res_2) def test_name(self): with paddle.static.program_guard(paddle.static.Program()): - out = paddle.linspace(0, - 10, - 5, - dtype='float32', - name='linspace_res') + out = paddle.linspace( + 0, 10, 5, dtype='float32', name='linspace_res' + ) assert 'linspace_res' in out.name def test_imperative(self): @@ -139,7 +134,6 @@ def test_api_eager_dygraph(self): class TestLinspaceOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): diff --git a/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py b/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py index 897689d559714..3bac2d40ae3a2 100644 --- a/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py +++ b/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py @@ -50,18 +50,21 @@ def run_pserver(use_cuda, sync_mode, ip, port, trainers, trainer_id): config = fluid.DistributeTranspilerConfig() config.sync_mode = sync_mode t = fluid.DistributeTranspiler(config=config) - t.transpile(trainer_id, - pservers=pserver_endpoints, - trainers=trainers, - sync_mode=sync_mode) + t.transpile( + trainer_id, + pservers=pserver_endpoints, + trainers=trainers, + sync_mode=sync_mode, + ) pserver_prog = t.get_pserver_program(current_endpoint) pserver_startup = t.get_startup_program(current_endpoint, pserver_prog) exe.run(pserver_startup) exe.run(pserver_prog) -def run_pserver_with_empty_block(use_cuda, sync_mode, ip, port, trainers, - trainer_id): +def run_pserver_with_empty_block( + use_cuda, sync_mode, ip, port, trainers, trainer_id +): remove_ps_flag(os.getpid()) x = fluid.layers.data(name='x', shape=[1], dtype='float32') y_predict = fluid.layers.fc(input=x, size=1, act=None, bias_attr=False) @@ -87,15 +90,17 @@ def run_pserver_with_empty_block(use_cuda, sync_mode, ip, port, trainers, config.slice_var_up = False t = fluid.DistributeTranspiler(config=config) - t.transpile(trainer_id, - pservers=pserver_endpoints, - trainers=trainers, - sync_mode=sync_mode) + t.transpile( + trainer_id, + pservers=pserver_endpoints, + trainers=trainers, + sync_mode=sync_mode, + ) pserver_prog = t.get_pserver_program(ps2) # pserver2 have no parameter - assert (len(pserver_prog.blocks) == 2) - assert (len(pserver_prog.blocks[1].ops) == 0) + assert len(pserver_prog.blocks) == 2 + assert len(pserver_prog.blocks[1].ops) == 0 pserver_startup = t.get_startup_program(ps2, pserver_prog) exe.run(pserver_startup) @@ -108,7 +113,6 @@ def gen_complete_file_flag(flag_file): class TestListenAndServOp(unittest.TestCase): - def setUp(self): self.ps_timeout = 200 self.ip = "127.0.0.1" @@ -117,9 +121,17 @@ def setUp(self): self.trainer_id = 0 def _start_pserver(self, use_cuda, sync_mode, pserver_func): - p = Process(target=pserver_func, - args=(use_cuda, sync_mode, self.ip, self.port, - self.trainers, self.trainer_id)) + p = Process( + target=pserver_func, + args=( + use_cuda, + sync_mode, + self.ip, + self.port, + self.trainers, + self.trainer_id, + ), + ) p.daemon = True p.start() return p diff --git a/python/paddle/fluid/tests/unittests/test_load_op.py b/python/paddle/fluid/tests/unittests/test_load_op.py index 1b81f3b8f4096..ed123e06a03c5 100644 --- a/python/paddle/fluid/tests/unittests/test_load_op.py +++ b/python/paddle/fluid/tests/unittests/test_load_op.py @@ -21,8 +21,7 @@ class TestLoadOp(unittest.TestCase): - """ Test load operator. - """ + """Test load operator.""" def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -37,13 +36,17 @@ def setUp(self): param_attr=fluid.ParamAttr( name='w', initializer=fluid.initializer.NumpyArrayInitializer( - self.ones))) + self.ones + ), + ), + ) exe = fluid.Executor(fluid.CPUPlace()) exe.run(start_prog) - fluid.io.save_persistables(exe, - dirname=os.path.join(self.temp_dir.name, - "./model"), - main_program=main_prog) + fluid.io.save_persistables( + exe, + dirname=os.path.join(self.temp_dir.name, "./model"), + main_program=main_prog, + ) def tearDown(self): self.temp_dir.cleanup() @@ -53,8 +56,9 @@ def test_load(self): start_prog = fluid.Program() with fluid.program_guard(main_prog, start_prog): var = layers.create_tensor(dtype='float32') - layers.load(var, - file_path=os.path.join(self.temp_dir.name, './model/w')) + layers.load( + var, file_path=os.path.join(self.temp_dir.name, './model/w') + ) exe = fluid.Executor(fluid.CPUPlace()) exe.run(start_prog) diff --git a/python/paddle/fluid/tests/unittests/test_load_op_xpu.py b/python/paddle/fluid/tests/unittests/test_load_op_xpu.py index 957fe5aa42075..05ad3dc77626e 100644 --- a/python/paddle/fluid/tests/unittests/test_load_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/test_load_op_xpu.py @@ -21,11 +21,11 @@ import paddle -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestLoadOpXpu(unittest.TestCase): - """ Test load operator. - """ + """Test load operator.""" def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -41,12 +41,15 @@ def setUp(self): param_attr=fluid.ParamAttr( name='w', initializer=fluid.initializer.NumpyArrayInitializer( - self.ones))) + self.ones + ), + ), + ) exe = fluid.Executor(fluid.XPUPlace(0)) exe.run(start_prog) - fluid.io.save_persistables(exe, - dirname=self.model_path, - main_program=main_prog) + fluid.io.save_persistables( + exe, dirname=self.model_path, main_program=main_prog + ) def tearDown(self): self.temp_dir.cleanup() diff --git a/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py b/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py index 4d47c94f2fabf..6bace212953a6 100644 --- a/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py +++ b/python/paddle/fluid/tests/unittests/test_load_state_dict_from_old_format.py @@ -24,19 +24,23 @@ def convolutional_neural_network(img): - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) prediction = fluid.layers.fc(input=conv_pool_2, size=10, act='softmax') return prediction @@ -54,7 +58,6 @@ def static_train_net(img, label): class TestLoadStateDictFromSaveInferenceModel(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.seed = 90 @@ -72,30 +75,38 @@ def train_and_save_model(self, only_params=False): startup_program = fluid.default_startup_program() main_program = fluid.default_main_program() - img = fluid.data(name='img', - shape=[None, 1, 28, 28], - dtype='float32') + img = fluid.data( + name='img', shape=[None, 1, 28, 28], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') prediction, avg_loss = static_train_net(img, label) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=[img, label], place=place) exe.run(startup_program) - train_reader = paddle.batch(paddle.reader.shuffle( - paddle.dataset.mnist.train(), buf_size=100), - batch_size=self.batch_size) + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.mnist.train(), buf_size=100 + ), + batch_size=self.batch_size, + ) for _ in range(0, self.epoch_num): for batch_id, data in enumerate(train_reader()): - exe.run(main_program, - feed=feeder.feed(data), - fetch_list=[avg_loss]) + exe.run( + main_program, + feed=feeder.feed(data), + fetch_list=[avg_loss], + ) if batch_id > self.batch_num: break @@ -103,18 +114,22 @@ def train_and_save_model(self, only_params=False): static_param_dict = {} for param in fluid.default_main_program().all_parameters(): static_param_dict[param.name] = fluid.executor._fetch_var( - param.name) + param.name + ) if only_params: - fluid.io.save_params(exe, - self.save_dirname, - filename=self.params_filename) + fluid.io.save_params( + exe, self.save_dirname, filename=self.params_filename + ) else: fluid.io.save_inference_model( - self.save_dirname, ["img"], [prediction], + self.save_dirname, + ["img"], + [prediction], exe, model_filename=self.model_filename, - params_filename=self.params_filename) + params_filename=self.params_filename, + ) return static_param_dict @@ -124,7 +139,8 @@ def check_load_state_dict(self, orig_dict, load_dict): def test_load_default(self): self.save_dirname = os.path.join( - self.temp_dir.name, "static_mnist.load_state_dict.default") + self.temp_dir.name, "static_mnist.load_state_dict.default" + ) self.model_filename = None self.params_filename = None orig_param_dict = self.train_and_save_model() @@ -137,38 +153,45 @@ def test_load_default(self): def test_load_with_model_filename(self): self.save_dirname = os.path.join( - self.temp_dir.name, "static_mnist.load_state_dict.model_filename") + self.temp_dir.name, "static_mnist.load_state_dict.model_filename" + ) self.model_filename = "static_mnist.model" self.params_filename = None orig_param_dict = self.train_and_save_model() load_param_dict, _ = fluid.load_dygraph( - self.save_dirname, model_filename=self.model_filename) + self.save_dirname, model_filename=self.model_filename + ) self.check_load_state_dict(orig_param_dict, load_param_dict) - new_load_param_dict = paddle.load(self.save_dirname, - model_filename=self.model_filename) + new_load_param_dict = paddle.load( + self.save_dirname, model_filename=self.model_filename + ) self.check_load_state_dict(orig_param_dict, new_load_param_dict) def test_load_with_param_filename(self): self.save_dirname = os.path.join( - self.temp_dir.name, "static_mnist.load_state_dict.param_filename") + self.temp_dir.name, "static_mnist.load_state_dict.param_filename" + ) self.model_filename = None self.params_filename = "static_mnist.params" orig_param_dict = self.train_and_save_model() load_param_dict, _ = fluid.load_dygraph( - self.save_dirname, params_filename=self.params_filename) + self.save_dirname, params_filename=self.params_filename + ) self.check_load_state_dict(orig_param_dict, load_param_dict) - new_load_param_dict = paddle.load(self.save_dirname, - params_filename=self.params_filename) + new_load_param_dict = paddle.load( + self.save_dirname, params_filename=self.params_filename + ) self.check_load_state_dict(orig_param_dict, new_load_param_dict) def test_load_with_model_and_param_filename(self): self.save_dirname = os.path.join( self.temp_dir.name, - "static_mnist.load_state_dict.model_and_param_filename") + "static_mnist.load_state_dict.model_and_param_filename", + ) self.model_filename = "static_mnist.model" self.params_filename = "static_mnist.params" orig_param_dict = self.train_and_save_model() @@ -176,17 +199,21 @@ def test_load_with_model_and_param_filename(self): load_param_dict, _ = fluid.load_dygraph( self.save_dirname, params_filename=self.params_filename, - model_filename=self.model_filename) + model_filename=self.model_filename, + ) self.check_load_state_dict(orig_param_dict, load_param_dict) - new_load_param_dict = paddle.load(self.save_dirname, - params_filename=self.params_filename, - model_filename=self.model_filename) + new_load_param_dict = paddle.load( + self.save_dirname, + params_filename=self.params_filename, + model_filename=self.model_filename, + ) self.check_load_state_dict(orig_param_dict, new_load_param_dict) def test_load_state_dict_from_save_params(self): self.save_dirname = os.path.join( - self.temp_dir.name, "static_mnist.load_state_dict.save_params") + self.temp_dir.name, "static_mnist.load_state_dict.save_params" + ) self.params_filename = None orig_param_dict = self.train_and_save_model(True) diff --git a/python/paddle/fluid/tests/unittests/test_load_vars_shape_check.py b/python/paddle/fluid/tests/unittests/test_load_vars_shape_check.py index 2d9bd1ac65090..756b2ab77320e 100644 --- a/python/paddle/fluid/tests/unittests/test_load_vars_shape_check.py +++ b/python/paddle/fluid/tests/unittests/test_load_vars_shape_check.py @@ -20,7 +20,6 @@ class TestLoadVarsShapeCheck(unittest.TestCase): - def setUp(self): self.model_path = "./model_temp/" diff --git a/python/paddle/fluid/tests/unittests/test_locality_aware_nms_op.py b/python/paddle/fluid/tests/unittests/test_locality_aware_nms_op.py index b79b1bff8b94d..a09707c5767a9 100644 --- a/python/paddle/fluid/tests/unittests/test_locality_aware_nms_op.py +++ b/python/paddle/fluid/tests/unittests/test_locality_aware_nms_op.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np @@ -25,13 +25,15 @@ def weight_merge(box1, box2, score1, score2): box2[i] = (box1[i] * score1 + box2[i] * score2) / (score1 + score2) -def nms(boxes, - scores, - score_threshold, - nms_threshold, - top_k=200, - normalized=True, - eta=1.0): +def nms( + boxes, + scores, + score_threshold, + nms_threshold, + top_k=200, + normalized=True, + eta=1.0, +): """Apply non-maximum suppression at test time to avoid detecting too many overlapping bounding boxes for a given object. Args: @@ -48,11 +50,13 @@ def nms(boxes, """ index = -1 for i in range(boxes.shape[0]): - if index > -1 and iou(boxes[i], boxes[index], - normalized) > nms_threshold: + if ( + index > -1 + and iou(boxes[i], boxes[index], normalized) > nms_threshold + ): weight_merge(boxes[i], boxes[index], scores[i], scores[index]) scores[index] += scores[i] - scores[i] = score_threshold - 1. + scores[i] = score_threshold - 1.0 else: index = i @@ -90,8 +94,17 @@ def nms(boxes, return selected_indices -def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, - nms_top_k, keep_top_k, normalized, shared): +def multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + normalized, + shared, +): if shared: class_num = scores.shape[0] priorbox_num = scores.shape[1] @@ -102,13 +115,26 @@ def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, selected_indices = {} num_det = 0 for c in range(class_num): - if c == background: continue + if c == background: + continue if shared: - indices = nms(boxes, scores[c], score_threshold, nms_threshold, - nms_top_k, normalized) + indices = nms( + boxes, + scores[c], + score_threshold, + nms_threshold, + nms_top_k, + normalized, + ) else: - indices = nms(boxes[:, c, :], scores[:, c], score_threshold, - nms_threshold, nms_top_k, normalized) + indices = nms( + boxes[:, c, :], + scores[:, c], + score_threshold, + nms_threshold, + nms_top_k, + normalized, + ) selected_indices[c] = indices num_det += len(indices) @@ -121,9 +147,9 @@ def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, else: score_index.append((scores[idx][c], c, idx)) - sorted_score_index = sorted(score_index, - key=lambda tup: tup[0], - reverse=True) + sorted_score_index = sorted( + score_index, key=lambda tup: tup[0], reverse=True + ) sorted_score_index = sorted_score_index[:keep_top_k] selected_indices = {} @@ -139,29 +165,33 @@ def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, return selected_indices, num_det -def batched_multiclass_nms(boxes, - scores, - background, - score_threshold, - nms_threshold, - nms_top_k, - keep_top_k, - normalized=True): +def batched_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + normalized=True, +): batch_size = scores.shape[0] num_boxes = scores.shape[2] det_outs = [] lod = [] for n in range(batch_size): - nmsed_outs, nmsed_num = multiclass_nms(boxes[n], - scores[n], - background, - score_threshold, - nms_threshold, - nms_top_k, - keep_top_k, - normalized, - shared=True) + nmsed_outs, nmsed_num = multiclass_nms( + boxes[n], + scores[n], + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + normalized, + shared=True, + ) lod.append(nmsed_num) if nmsed_num == 0: @@ -170,19 +200,25 @@ def batched_multiclass_nms(boxes, for c, indices in nmsed_outs.items(): for idx in indices: xmin, ymin, xmax, ymax = boxes[n][idx][:] - tmp_det_out.append([ - c, scores[n][c][idx], xmin, ymin, xmax, ymax, - idx + n * num_boxes - ]) - sorted_det_out = sorted(tmp_det_out, - key=lambda tup: tup[0], - reverse=False) + tmp_det_out.append( + [ + c, + scores[n][c][idx], + xmin, + ymin, + xmax, + ymax, + idx + n * num_boxes, + ] + ) + sorted_det_out = sorted( + tmp_det_out, key=lambda tup: tup[0], reverse=False + ) det_outs.extend(sorted_det_out) return det_outs, lod class TestLocalAwareNMSOp(OpTest): - def set_argument(self): self.score_threshold = 0.01 @@ -203,7 +239,7 @@ def setUp(self): def softmax(x): # clip to shiftx, otherwise, when calc loss with # log(exp(shiftx)), may get log(0)=INF - shiftx = (x - np.max(x)).clip(-64.) + shiftx = (x - np.max(x)).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) @@ -217,10 +253,15 @@ def softmax(x): boxes_copy = copy.deepcopy(boxes) scores_copy = copy.deepcopy(scores) - det_outs, lod = batched_multiclass_nms(boxes_copy, scores_copy, - background, score_threshold, - nms_threshold, nms_top_k, - keep_top_k) + det_outs, lod = batched_multiclass_nms( + boxes_copy, + scores_copy, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + ) lod = [1] if not det_outs else lod det_outs = [[-1, 0]] if not det_outs else det_outs @@ -245,13 +286,11 @@ def test_check_output(self): class TestLocalAwareNMSOpNoBoxes(TestLocalAwareNMSOp): - def set_argument(self): self.score_threshold = 2.0 class TestLocalAwareNMSOp4Points(OpTest): - def set_argument(self): self.score_threshold = 0.01 @@ -266,41 +305,93 @@ def setUp(self): nms_threshold = 0.3 score_threshold = self.score_threshold - scores = np.array([[[0.76319082, 0.73770091]], - [[0.68513154, 0.45952697]]]) - boxes = np.array([[[ - 0.42078365, 0.58117018, 2.92776169, 3.28557757, 4.24344318, - 0.92196165, 2.72370856, -1.66141214 - ], - [ - 0.13856006, 1.86871034, 2.81287224, 3.61381734, - 4.5505249, 0.51766346, 2.75630304, -1.91459389 - ]], - [[ - 1.57533883, 1.3217477, 3.07904942, 3.89512545, - 4.78680923, 1.96914586, 3.539482, -1.59739244 - ], - [ - 0.55084125, 1.71596215, 2.52476074, 3.18940435, - 5.09035159, 0.91959482, 3.71442385, -0.57299128 - ]]]) - - det_outs = np.array([[ - 0., 1.5008917, 0.28206837, 1.2140071, 2.8712926, 3.4469104, - 4.3943763, 0.7232457, 2.7397292, -1.7858533 - ], - [ - 0., 1.1446586, 1.1640508, 1.4800063, 2.856528, - 3.6118112, 4.908667, 1.5478, 3.609713, - -1.1861432 - ]]) + scores = np.array( + [[[0.76319082, 0.73770091]], [[0.68513154, 0.45952697]]] + ) + boxes = np.array( + [ + [ + [ + 0.42078365, + 0.58117018, + 2.92776169, + 3.28557757, + 4.24344318, + 0.92196165, + 2.72370856, + -1.66141214, + ], + [ + 0.13856006, + 1.86871034, + 2.81287224, + 3.61381734, + 4.5505249, + 0.51766346, + 2.75630304, + -1.91459389, + ], + ], + [ + [ + 1.57533883, + 1.3217477, + 3.07904942, + 3.89512545, + 4.78680923, + 1.96914586, + 3.539482, + -1.59739244, + ], + [ + 0.55084125, + 1.71596215, + 2.52476074, + 3.18940435, + 5.09035159, + 0.91959482, + 3.71442385, + -0.57299128, + ], + ], + ] + ) + + det_outs = np.array( + [ + [ + 0.0, + 1.5008917, + 0.28206837, + 1.2140071, + 2.8712926, + 3.4469104, + 4.3943763, + 0.7232457, + 2.7397292, + -1.7858533, + ], + [ + 0.0, + 1.1446586, + 1.1640508, + 1.4800063, + 2.856528, + 3.6118112, + 4.908667, + 1.5478, + 3.609713, + -1.1861432, + ], + ] + ) lod = [1, 1] nmsed_outs = det_outs.astype('float32') self.op_type = 'locality_aware_nms' self.inputs = { 'BBoxes': boxes.astype('float32'), - 'Scores': scores.astype('float32') + 'Scores': scores.astype('float32'), } self.outputs = {'Out': (nmsed_outs, [lod])} self.attrs = { @@ -309,7 +400,7 @@ def setUp(self): 'nms_top_k': nms_top_k, 'keep_top_k': keep_top_k, 'background_label': -1, - 'normalized': False + 'normalized': False, } def test_check_output(self): @@ -317,75 +408,151 @@ def test_check_output(self): class TestLocalityAwareNMSAPI(unittest.TestCase): - def test_api(self): boxes = fluid.data(name='bboxes', shape=[None, 81, 8], dtype='float32') scores = fluid.data(name='scores', shape=[None, 1, 81], dtype='float32') - fluid.layers.locality_aware_nms(bboxes=boxes, - scores=scores, - score_threshold=0.5, - nms_top_k=400, - nms_threshold=0.3, - keep_top_k=200, - normalized=False) + fluid.layers.locality_aware_nms( + bboxes=boxes, + scores=scores, + score_threshold=0.5, + nms_top_k=400, + nms_threshold=0.3, + keep_top_k=200, + normalized=False, + ) class TestLocalityAwareNMSError(unittest.TestCase): - def test_error(self): boxes = fluid.data(name='bboxes', shape=[None, 81, 8], dtype='float32') scores = fluid.data(name='scores', shape=[None, 1, 81], dtype='float32') - boxes_int = fluid.data(name='bboxes_int', - shape=[None, 81, 8], - dtype='int32') - scores_int = fluid.data(name='scores_int', - shape=[None, 1, 81], - dtype='int32') + boxes_int = fluid.data( + name='bboxes_int', shape=[None, 81, 8], dtype='int32' + ) + scores_int = fluid.data( + name='scores_int', shape=[None, 1, 81], dtype='int32' + ) boxes_tmp = [1, 2] scores_tmp = [1, 2] # type of boxes and scores must be variable - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes_tmp, - scores, 0.5, 400, 200) - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores_tmp, 0.5, 400, 200) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes_tmp, + scores, + 0.5, + 400, + 200, + ) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores_tmp, + 0.5, + 400, + 200, + ) # dtype of boxes and scores must in ['float32', 'float64'] - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes_int, - scores, 0.5, 400, 200) - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores_int, 0.5, 400, 200) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes_int, + scores, + 0.5, + 400, + 200, + ) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores_int, + 0.5, + 400, + 200, + ) score_threshold = int(1) # type of score_threshold must be float - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores, score_threshold, 400, 200) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores, + score_threshold, + 400, + 200, + ) nms_top_k = 400.5 # type of num_top_k must be int - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores, 0.5, nms_top_k, 200) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores, + 0.5, + nms_top_k, + 200, + ) keep_top_k = 200.5 # type of keep_top_k must be int - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores, 0.5, 400, keep_top_k) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores, + 0.5, + 400, + keep_top_k, + ) nms_threshold = int(0) # type of nms_threshold must be int - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores, 0.5, 400, 200, nms_threshold) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores, + 0.5, + 400, + 200, + nms_threshold, + ) nms_eta = int(1) # type of nms_eta must be float - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores, 0.5, 400, 200, 0.5, nms_eta) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores, + 0.5, + 400, + 200, + 0.5, + nms_eta, + ) bg_label = 1.5 # type of background_label must be int - self.assertRaises(TypeError, fluid.layers.locality_aware_nms, boxes, - scores, 0.5, 400, 200, 0.5, 1.0, bg_label) + self.assertRaises( + TypeError, + fluid.layers.locality_aware_nms, + boxes, + scores, + 0.5, + 400, + 200, + 0.5, + 1.0, + bg_label, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_lod_append_op.py b/python/paddle/fluid/tests/unittests/test_lod_append_op.py index 23d828b3cd78a..721a247d5d600 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_append_op.py +++ b/python/paddle/fluid/tests/unittests/test_lod_append_op.py @@ -1,4 +1,4 @@ -#Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,15 +19,13 @@ class TestLoDAppendAPI(unittest.TestCase): - def test_api(self, use_cuda=False): main_program = Program() with fluid.program_guard(main_program): x = fluid.layers.data(name='x', shape=[6], dtype='float32') - level = fluid.layers.data(name='level', - shape=[3], - dtype='int32', - lod_level=0) + level = fluid.layers.data( + name='level', shape=[3], dtype='int32', lod_level=0 + ) result = fluid.layers.lod_append(x, level) x_i = np.array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0]).astype("float32") @@ -38,35 +36,32 @@ def test_api(self, use_cuda=False): return place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) - [out] = exe.run(fluid.default_main_program(), - feed={ - 'x': x_i, - 'level': level_i - }, - fetch_list=[result], - return_numpy=False) + [out] = exe.run( + fluid.default_main_program(), + feed={'x': x_i, 'level': level_i}, + fetch_list=[result], + return_numpy=False, + ) self.assertEqual(out.recursive_sequence_lengths(), [[2, 4]]) class TestLodAppendOpError(unittest.TestCase): - def test_error(self): # The input(x) must be Variable. x1 = np.array([0.9383, 0.1983, 3.2, 1.2]).astype("float64") level1 = [0, 2, 4] self.assertRaises(TypeError, fluid.layers.lod_append, x1, level1) - #The input(level) must be Variable or list. + # The input(level) must be Variable or list. x2 = fluid.layers.data(name='x2', shape=[4], dtype='float32') self.assertRaises(ValueError, fluid.layers.lod_append, x2, 2) # Input(x) dtype must be float32 or float64 or int32 or int64 for dtype in ["bool", "float16"]: x3 = fluid.layers.data(name='x3_' + dtype, shape=[4], dtype=dtype) - level3 = fluid.layers.data(name='level3' + dtype, - shape=[4], - dtype='int32', - lod_level=2) + level3 = fluid.layers.data( + name='level3' + dtype, shape=[4], dtype='int32', lod_level=2 + ) self.assertRaises(TypeError, fluid.layers.lod_append, x3, level3) diff --git a/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py b/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py index 4129c59edca1e..4b702d0a0f0ac 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py +++ b/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py @@ -24,7 +24,6 @@ class TestLoDArrayLength(unittest.TestCase): - def test_array_length(self): tmp = layers.zeros(shape=[10], dtype='int32') i = layers.fill_constant(shape=[1], dtype='int64', value=10) @@ -37,17 +36,15 @@ def test_array_length(self): class TestLoDArrayLengthOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - #for ci coverage + # for ci coverage x1 = numpy.random.randn(2, 4).astype('int32') self.assertRaises(TypeError, fluid.layers.array_length, array=x1) class TestArrayLengthApi(unittest.TestCase): - def test_api(self): paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_lod_rank_table.py b/python/paddle/fluid/tests/unittests/test_lod_rank_table.py index 7a6b6857ffbfa..88ad04a52c205 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_rank_table.py +++ b/python/paddle/fluid/tests/unittests/test_lod_rank_table.py @@ -21,7 +21,6 @@ class TestLoDRankTable(unittest.TestCase): - def test_lod_rank_table(self): x = data(name='x', shape=[100]) cpu = core.CPUPlace() @@ -32,8 +31,9 @@ def test_lod_rank_table(self): tensor = core.LoDTensor() tensor.set(numpy.random.random(size=(17, 100)), cpu) - tensor.set_recursive_sequence_lengths([[1, 2], [5, 1, 1], - [3, 1, 5, 1, 3, 3, 1]]) + tensor.set_recursive_sequence_lengths( + [[1, 2], [5, 1, 1], [3, 1, 5, 1, 3, 3, 1]] + ) exe.run(scope=scope, feed={'x': tensor}) var = scope.find_var(rank_table.name) table = var.get_lod_rank_table() @@ -41,7 +41,6 @@ def test_lod_rank_table(self): class TestLoDRankTableError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x = numpy.random.random((2, 4)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_lod_reset_op.py b/python/paddle/fluid/tests/unittests/test_lod_reset_op.py index 8c814e984e28c..5e9bd45907a40 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_reset_op.py +++ b/python/paddle/fluid/tests/unittests/test_lod_reset_op.py @@ -20,7 +20,6 @@ class TestLodResetOpByAttr(OpTest): - def setUp(self): self.op_type = "lod_reset" x = np.random.random((10, 20)).astype("float64") @@ -44,7 +43,6 @@ def test_check_grad(self): class TestLodResetOpByInput(OpTest): - def setUp(self): self.op_type = "lod_reset" x = np.random.random((10, 20)).astype("float64") @@ -55,7 +53,7 @@ def setUp(self): target_lod = [4, 3, 3] self.inputs = { 'X': (x, lod), - 'Y': np.array([target_offset_lod]).astype('int32') + 'Y': np.array([target_offset_lod]).astype('int32'), } self.outputs = {'Out': (x, [target_lod])} @@ -69,7 +67,6 @@ def test_check_grad(self): class TestLodResetOpBoth(OpTest): - def setUp(self): self.op_type = "lod_reset" x = np.random.random((10, 20)).astype("float64") @@ -79,7 +76,7 @@ def setUp(self): target_lod_in = [4, 3, 3] self.inputs = { 'X': (x, lod), - 'Y': np.array(target_offset_lod_in).astype('int32') + 'Y': np.array(target_offset_lod_in).astype('int32'), } self.attrs = {'target_lod': target_offset_lod_attr} self.outputs = {'Out': (x, [target_lod_in])} @@ -94,7 +91,6 @@ def test_check_grad(self): class TestLodResetOpYIsLoDTensor(OpTest): - def setUp(self): self.op_type = "lod_reset" x = np.random.random((10, 20)).astype("float64") @@ -114,7 +110,6 @@ def test_check_grad(self): class TestLodAppendOpByAttr(OpTest): - def setUp(self): self.op_type = "lod_reset" x = np.random.random((10, 20)).astype("float64") @@ -138,7 +133,6 @@ def test_check_grad(self): class TestLodResetOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input must be Variable. @@ -148,13 +142,12 @@ def test_errors(self): # Input(x) dtype must be float32 or float64 or int32 or int64 for dtype in ["bool", "float16"]: - x2 = fluid.layers.data(name='x2' + dtype, - shape=[4], - dtype=dtype) - y2 = fluid.layers.data(name='y2' + dtype, - shape=[4], - dtype='int32', - lod_level=2) + x2 = fluid.layers.data( + name='x2' + dtype, shape=[4], dtype=dtype + ) + y2 = fluid.layers.data( + name='y2' + dtype, shape=[4], dtype='int32', lod_level=2 + ) self.assertRaises(TypeError, fluid.layers.lod_reset, x2, y2) diff --git a/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py b/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py index 685c48e96877c..c6656ff1caa66 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py +++ b/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py @@ -19,7 +19,6 @@ class TestLoDTensorArray(unittest.TestCase): - def test_get_set(self): scope = core.Scope() arr = scope.var('tmp_lod_tensor_array') @@ -49,7 +48,6 @@ def test_get_set(self): class TestCreateArray(unittest.TestCase): - def setUp(self): self.place = paddle.CPUPlace() self.shapes = [[10, 4], [8, 12], [1]] @@ -60,7 +58,8 @@ def test_initialized_list_and_error(self): np.random.random(shape).astype('float32') for shape in self.shapes ] array = paddle.tensor.create_array( - 'float32', [paddle.to_tensor(x) for x in init_data]) + 'float32', [paddle.to_tensor(x) for x in init_data] + ) for res, gt in zip(array, init_data): np.testing.assert_array_equal(res, gt) @@ -82,8 +81,9 @@ def test_static(self): # test error with nest list with self.assertRaises(TypeError): - paddle.tensor.create_array('float32', - [init_data[0], [init_data[1]]]) + paddle.tensor.create_array( + 'float32', [init_data[0], [init_data[1]]] + ) # test error with not variable with self.assertRaises(TypeError): diff --git a/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py b/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py index 9cc9a5540aa35..418c95901edd0 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py +++ b/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py @@ -28,7 +28,6 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): - def place(self): return core.CPUPlace() @@ -40,10 +39,12 @@ def test_lod_tensor_to_array_level_0(self): np.array(x).astype('int32') for x in [[3, 0, 9], [4, 1], [5, 2], [6], [7], [8]] ] - self.main(tensor=tensor, - expect_array=expect, - expect_lod=[] * 6, - expect_max_len=6) + self.main( + tensor=tensor, + expect_array=expect, + expect_lod=[] * 6, + expect_max_len=6, + ) def test_lod_tensor_to_array_level_0_empty_seq(self): tensor = core.LoDTensor() @@ -53,10 +54,12 @@ def test_lod_tensor_to_array_level_0_empty_seq(self): np.array(x).astype('int32') for x in [[3, 0, 9], [4, 1], [5, 2], [6], [7], [8]] ] - self.main(tensor=tensor, - expect_array=expect, - expect_lod=[] * 6, - expect_max_len=6) + self.main( + tensor=tensor, + expect_array=expect, + expect_lod=[] * 6, + expect_max_len=6, + ) def test_lod_tensor_to_array_level_1(self): tensor = core.LoDTensor() @@ -66,65 +69,82 @@ def test_lod_tensor_to_array_level_1(self): expect = [ np.array([9, 10, 0, 1, 2], dtype='int32'), np.array([11, 12, 13, 14, 15, 16, 3, 4, 5, 6, 7, 8], dtype='int32'), - np.array([17, 18, 19], dtype='int32') + np.array([17, 18, 19], dtype='int32'), ] lod = [[[2, 3]], [[6, 6]], [[3]]] - self.main(tensor=tensor, - expect_array=expect, - expect_lod=lod, - expect_max_len=3) + self.main( + tensor=tensor, expect_array=expect, expect_lod=lod, expect_max_len=3 + ) def test_lod_tensor_to_array_level_1_empty_seq(self): tensor = core.LoDTensor() tensor.set(np.arange(31).reshape(31, 1).astype('int32'), self.place()) tensor.set_recursive_sequence_lengths( - [[3, 2, 4, 2], [3, 4, 4, 0, 1, 5, 2, 2, 2, 7, 1]]) + [[3, 2, 4, 2], [3, 4, 4, 0, 1, 5, 2, 2, 2, 7, 1]] + ) expect = [ - np.array(item, dtype='int32') for item in [[ - 12, 13, 14, 15, 16, 0, 1, 2, 23, 24, 25, 26, 27, 28, 29 - ], [17, 18, 3, 4, 5, 6, 11, 30], [19, 20, 7, 8, 9, 10], [21, 22]] + np.array(item, dtype='int32') + for item in [ + [12, 13, 14, 15, 16, 0, 1, 2, 23, 24, 25, 26, 27, 28, 29], + [17, 18, 3, 4, 5, 6, 11, 30], + [19, 20, 7, 8, 9, 10], + [21, 22], + ] ] lod = [[[5, 3, 0, 7]], [[2, 4, 1, 1]], [[2, 4]], [[2]]] - self.main(tensor=tensor, - expect_array=expect, - expect_lod=lod, - expect_max_len=4) + self.main( + tensor=tensor, expect_array=expect, expect_lod=lod, expect_max_len=4 + ) def test_lod_tensor_to_array_level_2(self): tensor = core.LoDTensor() tensor.set(np.arange(50).reshape(50, 1).astype('int32'), self.place()) tensor.set_recursive_sequence_lengths( - [[2, 3, 1], [2, 3, 1, 4, 2, 1], - [3, 4, 4, 6, 4, 1, 1, 4, 4, 8, 6, 1, 4]]) + [ + [2, 3, 1], + [2, 3, 1, 4, 2, 1], + [3, 4, 4, 6, 4, 1, 1, 4, 4, 8, 6, 1, 4], + ] + ) expect = [ np.array(item, dtype='int32') - for item in [[21, 0, 1, 2, 3, 4, 5, 6, 46, 47, 48, 49], - list(range(22, 39)) + list(range(7, 21)), - list(range(39, 46))] + for item in [ + [21, 0, 1, 2, 3, 4, 5, 6, 46, 47, 48, 49], + list(range(22, 39)) + list(range(7, 21)), + list(range(39, 46)), + ] ] - lod = [[[1, 2, 1], [1, 3, 4, 4]], [[4, 3], [1, 4, 4, 8, 4, 6, 4]], - [[2], [6, 1]]] - self.main(tensor=tensor, - expect_array=expect, - expect_lod=lod, - expect_max_len=3) + lod = [ + [[1, 2, 1], [1, 3, 4, 4]], + [[4, 3], [1, 4, 4, 8, 4, 6, 4]], + [[2], [6, 1]], + ] + self.main( + tensor=tensor, expect_array=expect, expect_lod=lod, expect_max_len=3 + ) def test_lod_tensor_to_array_level_2_skip_level(self): tensor = core.LoDTensor() tensor.set(np.arange(50).reshape(50, 1).astype('int32'), self.place()) tensor.set_recursive_sequence_lengths( - [[2, 3, 1], [2, 3, 1, 4, 2, 1], - [3, 4, 4, 6, 4, 1, 1, 4, 4, 8, 6, 1, 4]]) - self.main(tensor=tensor, - expect_array=None, - expect_lod=None, - expect_max_len=4, - level=1) + [ + [2, 3, 1], + [2, 3, 1, 4, 2, 1], + [3, 4, 4, 6, 4, 1, 1, 4, 4, 8, 6, 1, 4], + ] + ) + self.main( + tensor=tensor, + expect_array=None, + expect_lod=None, + expect_max_len=4, + level=1, + ) def main(self, tensor, expect_array, expect_lod, expect_max_len, level=0): place = self.place() @@ -151,37 +171,38 @@ def main(self, tensor, expect_array, expect_lod, expect_max_len, level=0): self.assertEqual( np.array(scope.find_var(max_len.name).get_tensor())[0], - expect_max_len) + expect_max_len, + ) def check_array_same(self, array, expect_tensor, expect_lod): self.assertEqual(len(expect_tensor), len(array)) for i, exp in enumerate(zip(expect_tensor, expect_lod)): exp_tensor, exp_lod = exp exp_tensor = np.expand_dims(exp_tensor, axis=1) - np.testing.assert_allclose(exp_tensor, - np.array(array[i]), - rtol=1e-05) + np.testing.assert_allclose( + exp_tensor, np.array(array[i]), rtol=1e-05 + ) self.assertEqual(exp_lod, array[i].recursive_sequence_lengths()) def check_tensor_same(self, actual, expect): - np.testing.assert_allclose(np.array(actual), - np.array(expect), - rtol=1e-05) - self.assertEqual(actual.recursive_sequence_lengths(), - expect.recursive_sequence_lengths()) + np.testing.assert_allclose( + np.array(actual), np.array(expect), rtol=1e-05 + ) + self.assertEqual( + actual.recursive_sequence_lengths(), + expect.recursive_sequence_lengths(), + ) class TestCPULoDTensorArrayOpGrad(unittest.TestCase): - def test_grad(self): place = core.CPUPlace() program = Program() with program_guard(program): - x = layers.data(name='x', - shape=[1], - dtype='float32', - stop_gradient=False) + x = layers.data( + name='x', shape=[1], dtype='float32', stop_gradient=False + ) table = lod_rank_table(x, level=0) array = lod_tensor_to_array(x, table) result = array_to_lod_tensor(array, table) @@ -198,10 +219,13 @@ def test_grad(self): exe = Executor(place) g_out = [ - np.array(item).sum() for item in exe.run(program, - feed={'x': tensor}, - fetch_list=[g_vars], - return_numpy=False) + np.array(item).sum() + for item in exe.run( + program, + feed={'x': tensor}, + fetch_list=[g_vars], + return_numpy=False, + ) ] g_out_sum = np.array(g_out).sum() @@ -209,7 +233,6 @@ def test_grad(self): class TestLoDTensorArrayError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x = np.random.random((10)).astype("float32") @@ -242,7 +265,6 @@ def test_table_list_Variable(): class TestArrayLoDTensorError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x = np.random.random((10)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_log_loss_op.py b/python/paddle/fluid/tests/unittests/test_log_loss_op.py index 28681da7882c5..35d31128732bd 100644 --- a/python/paddle/fluid/tests/unittests/test_log_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_log_loss_op.py @@ -23,7 +23,6 @@ def sigmoid_array(x): class TestLogLossOp(OpTest): - def setUp(self): self.op_type = 'log_loss' samples_num = 100 @@ -38,8 +37,9 @@ def setUp(self): } self.attrs = {'epsilon': epsilon} - loss = -labels * np.log(predicted + epsilon) - ( - 1 - labels) * np.log(1 - predicted + epsilon) + loss = -labels * np.log(predicted + epsilon) - (1 - labels) * np.log( + 1 - predicted + epsilon + ) self.outputs = {'Loss': loss} def test_check_output(self): @@ -50,7 +50,6 @@ def test_check_grad(self): class TestLogLossOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program()): diff --git a/python/paddle/fluid/tests/unittests/test_log_softmax.py b/python/paddle/fluid/tests/unittests/test_log_softmax.py index 8f1fb2c1da28f..4be4319d805f3 100644 --- a/python/paddle/fluid/tests/unittests/test_log_softmax.py +++ b/python/paddle/fluid/tests/unittests/test_log_softmax.py @@ -23,7 +23,7 @@ def ref_log_softmax(x): - shiftx = (x - np.max(x)) + shiftx = x - np.max(x) out = shiftx - np.log(np.exp(shiftx).sum()) return out @@ -33,14 +33,14 @@ def ref_log_softmax_grad(x, axis): axis += len(x.shape) out = np.apply_along_axis(ref_log_softmax, axis, x) axis_dim = x.shape[axis] - dout = np.full_like(x, fill_value=1. / x.size) + dout = np.full_like(x, fill_value=1.0 / x.size) dx = dout - np.exp(out) * dout.copy().sum(axis=axis, keepdims=True).repeat( - axis_dim, axis=axis) + axis_dim, axis=axis + ) return dx class TestLogSoftmaxOp(OpTest): - def setUp(self): self.op_type = 'log_softmax' self.python_api = F.log_softmax @@ -49,7 +49,7 @@ def setUp(self): self.axis = -1 self.set_attrs() - x = np.random.uniform(0.1, 1., self.shape).astype(self.dtype) + x = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) out = np.apply_along_axis(ref_log_softmax, self.axis, x) self.x_grad = ref_log_softmax_grad(x, self.axis) @@ -64,27 +64,25 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], ['Out'], - user_defined_grads=[self.x_grad], - check_eager=True) + self.check_grad( + ['X'], ['Out'], user_defined_grads=[self.x_grad], check_eager=True + ) class TestLogSoftmaxShape(TestLogSoftmaxOp): - def set_attrs(self): self.shape = [12, 10] class TestLogSoftmaxAxis(TestLogSoftmaxOp): - def set_attrs(self): self.axis = 1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestLogSoftmaxBF16Op(OpTest): - def setUp(self): self.op_type = 'log_softmax' self.python_api = F.log_softmax @@ -92,7 +90,7 @@ def setUp(self): self.shape = [2, 3, 4, 5] self.axis = -1 - x = np.random.uniform(0.1, 1., self.shape).astype(np.float32) + x = np.random.uniform(0.1, 1.0, self.shape).astype(np.float32) out = np.apply_along_axis(ref_log_softmax, self.axis, x) self.x_grad = ref_log_softmax_grad(x, self.axis) @@ -106,19 +104,24 @@ def test_check_output(self): def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], ['Out'], - user_defined_grads=[self.x_grad], - check_eager=True) + self.check_grad_with_place( + place, + ['X'], + ['Out'], + user_defined_grads=[self.x_grad], + check_eager=True, + ) class TestNNLogSoftmaxAPI(unittest.TestCase): - def setUp(self): self.x_shape = [2, 3, 4, 5] - self.x = np.random.uniform(-1., 1., self.x_shape).astype(np.float32) - self.place = paddle.CUDAPlace(0) \ - if paddle.fluid.core.is_compiled_with_cuda() \ + self.x = np.random.uniform(-1.0, 1.0, self.x_shape).astype(np.float32) + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def check_api(self, axis=-1): ref_out = np.apply_along_axis(ref_log_softmax, axis, self.x) @@ -145,13 +148,14 @@ def test_check_api(self): class TestNNFunctionalLogSoftmaxAPI(unittest.TestCase): - def setUp(self): self.x_shape = [2, 3, 4, 5] self.x = np.random.uniform(-1, 1, self.x_shape).astype(np.float32) - self.place = paddle.CUDAPlace(0) \ - if paddle.fluid.core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def check_api(self, axis=-1, dtype=None): x = self.x.copy() diff --git a/python/paddle/fluid/tests/unittests/test_logcumsumexp_op.py b/python/paddle/fluid/tests/unittests/test_logcumsumexp_op.py index 6064303a52178..96f0e4bff6a66 100644 --- a/python/paddle/fluid/tests/unittests/test_logcumsumexp_op.py +++ b/python/paddle/fluid/tests/unittests/test_logcumsumexp_op.py @@ -26,11 +26,13 @@ def np_naive_logcumsumexp(x: np.ndarray, axis: Optional[int] = None): return np.log(np.cumsum(np.exp(x), axis=axis)) -def np_logcumsumexp(x: np.ndarray, - axis: Optional[int] = None, - flatten: Optional[bool] = None, - reverse: bool = False, - exclusive: bool = False): +def np_logcumsumexp( + x: np.ndarray, + axis: Optional[int] = None, + flatten: Optional[bool] = None, + reverse: bool = False, + exclusive: bool = False, +): # `flatten` aligns with c++ op if flatten: assert axis in [0, None] @@ -76,23 +78,30 @@ def np_logcumsumexp_grad( log_grad_negative = np.where(dout < 0, np.log(-dout), np.finfo(x.dtype).min) output_pos = np.exp( - np_logcumsumexp(log_grad_positive - out, - axis=axis, - flatten=flatten, - reverse=not reverse, - exclusive=exclusive).reshape(x.shape) + x) + np_logcumsumexp( + log_grad_positive - out, + axis=axis, + flatten=flatten, + reverse=not reverse, + exclusive=exclusive, + ).reshape(x.shape) + + x + ) output_neg = np.exp( - np_logcumsumexp(log_grad_negative - out, - axis=axis, - flatten=flatten, - reverse=not reverse, - exclusive=exclusive).reshape(x.shape) + x) + np_logcumsumexp( + log_grad_negative - out, + axis=axis, + flatten=flatten, + reverse=not reverse, + exclusive=exclusive, + ).reshape(x.shape) + + x + ) return output_pos - output_neg class TestLogcumsumexp(unittest.TestCase): - def run_imperative(self): data_np = np.arange(12, dtype=np.float32).reshape(3, 4) data = paddle.to_tensor(data_np) @@ -146,14 +155,16 @@ def run_static(self, use_gpu=False): place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - out = exe.run(feed={'X': data_np}, - fetch_list=[ - y.name, - y2.name, - y3.name, - y4.name, - y5.name, - ]) + out = exe.run( + feed={'X': data_np}, + fetch_list=[ + y.name, + y2.name, + y3.name, + y4.name, + y5.name, + ], + ) z = np_logcumsumexp(data_np) np.testing.assert_allclose(z, out[0], rtol=1e-05) @@ -202,9 +213,7 @@ def test_type_error(self): class BaseTestCases: - class BaseOpTest(OpTest): - def setUp(self): self.op_type = "logcumsumexp" input, attrs = self.input_and_attrs() @@ -218,56 +227,55 @@ def test_check_output(self): self.check_output() def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[ - np_logcumsumexp_grad(self.inputs['X'], - 1 / self.inputs['X'].size, - **self.attrs) - ]) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[ + np_logcumsumexp_grad( + self.inputs['X'], + 1 / self.inputs['X'].size, + **self.attrs + ) + ], + ) def input_and_attrs(self): raise NotImplementedError() class TestLogcumsumexpOp1(BaseTestCases.BaseOpTest): - def input_and_attrs(self): return np.arange(100, dtype=np.float64).reshape(10, 10), { 'axis': 0, 'flatten': True, - 'reverse': True + 'reverse': True, } class TestLogcumsumexpOp2(BaseTestCases.BaseOpTest): - def input_and_attrs(self): return np.arange(100, dtype=np.float64).reshape(10, 10), { 'axis': 1, - 'reverse': True + 'reverse': True, } class TestLogcumsumexpOp3(BaseTestCases.BaseOpTest): - def input_and_attrs(self): return np.arange(100, dtype=np.float64).reshape(10, 10), {'axis': 1} class TestLogcumsumexpOp4(BaseTestCases.BaseOpTest): - def input_and_attrs(self): return np.arange(100, dtype=np.float64).reshape(10, 10), { 'axis': 0, 'flatten': True, 'reverse': True, - 'exclusive': True + 'exclusive': True, } class TestLogcumsumexpFP16(unittest.TestCase): - def check_main(self, x_np, dtype, axis=None): paddle.disable_static() x = paddle.to_tensor(x_np.astype(dtype)) diff --git a/python/paddle/fluid/tests/unittests/test_logical_op.py b/python/paddle/fluid/tests/unittests/test_logical_op.py index c36445e7b0710..31490961c84ce 100755 --- a/python/paddle/fluid/tests/unittests/test_logical_op.py +++ b/python/paddle/fluid/tests/unittests/test_logical_op.py @@ -20,87 +20,41 @@ from paddle.fluid.framework import _test_eager_guard SUPPORTED_DTYPES = [ - bool, np.int8, np.int16, np.int32, np.int64, np.float32, np.float64 + bool, + np.int8, + np.int16, + np.int32, + np.int64, + np.float32, + np.float64, ] -TEST_META_OP_DATA = [{ - 'op_str': 'logical_and', - 'binary_op': True -}, { - 'op_str': 'logical_or', - 'binary_op': True -}, { - 'op_str': 'logical_xor', - 'binary_op': True -}, { - 'op_str': 'logical_not', - 'binary_op': False -}] +TEST_META_OP_DATA = [ + {'op_str': 'logical_and', 'binary_op': True}, + {'op_str': 'logical_or', 'binary_op': True}, + {'op_str': 'logical_xor', 'binary_op': True}, + {'op_str': 'logical_not', 'binary_op': False}, +] TEST_META_SHAPE_DATA = { - 'XDimLargerThanYDim1': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 5] - }, - 'XDimLargerThanYDim2': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 1] - }, - 'XDimLargerThanYDim3': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [1, 4, 1] - }, - 'XDimLargerThanYDim4': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [3, 4, 1] - }, - 'XDimLargerThanYDim5': { - 'x_shape': [2, 3, 1, 5], - 'y_shape': [3, 1, 1] - }, - 'XDimLessThanYDim1': { - 'x_shape': [4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim2': { - 'x_shape': [1, 4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim3': { - 'x_shape': [3, 4, 1], - 'y_shape': [2, 3, 4, 5] - }, - 'XDimLessThanYDim4': { - 'x_shape': [3, 1, 1], - 'y_shape': [2, 3, 1, 5] - }, - 'XDimLessThanYDim5': { - 'x_shape': [4, 5], - 'y_shape': [2, 3, 4, 5] - }, - 'Axis1InLargerDim': { - 'x_shape': [1, 4, 5], - 'y_shape': [2, 3, 1, 5] - }, - 'EqualDim1': { - 'x_shape': [10, 7], - 'y_shape': [10, 7] - }, - 'EqualDim2': { - 'x_shape': [1, 1, 4, 5], - 'y_shape': [2, 3, 1, 5] - } + 'XDimLargerThanYDim1': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 5]}, + 'XDimLargerThanYDim2': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 1]}, + 'XDimLargerThanYDim3': {'x_shape': [2, 3, 4, 5], 'y_shape': [1, 4, 1]}, + 'XDimLargerThanYDim4': {'x_shape': [2, 3, 4, 5], 'y_shape': [3, 4, 1]}, + 'XDimLargerThanYDim5': {'x_shape': [2, 3, 1, 5], 'y_shape': [3, 1, 1]}, + 'XDimLessThanYDim1': {'x_shape': [4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim2': {'x_shape': [1, 4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim3': {'x_shape': [3, 4, 1], 'y_shape': [2, 3, 4, 5]}, + 'XDimLessThanYDim4': {'x_shape': [3, 1, 1], 'y_shape': [2, 3, 1, 5]}, + 'XDimLessThanYDim5': {'x_shape': [4, 5], 'y_shape': [2, 3, 4, 5]}, + 'Axis1InLargerDim': {'x_shape': [1, 4, 5], 'y_shape': [2, 3, 1, 5]}, + 'EqualDim1': {'x_shape': [10, 7], 'y_shape': [10, 7]}, + 'EqualDim2': {'x_shape': [1, 1, 4, 5], 'y_shape': [2, 3, 1, 5]}, } TEST_META_WRONG_SHAPE_DATA = { - 'ErrorDim1': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [3, 4] - }, - 'ErrorDim2': { - 'x_shape': [2, 3, 4, 5], - 'y_shape': [4, 3] - } + 'ErrorDim1': {'x_shape': [2, 3, 4, 5], 'y_shape': [3, 4]}, + 'ErrorDim2': {'x_shape': [2, 3, 4, 5], 'y_shape': [4, 3]}, } @@ -175,16 +129,20 @@ def test(unit_test, use_gpu=False, test_error=False): META_DATA = dict(TEST_META_WRONG_SHAPE_DATA) for shape_data in META_DATA.values(): for data_type in SUPPORTED_DTYPES: - meta_data['x_np'] = np_data_generator(shape_data['x_shape'], - dtype=data_type) - meta_data['y_np'] = np_data_generator(shape_data['y_shape'], - dtype=data_type) + meta_data['x_np'] = np_data_generator( + shape_data['x_shape'], dtype=data_type + ) + meta_data['y_np'] = np_data_generator( + shape_data['y_shape'], dtype=data_type + ) if meta_data['binary_op'] and test_error: # catch C++ Exception - unit_test.assertRaises(BaseException, run_static, - **meta_data) - unit_test.assertRaises(BaseException, run_dygraph, - **meta_data) + unit_test.assertRaises( + BaseException, run_static, **meta_data + ) + unit_test.assertRaises( + BaseException, run_dygraph, **meta_data + ) continue static_result = run_static(**meta_data) dygraph_result = run_dygraph(**meta_data) @@ -195,12 +153,12 @@ def test(unit_test, use_gpu=False, test_error=False): np_result = np_op(meta_data['x_np']) unit_test.assertTrue((static_result == np_result).all()) unit_test.assertTrue( - (dygraph_result.numpy() == np_result).all()) + (dygraph_result.numpy() == np_result).all() + ) unit_test.assertTrue((eager_result.numpy() == np_result).all()) def test_type_error(unit_test, use_gpu, type_str_map): - def check_type(op_str, x, y, binary_op): op = getattr(paddle, op_str) error_type = ValueError @@ -235,24 +193,24 @@ def check_type(op_str, x, y, binary_op): startup_program = paddle.static.Program() main_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - x = paddle.static.data(name='x', - shape=[10], - dtype=type_str_map['x']) - y = paddle.static.data(name='y', - shape=[10], - dtype=type_str_map['y']) + x = paddle.static.data( + name='x', shape=[10], dtype=type_str_map['x'] + ) + y = paddle.static.data( + name='y', shape=[10], dtype=type_str_map['y'] + ) check_type(meta_data['op_str'], x, y, binary_op) def type_map_factory(): - return [{ - 'x': x_type, - 'y': y_type - } for x_type in SUPPORTED_DTYPES for y_type in SUPPORTED_DTYPES] + return [ + {'x': x_type, 'y': y_type} + for x_type in SUPPORTED_DTYPES + for y_type in SUPPORTED_DTYPES + ] class TestCPU(unittest.TestCase): - def test(self): test(self) @@ -266,7 +224,6 @@ def test_type_error(self): class TestCUDA(unittest.TestCase): - def test(self): test(self, True) diff --git a/python/paddle/fluid/tests/unittests/test_logit_op.py b/python/paddle/fluid/tests/unittests/test_logit_op.py index 7240fad1a441f..74f645fb68d1c 100644 --- a/python/paddle/fluid/tests/unittests/test_logit_op.py +++ b/python/paddle/fluid/tests/unittests/test_logit_op.py @@ -22,22 +22,23 @@ def logit(x, eps): - x_min = np.minimum(x, 1. - eps) + x_min = np.minimum(x, 1.0 - eps) x_max = np.maximum(x_min, eps) - return np.log(x_max / (1. - x_max)) + return np.log(x_max / (1.0 - x_max)) def logit_grad(x, eps=1e-8): - tmp_x = np.select([x < eps, x > (1. - eps)], [x * 0., x * 0.], default=-1.0) - x_1 = 1. - x + tmp_x = np.select( + [x < eps, x > (1.0 - eps)], [x * 0.0, x * 0.0], default=-1.0 + ) + x_1 = 1.0 - x _x = np.select([tmp_x == -1.0], [np.reciprocal(x * x_1)], default=0.0) - dout = np.full_like(x, fill_value=1. / _x.size) + dout = np.full_like(x, fill_value=1.0 / _x.size) dx = dout * _x return dx class TestLogitOp(OpTest): - def setUp(self): self.op_type = 'logit' self.python_api = paddle.logit @@ -45,7 +46,7 @@ def setUp(self): self.shape = [120] self.eps = 1e-8 self.set_attrs() - x = np.random.uniform(-1., 1., self.shape).astype(self.dtype) + x = np.random.uniform(-1.0, 1.0, self.shape).astype(self.dtype) out = logit(x, self.eps) self.x_grad = logit_grad(x, self.eps) self.inputs = {'X': x} @@ -59,31 +60,30 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], ['Out'], - user_defined_grads=[self.x_grad], - check_eager=True) + self.check_grad( + ['X'], ['Out'], user_defined_grads=[self.x_grad], check_eager=True + ) class TestLogitShape(TestLogitOp): - def set_attrs(self): self.shape = [2, 60] class TestLogitEps(TestLogitOp): - def set_attrs(self): self.eps = 1e-8 class TestLogitAPI(unittest.TestCase): - def setUp(self): self.x_shape = [120] - self.x = np.random.uniform(0., 1., self.x_shape).astype(np.float32) - self.place = paddle.CUDAPlace(0) \ - if paddle.fluid.core.is_compiled_with_cuda() \ + self.x = np.random.uniform(0.0, 1.0, self.x_shape).astype(np.float32) + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def check_api(self, eps=1e-8): ref_out = logit(self.x, eps) diff --git a/python/paddle/fluid/tests/unittests/test_logspace.py b/python/paddle/fluid/tests/unittests/test_logspace.py index 8b5501a26ae5d..170d056895a94 100644 --- a/python/paddle/fluid/tests/unittests/test_logspace.py +++ b/python/paddle/fluid/tests/unittests/test_logspace.py @@ -19,7 +19,6 @@ class TestLogspaceOpCommonCase(OpTest): - def setUp(self): self.op_type = "logspace" dtype = 'float32' @@ -38,7 +37,6 @@ def test_check_output(self): class TestLogspaceOpReverseCase(OpTest): - def setUp(self): self.op_type = "logspace" dtype = 'float32' @@ -46,7 +44,7 @@ def setUp(self): 'Start': np.array([10]).astype(dtype), 'Stop': np.array([0]).astype(dtype), 'Num': np.array([11]).astype('int32'), - 'Base': np.array([2]).astype(dtype) + 'Base': np.array([2]).astype(dtype), } self.attrs = {'dtype': int(paddle.float32)} @@ -57,7 +55,6 @@ def test_check_output(self): class TestLogspaceOpNumOneCase(OpTest): - def setUp(self): self.op_type = "logspace" dtype = 'float32' @@ -65,7 +62,7 @@ def setUp(self): 'Start': np.array([10]).astype(dtype), 'Stop': np.array([0]).astype(dtype), 'Num': np.array([1]).astype('int32'), - 'Base': np.array([2]).astype(dtype) + 'Base': np.array([2]).astype(dtype), } self.attrs = {'dtype': int(paddle.float32)} @@ -76,7 +73,6 @@ def test_check_output(self): class TestLogspaceOpMinusBaseCase(OpTest): - def setUp(self): self.op_type = "logspace" dtype = 'float32' @@ -95,7 +91,6 @@ def test_check_output(self): class TestLogspaceOpZeroBaseCase(OpTest): - def setUp(self): self.op_type = "logspace" dtype = 'float32' @@ -114,7 +109,6 @@ def test_check_output(self): class TestLogspaceAPI(unittest.TestCase): - def test_variable_input1(self): paddle.enable_static() prog = paddle.static.Program() @@ -156,12 +150,9 @@ def test_dtype(self): def test_name(self): with paddle.static.program_guard(paddle.static.Program()): - out = paddle.logspace(0, - 10, - 5, - 2, - dtype='float32', - name='logspace_res') + out = paddle.logspace( + 0, 10, 5, 2, dtype='float32', name='logspace_res' + ) assert 'logspace_res' in out.name def test_imperative(self): @@ -179,7 +170,6 @@ def test_imperative(self): class TestLogspaceOpError(unittest.TestCase): - def test_errors(self): with paddle.static.program_guard(paddle.static.Program()): @@ -209,9 +199,9 @@ def test_num_type(): self.assertRaises(TypeError, test_num_type) def test_start_dtype(): - start = paddle.static.data(shape=[1], - dtype="float64", - name="start") + start = paddle.static.data( + shape=[1], dtype="float64", name="start" + ) paddle.logspace(start, 10, 1, 2, dtype="float32") self.assertRaises(ValueError, test_start_dtype) @@ -223,17 +213,17 @@ def test_end_dtype(): self.assertRaises(ValueError, test_end_dtype) def test_num_dtype(): - num = paddle.static.data(shape=[1], - dtype="float32", - name="step") + num = paddle.static.data( + shape=[1], dtype="float32", name="step" + ) paddle.logspace(0, 10, num, 2, dtype="float32") self.assertRaises(TypeError, test_num_dtype) def test_base_dtype(): - base = paddle.static.data(shape=[1], - dtype="float64", - name="end") + base = paddle.static.data( + shape=[1], dtype="float64", name="end" + ) paddle.logspace(0, 10, 1, base, dtype="float32") self.assertRaises(ValueError, test_base_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_logsumexp.py b/python/paddle/fluid/tests/unittests/test_logsumexp.py index ca732d6b0a766..ac29603713e16 100644 --- a/python/paddle/fluid/tests/unittests/test_logsumexp.py +++ b/python/paddle/fluid/tests/unittests/test_logsumexp.py @@ -21,7 +21,7 @@ def ref_logsumexp(x, axis=None, keepdim=False, reduce_all=False): if isinstance(axis, int): - axis = (axis, ) + axis = (axis,) elif isinstance(axis, list): axis = tuple(axis) if reduce_all: @@ -53,7 +53,6 @@ def logsumexp_ref_grad(x): class TestLogsumexp(OpTest): - def setUp(self): self.op_type = 'logsumexp' self.python_api = logsumexp_wrapper @@ -73,7 +72,7 @@ def setUp(self): self.attrs = { 'axis': self.axis, 'keepdim': self.keepdim, - 'reduce_all': self.reduce_all + 'reduce_all': self.reduce_all, } self.user_defined_grads = None self.user_defined_grad_outputs = None @@ -90,10 +89,12 @@ def test_check_output(self): def test_check_grad(self): self.check_grad( - ['X'], ['Out'], + ['X'], + ['Out'], user_defined_grads=self.user_defined_grads, user_defined_grad_outputs=self.user_defined_grad_outputs, - check_eager=True) + check_eager=True, + ) def calc_grad(self): dy = np.ones(1, dtype=self.dtype) @@ -103,19 +104,16 @@ def calc_grad(self): class TestLogsumexp_shape(TestLogsumexp): - def set_attrs(self): self.shape = [4, 5, 6] class TestLogsumexp_axis(TestLogsumexp): - def set_attrs(self): self.axis = [0, -1] class TestLogsumexp_axis_all(TestLogsumexp): - def set_attrs(self): self.axis = [0, 1, 2, 3] @@ -126,13 +124,11 @@ def set_attrs_addition(self): class TestLogsumexp_keepdim(TestLogsumexp): - def set_attrs(self): self.keepdim = True class TestLogsumexp_reduce_all(TestLogsumexp): - def set_attrs(self): self.reduce_all = True @@ -143,7 +139,6 @@ def set_attrs_addition(self): class TestLogsumexp_FP32(TestLogsumexp): - def set_attrs(self): self.dtype = 'float32' @@ -154,10 +149,10 @@ def test_check_grad(self): np.testing.assert_allclose(x_grad, ref_x_grad, rtol=1e-08, atol=1e-08) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestLogsumexp_FP16(TestLogsumexp): - def set_attrs(self): self.dtype = 'float16' @@ -169,10 +164,9 @@ def test_check_output(self): tensor_x = paddle.to_tensor(x) out_pad = logsumexp_wrapper(tensor_x) paddle.enable_static() - np.testing.assert_allclose(out_pad.numpy(), - out_ref, - rtol=1e-03, - atol=1e-08) + np.testing.assert_allclose( + out_pad.numpy(), out_ref, rtol=1e-03, atol=1e-08 + ) def test_check_grad(self): self.__class__.dtype = self.dtype @@ -184,7 +178,6 @@ def test_check_grad(self): class TestLogsumexpError(unittest.TestCase): - def test_errors(self): with paddle.static.program_guard(paddle.static.Program()): self.assertRaises(TypeError, paddle.logsumexp, 1) @@ -193,12 +186,14 @@ def test_errors(self): class TestLogsumexpAPI(unittest.TestCase): - def setUp(self): self.shape = [2, 3, 4, 5] self.x = np.random.uniform(-1, 1, self.shape).astype(np.float32) - self.place = paddle.CUDAPlace(0) if paddle.fluid.core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def api_case(self, axis=None, keepdim=False): out_ref = ref_logsumexp(self.x, axis, keepdim) diff --git a/python/paddle/fluid/tests/unittests/test_lookahead.py b/python/paddle/fluid/tests/unittests/test_lookahead.py index 10c2ef78b8618..6f7c052f99256 100644 --- a/python/paddle/fluid/tests/unittests/test_lookahead.py +++ b/python/paddle/fluid/tests/unittests/test_lookahead.py @@ -25,7 +25,6 @@ class TestLookAhead(unittest.TestCase): - def test_lookahead_static(self): paddle.enable_static() place = fluid.CPUPlace() @@ -41,7 +40,8 @@ def test_lookahead_static(self): optimizer = paddle.optimizer.SGD(learning_rate=SGD_LR) lookahead = paddle.incubate.optimizer.LookAhead( - optimizer, alpha=LOOKAHEAD_ALPHA, k=LOOKAHEAD_K) + optimizer, alpha=LOOKAHEAD_ALPHA, k=LOOKAHEAD_K + ) lookahead.minimize(loss) exe.run(startup) @@ -49,21 +49,24 @@ def test_lookahead_static(self): fast_param = None for i in range(10): if (i + 1) % LOOKAHEAD_K == 0: - slow_param = slow_param + LOOKAHEAD_ALPHA * (fast_param - - slow_param) + slow_param = slow_param + LOOKAHEAD_ALPHA * ( + fast_param - slow_param + ) x = np.random.random(size=(10, 1)).astype('float32') - latest_b, b_grad = exe.run(program=train_program, - feed={'X': x}, - fetch_list=[ - 'fc_0.b_0', - 'fc_0.b_0@GRAD', - ]) + latest_b, b_grad = exe.run( + program=train_program, + feed={'X': x}, + fetch_list=[ + 'fc_0.b_0', + 'fc_0.b_0@GRAD', + ], + ) if i == 0: slow_param = latest_b if (i + 1) % LOOKAHEAD_K == 0: - self.assertAlmostEqual(slow_param.all(), - latest_b.all(), - delta=5e-3) + self.assertAlmostEqual( + slow_param.all(), latest_b.all(), delta=5e-3 + ) fast_param = latest_b - SGD_LR * b_grad def func_test_look_ahead_dygraph(self): @@ -76,21 +79,20 @@ def func_test_look_ahead_dygraph(self): # define a random dataset class RandomDataset(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, - (1, )).astype('int64') + label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype( + 'int64' + ) return image, label def __len__(self): return self.num_samples class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) @@ -110,34 +112,41 @@ def train(layer, loader, loss_fn, opt): out = layer(image) loss = loss_fn(out, label) loss.backward() - fast_param = (layer.bias.numpy() - - SGD_LR * layer.bias.grad.numpy()) + fast_param = ( + layer.bias.numpy() - SGD_LR * layer.bias.grad.numpy() + ) opt.step() if idx == 1: slow_param = fast_param if idx % LOOKAHEAD_K == 0: slow_param = slow_param + LOOKAHEAD_ALPHA * ( - fast_param - slow_param) - self.assertAlmostEqual(np.mean(slow_param), - np.mean(layer.bias.numpy()), - delta=5e-3) + fast_param - slow_param + ) + self.assertAlmostEqual( + np.mean(slow_param), + np.mean(layer.bias.numpy()), + delta=5e-3, + ) opt.clear_grad() layer = LinearNet() loss_fn = nn.CrossEntropyLoss() - optimizer = paddle.optimizer.SGD(learning_rate=SGD_LR, - parameters=layer.parameters()) - lookahead = paddle.incubate.optimizer.LookAhead(optimizer, - alpha=LOOKAHEAD_ALPHA, - k=LOOKAHEAD_K) + optimizer = paddle.optimizer.SGD( + learning_rate=SGD_LR, parameters=layer.parameters() + ) + lookahead = paddle.incubate.optimizer.LookAhead( + optimizer, alpha=LOOKAHEAD_ALPHA, k=LOOKAHEAD_K + ) # create data loader dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) - loader = paddle.io.DataLoader(dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) + loader = paddle.io.DataLoader( + dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) train(layer, loader, loss_fn, lookahead) diff --git a/python/paddle/fluid/tests/unittests/test_lookup_table_bf16_op.py b/python/paddle/fluid/tests/unittests/test_lookup_table_bf16_op.py index 8c6b3421e24a9..4c572622abe7b 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_table_bf16_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_table_bf16_op.py @@ -14,10 +14,12 @@ import unittest import numpy as np -from paddle.fluid.tests.unittests.op_test import (OpTest, - convert_float_to_uint16, - convert_uint16_to_float, - skip_check_grad_ci) +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + convert_float_to_uint16, + convert_uint16_to_float, + skip_check_grad_ci, +) import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.op import Operator @@ -26,8 +28,11 @@ def _lookup(weights, ids, flat_ids, op_version="lookup_table"): w_shape = weights.shape - out_shape = list(ids.shape[:-1]) if op_version == "lookup_table" else list( - ids.shape) + out_shape = ( + list(ids.shape[:-1]) + if op_version == "lookup_table" + else list(ids.shape) + ) out_shape.append(w_shape[-1]) out = weights[flat_ids].reshape(out_shape) return out @@ -36,8 +41,11 @@ def _lookup(weights, ids, flat_ids, op_version="lookup_table"): def _get_grad(weights, ids, flat_ids, op_version="lookup_table"): w_shape = weights.shape w_grad = np.zeros((w_shape), dtype=weights.dtype) - out_shape = list(ids.shape[:-1]) if op_version == "lookup_table" else list( - ids.shape) + out_shape = ( + list(ids.shape[:-1]) + if op_version == "lookup_table" + else list(ids.shape) + ) out_grad_shape = (np.prod(out_shape), w_shape[-1]) out_grad = weights[flat_ids].reshape(out_grad_shape) for i, idx in enumerate(flat_ids): @@ -45,10 +53,10 @@ def _get_grad(weights, ids, flat_ids, op_version="lookup_table"): return w_grad -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestLookupTableBF16Op(OpTest): - def init_test(self): self.op_type = "lookup_table" self.ids_shape = (4, 1) @@ -62,11 +70,13 @@ def setUp(self): self.flat_ids = self.ids.flatten() self.w_bf16 = convert_float_to_uint16(table) - self.out_bf16 = _lookup(self.w_bf16, self.ids, self.flat_ids, - self.op_type) + self.out_bf16 = _lookup( + self.w_bf16, self.ids, self.flat_ids, self.op_type + ) self.out_fp32 = _lookup(table, self.ids, self.flat_ids, self.op_type) - self.w_grad_fp32 = _get_grad(table, self.ids, self.flat_ids, - self.op_type) + self.w_grad_fp32 = _get_grad( + table, self.ids, self.flat_ids, self.op_type + ) self.inputs = {'W': self.w_bf16, 'Ids': self.ids} self.outputs = {'Out': self.out_fp32} @@ -75,36 +85,40 @@ def test_check_output(self): self.check_output_with_place(core.CPUPlace(), check_dygraph=False) def test_check_grad(self): - self.check_grad_with_place(core.CPUPlace(), ['W'], - 'Out', - no_grad_set=set('Ids'), - check_dygraph=False, - max_relative_error=1.5e-2, - user_defined_grads=[self.w_grad_fp32], - user_defined_grad_outputs=[self.out_bf16]) - - -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") + self.check_grad_with_place( + core.CPUPlace(), + ['W'], + 'Out', + no_grad_set=set('Ids'), + check_dygraph=False, + max_relative_error=1.5e-2, + user_defined_grads=[self.w_grad_fp32], + user_defined_grad_outputs=[self.out_bf16], + ) + + +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestLookupTableBF16OpIds4D(TestLookupTableBF16Op): - def init_test(self): self.op_type = "lookup_table" self.ids_shape = (2, 4, 5, 1) -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestLookupTableBF16OpWIsSelectedRows(unittest.TestCase): - def init_test(self): self.op_type = "lookup_table" self.ids_shape = (10, 1) def setUp(self): self.init_test() - self.ids = np.random.randint(low=0, high=15, - size=self.ids_shape).astype("int64") + self.ids = np.random.randint( + low=0, high=15, size=self.ids_shape + ).astype("int64") self.flat_ids = self.ids.flatten() self.w_fp32 = np.random.random((15, 32)).astype("float32") self.w_bf16 = convert_float_to_uint16(self.w_fp32) @@ -144,11 +158,12 @@ def test_check_output(self): self._check_output(ref, result_array) -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestLookupTableBF16OpWIsSelectedRows4DIds( - TestLookupTableBF16OpWIsSelectedRows): - + TestLookupTableBF16OpWIsSelectedRows +): def init_test(self): self.op_type = "lookup_table" self.ids_shape = (3, 4, 5, 1) @@ -161,11 +176,12 @@ def setUp(self): @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") + "test the gradient here." +) +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestLookupTableBF16OpWithPadding(TestLookupTableBF16Op): - def test_check_output(self): ids = np.squeeze(self.inputs['Ids']) padding_idx = np.random.choice(ids, 1)[0] @@ -177,11 +193,12 @@ def test_check_output(self): @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") + "test the gradient here." +) +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestLookupTableBF16OpIds4DPadding(TestLookupTableBF16OpIds4D): - def test_check_output(self): ids = self.inputs['Ids'] flatten_idx = ids.flatten() @@ -202,8 +219,9 @@ def set_initializer(self): def setUp(self): self.ids_shape = [4, 1] self.w_shape = [10, 64] - self.ids = np.random.randint(low=0, high=9, - size=self.ids_shape).astype("int64") + self.ids = np.random.randint(low=0, high=9, size=self.ids_shape).astype( + "int64" + ) self.flat_ids = self.ids.flatten() self.value = 3.0 self.w_fp32 = np.full(self.w_shape, self.value) @@ -214,18 +232,20 @@ def setUp(self): with fluid.program_guard(self.prog, self.startup_prog): x = fluid.layers.data(name='x', shape=self.ids_shape, dtype='int64') - self.emb = fluid.layers.embedding(input=x, - size=self.w_shape, - param_attr=fluid.ParamAttr( - name="emb_weight", - initializer=self.initializer), - is_sparse=False, - dtype="uint16") # bfloat16 + self.emb = fluid.layers.embedding( + input=x, + size=self.w_shape, + param_attr=fluid.ParamAttr( + name="emb_weight", initializer=self.initializer + ), + is_sparse=False, + dtype="uint16", + ) # bfloat16 exe = fluid.Executor(self.place) exe.run(self.startup_prog) - self.result = exe.run(self.prog, - feed={'x': self.ids}, - fetch_list=['emb_weight', self.emb]) + self.result = exe.run( + self.prog, feed={'x': self.ids}, fetch_list=['emb_weight', self.emb] + ) def test_embedding_weights(self): result = convert_uint16_to_float(self.result[0]) diff --git a/python/paddle/fluid/tests/unittests/test_lookup_table_dequant_op.py b/python/paddle/fluid/tests/unittests/test_lookup_table_dequant_op.py index bd15a3ac1cbf0..42a491344c62c 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_table_dequant_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_table_dequant_op.py @@ -19,7 +19,6 @@ class TestLookupTableDequantOp(OpTest): - def setUp(self): self.op_type = "lookup_table_dequant" table = np.random.random((17, 32)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_lookup_table_op.py b/python/paddle/fluid/tests/unittests/test_lookup_table_op.py index 2a57ca7e5968f..02fccaaca2232 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_table_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_table_op.py @@ -23,7 +23,6 @@ class TestLookupTableOp(OpTest): - def setUp(self): self.op_type = "lookup_table" table = np.random.random((17, 31)).astype("float64") @@ -40,12 +39,12 @@ def test_check_grad(self): class TestLookupTableOpWithTensorIds(OpTest): - def setUp(self): self.op_type = "lookup_table" table = np.random.random((17, 31)).astype("float64") - ids = np.random.randint(low=0, high=17, - size=(2, 4, 5, 1)).astype("int64") + ids = np.random.randint(low=0, high=17, size=(2, 4, 5, 1)).astype( + "int64" + ) self.inputs = {'W': table, 'Ids': ids} self.outputs = {'Out': table[ids.flatten()].reshape((2, 4, 5, 31))} @@ -59,9 +58,9 @@ def test_check_grad(self): @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") + "test the gradient here." +) class TestLookupTableOpWithPadding(TestLookupTableOp): - def test_check_output(self): ids = np.squeeze(self.inputs['Ids']) padding_idx = np.random.choice(ids, 1)[0] @@ -73,9 +72,9 @@ def test_check_output(self): @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") + "test the gradient here." +) class TestLookupTableOpWithTensorIdsAndPadding(TestLookupTableOpWithTensorIds): - def test_check_output(self): ids = self.inputs['Ids'] flatten_idx = ids.flatten() @@ -86,7 +85,6 @@ def test_check_output(self): class TestLookupTableWIsSelectedRows(unittest.TestCase): - def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() ids_array = np.array([[0], [4], [3], [5]]).astype("int64") @@ -139,13 +137,14 @@ def test_w_is_selected_rows(self): self.check_with_place(place) -class TestLookupTableWithTensorIdsWIsSelectedRows(TestLookupTableWIsSelectedRows - ): - +class TestLookupTableWithTensorIdsWIsSelectedRows( + TestLookupTableWIsSelectedRows +): def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() - ids_array = np.random.randint(low=0, high=6, - size=(2, 4, 3, 1)).astype("int64") + ids_array = np.random.randint(low=0, high=6, size=(2, 4, 3, 1)).astype( + "int64" + ) ids_tensor.set(ids_array, place) return ids_array @@ -155,7 +154,6 @@ def check_result(self, ids_array, result_array): class TestEmbedOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.randint(0, 10, (4, 1)).astype("int64") @@ -176,9 +174,9 @@ def test_input_dtype(): def test_param_dtype(): # dtype must be float32 or float64 input2 = fluid.data(name='x2', shape=[4, 1], dtype='int64') - fluid.layers.embedding(input=input2, - size=(10, 64), - dtype='int64') + fluid.layers.embedding( + input=input2, size=(10, 64), dtype='int64' + ) self.assertRaises(TypeError, test_param_dtype) @@ -187,11 +185,11 @@ def test_param_dtype(): class TestLookupTableOpInt8(OpTest): - def setUp(self): self.op_type = "lookup_table" - table = np.random.randint(low=-128, high=127, - size=(17, 31)).astype("int8") + table = np.random.randint(low=-128, high=127, size=(17, 31)).astype( + "int8" + ) ids = np.random.randint(0, 17, 4).astype("int64") ids_expand = np.expand_dims(ids, axis=1) self.inputs = {'W': table, 'Ids': ids_expand} @@ -207,13 +205,14 @@ def test_check_grad(self): class TestLookupTableOpWithTensorIdsInt8(OpTest): - def setUp(self): self.op_type = "lookup_table" - table = np.random.randint(low=-128, high=127, - size=(17, 31)).astype("int8") - ids = np.random.randint(low=0, high=17, - size=(2, 4, 5, 1)).astype("int64") + table = np.random.randint(low=-128, high=127, size=(17, 31)).astype( + "int8" + ) + ids = np.random.randint(low=0, high=17, size=(2, 4, 5, 1)).astype( + "int64" + ) self.inputs = {'W': table, 'Ids': ids} self.outputs = {'Out': table[ids.flatten()].reshape((2, 4, 5, 31))} @@ -227,7 +226,6 @@ def test_check_grad(self): class TestLookupTableOpWithPaddingInt8(TestLookupTableOpInt8): - def test_check_output(self): ids = np.squeeze(self.inputs['Ids']) padding_idx = np.random.choice(ids, 1)[0] @@ -242,8 +240,8 @@ def test_check_grad(self): class TestLookupTableOpWithTensorIdsAndPaddingInt8( - TestLookupTableOpWithTensorIdsInt8): - + TestLookupTableOpWithTensorIdsInt8 +): def test_check_output(self): ids = self.inputs['Ids'] flatten_idx = ids.flatten() @@ -259,7 +257,6 @@ def test_check_grad(self): class TestLookupTableWIsSelectedRowsInt8(unittest.TestCase): - def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() ids_array = np.array([[0], [4], [3], [5]]).astype("int64") @@ -313,12 +310,13 @@ def test_w_is_selected_rows(self): class TestLookupTableWithTensorIdsWIsSelectedRowsInt8( - TestLookupTableWIsSelectedRowsInt8): - + TestLookupTableWIsSelectedRowsInt8 +): def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() - ids_array = np.random.randint(low=0, high=6, - size=(2, 4, 3, 1)).astype("int64") + ids_array = np.random.randint(low=0, high=6, size=(2, 4, 3, 1)).astype( + "int64" + ) ids_tensor.set(ids_array, place) return ids_array @@ -329,11 +327,11 @@ def check_result(self, ids_array, result_array): @skip_check_grad_ci(reason="Int16 type only be used in test and inference.") class TestLookupTableOpInt16(OpTest): - def setUp(self): self.op_type = "lookup_table" - table = np.random.randint(low=-128, high=127, - size=(17, 31)).astype("int16") + table = np.random.randint(low=-128, high=127, size=(17, 31)).astype( + "int16" + ) ids = np.random.randint(0, 17, 4).astype("int64") ids_expand = np.expand_dims(ids, axis=1) self.inputs = {'W': table, 'Ids': ids_expand} @@ -345,13 +343,14 @@ def test_check_output(self): @skip_check_grad_ci(reason="Int16 type only be used in test and inference.") class TestLookupTableOpWithTensorIdsInt16(OpTest): - def setUp(self): self.op_type = "lookup_table" - table = np.random.randint(low=-128, high=127, - size=(17, 31)).astype("int16") - ids = np.random.randint(low=0, high=17, - size=(2, 4, 5, 1)).astype("int64") + table = np.random.randint(low=-128, high=127, size=(17, 31)).astype( + "int16" + ) + ids = np.random.randint(low=0, high=17, size=(2, 4, 5, 1)).astype( + "int64" + ) self.inputs = {'W': table, 'Ids': ids} self.outputs = {'Out': table[ids.flatten()].reshape((2, 4, 5, 31))} @@ -361,7 +360,6 @@ def test_check_output(self): @skip_check_grad_ci(reason="Int16 type only be used in test and inference.") class TestLookupTableOpWithPaddingInt16(TestLookupTableOpInt16): - def test_check_output(self): ids = np.squeeze(self.inputs['Ids']) padding_idx = np.random.choice(ids, 1)[0] @@ -372,8 +370,8 @@ def test_check_output(self): @skip_check_grad_ci(reason="Int16 type only be used in test and inference.") class TestLookupTableOpWithTensorIdsAndPaddingInt16( - TestLookupTableOpWithTensorIdsInt16): - + TestLookupTableOpWithTensorIdsInt16 +): def test_check_output(self): ids = self.inputs['Ids'] flatten_idx = ids.flatten() @@ -384,7 +382,6 @@ def test_check_output(self): class TestLookupTableWIsSelectedRowsInt16(unittest.TestCase): - def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() ids_array = np.array([[0], [4], [3], [5]]).astype("int64") @@ -437,12 +434,13 @@ def test_w_is_selected_rows(self): class TestLookupTableWithTensorIdsWIsSelectedRowsInt16( - TestLookupTableWIsSelectedRowsInt16): - + TestLookupTableWIsSelectedRowsInt16 +): def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() - ids_array = np.random.randint(low=0, high=6, - size=(2, 4, 3, 1)).astype("int64") + ids_array = np.random.randint(low=0, high=6, size=(2, 4, 3, 1)).astype( + "int64" + ) ids_tensor.set(ids_array, place) return ids_array @@ -452,13 +450,14 @@ def check_result(self, ids_array, result_array): class TestOutDtype(unittest.TestCase): - def test_dtype(self): api_fn = F.embedding - check_out_dtype(api_fn, - in_specs=[([10, 16], 'int64'), ([100, 64], )], - expect_dtypes=['float32', 'float64'], - target_index=1) + check_out_dtype( + api_fn, + in_specs=[([10, 16], 'int64'), ([100, 64],)], + expect_dtypes=['float32', 'float64'], + target_index=1, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_lookup_table_v2_bf16_op.py b/python/paddle/fluid/tests/unittests/test_lookup_table_v2_bf16_op.py index a479693d518b2..e0dbf743430c6 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_table_v2_bf16_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_table_v2_bf16_op.py @@ -17,23 +17,24 @@ import paddle from paddle.fluid.tests.unittests.op_test import convert_uint16_to_float from paddle.fluid.tests.unittests.test_lookup_table_bf16_op import ( - _lookup, TestLookupTableBF16Op, TestLookupTableBF16OpIds4D, + _lookup, + TestLookupTableBF16Op, + TestLookupTableBF16OpIds4D, TestLookupTableBF16OpWIsSelectedRows, - TestLookupTableBF16OpWIsSelectedRows4DIds) + TestLookupTableBF16OpWIsSelectedRows4DIds, +) import paddle.fluid as fluid import paddle.fluid.core as core class TestLookupTableV2BF16Op(TestLookupTableBF16Op): - def init_test(self): self.op_type = "lookup_table_v2" - self.ids_shape = (4) + self.ids_shape = 4 self.mkldnn_data_type = "bfloat16" class TestLookupTableV2BF16OpIds4D(TestLookupTableBF16OpIds4D): - def init_test(self): self.op_type = "lookup_table_v2" self.ids_shape = (2, 4, 5) @@ -41,23 +42,22 @@ def init_test(self): class TestLookupTableV2BF16OpWIsSelectedRows( - TestLookupTableBF16OpWIsSelectedRows): - + TestLookupTableBF16OpWIsSelectedRows +): def init_test(self): self.op_type = "lookup_table_v2" - self.ids_shape = (10) + self.ids_shape = 10 class TestLookupTableV2BF16OpWIsSelectedRows4DIds( - TestLookupTableBF16OpWIsSelectedRows4DIds): - + TestLookupTableBF16OpWIsSelectedRows4DIds +): def init_test(self): self.op_type = "lookup_table_v2" self.ids_shape = (3, 4, 5) class TestLookupTableBF16OpWithPadding(TestLookupTableV2BF16Op): - def test_check_output(self): ids = np.squeeze(self.inputs['Ids']) padding_idx = np.random.choice(ids, 1)[0] @@ -67,7 +67,6 @@ def test_check_output(self): class TestLookupTableBF16OpIds4DPadding(TestLookupTableV2BF16OpIds4D): - def test_check_output(self): ids = self.inputs['Ids'] flatten_idx = ids.flatten() @@ -89,8 +88,9 @@ def setUp(self): self.op_type = "lookup_table_v2" self.ids_shape = [4] self.w_shape = [10, 64] - self.ids = np.random.randint(low=0, high=9, - size=self.ids_shape).astype("int64") + self.ids = np.random.randint(low=0, high=9, size=self.ids_shape).astype( + "int64" + ) self.flat_ids = self.ids.flatten() self.value = 3.0 self.w_fp32 = np.full(self.w_shape, self.value) @@ -101,18 +101,20 @@ def setUp(self): with fluid.program_guard(self.prog, self.startup_prog): x = fluid.layers.data(name='x', shape=self.ids_shape, dtype='int64') - self.emb = fluid.input.embedding(input=x, - size=self.w_shape, - param_attr=fluid.ParamAttr( - name="emb_weight", - initializer=self.initializer), - is_sparse=False, - dtype="uint16") # bfloat16 + self.emb = fluid.input.embedding( + input=x, + size=self.w_shape, + param_attr=fluid.ParamAttr( + name="emb_weight", initializer=self.initializer + ), + is_sparse=False, + dtype="uint16", + ) # bfloat16 exe = fluid.Executor(self.place) exe.run(self.startup_prog) - self.result = exe.run(self.prog, - feed={'x': self.ids}, - fetch_list=['emb_weight', self.emb]) + self.result = exe.run( + self.prog, feed={'x': self.ids}, fetch_list=['emb_weight', self.emb] + ) def test_embedding_weights(self): result = convert_uint16_to_float(self.result[0]) diff --git a/python/paddle/fluid/tests/unittests/test_lookup_table_v2_op.py b/python/paddle/fluid/tests/unittests/test_lookup_table_v2_op.py index 4919c9e8c0089..0f73c052bb51b 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_table_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_table_v2_op.py @@ -24,7 +24,6 @@ class TestStaticGraphSupportMultipleInt(unittest.TestCase): - def test_main(self): dtypes = ['uint8', 'int8', 'int16', 'int32', 'int64'] if paddle.in_dynamic_mode(): @@ -33,8 +32,9 @@ def test_main(self): else: disable_static = False for i, dtype in enumerate(dtypes): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.static.data(name='x', shape=[-1, 7, 30], dtype=dtype) emb = paddle.nn.Embedding(10, 20) y = emb(x) @@ -44,7 +44,6 @@ def test_main(self): class TestLookupTableOp(OpTest): - def setUp(self): self.op_type = "lookup_table_v2" self.python_api = paddle.nn.functional.embedding @@ -64,25 +63,21 @@ def test_check_grad(self): class TestLookupTableOpInt16(OpTest): - def id_dtype(self): return "int16" class TestLookupTableOpInt8(OpTest): - def id_dtype(self): return "int8" class TestLookupTableOpUInt8(OpTest): - def id_dtype(self): return "uint8" class TestLookupTableOpWithTensorIds(OpTest): - def setUp(self): self.op_type = "lookup_table_v2" table = np.random.random((17, 31)).astype("float64") @@ -100,9 +95,9 @@ def test_check_grad(self): @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") + "test the gradient here." +) class TestLookupTableOpWithPadding(TestLookupTableOp): - def test_check_output(self): ids = np.squeeze(self.inputs['Ids']) padding_idx = np.random.choice(ids, 1)[0] @@ -114,9 +109,9 @@ def test_check_output(self): @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") + "test the gradient here." +) class TestLookupTableOpWithTensorIdsAndPadding(TestLookupTableOpWithTensorIds): - def test_check_output(self): ids = self.inputs['Ids'] flatten_idx = ids.flatten() @@ -127,7 +122,6 @@ def test_check_output(self): class TestLookupTableWIsSelectedRows(unittest.TestCase): - def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() ids_array = np.array([0, 4, 3, 5]).astype("int32") @@ -180,13 +174,14 @@ def test_w_is_selected_rows(self): self.check_with_place(place) -class TestLookupTableWithTensorIdsWIsSelectedRows(TestLookupTableWIsSelectedRows - ): - +class TestLookupTableWithTensorIdsWIsSelectedRows( + TestLookupTableWIsSelectedRows +): def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() - ids_array = np.random.randint(low=0, high=6, - size=(2, 4, 3)).astype("int64") + ids_array = np.random.randint(low=0, high=6, size=(2, 4, 3)).astype( + "int64" + ) ids_tensor.set(ids_array, place) return ids_array @@ -196,7 +191,6 @@ def check_result(self, ids_array, result_array): class TestLookupTableIsSparse(unittest.TestCase): - def init_data(self): self.x_data = np.array([[1, 3, 0, 4, 7]]).astype("int64") self.y_data = np.array([[0.1, 0.3, 0, 0.4, 0.7]]).astype("float32") @@ -214,8 +208,11 @@ def get_w_grad(self, is_sparse): name="emb_weight", learning_rate=10, initializer=fluid.initializer.NumpyArrayInitializer( - self.w_data)), - is_sparse=is_sparse) + self.w_data + ), + ), + is_sparse=is_sparse, + ) y = fluid.layers.reduce_sum(emb, dim=-1) loss = fluid.layers.square_error_cost(input=y, label=y_) @@ -227,12 +224,11 @@ def get_w_grad(self, is_sparse): place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'x': self.x_data, - 'y_': self.y_data - }, - fetch_list=['emb_weight'], - return_numpy=False) + ret = exe.run( + feed={'x': self.x_data, 'y_': self.y_data}, + fetch_list=['emb_weight'], + return_numpy=False, + ) return np.array(ret[0]) def test_w_grad(self): @@ -242,14 +238,12 @@ def test_w_grad(self): self.check_grad(w_grad, w_grad_with_sparse) def check_grad(self, w_grad1, w_grad2, tolerance=1e-6): - np.testing.assert_allclose(w_grad1, - w_grad2, - rtol=tolerance, - atol=tolerance) + np.testing.assert_allclose( + w_grad1, w_grad2, rtol=tolerance, atol=tolerance + ) class TestLookupTableApi(unittest.TestCase): - def test_api(self): x = fluid.layers.data(name='x', shape=[20], dtype='int64') emb = fluid.embedding(input=x, size=[128, 64]) @@ -259,15 +253,16 @@ def test_api(self): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'x': x_data, - }, - fetch_list=[emb], - return_numpy=False) + ret = exe.run( + feed={ + 'x': x_data, + }, + fetch_list=[emb], + return_numpy=False, + ) class TestEmbedOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.randint(0, 10, (4, 6)).astype("int64") diff --git a/python/paddle/fluid/tests/unittests/test_lr_scheduler.py b/python/paddle/fluid/tests/unittests/test_lr_scheduler.py index 2d76f141f9ff8..55d26bc509379 100644 --- a/python/paddle/fluid/tests/unittests/test_lr_scheduler.py +++ b/python/paddle/fluid/tests/unittests/test_lr_scheduler.py @@ -21,9 +21,9 @@ import paddle.fluid.core as core -def reduce_lr_on_plateau(decay_rate, threshold, cooldown, patience, m, n, loss, - var_list): - +def reduce_lr_on_plateau( + decay_rate, threshold, cooldown, patience, m, n, loss, var_list +): def is_better(current, best, m, n): if m == 'min' and n == 'rel': return current < best - best * threshold @@ -53,7 +53,6 @@ def is_better(current, best, m, n): class TestReduceOnPlateauDecay(object): - def test_ReduceLR(self): # the decay rate must be less than 1.0 with self.assertRaises(ValueError): @@ -63,8 +62,9 @@ def test_ReduceLR(self): paddle.optimizer.lr.ReduceOnPlateau(learning_rate=1.0, mode="test") # the threshold_mode must be "rel" or "abs" with self.assertRaises(ValueError): - paddle.optimizer.lr.ReduceOnPlateau(learning_rate=1.0, - threshold_mode="test") + paddle.optimizer.lr.ReduceOnPlateau( + learning_rate=1.0, threshold_mode="test" + ) with self.assertRaises(TypeError): paddle.optimizer.lr.ReduceOnPlateau(learning_rate="test") with self.assertRaises(TypeError): @@ -75,8 +75,9 @@ def test_ReduceLR(self): places.append(paddle.CUDAPlace(0)) for place in places: - for m, n in zip(['min', 'max', 'min', 'max'], - ['rel', 'rel', 'abs', 'abs']): + for m, n in zip( + ['min', 'max', 'min', 'max'], ['rel', 'rel', 'abs', 'abs'] + ): kwargs = { 'learning_rate': 1.0, 'mode': m, @@ -107,10 +108,9 @@ def _test_static(self, place, kwargs): main_prog = paddle.static.Program() start_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, start_prog): - x = fluid.layers.create_global_var([1], - 1, - 'float32', - persistable=True) + x = fluid.layers.create_global_var( + [1], 1, 'float32', persistable=True + ) paddle.increment(x) loss = paddle.sin(x) scheduler = paddle.optimizer.lr.ReduceOnPlateau(**kwargs) @@ -124,12 +124,19 @@ def _test_static(self, place, kwargs): for epoch in range(20): for batch_id in range(1): - out, actual_lr = exe.run(main_prog, - fetch_list=[loss.name, lr_var.name]) + out, actual_lr = exe.run( + main_prog, fetch_list=[loss.name, lr_var.name] + ) expected_lr = reduce_lr_on_plateau( - kwargs['factor'], kwargs['threshold'], kwargs['cooldown'], - kwargs['patience'], kwargs['mode'], - kwargs['threshold_mode'], out[0], var_list) + kwargs['factor'], + kwargs['threshold'], + kwargs['cooldown'], + kwargs['patience'], + kwargs['mode'], + kwargs['threshold_mode'], + out[0], + var_list, + ) scheduler.step(out[0]) actual_lr = scheduler() @@ -137,12 +144,19 @@ def _test_static(self, place, kwargs): for epoch in range(10): for batch_id in range(1): - out, actual_lr = exe.run(test_prog, - fetch_list=[loss.name, lr_var.name]) + out, actual_lr = exe.run( + test_prog, fetch_list=[loss.name, lr_var.name] + ) expected_lr = reduce_lr_on_plateau( - kwargs['factor'], kwargs['threshold'], kwargs['cooldown'], - kwargs['patience'], kwargs['mode'], - kwargs['threshold_mode'], out[0], var_list) + kwargs['factor'], + kwargs['threshold'], + kwargs['cooldown'], + kwargs['patience'], + kwargs['mode'], + kwargs['threshold_mode'], + out[0], + var_list, + ) scheduler.step(out[0]) actual_lr = scheduler() self.assertEqual(actual_lr, np.array(expected_lr)) @@ -158,8 +172,9 @@ def _test_dygraph(self, place, kwargs): linear = paddle.nn.Linear(10, 10) scheduler = paddle.optimizer.lr.ReduceOnPlateau(**kwargs) - adam = paddle.optimizer.Adam(learning_rate=scheduler, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + learning_rate=scheduler, parameters=linear.parameters() + ) for epoch in range(20): for batch_id in range(1): @@ -174,17 +189,25 @@ def _test_dygraph(self, place, kwargs): current_lr = adam.get_lr() # get lr form python expected_lr = reduce_lr_on_plateau( - kwargs['factor'], kwargs['threshold'], kwargs['cooldown'], - kwargs['patience'], kwargs['mode'], kwargs['threshold_mode'], - loss, var_list) + kwargs['factor'], + kwargs['threshold'], + kwargs['cooldown'], + kwargs['patience'], + kwargs['mode'], + kwargs['threshold_mode'], + loss, + var_list, + ) self.assertEqual(current_lr, expected_lr) state_dict = adam.state_dict() scheduler1 = paddle.optimizer.lr.ReduceOnPlateau(**kwargs) - adam1 = paddle.optimizer.Adam(learning_rate=scheduler1, - parameters=linear.parameters()) + adam1 = paddle.optimizer.Adam( + learning_rate=scheduler1, parameters=linear.parameters() + ) adam1.set_state_dict(state_dict) - self.assertEqual(scheduler.cooldown_counter, - scheduler1.cooldown_counter) + self.assertEqual( + scheduler.cooldown_counter, scheduler1.cooldown_counter + ) self.assertEqual(scheduler.best.numpy()[0], scheduler1.best) self.assertEqual(scheduler.num_bad_epochs, scheduler1.num_bad_epochs) self.assertEqual(scheduler.last_epoch, scheduler1.last_epoch) @@ -231,13 +254,15 @@ def inverse_time_lr(epoch_num, learning_rate, gamma, verbose=False): return learning_rate / (1 + gamma * epoch_num) -def polynomial_lr(epoch_num, - learning_rate, - decay_steps, - end_lr=0.0001, - power=1.0, - cycle=False, - verbose=False): +def polynomial_lr( + epoch_num, + learning_rate, + decay_steps, + end_lr=0.0001, + power=1.0, + cycle=False, + verbose=False, +): if cycle: div = math.ceil(epoch_num / float(decay_steps)) @@ -247,53 +272,58 @@ def polynomial_lr(epoch_num, else: epoch_num = min(epoch_num, decay_steps) return (learning_rate - end_lr) * ( - (1 - float(epoch_num) / float(decay_steps))**power) + end_lr + (1 - float(epoch_num) / float(decay_steps)) ** power + ) + end_lr def get_lr(self): if self.last_epoch == 0: return self.base_lr elif (self.last_epoch - 1 - self.T_max) % (2 * self.T_max) == 0: - return self.last_lr + (self.base_lr - self.eta_min) * ( - 1 - math.cos(math.pi / self.T_max)) / 2 + return ( + self.last_lr + + (self.base_lr - self.eta_min) + * (1 - math.cos(math.pi / self.T_max)) + / 2 + ) return (1 + math.cos(math.pi * self.last_epoch / self.T_max)) / ( - 1 + math.cos(math.pi * (self.last_epoch - 1) / self.T_max)) * ( - self.last_lr - self.eta_min) + self.eta_min + 1 + math.cos(math.pi * (self.last_epoch - 1) / self.T_max) + ) * (self.last_lr - self.eta_min) + self.eta_min cosine_annealing_lr_current = None -def cosine_annealing_lr(epoch_num, - learning_rate, - T_max, - eta_min=0, - verbose=False): +def cosine_annealing_lr( + epoch_num, learning_rate, T_max, eta_min=0, verbose=False +): global cosine_annealing_lr_current if epoch_num == 0: cosine_annealing_lr_current = learning_rate elif (epoch_num - 1 - T_max) % (2 * T_max) == 0: - cosine_annealing_lr_current = cosine_annealing_lr_current + ( - learning_rate - eta_min) * (1 - - math.cos(math.pi / float(T_max))) / 2 + cosine_annealing_lr_current = ( + cosine_annealing_lr_current + + (learning_rate - eta_min) + * (1 - math.cos(math.pi / float(T_max))) + / 2 + ) else: cosine_annealing_lr_current = ( - 1 + math.cos(math.pi * epoch_num / float(T_max))) / ( - 1 + math.cos(math.pi * (epoch_num - 1) / float(T_max))) * ( - cosine_annealing_lr_current - eta_min) + eta_min + 1 + math.cos(math.pi * epoch_num / float(T_max)) + ) / (1 + math.cos(math.pi * (epoch_num - 1) / float(T_max))) * ( + cosine_annealing_lr_current - eta_min + ) + eta_min return cosine_annealing_lr_current -def linear_warmup_lr(epoch_num, - learning_rate, - warmup_steps, - start_lr, - end_lr, - verbose=False): +def linear_warmup_lr( + epoch_num, learning_rate, warmup_steps, start_lr, end_lr, verbose=False +): tmp = epoch_num - warmup_steps if tmp < 0: - return start_lr + (end_lr - start_lr) * (float(epoch_num) / - float(warmup_steps)) + return start_lr + (end_lr - start_lr) * ( + float(epoch_num) / float(warmup_steps) + ) elif paddle.in_dynamic_mode(): if tmp < 3: return 0.5 @@ -305,35 +335,36 @@ def linear_warmup_lr(epoch_num, return 0.5 -def multi_step_lr(epoch_num, - learning_rate, - milestones, - gamma=0.1, - verbose=False): +def multi_step_lr( + epoch_num, learning_rate, milestones, gamma=0.1, verbose=False +): for i in range(len(milestones)): if epoch_num < milestones[i]: return learning_rate * (gamma**i) - return learning_rate * (gamma**len(milestones)) + return learning_rate * (gamma ** len(milestones)) def step_lr(epoch_num, learning_rate, step_size, gamma=0.1, verbose=False): return learning_rate * math.pow(gamma, epoch_num // step_size) -def one_cycle_lr(epoch_num, - max_learning_rate, - total_steps, - divide_factor=25, - end_learning_rate=0.0001, - phase_pct=0.3, - anneal_strategy='cos', - three_phase=False, - verbose=False): +def one_cycle_lr( + epoch_num, + max_learning_rate, + total_steps, + divide_factor=25, + end_learning_rate=0.0001, + phase_pct=0.3, + anneal_strategy='cos', + three_phase=False, + verbose=False, +): initial_lr = max_learning_rate / divide_factor if three_phase: _end_steps = [ float(phase_pct * total_steps) - 1, - float(2 * phase_pct * total_steps) - 2, total_steps - 1 + float(2 * phase_pct * total_steps) - 2, + total_steps - 1, ] _schedule_phases = [ { @@ -367,6 +398,7 @@ def one_cycle_lr(epoch_num, def anneal_func(start, end, pct): cos_out = math.cos(math.pi * pct) + 1 return end + (start - end) / 2.0 * cos_out + else: def anneal_func(start, end, pct): @@ -384,24 +416,26 @@ def anneal_func(start, end, pct): return computed_lr -def cyclic_lr(epoch_num, - base_learning_rate, - max_learning_rate, - step_size_up, - step_size_down, - mode, - exp_gamma=0.1, - scale_fn=None, - scale_mode='cycle', - verbose=False): +def cyclic_lr( + epoch_num, + base_learning_rate, + max_learning_rate, + step_size_up, + step_size_down, + mode, + exp_gamma=0.1, + scale_fn=None, + scale_mode='cycle', + verbose=False, +): total_steps = step_size_up + step_size_down step_ratio = step_size_up / total_steps def triangular(x): - return 1. + return 1.0 def triangular2(x): - return 1 / (2.**(x - 1)) + return 1 / (2.0 ** (x - 1)) def exp_range(x): return exp_gamma**x @@ -419,7 +453,7 @@ def exp_range(x): cycle = math.floor(1 + epoch_num / total_steps) iterations = epoch_num - x = 1. + epoch_num / total_steps - cycle + x = 1.0 + epoch_num / total_steps - cycle if x <= step_ratio: scale_factor = x / step_ratio @@ -432,7 +466,6 @@ def exp_range(x): class TestLRScheduler(unittest.TestCase): - def _test_static(self, python_func, paddle_api, kwarg, place): scheduler = paddle_api(**kwarg) adam = paddle.optimizer.Adam(learning_rate=scheduler) @@ -456,7 +489,8 @@ def _test_static(self, python_func, paddle_api, kwarg, place): out = exe.run( main_prog, feed={'x': np.random.randn(3, 4, 5).astype('float32')}, - fetch_list=lr_var.name) + fetch_list=lr_var.name, + ) self.assertEqual(out, np.array(python_func(num, **kwarg))) scheduler.step() num += 1 @@ -466,22 +500,26 @@ def _test_static(self, python_func, paddle_api, kwarg, place): out = exe.run( test_prog, feed={'x': np.random.randn(3, 4, 5).astype('float32')}, - fetch_list=lr_var.name) + fetch_list=lr_var.name, + ) self.assertEqual(out, np.array(python_func(num, **kwarg))) scheduler.step() num += 1 if isinstance(place, paddle.CPUPlace): compiled_train_prog = paddle.static.CompiledProgram( - main_prog).with_data_parallel(loss_name=loss.name, - places=fluid.cpu_places(4)) + main_prog + ).with_data_parallel( + loss_name=loss.name, places=fluid.cpu_places(4) + ) for epoch in range(5): python_result = python_func(num, **kwarg) for batch_id in range(2): _ = exe.run( compiled_train_prog, feed={'x': np.random.randn(12, 4, 5).astype('float32')}, - fetch_list=lr_var.name) + fetch_list=lr_var.name, + ) scopes = compiled_train_prog._executor.local_scopes() out = np.array(scopes[0].var(lr_var.name).get_tensor()) self.assertEqual(out, np.array(python_result)) @@ -495,17 +533,20 @@ def _test_static(self, python_func, paddle_api, kwarg, place): num += 1 compiled_test_prog = paddle.static.CompiledProgram( - test_prog).with_data_parallel( - loss_name=loss.name, - share_vars_from=compiled_train_prog, - places=fluid.cpu_places(4)) + test_prog + ).with_data_parallel( + loss_name=loss.name, + share_vars_from=compiled_train_prog, + places=fluid.cpu_places(4), + ) for epoch in range(5): python_result = python_func(num, **kwarg) for batch_id in range(2): _ = exe.run( compiled_test_prog, feed={'x': np.random.randn(12, 4, 5).astype('float32')}, - fetch_list=lr_var.name) + fetch_list=lr_var.name, + ) scopes = compiled_test_prog._executor.local_scopes() out = np.array(scopes[0].var(lr_var.name).get_tensor()) self.assertEqual(out, np.array(python_result)) @@ -524,10 +565,12 @@ def _test_dygraph(self, python_func, paddle_api, kwarg, place): linear = paddle.nn.Linear(10, 10) if paddle_api.__name__ == "LinearWarmup": kwarg['learning_rate'] = paddle.optimizer.lr.PiecewiseDecay( - [3, 6], [0.5, 0.2, 0.1]) + [3, 6], [0.5, 0.2, 0.1] + ) scheduler = paddle_api(**kwarg) - adam = paddle.optimizer.Adam(learning_rate=scheduler, - parameters=linear.parameters()) + adam = paddle.optimizer.Adam( + learning_rate=scheduler, parameters=linear.parameters() + ) for epoch in range(20): for batch_id in range(2): x = paddle.to_tensor(x) @@ -545,15 +588,20 @@ def _test_dygraph(self, python_func, paddle_api, kwarg, place): self.assertAlmostEqual(current_lr, expected_lr) state_dict = adam.state_dict() scheduler1 = paddle.optimizer.lr.LinearWarmup(**kwarg) - adam1 = paddle.optimizer.Adam(learning_rate=scheduler1, - parameters=linear.parameters()) + adam1 = paddle.optimizer.Adam( + learning_rate=scheduler1, parameters=linear.parameters() + ) adam1.set_state_dict(state_dict) self.assertEqual(scheduler.last_epoch, scheduler1.last_epoch) self.assertEqual(scheduler.last_lr, scheduler1.last_lr) - self.assertEqual(scheduler.learning_rate.last_lr, - scheduler1.learning_rate.last_lr) - self.assertEqual(scheduler.learning_rate.last_epoch, - scheduler1.learning_rate.last_epoch) + self.assertEqual( + scheduler.learning_rate.last_lr, + scheduler1.learning_rate.last_lr, + ) + self.assertEqual( + scheduler.learning_rate.last_epoch, + scheduler1.learning_rate.last_epoch, + ) scheduler.step() else: self.assertEqual(current_lr, expected_lr) @@ -563,266 +611,355 @@ def test_scheduler(self): with self.assertRaises(NotImplementedError): paddle.optimizer.lr.LRScheduler().step() with self.assertRaises(TypeError): - paddle.optimizer.lr.MultiStepDecay(learning_rate="test", - milestones=[1, 2, 3]) + paddle.optimizer.lr.MultiStepDecay( + learning_rate="test", milestones=[1, 2, 3] + ) with self.assertRaises(TypeError): - paddle.optimizer.lr.MultiStepDecay(learning_rate=0.5, - milestones='test') + paddle.optimizer.lr.MultiStepDecay( + learning_rate=0.5, milestones='test' + ) with self.assertRaises(ValueError): - paddle.optimizer.lr.MultiStepDecay(learning_rate=0.5, - milestones=[3, 2, 1]) + paddle.optimizer.lr.MultiStepDecay( + learning_rate=0.5, milestones=[3, 2, 1] + ) with self.assertRaises(ValueError): - paddle.optimizer.lr.MultiStepDecay(learning_rate=0.5, - milestones=[1, 2, 3], - gamma=2) + paddle.optimizer.lr.MultiStepDecay( + learning_rate=0.5, milestones=[1, 2, 3], gamma=2 + ) # check type of max_learning_rate with self.assertRaises(TypeError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate='test', - total_steps=20) + paddle.optimizer.lr.OneCycleLR( + max_learning_rate='test', total_steps=20 + ) # check value of max_learning_rate with self.assertRaises(ValueError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate=-1.5, - total_steps=20) + paddle.optimizer.lr.OneCycleLR( + max_learning_rate=-1.5, total_steps=20 + ) # check type of end_learning_rate with self.assertRaises(TypeError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate=0.1, - total_steps=20, - end_learning_rate='test') + paddle.optimizer.lr.OneCycleLR( + max_learning_rate=0.1, total_steps=20, end_learning_rate='test' + ) # check value of end_learning_rate with self.assertRaises(ValueError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate=0.1, - total_steps=20, - end_learning_rate=-1) + paddle.optimizer.lr.OneCycleLR( + max_learning_rate=0.1, total_steps=20, end_learning_rate=-1 + ) # check type of total_steps with self.assertRaises(TypeError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate=0.1, - total_steps='test') + paddle.optimizer.lr.OneCycleLR( + max_learning_rate=0.1, total_steps='test' + ) # check value of total_steps with self.assertRaises(ValueError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate=0.1, - total_steps=-10) + paddle.optimizer.lr.OneCycleLR( + max_learning_rate=0.1, total_steps=-10 + ) # check value of anneal_strategy with self.assertRaises(ValueError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate=0.1, - total_steps=20, - anneal_strategy='test') + paddle.optimizer.lr.OneCycleLR( + max_learning_rate=0.1, total_steps=20, anneal_strategy='test' + ) # check value of phase_pct when three_phase is True with self.assertRaises(ValueError): - paddle.optimizer.lr.OneCycleLR(max_learning_rate=0.1, - total_steps=20, - phase_pct=0.6, - three_phase=True) + paddle.optimizer.lr.OneCycleLR( + max_learning_rate=0.1, + total_steps=20, + phase_pct=0.6, + three_phase=True, + ) # check type of max_learning_rate with self.assertRaises(TypeError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate='test', - step_size_up=10) + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, + max_learning_rate='test', + step_size_up=10, + ) # check value of max_learning_rate with self.assertRaises(ValueError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate=-1, - step_size_up=10) + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, max_learning_rate=-1, step_size_up=10 + ) # check type of step_size_up with self.assertRaises(TypeError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate=1.0, - step_size_up='test') + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, + max_learning_rate=1.0, + step_size_up='test', + ) # check value of step_size_up with self.assertRaises(ValueError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate=1.0, - step_size_up=-1) + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, max_learning_rate=1.0, step_size_up=-1 + ) # check type of step_size_down with self.assertRaises(TypeError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate=1.0, - step_size_up=500, - step_size_down='test') + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, + max_learning_rate=1.0, + step_size_up=500, + step_size_down='test', + ) # check type of step_size_down with self.assertRaises(ValueError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate=1.0, - step_size_up=500, - step_size_down=-1) + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, + max_learning_rate=1.0, + step_size_up=500, + step_size_down=-1, + ) # check value of mode with self.assertRaises(ValueError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate=1.0, - step_size_up=500, - step_size_down=500, - mode='test') + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, + max_learning_rate=1.0, + step_size_up=500, + step_size_down=500, + mode='test', + ) # check type value of scale_mode with self.assertRaises(ValueError): - paddle.optimizer.lr.CyclicLR(base_learning_rate=0.5, - max_learning_rate=1.0, - step_size_up=500, - step_size_down=-1, - scale_mode='test') + paddle.optimizer.lr.CyclicLR( + base_learning_rate=0.5, + max_learning_rate=1.0, + step_size_up=500, + step_size_down=-1, + scale_mode='test', + ) func_api_kwargs = [ - (noam_lr, paddle.optimizer.lr.NoamDecay, { - "d_model": 0.01, - "warmup_steps": 100, - "verbose": False - }), - (piecewise_lr, paddle.optimizer.lr.PiecewiseDecay, { - "boundaries": [3, 6, 9, 15, 20], - "values": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6], - "verbose": False - }), - (natural_exp_lr, paddle.optimizer.lr.NaturalExpDecay, { - "learning_rate": 0.5, - "gamma": 0.1, - "verbose": True - }), - (inverse_time_lr, paddle.optimizer.lr.InverseTimeDecay, { - "learning_rate": 0.5, - "gamma": 0.1, - "verbose": False - }), - (polynomial_lr, paddle.optimizer.lr.PolynomialDecay, { - "learning_rate": 0.5, - "decay_steps": 20, - "end_lr": 0, - "power": 1.0, - "cycle": False - }), - (polynomial_lr, paddle.optimizer.lr.PolynomialDecay, { - "learning_rate": 0.5, - "decay_steps": 20, - "end_lr": 0, - "power": 1.0, - "cycle": True, - "verbose": False - }), - (linear_warmup_lr, paddle.optimizer.lr.LinearWarmup, { - 'learning_rate': 0.5, - 'warmup_steps': 10, - 'start_lr': 0, - 'end_lr': 0.5 - }), - (exponential_lr, paddle.optimizer.lr.ExponentialDecay, { - "learning_rate": 0.5, - "gamma": 0.9, - "verbose": False - }), - (multi_step_lr, paddle.optimizer.lr.MultiStepDecay, { - "learning_rate": 0.5, - "milestones": [3, 6, 9, 15, 20], - "gamma": 0.8 - }), - (step_lr, paddle.optimizer.lr.StepDecay, { - "learning_rate": 0.5, - "step_size": 2, - "gamma": 0.8, - "verbose": False - }), - (lambda_lr, paddle.optimizer.lr.LambdaDecay, { - "learning_rate": 0.5, - "lr_lambda": lambda x: 0.95**x, - "verbose": True - }), - (multiplicative_lr, paddle.optimizer.lr.MultiplicativeDecay, { - "learning_rate": 0.5, - "lr_lambda": lambda x: 0.95, - "verbose": True - }), - (cosine_annealing_lr, paddle.optimizer.lr.CosineAnnealingDecay, { - "learning_rate": 0.5, - "T_max": 10, - "verbose": False - }), - (one_cycle_lr, paddle.optimizer.lr.OneCycleLR, { - "max_learning_rate": 0.1, - "total_steps": 20, - "divide_factor": 5, - "end_learning_rate": 0.0001, - "anneal_strategy": 'cos', - "phase_pct": 0.3, - "three_phase": False, - }), - (one_cycle_lr, paddle.optimizer.lr.OneCycleLR, { - "max_learning_rate": 0.5, - "total_steps": 20, - "divide_factor": 10, - "end_learning_rate": 0.001, - "anneal_strategy": 'linear', - "phase_pct": 0.4, - "three_phase": False, - }), - (one_cycle_lr, paddle.optimizer.lr.OneCycleLR, { - "max_learning_rate": 1.0, - "total_steps": 20, - "divide_factor": 9, - "end_learning_rate": 0.0001, - "anneal_strategy": 'cos', - "phase_pct": 0.3, - "three_phase": True, - }), - (one_cycle_lr, paddle.optimizer.lr.OneCycleLR, { - "max_learning_rate": 0.3, - "total_steps": 20, - "divide_factor": 25, - "end_learning_rate": 0.0005, - "anneal_strategy": 'linear', - "phase_pct": 0.2, - "three_phase": True, - }), - (cyclic_lr, paddle.optimizer.lr.CyclicLR, { - "base_learning_rate": 0.5, - "max_learning_rate": 1.0, - "step_size_up": 15, - "step_size_down": 5, - "mode": 'triangular', - "exp_gamma": 1., - "scale_fn": None, - "scale_mode": 'cycle', - "verbose": False - }), - (cyclic_lr, paddle.optimizer.lr.CyclicLR, { - "base_learning_rate": 0.5, - "max_learning_rate": 1.0, - "step_size_up": 15, - "step_size_down": 5, - "mode": 'triangular2', - "exp_gamma": 1., - "scale_fn": None, - "scale_mode": 'cycle', - "verbose": False - }), - (cyclic_lr, paddle.optimizer.lr.CyclicLR, { - "base_learning_rate": 0.5, - "max_learning_rate": 1.0, - "step_size_up": 15, - "step_size_down": 5, - "mode": 'exp_range', - "exp_gamma": 0.8, - "scale_fn": None, - "scale_mode": 'cycle', - "verbose": False - }), - (cyclic_lr, paddle.optimizer.lr.CyclicLR, { - "base_learning_rate": 0.5, - "max_learning_rate": 1.0, - "step_size_up": 15, - "step_size_down": 5, - "mode": 'exp_range', - "exp_gamma": 1., - "scale_fn": lambda x: 0.95**x, - "scale_mode": 'cycle', - "verbose": False - }), - (cyclic_lr, paddle.optimizer.lr.CyclicLR, { - "base_learning_rate": 0.5, - "max_learning_rate": 1.0, - "step_size_up": 15, - "step_size_down": 5, - "mode": 'exp_range', - "exp_gamma": 1., - "scale_fn": lambda x: 0.95, - "scale_mode": 'iterations', - "verbose": False - }) + ( + noam_lr, + paddle.optimizer.lr.NoamDecay, + {"d_model": 0.01, "warmup_steps": 100, "verbose": False}, + ), + ( + piecewise_lr, + paddle.optimizer.lr.PiecewiseDecay, + { + "boundaries": [3, 6, 9, 15, 20], + "values": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6], + "verbose": False, + }, + ), + ( + natural_exp_lr, + paddle.optimizer.lr.NaturalExpDecay, + {"learning_rate": 0.5, "gamma": 0.1, "verbose": True}, + ), + ( + inverse_time_lr, + paddle.optimizer.lr.InverseTimeDecay, + {"learning_rate": 0.5, "gamma": 0.1, "verbose": False}, + ), + ( + polynomial_lr, + paddle.optimizer.lr.PolynomialDecay, + { + "learning_rate": 0.5, + "decay_steps": 20, + "end_lr": 0, + "power": 1.0, + "cycle": False, + }, + ), + ( + polynomial_lr, + paddle.optimizer.lr.PolynomialDecay, + { + "learning_rate": 0.5, + "decay_steps": 20, + "end_lr": 0, + "power": 1.0, + "cycle": True, + "verbose": False, + }, + ), + ( + linear_warmup_lr, + paddle.optimizer.lr.LinearWarmup, + { + 'learning_rate': 0.5, + 'warmup_steps': 10, + 'start_lr': 0, + 'end_lr': 0.5, + }, + ), + ( + exponential_lr, + paddle.optimizer.lr.ExponentialDecay, + {"learning_rate": 0.5, "gamma": 0.9, "verbose": False}, + ), + ( + multi_step_lr, + paddle.optimizer.lr.MultiStepDecay, + { + "learning_rate": 0.5, + "milestones": [3, 6, 9, 15, 20], + "gamma": 0.8, + }, + ), + ( + step_lr, + paddle.optimizer.lr.StepDecay, + { + "learning_rate": 0.5, + "step_size": 2, + "gamma": 0.8, + "verbose": False, + }, + ), + ( + lambda_lr, + paddle.optimizer.lr.LambdaDecay, + { + "learning_rate": 0.5, + "lr_lambda": lambda x: 0.95**x, + "verbose": True, + }, + ), + ( + multiplicative_lr, + paddle.optimizer.lr.MultiplicativeDecay, + { + "learning_rate": 0.5, + "lr_lambda": lambda x: 0.95, + "verbose": True, + }, + ), + ( + cosine_annealing_lr, + paddle.optimizer.lr.CosineAnnealingDecay, + {"learning_rate": 0.5, "T_max": 10, "verbose": False}, + ), + ( + one_cycle_lr, + paddle.optimizer.lr.OneCycleLR, + { + "max_learning_rate": 0.1, + "total_steps": 20, + "divide_factor": 5, + "end_learning_rate": 0.0001, + "anneal_strategy": 'cos', + "phase_pct": 0.3, + "three_phase": False, + }, + ), + ( + one_cycle_lr, + paddle.optimizer.lr.OneCycleLR, + { + "max_learning_rate": 0.5, + "total_steps": 20, + "divide_factor": 10, + "end_learning_rate": 0.001, + "anneal_strategy": 'linear', + "phase_pct": 0.4, + "three_phase": False, + }, + ), + ( + one_cycle_lr, + paddle.optimizer.lr.OneCycleLR, + { + "max_learning_rate": 1.0, + "total_steps": 20, + "divide_factor": 9, + "end_learning_rate": 0.0001, + "anneal_strategy": 'cos', + "phase_pct": 0.3, + "three_phase": True, + }, + ), + ( + one_cycle_lr, + paddle.optimizer.lr.OneCycleLR, + { + "max_learning_rate": 0.3, + "total_steps": 20, + "divide_factor": 25, + "end_learning_rate": 0.0005, + "anneal_strategy": 'linear', + "phase_pct": 0.2, + "three_phase": True, + }, + ), + ( + cyclic_lr, + paddle.optimizer.lr.CyclicLR, + { + "base_learning_rate": 0.5, + "max_learning_rate": 1.0, + "step_size_up": 15, + "step_size_down": 5, + "mode": 'triangular', + "exp_gamma": 1.0, + "scale_fn": None, + "scale_mode": 'cycle', + "verbose": False, + }, + ), + ( + cyclic_lr, + paddle.optimizer.lr.CyclicLR, + { + "base_learning_rate": 0.5, + "max_learning_rate": 1.0, + "step_size_up": 15, + "step_size_down": 5, + "mode": 'triangular2', + "exp_gamma": 1.0, + "scale_fn": None, + "scale_mode": 'cycle', + "verbose": False, + }, + ), + ( + cyclic_lr, + paddle.optimizer.lr.CyclicLR, + { + "base_learning_rate": 0.5, + "max_learning_rate": 1.0, + "step_size_up": 15, + "step_size_down": 5, + "mode": 'exp_range', + "exp_gamma": 0.8, + "scale_fn": None, + "scale_mode": 'cycle', + "verbose": False, + }, + ), + ( + cyclic_lr, + paddle.optimizer.lr.CyclicLR, + { + "base_learning_rate": 0.5, + "max_learning_rate": 1.0, + "step_size_up": 15, + "step_size_down": 5, + "mode": 'exp_range', + "exp_gamma": 1.0, + "scale_fn": lambda x: 0.95**x, + "scale_mode": 'cycle', + "verbose": False, + }, + ), + ( + cyclic_lr, + paddle.optimizer.lr.CyclicLR, + { + "base_learning_rate": 0.5, + "max_learning_rate": 1.0, + "step_size_up": 15, + "step_size_down": 5, + "mode": 'exp_range', + "exp_gamma": 1.0, + "scale_fn": lambda x: 0.95, + "scale_mode": 'iterations', + "verbose": False, + }, + ), ] for python_func, paddle_api, kwarg in func_api_kwargs: @@ -838,14 +975,17 @@ def test_scheduler(self): paddle.enable_static() def test_linear_warmp(self): - natural_lr = paddle.optimizer.lr.NaturalExpDecay(learning_rate=0.5, - gamma=0.1) + natural_lr = paddle.optimizer.lr.NaturalExpDecay( + learning_rate=0.5, gamma=0.1 + ) natural_lr_warmup = paddle.optimizer.lr.LinearWarmup( - learning_rate=natural_lr, warmup_steps=10, start_lr=0.0, end_lr=0.1) + learning_rate=natural_lr, warmup_steps=10, start_lr=0.0, end_lr=0.1 + ) for idx in range(30): if idx >= 10: - self.assertEqual(natural_lr_warmup.get_lr(), - natural_lr.get_lr()) + self.assertEqual( + natural_lr_warmup.get_lr(), natural_lr.get_lr() + ) natural_lr.step() natural_lr_warmup.step() diff --git a/python/paddle/fluid/tests/unittests/test_lrn_op.py b/python/paddle/fluid/tests/unittests/test_lrn_op.py index c87d3067cc4fd..6044628460000 100644 --- a/python/paddle/fluid/tests/unittests/test_lrn_op.py +++ b/python/paddle/fluid/tests/unittests/test_lrn_op.py @@ -22,7 +22,6 @@ class TestLRNOp(OpTest): - def get_input(self): r''' TODO(gongweibao): why it's grad diff is so large? x = np.ndarray( @@ -64,7 +63,7 @@ def get_attrs(self): 'k': self.k, 'alpha': self.alpha, 'beta': self.beta, - 'data_format': self.data_format + 'data_format': self.data_format, } return attrs @@ -103,13 +102,11 @@ def test_check_grad_normal(self): class TestLRNOpAttrDataFormat(TestLRNOp): - def init_test_case(self): self.data_format = 'NHWC' class TestLRNAPI(unittest.TestCase): - def test_case(self): data1 = fluid.data(name='data1', shape=[2, 4, 5, 5], dtype='float32') data2 = fluid.data(name='data2', shape=[2, 5, 5, 4], dtype='float32') @@ -124,23 +121,22 @@ def test_case(self): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "data1": data1_np, - "data2": data2_np - }, - fetch_list=[out1, out2], - return_numpy=True) - - np.testing.assert_allclose(results[0], - np.transpose(results[1], (0, 3, 1, 2)), - rtol=1e-05) + results = exe.run( + fluid.default_main_program(), + feed={"data1": data1_np, "data2": data2_np}, + fetch_list=[out1, out2], + return_numpy=True, + ) + + np.testing.assert_allclose( + results[0], np.transpose(results[1], (0, 3, 1, 2)), rtol=1e-05 + ) def test_exception(self): input1 = fluid.data(name="input1", shape=[2, 4, 5, 5], dtype="float32") - input2 = fluid.data(name="input2", - shape=[2, 4, 5, 5, 5], - dtype="float32") + input2 = fluid.data( + name="input2", shape=[2, 4, 5, 5, 5], dtype="float32" + ) def _attr_data_fromat(): out = fluid.layers.lrn(input1, data_format='NDHW') @@ -153,7 +149,6 @@ def _input_dim_size(): class TestLRNOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input must be float32 @@ -162,7 +157,6 @@ def test_errors(self): class TestLocalResponseNormFAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -174,84 +168,81 @@ def check_static_3d_input(self, place): in_np1 = np.random.random([3, 40, 40]).astype("float32") in_np2 = np.transpose(in_np1, (0, 2, 1)) - input1 = fluid.data(name="input1", - shape=[3, 40, 40], - dtype="float32") - input2 = fluid.data(name="input2", - shape=[3, 40, 40], - dtype="float32") - res1 = paddle.nn.functional.local_response_norm(x=input1, - size=5, - data_format='NCL') - res2 = paddle.nn.functional.local_response_norm(x=input2, - size=5, - data_format='NLC') + input1 = fluid.data( + name="input1", shape=[3, 40, 40], dtype="float32" + ) + input2 = fluid.data( + name="input2", shape=[3, 40, 40], dtype="float32" + ) + res1 = paddle.nn.functional.local_response_norm( + x=input1, size=5, data_format='NCL' + ) + res2 = paddle.nn.functional.local_response_norm( + x=input2, size=5, data_format='NLC' + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input1": in_np1, - "input2": in_np2 - }, - fetch_list=[res1, res2]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input1": in_np1, "input2": in_np2}, + fetch_list=[res1, res2], + ) fetches1_tran = np.transpose(fetches[1], (0, 2, 1)) np.testing.assert_allclose(fetches[0], fetches1_tran, rtol=1e-05) def check_static_4d_input(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input1 = fluid.data(name="input1", - shape=[3, 3, 40, 40], - dtype="float32") - input2 = fluid.data(name="input2", - shape=[3, 40, 40, 3], - dtype="float32") - - res1 = paddle.nn.functional.local_response_norm(x=input1, - size=5, - data_format='NCHW') - res2 = paddle.nn.functional.local_response_norm(x=input2, - size=5, - data_format='NHWC') + input1 = fluid.data( + name="input1", shape=[3, 3, 40, 40], dtype="float32" + ) + input2 = fluid.data( + name="input2", shape=[3, 40, 40, 3], dtype="float32" + ) + + res1 = paddle.nn.functional.local_response_norm( + x=input1, size=5, data_format='NCHW' + ) + res2 = paddle.nn.functional.local_response_norm( + x=input2, size=5, data_format='NHWC' + ) in_np1 = np.random.random([3, 3, 40, 40]).astype("float32") in_np2 = np.transpose(in_np1, (0, 2, 3, 1)) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input1": in_np1, - "input2": in_np2 - }, - fetch_list=[res1, res2]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input1": in_np1, "input2": in_np2}, + fetch_list=[res1, res2], + ) fetches1_tran = np.transpose(fetches[1], (0, 3, 1, 2)) np.testing.assert_allclose(fetches[0], fetches1_tran, rtol=1e-05) def check_static_5d_input(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input1 = fluid.data(name="input1", - shape=[3, 3, 3, 40, 40], - dtype="float32") - input2 = fluid.data(name="input2", - shape=[3, 3, 40, 40, 3], - dtype="float32") - res1 = paddle.nn.functional.local_response_norm(x=input1, - size=5, - data_format='NCDHW') - res2 = paddle.nn.functional.local_response_norm(x=input2, - size=5, - data_format='NDHWC') + input1 = fluid.data( + name="input1", shape=[3, 3, 3, 40, 40], dtype="float32" + ) + input2 = fluid.data( + name="input2", shape=[3, 3, 40, 40, 3], dtype="float32" + ) + res1 = paddle.nn.functional.local_response_norm( + x=input1, size=5, data_format='NCDHW' + ) + res2 = paddle.nn.functional.local_response_norm( + x=input2, size=5, data_format='NDHWC' + ) in_np1 = np.random.random([3, 3, 3, 40, 40]).astype("float32") in_np2 = np.transpose(in_np1, (0, 2, 3, 4, 1)) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input1": in_np1, - "input2": in_np2 - }, - fetch_list=[res1, res2]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input1": in_np1, "input2": in_np2}, + fetch_list=[res1, res2], + ) fetches1_tran = np.transpose(fetches[1], (0, 4, 1, 2, 3)) np.testing.assert_allclose(fetches[0], fetches1_tran, rtol=1e-05) @@ -270,12 +261,12 @@ def check_dygraph_3d_input(self, place): in1 = paddle.to_tensor(in_np1) in2 = paddle.to_tensor(in_np2) - res1 = paddle.nn.functional.local_response_norm(x=in1, - size=5, - data_format='NCL') - res2 = paddle.nn.functional.local_response_norm(x=in2, - size=5, - data_format='NLC') + res1 = paddle.nn.functional.local_response_norm( + x=in1, size=5, data_format='NCL' + ) + res2 = paddle.nn.functional.local_response_norm( + x=in2, size=5, data_format='NLC' + ) res2_tran = np.transpose(res2.numpy(), (0, 2, 1)) np.testing.assert_allclose(res1.numpy(), res2_tran, rtol=1e-05) @@ -288,12 +279,12 @@ def check_dygraph_4d_input(self, place): in1 = paddle.to_tensor(in_np1) in2 = paddle.to_tensor(in_np2) - res1 = paddle.nn.functional.local_response_norm(x=in1, - size=5, - data_format='NCHW') - res2 = paddle.nn.functional.local_response_norm(x=in2, - size=5, - data_format='NHWC') + res1 = paddle.nn.functional.local_response_norm( + x=in1, size=5, data_format='NCHW' + ) + res2 = paddle.nn.functional.local_response_norm( + x=in2, size=5, data_format='NHWC' + ) res2_tran = np.transpose(res2.numpy(), (0, 3, 1, 2)) np.testing.assert_allclose(res1.numpy(), res2_tran, rtol=1e-05) @@ -306,12 +297,12 @@ def check_dygraph_5d_input(self, place): in1 = paddle.to_tensor(in_np1) in2 = paddle.to_tensor(in_np2) - res1 = paddle.nn.functional.local_response_norm(x=in1, - size=5, - data_format='NCDHW') - res2 = paddle.nn.functional.local_response_norm(x=in2, - size=5, - data_format='NDHWC') + res1 = paddle.nn.functional.local_response_norm( + x=in1, size=5, data_format='NCDHW' + ) + res2 = paddle.nn.functional.local_response_norm( + x=in2, size=5, data_format='NDHWC' + ) res2_tran = np.transpose(res2.numpy(), (0, 4, 1, 2, 3)) np.testing.assert_allclose(res1.numpy(), res2_tran, rtol=1e-05) @@ -324,14 +315,14 @@ def test_dygraph(self): class TestLocalResponseNormFAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of lrn must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace() + ) paddle.nn.functional.local_response_norm(x1, size=5) self.assertRaises(TypeError, test_Variable) @@ -344,9 +335,9 @@ def test_datatype(): def test_dataformat(): x = fluid.data(name='x', shape=[3, 4, 5, 6], dtype="float32") - paddle.nn.functional.local_response_norm(x, - size=5, - data_format="NCTHW") + paddle.nn.functional.local_response_norm( + x, size=5, data_format="NCTHW" + ) self.assertRaises(ValueError, test_dataformat) @@ -364,7 +355,6 @@ def test_shape(): class TestLocalResponseNormCAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_lstm_cudnn_op.py b/python/paddle/fluid/tests/unittests/test_lstm_cudnn_op.py index 53949bbb738e9..3be8fa8e3c170 100644 --- a/python/paddle/fluid/tests/unittests/test_lstm_cudnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstm_cudnn_op.py @@ -33,7 +33,6 @@ class RandomWeight: - def __init__(self): pass @@ -43,34 +42,29 @@ def updata_weight(self, hidden_size, input_size, dtype): self.input_size = input_size self.dtype = dtype - self.weight_ih = np.random.uniform(low=-std, - high=std, - size=(4 * self.hidden_size, - self.input_size)).astype(dtype) + self.weight_ih = np.random.uniform( + low=-std, high=std, size=(4 * self.hidden_size, self.input_size) + ).astype(dtype) self.weight_hh = np.random.uniform( - low=-std, high=std, - size=(4 * self.hidden_size, self.hidden_size)).astype(dtype) - self.bias_ih = np.random.uniform(low=-std, - high=std, - size=(4 * - self.hidden_size)).astype(dtype) - self.bias_hh = np.random.uniform(low=-std, - high=std, - size=(4 * - self.hidden_size)).astype(dtype) + low=-std, high=std, size=(4 * self.hidden_size, self.hidden_size) + ).astype(dtype) + self.bias_ih = np.random.uniform( + low=-std, high=std, size=(4 * self.hidden_size) + ).astype(dtype) + self.bias_hh = np.random.uniform( + low=-std, high=std, size=(4 * self.hidden_size) + ).astype(dtype) weight = RandomWeight() class LayerMixin(object): - def __call__(self, *args, **kwargs): return self.forward(*args, **kwargs) class LayerListMixin(LayerMixin): - def __init__(self, layers=None): self._layers = list(layers) if layers else [] @@ -82,7 +76,6 @@ def __iter__(self): class LSTMCell(LayerMixin): - def __init__(self, input_size, hidden_size, bias=True): self.input_size = input_size self.hidden_size = hidden_size @@ -145,12 +138,14 @@ def update_state(mask, new, old): return tuple(map(lambda x, y: np.where(mask, x, y), new, old)) -def rnn(cell, - inputs, - initial_states, - sequence_length=None, - time_major=False, - is_reverse=False): +def rnn( + cell, + inputs, + initial_states, + sequence_length=None, + time_major=False, + is_reverse=False, +): if not time_major: inputs = np.transpose(inputs, [1, 0, 2]) if is_reverse: @@ -172,7 +167,7 @@ def rnn(cell, if mask is not None: m_t = mask[t] y, new_state = cell(x_t, state) - y = np.where(m_t, y, 0.) + y = np.where(m_t, y, 0.0) outputs.append(y) state = update_state(m_t, new_state, state) else: @@ -190,25 +185,27 @@ def rnn(cell, return outputs, final_state -def birnn(cell_fw, - cell_bw, - inputs, - initial_states, - sequence_length=None, - time_major=False): +def birnn( + cell_fw, + cell_bw, + inputs, + initial_states, + sequence_length=None, + time_major=False, +): states_fw, states_bw = initial_states - outputs_fw, states_fw = rnn(cell_fw, - inputs, - states_fw, - sequence_length, - time_major=time_major) - - outputs_bw, states_bw = rnn(cell_bw, - inputs, - states_bw, - sequence_length, - time_major=time_major, - is_reverse=True) + outputs_fw, states_fw = rnn( + cell_fw, inputs, states_fw, sequence_length, time_major=time_major + ) + + outputs_bw, states_bw = rnn( + cell_bw, + inputs, + states_bw, + sequence_length, + time_major=time_major, + is_reverse=True, + ) outputs = np.concatenate((outputs_fw, outputs_bw), -1) final_states = (states_fw, states_bw) @@ -271,7 +268,6 @@ def concat_states(states, bidirectional=False, state_components=1): class RNN(LayerMixin): - def __init__(self, cell, is_reverse=False, time_major=False): super(RNN, self).__init__() self.cell = cell @@ -282,59 +278,69 @@ def __init__(self, cell, is_reverse=False, time_major=False): self.time_major = time_major def forward(self, inputs, initial_states=None, sequence_length=None): - final_outputs, final_states = rnn(self.cell, - inputs, - initial_states=initial_states, - sequence_length=sequence_length, - time_major=self.time_major, - is_reverse=self.is_reverse) + final_outputs, final_states = rnn( + self.cell, + inputs, + initial_states=initial_states, + sequence_length=sequence_length, + time_major=self.time_major, + is_reverse=self.is_reverse, + ) return final_outputs, final_states class BiRNN(LayerMixin): - def __init__(self, cell_fw, cell_bw, time_major=False): super(BiRNN, self).__init__() self.cell_fw = cell_fw self.cell_bw = cell_bw self.time_major = time_major - def forward(self, - inputs, - initial_states=None, - sequence_length=None, - **kwargs): + def forward( + self, inputs, initial_states=None, sequence_length=None, **kwargs + ): if isinstance(initial_states, (list, tuple)): - assert len(initial_states) == 2, \ - "length of initial_states should be 2 when it is a list/tuple" + assert ( + len(initial_states) == 2 + ), "length of initial_states should be 2 when it is a list/tuple" else: initial_states = [initial_states, initial_states] - outputs, final_states = birnn(self.cell_fw, self.cell_bw, inputs, - initial_states, sequence_length, - self.time_major) + outputs, final_states = birnn( + self.cell_fw, + self.cell_bw, + inputs, + initial_states, + sequence_length, + self.time_major, + ) return outputs, final_states class RNNMixin(LayerListMixin): - def forward(self, inputs, initial_states=None, sequence_length=None): batch_index = 1 if self.time_major else 0 batch_size = inputs.shape[batch_index] dtype = inputs.dtype if initial_states is None: - state_shape = (self.num_layers * self.num_directions, batch_size, - self.hidden_size) + state_shape = ( + self.num_layers * self.num_directions, + batch_size, + self.hidden_size, + ) if self.state_components == 1: initial_states = np.zeros(state_shape, dtype) else: - initial_states = tuple([ - np.zeros(state_shape, dtype) - for _ in range(self.state_components) - ]) - - states = split_states(initial_states, self.num_directions == 2, - self.state_components) + initial_states = tuple( + [ + np.zeros(state_shape, dtype) + for _ in range(self.state_components) + ] + ) + + states = split_states( + initial_states, self.num_directions == 2, self.state_components + ) final_states = [] for i, rnn_layer in enumerate(self): @@ -344,20 +350,22 @@ def forward(self, inputs, initial_states=None, sequence_length=None): final_states.append(final_state) inputs = outputs - final_states = concat_states(final_states, self.num_directions == 2, - self.state_components) + final_states = concat_states( + final_states, self.num_directions == 2, self.state_components + ) return outputs, final_states class LSTM(RNNMixin): - - def __init__(self, - input_size, - hidden_size, - num_layers=1, - direction="forward", - dropout=0., - time_major=False): + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + direction="forward", + dropout=0.0, + time_major=False, + ): super(LSTM, self).__init__() if direction in ["forward", "backward"]: @@ -378,7 +386,8 @@ def __init__(self, else: raise ValueError( "direction should be forward, backward or bidirectional, " - "received direction = {}".format(direction)) + "received direction = {}".format(direction) + ) self.input_size = input_size self.hidden_size = hidden_size @@ -389,10 +398,10 @@ def __init__(self, self.state_components = 2 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNLstmOp(OpTest): - def get_weight_names(self): weight_names = [] for i in range(2 * self.num_layers): @@ -404,8 +413,11 @@ def get_weight_names(self): def setUp(self): self.op_type = "cudnn_lstm" self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.sequence_length = None if core.is_compiled_with_rocm( - ) else np.array([12, 11, 10, 9, 8], dtype=np.int32) + self.sequence_length = ( + None + if core.is_compiled_with_rocm() + else np.array([12, 11, 10, 9, 8], dtype=np.int32) + ) self.num_layers = 1 self.set_attrs() @@ -414,24 +426,26 @@ def setUp(self): input_size = 21 hidden_size = 21 - input = np.random.uniform(low=-0.1, - high=0.1, - size=(seq_length, batch_size, - input_size)).astype(self.dtype) + input = np.random.uniform( + low=-0.1, high=0.1, size=(seq_length, batch_size, input_size) + ).astype(self.dtype) input[11][1:][:] = 0 input[10][2:][:] = 0 input[9][3:][:] = 0 input[8][4:][:] = 0 weight.updata_weight(hidden_size, input_size, self.dtype) - rnn1 = LSTM(input_size, - hidden_size, - num_layers=self.num_layers, - time_major=True, - direction="forward") - - output, (last_hidden, - last_cell) = rnn1(input, sequence_length=self.sequence_length) + rnn1 = LSTM( + input_size, + hidden_size, + num_layers=self.num_layers, + time_major=True, + direction="forward", + ) + + output, (last_hidden, last_cell) = rnn1( + input, sequence_length=self.sequence_length + ) flat_w = [] num = 0 @@ -455,10 +469,12 @@ def setUp(self): bias_hh = weight.bias_hh flat_w.append(("bias" + str(num), bias_hh)) num += 1 - init_h = np.zeros( - (self.num_layers, batch_size, hidden_size)).astype(self.dtype) - init_c = np.zeros( - (self.num_layers, batch_size, hidden_size)).astype(self.dtype) + init_h = np.zeros((self.num_layers, batch_size, hidden_size)).astype( + self.dtype + ) + init_c = np.zeros((self.num_layers, batch_size, hidden_size)).astype( + self.dtype + ) state_out = np.ndarray((300)).astype("uint8") if core.is_compiled_with_rocm(): @@ -473,7 +489,7 @@ def setUp(self): 'WeightList': flat_w, 'InitH': init_h, 'InitC': init_c, - 'SequenceLength': self.sequence_length + 'SequenceLength': self.sequence_length, } if self.sequence_length is None: self.inputs = { @@ -494,7 +510,7 @@ def setUp(self): "LastH": last_hidden, 'LastC': last_cell, 'Reserve': np.ndarray((400)).astype("uint8"), - 'StateOut': state_out + 'StateOut': state_out, } def set_attrs(self): @@ -503,26 +519,29 @@ def set_attrs(self): def test_output_with_place(self): place = core.CUDAPlace(0) if core.is_compiled_with_rocm(): - self.check_output_with_place(place, - atol=1e-5, - no_check_set=['Reserve', 'StateOut']) + self.check_output_with_place( + place, atol=1e-5, no_check_set=['Reserve', 'StateOut'] + ) else: - self.check_output_with_place(place, - no_check_set=['Reserve', 'StateOut']) + self.check_output_with_place( + place, no_check_set=['Reserve', 'StateOut'] + ) def test_grad_with_place(self): place = core.CUDAPlace(0) var_name_list = self.get_weight_names() for var_name in var_name_list: self.check_grad_with_place( - place, set(['Input', var_name, 'InitH', 'InitC']), - ['Out', 'LastH', 'LastC']) + place, + set(['Input', var_name, 'InitH', 'InitC']), + ['Out', 'LastH', 'LastC'], + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNlstmAPI(unittest.TestCase): - def test_lstm(self): seq_len = 20 batch_size = 5 @@ -530,31 +549,41 @@ def test_lstm(self): dropout_prob = 0.0 num_layers = 1 dtype = 'float32' if core.is_compiled_with_rocm() else 'float64' - input = fluid.data(name='input', - shape=[seq_len, batch_size, hidden_size], - dtype=dtype) - init_h = layers.fill_constant([num_layers, batch_size, hidden_size], - dtype, 0.0) - init_c = layers.fill_constant([num_layers, batch_size, hidden_size], - dtype, 0.0) - rnn_out, last_h, last_c = layers.lstm(input, init_h, init_c, seq_len, - hidden_size, num_layers, - dropout_prob, False) + input = fluid.data( + name='input', shape=[seq_len, batch_size, hidden_size], dtype=dtype + ) + init_h = layers.fill_constant( + [num_layers, batch_size, hidden_size], dtype, 0.0 + ) + init_c = layers.fill_constant( + [num_layers, batch_size, hidden_size], dtype, 0.0 + ) + rnn_out, last_h, last_c = layers.lstm( + input, + init_h, + init_c, + seq_len, + hidden_size, + num_layers, + dropout_prob, + False, + ) exe = fluid.Executor(fluid.CUDAPlace(0)) exe.run(fluid.default_startup_program()) - input_i = np.random.uniform(low=-0.1, - high=0.1, - size=(seq_len, batch_size, - hidden_size)).astype("float64") - out = exe.run(fluid.default_main_program(), - feed={'input': input_i}, - fetch_list=[rnn_out, last_h, last_c, 'cudnn_lstm_0.w_0']) - - -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + input_i = np.random.uniform( + low=-0.1, high=0.1, size=(seq_len, batch_size, hidden_size) + ).astype("float64") + out = exe.run( + fluid.default_main_program(), + feed={'input': input_i}, + fetch_list=[rnn_out, last_h, last_c, 'cudnn_lstm_0.w_0'], + ) + + +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNlstmAPI(unittest.TestCase): - def test_lstm(self): seq_len = 20 batch_size = 5 @@ -562,25 +591,36 @@ def test_lstm(self): dropout_prob = 0.0 num_layers = 2 dtype = 'float32' if core.is_compiled_with_rocm() else 'float64' - input = fluid.data(name='input', - shape=[seq_len, batch_size, hidden_size], - dtype=dtype) - init_h = layers.fill_constant([num_layers, batch_size, hidden_size], - dtype, 0.0) - init_c = layers.fill_constant([num_layers, batch_size, hidden_size], - dtype, 0.0) - rnn_out, last_h, last_c = layers.lstm(input, init_h, init_c, seq_len, - hidden_size, num_layers, - dropout_prob, False, True) + input = fluid.data( + name='input', shape=[seq_len, batch_size, hidden_size], dtype=dtype + ) + init_h = layers.fill_constant( + [num_layers, batch_size, hidden_size], dtype, 0.0 + ) + init_c = layers.fill_constant( + [num_layers, batch_size, hidden_size], dtype, 0.0 + ) + rnn_out, last_h, last_c = layers.lstm( + input, + init_h, + init_c, + seq_len, + hidden_size, + num_layers, + dropout_prob, + False, + True, + ) exe = fluid.Executor(fluid.CUDAPlace(0)) exe.run(fluid.default_startup_program()) - input_i = np.random.uniform(low=-0.1, - high=0.1, - size=(seq_len, batch_size, - hidden_size)).astype(dtype) - out = exe.run(fluid.default_main_program(), - feed={'input': input_i}, - fetch_list=[rnn_out, last_h, last_c, 'cudnn_lstm_0.w_0']) + input_i = np.random.uniform( + low=-0.1, high=0.1, size=(seq_len, batch_size, hidden_size) + ).astype(dtype) + out = exe.run( + fluid.default_main_program(), + feed={'input': input_i}, + fetch_list=[rnn_out, last_h, last_c, 'cudnn_lstm_0.w_0'], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_lstm_op.py b/python/paddle/fluid/tests/unittests/test_lstm_op.py index db701a7b733ce..aeb9b28c860c0 100644 --- a/python/paddle/fluid/tests/unittests/test_lstm_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstm_op.py @@ -33,13 +33,13 @@ def sigmoid(x): y = np.copy(x) y[x < SIGMOID_THRESHOLD_MIN] = SIGMOID_THRESHOLD_MIN y[x > SIGMOID_THRESHOLD_MAX] = SIGMOID_THRESHOLD_MAX - return 1. / (1. + np.exp(-y)) + return 1.0 / (1.0 + np.exp(-y)) def tanh(x): - y = -2. * x + y = -2.0 * x y[y > EXP_MAX_INPUT] = EXP_MAX_INPUT - return (2. / (1. + np.exp(y))) - 1. + return (2.0 / (1.0 + np.exp(y))) - 1.0 def relu(x): @@ -50,23 +50,23 @@ def relu(x): 'identity': identity, 'sigmoid': sigmoid, 'tanh': tanh, - 'relu': relu + 'relu': relu, } def lstm( - input, # T x 4D - lod, # 1 x N - h0=None, # N x D - c0=None, # N x D - w_h=None, # D x 4D - w_b=None, # 1 x 4D - w_c=None, # 1 x 3D - is_reverse=False, - act_gate=None, - act_cell=None, - act_cand=None): - + input, # T x 4D + lod, # 1 x N + h0=None, # N x D + c0=None, # N x D + w_h=None, # D x 4D + w_b=None, # 1 x 4D + w_c=None, # 1 x 3D + is_reverse=False, + act_gate=None, + act_cell=None, + act_cand=None, +): def _step(x, w_h, w_c, h_pre, c_pre, act_gate, act_cell, act_cand): g = np.dot(h_pre, w_h) # 1 x 4D g = g + x @@ -108,13 +108,14 @@ def _reverse(x, offset): for i in range(batch_size): # compute one sequence seq_len = lod[0][i] - x = input[offset[i]:offset[i + 1], :] + x = input[offset[i] : offset[i + 1], :] h_pre = h0[i] # 1 x D c_pre = c0[i] # 1 x D for j in range(seq_len): # compute one step - h_pre, c_pre = _step(x[j], w_h, w_c, h_pre, c_pre, act_gate, - act_cell, act_cand) + h_pre, c_pre = _step( + x[j], w_h, w_c, h_pre, c_pre, act_gate, act_cell, act_cand + ) hidden.append(h_pre.flatten()) cell.append(c_pre.flatten()) @@ -130,7 +131,6 @@ def _reverse(x, offset): class LstmUnitTestError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_size = 20 @@ -138,77 +138,123 @@ def test_errors(self): dropout_prob = 0.2 hidden_size = 150 num_layers = 1 - input = fluid.data(name='input', - shape=[batch_size, seq_len, hidden_size], - dtype='float32') - pre_hidden = fill_constant([num_layers, batch_size, hidden_size], - 'float32', 0.0) - pre_cell = fill_constant([num_layers, batch_size, hidden_size], - 'float32', 0.0) + input = fluid.data( + name='input', + shape=[batch_size, seq_len, hidden_size], + dtype='float32', + ) + pre_hidden = fill_constant( + [num_layers, batch_size, hidden_size], 'float32', 0.0 + ) + pre_cell = fill_constant( + [num_layers, batch_size, hidden_size], 'float32', 0.0 + ) np_input = np.random.uniform( - -0.1, 0.1, (batch_size, seq_len, hidden_size)).astype('float64') + -0.1, 0.1, (batch_size, seq_len, hidden_size) + ).astype('float64') np_pre_hidden = np.random.uniform( - -0.1, 0.1, - (num_layers, batch_size, hidden_size)).astype('float64') + -0.1, 0.1, (num_layers, batch_size, hidden_size) + ).astype('float64') np_pre_cell = np.random.uniform( - -0.1, 0.1, - (num_layers, batch_size, hidden_size)).astype('float64') + -0.1, 0.1, (num_layers, batch_size, hidden_size) + ).astype('float64') def test_input_Variable(): - LSTM(np_input, pre_hidden, pre_cell, \ - seq_len, hidden_size, num_layers, \ - dropout_prob=dropout_prob) + LSTM( + np_input, + pre_hidden, + pre_cell, + seq_len, + hidden_size, + num_layers, + dropout_prob=dropout_prob, + ) self.assertRaises(TypeError, test_input_Variable) def test_pre_hidden_Variable(): - LSTM(np_input, np_pre_hidden, pre_cell, \ - seq_len, hidden_size, num_layers, \ - dropout_prob=dropout_prob) + LSTM( + np_input, + np_pre_hidden, + pre_cell, + seq_len, + hidden_size, + num_layers, + dropout_prob=dropout_prob, + ) self.assertRaises(TypeError, test_pre_hidden_Variable) def test_pre_cell_Variable(): - LSTM(np_input, pre_hidden, np_pre_cell, \ - seq_len, hidden_size, num_layers, \ - dropout_prob=dropout_prob) + LSTM( + np_input, + pre_hidden, + np_pre_cell, + seq_len, + hidden_size, + num_layers, + dropout_prob=dropout_prob, + ) self.assertRaises(TypeError, test_pre_cell_Variable) def test_input_type(): - error_input = fluid.data(name='error_input', - shape=[None, hidden_size * 3], - dtype='int32') - LSTM(error_input, pre_hidden, pre_cell, \ - seq_len, hidden_size, num_layers, \ - dropout_prob=dropout_prob) + error_input = fluid.data( + name='error_input', + shape=[None, hidden_size * 3], + dtype='int32', + ) + LSTM( + error_input, + pre_hidden, + pre_cell, + seq_len, + hidden_size, + num_layers, + dropout_prob=dropout_prob, + ) self.assertRaises(TypeError, test_input_type) def test_pre_hidden_type(): - error_pre_hidden = fluid.data(name='error_pre_hidden', - shape=[None, hidden_size], - dtype='int32') - LSTM(input, error_pre_hidden, pre_cell, \ - seq_len, hidden_size, num_layers, \ - dropout_prob=dropout_prob) + error_pre_hidden = fluid.data( + name='error_pre_hidden', + shape=[None, hidden_size], + dtype='int32', + ) + LSTM( + input, + error_pre_hidden, + pre_cell, + seq_len, + hidden_size, + num_layers, + dropout_prob=dropout_prob, + ) self.assertRaises(TypeError, test_pre_hidden_type) def test_pre_cell_type(): - error_pre_cell = fluid.data(name='error_pre_cell', - shape=[None, hidden_size], - dtype='int32') - LSTM(input, pre_hidden, error_pre_cell, \ - seq_len, hidden_size, num_layers, \ - dropout_prob=dropout_prob) + error_pre_cell = fluid.data( + name='error_pre_cell', + shape=[None, hidden_size], + dtype='int32', + ) + LSTM( + input, + pre_hidden, + error_pre_cell, + seq_len, + hidden_size, + num_layers, + dropout_prob=dropout_prob, + ) self.assertRaises(TypeError, test_pre_cell_type) class TestLstmOp(OpTest): - def set_is_test(self): self.is_test = False @@ -247,11 +293,21 @@ def setUp(self): else: b = np.random.normal(size=(1, 4 * self.D)).astype('float64') - w_b = b[:, 0:4 * self.D] - w_c = b[:, 4 * self.D:] if self.use_peepholes else None - h, c = lstm(x, self.lod, h0, c0, w, w_b, w_c, self.is_reverse, - ACTIVATION[self.act_gate], ACTIVATION[self.act_cell], - ACTIVATION[self.act_cand]) + w_b = b[:, 0 : 4 * self.D] + w_c = b[:, 4 * self.D :] if self.use_peepholes else None + h, c = lstm( + x, + self.lod, + h0, + c0, + w, + w_b, + w_c, + self.is_reverse, + ACTIVATION[self.act_gate], + ACTIVATION[self.act_cell], + ACTIVATION[self.act_cand], + ) self.inputs = {'Input': (x, self.lod), 'Weight': w} @@ -271,7 +327,7 @@ def setUp(self): 'gate_activation': self.act_gate, 'cell_activation': self.act_cell, 'candidate_activation': self.act_cand, - 'is_test': self.is_test + 'is_test': self.is_test, } def test_check_output(self): @@ -281,33 +337,33 @@ def test_check_grad(self): # TODO(qingqing) remove folowing lines after the check_grad is refined. N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'Weight', 'Bias'], ['Hidden'], - max_relative_error=5e-4, - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'Weight', 'Bias'], + ['Hidden'], + max_relative_error=5e-4, + check_dygraph=False, + ) class TestLstmOpCase1(TestLstmOp): - def set_lod(self): self.lod = [[0, 3, 2]] class TestLstmOpCase2(TestLstmOp): - def set_lod(self): self.lod = [[0, 3, 0]] class TestLstmOpCase3(TestLstmOp): - def set_lod(self): self.lod = [[2, 0, 4]] class TestLstmOpInference(TestLstmOp): - def set_is_test(self): self.is_test = True @@ -317,43 +373,42 @@ def test_check_grad(self): class TestLstmOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): input_data = np.random.random((1, 2048)).astype("float32") - fluid.layers.dynamic_lstm(input=input_data, - size=2048, - use_peepholes=False) + fluid.layers.dynamic_lstm( + input=input_data, size=2048, use_peepholes=False + ) self.assertRaises(TypeError, test_Variable) def test_h_0(): - in_data = fluid.data(name="input", - shape=[None, 2048], - dtype="float32") + in_data = fluid.data( + name="input", shape=[None, 2048], dtype="float32" + ) h = fluid.data(name="h", shape=[None, 512], dtype="int32") c = fluid.data(name="c", shape=[None, 512], dtype="float32") - fluid.layers.dynamic_lstm(input=in_data, - size=2048, - use_peepholes=False, - h_0=h, - c_0=c) + fluid.layers.dynamic_lstm( + input=in_data, size=2048, use_peepholes=False, h_0=h, c_0=c + ) self.assertRaises(TypeError, test_h_0) def test_c_0(): - in_data_ = fluid.data(name="input_", - shape=[None, 2048], - dtype="float32") + in_data_ = fluid.data( + name="input_", shape=[None, 2048], dtype="float32" + ) h_ = fluid.data(name="h_", shape=[None, 512], dtype="float32") c_ = fluid.data(name="c_", shape=[None, 512], dtype="int32") - fluid.layers.dynamic_lstm(input=in_data_, - size=2048, - use_peepholes=False, - h_0=h_, - c_0=c_) + fluid.layers.dynamic_lstm( + input=in_data_, + size=2048, + use_peepholes=False, + h_0=h_, + c_0=c_, + ) self.assertRaises(TypeError, test_c_0) diff --git a/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py b/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py index b4e6690117d20..d46e8e4719e81 100644 --- a/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py @@ -21,35 +21,39 @@ def sigmoid_np(x): - return 1. / (1. + np.exp(-x)) + return 1.0 / (1.0 + np.exp(-x)) def tanh_np(x): - return 2 * sigmoid_np(2. * x) - 1. + return 2 * sigmoid_np(2.0 * x) - 1.0 class LstmUnitTestError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_size, dict_dim, emb_dim, hidden_dim = 32, 128, 64, 512 - data = fluid.data(name='step_data', - shape=[batch_size], - dtype='int64') + data = fluid.data( + name='step_data', shape=[batch_size], dtype='int64' + ) inputs = fluid.embedding(input=data, size=[dict_dim, emb_dim]) - pre_hidden = fluid.data(name='pre_hidden', - shape=[batch_size, hidden_dim], - dtype='float32') - pre_cell = fluid.data(name='pre_cell', - shape=[batch_size, hidden_dim], - dtype='float32') + pre_hidden = fluid.data( + name='pre_hidden', + shape=[batch_size, hidden_dim], + dtype='float32', + ) + pre_cell = fluid.data( + name='pre_cell', shape=[batch_size, hidden_dim], dtype='float32' + ) np_input = np.random.uniform( - -0.1, 0.1, (batch_size, emb_dim)).astype('float64') + -0.1, 0.1, (batch_size, emb_dim) + ).astype('float64') np_pre_hidden = np.random.uniform( - -0.1, 0.1, (batch_size, hidden_dim)).astype('float64') + -0.1, 0.1, (batch_size, hidden_dim) + ).astype('float64') np_pre_cell = np.random.uniform( - -0.1, 0.1, (batch_size, hidden_dim)).astype('float64') + -0.1, 0.1, (batch_size, hidden_dim) + ).astype('float64') def test_input_Variable(): lstm_unit(np_input, pre_hidden, pre_cell) @@ -67,42 +71,48 @@ def test_pre_cell_Variable(): self.assertRaises(TypeError, test_pre_cell_Variable) def test_input_type(): - error_input = fluid.data(name='error_input', - shape=[batch_size, emb_dim], - dtype='int32') + error_input = fluid.data( + name='error_input', + shape=[batch_size, emb_dim], + dtype='int32', + ) lstm_unit(error_input, pre_hidden, pre_cell) self.assertRaises(TypeError, test_input_type) def test_pre_hidden_type(): - error_pre_hidden = fluid.data(name='error_pre_hidden', - shape=[batch_size, hidden_dim], - dtype='int32') + error_pre_hidden = fluid.data( + name='error_pre_hidden', + shape=[batch_size, hidden_dim], + dtype='int32', + ) lstm_unit(inputs, error_pre_hidden, pre_cell) self.assertRaises(TypeError, test_pre_hidden_type) def test_pre_cell_type(): - error_pre_cell = fluid.data(name='error_pre_cell', - shape=[batch_size, hidden_dim], - dtype='int32') + error_pre_cell = fluid.data( + name='error_pre_cell', + shape=[batch_size, hidden_dim], + dtype='int32', + ) lstm_unit(inputs, pre_hidden, error_pre_cell) self.assertRaises(TypeError, test_pre_cell_type) class LstmUnitTest(OpTest): - def setUp(self): self.op_type = "lstm_unit" x_np = np.random.normal(size=(15, 160)).astype("float64") c_np = np.random.normal(size=(15, 40)).astype("float64") i_np, f_np, o_np, j_np = np.split(x_np, 4, axis=1) - forget_bias_np = 0. - self.attrs = {'forget_bias': 0.} + forget_bias_np = 0.0 + self.attrs = {'forget_bias': 0.0} new_c = c_np * sigmoid_np(f_np + forget_bias_np) + sigmoid_np( - i_np) * tanh_np(j_np) + i_np + ) * tanh_np(j_np) new_h = tanh_np(new_c) * sigmoid_np(o_np) self.inputs = {'X': x_np, 'C_prev': c_np} diff --git a/python/paddle/fluid/tests/unittests/test_lstmp_op.py b/python/paddle/fluid/tests/unittests/test_lstmp_op.py index 721a61a7e7d55..730af353dce6b 100644 --- a/python/paddle/fluid/tests/unittests/test_lstmp_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstmp_op.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np @@ -22,30 +22,42 @@ 'identity': LstmTest.identity, 'sigmoid': LstmTest.sigmoid, 'tanh': LstmTest.tanh, - 'relu': LstmTest.relu + 'relu': LstmTest.relu, } # LSTM with recurrent projection Layer def lstmp( - input, # T x 4D - lod, # 1 x N - h0=None, # N x D - c0=None, # N x D - w_r=None, # P x 4D - w_rh=None, # D x P - w_b=None, # 1 x 4D - w_c=None, # 1 x 3D - is_reverse=False, - proj_clip=0.0, - cell_clip=0.0, - act_gate=None, - act_cell=None, - act_cand=None, - act_proj=None): - - def _step(x, w_r, w_rh, w_c, r_pre, c_pre, proj_clip, cell_clip, act_gate, - act_cell, act_cand, act_proj): + input, # T x 4D + lod, # 1 x N + h0=None, # N x D + c0=None, # N x D + w_r=None, # P x 4D + w_rh=None, # D x P + w_b=None, # 1 x 4D + w_c=None, # 1 x 3D + is_reverse=False, + proj_clip=0.0, + cell_clip=0.0, + act_gate=None, + act_cell=None, + act_cand=None, + act_proj=None, +): + def _step( + x, + w_r, + w_rh, + w_c, + r_pre, + c_pre, + proj_clip, + cell_clip, + act_gate, + act_cell, + act_cand, + act_proj, + ): g = np.dot(r_pre, w_r) # 1 x 4D g = g + x g = np.reshape(g, (1, g.size)) @@ -103,14 +115,25 @@ def _reverse(x, offset): for i in range(batch_size): # compute one sequence seq_len = lod[0][i] - x = input[offset[i]:offset[i + 1], :] + x = input[offset[i] : offset[i + 1], :] r_pre = h0[i] c_pre = c0[i] # 1 x D for j in range(seq_len): # compute one step - r_pre, c_pre = _step(x[j], w_r, w_rh, w_c, r_pre, c_pre, proj_clip, - cell_clip, act_gate, act_cell, act_cand, - act_proj) + r_pre, c_pre = _step( + x[j], + w_r, + w_rh, + w_c, + r_pre, + c_pre, + proj_clip, + cell_clip, + act_gate, + act_cell, + act_cand, + act_proj, + ) projection.append(r_pre.flatten()) cell.append(c_pre.flatten()) @@ -126,7 +149,6 @@ def _reverse(x, offset): class TestLstmpOp(LstmTest.TestLstmOp): - def reset_argument(self): pass @@ -154,15 +176,28 @@ def setUp(self): else: b = np.random.normal(size=(1, 4 * self.D)).astype('float64') - w_b = b[:, 0:4 * self.D] - w_c = b[:, 4 * self.D:] if self.use_peepholes else None + w_b = b[:, 0 : 4 * self.D] + w_c = b[:, 4 * self.D :] if self.use_peepholes else None w_rh = np.random.normal(size=(self.D, self.P)).astype('float64') proj_clip = 0.1 cell_clip = 0.1 - r, c = lstmp(x, self.lod, h0, c0, w, w_rh, w_b, w_c, self.is_reverse, - proj_clip, cell_clip, ACTIVATION[self.act_gate], - ACTIVATION[self.act_cell], ACTIVATION[self.act_cand], - ACTIVATION[self.act_proj]) + r, c = lstmp( + x, + self.lod, + h0, + c0, + w, + w_rh, + w_b, + w_c, + self.is_reverse, + proj_clip, + cell_clip, + ACTIVATION[self.act_gate], + ACTIVATION[self.act_cell], + ACTIVATION[self.act_cand], + ACTIVATION[self.act_proj], + ) self.inputs = {'Input': (x, self.lod), 'Weight': w, 'ProjWeight': w_rh} @@ -184,7 +219,7 @@ def setUp(self): 'gate_activation': self.act_gate, 'cell_activation': self.act_cell, 'candidate_activation': self.act_cand, - 'proj_activation': self.act_proj + 'proj_activation': self.act_proj, } def test_check_output(self): @@ -195,16 +230,18 @@ def test_check_grad(self): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'Weight', 'ProjWeight', 'Bias'], - ['Projection'], - numeric_grad_delta=0.0000005, - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'Weight', 'ProjWeight', 'Bias'], + ['Projection'], + numeric_grad_delta=0.0000005, + check_dygraph=False, + ) class TestLstmpOpHasInitial(TestLstmpOp): - def reset_argument(self): self.has_initial_state = True @@ -213,162 +250,187 @@ def test_check_grad(self): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'Weight', 'ProjWeight', 'Bias', 'H0', 'C0'], - ['Projection'], - numeric_grad_delta=0.0000005, - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'Weight', 'ProjWeight', 'Bias', 'H0', 'C0'], + ['Projection'], + numeric_grad_delta=0.0000005, + check_dygraph=False, + ) def test_check_grad_ingore_bias(self): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'ProjWeight', 'Weight'], ['Projection'], - numeric_grad_delta=0.0000005, - no_grad_set=set('Bias'), - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'ProjWeight', 'Weight'], + ['Projection'], + numeric_grad_delta=0.0000005, + no_grad_set=set('Bias'), + check_dygraph=False, + ) def test_check_grad_ingore_weight(self): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'ProjWeight', 'Bias'], ['Projection'], - numeric_grad_delta=0.0000005, - no_grad_set=set('Weight'), - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'ProjWeight', 'Bias'], + ['Projection'], + numeric_grad_delta=0.0000005, + no_grad_set=set('Weight'), + check_dygraph=False, + ) def test_check_grad_ingore_proj_weight(self): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'Weight', 'Bias'], ['Projection'], - numeric_grad_delta=0.0000005, - no_grad_set=set('ProjWeight'), - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'Weight', 'Bias'], + ['Projection'], + numeric_grad_delta=0.0000005, + no_grad_set=set('ProjWeight'), + check_dygraph=False, + ) def test_check_grad_ingore_input(self): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Weight', 'ProjWeight', 'Bias'], ['Projection'], - numeric_grad_delta=0.0000005, - no_grad_set=set('Input'), - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Weight', 'ProjWeight', 'Bias'], + ['Projection'], + numeric_grad_delta=0.0000005, + no_grad_set=set('Input'), + check_dygraph=False, + ) def test_check_grad_ingore_h0(self): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'Weight', 'ProjWeight', 'Bias', 'C0'], - ['Projection'], - numeric_grad_delta=0.0000005, - no_grad_set=set('H0'), - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'Weight', 'ProjWeight', 'Bias', 'C0'], + ['Projection'], + numeric_grad_delta=0.0000005, + no_grad_set=set('H0'), + check_dygraph=False, + ) def test_check_grad_ingore_c0(self): N = len(self.lod[0]) self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') self.outputs['BatchHidden'] = np.zeros((N, self.D)).astype('float64') - self.outputs['BatchCellPreAct'] = np.zeros( - (N, self.D)).astype('float64') - self.check_grad(['Input', 'Weight', 'ProjWeight', 'Bias', 'H0'], - ['Projection'], - numeric_grad_delta=0.0000005, - no_grad_set=set('C0'), - check_dygraph=False) + self.outputs['BatchCellPreAct'] = np.zeros((N, self.D)).astype( + 'float64' + ) + self.check_grad( + ['Input', 'Weight', 'ProjWeight', 'Bias', 'H0'], + ['Projection'], + numeric_grad_delta=0.0000005, + no_grad_set=set('C0'), + check_dygraph=False, + ) class TestLstmpOpRerverse(TestLstmpOp): - def reset_argument(self): self.is_reverse = True class TestLstmpOpNotUsePeepholes(TestLstmpOp): - def reset_argument(self): self.use_peepholes = False class TestLstmpOpLinearProjection(TestLstmpOp): - def reset_argument(self): self.act_proj = 'identity' class TestLstmpOpLen0Case1(TestLstmpOp): - def reset_argument(self): self.lod = [[0, 4, 0]] class TestLstmpOpLen0Case2(TestLstmpOp): - def reset_argument(self): self.lod = [[2, 0, 3]] class TestLstmpOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): input_data = np.random.random((1, 2048)).astype("float32") - fluid.layers.dynamic_lstmp(input=input_data, - size=2048, - proj_size=256, - use_peepholes=False, - is_reverse=True, - cell_activation="tanh", - proj_activation="tanh") + fluid.layers.dynamic_lstmp( + input=input_data, + size=2048, + proj_size=256, + use_peepholes=False, + is_reverse=True, + cell_activation="tanh", + proj_activation="tanh", + ) self.assertRaises(TypeError, test_Variable) def test_h_0(): - in_data = fluid.data(name="input", - shape=[None, 2048], - dtype="float32") + in_data = fluid.data( + name="input", shape=[None, 2048], dtype="float32" + ) h = fluid.data(name="h", shape=[None, 512], dtype="int32") c = fluid.data(name="c", shape=[None, 512], dtype="float32") - fluid.layers.dynamic_lstmp(input=in_data, - size=2048, - proj_size=256, - use_peepholes=False, - is_reverse=True, - cell_activation="tanh", - proj_activation="tanh", - h_0=h, - c_0=c) + fluid.layers.dynamic_lstmp( + input=in_data, + size=2048, + proj_size=256, + use_peepholes=False, + is_reverse=True, + cell_activation="tanh", + proj_activation="tanh", + h_0=h, + c_0=c, + ) self.assertRaises(TypeError, test_h_0) def test_c_0(): - in_data_ = fluid.data(name="input_", - shape=[None, 2048], - dtype="float32") + in_data_ = fluid.data( + name="input_", shape=[None, 2048], dtype="float32" + ) h_ = fluid.data(name="h_", shape=[None, 512], dtype="float32") c_ = fluid.data(name="c_", shape=[None, 512], dtype="int32") - fluid.layers.dynamic_lstmp(input=in_data_, - size=2048, - proj_size=256, - use_peepholes=False, - is_reverse=True, - cell_activation="tanh", - proj_activation="tanh", - h_0=h_, - c_0=c_) + fluid.layers.dynamic_lstmp( + input=in_data_, + size=2048, + proj_size=256, + use_peepholes=False, + is_reverse=True, + cell_activation="tanh", + proj_activation="tanh", + h_0=h_, + c_0=c_, + ) self.assertRaises(TypeError, test_c_0) diff --git a/python/paddle/fluid/tests/unittests/test_lu_op.py b/python/paddle/fluid/tests/unittests/test_lu_op.py index 0c24bcd68650f..d76fdf16ad933 100644 --- a/python/paddle/fluid/tests/unittests/test_lu_op.py +++ b/python/paddle/fluid/tests/unittests/test_lu_op.py @@ -43,8 +43,11 @@ def scipy_lu(A, pivot): PP.append(P) PL.append(L) PU.append(U) - return np.array(PP).reshape(preshape + pshape), np.array(PL).reshape( - preshape + lshape), np.array(PU).reshape(preshape + ushape) + return ( + np.array(PP).reshape(preshape + pshape), + np.array(PL).reshape(preshape + lshape), + np.array(PU).reshape(preshape + ushape), + ) def Pmat_to_perm(Pmat_org, cut): @@ -66,9 +69,15 @@ def Pmat_to_perm(Pmat_org, cut): sP[idx, :] = tmp permmat.append(permlst) - Pivot = np.array(permmat).reshape(list(shape[:-2]) + [ - rows, - ]) + 1 + Pivot = ( + np.array(permmat).reshape( + list(shape[:-2]) + + [ + rows, + ] + ) + + 1 + ) return Pivot[..., :cut] @@ -112,17 +121,20 @@ def set_output(self): ushape = np.array(sU.shape) lpad = (len(sL.shape) - 2) * [(0, 0)] + list( - ((0, (ashape - lshape)[-2]), (0, (ashape - lshape)[-1]))) + ((0, (ashape - lshape)[-2]), (0, (ashape - lshape)[-1])) + ) upad = (len(sU.shape) - 2) * [(0, 0)] + list( - ((0, (ashape - ushape)[-2]), (0, (ashape - ushape)[-1]))) + ((0, (ashape - ushape)[-2]), (0, (ashape - ushape)[-1])) + ) NsL = np.pad(sL, lpad) NsU = np.pad(sU, upad) NLU = NsL + NsU self.output = NLU self.Pivots = Pmat_to_perm(sP, min(ashape[-2], ashape[-1])) - self.Infos = np.zeros( - self.x_shape[:-2]) if len(X.shape) > 2 else np.array([0]) + self.Infos = ( + np.zeros(self.x_shape[:-2]) if len(X.shape) > 2 else np.array([0]) + ) def setUp(self): self.op_type = "lu" @@ -136,7 +148,7 @@ def setUp(self): self.outputs = { 'Out': self.output, 'Pivots': self.Pivots, - 'Infos': self.Infos + 'Infos': self.Infos, } def test_check_output(self): @@ -173,9 +185,7 @@ def config(self): class TestLUAPI(unittest.TestCase): - def test_dygraph(self): - def run_lu_dygraph(shape, dtype): if dtype == "float32": np_dtype = np.float32 @@ -217,7 +227,7 @@ def run_lu_dygraph(shape, dtype): (4, 5, 3), # 3-dim Tensors (2, 5, 3, 5), (3, 5, 5, 5), - (4, 5, 5, 3) # 4-dim Tensors + (4, 5, 5, 3), # 4-dim Tensors ] dtypes = ["float32", "float64"] for tensor_shape, dtype in itertools.product(tensor_shapes, dtypes): @@ -251,28 +261,29 @@ def run_lu_static(shape, dtype): ushape = np.array(sU.shape) lpad = (len(sL.shape) - 2) * [(0, 0)] + list( - ((0, (ashape - lshape)[-2]), (0, - (ashape - lshape)[-1]))) + ((0, (ashape - lshape)[-2]), (0, (ashape - lshape)[-1])) + ) upad = (len(sU.shape) - 2) * [(0, 0)] + list( - ((0, (ashape - ushape)[-2]), (0, - (ashape - ushape)[-1]))) + ((0, (ashape - ushape)[-2]), (0, (ashape - ushape)[-1])) + ) NsL = np.pad(sL, lpad) NsU = np.pad(sU, upad) NLU = NsL + NsU - x = paddle.fluid.data(name="input", - shape=shape, - dtype=dtype) + x = paddle.fluid.data( + name="input", shape=shape, dtype=dtype + ) lu, p = paddle.linalg.lu(x, pivot=pivot) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": a}, - fetch_list=[lu, p]) - np.testing.assert_allclose(fetches[0], - NLU, - rtol=1e-05, - atol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": a}, + fetch_list=[lu, p], + ) + np.testing.assert_allclose( + fetches[0], NLU, rtol=1e-05, atol=1e-05 + ) tensor_shapes = [ (3, 5), @@ -283,7 +294,7 @@ def run_lu_static(shape, dtype): (4, 5, 3), # 3-dim Tensors (2, 5, 3, 5), (3, 5, 5, 5), - (4, 5, 5, 3) # 4-dim Tensors + (4, 5, 5, 3), # 4-dim Tensors ] dtypes = ["float32", "float64"] for tensor_shape, dtype in itertools.product(tensor_shapes, dtypes): diff --git a/python/paddle/fluid/tests/unittests/test_lu_unpack_op.py b/python/paddle/fluid/tests/unittests/test_lu_unpack_op.py index 1ec36680fa4c2..0c00fbea82088 100644 --- a/python/paddle/fluid/tests/unittests/test_lu_unpack_op.py +++ b/python/paddle/fluid/tests/unittests/test_lu_unpack_op.py @@ -48,9 +48,11 @@ def scipy_lu_unpack(A): Llst.append(L) Ulst.append(U) - return np.array(Plst).reshape(preshape + pshape), np.array( - Llst).reshape(preshape + lshape), np.array(Ulst).reshape(preshape + - ushape) + return ( + np.array(Plst).reshape(preshape + pshape), + np.array(Llst).reshape(preshape + lshape), + np.array(Ulst).reshape(preshape + ushape), + ) def Pmat_to_perm(Pmat_org, cut): @@ -72,9 +74,15 @@ def Pmat_to_perm(Pmat_org, cut): sP[idx, :] = tmp permmat.append(permlst) - Pivot = np.array(permmat).reshape(list(shape[:-2]) + [ - rows, - ]) + 1 + Pivot = ( + np.array(permmat).reshape( + list(shape[:-2]) + + [ + rows, + ] + ) + + 1 + ) return Pivot[..., :cut] @@ -132,21 +140,23 @@ def setUp(self): place = fluid.CPUPlace() if core.is_compiled_with_cuda(): place = fluid.CUDAPlace(0) - xv = paddle.fluid.data(name="input", - shape=self.x_shape, - dtype=self.dtype) + xv = paddle.fluid.data( + name="input", shape=self.x_shape, dtype=self.dtype + ) lu, p = paddle.linalg.lu(xv) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": x}, - fetch_list=[lu, p]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": x}, + fetch_list=[lu, p], + ) lu, pivots = fetches[0], fetches[1] self.inputs = {'X': lu, 'Pivots': pivots} self.attrs = { 'unpack_ludata': self.unpack_ludata, - 'unpack_pivots': self.unpack_pivots + 'unpack_pivots': self.unpack_pivots, } self.set_output(x) self.outputs = { @@ -189,12 +199,10 @@ def config(self): class TestLU_UnpackAPI(unittest.TestCase): - def setUp(self): np.random.seed(2022) def test_dygraph(self): - def run_lu_unpack_dygraph(shape, dtype): if dtype == "float32": np_dtype = np.float32 @@ -229,7 +237,7 @@ def run_lu_unpack_dygraph(shape, dtype): (4, 5, 3), # 3-dim Tensors (2, 5, 3, 5), (3, 5, 5, 5), - (4, 5, 5, 3) # 4-dim Tensors + (4, 5, 5, 3), # 4-dim Tensors ] dtypes = ["float32", "float64"] for tensor_shape, dtype in itertools.product(tensor_shapes, dtypes): @@ -255,27 +263,26 @@ def run_lu_static(shape, dtype): with fluid.program_guard(fluid.Program(), fluid.Program()): sP, sL, sU = scipy_lu_unpack(a) - x = paddle.fluid.data(name="input", - shape=shape, - dtype=dtype) + x = paddle.fluid.data( + name="input", shape=shape, dtype=dtype + ) lu, p = paddle.linalg.lu(x) pP, pL, pU = paddle.linalg.lu_unpack(lu, p) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": a}, - fetch_list=[pP, pL, pU]) - np.testing.assert_allclose(fetches[0], - sP, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(fetches[1], - sL, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(fetches[2], - sU, - rtol=1e-05, - atol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": a}, + fetch_list=[pP, pL, pU], + ) + np.testing.assert_allclose( + fetches[0], sP, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + fetches[1], sL, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + fetches[2], sU, rtol=1e-05, atol=1e-05 + ) tensor_shapes = [ (3, 5), @@ -286,7 +293,7 @@ def run_lu_static(shape, dtype): (4, 5, 3), # 3-dim Tensors (2, 5, 3, 5), (3, 5, 5, 5), - (4, 5, 5, 3) # 4-dim Tensors + (4, 5, 5, 3), # 4-dim Tensors ] dtypes = ["float32", "float64"] for tensor_shape, dtype in itertools.product(tensor_shapes, dtypes): diff --git a/python/paddle/fluid/tests/unittests/test_manual_seed.py b/python/paddle/fluid/tests/unittests/test_manual_seed.py index 7d9762c788abb..ac697a49dbd66 100644 --- a/python/paddle/fluid/tests/unittests/test_manual_seed.py +++ b/python/paddle/fluid/tests/unittests/test_manual_seed.py @@ -20,7 +20,6 @@ class TestManualSeed(unittest.TestCase): - def test_seed(self): fluid.enable_dygraph() diff --git a/python/paddle/fluid/tests/unittests/test_margin_cross_entropy_op.py b/python/paddle/fluid/tests/unittests/test_margin_cross_entropy_op.py index 6473ab7568f19..d8a5f653a9ae6 100644 --- a/python/paddle/fluid/tests/unittests/test_margin_cross_entropy_op.py +++ b/python/paddle/fluid/tests/unittests/test_margin_cross_entropy_op.py @@ -21,20 +21,15 @@ def stable_softmax_comm(x): - shiftx = (x - np.max(x)) + shiftx = x - np.max(x) deno = np.log(np.sum(np.exp(shiftx))) comm = shiftx - deno return comm -def margin_cross_entropy(logits, - label, - axis, - margin1, - margin2, - margin3, - scale, - reduction=None): +def margin_cross_entropy( + logits, label, axis, margin1, margin2, margin3, scale, reduction=None +): one_hot_label = np.zeros_like(logits, dtype=logits.dtype) for i, lb in enumerate(label): one_hot_label[i, lb] = 1.0 @@ -61,16 +56,18 @@ def margin_cross_entropy(logits, return loss, softmax -def python_api(logits, - label, - return_softmax=False, - ring_id=0, - rank=0, - nrank=0, - margin1=1.0, - margin2=0.5, - margin3=0.0, - scale=64.0): +def python_api( + logits, + label, + return_softmax=False, + ring_id=0, + rank=0, + nrank=0, + margin1=1.0, + margin2=0.5, + margin3=0.0, + scale=64.0, +): return paddle.nn.functional.margin_cross_entropy( logits, label, @@ -80,13 +77,14 @@ def python_api(logits, margin3=margin3, scale=scale, group=None, - reduction=None) + reduction=None, + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOp(OpTest): - def initParams(self): self.python_api = python_api self.op_type = "margin_cross_entropy" @@ -111,26 +109,35 @@ def setUp(self): self.init_dtype() datas = np.random.uniform( - -0.99, 0.99, [self.batch_dim, self.feat_dim]).astype(self.dtype) + -0.99, 0.99, [self.batch_dim, self.feat_dim] + ).astype(self.dtype) datas = datas / np.sqrt(np.sum(np.square(datas), axis=1, keepdims=True)) weights = np.random.uniform( - -0.99, 0.99, [self.feat_dim, self.num_class]).astype(self.dtype) + -0.99, 0.99, [self.feat_dim, self.num_class] + ).astype(self.dtype) weights = weights / np.sqrt( - np.sum(np.square(weights), axis=0, keepdims=True)) + np.sum(np.square(weights), axis=0, keepdims=True) + ) logits = np.matmul(datas, weights) - labels = np.random.randint(0, - self.num_class, (self.batch_dim, ), - dtype="int64") + labels = np.random.randint( + 0, self.num_class, (self.batch_dim,), dtype="int64" + ) - loss, softmax = margin_cross_entropy(logits, labels, self.axis, - self.margin1, self.margin2, - self.margin3, self.scale) + loss, softmax = margin_cross_entropy( + logits, + labels, + self.axis, + self.margin1, + self.margin2, + self.margin3, + self.scale, + ) self.inputs = {"Logits": logits, "Label": labels} self.outputs = { "Softmax": softmax.astype(self.dtype), - "Loss": loss.astype(self.dtype) + "Loss": loss.astype(self.dtype), } self.attrs = { 'margin1': self.margin1, @@ -140,55 +147,61 @@ def setUp(self): } def test_check_output(self): - self.check_output_with_place(core.CUDAPlace(0), - atol=1e-5, - check_eager=True) + self.check_output_with_place( + core.CUDAPlace(0), atol=1e-5, check_eager=True + ) def test_check_grad(self): - self.check_grad_with_place(core.CUDAPlace(0), ["Logits"], - "Loss", - check_eager=True) + self.check_grad_with_place( + core.CUDAPlace(0), ["Logits"], "Loss", check_eager=True + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpFP32(TestMarginCrossEntropyOp): - def init_dtype(self): self.dtype = np.float32 def test_check_grad(self): - self.check_grad_with_place(core.CUDAPlace(0), ["Logits"], - "Loss", - numeric_grad_delta=5e-2, - max_relative_error=5e-2, - check_eager=True) - - -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + self.check_grad_with_place( + core.CUDAPlace(0), + ["Logits"], + "Loss", + numeric_grad_delta=5e-2, + max_relative_error=5e-2, + check_eager=True, + ) + + +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpFP16(TestMarginCrossEntropyOp): - def init_dtype(self): self.dtype = np.float16 def test_check_output(self): - self.check_output_with_place(core.CUDAPlace(0), - atol=5e-2, - check_eager=True) + self.check_output_with_place( + core.CUDAPlace(0), atol=5e-2, check_eager=True + ) def test_check_grad(self): - self.check_grad_with_place(core.CUDAPlace(0), ["Logits"], - "Loss", - numeric_grad_delta=6e-1, - max_relative_error=6e-1, - check_eager=True) - - -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + self.check_grad_with_place( + core.CUDAPlace(0), + ["Logits"], + "Loss", + numeric_grad_delta=6e-1, + max_relative_error=6e-1, + check_eager=True, + ) + + +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpCosFace(TestMarginCrossEntropyOp): - def init_loss_params(self): self.margin1 = 1.0 self.margin2 = 0.0 @@ -196,10 +209,10 @@ def init_loss_params(self): self.scale = 2.0 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpSphereFace(TestMarginCrossEntropyOp): - def init_loss_params(self): self.margin1 = 1.35 self.margin2 = 0.0 @@ -208,28 +221,27 @@ def init_loss_params(self): class TestMarginCrossEntropyOpCPU(TestMarginCrossEntropyOp): - def test_check_output(self): try: - self.check_output_with_place(core.CPUPlace(), - atol=1e-5, - check_eager=True) + self.check_output_with_place( + core.CPUPlace(), atol=1e-5, check_eager=True + ) except RuntimeError: pass def test_check_grad(self): try: - self.check_grad_with_place(core.CPUPlace(), ["Logits"], - "Loss", - check_eager=True) + self.check_grad_with_place( + core.CPUPlace(), ["Logits"], "Loss", check_eager=True + ) except RuntimeError: pass -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpV2(unittest.TestCase): - def setUp(self): self.initParams() np.random.seed(self.seed) @@ -268,31 +280,42 @@ def test_static(self): def check_static_result(self, place): with program_guard(Program(), Program()): datas = np.random.uniform( - -0.99, 0.99, [self.batch_dim, self.feat_dim]).astype(self.dtype) + -0.99, 0.99, [self.batch_dim, self.feat_dim] + ).astype(self.dtype) datas = datas / np.sqrt( - np.sum(np.square(datas), axis=1, keepdims=True)) + np.sum(np.square(datas), axis=1, keepdims=True) + ) weights = np.random.uniform( - -0.99, 0.99, [self.feat_dim, self.num_class]).astype(self.dtype) + -0.99, 0.99, [self.feat_dim, self.num_class] + ).astype(self.dtype) weights = weights / np.sqrt( - np.sum(np.square(weights), axis=0, keepdims=True)) + np.sum(np.square(weights), axis=0, keepdims=True) + ) logits_np = np.matmul(datas, weights) - labels_np = np.random.randint(0, - self.num_class, (self.batch_dim, ), - dtype="int64") - - loss_np, softmax_np = margin_cross_entropy(logits_np, labels_np, - self.axis, self.margin1, - self.margin2, - self.margin3, self.scale, - self.reduction) - - logits = paddle.static.data(name='logits', - shape=[self.batch_dim, self.num_class], - dtype=self.dtype) - label = paddle.static.data(name='label', - shape=[self.batch_dim], - dtype="int64") + labels_np = np.random.randint( + 0, self.num_class, (self.batch_dim,), dtype="int64" + ) + + loss_np, softmax_np = margin_cross_entropy( + logits_np, + labels_np, + self.axis, + self.margin1, + self.margin2, + self.margin3, + self.scale, + self.reduction, + ) + + logits = paddle.static.data( + name='logits', + shape=[self.batch_dim, self.num_class], + dtype=self.dtype, + ) + label = paddle.static.data( + name='label', shape=[self.batch_dim], dtype="int64" + ) loss, softmax = paddle.nn.functional.margin_cross_entropy( logits, label, @@ -301,16 +324,15 @@ def check_static_result(self, place): margin3=self.margin3, scale=self.scale, return_softmax=True, - reduction=self.reduction) + reduction=self.reduction, + ) exe = paddle.fluid.Executor(place) - [loss_res, - softmax_res] = exe.run(paddle.fluid.default_main_program(), - feed={ - 'logits': logits_np, - 'label': labels_np - }, - fetch_list=[loss, softmax]) + [loss_res, softmax_res] = exe.run( + paddle.fluid.default_main_program(), + feed={'logits': logits_np, 'label': labels_np}, + fetch_list=[loss, softmax], + ) np.testing.assert_allclose(loss_res, loss_np) np.testing.assert_allclose(softmax_res, softmax_np) @@ -321,24 +343,33 @@ def test_dynamic(self): def check_dynamic_result(self, place): with paddle.fluid.dygraph.guard(place): datas = np.random.uniform( - -0.99, 0.99, [self.batch_dim, self.feat_dim]).astype(self.dtype) + -0.99, 0.99, [self.batch_dim, self.feat_dim] + ).astype(self.dtype) datas = datas / np.sqrt( - np.sum(np.square(datas), axis=1, keepdims=True)) + np.sum(np.square(datas), axis=1, keepdims=True) + ) weights = np.random.uniform( - -0.99, 0.99, [self.feat_dim, self.num_class]).astype(self.dtype) + -0.99, 0.99, [self.feat_dim, self.num_class] + ).astype(self.dtype) weights = weights / np.sqrt( - np.sum(np.square(weights), axis=0, keepdims=True)) + np.sum(np.square(weights), axis=0, keepdims=True) + ) logits_np = np.matmul(datas, weights) - labels_np = np.random.randint(0, - self.num_class, (self.batch_dim, ), - dtype="int64") - - loss_np, softmax_np = margin_cross_entropy(logits_np, labels_np, - self.axis, self.margin1, - self.margin2, - self.margin3, self.scale, - self.reduction) + labels_np = np.random.randint( + 0, self.num_class, (self.batch_dim,), dtype="int64" + ) + + loss_np, softmax_np = margin_cross_entropy( + logits_np, + labels_np, + self.axis, + self.margin1, + self.margin2, + self.margin3, + self.scale, + self.reduction, + ) logits = paddle.to_tensor(logits_np, dtype=self.dtype) labels = paddle.to_tensor(labels_np, dtype="int64") @@ -351,7 +382,8 @@ def check_dynamic_result(self, place): margin3=self.margin3, scale=self.scale, return_softmax=True, - reduction=self.reduction) + reduction=self.reduction, + ) loss_res = loss.numpy() softmax_res = softmax.numpy() @@ -359,26 +391,26 @@ def check_dynamic_result(self, place): np.testing.assert_allclose(softmax_res, softmax_np) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpV3(TestMarginCrossEntropyOpV2): - def init_reduction(self): self.reduction = 'mean' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpV4(TestMarginCrossEntropyOpV2): - def init_reduction(self): self.reduction = 'sum' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMarginCrossEntropyOpAPIError(unittest.TestCase): - def setUp(self): self.initParams() np.random.seed(self.seed) @@ -408,17 +440,15 @@ def init_dtype(self): self.dtype = np.float64 def test_dynamic_errors(self): - def test_dim(): for place in self.places: with paddle.fluid.dygraph.guard(place): - labels_np = np.random.randint(0, - self.num_class, - (self.batch_dim, 2), - dtype="int64") + labels_np = np.random.randint( + 0, self.num_class, (self.batch_dim, 2), dtype="int64" + ) logits_np = np.random.uniform( - -0.99, 0.99, - [self.batch_dim, self.num_class]).astype(self.dtype) + -0.99, 0.99, [self.batch_dim, self.num_class] + ).astype(self.dtype) labels = paddle.to_tensor(labels_np) logits = paddle.to_tensor(logits_np) @@ -430,17 +460,18 @@ def test_dim(): margin3=self.margin3, scale=self.scale, return_softmax=True, - reduction=None) + reduction=None, + ) def test_label_type(): for place in self.places: with paddle.fluid.dygraph.guard(place): - labels_np = np.random.uniform(0, self.num_class, - (self.batch_dim, 1)).astype( - self.dtype) + labels_np = np.random.uniform( + 0, self.num_class, (self.batch_dim, 1) + ).astype(self.dtype) logits_np = np.random.uniform( - -0.99, 0.99, - [self.batch_dim, self.num_class]).astype(self.dtype) + -0.99, 0.99, [self.batch_dim, self.num_class] + ).astype(self.dtype) labels = paddle.to_tensor(labels_np) logits = paddle.to_tensor(logits_np) @@ -452,18 +483,18 @@ def test_label_type(): margin3=self.margin3, scale=self.scale, return_softmax=True, - reduction=None) + reduction=None, + ) def test_group_value(): for place in self.places: with paddle.fluid.dygraph.guard(place): - labels_np = np.random.randint(0, - self.num_class, - (self.batch_dim, ), - dtype="int64") + labels_np = np.random.randint( + 0, self.num_class, (self.batch_dim,), dtype="int64" + ) logits_np = np.random.uniform( - -0.99, 0.99, - [self.batch_dim, self.num_class]).astype(self.dtype) + -0.99, 0.99, [self.batch_dim, self.num_class] + ).astype(self.dtype) labels = paddle.to_tensor(labels_np) logits = paddle.to_tensor(logits_np) @@ -476,7 +507,8 @@ def test_group_value(): scale=self.scale, return_softmax=True, reduction=None, - group=True) + group=True, + ) self.assertRaises(ValueError, test_dim) self.assertRaises(NotImplementedError, test_label_type) diff --git a/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py b/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py index 95919ff53292f..f68b995a1ff7a 100644 --- a/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py @@ -19,20 +19,21 @@ class TestMarginRankLossOp(OpTest): - def setUp(self): self.op_type = "margin_rank_loss" batch_size = 5 margin = 0.5 # labels_{i} = {-1, 1} - label = 2 * np.random.randint( - 0, 2, size=(batch_size, 1)).astype("float32") - 1 + label = ( + 2 * np.random.randint(0, 2, size=(batch_size, 1)).astype("float32") + - 1 + ) x1 = np.random.random((batch_size, 1)).astype("float32") x2 = np.random.random((batch_size, 1)).astype("float32") # loss = max(0, -label * (x1 - x2) + margin) loss = -label * (x1 - x2) + margin loss = np.where(loss > 0, loss, 0) - act = np.where(loss > 0, 1., 0.) + act = np.where(loss > 0, 1.0, 0.0) self.attrs = {'margin': margin} self.inputs = {'Label': label, 'X1': x1, 'X2': x2} @@ -52,13 +53,17 @@ def test_check_grad_ignore_x2(self): class TestMarginRankLossLayer(unittest.TestCase): - def setUp(self): self.batch_size = 5 self.margin = 0.5 # labels_{i} = {-1, 1} - self.label = 2 * np.random.randint( - 0, 2, size=(self.batch_size, 1)).astype("float32") - 1 + self.label = ( + 2 + * np.random.randint(0, 2, size=(self.batch_size, 1)).astype( + "float32" + ) + - 1 + ) self.x1 = np.random.random((self.batch_size, 1)).astype("float32") self.x2 = np.random.random((self.batch_size, 1)).astype("float32") # loss = max(0, -label * (x1 - x2) + margin) @@ -86,13 +91,11 @@ def check_identity(self, place): exe = fluid.Executor(place) exe.run(start) - out_np, = exe.run(main, - feed={ - "label": self.label, - "x1": self.x1, - "x2": self.x2 - }, - fetch_list=[out]) + (out_np,) = exe.run( + main, + feed={"label": self.label, "x1": self.x1, "x2": self.x2}, + fetch_list=[out], + ) np.testing.assert_allclose(out_np, self.loss) diff --git a/python/paddle/fluid/tests/unittests/test_marker_op.py b/python/paddle/fluid/tests/unittests/test_marker_op.py index f0cb6ec1bbbf2..ed19915c41e00 100644 --- a/python/paddle/fluid/tests/unittests/test_marker_op.py +++ b/python/paddle/fluid/tests/unittests/test_marker_op.py @@ -17,14 +17,13 @@ class TestMarkerOp(OpTest): - def setUp(self): self.op_type = "marker" self.inputs = {} self.attrs = { 'marker_role': 'forward', 'marker_pos': 'B', - 'op_role': OpRole.Forward + 'op_role': OpRole.Forward, } self.outputs = {} diff --git a/python/paddle/fluid/tests/unittests/test_masked_select_op.py b/python/paddle/fluid/tests/unittests/test_masked_select_op.py index 4de07192dd08d..7774eb23dc171 100644 --- a/python/paddle/fluid/tests/unittests/test_masked_select_op.py +++ b/python/paddle/fluid/tests/unittests/test_masked_select_op.py @@ -27,7 +27,6 @@ def np_masked_select(x, mask): class TestMaskedSelectOp(OpTest): - def setUp(self): self.init() self.op_type = "masked_select" @@ -49,19 +48,16 @@ def init(self): class TestMaskedSelectOp1(TestMaskedSelectOp): - def init(self): self.shape = (6, 8, 9, 18) class TestMaskedSelectOp2(TestMaskedSelectOp): - def init(self): - self.shape = (168, ) + self.shape = (168,) class TestMaskedSelectAPI(unittest.TestCase): - def test_imperative_mode(self): paddle.disable_static() shape = (88, 6, 8) @@ -86,27 +82,26 @@ def test_static_mode(self): exe = paddle.static.Executor(place=paddle.CPUPlace()) - res, = exe.run(paddle.static.default_main_program(), - feed={ - "x": np_x, - "mask": np_mask - }, - fetch_list=[out]) + (res,) = exe.run( + paddle.static.default_main_program(), + feed={"x": np_x, "mask": np_mask}, + fetch_list=[out], + ) np.testing.assert_allclose(res, np_out, rtol=1e-05) class TestMaskedSelectError(unittest.TestCase): - def test_error(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): shape = [8, 9, 6] x = paddle.fluid.data(shape=shape, dtype='float32', name='x') mask = paddle.fluid.data(shape=shape, dtype='bool', name='mask') - mask_float = paddle.fluid.data(shape=shape, - dtype='float32', - name='mask_float') + mask_float = paddle.fluid.data( + shape=shape, dtype='float32', name='mask_float' + ) np_x = np.random.random(shape).astype('float32') np_mask = np.array(np.random.randint(2, size=shape, dtype=bool)) diff --git a/python/paddle/fluid/tests/unittests/test_match_matrix_tensor_op.py b/python/paddle/fluid/tests/unittests/test_match_matrix_tensor_op.py index 56e6ab07a3019..8b46676704e05 100644 --- a/python/paddle/fluid/tests/unittests/test_match_matrix_tensor_op.py +++ b/python/paddle/fluid/tests/unittests/test_match_matrix_tensor_op.py @@ -19,7 +19,6 @@ class TestMatchMatrixTensorOp(OpTest): - def setUp(self): self.init_op_type() self.set_data() @@ -56,8 +55,8 @@ def compute(self): for idx in range(len(x_lod[0])): x_len = x_lod[0][idx] y_len = y_lod[0][idx] - x_sub = x_data[x_offset:(x_offset + x_len), :] - y_sub = y_data[y_offset:(y_offset + y_len), :] + x_sub = x_data[x_offset : (x_offset + x_len), :] + y_sub = y_data[y_offset : (y_offset + y_len), :] tmp_sub = np.dot(x_sub, w_data) tmp = np.vstack((tmp, tmp_sub.reshape(tmp_sub.size, 1))) @@ -77,7 +76,6 @@ def test_check_grad(self): class TestMatchMatrixTensorOpCase1(TestMatchMatrixTensorOp): - def set_data(self): ix, iy, h, dim_t = [5, 8, 25, 4] x_lod = [[5]] @@ -86,7 +84,6 @@ def set_data(self): class TestMatchMatrixTensorOpCase2(TestMatchMatrixTensorOp): - def set_data(self): ix, iy, h, dim_t = [105, 120, 1, 4] x_lod = [[30, 45, 30]] @@ -95,7 +92,6 @@ def set_data(self): class TestMatchMatrixTensorOpCase3(TestMatchMatrixTensorOp): - def set_data(self): ix, iy, h, dim_t = [5, 9, 32, 1] x_lod = [[1, 2, 2]] @@ -104,7 +100,6 @@ def set_data(self): class TestMatchMatrixTensorOpCase4(TestMatchMatrixTensorOp): - def set_data(self): ix, iy, h, dim_t = [8, 12, 16, 5] x_lod = [[1, 2, 3, 1, 1]] @@ -114,9 +109,9 @@ def set_data(self): def test_api(self): x_lod_tensor = fluid.layers.data(name='x', shape=[10], lod_level=1) y_lod_tensor = fluid.layers.data(name='y', shape=[10], lod_level=1) - out, out_tmp = fluid.contrib.match_matrix_tensor(x=x_lod_tensor, - y=y_lod_tensor, - channel_num=3) + out, out_tmp = fluid.contrib.match_matrix_tensor( + x=x_lod_tensor, y=y_lod_tensor, channel_num=3 + ) place = fluid.CPUPlace() x_data = np.random.rand(7, 10).astype('float32') @@ -126,12 +121,9 @@ def test_api(self): exe = fluid.Executor(place=place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'x': x, - 'y': y - }, - fetch_list=[out], - return_numpy=False) + ret = exe.run( + feed={'x': x, 'y': y}, fetch_list=[out], return_numpy=False + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_math_op_patch.py b/python/paddle/fluid/tests/unittests/test_math_op_patch.py index 9e36c3b382f21..9efdb268a4b69 100644 --- a/python/paddle/fluid/tests/unittests/test_math_op_patch.py +++ b/python/paddle/fluid/tests/unittests/test_math_op_patch.py @@ -21,7 +21,6 @@ class TestMathOpPatches(unittest.TestCase): - def setUp(self): paddle.enable_static() @@ -36,9 +35,9 @@ def test_add_scalar(self): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') - b_np, c_np, d_np = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b, c, d]) + b_np, c_np, d_np = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b, c, d] + ) np.testing.assert_allclose(a_np + 10, b_np, rtol=1e-05) ab_np = np.concatenate([a_np, b_np], axis=1) np.testing.assert_allclose(ab_np + 10, c_np, rtol=1e-05) @@ -52,9 +51,9 @@ def test_radd_scalar(self): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') - b_np = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + b_np = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(a_np + 10, b_np, rtol=1e-05) @prog_scope() @@ -64,9 +63,9 @@ def test_sub_scalar(self): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(a_np - 10, b_np, rtol=1e-05) @prog_scope() @@ -76,9 +75,9 @@ def test_radd_scalar(self): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(10 - a_np, b_np, rtol=1e-05) @prog_scope() @@ -88,9 +87,9 @@ def test_mul_scalar(self): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(a_np * 10, b_np, rtol=1e-05) @prog_scope() @@ -100,9 +99,9 @@ def test_rmul_scalar(self): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(10 * a_np, b_np, rtol=1e-05) @prog_scope() @@ -112,9 +111,9 @@ def test_div_scalar(self): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(a_np / 10, b_np, rtol=1e-05) @prog_scope() @@ -125,9 +124,9 @@ def test_rdiv_scalar(self): exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') + 1e-2 - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(10 / a_np, b_np, rtol=1e-05) @prog_scope() @@ -139,12 +138,11 @@ def test_div_two_tensor(self): exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') b_np = np.random.random(size=[10, 1]).astype('float32') + 1e-2 - c_np, = exe.run(fluid.default_main_program(), - feed={ - "a": a_np, - 'b': b_np - }, - fetch_list=[c]) + (c_np,) = exe.run( + fluid.default_main_program(), + feed={"a": a_np, 'b': b_np}, + fetch_list=[c], + ) np.testing.assert_allclose(a_np / b_np, c_np, rtol=1e-05) @prog_scope() @@ -156,12 +154,11 @@ def test_mul_two_tensor(self): exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') b_np = np.random.random(size=[10, 1]).astype('float32') - c_np, = exe.run(fluid.default_main_program(), - feed={ - "a": a_np, - 'b': b_np - }, - fetch_list=[c]) + (c_np,) = exe.run( + fluid.default_main_program(), + feed={"a": a_np, 'b': b_np}, + fetch_list=[c], + ) np.testing.assert_allclose(a_np * b_np, c_np, rtol=1e-05) @prog_scope() @@ -173,12 +170,11 @@ def test_add_two_tensor(self): exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') b_np = np.random.random(size=[10, 1]).astype('float32') - c_np, = exe.run(fluid.default_main_program(), - feed={ - "a": a_np, - 'b': b_np - }, - fetch_list=[c]) + (c_np,) = exe.run( + fluid.default_main_program(), + feed={"a": a_np, 'b': b_np}, + fetch_list=[c], + ) np.testing.assert_allclose(a_np + b_np, c_np, rtol=1e-05) @prog_scope() @@ -190,12 +186,11 @@ def test_sub_two_tensor(self): exe = fluid.Executor(place) a_np = np.random.random(size=[10, 1]).astype('float32') b_np = np.random.random(size=[10, 1]).astype('float32') - c_np, = exe.run(fluid.default_main_program(), - feed={ - "a": a_np, - 'b': b_np - }, - fetch_list=[c]) + (c_np,) = exe.run( + fluid.default_main_program(), + feed={"a": a_np, 'b': b_np}, + fetch_list=[c], + ) np.testing.assert_allclose(a_np - b_np, c_np, rtol=1e-05) @prog_scope() @@ -205,9 +200,9 @@ def test_integer_div(self): place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.array([3, 4, 10, 14, 9, 18]).astype('int64') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) b_np_actual = (a_np / 7).astype('float32') np.testing.assert_allclose(b_np, b_np_actual, rtol=1e-05) @@ -216,19 +211,18 @@ def test_integer_div(self): def test_equal(self): a = fluid.layers.data(name="a", shape=[1], dtype='float32') b = fluid.layers.data(name="b", shape=[1], dtype='float32') - c = (a == b) + c = a == b place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.array([3, 4, 10, 14, 9, 18]).astype('float32') b_np = np.array([3, 4, 11, 15, 8, 18]).astype('float32') - c_np, = exe.run(fluid.default_main_program(), - feed={ - "a": a_np, - "b": b_np - }, - fetch_list=[c]) + (c_np,) = exe.run( + fluid.default_main_program(), + feed={"a": a_np, "b": b_np}, + fetch_list=[c], + ) np.testing.assert_array_equal(c_np, a_np == b_np) self.assertEqual(c.dtype, fluid.core.VarDesc.VarType.BOOL) @@ -240,19 +234,18 @@ def test_equal_and_cond(self): one = fluid.layers.ones(shape=[1], dtype='int32') zero = fluid.layers.zeros(shape=[1], dtype='int32') - cond = (one == zero) + cond = one == zero c = fluid.layers.cond(cond, lambda: a + b, lambda: a - b) place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = np.array([3, 4, 10, 14, 9, 18]).astype('float') b_np = np.array([3, 4, 11, 15, 8, 18]).astype('float') - c_np, = exe.run(fluid.default_main_program(), - feed={ - "a": a_np, - "b": b_np - }, - fetch_list=[c]) + (c_np,) = exe.run( + fluid.default_main_program(), + feed={"a": a_np, "b": b_np}, + fetch_list=[c], + ) np.testing.assert_array_equal(c_np, a_np - b_np) @@ -264,9 +257,9 @@ def test_neg(self): exe = fluid.Executor(place) a_np = np.random.uniform(-1, 1, size=[10, 1]).astype('float32') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(-a_np, b_np, rtol=1e-05) @prog_scope() @@ -277,9 +270,9 @@ def test_astype(self): exe = fluid.Executor(place) a_np = np.random.uniform(-1, 1, size=[10, 1]).astype('float64') - b_np, = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + (b_np,) = exe.run( + fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b] + ) np.testing.assert_allclose(a_np.astype('float32'), b_np, rtol=1e-05) def test_bitwise_and(self): @@ -292,12 +285,11 @@ def test_bitwise_and(self): z = x & y exe = fluid.Executor() - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_np, - "y": y_np - }, - fetch_list=[z]) + out = exe.run( + fluid.default_main_program(), + feed={"x": x_np, "y": y_np}, + fetch_list=[z], + ) np.testing.assert_array_equal(out[0], out_np) @prog_scope() @@ -311,12 +303,11 @@ def test_bitwise_or(self): z = x | y exe = fluid.Executor() - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_np, - "y": y_np - }, - fetch_list=[z]) + out = exe.run( + fluid.default_main_program(), + feed={"x": x_np, "y": y_np}, + fetch_list=[z], + ) np.testing.assert_array_equal(out[0], out_np) @prog_scope() @@ -330,12 +321,11 @@ def test_bitwise_xor(self): z = x ^ y exe = fluid.Executor() - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_np, - "y": y_np - }, - fetch_list=[z]) + out = exe.run( + fluid.default_main_program(), + feed={"x": x_np, "y": y_np}, + fetch_list=[z], + ) np.testing.assert_array_equal(out[0], out_np) @prog_scope() @@ -347,9 +337,9 @@ def test_bitwise_not(self): z = ~x exe = fluid.Executor() - out = exe.run(fluid.default_main_program(), - feed={"x": x_np}, - fetch_list=[z]) + out = exe.run( + fluid.default_main_program(), feed={"x": x_np}, fetch_list=[z] + ) np.testing.assert_array_equal(out[0], out_np) @prog_scope() @@ -361,9 +351,9 @@ def test_T(self): z = x.T exe = fluid.Executor() - out = exe.run(fluid.default_main_program(), - feed={"x": x_np}, - fetch_list=[z]) + out = exe.run( + fluid.default_main_program(), feed={"x": x_np}, fetch_list=[z] + ) np.testing.assert_array_equal(out[0], out_np) @prog_scope() @@ -382,12 +372,11 @@ def test_matmul(self): b_np = np.random.uniform(-1, 1, size=[3, 5]).astype('float32') place = paddle.CPUPlace() exe = paddle.static.Executor(place) - c_np, = exe.run(paddle.static.default_main_program(), - feed={ - "a": a_np, - "b": b_np - }, - fetch_list=[c]) + (c_np,) = exe.run( + paddle.static.default_main_program(), + feed={"a": a_np, "b": b_np}, + fetch_list=[c], + ) np.testing.assert_allclose(a_np @ b_np, c_np, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_math_op_patch_var_base.py b/python/paddle/fluid/tests/unittests/test_math_op_patch_var_base.py index 5c66d4cd6b5f1..5330dc8b4f852 100644 --- a/python/paddle/fluid/tests/unittests/test_math_op_patch_var_base.py +++ b/python/paddle/fluid/tests/unittests/test_math_op_patch_var_base.py @@ -21,7 +21,6 @@ class TestMathOpPatchesVarBase(unittest.TestCase): - def setUp(self): self.shape = [10, 1024] self.dtype = np.float32 @@ -75,7 +74,7 @@ def func_test_div(self): a = fluid.dygraph.to_variable(a_np) b = fluid.dygraph.to_variable(b_np) res = a / b - #NOTE: Not sure why array_equal fails on windows, allclose is acceptable + # NOTE: Not sure why array_equal fails on windows, allclose is acceptable np.testing.assert_allclose(res.numpy(), a_np / b_np, rtol=1e-05) def test_div(self): @@ -244,8 +243,8 @@ def func_test_equal(self): a = fluid.dygraph.to_variable(a_np) b = fluid.dygraph.to_variable(b_np) c = fluid.dygraph.to_variable(c_np) - res1 = (a == b) - res2 = (a == c) + res1 = a == b + res2 = a == c np.testing.assert_array_equal(res1.numpy(), a_np == b_np) np.testing.assert_array_equal(res2.numpy(), a_np == c_np) @@ -262,8 +261,8 @@ def func_test_not_equal(self): a = fluid.dygraph.to_variable(a_np) b = fluid.dygraph.to_variable(b_np) c = fluid.dygraph.to_variable(c_np) - res1 = (a != b) - res2 = (a != c) + res1 = a != b + res2 = a != c np.testing.assert_array_equal(res1.numpy(), a_np != b_np) np.testing.assert_array_equal(res2.numpy(), a_np != c_np) @@ -278,7 +277,7 @@ def func_test_less_than(self): with fluid.dygraph.guard(): a = fluid.dygraph.to_variable(a_np) b = fluid.dygraph.to_variable(b_np) - res = (a < b) + res = a < b np.testing.assert_array_equal(res.numpy(), a_np < b_np) def test_less_than(self): @@ -292,7 +291,7 @@ def func_test_less_equal(self): with fluid.dygraph.guard(): a = fluid.dygraph.to_variable(a_np) b = fluid.dygraph.to_variable(b_np) - res = (a <= b) + res = a <= b np.testing.assert_array_equal(res.numpy(), a_np <= b_np) def test_less_equal(self): @@ -306,7 +305,7 @@ def func_test_greater_than(self): with fluid.dygraph.guard(): a = fluid.dygraph.to_variable(a_np) b = fluid.dygraph.to_variable(b_np) - res = (a > b) + res = a > b np.testing.assert_array_equal(res.numpy(), a_np > b_np) def test_greater_than(self): @@ -320,7 +319,7 @@ def func_test_greater_equal(self): with fluid.dygraph.guard(): a = fluid.dygraph.to_variable(a_np) b = fluid.dygraph.to_variable(b_np) - res = (a >= b) + res = a >= b np.testing.assert_array_equal(res.numpy(), a_np >= b_np) def test_greater_equal(self): @@ -386,10 +385,12 @@ def func_test_np_left_mul(self): x = fluid.layers.ones((2, 2), dtype="float32") y = t * x - np.testing.assert_allclose(y.numpy(), - t * np.ones((2, 2), dtype='float32'), - rtol=1e-05, - atol=0.0) + np.testing.assert_allclose( + y.numpy(), + t * np.ones((2, 2), dtype='float32'), + rtol=1e-05, + atol=0.0, + ) def test_np_left_mul(self): with _test_eager_guard(): @@ -483,92 +484,112 @@ def func_test_tensor_patch_method(self): np.testing.assert_array_equal(x.abs().numpy(), paddle.abs(x).numpy()) m = x.abs() np.testing.assert_array_equal(m.sqrt().numpy(), paddle.sqrt(m).numpy()) - np.testing.assert_array_equal(m.rsqrt().numpy(), - paddle.rsqrt(m).numpy()) + np.testing.assert_array_equal( + m.rsqrt().numpy(), paddle.rsqrt(m).numpy() + ) np.testing.assert_array_equal(x.ceil().numpy(), paddle.ceil(x).numpy()) - np.testing.assert_array_equal(x.floor().numpy(), - paddle.floor(x).numpy()) + np.testing.assert_array_equal( + x.floor().numpy(), paddle.floor(x).numpy() + ) np.testing.assert_array_equal(x.cos().numpy(), paddle.cos(x).numpy()) np.testing.assert_array_equal(x.acos().numpy(), paddle.acos(x).numpy()) np.testing.assert_array_equal(x.asin().numpy(), paddle.asin(x).numpy()) np.testing.assert_array_equal(x.sin().numpy(), paddle.sin(x).numpy()) np.testing.assert_array_equal(x.sinh().numpy(), paddle.sinh(x).numpy()) np.testing.assert_array_equal(x.cosh().numpy(), paddle.cosh(x).numpy()) - np.testing.assert_array_equal(x.round().numpy(), - paddle.round(x).numpy()) - np.testing.assert_array_equal(x.reciprocal().numpy(), - paddle.reciprocal(x).numpy()) - np.testing.assert_array_equal(x.square().numpy(), - paddle.square(x).numpy()) + np.testing.assert_array_equal( + x.round().numpy(), paddle.round(x).numpy() + ) + np.testing.assert_array_equal( + x.reciprocal().numpy(), paddle.reciprocal(x).numpy() + ) + np.testing.assert_array_equal( + x.square().numpy(), paddle.square(x).numpy() + ) np.testing.assert_array_equal(x.rank().numpy(), paddle.rank(x).numpy()) np.testing.assert_array_equal(x[0].t().numpy(), paddle.t(x[0]).numpy()) - np.testing.assert_array_equal(x.asinh().numpy(), - paddle.asinh(x).numpy()) + np.testing.assert_array_equal( + x.asinh().numpy(), paddle.asinh(x).numpy() + ) ### acosh(x) = nan, need to change input t_np = np.random.uniform(1, 2, [2, 3]).astype(self.dtype) t = paddle.to_tensor(t_np) - np.testing.assert_array_equal(t.acosh().numpy(), - paddle.acosh(t).numpy()) - np.testing.assert_array_equal(x.atanh().numpy(), - paddle.atanh(x).numpy()) - d = paddle.to_tensor([[1.2285208, 1.3491015, 1.4899898], - [1.30058, 1.0688717, 1.4928783], - [1.0958099, 1.3724753, 1.8926544]]) + np.testing.assert_array_equal( + t.acosh().numpy(), paddle.acosh(t).numpy() + ) + np.testing.assert_array_equal( + x.atanh().numpy(), paddle.atanh(x).numpy() + ) + d = paddle.to_tensor( + [ + [1.2285208, 1.3491015, 1.4899898], + [1.30058, 1.0688717, 1.4928783], + [1.0958099, 1.3724753, 1.8926544], + ] + ) d = d.matmul(d.t()) # ROCM not support cholesky if not fluid.core.is_compiled_with_rocm(): - np.testing.assert_array_equal(d.cholesky().numpy(), - paddle.cholesky(d).numpy()) + np.testing.assert_array_equal( + d.cholesky().numpy(), paddle.cholesky(d).numpy() + ) - np.testing.assert_array_equal(x.is_empty().numpy(), - paddle.is_empty(x).numpy()) - np.testing.assert_array_equal(x.isfinite().numpy(), - paddle.isfinite(x).numpy()) np.testing.assert_array_equal( - x.cast('int32').numpy(), - paddle.cast(x, 'int32').numpy()) + x.is_empty().numpy(), paddle.is_empty(x).numpy() + ) + np.testing.assert_array_equal( + x.isfinite().numpy(), paddle.isfinite(x).numpy() + ) np.testing.assert_array_equal( - x.expand([3, 2, 3]).numpy(), - paddle.expand(x, [3, 2, 3]).numpy()) + x.cast('int32').numpy(), paddle.cast(x, 'int32').numpy() + ) np.testing.assert_array_equal( - x.tile([2, 2]).numpy(), - paddle.tile(x, [2, 2]).numpy()) - np.testing.assert_array_equal(x.flatten().numpy(), - paddle.flatten(x).numpy()) + x.expand([3, 2, 3]).numpy(), paddle.expand(x, [3, 2, 3]).numpy() + ) + np.testing.assert_array_equal( + x.tile([2, 2]).numpy(), paddle.tile(x, [2, 2]).numpy() + ) + np.testing.assert_array_equal( + x.flatten().numpy(), paddle.flatten(x).numpy() + ) index = paddle.to_tensor([0, 1]) np.testing.assert_array_equal( - x.gather(index).numpy(), - paddle.gather(x, index).numpy()) + x.gather(index).numpy(), paddle.gather(x, index).numpy() + ) index = paddle.to_tensor([[0, 1], [1, 2]]) np.testing.assert_array_equal( - x.gather_nd(index).numpy(), - paddle.gather_nd(x, index).numpy()) + x.gather_nd(index).numpy(), paddle.gather_nd(x, index).numpy() + ) np.testing.assert_array_equal( - x.reverse([0, 1]).numpy(), - paddle.reverse(x, [0, 1]).numpy()) + x.reverse([0, 1]).numpy(), paddle.reverse(x, [0, 1]).numpy() + ) np.testing.assert_array_equal( - a.reshape([3, 2]).numpy(), - paddle.reshape(a, [3, 2]).numpy()) + a.reshape([3, 2]).numpy(), paddle.reshape(a, [3, 2]).numpy() + ) np.testing.assert_array_equal( x.slice([0, 1], [0, 0], [1, 2]).numpy(), - paddle.slice(x, [0, 1], [0, 0], [1, 2]).numpy()) + paddle.slice(x, [0, 1], [0, 0], [1, 2]).numpy(), + ) np.testing.assert_array_equal( - x.split(2)[0].numpy(), - paddle.split(x, 2)[0].numpy()) + x.split(2)[0].numpy(), paddle.split(x, 2)[0].numpy() + ) m = paddle.to_tensor( - np.random.uniform(-1, 1, [1, 6, 1, 1]).astype(self.dtype)) + np.random.uniform(-1, 1, [1, 6, 1, 1]).astype(self.dtype) + ) np.testing.assert_array_equal( - m.squeeze([]).numpy(), - paddle.squeeze(m, []).numpy()) + m.squeeze([]).numpy(), paddle.squeeze(m, []).numpy() + ) np.testing.assert_array_equal( - m.squeeze([1, 2]).numpy(), - paddle.squeeze(m, [1, 2]).numpy()) + m.squeeze([1, 2]).numpy(), paddle.squeeze(m, [1, 2]).numpy() + ) m = paddle.to_tensor([2, 3, 3, 1, 5, 3], 'float32') - np.testing.assert_array_equal(m.unique()[0].numpy(), - paddle.unique(m)[0].numpy()) + np.testing.assert_array_equal( + m.unique()[0].numpy(), paddle.unique(m)[0].numpy() + ) np.testing.assert_array_equal( m.unique(return_counts=True)[1], - paddle.unique(m, return_counts=True)[1]) + paddle.unique(m, return_counts=True)[1], + ) np.testing.assert_array_equal(x.flip([0]), paddle.flip(x, [0])) np.testing.assert_array_equal(x.unbind(0), paddle.unbind(x, 0)) np.testing.assert_array_equal(x.roll(1), paddle.roll(x, 1)) @@ -582,84 +603,86 @@ def func_test_tensor_patch_method(self): # 2. Binary operation np.testing.assert_array_equal( - x.divide(y).numpy(), - paddle.divide(x, y).numpy()) + x.divide(y).numpy(), paddle.divide(x, y).numpy() + ) np.testing.assert_array_equal( x.matmul(y, True, False).numpy(), - paddle.matmul(x, y, True, False).numpy()) + paddle.matmul(x, y, True, False).numpy(), + ) np.testing.assert_array_equal( x.norm(p='fro', axis=[0, 1]).numpy(), - paddle.norm(x, p='fro', axis=[0, 1]).numpy()) + paddle.norm(x, p='fro', axis=[0, 1]).numpy(), + ) np.testing.assert_array_equal( - x.dist(y).numpy(), - paddle.dist(x, y).numpy()) + x.dist(y).numpy(), paddle.dist(x, y).numpy() + ) np.testing.assert_array_equal( - x.cross(y).numpy(), - paddle.cross(x, y).numpy()) + x.cross(y).numpy(), paddle.cross(x, y).numpy() + ) m = x.expand([2, 2, 3]) n = y.expand([2, 2, 3]).transpose([0, 2, 1]) np.testing.assert_array_equal( - m.bmm(n).numpy(), - paddle.bmm(m, n).numpy()) + m.bmm(n).numpy(), paddle.bmm(m, n).numpy() + ) np.testing.assert_array_equal( - x.histogram(5, -1, 1).numpy(), - paddle.histogram(x, 5, -1, 1).numpy()) + x.histogram(5, -1, 1).numpy(), paddle.histogram(x, 5, -1, 1).numpy() + ) np.testing.assert_array_equal( - x.equal(y).numpy(), - paddle.equal(x, y).numpy()) + x.equal(y).numpy(), paddle.equal(x, y).numpy() + ) np.testing.assert_array_equal( - x.greater_equal(y).numpy(), - paddle.greater_equal(x, y).numpy()) + x.greater_equal(y).numpy(), paddle.greater_equal(x, y).numpy() + ) np.testing.assert_array_equal( - x.greater_than(y).numpy(), - paddle.greater_than(x, y).numpy()) + x.greater_than(y).numpy(), paddle.greater_than(x, y).numpy() + ) np.testing.assert_array_equal( - x.less_equal(y).numpy(), - paddle.less_equal(x, y).numpy()) + x.less_equal(y).numpy(), paddle.less_equal(x, y).numpy() + ) np.testing.assert_array_equal( - x.less_than(y).numpy(), - paddle.less_than(x, y).numpy()) + x.less_than(y).numpy(), paddle.less_than(x, y).numpy() + ) np.testing.assert_array_equal( - x.not_equal(y).numpy(), - paddle.not_equal(x, y).numpy()) + x.not_equal(y).numpy(), paddle.not_equal(x, y).numpy() + ) np.testing.assert_array_equal( - x.equal_all(y).numpy(), - paddle.equal_all(x, y).numpy()) + x.equal_all(y).numpy(), paddle.equal_all(x, y).numpy() + ) np.testing.assert_array_equal( - x.allclose(y).numpy(), - paddle.allclose(x, y).numpy()) + x.allclose(y).numpy(), paddle.allclose(x, y).numpy() + ) m = x.expand([2, 2, 3]) np.testing.assert_array_equal( - x.expand_as(m).numpy(), - paddle.expand_as(x, m).numpy()) + x.expand_as(m).numpy(), paddle.expand_as(x, m).numpy() + ) index = paddle.to_tensor([2, 1, 0]) np.testing.assert_array_equal( - a.scatter(index, b).numpy(), - paddle.scatter(a, index, b).numpy()) + a.scatter(index, b).numpy(), paddle.scatter(a, index, b).numpy() + ) # 3. Bool tensor operation x = paddle.to_tensor([[True, False], [True, False]]) y = paddle.to_tensor([[False, False], [False, True]]) np.testing.assert_array_equal( - x.logical_and(y).numpy(), - paddle.logical_and(x, y).numpy()) + x.logical_and(y).numpy(), paddle.logical_and(x, y).numpy() + ) np.testing.assert_array_equal( - x.logical_not(y).numpy(), - paddle.logical_not(x, y).numpy()) + x.logical_not(y).numpy(), paddle.logical_not(x, y).numpy() + ) np.testing.assert_array_equal( - x.logical_or(y).numpy(), - paddle.logical_or(x, y).numpy()) + x.logical_or(y).numpy(), paddle.logical_or(x, y).numpy() + ) np.testing.assert_array_equal( - x.logical_xor(y).numpy(), - paddle.logical_xor(x, y).numpy()) + x.logical_xor(y).numpy(), paddle.logical_xor(x, y).numpy() + ) np.testing.assert_array_equal( - x.logical_and(y).numpy(), - paddle.logical_and(x, y).numpy()) + x.logical_and(y).numpy(), paddle.logical_and(x, y).numpy() + ) a = paddle.to_tensor([[1, 2], [3, 4]]) b = paddle.to_tensor([[4, 3], [2, 1]]) np.testing.assert_array_equal( - x.where(a, b).numpy(), - paddle.where(x, a, b).numpy()) + x.where(a, b).numpy(), paddle.where(x, a, b).numpy() + ) x_np = np.random.randn(3, 6, 9, 7) x = paddle.to_tensor(x_np) @@ -724,7 +747,7 @@ def func_test_complex_scalar(self): a_np = np.random.random(self.shape).astype(self.dtype) with fluid.dygraph.guard(): a = fluid.dygraph.to_variable(a_np) - res = 1J * a + res = 1j * a np.testing.assert_array_equal(res.numpy(), 1j * a_np) def test_complex_scalar(self): diff --git a/python/paddle/fluid/tests/unittests/test_matmul_op.py b/python/paddle/fluid/tests/unittests/test_matmul_op.py index b8ca9b9dc1a92..62c7e9b7bb716 100644 --- a/python/paddle/fluid/tests/unittests/test_matmul_op.py +++ b/python/paddle/fluid/tests/unittests/test_matmul_op.py @@ -86,7 +86,6 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): class Generator(object): - def setUp(self): self.op_type = "matmul" X = np.random.random(self.shape_X).astype("float32") @@ -95,7 +94,7 @@ def setUp(self): self.inputs = {'X': X, 'Y': Y} self.attrs = { 'transpose_X': self.transpose_X, - 'transpose_Y': self.transpose_Y + 'transpose_Y': self.transpose_Y, } self.outputs = {'Out': Out} @@ -106,39 +105,37 @@ def test_check_grad_normal(self): self.check_grad(['X', 'Y'], 'Out', max_relative_error=1e-3) def test_check_grad_ignore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=1e-3, - no_grad_set=set("X")) + self.check_grad( + ['Y'], 'Out', max_relative_error=1e-3, no_grad_set=set("X") + ) def test_check_grad_ignore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=1e-3, - no_grad_set=set('Y')) + self.check_grad( + ['X'], 'Out', max_relative_error=1e-3, no_grad_set=set('Y') + ) class TestMatmulOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The inputs type of matmul_op must be Variable. input1 = 12 self.assertRaises(TypeError, fluid.layers.matmul, input1, input1) # The inputs dtype of matmul_op must be float32, float64. - input2 = fluid.layers.data(name='input2', - shape=[10, 10], - dtype="int32") + input2 = fluid.layers.data( + name='input2', shape=[10, 10], dtype="int32" + ) self.assertRaises(TypeError, fluid.layers.matmul, input2, input2) - input3 = fluid.layers.data(name='input3', - shape=[2, 2], - dtype="float16") + input3 = fluid.layers.data( + name='input3', shape=[2, 2], dtype="float16" + ) fluid.layers.matmul(input3, input3) # Negative dimension generation def generate_negative_dims(in_shape): from itertools import combinations + size = len(in_shape) indexs = list() shapes = list() @@ -146,7 +143,8 @@ def generate_negative_dims(in_shape): indexs.extend(list(combinations([j for j in range(size)], i + 1))) for idx in indexs: shapes.append( - [in_shape[i] if i not in idx else -1 for i in range(size)]) + [in_shape[i] if i not in idx else -1 for i in range(size)] + ) return shapes @@ -160,51 +158,61 @@ def test_negative_dims_program(obj): with program_guard(Program(), Program()): x = fluid.data(name='x', shape=shape_x, dtype='float32') y = fluid.data(name='y', shape=shape_y, dtype='float32') - output = fluid.layers.matmul(x, y, obj.transpose_X, - obj.transpose_Y) + output = fluid.layers.matmul( + x, y, obj.transpose_X, obj.transpose_Y + ) obj.assertEqual(len(Ref.shape), len(output.shape)) for idx in range(len(Ref.shape)): if output.shape[idx] != -1: obj.assertEqual(Ref.shape[idx], output.shape[idx]) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(fluid.default_main_program(), - feed={ - 'x': X, - 'y': Y - }, - fetch_list=[output]) + (res,) = exe.run( + fluid.default_main_program(), + feed={'x': X, 'y': Y}, + fetch_list=[output], + ) np.allclose(res, Ref, atol=1e-5) # Generate program api cases for all negative possibilities def api_test(dim_x, dim_y, trans_x, trans_y): - test_name = ('TestMatMulAPI_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( - dim_x, dim_y, trans_x, trans_y)) - shape_x, shape_y = generate_compatible_shapes(dim_x, dim_y, trans_x, - trans_y) + test_name = 'TestMatMulAPI_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( + dim_x, dim_y, trans_x, trans_y + ) + shape_x, shape_y = generate_compatible_shapes( + dim_x, dim_y, trans_x, trans_y + ) globals()[test_name] = type( - test_name, (unittest.TestCase, ), { + test_name, + (unittest.TestCase,), + { 'shape_X': shape_x, 'shape_Y': shape_y, 'transpose_X': trans_x, 'transpose_Y': trans_y, 'test_propram': test_negative_dims_program, - }) + }, + ) # Generate operators cases for all possibilities def inject_test(dim_x, dim_y, trans_x, trans_y): - test_name = ('TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( - dim_x, dim_y, trans_x, trans_y)) - shape_x, shape_y = generate_compatible_shapes(dim_x, dim_y, trans_x, - trans_y) + test_name = 'TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( + dim_x, dim_y, trans_x, trans_y + ) + shape_x, shape_y = generate_compatible_shapes( + dim_x, dim_y, trans_x, trans_y + ) globals()[test_name] = type( - test_name, (Generator, OpTest), { + test_name, + (Generator, OpTest), + { 'shape_X': shape_x, 'shape_Y': shape_y, 'transpose_X': trans_x, 'transpose_Y': trans_y, - }) + }, + ) for dim_X in (1, 2, 3): @@ -216,8 +224,9 @@ def inject_test(dim_x, dim_y, trans_x, trans_y): # Test case more batch_size and N, M, K -def generate_compatible_shapes_batch(dim_X, dim_Y, transpose_X, transpose_Y, - batch_size): +def generate_compatible_shapes_batch( + dim_X, dim_Y, transpose_X, transpose_Y, batch_size +): BATCH_SIZE = 2 M = 3 N = 4 @@ -278,20 +287,25 @@ def generate_compatible_shapes_ndim(dim, transpose_X, transpose_Y): for transpose_Y in [False, True]: test_name = ( 'TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( - dim, dim, transpose_X, transpose_Y)) + dim, dim, transpose_X, transpose_Y + ) + ) shape_X, shape_Y = generate_compatible_shapes_ndim( - dim, transpose_X, transpose_Y) + dim, transpose_X, transpose_Y + ) globals()[test_name] = type( - test_name, (Generator, OpTest), { + test_name, + (Generator, OpTest), + { 'shape_X': shape_X, 'shape_Y': shape_Y, 'transpose_X': transpose_X, 'transpose_Y': transpose_Y, - }) + }, + ) class API_TestMm(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program()): x = fluid.data(name="x", shape=[2], dtype="float64") @@ -302,8 +316,9 @@ def test_out(self): data1 = np.random.rand(2) data2 = np.random.rand(2) np_res = exe.run(feed={'x': data1, 'y': data2}, fetch_list=[result]) - expected_result = np.matmul(data1.reshape(1, 2), - data2.reshape(2, 1)) + expected_result = np.matmul( + data1.reshape(1, 2), data2.reshape(2, 1) + ) np.testing.assert_allclose( np_res, @@ -311,7 +326,9 @@ def test_out(self): rtol=1e-05, atol=1e-05, err_msg='two value is {}\n{}, check diff!'.format( - np_res, expected_result)) + np_res, expected_result + ), + ) def test_dygraph_without_out(self): device = fluid.CPUPlace() @@ -326,7 +343,6 @@ def test_dygraph_without_out(self): class Test_API_Matmul(unittest.TestCase): - def test_dygraph_without_out(self): device = fluid.CPUPlace() with fluid.dygraph.guard(device): @@ -340,9 +356,7 @@ def test_dygraph_without_out(self): class API_TestMmError(unittest.TestCase): - def test_errors(self): - def test_error1(): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.data(name="data1", shape=[10, 2], dtype="float32") @@ -353,24 +367,24 @@ def test_error1(): def test_error2(): with fluid.program_guard(fluid.Program(), fluid.Program()): - data1 = fluid.data(name="data1", - shape=[-1, 10, 2], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[-1, 2, 10], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[-1, 10, 2], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[-1, 2, 10], dtype="float32" + ) paddle.mm(data1, data2) test_error2() def test_error3(): with fluid.program_guard(fluid.Program(), fluid.Program()): - data1 = fluid.data(name="data1", - shape=[10, 10, 2], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[3, 2, 10], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[10, 10, 2], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[3, 2, 10], dtype="float32" + ) paddle.mm(data1, data2) self.assertRaises(ValueError, test_error3) diff --git a/python/paddle/fluid/tests/unittests/test_matmul_op_with_head.py b/python/paddle/fluid/tests/unittests/test_matmul_op_with_head.py index 77e3c7f4c1fca..bd6b1877ecd29 100644 --- a/python/paddle/fluid/tests/unittests/test_matmul_op_with_head.py +++ b/python/paddle/fluid/tests/unittests/test_matmul_op_with_head.py @@ -59,16 +59,18 @@ def matmul_head(X, Y, head_number=1): sub_y_height = Y.shape[-2] // head_number if np.ndim(X) == 2: for i in range(0, head_number): - x.append(X[:, i * sub_x_width:i * sub_x_width + sub_x_width]) - y.append(Y[i * sub_y_height:i * sub_y_height + sub_y_height, :]) + x.append(X[:, i * sub_x_width : i * sub_x_width + sub_x_width]) + y.append(Y[i * sub_y_height : i * sub_y_height + sub_y_height, :]) for i in range(0, head_number): z.append(np.matmul(x[i], y[i])) Z = np.concatenate((z), axis=1) elif np.ndim(X) == 3: for i in range(0, head_number): - x.append(X[:, :, i * sub_x_width:i * sub_x_width + sub_x_width]) - y.append(Y[:, i * sub_y_height:i * sub_y_height + sub_y_height, :]) + x.append(X[:, :, i * sub_x_width : i * sub_x_width + sub_x_width]) + y.append( + Y[:, i * sub_y_height : i * sub_y_height + sub_y_height, :] + ) for i in range(0, head_number): z.append(np.matmul(x[i], y[i])) Z = np.concatenate((z), axis=2) @@ -87,11 +89,9 @@ def transpose_mat(X): return X -def reference_matmul_mul_head(X, - Y, - head_number=1, - transpose_X=False, - transpose_Y=False): +def reference_matmul_mul_head( + X, Y, head_number=1, transpose_X=False, transpose_Y=False +): """Reference forward implementation using np.matmul.""" # np.matmul does not support the transpose flags, so we manually # transpose X and Y appropriately. @@ -112,19 +112,19 @@ def reference_matmul_mul_head(X, # Generator for multiple head class GeneratorMulHead(object): - def setUp(self): self.op_type = "matmul" X = np.random.random(self.shape_X).astype("float32") Y = np.random.random(self.shape_Y).astype("float32") - Out = reference_matmul_mul_head(X, Y, 4, self.transpose_X, - self.transpose_Y) + Out = reference_matmul_mul_head( + X, Y, 4, self.transpose_X, self.transpose_Y + ) self.inputs = {'X': X, 'Y': Y} self.attrs = { 'transpose_X': self.transpose_X, 'transpose_Y': self.transpose_Y, - 'head_number': self.head_number + 'head_number': self.head_number, } self.outputs = {'Out': Out} @@ -135,17 +135,23 @@ def test_check_output(self): def inject_test_multiple_head(dim_x, dim_y, trans_x, trans_y, head_number): test_name = ( 'TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}_head_{}'.format( - dim_x, dim_y, trans_x, trans_y, head_number)) + dim_x, dim_y, trans_x, trans_y, head_number + ) + ) shape_x, shape_y = generate_compatible_shapes_mul_head( - dim_x, dim_y, trans_x, trans_y) + dim_x, dim_y, trans_x, trans_y + ) globals()[test_name] = type( - test_name, (GeneratorMulHead, OpTest), { + test_name, + (GeneratorMulHead, OpTest), + { 'shape_X': shape_x, 'shape_Y': shape_y, 'transpose_X': trans_x, 'transpose_Y': trans_y, - 'head_number': head_number - }) + 'head_number': head_number, + }, + ) def matmul_head2(X, Y, head_number=1): @@ -154,20 +160,21 @@ def matmul_head2(X, Y, head_number=1): z = [] sub_x_width = X.shape[-1] // head_number sub_y_width = Y.shape[-1] // head_number - assert (sub_x_width == Y.shape[-2] - ), "Error: incompatible head number or matrix size!" + assert ( + sub_x_width == Y.shape[-2] + ), "Error: incompatible head number or matrix size!" if np.ndim(X) == 2: for i in range(0, head_number): - x.append(X[:, i * sub_x_width:i * sub_x_width + sub_x_width]) - y.append(Y[:, i * sub_y_width:i * sub_y_width + sub_y_width]) + x.append(X[:, i * sub_x_width : i * sub_x_width + sub_x_width]) + y.append(Y[:, i * sub_y_width : i * sub_y_width + sub_y_width]) for i in range(0, head_number): z.append(np.matmul(x[i], y[i])) Z = np.concatenate((z), axis=1) elif np.ndim(X) == 3: for i in range(0, head_number): - x.append(X[:, :, i * sub_x_width:i * sub_x_width + sub_x_width]) - y.append(Y[:, :, i * sub_y_width:i * sub_y_width + sub_y_width]) + x.append(X[:, :, i * sub_x_width : i * sub_x_width + sub_x_width]) + y.append(Y[:, :, i * sub_y_width : i * sub_y_width + sub_y_width]) for i in range(0, head_number): z.append(np.matmul(x[i], y[i])) Z = np.concatenate((z), axis=2) @@ -176,11 +183,9 @@ def matmul_head2(X, Y, head_number=1): return Z -def reference_matmul_mul_head2(X, - Y, - head_number=1, - transpose_X=False, - transpose_Y=False): +def reference_matmul_mul_head2( + X, Y, head_number=1, transpose_X=False, transpose_Y=False +): """Reference forward implementation using np.matmul.""" # np.matmul does not support the transpose flags, so we manually # transpose X and Y appropriately. @@ -199,8 +204,9 @@ def reference_matmul_mul_head2(X, return Out -def generate_compatible_shapes_mul_head2(dim_X, dim_Y, transpose_X, - transpose_Y): +def generate_compatible_shapes_mul_head2( + dim_X, dim_Y, transpose_X, transpose_Y +): BATCH_SIZE = 2 # Assume head number H is 4. We need make sure K1/H = M2 M1 = 3 @@ -227,38 +233,36 @@ def generate_compatible_shapes_mul_head2(dim_X, dim_Y, transpose_X, # Generator for multiple head, case 2 when width of X is not same as height of Y class GeneratorMulHead2(object): - def setUp(self): self.op_type = "matmul" X = np.zeros(self.shape_X) Y = np.zeros(self.shape_Y) if len(self.shape_X) == 2: - X = np.arange(0, - self.shape_X[-1] * self.shape_X[-2], - dtype=np.float32).reshape(self.shape_X) - Y = np.arange(0, - self.shape_Y[-1] * self.shape_Y[-2], - dtype=np.float32).reshape(self.shape_Y) + X = np.arange( + 0, self.shape_X[-1] * self.shape_X[-2], dtype=np.float32 + ).reshape(self.shape_X) + Y = np.arange( + 0, self.shape_Y[-1] * self.shape_Y[-2], dtype=np.float32 + ).reshape(self.shape_Y) else: for i in range(0, len(self.shape_X) - 1): - X[i, :, :] = np.arange(0, - self.shape_X[-1] * self.shape_X[-2], - dtype=np.float32).reshape( - list(self.shape_X)[-2:]) - Y[i, :, :] = np.arange(0, - self.shape_Y[-1] * self.shape_Y[-2], - dtype=np.float32).reshape( - list(self.shape_Y)[-2:]) - - Out = reference_matmul_mul_head2(X, Y, 4, self.transpose_X, - self.transpose_Y) + X[i, :, :] = np.arange( + 0, self.shape_X[-1] * self.shape_X[-2], dtype=np.float32 + ).reshape(list(self.shape_X)[-2:]) + Y[i, :, :] = np.arange( + 0, self.shape_Y[-1] * self.shape_Y[-2], dtype=np.float32 + ).reshape(list(self.shape_Y)[-2:]) + + Out = reference_matmul_mul_head2( + X, Y, 4, self.transpose_X, self.transpose_Y + ) self.inputs = {'X': X, 'Y': Y} self.attrs = { 'transpose_X': self.transpose_X, 'transpose_Y': self.transpose_Y, - 'head_number': self.head_number + 'head_number': self.head_number, } self.outputs = {'Out': Out} @@ -269,26 +273,32 @@ def test_check_output(self): def inject_test_multiple_head2(dim_x, dim_y, trans_x, trans_y, head_number): test_name = ( 'TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}_head2_{}'.format( - dim_x, dim_y, trans_x, trans_y, head_number)) + dim_x, dim_y, trans_x, trans_y, head_number + ) + ) shape_x, shape_y = generate_compatible_shapes_mul_head2( - dim_x, dim_y, trans_x, trans_y) + dim_x, dim_y, trans_x, trans_y + ) globals()[test_name] = type( - test_name, (GeneratorMulHead2, OpTest), { + test_name, + (GeneratorMulHead2, OpTest), + { 'shape_X': shape_x, 'shape_Y': shape_y, 'transpose_X': trans_x, 'transpose_Y': trans_y, - 'head_number': head_number - }) + 'head_number': head_number, + }, + ) -#test case for multiple head +# test case for multiple head for dim in (2, 3): for transose_x in (False, True): for transose_y in (False, True): inject_test_multiple_head(dim, dim, transose_x, transose_y, 4) -#test case for multiple head when X.width != Y.height +# test case for multiple head when X.width != Y.height for dim in (2, 3): for transose_x in (False, True): for transose_y in (False, True): diff --git a/python/paddle/fluid/tests/unittests/test_matmul_v2_op.py b/python/paddle/fluid/tests/unittests/test_matmul_v2_op.py index 4caffe5019f7c..9af6d6598d29a 100644 --- a/python/paddle/fluid/tests/unittests/test_matmul_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_matmul_v2_op.py @@ -29,7 +29,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): # transpose X and Y appropriately. if transpose_X: if X.ndim == 1: - X = X.reshape((X.size, )) + X = X.reshape((X.size,)) elif X.ndim == 2: X = X.T else: @@ -38,7 +38,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): X = np.transpose(X, tuple(dim)) if transpose_Y: if Y.ndim == 1: - Y = Y.reshape((Y.size, )) + Y = Y.reshape((Y.size,)) else: dim = [i for i in range(len(Y.shape))] dim[-1], dim[len(Y.shape) - 2] = dim[len(Y.shape) - 2], dim[-1] @@ -60,8 +60,8 @@ class TestMatMulV2Op(OpTest): """ def config(self): - self.x_shape = (100, ) - self.y_shape = (100, ) + self.x_shape = (100,) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -106,10 +106,9 @@ def test_check_output(self): def test_check_grad(self): if core.is_compiled_with_rocm(): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=1e-2, - check_eager=False) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=1e-2, check_eager=False + ) else: self.check_grad(['X', 'Y'], 'Out', check_eager=False) @@ -120,7 +119,7 @@ class TestMatMulOp2(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 3, 2, 100) self.trans_x = False self.trans_y = True @@ -132,7 +131,7 @@ class TestMatMulOp3(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 1, 100, 2) self.trans_x = False self.trans_y = False @@ -144,7 +143,7 @@ class TestMatMulOp4(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 2, 100, 2) self.trans_x = False self.trans_y = False @@ -157,7 +156,7 @@ class TestMatMulOp5(TestMatMulV2Op): def config(self): self.x_shape = (1, 1, 100, 1) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = True self.trans_y = False @@ -169,7 +168,7 @@ class TestMatMulOp6(TestMatMulV2Op): def config(self): self.x_shape = (1, 2, 102, 1) - self.y_shape = (102, ) + self.y_shape = (102,) self.trans_x = True self.trans_y = False @@ -181,7 +180,7 @@ class TestMatMulOp7(TestMatMulV2Op): def config(self): self.x_shape = (1, 2, 1, 100) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -288,7 +287,7 @@ class TestMatMulOp16(TestMatMulV2Op): """ def config(self): - self.x_shape = (100) + self.x_shape = 100 self.y_shape = (1, 2, 2, 100, 2) self.trans_x = False self.trans_y = False @@ -301,7 +300,7 @@ class TestMatMulOp17(TestMatMulV2Op): def config(self): self.x_shape = (2, 1, 100) - self.y_shape = (100) + self.y_shape = 100 self.trans_x = False self.trans_y = False @@ -330,15 +329,14 @@ def config(self): self.trans_y = True -#--------------------test matmul fp16-------------------- +# --------------------test matmul fp16-------------------- def create_test_fp16_class(parent, atol=0.001, max_relative_error=1.0): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestMatMulOpFp16Case(parent): - def init_kernel_type(self): self.dtype = np.float16 @@ -346,18 +344,20 @@ def test_check_output(self): if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_output_with_place(place, - atol=atol, - check_eager=False) + self.check_output_with_place( + place, atol=atol, check_eager=False + ) def test_check_grad(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): self.check_grad_with_place( - place, ['X', 'Y'], + place, + ['X', 'Y'], 'Out', max_relative_error=max_relative_error, - check_eager=False) + check_eager=False, + ) cls_name = "{0}_{1}".format(parent.__name__, "Fp16") TestMatMulOpFp16Case.__name__ = cls_name @@ -382,24 +382,25 @@ def test_check_grad(self): create_test_fp16_class(TestMatMulOp16) create_test_fp16_class(TestMatMulOp17) -#--------------------test matmul bf16-------------------- +# --------------------test matmul bf16-------------------- def create_test_bf16_class(parent, atol=0.01): - @unittest.skipIf( not core.is_compiled_with_cuda() or not core.is_bfloat16_supported(core.CUDAPlace(0)), - "core is not compiled with CUDA and not support the bfloat16") + "core is not compiled with CUDA and not support the bfloat16", + ) class TestMatMulOpBf16Case(parent): - def get_numeric_grad(self, place, check_name): scope = core.Scope() self._check_grad_helper() - op = create_op(scope, self.op_type, self.inputs, self.outputs, - self.attrs) - return get_numeric_gradient(place, scope, op, self.inputs_fp32, - check_name, ['Out']) + op = create_op( + scope, self.op_type, self.inputs, self.outputs, self.attrs + ) + return get_numeric_gradient( + place, scope, op, self.inputs_fp32, check_name, ['Out'] + ) def init_kernel_type(self): self.dtype = np.uint16 @@ -411,18 +412,24 @@ def test_check_output(self): def test_check_grad_x(self): place = core.CUDAPlace(0) numeric_grads = self.get_numeric_grad(place, 'X') - self.check_grad_with_place(place, ['X'], - 'Out', - no_grad_set=set(['Y']), - user_defined_grads=[numeric_grads]) + self.check_grad_with_place( + place, + ['X'], + 'Out', + no_grad_set=set(['Y']), + user_defined_grads=[numeric_grads], + ) def test_check_grad_y(self): place = core.CUDAPlace(0) numeric_grads = self.get_numeric_grad(place, 'Y') - self.check_grad_with_place(place, ['Y'], - 'Out', - no_grad_set=set(['X']), - user_defined_grads=[numeric_grads]) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + no_grad_set=set(['X']), + user_defined_grads=[numeric_grads], + ) def test_check_grad(self): pass @@ -452,7 +459,6 @@ def test_check_grad(self): class TestMatMulV2API(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -469,12 +475,11 @@ def check_static_result(self, place): y_np = np.random.random([3, 4]).astype("float32") exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input_x": x_np, - "input_y": y_np - }, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": x_np, "input_y": y_np}, + fetch_list=[result], + ) def test_static(self): for place in self.places: @@ -506,7 +511,8 @@ def test_compute_type_fp32(self): if core.is_float16_supported(place): with fluid.dygraph.guard(place): paddle.set_flags( - {'FLAGS_gemm_use_half_precision_compute_type': False}) + {'FLAGS_gemm_use_half_precision_compute_type': False} + ) input_x = np.random.random([2, 8, 16]).astype("float16") input_y = np.random.random([2, 16, 8]).astype("float16") for i in range(0, 16, 2): @@ -522,7 +528,8 @@ def test_compute_type_fp32(self): self.assertTrue(np.isfinite(result_np)[0, 0, 0]) np.testing.assert_array_equal(result_np, result.numpy()) paddle.set_flags( - {'FLAGS_gemm_use_half_precision_compute_type': True}) + {'FLAGS_gemm_use_half_precision_compute_type': True} + ) def test_compute_type_fp16_nan(self): if core.is_compiled_with_cuda(): @@ -530,7 +537,8 @@ def test_compute_type_fp16_nan(self): if core.is_float16_supported(place): with fluid.dygraph.guard(place): paddle.set_flags( - {'FLAGS_gemm_use_half_precision_compute_type': True}) + {'FLAGS_gemm_use_half_precision_compute_type': True} + ) input_x = np.random.random([2, 8, 16]).astype("float16") input_y = np.random.random([2, 16, 8]).astype("float16") for i in range(0, 16, 2): @@ -543,10 +551,12 @@ def test_compute_type_fp16_nan(self): result = paddle.matmul(x, y) result_np = np.matmul(input_x, input_y) self.assertFalse( - paddle.isfinite(result)[0, 0, 0]) # contains nan/inf + paddle.isfinite(result)[0, 0, 0] + ) # contains nan/inf self.assertTrue(np.isfinite(result_np)[0, 0, 0]) paddle.set_flags( - {'FLAGS_gemm_use_half_precision_compute_type': False}) + {'FLAGS_gemm_use_half_precision_compute_type': False} + ) def test_api_eager_dygraph(self): with _test_eager_guard(): @@ -555,7 +565,6 @@ def test_api_eager_dygraph(self): class TestComplexMatMulOp(OpTest): - def setUp(self): self.op_type = "matmul_v2" self.init_base_dtype() @@ -564,7 +573,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': -1, 'use_mkldnn': False} self.outputs = {'Out': self.out} @@ -573,17 +582,18 @@ def init_base_dtype(self): self.dtype = np.float64 def init_input_output(self): - self.x = np.random.random( - (10, 10)).astype(self.dtype) + 1J * np.random.random( - (10, 10)).astype(self.dtype) - self.y = np.random.random( - (10, 10)).astype(self.dtype) + 1J * np.random.random( - (10, 10)).astype(self.dtype) + self.x = np.random.random((10, 10)).astype( + self.dtype + ) + 1j * np.random.random((10, 10)).astype(self.dtype) + self.y = np.random.random((10, 10)).astype( + self.dtype + ) + 1j * np.random.random((10, 10)).astype(self.dtype) self.out = np.dot(self.x, self.y) def init_grad_input_output(self): - self.grad_out = np.ones((10, 10), self.dtype) + 1J * np.ones( - (10, 10), self.dtype) + self.grad_out = np.ones((10, 10), self.dtype) + 1j * np.ones( + (10, 10), self.dtype + ) self.grad_x = np.matmul(self.grad_out, np.conj(self.y).T) self.grad_y = np.matmul(np.conj(self.x).T, self.grad_out) @@ -591,31 +601,36 @@ def test_check_output(self): self.check_output(check_eager=False) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) class TestComplexMatMulOpBroadcast(OpTest): - def setUp(self): self.op_type = "matmul_v2" self.init_base_dtype() @@ -624,7 +639,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': -1, 'use_mkldnn': False} self.outputs = {'Out': self.out} @@ -633,61 +648,68 @@ def init_base_dtype(self): self.dtype = np.float64 def init_input_output(self): - self.x = np.random.random( - (10, 2, 5)).astype(self.dtype) + 1J * np.random.random( - (10, 2, 5)).astype(self.dtype) - self.y = np.random.random( - (5, 20)).astype(self.dtype) + 1J * np.random.random( - (5, 20)).astype(self.dtype) + self.x = np.random.random((10, 2, 5)).astype( + self.dtype + ) + 1j * np.random.random((10, 2, 5)).astype(self.dtype) + self.y = np.random.random((5, 20)).astype( + self.dtype + ) + 1j * np.random.random((5, 20)).astype(self.dtype) self.out = np.dot(self.x, self.y) def init_grad_input_output(self): - self.grad_out = np.ones((10, 2, 20), self.dtype) + 1J * np.ones( - (10, 2, 20), self.dtype) + self.grad_out = np.ones((10, 2, 20), self.dtype) + 1j * np.ones( + (10, 2, 20), self.dtype + ) self.grad_x = np.matmul(self.grad_out, np.conj(self.y).T) - self.grad_y = np.sum(np.matmul( - np.conj(self.x).transpose(0, 2, 1), self.grad_out), - axis=0) + self.grad_y = np.sum( + np.matmul(np.conj(self.x).transpose(0, 2, 1), self.grad_out), axis=0 + ) def test_check_output(self): self.check_output(check_eager=False) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[self.grad_x, self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[self.grad_x, self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - no_grad_set=set("X"), - user_defined_grads=[self.grad_y], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['Y'], + 'Out', + no_grad_set=set("X"), + user_defined_grads=[self.grad_y], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Y'), - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=False) + self.check_grad( + ['X'], + 'Out', + no_grad_set=set('Y'), + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=False, + ) class TestMatMulTypePromotion(TestComplexMatMulOp): - def init_input_output(self): self.x = np.random.random((10, 10)).astype(self.dtype) - self.y = np.random.random( - (10, 10)).astype(self.dtype) + 1J * np.random.random( - (10, 10)).astype(self.dtype) + self.y = np.random.random((10, 10)).astype( + self.dtype + ) + 1j * np.random.random((10, 10)).astype(self.dtype) self.out = np.dot(self.x, self.y) def init_grad_input_output(self): - self.grad_out = np.ones((10, 10), self.dtype) + 1J * np.ones( - (10, 10), self.dtype) + self.grad_out = np.ones((10, 10), self.dtype) + 1j * np.ones( + (10, 10), self.dtype + ) self.grad_x = np.matmul(self.grad_out, np.conj(self.y).T).real self.grad_y = np.matmul(np.conj(self.x).T, self.grad_out) diff --git a/python/paddle/fluid/tests/unittests/test_matrix_nms_op.py b/python/paddle/fluid/tests/unittests/test_matrix_nms_op.py index d0b9c3021284a..fe43d42cb45bf 100644 --- a/python/paddle/fluid/tests/unittests/test_matrix_nms_op.py +++ b/python/paddle/fluid/tests/unittests/test_matrix_nms_op.py @@ -21,22 +21,34 @@ import paddle -def python_matrix_nms(bboxes, - scores, - score_threshold, - nms_top_k, - keep_top_k, - post_threshold, - use_gaussian=False, - gaussian_sigma=2., - background_label=0, - normalized=True, - return_index=True, - return_rois_num=True): +def python_matrix_nms( + bboxes, + scores, + score_threshold, + nms_top_k, + keep_top_k, + post_threshold, + use_gaussian=False, + gaussian_sigma=2.0, + background_label=0, + normalized=True, + return_index=True, + return_rois_num=True, +): out, rois_num, index = paddle.vision.ops.matrix_nms( - bboxes, scores, score_threshold, post_threshold, nms_top_k, keep_top_k, - use_gaussian, gaussian_sigma, background_label, normalized, - return_index, return_rois_num) + bboxes, + scores, + score_threshold, + post_threshold, + nms_top_k, + keep_top_k, + use_gaussian, + gaussian_sigma, + background_label, + normalized, + return_index, + return_rois_num, + ) if not return_index: index = None if not return_rois_num: @@ -47,7 +59,7 @@ def python_matrix_nms(bboxes, def softmax(x): # clip to shiftx, otherwise, when calc loss with # log(exp(shiftx)), may get log(0)=INF - shiftx = (x - np.max(x)).clip(-64.) + shiftx = (x - np.max(x)).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) @@ -61,18 +73,20 @@ def iou_matrix(a, b, norm=True): area_i = np.prod(br_i - tl_i + pad, axis=2) * (tl_i < br_i).all(axis=2) area_a = np.prod(a[:, 2:] - a[:, :2] + pad, axis=1) area_b = np.prod(b[:, 2:] - b[:, :2] + pad, axis=1) - area_o = (area_a[:, np.newaxis] + area_b - area_i) + area_o = area_a[:, np.newaxis] + area_b - area_i return area_i / (area_o + 1e-10) -def matrix_nms(boxes, - scores, - score_threshold, - post_threshold=0., - nms_top_k=400, - normalized=True, - use_gaussian=False, - gaussian_sigma=2.): +def matrix_nms( + boxes, + scores, + score_threshold, + post_threshold=0.0, + nms_top_k=400, + normalized=True, + use_gaussian=False, + gaussian_sigma=2.0, +): all_scores = copy.deepcopy(scores) all_scores = all_scores.flatten() selected_indices = np.where(all_scores > score_threshold)[0] @@ -99,7 +113,7 @@ def matrix_nms(boxes, decay = decay.min(0) decayed_scores = sorted_scores * decay - if post_threshold > 0.: + if post_threshold > 0.0: inds = np.where(decayed_scores > post_threshold)[0] selected_boxes = selected_boxes[inds, :] decayed_scores = decayed_scores[inds] @@ -108,9 +122,18 @@ def matrix_nms(boxes, return decayed_scores, selected_boxes, sorted_indices -def multiclass_nms(boxes, scores, background, score_threshold, post_threshold, - nms_top_k, keep_top_k, normalized, use_gaussian, - gaussian_sigma): +def multiclass_nms( + boxes, + scores, + background, + score_threshold, + post_threshold, + nms_top_k, + keep_top_k, + normalized, + use_gaussian, + gaussian_sigma, +): all_boxes = [] all_cls = [] all_scores = [] @@ -119,8 +142,15 @@ def multiclass_nms(boxes, scores, background, score_threshold, post_threshold, if c == background: continue decayed_scores, selected_boxes, indices = matrix_nms( - boxes, scores[c], score_threshold, post_threshold, nms_top_k, - normalized, use_gaussian, gaussian_sigma) + boxes, + scores[c], + score_threshold, + post_threshold, + nms_top_k, + normalized, + use_gaussian, + gaussian_sigma, + ) all_cls.append(np.full(len(decayed_scores), c, decayed_scores.dtype)) all_boxes.append(selected_boxes) all_scores.append(decayed_scores) @@ -131,7 +161,8 @@ def multiclass_nms(boxes, scores, background, score_threshold, post_threshold, all_scores = np.concatenate(all_scores) all_indices = np.concatenate(all_indices) all_pred = np.concatenate( - (all_cls[:, np.newaxis], all_scores[:, np.newaxis], all_boxes), axis=1) + (all_cls[:, np.newaxis], all_scores[:, np.newaxis], all_boxes), axis=1 + ) num_det = len(all_pred) if num_det == 0: @@ -149,25 +180,35 @@ def multiclass_nms(boxes, scores, background, score_threshold, post_threshold, return all_pred, all_indices -def batched_multiclass_nms(boxes, - scores, - background, - score_threshold, - post_threshold, - nms_top_k, - keep_top_k, - normalized=True, - use_gaussian=False, - gaussian_sigma=2.): +def batched_multiclass_nms( + boxes, + scores, + background, + score_threshold, + post_threshold, + nms_top_k, + keep_top_k, + normalized=True, + use_gaussian=False, + gaussian_sigma=2.0, +): batch_size = scores.shape[0] det_outs = [] index_outs = [] lod = [] for n in range(batch_size): - nmsed_outs, indices = multiclass_nms(boxes[n], scores[n], background, - score_threshold, post_threshold, - nms_top_k, keep_top_k, normalized, - use_gaussian, gaussian_sigma) + nmsed_outs, indices = multiclass_nms( + boxes[n], + scores[n], + background, + score_threshold, + post_threshold, + nms_top_k, + keep_top_k, + normalized, + use_gaussian, + gaussian_sigma, + ) nmsed_num = len(nmsed_outs) lod.append(nmsed_num) if nmsed_num == 0: @@ -182,9 +223,8 @@ def batched_multiclass_nms(boxes, class TestMatrixNMSOp(OpTest): - def set_argument(self): - self.post_threshold = 0. + self.post_threshold = 0.0 self.use_gaussian = False def setUp(self): @@ -202,7 +242,7 @@ def setUp(self): use_gaussian = False if hasattr(self, 'use_gaussian'): use_gaussian = self.use_gaussian - gaussian_sigma = 2. + gaussian_sigma = 2.0 scores = np.random.random((N * M, C)).astype('float32') @@ -215,8 +255,17 @@ def setUp(self): boxes[:, :, 2:4] = boxes[:, :, 2:4] * 0.5 + 0.5 det_outs, index_outs, lod = batched_multiclass_nms( - boxes, scores, background, score_threshold, post_threshold, - nms_top_k, keep_top_k, True, use_gaussian, gaussian_sigma) + boxes, + scores, + background, + score_threshold, + post_threshold, + nms_top_k, + keep_top_k, + True, + use_gaussian, + gaussian_sigma, + ) empty = len(det_outs) == 0 det_outs = np.array([], dtype=np.float32) if empty else det_outs @@ -228,7 +277,7 @@ def setUp(self): self.outputs = { 'Out': nmsed_outs, 'Index': index_outs[:, None], - 'RoisNum': np.array(lod).astype('int32') + 'RoisNum': np.array(lod).astype('int32'), } self.attrs = { 'score_threshold': score_threshold, @@ -246,20 +295,17 @@ def test_check_output(self): class TestMatrixNMSOpNoOutput(TestMatrixNMSOp): - def set_argument(self): self.post_threshold = 2.0 class TestMatrixNMSOpGaussian(TestMatrixNMSOp): - def set_argument(self): - self.post_threshold = 0. + self.post_threshold = 0.0 self.use_gaussian = True class TestMatrixNMSError(unittest.TestCase): - def test_errors(self): M = 1200 N = 7 @@ -268,7 +314,7 @@ def test_errors(self): nms_top_k = 400 keep_top_k = 200 score_threshold = 0.01 - post_threshold = 0. + post_threshold = 0.0 boxes_np = np.random.random((M, C, BOX_SIZE)).astype('float32') scores = np.random.random((N * M, C)).astype('float32') @@ -277,52 +323,62 @@ def test_errors(self): scores_np = np.transpose(scores, (0, 2, 1)) with program_guard(Program(), Program()): - boxes_data = fluid.data(name='bboxes', - shape=[M, C, BOX_SIZE], - dtype='float32') - scores_data = fluid.data(name='scores', - shape=[N, C, M], - dtype='float32') + boxes_data = fluid.data( + name='bboxes', shape=[M, C, BOX_SIZE], dtype='float32' + ) + scores_data = fluid.data( + name='scores', shape=[N, C, M], dtype='float32' + ) def test_bboxes_Variable(): # the bboxes type must be Variable - fluid.layers.matrix_nms(bboxes=boxes_np, - scores=scores_data, - score_threshold=score_threshold, - post_threshold=post_threshold, - nms_top_k=nms_top_k, - keep_top_k=keep_top_k) - paddle.vision.ops.matrix_nms(bboxes=boxes_np, - scores=scores_data, - score_threshold=score_threshold, - post_threshold=post_threshold, - nms_top_k=nms_top_k, - keep_top_k=keep_top_k) + fluid.layers.matrix_nms( + bboxes=boxes_np, + scores=scores_data, + score_threshold=score_threshold, + post_threshold=post_threshold, + nms_top_k=nms_top_k, + keep_top_k=keep_top_k, + ) + paddle.vision.ops.matrix_nms( + bboxes=boxes_np, + scores=scores_data, + score_threshold=score_threshold, + post_threshold=post_threshold, + nms_top_k=nms_top_k, + keep_top_k=keep_top_k, + ) def test_scores_Variable(): # the scores type must be Variable - fluid.layers.matrix_nms(bboxes=boxes_data, - scores=scores_np, - score_threshold=score_threshold, - post_threshold=post_threshold, - nms_top_k=nms_top_k, - keep_top_k=keep_top_k) - paddle.vision.ops.matrix_nms(bboxes=boxes_data, - scores=scores_np, - score_threshold=score_threshold, - post_threshold=post_threshold, - nms_top_k=nms_top_k, - keep_top_k=keep_top_k) + fluid.layers.matrix_nms( + bboxes=boxes_data, + scores=scores_np, + score_threshold=score_threshold, + post_threshold=post_threshold, + nms_top_k=nms_top_k, + keep_top_k=keep_top_k, + ) + paddle.vision.ops.matrix_nms( + bboxes=boxes_data, + scores=scores_np, + score_threshold=score_threshold, + post_threshold=post_threshold, + nms_top_k=nms_top_k, + keep_top_k=keep_top_k, + ) def test_empty(): # when all score are lower than threshold try: - fluid.layers.matrix_nms(bboxes=boxes_data, - scores=scores_data, - score_threshold=score_threshold, - post_threshold=post_threshold, - nms_top_k=nms_top_k, - keep_top_k=keep_top_k) + fluid.layers.matrix_nms( + bboxes=boxes_data, + scores=scores_data, + score_threshold=score_threshold, + post_threshold=post_threshold, + nms_top_k=nms_top_k, + keep_top_k=keep_top_k, + ) except Exception as e: self.fail(e) try: @@ -332,19 +388,22 @@ def test_empty(): score_threshold=score_threshold, post_threshold=post_threshold, nms_top_k=nms_top_k, - keep_top_k=keep_top_k) + keep_top_k=keep_top_k, + ) except Exception as e: self.fail(e) def test_coverage(): # cover correct workflow try: - fluid.layers.matrix_nms(bboxes=boxes_data, - scores=scores_data, - score_threshold=score_threshold, - post_threshold=post_threshold, - nms_top_k=nms_top_k, - keep_top_k=keep_top_k) + fluid.layers.matrix_nms( + bboxes=boxes_data, + scores=scores_data, + score_threshold=score_threshold, + post_threshold=post_threshold, + nms_top_k=nms_top_k, + keep_top_k=keep_top_k, + ) except Exception as e: self.fail(e) try: @@ -354,7 +413,8 @@ def test_coverage(): score_threshold=score_threshold, post_threshold=post_threshold, nms_top_k=nms_top_k, - keep_top_k=keep_top_k) + keep_top_k=keep_top_k, + ) except Exception as e: self.fail(e) diff --git a/python/paddle/fluid/tests/unittests/test_matrix_power_op.py b/python/paddle/fluid/tests/unittests/test_matrix_power_op.py index af400b14615c0..0632af3a3a323 100644 --- a/python/paddle/fluid/tests/unittests/test_matrix_power_op.py +++ b/python/paddle/fluid/tests/unittests/test_matrix_power_op.py @@ -23,7 +23,6 @@ class TestMatrixPowerOp(OpTest): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -45,14 +44,12 @@ def test_check_output(self): self.check_output() def test_grad(self): - self.check_grad(["X"], - "Out", - numeric_grad_delta=1e-5, - max_relative_error=1e-7) + self.check_grad( + ["X"], "Out", numeric_grad_delta=1e-5, max_relative_error=1e-7 + ) class TestMatrixPowerOpN1(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -60,7 +57,6 @@ def config(self): class TestMatrixPowerOpN2(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -68,7 +64,6 @@ def config(self): class TestMatrixPowerOpN3(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -76,7 +71,6 @@ def config(self): class TestMatrixPowerOpN4(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -84,7 +78,6 @@ def config(self): class TestMatrixPowerOpN5(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -92,7 +85,6 @@ def config(self): class TestMatrixPowerOpN6(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -100,7 +92,6 @@ def config(self): class TestMatrixPowerOpN10(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -108,21 +99,18 @@ def config(self): class TestMatrixPowerOpNMinus(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" self.n = -1 def test_grad(self): - self.check_grad(["X"], - "Out", - numeric_grad_delta=1e-5, - max_relative_error=1e-6) + self.check_grad( + ["X"], "Out", numeric_grad_delta=1e-5, max_relative_error=1e-6 + ) class TestMatrixPowerOpNMinus2(TestMatrixPowerOpNMinus): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -130,7 +118,6 @@ def config(self): class TestMatrixPowerOpNMinus3(TestMatrixPowerOpNMinus): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -138,7 +125,6 @@ def config(self): class TestMatrixPowerOpNMinus4(TestMatrixPowerOpNMinus): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -146,7 +132,6 @@ def config(self): class TestMatrixPowerOpNMinus5(TestMatrixPowerOpNMinus): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -154,7 +139,6 @@ def config(self): class TestMatrixPowerOpNMinus6(TestMatrixPowerOpNMinus): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -162,21 +146,18 @@ def config(self): class TestMatrixPowerOpNMinus10(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" self.n = -10 def test_grad(self): - self.check_grad(["X"], - "Out", - numeric_grad_delta=1e-5, - max_relative_error=1e-6) + self.check_grad( + ["X"], "Out", numeric_grad_delta=1e-5, max_relative_error=1e-6 + ) class TestMatrixPowerOpBatched1(TestMatrixPowerOp): - def config(self): self.matrix_shape = [8, 4, 4] self.dtype = "float64" @@ -184,7 +165,6 @@ def config(self): class TestMatrixPowerOpBatched2(TestMatrixPowerOp): - def config(self): self.matrix_shape = [2, 6, 4, 4] self.dtype = "float64" @@ -192,7 +172,6 @@ def config(self): class TestMatrixPowerOpBatched3(TestMatrixPowerOp): - def config(self): self.matrix_shape = [2, 6, 4, 4] self.dtype = "float64" @@ -200,7 +179,6 @@ def config(self): class TestMatrixPowerOpBatchedLong(TestMatrixPowerOp): - def config(self): self.matrix_shape = [1, 2, 3, 4, 4, 3, 3] self.dtype = "float64" @@ -208,7 +186,6 @@ def config(self): class TestMatrixPowerOpLarge1(TestMatrixPowerOp): - def config(self): self.matrix_shape = [32, 32] self.dtype = "float64" @@ -216,7 +193,6 @@ def config(self): class TestMatrixPowerOpLarge2(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float64" @@ -224,7 +200,6 @@ def config(self): class TestMatrixPowerOpFP32(TestMatrixPowerOp): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float32" @@ -235,7 +210,6 @@ def test_grad(self): class TestMatrixPowerOpBatchedFP32(TestMatrixPowerOpFP32): - def config(self): self.matrix_shape = [2, 8, 4, 4] self.dtype = "float32" @@ -243,7 +217,6 @@ def config(self): class TestMatrixPowerOpLarge1FP32(TestMatrixPowerOpFP32): - def config(self): self.matrix_shape = [32, 32] self.dtype = "float32" @@ -251,7 +224,6 @@ def config(self): class TestMatrixPowerOpLarge2FP32(TestMatrixPowerOpFP32): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float32" @@ -259,7 +231,6 @@ def config(self): class TestMatrixPowerOpFP32Minus(TestMatrixPowerOpFP32): - def config(self): self.matrix_shape = [10, 10] self.dtype = "float32" @@ -267,7 +238,6 @@ def config(self): class TestMatrixPowerAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -282,12 +252,14 @@ def check_static_result(self, place): result_np = np.linalg.matrix_power(input_np, -2) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input_x": input_np}, - fetch_list=[result]) - np.testing.assert_allclose(fetches[0], - np.linalg.matrix_power(input_np, -2), - rtol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": input_np}, + fetch_list=[result], + ) + np.testing.assert_allclose( + fetches[0], np.linalg.matrix_power(input_np, -2), rtol=1e-05 + ) def test_static(self): for place in self.places: @@ -299,13 +271,14 @@ def test_dygraph(self): input_np = np.random.random([4, 4]).astype("float64") input = paddle.to_tensor(input_np) result = paddle.linalg.matrix_power(input, -2) - np.testing.assert_allclose(result.numpy(), - np.linalg.matrix_power(input_np, -2), - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), + np.linalg.matrix_power(input_np, -2), + rtol=1e-05, + ) class TestMatrixPowerAPIError(unittest.TestCase): - def test_errors(self): input_np = np.random.random([4, 4]).astype("float64") @@ -314,9 +287,9 @@ def test_errors(self): # n must be int for n in [2.0, '2', -2.0]: - input = fluid.data(name="input_float32", - shape=[4, 4], - dtype='float32') + input = fluid.data( + name="input_float32", shape=[4, 4], dtype='float32' + ) self.assertRaises(TypeError, paddle.linalg.matrix_power, input, n) # The data type of input must be float32 or float64. @@ -339,7 +312,6 @@ def test_errors(self): class TestMatrixPowerSingularAPI(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -354,9 +326,11 @@ def check_static_result(self, place): exe = fluid.Executor(place) try: - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) except RuntimeError as ex: print("The mat is singular") except ValueError as ex: diff --git a/python/paddle/fluid/tests/unittests/test_matrix_rank_op.py b/python/paddle/fluid/tests/unittests/test_matrix_rank_op.py index 3c3dba880047d..72c104ad85c28 100644 --- a/python/paddle/fluid/tests/unittests/test_matrix_rank_op.py +++ b/python/paddle/fluid/tests/unittests/test_matrix_rank_op.py @@ -31,7 +31,6 @@ def matrix_rank_wraper(x, tol=None, use_default_tol=True, hermitian=False): class TestMatrixRankOP(OpTest): - def setUp(self): self.python_api = matrix_rank_wraper self.op_type = "matrix_rank" @@ -58,91 +57,90 @@ def init_data(self): class TestMatrixRankOP1(TestMatrixRankOP): - def init_data(self): self.x = np.eye(3, k=1, dtype=np.float64) self.tol_tensor = None self.tol = None self.use_default_tol = True self.hermitian = False - self.out = np.linalg.matrix_rank(self.x, self.tol_tensor, - self.hermitian) + self.out = np.linalg.matrix_rank( + self.x, self.tol_tensor, self.hermitian + ) class TestMatrixRankOP2(TestMatrixRankOP): - def init_data(self): self.x = np.random.rand(3, 4, 5, 6).astype(np.float32) self.tol_tensor = np.random.random([3, 4]).astype(self.x.dtype) self.tol = None self.use_default_tol = False self.hermitian = False - self.out = np.linalg.matrix_rank(self.x, self.tol_tensor, - self.hermitian) + self.out = np.linalg.matrix_rank( + self.x, self.tol_tensor, self.hermitian + ) class TestMatrixRankOP3(TestMatrixRankOP): - def init_data(self): self.x = np.eye(200, dtype=np.float64) self.tol_tensor = None self.tol = None self.use_default_tol = True self.hermitian = True - self.out = np.linalg.matrix_rank(self.x, self.tol_tensor, - self.hermitian) + self.out = np.linalg.matrix_rank( + self.x, self.tol_tensor, self.hermitian + ) class TestMatrixRankOP4(TestMatrixRankOP): - def init_data(self): self.x = np.random.rand(1, 10).astype(np.float32) self.tol_tensor = None self.tol = None self.use_default_tol = True self.hermitian = False - self.out = np.linalg.matrix_rank(self.x, self.tol_tensor, - self.hermitian) + self.out = np.linalg.matrix_rank( + self.x, self.tol_tensor, self.hermitian + ) class TestMatrixRankOP5(TestMatrixRankOP): - def init_data(self): self.x = np.random.rand(5, 1).astype(np.float64) self.tol_tensor = np.random.random([1, 4]).astype(self.x.dtype) self.tol = None self.use_default_tol = False self.hermitian = False - self.out = np.linalg.matrix_rank(self.x, self.tol_tensor, - self.hermitian) + self.out = np.linalg.matrix_rank( + self.x, self.tol_tensor, self.hermitian + ) class TestMatrixRankOP6(TestMatrixRankOP): - def init_data(self): self.x = np.random.rand(3, 4, 5, 6).astype(np.float32) self.tol_tensor = None self.tol = None self.use_default_tol = False self.hermitian = False - self.out = np.linalg.matrix_rank(self.x, self.tol_tensor, - self.hermitian) + self.out = np.linalg.matrix_rank( + self.x, self.tol_tensor, self.hermitian + ) class TestMatrixRankOP7(TestMatrixRankOP): - def init_data(self): self.x = np.eye(200, dtype=np.float64) self.tol_tensor = np.random.random([200, 200]).astype(self.x.dtype) self.tol = None self.use_default_tol = True self.hermitian = True - self.out = np.linalg.matrix_rank(self.x, self.tol_tensor, - self.hermitian) + self.out = np.linalg.matrix_rank( + self.x, self.tol_tensor, self.hermitian + ) class TestMatrixRankAPI(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() @@ -177,51 +175,54 @@ def test_static(self): with fluid.program_guard(fluid.Program(), fluid.Program()): x_np = np.random.rand(3, 4, 7, 7).astype(np.float64) tol_np = np.random.random([3, 4]).astype(np.float32) - x_pd = paddle.fluid.data(name="X", - shape=[3, 4, 7, 7], - dtype='float64') - tol_pd = paddle.fluid.data(name="TolTensor", - shape=[3, 4], - dtype='float32') + x_pd = paddle.fluid.data( + name="X", shape=[3, 4, 7, 7], dtype='float64' + ) + tol_pd = paddle.fluid.data( + name="TolTensor", shape=[3, 4], dtype='float32' + ) rank_np = np.linalg.matrix_rank(x_np, tol_np, hermitian=False) - rank_pd = paddle.linalg.matrix_rank(x_pd, - tol_pd, - hermitian=False) + rank_pd = paddle.linalg.matrix_rank( + x_pd, tol_pd, hermitian=False + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "X": x_np, - "TolTensor": tol_np - }, - fetch_list=[rank_pd]) + fetches = exe.run( + fluid.default_main_program(), + feed={"X": x_np, "TolTensor": tol_np}, + fetch_list=[rank_pd], + ) np.testing.assert_allclose(fetches[0], rank_np, rtol=1e-05) for place in places: with fluid.program_guard(fluid.Program(), fluid.Program()): x_np = np.random.rand(3, 4, 7, 7).astype(np.float64) - x_pd = paddle.fluid.data(name="X", - shape=[3, 4, 7, 7], - dtype='float64') + x_pd = paddle.fluid.data( + name="X", shape=[3, 4, 7, 7], dtype='float64' + ) rank_np = np.linalg.matrix_rank(x_np, hermitian=True) rank_pd = paddle.linalg.matrix_rank(x_pd, hermitian=True) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"X": x_np}, - fetch_list=[rank_pd]) + fetches = exe.run( + fluid.default_main_program(), + feed={"X": x_np}, + fetch_list=[rank_pd], + ) np.testing.assert_allclose(fetches[0], rank_np, rtol=1e-05) for place in places: with fluid.program_guard(fluid.Program(), fluid.Program()): x_np = np.random.rand(3, 4, 7, 7).astype(np.float64) - x_pd = paddle.fluid.data(name="X", - shape=[3, 4, 7, 7], - dtype='float64') + x_pd = paddle.fluid.data( + name="X", shape=[3, 4, 7, 7], dtype='float64' + ) rank_np = np.linalg.matrix_rank(x_np, 0.1, hermitian=False) rank_pd = paddle.linalg.matrix_rank(x_pd, 0.1, hermitian=False) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"X": x_np}, - fetch_list=[rank_pd]) + fetches = exe.run( + fluid.default_main_program(), + feed={"X": x_np}, + fetch_list=[rank_pd], + ) np.testing.assert_allclose(fetches[0], rank_np, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_max_min_amax_amin_op.py b/python/paddle/fluid/tests/unittests/test_max_min_amax_amin_op.py index ee316bb0431e3..e323d91ba5611 100644 --- a/python/paddle/fluid/tests/unittests/test_max_min_amax_amin_op.py +++ b/python/paddle/fluid/tests/unittests/test_max_min_amax_amin_op.py @@ -22,12 +22,14 @@ class TestMaxMinAmaxAminAPI(unittest.TestCase): - def setUp(self): self.init_case() self.cal_np_out_and_gradient() - self.place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def init_case(self): self.x_np = np.array([[0.2, 0.3, 0.5, 0.9], [0.1, 0.2, 0.6, 0.7]]) @@ -40,7 +42,6 @@ def init_case(self): # its gradient check is not supported by unittest framework, # thus we calculate the gradient by numpy function. def cal_np_out_and_gradient(self): - def _cal_np_out_and_gradient(func): if func == 'amax': out = np.amax(self.x_np, axis=self.axis, keepdims=self.keepdim) @@ -51,8 +52,10 @@ def _cal_np_out_and_gradient(func): elif func == 'min': out = np.min(self.x_np, axis=self.axis, keepdims=self.keepdim) else: - print('This unittest only test amax/amin/max/min, but now is', - func) + print( + 'This unittest only test amax/amin/max/min, but now is', + func, + ) self.np_out[func] = out grad = np.zeros(self.shape) out_b = np.broadcast_to(out.view(), self.shape) @@ -86,7 +89,6 @@ def _choose_paddle_func(self, func, x): # We check the output between paddle API and numpy in static graph. def test_static_graph(self): - def _test_static_graph(func): startup_program = fluid.Program() train_program = fluid.Program() @@ -96,9 +98,11 @@ def _test_static_graph(func): out = self._choose_paddle_func(func, x) exe = fluid.Executor(self.place) - res = exe.run(fluid.default_main_program(), - feed={'input': self.x_np}, - fetch_list=[out]) + res = exe.run( + fluid.default_main_program(), + feed={'input': self.x_np}, + fetch_list=[out], + ) self.assertTrue((np.array(res[0]) == self.np_out[func]).all()) _test_static_graph('amax') @@ -109,19 +113,18 @@ def _test_static_graph(func): # As dygraph is easy to compute gradient, we check the gradient between # paddle API and numpy in dygraph. def test_dygraph(self): - def _test_dygraph(func): paddle.disable_static() - x = paddle.to_tensor(self.x_np, - dtype=self.dtype, - stop_gradient=False) + x = paddle.to_tensor( + self.x_np, dtype=self.dtype, stop_gradient=False + ) out = self._choose_paddle_func(func, x) grad_tensor = paddle.ones_like(x) paddle.autograd.backward([out], [grad_tensor], True) - np.testing.assert_allclose(self.np_out[func], - out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + self.np_out[func], out.numpy(), rtol=1e-05 + ) np.testing.assert_allclose(self.np_grad[func], x.grad, rtol=1e-05) paddle.enable_static() @@ -130,10 +133,10 @@ def _test_dygraph(func): _test_dygraph('max') _test_dygraph('min') - # test two minimum or maximum elements -class TestMaxMinAmaxAminAPI2(TestMaxMinAmaxAminAPI): + +class TestMaxMinAmaxAminAPI2(TestMaxMinAmaxAminAPI): def init_case(self): self.x_np = np.array([[0.2, 0.3, 0.9, 0.9], [0.1, 0.1, 0.6, 0.7]]) self.shape = [2, 4] @@ -144,7 +147,6 @@ def init_case(self): # test different axis class TestMaxMinAmaxAminAPI3(TestMaxMinAmaxAminAPI): - def init_case(self): self.x_np = np.array([[0.2, 0.3, 0.9, 0.9], [0.1, 0.1, 0.6, 0.7]]) self.shape = [2, 4] @@ -155,7 +157,6 @@ def init_case(self): # test keepdim = True class TestMaxMinAmaxAminAPI4(TestMaxMinAmaxAminAPI): - def init_case(self): self.x_np = np.array([[0.2, 0.3, 0.9, 0.9], [0.1, 0.1, 0.6, 0.7]]) self.shape = [2, 4] @@ -166,10 +167,10 @@ def init_case(self): # test axis is tuple class TestMaxMinAmaxAminAPI5(TestMaxMinAmaxAminAPI): - def init_case(self): - self.x_np = np.array([[[1, 2], [3, 4]], [[5, 6], [7, - 8]]]).astype(np.int32) + self.x_np = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]).astype( + np.int32 + ) self.shape = [2, 2, 2] self.dtype = 'int32' self.axis = (0, 1) @@ -178,7 +179,6 @@ def init_case(self): # test multiple minimum or maximum elements class TestMaxMinAmaxAminAPI6(TestMaxMinAmaxAminAPI): - def init_case(self): self.x_np = np.array([[0.2, 0.9, 0.9, 0.9], [0.9, 0.9, 0.2, 0.2]]) self.shape = [2, 4] diff --git a/python/paddle/fluid/tests/unittests/test_max_op.py b/python/paddle/fluid/tests/unittests/test_max_op.py index cf52cdb8d4789..181343bd5f66c 100644 --- a/python/paddle/fluid/tests/unittests/test_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_max_op.py @@ -22,7 +22,6 @@ class ApiMaxTest(unittest.TestCase): - def setUp(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -31,39 +30,43 @@ def setUp(self): def test_api(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.static.data("data", shape=[10, 10], dtype="float32") result_max = paddle.max(x=data, axis=1) exe = paddle.static.Executor(self.place) input_data = np.random.rand(10, 10).astype(np.float32) - res, = exe.run(feed={"data": input_data}, fetch_list=[result_max]) + (res,) = exe.run(feed={"data": input_data}, fetch_list=[result_max]) self.assertEqual((res == np.max(input_data, axis=1)).all(), True) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.static.data("data", shape=[10, 10], dtype="int64") result_max = paddle.max(x=data, axis=0) exe = paddle.static.Executor(self.place) input_data = np.random.randint(10, size=(10, 10)).astype(np.int64) - res, = exe.run(feed={"data": input_data}, fetch_list=[result_max]) + (res,) = exe.run(feed={"data": input_data}, fetch_list=[result_max]) self.assertEqual((res == np.max(input_data, axis=0)).all(), True) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.static.data("data", shape=[10, 10], dtype="int64") result_max = paddle.max(x=data, axis=(0, 1)) exe = paddle.static.Executor(self.place) input_data = np.random.randint(10, size=(10, 10)).astype(np.int64) - res, = exe.run(feed={"data": input_data}, fetch_list=[result_max]) + (res,) = exe.run(feed={"data": input_data}, fetch_list=[result_max]) self.assertEqual((res == np.max(input_data, axis=(0, 1))).all(), True) def test_errors(self): paddle.enable_static() def test_input_type(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = np.random.rand(10, 10) result_max = paddle.max(x=data, axis=0) @@ -105,17 +108,17 @@ def test_all_negative_axis(self): class TestOutDtype(unittest.TestCase): - def test_max(self): api_fn = paddle.max shape = [10, 16] - check_out_dtype(api_fn, - in_specs=[(shape, )], - expect_dtypes=['float32', 'float64', 'int32', 'int64']) + check_out_dtype( + api_fn, + in_specs=[(shape,)], + expect_dtypes=['float32', 'float64', 'int32', 'int64'], + ) class TestMaxWithTensorAxis1(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.max self.np_api = np.max @@ -125,7 +128,6 @@ def init_data(self): class TestMaxWithTensorAxis2(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.max self.np_api = np.max @@ -134,7 +136,7 @@ def init_data(self): self.tensor_axis = [ 0, paddle.to_tensor([1], 'int64'), - paddle.to_tensor([2], 'int64') + paddle.to_tensor([2], 'int64'), ] diff --git a/python/paddle/fluid/tests/unittests/test_maximum_op.py b/python/paddle/fluid/tests/unittests/test_maximum_op.py index 98c7aaa262f0e..0e5ce1c9ff992 100644 --- a/python/paddle/fluid/tests/unittests/test_maximum_op.py +++ b/python/paddle/fluid/tests/unittests/test_maximum_op.py @@ -19,7 +19,6 @@ class ApiMaximumTest(unittest.TestCase): - def setUp(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -40,56 +39,56 @@ def setUp(self): def test_static_api(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_y = paddle.static.data("y", shape=[10, 15], dtype="float32") result_max = paddle.maximum(data_x, data_y) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "y": self.input_y - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"x": self.input_x, "y": self.input_y}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected1, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_z = paddle.static.data("z", shape=[15], dtype="float32") result_max = paddle.maximum(data_x, data_z) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "z": self.input_z - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"x": self.input_x, "z": self.input_z}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected2, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_a = paddle.static.data("a", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_max = paddle.maximum(data_a, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "a": self.input_a, - "c": self.input_c - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"a": self.input_a, "c": self.input_c}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected3, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_b = paddle.static.data("b", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_max = paddle.maximum(data_b, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "b": self.input_b, - "c": self.input_c - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"b": self.input_b, "c": self.input_c}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected4, rtol=1e-05) def test_dynamic_api(self): diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py index 019d49072c41f..9334c37f94bd4 100644 --- a/python/paddle/fluid/tests/unittests/test_maxout_op.py +++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py @@ -28,14 +28,15 @@ def maxout_forward_naive(x, groups, channel_axis): s0, s1, s2, s3 = x.shape if channel_axis == 1: - return np.ndarray([s0, s1 // groups, groups, s2, s3], \ - buffer = x, dtype=x.dtype).max(axis=2) - return np.ndarray([s0, s1, s2, s3 // groups, groups], \ - buffer = x, dtype=x.dtype).max(axis=4) + return np.ndarray( + [s0, s1 // groups, groups, s2, s3], buffer=x, dtype=x.dtype + ).max(axis=2) + return np.ndarray( + [s0, s1, s2, s3 // groups, groups], buffer=x, dtype=x.dtype + ).max(axis=4) class TestMaxOutOp(OpTest): - def setUp(self): self.op_type = "maxout" self.python_api = paddle.nn.functional.maxout @@ -63,25 +64,21 @@ def test_check_grad(self): class TestMaxOutOpAxis0(TestMaxOutOp): - def set_attrs(self): self.axis = -1 class TestMaxOutOpAxis1(TestMaxOutOp): - def set_attrs(self): self.axis = 3 class TestMaxOutOpFP32(TestMaxOutOp): - def set_attrs(self): self.dtype = 'float32' class TestMaxOutOpGroups(TestMaxOutOp): - def set_attrs(self): self.groups = 3 @@ -92,8 +89,11 @@ def setUp(self): self.x_np = np.random.uniform(-1, 1, [2, 6, 5, 4]).astype(np.float64) self.groups = 2 self.axis = 1 - self.place=paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): with paddle.static.program_guard(paddle.static.Program()): @@ -142,9 +142,9 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.maxout, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[2, 4, 6, 8], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[2, 4, 6, 8], dtype='int32' + ) self.assertRaises(TypeError, F.maxout, x_int32) x_float32 = paddle.fluid.data(name='x_float32', shape=[2, 4, 6, 8]) diff --git a/python/paddle/fluid/tests/unittests/test_mean_iou.py b/python/paddle/fluid/tests/unittests/test_mean_iou.py index d55d1507f963e..3c78395755fb8 100644 --- a/python/paddle/fluid/tests/unittests/test_mean_iou.py +++ b/python/paddle/fluid/tests/unittests/test_mean_iou.py @@ -18,8 +18,9 @@ import paddle.fluid as fluid -def compute_mean_iou(predictions, labels, num_classes, in_wrongs, in_corrects, - in_mean_ious): +def compute_mean_iou( + predictions, labels, num_classes, in_wrongs, in_corrects, in_mean_ious +): assert predictions.shape == labels.shape predictions = predictions.flatten() labels = labels.flatten() @@ -40,8 +41,9 @@ def compute_mean_iou(predictions, labels, num_classes, in_wrongs, in_corrects, denominator = out_wrong + out_correct valid_count = (denominator != 0).sum() - denominator = np.where(denominator > 0, denominator, - np.ones(denominator.shape)) + denominator = np.where( + denominator > 0, denominator, np.ones(denominator.shape) + ) mean_iou = (out_correct / denominator).sum() / valid_count for _, in_mean_iou in in_mean_ious: @@ -50,48 +52,67 @@ def compute_mean_iou(predictions, labels, num_classes, in_wrongs, in_corrects, class TestMeanIOUOp(OpTest): - def setUp(self): self.config() self.op_type = "mean_iou" - predictions = np.random.randint(0, self.num_classes, - self.image_size).astype("int32") - labels = np.random.randint(0, self.num_classes, - self.image_size).astype("int32") + predictions = np.random.randint( + 0, self.num_classes, self.image_size + ).astype("int32") + labels = np.random.randint(0, self.num_classes, self.image_size).astype( + "int32" + ) in_wrongs = [] for i in range(self.in_wrong_num): in_wrongs.append( - ("in_wrong_%d" % i, - np.random.randint(0, 10, [self.num_classes]).astype("int32"))) + ( + "in_wrong_%d" % i, + np.random.randint(0, 10, [self.num_classes]).astype( + "int32" + ), + ) + ) in_corrects = [] for i in range(self.in_correct_num): in_corrects.append( - ("in_correct_%d" % i, - np.random.randint(0, 10, [self.num_classes]).astype("int32"))) + ( + "in_correct_%d" % i, + np.random.randint(0, 10, [self.num_classes]).astype( + "int32" + ), + ) + ) in_mean_ious = [] for i in range(self.in_mean_iou_num): - in_mean_ious.append(("in_mean_iou_%d" % i, - np.random.uniform(0, 1, - [1]).astype("float32"))) + in_mean_ious.append( + ( + "in_mean_iou_%d" % i, + np.random.uniform(0, 1, [1]).astype("float32"), + ) + ) self.inputs = { 'Predictions': predictions, 'Labels': labels, 'InWrongs': in_wrongs, 'InCorrects': in_corrects, - 'InMeanIou': in_mean_ious + 'InMeanIou': in_mean_ious, } self.attrs = {'num_classes': int(self.num_classes)} mean_iou, out_wrong, out_correct = compute_mean_iou( - predictions, labels, self.num_classes, in_wrongs, in_corrects, - in_mean_ious) + predictions, + labels, + self.num_classes, + in_wrongs, + in_corrects, + in_mean_ious, + ) self.outputs = { 'OutMeanIou': mean_iou, 'OutWrong': out_wrong, - 'OutCorrect': out_correct + 'OutCorrect': out_correct, } def config(self): @@ -106,7 +127,6 @@ def test_check_output(self): class TestCase1(TestMeanIOUOp): - def config(self): self.num_classes = 5 self.image_size = [100, 128] @@ -121,14 +141,15 @@ def test_check_output(self): class TestMeanIOUOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): # The input type of accuracy_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.mean_iou, x1, y1) # The input dtype of accuracy_op must be float32 or float64. x2 = fluid.layers.data(name='x2', shape=[4], dtype="float32") diff --git a/python/paddle/fluid/tests/unittests/test_mean_op.py b/python/paddle/fluid/tests/unittests/test_mean_op.py index 6ba980879a6c9..ed9313b054696 100644 --- a/python/paddle/fluid/tests/unittests/test_mean_op.py +++ b/python/paddle/fluid/tests/unittests/test_mean_op.py @@ -40,7 +40,6 @@ def reduce_mean_wrapper(x, axis=0, keepdim=False, reduce_all=False): class TestMeanOp(OpTest): - def setUp(self): self.op_type = "mean" self.python_api = paddle.mean @@ -60,27 +59,26 @@ def test_checkout_grad(self): class TestMeanOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of mean_op must be Variable. input1 = 12 self.assertRaises(TypeError, paddle.mean, input1) # The input dtype of mean_op must be float16, float32, float64. - input2 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") + input2 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) self.assertRaises(TypeError, paddle.mean, input2) - input3 = fluid.layers.data(name='input3', - shape=[4], - dtype="float16") + input3 = fluid.layers.data( + name='input3', shape=[4], dtype="float16" + ) fluid.layers.softmax(input3) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16MeanOp(TestMeanOp): - def init_dtype_type(self): self.dtype = np.float16 self.__class__.no_need_check_grad = True @@ -100,13 +98,13 @@ def test_checkout_grad(self): y = paddle.mean(x) dx = paddle.grad(y, x)[0].numpy() dx_expected = self.dtype(1.0 / np.prod(x_np.shape)) * np.ones( - x_np.shape).astype(self.dtype) + x_np.shape + ).astype(self.dtype) np.testing.assert_array_equal(dx, dx_expected) @OpTestTool.skip_if_not_cpu_bf16() class TestBF16MeanOp(TestMeanOp): - def init_dtype_type(self): self.dtype = np.uint16 @@ -136,7 +134,6 @@ def ref_reduce_mean_grad(x, axis, dtype): class TestReduceMeanOp(OpTest): - def setUp(self): self.op_type = 'reduce_mean' self.python_api = reduce_mean_wrapper @@ -157,7 +154,7 @@ def setUp(self): self.attrs = { 'dim': self.axis, 'keep_dim': self.keepdim, - 'reduce_all': self.reduce_all + 'reduce_all': self.reduce_all, } if self.dtype == 'float16': @@ -187,18 +184,17 @@ def test_check_grad(self): return with fluid.dygraph.guard(place=place): x = paddle.tensor(self.inputs['X']) - y = paddle.mean(x, - axis=self.attrs['dim'], - keepdim=self.attrs['keep_dim']) + y = paddle.mean( + x, axis=self.attrs['dim'], keepdim=self.attrs['keep_dim'] + ) dx = paddle.grad(y, x)[0].numpy() - dx_expected = ref_reduce_mean_grad(self.inputs['X'], - self.attrs['dim'], - self.dtype) + dx_expected = ref_reduce_mean_grad( + self.inputs['X'], self.attrs['dim'], self.dtype + ) np.testing.assert_array_equal(dx, dx_expected) class TestReduceMeanOpDefaultAttrs(TestReduceMeanOp): - def setUp(self): self.op_type = 'reduce_mean' self.python_api = reduce_mean_wrapper @@ -212,104 +208,88 @@ def setUp(self): class TestReduceMeanOpFloat32(TestReduceMeanOp): - def set_attrs(self): self.dtype = 'float32' class TestReduceMeanOpFloat16(TestReduceMeanOp): - def set_attrs(self): self.dtype = 'float16' class TestReduceMeanOpShape1D(TestReduceMeanOp): - def set_attrs(self): self.shape = [100] class TestReduceMeanOpShape1DFP16(TestReduceMeanOp): - def set_attrs(self): self.shape = [100] self.dtype = 'float16' class TestReduceMeanOpShape6D(TestReduceMeanOp): - def set_attrs(self): self.shape = [2, 3, 4, 5, 6, 7] class TestReduceMeanOpShape6DFP16(TestReduceMeanOp): - def set_attrs(self): self.shape = [2, 3, 4, 5, 6, 7] self.dtype = 'float16' class TestReduceMeanOpAxisAll(TestReduceMeanOp): - def set_attrs(self): self.axis = [0, 1, 2, 3] class TestReduceMeanOpAxisAllFP16(TestReduceMeanOp): - def set_attrs(self): self.axis = [0, 1, 2, 3] self.dtype = 'float16' class TestReduceMeanOpAxisTuple(TestReduceMeanOp): - def set_attrs(self): self.axis = (0, 1, 2) class TestReduceMeanOpAxisTupleFP16(TestReduceMeanOp): - def set_attrs(self): self.axis = (0, 1, 2) self.dtype = 'float16' class TestReduceMeanOpAxisNegative(TestReduceMeanOp): - def set_attrs(self): self.axis = [-2, -1] class TestReduceMeanOpAxisNegativeFP16(TestReduceMeanOp): - def set_attrs(self): self.axis = [-2, -1] self.dtype = 'float16' class TestReduceMeanOpKeepdimTrue1(TestReduceMeanOp): - def set_attrs(self): self.keepdim = True class TestReduceMeanOpKeepdimTrue1FP16(TestReduceMeanOp): - def set_attrs(self): self.keepdim = True self.dtype = 'float16' class TestReduceMeanOpKeepdimTrue2(TestReduceMeanOp): - def set_attrs(self): self.axis = [0, 1, 2, 3] self.keepdim = True class TestReduceMeanOpKeepdimTrue2FP16(TestReduceMeanOp): - def set_attrs(self): self.axis = [0, 1, 2, 3] self.keepdim = True @@ -317,13 +297,11 @@ def set_attrs(self): class TestReduceMeanOpReduceAllTrue(TestReduceMeanOp): - def set_attrs(self): self.reduce_all = True class TestReduceMeanOpReduceAllTrueFP16(TestReduceMeanOp): - def set_attrs(self): self.reduce_all = True self.dtype = 'float16' @@ -335,8 +313,11 @@ class TestMeanAPI(unittest.TestCase): def setUp(self): self.x_shape = [2, 3, 4, 5] self.x = np.random.uniform(-1, 1, self.x_shape).astype(np.float32) - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_api_static(self): paddle.enable_static() @@ -350,8 +331,9 @@ def test_api_static(self): out5 = paddle.mean(x, tuple(axis)) exe = paddle.static.Executor(self.place) - res = exe.run(feed={'X': self.x}, - fetch_list=[out1, out2, out3, out4, out5]) + res = exe.run( + feed={'X': self.x}, fetch_list=[out1, out2, out3, out4, out5] + ) out_ref = np.mean(self.x) for out in res: np.testing.assert_allclose(out, out_ref, rtol=0.0001) @@ -393,9 +375,9 @@ def test_fluid_api(self): x_np = np.random.rand(10, 10).astype(np.float32) x = fluid.dygraph.to_variable(x_np) out = fluid.layers.reduce_mean(input=x, dim=1) - np.testing.assert_allclose(out.numpy(), - np.mean(x_np, axis=1), - rtol=1e-05) + np.testing.assert_allclose( + out.numpy(), np.mean(x_np, axis=1), rtol=1e-05 + ) def test_errors(self): paddle.disable_static() @@ -410,7 +392,6 @@ def test_errors(self): class TestMeanWithTensorAxis1(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.mean self.np_api = np.mean @@ -420,7 +401,6 @@ def init_data(self): class TestMeanWithTensorAxis2(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.mean self.np_api = np.mean @@ -429,12 +409,11 @@ def init_data(self): self.tensor_axis = [ 0, paddle.to_tensor([1], 'int64'), - paddle.to_tensor([2], 'int64') + paddle.to_tensor([2], 'int64'), ] class TestMeanDoubleGradCheck(unittest.TestCase): - def mean_wrapper(self, x): return paddle.mean(x[0]) @@ -449,17 +428,13 @@ def func(self, place): out = paddle.mean(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.mean_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.mean_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -471,7 +446,6 @@ def test_grad(self): class TestMeanTripleGradCheck(unittest.TestCase): - def mean_wrapper(self, x): return paddle.mean(x[0]) @@ -486,17 +460,13 @@ def func(self, place): out = paddle.mean(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.mean_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.mean_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_median.py b/python/paddle/fluid/tests/unittests/test_median.py index 8f33e4713d270..c717293b4cd87 100644 --- a/python/paddle/fluid/tests/unittests/test_median.py +++ b/python/paddle/fluid/tests/unittests/test_median.py @@ -21,7 +21,6 @@ class TestMedian(unittest.TestCase): - def check_numpy_res(self, np1, np2): self.assertEqual(np1.shape, np2.shape) mismatch = np.sum((np1 - np2) * (np1 - np2)) @@ -56,8 +55,11 @@ def test_median_static(self): w = 4 l = 2 x = np.arange(h * w * l).reshape([h, w, l]) - lis_tests = [[x, axis, keepdims] for axis in [-1, 0, 1, 2, None] - for keepdims in [False, True]] + lis_tests = [ + [x, axis, keepdims] + for axis in [-1, 0, 1, 2, None] + for keepdims in [False, True] + ] for lis_test in lis_tests: self.static_single_test_median(lis_test) @@ -67,8 +69,11 @@ def test_median_dygraph(self): w = 4 l = 2 x = np.arange(h * w * l).reshape([h, w, l]) - lis_tests = [[x, axis, keepdims] for axis in [-1, 0, 1, 2, None] - for keepdims in [False, True]] + lis_tests = [ + [x, axis, keepdims] + for axis in [-1, 0, 1, 2, None] + for keepdims in [False, True] + ] for lis_test in lis_tests: self.dygraph_single_test_median(lis_test) diff --git a/python/paddle/fluid/tests/unittests/test_memcpy_op.py b/python/paddle/fluid/tests/unittests/test_memcpy_op.py index 8ab07ad9583ef..65a246d963624 100755 --- a/python/paddle/fluid/tests/unittests/test_memcpy_op.py +++ b/python/paddle/fluid/tests/unittests/test_memcpy_op.py @@ -21,7 +21,6 @@ class TestMemcpy_FillConstant(unittest.TestCase): - def get_prog(self): paddle.enable_static() main_program = Program() @@ -33,20 +32,25 @@ def get_prog(self): shape=[10, 10], dtype='float32', persistable=False, - stop_gradient=True) - gpu_var = main_program.global_block().create_var(name=gpu_var_name, - shape=[10, 10], - dtype='float32', - persistable=False, - stop_gradient=True) - main_program.global_block().append_op(type="fill_constant", - outputs={"Out": gpu_var_name}, - attrs={ - "shape": [10, 10], - "dtype": gpu_var.dtype, - "value": 1.0, - "place_type": 1 - }) + stop_gradient=True, + ) + gpu_var = main_program.global_block().create_var( + name=gpu_var_name, + shape=[10, 10], + dtype='float32', + persistable=False, + stop_gradient=True, + ) + main_program.global_block().append_op( + type="fill_constant", + outputs={"Out": gpu_var_name}, + attrs={ + "shape": [10, 10], + "dtype": gpu_var.dtype, + "value": 1.0, + "place_type": 1, + }, + ) main_program.global_block().append_op( type="fill_constant", outputs={"Out": pinned_var_name}, @@ -54,35 +58,40 @@ def get_prog(self): "shape": [10, 10], "dtype": gpu_var.dtype, "value": 0.0, - "place_type": 2 - }) + "place_type": 2, + }, + ) return main_program, gpu_var, pinned_var def test_gpu_copy_to_pinned(self): main_program, gpu_var, pinned_var = self.get_prog() - main_program.global_block().append_op(type='memcpy', - inputs={'X': gpu_var}, - outputs={'Out': pinned_var}, - attrs={'dst_place_type': 2}) + main_program.global_block().append_op( + type='memcpy', + inputs={'X': gpu_var}, + outputs={'Out': pinned_var}, + attrs={'dst_place_type': 2}, + ) place = fluid.CUDAPlace(0) exe = fluid.Executor(place) - gpu_, pinned_ = exe.run(main_program, - feed={}, - fetch_list=[gpu_var.name, pinned_var.name]) + gpu_, pinned_ = exe.run( + main_program, feed={}, fetch_list=[gpu_var.name, pinned_var.name] + ) np.testing.assert_allclose(gpu_, pinned_, rtol=1e-05) np.testing.assert_allclose(pinned_, np.ones((10, 10)), rtol=1e-05) def test_pinned_copy_gpu(self): main_program, gpu_var, pinned_var = self.get_prog() - main_program.global_block().append_op(type='memcpy', - inputs={'X': pinned_var}, - outputs={'Out': gpu_var}, - attrs={'dst_place_type': 1}) + main_program.global_block().append_op( + type='memcpy', + inputs={'X': pinned_var}, + outputs={'Out': gpu_var}, + attrs={'dst_place_type': 1}, + ) place = fluid.CUDAPlace(0) exe = fluid.Executor(place) - gpu_, pinned_ = exe.run(main_program, - feed={}, - fetch_list=[gpu_var.name, pinned_var.name]) + gpu_, pinned_ = exe.run( + main_program, feed={}, fetch_list=[gpu_var.name, pinned_var.name] + ) np.testing.assert_allclose(gpu_, pinned_, rtol=1e-05) np.testing.assert_allclose(gpu_, np.zeros((10, 10)), rtol=1e-05) @@ -98,13 +107,15 @@ def test_hip_copy_bool_value(self): shape=[1], dtype='bool', persistable=False, - stop_gradient=True) + stop_gradient=True, + ) gpu_var = main_program.global_block().create_var( name=gpu_var_name, shape=[1], dtype='bool', persistable=False, - stop_gradient=True) + stop_gradient=True, + ) main_program.global_block().append_op( type="fill_constant", outputs={"Out": gpu_var_name}, @@ -112,8 +123,9 @@ def test_hip_copy_bool_value(self): "shape": [1], "dtype": gpu_var.dtype, "value": False, - "place_type": 1 - }) + "place_type": 1, + }, + ) main_program.global_block().append_op( type="fill_constant", outputs={"Out": pinned_var_name}, @@ -121,18 +133,23 @@ def test_hip_copy_bool_value(self): "shape": [1], "dtype": gpu_var.dtype, "value": True, - "place_type": 2 - }) + "place_type": 2, + }, + ) - main_program.global_block().append_op(type='memcpy', - inputs={'X': pinned_var}, - outputs={'Out': gpu_var}, - attrs={'dst_place_type': 1}) + main_program.global_block().append_op( + type='memcpy', + inputs={'X': pinned_var}, + outputs={'Out': gpu_var}, + attrs={'dst_place_type': 1}, + ) place = fluid.CUDAPlace(0) exe = fluid.Executor(place) - gpu_, pinned_ = exe.run(main_program, - feed={}, - fetch_list=[gpu_var.name, pinned_var.name]) + gpu_, pinned_ = exe.run( + main_program, + feed={}, + fetch_list=[gpu_var.name, pinned_var.name], + ) expect_value = np.array([1]).astype('bool') np.testing.assert_array_equal(gpu_, expect_value) else: @@ -140,7 +157,6 @@ def test_hip_copy_bool_value(self): class TestMemcpyOPError(unittest.TestCase): - def get_prog(self): paddle.enable_static() main_program = Program() @@ -150,7 +166,8 @@ def get_prog(self): shape=[10, 10], dtype='float32', persistable=False, - stop_gradient=True) + stop_gradient=True, + ) main_program.global_block().append_op( type="fill_constant", outputs={"Out": "tensor@Pinned_0"}, @@ -158,41 +175,47 @@ def get_prog(self): "shape": [10, 10], "dtype": pinned_var.dtype, "value": 0.0, - "place_type": 2 - }) + "place_type": 2, + }, + ) return main_program, pinned_var def test_SELECTED_ROWS(self): main_program, pinned_var = self.get_prog() - selected_row_var = main_program.global_block().create_var( \ - name="selected_row_0", dtype="float32", persistable=False, \ - type=fluid.core.VarDesc.VarType.SELECTED_ROWS, stop_gradient=True) - main_program.global_block().append_op(type="fill_constant", - outputs={"Out": selected_row_var}, - attrs={ - "shape": - selected_row_var.shape, - "dtype": - selected_row_var.dtype, - "value": 1.0, - "place_type": 1 - }) + selected_row_var = main_program.global_block().create_var( + name="selected_row_0", + dtype="float32", + persistable=False, + type=fluid.core.VarDesc.VarType.SELECTED_ROWS, + stop_gradient=True, + ) + main_program.global_block().append_op( + type="fill_constant", + outputs={"Out": selected_row_var}, + attrs={ + "shape": selected_row_var.shape, + "dtype": selected_row_var.dtype, + "value": 1.0, + "place_type": 1, + }, + ) with self.assertRaises(RuntimeError): main_program.global_block().append_op( type='memcpy', inputs={'X': selected_row_var}, outputs={'Out': pinned_var}, - attrs={'dst_place_type': 2}) + attrs={'dst_place_type': 2}, + ) place = fluid.CUDAPlace(0) exe = fluid.Executor(place) selected_row_var_, pinned_ = exe.run( main_program, feed={}, - fetch_list=[selected_row_var.name, pinned_var.name]) + fetch_list=[selected_row_var.name, pinned_var.name], + ) class TestMemcpyApi(unittest.TestCase): - def test_api(self): a = paddle.ones([1024, 1024]) b = paddle.tensor.creation._memcpy(a, paddle.CUDAPinnedPlace()) diff --git a/python/paddle/fluid/tests/unittests/test_memory_analysis.py b/python/paddle/fluid/tests/unittests/test_memory_analysis.py index 1672e7371cbc6..8f38433d20c9b 100644 --- a/python/paddle/fluid/tests/unittests/test_memory_analysis.py +++ b/python/paddle/fluid/tests/unittests/test_memory_analysis.py @@ -14,12 +14,14 @@ import unittest import paddle -from paddle.fluid.memory_analysis import pre_allocate_memory, get_max_memory_info +from paddle.fluid.memory_analysis import ( + pre_allocate_memory, + get_max_memory_info, +) from simple_nets import simple_fc_net class TestMemoryAnalysis(unittest.TestCase): - def setUp(self): paddle.enable_static() @@ -28,18 +30,19 @@ def test_get_memory_info(self): optimizer = paddle.optimizer.Adam(learning_rate=1e-3) optimizer.minimize(loss) main_prog = paddle.static.default_main_program() - max_tmp_mem_1, max_persitable_mem_1 = get_max_memory_info(main_prog, - batch_size=32) + max_tmp_mem_1, max_persitable_mem_1 = get_max_memory_info( + main_prog, batch_size=32 + ) self.assertGreater(max_tmp_mem_1, 0) self.assertGreater(max_persitable_mem_1, 0) - max_tmp_mem_2, max_persitable_mem_2 = get_max_memory_info(main_prog, - batch_size=64) + max_tmp_mem_2, max_persitable_mem_2 = get_max_memory_info( + main_prog, batch_size=64 + ) self.assertEqual(max_persitable_mem_1, max_persitable_mem_2) self.assertLess(max_tmp_mem_1, max_tmp_mem_2) class TestPreAllocateMemory(unittest.TestCase): - def setUp(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_memory_reuse_exclude_feed_var.py b/python/paddle/fluid/tests/unittests/test_memory_reuse_exclude_feed_var.py index 7b335bf83d6ce..839c21b151a26 100644 --- a/python/paddle/fluid/tests/unittests/test_memory_reuse_exclude_feed_var.py +++ b/python/paddle/fluid/tests/unittests/test_memory_reuse_exclude_feed_var.py @@ -18,15 +18,14 @@ class TestMemoryReuseExcludeFeedVar(unittest.TestCase): - def setUp(self): self.image_shape = [28, 28] self.iteration = 10 def main_impl(self, place): - image = fluid.layers.data(name='image', - shape=self.image_shape, - dtype='float32') + image = fluid.layers.data( + name='image', shape=self.image_shape, dtype='float32' + ) relu_image = fluid.layers.relu(image) loss = fluid.layers.reduce_mean(relu_image) @@ -38,12 +37,13 @@ def main_impl(self, place): exe.run(fluid.default_startup_program()) compiled_prog = fluid.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + fluid.default_main_program() + ).with_data_parallel(loss_name=loss.name, build_strategy=build_strategy) image_tensor = fluid.LoDTensor() - np_image = np.random.uniform(low=-10, high=10, - size=self.image_shape).astype('float32') + np_image = np.random.uniform( + low=-10, high=10, size=self.image_shape + ).astype('float32') image_tensor.set(np_image, place) feed_dict = [{image.name: image_tensor}] diff --git a/python/paddle/fluid/tests/unittests/test_memory_usage.py b/python/paddle/fluid/tests/unittests/test_memory_usage.py index 2ddfd1ceb0270..8c41d0853fc64 100644 --- a/python/paddle/fluid/tests/unittests/test_memory_usage.py +++ b/python/paddle/fluid/tests/unittests/test_memory_usage.py @@ -20,8 +20,10 @@ def train_simulator(test_batch_size=10): if test_batch_size <= 0: - raise ValueError("batch_size should be a positive integeral value, " - "but got batch_size={}".format(test_batch_size)) + raise ValueError( + "batch_size should be a positive integeral value, " + "but got batch_size={}".format(test_batch_size) + ) x = fluid.layers.data(name='x', shape=[13], dtype='float32') y_predict = fluid.layers.fc(input=x, size=1, act=None) @@ -35,14 +37,16 @@ def train_simulator(test_batch_size=10): # Calculate memory usage in current network config lower_usage, upper_usage, unit = fluid.contrib.memory_usage( - fluid.default_main_program(), batch_size=test_batch_size) + fluid.default_main_program(), batch_size=test_batch_size + ) - print("memory usage is about %.3f - %.3f %s" % - (lower_usage, upper_usage, unit)) + print( + "memory usage is about %.3f - %.3f %s" + % (lower_usage, upper_usage, unit) + ) class TestMemoryUsage(unittest.TestCase): - def test_with_unit_B(self): with self.program_scope_guard(): train_simulator() diff --git a/python/paddle/fluid/tests/unittests/test_merge_selectedrows_op.py b/python/paddle/fluid/tests/unittests/test_merge_selectedrows_op.py index 65fd3b9e40512..a22571289b28e 100644 --- a/python/paddle/fluid/tests/unittests/test_merge_selectedrows_op.py +++ b/python/paddle/fluid/tests/unittests/test_merge_selectedrows_op.py @@ -19,7 +19,6 @@ class TestMergeSelectedRows(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_merged_adam_op.py b/python/paddle/fluid/tests/unittests/test_merged_adam_op.py index c6aadeda5fc15..49d1cf4bd79a0 100644 --- a/python/paddle/fluid/tests/unittests/test_merged_adam_op.py +++ b/python/paddle/fluid/tests/unittests/test_merged_adam_op.py @@ -19,20 +19,22 @@ from paddle.fluid.framework import in_dygraph_mode -def run_adam_op(params, - grads, - lrs, - moment1s, - moment2s, - beta1_pows, - beta2_pows, - master_params, - epsilon, - beta1, - beta2, - place, - multi_precision=False, - use_merged=False): +def run_adam_op( + params, + grads, + lrs, + moment1s, + moment2s, + beta1_pows, + beta2_pows, + master_params, + epsilon, + beta1, + beta2, + place, + multi_precision=False, + use_merged=False, +): assert len(params) == len(grads) assert len(params) == len(lrs) assert len(params) == len(moment1s) @@ -57,25 +59,71 @@ def run_adam_op(params, if not use_merged: for i in range(len(param_vars)): _, _, _, _, _, _ = _legacy_C_ops.adam( - param_vars[i], grad_vars[i], lr_vars[i], moment1_vars[i], - moment2_vars[i], beta1_pow_vars[i], beta2_pow_vars[i], - master_param_vars[i], param_vars[i], moment1_vars[i], - moment2_vars[i], beta1_pow_vars[i], beta2_pow_vars[i], - master_param_vars[i], 'epsilon', epsilon, 'beta1', beta1, - 'beta2', beta2, 'multi_precision', multi_precision) + param_vars[i], + grad_vars[i], + lr_vars[i], + moment1_vars[i], + moment2_vars[i], + beta1_pow_vars[i], + beta2_pow_vars[i], + master_param_vars[i], + param_vars[i], + moment1_vars[i], + moment2_vars[i], + beta1_pow_vars[i], + beta2_pow_vars[i], + master_param_vars[i], + 'epsilon', + epsilon, + 'beta1', + beta1, + 'beta2', + beta2, + 'multi_precision', + multi_precision, + ) else: if in_dygraph_mode(): _, _, _, _, _, _ = _C_ops.merged_adam_( - param_vars, grad_vars, lr_vars, moment1_vars, moment2_vars, - beta1_pow_vars, beta2_pow_vars, master_param_vars, beta1, beta2, - epsilon, multi_precision, False) + param_vars, + grad_vars, + lr_vars, + moment1_vars, + moment2_vars, + beta1_pow_vars, + beta2_pow_vars, + master_param_vars, + beta1, + beta2, + epsilon, + multi_precision, + False, + ) else: _, _, _, _, _, _ = _legacy_C_ops.merged_adam( - param_vars, grad_vars, lr_vars, moment1_vars, moment2_vars, - beta1_pow_vars, beta2_pow_vars, master_param_vars, param_vars, - moment1_vars, moment2_vars, beta1_pow_vars, beta2_pow_vars, - master_param_vars, 'epsilon', epsilon, 'beta1', beta1, 'beta2', - beta2, 'multi_precision', multi_precision) + param_vars, + grad_vars, + lr_vars, + moment1_vars, + moment2_vars, + beta1_pow_vars, + beta2_pow_vars, + master_param_vars, + param_vars, + moment1_vars, + moment2_vars, + beta1_pow_vars, + beta2_pow_vars, + master_param_vars, + 'epsilon', + epsilon, + 'beta1', + beta1, + 'beta2', + beta2, + 'multi_precision', + multi_precision, + ) outputs = { 'ParamOut': param_vars, @@ -83,14 +131,13 @@ def run_adam_op(params, 'Moment2Out': moment2_vars, 'Beta1PowOut': beta1_pow_vars, 'Beta2PowOut': beta2_pow_vars, - 'MasterParamOut': master_param_vars + 'MasterParamOut': master_param_vars, } return outputs class TestMergedAdam(unittest.TestCase): - def setUp(self): paddle.disable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -111,27 +158,46 @@ def prepare_data(self, shapes, multi_precision, seed, place): beta1_pows = self.gen_rand_data([[1], [1], [1], [1]], mp_dtype) beta2_pows = self.gen_rand_data([[1], [1], [1], [1]], mp_dtype) master_params = [p.astype(mp_dtype) for p in params] - return params, grads, lrs, moment1s, moment2s, beta1_pows, beta2_pows, master_params + return ( + params, + grads, + lrs, + moment1s, + moment2s, + beta1_pows, + beta2_pows, + master_params, + ) def check_with_place(self, place, multi_precision): - params, grads, lrs, moment1s, moment2s, beta1_pows, beta2_pows, master_params = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + lrs, + moment1s, + moment2s, + beta1_pows, + beta2_pows, + master_params, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_merged): - return run_adam_op(params=params, - grads=grads, - lrs=lrs, - moment1s=moment1s, - moment2s=moment2s, - beta1_pows=beta1_pows, - beta2_pows=beta2_pows, - master_params=master_params, - epsilon=0.9, - beta1=0.9, - beta2=0.99, - place=place, - multi_precision=multi_precision, - use_merged=use_merged) + return run_adam_op( + params=params, + grads=grads, + lrs=lrs, + moment1s=moment1s, + moment2s=moment2s, + beta1_pows=beta1_pows, + beta2_pows=beta2_pows, + master_params=master_params, + epsilon=0.9, + beta1=0.9, + beta2=0.99, + place=place, + multi_precision=multi_precision, + use_merged=use_merged, + ) outs1 = run_op(True) outs2 = run_op(False) @@ -144,10 +210,9 @@ def run_op(use_merged): if place == 'gpu': np.testing.assert_array_equal(value1[i], value2[i]) else: - np.testing.assert_allclose(value1[i], - value2[i], - rtol=1e-05, - atol=1e-07) + np.testing.assert_allclose( + value1[i], value2[i], rtol=1e-05, atol=1e-07 + ) def get_places(self): places = ['cpu'] diff --git a/python/paddle/fluid/tests/unittests/test_merged_momentum_op.py b/python/paddle/fluid/tests/unittests/test_merged_momentum_op.py index 466ee3afd58c5..dcba88284fc10 100644 --- a/python/paddle/fluid/tests/unittests/test_merged_momentum_op.py +++ b/python/paddle/fluid/tests/unittests/test_merged_momentum_op.py @@ -19,16 +19,18 @@ from collections import OrderedDict -def run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - mu=0.9, - rescale_grad=0.01, - use_merged=False): +def run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + mu=0.9, + rescale_grad=0.01, + use_merged=False, +): assert len(params) == len(grads) assert len(params) == len(velocitys) if multi_precision: @@ -45,48 +47,70 @@ def run_momentum_op(params, } param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in params ] grad_vars = [ helper.create_variable(shape=g.shape, dtype=g.dtype) for g in grads ] velocity_vars = [ - helper.create_variable(persistable=True, - shape=v.shape, - dtype=v.dtype) for v in velocitys + helper.create_variable( + persistable=True, shape=v.shape, dtype=v.dtype + ) + for v in velocitys ] - lr_var = helper.create_variable(persistable=True, - shape=learning_rate.shape, - dtype=learning_rate.dtype) + lr_var = helper.create_variable( + persistable=True, + shape=learning_rate.shape, + dtype=learning_rate.dtype, + ) feed_dict = OrderedDict() feed_dict.update( - OrderedDict([(p_var.name, p_val) - for p_var, p_val in zip(param_vars, params)])) + OrderedDict( + [ + (p_var.name, p_val) + for p_var, p_val in zip(param_vars, params) + ] + ) + ) feed_dict.update( - OrderedDict([(v_var.name, v_val) - for v_var, v_val in zip(velocity_vars, velocitys)])) + OrderedDict( + [ + (v_var.name, v_val) + for v_var, v_val in zip(velocity_vars, velocitys) + ] + ) + ) fetch_list = list(feed_dict.keys()) feed_dict.update( - OrderedDict([(g_var.name, g_val) - for g_var, g_val in zip(grad_vars, grads)])) + OrderedDict( + [(g_var.name, g_val) for g_var, g_val in zip(grad_vars, grads)] + ) + ) feed_dict.update({lr_var.name: learning_rate}) if multi_precision: master_param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in master_params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in master_params ] feed_dict.update( - OrderedDict([ - (mp_var.name, mp_val) - for mp_var, mp_val in zip(master_param_vars, master_params) - ])) + OrderedDict( + [ + (mp_var.name, mp_val) + for mp_var, mp_val in zip( + master_param_vars, master_params + ) + ] + ) + ) # CPUPlace does not use MasterParam if isinstance(place, paddle.CUDAPlace): fetch_list = fetch_list + [ @@ -96,8 +120,9 @@ def run_momentum_op(params, master_param_vars = None if not use_merged: - for i, (p, g, - v) in enumerate(zip(param_vars, grad_vars, velocity_vars)): + for i, (p, g, v) in enumerate( + zip(param_vars, grad_vars, velocity_vars) + ): inputs = { 'Param': p, 'Grad': g, @@ -108,10 +133,9 @@ def run_momentum_op(params, if multi_precision: inputs['MasterParam'] = master_param_vars[i] outputs['MasterParamOut'] = master_param_vars[i] - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) else: inputs = { 'Param': param_vars, @@ -123,10 +147,9 @@ def run_momentum_op(params, if multi_precision: inputs['MasterParam'] = master_param_vars outputs['MasterParamOut'] = master_param_vars - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): @@ -134,17 +157,19 @@ def run_momentum_op(params, return exe.run(main, feed=feed_dict, fetch_list=fetch_list) -def run_momentum_op2(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - mu=0.9, - rescale_grad=0.01, - use_merged=False, - use_nesterov=True): +def run_momentum_op2( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + mu=0.9, + rescale_grad=0.01, + use_merged=False, + use_nesterov=True, +): assert len(params) == len(grads) assert len(params) == len(velocitys) if multi_precision: @@ -156,48 +181,70 @@ def run_momentum_op2(params, helper = LayerHelper(op_type, **locals()) param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in params ] grad_vars = [ helper.create_variable(shape=g.shape, dtype=g.dtype) for g in grads ] velocity_vars = [ - helper.create_variable(persistable=True, - shape=v.shape, - dtype=v.dtype) for v in velocitys + helper.create_variable( + persistable=True, shape=v.shape, dtype=v.dtype + ) + for v in velocitys ] - lr_var = helper.create_variable(persistable=True, - shape=learning_rate.shape, - dtype=learning_rate.dtype) + lr_var = helper.create_variable( + persistable=True, + shape=learning_rate.shape, + dtype=learning_rate.dtype, + ) feed_dict = OrderedDict() feed_dict.update( - OrderedDict([(p_var.name, p_val) - for p_var, p_val in zip(param_vars, params)])) + OrderedDict( + [ + (p_var.name, p_val) + for p_var, p_val in zip(param_vars, params) + ] + ) + ) feed_dict.update( - OrderedDict([(v_var.name, v_val) - for v_var, v_val in zip(velocity_vars, velocitys)])) + OrderedDict( + [ + (v_var.name, v_val) + for v_var, v_val in zip(velocity_vars, velocitys) + ] + ) + ) fetch_list = list(feed_dict.keys()) feed_dict.update( - OrderedDict([(g_var.name, g_val) - for g_var, g_val in zip(grad_vars, grads)])) + OrderedDict( + [(g_var.name, g_val) for g_var, g_val in zip(grad_vars, grads)] + ) + ) feed_dict.update({lr_var.name: learning_rate}) if multi_precision: master_param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in master_params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in master_params ] feed_dict.update( - OrderedDict([ - (mp_var.name, mp_val) - for mp_var, mp_val in zip(master_param_vars, master_params) - ])) + OrderedDict( + [ + (mp_var.name, mp_val) + for mp_var, mp_val in zip( + master_param_vars, master_params + ) + ] + ) + ) # CPUPlace does not use MasterParam if isinstance(place, paddle.CUDAPlace): fetch_list = fetch_list + [ @@ -207,8 +254,9 @@ def run_momentum_op2(params, master_param_vars = None if not use_merged: - for i, (p, g, - v) in enumerate(zip(param_vars, grad_vars, velocity_vars)): + for i, (p, g, v) in enumerate( + zip(param_vars, grad_vars, velocity_vars) + ): inputs = { 'Param': p, 'Grad': g, @@ -227,10 +275,9 @@ def run_momentum_op2(params, 'regularization_method': 'l2_decay', 'regularization_coeff': 2.0, } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) else: inputs = { 'Param': param_vars, @@ -243,22 +290,18 @@ def run_momentum_op2(params, inputs['MasterParam'] = master_param_vars outputs['MasterParamOut'] = master_param_vars attrs = { - 'mu': - mu, - 'multi_precision': - multi_precision, - 'rescale_grad': - rescale_grad, - 'use_nesterov': - use_nesterov, - 'regularization_method': - ['l2_decay' for i in range(len(param_vars))], + 'mu': mu, + 'multi_precision': multi_precision, + 'rescale_grad': rescale_grad, + 'use_nesterov': use_nesterov, + 'regularization_method': [ + 'l2_decay' for i in range(len(param_vars)) + ], 'regularization_coeff': [2.0 for i in range(len(param_vars))], } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): @@ -267,7 +310,6 @@ def run_momentum_op2(params, class TestMergedMomentum(unittest.TestCase): - def setUp(self): paddle.enable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -280,8 +322,11 @@ def gen_rand_data(self, shapes, dtype): def prepare_data(self, shapes, multi_precision, seed, place): np.random.seed(seed) mp_dtype = np.float32 - dtype = np.float16 if multi_precision and isinstance( - place, paddle.CUDAPlace) else np.float32 + dtype = ( + np.float16 + if multi_precision and isinstance(place, paddle.CUDAPlace) + else np.float32 + ) params = self.gen_rand_data(shapes, dtype) grads = self.gen_rand_data(shapes, dtype) velocitys = self.gen_rand_data(shapes, mp_dtype) @@ -293,21 +338,28 @@ def prepare_data(self, shapes, multi_precision, seed, place): return params, grads, velocitys, master_params, learning_rate def check_with_place(self, place, multi_precision): - params, grads, velocitys, master_params, learning_rate = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + velocitys, + master_params, + learning_rate, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_merged): # FIXME(zengjinle): CPU Momentum Op does not support rescale_grad rescale_grad = 1.0 if isinstance(place, paddle.CPUPlace) else 0.01 - return run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - rescale_grad=rescale_grad, - use_merged=use_merged) + return run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + rescale_grad=rescale_grad, + use_merged=use_merged, + ) outs1 = run_op(True) outs2 = run_op(False) @@ -331,7 +383,6 @@ def test_main(self): class TestMergedMomentum2(unittest.TestCase): - def setUp(self): paddle.enable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -343,8 +394,11 @@ def gen_rand_data(self, shapes, dtype): def prepare_data(self, shapes, multi_precision, seed, place): np.random.seed(seed) mp_dtype = np.float32 - dtype = np.float16 if multi_precision and isinstance( - place, paddle.CUDAPlace) else np.float32 + dtype = ( + np.float16 + if multi_precision and isinstance(place, paddle.CUDAPlace) + else np.float32 + ) params = self.gen_rand_data(shapes, dtype) grads = self.gen_rand_data(shapes, dtype) velocitys = self.gen_rand_data(shapes, mp_dtype) @@ -356,22 +410,29 @@ def prepare_data(self, shapes, multi_precision, seed, place): return params, grads, velocitys, master_params, learning_rate def check_with_place(self, place, multi_precision): - params, grads, velocitys, master_params, learning_rate = self.prepare_data( - self.shapes, multi_precision, self.seed, place) + ( + params, + grads, + velocitys, + master_params, + learning_rate, + ) = self.prepare_data(self.shapes, multi_precision, self.seed, place) def run_op(use_nesterov, use_merged): # FIXME(zengjinle): CPU Momentum Op does not support rescale_grad rescale_grad = 1.0 if isinstance(place, paddle.CPUPlace) else 0.01 - return run_momentum_op2(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - rescale_grad=rescale_grad, - use_merged=use_merged, - use_nesterov=use_nesterov) + return run_momentum_op2( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + rescale_grad=rescale_grad, + use_merged=use_merged, + use_nesterov=use_nesterov, + ) outs1 = run_op(use_nesterov=True, use_merged=True) outs2 = run_op(use_nesterov=True, use_merged=False) diff --git a/python/paddle/fluid/tests/unittests/test_meshgrid_op.py b/python/paddle/fluid/tests/unittests/test_meshgrid_op.py index f49ac0d1d5279..3c3da16c958c4 100644 --- a/python/paddle/fluid/tests/unittests/test_meshgrid_op.py +++ b/python/paddle/fluid/tests/unittests/test_meshgrid_op.py @@ -21,7 +21,6 @@ class TestMeshgridOp(OpTest): - def setUp(self): self.op_type = "meshgrid" self.dtype = self.get_dtype() @@ -46,7 +45,7 @@ def init_test_data(self): ins = [] outs = [] for i in range(len(self.shape)): - ins.append(np.random.random((self.shape[i], )).astype(self.dtype)) + ins.append(np.random.random((self.shape[i],)).astype(self.dtype)) for i in range(len(self.shape)): out_reshape = [1] * len(self.shape) @@ -60,23 +59,29 @@ def get_x_shape(self): class TestMeshgridOp2(TestMeshgridOp): - def get_x_shape(self): return [100, 300] class TestMeshgridOp3(unittest.TestCase): - def test_api(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -85,28 +90,34 @@ def test_api(self): exe = fluid.Executor(place=fluid.CPUPlace()) grid_x, grid_y = paddle.tensor.meshgrid(x, y) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) assert np.array_equal(res_1, out_1) assert np.array_equal(res_2, out_2) class TestMeshgridOp4(unittest.TestCase): - def test_list_input(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -115,29 +126,35 @@ def test_list_input(self): exe = fluid.Executor(place=fluid.CPUPlace()) grid_x, grid_y = paddle.tensor.meshgrid([x, y]) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) assert np.array_equal(res_1, out_1) assert np.array_equal(res_2, out_2) class TestMeshgridOp5(unittest.TestCase): - def test_tuple_input(self): x = fluid.data(shape=[100], dtype='int32', name='x') y = fluid.data(shape=[200], dtype='int32', name='y') - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') out_1 = np.reshape(input_1, [100, 1]) out_1 = np.broadcast_to(out_1, [100, 200]) @@ -146,26 +163,32 @@ def test_tuple_input(self): exe = fluid.Executor(place=fluid.CPUPlace()) grid_x, grid_y = paddle.tensor.meshgrid((x, y)) - res_1, res_2 = exe.run(fluid.default_main_program(), - feed={ - 'x': input_1, - 'y': input_2 - }, - fetch_list=[grid_x, grid_y]) + res_1, res_2 = exe.run( + fluid.default_main_program(), + feed={'x': input_1, 'y': input_2}, + fetch_list=[grid_x, grid_y], + ) assert np.array_equal(res_1, out_1) assert np.array_equal(res_2, out_2) class TestMeshgridOp6(unittest.TestCase): - def test_api_with_dygraph(self): - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') with fluid.dygraph.guard(): tensor_3 = fluid.dygraph.to_variable(input_3) @@ -181,14 +204,21 @@ def test_api_eager_dygraph(self): class TestMeshgridOp7(unittest.TestCase): - def test_api_with_dygraph_list_input(self): - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') with fluid.dygraph.guard(): tensor_3 = fluid.dygraph.to_variable(input_3) @@ -204,14 +234,21 @@ def test_api_eager_dygraph(self): class TestMeshgridOp8(unittest.TestCase): - def test_api_with_dygraph_tuple_input(self): - input_3 = np.random.randint(0, 100, [ + input_3 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_4 = np.random.randint( + 0, 100, - ]).astype('int32') - input_4 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') with fluid.dygraph.guard(): tensor_3 = fluid.dygraph.to_variable(input_3) @@ -227,14 +264,21 @@ def test_api_eager_dygraph(self): class TestMeshgridEager(unittest.TestCase): - def test_dygraph_api(self): - input_1 = np.random.randint(0, 100, [ + input_1 = np.random.randint( + 0, + 100, + [ + 100, + ], + ).astype('int32') + input_2 = np.random.randint( + 0, 100, - ]).astype('int32') - input_2 = np.random.randint(0, 100, [ - 200, - ]).astype('int32') + [ + 200, + ], + ).astype('int32') with fluid.dygraph.guard(): tensor_1 = fluid.dygraph.to_variable(input_1) @@ -250,15 +294,22 @@ def test_dygraph_api(self): tensor_eager_1.stop_gradient = False tensor_eager_2.stop_gradient = False res_eager_1, res_eager_2 = paddle.tensor.meshgrid( - (tensor_eager_1, tensor_eager_2)) + (tensor_eager_1, tensor_eager_2) + ) sum_eager = paddle.add_n([res_eager_1, res_eager_2]) sum_eager.backward() - self.assertEqual(( - tensor_1.grad.numpy() == tensor_eager_1.grad.numpy()).all(), - True) - self.assertEqual(( - tensor_2.grad.numpy() == tensor_eager_2.grad.numpy()).all(), - True) + self.assertEqual( + ( + tensor_1.grad.numpy() == tensor_eager_1.grad.numpy() + ).all(), + True, + ) + self.assertEqual( + ( + tensor_2.grad.numpy() == tensor_eager_2.grad.numpy() + ).all(), + True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_min_op.py b/python/paddle/fluid/tests/unittests/test_min_op.py index 1f4cd92624138..a81b63dbb586d 100644 --- a/python/paddle/fluid/tests/unittests/test_min_op.py +++ b/python/paddle/fluid/tests/unittests/test_min_op.py @@ -22,7 +22,6 @@ class ApiMinTest(unittest.TestCase): - def setUp(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -31,39 +30,43 @@ def setUp(self): def test_api(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.static.data("data", shape=[10, 10], dtype="float32") result_min = paddle.min(x=data, axis=1) exe = paddle.static.Executor(self.place) input_data = np.random.rand(10, 10).astype(np.float32) - res, = exe.run(feed={"data": input_data}, fetch_list=[result_min]) + (res,) = exe.run(feed={"data": input_data}, fetch_list=[result_min]) self.assertEqual((res == np.min(input_data, axis=1)).all(), True) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.static.data("data", shape=[10, 10], dtype="int64") result_min = paddle.min(x=data, axis=0) exe = paddle.static.Executor(self.place) input_data = np.random.randint(10, size=(10, 10)).astype(np.int64) - res, = exe.run(feed={"data": input_data}, fetch_list=[result_min]) + (res,) = exe.run(feed={"data": input_data}, fetch_list=[result_min]) self.assertEqual((res == np.min(input_data, axis=0)).all(), True) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.static.data("data", shape=[10, 10], dtype="int64") result_min = paddle.min(x=data, axis=(0, 1)) exe = paddle.static.Executor(self.place) input_data = np.random.randint(10, size=(10, 10)).astype(np.int64) - res, = exe.run(feed={"data": input_data}, fetch_list=[result_min]) + (res,) = exe.run(feed={"data": input_data}, fetch_list=[result_min]) self.assertEqual((res == np.min(input_data, axis=(0, 1))).all(), True) def test_errors(self): paddle.enable_static() def test_input_type(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = np.random.rand(10, 10) result_min = paddle.min(x=data, axis=0) @@ -84,17 +87,17 @@ def test_eager_api(self): class TestOutDtype(unittest.TestCase): - def test_min(self): api_fn = paddle.min shape = [10, 16] - check_out_dtype(api_fn, - in_specs=[(shape, )], - expect_dtypes=['float32', 'float64', 'int32', 'int64']) + check_out_dtype( + api_fn, + in_specs=[(shape,)], + expect_dtypes=['float32', 'float64', 'int32', 'int64'], + ) class TestMinWithTensorAxis1(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.min self.np_api = np.min @@ -104,7 +107,6 @@ def init_data(self): class TestMinWithTensorAxis2(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.min self.np_api = np.min @@ -113,7 +115,7 @@ def init_data(self): self.tensor_axis = [ 0, paddle.to_tensor([1], 'int64'), - paddle.to_tensor([2], 'int64') + paddle.to_tensor([2], 'int64'), ] self.keepdim = True diff --git a/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py b/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py index 67329652726a1..7afb4860211be 100644 --- a/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py +++ b/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py @@ -18,26 +18,25 @@ class TestMineHardExamplesOp(OpTest): - def set_data(self): self.init_test_data() self.inputs = { 'ClsLoss': self.cls_loss, 'LocLoss': self.loc_loss, 'MatchIndices': self.match_indices, - 'MatchDist': self.match_dis + 'MatchDist': self.match_dis, } self.attrs = { 'neg_pos_ratio': self.neg_pos_ratio, 'neg_overlap': self.neg_overlap, 'sample_size': self.sample_size, - 'mining_type': self.mining_type + 'mining_type': self.mining_type, } self.outputs = { 'NegIndices': (self.neg_indices, self.neg_indices_lod), - 'UpdatedMatchIndices': self.updated_match_indices + 'UpdatedMatchIndices': self.updated_match_indices, } def test_check_output(self): @@ -55,17 +54,21 @@ def init_test_data(self): self.neg_overlap = 0.5 self.sample_size = 0 self.mining_type = "max_negative" - self.cls_loss = np.array([[0.1, 0.1, 0.3], [0.3, 0.1, - 0.1]]).astype('float64') + self.cls_loss = np.array([[0.1, 0.1, 0.3], [0.3, 0.1, 0.1]]).astype( + 'float64' + ) - self.loc_loss = np.array([[0.1, 0.2, 0.3], [0.3, 0.4, - 0.1]]).astype('float64') + self.loc_loss = np.array([[0.1, 0.2, 0.3], [0.3, 0.4, 0.1]]).astype( + 'float64' + ) - self.match_dis = np.array([[0.2, 0.4, 0.8], [0.1, 0.9, - 0.3]]).astype('float64') + self.match_dis = np.array([[0.2, 0.4, 0.8], [0.1, 0.9, 0.3]]).astype( + 'float64' + ) - self.match_indices = np.array([[0, -1, -1], [-1, 0, - -1]]).astype('int32') + self.match_indices = np.array([[0, -1, -1], [-1, 0, -1]]).astype( + 'int32' + ) self.updated_match_indices = self.match_indices @@ -74,23 +77,26 @@ def init_test_data(self): class TestMineHardExamplesOpHardExample(TestMineHardExamplesOp): - def init_test_data(self): super(TestMineHardExamplesOpHardExample, self).init_test_data() self.mining_type = "hard_example" self.sample_size = 2 - self.cls_loss = np.array([[0.5, 0.1, 0.3], [0.3, 0.1, - 0.1]]).astype('float64') + self.cls_loss = np.array([[0.5, 0.1, 0.3], [0.3, 0.1, 0.1]]).astype( + 'float64' + ) - self.loc_loss = np.array([[0.2, 0.2, 0.3], [0.3, 0.1, - 0.2]]).astype('float64') + self.loc_loss = np.array([[0.2, 0.2, 0.3], [0.3, 0.1, 0.2]]).astype( + 'float64' + ) - self.match_indices = np.array([[0, -1, -1], [-1, 0, - -1]]).astype('int32') + self.match_indices = np.array([[0, -1, -1], [-1, 0, -1]]).astype( + 'int32' + ) - self.updated_match_indices = np.array([[0, -1, -1], - [-1, -1, -1]]).astype('int32') + self.updated_match_indices = np.array( + [[0, -1, -1], [-1, -1, -1]] + ).astype('int32') self.neg_indices_lod = [[1, 2]] self.neg_indices = np.array([[2], [0], [2]]).astype('int32') diff --git a/python/paddle/fluid/tests/unittests/test_minimum_op.py b/python/paddle/fluid/tests/unittests/test_minimum_op.py index 4dbdfdcad4cd1..81771950c0823 100644 --- a/python/paddle/fluid/tests/unittests/test_minimum_op.py +++ b/python/paddle/fluid/tests/unittests/test_minimum_op.py @@ -19,7 +19,6 @@ class ApiMinimumTest(unittest.TestCase): - def setUp(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -40,56 +39,56 @@ def setUp(self): def test_static_api(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_y = paddle.static.data("y", shape=[10, 15], dtype="float32") result_max = paddle.minimum(data_x, data_y) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "y": self.input_y - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"x": self.input_x, "y": self.input_y}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected1, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_z = paddle.static.data("z", shape=[15], dtype="float32") result_max = paddle.minimum(data_x, data_z) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "z": self.input_z - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"x": self.input_x, "z": self.input_z}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected2, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_a = paddle.static.data("a", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_max = paddle.minimum(data_a, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "a": self.input_a, - "c": self.input_c - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"a": self.input_a, "c": self.input_c}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected3, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_b = paddle.static.data("b", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_max = paddle.minimum(data_b, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "b": self.input_b, - "c": self.input_c - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"b": self.input_b, "c": self.input_c}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected4, rtol=1e-05) def test_dynamic_api(self): diff --git a/python/paddle/fluid/tests/unittests/test_minus_op.py b/python/paddle/fluid/tests/unittests/test_minus_op.py index aaa8744061c96..40b4ddd3b579b 100644 --- a/python/paddle/fluid/tests/unittests/test_minus_op.py +++ b/python/paddle/fluid/tests/unittests/test_minus_op.py @@ -19,12 +19,11 @@ class TestMinusOp(OpTest): - def setUp(self): self.op_type = "minus" self.inputs = { 'X': np.random.random((32, 84)).astype("float32"), - 'Y': np.random.random((32, 84)).astype("float32") + 'Y': np.random.random((32, 84)).astype("float32"), } self.outputs = {'Out': (self.inputs['X'] - self.inputs['Y'])} diff --git a/python/paddle/fluid/tests/unittests/test_mix_precision_all_reduce_fuse.py b/python/paddle/fluid/tests/unittests/test_mix_precision_all_reduce_fuse.py index bc57d38b77c5a..62243890305e5 100644 --- a/python/paddle/fluid/tests/unittests/test_mix_precision_all_reduce_fuse.py +++ b/python/paddle/fluid/tests/unittests/test_mix_precision_all_reduce_fuse.py @@ -36,21 +36,25 @@ def conv_net(use_feed): img = fluid.layers.data(name='image', shape=img_shape, dtype='float16') label = fluid.layers.data(name='label', shape=[1], dtype='int64') - conv_pool_1 = fluid.nets.simple_img_conv_pool(input=img, - filter_size=5, - num_filters=20, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_1 = fluid.nets.simple_img_conv_pool( + input=img, + filter_size=5, + num_filters=20, + pool_size=2, + pool_stride=2, + act="relu", + ) conv_pool_1 = fluid.layers.batch_norm(conv_pool_1) conv_pool_1 = fluid.layers.cast(conv_pool_1, np.float32) - conv_pool_2 = fluid.nets.simple_img_conv_pool(input=conv_pool_1, - filter_size=5, - num_filters=50, - pool_size=2, - pool_stride=2, - act="relu") + conv_pool_2 = fluid.nets.simple_img_conv_pool( + input=conv_pool_1, + filter_size=5, + num_filters=50, + pool_size=2, + pool_stride=2, + act="relu", + ) hidden = fluid.layers.cast(conv_pool_2, np.float32) return loss_net(hidden, label) @@ -61,11 +65,10 @@ def _optimizer(learning_rate=1e-6): class TestResnet(TestParallelExecutorBase): - def check_model(self, use_device): - img, label = init_data(batch_size=batch_size, - img_shape=img_shape, - label_range=9) + img, label = init_data( + batch_size=batch_size, img_shape=img_shape, label_range=9 + ) img = np.float16(img) feed_dict = {"image": img, "label": label} @@ -75,7 +78,8 @@ def check_model(self, use_device): iter=10, use_device=use_device, fuse_all_reduce_ops=True, - optimizer=_optimizer) + optimizer=_optimizer, + ) def test_model(self): if core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_mode_op.py b/python/paddle/fluid/tests/unittests/test_mode_op.py index 8b2dcdf529525..bec5707ac9565 100644 --- a/python/paddle/fluid/tests/unittests/test_mode_op.py +++ b/python/paddle/fluid/tests/unittests/test_mode_op.py @@ -40,7 +40,7 @@ def cal_mode(a, axis, keepdim=False): if axis < 0: axis = len(a.shape) + axis in_dims = list(range(a.ndim)) - a_view = np.transpose(a, in_dims[:axis] + in_dims[axis + 1:] + [axis]) + a_view = np.transpose(a, in_dims[:axis] + in_dims[axis + 1 :] + [axis]) inds = np.ndindex(a_view.shape[:-1]) modes = np.empty(a_view.shape[:-1], dtype=a.dtype) indexes = np.empty(a_view.shape[:-1], dtype=np.int64) @@ -55,7 +55,6 @@ def cal_mode(a, axis, keepdim=False): class TestModeOp(OpTest): - def init_args(self): self.axis = 1 @@ -81,7 +80,6 @@ def test_check_grad(self): class TestModeOpLastdim(OpTest): - def init_args(self): self.axis = -1 @@ -107,14 +105,12 @@ def test_check_grad(self): class TestModeOpKernels(unittest.TestCase): - def setUp(self): self.axises = [-1, 1] np.random.seed(666) self.inputs = np.ceil(np.random.rand(2, 10, 10) * 1000) def test_mode_op(self): - def test_cpu_kernel(): paddle.set_device('cpu') tensor = paddle.to_tensor(self.inputs) @@ -123,9 +119,9 @@ def test_cpu_kernel(): v, inds = paddle.mode(tensor, axis) np.testing.assert_allclose(v.numpy(), value_expect, rtol=1e-05) - value_expect, indice_expect = cal_mode(self.inputs, - axis, - keepdim=True) + value_expect, indice_expect = cal_mode( + self.inputs, axis, keepdim=True + ) v, inds = paddle.mode(tensor, axis, keepdim=True) np.testing.assert_allclose(v.numpy(), value_expect, rtol=1e-05) @@ -137,9 +133,9 @@ def test_gpu_kernel(): v, inds = paddle.mode(tensor, axis) np.testing.assert_allclose(v.numpy(), value_expect, rtol=1e-05) - value_expect, indice_expect = cal_mode(self.inputs, - axis, - keepdim=True) + value_expect, indice_expect = cal_mode( + self.inputs, axis, keepdim=True + ) v, inds = paddle.mode(tensor, axis, keepdim=True) np.testing.assert_allclose(v.numpy(), value_expect, rtol=1e-05) @@ -150,7 +146,6 @@ def test_gpu_kernel(): class TestModeOpErrors(unittest.TestCase): - def setUp(self): self.x = paddle.uniform([2, 10, 20, 25], dtype='float32') @@ -161,25 +156,27 @@ def test_dim_range_error(): class TestModeOpInStatic(unittest.TestCase): - def setUp(self): np.random.seed(666) - self.input_data = np.ceil(np.random.random((2, 10, 10)) * 1000, - dtype=np.float64) + self.input_data = np.ceil( + np.random.random((2, 10, 10)) * 1000, dtype=np.float64 + ) def test_run_static(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - input_tensor = paddle.static.data(name="x", - shape=[2, 10, 10], - dtype="float64") + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + input_tensor = paddle.static.data( + name="x", shape=[2, 10, 10], dtype="float64" + ) result = paddle.mode(input_tensor, axis=1) expect_value = cal_mode(self.input_data, axis=1)[0] exe = paddle.static.Executor(paddle.CPUPlace()) - paddle_result = exe.run(feed={"x": self.input_data}, - fetch_list=[result])[0] + paddle_result = exe.run( + feed={"x": self.input_data}, fetch_list=[result] + )[0] np.testing.assert_allclose(paddle_result, expect_value, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_modelaverage.py b/python/paddle/fluid/tests/unittests/test_modelaverage.py index 474fa692b476b..0ab8aad42719c 100644 --- a/python/paddle/fluid/tests/unittests/test_modelaverage.py +++ b/python/paddle/fluid/tests/unittests/test_modelaverage.py @@ -20,7 +20,6 @@ class TestModelAverage(unittest.TestCase): - def test_model_average_static(self): paddle.enable_static() place = fluid.CPUPlace() @@ -35,50 +34,75 @@ def test_model_average_static(self): hidden = fluid.layers.fc(input=data, size=10) loss = paddle.mean(hidden) test_program = train_program.clone() - optimizer = paddle.optimizer.Momentum(learning_rate=0.2, - momentum=0.1) + optimizer = paddle.optimizer.Momentum( + learning_rate=0.2, momentum=0.1 + ) optimizer.minimize(loss) # build ModelAverage optimizer model_average = paddle.incubate.optimizer.ModelAverage( - 0.15, min_average_window=2, max_average_window=10) + 0.15, min_average_window=2, max_average_window=10 + ) exe.run(startup) for i in range(10): x = np.random.random(size=(10, 1)).astype('float32') - latest_b, sum_1, sum_2, sum_3, num_accumulates, old_num_accumulates, num_updates = exe.run( + ( + latest_b, + sum_1, + sum_2, + sum_3, + num_accumulates, + old_num_accumulates, + num_updates, + ) = exe.run( program=train_program, feed={'X': x}, fetch_list=[ - 'fc_0.b_0', 'fc_0.b_0_sum_1_0', 'fc_0.b_0_sum_2_0', - 'fc_0.b_0_sum_3_0', 'fc_0.b_0_num_accumulates_0', - 'fc_0.b_0_old_num_accumulates_0', 'fc_0.b_0_num_updates_0' - ]) + 'fc_0.b_0', + 'fc_0.b_0_sum_1_0', + 'fc_0.b_0_sum_2_0', + 'fc_0.b_0_sum_3_0', + 'fc_0.b_0_num_accumulates_0', + 'fc_0.b_0_old_num_accumulates_0', + 'fc_0.b_0_num_updates_0', + ], + ) self.assertTrue( - np.equal(sum_1, np.zeros(shape=[10], dtype='float32')).all()) + np.equal(sum_1, np.zeros(shape=[10], dtype='float32')).all() + ) self.assertTrue( - np.equal(sum_2, np.zeros(shape=[10], dtype='float32')).all()) + np.equal(sum_2, np.zeros(shape=[10], dtype='float32')).all() + ) self.assertTrue( - np.equal(num_accumulates, np.array([0], dtype='int64')).all()) + np.equal(num_accumulates, np.array([0], dtype='int64')).all() + ) self.assertTrue( - np.equal(old_num_accumulates, np.array([2], dtype='int64')).all()) + np.equal(old_num_accumulates, np.array([2], dtype='int64')).all() + ) self.assertTrue( - np.equal(num_updates, np.array([10], dtype='int64')).all()) + np.equal(num_updates, np.array([10], dtype='int64')).all() + ) - average_b = (sum_1 + sum_2 + sum_3) / (num_accumulates + - old_num_accumulates) + average_b = (sum_1 + sum_2 + sum_3) / ( + num_accumulates + old_num_accumulates + ) # apply ModelAverage with model_average.apply(exe): x = np.random.random(size=(10, 1)).astype('float32') - outs, b = exe.run(program=test_program, - feed={'X': x}, - fetch_list=[loss.name, 'fc_0.b_0']) + outs, b = exe.run( + program=test_program, + feed={'X': x}, + fetch_list=[loss.name, 'fc_0.b_0'], + ) self.assertAlmostEqual(np.mean(average_b), np.mean(b)) x = np.random.random(size=(10, 1)).astype('float32') - outs, b = exe.run(program=test_program, - feed={'X': x}, - fetch_list=[loss.name, 'fc_0.b_0']) + outs, b = exe.run( + program=test_program, + feed={'X': x}, + fetch_list=[loss.name, 'fc_0.b_0'], + ) self.assertAlmostEqual(np.mean(latest_b), np.mean(b)) def test_model_average_dygraph(self): @@ -91,21 +115,20 @@ def test_model_average_dygraph(self): # define a random dataset class RandomDataset(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, - (1, )).astype('int64') + label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype( + 'int64' + ) return image, label def __len__(self): return self.num_samples class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) @@ -131,23 +154,30 @@ def train(layer, loader, loss_fn, opt, model_average): sum_2 = model_average._get_accumulator('sum_2', layer.bias) sum_3 = model_average._get_accumulator('sum_3', layer.bias) num_accumulates = model_average._get_accumulator( - 'num_accumulates', layer.bias) + 'num_accumulates', layer.bias + ) old_num_accumulates = model_average._get_accumulator( - 'old_num_accumulates', layer.bias) + 'old_num_accumulates', layer.bias + ) num_updates = model_average._get_accumulator( - 'num_updates', layer.bias) + 'num_updates', layer.bias + ) - return ((sum_1 + sum_2 + sum_3) / - (num_accumulates + old_num_accumulates)).numpy() + return ( + (sum_1 + sum_2 + sum_3) + / (num_accumulates + old_num_accumulates) + ).numpy() def evaluate(layer, loader, loss_fn, check_param): for batch_id, (image, label) in enumerate(loader()): out = layer(image) loss = loss_fn(out, label) loss.backward() - self.assertAlmostEqual(np.mean(layer.bias.numpy()), - np.mean(check_param), - delta=5e-3) + self.assertAlmostEqual( + np.mean(layer.bias.numpy()), + np.mean(check_param), + delta=5e-3, + ) # print("Evaluate batch {}: loss = {}, bias = {}".format( # batch_id, np.mean(loss.numpy()), layer.bias.numpy())) @@ -155,36 +185,42 @@ def evaluate(layer, loader, loss_fn, check_param): layer = LinearNet() loss_fn = nn.CrossEntropyLoss() - optimizer = paddle.optimizer.Momentum(learning_rate=0.2, - momentum=0.1, - parameters=layer.parameters()) + optimizer = paddle.optimizer.Momentum( + learning_rate=0.2, momentum=0.1, parameters=layer.parameters() + ) # build ModelAverage optimizer model_average = paddle.incubate.optimizer.ModelAverage( 0.15, parameters=layer.parameters(), min_average_window=2, - max_average_window=10) + max_average_window=10, + ) # create data loader dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) - loader = paddle.io.DataLoader(dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=2) - eval_loader = paddle.io.DataLoader(dataset, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=1) + loader = paddle.io.DataLoader( + dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2, + ) + eval_loader = paddle.io.DataLoader( + dataset, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=1, + ) # train check_param = train(layer, loader, loss_fn, optimizer, model_average) # print(check_param) with model_average.apply(need_restore=False): evaluate(layer, eval_loader, loss_fn, check_param) - check_param = (model_average._get_accumulator('restore', - layer.bias)).numpy() + check_param = ( + model_average._get_accumulator('restore', layer.bias) + ).numpy() # print(check_param) # print("\nEvaluate With Restored Paramters") model_average.restore() diff --git a/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py b/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py index 1e81a6ae6b33a..99ce37834b5c3 100644 --- a/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py @@ -19,27 +19,29 @@ def modified_huber_loss_forward(val): if val < -1: - return -4. * val + return -4.0 * val elif val < 1: - return (1. - val) * (1. - val) + return (1.0 - val) * (1.0 - val) else: - return 0. + return 0.0 class TestModifiedHuberLossOp(OpTest): - def setUp(self): self.op_type = 'modified_huber_loss' samples_num = 100 - x_np = np.random.uniform(-2., 2., (samples_num, 1)).astype('float32') - y_np = np.random.choice([0, 1], samples_num).reshape( - (samples_num, 1)).astype('float32') - product_res = x_np * (2. * y_np - 1.) + x_np = np.random.uniform(-2.0, 2.0, (samples_num, 1)).astype('float32') + y_np = ( + np.random.choice([0, 1], samples_num) + .reshape((samples_num, 1)) + .astype('float32') + ) + product_res = x_np * (2.0 * y_np - 1.0) # keep away from the junction of piecewise function for pos, val in np.ndenumerate(product_res): - while abs(val - 1.) < 0.05: - x_np[pos] = np.random.uniform(-2., 2.) + while abs(val - 1.0) < 0.05: + x_np[pos] = np.random.uniform(-2.0, 2.0) y_np[pos] = np.random.choice([0, 1]) product_res[pos] = x_np[pos] * (2 * y_np[pos] - 1) val = product_res[pos] @@ -49,7 +51,7 @@ def setUp(self): self.outputs = { 'IntermediateVal': product_res.astype('float32'), - 'Out': loss.reshape((samples_num, 1)).astype('float32') + 'Out': loss.reshape((samples_num, 1)).astype('float32'), } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/test_momentum_op.py b/python/paddle/fluid/tests/unittests/test_momentum_op.py index e16dfa969f119..991291e36fb7c 100644 --- a/python/paddle/fluid/tests/unittests/test_momentum_op.py +++ b/python/paddle/fluid/tests/unittests/test_momentum_op.py @@ -23,14 +23,16 @@ from paddle.fluid.framework import _test_eager_guard -def calculate_momentum_by_numpy(param, - grad, - mu, - velocity, - use_nesterov, - learning_rate, - regularization_method=None, - regularization_coeff=1.0): +def calculate_momentum_by_numpy( + param, + grad, + mu, + velocity, + use_nesterov, + learning_rate, + regularization_method=None, + regularization_coeff=1.0, +): if regularization_method == "l2_decay": grad = grad + regularization_coeff * param @@ -42,8 +44,9 @@ def calculate_momentum_by_numpy(param, else: velocity_out = mu * velocity + grad if use_nesterov: - param_out = param - grad * learning_rate - \ - velocity_out * mu * learning_rate + param_out = ( + param - grad * learning_rate - velocity_out * mu * learning_rate + ) else: param_out = param - learning_rate * velocity_out @@ -51,7 +54,6 @@ def calculate_momentum_by_numpy(param, class TestMomentumOp1(OpTest): - def setUp(self): self.op_type = "momentum" self.dtype = np.float32 @@ -68,7 +70,7 @@ def setUp(self): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = {'mu': mu} @@ -79,7 +81,8 @@ def setUp(self): mu=mu, velocity=velocity, use_nesterov=use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -91,7 +94,6 @@ def test_check_output(self): class TestMomentumOpFp16(TestMomentumOp1): - def init_dtype(self): self.dtype = np.float16 @@ -100,8 +102,7 @@ def test_check_output(self): class TestMomentumOp2(OpTest): - '''Test Momentum with default values for attributes - ''' + '''Test Momentum with default values for attributes''' def setUp(self): self.op_type = "momentum" @@ -117,7 +118,7 @@ def setUp(self): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = {'mu': mu, 'use_nesterov': use_nesterov} @@ -128,7 +129,8 @@ def setUp(self): mu=mu, velocity=velocity, use_nesterov=use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -136,10 +138,10 @@ def test_check_output(self): self.check_output() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestLarsMomentumOpWithMP(OpTest): - def setUp(self): self.config() self.op_type = "lars_momentum" @@ -166,11 +168,16 @@ def setUp(self): fp32_grad = grad.astype("float32") pnorm = np.sqrt(np.square(master_param).sum()) gnorm = np.sqrt(np.square(fp32_grad).sum()) - local_lr = learning_rate * lars_coeff * pnorm / ( - gnorm + lars_weight_decay * pnorm) + local_lr = ( + learning_rate + * lars_coeff + * pnorm + / (gnorm + lars_weight_decay * pnorm) + ) fp32_grad = fp32_grad * rescale_grad velocity_out = mu * velocity + local_lr * ( - fp32_grad + lars_weight_decay * master_param) + fp32_grad + lars_weight_decay * master_param + ) p_new = master_param - velocity_out param_out = p_new.astype("float16") master_param_out = p_new @@ -183,7 +190,8 @@ def setUp(self): param_outs.append(("SubParam_out_" + str(i), param_out)) master_params.append(("SubMasterParam_" + str(i), master_param)) master_param_outs.append( - ("SubMasterParamOut_" + str(i), master_param_out)) + ("SubMasterParamOut_" + str(i), master_param_out) + ) self.inputs = { 'Param': params, @@ -198,13 +206,13 @@ def setUp(self): 'lars_coeff': lars_coeff, 'lars_weight_decay': [lars_weight_decay], 'multi_precision': True, - 'rescale_grad': rescale_grad + 'rescale_grad': rescale_grad, } self.outputs = { 'ParamOut': param_outs, 'VelocityOut': velocity_outs, - 'MasterParamOut': master_param_outs + 'MasterParamOut': master_param_outs, } def test_check_output(self): @@ -219,7 +227,6 @@ def config(self): class TestLarsMomentumOp(OpTest): - def setUp(self): self.config() self.op_type = "lars_momentum" @@ -240,10 +247,15 @@ def setUp(self): learning_rate = np.array([0.001]).astype("float32") pnorm = np.sqrt(np.square(param).sum()) gnorm = np.sqrt(np.square(grad).sum()) - local_lr = learning_rate * lars_coeff * pnorm / ( - gnorm + lars_weight_decay * param) + local_lr = ( + learning_rate + * lars_coeff + * pnorm + / (gnorm + lars_weight_decay * param) + ) velocity_out = mu * velocity + local_lr * ( - grad + lars_weight_decay * param) + grad + lars_weight_decay * param + ) param_out = param - velocity_out params.append(("SubParam_" + str(i), param)) @@ -257,13 +269,13 @@ def setUp(self): 'Param': params, 'Grad': grads, 'Velocity': velocitys, - 'LearningRate': learning_rates + 'LearningRate': learning_rates, } self.attrs = { 'mu': mu, 'lars_coeff': lars_coeff, - 'lars_weight_decay': [lars_weight_decay] + 'lars_weight_decay': [lars_weight_decay], } self.outputs = {'ParamOut': param_outs, 'VelocityOut': velocity_outs} @@ -276,7 +288,6 @@ def config(self): class TestSparseMomentumOp(unittest.TestCase): - def setUp(self): self.use_nesterov = False self.regularization_method = "" @@ -315,8 +326,9 @@ def check_with_place(self, place): velocity_np_array = np.ones((height, row_numel)).astype("float32") velocity.set(velocity_np_array, place) velocity_out = scope.var('VelocityOut').get_tensor() - velocity_out_np_array = np.full((height, row_numel), - 0.0).astype("float32") + velocity_out_np_array = np.full((height, row_numel), 0.0).astype( + "float32" + ) velocity_out.set(velocity_out_np_array, place) # create and initialize LearningRate Variable @@ -325,17 +337,19 @@ def check_with_place(self, place): lr.set(lr_array, place) # create and run operator - op = Operator("momentum", - Param='Param', - Grad='Grad', - Velocity='Velocity', - ParamOut='ParamOut', - VelocityOut='VelocityOut', - LearningRate='LearningRate', - mu=mu, - use_nesterov=use_nesterov, - regularization_method=regularization_method, - regularization_coeff=regularization_coeff) + op = Operator( + "momentum", + Param='Param', + Grad='Grad', + Velocity='Velocity', + ParamOut='ParamOut', + VelocityOut='VelocityOut', + LearningRate='LearningRate', + mu=mu, + use_nesterov=use_nesterov, + regularization_method=regularization_method, + regularization_coeff=regularization_coeff, + ) op.run(scope, place) # get and compare result @@ -358,7 +372,8 @@ def check_with_place(self, place): use_nesterov=use_nesterov, learning_rate=lr_array, regularization_method=regularization_method, - regularization_coeff=regularization_coeff) + regularization_coeff=regularization_coeff, + ) self.assertTrue((_velocity_out == velocity_out_np_array).all()) self.assertTrue((_param_out == param_out_np_array).all()) @@ -375,13 +390,11 @@ def test_sparse_momentum(self): class TestSparseMomentumOp2(TestSparseMomentumOp): - def init_kernel(self): self.use_nesterov = True class TestSparseMomentumOpWithMultiPrecision(unittest.TestCase): - def setUp(self): self.init_args() self.regularization_method = "" @@ -425,8 +438,9 @@ def check_with_place(self, place): velocity_np_array = np.ones((height, row_numel)).astype("float32") velocity.set(velocity_np_array, place) velocity_out = scope.var('VelocityOut').get_tensor() - velocity_out_np_array = np.full((height, row_numel), - 0.0).astype("float32") + velocity_out_np_array = np.full((height, row_numel), 0.0).astype( + "float32" + ) velocity_out.set(velocity_out_np_array, place) # create and initialize LearningRate Variable @@ -435,21 +449,23 @@ def check_with_place(self, place): lr.set(lr_array, place) # create and run operator - op = Operator("momentum", - Param='Param', - Grad='Grad', - Velocity='Velocity', - MasterParam='MasterParam', - ParamOut='ParamOut', - VelocityOut='VelocityOut', - MasterParamOut='MasterParamOut', - LearningRate='LearningRate', - mu=mu, - use_nesterov=use_nesterov, - regularization_method=regularization_method, - regularization_coeff=regularization_coeff, - multi_precision=True, - rescale_grad=1.0) + op = Operator( + "momentum", + Param='Param', + Grad='Grad', + Velocity='Velocity', + MasterParam='MasterParam', + ParamOut='ParamOut', + VelocityOut='VelocityOut', + MasterParamOut='MasterParamOut', + LearningRate='LearningRate', + mu=mu, + use_nesterov=use_nesterov, + regularization_method=regularization_method, + regularization_coeff=regularization_coeff, + multi_precision=True, + rescale_grad=1.0, + ) op.run(scope, place) # get and compare result @@ -470,7 +486,8 @@ def check_with_place(self, place): use_nesterov=use_nesterov, learning_rate=lr_array, regularization_method=regularization_method, - regularization_coeff=regularization_coeff) + regularization_coeff=regularization_coeff, + ) self.assertTrue((_velocity_out == velocity_out_np_array).all()) self.assertTrue((_param_out == param_out_np_array).all()) @@ -484,23 +501,22 @@ def test_sparse_momentum(self): class TestSparseMomentumOpWithMultiPrecision2( - TestSparseMomentumOpWithMultiPrecision): - + TestSparseMomentumOpWithMultiPrecision +): def init_args(self): self.use_nesterov = True class TestMomentumV2(unittest.TestCase): - def test_momentum_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Momentum(learning_rate=0.01, - momentum=0.9, - parameters=linear.parameters()) + adam = paddle.optimizer.Momentum( + learning_rate=0.01, momentum=0.9, parameters=linear.parameters() + ) out = linear(a) out.backward() adam.step() @@ -517,13 +533,15 @@ def test_momentum(self): cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = paddle.mean(cost) - rms_optimizer = paddle.optimizer.Momentum(learning_rate=0.1, - momentum=0.9) + rms_optimizer = paddle.optimizer.Momentum( + learning_rate=0.1, momentum=0.9 + ) rms_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -531,9 +549,9 @@ def test_momentum(self): exe.run(main, feed=feeder.feed(data), fetch_list=fetch_list) def test_raise_error(self): - self.assertRaises(ValueError, - paddle.optimizer.Momentum, - learning_rate=None) + self.assertRaises( + ValueError, paddle.optimizer.Momentum, learning_rate=None + ) self.assertRaises(ValueError, paddle.optimizer.Momentum, momentum=None) def test_api_eager_dygraph(self): @@ -543,7 +561,6 @@ def test_api_eager_dygraph(self): class TestMomentumOpWithDecay(OpTest): - def setUp(self): self.op_type = "momentum" self.dtype = np.float32 @@ -565,14 +582,14 @@ def setUp(self): 'Param': param, 'Grad': grad, 'Velocity': velocity, - 'LearningRate': learning_rate + 'LearningRate': learning_rate, } self.attrs = { 'mu': mu, 'use_nesterov': use_nesterov, 'regularization_method': regularization_method, - 'regularization_coeff': regularization_coeff + 'regularization_coeff': regularization_coeff, } grad = grad + regularization_coeff * param @@ -583,7 +600,8 @@ def setUp(self): mu=mu, velocity=velocity, use_nesterov=use_nesterov, - learning_rate=learning_rate) + learning_rate=learning_rate, + ) self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -596,7 +614,6 @@ def test_check_output(self): class TestMomentumOpWithDecayFP16(TestMomentumOpWithDecay): - def init_config(self): self.dtype = np.float16 @@ -606,13 +623,11 @@ def test_check_output(self): class TestMomentumOpWithDecay2(TestMomentumOpWithDecay): - def init_config(self): self.use_nesterov = False class TestSparseMomentumOpWithDecay(TestSparseMomentumOp): - def setUp(self): self.use_nesterov = False self.regularization_method = 'l2_decay' @@ -620,13 +635,11 @@ def setUp(self): class TestSparseMomentumOpWithDecay2(TestSparseMomentumOpWithDecay): - def init_kernel(self): self.use_nesterov = True class TestMomentumOpWithDecayAPI(unittest.TestCase): - def _test_momentum_dygraph_common(self, regularization): paddle.disable_static() inp = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32") @@ -639,13 +652,16 @@ def _test_momentum_dygraph_common(self, regularization): learning_rate=0.01, momentum=0.9, parameter_list=linear.parameters(), - regularization=regularization) + regularization=regularization, + ) momentum.minimize(loss) def test_momentum_dygraph_1(self): self._test_momentum_dygraph_common( regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ) + ) def test_momentum_static(self): paddle.enable_static() @@ -659,12 +675,14 @@ def test_momentum_static(self): avg_cost = paddle.mean(cost) momentum_optimizer = paddle.fluid.contrib.optimizer.Momentum( - learning_rate=0.1, momentum=0.9) + learning_rate=0.1, momentum=0.9 + ) momentum_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -673,23 +691,23 @@ def test_momentum_static(self): class TestFusedMomentumWithDecayAPI(unittest.TestCase): - def get_program(self, weight_attr, bias_attr=False): main_program = paddle.static.Program() startup_program = paddle.static.Program() - with paddle.static.program_guard(main_program=main_program, - startup_program=startup_program): + with paddle.static.program_guard( + main_program=main_program, startup_program=startup_program + ): x = paddle.static.data(name='x', shape=[10, 10]) - linear = paddle.nn.Linear(10, - 10, - weight_attr=weight_attr, - bias_attr=bias_attr) + linear = paddle.nn.Linear( + 10, 10, weight_attr=weight_attr, bias_attr=bias_attr + ) out = linear(x) loss = paddle.mean(out) optimizer = paddle.optimizer.Momentum( learning_rate=0.01, momentum=0.9, - weight_decay=paddle.regularizer.L2Decay(0.5)) + weight_decay=paddle.regularizer.L2Decay(0.5), + ) optimizer.minimize(loss) return main_program @@ -698,7 +716,8 @@ def test_param_has_l2decay(self): weight_attr = paddle.ParamAttr( name="weight", initializer=paddle.nn.initializer.Constant(value=0.5), - regularizer=paddle.regularizer.L2Decay(0.1)) + regularizer=paddle.regularizer.L2Decay(0.1), + ) program = self.get_program(weight_attr, bias_attr=False) ops = program.global_block().ops @@ -713,11 +732,13 @@ def test_param_has_l1decay(self): weight_attr = paddle.ParamAttr( name="weight", initializer=paddle.nn.initializer.Constant(value=0.5), - regularizer=paddle.regularizer.L1Decay(0.1)) + regularizer=paddle.regularizer.L1Decay(0.1), + ) bias_attr = paddle.ParamAttr( name="bias", - initializer=paddle.nn.initializer.Constant(value=0.), - regularizer=None) + initializer=paddle.nn.initializer.Constant(value=0.0), + regularizer=None, + ) program = self.get_program(weight_attr, bias_attr) ops = program.global_block().ops @@ -732,8 +753,9 @@ def test_param_has_l1decay(self): self.assertEqual(ops[-1].attr('regularization_coeff'), 0) if 'bias' in ops[-2].input('Param'): self.assertEqual(ops[-2].attr('regularization_method'), 'l2_decay') - self.assertEqual(ops[-2].attr('regularization_coeff'), - np.float32(0.5)) + self.assertEqual( + ops[-2].attr('regularization_coeff'), np.float32(0.5) + ) def test_param_has_no_regularizer(self): paddle.enable_static() @@ -747,11 +769,11 @@ def test_param_has_no_regularizer(self): class TestMomentumOpVsMomentumOpWithDecayAPI(unittest.TestCase): - def __update_params(self, momentum, linear): for i in range(10): - inp = paddle.full(shape=[2, 2], fill_value=i, - dtype='float32').astype("float32") + inp = paddle.full( + shape=[2, 2], fill_value=i, dtype='float32' + ).astype("float32") inp = paddle.to_tensor(inp) out = linear(inp) loss = paddle.mean(out) @@ -766,32 +788,39 @@ def __test_vs(self, place=fluid.CPUPlace()): 2, 2, weight_attr=paddle.nn.initializer.Constant(value=2.0), - bias_attr=paddle.nn.initializer.Constant(value=2.0)) + bias_attr=paddle.nn.initializer.Constant(value=2.0), + ) momentum_old = paddle.fluid.optimizer.Momentum( learning_rate=0.01, momentum=0.9, parameter_list=linear_old.parameters(), regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ), + ) self.__update_params(momentum=momentum_old, linear=linear_old) linear_new = paddle.nn.Linear( 2, 2, weight_attr=paddle.nn.initializer.Constant(value=2.0), - bias_attr=paddle.nn.initializer.Constant(value=2.0)) + bias_attr=paddle.nn.initializer.Constant(value=2.0), + ) momentum_new = paddle.fluid.contrib.optimizer.Momentum( learning_rate=0.01, momentum=0.9, parameter_list=linear_new.parameters(), regularization=paddle.fluid.regularizer.L2Decay( - regularization_coeff=0.1)) + regularization_coeff=0.1 + ), + ) self.__update_params(momentum=momentum_new, linear=linear_new) self.assertEqual( (linear_old.weight.numpy() == linear_new.weight.numpy()).all(), True, - 'the param weight updated by two Momentum optimizers should equal') + 'the param weight updated by two Momentum optimizers should equal', + ) def test_vs(self, place=fluid.CPUPlace()): places = [fluid.CPUPlace()] @@ -803,7 +832,6 @@ def test_vs(self, place=fluid.CPUPlace()): class TestMomentumV2Group(TestMomentumV2): - def test_momentum_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") @@ -811,22 +839,20 @@ def test_momentum_dygraph(self): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.Momentum(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001, - 'learning_rate': - 0.1, - 'momentum': - 0.99 - }], - weight_decay=0.1, - momentum=0.9) + adam = paddle.optimizer.Momentum( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + 'learning_rate': 0.1, + 'momentum': 0.99, + }, + ], + weight_decay=0.1, + momentum=0.9, + ) out = linear_1(a) out = linear_2(out) out.backward() @@ -835,13 +861,14 @@ def test_momentum_dygraph(self): class TestMultiTensorMomentumDygraph(unittest.TestCase): - - def _momentum_optimize_dygraph(self, - place, - use_param_attr=False, - use_param_group=False, - use_amp=False, - use_multi_tensor=False): + def _momentum_optimize_dygraph( + self, + place, + use_param_attr=False, + use_param_group=False, + use_amp=False, + use_multi_tensor=False, + ): paddle.disable_static() paddle.seed(10) paddle.set_device(place) @@ -849,7 +876,8 @@ def _momentum_optimize_dygraph(self, weight_attr = paddle.ParamAttr( learning_rate=0.5, regularizer=paddle.regularizer.L2Decay(1.0), - trainable=True) + trainable=True, + ) if use_param_attr: model = paddle.nn.Linear(5, 5, weight_attr) else: @@ -858,17 +886,21 @@ def _momentum_optimize_dygraph(self, optimizer = paddle.optimizer.Momentum( parameters=model.parameters(), use_multi_tensor=use_multi_tensor, - multi_precision=use_amp) + multi_precision=use_amp, + ) else: optimizer = paddle.optimizer.Momentum( - parameters=[{ - 'params': model.parameters(), - 'weight_decay': 0.001, - 'learning_rate': 0.1, - 'momentum': 0.99 - }], + parameters=[ + { + 'params': model.parameters(), + 'weight_decay': 0.001, + 'learning_rate': 0.1, + 'momentum': 0.99, + } + ], use_multi_tensor=use_multi_tensor, - multi_precision=use_amp) + multi_precision=use_amp, + ) for idx in range(5): if place == 'gpu' and use_amp == True: model = paddle.amp.decorate(models=model, level='O2') @@ -898,9 +930,11 @@ def _get_places(self): def _check_with_place_amp(self, place, use_amp): output1, params1 = self._momentum_optimize_dygraph( - place=place, use_amp=use_amp, use_multi_tensor=True) + place=place, use_amp=use_amp, use_multi_tensor=True + ) output2, params2 = self._momentum_optimize_dygraph( - place=place, use_amp=use_amp, use_multi_tensor=False) + place=place, use_amp=use_amp, use_multi_tensor=False + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): @@ -911,12 +945,14 @@ def _check_with_param_arrt(self, place, use_amp): place=place, use_amp=use_amp, use_param_attr=True, - use_multi_tensor=True) + use_multi_tensor=True, + ) output2, params2 = self._momentum_optimize_dygraph( place=place, use_amp=use_amp, use_param_attr=True, - use_multi_tensor=False) + use_multi_tensor=False, + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): np.testing.assert_allclose(params1[idx], params2[idx], rtol=1e-05) @@ -926,12 +962,14 @@ def _check_with_param_group(self, place, use_amp): place=place, use_amp=use_amp, use_param_group=True, - use_multi_tensor=True) + use_multi_tensor=True, + ) output2, params2 = self._momentum_optimize_dygraph( place=place, use_amp=use_amp, use_param_group=True, - use_multi_tensor=False) + use_multi_tensor=False, + ) np.testing.assert_allclose(output1, output2, rtol=1e-05) for idx in range(len(params1)): np.testing.assert_allclose(params1[idx], params2[idx], rtol=1e-05) @@ -950,11 +988,9 @@ def test_api_eager_dygraph(self): class TestMultiTensorMomentumStatic(unittest.TestCase): - - def _momentum_optimize_static(self, - place, - use_amp=False, - use_multi_tensor=False): + def _momentum_optimize_static( + self, place, use_amp=False, use_multi_tensor=False + ): paddle.enable_static() paddle.seed(10) np.random.seed(10) @@ -963,24 +999,26 @@ def _momentum_optimize_static(self, exe = paddle.static.Executor(place=place) train_program = paddle.static.Program() startup_program = paddle.static.Program() - optimizer = paddle.optimizer.Momentum(multi_precision=use_amp, - use_multi_tensor=use_multi_tensor) + optimizer = paddle.optimizer.Momentum( + multi_precision=use_amp, use_multi_tensor=use_multi_tensor + ) if use_amp: optimizer = paddle.static.amp.decorate( optimizer, init_loss_scaling=128.0, use_dynamic_loss_scaling=True, use_pure_fp16=True, - use_fp16_guard=False) + use_fp16_guard=False, + ) with paddle.static.program_guard(train_program, startup_program): if use_amp: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float16') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float16' + ) else: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float32') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float32' + ) hidden = paddle.static.nn.fc(x=data, size=10) loss = paddle.mean(hidden) optimizer.minimize(loss) @@ -992,9 +1030,9 @@ def _momentum_optimize_static(self, x = numpy.random.random(size=(2, 2)).astype('float32') out = [] for idx in range(5): - loss_data, = exe.run(train_program, - feed={"X": x}, - fetch_list=[loss.name]) + (loss_data,) = exe.run( + train_program, feed={"X": x}, fetch_list=[loss.name] + ) out.append(loss_data) return out @@ -1005,12 +1043,12 @@ def _get_places(self): return places def _check_with_place_amp(self, place, use_amp): - output1 = self._momentum_optimize_static(place=place, - use_amp=use_amp, - use_multi_tensor=True) - output2 = self._momentum_optimize_static(place=place, - use_amp=use_amp, - use_multi_tensor=False) + output1 = self._momentum_optimize_static( + place=place, use_amp=use_amp, use_multi_tensor=True + ) + output2 = self._momentum_optimize_static( + place=place, use_amp=use_amp, use_multi_tensor=False + ) for idx in range(len(output1)): np.testing.assert_allclose(output1[idx], output2[idx], rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_monitor.py b/python/paddle/fluid/tests/unittests/test_monitor.py index 8649f11821eb6..dafb23c7d411e 100644 --- a/python/paddle/fluid/tests/unittests/test_monitor.py +++ b/python/paddle/fluid/tests/unittests/test_monitor.py @@ -27,7 +27,7 @@ class TestDatasetWithStat(unittest.TestCase): - """ TestCases for Dataset. """ + """TestCases for Dataset.""" def setUp(self): self.use_data_loader = False @@ -53,10 +53,9 @@ def test_dataset_run_with_stat(self): slots = ["slot1", "slot2", "slot3", "slot4"] slots_vars = [] for slot in slots: - var = fluid.layers.data(name=slot, - shape=[1], - dtype="int64", - lod_level=1) + var = fluid.layers.data( + name=slot, shape=[1], dtype="int64", lod_level=1 + ) slots_vars.append(var) embs = [] @@ -78,7 +77,8 @@ def test_dataset_run_with_stat(self): exe.run(fluid.default_startup_program()) if self.use_data_loader: data_loader = fluid.io.DataLoader.from_dataset( - dataset, fluid.cpu_places(), self.drop_last) + dataset, fluid.cpu_places(), self.drop_last + ) for i in range(self.epoch_num): for data in data_loader(): exe.run(fluid.default_main_program(), feed=data) @@ -86,11 +86,13 @@ def test_dataset_run_with_stat(self): else: for i in range(self.epoch_num): try: - exe.train_from_dataset(fluid.default_main_program(), - dataset, - fetch_list=[embs[0], embs[1]], - fetch_info=["emb0", "emb1"], - print_period=1) + exe.train_from_dataset( + fluid.default_main_program(), + dataset, + fetch_list=[embs[0], embs[1]], + fetch_info=["emb0", "emb1"], + print_period=1, + ) except Exception as e: self.assertTrue(False) diff --git a/python/paddle/fluid/tests/unittests/test_mse_loss.py b/python/paddle/fluid/tests/unittests/test_mse_loss.py index ef2a494c219e7..b0b59694406a7 100644 --- a/python/paddle/fluid/tests/unittests/test_mse_loss.py +++ b/python/paddle/fluid/tests/unittests/test_mse_loss.py @@ -22,7 +22,6 @@ class TestMseLoss(unittest.TestCase): - def test_mse_loss(self): input_val = np.random.uniform(0.1, 0.5, (2, 3)).astype("float32") label_val = np.random.uniform(0.1, 0.5, (2, 3)).astype("float32") @@ -34,24 +33,22 @@ def test_mse_loss(self): label_var = fluid.data(name="label", shape=[-1, 3], dtype="float32") output = layers.mse_loss(input=input_var, label=label_var) - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = Executor(place) - result, = exe.run(fluid.default_main_program(), - feed={ - "input": input_val, - "label": label_val - }, - fetch_list=[output]) + (result,) = exe.run( + fluid.default_main_program(), + feed={"input": input_val, "label": label_val}, + fetch_list=[output], + ) np.testing.assert_allclose(np_result, result, rtol=1e-05) class TestMseInvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_input(): input = [256, 3] label = fluid.data(name='label1', shape=[None, 3], dtype='float32') @@ -68,7 +65,6 @@ def test_invalid_label(): class TestNNMseLoss(unittest.TestCase): - def test_NNMseLoss_mean(self): for dim in [[10, 10], [2, 10, 10], [3, 3, 10, 10]]: input_np = np.random.uniform(0.1, 0.5, dim).astype("float32") @@ -76,30 +72,34 @@ def test_NNMseLoss_mean(self): paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.layers.data(name='input', - shape=dim, - dtype='float32') - label = fluid.layers.data(name='label', - shape=dim, - dtype='float32') + input = fluid.layers.data( + name='input', shape=dim, dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=dim, dtype='float32' + ) mse_loss = paddle.nn.loss.MSELoss() ret = mse_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[ret]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[ret], + ) with fluid.dygraph.guard(): mse_loss = paddle.nn.loss.MSELoss() - dy_ret = mse_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = mse_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_result = dy_ret.numpy() sub = input_np - label_np @@ -116,30 +116,34 @@ def test_NNMseLoss_sum(self): paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.layers.data(name='input', - shape=dim, - dtype='float32') - label = fluid.layers.data(name='label', - shape=dim, - dtype='float32') + input = fluid.layers.data( + name='input', shape=dim, dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=dim, dtype='float32' + ) mse_loss = paddle.nn.loss.MSELoss(reduction='sum') ret = mse_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[ret]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[ret], + ) with fluid.dygraph.guard(): mse_loss = paddle.nn.loss.MSELoss(reduction='sum') - dy_ret = mse_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = mse_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_result = dy_ret.numpy() sub = input_np - label_np @@ -156,34 +160,38 @@ def test_NNMseLoss_none(self): paddle.enable_static() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.layers.data(name='input', - shape=dim, - dtype='float32') - label = fluid.layers.data(name='label', - shape=dim, - dtype='float32') + input = fluid.layers.data( + name='input', shape=dim, dtype='float32' + ) + label = fluid.layers.data( + name='label', shape=dim, dtype='float32' + ) mse_loss = paddle.nn.loss.MSELoss(reduction='none') ret = mse_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[ret]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[ret], + ) with fluid.dygraph.guard(): mse_loss = paddle.nn.loss.MSELoss(reduction='none') - dy_ret = mse_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = mse_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_result = dy_ret.numpy() sub = input_np - label_np - expected = (sub * sub) + expected = sub * sub np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) @@ -191,7 +199,6 @@ def test_NNMseLoss_none(self): class TestNNFunctionalMseLoss(unittest.TestCase): - def test_NNFunctionalMseLoss_mean(self): for dim in [[10, 10], [2, 10, 10], [3, 3, 10, 10]]: input_np = np.random.uniform(0.1, 0.5, dim).astype("float32") @@ -199,30 +206,32 @@ def test_NNFunctionalMseLoss_mean(self): paddle.enable_static() prog = paddle.static.Program() startup_prog = paddle.static.Program() - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=dim, - dtype='float32') - target = paddle.fluid.data(name='target', - shape=dim, - dtype='float32') + input = paddle.fluid.data( + name='input', shape=dim, dtype='float32' + ) + target = paddle.fluid.data( + name='target', shape=dim, dtype='float32' + ) mse_loss = paddle.nn.functional.mse_loss(input, target, 'mean') exe = paddle.static.Executor(place) exe.run(startup_prog) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "target": target_np - }, - fetch_list=[mse_loss]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "target": target_np}, + fetch_list=[mse_loss], + ) paddle.disable_static() - dy_ret = paddle.nn.functional.mse_loss(paddle.to_tensor(input_np), - paddle.to_tensor(target_np), - 'mean') + dy_ret = paddle.nn.functional.mse_loss( + paddle.to_tensor(input_np), paddle.to_tensor(target_np), 'mean' + ) dy_result = dy_ret.numpy() sub = input_np - target_np @@ -239,30 +248,32 @@ def test_NNFunctionalMseLoss_sum(self): paddle.enable_static() prog = paddle.static.Program() startup_prog = paddle.static.Program() - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=dim, - dtype='float32') - target = paddle.fluid.data(name='target', - shape=dim, - dtype='float32') + input = paddle.fluid.data( + name='input', shape=dim, dtype='float32' + ) + target = paddle.fluid.data( + name='target', shape=dim, dtype='float32' + ) mse_loss = paddle.nn.functional.mse_loss(input, target, 'sum') exe = paddle.static.Executor(place) exe.run(startup_prog) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "target": target_np - }, - fetch_list=[mse_loss]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "target": target_np}, + fetch_list=[mse_loss], + ) paddle.disable_static() - dy_ret = paddle.nn.functional.mse_loss(paddle.to_tensor(input_np), - paddle.to_tensor(target_np), - 'sum') + dy_ret = paddle.nn.functional.mse_loss( + paddle.to_tensor(input_np), paddle.to_tensor(target_np), 'sum' + ) dy_result = dy_ret.numpy() sub = input_np - target_np @@ -279,30 +290,32 @@ def test_NNFunctionalMseLoss_none(self): paddle.enable_static() prog = paddle.static.Program() startup_prog = paddle.static.Program() - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) with paddle.static.program_guard(prog, startup_prog): - input = paddle.fluid.data(name='input', - shape=dim, - dtype='float32') - target = paddle.fluid.data(name='target', - shape=dim, - dtype='float32') + input = paddle.fluid.data( + name='input', shape=dim, dtype='float32' + ) + target = paddle.fluid.data( + name='target', shape=dim, dtype='float32' + ) mse_loss = paddle.nn.functional.mse_loss(input, target, 'none') exe = paddle.static.Executor(place) exe.run(startup_prog) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "target": target_np - }, - fetch_list=[mse_loss]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "target": target_np}, + fetch_list=[mse_loss], + ) paddle.disable_static() - dy_ret = paddle.nn.functional.mse_loss(paddle.to_tensor(input_np), - paddle.to_tensor(target_np), - 'none') + dy_ret = paddle.nn.functional.mse_loss( + paddle.to_tensor(input_np), paddle.to_tensor(target_np), 'none' + ) dy_result = dy_ret.numpy() sub = input_np - target_np diff --git a/python/paddle/fluid/tests/unittests/test_mul_nn_grad.py b/python/paddle/fluid/tests/unittests/test_mul_nn_grad.py index 22000cb1808ba..94e12e337ec91 100644 --- a/python/paddle/fluid/tests/unittests/test_mul_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_mul_nn_grad.py @@ -26,7 +26,6 @@ class TestMulGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): prog = fluid.Program() @@ -45,7 +44,6 @@ def test_grad(self): class TestMulDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -62,11 +60,9 @@ def func(self, place): x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, y_shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -77,7 +73,6 @@ def test_grad(self): class TestMatmulDoubleGradCheck(unittest.TestCase): - def setUp(self): self.init_test() @@ -92,25 +87,21 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = layers.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = layers.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = layers.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = layers.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = layers.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = layers.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -121,7 +112,6 @@ def test_grad(self): def TestMatmulDoubleGradCheckCase1(TestMatmulDoubleGradCheck): - def init_test(self): self.x_shape = [2, 3] self.y_shape = [3, 2] @@ -130,7 +120,6 @@ def init_test(self): def TestMatmulDoubleGradCheckCase2(TestMatmulDoubleGradCheck): - def init_test(self): self.x_shape = [2, 4, 3] self.y_shape = [2, 4, 5] @@ -139,7 +128,6 @@ def init_test(self): def TestMatmulDoubleGradCheckCase3(TestMatmulDoubleGradCheck): - def init_test(self): self.x_shape = [2, 3, 4, 5] self.y_shape = [2, 3, 3, 5] @@ -148,7 +136,6 @@ def init_test(self): def TestMatmulDoubleGradCheckCase4(TestMatmulDoubleGradCheck): - def init_test(self): self.x_shape = [2, 3, 4] self.y_shape = [4, 3] diff --git a/python/paddle/fluid/tests/unittests/test_mul_op.py b/python/paddle/fluid/tests/unittests/test_mul_op.py index 0236f34ea8bf9..5469caed15753 100644 --- a/python/paddle/fluid/tests/unittests/test_mul_op.py +++ b/python/paddle/fluid/tests/unittests/test_mul_op.py @@ -24,14 +24,13 @@ class TestMulOp(OpTest): - def setUp(self): self.op_type = "mul" self.dtype = np.float64 self.init_dtype_type() self.inputs = { 'X': np.random.random((20, 5)).astype(self.dtype), - 'Y': np.random.random((5, 21)).astype(self.dtype) + 'Y': np.random.random((5, 21)).astype(self.dtype), } self.outputs = {'Out': np.dot(self.inputs['X'], self.inputs['Y'])} @@ -45,27 +44,26 @@ def test_check_grad_normal(self): self.check_grad(['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.5, - no_grad_set=set("X")) + self.check_grad( + ['Y'], 'Out', max_relative_error=0.5, no_grad_set=set("X") + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.5, - no_grad_set=set('Y')) + self.check_grad( + ['X'], 'Out', max_relative_error=0.5, no_grad_set=set('Y') + ) class TestMulOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of mul_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - x2 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + x2 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.mul, x1, x2) # The input dtype of mul_op must be float32 or float64. x3 = fluid.layers.data(name='x3', shape=[4], dtype="int32") @@ -74,21 +72,22 @@ def test_errors(self): class TestMulOp2(OpTest): - def setUp(self): self.op_type = "mul" self.dtype = np.float64 self.init_dtype_type() self.inputs = { 'X': np.random.random((3, 4, 2, 9)).astype(self.dtype), - 'Y': np.random.random((3, 6, 1, 2, 3)).astype(self.dtype) + 'Y': np.random.random((3, 6, 1, 2, 3)).astype(self.dtype), } self.attrs = { 'x_num_col_dims': 2, 'y_num_col_dims': 2, } - result = np.dot(self.inputs['X'].reshape(3 * 4, 2 * 9), - self.inputs['Y'].reshape(3 * 6, 1 * 2 * 3)) + result = np.dot( + self.inputs['X'].reshape(3 * 4, 2 * 9), + self.inputs['Y'].reshape(3 * 6, 1 * 2 * 3), + ) result = result.reshape(3, 4, 1, 2, 3) self.outputs = {'Out': result} @@ -102,22 +101,20 @@ def test_check_grad_normal(self): self.check_grad(['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.5, - no_grad_set=set('X')) + self.check_grad( + ['Y'], 'Out', max_relative_error=0.5, no_grad_set=set('X') + ) def test_check_grad_ignore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.5, - no_grad_set=set('Y')) + self.check_grad( + ['X'], 'Out', max_relative_error=0.5, no_grad_set=set('Y') + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16MulOp1(TestMulOp): - def init_dtype_type(self): self.dtype = np.float16 @@ -129,31 +126,37 @@ def test_check_output(self): def test_check_grad_normal(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['X', 'Y'], - 'Out', - max_relative_error=0.5) + self.check_grad_with_place( + place, ['X', 'Y'], 'Out', max_relative_error=0.5 + ) def test_check_grad_ingore_x(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.5, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.5, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.5, - no_grad_set=set('Y')) - - -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.5, + no_grad_set=set('Y'), + ) + + +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16MulOp2(TestMulOp2): - def init_dtype_type(self): self.dtype = np.float16 @@ -165,25 +168,31 @@ def test_check_output(self): def test_check_grad_normal(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['X', 'Y'], - 'Out', - max_relative_error=0.9) + self.check_grad_with_place( + place, ['X', 'Y'], 'Out', max_relative_error=0.9 + ) def test_check_grad_ingore_x(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.5, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.5, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.9, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.9, + no_grad_set=set('Y'), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_multi_dot_op.py b/python/paddle/fluid/tests/unittests/test_multi_dot_op.py index ce188cac5c7d2..96d8c7572eef0 100644 --- a/python/paddle/fluid/tests/unittests/test_multi_dot_op.py +++ b/python/paddle/fluid/tests/unittests/test_multi_dot_op.py @@ -23,10 +23,9 @@ paddle.enable_static() -#the unittest of multi_dot -#compare the result of paddle multi_dot and numpy multi_dot +# the unittest of multi_dot +# compare the result of paddle multi_dot and numpy multi_dot class TestMultiDotOp(OpTest): - def setUp(self): self.op_type = "multi_dot" self.python_api = paddle.linalg.multi_dot @@ -50,9 +49,8 @@ def test_check_grad(self): self.check_grad(['x1'], 'Out', check_eager=True) -#(A*B)*C +# (A*B)*C class TestMultiDotOp3Mat(TestMultiDotOp): - def get_inputs_and_outputs(self): self.A = np.random.random((2, 10)).astype(self.dtype) self.B = np.random.random((10, 4)).astype(self.dtype) @@ -66,9 +64,8 @@ def test_check_grad(self): self.check_grad(['x2'], 'Out', check_eager=True) -#A*(B*C) +# A*(B*C) class TestMultiDotOp3Mat2(TestMultiDotOp): - def get_inputs_and_outputs(self): self.A = np.random.random((3, 4)).astype(self.dtype) self.B = np.random.random((4, 8)).astype(self.dtype) @@ -83,15 +80,18 @@ def test_check_grad(self): class TestMultiDotOp4Mat(TestMultiDotOp): - def get_inputs_and_outputs(self): self.A = np.random.random((8, 6)).astype(self.dtype) self.B = np.random.random((6, 3)).astype(self.dtype) self.C = np.random.random((3, 4)).astype(self.dtype) self.D = np.random.random((4, 5)).astype(self.dtype) self.inputs = { - 'X': [('x0', self.A), ('x1', self.B), ('x2', self.C), - ('x3', self.D)] + 'X': [ + ('x0', self.A), + ('x1', self.B), + ('x2', self.C), + ('x3', self.D), + ] } self.outputs = {'Out': multi_dot([self.A, self.B, self.C, self.D])} @@ -103,7 +103,6 @@ def test_check_grad(self): class TestMultiDotOpFirst1D(TestMultiDotOp): - def get_inputs_and_outputs(self): self.A = np.random.random((4)).astype(self.dtype) self.B = np.random.random((4, 3)).astype(self.dtype) @@ -112,7 +111,6 @@ def get_inputs_and_outputs(self): class TestMultiDotOp3MatFirst1D(TestMultiDotOp3Mat): - def get_inputs_and_outputs(self): self.A = np.random.random((4)).astype(self.dtype) self.B = np.random.random((4, 3)).astype(self.dtype) @@ -122,21 +120,23 @@ def get_inputs_and_outputs(self): class TestMultiDotOp4MatFirst1D(TestMultiDotOp4Mat): - def get_inputs_and_outputs(self): self.A = np.random.random((4)).astype(self.dtype) self.B = np.random.random((4, 3)).astype(self.dtype) self.C = np.random.random((3, 4)).astype(self.dtype) self.D = np.random.random((4, 5)).astype(self.dtype) self.inputs = { - 'X': [('x0', self.A), ('x1', self.B), ('x2', self.C), - ('x3', self.D)] + 'X': [ + ('x0', self.A), + ('x1', self.B), + ('x2', self.C), + ('x3', self.D), + ] } self.outputs = {'Out': multi_dot([self.A, self.B, self.C, self.D])} class TestMultiDotOpLast1D(TestMultiDotOp): - def get_inputs_and_outputs(self): self.A = np.random.random((3, 6)).astype(self.dtype) self.B = np.random.random((6)).astype(self.dtype) @@ -145,7 +145,6 @@ def get_inputs_and_outputs(self): class TestMultiDotOp3MatLast1D(TestMultiDotOp3Mat): - def get_inputs_and_outputs(self): self.A = np.random.random((2, 4)).astype(self.dtype) self.B = np.random.random((4, 3)).astype(self.dtype) @@ -160,32 +159,33 @@ def test_check_grad(self): class TestMultiDotOp4MatLast1D(TestMultiDotOp4Mat): - def get_inputs_and_outputs(self): self.A = np.random.random((2, 3)).astype(self.dtype) self.B = np.random.random((3, 2)).astype(self.dtype) self.C = np.random.random((2, 3)).astype(self.dtype) self.D = np.random.random((3)).astype(self.dtype) self.inputs = { - 'X': [('x0', self.A), ('x1', self.B), ('x2', self.C), - ('x3', self.D)] + 'X': [ + ('x0', self.A), + ('x1', self.B), + ('x2', self.C), + ('x3', self.D), + ] } self.outputs = {'Out': multi_dot([self.A, self.B, self.C, self.D])} class TestMultiDotOpFirstAndLast1D(TestMultiDotOp): - def get_inputs_and_outputs(self): - self.A = np.random.random((4, )).astype(self.dtype) + self.A = np.random.random((4,)).astype(self.dtype) self.B = np.random.random((4)).astype(self.dtype) self.inputs = {'X': [('x0', self.A), ('x1', self.B)]} self.outputs = {'Out': multi_dot([self.A, self.B])} class TestMultiDotOp3MatFirstAndLast1D(TestMultiDotOp3Mat): - def get_inputs_and_outputs(self): - self.A = np.random.random((6, )).astype(self.dtype) + self.A = np.random.random((6,)).astype(self.dtype) self.B = np.random.random((6, 4)).astype(self.dtype) self.C = np.random.random((4)).astype(self.dtype) self.inputs = {'X': [('x0', self.A), ('x1', self.B), ('x2', self.C)]} @@ -193,52 +193,57 @@ def get_inputs_and_outputs(self): class TestMultiDotOp4MatFirstAndLast1D(TestMultiDotOp4Mat): - def get_inputs_and_outputs(self): - self.A = np.random.random((3, )).astype(self.dtype) + self.A = np.random.random((3,)).astype(self.dtype) self.B = np.random.random((3, 4)).astype(self.dtype) self.C = np.random.random((4, 2)).astype(self.dtype) self.D = np.random.random((2)).astype(self.dtype) self.inputs = { - 'X': [('x0', self.A), ('x1', self.B), ('x2', self.C), - ('x3', self.D)] + 'X': [ + ('x0', self.A), + ('x1', self.B), + ('x2', self.C), + ('x3', self.D), + ] } self.outputs = {'Out': multi_dot([self.A, self.B, self.C, self.D])} #####python API test####### class TestMultiDotOpError(unittest.TestCase): - def test_errors(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): # The inputs type of multi_dot must be list matrix. input1 = 12 - self.assertRaises(TypeError, paddle.linalg.multi_dot, - [input1, input1]) + self.assertRaises( + TypeError, paddle.linalg.multi_dot, [input1, input1] + ) # The inputs dtype of multi_dot must be float64, float64 or float16. - input2 = paddle.static.data(name='input2', - shape=[10, 10], - dtype="int32") - self.assertRaises(TypeError, paddle.linalg.multi_dot, - [input2, input2]) + input2 = paddle.static.data( + name='input2', shape=[10, 10], dtype="int32" + ) + self.assertRaises( + TypeError, paddle.linalg.multi_dot, [input2, input2] + ) # the number of tensor must be larger than 1 x0 = paddle.static.data(name='x0', shape=[3, 2], dtype="float64") self.assertRaises(ValueError, paddle.linalg.multi_dot, [x0]) - #the first tensor must be 1D or 2D + # the first tensor must be 1D or 2D x1 = paddle.static.data(name='x1', shape=[3, 2, 3], dtype="float64") x2 = paddle.static.data(name='x2', shape=[3, 2], dtype="float64") self.assertRaises(ValueError, paddle.linalg.multi_dot, [x1, x2]) - #the last tensor must be 1D or 2D + # the last tensor must be 1D or 2D x3 = paddle.static.data(name='x3', shape=[3, 2], dtype="float64") x4 = paddle.static.data(name='x4', shape=[3, 2, 2], dtype="float64") self.assertRaises(ValueError, paddle.linalg.multi_dot, [x3, x4]) - #the tensor must be 2D, except first and last tensor + # the tensor must be 2D, except first and last tensor x5 = paddle.static.data(name='x5', shape=[3, 2], dtype="float64") x6 = paddle.static.data(name='x6', shape=[2], dtype="float64") x7 = paddle.static.data(name='x7', shape=[2, 2], dtype="float64") @@ -246,7 +251,6 @@ def test_errors(self): class APITestMultiDot(unittest.TestCase): - def test_out(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): @@ -256,11 +260,9 @@ def test_out(self): exe = paddle.static.Executor(paddle.CPUPlace()) data1 = np.random.rand(3, 2).astype("float64") data2 = np.random.rand(2, 3).astype("float64") - np_res, = exe.run(feed={ - 'x0': data1, - 'x1': data2 - }, - fetch_list=[result]) + (np_res,) = exe.run( + feed={'x0': data1, 'x1': data2}, fetch_list=[result] + ) expected_result = np.linalg.multi_dot([data1, data2]) np.testing.assert_allclose( @@ -269,7 +271,9 @@ def test_out(self): rtol=1e-05, atol=1e-05, err_msg='two value is {}\n{}, check diff!'.format( - np_res, expected_result)) + np_res, expected_result + ), + ) def test_dygraph_without_out(self): paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_multi_label_soft_margin_loss.py b/python/paddle/fluid/tests/unittests/test_multi_label_soft_margin_loss.py index b26007a83ab86..b5236553ca665 100644 --- a/python/paddle/fluid/tests/unittests/test_multi_label_soft_margin_loss.py +++ b/python/paddle/fluid/tests/unittests/test_multi_label_soft_margin_loss.py @@ -24,7 +24,8 @@ def call_MultiLabelSoftMarginLoss_layer( reduction='mean', ): multilabel_margin_loss = paddle.nn.MultiLabelSoftMarginLoss( - weight=weight, reduction=reduction) + weight=weight, reduction=reduction + ) res = multilabel_margin_loss( input=input, label=label, @@ -47,55 +48,52 @@ def call_MultiLabelSoftMarginLoss_functional( return res -def test_static(place, - input_np, - label_np, - weight_np=None, - reduction='mean', - functional=False): +def test_static( + place, + input_np, + label_np, + weight_np=None, + reduction='mean', + functional=False, +): paddle.enable_static() prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.static.data(name='input', - shape=input_np.shape, - dtype='float64') - label = paddle.static.data(name='label', - shape=label_np.shape, - dtype='float64') + input = paddle.static.data( + name='input', shape=input_np.shape, dtype='float64' + ) + label = paddle.static.data( + name='label', shape=label_np.shape, dtype='float64' + ) feed_dict = { "input": input_np, "label": label_np, } weight = None if weight_np is not None: - weight = paddle.static.data(name='weight', - shape=weight_np.shape, - dtype='float64') + weight = paddle.static.data( + name='weight', shape=weight_np.shape, dtype='float64' + ) feed_dict['weight'] = weight_np if functional: - res = call_MultiLabelSoftMarginLoss_functional(input=input, - label=label, - weight=weight, - reduction=reduction) + res = call_MultiLabelSoftMarginLoss_functional( + input=input, label=label, weight=weight, reduction=reduction + ) else: - res = call_MultiLabelSoftMarginLoss_layer(input=input, - label=label, - weight=weight, - reduction=reduction) + res = call_MultiLabelSoftMarginLoss_layer( + input=input, label=label, weight=weight, reduction=reduction + ) exe = paddle.static.Executor(place) - static_result, = exe.run(prog, feed=feed_dict, fetch_list=[res]) + (static_result,) = exe.run(prog, feed=feed_dict, fetch_list=[res]) return static_result -def test_dygraph(place, - input_np, - label_np, - weight=None, - reduction='mean', - functional=False): +def test_dygraph( + place, input_np, label_np, weight=None, reduction='mean', functional=False +): with paddle.fluid.dygraph.base.guard(): input = paddle.to_tensor(input_np) label = paddle.to_tensor(label_np) @@ -104,12 +102,12 @@ def test_dygraph(place, if functional: dy_res = call_MultiLabelSoftMarginLoss_functional( - input=input, label=label, weight=weight, reduction=reduction) + input=input, label=label, weight=weight, reduction=reduction + ) else: - dy_res = call_MultiLabelSoftMarginLoss_layer(input=input, - label=label, - weight=weight, - reduction=reduction) + dy_res = call_MultiLabelSoftMarginLoss_layer( + input=input, label=label, weight=weight, reduction=reduction + ) dy_result = dy_res.numpy() return dy_result @@ -120,7 +118,6 @@ def calc_multilabel_margin_loss( weight=None, reduction="mean", ): - def LogSigmoid(x): return np.log(1 / (1 + np.exp(-x))) @@ -140,7 +137,6 @@ def LogSigmoid(x): class TestMultiLabelMarginLoss(unittest.TestCase): - def test_MultiLabelSoftMarginLoss(self): input = np.random.uniform(0.1, 0.8, size=(5, 5)).astype(np.float64) label = np.random.randint(0, 2, size=(5, 5)).astype(np.float64) @@ -151,52 +147,64 @@ def test_MultiLabelSoftMarginLoss(self): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - expected = calc_multilabel_margin_loss(input=input, - label=label, - reduction=reduction) - - dy_result = test_dygraph(place=place, - input_np=input, - label_np=label, - reduction=reduction) - - static_result = test_static(place=place, - input_np=input, - label_np=label, - reduction=reduction) + expected = calc_multilabel_margin_loss( + input=input, label=label, reduction=reduction + ) + + dy_result = test_dygraph( + place=place, + input_np=input, + label_np=label, + reduction=reduction, + ) + + static_result = test_static( + place=place, + input_np=input, + label_np=label, + reduction=reduction, + ) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) - static_functional = test_static(place=place, - input_np=input, - label_np=label, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - input_np=input, - label_np=label, - reduction=reduction, - functional=True) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-05) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-05) + static_functional = test_static( + place=place, + input_np=input, + label_np=label, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + input_np=input, + label_np=label, + reduction=reduction, + functional=True, + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-05 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-05 + ) np.testing.assert_allclose(dy_functional, expected, rtol=1e-05) def test_MultiLabelSoftMarginLoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.MultiLabelSoftMarginLoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.MultiLabelSoftMarginLoss, + reduction="unsupport reduction", + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([[0.0, 1.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.multi_label_soft_margin_loss, - input=input, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.multi_label_soft_margin_loss, + input=input, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() def test_MultiLabelSoftMarginLoss_weights(self): @@ -205,37 +213,44 @@ def test_MultiLabelSoftMarginLoss_weights(self): weight = np.random.randint(0, 2, size=(5, 5)).astype(np.float64) place = 'cpu' reduction = 'mean' - expected = calc_multilabel_margin_loss(input=input, - label=label, - weight=weight, - reduction=reduction) - - dy_result = test_dygraph(place=place, - input_np=input, - label_np=label, - weight=weight, - reduction=reduction) - - static_result = test_static(place=place, - input_np=input, - label_np=label, - weight_np=weight, - reduction=reduction) + expected = calc_multilabel_margin_loss( + input=input, label=label, weight=weight, reduction=reduction + ) + + dy_result = test_dygraph( + place=place, + input_np=input, + label_np=label, + weight=weight, + reduction=reduction, + ) + + static_result = test_static( + place=place, + input_np=input, + label_np=label, + weight_np=weight, + reduction=reduction, + ) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) - static_functional = test_static(place=place, - input_np=input, - label_np=label, - weight_np=weight, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - input_np=input, - label_np=label, - weight=weight, - reduction=reduction, - functional=True) + static_functional = test_static( + place=place, + input_np=input, + label_np=label, + weight_np=weight, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + input_np=input, + label_np=label, + weight=weight, + reduction=reduction, + functional=True, + ) np.testing.assert_allclose(static_functional, expected, rtol=1e-05) np.testing.assert_allclose(static_functional, dy_functional, rtol=1e-05) np.testing.assert_allclose(dy_functional, expected, rtol=1e-05) @@ -245,10 +260,12 @@ def test_MultiLabelSoftMarginLoss_dimension(self): input = paddle.to_tensor([[0.1, 0.3], [1, 2]], dtype='float32') label = paddle.to_tensor([[0.2, 0.1]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.multi_label_soft_margin_loss, - input=input, - label=label) + self.assertRaises( + ValueError, + paddle.nn.functional.multi_label_soft_margin_loss, + input=input, + label=label, + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py b/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py index bd03b646ddc4a..cde6c8daf96be 100644 --- a/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py +++ b/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np @@ -18,42 +18,70 @@ from op_test import OpTest import paddle import paddle.fluid as fluid -from paddle.fluid import Program, program_guard, in_dygraph_mode, _non_static_mode +from paddle.fluid import ( + Program, + program_guard, + in_dygraph_mode, + _non_static_mode, +) from paddle.fluid.layer_helper import LayerHelper from paddle import _C_ops, _legacy_C_ops -def multiclass_nms3(bboxes, - scores, - rois_num=None, - score_threshold=0.3, - nms_top_k=1000, - keep_top_k=100, - nms_threshold=0.3, - normalized=True, - nms_eta=1., - background_label=-1, - return_index=True, - return_rois_num=True, - name=None): +def multiclass_nms3( + bboxes, + scores, + rois_num=None, + score_threshold=0.3, + nms_top_k=1000, + keep_top_k=100, + nms_threshold=0.3, + normalized=True, + nms_eta=1.0, + background_label=-1, + return_index=True, + return_rois_num=True, + name=None, +): helper = LayerHelper('multiclass_nms3', **locals()) if in_dygraph_mode(): - attrs = (score_threshold, nms_top_k, keep_top_k, nms_threshold, - normalized, nms_eta, background_label) + attrs = ( + score_threshold, + nms_top_k, + keep_top_k, + nms_threshold, + normalized, + nms_eta, + background_label, + ) output, index, nms_rois_num = _C_ops.multiclass_nms3( - bboxes, scores, rois_num, *attrs) + bboxes, scores, rois_num, *attrs + ) if not return_index: index = None return output, index, nms_rois_num elif _non_static_mode(): - attrs = ('background_label', background_label, 'score_threshold', - score_threshold, 'nms_top_k', nms_top_k, 'nms_threshold', - nms_threshold, 'keep_top_k', keep_top_k, 'nms_eta', nms_eta, - 'normalized', normalized) + attrs = ( + 'background_label', + background_label, + 'score_threshold', + score_threshold, + 'nms_top_k', + nms_top_k, + 'nms_threshold', + nms_threshold, + 'keep_top_k', + keep_top_k, + 'nms_eta', + nms_eta, + 'normalized', + normalized, + ) output, index, nms_rois_num = _legacy_C_ops.multiclass_nms3( - bboxes, scores, rois_num, *attrs) + bboxes, scores, rois_num, *attrs + ) if not return_index: index = None return output, index, nms_rois_num @@ -70,21 +98,24 @@ def multiclass_nms3(bboxes, if return_rois_num: nms_rois_num = helper.create_variable_for_type_inference( - dtype='int32') + dtype='int32' + ) outputs['NmsRoisNum'] = nms_rois_num - helper.append_op(type="multiclass_nms3", - inputs=inputs, - attrs={ - 'background_label': background_label, - 'score_threshold': score_threshold, - 'nms_top_k': nms_top_k, - 'nms_threshold': nms_threshold, - 'keep_top_k': keep_top_k, - 'nms_eta': nms_eta, - 'normalized': normalized - }, - outputs=outputs) + helper.append_op( + type="multiclass_nms3", + inputs=inputs, + attrs={ + 'background_label': background_label, + 'score_threshold': score_threshold, + 'nms_top_k': nms_top_k, + 'nms_threshold': nms_threshold, + 'keep_top_k': keep_top_k, + 'nms_eta': nms_eta, + 'normalized': normalized, + }, + outputs=outputs, + ) output.stop_gradient = True index.stop_gradient = True if not return_index: @@ -98,14 +129,13 @@ def multiclass_nms3(bboxes, def softmax(x): # clip to shiftx, otherwise, when calc loss with # log(exp(shiftx)), may get log(0)=INF - shiftx = (x - np.max(x)).clip(-64.) + shiftx = (x - np.max(x)).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) def iou(box_a, box_b, norm): - """Apply intersection-over-union overlap between box_a and box_b - """ + """Apply intersection-over-union overlap between box_a and box_b""" xmin_a = min(box_a[0], box_a[2]) ymin_a = min(box_a[1], box_a[3]) xmax_a = max(box_a[0], box_a[2]) @@ -116,10 +146,12 @@ def iou(box_a, box_b, norm): xmax_b = max(box_b[0], box_b[2]) ymax_b = max(box_b[1], box_b[3]) - area_a = (ymax_a - ymin_a + (norm == False)) * (xmax_a - xmin_a + - (norm == False)) - area_b = (ymax_b - ymin_b + (norm == False)) * (xmax_b - xmin_b + - (norm == False)) + area_a = (ymax_a - ymin_a + (norm == False)) * ( + xmax_a - xmin_a + (norm == False) + ) + area_b = (ymax_b - ymin_b + (norm == False)) * ( + xmax_b - xmin_b + (norm == False) + ) if area_a <= 0 and area_b <= 0: return 0.0 @@ -128,21 +160,24 @@ def iou(box_a, box_b, norm): xb = min(xmax_a, xmax_b) yb = min(ymax_a, ymax_b) - inter_area = max(xb - xa + - (norm == False), 0.0) * max(yb - ya + (norm == False), 0.0) + inter_area = max(xb - xa + (norm == False), 0.0) * max( + yb - ya + (norm == False), 0.0 + ) iou_ratio = inter_area / (area_a + area_b - inter_area) return iou_ratio -def nms(boxes, - scores, - score_threshold, - nms_threshold, - top_k=200, - normalized=True, - eta=1.0): +def nms( + boxes, + scores, + score_threshold, + nms_threshold, + top_k=200, + normalized=True, + eta=1.0, +): """Apply non-maximum suppression at test time to avoid detecting too many overlapping bounding boxes for a given object. Args: @@ -189,8 +224,17 @@ def nms(boxes, return selected_indices -def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, - nms_top_k, keep_top_k, normalized, shared): +def multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + normalized, + shared, +): if shared: class_num = scores.shape[0] priorbox_num = scores.shape[1] @@ -201,13 +245,26 @@ def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, selected_indices = {} num_det = 0 for c in range(class_num): - if c == background: continue + if c == background: + continue if shared: - indices = nms(boxes, scores[c], score_threshold, nms_threshold, - nms_top_k, normalized) + indices = nms( + boxes, + scores[c], + score_threshold, + nms_threshold, + nms_top_k, + normalized, + ) else: - indices = nms(boxes[:, c, :], scores[:, c], score_threshold, - nms_threshold, nms_top_k, normalized) + indices = nms( + boxes[:, c, :], + scores[:, c], + score_threshold, + nms_threshold, + nms_top_k, + normalized, + ) selected_indices[c] = indices num_det += len(indices) @@ -220,9 +277,9 @@ def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, else: score_index.append((scores[idx][c], c, idx)) - sorted_score_index = sorted(score_index, - key=lambda tup: tup[0], - reverse=True) + sorted_score_index = sorted( + score_index, key=lambda tup: tup[0], reverse=True + ) sorted_score_index = sorted_score_index[:keep_top_k] selected_indices = {} @@ -238,9 +295,17 @@ def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, return selected_indices, num_det -def lod_multiclass_nms(boxes, scores, background, score_threshold, - nms_threshold, nms_top_k, keep_top_k, box_lod, - normalized): +def lod_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + box_lod, + normalized, +): num_class = boxes.shape[1] det_outs = [] lod = [] @@ -249,19 +314,21 @@ def lod_multiclass_nms(boxes, scores, background, score_threshold, if box_lod[0][n] == 0: lod.append(0) continue - box = boxes[head:head + box_lod[0][n]] - score = scores[head:head + box_lod[0][n]] + box = boxes[head : head + box_lod[0][n]] + score = scores[head : head + box_lod[0][n]] offset = head head = head + box_lod[0][n] - nmsed_outs, nmsed_num = multiclass_nms(box, - score, - background, - score_threshold, - nms_threshold, - nms_top_k, - keep_top_k, - normalized, - shared=False) + nmsed_outs, nmsed_num = multiclass_nms( + box, + score, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + normalized, + shared=False, + ) lod.append(nmsed_num) if nmsed_num == 0: @@ -270,41 +337,52 @@ def lod_multiclass_nms(boxes, scores, background, score_threshold, for c, indices in nmsed_outs.items(): for idx in indices: xmin, ymin, xmax, ymax = box[idx, c, :] - tmp_det_out.append([ - c, score[idx][c], xmin, ymin, xmax, ymax, - offset * num_class + idx * num_class + c - ]) - sorted_det_out = sorted(tmp_det_out, - key=lambda tup: tup[0], - reverse=False) + tmp_det_out.append( + [ + c, + score[idx][c], + xmin, + ymin, + xmax, + ymax, + offset * num_class + idx * num_class + c, + ] + ) + sorted_det_out = sorted( + tmp_det_out, key=lambda tup: tup[0], reverse=False + ) det_outs.extend(sorted_det_out) return det_outs, lod -def batched_multiclass_nms(boxes, - scores, - background, - score_threshold, - nms_threshold, - nms_top_k, - keep_top_k, - normalized=True): +def batched_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + normalized=True, +): batch_size = scores.shape[0] num_boxes = scores.shape[2] det_outs = [] index_outs = [] lod = [] for n in range(batch_size): - nmsed_outs, nmsed_num = multiclass_nms(boxes[n], - scores[n], - background, - score_threshold, - nms_threshold, - nms_top_k, - keep_top_k, - normalized, - shared=True) + nmsed_outs, nmsed_num = multiclass_nms( + boxes[n], + scores[n], + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + normalized, + shared=True, + ) lod.append(nmsed_num) if nmsed_num == 0: @@ -313,19 +391,25 @@ def batched_multiclass_nms(boxes, for c, indices in nmsed_outs.items(): for idx in indices: xmin, ymin, xmax, ymax = boxes[n][idx][:] - tmp_det_out.append([ - c, scores[n][c][idx], xmin, ymin, xmax, ymax, - idx + n * num_boxes - ]) - sorted_det_out = sorted(tmp_det_out, - key=lambda tup: tup[0], - reverse=False) + tmp_det_out.append( + [ + c, + scores[n][c][idx], + xmin, + ymin, + xmax, + ymax, + idx + n * num_boxes, + ] + ) + sorted_det_out = sorted( + tmp_det_out, key=lambda tup: tup[0], reverse=False + ) det_outs.extend(sorted_det_out) return det_outs, lod class TestMulticlassNMSOp(OpTest): - def set_argument(self): self.score_threshold = 0.01 @@ -351,9 +435,15 @@ def setUp(self): boxes[:, :, 0:2] = boxes[:, :, 0:2] * 0.5 boxes[:, :, 2:4] = boxes[:, :, 2:4] * 0.5 + 0.5 - det_outs, lod = batched_multiclass_nms(boxes, scores, background, - score_threshold, nms_threshold, - nms_top_k, keep_top_k) + det_outs, lod = batched_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + ) lod = [1] if not det_outs else lod det_outs = [[-1, 0]] if not det_outs else det_outs det_outs = np.array(det_outs) @@ -377,7 +467,6 @@ def test_check_output(self): class TestMulticlassNMSOpNoOutput(TestMulticlassNMSOp): - def set_argument(self): # Here set 2.0 to test the case there is no outputs. # In practical use, 0.0 < score_threshold < 1.0 @@ -385,7 +474,6 @@ def set_argument(self): class TestMulticlassNMSLoDInput(OpTest): - def set_argument(self): self.score_threshold = 0.01 @@ -412,13 +500,21 @@ def setUp(self): boxes[:, :, 2] = boxes[:, :, 2] * 10 + 10 boxes[:, :, 3] = boxes[:, :, 3] * 10 + 10 - det_outs, lod = lod_multiclass_nms(boxes, scores, background, - score_threshold, nms_threshold, - nms_top_k, keep_top_k, box_lod, - normalized) + det_outs, lod = lod_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + box_lod, + normalized, + ) det_outs = np.array(det_outs).astype('float32') - nmsed_outs = det_outs[:, :-1].astype('float32') if len( - det_outs) else det_outs + nmsed_outs = ( + det_outs[:, :-1].astype('float32') if len(det_outs) else det_outs + ) self.op_type = 'multiclass_nms' self.inputs = { 'BBoxes': (boxes, box_lod), @@ -440,7 +536,6 @@ def test_check_output(self): class TestMulticlassNMSNoBox(TestMulticlassNMSLoDInput): - def setUp(self): self.set_argument() M = 1200 @@ -464,13 +559,21 @@ def setUp(self): boxes[:, :, 2] = boxes[:, :, 2] * 10 + 10 boxes[:, :, 3] = boxes[:, :, 3] * 10 + 10 - det_outs, lod = lod_multiclass_nms(boxes, scores, background, - score_threshold, nms_threshold, - nms_top_k, keep_top_k, box_lod, - normalized) + det_outs, lod = lod_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + box_lod, + normalized, + ) det_outs = np.array(det_outs).astype('float32') - nmsed_outs = det_outs[:, :-1].astype('float32') if len( - det_outs) else det_outs + nmsed_outs = ( + det_outs[:, :-1].astype('float32') if len(det_outs) else det_outs + ) self.op_type = 'multiclass_nms' self.inputs = { 'BBoxes': (boxes, box_lod), @@ -489,7 +592,6 @@ def setUp(self): class TestIOU(unittest.TestCase): - def test_iou(self): box1 = np.array([4.0, 3.0, 7.0, 5.0]).astype('float32') box2 = np.array([3.0, 4.0, 6.0, 8.0]).astype('float32') @@ -500,7 +602,6 @@ def test_iou(self): class TestMulticlassNMS2Op(TestMulticlassNMSOp): - def setUp(self): self.set_argument() N = 7 @@ -523,20 +624,28 @@ def setUp(self): boxes[:, :, 0:2] = boxes[:, :, 0:2] * 0.5 boxes[:, :, 2:4] = boxes[:, :, 2:4] * 0.5 + 0.5 - det_outs, lod = batched_multiclass_nms(boxes, scores, background, - score_threshold, nms_threshold, - nms_top_k, keep_top_k) + det_outs, lod = batched_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + ) det_outs = np.array(det_outs) - nmsed_outs = det_outs[:, :-1].astype('float32') if len( - det_outs) else det_outs - index_outs = det_outs[:, - -1:].astype('int') if len(det_outs) else det_outs + nmsed_outs = ( + det_outs[:, :-1].astype('float32') if len(det_outs) else det_outs + ) + index_outs = ( + det_outs[:, -1:].astype('int') if len(det_outs) else det_outs + ) self.op_type = 'multiclass_nms2' self.inputs = {'BBoxes': boxes, 'Scores': scores} self.outputs = { 'Out': (nmsed_outs, [lod]), - 'Index': (index_outs, [lod]) + 'Index': (index_outs, [lod]), } self.attrs = { 'background_label': 0, @@ -553,7 +662,6 @@ def test_check_output(self): class TestMulticlassNMS2OpNoOutput(TestMulticlassNMS2Op): - def set_argument(self): # Here set 2.0 to test the case there is no outputs. # In practical use, 0.0 < score_threshold < 1.0 @@ -561,7 +669,6 @@ def set_argument(self): class TestMulticlassNMS2LoDInput(TestMulticlassNMSLoDInput): - def setUp(self): self.set_argument() M = 1200 @@ -585,16 +692,25 @@ def setUp(self): boxes[:, :, 2] = boxes[:, :, 2] * 10 + 10 boxes[:, :, 3] = boxes[:, :, 3] * 10 + 10 - det_outs, lod = lod_multiclass_nms(boxes, scores, background, - score_threshold, nms_threshold, - nms_top_k, keep_top_k, box_lod, - normalized) + det_outs, lod = lod_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + box_lod, + normalized, + ) det_outs = np.array(det_outs) - nmsed_outs = det_outs[:, :-1].astype('float32') if len( - det_outs) else det_outs - index_outs = det_outs[:, - -1:].astype('int') if len(det_outs) else det_outs + nmsed_outs = ( + det_outs[:, :-1].astype('float32') if len(det_outs) else det_outs + ) + index_outs = ( + det_outs[:, -1:].astype('int') if len(det_outs) else det_outs + ) self.op_type = 'multiclass_nms2' self.inputs = { 'BBoxes': (boxes, box_lod), @@ -602,7 +718,7 @@ def setUp(self): } self.outputs = { 'Out': (nmsed_outs, [lod]), - 'Index': (index_outs, [lod]) + 'Index': (index_outs, [lod]), } self.attrs = { 'background_label': 0, @@ -620,7 +736,6 @@ def test_check_output(self): class TestMulticlassNMS2LoDNoOutput(TestMulticlassNMS2LoDInput): - def set_argument(self): # Here set 2.0 to test the case there is no outputs. # In practical use, 0.0 < score_threshold < 1.0 @@ -628,7 +743,6 @@ def set_argument(self): class TestMulticlassNMSError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): M = 1200 @@ -642,12 +756,12 @@ def test_errors(self): scores = np.reshape(scores, (N, M, C)) scores_np = np.transpose(scores, (0, 2, 1)) - boxes_data = fluid.data(name='bboxes', - shape=[M, C, BOX_SIZE], - dtype='float32') - scores_data = fluid.data(name='scores', - shape=[N, C, M], - dtype='float32') + boxes_data = fluid.data( + name='bboxes', shape=[M, C, BOX_SIZE], dtype='float32' + ) + scores_data = fluid.data( + name='scores', shape=[N, C, M], dtype='float32' + ) def test_bboxes_Variable(): # the bboxes type must be Variable @@ -662,7 +776,6 @@ def test_scores_Variable(): class TestMulticlassNMS3Op(TestMulticlassNMS2Op): - def setUp(self): self.python_api = multiclass_nms3 self.set_argument() @@ -686,21 +799,29 @@ def setUp(self): boxes[:, :, 0:2] = boxes[:, :, 0:2] * 0.5 boxes[:, :, 2:4] = boxes[:, :, 2:4] * 0.5 + 0.5 - det_outs, lod = batched_multiclass_nms(boxes, scores, background, - score_threshold, nms_threshold, - nms_top_k, keep_top_k) + det_outs, lod = batched_multiclass_nms( + boxes, + scores, + background, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + ) det_outs = np.array(det_outs) - nmsed_outs = det_outs[:, :-1].astype('float32') if len( - det_outs) else det_outs - index_outs = det_outs[:, - -1:].astype('int') if len(det_outs) else det_outs + nmsed_outs = ( + det_outs[:, :-1].astype('float32') if len(det_outs) else det_outs + ) + index_outs = ( + det_outs[:, -1:].astype('int') if len(det_outs) else det_outs + ) self.op_type = 'multiclass_nms3' self.inputs = {'BBoxes': boxes, 'Scores': scores} self.outputs = { 'Out': nmsed_outs, 'Index': index_outs, - 'NmsRoisNum': np.array(lod).astype('int32') + 'NmsRoisNum': np.array(lod).astype('int32'), } self.attrs = { 'background_label': 0, @@ -717,7 +838,6 @@ def test_check_output(self): class TestMulticlassNMS3OpNoOutput(TestMulticlassNMS3Op): - def set_argument(self): # Here set 2.0 to test the case there is no outputs. # In practical use, 0.0 < score_threshold < 1.0 diff --git a/python/paddle/fluid/tests/unittests/test_multihead_attention.py b/python/paddle/fluid/tests/unittests/test_multihead_attention.py index 589b62d764192..8c6b3f96f31a4 100644 --- a/python/paddle/fluid/tests/unittests/test_multihead_attention.py +++ b/python/paddle/fluid/tests/unittests/test_multihead_attention.py @@ -19,42 +19,44 @@ class TestMultiheadAttention(unittest.TestCase): - def gen_random_input(self): - """Generate random input data. - """ + """Generate random input data.""" # batch_size, max_sequence_length, hidden dimension self.input_shape = (3, 13, 16) self.queries = np.random.random(size=self.input_shape).astype("float32") self.keys = np.random.random(size=self.input_shape).astype("float32") def set_program(self): - """Build the test program. - """ - queries = fluid.layers.data(name="queries", - shape=self.input_shape, - dtype="float32", - append_batch_size=False) + """Build the test program.""" + queries = fluid.layers.data( + name="queries", + shape=self.input_shape, + dtype="float32", + append_batch_size=False, + ) queries.stop_gradient = False - keys = fluid.layers.data(name="keys", - shape=self.input_shape, - dtype="float32", - append_batch_size=False) + keys = fluid.layers.data( + name="keys", + shape=self.input_shape, + dtype="float32", + append_batch_size=False, + ) keys.stop_gradient = False - contexts = fluid.nets.scaled_dot_product_attention(queries=queries, - keys=keys, - values=keys, - num_heads=8, - dropout_rate=0.) + contexts = fluid.nets.scaled_dot_product_attention( + queries=queries, + keys=keys, + values=keys, + num_heads=8, + dropout_rate=0.0, + ) out = fluid.layers.reduce_sum(contexts, dim=None) fluid.backward.append_backward(loss=out) self.fetch_list = [contexts] def run_program(self): - """Run the test program. - """ + """Run the test program.""" places = [core.CPUPlace()] if core.is_compiled_with_cuda(): places.append(core.CUDAPlace(0)) @@ -64,15 +66,16 @@ def run_program(self): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - output = exe.run(fluid.default_main_program(), - feed=self.inputs, - fetch_list=self.fetch_list, - return_numpy=True) + output = exe.run( + fluid.default_main_program(), + feed=self.inputs, + fetch_list=self.fetch_list, + return_numpy=True, + ) self.op_output = output def set_inputs(self, place): - """Set the randomly generated data to the test program. - """ + """Set the randomly generated data to the test program.""" self.inputs = {} queries = fluid.Tensor() queries.set(self.queries, place) @@ -89,7 +92,7 @@ def test_multihead_attention(self): self.set_program() self.run_program() - #fixme(caoying) add more meaningfull unittest. + # fixme(caoying) add more meaningfull unittest. if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_multimarginloss.py b/python/paddle/fluid/tests/unittests/test_multimarginloss.py index 1eff1deb69295..3a54f752eb2fb 100644 --- a/python/paddle/fluid/tests/unittests/test_multimarginloss.py +++ b/python/paddle/fluid/tests/unittests/test_multimarginloss.py @@ -26,10 +26,9 @@ def call_MultiMarginLoss_layer( weight=None, reduction='mean', ): - triplet_margin_loss = paddle.nn.MultiMarginLoss(p=p, - margin=margin, - weight=weight, - reduction=reduction) + triplet_margin_loss = paddle.nn.MultiMarginLoss( + p=p, margin=margin, weight=weight, reduction=reduction + ) res = triplet_margin_loss( input=input, label=label, @@ -45,82 +44,92 @@ def call_MultiMarginLoss_functional( weight=None, reduction='mean', ): - res = paddle.nn.functional.multi_margin_loss(input=input, - label=label, - p=p, - margin=margin, - weight=weight, - reduction=reduction) + res = paddle.nn.functional.multi_margin_loss( + input=input, + label=label, + p=p, + margin=margin, + weight=weight, + reduction=reduction, + ) return res -def test_static(place, - input_np, - label_np, - p=1, - margin=1.0, - weight_np=None, - reduction='mean', - functional=False): +def test_static( + place, + input_np, + label_np, + p=1, + margin=1.0, + weight_np=None, + reduction='mean', + functional=False, +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.static.data(name='input', - shape=input_np.shape, - dtype=input_np.dtype) - label = paddle.static.data(name='label', - shape=label_np.shape, - dtype=label_np.dtype) + input = paddle.static.data( + name='input', shape=input_np.shape, dtype=input_np.dtype + ) + label = paddle.static.data( + name='label', shape=label_np.shape, dtype=label_np.dtype + ) feed_dict = { "input": input_np, "label": label_np, } weight = None if weight_np is not None: - weight = paddle.static.data(name='weight', - shape=weight_np.shape, - dtype=weight_np.dtype) + weight = paddle.static.data( + name='weight', shape=weight_np.shape, dtype=weight_np.dtype + ) feed_dict['weight'] = weight_np if functional: - res = call_MultiMarginLoss_functional(input=input, - label=label, - p=p, - margin=margin, - weight=weight, - reduction=reduction) + res = call_MultiMarginLoss_functional( + input=input, + label=label, + p=p, + margin=margin, + weight=weight, + reduction=reduction, + ) else: - res = call_MultiMarginLoss_layer(input=input, - label=label, - p=p, - margin=margin, - weight=weight, - reduction=reduction) + res = call_MultiMarginLoss_layer( + input=input, + label=label, + p=p, + margin=margin, + weight=weight, + reduction=reduction, + ) exe = paddle.static.Executor(place) static_result = exe.run(prog, feed=feed_dict, fetch_list=[res]) return static_result[0] -def test_static_data_shape(place, - input_np, - label_np, - wrong_label_shape=None, - weight_np=None, - wrong_weight_shape=None, - functional=False): +def test_static_data_shape( + place, + input_np, + label_np, + wrong_label_shape=None, + weight_np=None, + wrong_weight_shape=None, + functional=False, +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.static.data(name='input', - shape=input_np.shape, - dtype=input_np.dtype) + input = paddle.static.data( + name='input', shape=input_np.shape, dtype=input_np.dtype + ) if wrong_label_shape is None: label_shape = label_np.shape else: label_shape = wrong_label_shape - label = paddle.static.data(name='label', - shape=label_shape, - dtype=label_np.dtype) + label = paddle.static.data( + name='label', shape=label_shape, dtype=label_np.dtype + ) feed_dict = { "input": input_np, "label": label_np, @@ -131,9 +140,9 @@ def test_static_data_shape(place, weight_shape = weight_np.shape else: weight_shape = wrong_weight_shape - weight = paddle.static.data(name='weight', - shape=weight_shape, - dtype=weight_np.dtype) + weight = paddle.static.data( + name='weight', shape=weight_shape, dtype=weight_np.dtype + ) feed_dict['weight'] = weight_np if functional: res = call_MultiMarginLoss_functional( @@ -153,14 +162,16 @@ def test_static_data_shape(place, return static_result -def test_dygraph(place, - input, - label, - p=1, - margin=1.0, - weight=None, - reduction='mean', - functional=False): +def test_dygraph( + place, + input, + label, + p=1, + margin=1.0, + weight=None, + reduction='mean', + functional=False, +): paddle.disable_static() input = paddle.to_tensor(input) label = paddle.to_tensor(label) @@ -168,19 +179,23 @@ def test_dygraph(place, if weight is not None: weight = paddle.to_tensor(weight) if functional: - dy_res = call_MultiMarginLoss_functional(input=input, - label=label, - p=p, - margin=margin, - weight=weight, - reduction=reduction) + dy_res = call_MultiMarginLoss_functional( + input=input, + label=label, + p=p, + margin=margin, + weight=weight, + reduction=reduction, + ) else: - dy_res = call_MultiMarginLoss_layer(input=input, - label=label, - p=p, - margin=margin, - weight=weight, - reduction=reduction) + dy_res = call_MultiMarginLoss_layer( + input=input, + label=label, + p=p, + margin=margin, + weight=weight, + reduction=reduction, + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result @@ -194,16 +209,22 @@ def calc_multi_margin_loss( weight=None, reduction='mean', ): - index_sample = np.array([input[i, label[i]] - for i in range(label.size)]).reshape(-1, 1) + index_sample = np.array( + [input[i, label[i]] for i in range(label.size)] + ).reshape(-1, 1) if weight is None: - expected = np.mean(np.maximum(margin + input - index_sample, 0.0)**p, - axis=1) - margin**p / input.shape[1] + expected = ( + np.mean(np.maximum(margin + input - index_sample, 0.0) ** p, axis=1) + - margin**p / input.shape[1] + ) else: - weight = np.array([weight[label[i]] - for i in range(label.size)]).reshape(-1, 1) - expected = np.mean(np.maximum(weight * (margin + input - index_sample), 0.0) ** p, axis=1) - weight*(margin ** p / \ - input.shape[1]) + weight = np.array( + [weight[label[i]] for i in range(label.size)] + ).reshape(-1, 1) + expected = np.mean( + np.maximum(weight * (margin + input - index_sample), 0.0) ** p, + axis=1, + ) - weight * (margin**p / input.shape[1]) if reduction == 'mean': expected = np.mean(expected) @@ -216,14 +237,14 @@ def calc_multi_margin_loss( class TestMultiMarginLoss(unittest.TestCase): - def test_MultiMarginLoss(self): batch_size = 5 num_classes = 2 shape = (batch_size, num_classes) input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) - label = np.random.uniform(0, input.shape[1], - size=(batch_size, )).astype(np.int64) + label = np.random.uniform(0, input.shape[1], size=(batch_size,)).astype( + np.int64 + ) places = [paddle.CPUPlace()] if paddle.device.is_compiled_with_cuda(): @@ -231,9 +252,9 @@ def test_MultiMarginLoss(self): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - expected = calc_multi_margin_loss(input=input, - label=label, - reduction=reduction) + expected = calc_multi_margin_loss( + input=input, label=label, reduction=reduction + ) dy_result = test_dygraph( place=place, @@ -251,32 +272,40 @@ def test_MultiMarginLoss(self): np.testing.assert_allclose(static_result, expected) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected) - static_functional = test_static(place=place, - input_np=input, - label_np=label, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - input=input, - label=label, - reduction=reduction, - functional=True) + static_functional = test_static( + place=place, + input_np=input, + label_np=label, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + input=input, + label=label, + reduction=reduction, + functional=True, + ) np.testing.assert_allclose(static_functional, expected) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected) def test_MultiMarginLoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.MultiMarginLoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.MultiMarginLoss, + reduction="unsupport reduction", + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([0], dtype='int32') - self.assertRaises(ValueError, - paddle.nn.functional.multi_margin_loss, - input=input, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.multi_margin_loss, + input=input, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() def test_MultiMarginLoss_dimension(self): @@ -308,12 +337,12 @@ def test_MultiMarginLoss_p(self): reduction = 'mean' place = paddle.CPUPlace() input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) - label = np.random.uniform(0, input.shape[1], - size=(batch_size, )).astype(np.int64) - expected = calc_multi_margin_loss(input=input, - p=p, - label=label, - reduction=reduction) + label = np.random.uniform(0, input.shape[1], size=(batch_size,)).astype( + np.int64 + ) + expected = calc_multi_margin_loss( + input=input, p=p, label=label, reduction=reduction + ) dy_result = test_dygraph( place=place, @@ -333,18 +362,22 @@ def test_MultiMarginLoss_p(self): np.testing.assert_allclose(static_result, expected) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected) - static_functional = test_static(place=place, - p=p, - input_np=input, - label_np=label, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - p=p, - input=input, - label=label, - reduction=reduction, - functional=True) + static_functional = test_static( + place=place, + p=p, + input_np=input, + label_np=label, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + p=p, + input=input, + label=label, + reduction=reduction, + functional=True, + ) np.testing.assert_allclose(static_functional, expected) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected) @@ -356,14 +389,13 @@ def test_MultiMarginLoss_weight(self): reduction = 'mean' place = paddle.CPUPlace() input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) - label = np.random.uniform(0, input.shape[1], - size=(batch_size, )).astype(np.int64) - weight = np.random.uniform(0, 2, - size=(num_classes, )).astype(np.float64) - expected = calc_multi_margin_loss(input=input, - label=label, - weight=weight, - reduction=reduction) + label = np.random.uniform(0, input.shape[1], size=(batch_size,)).astype( + np.int64 + ) + weight = np.random.uniform(0, 2, size=(num_classes,)).astype(np.float64) + expected = calc_multi_margin_loss( + input=input, label=label, weight=weight, reduction=reduction + ) dy_result = test_dygraph( place=place, @@ -383,18 +415,22 @@ def test_MultiMarginLoss_weight(self): np.testing.assert_allclose(static_result, expected) np.testing.assert_allclose(static_result, dy_result) np.testing.assert_allclose(dy_result, expected) - static_functional = test_static(place=place, - input_np=input, - label_np=label, - weight_np=weight, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - input=input, - label=label, - weight=weight, - reduction=reduction, - functional=True) + static_functional = test_static( + place=place, + input_np=input, + label_np=label, + weight_np=weight, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + input=input, + label=label, + weight=weight, + reduction=reduction, + functional=True, + ) np.testing.assert_allclose(static_functional, expected) np.testing.assert_allclose(static_functional, dy_functional) np.testing.assert_allclose(dy_functional, expected) @@ -405,10 +441,10 @@ def test_MultiMarginLoss_static_data_shape(self): shape = (batch_size, num_classes) place = paddle.CPUPlace() input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) - label = np.random.uniform(0, input.shape[1], - size=(batch_size, )).astype(np.int64) - weight = np.random.uniform(0, 2, - size=(num_classes, )).astype(np.float64) + label = np.random.uniform(0, input.shape[1], size=(batch_size,)).astype( + np.int64 + ) + weight = np.random.uniform(0, 2, size=(num_classes,)).astype(np.float64) self.assertRaises( ValueError, @@ -416,7 +452,7 @@ def test_MultiMarginLoss_static_data_shape(self): place=place, input_np=input, label_np=label, - wrong_label_shape=(10, ), + wrong_label_shape=(10,), functional=True, ) self.assertRaises( @@ -425,7 +461,7 @@ def test_MultiMarginLoss_static_data_shape(self): place=place, input_np=input, label_np=label, - wrong_label_shape=(10, ), + wrong_label_shape=(10,), functional=False, ) self.assertRaises( @@ -435,7 +471,7 @@ def test_MultiMarginLoss_static_data_shape(self): input_np=input, label_np=label, weight_np=weight, - wrong_weight_shape=(3, ), + wrong_weight_shape=(3,), functional=True, ) self.assertRaises( @@ -445,7 +481,7 @@ def test_MultiMarginLoss_static_data_shape(self): input_np=input, label_np=label, weight_np=weight, - wrong_weight_shape=(3, ), + wrong_weight_shape=(3,), functional=False, ) diff --git a/python/paddle/fluid/tests/unittests/test_multinomial_op.py b/python/paddle/fluid/tests/unittests/test_multinomial_op.py index 5df70185dda58..f79dee65250d5 100644 --- a/python/paddle/fluid/tests/unittests/test_multinomial_op.py +++ b/python/paddle/fluid/tests/unittests/test_multinomial_op.py @@ -43,7 +43,6 @@ def sample_output_two_dimension(out, shape): class TestMultinomialOp(OpTest): - def setUp(self): paddle.enable_static() self.op_type = "multinomial" @@ -66,16 +65,16 @@ def verify_output(self, outs): # normalize the input to get the probability prob = self.input_np / self.input_np.sum(axis=-1, keepdims=True) sample_prob = self.sample_output(np.array(outs[0])) - np.testing.assert_allclose(sample_prob, - prob, - rtol=0, - atol=0.01, - err_msg='sample_prob: ' + str(sample_prob) + - '\nprob: ' + str(prob)) + np.testing.assert_allclose( + sample_prob, + prob, + rtol=0, + atol=0.01, + err_msg='sample_prob: ' + str(sample_prob) + '\nprob: ' + str(prob), + ) class TestMultinomialOp2(TestMultinomialOp): - def init_data(self): # input probability is a matrix self.input_np = np.random.rand(3, 4) @@ -87,7 +86,6 @@ def sample_output(self, out): class TestMultinomialOp3(TestMultinomialOp): - def init_data(self): # replacement is False. number of samples must be less than number of categories. self.input_np = np.random.rand(1000) @@ -98,12 +96,13 @@ def verify_output(self, outs): out = np.array(outs[0]) unique_out = np.unique(out) self.assertEqual( - len(unique_out), 100, - "replacement is False. categories can't be sampled repeatedly") + len(unique_out), + 100, + "replacement is False. categories can't be sampled repeatedly", + ) class TestMultinomialApi(unittest.TestCase): - def test_dygraph(self): # input probability is a vector, and replacement is True paddle.disable_static() @@ -114,12 +113,13 @@ def test_dygraph(self): sample_prob = sample_output_one_dimension(out.numpy(), 4) prob = x_numpy / x_numpy.sum(axis=-1, keepdims=True) - np.testing.assert_allclose(sample_prob, - prob, - rtol=0, - atol=0.01, - err_msg='sample_prob: ' + str(sample_prob) + - '\nprob: ' + str(prob)) + np.testing.assert_allclose( + sample_prob, + prob, + rtol=0, + atol=0.01, + err_msg='sample_prob: ' + str(sample_prob) + '\nprob: ' + str(prob), + ) def test_dygraph2(self): # input probability is a matrix, and replacement is True @@ -130,12 +130,13 @@ def test_dygraph2(self): sample_prob = sample_output_two_dimension(out.numpy(), [3, 4]) prob = x_numpy / x_numpy.sum(axis=-1, keepdims=True) - np.testing.assert_allclose(sample_prob, - prob, - rtol=0, - atol=0.01, - err_msg='sample_prob: ' + str(sample_prob) + - '\nprob: ' + str(prob)) + np.testing.assert_allclose( + sample_prob, + prob, + rtol=0, + atol=0.01, + err_msg='sample_prob: ' + str(sample_prob) + '\nprob: ' + str(prob), + ) paddle.enable_static() def test_dygraph3(self): @@ -147,8 +148,10 @@ def test_dygraph3(self): unique_out = np.unique(out.numpy()) self.assertEqual( - len(unique_out), 100, - "replacement is False. categories can't be sampled repeatedly") + len(unique_out), + 100, + "replacement is False. categories can't be sampled repeatedly", + ) paddle.enable_static() def test_dygraph4(self): @@ -178,16 +181,16 @@ def test_static(self): sample_prob = sample_output_one_dimension(out, 4) prob = x_np / x_np.sum(axis=-1, keepdims=True) - np.testing.assert_allclose(sample_prob, - prob, - rtol=0, - atol=0.01, - err_msg='sample_prob: ' + str(sample_prob) + - '\nprob: ' + str(prob)) + np.testing.assert_allclose( + sample_prob, + prob, + rtol=0, + atol=0.01, + err_msg='sample_prob: ' + str(sample_prob) + '\nprob: ' + str(prob), + ) class TestMultinomialAlias(unittest.TestCase): - def test_alias(self): paddle.disable_static() x = paddle.rand([4]) @@ -197,12 +200,10 @@ def test_alias(self): class TestMultinomialError(unittest.TestCase): - def setUp(self): paddle.disable_static() def test_num_sample(self): - def test_num_sample_less_than_0(): x = paddle.rand([4]) paddle.multinomial(x, num_samples=-2) @@ -210,7 +211,6 @@ def test_num_sample_less_than_0(): self.assertRaises(ValueError, test_num_sample_less_than_0) def test_replacement_False(self): - def test_samples_larger_than_categories(): x = paddle.rand([4]) paddle.multinomial(x, num_samples=5, replacement=False) @@ -218,7 +218,6 @@ def test_samples_larger_than_categories(): self.assertRaises(ValueError, test_samples_larger_than_categories) def test_input_probs_dim(self): - def test_dim_larger_than_2(): x = paddle.rand([2, 3, 3]) paddle.multinomial(x) @@ -233,7 +232,7 @@ def test_dim_less_than_1(): self.assertRaises(ValueError, test_dim_less_than_1) with self.assertRaises(ValueError): - y = paddle.multinomial(paddle.to_tensor([1., 2., -3.])) + y = paddle.multinomial(paddle.to_tensor([1.0, 2.0, -3.0])) with self.assertRaises(ValueError): prob = paddle.rand([20, 1000]) @@ -242,7 +241,6 @@ def test_dim_less_than_1(): class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -294,7 +292,6 @@ def test_fixed_random_number(self): class TestMultinomialTensorNumSamples(UnittestBase): - def init_info(self): self.shapes = [[3, 4]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -326,8 +323,9 @@ def test_static(self): exe = paddle.static.Executor() exe.run(starup_prog) res = exe.run(fetch_list=[feat, out]) - paddle.static.save_inference_model(self.save_path, [x], [feat, out], - exe) + paddle.static.save_inference_model( + self.save_path, [x], [feat, out], exe + ) np.testing.assert_equal(res[1].shape, (3, 3)) # Test for Inference Predictor diff --git a/python/paddle/fluid/tests/unittests/test_multiplex_op.py b/python/paddle/fluid/tests/unittests/test_multiplex_op.py index 10ad40082f777..75a08f99eccac 100644 --- a/python/paddle/fluid/tests/unittests/test_multiplex_op.py +++ b/python/paddle/fluid/tests/unittests/test_multiplex_op.py @@ -21,7 +21,6 @@ class TestMultiplexOp(OpTest): - def setUp(self): self.op_type = "multiplex" rows = 4 @@ -34,7 +33,7 @@ def setUp(self): ins4 = np.random.random((rows, 25)).astype("float64") self.inputs = { 'Ids': index, - 'X': [('x1', ins1), ('x2', ins2), ('x3', ins3), ('x4', ins4)] + 'X': [('x1', ins1), ('x2', ins2), ('x3', ins3), ('x4', ins4)], } # multiplex output output = np.zeros_like(ins1) @@ -60,7 +59,6 @@ def test_check_grad_ignore_x3(self): class TestMultiplexOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): x1 = fluid.data(name='x1', shape=[None, 2], dtype='int64') @@ -86,16 +84,15 @@ def test_type(): self.assertRaises(TypeError, test_type) def test_type2(): - index2 = fluid.data(name='index2', - shape=[None, 1], - dtype='int16') + index2 = fluid.data( + name='index2', shape=[None, 1], dtype='int16' + ) paddle.multiplex(inputs=[x1, x2], index=index2) self.assertRaises(TypeError, test_type2) class TestMultiplexODygrap(unittest.TestCase): - def test_multiplex_dygraph(self): paddle.disable_static() img1 = np.array([[1, 2], [3, 4]]).astype(np.float32) @@ -118,18 +115,25 @@ def test_dygraph_api(self): with _test_eager_guard(): inputs_eager = [paddle.to_tensor(img1), paddle.to_tensor(img2)] index_eager = paddle.to_tensor( - np.array([[1], [0]]).astype(np.int32)) + np.array([[1], [0]]).astype(np.int32) + ) inputs_eager[0].stop_gradient = False inputs_eager[1].stop_gradient = False res_eager = paddle.multiplex(inputs_eager, index_eager) res_eager.backward() self.assertEqual((res.numpy() == res_eager.numpy()).all(), True) self.assertEqual( - (inputs[0].grad.numpy() == inputs_eager[0].grad.numpy() - ).all(), True) + ( + inputs[0].grad.numpy() == inputs_eager[0].grad.numpy() + ).all(), + True, + ) self.assertEqual( - (inputs[1].grad.numpy() == inputs_eager[1].grad.numpy() - ).all(), True) + ( + inputs[1].grad.numpy() == inputs_eager[1].grad.numpy() + ).all(), + True, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_multiply.py b/python/paddle/fluid/tests/unittests/test_multiply.py index e32245990a847..a2c213ade9c4c 100755 --- a/python/paddle/fluid/tests/unittests/test_multiply.py +++ b/python/paddle/fluid/tests/unittests/test_multiply.py @@ -23,27 +23,28 @@ class TestMultiplyApi(unittest.TestCase): - def _run_static_graph_case(self, x_data, y_data): with program_guard(Program(), Program()): paddle.enable_static() - x = paddle.static.data(name='x', - shape=x_data.shape, - dtype=x_data.dtype) - y = paddle.static.data(name='y', - shape=y_data.shape, - dtype=y_data.dtype) + x = paddle.static.data( + name='x', shape=x_data.shape, dtype=x_data.dtype + ) + y = paddle.static.data( + name='y', shape=y_data.shape, dtype=y_data.dtype + ) res = tensor.multiply(x, y) - place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) - outs = exe.run(paddle.static.default_main_program(), - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[res]) + outs = exe.run( + paddle.static.default_main_program(), + feed={'x': x_data, 'y': y_data}, + fetch_list=[res], + ) res = outs[0] return res @@ -112,7 +113,6 @@ def test_multiply(self): class TestMultiplyError(unittest.TestCase): - def func_test_errors(self): # test static computation graph: dtype can not be int8 paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py index 4cf28bbd30b19..f3872a388c061 100755 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dataset.py @@ -17,15 +17,20 @@ import paddle import paddle.fluid as fluid -from paddle.io import Dataset, IterableDataset, TensorDataset, \ - ComposeDataset, ChainDataset, DataLoader +from paddle.io import ( + Dataset, + IterableDataset, + TensorDataset, + ComposeDataset, + ChainDataset, + DataLoader, +) from paddle.fluid.framework import _test_eager_guard IMAGE_SIZE = 32 class RandomDataset(Dataset): - def __init__(self, sample_num): self.sample_num = sample_num @@ -35,12 +40,11 @@ def __len__(self): def __getitem__(self, idx): np.random.seed(idx) image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, 9, (1, )).astype('int64') + label = np.random.randint(0, 9, (1,)).astype('int64') return image, label class RandomIterableDataset(IterableDataset): - def __init__(self, sample_num): self.sample_num = sample_num @@ -48,12 +52,11 @@ def __iter__(self): for i in range(self.sample_num): np.random.seed(i) image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, 9, (1, )).astype('int64') + label = np.random.randint(0, 9, (1,)).astype('int64') yield image, label class TestTensorDataset(unittest.TestCase): - def run_main(self, num_workers, places): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -66,21 +69,25 @@ def run_main(self, num_workers, places): dataset = TensorDataset([input, label]) assert len(dataset) == 16 - dataloader = DataLoader(dataset, - places=place, - num_workers=num_workers, - batch_size=1, - drop_last=True) + dataloader = DataLoader( + dataset, + places=place, + num_workers=num_workers, + batch_size=1, + drop_last=True, + ) for i, (input, label) in enumerate(dataloader()): assert len(input) == 1 assert len(label) == 1 assert input.shape == [1, 3, 4] assert label.shape == [1, 1] - assert isinstance(input, - (fluid.core.VarBase, fluid.core.eager.Tensor)) - assert isinstance(label, - (fluid.core.VarBase, fluid.core.eager.Tensor)) + assert isinstance( + input, (fluid.core.VarBase, fluid.core.eager.Tensor) + ) + assert isinstance( + label, (fluid.core.VarBase, fluid.core.eager.Tensor) + ) assert np.allclose(input.numpy(), input_np[i]) assert np.allclose(label.numpy(), label_np[i]) @@ -98,7 +105,6 @@ def test_main(self): class TestComposeDataset(unittest.TestCase): - def func_test_main(self): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -124,7 +130,6 @@ def test_main(self): class TestRandomSplitApi(unittest.TestCase): - def func_test_main(self): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -151,7 +156,6 @@ def test_main(self): class TestRandomSplitError(unittest.TestCase): - def func_test_errors(self): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -167,7 +171,6 @@ def test_errors(self): class TestSubsetDataset(unittest.TestCase): - def run_main(self, num_workers, places): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -184,11 +187,13 @@ def run_main(self, num_workers, places): assert len(dataset) == 5 def prepare_dataloader(dataset): - return DataLoader(dataset, - places=places, - num_workers=num_workers, - batch_size=1, - drop_last=True) + return DataLoader( + dataset, + places=places, + num_workers=num_workers, + batch_size=1, + drop_last=True, + ) dataloader = prepare_dataloader(dataset) dataloader_even = prepare_dataloader(even_subset) @@ -199,10 +204,12 @@ def assert_basic(input, label): assert len(label) == 1 assert input.shape == [1, 3, 4] assert label.shape == [1, 1] - assert isinstance(input, - (fluid.core.VarBase, fluid.core.eager.Tensor)) - assert isinstance(label, - (fluid.core.VarBase, fluid.core.eager.Tensor)) + assert isinstance( + input, (fluid.core.VarBase, fluid.core.eager.Tensor) + ) + assert isinstance( + label, (fluid.core.VarBase, fluid.core.eager.Tensor) + ) elements_list = list() for _, (input, label) in enumerate(dataloader()): @@ -237,7 +244,6 @@ def test_main(self): class TestChainDataset(unittest.TestCase): - def run_main(self, num_workers, places): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -275,7 +281,6 @@ def test_main(self): class NumpyMixTensorDataset(Dataset): - def __init__(self, sample_num): self.sample_num = sample_num @@ -285,12 +290,11 @@ def __len__(self): def __getitem__(self, idx): np.random.seed(idx) image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, 9, (1, )).astype('int64') + label = np.random.randint(0, 9, (1,)).astype('int64') return paddle.to_tensor(image, place=paddle.CPUPlace()), label class TestNumpyMixTensorDataset(TestTensorDataset): - def run_main(self, num_workers, places): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -298,25 +302,28 @@ def run_main(self, num_workers, places): with fluid.dygraph.guard(place): dataset = NumpyMixTensorDataset(16) assert len(dataset) == 16 - dataloader = DataLoader(dataset, - places=place, - num_workers=num_workers, - batch_size=1, - drop_last=True) + dataloader = DataLoader( + dataset, + places=place, + num_workers=num_workers, + batch_size=1, + drop_last=True, + ) for i, (input, label) in enumerate(dataloader()): assert len(input) == 1 assert len(label) == 1 assert input.shape == [1, IMAGE_SIZE] assert label.shape == [1, 1] - assert isinstance(input, - (fluid.core.VarBase, fluid.core.eager.Tensor)) - assert isinstance(label, - (fluid.core.VarBase, fluid.core.eager.Tensor)) + assert isinstance( + input, (fluid.core.VarBase, fluid.core.eager.Tensor) + ) + assert isinstance( + label, (fluid.core.VarBase, fluid.core.eager.Tensor) + ) class ComplextDataset(Dataset): - def __init__(self, sample_num): self.sample_num = sample_num @@ -324,18 +331,19 @@ def __len__(self): return self.sample_num def __getitem__(self, idx): - return (3.1, 'abc', - paddle.to_tensor(np.random.random([IMAGE_SIZE - ]).astype('float32'), - place=paddle.CPUPlace()), - [1, np.random.random([2]).astype('float32')], { - 'a': 2.0, - 'b': np.random.random([2]).astype('float32') - }) + return ( + 3.1, + 'abc', + paddle.to_tensor( + np.random.random([IMAGE_SIZE]).astype('float32'), + place=paddle.CPUPlace(), + ), + [1, np.random.random([2]).astype('float32')], + {'a': 2.0, 'b': np.random.random([2]).astype('float32')}, + ) class TestComplextDataset(unittest.TestCase): - def run_main(self, num_workers): paddle.static.default_startup_program().random_seed = 1 paddle.static.default_main_program().random_seed = 1 @@ -343,11 +351,13 @@ def run_main(self, num_workers): with fluid.dygraph.guard(place): dataset = ComplextDataset(16) assert len(dataset) == 16 - dataloader = DataLoader(dataset, - places=place, - num_workers=num_workers, - batch_size=2, - drop_last=True) + dataloader = DataLoader( + dataset, + places=place, + num_workers=num_workers, + batch_size=2, + drop_last=True, + ) for i, data in enumerate(dataloader()): assert len(data) == 5 @@ -380,7 +390,6 @@ def test_main(self): class SingleFieldDataset(Dataset): - def __init__(self, sample_num): self.sample_num = sample_num @@ -392,7 +401,6 @@ def __getitem__(self, idx): class TestSingleFieldDataset(unittest.TestCase): - def init_dataset(self): self.sample_num = 16 self.dataset = SingleFieldDataset(self.sample_num) @@ -403,15 +411,18 @@ def run_main(self, num_workers): place = paddle.CPUPlace() with fluid.dygraph.guard(place): self.init_dataset() - dataloader = DataLoader(self.dataset, - places=place, - num_workers=num_workers, - batch_size=2, - drop_last=True) + dataloader = DataLoader( + self.dataset, + places=place, + num_workers=num_workers, + batch_size=2, + drop_last=True, + ) for i, data in enumerate(dataloader()): - assert isinstance(data, - (fluid.core.VarBase, fluid.core.eager.Tensor)) + assert isinstance( + data, (fluid.core.VarBase, fluid.core.eager.Tensor) + ) assert data.shape == [2, 2, 3] def func_test_main(self): @@ -425,7 +436,6 @@ def test_main(self): class SingleFieldIterableDataset(IterableDataset): - def __init__(self, sample_num): self.sample_num = sample_num @@ -435,22 +445,23 @@ def __iter__(self): class TestSingleFieldIterableDataset(TestSingleFieldDataset): - def init_dataset(self): self.sample_num = 16 self.dataset = SingleFieldIterableDataset(self.sample_num) class TestDataLoaderGenerateStates(unittest.TestCase): - def setUp(self): self.inputs = [(0, 1), (0, 2), (1, 3)] - self.outputs = [[1835504127, 1731038949, 1320224556, 2330041505], - [2834126987, 2358157858, 1860244682, 1437227251], - [457190280, 2660306227, 859341110, 354512857]] + self.outputs = [ + [1835504127, 1731038949, 1320224556, 2330041505], + [2834126987, 2358157858, 1860244682, 1437227251], + [457190280, 2660306227, 859341110, 354512857], + ] def func_test_main(self): from paddle.fluid.dataloader.worker import _generate_states + for inp, outp in zip(self.inputs, self.outputs): out = _generate_states(*inp) assert out == outp @@ -462,16 +473,18 @@ def test_main(self): class TestDatasetWithDropLast(unittest.TestCase): - def run_main(self, dataset, num_samples, batch_size): for num_workers in [0, 1]: for drop_last in [True, False]: - steps = (num_samples + (1 - int(drop_last)) * \ - (batch_size - 1)) // batch_size - dataloader = DataLoader(dataset, - batch_size=batch_size, - drop_last=drop_last, - num_workers=num_workers) + steps = ( + num_samples + (1 - int(drop_last)) * (batch_size - 1) + ) // batch_size + dataloader = DataLoader( + dataset, + batch_size=batch_size, + drop_last=drop_last, + num_workers=num_workers, + ) datas = [] for data in dataloader: datas.append(data) diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dynamic.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dynamic.py index d0e592d488c1e..e4d678d8d1707 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dynamic.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_dynamic.py @@ -21,35 +21,52 @@ from paddle.io import DataLoader from paddle.fluid.dygraph.nn import Linear -from test_multiprocess_dataloader_static import RandomDataset, RandomBatchedDataset, prepare_places -from test_multiprocess_dataloader_static import EPOCH_NUM, BATCH_SIZE, IMAGE_SIZE, SAMPLE_NUM, CLASS_NUM +from test_multiprocess_dataloader_static import ( + RandomDataset, + RandomBatchedDataset, + prepare_places, +) +from test_multiprocess_dataloader_static import ( + EPOCH_NUM, + BATCH_SIZE, + IMAGE_SIZE, + SAMPLE_NUM, + CLASS_NUM, +) class SimpleFCNet(fluid.dygraph.Layer): - def __init__(self): super(SimpleFCNet, self).__init__() - param_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.8)) - bias_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)) + param_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.8) + ) + bias_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ) self._fcs = [] in_channel = IMAGE_SIZE for hidden_size in [10, 20, 30]: self._fcs.append( - Linear(in_channel, - hidden_size, - act='tanh', - param_attr=param_attr, - bias_attr=bias_attr)) + Linear( + in_channel, + hidden_size, + act='tanh', + param_attr=param_attr, + bias_attr=bias_attr, + ) + ) in_channel = hidden_size self._fcs.append( - Linear(in_channel, - CLASS_NUM, - act='softmax', - param_attr=param_attr, - bias_attr=bias_attr)) + Linear( + in_channel, + CLASS_NUM, + act='softmax', + param_attr=param_attr, + bias_attr=bias_attr, + ) + ) def forward(self, image): out = image @@ -59,7 +76,6 @@ def forward(self, image): class TestDygraphDataLoader(unittest.TestCase): - def run_main(self, num_workers, places, persistent_workers): fluid.default_startup_program().random_seed = 1 fluid.default_main_program().random_seed = 1 @@ -68,11 +84,13 @@ def run_main(self, num_workers, places, persistent_workers): optimizer = fluid.optimizer.Adam(parameter_list=fc_net.parameters()) dataset = RandomDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - num_workers=num_workers, - batch_size=BATCH_SIZE, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + num_workers=num_workers, + batch_size=BATCH_SIZE, + drop_last=True, + persistent_workers=persistent_workers, + ) assert len(dataloader) == int(SAMPLE_NUM / BATCH_SIZE) step_list = [] @@ -96,7 +114,7 @@ def run_main(self, num_workers, places, persistent_workers): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret @@ -107,21 +125,27 @@ def test_main(self): for persistent_workers in [False, True]: results = [] for num_workers in [0, 2]: - print(self.__class__.__name__, p, num_workers, - persistent_workers) + print( + self.__class__.__name__, + p, + num_workers, + persistent_workers, + ) sys.stdout.flush() - ret = self.run_main(num_workers=num_workers, - places=p, - persistent_workers=persistent_workers) + ret = self.run_main( + num_workers=num_workers, + places=p, + persistent_workers=persistent_workers, + ) results.append(ret) diff = np.max( - np.abs(results[0]['loss'] - results[1]['loss']) / - np.abs(results[0]['loss'])) + np.abs(results[0]['loss'] - results[1]['loss']) + / np.abs(results[0]['loss']) + ) self.assertLess(diff, 1e-2) class TestDygraphDataLoaderWithBatchedDataset(TestDygraphDataLoader): - def run_main(self, num_workers, places, persistent_workers): fluid.default_startup_program().random_seed = 1 fluid.default_main_program().random_seed = 1 @@ -130,11 +154,13 @@ def run_main(self, num_workers, places, persistent_workers): optimizer = fluid.optimizer.Adam(parameter_list=fc_net.parameters()) dataset = RandomBatchedDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - num_workers=num_workers, - batch_size=None, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + num_workers=num_workers, + batch_size=None, + drop_last=True, + persistent_workers=persistent_workers, + ) assert len(dataloader) == int(SAMPLE_NUM / BATCH_SIZE) step_list = [] @@ -158,7 +184,7 @@ def run_main(self, num_workers, places, persistent_workers): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py index ea4b9363bfce2..f60c73f45773a 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py @@ -23,14 +23,13 @@ class RandomDataset(Dataset): - def __init__(self, sample_num): self.sample_num = sample_num def __getitem__(self, idx): np.random.seed(idx) image = np.random.random([784]).astype('float32') - label = np.random.randint(0, 9, (1, )).astype('int64') + label = np.random.randint(0, 9, (1,)).astype('int64') return image, label def __len__(self): @@ -38,7 +37,6 @@ def __len__(self): class TestDataLoaderAssert(unittest.TestCase): - def test_main(self): place = fluid.cpu_places()[0] with fluid.dygraph.guard(place): @@ -61,9 +59,9 @@ def test_main(self): # num_workers < 0 try: - loader = DataLoader(dataset=dataset, - places=place, - num_workers=-1) + loader = DataLoader( + dataset=dataset, places=place, num_workers=-1 + ) self.assertTrue(False) except AssertionError: pass @@ -77,27 +75,28 @@ def test_main(self): # set batch_sampler and shuffle/batch_size/drop_last try: - loader = DataLoader(dataset=dataset, - places=place, - batch_sampler=batch_sampler, - shuffle=True, - drop_last=True) + loader = DataLoader( + dataset=dataset, + places=place, + batch_sampler=batch_sampler, + shuffle=True, + drop_last=True, + ) self.assertTrue(False) except AssertionError: pass # set batch_sampler correctly try: - loader = DataLoader(dataset=dataset, - places=place, - batch_sampler=batch_sampler) + loader = DataLoader( + dataset=dataset, places=place, batch_sampler=batch_sampler + ) self.assertTrue(True) except AssertionError: self.assertTrue(False) class TestDatasetRuntimeError(unittest.TestCase): - def test_main(self): dataset = Dataset() @@ -141,10 +140,10 @@ def test_main(self): # CI Converage cannot record stub in subprocess, # HACK a _worker_loop in main process call here -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestDataLoaderWorkerLoop(unittest.TestCase): - def run_without_worker_done(self, use_shared_memory=True): try: place = fluid.cpu_places()[0] @@ -161,12 +160,15 @@ def _collate_fn(sample_list): np.stack(s, axis=0) for s in list(zip(*sample_list)) ] - loader = DataLoader(dataset, - num_workers=1, - places=place, - use_shared_memory=use_shared_memory) - assert loader.num_workers > 0, \ - "go to AssertionError and pass in Mac and Windows" + loader = DataLoader( + dataset, + num_workers=1, + places=place, + use_shared_memory=use_shared_memory, + ) + assert ( + loader.num_workers > 0 + ), "go to AssertionError and pass in Mac and Windows" loader = iter(loader) print("loader length", len(loader)) indices_queue = multiprocessing.Queue() @@ -174,16 +176,28 @@ def _collate_fn(sample_list): indices_queue.put([i, i + 10]) indices_queue.put(None) base_seed = 1234 - _worker_loop(loader._dataset, 0, indices_queue, - loader._data_queue, loader._workers_done_event, - True, _collate_fn, True, _init_fn, 0, 1, - loader._use_shared_memory, base_seed) + _worker_loop( + loader._dataset, + 0, + indices_queue, + loader._data_queue, + loader._workers_done_event, + True, + _collate_fn, + True, + _init_fn, + 0, + 1, + loader._use_shared_memory, + base_seed, + ) self.assertTrue(False) except AssertionError: pass except Exception as e: print("Exception", e) import sys + sys.stdout.flush() self.assertTrue(False) @@ -203,12 +217,15 @@ def _collate_fn(sample_list): np.stack(s, axis=0) for s in list(zip(*sample_list)) ] - loader = DataLoader(dataset, - num_workers=1, - places=place, - use_shared_memory=use_shared_memory) - assert loader.num_workers > 0, \ - "go to AssertionError and pass in Mac and Windows" + loader = DataLoader( + dataset, + num_workers=1, + places=place, + use_shared_memory=use_shared_memory, + ) + assert ( + loader.num_workers > 0 + ), "go to AssertionError and pass in Mac and Windows" loader = iter(loader) print("loader length", len(loader)) indices_queue = multiprocessing.Queue() @@ -217,10 +234,21 @@ def _collate_fn(sample_list): indices_queue.put(None) loader._workers_done_event.set() base_seed = 1234 - _worker_loop(loader._dataset, 0, indices_queue, - loader._data_queue, loader._workers_done_event, - True, _collate_fn, True, _init_fn, 0, 1, - loader._use_shared_memory, base_seed) + _worker_loop( + loader._dataset, + 0, + indices_queue, + loader._data_queue, + loader._workers_done_event, + True, + _collate_fn, + True, + _init_fn, + 0, + 1, + loader._use_shared_memory, + base_seed, + ) self.assertTrue(True) except AssertionError: pass diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_dynamic.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_dynamic.py index 98c8a19d520f4..fe5cf6e27a831 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_dynamic.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_dynamic.py @@ -21,35 +21,52 @@ from paddle.io import DataLoader from paddle.fluid.dygraph.nn import Linear -from test_multiprocess_dataloader_iterable_dataset_static import RandomDataset, RandomBatchedDataset, prepare_places -from test_multiprocess_dataloader_iterable_dataset_static import EPOCH_NUM, BATCH_SIZE, IMAGE_SIZE, SAMPLE_NUM, CLASS_NUM +from test_multiprocess_dataloader_iterable_dataset_static import ( + RandomDataset, + RandomBatchedDataset, + prepare_places, +) +from test_multiprocess_dataloader_iterable_dataset_static import ( + EPOCH_NUM, + BATCH_SIZE, + IMAGE_SIZE, + SAMPLE_NUM, + CLASS_NUM, +) class SimpleFCNet(fluid.dygraph.Layer): - def __init__(self): super(SimpleFCNet, self).__init__() - param_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.8)) - bias_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)) + param_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.8) + ) + bias_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ) self._fcs = [] in_channel = IMAGE_SIZE for hidden_size in [10, 20, 30]: self._fcs.append( - Linear(in_channel, - hidden_size, - act='tanh', - param_attr=param_attr, - bias_attr=bias_attr)) + Linear( + in_channel, + hidden_size, + act='tanh', + param_attr=param_attr, + bias_attr=bias_attr, + ) + ) in_channel = hidden_size self._fcs.append( - Linear(in_channel, - CLASS_NUM, - act='softmax', - param_attr=param_attr, - bias_attr=bias_attr)) + Linear( + in_channel, + CLASS_NUM, + act='softmax', + param_attr=param_attr, + bias_attr=bias_attr, + ) + ) def forward(self, image): out = image @@ -59,7 +76,6 @@ def forward(self, image): class TestDygraphDataLoader(unittest.TestCase): - def run_main(self, num_workers, places, persistent_workers): fluid.default_startup_program().random_seed = 1 fluid.default_main_program().random_seed = 1 @@ -68,11 +84,13 @@ def run_main(self, num_workers, places, persistent_workers): optimizer = fluid.optimizer.Adam(parameter_list=fc_net.parameters()) dataset = RandomDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - num_workers=num_workers, - batch_size=BATCH_SIZE, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + num_workers=num_workers, + batch_size=BATCH_SIZE, + drop_last=True, + persistent_workers=persistent_workers, + ) step_list = [] loss_list = [] @@ -95,7 +113,7 @@ def run_main(self, num_workers, places, persistent_workers): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret @@ -106,19 +124,26 @@ def test_main(self): for persistent_workers in [False, True]: results = [] for num_workers in [0, 2]: - print(self.__class__.__name__, p, num_workers, - persistent_workers) + print( + self.__class__.__name__, + p, + num_workers, + persistent_workers, + ) sys.stdout.flush() - ret = self.run_main(num_workers=num_workers, - places=p, - persistent_workers=persistent_workers) + ret = self.run_main( + num_workers=num_workers, + places=p, + persistent_workers=persistent_workers, + ) results.append(ret) - assert results[0]['loss'].shape[0] * 2 == results[1][ - 'loss'].shape[0] + assert ( + results[0]['loss'].shape[0] * 2 + == results[1]['loss'].shape[0] + ) class TestDygraphDataLoaderWithBatchedDataset(TestDygraphDataLoader): - def run_main(self, num_workers, places, persistent_workers): fluid.default_startup_program().random_seed = 1 fluid.default_main_program().random_seed = 1 @@ -127,11 +152,13 @@ def run_main(self, num_workers, places, persistent_workers): optimizer = fluid.optimizer.Adam(parameter_list=fc_net.parameters()) dataset = RandomBatchedDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - num_workers=num_workers, - batch_size=None, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + num_workers=num_workers, + batch_size=None, + drop_last=True, + persistent_workers=persistent_workers, + ) step_list = [] loss_list = [] @@ -154,7 +181,7 @@ def run_main(self, num_workers, places, persistent_workers): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_split.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_split.py index aef0c8c7d0812..57ae8bf46f863 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_split.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_split.py @@ -21,7 +21,6 @@ class RangeIterableDatasetSplit(IterableDataset): - def __init__(self, start, end): self.start = start self.end = end @@ -34,7 +33,9 @@ def __iter__(self): else: per_worker = int( math.ceil( - (self.end - self.start) / float(worker_info.num_workers))) + (self.end - self.start) / float(worker_info.num_workers) + ) + ) worker_id = worker_info.id iter_start = self.start + worker_id * per_worker iter_end = min(iter_start + per_worker, self.end) @@ -44,16 +45,17 @@ def __iter__(self): class TestDynamicDataLoaderIterSplit(unittest.TestCase): - def test_main(self): place = fluid.CPUPlace() with fluid.dygraph.guard(place): dataset = RangeIterableDatasetSplit(0, 10) - dataloader = DataLoader(dataset, - places=place, - num_workers=2, - batch_size=1, - drop_last=True) + dataloader = DataLoader( + dataset, + places=place, + num_workers=2, + batch_size=1, + drop_last=True, + ) rets = [] for d in dataloader: @@ -63,7 +65,6 @@ def test_main(self): class RangeIterableDataset(IterableDataset): - def __init__(self, start, end): self.start = start self.end = end @@ -74,7 +75,6 @@ def __iter__(self): class TestDynamicDataLoaderIterInitFuncSplit(unittest.TestCase): - def test_main(self): place = fluid.CPUPlace() with fluid.dygraph.guard(place): @@ -87,18 +87,21 @@ def worker_spliter(worker_id): start = dataset.start end = dataset.end num_per_worker = int( - math.ceil((end - start) / float(worker_info.num_workers))) + math.ceil((end - start) / float(worker_info.num_workers)) + ) worker_id = worker_info.id dataset.start = start + worker_id * num_per_worker dataset.end = min(dataset.start + num_per_worker, end) - dataloader = DataLoader(dataset, - places=place, - num_workers=1, - batch_size=1, - drop_last=True, - worker_init_fn=worker_spliter) + dataloader = DataLoader( + dataset, + places=place, + num_workers=1, + batch_size=1, + drop_last=True, + worker_init_fn=worker_spliter, + ) rets = [] for d in dataloader: diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_static.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_static.py index 4c78edf3adec5..49f12ea657e93 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_static.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_iterable_dataset_static.py @@ -28,7 +28,6 @@ class RandomDataset(IterableDataset): - def __init__(self, sample_num, class_num): self.sample_num = sample_num self.class_num = class_num @@ -37,8 +36,9 @@ def __iter__(self): for i in range(self.sample_num): np.random.seed(i) image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, self.class_num - 1, - (1, )).astype('int64') + label = np.random.randint(0, self.class_num - 1, (1,)).astype( + 'int64' + ) yield image, label @@ -50,29 +50,36 @@ def simple_fc_net_static(): with fluid.unique_name.guard(): with fluid.program_guard(main_prog, startup_prog): - image = fluid.data(name='image', - shape=[None, IMAGE_SIZE], - dtype='float32') + image = fluid.data( + name='image', shape=[None, IMAGE_SIZE], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') hidden = image - param_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.8)) - bias_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)) + param_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.8) + ) + bias_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ) for hidden_size in [10, 20, 30]: - hidden = fluid.layers.fc(hidden, - size=hidden_size, - act='tanh', - param_attr=param_attr, - bias_attr=bias_attr) - - predict_label = fluid.layers.fc(hidden, - size=CLASS_NUM, - act='softmax', - param_attr=param_attr, - bias_attr=bias_attr) + hidden = fluid.layers.fc( + hidden, + size=hidden_size, + act='tanh', + param_attr=param_attr, + bias_attr=bias_attr, + ) + + predict_label = fluid.layers.fc( + hidden, + size=CLASS_NUM, + act='softmax', + param_attr=param_attr, + bias_attr=bias_attr, + ) loss = fluid.layers.reduce_mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) @@ -96,21 +103,22 @@ def prepare_places(with_data_parallel, with_cpu=False, with_gpu=True): class TestStaticDataLoader(unittest.TestCase): - def run_main(self, num_workers, places, persistent_workers): scope = fluid.Scope() with fluid.scope_guard(scope): startup_prog, main_prog, image, label, loss = simple_fc_net_static() dataset = RandomDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - feed_list=[image, label], - places=places, - num_workers=num_workers, - batch_size=BATCH_SIZE, - return_list=False, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + feed_list=[image, label], + places=places, + num_workers=num_workers, + batch_size=BATCH_SIZE, + return_list=False, + drop_last=True, + persistent_workers=persistent_workers, + ) # assert len(dataloader) == int(SAMPLE_NUM / BATCH_SIZE) exe = fluid.Executor(place=places[0]) @@ -118,8 +126,9 @@ def run_main(self, num_workers, places, persistent_workers): prog = fluid.CompiledProgram(main_prog) if len(places) > 1: - prog = prog.with_data_parallel(loss_name=loss.name, - places=places) + prog = prog.with_data_parallel( + loss_name=loss.name, places=places + ) step_list = [] loss_list = [] @@ -128,7 +137,8 @@ def run_main(self, num_workers, places, persistent_workers): step = 0 for d in dataloader: assert len(d) == len(places), "{} != {}".format( - len(d), len(places)) + len(d), len(places) + ) for i, item in enumerate(d): image = item['image'] label = item['label'] @@ -136,10 +146,12 @@ def run_main(self, num_workers, places, persistent_workers): assert label.shape() == [BATCH_SIZE, 1] assert image._place()._equals(places[i]) assert label._place()._equals(places[i]) - L, = exe.run(program=prog, - feed=d, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + feed=d, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 step_list.append(step) @@ -148,7 +160,7 @@ def run_main(self, num_workers, places, persistent_workers): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret @@ -158,19 +170,26 @@ def test_main(self): for persistent_workers in [False, True]: results = [] for num_workers in [0, 2]: - print(self.__class__.__name__, p, num_workers, - persistent_workers) + print( + self.__class__.__name__, + p, + num_workers, + persistent_workers, + ) sys.stdout.flush() - ret = self.run_main(num_workers=num_workers, - places=p, - persistent_workers=persistent_workers) + ret = self.run_main( + num_workers=num_workers, + places=p, + persistent_workers=persistent_workers, + ) results.append(ret) - assert results[0]['loss'].shape[0] * 2 == results[1][ - 'loss'].shape[0] + assert ( + results[0]['loss'].shape[0] * 2 + == results[1]['loss'].shape[0] + ) class RandomBatchedDataset(IterableDataset): - def __init__(self, sample_num, class_num): self.sample_num = sample_num // BATCH_SIZE self.class_num = class_num @@ -182,37 +201,40 @@ def __iter__(self): labels = [] for _ in range(BATCH_SIZE): image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, self.class_num - 1, - (1, )).astype('int64') + label = np.random.randint(0, self.class_num - 1, (1,)).astype( + 'int64' + ) images.append(image) labels.append(label) yield np.stack(images, axis=0), np.stack(labels, axis=0) class TestStaticDataLoaderWithBatchedDataset(TestStaticDataLoader): - def run_main(self, num_workers, places, persistent_workers): scope = fluid.Scope() with fluid.scope_guard(scope): startup_prog, main_prog, image, label, loss = simple_fc_net_static() dataset = RandomBatchedDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - feed_list=[image, label], - places=places, - num_workers=num_workers, - batch_size=None, - return_list=False, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + feed_list=[image, label], + places=places, + num_workers=num_workers, + batch_size=None, + return_list=False, + drop_last=True, + persistent_workers=persistent_workers, + ) exe = fluid.Executor(place=places[0]) exe.run(startup_prog) prog = fluid.CompiledProgram(main_prog) if len(places) > 1: - prog = prog.with_data_parallel(loss_name=loss.name, - places=places) + prog = prog.with_data_parallel( + loss_name=loss.name, places=places + ) step_list = [] loss_list = [] @@ -221,7 +243,8 @@ def run_main(self, num_workers, places, persistent_workers): step = 0 for d in dataloader: assert len(d) == len(places), "{} != {}".format( - len(d), len(places)) + len(d), len(places) + ) for i, item in enumerate(d): image = item['image'] label = item['label'] @@ -229,10 +252,12 @@ def run_main(self, num_workers, places, persistent_workers): assert label.shape() == [BATCH_SIZE, 1] assert image._place()._equals(places[i]) assert label._place()._equals(places[i]) - L, = exe.run(program=prog, - feed=d, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + feed=d, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 step_list.append(step) @@ -241,7 +266,7 @@ def run_main(self, num_workers, places, persistent_workers): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_static.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_static.py index 6a44c9f48430a..853d7ec192405 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_static.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_static.py @@ -29,7 +29,6 @@ class RandomDataset(Dataset): - def __init__(self, sample_num, class_num): self.sample_num = sample_num self.class_num = class_num @@ -37,7 +36,7 @@ def __init__(self, sample_num, class_num): def __getitem__(self, idx): np.random.seed(idx) image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, self.class_num - 1, (1, )).astype('int64') + label = np.random.randint(0, self.class_num - 1, (1,)).astype('int64') return image, label def __len__(self): @@ -52,29 +51,36 @@ def simple_fc_net_static(): with fluid.unique_name.guard(): with fluid.program_guard(main_prog, startup_prog): - image = fluid.data(name='image', - shape=[None, IMAGE_SIZE], - dtype='float32') + image = fluid.data( + name='image', shape=[None, IMAGE_SIZE], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') hidden = image - param_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.8)) - bias_attr = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)) + param_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.8) + ) + bias_attr = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ) for hidden_size in [10, 20, 30]: - hidden = fluid.layers.fc(hidden, - size=hidden_size, - act='tanh', - param_attr=param_attr, - bias_attr=bias_attr) - - predict_label = fluid.layers.fc(hidden, - size=CLASS_NUM, - act='softmax', - param_attr=param_attr, - bias_attr=bias_attr) + hidden = fluid.layers.fc( + hidden, + size=hidden_size, + act='tanh', + param_attr=param_attr, + bias_attr=bias_attr, + ) + + predict_label = fluid.layers.fc( + hidden, + size=CLASS_NUM, + act='softmax', + param_attr=param_attr, + bias_attr=bias_attr, + ) loss = fluid.layers.reduce_mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) @@ -98,21 +104,22 @@ def prepare_places(with_data_parallel, with_cpu=False, with_gpu=True): class TestStaticDataLoader(unittest.TestCase): - def run_main(self, num_workers, places, persistent_workers, use_pe=True): scope = fluid.Scope() with fluid.scope_guard(scope): startup_prog, main_prog, image, label, loss = simple_fc_net_static() dataset = RandomDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - feed_list=[image, label], - places=places, - num_workers=num_workers, - batch_size=BATCH_SIZE, - return_list=False, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + feed_list=[image, label], + places=places, + num_workers=num_workers, + batch_size=BATCH_SIZE, + return_list=False, + drop_last=True, + persistent_workers=persistent_workers, + ) assert len(dataloader) == int(SAMPLE_NUM / BATCH_SIZE) exe = fluid.Executor(place=places[0]) @@ -121,8 +128,9 @@ def run_main(self, num_workers, places, persistent_workers, use_pe=True): if use_pe: prog = fluid.CompiledProgram(main_prog) if len(places) > 1: - prog = prog.with_data_parallel(loss_name=loss.name, - places=places) + prog = prog.with_data_parallel( + loss_name=loss.name, places=places + ) else: prog = main_prog @@ -133,7 +141,8 @@ def run_main(self, num_workers, places, persistent_workers, use_pe=True): step = 0 for d in dataloader: assert len(d) == len(places), "{} != {}".format( - len(d), len(places)) + len(d), len(places) + ) for i, item in enumerate(d): image = item['image'] label = item['label'] @@ -141,10 +150,12 @@ def run_main(self, num_workers, places, persistent_workers, use_pe=True): assert label.shape() == [BATCH_SIZE, 1] assert image._place()._equals(places[i]) assert label._place()._equals(places[i]) - L, = exe.run(program=prog, - feed=d, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + feed=d, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 step_list.append(step) @@ -153,7 +164,7 @@ def run_main(self, num_workers, places, persistent_workers, use_pe=True): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret @@ -163,35 +174,43 @@ def test_main(self): for persistent_workers in [True, False]: results = [] for num_workers in [0, 2]: - print(self.__class__.__name__, p, num_workers, - persistent_workers) + print( + self.__class__.__name__, + p, + num_workers, + persistent_workers, + ) sys.stdout.flush() - ret = self.run_main(num_workers=num_workers, - places=p, - persistent_workers=persistent_workers) + ret = self.run_main( + num_workers=num_workers, + places=p, + persistent_workers=persistent_workers, + ) results.append(ret) diff = np.max( - np.abs(results[0]['loss'] - results[1]['loss']) / - np.abs(results[0]['loss'])) + np.abs(results[0]['loss'] - results[1]['loss']) + / np.abs(results[0]['loss']) + ) self.assertLess(diff, 1e-2) class TestStaticDataLoaderReturnList(unittest.TestCase): - def run_single_place(self, num_workers): scope = fluid.Scope() - image = fluid.data(name='image', - shape=[None, IMAGE_SIZE], - dtype='float32') + image = fluid.data( + name='image', shape=[None, IMAGE_SIZE], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') with fluid.scope_guard(scope): dataset = RandomDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - feed_list=[image, label], - num_workers=num_workers, - batch_size=BATCH_SIZE, - drop_last=True, - return_list=True) + dataloader = DataLoader( + dataset, + feed_list=[image, label], + num_workers=num_workers, + batch_size=BATCH_SIZE, + drop_last=True, + return_list=True, + ) for d in dataloader: assert isinstance(d, list) @@ -201,19 +220,21 @@ def run_single_place(self, num_workers): def run_multi_place(self, num_workers): scope = fluid.Scope() - image = fluid.data(name='image', - shape=[None, IMAGE_SIZE], - dtype='float32') + image = fluid.data( + name='image', shape=[None, IMAGE_SIZE], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') with fluid.scope_guard(scope): dataset = RandomDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - feed_list=[image, label], - num_workers=num_workers, - batch_size=BATCH_SIZE, - places=[fluid.CPUPlace()] * 2, - drop_last=True, - return_list=True) + dataloader = DataLoader( + dataset, + feed_list=[image, label], + num_workers=num_workers, + batch_size=BATCH_SIZE, + places=[fluid.CPUPlace()] * 2, + drop_last=True, + return_list=True, + ) for d in dataloader: assert isinstance(d, list) @@ -229,7 +250,6 @@ def test_main(self): class RandomBatchedDataset(Dataset): - def __init__(self, sample_num, class_num): self.sample_num = int(sample_num / BATCH_SIZE) self.class_num = class_num @@ -240,8 +260,9 @@ def __getitem__(self, idx): labels = [] for _ in range(BATCH_SIZE): image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, self.class_num - 1, - (1, )).astype('int64') + label = np.random.randint(0, self.class_num - 1, (1,)).astype( + 'int64' + ) images.append(image) labels.append(label) return np.stack(images, axis=0), np.stack(labels, axis=0) @@ -251,21 +272,22 @@ def __len__(self): class TestStaticDataLoaderWithBatchedDataset(TestStaticDataLoader): - def run_main(self, num_workers, places, persistent_workers): scope = fluid.Scope() with fluid.scope_guard(scope): startup_prog, main_prog, image, label, loss = simple_fc_net_static() dataset = RandomBatchedDataset(SAMPLE_NUM, CLASS_NUM) - dataloader = DataLoader(dataset, - feed_list=[image, label], - places=places, - num_workers=num_workers, - batch_size=None, - return_list=False, - drop_last=True, - persistent_workers=persistent_workers) + dataloader = DataLoader( + dataset, + feed_list=[image, label], + places=places, + num_workers=num_workers, + batch_size=None, + return_list=False, + drop_last=True, + persistent_workers=persistent_workers, + ) assert len(dataloader) == int(SAMPLE_NUM / BATCH_SIZE) exe = fluid.Executor(place=places[0]) @@ -273,8 +295,9 @@ def run_main(self, num_workers, places, persistent_workers): prog = fluid.CompiledProgram(main_prog) if len(places) > 1: - prog = prog.with_data_parallel(loss_name=loss.name, - places=places) + prog = prog.with_data_parallel( + loss_name=loss.name, places=places + ) step_list = [] loss_list = [] @@ -283,7 +306,8 @@ def run_main(self, num_workers, places, persistent_workers): step = 0 for d in dataloader: assert len(d) == len(places), "{} != {}".format( - len(d), len(places)) + len(d), len(places) + ) for i, item in enumerate(d): image = item['image'] label = item['label'] @@ -291,10 +315,12 @@ def run_main(self, num_workers, places, persistent_workers): assert label.shape() == [BATCH_SIZE, 1] assert image._place()._equals(places[i]) assert label._place()._equals(places[i]) - L, = exe.run(program=prog, - feed=d, - fetch_list=[loss], - use_program_cache=True) + (L,) = exe.run( + program=prog, + feed=d, + fetch_list=[loss], + use_program_cache=True, + ) loss_list.append(np.mean(L)) step += 1 step_list.append(step) @@ -303,7 +329,7 @@ def run_main(self, num_workers, places, persistent_workers): ret = { "time": end_t - start_t, "step": step_list, - "loss": np.array(loss_list) + "loss": np.array(loss_list), } print("time cost", ret['time'], 'step_list', ret['step']) return ret diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_reader_exception.py b/python/paddle/fluid/tests/unittests/test_multiprocess_reader_exception.py index e1033ae881e81..3eae95ef34b05 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_reader_exception.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_reader_exception.py @@ -24,7 +24,6 @@ class ReaderException(Exception): class TestMultiprocessReaderExceptionWithQueueSuccess(unittest.TestCase): - def setUp(self): self.use_pipe = False self.raise_exception = False @@ -40,12 +39,12 @@ def main_impl(self, place, iterable): batch_size = 4 def fake_reader(): - def __impl__(): for _ in range(sample_num): if not self.raise_exception: - yield list(np.random.uniform(low=-1, high=1, - size=[10])), + yield list( + np.random.uniform(low=-1, high=1, size=[10]) + ), else: raise ValueError() @@ -53,23 +52,28 @@ def __impl__(): with fluid.program_guard(fluid.Program(), fluid.Program()): image = fluid.data(name='image', dtype='float32', shape=[None, 10]) - reader = fluid.io.DataLoader.from_generator(feed_list=[image], - capacity=2, - iterable=iterable) + reader = fluid.io.DataLoader.from_generator( + feed_list=[image], capacity=2, iterable=iterable + ) image_p_1 = image + 1 decorated_reader = multiprocess_reader( - [fake_reader(), fake_reader()], use_pipe=self.use_pipe) + [fake_reader(), fake_reader()], use_pipe=self.use_pipe + ) if isinstance(place, fluid.CUDAPlace): - reader.set_sample_generator(decorated_reader, - batch_size=batch_size, - places=fluid.cuda_places(0)) + reader.set_sample_generator( + decorated_reader, + batch_size=batch_size, + places=fluid.cuda_places(0), + ) else: - reader.set_sample_generator(decorated_reader, - batch_size=batch_size, - places=fluid.cpu_places(1)) + reader.set_sample_generator( + decorated_reader, + batch_size=batch_size, + places=fluid.cpu_places(1), + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -117,24 +121,24 @@ def test_main(self): class TestMultiprocessReaderExceptionWithQueueFailed( - TestMultiprocessReaderExceptionWithQueueSuccess): - + TestMultiprocessReaderExceptionWithQueueSuccess +): def setUp(self): self.use_pipe = False self.raise_exception = True class TestMultiprocessReaderExceptionWithPipeSuccess( - TestMultiprocessReaderExceptionWithQueueSuccess): - + TestMultiprocessReaderExceptionWithQueueSuccess +): def setUp(self): self.use_pipe = True self.raise_exception = False class TestMultiprocessReaderExceptionWithPipeFailed( - TestMultiprocessReaderExceptionWithQueueSuccess): - + TestMultiprocessReaderExceptionWithQueueSuccess +): def setUp(self): self.use_pipe = True self.raise_exception = True diff --git a/python/paddle/fluid/tests/unittests/test_mv_op.py b/python/paddle/fluid/tests/unittests/test_mv_op.py index fbfd18a477801..d2381a21e99fd 100644 --- a/python/paddle/fluid/tests/unittests/test_mv_op.py +++ b/python/paddle/fluid/tests/unittests/test_mv_op.py @@ -1,4 +1,4 @@ -#Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,7 +20,6 @@ class TestMVOp(OpTest): - def setUp(self): self.op_type = "mv" self.python_api = paddle.mv @@ -40,7 +39,6 @@ def init_config(self): class TestMVAPI(unittest.TestCase): - def test_dygraph_api_out(self): paddle.disable_static() @@ -68,12 +66,12 @@ def test_static_graph(self): self.input_vec = np.random.rand(100).astype("float64") with program_guard(train_program, startup_program): - data_x = paddle.static.data("x", - shape=[5, 100], - dtype="float64") - data_vec = paddle.static.data("vec", - shape=[100], - dtype="float64") + data_x = paddle.static.data( + "x", shape=[5, 100], dtype="float64" + ) + data_vec = paddle.static.data( + "vec", shape=[100], dtype="float64" + ) data_x.stop_gradient = x_stop_gradient data_vec.stop_gradient = vec_stop_gradient @@ -82,19 +80,16 @@ def test_static_graph(self): self.place = paddle.CPUPlace() exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "vec": self.input_vec - }, - fetch_list=[result_vec]) + (res,) = exe.run( + feed={"x": self.input_x, "vec": self.input_vec}, + fetch_list=[result_vec], + ) z_expected = np.array(np.dot(self.input_x, self.input_vec)) np.testing.assert_allclose(res, z_expected, rtol=1e-05) class TestMVError(unittest.TestCase): - def test_input(self): - def test_shape(): paddle.enable_static() @@ -102,9 +97,9 @@ def test_shape(): self.input_vec = np.random.rand(100).astype("float64") data_x = paddle.static.data("x", shape=[5, 100], dtype="float64") - data_vec = paddle.static.data("vec", - shape=[100, 2], - dtype="float64") + data_vec = paddle.static.data( + "vec", shape=[100, 2], dtype="float64" + ) result_vec = paddle.mv(data_x, data_vec) self.assertRaises(ValueError, test_shape) diff --git a/python/paddle/fluid/tests/unittests/test_naive_best_fit_gpu_memory_limit.py b/python/paddle/fluid/tests/unittests/test_naive_best_fit_gpu_memory_limit.py index 6994bf30523be..c6846d7068e0c 100644 --- a/python/paddle/fluid/tests/unittests/test_naive_best_fit_gpu_memory_limit.py +++ b/python/paddle/fluid/tests/unittests/test_naive_best_fit_gpu_memory_limit.py @@ -23,7 +23,6 @@ class TestBase(unittest.TestCase): - def setUp(self): if fluid.is_compiled_with_cuda(): self._limit = fluid.core.globals()['FLAGS_gpu_memory_limit_mb'] @@ -36,8 +35,10 @@ def test_allocate(self): place = fluid.CUDAPlace(0) t = fluid.LoDTensor() - t.set(np.ndarray([int(self._limit / 2), other_dim], dtype='float32'), - place) + t.set( + np.ndarray([int(self._limit / 2), other_dim], dtype='float32'), + place, + ) del t t = fluid.LoDTensor() diff --git a/python/paddle/fluid/tests/unittests/test_name_scope.py b/python/paddle/fluid/tests/unittests/test_name_scope.py index a0e37384eb2f0..475ebec9ecee2 100644 --- a/python/paddle/fluid/tests/unittests/test_name_scope.py +++ b/python/paddle/fluid/tests/unittests/test_name_scope.py @@ -17,7 +17,6 @@ class TestNameScope(unittest.TestCase): - def test_name_scope(self): with fluid.name_scope("s1"): a = fluid.layers.data(name='data', shape=[1], dtype='int32') diff --git a/python/paddle/fluid/tests/unittests/test_nan_inf.py b/python/paddle/fluid/tests/unittests/test_nan_inf.py index 65a537981ad25..f062adb6a7a43 100644 --- a/python/paddle/fluid/tests/unittests/test_nan_inf.py +++ b/python/paddle/fluid/tests/unittests/test_nan_inf.py @@ -22,7 +22,6 @@ class TestNanInf(unittest.TestCase): - def setUp(self): self._python_interp = sys.executable if os.getenv('WITH_COVERAGE', 'OFF') == 'ON': @@ -33,10 +32,12 @@ def setUp(self): def check_nan_inf(self): cmd = self._python_interp - proc = subprocess.Popen(cmd.split(" "), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=self.env) + proc = subprocess.Popen( + cmd.split(" "), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.env, + ) out, err = proc.communicate() returncode = proc.returncode @@ -49,7 +50,8 @@ def check_nan_inf(self): assert (out + err).find('find nan or inf==='.encode()) != -1 else: assert (out + err).find( - 'There are `nan` or `inf` in tensor'.encode()) != -1 + 'There are `nan` or `inf` in tensor'.encode() + ) != -1 def test_nan_inf_in_static_mode(self): self._python_interp += " check_nan_inf_base.py" @@ -61,7 +63,6 @@ def test_nan_inf_in_dynamic_mode(self): class TestNanInfEnv(TestNanInf): - def setUp(self): super(TestNanInfEnv, self).setUp() # windows python have some bug with env, so need use str to pass ci @@ -69,7 +70,8 @@ def setUp(self): self.env[str("PADDLE_INF_NAN_SKIP_OP")] = str("mul") self.env[str("PADDLE_INF_NAN_SKIP_ROLE")] = str("loss") self.env[str("PADDLE_INF_NAN_SKIP_VAR")] = str( - "elementwise_add:fc_0.tmp_1") + "elementwise_add:fc_0.tmp_1" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_nanmean_api.py b/python/paddle/fluid/tests/unittests/test_nanmean_api.py index 3155140518de5..bcc9ac5c77a2c 100644 --- a/python/paddle/fluid/tests/unittests/test_nanmean_api.py +++ b/python/paddle/fluid/tests/unittests/test_nanmean_api.py @@ -27,10 +27,14 @@ def setUp(self): self.x_shape = [2, 3, 4, 5] self.x = np.random.uniform(-1, 1, self.x_shape).astype(np.float32) self.x[0, :, :, :] = np.nan - self.x_grad = np.array([[np.nan, np.nan, 3.], [0., np.nan, - 2.]]).astype(np.float32) - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.x_grad = np.array( + [[np.nan, np.nan, 3.0], [0.0, np.nan, 2.0]] + ).astype(np.float32) + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_api_static(self): paddle.enable_static() @@ -43,8 +47,9 @@ def test_api_static(self): out4 = paddle.nanmean(x, axis) out5 = paddle.nanmean(x, tuple(axis)) exe = paddle.static.Executor(self.place) - res = exe.run(feed={'X': self.x}, - fetch_list=[out1, out2, out3, out4, out5]) + res = exe.run( + feed={'X': self.x}, fetch_list=[out1, out2, out3, out4, out5] + ) out_ref = np.nanmean(self.x) for out in res: np.testing.assert_allclose(out, out_ref, rtol=0.0001) @@ -100,9 +105,9 @@ def test_case(x, axis=None, keepdim=False): sum_dx_ref = np.prod(y.shape) if np.isnan(y.numpy()).sum(): sum_dx_ref -= np.isnan(y.numpy()).sum() - cnt = paddle.sum(~paddle.isnan(x_tensor), - axis=axis, - keepdim=keepdim) + cnt = paddle.sum( + ~paddle.isnan(x_tensor), axis=axis, keepdim=keepdim + ) if (cnt == 0).sum(): dx[np.isnan(dx)] = 0 sum_dx = dx.sum() diff --git a/python/paddle/fluid/tests/unittests/test_nanmedian.py b/python/paddle/fluid/tests/unittests/test_nanmedian.py index 76bea45881338..952263ae94b87 100644 --- a/python/paddle/fluid/tests/unittests/test_nanmedian.py +++ b/python/paddle/fluid/tests/unittests/test_nanmedian.py @@ -21,20 +21,19 @@ class TestNanmedian(unittest.TestCase): - def setUp(self): - single_axis_shape = (120) + single_axis_shape = 120 multi_axis_shape = (2, 3, 4, 5) self.fake_data = { - "single_axis_normal": - np.random.uniform(-1, 1, single_axis_shape).astype(np.float32), - "multi_axis_normal": - np.random.uniform(-1, 1, multi_axis_shape).astype(np.float32), - "single_axis_all_nan": - np.full(single_axis_shape, np.nan), - "multi_axis_all_nan": - np.full(multi_axis_shape, np.nan), + "single_axis_normal": np.random.uniform( + -1, 1, single_axis_shape + ).astype(np.float32), + "multi_axis_normal": np.random.uniform( + -1, 1, multi_axis_shape + ).astype(np.float32), + "single_axis_all_nan": np.full(single_axis_shape, np.nan), + "multi_axis_all_nan": np.full(multi_axis_shape, np.nan), } single_partial_nan = self.fake_data["single_axis_normal"].copy() @@ -59,11 +58,22 @@ def setUp(self): col_data[:, :, 2, 3:] = np.nan self.fake_data["col_nan_odd"] = col_data - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) self.axis_candiate_list = [ - None, 0, 2, -1, -2, (1, 2), [0, -1], [0, 1, 3], (1, 2, 3), - [0, 2, 1, 3] + None, + 0, + 2, + -1, + -2, + (1, 2), + [0, -1], + [0, 1, 3], + (1, 2, 3), + [0, 2, 1, 3], ] def test_api_static(self): @@ -79,8 +89,9 @@ def test_api_static(self): out4 = paddle.nanmedian(x, axis=axis, keepdim=True) out5 = paddle.nanmedian(x, axis=tuple(axis), keepdim=True) exe = paddle.static.Executor(self.place) - res = exe.run(feed={'X': data}, - fetch_list=[out1, out2, out3, out4, out5]) + res = exe.run( + feed={'X': data}, fetch_list=[out1, out2, out3, out4, out5] + ) for out in res: np.testing.assert_allclose(np_res, out, rtol=1e-05, equal_nan=True) @@ -109,23 +120,22 @@ def test_data_case(data): continue np_res = np.nanmedian(data, keepdims=keep_dim) - pd_res = paddle.nanmedian(paddle.to_tensor(data), - keepdim=keep_dim) - np.testing.assert_allclose(np_res, - pd_res.numpy(), - rtol=1e-05, - equal_nan=True) + pd_res = paddle.nanmedian( + paddle.to_tensor(data), keepdim=keep_dim + ) + np.testing.assert_allclose( + np_res, pd_res.numpy(), rtol=1e-05, equal_nan=True + ) def test_axis_case(data, axis): - pd_res = paddle.nanmedian(paddle.to_tensor(data), - axis=axis, - keepdim=False) + pd_res = paddle.nanmedian( + paddle.to_tensor(data), axis=axis, keepdim=False + ) axis = clean_axis_numpy(axis, len(data.shape)) np_res = np.nanmedian(data, axis=axis, keepdims=False) - np.testing.assert_allclose(np_res, - pd_res.numpy(), - rtol=1e-05, - equal_nan=True) + np.testing.assert_allclose( + np_res, pd_res.numpy(), rtol=1e-05, equal_nan=True + ) for name, data in self.fake_data.items(): test_data_case(data) diff --git a/python/paddle/fluid/tests/unittests/test_nansum_api.py b/python/paddle/fluid/tests/unittests/test_nansum_api.py index c0fe5da0cbccb..9abc0e4e4e7aa 100644 --- a/python/paddle/fluid/tests/unittests/test_nansum_api.py +++ b/python/paddle/fluid/tests/unittests/test_nansum_api.py @@ -19,7 +19,6 @@ class API_Test_Nansum(unittest.TestCase): - def test_static_graph(self): paddle.enable_static() startup_program = fluid.Program() @@ -36,12 +35,14 @@ def test_static_graph(self): exe = fluid.Executor(place) exe.run(startup_program) - x = np.array([[float('nan'), 3, 5, 9], [1, 2, - float('-nan'), - 7]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': x}, - fetch_list=[out1, out2, out3, out4]) + x = np.array( + [[float('nan'), 3, 5, 9], [1, 2, float('-nan'), 7]] + ).astype(np.float32) + res = exe.run( + train_program, + feed={'input': x}, + fetch_list=[out1, out2, out3, out4], + ) out1_np = np.array(res[0]) out2_np = np.array(res[1]) @@ -52,14 +53,22 @@ def test_static_graph(self): out3_ref = np.array([17, 10]).astype(np.float32) out4_ref = np.array([[17], [10]]).astype(np.float32) - self.assertTrue((out1_np == out1_ref).all(), - msg='nansum output is wrong, out =' + str(out1_np)) - self.assertTrue((out2_np == out2_ref).all(), - msg='nansum output is wrong, out =' + str(out2_np)) - self.assertTrue((out3_np == out3_ref).all(), - msg='nansum output is wrong, out =' + str(out3_np)) - self.assertTrue((out4_np == out4_ref).all(), - msg='nansum output is wrong, out =' + str(out4_np)) + self.assertTrue( + (out1_np == out1_ref).all(), + msg='nansum output is wrong, out =' + str(out1_np), + ) + self.assertTrue( + (out2_np == out2_ref).all(), + msg='nansum output is wrong, out =' + str(out2_np), + ) + self.assertTrue( + (out3_np == out3_ref).all(), + msg='nansum output is wrong, out =' + str(out3_np), + ) + self.assertTrue( + (out4_np == out4_ref).all(), + msg='nansum output is wrong, out =' + str(out4_np), + ) def test_error_api(self): paddle.enable_static() @@ -79,8 +88,9 @@ def run2(): self.assertRaises(TypeError, run2) def test_dygraph(self): - x = np.array([[float('nan'), 3, 5, 9], [1, 2, float('-nan'), - 7]]).astype(np.float32) + x = np.array( + [[float('nan'), 3, 5, 9], [1, 2, float('-nan'), 7]] + ).astype(np.float32) with fluid.dygraph.guard(): inputs = fluid.dygraph.to_variable(x) out = paddle.nansum(inputs) @@ -88,7 +98,8 @@ def test_dygraph(self): self.assertTrue( (out.numpy() == out_ref).all(), - msg='nansum output is wrong, out =' + str(out.numpy())) + msg='nansum output is wrong, out =' + str(out.numpy()), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_nce.py b/python/paddle/fluid/tests/unittests/test_nce.py index 484dbfabd9d46..885d12c6fa204 100644 --- a/python/paddle/fluid/tests/unittests/test_nce.py +++ b/python/paddle/fluid/tests/unittests/test_nce.py @@ -22,8 +22,9 @@ from op_test import OpTest -def nce(input, weight, bias, sample_weight, labels, num_classes, - num_sample_class): +def nce( + input, weight, bias, sample_weight, labels, num_classes, num_sample_class +): samples = [] sample_labels = [] batch_size = input.shape[0] @@ -54,23 +55,34 @@ def nce(input, weight, bias, sample_weight, labels, num_classes, o = sample_out[i] cost = -np.log(o / (o + b)) if samples[i][2] else -np.log(b / (o + b)) out[samples[i][0]] += cost * samples[i][3] - return (out[:, np.newaxis], - np.array(sample_out).reshape(batch_size, - num_sample_class + num_true_class), - np.array(sample_labels).reshape(batch_size, - num_sample_class + num_true_class)) + return ( + out[:, np.newaxis], + np.array(sample_out).reshape( + batch_size, num_sample_class + num_true_class + ), + np.array(sample_labels).reshape( + batch_size, num_sample_class + num_true_class + ), + ) class TestNCE(OpTest): - - def generate_data(self, dim, batch_size, num_classes, num_true_class, - num_neg_samples, is_sparse): + def generate_data( + self, + dim, + batch_size, + num_classes, + num_true_class, + num_neg_samples, + is_sparse, + ): input = np.random.randn(batch_size, dim).astype(np.float32) weight = np.random.randn(num_classes, dim).astype(np.float32) bias = np.random.randn(num_classes).astype(np.float32) sample_weight = np.random.randn(batch_size).astype(np.float32) - labels = np.random.randint(0, num_classes, - (batch_size, num_true_class)).astype("int64") + labels = np.random.randint( + 0, num_classes, (batch_size, num_true_class) + ).astype("int64") self.attrs = { 'num_total_classes': num_classes, 'num_neg_samples': num_neg_samples, @@ -78,14 +90,14 @@ def generate_data(self, dim, batch_size, num_classes, num_true_class, 'seed': 0, 'sampler': 0, 'is_sparse': is_sparse, - 'is_test': self.is_test + 'is_test': self.is_test, } self.inputs = { 'Input': input, 'Label': labels, 'Weight': weight, 'Bias': bias, - 'SampleWeight': sample_weight + 'SampleWeight': sample_weight, } def set_is_test(self): @@ -95,17 +107,22 @@ def set_data(self): self.generate_data(5, 25, 100, 1, 2, False) def compute(self): - out = nce(self.inputs['Input'], self.inputs['Weight'], - self.inputs['Bias'], self.inputs['SampleWeight'], - self.inputs['Label'], self.attrs['num_total_classes'], - self.attrs['num_neg_samples']) + out = nce( + self.inputs['Input'], + self.inputs['Weight'], + self.inputs['Bias'], + self.inputs['SampleWeight'], + self.inputs['Label'], + self.attrs['num_total_classes'], + self.attrs['num_neg_samples'], + ) if self.is_test: self.outputs = {'Cost': out[0]} else: self.outputs = { 'Cost': out[0], 'SampleLogits': out[1], - 'SampleLabels': out[2] + 'SampleLabels': out[2], } def setUp(self): @@ -118,13 +135,12 @@ def test_check_output(self): self.check_output() def test_check_grad(self): - self.check_grad(["Input", "Weight", "Bias"], - "Cost", - max_relative_error=0.02) + self.check_grad( + ["Input", "Weight", "Bias"], "Cost", max_relative_error=0.02 + ) class TestNCECase1Tensor(TestNCE): - def set_data(self): self.generate_data(10, 20, 100, 2, 5, False) @@ -139,7 +155,6 @@ def test_check_grad(self): class TestNCECase1SelectedRows(unittest.TestCase): - def setUp(self): self.base_lr = 0.0001 self.batch_size = 8 @@ -163,33 +178,51 @@ def get_optimizer(self): optimizer = fluid.optimizer.SGD(learning_rate=self.base_lr) return optimizer - def train_network(self, num_total_classes, num_neg_samples, sampler, - custom_dist, is_sparse): + def train_network( + self, + num_total_classes, + num_neg_samples, + sampler, + custom_dist, + is_sparse, + ): input = fluid.layers.data(name="input", shape=[10], dtype="float32") label = fluid.layers.data(name="label", shape=[1], dtype="int64") - w_param = fluid.default_main_program().global_block().create_parameter( - shape=[num_total_classes, 10], - dtype='float32', - name='nce_w', - initializer=initializer.ConstantInitializer()) - b_param = fluid.default_main_program().global_block().create_parameter( - shape=[num_total_classes, 1], - dtype='float32', - name='nce_b', - initializer=initializer.ConstantInitializer()) - - cost = fluid.layers.nce(input=input, - label=label, - num_total_classes=num_total_classes, - sampler=sampler, - custom_dist=custom_dist, - sample_weight=None, - param_attr='nce_w', - bias_attr='nce_b', - seed=1, - num_neg_samples=num_neg_samples, - is_sparse=is_sparse) + w_param = ( + fluid.default_main_program() + .global_block() + .create_parameter( + shape=[num_total_classes, 10], + dtype='float32', + name='nce_w', + initializer=initializer.ConstantInitializer(), + ) + ) + b_param = ( + fluid.default_main_program() + .global_block() + .create_parameter( + shape=[num_total_classes, 1], + dtype='float32', + name='nce_b', + initializer=initializer.ConstantInitializer(), + ) + ) + + cost = fluid.layers.nce( + input=input, + label=label, + num_total_classes=num_total_classes, + sampler=sampler, + custom_dist=custom_dist, + sample_weight=None, + param_attr='nce_w', + bias_attr='nce_b', + seed=1, + num_neg_samples=num_neg_samples, + is_sparse=is_sparse, + ) avg_cost = paddle.mean(cost) # optimizer optimizer = self.get_optimizer() @@ -210,15 +243,19 @@ def test_input_is_selected_rows(self): dense_startup_program = fluid.framework.Program() dense_train_program = fluid.framework.Program() with fluid.scope_guard(dense_scope): - with fluid.program_guard(dense_train_program, - dense_startup_program): - cost, feeds = self.train_network(20, 5, "custom_dist", - nid_freq_arr.tolist(), False) + with fluid.program_guard( + dense_train_program, dense_startup_program + ): + cost, feeds = self.train_network( + 20, 5, "custom_dist", nid_freq_arr.tolist(), False + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) exe.run(dense_startup_program) - loss_val = exe.run(dense_train_program, - feed=feeder.feed(data), - fetch_list=[cost.name]) + loss_val = exe.run( + dense_train_program, + feed=feeder.feed(data), + fetch_list=[cost.name], + ) rets.append(np.mean(loss_val)) # for sparse @@ -226,106 +263,113 @@ def test_input_is_selected_rows(self): sparse_startup_program = fluid.framework.Program() sparse_train_program = fluid.framework.Program() with fluid.scope_guard(sparse_scope): - with fluid.program_guard(sparse_train_program, - sparse_startup_program): - cost, feeds = self.train_network(20, 5, "custom_dist", - nid_freq_arr.tolist(), True) + with fluid.program_guard( + sparse_train_program, sparse_startup_program + ): + cost, feeds = self.train_network( + 20, 5, "custom_dist", nid_freq_arr.tolist(), True + ) feeder = fluid.DataFeeder(feed_list=feeds, place=place) exe.run(sparse_startup_program) - loss_val = exe.run(sparse_train_program, - feed=feeder.feed(data), - fetch_list=[cost.name]) + loss_val = exe.run( + sparse_train_program, + feed=feeder.feed(data), + fetch_list=[cost.name], + ) rets.append(np.mean(loss_val)) self.assertEqual(rets[0], rets[1]) class TestNCE_OpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - input1 = fluid.create_lod_tensor(np.array([0.0, 3.0, 2.0, 4.0]), - [[1, 1, 2]], fluid.CPUPlace()) - label1 = fluid.layers.data(name='label1', - shape=[-1, 4], - dtype="int64") + input1 = fluid.create_lod_tensor( + np.array([0.0, 3.0, 2.0, 4.0]), [[1, 1, 2]], fluid.CPUPlace() + ) + label1 = fluid.layers.data( + name='label1', shape=[-1, 4], dtype="int64" + ) # the input(input) of nce layer must be Variable. self.assertRaises(TypeError, fluid.layers.nce, input1, label1, 5) - input2 = fluid.layers.data(name='input2', - shape=[-1, 4], - dtype="float32") - label2 = fluid.create_lod_tensor(np.array([0.0, 3.0, 2.0, 4.0]), - [[1, 1, 2]], fluid.CPUPlace()) + input2 = fluid.layers.data( + name='input2', shape=[-1, 4], dtype="float32" + ) + label2 = fluid.create_lod_tensor( + np.array([0.0, 3.0, 2.0, 4.0]), [[1, 1, 2]], fluid.CPUPlace() + ) # the input(label) of nce layer must be Variable. self.assertRaises(TypeError, fluid.layers.nce, input2, label2, 5) - input3 = fluid.layers.data(name='input3', - shape=[-1, 4], - dtype="float16") - label3 = fluid.layers.data(name='label3', - shape=[-1, 1], - dtype="int64") + input3 = fluid.layers.data( + name='input3', shape=[-1, 4], dtype="float16" + ) + label3 = fluid.layers.data( + name='label3', shape=[-1, 1], dtype="int64" + ) # the data type of input(input) must be float32 or float64. self.assertRaises(TypeError, fluid.layers.nce, input3, label3, 5) - input4 = fluid.layers.data(name='input4', - shape=[-1, 4], - dtype="float32") - label4 = fluid.layers.data(name='label4', - shape=[-1, 1], - dtype="int32") + input4 = fluid.layers.data( + name='input4', shape=[-1, 4], dtype="float32" + ) + label4 = fluid.layers.data( + name='label4', shape=[-1, 1], dtype="int32" + ) # the data type of input(label) must be int64. self.assertRaises(TypeError, fluid.layers.nce, input4, label4, 5) class TestDygraphNCE_OpError(unittest.TestCase): - def test_NCE_errors(self): with program_guard(Program(), Program()): nce = fluid.NCE(20, 5) - input1 = fluid.create_lod_tensor(np.array([0.0, 3.0, 2.0, 4.0]), - [[1, 1, 2]], fluid.CPUPlace()) - label1 = fluid.layers.data(name='label1', - shape=[-1, 4], - dtype="int64") + input1 = fluid.create_lod_tensor( + np.array([0.0, 3.0, 2.0, 4.0]), [[1, 1, 2]], fluid.CPUPlace() + ) + label1 = fluid.layers.data( + name='label1', shape=[-1, 4], dtype="int64" + ) # the input(input) of NCE layer must be Variable. self.assertRaises(TypeError, nce, input1, label1) - input2 = fluid.layers.data(name='input2', - shape=[-1, 4], - dtype="float32") - label2 = fluid.create_lod_tensor(np.array([0.0, 3.0, 2.0, 4.0]), - [[1, 1, 2]], fluid.CPUPlace()) + input2 = fluid.layers.data( + name='input2', shape=[-1, 4], dtype="float32" + ) + label2 = fluid.create_lod_tensor( + np.array([0.0, 3.0, 2.0, 4.0]), [[1, 1, 2]], fluid.CPUPlace() + ) # the input(label) of NCE layer must be Variable. self.assertRaises(TypeError, nce, input2, label2) - input3 = fluid.layers.data(name='input3', - shape=[-1, 4], - dtype="float16") - label3 = fluid.layers.data(name='label3', - shape=[-1, 1], - dtype="int64") + input3 = fluid.layers.data( + name='input3', shape=[-1, 4], dtype="float16" + ) + label3 = fluid.layers.data( + name='label3', shape=[-1, 1], dtype="int64" + ) # the data type of input(input) must be float32 or float64. self.assertRaises(TypeError, nce, input3, label3) - input4 = fluid.layers.data(name='input4', - shape=[-1, 4], - dtype="float32") - label4 = fluid.layers.data(name='label4', - shape=[-1, 1], - dtype="int32") + input4 = fluid.layers.data( + name='input4', shape=[-1, 4], dtype="float32" + ) + label4 = fluid.layers.data( + name='label4', shape=[-1, 1], dtype="int32" + ) # the data type of input(label) must be int64. self.assertRaises(TypeError, nce, input4, label4) - input5 = fluid.layers.data(name='input5', - shape=[-1, 4], - dtype="float32") - label5 = fluid.layers.data(name='label5', - shape=[-1, 1], - dtype="int64") + input5 = fluid.layers.data( + name='input5', shape=[-1, 4], dtype="float32" + ) + label5 = fluid.layers.data( + name='label5', shape=[-1, 1], dtype="int64" + ) sample_weight = fluid.create_lod_tensor( - np.array([0.0, 3.0, 2.0, 4.0]), [[1, 1, 2]], fluid.CPUPlace()) + np.array([0.0, 3.0, 2.0, 4.0]), [[1, 1, 2]], fluid.CPUPlace() + ) # the sample_weight of nce must be Variable or None. self.assertRaises(TypeError, nce, input5, label5, sample_weight) diff --git a/python/paddle/fluid/tests/unittests/test_nearest_interp_op.py b/python/paddle/fluid/tests/unittests/test_nearest_interp_op.py index 42b59e57653a7..f8a997e4fbab6 100755 --- a/python/paddle/fluid/tests/unittests/test_nearest_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_nearest_interp_op.py @@ -19,13 +19,15 @@ import paddle.fluid as fluid -def nearest_neighbor_interp_np(X, - out_h, - out_w, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='NCHW'): +def nearest_neighbor_interp_np( + X, + out_h, + out_w, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='NCHW', +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 3, 1, 2)) # NHWC => NCHW @@ -38,13 +40,13 @@ def nearest_neighbor_interp_np(X, n, c, in_h, in_w = X.shape ratio_h = ratio_w = 0.0 - if (out_h > 1): - if (align_corners): + if out_h > 1: + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: ratio_h = 1.0 * in_h / out_h - if (out_w > 1): - if (align_corners): + if out_w > 1: + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: ratio_w = 1.0 * in_w / out_w @@ -71,7 +73,6 @@ def nearest_neighbor_interp_np(X, class TestNearestInterpOp(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -95,10 +96,15 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners, - self.data_layout) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -112,7 +118,7 @@ def setUp(self): 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } self.outputs = {'Out': output_np} @@ -120,136 +126,126 @@ def test_check_output(self): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 4, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True class TestNearestNeighborInterpCase1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpCase2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpCase3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpCase4(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = True class TestNearestNeighborInterpCase5(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") self.align_corners = True class TestNearestNeighborInterpCase6(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 129]).astype("int32") self.align_corners = True class TestNearestNeighborInterpSame(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpActualShape(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True class TestNearestNeighborInterpDataLayout(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 4, 4, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 8]).astype("int32") self.align_corners = True self.data_layout = "NHWC" class TestNearestInterpOpUint8(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None self.init_test_case() self.op_type = "nearest_interp" self.check_eager = True - input_np = np.random.randint(low=0, high=256, - size=self.input_shape).astype("uint8") + input_np = np.random.randint( + low=0, high=256, size=self.input_shape + ).astype("uint8") if self.scale > 0: out_h = int(self.input_shape[2] * self.scale) @@ -258,9 +254,14 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -270,67 +271,62 @@ def setUp(self): 'out_w': self.out_w, 'scale': self.scale, 'interp_method': self.interp_method, - 'align_corners': self.align_corners + 'align_corners': self.align_corners, } self.outputs = {'Out': output_np} def test_check_output(self): - self.check_output_with_place(place=core.CPUPlace(), - atol=1, - check_eager=self.check_eager) + self.check_output_with_place( + place=core.CPUPlace(), atol=1, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 3, 9, 6] self.out_h = 10 self.out_w = 9 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpCase1Uint8(TestNearestInterpOpUint8): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] self.out_h = 80 self.out_w = 40 - self.scale = 0. + self.scale = 0.0 self.align_corners = True class TestNearestNeighborInterpCase2Uint8(TestNearestInterpOpUint8): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] self.out_h = 5 self.out_w = 13 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([6, 15]).astype("int32") self.align_corners = True class TestNearestInterpWithoutCorners(TestNearestInterpOp): - def set_align_corners(self): self.align_corners = False class TestNearestNeighborInterpScale1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 2. + self.scale = 2.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True class TestNearestNeighborInterpScale2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 5, 7] @@ -342,19 +338,17 @@ def init_test_case(self): class TestNearestNeighborInterpScale3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 1. + self.scale = 1.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True class TestNearestInterpOp_attr_tensor(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -389,59 +383,62 @@ def setUp(self): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.check_eager = False self.attrs['out_h'] = self.out_h self.attrs['out_w'] = self.out_w - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 5, 4, 4] self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [3, 3] self.align_corners = True # out_size is a tensor list class TestNearestInterp_attr_tensor_Case1(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = [8, 12] self.align_corners = True # out_size is a 1-D tensor class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.shape_by_1Dtensor = True @@ -449,7 +446,6 @@ def init_test_case(self): # scale is a 1-D tensor class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] @@ -462,7 +458,6 @@ def init_test_case(self): class TestNearestAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") y = fluid.data(name="y", shape=[2, 6, 6, 3], dtype="float32") @@ -470,18 +465,18 @@ def test_case(self): dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[2], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) - out1 = fluid.layers.resize_nearest(y, - out_shape=[12, 12], - data_format='NHWC') + out1 = fluid.layers.resize_nearest( + y, out_shape=[12, 12], data_format='NHWC' + ) out2 = fluid.layers.resize_nearest(x, out_shape=[12, dim]) out3 = fluid.layers.resize_nearest(x, out_shape=shape_tensor) - out4 = fluid.layers.resize_nearest(x, - out_shape=[4, 4], - actual_shape=actual_size) + out4 = fluid.layers.resize_nearest( + x, out_shape=[4, 4], actual_shape=actual_size + ) out5 = fluid.layers.resize_nearest(x, scale=scale_tensor) x_data = np.random.random((2, 3, 6, 6)).astype("float32") @@ -496,39 +491,39 @@ def test_case(self): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": np.transpose(x_data, (0, 2, 3, 1)), - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = nearest_neighbor_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=True) - np.testing.assert_allclose(results[0], - np.transpose(expect_res, (0, 2, 3, 1)), - rtol=1e-05) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": np.transpose(x_data, (0, 2, 3, 1)), + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = nearest_neighbor_interp_np( + x_data, out_h=12, out_w=12, align_corners=True + ) + np.testing.assert_allclose( + results[0], np.transpose(expect_res, (0, 2, 3, 1)), rtol=1e-05 + ) for i in range(len(results) - 1): np.testing.assert_allclose(results[i + 1], expect_res, rtol=1e-05) class TestNearestInterpException(unittest.TestCase): - def test_exception(self): input = fluid.data(name="input", shape=[1, 3, 6, 6], dtype="float32") def attr_data_format(): # for 4-D input, data_format can only be NCHW or NHWC - out = fluid.layers.resize_nearest(input, - out_shape=[4, 8], - data_format='NDHWC') + out = fluid.layers.resize_nearest( + input, out_shape=[4, 8], data_format='NDHWC' + ) def attr_scale_type(): out = fluid.layers.resize_nearest(input, scale='scale') @@ -543,5 +538,6 @@ def attr_scale_value(): if __name__ == "__main__": import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_nearest_interp_v2_op.py b/python/paddle/fluid/tests/unittests/test_nearest_interp_v2_op.py index c3157a6e9a155..fb1e1b08ceab9 100755 --- a/python/paddle/fluid/tests/unittests/test_nearest_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_nearest_interp_v2_op.py @@ -23,18 +23,20 @@ paddle.enable_static() -def nearest_interp_test(x, - OutSize=None, - SizeTensor=None, - Scale=None, - data_layout='NCHW', - out_d=-1, - out_h=-1, - out_w=-1, - scale=[], - interp_method='nearest', - align_corners=True, - align_mode=0): +def nearest_interp_test( + x, + OutSize=None, + SizeTensor=None, + Scale=None, + data_layout='NCHW', + out_d=-1, + out_h=-1, + out_w=-1, + scale=[], + interp_method='nearest', + align_corners=True, + align_mode=0, +): if isinstance(scale, float) or isinstance(scale, int): scale_list = [] for _ in range(len(x.shape) - 2): @@ -44,23 +46,36 @@ def nearest_interp_test(x, scale = list(map(float, scale)) if SizeTensor is not None: if not isinstance(SizeTensor, list) and not isinstance( - SizeTensor, tuple): + SizeTensor, tuple + ): SizeTensor = [SizeTensor] - return paddle._C_ops.nearest_interp(x, OutSize, SizeTensor, Scale, - data_layout, out_d, out_h, out_w, scale, - interp_method, align_corners, - align_mode) - - -def nearest_neighbor_interp_np(X, - out_h, - out_w, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='NCHW'): + return paddle._C_ops.nearest_interp( + x, + OutSize, + SizeTensor, + Scale, + data_layout, + out_d, + out_h, + out_w, + scale, + interp_method, + align_corners, + align_mode, + ) + + +def nearest_neighbor_interp_np( + X, + out_h, + out_w, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='NCHW', +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 3, 1, 2)) # NHWC => NCHW @@ -73,16 +88,16 @@ def nearest_neighbor_interp_np(X, n, c, in_h, in_w = X.shape ratio_h = ratio_w = 0.0 - if (out_h > 1): - if (align_corners): + if out_h > 1: + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: ratio_h = 1.0 / scale_h else: ratio_h = 1.0 * in_h / out_h - if (out_w > 1): - if (align_corners): + if out_w > 1: + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -110,17 +125,19 @@ def nearest_neighbor_interp_np(X, return out.astype(X.dtype) -def nearest_neighbor_interp3d_np(X, - out_d, - out_h, - out_w, - scale_d=0, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='NCHW'): +def nearest_neighbor_interp3d_np( + X, + out_d, + out_h, + out_w, + scale_d=0, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='NCHW', +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 4, 1, 2, 3)) # NDHWC => NCDHW @@ -135,24 +152,24 @@ def nearest_neighbor_interp3d_np(X, n, c, in_d, in_h, in_w = X.shape ratio_d = ratio_h = ratio_w = 0.0 - if (out_d > 1): - if (align_corners): + if out_d > 1: + if align_corners: ratio_d = (in_d - 1.0) / (out_d - 1.0) else: if scale_d > 0: ratio_d = 1.0 / scale_d else: ratio_d = 1.0 * in_d / out_d - if (out_h > 1): - if (align_corners): + if out_h > 1: + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: ratio_h = 1.0 / scale_h else: ratio_h = 1.0 * in_h / out_h - if (out_w > 1): - if (align_corners): + if out_w > 1: + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -184,7 +201,6 @@ def nearest_neighbor_interp3d_np(X, class TestNearestInterpOp(OpTest): - def setUp(self): self.python_api = nearest_interp_test self.out_size = None @@ -240,15 +256,30 @@ def setUp(self): if len(self.input_shape) == 4: output_np = nearest_neighbor_interp_np( - input_np, out_h, out_w, scale_h, scale_w, self.out_size, - self.actual_shape, self.align_corners, self.data_layout) + input_np, + out_h, + out_w, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) elif len(self.input_shape) == 5: - output_np = nearest_neighbor_interp3d_np(input_np, out_d, out_h, - out_w, scale_d, scale_h, - scale_w, self.out_size, - self.actual_shape, - self.align_corners, - self.data_layout) + output_np = nearest_neighbor_interp3d_np( + input_np, + out_d, + out_h, + out_w, + scale_d, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -261,7 +292,7 @@ def setUp(self): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } else: self.attrs = { @@ -269,7 +300,7 @@ def setUp(self): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -297,7 +328,6 @@ def init_test_case(self): class TestNearestNeighborInterpCase1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 1, 7, 8] @@ -309,7 +339,6 @@ def init_test_case(self): class TestNearestNeighborInterpCase2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] @@ -320,7 +349,6 @@ def init_test_case(self): class TestNearestNeighborInterpCase3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] @@ -331,7 +359,6 @@ def init_test_case(self): class TestNearestNeighborInterpCase4(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] @@ -343,7 +370,6 @@ def init_test_case(self): class TestNearestNeighborInterpCase5(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] @@ -355,7 +381,6 @@ def init_test_case(self): class TestNearestNeighborInterpCase6(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [1, 1, 32, 64] @@ -367,7 +392,6 @@ def init_test_case(self): class TestNearestNeighborInterpSame(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] @@ -378,7 +402,6 @@ def init_test_case(self): class TestNearestNeighborInterpActualShape(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] @@ -390,7 +413,6 @@ def init_test_case(self): class TestNearestNeighborInterpDataLayout(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 4, 4, 5] @@ -403,15 +425,15 @@ def init_test_case(self): class TestNearestInterpOpUint8(OpTest): - def setUp(self): self.python_api = nearest_interp_test self.out_size = None self.actual_shape = None self.init_test_case() self.op_type = "nearest_interp_v2" - input_np = np.random.randint(low=0, high=256, - size=self.input_shape).astype("uint8") + input_np = np.random.randint( + low=0, high=256, size=self.input_shape + ).astype("uint8") if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -428,9 +450,16 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -438,7 +467,7 @@ def setUp(self): 'out_h': self.out_h, 'out_w': self.out_w, 'interp_method': self.interp_method, - 'align_corners': self.align_corners + 'align_corners': self.align_corners, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -450,9 +479,9 @@ def setUp(self): self.outputs = {'Out': output_np} def test_check_output(self): - self.check_output_with_place(place=core.CPUPlace(), - atol=1, - check_eager=True) + self.check_output_with_place( + place=core.CPUPlace(), atol=1, check_eager=True + ) def init_test_case(self): self.interp_method = 'nearest' @@ -464,7 +493,6 @@ def init_test_case(self): class TestNearestNeighborInterpCase1Uint8(TestNearestInterpOpUint8): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 3, 32, 64] @@ -475,7 +503,6 @@ def init_test_case(self): class TestNearestNeighborInterpCase2Uint8(TestNearestInterpOpUint8): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [4, 1, 7, 8] @@ -487,25 +514,22 @@ def init_test_case(self): class TestNearestInterpWithoutCorners(TestNearestInterpOp): - def set_align_corners(self): self.align_corners = False class TestNearestNeighborInterpScale1(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 2. + self.scale = 2.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True class TestNearestNeighborInterpScale2(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 5, 7] @@ -517,7 +541,6 @@ def init_test_case(self): class TestNearestNeighborInterpScale3(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 7, 5] @@ -529,7 +552,6 @@ def init_test_case(self): class TestNearestNeighbor3DInterp(TestNearestInterpOp): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 4, 7, 5] @@ -542,7 +564,6 @@ def init_test_case(self): class TestNearestInterpOp_attr_tensor(OpTest): - def setUp(self): self.python_api = nearest_interp_test self.out_size = None @@ -581,8 +602,9 @@ def setUp(self): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h @@ -594,9 +616,16 @@ def setUp(self): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): @@ -617,7 +646,6 @@ def init_test_case(self): # out_size is a tensor list class TestNearestInterp_attr_tensor_Case1(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 3, 9, 6] @@ -630,7 +658,6 @@ def init_test_case(self): # out_size is a 1-D tensor class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] @@ -644,7 +671,6 @@ def init_test_case(self): # scale is a 1-D tensor class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [3, 2, 32, 16] @@ -657,7 +683,6 @@ def init_test_case(self): class TestNearestAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") y = fluid.data(name="y", shape=[2, 6, 6, 3], dtype="float32") @@ -665,18 +690,18 @@ def test_case(self): dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[2], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[2], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) - out1 = fluid.layers.resize_nearest(y, - out_shape=[12, 12], - data_format='NHWC') + out1 = fluid.layers.resize_nearest( + y, out_shape=[12, 12], data_format='NHWC' + ) out2 = fluid.layers.resize_nearest(x, out_shape=[12, dim]) out3 = fluid.layers.resize_nearest(x, out_shape=shape_tensor) - out4 = fluid.layers.resize_nearest(x, - out_shape=[4, 4], - actual_shape=actual_size) + out4 = fluid.layers.resize_nearest( + x, out_shape=[4, 4], actual_shape=actual_size + ) out5 = fluid.layers.resize_nearest(x, scale=scale_tensor) x_data = np.random.random((2, 3, 6, 6)).astype("float32") @@ -691,33 +716,34 @@ def test_case(self): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": np.transpose(x_data, (0, 2, 3, 1)), - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = nearest_neighbor_interp_np(x_data, - out_h=12, - out_w=12, - align_corners=True) - np.testing.assert_allclose(results[0], - np.transpose(expect_res, (0, 2, 3, 1)), - rtol=1e-05) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": np.transpose(x_data, (0, 2, 3, 1)), + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = nearest_neighbor_interp_np( + x_data, out_h=12, out_w=12, align_corners=True + ) + np.testing.assert_allclose( + results[0], np.transpose(expect_res, (0, 2, 3, 1)), rtol=1e-05 + ) for i in range(len(results) - 1): np.testing.assert_allclose(results[i + 1], expect_res, rtol=1e-05) class TestNearestInterpOpAPI_dy(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -727,21 +753,22 @@ def test_case(self): scale_np = np.array([2, 2]).astype("int64") input_x = paddle.to_tensor(input_data) scale = paddle.to_tensor(scale_np) - expect_res = nearest_neighbor_interp_np(input_data, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - scale_factor=scale, - mode="nearest", - align_corners=False) + expect_res = nearest_neighbor_interp_np( + input_data, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, + scale_factor=scale, + mode="nearest", + align_corners=False, + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) class TestNearestInterp3DOpAPI_dy(unittest.TestCase): - def test_case(self): import paddle + if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -751,30 +778,30 @@ def test_case(self): scale_np = np.array([2, 2, 2]).astype("int64") input_x = paddle.to_tensor(input_data) scale = paddle.to_tensor(scale_np) - expect_res = nearest_neighbor_interp3d_np(input_data, - out_d=12, - out_h=12, - out_w=12, - align_corners=False) - out = interpolate(x=input_x, - scale_factor=scale, - mode="nearest", - align_corners=False, - data_format="NCDHW") + expect_res = nearest_neighbor_interp3d_np( + input_data, out_d=12, out_h=12, out_w=12, align_corners=False + ) + out = interpolate( + x=input_x, + scale_factor=scale, + mode="nearest", + align_corners=False, + data_format="NCDHW", + ) np.testing.assert_allclose(out.numpy(), expect_res, rtol=1e-05) class TestNearestInterpException(unittest.TestCase): - def test_exception(self): import paddle + input = fluid.data(name="input", shape=[1, 3, 6, 6], dtype="float32") def attr_data_format(): # for 4-D input, data_format can only be NCHW or NHWC - out = fluid.layers.resize_nearest(input, - out_shape=[4, 8], - data_format='NDHWC') + out = fluid.layers.resize_nearest( + input, out_shape=[4, 8], data_format='NDHWC' + ) def attr_scale_type(): out = fluid.layers.resize_nearest(input, scale='scale') @@ -788,9 +815,9 @@ def input_shape_error(): def mode_error(): x = paddle.randn([1, 3]) - out = paddle.nn.functional.interpolate(x, - scale_factor='scale', - mode="BILINEAR") + out = paddle.nn.functional.interpolate( + x, scale_factor='scale', mode="BILINEAR" + ) self.assertRaises(ValueError, attr_data_format) self.assertRaises(TypeError, attr_scale_type) @@ -799,10 +826,10 @@ def mode_error(): self.assertRaises(ValueError, mode_error) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestNearestInterp3DOpForFloat16(unittest.TestCase): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 2, 6, 6, 6] @@ -815,11 +842,13 @@ def check_main(self, x_np, dtype): x_np = x_np.astype(dtype) x = paddle.to_tensor(x_np) x.stop_gradient = False - y = interpolate(x, - scale_factor=self.scale, - mode=self.interp_method, - align_corners=self.align_corners, - data_format=self.data_layout) + y = interpolate( + x, + scale_factor=self.scale, + mode=self.interp_method, + align_corners=self.align_corners, + data_format=self.data_layout, + ) x_g = paddle.grad(y, x) y_np = y[0].numpy().astype('float32') x_g_np = x_g[0].numpy().astype('float32') @@ -838,10 +867,10 @@ def test_main(self): np.testing.assert_allclose(x_g_np_1, x_g_np_2) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestNearestInterpOpForFloat16(unittest.TestCase): - def init_test_case(self): self.interp_method = 'nearest' self.input_shape = [2, 2, 6, 6] @@ -853,10 +882,12 @@ def check_main(self, x_np, dtype): x_np = x_np.astype(dtype) x = paddle.to_tensor(x_np) x.stop_gradient = False - y = interpolate(x, - scale_factor=self.scale, - mode=self.interp_method, - align_corners=self.align_corners) + y = interpolate( + x, + scale_factor=self.scale, + mode=self.interp_method, + align_corners=self.align_corners, + ) x_g = paddle.grad(y, x) y_np = y[0].numpy().astype('float32') x_g_np = x_g[0].numpy().astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_neg_op.py b/python/paddle/fluid/tests/unittests/test_neg_op.py index 77c961408a5c2..260d0fa35dc4b 100644 --- a/python/paddle/fluid/tests/unittests/test_neg_op.py +++ b/python/paddle/fluid/tests/unittests/test_neg_op.py @@ -18,7 +18,6 @@ class TestNegOp(unittest.TestCase): - def setUp(self): self.init_dtype_type() self.input = (np.random.random((32, 8)) * 100).astype(self.dtype) @@ -30,9 +29,9 @@ def run_imperative(self): input = paddle.to_tensor(self.input) dy_result = paddle.neg(input) expected_result = np.negative(self.input) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05 + ) def run_static(self, use_gpu=False): input = paddle.fluid.data(name='input', shape=[32, 8], dtype=self.dtype) @@ -66,31 +65,26 @@ def test_gpu(self): class TestNegOpFp32(TestNegOp): - def init_dtype_type(self): self.dtype = np.float32 class TestNegOpInt64(TestNegOp): - def init_dtype_type(self): self.dtype = np.int64 class TestNegOpInt32(TestNegOp): - def init_dtype_type(self): self.dtype = np.int32 class TestNegOpInt16(TestNegOp): - def init_dtype_type(self): self.dtype = np.int16 class TestNegOpInt8(TestNegOp): - def init_dtype_type(self): self.dtype = np.int8 diff --git a/python/paddle/fluid/tests/unittests/test_network_with_dtype.py b/python/paddle/fluid/tests/unittests/test_network_with_dtype.py index 203dcdc38cb27..f230cc66c20a1 100644 --- a/python/paddle/fluid/tests/unittests/test_network_with_dtype.py +++ b/python/paddle/fluid/tests/unittests/test_network_with_dtype.py @@ -22,7 +22,6 @@ class TestNetWithDtype(unittest.TestCase): - def setUp(self): self.dtype = "float64" self.init_dtype() @@ -40,8 +39,9 @@ def run_net_on_place(self, place): sgd_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=BATCH_SIZE) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=BATCH_SIZE + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(startup) diff --git a/python/paddle/fluid/tests/unittests/test_newprofiler.py b/python/paddle/fluid/tests/unittests/test_newprofiler.py index afa09462cb67e..0d42de0be7c91 100755 --- a/python/paddle/fluid/tests/unittests/test_newprofiler.py +++ b/python/paddle/fluid/tests/unittests/test_newprofiler.py @@ -25,62 +25,67 @@ class TestProfiler(unittest.TestCase): - def tearDown(self): self.temp_dir.cleanup() def test_profiler(self): - def my_trace_back(prof): - path = os.path.join(self.temp_dir.name, - './test_profiler_chrometracing') + path = os.path.join( + self.temp_dir.name, './test_profiler_chrometracing' + ) profiler.export_chrome_tracing(path)(prof) path = os.path.join(self.temp_dir.name, './test_profiler_pb') profiler.export_protobuf(path)(prof) self.temp_dir = tempfile.TemporaryDirectory() x_value = np.random.randn(2, 3, 3) - x = paddle.to_tensor(x_value, - stop_gradient=False, - place=paddle.CPUPlace()) + x = paddle.to_tensor( + x_value, stop_gradient=False, place=paddle.CPUPlace() + ) y = x / 2.0 ones_like_y = paddle.ones_like(y) - with profiler.Profiler(targets=[profiler.ProfilerTarget.CPU], ) as prof: + with profiler.Profiler( + targets=[profiler.ProfilerTarget.CPU], + ) as prof: y = x / 2.0 prof = None self.assertEqual(utils._is_profiler_used, False) with profiler.RecordEvent(name='test'): y = x / 2.0 - with profiler.Profiler(targets=[profiler.ProfilerTarget.CPU], - scheduler=(1, 2)) as prof: + with profiler.Profiler( + targets=[profiler.ProfilerTarget.CPU], scheduler=(1, 2) + ) as prof: self.assertEqual(utils._is_profiler_used, True) with profiler.RecordEvent(name='test'): y = x / 2.0 prof = None - with profiler.Profiler(targets=[profiler.ProfilerTarget.CPU], - scheduler=profiler.make_scheduler(closed=0, - ready=1, - record=1, - repeat=1), - on_trace_ready=my_trace_back) as prof: + with profiler.Profiler( + targets=[profiler.ProfilerTarget.CPU], + scheduler=profiler.make_scheduler( + closed=0, ready=1, record=1, repeat=1 + ), + on_trace_ready=my_trace_back, + ) as prof: y = x / 2.0 prof = None - with profiler.Profiler(targets=[profiler.ProfilerTarget.CPU], - scheduler=profiler.make_scheduler(closed=0, - ready=0, - record=2, - repeat=1), - on_trace_ready=my_trace_back) as prof: + with profiler.Profiler( + targets=[profiler.ProfilerTarget.CPU], + scheduler=profiler.make_scheduler( + closed=0, ready=0, record=2, repeat=1 + ), + on_trace_ready=my_trace_back, + ) as prof: for i in range(3): y = x / 2.0 prof.step() prof = None with profiler.Profiler( - targets=[profiler.ProfilerTarget.CPU], - scheduler=lambda x: profiler.ProfilerState.RECORD_AND_RETURN, - on_trace_ready=my_trace_back) as prof: + targets=[profiler.ProfilerTarget.CPU], + scheduler=lambda x: profiler.ProfilerState.RECORD_AND_RETURN, + on_trace_ready=my_trace_back, + ) as prof: for i in range(2): y = x / 2.0 prof.step() @@ -107,35 +112,39 @@ def my_sheduler1(num_step): prof = None with profiler.Profiler( - targets=[profiler.ProfilerTarget.CPU], - scheduler=lambda x: profiler.ProfilerState.RECORD_AND_RETURN, - on_trace_ready=my_trace_back) as prof: + targets=[profiler.ProfilerTarget.CPU], + scheduler=lambda x: profiler.ProfilerState.RECORD_AND_RETURN, + on_trace_ready=my_trace_back, + ) as prof: for i in range(2): y = x / 2.0 prof.step() prof = None - with profiler.Profiler(targets=[profiler.ProfilerTarget.CPU], - scheduler=my_sheduler, - on_trace_ready=my_trace_back) as prof: + with profiler.Profiler( + targets=[profiler.ProfilerTarget.CPU], + scheduler=my_sheduler, + on_trace_ready=my_trace_back, + ) as prof: for i in range(5): y = x / 2.0 prof.step() prof = None - with profiler.Profiler(targets=[profiler.ProfilerTarget.CPU], - scheduler=my_sheduler1) as prof: + with profiler.Profiler( + targets=[profiler.ProfilerTarget.CPU], scheduler=my_sheduler1 + ) as prof: for i in range(5): y = x / 2.0 prof.step() prof = None - with profiler.Profiler(targets=[profiler.ProfilerTarget.CPU], - scheduler=profiler.make_scheduler(closed=1, - ready=1, - record=2, - repeat=1, - skip_first=1), - on_trace_ready=my_trace_back, - profile_memory=True, - record_shapes=True) as prof: + with profiler.Profiler( + targets=[profiler.ProfilerTarget.CPU], + scheduler=profiler.make_scheduler( + closed=1, ready=1, record=2, repeat=1, skip_first=1 + ), + on_trace_ready=my_trace_back, + profile_memory=True, + record_shapes=True, + ) as prof: for i in range(5): y = x / 2.0 paddle.grad(outputs=y, inputs=[x], grad_outputs=ones_like_y) @@ -148,13 +157,12 @@ def my_sheduler1(num_step): prof = None dataset = RandomDataset(10 * 4) simple_net = SimpleNet() - opt = paddle.optimizer.SGD(learning_rate=1e-3, - parameters=simple_net.parameters()) - loader = DataLoader(dataset, - batch_size=4, - shuffle=True, - drop_last=True, - num_workers=2) + opt = paddle.optimizer.SGD( + learning_rate=1e-3, parameters=simple_net.parameters() + ) + loader = DataLoader( + dataset, batch_size=4, shuffle=True, drop_last=True, num_workers=2 + ) prof = profiler.Profiler(on_trace_ready=lambda prof: None) prof.start() for i, (image, label) in enumerate(loader()): @@ -171,8 +179,9 @@ def my_sheduler1(num_step): dataset = RandomDataset(10 * 4) simple_net = SimpleNet() loader = DataLoader(dataset, batch_size=4, shuffle=True, drop_last=True) - opt = paddle.optimizer.Adam(learning_rate=1e-3, - parameters=simple_net.parameters()) + opt = paddle.optimizer.Adam( + learning_rate=1e-3, parameters=simple_net.parameters() + ) prof = profiler.Profiler(on_trace_ready=lambda prof: None) prof.start() for i, (image, label) in enumerate(loader()): @@ -187,19 +196,17 @@ def my_sheduler1(num_step): class TestNvprof(unittest.TestCase): - def test_nvprof(self): for i in range(10): paddle.fluid.profiler._nvprof_range(i, 10, 20) x_value = np.random.randn(2, 3, 3) - x = paddle.to_tensor(x_value, - stop_gradient=False, - place=paddle.CPUPlace()) + x = paddle.to_tensor( + x_value, stop_gradient=False, place=paddle.CPUPlace() + ) y = x / 2.0 class TestGetProfiler(unittest.TestCase): - def test_getprofiler(self): config_content = ''' { @@ -222,11 +229,12 @@ def test_getprofiler(self): filehandle.write(config_content) filehandle.flush() import paddle.profiler.profiler as profiler + profiler = profiler.get_profiler(filehandle.name) x_value = np.random.randn(2, 3, 3) - x = paddle.to_tensor(x_value, - stop_gradient=False, - place=paddle.CPUPlace()) + x = paddle.to_tensor( + x_value, stop_gradient=False, place=paddle.CPUPlace() + ) with profiler: for i in range(5): y = x / 2.0 @@ -262,6 +270,7 @@ def test_getprofiler(self): filehandle.write(config_content) filehandle.flush() import paddle.profiler.profiler as profiler + try: profiler = profiler.get_profiler(filehandle.name) except: @@ -299,6 +308,7 @@ def test_getprofiler(self): filehandle.write(config_content) filehandle.flush() import paddle.profiler.profiler as profiler + profiler = profiler.get_profiler(filehandle.name) # test exception @@ -334,20 +344,21 @@ def test_getprofiler(self): filehandle.write(config_content) filehandle.flush() import paddle.profiler.profiler as profiler + profiler = profiler.get_profiler(filehandle.name) # test path error import paddle.profiler.profiler as profiler + profiler = profiler.get_profiler('nopath.json') class RandomDataset(Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): image = np.random.random([100]).astype('float32') - label = np.random.randint(0, 10 - 1, (1, )).astype('int64') + label = np.random.randint(0, 10 - 1, (1,)).astype('int64') return image, label def __len__(self): @@ -355,7 +366,6 @@ def __len__(self): class SimpleNet(nn.Layer): - def __init__(self): super(SimpleNet, self).__init__() self.fc = nn.Linear(100, 10) @@ -365,19 +375,20 @@ def forward(self, image, label=None): class TestTimerOnly(unittest.TestCase): - def test_with_dataloader(self): - def train(step_num_samples=None): dataset = RandomDataset(20 * 4) simple_net = SimpleNet() - opt = paddle.optimizer.SGD(learning_rate=1e-3, - parameters=simple_net.parameters()) - loader = DataLoader(dataset, - batch_size=4, - shuffle=True, - drop_last=True, - num_workers=2) + opt = paddle.optimizer.SGD( + learning_rate=1e-3, parameters=simple_net.parameters() + ) + loader = DataLoader( + dataset, + batch_size=4, + shuffle=True, + drop_last=True, + num_workers=2, + ) step_info = '' p = profiler.Profiler(timer_only=True) p.start() diff --git a/python/paddle/fluid/tests/unittests/test_newprofiler_helper.py b/python/paddle/fluid/tests/unittests/test_newprofiler_helper.py index e03ee36cad1c9..706ae74113f11 100755 --- a/python/paddle/fluid/tests/unittests/test_newprofiler_helper.py +++ b/python/paddle/fluid/tests/unittests/test_newprofiler_helper.py @@ -18,7 +18,6 @@ class TestStatisticHelper(unittest.TestCase): - def test_sum_ranges_case1(self): src = [(1, 3), (4, 10), (11, 15)] self.assertEqual(statistic_helper.sum_ranges(src), 12) diff --git a/python/paddle/fluid/tests/unittests/test_nll_loss.py b/python/paddle/fluid/tests/unittests/test_nll_loss.py index b8158bc0e6bfc..a598c501ec052 100644 --- a/python/paddle/fluid/tests/unittests/test_nll_loss.py +++ b/python/paddle/fluid/tests/unittests/test_nll_loss.py @@ -20,11 +20,9 @@ from paddle.fluid.framework import _test_eager_guard -def nll_loss_1d(logs, - targets, - weight=None, - reduction='mean', - ignore_index=-100): +def nll_loss_1d( + logs, targets, weight=None, reduction='mean', ignore_index=-100 +): input_shape = logs.shape N = input_shape[0] C = input_shape[1] @@ -41,17 +39,16 @@ def nll_loss_1d(logs, if reduction == 'sum': return np.sum(out), np.array([total_weight]).astype('float64') elif reduction == 'mean': - return out.sum() / total_weight, np.array([total_weight - ]).astype('float64') + return out.sum() / total_weight, np.array([total_weight]).astype( + 'float64' + ) elif reduction == 'none': return out -def nll_loss_2d(logs, - targets, - weight=None, - reduction='mean', - ignore_index=-100): +def nll_loss_2d( + logs, targets, weight=None, reduction='mean', ignore_index=-100 +): input_shape = logs.shape N = input_shape[0] H = input_shape[2] @@ -71,24 +68,27 @@ def nll_loss_2d(logs, if reduction == 'sum': return np.sum(out), np.array([total_weight]).astype('float64') elif reduction == 'mean': - return out.sum() / total_weight, np.array([total_weight - ]).astype('float64') + return out.sum() / total_weight, np.array([total_weight]).astype( + 'float64' + ) elif reduction == 'none': return out class TestNLLLoss(unittest.TestCase): - def test_NLLLoss_1D_mean(self): np.random.seed(200) input_np = np.random.random(size=(10, 10)).astype(np.float64) np.random.seed(200) - label_np = np.random.randint(0, 10, size=(10, )).astype(np.int64) + label_np = np.random.randint(0, 10, size=(10,)).astype(np.int64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[10, 10], dtype='float64') label = fluid.data(name='label', shape=[10], dtype='int64') @@ -96,24 +96,25 @@ def test_NLLLoss_1D_mean(self): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss() - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() with fluid.dygraph.guard(): with _test_eager_guard(): nll_loss = paddle.nn.loss.NLLLoss() - eager_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + eager_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) eager_result = eager_res.numpy() expected = nll_loss_1d(input_np, label_np)[0] @@ -126,12 +127,15 @@ def test_NLLLoss_1D_sum(self): np.random.seed(200) input_np = np.random.random(size=(10, 10)).astype(np.float64) np.random.seed(200) - label_np = np.random.randint(0, 10, size=(10, )).astype(np.int64) + label_np = np.random.randint(0, 10, size=(10,)).astype(np.int64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[10, 10], dtype='float64') label = fluid.data(name='label', shape=[10], dtype='int64') @@ -139,17 +143,17 @@ def test_NLLLoss_1D_sum(self): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss(reduction='sum') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() with _test_eager_guard(): @@ -172,12 +176,15 @@ def test_NLLLoss_1D_with_weight_mean(self): np.random.seed(200) input_np = np.random.random(size=(10, 10)).astype(np.float64) np.random.seed(200) - label_np = np.random.randint(0, 10, size=(10, )).astype(np.int64) - weight_np = np.random.random(size=(10, )).astype(np.float64) + label_np = np.random.randint(0, 10, size=(10,)).astype(np.int64) + weight_np = np.random.random(size=(10,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[10, 10], dtype='float64') @@ -187,26 +194,32 @@ def test_NLLLoss_1D_with_weight_mean(self): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np)) - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np) + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() with _test_eager_guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np)) - eager_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np) + ) + eager_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) loss = eager_res.sum() loss.backward() eager_result = eager_res.numpy() @@ -222,12 +235,15 @@ def test_NLLLoss_1D_with_weight_sum(self): np.random.seed(200) input_np = np.random.random(size=(10, 10)).astype(np.float64) np.random.seed(200) - label_np = np.random.randint(0, 10, size=(10, )).astype(np.int64) - weight_np = np.random.random(size=(10, )).astype(np.float64) + label_np = np.random.randint(0, 10, size=(10,)).astype(np.int64) + weight_np = np.random.random(size=(10,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[10, 10], dtype='float64') @@ -237,24 +253,27 @@ def test_NLLLoss_1D_with_weight_sum(self): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np), reduction='sum') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np), reduction='sum' + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() - expected = nll_loss_1d(input_np, - label_np, - weight=weight_np, - reduction='sum')[0] + expected = nll_loss_1d( + input_np, label_np, weight=weight_np, reduction='sum' + )[0] np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) @@ -264,8 +283,8 @@ def test_NLLLoss_1D_with_weight_mean_cpu(self): np.random.seed(200) input_np = np.random.random(size=(10, 10)).astype(np.float64) np.random.seed(200) - label_np = np.random.randint(0, 10, size=(10, )).astype(np.int64) - weight_np = np.random.random(size=(10, )).astype(np.float64) + label_np = np.random.randint(0, 10, size=(10,)).astype(np.int64) + weight_np = np.random.random(size=(10,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() place = fluid.CPUPlace() @@ -277,19 +296,23 @@ def test_NLLLoss_1D_with_weight_mean_cpu(self): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np)) - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np) + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() expected = nll_loss_1d(input_np, label_np, weight=weight_np)[0] @@ -301,8 +324,8 @@ def test_NLLLoss_1D_with_weight_no_reduce_cpu(self): np.random.seed(200) input_np = np.random.random(size=(10, 10)).astype(np.float64) np.random.seed(200) - label_np = np.random.randint(0, 10, size=(10, )).astype(np.int64) - weight_np = np.random.random(size=(10, )).astype(np.float64) + label_np = np.random.randint(0, 10, size=(10,)).astype(np.int64) + weight_np = np.random.random(size=(10,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() place = fluid.CPUPlace() @@ -314,24 +337,27 @@ def test_NLLLoss_1D_with_weight_no_reduce_cpu(self): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np), reduction='none') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np), reduction='none' + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() - expected = nll_loss_1d(input_np, - label_np, - weight=weight_np, - reduction='none') + expected = nll_loss_1d( + input_np, label_np, weight=weight_np, reduction='none' + ) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) @@ -344,29 +370,32 @@ def test_NLLLoss_2D_mean(self): label_np = np.random.randint(0, 3, size=(5, 5, 5)).astype(np.int64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5], dtype='int64') nll_loss = paddle.nn.loss.NLLLoss() res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss() - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() expected = nll_loss_2d(input_np, label_np)[0] @@ -382,29 +411,32 @@ def test_NLLLoss_2D_sum(self): label_np = np.random.randint(0, 3, size=(5, 5, 5)).astype(np.int64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5], dtype='int64') nll_loss = paddle.nn.loss.NLLLoss(reduction='sum') res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss(reduction='sum') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() expected = nll_loss_2d(input_np, label_np, reduction='sum')[0] @@ -418,16 +450,19 @@ def test_NLLLoss_2D_with_weight_mean(self): input_np = np.random.random(size=(5, 3, 5, 5)).astype(np.float64) np.random.seed(200) label_np = np.random.randint(0, 3, size=(5, 5, 5)).astype(np.int64) - weight_np = np.random.random(size=(3, )).astype(np.float64) + weight_np = np.random.random(size=(3,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype='float64') @@ -435,19 +470,23 @@ def test_NLLLoss_2D_with_weight_mean(self): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np)) - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np) + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() expected = nll_loss_2d(input_np, label_np, weight=weight_np)[0] @@ -461,14 +500,14 @@ def test_NLLLoss_2D_with_weight_mean_cpu(self): input_np = np.random.random(size=(5, 3, 5, 5)).astype(np.float64) np.random.seed(200) label_np = np.random.randint(0, 3, size=(5, 5, 5)).astype(np.int64) - weight_np = np.random.random(size=(3, )).astype(np.float64) + weight_np = np.random.random(size=(3,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype='float64') @@ -476,19 +515,23 @@ def test_NLLLoss_2D_with_weight_mean_cpu(self): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np)) - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np) + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() expected = nll_loss_2d(input_np, label_np, weight=weight_np)[0] @@ -502,15 +545,18 @@ def test_NLLLoss_2D_with_weight_sum(self): input_np = np.random.random(size=(5, 3, 5, 5)).astype(np.float64) np.random.seed(200) label_np = np.random.randint(0, 3, size=(5, 5, 5)).astype(np.int64) - weight_np = np.random.random(size=(3, )).astype(np.float64) + weight_np = np.random.random(size=(3,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype='float64') @@ -518,25 +564,28 @@ def test_NLLLoss_2D_with_weight_sum(self): res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np), reduction='sum') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np), reduction='sum' + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() - expected = nll_loss_2d(input_np, - label_np, - weight=weight_np, - reduction='sum')[0] + expected = nll_loss_2d( + input_np, label_np, weight=weight_np, reduction='sum' + )[0] np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) @@ -549,35 +598,39 @@ def test_NLLLoss_in_dims_not_2or4_mean(self): label_np = np.random.randint(0, 3, size=(5, 5, 5, 5)).astype(np.int64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5, 5], dtype='int64') nll_loss = paddle.nn.loss.NLLLoss() res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={"input": input_np, "label": label_np}, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss() - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() input_shape = input_np.shape label_shape = label_np.shape - input_np_reshape = np.reshape(input_np, - (input_shape[0], input_shape[1], 1, -1)) + input_np_reshape = np.reshape( + input_np, (input_shape[0], input_shape[1], 1, -1) + ) label_np_reshape = np.reshape(label_np, (label_shape[0], 1, -1)) expected = nll_loss_2d(input_np_reshape, label_np_reshape)[0] @@ -590,45 +643,53 @@ def test_NLLLoss_in_dims_not_2or4_with_weight_mean(self): input_np = np.random.random(size=(5, 3, 5, 5, 5)).astype(np.float64) np.random.seed(200) label_np = np.random.randint(0, 3, size=(5, 5, 5, 5)).astype(np.int64) - weight_np = np.random.random(size=(3, )).astype(np.float64) + weight_np = np.random.random(size=(3,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5, 5], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype='float64') nll_loss = paddle.nn.loss.NLLLoss(weight=weight) res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np)) - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np) + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() input_shape = input_np.shape label_shape = label_np.shape - input_np_reshape = np.reshape(input_np, - (input_shape[0], input_shape[1], 1, -1)) + input_np_reshape = np.reshape( + input_np, (input_shape[0], input_shape[1], 1, -1) + ) label_np_reshape = np.reshape(label_np, (label_shape[0], 1, -1)) - expected = nll_loss_2d(input_np_reshape, - label_np_reshape, - weight=weight_np)[0] + expected = nll_loss_2d( + input_np_reshape, label_np_reshape, weight=weight_np + )[0] np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) @@ -639,46 +700,56 @@ def test_NLLLoss_in_dims_not_2or4_with_weight_sum(self): input_np = np.random.random(size=(5, 3, 5, 5, 5)).astype(np.float64) np.random.seed(200) label_np = np.random.randint(0, 3, size=(5, 5, 5, 5)).astype(np.int64) - weight_np = np.random.random(size=(3, )).astype(np.float64) + weight_np = np.random.random(size=(3,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5, 5], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype='float64') nll_loss = paddle.nn.loss.NLLLoss(weight=weight, reduction='sum') res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np), reduction='sum') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np), reduction='sum' + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() input_shape = input_np.shape label_shape = label_np.shape - input_np_reshape = np.reshape(input_np, - (input_shape[0], input_shape[1], 1, -1)) + input_np_reshape = np.reshape( + input_np, (input_shape[0], input_shape[1], 1, -1) + ) label_np_reshape = np.reshape(label_np, (label_shape[0], 1, -1)) - expected = nll_loss_2d(input_np_reshape, - label_np_reshape, - weight=weight_np, - reduction='sum')[0] + expected = nll_loss_2d( + input_np_reshape, + label_np_reshape, + weight=weight_np, + reduction='sum', + )[0] np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) @@ -689,47 +760,57 @@ def test_NLLLoss_in_dims_not_2or4_with_weight_no_reduce(self): input_np = np.random.random(size=(5, 3, 5, 5, 5)).astype(np.float64) np.random.seed(200) label_np = np.random.randint(0, 3, size=(5, 5, 5, 5)).astype(np.int64) - weight_np = np.random.random(size=(3, )).astype(np.float64) + weight_np = np.random.random(size=(3,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() - #place = fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) + # place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5, 5], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype='float64') nll_loss = paddle.nn.loss.NLLLoss(weight=weight, reduction='none') res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np), reduction='none') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np), reduction='none' + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() input_shape = input_np.shape label_shape = label_np.shape - out_shape = (input_shape[0], ) + input_shape[2:] - input_np_reshape = np.reshape(input_np, - (input_shape[0], input_shape[1], 1, -1)) + out_shape = (input_shape[0],) + input_shape[2:] + input_np_reshape = np.reshape( + input_np, (input_shape[0], input_shape[1], 1, -1) + ) label_np_reshape = np.reshape(label_np, (label_shape[0], 1, -1)) - expected = nll_loss_2d(input_np_reshape, - label_np_reshape, - weight=weight_np, - reduction='none') + expected = nll_loss_2d( + input_np_reshape, + label_np_reshape, + weight=weight_np, + reduction='none', + ) expected = np.reshape(expected, out_shape) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) @@ -740,45 +821,52 @@ def test_NLLLoss_in_dims_not_2or4_with_weight_no_reduce_cpu(self): input_np = np.random.random(size=(5, 3, 5, 5, 5)).astype(np.float64) np.random.seed(200) label_np = np.random.randint(0, 3, size=(5, 5, 5, 5)).astype(np.int64) - weight_np = np.random.random(size=(3, )).astype(np.float64) + weight_np = np.random.random(size=(3,)).astype(np.float64) prog = fluid.Program() startup_prog = fluid.Program() place = fluid.CPUPlace() with fluid.program_guard(prog, startup_prog): - input = fluid.data(name='input', - shape=[5, 3, 5, 5, 5], - dtype='float64') + input = fluid.data( + name='input', shape=[5, 3, 5, 5, 5], dtype='float64' + ) label = fluid.data(name='label', shape=[5, 5, 5, 5], dtype='int64') weight = fluid.data(name='weight', shape=[3], dtype='float64') nll_loss = paddle.nn.loss.NLLLoss(weight=weight, reduction='none') res = nll_loss(input, label) exe = fluid.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np, - "weight": weight_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, + feed={ + "input": input_np, + "label": label_np, + "weight": weight_np, + }, + fetch_list=[res], + ) with fluid.dygraph.guard(): nll_loss = paddle.nn.loss.NLLLoss( - weight=paddle.to_tensor(weight_np), reduction='none') - dy_res = nll_loss(paddle.to_tensor(input_np), - paddle.to_tensor(label_np)) + weight=paddle.to_tensor(weight_np), reduction='none' + ) + dy_res = nll_loss( + paddle.to_tensor(input_np), paddle.to_tensor(label_np) + ) dy_result = dy_res.numpy() input_shape = input_np.shape label_shape = label_np.shape - out_shape = (input_shape[0], ) + input_shape[2:] - input_np_reshape = np.reshape(input_np, - (input_shape[0], input_shape[1], 1, -1)) + out_shape = (input_shape[0],) + input_shape[2:] + input_np_reshape = np.reshape( + input_np, (input_shape[0], input_shape[1], 1, -1) + ) label_np_reshape = np.reshape(label_np, (label_shape[0], 1, -1)) - expected = nll_loss_2d(input_np_reshape, - label_np_reshape, - weight=weight_np, - reduction='none') + expected = nll_loss_2d( + input_np_reshape, + label_np_reshape, + weight=weight_np, + reduction='none', + ) expected = np.reshape(expected, out_shape) np.testing.assert_allclose(static_result, expected, rtol=1e-05) np.testing.assert_allclose(static_result, dy_result, rtol=1e-05) @@ -786,7 +874,6 @@ def test_NLLLoss_in_dims_not_2or4_with_weight_no_reduce_cpu(self): class TestNLLLossOp1DWithReduce(OpTest): - def setUp(self): self.init_test_case() self.op_type = "nll_loss" @@ -796,20 +883,23 @@ def setUp(self): self.python_api = paddle.nn.functional.nll_loss self.python_out_sig = ["Out"] np.random.seed(200) - input_np = np.random.uniform(0.1, 0.8, - self.input_shape).astype("float64") + input_np = np.random.uniform(0.1, 0.8, self.input_shape).astype( + "float64" + ) np.random.seed(200) - label_np = np.random.randint(0, self.input_shape[1], - self.label_shape).astype("int64") + label_np = np.random.randint( + 0, self.input_shape[1], self.label_shape + ).astype("int64") output_np, total_weight_np = nll_loss_1d(input_np, label_np) self.inputs = {'X': input_np, 'Label': label_np} if self.with_weight: np.random.seed(200) - weight_np = np.random.uniform(0.1, 0.8, - self.input_shape[1]).astype("float64") - output_np, total_weight_np = nll_loss_1d(input_np, - label_np, - weight=weight_np) + weight_np = np.random.uniform(0.1, 0.8, self.input_shape[1]).astype( + "float64" + ) + output_np, total_weight_np = nll_loss_1d( + input_np, label_np, weight=weight_np + ) self.inputs['Weight'] = weight_np self.outputs = {'Out': output_np, 'Total_weight': total_weight_np} @@ -836,7 +926,6 @@ def init_test_case(self): class TestNLLLossOp1DNoReduce(OpTest): - def setUp(self): self.init_test_case() self.op_type = "nll_loss" @@ -844,22 +933,24 @@ def setUp(self): self.python_out_sig = ["Out"] self.with_weight = False np.random.seed(200) - input_np = np.random.uniform(0.1, 0.8, - self.input_shape).astype("float64") + input_np = np.random.uniform(0.1, 0.8, self.input_shape).astype( + "float64" + ) np.random.seed(200) - label_np = np.random.randint(0, self.input_shape[1], - self.label_shape).astype("int64") + label_np = np.random.randint( + 0, self.input_shape[1], self.label_shape + ).astype("int64") output_np = nll_loss_1d(input_np, label_np, reduction='none') total_weight_np = np.array([0]).astype('float64') self.inputs = {'X': input_np, 'Label': label_np} if self.with_weight: np.random.seed(200) - weight_np = np.random.uniform(0.1, 0.8, - self.input_shape[1]).astype("float64") - output_np, total_weight_np = nll_loss_1d(input_np, - label_np, - weight=weight_np, - reduction='none') + weight_np = np.random.uniform(0.1, 0.8, self.input_shape[1]).astype( + "float64" + ) + output_np, total_weight_np = nll_loss_1d( + input_np, label_np, weight=weight_np, reduction='none' + ) self.inputs['Weight'] = weight_np self.outputs = {'Out': output_np, 'Total_weight': total_weight_np} @@ -886,7 +977,6 @@ def init_test_case(self): class TestNLLLossOp2DWithReduce(OpTest): - def setUp(self): self.init_test_case() self.op_type = "nll_loss" @@ -894,20 +984,23 @@ def setUp(self): self.python_out_sig = ["Out"] self.with_weight = False np.random.seed(200) - input_np = np.random.uniform(0.1, 0.8, - self.input_shape).astype("float64") + input_np = np.random.uniform(0.1, 0.8, self.input_shape).astype( + "float64" + ) np.random.seed(200) - label_np = np.random.randint(0, self.input_shape[1], - self.label_shape).astype("int64") + label_np = np.random.randint( + 0, self.input_shape[1], self.label_shape + ).astype("int64") output_np, total_weight_np = nll_loss_2d(input_np, label_np) self.inputs = {'X': input_np, 'Label': label_np} if self.with_weight: np.random.seed(200) - weight_np = np.random.uniform(0.1, 0.8, - self.input_shape[1]).astype("float64") - output_np, total_weight_np = nll_loss_2d(input_np, - label_np, - weight=weight_np) + weight_np = np.random.uniform(0.1, 0.8, self.input_shape[1]).astype( + "float64" + ) + output_np, total_weight_np = nll_loss_2d( + input_np, label_np, weight=weight_np + ) self.inputs['Weight'] = weight_np self.outputs = {'Out': output_np, 'Total_weight': total_weight_np} @@ -934,7 +1027,6 @@ def init_test_case(self): class TestNLLLossOp2DNoReduce(OpTest): - def setUp(self): self.init_test_case() self.op_type = "nll_loss" @@ -942,22 +1034,24 @@ def setUp(self): self.python_out_sig = ["Out"] self.with_weight = False np.random.seed(200) - input_np = np.random.uniform(0.1, 0.8, - self.input_shape).astype("float64") + input_np = np.random.uniform(0.1, 0.8, self.input_shape).astype( + "float64" + ) np.random.seed(200) - label_np = np.random.randint(0, self.input_shape[1], - self.label_shape).astype("int64") + label_np = np.random.randint( + 0, self.input_shape[1], self.label_shape + ).astype("int64") output_np = nll_loss_2d(input_np, label_np, reduction='none') total_weight_np = np.array([0]).astype('float64') self.inputs = {'X': input_np, 'Label': label_np} if self.with_weight: np.random.seed(200) - weight_np = np.random.uniform(0.1, 0.8, - self.input_shape[1]).astype("float64") - output_np, total_weight_np = nll_loss_2d(input_np, - label_np, - weight=weight_np, - reduction='none') + weight_np = np.random.uniform(0.1, 0.8, self.input_shape[1]).astype( + "float64" + ) + output_np, total_weight_np = nll_loss_2d( + input_np, label_np, weight=weight_np, reduction='none' + ) self.inputs['Weight'] = weight_np self.outputs = {'Out': output_np, 'Total_weight': total_weight_np} @@ -984,7 +1078,6 @@ def init_test_case(self): class TestNLLLossName(unittest.TestCase): - def test_name(self): prog = paddle.static.Program() startup_prog = paddle.static.Program() @@ -998,22 +1091,26 @@ def test_name(self): class TestNLLLossInvalidArgs(unittest.TestCase): - def test_x_dim_value_error(self): - def test_x_dim_lt_2(): prog = paddle.static.Program() startup_prog = paddle.static.Program() place = paddle.CPUPlace() with paddle.static.program_guard(prog, startup_prog): - x = paddle.fluid.data(name='x', shape=[ - 10, - ], dtype='float64') - label = paddle.fluid.data(name='label', - shape=[ - 10, - ], - dtype='float64') + x = paddle.fluid.data( + name='x', + shape=[ + 10, + ], + dtype='float64', + ) + label = paddle.fluid.data( + name='label', + shape=[ + 10, + ], + dtype='float64', + ) nll_loss = paddle.nn.loss.NLLLoss() res = nll_loss(x, label) @@ -1021,8 +1118,8 @@ def test_x_dim_lt_2(): def test_x_dim_imperative_lt_2(): with fluid.dygraph.guard(): - x_np = np.random.random(size=(5, )).astype(np.float64) - label_np = np.random.randint(0, 10, size=(5, )).astype(np.int64) + x_np = np.random.random(size=(5,)).astype(np.float64) + label_np = np.random.randint(0, 10, size=(5,)).astype(np.int64) x = paddle.to_tensor(x_np) label = paddle.to_tensor(label_np) nll_loss = paddle.nn.loss.NLLLoss() @@ -1031,16 +1128,15 @@ def test_x_dim_imperative_lt_2(): self.assertRaises(ValueError, test_x_dim_imperative_lt_2) def test_reduction_value_error(self): - def test_NLLLoss_reduction_not_sum_mean_none(): prog = paddle.static.Program() startup_prog = paddle.static.Program() place = paddle.CPUPlace() with paddle.static.program_guard(prog, startup_prog): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float64') - label = paddle.fluid.data(name='label', - shape=[10], - dtype='int64') + label = paddle.fluid.data( + name='label', shape=[10], dtype='int64' + ) nll_loss = paddle.nn.loss.NLLLoss(reduction='') res = nll_loss(x, label) @@ -1049,14 +1145,15 @@ def test_NLLLoss_reduction_not_sum_mean_none(): def test_NLLLoss_reduction_imperative_not_sum_mean_none(): with fluid.dygraph.guard(): x_np = np.random.random(size=(5, 3)).astype(np.float64) - label_np = np.random.randint(0, 3, size=(5, )).astype(np.int64) + label_np = np.random.randint(0, 3, size=(5,)).astype(np.int64) x = paddle.to_tensor(x_np) label = paddle.to_tensor(label_np) nll_loss = paddle.nn.loss.NLLLoss(reduction='') res = nll_loss(x, label) - self.assertRaises(ValueError, - test_NLLLoss_reduction_imperative_not_sum_mean_none) + self.assertRaises( + ValueError, test_NLLLoss_reduction_imperative_not_sum_mean_none + ) def test_nll_loss_function_reduction_not_sum_mean_none(): prog = paddle.static.Program() @@ -1064,25 +1161,27 @@ def test_nll_loss_function_reduction_not_sum_mean_none(): place = paddle.CPUPlace() with paddle.static.program_guard(prog, startup_prog): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float64') - label = paddle.fluid.data(name='label', - shape=[10], - dtype='int64') + label = paddle.fluid.data( + name='label', shape=[10], dtype='int64' + ) res = paddle.nn.functional.nll_loss(x, label, reduction='') - self.assertRaises(ValueError, - test_nll_loss_function_reduction_not_sum_mean_none) + self.assertRaises( + ValueError, test_nll_loss_function_reduction_not_sum_mean_none + ) def test_nll_loss_function_reduction_imperative_not_sum_mean_none(): with fluid.dygraph.guard(): x_np = np.random.random(size=(5, 3)).astype(np.float64) - label_np = np.random.randint(0, 3, size=(5, )).astype(np.int64) + label_np = np.random.randint(0, 3, size=(5,)).astype(np.int64) x = paddle.to_tensor(x_np) label = paddle.to_tensor(label_np) res = paddle.nn.functional.nll_loss(x, label, reduction='') self.assertRaises( ValueError, - test_nll_loss_function_reduction_imperative_not_sum_mean_none) + test_nll_loss_function_reduction_imperative_not_sum_mean_none, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_nms_op.py b/python/paddle/fluid/tests/unittests/test_nms_op.py index cbd24d4ddf22e..80f2da013251d 100755 --- a/python/paddle/fluid/tests/unittests/test_nms_op.py +++ b/python/paddle/fluid/tests/unittests/test_nms_op.py @@ -19,8 +19,7 @@ def iou(box_a, box_b): - """Apply intersection-over-union overlap between box_a and box_b - """ + """Apply intersection-over-union overlap between box_a and box_b""" xmin_a = min(box_a[0], box_a[2]) ymin_a = min(box_a[1], box_a[3]) xmax_a = max(box_a[0], box_a[2]) @@ -69,7 +68,6 @@ def nms(boxes, nms_threshold): class TestNMSOp(OpTest): - def setUp(self): self.op_type = 'nms' self.python_api = paddle.vision.ops.nms diff --git a/python/paddle/fluid/tests/unittests/test_nn_dice_loss.py b/python/paddle/fluid/tests/unittests/test_nn_dice_loss.py index f5a7d2ed4ed8e..f1e1f6f6e4b2a 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_dice_loss.py +++ b/python/paddle/fluid/tests/unittests/test_nn_dice_loss.py @@ -22,7 +22,6 @@ class TestDiceLossValue(unittest.TestCase): - def test_dice_loss(self): input_ = paddle.rand([2, 3, num_classes]) label_ = paddle.randint(0, num_classes, [2, 3, 1], dtype=paddle.int64) @@ -40,23 +39,21 @@ def test_dice_loss(self): class TestDiceLossInvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_dtype(): input_ = paddle.rand([2, 3, num_classes], dtype=paddle.float32) - label_ = paddle.randint(0, - num_classes, [2, 3, 1], - dtype=paddle.int64) + label_ = paddle.randint( + 0, num_classes, [2, 3, 1], dtype=paddle.int64 + ) nn.dice_loss(input_, label_.astype(paddle.float32)) self.assertRaises(AssertionError, test_invalid_dtype) def test_zero_shape_input(): input_ = paddle.rand([0, 3, num_classes], dtype=paddle.float32) - label_ = paddle.randint(0, - num_classes, [0, 3, 1], - dtype=paddle.int64) + label_ = paddle.randint( + 0, num_classes, [0, 3, 1], dtype=paddle.int64 + ) nn.dice_loss(input_, label_) self.assertRaises(AssertionError, test_zero_shape_input) diff --git a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py index d658fcec836db..a7062e600258e 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py +++ b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py @@ -22,7 +22,6 @@ class EmbeddingDygraph(unittest.TestCase): - def func_1(self): x_data = np.arange(3, 6).reshape((3, 1)).astype(np.int64) paddle.disable_static(paddle.CPUPlace()) @@ -33,8 +32,9 @@ def func_1(self): w0 = np.full(shape=(10, 3), fill_value=2).astype(np.float32) embedding.weight.set_value(w0) - adam = paddle.optimizer.Adam(parameters=[embedding.weight], - learning_rate=0.01) + adam = paddle.optimizer.Adam( + parameters=[embedding.weight], learning_rate=0.01 + ) adam.clear_grad() out = embedding(x) diff --git a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_static.py b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_static.py index bb1541beb1f44..b5c354c7486f6 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_static.py +++ b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_static.py @@ -19,33 +19,36 @@ class EmbeddingStatic(unittest.TestCase): - def test_1(self): prog = fluid.Program() with fluid.program_guard(prog): def test_bad_x(): initializer = fluid.initializer.NumpyArrayInitializer( - np.random.random(size=(128, 100))) - - param_attr = fluid.ParamAttr(name="emb_weight", - learning_rate=0.5, - initializer=initializer, - trainable=True) - - weight = prog.global_block().create_parameter((128, 100), - attr=param_attr, - dtype="float32") - - label = fluid.layers.data(name="label", - shape=[4], - append_batch_size=False, - dtype="int64") - - emb = functional.embedding(x=label, - weight=weight, - sparse=True, - name="embedding") + np.random.random(size=(128, 100)) + ) + + param_attr = fluid.ParamAttr( + name="emb_weight", + learning_rate=0.5, + initializer=initializer, + trainable=True, + ) + + weight = prog.global_block().create_parameter( + (128, 100), attr=param_attr, dtype="float32" + ) + + label = fluid.layers.data( + name="label", + shape=[4], + append_batch_size=False, + dtype="int64", + ) + + emb = functional.embedding( + x=label, weight=weight, sparse=True, name="embedding" + ) test_bad_x() @@ -55,27 +58,34 @@ def test_2(self): def test_bad_x(): initializer = fluid.initializer.NumpyArrayInitializer( - np.random.random(size=(128, 100))) - - param_attr = fluid.ParamAttr(name="emb_weight", - learning_rate=0.5, - initializer=initializer, - trainable=True) - - weight = prog.global_block().create_parameter((128, 100), - attr=param_attr, - dtype="float32") - - label = fluid.layers.data(name="label", - shape=[4], - append_batch_size=False, - dtype="int32") - - emb = functional.embedding(x=label, - weight=weight, - padding_idx=129, - sparse=True, - name="embedding") + np.random.random(size=(128, 100)) + ) + + param_attr = fluid.ParamAttr( + name="emb_weight", + learning_rate=0.5, + initializer=initializer, + trainable=True, + ) + + weight = prog.global_block().create_parameter( + (128, 100), attr=param_attr, dtype="float32" + ) + + label = fluid.layers.data( + name="label", + shape=[4], + append_batch_size=False, + dtype="int32", + ) + + emb = functional.embedding( + x=label, + weight=weight, + padding_idx=129, + sparse=True, + name="embedding", + ) with self.assertRaises(ValueError): test_bad_x() diff --git a/python/paddle/fluid/tests/unittests/test_nn_functional_hot_op.py b/python/paddle/fluid/tests/unittests/test_nn_functional_hot_op.py index 27a7af9448520..f7126a998818f 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_functional_hot_op.py +++ b/python/paddle/fluid/tests/unittests/test_nn_functional_hot_op.py @@ -23,7 +23,6 @@ class TestOneHotOp(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -47,7 +46,6 @@ def test_check_output(self): class TestOneHotOp_attr(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -56,8 +54,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -71,7 +70,6 @@ def test_check_output(self): class TestOneHotOp_default_dtype(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -95,7 +93,6 @@ def test_check_output(self): class TestOneHotOp_default_dtype_attr(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -104,8 +101,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -119,7 +117,6 @@ def test_check_output(self): class TestOneHotOp_exception(unittest.TestCase): - def setUp(self): self.op_type = 'one_hot_v2' self.depth = 10 @@ -135,30 +132,34 @@ def setUp(self): def test_check_output(self): program = Program() with program_guard(program): - x = fluid.layers.data(name='x', - shape=[self.dimension], - dtype='float32', - lod_level=1) + x = fluid.layers.data( + name='x', shape=[self.dimension], dtype='float32', lod_level=1 + ) block = program.current_block() - one_hot_out = block.create_var(name="one_hot_out", - type=core.VarDesc.VarType.LOD_TENSOR, - dtype='float32') - block.append_op(type='one_hot', - inputs={'X': x}, - attrs={'depth': self.depth}, - outputs={'Out': one_hot_out}) + one_hot_out = block.create_var( + name="one_hot_out", + type=core.VarDesc.VarType.LOD_TENSOR, + dtype='float32', + ) + block.append_op( + type='one_hot', + inputs={'X': x}, + attrs={'depth': self.depth}, + outputs={'Out': one_hot_out}, + ) exe = fluid.Executor(self.place) def run(): - exe.run(feed={'x': self.x}, - fetch_list=[one_hot_out], - return_numpy=False) + exe.run( + feed={'x': self.x}, + fetch_list=[one_hot_out], + return_numpy=False, + ) self.assertRaises(ValueError, run) class TestOneHotOpApi(unittest.TestCase): - def test_api(self): num_classes = 10 self._run(num_classes) @@ -169,40 +170,45 @@ def test_api_with_depthTensor(self): def test_api_with_dygraph(self): num_classes = 10 - label = np.array([ - np.random.randint(0, num_classes - 1) for i in range(6) - ]).reshape([6, 1]) + label = np.array( + [np.random.randint(0, num_classes - 1) for i in range(6)] + ).reshape([6, 1]) with fluid.dygraph.guard(): one_hot_label = functional.one_hot( - x=fluid.dygraph.to_variable(label), num_classes=num_classes) + x=fluid.dygraph.to_variable(label), num_classes=num_classes + ) def _run(self, num_classes): label = fluid.layers.data(name="label", shape=[1], dtype="int64") one_hot_label = functional.one_hot(x=label, num_classes=num_classes) place = fluid.CPUPlace() - label_data = np.array([np.random.randint(0, 10 - 1) - for i in range(6)]).reshape([6, 1]) + label_data = np.array( + [np.random.randint(0, 10 - 1) for i in range(6)] + ).reshape([6, 1]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'label': label_data, - }, - fetch_list=[one_hot_label], - return_numpy=False) + ret = exe.run( + feed={ + 'label': label_data, + }, + fetch_list=[one_hot_label], + return_numpy=False, + ) class BadInputTestOnehotV2(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): def test_bad_x(): - label = fluid.layers.data(name="label", - shape=[4], - append_batch_size=False, - dtype="float32") + label = fluid.layers.data( + name="label", + shape=[4], + append_batch_size=False, + dtype="float32", + ) one_hot_label = functional.one_hot(x=label, num_classes=4) self.assertRaises(TypeError, test_bad_x) diff --git a/python/paddle/fluid/tests/unittests/test_nn_grad.py b/python/paddle/fluid/tests/unittests/test_nn_grad.py index 7b806491bc560..4b3d92c92dcea 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_nn_grad.py @@ -26,28 +26,25 @@ class TestSliceOpDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): self.config() - out = fluid.layers.slice(self.inputs, - axes=self.axes, - starts=self.starts, - ends=self.ends) - gradient_checker.double_grad_check([self.inputs], - out, - x_init=self.x_arr, - place=place) + out = fluid.layers.slice( + self.inputs, axes=self.axes, starts=self.starts, ends=self.ends + ) + gradient_checker.double_grad_check( + [self.inputs], out, x_init=self.x_arr, place=place + ) def config(self): self.starts = [1, 0, -1] self.ends = [3, 3, 6] self.axes = [0, 1, 2] self.x_arr = np.random.random([3, 4, 5, 2]).astype("float64") - self.inputs = layers.create_parameter(dtype="float64", - shape=[3, 4, 5, 2], - name='x') + self.inputs = layers.create_parameter( + dtype="float64", shape=[3, 4, 5, 2], name='x' + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -58,19 +55,17 @@ def test_grad(self): class TestSliceOpDoubleGradCheckCase3(TestSliceOpDoubleGradCheck): - def config(self): self.starts = [1, -1, 1] self.ends = [3, 3, 3] self.axes = [0, 1, 2] self.x_arr = np.random.random([3, 3, 3]).astype("float64") - self.inputs = layers.create_parameter(dtype="float64", - shape=[3, 3, 3], - name='x3') + self.inputs = layers.create_parameter( + dtype="float64", shape=[3, 3, 3], name='x3' + ) class TestReduceMeanWithDimDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): shape = [7, 11] @@ -82,11 +77,9 @@ def func(self, place): y = layers.reduce_mean(x, dim=0) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -97,7 +90,6 @@ def test_grad(self): class TestReduceSumWithDimDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): shape = [7, 11] @@ -109,11 +101,9 @@ def func(self, place): y = layers.reduce_sum(x, dim=0) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - y, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -124,7 +114,6 @@ def test_grad(self): class TestReshapeDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): x_shape = [3, 12] @@ -137,11 +126,9 @@ def func(self, place): out = layers.expand(x, expand_times) x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x], - out, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -152,7 +139,6 @@ def test_grad(self): class TestExpandDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): x_shape = [3, 12] @@ -165,11 +151,9 @@ def func(self, place): out = layers.reshape(x, new_shape) x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x], - out, - x_init=x_arr, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -180,7 +164,6 @@ def test_grad(self): class TestTileDoubleGradCheck(unittest.TestCase): - def tile_wrapper(self, x): return paddle.tile(x[0], [4, 9]) @@ -196,15 +179,12 @@ def func(self, place): out = paddle.tile(x, repeat_times) x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x], - out, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.tile_wrapper, [x], - out, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.tile_wrapper, [x], out, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -215,7 +195,6 @@ def test_grad(self): class TestExpandV2DoubleGradCheck(unittest.TestCase): - def expand_wrapper(self, x): return paddle.expand(x[0], [4, 12]) @@ -231,15 +210,12 @@ def func(self, place): out = paddle.expand(x, new_shape) x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x], - out, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.expand_wrapper, [x], - out, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.expand_wrapper, [x], out, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -250,7 +226,6 @@ def test_grad(self): class TestSqueezeDoubleGradCheck(unittest.TestCase): - def squeeze_warpper(self, x): axes = [0, 2] return paddle.squeeze(x[0], axes) @@ -267,16 +242,12 @@ def func(self, place): out = paddle.squeeze(x, axes) x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x], - out, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.squeeze_warpper, - [x], - out, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.squeeze_warpper, [x], out, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -287,7 +258,6 @@ def test_grad(self): class TestUnsqueezeDoubleGradCheck(unittest.TestCase): - def unsqueeze_wrapper(self, x): axes = [1, 2] return paddle.unsqueeze(x[0], axes) @@ -304,16 +274,12 @@ def func(self, place): out = paddle.unsqueeze(x, axes) x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x], - out, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.unsqueeze_wrapper, - [x], - out, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.unsqueeze_wrapper, [x], out, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -324,9 +290,8 @@ def test_grad(self): class TestClipDoubleGradCheck(unittest.TestCase): - def clip_wrapper(self, x): - return paddle.clip(x[0], min=-1., max=1.) + return paddle.clip(x[0], min=-1.0, max=1.0) @prog_scope() def func(self, place): @@ -335,14 +300,13 @@ def func(self, place): x = layers.data('x', x_shape, False, dtype) x.persistable = True - out = paddle.clip(x, min=-1., max=1.) - x_arr = np.random.uniform(-5., 5., x_shape).astype(dtype) + out = paddle.clip(x, min=-1.0, max=1.0) + x_arr = np.random.uniform(-5.0, 5.0, x_shape).astype(dtype) gradient_checker.double_grad_check([x], out, x_init=x_arr, place=place) - gradient_checker.double_grad_check_for_dygraph(self.clip_wrapper, [x], - out, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.clip_wrapper, [x], out, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -353,7 +317,6 @@ def test_grad(self): class TestTransposeDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): x_shape = [3, 40] @@ -376,7 +339,6 @@ def test_grad(self): class TestTransposeDoubleGradCheckCase1(unittest.TestCase): - @prog_scope() def func(self, place): x_shape = [2, 3, 4, 5] @@ -399,7 +361,6 @@ def test_grad(self): class TestConstantPadDoubleGradCheck(unittest.TestCase): - def pad_wrapper(self, x): pad = [1, 1, 1, 1] return paddle.nn.functional.pad(x[0], pad) @@ -416,15 +377,12 @@ def func(self, place): out = paddle.nn.functional.pad(x, pad) x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x], - out, - x_init=x_arr, - place=place, - eps=eps) - gradient_checker.double_grad_check_for_dygraph(self.pad_wrapper, [x], - out, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps + ) + gradient_checker.double_grad_check_for_dygraph( + self.pad_wrapper, [x], out, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -435,7 +393,6 @@ def test_grad(self): class TestConstantPadDoubleGradCheckCase1(TestConstantPadDoubleGradCheck): - @prog_scope() def func(self, place): x_shape = [2, 3, 4, 5] @@ -451,7 +408,6 @@ def func(self, place): class TestConcatDoubleGradCheck(unittest.TestCase): - def concat_wrapper(self, x): return paddle.concat(x, axis=0) @@ -469,15 +425,16 @@ def func(self, place): x2_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) x1_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) - gradient_checker.double_grad_check([x1, x2], - out, - x_init=[x1_arr, x2_arr], - place=place) - gradient_checker.double_grad_check_for_dygraph(self.concat_wrapper, - [x1, x2], - out, - x_init=[x1_arr, x2_arr], - place=place) + gradient_checker.double_grad_check( + [x1, x2], out, x_init=[x1_arr, x2_arr], place=place + ) + gradient_checker.double_grad_check_for_dygraph( + self.concat_wrapper, + [x1, x2], + out, + x_init=[x1_arr, x2_arr], + place=place, + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -488,23 +445,22 @@ def test_grad(self): class TestAvgPool2DDoubleGradCheckCase1(unittest.TestCase): - @prog_scope() def func(self, place): - input_NCHW = fluid.layers.data(name="input_NCHW", - shape=[2, 3, 5, 5], - append_batch_size=False, - dtype="float32") + input_NCHW = fluid.layers.data( + name="input_NCHW", + shape=[2, 3, 5, 5], + append_batch_size=False, + dtype="float32", + ) input_NCHW.persistable = True y = layers.pool2d(input_NCHW, pool_size=2, pool_type="avg") x_arr = np.random.uniform(-1, 1, [2, 3, 5, 5]).astype(np.float32) - gradient_checker.double_grad_check([input_NCHW], - y, - x_init=x_arr, - place=place, - eps=0.05) + gradient_checker.double_grad_check( + [input_NCHW], y, x_init=x_arr, place=place, eps=0.05 + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -515,36 +471,33 @@ def test_grad(self): class TestAvgPool2DDoubleGradCheckCase2(unittest.TestCase): - def pool2d_wrapper(self, x): - return paddle.nn.functional.avg_pool2d(x[0], - kernel_size=2, - data_format="NHWC") + return paddle.nn.functional.avg_pool2d( + x[0], kernel_size=2, data_format="NHWC" + ) @prog_scope() def func(self, place): - input_NHWC = fluid.layers.data(name="input_NHWC", - shape=[2, 5, 5, 3], - append_batch_size=False, - dtype="float32") + input_NHWC = fluid.layers.data( + name="input_NHWC", + shape=[2, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) input_NHWC.persistable = True - y = paddle.nn.functional.avg_pool2d(input_NHWC, - kernel_size=2, - data_format="NHWC") + y = paddle.nn.functional.avg_pool2d( + input_NHWC, kernel_size=2, data_format="NHWC" + ) x_arr = np.random.uniform(-1, 1, [2, 5, 5, 3]).astype(np.float32) - gradient_checker.double_grad_check([input_NHWC], - y, - x_init=x_arr, - place=place, - eps=0.05) + gradient_checker.double_grad_check( + [input_NHWC], y, x_init=x_arr, place=place, eps=0.05 + ) - gradient_checker.double_grad_check_for_dygraph(self.pool2d_wrapper, - [input_NHWC], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.pool2d_wrapper, [input_NHWC], y, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -555,35 +508,32 @@ def test_grad(self): class TestAvgPool2DDoubleGradCheckCase3(unittest.TestCase): - def pool2d_wrapper(self, x): - return paddle.nn.functional.avg_pool2d(x[0], - kernel_size=2, - padding=[1, 1]) + return paddle.nn.functional.avg_pool2d( + x[0], kernel_size=2, padding=[1, 1] + ) @prog_scope() def func(self, place): - input_NCHW = fluid.layers.data(name="input_NCHW", - shape=[2, 3, 5, 5], - append_batch_size=False, - dtype="float32") + input_NCHW = fluid.layers.data( + name="input_NCHW", + shape=[2, 3, 5, 5], + append_batch_size=False, + dtype="float32", + ) input_NCHW.persistable = True - y = paddle.nn.functional.avg_pool2d(input_NCHW, - kernel_size=2, - padding=[1, 1]) + y = paddle.nn.functional.avg_pool2d( + input_NCHW, kernel_size=2, padding=[1, 1] + ) x_arr = np.random.uniform(-1, 1, [2, 3, 5, 5]).astype(np.float32) - gradient_checker.double_grad_check([input_NCHW], - y, - x_init=x_arr, - place=place, - eps=0.05) - gradient_checker.double_grad_check_for_dygraph(self.pool2d_wrapper, - [input_NCHW], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [input_NCHW], y, x_init=x_arr, place=place, eps=0.05 + ) + gradient_checker.double_grad_check_for_dygraph( + self.pool2d_wrapper, [input_NCHW], y, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -594,32 +544,29 @@ def test_grad(self): class TestAvgPool2DDoubleGradCheckCase4(unittest.TestCase): - def pool2d_wrapper(self, x): return paddle.nn.functional.avg_pool2d(x[0], kernel_size=[4, 4]) @prog_scope() def func(self, place): - input_NCHW = fluid.layers.data(name="input_NCHW", - shape=[2, 3, 5, 5], - append_batch_size=False, - dtype="float32") + input_NCHW = fluid.layers.data( + name="input_NCHW", + shape=[2, 3, 5, 5], + append_batch_size=False, + dtype="float32", + ) input_NCHW.persistable = True y = layers.pool2d(input_NCHW, pool_size=[4, 4], pool_type="avg") y = paddle.nn.functional.avg_pool2d(input_NCHW, kernel_size=[4, 4]) x_arr = np.random.uniform(-1, 1, [2, 3, 5, 5]).astype(np.float32) - gradient_checker.double_grad_check([input_NCHW], - y, - x_init=x_arr, - place=place, - eps=0.05) - gradient_checker.double_grad_check_for_dygraph(self.pool2d_wrapper, - [input_NCHW], - y, - x_init=x_arr, - place=place) + gradient_checker.double_grad_check( + [input_NCHW], y, x_init=x_arr, place=place, eps=0.05 + ) + gradient_checker.double_grad_check_for_dygraph( + self.pool2d_wrapper, [input_NCHW], y, x_init=x_arr, place=place + ) def test_grad(self): places = [fluid.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_nn_margin_rank_loss.py b/python/paddle/fluid/tests/unittests/test_nn_margin_rank_loss.py index 2f407edc722bc..2ddfd653a40f3 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_margin_rank_loss.py +++ b/python/paddle/fluid/tests/unittests/test_nn_margin_rank_loss.py @@ -32,14 +32,13 @@ def calc_margin_rank_loss(x, y, label, margin=0.0, reduction='none'): def create_test_case(margin, reduction): - class MarginRankingLossCls(unittest.TestCase): - def setUp(self): self.x_data = np.random.rand(10, 10).astype("float64") self.y_data = np.random.rand(10, 10).astype("float64") - self.label_data = np.random.choice([-1, 1], - size=[10, 10]).astype("float64") + self.label_data = np.random.choice([-1, 1], size=[10, 10]).astype( + "float64" + ) self.places = [] self.places.append(fluid.CPUPlace()) if core.is_compiled_with_cuda(): @@ -47,59 +46,69 @@ def setUp(self): def run_static_functional_api(self, place): paddle.enable_static() - expected = calc_margin_rank_loss(self.x_data, - self.y_data, - self.label_data, - margin=margin, - reduction=reduction) + expected = calc_margin_rank_loss( + self.x_data, + self.y_data, + self.label_data, + margin=margin, + reduction=reduction, + ) with program_guard(Program(), Program()): - x = paddle.static.data(name="x", - shape=[10, 10], - dtype="float64") - y = paddle.static.data(name="y", - shape=[10, 10], - dtype="float64") - label = paddle.static.data(name="label", - shape=[10, 10], - dtype="float64") + x = paddle.static.data( + name="x", shape=[10, 10], dtype="float64" + ) + y = paddle.static.data( + name="y", shape=[10, 10], dtype="float64" + ) + label = paddle.static.data( + name="label", shape=[10, 10], dtype="float64" + ) result = paddle.nn.functional.margin_ranking_loss( - x, y, label, margin, reduction) + x, y, label, margin, reduction + ) exe = paddle.static.Executor(place) - result_numpy, = exe.run(feed={ - "x": self.x_data, - "y": self.y_data, - "label": self.label_data - }, - fetch_list=[result]) + (result_numpy,) = exe.run( + feed={ + "x": self.x_data, + "y": self.y_data, + "label": self.label_data, + }, + fetch_list=[result], + ) np.testing.assert_allclose(result_numpy, expected, rtol=1e-05) def run_static_api(self, place): paddle.enable_static() - expected = calc_margin_rank_loss(self.x_data, - self.y_data, - self.label_data, - margin=margin, - reduction=reduction) + expected = calc_margin_rank_loss( + self.x_data, + self.y_data, + self.label_data, + margin=margin, + reduction=reduction, + ) with program_guard(Program(), Program()): - x = paddle.static.data(name="x", - shape=[10, 10], - dtype="float64") - y = paddle.static.data(name="y", - shape=[10, 10], - dtype="float64") - label = paddle.static.data(name="label", - shape=[10, 10], - dtype="float64") + x = paddle.static.data( + name="x", shape=[10, 10], dtype="float64" + ) + y = paddle.static.data( + name="y", shape=[10, 10], dtype="float64" + ) + label = paddle.static.data( + name="label", shape=[10, 10], dtype="float64" + ) margin_rank_loss = paddle.nn.loss.MarginRankingLoss( - margin=margin, reduction=reduction) + margin=margin, reduction=reduction + ) result = margin_rank_loss(x, y, label) exe = paddle.static.Executor(place) - result_numpy, = exe.run(feed={ - "x": self.x_data, - "y": self.y_data, - "label": self.label_data - }, - fetch_list=[result]) + (result_numpy,) = exe.run( + feed={ + "x": self.x_data, + "y": self.y_data, + "label": self.label_data, + }, + fetch_list=[result], + ) np.testing.assert_allclose(result_numpy, expected, rtol=1e-05) self.assertTrue('loss' in result.name) @@ -110,12 +119,15 @@ def run_dynamic_functional_api(self, place): label = paddle.to_tensor(self.label_data) result = paddle.nn.functional.margin_ranking_loss( - x, y, label, margin, reduction) - expected = calc_margin_rank_loss(self.x_data, - self.y_data, - self.label_data, - margin=margin, - reduction=reduction) + x, y, label, margin, reduction + ) + expected = calc_margin_rank_loss( + self.x_data, + self.y_data, + self.label_data, + margin=margin, + reduction=reduction, + ) np.testing.assert_allclose(result.numpy(), expected, rtol=1e-05) def run_dynamic_api(self, place): @@ -124,13 +136,16 @@ def run_dynamic_api(self, place): y = paddle.to_tensor(self.y_data) label = paddle.to_tensor(self.label_data) margin_rank_loss = paddle.nn.loss.MarginRankingLoss( - margin=margin, reduction=reduction) + margin=margin, reduction=reduction + ) result = margin_rank_loss(x, y, label) - expected = calc_margin_rank_loss(self.x_data, - self.y_data, - self.label_data, - margin=margin, - reduction=reduction) + expected = calc_margin_rank_loss( + self.x_data, + self.y_data, + self.label_data, + margin=margin, + reduction=reduction, + ) np.testing.assert_allclose(result.numpy(), expected, rtol=1e-05) def run_dynamic_broadcast_api(self, place): @@ -140,13 +155,16 @@ def run_dynamic_broadcast_api(self, place): y = paddle.to_tensor(self.y_data) label = paddle.to_tensor(label_data) margin_rank_loss = paddle.nn.loss.MarginRankingLoss( - margin=margin, reduction=reduction) + margin=margin, reduction=reduction + ) result = margin_rank_loss(x, y, label) - expected = calc_margin_rank_loss(self.x_data, - self.y_data, - label_data, - margin=margin, - reduction=reduction) + expected = calc_margin_rank_loss( + self.x_data, + self.y_data, + label_data, + margin=margin, + reduction=reduction, + ) np.testing.assert_allclose(result.numpy(), expected, rtol=1e-05) def test_case(self): @@ -172,21 +190,22 @@ class MarginRakingLossError(unittest.TestCase): paddle.enable_static() def test_errors(self): - def test_margin_value_error(): margin_rank_loss = paddle.nn.loss.MarginRankingLoss( - margin=0.1, reduction="reduce_mean") + margin=0.1, reduction="reduce_mean" + ) self.assertRaises(ValueError, test_margin_value_error) def test_functional_margin_value_error(): x = paddle.static.data(name="x", shape=[10, 10], dtype="float64") y = paddle.static.data(name="y", shape=[10, 10], dtype="float64") - label = paddle.static.data(name="label", - shape=[10, 10], - dtype="float64") + label = paddle.static.data( + name="label", shape=[10, 10], dtype="float64" + ) result = paddle.nn.functional.margin_ranking_loss( - x, y, label, margin=0.1, reduction="reduction_mean") + x, y, label, margin=0.1, reduction="reduction_mean" + ) self.assertRaises(ValueError, test_functional_margin_value_error) diff --git a/python/paddle/fluid/tests/unittests/test_nn_matmul_v2_grad.py b/python/paddle/fluid/tests/unittests/test_nn_matmul_v2_grad.py index 567ac1404c669..0c3bca6ba0a88 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_matmul_v2_grad.py +++ b/python/paddle/fluid/tests/unittests/test_nn_matmul_v2_grad.py @@ -25,7 +25,6 @@ class TestMatmulDoubleGradCheck(unittest.TestCase): - def setUp(self): self.init_test() @@ -40,25 +39,21 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -69,7 +64,6 @@ def test_grad(self): class TestMatmulDoubleGradCheckCase1(TestMatmulDoubleGradCheck): - def init_test(self): self.x_shape = [2, 3] self.y_shape = [3, 2] @@ -85,7 +79,6 @@ def test_grad(self): class TestMatmulDoubleGradCheck2(unittest.TestCase): - def setUp(self): self.init_test() @@ -100,25 +93,21 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -129,7 +118,6 @@ def test_grad(self): class TestMatmulDoubleGradCheckCase3(unittest.TestCase): - def setUp(self): self.init_test() @@ -144,25 +132,21 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.double_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -173,7 +157,6 @@ def test_grad(self): class TestMatmulTripleGradCheckDotCase(unittest.TestCase): - def setUp(self): self.init_test() @@ -190,21 +173,19 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) out = paddle.matmul(x, y, self.transpose_x, self.transpose_y, name='out') np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): @@ -216,7 +197,6 @@ def test_grad(self): class TestMatmulTripleGradCheckNormalCase1(unittest.TestCase): - def setUp(self): self.init_test() @@ -231,25 +211,21 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -260,7 +236,6 @@ def test_grad(self): class TestMatmulTripleGradCheckNormalCase2(unittest.TestCase): - def setUp(self): self.init_test() @@ -275,25 +250,21 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -304,7 +275,6 @@ def test_grad(self): class TestMatmulTripleGradCheckNormalCase3(unittest.TestCase): - def setUp(self): self.init_test() @@ -319,25 +289,21 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -348,7 +314,6 @@ def test_grad(self): class TestMatmulTripleGradCheckNormalCase4(unittest.TestCase): - def setUp(self): self.init_test() @@ -363,25 +328,21 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -392,7 +353,6 @@ def test_grad(self): class TestMatmulTripleGradCheckBroadcastCase1(unittest.TestCase): - def setUp(self): self.init_test() @@ -407,25 +367,21 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -436,7 +392,6 @@ def test_grad(self): class TestMatmulTripleGradCheckBroadcastCase2(unittest.TestCase): - def setUp(self): self.init_test() @@ -451,25 +406,21 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -480,7 +431,6 @@ def test_grad(self): class TestMatmulTripleGradCheckBroadcastCase3(unittest.TestCase): - def setUp(self): self.init_test() @@ -495,25 +445,21 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -524,7 +470,6 @@ def test_grad(self): class TestMatmulTripleGradCheckBroadcastCase4(unittest.TestCase): - def setUp(self): self.init_test() @@ -539,25 +484,21 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -568,7 +509,6 @@ def test_grad(self): class TestMatmulTripleGradCheckBroadcastCase5(unittest.TestCase): - def setUp(self): self.init_test() @@ -583,25 +523,21 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -612,7 +548,6 @@ def test_grad(self): class TestMatmulTripleGradCheckSpecialCase1(unittest.TestCase): - def setUp(self): self.init_test() @@ -627,25 +562,21 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] @@ -656,7 +587,6 @@ def test_grad(self): class TestMatmulTripleGradCheckSpecialCase2(unittest.TestCase): - def setUp(self): self.init_test() @@ -671,25 +601,21 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter(dtype=typename, - shape=self.x_shape, - name='x') - y = paddle.static.create_parameter(dtype=typename, - shape=self.y_shape, - name='y') - out = paddle.matmul(x, - y, - self.transpose_x, - self.transpose_y, - name='out') + x = paddle.static.create_parameter( + dtype=typename, shape=self.x_shape, name='x' + ) + y = paddle.static.create_parameter( + dtype=typename, shape=self.y_shape, name='y' + ) + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) np.random.seed(2021) x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check([x, y], - out, - x_init=[x_arr, y_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) def test_grad(self): places = [fluid.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_nn_quant_functional_layers.py b/python/paddle/fluid/tests/unittests/test_nn_quant_functional_layers.py index 9ef7e19fbe098..6edc691f23a08 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_quant_functional_layers.py +++ b/python/paddle/fluid/tests/unittests/test_nn_quant_functional_layers.py @@ -18,8 +18,7 @@ class TestFunctionalLayers(unittest.TestCase): - """ - """ + """ """ def setUp(self): paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_nn_sigmoid_op.py b/python/paddle/fluid/tests/unittests/test_nn_sigmoid_op.py index 7af7b96c999b6..45840d2031915 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_sigmoid_op.py +++ b/python/paddle/fluid/tests/unittests/test_nn_sigmoid_op.py @@ -23,7 +23,6 @@ class TestNNSigmoidAPI(unittest.TestCase): - def setUp(self): self.init_data() @@ -69,7 +68,6 @@ def test_check_api(self): class TestNNFunctionalSigmoidAPI(unittest.TestCase): - def setUp(self): self.init_data() diff --git a/python/paddle/fluid/tests/unittests/test_nonzero_api.py b/python/paddle/fluid/tests/unittests/test_nonzero_api.py index cd4d227e823b9..77363d1d3e73b 100644 --- a/python/paddle/fluid/tests/unittests/test_nonzero_api.py +++ b/python/paddle/fluid/tests/unittests/test_nonzero_api.py @@ -20,7 +20,6 @@ class TestNonZeroAPI(unittest.TestCase): - def test_nonzero_api_as_tuple(self): data = np.array([[True, False], [False, True]]) with program_guard(Program(), Program()): @@ -31,9 +30,9 @@ def test_nonzero_api_as_tuple(self): z = fluid.layers.concat(list(y), axis=1) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': data}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[z.name], return_numpy=False + ) expect_out = np.array([[0, 0], [1, 1]]) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) @@ -45,9 +44,9 @@ def test_nonzero_api_as_tuple(self): self.assertEqual(len(y), 1) z = fluid.layers.concat(list(y), axis=1) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': data}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[z.name], return_numpy=False + ) expect_out = np.array([[0], [1]]) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) @@ -57,9 +56,9 @@ def test_nonzero_api(self): x = fluid.layers.data(name='x', shape=[-1, 2]) y = paddle.nonzero(x) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': data}, - fetch_list=[y.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[y.name], return_numpy=False + ) expect_out = np.array([[0, 0], [1, 1]]) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) @@ -68,9 +67,9 @@ def test_nonzero_api(self): x = fluid.layers.data(name='x', shape=[-1]) y = paddle.nonzero(x) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': data}, - fetch_list=[y.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[y.name], return_numpy=False + ) expect_out = np.array([[0], [1]]) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_norm_all.py b/python/paddle/fluid/tests/unittests/test_norm_all.py index f9a62d576c844..17047d349cd16 100644 --- a/python/paddle/fluid/tests/unittests/test_norm_all.py +++ b/python/paddle/fluid/tests/unittests/test_norm_all.py @@ -23,18 +23,25 @@ # hack method for test p_norm final state -def p_norm_python_api(x, - p=2.0, - axis=-1, - epsilon=1e-12, - keepdim=False, - as_vector=False): +def p_norm_python_api( + x, p=2.0, axis=-1, epsilon=1e-12, keepdim=False, as_vector=False +): if in_dygraph_mode(): return _C_ops.p_norm(x, p, axis, epsilon, keepdim, as_vector) if _in_legacy_dygraph(): - return _legacy_C_ops.p_norm(x, 'axis', axis, 'porder', float(p), - 'keepdim', keepdim, 'epsilon', epsilon, - 'as_vector', as_vector) + return _legacy_C_ops.p_norm( + x, + 'axis', + axis, + 'porder', + float(p), + 'keepdim', + keepdim, + 'epsilon', + epsilon, + 'as_vector', + as_vector, + ) def p_norm(x, axis, porder, keepdims=False, reduce_all=False): @@ -76,10 +83,13 @@ def p_norm(x, axis, porder, keepdims=False, reduce_all=False): def numpy_frobenius_norm(x, axis=None, keepdims=False): - if isinstance(axis, list): axis = tuple(axis) - if axis is None: x = x.reshape(1, x.size) - r = np.linalg.norm(x, ord='fro', axis=axis, - keepdims=keepdims).astype(x.dtype) + if isinstance(axis, list): + axis = tuple(axis) + if axis is None: + x = x.reshape(1, x.size) + r = np.linalg.norm(x, ord='fro', axis=axis, keepdims=keepdims).astype( + x.dtype + ) return r @@ -88,19 +98,18 @@ def frobenius_norm(x, dim, keep_dim, reduce_all): class TestFrobeniusNormOp(OpTest): - def setUp(self): self.python_api = frobenius_norm self.op_type = "frobenius_norm" self.init_test_case() x = (np.random.random(self.shape) + 1.0).astype(self.dtype) norm = numpy_frobenius_norm(x, self.axis, self.keepdim) - self.reduce_all = (len(self.axis) == len(self.shape)) + self.reduce_all = len(self.axis) == len(self.shape) self.inputs = {'X': x} self.attrs = { 'dim': list(self.axis), 'keep_dim': self.keepdim, - 'reduce_all': self.reduce_all + 'reduce_all': self.reduce_all, } self.outputs = {'Out': norm} @@ -118,7 +127,6 @@ def init_test_case(self): class TestFrobeniusNormOp2(TestFrobeniusNormOp): - def init_test_case(self): self.shape = [5, 5, 5] self.axis = (0, 1) @@ -130,7 +138,6 @@ def test_check_grad(self): class TestPnormOp(OpTest): - def setUp(self): self.op_type = "p_norm" self.python_api = p_norm_python_api @@ -143,7 +150,7 @@ def setUp(self): 'axis': self.axis, 'keepdim': self.keepdim, 'porder': float(self.porder), - 'asvector': self.asvector + 'asvector': self.asvector, } self.outputs = {'Out': norm} self.gradient = self.calc_gradient() @@ -169,7 +176,7 @@ def calc_gradient(self): 'axis': self.axis, 'keepdim': self.keepdim, 'porder': float(self.porder), - 'asvector': self.asvector + 'asvector': self.asvector, } x = self.inputs["X"] porder = self.attrs["porder"] @@ -180,22 +187,21 @@ def calc_gradient(self): if porder == 0: grad = np.zeros(x.shape).astype(x.dtype) elif porder in [float("inf"), float("-inf")]: - norm = p_norm(x, - axis=axis, - porder=porder, - keepdims=True, - reduce_all=asvector) + norm = p_norm( + x, axis=axis, porder=porder, keepdims=True, reduce_all=asvector + ) x_abs = np.abs(x) grad = np.sign(x) grad[x_abs != norm] = 0.0 else: - norm = p_norm(x, - axis=axis, - porder=porder, - keepdims=True, - reduce_all=asvector) - grad = np.power(norm, 1 - porder) * np.power( - np.abs(x), porder - 1) * np.sign(x) + norm = p_norm( + x, axis=axis, porder=porder, keepdims=True, reduce_all=asvector + ) + grad = ( + np.power(norm, 1 - porder) + * np.power(np.abs(x), porder - 1) + * np.sign(x) + ) numel = 1 for s in x.shape: @@ -206,7 +212,6 @@ def calc_gradient(self): class TestPnormOp2(TestPnormOp): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -221,7 +226,6 @@ def test_check_grad(self): class TestPnormOp3(TestPnormOp): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -236,7 +240,6 @@ def test_check_grad(self): class TestPnormOp4(TestPnormOp): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -251,7 +254,6 @@ def test_check_grad(self): class TestPnormOp5(TestPnormOp): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = 2 @@ -266,7 +268,6 @@ def test_check_grad(self): class TestPnormOp6(TestPnormOp): - def init_test_case(self): self.shape = [3, 20, 3] self.axis = -1 @@ -280,10 +281,10 @@ def test_check_grad(self): self.check_grad(['X'], 'Out', user_defined_grads=self.gradient) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestPnormOpFP16(TestPnormOp): - def init_test_case(self): self.shape = [2, 3, 4, 5] self.axis = 1 @@ -301,15 +302,15 @@ def test_check_output(self): def test_check_grad(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['X'], - 'Out', - user_defined_grads=self.gradient) + self.check_grad_with_place( + place, ['X'], 'Out', user_defined_grads=self.gradient + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestPnormOpFP161(TestPnormOpFP16): - def init_test_case(self): self.shape = [2, 3, 4, 5] self.axis = -1 @@ -320,17 +321,18 @@ def init_test_case(self): self.asvector = True -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestPnormBF16Op(OpTest): - def setUp(self): self.op_type = "p_norm" self.python_api = p_norm_python_api self.init_test_case() self.x = (np.random.random(self.shape) + 0.5).astype(np.float32) - self.norm = p_norm(self.x, self.axis, self.porder, self.keepdim, - self.asvector) + self.norm = p_norm( + self.x, self.axis, self.porder, self.keepdim, self.asvector + ) self.gradient = self.calc_gradient() self.inputs = {'X': convert_float_to_uint16(self.x)} self.attrs = { @@ -338,7 +340,7 @@ def setUp(self): 'axis': self.axis, 'keepdim': self.keepdim, 'porder': float(self.porder), - 'asvector': self.asvector + 'asvector': self.asvector, } self.outputs = {'Out': convert_float_to_uint16(self.norm)} @@ -348,10 +350,13 @@ def test_check_output(self): def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - user_defined_grads=self.gradient, - check_eager=True) + self.check_grad_with_place( + place, + ['X'], + 'Out', + user_defined_grads=self.gradient, + check_eager=True, + ) def init_test_case(self): self.shape = [2, 3, 4, 5] @@ -368,7 +373,7 @@ def calc_gradient(self): 'axis': self.axis, 'keepdim': self.keepdim, 'porder': float(self.porder), - 'asvector': self.asvector + 'asvector': self.asvector, } x = self.x porder = self.attrs["porder"] @@ -379,22 +384,21 @@ def calc_gradient(self): if porder == 0: grad = np.zeros(x.shape).astype(x.dtype) elif porder in [float("inf"), float("-inf")]: - norm = p_norm(x, - axis=axis, - porder=porder, - keepdims=True, - reduce_all=asvector) + norm = p_norm( + x, axis=axis, porder=porder, keepdims=True, reduce_all=asvector + ) x_abs = np.abs(x) grad = np.sign(x) grad[x_abs != norm] = 0.0 else: - norm = p_norm(x, - axis=axis, - porder=porder, - keepdims=True, - reduce_all=asvector) - grad = np.power(norm, 1 - porder) * np.power( - np.abs(x), porder - 1) * np.sign(x) + norm = p_norm( + x, axis=axis, porder=porder, keepdims=True, reduce_all=asvector + ) + grad = ( + np.power(norm, 1 - porder) + * np.power(np.abs(x), porder - 1) + * np.sign(x) + ) numel = 1 for s in x.shape: @@ -411,15 +415,19 @@ def run_fro(self, p, axis, shape_x, dtype, keep_dim, check_dim=False): place = fluid.CPUPlace() exe = fluid.Executor(place) np_input = (np.random.rand(*shape_x) + 1.0).astype(dtype) - expected_result = numpy_frobenius_norm(np_input, - axis=axis, - keepdims=keep_dim) - result, = exe.run(feed={"X": np_input}, fetch_list=[out]) + expected_result = numpy_frobenius_norm( + np_input, axis=axis, keepdims=keep_dim + ) + (result,) = exe.run(feed={"X": np_input}, fetch_list=[out]) self.assertEqual((np.abs(result - expected_result) < 1e-6).all(), True) if keep_dim and check_dim: self.assertEqual( - (np.abs(np.array(result.shape) - np.array(expected_result.shape)) < - 1e-6).all(), True) + ( + np.abs(np.array(result.shape) - np.array(expected_result.shape)) + < 1e-6 + ).all(), + True, + ) def run_pnorm(self, p, axis, shape_x, dtype, keep_dim, check_dim=False): @@ -429,16 +437,19 @@ def run_pnorm(self, p, axis, shape_x, dtype, keep_dim, check_dim=False): place = fluid.CPUPlace() exe = fluid.Executor(place) np_input = (np.random.rand(*shape_x) + 1.0).astype(dtype) - expected_result = p_norm(np_input, - porder=p, - axis=axis, - keepdims=keep_dim).astype(dtype) - result, = exe.run(feed={"X": np_input}, fetch_list=[out]) + expected_result = p_norm( + np_input, porder=p, axis=axis, keepdims=keep_dim + ).astype(dtype) + (result,) = exe.run(feed={"X": np_input}, fetch_list=[out]) self.assertEqual((np.abs(result - expected_result) < 1e-6).all(), True) if keep_dim and check_dim: self.assertEqual( - (np.abs(np.array(result.shape) - np.array(expected_result.shape)) < - 1e-6).all(), True) + ( + np.abs(np.array(result.shape) - np.array(expected_result.shape)) + < 1e-6 + ).all(), + True, + ) def run_graph(self, p, axis, shape_x, dtype): @@ -447,7 +458,7 @@ def run_graph(self, p, axis, shape_x, dtype): np_input = np.arange(24).astype('float32') - 12 np_input = np_input.reshape(shape) x = paddle.to_tensor(np_input) - #[[[-12. -11. -10. -9.] [ -8. -7. -6. -5.] [ -4. -3. -2. -1.]] + # [[[-12. -11. -10. -9.] [ -8. -7. -6. -5.] [ -4. -3. -2. -1.]] # [[ 0. 1. 2. 3.] [ 4. 5. 6. 7.] [ 8. 9. 10. 11.]]] out_pnorm = paddle.norm(x, p=2, axis=-1) @@ -458,135 +469,166 @@ def run_graph(self, p, axis, shape_x, dtype): # compute 2-order norm along [0,1] dimension. out_pnorm = paddle.norm(x, p=2, axis=[0, 1]) out_pnorm = paddle.norm(x, p=2) - #out_pnorm = [17.43559577 16.91153453 16.73320053 16.91153453] + # out_pnorm = [17.43559577 16.91153453 16.73320053 16.91153453] # compute inf-order norm out_pnorm = paddle.norm(x, p=np.inf) - #out_pnorm = [12.] + # out_pnorm = [12.] out_pnorm = paddle.norm(x, p=np.inf, axis=0) - #out_pnorm = [[0. 1. 2. 3.] [4. 5. 6. 5.] [4. 3. 2. 1.]] + # out_pnorm = [[0. 1. 2. 3.] [4. 5. 6. 5.] [4. 3. 2. 1.]] # compute -inf-order norm out_pnorm = paddle.norm(x, p=-np.inf) - #out_pnorm = [0.] + # out_pnorm = [0.] out_pnorm = paddle.norm(x, p=-np.inf, axis=0) # out_fro = [17.43559577 16.91153453 16.73320053 16.91153453] paddle.enable_static() class API_NormTest(unittest.TestCase): - def test_basic(self): keep_dims = {False, True} for keep in keep_dims: - run_fro(self, - p='fro', - axis=None, - shape_x=[2, 3, 4], - dtype="float32", - keep_dim=keep) - run_fro(self, - p='fro', - axis=[0, 1], - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=2, - axis=None, - shape_x=[3, 4], - dtype="float32", - keep_dim=keep) - run_pnorm(self, - p=2, - axis=1, - shape_x=[3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=np.inf, - axis=0, - shape_x=[2, 3, 4], - dtype="float32", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=np.inf, - axis=None, - shape_x=[2, 3, 4], - dtype="float32", - keep_dim=keep) - run_pnorm(self, - p=-np.inf, - axis=0, - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=-np.inf, - axis=None, - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep) - run_pnorm(self, - p=0, - axis=1, - shape_x=[3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - - run_pnorm(self, - p=1, - axis=1, - shape_x=[3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=0, - axis=None, - shape_x=[3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=2, - axis=[0, 1], - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=2, - axis=-1, - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=1, - axis=[0, 1], - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=np.inf, - axis=[0, 1], - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) - run_pnorm(self, - p=-np.inf, - axis=[0, 1], - shape_x=[2, 3, 4], - dtype="float64", - keep_dim=keep, - check_dim=True) + run_fro( + self, + p='fro', + axis=None, + shape_x=[2, 3, 4], + dtype="float32", + keep_dim=keep, + ) + run_fro( + self, + p='fro', + axis=[0, 1], + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=2, + axis=None, + shape_x=[3, 4], + dtype="float32", + keep_dim=keep, + ) + run_pnorm( + self, + p=2, + axis=1, + shape_x=[3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=np.inf, + axis=0, + shape_x=[2, 3, 4], + dtype="float32", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=np.inf, + axis=None, + shape_x=[2, 3, 4], + dtype="float32", + keep_dim=keep, + ) + run_pnorm( + self, + p=-np.inf, + axis=0, + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=-np.inf, + axis=None, + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + ) + run_pnorm( + self, + p=0, + axis=1, + shape_x=[3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + + run_pnorm( + self, + p=1, + axis=1, + shape_x=[3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=0, + axis=None, + shape_x=[3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=2, + axis=[0, 1], + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=2, + axis=-1, + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=1, + axis=[0, 1], + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=np.inf, + axis=[0, 1], + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) + run_pnorm( + self, + p=-np.inf, + axis=[0, 1], + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True, + ) def test_dygraph(self): run_graph(self, p='fro', axis=None, shape_x=[2, 3, 4], dtype="float32") @@ -609,8 +651,9 @@ def err_dtype(p, shape_x, xdtype, out=None): self.assertRaises(TypeError, err_dtype, "fro", [2, 2], "int64") self.assertRaises(ValueError, paddle.norm, "inf", [2], "int64") out = fluid.data(name="out", shape=[1], dtype="int64") - self.assertRaises(TypeError, err_dtype, "fro", [2, 2], "float64", - out) + self.assertRaises( + TypeError, err_dtype, "fro", [2, 2], "float64", out + ) self.assertRaises(TypeError, err_dtype, 2, [10], "int64") self.assertRaises(TypeError, err_dtype, 2, [10], "float64", out) @@ -620,11 +663,9 @@ def err_dtype(p, shape_x, xdtype, out=None): self.assertRaises(ValueError, paddle.norm, data, p=[1], axis=-1) self.assertRaises(ValueError, paddle.norm, 0, [1, 0], "float64") data = fluid.data(name="data_3d", shape=[2, 2, 2], dtype="float64") - self.assertRaises(ValueError, - paddle.norm, - data, - p='unspport', - axis=[-3, -2, -1]) + self.assertRaises( + ValueError, paddle.norm, data, p='unspport', axis=[-3, -2, -1] + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_norm_nn_grad.py b/python/paddle/fluid/tests/unittests/test_norm_nn_grad.py index 28622d44d18bb..26dea91aecce4 100644 --- a/python/paddle/fluid/tests/unittests/test_norm_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_norm_nn_grad.py @@ -25,7 +25,6 @@ class TestInstanceNormDoubleGradCheck(unittest.TestCase): - @prog_scope() def func(self, place): prog = fluid.Program() @@ -38,12 +37,9 @@ def func(self, place): x = layers.create_parameter(dtype=dtype, shape=shape, name='x') z = fluid.layers.instance_norm(input=x) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - z, - x_init=x_arr, - atol=atol, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], z, x_init=x_arr, atol=atol, place=place, eps=eps + ) def test_grad(self): paddle.enable_static() @@ -55,8 +51,8 @@ def test_grad(self): class TestInstanceNormDoubleGradCheckWithoutParamBias( - TestInstanceNormDoubleGradCheck): - + TestInstanceNormDoubleGradCheck +): @prog_scope() def func(self, place): prog = fluid.Program() @@ -67,20 +63,16 @@ def func(self, place): eps = 0.005 atol = 1e-4 x = layers.create_parameter(dtype=dtype, shape=shape, name='x') - z = fluid.layers.instance_norm(input=x, - param_attr=False, - bias_attr=False) + z = fluid.layers.instance_norm( + input=x, param_attr=False, bias_attr=False + ) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) - gradient_checker.double_grad_check([x], - z, - x_init=x_arr, - atol=atol, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], z, x_init=x_arr, atol=atol, place=place, eps=eps + ) class TestInstanceNormDoubleGradEagerCheck(unittest.TestCase): - def instance_norm_wrapper(self, x): return paddle.nn.functional.instance_norm(x[0]) @@ -97,19 +89,18 @@ def func(self, place): z = paddle.nn.functional.instance_norm(x) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) # check for static mode - gradient_checker.double_grad_check([x], - z, - x_init=x_arr, - atol=atol, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], z, x_init=x_arr, atol=atol, place=place, eps=eps + ) # check for eager mode gradient_checker.double_grad_check_for_dygraph( - self.instance_norm_wrapper, [x], + self.instance_norm_wrapper, + [x], z, x_init=x_arr, atol=atol, - place=place) + place=place, + ) def test_grad(self): paddle.enable_static() @@ -121,8 +112,8 @@ def test_grad(self): class TestInstanceNormDoubleGradEagerCheckWithParams( - TestInstanceNormDoubleGradEagerCheck): - + TestInstanceNormDoubleGradEagerCheck +): def instance_norm_wrapper(self, x): instance_norm = paddle.nn.InstanceNorm2D(3) return instance_norm(x[0]) @@ -140,23 +131,21 @@ def func(self, place): z = paddle.nn.InstanceNorm2D(3)(x) x_arr = np.random.uniform(-1, 1, shape).astype(dtype) # check for static mode - gradient_checker.double_grad_check([x], - z, - x_init=x_arr, - atol=atol, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], z, x_init=x_arr, atol=atol, place=place, eps=eps + ) # check for eager mode gradient_checker.double_grad_check_for_dygraph( - self.instance_norm_wrapper, [x], + self.instance_norm_wrapper, + [x], z, x_init=x_arr, atol=atol, - place=place) + place=place, + ) class TestBatchNormDoubleGradCheck(unittest.TestCase): - def setUp(self): self.init_test() @@ -170,7 +159,8 @@ def batch_norm_wrapper(self, x): batch_norm = paddle.nn.BatchNorm2D( self.shape[self.channel_index], data_format=self.data_layout, - use_global_stats=self.use_global_stats) + use_global_stats=self.use_global_stats, + ) return batch_norm(x[0]) @prog_scope() @@ -182,22 +172,23 @@ def func(self, place): eps = 0.005 atol = 1e-4 x = layers.create_parameter(dtype=dtype, shape=self.shape, name='x') - z = fluid.layers.batch_norm(input=x, - data_layout=self.data_layout, - use_global_stats=self.use_global_stats) + z = fluid.layers.batch_norm( + input=x, + data_layout=self.data_layout, + use_global_stats=self.use_global_stats, + ) x_arr = np.random.uniform(-1, 1, self.shape).astype(dtype) - gradient_checker.double_grad_check([x], - z, - x_init=x_arr, - atol=atol, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x], z, x_init=x_arr, atol=atol, place=place, eps=eps + ) gradient_checker.double_grad_check_for_dygraph( - self.batch_norm_wrapper, [x], + self.batch_norm_wrapper, + [x], z, x_init=x_arr, atol=atol, - place=place) + place=place, + ) def test_grad(self): paddle.enable_static() @@ -209,7 +200,6 @@ def test_grad(self): class TestBatchNormDoubleGradCheckCase1(TestBatchNormDoubleGradCheck): - def init_test(self): self.data_layout = 'NHWC' self.use_global_stats = False @@ -218,7 +208,6 @@ def init_test(self): class TestBatchNormDoubleGradCheckCase2(TestBatchNormDoubleGradCheck): - def init_test(self): self.data_layout = 'NCHW' self.use_global_stats = True @@ -227,7 +216,6 @@ def init_test(self): class TestBatchNormDoubleGradCheckCase3(TestBatchNormDoubleGradCheck): - def init_test(self): self.data_layout = 'NHWC' self.use_global_stats = True @@ -236,7 +224,6 @@ def init_test(self): class TestBatchNormDoubleGradCheckCase4(TestBatchNormDoubleGradCheck): - def init_test(self): self.data_layout = 'NCHW' self.use_global_stats = False @@ -247,12 +234,12 @@ def batch_norm_wrapper(self, x): batch_norm = paddle.nn.BatchNorm3D( self.shape[self.channel_index], data_format=self.data_layout, - use_global_stats=self.use_global_stats) + use_global_stats=self.use_global_stats, + ) return batch_norm(x[0]) class TestBatchNormDoubleGradCheckCase5(TestBatchNormDoubleGradCheck): - @prog_scope() def func(self, place): prog = fluid.Program() @@ -261,25 +248,30 @@ def func(self, place): dtype = "float32" eps = 0.005 atol = 2e-4 - chn = self.shape[1] if self.data_layout == 'NCHW' else self.shape[-1] + chn = ( + self.shape[1] if self.data_layout == 'NCHW' else self.shape[-1] + ) x = layers.create_parameter(dtype=dtype, shape=self.shape, name='x') - z = fluid.layers.batch_norm(input=x, - data_layout=self.data_layout, - use_global_stats=self.use_global_stats) + z = fluid.layers.batch_norm( + input=x, + data_layout=self.data_layout, + use_global_stats=self.use_global_stats, + ) x_arr = np.random.uniform(-1, 1, self.shape).astype(dtype) w, b = prog.global_block().all_parameters()[1:3] w_arr = np.ones(chn).astype(dtype) b_arr = np.zeros(chn).astype(dtype) - gradient_checker.double_grad_check([x, w, b], - z, - x_init=[x_arr, w_arr, b_arr], - atol=atol, - place=place, - eps=eps) + gradient_checker.double_grad_check( + [x, w, b], + z, + x_init=[x_arr, w_arr, b_arr], + atol=atol, + place=place, + eps=eps, + ) class TestBatchNormDoubleGradCheckCase6(TestBatchNormDoubleGradCheckCase5): - def init_test(self): self.data_layout = 'NCHW' self.use_global_stats = True diff --git a/python/paddle/fluid/tests/unittests/test_norm_op.py b/python/paddle/fluid/tests/unittests/test_norm_op.py index a5c643f4612df..d5b360cb729c9 100644 --- a/python/paddle/fluid/tests/unittests/test_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_norm_op.py @@ -28,7 +28,6 @@ def l2_norm(x, axis, epsilon): class TestNormOp(OpTest): - def setUp(self): self.op_type = "norm" self.python_api = paddle.fluid.layers.l2_normalize @@ -56,7 +55,6 @@ def init_dtype(self): class TestNormOp2(TestNormOp): - def init_test_case(self): self.shape = [5, 3, 9, 7] self.axis = 0 @@ -64,17 +62,17 @@ def init_test_case(self): class TestNormOp3(TestNormOp): - def init_test_case(self): self.shape = [5, 3, 2, 7] self.axis = -1 self.epsilon = 1e-8 -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class TestNormOp4(TestNormOp): - def init_test_case(self): self.shape = [128, 1024, 14, 14] self.axis = 2 @@ -84,10 +82,11 @@ def test_check_grad(self): pass -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class TestNormOp5(TestNormOp): - def init_test_case(self): self.shape = [2048, 2048] self.axis = 1 @@ -98,7 +97,6 @@ def test_check_grad(self): class TestNormOp6(TestNormOp): - def init_dtype(self): self.dtype = "float32" @@ -106,10 +104,10 @@ def test_check_grad(self): self.check_grad(['X'], 'Out', max_relative_error=0.008) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestNormOp7(TestNormOp): - def init_dtype(self): self.dtype = "float16" @@ -117,14 +115,13 @@ def test_check_output(self): self.check_output_with_place(fluid.core.CUDAPlace(0), atol=5e-2) def test_check_grad(self): - self.check_grad_with_place(fluid.core.CUDAPlace(0), ['X'], - 'Out', - max_relative_error=0.05) + self.check_grad_with_place( + fluid.core.CUDAPlace(0), ['X'], 'Out', max_relative_error=0.05 + ) @skip_check_grad_ci(reason="skip check grad for test mode.") class TestNormTestOp(OpTest): - def setUp(self): self.op_type = "norm" self.init_test_case() @@ -134,7 +131,7 @@ def setUp(self): self.attrs = { 'epsilon': self.epsilon, 'axis': self.axis, - 'is_test': True + 'is_test': True, } self.outputs = {'Out': y} @@ -151,7 +148,6 @@ def init_test_case(self): class API_NormTest(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program()): @@ -164,5 +160,6 @@ def test_norm_x_type(): if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_normal.py b/python/paddle/fluid/tests/unittests/test_normal.py index 3458bbdf7fd86..d24f5f02d2bc0 100644 --- a/python/paddle/fluid/tests/unittests/test_normal.py +++ b/python/paddle/fluid/tests/unittests/test_normal.py @@ -22,7 +22,6 @@ class TestNormalAPI(unittest.TestCase): - def setUp(self): self.mean = 1.0 self.std = 0.0 @@ -30,9 +29,11 @@ def setUp(self): self.repeat_num = 2000 self.set_attrs() self.dtype = self.get_dtype() - self.place=paddle.CUDAPlace(0) \ - if paddle.fluid.core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def set_attrs(self): self.shape = [8, 12] @@ -59,27 +60,32 @@ def static_api(self): ret_all_shape = copy.deepcopy(shape) ret_all_shape.insert(0, self.repeat_num) ret_all = np.zeros(ret_all_shape, self.dtype) - if isinstance(self.mean, np.ndarray) \ - and isinstance(self.std, np.ndarray): + if isinstance(self.mean, np.ndarray) and isinstance( + self.std, np.ndarray + ): with paddle.static.program_guard(paddle.static.Program()): - mean = paddle.fluid.data('Mean', self.mean.shape, - self.mean.dtype) + mean = paddle.fluid.data( + 'Mean', self.mean.shape, self.mean.dtype + ) std = paddle.fluid.data('Std', self.std.shape, self.std.dtype) out = paddle.normal(mean, std, self.shape) exe = paddle.static.Executor(self.place) for i in range(self.repeat_num): - ret = exe.run(feed={ - 'Mean': self.mean, - 'Std': self.std.reshape(shape) - }, - fetch_list=[out]) + ret = exe.run( + feed={ + 'Mean': self.mean, + 'Std': self.std.reshape(shape), + }, + fetch_list=[out], + ) ret_all[i] = ret[0] return ret_all elif isinstance(self.mean, np.ndarray): with paddle.static.program_guard(paddle.static.Program()): - mean = paddle.fluid.data('Mean', self.mean.shape, - self.mean.dtype) + mean = paddle.fluid.data( + 'Mean', self.mean.shape, self.mean.dtype + ) out = paddle.normal(mean, self.std, self.shape) exe = paddle.static.Executor(self.place) @@ -114,10 +120,16 @@ def dygraph_api(self): ret_all_shape.insert(0, self.repeat_num) ret_all = np.zeros(ret_all_shape, self.dtype) - mean = paddle.to_tensor(self.mean) \ - if isinstance(self.mean, np.ndarray) else self.mean - std = paddle.to_tensor(self.std) \ - if isinstance(self.std, np.ndarray) else self.std + mean = ( + paddle.to_tensor(self.mean) + if isinstance(self.mean, np.ndarray) + else self.mean + ) + std = ( + paddle.to_tensor(self.std) + if isinstance(self.std, np.ndarray) + else self.std + ) for i in range(self.repeat_num): out = paddle.normal(mean, std, self.shape) ret_all[i] = out.numpy() @@ -134,42 +146,43 @@ def test_api(self): ret = ret.flatten().reshape([self.repeat_num, -1]) mean = np.mean(ret, axis=0) std = np.std(ret, axis=0) - mean_ref=self.mean.flatten() \ - if isinstance(self.mean, np.ndarray) else self.mean - std_ref=self.std.flatten() \ - if isinstance(self.std, np.ndarray) else self.std + mean_ref = ( + self.mean.flatten() + if isinstance(self.mean, np.ndarray) + else self.mean + ) + std_ref = ( + self.std.flatten() + if isinstance(self.std, np.ndarray) + else self.std + ) np.testing.assert_allclose(mean_ref, mean, rtol=0.2, atol=0.2) np.testing.assert_allclose(std_ref, std, rtol=0.2, atol=0.2) class TestNormalAPI_mean_is_tensor(TestNormalAPI): - def set_attrs(self): self.mean = np.random.uniform(-2, -1, [2, 3, 4, 5]).astype('float64') class TestNormalAPI_std_is_tensor(TestNormalAPI): - def set_attrs(self): self.std = np.random.uniform(0.7, 1, [2, 3, 17]).astype('float64') class TestNormalAPI_mean_std_are_tensor(TestNormalAPI): - def set_attrs(self): self.mean = np.random.uniform(1, 2, [1, 100]).astype('float64') self.std = np.random.uniform(0.5, 1, [1, 100]).astype('float64') class TestNormalAPI_mean_std_are_tensor_with_different_dtype(TestNormalAPI): - def set_attrs(self): self.mean = np.random.uniform(1, 2, [100]).astype('float64') self.std = np.random.uniform(1, 2, [100]).astype('float32') class TestNormalAlias(unittest.TestCase): - def test_alias(self): paddle.disable_static() shape = [1, 2, 3] @@ -180,7 +193,6 @@ def test_alias(self): class TestNormalErrors(unittest.TestCase): - def test_errors(self): with paddle.static.program_guard(paddle.static.Program()): mean = [1, 2, 3] diff --git a/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py b/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py index e35e0283332d8..5a5b4f4eeb0cc 100644 --- a/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py +++ b/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py @@ -22,18 +22,19 @@ class TestNormalization(unittest.TestCase): data_desc = {"name": "input", "shape": (2, 3, 7)} def gen_random_input(self): - """Generate random input data. - """ - self.data = np.random.random( - size=self.data_desc["shape"]).astype("float32") + """Generate random input data.""" + self.data = np.random.random(size=self.data_desc["shape"]).astype( + "float32" + ) def set_program(self, axis, epsilon): - """Build the test program. - """ - data = fluid.layers.data(name=self.data_desc["name"], - shape=self.data_desc["shape"], - dtype="float32", - append_batch_size=False) + """Build the test program.""" + data = fluid.layers.data( + name=self.data_desc["name"], + shape=self.data_desc["shape"], + dtype="float32", + append_batch_size=False, + ) data.stop_gradient = False l2_norm = fluid.layers.l2_normalize(x=data, axis=axis, epsilon=epsilon) out = fluid.layers.reduce_sum(l2_norm, dim=None) @@ -42,8 +43,7 @@ def set_program(self, axis, epsilon): self.fetch_list = [l2_norm] def run_program(self): - """Run the test program. - """ + """Run the test program.""" places = [core.CPUPlace()] if core.is_compiled_with_cuda(): places.append(core.CUDAPlace(0)) @@ -52,33 +52,33 @@ def run_program(self): self.set_inputs(place) exe = fluid.Executor(place) - output, = exe.run(fluid.default_main_program(), - feed=self.inputs, - fetch_list=self.fetch_list, - return_numpy=True) + (output,) = exe.run( + fluid.default_main_program(), + feed=self.inputs, + fetch_list=self.fetch_list, + return_numpy=True, + ) self.op_output = output def set_inputs(self, place): - """Set the randomly generated data to the test program. - """ + """Set the randomly generated data to the test program.""" self.inputs = {} tensor = fluid.Tensor() tensor.set(self.data, place) self.inputs[self.data_desc["name"]] = tensor def l2_normalize(self, data, axis, epsilon): - """ Compute the groundtruth. - """ + """Compute the groundtruth.""" output = data / np.broadcast_to( np.sqrt(np.sum(np.square(data), axis=axis, keepdims=True)), - data.shape) + data.shape, + ) return output def test_l2_normalize(self): - """ Test the python wrapper for l2_normalize. - """ + """Test the python wrapper for l2_normalize.""" axis = 1 - #TODO(caoying) epsilon is not supported due to lack of a maximum_op. + # TODO(caoying) epsilon is not supported due to lack of a maximum_op. epsilon = 1e-6 self.gen_random_input() @@ -89,10 +89,9 @@ def test_l2_normalize(self): expect_output = self.l2_normalize(self.data, axis, epsilon) # check output - np.testing.assert_allclose(self.op_output, - expect_output, - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + self.op_output, expect_output, rtol=1e-05, atol=0.001 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_normalize.py b/python/paddle/fluid/tests/unittests/test_normalize.py index bb8b4881d3f47..e4fed2904e977 100644 --- a/python/paddle/fluid/tests/unittests/test_normalize.py +++ b/python/paddle/fluid/tests/unittests/test_normalize.py @@ -28,7 +28,6 @@ def p_normalize(x, axis=1, p=2, epsilon=1e-12, keepdims=True): class TestNNFunctionalNormalize(unittest.TestCase): - def setUp(self): self.input_np = np.random.random(size=(10, 10)).astype(np.float32) self.input_np2 = np.array([0.0, 0.0]).astype(np.float32) @@ -66,11 +65,10 @@ def run_static(self, use_gpu=False): place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - static_result = exe.run(feed={ - "input": self.input_np, - "input2": self.input_np2 - }, - fetch_list=[result0, result1, result2, result4]) + static_result = exe.run( + feed={"input": self.input_np, "input2": self.input_np2}, + fetch_list=[result0, result1, result2, result4], + ) np.testing.assert_allclose(static_result[0], self.expected0, rtol=1e-05) np.testing.assert_allclose(static_result[1], self.expected1, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_npair_loss_op.py b/python/paddle/fluid/tests/unittests/test_npair_loss_op.py index 557c5bac866b0..f90b3f4cdff60 100644 --- a/python/paddle/fluid/tests/unittests/test_npair_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_npair_loss_op.py @@ -20,13 +20,13 @@ def npairloss(anchor, positive, labels, l2_reg=0.002): - def softmax_cross_entropy_with_logits(logits, labels): logits = np.exp(logits) logits = logits / np.sum(logits, axis=1).reshape(-1, 1) - return np.mean(-np.sum(labels * np.log(logits), axis=1), - dtype=np.float32) + return np.mean( + -np.sum(labels * np.log(logits), axis=1), dtype=np.float32 + ) batch_size = labels.shape[0] @@ -35,27 +35,26 @@ def softmax_cross_entropy_with_logits(logits, labels): labels = labels / np.sum(labels, axis=1, keepdims=True) l2loss = np.mean(np.sum(np.power(anchor, 2), 1)) + np.mean( - np.sum(np.power(positive, 2), 1)) + np.sum(np.power(positive, 2), 1) + ) l2loss = (l2loss * 0.25 * l2_reg).astype(np.float32) similarity_matrix = np.matmul(anchor, positive.transpose()) celoss = np.mean( - softmax_cross_entropy_with_logits(similarity_matrix, labels)) + softmax_cross_entropy_with_logits(similarity_matrix, labels) + ) return l2loss + celoss class TestNpairLossOp(unittest.TestCase): - def setUp(self): self.dtype = np.float32 def __assert_close(self, tensor, np_array, msg, atol=1e-4): - np.testing.assert_allclose(np.array(tensor), - np_array, - rtol=1e-05, - atol=atol, - err_msg=msg) + np.testing.assert_allclose( + np.array(tensor), np_array, rtol=1e-05, atol=atol, err_msg=msg + ) def test_npair_loss(self): reg_lambda = 0.002 @@ -65,81 +64,97 @@ def test_npair_loss(self): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - embeddings_anchor = np.random.rand(num_data, - feat_dim).astype(np.float32) - embeddings_positive = np.random.rand(num_data, - feat_dim).astype(np.float32) - row_labels = np.random.randint(0, num_classes, - size=(num_data)).astype(np.float32) - out_loss = npairloss(embeddings_anchor, - embeddings_positive, - row_labels, - l2_reg=reg_lambda) - - anc = fluid.layers.data(dtype='float32', - name='anc', - shape=embeddings_anchor.shape, - append_batch_size=False) - pos = fluid.layers.data(dtype='float32', - name='pos', - shape=embeddings_positive.shape, - append_batch_size=False) - lab = fluid.layers.data(dtype='float32', - name='lab', - shape=row_labels.shape, - append_batch_size=False) - - npair_loss_op = fluid.layers.npair_loss(anchor=anc, - positive=pos, - labels=lab, - l2_reg=reg_lambda) - out_tensor = exe.run(feed={ - 'anc': embeddings_anchor, - 'pos': embeddings_positive, - 'lab': row_labels - }, - fetch_list=[npair_loss_op.name]) - - self.__assert_close(out_tensor, - out_loss, - "inference output are different at " + str(place) + - ", " + str(np.dtype('float32')) + - str(np.array(out_tensor)) + str(out_loss), - atol=1e-3) + embeddings_anchor = np.random.rand(num_data, feat_dim).astype( + np.float32 + ) + embeddings_positive = np.random.rand(num_data, feat_dim).astype( + np.float32 + ) + row_labels = np.random.randint(0, num_classes, size=(num_data)).astype( + np.float32 + ) + out_loss = npairloss( + embeddings_anchor, + embeddings_positive, + row_labels, + l2_reg=reg_lambda, + ) + + anc = fluid.layers.data( + dtype='float32', + name='anc', + shape=embeddings_anchor.shape, + append_batch_size=False, + ) + pos = fluid.layers.data( + dtype='float32', + name='pos', + shape=embeddings_positive.shape, + append_batch_size=False, + ) + lab = fluid.layers.data( + dtype='float32', + name='lab', + shape=row_labels.shape, + append_batch_size=False, + ) + + npair_loss_op = fluid.layers.npair_loss( + anchor=anc, positive=pos, labels=lab, l2_reg=reg_lambda + ) + out_tensor = exe.run( + feed={ + 'anc': embeddings_anchor, + 'pos': embeddings_positive, + 'lab': row_labels, + }, + fetch_list=[npair_loss_op.name], + ) + + self.__assert_close( + out_tensor, + out_loss, + "inference output are different at " + + str(place) + + ", " + + str(np.dtype('float32')) + + str(np.array(out_tensor)) + + str(out_loss), + atol=1e-3, + ) class TestNpairLossOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): anchor_np = np.random.random((2, 4)).astype("float32") positive_np = np.random.random((2, 4)).astype("float32") labels_np = np.random.random((2)).astype("float32") - anchor_data = fluid.data(name='anchor', - shape=[2, 4], - dtype='float32') - positive_data = fluid.data(name='positive', - shape=[2, 4], - dtype='float32') + anchor_data = fluid.data( + name='anchor', shape=[2, 4], dtype='float32' + ) + positive_data = fluid.data( + name='positive', shape=[2, 4], dtype='float32' + ) labels_data = fluid.data(name='labels', shape=[2], dtype='float32') def test_anchor_Variable(): # the anchor type must be Variable - fluid.layers.npair_loss(anchor=anchor_np, - positive=positive_data, - labels=labels_data) + fluid.layers.npair_loss( + anchor=anchor_np, positive=positive_data, labels=labels_data + ) def test_positive_Variable(): # the positive type must be Variable - fluid.layers.npair_loss(anchor=anchor_data, - positive=positive_np, - labels=labels_data) + fluid.layers.npair_loss( + anchor=anchor_data, positive=positive_np, labels=labels_data + ) def test_labels_Variable(): # the labels type must be Variable - fluid.layers.npair_loss(anchor=anchor_data, - positive=positive_data, - labels=labels_np) + fluid.layers.npair_loss( + anchor=anchor_data, positive=positive_data, labels=labels_np + ) self.assertRaises(TypeError, test_anchor_Variable) self.assertRaises(TypeError, test_positive_Variable) @@ -147,30 +162,34 @@ def test_labels_Variable(): def test_anchor_type(): # dtype must be float32 or float64 - anchor_data1 = fluid.data(name='anchor1', - shape=[2, 4], - dtype='int32') - fluid.layers.npair_loss(anchor=anchor_data, - positive=positive_data, - labels=labels_np) + anchor_data1 = fluid.data( + name='anchor1', shape=[2, 4], dtype='int32' + ) + fluid.layers.npair_loss( + anchor=anchor_data, positive=positive_data, labels=labels_np + ) def test_positive_type(): # dtype must be float32 or float64 - positive_data1 = fluid.data(name='positive1', - shape=[2, 4], - dtype='int32') - fluid.layers.npair_loss(anchor=anchor_data, - positive=positive_data1, - labels=labels_np) + positive_data1 = fluid.data( + name='positive1', shape=[2, 4], dtype='int32' + ) + fluid.layers.npair_loss( + anchor=anchor_data, + positive=positive_data1, + labels=labels_np, + ) def test_labels_type(): # dtype must be float32 or float64 - labels_data1 = fluid.data(name='labels1', - shape=[2], - dtype='int32') - fluid.layers.npair_loss(anchor=anchor_data, - positive=positive_data, - labels=labels_data1) + labels_data1 = fluid.data( + name='labels1', shape=[2], dtype='int32' + ) + fluid.layers.npair_loss( + anchor=anchor_data, + positive=positive_data, + labels=labels_data1, + ) self.assertRaises(TypeError, test_anchor_type) self.assertRaises(TypeError, test_positive_type) diff --git a/python/paddle/fluid/tests/unittests/test_number_count_op.py b/python/paddle/fluid/tests/unittests/test_number_count_op.py index 58e24e6692ad3..a31fb1a5978c1 100644 --- a/python/paddle/fluid/tests/unittests/test_number_count_op.py +++ b/python/paddle/fluid/tests/unittests/test_number_count_op.py @@ -22,17 +22,17 @@ def count(x, upper_num): - res = np.zeros((upper_num, )).astype(int) + res = np.zeros((upper_num,)).astype(int) for i in x.reshape(-1): if i >= 0 and i < len(res): res[i] += 1 return res -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestNumberCountOpInt64(op_test.OpTest): - def setUp(self): upper_num = 16 self.op_type = "number_count" @@ -45,14 +45,15 @@ def test_forward(self): self.check_output_with_place(paddle.CUDAPlace(0)) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestNumberCountAPI(unittest.TestCase): - def setUp(self): self.upper_num = 320 - self.x = np.random.randint(-1, self.upper_num, - size=(6000, 200)).astype('int64') + self.x = np.random.randint(-1, self.upper_num, size=(6000, 200)).astype( + 'int64' + ) self.out = count(self.x, self.upper_num) self.place = paddle.CUDAPlace(0) diff --git a/python/paddle/fluid/tests/unittests/test_numel_op.py b/python/paddle/fluid/tests/unittests/test_numel_op.py index 7bb76c531a31a..4bb359a7bd2e0 100644 --- a/python/paddle/fluid/tests/unittests/test_numel_op.py +++ b/python/paddle/fluid/tests/unittests/test_numel_op.py @@ -20,7 +20,6 @@ class TestNumelOp(OpTest): - def setUp(self): self.op_type = "size" self.init() @@ -38,19 +37,16 @@ def init(self): class TestNumelOp1(TestNumelOp): - def init(self): self.shape = (11, 66) class TestNumelOp2(TestNumelOp): - def init(self): - self.shape = (0, ) + self.shape = (0,) class TestNumelAPI(unittest.TestCase): - def test_numel_static(self): main_program = fluid.Program() startup_program = fluid.Program() @@ -64,17 +60,19 @@ def test_numel_static(self): out_1 = paddle.numel(x_1) out_2 = paddle.numel(x_2) exe = paddle.static.Executor(place=paddle.CPUPlace()) - res_1, res_2 = exe.run(feed={ - "x_1": input_1, - "x_2": input_2, - }, - fetch_list=[out_1, out_2]) - assert (np.array_equal(res_1, - np.array([np.size(input_1) - ]).astype("int64"))) - assert (np.array_equal(res_2, - np.array([np.size(input_2) - ]).astype("int64"))) + res_1, res_2 = exe.run( + feed={ + "x_1": input_1, + "x_2": input_2, + }, + fetch_list=[out_1, out_2], + ) + assert np.array_equal( + res_1, np.array([np.size(input_1)]).astype("int64") + ) + assert np.array_equal( + res_2, np.array([np.size(input_2)]).astype("int64") + ) def test_numel_imperative(self): paddle.disable_static(paddle.CPUPlace()) @@ -84,8 +82,8 @@ def test_numel_imperative(self): x_2 = paddle.to_tensor(input_2) out_1 = paddle.numel(x_1) out_2 = paddle.numel(x_2) - assert (np.array_equal(out_1.numpy().item(0), np.size(input_1))) - assert (np.array_equal(out_2.numpy().item(0), np.size(input_2))) + assert np.array_equal(out_1.numpy().item(0), np.size(input_1)) + assert np.array_equal(out_2.numpy().item(0), np.size(input_2)) paddle.enable_static() def test_error(self): diff --git a/python/paddle/fluid/tests/unittests/test_one_hot_op.py b/python/paddle/fluid/tests/unittests/test_one_hot_op.py index 55abf26ea231e..4ca0017052eb9 100644 --- a/python/paddle/fluid/tests/unittests/test_one_hot_op.py +++ b/python/paddle/fluid/tests/unittests/test_one_hot_op.py @@ -22,7 +22,6 @@ class TestOneHotOp(OpTest): - def setUp(self): self.op_type = 'one_hot' depth = 10 @@ -32,8 +31,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -47,7 +47,6 @@ def test_check_output(self): class TestOneHotOp_attr(OpTest): - def setUp(self): self.op_type = 'one_hot' depth = 10 @@ -56,8 +55,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -71,7 +71,6 @@ def test_check_output(self): class TestOneHotOp_default_dtype(OpTest): - def setUp(self): self.op_type = 'one_hot' depth = 10 @@ -81,8 +80,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -96,7 +96,6 @@ def test_check_output(self): class TestOneHotOp_default_dtype_attr(OpTest): - def setUp(self): self.op_type = 'one_hot' depth = 10 @@ -105,8 +104,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -120,7 +120,6 @@ def test_check_output(self): class TestOneHotOp_out_of_range(OpTest): - def setUp(self): self.op_type = 'one_hot' depth = 10 @@ -128,8 +127,9 @@ def setUp(self): x = [np.random.choice([-1, depth]) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype( + 'float32' + ) self.inputs = {'X': (x, x_lod)} self.attrs = {'depth': depth, 'allow_out_of_range': True} @@ -140,7 +140,6 @@ def test_check_output(self): class TestOneHotOp_exception(unittest.TestCase): - def setUp(self): self.op_type = 'one_hot' self.depth = 10 @@ -156,46 +155,54 @@ def setUp(self): def test_check_output(self): program = Program() with program_guard(program): - x = fluid.layers.data(name='x', - shape=[self.dimension], - dtype='float32', - lod_level=1) + x = fluid.layers.data( + name='x', shape=[self.dimension], dtype='float32', lod_level=1 + ) block = program.current_block() - one_hot_out = block.create_var(name="one_hot_out", - type=core.VarDesc.VarType.LOD_TENSOR, - dtype='float32') - block.append_op(type='one_hot', - inputs={'X': x}, - attrs={'depth': self.depth}, - outputs={'Out': one_hot_out}) + one_hot_out = block.create_var( + name="one_hot_out", + type=core.VarDesc.VarType.LOD_TENSOR, + dtype='float32', + ) + block.append_op( + type='one_hot', + inputs={'X': x}, + attrs={'depth': self.depth}, + outputs={'Out': one_hot_out}, + ) exe = fluid.Executor(self.place) def run(): - exe.run(feed={'x': self.x}, - fetch_list=[one_hot_out], - return_numpy=False) + exe.run( + feed={'x': self.x}, + fetch_list=[one_hot_out], + return_numpy=False, + ) self.assertRaises(ValueError, run) class TestOneHotOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input must be Variable in_w = np.random.random((4, 1)).astype("int32") self.assertRaises(TypeError, fluid.layers.one_hot, in_w) # the input must be int32 or int 64 - in_w2 = fluid.layers.data(name="in_w2", - shape=[4, 1], - append_batch_size=False, - dtype="float32") + in_w2 = fluid.layers.data( + name="in_w2", + shape=[4, 1], + append_batch_size=False, + dtype="float32", + ) self.assertRaises(TypeError, fluid.layers.one_hot, in_w2) # the depth must be int, long or Variable - in_r = fluid.layers.data(name="in_r", - shape=[4, 1], - append_batch_size=False, - dtype="int32") + in_r = fluid.layers.data( + name="in_r", + shape=[4, 1], + append_batch_size=False, + dtype="int32", + ) depth_w = np.array([4]) self.assertRaises(TypeError, fluid.layers.one_hot, in_r, 4.1) self.assertRaises(TypeError, fluid.layers.one_hot, in_r, depth_w) diff --git a/python/paddle/fluid/tests/unittests/test_one_hot_v2_op.py b/python/paddle/fluid/tests/unittests/test_one_hot_v2_op.py index 284957a5318f1..23d277c04e59f 100644 --- a/python/paddle/fluid/tests/unittests/test_one_hot_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_one_hot_v2_op.py @@ -22,7 +22,6 @@ class TestOneHotOp(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -46,7 +45,6 @@ def test_check_output(self): class TestOneHotOp_attr(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -55,8 +53,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -70,7 +69,6 @@ def test_check_output(self): class TestOneHotOp_default_dtype(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -94,7 +92,6 @@ def test_check_output(self): class TestOneHotOp_default_dtype_attr(OpTest): - def setUp(self): self.op_type = 'one_hot_v2' depth = 10 @@ -103,8 +100,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype('float32') + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + 'float32' + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -118,7 +116,6 @@ def test_check_output(self): class TestOneHotOp_exception(unittest.TestCase): - def setUp(self): self.op_type = 'one_hot_v2' self.depth = 10 @@ -134,30 +131,34 @@ def setUp(self): def test_check_output(self): program = Program() with program_guard(program): - x = fluid.layers.data(name='x', - shape=[self.dimension], - dtype='float32', - lod_level=1) + x = fluid.layers.data( + name='x', shape=[self.dimension], dtype='float32', lod_level=1 + ) block = program.current_block() - one_hot_out = block.create_var(name="one_hot_out", - type=core.VarDesc.VarType.LOD_TENSOR, - dtype='float32') - block.append_op(type='one_hot', - inputs={'X': x}, - attrs={'depth': self.depth}, - outputs={'Out': one_hot_out}) + one_hot_out = block.create_var( + name="one_hot_out", + type=core.VarDesc.VarType.LOD_TENSOR, + dtype='float32', + ) + block.append_op( + type='one_hot', + inputs={'X': x}, + attrs={'depth': self.depth}, + outputs={'Out': one_hot_out}, + ) exe = fluid.Executor(self.place) def run(): - exe.run(feed={'x': self.x}, - fetch_list=[one_hot_out], - return_numpy=False) + exe.run( + feed={'x': self.x}, + fetch_list=[one_hot_out], + return_numpy=False, + ) self.assertRaises(ValueError, run) class TestOneHotOpApi(unittest.TestCase): - def test_api(self): depth = 10 self._run(depth) @@ -168,45 +169,53 @@ def test_api_with_depthTensor(self): def test_api_with_dygraph(self): depth = 10 - label = np.array([np.random.randint(0, depth - 1) - for i in range(6)]).reshape([6, 1]) + label = np.array( + [np.random.randint(0, depth - 1) for i in range(6)] + ).reshape([6, 1]) with fluid.dygraph.guard(): one_hot_label = fluid.one_hot( - input=fluid.dygraph.to_variable(label), depth=depth) + input=fluid.dygraph.to_variable(label), depth=depth + ) one_hot_label = paddle.nn.functional.one_hot( - fluid.dygraph.to_variable(label), depth) + fluid.dygraph.to_variable(label), depth + ) with _test_eager_guard(): one_hot_label = paddle.nn.functional.one_hot( - paddle.to_tensor(label), depth) + paddle.to_tensor(label), depth + ) def _run(self, depth): label = fluid.layers.data(name="label", shape=[1], dtype="int64") one_hot_label = fluid.one_hot(input=label, depth=depth) place = fluid.CPUPlace() - label_data = np.array([np.random.randint(0, 10 - 1) - for i in range(6)]).reshape([6, 1]) + label_data = np.array( + [np.random.randint(0, 10 - 1) for i in range(6)] + ).reshape([6, 1]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'label': label_data, - }, - fetch_list=[one_hot_label], - return_numpy=False) + ret = exe.run( + feed={ + 'label': label_data, + }, + fetch_list=[one_hot_label], + return_numpy=False, + ) class BadInputTestOnehotV2(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): def test_bad_x(): - label = fluid.layers.data(name="label", - shape=[4], - append_batch_size=False, - dtype="float32") + label = fluid.layers.data( + name="label", + shape=[4], + append_batch_size=False, + dtype="float32", + ) one_hot_label = fluid.one_hot(input=label, depth=4) self.assertRaises(TypeError, test_bad_x) diff --git a/python/paddle/fluid/tests/unittests/test_ones_like.py b/python/paddle/fluid/tests/unittests/test_ones_like.py index 8c0ceadbf581e..fe7a66acecdf4 100644 --- a/python/paddle/fluid/tests/unittests/test_ones_like.py +++ b/python/paddle/fluid/tests/unittests/test_ones_like.py @@ -23,7 +23,6 @@ class TestOnesLikeAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x = paddle.fluid.data('x', [3, 4]) @@ -31,7 +30,6 @@ def test_errors(self): class TestOnesLikeAPI(unittest.TestCase): - def test_api(self): shape = [3, 4] startup_program = Program() @@ -46,25 +44,33 @@ def test_api(self): out4 = ones_like(x, 'int32') out5 = ones_like(x, 'int64') - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - outs = exe.run(train_program, - feed={'X': np.ones(shape).astype('float32')}, - fetch_list=[out1, out2, out3, out4, out5]) + outs = exe.run( + train_program, + feed={'X': np.ones(shape).astype('float32')}, + fetch_list=[out1, out2, out3, out4, out5], + ) for i, dtype in enumerate( - [np.float32, np.bool_, np.float64, np.int32, np.int64]): + [np.float32, np.bool_, np.float64, np.int32, np.int64] + ): self.assertEqual(outs[i].dtype, dtype) self.assertEqual((outs[i] == np.ones(shape, dtype)).all(), True) class TestOnesLikeImpeartive(unittest.TestCase): - def test_out(self): shape = [3, 4] - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) paddle.disable_static(place) x = paddle.to_tensor(np.ones(shape)) for dtype in [np.bool_, np.float32, np.float64, np.int32, np.int64]: @@ -80,11 +86,13 @@ def test_out(self): class TestOnesAPI(unittest.TestCase): - def test_api(self): shape = [3, 4] - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) paddle.disable_static(place) for dtype in [np.float32, np.float64, np.int32, np.int64]: diff --git a/python/paddle/fluid/tests/unittests/test_ones_op.py b/python/paddle/fluid/tests/unittests/test_ones_op.py index ac02627f9cc1a..a82842b3c72aa 100644 --- a/python/paddle/fluid/tests/unittests/test_ones_op.py +++ b/python/paddle/fluid/tests/unittests/test_ones_op.py @@ -21,13 +21,12 @@ class ApiOnesTest(unittest.TestCase): - def test_paddle_ones(self): with paddle.static.program_guard(paddle.static.Program()): ones = paddle.ones(shape=[10]) place = paddle.CPUPlace() exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[ones]) + (result,) = exe.run(fetch_list=[ones]) expected_result = np.ones(10, dtype="float32") self.assertEqual((result == expected_result).all(), True) @@ -35,7 +34,7 @@ def test_paddle_ones(self): ones = paddle.ones(shape=[10], dtype="float64") place = paddle.CPUPlace() exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[ones]) + (result,) = exe.run(fetch_list=[ones]) expected_result = np.ones(10, dtype="float64") self.assertEqual((result == expected_result).all(), True) @@ -43,7 +42,7 @@ def test_paddle_ones(self): ones = paddle.ones(shape=[10], dtype="int64") place = paddle.CPUPlace() exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[ones]) + (result,) = exe.run(fetch_list=[ones]) expected_result = np.ones(10, dtype="int64") self.assertEqual((result == expected_result).all(), True) @@ -52,15 +51,13 @@ def test_fluid_ones(self): ones = fluid.layers.ones(shape=[10], dtype="int64") place = paddle.CPUPlace() exe = paddle.static.Executor(place) - result, = exe.run(fetch_list=[ones]) + (result,) = exe.run(fetch_list=[ones]) expected_result = np.ones(10, dtype="int64") self.assertEqual((result == expected_result).all(), True) class ApiOnesZerosError(unittest.TestCase): - def test_errors(self): - def test_error1(): with paddle.static.program_guard(paddle.static.Program()): ones = paddle.ones(shape=10, dtype="int64") diff --git a/python/paddle/fluid/tests/unittests/test_onnx_export.py b/python/paddle/fluid/tests/unittests/test_onnx_export.py index 6cb4ed2fd924e..4a4e260004462 100644 --- a/python/paddle/fluid/tests/unittests/test_onnx_export.py +++ b/python/paddle/fluid/tests/unittests/test_onnx_export.py @@ -20,7 +20,6 @@ class LinearNet(paddle.nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = paddle.nn.Linear(128, 10) @@ -30,7 +29,6 @@ def forward(self, x): class Logic(paddle.nn.Layer): - def __init__(self): super(Logic, self).__init__() @@ -42,10 +40,10 @@ def forward(self, x, y, z): class TestExportWithTensor(unittest.TestCase): - def func_with_tensor(self): - self.x_spec = paddle.static.InputSpec(shape=[None, 128], - dtype='float32') + self.x_spec = paddle.static.InputSpec( + shape=[None, 128], dtype='float32' + ) model = LinearNet() paddle.onnx.export(model, 'linear_net', input_spec=[self.x_spec]) @@ -56,7 +54,6 @@ def test_with_tensor(self): class TestExportWithTensor1(unittest.TestCase): - def func_with_tensor(self): self.x = paddle.to_tensor(np.random.random((1, 128))) model = LinearNet() @@ -69,17 +66,15 @@ def test_with_tensor(self): class TestExportPrunedGraph(unittest.TestCase): - def func_prune_graph(self): model = Logic() self.x = paddle.to_tensor(np.array([1])) self.y = paddle.to_tensor(np.array([-1])) paddle.jit.to_static(model) out = model(self.x, self.y, z=True) - paddle.onnx.export(model, - 'pruned', - input_spec=[self.x], - output_spec=[out]) + paddle.onnx.export( + model, 'pruned', input_spec=[self.x], output_spec=[out] + ) def test_prune_graph(self): # test eager diff --git a/python/paddle/fluid/tests/unittests/test_op_function_generator.py b/python/paddle/fluid/tests/unittests/test_op_function_generator.py index a8e4e7d3e8211..3163a76888802 100644 --- a/python/paddle/fluid/tests/unittests/test_op_function_generator.py +++ b/python/paddle/fluid/tests/unittests/test_op_function_generator.py @@ -22,7 +22,6 @@ class TestTracedLayer(fluid.dygraph.Layer): - def __init__(self, name_scope): super(TestTracedLayer, self).__init__(name_scope) @@ -31,7 +30,6 @@ def forward(self, input): class TestVariable(unittest.TestCase): - def setUp(self): self.shape = [512, 768] self.dtype = np.float32 @@ -100,7 +98,8 @@ def test_traced_layer(self): a = np.random.uniform(-1, 1, self.shape).astype(self.dtype) x = fluid.dygraph.to_variable(a) res_dygraph, static_layer = TracedLayer.trace( - layer, inputs=x) # dygraph out + layer, inputs=x + ) # dygraph out res_static_graph = static_layer([x])[0] np.testing.assert_array_equal(res_dygraph.numpy(), res_static_graph) diff --git a/python/paddle/fluid/tests/unittests/test_op_name_conflict.py b/python/paddle/fluid/tests/unittests/test_op_name_conflict.py index 86f82f0a9e22e..10e9b4d3028c3 100644 --- a/python/paddle/fluid/tests/unittests/test_op_name_conflict.py +++ b/python/paddle/fluid/tests/unittests/test_op_name_conflict.py @@ -18,7 +18,6 @@ class TestOpNameConflict(unittest.TestCase): - def test_conflict(self): main = fluid.Program() startup = fluid.Program() @@ -34,12 +33,14 @@ def test_conflict(self): place = fluid.CPUPlace() exe = fluid.Executor(place) - m_v, n_v, p_v = exe.run(feed={ - "x": np.ones((1), "float32") * 2, - "y": np.ones((1), "float32") * 3, - "z": np.ones((1), "float32") * 5 - }, - fetch_list=[m, n, p]) + m_v, n_v, p_v = exe.run( + feed={ + "x": np.ones((1), "float32") * 2, + "y": np.ones((1), "float32") * 3, + "z": np.ones((1), "float32") * 5, + }, + fetch_list=[m, n, p], + ) self.assertEqual(m_v[0], 5.0) self.assertEqual(n_v[0], 8.0) @@ -50,37 +51,42 @@ def test_layers(self): startup = fluid.Program() with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - place = fluid.CUDAPlace(0) if fluid.core.is_compiled_with_cuda( - ) else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - data = fluid.data(name='data', - shape=[None, 1, 2, 2], - dtype='float32') - tensor = fluid.data(name='tensor', - shape=[None, 32, 64], - dtype='float32') - x = fluid.data(name='x', - shape=[None, 1], - dtype='float32', - lod_level=1) + data = fluid.data( + name='data', shape=[None, 1, 2, 2], dtype='float32' + ) + tensor = fluid.data( + name='tensor', shape=[None, 32, 64], dtype='float32' + ) + x = fluid.data( + name='x', shape=[None, 1], dtype='float32', lod_level=1 + ) input_scale = fluid.layers.create_parameter( shape=[1], dtype="float32", - default_initializer=fluid.initializer.Constant(2.0)) + default_initializer=fluid.initializer.Constant(2.0), + ) input_bias = fluid.layers.create_parameter( shape=[1], dtype="float32", - default_initializer=fluid.initializer.Constant(0.5)) - out_affine = fluid.layers.affine_channel(data, - scale=input_scale, - bias=input_bias) - out_similarity = fluid.layers.similarity_focus(input=data, - axis=1, - indexes=[0]) + default_initializer=fluid.initializer.Constant(0.5), + ) + out_affine = fluid.layers.affine_channel( + data, scale=input_scale, bias=input_bias + ) + out_similarity = fluid.layers.similarity_focus( + input=data, axis=1, indexes=[0] + ) position_tensor = fluid.layers.add_position_encoding( - input=tensor, alpha=1.0, beta=1.0) + input=tensor, alpha=1.0, beta=1.0 + ) x_reversed = fluid.layers.sequence_reverse(x) exe.run(fluid.default_startup_program()) @@ -88,21 +94,24 @@ def test_layers(self): x_d = fluid.create_lod_tensor( np.array([[1.1], [2.2], [3.3], [4.4]]).astype('float32'), - [[1, 3]], place) - outs = exe.run(test_program, - fetch_list=[ - out_affine, out_similarity, position_tensor, - x_reversed - ], - feed={ - data.name: - np.ones([1, 1, 2, 2]).astype('float32'), - tensor.name: - np.ones([1, 32, 64]).astype('float32'), - x.name: - x_d - }, - return_numpy=False) + [[1, 3]], + place, + ) + outs = exe.run( + test_program, + fetch_list=[ + out_affine, + out_similarity, + position_tensor, + x_reversed, + ], + feed={ + data.name: np.ones([1, 1, 2, 2]).astype('float32'), + tensor.name: np.ones([1, 32, 64]).astype('float32'), + x.name: x_d, + }, + return_numpy=False, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_op_support_gpu.py b/python/paddle/fluid/tests/unittests/test_op_support_gpu.py index b13301517b34c..fe7fd6ab3d5f8 100644 --- a/python/paddle/fluid/tests/unittests/test_op_support_gpu.py +++ b/python/paddle/fluid/tests/unittests/test_op_support_gpu.py @@ -17,10 +17,10 @@ class TestOpSupportGPU(unittest.TestCase): - def test_case(self): - self.assertEqual(core.is_compiled_with_cuda(), - core.op_support_gpu("sum")) + self.assertEqual( + core.is_compiled_with_cuda(), core.op_support_gpu("sum") + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_op_version.py b/python/paddle/fluid/tests/unittests/test_op_version.py index 49b3c69b8909c..47f49d21864aa 100644 --- a/python/paddle/fluid/tests/unittests/test_op_version.py +++ b/python/paddle/fluid/tests/unittests/test_op_version.py @@ -19,7 +19,6 @@ class OpLastCheckpointCheckerTest(unittest.TestCase): - def __init__(self, methodName='runTest'): super(OpLastCheckpointCheckerTest, self).__init__(methodName) self.checker = utils.OpLastCheckpointChecker() @@ -27,8 +26,9 @@ def __init__(self, methodName='runTest'): def test_op_attr_info(self): update_type = fluid.core.OpUpdateType.kNewAttr - info_list = self.checker.filter_updates(self.fake_op, update_type, - 'STRINGS') + info_list = self.checker.filter_updates( + self.fake_op, update_type, 'STRINGS' + ) self.assertTrue(info_list) self.assertEqual(info_list[0].name(), 'STRINGS') self.assertEqual(info_list[0].default_value(), ['str1', 'str2']) @@ -36,8 +36,9 @@ def test_op_attr_info(self): def test_op_input_output_info(self): update_type = fluid.core.OpUpdateType.kNewInput - info_list = self.checker.filter_updates(self.fake_op, update_type, - 'NewInput') + info_list = self.checker.filter_updates( + self.fake_op, update_type, 'NewInput' + ) self.assertTrue(info_list) self.assertEqual(info_list[0].name(), 'NewInput') self.assertEqual(info_list[0].remark(), 'NewInput_') @@ -50,7 +51,6 @@ def test_op_bug_fix_info(self): class OpVersionTest(unittest.TestCase): - def __init__(self, methodName='runTest'): super(OpVersionTest, self).__init__(methodName) self.vmap = fluid.core.get_op_version_map() @@ -72,11 +72,13 @@ def test_checkpoints(self): true_l = [2.56, 1.28] self.assertEqual(len(true_l), len(desc_2[1].info().default_value())) for i in range(len(true_l)): - self.assertAlmostEqual(desc_2[1].info().default_value()[i], - true_l[i], 2) + self.assertAlmostEqual( + desc_2[1].info().default_value()[i], true_l[i], 2 + ) self.assertEqual(desc_2[2].info().default_value(), [10, 100]) - self.assertEqual(desc_2[3].info().default_value(), - [10000001, -10000001]) + self.assertEqual( + desc_2[3].info().default_value(), [10000001, -10000001] + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_operator.py b/python/paddle/fluid/tests/unittests/test_operator.py index 62db058682bd9..0b7d9a4c1bb42 100644 --- a/python/paddle/fluid/tests/unittests/test_operator.py +++ b/python/paddle/fluid/tests/unittests/test_operator.py @@ -21,7 +21,6 @@ class TestGetAllProtos(unittest.TestCase): - def test_all(self): all_protos = op.get_all_op_protos() self.assertNotEqual(0, len(all_protos)) @@ -31,7 +30,6 @@ def test_all(self): class TestOpDescCreationMethod(unittest.TestCase): - def test_plain_input_output(self): op_proto = framework_pb2.OpProto() op_proto.type = "test" @@ -112,10 +110,9 @@ def test_multiple_input_plain_output(self): expected1.type = 'fc' self.assertEqual(expected1, generated1) - generated2 = method(X=['x1', 'x2', 'x3'], - b='b', - W=['w1', 'w2', 'w3'], - Y='y') + generated2 = method( + X=['x1', 'x2', 'x3'], b='b', W=['w1', 'w2', 'w3'], Y='y' + ) expected2 = framework_pb2.OpDesc() tmp = expected2.inputs.add() @@ -163,14 +160,16 @@ def __add_attr__(name, type): method = op.OpDescCreationMethod(op_proto) - generated = method(X="a", - int_attr=10, - float_attr=3.2, - float64_attr=np.finfo("float64").max, - string_attr="test_str", - ints_attr=[0, 1, 2, 3, 4], - floats_attr=[0.2, 3.2, 4.5], - strings_attr=["a", "b", "c"]) + generated = method( + X="a", + int_attr=10, + float_attr=3.2, + float64_attr=np.finfo("float64").max, + string_attr="test_str", + ints_attr=[0, 1, 2, 3, 4], + floats_attr=[0.2, 3.2, 4.5], + strings_attr=["a", "b", "c"], + ) expected = framework_pb2.OpDesc() expected.type = "test" @@ -218,13 +217,13 @@ def __add_attr__(name, type): class TestOpCreations(unittest.TestCase): - def test_all(self): add_op = op.Operator("sum", X=["a", "b"], Out="z") self.assertIsNotNone(add_op) # Invoke C++ DebugString() - self.assertEqual('Op(sum), inputs:{X[a, b]}, outputs:{Out[z]}.', - str(add_op)) + self.assertEqual( + 'Op(sum), inputs:{X[a, b]}, outputs:{Out[z]}.', str(add_op) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_operator_desc.py b/python/paddle/fluid/tests/unittests/test_operator_desc.py index eeb7dfe95db6a..af3eecec826c2 100644 --- a/python/paddle/fluid/tests/unittests/test_operator_desc.py +++ b/python/paddle/fluid/tests/unittests/test_operator_desc.py @@ -22,7 +22,6 @@ class TestOperator(unittest.TestCase): - def test_error_type(self): block = main_program._create_block() try: @@ -30,37 +29,34 @@ def test_error_type(self): self.assertFail() except ValueError as v_err: self.assertEqual( - str(v_err), - "`type` to initialized an Operator can not be None.") + str(v_err), "`type` to initialized an Operator can not be None." + ) try: block.append_op(type="no_such_op") self.assertFail() except ValueError as a_err: self.assertEqual( - str(a_err), "Operator \"no_such_op\" has not been registered.") + str(a_err), "Operator \"no_such_op\" has not been registered." + ) def test_op_desc_creation(self): program = Program() block = program.current_block() - mul_x = block.create_var(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x") - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - mul_op = block.append_op(type="mul", - inputs={ - "X": [mul_x], - "Y": mul_y - }, - outputs={"Out": [mul_out]}, - attrs={"x_num_col_dims": 1}) + mul_x = block.create_var( + dtype="float32", shape=[5, 10], lod_level=0, name="mul.x" + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + mul_op = block.append_op( + type="mul", + inputs={"X": [mul_x], "Y": mul_y}, + outputs={"Out": [mul_out]}, + attrs={"x_num_col_dims": 1}, + ) self.assertNotEqual(str(mul_op), "") self.assertEqual(mul_op.type, "mul") @@ -71,10 +67,19 @@ def test_op_desc_creation(self): self.assertEqual(mul_op.output("Out"), ["mul.out"]) self.assertEqual( set(mul_op.attr_names), - set([ - "x_num_col_dims", "y_num_col_dims", "op_role", "op_role_var", - "op_namescope", "op_callstack", "op_device", "with_quant_attr" - ])) + set( + [ + "x_num_col_dims", + "y_num_col_dims", + "op_role", + "op_role_var", + "op_namescope", + "op_callstack", + "op_device", + "with_quant_attr", + ] + ), + ) self.assertEqual(mul_op.has_attr("x_num_col_dims"), True) self.assertEqual(mul_op.attr_type("x_num_col_dims"), core.AttrType.INT) self.assertEqual(mul_op.attr("x_num_col_dims"), 1) @@ -89,25 +94,23 @@ def test_op_desc_creation(self): def test_mult_input(self): program = Program() block = program.current_block() - sum_x1 = block.create_var(dtype="int", - shape=[3, 4], - lod_level=0, - name="sum.x1") - sum_x2 = block.create_var(dtype="int", - shape=[3, 4], - lod_level=0, - name="sum.x2") - sum_x3 = block.create_var(dtype="int", - shape=[3, 4], - lod_level=0, - name="sum.x3") - sum_out = block.create_var(dtype="int", - shape=[3, 4], - lod_level=0, - name="sum.out") - sum_op = block.append_op(type="sum", - inputs={"X": [sum_x1, sum_x2, sum_x3]}, - outputs={"Out": sum_out}) + sum_x1 = block.create_var( + dtype="int", shape=[3, 4], lod_level=0, name="sum.x1" + ) + sum_x2 = block.create_var( + dtype="int", shape=[3, 4], lod_level=0, name="sum.x2" + ) + sum_x3 = block.create_var( + dtype="int", shape=[3, 4], lod_level=0, name="sum.x3" + ) + sum_out = block.create_var( + dtype="int", shape=[3, 4], lod_level=0, name="sum.out" + ) + sum_op = block.append_op( + type="sum", + inputs={"X": [sum_x1, sum_x2, sum_x3]}, + outputs={"Out": sum_out}, + ) self.assertEqual(sum_op.type, "sum") self.assertEqual(sum_op.input_names, ["X"]) self.assertEqual(sum_op.input("X"), ["sum.x1", "sum.x2", "sum.x3"]) diff --git a/python/paddle/fluid/tests/unittests/test_ops_nms.py b/python/paddle/fluid/tests/unittests/test_ops_nms.py index 4004f29b446a3..573231a8a725a 100644 --- a/python/paddle/fluid/tests/unittests/test_ops_nms.py +++ b/python/paddle/fluid/tests/unittests/test_ops_nms.py @@ -46,10 +46,12 @@ def multiclass_nms(boxes, scores, category_idxs, iou_threshold, top_k): cur_category_scores = scores[cur_category_boxes_idxs] cur_category_sorted_indices = np.argsort(-cur_category_scores) cur_category_sorted_boxes = cur_category_boxes[ - cur_category_sorted_indices] + cur_category_sorted_indices + ] - cur_category_keep_boxes_sub_idxs = cur_category_sorted_indices[nms( - cur_category_sorted_boxes, iou_threshold)] + cur_category_keep_boxes_sub_idxs = cur_category_sorted_indices[ + nms(cur_category_sorted_boxes, iou_threshold) + ] mask[cur_category_boxes_idxs[cur_category_keep_boxes_sub_idxs]] = True @@ -72,7 +74,6 @@ def gen_args(num_boxes, dtype): class TestOpsNMS(unittest.TestCase): - def setUp(self): self.num_boxes = 64 self.threshold = 0.5 @@ -91,78 +92,100 @@ def test_nms(self): for device in self.devices: for dtype in self.dtypes: boxes, scores, category_idxs, categories = gen_args( - self.num_boxes, dtype) + self.num_boxes, dtype + ) paddle.set_device(device) - out = paddle.vision.ops.nms(paddle.to_tensor(boxes), - self.threshold, - paddle.to_tensor(scores)) - out = paddle.vision.ops.nms(paddle.to_tensor(boxes), - self.threshold) + out = paddle.vision.ops.nms( + paddle.to_tensor(boxes), + self.threshold, + paddle.to_tensor(scores), + ) + out = paddle.vision.ops.nms( + paddle.to_tensor(boxes), self.threshold + ) out_py = nms(boxes, self.threshold) np.testing.assert_array_equal( out.numpy(), out_py, - err_msg='paddle out: {}\n py out: {}\n'.format(out, out_py)) + err_msg='paddle out: {}\n py out: {}\n'.format(out, out_py), + ) def test_multiclass_nms_dynamic(self): for device in self.devices: for dtype in self.dtypes: boxes, scores, category_idxs, categories = gen_args( - self.num_boxes, dtype) + self.num_boxes, dtype + ) paddle.set_device(device) - out = paddle.vision.ops.nms(paddle.to_tensor(boxes), - self.threshold, - paddle.to_tensor(scores), - paddle.to_tensor(category_idxs), - categories, self.topk) - out_py = multiclass_nms(boxes, scores, category_idxs, - self.threshold, self.topk) + out = paddle.vision.ops.nms( + paddle.to_tensor(boxes), + self.threshold, + paddle.to_tensor(scores), + paddle.to_tensor(category_idxs), + categories, + self.topk, + ) + out_py = multiclass_nms( + boxes, scores, category_idxs, self.threshold, self.topk + ) np.testing.assert_array_equal( out.numpy(), out_py, - err_msg='paddle out: {}\n py out: {}\n'.format(out, out_py)) + err_msg='paddle out: {}\n py out: {}\n'.format(out, out_py), + ) def test_multiclass_nms_static(self): for device in self.devices: for dtype in self.dtypes: paddle.enable_static() boxes, scores, category_idxs, categories = gen_args( - self.num_boxes, dtype) - boxes_static = paddle.static.data(shape=boxes.shape, - dtype=boxes.dtype, - name="boxes") - scores_static = paddle.static.data(shape=scores.shape, - dtype=scores.dtype, - name="scores") + self.num_boxes, dtype + ) + boxes_static = paddle.static.data( + shape=boxes.shape, dtype=boxes.dtype, name="boxes" + ) + scores_static = paddle.static.data( + shape=scores.shape, dtype=scores.dtype, name="scores" + ) category_idxs_static = paddle.static.data( shape=category_idxs.shape, dtype=category_idxs.dtype, - name="category_idxs") - out = paddle.vision.ops.nms(boxes_static, self.threshold, - scores_static, category_idxs_static, - categories, self.topk) + name="category_idxs", + ) + out = paddle.vision.ops.nms( + boxes_static, + self.threshold, + scores_static, + category_idxs_static, + categories, + self.topk, + ) place = paddle.CPUPlace() if device == 'gpu': place = paddle.CUDAPlace(0) exe = paddle.static.Executor(place) - out = exe.run(paddle.static.default_main_program(), - feed={ - 'boxes': boxes, - 'scores': scores, - 'category_idxs': category_idxs - }, - fetch_list=[out]) + out = exe.run( + paddle.static.default_main_program(), + feed={ + 'boxes': boxes, + 'scores': scores, + 'category_idxs': category_idxs, + }, + fetch_list=[out], + ) paddle.disable_static() - out_py = multiclass_nms(boxes, scores, category_idxs, - self.threshold, self.topk) + out_py = multiclass_nms( + boxes, scores, category_idxs, self.threshold, self.topk + ) out = np.array(out) out = np.squeeze(out) np.testing.assert_array_equal( out, out_py, - err_msg='paddle out: {}\n py out: {}\n'.format(out, out_py)) + err_msg='paddle out: {}\n py out: {}\n'.format(out, out_py), + ) def test_multiclass_nms_dynamic_to_static(self): for device in self.devices: @@ -173,10 +196,14 @@ def fun(x): scores = np.arange(0, 64).astype('float32') categories = np.array([0, 1, 2, 3]) category_idxs = categories.repeat(16) - out = paddle.vision.ops.nms(x, 0.1, - paddle.to_tensor(scores), - paddle.to_tensor(category_idxs), - categories, 10) + out = paddle.vision.ops.nms( + x, + 0.1, + paddle.to_tensor(scores), + paddle.to_tensor(category_idxs), + categories, + 10, + ) return out boxes = np.random.rand(64, 4).astype('float32') @@ -188,9 +215,9 @@ def fun(x): fun, self.path, input_spec=[ - paddle.static.InputSpec(shape=[None, 4], - dtype='float32', - name='x') + paddle.static.InputSpec( + shape=[None, 4], dtype='float32', name='x' + ) ], ) load_func = paddle.jit.load(self.path) @@ -199,20 +226,23 @@ def fun(x): origin, res, err_msg='origin out: {}\n inference model out: {}\n'.format( - origin, res)) + origin, res + ), + ) def test_matrix_nms_dynamic(self): for device in self.devices: for dtype in self.dtypes: boxes, scores, category_idxs, categories = gen_args( - self.num_boxes, dtype) + self.num_boxes, dtype + ) scores = np.random.rand(1, 4, self.num_boxes).astype(dtype) paddle.set_device(device) out = paddle.vision.ops.matrix_nms( paddle.to_tensor(boxes).unsqueeze(0), paddle.to_tensor(scores), self.threshold, - post_threshold=0., + post_threshold=0.0, nms_top_k=400, keep_top_k=100, ) diff --git a/python/paddle/fluid/tests/unittests/test_optimizer.py b/python/paddle/fluid/tests/unittests/test_optimizer.py index 47d42e354cadb..17ad33f67ab99 100644 --- a/python/paddle/fluid/tests/unittests/test_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_optimizer.py @@ -22,7 +22,11 @@ import paddle.fluid.core as core import numpy as np from paddle.fluid.backward import append_backward -from paddle.fluid.framework import Program, program_guard, convert_np_dtype_to_dtype_ +from paddle.fluid.framework import ( + Program, + program_guard, + convert_np_dtype_to_dtype_, +) from paddle.fluid.framework import _test_eager_guard import paddle from paddle.io import Dataset @@ -30,40 +34,36 @@ class TestOptimizer(unittest.TestCase): - def test_sgd_optimizer(self): - def check_sgd_optimizer(optimizer_attr): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr=optimizer_attr) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr=optimizer_attr, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.01) opts, _ = sgd_optimizer.minimize(mean_out, init_program) return opts @@ -78,40 +78,36 @@ def check_sgd_optimizer(optimizer_attr): class TestOptimizerBackwardApplygrad(unittest.TestCase): - def test_sgd_optimizer(self): - def check_sgd_optimizer(optimizer_attr): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr=optimizer_attr) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr=optimizer_attr, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.01) with framework.program_guard(program, init_program): p_g = sgd_optimizer.backward(mean_out) @@ -128,9 +124,7 @@ def check_sgd_optimizer(optimizer_attr): class TestMomentumOptimizer(unittest.TestCase): - class MockMomentum(optimizer.MomentumOptimizer): - def get_accumulators(self): return self._accumulators @@ -141,36 +135,35 @@ def test_vanilla_momentum_optimizer(self): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) learning_rate = 0.01 - momentum_optimizer = self.MockMomentum(learning_rate=learning_rate, - momentum=0.2) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + momentum_optimizer = self.MockMomentum( + learning_rate=learning_rate, momentum=0.2 + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(momentum_optimizer.get_accumulators()), 0) @@ -201,37 +194,35 @@ def test_nesterov_momentum_optimizer(self): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) learning_rate = 0.01 - momentum_optimizer = self.MockMomentum(learning_rate=learning_rate, - momentum=0.2, - use_nesterov=True) + momentum_optimizer = self.MockMomentum( + learning_rate=learning_rate, momentum=0.2, use_nesterov=True + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(momentum_optimizer.get_accumulators()), 0) @@ -260,9 +251,7 @@ def test_nesterov_momentum_optimizer(self): class TestAdagradOptimizer(unittest.TestCase): - class MockAdagrad(optimizer.AdagradOptimizer): - def get_accumulators(self): return self._accumulators @@ -273,36 +262,35 @@ def test_adagrad_optimizer(self): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) learning_rate = 0.01 - adagrad_optimizer = self.MockAdagrad(learning_rate=learning_rate, - epsilon=1.0e-6) + adagrad_optimizer = self.MockAdagrad( + learning_rate=learning_rate, epsilon=1.0e-6 + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(adagrad_optimizer.get_accumulators()), 0) @@ -329,9 +317,7 @@ def test_adagrad_optimizer(self): class TestAdamOptimizer(unittest.TestCase): - class MockAdam(optimizer.AdamOptimizer): - def get_accumulators(self): return self._accumulators @@ -345,37 +331,35 @@ def test_adam_optimizer(self): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) learning_rate = 0.01 - adam_optimizer = self.MockAdam(learning_rate=learning_rate, - beta1=0.9, - beta2=0.999) + adam_optimizer = self.MockAdam( + learning_rate=learning_rate, beta1=0.9, beta2=0.999 + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(adam_optimizer.get_accumulators()), 0) @@ -404,9 +388,7 @@ def test_adam_optimizer(self): class TestAdamaxOptimizer(unittest.TestCase): - class MockAdamax(optimizer.AdamaxOptimizer): - def get_accumulators(self): return self._accumulators @@ -420,37 +402,35 @@ def test_adamax_optimizer(self): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) learning_rate = 0.01 - adamax_optimizer = self.MockAdamax(learning_rate=learning_rate, - beta1=0.9, - beta2=0.999) + adamax_optimizer = self.MockAdamax( + learning_rate=learning_rate, beta1=0.9, beta2=0.999 + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(adamax_optimizer.get_accumulators()), 0) @@ -479,61 +459,56 @@ def test_adamax_optimizer(self): class TestDpsgdOptimizer(unittest.TestCase): - def test_dpsgd_optimizer(self): - def check_dpsgd_optimizer(optimizer_attr): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr=optimizer_attr) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) - dpsgd_optimizer = optimizer.DpsgdOptimizer(learning_rate=0.01, - clip=100.0, - batch_size=16.0, - sigma=0.0) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr=optimizer_attr, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) + dpsgd_optimizer = optimizer.DpsgdOptimizer( + learning_rate=0.01, clip=100.0, batch_size=16.0, sigma=0.0 + ) opts, _ = dpsgd_optimizer.minimize(mean_out, init_program) return opts - opts = check_dpsgd_optimizer({ - 'learning_rate': 1.1, - 'clip': 100.0, - 'batch_size': 16.0, - 'sigma': 4.0 - }) + opts = check_dpsgd_optimizer( + { + 'learning_rate': 1.1, + 'clip': 100.0, + 'batch_size': 16.0, + 'sigma': 4.0, + } + ) self.assertEqual(len(opts), 2) self.assertEqual([op.type for op in opts], ["scale", "dpsgd"]) class TestDecayedAdagradOptimizer(unittest.TestCase): - class MockDecayedAdagrad(optimizer.DecayedAdagradOptimizer): - def get_accumulators(self): return self._accumulators @@ -544,36 +519,35 @@ def test_decayed_adagrad_optimizer(self): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) learning_rate = 0.01 decayed_adagrad_optimizer = self.MockDecayedAdagrad( - learning_rate=learning_rate, decay=0.95, epsilon=1.0e-6) + learning_rate=learning_rate, decay=0.95, epsilon=1.0e-6 + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(decayed_adagrad_optimizer.get_accumulators()), 0) @@ -586,7 +560,8 @@ def test_decayed_adagrad_optimizer(self): accumulators = decayed_adagrad_optimizer.get_accumulators() self.assertEqual(len(accumulators), 1) self.assertTrue( - decayed_adagrad_optimizer.get_moment_str() in accumulators) + decayed_adagrad_optimizer.get_moment_str() in accumulators + ) moment_acc = accumulators[decayed_adagrad_optimizer.get_moment_str()] self.assertEqual(len(moment_acc), 1) self.assertTrue(mul_x.name in moment_acc) @@ -601,9 +576,7 @@ def test_decayed_adagrad_optimizer(self): class TestFtrlOptimizer(unittest.TestCase): - class MockFtrl(optimizer.FtrlOptimizer): - def get_accumulators(self): return self._accumulators @@ -617,38 +590,35 @@ def test_ftrl_optimizer(self): init_program = framework.Program() program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) learning_rate = 0.01 - ftrl_optimizer = self.MockFtrl(learning_rate=learning_rate, - l1=0.0, - l2=0.0, - lr_power=-0.5) + ftrl_optimizer = self.MockFtrl( + learning_rate=learning_rate, l1=0.0, l2=0.0, lr_power=-0.5 + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(ftrl_optimizer.get_accumulators()), 0) @@ -677,44 +647,40 @@ def test_ftrl_optimizer(self): class TestLookaheadOptimizer(unittest.TestCase): - def test_lookahead_optimizer(self): init_program = framework.Program() program = framework.Program() block = program.global_block() init_block = init_program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr={'learning_rate': 1.1}) - init_mul_x = init_block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x") - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr={'learning_rate': 1.1}, + ) + init_mul_x = init_block.create_parameter( + dtype="float32", shape=[5, 10], lod_level=0, name="mul.x" + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) sgd = optimizer.SGD(learning_rate=0.01) lookahead = optimizer.LookaheadOptimizer(sgd, alpha=0.5, k=5) @@ -725,109 +691,98 @@ def test_lookahead_optimizer(self): class TestRecomputeOptimizer(unittest.TestCase): - def net(self, return_input=False, with_dropout=False, with_seed=False): program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x") - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") + mul_x = block.create_parameter( + dtype="float32", shape=[5, 10], lod_level=0, name="mul.x" + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) if with_dropout is True: - mul_out_drop = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out.dropout") - mul_out_mask = block.create_var(dtype="uint8", - shape=[5, 8], - lod_level=0, - name="mul.out.mask") + mul_out_drop = block.create_var( + dtype="float32", + shape=[5, 8], + lod_level=0, + name="mul.out.dropout", + ) + mul_out_mask = block.create_var( + dtype="uint8", shape=[5, 8], lod_level=0, name="mul.out.mask" + ) if with_seed is True: - seed_out = block.create_var(dtype="int32", - shape=[1], - name="seed.out") - - b1 = block.create_parameter(dtype="float32", - shape=[5, 8], - lod_level=0, - name="b1") - b1_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="b1_out") - b2 = block.create_parameter(dtype="float32", - shape=[5, 8], - lod_level=0, - name="b2") - b2_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="b2_out") - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) + seed_out = block.create_var( + dtype="int32", shape=[1], name="seed.out" + ) + + b1 = block.create_parameter( + dtype="float32", shape=[5, 8], lod_level=0, name="b1" + ) + b1_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="b1_out" + ) + b2 = block.create_parameter( + dtype="float32", shape=[5, 8], lod_level=0, name="b2" + ) + b2_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="b2_out" + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) if with_dropout is True: dropout_inputs = {'X': [mul_out]} if with_seed is True: - block.append_op(type='seed', - outputs={'Out': seed_out}, - attrs={ - 'deterministic': True, - 'rng_name': 'rng0', - 'force_cpu': True - }) + block.append_op( + type='seed', + outputs={'Out': seed_out}, + attrs={ + 'deterministic': True, + 'rng_name': 'rng0', + 'force_cpu': True, + }, + ) dropout_inputs = {'X': [mul_out], 'Seed': [seed_out]} - block.append_op(type='dropout', - inputs=dropout_inputs, - outputs={ - 'Out': [mul_out_drop], - 'Mask': [mul_out_mask] - }, - attrs={ - 'dropout_prob': 0.5, - }) - block.append_op(type="elementwise_add", - inputs={ - "X": mul_out_drop, - "Y": b1 - }, - outputs={"Out": b1_out}) + block.append_op( + type='dropout', + inputs=dropout_inputs, + outputs={'Out': [mul_out_drop], 'Mask': [mul_out_mask]}, + attrs={ + 'dropout_prob': 0.5, + }, + ) + block.append_op( + type="elementwise_add", + inputs={"X": mul_out_drop, "Y": b1}, + outputs={"Out": b1_out}, + ) else: - block.append_op(type="elementwise_add", - inputs={ - "X": mul_out, - "Y": b1 - }, - outputs={"Out": b1_out}) - - block.append_op(type="elementwise_add", - inputs={ - "X": b1_out, - "Y": b2 - }, - outputs={"Out": b2_out}) - block.append_op(type="mean", - inputs={"X": b2_out}, - outputs={"Out": mean_out}) + block.append_op( + type="elementwise_add", + inputs={"X": mul_out, "Y": b1}, + outputs={"Out": b1_out}, + ) + + block.append_op( + type="elementwise_add", + inputs={"X": b1_out, "Y": b2}, + outputs={"Out": b2_out}, + ) + block.append_op( + type="mean", inputs={"X": b2_out}, outputs={"Out": mean_out} + ) if return_input == True: return mul_x, mul_out, b1_out, b2_out, mean_out @@ -836,124 +791,225 @@ def net(self, return_input=False, with_dropout=False, with_seed=False): def test_no_checkpoint(self): mul_out, b1_out, b2_out, mean_out = self.net() self.assertEqual(len(mean_out.block.ops), 4) - self.assertEqual([op.type for op in mean_out.block.ops], - ["mul", "elementwise_add", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + ["mul", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 12) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "elementwise_add_grad", - "elementwise_add_grad", "mul_grad", "sgd", "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add_grad", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_one_checkpoint(self): mul_out, b1_out, b2_out, mean_out = self.net() self.assertEqual(len(mean_out.block.ops), 4) - self.assertEqual([op.type for op in mean_out.block.ops], - ["mul", "elementwise_add", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + ["mul", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([b1_out]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 13) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "elementwise_add_grad", "mul", - "elementwise_add_grad", "mul_grad", "sgd", "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add_grad", + "mul", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_str_checkpoints(self): mul_out, b1_out, b2_out, mean_out = self.net() self.assertEqual(len(mean_out.block.ops), 4) - self.assertEqual([op.type for op in mean_out.block.ops], - ["mul", "elementwise_add", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + ["mul", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([b1_out.name]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 13) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "elementwise_add_grad", "mul", - "elementwise_add_grad", "mul_grad", "sgd", "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add_grad", + "mul", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_multi_checkpoint(self): mul_out, b1_out, b2_out, mean_out = self.net() self.assertEqual(len(mean_out.block.ops), 4) - self.assertEqual([op.type for op in mean_out.block.ops], - ["mul", "elementwise_add", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + ["mul", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([mul_out, b2_out]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 13) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "elementwise_add", - "elementwise_add_grad", "elementwise_add_grad", "mul_grad", "sgd", - "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add", + "elementwise_add_grad", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_adjacent_checkpoint(self): mul_out, b1_out, b2_out, mean_out = self.net() self.assertEqual(len(mean_out.block.ops), 4) - self.assertEqual([op.type for op in mean_out.block.ops], - ["mul", "elementwise_add", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + ["mul", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([mul_out, b1_out]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 12) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "elementwise_add_grad", - "elementwise_add_grad", "mul_grad", "sgd", "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add_grad", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_out_of_order_checkpoint(self): mul_out, b1_out, b2_out, mean_out = self.net() self.assertEqual(len(mean_out.block.ops), 4) - self.assertEqual([op.type for op in mean_out.block.ops], - ["mul", "elementwise_add", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + ["mul", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([b2_out, mul_out]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 13) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "elementwise_add", - "elementwise_add_grad", "elementwise_add_grad", "mul_grad", "sgd", - "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add", + "elementwise_add_grad", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_input_as_checkpoints(self): mul_x, mul_out, b1_out, b2_out, mean_out = self.net(return_input=True) self.assertEqual(len(mean_out.block.ops), 4) - self.assertEqual([op.type for op in mean_out.block.ops], - ["mul", "elementwise_add", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + ["mul", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([mul_x, b2_out]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 14) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "mul", "elementwise_add", - "elementwise_add_grad", "elementwise_add_grad", "mul_grad", "sgd", - "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "mul", + "elementwise_add", + "elementwise_add_grad", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_apply_gradients(self): mul_out, b1_out, b2_out, mean_out = self.net() @@ -961,10 +1017,12 @@ def test_apply_gradients(self): recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([b1_out]) # apply backward - params_grads = recompute_optimizer.backward(mean_out, - startup_program=None, - parameter_list=None, - no_grad_set=None) + params_grads = recompute_optimizer.backward( + mean_out, + startup_program=None, + parameter_list=None, + no_grad_set=None, + ) # apply gradient program = mean_out.block.program @@ -972,11 +1030,24 @@ def test_apply_gradients(self): optimize_ops = recompute_optimizer.apply_gradients(params_grads) self.assertEqual(len(mean_out.block.ops), 13) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "elementwise_add", "elementwise_add", "mean", - "fill_constant", "mean_grad", "elementwise_add_grad", "mul", - "elementwise_add_grad", "mul_grad", "sgd", "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add_grad", + "mul", + "elementwise_add_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_load(self): mul_out, b1_out, b2_out, mean_out = self.net() @@ -989,7 +1060,8 @@ def test_load(self): except NotImplementedError as e: self.assertEqual( "load function is not supported by Recompute Optimizer for now", - str(e)) + str(e), + ) def test_dropout(self): """ @@ -1000,40 +1072,81 @@ def test_dropout(self): self.assertEqual(len(mean_out.block.ops), 5) self.assertEqual( [op.type for op in mean_out.block.ops], - ["mul", "dropout", "elementwise_add", "elementwise_add", "mean"]) + ["mul", "dropout", "elementwise_add", "elementwise_add", "mean"], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([b1_out]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 17) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "seed", "dropout", "elementwise_add", "elementwise_add", - "mean", "fill_constant", "mean_grad", "elementwise_add_grad", "mul", - "dropout", "elementwise_add_grad", "dropout_grad", "mul_grad", - "sgd", "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "seed", + "dropout", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add_grad", + "mul", + "dropout", + "elementwise_add_grad", + "dropout_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_dropout_with_determinate_seed(self): - mul_out, b1_out, b2_out, mean_out = self.net(with_dropout=True, - with_seed=True) + mul_out, b1_out, b2_out, mean_out = self.net( + with_dropout=True, with_seed=True + ) self.assertEqual(len(mean_out.block.ops), 6) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "seed", "dropout", "elementwise_add", "elementwise_add", - "mean" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "seed", + "dropout", + "elementwise_add", + "elementwise_add", + "mean", + ], + ) sgd_optimizer = optimizer.SGD(learning_rate=1.0) recompute_optimizer = optimizer.RecomputeOptimizer(sgd_optimizer) recompute_optimizer._set_checkpoints([b1_out]) opts, params_grads = recompute_optimizer.minimize(mean_out) self.assertEqual(len(mean_out.block.ops), 17) - self.assertEqual([op.type for op in mean_out.block.ops], [ - "mul", "seed", "dropout", "elementwise_add", "elementwise_add", - "mean", "fill_constant", "mean_grad", "elementwise_add_grad", "mul", - "dropout", "elementwise_add_grad", "dropout_grad", "mul_grad", - "sgd", "sgd", "sgd" - ]) + self.assertEqual( + [op.type for op in mean_out.block.ops], + [ + "mul", + "seed", + "dropout", + "elementwise_add", + "elementwise_add", + "mean", + "fill_constant", + "mean_grad", + "elementwise_add_grad", + "mul", + "dropout", + "elementwise_add_grad", + "dropout_grad", + "mul_grad", + "sgd", + "sgd", + "sgd", + ], + ) def test_dropout_with_seed(self): """ @@ -1044,16 +1157,16 @@ def test_dropout_with_seed(self): def gen_data(): return { "x": np.random.random(size=(100, 3)).astype('float32'), - "y": np.random.randint(2, size=(100, 1)).astype('int64') + "y": np.random.randint(2, size=(100, 1)).astype('int64'), } def mlp(input_x, input_y): - drop_res = fluid.layers.dropout(input_x, - dropout_prob=0.5, - name="dropout_with_seed_cpu") - prediction = fluid.layers.fc(input=[drop_res], - size=2, - act='softmax') + drop_res = fluid.layers.dropout( + input_x, dropout_prob=0.5, name="dropout_with_seed_cpu" + ) + prediction = fluid.layers.fc( + input=[drop_res], size=2, act='softmax' + ) cost = fluid.layers.cross_entropy(input=prediction, label=input_y) sum_cost = fluid.layers.reduce_mean(cost) return drop_res, prediction, sum_cost @@ -1063,9 +1176,9 @@ def mlp(input_x, input_y): scope = fluid.Scope() with fluid.scope_guard(scope): with program_guard(main_program, startup_program): - input_x = fluid.layers.data(name="x", - shape=[3], - dtype='float32') + input_x = fluid.layers.data( + name="x", shape=[3], dtype='float32' + ) input_y = fluid.layers.data(name="y", shape=[1], dtype='int64') drop_res, prediction, cost = mlp(input_x, input_y) sgd = fluid.optimizer.Adam(learning_rate=0.01) @@ -1077,19 +1190,21 @@ def mlp(input_x, input_y): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) feed_data = gen_data() - drop_vec = exe.run(feed=feed_data, - program=fluid.default_main_program(), - fetch_list=[ - "dropout_with_seed_cpu.tmp_1", - "dropout_with_seed_cpu.tmp_1.subprog_0" - ]) + drop_vec = exe.run( + feed=feed_data, + program=fluid.default_main_program(), + fetch_list=[ + "dropout_with_seed_cpu.tmp_1", + "dropout_with_seed_cpu.tmp_1.subprog_0", + ], + ) self.assertEqual(drop_vec[0].tolist(), drop_vec[1].tolist()) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestRecomputeOptimizerCUDA(unittest.TestCase): - def test_dropout_with_seed(self): """ when we recompute a dropout op, make sure that the recomputed one @@ -1099,16 +1214,16 @@ def test_dropout_with_seed(self): def gen_data(): return { "x": np.random.random(size=(100, 3)).astype('float32'), - "y": np.random.randint(2, size=(100, 1)).astype('int64') + "y": np.random.randint(2, size=(100, 1)).astype('int64'), } def mlp(input_x, input_y): - drop_res = fluid.layers.dropout(input_x, - dropout_prob=0.5, - name="dropout_with_seed_gpu") - prediction = fluid.layers.fc(input=[drop_res], - size=2, - act='softmax') + drop_res = fluid.layers.dropout( + input_x, dropout_prob=0.5, name="dropout_with_seed_gpu" + ) + prediction = fluid.layers.fc( + input=[drop_res], size=2, act='softmax' + ) cost = fluid.layers.cross_entropy(input=prediction, label=input_y) sum_cost = fluid.layers.reduce_mean(cost) return drop_res, prediction, sum_cost @@ -1118,9 +1233,9 @@ def mlp(input_x, input_y): scope = fluid.Scope() with fluid.scope_guard(scope): with program_guard(main_program, startup_program): - input_x = fluid.layers.data(name="x", - shape=[3], - dtype='float32') + input_x = fluid.layers.data( + name="x", shape=[3], dtype='float32' + ) input_y = fluid.layers.data(name="y", shape=[1], dtype='int64') drop_res, prediction, cost = mlp(input_x, input_y) sgd = fluid.optimizer.Adam(learning_rate=0.01) @@ -1132,70 +1247,67 @@ def mlp(input_x, input_y): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) feed_data = gen_data() - drop_vec = exe.run(feed=feed_data, - program=fluid.default_main_program(), - fetch_list=[ - "dropout_with_seed_gpu.tmp_1", - "dropout_with_seed_gpu.tmp_1.subprog_0" - ]) + drop_vec = exe.run( + feed=feed_data, + program=fluid.default_main_program(), + fetch_list=[ + "dropout_with_seed_gpu.tmp_1", + "dropout_with_seed_gpu.tmp_1.subprog_0", + ], + ) self.assertEqual(drop_vec[0].tolist(), drop_vec[1].tolist()) class TestGradientMergeOptimizer(unittest.TestCase): - def net(self): program = framework.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x") - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - b1 = block.create_parameter(dtype="float32", - shape=[5, 8], - lod_level=0, - name="b1") - b1_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="b1_out") - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - block.append_op(type="elementwise_add", - inputs={ - "X": mul_out, - "Y": b1 - }, - outputs={"Out": b1_out}) - block.append_op(type="mean", - inputs={"X": b1_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", shape=[5, 10], lod_level=0, name="mul.x" + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + b1 = block.create_parameter( + dtype="float32", shape=[5, 8], lod_level=0, name="b1" + ) + b1_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="b1_out" + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + block.append_op( + type="elementwise_add", + inputs={"X": mul_out, "Y": b1}, + outputs={"Out": b1_out}, + ) + block.append_op( + type="mean", inputs={"X": b1_out}, outputs={"Out": mean_out} + ) return mean_out - def test_program_desc(self, ): + def test_program_desc( + self, + ): cost = self.net() main_program = cost.block.program init_program = framework.Program() self.assertEqual(main_program.num_blocks, 1) self.assertEqual(len(cost.block.ops), 3) - self.assertEqual([op.type for op in cost.block.ops], - ["mul", "elementwise_add", "mean"]) + self.assertEqual( + [op.type for op in cost.block.ops], + ["mul", "elementwise_add", "mean"], + ) opt = optimizer.SGD(learning_rate=1.0) opt = optimizer.GradientMergeOptimizer(opt, k_steps=4) @@ -1222,13 +1334,15 @@ def test_program_desc(self, ): 'elementwise_add', 'elementwise_add', 'conditional_block', - ]) + ], + ) # optimize block self.assertEqual(len(main_program.block(1).ops), 6) self.assertEqual( [op.type for op in main_program.block(1).ops], - ['scale', 'scale', 'sgd', 'sgd', 'fill_constant', 'fill_constant']) + ['scale', 'scale', 'sgd', 'sgd', 'fill_constant', 'fill_constant'], + ) class TestOptimizerDtype(unittest.TestCase): @@ -1238,9 +1352,7 @@ class TestOptimizerDtype(unittest.TestCase): ''' def check_with_dtype(self, dtype): - class MyLayer(paddle.nn.Layer): - def __init__(self, dtype): super(MyLayer, self).__init__() self._w = self.create_parameter([2, 3], dtype=dtype) @@ -1287,12 +1399,14 @@ def check_with_opt_state_dict(self, use_save_load=True): numpy.random.seed(100) class SimpleNet(paddle.nn.Layer): - def __init__(self, input_size, output_size): super(SimpleNet, self).__init__() - self.linears = paddle.nn.LayerList([ - paddle.nn.Linear(input_size, output_size) for i in range(1) - ]) + self.linears = paddle.nn.LayerList( + [ + paddle.nn.Linear(input_size, output_size) + for i in range(1) + ] + ) def forward(self, x): for i, l in enumerate(self.linears): @@ -1305,7 +1419,6 @@ def forward(self, x): nums_batch = 10 class RandomDataset(Dataset): - def __init__(self, num_samples): self.num_samples = num_samples @@ -1318,17 +1431,21 @@ def __len__(self): return self.num_samples dataset = RandomDataset(nums_batch * batch_size) - loader = paddle.io.DataLoader(dataset, - batch_size=batch_size, - shuffle=False, - drop_last=True, - num_workers=0) + loader = paddle.io.DataLoader( + dataset, + batch_size=batch_size, + shuffle=False, + drop_last=True, + num_workers=0, + ) mse = paddle.nn.MSELoss() model = SimpleNet(input_size, output_size) # 定义模型 - optimizer = paddle.optimizer.Momentum(learning_rate=0.0001, - parameters=model.parameters(), - multi_precision=True) # 定义优化器 + optimizer = paddle.optimizer.Momentum( + learning_rate=0.0001, + parameters=model.parameters(), + multi_precision=True, + ) # 定义优化器 scaler = paddle.amp.GradScaler(init_loss_scaling=1024) model = paddle.amp.decorate(models=model, level='O2') @@ -1356,9 +1473,11 @@ def test_with_state_dict(self): if core.is_compiled_with_cuda(): with fluid.dygraph.guard(): out_use_state_dict = self.check_with_opt_state_dict( - use_save_load=True) + use_save_load=True + ) out_no_state_dict = self.check_with_opt_state_dict( - use_save_load=False) + use_save_load=False + ) np.testing.assert_array_equal(out_use_state_dict, out_no_state_dict) diff --git a/python/paddle/fluid/tests/unittests/test_optimizer_for_varbase.py b/python/paddle/fluid/tests/unittests/test_optimizer_for_varbase.py index 06cdc599f6d00..fd73c042cb2da 100644 --- a/python/paddle/fluid/tests/unittests/test_optimizer_for_varbase.py +++ b/python/paddle/fluid/tests/unittests/test_optimizer_for_varbase.py @@ -21,7 +21,6 @@ class TestOptimizerForVarBase(unittest.TestCase): - def setUp(self): self.lr = 0.01 @@ -32,16 +31,16 @@ def run_optimizer_step_with_varbase_list_input(self, optimizer): z = x + y - opt = optimizer(learning_rate=self.lr, - parameters=[x], - weight_decay=0.01) + opt = optimizer( + learning_rate=self.lr, parameters=[x], weight_decay=0.01 + ) z.backward() opt.step() - np.testing.assert_allclose(x.numpy(), - np.full([2, 3], -self.lr), - rtol=1e-05) + np.testing.assert_allclose( + x.numpy(), np.full([2, 3], -self.lr), rtol=1e-05 + ) def run_optimizer_minimize_with_varbase_list_input(self, optimizer): x = paddle.zeros([2, 3]) @@ -55,9 +54,9 @@ def run_optimizer_minimize_with_varbase_list_input(self, optimizer): z.backward() opt.minimize(z) - np.testing.assert_allclose(x.numpy(), - np.full([2, 3], -self.lr), - rtol=1e-05) + np.testing.assert_allclose( + x.numpy(), np.full([2, 3], -self.lr), rtol=1e-05 + ) def func_test_adam_with_varbase_list_input(self): self.run_optimizer_step_with_varbase_list_input(optimizer.Adam) @@ -130,17 +129,20 @@ def func_test_create_param_lr_with_1_for_coverage(self): shape=[5, 10], lod_level=0, name="x", - optimize_attr={'learning_rate': 1.0}) + optimize_attr={'learning_rate': 1.0}, + ) else: x = paddle.fluid.framework.EagerParamBase( dtype="float32", shape=[5, 10], lod_level=0, name="x", - optimize_attr={'learning_rate': 1.0}) + optimize_attr={'learning_rate': 1.0}, + ) x.value().get_tensor().set( np.random.random((5, 10)).astype('float32'), - paddle.fluid.framework._current_expected_place()) + paddle.fluid.framework._current_expected_place(), + ) y = paddle.ones([5, 10]) z = x + y @@ -160,17 +162,20 @@ def func_test_create_param_lr_with_no_1_value_for_coverage(self): shape=[5, 10], lod_level=0, name="x", - optimize_attr={'learning_rate': 0.12}) + optimize_attr={'learning_rate': 0.12}, + ) else: x = paddle.fluid.framework.EagerParamBase( dtype="float32", shape=[5, 10], lod_level=0, name="x", - optimize_attr={'learning_rate': 0.12}) + optimize_attr={'learning_rate': 0.12}, + ) x.value().get_tensor().set( np.random.random((5, 10)).astype('float32'), - paddle.fluid.framework._current_expected_place()) + paddle.fluid.framework._current_expected_place(), + ) y = paddle.ones([5, 10]) z = x + y diff --git a/python/paddle/fluid/tests/unittests/test_optimizer_grad.py b/python/paddle/fluid/tests/unittests/test_optimizer_grad.py index a29fb920d7884..87e39a1efb0c5 100644 --- a/python/paddle/fluid/tests/unittests/test_optimizer_grad.py +++ b/python/paddle/fluid/tests/unittests/test_optimizer_grad.py @@ -43,8 +43,8 @@ def __init__(self, test_optimizer, param_lr=1.0, y_no_grad=False): def _init_param(self): self.x = np.ones(self.shape).astype('float32') - self.y = np.ones(self.shape).astype('float32') * 2. - self.z = np.ones(self.shape).astype('float32') * 3. + self.y = np.ones(self.shape).astype('float32') * 2.0 + self.z = np.ones(self.shape).astype('float32') * 3.0 def _calc_gradient(self, cond_i): """ @@ -81,39 +81,42 @@ def build_net(self, cond_i, use_bf16=False): dtype="float32", shape=self.shape, attr=fluid.ParamAttr(learning_rate=self.param_lr, name="param_x"), - default_initializer=fluid.initializer.NumpyArrayInitializer(self.x)) + default_initializer=fluid.initializer.NumpyArrayInitializer(self.x), + ) param_y = fluid.layers.create_parameter( dtype="float32", shape=self.shape, attr=fluid.ParamAttr(learning_rate=self.param_lr, name="param_y"), - default_initializer=fluid.initializer.NumpyArrayInitializer(self.y)) + default_initializer=fluid.initializer.NumpyArrayInitializer(self.y), + ) param_z = fluid.layers.create_parameter( dtype="float32", shape=self.shape, attr=fluid.ParamAttr(learning_rate=self.param_lr, name="param_z"), - default_initializer=fluid.initializer.NumpyArrayInitializer(self.z)) + default_initializer=fluid.initializer.NumpyArrayInitializer(self.z), + ) sum_xy = fluid.layers.elementwise_add(param_x, param_y, name='sum_xy') sub_yz = fluid.layers.elementwise_sub(param_y, param_z, name='sub_yz') useless = fluid.layers.fc(param_x, size=1, name='fc_useless') def cond_true(): - cond_yz = fluid.layers.elementwise_add(param_y, - param_z, - name='sum_cond_yz') + cond_yz = fluid.layers.elementwise_add( + param_y, param_z, name='sum_cond_yz' + ) # param_y will not be updated param_y.stop_gradient = self.y_no_grad - cond_res = fluid.layers.elementwise_add(cond_yz, - param_z, - name='sum_cond_true') + cond_res = fluid.layers.elementwise_add( + cond_yz, param_z, name='sum_cond_true' + ) cond_useless = fluid.layers.elementwise_mul(param_x, param_y) return cond_res def cond_false(): - cond_res = fluid.layers.elementwise_add(param_y, - param_z, - name='sum_cond_false') + cond_res = fluid.layers.elementwise_add( + param_y, param_z, name='sum_cond_false' + ) cond_useless = fluid.layers.elementwise_mul(param_z, param_z) return cond_res @@ -123,18 +126,23 @@ def cond_false(): mean_out = paddle.mean(sum_all) if use_bf16: import paddle.static.amp as amp + self.optimizer = amp.bf16.decorate_bf16( self.optimizer, amp_lists=amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_list={'elementwise_add'}), + custom_fp32_list={'elementwise_add'} + ), use_bf16_guard=False, - use_pure_bf16=True) + use_pure_bf16=True, + ) self.optimizer.minimize(mean_out) - fetch_list = ["param_x", "param_z"] if self.y_no_grad else [ - "param_x", "param_y", "param_z" - ] + fetch_list = ( + ["param_x", "param_z"] + if self.y_no_grad + else ["param_x", "param_y", "param_z"] + ) fetch_list += [_append_grad_suffix_(param) for param in fetch_list] return fetch_list, self.optimizer @@ -200,8 +208,9 @@ def _check_grads(self, use_bf16=False): for param_lr in self.param_lr: for cond_i in self.cond_i: for y_no_grad in self.y_no_grad: - self.attr[ - 'lr'] = param_lr * self.optimizer._learning_rate + self.attr['lr'] = ( + param_lr * self.optimizer._learning_rate + ) self._init_param_attr() main_program = fluid.Program() @@ -209,11 +218,15 @@ def _check_grads(self, use_bf16=False): with fluid.program_guard(main_program, init_program): # reset optimizer._accumulators to avoid duplicate name in loop. self.optimizer._accumulators = defaultdict( - lambda: dict()) - test_net = self.NetClass(self.optimizer, param_lr, - y_no_grad) - fetch_list, decorated_optimizer = test_net.build_net( - cond_i, use_bf16) + lambda: dict() + ) + test_net = self.NetClass( + self.optimizer, param_lr, y_no_grad + ) + ( + fetch_list, + decorated_optimizer, + ) = test_net.build_net(cond_i, use_bf16) if use_bf16: self.optimizer = decorated_optimizer @@ -225,24 +238,28 @@ def _check_grads(self, use_bf16=False): # Train 2 steps to check validity for batch_i in range(2): - res = exe.run(main_program, - fetch_list=fetch_list) + res = exe.run( + main_program, fetch_list=fetch_list + ) gt_grads = test_net._calc_gradient(cond_i) gt_params = self._apply_optimize( - test_net, gt_grads) + test_net, gt_grads + ) param_grads = gt_params + gt_grads for i in range(len(res)): np.testing.assert_allclose( - res[i], param_grads[i]) + res[i], param_grads[i] + ) -@unittest.skipIf(not fluid.core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not fluid.core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestSGDOptimizer(TestOptimizer): - def test_optimizer_multiblock_except(self): - with self.assertRaisesRegexp(ValueError, - "var param_y not in this block"): + with self.assertRaisesRegexp( + ValueError, "var param_y not in this block" + ): self._check_grads(use_bf16=True) @@ -256,10 +273,9 @@ class TestAdamOptimizer(TestOptimizer): def setUp(self): self._init_config() beta1, beta2, epsilon = 0.9, 0.999, 1e-8 - self.optimizer = optimizer.AdamOptimizer(learning_rate=0.01, - beta1=beta1, - beta2=beta2, - epsilon=epsilon) + self.optimizer = optimizer.AdamOptimizer( + learning_rate=0.01, beta1=beta1, beta2=beta2, epsilon=epsilon + ) self.attr = { "beta1": beta1, "beta2": beta2, @@ -267,7 +283,7 @@ def setUp(self): "beta2_pow": beta2, "moment1": np.zeros(SHAPE).astype("float32"), "moment2": np.zeros(SHAPE).astype("float32"), - "epsilon": epsilon + "epsilon": epsilon, } def _apply_gradient(self, param, grad, name): @@ -280,13 +296,14 @@ def _apply_gradient(self, param, grad, name): beta1_pow, beta2_pow = attr['beta1_pow'], attr['beta2_pow'] epsilon = attr['epsilon'] - moment1_out = beta1 * moment1 + (1. - beta1) * grad - moment2_out = beta2 * moment2 + (1. - beta2) * np.square(grad) + moment1_out = beta1 * moment1 + (1.0 - beta1) * grad + moment2_out = beta2 * moment2 + (1.0 - beta2) * np.square(grad) - lr = attr['lr'] * np.sqrt(1. - beta2_pow) / (1. - beta1_pow) + lr = attr['lr'] * np.sqrt(1.0 - beta2_pow) / (1.0 - beta1_pow) param_out = param - lr * ( - moment1_out / - (np.sqrt(moment2_out) + epsilon * np.sqrt(1 - beta2_pow))) + moment1_out + / (np.sqrt(moment2_out) + epsilon * np.sqrt(1 - beta2_pow)) + ) # update hyper-parameter of optimizer self.param_attr[name]['beta1_pow'] = beta1_pow * beta1 diff --git a/python/paddle/fluid/tests/unittests/test_optimizer_in_control_flow.py b/python/paddle/fluid/tests/unittests/test_optimizer_in_control_flow.py index 6034833028430..8bc2b79675882 100644 --- a/python/paddle/fluid/tests/unittests/test_optimizer_in_control_flow.py +++ b/python/paddle/fluid/tests/unittests/test_optimizer_in_control_flow.py @@ -34,10 +34,9 @@ paddle.enable_static() -def static(train_data, - loss_in_switch=True, - use_cuda=False, - use_parallel_exe=False): +def static( + train_data, loss_in_switch=True, use_cuda=False, use_parallel_exe=False +): startup_program = Program() main_program = Program() startup_program.random_seed = SEED @@ -51,20 +50,26 @@ def double_fc_net(image): size=FC_SIZE, act='relu', param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.99)), + initializer=fluid.initializer.Constant(value=0.99) + ), bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.5)), - name="hidden") + initializer=fluid.initializer.Constant(value=0.5) + ), + name="hidden", + ) prediction = layers.fc( hidden, size=CLASS_NUM, act='softmax', param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.2)), + initializer=fluid.initializer.Constant(value=1.2) + ), bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.8)), - name="prediction") + initializer=fluid.initializer.Constant(value=0.8) + ), + name="prediction", + ) return hidden, prediction def fn_1(opt, avg_loss=None, pred=None, label=None): @@ -76,8 +81,9 @@ def fn_1(opt, avg_loss=None, pred=None, label=None): def fn_2(opt, avg_loss=None, pred=None, label=None): if avg_loss is None: - loss = layers.softmax_with_cross_entropy(logits=pred, - label=label) + loss = layers.softmax_with_cross_entropy( + logits=pred, label=label + ) avg_loss = paddle.mean(loss, name='mean_softmax_loss') opt.minimize(avg_loss) return avg_loss @@ -96,15 +102,19 @@ def fn_2(opt, avg_loss=None, pred=None, label=None): if loss_in_switch: avg_loss = layers.case( [(mod_two, lambda: fn_1(adam, None, prediction, label))], - lambda: fn_2(sgd, None, prediction, label)) + lambda: fn_2(sgd, None, prediction, label), + ) else: loss_1 = layers.cross_entropy(input=prediction, label=label) avg_loss_1 = paddle.mean(loss_1) - loss_2 = layers.softmax_with_cross_entropy(logits=prediction, - label=label) + loss_2 = layers.softmax_with_cross_entropy( + logits=prediction, label=label + ) avg_loss_2 = paddle.mean(loss_2) - avg_loss = layers.case([(mod_two, lambda: fn_1(adam, avg_loss_1))], - lambda: fn_2(sgd, avg_loss_2)) + avg_loss = layers.case( + [(mod_two, lambda: fn_1(adam, avg_loss_1))], + lambda: fn_2(sgd, avg_loss_2), + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) @@ -116,7 +126,7 @@ def fn_2(opt, avg_loss=None, pred=None, label=None): feed = { 'image': feed_image, 'label': feed_label, - 'id': np.array([epoch]).astype('int32') + 'id': np.array([epoch]).astype('int32'), } out = exe.run(main_program, feed=feed, fetch_list=fetch_list) out_hidden, out_pred, loss = out @@ -125,27 +135,31 @@ def fn_2(opt, avg_loss=None, pred=None, label=None): class DygraphLayer(fluid.dygraph.Layer): - def __init__(self): super(DygraphLayer, self).__init__() self.fc_1 = fluid.dygraph.nn.Linear( INPUT_SIZE, FC_SIZE, act='relu', - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.99)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.5)), + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.99) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.5) + ), ) self.fc_2 = fluid.dygraph.nn.Linear( FC_SIZE, CLASS_NUM, act='softmax', - param_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.2)), - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.8))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.2) + ), + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.8) + ), + ) def forward(self, inputs): hidden = self.fc_1(inputs) @@ -159,10 +173,12 @@ def dynamic(train_data, use_cuda=False, use_parallel_exe=False): fluid.default_startup_program().random_seed = SEED fluid.default_main_program().random_seed = SEED dy_layer = DygraphLayer() - adam = fluid.optimizer.Adam(learning_rate=LR, - parameter_list=dy_layer.parameters()) - sgd = fluid.optimizer.SGD(learning_rate=LR, - parameter_list=dy_layer.parameters()) + adam = fluid.optimizer.Adam( + learning_rate=LR, parameter_list=dy_layer.parameters() + ) + sgd = fluid.optimizer.SGD( + learning_rate=LR, parameter_list=dy_layer.parameters() + ) for epoch in range(EPOCH_NUM): image_data, label = train_data[epoch] @@ -177,7 +193,8 @@ def dynamic(train_data, use_cuda=False, use_parallel_exe=False): adam.minimize(loss) else: softmax_loss = layers.softmax_with_cross_entropy( - prediction, var_label) + prediction, var_label + ) loss = paddle.mean(softmax_loss) loss.backward() sgd.minimize(loss) @@ -192,16 +209,18 @@ class TestMultiTask(unittest.TestCase): Todo(liym27): add parallel GPU train. ''' - def random_input(self, - seed, - image_shape=[BATCH_SIZE, INPUT_SIZE], - label_shape=[BATCH_SIZE, 1]): + def random_input( + self, + seed, + image_shape=[BATCH_SIZE, INPUT_SIZE], + label_shape=[BATCH_SIZE, 1], + ): np.random.seed(seed) image_np = np.random.random(size=image_shape).astype('float32') np.random.seed(seed) - label_np = np.random.randint(low=0, - high=CLASS_NUM - 1, - size=label_shape).astype('int64') + label_np = np.random.randint( + low=0, high=CLASS_NUM - 1, size=label_shape + ).astype('int64') return image_np, label_np def init_train_data(self): @@ -214,15 +233,15 @@ def test_optimzier_in_switch(self): use_cuda = core.is_compiled_with_cuda() hidden_2, pre_2, loss_2 = dynamic(self.train_data, use_cuda) for loss_in_switch in [True, False]: - hidden_1, pre_1, loss_1 = static(self.train_data, loss_in_switch, - use_cuda) + hidden_1, pre_1, loss_1 = static( + self.train_data, loss_in_switch, use_cuda + ) np.testing.assert_allclose(hidden_1, hidden_2, rtol=1e-05) np.testing.assert_allclose(pre_1, pre_2, rtol=1e-05) np.testing.assert_allclose(loss_1, loss_2, rtol=1e-05) class TestMultiOptimizersMultiCardsError(unittest.TestCase): - def test_error(self): startup_program = Program() main_program = Program() @@ -244,8 +263,10 @@ def fn_2(opt, avg_loss): cond = layers.fill_constant([1], 'bool', True) - layers.case([(cond, lambda: fn_1(adam, avg_loss))], - lambda: fn_2(sgd, avg_loss)) + layers.case( + [(cond, lambda: fn_1(adam, avg_loss))], + lambda: fn_2(sgd, avg_loss), + ) cpu_place = fluid.CPUPlace() cuda_place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() @@ -263,17 +284,20 @@ def fn_2(opt, avg_loss): # to use multi cards ** only on CPU ** not GPU to reduce CI time. os.environ['CPU_NUM'] = str(2) - pe_exe = fluid.ParallelExecutor(use_cuda=use_cuda, - main_program=main_program, - loss_name=avg_loss.name) + pe_exe = fluid.ParallelExecutor( + use_cuda=use_cuda, + main_program=main_program, + loss_name=avg_loss.name, + ) num_devices = pe_exe.device_count def not_implemented_error(): - pe_exe.run(feed={ - 'X': - np.random.random(size=[64, 10]).astype('float32'), - }, - fetch_list=[avg_loss.name]) + pe_exe.run( + feed={ + 'X': np.random.random(size=[64, 10]).astype('float32'), + }, + fetch_list=[avg_loss.name], + ) if num_devices > 1: self.assertRaises(NotImplementedError, not_implemented_error) diff --git a/python/paddle/fluid/tests/unittests/test_outer.py b/python/paddle/fluid/tests/unittests/test_outer.py index a887615f0f306..5625618776c29 100644 --- a/python/paddle/fluid/tests/unittests/test_outer.py +++ b/python/paddle/fluid/tests/unittests/test_outer.py @@ -22,27 +22,28 @@ class TestMultiplyApi(unittest.TestCase): - def _run_static_graph_case(self, x_data, y_data): with program_guard(Program(), Program()): paddle.enable_static() - x = paddle.static.data(name='x', - shape=x_data.shape, - dtype=x_data.dtype) - y = paddle.static.data(name='y', - shape=y_data.shape, - dtype=y_data.dtype) + x = paddle.static.data( + name='x', shape=x_data.shape, dtype=x_data.dtype + ) + y = paddle.static.data( + name='y', shape=y_data.shape, dtype=y_data.dtype + ) res = paddle.outer(x, y) - place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) - outs = exe.run(paddle.static.default_main_program(), - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[res]) + outs = exe.run( + paddle.static.default_main_program(), + feed={'x': x_data, 'y': y_data}, + fetch_list=[res], + ) res = outs[0] return res @@ -94,17 +95,21 @@ def func_test_multiply(self): # test dynamic computation graph: 2-d array Complex x_data = np.random.rand(20, 50).astype( - np.float64) + 1J * np.random.rand(20, 50).astype(np.float64) - y_data = np.random.rand(50).astype( - np.float64) + 1J * np.random.rand(50).astype(np.float64) + np.float64 + ) + 1j * np.random.rand(20, 50).astype(np.float64) + y_data = np.random.rand(50).astype(np.float64) + 1j * np.random.rand( + 50 + ).astype(np.float64) res = self._run_dynamic_graph_case(x_data, y_data) np.testing.assert_allclose(res, np.outer(x_data, y_data), rtol=1e-05) # test dynamic computation graph: 3-d array Complex x_data = np.random.rand(5, 10, 10).astype( - np.float64) + 1J * np.random.rand(5, 10, 10).astype(np.float64) - y_data = np.random.rand(2, 10).astype( - np.float64) + 1J * np.random.rand(2, 10).astype(np.float64) + np.float64 + ) + 1j * np.random.rand(5, 10, 10).astype(np.float64) + y_data = np.random.rand(2, 10).astype(np.float64) + 1j * np.random.rand( + 2, 10 + ).astype(np.float64) res = self._run_dynamic_graph_case(x_data, y_data) np.testing.assert_allclose(res, np.outer(x_data, y_data), rtol=1e-05) @@ -115,7 +120,6 @@ def test_multiply(self): class TestMultiplyError(unittest.TestCase): - def func_test_errors(self): # test static computation graph: dtype can not be int8 paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_overlap_add_op.py b/python/paddle/fluid/tests/unittests/test_overlap_add_op.py index f3815ba496566..430fde53aceed 100644 --- a/python/paddle/fluid/tests/unittests/test_overlap_add_op.py +++ b/python/paddle/fluid/tests/unittests/test_overlap_add_op.py @@ -33,8 +33,9 @@ def overlap_add(x, hop_length, axis=-1): frame_length = x.shape[1] if axis == 0 else x.shape[-2] # Assure no gaps between frames. - assert 0 < hop_length <= frame_length, \ - f'hop_length should be in (0, frame_length({frame_length})], but got {hop_length}.' + assert ( + 0 < hop_length <= frame_length + ), f'hop_length should be in (0, frame_length({frame_length})], but got {hop_length}.' seq_length = (n_frames - 1) * hop_length + frame_length @@ -55,7 +56,7 @@ def overlap_add(x, hop_length, axis=-1): for i in range(x.shape[0]): for frame in range(x.shape[-1]): sample = frame * hop_length - y[i, sample:sample + frame_length] += x[i, :, frame] + y[i, sample : sample + frame_length] += x[i, :, frame] if axis == 0: y = y.transpose((1, 0)) @@ -70,7 +71,6 @@ def overlap_add(x, hop_length, axis=-1): class TestOverlapAddOp(OpTest): - def setUp(self): self.op_type = "overlap_add" self.python_api = paddle.signal.overlap_add @@ -101,7 +101,6 @@ def test_check_grad_normal(self): class TestCase1(TestOverlapAddOp): - def initTestCase(self): input_shape = (3, 50) input_type = 'float64' @@ -113,7 +112,6 @@ def initTestCase(self): class TestCase2(TestOverlapAddOp): - def initTestCase(self): input_shape = (2, 40, 5) input_type = 'float64' @@ -125,7 +123,6 @@ def initTestCase(self): class TestCase3(TestOverlapAddOp): - def initTestCase(self): input_shape = (5, 40, 2) input_type = 'float64' @@ -137,7 +134,6 @@ def initTestCase(self): class TestCase4(TestOverlapAddOp): - def initTestCase(self): input_shape = (3, 5, 12, 8) input_type = 'float64' @@ -149,7 +145,6 @@ def initTestCase(self): class TestCase5(TestOverlapAddOp): - def initTestCase(self): input_shape = (8, 12, 5, 3) input_type = 'float64' diff --git a/python/paddle/fluid/tests/unittests/test_pad2d_op.py b/python/paddle/fluid/tests/unittests/test_pad2d_op.py index 0f43ddbd8fc82..aeca9becf84b1 100644 --- a/python/paddle/fluid/tests/unittests/test_pad2d_op.py +++ b/python/paddle/fluid/tests/unittests/test_pad2d_op.py @@ -20,7 +20,6 @@ class TestPad2dOp(OpTest): - def setUp(self): self.pad_value = 0.0 self.variable_paddings = False @@ -30,25 +29,37 @@ def setUp(self): self.attrs = {} if self.variable_paddings: self.attrs['paddings'] = [] - self.inputs['Paddings'] = np.array( - self.paddings).flatten().astype("int32") + self.inputs['Paddings'] = ( + np.array(self.paddings).flatten().astype("int32") + ) else: - self.attrs['paddings'] = np.array( - self.paddings).flatten().astype("int32") + self.attrs['paddings'] = ( + np.array(self.paddings).flatten().astype("int32") + ) self.attrs['pad_value'] = self.pad_value self.attrs['mode'] = self.mode self.attrs['data_format'] = self.data_format if self.data_format == "NCHW": - paddings = [(0, 0), (0, 0), (self.paddings[0], self.paddings[1]), - (self.paddings[2], self.paddings[3])] + paddings = [ + (0, 0), + (0, 0), + (self.paddings[0], self.paddings[1]), + (self.paddings[2], self.paddings[3]), + ] else: - paddings = [(0, 0), (self.paddings[0], self.paddings[1]), - (self.paddings[2], self.paddings[3]), (0, 0)] + paddings = [ + (0, 0), + (self.paddings[0], self.paddings[1]), + (self.paddings[2], self.paddings[3]), + (0, 0), + ] if self.mode == "constant": - out = np.pad(self.inputs['X'], - paddings, - mode=self.mode, - constant_values=self.pad_value) + out = np.pad( + self.inputs['X'], + paddings, + mode=self.mode, + constant_values=self.pad_value, + ) else: out = np.pad(self.inputs['X'], paddings, mode=self.mode) self.outputs = {'Out': out} @@ -68,7 +79,6 @@ def initTestCase(self): class TestCase1(TestPad2dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.paddings = [0, 1, 2, 3] @@ -77,7 +87,6 @@ def initTestCase(self): class TestCase2(TestPad2dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.paddings = [0, 1, 2, 3] @@ -86,7 +95,6 @@ def initTestCase(self): class TestCase3(TestPad2dOp): - def initTestCase(self): self.shape = (2, 4, 4, 4) self.paddings = [0, 1, 2, 3] @@ -95,7 +103,6 @@ def initTestCase(self): class TestCase4(TestPad2dOp): - def initTestCase(self): self.shape = (2, 4, 4, 4) self.paddings = [0, 1, 2, 3] @@ -104,7 +111,6 @@ def initTestCase(self): class TestCase5(TestPad2dOp): - def initTestCase(self): self.shape = (2, 4, 4, 4) self.paddings = [0, 1, 2, 3] @@ -114,7 +120,6 @@ def initTestCase(self): class TestCase6(TestPad2dOp): - def initTestCase(self): self.shape = (2, 4, 4, 4) self.paddings = [0, 1, 2, 3] @@ -125,7 +130,6 @@ def initTestCase(self): class TestCase7(TestPad2dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.paddings = [0, 1, 2, 3] @@ -135,7 +139,6 @@ def initTestCase(self): class TestPad2dOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.random((2, 2, 2, 2)).astype("float32") @@ -145,9 +148,9 @@ def test_Variable(): self.assertRaises(TypeError, test_Variable) - data = fluid.data(name='data', - shape=[None, 3, 20, 20], - dtype='float16') + data = fluid.data( + name='data', shape=[None, 3, 20, 20], dtype='float16' + ) fluid.layers.pad2d(input=data, paddings=[1, 1, 1, 1]) diff --git a/python/paddle/fluid/tests/unittests/test_pad3d_op.py b/python/paddle/fluid/tests/unittests/test_pad3d_op.py index c1c0aa7defb46..ee4a9aed67073 100644 --- a/python/paddle/fluid/tests/unittests/test_pad3d_op.py +++ b/python/paddle/fluid/tests/unittests/test_pad3d_op.py @@ -24,7 +24,6 @@ class TestPad3dOp(OpTest): - def setUp(self): paddle.enable_static() self.value = 0.0 @@ -35,11 +34,13 @@ def setUp(self): self.attrs = {} if self.variable_paddings: self.attrs['paddings'] = [] - self.inputs['Paddings'] = np.array( - self.paddings).flatten().astype("int32") + self.inputs['Paddings'] = ( + np.array(self.paddings).flatten().astype("int32") + ) else: - self.attrs['paddings'] = np.array( - self.paddings).flatten().astype("int32") + self.attrs['paddings'] = ( + np.array(self.paddings).flatten().astype("int32") + ) self.attrs['value'] = self.value self.attrs['mode'] = self.mode self.attrs['data_format'] = self.data_format @@ -60,10 +61,12 @@ def setUp(self): (0, 0), ] if self.mode == "constant": - out = np.pad(self.inputs['X'], - paddings, - mode=self.mode, - constant_values=self.value) + out = np.pad( + self.inputs['X'], + paddings, + mode=self.mode, + constant_values=self.value, + ) elif self.mode == "reflect": out = np.pad(self.inputs['X'], paddings, mode=self.mode) elif self.mode == "replicate": @@ -88,7 +91,6 @@ def initTestCase(self): class TestCase1(TestPad3dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.paddings = [0, 1, 2, 3, 4, 5] @@ -99,7 +101,6 @@ def initTestCase(self): class TestCase2(TestPad3dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.paddings = [1, 1, 1, 1, 1, 1] @@ -110,7 +111,6 @@ def initTestCase(self): class TestCase3(TestPad3dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.paddings = [0, 1, 1, 0, 2, 3] @@ -120,7 +120,6 @@ def initTestCase(self): class TestCase4(TestPad3dOp): - def initTestCase(self): self.shape = (4, 4, 4, 4, 4) self.paddings = [0, 1, 2, 1, 2, 3] @@ -130,7 +129,6 @@ def initTestCase(self): class TestCase5(TestPad3dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.paddings = [0, 1, 2, 3, 2, 1] @@ -140,7 +138,6 @@ def initTestCase(self): class TestCase6(TestPad3dOp): - def initTestCase(self): self.shape = (4, 4, 4, 4, 4) self.paddings = [5, 4, 2, 1, 2, 3] @@ -150,7 +147,6 @@ def initTestCase(self): class TestCase7(TestPad3dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.paddings = [0, 1, 2, 3, 2, 1] @@ -160,7 +156,6 @@ def initTestCase(self): class TestCase8(TestPad3dOp): - def initTestCase(self): self.shape = (4, 4, 4, 4, 4) self.paddings = [0, 1, 2, 1, 2, 3] @@ -170,7 +165,6 @@ def initTestCase(self): class TestCase9(TestPad3dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.paddings = [0, 1, 2, 3, 4, 5] @@ -181,7 +175,6 @@ def initTestCase(self): class TestCase10(TestPad3dOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.paddings = [0, 1, 2, 3, 4, 5] @@ -192,7 +185,6 @@ def initTestCase(self): class TestPadAPI(unittest.TestCase): - def setUp(self): self.places = [paddle.CPUPlace()] if core.is_compiled_with_cuda(): @@ -207,15 +199,15 @@ def check_static_result_1(self, place): value = 100 input_data = np.random.rand(*input_shape).astype(np.float32) x = paddle.fluid.data(name="x", shape=input_shape) - result = F.pad(x=x, - pad=pad, - value=value, - mode=mode, - data_format="NCDHW") + result = F.pad( + x=x, pad=pad, value=value, mode=mode, data_format="NCDHW" + ) exe = Executor(place) - fetches = exe.run(default_main_program(), - feed={"x": input_data}, - fetch_list=[result]) + fetches = exe.run( + default_main_program(), + feed={"x": input_data}, + fetch_list=[result], + ) np_out = self._get_numpy_out(input_data, pad, mode, value) np.testing.assert_allclose(fetches[0], np_out, rtol=1e-05) @@ -231,18 +223,18 @@ def check_static_result_2(self, place): result1 = F.pad(x=x, pad=pad, mode=mode, data_format="NCDHW") result2 = F.pad(x=x, pad=pad, mode=mode, data_format="NDHWC") exe = Executor(place) - fetches = exe.run(default_main_program(), - feed={"x": input_data}, - fetch_list=[result1, result2]) - - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - data_format="NCDHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - data_format="NDHWC") + fetches = exe.run( + default_main_program(), + feed={"x": input_data}, + fetch_list=[result1, result2], + ) + + np_out1 = self._get_numpy_out( + input_data, pad, mode, data_format="NCDHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, data_format="NDHWC" + ) np.testing.assert_allclose(fetches[0], np_out1, rtol=1e-05) np.testing.assert_allclose(fetches[1], np_out2, rtol=1e-05) @@ -257,18 +249,18 @@ def check_static_result_3(self, place): result1 = F.pad(x=x, pad=pad, mode=mode, data_format="NCDHW") result2 = F.pad(x=x, pad=pad, mode=mode, data_format="NDHWC") exe = Executor(place) - fetches = exe.run(default_main_program(), - feed={"x": input_data}, - fetch_list=[result1, result2]) - - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - data_format="NCDHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - data_format="NDHWC") + fetches = exe.run( + default_main_program(), + feed={"x": input_data}, + fetch_list=[result1, result2], + ) + + np_out1 = self._get_numpy_out( + input_data, pad, mode, data_format="NCDHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, data_format="NDHWC" + ) np.testing.assert_allclose(fetches[0], np_out1, rtol=1e-05) np.testing.assert_allclose(fetches[1], np_out2, rtol=1e-05) @@ -283,27 +275,24 @@ def check_static_result_4(self, place): result1 = F.pad(x=x, pad=pad, mode=mode, data_format="NCDHW") result2 = F.pad(x=x, pad=pad, mode=mode, data_format="NDHWC") exe = Executor(place) - fetches = exe.run(default_main_program(), - feed={"x": input_data}, - fetch_list=[result1, result2]) - - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - data_format="NCDHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - data_format="NDHWC") + fetches = exe.run( + default_main_program(), + feed={"x": input_data}, + fetch_list=[result1, result2], + ) + + np_out1 = self._get_numpy_out( + input_data, pad, mode, data_format="NCDHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, data_format="NDHWC" + ) np.testing.assert_allclose(fetches[0], np_out1, rtol=1e-05) np.testing.assert_allclose(fetches[1], np_out2, rtol=1e-05) - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0, - data_format="NCDHW"): + def _get_numpy_out( + self, input_data, pad, mode, value=0, data_format="NCDHW" + ): if mode == "constant" and len(pad) == len(input_data.shape) * 2: pad = np.reshape(pad, (-1, 2)).tolist() elif data_format == "NCDHW": @@ -375,38 +364,26 @@ def test_dygraph_1(self): mode = "constant" value = 100 input_data = np.random.rand(*input_shape).astype(np.float32) - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NCDHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NDHWC") - np_out3 = self._get_numpy_out(input_data, - pad_3, - mode, - value, - data_format="NCDHW") + np_out1 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NCDHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NDHWC" + ) + np_out3 = self._get_numpy_out( + input_data, pad_3, mode, value, data_format="NCDHW" + ) tensor_data = paddle.to_tensor(input_data) - y1 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NCDHW") - y2 = F.pad(tensor_data, - pad=pad, - mode=mode, - value=value, - data_format="NDHWC") - y3 = F.pad(tensor_data, - pad=pad_3, - mode=mode, - value=value, - data_format="NCDHW") + y1 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NCDHW" + ) + y2 = F.pad( + tensor_data, pad=pad, mode=mode, value=value, data_format="NDHWC" + ) + y3 = F.pad( + tensor_data, pad=pad_3, mode=mode, value=value, data_format="NCDHW" + ) np.testing.assert_allclose(y1.numpy(), np_out1, rtol=1e-05) np.testing.assert_allclose(y2.numpy(), np_out2, rtol=1e-05) @@ -420,40 +397,36 @@ def test_dygraph_2(self): mode = "constant" value = 100 input_data = np.random.rand(*input_shape).astype(np.float32) - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NCHW") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NHWC") - np_out3 = self._get_numpy_out(input_data, - pad_3, - mode, - value, - data_format="NCHW") + np_out1 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NCHW" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NHWC" + ) + np_out3 = self._get_numpy_out( + input_data, pad_3, mode, value, data_format="NCHW" + ) tensor_data = paddle.to_tensor(input_data) tensor_pad = paddle.to_tensor(pad, dtype="int32") - y1 = F.pad(tensor_data, - pad=tensor_pad, - mode=mode, - value=value, - data_format="NCHW") - y2 = F.pad(tensor_data, - pad=tensor_pad, - mode=mode, - value=value, - data_format="NHWC") - y3 = F.pad(tensor_data, - pad=pad_3, - mode=mode, - value=value, - data_format="NCHW") + y1 = F.pad( + tensor_data, + pad=tensor_pad, + mode=mode, + value=value, + data_format="NCHW", + ) + y2 = F.pad( + tensor_data, + pad=tensor_pad, + mode=mode, + value=value, + data_format="NHWC", + ) + y3 = F.pad( + tensor_data, pad=pad_3, mode=mode, value=value, data_format="NCHW" + ) np.testing.assert_allclose(y1.numpy(), np_out1, rtol=1e-05) np.testing.assert_allclose(y2.numpy(), np_out2, rtol=1e-05) @@ -467,39 +440,35 @@ def test_dygraph_3(self): mode = "constant" value = 100 input_data = np.random.rand(*input_shape).astype(np.float32) - np_out1 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NCL") - np_out2 = self._get_numpy_out(input_data, - pad, - mode, - value, - data_format="NLC") - np_out3 = self._get_numpy_out(input_data, - pad_3, - mode, - value, - data_format="NCL") + np_out1 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NCL" + ) + np_out2 = self._get_numpy_out( + input_data, pad, mode, value, data_format="NLC" + ) + np_out3 = self._get_numpy_out( + input_data, pad_3, mode, value, data_format="NCL" + ) tensor_data = paddle.to_tensor(input_data) tensor_pad = paddle.to_tensor(pad, dtype="int32") - y1 = F.pad(tensor_data, - pad=tensor_pad, - mode=mode, - value=value, - data_format="NCL") - y2 = F.pad(tensor_data, - pad=tensor_pad, - mode=mode, - value=value, - data_format="NLC") - y3 = F.pad(tensor_data, - pad=pad_3, - mode=mode, - value=value, - data_format="NCL") + y1 = F.pad( + tensor_data, + pad=tensor_pad, + mode=mode, + value=value, + data_format="NCL", + ) + y2 = F.pad( + tensor_data, + pad=tensor_pad, + mode=mode, + value=value, + data_format="NLC", + ) + y3 = F.pad( + tensor_data, pad=pad_3, mode=mode, value=value, data_format="NCL" + ) np.testing.assert_allclose(y1.numpy(), np_out1, rtol=1e-05) np.testing.assert_allclose(y2.numpy(), np_out2, rtol=1e-05) @@ -507,13 +476,9 @@ def test_dygraph_3(self): class TestPad1dAPI(unittest.TestCase): - - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0.0, - data_format="NCL"): + def _get_numpy_out( + self, input_data, pad, mode, value=0.0, data_format="NCL" + ): if data_format == "NCL": pad = [ (0, 0), @@ -555,59 +520,52 @@ def test_class(self): pad_reflection = nn.Pad1D(padding=pad, mode="reflect") pad_replication = nn.Pad1D(padding=pad, mode="replicate") pad_constant = nn.Pad1D(padding=pad, mode="constant", value=value) - pad_constant_int = nn.Pad1D(padding=pad_int, - mode="constant", - value=value) + pad_constant_int = nn.Pad1D( + padding=pad_int, mode="constant", value=value + ) pad_circular = nn.Pad1D(padding=pad, mode="circular") data = paddle.to_tensor(input_data) output = pad_reflection(data) - np_out = self._get_numpy_out(input_data, - pad, - "reflect", - data_format="NCL") + np_out = self._get_numpy_out( + input_data, pad, "reflect", data_format="NCL" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_replication(data) - np_out = self._get_numpy_out(input_data, - pad, - "replicate", - data_format="NCL") + np_out = self._get_numpy_out( + input_data, pad, "replicate", data_format="NCL" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_constant(data) - np_out = self._get_numpy_out(input_data, - pad, - "constant", - value=value, - data_format="NCL") + np_out = self._get_numpy_out( + input_data, pad, "constant", value=value, data_format="NCL" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_constant_int(data) - np_out = self._get_numpy_out(input_data, [pad_int] * 2, - "constant", - value=value, - data_format="NCL") + np_out = self._get_numpy_out( + input_data, + [pad_int] * 2, + "constant", + value=value, + data_format="NCL", + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_circular(data) - np_out = self._get_numpy_out(input_data, - pad, - "circular", - value=value, - data_format="NCL") + np_out = self._get_numpy_out( + input_data, pad, "circular", value=value, data_format="NCL" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) class TestPad2dAPI(unittest.TestCase): - - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0.0, - data_format="NCHW"): + def _get_numpy_out( + self, input_data, pad, mode, value=0.0, data_format="NCHW" + ): if data_format == "NCHW": pad = [ (0, 0), @@ -651,58 +609,52 @@ def test_class(self): pad_reflection = nn.Pad2D(padding=pad, mode="reflect") pad_replication = nn.Pad2D(padding=pad, mode="replicate") pad_constant = nn.Pad2D(padding=pad, mode="constant", value=value) - pad_constant_int = nn.Pad2D(padding=pad_int, - mode="constant", - value=value) + pad_constant_int = nn.Pad2D( + padding=pad_int, mode="constant", value=value + ) pad_circular = nn.Pad2D(padding=pad, mode="circular") data = paddle.to_tensor(input_data) output = pad_reflection(data) - np_out = self._get_numpy_out(input_data, - pad, - "reflect", - data_format="NCHW") + np_out = self._get_numpy_out( + input_data, pad, "reflect", data_format="NCHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_replication(data) - np_out = self._get_numpy_out(input_data, - pad, - "replicate", - data_format="NCHW") + np_out = self._get_numpy_out( + input_data, pad, "replicate", data_format="NCHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_constant(data) - np_out = self._get_numpy_out(input_data, - pad, - "constant", - value=value, - data_format="NCHW") + np_out = self._get_numpy_out( + input_data, pad, "constant", value=value, data_format="NCHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_constant_int(data) - np_out = self._get_numpy_out(input_data, [pad_int] * 4, - "constant", - value=value, - data_format="NCHW") + np_out = self._get_numpy_out( + input_data, + [pad_int] * 4, + "constant", + value=value, + data_format="NCHW", + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_circular(data) - np_out = self._get_numpy_out(input_data, - pad, - "circular", - data_format="NCHW") + np_out = self._get_numpy_out( + input_data, pad, "circular", data_format="NCHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) class TestPad3dAPI(unittest.TestCase): - - def _get_numpy_out(self, - input_data, - pad, - mode, - value=0.0, - data_format="NCDHW"): + def _get_numpy_out( + self, input_data, pad, mode, value=0.0, data_format="NCDHW" + ): if data_format == "NCDHW": pad = [ (0, 0), @@ -748,47 +700,45 @@ def test_class(self): pad_reflection = nn.Pad3D(padding=pad, mode="reflect") pad_replication = nn.Pad3D(padding=pad, mode="replicate") pad_constant = nn.Pad3D(padding=pad, mode="constant", value=value) - pad_constant_int = nn.Pad3D(padding=pad_int, - mode="constant", - value=value) + pad_constant_int = nn.Pad3D( + padding=pad_int, mode="constant", value=value + ) pad_circular = nn.Pad3D(padding=pad, mode="circular") data = paddle.to_tensor(input_data) output = pad_reflection(data) - np_out = self._get_numpy_out(input_data, - pad, - "reflect", - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, pad, "reflect", data_format="NCDHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_replication(data) - np_out = self._get_numpy_out(input_data, - pad, - "replicate", - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, pad, "replicate", data_format="NCDHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_constant(data) - np_out = self._get_numpy_out(input_data, - pad, - "constant", - value=value, - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, pad, "constant", value=value, data_format="NCDHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_constant_int(data) - np_out = self._get_numpy_out(input_data, [pad_int] * 6, - "constant", - value=value, - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, + [pad_int] * 6, + "constant", + value=value, + data_format="NCDHW", + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_circular(data) - np_out = self._get_numpy_out(input_data, - pad, - "circular", - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, pad, "circular", data_format="NCDHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) def test_pad_tensor(self): @@ -799,38 +749,34 @@ def test_pad_tensor(self): pad_tensor = paddle.to_tensor(pad) input_data = np.random.rand(*input_shape).astype(np.float32) - pad_reflection_ncdhw = nn.Pad3D(padding=pad_tensor, - mode="reflect", - data_format="NCDHW") - pad_reflection_ndhwc = nn.Pad3D(padding=pad_tensor, - mode="reflect", - data_format="NDHWC") + pad_reflection_ncdhw = nn.Pad3D( + padding=pad_tensor, mode="reflect", data_format="NCDHW" + ) + pad_reflection_ndhwc = nn.Pad3D( + padding=pad_tensor, mode="reflect", data_format="NDHWC" + ) data = paddle.to_tensor(input_data) output = pad_reflection_ncdhw(data) - np_out = self._get_numpy_out(input_data, - pad, - "reflect", - data_format="NCDHW") + np_out = self._get_numpy_out( + input_data, pad, "reflect", data_format="NCDHW" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) output = pad_reflection_ndhwc(data) - np_out = self._get_numpy_out(input_data, - pad, - "reflect", - data_format="NDHWC") + np_out = self._get_numpy_out( + input_data, pad, "reflect", data_format="NDHWC" + ) np.testing.assert_allclose(output.numpy(), np_out, rtol=1e-05) class TestPad3dOpError(unittest.TestCase): - def setUp(self): self.places = [paddle.CPUPlace()] if core.is_compiled_with_cuda(): self.places.append(paddle.CUDAPlace(0)) def test_errors(self): - def test_variable(): input_shape = (1, 2, 3, 4, 5) data = np.random.rand(*input_shape).astype(np.float32) @@ -840,49 +786,53 @@ def test_reflect_1(): input_shape = (1, 2, 3, 4, 5) data = np.random.rand(*input_shape).astype(np.float32) x = paddle.to_tensor(data) - y = F.pad(x, - pad=[5, 6, 1, 1, 1, 1], - value=1, - mode='reflect', - data_format="NCDHW") + y = F.pad( + x, + pad=[5, 6, 1, 1, 1, 1], + value=1, + mode='reflect', + data_format="NCDHW", + ) def test_reflect_2(): input_shape = (1, 2, 3, 4, 5) data = np.random.rand(*input_shape).astype(np.float32) x = paddle.to_tensor(data) - y = F.pad(x, - pad=[1, 1, 4, 3, 1, 1], - value=1, - mode='reflect', - data_format="NCDHW") + y = F.pad( + x, + pad=[1, 1, 4, 3, 1, 1], + value=1, + mode='reflect', + data_format="NCDHW", + ) def test_reflect_3(): input_shape = (1, 2, 3, 4, 5) data = np.random.rand(*input_shape).astype(np.float32) x = paddle.to_tensor(data) - y = F.pad(x, - pad=[1, 1, 1, 1, 2, 3], - value=1, - mode='reflect', - data_format="NCDHW") + y = F.pad( + x, + pad=[1, 1, 1, 1, 2, 3], + value=1, + mode='reflect', + data_format="NCDHW", + ) def test_circular_1(): input_shape = (1, 2, 0, 4, 5) data = np.random.rand(*input_shape).astype(np.float32) x = paddle.to_tensor(data) - y = F.pad(x, - pad=[1, 1, 1, 1, 2, 3], - mode='circular', - data_format="NCDHW") + y = F.pad( + x, pad=[1, 1, 1, 1, 2, 3], mode='circular', data_format="NCDHW" + ) def test_replicate_1(): input_shape = (1, 2, 0, 4, 5) data = np.random.rand(*input_shape).astype(np.float32) x = paddle.to_tensor(data) - y = F.pad(x, - pad=[1, 1, 1, 1, 2, 3], - mode='replicate', - data_format="NCDHW") + y = F.pad( + x, pad=[1, 1, 1, 1, 2, 3], mode='replicate', data_format="NCDHW" + ) paddle.disable_static() for place in self.places: @@ -896,14 +846,16 @@ def test_replicate_1(): class TestPadDataformatError(unittest.TestCase): - def test_errors(self): - def test_ncl(): input_shape = (1, 2, 3, 4) pad = paddle.to_tensor(np.array([2, 1, 2, 1]).astype('int32')) - data = np.arange(np.prod(input_shape), - dtype=np.float64).reshape(input_shape) + 1 + data = ( + np.arange(np.prod(input_shape), dtype=np.float64).reshape( + input_shape + ) + + 1 + ) my_pad = nn.Pad1D(padding=pad, mode="replicate", data_format="NCL") data = paddle.to_tensor(data) result = my_pad(data) @@ -911,8 +863,12 @@ def test_ncl(): def test_nchw(): input_shape = (1, 2, 4) pad = paddle.to_tensor(np.array([2, 1, 2, 1]).astype('int32')) - data = np.arange(np.prod(input_shape), - dtype=np.float64).reshape(input_shape) + 1 + data = ( + np.arange(np.prod(input_shape), dtype=np.float64).reshape( + input_shape + ) + + 1 + ) my_pad = nn.Pad1D(padding=pad, mode="replicate", data_format="NCHW") data = paddle.to_tensor(data) result = my_pad(data) @@ -920,11 +876,15 @@ def test_nchw(): def test_ncdhw(): input_shape = (1, 2, 3, 4) pad = paddle.to_tensor(np.array([2, 1, 2, 1]).astype('int32')) - data = np.arange(np.prod(input_shape), - dtype=np.float64).reshape(input_shape) + 1 - my_pad = nn.Pad1D(padding=pad, - mode="replicate", - data_format="NCDHW") + data = ( + np.arange(np.prod(input_shape), dtype=np.float64).reshape( + input_shape + ) + + 1 + ) + my_pad = nn.Pad1D( + padding=pad, mode="replicate", data_format="NCDHW" + ) data = paddle.to_tensor(data) result = my_pad(data) diff --git a/python/paddle/fluid/tests/unittests/test_pad_constant_like.py b/python/paddle/fluid/tests/unittests/test_pad_constant_like.py index 42fad15c97f89..fda844eb09b5d 100644 --- a/python/paddle/fluid/tests/unittests/test_pad_constant_like.py +++ b/python/paddle/fluid/tests/unittests/test_pad_constant_like.py @@ -20,22 +20,22 @@ class TestPadConstantLikeOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = "pad_constant_like" self.inputs = { 'X': np.random.random(self.x_shape).astype("float64"), - 'Y': np.random.random(self.y_shape).astype("float64") + 'Y': np.random.random(self.y_shape).astype("float64"), } self.attrs = {} self.attrs['pad_value'] = self.pad_value self.outputs = { - 'Out': - np.pad(self.inputs['Y'], - self.paddings, - mode='constant', - constant_values=self.pad_value) + 'Out': np.pad( + self.inputs['Y'], + self.paddings, + mode='constant', + constant_values=self.pad_value, + ) } def test_check_output(self): @@ -52,7 +52,6 @@ def initTestCase(self): class TestCase1(TestPadConstantLikeOp): - def initTestCase(self): self.x_shape = (4, 3, 4, 5) self.y_shape = (2, 3, 4, 5) @@ -61,7 +60,6 @@ def initTestCase(self): class TestCase2(TestPadConstantLikeOp): - def initTestCase(self): self.x_shape = (4, 3, 4, 10) self.y_shape = (2, 3, 2, 10) @@ -70,38 +68,38 @@ def initTestCase(self): class TestPadConstantLikeOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x_data = np.random.random((2, 2, 2, 2)).astype("float32") y_data = np.random.random((2, 2, 2, 2)).astype("float32") def test_Variable_x(): - var_y = fluid.data(name="data_y", - shape=[2, 2, 2, 2], - dtype="float32") + var_y = fluid.data( + name="data_y", shape=[2, 2, 2, 2], dtype="float32" + ) fluid.layers.pad_constant_like(x=x_data, y=var_y) self.assertRaises(TypeError, test_Variable_x) def test_Variable_y(): - var_x = fluid.data(name="data_x", - shape=[2, 2, 2, 2], - dtype="float32") + var_x = fluid.data( + name="data_x", shape=[2, 2, 2, 2], dtype="float32" + ) fluid.layers.pad_constant_like(x=var_x, y=y_data) self.assertRaises(TypeError, test_Variable_y) class TestOutDtype(unittest.TestCase): - def test_dtype(self): api_fn = fluid.layers.pad_constant_like - check_out_dtype(api_fn, - in_specs=[([2, 3, 2, 3], 'float64'), ([1, 3, 1, 3], )], - expect_dtypes=['float32', 'float64', 'int32', 'int64'], - target_index=1, - pad_value=0.) + check_out_dtype( + api_fn, + in_specs=[([2, 3, 2, 3], 'float64'), ([1, 3, 1, 3],)], + expect_dtypes=['float32', 'float64', 'int32', 'int64'], + target_index=1, + pad_value=0.0, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_pad_op.py b/python/paddle/fluid/tests/unittests/test_pad_op.py index fa31836ecf3e6..916f0399d9f01 100644 --- a/python/paddle/fluid/tests/unittests/test_pad_op.py +++ b/python/paddle/fluid/tests/unittests/test_pad_op.py @@ -25,7 +25,6 @@ class TestPadOp(OpTest): - def setUp(self): self.initTestCase() self.dtype = self.get_dtype() @@ -37,11 +36,12 @@ def setUp(self): self.attrs['paddings'] = np.array(self.paddings).flatten() self.attrs['pad_value'] = self.pad_value self.outputs = { - 'Out': - np.pad(self.inputs['X'], - self.paddings, - mode='constant', - constant_values=self.pad_value) + 'Out': np.pad( + self.inputs['X'], + self.paddings, + mode='constant', + constant_values=self.pad_value, + ) } def get_dtype(self): @@ -60,7 +60,6 @@ def initTestCase(self): class TestCase1(TestPadOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.paddings = [(0, 1), (2, 3), (2, 1), (1, 1)] @@ -68,7 +67,6 @@ def initTestCase(self): class TestCase2(TestPadOp): - def initTestCase(self): self.shape = (5, 5, 5) self.paddings = [(0, 0), (0, 0), (1, 2)] @@ -76,22 +74,20 @@ def initTestCase(self): class TestCase3(TestPadOp): - def initTestCase(self): - self.shape = (100) + self.shape = 100 self.paddings = [(0, 1)] self.pad_value = 0.9 -#----------------Pad Fp16---------------- +# ----------------Pad Fp16---------------- def create_test_fp16(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestPadFp16(parent): - def get_dtype(self): return np.float16 @@ -110,7 +106,6 @@ def test_check_grad_normal(self): class TestPadOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.random((2, 2)).astype("float32") @@ -125,7 +120,6 @@ def test_Variable(): class TestPaddingValueTensor(UnittestBase): - def init_info(self): self.shapes = [[2, 4]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -148,15 +142,16 @@ def test_static(self): exe = paddle.static.Executor() exe.run(starup_prog) res = exe.run(fetch_list=[feat, out]) - gt = np.pad(res[0], [1, 1], 'constant', constant_values=[1., 1.]) + gt = np.pad(res[0], [1, 1], 'constant', constant_values=[1.0, 1.0]) np.testing.assert_allclose(res[1], gt) - paddle.static.save_inference_model(self.save_path, [x], [feat, out], - exe) + paddle.static.save_inference_model( + self.save_path, [x], [feat, out], exe + ) # Test for Inference Predictor infer_outs = self.infer_prog() - gt = np.pad(infer_outs[0], [1, 1], - 'constant', - constant_values=[1., 1.]) + gt = np.pad( + infer_outs[0], [1, 1], 'constant', constant_values=[1.0, 1.0] + ) np.testing.assert_allclose(infer_outs[1], gt) def path_prefix(self): @@ -167,22 +162,20 @@ def var_prefix(self): def call_func(self, x): padding_value = paddle.assign([1.0]) - out = paddle.nn.functional.pad(x, - pad=[1, 1, 1, 1], - value=padding_value, - mode='constant') + out = paddle.nn.functional.pad( + x, pad=[1, 1, 1, 1], value=padding_value, mode='constant' + ) return out class TestPaddingValueTensor2(TestPaddingValueTensor): - def call_func(self, x): padding_value = paddle.assign([1.0]) # test for int value tmp = paddle.fluid.layers.pad(x, paddings=[1, 1, 1, 1], pad_value=1) - out = paddle.fluid.layers.pad(x, - paddings=[1, 1, 1, 1], - pad_value=padding_value) + out = paddle.fluid.layers.pad( + x, paddings=[1, 1, 1, 1], pad_value=padding_value + ) return out diff --git a/python/paddle/fluid/tests/unittests/test_paddle_imperative_double_grad.py b/python/paddle/fluid/tests/unittests/test_paddle_imperative_double_grad.py index 99b797fbf9e4c..dd958243fade6 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_imperative_double_grad.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_imperative_double_grad.py @@ -22,7 +22,6 @@ def _dygraph_guard_(func): - def __impl__(*args, **kwargs): if paddle.in_dynamic_mode(): return func(*args, **kwargs) @@ -42,26 +41,29 @@ def random_var(size, low=-1, high=1, dtype='float32'): class TestDygraphDoubleGrad(TestCase): - def setUp(self): self.sort_sum_gradient = False self.shape = [5, 10] - def grad(self, - outputs, - inputs, - grad_outputs=None, - no_grad_vars=None, - retain_graph=None, - create_graph=False, - allow_unused=False): - return paddle.grad(outputs=outputs, - inputs=inputs, - grad_outputs=grad_outputs, - no_grad_vars=no_grad_vars, - retain_graph=retain_graph, - create_graph=create_graph, - allow_unused=allow_unused) + def grad( + self, + outputs, + inputs, + grad_outputs=None, + no_grad_vars=None, + retain_graph=None, + create_graph=False, + allow_unused=False, + ): + return paddle.grad( + outputs=outputs, + inputs=inputs, + grad_outputs=grad_outputs, + no_grad_vars=no_grad_vars, + retain_graph=retain_graph, + create_graph=create_graph, + allow_unused=allow_unused, + ) @dygraph_guard def func_exception(self): @@ -83,12 +85,16 @@ def func_exception(self): self.grad([random_var(shape)], [1]) with self.assertRaises(AssertionError): - self.grad([random_var(shape), random_var(shape)], - [random_var(shape)], [random_var(shape)]) + self.grad( + [random_var(shape), random_var(shape)], + [random_var(shape)], + [random_var(shape)], + ) with self.assertRaises(AssertionError): - self.grad([random_var(shape)], [random_var(shape)], - no_grad_vars=[1]) + self.grad( + [random_var(shape)], [random_var(shape)], no_grad_vars=[1] + ) with self.assertRaises(AssertionError): self.grad([random_var(shape)], [random_var(shape)], no_grad_vars=1) @@ -105,31 +111,33 @@ def func_simple_example(self): y = x + 1 for create_graph in [False, True]: - dx, = self.grad([x], [x], - create_graph=create_graph, - retain_graph=True) + (dx,) = self.grad( + [x], [x], create_graph=create_graph, retain_graph=True + ) self.assertEqual(dx.shape, x.shape) self.assertTrue(np.all(dx.numpy() == 1)) self.assertNotEqual(dx.stop_gradient, create_graph) - dx_mul_2, = self.grad([y, x], [x], - create_graph=create_graph, - retain_graph=True) + (dx_mul_2,) = self.grad( + [y, x], [x], create_graph=create_graph, retain_graph=True + ) self.assertEqual(dx_mul_2.shape, x.shape) self.assertTrue(np.all(dx_mul_2.numpy() == 2)) self.assertNotEqual(dx_mul_2.stop_gradient, create_graph) - none_grad, = self.grad([x], [y], - create_graph=create_graph, - allow_unused=True) + (none_grad,) = self.grad( + [x], [y], create_graph=create_graph, allow_unused=True + ) self.assertTrue(none_grad is None) - grad_with_none_and_not_none, = self.grad([x, y], [y], - create_graph=create_graph) + (grad_with_none_and_not_none,) = self.grad( + [x, y], [y], create_graph=create_graph + ) self.assertTrue(grad_with_none_and_not_none.shape, x.shape) self.assertTrue(np.all(grad_with_none_and_not_none.numpy() == 1)) - self.assertNotEqual(grad_with_none_and_not_none.stop_gradient, - create_graph) + self.assertNotEqual( + grad_with_none_and_not_none.stop_gradient, create_graph + ) def test_simple_example(self): with _test_eager_guard(): @@ -144,11 +152,12 @@ def func_none_one_initial_gradient(self): half_numel = int(numel / 2) half_x_positive = np.random.uniform(low=1, high=2, size=[half_numel]) - half_x_negative = np.random.uniform(low=-2, - high=-1, - size=[numel - half_numel]) - x_np = np.array(list(half_x_positive) + - list(half_x_negative)).astype('float32') + half_x_negative = np.random.uniform( + low=-2, high=-1, size=[numel - half_numel] + ) + x_np = np.array(list(half_x_positive) + list(half_x_negative)).astype( + 'float32' + ) np.random.shuffle(x_np) x = fluid.dygraph.to_variable(x_np) @@ -163,8 +172,9 @@ def func_none_one_initial_gradient(self): relu_x_np = np.maximum(x_np, alpha * x_np).astype('float32') relu_x_grad_np = ((x_np > 0) + (x_np < 0) * alpha).astype('float32') dy_expected = (relu_x_np * relu_x_grad_np * 2).astype('float32') - dz_expected = (np.power(relu_x_np, 3) * relu_x_grad_np * - 4).astype('float32') + dz_expected = (np.power(relu_x_np, 3) * relu_x_grad_np * 4).astype( + 'float32' + ) random_grad_y = random_var(y.shape, low=1, high=2) random_grad_z = random_var(z.shape, low=1, high=2) @@ -177,31 +187,39 @@ def func_none_one_initial_gradient(self): for grad_y in [random_grad_y]: for grad_z in [random_grad_z]: for create_graph in [False, True]: - dx_actual, = self.grad(outputs=[y, z], - inputs=[x], - grad_outputs=[grad_y, grad_z], - create_graph=create_graph, - retain_graph=True) + (dx_actual,) = self.grad( + outputs=[y, z], + inputs=[x], + grad_outputs=[grad_y, grad_z], + create_graph=create_graph, + retain_graph=True, + ) - grad_y_np = ones_grad_y if grad_y is None else grad_y.numpy( + grad_y_np = ( + ones_grad_y if grad_y is None else grad_y.numpy() ) - grad_z_np = ones_grad_z if grad_z is None else grad_z.numpy( + grad_z_np = ( + ones_grad_z if grad_z is None else grad_z.numpy() ) - dx_expected = dy_expected * grad_y_np + dz_expected * grad_z_np - np.testing.assert_allclose(dx_actual.numpy(), - dx_expected, - rtol=1e-05) + dx_expected = ( + dy_expected * grad_y_np + dz_expected * grad_z_np + ) + np.testing.assert_allclose( + dx_actual.numpy(), dx_expected, rtol=1e-05 + ) if grad_y is not None: self.assertTrue(grad_y.stop_gradient) - np.testing.assert_array_equal(grad_y.numpy(), - original_random_grad_y) + np.testing.assert_array_equal( + grad_y.numpy(), original_random_grad_y + ) if grad_z is not None: self.assertTrue(grad_z.stop_gradient) - np.testing.assert_array_equal(grad_z.numpy(), - original_random_grad_z) + np.testing.assert_array_equal( + grad_z.numpy(), original_random_grad_z + ) def test_none_one_initial_gradient(self): with _test_eager_guard(): @@ -222,14 +240,15 @@ def func_example_with_gradient_accumulation_and_create_graph(self): w_mean = fluid.layers.reduce_mean(w) del y, z, w - dx_actual, = self.grad([w_mean], [x], create_graph=True) + (dx_actual,) = self.grad([w_mean], [x], create_graph=True) del w_mean self.assertFalse(dx_actual.stop_gradient) # Theoritical result based on math calculation - dx_expected = (1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * - (x_np > 0) * 2).astype('float32') + dx_expected = ( + 1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * (x_np > 0) * 2 + ).astype('float32') np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) if not _in_legacy_dygraph(): @@ -240,12 +259,13 @@ def func_example_with_gradient_accumulation_and_create_graph(self): x_grad_actual = x.gradient() x_grad_expected = ( - 2.0 / float(numel) * - (x_np + dx_expected * - (x_np > 0) * 2 / float(numel))).astype('float32') - np.testing.assert_allclose(x_grad_actual, - x_grad_expected, - rtol=1e-05) + 2.0 + / float(numel) + * (x_np + dx_expected * (x_np > 0) * 2 / float(numel)) + ).astype('float32') + np.testing.assert_allclose( + x_grad_actual, x_grad_expected, rtol=1e-05 + ) def test_example_with_gradient_accumulation_and_create_graph(self): with _test_eager_guard(): @@ -267,15 +287,20 @@ def func_example_with_gradient_accumulation_and_no_grad_vars(self): w_mean = fluid.layers.reduce_mean(w) del y1, z, w - dx_actual, = self.grad([w_mean], [x], - create_graph=True, - no_grad_vars=[y2]) + (dx_actual,) = self.grad( + [w_mean], [x], create_graph=True, no_grad_vars=[y2] + ) self.assertFalse(y2.stop_gradient) self.assertFalse(dx_actual.stop_gradient) - dx_expected = (1.0 / float(numel) * (np.maximum(x_np, 0) + y2.numpy()) * - (x_np > 0) * 2).astype('float32') + dx_expected = ( + 1.0 + / float(numel) + * (np.maximum(x_np, 0) + y2.numpy()) + * (x_np > 0) + * 2 + ).astype('float32') np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) if not _in_legacy_dygraph(): @@ -286,12 +311,13 @@ def func_example_with_gradient_accumulation_and_no_grad_vars(self): x_grad_actual = x.gradient() x_grad_expected = ( - 2.0 / float(numel) * - (x_np + dx_expected * - (x_np > 0) * 4 / float(numel))).astype('float32') - np.testing.assert_allclose(x_grad_actual, - x_grad_expected, - rtol=1e-05) + 2.0 + / float(numel) + * (x_np + dx_expected * (x_np > 0) * 4 / float(numel)) + ).astype('float32') + np.testing.assert_allclose( + x_grad_actual, x_grad_expected, rtol=1e-05 + ) def test_example_with_gradient_accumulation_and_no_grad_vars(self): with _test_eager_guard(): @@ -312,13 +338,14 @@ def func_example_with_gradient_accumulation_and_not_create_graph(self): w_mean = fluid.layers.reduce_mean(w) del y, z, w - dx_actual, = self.grad([w_mean], [x], create_graph=False) + (dx_actual,) = self.grad([w_mean], [x], create_graph=False) del w_mean self.assertTrue(dx_actual.stop_gradient) - dx_expected = (1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * - (x_np > 0) * 2).astype('float32') + dx_expected = ( + 1.0 / float(numel) * (np.maximum(x_np, 0) + 1) * (x_np > 0) * 2 + ).astype('float32') np.testing.assert_allclose(dx_actual.numpy(), dx_expected, rtol=1e-05) @@ -330,9 +357,9 @@ def func_example_with_gradient_accumulation_and_not_create_graph(self): x_grad_actual = x.gradient() x_grad_expected = (2.0 * x_np / float(numel)).astype('float32') - np.testing.assert_allclose(x_grad_actual, - x_grad_expected, - rtol=1e-05) + np.testing.assert_allclose( + x_grad_actual, x_grad_expected, rtol=1e-05 + ) def test_example_with_gradient_accumulation_and_not_create_graph(self): with _test_eager_guard(): @@ -341,7 +368,6 @@ def test_example_with_gradient_accumulation_and_not_create_graph(self): class TestDygraphDoubleGradSortGradient(TestDygraphDoubleGrad): - def setUp(self): self.sort_sum_gradient = True self.shape = [5, 10] diff --git a/python/paddle/fluid/tests/unittests/test_paddle_multiprocessing.py b/python/paddle/fluid/tests/unittests/test_paddle_multiprocessing.py index 8f4131c8d6640..a6e3bf827d356 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_multiprocessing.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_multiprocessing.py @@ -18,7 +18,11 @@ import time import paddle import paddle.incubate.multiprocessing as mp -from paddle.fluid.framework import _enable_legacy_dygraph, _test_eager_guard, in_dygraph_mode +from paddle.fluid.framework import ( + _enable_legacy_dygraph, + _test_eager_guard, + in_dygraph_mode, +) REPEAT = 20 HAS_SHM_FILES = os.path.isdir('/dev/shm') @@ -47,14 +51,14 @@ def send_parambase(queue, event, device, dtype): tensor = paddle.nn.Layer().create_parameter( [5, 5], dtype=dtype, - default_initializer=paddle.nn.initializer.Constant(value=1.0)) + default_initializer=paddle.nn.initializer.Constant(value=1.0), + ) queue.put(tensor) queue.put(tensor) event.wait() class leak_checker(object): - def __init__(self, test_case): self.checked_pids = [os.getpid()] self.test_case = test_case @@ -98,7 +102,6 @@ def _has_shm_files(self): class TestMultiprocessingBase(unittest.TestCase): - def get_tensor(self, device="cpu"): self.device = device.lower() place = None @@ -108,7 +111,8 @@ def get_tensor(self, device="cpu"): def get_parameter(self): w = paddle.nn.Layer().create_parameter( [10, 10], - default_initializer=paddle.nn.initializer.Constant(value=0.0)) + default_initializer=paddle.nn.initializer.Constant(value=0.0), + ) return w def _test_empty(self, dtype="float32"): @@ -118,13 +122,9 @@ def _test_empty(self, dtype="float32"): out = q.get(timeout=1) self.assertEqual(str(out), str(empty)) - def _test_sharing(self, - ctx=mp, - device='cpu', - dtype="float32", - repeat=1, - param=False): - + def _test_sharing( + self, ctx=mp, device='cpu', dtype="float32", repeat=1, param=False + ): def test_fill(): if param: x = self.get_parameter() @@ -159,7 +159,8 @@ def test_receive(): process = ctx.Process( target=send_parambase if param else send_tensor, - args=(queue, event, device, dtype)) + args=(queue, event, device, dtype), + ) process.daemon = True lc.check_pid(process.pid) process.start() @@ -180,7 +181,6 @@ def test_receive(): class TestMultiprocessingCpu(TestMultiprocessingBase): - def func_test_pass_tensor(self): if in_dygraph_mode(): return @@ -216,9 +216,10 @@ def test_pass_empty(self): class TestMultiprocessingGpu(TestMultiprocessingBase): - - @unittest.skipIf(not paddle.fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not paddle.fluid.core.is_compiled_with_cuda(), + "core is not compiled with CUDA", + ) def func_test_pass_tensor(self): if in_dygraph_mode(): return diff --git a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py index f5aea5c303960..b41116973940f 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_save_load.py @@ -40,14 +40,17 @@ def random_batch_reader(): - def _get_random_inputs_and_labels(): np.random.seed(SEED) image = np.random.random([BATCH_SIZE, IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, ( - BATCH_SIZE, - 1, - )).astype('int64') + label = np.random.randint( + 0, + CLASS_NUM - 1, + ( + BATCH_SIZE, + 1, + ), + ).astype('int64') return image, label def __reader__(): @@ -61,7 +64,6 @@ def __reader__(): class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) @@ -71,7 +73,6 @@ def forward(self, x): class LayerWithLargeParameters(paddle.nn.Layer): - def __init__(self): super(LayerWithLargeParameters, self).__init__() self._l = paddle.nn.Linear(10, LARGE_PARAM) @@ -92,7 +93,6 @@ def train(layer, loader, loss_fn, opt): class TestSaveLoadLargeParameters(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -107,9 +107,11 @@ def test_large_parameters_paddle_save(self): layer = LayerWithLargeParameters() save_dict = layer.state_dict() - path = os.path.join(self.temp_dir.name, - "test_paddle_save_load_large_param_save", - "layer.pdparams") + path = os.path.join( + self.temp_dir.name, + "test_paddle_save_load_large_param_save", + "layer.pdparams", + ) protocol = 4 paddle.save(save_dict, path, protocol=protocol) dict_load = paddle.load(path, return_numpy=True) @@ -119,7 +121,6 @@ def test_large_parameters_paddle_save(self): class TestSaveLoadPickle(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -133,9 +134,11 @@ def test_pickle_protocol(self): layer = LinearNet() save_dict = layer.state_dict() - path = os.path.join(self.temp_dir.name, - "test_paddle_save_load_pickle_protocol", - "layer.pdparams") + path = os.path.join( + self.temp_dir.name, + "test_paddle_save_load_pickle_protocol", + "layer.pdparams", + ) with self.assertRaises(ValueError): paddle.save(save_dict, path, 2.0) @@ -156,12 +159,12 @@ def test_pickle_protocol(self): dict_load = paddle.load(path) # compare results before and after saving for key, value in save_dict.items(): - np.testing.assert_array_equal(dict_load[key].numpy(), - value.numpy()) + np.testing.assert_array_equal( + dict_load[key].numpy(), value.numpy() + ) class TestSaveLoadAny(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -209,9 +212,9 @@ def replace_static_load(self, program, model_path): def test_replace_static_save_load(self): paddle.enable_static() with new_program_scope(): - x = paddle.static.data(name="static_x", - shape=[None, IMAGE_SIZE], - dtype='float32') + x = paddle.static.data( + name="static_x", shape=[None, IMAGE_SIZE], dtype='float32' + ) z = paddle.static.nn.fc(x, 10) z = paddle.static.nn.fc(z, 10, bias_attr=False) loss = fluid.layers.reduce_mean(z) @@ -226,19 +229,22 @@ def test_replace_static_save_load(self): base_map = {} for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_map[var.name] = t - path = os.path.join(self.temp_dir.name, - "test_replace_static_save_load", "model") + path = os.path.join( + self.temp_dir.name, "test_replace_static_save_load", "model" + ) # paddle.save, legacy paddle.fluid.load self.replace_static_save(prog, path) self.set_zero(prog, place) paddle.fluid.io.load(prog, path) for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, np.array(base_t)) # legacy paddle.fluid.save, paddle.load @@ -247,8 +253,9 @@ def test_replace_static_save_load(self): self.replace_static_load(prog, path) for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) # test for return tensor @@ -258,7 +265,8 @@ def test_replace_static_save_load(self): tensor = var.get_value(fluid.global_scope()) paddle.save( tensor, - os.path.join(self.temp_dir.name, path_vars, var.name)) + os.path.join(self.temp_dir.name, path_vars, var.name), + ) with self.assertRaises(TypeError): var.get_value('fluid.global_scope()') with self.assertRaises(ValueError): @@ -276,12 +284,14 @@ def test_replace_static_save_load(self): self.set_zero(prog, place) for var in prog.list_vars(): if var.persistable: - tensor = paddle.load(os.path.join(self.temp_dir.name, - path_vars, var.name), - return_numpy=False) + tensor = paddle.load( + os.path.join(self.temp_dir.name, path_vars, var.name), + return_numpy=False, + ) var.set_value(tensor) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -289,17 +299,19 @@ def test_paddle_save_load_v2(self): paddle.disable_static() class StepDecay(LRScheduler): - - def __init__(self, - learning_rate, - step_size, - gamma=0.1, - last_epoch=-1, - verbose=False): + def __init__( + self, + learning_rate, + step_size, + gamma=0.1, + last_epoch=-1, + verbose=False, + ): self.step_size = step_size self.gamma = gamma - super(StepDecay, self).__init__(learning_rate, last_epoch, - verbose) + super(StepDecay, self).__init__( + learning_rate, last_epoch, verbose + ) def get_lr(self): i = self.last_epoch // self.step_size @@ -307,14 +319,16 @@ def get_lr(self): layer = LinearNet() inps = paddle.randn([2, IMAGE_SIZE]) - adam = opt.Adam(learning_rate=StepDecay(0.1, 1), - parameters=layer.parameters()) + adam = opt.Adam( + learning_rate=StepDecay(0.1, 1), parameters=layer.parameters() + ) y = layer(inps) y.mean().backward() adam.step() state_dict = adam.state_dict() - path = os.path.join(self.temp_dir.name, - 'paddle_save_load_v2/model.pdparams') + path = os.path.join( + self.temp_dir.name, 'paddle_save_load_v2/model.pdparams' + ) with self.assertRaises(TypeError): paddle.save(state_dict, path, use_binary_format='False') # legacy paddle.save, paddle.load @@ -327,8 +341,9 @@ def get_lr(self): if isinstance(v, dict): self.assertTrue(v == load_dict_tensor[k]) else: - np.testing.assert_array_equal(v.numpy(), - load_dict_tensor[k].numpy()) + np.testing.assert_array_equal( + v.numpy(), load_dict_tensor[k].numpy() + ) if not np.array_equal(v.numpy(), load_dict_np[k]): print(v.numpy()) print(load_dict_np[k]) @@ -338,8 +353,9 @@ def test_single_pickle_var_dygraph(self): # enable dygraph mode paddle.disable_static() layer = LinearNet() - path = os.path.join(self.temp_dir.name, - 'paddle_save_load_v2/var_dygraph') + path = os.path.join( + self.temp_dir.name, 'paddle_save_load_v2/var_dygraph' + ) tensor = layer._linear.weight with self.assertRaises(ValueError): paddle.save(tensor, path, pickle_protocol='3') @@ -351,7 +367,9 @@ def test_single_pickle_var_dygraph(self): self.assertTrue( isinstance( t_dygraph, - (paddle.fluid.core.VarBase, paddle.fluid.core.eager.Tensor))) + (paddle.fluid.core.VarBase, paddle.fluid.core.eager.Tensor), + ) + ) np.testing.assert_array_equal(tensor.numpy(), np_dygraph) np.testing.assert_array_equal(tensor.numpy(), t_dygraph.numpy()) paddle.enable_static() @@ -366,14 +384,16 @@ def test_single_pickle_var_static(self): paddle.enable_static() with new_program_scope(): # create network - x = paddle.static.data(name="x", - shape=[None, IMAGE_SIZE], - dtype='float32') + x = paddle.static.data( + name="x", shape=[None, IMAGE_SIZE], dtype='float32' + ) z = paddle.static.nn.fc(x, 128) loss = fluid.layers.reduce_mean(z) - place = fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) prog = paddle.static.default_main_program() @@ -383,8 +403,9 @@ def test_single_pickle_var_static(self): break scope = fluid.global_scope() origin_tensor = np.array(tensor) - path = os.path.join(self.temp_dir.name, - 'test_single_pickle_var_static/var') + path = os.path.join( + self.temp_dir.name, 'test_single_pickle_var_static/var' + ) paddle.save(tensor, path) self.set_zero(prog, place, scope) # static load @@ -406,8 +427,10 @@ def test_single_pickle_var_static(self): def test_dygraph_save_static_load(self): inps = np.random.randn(1, IMAGE_SIZE).astype('float32') - path = os.path.join(self.temp_dir.name, - 'test_dygraph_save_static_load/dy-static.pdparams') + path = os.path.join( + self.temp_dir.name, + 'test_dygraph_save_static_load/dy-static.pdparams', + ) paddle.disable_static() with paddle.utils.unique_name.guard(): layer = LinearNet() @@ -416,14 +439,16 @@ def test_dygraph_save_static_load(self): paddle.enable_static() with new_program_scope(): layer = LinearNet() - data = paddle.static.data(name='x_static_save', - shape=(None, IMAGE_SIZE), - dtype='float32') + data = paddle.static.data( + name='x_static_save', shape=(None, IMAGE_SIZE), dtype='float32' + ) y_static = layer(data) program = paddle.static.default_main_program() - place = fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe = paddle.static.Executor(paddle.CPUPlace()) exe.run(paddle.static.default_startup_program()) state_dict = paddle.load(path, keep_name_table=True) @@ -431,7 +456,8 @@ def test_dygraph_save_static_load(self): state_dict_param = program.state_dict("param") for name, tensor in state_dict_dy.items(): np.testing.assert_array_equal( - tensor.numpy(), np.array(state_dict_param[tensor.name])) + tensor.numpy(), np.array(state_dict_param[tensor.name]) + ) def test_save_load_complex_object_dygraph_save(self): paddle.disable_static() @@ -440,24 +466,28 @@ def test_save_load_complex_object_dygraph_save(self): obj1 = [ paddle.randn([3, 4], dtype='float32'), np.random.randn(5, 6), - ('fake_weight', np.ones([7, 8], dtype='float32')) + ('fake_weight', np.ones([7, 8], dtype='float32')), ] obj2 = {'k1': obj1, 'k2': state_dict, 'epoch': 123} - obj3 = (paddle.randn([5, 4], dtype='float32'), - np.random.randn(3, 4).astype("float32"), { - "state_dict": state_dict, - "opt": state_dict - }) - obj4 = (np.random.randn(5, 6), (123, )) - - path1 = os.path.join(self.temp_dir.name, - "test_save_load_any_complex_object_dygraph/obj1") - path2 = os.path.join(self.temp_dir.name, - "test_save_load_any_complex_object_dygraph/obj2") - path3 = os.path.join(self.temp_dir.name, - "test_save_load_any_complex_object_dygraph/obj3") - path4 = os.path.join(self.temp_dir.name, - "test_save_load_any_complex_object_dygraph/obj4") + obj3 = ( + paddle.randn([5, 4], dtype='float32'), + np.random.randn(3, 4).astype("float32"), + {"state_dict": state_dict, "opt": state_dict}, + ) + obj4 = (np.random.randn(5, 6), (123,)) + + path1 = os.path.join( + self.temp_dir.name, "test_save_load_any_complex_object_dygraph/obj1" + ) + path2 = os.path.join( + self.temp_dir.name, "test_save_load_any_complex_object_dygraph/obj2" + ) + path3 = os.path.join( + self.temp_dir.name, "test_save_load_any_complex_object_dygraph/obj3" + ) + path4 = os.path.join( + self.temp_dir.name, "test_save_load_any_complex_object_dygraph/obj4" + ) paddle.save(obj1, path1) paddle.save(obj2, path2) paddle.save(obj3, path3) @@ -473,10 +503,12 @@ def test_save_load_complex_object_dygraph_save(self): np.testing.assert_array_equal(load_tensor1[2].numpy(), obj1[2][1]) for i in range(len(load_tensor1)): self.assertTrue( - type(load_tensor1[i]) == type(load_tensor2['k1'][i])) + type(load_tensor1[i]) == type(load_tensor2['k1'][i]) + ) for k, v in state_dict.items(): - np.testing.assert_array_equal(v.numpy(), - load_tensor2['k2'][k].numpy()) + np.testing.assert_array_equal( + v.numpy(), load_tensor2['k2'][k].numpy() + ) self.assertTrue(load_tensor2['epoch'] == 123) np.testing.assert_array_equal(load_tensor3[0].numpy(), obj3[0].numpy()) @@ -484,11 +516,13 @@ def test_save_load_complex_object_dygraph_save(self): for k, v in state_dict.items(): np.testing.assert_array_equal( - load_tensor3[2]['state_dict'][k].numpy(), v.numpy()) + load_tensor3[2]['state_dict'][k].numpy(), v.numpy() + ) for k, v in state_dict.items(): - np.testing.assert_array_equal(load_tensor3[2]['opt'][k].numpy(), - v.numpy()) + np.testing.assert_array_equal( + load_tensor3[2]['opt'][k].numpy(), v.numpy() + ) np.testing.assert_array_equal(load_tensor4[0].numpy(), obj4[0]) @@ -510,8 +544,9 @@ def test_save_load_complex_object_dygraph_save(self): np.testing.assert_array_equal(load_array3[1], obj3[1]) for k, v in state_dict.items(): - np.testing.assert_array_equal(load_array3[2]['state_dict'][k], - v.numpy()) + np.testing.assert_array_equal( + load_array3[2]['state_dict'][k], v.numpy() + ) for k, v in state_dict.items(): np.testing.assert_array_equal(load_array3[2]['opt'][k], v.numpy()) @@ -526,38 +561,50 @@ def test_save_load_complex_object_dygraph_save(self): load_tensor3 = paddle.load(path3, return_numpy=False) load_tensor4 = paddle.load(path4, return_numpy=False) - np.testing.assert_array_equal(np.array(load_tensor1[0]), - obj1[0].numpy()) + np.testing.assert_array_equal( + np.array(load_tensor1[0]), obj1[0].numpy() + ) np.testing.assert_array_equal(np.array(load_tensor1[1]), obj1[1]) np.testing.assert_array_equal(np.array(load_tensor1[2]), obj1[2][1]) for i in range(len(load_tensor1)): self.assertTrue( - type(load_tensor1[i]) == type(load_tensor2['k1'][i])) + type(load_tensor1[i]) == type(load_tensor2['k1'][i]) + ) for k, v in state_dict.items(): - np.testing.assert_array_equal(v.numpy(), - np.array(load_tensor2['k2'][k])) + np.testing.assert_array_equal( + v.numpy(), np.array(load_tensor2['k2'][k]) + ) self.assertTrue(load_tensor2['epoch'] == 123) - self.assertTrue(isinstance(load_tensor3[0], - paddle.fluid.core.LoDTensor)) - np.testing.assert_array_equal(np.array(load_tensor3[0]), - obj3[0].numpy()) + self.assertTrue( + isinstance(load_tensor3[0], paddle.fluid.core.LoDTensor) + ) + np.testing.assert_array_equal( + np.array(load_tensor3[0]), obj3[0].numpy() + ) np.testing.assert_array_equal(np.array(load_tensor3[1]), obj3[1]) for k, v in state_dict.items(): self.assertTrue( - isinstance(load_tensor3[2]["state_dict"][k], - paddle.fluid.core.LoDTensor)) + isinstance( + load_tensor3[2]["state_dict"][k], + paddle.fluid.core.LoDTensor, + ) + ) np.testing.assert_array_equal( - np.array(load_tensor3[2]['state_dict'][k]), v.numpy()) + np.array(load_tensor3[2]['state_dict'][k]), v.numpy() + ) for k, v in state_dict.items(): self.assertTrue( - isinstance(load_tensor3[2]["opt"][k], - paddle.fluid.core.LoDTensor)) - np.testing.assert_array_equal(np.array(load_tensor3[2]['opt'][k]), - v.numpy()) + isinstance( + load_tensor3[2]["opt"][k], paddle.fluid.core.LoDTensor + ) + ) + np.testing.assert_array_equal( + np.array(load_tensor3[2]['opt'][k]), v.numpy() + ) self.assertTrue(load_tensor4[0], paddle.fluid.core.LoDTensor) np.testing.assert_array_equal(np.array(load_tensor4[0]), obj4[0]) @@ -581,8 +628,9 @@ def test_save_load_complex_object_dygraph_save(self): np.testing.assert_array_equal(load_array3[1], obj3[1]) for k, v in state_dict.items(): - np.testing.assert_array_equal(load_array3[2]['state_dict'][k], - v.numpy()) + np.testing.assert_array_equal( + load_array3[2]['state_dict'][k], v.numpy() + ) for k, v in state_dict.items(): np.testing.assert_array_equal(load_array3[2]['opt'][k], v.numpy()) @@ -593,15 +641,17 @@ def test_save_load_complex_object_static_save(self): paddle.enable_static() with new_program_scope(): # create network - x = paddle.static.data(name="x", - shape=[None, IMAGE_SIZE], - dtype='float32') + x = paddle.static.data( + name="x", shape=[None, IMAGE_SIZE], dtype='float32' + ) z = paddle.static.nn.fc(x, 10, bias_attr=False) z = paddle.static.nn.fc(z, 128, bias_attr=False) loss = fluid.layers.reduce_mean(z) - place = fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) prog = paddle.static.default_main_program() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) @@ -611,27 +661,32 @@ def test_save_load_complex_object_static_save(self): obj1 = [ state_dict[keys[0]], np.random.randn(5, 6), - ('fake_weight', np.ones([7, 8], dtype='float32')) + ('fake_weight', np.ones([7, 8], dtype='float32')), ] obj2 = {'k1': obj1, 'k2': state_dict, 'epoch': 123} - obj3 = (state_dict[keys[0]], np.ndarray([3, 4], dtype="float32"), { - "state_dict": state_dict, - "opt": state_dict - }) - obj4 = (np.ndarray([3, 4], dtype="float32"), ) + obj3 = ( + state_dict[keys[0]], + np.ndarray([3, 4], dtype="float32"), + {"state_dict": state_dict, "opt": state_dict}, + ) + obj4 = (np.ndarray([3, 4], dtype="float32"),) path1 = os.path.join( self.temp_dir.name, - "test_save_load_any_complex_object_static/obj1") + "test_save_load_any_complex_object_static/obj1", + ) path2 = os.path.join( self.temp_dir.name, - "test_save_load_any_complex_object_static/obj2") + "test_save_load_any_complex_object_static/obj2", + ) path3 = os.path.join( self.temp_dir.name, - "test_save_load_any_complex_object_static/obj3") + "test_save_load_any_complex_object_static/obj3", + ) path4 = os.path.join( self.temp_dir.name, - "test_save_load_any_complex_object_static/obj4") + "test_save_load_any_complex_object_static/obj4", + ) paddle.save(obj1, path1) paddle.save(obj2, path2) paddle.save(obj3, path3) @@ -642,16 +697,19 @@ def test_save_load_complex_object_static_save(self): load_tensor3 = paddle.load(path3, return_numpy=False) load_tensor4 = paddle.load(path4, return_numpy=False) - np.testing.assert_array_equal(np.array(load_tensor1[0]), - np.array(obj1[0])) + np.testing.assert_array_equal( + np.array(load_tensor1[0]), np.array(obj1[0]) + ) np.testing.assert_array_equal(np.array(load_tensor1[1]), obj1[1]) np.testing.assert_array_equal(np.array(load_tensor1[2]), obj1[2][1]) for i in range(len(load_tensor1)): self.assertTrue( - type(load_tensor1[i]) == type(load_tensor2['k1'][i])) + type(load_tensor1[i]) == type(load_tensor2['k1'][i]) + ) for k, v in state_dict.items(): - np.testing.assert_array_equal(np.array(v), - np.array(load_tensor2['k2'][k])) + np.testing.assert_array_equal( + np.array(v), np.array(load_tensor2['k2'][k]) + ) self.assertTrue(load_tensor2['epoch'] == 123) self.assertTrue(isinstance(load_tensor3[0], fluid.core.LoDTensor)) @@ -661,16 +719,21 @@ def test_save_load_complex_object_static_save(self): for k, v in state_dict.items(): self.assertTrue( - isinstance(load_tensor3[2]["state_dict"][k], - fluid.core.LoDTensor)) + isinstance( + load_tensor3[2]["state_dict"][k], fluid.core.LoDTensor + ) + ) np.testing.assert_array_equal( - np.array(load_tensor3[2]['state_dict'][k]), np.array(v)) + np.array(load_tensor3[2]['state_dict'][k]), np.array(v) + ) for k, v in state_dict.items(): self.assertTrue( - isinstance(load_tensor3[2]["opt"][k], fluid.core.LoDTensor)) + isinstance(load_tensor3[2]["opt"][k], fluid.core.LoDTensor) + ) np.testing.assert_array_equal( - np.array(load_tensor3[2]['opt'][k]), np.array(v)) + np.array(load_tensor3[2]['opt'][k]), np.array(v) + ) self.assertTrue(isinstance(load_tensor4[0], fluid.core.LoDTensor)) np.testing.assert_array_equal(np.array(load_tensor4[0]), obj4[0]) @@ -685,7 +748,8 @@ def test_save_load_complex_object_static_save(self): np.testing.assert_array_equal(load_array1[2], obj1[2][1]) for i in range(len(load_array1)): self.assertTrue( - type(load_array1[i]) == type(load_array2['k1'][i])) + type(load_array1[i]) == type(load_array2['k1'][i]) + ) for k, v in state_dict.items(): np.testing.assert_array_equal(np.array(v), load_array2['k2'][k]) self.assertTrue(load_array2['epoch'] == 123) @@ -694,12 +758,14 @@ def test_save_load_complex_object_static_save(self): np.testing.assert_array_equal(load_array3[1], obj3[1]) for k, v in state_dict.items(): - np.testing.assert_array_equal(load_array3[2]['state_dict'][k], - np.array(v)) + np.testing.assert_array_equal( + load_array3[2]['state_dict'][k], np.array(v) + ) for k, v in state_dict.items(): - np.testing.assert_array_equal(load_array3[2]['opt'][k], - np.array(v)) + np.testing.assert_array_equal( + load_array3[2]['opt'][k], np.array(v) + ) np.testing.assert_array_equal(load_array4[0], obj4[0]) @@ -711,44 +777,64 @@ def test_save_load_complex_object_static_save(self): load_tensor3 = paddle.load(path3, return_numpy=False) load_tensor4 = paddle.load(path4, return_numpy=False) - np.testing.assert_array_equal(np.array(load_tensor1[0]), - np.array(obj1[0])) + np.testing.assert_array_equal( + np.array(load_tensor1[0]), np.array(obj1[0]) + ) np.testing.assert_array_equal(np.array(load_tensor1[1]), obj1[1]) np.testing.assert_array_equal(load_tensor1[2].numpy(), obj1[2][1]) for i in range(len(load_tensor1)): self.assertTrue( - type(load_tensor1[i]) == type(load_tensor2['k1'][i])) + type(load_tensor1[i]) == type(load_tensor2['k1'][i]) + ) for k, v in state_dict.items(): - np.testing.assert_array_equal(np.array(v), - np.array(load_tensor2['k2'][k])) + np.testing.assert_array_equal( + np.array(v), np.array(load_tensor2['k2'][k]) + ) self.assertTrue(load_tensor2['epoch'] == 123) self.assertTrue( - isinstance(load_tensor3[0], - (fluid.core.VarBase, fluid.core.eager.Tensor))) + isinstance( + load_tensor3[0], + (fluid.core.VarBase, fluid.core.eager.Tensor), + ) + ) np.testing.assert_array_equal(load_tensor3[0].numpy(), obj3[0]) self.assertTrue( - isinstance(load_tensor3[1], - (fluid.core.VarBase, fluid.core.eager.Tensor))) + isinstance( + load_tensor3[1], + (fluid.core.VarBase, fluid.core.eager.Tensor), + ) + ) np.testing.assert_array_equal(load_tensor3[1].numpy(), obj3[1]) for k, v in state_dict.items(): self.assertTrue( - isinstance(load_tensor3[2]["state_dict"][k], - (fluid.core.VarBase, fluid.core.eager.Tensor))) + isinstance( + load_tensor3[2]["state_dict"][k], + (fluid.core.VarBase, fluid.core.eager.Tensor), + ) + ) np.testing.assert_array_equal( - load_tensor3[2]['state_dict'][k].numpy(), np.array(v)) + load_tensor3[2]['state_dict'][k].numpy(), np.array(v) + ) for k, v in state_dict.items(): self.assertTrue( - isinstance(load_tensor3[2]["opt"][k], - (fluid.core.VarBase, fluid.core.eager.Tensor))) - np.testing.assert_array_equal(load_tensor3[2]['opt'][k].numpy(), - np.array(v)) + isinstance( + load_tensor3[2]["opt"][k], + (fluid.core.VarBase, fluid.core.eager.Tensor), + ) + ) + np.testing.assert_array_equal( + load_tensor3[2]['opt'][k].numpy(), np.array(v) + ) self.assertTrue( - isinstance(load_tensor4[0], - (fluid.core.VarBase, fluid.core.eager.Tensor))) + isinstance( + load_tensor4[0], + (fluid.core.VarBase, fluid.core.eager.Tensor), + ) + ) np.testing.assert_array_equal(load_tensor4[0].numpy(), obj4[0]) load_array1 = paddle.load(path1, return_numpy=True) @@ -761,7 +847,8 @@ def test_save_load_complex_object_static_save(self): np.testing.assert_array_equal(load_array1[2], obj1[2][1]) for i in range(len(load_array1)): self.assertTrue( - type(load_array1[i]) == type(load_array2['k1'][i])) + type(load_array1[i]) == type(load_array2['k1'][i]) + ) for k, v in state_dict.items(): np.testing.assert_array_equal(np.array(v), load_array2['k2'][k]) self.assertTrue(load_array2['epoch'] == 123) @@ -770,12 +857,14 @@ def test_save_load_complex_object_static_save(self): np.testing.assert_array_equal(load_array3[1], obj3[1]) for k, v in state_dict.items(): - np.testing.assert_array_equal(load_array3[2]['state_dict'][k], - np.array(v)) + np.testing.assert_array_equal( + load_array3[2]['state_dict'][k], np.array(v) + ) for k, v in state_dict.items(): - np.testing.assert_array_equal(load_array3[2]['opt'][k], - np.array(v)) + np.testing.assert_array_equal( + load_array3[2]['opt'][k], np.array(v) + ) self.assertTrue(isinstance(load_array4[0], np.ndarray)) np.testing.assert_array_equal(load_array4[0], obj4[0]) @@ -783,8 +872,10 @@ def test_save_load_complex_object_static_save(self): def test_varbase_binary_var(self): paddle.disable_static() varbase = paddle.randn([3, 2], dtype='float32') - path = os.path.join(self.temp_dir.name, - 'test_paddle_save_load_varbase_binary_var/varbase') + path = os.path.join( + self.temp_dir.name, + 'test_paddle_save_load_varbase_binary_var/varbase', + ) paddle.save(varbase, path, use_binary_format=True) load_array = paddle.load(path, return_numpy=True) load_tensor = paddle.load(path, return_numpy=False) @@ -797,7 +888,6 @@ def test_varbase_binary_var(self): class TestSaveLoadToMemory(unittest.TestCase): - def test_dygraph_save_to_memory(self): paddle.disable_static() linear = LinearNet() @@ -826,15 +916,17 @@ def test_static_save_to_memory(self): paddle.enable_static() with new_program_scope(): # create network - x = paddle.static.data(name="x", - shape=[None, IMAGE_SIZE], - dtype='float32') + x = paddle.static.data( + name="x", shape=[None, IMAGE_SIZE], dtype='float32' + ) z = paddle.static.nn.fc(x, 10, bias_attr=False) z = paddle.static.nn.fc(z, 128, bias_attr=False) loss = fluid.layers.reduce_mean(z) - place = fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) prog = paddle.static.default_main_program() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) @@ -852,8 +944,10 @@ def test_static_save_to_memory(self): byio2.seek(0) prog_load = paddle.load(byio2) - self.assertTrue(prog.desc.serialize_to_string() == - prog_load.desc.serialize_to_string()) + self.assertTrue( + prog.desc.serialize_to_string() + == prog_load.desc.serialize_to_string() + ) tensor_load = paddle.load(byio, return_numpy=True) np.testing.assert_array_equal(tensor_load, np.array(tensor)) @@ -864,7 +958,6 @@ def test_static_save_to_memory(self): class TestSaveLoad(unittest.TestCase): - def setUp(self): # enable dygraph mode paddle.disable_static() @@ -895,18 +988,23 @@ def build_and_train_model(self): def check_load_state_dict(self, orig_dict, load_dict): for var_name, value in orig_dict.items(): - load_value = load_dict[var_name].numpy() if hasattr( - load_dict[var_name], 'numpy') else np.array(load_dict[var_name]) + load_value = ( + load_dict[var_name].numpy() + if hasattr(load_dict[var_name], 'numpy') + else np.array(load_dict[var_name]) + ) np.testing.assert_array_equal(value.numpy(), load_value) def test_save_load(self): layer, opt = self.build_and_train_model() # save - layer_save_path = os.path.join(self.temp_dir.name, - "test_paddle_save_load.linear.pdparams") - opt_save_path = os.path.join(self.temp_dir.name, - "test_paddle_save_load.linear.pdopt") + layer_save_path = os.path.join( + self.temp_dir.name, "test_paddle_save_load.linear.pdparams" + ) + opt_save_path = os.path.join( + self.temp_dir.name, "test_paddle_save_load.linear.pdopt" + ) layer_state_dict = layer.state_dict() opt_state_dict = opt.state_dict() @@ -924,7 +1022,8 @@ def test_save_load(self): paddle.enable_static() static_save_path = os.path.join( self.temp_dir.name, - "static_mode_test/test_paddle_save_load.linear.pdparams") + "static_mode_test/test_paddle_save_load.linear.pdparams", + ) paddle.save(layer_state_dict, static_save_path) load_static_state_dict = paddle.load(static_save_path) self.check_load_state_dict(layer_state_dict, load_static_state_dict) @@ -937,33 +1036,36 @@ def test_save_load(self): with self.assertRaises(ValueError): paddle.save( layer_state_dict, - os.path.join(self.temp_dir.name, - "test_paddle_save_load.linear.model/")) + os.path.join( + self.temp_dir.name, "test_paddle_save_load.linear.model/" + ), + ) # 3. test load path not exist error with self.assertRaises(ValueError): paddle.load( - os.path.join(self.temp_dir.name, - "test_paddle_save_load.linear.params")) + os.path.join( + self.temp_dir.name, "test_paddle_save_load.linear.params" + ) + ) # 4. test load old save path error with self.assertRaises(ValueError): paddle.load( - os.path.join(self.temp_dir.name, - "test_paddle_save_load.linear")) + os.path.join(self.temp_dir.name, "test_paddle_save_load.linear") + ) class TestSaveLoadProgram(unittest.TestCase): - def test_save_load_program(self): paddle.enable_static() temp_dir = tempfile.TemporaryDirectory() with new_program_scope(): layer = LinearNet() - data = paddle.static.data(name='x_static_save', - shape=(None, IMAGE_SIZE), - dtype='float32') + data = paddle.static.data( + name='x_static_save', shape=(None, IMAGE_SIZE), dtype='float32' + ) y_static = layer(data) main_program = paddle.static.default_main_program() startup_program = paddle.static.default_startup_program() @@ -971,10 +1073,12 @@ def test_save_load_program(self): origin_startup = startup_program.desc.serialize_to_string() path1 = os.path.join( temp_dir.name, - "test_paddle_save_load_program/main_program.pdmodel") + "test_paddle_save_load_program/main_program.pdmodel", + ) path2 = os.path.join( temp_dir.name, - "test_paddle_save_load_program/startup_program.pdmodel") + "test_paddle_save_load_program/startup_program.pdmodel", + ) paddle.save(main_program, path1) paddle.save(startup_program, path2) @@ -987,7 +1091,6 @@ def test_save_load_program(self): class TestSaveLoadLayer(unittest.TestCase): - def test_save_load_layer(self): paddle.disable_static() temp_dir = tempfile.TemporaryDirectory() @@ -998,8 +1101,9 @@ def test_save_load_layer(self): layer2.eval() origin_layer = (layer1, layer2) origin = (layer1(inps), layer2(inps)) - path = os.path.join(temp_dir.name, - "test_save_load_layer_/layer.pdmodel") + path = os.path.join( + temp_dir.name, "test_save_load_layer_/layer.pdmodel" + ) with self.assertRaises(ValueError): paddle.save(origin_layer, path) temp_dir.cleanup() diff --git a/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py b/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py index 32ee64d3c70f6..5eca9492ee396 100644 --- a/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py +++ b/python/paddle/fluid/tests/unittests/test_paddle_save_load_binary.py @@ -28,7 +28,6 @@ class TestSaveLoadBinaryFormat(unittest.TestCase): - def setUp(self): # enable static graph mode paddle.enable_static() @@ -49,18 +48,18 @@ def set_zero(self, prog, place, scope=None): self.assertTrue(np.sum(np.abs(new_t)) == 0) def replace_save_vars(self, program, dirname): - def predicate(var): return var.persistable vars = filter(predicate, program.list_vars()) for var in vars: - paddle.save(var.get_value(), - os.path.join(dirname, var.name), - use_binary_format=True) + paddle.save( + var.get_value(), + os.path.join(dirname, var.name), + use_binary_format=True, + ) def replace_load_vars(self, program, dirname): - def predicate(var): return var.persistable @@ -74,60 +73,65 @@ def test_replace_save_load_vars(self): paddle.enable_static() with new_program_scope(): # create network - x = paddle.static.data(name="x", - shape=[None, IMAGE_SIZE], - dtype='float32') + x = paddle.static.data( + name="x", shape=[None, IMAGE_SIZE], dtype='float32' + ) z = paddle.static.nn.fc(x, 10, bias_attr=False) z = paddle.static.nn.fc(z, 128, bias_attr=False) loss = fluid.layers.reduce_mean(z) - place = fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) prog = paddle.static.default_main_program() base_map = {} for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t # test for replace_save_vars/io.load_vars path_vars1 = os.path.join( - self.temp_dir.name, 'test_replace_save_load_vars_binary1/model') + self.temp_dir.name, 'test_replace_save_load_vars_binary1/model' + ) self.replace_save_vars(prog, path_vars1) # set var to zero self.set_zero(prog, place) var_list = list( - filter(lambda var: var.persistable, prog.list_vars())) - fluid.io.load_vars(exe, - path_vars1, - main_program=prog, - vars=var_list) + filter(lambda var: var.persistable, prog.list_vars()) + ) + fluid.io.load_vars( + exe, path_vars1, main_program=prog, vars=var_list + ) for var in prog.list_vars(): if var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) # test for io.save_vars/replace_load_vars path_vars2 = os.path.join( - self.temp_dir.name, - 'test_replace_save_load_vars_binary2/model/') - fluid.io.save_vars(exe, - path_vars2, - main_program=prog, - vars=var_list) + self.temp_dir.name, 'test_replace_save_load_vars_binary2/model/' + ) + fluid.io.save_vars( + exe, path_vars2, main_program=prog, vars=var_list + ) self.set_zero(prog, place) self.replace_load_vars(prog, path_vars2) for var in prog.list_vars(): if var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -143,22 +147,27 @@ def test_save_load_lod_tensor(self): name='fc_vars', ) prog = fluid.default_main_program() - place = fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) exe = fluid.Executor(place) prog = paddle.static.default_main_program() exe.run(fluid.default_startup_program()) - dirname = os.path.join(self.temp_dir.name, - 'test_save_load_lod_tensor1/tensor_') + dirname = os.path.join( + self.temp_dir.name, 'test_save_load_lod_tensor1/tensor_' + ) for var in prog.list_vars(): - if var.persistable and list( - var.shape) == [IMAGE_SIZE, OUTPUT_NUM]: + if var.persistable and list(var.shape) == [ + IMAGE_SIZE, + OUTPUT_NUM, + ]: tensor = var.get_value() - paddle.save(tensor, - dirname + 'fc_vars.w_0', - use_binary_format=True) + paddle.save( + tensor, dirname + 'fc_vars.w_0', use_binary_format=True + ) break origin = np.array(var.get_value()) @@ -168,7 +177,8 @@ def test_save_load_lod_tensor(self): loaded_tensor = paddle.load(dirname + 'fc_vars.w_0') self.assertTrue(isinstance(loaded_tensor, fluid.core.LoDTensor)) self.assertTrue( - list(loaded_tensor.shape()) == [IMAGE_SIZE, OUTPUT_NUM]) + list(loaded_tensor.shape()) == [IMAGE_SIZE, OUTPUT_NUM] + ) to_array = np.array(loaded_tensor) np.testing.assert_array_equal(origin, to_array) @@ -178,8 +188,9 @@ def test_save_load_lod_tensor(self): # On the Windows platform, when parsing a string that can't be parsed as a `Program`, `desc_.ParseFromString` has a timeout risk. if 'Windows' != platform.system(): with self.assertRaises(ValueError): - path = os.path.join(self.temp_dir.name, - 'test_save_load_error/temp') + path = os.path.join( + self.temp_dir.name, 'test_save_load_error/temp' + ) with open(path, "w") as f: f.write('\0') paddle.load(path) @@ -193,14 +204,18 @@ def test_save_load_lod_tensor(self): temp_lod, os.path.join( self.temp_dir.name, - 'test_save_load_error_not_exist_file/not_exist_file')) + 'test_save_load_error_not_exist_file/not_exist_file', + ), + ) with self.assertRaises(RuntimeError): fluid.core.load_lod_tensor( temp_lod, os.path.join( self.temp_dir.name, - 'test_save_load_error_not_exist_file/not_exist_file')) + 'test_save_load_error_not_exist_file/not_exist_file', + ), + ) # save to memory byio = BytesIO() @@ -218,14 +233,18 @@ def test_save_load_lod_tensor(self): def test_save_load_selected_rows(self): paddle.enable_static() - place = fluid.CPUPlace() if not paddle.fluid.core.is_compiled_with_cuda( - ) else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) height = 10 rows = [0, 4, 7] row_numel = 12 selected_rows = fluid.core.SelectedRows(rows, height) - path = os.path.join(self.temp_dir.name, - 'test_paddle_save_load_selected_rows/sr.pdsr') + path = os.path.join( + self.temp_dir.name, 'test_paddle_save_load_selected_rows/sr.pdsr' + ) with self.assertRaises(ValueError): paddle.save(selected_rows, path, use_binary_format=True) @@ -247,13 +266,17 @@ def test_save_load_selected_rows(self): selected_rows, os.path.join( self.temp_dir.name, - 'test_paddle_save_load_selected_rows_not_exist_file/temp')) + 'test_paddle_save_load_selected_rows_not_exist_file/temp', + ), + ) with self.assertRaises(RuntimeError): fluid.core.load_selected_rows( selected_rows, os.path.join( self.temp_dir.name, - 'test_paddle_save_load_selected_rows_not_exist_file/temp')) + 'test_paddle_save_load_selected_rows_not_exist_file/temp', + ), + ) # save to memory byio = BytesIO() @@ -265,8 +288,9 @@ def test_save_load_selected_rows(self): self.assertTrue(isinstance(selected_rows_mem, fluid.core.SelectedRows)) self.assertTrue(list(selected_rows_mem.rows()) == rows) self.assertTrue(selected_rows_mem.height() == height) - np.testing.assert_array_equal(np.array(selected_rows_mem.get_tensor()), - np_array) + np.testing.assert_array_equal( + np.array(selected_rows_mem.get_tensor()), np_array + ) with self.assertRaises(NotImplementedError): paddle.framework.io._save_selected_rows(selected_rows, 1) diff --git a/python/paddle/fluid/tests/unittests/test_pairwise_distance.py b/python/paddle/fluid/tests/unittests/test_pairwise_distance.py index 419084cf2715e..675196f57ebb2 100644 --- a/python/paddle/fluid/tests/unittests/test_pairwise_distance.py +++ b/python/paddle/fluid/tests/unittests/test_pairwise_distance.py @@ -27,121 +27,92 @@ def np_pairwise_distance(x, y, p=2.0, epsilon=1e-6, keepdim=False): return distance -def call_pairwise_distance_layer(x, y, p=2., epsilon=1e-6, keepdim='False'): - pairwise_distance = paddle.nn.PairwiseDistance(p=p, - epsilon=epsilon, - keepdim=keepdim) +def call_pairwise_distance_layer(x, y, p=2.0, epsilon=1e-6, keepdim='False'): + pairwise_distance = paddle.nn.PairwiseDistance( + p=p, epsilon=epsilon, keepdim=keepdim + ) distance = pairwise_distance(x=x, y=y) return distance -def call_pairwise_distance_functional(x, - y, - p=2., - epsilon=1e-6, - keepdim='False'): - distance = paddle.nn.functional.pairwise_distance(x=x, - y=y, - p=p, - epsilon=epsilon, - keepdim=keepdim) +def call_pairwise_distance_functional( + x, y, p=2.0, epsilon=1e-6, keepdim='False' +): + distance = paddle.nn.functional.pairwise_distance( + x=x, y=y, p=p, epsilon=epsilon, keepdim=keepdim + ) return distance -def test_static(place, - x_np, - y_np, - p=2.0, - epsilon=1e-6, - keepdim=False, - functional=False): +def test_static( + place, x_np, y_np, p=2.0, epsilon=1e-6, keepdim=False, functional=False +): prog = paddle.static.Program() startup_prog = paddle.static.Program() - place = fluid.CUDAPlace( - 0) if paddle.fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) paddle.enable_static() with paddle.static.program_guard(prog, startup_prog): x = paddle.fluid.data(name='x', shape=x_np.shape, dtype=x_np.dtype) y = paddle.fluid.data(name='y', shape=y_np.shape, dtype=x_np.dtype) if functional: - distance = call_pairwise_distance_functional(x=x, - y=y, - p=p, - epsilon=epsilon, - keepdim=keepdim) + distance = call_pairwise_distance_functional( + x=x, y=y, p=p, epsilon=epsilon, keepdim=keepdim + ) else: - distance = call_pairwise_distance_layer(x=x, - y=y, - p=p, - epsilon=epsilon, - keepdim=keepdim) + distance = call_pairwise_distance_layer( + x=x, y=y, p=p, epsilon=epsilon, keepdim=keepdim + ) exe = paddle.static.Executor(place) - static_ret = exe.run(prog, - feed={ - 'x': x_np, - 'y': y_np - }, - fetch_list=[distance]) + static_ret = exe.run( + prog, feed={'x': x_np, 'y': y_np}, fetch_list=[distance] + ) static_ret = static_ret[0] paddle.disable_static() return static_ret -def test_dygraph(place, - x_np, - y_np, - p=2.0, - epsilon=1e-6, - keepdim=False, - functional=False): +def test_dygraph( + place, x_np, y_np, p=2.0, epsilon=1e-6, keepdim=False, functional=False +): x = paddle.to_tensor(x_np) y = paddle.to_tensor(y_np) if functional: - dy_distance = call_pairwise_distance_functional(x=x, - y=y, - p=p, - epsilon=epsilon, - keepdim=keepdim) + dy_distance = call_pairwise_distance_functional( + x=x, y=y, p=p, epsilon=epsilon, keepdim=keepdim + ) else: - dy_distance = call_pairwise_distance_layer(x=x, - y=y, - p=p, - epsilon=epsilon, - keepdim=keepdim) + dy_distance = call_pairwise_distance_layer( + x=x, y=y, p=p, epsilon=epsilon, keepdim=keepdim + ) dygraph_ret = dy_distance.numpy() return dygraph_ret -def test_legacy_dygraph(place, - x_np, - y_np, - p=2.0, - epsilon=1e-6, - keepdim=False, - functional=False): +def test_legacy_dygraph( + place, x_np, y_np, p=2.0, epsilon=1e-6, keepdim=False, functional=False +): paddle.fluid.framework._enable_legacy_dygraph() x = paddle.to_tensor(x_np) y = paddle.to_tensor(y_np) if functional: - legacy_distance = call_pairwise_distance_functional(x=x, - y=y, - p=p, - epsilon=epsilon, - keepdim=keepdim) + legacy_distance = call_pairwise_distance_functional( + x=x, y=y, p=p, epsilon=epsilon, keepdim=keepdim + ) else: - legacy_distance = call_pairwise_distance_layer(x=x, - y=y, - p=p, - epsilon=epsilon, - keepdim=keepdim) + legacy_distance = call_pairwise_distance_layer( + x=x, y=y, p=p, epsilon=epsilon, keepdim=keepdim + ) legacy_ret = legacy_distance.numpy() paddle.fluid.framework._disable_legacy_dygraph() return legacy_ret class TestPairwiseDistance(unittest.TestCase): - def test_pairwise_distance(self): epsilon = 1e-6 all_shape = [[5], [100, 100]] @@ -159,81 +130,107 @@ def test_pairwise_distance(self): x_np = np.random.random(shape).astype(dtype) y_np = np.random.random(shape).astype(dtype) - static_ret = test_static(place, - x_np, - y_np, - p, - epsilon=epsilon, - keepdim=keepdim) - dygraph_ret = test_dygraph(place, - x_np, - y_np, - p, - epsilon=epsilon, - keepdim=keepdim) - legacy_ret = test_legacy_dygraph(place, - x_np, - y_np, - p, - epsilon=epsilon, - keepdim=keepdim) + static_ret = test_static( + place, + x_np, + y_np, + p, + epsilon=epsilon, + keepdim=keepdim, + ) + dygraph_ret = test_dygraph( + place, + x_np, + y_np, + p, + epsilon=epsilon, + keepdim=keepdim, + ) + legacy_ret = test_legacy_dygraph( + place, + x_np, + y_np, + p, + epsilon=epsilon, + keepdim=keepdim, + ) excepted_value = np_pairwise_distance( - x_np, y_np, p, epsilon=epsilon, keepdim=keepdim) - - self.assertEqual(static_ret.shape, - excepted_value.shape) - self.assertEqual(dygraph_ret.shape, - excepted_value.shape) - self.assertEqual(legacy_ret.shape, - excepted_value.shape) - - np.testing.assert_allclose(static_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(dygraph_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(legacy_ret, - excepted_value, - rtol=1e-05) - - static_functional_ret = test_static(place, - x_np, - y_np, - p, - epsilon=epsilon, - keepdim=keepdim) + x_np, y_np, p, epsilon=epsilon, keepdim=keepdim + ) + + self.assertEqual( + static_ret.shape, excepted_value.shape + ) + self.assertEqual( + dygraph_ret.shape, excepted_value.shape + ) + self.assertEqual( + legacy_ret.shape, excepted_value.shape + ) + + np.testing.assert_allclose( + static_ret, excepted_value, rtol=1e-05 + ) + np.testing.assert_allclose( + dygraph_ret, excepted_value, rtol=1e-05 + ) + np.testing.assert_allclose( + legacy_ret, excepted_value, rtol=1e-05 + ) + + static_functional_ret = test_static( + place, + x_np, + y_np, + p, + epsilon=epsilon, + keepdim=keepdim, + ) dygraph_functional_ret = test_dygraph( place, x_np, y_np, p, epsilon=epsilon, - keepdim=keepdim) + keepdim=keepdim, + ) legacy_functional_ret = test_legacy_dygraph( place, x_np, y_np, p, epsilon=epsilon, - keepdim=keepdim) - - self.assertEqual(static_functional_ret.shape, - excepted_value.shape) - self.assertEqual(dygraph_functional_ret.shape, - excepted_value.shape) - self.assertEqual(legacy_functional_ret.shape, - excepted_value.shape) - - np.testing.assert_allclose(static_functional_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(dygraph_functional_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(legacy_functional_ret, - excepted_value, - rtol=1e-05) + keepdim=keepdim, + ) + + self.assertEqual( + static_functional_ret.shape, + excepted_value.shape, + ) + self.assertEqual( + dygraph_functional_ret.shape, + excepted_value.shape, + ) + self.assertEqual( + legacy_functional_ret.shape, + excepted_value.shape, + ) + + np.testing.assert_allclose( + static_functional_ret, + excepted_value, + rtol=1e-05, + ) + np.testing.assert_allclose( + dygraph_functional_ret, + excepted_value, + rtol=1e-05, + ) + np.testing.assert_allclose( + legacy_functional_ret, + excepted_value, + rtol=1e-05, + ) def test_pairwise_distance_broadcast_1(self): shape_x = [100, 100] @@ -243,25 +240,18 @@ def test_pairwise_distance_broadcast_1(self): place = paddle.CPUPlace() x_np = np.random.random(shape_x).astype('float32') y_np = np.random.random(shape_y).astype('float32') - static_ret = test_static(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim) - dygraph_ret = test_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim) - legacy_ret = test_legacy_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim) - excepted_value = np_pairwise_distance(x_np, - y_np, - epsilon=epsilon, - keepdim=keepdim) + static_ret = test_static( + place=place, x_np=x_np, y_np=y_np, epsilon=epsilon, keepdim=keepdim + ) + dygraph_ret = test_dygraph( + place=place, x_np=x_np, y_np=y_np, epsilon=epsilon, keepdim=keepdim + ) + legacy_ret = test_legacy_dygraph( + place=place, x_np=x_np, y_np=y_np, epsilon=epsilon, keepdim=keepdim + ) + excepted_value = np_pairwise_distance( + x_np, y_np, epsilon=epsilon, keepdim=keepdim + ) self.assertEqual(static_ret.shape, excepted_value.shape) self.assertEqual(dygraph_ret.shape, excepted_value.shape) @@ -271,38 +261,44 @@ def test_pairwise_distance_broadcast_1(self): np.testing.assert_allclose(dygraph_ret, excepted_value, rtol=1e-05) np.testing.assert_allclose(legacy_ret, excepted_value, rtol=1e-05) - static_functional_ret = test_static(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim, - functional=True) - dygraph_functional_ret = test_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim, - functional=True) - legacy_functional_ret = test_legacy_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim, - functional=True) + static_functional_ret = test_static( + place=place, + x_np=x_np, + y_np=y_np, + epsilon=epsilon, + keepdim=keepdim, + functional=True, + ) + dygraph_functional_ret = test_dygraph( + place=place, + x_np=x_np, + y_np=y_np, + epsilon=epsilon, + keepdim=keepdim, + functional=True, + ) + legacy_functional_ret = test_legacy_dygraph( + place=place, + x_np=x_np, + y_np=y_np, + epsilon=epsilon, + keepdim=keepdim, + functional=True, + ) self.assertEqual(static_functional_ret.shape, excepted_value.shape) self.assertEqual(dygraph_functional_ret.shape, excepted_value.shape) self.assertEqual(legacy_functional_ret.shape, excepted_value.shape) - np.testing.assert_allclose(static_functional_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(dygraph_functional_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(legacy_functional_ret, - excepted_value, - rtol=1e-05) + np.testing.assert_allclose( + static_functional_ret, excepted_value, rtol=1e-05 + ) + np.testing.assert_allclose( + dygraph_functional_ret, excepted_value, rtol=1e-05 + ) + np.testing.assert_allclose( + legacy_functional_ret, excepted_value, rtol=1e-05 + ) def test_pairwise_distance_broadcast_2(self): shape_x = [100, 100] @@ -312,25 +308,18 @@ def test_pairwise_distance_broadcast_2(self): place = paddle.CPUPlace() x_np = np.random.random(shape_x).astype('float32') y_np = np.random.random(shape_y).astype('float32') - static_ret = test_static(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim) - dygraph_ret = test_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim) - legacy_ret = test_legacy_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim) - excepted_value = np_pairwise_distance(x_np, - y_np, - epsilon=epsilon, - keepdim=keepdim) + static_ret = test_static( + place=place, x_np=x_np, y_np=y_np, epsilon=epsilon, keepdim=keepdim + ) + dygraph_ret = test_dygraph( + place=place, x_np=x_np, y_np=y_np, epsilon=epsilon, keepdim=keepdim + ) + legacy_ret = test_legacy_dygraph( + place=place, x_np=x_np, y_np=y_np, epsilon=epsilon, keepdim=keepdim + ) + excepted_value = np_pairwise_distance( + x_np, y_np, epsilon=epsilon, keepdim=keepdim + ) self.assertEqual(static_ret.shape, excepted_value.shape) self.assertEqual(dygraph_ret.shape, excepted_value.shape) @@ -340,38 +329,44 @@ def test_pairwise_distance_broadcast_2(self): np.testing.assert_allclose(dygraph_ret, excepted_value, rtol=1e-05) np.testing.assert_allclose(legacy_ret, excepted_value, rtol=1e-05) - static_functional_ret = test_static(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim, - functional=True) - dygraph_functional_ret = test_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim, - functional=True) - legacy_functional_ret = test_legacy_dygraph(place=place, - x_np=x_np, - y_np=y_np, - epsilon=epsilon, - keepdim=keepdim, - functional=True) + static_functional_ret = test_static( + place=place, + x_np=x_np, + y_np=y_np, + epsilon=epsilon, + keepdim=keepdim, + functional=True, + ) + dygraph_functional_ret = test_dygraph( + place=place, + x_np=x_np, + y_np=y_np, + epsilon=epsilon, + keepdim=keepdim, + functional=True, + ) + legacy_functional_ret = test_legacy_dygraph( + place=place, + x_np=x_np, + y_np=y_np, + epsilon=epsilon, + keepdim=keepdim, + functional=True, + ) self.assertEqual(static_functional_ret.shape, excepted_value.shape) self.assertEqual(dygraph_functional_ret.shape, excepted_value.shape) self.assertEqual(legacy_functional_ret.shape, excepted_value.shape) - np.testing.assert_allclose(static_functional_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(dygraph_functional_ret, - excepted_value, - rtol=1e-05) - np.testing.assert_allclose(legacy_functional_ret, - excepted_value, - rtol=1e-05) + np.testing.assert_allclose( + static_functional_ret, excepted_value, rtol=1e-05 + ) + np.testing.assert_allclose( + dygraph_functional_ret, excepted_value, rtol=1e-05 + ) + np.testing.assert_allclose( + legacy_functional_ret, excepted_value, rtol=1e-05 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py index fb69986a348a7..a452f2b2b15d6 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel.py @@ -19,7 +19,12 @@ import os import subprocess -from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc +from paddle.distributed.utils.launch_utils import ( + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, +) def get_cluster_from_args(selected_gpus): @@ -47,10 +52,9 @@ def get_gpus(selected_gpus): return selected_gpus -def start_local_trainers_cpu(trainer_endpoints, - training_script, - training_script_args, - log_dir=None): +def start_local_trainers_cpu( + trainer_endpoints, training_script, training_script_args, log_dir=None +): current_env = copy.copy(os.environ.copy()) current_env.pop("http_proxy", None) current_env.pop("https_proxy", None) @@ -64,15 +68,16 @@ def start_local_trainers_cpu(trainer_endpoints, "PADDLE_TRAINER_ID": "%d" % rank_id, "PADDLE_CURRENT_ENDPOINT": "%s" % endpoint, "PADDLE_TRAINERS_NUM": "%d" % n_rank, - "PADDLE_TRAINER_ENDPOINTS": ",".join(trainer_endpoints) + "PADDLE_TRAINER_ENDPOINTS": ",".join(trainer_endpoints), } current_env.update(proc_env) print("trainer proc env:{}".format(current_env)) - assert os.getenv('WITH_COVERAGE', - 'OFF') == 'OFF', "Gloo don't support WITH_COVERAGE." + assert ( + os.getenv('WITH_COVERAGE', 'OFF') == 'OFF' + ), "Gloo don't support WITH_COVERAGE." cmd = "python -u " + training_script print("start trainer proc:{} env:{}".format(cmd, proc_env)) @@ -92,17 +97,19 @@ def start_local_trainers_cpu(trainer_endpoints, return procs -def start_local_trainers(cluster, - pod, - training_script, - training_script_args, - eager_mode=True, - log_dir=None): +def start_local_trainers( + cluster, + pod, + training_script, + training_script_args, + eager_mode=True, + log_dir=None, +): current_env = copy.copy(os.environ.copy()) - #paddle broadcast ncclUniqueId use socket, and - #proxy maybe make trainers unreachable, so delete them. - #if we set them to "", grpc will log error message "bad uri" - #so just delete them. + # paddle broadcast ncclUniqueId use socket, and + # proxy maybe make trainers unreachable, so delete them. + # if we set them to "", grpc will log error message "bad uri" + # so just delete them. current_env.pop("http_proxy", None) current_env.pop("https_proxy", None) @@ -113,7 +120,7 @@ def start_local_trainers(cluster, "PADDLE_TRAINER_ID": "%d" % t.rank, "PADDLE_CURRENT_ENDPOINT": "%s" % t.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), - "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()) + "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } if not eager_mode: @@ -146,10 +153,11 @@ def start_local_trainers(cluster, class TestMultipleGpus(unittest.TestCase): - def run_mnist_2gpu(self, target_file_name, eager_mode=True): - if not fluid.core.is_compiled_with_cuda( - ) or fluid.core.get_cuda_device_count() == 0: + if ( + not fluid.core.is_compiled_with_cuda() + or fluid.core.get_cuda_device_count() == 0 + ): return selected_gpus = get_gpus('0,1') @@ -158,11 +166,13 @@ def run_mnist_2gpu(self, target_file_name, eager_mode=True): cluster, pod = get_cluster_from_args(selected_gpus) - procs = start_local_trainers(cluster, - pod, - eager_mode=eager_mode, - training_script=target_file_name, - training_script_args=[]) + procs = start_local_trainers( + cluster, + pod, + eager_mode=eager_mode, + training_script=target_file_name, + training_script_args=[], + ) while True: alive = watch_local_trainers(procs, cluster.trainers_endpoints()) @@ -174,15 +184,17 @@ def run_mnist_2gpu(self, target_file_name, eager_mode=True): class TestMultipleWithGloo(unittest.TestCase): - def run_mnist_2cpu(self, target_file_name): cluster, pod = get_cluster_from_args( - [0, 1]) #tmp use. for getting trainer_nranks() + [0, 1] + ) # tmp use. for getting trainer_nranks() - procs = start_local_trainers_cpu(cluster.trainers_endpoints(), - training_script=target_file_name, - training_script_args=[]) + procs = start_local_trainers_cpu( + cluster.trainers_endpoints(), + training_script=target_file_name, + training_script_args=[], + ) while True: alive = watch_local_trainers(procs, cluster.trainers_nranks()) @@ -194,22 +206,21 @@ def run_mnist_2cpu(self, target_file_name): class TestDataParallelGradientCheck(TestMultipleGpus): - def test_multiple_gpus_dynamic(self): - self.run_mnist_2gpu('parallel_dygraph_gradient_check.py', - eager_mode=False) + self.run_mnist_2gpu( + 'parallel_dygraph_gradient_check.py', eager_mode=False + ) class TestDataParallelWithPyLayer(TestMultipleGpus): - def test_parallel_dygraph_dataparallel_with_pylayer(self): self.run_mnist_2gpu('parallel_dygraph_dataparallel_with_pylayer.py') - self.run_mnist_2gpu('parallel_dygraph_dataparallel_with_pylayer.py', - eager_mode=False) + self.run_mnist_2gpu( + 'parallel_dygraph_dataparallel_with_pylayer.py', eager_mode=False + ) class TestGradientCheckInEagerMode(TestMultipleGpus): - def test_multiple_gpus_dynamic(self): self.run_mnist_2gpu('parallel_dygraph_gradient_check_in_eager_mode.py') diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel_cpuonly.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel_cpuonly.py index 8862a3310af94..540c668a6f8ed 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel_cpuonly.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_dataparallel_cpuonly.py @@ -18,7 +18,12 @@ import os import subprocess -from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc +from paddle.distributed.utils.launch_utils import ( + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, +) def get_cluster_from_args(selected_gpus): @@ -46,16 +51,14 @@ def get_gpus(selected_gpus): return selected_gpus -def start_local_trainers(cluster, - pod, - training_script, - training_script_args, - log_dir=None): +def start_local_trainers( + cluster, pod, training_script, training_script_args, log_dir=None +): current_env = copy.copy(os.environ.copy()) - #paddle broadcast ncclUniqueId use socket, and - #proxy maybe make trainers unreachable, so delete them. - #if we set them to "", grpc will log error message "bad uri" - #so just delete them. + # paddle broadcast ncclUniqueId use socket, and + # proxy maybe make trainers unreachable, so delete them. + # if we set them to "", grpc will log error message "bad uri" + # so just delete them. current_env.pop("http_proxy", None) current_env.pop("https_proxy", None) @@ -69,8 +72,7 @@ def start_local_trainers(cluster, "MASTER_ADDR": "127.0.0.1", "MASTER_PORT": "6170", "NCCL_DEBUG": "INFO", - "PADDLE_DISTRI_BACKEND": - "gloo", # make init_parallel_env get 'gloo' argument. + "PADDLE_DISTRI_BACKEND": "gloo", # make init_parallel_env get 'gloo' argument. } current_env.update(proc_env) @@ -100,10 +102,9 @@ def start_local_trainers(cluster, class TestMultipleGpus(unittest.TestCase): - def run_mnist_2gpu(self, target_file_name): - #if not fluid.core.is_compiled_with_cuda( - #) or fluid.core.get_cuda_device_count() == 0: + # if not fluid.core.is_compiled_with_cuda( + # ) or fluid.core.get_cuda_device_count() == 0: # return selected_gpus = get_gpus('0,1') @@ -111,10 +112,12 @@ def run_mnist_2gpu(self, target_file_name): pod = None cluster, pod = get_cluster_from_args(selected_gpus) - procs = start_local_trainers(cluster, - pod, - training_script=target_file_name, - training_script_args=[]) + procs = start_local_trainers( + cluster, + pod, + training_script=target_file_name, + training_script_args=[], + ) while True: alive = watch_local_trainers(procs, cluster.trainers_nranks()) @@ -126,13 +129,11 @@ def run_mnist_2gpu(self, target_file_name): class TestDataParallelGradientCheck(TestMultipleGpus): - def test_multiple_gpus_dynamic(self): self.run_mnist_2gpu('parallel_dygraph_gradient_check.py') class TestDataParallelGradientCheckInEagerMode(TestMultipleGpus): - def test_multiple_gpus_dynamic(self): self.run_mnist_2gpu('parallel_dygraph_gradient_check_in_eager_mode.py') diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_diff_length_gloo.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_diff_length_gloo.py index 278ea58de37a0..bc1c5bf4cfd9b 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_diff_length_gloo.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_diff_length_gloo.py @@ -21,7 +21,6 @@ class TestParallelDygraphSparseEmdedding_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True @@ -29,10 +28,12 @@ def _setup_config(self): self._diff_batch = True def test_sparse_embedding(self): - self.check_with_place("parallel_dygraph_sparse_embedding.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_sparse_embedding.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_gloo.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_gloo.py index c9e4d669179c7..ef56153af7feb 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_gloo.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_gloo.py @@ -21,31 +21,33 @@ class TestParallelDygraphSparseEmdedding_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True self._dygraph = True def test_sparse_embedding(self): - self.check_with_place("parallel_dygraph_sparse_embedding.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_sparse_embedding.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphSparseEmdeddingFP64_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True self._dygraph = True def test_sparse_embedding_fp64(self): - self.check_with_place("parallel_dygraph_sparse_embedding_fp64.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_sparse_embedding_fp64.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_over_height_gloo.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_over_height_gloo.py index 76f2d3ecaa88f..c56f63893aaec 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_over_height_gloo.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_sparse_embedding_over_height_gloo.py @@ -21,7 +21,6 @@ class TestParallelDygraphSparseEmdeddingOverHeight_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True @@ -32,7 +31,8 @@ def test_sparse_embedding(self): "parallel_dygraph_sparse_embedding_over_height.py", delta=1e-7, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_transformer_gloo.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_transformer_gloo.py index 6089ef5d8292f..1af28e65cd7ed 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_transformer_gloo.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_transformer_gloo.py @@ -22,21 +22,21 @@ class TestParallelDygraphTransformer_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True self._dygraph = True def test_transformer(self): - self.check_with_place("parallel_dygraph_transformer.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_transformer.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphTransformerAccGrad_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True @@ -46,10 +46,12 @@ def _setup_config(self): def test_transformer(self): if fluid.core.is_compiled_with_cuda(): - self.check_with_place("parallel_dygraph_transformer.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_transformer.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_unused_variables_gloo.py b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_unused_variables_gloo.py index 0cadd9a61b777..9278ff7280e27 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_dygraph_unused_variables_gloo.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_dygraph_unused_variables_gloo.py @@ -21,45 +21,48 @@ class TestParallelDygraphUnusedVar_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True self._dygraph = True def test_net(self): - self.check_with_place("parallel_dygraph_unused_variables.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_unused_variables.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphNoVar_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True self._dygraph = True def test_net(self): - self.check_with_place("parallel_dygraph_none_var.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_none_var.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) class TestParallelDygraphSharedUnusedVariables_GLOO(TestDistBase): - def _setup_config(self): self._sync_mode = False self._gloo_mode = True self._dygraph = True def test_mnist(self): - self.check_with_place("parallel_dygraph_shared_unused_var.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "parallel_dygraph_shared_unused_var.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py index 7ef548536a33c..7f30c15735ffb 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py @@ -33,27 +33,42 @@ embedding_name = 'emb' -def db_lstm(word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, - is_sparse, **ignored): +def db_lstm( + word, + predicate, + ctx_n2, + ctx_n1, + ctx_0, + ctx_p1, + ctx_p2, + mark, + is_sparse, + **ignored +): # 8 features - predicate_embedding = fluid.layers.embedding(input=predicate, - is_sparse=is_sparse, - size=[pred_dict_len, word_dim], - dtype='float32', - param_attr='vemb') - - mark_embedding = fluid.layers.embedding(input=mark, - is_sparse=is_sparse, - size=[mark_dict_len, mark_dim], - dtype='float32') + predicate_embedding = fluid.layers.embedding( + input=predicate, + is_sparse=is_sparse, + size=[pred_dict_len, word_dim], + dtype='float32', + param_attr='vemb', + ) + + mark_embedding = fluid.layers.embedding( + input=mark, + is_sparse=is_sparse, + size=[mark_dict_len, mark_dim], + dtype='float32', + ) word_input = [word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2] emb_layers = [ - fluid.layers.embedding(size=[word_dict_len, word_dim], - is_sparse=is_sparse, - input=x, - param_attr=fluid.ParamAttr(name=embedding_name, - trainable=False)) + fluid.layers.embedding( + size=[word_dict_len, word_dim], + is_sparse=is_sparse, + input=x, + param_attr=fluid.ParamAttr(name=embedding_name, trainable=False), + ) for x in word_input ] # TODO(zcd): if the parameter is not trainable, the @@ -71,92 +86,98 @@ def db_lstm(word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, hidden_0 = fluid.layers.sums(input=hidden_0_layers) - lstm_0 = fluid.layers.dynamic_lstm(input=hidden_0, - size=hidden_dim, - candidate_activation='relu', - gate_activation='sigmoid', - cell_activation='sigmoid') + lstm_0 = fluid.layers.dynamic_lstm( + input=hidden_0, + size=hidden_dim, + candidate_activation='relu', + gate_activation='sigmoid', + cell_activation='sigmoid', + ) # stack L-LSTM and R-LSTM with direct edges input_tmp = [hidden_0, lstm_0] for i in range(1, depth): - mix_hidden = fluid.layers.sums(input=[ - fluid.layers.fc(input=input_tmp[0], size=hidden_dim, act='tanh'), - fluid.layers.fc(input=input_tmp[1], size=hidden_dim, act='tanh') - ]) - - lstm = fluid.layers.dynamic_lstm(input=mix_hidden, - size=hidden_dim, - candidate_activation='relu', - gate_activation='sigmoid', - cell_activation='sigmoid', - is_reverse=((i % 2) == 1)) + mix_hidden = fluid.layers.sums( + input=[ + fluid.layers.fc( + input=input_tmp[0], size=hidden_dim, act='tanh' + ), + fluid.layers.fc( + input=input_tmp[1], size=hidden_dim, act='tanh' + ), + ] + ) + + lstm = fluid.layers.dynamic_lstm( + input=mix_hidden, + size=hidden_dim, + candidate_activation='relu', + gate_activation='sigmoid', + cell_activation='sigmoid', + is_reverse=((i % 2) == 1), + ) input_tmp = [mix_hidden, lstm] - feature_out = fluid.layers.sums(input=[ - fluid.layers.fc(input=input_tmp[0], size=label_dict_len, act='tanh'), - fluid.layers.fc(input=input_tmp[1], size=label_dict_len, act='tanh') - ]) + feature_out = fluid.layers.sums( + input=[ + fluid.layers.fc( + input=input_tmp[0], size=label_dict_len, act='tanh' + ), + fluid.layers.fc( + input=input_tmp[1], size=label_dict_len, act='tanh' + ), + ] + ) return feature_out class TestCRFModel(unittest.TestCase): - - def check_network_convergence(self, - is_sparse, - build_strategy=None, - use_cuda=True): + def check_network_convergence( + self, is_sparse, build_strategy=None, use_cuda=True + ): os.environ['CPU_NUM'] = str(4) main = fluid.Program() startup = fluid.Program() scope = fluid.Scope() with fluid.scope_guard(scope): with fluid.program_guard(main, startup): - word = fluid.layers.data(name='word_data', - shape=[1], - dtype='int64', - lod_level=1) - predicate = fluid.layers.data(name='verb_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_n2 = fluid.layers.data(name='ctx_n2_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_n1 = fluid.layers.data(name='ctx_n1_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_0 = fluid.layers.data(name='ctx_0_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_p1 = fluid.layers.data(name='ctx_p1_data', - shape=[1], - dtype='int64', - lod_level=1) - ctx_p2 = fluid.layers.data(name='ctx_p2_data', - shape=[1], - dtype='int64', - lod_level=1) - mark = fluid.layers.data(name='mark_data', - shape=[1], - dtype='int64', - lod_level=1) + word = fluid.layers.data( + name='word_data', shape=[1], dtype='int64', lod_level=1 + ) + predicate = fluid.layers.data( + name='verb_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_n2 = fluid.layers.data( + name='ctx_n2_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_n1 = fluid.layers.data( + name='ctx_n1_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_0 = fluid.layers.data( + name='ctx_0_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_p1 = fluid.layers.data( + name='ctx_p1_data', shape=[1], dtype='int64', lod_level=1 + ) + ctx_p2 = fluid.layers.data( + name='ctx_p2_data', shape=[1], dtype='int64', lod_level=1 + ) + mark = fluid.layers.data( + name='mark_data', shape=[1], dtype='int64', lod_level=1 + ) feature_out = db_lstm(**locals()) - target = fluid.layers.data(name='target', - shape=[1], - dtype='int64', - lod_level=1) + target = fluid.layers.data( + name='target', shape=[1], dtype='int64', lod_level=1 + ) crf_cost = fluid.layers.linear_chain_crf( input=feature_out, label=target, - param_attr=fluid.ParamAttr(name='crfw', learning_rate=1e-1)) + param_attr=fluid.ParamAttr(name='crfw', learning_rate=1e-1), + ) avg_cost = paddle.mean(crf_cost) sgd_optimizer = fluid.optimizer.SGD( @@ -164,41 +185,63 @@ def check_network_convergence(self, learning_rate=0.01, decay_steps=100000, decay_rate=0.5, - staircase=True)) + staircase=True, + ) + ) sgd_optimizer.minimize(avg_cost) - train_data = paddle.batch(paddle.reader.shuffle( - paddle.dataset.conll05.test(), buf_size=8192), - batch_size=8) + train_data = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.conll05.test(), buf_size=8192 + ), + batch_size=8, + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup) train_cp = compiler.CompiledProgram(main).with_data_parallel( - loss_name=avg_cost.name, build_strategy=build_strategy) - - feeder = fluid.DataFeeder(feed_list=[ - word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, predicate, - mark, target - ], - place=fluid.CPUPlace()) + loss_name=avg_cost.name, build_strategy=build_strategy + ) + + feeder = fluid.DataFeeder( + feed_list=[ + word, + ctx_n2, + ctx_n1, + ctx_0, + ctx_p1, + ctx_p2, + predicate, + mark, + target, + ], + place=fluid.CPUPlace(), + ) data = train_data() for i in range(4): cur_batch = next(data) print( - exe.run(train_cp, - feed=feeder.feed(cur_batch), - fetch_list=[avg_cost.name])[0]) + exe.run( + train_cp, + feed=feeder.feed(cur_batch), + fetch_list=[avg_cost.name], + )[0] + ) def _new_build_strategy(self, use_reduce=False): build_strategy = fluid.BuildStrategy() if use_reduce: - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce + build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + ) else: - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.AllReduce + build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.AllReduce + ) return build_strategy @@ -207,46 +250,54 @@ def test_update_sparse_parameter_all_reduce(self): self.check_network_convergence( is_sparse=True, build_strategy=self._new_build_strategy(), - use_cuda=True) + use_cuda=True, + ) self.check_network_convergence( is_sparse=True, build_strategy=self._new_build_strategy(), - use_cuda=False) + use_cuda=False, + ) def test_update_dense_parameter_all_reduce(self): if core.is_compiled_with_cuda(): self.check_network_convergence( is_sparse=False, build_strategy=self._new_build_strategy(), - use_cuda=True) + use_cuda=True, + ) self.check_network_convergence( is_sparse=False, build_strategy=self._new_build_strategy(), - use_cuda=False) + use_cuda=False, + ) def test_update_sparse_parameter_reduce(self): if core.is_compiled_with_cuda(): self.check_network_convergence( is_sparse=True, build_strategy=self._new_build_strategy(use_reduce=True), - use_cuda=True) + use_cuda=True, + ) self.check_network_convergence( is_sparse=True, build_strategy=self._new_build_strategy(use_reduce=True), - use_cuda=False) + use_cuda=False, + ) def test_update_dense_parameter_reduce(self): if core.is_compiled_with_cuda(): self.check_network_convergence( is_sparse=False, build_strategy=self._new_build_strategy(use_reduce=True), - use_cuda=True) + use_cuda=True, + ) self.check_network_convergence( is_sparse=False, build_strategy=self._new_build_strategy(use_reduce=True), - use_cuda=False) + use_cuda=False, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_drop_scope.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_drop_scope.py index 6ff24c19f46ee..9c8d342993739 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_drop_scope.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_drop_scope.py @@ -20,7 +20,6 @@ class TestParallelExecutorDropExeScope(unittest.TestCase): - def check_drop_scope(self, use_cuda=True): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() @@ -42,14 +41,18 @@ def check_drop_scope(self, use_cuda=True): exec_strateg = fluid.ExecutionStrategy() exec_strateg.num_iteration_per_drop_scope = 10 - train_exe = fluid.ParallelExecutor(use_cuda=use_cuda, - main_program=train_program, - loss_name=loss.name, - exec_strategy=exec_strateg) - test_exe = fluid.ParallelExecutor(use_cuda=use_cuda, - main_program=test_program, - share_vars_from=train_exe, - exec_strategy=exec_strateg) + train_exe = fluid.ParallelExecutor( + use_cuda=use_cuda, + main_program=train_program, + loss_name=loss.name, + exec_strategy=exec_strateg, + ) + test_exe = fluid.ParallelExecutor( + use_cuda=use_cuda, + main_program=test_program, + share_vars_from=train_exe, + exec_strategy=exec_strateg, + ) x = numpy.random.random(size=(10, 1)).astype('float32') train_exe.run(feed={"X": x}, fetch_list=[loss.name]) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_dry_run.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_dry_run.py index d2feddffc85da..51b234c371914 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_dry_run.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_dry_run.py @@ -23,16 +23,18 @@ class TestBase(unittest.TestCase): - - def main(self, - network_func, - iter=10, - iter_per_pe=10, - use_gpu=True, - use_experimental_executor=False): + def main( + self, + network_func, + iter=10, + iter_per_pe=10, + use_gpu=True, + use_experimental_executor=False, + ): if use_gpu and not fluid.core.is_compiled_with_cuda(): logging.warning( - "Paddle is not compiled with CUDA, skip GPU unittests") + "Paddle is not compiled with CUDA, skip GPU unittests" + ) return main_prog = fluid.Program() @@ -42,28 +44,34 @@ def main(self, with fluid.scope_guard(scope): loss = network_func() exe = fluid.Executor( - fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace()) + fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace() + ) exe.run(startup_prog) exe_strategy = fluid.ExecutionStrategy() exe_strategy._dry_run = True - exe_strategy.use_experimental_executor = use_experimental_executor + exe_strategy.use_experimental_executor = ( + use_experimental_executor + ) train_cp = compiler.CompiledProgram( - main_prog).with_data_parallel(loss_name=loss.name, - exec_strategy=exe_strategy) + main_prog + ).with_data_parallel( + loss_name=loss.name, exec_strategy=exe_strategy + ) for _ in range(iter): for _ in range(iter_per_pe): exe.run(train_cp) class TestMNISTDryRun(TestBase): - def test_mnist_dry_run(self): for use_gpu in (False, True): for use_experimental_executor in (False, True): - self.main(network_func=TestMNISTDryRun.network_func, - use_gpu=use_gpu, - use_experimental_executor=use_experimental_executor) + self.main( + network_func=TestMNISTDryRun.network_func, + use_gpu=use_gpu, + use_experimental_executor=use_experimental_executor, + ) @staticmethod def network_func(): diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_feed_persistable_var.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_feed_persistable_var.py index 622095b85a7ce..4b5781921e296 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_feed_persistable_var.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_feed_persistable_var.py @@ -23,26 +23,27 @@ class TestFeedPersistableVar(unittest.TestCase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) batch_size = 4 - cls.img, cls.label = init_data(batch_size, - img_shape=[784], - label_range=9) + cls.img, cls.label = init_data( + batch_size, img_shape=[784], label_range=9 + ) cls.feed_dict = { 'image': cls.img, 'label': cls.label, - 'learning_rate': numpy.array([1.0]).astype("float32") + 'learning_rate': numpy.array([1.0]).astype("float32"), } def optimizer(self): - learning_rate = fluid.layers.create_global_var(name="learning_rate", - shape=[1], - value=1.0, - dtype='float32', - persistable=True) + learning_rate = fluid.layers.create_global_var( + name="learning_rate", + shape=[1], + value=1.0, + dtype='float32', + persistable=True, + ) optimizer = fluid.optimizer.SGD(learning_rate=learning_rate) return optimizer @@ -62,7 +63,8 @@ def check_feed_persistable_var(self, feed_dict, use_cuda=False): exe.run(program=startup) compiled_prog = fluid.compiler.CompiledProgram( - main).with_data_parallel(loss_name=loss.name) + main + ).with_data_parallel(loss_name=loss.name) exe.run(program=compiled_prog, feed=feed_dict) @@ -70,12 +72,14 @@ def test_feed_persistable_var(self): self.check_feed_persistable_var(self.feed_dict) self.check_feed_persistable_var(self.feed_dict, use_cuda=True) - self.feed_dict['learning_rate'] = numpy.array([1.0, - 1.0]).astype("float32") + self.feed_dict['learning_rate'] = numpy.array([1.0, 1.0]).astype( + "float32" + ) self.check_feed_persistable_var(self.feed_dict, use_cuda=True) - self.feed_dict['learning_rate'] = numpy.array([1.0, - 1.0]).astype("float32") + self.feed_dict['learning_rate'] = numpy.array([1.0, 1.0]).astype( + "float32" + ) run = partial(self.check_feed_persistable_var, self.feed_dict) self.assertRaises(RuntimeError, run) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py index 3695de024f4c6..950ff45e86f39 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py @@ -37,24 +37,25 @@ def Lenet(data, class_dim): class TestFetchAndFeed(unittest.TestCase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) - def parallel_exe(self, - use_cuda, - run_parallel_exe, - use_faster_executor=False, - num_threads=4, - seed=1): + def parallel_exe( + self, + use_cuda, + run_parallel_exe, + use_faster_executor=False, + num_threads=4, + seed=1, + ): main_program = fluid.Program() startup = fluid.Program() startup.random_seed = seed with fluid.program_guard(main_program, startup): - data = fluid.layers.data(name='image', - shape=[3, 224, 224], - dtype='float32') + data = fluid.layers.data( + name='image', shape=[3, 224, 224], dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') out = Lenet(data, class_dim=102) loss = fluid.layers.cross_entropy(input=out, label=label) @@ -62,14 +63,15 @@ def parallel_exe(self, opt = fluid.optimizer.Momentum( learning_rate=0.1, momentum=0.9, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) opt.minimize(loss) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup) - #FIXME force disable enable_inplace and memory_optimize to pass the unittest + # FIXME force disable enable_inplace and memory_optimize to pass the unittest build_strategy = fluid.BuildStrategy() build_strategy.enable_inplace = False build_strategy.memory_optimize = False @@ -79,56 +81,66 @@ def parallel_exe(self, train_cp = compiler.CompiledProgram(main_program).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy, - exec_strategy=exec_strategy) + exec_strategy=exec_strategy, + ) run_parallel_exe(train_cp, exe, use_cuda, data, label, loss) - def run_parallel_exe_with_fetch(self, compiled_program, exe, use_cuda, data, - label, loss): - + def run_parallel_exe_with_fetch( + self, compiled_program, exe, use_cuda, data, label, loss + ): def get_data(batch_size=8): np.random.seed(5) while True: img = np.random.random(size=[batch_size, 3, 224, 224]).astype( - np.float32) + np.float32 + ) l = (np.random.random(size=[batch_size, 1]) * 10).astype( - np.int64) + np.int64 + ) yield img, l fetch_list = [] all_vars = compiled_program._program.global_block().vars for k, v in all_vars.items(): - if ('tmp' not in k) and ( - k[0] != '_' or v.persistable - ) and v.type == core.VarDesc.VarType.LOD_TENSOR: + if ( + ('tmp' not in k) + and (k[0] != '_' or v.persistable) + and v.type == core.VarDesc.VarType.LOD_TENSOR + ): fetch_list.append(k) for batch_id, img_label in enumerate(get_data()): img, l = img_label train_inputs = {data.name: img, label.name: l} - ret = exe.run(compiled_program, - fetch_list=fetch_list, - feed=train_inputs, - return_numpy=True) + ret = exe.run( + compiled_program, + fetch_list=fetch_list, + feed=train_inputs, + return_numpy=True, + ) for i in range(len(fetch_list)): - assert not math.isnan(np.sum(ret[i])) and \ - not math.isinf(np.sum(ret[i])) + assert not math.isnan(np.sum(ret[i])) and not math.isinf( + np.sum(ret[i]) + ) if batch_id == 2: break - def run_parallel_exe_with_feed(self, compiled_program, exe, use_cuda, data, - label, loss): - + def run_parallel_exe_with_feed( + self, compiled_program, exe, use_cuda, data, label, loss + ): def get_data(batch_size=8): np.random.seed(5) while True: train_data = [] for _ in range(batch_size): img = np.random.random(size=[1, 3, 224, 224]).astype( - np.float32) + np.float32 + ) label = (np.random.random(size=[1, 1]) * 10).astype( - np.int64) + np.int64 + ) train_data.append([img, label]) yield train_data @@ -137,37 +149,45 @@ def get_data(batch_size=8): reader = feeder.decorate_reader(get_data, multi_devices=True) for batch_id, data in enumerate(reader()): - loss_np = exe.run(compiled_program, - feed=data, - fetch_list=[loss.name])[0] + loss_np = exe.run( + compiled_program, feed=data, fetch_list=[loss.name] + )[0] print(batch_id, loss_np) if batch_id == 2: break def check_executor(self, use_faster_executor=False, num_threads=4): if core.is_compiled_with_cuda(): - self.parallel_exe(use_cuda=True, - run_parallel_exe=self.run_parallel_exe_with_fetch, - use_faster_executor=use_faster_executor, - num_threads=num_threads) - self.parallel_exe(use_cuda=False, - run_parallel_exe=self.run_parallel_exe_with_fetch, - use_faster_executor=use_faster_executor, - num_threads=num_threads) + self.parallel_exe( + use_cuda=True, + run_parallel_exe=self.run_parallel_exe_with_fetch, + use_faster_executor=use_faster_executor, + num_threads=num_threads, + ) + self.parallel_exe( + use_cuda=False, + run_parallel_exe=self.run_parallel_exe_with_fetch, + use_faster_executor=use_faster_executor, + num_threads=num_threads, + ) def test_fetch(self): for use_faster_executor in {True, False}: - self.check_executor(use_faster_executor=use_faster_executor, - num_threads=4) - self.check_executor(use_faster_executor=use_faster_executor, - num_threads=1) + self.check_executor( + use_faster_executor=use_faster_executor, num_threads=4 + ) + self.check_executor( + use_faster_executor=use_faster_executor, num_threads=1 + ) def test_feed(self): if core.is_compiled_with_cuda(): - self.parallel_exe(use_cuda=True, - run_parallel_exe=self.run_parallel_exe_with_feed) - self.parallel_exe(use_cuda=False, - run_parallel_exe=self.run_parallel_exe_with_feed) + self.parallel_exe( + use_cuda=True, run_parallel_exe=self.run_parallel_exe_with_feed + ) + self.parallel_exe( + use_cuda=False, run_parallel_exe=self.run_parallel_exe_with_feed + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_isolated_var.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_isolated_var.py index 3d2146ab36baa..9cfd502b3a376 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_isolated_var.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_isolated_var.py @@ -25,7 +25,6 @@ def enable_parallel_ssa_executor(enabled=True): class TestParallelExecutorFetchIsolatedVarBase(unittest.TestCase): - def build_network(self, is_training): x = fluid.data(name='x', shape=[-1, 10], dtype='float32') y = fluid.data(name='y', shape=[-1, 10], dtype='float32') @@ -55,19 +54,30 @@ def test_main(self): for use_experimental_executor in [False, True]: for use_parallel_ssa_executor in [False, True]: func = lambda: self.run_impl( - use_gpu, dev_cnt, is_training, + use_gpu, + dev_cnt, + is_training, use_experimental_executor, - use_parallel_ssa_executor) + use_parallel_ssa_executor, + ) self.run_func_with_guard(func) - def run_impl(self, use_gpu, dev_cnt, is_training, use_experimental_executor, - use_parallel_ssa_executor): + def run_impl( + self, + use_gpu, + dev_cnt, + is_training, + use_experimental_executor, + use_parallel_ssa_executor, + ): paddle.enable_static() enable_parallel_ssa_executor(use_parallel_ssa_executor) if fluid.is_compiled_with_cuda(): - if fluid.core.globals( - )['FLAGS_enable_parallel_graph'] and not use_gpu: + if ( + fluid.core.globals()['FLAGS_enable_parallel_graph'] + and not use_gpu + ): return # windows has only 1 GPU if use_gpu and dev_cnt > 1 and os.name == "nt": @@ -85,22 +95,23 @@ def run_impl(self, use_gpu, dev_cnt, is_training, use_experimental_executor, exe.run(fluid.default_startup_program()) prog = fluid.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - loss_name=loss_name, - exec_strategy=self.exec_strategy(use_experimental_executor), - places=places) + fluid.default_main_program() + ).with_data_parallel( + loss_name=loss_name, + exec_strategy=self.exec_strategy(use_experimental_executor), + places=places, + ) BATCH_SIZE = 8 * dev_cnt for _ in range(10): x_np = np.random.random(size=[BATCH_SIZE, 10]).astype('float32') y_np = np.random.random(size=[BATCH_SIZE, 10]).astype('float32') - _, y_np_fetch = exe.run(prog, - feed={ - 'x': x_np, - 'y': y_np - }, - fetch_list=[loss, isolated_var]) + _, y_np_fetch = exe.run( + prog, + feed={'x': x_np, 'y': y_np}, + fetch_list=[loss, isolated_var], + ) np.testing.assert_array_equal(y_np, y_np_fetch) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_fix_op_run_order.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_fix_op_run_order.py index 80da6b5ac61c8..4dcf4673a09ef 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_fix_op_run_order.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_fix_op_run_order.py @@ -21,7 +21,6 @@ class TestFixOpRunOrder(unittest.TestCase): - def setUp(self): paddle.enable_static() paddle.seed(1) @@ -30,8 +29,11 @@ def setUp(self): fluid.set_flags({'FLAGS_cudnn_deterministic': 1}) def get_place(self): - return paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + return ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) def get_feed(self): batch_size = 4 @@ -44,12 +46,12 @@ def create_model(self, fix_op_run_order): startup_prog = paddle.static.Program() scope = paddle.static.Scope() with paddle.static.program_guard(main_prog, startup_prog): - image = paddle.static.data(name="image", - shape=[None, 3, 224, 224], - dtype="float32") - label = paddle.static.data(name="label", - shape=[None, 1], - dtype="int64") + image = paddle.static.data( + name="image", shape=[None, 3, 224, 224], dtype="float32" + ) + label = paddle.static.data( + name="label", shape=[None, 1], dtype="int64" + ) model = resnet18() pred = model(image) loss_fn = CrossEntropyLoss() @@ -64,7 +66,8 @@ def create_model(self, fix_op_run_order): main_prog = paddle.static.CompiledProgram(main_prog).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy, - places=[self.get_place()]) + places=[self.get_place()], + ) exe = paddle.static.Executor(self.get_place()) with paddle.static.scope_guard(scope): diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_inference_feed_partial_data.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_inference_feed_partial_data.py index 926201e6eaaf4..37ccb429b6e96 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_inference_feed_partial_data.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_inference_feed_partial_data.py @@ -18,7 +18,6 @@ class TestInferencePartialFeed(unittest.TestCase): - def setUp(self): self.iterations = 10 self.size = 10 @@ -44,12 +43,15 @@ def run_network(self, places, use_split, has_persistable): exe.run(startup_prog) prog = fluid.CompiledProgram(main_prog).with_data_parallel( - places=places) + places=places + ) gen_random = lambda shape: np.random.uniform( - low=-1.0, high=1.0, size=shape).astype('float32') + low=-1.0, high=1.0, size=shape + ).astype('float32') assert_result = lambda feed, result: np.testing.assert_array_equal( - np.maximum(0, feed), result) + np.maximum(0, feed), result + ) def assert_merged_unmerged(merged, unmerged): unmerged = np.concatenate(unmerged, axis=0) @@ -68,10 +70,16 @@ def feed_split_test(): fetch_list = [relu_x, relu_y, relu_lr] relu_x_np, relu_y_np, relu_lr_np = exe.run( - prog, feed=feed, fetch_list=fetch_list, return_merged=True) + prog, feed=feed, fetch_list=fetch_list, return_merged=True + ) - relu_x_np_unmerged, relu_y_np_unmerged, relu_lr_np_unmerged = exe.run( - prog, feed=feed, fetch_list=fetch_list, return_merged=False) + ( + relu_x_np_unmerged, + relu_y_np_unmerged, + relu_lr_np_unmerged, + ) = exe.run( + prog, feed=feed, fetch_list=fetch_list, return_merged=False + ) assert_merged_unmerged(relu_x_np, relu_x_np_unmerged) assert_merged_unmerged(relu_y_np, relu_y_np_unmerged) @@ -99,24 +107,28 @@ def feed_list_test(): y_np_list.append(y_np) lr_np_list.append(lr_np) - feed_list.append({ - x.name: x_np, - y.name: y_np, - lr.name: lr_np - }) + feed_list.append( + {x.name: x_np, y.name: y_np, lr.name: lr_np} + ) fetch_list = [relu_x, relu_y, relu_lr] relu_x_np, relu_y_np, relu_lr_np = exe.run( prog, feed=feed_list, fetch_list=fetch_list, - return_merged=True) - - relu_x_np_unmerged, relu_y_np_unmerged, relu_lr_np_unmerged = exe.run( + return_merged=True, + ) + + ( + relu_x_np_unmerged, + relu_y_np_unmerged, + relu_lr_np_unmerged, + ) = exe.run( prog, feed=feed_list, fetch_list=fetch_list, - return_merged=False) + return_merged=False, + ) assert_merged_unmerged(relu_x_np, relu_x_np_unmerged) assert_merged_unmerged(relu_y_np, relu_y_np_unmerged) @@ -144,20 +156,18 @@ def test_main(self): for p in places: for has_persistable in [False, True]: for use_split in [False, True]: - self.run_network(p, - use_split=use_split, - has_persistable=has_persistable) + self.run_network( + p, use_split=use_split, has_persistable=has_persistable + ) class TestInferencePartialFeedUsingDataLoader(unittest.TestCase): - def setUp(self): self.epoch_num = 3 self.batch_num = 101 # a prime number self.batch_size = 32 def create_reader(self): - def __impl__(): for _ in range(self.batch_num): yield np.random.random([self.batch_size, 1]).astype('float32'), @@ -167,10 +177,9 @@ def __impl__(): def run_network(self, iterable, use_cuda, drop_last): x = fluid.data(shape=[None, 1], name='x', dtype='float32') places = fluid.cuda_places() if use_cuda else fluid.cpu_places(4) - loader = fluid.io.DataLoader.from_generator(feed_list=[x], - capacity=16, - iterable=iterable, - drop_last=drop_last) + loader = fluid.io.DataLoader.from_generator( + feed_list=[x], capacity=16, iterable=iterable, drop_last=drop_last + ) y = fluid.layers.fc(x, size=10) loss = fluid.layers.reduce_mean(y) @@ -178,17 +187,18 @@ def run_network(self, iterable, use_cuda, drop_last): exe.run(fluid.default_startup_program()) prog = fluid.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - places=places, loss_name=loss.name) + fluid.default_main_program() + ).with_data_parallel(places=places, loss_name=loss.name) - loader.set_batch_generator(self.create_reader(), - places=places if iterable else None) + loader.set_batch_generator( + self.create_reader(), places=places if iterable else None + ) for _ in range(self.epoch_num): actual_batch_num = 0 if loader.iterable: for feed_data in loader(): - x_data, = exe.run(prog, feed=feed_data, fetch_list=[x]) + (x_data,) = exe.run(prog, feed=feed_data, fetch_list=[x]) self.assertEqual(x_data.shape[0] % self.batch_size, 0) self.assertTrue(x_data.shape[0] != 0) actual_batch_num += int(x_data.shape[0] / self.batch_size) @@ -196,11 +206,12 @@ def run_network(self, iterable, use_cuda, drop_last): loader.start() try: while True: - x_data, = exe.run(prog, fetch_list=[x]) + (x_data,) = exe.run(prog, fetch_list=[x]) self.assertEqual(x_data.shape[0] % self.batch_size, 0) self.assertTrue(x_data.shape[0] != 0) - actual_batch_num += int(x_data.shape[0] / - self.batch_size) + actual_batch_num += int( + x_data.shape[0] / self.batch_size + ) except fluid.core.EOFException: loader.reset() @@ -210,8 +221,9 @@ def run_network(self, iterable, use_cuda, drop_last): self.assertGreater(self.batch_num, actual_batch_num) def test_main(self): - use_cuda_list = [False, True - ] if fluid.is_compiled_with_cuda() else [False] + use_cuda_list = ( + [False, True] if fluid.is_compiled_with_cuda() else [False] + ) iterable_list = [False, True] drop_last_list = [False, True] for iterable in iterable_list: diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py index b4a69e89e4bc4..1341ccaad31af 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py @@ -32,8 +32,10 @@ def simple_fc_net(use_feed): hidden, size=200, act='tanh', - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) prediction = fluid.layers.fc(hidden, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) loss = paddle.mean(loss) @@ -52,7 +54,9 @@ def fc_with_batchnorm(use_feed): size=200, act='tanh', bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0))) + initializer=fluid.initializer.Constant(value=1.0) + ), + ) hidden = fluid.layers.batch_norm(input=hidden) with fluid.name_scope("fc_layer"): @@ -71,16 +75,13 @@ def init_data(): class TestMNIST(TestParallelExecutorBase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) - def _compare_reduce_and_allreduce(self, - model, - use_device, - delta1=1e-6, - delta2=1e-4): + def _compare_reduce_and_allreduce( + self, model, use_device, delta1=1e-6, delta2=1e-4 + ): if use_device == DeviceType.CUDA and not core.is_compiled_with_cuda(): return @@ -89,23 +90,23 @@ def _compare_reduce_and_allreduce(self, img, label = init_data() - all_reduce_first_loss, all_reduce_last_loss, _ = self.check_network_convergence( + ( + all_reduce_first_loss, + all_reduce_last_loss, + _, + ) = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_reduce=False) + use_reduce=False, + ) reduce_first_loss, reduce_last_loss, _ = self.check_network_convergence( model, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_reduce=True) + use_reduce=True, + ) for loss in zip(all_reduce_first_loss, reduce_first_loss): self.assertAlmostEqual(loss[0], loss[1], delta=delta1) @@ -122,13 +123,12 @@ def check_simple_fc_convergence(self, use_device, use_reduce=False): img, label = init_data() - self.check_network_convergence(simple_fc_net, - feed_dict={ - "image": img, - "label": label - }, - use_device=use_device, - use_reduce=use_reduce) + self.check_network_convergence( + simple_fc_net, + feed_dict={"image": img, "label": label}, + use_device=use_device, + use_reduce=use_reduce, + ) def test_simple_fc(self): # use_device @@ -140,10 +140,12 @@ def test_simple_fc_with_new_strategy(self): # use_device, use_reduce # NOTE: the computation result of nccl_reduce is non-deterministic, # related issue: https://github.com/NVIDIA/nccl/issues/157 - self._compare_reduce_and_allreduce(simple_fc_net, DeviceType.CUDA, 1e-5, - 1e-2) - self._compare_reduce_and_allreduce(simple_fc_net, DeviceType.CPU, 1e-5, - 1e-2) + self._compare_reduce_and_allreduce( + simple_fc_net, DeviceType.CUDA, 1e-5, 1e-2 + ) + self._compare_reduce_and_allreduce( + simple_fc_net, DeviceType.CPU, 1e-5, 1e-2 + ) def check_simple_fc_parallel_accuracy(self, use_device): if use_device == DeviceType.CUDA and not core.is_compiled_with_cuda(): @@ -153,29 +155,29 @@ def check_simple_fc_parallel_accuracy(self, use_device): single_first_loss, single_last_loss, _ = self.check_network_convergence( method=simple_fc_net, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_parallel_executor=False) - parallel_first_loss, parallel_last_loss, _ = self.check_network_convergence( + use_parallel_executor=False, + ) + ( + parallel_first_loss, + parallel_last_loss, + _, + ) = self.check_network_convergence( method=simple_fc_net, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_parallel_executor=True) + use_parallel_executor=True, + ) self.assertAlmostEquals( np.mean(parallel_first_loss), single_first_loss, delta=1e-6, ) - self.assertAlmostEquals(np.mean(parallel_last_loss), - single_last_loss, - delta=1e-6) + self.assertAlmostEquals( + np.mean(parallel_last_loss), single_last_loss, delta=1e-6 + ) def test_simple_fc_parallel_accuracy(self): self.check_simple_fc_parallel_accuracy(DeviceType.CUDA) @@ -188,31 +190,32 @@ def check_batchnorm_fc_convergence(self, use_device, use_fast_executor): return img, label = init_data() - self.check_network_convergence(fc_with_batchnorm, - feed_dict={ - "image": img, - "label": label - }, - use_device=use_device, - use_fast_executor=use_fast_executor) + self.check_network_convergence( + fc_with_batchnorm, + feed_dict={"image": img, "label": label}, + use_device=use_device, + use_fast_executor=use_fast_executor, + ) def test_batchnorm_fc(self): for use_device in (DeviceType.CPU, DeviceType.CUDA): for use_fast_executor in (False, True): - self.check_batchnorm_fc_convergence(use_device, - use_fast_executor) + self.check_batchnorm_fc_convergence( + use_device, use_fast_executor + ) def test_batchnorm_fc_with_new_strategy(self): # NOTE: the computation result of nccl_reduce is non-deterministic, # related issue: https://github.com/NVIDIA/nccl/issues/157 - self._compare_reduce_and_allreduce(fc_with_batchnorm, DeviceType.CUDA, - 1e-5, 1e-2) - self._compare_reduce_and_allreduce(fc_with_batchnorm, DeviceType.CPU, - 1e-5, 1e-2) + self._compare_reduce_and_allreduce( + fc_with_batchnorm, DeviceType.CUDA, 1e-5, 1e-2 + ) + self._compare_reduce_and_allreduce( + fc_with_batchnorm, DeviceType.CPU, 1e-5, 1e-2 + ) class TestMNISTNoReduce(unittest.TestCase): - def run_program(self, device_type): if device_type == DeviceType.CUDA: if not paddle.is_compiled_with_cuda(): @@ -237,16 +240,18 @@ def run_program(self, device_type): build_strategy = paddle.static.BuildStrategy() build_strategy.reduce_strategy = no_reduce main_multi_place = paddle.static.CompiledProgram( - main).with_data_parallel(loss_name=loss.name, - build_strategy=build_strategy, - places=places) + main + ).with_data_parallel( + loss_name=loss.name, build_strategy=build_strategy, places=places + ) build_strategy = paddle.static.BuildStrategy() build_strategy.reduce_strategy = no_reduce main_single_place = paddle.static.CompiledProgram( - main.clone()).with_data_parallel(loss_name=loss.name, - build_strategy=build_strategy, - places=places[0]) + main.clone() + ).with_data_parallel( + loss_name=loss.name, build_strategy=build_strategy, places=places[0] + ) image, label = init_data() feed = {'image': image, 'label': label} @@ -254,9 +259,9 @@ def run_program(self, device_type): scope = paddle.static.Scope() with paddle.static.scope_guard(scope): exe.run(startup) - grads_multi_place = exe.run(main_multi_place, - feed=feed, - fetch_list=[grads]) + grads_multi_place = exe.run( + main_multi_place, feed=feed, fetch_list=[grads] + ) feeds = self.split_feed(feed, len(places)) grads_single_place = [list() for _ in range(len(grads))] @@ -266,8 +271,9 @@ def run_program(self, device_type): grads_single_place[i].append(g) for i in range(len(grads)): - grads_single_place[i] = np.concatenate(grads_single_place[i], - axis=0) / len(places) + grads_single_place[i] = np.concatenate( + grads_single_place[i], axis=0 + ) / len(places) self.assertEqual(len(grads_multi_place), len(grads_single_place)) for g1, g2 in zip(grads_multi_place, grads_single_place): diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_pg.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_pg.py index e312cdd125dcb..8813c962f8b6d 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_pg.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_pg.py @@ -25,7 +25,6 @@ class TestMNIST(TestParallelExecutorBase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) @@ -36,13 +35,12 @@ def check_simple_fc_convergence(self, use_device, use_reduce=False): return img, label = init_data() - self.check_network_convergence(simple_fc_net, - feed_dict={ - "image": img, - "label": label - }, - use_device=use_device, - use_reduce=use_reduce) + self.check_network_convergence( + simple_fc_net, + feed_dict={"image": img, "label": label}, + use_device=use_device, + use_reduce=use_reduce, + ) def test_simple_fc(self): # use_device @@ -55,29 +53,29 @@ def check_simple_fc_parallel_accuracy(self, use_device): img, label = init_data() single_first_loss, single_last_loss, _ = self.check_network_convergence( method=simple_fc_net, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_parallel_executor=False) - parallel_first_loss, parallel_last_loss, _ = self.check_network_convergence( + use_parallel_executor=False, + ) + ( + parallel_first_loss, + parallel_last_loss, + _, + ) = self.check_network_convergence( method=simple_fc_net, - feed_dict={ - "image": img, - "label": label - }, + feed_dict={"image": img, "label": label}, use_device=use_device, - use_parallel_executor=True) + use_parallel_executor=True, + ) self.assertAlmostEquals( np.mean(parallel_first_loss), single_first_loss, delta=1e-6, ) - self.assertAlmostEquals(np.mean(parallel_last_loss), - single_last_loss, - delta=1e-6) + self.assertAlmostEquals( + np.mean(parallel_last_loss), single_last_loss, delta=1e-6 + ) def test_simple_fc_parallel_accuracy(self): self.check_simple_fc_parallel_accuracy(DeviceType.CUDA) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_profiler.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_profiler.py index 6ba451b0e6806..4d8dc470fa9e4 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_profiler.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_profiler.py @@ -27,19 +27,20 @@ class TestPEProfiler(TestProfiler): - def test_cpu_profiler(self): exe = fluid.Executor(fluid.CPUPlace()) self.net_profiler(exe, 'CPU', "Default", use_parallel_executor=True) - @unittest.skipIf(not core.is_compiled_with_cuda(), - "profiler is enabled only with GPU") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "profiler is enabled only with GPU" + ) def test_cuda_profiler(self): exe = fluid.Executor(fluid.CUDAPlace(0)) self.net_profiler(exe, 'GPU', "OpDetail", use_parallel_executor=True) - @unittest.skipIf(not core.is_compiled_with_cuda(), - "profiler is enabled only with GPU") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "profiler is enabled only with GPU" + ) def test_all_profiler(self): exe = fluid.Executor(fluid.CUDAPlace(0)) self.net_profiler(exe, 'All', "AllOpDetail", use_parallel_executor=True) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_run_cinn.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_run_cinn.py index 40a1d087cdc81..3bb546eb91792 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_run_cinn.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_run_cinn.py @@ -22,8 +22,9 @@ paddle.enable_static() -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', - level=logging.INFO) +logging.basicConfig( + format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO +) logger = logging.getLogger("paddle_with_cinn") @@ -39,8 +40,9 @@ def set_cinn_flag(val): def reader(limit): for _ in range(limit): - yield np.random.random([1, 28]).astype('float32'), \ - np.random.randint(0, 2, size=[1]).astype('int64') + yield np.random.random([1, 28]).astype('float32'), np.random.randint( + 0, 2, size=[1] + ).astype('int64') def rand_data(img, label, loop_num=10): @@ -59,8 +61,12 @@ def build_program(main_program, startup_program): name="bias", shape=[1, 28], dtype="float32", - attr=paddle.ParamAttr(initializer=paddle.nn.initializer.Assign( - np.random.rand(1, 28).astype(np.float32)))) + attr=paddle.ParamAttr( + initializer=paddle.nn.initializer.Assign( + np.random.rand(1, 28).astype(np.float32) + ) + ), + ) label = paddle.static.data(name="label", shape=[1], dtype='int64') hidden = paddle.add(img, param) @@ -83,15 +89,19 @@ def train(dot_save_dir, prefix, seed=1234): main_program = paddle.static.Program() img, label, loss = build_program(main_program, startup_program) - place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) exe.run(startup_program) build_strategy = paddle.static.BuildStrategy() build_strategy.debug_graphviz_path = os.path.join(dot_save_dir, prefix) compiled_program = paddle.static.CompiledProgram( - main_program, build_strategy).with_data_parallel(loss_name=loss.name) + main_program, build_strategy + ).with_data_parallel(loss_name=loss.name) iters = 100 feed = rand_data(img.name, label.name, iters) @@ -104,7 +114,6 @@ def train(dot_save_dir, prefix, seed=1234): @unittest.skipIf(not set_cinn_flag(True), "Paddle is not compiled with CINN.") class TestParallelExecutorRunCinn(unittest.TestCase): - def setUp(self): self.tmpdir = tempfile.mkdtemp(prefix="dots_") @@ -115,10 +124,9 @@ def test_run_with_cinn(self): cinn_losses = train(self.tmpdir, "paddle") set_cinn_flag(False) pd_losses = train(self.tmpdir, "cinn") - np.testing.assert_allclose(cinn_losses, - pd_losses, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + cinn_losses, pd_losses, rtol=1e-05, atol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_run_load_infer_program.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_run_load_infer_program.py index c0cd03b71fa8f..461461f797529 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_run_load_infer_program.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_run_load_infer_program.py @@ -19,12 +19,15 @@ class TestMNIST(unittest.TestCase): - @classmethod def setUpClass(cls): cls.save_dirname = "./" - cls.model_filename = "test_parallel_executor_run_load_infer_program_model" - cls.params_filename = "test_parallel_executor_run_load_infer_program_parameter" + cls.model_filename = ( + "test_parallel_executor_run_load_infer_program_model" + ) + cls.params_filename = ( + "test_parallel_executor_run_load_infer_program_parameter" + ) cls.place = fluid.CPUPlace() cls.exe = fluid.Executor(cls.place) img, label = init_data() @@ -35,22 +38,30 @@ def setUpClass(cls): def test_simple_fc(self): exe_loss = self.run_with_executor() - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(self.save_dirname, - self.exe, - self.model_filename, - self.params_filename) - - train_exe = fluid.ParallelExecutor(use_cuda=False, - main_program=inference_program) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = fluid.io.load_inference_model( + self.save_dirname, + self.exe, + self.model_filename, + self.params_filename, + ) + + train_exe = fluid.ParallelExecutor( + use_cuda=False, main_program=inference_program + ) feed_vars = [ inference_program.global_block().var(var_name) for var_name in ["image", "label"] ] feeder = fluid.DataFeeder(place=self.place, feed_list=feed_vars) - pe_loss = train_exe.run(feed=feeder.feed(self.batch_data), - fetch_list=[fetch_targets[0].name]) + pe_loss = train_exe.run( + feed=feeder.feed(self.batch_data), + fetch_list=[fetch_targets[0].name], + ) assert exe_loss == pe_loss def run_with_executor(self): @@ -60,23 +71,25 @@ def run_with_executor(self): loss = simple_fc_net() feed_vars = [ - main.global_block().var(var_name) - for var_name in ["image", "label"] + main.global_block().var(var_name) for var_name in ["image", "label"] ] feeder = fluid.DataFeeder(place=self.place, feed_list=feed_vars) self.exe.run(startup) - loss_data = self.exe.run(main, - feed=feeder.feed(self.batch_data), - fetch_list=[loss.name]) - - fluid.io.save_inference_model(self.save_dirname, ["image", "label"], - [loss], - self.exe, - model_filename=self.model_filename, - params_filename=self.params_filename, - main_program=main) + loss_data = self.exe.run( + main, feed=feeder.feed(self.batch_data), fetch_list=[loss.name] + ) + + fluid.io.save_inference_model( + self.save_dirname, + ["image", "label"], + [loss], + self.exe, + model_filename=self.model_filename, + params_filename=self.params_filename, + main_program=main, + ) return loss_data diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_cpu.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_cpu.py index a3394833fd4be..79f5fd4c3ca30 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_cpu.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_cpu.py @@ -19,19 +19,22 @@ class TestResnetCPU(TestResnetBase): - def test_seresnext_with_learning_rate_decay(self): # NOTE(zcd): This test is compare the result of use parallel_executor # and executor, and the result of drop_out op and batch_norm op in # this two executor have diff, so the two ops should be removed # from the model. - check_func = partial(self.check_network_convergence, - optimizer=seresnext_net.optimizer, - use_parallel_executor=False) - self._compare_result_with_origin_model(check_func, - use_device=DeviceType.CPU, - compare_separately=False, - delta2=1e-3) + check_func = partial( + self.check_network_convergence, + optimizer=seresnext_net.optimizer, + use_parallel_executor=False, + ) + self._compare_result_with_origin_model( + check_func, + use_device=DeviceType.CPU, + compare_separately=False, + delta2=1e-3, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_gpu.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_gpu.py index e3f7d0a3462e8..6f7f0f507a4ed 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_gpu.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_base_gpu.py @@ -19,19 +19,22 @@ class TestResnetGPU(TestResnetBase): - def test_seresnext_with_learning_rate_decay(self): # NOTE(zcd): This test is compare the result of use parallel_executor # and executor, and the result of drop_out op and batch_norm op in # this two executor have diff, so the two ops should be removed # from the model. - check_func = partial(self.check_network_convergence, - optimizer=seresnext_net.optimizer, - use_parallel_executor=False) - self._compare_result_with_origin_model(check_func, - use_device=DeviceType.CUDA, - delta2=1e-5, - compare_separately=False) + check_func = partial( + self.check_network_convergence, + optimizer=seresnext_net.optimizer, + use_parallel_executor=False, + ) + self._compare_result_with_origin_model( + check_func, + use_device=DeviceType.CUDA, + delta2=1e-5, + compare_separately=False, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_cpu.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_cpu.py index 4ef6be1667cd8..46b7bb83147bd 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_cpu.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_cpu.py @@ -24,15 +24,17 @@ class TestResnetWithFuseAllReduceCPU(TestResnetBase): - def test_seresnext_with_fused_all_reduce(self): # NOTE(zcd): In order to make the program faster, # this unit test remove drop_out and batch_norm. - check_func = partial(self.check_network_convergence, - optimizer=seresnext_net.optimizer, - fuse_all_reduce_ops=True) - self._compare_result_with_origin_model(check_func, - use_device=DeviceType.CPU) + check_func = partial( + self.check_network_convergence, + optimizer=seresnext_net.optimizer, + fuse_all_reduce_ops=True, + ) + self._compare_result_with_origin_model( + check_func, use_device=DeviceType.CPU + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_gpu.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_gpu.py index 016c8f41cbe86..ca349fc110090 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_gpu.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_fuse_all_reduce_gpu.py @@ -24,16 +24,17 @@ class TestResnetWithFuseAllReduceGPU(TestResnetBase): - def test_seresnext_with_fused_all_reduce(self): # NOTE(zcd): In order to make the program faster, # this unit test remove drop_out and batch_norm. - check_func = partial(self.check_network_convergence, - optimizer=seresnext_net.optimizer, - fuse_all_reduce_ops=True) - self._compare_result_with_origin_model(check_func, - use_device=DeviceType.CUDA, - delta2=1e-2) + check_func = partial( + self.check_network_convergence, + optimizer=seresnext_net.optimizer, + fuse_all_reduce_ops=True, + ) + self._compare_result_with_origin_model( + check_func, use_device=DeviceType.CUDA, delta2=1e-2 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_cpu.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_cpu.py index d8d854fd315c9..abd650468c531 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_cpu.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_cpu.py @@ -19,19 +19,23 @@ class TestResnetWithReduceBase(TestParallelExecutorBase): - def _compare_reduce_and_allreduce(self, use_device, delta2=1e-5): if use_device == DeviceType.CUDA and not core.is_compiled_with_cuda(): return - all_reduce_first_loss, all_reduce_last_loss, _ = self.check_network_convergence( + ( + all_reduce_first_loss, + all_reduce_last_loss, + _, + ) = self.check_network_convergence( seresnext_net.model, feed_dict=seresnext_net.feed_dict(use_device), iter=seresnext_net.iter(use_device), batch_size=seresnext_net.batch_size(use_device), use_device=use_device, use_reduce=False, - optimizer=seresnext_net.optimizer) + optimizer=seresnext_net.optimizer, + ) reduce_first_loss, reduce_last_loss, _ = self.check_network_convergence( seresnext_net.model, feed_dict=seresnext_net.feed_dict(use_device), @@ -39,7 +43,8 @@ def _compare_reduce_and_allreduce(self, use_device, delta2=1e-5): batch_size=seresnext_net.batch_size(use_device), use_device=use_device, use_reduce=True, - optimizer=seresnext_net.optimizer) + optimizer=seresnext_net.optimizer, + ) for loss in zip(all_reduce_first_loss, reduce_first_loss): self.assertAlmostEquals(loss[0], loss[1], delta=1e-5) @@ -49,7 +54,11 @@ def _compare_reduce_and_allreduce(self, use_device, delta2=1e-5): if not use_device: return - all_reduce_first_loss_seq, all_reduce_last_loss_seq, _ = self.check_network_convergence( + ( + all_reduce_first_loss_seq, + all_reduce_last_loss_seq, + _, + ) = self.check_network_convergence( seresnext_net.model, feed_dict=seresnext_net.feed_dict(use_device), iter=seresnext_net.iter(use_device), @@ -57,9 +66,14 @@ def _compare_reduce_and_allreduce(self, use_device, delta2=1e-5): use_device=use_device, use_reduce=False, optimizer=seresnext_net.optimizer, - enable_sequential_execution=True) - - reduce_first_loss_seq, reduce_last_loss_seq, _ = self.check_network_convergence( + enable_sequential_execution=True, + ) + + ( + reduce_first_loss_seq, + reduce_last_loss_seq, + _, + ) = self.check_network_convergence( seresnext_net.model, feed_dict=seresnext_net.feed_dict(use_device), iter=seresnext_net.iter(use_device), @@ -67,7 +81,8 @@ def _compare_reduce_and_allreduce(self, use_device, delta2=1e-5): use_device=use_device, use_reduce=True, optimizer=seresnext_net.optimizer, - enable_sequential_execution=True) + enable_sequential_execution=True, + ) for loss in zip(all_reduce_first_loss, all_reduce_first_loss_seq): self.assertAlmostEquals(loss[0], loss[1], delta=1e-5) @@ -86,10 +101,10 @@ def _compare_reduce_and_allreduce(self, use_device, delta2=1e-5): class TestResnetWithReduceCPU(TestResnetWithReduceBase): - def test_seresnext_with_reduce(self): - self._compare_reduce_and_allreduce(use_device=DeviceType.CPU, - delta2=1e-3) + self._compare_reduce_and_allreduce( + use_device=DeviceType.CPU, delta2=1e-3 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_gpu.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_gpu.py index 726f42e12a079..e4cce04e2a413 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_gpu.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext_with_reduce_gpu.py @@ -13,14 +13,17 @@ # limitations under the License. import unittest -from test_parallel_executor_seresnext_with_reduce_cpu import TestResnetWithReduceBase, DeviceType +from test_parallel_executor_seresnext_with_reduce_cpu import ( + TestResnetWithReduceBase, + DeviceType, +) class TestResnetWithReduceGPU(TestResnetWithReduceBase): - def test_seresnext_with_reduce(self): - self._compare_reduce_and_allreduce(use_device=DeviceType.CUDA, - delta2=1e-2) + self._compare_reduce_and_allreduce( + use_device=DeviceType.CUDA, delta2=1e-2 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py index 8366f8f29cd80..9391424692b43 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py @@ -24,7 +24,6 @@ class ParallelExecutorTestingDuringTraining(unittest.TestCase): - def check_network_convergence(self, use_cuda, build_strategy=None): os.environ['CPU_NUM'] = str(4) main = fluid.Program() @@ -46,20 +45,22 @@ def check_network_convergence(self, use_cuda, build_strategy=None): feed_dict = {'image': image, 'label': label} train_cp = compiler.CompiledProgram(main).with_data_parallel( - loss_name=loss.name, build_strategy=build_strategy) + loss_name=loss.name, build_strategy=build_strategy + ) test_cp = compiler.CompiledProgram(test_program).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy, - share_vars_from=train_cp) + share_vars_from=train_cp, + ) for i in range(5): exe.run(train_cp, feed=feed_dict, fetch_list=[loss.name]) - test_loss, = exe.run(test_cp, - feed=feed_dict, - fetch_list=[loss.name]) - train_loss, = exe.run(train_cp, - feed=feed_dict, - fetch_list=[loss.name]) + (test_loss,) = exe.run( + test_cp, feed=feed_dict, fetch_list=[loss.name] + ) + (train_loss,) = exe.run( + train_cp, feed=feed_dict, fetch_list=[loss.name] + ) avg_test_loss_val = np.array(test_loss).mean() if math.isnan(float(avg_test_loss_val)): @@ -69,32 +70,41 @@ def check_network_convergence(self, use_cuda, build_strategy=None): if math.isnan(float(avg_train_loss_val)): sys.exit("got NaN loss, training failed.") - np.testing.assert_allclose(train_loss, - test_loss, - rtol=1e-05, - atol=0.01) + np.testing.assert_allclose( + train_loss, test_loss, rtol=1e-05, atol=0.01 + ) def test_parallel_testing(self): build_strategy = fluid.BuildStrategy() - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.AllReduce + build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.AllReduce + ) if core.is_compiled_with_cuda(): - self.check_network_convergence(use_cuda=True, - build_strategy=build_strategy) - self.check_network_convergence(use_cuda=False, - build_strategy=build_strategy) + self.check_network_convergence( + use_cuda=True, build_strategy=build_strategy + ) + self.check_network_convergence( + use_cuda=False, build_strategy=build_strategy + ) def test_parallel_testing_with_new_strategy_gpu(self): build_strategy = fluid.BuildStrategy() - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce + build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + ) if core.is_compiled_with_cuda(): - self.check_network_convergence(use_cuda=True, - build_strategy=build_strategy) + self.check_network_convergence( + use_cuda=True, build_strategy=build_strategy + ) def test_parallel_testing_with_new_strategy_cpu(self): build_strategy = fluid.BuildStrategy() - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce - self.check_network_convergence(use_cuda=False, - build_strategy=build_strategy) + build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + ) + self.check_network_convergence( + use_cuda=False, build_strategy=build_strategy + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py index 684be9023b64e..d447860862b60 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py @@ -77,12 +77,14 @@ def prepare_batch_input(insts, src_pad_idx, trg_pad_idx, n_head): data to tensors and return a dict mapping names to tensors. """ - def __pad_batch_data(insts, - pad_idx, - is_target=False, - return_pos=True, - return_attn_bias=True, - return_max_len=True): + def __pad_batch_data( + insts, + pad_idx, + is_target=False, + return_pos=True, + return_attn_bias=True, + return_max_len=True, + ): """ Pad the instances to the max sequence length in batch, and generate the corresponding position data and attention bias. @@ -90,13 +92,19 @@ def __pad_batch_data(insts, return_list = [] max_len = max(len(inst) for inst in insts) inst_data = np.array( - [inst + [pad_idx] * (max_len - len(inst)) for inst in insts]) + [inst + [pad_idx] * (max_len - len(inst)) for inst in insts] + ) return_list += [inst_data.astype("int64").reshape([-1, 1])] if return_pos: - inst_pos = np.array([[ - pos_i + 1 if w_i != pad_idx else 0 - for pos_i, w_i in enumerate(inst) - ] for inst in inst_data]) + inst_pos = np.array( + [ + [ + pos_i + 1 if w_i != pad_idx else 0 + for pos_i, w_i in enumerate(inst) + ] + for inst in inst_data + ] + ) return_list += [inst_pos.astype("int64").reshape([-1, 1])] if return_attn_bias: @@ -104,37 +112,55 @@ def __pad_batch_data(insts, # This is used to avoid attention on paddings and subsequent # words. slf_attn_bias_data = np.ones( - (inst_data.shape[0], max_len, max_len)) + (inst_data.shape[0], max_len, max_len) + ) slf_attn_bias_data = np.triu(slf_attn_bias_data, 1).reshape( - [-1, 1, max_len, max_len]) - slf_attn_bias_data = np.tile(slf_attn_bias_data, - [1, n_head, 1, 1]) * [-1e9] + [-1, 1, max_len, max_len] + ) + slf_attn_bias_data = np.tile( + slf_attn_bias_data, [1, n_head, 1, 1] + ) * [-1e9] else: # This is used to avoid attention on paddings. - slf_attn_bias_data = np.array([[0] * len(inst) + [-1e9] * - (max_len - len(inst)) - for inst in insts]) + slf_attn_bias_data = np.array( + [ + [0] * len(inst) + [-1e9] * (max_len - len(inst)) + for inst in insts + ] + ) slf_attn_bias_data = np.tile( slf_attn_bias_data.reshape([-1, 1, 1, max_len]), - [1, n_head, max_len, 1]) + [1, n_head, max_len, 1], + ) return_list += [slf_attn_bias_data.astype("float32")] if return_max_len: return_list += [max_len] return return_list if len(return_list) > 1 else return_list[0] src_word, src_pos, src_slf_attn_bias, src_max_len = __pad_batch_data( - [inst[0] for inst in insts], src_pad_idx, is_target=False) + [inst[0] for inst in insts], src_pad_idx, is_target=False + ) trg_word, trg_pos, trg_slf_attn_bias, trg_max_len = __pad_batch_data( - [inst[1] for inst in insts], trg_pad_idx, is_target=True) - trg_src_attn_bias = np.tile(src_slf_attn_bias[:, :, ::src_max_len, :], - [1, 1, trg_max_len, 1]).astype("float32") - lbl_word = __pad_batch_data([inst[2] for inst in insts], trg_pad_idx, False, - False, False, False) + [inst[1] for inst in insts], trg_pad_idx, is_target=True + ) + trg_src_attn_bias = np.tile( + src_slf_attn_bias[:, :, ::src_max_len, :], [1, 1, trg_max_len, 1] + ).astype("float32") + lbl_word = __pad_batch_data( + [inst[2] for inst in insts], trg_pad_idx, False, False, False, False + ) lbl_weight = (lbl_word != trg_pad_idx).astype("float32").reshape([-1, 1]) return [ - src_word, src_pos, trg_word, trg_pos, src_slf_attn_bias, - trg_slf_attn_bias, trg_src_attn_bias, lbl_word, lbl_weight + src_word, + src_pos, + trg_word, + trg_pos, + src_slf_attn_bias, + trg_slf_attn_bias, + trg_src_attn_bias, + lbl_word, + lbl_weight, ] @@ -145,12 +171,19 @@ def transformer(use_feed): assert not use_feed, "transfomer doesn't support feed yet" return transformer_model.transformer( ModelHyperParams.src_vocab_size + 1, - ModelHyperParams.trg_vocab_size + 1, ModelHyperParams.max_length + 1, - ModelHyperParams.n_layer, ModelHyperParams.n_head, - ModelHyperParams.d_key, ModelHyperParams.d_value, - ModelHyperParams.d_model, ModelHyperParams.d_inner_hid, - ModelHyperParams.dropout, ModelHyperParams.src_pad_idx, - ModelHyperParams.trg_pad_idx, ModelHyperParams.pos_pad_idx) + ModelHyperParams.trg_vocab_size + 1, + ModelHyperParams.max_length + 1, + ModelHyperParams.n_layer, + ModelHyperParams.n_head, + ModelHyperParams.d_key, + ModelHyperParams.d_value, + ModelHyperParams.d_model, + ModelHyperParams.d_inner_hid, + ModelHyperParams.dropout, + ModelHyperParams.src_pad_idx, + ModelHyperParams.trg_pad_idx, + ModelHyperParams.pos_pad_idx, + ) def get_feed_data_reader(): @@ -158,15 +191,21 @@ def get_feed_data_reader(): if feed_data_reader is not None: return feed_data_reader - reader = paddle.batch(wmt16.train(ModelHyperParams.src_vocab_size, - ModelHyperParams.trg_vocab_size), - batch_size=transformer_model.batch_size) + reader = paddle.batch( + wmt16.train( + ModelHyperParams.src_vocab_size, ModelHyperParams.trg_vocab_size + ), + batch_size=transformer_model.batch_size, + ) all_batch_tensors = [] for batch in reader(): tensors = [] - for tensor in prepare_batch_input(batch, ModelHyperParams.src_pad_idx, - ModelHyperParams.trg_pad_idx, - ModelHyperParams.n_head): + for tensor in prepare_batch_input( + batch, + ModelHyperParams.src_pad_idx, + ModelHyperParams.trg_pad_idx, + ModelHyperParams.n_head, + ): tensors.append(np.array(tensor)) all_batch_tensors.append(tensors) @@ -174,30 +213,36 @@ def __reader__(): for t in all_batch_tensors: yield t - feed_data_reader = FeedDataReader(feed_list=transformer_model.build_inputs( - ModelHyperParams.max_length + 1, ModelHyperParams.n_head), - reader=__reader__) + feed_data_reader = FeedDataReader( + feed_list=transformer_model.build_inputs( + ModelHyperParams.max_length + 1, ModelHyperParams.n_head + ), + reader=__reader__, + ) return feed_data_reader class TestTransformer(TestParallelExecutorBase): - def test_main(self): if core.is_compiled_with_cuda(): self.check_network_convergence( transformer, use_device=DeviceType.CUDA, - feed_data_reader=get_feed_data_reader()) + feed_data_reader=get_feed_data_reader(), + ) self.check_network_convergence( transformer, use_device=DeviceType.CUDA, enable_sequential_execution=True, - feed_data_reader=get_feed_data_reader()) - self.check_network_convergence(transformer, - use_device=DeviceType.CPU, - iter=2, - feed_data_reader=get_feed_data_reader()) + feed_data_reader=get_feed_data_reader(), + ) + self.check_network_convergence( + transformer, + use_device=DeviceType.CPU, + iter=2, + feed_data_reader=get_feed_data_reader(), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parameter.py b/python/paddle/fluid/tests/unittests/test_parameter.py index e066b96127195..d077fb9836285 100644 --- a/python/paddle/fluid/tests/unittests/test_parameter.py +++ b/python/paddle/fluid/tests/unittests/test_parameter.py @@ -16,7 +16,12 @@ import copy import paddle from paddle.fluid.dygraph import guard -from paddle.fluid.framework import default_main_program, Variable, _test_eager_guard, ParamBase +from paddle.fluid.framework import ( + default_main_program, + Variable, + _test_eager_guard, + ParamBase, +) import paddle.fluid.core as core from paddle.fluid.executor import Executor import paddle.fluid.io as io @@ -28,15 +33,16 @@ class ParameterChecks(unittest.TestCase): - def test_parameter(self): shape = [784, 100] val = 1.0625 b = main_program.global_block() - param = b.create_parameter(name='fc.w', - shape=shape, - dtype='float32', - initializer=ConstantInitializer(val)) + param = b.create_parameter( + name='fc.w', + shape=shape, + dtype='float32', + initializer=ConstantInitializer(val), + ) self.assertIsNotNone(param) self.assertEqual('fc.w', param.name) self.assertEqual((784, 100), param.shape) @@ -65,8 +71,9 @@ def func_parambase(self): np.testing.assert_array_equal(param_copy.numpy(), param.numpy()) self.assertEqual(param_copy.optimize_attr, param.optimize_attr) self.assertEqual(param_copy.regularizer, param.regularizer) - self.assertEqual(param_copy.do_model_average, - param.do_model_average) + self.assertEqual( + param_copy.do_model_average, param.do_model_average + ) self.assertEqual(param_copy.need_clip, param.need_clip) self.assertEqual(param_copy.is_distributed, param.is_distributed) @@ -84,30 +91,27 @@ def test_parambase(self): def func_exception(self): b = main_program.global_block() with self.assertRaises(ValueError): - b.create_parameter(name='test', - shape=None, - dtype='float32', - initializer=None) + b.create_parameter( + name='test', shape=None, dtype='float32', initializer=None + ) with self.assertRaises(ValueError): - b.create_parameter(name='test', - shape=[1], - dtype=None, - initializer=None) + b.create_parameter( + name='test', shape=[1], dtype=None, initializer=None + ) with self.assertRaises(ValueError): - b.create_parameter(name='test', - shape=[], - dtype='float32', - initializer=None) + b.create_parameter( + name='test', shape=[], dtype='float32', initializer=None + ) with self.assertRaises(ValueError): - b.create_parameter(name='test', - shape=[-1], - dtype='float32', - initializer=None) + b.create_parameter( + name='test', shape=[-1], dtype='float32', initializer=None + ) def func_parambase_to_vector(self): with guard(): initializer = paddle.ParamAttr( - initializer=paddle.nn.initializer.Constant(3.)) + initializer=paddle.nn.initializer.Constant(3.0) + ) linear1 = paddle.nn.Linear(10, 15, initializer) vec = paddle.nn.utils.parameters_to_vector(linear1.parameters()) @@ -120,10 +124,12 @@ def func_parambase_to_vector(self): paddle.nn.utils.vector_to_parameters(vec, linear2.parameters()) self.assertEqual(linear2.weight.shape, [10, 15]) self.assertEqual(linear2.bias.shape, [15]) - np.testing.assert_array_equal(linear1.weight.numpy(), - linear2.weight.numpy()) - np.testing.assert_array_equal(linear1.bias.numpy(), - linear2.bias.numpy()) + np.testing.assert_array_equal( + linear1.weight.numpy(), linear2.weight.numpy() + ) + np.testing.assert_array_equal( + linear1.bias.numpy(), linear2.bias.numpy() + ) self.assertTrue(linear2.weight.is_leaf, True) self.assertTrue(linear2.bias.is_leaf, True) diff --git a/python/paddle/fluid/tests/unittests/test_partial_concat_op.py b/python/paddle/fluid/tests/unittests/test_partial_concat_op.py index 1e4c0076da6e7..8046d8fa1d7e5 100644 --- a/python/paddle/fluid/tests/unittests/test_partial_concat_op.py +++ b/python/paddle/fluid/tests/unittests/test_partial_concat_op.py @@ -19,33 +19,34 @@ def np_partial_concat(inputs, start, length): - assert (len(inputs[0].shape) == 2) + assert len(inputs[0].shape) == 2 size = inputs[0].shape[1] - assert (start >= -size and start < size) + assert start >= -size and start < size if start < 0: start += size if length < 0: length = size - start - assert (size >= start + length) + assert size >= start + length elems = [] for elem in inputs: - assert (elem.shape == inputs[0].shape) - elems.append(elem[:, start:start + length]) + assert elem.shape == inputs[0].shape + elems.append(elem[:, start : start + length]) res = np.concatenate(elems, axis=1) return np.concatenate(elems, axis=1) class TestPartialConcatOp(OpTest): - def setUp(self): self.op_type = "partial_concat" self.init_kernel_type() self.init_para() self.var_names = ['x' + str(num) for num in range(self.var_num)] - self.vars = [np.random.random((self.batch_size, self.column)).astype(self.dtype)\ - for num in range(self.var_num) ] + self.vars = [ + np.random.random((self.batch_size, self.column)).astype(self.dtype) + for num in range(self.var_num) + ] self.inputs = {'X': list(zip(self.var_names, self.vars))} self.attrs = {'start_index': self.start_index, 'length': self.length} y = np_partial_concat(self.vars[:], self.start_index, self.length) @@ -70,7 +71,6 @@ def test_check_grad(self): class TestPartialConcatOp2(TestPartialConcatOp): - def init_para(self): self.batch_size = random.randint(1, 10) self.column = random.randint(101, 200) @@ -80,7 +80,6 @@ def init_para(self): class TestPartialConcatOp3(TestPartialConcatOp): - def init_para(self): self.batch_size = random.randint(1, 10) self.column = random.randint(101, 200) @@ -90,7 +89,6 @@ def init_para(self): class TestPartialConcatOp4(TestPartialConcatOp): - def init_para(self): self.batch_size = random.randint(1, 10) self.column = random.randint(101, 200) diff --git a/python/paddle/fluid/tests/unittests/test_partial_sum_op.py b/python/paddle/fluid/tests/unittests/test_partial_sum_op.py index 5eb416cadc82d..9a5304f7b42ae 100644 --- a/python/paddle/fluid/tests/unittests/test_partial_sum_op.py +++ b/python/paddle/fluid/tests/unittests/test_partial_sum_op.py @@ -19,7 +19,6 @@ class TestPartialSumOp(OpTest): - def setUp(self): self.op_type = "partial_sum" self.init_kernel_type() @@ -29,13 +28,15 @@ def setUp(self): else: end_index = self.start_index + self.length self.var_names = ['x' + str(num) for num in range(self.var_num)] - self.vars = [np.random.random((self.batch_size, self.column)).astype(self.dtype)\ - for num in range(self.var_num) ] + self.vars = [ + np.random.random((self.batch_size, self.column)).astype(self.dtype) + for num in range(self.var_num) + ] self.inputs = {'X': list(zip(self.var_names, self.vars))} self.attrs = {'start_index': self.start_index, 'length': self.length} - y = self.vars[0][:, self.start_index:end_index] + y = self.vars[0][:, self.start_index : end_index] for i in range(1, self.var_num): - y = y + self.vars[i][:, self.start_index:end_index] + y = y + self.vars[i][:, self.start_index : end_index] self.outputs = {'Out': y} @@ -58,7 +59,6 @@ def test_check_grad(self): class TestPartialSumOp2(TestPartialSumOp): - def init_para(self): self.batch_size = random.randint(1, 10) self.column = random.randint(101, 200) @@ -68,7 +68,6 @@ def init_para(self): class TestPartialSumOp3(TestPartialSumOp): - def init_para(self): self.batch_size = random.randint(1, 10) self.column = random.randint(101, 200) @@ -78,7 +77,6 @@ def init_para(self): class TestPartialSumOp4(TestPartialSumOp): - def init_para(self): self.batch_size = random.randint(1, 10) self.column = random.randint(101, 200) diff --git a/python/paddle/fluid/tests/unittests/test_pipeline_parallel.py b/python/paddle/fluid/tests/unittests/test_pipeline_parallel.py index e1abb2c2372b1..11e8aa2b7c3a3 100644 --- a/python/paddle/fluid/tests/unittests/test_pipeline_parallel.py +++ b/python/paddle/fluid/tests/unittests/test_pipeline_parallel.py @@ -18,13 +18,11 @@ class TestPipelineParallel(TestMultipleGpus): - def test_pipeline_parallel(self): self.run_mnist_2gpu('hybrid_parallel_pp_alexnet.py') class TestModelParallelWithRecompute(TestMultipleGpus): - def test_model_parallel_with_recompute(self): self.run_mnist_2gpu("dygraph_recompute_hybrid.py") diff --git a/python/paddle/fluid/tests/unittests/test_pixel_shuffle.py b/python/paddle/fluid/tests/unittests/test_pixel_shuffle.py index 996cd826908ee..2b20ed0fc88fd 100644 --- a/python/paddle/fluid/tests/unittests/test_pixel_shuffle.py +++ b/python/paddle/fluid/tests/unittests/test_pixel_shuffle.py @@ -25,8 +25,14 @@ def pixel_shuffle_np(x, up_factor, data_format="NCHW"): if data_format == "NCHW": n, c, h, w = x.shape - new_shape = (n, c // (up_factor * up_factor), up_factor, up_factor, h, - w) + new_shape = ( + n, + c // (up_factor * up_factor), + up_factor, + up_factor, + h, + w, + ) # reshape to (num,output_channel,upscale_factor,upscale_factor,h,w) npresult = np.reshape(x, new_shape) # transpose to (num,output_channel,h,upscale_factor,w,upscale_factor) @@ -36,8 +42,14 @@ def pixel_shuffle_np(x, up_factor, data_format="NCHW"): return npresult else: n, h, w, c = x.shape - new_shape = (n, h, w, c // (up_factor * up_factor), up_factor, - up_factor) + new_shape = ( + n, + h, + w, + c // (up_factor * up_factor), + up_factor, + up_factor, + ) # reshape to (num,h,w,output_channel,upscale_factor,upscale_factor) npresult = np.reshape(x, new_shape) # transpose to (num,h,upscale_factor,w,upscale_factor,output_channel) @@ -48,7 +60,6 @@ def pixel_shuffle_np(x, up_factor, data_format="NCHW"): class TestPixelShuffleOp(OpTest): - def setUp(self): self.op_type = "pixel_shuffle" self.python_api = paddle.nn.functional.pixel_shuffle @@ -80,13 +91,11 @@ def test_check_grad(self): class TestChannelLast(TestPixelShuffleOp): - def init_data_format(self): self.format = "NHWC" class TestPixelShuffleAPI(unittest.TestCase): - def setUp(self): self.x_1_np = np.random.random([2, 9, 4, 4]).astype("float64") self.x_2_np = np.random.random([2, 4, 4, 9]).astype("float64") @@ -94,47 +103,53 @@ def setUp(self): self.out_2_np = pixel_shuffle_np(self.x_2_np, 3, "NHWC") def test_static_graph_functional(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=[2, 9, 4, 4], - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=[2, 4, 4, 9], - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=[2, 9, 4, 4], dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=[2, 4, 4, 9], dtype="float64" + ) out_1 = F.pixel_shuffle(x_1, 3) out_2 = F.pixel_shuffle(x_2, 3, "NHWC") exe = paddle.static.Executor(place=place) - res_1 = exe.run(fluid.default_main_program(), - feed={"x": self.x_1_np}, - fetch_list=out_1, - use_prune=True) - - res_2 = exe.run(fluid.default_main_program(), - feed={"x2": self.x_2_np}, - fetch_list=out_2, - use_prune=True) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True, + ) + + res_2 = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True, + ) assert np.allclose(res_1, self.out_1_np) assert np.allclose(res_2, self.out_2_np) # same test between layer and functional in this op. def test_static_graph_layer(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=[2, 9, 4, 4], - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=[2, 4, 4, 9], - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=[2, 9, 4, 4], dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=[2, 4, 4, 9], dtype="float64" + ) # init instance ps_1 = paddle.nn.PixelShuffle(3) ps_2 = paddle.nn.PixelShuffle(3, "NHWC") @@ -144,15 +159,19 @@ def test_static_graph_layer(self): out_2_np = pixel_shuffle_np(self.x_2_np, 3, "NHWC") exe = paddle.static.Executor(place=place) - res_1 = exe.run(fluid.default_main_program(), - feed={"x": self.x_1_np}, - fetch_list=out_1, - use_prune=True) - - res_2 = exe.run(fluid.default_main_program(), - feed={"x2": self.x_2_np}, - fetch_list=out_2, - use_prune=True) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True, + ) + + res_2 = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True, + ) assert np.allclose(res_1, out_1_np) assert np.allclose(res_2, out_2_np) @@ -170,23 +189,26 @@ def run_dygraph(self, up_factor, data_format): npresult = pixel_shuffle_np(x, up_factor, data_format) - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) - pixel_shuffle = paddle.nn.PixelShuffle(up_factor, - data_format=data_format) + pixel_shuffle = paddle.nn.PixelShuffle( + up_factor, data_format=data_format + ) result = pixel_shuffle(paddle.to_tensor(x)) np.testing.assert_allclose(result.numpy(), npresult, rtol=1e-05) - result_functional = F.pixel_shuffle(paddle.to_tensor(x), 3, - data_format) - np.testing.assert_allclose(result_functional.numpy(), - npresult, - rtol=1e-05) + result_functional = F.pixel_shuffle( + paddle.to_tensor(x), 3, data_format + ) + np.testing.assert_allclose( + result_functional.numpy(), npresult, rtol=1e-05 + ) def test_dygraph1(self): self.run_dygraph(3, "NCHW") @@ -196,9 +218,7 @@ def test_dygraph2(self): class TestPixelShuffleError(unittest.TestCase): - def test_error_functional(self): - def error_upscale_factor(): with paddle.fluid.dygraph.guard(): x = np.random.random([2, 9, 4, 4]).astype("float64") @@ -214,7 +234,6 @@ def error_data_format(): self.assertRaises(ValueError, error_data_format) def test_error_layer(self): - def error_upscale_factor_layer(): with paddle.fluid.dygraph.guard(): x = np.random.random([2, 9, 4, 4]).astype("float64") diff --git a/python/paddle/fluid/tests/unittests/test_pixel_unshuffle.py b/python/paddle/fluid/tests/unittests/test_pixel_unshuffle.py index 5ab545222d8ee..8f4de9314d3c6 100644 --- a/python/paddle/fluid/tests/unittests/test_pixel_unshuffle.py +++ b/python/paddle/fluid/tests/unittests/test_pixel_unshuffle.py @@ -27,23 +27,41 @@ def pixel_unshuffle_np(x, down_factor, data_format="NCHW"): if data_format == "NCHW": n, c, h, w = x.shape - new_shape = (n, c, h // down_factor, down_factor, w // down_factor, - down_factor) + new_shape = ( + n, + c, + h // down_factor, + down_factor, + w // down_factor, + down_factor, + ) npresult = np.reshape(x, new_shape) npresult = npresult.transpose(0, 1, 3, 5, 2, 4) oshape = [ - n, c * down_factor * down_factor, h // down_factor, w // down_factor + n, + c * down_factor * down_factor, + h // down_factor, + w // down_factor, ] npresult = np.reshape(npresult, oshape) return npresult else: n, h, w, c = x.shape - new_shape = (n, h // down_factor, down_factor, w // down_factor, - down_factor, c) + new_shape = ( + n, + h // down_factor, + down_factor, + w // down_factor, + down_factor, + c, + ) npresult = np.reshape(x, new_shape) npresult = npresult.transpose(0, 1, 3, 5, 2, 4) oshape = [ - n, h // down_factor, w // down_factor, c * down_factor * down_factor + n, + h // down_factor, + w // down_factor, + c * down_factor * down_factor, ] npresult = np.reshape(npresult, oshape) return npresult @@ -73,7 +91,7 @@ def setUp(self): self.outputs = {"Out": npresult} self.attrs = { "downscale_factor": down_factor, - "data_format": self.format + "data_format": self.format, } def init_data_format(self): @@ -115,30 +133,35 @@ def setUp(self): def test_static_graph_functional(self): '''test_static_graph_functional''' - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=[2, 1, 12, 12], - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=[2, 12, 12, 1], - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=[2, 1, 12, 12], dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=[2, 12, 12, 1], dtype="float64" + ) out_1 = F.pixel_unshuffle(x_1, 3) out_2 = F.pixel_unshuffle(x_2, 3, "NHWC") exe = paddle.static.Executor(place=place) - res_1 = exe.run(fluid.default_main_program(), - feed={"x": self.x_1_np}, - fetch_list=out_1, - use_prune=True) - - res_2 = exe.run(fluid.default_main_program(), - feed={"x2": self.x_2_np}, - fetch_list=out_2, - use_prune=True) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True, + ) + + res_2 = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True, + ) assert np.allclose(res_1, self.out_1_np) assert np.allclose(res_2, self.out_2_np) @@ -147,17 +170,18 @@ def test_static_graph_functional(self): def test_static_graph_layer(self): '''test_static_graph_layer''' - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=[2, 1, 12, 12], - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=[2, 12, 12, 1], - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=[2, 1, 12, 12], dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=[2, 12, 12, 1], dtype="float64" + ) # init instance ps_1 = paddle.nn.PixelUnshuffle(3) ps_2 = paddle.nn.PixelUnshuffle(3, "NHWC") @@ -167,15 +191,19 @@ def test_static_graph_layer(self): out_2_np = pixel_unshuffle_np(self.x_2_np, 3, "NHWC") exe = paddle.static.Executor(place=place) - res_1 = exe.run(fluid.default_main_program(), - feed={"x": self.x_1_np}, - fetch_list=out_1, - use_prune=True) - - res_2 = exe.run(fluid.default_main_program(), - feed={"x2": self.x_2_np}, - fetch_list=out_2, - use_prune=True) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True, + ) + + res_2 = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True, + ) assert np.allclose(res_1, out_1_np) assert np.allclose(res_2, out_2_np) @@ -194,23 +222,26 @@ def run_dygraph(self, down_factor, data_format): npresult = pixel_unshuffle_np(x, down_factor, data_format) - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() paddle.disable_static(place=place) - pixel_unshuffle = paddle.nn.PixelUnshuffle(down_factor, - data_format=data_format) + pixel_unshuffle = paddle.nn.PixelUnshuffle( + down_factor, data_format=data_format + ) result = pixel_unshuffle(paddle.to_tensor(x)) np.testing.assert_allclose(result.numpy(), npresult, rtol=1e-05) - result_functional = F.pixel_unshuffle(paddle.to_tensor(x), 3, - data_format) - np.testing.assert_allclose(result_functional.numpy(), - npresult, - rtol=1e-05) + result_functional = F.pixel_unshuffle( + paddle.to_tensor(x), 3, data_format + ) + np.testing.assert_allclose( + result_functional.numpy(), npresult, rtol=1e-05 + ) pixel_unshuffle_str = 'downscale_factor={}'.format(down_factor) if data_format != 'NCHW': @@ -258,8 +289,9 @@ def error_downscale_factor_2(): def error_data_format(): with paddle.fluid.dygraph.guard(): x = np.random.random([2, 1, 12, 12]).astype("float64") - pixel_unshuffle = F.pixel_unshuffle(paddle.to_tensor(x), 3, - "WOW") + pixel_unshuffle = F.pixel_unshuffle( + paddle.to_tensor(x), 3, "WOW" + ) self.assertRaises(ValueError, error_data_format) diff --git a/python/paddle/fluid/tests/unittests/test_poisson_op.py b/python/paddle/fluid/tests/unittests/test_poisson_op.py index d6f878c94738b..4e4a20c77cb5f 100644 --- a/python/paddle/fluid/tests/unittests/test_poisson_op.py +++ b/python/paddle/fluid/tests/unittests/test_poisson_op.py @@ -38,7 +38,6 @@ def output_hist(out, lam, a, b): class TestPoissonOp1(OpTest): - def setUp(self): self.op_type = "poisson" self.config() @@ -67,11 +66,11 @@ def test_check_grad_normal(self): user_defined_grads=[np.zeros([2048, 1024], dtype=self.dtype)], user_defined_grad_outputs=[ np.random.rand(2048, 1024).astype(self.dtype) - ]) + ], + ) class TestPoissonOp2(TestPoissonOp1): - def config(self): self.lam = 5 self.a = 1 @@ -80,18 +79,20 @@ def config(self): class TestPoissonAPI(unittest.TestCase): - def test_static(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x_np = np.random.rand(10, 10) x = paddle.static.data(name="x", shape=[10, 10], dtype='float64') y = paddle.poisson(x) exe = paddle.static.Executor() - y_np = exe.run(paddle.static.default_main_program(), - feed={"x": x_np}, - fetch_list=[y]) + y_np = exe.run( + paddle.static.default_main_program(), + feed={"x": x_np}, + fetch_list=[y], + ) self.assertTrue(np.min(y_np) >= 0) def test_dygraph(self): @@ -117,70 +118,250 @@ def test_fixed_random_number(self): paddle.disable_static() paddle.set_device('gpu') paddle.seed(2021) - x = paddle.full([32, 3, 1024, 768], 10., dtype="float32") + x = paddle.full([32, 3, 1024, 768], 10.0, dtype="float32") y = paddle.poisson(x) y_np = y.numpy() expect = [ - 13., 13., 11., 8., 12., 6., 9., 15., 16., 6., 13., 12., 9., 15., - 17., 8., 11., 16., 11., 10. + 13.0, + 13.0, + 11.0, + 8.0, + 12.0, + 6.0, + 9.0, + 15.0, + 16.0, + 6.0, + 13.0, + 12.0, + 9.0, + 15.0, + 17.0, + 8.0, + 11.0, + 16.0, + 11.0, + 10.0, ] np.testing.assert_array_equal(y_np[0, 0, 0, 0:20], expect) expect = [ - 15., 7., 12., 8., 14., 10., 10., 11., 11., 11., 21., 6., 9., 13., - 13., 11., 6., 9., 12., 12. + 15.0, + 7.0, + 12.0, + 8.0, + 14.0, + 10.0, + 10.0, + 11.0, + 11.0, + 11.0, + 21.0, + 6.0, + 9.0, + 13.0, + 13.0, + 11.0, + 6.0, + 9.0, + 12.0, + 12.0, ] np.testing.assert_array_equal(y_np[8, 1, 300, 200:220], expect) expect = [ - 10., 15., 9., 6., 4., 13., 10., 10., 13., 12., 9., 7., 10., 14., 7., - 10., 8., 5., 10., 14. + 10.0, + 15.0, + 9.0, + 6.0, + 4.0, + 13.0, + 10.0, + 10.0, + 13.0, + 12.0, + 9.0, + 7.0, + 10.0, + 14.0, + 7.0, + 10.0, + 8.0, + 5.0, + 10.0, + 14.0, ] np.testing.assert_array_equal(y_np[16, 1, 600, 400:420], expect) expect = [ - 10., 9., 14., 12., 8., 9., 7., 8., 11., 10., 13., 8., 12., 9., 7., - 8., 11., 11., 12., 5. + 10.0, + 9.0, + 14.0, + 12.0, + 8.0, + 9.0, + 7.0, + 8.0, + 11.0, + 10.0, + 13.0, + 8.0, + 12.0, + 9.0, + 7.0, + 8.0, + 11.0, + 11.0, + 12.0, + 5.0, ] np.testing.assert_array_equal(y_np[24, 2, 900, 600:620], expect) expect = [ - 15., 5., 11., 13., 12., 12., 13., 16., 9., 9., 7., 9., 13., 11., - 15., 6., 11., 9., 10., 10. + 15.0, + 5.0, + 11.0, + 13.0, + 12.0, + 12.0, + 13.0, + 16.0, + 9.0, + 9.0, + 7.0, + 9.0, + 13.0, + 11.0, + 15.0, + 6.0, + 11.0, + 9.0, + 10.0, + 10.0, ] np.testing.assert_array_equal(y_np[31, 2, 1023, 748:768], expect) - x = paddle.full([16, 1024, 1024], 5., dtype="float32") + x = paddle.full([16, 1024, 1024], 5.0, dtype="float32") y = paddle.poisson(x) y_np = y.numpy() expect = [ - 4., 5., 2., 9., 8., 7., 4., 7., 4., 7., 6., 3., 10., 7., 5., 7., 2., - 5., 5., 6. + 4.0, + 5.0, + 2.0, + 9.0, + 8.0, + 7.0, + 4.0, + 7.0, + 4.0, + 7.0, + 6.0, + 3.0, + 10.0, + 7.0, + 5.0, + 7.0, + 2.0, + 5.0, + 5.0, + 6.0, ] np.testing.assert_array_equal(y_np[0, 0, 100:120], expect) expect = [ - 1., 4., 8., 11., 6., 5., 4., 4., 7., 4., 4., 7., 11., 6., 5., 3., - 4., 6., 3., 3. + 1.0, + 4.0, + 8.0, + 11.0, + 6.0, + 5.0, + 4.0, + 4.0, + 7.0, + 4.0, + 4.0, + 7.0, + 11.0, + 6.0, + 5.0, + 3.0, + 4.0, + 6.0, + 3.0, + 3.0, ] np.testing.assert_array_equal(y_np[4, 300, 300:320], expect) expect = [ - 7., 5., 4., 6., 8., 5., 6., 7., 7., 7., 3., 10., 5., 10., 4., 5., - 8., 7., 5., 7. + 7.0, + 5.0, + 4.0, + 6.0, + 8.0, + 5.0, + 6.0, + 7.0, + 7.0, + 7.0, + 3.0, + 10.0, + 5.0, + 10.0, + 4.0, + 5.0, + 8.0, + 7.0, + 5.0, + 7.0, ] np.testing.assert_array_equal(y_np[8, 600, 600:620], expect) expect = [ - 8., 6., 7., 4., 3., 0., 4., 6., 6., 4., 3., 10., 5., 1., 3., 8., 8., - 2., 1., 4. + 8.0, + 6.0, + 7.0, + 4.0, + 3.0, + 0.0, + 4.0, + 6.0, + 6.0, + 4.0, + 3.0, + 10.0, + 5.0, + 1.0, + 3.0, + 8.0, + 8.0, + 2.0, + 1.0, + 4.0, ] np.testing.assert_array_equal(y_np[12, 900, 900:920], expect) expect = [ - 2., 1., 14., 3., 6., 5., 2., 2., 6., 5., 7., 4., 8., 4., 8., 4., 5., - 7., 1., 7. + 2.0, + 1.0, + 14.0, + 3.0, + 6.0, + 5.0, + 2.0, + 2.0, + 6.0, + 5.0, + 7.0, + 4.0, + 8.0, + 4.0, + 8.0, + 4.0, + 5.0, + 7.0, + 1.0, + 7.0, ] np.testing.assert_array_equal(y_np[15, 1023, 1000:1020], expect) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py b/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py index 84f61f7bebc4e..b5deecc47007d 100644 --- a/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py +++ b/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py @@ -24,23 +24,27 @@ def PolygonBoxRestore(input): geo_channels = shape[1] h = shape[2] w = shape[3] - h_indexes = np.array(list(range(h)) * w).reshape( - [w, h]).transpose()[np.newaxis, :] # [1, h, w] - w_indexes = np.array(list(range(w)) * h).reshape( - [h, w])[np.newaxis, :] # [1, h, w] - indexes = np.concatenate( - (w_indexes, h_indexes))[np.newaxis, :] # [1, 2, h, w] + h_indexes = ( + np.array(list(range(h)) * w).reshape([w, h]).transpose()[np.newaxis, :] + ) # [1, h, w] + w_indexes = np.array(list(range(w)) * h).reshape([h, w])[ + np.newaxis, : + ] # [1, h, w] + indexes = np.concatenate((w_indexes, h_indexes))[ + np.newaxis, : + ] # [1, 2, h, w] + indexes = indexes.repeat([geo_channels / 2], axis=0)[ + np.newaxis, : + ] # [1, geo_channels/2, 2, h, w] indexes = indexes.repeat( - [geo_channels / 2], - axis=0)[np.newaxis, :] # [1, geo_channels/2, 2, h, w] - indexes = indexes.repeat([batch_size], - axis=0) # [batch_size, geo_channels/2, 2, h, w] - return indexes.reshape( - input.shape) * 4 - input # [batch_size, geo_channels, h, w] + [batch_size], axis=0 + ) # [batch_size, geo_channels/2, 2, h, w] + return ( + indexes.reshape(input.shape) * 4 - input + ) # [batch_size, geo_channels, h, w] class TestPolygonBoxRestoreOp(OpTest): - def config(self): self.input_shape = (1, 8, 2, 2) @@ -57,25 +61,21 @@ def test_check_output(self): class TestCase1(TestPolygonBoxRestoreOp): - def config(self): self.input_shape = (2, 10, 3, 2) class TestCase2(TestPolygonBoxRestoreOp): - def config(self): self.input_shape = (3, 12, 4, 5) class TestPolygonBoxInvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_input(): - input = fluid.data(name='input', - shape=[None, 3, 32, 32], - dtype='int64') + input = fluid.data( + name='input', shape=[None, 3, 32, 32], dtype='int64' + ) out = fluid.layers.polygon_box_transform(input) self.assertRaises(TypeError, test_invalid_input) diff --git a/python/paddle/fluid/tests/unittests/test_pool1d_api.py b/python/paddle/fluid/tests/unittests/test_pool1d_api.py index 19d2081734216..a3aa60e0ebd10 100644 --- a/python/paddle/fluid/tests/unittests/test_pool1d_api.py +++ b/python/paddle/fluid/tests/unittests/test_pool1d_api.py @@ -29,24 +29,28 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def max_pool1D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=False, - adaptive=False, - data_type=np.float64): +def max_pool1D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=False, + adaptive=False, + data_type=np.float64, +): N, C, L = x.shape if global_pool == 1: ksize = [L] if adaptive: L_out = ksize[0] else: - L_out = (L - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + L_out = ( + (L - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) out = np.zeros((N, C, L_out)) for i in range(L_out): @@ -62,24 +66,28 @@ def max_pool1D_forward_naive(x, return out -def avg_pool1D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=False, - adaptive=False, - data_type=np.float64): +def avg_pool1D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=False, + adaptive=False, + data_type=np.float64, +): N, C, L = x.shape if global_pool == 1: ksize = [L] if adaptive: L_out = ksize[0] else: - L_out = (L - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + L_out = ( + (L - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (L - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) out = np.zeros((N, C, L_out)) for i in range(L_out): @@ -91,19 +99,21 @@ def avg_pool1D_forward_naive(x, r_end = np.min((i * strides[0] + ksize[0] - paddings[0], L)) x_masked = x[:, :, r_start:r_end] - field_size = (r_end - r_start) \ - if (exclusive or adaptive) else (ksize[0]) + field_size = ( + (r_end - r_start) if (exclusive or adaptive) else (ksize[0]) + ) if data_type == np.int8 or data_type == np.uint8: - out[:, :, i] = (np.rint(np.sum(x_masked, axis=(2, 3)) / - field_size)).astype(data_type) + out[:, :, i] = ( + np.rint(np.sum(x_masked, axis=(2, 3)) / field_size) + ).astype(data_type) else: - out[:, :, - i] = (np.sum(x_masked, axis=(2)) / field_size).astype(data_type) + out[:, :, i] = (np.sum(x_masked, axis=(2)) / field_size).astype( + data_type + ) return out class TestPool1D_API(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -116,16 +126,16 @@ def check_avg_static_results(self, place): result = F.avg_pool1d(input, kernel_size=2, stride=2, padding=0) input_np = np.random.random([2, 3, 32]).astype("float32") - result_np = avg_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[0], - ceil_mode=False) + result_np = avg_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[0], ceil_mode=False + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def check_avg_dygraph_results(self, place): @@ -134,16 +144,15 @@ def check_avg_dygraph_results(self, place): input = fluid.dygraph.to_variable(input_np) result = F.avg_pool1d(input, kernel_size=2, stride=2, padding=[0]) - result_np = avg_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[0]) + result_np = avg_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[0] + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool1d_dg = paddle.nn.layer.AvgPool1D(kernel_size=2, - stride=None, - padding=0) + avg_pool1d_dg = paddle.nn.layer.AvgPool1D( + kernel_size=2, stride=None, padding=0 + ) result = avg_pool1d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -151,24 +160,19 @@ def check_avg_dygraph_padding_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = F.avg_pool1d(input, - kernel_size=2, - stride=2, - padding=[1], - exclusive=True) - - result_np = avg_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[1], - exclusive=False) + result = F.avg_pool1d( + input, kernel_size=2, stride=2, padding=[1], exclusive=True + ) + + result_np = avg_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[1], exclusive=False + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool1d_dg = paddle.nn.AvgPool1D(kernel_size=2, - stride=None, - padding=1, - exclusive=True) + avg_pool1d_dg = paddle.nn.AvgPool1D( + kernel_size=2, stride=None, padding=1, exclusive=True + ) result = avg_pool1d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -179,15 +183,16 @@ def check_max_static_results(self, place): result = F.max_pool1d(input, kernel_size=2, stride=2, padding=[0]) input_np = np.random.random([2, 3, 32]).astype("float32") - result_np = max_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[0]) + result_np = max_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[0] + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def check_max_dygraph_results(self, place): @@ -196,16 +201,15 @@ def check_max_dygraph_results(self, place): input = fluid.dygraph.to_variable(input_np) result = F.max_pool1d(input, kernel_size=2, stride=2, padding=0) - result_np = max_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[0]) + result_np = max_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[0] + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool1d_dg = paddle.nn.layer.MaxPool1D(kernel_size=2, - stride=None, - padding=0) + max_pool1d_dg = paddle.nn.layer.MaxPool1D( + kernel_size=2, stride=None, padding=0 + ) result = max_pool1d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -213,22 +217,19 @@ def check_max_dygraph_return_index_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result, index = F.max_pool1d(input, - kernel_size=2, - stride=2, - padding=0, - return_mask=True) + result, index = F.max_pool1d( + input, kernel_size=2, stride=2, padding=0, return_mask=True + ) - result_np = max_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[0]) + result_np = max_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[0] + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool1d_dg = paddle.nn.layer.MaxPool1D(kernel_size=2, - stride=None, - padding=0) + max_pool1d_dg = paddle.nn.layer.MaxPool1D( + kernel_size=2, stride=None, padding=0 + ) result = max_pool1d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -236,15 +237,13 @@ def check_max_dygraph_padding_same(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = F.max_pool1d(input, - kernel_size=2, - stride=2, - padding="SAME") + result = F.max_pool1d( + input, kernel_size=2, stride=2, padding="SAME" + ) - result_np = max_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[0]) + result_np = max_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[0] + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -252,15 +251,13 @@ def check_avg_dygraph_padding_same(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = F.avg_pool1d(input, - kernel_size=2, - stride=2, - padding="SAME") + result = F.avg_pool1d( + input, kernel_size=2, stride=2, padding="SAME" + ) - result_np = avg_pool1D_forward_naive(input_np, - ksize=[2], - strides=[2], - paddings=[0]) + result_np = avg_pool1D_forward_naive( + input_np, ksize=[2], strides=[2], paddings=[0] + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -281,129 +278,145 @@ def test_dygraph_api(self): class TestPool2DError_API(unittest.TestCase): - def test_error_api(self): - def run1(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = [[2]] - res_pd = F.max_pool1d(input_pd, - kernel_size=2, - stride=2, - padding=padding) + res_pd = F.max_pool1d( + input_pd, kernel_size=2, stride=2, padding=padding + ) self.assertRaises(ValueError, run1) def run2(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = [[2]] - res_pd = F.max_pool1d(input_pd, - kernel_size=2, - stride=2, - padding=padding) + res_pd = F.max_pool1d( + input_pd, kernel_size=2, stride=2, padding=padding + ) self.assertRaises(ValueError, run2) def run3(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "padding" - res_pd = F.max_pool1d(input_pd, - kernel_size=2, - stride=2, - padding=padding) + res_pd = F.max_pool1d( + input_pd, kernel_size=2, stride=2, padding=padding + ) self.assertRaises(ValueError, run3) def run4(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = F.max_pool1d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=True) + res_pd = F.max_pool1d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=True, + ) self.assertRaises(ValueError, run4) def run5(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = F.max_pool1d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=True) + res_pd = F.max_pool1d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=True, + ) self.assertRaises(ValueError, run5) def run6(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = F.avg_pool1d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=True) + res_pd = F.avg_pool1d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=True, + ) self.assertRaises(ValueError, run6) def run7(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "paddle" - res_pd = F.avg_pool1d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=True) + res_pd = F.avg_pool1d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=True, + ) self.assertRaises(ValueError, run7) def run_kernel_out_of_range(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = 0 - res_pd = F.avg_pool1d(input_pd, - kernel_size=-1, - stride=2, - padding=padding, - ceil_mode=True) + res_pd = F.avg_pool1d( + input_pd, + kernel_size=-1, + stride=2, + padding=padding, + ceil_mode=True, + ) self.assertRaises(ValueError, run_kernel_out_of_range) def run_stride_out_of_range(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = 0 - res_pd = F.avg_pool1d(input_pd, - kernel_size=2, - stride=0, - padding=padding, - ceil_mode=True) + res_pd = F.avg_pool1d( + input_pd, + kernel_size=2, + stride=0, + padding=padding, + ceil_mode=True, + ) self.assertRaises(ValueError, run_stride_out_of_range) diff --git a/python/paddle/fluid/tests/unittests/test_pool2d_api.py b/python/paddle/fluid/tests/unittests/test_pool2d_api.py index d20844d255876..7e100d4c90558 100644 --- a/python/paddle/fluid/tests/unittests/test_pool2d_api.py +++ b/python/paddle/fluid/tests/unittests/test_pool2d_api.py @@ -19,11 +19,14 @@ import paddle.fluid.core as core from paddle.fluid.framework import _test_eager_guard from paddle.nn.functional import avg_pool2d, max_pool2d -from test_pool2d_op import avg_pool2D_forward_naive, max_pool2D_forward_naive, pool2D_forward_naive +from test_pool2d_op import ( + avg_pool2D_forward_naive, + max_pool2D_forward_naive, + pool2D_forward_naive, +) class TestPool2D_API(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -32,22 +35,26 @@ def setUp(self): def check_avg_static_results(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.data(name="input", - shape=[2, 3, 32, 32], - dtype="float32") + input = fluid.data( + name="input", shape=[2, 3, 32, 32], dtype="float32" + ) result = avg_pool2d(input, kernel_size=2, stride=2, padding=0) input_np = np.random.random([2, 3, 32, 32]).astype("float32") - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='avg') + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='avg', + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def check_avg_dygraph_results(self, place): @@ -56,16 +63,18 @@ def check_avg_dygraph_results(self, place): input = fluid.dygraph.to_variable(input_np) result = avg_pool2d(input, kernel_size=2, stride=2, padding=0) - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='avg') + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='avg', + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool2d_dg = paddle.nn.layer.AvgPool2D(kernel_size=2, - stride=2, - padding=0) + avg_pool2d_dg = paddle.nn.layer.AvgPool2D( + kernel_size=2, stride=2, padding=0 + ) result = avg_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -73,24 +82,23 @@ def check_avg_dygraph_padding_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = avg_pool2d(input, - kernel_size=2, - stride=2, - padding=1, - ceil_mode=False) - - result_np = avg_pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[1, 1], - ceil_mode=False, - exclusive=False) + result = avg_pool2d( + input, kernel_size=2, stride=2, padding=1, ceil_mode=False + ) + + result_np = avg_pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[1, 1], + ceil_mode=False, + exclusive=False, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool2d_dg = paddle.nn.layer.AvgPool2D(kernel_size=2, - stride=2, - padding=1, - ceil_mode=False) + avg_pool2d_dg = paddle.nn.layer.AvgPool2D( + kernel_size=2, stride=2, padding=1, ceil_mode=False + ) result = avg_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -98,66 +106,69 @@ def check_avg_dygraph_ceilmode_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = avg_pool2d(input, - kernel_size=2, - stride=2, - padding=0, - ceil_mode=True) - - result_np = avg_pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - ceil_mode=True) + result = avg_pool2d( + input, kernel_size=2, stride=2, padding=0, ceil_mode=True + ) + + result_np = avg_pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + ceil_mode=True, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool2d_dg = paddle.nn.layer.AvgPool2D(kernel_size=2, - stride=2, - padding=0, - ceil_mode=True) + avg_pool2d_dg = paddle.nn.layer.AvgPool2D( + kernel_size=2, stride=2, padding=0, ceil_mode=True + ) result = avg_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) def check_max_static_results(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.data(name="input", - shape=[2, 3, 32, 32], - dtype="float32") + input = fluid.data( + name="input", shape=[2, 3, 32, 32], dtype="float32" + ) result = max_pool2d(input, kernel_size=2, stride=2, padding=0) input_np = np.random.random([2, 3, 32, 32]).astype("float32") - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='max') + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='max', + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def check_max_dygraph_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = max_pool2d(input, - kernel_size=2, - stride=2, - padding=0, - return_mask=False) - - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='max') + result = max_pool2d( + input, kernel_size=2, stride=2, padding=0, return_mask=False + ) + + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='max', + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool2d_dg = paddle.nn.layer.MaxPool2D(kernel_size=2, - stride=2, - padding=0) + max_pool2d_dg = paddle.nn.layer.MaxPool2D( + kernel_size=2, stride=2, padding=0 + ) result = max_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -165,46 +176,51 @@ def check_max_dygraph_nhwc_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable( - np.transpose(input_np, [0, 2, 3, 1])) - result = max_pool2d(input, - kernel_size=2, - stride=2, - padding=0, - return_mask=False, - data_format="NHWC") - - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='max') - np.testing.assert_allclose(np.transpose(result.numpy(), - [0, 3, 1, 2]), - result_np, - rtol=1e-05) + np.transpose(input_np, [0, 2, 3, 1]) + ) + result = max_pool2d( + input, + kernel_size=2, + stride=2, + padding=0, + return_mask=False, + data_format="NHWC", + ) + + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='max', + ) + np.testing.assert_allclose( + np.transpose(result.numpy(), [0, 3, 1, 2]), + result_np, + rtol=1e-05, + ) def check_max_dygraph_padding_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = max_pool2d(input, - kernel_size=2, - stride=2, - padding=1, - ceil_mode=False) - - result_np = max_pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[1, 1], - ceil_mode=False, - exclusive=False) + result = max_pool2d( + input, kernel_size=2, stride=2, padding=1, ceil_mode=False + ) + + result_np = max_pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[1, 1], + ceil_mode=False, + exclusive=False, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool2d_dg = paddle.nn.layer.MaxPool2D(kernel_size=2, - stride=2, - padding=1, - ceil_mode=False) + max_pool2d_dg = paddle.nn.layer.MaxPool2D( + kernel_size=2, stride=2, padding=1, ceil_mode=False + ) result = max_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -212,23 +228,22 @@ def check_max_dygraph_ceilmode_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = max_pool2d(input, - kernel_size=2, - stride=2, - padding=0, - ceil_mode=True) - - result_np = max_pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - ceil_mode=True) + result = max_pool2d( + input, kernel_size=2, stride=2, padding=0, ceil_mode=True + ) + + result_np = max_pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + ceil_mode=True, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool2d_dg = paddle.nn.layer.MaxPool2D(kernel_size=2, - stride=2, - padding=0, - ceil_mode=True) + max_pool2d_dg = paddle.nn.layer.MaxPool2D( + kernel_size=2, stride=2, padding=0, ceil_mode=True + ) result = max_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -236,23 +251,27 @@ def check_max_dygraph_stride_is_none(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result, indices = max_pool2d(input, - kernel_size=2, - stride=None, - padding="SAME", - return_mask=True) - - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='max', - padding_algorithm="SAME") + result, indices = max_pool2d( + input, + kernel_size=2, + stride=None, + padding="SAME", + return_mask=True, + ) + + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='max', + padding_algorithm="SAME", + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool2d_dg = paddle.nn.layer.MaxPool2D(kernel_size=2, - stride=2, - padding=0) + max_pool2d_dg = paddle.nn.layer.MaxPool2D( + kernel_size=2, stride=2, padding=0 + ) result = max_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -260,22 +279,23 @@ def check_avg_dygraph_stride_is_none(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = avg_pool2d(input, - kernel_size=2, - stride=None, - padding="SAME") - - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='avg', - padding_algorithm="SAME") + result = avg_pool2d( + input, kernel_size=2, stride=None, padding="SAME" + ) + + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='avg', + padding_algorithm="SAME", + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool2d_dg = paddle.nn.layer.AvgPool2D(kernel_size=2, - stride=2, - padding=0) + avg_pool2d_dg = paddle.nn.layer.AvgPool2D( + kernel_size=2, stride=2, padding=0 + ) result = avg_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -284,22 +304,26 @@ def check_max_dygraph_padding(self, place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) padding = [[0, 0], [0, 0], [0, 0], [0, 0]] - result = max_pool2d(input, - kernel_size=2, - stride=2, - padding=padding, - return_mask=False) - - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='max') + result = max_pool2d( + input, + kernel_size=2, + stride=2, + padding=padding, + return_mask=False, + ) + + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='max', + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool2d_dg = paddle.nn.layer.MaxPool2D(kernel_size=2, - stride=2, - padding=0) + max_pool2d_dg = paddle.nn.layer.MaxPool2D( + kernel_size=2, stride=2, padding=0 + ) result = max_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -308,22 +332,26 @@ def check_avg_divisor(self, place): input_np = np.random.random([2, 3, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) padding = [[0, 0], [0, 0], [0, 0], [0, 0]] - result = avg_pool2d(input, - kernel_size=2, - stride=2, - padding=padding, - divisor_override=4) - - result_np = pool2D_forward_naive(input_np, - ksize=[2, 2], - strides=[2, 2], - paddings=[0, 0], - pool_type='avg') + result = avg_pool2d( + input, + kernel_size=2, + stride=2, + padding=padding, + divisor_override=4, + ) + + result_np = pool2D_forward_naive( + input_np, + ksize=[2, 2], + strides=[2, 2], + paddings=[0, 0], + pool_type='avg', + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool2d_dg = paddle.nn.layer.AvgPool2D(kernel_size=2, - stride=2, - padding=0) + avg_pool2d_dg = paddle.nn.layer.AvgPool2D( + kernel_size=2, stride=2, padding=0 + ) result = avg_pool2d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -348,193 +376,227 @@ def test_dygraph_api(self): class TestPool2DError_API(unittest.TestCase): - def test_error_api(self): - def run1(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = [[0, 1], [0, 0], [0, 0], [0, 0]] - res_pd = max_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding) + res_pd = max_pool2d( + input_pd, kernel_size=2, stride=2, padding=padding + ) self.assertRaises(ValueError, run1) def run2(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = [[0, 1], [0, 0], [0, 0], [0, 0]] - res_pd = max_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - data_format='NHWC') + res_pd = max_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + data_format='NHWC', + ) self.assertRaises(ValueError, run2) def run3(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "padding" - res_pd = max_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - data_format='NHWC') + res_pd = max_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + data_format='NHWC', + ) self.assertRaises(ValueError, run3) def run3_avg(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "padding" - res_pd = avg_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - data_format='NHWC') + res_pd = avg_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + data_format='NHWC', + ) self.assertRaises(ValueError, run3_avg) def run4(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = max_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=True, - data_format='NHWC') + res_pd = max_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=True, + data_format='NHWC', + ) self.assertRaises(ValueError, run4) def run4_avg(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = avg_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=True, - data_format='NHWC') + res_pd = avg_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=True, + data_format='NHWC', + ) self.assertRaises(ValueError, run4_avg) def run5(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "padding" - res_pd = avg_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - data_format='NHWC') + res_pd = avg_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + data_format='NHWC', + ) self.assertRaises(ValueError, run5) def run6(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = avg_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=True, - data_format='NHWC') + res_pd = avg_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=True, + data_format='NHWC', + ) self.assertRaises(ValueError, run6) def run7(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = avg_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=False, - data_format='NNNN') + res_pd = avg_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=False, + data_format='NNNN', + ) self.assertRaises(ValueError, run7) def run8(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = "VALID" - res_pd = max_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - ceil_mode=False, - data_format='NNNN') + res_pd = max_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + ceil_mode=False, + data_format='NNNN', + ) self.assertRaises(ValueError, run8) def run9(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = max_pool2d(input_pd, - kernel_size=2, - stride=2, - padding=0, - ceil_mode=False, - data_format='NHWC', - return_mask=True) + res_pd = max_pool2d( + input_pd, + kernel_size=2, + stride=2, + padding=0, + ceil_mode=False, + data_format='NHWC', + return_mask=True, + ) self.assertRaises(ValueError, run9) def run_kernel_out_of_range(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = avg_pool2d(input_pd, - kernel_size=[-1, 2], - stride=2, - padding=0, - ceil_mode=False, - data_format='NHWC') + res_pd = avg_pool2d( + input_pd, + kernel_size=[-1, 2], + stride=2, + padding=0, + ceil_mode=False, + data_format='NHWC', + ) self.assertRaises(ValueError, run_kernel_out_of_range) def run_stride_out_of_range(): with fluid.dygraph.guard(): - input_np = np.random.uniform(-1, 1, - [2, 3, 32, 32]).astype(np.float32) + input_np = np.random.uniform(-1, 1, [2, 3, 32, 32]).astype( + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = avg_pool2d(input_pd, - kernel_size=3, - stride=[0, 2], - padding=0, - ceil_mode=False, - data_format='NHWC') + res_pd = avg_pool2d( + input_pd, + kernel_size=3, + stride=[0, 2], + padding=0, + ceil_mode=False, + data_format='NHWC', + ) self.assertRaises(ValueError, run_stride_out_of_range) diff --git a/python/paddle/fluid/tests/unittests/test_pool2d_op.py b/python/paddle/fluid/tests/unittests/test_pool2d_op.py index 46a822a513893..7c44827262d6c 100644 --- a/python/paddle/fluid/tests/unittests/test_pool2d_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool2d_op.py @@ -29,15 +29,17 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def max_pool2D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_type=np.float64): +def max_pool2D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_type=np.float64, +): if data_type == np.float64 and core.is_compiled_with_rocm(): data_type = np.float32 N, C, H, W = x.shape @@ -46,12 +48,16 @@ def max_pool2D_forward_naive(x, if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 - W_out = (W - ksize[1] + 2 * paddings[1] + strides[1] - - 1) // strides[1] + 1 if ceil_mode else ( - W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + H_out = ( + (H - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + 2 * paddings[1] + strides[1] - 1) // strides[1] + 1 + if ceil_mode + else (W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + ) out = np.zeros((N, C, H_out, W_out)) for i in range(H_out): for j in range(W_out): @@ -71,15 +77,17 @@ def max_pool2D_forward_naive(x, return out -def avg_pool2D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_type=np.float64): +def avg_pool2D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_type=np.float64, +): if data_type == np.float64 and core.is_compiled_with_rocm(): data_type = np.float32 N, C, H, W = x.shape @@ -88,12 +96,16 @@ def avg_pool2D_forward_naive(x, if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 - W_out = (W - ksize[1] + 2 * paddings[1] + strides[1] - - 1) // strides[1] + 1 if ceil_mode else ( - W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + H_out = ( + (H - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + 2 * paddings[1] + strides[1] - 1) // strides[1] + 1 + if ceil_mode + else (W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + ) out = np.zeros((N, C, H_out, W_out)) for i in range(H_out): for j in range(W_out): @@ -115,39 +127,44 @@ def avg_pool2D_forward_naive(x, x_masked = x[:, :, r_start:r_end, c_start:c_end] - if (exclusive or adaptive): + if exclusive or adaptive: field_size = (r_end - r_start) * (c_end - c_start) if data_type == np.int8 or data_type == np.uint8: - out[:, :, i, - j] = (np.rint(np.sum(x_masked, axis=(2, 3)) / - field_size)).astype(data_type) + out[:, :, i, j] = ( + np.rint(np.sum(x_masked, axis=(2, 3)) / field_size) + ).astype(data_type) else: - out[:, :, i, j] = (np.sum(x_masked, axis=(2, 3)) / - field_size).astype(data_type) + out[:, :, i, j] = ( + np.sum(x_masked, axis=(2, 3)) / field_size + ).astype(data_type) return out -def pool2D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_format='NCHW', - pool_type="max", - padding_algorithm="EXPLICIT"): +def pool2D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_format='NCHW', + pool_type="max", + padding_algorithm="EXPLICIT", +): # update paddings def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -157,9 +174,10 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): if isinstance(padding_algorithm, str): padding_algorithm = padding_algorithm.upper() if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if padding_algorithm == "VALID": paddings = [0, 0, 0, 0] @@ -167,7 +185,8 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): raise ValueError( "When Attr(pool_padding) is \"VALID\", Attr(ceil_mode)" " must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) elif padding_algorithm == "SAME": input_data_shape = [] if data_format == "NCHW": @@ -180,8 +199,11 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): is_sys = True if len(paddings) == 2 else False N = x.shape[0] - C, H, W = [x.shape[1], x.shape[2], x.shape[3]] if data_format == 'NCHW' \ + C, H, W = ( + [x.shape[1], x.shape[2], x.shape[3]] + if data_format == 'NCHW' else [x.shape[3], x.shape[1], x.shape[2]] + ) if global_pool == 1: ksize = [H, W] @@ -195,13 +217,26 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) // strides[0] + 1 \ - if ceil_mode else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 - W_out = (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) // strides[1] + 1 \ - if ceil_mode else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 - - out = np.zeros((N, C, H_out, W_out)) if data_format=='NCHW' \ + H_out = ( + (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) + // strides[0] + + 1 + if ceil_mode + else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) + // strides[1] + + 1 + if ceil_mode + else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 + ) + + out = ( + np.zeros((N, C, H_out, W_out)) + if data_format == 'NCHW' else np.zeros((N, H_out, W_out, C)) + ) for i in range(H_out): if adaptive: in_h_start = adaptive_start_index(i, H, ksize[0]) @@ -229,21 +264,22 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): if data_format == 'NCHW': x_masked = x[:, :, in_h_start:in_h_end, in_w_start:in_w_end] if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (in_h_end - in_h_start) * (in_w_end - - in_w_start) - + if exclusive or adaptive: + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) -# if (exclusive or adaptive) else (ksize[0] * ksize[1]) + # if (exclusive or adaptive) else (ksize[0] * ksize[1]) out[:, :, i, j] = np.sum(x_masked, axis=(2, 3)) / field_size elif pool_type == 'max': out[:, :, i, j] = np.max(x_masked, axis=(2, 3)) elif data_format == 'NHWC': x_masked = x[:, in_h_start:in_h_end, in_w_start:in_w_end, :] if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (in_h_end - in_h_start) * (in_w_end - - in_w_start) + if exclusive or adaptive: + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) out[:, i, j, :] = np.sum(x_masked, axis=(1, 2)) / field_size elif pool_type == 'max': out[:, i, j, :] = np.max(x_masked, axis=(1, 2)) @@ -251,7 +287,6 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): class TestPool2D_Op_Mixin(object): - def setUp(self): self.op_type = "pool2d" self.use_cudnn = False @@ -271,12 +306,19 @@ def setUp(self): self.init_shape() input = np.random.random(self.shape).astype(self.dtype) - output = pool2D_forward_naive(input, self.ksize, self.strides, - self.paddings, self.global_pool, - self.ceil_mode, self.exclusive, - self.adaptive, self.data_format, - self.pool_type, - self.padding_algorithm).astype(self.dtype) + output = pool2D_forward_naive( + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + self.data_format, + self.pool_type, + self.padding_algorithm, + ).astype(self.dtype) self.inputs = {'X': OpTest.np_dtype_to_fluid_dtype(input)} self.attrs = { @@ -304,7 +346,8 @@ def test_check_output(self): if self.has_cudnn(): place = core.CUDAPlace(0) self.check_output_with_place( - place, atol=1e-5, check_dygraph=(self.use_mkldnn == False)) + place, atol=1e-5, check_dygraph=(self.use_mkldnn == False) + ) else: self.check_output(check_dygraph=(self.use_mkldnn == False)) @@ -314,16 +357,20 @@ def test_check_grad(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode if self.has_cudnn() and self.pool_type != "max": place = core.CUDAPlace(0) - self.check_grad_with_place(place, - set(['X']), - 'Out', - max_relative_error=0.07, - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + set(['X']), + 'Out', + max_relative_error=0.07, + check_dygraph=(self.use_mkldnn == False), + ) elif self.pool_type != "max": - self.check_grad(set(['X']), - 'Out', - max_relative_error=0.07, - check_dygraph=(self.use_mkldnn == False)) + self.check_grad( + set(['X']), + 'Out', + max_relative_error=0.07, + check_dygraph=(self.use_mkldnn == False), + ) def init_data_format(self): self.data_format = "NCHW" @@ -367,7 +414,6 @@ class TestPool2D_Op(TestPool2D_Op_Mixin, OpTest): class TestCase1(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -387,7 +433,6 @@ def init_shape(self): class TestCase2(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -407,35 +452,31 @@ def init_shape(self): class TestCase3(TestPool2D_Op): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase4(TestCase1): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase5(TestCase2): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive -#--------------------test pool2d cudnn-------------------- +# --------------------test pool2d cudnn-------------------- def create_test_cudnn_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNCase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -451,15 +492,14 @@ def init_kernel_type(self): create_test_cudnn_class(TestCase4) create_test_cudnn_class(TestCase5) -#--------------------test pool2d cudnn_fp16-------------------- +# --------------------test pool2d cudnn_fp16-------------------- def create_test_cudnn_fp16_class(parent, check_grad=True): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNFp16Case(parent): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -472,19 +512,24 @@ def test_check_output(self): self.check_output_with_place( place, atol=1e-3, - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode place = core.CUDAPlace(0) - if core.is_float16_supported( - place) and self.pool_type != "max" and check_grad: + if ( + core.is_float16_supported(place) + and self.pool_type != "max" + and check_grad + ): self.check_grad_with_place( place, set(['X']), 'Out', max_relative_error=0.07, - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) cls_name = "{0}_{1}".format(parent.__name__, "CUDNNFp16Op") TestCUDNNFp16Case.__name__ = cls_name @@ -492,11 +537,10 @@ def test_check_grad(self): def create_test_fp16_class(parent, check_grad=True): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestFp16Case(parent): - def init_kernel_type(self): self.use_cudnn = False self.dtype = np.float16 @@ -509,19 +553,24 @@ def test_check_output(self): self.check_output_with_place( place, atol=1e-3, - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode place = core.CUDAPlace(0) - if core.is_float16_supported( - place) and self.pool_type != "max" and check_grad: + if ( + core.is_float16_supported(place) + and self.pool_type != "max" + and check_grad + ): self.check_grad_with_place( place, set(['X']), 'Out', max_relative_error=0.07, - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) cls_name = "{0}_{1}".format(parent.__name__, "Fp16Op") TestFp16Case.__name__ = cls_name @@ -542,15 +591,14 @@ def test_check_grad(self): create_test_fp16_class(TestCase4) create_test_fp16_class(TestCase5) -#--------------------test pool2d use ceil mode-------------------- +# --------------------test pool2d use ceil mode-------------------- def create_test_cudnn_use_ceil_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestPool2DUseCeilCase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -567,9 +615,7 @@ def init_ceil_mode(self): def create_test_use_ceil_class(parent): - class TestPool2DUseCeilCase(parent): - def init_ceil_mode(self): self.ceil_mode = True @@ -583,13 +629,11 @@ def init_ceil_mode(self): class TestAvgInclude(TestCase2): - def init_exclusive(self): self.exclusive = False class TestCUDNNAvgInclude(TestCase2): - def init_kernel_type(self): self.use_cudnn = True @@ -598,13 +642,11 @@ def init_exclusive(self): class TestAvgPoolAdaptive(TestCase1): - def init_adaptive(self): self.adaptive = True class TestAvgPoolAdaptiveAsyOutSize(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -617,11 +659,10 @@ def init_test_case(self): self.paddings = [0, 0, 0, 0] -#-------test pool2d with asymmetric padding----- +# -------test pool2d with asymmetric padding----- class TestPool2D_AsyPadding(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -632,7 +673,6 @@ def init_shape(self): class TestCase1_AsyPadding(TestCase1): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -643,7 +683,6 @@ def init_shape(self): class TestCase2_AsyPadding(TestCase2): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -654,7 +693,6 @@ def init_shape(self): class TestCase3_AsyPadding(TestCase3): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -665,7 +703,6 @@ def init_shape(self): class TestCase4_AsyPadding(TestCase4): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -676,7 +713,6 @@ def init_shape(self): class TestCase5_AsyPadding((TestCase5)): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -708,7 +744,6 @@ def init_shape(self): class TestAvgInclude_AsyPadding(TestCase2): - def init_exclusive(self): self.exclusive = False @@ -722,7 +757,6 @@ def init_shape(self): class TestCUDNNAvgInclude_AsyPadding(TestCase2): - def init_kernel_type(self): self.use_cudnn = True @@ -739,7 +773,6 @@ def init_shape(self): class TestAvgPoolAdaptive_AsyPadding(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -752,9 +785,8 @@ def init_shape(self): self.shape = [2, 3, 7, 7] -#----------- test channel_last -------------- +# ----------- test channel_last -------------- class TestPool2D_channel_last(TestPool2D_Op): - def init_data_format(self): self.data_format = "NHWC" @@ -763,7 +795,6 @@ def init_shape(self): class TestCase1_channel_last(TestCase1): - def init_data_format(self): self.data_format = "NHWC" @@ -772,7 +803,6 @@ def init_shape(self): class TestCase2_channel_last(TestCase2): - def init_data_format(self): self.data_format = "NHWC" @@ -781,7 +811,6 @@ def init_shape(self): class TestCase3_channel_last(TestCase3): - def init_data_format(self): self.data_format = "NHWC" @@ -790,7 +819,6 @@ def init_shape(self): class TestCase4_channel_last(TestCase4): - def init_data_format(self): self.data_format = "NHWC" @@ -799,7 +827,6 @@ def init_shape(self): class TestCase5_channel_last(TestCase5): - def init_data_format(self): self.data_format = "NHWC" @@ -829,7 +856,6 @@ def init_shape(self): class TestCase5_Max(TestCase2): - def init_pool_type(self): self.pool_type = "max" @@ -838,16 +864,14 @@ def test_check_grad(self): return if self.has_cudnn() and self.pool_type == "max": place = core.CUDAPlace(0) - self.check_grad_with_place(place, - set(['X']), - 'Out', - max_relative_error=1.00) + self.check_grad_with_place( + place, set(['X']), 'Out', max_relative_error=1.00 + ) elif self.pool_type == "max": self.check_grad(set(['X']), 'Out', max_relative_error=1.00) class TestCase5_channel_last_Max(TestCase5_Max): - def init_data_format(self): self.data_format = "NHWC" @@ -860,13 +884,11 @@ def init_shape(self): class TestAvgInclude_channel_last(TestCase2_channel_last): - def init_exclusive(self): self.exclusive = False class TestCUDNNAvgInclude_channel_last(TestCase2_channel_last): - def init_kernel_type(self): self.use_cudnn = True @@ -875,13 +897,11 @@ def init_exclusive(self): class TestAvgPoolAdaptive_channel_last(TestCase1_channel_last): - def init_adaptive(self): self.adaptive = True class TestPool2D_AsyPadding_channel_last(TestPool2D_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -890,7 +910,6 @@ def init_shape(self): class TestCase1_AsyPadding_channel_last(TestCase1_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -899,7 +918,6 @@ def init_shape(self): class TestCase2_AsyPadding_channel_last(TestCase2_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -908,7 +926,6 @@ def init_shape(self): class TestCase3_AsyPadding_channel_last(TestCase3_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -917,7 +934,6 @@ def init_shape(self): class TestCase4_AsyPadding_channel_last(TestCase4_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -926,7 +942,6 @@ def init_shape(self): class TestCase5_AsyPadding_channel_last(TestCase5_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -942,8 +957,9 @@ def init_shape(self): create_test_cudnn_class(TestCase5_AsyPadding_channel_last) create_test_cudnn_fp16_class(TestPool2D_AsyPadding_channel_last) -create_test_cudnn_fp16_class(TestCase1_AsyPadding_channel_last, - check_grad=False) +create_test_cudnn_fp16_class( + TestCase1_AsyPadding_channel_last, check_grad=False +) create_test_cudnn_fp16_class(TestCase2_AsyPadding_channel_last) create_test_cudnn_fp16_class(TestCase3_AsyPadding_channel_last) create_test_cudnn_fp16_class(TestCase4_AsyPadding_channel_last) @@ -957,7 +973,6 @@ def init_shape(self): class TestAvgInclude_AsyPadding_channel_last(TestAvgInclude_AsyPadding): - def init_data_format(self): self.data_format = "NHWC" @@ -965,9 +980,9 @@ def init_shape(self): self.shape = [2, 7, 7, 3] -class TestCUDNNAvgInclude_AsyPadding_channel_last(TestCUDNNAvgInclude_AsyPadding - ): - +class TestCUDNNAvgInclude_AsyPadding_channel_last( + TestCUDNNAvgInclude_AsyPadding +): def init_data_format(self): self.data_format = "NHWC" @@ -975,9 +990,9 @@ def init_shape(self): self.shape = [2, 7, 7, 3] -class TestAvgPoolAdaptive_AsyPadding_channel_last(TestAvgPoolAdaptive_AsyPadding - ): - +class TestAvgPoolAdaptive_AsyPadding_channel_last( + TestAvgPoolAdaptive_AsyPadding +): def init_data_format(self): self.data_format = "NHWC" @@ -989,9 +1004,7 @@ def init_shape(self): def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.paddings = [0, 0] self.padding_algorithm = "SAME" @@ -1017,11 +1030,10 @@ def init_paddings(self): def create_test_cudnn_padding_SAME_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingSMAECase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -1050,9 +1062,7 @@ def init_paddings(self): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.paddings = [1, 1] self.padding_algorithm = "VALID" @@ -1078,11 +1088,10 @@ def init_paddings(self): def create_test_cudnn_padding_VALID_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingVALIDCase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -1111,7 +1120,6 @@ def init_paddings(self): class TestCase1_strides(TestCase1): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 2] @@ -1127,106 +1135,131 @@ def init_shape(self): # ----- test API class TestPool2DAPI(unittest.TestCase): - def test_api(self): x_NHWC = np.random.random([2, 5, 5, 3]).astype("float32") x_NCHW = np.random.random([2, 3, 5, 5]).astype("float32") - input_NHWC = fluid.layers.data(name="input_NHWC", - shape=[2, 5, 5, 3], - append_batch_size=False, - dtype="float32") - - input_NCHW = fluid.layers.data(name="input_NCHW", - shape=[2, 3, 5, 5], - append_batch_size=False, - dtype="float32") - - input_NHWC_negetive = fluid.layers.data(name="input_NHWC_negetive", - shape=[2, -1, 5, 3], - append_batch_size=False, - dtype="float32") - - input_NCHW_negetive = fluid.layers.data(name="input_NCHW_negetive", - shape=[2, 3, -1, -1], - append_batch_size=False, - dtype="float32") + input_NHWC = fluid.layers.data( + name="input_NHWC", + shape=[2, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + input_NCHW = fluid.layers.data( + name="input_NCHW", + shape=[2, 3, 5, 5], + append_batch_size=False, + dtype="float32", + ) + + input_NHWC_negetive = fluid.layers.data( + name="input_NHWC_negetive", + shape=[2, -1, 5, 3], + append_batch_size=False, + dtype="float32", + ) + + input_NCHW_negetive = fluid.layers.data( + name="input_NCHW_negetive", + shape=[2, 3, -1, -1], + append_batch_size=False, + dtype="float32", + ) ksize = [3, 3] - out_1 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1], - use_cudnn=False, - data_format="NHWC") - - out_2 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="avg", - pool_padding=[[0, 0], [1, 1], [1, 1], - [0, 0]], - use_cudnn=False, - data_format="NHWC") - - out_3 = fluid.layers.pool2d(input=input_NCHW, - pool_size=ksize, - pool_type="avg", - pool_padding=[[0, 0], [0, 0], [1, 1], - [1, 1]], - use_cudnn=False, - data_format="NCHW") - - out_4 = fluid.layers.pool2d(input=input_NCHW, - pool_size=ksize, - pool_type="avg", - pool_padding=[1, 2, 1, 0], - use_cudnn=False, - data_format="NCHW") + out_1 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1], + use_cudnn=False, + data_format="NHWC", + ) + + out_2 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="avg", + pool_padding=[[0, 0], [1, 1], [1, 1], [0, 0]], + use_cudnn=False, + data_format="NHWC", + ) + + out_3 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=ksize, + pool_type="avg", + pool_padding=[[0, 0], [0, 0], [1, 1], [1, 1]], + use_cudnn=False, + data_format="NCHW", + ) + + out_4 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=ksize, + pool_type="avg", + pool_padding=[1, 2, 1, 0], + use_cudnn=False, + data_format="NCHW", + ) # test VALID - out_5 = fluid.layers.pool2d(input=input_NCHW, - pool_size=ksize, - pool_type="avg", - pool_padding="VALID", - use_cudnn=False, - data_format="NCHW") - - out_6 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALID", - use_cudnn=False, - data_format="NHWC") + out_5 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=ksize, + pool_type="avg", + pool_padding="VALID", + use_cudnn=False, + data_format="NCHW", + ) + + out_6 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALID", + use_cudnn=False, + data_format="NHWC", + ) # test SAME - out_7 = fluid.layers.pool2d(input=input_NCHW, - pool_size=[4, 4], - pool_type="avg", - pool_padding="SAME", - use_cudnn=False, - data_format="NCHW") - - out_8 = fluid.layers.pool2d(input=input_NHWC, - pool_size=[4, 4], - pool_type="max", - pool_padding="SAME", - use_cudnn=False, - data_format="NHWC") + out_7 = fluid.layers.pool2d( + input=input_NCHW, + pool_size=[4, 4], + pool_type="avg", + pool_padding="SAME", + use_cudnn=False, + data_format="NCHW", + ) + + out_8 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=[4, 4], + pool_type="max", + pool_padding="SAME", + use_cudnn=False, + data_format="NHWC", + ) # test negetive - out_9 = fluid.layers.pool2d(input=input_NHWC_negetive, - pool_size=ksize, - pool_type="avg", - pool_padding=[0, 0], - use_cudnn=False, - data_format="NHWC") + out_9 = fluid.layers.pool2d( + input=input_NHWC_negetive, + pool_size=ksize, + pool_type="avg", + pool_padding=[0, 0], + use_cudnn=False, + data_format="NHWC", + ) assert out_9.shape == (2, -1, 3, 3) - out_10 = fluid.layers.pool2d(input=input_NCHW_negetive, - pool_size=ksize, - pool_type="avg", - pool_padding=[0, 0], - use_cudnn=False, - data_format="NCHW") + out_10 = fluid.layers.pool2d( + input=input_NCHW_negetive, + pool_size=ksize, + pool_type="avg", + pool_padding=[0, 0], + use_cudnn=False, + data_format="NCHW", + ) assert out_10.shape == (2, 3, -1, -1) exe = fluid.Executor(place=fluid.CPUPlace()) @@ -1236,46 +1269,61 @@ def test_api(self): "input_NHWC": x_NHWC, "input_NCHW": x_NCHW, "input_NHWC_negetive": x_NHWC, - "input_NCHW_negetive": x_NCHW + "input_NCHW_negetive": x_NCHW, }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8], + ) assert np.allclose( res_1, - pool2D_forward_naive(x=x_NHWC, - ksize=ksize, - pool_type="max", - strides=[1, 1], - paddings=[1, 1], - data_format="NHWC")) + pool2D_forward_naive( + x=x_NHWC, + ksize=ksize, + pool_type="max", + strides=[1, 1], + paddings=[1, 1], + data_format="NHWC", + ), + ) assert np.allclose( res_2, - pool2D_forward_naive(x=x_NHWC, - ksize=ksize, - pool_type="avg", - strides=[1, 1], - paddings=[1, 1, 1, 1], - data_format="NHWC")) - assert np.allclose(res_3, - pool2D_forward_naive(x=x_NCHW, - ksize=ksize, - pool_type="avg", - strides=[1, 1], - paddings=[1, 1, 1, 1], - data_format="NCHW"), - rtol=0.07, - atol=1e-05) - - assert np.allclose(res_4, - pool2D_forward_naive(x=x_NCHW, - ksize=ksize, - pool_type="avg", - strides=[1, 1], - paddings=[1, 2, 1, 0], - data_format="NCHW"), - rtol=0.07, - atol=1e-05) + pool2D_forward_naive( + x=x_NHWC, + ksize=ksize, + pool_type="avg", + strides=[1, 1], + paddings=[1, 1, 1, 1], + data_format="NHWC", + ), + ) + assert np.allclose( + res_3, + pool2D_forward_naive( + x=x_NCHW, + ksize=ksize, + pool_type="avg", + strides=[1, 1], + paddings=[1, 1, 1, 1], + data_format="NCHW", + ), + rtol=0.07, + atol=1e-05, + ) + + assert np.allclose( + res_4, + pool2D_forward_naive( + x=x_NCHW, + ksize=ksize, + pool_type="avg", + strides=[1, 1], + paddings=[1, 2, 1, 0], + data_format="NCHW", + ), + rtol=0.07, + atol=1e-05, + ) # VALID assert np.allclose( @@ -1287,192 +1335,234 @@ def test_api(self): strides=[1, 1], paddings=[10, 20], # any ele is ok padding_algorithm="VALID", - data_format="NCHW"), + data_format="NCHW", + ), rtol=0.07, - atol=1e-05) + atol=1e-05, + ) assert np.allclose( res_6, - pool2D_forward_naive(x=x_NHWC, - ksize=ksize, - pool_type="max", - strides=[1, 1], - paddings=[10, 20], - padding_algorithm="VALID", - data_format="NHWC")) + pool2D_forward_naive( + x=x_NHWC, + ksize=ksize, + pool_type="max", + strides=[1, 1], + paddings=[10, 20], + padding_algorithm="VALID", + data_format="NHWC", + ), + ) # SAME - assert np.allclose(res_7, - pool2D_forward_naive(x=x_NCHW, - ksize=[4, 4], - pool_type="avg", - strides=[1, 1], - paddings=[10, 20], - padding_algorithm="SAME", - data_format="NCHW"), - rtol=0.07, - atol=1e-05) + assert np.allclose( + res_7, + pool2D_forward_naive( + x=x_NCHW, + ksize=[4, 4], + pool_type="avg", + strides=[1, 1], + paddings=[10, 20], + padding_algorithm="SAME", + data_format="NCHW", + ), + rtol=0.07, + atol=1e-05, + ) assert np.allclose( res_8, - pool2D_forward_naive(x=x_NHWC, - ksize=[4, 4], - pool_type="max", - strides=[1, 1], - paddings=[10, 20], - padding_algorithm="SAME", - data_format="NHWC")) + pool2D_forward_naive( + x=x_NHWC, + ksize=[4, 4], + pool_type="max", + strides=[1, 1], + paddings=[10, 20], + padding_algorithm="SAME", + data_format="NHWC", + ), + ) class TestPool2DAPI_Error(unittest.TestCase): - def test_api(self): - input_NHWC = fluid.layers.data(name="input_NHWC", - shape=[2, 5, 5, 3], - append_batch_size=False, - dtype="float32") + input_NHWC = fluid.layers.data( + name="input_NHWC", + shape=[2, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) ksize = [3, 3] # cudnn type error def run_1(): - out_1 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1], - use_cudnn=[0], - data_format="NHWC") + out_1 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1], + use_cudnn=[0], + data_format="NHWC", + ) self.assertRaises(TypeError, run_1) # data_format value error def run_2(): - out_2 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1], - use_cudnn=False, - data_format="NHWCC") + out_2 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1], + use_cudnn=False, + data_format="NHWCC", + ) self.assertRaises(ValueError, run_2) # padding str value error def run_3(): - out_3 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALIDSAME", - use_cudnn=False, - data_format="NHWC") + out_3 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALIDSAME", + use_cudnn=False, + data_format="NHWC", + ) self.assertRaises(ValueError, run_3) # padding str valid and ceil_mode value error def run_4(): - out_4 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALID", - use_cudnn=False, - ceil_mode=True, - data_format="NHWC") + out_4 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALID", + use_cudnn=False, + ceil_mode=True, + data_format="NHWC", + ) self.assertRaises(ValueError, run_4) # padding with 8 ele. value error def run_5(): - out_5 = fluid.layers.pool2d(input=input_NHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[[1, 1], [0, 0], [0, 0], - [1, 1]], - use_cudnn=False, - data_format="NHWC") + out_5 = fluid.layers.pool2d( + input=input_NHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[[1, 1], [0, 0], [0, 0], [1, 1]], + use_cudnn=False, + data_format="NHWC", + ) self.assertRaises(ValueError, run_5) class TestDygraphPool2DAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of Pool2D must be Variable. data1 = np.random.random((3, 32, 32, 5)).astype('float32') - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='max', - pool_stride=1, - global_pooling=False) + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='max', + pool_stride=1, + global_pooling=False, + ) self.assertRaises(TypeError, pool2d, data1) # the input dtype of Pool2D must be uint8 or int8 or float16 or float32 or float64 # uint8 and int8 only can be set on mkldnn # float16 only can be set on GPU place - data2 = fluid.layers.data(name='x1', - shape=[3, 32, 32, 5], - dtype="int32") + data2 = fluid.layers.data( + name='x1', shape=[3, 32, 32, 5], dtype="int32" + ) self.assertRaises(TypeError, pool2d, data2) def test_data_format_error(self): with program_guard(Program(), Program()): # the data_format must be 'NCHW' or 'NHWC' data1 = np.random.random((3, 32, 32, 5)).astype('float32') - self.assertRaises(ValueError, - fluid.dygraph.Pool2D, - pool_size=2, - pool_type='max', - pool_stride=1, - global_pooling=False, - data_format='NWHC') + self.assertRaises( + ValueError, + fluid.dygraph.Pool2D, + pool_size=2, + pool_type='max', + pool_stride=1, + global_pooling=False, + data_format='NWHC', + ) class TestDygraphPool2DAPI(unittest.TestCase): - def test_nhwc(self): with fluid.dygraph.guard(): data = np.random.random((3, 32, 32, 5)).astype('float32') x = fluid.dygraph.to_variable(data) - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='max', - pool_stride=1, - pool_padding=[0, 0], - global_pooling=False, - data_format='NHWC') + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='max', + pool_stride=1, + pool_padding=[0, 0], + global_pooling=False, + data_format='NHWC', + ) out1 = pool2d(x) - out2 = pool2D_forward_naive(data, [2, 2], [1, 1], - paddings=[0, 0], - pool_type='max', - data_format='NHWC') + out2 = pool2D_forward_naive( + data, + [2, 2], + [1, 1], + paddings=[0, 0], + pool_type='max', + data_format='NHWC', + ) np.testing.assert_allclose(out1.numpy(), out2, rtol=1e-05) def test_lower_case(self): with fluid.dygraph.guard(): data = np.random.random((3, 32, 32, 5)).astype('float32') x = fluid.dygraph.to_variable(data) - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='max', - pool_stride=1, - pool_padding=[0, 0], - global_pooling=False, - data_format='nhwc') + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='max', + pool_stride=1, + pool_padding=[0, 0], + global_pooling=False, + data_format='nhwc', + ) out1 = pool2d(x) - out2 = pool2D_forward_naive(data, [2, 2], [1, 1], - paddings=[0, 0], - pool_type='max', - data_format='NHWC') + out2 = pool2D_forward_naive( + data, + [2, 2], + [1, 1], + paddings=[0, 0], + pool_type='max', + data_format='NHWC', + ) np.testing.assert_allclose(out1.numpy(), out2, rtol=1e-05) def test_upper_case(self): with fluid.dygraph.guard(): data = np.random.random((3, 32, 32, 5)).astype('float32') x = fluid.dygraph.to_variable(data) - pool2d = fluid.dygraph.Pool2D(pool_size=2, - pool_type='MAX', - pool_stride=1, - pool_padding=[0, 0], - global_pooling=False, - data_format='nhwc') + pool2d = fluid.dygraph.Pool2D( + pool_size=2, + pool_type='MAX', + pool_stride=1, + pool_padding=[0, 0], + global_pooling=False, + data_format='nhwc', + ) out1 = pool2d(x) - out2 = pool2D_forward_naive(data, [2, 2], [1, 1], - paddings=[0, 0], - pool_type='max', - data_format='NHWC') + out2 = pool2D_forward_naive( + data, + [2, 2], + [1, 1], + paddings=[0, 0], + pool_type='max', + data_format='NHWC', + ) np.testing.assert_allclose(out1.numpy(), out2, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_pool3d_api.py b/python/paddle/fluid/tests/unittests/test_pool3d_api.py index 31426fe541d72..29882a6c8bce8 100644 --- a/python/paddle/fluid/tests/unittests/test_pool3d_api.py +++ b/python/paddle/fluid/tests/unittests/test_pool3d_api.py @@ -20,11 +20,14 @@ from paddle.fluid.framework import _test_eager_guard from paddle.nn.functional import avg_pool3d, max_pool3d from paddle.fluid.framework import _test_eager_guard -from test_pool3d_op import avg_pool3D_forward_naive, max_pool3D_forward_naive, pool3D_forward_naive +from test_pool3d_op import ( + avg_pool3D_forward_naive, + max_pool3D_forward_naive, + pool3D_forward_naive, +) class TestPool3D_API(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -33,22 +36,26 @@ def setUp(self): def check_avg_static_results(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.data(name="input", - shape=[2, 3, 32, 32, 32], - dtype="float32") + input = fluid.data( + name="input", shape=[2, 3, 32, 32, 32], dtype="float32" + ) result = avg_pool3d(input, kernel_size=2, stride=2, padding=0) input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='avg') + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='avg', + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def check_avg_dygraph_results(self, place): @@ -57,18 +64,20 @@ def check_avg_dygraph_results(self, place): input = fluid.dygraph.to_variable(input_np) result = avg_pool3d(input, kernel_size=2, stride=2, padding="SAME") - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='avg', - padding_algorithm="SAME") + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='avg', + padding_algorithm="SAME", + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool3d_dg = paddle.nn.layer.AvgPool3D(kernel_size=2, - stride=None, - padding="SAME") + avg_pool3d_dg = paddle.nn.layer.AvgPool3D( + kernel_size=2, stride=None, padding="SAME" + ) result = avg_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -76,27 +85,33 @@ def check_avg_dygraph_padding_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = avg_pool3d(input, - kernel_size=2, - stride=2, - padding=1, - ceil_mode=False, - exclusive=True) - - result_np = avg_pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[1, 1, 1], - ceil_mode=False, - exclusive=False) + result = avg_pool3d( + input, + kernel_size=2, + stride=2, + padding=1, + ceil_mode=False, + exclusive=True, + ) + + result_np = avg_pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[1, 1, 1], + ceil_mode=False, + exclusive=False, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool3d_dg = paddle.nn.layer.AvgPool3D(kernel_size=2, - stride=None, - padding=1, - ceil_mode=False, - exclusive=True) + avg_pool3d_dg = paddle.nn.layer.AvgPool3D( + kernel_size=2, + stride=None, + padding=1, + ceil_mode=False, + exclusive=True, + ) result = avg_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -104,45 +119,48 @@ def check_avg_dygraph_ceilmode_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = avg_pool3d(input, - kernel_size=2, - stride=2, - padding=0, - ceil_mode=True) - - result_np = avg_pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - ceil_mode=True) + result = avg_pool3d( + input, kernel_size=2, stride=2, padding=0, ceil_mode=True + ) + + result_np = avg_pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + ceil_mode=True, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool3d_dg = paddle.nn.layer.AvgPool3D(kernel_size=2, - stride=None, - padding=0, - ceil_mode=True) + avg_pool3d_dg = paddle.nn.layer.AvgPool3D( + kernel_size=2, stride=None, padding=0, ceil_mode=True + ) result = avg_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) def check_max_static_results(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.data(name="input", - shape=[2, 3, 32, 32, 32], - dtype="float32") + input = fluid.data( + name="input", shape=[2, 3, 32, 32, 32], dtype="float32" + ) result = max_pool3d(input, kernel_size=2, stride=2, padding=0) input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='max') + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='max', + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], result_np, rtol=1e-05) def check_max_dygraph_results(self, place): @@ -151,16 +169,18 @@ def check_max_dygraph_results(self, place): input = fluid.dygraph.to_variable(input_np) result = max_pool3d(input, kernel_size=2, stride=2, padding=0) - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='max') + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='max', + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool3d_dg = paddle.nn.layer.MaxPool3D(kernel_size=2, - stride=None, - padding=0) + max_pool3d_dg = paddle.nn.layer.MaxPool3D( + kernel_size=2, stride=None, padding=0 + ) result = max_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -168,47 +188,52 @@ def check_max_dygraph_ndhwc_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable( - np.transpose(input_np, [0, 2, 3, 4, 1])) - result = max_pool3d(input, - kernel_size=2, - stride=2, - padding=0, - data_format="NDHWC", - return_mask=False) - - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='max') - - np.testing.assert_allclose(np.transpose(result.numpy(), - [0, 4, 1, 2, 3]), - result_np, - rtol=1e-05) + np.transpose(input_np, [0, 2, 3, 4, 1]) + ) + result = max_pool3d( + input, + kernel_size=2, + stride=2, + padding=0, + data_format="NDHWC", + return_mask=False, + ) + + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='max', + ) + + np.testing.assert_allclose( + np.transpose(result.numpy(), [0, 4, 1, 2, 3]), + result_np, + rtol=1e-05, + ) def check_max_dygraph_ceilmode_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = max_pool3d(input, - kernel_size=2, - stride=2, - padding=0, - ceil_mode=True) - - result_np = max_pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - ceil_mode=True) + result = max_pool3d( + input, kernel_size=2, stride=2, padding=0, ceil_mode=True + ) + + result_np = max_pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + ceil_mode=True, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool3d_dg = paddle.nn.layer.MaxPool3D(kernel_size=2, - stride=None, - padding=0, - ceil_mode=True) + max_pool3d_dg = paddle.nn.layer.MaxPool3D( + kernel_size=2, stride=None, padding=0, ceil_mode=True + ) result = max_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -216,24 +241,23 @@ def check_max_dygraph_padding_results(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result = max_pool3d(input, - kernel_size=2, - stride=2, - padding=1, - ceil_mode=False) - - result_np = max_pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[1, 1, 1], - ceil_mode=False) + result = max_pool3d( + input, kernel_size=2, stride=2, padding=1, ceil_mode=False + ) + + result_np = max_pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[1, 1, 1], + ceil_mode=False, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool3d_dg = paddle.nn.layer.MaxPool3D(kernel_size=2, - stride=None, - padding=1, - ceil_mode=False) + max_pool3d_dg = paddle.nn.layer.MaxPool3D( + kernel_size=2, stride=None, padding=1, ceil_mode=False + ) result = max_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -241,23 +265,27 @@ def check_max_dygraph_stride_is_none(self, place): with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) - result, indices = max_pool3d(input, - kernel_size=2, - stride=None, - padding="SAME", - return_mask=True) - - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='max', - padding_algorithm="SAME") + result, indices = max_pool3d( + input, + kernel_size=2, + stride=None, + padding="SAME", + return_mask=True, + ) + + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='max', + padding_algorithm="SAME", + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool3d_dg = paddle.nn.layer.MaxPool3D(kernel_size=2, - stride=2, - padding=0) + max_pool3d_dg = paddle.nn.layer.MaxPool3D( + kernel_size=2, stride=2, padding=0 + ) result = max_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -268,16 +296,18 @@ def check_max_dygraph_padding(self, place): padding = [[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]] result = max_pool3d(input, kernel_size=2, stride=2, padding=padding) - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='max') + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='max', + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - max_pool3d_dg = paddle.nn.layer.MaxPool3D(kernel_size=2, - stride=2, - padding=0) + max_pool3d_dg = paddle.nn.layer.MaxPool3D( + kernel_size=2, stride=2, padding=0 + ) result = max_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) @@ -290,31 +320,37 @@ def check_avg_divisor(self, place): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) padding = 0 - result = avg_pool3d(input, - kernel_size=2, - stride=2, - padding=padding, - divisor_override=8) - - result_np = pool3D_forward_naive(input_np, - ksize=[2, 2, 2], - strides=[2, 2, 2], - paddings=[0, 0, 0], - pool_type='avg') + result = avg_pool3d( + input, + kernel_size=2, + stride=2, + padding=padding, + divisor_override=8, + ) + + result_np = pool3D_forward_naive( + input_np, + ksize=[2, 2, 2], + strides=[2, 2, 2], + paddings=[0, 0, 0], + pool_type='avg', + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) - avg_pool3d_dg = paddle.nn.layer.AvgPool3D(kernel_size=2, - stride=2, - padding=0) + avg_pool3d_dg = paddle.nn.layer.AvgPool3D( + kernel_size=2, stride=2, padding=0 + ) result = avg_pool3d_dg(input) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) padding = [0, 0, 0, 0, 0, 0] - result = avg_pool3d(input, - kernel_size=2, - stride=2, - padding=padding, - divisor_override=8) + result = avg_pool3d( + input, + kernel_size=2, + stride=2, + padding=padding, + divisor_override=8, + ) np.testing.assert_allclose(result.numpy(), result_np, rtol=1e-05) def test_pool3d(self): @@ -336,167 +372,198 @@ def test_dygraph_api(self): class TestPool3DError_API(unittest.TestCase): - def test_error_api(self): - def run1(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = [[0, 1], [0, 0], [0, 0], [0, 0], [0, 0]] - res_pd = avg_pool3d(input_pd, - kernel_size=2, - stride=2, - padding=padding) + res_pd = avg_pool3d( + input_pd, kernel_size=2, stride=2, padding=padding + ) self.assertRaises(ValueError, run1) def run2(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = [[0, 1], [0, 0], [0, 0], [0, 0], [0, 0]] - res_pd = avg_pool3d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - data_format='NCDHW') + res_pd = avg_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + data_format='NCDHW', + ) self.assertRaises(ValueError, run2) def run3(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) padding = [[0, 1], [0, 0], [0, 0], [0, 0], [0, 0]] - res_pd = avg_pool3d(input_pd, - kernel_size=2, - stride=2, - padding=padding, - data_format='NDHWC') + res_pd = avg_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding=padding, + data_format='NDHWC', + ) self.assertRaises(ValueError, run3) def run4(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = avg_pool3d(input_pd, - kernel_size=2, - stride=2, - padding=0, - data_format='NNNN') + res_pd = avg_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding=0, + data_format='NNNN', + ) self.assertRaises(ValueError, run4) def run5(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = max_pool3d(input_pd, - kernel_size=2, - stride=2, - padding=0, - data_format='NNNN') + res_pd = max_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding=0, + data_format='NNNN', + ) self.assertRaises(ValueError, run5) def run6(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = avg_pool3d(input_pd, - kernel_size=2, - stride=2, - padding="padding", - data_format='NNNN') + res_pd = avg_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding="padding", + data_format='NNNN', + ) self.assertRaises(ValueError, run6) def run7(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = max_pool3d(input_pd, - kernel_size=2, - stride=2, - padding="padding", - data_format='NNNN') + res_pd = max_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding="padding", + data_format='NNNN', + ) self.assertRaises(ValueError, run7) def run8(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = avg_pool3d(input_pd, - kernel_size=2, - stride=2, - padding="VALID", - ceil_mode=True, - data_format='NNNN') + res_pd = avg_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding="VALID", + ceil_mode=True, + data_format='NNNN', + ) self.assertRaises(ValueError, run8) def run9(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = max_pool3d(input_pd, - kernel_size=2, - stride=2, - padding="VALID", - ceil_mode=True, - data_format='NNNN') + res_pd = max_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding="VALID", + ceil_mode=True, + data_format='NNNN', + ) self.assertRaises(ValueError, run9) def run10(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = max_pool3d(input_pd, - kernel_size=2, - stride=2, - padding=0, - data_format='NDHWC', - return_mask=True) + res_pd = max_pool3d( + input_pd, + kernel_size=2, + stride=2, + padding=0, + data_format='NDHWC', + return_mask=True, + ) self.assertRaises(ValueError, run10) def run_kernel_out_of_range(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = avg_pool3d(input_pd, - kernel_size=-1, - stride=2, - padding="VALID", - ceil_mode=True) + res_pd = avg_pool3d( + input_pd, + kernel_size=-1, + stride=2, + padding="VALID", + ceil_mode=True, + ) self.assertRaises(ValueError, run_kernel_out_of_range) def run_size_out_of_range(): with fluid.dygraph.guard(): input_np = np.random.uniform(-1, 1, [2, 3, 32, 32, 32]).astype( - np.float32) + np.float32 + ) input_pd = fluid.dygraph.to_variable(input_np) - res_pd = avg_pool3d(input_pd, - kernel_size=2, - stride=0, - padding="VALID", - ceil_mode=True) + res_pd = avg_pool3d( + input_pd, + kernel_size=2, + stride=0, + padding="VALID", + ceil_mode=True, + ) self.assertRaises(ValueError, run_size_out_of_range) diff --git a/python/paddle/fluid/tests/unittests/test_pool3d_op.py b/python/paddle/fluid/tests/unittests/test_pool3d_op.py index 37fde90a45788..a5bf506695648 100644 --- a/python/paddle/fluid/tests/unittests/test_pool3d_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool3d_op.py @@ -29,25 +29,29 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def pool3D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_format='NCDHW', - pool_type='max', - padding_algorithm="EXPLICIT"): +def pool3D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_format='NCDHW', + pool_type='max', + padding_algorithm="EXPLICIT", +): # update paddings def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -57,9 +61,10 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): if isinstance(padding_algorithm, str): padding_algorithm = padding_algorithm.upper() if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if padding_algorithm == "VALID": paddings = [0, 0, 0, 0, 0, 0] @@ -67,7 +72,8 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): raise ValueError( "When Attr(pool_padding) is \"VALID\", Attr(ceil_mode)" " must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) elif padding_algorithm == "SAME": input_data_shape = [] if data_format == "NCDHW": @@ -80,8 +86,11 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): is_sys = True if len(paddings) == 3 else False N = x.shape[0] - C,D, H, W = [x.shape[1], x.shape[2], x.shape[3], x.shape[4]] \ - if data_format == 'NCDHW' else [x.shape[4], x.shape[1], x.shape[2],x.shape[3]] + C, D, H, W = ( + [x.shape[1], x.shape[2], x.shape[3], x.shape[4]] + if data_format == 'NCDHW' + else [x.shape[4], x.shape[1], x.shape[2], x.shape[3]] + ) if global_pool == 1: ksize = [D, H, W] @@ -98,18 +107,35 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): D_out, H_out, W_out = ksize else: - D_out = (D - ksize[0] + pad_d_forth+pad_d_back + strides[0] - 1) // strides[0] + 1 \ - if ceil_mode else (D - ksize[0] + pad_d_forth+pad_d_back) // strides[0] + 1 - - H_out = (H - ksize[1] + pad_h_up + pad_h_down + strides[1] - 1) // strides[1] + 1 \ - if ceil_mode else (H - ksize[1] + pad_h_up + pad_h_down) // strides[1] + 1 - - W_out = (W - ksize[2] + pad_w_left + pad_w_right + strides[2] - 1) // strides[2] + 1 \ - if ceil_mode else (W - ksize[2] + pad_w_left + pad_w_right) // strides[2] + 1 - - - out = np.zeros((N, C, D_out, H_out, W_out)) if data_format=='NCDHW' \ + D_out = ( + (D - ksize[0] + pad_d_forth + pad_d_back + strides[0] - 1) + // strides[0] + + 1 + if ceil_mode + else (D - ksize[0] + pad_d_forth + pad_d_back) // strides[0] + 1 + ) + + H_out = ( + (H - ksize[1] + pad_h_up + pad_h_down + strides[1] - 1) + // strides[1] + + 1 + if ceil_mode + else (H - ksize[1] + pad_h_up + pad_h_down) // strides[1] + 1 + ) + + W_out = ( + (W - ksize[2] + pad_w_left + pad_w_right + strides[2] - 1) + // strides[2] + + 1 + if ceil_mode + else (W - ksize[2] + pad_w_left + pad_w_right) // strides[2] + 1 + ) + + out = ( + np.zeros((N, C, D_out, H_out, W_out)) + if data_format == 'NCDHW' else np.zeros((N, D_out, H_out, W_out, C)) + ) for k in range(D_out): if adaptive: d_start = adaptive_start_index(k, D, ksize[0]) @@ -127,17 +153,29 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): else: d_start = k * strides[0] - pad_d_forth - d_end = np.min((k * strides[0] + ksize[0] - pad_d_forth, - D + pad_d_back)) + d_end = np.min( + ( + k * strides[0] + ksize[0] - pad_d_forth, + D + pad_d_back, + ) + ) h_start = i * strides[1] - pad_h_up h_end = np.min( - (i * strides[1] + ksize[1] - pad_h_up, H + pad_h_down)) + (i * strides[1] + ksize[1] - pad_h_up, H + pad_h_down) + ) w_start = j * strides[2] - pad_w_left - w_end = np.min((j * strides[2] + ksize[2] - pad_w_left, - W + pad_w_right)) - - field_size = (d_end - d_start) * (h_end - h_start) * ( - w_end - w_start) + w_end = np.min( + ( + j * strides[2] + ksize[2] - pad_w_left, + W + pad_w_right, + ) + ) + + field_size = ( + (d_end - d_start) + * (h_end - h_start) + * (w_end - w_start) + ) w_start = np.max((w_start, 0)) d_start = np.max((d_start, 0)) h_start = np.max((h_start, 0)) @@ -145,78 +183,95 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): d_end = np.min((d_end, D)) h_end = np.min((h_end, H)) if data_format == 'NCDHW': - x_masked = x[:, :, d_start:d_end, h_start:h_end, - w_start:w_end] + x_masked = x[ + :, :, d_start:d_end, h_start:h_end, w_start:w_end + ] if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (d_end - d_start) * ( - h_end - h_start) * (w_end - w_start) - - out[:, :, k, i, - j] = np.sum(x_masked, axis=(2, 3, 4)) / field_size + if exclusive or adaptive: + field_size = ( + (d_end - d_start) + * (h_end - h_start) + * (w_end - w_start) + ) + + out[:, :, k, i, j] = ( + np.sum(x_masked, axis=(2, 3, 4)) / field_size + ) elif pool_type == 'max': out[:, :, k, i, j] = np.max(x_masked, axis=(2, 3, 4)) elif data_format == 'NDHWC': - x_masked = x[:, d_start:d_end, h_start:h_end, - w_start:w_end, :] + x_masked = x[ + :, d_start:d_end, h_start:h_end, w_start:w_end, : + ] if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (d_end - d_start) * ( - h_end - h_start) * (w_end - w_start) - - out[:, k, i, j, :] = np.sum(x_masked, - axis=(1, 2, 3)) / field_size + if exclusive or adaptive: + field_size = ( + (d_end - d_start) + * (h_end - h_start) + * (w_end - w_start) + ) + + out[:, k, i, j, :] = ( + np.sum(x_masked, axis=(1, 2, 3)) / field_size + ) elif pool_type == 'max': out[:, k, i, j, :] = np.max(x_masked, axis=(1, 2, 3)) return out -def max_pool3D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False): - out = pool3D_forward_naive(x=x, - ksize=ksize, - strides=strides, - paddings=paddings, - global_pool=global_pool, - ceil_mode=ceil_mode, - exclusive=exclusive, - adaptive=adaptive, - data_format='NCDHW', - pool_type="max") +def max_pool3D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, +): + out = pool3D_forward_naive( + x=x, + ksize=ksize, + strides=strides, + paddings=paddings, + global_pool=global_pool, + ceil_mode=ceil_mode, + exclusive=exclusive, + adaptive=adaptive, + data_format='NCDHW', + pool_type="max", + ) return out -def avg_pool3D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False): - out = pool3D_forward_naive(x=x, - ksize=ksize, - strides=strides, - paddings=paddings, - global_pool=global_pool, - ceil_mode=ceil_mode, - exclusive=exclusive, - adaptive=adaptive, - data_format='NCDHW', - pool_type="avg") +def avg_pool3D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, +): + out = pool3D_forward_naive( + x=x, + ksize=ksize, + strides=strides, + paddings=paddings, + global_pool=global_pool, + ceil_mode=ceil_mode, + exclusive=exclusive, + adaptive=adaptive, + data_format='NCDHW', + pool_type="avg", + ) return out class TestPool3D_Op(OpTest): - def setUp(self): self.op_type = "pool3d" self.init_kernel_type() @@ -235,12 +290,19 @@ def setUp(self): paddle.enable_static() input = np.random.random(self.shape).astype(self.dtype) - output = pool3D_forward_naive(input, self.ksize, self.strides, - self.paddings, self.global_pool, - self.ceil_mode, self.exclusive, - self.adaptive, self.data_format, - self.pool_type, - self.padding_algorithm).astype(self.dtype) + output = pool3D_forward_naive( + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + self.data_format, + self.pool_type, + self.padding_algorithm, + ).astype(self.dtype) self.inputs = {'X': OpTest.np_dtype_to_fluid_dtype(input)} @@ -276,10 +338,9 @@ def test_check_grad(self): if self.has_cudnn() and self.pool_type != "max": place = core.CUDAPlace(0) if core.is_compiled_with_rocm(): - self.check_grad_with_place(place, - set(['X']), - 'Out', - max_relative_error=1e-2) + self.check_grad_with_place( + place, set(['X']), 'Out', max_relative_error=1e-2 + ) else: self.check_grad_with_place(place, set(['X']), 'Out') elif self.pool_type != "max": @@ -322,7 +383,6 @@ def init_adaptive(self): class TestCase1(TestPool3D_Op): - def init_shape(self): self.shape = [1, 3, 7, 7, 7] @@ -341,7 +401,6 @@ def init_global_pool(self): class TestCase2(TestPool3D_Op): - def init_shape(self): self.shape = [1, 3, 6, 7, 7] @@ -360,32 +419,28 @@ def init_global_pool(self): class TestCase3(TestPool3D_Op): - def init_pool_type(self): self.pool_type = "max" class TestCase4(TestCase1): - def init_pool_type(self): self.pool_type = "max" class TestCase5(TestCase2): - def init_pool_type(self): self.pool_type = "max" -#--------------------test pool3d cudnn-------------------- +# --------------------test pool3d cudnn-------------------- def create_test_cudnn_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNCase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -403,11 +458,10 @@ def init_kernel_type(self): def create_test_cudnn_fp16_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNFp16Case(parent): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -427,11 +481,10 @@ def test_check_output(self): def create_test_fp16_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestFp16Case(parent): - def init_kernel_type(self): self.use_cudnn = False self.dtype = np.float16 @@ -464,11 +517,10 @@ def test_check_output(self): # ---- test ceil mode ------ def create_test_cudnn_use_ceil_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestPool3DUseCeilCase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -485,9 +537,7 @@ def init_ceil_mode(self): def create_test_use_ceil_class(parent): - class TestPool3DUseCeilCase(parent): - def init_ceil_mode(self): self.ceil_mode = True @@ -501,15 +551,14 @@ def init_ceil_mode(self): class TestAvgInclude(TestCase2): - def init_exclusive(self): self.exclusive = False -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNAvgInclude(TestCase2): - def init_kernel_type(self): self.use_cudnn = True @@ -518,13 +567,11 @@ def init_exclusive(self): class TestAvgPoolAdaptive(TestCase1): - def init_adaptive(self): self.adaptive = True class TestAvgPoolAdaptiveAsyOutSize(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -536,9 +583,8 @@ def init_test_case(self): self.strides = [1, 1, 1] -#-------test pool3d with asymmetric padding------ +# -------test pool3d with asymmetric padding------ class TestPool3D_Op_AsyPadding(TestPool3D_Op): - def init_test_case(self): self.ksize = [3, 4, 3] self.strides = [1, 1, 2] @@ -551,7 +597,6 @@ def init_shape(self): class TestCase1_AsyPadding(TestCase1): - def init_test_case(self): self.ksize = [3, 3, 4] self.strides = [1, 1, 2] @@ -564,7 +609,6 @@ def init_shape(self): class TestCase2_AsyPadding(TestCase2): - def init_test_case(self): self.ksize = [3, 3, 3] self.strides = [1, 1, 1] @@ -577,7 +621,6 @@ def init_shape(self): class TestCase3_AsyPadding(TestCase3): - def init_test_case(self): self.ksize = [3, 3, 3] self.strides = [1, 1, 1] @@ -590,7 +633,6 @@ def init_shape(self): class TestCase4_AsyPadding(TestCase4): - def init_test_case(self): self.ksize = [3, 3, 3] self.strides = [1, 1, 1] @@ -603,7 +645,6 @@ def init_shape(self): class TestCase5_AsyPadding(TestCase5): - def init_test_case(self): self.ksize = [3, 3, 3] self.strides = [1, 1, 1] @@ -637,7 +678,6 @@ def init_shape(self): class TestAvgInclude_AsyPadding(TestCase2): - def init_exclusive(self): self.exclusive = False @@ -645,10 +685,10 @@ def init_paddings(self): self.paddings = [2, 2, 1, 1, 0, 0] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNAvgInclude_AsyPadding(TestCase2): - def init_kernel_type(self): self.use_cudnn = True @@ -663,7 +703,6 @@ def init_shape(self): class TestAvgPoolAdaptive_AsyPadding(TestCase1): - def init_adaptive(self): self.adaptive = True @@ -673,7 +712,6 @@ def init_paddings(self): # ------------ test channel_last -------------- class TestPool3D_channel_last(TestPool3D_Op): - def init_data_format(self): self.data_format = "NDHWC" @@ -682,7 +720,6 @@ def init_shape(self): class TestCase1_channel_last(TestCase1): - def init_data_format(self): self.data_format = "NDHWC" @@ -691,7 +728,6 @@ def init_shape(self): class TestCase2_channel_last(TestCase2): - def init_data_format(self): self.data_format = "NDHWC" @@ -700,7 +736,6 @@ def init_shape(self): class TestCase3_channel_last(TestCase3): - def init_data_format(self): self.data_format = "NDHWC" @@ -709,7 +744,6 @@ def init_shape(self): class TestCase4_channel_last(TestCase4): - def init_data_format(self): self.data_format = "NDHWC" @@ -718,7 +752,6 @@ def init_shape(self): class TestCase5_channel_last(TestCase5): - def init_data_format(self): self.data_format = "NDHWC" @@ -741,7 +774,6 @@ def init_shape(self): class TestCase5_Max(TestCase2): - def init_pool_type(self): self.pool_type = "max" @@ -750,16 +782,14 @@ def test_check_grad(self): return if self.has_cudnn() and self.pool_type == "max": place = core.CUDAPlace(0) - self.check_grad_with_place(place, - set(['X']), - 'Out', - max_relative_error=1.00) + self.check_grad_with_place( + place, set(['X']), 'Out', max_relative_error=1.00 + ) elif self.pool_type == "max": self.check_grad(set(['X']), 'Out', max_relative_error=1.00) class TestCase5_channel_last_Max(TestCase5_Max): - def init_data_format(self): self.data_format = "NDHWC" @@ -772,15 +802,14 @@ def init_shape(self): class TestAvgInclude_channel_last(TestCase2_channel_last): - def init_exclusive(self): self.exclusive = False -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestCUDNNAvgInclude_channel_last(TestCase2_channel_last): - def init_kernel_type(self): self.use_cudnn = True @@ -789,14 +818,12 @@ def init_exclusive(self): class TestAvgPoolAdaptive_channel_last(TestCase1_channel_last): - def init_adaptive(self): self.adaptive = True # --- asy padding class TestPool3D_Op_AsyPadding_channel_last(TestPool3D_Op_AsyPadding): - def init_data_format(self): self.data_format = "NDHWC" @@ -805,7 +832,6 @@ def init_shape(self): class TestCase1_AsyPadding_channel_last(TestCase1_AsyPadding): - def init_data_format(self): self.data_format = "NDHWC" @@ -814,7 +840,6 @@ def init_shape(self): class TestCase2_AsyPadding_channel_last(TestCase2_AsyPadding): - def init_data_format(self): self.data_format = "NDHWC" @@ -823,7 +848,6 @@ def init_shape(self): class TestCase3_AsyPadding_channel_last(TestCase3_AsyPadding): - def init_data_format(self): self.data_format = "NDHWC" @@ -832,7 +856,6 @@ def init_shape(self): class TestCase4_AsyPadding_channel_last(TestCase4_AsyPadding): - def init_data_format(self): self.data_format = "NDHWC" @@ -841,7 +864,6 @@ def init_shape(self): class TestCase5_AsyPadding_channel_last(TestCase5_AsyPadding): - def init_data_format(self): self.data_format = "NDHWC" @@ -864,23 +886,23 @@ def init_shape(self): class TestAvgInclude_AsyPadding_channel_last(TestAvgInclude_AsyPadding): - def init_data_format(self): self.data_format = "NDHWC" -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") -class TestCUDNNAvgInclude_AsyPadding_channel_last(TestCUDNNAvgInclude_AsyPadding - ): - +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) +class TestCUDNNAvgInclude_AsyPadding_channel_last( + TestCUDNNAvgInclude_AsyPadding +): def init_data_format(self): self.data_format = "NDHWC" -class TestAvgPoolAdaptive_AsyPadding_channel_last(TestAvgPoolAdaptive_AsyPadding - ): - +class TestAvgPoolAdaptive_AsyPadding_channel_last( + TestAvgPoolAdaptive_AsyPadding +): def init_data_format(self): self.data_format = "NDHWC" @@ -888,11 +910,9 @@ def init_shape(self): self.shape = [1, 7, 7, 7, 3] -#test padding = SAME VALID +# test padding = SAME VALID def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.paddings = [0, 0, 0] self.padding_algorithm = "SAME" @@ -918,11 +938,10 @@ def init_paddings(self): def create_test_cudnn_padding_SAME_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingSMAECase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -951,9 +970,7 @@ def init_paddings(self): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.paddings = [1, 1, 1] self.padding_algorithm = "VALID" @@ -979,11 +996,10 @@ def init_paddings(self): def create_test_cudnn_padding_VALID_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestCUDNNPaddingVALIDCase(parent): - def init_kernel_type(self): self.use_cudnn = True @@ -1011,236 +1027,289 @@ def init_paddings(self): create_test_cudnn_padding_VALID_class(TestCase5_channel_last) -#test API +# test API class TestPool3DAPI(unittest.TestCase): - def test_api(self): x_NDHWC = np.random.random([2, 5, 5, 5, 3]).astype("float32") x_NCDHW = np.random.random([2, 3, 5, 5, 5]).astype("float32") - input_NDHWC = fluid.layers.data(name="input_NDHWC", - shape=[2, 5, 5, 5, 3], - append_batch_size=False, - dtype="float32") + input_NDHWC = fluid.layers.data( + name="input_NDHWC", + shape=[2, 5, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) - input_NCDHW = fluid.layers.data(name="input_NCDHW", - shape=[2, 3, 5, 5, 5], - append_batch_size=False, - dtype="float32") + input_NCDHW = fluid.layers.data( + name="input_NCDHW", + shape=[2, 3, 5, 5, 5], + append_batch_size=False, + dtype="float32", + ) ksize = [3, 3, 3] - out_1 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1, 1], - use_cudnn=False, - data_format="NDHWC") - - out_2 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="avg", - pool_padding=[[0, 0], [1, 1], [1, 1], - [1, 1], [0, 0]], - use_cudnn=False, - data_format="NDHWC") - - out_3 = fluid.layers.pool3d(input=input_NCDHW, - pool_size=ksize, - pool_type="avg", - pool_padding=[[0, 0], [0, 0], [1, 1], - [1, 1], [1, 1]], - use_cudnn=False, - data_format="NCDHW") - - out_4 = fluid.layers.pool3d(input=input_NCDHW, - pool_size=ksize, - pool_type="avg", - pool_padding=[1, 2, 1, 0, 0, 1], - use_cudnn=False, - data_format="NCDHW") + out_1 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1, 1], + use_cudnn=False, + data_format="NDHWC", + ) + + out_2 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="avg", + pool_padding=[[0, 0], [1, 1], [1, 1], [1, 1], [0, 0]], + use_cudnn=False, + data_format="NDHWC", + ) + + out_3 = fluid.layers.pool3d( + input=input_NCDHW, + pool_size=ksize, + pool_type="avg", + pool_padding=[[0, 0], [0, 0], [1, 1], [1, 1], [1, 1]], + use_cudnn=False, + data_format="NCDHW", + ) + + out_4 = fluid.layers.pool3d( + input=input_NCDHW, + pool_size=ksize, + pool_type="avg", + pool_padding=[1, 2, 1, 0, 0, 1], + use_cudnn=False, + data_format="NCDHW", + ) # test VALID - out_5 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="avg", - pool_padding="VALID", - use_cudnn=False, - data_format="NDHWC") - - out_6 = fluid.layers.pool3d(input=input_NCDHW, - pool_size=ksize, - pool_type="avg", - pool_padding="VALID", - use_cudnn=False, - data_format="NCDHW") + out_5 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="avg", + pool_padding="VALID", + use_cudnn=False, + data_format="NDHWC", + ) + + out_6 = fluid.layers.pool3d( + input=input_NCDHW, + pool_size=ksize, + pool_type="avg", + pool_padding="VALID", + use_cudnn=False, + data_format="NCDHW", + ) # test SAME - out_7 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_stride=[1, 1, 2], - pool_type="avg", - pool_padding="SAME", - use_cudnn=False, - data_format="NDHWC") - - out_8 = fluid.layers.pool3d(input=input_NCDHW, - pool_size=[4, 4, 4], - pool_type="avg", - pool_padding="SAME", - use_cudnn=False, - data_format="NCDHW") + out_7 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_stride=[1, 1, 2], + pool_type="avg", + pool_padding="SAME", + use_cudnn=False, + data_format="NDHWC", + ) + + out_8 = fluid.layers.pool3d( + input=input_NCDHW, + pool_size=[4, 4, 4], + pool_type="avg", + pool_padding="SAME", + use_cudnn=False, + data_format="NCDHW", + ) exe = fluid.Executor(place=fluid.CPUPlace()) [res_1, res_2, res_3, res_4, res_5, res_6, res_7, res_8] = exe.run( fluid.default_main_program(), - feed={ - "input_NDHWC": x_NDHWC, - "input_NCDHW": x_NCDHW - }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8]) + feed={"input_NDHWC": x_NDHWC, "input_NCDHW": x_NCDHW}, + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8], + ) assert np.allclose( res_1, - pool3D_forward_naive(x=x_NDHWC, - ksize=ksize, - pool_type="max", - strides=[1, 1, 1], - paddings=[1, 1, 1], - data_format="NDHWC")) + pool3D_forward_naive( + x=x_NDHWC, + ksize=ksize, + pool_type="max", + strides=[1, 1, 1], + paddings=[1, 1, 1], + data_format="NDHWC", + ), + ) assert np.allclose( res_2, - pool3D_forward_naive(x=x_NDHWC, - ksize=ksize, - pool_type="avg", - strides=[1, 1, 1], - paddings=[1, 1, 1, 1, 1, 1], - data_format="NDHWC")) - assert np.allclose(res_3, - pool3D_forward_naive(x=x_NCDHW, - ksize=ksize, - pool_type="avg", - strides=[1, 1, 1], - paddings=[1, 1, 1, 1, 1, 1], - data_format="NCDHW"), - rtol=0.07, - atol=1e-05) - - assert np.allclose(res_4, - pool3D_forward_naive(x=x_NCDHW, - ksize=ksize, - pool_type="avg", - strides=[1, 1, 1], - paddings=[1, 2, 1, 0, 0, 1], - data_format="NCDHW"), - rtol=0.07, - atol=1e-05) + pool3D_forward_naive( + x=x_NDHWC, + ksize=ksize, + pool_type="avg", + strides=[1, 1, 1], + paddings=[1, 1, 1, 1, 1, 1], + data_format="NDHWC", + ), + ) + assert np.allclose( + res_3, + pool3D_forward_naive( + x=x_NCDHW, + ksize=ksize, + pool_type="avg", + strides=[1, 1, 1], + paddings=[1, 1, 1, 1, 1, 1], + data_format="NCDHW", + ), + rtol=0.07, + atol=1e-05, + ) + + assert np.allclose( + res_4, + pool3D_forward_naive( + x=x_NCDHW, + ksize=ksize, + pool_type="avg", + strides=[1, 1, 1], + paddings=[1, 2, 1, 0, 0, 1], + data_format="NCDHW", + ), + rtol=0.07, + atol=1e-05, + ) # VALID assert np.allclose( res_5, - pool3D_forward_naive(x=x_NDHWC, - ksize=ksize, - pool_type="avg", - strides=[1, 1, 1], - paddings=[10, 20], - padding_algorithm="VALID", - data_format="NDHWC")) - - assert np.allclose(res_6, - pool3D_forward_naive(x=x_NCDHW, - ksize=ksize, - pool_type="avg", - strides=[1, 1, 1], - paddings=[10, 20], - padding_algorithm="VALID", - data_format="NCDHW"), - rtol=0.07, - atol=1e-05) + pool3D_forward_naive( + x=x_NDHWC, + ksize=ksize, + pool_type="avg", + strides=[1, 1, 1], + paddings=[10, 20], + padding_algorithm="VALID", + data_format="NDHWC", + ), + ) + + assert np.allclose( + res_6, + pool3D_forward_naive( + x=x_NCDHW, + ksize=ksize, + pool_type="avg", + strides=[1, 1, 1], + paddings=[10, 20], + padding_algorithm="VALID", + data_format="NCDHW", + ), + rtol=0.07, + atol=1e-05, + ) # SAME assert np.allclose( res_7, - pool3D_forward_naive(x=x_NDHWC, - ksize=ksize, - pool_type="avg", - strides=[1, 1, 2], - paddings=[10, 20], - padding_algorithm="SAME", - data_format="NDHWC")) - - assert np.allclose(res_8, - pool3D_forward_naive(x=x_NCDHW, - ksize=[4, 4, 4], - pool_type="avg", - strides=[1, 1, 1], - paddings=[10, 20], - padding_algorithm="SAME", - data_format="NCDHW"), - rtol=0.07, - atol=1e-05) + pool3D_forward_naive( + x=x_NDHWC, + ksize=ksize, + pool_type="avg", + strides=[1, 1, 2], + paddings=[10, 20], + padding_algorithm="SAME", + data_format="NDHWC", + ), + ) + + assert np.allclose( + res_8, + pool3D_forward_naive( + x=x_NCDHW, + ksize=[4, 4, 4], + pool_type="avg", + strides=[1, 1, 1], + paddings=[10, 20], + padding_algorithm="SAME", + data_format="NCDHW", + ), + rtol=0.07, + atol=1e-05, + ) class TestPool3DAPI_Error(unittest.TestCase): - def test_api(self): - input_NDHWC = fluid.layers.data(name="input_NDHWC", - shape=[2, 5, 5, 5, 3], - append_batch_size=False, - dtype="float32") + input_NDHWC = fluid.layers.data( + name="input_NDHWC", + shape=[2, 5, 5, 5, 3], + append_batch_size=False, + dtype="float32", + ) ksize = [3, 3, 3] # cudnn type error def run_1(): - out_1 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1, 1], - use_cudnn=[0], - data_format="NDHWC") + out_1 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1, 1], + use_cudnn=[0], + data_format="NDHWC", + ) self.assertRaises(TypeError, run_1) # data_format value error def run_2(): - out_2 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[1, 1, 1], - use_cudnn=False, - data_format="NDHWCC") + out_2 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[1, 1, 1], + use_cudnn=False, + data_format="NDHWCC", + ) self.assertRaises(ValueError, run_2) # padding str value error def run_3(): - out_3 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALIDSAME", - use_cudnn=False, - data_format="NDHWC") + out_3 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALIDSAME", + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_3) # padding str valid and ceil_mode value error def run_4(): - out_4 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="max", - pool_padding="VALID", - use_cudnn=False, - ceil_mode=True, - data_format="NDHWC") + out_4 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="max", + pool_padding="VALID", + use_cudnn=False, + ceil_mode=True, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_4) # padding with 8 ele. value error def run_5(): - out_5 = fluid.layers.pool3d(input=input_NDHWC, - pool_size=ksize, - pool_type="max", - pool_padding=[[1, 1], [0, 0], [0, 0], - [1, 1], [1, 1]], - use_cudnn=False, - data_format="NDHWC") + out_5 = fluid.layers.pool3d( + input=input_NDHWC, + pool_size=ksize, + pool_type="max", + pool_padding=[[1, 1], [0, 0], [0, 0], [1, 1], [1, 1]], + use_cudnn=False, + data_format="NDHWC", + ) self.assertRaises(ValueError, run_5) diff --git a/python/paddle/fluid/tests/unittests/test_pool_max_op.py b/python/paddle/fluid/tests/unittests/test_pool_max_op.py index 64204aa0ff26f..f324adb6c8d54 100644 --- a/python/paddle/fluid/tests/unittests/test_pool_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool_max_op.py @@ -25,12 +25,9 @@ def adaptive_end_index(index, input_size, output_size): return int(np.ceil((index + 1) * input_size / output_size)) -def max_pool3D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=False, - adaptive=False): +def max_pool3D_forward_naive( + x, ksize, strides, paddings, global_pool=False, adaptive=False +): N, C, D, H, W = x.shape if global_pool: @@ -77,19 +74,19 @@ def max_pool3D_forward_naive(x, sub_deep = index[0][0] sub_row = index[1][0] sub_col = index[2][0] - index = ((d_start + sub_deep) * H + - (h_start + sub_row)) * W + w_start + sub_col + index = ( + ((d_start + sub_deep) * H + (h_start + sub_row)) * W + + w_start + + sub_col + ) mask[n, c, k, i, j] = index return out, mask -def max_pool2D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=False, - adaptive=False): +def max_pool2D_forward_naive( + x, ksize, strides, paddings, global_pool=False, adaptive=False +): N, C, H, W = x.shape if global_pool: @@ -132,17 +129,21 @@ def max_pool2D_forward_naive(x, class TestMaxPoolWithIndex_Op(OpTest): - def setUp(self): self.init_test_case() self.init_global() self.init_adaptive() input = np.random.random(self.shape).astype("float64") - input = np.round(input * 100., 2) - output, mask = self.pool_forward_naive(input, self.ksize, self.strides, - self.paddings, self.global_pool, - self.adaptive) + input = np.round(input * 100.0, 2) + output, mask = self.pool_forward_naive( + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.adaptive, + ) output = output.astype("float64") mask = mask.astype("int32") @@ -179,13 +180,11 @@ def init_adaptive(self): class TestCase1(TestMaxPoolWithIndex_Op): - def init_global(self): self.global_pool = True class TestCase2(TestMaxPoolWithIndex_Op): - def init_test_case(self): self.op_type = "max_pool3d_with_index" self.pool_forward_naive = max_pool3D_forward_naive @@ -199,14 +198,12 @@ def init_global(self): class TestCase3(TestCase2): - def init_global(self): self.global_pool = False -#----------------max_pool2d_with_index---------------- +# ----------------max_pool2d_with_index---------------- class TestCase4(TestMaxPoolWithIndex_Op): - def init_test_case(self): self.op_type = "max_pool2d_with_index" self.pool_forward_naive = max_pool2D_forward_naive @@ -220,13 +217,11 @@ def init_global(self): class TestCase5(TestCase4): - def init_global(self): self.global_pool = False class TestCase6(TestMaxPoolWithIndex_Op): - def init_test_case(self): self.op_type = "max_pool2d_with_index" self.pool_forward_naive = max_pool2D_forward_naive @@ -240,19 +235,16 @@ def init_global(self): class TestCase7(TestCase6): - def init_global(self): self.global_pool = False class TestCastAdaptive2d(TestCase6): - def init_adaptive(self): self.adaptive = True class TestCastAdaptive3d(TestMaxPoolWithIndex_Op): - def init_adaptive(self): self.adaptive = True diff --git a/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py b/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py index dcbd5928fc4b1..d666157e7d287 100644 --- a/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py +++ b/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py @@ -45,12 +45,14 @@ def py_pnpair_op(score, label, query, column=-1, weight=None): else: neg += w - return np.array(pos).astype('float32'), np.array(neg).astype( - 'float32'), np.array(neu).astype('float32') + return ( + np.array(pos).astype('float32'), + np.array(neg).astype('float32'), + np.array(neu).astype('float32'), + ) class TestPositiveNegativePairOp(OpTest): - def setUp(self): self.op_type = 'positive_negative_pair' batch_size = 20 @@ -58,7 +60,8 @@ def setUp(self): score = np.random.normal(size=(batch_size, 1)).astype('float32') label = np.random.normal(size=(batch_size, 1)).astype('float32') query = np.array( - [np.random.randint(max_query_id) for i in range(batch_size)]) + [np.random.randint(max_query_id) for i in range(batch_size)] + ) query = np.reshape(query, newshape=(batch_size, 1)).astype('int64') pos, neg, neu = py_pnpair_op(score, label, query) @@ -67,7 +70,7 @@ def setUp(self): self.outputs = { 'PositivePair': pos, 'NegativePair': neg, - 'NeutralPair': neu + 'NeutralPair': neu, } def test_check_output(self): @@ -75,7 +78,6 @@ def test_check_output(self): class TestPositiveNegativePairOpAccumulateWeight(OpTest): - def setUp(self): self.op_type = 'positive_negative_pair' batch_size = 20 @@ -86,21 +88,23 @@ def setUp(self): label = np.random.normal(size=(batch_size, 1)).astype('float32') weight = np.random.normal(size=(batch_size, 1)).astype('float32') query = np.array( - [np.random.randint(max_query_id) for i in range(batch_size)]) + [np.random.randint(max_query_id) for i in range(batch_size)] + ) query = np.reshape(query, newshape=(batch_size, 1)).astype('int64') - acc_pos = np.reshape(np.random.randint(max_random_num), - newshape=(1)).astype('float32') - acc_neg = np.reshape(np.random.randint(max_random_num), - newshape=(1)).astype('float32') - acc_neu = np.reshape(np.random.randint(max_random_num), - newshape=(1)).astype('float32') + acc_pos = np.reshape( + np.random.randint(max_random_num), newshape=(1) + ).astype('float32') + acc_neg = np.reshape( + np.random.randint(max_random_num), newshape=(1) + ).astype('float32') + acc_neu = np.reshape( + np.random.randint(max_random_num), newshape=(1) + ).astype('float32') column = np.random.randint(score_dim) - pos, neg, neu = py_pnpair_op(score, - label, - query, - column=column, - weight=weight) + pos, neg, neu = py_pnpair_op( + score, label, query, column=column, weight=weight + ) self.inputs = { 'Score': score, 'Label': label, @@ -108,13 +112,13 @@ def setUp(self): 'AccumulatePositivePair': acc_pos, 'AccumulateNegativePair': acc_neg, 'AccumulateNeutralPair': acc_neu, - 'Weight': weight + 'Weight': weight, } self.attrs = {'column': column} self.outputs = { 'PositivePair': pos + acc_pos, 'NegativePair': neg + acc_neg, - 'NeutralPair': neu + acc_neu + 'NeutralPair': neu + acc_neu, } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/test_pow.py b/python/paddle/fluid/tests/unittests/test_pow.py index 9df1af1375c72..79282f3460f1a 100755 --- a/python/paddle/fluid/tests/unittests/test_pow.py +++ b/python/paddle/fluid/tests/unittests/test_pow.py @@ -71,7 +71,7 @@ def test_power(self): """test_power.""" np.random.seed(7) # test 1-d float tensor ** float scalar - dims = (np.random.randint(200, 300), ) + dims = (np.random.randint(200, 300),) x = (np.random.rand(*dims) * 10).astype(np.float64) y = np.random.rand() * 10 res = _run_power(DYNAMIC, x, y) @@ -80,7 +80,7 @@ def test_power(self): np.testing.assert_allclose(res, np.power(x, y), rtol=1e-05) # test 1-d float tensor ** int scalar - dims = (np.random.randint(200, 300), ) + dims = (np.random.randint(200, 300),) x = (np.random.rand(*dims) * 10).astype(np.float64) y = int(np.random.rand() * 10) res = _run_power(DYNAMIC, x, y) @@ -96,7 +96,7 @@ def test_power(self): np.testing.assert_allclose(res, np.power(x, y), rtol=1e-05) # test 1-d float tensor ** 1-d float tensor - dims = (np.random.randint(200, 300), ) + dims = (np.random.randint(200, 300),) x = (np.random.rand(*dims) * 10).astype(np.float64) y = (np.random.rand(*dims) * 10).astype(np.float64) res = _run_power(DYNAMIC, x, y) @@ -105,7 +105,7 @@ def test_power(self): np.testing.assert_allclose(res, np.power(x, y), rtol=1e-05) # test 1-d int tensor ** 1-d int tensor - dims = (np.random.randint(200, 300), ) + dims = (np.random.randint(200, 300),) x = (np.random.rand(*dims) * 10).astype(np.int64) y = (np.random.rand(*dims) * 10).astype(np.int64) res = _run_power(DYNAMIC, x, y) @@ -114,7 +114,7 @@ def test_power(self): np.testing.assert_allclose(res, np.power(x, y), rtol=1e-05) # test 1-d int tensor ** 1-d int tensor - dims = (np.random.randint(200, 300), ) + dims = (np.random.randint(200, 300),) x = (np.random.rand(*dims) * 10).astype(np.int32) y = (np.random.rand(*dims) * 10).astype(np.int32) res = _run_power(DYNAMIC, x, y) @@ -123,7 +123,7 @@ def test_power(self): np.testing.assert_allclose(res, np.power(x, y), rtol=1e-05) # test 1-d int tensor ** 1-d int tensor - dims = (np.random.randint(200, 300), ) + dims = (np.random.randint(200, 300),) x = (np.random.rand(*dims) * 10).astype(np.float32) y = (np.random.rand(*dims) * 10).astype(np.float32) res = _run_power(DYNAMIC, x, y) @@ -132,8 +132,11 @@ def test_power(self): np.testing.assert_allclose(res, np.power(x, y), rtol=1e-05) # test broadcast - dims = (np.random.randint(1, 10), np.random.randint(5, 10), - np.random.randint(5, 10)) + dims = ( + np.random.randint(1, 10), + np.random.randint(5, 10), + np.random.randint(5, 10), + ) x = (np.random.rand(*dims) * 10).astype(np.float64) y = (np.random.rand(dims[-1]) * 10).astype(np.float64) res = _run_power(DYNAMIC, x, y) @@ -150,22 +153,28 @@ def test_errors(self): np.random.seed(7) # test dynamic computation graph: inputs must be broadcastable - dims = (np.random.randint(1, 10), np.random.randint(5, 10), - np.random.randint(5, 10)) + dims = ( + np.random.randint(1, 10), + np.random.randint(5, 10), + np.random.randint(5, 10), + ) x = (np.random.rand(*dims) * 10).astype(np.float64) y = (np.random.rand(dims[-1] + 1) * 10).astype(np.float64) self.assertRaises(ValueError, _run_power, DYNAMIC, x, y) self.assertRaises(ValueError, _run_power, STATIC, x, y) # test dynamic computation graph: inputs must be broadcastable - dims = (np.random.randint(1, 10), np.random.randint(5, 10), - np.random.randint(5, 10)) + dims = ( + np.random.randint(1, 10), + np.random.randint(5, 10), + np.random.randint(5, 10), + ) x = (np.random.rand(*dims) * 10).astype(np.float64) y = (np.random.rand(dims[-1] + 1) * 10).astype(np.int8) self.assertRaises(TypeError, paddle.pow, x, y) # test 1-d float tensor ** int string - dims = (np.random.randint(200, 300), ) + dims = (np.random.randint(200, 300),) x = (np.random.rand(*dims) * 10).astype(np.float64) y = int(np.random.rand() * 10) self.assertRaises(TypeError, paddle.pow, x, str(y)) diff --git a/python/paddle/fluid/tests/unittests/test_pow2_decay_with_linear_warmup_op.py b/python/paddle/fluid/tests/unittests/test_pow2_decay_with_linear_warmup_op.py index 43f98cada4235..69619ed02af84 100644 --- a/python/paddle/fluid/tests/unittests/test_pow2_decay_with_linear_warmup_op.py +++ b/python/paddle/fluid/tests/unittests/test_pow2_decay_with_linear_warmup_op.py @@ -23,8 +23,9 @@ def gen_pow2_warmup_op_lr(warmup_steps, total_steps, base_lr, end_lr, place): main = paddle.static.Program() startup = paddle.static.Program() with paddle.static.program_guard(main, startup): - lr = pow2_decay_with_linear_warmup(warmup_steps, total_steps, base_lr, - end_lr) + lr = pow2_decay_with_linear_warmup( + warmup_steps, total_steps, base_lr, end_lr + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): exe.run(startup) @@ -34,18 +35,21 @@ def gen_pow2_warmup_op_lr(warmup_steps, total_steps, base_lr, end_lr, place): class Pow2Warmup(LinearWarmup): - def __init__(self, warmup_steps, total_steps, base_lr, end_lr): assert total_steps > warmup_steps - lr_sch = PolynomialDecay(learning_rate=base_lr, - decay_steps=total_steps - warmup_steps, - end_lr=end_lr, - power=2) + lr_sch = PolynomialDecay( + learning_rate=base_lr, + decay_steps=total_steps - warmup_steps, + end_lr=end_lr, + power=2, + ) - super(Pow2Warmup, self).__init__(learning_rate=lr_sch, - warmup_steps=warmup_steps, - start_lr=0.0, - end_lr=base_lr) + super(Pow2Warmup, self).__init__( + learning_rate=lr_sch, + warmup_steps=warmup_steps, + start_lr=0.0, + end_lr=base_lr, + ) def gen_pow2_warmup_py_lr(warmup_steps, total_steps, base_lr, end_lr, place): @@ -57,7 +61,6 @@ def gen_pow2_warmup_py_lr(warmup_steps, total_steps, base_lr, end_lr, place): class TestPow2WarmupLRScheduler(unittest.TestCase): - def setUp(self): paddle.enable_static() self.params = { diff --git a/python/paddle/fluid/tests/unittests/test_precision_recall_op.py b/python/paddle/fluid/tests/unittests/test_precision_recall_op.py index 26e12b0deb0b4..8bc80de3429d5 100644 --- a/python/paddle/fluid/tests/unittests/test_precision_recall_op.py +++ b/python/paddle/fluid/tests/unittests/test_precision_recall_op.py @@ -85,16 +85,21 @@ def compute_metrics(states, cls_num): class TestPrecisionRecallOp_0(OpTest): - def setUp(self): self.op_type = "precision_recall" ins_num = 64 cls_num = 10 max_probs = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') - idxs = np.random.choice(range(cls_num), ins_num).reshape( - (ins_num, 1)).astype('int32') - labels = np.random.choice(range(cls_num), ins_num).reshape( - (ins_num, 1)).astype('int32') + idxs = ( + np.random.choice(range(cls_num), ins_num) + .reshape((ins_num, 1)) + .astype('int32') + ) + labels = ( + np.random.choice(range(cls_num), ins_num) + .reshape((ins_num, 1)) + .astype('int32') + ) states = get_states(idxs, labels, cls_num) metrics = compute_metrics(states, cls_num) @@ -105,7 +110,7 @@ def setUp(self): self.outputs = { 'BatchMetrics': metrics, 'AccumMetrics': metrics, - 'AccumStatesInfo': states + 'AccumStatesInfo': states, } def test_check_output(self): @@ -113,17 +118,22 @@ def test_check_output(self): class TestPrecisionRecallOp_1(OpTest): - def setUp(self): self.op_type = "precision_recall" ins_num = 64 cls_num = 10 max_probs = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') - idxs = np.random.choice(range(cls_num), ins_num).reshape( - (ins_num, 1)).astype('int32') + idxs = ( + np.random.choice(range(cls_num), ins_num) + .reshape((ins_num, 1)) + .astype('int32') + ) weights = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') - labels = np.random.choice(range(cls_num), ins_num).reshape( - (ins_num, 1)).astype('int32') + labels = ( + np.random.choice(range(cls_num), ins_num) + .reshape((ins_num, 1)) + .astype('int32') + ) states = get_states(idxs, labels, cls_num, weights) metrics = compute_metrics(states, cls_num) @@ -134,13 +144,13 @@ def setUp(self): 'MaxProbs': max_probs, 'Indices': idxs, 'Labels': labels, - 'Weights': weights + 'Weights': weights, } self.outputs = { 'BatchMetrics': metrics, 'AccumMetrics': metrics, - 'AccumStatesInfo': states + 'AccumStatesInfo': states, } def test_check_output(self): @@ -148,17 +158,22 @@ def test_check_output(self): class TestPrecisionRecallOp_2(OpTest): - def setUp(self): self.op_type = "precision_recall" ins_num = 64 cls_num = 10 max_probs = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') - idxs = np.random.choice(range(cls_num), ins_num).reshape( - (ins_num, 1)).astype('int32') + idxs = ( + np.random.choice(range(cls_num), ins_num) + .reshape((ins_num, 1)) + .astype('int32') + ) weights = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') - labels = np.random.choice(range(cls_num), ins_num).reshape( - (ins_num, 1)).astype('int32') + labels = ( + np.random.choice(range(cls_num), ins_num) + .reshape((ins_num, 1)) + .astype('int32') + ) states = np.random.randint(0, 30, (cls_num, 4)).astype('float32') accum_states = get_states(idxs, labels, cls_num, weights) @@ -173,13 +188,13 @@ def setUp(self): 'Indices': idxs, 'Labels': labels, 'Weights': weights, - 'StatesInfo': states + 'StatesInfo': states, } self.outputs = { 'BatchMetrics': batch_metrics, 'AccumMetrics': accum_metrics, - 'AccumStatesInfo': accum_states + 'AccumStatesInfo': accum_states, } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/test_prelu_op.py b/python/paddle/fluid/tests/unittests/test_prelu_op.py index bf6de7911da86..c267286be9dc4 100644 --- a/python/paddle/fluid/tests/unittests/test_prelu_op.py +++ b/python/paddle/fluid/tests/unittests/test_prelu_op.py @@ -38,11 +38,13 @@ def ref_prelu_nn(x, num_parameters, init): class TestFunctionalPReluAPI(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() - self.x_np = np.random.uniform(-1., 1., [1, 2, 3, 4]).astype('float32') + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) + self.x_np = np.random.uniform(-1.0, 1.0, [1, 2, 3, 4]).astype('float32') self.weight_np_0 = np.random.randn(1).astype('float32') self.weight_np_1 = np.random.randn(self.x_np.shape[1]).astype('float32') @@ -52,11 +54,9 @@ def static_check(self, weight_np): weight = paddle.fluid.data('Alpha', weight_np.shape, 'float32') out = F.prelu(x, weight) exe = paddle.static.Executor(self.place) - res = exe.run(feed={ - 'X': self.x_np, - 'Alpha': weight_np - }, - fetch_list=[out]) + res = exe.run( + feed={'X': self.x_np, 'Alpha': weight_np}, fetch_list=[out] + ) out_ref = ref_prelu(self.x_np, weight_np) np.testing.assert_allclose(out_ref, res[0], rtol=1e-05) @@ -83,44 +83,46 @@ def test_dygraph_api_eager(self): def test_error(self): with paddle.static.program_guard(paddle.static.Program()): - weight_fp32 = paddle.fluid.data(name='weight_fp32', - shape=[1], - dtype='float32') + weight_fp32 = paddle.fluid.data( + name='weight_fp32', shape=[1], dtype='float32' + ) # The input type must be Variable. self.assertRaises(TypeError, F.prelu, x=1, weight=weight_fp32) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[2, 3], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[2, 3], dtype='int32' + ) self.assertRaises(TypeError, F.prelu, x=x_int32, weight=weight_fp32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[2, 3], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[2, 3], dtype='float16' + ) F.prelu(x=x_fp16, weight=weight_fp32) class TestNNPReluAPI(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.x_np = np.ones([1, 2, 3, 4]).astype('float32') def test_static_api(self): startup_program = paddle.static.Program() train_program = paddle.static.Program() with paddle.static.program_guard(train_program, startup_program): - x = paddle.fluid.data(name='X', - shape=self.x_np.shape, - dtype='float32') + x = paddle.fluid.data( + name='X', shape=self.x_np.shape, dtype='float32' + ) m = paddle.nn.PReLU() out = m(x) exe = paddle.static.Executor(self.place) exe.run(startup_program) - res = exe.run(train_program, - feed={'X': self.x_np}, - fetch_list=[out]) + res = exe.run( + train_program, feed={'X': self.x_np}, fetch_list=[out] + ) out_ref = ref_prelu_nn(self.x_np, 1, 0.25) np.testing.assert_allclose(out_ref, res[0], rtol=1e-05) @@ -152,8 +154,11 @@ def test_dygraph_api(self): np.testing.assert_allclose(out_ref, out.numpy(), rtol=1e-05) x = paddle.to_tensor(self.x_np) - m = paddle.nn.PReLU(weight_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(0.5))) + m = paddle.nn.PReLU( + weight_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(0.5) + ) + ) out = m(x) out_ref = ref_prelu_nn(self.x_np, 1, 0.5) np.testing.assert_allclose(out_ref, out.numpy(), rtol=1e-05) @@ -167,7 +172,6 @@ def prelu_api_wrapper(x, weight, data_format="NCHW"): class PReluTest(OpTest): - def setUp(self): self.init_dtype() self.init_input_shape() @@ -182,12 +186,9 @@ def setUp(self): x_np[np.abs(x_np) < 0.005] = 0.02 if self.attrs == { - 'mode': "all", - "data_format": "NCHW" - } or self.attrs == { - 'mode': "all", - "data_format": "NHWC" - }: + 'mode': "all", + "data_format": "NCHW", + } or self.attrs == {'mode': "all", "data_format": "NHWC"}: alpha_np = np.random.uniform(-1, -0.5, (1)) elif self.attrs == {'mode': "channel", "data_format": "NCHW"}: alpha_np = np.random.uniform(-1, -0.5, [1, self.x_shape[1], 1, 1]) @@ -205,15 +206,17 @@ def setUp(self): # since np operands could not be broadcast together with shapes (1,100,2,2,2,3) (1,100,1,1) reshaped_alpha = self.inputs['Alpha'] if self.attrs == {'mode': "channel", "data_format": "NCHW"}: - reshaped_alpha = np.reshape(self.inputs['Alpha'], - [1, self.x_shape[1]] + - [1] * len(self.x_shape[2:])) + reshaped_alpha = np.reshape( + self.inputs['Alpha'], + [1, self.x_shape[1]] + [1] * len(self.x_shape[2:]), + ) elif self.attrs == {'mode': "channel", "data_format": "NHWC"}: - reshaped_alpha = np.reshape(self.inputs['Alpha'], - [1] + [1] * len(self.x_shape[1:-1]) + - [self.x_shape[-1]]) - out_np = np.maximum(self.inputs['X'], 0.) - out_np = out_np + np.minimum(self.inputs['X'], 0.) * reshaped_alpha + reshaped_alpha = np.reshape( + self.inputs['Alpha'], + [1] + [1] * len(self.x_shape[1:-1]) + [self.x_shape[-1]], + ) + out_np = np.maximum(self.inputs['X'], 0.0) + out_np = out_np + np.minimum(self.inputs['X'], 0.0) * reshaped_alpha assert out_np is not self.inputs['X'] self.outputs = {'Out': out_np} @@ -234,11 +237,9 @@ def test_check_grad(self): @skip_check_grad_ci( - reason= - "[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" + reason="[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" ) class TestModeAll(PReluTest): - def init_input_shape(self): self.x_shape = [2, 3, 4, 5] @@ -247,11 +248,9 @@ def init_attr(self): @skip_check_grad_ci( - reason= - "[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" + reason="[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" ) class TestModeAllNHWC(PReluTest): - def init_input_shape(self): self.x_shape = [2, 3, 4, 50] @@ -260,7 +259,6 @@ def init_attr(self): class TestModeElt(PReluTest): - def init_input_shape(self): self.x_shape = [3, 2, 5, 10] @@ -269,7 +267,6 @@ def init_attr(self): class TestModeEltNHWC(PReluTest): - def init_input_shape(self): self.x_shape = [3, 2, 5, 10] @@ -278,11 +275,9 @@ def init_attr(self): @skip_check_grad_ci( - reason= - "[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" + reason="[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" ) class TestModeAllRank3(PReluTest): - def init_input_shape(self): self.x_shape = [1, 200, 3] @@ -291,11 +286,9 @@ def init_attr(self): @skip_check_grad_ci( - reason= - "[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" + reason="[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" ) class TestModeAllRank3NHWC(PReluTest): - def init_input_shape(self): self.x_shape = [1, 200, 3] @@ -304,11 +297,9 @@ def init_attr(self): @skip_check_grad_ci( - reason= - "[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" + reason="[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" ) class TestModeAllRank6(PReluTest): - def init_input_shape(self): self.x_shape = [1, 2, 3, 4, 5, 6] @@ -317,11 +308,9 @@ def init_attr(self): @skip_check_grad_ci( - reason= - "[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" + reason="[skip shape check] Input(Alpha) must be 1-D and only has one data in 'all' mode" ) class TestModeAllRank6NHWC(PReluTest): - def init_input_shape(self): self.x_shape = [1, 2, 3, 4, 5, 6] @@ -330,7 +319,6 @@ def init_attr(self): class TestModeChannelRank3(PReluTest): - def init_input_shape(self): self.x_shape = [1, 200, 3] @@ -339,7 +327,6 @@ def init_attr(self): class TestModeChannelRank3NHWC(PReluTest): - def init_input_shape(self): self.x_shape = [1, 3, 100] @@ -348,7 +335,6 @@ def init_attr(self): class TestModeChannelRank6(PReluTest): - def init_input_shape(self): self.x_shape = [1, 100, 2, 2, 2, 2] @@ -357,7 +343,6 @@ def init_attr(self): class TestModeChannelRank6NHWC(PReluTest): - def init_input_shape(self): self.x_shape = [1, 2, 2, 2, 2, 100] @@ -366,7 +351,6 @@ def init_attr(self): class TestModeElementRank3(PReluTest): - def init_input_shape(self): self.x_shape = [3, 10, 10] @@ -375,7 +359,6 @@ def init_attr(self): class TestModeElementRank3NHWC(PReluTest): - def init_input_shape(self): self.x_shape = [3, 10, 10] @@ -384,7 +367,6 @@ def init_attr(self): class TestModeElementRank6(PReluTest): - def init_input_shape(self): self.x_shape = [3, 2, 2, 4, 5, 2] @@ -393,7 +375,6 @@ def init_attr(self): class TestModeElementRank6NHWC(PReluTest): - def init_input_shape(self): self.x_shape = [3, 2, 2, 4, 5, 2] @@ -401,15 +382,13 @@ def init_attr(self): self.attrs = {'mode': "element", "data_format": "NHWC"} -def create_test_fp16_class(parent, - check_grad=True, - atol=1e-3, - max_relative_error=0.05): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +def create_test_fp16_class( + parent, check_grad=True, atol=1e-3, max_relative_error=0.05 +): + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestPReluFp16Case(parent): - def init_dtype(self): self.dtype = np.float16 @@ -417,18 +396,20 @@ def test_check_output(self): if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_output_with_place(place, - atol=atol, - check_eager=self.eager_mode) + self.check_output_with_place( + place, atol=atol, check_eager=self.eager_mode + ) def test_check_grad(self): place = core.CUDAPlace(0) if core.is_float16_supported(place) and check_grad: self.check_grad_with_place( - place, ['X', 'Alpha'], + place, + ['X', 'Alpha'], 'Out', max_relative_error=max_relative_error, - check_eager=self.eager_mode) + check_eager=self.eager_mode, + ) cls_name = "{0}_{1}".format(parent.__name__, "Fp16Op") TestPReluFp16Case.__name__ = cls_name @@ -460,27 +441,26 @@ def prelu_t(x, mode, param_attr=None, name=None, data_format='NCHW'): shape=alpha_shape, dtype='float32', is_bias=False, - default_initializer=fluid.initializer.ConstantInitializer(0.25)) + default_initializer=fluid.initializer.ConstantInitializer(0.25), + ) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="prelu", - inputs={ - "X": x, - 'Alpha': alpha - }, - attrs={ - "mode": mode, - 'data_format': data_format - }, - outputs={"Out": out}) + helper.append_op( + type="prelu", + inputs={"X": x, 'Alpha': alpha}, + attrs={"mode": mode, 'data_format': data_format}, + outputs={"Out": out}, + ) return out # error message test if mode is not one of 'all', 'channel', 'element' class TestModeError(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.x_np = np.ones([1, 2, 3, 4]).astype('float32') def test_mode_error(self): @@ -490,7 +470,7 @@ def test_mode_error(self): try: y = prelu_t(x, 'any') except Exception as e: - assert (e.args[0].find('InvalidArgument') != -1) + assert e.args[0].find('InvalidArgument') != -1 def test_data_format_error1(self): main_program = Program() @@ -499,7 +479,7 @@ def test_data_format_error1(self): try: y = prelu_t(x, 'channel', data_format='N') except Exception as e: - assert (e.args[0].find('InvalidArgument') != -1) + assert e.args[0].find('InvalidArgument') != -1 def test_data_format_error2(self): main_program = Program() diff --git a/python/paddle/fluid/tests/unittests/test_print_op.py b/python/paddle/fluid/tests/unittests/test_print_op.py index 1dc6cca53cb43..0ce1230f52e86 100755 --- a/python/paddle/fluid/tests/unittests/test_print_op.py +++ b/python/paddle/fluid/tests/unittests/test_print_op.py @@ -28,7 +28,6 @@ class TestPrintOpCPU(unittest.TestCase): - def setUp(self): self.place = paddle.CPUPlace() self.x_tensor = fluid.core.LoDTensor() @@ -48,17 +47,17 @@ def test_forward(self): switch_main_program(Program()) printed = self.build_network(True, print_phase='forward') exe = paddle.static.Executor(self.place) - outs = exe.run(feed={'x': self.x_tensor}, - fetch_list=[printed], - return_numpy=False) + outs = exe.run( + feed={'x': self.x_tensor}, fetch_list=[printed], return_numpy=False + ) def test_backward(self): switch_main_program(Program()) loss = self.build_network(False, print_phase='backward') exe = paddle.static.Executor(self.place) - outs = exe.run(feed={'x': self.x_tensor}, - fetch_list=[loss], - return_numpy=False) + outs = exe.run( + feed={'x': self.x_tensor}, fetch_list=[loss], return_numpy=False + ) def test_all_parameters(self): x = layers.data('x', shape=[3], dtype='float32', lod_level=1) @@ -78,36 +77,36 @@ def test_all_parameters(self): loss = paddle.mean(x) paddle.static.append_backward(loss=loss) exe = paddle.static.Executor(self.place) - outs = exe.run(feed={'x': self.x_tensor}, - fetch_list=[loss], - return_numpy=False) + outs = exe.run( + feed={'x': self.x_tensor}, fetch_list=[loss], return_numpy=False + ) def test_no_summarize(self): switch_main_program(Program()) printed = self.build_network(True, summarize=-1, print_phase='forward') exe = paddle.static.Executor(self.place) - outs = exe.run(feed={'x': self.x_tensor}, - fetch_list=[printed], - return_numpy=False) + outs = exe.run( + feed={'x': self.x_tensor}, fetch_list=[printed], return_numpy=False + ) class TestPrintOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of Print_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - paddle.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], paddle.CPUPlace() + ) self.assertRaises(TypeError, paddle.static.Print, x1) # The input dtype of Print_op must be float32, float64, int32_t, int64_t or bool. x2 = paddle.static.data(name='x2', shape=[4], dtype="float16") self.assertRaises(TypeError, paddle.static.Print, x2) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestPrintOpGPU(TestPrintOpCPU): - def setUp(self): self.place = paddle.CUDAPlace(0) self.x_tensor = fluid.core.LoDTensor() @@ -117,7 +116,6 @@ def setUp(self): class TestPrintOpBackward(unittest.TestCase): - def check_backward(self, use_cuda): main = paddle.static.Program() startup = paddle.static.Program() @@ -135,7 +133,8 @@ def check_backward(self, use_cuda): exe.run(startup) binary = paddle.static.CompiledProgram(main).with_data_parallel( - loss_name=loss.name) + loss_name=loss.name + ) img, label = init_data() feed_dict = {"image": img, "label": label} diff --git a/python/paddle/fluid/tests/unittests/test_prior_box_op.py b/python/paddle/fluid/tests/unittests/test_prior_box_op.py index 67d468ac626e6..80e567be34c32 100644 --- a/python/paddle/fluid/tests/unittests/test_prior_box_op.py +++ b/python/paddle/fluid/tests/unittests/test_prior_box_op.py @@ -19,19 +19,21 @@ import paddle -def python_prior_box(input, - image, - min_sizes, - aspect_ratios=[1.], - variances=[0.1, 0.1, 0.2, 0.2], - max_sizes=None, - flip=False, - clip=False, - step_w=0, - step_h=0, - offset=0.5, - min_max_aspect_ratios_order=False, - name=None): +def python_prior_box( + input, + image, + min_sizes, + aspect_ratios=[1.0], + variances=[0.1, 0.1, 0.2, 0.2], + max_sizes=None, + flip=False, + clip=False, + step_w=0, + step_h=0, + offset=0.5, + min_max_aspect_ratios_order=False, + name=None, +): return paddle.fluid.layers.detection.prior_box( input, image, @@ -44,11 +46,11 @@ def python_prior_box(input, steps=[step_w, step_h], offset=offset, name=name, - min_max_aspect_ratios_order=min_max_aspect_ratios_order) + min_max_aspect_ratios_order=min_max_aspect_ratios_order, + ) class TestPriorBoxOp(OpTest): - def set_data(self): self.init_test_params() self.init_test_input() @@ -107,8 +109,9 @@ def init_test_params(self): self.flip = True self.set_min_max_aspect_ratios_order() self.real_aspect_ratios = [1, 2.0, 1.0 / 2.0, 3.0, 1.0 / 3.0] - self.aspect_ratios = np.array(self.aspect_ratios, - dtype=np.float64).flatten() + self.aspect_ratios = np.array( + self.aspect_ratios, dtype=np.float64 + ).flatten() self.variances = [0.1, 0.1, 0.2, 0.2] self.variances = np.array(self.variances, dtype=np.float64).flatten() @@ -120,12 +123,12 @@ def init_test_params(self): def init_test_input(self): self.image = np.random.random( - (self.batch_size, self.image_channels, self.image_w, - self.image_h)).astype('float32') + (self.batch_size, self.image_channels, self.image_w, self.image_h) + ).astype('float32') self.input = np.random.random( - (self.batch_size, self.input_channels, self.layer_w, - self.layer_h)).astype('float32') + (self.batch_size, self.input_channels, self.layer_w, self.layer_h) + ).astype('float32') def init_test_output(self): out_dim = (self.layer_h, self.layer_w, self.num_priors, 4) @@ -146,73 +149,78 @@ def init_test_output(self): ar = self.real_aspect_ratios[r] c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 else: - c_w = c_h = min_size / 2. - out_boxes[h, w, idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + c_w = c_h = min_size / 2.0 + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # rest of priors for r in range(len(self.real_aspect_ratios)): ar = self.real_aspect_ratios[r] - if abs(ar - 1.) < 1e-6: + if abs(ar - 1.0) < 1e-6: continue c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # clip the prior's coordidate such that it is within[0, 1] if self.clip: out_boxes = np.clip(out_boxes, 0.0, 1.0) # set the variance. - out_var = np.tile(self.variances, - (self.layer_h, self.layer_w, self.num_priors, 1)) + out_var = np.tile( + self.variances, (self.layer_h, self.layer_w, self.num_priors, 1) + ) self.out_boxes = out_boxes.astype('float32') self.out_var = out_var.astype('float32') class TestPriorBoxOpWithoutMaxSize(TestPriorBoxOp): - def set_max_sizes(self): self.max_sizes = [] class TestPriorBoxOpWithSpecifiedOutOrder(TestPriorBoxOp): - def set_min_max_aspect_ratios_order(self): self.min_max_aspect_ratios_order = True diff --git a/python/paddle/fluid/tests/unittests/test_prod_op.py b/python/paddle/fluid/tests/unittests/test_prod_op.py index cd754f7dbaf26..24605da11e8a7 100644 --- a/python/paddle/fluid/tests/unittests/test_prod_op.py +++ b/python/paddle/fluid/tests/unittests/test_prod_op.py @@ -19,7 +19,6 @@ class TestProdOp(unittest.TestCase): - def setUp(self): self.input = np.random.random(size=(10, 10, 5)).astype(np.float32) @@ -27,54 +26,52 @@ def run_imperative(self): input = paddle.to_tensor(self.input) dy_result = paddle.prod(input) expected_result = np.prod(self.input) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05 + ) dy_result = paddle.prod(input, axis=1) expected_result = np.prod(self.input, axis=1) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05 + ) dy_result = paddle.prod(input, axis=-1) expected_result = np.prod(self.input, axis=-1) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05 + ) dy_result = paddle.prod(input, axis=[0, 1]) expected_result = np.prod(self.input, axis=(0, 1)) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05, - atol=1e-8) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05, atol=1e-8 + ) dy_result = paddle.prod(input, axis=1, keepdim=True) expected_result = np.prod(self.input, axis=1, keepdims=True) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05 + ) dy_result = paddle.prod(input, axis=1, dtype='int64') expected_result = np.prod(self.input, axis=1, dtype=np.int64) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05 + ) dy_result = paddle.prod(input, axis=1, keepdim=True, dtype='int64') - expected_result = np.prod(self.input, - axis=1, - keepdims=True, - dtype=np.int64) - np.testing.assert_allclose(dy_result.numpy(), - expected_result, - rtol=1e-05) + expected_result = np.prod( + self.input, axis=1, keepdims=True, dtype=np.int64 + ) + np.testing.assert_allclose( + dy_result.numpy(), expected_result, rtol=1e-05 + ) def run_static(self, use_gpu=False): - input = paddle.fluid.data(name='input', - shape=[10, 10, 5], - dtype='float32') + input = paddle.fluid.data( + name='input', shape=[10, 10, 5], dtype='float32' + ) result0 = paddle.prod(input) result1 = paddle.prod(input, axis=1) result2 = paddle.prod(input, axis=-1) @@ -86,44 +83,49 @@ def run_static(self, use_gpu=False): place = paddle.CUDAPlace(0) if use_gpu else paddle.CPUPlace() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) - static_result = exe.run(feed={"input": self.input}, - fetch_list=[ - result0, result1, result2, result3, result4, - result5, result6 - ]) + static_result = exe.run( + feed={"input": self.input}, + fetch_list=[ + result0, + result1, + result2, + result3, + result4, + result5, + result6, + ], + ) expected_result = np.prod(self.input) - np.testing.assert_allclose(static_result[0], - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + static_result[0], expected_result, rtol=1e-05 + ) expected_result = np.prod(self.input, axis=1) - np.testing.assert_allclose(static_result[1], - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + static_result[1], expected_result, rtol=1e-05 + ) expected_result = np.prod(self.input, axis=-1) - np.testing.assert_allclose(static_result[2], - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + static_result[2], expected_result, rtol=1e-05 + ) expected_result = np.prod(self.input, axis=(0, 1)) - np.testing.assert_allclose(static_result[3], - expected_result, - rtol=1e-05, - atol=1e-8) + np.testing.assert_allclose( + static_result[3], expected_result, rtol=1e-05, atol=1e-8 + ) expected_result = np.prod(self.input, axis=1, keepdims=True) - np.testing.assert_allclose(static_result[4], - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + static_result[4], expected_result, rtol=1e-05 + ) expected_result = np.prod(self.input, axis=1, dtype=np.int64) - np.testing.assert_allclose(static_result[5], - expected_result, - rtol=1e-05) - expected_result = np.prod(self.input, - axis=1, - keepdims=True, - dtype=np.int64) - np.testing.assert_allclose(static_result[6], - expected_result, - rtol=1e-05) + np.testing.assert_allclose( + static_result[5], expected_result, rtol=1e-05 + ) + expected_result = np.prod( + self.input, axis=1, keepdims=True, dtype=np.int64 + ) + np.testing.assert_allclose( + static_result[6], expected_result, rtol=1e-05 + ) def test_cpu(self): paddle.disable_static(place=paddle.CPUPlace()) @@ -146,14 +148,14 @@ def test_gpu(self): class TestProdOpError(unittest.TestCase): - def test_error(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[2, 2, 4], dtype='float32') - bool_x = paddle.fluid.data(name='bool_x', - shape=[2, 2, 4], - dtype='bool') + bool_x = paddle.fluid.data( + name='bool_x', shape=[2, 2, 4], dtype='bool' + ) # The argument x shoule be a Tensor self.assertRaises(TypeError, paddle.prod, [1]) @@ -168,7 +170,6 @@ def test_error(self): class TestProdWithTensorAxis1(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.prod self.np_api = np.prod @@ -178,7 +179,6 @@ def init_data(self): class TestProdWithTensorAxis2(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.prod self.np_api = np.prod @@ -187,7 +187,7 @@ def init_data(self): self.tensor_axis = [ 0, paddle.to_tensor([1], 'int64'), - paddle.to_tensor([2], 'int64') + paddle.to_tensor([2], 'int64'), ] diff --git a/python/paddle/fluid/tests/unittests/test_profiler.py b/python/paddle/fluid/tests/unittests/test_profiler.py index 2b630ed384046..b7dd11f7b8a80 100644 --- a/python/paddle/fluid/tests/unittests/test_profiler.py +++ b/python/paddle/fluid/tests/unittests/test_profiler.py @@ -26,7 +26,6 @@ class TestProfiler(unittest.TestCase): - @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) @@ -38,9 +37,9 @@ def build_program(self, compile_program=True): image = fluid.layers.data(name='x', shape=[784], dtype='float32') hidden1 = fluid.layers.fc(input=image, size=64, act='relu') i = layers.zeros(shape=[1], dtype='int64') - counter = fluid.layers.zeros(shape=[1], - dtype='int64', - force_cpu=True) + counter = fluid.layers.zeros( + shape=[1], dtype='int64', force_cpu=True + ) until = layers.fill_constant([1], dtype='int64', value=10) data_arr = layers.array_write(hidden1, i) cond = fluid.layers.less_than(x=counter, y=until) @@ -58,9 +57,9 @@ def build_program(self, compile_program=True): cost = fluid.layers.cross_entropy(input=predict, label=label) avg_cost = paddle.mean(cost) batch_size = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=predict, - label=label, - total=batch_size) + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size + ) optimizer = fluid.optimizer.Momentum(learning_rate=0.001, momentum=0.9) opts = optimizer.minimize(avg_cost, startup_program=startup_program) @@ -71,8 +70,10 @@ def build_program(self, compile_program=True): exec_strategy = fluid.ExecutionStrategy() exec_strategy.num_threads = 1 train_program = fluid.compiler.CompiledProgram( - main_program).with_data_parallel(loss_name=avg_cost.name, - exec_strategy=exec_strategy) + main_program + ).with_data_parallel( + loss_name=avg_cost.name, exec_strategy=exec_strategy + ) else: train_program = main_program return train_program, startup_program, avg_cost, batch_size, batch_acc @@ -84,41 +85,48 @@ def get_profile_path(self): def check_profile_result(self, profile_path): data = open(profile_path, 'rb').read() - if (len(data) > 0): + if len(data) > 0: profile_pb = profiler_pb2.Profile() profile_pb.ParseFromString(data) self.assertGreater(len(profile_pb.events), 0) for event in profile_pb.events: if event.type == profiler_pb2.Event.GPUKernel: if not event.detail_info and not event.name.startswith( - "MEM"): + "MEM" + ): raise Exception( "Kernel %s missing event. Has this kernel been recorded by RecordEvent?" - % event.name) + % event.name + ) elif event.type == profiler_pb2.Event.CPU and ( - event.name.startswith("Driver API") - or event.name.startswith("Runtime API")): + event.name.startswith("Driver API") + or event.name.startswith("Runtime API") + ): print("Warning: unregister", event.name) def run_iter(self, exe, main_program, fetch_list): x = np.random.random((32, 784)).astype("float32") y = np.random.randint(0, 10, (32, 1)).astype("int64") - outs = exe.run(main_program, - feed={ - 'x': x, - 'y': y - }, - fetch_list=fetch_list) - - def net_profiler(self, - exe, - state, - tracer_option, - batch_range=None, - use_parallel_executor=False, - use_new_api=False): - main_program, startup_program, avg_cost, batch_size, batch_acc = self.build_program( - compile_program=use_parallel_executor) + outs = exe.run( + main_program, feed={'x': x, 'y': y}, fetch_list=fetch_list + ) + + def net_profiler( + self, + exe, + state, + tracer_option, + batch_range=None, + use_parallel_executor=False, + use_new_api=False, + ): + ( + main_program, + startup_program, + avg_cost, + batch_size, + batch_acc, + ) = self.build_program(compile_program=use_parallel_executor) exe.run(startup_program) profile_path = self.get_profile_path() @@ -127,22 +135,26 @@ def net_profiler(self, for iter in range(10): if iter == 2: profiler.reset_profiler() - self.run_iter(exe, main_program, - [avg_cost, batch_acc, batch_size]) + self.run_iter( + exe, main_program, [avg_cost, batch_acc, batch_size] + ) else: options = utils.ProfilerOptions( options={ 'state': state, 'sorted_key': 'total', 'tracer_level': tracer_option, - 'batch_range': - [0, 10] if batch_range is None else batch_range, - 'profile_path': profile_path - }) + 'batch_range': [0, 10] + if batch_range is None + else batch_range, + 'profile_path': profile_path, + } + ) with utils.Profiler(enabled=True, options=options) as prof: for iter in range(10): - self.run_iter(exe, main_program, - [avg_cost, batch_acc, batch_size]) + self.run_iter( + exe, main_program, [avg_cost, batch_acc, batch_size] + ) utils.get_profiler().record_step() if batch_range is None and iter == 2: utils.get_profiler().reset() @@ -153,37 +165,44 @@ def net_profiler(self, def test_cpu_profiler(self): exe = fluid.Executor(fluid.CPUPlace()) for use_new_api in [False, True]: - self.net_profiler(exe, - 'CPU', - "Default", - batch_range=[5, 10], - use_new_api=use_new_api) - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "profiler is enabled only with GPU") + self.net_profiler( + exe, + 'CPU', + "Default", + batch_range=[5, 10], + use_new_api=use_new_api, + ) + + @unittest.skipIf( + not core.is_compiled_with_cuda(), "profiler is enabled only with GPU" + ) def test_cuda_profiler(self): exe = fluid.Executor(fluid.CUDAPlace(0)) for use_new_api in [False, True]: - self.net_profiler(exe, - 'GPU', - "OpDetail", - batch_range=[0, 10], - use_new_api=use_new_api) - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "profiler is enabled only with GPU") + self.net_profiler( + exe, + 'GPU', + "OpDetail", + batch_range=[0, 10], + use_new_api=use_new_api, + ) + + @unittest.skipIf( + not core.is_compiled_with_cuda(), "profiler is enabled only with GPU" + ) def test_all_profiler(self): exe = fluid.Executor(fluid.CUDAPlace(0)) for use_new_api in [False, True]: - self.net_profiler(exe, - 'All', - "AllOpDetail", - batch_range=None, - use_new_api=use_new_api) + self.net_profiler( + exe, + 'All', + "AllOpDetail", + batch_range=None, + use_new_api=use_new_api, + ) class TestProfilerAPIError(unittest.TestCase): - def test_errors(self): options = utils.ProfilerOptions() self.assertTrue(options['profile_path'] is None) diff --git a/python/paddle/fluid/tests/unittests/test_profiler_statistic.py b/python/paddle/fluid/tests/unittests/test_profiler_statistic.py index e6e7a9c8b6209..0483f4995d97a 100644 --- a/python/paddle/fluid/tests/unittests/test_profiler_statistic.py +++ b/python/paddle/fluid/tests/unittests/test_profiler_statistic.py @@ -19,7 +19,6 @@ class HostPythonNode: - def __init__(self, name, type, start_ns, end_ns, process_id, thread_id): self.name = name self.type = type @@ -34,9 +33,9 @@ def __init__(self, name, type, start_ns, end_ns, process_id, thread_id): class DevicePythonNode: - - def __init__(self, name, type, start_ns, end_ns, device_id, context_id, - stream_id): + def __init__( + self, name, type, start_ns, end_ns, device_id, context_id, stream_id + ): self.name = name self.type = type self.start_ns = start_ns @@ -47,8 +46,20 @@ def __init__(self, name, type, start_ns, end_ns, device_id, context_id, class MemPythonNode: - def __init__(self, timestamp_ns, addr, type, process_id, thread_id, increase_bytes, place, current_allocated, \ - current_reserved, peak_allocated, peak_reserved): + def __init__( + self, + timestamp_ns, + addr, + type, + process_id, + thread_id, + increase_bytes, + place, + current_allocated, + current_reserved, + peak_allocated, + peak_reserved, + ): self.timestamp_ns = timestamp_ns self.addr = addr self.type = type @@ -63,155 +74,308 @@ def __init__(self, timestamp_ns, addr, type, process_id, thread_id, increase_byt class TestProfilerStatistic(unittest.TestCase): - def test_statistic_case1(self): - root_node = HostPythonNode('Root Node', - profiler.TracerEventType.UserDefined, 0, - float('inf'), 1000, 1001) - profilerstep_node = HostPythonNode('ProfileStep#1', - profiler.TracerEventType.ProfileStep, - 0, 400, 1000, 1001) - dataloader_node = HostPythonNode('Dataloader', - profiler.TracerEventType.Dataloader, 5, - 15, 1000, 1001) - mobilenet_node = HostPythonNode('MobileNet', - profiler.TracerEventType.Forward, 20, - 50, 1000, 1001) - yolonet_node = HostPythonNode('Yolov3Net', - profiler.TracerEventType.Forward, 50, 110, - 1000, 1001) + root_node = HostPythonNode( + 'Root Node', + profiler.TracerEventType.UserDefined, + 0, + float('inf'), + 1000, + 1001, + ) + profilerstep_node = HostPythonNode( + 'ProfileStep#1', + profiler.TracerEventType.ProfileStep, + 0, + 400, + 1000, + 1001, + ) + dataloader_node = HostPythonNode( + 'Dataloader', profiler.TracerEventType.Dataloader, 5, 15, 1000, 1001 + ) + mobilenet_node = HostPythonNode( + 'MobileNet', profiler.TracerEventType.Forward, 20, 50, 1000, 1001 + ) + yolonet_node = HostPythonNode( + 'Yolov3Net', profiler.TracerEventType.Forward, 50, 110, 1000, 1001 + ) userdefined_node = HostPythonNode( - 'Communication Time', profiler.TracerEventType.PythonUserDefined, - 100, 110, 1000, 1001) + 'Communication Time', + profiler.TracerEventType.PythonUserDefined, + 100, + 110, + 1000, + 1001, + ) communication_node = HostPythonNode( - 'Communication', profiler.TracerEventType.Communication, 105, 110, - 1000, 1001) - backward_node = HostPythonNode('Gradient Backward', - profiler.TracerEventType.Backward, 120, - 200, 1000, 1001) + 'Communication', + profiler.TracerEventType.Communication, + 105, + 110, + 1000, + 1001, + ) + backward_node = HostPythonNode( + 'Gradient Backward', + profiler.TracerEventType.Backward, + 120, + 200, + 1000, + 1001, + ) optimization_node = HostPythonNode( - 'Optimization', profiler.TracerEventType.Optimization, 220, 300, - 1000, 1001) - conv2d_node = HostPythonNode('conv2d', - profiler.TracerEventType.Operator, 25, 40, - 1000, 1001) - sync_batch_norm_node = HostPythonNode('sync_batch_norm', - profiler.TracerEventType.Operator, - 60, 100, 1000, 1001) + 'Optimization', + profiler.TracerEventType.Optimization, + 220, + 300, + 1000, + 1001, + ) + conv2d_node = HostPythonNode( + 'conv2d', profiler.TracerEventType.Operator, 25, 40, 1000, 1001 + ) + sync_batch_norm_node = HostPythonNode( + 'sync_batch_norm', + profiler.TracerEventType.Operator, + 60, + 100, + 1000, + 1001, + ) conv2d_infer_shape = HostPythonNode( - 'conv2d::infer_shape', profiler.TracerEventType.OperatorInner, 25, - 30, 1000, 1001) - conv2d_compute = HostPythonNode('conv2d::compute', - profiler.TracerEventType.OperatorInner, - 30, 40, 1000, 1001) + 'conv2d::infer_shape', + profiler.TracerEventType.OperatorInner, + 25, + 30, + 1000, + 1001, + ) + conv2d_compute = HostPythonNode( + 'conv2d::compute', + profiler.TracerEventType.OperatorInner, + 30, + 40, + 1000, + 1001, + ) conv2d_compute.mem_node.append( - MemPythonNode(33, 0, profiler_statistic.TracerMemEventType.Allocate, - 1000, 1001, 20, 'place(gpu:0)', 200, 200, 800, 800)) + MemPythonNode( + 33, + 0, + profiler_statistic.TracerMemEventType.Allocate, + 1000, + 1001, + 20, + 'place(gpu:0)', + 200, + 200, + 800, + 800, + ) + ) conv2d_launchkernel = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 30, 35, - 1000, 1001) - conv2d_MemCpy = HostPythonNode('AsyncMemcpy', - profiler.TracerEventType.UserDefined, 35, - 40, 1000, 1001) - conv2d_cudaMemCpy = HostPythonNode('cudaMemcpy', - profiler.TracerEventType.CudaRuntime, - 35, 40, 1000, 1001) - conv2d_kernel = DevicePythonNode('conv2d_kernel', - profiler.TracerEventType.Kernel, 35, - 50, 0, 0, 0) - conv2d_memcpy = DevicePythonNode('conv2d_memcpy', - profiler.TracerEventType.Memcpy, 50, - 60, 0, 0, 0) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 30, + 35, + 1000, + 1001, + ) + conv2d_MemCpy = HostPythonNode( + 'AsyncMemcpy', + profiler.TracerEventType.UserDefined, + 35, + 40, + 1000, + 1001, + ) + conv2d_cudaMemCpy = HostPythonNode( + 'cudaMemcpy', + profiler.TracerEventType.CudaRuntime, + 35, + 40, + 1000, + 1001, + ) + conv2d_kernel = DevicePythonNode( + 'conv2d_kernel', profiler.TracerEventType.Kernel, 35, 50, 0, 0, 0 + ) + conv2d_memcpy = DevicePythonNode( + 'conv2d_memcpy', profiler.TracerEventType.Memcpy, 50, 60, 0, 0, 0 + ) sync_batch_norm_infer_shape = HostPythonNode( 'sync_batch_norm::infer_shape', - profiler.TracerEventType.OperatorInner, 60, 70, 1000, 1001) + profiler.TracerEventType.OperatorInner, + 60, + 70, + 1000, + 1001, + ) sync_batch_norm_compute = HostPythonNode( - 'sync_batch_norm::compute', profiler.TracerEventType.OperatorInner, - 80, 100, 1000, 1001) + 'sync_batch_norm::compute', + profiler.TracerEventType.OperatorInner, + 80, + 100, + 1000, + 1001, + ) sync_batch_norm_launchkernel = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 80, 90, - 1000, 1001) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 80, + 90, + 1000, + 1001, + ) sync_batch_norm_MemCpy = HostPythonNode( - 'AsyncMemcpy', profiler.TracerEventType.UserDefined, 90, 100, 1000, - 1001) + 'AsyncMemcpy', + profiler.TracerEventType.UserDefined, + 90, + 100, + 1000, + 1001, + ) sync_batch_norm_cudaMemCpy = HostPythonNode( - 'cudaMemcpy', profiler.TracerEventType.CudaRuntime, 90, 100, 1000, - 1001) + 'cudaMemcpy', + profiler.TracerEventType.CudaRuntime, + 90, + 100, + 1000, + 1001, + ) sync_batch_norm_kernel = DevicePythonNode( - 'sync_batch_norm_kernel', profiler.TracerEventType.Kernel, 95, 155, - 0, 0, 0) + 'sync_batch_norm_kernel', + profiler.TracerEventType.Kernel, + 95, + 155, + 0, + 0, + 0, + ) sync_batch_norm_memcpy = DevicePythonNode( - 'sync_batch_norm_memcpy', profiler.TracerEventType.Memcpy, 150, 200, - 0, 0, 1) + 'sync_batch_norm_memcpy', + profiler.TracerEventType.Memcpy, + 150, + 200, + 0, + 0, + 1, + ) root_node.children_node.append(profilerstep_node) - profilerstep_node.children_node.extend([ - dataloader_node, mobilenet_node, yolonet_node, backward_node, - optimization_node - ]) + profilerstep_node.children_node.extend( + [ + dataloader_node, + mobilenet_node, + yolonet_node, + backward_node, + optimization_node, + ] + ) mobilenet_node.children_node.append(conv2d_node) yolonet_node.children_node.extend( - [sync_batch_norm_node, userdefined_node]) + [sync_batch_norm_node, userdefined_node] + ) userdefined_node.children_node.append(communication_node) conv2d_node.children_node.extend( - [conv2d_infer_shape, conv2d_compute, conv2d_MemCpy]) + [conv2d_infer_shape, conv2d_compute, conv2d_MemCpy] + ) conv2d_compute.runtime_node.append(conv2d_launchkernel) conv2d_MemCpy.runtime_node.append(conv2d_cudaMemCpy) conv2d_launchkernel.device_node.append(conv2d_kernel) conv2d_cudaMemCpy.device_node.append(conv2d_memcpy) - sync_batch_norm_node.children_node.extend([ - sync_batch_norm_infer_shape, sync_batch_norm_compute, - sync_batch_norm_MemCpy - ]) + sync_batch_norm_node.children_node.extend( + [ + sync_batch_norm_infer_shape, + sync_batch_norm_compute, + sync_batch_norm_MemCpy, + ] + ) sync_batch_norm_compute.runtime_node.append( - sync_batch_norm_launchkernel) + sync_batch_norm_launchkernel + ) sync_batch_norm_MemCpy.runtime_node.append(sync_batch_norm_cudaMemCpy) sync_batch_norm_launchkernel.device_node.append(sync_batch_norm_kernel) sync_batch_norm_cudaMemCpy.device_node.append(sync_batch_norm_memcpy) thread_tree = {'thread1001': root_node} extra_info = { 'Process Cpu Utilization': '1.02', - 'System Cpu Utilization': '0.68' + 'System Cpu Utilization': '0.68', } statistic_data = profiler.profiler_statistic.StatisticData( - thread_tree, extra_info) + thread_tree, extra_info + ) time_range_summary = statistic_data.time_range_summary event_summary = statistic_data.event_summary self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.ProfileStep), 400) + profiler.TracerEventType.ProfileStep + ), + 400, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Forward), 90) + profiler.TracerEventType.Forward + ), + 90, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Backward), 80) + profiler.TracerEventType.Backward + ), + 80, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Optimization), 80) + profiler.TracerEventType.Optimization + ), + 80, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Operator), 55) + profiler.TracerEventType.Operator + ), + 55, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.OperatorInner), 45) + profiler.TracerEventType.OperatorInner + ), + 45, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.CudaRuntime), 30) + profiler.TracerEventType.CudaRuntime + ), + 30, + ) self.assertEqual( time_range_summary.get_gpu_range_sum( - 0, profiler.TracerEventType.Kernel), 75) + 0, profiler.TracerEventType.Kernel + ), + 75, + ) self.assertEqual( time_range_summary.get_gpu_range_sum( - 0, profiler.TracerEventType.Memcpy), 60) + 0, profiler.TracerEventType.Memcpy + ), + 60, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.UserDefined), 15) + profiler.TracerEventType.UserDefined + ), + 15, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Communication), 5) + profiler.TracerEventType.Communication + ), + 5, + ) self.assertEqual(len(event_summary.items), 2) self.assertEqual(len(event_summary.userdefined_items), 1) self.assertEqual(len(event_summary.model_perspective_items), 5) @@ -219,171 +383,347 @@ def test_statistic_case1(self): self.assertEqual(event_summary.items['conv2d'].cpu_time, 15) self.assertEqual(event_summary.items['conv2d'].general_gpu_time, 25) self.assertEqual( - event_summary.model_perspective_items['Forward'].cpu_time, 90) + event_summary.model_perspective_items['Forward'].cpu_time, 90 + ) self.assertEqual( event_summary.model_perspective_items['Forward'].general_gpu_time, - 135) + 135, + ) self.assertEqual( event_summary.model_perspective_items['Backward'].general_gpu_time, - 0) + 0, + ) self.assertEqual( - event_summary.memory_manipulation_items['AsyncMemcpy'].cpu_time, 15) + event_summary.memory_manipulation_items['AsyncMemcpy'].cpu_time, 15 + ) self.assertEqual( - event_summary.memory_manipulation_items['AsyncMemcpy']. - general_gpu_time, 60) + event_summary.memory_manipulation_items[ + 'AsyncMemcpy' + ].general_gpu_time, + 60, + ) self.assertEqual( - statistic_data.memory_summary.allocated_items['place(gpu:0)'] - ['conv2d'].allocation_count, 1) + statistic_data.memory_summary.allocated_items['place(gpu:0)'][ + 'conv2d' + ].allocation_count, + 1, + ) self.assertEqual( - statistic_data.memory_summary.allocated_items['place(gpu:0)'] - ['conv2d'].allocation_size, 20) + statistic_data.memory_summary.allocated_items['place(gpu:0)'][ + 'conv2d' + ].allocation_size, + 20, + ) self.assertEqual( - statistic_data.memory_summary.allocated_items['place(gpu:0)'] - ['conv2d'].increase_size, 20) + statistic_data.memory_summary.allocated_items['place(gpu:0)'][ + 'conv2d' + ].increase_size, + 20, + ) self.assertEqual( - statistic_data.memory_summary.allocated_items['place(gpu:0)'] - ['conv2d'].increase_size, 20) + statistic_data.memory_summary.allocated_items['place(gpu:0)'][ + 'conv2d' + ].increase_size, + 20, + ) self.assertEqual( - statistic_data.memory_summary. - peak_allocation_values['place(gpu:0)'], 800) + statistic_data.memory_summary.peak_allocation_values[ + 'place(gpu:0)' + ], + 800, + ) self.assertEqual( statistic_data.memory_summary.peak_reserved_values['place(gpu:0)'], - 800) + 800, + ) print( profiler.profiler_statistic._build_table( statistic_data, sorted_by=profiler.SortedKeys.CPUTotal, op_detail=True, thread_sep=False, - time_unit='ms')) + time_unit='ms', + ) + ) def test_statistic_case2(self): - root_node = HostPythonNode('Root Node', - profiler.TracerEventType.UserDefined, 0, - float('inf'), 1000, 1001) - profilerstep_node = HostPythonNode('ProfileStep#1', - profiler.TracerEventType.ProfileStep, - 0, 400, 1000, 1001) - - dataloader_node = HostPythonNode('Dataloader', - profiler.TracerEventType.Dataloader, 5, - 15, 1000, 1001) - - mobilenet_node = HostPythonNode('MobileNet', - profiler.TracerEventType.Forward, 20, - 50, 1000, 1001) - yolonet_node = HostPythonNode('Yolov3Net', - profiler.TracerEventType.Forward, 50, 110, - 1000, 1001) + root_node = HostPythonNode( + 'Root Node', + profiler.TracerEventType.UserDefined, + 0, + float('inf'), + 1000, + 1001, + ) + profilerstep_node = HostPythonNode( + 'ProfileStep#1', + profiler.TracerEventType.ProfileStep, + 0, + 400, + 1000, + 1001, + ) + + dataloader_node = HostPythonNode( + 'Dataloader', profiler.TracerEventType.Dataloader, 5, 15, 1000, 1001 + ) + + mobilenet_node = HostPythonNode( + 'MobileNet', profiler.TracerEventType.Forward, 20, 50, 1000, 1001 + ) + yolonet_node = HostPythonNode( + 'Yolov3Net', profiler.TracerEventType.Forward, 50, 110, 1000, 1001 + ) userdefined_node = HostPythonNode( - 'Communication Time', profiler.TracerEventType.PythonUserDefined, - 100, 110, 1000, 1001) + 'Communication Time', + profiler.TracerEventType.PythonUserDefined, + 100, + 110, + 1000, + 1001, + ) allreduce_launchkernel0 = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 102, 104, - 1000, 1001) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 102, + 104, + 1000, + 1001, + ) nccl_allreduce_kernel0 = DevicePythonNode( - 'nccl_allreduce_kernel', profiler.TracerEventType.Kernel, 105, 120, - 0, 0, 2) + 'nccl_allreduce_kernel', + profiler.TracerEventType.Kernel, + 105, + 120, + 0, + 0, + 2, + ) communication_node = HostPythonNode( - 'Communication', profiler.TracerEventType.Communication, 105, 110, - 1000, 1001) + 'Communication', + profiler.TracerEventType.Communication, + 105, + 110, + 1000, + 1001, + ) - allreduce_op1 = HostPythonNode('allreduce_op1', - profiler.TracerEventType.Operator, 105, - 108, 1000, 1001) + allreduce_op1 = HostPythonNode( + 'allreduce_op1', + profiler.TracerEventType.Operator, + 105, + 108, + 1000, + 1001, + ) allreduce_op1_infershape = HostPythonNode( - 'allreduce_op1::infershape', profiler.TracerEventType.OperatorInner, - 105, 106, 1000, 1001) + 'allreduce_op1::infershape', + profiler.TracerEventType.OperatorInner, + 105, + 106, + 1000, + 1001, + ) allreduce_launchkernel1 = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 106, 107, - 1000, 1001) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 106, + 107, + 1000, + 1001, + ) nccl_allreduce_kernel1 = DevicePythonNode( - 'nccl_allreduce_kernel', profiler.TracerEventType.Kernel, 130, 150, - 0, 0, 2) + 'nccl_allreduce_kernel', + profiler.TracerEventType.Kernel, + 130, + 150, + 0, + 0, + 2, + ) - backward_node = HostPythonNode('Gradient Backward', - profiler.TracerEventType.Backward, 120, - 200, 1000, 1001) + backward_node = HostPythonNode( + 'Gradient Backward', + profiler.TracerEventType.Backward, + 120, + 200, + 1000, + 1001, + ) optimization_node = HostPythonNode( - 'Optimization', profiler.TracerEventType.Optimization, 220, 300, - 1000, 1001) - conv2d_node = HostPythonNode('conv2d', - profiler.TracerEventType.Operator, 25, 40, - 1000, 1001) - sync_batch_norm_node = HostPythonNode('sync_batch_norm', - profiler.TracerEventType.Operator, - 60, 100, 1000, 1001) + 'Optimization', + profiler.TracerEventType.Optimization, + 220, + 300, + 1000, + 1001, + ) + conv2d_node = HostPythonNode( + 'conv2d', profiler.TracerEventType.Operator, 25, 40, 1000, 1001 + ) + sync_batch_norm_node = HostPythonNode( + 'sync_batch_norm', + profiler.TracerEventType.Operator, + 60, + 100, + 1000, + 1001, + ) conv2d_infer_shape = HostPythonNode( - 'conv2d::infer_shape', profiler.TracerEventType.OperatorInner, 25, - 30, 1000, 1001) - conv2d_compute = HostPythonNode('conv2d::compute', - profiler.TracerEventType.OperatorInner, - 30, 40, 1000, 1001) + 'conv2d::infer_shape', + profiler.TracerEventType.OperatorInner, + 25, + 30, + 1000, + 1001, + ) + conv2d_compute = HostPythonNode( + 'conv2d::compute', + profiler.TracerEventType.OperatorInner, + 30, + 40, + 1000, + 1001, + ) conv2d_launchkernel = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 30, 35, - 1000, 1001) - conv2d_MemCpy = HostPythonNode('AsyncMemcpy', - profiler.TracerEventType.UserDefined, 35, - 40, 1000, 1001) - conv2d_cudaMemCpy = HostPythonNode('cudaMemcpy', - profiler.TracerEventType.CudaRuntime, - 35, 40, 1000, 1001) - conv2d_kernel = DevicePythonNode('conv2d_kernel', - profiler.TracerEventType.Kernel, 35, - 50, 0, 0, 0) - conv2d_memcpy = DevicePythonNode('conv2d_memcpy', - profiler.TracerEventType.Memcpy, 50, - 60, 0, 0, 0) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 30, + 35, + 1000, + 1001, + ) + conv2d_MemCpy = HostPythonNode( + 'AsyncMemcpy', + profiler.TracerEventType.UserDefined, + 35, + 40, + 1000, + 1001, + ) + conv2d_cudaMemCpy = HostPythonNode( + 'cudaMemcpy', + profiler.TracerEventType.CudaRuntime, + 35, + 40, + 1000, + 1001, + ) + conv2d_kernel = DevicePythonNode( + 'conv2d_kernel', profiler.TracerEventType.Kernel, 35, 50, 0, 0, 0 + ) + conv2d_memcpy = DevicePythonNode( + 'conv2d_memcpy', profiler.TracerEventType.Memcpy, 50, 60, 0, 0, 0 + ) sync_batch_norm_infer_shape = HostPythonNode( 'sync_batch_norm::infer_shape', - profiler.TracerEventType.OperatorInner, 60, 70, 1000, 1001) + profiler.TracerEventType.OperatorInner, + 60, + 70, + 1000, + 1001, + ) sync_batch_norm_compute = HostPythonNode( - 'sync_batch_norm::compute', profiler.TracerEventType.OperatorInner, - 80, 100, 1000, 1001) + 'sync_batch_norm::compute', + profiler.TracerEventType.OperatorInner, + 80, + 100, + 1000, + 1001, + ) sync_batch_norm_launchkernel = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 80, 90, - 1000, 1001) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 80, + 90, + 1000, + 1001, + ) sync_batch_norm_MemCpy = HostPythonNode( - 'AsyncMemcpy', profiler.TracerEventType.UserDefined, 90, 100, 1000, - 1001) + 'AsyncMemcpy', + profiler.TracerEventType.UserDefined, + 90, + 100, + 1000, + 1001, + ) sync_batch_norm_cudaMemCpy = HostPythonNode( - 'cudaMemcpy', profiler.TracerEventType.CudaRuntime, 90, 100, 1000, - 1001) + 'cudaMemcpy', + profiler.TracerEventType.CudaRuntime, + 90, + 100, + 1000, + 1001, + ) sync_batch_norm_kernel = DevicePythonNode( - 'sync_batch_norm_kernel', profiler.TracerEventType.Kernel, 95, 300, - 0, 0, 0) + 'sync_batch_norm_kernel', + profiler.TracerEventType.Kernel, + 95, + 300, + 0, + 0, + 0, + ) sync_batch_norm_memcpy = DevicePythonNode( - 'sync_batch_norm_memcpy', profiler.TracerEventType.Memcpy, 150, 200, - 0, 0, 1) + 'sync_batch_norm_memcpy', + profiler.TracerEventType.Memcpy, + 150, + 200, + 0, + 0, + 1, + ) - allreduce_node2 = HostPythonNode('allreduce', - profiler.TracerEventType.Operator, 230, - 250, 1000, 1001) + allreduce_node2 = HostPythonNode( + 'allreduce', profiler.TracerEventType.Operator, 230, 250, 1000, 1001 + ) allreduce_node2_infershape = HostPythonNode( 'allreduce_node2::infershape', - profiler.TracerEventType.OperatorInner, 231, 232, 1000, 1001) + profiler.TracerEventType.OperatorInner, + 231, + 232, + 1000, + 1001, + ) allreduce_launchkernel2 = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 235, 240, - 1000, 1001) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 235, + 240, + 1000, + 1001, + ) nccl_allreduce_kernel2 = DevicePythonNode( - 'nccl_allreduce_kernel', profiler.TracerEventType.Kernel, 250, 280, - 0, 0, 2) + 'nccl_allreduce_kernel', + profiler.TracerEventType.Kernel, + 250, + 280, + 0, + 0, + 2, + ) root_node.children_node.append(profilerstep_node) - profilerstep_node.children_node.extend([ - dataloader_node, mobilenet_node, yolonet_node, backward_node, - optimization_node - ]) + profilerstep_node.children_node.extend( + [ + dataloader_node, + mobilenet_node, + yolonet_node, + backward_node, + optimization_node, + ] + ) mobilenet_node.children_node.append(conv2d_node) yolonet_node.children_node.extend( - [sync_batch_norm_node, userdefined_node]) + [sync_batch_norm_node, userdefined_node] + ) userdefined_node.children_node.append(communication_node) userdefined_node.runtime_node.append(allreduce_launchkernel0) allreduce_launchkernel0.device_node.append(nccl_allreduce_kernel0) @@ -392,17 +732,22 @@ def test_statistic_case2(self): allreduce_op1.runtime_node.append(allreduce_launchkernel1) allreduce_launchkernel1.device_node.append(nccl_allreduce_kernel1) conv2d_node.children_node.extend( - [conv2d_infer_shape, conv2d_compute, conv2d_MemCpy]) + [conv2d_infer_shape, conv2d_compute, conv2d_MemCpy] + ) conv2d_compute.runtime_node.append(conv2d_launchkernel) conv2d_MemCpy.runtime_node.append(conv2d_cudaMemCpy) conv2d_launchkernel.device_node.append(conv2d_kernel) conv2d_cudaMemCpy.device_node.append(conv2d_memcpy) - sync_batch_norm_node.children_node.extend([ - sync_batch_norm_infer_shape, sync_batch_norm_compute, - sync_batch_norm_MemCpy - ]) + sync_batch_norm_node.children_node.extend( + [ + sync_batch_norm_infer_shape, + sync_batch_norm_compute, + sync_batch_norm_MemCpy, + ] + ) sync_batch_norm_compute.runtime_node.append( - sync_batch_norm_launchkernel) + sync_batch_norm_launchkernel + ) sync_batch_norm_MemCpy.runtime_node.append(sync_batch_norm_cudaMemCpy) sync_batch_norm_launchkernel.device_node.append(sync_batch_norm_kernel) sync_batch_norm_cudaMemCpy.device_node.append(sync_batch_norm_memcpy) @@ -413,62 +758,111 @@ def test_statistic_case2(self): thread_tree = {'thread1001': root_node} extra_info = { 'Process Cpu Utilization': '1.02', - 'System Cpu Utilization': '0.68' + 'System Cpu Utilization': '0.68', } statistic_data = profiler.profiler_statistic.StatisticData( - thread_tree, extra_info) + thread_tree, extra_info + ) time_range_summary = statistic_data.time_range_summary event_summary = statistic_data.event_summary distributed_summary = statistic_data.distributed_summary self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.ProfileStep), 400) + profiler.TracerEventType.ProfileStep + ), + 400, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Forward), 90) + profiler.TracerEventType.Forward + ), + 90, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Backward), 80) + profiler.TracerEventType.Backward + ), + 80, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Optimization), 80) + profiler.TracerEventType.Optimization + ), + 80, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Operator), 78) + profiler.TracerEventType.Operator + ), + 78, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.OperatorInner), 47) + profiler.TracerEventType.OperatorInner + ), + 47, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.CudaRuntime), 38) + profiler.TracerEventType.CudaRuntime + ), + 38, + ) self.assertEqual( time_range_summary.get_gpu_range_sum( - 0, profiler.TracerEventType.Kernel), 220) + 0, profiler.TracerEventType.Kernel + ), + 220, + ) self.assertEqual( time_range_summary.get_gpu_range_sum( - 0, profiler.TracerEventType.Memcpy), 60) + 0, profiler.TracerEventType.Memcpy + ), + 60, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.UserDefined), 15) + profiler.TracerEventType.UserDefined + ), + 15, + ) self.assertEqual( time_range_summary.get_cpu_range_sum( - profiler.TracerEventType.Communication), 5) + profiler.TracerEventType.Communication + ), + 5, + ) self.assertEqual( profiler.statistic_helper.sum_ranges( - distributed_summary.cpu_communication_range), 25) + distributed_summary.cpu_communication_range + ), + 25, + ) self.assertEqual( profiler.statistic_helper.sum_ranges( - distributed_summary.gpu_communication_range), 65) + distributed_summary.gpu_communication_range + ), + 65, + ) self.assertEqual( profiler.statistic_helper.sum_ranges( - distributed_summary.communication_range), 85) + distributed_summary.communication_range + ), + 85, + ) self.assertEqual( profiler.statistic_helper.sum_ranges( - distributed_summary.computation_range), 220) + distributed_summary.computation_range + ), + 220, + ) self.assertEqual( profiler.statistic_helper.sum_ranges( - distributed_summary.overlap_range), 85) + distributed_summary.overlap_range + ), + 85, + ) self.assertEqual(len(event_summary.items), 4) self.assertEqual(len(event_summary.userdefined_items), 1) self.assertEqual(len(event_summary.model_perspective_items), 5) @@ -476,76 +870,136 @@ def test_statistic_case2(self): self.assertEqual(event_summary.items['conv2d'].cpu_time, 15) self.assertEqual(event_summary.items['conv2d'].general_gpu_time, 25) self.assertEqual( - event_summary.model_perspective_items['Forward'].cpu_time, 90) + event_summary.model_perspective_items['Forward'].cpu_time, 90 + ) self.assertEqual( event_summary.model_perspective_items['Forward'].general_gpu_time, - 315) + 315, + ) self.assertEqual( event_summary.model_perspective_items['Backward'].general_gpu_time, - 0) + 0, + ) self.assertEqual( - event_summary.memory_manipulation_items['AsyncMemcpy'].cpu_time, 15) + event_summary.memory_manipulation_items['AsyncMemcpy'].cpu_time, 15 + ) self.assertEqual( - event_summary.memory_manipulation_items['AsyncMemcpy']. - general_gpu_time, 60) + event_summary.memory_manipulation_items[ + 'AsyncMemcpy' + ].general_gpu_time, + 60, + ) print( profiler.profiler_statistic._build_table( statistic_data, sorted_by=profiler.SortedKeys.CPUTotal, op_detail=True, thread_sep=False, - time_unit='ms')) + time_unit='ms', + ) + ) def test_statistic_case3(self): # for coverage, test all time is 0 - root_node = HostPythonNode('Root Node', - profiler.TracerEventType.UserDefined, 0, - float('inf'), 1000, 1001) - profilerstep_node = HostPythonNode('ProfileStep#1', - profiler.TracerEventType.ProfileStep, - 0, 400, 1000, 1001) - dataloader_node = HostPythonNode('Dataloader', - profiler.TracerEventType.Dataloader, 5, - 15, 1000, 1001) - mobilenet_node = HostPythonNode('MobileNet', - profiler.TracerEventType.Forward, 20, - 50, 1000, 1001) - - backward_node = HostPythonNode('Gradient Backward', - profiler.TracerEventType.Backward, 120, - 200, 1000, 1001) + root_node = HostPythonNode( + 'Root Node', + profiler.TracerEventType.UserDefined, + 0, + float('inf'), + 1000, + 1001, + ) + profilerstep_node = HostPythonNode( + 'ProfileStep#1', + profiler.TracerEventType.ProfileStep, + 0, + 400, + 1000, + 1001, + ) + dataloader_node = HostPythonNode( + 'Dataloader', profiler.TracerEventType.Dataloader, 5, 15, 1000, 1001 + ) + mobilenet_node = HostPythonNode( + 'MobileNet', profiler.TracerEventType.Forward, 20, 50, 1000, 1001 + ) + + backward_node = HostPythonNode( + 'Gradient Backward', + profiler.TracerEventType.Backward, + 120, + 200, + 1000, + 1001, + ) optimization_node = HostPythonNode( - 'Optimization', profiler.TracerEventType.Optimization, 220, 300, - 1000, 1001) + 'Optimization', + profiler.TracerEventType.Optimization, + 220, + 300, + 1000, + 1001, + ) userdefined_node = HostPythonNode( - 'Communication Time', profiler.TracerEventType.PythonUserDefined, - 60, 70, 1000, 1001) + 'Communication Time', + profiler.TracerEventType.PythonUserDefined, + 60, + 70, + 1000, + 1001, + ) - conv2d_node = HostPythonNode('conv2d', - profiler.TracerEventType.Operator, 25, 25, - 1000, 1001) + conv2d_node = HostPythonNode( + 'conv2d', profiler.TracerEventType.Operator, 25, 25, 1000, 1001 + ) conv2d_infer_shape = HostPythonNode( - 'conv2d::infer_shape', profiler.TracerEventType.OperatorInner, 25, - 25, 1000, 1001) - conv2d_compute = HostPythonNode('conv2d::compute', - profiler.TracerEventType.OperatorInner, - 25, 25, 1000, 1001) + 'conv2d::infer_shape', + profiler.TracerEventType.OperatorInner, + 25, + 25, + 1000, + 1001, + ) + conv2d_compute = HostPythonNode( + 'conv2d::compute', + profiler.TracerEventType.OperatorInner, + 25, + 25, + 1000, + 1001, + ) conv2d_launchkernel = HostPythonNode( - 'cudalaunchkernel', profiler.TracerEventType.CudaRuntime, 25, 25, - 1000, 1001) + 'cudalaunchkernel', + profiler.TracerEventType.CudaRuntime, + 25, + 25, + 1000, + 1001, + ) - conv2d_kernel = DevicePythonNode('conv2d_kernel', - profiler.TracerEventType.Kernel, 35, - 35, 0, 0, 0) + conv2d_kernel = DevicePythonNode( + 'conv2d_kernel', profiler.TracerEventType.Kernel, 35, 35, 0, 0, 0 + ) another_kernel = DevicePythonNode( 'void phi::funcs::VectorizedBroadcastKernel, phi::funcs::AddFunctor>()', - profiler.TracerEventType.Kernel, 35, 35, 0, 0, 0) + profiler.TracerEventType.Kernel, + 35, + 35, + 0, + 0, + 0, + ) root_node.children_node.append(profilerstep_node) - profilerstep_node.children_node.extend([ - dataloader_node, mobilenet_node, userdefined_node, backward_node, - optimization_node - ]) + profilerstep_node.children_node.extend( + [ + dataloader_node, + mobilenet_node, + userdefined_node, + backward_node, + optimization_node, + ] + ) mobilenet_node.children_node.append(conv2d_node) conv2d_node.children_node.extend([conv2d_infer_shape, conv2d_compute]) conv2d_compute.runtime_node.append(conv2d_launchkernel) @@ -554,30 +1008,41 @@ def test_statistic_case3(self): thread_tree = {'thread1001': root_node} extra_info = { 'Process Cpu Utilization': '1.02', - 'System Cpu Utilization': '0.68' + 'System Cpu Utilization': '0.68', } statistic_data = profiler.profiler_statistic.StatisticData( - thread_tree, extra_info) + thread_tree, extra_info + ) time_range_summary = statistic_data.time_range_summary event_summary = statistic_data.event_summary self.assertEqual(event_summary.items['conv2d'].cpu_time, 0) self.assertEqual(event_summary.items['conv2d'].general_gpu_time, 0) self.assertEqual( - event_summary.userdefined_items['Communication Time']. - general_gpu_time, 0) + event_summary.userdefined_items[ + 'Communication Time' + ].general_gpu_time, + 0, + ) for sort_key in [ - profiler.SortedKeys.CPUTotal, profiler.SortedKeys.CPUMax, - profiler.SortedKeys.CPUMin, profiler.SortedKeys.CPUAvg, - profiler.SortedKeys.GPUTotal, profiler.SortedKeys.GPUMax, - profiler.SortedKeys.GPUMin, profiler.SortedKeys.GPUAvg + profiler.SortedKeys.CPUTotal, + profiler.SortedKeys.CPUMax, + profiler.SortedKeys.CPUMin, + profiler.SortedKeys.CPUAvg, + profiler.SortedKeys.GPUTotal, + profiler.SortedKeys.GPUMax, + profiler.SortedKeys.GPUMin, + profiler.SortedKeys.GPUAvg, ]: print( - profiler.profiler_statistic._build_table(statistic_data, - sorted_by=sort_key, - op_detail=True, - thread_sep=False, - time_unit='ms')) + profiler.profiler_statistic._build_table( + statistic_data, + sorted_by=sort_key, + op_detail=True, + thread_sep=False, + time_unit='ms', + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_program.py b/python/paddle/fluid/tests/unittests/test_program.py index 71c1e9239c04f..a83574b10013f 100644 --- a/python/paddle/fluid/tests/unittests/test_program.py +++ b/python/paddle/fluid/tests/unittests/test_program.py @@ -23,7 +23,6 @@ class TestProgram(unittest.TestCase): - def test_program(self): b = main_program.current_block() self.assertEqual(-1, b.parent_idx) @@ -55,20 +54,17 @@ def test_program(self): def test_program_clone(self): prog = Program() - x = prog.global_block().create_var(name='X', - shape=[1000, 784], - dtype='float32') + x = prog.global_block().create_var( + name='X', shape=[1000, 784], dtype='float32' + ) - y = prog.global_block().create_var(name='Y', - shape=[784, 100], - dtype='float32') + y = prog.global_block().create_var( + name='Y', shape=[784, 100], dtype='float32' + ) out = prog.global_block().create_var(name='Out', dtype='float32') - prog.global_block().append_op(type="mul", - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + prog.global_block().append_op( + type="mul", inputs={'X': [x], 'Y': [y]}, outputs={'Out': [out]} + ) # FIXME(yuyang18): We manual compare the output string, since the order # of variable could be changed. @@ -78,20 +74,17 @@ def test_program_clone(self): def test_parse_program_from_string(self): prog = Program() - x = prog.global_block().create_var(name='X', - shape=[1000, 784], - dtype='float32') + x = prog.global_block().create_var( + name='X', shape=[1000, 784], dtype='float32' + ) - y = prog.global_block().create_var(name='Y', - shape=[784, 100], - dtype='float32') + y = prog.global_block().create_var( + name='Y', shape=[784, 100], dtype='float32' + ) out = prog.global_block().create_var(name='Out', dtype='float32') - prog.global_block().append_op(type="mul", - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + prog.global_block().append_op( + type="mul", inputs={'X': [x], 'Y': [y]}, outputs={'Out': [out]} + ) binary_str = prog.desc.serialize_to_string() prog_restored = Program.parse_from_string(binary_str) @@ -111,17 +104,19 @@ def test_program_clone_with_parameter(self): self.assertNotEqual(0, len(new_program.blocks[0].all_parameters())) def test_program_inference_optimize(self): - def net(): - reader = fluid.layers.py_reader(capacity=10, - shapes=[[-1, 10], [-1, 1]], - lod_levels=[0, 0], - dtypes=['float32', 'int64'], - use_double_buffer=True) + reader = fluid.layers.py_reader( + capacity=10, + shapes=[[-1, 10], [-1, 1]], + lod_levels=[0, 0], + dtypes=['float32', 'int64'], + use_double_buffer=True, + ) in_data, label = fluid.layers.read_file(reader) predict_label = fluid.layers.fc(in_data, size=2, act='softmax') loss = paddle.mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) @@ -132,7 +127,8 @@ def net(): net() no_read_program = main_program._inference_optimize() keep_read_program = main_program._inference_optimize( - prune_read_op=False) + prune_read_op=False + ) no_read_ops = no_read_program.global_block().ops keep_read_ops = keep_read_program.global_block().ops self.assertEqual(len(keep_read_ops) - len(no_read_ops), 2) @@ -158,8 +154,9 @@ def test_program_all_parameters(self): def test_prune_with_input_type_error(self): program = fluid.default_main_program() feed_var_names = [2, 3, 4] - self.assertRaises(ValueError, program._prune_with_input, feed_var_names, - []) + self.assertRaises( + ValueError, program._prune_with_input, feed_var_names, [] + ) def test_random_seed_error(self): program = fluid.default_main_program() @@ -169,21 +166,24 @@ def test_random_seed_error(self): def test_copy_info_from_error(self): program = fluid.default_main_program() self.assertRaises(TypeError, program._copy_param_info_from, "program") - self.assertRaises(TypeError, program._copy_dist_param_info_from, - "program") + self.assertRaises( + TypeError, program._copy_dist_param_info_from, "program" + ) def test_remove_training_info(self): - def net(): - reader = fluid.layers.py_reader(capacity=10, - shapes=[[-1, 10], [-1, 1]], - lod_levels=[0, 0], - dtypes=['float32', 'int64'], - use_double_buffer=True) + reader = fluid.layers.py_reader( + capacity=10, + shapes=[[-1, 10], [-1, 1]], + lod_levels=[0, 0], + dtypes=['float32', 'int64'], + use_double_buffer=True, + ) in_data, label = fluid.layers.read_file(reader) predict_label = fluid.layers.fc(in_data, size=2, act='softmax') loss = paddle.mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) @@ -212,7 +212,6 @@ def build_program(): class TestProgramProto(unittest.TestCase): - def test_update_op(self): program = build_program() a = program.desc.serialize_to_string() @@ -241,7 +240,6 @@ def test_update_var_attr(self): class TestProgramHash(unittest.TestCase): - def build_program(self): main_program = paddle.static.Program() startuo_program = paddle.static.Program() @@ -264,8 +262,8 @@ def test_program_hash_equal(self): program1, program2 = programs[0], programs[1] # why not write as below? # since the callstack attribute are not equal - #program1 = self.build_program() - #program2 = self.build_program() + # program1 = self.build_program() + # program2 = self.build_program() self.assertTrue(program1.desc.need_update()) self.assertTrue(program2.desc.need_update()) @@ -273,7 +271,8 @@ def test_program_hash_equal(self): self.assertFalse(id(program1) == id(program2)) # print(program1, program2) self.assertTrue( - program1.desc.cached_hash_str() == program2.desc.cached_hash_str()) + program1.desc.cached_hash_str() == program2.desc.cached_hash_str() + ) self.assertFalse(program1.desc.need_update()) self.assertFalse(program2.desc.need_update()) @@ -283,8 +282,10 @@ def test_program_clone(self): program_clone = program.clone() self.assertFalse(id(program) == id(program_clone)) - self.assertTrue(program.desc.cached_hash_str() == - program_clone.desc.cached_hash_str()) + self.assertTrue( + program.desc.cached_hash_str() + == program_clone.desc.cached_hash_str() + ) def test_program_update(self): program = self.build_program() diff --git a/python/paddle/fluid/tests/unittests/test_program_code.py b/python/paddle/fluid/tests/unittests/test_program_code.py index 8bdb084a5f4cd..449f97c22f030 100644 --- a/python/paddle/fluid/tests/unittests/test_program_code.py +++ b/python/paddle/fluid/tests/unittests/test_program_code.py @@ -19,25 +19,22 @@ class TestProgramToReadableCode(unittest.TestCase): - def setUp(self): self.program = fluid.Program() self.block = self.program.current_block() - self.var = self.block.create_var(name="X", - shape=[-1, 23, 48], - dtype='float32') - self.param = self.block.create_parameter(name="W", - shape=[23, 48], - dtype='float32', - trainable=True) - self.op = self.block.append_op(type="abs", - inputs={"X": [self.var]}, - outputs={"Out": [self.var]}) + self.var = self.block.create_var( + name="X", shape=[-1, 23, 48], dtype='float32' + ) + self.param = self.block.create_parameter( + name="W", shape=[23, 48], dtype='float32', trainable=True + ) + self.op = self.block.append_op( + type="abs", inputs={"X": [self.var]}, outputs={"Out": [self.var]} + ) # add control flow op and sub block self.append_cond_op(self.program) def append_cond_op(self, program): - def true_func(): return layers.fill_constant(shape=[2, 3], dtype='int32', value=2) diff --git a/python/paddle/fluid/tests/unittests/test_program_prune_backward.py b/python/paddle/fluid/tests/unittests/test_program_prune_backward.py index 1fb423bc1f1fe..ad7617654d275 100755 --- a/python/paddle/fluid/tests/unittests/test_program_prune_backward.py +++ b/python/paddle/fluid/tests/unittests/test_program_prune_backward.py @@ -20,7 +20,11 @@ import paddle.fluid.core as core from simple_nets import init_data, simple_fc_net, fc_with_batchnorm import seresnext_net -from test_parallel_executor_transformer import transformer, get_feed_data_reader, DeviceType +from test_parallel_executor_transformer import ( + transformer, + get_feed_data_reader, + DeviceType, +) from fake_reader import fake_imdb_reader import paddle @@ -32,19 +36,19 @@ def lstm_net(use_feed): hid_dim2 = 96 class_dim = 2 emb_lr = 30.0 - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") emb = fluid.layers.embedding( input=data, size=[dict_dim, emb_dim], - param_attr=fluid.ParamAttr(learning_rate=emb_lr)) + param_attr=fluid.ParamAttr(learning_rate=emb_lr), + ) fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4) - lstm_h, c = fluid.layers.dynamic_lstm(input=fc0, - size=hid_dim * 4, - is_reverse=False) + lstm_h, c = fluid.layers.dynamic_lstm( + input=fc0, size=hid_dim * 4, is_reverse=False + ) lstm_max = fluid.layers.sequence_pool(input=lstm_h, pool_type='max') lstm_max_tanh = fluid.layers.tanh(lstm_max) fc1 = fluid.layers.fc(input=lstm_max_tanh, size=hid_dim2, act='tanh') @@ -64,8 +68,10 @@ def simple_fc_net_with_accuracy(use_feed): hidden, size=200, act='relu', - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) prediction = fluid.layers.fc(hidden, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) loss = paddle.mean(loss) @@ -90,9 +96,11 @@ def loss2(pred, label): return avg_loss two = fluid.layers.fill_constant([1], 'int32', 2) - pred = (two == 0) - avg_loss = fluid.layers.case([(pred, lambda: loss1(prediction, label))], - lambda: loss2(prediction, label)) + pred = two == 0 + avg_loss = fluid.layers.case( + [(pred, lambda: loss1(prediction, label))], + lambda: loss2(prediction, label), + ) return avg_loss @@ -118,15 +126,15 @@ def loss2(opt, pred, label, with_optimize): sgd = fluid.optimizer.SGD(learning_rate=0.1) two = fluid.layers.fill_constant([1], 'int32', 2) - pred = (two == 0) + pred = two == 0 avg_loss = fluid.layers.case( [(pred, lambda: loss1(sgd, prediction, label, with_optimize))], - lambda: loss2(sgd, prediction, label, with_optimize)) + lambda: loss2(sgd, prediction, label, with_optimize), + ) return avg_loss class TestProgramPruneBackward(unittest.TestCase): - def program_compare(self, program_a, program_b): assert isinstance( program_a, fluid.framework.Program @@ -142,8 +150,9 @@ def program_compare(self, program_a, program_b): self.assertEqual(len(block_a.ops), len(block_b.ops)) self.assertEqual(len(block_a.vars), len(block_b.vars)) for op_idx in range(len(block_a.ops)): - self.assertEqual(block_a.ops[op_idx].type, - block_b.ops[op_idx].type) + self.assertEqual( + block_a.ops[op_idx].type, block_b.ops[op_idx].type + ) for var_key in list(block_a.vars.keys()): self.assertTrue(block_b.has_var(var_key)) @@ -165,113 +174,111 @@ def check_prune_correctness(self, method, feed_dict, optimizer): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - loss_data_prune, = exe.run(test_prog_prune, - feed=feed_dict, - fetch_list=[loss.name]) - loss_data_orig, = exe.run(test_prog_orig, - feed=feed_dict, - fetch_list=[loss.name]) + (loss_data_prune,) = exe.run( + test_prog_prune, feed=feed_dict, fetch_list=[loss.name] + ) + (loss_data_orig,) = exe.run( + test_prog_orig, feed=feed_dict, fetch_list=[loss.name] + ) self.assertEqual(loss_data_orig, loss_data_prune) def test_simple_fc_net(self): - def optimizer(): optimizer = fluid.optimizer.SGD( learning_rate=0.001, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) return optimizer with self.program_scope_guard(): img, label = init_data() - self.check_prune_correctness(method=simple_fc_net, - feed_dict={ - "image": img, - "label": label - }, - optimizer=optimizer) + self.check_prune_correctness( + method=simple_fc_net, + feed_dict={"image": img, "label": label}, + optimizer=optimizer, + ) def test_simple_fc_net_with_accuracy(self): - def optimizer(): optimizer = fluid.optimizer.SGD( learning_rate=0.001, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) return optimizer with self.program_scope_guard(): img, label = init_data() - self.check_prune_correctness(method=simple_fc_net_with_accuracy, - feed_dict={ - "image": img, - "label": label - }, - optimizer=optimizer) + self.check_prune_correctness( + method=simple_fc_net_with_accuracy, + feed_dict={"image": img, "label": label}, + optimizer=optimizer, + ) def test_batchnorm_fc(self): - def optimizer(): optimizer = fluid.optimizer.SGD( learning_rate=0.001, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) return optimizer with self.program_scope_guard(): img, label = init_data() - self.check_prune_correctness(method=fc_with_batchnorm, - feed_dict={ - "image": img, - "label": label - }, - optimizer=optimizer) + self.check_prune_correctness( + method=fc_with_batchnorm, + feed_dict={"image": img, "label": label}, + optimizer=optimizer, + ) def test_seresnet(self): with self.program_scope_guard(): self.check_prune_correctness( method=seresnext_net.model, feed_dict=seresnext_net.feed_dict(use_device=DeviceType.CPU), - optimizer=seresnext_net.optimizer) + optimizer=seresnext_net.optimizer, + ) def test_transformer(self): - def optimizer(): optimizer = fluid.optimizer.Adam( learning_rate=0.001, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) return optimizer with self.program_scope_guard(): # the program argument is used to distinguish Program and CompiledProgram feed_dict = get_feed_data_reader().get_next( - fluid.Executor(core.CPUPlace()), fluid.default_main_program()) - self.check_prune_correctness(method=transformer, - feed_dict=feed_dict, - optimizer=optimizer) + fluid.Executor(core.CPUPlace()), fluid.default_main_program() + ) + self.check_prune_correctness( + method=transformer, feed_dict=feed_dict, optimizer=optimizer + ) def test_lstm(self): - def optimizer(): optimizer = fluid.optimizer.Adagrad( learning_rate=0.001, - regularization=fluid.regularizer.L2Decay(1e-4)) + regularization=fluid.regularizer.L2Decay(1e-4), + ) return optimizer with self.program_scope_guard(): word_dict_size = 5147 reader = fake_imdb_reader(word_dict_size, 1) - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") - feeder = fluid.DataFeeder(feed_list=[data, label], - place=core.CPUPlace()) + feeder = fluid.DataFeeder( + feed_list=[data, label], place=core.CPUPlace() + ) feed_data = feeder.feed(reader()) - self.check_prune_correctness(method=lstm_net, - feed_dict=feed_data, - optimizer=optimizer) + self.check_prune_correctness( + method=lstm_net, feed_dict=feed_data, optimizer=optimizer + ) def test_cond(self): - def optimizer(): optimizer = fluid.optimizer.SGD(learning_rate=0.01) return optimizer @@ -280,9 +287,9 @@ def optimizer(): x_in = np.random.random(size=(10, 4)).astype('float32') label_in = np.random.randint(1, size=(10, 1)).astype('int64') feed_dict = {'x': x_in, 'label': label_in} - self.check_prune_correctness(method=cond_net, - feed_dict=feed_dict, - optimizer=optimizer) + self.check_prune_correctness( + method=cond_net, feed_dict=feed_dict, optimizer=optimizer + ) def test_optimization_in_cond(self): x_in = np.random.random(size=(10, 4)).astype('float32') @@ -295,9 +302,9 @@ def test_optimization_in_cond(self): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - loss_data_orig, = exe.run(test_prog_orig, - feed=feed_dict, - fetch_list=[loss.name]) + (loss_data_orig,) = exe.run( + test_prog_orig, feed=feed_dict, fetch_list=[loss.name] + ) with self.program_scope_guard(): loss = optimization_in_cond_net(True) @@ -307,9 +314,9 @@ def test_optimization_in_cond(self): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - loss_data_prune, = exe.run(test_prog_prune, - feed=feed_dict, - fetch_list=[loss.name]) + (loss_data_prune,) = exe.run( + test_prog_prune, feed=feed_dict, fetch_list=[loss.name] + ) self.program_compare(test_prog_orig, test_prog_prune) self.assertEqual(loss_data_orig, loss_data_prune) diff --git a/python/paddle/fluid/tests/unittests/test_program_to_string.py b/python/paddle/fluid/tests/unittests/test_program_to_string.py index 8c010b90671b1..fda96d9046770 100644 --- a/python/paddle/fluid/tests/unittests/test_program_to_string.py +++ b/python/paddle/fluid/tests/unittests/test_program_to_string.py @@ -17,17 +17,16 @@ class TestProgram(unittest.TestCase): - def test_program_to_string(self): prog = fluid.default_main_program() - a = fluid.layers.data(name="X", - shape=[2, 3], - dtype="float32", - append_batch_size=False) + a = fluid.layers.data( + name="X", shape=[2, 3], dtype="float32", append_batch_size=False + ) c = fluid.layers.fc(a, size=3) prog_string = prog.to_string(throw_on_error=True, with_details=False) - prog_string_with_details = prog.to_string(throw_on_error=False, - with_details=True) + prog_string_with_details = prog.to_string( + throw_on_error=False, with_details=True + ) assert prog_string is not None assert len(prog_string_with_details) > len(prog_string) diff --git a/python/paddle/fluid/tests/unittests/test_protobuf.py b/python/paddle/fluid/tests/unittests/test_protobuf.py index 39f32fb1a2da0..c3f1fa80185bf 100644 --- a/python/paddle/fluid/tests/unittests/test_protobuf.py +++ b/python/paddle/fluid/tests/unittests/test_protobuf.py @@ -17,7 +17,6 @@ class TestFrameworkProto(unittest.TestCase): - def test_all(self): op_proto = framework_pb2.OpProto() ipt0 = op_proto.inputs.add() diff --git a/python/paddle/fluid/tests/unittests/test_protobuf_descs.py b/python/paddle/fluid/tests/unittests/test_protobuf_descs.py index 6022bf02602cb..1099855eec7eb 100644 --- a/python/paddle/fluid/tests/unittests/test_protobuf_descs.py +++ b/python/paddle/fluid/tests/unittests/test_protobuf_descs.py @@ -18,7 +18,6 @@ class TestOpDesc(unittest.TestCase): - def test_op_desc(self): program_desc = core.ProgramDesc() self.assertIsNotNone(program_desc) @@ -79,7 +78,6 @@ def test_op_desc(self): class TestProgramDesc(unittest.TestCase): - def test_instance(self): program_desc = core.ProgramDesc() self.assertIsNotNone(program_desc) @@ -107,7 +105,6 @@ def test_append_block(self): class TestVarDesc(unittest.TestCase): - def test_shape(self): program_desc = core.ProgramDesc() block = program_desc.block(0) @@ -145,8 +142,9 @@ def test_multiple_dtype(self): var = block.var(b'my_reader') var.set_type(core.VarDesc.VarType.READER) src_types = [ - core.VarDesc.VarType.INT32, core.VarDesc.VarType.FP64, - core.VarDesc.VarType.FP32 + core.VarDesc.VarType.INT32, + core.VarDesc.VarType.FP64, + core.VarDesc.VarType.FP32, ] var.set_dtypes(src_types) self.assertEqual(src_types, var.dtypes()) @@ -164,7 +162,6 @@ def test_multiple_lod_level(self): class TestBlockDesc(unittest.TestCase): - def test_add_var(self): program_desc = core.ProgramDesc() self.assertIsNotNone(program_desc) diff --git a/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py b/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py index fb4dbf32630c7..3c2689585061a 100644 --- a/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py +++ b/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py @@ -18,7 +18,6 @@ class TestProximalAdagradOp(OpTest): - def setUp(self): self.op_type = "proximal_adagrad" w = np.random.random((102, 105)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py b/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py index bd78a5e8fd9e5..137594b9a08e1 100644 --- a/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py +++ b/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py @@ -18,7 +18,6 @@ class TestProximalGDOp(OpTest): - def setUp(self): self.op_type = "proximal_gd" w = np.random.random((102, 105)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_prroi_pool_op.py b/python/paddle/fluid/tests/unittests/test_prroi_pool_op.py index a927c2a5e820c..4a953b463bea5 100644 --- a/python/paddle/fluid/tests/unittests/test_prroi_pool_op.py +++ b/python/paddle/fluid/tests/unittests/test_prroi_pool_op.py @@ -22,22 +22,24 @@ class TestPRROIPoolOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() self.prRoIPool = PyPrRoIPool() - self.outs = self.prRoIPool.compute(self.x, self.rois, - self.output_channels, - self.spatial_scale, - self.pooled_height, - self.pooled_width).astype('float32') + self.outs = self.prRoIPool.compute( + self.x, + self.rois, + self.output_channels, + self.spatial_scale, + self.pooled_height, + self.pooled_width, + ).astype('float32') self.inputs = {'X': self.x, 'ROIs': (self.rois[:, 1:5], self.rois_lod)} self.attrs = { 'output_channels': self.output_channels, 'spatial_scale': self.spatial_scale, 'pooled_height': self.pooled_height, - 'pooled_width': self.pooled_width + 'pooled_width': self.pooled_width, } self.outputs = {'Out': self.outs} @@ -63,14 +65,18 @@ def make_rois(self): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x1 = np.random.uniform( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.uniform( - 0, self.height // self.spatial_scale - self.pooled_height) - - x2 = np.random.uniform(x1 + self.pooled_width, - self.width // self.spatial_scale) - y2 = np.random.uniform(y1 + self.pooled_height, - self.height // self.spatial_scale) + 0, self.height // self.spatial_scale - self.pooled_height + ) + + x2 = np.random.uniform( + x1 + self.pooled_width, self.width // self.spatial_scale + ) + y2 = np.random.uniform( + y1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) self.rois_num = len(rois) @@ -95,24 +101,24 @@ def run_net(self, place): x = fluid.layers.data( name="X", shape=[self.channels, self.height, self.width], - dtype="float32") - rois = fluid.layers.data(name="ROIs", - shape=[4], - dtype="float32", - lod_level=1) + dtype="float32", + ) + rois = fluid.layers.data( + name="ROIs", shape=[4], dtype="float32", lod_level=1 + ) output = fluid.layers.prroi_pool(x, rois, 0.25, 2, 2) loss = paddle.mean(output) optimizer = fluid.optimizer.SGD(learning_rate=1e-3) optimizer.minimize(loss) input_x = fluid.create_lod_tensor(self.x, [], place) - input_rois = fluid.create_lod_tensor(self.rois[:, 1:5], - self.rois_lod, place) + input_rois = fluid.create_lod_tensor( + self.rois[:, 1:5], self.rois_lod, place + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - exe.run(fluid.default_main_program(), { - 'X': input_x, - "ROIs": input_rois - }) + exe.run( + fluid.default_main_program(), {'X': input_x, "ROIs": input_rois} + ) def test_net(self): places = [fluid.CPUPlace()] @@ -123,47 +129,51 @@ def test_net(self): def test_errors(self): with program_guard(Program(), Program()): - x = fluid.layers.data(name="x", - shape=[245, 30, 30], - dtype="float32") - rois = fluid.layers.data(name="rois", - shape=[4], - dtype="float32", - lod_level=1) + x = fluid.layers.data( + name="x", shape=[245, 30, 30], dtype="float32" + ) + rois = fluid.layers.data( + name="rois", shape=[4], dtype="float32", lod_level=1 + ) # spatial_scale must be float type - self.assertRaises(TypeError, fluid.layers.prroi_pool, x, rois, 2, 7, - 7) + self.assertRaises( + TypeError, fluid.layers.prroi_pool, x, rois, 2, 7, 7 + ) # pooled_height must be int type - self.assertRaises(TypeError, fluid.layers.prroi_pool, x, rois, 0.25, - 0.7, 7) + self.assertRaises( + TypeError, fluid.layers.prroi_pool, x, rois, 0.25, 0.7, 7 + ) # pooled_width must be int type - self.assertRaises(TypeError, fluid.layers.prroi_pool, x, rois, 0.25, - 7, 0.7) + self.assertRaises( + TypeError, fluid.layers.prroi_pool, x, rois, 0.25, 7, 0.7 + ) class TestPRROIPoolOpTensorRoIs(OpTest): - def set_data(self): self.init_test_case() self.make_rois() self.prRoIPool = PyPrRoIPool() - self.outs = self.prRoIPool.compute(self.x, self.rois, - self.output_channels, - self.spatial_scale, - self.pooled_height, - self.pooled_width).astype('float32') + self.outs = self.prRoIPool.compute( + self.x, + self.rois, + self.output_channels, + self.spatial_scale, + self.pooled_height, + self.pooled_width, + ).astype('float32') self.rois_index = np.array(self.rois_lod).reshape([-1]).astype(np.int64) self.inputs = { 'X': self.x, 'ROIs': self.rois[:, 1:5], - 'BatchRoINums': self.rois_index + 'BatchRoINums': self.rois_index, } self.attrs = { 'output_channels': self.output_channels, 'spatial_scale': self.spatial_scale, 'pooled_height': self.pooled_height, - 'pooled_width': self.pooled_width + 'pooled_width': self.pooled_width, } self.outputs = {'Out': self.outs} @@ -189,14 +199,18 @@ def make_rois(self): self.rois_lod.append(bno + 1) for i in range(bno + 1): x1 = np.random.uniform( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.uniform( - 0, self.height // self.spatial_scale - self.pooled_height) - - x2 = np.random.uniform(x1 + self.pooled_width, - self.width // self.spatial_scale) - y2 = np.random.uniform(y1 + self.pooled_height, - self.height // self.spatial_scale) + 0, self.height // self.spatial_scale - self.pooled_height + ) + + x2 = np.random.uniform( + x1 + self.pooled_width, self.width // self.spatial_scale + ) + y2 = np.random.uniform( + y1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) self.rois_num = len(rois) @@ -221,27 +235,28 @@ def run_net(self, place): x = fluid.layers.data( name="X", shape=[self.channels, self.height, self.width], - dtype="float32") + dtype="float32", + ) rois = fluid.layers.data(name="ROIs", shape=[4], dtype="float32") - rois_index = fluid.layers.data(name='rois_idx', - shape=[], - dtype="int64") - output = fluid.layers.prroi_pool(x, - rois, - 0.25, - 2, - 2, - batch_roi_nums=rois_index) + rois_index = fluid.layers.data( + name='rois_idx', shape=[], dtype="int64" + ) + output = fluid.layers.prroi_pool( + x, rois, 0.25, 2, 2, batch_roi_nums=rois_index + ) loss = paddle.mean(output) optimizer = fluid.optimizer.SGD(learning_rate=1e-3) optimizer.minimize(loss) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - exe.run(fluid.default_main_program(), { - 'X': self.x, - "ROIs": self.rois[:, 1:5], - "rois_idx": self.rois_index - }) + exe.run( + fluid.default_main_program(), + { + 'X': self.x, + "ROIs": self.rois[:, 1:5], + "rois_idx": self.rois_index, + }, + ) def test_net(self): places = [fluid.CPUPlace()] @@ -252,42 +267,50 @@ def test_net(self): def test_errors(self): with program_guard(Program(), Program()): - x = fluid.layers.data(name="x", - shape=[245, 30, 30], - dtype="float32") - rois = fluid.layers.data(name="rois", - shape=[4], - dtype="float32", - lod_level=1) + x = fluid.layers.data( + name="x", shape=[245, 30, 30], dtype="float32" + ) + rois = fluid.layers.data( + name="rois", shape=[4], dtype="float32", lod_level=1 + ) # spatial_scale must be float type - self.assertRaises(TypeError, fluid.layers.prroi_pool, x, rois, 2, 7, - 7) + self.assertRaises( + TypeError, fluid.layers.prroi_pool, x, rois, 2, 7, 7 + ) # pooled_height must be int type - self.assertRaises(TypeError, fluid.layers.prroi_pool, x, rois, 0.25, - 0.7, 7) + self.assertRaises( + TypeError, fluid.layers.prroi_pool, x, rois, 0.25, 0.7, 7 + ) # pooled_width must be int type - self.assertRaises(TypeError, fluid.layers.prroi_pool, x, rois, 0.25, - 7, 0.7) + self.assertRaises( + TypeError, fluid.layers.prroi_pool, x, rois, 0.25, 7, 0.7 + ) def test_bad_x(): - x = fluid.layers.data(name='data1', - shape=[2, 3, 16, 16], - dtype='int64', - append_batch_size=False) - label = fluid.layers.data(name='label1', - shape=[2, 4], - dtype='float32', - lod_level=1, - append_batch_size=False) + x = fluid.layers.data( + name='data1', + shape=[2, 3, 16, 16], + dtype='int64', + append_batch_size=False, + ) + label = fluid.layers.data( + name='label1', + shape=[2, 4], + dtype='float32', + lod_level=1, + append_batch_size=False, + ) output = fluid.layers.prroi_pool(x, label, 0.25, 2, 2) self.assertRaises(TypeError, test_bad_x) def test_bad_y(): - x = fluid.layers.data(name='data2', - shape=[2, 3, 16, 16], - dtype='float32', - append_batch_size=False) + x = fluid.layers.data( + name='data2', + shape=[2, 3, 16, 16], + dtype='float32', + append_batch_size=False, + ) label = [[1, 2, 3, 4], [2, 3, 4, 5]] output = fluid.layers.prroi_pool(x, label, 0.25, 2, 2) @@ -296,5 +319,6 @@ def test_bad_y(): if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_prune.py b/python/paddle/fluid/tests/unittests/test_prune.py index b6386e6967466..5b7596d813d74 100644 --- a/python/paddle/fluid/tests/unittests/test_prune.py +++ b/python/paddle/fluid/tests/unittests/test_prune.py @@ -23,7 +23,6 @@ class TestPrune(unittest.TestCase): - def net(self): x = fluid.layers.data(name='x', shape=[2], dtype='float32') label = fluid.layers.data(name="label", shape=[1], dtype="int64") @@ -39,14 +38,24 @@ def test_prune_with_input(self): with fluid.program_guard(program, startup_program): (x, y, label, loss) = self.net() self.assertEqual(len(block.ops), 5) - self.assertEqual([op.type for op in block.ops], [ - "mul", "elementwise_add", "softmax", "cross_entropy2", "reduce_mean" - ]) + self.assertEqual( + [op.type for op in block.ops], + [ + "mul", + "elementwise_add", + "softmax", + "cross_entropy2", + "reduce_mean", + ], + ) pruned_program = program._prune_with_input( - feeded_var_names=[y.name, label.name], targets=[loss]) + feeded_var_names=[y.name, label.name], targets=[loss] + ) self.assertEqual(len(pruned_program.global_block().ops), 2) - self.assertEqual([op.type for op in pruned_program.global_block().ops], - ["cross_entropy2", "reduce_mean"]) + self.assertEqual( + [op.type for op in pruned_program.global_block().ops], + ["cross_entropy2", "reduce_mean"], + ) def test_prune(self): program = framework.Program() @@ -55,16 +64,28 @@ def test_prune(self): with fluid.program_guard(program, startup_program): (x, y, label, loss) = self.net() self.assertEqual(len(block.ops), 5) - self.assertEqual([op.type for op in block.ops], [ - "mul", "elementwise_add", "softmax", "cross_entropy2", "reduce_mean" - ]) + self.assertEqual( + [op.type for op in block.ops], + [ + "mul", + "elementwise_add", + "softmax", + "cross_entropy2", + "reduce_mean", + ], + ) pruned_program = program._prune(targets=[loss]) self.assertEqual(len(pruned_program.global_block().ops), 5) - self.assertEqual([op.type for op in pruned_program.global_block().ops], - [ - "mul", "elementwise_add", "softmax", - "cross_entropy2", "reduce_mean" - ]) + self.assertEqual( + [op.type for op in pruned_program.global_block().ops], + [ + "mul", + "elementwise_add", + "softmax", + "cross_entropy2", + "reduce_mean", + ], + ) def test_prune_target_not_list(self): program = framework.Program() @@ -73,16 +94,28 @@ def test_prune_target_not_list(self): with fluid.program_guard(program, startup_program): (x, y, label, loss) = self.net() self.assertEqual(len(block.ops), 5) - self.assertEqual([op.type for op in block.ops], [ - "mul", "elementwise_add", "softmax", "cross_entropy2", "reduce_mean" - ]) + self.assertEqual( + [op.type for op in block.ops], + [ + "mul", + "elementwise_add", + "softmax", + "cross_entropy2", + "reduce_mean", + ], + ) pruned_program = program._prune(targets=loss) self.assertEqual(len(pruned_program.global_block().ops), 5) - self.assertEqual([op.type for op in pruned_program.global_block().ops], - [ - "mul", "elementwise_add", "softmax", - "cross_entropy2", "reduce_mean" - ]) + self.assertEqual( + [op.type for op in pruned_program.global_block().ops], + [ + "mul", + "elementwise_add", + "softmax", + "cross_entropy2", + "reduce_mean", + ], + ) def test_prune_target_none(self): program = framework.Program() @@ -91,15 +124,23 @@ def test_prune_target_none(self): with fluid.program_guard(program, startup_program): (x, y, label, loss) = self.net() self.assertEqual(len(block.ops), 5) - self.assertEqual([op.type for op in block.ops], [ - "mul", "elementwise_add", "softmax", "cross_entropy2", "reduce_mean" - ]) + self.assertEqual( + [op.type for op in block.ops], + [ + "mul", + "elementwise_add", + "softmax", + "cross_entropy2", + "reduce_mean", + ], + ) try: pruned_program = program._prune(targets=None) except ValueError as e: self.assertIn( "All targets of Program._prune_with_input() can only be Variable or Operator", - str(e)) + str(e), + ) def mock(self, program, feed, fetch, optimize_ops): @@ -116,7 +157,6 @@ def _mock_guard(mock): class TestExecutorRunAutoPrune(unittest.TestCase): - def net1(self): x = fluid.layers.data(name='x', shape=[2], dtype='float32') label = fluid.layers.data(name="label", shape=[1], dtype="int64") @@ -124,11 +164,11 @@ def net1(self): name="fc_weight", learning_rate=0.5, initializer=fluid.initializer.Constant(1.0), - trainable=True) - y = fluid.layers.fc(input=[x], - size=2, - act="softmax", - param_attr=w_param_attrs) + trainable=True, + ) + y = fluid.layers.fc( + input=[x], size=2, act="softmax", param_attr=w_param_attrs + ) loss1 = fluid.layers.cross_entropy(input=y, label=label) loss1 = paddle.mean(x=loss1) loss2 = fluid.layers.cross_entropy(input=y, label=label) @@ -145,25 +185,35 @@ def net2(self): name="fc_weight1", learning_rate=0.5, initializer=fluid.initializer.Constant(1.0), - trainable=True) + trainable=True, + ) w2_param_attrs = fluid.ParamAttr( name="fc_weight2", learning_rate=0.5, initializer=fluid.initializer.Constant(1.0), - trainable=True) - y1 = fluid.layers.fc(input=[x1], - size=2, - act="softmax", - param_attr=w1_param_attrs) - y2 = fluid.layers.fc(input=[x2], - size=2, - act="softmax", - param_attr=w2_param_attrs) + trainable=True, + ) + y1 = fluid.layers.fc( + input=[x1], size=2, act="softmax", param_attr=w1_param_attrs + ) + y2 = fluid.layers.fc( + input=[x2], size=2, act="softmax", param_attr=w2_param_attrs + ) loss1 = fluid.layers.cross_entropy(input=y1, label=label) loss1 = paddle.mean(x=loss1) loss2 = fluid.layers.cross_entropy(input=y2, label=label) loss2 = paddle.mean(x=loss2) - return x1, x2, y1, y2, label, loss1, loss2, w1_param_attrs, w2_param_attrs + return ( + x1, + x2, + y1, + y2, + label, + loss1, + loss2, + w1_param_attrs, + w2_param_attrs, + ) def test_not_prune(self): """ @@ -179,13 +229,12 @@ def test_not_prune(self): exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) self.assertIsNotNone(scope.find_var(loss1.name)) self.assertIsNotNone(scope.find_var(loss2.name)) @@ -202,22 +251,24 @@ def test_prune_fetches_without_optimizer(self): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) weight_init = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=True) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(loss1.name)) - self.assertIsNone(scope.find_var(loss2.name)) #loss2 is pruned + self.assertIsNone(scope.find_var(loss2.name)) # loss2 is pruned weight = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) - np.testing.assert_array_equal(weight_init, - weight) # weight not changed + scope.find_var(w_param_attrs.name).get_tensor() + ) + np.testing.assert_array_equal( + weight_init, weight + ) # weight not changed def test_prune_fetches_with_optimizer(self): """ @@ -235,22 +286,24 @@ def test_prune_fetches_with_optimizer(self): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) weight_init = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=True) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(loss1.name)) - self.assertIsNone(scope.find_var(loss2.name)) #loss2 is pruned + self.assertIsNone(scope.find_var(loss2.name)) # loss2 is pruned weight = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) - self.assertFalse(np.array_equal(weight_init, - weight)) # weight changed + scope.find_var(w_param_attrs.name).get_tensor() + ) + self.assertFalse( + np.array_equal(weight_init, weight) + ) # weight changed def test_prune_compiled_program(self): program = framework.Program() @@ -264,25 +317,29 @@ def test_prune_compiled_program(self): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) compiled_prog = fluid.CompiledProgram( - program).with_data_parallel(loss_name=loss1.name, - places=fluid.CPUPlace()) + program + ).with_data_parallel( + loss_name=loss1.name, places=fluid.CPUPlace() + ) weight_init = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(compiled_prog, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=True) + res = exe.run( + compiled_prog, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(loss1.name)) self.assertIsNone(scope.find_var(loss2.name)) weight = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) - self.assertFalse(np.array_equal(weight_init, - weight)) # weight changed + scope.find_var(w_param_attrs.name).get_tensor() + ) + self.assertFalse( + np.array_equal(weight_init, weight) + ) # weight changed def test_prune_feed_without_optimizer(self): program = framework.Program() @@ -294,22 +351,24 @@ def test_prune_feed_without_optimizer(self): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) weight_init = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - y.name: x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=True) + res = exe.run( + program, + feed={y.name: x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(loss1.name)) self.assertIsNone(scope.find_var(loss2.name)) weight = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) - np.testing.assert_array_equal(weight_init, - weight) # weight unchanged + scope.find_var(w_param_attrs.name).get_tensor() + ) + np.testing.assert_array_equal( + weight_init, weight + ) # weight unchanged def test_prune_feed_with_optimizer(self): program = framework.Program() @@ -324,15 +383,14 @@ def test_prune_feed_with_optimizer(self): exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - self.assertRaises(Exception, - exe.run, - program, - feed={ - y.name: x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=True) + self.assertRaises( + Exception, + exe.run, + program, + feed={y.name: x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(loss1.name)) self.assertIsNone(scope.find_var(loss2.name)) @@ -358,16 +416,16 @@ def test_prune_with_cache_program(self): sgd_optimizer.minimize(loss1) exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') - label_np = np.random.randint(1, - size=(10, 1)).astype('int64') + label_np = np.random.randint(1, size=(10, 1)).astype( + 'int64' + ) for i in range(10): - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=True) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=True, + ) if i == 0: self.assertEqual(exe.prune_called_times, 1) else: @@ -387,38 +445,54 @@ def test_prune_with_cache_program2(self): scope = fluid.Scope() with fluid.scope_guard(scope): with fluid.program_guard(program, startup_program): - (x1, x2, y1, y2, label, loss1, loss2, w1_param_attrs, - w2_param_attrs) = self.net2() + ( + x1, + x2, + y1, + y2, + label, + loss1, + loss2, + w1_param_attrs, + w2_param_attrs, + ) = self.net2() adam_optimizer1 = fluid.optimizer.AdamOptimizer( - learning_rate=0.5) + learning_rate=0.5 + ) train1 = adam_optimizer1.minimize(loss1) adam_optimizer2 = fluid.optimizer.AdamOptimizer( - learning_rate=0.5) + learning_rate=0.5 + ) train2 = adam_optimizer2.minimize(loss2) exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') - label_np = np.random.randint(1, - size=(10, 1)).astype('int64') + label_np = np.random.randint(1, size=(10, 1)).astype( + 'int64' + ) for i in range(10): if i % 2: - res = exe.run(program, - feed={ - 'x1': x_np, - 'x2': x_np, - 'label': label_np - }, - fetch_list=[loss1, loss2, train1], - use_prune=True) + res = exe.run( + program, + feed={ + 'x1': x_np, + 'x2': x_np, + 'label': label_np, + }, + fetch_list=[loss1, loss2, train1], + use_prune=True, + ) else: - res = exe.run(program, - feed={ - 'x1': x_np, - 'x2': x_np, - 'label': label_np - }, - fetch_list=[loss1, loss2, train2], - use_prune=True) + res = exe.run( + program, + feed={ + 'x1': x_np, + 'x2': x_np, + 'label': label_np, + }, + fetch_list=[loss1, loss2, train2], + use_prune=True, + ) if i == 0: self.assertEqual(exe.prune_called_times, 1) elif i == 1: @@ -448,19 +522,21 @@ def test_prune_with_cache_compiled_program(self): sgd_optimizer.minimize(loss1) exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') - label_np = np.random.randint(1, - size=(10, 1)).astype('int64') + label_np = np.random.randint(1, size=(10, 1)).astype( + 'int64' + ) compiled_prog = fluid.CompiledProgram( - program).with_data_parallel(loss_name=loss1.name, - places=fluid.CPUPlace()) + program + ).with_data_parallel( + loss_name=loss1.name, places=fluid.CPUPlace() + ) for i in range(10): - res = exe.run(compiled_prog, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=True) + res = exe.run( + compiled_prog, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=True, + ) if i == 0: self.assertEqual(exe.prune_called_times, 1) else: @@ -486,43 +562,43 @@ def test_prune_with_multi_optimizers(self): exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) weight_without_prune = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) scope = fluid.Scope() # use_prune with fluid.scope_guard(scope): exe.run(startup_program) - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name, train1], - use_prune=True) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name, train1], + use_prune=True, + ) weight_with_prune = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) # expected scope = fluid.Scope() with fluid.scope_guard(scope): exe.run(startup_program) - exe.run(cloned_program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + exe.run( + cloned_program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) weight_expected = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) np.testing.assert_array_equal(weight_with_prune, weight_expected) self.assertFalse(np.array_equal(weight_without_prune, weight_expected)) @@ -540,60 +616,72 @@ def test_prune_with_multi_devices(self): # do not use_prune with fluid.scope_guard(scope): with fluid.program_guard(program, startup_program): - (x1, x2, y1, y2, label, loss1, loss2, w1_param_attrs, - w2_param_attrs) = self.net2() + ( + x1, + x2, + y1, + y2, + label, + loss1, + loss2, + w1_param_attrs, + w2_param_attrs, + ) = self.net2() adam_optimizer1 = fluid.optimizer.AdamOptimizer( - learning_rate=0.5) + learning_rate=0.5 + ) train1 = adam_optimizer1.minimize(loss1) cloned_program = program.clone() adam_optimizer2 = fluid.optimizer.AdamOptimizer( - learning_rate=0.5) + learning_rate=0.5 + ) train2 = adam_optimizer2.minimize(loss2) exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') compiled_prog1 = fluid.CompiledProgram( - program).with_data_parallel(loss_name=loss1.name, - places=[fluid.CPUPlace()] * 2) + program + ).with_data_parallel( + loss_name=loss1.name, places=[fluid.CPUPlace()] * 2 + ) compiled_prog2 = fluid.CompiledProgram( - program).with_data_parallel(loss_name=loss2.name, - places=[fluid.CPUPlace()] * 2) + program + ).with_data_parallel( + loss_name=loss2.name, places=[fluid.CPUPlace()] * 2 + ) for i in range(10): if i % 2 == 1: - res = exe.run(compiled_prog1, - feed=[{ - 'x1': x_np[0:5, :], - 'label': label_np[0:5, :] - }, { - 'x1': x_np[5:, :], - 'label': label_np[5:, :] - }], - fetch_list=[loss1.name, train1], - use_prune=True) + res = exe.run( + compiled_prog1, + feed=[ + {'x1': x_np[0:5, :], 'label': label_np[0:5, :]}, + {'x1': x_np[5:, :], 'label': label_np[5:, :]}, + ], + fetch_list=[loss1.name, train1], + use_prune=True, + ) else: - res = exe.run(compiled_prog2, - feed={ - 'x2': x_np, - 'label': label_np - }, - fetch_list=[loss2.name, train2], - use_prune=True) + res = exe.run( + compiled_prog2, + feed={'x2': x_np, 'label': label_np}, + fetch_list=[loss2.name, train2], + use_prune=True, + ) weight1 = np.array( - scope.find_var(w1_param_attrs.name).get_tensor()) + scope.find_var(w1_param_attrs.name).get_tensor() + ) # expected scope = fluid.Scope() with fluid.scope_guard(scope): exe.run(startup_program) for i in range(10): if i % 2 == 1: - exe.run(cloned_program, - feed={ - 'x1': x_np, - 'x2': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + exe.run( + cloned_program, + feed={'x1': x_np, 'x2': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) weight2 = np.array(scope.find_var(w1_param_attrs.name).get_tensor()) np.testing.assert_allclose(weight1, weight2, rtol=1e-05) @@ -619,44 +707,44 @@ def test_prune_program_with_tupe_in_fetch_list(self): x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) weight_without_prune = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) scope = fluid.Scope() # use_prune with fluid.scope_guard(scope): exe.run(startup_program) - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name, train1], - use_prune=True) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name, train1], + use_prune=True, + ) weight_with_prune = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) # expected scope = fluid.Scope() with fluid.scope_guard(scope): exe.run(startup_program) - exe.run(cloned_program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + exe.run( + cloned_program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) weight_expected = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) np.testing.assert_array_equal(weight_with_prune, weight_expected) self.assertFalse(np.array_equal(weight_without_prune, weight_expected)) @@ -671,8 +759,17 @@ def test_prune_program_partial_parameter_updated(self): scope = fluid.Scope() with fluid.scope_guard(scope): with fluid.program_guard(program, startup_program): - (x1, x2, y1, y2, label, loss1, loss2, w1_param_attrs, - w2_param_attrs) = self.net2() + ( + x1, + x2, + y1, + y2, + label, + loss1, + loss2, + w1_param_attrs, + w2_param_attrs, + ) = self.net2() loss1.persistable = True loss2.persistable = True sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.5) @@ -682,31 +779,36 @@ def test_prune_program_partial_parameter_updated(self): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) weight1_init = np.array( - scope.find_var(w1_param_attrs.name).get_tensor()) + scope.find_var(w1_param_attrs.name).get_tensor() + ) weight2_init = np.array( - scope.find_var(w2_param_attrs.name).get_tensor()) + scope.find_var(w2_param_attrs.name).get_tensor() + ) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - 'x1': x_np, - 'label': label_np - }, - fetch_list=[loss1.name, train1], - use_prune=True) + res = exe.run( + program, + feed={'x1': x_np, 'label': label_np}, + fetch_list=[loss1.name, train1], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(w1_param_attrs.name)) self.assertIsNotNone(scope.find_var(w2_param_attrs.name)) self.assertIsNotNone(scope.find_var(loss1.name)) self.assertIsNone(scope.find_var(loss2.name)) weight1 = np.array( - scope.find_var(w1_param_attrs.name).get_tensor()) + scope.find_var(w1_param_attrs.name).get_tensor() + ) weight2 = np.array( - scope.find_var(w2_param_attrs.name).get_tensor()) - self.assertFalse(np.array_equal(weight1_init, - weight1)) # weight changed - np.testing.assert_array_equal(weight2_init, - weight2) # weight2 unchanged + scope.find_var(w2_param_attrs.name).get_tensor() + ) + self.assertFalse( + np.array_equal(weight1_init, weight1) + ) # weight changed + np.testing.assert_array_equal( + weight2_init, weight2 + ) # weight2 unchanged def test_prune_override_use_prune(self): ''' @@ -727,43 +829,43 @@ def test_prune_override_use_prune(self): exe.run(startup_program) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) weight_without_prune = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) scope = fluid.Scope() # use_prune with fluid.scope_guard(scope): exe.run(startup_program) - res = exe.run(program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name, train1]) + res = exe.run( + program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name, train1], + ) weight_with_prune = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) # expected scope = fluid.Scope() with fluid.scope_guard(scope): exe.run(startup_program) - exe.run(cloned_program, - feed={ - 'x': x_np, - 'label': label_np - }, - fetch_list=[loss1.name], - use_prune=False) + exe.run( + cloned_program, + feed={'x': x_np, 'label': label_np}, + fetch_list=[loss1.name], + use_prune=False, + ) weight_expected = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) np.testing.assert_array_equal(weight_with_prune, weight_expected) self.assertFalse(np.array_equal(weight_without_prune, weight_expected)) @@ -779,23 +881,25 @@ def test_prune_feed_var_in_fetchlist_1(self): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) weight_init = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - y.name: x_np, - 'label': label_np - }, - fetch_list=[y.name, loss1.name], - use_prune=True) + res = exe.run( + program, + feed={y.name: x_np, 'label': label_np}, + fetch_list=[y.name, loss1.name], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(loss1.name)) self.assertIsNone(scope.find_var(loss2.name)) self.assertIsNone(scope.find_var(x.name)) weight = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) - np.testing.assert_array_equal(weight_init, - weight) # weight unchanged + scope.find_var(w_param_attrs.name).get_tensor() + ) + np.testing.assert_array_equal( + weight_init, weight + ) # weight unchanged def test_prune_feed_var_in_fetchlist_2(self): # the variable to be fed is leaf @@ -808,22 +912,24 @@ def test_prune_feed_var_in_fetchlist_2(self): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) weight_init = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) + scope.find_var(w_param_attrs.name).get_tensor() + ) x_np = np.random.random(size=(10, 2)).astype('float32') label_np = np.random.randint(1, size=(10, 1)).astype('int64') - res = exe.run(program, - feed={ - x.name: x_np, - 'label': label_np - }, - fetch_list=[x.name, loss1.name], - use_prune=True) + res = exe.run( + program, + feed={x.name: x_np, 'label': label_np}, + fetch_list=[x.name, loss1.name], + use_prune=True, + ) self.assertIsNotNone(scope.find_var(loss1.name)) self.assertIsNone(scope.find_var(loss2.name)) weight = np.array( - scope.find_var(w_param_attrs.name).get_tensor()) - np.testing.assert_array_equal(weight_init, - weight) # weight unchanged + scope.find_var(w_param_attrs.name).get_tensor() + ) + np.testing.assert_array_equal( + weight_init, weight + ) # weight unchanged if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_prune_gate_by_capacity_op.py b/python/paddle/fluid/tests/unittests/test_prune_gate_by_capacity_op.py index 4b73e05f502c6..0caab5573470e 100644 --- a/python/paddle/fluid/tests/unittests/test_prune_gate_by_capacity_op.py +++ b/python/paddle/fluid/tests/unittests/test_prune_gate_by_capacity_op.py @@ -21,7 +21,7 @@ def count(x, upper_num): - res = np.zeros((upper_num, )).astype(int) + res = np.zeros((upper_num,)).astype(int) for i in x.reshape(-1): if i >= 0 and i < len(res): res[i] += 1 @@ -64,20 +64,22 @@ def assert_allclose(output, expected, n_expert): assert np.allclose(c1, c2) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestPruneGateByCapacityAPI1(unittest.TestCase): - def init_test_case(self): - self.gate_idx = np.random.randint(0, self.n_expert, - size=(200, )).astype(self.dtype) + self.gate_idx = np.random.randint(0, self.n_expert, size=(200,)).astype( + self.dtype + ) expert_count = count(self.gate_idx, self.n_expert * self.n_worker) - capacity = np.random.randint(10, 200, size=(self.n_expert, )) - self.expert_count = limit_by_capacity(expert_count, capacity, - self.n_worker).astype(self.dtype) - self.out = prune_gate_by_capacity(self.gate_idx, self.expert_count, - self.n_expert, - self.n_worker).astype(self.dtype) + capacity = np.random.randint(10, 200, size=(self.n_expert,)) + self.expert_count = limit_by_capacity( + expert_count, capacity, self.n_worker + ).astype(self.dtype) + self.out = prune_gate_by_capacity( + self.gate_idx, self.expert_count, self.n_expert, self.n_worker + ).astype(self.dtype) self.place = paddle.CUDAPlace(0) def setUp(self): @@ -89,29 +91,35 @@ def setUp(self): def test_static_api(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): - gate_idx_tensor = paddle.static.data('GateIdx', - shape=self.gate_idx.shape, - dtype="int64") + gate_idx_tensor = paddle.static.data( + 'GateIdx', shape=self.gate_idx.shape, dtype="int64" + ) expert_count_tensor = paddle.static.data( - 'ExpertCount', shape=self.expert_count.shape, dtype="int64") - out = utils._prune_gate_by_capacity(gate_idx_tensor, - expert_count_tensor, - self.n_expert, self.n_worker) + 'ExpertCount', shape=self.expert_count.shape, dtype="int64" + ) + out = utils._prune_gate_by_capacity( + gate_idx_tensor, + expert_count_tensor, + self.n_expert, + self.n_worker, + ) exe = paddle.static.Executor(self.place) - res = exe.run(feed={ - 'GateIdx': self.gate_idx, - 'ExpertCount': self.expert_count, - }, - fetch_list=out) + res = exe.run( + feed={ + 'GateIdx': self.gate_idx, + 'ExpertCount': self.expert_count, + }, + fetch_list=out, + ) assert_allclose(res[0], self.out, self.n_expert) def func_dygraph_api(self): paddle.disable_static(self.place) gate_idx_tensor = paddle.to_tensor(self.gate_idx) expert_count_tensor = paddle.to_tensor(self.expert_count) - out = utils._prune_gate_by_capacity(gate_idx_tensor, - expert_count_tensor, self.n_expert, - self.n_worker) + out = utils._prune_gate_by_capacity( + gate_idx_tensor, expert_count_tensor, self.n_expert, self.n_worker + ) assert_allclose(out.numpy(), self.out, self.n_expert) def test_dygraph_api(self): @@ -120,10 +128,10 @@ def test_dygraph_api(self): self.func_dygraph_api() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestPruneGateByCapacityAPI2(TestPruneGateByCapacityAPI1): - def setUp(self): self.n_expert = 12 self.n_worker = 1 diff --git a/python/paddle/fluid/tests/unittests/test_ps_dispatcher.py b/python/paddle/fluid/tests/unittests/test_ps_dispatcher.py index f290d164333de..88d6c6f45ac47 100644 --- a/python/paddle/fluid/tests/unittests/test_ps_dispatcher.py +++ b/python/paddle/fluid/tests/unittests/test_ps_dispatcher.py @@ -13,15 +13,20 @@ # limitations under the License. import unittest -from paddle.fluid.incubate.fleet.parameter_server.ir.ps_dispatcher import RoundRobin, HashName, PSDispatcher +from paddle.fluid.incubate.fleet.parameter_server.ir.ps_dispatcher import ( + RoundRobin, + HashName, + PSDispatcher, +) class TestPsDispatcher(unittest.TestCase): - def setUp(self): self.points = [ - "127.0.0.1:1001", "127.0.0.1:1002", "127.0.0.1:1003", - "127.0.0.1:1004" + "127.0.0.1:1001", + "127.0.0.1:1002", + "127.0.0.1:1003", + "127.0.0.1:1004", ] def test_base(self): @@ -33,9 +38,7 @@ def test_base(self): base.dispatch([]) def test_hash(self): - class Var: - def __init__(self, index): self._name = "var_{}".format(index) @@ -53,9 +56,7 @@ def name(self): self.assertEqual(len(eplist), 4) def test_round_rodin(self): - class Var: - def __init__(self, index): self._name = "var_{}".format(index) diff --git a/python/paddle/fluid/tests/unittests/test_psroi_pool_op.py b/python/paddle/fluid/tests/unittests/test_psroi_pool_op.py index d386c7145ddb4..0bb1adb4ad65f 100644 --- a/python/paddle/fluid/tests/unittests/test_psroi_pool_op.py +++ b/python/paddle/fluid/tests/unittests/test_psroi_pool_op.py @@ -19,8 +19,15 @@ from op_test import OpTest -def calc_psroi_pool(x, rois, rois_num_per_img, output_channels, spatial_scale, - pooled_height, pooled_width): +def calc_psroi_pool( + x, + rois, + rois_num_per_img, + output_channels, + spatial_scale, + pooled_height, + pooled_width, +): """ Psroi_pool implemented by Numpy. x: 4-D as (N, C, H, W), @@ -43,8 +50,8 @@ def calc_psroi_pool(x, rois, rois_num_per_img, output_channels, spatial_scale, batch_id += 1 roi_start_w = round(roi[0]) * spatial_scale roi_start_h = round(roi[1]) * spatial_scale - roi_end_w = (round(roi[2]) + 1.) * spatial_scale - roi_end_h = (round(roi[3]) + 1.) * spatial_scale + roi_end_w = (round(roi[2]) + 1.0) * spatial_scale + roi_end_h = (round(roi[3]) + 1.0) * spatial_scale roi_height = max(roi_end_h - roi_start_h, 0.1) roi_width = max(roi_end_w - roi_start_w, 0.1) @@ -58,13 +65,17 @@ def calc_psroi_pool(x, rois, rois_num_per_img, output_channels, spatial_scale, for ph in range(pooled_height): for pw in range(pooled_width): hstart = int( - math.floor(float(ph) * bin_size_h + roi_start_h)) + math.floor(float(ph) * bin_size_h + roi_start_h) + ) wstart = int( - math.floor(float(pw) * bin_size_w + roi_start_w)) + math.floor(float(pw) * bin_size_w + roi_start_w) + ) hend = int( - math.ceil(float(ph + 1) * bin_size_h + roi_start_h)) + math.ceil(float(ph + 1) * bin_size_h + roi_start_h) + ) wend = int( - math.ceil(float(pw + 1) * bin_size_w + roi_start_w)) + math.ceil(float(pw + 1) * bin_size_w + roi_start_w) + ) hstart = min(max(hstart, 0), x.shape[2]) hend = min(max(hend, 0), x.shape[2]) wstart = min(max(wstart, 0), x.shape[3]) @@ -72,37 +83,41 @@ def calc_psroi_pool(x, rois, rois_num_per_img, output_channels, spatial_scale, c_in = (c * pooled_height + ph) * pooled_width + pw is_empty = (hend <= hstart) or (wend <= wstart) - out_sum = 0. + out_sum = 0.0 for ih in range(hstart, hend): for iw in range(wstart, wend): out_sum += x_i[c_in, ih, iw] bin_area = (hend - hstart) * (wend - wstart) - out_data[i, c, ph, - pw] = 0. if is_empty else (out_sum / - float(bin_area)) + out_data[i, c, ph, pw] = ( + 0.0 if is_empty else (out_sum / float(bin_area)) + ) return out_data class TestPSROIPoolOp(OpTest): - def set_data(self): paddle.enable_static() self.init_test_case() self.make_rois() - self.outs = calc_psroi_pool(self.x, self.boxes, self.boxes_num, - self.output_channels, self.spatial_scale, - self.pooled_height, - self.pooled_width).astype('float64') + self.outs = calc_psroi_pool( + self.x, + self.boxes, + self.boxes_num, + self.output_channels, + self.spatial_scale, + self.pooled_height, + self.pooled_width, + ).astype('float64') self.inputs = { 'X': self.x, 'ROIs': (self.rois_with_batch_id[:, 1:5], self.rois_lod), - 'RoisNum': self.boxes_num + 'RoisNum': self.boxes_num, } self.attrs = { 'output_channels': self.output_channels, 'spatial_scale': self.spatial_scale, 'pooled_height': self.pooled_height, - 'pooled_width': self.pooled_width + 'pooled_width': self.pooled_width, } self.outputs = {'Out': self.outs} @@ -128,26 +143,32 @@ def make_rois(self): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x1 = np.random.random_integers( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.random_integers( - 0, self.height // self.spatial_scale - self.pooled_height) + 0, self.height // self.spatial_scale - self.pooled_height + ) - x2 = np.random.random_integers(x1 + self.pooled_width, - self.width // self.spatial_scale) + x2 = np.random.random_integers( + x1 + self.pooled_width, self.width // self.spatial_scale + ) y2 = np.random.random_integers( - y1 + self.pooled_height, self.height // self.spatial_scale) + y1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) self.rois_num = len(rois) self.rois_with_batch_id = np.array(rois).astype('float64') self.boxes = self.rois_with_batch_id[:, 1:] - self.boxes_num = np.array([bno + 1 for bno in range(self.batch_size) - ]).astype('int32') + self.boxes_num = np.array( + [bno + 1 for bno in range(self.batch_size)] + ).astype('int32') def setUp(self): self.op_type = 'psroi_pool' self.python_api = lambda x, boxes, boxes_num, pooled_height, pooled_width, output_channels, spatial_scale: paddle.vision.ops.psroi_pool( - x, boxes, boxes_num, (pooled_height, pooled_width), spatial_scale) + x, boxes, boxes_num, (pooled_height, pooled_width), spatial_scale + ) self.set_data() def test_check_output(self): @@ -158,33 +179,38 @@ def test_check_grad(self): class TestPSROIPoolDynamicFunctionAPI(unittest.TestCase): - def setUp(self): self.x = np.random.random([2, 490, 28, 28]).astype(np.float32) - self.boxes = np.array([[1, 5, 8, 10], [4, 2, 6, 7], - [12, 12, 19, 21]]).astype(np.float32) + self.boxes = np.array( + [[1, 5, 8, 10], [4, 2, 6, 7], [12, 12, 19, 21]] + ).astype(np.float32) self.boxes_num = np.array([1, 2]).astype(np.int32) def test_output_size(self): - def test_output_size_is_int(): output_size = 7 - out = paddle.vision.ops.psroi_pool(paddle.to_tensor(self.x), - paddle.to_tensor(self.boxes), - paddle.to_tensor(self.boxes_num), - output_size).numpy() - expect_out = calc_psroi_pool(self.x, self.boxes, self.boxes_num, 10, - 1.0, 7, 7) + out = paddle.vision.ops.psroi_pool( + paddle.to_tensor(self.x), + paddle.to_tensor(self.boxes), + paddle.to_tensor(self.boxes_num), + output_size, + ).numpy() + expect_out = calc_psroi_pool( + self.x, self.boxes, self.boxes_num, 10, 1.0, 7, 7 + ) np.testing.assert_allclose(out, expect_out, rtol=1e-05) def test_output_size_is_tuple(): output_size = (7, 7) - out = paddle.vision.ops.psroi_pool(paddle.to_tensor(self.x), - paddle.to_tensor(self.boxes), - paddle.to_tensor(self.boxes_num), - output_size).numpy() - expect_out = calc_psroi_pool(self.x, self.boxes, self.boxes_num, 10, - 1.0, 7, 7) + out = paddle.vision.ops.psroi_pool( + paddle.to_tensor(self.x), + paddle.to_tensor(self.boxes), + paddle.to_tensor(self.boxes_num), + output_size, + ).numpy() + expect_out = calc_psroi_pool( + self.x, self.boxes, self.boxes_num, 10, 1.0, 7, 7 + ) np.testing.assert_allclose(out, expect_out, rtol=1e-05) def test_dytype_is_float64(): @@ -192,9 +218,12 @@ def test_dytype_is_float64(): out = paddle.vision.ops.psroi_pool( paddle.to_tensor(self.x, 'float64'), paddle.to_tensor(self.boxes, 'float64'), - paddle.to_tensor(self.boxes_num, 'int32'), output_size).numpy() - expect_out = calc_psroi_pool(self.x, self.boxes, self.boxes_num, 10, - 1.0, 7, 7) + paddle.to_tensor(self.boxes_num, 'int32'), + output_size, + ).numpy() + expect_out = calc_psroi_pool( + self.x, self.boxes, self.boxes_num, 10, 1.0, 7, 7 + ) np.testing.assert_allclose(out, expect_out, rtol=1e-05) places = ['cpu'] @@ -208,41 +237,48 @@ def test_dytype_is_float64(): class TestPSROIPoolDynamicClassAPI(unittest.TestCase): - def setUp(self): self.x = np.random.random([2, 128, 32, 32]).astype(np.float32) - self.boxes = np.array([[3, 5, 6, 13], [7, 4, 22, 18], [4, 5, 7, 10], - [5, 3, 25, 21]]).astype(np.float32) + self.boxes = np.array( + [[3, 5, 6, 13], [7, 4, 22, 18], [4, 5, 7, 10], [5, 3, 25, 21]] + ).astype(np.float32) self.boxes_num = np.array([2, 2]).astype(np.int32) def test_output_size(self): - def test_output_size_is_int(): psroi_module = paddle.vision.ops.PSRoIPool(8, 1.1) - out = psroi_module(paddle.to_tensor(self.x), - paddle.to_tensor(self.boxes), - paddle.to_tensor(self.boxes_num)).numpy() - expect_out = calc_psroi_pool(self.x, self.boxes, self.boxes_num, 2, - 1.1, 8, 8) + out = psroi_module( + paddle.to_tensor(self.x), + paddle.to_tensor(self.boxes), + paddle.to_tensor(self.boxes_num), + ).numpy() + expect_out = calc_psroi_pool( + self.x, self.boxes, self.boxes_num, 2, 1.1, 8, 8 + ) np.testing.assert_allclose(out, expect_out, rtol=1e-05) def test_output_size_is_tuple(): psroi_pool_module = paddle.vision.ops.PSRoIPool(8, 1.1) - out = psroi_pool_module(paddle.to_tensor(self.x), - paddle.to_tensor(self.boxes), - paddle.to_tensor(self.boxes_num)).numpy() - expect_out = calc_psroi_pool(self.x, self.boxes, self.boxes_num, 2, - 1.1, 8, 8) + out = psroi_pool_module( + paddle.to_tensor(self.x), + paddle.to_tensor(self.boxes), + paddle.to_tensor(self.boxes_num), + ).numpy() + expect_out = calc_psroi_pool( + self.x, self.boxes, self.boxes_num, 2, 1.1, 8, 8 + ) np.testing.assert_allclose(out, expect_out, rtol=1e-05) def test_dytype_is_float64(): psroi_pool_module = paddle.vision.ops.PSRoIPool(8, 1.1) - out = psroi_pool_module(paddle.to_tensor(self.x, 'float64'), - paddle.to_tensor(self.boxes, 'float64'), - paddle.to_tensor(self.boxes_num, - 'int32')).numpy() - expect_out = calc_psroi_pool(self.x, self.boxes, self.boxes_num, 2, - 1.1, 8, 8) + out = psroi_pool_module( + paddle.to_tensor(self.x, 'float64'), + paddle.to_tensor(self.boxes, 'float64'), + paddle.to_tensor(self.boxes_num, 'int32'), + ).numpy() + expect_out = calc_psroi_pool( + self.x, self.boxes, self.boxes_num, 2, 1.1, 8, 8 + ) np.testing.assert_allclose(out, expect_out, rtol=1e-05) paddle.disable_static() @@ -257,87 +293,89 @@ def test_dytype_is_float64(): class TestPSROIPoolBoxesNumError(unittest.TestCase): - def setUp(self): paddle.disable_static() self.x = paddle.uniform([2, 490, 28, 28], dtype='float32') self.boxes = paddle.to_tensor( - [[1, 5, 8, 10], [4, 2, 6, 7], [12, 12, 19, 21]], 'float32') + [[1, 5, 8, 10], [4, 2, 6, 7], [12, 12, 19, 21]], 'float32' + ) def test_errors(self): - def test_boxes_num_nums_error(): boxes_num = paddle.to_tensor([1, 5], 'int32') - out = paddle.vision.ops.psroi_pool(self.x, - self.boxes, - boxes_num, - output_size=7) + out = paddle.vision.ops.psroi_pool( + self.x, self.boxes, boxes_num, output_size=7 + ) self.assertRaises(ValueError, test_boxes_num_nums_error) def test_boxes_num_length_error(): boxes_num = paddle.to_tensor([1, 1, 1], 'int32') - out = paddle.vision.ops.psroi_pool(self.x, - self.boxes, - boxes_num, - output_size=7) + out = paddle.vision.ops.psroi_pool( + self.x, self.boxes, boxes_num, output_size=7 + ) self.assertRaises(ValueError, test_boxes_num_length_error) class TestPSROIPoolChannelError(unittest.TestCase): - def setUp(self): paddle.disable_static() self.x = paddle.uniform([2, 490, 28, 28], dtype='float32') self.boxes = paddle.to_tensor( - [[1, 5, 8, 10], [4, 2, 6, 7], [12, 12, 19, 21]], 'float32') + [[1, 5, 8, 10], [4, 2, 6, 7], [12, 12, 19, 21]], 'float32' + ) self.output_size = 4 def test_errors(self): - def test_channel_error(): boxes_num = paddle.to_tensor([2, 1], 'int32') - out = paddle.vision.ops.psroi_pool(self.x, self.boxes, boxes_num, - self.output_size) + out = paddle.vision.ops.psroi_pool( + self.x, self.boxes, boxes_num, self.output_size + ) self.assertRaises(ValueError, test_channel_error) class TestPSROIPoolStaticAPI(unittest.TestCase): - def setUp(self): paddle.enable_static() - self.x_placeholder = paddle.static.data(name='x', - shape=[2, 490, 28, 28]) + self.x_placeholder = paddle.static.data( + name='x', shape=[2, 490, 28, 28] + ) self.x = np.random.random([2, 490, 28, 28]).astype(np.float32) - self.boxes_placeholder = paddle.static.data(name='boxes', - shape=[3, 4], - lod_level=1) - self.boxes = np.array([[1, 5, 8, 10], [4, 2, 6, 7], - [12, 12, 19, 21]]).astype(np.float32) + self.boxes_placeholder = paddle.static.data( + name='boxes', shape=[3, 4], lod_level=1 + ) + self.boxes = np.array( + [[1, 5, 8, 10], [4, 2, 6, 7], [12, 12, 19, 21]] + ).astype(np.float32) self.boxes_num = np.array([1, 2]).astype(np.int32) def test_function_in_static(self): output_size = 7 - out = paddle.vision.ops.psroi_pool(self.x_placeholder, - self.boxes_placeholder, - self.boxes_num, output_size) - expect_out = calc_psroi_pool(self.x, self.boxes, self.boxes_num, 10, - 1.0, 7, 7) + out = paddle.vision.ops.psroi_pool( + self.x_placeholder, + self.boxes_placeholder, + self.boxes_num, + output_size, + ) + expect_out = calc_psroi_pool( + self.x, self.boxes, self.boxes_num, 10, 1.0, 7, 7 + ) places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): places.append(paddle.CUDAPlace(0)) for place in places: exe = paddle.static.Executor(place) boxes_lod_data = paddle.fluid.create_lod_tensor( - self.boxes, [[1, 2]], place) - out_res, = exe.run(paddle.static.default_main_program(), - feed={ - 'x': self.x, - 'boxes': boxes_lod_data - }, - fetch_list=[out.name]) + self.boxes, [[1, 2]], place + ) + (out_res,) = exe.run( + paddle.static.default_main_program(), + feed={'x': self.x, 'boxes': boxes_lod_data}, + fetch_list=[out.name], + ) np.testing.assert_allclose(out_res, expect_out, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_pull_gpups_sparse_op.py b/python/paddle/fluid/tests/unittests/test_pull_gpups_sparse_op.py index 8bddcb6d50f5f..3b2eb69a549bf 100644 --- a/python/paddle/fluid/tests/unittests/test_pull_gpups_sparse_op.py +++ b/python/paddle/fluid/tests/unittests/test_pull_gpups_sparse_op.py @@ -30,15 +30,13 @@ def test_static_graph(self): slots = [] with fluid.program_guard(train_program, startup_program): - l = fluid.layers.data(name='input', - shape=[1], - dtype="int64", - lod_level=1) + l = fluid.layers.data( + name='input', shape=[1], dtype="int64", lod_level=1 + ) slots.append(l) - output = _pull_gpups_sparse(slots, - size=[11], - is_distributed=True, - is_sparse=True) + output = _pull_gpups_sparse( + slots, size=[11], is_distributed=True, is_sparse=True + ) cost = paddle.mean(output) sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001) sgd_optimizer.minimize(cost, train_program) @@ -49,9 +47,9 @@ def test_static_graph(self): exe = fluid.Executor(place) exe.run(startup_program) img = np.array([1]).astype(np.int64) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_put_along_axis_op.py b/python/paddle/fluid/tests/unittests/test_put_along_axis_op.py index 43771731f9b57..b5a1cb25e4d58 100644 --- a/python/paddle/fluid/tests/unittests/test_put_along_axis_op.py +++ b/python/paddle/fluid/tests/unittests/test_put_along_axis_op.py @@ -23,7 +23,6 @@ class TestPutAlongAxisOp(OpTest): - def setUp(self): self.init_data() self.reduce_op = "assign" @@ -42,7 +41,7 @@ def setUp(self): self.inputs = { 'Input': self.xnp, 'Index': self.index_broadcast, - 'Value': self.value_broadcast + 'Value': self.value_broadcast, } self.attrs = {'Axis': self.axis, 'Reduce': self.reduce_op} self.outputs = {'Result': self.target} @@ -65,7 +64,6 @@ def init_data(self): class TestPutAlongAxisAPI(unittest.TestCase): - def setUp(self): np.random.seed(0) self.shape = [1, 3] @@ -90,15 +88,18 @@ def run(place): value = paddle.fluid.data('Value', self.value_shape) out = paddle.put_along_axis(x, index, value, self.axis) exe = paddle.static.Executor(self.place[0]) - res = exe.run(feed={ - 'X': self.x_feed, - 'Value': self.value_np, - 'Index': self.index_np - }, - fetch_list=[out]) - - np.put_along_axis(self.x_np, self.index_np, self.value_np, - self.axis) + res = exe.run( + feed={ + 'X': self.x_feed, + 'Value': self.value_np, + 'Index': self.index_np, + }, + fetch_list=[out], + ) + + np.put_along_axis( + self.x_np, self.index_np, self.value_np, self.axis + ) # numpy put_along_axis is an inplace opearion. out_ref = self.x_np @@ -109,25 +110,29 @@ def run(place): run(place) def test_api_dygraph(self): - def run(place): paddle.disable_static(place) x_tensor = paddle.to_tensor(self.x_np) index_tensor = paddle.to_tensor(self.index_np) value_tensor = paddle.to_tensor(self.value_np) - out = paddle.put_along_axis(x_tensor, index_tensor, value_tensor, - self.axis) + out = paddle.put_along_axis( + x_tensor, index_tensor, value_tensor, self.axis + ) np.array( - np.put_along_axis(self.x_np, self.index_np, self.value_np, - self.axis)) + np.put_along_axis( + self.x_np, self.index_np, self.value_np, self.axis + ) + ) out_ref = self.x_np np.testing.assert_allclose(out.numpy(), out_ref, rtol=0.001) # for ci coverage, numpy put_along_axis did not support argument of 'reduce' - paddle.put_along_axis(x_tensor, index_tensor, value_tensor, - self.axis, 'mul') - paddle.put_along_axis(x_tensor, index_tensor, value_tensor, - self.axis, 'add') + paddle.put_along_axis( + x_tensor, index_tensor, value_tensor, self.axis, 'mul' + ) + paddle.put_along_axis( + x_tensor, index_tensor, value_tensor, self.axis, 'add' + ) paddle.enable_static() @@ -135,7 +140,6 @@ def run(place): run(place) def test_inplace_dygraph(self): - def run(place): paddle.disable_static(place) x_tensor = paddle.to_tensor(self.x_np) @@ -145,8 +149,10 @@ def run(place): x_tensor.put_along_axis_(index_tensor, value_tensor, self.axis) np.array( - np.put_along_axis(self.x_np, self.index_np, self.value_np, - self.axis)) + np.put_along_axis( + self.x_np, self.index_np, self.value_np, self.axis + ) + ) out_ref = self.x_np np.testing.assert_allclose(x_tensor.numpy(), out_ref, rtol=0.001) @@ -157,7 +163,6 @@ def run(place): class TestPutAlongAxisAPICase2(TestPutAlongAxisAPI): - def setUp(self): np.random.seed(0) self.shape = [2, 2] @@ -174,13 +179,13 @@ def setUp(self): class TestPutAlongAxisAPICase3(TestPutAlongAxisAPI): - def setUp(self): np.random.seed(0) self.shape = [2, 2] self.index_shape = [4, 2] - self.index_np = np.array([[0, 0], [1, 0], [0, 0], [1, - 0]]).astype('int64') + self.index_np = np.array([[0, 0], [1, 0], [0, 0], [1, 0]]).astype( + 'int64' + ) self.x_np = np.random.random(self.shape).astype(np.float32) self.place = [paddle.CPUPlace()] self.axis = 0 diff --git a/python/paddle/fluid/tests/unittests/test_py_func_op.py b/python/paddle/fluid/tests/unittests/test_py_func_op.py index 91e41a28ee54f..961216795671f 100644 --- a/python/paddle/fluid/tests/unittests/test_py_func_op.py +++ b/python/paddle/fluid/tests/unittests/test_py_func_op.py @@ -64,8 +64,9 @@ def cross_entropy_grad(logits, labels, bwd_dout): N = logits.shape[1] dlogits = np.zeros([M, N]).astype(logits.dtype) for idx in range(M): - dlogits[idx][labels[idx] - [0]] = -bwd_dout[idx] / logits[idx][labels[idx][0]] + dlogits[idx][labels[idx][0]] = ( + -bwd_dout[idx] / logits[idx][labels[idx][0]] + ) return dlogits, None @@ -75,56 +76,85 @@ def simple_fc_net(img, label, use_py_func_op): hidden = fluid.layers.fc( hidden, size=200, - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) if not use_py_func_op: hidden = fluid.layers.tanh(hidden) else: - new_hidden = fluid.default_main_program().current_block( - ).create_var(name='hidden_{}'.format(idx), - dtype='float32', - shape=hidden.shape) - hidden = fluid.layers.py_func(func=tanh, - x=hidden, - out=new_hidden, - backward_func=tanh_grad, - skip_vars_in_backward_input=hidden) + new_hidden = ( + fluid.default_main_program() + .current_block() + .create_var( + name='hidden_{}'.format(idx), + dtype='float32', + shape=hidden.shape, + ) + ) + hidden = fluid.layers.py_func( + func=tanh, + x=hidden, + out=new_hidden, + backward_func=tanh_grad, + skip_vars_in_backward_input=hidden, + ) prediction = fluid.layers.fc(hidden, size=10, act='softmax') if not use_py_func_op: loss = fluid.layers.cross_entropy(input=prediction, label=label) else: - loss = fluid.default_main_program().current_block().create_var( - name='loss', dtype='float32', shape=[-1, 1]) - loss = fluid.layers.py_func(func=cross_entropy, - x=[prediction, label], - out=loss, - backward_func=cross_entropy_grad, - skip_vars_in_backward_input=loss) - - dummy_var = fluid.default_main_program().current_block().create_var( - name='test_tmp_var', dtype='float32', shape=[1]) - fluid.layers.py_func(func=dummy_func_with_no_input, - x=None, - out=dummy_var) + loss = ( + fluid.default_main_program() + .current_block() + .create_var(name='loss', dtype='float32', shape=[-1, 1]) + ) + loss = fluid.layers.py_func( + func=cross_entropy, + x=[prediction, label], + out=loss, + backward_func=cross_entropy_grad, + skip_vars_in_backward_input=loss, + ) + + dummy_var = ( + fluid.default_main_program() + .current_block() + .create_var(name='test_tmp_var', dtype='float32', shape=[1]) + ) + fluid.layers.py_func( + func=dummy_func_with_no_input, x=None, out=dummy_var + ) loss += dummy_var fluid.layers.py_func(func=dummy_func_with_no_output, x=loss, out=None) - loss_out = fluid.default_main_program().current_block().create_var( - dtype='float32', shape=[-1, 1]) - dummy_var_out = fluid.default_main_program().current_block().create_var( - dtype='float32', shape=[1]) - fluid.layers.py_func(func=dummy_func_with_multi_input_output, - x=(loss, dummy_var), - out=(loss_out, dummy_var_out)) - assert loss == loss_out and dummy_var == dummy_var_out, \ - "py_func failed with multi input and output" - - fluid.layers.py_func(func=dummy_func_with_multi_input_output, - x=[loss, dummy_var], - out=[loss_out, dummy_var_out]) - assert loss == loss_out and dummy_var == dummy_var_out, \ - "py_func failed with multi input and output" + loss_out = ( + fluid.default_main_program() + .current_block() + .create_var(dtype='float32', shape=[-1, 1]) + ) + dummy_var_out = ( + fluid.default_main_program() + .current_block() + .create_var(dtype='float32', shape=[1]) + ) + fluid.layers.py_func( + func=dummy_func_with_multi_input_output, + x=(loss, dummy_var), + out=(loss_out, dummy_var_out), + ) + assert ( + loss == loss_out and dummy_var == dummy_var_out + ), "py_func failed with multi input and output" + + fluid.layers.py_func( + func=dummy_func_with_multi_input_output, + x=[loss, dummy_var], + out=[loss_out, dummy_var_out], + ) + assert ( + loss == loss_out and dummy_var == dummy_var_out + ), "py_func failed with multi input and output" loss = paddle.mean(loss) return loss @@ -132,9 +162,9 @@ def simple_fc_net(img, label, use_py_func_op): def reader(): for _ in range(dev_cnt * 100): - yield np.random.random([784]), np.random.random_integers(size=[1], - low=0, - high=9) + yield np.random.random([784]), np.random.random_integers( + size=[1], low=0, high=9 + ) def test_main(use_cuda, use_py_func_op, use_parallel_executor): @@ -162,7 +192,8 @@ def test_main(use_cuda, use_py_func_op, use_parallel_executor): if use_parallel_executor: train_cp = compiler.CompiledProgram( - fluid.default_main_program()) + fluid.default_main_program() + ) train_cp = train_cp.with_data_parallel(loss_name=loss.name) fetch_list = [loss.name] else: @@ -171,15 +202,14 @@ def test_main(use_cuda, use_py_func_op, use_parallel_executor): ret = [] for epoch_id in range(2): for d in r(): - L, = exe.run(train_cp, - feed=feeder.feed(d), - fetch_list=fetch_list) + (L,) = exe.run( + train_cp, feed=feeder.feed(d), fetch_list=fetch_list + ) ret.append(L) return np.array(ret) class TestPyFuncOpUseExecutor(unittest.TestCase): - def setUp(self): self.use_parallel_executor = False @@ -187,8 +217,9 @@ def test_loss_diff(self): for use_cuda in [True, False]: losses = [] for use_py_func_op in [True, False]: - L = test_main(use_cuda, use_py_func_op, - self.use_parallel_executor) + L = test_main( + use_cuda, use_py_func_op, self.use_parallel_executor + ) if L is not None: losses.append(L) @@ -198,7 +229,6 @@ def test_loss_diff(self): class TestPyFuncOpUseParallelExecutor(TestPyFuncOpUseExecutor): - def setUp(self): self.use_parallel_executor = True diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_combination.py b/python/paddle/fluid/tests/unittests/test_py_reader_combination.py index 84ea5f5f7bb7e..1c399b89cec57 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_combination.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_combination.py @@ -19,23 +19,20 @@ class TestPyReaderCombination(unittest.TestCase): - def setUp(self): self.n1 = 10 self.n2 = 20 self.batch_size = 2 def create_reader(self, batch_num): - def __impl__(): for _ in range(batch_num): - image = np.random.uniform(low=-1, high=1, - size=[batch_num, - 784]).astype('float32') - label = np.random.random_integers(low=0, - high=9, - size=[batch_num, - 1]).astype('int64') + image = np.random.uniform( + low=-1, high=1, size=[batch_num, 784] + ).astype('float32') + label = np.random.random_integers( + low=0, high=9, size=[batch_num, 1] + ).astype('int64') yield image, label return __impl__ @@ -57,17 +54,17 @@ def _reset_iterable_reader(self, py_reader): def main_impl(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - image = fluid.layers.data(name='image', - dtype='float32', - shape=[784]) + image = fluid.layers.data( + name='image', dtype='float32', shape=[784] + ) label = fluid.layers.data(name='label', dtype='int64', shape=[1]) - py_reader1 = fluid.io.PyReader(feed_list=[image, label], - capacity=16, - iterable=True) - py_reader2 = fluid.io.PyReader(feed_list=[image, label], - capacity=16, - iterable=True) + py_reader1 = fluid.io.PyReader( + feed_list=[image, label], capacity=16, iterable=True + ) + py_reader2 = fluid.io.PyReader( + feed_list=[image, label], capacity=16, iterable=True + ) reader1 = paddle.reader.cache(self.create_reader(self.n1)) reader2 = paddle.reader.cache(self.create_reader(self.n2)) @@ -77,8 +74,12 @@ def main_impl(self, place): for _ in range(10): max_num = min(self.n1, self.n2) batch_num = 0 - for reader_np1, py_reader_dict1, reader_np2, py_reader_dict2 in zip( - reader1(), py_reader1(), reader2(), py_reader2()): + for ( + reader_np1, + py_reader_dict1, + reader_np2, + py_reader_dict2, + ) in zip(reader1(), py_reader1(), reader2(), py_reader2()): self.assertFeedVarEqual(reader_np1, py_reader_dict1) self.assertFeedVarEqual(reader_np2, py_reader_dict2) batch_num += 1 @@ -100,7 +101,6 @@ def test_main(self): class TestPyReaderCombination2(TestPyReaderCombination): - def setUp(self): self.n1 = 20 self.n2 = 10 @@ -108,7 +108,6 @@ def setUp(self): class TestPyReaderCombination3(TestPyReaderCombination): - def setUp(self): self.n1 = 10 self.n2 = 10 diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_error_msg.py b/python/paddle/fluid/tests/unittests/test_py_reader_error_msg.py index 337cafbb1246f..40cdcdcc19f3a 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_error_msg.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_error_msg.py @@ -19,32 +19,38 @@ class TestPyReaderErrorMsg(unittest.TestCase): - def test_check_input_array(self): - fluid.reader.GeneratorLoader._check_input_array([ - np.random.randint(100, size=[2]), - np.random.randint(100, size=[2]), - np.random.randint(100, size=[2]) - ]) - self.assertRaises(TypeError, - fluid.reader.GeneratorLoader._check_input_array, [ - np.random.randint(100, size=[2]), - np.random.randint(100, size=[1]), - np.random.randint(100, size=[3]) - ]) + fluid.reader.GeneratorLoader._check_input_array( + [ + np.random.randint(100, size=[2]), + np.random.randint(100, size=[2]), + np.random.randint(100, size=[2]), + ] + ) + self.assertRaises( + TypeError, + fluid.reader.GeneratorLoader._check_input_array, + [ + np.random.randint(100, size=[2]), + np.random.randint(100, size=[1]), + np.random.randint(100, size=[3]), + ], + ) class TestDoubleBufferAPI(unittest.TestCase): - def test_double_buffer(self): paddle.enable_static() if fluid.core.is_compiled_with_cuda(): - reader = fluid.layers.py_reader(capacity=64, - shapes=[(-1, 1, 28, 28), (-1, 1)], - dtypes=['float32', 'int64'], - use_double_buffer=False) - reader = fluid.layers.double_buffer(reader, - place=fluid.core.CUDAPlace(0)) + reader = fluid.layers.py_reader( + capacity=64, + shapes=[(-1, 1, 28, 28), (-1, 1)], + dtypes=['float32', 'int64'], + use_double_buffer=False, + ) + reader = fluid.layers.double_buffer( + reader, place=fluid.core.CUDAPlace(0) + ) image, label = fluid.layers.read_file(reader) diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_lod_level_share.py b/python/paddle/fluid/tests/unittests/test_py_reader_lod_level_share.py index 4b5e2b9711ee1..ef55a226dea43 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_lod_level_share.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_lod_level_share.py @@ -17,7 +17,6 @@ class TestLoDLevelShare(unittest.TestCase): - def setUp(self): self.use_double_buffer = False @@ -27,7 +26,8 @@ def test_lod_level_share(self): shapes=([-1, 256], [-1, 512], [-1, 100]), dtypes=('float32', 'int64', 'double'), lod_levels=(1, 2, 0), - use_double_buffer=self.use_double_buffer) + use_double_buffer=self.use_double_buffer, + ) x, y, z = fluid.layers.read_file(reader) self.assertEqual(x.lod_level, 1) @@ -36,7 +36,6 @@ def test_lod_level_share(self): class TestLoDLevelShare2(TestLoDLevelShare): - def setUp(self): self.use_double_buffer = True diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_pin_memory.py b/python/paddle/fluid/tests/unittests/test_py_reader_pin_memory.py index 596bc2ff83942..4268d6fbcc196 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_pin_memory.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_pin_memory.py @@ -20,7 +20,6 @@ def user_reader(inputs): - def _reader(): for d in inputs: yield d @@ -29,7 +28,6 @@ def _reader(): def batch_feeder(batch_reader, pin_memory=False, img_dtype="float32"): - def _feeder(): for batch_data in batch_reader(): sample_batch = [] @@ -48,7 +46,6 @@ def _feeder(): class TestPyReader(unittest.TestCase): - def setUp(self): self.capacity = 10 self.shapes = [(-1, 3, 2, 1), (-1, 1)] @@ -57,29 +54,37 @@ def setUp(self): def test_pin_memory_pyreader(self): with fluid.program_guard(fluid.Program(), fluid.Program()): - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) executor = fluid.Executor(place) - data_file = fluid.layers.py_reader(capacity=self.capacity, - dtypes=self.dtypes, - lod_levels=self.lod_levels, - shapes=self.shapes) + data_file = fluid.layers.py_reader( + capacity=self.capacity, + dtypes=self.dtypes, + lod_levels=self.lod_levels, + shapes=self.shapes, + ) # feed_queue = data_file.queue read_out_data = fluid.layers.read_file(data_file) self.inputs = [] for _ in range(10): - sample = np.random.uniform(low=0, high=1, - size=[3, 2, 1]).astype("float32") + sample = np.random.uniform( + low=0, high=1, size=[3, 2, 1] + ).astype("float32") label = np.random.randint(low=0, high=10, dtype="int64") self.inputs.append((sample, label)) self.input_tensors = [] for d, l in batch_feeder( - paddle.batch(user_reader(self.inputs), batch_size=2), - pin_memory=True - if fluid.core.is_compiled_with_cuda() else False)(): + paddle.batch(user_reader(self.inputs), batch_size=2), + pin_memory=True + if fluid.core.is_compiled_with_cuda() + else False, + )(): ta = fluid.LoDTensorArray() ta.append(d) ta.append(l) @@ -95,10 +100,13 @@ def test_pin_memory_pyreader(self): self.batched_inputs.append([feed_d, feed_l]) data_file.decorate_tensor_provider( - batch_feeder(paddle.batch(user_reader(self.inputs), - batch_size=2), - pin_memory=True - if fluid.core.is_compiled_with_cuda() else False)) + batch_feeder( + paddle.batch(user_reader(self.inputs), batch_size=2), + pin_memory=True + if fluid.core.is_compiled_with_cuda() + else False, + ) + ) executor.run(fluid.default_startup_program()) self.outputs = [] @@ -106,14 +114,16 @@ def test_pin_memory_pyreader(self): data_file.start() for _ in self.input_tensors: self.outputs.append( - executor.run(fetch_list=list(read_out_data))) + executor.run(fetch_list=list(read_out_data)) + ) data_file.reset() self.validate() def validate(self): self.assertEqual(len(self.batched_inputs), len(self.outputs)) - for in_data_list, out_data_list in zip(self.batched_inputs, - self.outputs): + for in_data_list, out_data_list in zip( + self.batched_inputs, self.outputs + ): self.assertEqual(len(in_data_list), len(out_data_list)) in_data_list_np = [ np.array(in_lod_tensor) for in_lod_tensor in in_data_list diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py b/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py index cdca6695e947b..4ccaaf0274669 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py @@ -24,7 +24,6 @@ def feed_data(feed_queue, inputs): class TestPyReader(unittest.TestCase): - def setUp(self): self.capacity = 10 self.batch_size_min = 10 @@ -42,46 +41,55 @@ def test_multiple_thread_main(self): def main(self, use_thread=False): with fluid.program_guard(fluid.Program(), fluid.Program()): - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) executor = fluid.Executor(place) - data_file = fluid.layers.py_reader(capacity=self.capacity, - dtypes=self.dtypes, - lod_levels=self.lod_levels, - shapes=self.shapes) + data_file = fluid.layers.py_reader( + capacity=self.capacity, + dtypes=self.dtypes, + lod_levels=self.lod_levels, + shapes=self.shapes, + ) feed_queue = data_file.queue read_out_data = fluid.layers.read_file(data_file) self.inputs = [] for i in range(self.iterations): in_data = fluid.LoDTensorArray() - batch_size = np.random.random_integers(self.batch_size_min, - self.batch_size_max) + batch_size = np.random.random_integers( + self.batch_size_min, self.batch_size_max + ) for shape, dtype in zip(self.shapes, self.dtypes): - next_data = np.random.uniform(low=0, - high=1000, - size=(batch_size, ) + - shape[1:]).astype(dtype) + next_data = np.random.uniform( + low=0, high=1000, size=(batch_size,) + shape[1:] + ).astype(dtype) in_data.append( - fluid.executor._as_lodtensor(next_data, place)) + fluid.executor._as_lodtensor(next_data, place) + ) self.inputs.append(in_data) executor.run(fluid.default_startup_program()) self.outputs = [] if use_thread: - thread = Thread(target=feed_data, - args=(feed_queue, self.inputs)) + thread = Thread( + target=feed_data, args=(feed_queue, self.inputs) + ) thread.start() for in_data in self.inputs: self.outputs.append( - executor.run(fetch_list=list(read_out_data))) + executor.run(fetch_list=list(read_out_data)) + ) else: for in_data in self.inputs: feed_queue.push(in_data) self.outputs.append( - executor.run(fetch_list=list(read_out_data))) + executor.run(fetch_list=list(read_out_data)) + ) feed_queue.close() self.validate() diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_return_list.py b/python/paddle/fluid/tests/unittests/test_py_reader_return_list.py index c6e951997eabf..bfc08245ee33b 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_return_list.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_return_list.py @@ -19,39 +19,40 @@ class TestPyReader(unittest.TestCase): - def setUp(self): self.batch_size = 32 self.epoch_num = 2 self.sample_num = 10 def test_returnlist(self): - def reader_creator_random_image(height, width): - def reader(): for i in range(self.sample_num): - yield np.random.uniform(low=0, - high=255, - size=[height, width]), + yield np.random.uniform( + low=0, high=255, size=[height, width] + ), return reader for return_list in [True, False]: with fluid.program_guard(fluid.Program(), fluid.Program()): - image = fluid.layers.data(name='image', - shape=[784, 784], - dtype='float32') - reader = fluid.io.PyReader(feed_list=[image], - capacity=4, - iterable=True, - return_list=return_list) + image = fluid.layers.data( + name='image', shape=[784, 784], dtype='float32' + ) + reader = fluid.io.PyReader( + feed_list=[image], + capacity=4, + iterable=True, + return_list=return_list, + ) user_defined_reader = reader_creator_random_image(784, 784) reader.decorate_sample_list_generator( - paddle.batch(user_defined_reader, - batch_size=self.batch_size), - fluid.core.CPUPlace()) + paddle.batch( + user_defined_reader, batch_size=self.batch_size + ), + fluid.core.CPUPlace(), + ) # definition of network is omitted executor = fluid.Executor(fluid.core.CPUPlace()) executor.run(fluid.default_main_program()) @@ -69,7 +70,8 @@ def reader(): batch_py_reader.decorate_sample_generator( user_defined_reader, batch_size=self.batch_size, - places=fluid.core.CPUPlace()) + places=fluid.core.CPUPlace(), + ) for epoch in range(self.epoch_num): for _, data in enumerate(batch_py_reader()): diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_sample_generator.py b/python/paddle/fluid/tests/unittests/test_py_reader_sample_generator.py index 7f0cf633ed25c..fa1297d46b6b9 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_sample_generator.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_sample_generator.py @@ -23,18 +23,18 @@ def random_reader(sample_num): - def __impl__(): for _ in range(sample_num): - yield np.random.random( - size=[784]).astype('float32'), np.random.random_integers( - low=0, high=9, size=[1]).astype('int64') + yield np.random.random(size=[784]).astype( + 'float32' + ), np.random.random_integers(low=0, high=9, size=[1]).astype( + 'int64' + ) return paddle.reader.cache(__impl__) class TestCaseBase(unittest.TestCase): - def setUp(self): self.batch_size = 32 self.epoch_num = 2 @@ -54,22 +54,24 @@ def generate_all_data(self, reader): def run_main(self, reader, use_sample_generator, iterable, drop_last): image = fluid.layers.data(name='image', dtype='float32', shape=[784]) label = fluid.layers.data(name='label', dtype='int64', shape=[1]) - py_reader = fluid.io.PyReader(feed_list=[image, label], - capacity=16, - iterable=iterable, - use_double_buffer=False) + py_reader = fluid.io.PyReader( + feed_list=[image, label], + capacity=16, + iterable=iterable, + use_double_buffer=False, + ) batch_reader = paddle.batch(reader, self.batch_size, drop_last) all_datas = self.generate_all_data(batch_reader) if not use_sample_generator: - py_reader.decorate_sample_list_generator(batch_reader, - places=fluid.cpu_places()) + py_reader.decorate_sample_list_generator( + batch_reader, places=fluid.cpu_places() + ) else: - py_reader.decorate_sample_generator(reader, - self.batch_size, - drop_last, - places=fluid.cpu_places()) + py_reader.decorate_sample_generator( + reader, self.batch_size, drop_last, places=fluid.cpu_places() + ) if drop_last: batch_num = int(self.sample_num / self.batch_size) @@ -111,12 +113,12 @@ def test_main(self): for iterable in [False, True]: for drop_last in [False, True]: with fluid.program_guard(fluid.Program(), fluid.Program()): - self.run_main(reader, use_sample_generator, iterable, - drop_last) + self.run_main( + reader, use_sample_generator, iterable, drop_last + ) class TestCase1(TestCaseBase): - def setUp(self): self.batch_size = 32 self.epoch_num = 10 @@ -124,7 +126,6 @@ def setUp(self): class TestCase2(TestCaseBase): - def setUp(self): self.batch_size = 32 self.epoch_num = 2 @@ -132,7 +133,6 @@ def setUp(self): class TestCase3(TestCaseBase): - def setUp(self): self.batch_size = 32 self.epoch_num = 2 diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py b/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py index a0ab41e4fe24f..dca3b061cbff0 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py @@ -39,8 +39,11 @@ def as_tensor(np_array_or_tensor, place=None): def as_numpy(tensor_or_numpy): - return tensor_or_numpy if isinstance( - tensor_or_numpy, np.ndarray) else np.array(tensor_or_numpy) + return ( + tensor_or_numpy + if isinstance(tensor_or_numpy, np.ndarray) + else np.array(tensor_or_numpy) + ) def sample_list_to_tensor_array(sample_list): @@ -80,13 +83,15 @@ def feed_data(feed_queue, batch_reader): feed_queue.close() -def simple_fc_net(in_size, - class_num, - hidden_sizes, - batch_size, - queue_capacity, - use_double_buffer=False, - use_feed_list=True): +def simple_fc_net( + in_size, + class_num, + hidden_sizes, + batch_size, + queue_capacity, + use_double_buffer=False, + use_feed_list=True, +): in_data = fluid.layers.data(name="data", dtype='float32', shape=[in_size]) label = fluid.layers.data(name='label', dtype='int64', shape=[1]) if use_feed_list: @@ -94,14 +99,16 @@ def simple_fc_net(in_size, capacity=queue_capacity, use_double_buffer=use_double_buffer, feed_list=[in_data, label], - name=unique_name.generate('py_reader_name')) + name=unique_name.generate('py_reader_name'), + ) else: py_reader = fluid.layers.py_reader( capacity=queue_capacity, shapes=[in_data.shape, label.shape], dtypes=['float32', 'int64'], name=unique_name.generate('py_reader_name'), - use_double_buffer=use_double_buffer) + use_double_buffer=use_double_buffer, + ) in_data, label = fluid.layers.read_file(py_reader) @@ -113,12 +120,15 @@ def simple_fc_net(in_size, hidden, size=hidden_size, act='tanh', - bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0))) + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ), + ) predict_label = fluid.layers.fc(hidden, size=class_num, act='softmax') loss = paddle.mean( - fluid.layers.cross_entropy(input=predict_label, label=label)) + fluid.layers.cross_entropy(input=predict_label, label=label) + ) optimizer = fluid.optimizer.Adam() optimizer.minimize(loss) @@ -126,7 +136,6 @@ def simple_fc_net(in_size, class TestPyReaderUsingExecutor(unittest.TestCase): - def setUp(self): self.in_size = 1000 self.hidden_sizes = [50, 30, 20] @@ -136,39 +145,42 @@ def setUp(self): self.queue_capacity = 50 def test(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): for use_parallel_executor in [False, True]: for use_double_buffer in [False, True]: for use_feed_list in [False, True]: for use_decorate_paddle_reader in [False, True]: print('Test Parameters:'), - print({ - 'use_cuda': + print( + { + 'use_cuda': use_cuda, + 'use_parallel_executor': use_parallel_executor, + 'use_double_buffer': use_double_buffer, + 'use_feed_list': use_feed_list, + 'use_decorate_paddle_reader': use_decorate_paddle_reader, + } + ) + self.main( use_cuda, - 'use_parallel_executor': use_parallel_executor, - 'use_double_buffer': use_double_buffer, - 'use_feed_list': use_feed_list, - 'use_decorate_paddle_reader': - use_decorate_paddle_reader - }) - self.main(use_cuda, use_parallel_executor, - use_double_buffer, use_feed_list, - use_decorate_paddle_reader) + use_decorate_paddle_reader, + ) def tensor_reader(self, use_decorate_paddle_reader): - def reader(): - for sample_id in range(self.batch_size * self.iterations * - self.batch_size_times): + for sample_id in range( + self.batch_size * self.iterations * self.batch_size_times + ): in_data = np.random.uniform( - low=0, high=1, size=(self.in_size, )).astype('float32') - label = np.random.random_integers(low=0, - high=self.class_num - 1, - size=(1, )).astype('int64') + low=0, high=1, size=(self.in_size,) + ).astype('float32') + label = np.random.random_integers( + low=0, high=self.class_num - 1, size=(1,) + ).astype('int64') reshaped_in_data = np.reshape(in_data, [1, -1]) reshaped_label = np.reshape(label, [1, -1]) @@ -176,9 +188,11 @@ def reader(): self.inputs.append([reshaped_in_data, reshaped_label]) else: self.inputs[-1][0] = np.concatenate( - (self.inputs[-1][0], reshaped_in_data), axis=0) + (self.inputs[-1][0], reshaped_in_data), axis=0 + ) self.inputs[-1][1] = np.concatenate( - (self.inputs[-1][1], reshaped_label), axis=0) + (self.inputs[-1][1], reshaped_label), axis=0 + ) yield in_data, label @@ -187,12 +201,14 @@ def reader(): return reader - def main(self, - use_cuda=True, - use_parallel_executor=False, - use_double_buffer=False, - use_feed_list=False, - use_decorate_paddle_reader=False): + def main( + self, + use_cuda=True, + use_parallel_executor=False, + use_double_buffer=False, + use_feed_list=False, + use_decorate_paddle_reader=False, + ): assert not use_cuda or use_cuda and core.is_compiled_with_cuda() self.use_cuda = use_cuda @@ -205,14 +221,22 @@ def main(self, main_program = fluid.Program() with fluid.program_guard(main_program, startup_program): - in_data, label, loss, optimizer, feed_queue, py_reader = simple_fc_net( + ( + in_data, + label, + loss, + optimizer, + feed_queue, + py_reader, + ) = simple_fc_net( in_size=self.in_size, class_num=self.class_num, hidden_sizes=self.hidden_sizes, batch_size=self.batch_size, queue_capacity=self.queue_capacity, use_double_buffer=self.use_double_buffer, - use_feed_list=self.use_feed_list) + use_feed_list=self.use_feed_list, + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() @@ -222,12 +246,14 @@ def main(self, train_cp = main_program if use_parallel_executor: train_cp = compiler.CompiledProgram( - main_program).with_data_parallel(loss_name=loss.name) + main_program + ).with_data_parallel(loss_name=loss.name) if use_cuda: self.batch_size_times = core.get_cuda_device_count() else: self.batch_size_times = int( - os.environ.get('CPU_NUM', multiprocessing.cpu_count())) + os.environ.get('CPU_NUM', multiprocessing.cpu_count()) + ) else: self.batch_size_times = 1 @@ -244,15 +270,17 @@ def main(self, py_reader.decorate_sample_list_generator(batch_reader) py_reader.start() else: - thread = threading.Thread(target=feed_data, - args=(feed_queue, batch_reader)) + thread = threading.Thread( + target=feed_data, args=(feed_queue, batch_reader) + ) thread.daemon = True thread.start() try: while True: - fetches = exe.run(train_cp, - fetch_list=[in_data.name, label.name]) + fetches = exe.run( + train_cp, fetch_list=[in_data.name, label.name] + ) fetches = [as_numpy(fetch) for fetch in fetches] self.outputs.append(fetches) except fluid.core.EOFException: diff --git a/python/paddle/fluid/tests/unittests/test_pylayer_op.py b/python/paddle/fluid/tests/unittests/test_pylayer_op.py index 6a959dab07e86..becab9796dff3 100644 --- a/python/paddle/fluid/tests/unittests/test_pylayer_op.py +++ b/python/paddle/fluid/tests/unittests/test_pylayer_op.py @@ -21,17 +21,13 @@ class FakeTensor(paddle.fluid.core.VarBase): - def __init__(self): pass class TestPyLayer(unittest.TestCase): - def func_test_simple_pylayer_multiple_output(self): - class tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x1, x2, func1, func2=paddle.square): ctx.func = func2 @@ -59,7 +55,8 @@ def backward(ctx, dy1, dy2): z2.mean().backward() self.assertTrue( - np.max(np.abs((input1.grad.numpy() - input2.grad.numpy()))) < 1e-10) + np.max(np.abs((input1.grad.numpy() - input2.grad.numpy()))) < 1e-10 + ) def test_simple_pylayer_multiple_output(self): with _test_eager_guard(): @@ -67,9 +64,7 @@ def test_simple_pylayer_multiple_output(self): self.func_test_simple_pylayer_multiple_output() def func_test_simple_pylayer_return_none_with_no_grad(self): - class tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x1, x2, func1, func2=paddle.square): ctx.func = func2 @@ -101,7 +96,8 @@ def backward(ctx, dy1, dy2): z2.mean().backward() self.assertTrue( - np.max(np.abs((input1.grad.numpy() - input2.grad.numpy()))) < 1e-10) + np.max(np.abs((input1.grad.numpy() - input2.grad.numpy()))) < 1e-10 + ) def test_simple_pylayer_return_none_with_no_grad(self): with _test_eager_guard(): @@ -109,9 +105,7 @@ def test_simple_pylayer_return_none_with_no_grad(self): self.func_test_simple_pylayer_return_none_with_no_grad() def func_test_simple_pylayer_single_output(self): - class tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x1, func1, func2=paddle.square): ctx.func = func2 @@ -121,7 +115,7 @@ def forward(ctx, x1, func1, func2=paddle.square): @staticmethod def backward(ctx, dy1): - y1, = ctx.saved_tensor() + (y1,) = ctx.saved_tensor() re1 = dy1 * (1 - ctx.func(y1)) return re1 @@ -135,7 +129,8 @@ def backward(ctx, dy1): z2.mean().backward() self.assertTrue( - np.max(np.abs((input1.grad.numpy() - input2.grad.numpy()))) < 1e-10) + np.max(np.abs((input1.grad.numpy() - input2.grad.numpy()))) < 1e-10 + ) def test_simple_pylayer_single_output(self): with _test_eager_guard(): @@ -143,9 +138,7 @@ def test_simple_pylayer_single_output(self): self.func_test_simple_pylayer_single_output() def func_test_pylayer_num_output_match(self): - class tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward( ctx, @@ -172,9 +165,7 @@ def test_pylayer_num_output_match(self): self.func_test_pylayer_num_output_match() def func_test_pylayer_dtype(self): - class tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x, dtype): y = paddle.cast(x, dtype) @@ -185,10 +176,16 @@ def backward(ctx, dy1): return dy1 dtypes = [ - 'bool', 'float16', 'float32', 'float64', 'uint8', 'int32', 'int64' + 'bool', + 'float16', + 'float32', + 'float64', + 'uint8', + 'int32', + 'int64', ] for dtype in dtypes: - input1 = (paddle.randn([2, 3])) + input1 = paddle.randn([2, 3]) input1.stop_gradient = False self.assertTrue(input1.grad is None) @@ -203,9 +200,7 @@ def test_pylayer_dtype(self): self.func_test_pylayer_dtype() def func_test_pylayer_Exception_forward(self): - class Layer_None1(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, *args): return None @@ -219,7 +214,6 @@ def backward(ctx, *args): z = Layer_None1.apply(input1) class Layer_None2(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, *args): return [None, args[0]] @@ -233,7 +227,6 @@ def backward(ctx, *args): z = Layer_None2.apply(input1) class Layer_one1(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, *args): return 1 @@ -248,7 +241,6 @@ def backward(ctx, *args): z = Layer_one1.apply(input1) class Layer_one2(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, *args): return [1, 2, args[0]] @@ -262,7 +254,6 @@ def backward(ctx, *args): z = Layer_one2.apply(input1) class Layer_no_fw(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def backward(ctx, *args): return args @@ -277,9 +268,7 @@ def test_pylayer_Exception_forward(self): self.func_test_pylayer_Exception_forward() def func_test_pylayer_nograd(self): - class tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x1, func1, func2=paddle.square, xx=None): ctx.func = func2 @@ -302,10 +291,9 @@ def test_pylayer_nograd(self): self.func_test_pylayer_nograd() def func_test_pylayer_Exception_bk(self): - class Layer_bk_none1( - EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x): return x * 2 @@ -322,8 +310,8 @@ def backward(ctx, dy1): z.sum().backward() class Layer_bk_none2( - EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x1, x2): return x1 + x2 @@ -339,9 +327,9 @@ def backward(ctx, dy1): with self.assertRaises(ValueError): z.mean().backward() - class Layer_bk_one1(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer - ): - + class Layer_bk_one1( + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x): return x + x @@ -357,9 +345,9 @@ def backward(ctx, dy): with self.assertRaises(ValueError): z.mean().backward() - class Layer_bk_one2(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer - ): - + class Layer_bk_one2( + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x1, x2): return x1 * 2, x2 * 5 @@ -377,7 +365,6 @@ def backward(ctx, *args): z.mean().backward() class Layer_no_bk(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x): return x * 2, x * 5 @@ -391,8 +378,8 @@ def forward(ctx, x): z.mean().backward() class Layer_bk_match( - EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x): return x * 2, x * 5 @@ -414,10 +401,9 @@ def test_pylayer_Exception_bk(self): self.func_test_pylayer_Exception_bk() def func_test_pylayer_bk_return_none(self): - class Layer_bk_none1( - EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x1, x2): return x1 + x2 @@ -436,8 +422,8 @@ def backward(ctx, dy): z.mean().backward() class Layer_bk_none2( - EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x1, x2): return x1 * 2, x2 * 5 @@ -461,9 +447,7 @@ def test_pylayer_bk_return_none(self): self.func_test_pylayer_bk_return_none() def func_test_pylayer_inplace(self): - class cus_tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x): return x @@ -473,7 +457,6 @@ def backward(ctx, dy): return dy class Layer(paddle.nn.Layer): - def __init__(self): super(Layer, self).__init__() @@ -499,9 +482,9 @@ def test_pylayer_inplace(self): def test_pylayer_inplace_backward_error(self): with _test_eager_guard(): - class cus_tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer - ): - + class cus_tanh( + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x): return x @@ -511,7 +494,6 @@ def backward(ctx, dy): return dy class Layer(paddle.nn.Layer): - def __init__(self): super(Layer, self).__init__() @@ -527,17 +509,19 @@ def forward(self, data): layer = Layer() z = layer(data) with self.assertRaisesRegexp( - RuntimeError, - "received tensor_version:{} != wrapper_version_snapshot:{}". - format(1, 0)): + RuntimeError, + "received tensor_version:{} != wrapper_version_snapshot:{}".format( + 1, 0 + ), + ): z.backward() def test_pylayer_inplace_backward_success_1(self): with _test_eager_guard(): - class cus_tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer - ): - + class cus_tanh( + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x): return x @@ -547,7 +531,6 @@ def backward(ctx, dy): return dy class Layer(paddle.nn.Layer): - def __init__(self): super(Layer, self).__init__() @@ -569,9 +552,9 @@ def forward(self, data): def test_pylayer_inplace_backward_success_2(self): with _test_eager_guard(): - class cus_tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer - ): - + class cus_tanh( + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x): return x @@ -581,7 +564,6 @@ def backward(ctx, dy): return dy class Layer(paddle.nn.Layer): - def __init__(self): super(Layer, self).__init__() @@ -601,10 +583,9 @@ def forward(self, data): self.assertTrue(data.grad is not None) def func_test_pylayer_inplace_and_leaf_exception(self): - class cus_pylayer_op( - EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - + EagerPyLayer if in_dygraph_mode() else LegacyPyLayer + ): @staticmethod def forward(ctx, x): return x @@ -614,7 +595,6 @@ def backward(ctx, dy): return dy class Layer(paddle.nn.Layer): - def __init__(self): super(Layer, self).__init__() @@ -636,9 +616,7 @@ def test_pylayer_inplace_and_leaf_exception(self): self.func_test_pylayer_inplace_and_leaf_exception() def func_test_backward_in_backward(self): - class cus_tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x): temp = x.detach() @@ -668,9 +646,7 @@ def test_backward_in_backward(self): self.func_test_backward_in_backward() def func_test_return_to_tensor(self): - class Tanh(EagerPyLayer if in_dygraph_mode() else LegacyPyLayer): - @staticmethod def forward(ctx, x1): y1 = paddle.tanh(x1) @@ -680,7 +656,7 @@ def forward(ctx, x1): @staticmethod def backward(ctx, dy1, dy2): - y1, = ctx.saved_tensor() + (y1,) = ctx.saved_tensor() re1 = dy1 * (1 - paddle.square(y1)) return dy1 @@ -700,7 +676,6 @@ def test_materialize_grads(self): with _test_eager_guard(): class Tanh(EagerPyLayer): - @staticmethod def forward(ctx, x): ctx.mark_not_inplace(x) @@ -719,7 +694,6 @@ def test_dont_materialize_grads(self): with _test_eager_guard(): class Tanh(EagerPyLayer): - @staticmethod def forward(ctx, x): ctx.mark_not_inplace(x) @@ -739,7 +713,6 @@ def test_mark_non_differentiable(self): with _test_eager_guard(): class Tanh(EagerPyLayer): - @staticmethod def forward(ctx, x): a = x + x @@ -760,7 +733,6 @@ def test_mark_non_differentiable2(self): with _test_eager_guard(): class Tanh(EagerPyLayer): - @staticmethod def forward(ctx, x): a = x + x @@ -782,11 +754,8 @@ def backward(ctx, grad_a, grad_b): class TestPyLayerReturnType(unittest.TestCase): - def test_forward_args_fake_tensor(self): - class Tanh(LegacyPyLayer): - @staticmethod def forward(ctx, x1): y1 = FakeTensor() @@ -802,9 +771,7 @@ def backward(ctx, dy1, dy2): y1, y2 = Tanh.apply(input1) def test_forward_kwargs_fake_tensor(self): - class Tanh(LegacyPyLayer): - @staticmethod def forward(ctx, x1): @@ -820,9 +787,7 @@ def backward(ctx, dy1, dy2): y = Tanh.apply(x1=input1) def test_forward_return_fake_tensor(self): - class Tanh(LegacyPyLayer): - @staticmethod def forward(ctx, x1): @@ -838,9 +803,7 @@ def backward(ctx, dy1, dy2): y = Tanh.apply(x1=input1) def test_forward_return_fake_tensor_tuple(self): - class Tanh(LegacyPyLayer): - @staticmethod def forward(ctx, x1): @@ -856,9 +819,7 @@ def backward(ctx, dy1, dy2): y = Tanh.apply(x1=input1) def test_backward_return_fake_tensor_tuple(self): - class Tanh(LegacyPyLayer): - @staticmethod def forward(ctx, x1, x2): return x1 + 1, x1 + 2 @@ -876,9 +837,7 @@ def backward(ctx, dy1, dy2): y.mean().backward() def test_backward_return_fake_tensor(self): - class Tanh(LegacyPyLayer): - @staticmethod def forward(ctx, x1): return x1 + 1, x1 + 2 diff --git a/python/paddle/fluid/tests/unittests/test_pyramid_hash_op.py b/python/paddle/fluid/tests/unittests/test_pyramid_hash_op.py index 6f3f94253c83d..d84730fa3bd12 100644 --- a/python/paddle/fluid/tests/unittests/test_pyramid_hash_op.py +++ b/python/paddle/fluid/tests/unittests/test_pyramid_hash_op.py @@ -18,7 +18,6 @@ class TestPyramidHashOpApi(unittest.TestCase): - def test_api(self): num_voc = 128 embed_dim = 64 @@ -52,14 +51,14 @@ def test_api(self): place = fluid.CPUPlace() x_tensor = fluid.create_lod_tensor( - np.random.randint(0, num_voc, x_shape).astype('int32'), x_lod, - place) + np.random.randint(0, num_voc, x_shape).astype('int32'), x_lod, place + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={'x': x_tensor}, - fetch_list=[hash_embd], - return_numpy=False) + ret = exe.run( + feed={'x': x_tensor}, fetch_list=[hash_embd], return_numpy=False + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_python_bf16_numpy_datatype.py b/python/paddle/fluid/tests/unittests/test_python_bf16_numpy_datatype.py index 3144e5da55637..4a650dc07b973 100644 --- a/python/paddle/fluid/tests/unittests/test_python_bf16_numpy_datatype.py +++ b/python/paddle/fluid/tests/unittests/test_python_bf16_numpy_datatype.py @@ -18,7 +18,6 @@ class TestBF16DataType(unittest.TestCase): - def test_matmul(self): a_bf16 = np.random.random((6, 7)).astype(bfloat16) b_bf16 = np.random.random((7, 8)).astype(bfloat16) diff --git a/python/paddle/fluid/tests/unittests/test_qr_op.py b/python/paddle/fluid/tests/unittests/test_qr_op.py index 4d8bfb220fcac..39360d2e72158 100644 --- a/python/paddle/fluid/tests/unittests/test_qr_op.py +++ b/python/paddle/fluid/tests/unittests/test_qr_op.py @@ -22,7 +22,6 @@ class TestQrOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = paddle.linalg.qr @@ -73,38 +72,36 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X'], ['Q', 'R'], - check_eager=True, - numeric_grad_delta=1e-5, - max_relative_error=1e-6) + self.check_grad( + ['X'], + ['Q', 'R'], + check_eager=True, + numeric_grad_delta=1e-5, + max_relative_error=1e-6, + ) class TestQrOpCase1(TestQrOp): - def get_shape(self): return (10, 12) class TestQrOpCase2(TestQrOp): - def get_shape(self): return (16, 15) class TestQrOpCase3(TestQrOp): - def get_shape(self): return (2, 12, 16) class TestQrOpCase4(TestQrOp): - def get_shape(self): return (3, 16, 15) class TestQrOpCase5(TestQrOp): - def get_mode(self): return "complete" @@ -113,7 +110,6 @@ def get_shape(self): class TestQrOpCase6(TestQrOp): - def get_mode(self): return "complete" @@ -122,7 +118,6 @@ def get_shape(self): class TestQrAPI(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() np.random.seed(7) @@ -180,12 +175,13 @@ def run_qr_dygraph(shape, mode, dtype): (4, 5, 3), # 3-dim Tensors (2, 5, 3, 5), (3, 5, 5, 5), - (4, 5, 5, 3) # 4-dim Tensors + (4, 5, 5, 3), # 4-dim Tensors ] modes = ["reduced", "complete", "r"] dtypes = ["float32", "float64"] for tensor_shape, mode, dtype in itertools.product( - tensor_shapes, modes, dtypes): + tensor_shapes, modes, dtypes + ): run_qr_dygraph(tensor_shape, mode, dtype) def test_static(self): @@ -227,33 +223,34 @@ def run_qr_static(shape, mode, dtype): tmp_q, tmp_r = np.linalg.qr(a[coord], mode=mode) np_q[coord] = tmp_q np_r[coord] = tmp_r - x = paddle.fluid.data(name="input", - shape=shape, - dtype=dtype) + x = paddle.fluid.data( + name="input", shape=shape, dtype=dtype + ) if mode == "r": r = paddle.linalg.qr(x, mode=mode) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": a}, - fetch_list=[r]) - np.testing.assert_allclose(fetches[0], - np_r, - rtol=1e-05, - atol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": a}, + fetch_list=[r], + ) + np.testing.assert_allclose( + fetches[0], np_r, rtol=1e-05, atol=1e-05 + ) else: q, r = paddle.linalg.qr(x, mode=mode) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": a}, - fetch_list=[q, r]) - np.testing.assert_allclose(fetches[0], - np_q, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(fetches[1], - np_r, - rtol=1e-05, - atol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": a}, + fetch_list=[q, r], + ) + np.testing.assert_allclose( + fetches[0], np_q, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + fetches[1], np_r, rtol=1e-05, atol=1e-05 + ) tensor_shapes = [ (3, 5), @@ -264,12 +261,13 @@ def run_qr_static(shape, mode, dtype): (4, 5, 3), # 3-dim Tensors (2, 5, 3, 5), (3, 5, 5, 5), - (4, 5, 5, 3) # 4-dim Tensors + (4, 5, 5, 3), # 4-dim Tensors ] modes = ["reduced", "complete", "r"] dtypes = ["float32", "float64"] for tensor_shape, mode, dtype in itertools.product( - tensor_shapes, modes, dtypes): + tensor_shapes, modes, dtypes + ): run_qr_static(tensor_shape, mode, dtype) diff --git a/python/paddle/fluid/tests/unittests/test_quantile_and_nanquantile.py b/python/paddle/fluid/tests/unittests/test_quantile_and_nanquantile.py index 259e5556deefe..3d7406c457e47 100644 --- a/python/paddle/fluid/tests/unittests/test_quantile_and_nanquantile.py +++ b/python/paddle/fluid/tests/unittests/test_quantile_and_nanquantile.py @@ -16,15 +16,17 @@ import numpy as np import paddle -API_list = [(paddle.quantile, np.quantile), - (paddle.nanquantile, np.nanquantile)] +API_list = [ + (paddle.quantile, np.quantile), + (paddle.nanquantile, np.nanquantile), +] class TestQuantileAndNanquantile(unittest.TestCase): """ - This class is used for numerical precision testing. If there is - a corresponding numpy API, the precision comparison can be performed directly. - Otherwise, it needs to be verified by numpy implementated function. + This class is used for numerical precision testing. If there is + a corresponding numpy API, the precision comparison can be performed directly. + Otherwise, it needs to be verified by numpy implementated function. """ def setUp(self): @@ -99,10 +101,9 @@ def test_quantile_include_NaN(self): x = paddle.to_tensor(input_data) paddle_res = paddle.quantile(x, q=0.35, axis=0) np_res = np.quantile(input_data, q=0.35, axis=0) - np.testing.assert_allclose(paddle_res.numpy(), - np_res, - rtol=1e-05, - equal_nan=True) + np.testing.assert_allclose( + paddle_res.numpy(), np_res, rtol=1e-05, equal_nan=True + ) # Test correctness when input filled with NaN. def test_nanquantile_all_NaN(self): @@ -111,15 +112,14 @@ def test_nanquantile_all_NaN(self): x = paddle.to_tensor(input_data) paddle_res = paddle.nanquantile(x, q=0.35, axis=0) np_res = np.nanquantile(input_data, q=0.35, axis=0) - np.testing.assert_allclose(paddle_res.numpy(), - np_res, - rtol=1e-05, - equal_nan=True) + np.testing.assert_allclose( + paddle_res.numpy(), np_res, rtol=1e-05, equal_nan=True + ) class TestMuitlpleQ(unittest.TestCase): """ - This class is used to test multiple input of q. + This class is used to test multiple input of q. """ def setUp(self): @@ -139,21 +139,19 @@ def test_quantile_multiple_axis(self): def test_quantile_multiple_axis_keepdim(self): x = paddle.to_tensor(self.input_data) - paddle_res = paddle.quantile(x, - q=[0.1, 0.2, 0.3], - axis=[1, 2], - keepdim=True) - np_res = np.quantile(self.input_data, - q=[0.1, 0.2, 0.3], - axis=[1, 2], - keepdims=True) + paddle_res = paddle.quantile( + x, q=[0.1, 0.2, 0.3], axis=[1, 2], keepdim=True + ) + np_res = np.quantile( + self.input_data, q=[0.1, 0.2, 0.3], axis=[1, 2], keepdims=True + ) np.testing.assert_allclose(paddle_res.numpy(), np_res, rtol=1e-05) class TestError(unittest.TestCase): """ - This class is used to test that exceptions are thrown correctly. - Validity of all parameter values and types should be considered. + This class is used to test that exceptions are thrown correctly. + Validity of all parameter values and types should be considered. """ def setUp(self): @@ -218,8 +216,8 @@ def test_axis_value_error_3(): class TestQuantileRuntime(unittest.TestCase): """ - This class is used to test the API could run correctly with - different devices, different data types, and dygraph/static mode. + This class is used to test the API could run correctly with + different devices, different data types, and dygraph/static mode. """ def setUp(self): @@ -241,20 +239,22 @@ def test_dygraph(self): x = paddle.to_tensor(np_input_data, dtype=dtype) paddle_res = func(x, q=0.5, axis=1) np_res = res_func(np_input_data, q=0.5, axis=1) - np.testing.assert_allclose(paddle_res.numpy(), - np_res, - rtol=1e-05) + np.testing.assert_allclose( + paddle_res.numpy(), np_res, rtol=1e-05 + ) def test_static(self): paddle.enable_static() for (func, res_func) in API_list: for device in self.devices: - x = paddle.static.data(name="x", - shape=self.input_data.shape, - dtype=paddle.float32) - x_fp64 = paddle.static.data(name="x_fp64", - shape=self.input_data.shape, - dtype=paddle.float64) + x = paddle.static.data( + name="x", shape=self.input_data.shape, dtype=paddle.float32 + ) + x_fp64 = paddle.static.data( + name="x_fp64", + shape=self.input_data.shape, + dtype=paddle.float64, + ) results = func(x, q=0.5, axis=1) np_input_data = self.input_data.astype('float32') @@ -264,16 +264,15 @@ def test_static(self): exe = paddle.static.Executor(device) paddle_res, paddle_res_fp64 = exe.run( paddle.static.default_main_program(), - feed={ - "x": np_input_data, - "x_fp64": np_input_data_fp64 - }, - fetch_list=[results, results_fp64]) + feed={"x": np_input_data, "x_fp64": np_input_data_fp64}, + fetch_list=[results, results_fp64], + ) np_res = res_func(np_input_data, q=0.5, axis=1) np_res_fp64 = res_func(np_input_data_fp64, q=0.5, axis=1) self.assertTrue( np.allclose(paddle_res, np_res) - and np.allclose(paddle_res_fp64, np_res_fp64)) + and np.allclose(paddle_res_fp64, np_res_fp64) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_query_op.py b/python/paddle/fluid/tests/unittests/test_query_op.py index 162c1f440e619..a97530febfa61 100644 --- a/python/paddle/fluid/tests/unittests/test_query_op.py +++ b/python/paddle/fluid/tests/unittests/test_query_op.py @@ -18,7 +18,6 @@ class TestCudnnVersion(unittest.TestCase): - def test_no_cudnn(self): cudnn_version = paddle.get_cudnn_version() if not core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_queue.py b/python/paddle/fluid/tests/unittests/test_queue.py index 703c11e710cab..bfc2f1e0e6f22 100644 --- a/python/paddle/fluid/tests/unittests/test_queue.py +++ b/python/paddle/fluid/tests/unittests/test_queue.py @@ -21,7 +21,6 @@ class TestQueue(unittest.TestCase): - def test_eq(self): """ test queue_generator op, enqueue op and dequeue op. @@ -31,39 +30,51 @@ def test_eq(self): startup_program = fluid.Program() value = np.random.rand(1) with fluid.program_guard(main_program, startup_program): - data_in = layers.create_global_var(shape=[2, 3], - value=value, - dtype="float32", - persistable=True, - name='var_in') - data_out = layers.create_global_var(shape=[2, 3], - value=value - 1.0, - dtype="float32", - persistable=True, - name='var_out') + data_in = layers.create_global_var( + shape=[2, 3], + value=value, + dtype="float32", + persistable=True, + name='var_in', + ) + data_out = layers.create_global_var( + shape=[2, 3], + value=value - 1.0, + dtype="float32", + persistable=True, + name='var_out', + ) startup_block = startup_program.block(0) queue_name = 'blocking_queue' - startup_block.create_var(name=queue_name, - persistable=True, - type=core.VarDesc.VarType.RAW) - startup_block.append_op(type="queue_generator", - attrs={'names': [queue_name]}) + startup_block.create_var( + name=queue_name, persistable=True, type=core.VarDesc.VarType.RAW + ) + startup_block.append_op( + type="queue_generator", attrs={'names': [queue_name]} + ) block = main_program.block(0) - block.append_op(type='enqueue', - inputs={'X': data_in}, - attrs={'queue_name': queue_name}) - block.append_op(type='dequeue', - outputs={'Out': [data_out]}, - attrs={'queue_name': queue_name}) + block.append_op( + type='enqueue', + inputs={'X': data_in}, + attrs={'queue_name': queue_name}, + ) + block.append_op( + type='dequeue', + outputs={'Out': [data_out]}, + attrs={'queue_name': queue_name}, + ) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(startup_program) - ret, = exe.run(main_program, fetch_list=[data_out.name]) - np.testing.assert_allclose(np.asarray(ret), - np.full((2, 3), value, np.float32), - rtol=1e-05) + (ret,) = exe.run(main_program, fetch_list=[data_out.name]) + np.testing.assert_allclose( + np.asarray(ret), np.full((2, 3), value, np.float32), rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_rad2deg.py b/python/paddle/fluid/tests/unittests/test_rad2deg.py index 00cf6e9fad575..220cf96f04f54 100644 --- a/python/paddle/fluid/tests/unittests/test_rad2deg.py +++ b/python/paddle/fluid/tests/unittests/test_rad2deg.py @@ -22,11 +22,11 @@ class TestRad2degAPI(unittest.TestCase): - def setUp(self): self.x_dtype = 'float64' - self.x_np = np.array([3.142, -3.142, 6.283, -6.283, 1.570, - -1.570]).astype(np.float64) + self.x_np = np.array( + [3.142, -3.142, 6.283, -6.283, 1.570, -1.570] + ).astype(np.float64) self.x_shape = [6] self.out_np = np.rad2deg(self.x_np) @@ -37,12 +37,17 @@ def test_static_graph(self): x = fluid.data(name='input', dtype=self.x_dtype, shape=self.x_shape) out = paddle.rad2deg(x) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(fluid.default_main_program(), - feed={'input': self.x_np}, - fetch_list=[out]) + res = exe.run( + fluid.default_main_program(), + feed={'input': self.x_np}, + fetch_list=[out], + ) self.assertTrue((np.array(out[0]) == self.out_np).all()) def test_dygraph(self): @@ -55,7 +60,6 @@ def test_dygraph(self): class TestRad2degAPI2(TestRad2degAPI): - def setUp(self): self.x_np = np.pi / 2 self.x_shape = [1] diff --git a/python/paddle/fluid/tests/unittests/test_rand_op.py b/python/paddle/fluid/tests/unittests/test_rand_op.py index b889d79b078f2..3503805414424 100644 --- a/python/paddle/fluid/tests/unittests/test_rand_op.py +++ b/python/paddle/fluid/tests/unittests/test_rand_op.py @@ -33,8 +33,9 @@ def test_errors(self): with program_guard(main_prog, start_prog): def test_Variable(): - x1 = fluid.create_lod_tensor(np.zeros((4, 784)), [[1, 1, 1, 1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.zeros((4, 784)), [[1, 1, 1, 1]], fluid.CPUPlace() + ) rand(x1) self.assertRaises(TypeError, test_Variable) @@ -69,9 +70,9 @@ def run_net(self, use_cuda=False): var_shape = fluid.data(name='var_shape', shape=[2], dtype="int64") result_3 = rand(var_shape) - var_shape_int32 = fluid.data(name='var_shape_int32', - shape=[2], - dtype="int32") + var_shape_int32 = fluid.data( + name='var_shape_int32', shape=[2], dtype="int32" + ) result_4 = rand(var_shape_int32) exe.run(startup_program) @@ -80,11 +81,9 @@ def run_net(self, use_cuda=False): x2 = np.array([4, 3]).astype('int32') ret = exe.run( train_program, - feed={ - "var_shape": x1, - "var_shape_int32": x2 - }, - fetch_list=[result_1, result_1, result_2, result_3, result_4]) + feed={"var_shape": x1, "var_shape_int32": x2}, + fetch_list=[result_1, result_1, result_2, result_3, result_4], + ) def test_run(self): self.run_net(False) @@ -118,7 +117,6 @@ def test_run(self): class TestRandDtype(unittest.TestCase): - def test_default_dtype(self): paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_randint_like.py b/python/paddle/fluid/tests/unittests/test_randint_like.py index 972243f24b59c..32a717d3815bf 100644 --- a/python/paddle/fluid/tests/unittests/test_randint_like.py +++ b/python/paddle/fluid/tests/unittests/test_randint_like.py @@ -20,7 +20,6 @@ # Test python API class TestRandintLikeAPI(unittest.TestCase): - def setUp(self): self.x_bool = np.zeros((10, 12)).astype("bool") self.x_int32 = np.zeros((10, 12)).astype("int32") @@ -30,16 +29,19 @@ def setUp(self): self.x_float64 = np.zeros((10, 12)).astype("float64") self.dtype = ["bool", "int32", "int64", "float16", "float32", "float64"] - self.place=paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() with program_guard(Program(), Program()): # results are from [-100, 100). - x_bool = paddle.fluid.data(name="x_bool", - shape=[10, 12], - dtype="bool") + x_bool = paddle.fluid.data( + name="x_bool", shape=[10, 12], dtype="bool" + ) exe = paddle.static.Executor(self.place) # x dtype is bool output dtype in ["bool", "int32", "int64", "float16", "float32", "float64"] outlist1 = [ @@ -51,9 +53,9 @@ def test_static_api(self): self.assertTrue(out.dtype, np.dtype(dtype)) self.assertTrue(((out >= -10) & (out <= 10)).all(), True) with program_guard(Program(), Program()): - x_int32 = paddle.fluid.data(name="x_int32", - shape=[10, 12], - dtype="int32") + x_int32 = paddle.fluid.data( + name="x_int32", shape=[10, 12], dtype="int32" + ) exe = paddle.static.Executor(self.place) # x dtype is int32 output dtype in ["bool", "int32", "int64", "float16", "float32", "float64"] outlist2 = [ @@ -66,9 +68,9 @@ def test_static_api(self): self.assertTrue(((out >= -5) & (out <= 10)).all(), True) with program_guard(Program(), Program()): - x_int64 = paddle.fluid.data(name="x_int64", - shape=[10, 12], - dtype="int64") + x_int64 = paddle.fluid.data( + name="x_int64", shape=[10, 12], dtype="int64" + ) exe = paddle.static.Executor(self.place) # x dtype is int64 output dtype in ["bool", "int32", "int64", "float16", "float32", "float64"] outlist3 = [ @@ -81,49 +83,52 @@ def test_static_api(self): self.assertTrue(((out >= -100) & (out <= 100)).all(), True) if paddle.is_compiled_with_cuda(): with program_guard(Program(), Program()): - x_float16 = paddle.fluid.data(name="x_float16", - shape=[10, 12], - dtype="float16") + x_float16 = paddle.fluid.data( + name="x_float16", shape=[10, 12], dtype="float16" + ) exe = paddle.static.Executor(self.place) # x dtype is float16 output dtype in ["bool", "int32", "int64", "float16", "float32", "float64"] outlist4 = [ paddle.randint_like(x_float16, low=-3, high=25, dtype=dtype) for dtype in self.dtype ] - outs4 = exe.run(feed={'x_float16': self.x_float16}, - fetch_list=outlist4) + outs4 = exe.run( + feed={'x_float16': self.x_float16}, fetch_list=outlist4 + ) for out, dtype in zip(outs4, self.dtype): self.assertTrue(out.dtype, np.dtype(dtype)) self.assertTrue(((out >= -3) & (out <= 25)).all(), True) with program_guard(Program(), Program()): - x_float32 = paddle.fluid.data(name="x_float32", - shape=[10, 12], - dtype="float32") + x_float32 = paddle.fluid.data( + name="x_float32", shape=[10, 12], dtype="float32" + ) exe = paddle.static.Executor(self.place) # x dtype is float32 output dtype in ["bool", "int32", "int64", "float16", "float32", "float64"] outlist5 = [ paddle.randint_like(x_float32, low=-25, high=25, dtype=dtype) for dtype in self.dtype ] - outs5 = exe.run(feed={'x_float32': self.x_float32}, - fetch_list=outlist5) + outs5 = exe.run( + feed={'x_float32': self.x_float32}, fetch_list=outlist5 + ) for out, dtype in zip(outs5, self.dtype): self.assertTrue(out.dtype, np.dtype(dtype)) self.assertTrue(((out >= -25) & (out <= 25)).all(), True) with program_guard(Program(), Program()): - x_float64 = paddle.fluid.data(name="x_float64", - shape=[10, 12], - dtype="float64") + x_float64 = paddle.fluid.data( + name="x_float64", shape=[10, 12], dtype="float64" + ) exe = paddle.static.Executor(self.place) # x dtype is float64 output dtype in ["bool", "int32", "int64", "float16", "float32", "float64"] outlist6 = [ paddle.randint_like(x_float64, low=-16, high=16, dtype=dtype) for dtype in self.dtype ] - outs6 = exe.run(feed={'x_float64': self.x_float64}, - fetch_list=outlist6) + outs6 = exe.run( + feed={'x_float64': self.x_float64}, fetch_list=outlist6 + ) for out, dtype in zip(outs6, self.dtype): self.assertTrue(out.dtype, dtype) self.assertTrue(((out >= -16) & (out <= 16)).all(), True) @@ -132,62 +137,63 @@ def test_dygraph_api(self): paddle.disable_static(self.place) # x dtype ["bool", "int32", "int64", "float32", "float64"] for x in [ - self.x_bool, self.x_int32, self.x_int64, self.x_float32, - self.x_float64 + self.x_bool, + self.x_int32, + self.x_int64, + self.x_float32, + self.x_float64, ]: x_inputs = paddle.to_tensor(x) # self.dtype ["bool", "int32", "int64", "float16", "float32", "float64"] for dtype in self.dtype: - out = paddle.randint_like(x_inputs, - low=-100, - high=100, - dtype=dtype) + out = paddle.randint_like( + x_inputs, low=-100, high=100, dtype=dtype + ) self.assertTrue(out.numpy().dtype, np.dtype(dtype)) self.assertTrue( - ((out.numpy() >= -100) & (out.numpy() <= 100)).all(), True) + ((out.numpy() >= -100) & (out.numpy() <= 100)).all(), True + ) # x dtype ["float16"] if paddle.is_compiled_with_cuda(): x_inputs = paddle.to_tensor(self.x_float16) # self.dtype ["bool", "int32", "int64", "float16", "float32", "float64"] for dtype in self.dtype: - out = paddle.randint_like(x_inputs, - low=-100, - high=100, - dtype=dtype) + out = paddle.randint_like( + x_inputs, low=-100, high=100, dtype=dtype + ) self.assertTrue(out.numpy().dtype, np.dtype(dtype)) self.assertTrue( - ((out.numpy() >= -100) & (out.numpy() <= 100)).all(), True) + ((out.numpy() >= -100) & (out.numpy() <= 100)).all(), True + ) paddle.enable_static() def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): - x_bool = paddle.fluid.data(name="x_bool", - shape=[10, 12], - dtype="bool") - x_int32 = paddle.fluid.data(name="x_int32", - shape=[10, 12], - dtype="int32") - x_int64 = paddle.fluid.data(name="x_int64", - shape=[10, 12], - dtype="int64") - x_float16 = paddle.fluid.data(name="x_float16", - shape=[10, 12], - dtype="float16") - x_float32 = paddle.fluid.data(name="x_float32", - shape=[10, 12], - dtype="float32") - x_float64 = paddle.fluid.data(name="x_float64", - shape=[10, 12], - dtype="float64") + x_bool = paddle.fluid.data( + name="x_bool", shape=[10, 12], dtype="bool" + ) + x_int32 = paddle.fluid.data( + name="x_int32", shape=[10, 12], dtype="int32" + ) + x_int64 = paddle.fluid.data( + name="x_int64", shape=[10, 12], dtype="int64" + ) + x_float16 = paddle.fluid.data( + name="x_float16", shape=[10, 12], dtype="float16" + ) + x_float32 = paddle.fluid.data( + name="x_float32", shape=[10, 12], dtype="float32" + ) + x_float64 = paddle.fluid.data( + name="x_float64", shape=[10, 12], dtype="float64" + ) # x dtype is bool # low is 5 and high is 5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_bool, - low=5, - high=5) + self.assertRaises( + ValueError, paddle.randint_like, x_bool, low=5, high=5 + ) # low(default value) is 0 and high is -5, low must less then high self.assertRaises(ValueError, paddle.randint_like, x_bool, high=-5) # if high is None, low must be greater than 0 @@ -195,11 +201,9 @@ def test_errors(self): # x dtype is int32 # low is 5 and high is 5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_int32, - low=5, - high=5) + self.assertRaises( + ValueError, paddle.randint_like, x_int32, low=5, high=5 + ) # low(default value) is 0 and high is -5, low must less then high self.assertRaises(ValueError, paddle.randint_like, x_int32, high=-5) # if high is None, low must be greater than 0 @@ -207,11 +211,9 @@ def test_errors(self): # x dtype is int64 # low is 5 and high is 5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_int64, - low=5, - high=5) + self.assertRaises( + ValueError, paddle.randint_like, x_int64, low=5, high=5 + ) # low(default value) is 0 and high is -5, low must less then high self.assertRaises(ValueError, paddle.randint_like, x_int64, high=-5) # if high is None, low must be greater than 0 @@ -220,57 +222,45 @@ def test_errors(self): # x dtype is float16 # low is 5 and high is 5, low must less then high if paddle.is_compiled_with_cuda(): - self.assertRaises(ValueError, - paddle.randint_like, - x_float16, - low=5, - high=5) + self.assertRaises( + ValueError, paddle.randint_like, x_float16, low=5, high=5 + ) # low(default value) is 0 and high is -5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_float16, - high=-5) + self.assertRaises( + ValueError, paddle.randint_like, x_float16, high=-5 + ) # if high is None, low must be greater than 0 - self.assertRaises(ValueError, - paddle.randint_like, - x_float16, - low=-5) + self.assertRaises( + ValueError, paddle.randint_like, x_float16, low=-5 + ) # x dtype is float32 # low is 5 and high is 5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_float32, - low=5, - high=5) + self.assertRaises( + ValueError, paddle.randint_like, x_float32, low=5, high=5 + ) # low(default value) is 0 and high is -5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_float32, - high=-5) + self.assertRaises( + ValueError, paddle.randint_like, x_float32, high=-5 + ) # if high is None, low must be greater than 0 - self.assertRaises(ValueError, - paddle.randint_like, - x_float32, - low=-5) + self.assertRaises( + ValueError, paddle.randint_like, x_float32, low=-5 + ) # x dtype is float64 # low is 5 and high is 5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_float64, - low=5, - high=5) + self.assertRaises( + ValueError, paddle.randint_like, x_float64, low=5, high=5 + ) # low(default value) is 0 and high is -5, low must less then high - self.assertRaises(ValueError, - paddle.randint_like, - x_float64, - high=-5) + self.assertRaises( + ValueError, paddle.randint_like, x_float64, high=-5 + ) # if high is None, low must be greater than 0 - self.assertRaises(ValueError, - paddle.randint_like, - x_float64, - low=-5) + self.assertRaises( + ValueError, paddle.randint_like, x_float64, low=-5 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_randint_op.py b/python/paddle/fluid/tests/unittests/test_randint_op.py index 549554733febc..505cd43923a83 100644 --- a/python/paddle/fluid/tests/unittests/test_randint_op.py +++ b/python/paddle/fluid/tests/unittests/test_randint_op.py @@ -32,7 +32,6 @@ def output_hist(out): class TestRandintOp(OpTest): - def setUp(self): self.op_type = "randint" self.inputs = {} @@ -56,7 +55,6 @@ def test_check_output_eager(self): class TestRandintOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): self.assertRaises(TypeError, paddle.randint, 5, shape=np.array([2])) @@ -66,10 +64,9 @@ def test_errors(self): self.assertRaises(TypeError, paddle.randint, 5, shape=['2']) shape_tensor = paddle.static.data('X', [1]) self.assertRaises(TypeError, paddle.randint, 5, shape=shape_tensor) - self.assertRaises(TypeError, - paddle.randint, - 5, - shape=[shape_tensor]) + self.assertRaises( + TypeError, paddle.randint, 5, shape=[shape_tensor] + ) def test_errors_eager(self): with _test_eager_guard(): @@ -77,14 +74,14 @@ def test_errors_eager(self): class TestRandintOp_attr_tensorlist(OpTest): - def setUp(self): self.op_type = "randint" self.new_shape = (10000, 784) shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype("int64") * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype("int64") * ele) + ) self.inputs = {'ShapeTensorList': shape_tensor} self.init_attrs() self.outputs = {"Out": np.zeros((10000, 784)).astype("int32")} @@ -106,7 +103,6 @@ def test_check_output_eager(self): class TestRandint_attr_tensor(OpTest): - def setUp(self): self.op_type = "randint" self.inputs = {"ShapeTensor": np.array([10000, 784]).astype("int64")} @@ -131,43 +127,42 @@ def test_check_output_eager(self): # Test python API class TestRandintAPI(unittest.TestCase): - def test_api(self): with program_guard(Program(), Program()): # results are from [0, 5). out1 = paddle.randint(5) # shape is a list and dtype is 'int32' - out2 = paddle.randint(low=-100, - high=100, - shape=[64, 64], - dtype='int32') + out2 = paddle.randint( + low=-100, high=100, shape=[64, 64], dtype='int32' + ) # shape is a tuple and dtype is 'int64' - out3 = paddle.randint(low=-100, - high=100, - shape=(32, 32, 3), - dtype='int64') + out3 = paddle.randint( + low=-100, high=100, shape=(32, 32, 3), dtype='int64' + ) # shape is a tensorlist and dtype is 'float32' dim_1 = paddle.fluid.layers.fill_constant([1], "int64", 32) dim_2 = paddle.fluid.layers.fill_constant([1], "int32", 50) - out4 = paddle.randint(low=-100, - high=100, - shape=[dim_1, 5, dim_2], - dtype='int32') + out4 = paddle.randint( + low=-100, high=100, shape=[dim_1, 5, dim_2], dtype='int32' + ) # shape is a tensor and dtype is 'float64' - var_shape = paddle.static.data(name='var_shape', - shape=[2], - dtype="int64") - out5 = paddle.randint(low=1, - high=1000, - shape=var_shape, - dtype='int64') - - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + var_shape = paddle.static.data( + name='var_shape', shape=[2], dtype="int64" + ) + out5 = paddle.randint( + low=1, high=1000, shape=var_shape, dtype='int64' + ) + + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) outs = exe.run( feed={'var_shape': np.array([100, 100]).astype('int64')}, - fetch_list=[out1, out2, out3, out4, out5]) + fetch_list=[out1, out2, out3, out4, out5], + ) def test_api_eager(self): with _test_eager_guard(): @@ -175,7 +170,6 @@ def test_api_eager(self): class TestRandintImperative(unittest.TestCase): - def test_api(self): paddle.disable_static() @@ -197,7 +191,6 @@ def run_test_case(self): class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -221,8 +214,9 @@ def run_test_case(self): paddle.set_device('gpu') paddle.seed(100) - x = paddle.randint(-10000, 10000, [32, 3, 1024, 1024], - dtype='int32').numpy() + x = paddle.randint( + -10000, 10000, [32, 3, 1024, 1024], dtype='int32' + ).numpy() self.assertTrue(x.mean(), -0.7517569760481516) self.assertTrue(x.std(), 5773.696619107639) expect = [2535, 2109, 5916, -5011, -261] @@ -232,8 +226,9 @@ def run_test_case(self): expect = [881, 1560, 1100, 9664, 1669] np.testing.assert_array_equal(x[30, 2, 1000, 1000:1005], expect) - x = paddle.randint(-10000, 10000, [32, 3, 1024, 1024], - dtype='int64').numpy() + x = paddle.randint( + -10000, 10000, [32, 3, 1024, 1024], dtype='int64' + ).numpy() self.assertTrue(x.mean(), -1.461287518342336) self.assertTrue(x.std(), 5773.023477548159) expect = [7213, -9597, 754, 8129, -1158] diff --git a/python/paddle/fluid/tests/unittests/test_randn_op.py b/python/paddle/fluid/tests/unittests/test_randn_op.py index 4590af0f10f81..d136e3fd57e14 100644 --- a/python/paddle/fluid/tests/unittests/test_randn_op.py +++ b/python/paddle/fluid/tests/unittests/test_randn_op.py @@ -20,7 +20,6 @@ class TestRandnOp(unittest.TestCase): - def test_api(self): shape = [1000, 784] train_program = Program() @@ -36,24 +35,31 @@ def test_api(self): var_shape = paddle.static.data('X', [2], 'int32') x4 = paddle.randn(var_shape) - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) - res = exe.run(train_program, - feed={'X': np.array(shape, dtype='int32')}, - fetch_list=[x1, x2, x3, x4]) + res = exe.run( + train_program, + feed={'X': np.array(shape, dtype='int32')}, + fetch_list=[x1, x2, x3, x4], + ) for out in res: - self.assertAlmostEqual(np.mean(out), .0, delta=0.1) - self.assertAlmostEqual(np.std(out), 1., delta=0.1) + self.assertAlmostEqual(np.mean(out), 0.0, delta=0.1) + self.assertAlmostEqual(np.std(out), 1.0, delta=0.1) class TestRandnOpForDygraph(unittest.TestCase): - def test_api(self): shape = [1000, 784] - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) paddle.disable_static(place) x1 = paddle.randn(shape, 'float32') x2 = paddle.randn(shape, 'float64') @@ -66,13 +72,12 @@ def test_api(self): x4 = paddle.randn(var_shape) for out in [x1, x2, x3, x4]: - self.assertAlmostEqual(np.mean(out.numpy()), .0, delta=0.1) - self.assertAlmostEqual(np.std(out.numpy()), 1., delta=0.1) + self.assertAlmostEqual(np.mean(out.numpy()), 0.0, delta=0.1) + self.assertAlmostEqual(np.std(out.numpy()), 1.0, delta=0.1) paddle.enable_static() class TestRandnOpError(unittest.TestCase): - def test_error(self): with program_guard(Program(), Program()): # The argument shape's type of randn_op should be list or tuple. diff --git a/python/paddle/fluid/tests/unittests/test_random_crop_op.py b/python/paddle/fluid/tests/unittests/test_random_crop_op.py index 580d348156c2e..ae474618733cb 100644 --- a/python/paddle/fluid/tests/unittests/test_random_crop_op.py +++ b/python/paddle/fluid/tests/unittests/test_random_crop_op.py @@ -19,15 +19,15 @@ class TestRandomCropOp(OpTest): - def setUp(self): - to_crop = np.array([[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]] * - 5).astype(np.int32) + to_crop = np.array( + [[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]] * 5 + ).astype(np.int32) self.possible_res = [ np.array([[1, 2, 3], [5, 6, 7]]).astype(np.int32), np.array([[2, 3, 4], [6, 7, 8]]).astype(np.int32), np.array([[5, 6, 7], [9, 10, 11]]).astype(np.int32), - np.array([[6, 7, 8], [10, 11, 12]]).astype(np.int32) + np.array([[6, 7, 8], [10, 11, 12]]).astype(np.int32), ] self.op_type = "random_crop" self.inputs = {'X': to_crop, 'Seed': np.array([10]).astype('int64')} @@ -45,7 +45,6 @@ def verify_output(self, outs): class TestRandomCropOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program()): @@ -56,17 +55,17 @@ def test_x_type(): self.assertRaises(TypeError, test_x_type) def test_x_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[None, 3, 256, 256], - dtype='float16') + x2 = fluid.layers.data( + name='x2', shape=[None, 3, 256, 256], dtype='float16' + ) fluid.layers.random_crop(x2) self.assertRaises(TypeError, test_x_dtype) def test_shape_type(): - x3 = fluid.layers.data(name='x3', - shape=[None, 3, 256, 256], - dtype='float32') + x3 = fluid.layers.data( + name='x3', shape=[None, 3, 256, 256], dtype='float32' + ) fluid.layers.random_crop(x3, shape=1) self.assertRaises(TypeError, test_shape_type) diff --git a/python/paddle/fluid/tests/unittests/test_random_routing_op.py b/python/paddle/fluid/tests/unittests/test_random_routing_op.py index 54313b4c10cff..03cce6c9caa8c 100644 --- a/python/paddle/fluid/tests/unittests/test_random_routing_op.py +++ b/python/paddle/fluid/tests/unittests/test_random_routing_op.py @@ -32,22 +32,24 @@ def random_routing(topk_idx, topk_value, prob, topk=2): raise RuntimeError("only topk=2 is supported now") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestNumberCountAPIFp32(unittest.TestCase): - def setUp(self): self.dtype = "float32" self.init() def init(self): self.upper_range = 8 - self.x = np.random.randint(-1, self.upper_range, - size=(200, 2)).astype('int64') - self.prob = np.random.random((self.x.shape[0], )).astype(self.dtype) + self.x = np.random.randint(-1, self.upper_range, size=(200, 2)).astype( + 'int64' + ) + self.prob = np.random.random((self.x.shape[0],)).astype(self.dtype) self.topk_value = np.random.random(self.x.shape).astype(self.dtype) - self.out = random_routing(self.x, self.topk_value, - self.prob).astype(self.dtype) + self.out = random_routing(self.x, self.topk_value, self.prob).astype( + self.dtype + ) self.place = paddle.CUDAPlace(0) def func_api_dygraph(self): @@ -64,10 +66,10 @@ def test_api_dygraph(self): self.func_api_dygraph() -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestNumberCountAPIFp16(TestNumberCountAPIFp32): - def setUp(self): self.dtype = "float16" self.init() diff --git a/python/paddle/fluid/tests/unittests/test_random_seed.py b/python/paddle/fluid/tests/unittests/test_random_seed.py index c49808f3ae54d..97f3fa56dc526 100644 --- a/python/paddle/fluid/tests/unittests/test_random_seed.py +++ b/python/paddle/fluid/tests/unittests/test_random_seed.py @@ -36,23 +36,20 @@ def test_generator_uniform_random_dygraph(self): x = fluid.layers.uniform_random([10], dtype="float32", min=0.0, max=1.0) st1 = gen.get_state() - x1 = fluid.layers.uniform_random([10], - dtype="float32", - min=0.0, - max=1.0) + x1 = fluid.layers.uniform_random( + [10], dtype="float32", min=0.0, max=1.0 + ) gen.set_state(st1) print(gen.get_state()) - x2 = fluid.layers.uniform_random([10], - dtype="float32", - min=0.0, - max=1.0) + x2 = fluid.layers.uniform_random( + [10], dtype="float32", min=0.0, max=1.0 + ) paddle.seed(12312321111) - x3 = fluid.layers.uniform_random([10], - dtype="float32", - min=0.0, - max=1.0) + x3 = fluid.layers.uniform_random( + [10], dtype="float32", min=0.0, max=1.0 + ) x_np = x.numpy() x1_np = x1.numpy() @@ -78,14 +75,14 @@ def test_generator_uniform_random_static(self): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) - #gen.set_state(cur_state) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) + # gen.set_state(cur_state) gen.manual_seed(123123143) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) @@ -103,17 +100,15 @@ def test_gen_dropout_dygraph(self): gen = paddle.seed(111111111) st = gen.get_state() # x = np.arange(1,101).reshape(2,50).astype("float32") - x = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) + x = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) y = fluid.layers.dropout(x, 0.5) gen.manual_seed(111111111) - #gen.set_state(st) - x1 = fluid.layers.uniform_random([2, 10], - dtype="float32", - min=0.0, - max=1.0) + # gen.set_state(st) + x1 = fluid.layers.uniform_random( + [2, 10], dtype="float32", min=0.0, max=1.0 + ) y1 = fluid.layers.dropout(x1, 0.5) y_np = y.numpy() y1_np = y1.numpy() @@ -137,7 +132,7 @@ def test_gen_dropout_static(self): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) out1 = exe.run(train_program, feed={}, fetch_list=[y_1]) - #gen.set_state(cur_state) + # gen.set_state(cur_state) gen.manual_seed(123123143) out2 = exe.run(train_program, feed={}, fetch_list=[y_1]) out1_np = np.array(out1[0]) @@ -184,14 +179,14 @@ def test_generator_gaussian_random_static(self): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) - #gen.set_state(cur_state) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) + # gen.set_state(cur_state) gen.manual_seed(123123143) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) @@ -243,14 +238,14 @@ def test_generator_uniform_random_static_1(self): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) - #gen.set_state(cur_state) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) + # gen.set_state(cur_state) gen.manual_seed(123123143) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) @@ -297,14 +292,14 @@ def test_generator_ranint_static(self): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) - #gen.set_state(cur_state) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) + # gen.set_state(cur_state) gen.manual_seed(123123143) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) @@ -356,14 +351,14 @@ def test_generator_randperm_static(self): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) paddle.seed(123123143) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) @@ -383,31 +378,27 @@ def test_generator_sampling_id_dygraph(self): fluid.enable_dygraph() gen.manual_seed(12312321111) - x = fluid.layers.uniform_random([10, 10], - dtype="float32", - min=0.0, - max=1.0) + x = fluid.layers.uniform_random( + [10, 10], dtype="float32", min=0.0, max=1.0 + ) y = fluid.layers.sampling_id(x) st1 = gen.get_state() - x1 = fluid.layers.uniform_random([10, 10], - dtype="float32", - min=0.0, - max=1.0) + x1 = fluid.layers.uniform_random( + [10, 10], dtype="float32", min=0.0, max=1.0 + ) y1 = fluid.layers.sampling_id(x) gen.set_state(st1) - x2 = fluid.layers.uniform_random([10, 10], - dtype="float32", - min=0.0, - max=1.0) + x2 = fluid.layers.uniform_random( + [10, 10], dtype="float32", min=0.0, max=1.0 + ) y2 = fluid.layers.sampling_id(x) gen.manual_seed(12312321111) - x3 = fluid.layers.uniform_random([10, 10], - dtype="float32", - min=0.0, - max=1.0) + x3 = fluid.layers.uniform_random( + [10, 10], dtype="float32", min=0.0, max=1.0 + ) y3 = fluid.layers.sampling_id(x) x_np = y.numpy() @@ -437,14 +428,14 @@ def test_generator_randperm_static_1(self): exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) paddle.seed(123123143) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) @@ -472,26 +463,30 @@ def test_gen_TruncatedNormal_initializer(self): result_1 = fluid.layers.fc( input=x, size=10, - param_attr=fluid.initializer.TruncatedNormal(loc=0.0, - scale=2.0)) + param_attr=fluid.initializer.TruncatedNormal( + loc=0.0, scale=2.0 + ), + ) result_2 = fluid.layers.fc( input=x, size=10, - param_attr=fluid.initializer.TruncatedNormal(loc=0.0, - scale=2.0)) + param_attr=fluid.initializer.TruncatedNormal( + loc=0.0, scale=2.0 + ), + ) exe = fluid.Executor(fluid.CPUPlace()) exe.run(startup_program) - out1 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out1 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) gen.manual_seed(123123143) with fluid.program_guard(train_program, startup_program): exe.run(startup_program) - out2 = exe.run(train_program, - feed={}, - fetch_list=[result_1, result_2]) + out2 = exe.run( + train_program, feed={}, fetch_list=[result_1, result_2] + ) out1_res1 = np.array(out1[0]) out1_res2 = np.array(out1[1]) diff --git a/python/paddle/fluid/tests/unittests/test_randperm_op.py b/python/paddle/fluid/tests/unittests/test_randperm_op.py index bb1e493a65dd9..12543a727a211 100644 --- a/python/paddle/fluid/tests/unittests/test_randperm_op.py +++ b/python/paddle/fluid/tests/unittests/test_randperm_op.py @@ -22,31 +22,38 @@ def check_randperm_out(n, data_np): - assert isinstance(data_np, np.ndarray), \ - "The input data_np should be np.ndarray." + assert isinstance( + data_np, np.ndarray + ), "The input data_np should be np.ndarray." gt_sorted = np.arange(n) out_sorted = np.sort(data_np) return list(gt_sorted == out_sorted) def error_msg(data_np): - return "The sorted ground truth and sorted out should " + \ - "be equal, out = " + str(data_np) + return ( + "The sorted ground truth and sorted out should " + + "be equal, out = " + + str(data_np) + ) def convert_dtype(dtype_str): dtype_str_list = ["int32", "int64", "float32", "float64"] dtype_num_list = [ - core.VarDesc.VarType.INT32, core.VarDesc.VarType.INT64, - core.VarDesc.VarType.FP32, core.VarDesc.VarType.FP64 + core.VarDesc.VarType.INT32, + core.VarDesc.VarType.INT64, + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP64, ] - assert dtype_str in dtype_str_list, dtype_str + \ - " should in " + str(dtype_str_list) + assert dtype_str in dtype_str_list, ( + dtype_str + " should in " + str(dtype_str_list) + ) return dtype_num_list[dtype_str_list.index(dtype_str)] class TestRandpermOp(OpTest): - """ Test randperm op.""" + """Test randperm op.""" def setUp(self): self.op_type = "randperm" @@ -70,8 +77,9 @@ def test_check_output(self): def verify_output(self, outs): out_np = np.array(outs[0]) - self.assertTrue(check_randperm_out(self.n, out_np), - msg=error_msg(out_np)) + self.assertTrue( + check_randperm_out(self.n, out_np), msg=error_msg(out_np) + ) def test_eager(self): with _test_eager_guard(): @@ -79,31 +87,26 @@ def test_eager(self): class TestRandpermOpN(TestRandpermOp): - def init_attrs(self): self.n = 10000 class TestRandpermOpInt32(TestRandpermOp): - def init_attrs(self): self.dtype = "int32" class TestRandpermOpFloat32(TestRandpermOp): - def init_attrs(self): self.dtype = "float32" class TestRandpermOpFloat64(TestRandpermOp): - def init_attrs(self): self.dtype = "float64" class TestRandpermOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): self.assertRaises(ValueError, paddle.randperm, -3) @@ -111,11 +114,13 @@ def test_errors(self): class TestRandpermAPI(unittest.TestCase): - def test_out(self): n = 10 - place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) with program_guard(Program(), Program()): x1 = paddle.randperm(n) x2 = paddle.randperm(n, 'float32') @@ -130,20 +135,19 @@ def test_out(self): class TestRandpermImperative(unittest.TestCase): - def test_out(self): paddle.disable_static() n = 10 for dtype in ['int32', np.int64, 'float32', 'float64']: data_p = paddle.randperm(n, dtype) data_np = data_p.numpy() - self.assertTrue(check_randperm_out(n, data_np), - msg=error_msg(data_np)) + self.assertTrue( + check_randperm_out(n, data_np), msg=error_msg(data_np) + ) paddle.enable_static() class TestRandpermEager(unittest.TestCase): - def test_out(self): paddle.disable_static() n = 10 @@ -151,13 +155,13 @@ def test_out(self): for dtype in ['int32', np.int64, 'float32', 'float64']: data_p = paddle.randperm(n, dtype) data_np = data_p.numpy() - self.assertTrue(check_randperm_out(n, data_np), - msg=error_msg(data_np)) + self.assertTrue( + check_randperm_out(n, data_np), msg=error_msg(data_np) + ) paddle.enable_static() class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -170,25 +174,70 @@ def test_fixed_random_number(self): x = paddle.randperm(30000, dtype='int32').numpy() expect = [ - 24562, 8409, 9379, 10328, 20503, 18059, 9681, 21883, 11783, 27413 + 24562, + 8409, + 9379, + 10328, + 20503, + 18059, + 9681, + 21883, + 11783, + 27413, ] np.testing.assert_array_equal(x[0:10], expect) expect = [ - 29477, 27100, 9643, 16637, 8605, 16892, 27767, 2724, 1612, 13096 + 29477, + 27100, + 9643, + 16637, + 8605, + 16892, + 27767, + 2724, + 1612, + 13096, ] np.testing.assert_array_equal(x[10000:10010], expect) expect = [ - 298, 4104, 16479, 22714, 28684, 7510, 14667, 9950, 15940, 28343 + 298, + 4104, + 16479, + 22714, + 28684, + 7510, + 14667, + 9950, + 15940, + 28343, ] np.testing.assert_array_equal(x[20000:20010], expect) x = paddle.randperm(30000, dtype='int64').numpy() expect = [ - 6587, 1909, 5525, 23001, 6488, 14981, 14355, 3083, 29561, 8171 + 6587, + 1909, + 5525, + 23001, + 6488, + 14981, + 14355, + 3083, + 29561, + 8171, ] np.testing.assert_array_equal(x[0:10], expect) expect = [ - 23460, 12394, 22501, 5427, 20185, 9100, 5127, 1651, 25806, 4818 + 23460, + 12394, + 22501, + 5427, + 20185, + 9100, + 5127, + 1651, + 25806, + 4818, ] np.testing.assert_array_equal(x[10000:10010], expect) expect = [5829, 4508, 16193, 24836, 8526, 242, 9984, 9243, 1977, 11839] @@ -196,35 +245,83 @@ def test_fixed_random_number(self): x = paddle.randperm(30000, dtype='float32').numpy() expect = [ - 5154., 10537., 14362., 29843., 27185., 28399., 27561., 4144., - 22906., 10705. + 5154.0, + 10537.0, + 14362.0, + 29843.0, + 27185.0, + 28399.0, + 27561.0, + 4144.0, + 22906.0, + 10705.0, ] np.testing.assert_array_equal(x[0:10], expect) expect = [ - 1958., 18414., 20090., 21910., 22746., 27346., 22347., 3002., 4564., - 26991. + 1958.0, + 18414.0, + 20090.0, + 21910.0, + 22746.0, + 27346.0, + 22347.0, + 3002.0, + 4564.0, + 26991.0, ] np.testing.assert_array_equal(x[10000:10010], expect) expect = [ - 25580., 12606., 553., 16387., 29536., 4241., 20946., 16899., 16339., - 4662. + 25580.0, + 12606.0, + 553.0, + 16387.0, + 29536.0, + 4241.0, + 20946.0, + 16899.0, + 16339.0, + 4662.0, ] np.testing.assert_array_equal(x[20000:20010], expect) x = paddle.randperm(30000, dtype='float64').numpy() expect = [ - 19051., 2449., 21940., 11121., 282., 7330., 13747., 24321., 21147., - 9163. + 19051.0, + 2449.0, + 21940.0, + 11121.0, + 282.0, + 7330.0, + 13747.0, + 24321.0, + 21147.0, + 9163.0, ] np.testing.assert_array_equal(x[0:10], expect) expect = [ - 15483., 1315., 5723., 20954., 13251., 25539., 5074., 1823., 14945., - 17624. + 15483.0, + 1315.0, + 5723.0, + 20954.0, + 13251.0, + 25539.0, + 5074.0, + 1823.0, + 14945.0, + 17624.0, ] np.testing.assert_array_equal(x[10000:10010], expect) expect = [ - 10516., 2552., 29970., 5941., 986., 8007., 24805., 26753., 12202., - 21404. + 10516.0, + 2552.0, + 29970.0, + 5941.0, + 986.0, + 8007.0, + 24805.0, + 26753.0, + 12202.0, + 21404.0, ] np.testing.assert_array_equal(x[20000:20010], expect) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_range.py b/python/paddle/fluid/tests/unittests/test_range.py index cdf4e8cd5e5bc..cd6057e13b5c9 100644 --- a/python/paddle/fluid/tests/unittests/test_range.py +++ b/python/paddle/fluid/tests/unittests/test_range.py @@ -24,20 +24,19 @@ def arange_wrapper(start, end, step, dtype=None): class TestRangeOp(OpTest): - def setUp(self): self.op_type = "range" self.init_config() self.inputs = { 'Start': np.array([self.case[0]]).astype(self.dtype), 'End': np.array([self.case[1]]).astype(self.dtype), - 'Step': np.array([self.case[2]]).astype(self.dtype) + 'Step': np.array([self.case[2]]).astype(self.dtype), } self.outputs = { - 'Out': - np.arange(self.case[0], self.case[1], - self.case[2]).astype(self.dtype) + 'Out': np.arange(self.case[0], self.case[1], self.case[2]).astype( + self.dtype + ) } def init_config(self): @@ -50,7 +49,6 @@ def test_check_output(self): class TestFloatRangeOpCase0(TestRangeOp): - def init_config(self): self.dtype = np.float32 self.python_api = partial(arange_wrapper, dtype=self.dtype) @@ -58,7 +56,6 @@ def init_config(self): class TestInt32RangeOpCase0(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.python_api = partial(arange_wrapper, dtype=self.dtype) @@ -66,7 +63,6 @@ def init_config(self): class TestInt32RangeOpCase1(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.python_api = partial(arange_wrapper, dtype=self.dtype) @@ -74,7 +70,6 @@ def init_config(self): class TestInt32RangeOpCase2(TestRangeOp): - def init_config(self): self.dtype = np.int32 self.python_api = partial(arange_wrapper, dtype=self.dtype) diff --git a/python/paddle/fluid/tests/unittests/test_rank_attention_op.py b/python/paddle/fluid/tests/unittests/test_rank_attention_op.py index 4974fd26d1c6d..679475a58843c 100644 --- a/python/paddle/fluid/tests/unittests/test_rank_attention_op.py +++ b/python/paddle/fluid/tests/unittests/test_rank_attention_op.py @@ -60,7 +60,7 @@ def gen_param_help(input, rank_offset, param, max_rank): output_param_row = block_matrix_row * input_row output_param_col = param_col - output_param = np.zeros((output_param_row * output_param_col, )) + output_param = np.zeros((output_param_row * output_param_col,)) for idx in range(output_param_row * output_param_col): output_col_idx = idx % output_param_col @@ -75,7 +75,11 @@ def gen_param_help(input, rank_offset, param, max_rank): if lower < 0 or faster < 0: continue start = lower * max_rank + faster - ori_idx = start * param_col * input_col + k_offset * param_col + output_col_idx + ori_idx = ( + start * param_col * input_col + + k_offset * param_col + + output_col_idx + ) output_param[idx] = param[int(ori_idx / param_col), ori_idx % param_col] output_param = output_param.reshape([output_param_row, output_param_col]) @@ -87,20 +91,25 @@ def np_rank_attention(input, rank_offset, rank_para, max_rank, max_size): rank_offset_row, rank_offset_col = rank_offset.shape rank_para_row, rank_para_col = rank_para.shape - assert (input_row == rank_offset_row) - assert (max_rank == ((rank_offset_col - 1) / 2)) - assert (rank_para_row == max_rank * max_rank * input_col) + assert input_row == rank_offset_row + assert max_rank == ((rank_offset_col - 1) / 2) + assert rank_para_row == max_rank * max_rank * input_col - input_help, ins_rank = gen_input_help(input, rank_offset, max_rank, - max_size) + input_help, ins_rank = gen_input_help( + input, rank_offset, max_rank, max_size + ) param_help = gen_param_help(input, rank_offset, rank_para, max_rank) block_matrix_row = input_col * max_rank res = np.zeros((input_row, rank_para_col)) for ins in range(input_row): - res[ins, :] = \ - np.dot(input_help[ins, :], - param_help[int(block_matrix_row * ins):int(block_matrix_row * (ins+1)),:]) + res[ins, :] = np.dot( + input_help[ins, :], + param_help[ + int(block_matrix_row * ins) : int(block_matrix_row * (ins + 1)), + :, + ], + ) return res, input_help, param_help, ins_rank @@ -142,7 +151,6 @@ def gen_rank_offset(pv_nums, max_rank): class TestRankAttentionOpComplex(OpTest): - def config(self): self.pv_num = 100 self.x_feat = 10 @@ -156,22 +164,27 @@ def setUp(self): ins_num, rank_offset = gen_rank_offset(self.pv_num, self.max_rank) input = np.random.random((ins_num, self.x_feat)).astype(self.dtype) rank_para_shape = [ - self.max_rank * self.max_rank * self.x_feat, self.y_feat + self.max_rank * self.max_rank * self.x_feat, + self.y_feat, ] rank_para = np.random.random(rank_para_shape).astype(self.dtype) np_out, np_input_help, np_param_help, np_ins_rank = np_rank_attention( - input, np.array(rank_offset), rank_para, self.max_rank, - self.pv_num * 7) + input, + np.array(rank_offset), + rank_para, + self.max_rank, + self.pv_num * 7, + ) self.inputs = { "X": input, "RankOffset": np.array(rank_offset).astype("int32"), - "RankParam": rank_para + "RankParam": rank_para, } self.attrs = {'MaxRank': self.max_rank, 'MaxSize': self.pv_num * 7} self.outputs = { "Out": np_out, "InputHelp": np_input_help, - "InsRank": np_ins_rank + "InsRank": np_ins_rank, } def test_check_output_gpu(self): @@ -184,7 +197,6 @@ def test_check_grad_gpu(self): class TestRankAttentionOpCpu(OpTest): - def config(self): self.pv_num = 100 self.x_feat = 10 @@ -198,22 +210,27 @@ def setUp(self): ins_num, rank_offset = gen_rank_offset(self.pv_num, self.max_rank) input = np.random.random((ins_num, self.x_feat)).astype(self.dtype) rank_para_shape = [ - self.max_rank * self.max_rank * self.x_feat, self.y_feat + self.max_rank * self.max_rank * self.x_feat, + self.y_feat, ] rank_para = np.random.random(rank_para_shape).astype(self.dtype) np_out, np_input_help, np_param_help, np_ins_rank = np_rank_attention( - input, np.array(rank_offset), rank_para, self.max_rank, - self.pv_num * 7) + input, + np.array(rank_offset), + rank_para, + self.max_rank, + self.pv_num * 7, + ) self.inputs = { "X": input, "RankOffset": np.array(rank_offset).astype("int32"), - "RankParam": rank_para + "RankParam": rank_para, } self.attrs = {'MaxRank': self.max_rank, 'MaxSize': self.pv_num * 7} self.outputs = { "Out": np_out, "InputHelp": np_input_help, - "InsRank": np_ins_rank + "InsRank": np_ins_rank, } def test_check_output_cpu(self): diff --git a/python/paddle/fluid/tests/unittests/test_rank_loss_op.py b/python/paddle/fluid/tests/unittests/test_rank_loss_op.py index 5ef05e16c967b..1ace41d2d24b2 100644 --- a/python/paddle/fluid/tests/unittests/test_rank_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_rank_loss_op.py @@ -20,7 +20,6 @@ class TestRankLossOp(OpTest): - def setUp(self): self.op_type = "rank_loss" shape = (100, 1) @@ -34,7 +33,7 @@ def setUp(self): self.inputs = { 'Label': label.reshape(label_shape), 'Left': left.reshape(left_shape), - 'Right': right.reshape(right_shape) + 'Right': right.reshape(right_shape), } self.outputs = {'Out': loss.reshape(label_shape)} @@ -56,42 +55,36 @@ def test_check_grad_ignore_right(self): class TestRankLossOp1(TestRankLossOp): - def set_shape(self): batch_size = 100 return (batch_size), (batch_size, 1), (batch_size, 1) class TestRankLossOp2(TestRankLossOp): - def set_shape(self): batch_size = 100 return (batch_size, 1), (batch_size), (batch_size, 1) class TestRankLossOp3(TestRankLossOp): - def set_shape(self): batch_size = 100 return (batch_size, 1), (batch_size, 1), (batch_size) class TestRankLossOp4(TestRankLossOp): - def set_shape(self): batch_size = 100 return (batch_size), (batch_size), (batch_size, 1) class TestRankLossOp5(TestRankLossOp): - def set_shape(self): batch_size = 100 return (batch_size), (batch_size), (batch_size) class TestRankLossOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): label = fluid.data(name="label", shape=[16, 1], dtype="float32") diff --git a/python/paddle/fluid/tests/unittests/test_raw_program_optimizer.py b/python/paddle/fluid/tests/unittests/test_raw_program_optimizer.py index eb11bf975dd02..fe86a667d1e88 100644 --- a/python/paddle/fluid/tests/unittests/test_raw_program_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_raw_program_optimizer.py @@ -22,7 +22,6 @@ class TestRawProgramOptimizer(unittest.TestCase): - def setUp(self): os.environ["PADDLE_TRAINER_ID"] = "0" os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" @@ -30,18 +29,19 @@ def setUp(self): def mlp(self, input_x, input_y, hid_dim=128, label_dim=2): fc_1 = paddle.static.nn.fc(x=input_x, size=hid_dim, activation='tanh') fc_2 = paddle.static.nn.fc(x=fc_1, size=hid_dim, activation='tanh') - prediction = paddle.static.nn.fc(x=[fc_2], - size=label_dim, - activation='softmax') - cost = paddle.nn.functional.cross_entropy(input=prediction, - label=input_y) + prediction = paddle.static.nn.fc( + x=[fc_2], size=label_dim, activation='softmax' + ) + cost = paddle.nn.functional.cross_entropy( + input=prediction, label=input_y + ) avg_cost = paddle.mean(x=cost) return avg_cost def gen_data(self): return { "x": np.random.random(size=(128, 32)).astype('float32'), - "y": np.random.randint(2, size=(128, 1)).astype('int64') + "y": np.random.randint(2, size=(128, 1)).astype('int64'), } def test_single_gpu(self): @@ -53,16 +53,17 @@ def test_single_gpu(self): strategy.without_graph_optimization = True with fluid.program_guard(sharding_program, sharding_startup_program): with fluid.unique_name.guard(): - input_x = paddle.static.data(name="x", - shape=[None, 32], - dtype='float32') - input_y = paddle.static.data(name="y", - shape=[None, 1], - dtype='int64') + input_x = paddle.static.data( + name="x", shape=[None, 32], dtype='float32' + ) + input_y = paddle.static.data( + name="y", shape=[None, 1], dtype='int64' + ) cost = self.mlp(input_x=input_x, input_y=input_y) output_name = cost.name - optimizer = fleet.distributed_optimizer(fluid.optimizer.Adam(), - strategy) + optimizer = fleet.distributed_optimizer( + fluid.optimizer.Adam(), strategy + ) optimizer.minimize(cost) trainer_id = fleet.worker_index() diff --git a/python/paddle/fluid/tests/unittests/test_reader_reset.py b/python/paddle/fluid/tests/unittests/test_reader_reset.py index f3611059b2dbb..edd3c6c28b113 100644 --- a/python/paddle/fluid/tests/unittests/test_reader_reset.py +++ b/python/paddle/fluid/tests/unittests/test_reader_reset.py @@ -23,9 +23,7 @@ class TestReaderReset(unittest.TestCase): - def prepare_data(self): - def fake_data_generator(): for n in range(self.total_ins_num): yield np.ones(self.ins_shape) * n, n @@ -46,15 +44,16 @@ def main(self, with_double_buffer): startup_prog = fluid.Program() with fluid.program_guard(main_prog, startup_prog): - image = fluid.layers.data(name='image', - shape=self.ins_shape, - dtype='float32') + image = fluid.layers.data( + name='image', shape=self.ins_shape, dtype='float32' + ) label = fluid.layers.data(name='label', shape=[1], dtype='int64') data_reader_handle = fluid.io.PyReader( feed_list=[image, label], capacity=16, iterable=False, - use_double_buffer=with_double_buffer) + use_double_buffer=with_double_buffer, + ) fetch_list = [image.name, label.name] place = fluid.CUDAPlace(0) if self.use_cuda else fluid.CPUPlace() @@ -62,10 +61,12 @@ def main(self, with_double_buffer): exe.run(startup_prog) data_reader_handle.decorate_sample_list_generator( - paddle.batch(self.prepare_data(), batch_size=self.batch_size)) + paddle.batch(self.prepare_data(), batch_size=self.batch_size) + ) train_cp = compiler.CompiledProgram(main_prog).with_data_parallel( - places=[place]) + places=[place] + ) batch_id = 0 pass_count = 0 @@ -73,13 +74,13 @@ def main(self, with_double_buffer): data_reader_handle.start() try: while True: - data_val, label_val = exe.run(train_cp, - fetch_list=fetch_list, - return_numpy=True) + data_val, label_val = exe.run( + train_cp, fetch_list=fetch_list, return_numpy=True + ) ins_num = data_val.shape[0] - broadcasted_label = np.ones(( - ins_num, ) + tuple(self.ins_shape)) * label_val.reshape( - (ins_num, 1)) + broadcasted_label = np.ones( + (ins_num,) + tuple(self.ins_shape) + ) * label_val.reshape((ins_num, 1)) self.assertEqual(data_val.all(), broadcasted_label.all()) batch_id += 1 except fluid.core.EOFException: diff --git a/python/paddle/fluid/tests/unittests/test_real_imag_op.py b/python/paddle/fluid/tests/unittests/test_real_imag_op.py index 8b0a0ad635b1a..bcd8f3c561edc 100644 --- a/python/paddle/fluid/tests/unittests/test_real_imag_op.py +++ b/python/paddle/fluid/tests/unittests/test_real_imag_op.py @@ -32,7 +32,6 @@ class TestRealOp(OpTest): - def setUp(self): # switch to static paddle.enable_static() @@ -46,31 +45,31 @@ def setUp(self): def init_input_output(self): self.inputs = { - 'X': - np.random.random( - (20, 5)).astype(self.dtype) + 1j * np.random.random( - (20, 5)).astype(self.dtype) + 'X': np.random.random((20, 5)).astype(self.dtype) + + 1j * np.random.random((20, 5)).astype(self.dtype) } self.outputs = {'Out': numpy_apis[self.op_type](self.inputs['X'])} def init_grad_input_output(self): self.grad_out = np.ones((20, 5), self.dtype) - self.grad_x = np.real( - self.grad_out) + 1j * np.zeros(self.grad_out.shape) + self.grad_x = np.real(self.grad_out) + 1j * np.zeros( + self.grad_out.shape + ) def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=[self.grad_x], - user_defined_grad_outputs=[self.grad_out], - check_eager=True) + self.check_grad( + ['X'], + 'Out', + user_defined_grads=[self.grad_x], + user_defined_grad_outputs=[self.grad_out], + check_eager=True, + ) class TestImagOp(TestRealOp): - def setUp(self): # switch to static paddle.enable_static() @@ -84,12 +83,12 @@ def setUp(self): def init_grad_input_output(self): self.grad_out = np.ones((20, 5), self.dtype) - self.grad_x = np.zeros( - self.grad_out.shape) + 1j * np.real(self.grad_out) + self.grad_x = np.zeros(self.grad_out.shape) + 1j * np.real( + self.grad_out + ) class TestRealAPI(unittest.TestCase): - def setUp(self): # switch to static paddle.enable_static() @@ -102,10 +101,10 @@ def setUp(self): self._shape = [2, 20, 2, 3] def test_in_static_mode(self): - def init_input_output(dtype): input = np.random.random(self._shape).astype( - dtype) + 1j * np.random.random(self._shape).astype(dtype) + dtype + ) + 1j * np.random.random(self._shape).astype(dtype) return {'x': input}, numpy_apis[self.api](input) for dtype in self.dtypes: @@ -122,7 +121,8 @@ def init_input_output(dtype): def test_in_dynamic_mode(self): for dtype in self.dtypes: input = np.random.random(self._shape).astype( - dtype) + 1j * np.random.random(self._shape).astype(dtype) + dtype + ) + 1j * np.random.random(self._shape).astype(dtype) np_res = numpy_apis[self.api](input) for place in self.places: # it is more convenient to use `guard` than `enable/disable_**` here @@ -130,8 +130,11 @@ def test_in_dynamic_mode(self): input_t = paddle.to_tensor(input) res = paddle_apis[self.api](input_t).numpy() np.testing.assert_array_equal(np_res, res) - res_t = input_t.real().numpy( - ) if self.api == "real" else input_t.imag().numpy() + res_t = ( + input_t.real().numpy() + if self.api == "real" + else input_t.imag().numpy() + ) np.testing.assert_array_equal(np_res, res_t) def test_name_argument(self): @@ -156,7 +159,6 @@ def test_dtype_error(self): class TestImagAPI(TestRealAPI): - def setUp(self): # switch to static paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_recurrent_op.py b/python/paddle/fluid/tests/unittests/test_recurrent_op.py index 1153689bb941a..37ebcb77003c1 100644 --- a/python/paddle/fluid/tests/unittests/test_recurrent_op.py +++ b/python/paddle/fluid/tests/unittests/test_recurrent_op.py @@ -28,7 +28,6 @@ class PyRNNBase(object): - def __init__(self, input_shape, output_shape): self.x = np.ones(shape=input_shape).astype("float32") self.y = np.zeros(shape=output_shape).astype("float32") @@ -46,13 +45,13 @@ def segment_inputs(self): class PySimpleRNN1(PyRNNBase): - def __init__(self, input_shape, output_shape): super(PySimpleRNN1, self).__init__(input_shape, output_shape) seq_len, batch_size, input_dim = input_shape - self.h_boot = np.random.normal(size=(batch_size, - input_dim)).astype("float32") + self.h_boot = np.random.normal(size=(batch_size, input_dim)).astype( + "float32" + ) self.scale = 1.0 / 2.0 men_dim = (seq_len, batch_size, input_dim) @@ -68,7 +67,6 @@ def step(self, step_id, x): class PySimpleRNN2(PyRNNBase): - def __init__(self, input_shape, output_shape): super(PySimpleRNN2, self).__init__(input_shape, output_shape) @@ -89,7 +87,7 @@ def step(self, step_id, x): hU = np.matmul(pre_mem, self.U).astype("float32") def py_sigmoid(x): - return 1. / (1. + np.exp(-x)) + return 1.0 / (1.0 + np.exp(-x)) self.mems[step_id] = py_sigmoid(xW + hU) self.y[step_id] = self.mems[step_id] @@ -136,14 +134,16 @@ def setUp(self): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False - h_boot = layers.data(shape=[self.input_dim], - dtype='float32', - name='h_boot') + h_boot = layers.data( + shape=[self.input_dim], dtype='float32', name='h_boot' + ) h_boot.stop_gradient = False rnn = layers.StaticRNN() @@ -151,8 +151,10 @@ def create_rnn_op(self): h_pre = rnn.memory(init=h_boot) x_t = rnn.step_input(x) - h = layers.scale(x=layers.elementwise_add(x=h_pre, y=x_t), - scale=self.py_rnn.scale) + h = layers.scale( + x=layers.elementwise_add(x=h_pre, y=x_t), + scale=self.py_rnn.scale, + ) rnn.update_memory(h_pre, h) rnn.output(h) @@ -165,9 +167,9 @@ def forward(self): for x in self.feed_data_field } exe = Executor(self.place) - out = exe.run(self.main_program, - feed=self.feed_map, - fetch_list=[self.output]) + out = exe.run( + self.main_program, feed=self.feed_map, fetch_list=[self.output] + ) return out[0] @@ -182,10 +184,12 @@ def backward(self): ] exe = Executor(self.place) - return exe.run(self.main_program, - feed=self.feed_map, - fetch_list=fetch_list, - return_numpy=False) + return exe.run( + self.main_program, + feed=self.feed_map, + fetch_list=fetch_list, + return_numpy=False, + ) def test_backward(self, rtol=0.01): self.check_forward() @@ -203,10 +207,18 @@ def test_backward(self, rtol=0.01): ana_grad[idx], rtol=rtol, atol=1e-8, - err_msg='num_grad (' + name + ') has diff at ' + - str(self.place) + '\nExpect ' + str(num_grad[idx]) + '\n' + - 'But Got' + str(ana_grad[idx]) + ' in class ' + - self.__class__.__name__) + err_msg='num_grad (' + + name + + ') has diff at ' + + str(self.place) + + '\nExpect ' + + str(num_grad[idx]) + + '\n' + + 'But Got' + + str(ana_grad[idx]) + + ' in class ' + + self.__class__.__name__, + ) def check_forward(self): pd_output = self.forward() @@ -268,14 +280,16 @@ def setUp(self): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False - h_boot = layers.data(shape=[self.input_dim], - dtype='float32', - name='h_boot') + h_boot = layers.data( + shape=[self.input_dim], dtype='float32', name='h_boot' + ) h_boot.stop_gradient = False rnn = layers.StaticRNN() @@ -288,15 +302,19 @@ def create_rnn_op(self): size=self.input_dim, param_attr=ParamAttr( name='W', - initializer=fluid.initializer.ConstantInitializer(1.0)), - bias_attr=False) + initializer=fluid.initializer.ConstantInitializer(1.0), + ), + bias_attr=False, + ) temp_r = layers.fc( input=h_pre, size=self.input_dim, param_attr=ParamAttr( name='U', - initializer=fluid.initializer.ConstantInitializer(0.0)), - bias_attr=False) + initializer=fluid.initializer.ConstantInitializer(0.0), + ), + bias_attr=False, + ) h = layers.sigmoid(x=layers.elementwise_add(x=temp_l, y=temp_r)) @@ -325,16 +343,18 @@ class RecurrentOpMultipleMemoryTest(RecurrentOpTest1): ''' class PySimpleRNN3(PyRNNBase): - def __init__(self, input_shape, output_shape): - super(RecurrentOpMultipleMemoryTest.PySimpleRNN3, - self).__init__(input_shape, output_shape) + super(RecurrentOpMultipleMemoryTest.PySimpleRNN3, self).__init__( + input_shape, output_shape + ) seq_len, batch_size, input_dim = input_shape - self.h_boot1 = np.random.normal(size=(batch_size, - input_dim)).astype("float32") - self.h_boot2 = np.random.normal(size=(batch_size, - input_dim)).astype("float32") + self.h_boot1 = np.random.normal( + size=(batch_size, input_dim) + ).astype("float32") + self.h_boot2 = np.random.normal( + size=(batch_size, input_dim) + ).astype("float32") men_dim = (seq_len, batch_size, input_dim) self.mems1 = np.zeros(shape=men_dim).astype("float32") @@ -364,26 +384,33 @@ def setUp(self): self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = RecurrentOpMultipleMemoryTest.PySimpleRNN3( - self.input_shape, self.output_shape) + self.input_shape, self.output_shape + ) with fluid.program_guard(self.main_program, self.startup_program): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False - h_boot1 = layers.data(shape=[self.batch_size, self.input_dim], - dtype='float32', - name='h_boot1', - append_batch_size=False) + h_boot1 = layers.data( + shape=[self.batch_size, self.input_dim], + dtype='float32', + name='h_boot1', + append_batch_size=False, + ) h_boot1.stop_gradient = False - h_boot2 = layers.data(shape=[self.batch_size, self.input_dim], - dtype='float32', - name='h_boot2', - append_batch_size=False) + h_boot2 = layers.data( + shape=[self.batch_size, self.input_dim], + dtype='float32', + name='h_boot2', + append_batch_size=False, + ) h_boot2.stop_gradient = False rnn = layers.StaticRNN() @@ -418,10 +445,10 @@ class RecurrentOpNoMemBootTest(RecurrentOpTest1): ''' class PySimpleRNN4(PyRNNBase): - def __init__(self, input_shape, output_shape): - super(RecurrentOpNoMemBootTest.PySimpleRNN4, - self).__init__(input_shape, output_shape) + super(RecurrentOpNoMemBootTest.PySimpleRNN4, self).__init__( + input_shape, output_shape + ) men_dim = input_shape self.mems = np.zeros(shape=men_dim).astype("float32") @@ -446,16 +473,19 @@ def setUp(self): self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = RecurrentOpNoMemBootTest.PySimpleRNN4( - self.input_shape, self.output_shape) + self.input_shape, self.output_shape + ) with fluid.program_guard(self.main_program, self.startup_program): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False rnn = layers.StaticRNN() @@ -491,31 +521,28 @@ class RecurrentOpSubBlockTest(RecurrentOpTest1): ''' class PySimpleRNN5(PyRNNBase): - def __init__(self, input_shape, output_shape): - super(RecurrentOpSubBlockTest.PySimpleRNN5, - self).__init__(input_shape, output_shape) + super(RecurrentOpSubBlockTest.PySimpleRNN5, self).__init__( + input_shape, output_shape + ) seq_len, batch_size, input_dim = input_shape - self.w1 = np.random.uniform(-0.1, 0.1, - size=(input_dim, - input_dim)).astype("float32") - self.w2 = np.random.uniform(-0.1, - 0.1, - size=(input_dim * 2, - input_dim)).astype("float32") - - self.emb = np.random.uniform(-0.1, - 0.1, - size=(seq_len, batch_size, - input_dim)).astype("float32") + self.w1 = np.random.uniform( + -0.1, 0.1, size=(input_dim, input_dim) + ).astype("float32") + self.w2 = np.random.uniform( + -0.1, 0.1, size=(input_dim * 2, input_dim) + ).astype("float32") + + self.emb = np.random.uniform( + -0.1, 0.1, size=(seq_len, batch_size, input_dim) + ).astype("float32") men_dim = (seq_len, batch_size, input_dim) self.mems = np.zeros(shape=men_dim).astype("float32") self.oy = np.matmul(self.emb, self.w1) def step(self, step_id, x): - def dot_attention(query, memory): attn = np.matmul(query, memory.transpose((0, 2, 1))) weight = softmax(attn) @@ -552,35 +579,43 @@ def setUp(self): self.input_shape = (self.sent_len, self.batch_size, self.input_dim) self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = RecurrentOpSubBlockTest.PySimpleRNN5( - self.input_shape, self.output_shape) + self.input_shape, self.output_shape + ) with fluid.program_guard(self.main_program, self.startup_program): rnn_out = self.create_rnn_op() self.output = paddle.mean(rnn_out) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype='float32', - name='x', - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype='float32', + name='x', + append_batch_size=False, + ) x.stop_gradient = False emb = layers.data( name='emb', shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', - append_batch_size=False) + append_batch_size=False, + ) emb.stop_gradient = False - w1 = layers.data(shape=[self.input_dim, self.input_dim], - dtype='float32', - name='w1', - append_batch_size=False) + w1 = layers.data( + shape=[self.input_dim, self.input_dim], + dtype='float32', + name='w1', + append_batch_size=False, + ) w1.stop_gradient = False - w2 = layers.data(shape=[self.input_dim * 2, self.input_dim], - dtype='float32', - name='w2', - append_batch_size=False) + w2 = layers.data( + shape=[self.input_dim * 2, self.input_dim], + dtype='float32', + name='w2', + append_batch_size=False, + ) w2.stop_gradient = False rnn = layers.StaticRNN() @@ -594,9 +629,11 @@ def dot_attention(query, memory): y = layers.matmul(emb, w1) with rnn.step(): - pre_h = rnn.memory(shape=(self.sent_len, self.input_dim), - batch_ref=x, - init_value=0.0) + pre_h = rnn.memory( + shape=(self.sent_len, self.input_dim), + batch_ref=x, + init_value=0.0, + ) step_in = rnn.step_input(x) concat_in = layers.concat([step_in, pre_h], 1) new_h = layers.matmul(concat_in, w2) @@ -643,14 +680,16 @@ def setUp(self): self.output = paddle.mean(self.create_rnn_op()) def create_rnn_op(self): - x = layers.data(shape=[self.sent_len, self.batch_size, self.input_dim], - dtype="float32", - name="x", - append_batch_size=False) + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + dtype="float32", + name="x", + append_batch_size=False, + ) x.stop_gradient = False - h_boot = layers.data(shape=[self.input_dim], - dtype="float32", - name="h_boot") + h_boot = layers.data( + shape=[self.input_dim], dtype="float32", name="h_boot" + ) h_boot.stop_gradient = True rnn = layers.StaticRNN() @@ -663,15 +702,19 @@ def create_rnn_op(self): size=self.input_dim, param_attr=ParamAttr( name="W", - initializer=fluid.initializer.ConstantInitializer(1.0)), - bias_attr=False) + initializer=fluid.initializer.ConstantInitializer(1.0), + ), + bias_attr=False, + ) temp_r = layers.fc( input=h_pre, size=self.input_dim, param_attr=ParamAttr( name="U", - initializer=fluid.initializer.ConstantInitializer(0.0)), - bias_attr=False) + initializer=fluid.initializer.ConstantInitializer(0.0), + ), + bias_attr=False, + ) h = layers.sigmoid(x=layers.elementwise_add(temp_l, temp_r)) diff --git a/python/paddle/fluid/tests/unittests/test_reduce_op.py b/python/paddle/fluid/tests/unittests/test_reduce_op.py index 8dba9dd1b56ca..8fa448a6927dd 100644 --- a/python/paddle/fluid/tests/unittests/test_reduce_op.py +++ b/python/paddle/fluid/tests/unittests/test_reduce_op.py @@ -23,7 +23,6 @@ class TestSumOp(OpTest): - def setUp(self): self.python_api = paddle.sum self.op_type = "reduce_sum" @@ -39,7 +38,6 @@ def test_check_grad(self): class TestSumOp_fp16(OpTest): - def setUp(self): self.python_api = paddle.sum self.op_type = "reduce_sum" @@ -58,19 +56,18 @@ def test_check_output(self): def calc_gradient(self): x = self.inputs["X"] grad = np.ones(x.shape, dtype=x.dtype) - return grad, + return (grad,) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=self.gradient, - check_eager=True) + self.check_grad( + ['X'], 'Out', user_defined_grads=self.gradient, check_eager=True + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSumOp_bf16(OpTest): - def setUp(self): np.random.seed(100) self.python_api = paddle.sum @@ -91,10 +88,13 @@ def test_check_output(self): def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - user_defined_grads=self.gradient, - check_eager=True) + self.check_grad_with_place( + place, + ['X'], + 'Out', + user_defined_grads=self.gradient, + check_eager=True, + ) def calc_gradient(self): x = self.x @@ -103,7 +103,6 @@ def calc_gradient(self): class TestSumOp_fp16_withInt(OpTest): - def setUp(self): self.python_api = paddle.sum self.op_type = "reduce_sum" @@ -124,17 +123,15 @@ def test_check_output(self): def calc_gradient(self): x = self.inputs["X"] grad = np.ones(x.shape, dtype=x.dtype) - return grad, + return (grad,) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - user_defined_grads=self.gradient, - check_eager=True) + self.check_grad( + ['X'], 'Out', user_defined_grads=self.gradient, check_eager=True + ) class TestSumOp5D(OpTest): - def setUp(self): self.python_api = paddle.sum self.op_type = "reduce_sum" @@ -152,7 +149,6 @@ def test_check_grad(self): class TestSumOp6D(OpTest): - def setUp(self): self.python_api = paddle.sum self.op_type = "reduce_sum" @@ -170,7 +166,6 @@ def test_check_grad(self): class TestSumOp8D(OpTest): - def setUp(self): self.python_api = paddle.sum self.op_type = "reduce_sum" @@ -189,7 +184,8 @@ def test_check_grad(self): @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestMaxOp(OpTest): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -208,7 +204,8 @@ def test_check_output(self): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestMinOp(OpTest): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -266,7 +263,6 @@ def raw_reduce_prod(x, dim=[0], keep_dim=False): class TestProdOp(OpTest): - def setUp(self): self.op_type = "reduce_prod" self.python_api = raw_reduce_prod @@ -275,8 +271,9 @@ def setUp(self): self.outputs = {'Out': self.inputs['X'].prod(axis=0)} def init_data_type(self): - self.data_type = "float32" if core.is_compiled_with_rocm( - ) else "float64" + self.data_type = ( + "float32" if core.is_compiled_with_rocm() else "float64" + ) def test_check_output(self): self.check_output(check_eager=True) @@ -286,7 +283,6 @@ def test_check_grad(self): class TestProd6DOp(OpTest): - def setUp(self): self.op_type = "reduce_prod" self.python_api = raw_reduce_prod @@ -300,8 +296,9 @@ def setUp(self): } def init_data_type(self): - self.data_type = "float32" if core.is_compiled_with_rocm( - ) else "float64" + self.data_type = ( + "float32" if core.is_compiled_with_rocm() else "float64" + ) def test_check_output(self): self.check_output(check_eager=True) @@ -311,14 +308,14 @@ def test_check_grad(self): class TestProd8DOp(OpTest): - def setUp(self): self.op_type = "reduce_prod" self.python_api = raw_reduce_prod self.init_data_type() self.inputs = { - 'X': np.random.random( - (2, 5, 3, 2, 2, 3, 4, 2)).astype(self.data_type) + 'X': np.random.random((2, 5, 3, 2, 2, 3, 4, 2)).astype( + self.data_type + ) } self.attrs = {'dim': [2, 3, 4]} self.outputs = { @@ -326,8 +323,9 @@ def setUp(self): } def init_data_type(self): - self.data_type = "float32" if core.is_compiled_with_rocm( - ) else "float64" + self.data_type = ( + "float32" if core.is_compiled_with_rocm() else "float64" + ) def test_check_output(self): self.check_output(check_eager=True) @@ -337,7 +335,6 @@ def test_check_grad(self): class TestAllOp(OpTest): - def setUp(self): self.op_type = "reduce_all" self.python_api = paddle.all @@ -350,13 +347,13 @@ def test_check_output(self): class TestAll8DOp(OpTest): - def setUp(self): self.op_type = "reduce_all" self.python_api = paddle.all self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.attrs = {'reduce_all': True, 'dim': (2, 3, 4)} self.outputs = {'Out': self.inputs['X'].all(axis=self.attrs['dim'])} @@ -366,12 +363,11 @@ def test_check_output(self): class TestAllOpWithDim(OpTest): - def setUp(self): self.op_type = "reduce_all" self.python_api = paddle.all self.inputs = {'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool")} - self.attrs = {'dim': (1, )} + self.attrs = {'dim': (1,)} self.outputs = {'Out': self.inputs['X'].all(axis=self.attrs['dim'])} def test_check_output(self): @@ -379,13 +375,13 @@ def test_check_output(self): class TestAll8DOpWithDim(OpTest): - def setUp(self): self.op_type = "reduce_all" self.python_api = paddle.all self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.attrs = {'dim': (1, 3, 4)} self.outputs = {'Out': self.inputs['X'].all(axis=self.attrs['dim'])} @@ -395,7 +391,6 @@ def test_check_output(self): class TestAllOpWithKeepDim(OpTest): - def setUp(self): self.op_type = "reduce_all" self.python_api = paddle.all @@ -410,18 +405,19 @@ def test_check_output(self): class TestAll8DOpWithKeepDim(OpTest): - def setUp(self): self.op_type = "reduce_all" self.python_api = paddle.all self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } - self.attrs = {'dim': (5, ), 'keep_dim': True} + self.attrs = {'dim': (5,), 'keep_dim': True} self.outputs = { - 'Out': - np.expand_dims(self.inputs['X'].all(axis=self.attrs['dim']), axis=5) + 'Out': np.expand_dims( + self.inputs['X'].all(axis=self.attrs['dim']), axis=5 + ) } def test_check_output(self): @@ -429,21 +425,19 @@ def test_check_output(self): class TestAllOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of reduce_all_op must be Variable. input1 = 12 self.assertRaises(TypeError, fluid.layers.reduce_all, input1) # The input dtype of reduce_all_op must be bool. - input2 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") + input2 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) self.assertRaises(TypeError, fluid.layers.reduce_all, input2) class TestAnyOp(OpTest): - def setUp(self): self.op_type = "reduce_any" self.python_api = paddle.any @@ -456,13 +450,13 @@ def test_check_output(self): class TestAny8DOp(OpTest): - def setUp(self): self.op_type = "reduce_any" self.python_api = paddle.any self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.attrs = {'reduce_all': True, 'dim': (3, 5, 4)} self.outputs = {'Out': self.inputs['X'].any(axis=self.attrs['dim'])} @@ -472,7 +466,6 @@ def test_check_output(self): class TestAnyOpWithDim(OpTest): - def setUp(self): self.op_type = "reduce_any" self.python_api = paddle.any @@ -485,13 +478,13 @@ def test_check_output(self): class TestAny8DOpWithDim(OpTest): - def setUp(self): self.op_type = "reduce_any" self.python_api = paddle.any self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.attrs = {'dim': (3, 6)} self.outputs = {'Out': self.inputs['X'].any(axis=self.attrs['dim'])} @@ -501,15 +494,15 @@ def test_check_output(self): class TestAnyOpWithKeepDim(OpTest): - def setUp(self): self.op_type = "reduce_any" self.python_api = paddle.any self.inputs = {'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool")} - self.attrs = {'dim': (1, ), 'keep_dim': True} + self.attrs = {'dim': (1,), 'keep_dim': True} self.outputs = { - 'Out': - np.expand_dims(self.inputs['X'].any(axis=self.attrs['dim']), axis=1) + 'Out': np.expand_dims( + self.inputs['X'].any(axis=self.attrs['dim']), axis=1 + ) } def test_check_output(self): @@ -517,18 +510,19 @@ def test_check_output(self): class TestAny8DOpWithKeepDim(OpTest): - def setUp(self): self.op_type = "reduce_any" self.python_api = paddle.any self.inputs = { - 'X': np.random.randint(0, 2, - (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } - self.attrs = {'dim': (1, ), 'keep_dim': True} + self.attrs = {'dim': (1,), 'keep_dim': True} self.outputs = { - 'Out': - np.expand_dims(self.inputs['X'].any(axis=self.attrs['dim']), axis=1) + 'Out': np.expand_dims( + self.inputs['X'].any(axis=self.attrs['dim']), axis=1 + ) } def test_check_output(self): @@ -536,21 +530,19 @@ def test_check_output(self): class TestAnyOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of reduce_any_op must be Variable. input1 = 12 self.assertRaises(TypeError, fluid.layers.reduce_any, input1) # The input dtype of reduce_any_op must be bool. - input2 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") + input2 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) self.assertRaises(TypeError, fluid.layers.reduce_any, input2) class Test1DReduce(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random(120).astype("float64")} @@ -564,7 +556,6 @@ def test_check_grad(self): class Test2DReduce0(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.attrs = {'dim': [0]} @@ -573,7 +564,6 @@ def setUp(self): class Test2DReduce1(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.attrs = {'dim': [1]} @@ -584,7 +574,6 @@ def setUp(self): class Test3DReduce0(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.attrs = {'dim': [1]} @@ -595,7 +584,6 @@ def setUp(self): class Test3DReduce1(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.attrs = {'dim': [2]} @@ -606,7 +594,6 @@ def setUp(self): class Test3DReduce2(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.attrs = {'dim': [-2]} @@ -617,7 +604,6 @@ def setUp(self): class Test3DReduce3(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.attrs = {'dim': [1, 2]} @@ -628,7 +614,6 @@ def setUp(self): class Test8DReduce0(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.attrs = {'dim': (4, 2, 3)} @@ -641,20 +626,18 @@ def setUp(self): class TestKeepDimReduce(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((5, 6, 10)).astype("float64")} self.attrs = {'dim': [1], 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].sum(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].sum( + axis=tuple(self.attrs['dim']), keepdims=self.attrs['keep_dim'] + ) } class TestKeepDim8DReduce(Test1DReduce): - def setUp(self): self.op_type = "reduce_sum" self.inputs = { @@ -662,15 +645,16 @@ def setUp(self): } self.attrs = {'dim': (3, 4, 5), 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].sum(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].sum( + axis=tuple(self.attrs['dim']), keepdims=self.attrs['keep_dim'] + ) } @skip_check_grad_ci( reason="reduce_max is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMaxOpMultiAxises(OpTest): """Remove Max with subgradient from gradient check to confirm the success of CI.""" @@ -689,7 +673,8 @@ def test_check_output(self): @skip_check_grad_ci( reason="reduce_min is discontinuous non-derivable function," - " its gradient check is not supported by unittest framework.") + " its gradient check is not supported by unittest framework." +) class TestReduceMinOpMultiAxises(OpTest): """Remove Min with subgradient from gradient check to confirm the success of CI.""" @@ -707,14 +692,14 @@ def test_check_output(self): class TestKeepDimReduceSumMultiAxises(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((5, 6, 10)).astype("float64")} self.attrs = {'dim': [-2, -1], 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].sum(axis=tuple(self.attrs['dim']), keepdims=True) + 'Out': self.inputs['X'].sum( + axis=tuple(self.attrs['dim']), keepdims=True + ) } def test_check_output(self): @@ -725,14 +710,14 @@ def test_check_grad(self): class TestReduceSumWithDimOne(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((100, 1, 1)).astype("float64")} self.attrs = {'dim': [1, 2], 'keep_dim': True} self.outputs = { - 'Out': - self.inputs['X'].sum(axis=tuple(self.attrs['dim']), keepdims=True) + 'Out': self.inputs['X'].sum( + axis=tuple(self.attrs['dim']), keepdims=True + ) } def test_check_output(self): @@ -743,14 +728,14 @@ def test_check_grad(self): class TestReduceSumWithNumelOne(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((100, 1)).astype("float64")} self.attrs = {'dim': [1], 'keep_dim': False} self.outputs = { - 'Out': - self.inputs['X'].sum(axis=tuple(self.attrs['dim']), keepdims=False) + 'Out': self.inputs['X'].sum( + axis=tuple(self.attrs['dim']), keepdims=False + ) } def test_check_output(self): @@ -761,7 +746,6 @@ def test_check_grad(self): class TestReduceAll(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((100, 1, 1)).astype("float64")} @@ -776,7 +760,6 @@ def test_check_grad(self): class Test1DReduceWithAxes1(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random(100).astype("float64")} @@ -791,18 +774,17 @@ def test_check_grad(self): class TestReduceWithDtype(OpTest): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((6, 2, 10)).astype("float64")} self.outputs = {'Out': self.inputs['X'].sum().astype('float64')} self.attrs = {'reduce_all': True} - self.attrs.update({ - 'in_dtype': - int(convert_np_dtype_to_dtype_(np.float32)), - 'out_dtype': - int(convert_np_dtype_to_dtype_(np.float64)) - }) + self.attrs.update( + { + 'in_dtype': int(convert_np_dtype_to_dtype_(np.float32)), + 'out_dtype': int(convert_np_dtype_to_dtype_(np.float64)), + } + ) def test_check_output(self): self.check_output() @@ -812,42 +794,40 @@ def test_check_grad(self): class TestReduceWithDtype1(TestReduceWithDtype): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((6, 2, 10)).astype("float64")} self.outputs = {'Out': self.inputs['X'].sum(axis=1)} self.attrs = {'dim': [1]} - self.attrs.update({ - 'in_dtype': - int(convert_np_dtype_to_dtype_(np.float32)), - 'out_dtype': - int(convert_np_dtype_to_dtype_(np.float64)) - }) + self.attrs.update( + { + 'in_dtype': int(convert_np_dtype_to_dtype_(np.float32)), + 'out_dtype': int(convert_np_dtype_to_dtype_(np.float64)), + } + ) class TestReduceWithDtype2(TestReduceWithDtype): - def setUp(self): self.op_type = "reduce_sum" self.inputs = {'X': np.random.random((6, 2, 10)).astype("float64")} self.outputs = {'Out': self.inputs['X'].sum(axis=1, keepdims=True)} self.attrs = {'dim': [1], 'keep_dim': True} - self.attrs.update({ - 'in_dtype': - int(convert_np_dtype_to_dtype_(np.float32)), - 'out_dtype': - int(convert_np_dtype_to_dtype_(np.float64)) - }) + self.attrs.update( + { + 'in_dtype': int(convert_np_dtype_to_dtype_(np.float32)), + 'out_dtype': int(convert_np_dtype_to_dtype_(np.float64)), + } + ) class TestReduceSumOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of reduce_sum_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.reduce_sum, x1) # The input dtype of reduce_sum_op must be float32 or float64 or int32 or int64. x2 = fluid.layers.data(name='x2', shape=[4], dtype="uint8") @@ -855,13 +835,9 @@ def test_errors(self): class API_TestSumOp(unittest.TestCase): - - def run_static(self, - shape, - x_dtype, - attr_axis, - attr_dtype=None, - np_axis=None): + def run_static( + self, shape, x_dtype, attr_axis, attr_dtype=None, np_axis=None + ): if np_axis is None: np_axis = attr_axis @@ -871,19 +847,21 @@ def run_static(self, for place in places: with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.data("data", shape=shape, dtype=x_dtype) - result_sum = paddle.sum(x=data, - axis=attr_axis, - dtype=attr_dtype) + result_sum = paddle.sum( + x=data, axis=attr_axis, dtype=attr_dtype + ) exe = fluid.Executor(place) input_data = np.random.rand(*shape).astype(x_dtype) - res, = exe.run(feed={"data": input_data}, - fetch_list=[result_sum]) + (res,) = exe.run( + feed={"data": input_data}, fetch_list=[result_sum] + ) - np.testing.assert_allclose(res, - np.sum(input_data.astype(attr_dtype), - axis=np_axis), - rtol=1e-05) + np.testing.assert_allclose( + res, + np.sum(input_data.astype(attr_dtype), axis=np_axis), + rtol=1e-05, + ) def test_static(self): shape = [10, 10] @@ -914,10 +892,9 @@ def test_static(self): shape = [5, 5, 5] self.run_static(shape, "int32", (0, 1), attr_dtype="int32") - self.run_static(shape, - "int32", (), - attr_dtype="int32", - np_axis=(0, 1, 2)) + self.run_static( + shape, "int32", (), attr_dtype="int32", np_axis=(0, 1, 2) + ) def test_dygraph(self): np_x = np.random.random([2, 3, 4]).astype('int32') @@ -935,7 +912,6 @@ def test_dygraph(self): class TestAllAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) paddle.enable_static() @@ -950,9 +926,11 @@ def check_static_result(self, place): input_np = np.random.randint(0, 2, [4, 4]).astype("bool") exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], np.all(input_np), rtol=1e-05) def test_static(self): @@ -991,7 +969,6 @@ def test_dygraph(self): class TestAnyAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) paddle.enable_static() @@ -1006,9 +983,11 @@ def check_static_result(self, place): input_np = np.random.randint(0, 2, [4, 4]).astype("bool") exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": input_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": input_np}, + fetch_list=[result], + ) np.testing.assert_allclose(fetches[0], np.any(input_np), rtol=1e-05) def test_static(self): @@ -1048,5 +1027,6 @@ def test_dygraph(self): if __name__ == '__main__': import paddle + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_reducescatter.py b/python/paddle/fluid/tests/unittests/test_reducescatter.py index 8320eb9ad59e8..ed46c272bca47 100644 --- a/python/paddle/fluid/tests/unittests/test_reducescatter.py +++ b/python/paddle/fluid/tests/unittests/test_reducescatter.py @@ -21,13 +21,13 @@ class TestReduceScatterOp(TestDistBase): - def _setup_config(self): pass def test_reducescatter(self): - self.check_with_place("collective_reducescatter_op.py", - "reduce_scatter") + self.check_with_place( + "collective_reducescatter_op.py", "reduce_scatter" + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_reducescatter_api.py b/python/paddle/fluid/tests/unittests/test_reducescatter_api.py index decb90ada9672..e5d6b26153685 100644 --- a/python/paddle/fluid/tests/unittests/test_reducescatter_api.py +++ b/python/paddle/fluid/tests/unittests/test_reducescatter_api.py @@ -22,7 +22,6 @@ class TestReduceScatterAPI(TestDistBase): - def _setup_config(self): pass diff --git a/python/paddle/fluid/tests/unittests/test_registry.py b/python/paddle/fluid/tests/unittests/test_registry.py index be0ee288178ac..3fc6370b6c754 100644 --- a/python/paddle/fluid/tests/unittests/test_registry.py +++ b/python/paddle/fluid/tests/unittests/test_registry.py @@ -21,7 +21,6 @@ class TestRegistry(unittest.TestCase): - @prog_scope() def test_registry_layer(self): x = fluid.layers.data(name='X', shape=[10, 10], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/test_regularizer.py b/python/paddle/fluid/tests/unittests/test_regularizer.py index 46cad0aaf754b..a3f1697032b55 100644 --- a/python/paddle/fluid/tests/unittests/test_regularizer.py +++ b/python/paddle/fluid/tests/unittests/test_regularizer.py @@ -26,7 +26,6 @@ class TestL2DecayRegularizer(unittest.TestCase): - def test_l2decay_regularizer(self): paddle.enable_static() program = framework.Program() @@ -36,32 +35,30 @@ def test_l2decay_regularizer(self): shape=[5, 10], lod_level=0, name="mul.x", - regularizer=regularizer.L2DecayRegularizer(0.5)) + regularizer=regularizer.L2DecayRegularizer(0.5), + ) self.assertTrue(mul_x.regularizer is not None) self.assertTrue( - isinstance(mul_x.regularizer, regularizer.L2DecayRegularizer)) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + isinstance(mul_x.regularizer, regularizer.L2DecayRegularizer) + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) count_ops = len(block.ops) @@ -74,7 +71,6 @@ def test_l2decay_regularizer(self): class TestL1DecayRegularizer(unittest.TestCase): - def test_l2decay_regularizer(self): paddle.enable_static() program = framework.Program() @@ -84,32 +80,30 @@ def test_l2decay_regularizer(self): shape=[5, 10], lod_level=0, name="mul.x", - regularizer=regularizer.L1DecayRegularizer(0.5)) + regularizer=regularizer.L1DecayRegularizer(0.5), + ) self.assertTrue(mul_x.regularizer is not None) self.assertTrue( - isinstance(mul_x.regularizer, regularizer.L1DecayRegularizer)) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + isinstance(mul_x.regularizer, regularizer.L1DecayRegularizer) + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) params_grads = append_backward(mean_out) self.assertEqual(len(params_grads), 1) count_ops = len(block.ops) @@ -122,22 +116,24 @@ def test_l2decay_regularizer(self): self.assertEqual(block.ops[-3].type, 'sign') -def bow_net(data, - label, - dict_dim, - is_sparse=False, - emb_dim=8, - hid_dim=8, - hid_dim2=6, - class_dim=2): +def bow_net( + data, + label, + dict_dim, + is_sparse=False, + emb_dim=8, + hid_dim=8, + hid_dim2=6, + class_dim=2, +): """ BOW net This model is from https://github.com/PaddlePaddle/models: fluid/PaddleNLP/text_classification/nets.py """ - emb = fluid.layers.embedding(input=data, - is_sparse=is_sparse, - size=[dict_dim, emb_dim]) + emb = fluid.layers.embedding( + input=data, is_sparse=is_sparse, size=[dict_dim, emb_dim] + ) bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') bow_tanh = fluid.layers.tanh(bow) fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh") @@ -149,11 +145,11 @@ def bow_net(data, class TestRegularizer(unittest.TestCase): - def setUp(self): self.word_len = 1500 - self.train_data = [[(random.sample(range(1000), 10), [0])] - for _ in range(2)] + self.train_data = [ + [(random.sample(range(1000), 10), [0])] for _ in range(2) + ] def get_places(self): places = [core.CPUPlace()] @@ -179,9 +175,9 @@ def run_program(self, place, feed_list): param_sum = [] for data in self.train_data: - out = exe.run(main_prog, - feed=feeder.feed(data), - fetch_list=param_list) + out = exe.run( + main_prog, feed=feeder.feed(data), fetch_list=param_list + ) p_sum = 0 for v in out: p_sum += np.sum(np.abs(v)) @@ -193,19 +189,19 @@ def check_l2decay_regularizer(self, place, model): paddle.framework.random._manual_program_seed(1) main_prog = fluid.framework.Program() startup_prog = fluid.framework.Program() - with self.scope_prog_guard(main_prog=main_prog, - startup_prog=startup_prog): - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + with self.scope_prog_guard( + main_prog=main_prog, startup_prog=startup_prog + ): + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") avg_cost = model(data, label, self.word_len) optimizer = fluid.optimizer.Adagrad( - learning_rate=0.1, - regularization=fluid.regularizer.L2Decay(1.0)) + learning_rate=0.1, regularization=fluid.regularizer.L2Decay(1.0) + ) optimizer.minimize(avg_cost) param_sum = self.run_program(place, [data, label]) return param_sum @@ -216,12 +212,12 @@ def check_l2decay(self, place, model): main_prog = fluid.framework.Program() startup_prog = fluid.framework.Program() - with self.scope_prog_guard(main_prog=main_prog, - startup_prog=startup_prog): - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + with self.scope_prog_guard( + main_prog=main_prog, startup_prog=startup_prog + ): + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") avg_cost_l2 = model(data, label, self.word_len) @@ -231,7 +227,7 @@ def check_l2decay(self, place, model): for para in param_list: para_mul = fluid.layers.square(x=para) para_sum.append(fluid.layers.reduce_sum(input=para_mul)) - avg_cost_l2 += fluid.layers.sums(para_sum) * .5 + avg_cost_l2 += fluid.layers.sums(para_sum) * 0.5 optimizer = fluid.optimizer.Adagrad(learning_rate=0.1) optimizer.minimize(avg_cost_l2) @@ -252,9 +248,11 @@ def test_l2(self): assert len(dense_sparse_p_sum[0]) == len(dense_sparse_p_sum[1]) for i in range(len(dense_sparse_p_sum[0])): - assert np.isclose(a=dense_sparse_p_sum[0][i], - b=dense_sparse_p_sum[1][i], - rtol=5e-5) + assert np.isclose( + a=dense_sparse_p_sum[0][i], + b=dense_sparse_p_sum[1][i], + rtol=5e-5, + ) def test_repeated_regularization(self): l1 = fluid.regularizer.L1Decay(regularization_coeff=0.1) @@ -268,43 +266,45 @@ def test_repeated_regularization(self): sgd.minimize(loss) with fluid.dygraph.guard(): input = fluid.dygraph.to_variable( - np.random.randn(3, 2).astype('float32')) + np.random.randn(3, 2).astype('float32') + ) paddle.seed(1) paddle.framework.random._manual_program_seed(1) - linear1 = fluid.dygraph.Linear(2, - 2, - param_attr=fc_param_attr, - bias_attr=fc_param_attr) - linear2 = fluid.dygraph.Linear(2, - 2, - param_attr=fc_param_attr, - bias_attr=fc_param_attr) + linear1 = fluid.dygraph.Linear( + 2, 2, param_attr=fc_param_attr, bias_attr=fc_param_attr + ) + linear2 = fluid.dygraph.Linear( + 2, 2, param_attr=fc_param_attr, bias_attr=fc_param_attr + ) loss1 = linear1(input) loss1.backward() # set l2 regularizer in optimizer, but l1 in fluid.ParamAttr - fluid.optimizer.SGD(parameter_list=linear1.parameters(), - learning_rate=1e-2, - regularization=l2).minimize(loss1) + fluid.optimizer.SGD( + parameter_list=linear1.parameters(), + learning_rate=1e-2, + regularization=l2, + ).minimize(loss1) # only set l1 in fluid.ParamAttr loss2 = linear2(input) loss2.backward() - fluid.optimizer.SGD(parameter_list=linear2.parameters(), - learning_rate=1e-2).minimize(loss2) + fluid.optimizer.SGD( + parameter_list=linear2.parameters(), learning_rate=1e-2 + ).minimize(loss2) # they should both be applied by l1, and keep the same np.testing.assert_allclose( linear1.weight.numpy(), linear2.weight.numpy(), rtol=1e-05, - err_msg= - 'weight should use the regularization in fluid.ParamAttr!') + err_msg='weight should use the regularization in fluid.ParamAttr!', + ) np.testing.assert_allclose( linear1.bias.numpy(), linear2.bias.numpy(), rtol=1e-05, - err_msg='bias should use the regularization in fluid.ParamAttr!' + err_msg='bias should use the regularization in fluid.ParamAttr!', ) diff --git a/python/paddle/fluid/tests/unittests/test_regularizer_api.py b/python/paddle/fluid/tests/unittests/test_regularizer_api.py index b4200970dcd99..930de8429f493 100644 --- a/python/paddle/fluid/tests/unittests/test_regularizer_api.py +++ b/python/paddle/fluid/tests/unittests/test_regularizer_api.py @@ -22,22 +22,24 @@ import paddle.fluid as fluid -def bow_net(data, - label, - dict_dim, - is_sparse=False, - emb_dim=8, - hid_dim=8, - hid_dim2=6, - class_dim=2): +def bow_net( + data, + label, + dict_dim, + is_sparse=False, + emb_dim=8, + hid_dim=8, + hid_dim2=6, + class_dim=2, +): """ BOW net This model is from https://github.com/PaddlePaddle/models: fluid/PaddleNLP/text_classification/nets.py """ - emb = fluid.layers.embedding(input=data, - is_sparse=is_sparse, - size=[dict_dim, emb_dim]) + emb = fluid.layers.embedding( + input=data, is_sparse=is_sparse, size=[dict_dim, emb_dim] + ) bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') bow_tanh = fluid.layers.tanh(bow) fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh") @@ -50,11 +52,11 @@ def bow_net(data, class TestRegularizer(unittest.TestCase): - def setUp(self): self.word_len = 1500 - self.train_data = [[(random.sample(range(1000), 10), [0])] - for _ in range(2)] + self.train_data = [ + [(random.sample(range(1000), 10), [0])] for _ in range(2) + ] def get_places(self): places = [core.CPUPlace()] @@ -80,9 +82,9 @@ def run_program(self, place, feed_list): param_sum = [] for data in self.train_data: - out = exe.run(main_prog, - feed=feeder.feed(data), - fetch_list=param_list) + out = exe.run( + main_prog, feed=feeder.feed(data), fetch_list=param_list + ) p_sum = 0 for v in out: p_sum += np.sum(np.abs(v)) @@ -94,19 +96,20 @@ def check_l2decay_regularizer(self, place, model): paddle.framework.random._manual_program_seed(1) main_prog = fluid.framework.Program() startup_prog = fluid.framework.Program() - with self.scope_prog_guard(main_prog=main_prog, - startup_prog=startup_prog): - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + with self.scope_prog_guard( + main_prog=main_prog, startup_prog=startup_prog + ): + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") avg_cost = model(data, label, self.word_len) optimizer = fluid.optimizer.Adagrad( learning_rate=0.1, - regularization=paddle.regularizer.L2Decay(1.0)) + regularization=paddle.regularizer.L2Decay(1.0), + ) optimizer.minimize(avg_cost) param_sum = self.run_program(place, [data, label]) return param_sum @@ -117,12 +120,12 @@ def check_l2decay(self, place, model): main_prog = fluid.framework.Program() startup_prog = fluid.framework.Program() - with self.scope_prog_guard(main_prog=main_prog, - startup_prog=startup_prog): - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + with self.scope_prog_guard( + main_prog=main_prog, startup_prog=startup_prog + ): + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") avg_cost_l2 = model(data, label, self.word_len) @@ -132,7 +135,7 @@ def check_l2decay(self, place, model): for para in param_list: para_mul = fluid.layers.square(x=para) para_sum.append(fluid.layers.reduce_sum(input=para_mul)) - avg_cost_l2 += fluid.layers.sums(para_sum) * .5 + avg_cost_l2 += fluid.layers.sums(para_sum) * 0.5 optimizer = fluid.optimizer.Adagrad(learning_rate=0.1) optimizer.minimize(avg_cost_l2) @@ -154,9 +157,11 @@ def test_l2(self): assert len(dense_sparse_p_sum[0]) == len(dense_sparse_p_sum[1]) for i in range(len(dense_sparse_p_sum[0])): - assert np.isclose(a=dense_sparse_p_sum[0][i], - b=dense_sparse_p_sum[1][i], - rtol=5e-5) + assert np.isclose( + a=dense_sparse_p_sum[0][i], + b=dense_sparse_p_sum[1][i], + rtol=5e-5, + ) def test_repeated_regularization(self): paddle.enable_static() @@ -171,43 +176,45 @@ def test_repeated_regularization(self): sgd.minimize(loss) with fluid.dygraph.guard(): input = fluid.dygraph.to_variable( - np.random.randn(3, 2).astype('float32')) + np.random.randn(3, 2).astype('float32') + ) paddle.seed(1) paddle.framework.random._manual_program_seed(1) - linear1 = fluid.dygraph.Linear(2, - 2, - param_attr=fc_param_attr, - bias_attr=fc_param_attr) - linear2 = fluid.dygraph.Linear(2, - 2, - param_attr=fc_param_attr, - bias_attr=fc_param_attr) + linear1 = fluid.dygraph.Linear( + 2, 2, param_attr=fc_param_attr, bias_attr=fc_param_attr + ) + linear2 = fluid.dygraph.Linear( + 2, 2, param_attr=fc_param_attr, bias_attr=fc_param_attr + ) loss1 = linear1(input) loss1.backward() # set l2 regularizer in optimizer, but l1 in fluid.ParamAttr - fluid.optimizer.SGD(parameter_list=linear1.parameters(), - learning_rate=1e-2, - regularization=l2).minimize(loss1) + fluid.optimizer.SGD( + parameter_list=linear1.parameters(), + learning_rate=1e-2, + regularization=l2, + ).minimize(loss1) # only set l1 in fluid.ParamAttr loss2 = linear2(input) loss2.backward() - fluid.optimizer.SGD(parameter_list=linear2.parameters(), - learning_rate=1e-2).minimize(loss2) + fluid.optimizer.SGD( + parameter_list=linear2.parameters(), learning_rate=1e-2 + ).minimize(loss2) # they should both be applied by l1, and keep the same np.testing.assert_allclose( linear1.weight.numpy(), linear2.weight.numpy(), rtol=1e-05, - err_msg= - 'weight should use the regularization in fluid.ParamAttr!') + err_msg='weight should use the regularization in fluid.ParamAttr!', + ) np.testing.assert_allclose( linear1.bias.numpy(), linear2.bias.numpy(), rtol=1e-05, - err_msg='bias should use the regularization in fluid.ParamAttr!' + err_msg='bias should use the regularization in fluid.ParamAttr!', ) diff --git a/python/paddle/fluid/tests/unittests/test_renorm_op.py b/python/paddle/fluid/tests/unittests/test_renorm_op.py index f2754eb16bd93..38362f039eaa2 100644 --- a/python/paddle/fluid/tests/unittests/test_renorm_op.py +++ b/python/paddle/fluid/tests/unittests/test_renorm_op.py @@ -22,10 +22,10 @@ class TestRenormAPI(unittest.TestCase): - def input_data(self): - self.data_x = np.array([[[2.0, 2, -2], [3, 0.3, 3]], - [[2, -8, 2], [3.1, 3.7, 3]]]) + self.data_x = np.array( + [[[2.0, 2, -2], [3, 0.3, 3]], [[2, -8, 2], [3.1, 3.7, 3]]] + ) self.p = 1.0 self.dim = 2 self.max_norm = 2.05 @@ -36,17 +36,25 @@ def test_renorm_api(self): # case 1: with program_guard(Program(), Program()): - #x = fluid.layers.data(name = 'x',shape=[-1, 2, 3]) + # x = fluid.layers.data(name = 'x',shape=[-1, 2, 3]) x = paddle.static.data(name="x", shape=[-1, 2, 3], dtype='float64') z = paddle.renorm(x, self.p, self.dim, self.max_norm) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={"x": self.data_x}, - fetch_list=[z], - return_numpy=False) - expected = np.array([[[0.40594056, 0.29285714, -0.41000000], - [0.60891086, 0.04392857, 0.61500001]], - [[0.40594056, -1.17142856, 0.41000000], - [0.62920785, 0.54178572, 0.61500001]]]) + (res,) = exe.run( + feed={"x": self.data_x}, fetch_list=[z], return_numpy=False + ) + expected = np.array( + [ + [ + [0.40594056, 0.29285714, -0.41000000], + [0.60891086, 0.04392857, 0.61500001], + ], + [ + [0.40594056, -1.17142856, 0.41000000], + [0.62920785, 0.54178572, 0.61500001], + ], + ] + ) np.testing.assert_allclose(expected, np.array(res), rtol=1e-05) def test_dygraph_api(self): @@ -56,20 +64,30 @@ def test_dygraph_api(self): input = [[[2.0, 2, -2], [3, 0.3, 3]], [[2, -8, 2], [3.1, 3.7, 3]]] x = paddle.to_tensor(input, stop_gradient=False) y = paddle.renorm(x, 1.0, 2, 2.05) - expected = np.array([[[0.40594056, 0.29285714, -0.41000000], - [0.60891086, 0.04392857, 0.61500001]], - [[0.40594056, -1.17142856, 0.41000000], - [0.62920785, 0.54178572, 0.61500001]]]) + expected = np.array( + [ + [ + [0.40594056, 0.29285714, -0.41000000], + [0.60891086, 0.04392857, 0.61500001], + ], + [ + [0.40594056, -1.17142856, 0.41000000], + [0.62920785, 0.54178572, 0.61500001], + ], + ] + ) np.testing.assert_allclose(expected, np.array(y), rtol=1e-05) z = paddle.mean(y) z.backward(retain_graph=True) - expected_grad = np.array([[[0, 0.01394558, 0.02733333], - [0, 0.01394558, 0.00683333]], - [[0, 0.01045918, 0.00683333], - [0, 0.01394558, 0.00683333]]]) - np.testing.assert_allclose(expected_grad, - np.array(x.grad), - rtol=1e-05) + expected_grad = np.array( + [ + [[0, 0.01394558, 0.02733333], [0, 0.01394558, 0.00683333]], + [[0, 0.01045918, 0.00683333], [0, 0.01394558, 0.00683333]], + ] + ) + np.testing.assert_allclose( + expected_grad, np.array(x.grad), rtol=1e-05 + ) # #test exception: with fluid.dygraph.guard(): input = [[[2.0, 2, -2], [3, 0.3, 3]], [[2, -8, 2], [3.1, 3.7, 3]]] @@ -87,10 +105,18 @@ def test_dygraph_api(self): exp = True self.assertTrue(exp) y = paddle.renorm(x, 1.0, -1, 2.05) - expected = np.array([[[0.40594056, 0.29285714, -0.41000000], - [0.60891086, 0.04392857, 0.61500001]], - [[0.40594056, -1.17142856, 0.41000000], - [0.62920785, 0.54178572, 0.61500001]]]) + expected = np.array( + [ + [ + [0.40594056, 0.29285714, -0.41000000], + [0.60891086, 0.04392857, 0.61500001], + ], + [ + [0.40594056, -1.17142856, 0.41000000], + [0.62920785, 0.54178572, 0.61500001], + ], + ] + ) np.testing.assert_allclose(expected, np.array(y), rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py b/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py index 1b7538f1bcc70..dbf14c81948f9 100644 --- a/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py @@ -40,14 +40,17 @@ def setUpClass(cls): @classmethod def set_program(cls): - dat = fluid.layers.data(name=cls.data_desc[0][0], - shape=cls.data_desc[0][1]) + dat = fluid.layers.data( + name=cls.data_desc[0][0], shape=cls.data_desc[0][1] + ) dat.stop_gradient = False - rank_dat = fluid.layers.data(name=cls.data_desc[1][0], - shape=cls.data_desc[1][1]) + rank_dat = fluid.layers.data( + name=cls.data_desc[1][0], shape=cls.data_desc[1][1] + ) table = lod_rank_table(rank_dat) - new_dat = fluid.layers.reorder_lod_tensor_by_rank(x=dat, - rank_table=table) + new_dat = fluid.layers.reorder_lod_tensor_by_rank( + x=dat, rank_table=table + ) loss = fluid.layers.reduce_sum(new_dat) fluid.backward.append_backward(loss=loss) cls.fetch_list = [new_dat, cls.data_desc[0][0] + '@GRAD'] @@ -61,10 +64,12 @@ def run_program(self): for place in places: self.set_inputs(place) exe = fluid.Executor(place) - output, input_grad = exe.run(fluid.default_main_program(), - feed=self.inputs, - fetch_list=self.fetch_list, - return_numpy=False) + output, input_grad = exe.run( + fluid.default_main_program(), + feed=self.inputs, + fetch_list=self.fetch_list, + return_numpy=False, + ) outputs.append(output) input_grads.append(input_grad) self.actual_outputs = outputs @@ -81,11 +86,13 @@ def set_data(self): lod_level_i = np.random.randint( low=1, high=5, - size=self.num_seq if i == 0 else sum(lod_level_i)).tolist() + size=self.num_seq if i == 0 else sum(lod_level_i), + ).tolist() data_lod.append(lod_level_i) data_value = np.random.random( - size=[sum(data_lod[-1]) if data_lod else self.num_seq] + - data_shape).astype('float32') + size=[sum(data_lod[-1]) if data_lod else self.num_seq] + + data_shape + ).astype('float32') self.data[data_name] = (data_value, data_lod) def set_inputs(self, place): @@ -104,8 +111,9 @@ def reorder(self): rank_table = [] # list of (index, length) for i in range(len(ref_lod)): rank_table.append((i, ref_lod[i])) - rank_table = sorted(rank_table, - key=functools.cmp_to_key(lambda x, y: y[1] - x[1])) + rank_table = sorted( + rank_table, key=functools.cmp_to_key(lambda x, y: y[1] - x[1]) + ) # compute the input sequence info according to input_lod input_value, input_lod = self.data[self.data_desc[0][0]] @@ -120,8 +128,9 @@ def reorder(self): for lod_level_i in offset_lod[level:]: sub_lod_i = [] for idx in range(start_idx, end_idx): - sub_lod_i.append(lod_level_i[idx + 1] - - lod_level_i[idx]) + sub_lod_i.append( + lod_level_i[idx + 1] - lod_level_i[idx] + ) sub_lod.append(sub_lod_i) start_idx = lod_level_i[start_idx] end_idx = lod_level_i[end_idx] @@ -137,8 +146,9 @@ def reorder(self): input_seq_start = input_table[index][0] input_seq_len = input_table[index][1] input_seq_end = input_seq_start + input_seq_len - output_value[offset:offset + input_seq_len] = input_value[ - input_seq_start:input_seq_end] + output_value[offset : offset + input_seq_len] = input_value[ + input_seq_start:input_seq_end + ] offset += input_seq_len input_seq_sub_lod = input_table[index][2] @@ -155,22 +165,22 @@ def test_reorder_lod_tensor(self): # check output expect_output, expect_output_lod = self.reorder() for actual_output in self.actual_outputs: - np.testing.assert_allclose(np.array(actual_output), - expect_output, - rtol=1e-05, - atol=0.001) - self.assertEqual(expect_output_lod, - actual_output.recursive_sequence_lengths()) + np.testing.assert_allclose( + np.array(actual_output), expect_output, rtol=1e-05, atol=0.001 + ) + self.assertEqual( + expect_output_lod, actual_output.recursive_sequence_lengths() + ) # check gradient expect_grad = np.ones_like(self.data[self.data_desc[0][0]][0]) expect_grad_lod = self.data[self.data_desc[0][0]][1] for actual_grad in self.actual_grads: - np.testing.assert_allclose(np.array(actual_grad), - expect_grad, - rtol=1e-05, - atol=0.001) - self.assertEqual(expect_grad_lod, - actual_grad.recursive_sequence_lengths()) + np.testing.assert_allclose( + np.array(actual_grad), expect_grad, rtol=1e-05, atol=0.001 + ) + self.assertEqual( + expect_grad_lod, actual_grad.recursive_sequence_lengths() + ) def test_reorder_tensor(self): self.data_desc[0][-1] = 0 # input is tensor @@ -179,43 +189,43 @@ def test_reorder_tensor(self): # check output expect_output, expect_output_lod = self.reorder() for actual_output in self.actual_outputs: - np.testing.assert_allclose(np.array(actual_output), - expect_output, - rtol=1e-05, - atol=0.001) - self.assertEqual(expect_output_lod, - actual_output.recursive_sequence_lengths()) + np.testing.assert_allclose( + np.array(actual_output), expect_output, rtol=1e-05, atol=0.001 + ) + self.assertEqual( + expect_output_lod, actual_output.recursive_sequence_lengths() + ) # check gradient expect_grad = np.ones_like(self.data[self.data_desc[0][0]][0]) expect_grad_lod = self.data[self.data_desc[0][0]][1] for actual_grad in self.actual_grads: - np.testing.assert_allclose(np.array(actual_grad), - expect_grad, - rtol=1e-05, - atol=0.001) - self.assertEqual(expect_grad_lod, - actual_grad.recursive_sequence_lengths()) + np.testing.assert_allclose( + np.array(actual_grad), expect_grad, rtol=1e-05, atol=0.001 + ) + self.assertEqual( + expect_grad_lod, actual_grad.recursive_sequence_lengths() + ) # compare outputs between LodTensors with explicit and implicit lod # use the same data but set the input lod explicitly input_lod = [[1] * len(self.data[self.data_desc[0][0]][0])] self.inputs[self.data_desc[0][0]].set_recursive_sequence_lengths( - input_lod) + input_lod + ) # preserve the output of LodTensor with implicit lod to compare expect_outputs = [ np.array(actual_output) for actual_output in self.actual_outputs ] self.run_program() - for actual_output, expect_output in zip(self.actual_outputs, - expect_outputs): - np.testing.assert_allclose(np.array(actual_output), - expect_output, - rtol=1e-05, - atol=0.001) + for actual_output, expect_output in zip( + self.actual_outputs, expect_outputs + ): + np.testing.assert_allclose( + np.array(actual_output), expect_output, rtol=1e-05, atol=0.001 + ) class TestReorderLoDTensorError(unittest.TestCase): - def test_errors(self): with program_guard(Program()): @@ -224,17 +234,19 @@ def test_Variable(): x1 = np.array([0.9383, 0.1983, 3.2, 1.2]).astype("float64") table1 = np.array([0.9383, 0.1983, 3.2, 1.2]).astype("float64") new_dat = fluid.layers.reorder_lod_tensor_by_rank( - x=x1, rank_table=table1) + x=x1, rank_table=table1 + ) self.assertRaises(TypeError, test_Variable) def test_type(): x2 = fluid.layers.data(name='x1', shape=[4], dtype='float32') - table2 = fluid.layers.data(name='table2', - shape=[4], - dtype='int32') + table2 = fluid.layers.data( + name='table2', shape=[4], dtype='int32' + ) new_dat2 = fluid.layers.reorder_lod_tensor_by_rank( - x=x2, rank_table=table2) + x=x2, rank_table=table2 + ) self.assertRaises(TypeError, test_type) diff --git a/python/paddle/fluid/tests/unittests/test_repeat_interleave_op.py b/python/paddle/fluid/tests/unittests/test_repeat_interleave_op.py index fcc8647fbcaa4..7332c36e1d284 100644 --- a/python/paddle/fluid/tests/unittests/test_repeat_interleave_op.py +++ b/python/paddle/fluid/tests/unittests/test_repeat_interleave_op.py @@ -21,20 +21,20 @@ class TestRepeatInterleaveOp(OpTest): - def setUp(self): self.op_type = "repeat_interleave" self.python_api = paddle.repeat_interleave self.init_dtype_type() index_np = np.random.randint( - low=0, high=3, size=self.index_size).astype(self.index_type) + low=0, high=3, size=self.index_size + ).astype(self.index_type) x_np = np.random.random(self.x_shape).astype(self.x_type) self.inputs = {'X': x_np, 'RepeatsTensor': index_np} self.attrs = {'dim': self.dim} - outer_loop = np.prod(self.x_shape[:self.dim]) - x_reshape = [outer_loop] + list(self.x_shape[self.dim:]) + outer_loop = np.prod(self.x_shape[: self.dim]) + x_reshape = [outer_loop] + list(self.x_shape[self.dim :]) x_np_reshape = np.reshape(x_np, tuple(x_reshape)) out_list = [] for i in range(outer_loop): @@ -63,18 +63,17 @@ def test_check_grad_normal(self): class TestRepeatInterleaveOp2(OpTest): - def setUp(self): self.op_type = "repeat_interleave" self.python_api = paddle.repeat_interleave self.init_dtype_type() index_np = 2 x_np = np.random.random(self.x_shape).astype(self.x_type) - self.inputs = {'X': x_np} #, 'RepeatsTensor': None} + self.inputs = {'X': x_np} # , 'RepeatsTensor': None} self.attrs = {'dim': self.dim, 'Repeats': index_np} - outer_loop = np.prod(self.x_shape[:self.dim]) - x_reshape = [outer_loop] + list(self.x_shape[self.dim:]) + outer_loop = np.prod(self.x_shape[: self.dim]) + x_reshape = [outer_loop] + list(self.x_shape[self.dim :]) x_np_reshape = np.reshape(x_np, tuple(x_reshape)) out_list = [] for i in range(outer_loop): @@ -102,10 +101,14 @@ def test_check_grad_normal(self): class TestIndexSelectAPI(unittest.TestCase): - def input_data(self): - self.data_x = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [9.0, 10.0, 11.0, 12.0]]) + self.data_x = np.array( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ] + ) self.data_index = np.array([0, 1, 2, 1]).astype('int32') def test_repeat_interleave_api(self): @@ -115,18 +118,19 @@ def test_repeat_interleave_api(self): # case 1: with program_guard(Program(), Program()): x = fluid.layers.data(name='x', shape=[-1, 4]) - index = fluid.layers.data(name='repeats_', - shape=[4], - dtype='int32', - append_batch_size=False) + index = fluid.layers.data( + name='repeats_', + shape=[4], + dtype='int32', + append_batch_size=False, + ) z = paddle.repeat_interleave(x, index, axis=1) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={ - 'x': self.data_x, - 'repeats_': self.data_index - }, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': self.data_x, 'repeats_': self.data_index}, + fetch_list=[z.name], + return_numpy=False, + ) expect_out = np.repeat(self.data_x, self.data_index, axis=1) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) @@ -134,18 +138,22 @@ def test_repeat_interleave_api(self): repeats = np.array([1, 2, 1]).astype('int32') with program_guard(Program(), Program()): x = fluid.layers.data(name='x', shape=[-1, 4]) - index = fluid.layers.data(name='repeats_', - shape=[3], - dtype='int32', - append_batch_size=False) + index = fluid.layers.data( + name='repeats_', + shape=[3], + dtype='int32', + append_batch_size=False, + ) z = paddle.repeat_interleave(x, index, axis=0) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={ - 'x': self.data_x, - 'repeats_': repeats, - }, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={ + 'x': self.data_x, + 'repeats_': repeats, + }, + fetch_list=[z.name], + return_numpy=False, + ) expect_out = np.repeat(self.data_x, repeats, axis=0) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) @@ -154,9 +162,9 @@ def test_repeat_interleave_api(self): x = fluid.layers.data(name='x', shape=[-1, 4]) z = paddle.repeat_interleave(x, repeats, axis=0) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': self.data_x}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': self.data_x}, fetch_list=[z.name], return_numpy=False + ) expect_out = np.repeat(self.data_x, repeats, axis=0) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_require_version.py b/python/paddle/fluid/tests/unittests/test_require_version.py index b5d9a1bfa5423..92066e95392c0 100644 --- a/python/paddle/fluid/tests/unittests/test_require_version.py +++ b/python/paddle/fluid/tests/unittests/test_require_version.py @@ -20,21 +20,30 @@ class VersionTest(unittest.TestCase): - def test_check_output(self): warnings.warn( "paddle.__version__: %s, fluid_version.full_version: %s, fluid_version.major: %s, fluid_version.minor: %s, fluid_version.patch: %s, fluid_version.rc: %s." - % (paddle.__version__, fluid_version.full_version, - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc)) + % ( + paddle.__version__, + fluid_version.full_version, + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, + ) + ) ori_full_version = fluid_version.full_version ori_sep_version = [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] = ['1', '4', '1', '0'] fluid.require_version('1') @@ -52,21 +61,24 @@ def test_check_output(self): # if installed version is 0.0.0.0, throw warning and skip the checking. [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] = ['0', '0', '0', '0'] fluid.require_version('0.0.0') fluid_version.full_version = ori_full_version [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] = ori_sep_version # Test Errors class TestErrors(unittest.TestCase): - def test_errors(self): # The type of params must be str. def test_input_type(): @@ -118,12 +130,16 @@ def test_version_2(): ori_full_version = fluid_version.full_version ori_sep_version = [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] = ['1', '4', '1', '0'] self.assertRaises(Exception, test_version) @@ -132,8 +148,10 @@ def test_version_2(): fluid_version.full_version = ori_full_version [ - fluid_version.major, fluid_version.minor, fluid_version.patch, - fluid_version.rc + fluid_version.major, + fluid_version.minor, + fluid_version.patch, + fluid_version.rc, ] = ori_sep_version diff --git a/python/paddle/fluid/tests/unittests/test_reset_grad_inplace_version.py b/python/paddle/fluid/tests/unittests/test_reset_grad_inplace_version.py index c28724b97ae6e..a13a0d2c9e880 100644 --- a/python/paddle/fluid/tests/unittests/test_reset_grad_inplace_version.py +++ b/python/paddle/fluid/tests/unittests/test_reset_grad_inplace_version.py @@ -22,7 +22,6 @@ # Test 1 def clear_grad_test_0(w, a): - @paddle.no_grad() def warp(*_): assert w.grad is not None @@ -33,7 +32,6 @@ def warp(*_): class TestInplaceAndClearGradient(unittest.TestCase): - def func_test(self): input_data = np.ones([1, 1]) w = paddle.to_tensor(input_data, 'float32', stop_gradient=False) @@ -43,8 +41,9 @@ def func_test(self): for i in range(2): print(" Step: ", i) out0 = _legacy_C_ops.scale(w, 'scale', 0.1) - out = _legacy_C_ops.matmul_v2(out0, w, 'trans_x', False, 'trans_y', - False) + out = _legacy_C_ops.matmul_v2( + out0, w, 'trans_x', False, 'trans_y', False + ) out.backward() assert w.grad[0] == 0.15 @@ -56,14 +55,12 @@ def test(self): # Test 2 class Counter: - def __init__(self): self.num_calls = 0 self.step = 0 def clear_grad_test_1(w, c): - @paddle.no_grad() def warp(*_): assert w.grad is not None @@ -77,7 +74,6 @@ def warp(*_): class TestInplaceClearGradAccumulation(unittest.TestCase): - def func_test(self): input_data = np.ones([1, 1]) w = paddle.to_tensor(input_data, 'float32', stop_gradient=False) @@ -88,8 +84,9 @@ def func_test(self): for c.step in range(5): out0 = _legacy_C_ops.scale(w, 'scale', 0.1) - out = _legacy_C_ops.matmul_v2(out0, w, 'trans_x', False, 'trans_y', - False) + out = _legacy_C_ops.matmul_v2( + out0, w, 'trans_x', False, 'trans_y', False + ) out.backward() @@ -106,7 +103,6 @@ def test(self): class TestInplaceClearGradAccumulationAlt(unittest.TestCase): - def func_test(self): input_data = np.ones([1, 1]) w = paddle.to_tensor(input_data, 'float32', stop_gradient=False) diff --git a/python/paddle/fluid/tests/unittests/test_reshape_op.py b/python/paddle/fluid/tests/unittests/test_reshape_op.py index b8cb8bfa88ac6..dad6e3fa3284a 100755 --- a/python/paddle/fluid/tests/unittests/test_reshape_op.py +++ b/python/paddle/fluid/tests/unittests/test_reshape_op.py @@ -23,7 +23,6 @@ # situation 1: have shape( list, no tensor), no actual shape(Tensor) class TestReshapeOp(OpTest): - def setUp(self): self.init_data() self.op_type = "reshape2" @@ -31,7 +30,7 @@ def setUp(self): self.attrs = {"shape": self.new_shape} self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def init_data(self): @@ -47,31 +46,27 @@ def test_check_grad(self): class TestReshapeOp_ZeroDim1(OpTest): - def init_data(self): self.ori_shape = () - self.new_shape = (1) - self.infered_shape = (1) + self.new_shape = 1 + self.infered_shape = 1 class TestReshapeOp_ZeroDim2(OpTest): - def init_data(self): self.ori_shape = () - self.new_shape = (-1) - self.infered_shape = (1) + self.new_shape = -1 + self.infered_shape = 1 class TestReshapeOp_ZeroDim3(OpTest): - def init_data(self): - self.ori_shape = (1) + self.ori_shape = 1 self.new_shape = () self.infered_shape = () class TestReshapeBF16Op(OpTest): - def setUp(self): self.init_data() self.op_type = "reshape2" @@ -81,11 +76,10 @@ def setUp(self): self.inputs = {"X": convert_float_to_uint16(x)} self.attrs = {"shape": self.new_shape} self.outputs = { - "Out": - convert_float_to_uint16(out), - 'XShape': - convert_float_to_uint16( - np.random.random(self.ori_shape).astype("float32")) + "Out": convert_float_to_uint16(out), + 'XShape': convert_float_to_uint16( + np.random.random(self.ori_shape).astype("float32") + ), } def init_data(self): @@ -101,7 +95,6 @@ def test_check_grad(self): class TestReshapeOpDimInfer1(TestReshapeOp): - def init_data(self): self.ori_shape = (5, 25) self.new_shape = (5, -1, 5) @@ -109,7 +102,6 @@ def init_data(self): class TestReshapeOpDimInfer2(TestReshapeOp): - def init_data(self): self.ori_shape = (10, 2, 6) self.new_shape = (10, 0, 3, -1) @@ -118,19 +110,18 @@ def init_data(self): # situation 2: have shape(list, no tensor), have actual shape(Tensor) class TestReshapeOpWithInputShape(OpTest): - def setUp(self): self.init_data() self.op_type = "reshape2" self.inputs = { "X": np.random.random(self.ori_shape).astype("float32"), - "Shape": np.array(self.actual_shape, dtype="int32") + "Shape": np.array(self.actual_shape, dtype="int32"), } self.attrs = {"shape": self.new_shape} self.outputs = { "Out": self.inputs["X"].reshape(self.actual_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def init_data(self): @@ -147,24 +138,24 @@ def test_check_grad(self): # Situation 3: have shape(list, have tensor), no actual shape(Tensor) class TestReshapeOp_attr_ShapeTensor(OpTest): - def setUp(self): self.init_data() self.op_type = "reshape2" shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { "X": np.random.random(self.ori_shape).astype("float32"), - 'ShapeTensor': shape_tensor + 'ShapeTensor': shape_tensor, } self.attrs = {'shape': self.shape} self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def init_data(self): @@ -181,7 +172,6 @@ def test_check_grad(self): class TestReshapeOpDimInfer1_attr_ShapeTensor(TestReshapeOp_attr_ShapeTensor): - def init_data(self): self.ori_shape = (5, 20) self.new_shape = (5, -1, 20) @@ -190,7 +180,6 @@ def init_data(self): class TestReshapeOpDimInfer2_attr_ShapeTensor(TestReshapeOp_attr_ShapeTensor): - def init_data(self): self.ori_shape = (10, 2, 6) self.new_shape = (10, 0, 3, -1) @@ -200,19 +189,18 @@ def init_data(self): # Situation 4: have shape(Tensor), no actual shape(Tensor) class TestReshapeOp_attr_OnlyShape(OpTest): - def setUp(self): self.init_data() self.op_type = "reshape2" self.inputs = { "X": np.random.random(self.ori_shape).astype("float32"), - "Shape": np.array(self.new_shape, dtype="int32") + "Shape": np.array(self.new_shape, dtype="int32"), } self.attrs = {} self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def init_data(self): @@ -228,7 +216,6 @@ def test_check_grad(self): class TestReshapeOpDimInfer1_attr_OnlyShape(TestReshapeOp_attr_OnlyShape): - def init_data(self): self.ori_shape = (5, 20) self.new_shape = (5, -1, 10) @@ -237,7 +224,6 @@ def init_data(self): class TestReshapeOpDimInfer2_attr_OnlyShape(TestReshapeOp_attr_OnlyShape): - def init_data(self): self.ori_shape = (10, 2, 6) self.new_shape = (10, 0, 3, -1) @@ -247,7 +233,6 @@ def init_data(self): # test int8 data type on CPU class TestReshapeInt8Op(OpTest): - def setUp(self): self.init_dtype() self.init_data() @@ -262,7 +247,7 @@ def setUp(self): } self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype(np.float32) + 'XShape': np.random.random(self.ori_shape).astype(np.float32), } def init_dtype(self): @@ -274,9 +259,9 @@ def init_data(self): self.infered_shape = (10, 2, 3, -1) def test_check_output(self): - self.check_output_with_place(fluid.core.CPUPlace(), - atol=1e-5, - no_check_set=['XShape']) + self.check_output_with_place( + fluid.core.CPUPlace(), atol=1e-5, no_check_set=['XShape'] + ) def test_check_grad(self): pass @@ -284,13 +269,11 @@ def test_check_grad(self): # test unt8 data type on CPU class TestReshapeUint8Op(TestReshapeInt8Op): - def init_dtype(self): self.dtype = np.uint8 class TestReshapeOpBool(TestReshapeOp): - def setUp(self): self.init_data() self.op_type = "reshape2" @@ -300,7 +283,7 @@ def setUp(self): self.attrs = {"shape": self.new_shape} self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype("float32") + 'XShape': np.random.random(self.ori_shape).astype("float32"), } def test_check_grad(self): @@ -309,7 +292,6 @@ def test_check_grad(self): # Test python API class TestReshapeAPI(unittest.TestCase): - def _set_paddle_api(self): self.fill_constant = paddle.fluid.layers.fill_constant self.data = paddle.static.data @@ -339,9 +321,9 @@ def _test_api(self): out_1 = self.reshape(x, shape) # situation 2: have shape(list, no tensor), have actual shape(Tensor) - out_2 = fluid.layers.reshape(x, - shape=shape, - actual_shape=actual_shape) + out_2 = fluid.layers.reshape( + x, shape=shape, actual_shape=actual_shape + ) # Situation 3: have shape(list, have tensor), no actual shape(Tensor) out_3 = self.reshape(x, shape=[positive_five, 10]) @@ -352,11 +334,9 @@ def _test_api(self): exe = paddle.static.Executor(place=paddle.CPUPlace()) res_1, res_2, res_3, res_4 = exe.run( main_prog, - feed={ - "x": input, - "shape": np.array([2, 5, 5]).astype("int32") - }, - fetch_list=[out_1, out_2, out_3, out_4]) + feed={"x": input, "shape": np.array([2, 5, 5]).astype("int32")}, + fetch_list=[out_1, out_2, out_3, out_4], + ) assert np.array_equal(res_1, input.reshape(shape)) assert np.array_equal(res_2, input.reshape(shape)) @@ -392,7 +372,6 @@ def test_imperative(self): class TestStaticReshape_(TestReshapeAPI): - def _executed_api(self): self.reshape = paddle.reshape_ @@ -418,7 +397,6 @@ def test_imperative(self): # Test Input Error class TestReshapeOpError(unittest.TestCase): - def _set_paddle_api(self): self.data = paddle.static.data self.reshape = paddle.reshape @@ -431,8 +409,9 @@ def _test_errors(self): with program_guard(Program(), Program()): # The x type of reshape_op must be Variable. def test_x_type(): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - paddle.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], paddle.CPUPlace() + ) self.reshape(x1, shape=[1]) self.assertRaises(TypeError, test_x_type) @@ -445,9 +424,9 @@ def test_x_dtype(): self.assertRaises(TypeError, test_x_dtype) def test_x_dtype_float16(): - x_float16 = self.data(name="x_float16", - shape=[2, 25], - dtype="float16") + x_float16 = self.data( + name="x_float16", shape=[2, 25], dtype="float16" + ) self.reshape(x_float16, shape=[2, 5, 5]) test_x_dtype_float16() @@ -494,7 +473,6 @@ def test_fluid_api_error(self): class TestDygraphReshapeAPI(unittest.TestCase): - def setUp(self): self.executed_api() @@ -530,13 +508,11 @@ def test_out_float32(self): class TestDygraphReshapeInplaceAPI(TestDygraphReshapeAPI): - def executed_api(self): self.reshape = paddle.reshape_ class TestReshapeZeroTensor(unittest.TestCase): - def test_reshape_zero_tensor_success(self): zero_tensor = paddle.zeros([0, 2, 3]) # since we use "0" as the dimension copy semantically in reshape, @@ -551,7 +527,6 @@ def test_reshape_zero_tensor_error(self): class TestReshapeAPI_ZeroDim(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -596,18 +571,18 @@ def test_static(self): # Test compile shape self.assertEqual(x.shape, ()) - self.assertEqual(out.shape, (1, )) + self.assertEqual(out.shape, (1,)) self.assertEqual(x_grad.shape, ()) - self.assertEqual(out_grad.shape, (1, )) + self.assertEqual(out_grad.shape, (1,)) exe = fluid.Executor() result = exe.run(main_prog, fetch_list=[x, out, x_grad, out_grad]) # Test runtime shape self.assertEqual(result[0].shape, ()) - self.assertEqual(result[1].shape, (1, )) + self.assertEqual(result[1].shape, (1,)) self.assertEqual(result[2].shape, ()) - self.assertEqual(result[3].shape, (1, )) + self.assertEqual(result[3].shape, (1,)) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_resnet50_with_cinn.py b/python/paddle/fluid/tests/unittests/test_resnet50_with_cinn.py index 2a16242fc60e2..754bb0dcb5918 100644 --- a/python/paddle/fluid/tests/unittests/test_resnet50_with_cinn.py +++ b/python/paddle/fluid/tests/unittests/test_resnet50_with_cinn.py @@ -19,8 +19,9 @@ paddle.enable_static() -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', - level=logging.INFO) +logging.basicConfig( + format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO +) logger = logging.getLogger(__name__) @@ -36,11 +37,14 @@ def set_cinn_flag(val): @unittest.skipIf(not set_cinn_flag(True), "Paddle is not compiled with CINN.") class TestResnet50Accuracy(unittest.TestCase): - def reader(self, limit): for _ in range(limit): - yield {'image': np.random.randint(0, 256, size=[32, 3, 224, 224]).astype('float32'), \ - 'label': np.random.randint(0, 1000, size=[32]).astype('int64')} + yield { + 'image': np.random.randint( + 0, 256, size=[32, 3, 224, 224] + ).astype('float32'), + 'label': np.random.randint(0, 1000, size=[32]).astype('int64'), + } def generate_random_data(self, loop_num=10): feed = [] @@ -51,9 +55,9 @@ def generate_random_data(self, loop_num=10): def build_program(self, main_program, startup_program): with paddle.static.program_guard(main_program, startup_program): - image = paddle.static.data(name='image', - shape=[32, 3, 224, 224], - dtype='float32') + image = paddle.static.data( + name='image', shape=[32, 3, 224, 224], dtype='float32' + ) label = paddle.static.data(name='label', shape=[32], dtype='int64') # TODO: stop_gradient slower training speed, need fix @@ -62,8 +66,9 @@ def build_program(self, main_program, startup_program): model = paddle.vision.models.resnet50() prediction = model(image) - loss = paddle.nn.functional.cross_entropy(input=prediction, - label=label) + loss = paddle.nn.functional.cross_entropy( + input=prediction, label=label + ) loss = paddle.mean(loss) adam = paddle.optimizer.Adam(learning_rate=0.001) adam.minimize(loss) @@ -83,23 +88,29 @@ def train(self, place, iters, feed, use_cinn=False, seed=1234): exe = paddle.static.Executor(place) compiled_prog = paddle.static.CompiledProgram( - main_program).with_data_parallel(loss_name=loss.name) + main_program + ).with_data_parallel(loss_name=loss.name) loss_vals = [] scope = paddle.static.Scope() with paddle.static.scope_guard(scope): exe.run(startup_program) for step in range(iters): - loss_v = exe.run(compiled_prog, - feed=feed[step], - fetch_list=[loss], - return_numpy=True) + loss_v = exe.run( + compiled_prog, + feed=feed[step], + fetch_list=[loss], + return_numpy=True, + ) loss_vals.append(loss_v[0][0]) return loss_vals def test_check_resnet50_accuracy(self): - place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) loop_num = 10 feed = self.generate_random_data(loop_num) diff --git a/python/paddle/fluid/tests/unittests/test_retain_graph.py b/python/paddle/fluid/tests/unittests/test_retain_graph.py index 71998f57e5f95..06f0be1b879ed 100644 --- a/python/paddle/fluid/tests/unittests/test_retain_graph.py +++ b/python/paddle/fluid/tests/unittests/test_retain_graph.py @@ -24,7 +24,6 @@ class Generator(fluid.dygraph.Layer): - def __init__(self): super(Generator, self).__init__() self.conv1 = paddle.nn.Conv2D(3, 3, 3, padding=1) @@ -36,7 +35,6 @@ def forward(self, x): class Discriminator(fluid.dygraph.Layer): - def __init__(self): super(Discriminator, self).__init__() self.convd = paddle.nn.Conv2D(6, 3, 1) @@ -47,15 +45,16 @@ def forward(self, x): class TestRetainGraph(unittest.TestCase): - - def cal_gradient_penalty(self, - netD, - real_data, - fake_data, - edge_data=None, - type='mixed', - constant=1.0, - lambda_gp=10.0): + def cal_gradient_penalty( + self, + netD, + real_data, + fake_data, + edge_data=None, + type='mixed', + constant=1.0, + lambda_gp=10.0, + ): if lambda_gp > 0.0: if type == 'real': interpolatesv = real_data @@ -63,10 +62,13 @@ def cal_gradient_penalty(self, interpolatesv = fake_data elif type == 'mixed': alpha = paddle.rand((real_data.shape[0], 1)) - alpha = paddle.expand(alpha, [ - real_data.shape[0], - np.prod(real_data.shape) // real_data.shape[0] - ]) + alpha = paddle.expand( + alpha, + [ + real_data.shape[0], + np.prod(real_data.shape) // real_data.shape[0], + ], + ) alpha = paddle.reshape(alpha, real_data.shape) interpolatesv = alpha * real_data + ((1 - alpha) * fake_data) else: @@ -76,21 +78,26 @@ def cal_gradient_penalty(self, fake_AB = paddle.concat((real_data.detach(), interpolatesv), 1) disc_interpolates = netD(fake_AB) - outs = paddle.fluid.layers.fill_constant(disc_interpolates.shape, - disc_interpolates.dtype, - 1.0) - gradients = paddle.grad(outputs=disc_interpolates, - inputs=fake_AB, - grad_outputs=outs, - create_graph=True, - retain_graph=True, - only_inputs=True) + outs = paddle.fluid.layers.fill_constant( + disc_interpolates.shape, disc_interpolates.dtype, 1.0 + ) + gradients = paddle.grad( + outputs=disc_interpolates, + inputs=fake_AB, + grad_outputs=outs, + create_graph=True, + retain_graph=True, + only_inputs=True, + ) gradients = paddle.reshape(gradients[0], [real_data.shape[0], -1]) - gradient_penalty = paddle.mean( - (paddle.norm(gradients + 1e-16, 2, 1) - constant)** - 2) * lambda_gp # added eps + gradient_penalty = ( + paddle.mean( + (paddle.norm(gradients + 1e-16, 2, 1) - constant) ** 2 + ) + * lambda_gp + ) # added eps return gradient_penalty, gradients else: return 0.0, None @@ -117,12 +124,12 @@ def run_retain(self, need_retain): G_pred_fake = d(fake_AB.detach()) false_target = paddle.fluid.layers.fill_constant( - G_pred_fake.shape, 'float32', 0.0) + G_pred_fake.shape, 'float32', 0.0 + ) - G_gradient_penalty, _ = self.cal_gradient_penalty(d, - realA, - fakeB, - lambda_gp=10.0) + G_gradient_penalty, _ = self.cal_gradient_penalty( + d, realA, fakeB, lambda_gp=10.0 + ) loss_d = gan_criterion(G_pred_fake, false_target) + G_gradient_penalty loss_d.backward(retain_graph=need_retain) @@ -131,10 +138,12 @@ def run_retain(self, need_retain): optim_g.clear_gradients() fake_AB = paddle.concat((realA, fakeB), 1) G_pred_fake = d(fake_AB) - true_target = paddle.fluid.layers.fill_constant(G_pred_fake.shape, - 'float32', 1.0) + true_target = paddle.fluid.layers.fill_constant( + G_pred_fake.shape, 'float32', 1.0 + ) loss_g = l1_criterion(fakeB, realB) + gan_criterion( - G_pred_fake, true_target) + G_pred_fake, true_target + ) loss_g.backward() optim_g.minimize(loss_g) diff --git a/python/paddle/fluid/tests/unittests/test_retinanet_detection_output.py b/python/paddle/fluid/tests/unittests/test_retinanet_detection_output.py index 61bcb87521780..5d8527477af7a 100644 --- a/python/paddle/fluid/tests/unittests/test_retinanet_detection_output.py +++ b/python/paddle/fluid/tests/unittests/test_retinanet_detection_output.py @@ -1,16 +1,16 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # -#Licensed under the Apache License, Version 2.0 (the "License") -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import numpy as np @@ -42,9 +42,9 @@ def multiclass_nms(prediction, class_num, keep_top_k, nms_threshold): for idx in indices: score_index.append((prediction[c][idx][4], c, idx)) - sorted_score_index = sorted(score_index, - key=lambda tup: tup[0], - reverse=True) + sorted_score_index = sorted( + score_index, key=lambda tup: tup[0], reverse=True + ) if keep_top_k > -1 and num_det > keep_top_k: sorted_score_index = sorted_score_index[:keep_top_k] num_det = keep_top_k @@ -59,9 +59,16 @@ def multiclass_nms(prediction, class_num, keep_top_k, nms_threshold): return nmsed_outs, num_det -def retinanet_detection_out(boxes_list, scores_list, anchors_list, im_info, - score_threshold, nms_threshold, nms_top_k, - keep_top_k): +def retinanet_detection_out( + boxes_list, + scores_list, + anchors_list, + im_info, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, +): class_num = scores_list[0].shape[-1] im_height, im_width, im_scale = im_info @@ -88,23 +95,37 @@ def retinanet_detection_out(boxes_list, scores_list, anchors_list, im_info, a = int(idx / class_num) c = int(idx % class_num) box_offset = a * 4 - anchor_box_width = anchors_per_level[ - box_offset + 2] - anchors_per_level[box_offset] + 1 - anchor_box_height = anchors_per_level[ - box_offset + 3] - anchors_per_level[box_offset + 1] + 1 - anchor_box_center_x = anchors_per_level[ - box_offset] + anchor_box_width / 2 - anchor_box_center_y = anchors_per_level[box_offset + - 1] + anchor_box_height / 2 - - target_box_center_x = bboxes_per_level[ - box_offset] * anchor_box_width + anchor_box_center_x - target_box_center_y = bboxes_per_level[ - box_offset + 1] * anchor_box_height + anchor_box_center_y - target_box_width = math.exp( - bboxes_per_level[box_offset + 2]) * anchor_box_width - target_box_height = math.exp( - bboxes_per_level[box_offset + 3]) * anchor_box_height + anchor_box_width = ( + anchors_per_level[box_offset + 2] + - anchors_per_level[box_offset] + + 1 + ) + anchor_box_height = ( + anchors_per_level[box_offset + 3] + - anchors_per_level[box_offset + 1] + + 1 + ) + anchor_box_center_x = ( + anchors_per_level[box_offset] + anchor_box_width / 2 + ) + anchor_box_center_y = ( + anchors_per_level[box_offset + 1] + anchor_box_height / 2 + ) + + target_box_center_x = ( + bboxes_per_level[box_offset] * anchor_box_width + + anchor_box_center_x + ) + target_box_center_y = ( + bboxes_per_level[box_offset + 1] * anchor_box_height + + anchor_box_center_y + ) + target_box_width = ( + math.exp(bboxes_per_level[box_offset + 2]) * anchor_box_width + ) + target_box_height = ( + math.exp(bboxes_per_level[box_offset + 3]) * anchor_box_height + ) pred_box_xmin = target_box_center_x - target_box_width / 2 pred_box_ymin = target_box_center_y - target_box_height / 2 @@ -117,33 +138,46 @@ def retinanet_detection_out(boxes_list, scores_list, anchors_list, im_info, pred_box_ymax = pred_box_ymax / im_scale pred_box_xmin = max( - min(pred_box_xmin, - np.round(im_width / im_scale) - 1), 0.) + min(pred_box_xmin, np.round(im_width / im_scale) - 1), 0.0 + ) pred_box_ymin = max( - min(pred_box_ymin, - np.round(im_height / im_scale) - 1), 0.) + min(pred_box_ymin, np.round(im_height / im_scale) - 1), 0.0 + ) pred_box_xmax = max( - min(pred_box_xmax, - np.round(im_width / im_scale) - 1), 0.) + min(pred_box_xmax, np.round(im_width / im_scale) - 1), 0.0 + ) pred_box_ymax = max( - min(pred_box_ymax, - np.round(im_height / im_scale) - 1), 0.) + min(pred_box_ymax, np.round(im_height / im_scale) - 1), 0.0 + ) if c not in prediction.keys(): prediction[c] = [] - prediction[c].append([ - pred_box_xmin, pred_box_ymin, pred_box_xmax, pred_box_ymax, - scores_per_level[idx] - ]) - - nmsed_outs, nmsed_num = multiclass_nms(prediction, class_num, keep_top_k, - nms_threshold) + prediction[c].append( + [ + pred_box_xmin, + pred_box_ymin, + pred_box_xmax, + pred_box_ymax, + scores_per_level[idx], + ] + ) + + nmsed_outs, nmsed_num = multiclass_nms( + prediction, class_num, keep_top_k, nms_threshold + ) return nmsed_outs, nmsed_num -def batched_retinanet_detection_out(boxes, scores, anchors, im_info, - score_threshold, nms_threshold, nms_top_k, - keep_top_k): +def batched_retinanet_detection_out( + boxes, + scores, + anchors, + im_info, + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, +): batch_size = scores[0].shape[0] det_outs = [] lod = [] @@ -158,8 +192,15 @@ def batched_retinanet_detection_out(boxes, scores, anchors, im_info, scores_per_batch.append(scores[lvl][n]) nmsed_outs, nmsed_num = retinanet_detection_out( - boxes_per_batch, scores_per_batch, anchors, im_info[n], - score_threshold, nms_threshold, nms_top_k, keep_top_k) + boxes_per_batch, + scores_per_batch, + anchors, + im_info[n], + score_threshold, + nms_threshold, + nms_top_k, + keep_top_k, + ) lod.append(nmsed_num) if nmsed_num == 0: continue @@ -169,7 +210,6 @@ def batched_retinanet_detection_out(boxes, scores, anchors, im_info, class TestRetinanetDetectionOutOp1(OpTest): - def set_argument(self): self.score_threshold = 0.05 self.min_level = 3 @@ -192,8 +232,8 @@ def set_argument(self): self.layer_w = [] num_levels = self.max_level - self.min_level + 1 for i in range(num_levels): - self.layer_h.append(2**(num_levels - i)) - self.layer_w.append(2**(num_levels - i)) + self.layer_h.append(2 ** (num_levels - i)) + self.layer_w.append(2 ** (num_levels - i)) def init_test_input(self): anchor_num = len(self.aspect_ratios) * self.scales_per_octave @@ -206,62 +246,89 @@ def init_test_input(self): layer_h = self.layer_h[i] layer_w = self.layer_w[i] - input_feat = np.random.random((self.batch_size, self.input_channels, - layer_h, layer_w)).astype('float32') + input_feat = np.random.random( + (self.batch_size, self.input_channels, layer_h, layer_w) + ).astype('float32') score = np.random.random( - (self.batch_size, self.class_num * anchor_num, layer_h, - layer_w)).astype('float32') + (self.batch_size, self.class_num * anchor_num, layer_h, layer_w) + ).astype('float32') score = np.transpose(score, [0, 2, 3, 1]) score = score.reshape((self.batch_size, -1, self.class_num)) - box = np.random.random((self.batch_size, self.box_size * anchor_num, - layer_h, layer_w)).astype('float32') + box = np.random.random( + (self.batch_size, self.box_size * anchor_num, layer_h, layer_w) + ).astype('float32') box = np.transpose(box, [0, 2, 3, 1]) box = box.reshape((self.batch_size, -1, self.box_size)) anchor_sizes = [] for octave in range(self.scales_per_octave): anchor_sizes.append( - float(self.anchor_strides[i] * (2**octave)) / - float(self.scales_per_octave) * self.anchor_scale) + float(self.anchor_strides[i] * (2**octave)) + / float(self.scales_per_octave) + * self.anchor_scale + ) anchor, var = anchor_generator_in_python( input_feat=input_feat, anchor_sizes=anchor_sizes, aspect_ratios=self.aspect_ratios, variances=[1.0, 1.0, 1.0, 1.0], stride=[self.anchor_strides[i], self.anchor_strides[i]], - offset=0.5) + offset=0.5, + ) anchor = np.reshape(anchor, [-1, 4]) self.scores_list.append(score.astype('float32')) self.bboxes_list.append(box.astype('float32')) self.anchors_list.append(anchor.astype('float32')) - self.im_info = np.array([[256., 256., 1.5]]).astype( - 'float32') #im_height, im_width, scale + self.im_info = np.array([[256.0, 256.0, 1.5]]).astype( + 'float32' + ) # im_height, im_width, scale def setUp(self): self.set_argument() self.init_test_input() nmsed_outs, lod = batched_retinanet_detection_out( - self.bboxes_list, self.scores_list, self.anchors_list, self.im_info, - self.score_threshold, self.nms_threshold, self.nms_top_k, - self.keep_top_k) + self.bboxes_list, + self.scores_list, + self.anchors_list, + self.im_info, + self.score_threshold, + self.nms_threshold, + self.nms_top_k, + self.keep_top_k, + ) nmsed_outs = np.array(nmsed_outs).astype('float32') self.op_type = 'retinanet_detection_output' self.inputs = { - 'BBoxes': [('b0', self.bboxes_list[0]), ('b1', self.bboxes_list[1]), - ('b2', self.bboxes_list[2]), ('b3', self.bboxes_list[3]), - ('b4', self.bboxes_list[4])], - 'Scores': [('s0', self.scores_list[0]), ('s1', self.scores_list[1]), - ('s2', self.scores_list[2]), ('s3', self.scores_list[3]), - ('s4', self.scores_list[4])], - 'Anchors': [('a0', self.anchors_list[0]), - ('a1', self.anchors_list[1]), - ('a2', self.anchors_list[2]), - ('a3', self.anchors_list[3]), - ('a4', self.anchors_list[4])], - 'ImInfo': (self.im_info, [[ - 1, - ]]) + 'BBoxes': [ + ('b0', self.bboxes_list[0]), + ('b1', self.bboxes_list[1]), + ('b2', self.bboxes_list[2]), + ('b3', self.bboxes_list[3]), + ('b4', self.bboxes_list[4]), + ], + 'Scores': [ + ('s0', self.scores_list[0]), + ('s1', self.scores_list[1]), + ('s2', self.scores_list[2]), + ('s3', self.scores_list[3]), + ('s4', self.scores_list[4]), + ], + 'Anchors': [ + ('a0', self.anchors_list[0]), + ('a1', self.anchors_list[1]), + ('a2', self.anchors_list[2]), + ('a3', self.anchors_list[3]), + ('a4', self.anchors_list[4]), + ], + 'ImInfo': ( + self.im_info, + [ + [ + 1, + ] + ], + ), } self.outputs = {'Out': (nmsed_outs, [lod])} self.attrs = { @@ -269,7 +336,7 @@ def setUp(self): 'nms_top_k': self.nms_top_k, 'nms_threshold': self.nms_threshold, 'keep_top_k': self.keep_top_k, - 'nms_eta': 1., + 'nms_eta': 1.0, } def test_check_output(self): @@ -277,7 +344,6 @@ def test_check_output(self): class TestRetinanetDetectionOutOp2(OpTest): - def set_argument(self): self.score_threshold = 0.05 self.min_level = 3 @@ -302,7 +368,6 @@ def set_argument(self): class TestRetinanetDetectionOutOpNo3(TestRetinanetDetectionOutOp1): - def set_argument(self): # Here set 2.0 to test the case there is no outputs. # In practical use, 0.0 < score_threshold < 1.0 @@ -327,12 +392,11 @@ def set_argument(self): self.layer_w = [] num_levels = self.max_level - self.min_level + 1 for i in range(num_levels): - self.layer_h.append(2**(num_levels - i)) - self.layer_w.append(2**(num_levels - i)) + self.layer_h.append(2 ** (num_levels - i)) + self.layer_w.append(2 ** (num_levels - i)) class TestRetinanetDetectionOutOpNo4(TestRetinanetDetectionOutOp1): - def set_argument(self): self.score_threshold = 0.05 self.min_level = 2 @@ -355,33 +419,52 @@ def set_argument(self): self.layer_w = [] num_levels = self.max_level - self.min_level + 1 for i in range(num_levels): - self.layer_h.append(2**(num_levels - i)) - self.layer_w.append(2**(num_levels - i)) + self.layer_h.append(2 ** (num_levels - i)) + self.layer_w.append(2 ** (num_levels - i)) def setUp(self): self.set_argument() self.init_test_input() nmsed_outs, lod = batched_retinanet_detection_out( - self.bboxes_list, self.scores_list, self.anchors_list, self.im_info, - self.score_threshold, self.nms_threshold, self.nms_top_k, - self.keep_top_k) + self.bboxes_list, + self.scores_list, + self.anchors_list, + self.im_info, + self.score_threshold, + self.nms_threshold, + self.nms_top_k, + self.keep_top_k, + ) nmsed_outs = np.array(nmsed_outs).astype('float32') self.op_type = 'retinanet_detection_output' self.inputs = { - 'BBoxes': [('b0', self.bboxes_list[0]), ('b1', self.bboxes_list[1]), - ('b2', self.bboxes_list[2]), - ('b3', self.bboxes_list[3])], - 'Scores': [('s0', self.scores_list[0]), ('s1', self.scores_list[1]), - ('s2', self.scores_list[2]), - ('s3', self.scores_list[3])], - 'Anchors': [('a0', self.anchors_list[0]), - ('a1', self.anchors_list[1]), - ('a2', self.anchors_list[2]), - ('a3', self.anchors_list[3])], - 'ImInfo': (self.im_info, [[ - 1, - ]]) + 'BBoxes': [ + ('b0', self.bboxes_list[0]), + ('b1', self.bboxes_list[1]), + ('b2', self.bboxes_list[2]), + ('b3', self.bboxes_list[3]), + ], + 'Scores': [ + ('s0', self.scores_list[0]), + ('s1', self.scores_list[1]), + ('s2', self.scores_list[2]), + ('s3', self.scores_list[3]), + ], + 'Anchors': [ + ('a0', self.anchors_list[0]), + ('a1', self.anchors_list[1]), + ('a2', self.anchors_list[2]), + ('a3', self.anchors_list[3]), + ], + 'ImInfo': ( + self.im_info, + [ + [ + 1, + ] + ], + ), } self.outputs = {'Out': (nmsed_outs, [lod])} self.attrs = { @@ -389,7 +472,7 @@ def setUp(self): 'nms_top_k': self.nms_top_k, 'nms_threshold': self.nms_threshold, 'keep_top_k': self.keep_top_k, - 'nms_eta': 1., + 'nms_eta': 1.0, } def test_check_output(self): @@ -397,7 +480,6 @@ def test_check_output(self): class TestRetinanetDetectionOutOpNo5(TestRetinanetDetectionOutOp1): - def set_argument(self): self.score_threshold = 0.05 self.min_level = 3 @@ -420,35 +502,34 @@ def set_argument(self): self.layer_w = [] num_levels = self.max_level - self.min_level + 1 for i in range(num_levels): - self.layer_h.append(2**(num_levels - i)) - self.layer_w.append(2**(num_levels - i)) + self.layer_h.append(2 ** (num_levels - i)) + self.layer_w.append(2 ** (num_levels - i)) class TestRetinanetDetectionOutOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - bboxes_low1 = fluid.data(name='bboxes_low1', - shape=[1, 44, 4], - dtype='float32') - bboxes_high1 = fluid.data(name='bboxes_high1', - shape=[1, 11, 4], - dtype='float32') - scores_low1 = fluid.data(name='scores_low1', - shape=[1, 44, 10], - dtype='float32') - scores_high1 = fluid.data(name='scores_high1', - shape=[1, 11, 10], - dtype='float32') - anchors_low1 = fluid.data(name='anchors_low1', - shape=[44, 4], - dtype='float32') - anchors_high1 = fluid.data(name='anchors_high1', - shape=[11, 4], - dtype='float32') - im_info1 = fluid.data(name="im_info1", - shape=[1, 3], - dtype='float32') + bboxes_low1 = fluid.data( + name='bboxes_low1', shape=[1, 44, 4], dtype='float32' + ) + bboxes_high1 = fluid.data( + name='bboxes_high1', shape=[1, 11, 4], dtype='float32' + ) + scores_low1 = fluid.data( + name='scores_low1', shape=[1, 44, 10], dtype='float32' + ) + scores_high1 = fluid.data( + name='scores_high1', shape=[1, 11, 10], dtype='float32' + ) + anchors_low1 = fluid.data( + name='anchors_low1', shape=[44, 4], dtype='float32' + ) + anchors_high1 = fluid.data( + name='anchors_high1', shape=[11, 4], dtype='float32' + ) + im_info1 = fluid.data( + name="im_info1", shape=[1, 3], dtype='float32' + ) # The `bboxes` must be list, each element must be Variable and # its Tensor data type must be one of float32 and float64. @@ -457,19 +538,21 @@ def test_bboxes_type(): bboxes=bboxes_low1, scores=[scores_low1, scores_high1], anchors=[anchors_low1, anchors_high1], - im_info=im_info1) + im_info=im_info1, + ) self.assertRaises(TypeError, test_bboxes_type) def test_bboxes_tensor_dtype(): - bboxes_high2 = fluid.data(name='bboxes_high2', - shape=[1, 11, 4], - dtype='int32') + bboxes_high2 = fluid.data( + name='bboxes_high2', shape=[1, 11, 4], dtype='int32' + ) fluid.layers.retinanet_detection_output( bboxes=[bboxes_high2, 5], scores=[scores_low1, scores_high1], anchors=[anchors_low1, anchors_high1], - im_info=im_info1) + im_info=im_info1, + ) self.assertRaises(TypeError, test_bboxes_tensor_dtype) @@ -480,19 +563,21 @@ def test_scores_type(): bboxes=[bboxes_low1, bboxes_high1], scores=scores_low1, anchors=[anchors_low1, anchors_high1], - im_info=im_info1) + im_info=im_info1, + ) self.assertRaises(TypeError, test_scores_type) def test_scores_tensor_dtype(): - scores_high2 = fluid.data(name='scores_high2', - shape=[1, 11, 10], - dtype='int32') + scores_high2 = fluid.data( + name='scores_high2', shape=[1, 11, 10], dtype='int32' + ) fluid.layers.retinanet_detection_output( bboxes=[bboxes_low1, bboxes_high1], scores=[scores_high2, 5], anchors=[anchors_low1, anchors_high1], - im_info=im_info1) + im_info=im_info1, + ) self.assertRaises(TypeError, test_scores_tensor_dtype) @@ -503,19 +588,21 @@ def test_anchors_type(): bboxes=[bboxes_low1, bboxes_high1], scores=[scores_low1, scores_high1], anchors=anchors_low1, - im_info=im_info1) + im_info=im_info1, + ) self.assertRaises(TypeError, test_anchors_type) def test_anchors_tensor_dtype(): - anchors_high2 = fluid.data(name='anchors_high2', - shape=[11, 4], - dtype='int32') + anchors_high2 = fluid.data( + name='anchors_high2', shape=[11, 4], dtype='int32' + ) fluid.layers.retinanet_detection_output( bboxes=[bboxes_low1, bboxes_high1], scores=[scores_low1, scores_high1], anchors=[anchors_high2, 5], - im_info=im_info1) + im_info=im_info1, + ) self.assertRaises(TypeError, test_anchors_tensor_dtype) @@ -526,19 +613,21 @@ def test_iminfo_type(): bboxes=[bboxes_low1, bboxes_high1], scores=[scores_low1, scores_high1], anchors=[anchors_low1, anchors_high1], - im_info=[2, 3, 4]) + im_info=[2, 3, 4], + ) self.assertRaises(TypeError, test_iminfo_type) def test_iminfo_tensor_dtype(): - im_info2 = fluid.data(name='im_info2', - shape=[1, 3], - dtype='int32') + im_info2 = fluid.data( + name='im_info2', shape=[1, 3], dtype='int32' + ) fluid.layers.retinanet_detection_output( bboxes=[bboxes_low1, bboxes_high1], scores=[scores_low1, scores_high1], anchors=[anchors_low1, anchors_high1], - im_info=im_info2) + im_info=im_info2, + ) self.assertRaises(TypeError, test_iminfo_tensor_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_reverse_op.py b/python/paddle/fluid/tests/unittests/test_reverse_op.py index fc304d516f50b..829ef2a883b25 100644 --- a/python/paddle/fluid/tests/unittests/test_reverse_op.py +++ b/python/paddle/fluid/tests/unittests/test_reverse_op.py @@ -28,7 +28,6 @@ class TestReverseOp(OpTest): - def initTestCase(self): self.x = np.random.random((3, 40)).astype('float64') self.axis = [0] @@ -52,63 +51,54 @@ def test_check_grad(self): class TestCase0(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 40)).astype('float64') self.axis = [1] class TestCase0_neg(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 40)).astype('float64') self.axis = [-1] class TestCase1(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 40)).astype('float64') self.axis = [0, 1] class TestCase1_neg(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 40)).astype('float64') self.axis = [0, -1] class TestCase2(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 4, 10)).astype('float64') self.axis = [0, 2] class TestCase2_neg(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 4, 10)).astype('float64') self.axis = [0, -2] class TestCase3(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 4, 10)).astype('float64') self.axis = [1, 2] class TestCase3_neg(TestReverseOp): - def initTestCase(self): self.x = np.random.random((3, 4, 10)).astype('float64') self.axis = [-1, -2] class TestCase4(unittest.TestCase): - def test_error(self): place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -116,9 +106,9 @@ def test_error(self): train_program = fluid.Program() startup_program = fluid.Program() with fluid.program_guard(train_program, startup_program): - label = fluid.layers.data(name="label", - shape=[1, 1, 1, 1, 1, 1, 1, 1], - dtype="int64") + label = fluid.layers.data( + name="label", shape=[1, 1, 1, 1, 1, 1, 1, 1], dtype="int64" + ) rev = fluid.layers.reverse(label, axis=[-1, -2]) def _run_program(): @@ -129,11 +119,13 @@ def _run_program(): class TestReverseLoDTensorArray(unittest.TestCase): - def setUp(self): self.shapes = [[5, 25], [5, 20], [5, 5]] - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.exe = fluid.Executor(self.place) def run_program(self, arr_len, axis=0): @@ -146,7 +138,8 @@ def run_program(self, arr_len, axis=0): x.stop_gradient = False inputs.append(x) inputs_data.append( - np.random.random(self.shapes[i]).astype('float32')) + np.random.random(self.shapes[i]).astype('float32') + ) tensor_array = fluid.layers.create_array(dtype='float32') for i in range(arr_len): @@ -158,13 +151,18 @@ def run_program(self, arr_len, axis=0): loss = fluid.layers.reduce_sum(output) fluid.backward.append_backward(loss) input_grads = list( - map(main_program.global_block().var, - [x.name + "@GRAD" for x in inputs])) + map( + main_program.global_block().var, + [x.name + "@GRAD" for x in inputs], + ) + ) feed_dict = dict(zip([x.name for x in inputs], inputs_data)) - res = self.exe.run(main_program, - feed=feed_dict, - fetch_list=input_grads + [output.name]) + res = self.exe.run( + main_program, + feed=feed_dict, + fetch_list=input_grads + [output.name], + ) return np.hstack(inputs_data[::-1]), res @@ -172,7 +170,7 @@ def test_case1(self): gt, res = self.run_program(arr_len=3) self.check_output(gt, res) # test with tuple type of axis - gt, res = self.run_program(arr_len=3, axis=(0, )) + gt, res = self.run_program(arr_len=3, axis=(0,)) self.check_output(gt, res) def test_case2(self): @@ -201,7 +199,6 @@ def test_raise_error(self): class TestReverseAxisTensor(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -227,8 +224,9 @@ def test_static(self): gt = res[0][::-1, :, ::-1] np.testing.assert_allclose(res[1], gt) - paddle.static.save_inference_model(self.save_path, [x], [feat, out], - exe) + paddle.static.save_inference_model( + self.save_path, [x], [feat, out], exe + ) # Test for Inference Predictor infer_outs = self.infer_prog() gt = infer_outs[0][::-1, :, ::-1] @@ -248,7 +246,6 @@ def call_func(self, x): class TestReverseAxisListTensor(TestReverseAxisTensor): - def path_prefix(self): return 'reverse_tensors' @@ -261,15 +258,18 @@ def call_func(self, x): out = paddle.fluid.layers.reverse(x, axes) # check attrs - axis_attrs = paddle.static.default_main_program().block( - 0).ops[-1].all_attrs()["axis"] + axis_attrs = ( + paddle.static.default_main_program() + .block(0) + .ops[-1] + .all_attrs()["axis"] + ) self.assertTrue(axis_attrs[0].name, axes[0].name) self.assertTrue(axis_attrs[1].name, axes[1].name) return out class TestReverseDoubleGradCheck(unittest.TestCase): - def reverse_wrapper(self, x): return fluid.layers.reverse(x[0], [0, 1]) @@ -284,17 +284,13 @@ def func(self, place): out = fluid.layers.reverse(data, [0, 1]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.reverse_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.reverse_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -306,7 +302,6 @@ def test_grad(self): class TestReverseTripleGradCheck(unittest.TestCase): - def reverse_wrapper(self, x): return fluid.layers.reverse(x[0], [0, 1]) @@ -321,17 +316,13 @@ def func(self, place): out = fluid.layers.reverse(data, [0, 1]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.reverse_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.reverse_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_rmsprop_op.py b/python/paddle/fluid/tests/unittests/test_rmsprop_op.py index 1948026b374d0..5845b421fb511 100644 --- a/python/paddle/fluid/tests/unittests/test_rmsprop_op.py +++ b/python/paddle/fluid/tests/unittests/test_rmsprop_op.py @@ -21,14 +21,19 @@ import paddle -def create_selected_rows_and_tensor(scope, place, height, row_num, - embedding_size): +def create_selected_rows_and_tensor( + scope, place, height, row_num, embedding_size +): sr = scope.var("@selected_rows@").get_selected_rows() tensor = scope.var("grad").get_tensor() - rows = np.random.random_integers(low=0, high=height - 1, size=[ - row_num, - ]).astype('int64') + rows = np.random.random_integers( + low=0, + high=height - 1, + size=[ + row_num, + ], + ).astype('int64') sr_val = np.random.random(size=[row_num, embedding_size]).astype('float32') sr.set_height(height) @@ -45,14 +50,9 @@ def create_selected_rows_and_tensor(scope, place, height, row_num, class TestBase(unittest.TestCase): - - def setup(self, - place, - is_sparse, - centered, - size, - row_num=None, - epsilon=1e-6): + def setup( + self, place, is_sparse, centered, size, row_num=None, epsilon=1e-6 + ): np.random.seed(5) # fix seed self.scope = fluid.global_scope() @@ -62,8 +62,9 @@ def setup(self, self.param = np.random.random(size).astype("float32") self.mean_square_name = "mean_square" - self.mean_square = np.random.uniform(low=1, high=2, - size=size).astype("float32") + self.mean_square = np.random.uniform(low=1, high=2, size=size).astype( + "float32" + ) self.mean_grad_name = "mean_grad" self.mean_grad = np.random.random(size).astype("float32") @@ -77,31 +78,44 @@ def setup(self, if self.is_sparse: self.grad_sr_name = "@selected_rows@" self.grad, self.grad_sr = create_selected_rows_and_tensor( - self.scope, place, size[0], row_num, size[1]) + self.scope, place, size[0], row_num, size[1] + ) else: self.grad = np.random.random(size).astype("float32") grad_tensor = self.scope.var(self.grad_name).get_tensor() grad_tensor.set(self.grad, place) self.moment_name = "moment" - self.moment = np.random.uniform(low=0, high=1, - size=size).astype("float32") + self.moment = np.random.uniform(low=0, high=1, size=size).astype( + "float32" + ) self.epsilon = epsilon self.decay = 0.9 self.momentum = 0.1 self.centered = centered - self.ms_out = self.decay * self.mean_square + ( - 1 - self.decay) * self.grad * self.grad + self.ms_out = ( + self.decay * self.mean_square + + (1 - self.decay) * self.grad * self.grad + ) if centered: - self.mg_out = self.decay * self.mean_grad + (1 - - self.decay) * self.grad - self.moment_out = self.momentum * self.moment + \ - self.learning_rate * self.grad / np.sqrt(self.ms_out - np.square(self.mg_out) + self.epsilon) + self.mg_out = ( + self.decay * self.mean_grad + (1 - self.decay) * self.grad + ) + self.moment_out = ( + self.momentum * self.moment + + self.learning_rate + * self.grad + / np.sqrt(self.ms_out - np.square(self.mg_out) + self.epsilon) + ) else: - self.moment_out = self.momentum * self.moment + \ - self.learning_rate * self.grad / np.sqrt(self.ms_out + self.epsilon) + self.moment_out = ( + self.momentum * self.moment + + self.learning_rate + * self.grad + / np.sqrt(self.ms_out + self.epsilon) + ) self.param_out = self.param - self.moment_out @@ -110,7 +124,8 @@ def setup(self, self.param_tensor.set(self.param, place) self.mean_square_tensor = self.scope.var( - self.mean_square_name).get_tensor() + self.mean_square_name + ).get_tensor() self.mean_square_tensor.set(self.mean_square, place) lr = self.scope.var(self.lr_name).get_tensor() @@ -121,7 +136,8 @@ def setup(self, if self.centered: self.mean_grad_tensor = self.scope.var( - self.mean_grad_name).get_tensor() + self.mean_grad_name + ).get_tensor() self.mean_grad_tensor.set(self.mean_grad, place) def check(self, actual_t, expect_t, place, out_name, atol=1e-5): @@ -130,19 +146,22 @@ def check(self, actual_t, expect_t, place, out_name, atol=1e-5): expect_t, rtol=1e-05, atol=atol, - err_msg='Output (' + out_name + ') has diff at ' + str(place) + - '\nExpect ' + str(expect_t) + '\n' + 'But Got' + str(actual_t)) + err_msg='Output (' + + out_name + + ') has diff at ' + + str(place) + + '\nExpect ' + + str(expect_t) + + '\n' + + 'But Got' + + str(actual_t), + ) class TestRmspropOp(TestBase): - - def check_with_place(self, - place, - is_sparse, - centered, - size, - row_num=None, - epsilon=1e-6): + def check_with_place( + self, place, is_sparse, centered, size, row_num=None, epsilon=1e-6 + ): self.setup(place, is_sparse, centered, size, row_num, epsilon) self.run_and_check() @@ -161,7 +180,7 @@ def run_and_check(self): 'epsilon': self.epsilon, 'decay': self.decay, 'momentum': self.momentum, - 'centered': self.centered + 'centered': self.centered, } if self.centered: @@ -173,25 +192,35 @@ def run_and_check(self): rmsprop_op.run(self.scope, self.place) - self.check(np.array(self.mean_square_tensor), - self.ms_out, - self.place, - self.mean_square_name, - atol=atol) - self.check(np.array(self.moment_tensor), - self.moment_out, - self.place, - self.moment_name, - atol=atol) - self.check(np.array(self.param_tensor), - self.param_out, - self.place, - self.param_name, - atol=atol) + self.check( + np.array(self.mean_square_tensor), + self.ms_out, + self.place, + self.mean_square_name, + atol=atol, + ) + self.check( + np.array(self.moment_tensor), + self.moment_out, + self.place, + self.moment_name, + atol=atol, + ) + self.check( + np.array(self.param_tensor), + self.param_out, + self.place, + self.param_name, + atol=atol, + ) if self.centered: - self.check(np.array(self.mean_grad_tensor), self.mg_out, self.place, - self.mean_grad_name) + self.check( + np.array(self.mean_grad_tensor), + self.mg_out, + self.place, + self.mean_grad_name, + ) def test_rmsprop(self): places = [core.CPUPlace()] @@ -202,37 +231,41 @@ def test_rmsprop(self): for place in places: for centered in [False, True]: with fluid.scope_guard(core.Scope()): - self.check_with_place(place, - is_sparse=False, - centered=centered, - size=size) + self.check_with_place( + place, is_sparse=False, centered=centered, size=size + ) with fluid.scope_guard(core.Scope()): - self.check_with_place(place, - is_sparse=True, - centered=centered, - row_num=512, - size=size) + self.check_with_place( + place, + is_sparse=True, + centered=centered, + row_num=512, + size=size, + ) with fluid.scope_guard(core.Scope()): - self.check_with_place(place, - is_sparse=True, - centered=centered, - row_num=60, - size=size) + self.check_with_place( + place, + is_sparse=True, + centered=centered, + row_num=60, + size=size, + ) class TestRMSPropV2(unittest.TestCase): - def test_rmsprop_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.RMSProp(learning_rate=0.01, - parameters=linear.parameters(), - weight_decay=0.01) + adam = paddle.optimizer.RMSProp( + learning_rate=0.01, + parameters=linear.parameters(), + weight_decay=0.01, + ) out = linear(a) out.backward() adam.step() @@ -253,8 +286,9 @@ def test_rmsprop(self): rms_optimizer.minimize(avg_cost) fetch_list = [avg_cost] - train_reader = paddle.batch(paddle.dataset.uci_housing.train(), - batch_size=1) + train_reader = paddle.batch( + paddle.dataset.uci_housing.train(), batch_size=1 + ) feeder = fluid.DataFeeder(place=place, feed_list=[x, y]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) @@ -263,38 +297,40 @@ def test_rmsprop(self): def test_raise_error(self): self.assertRaises(ValueError, paddle.optimizer.RMSProp, None) - self.assertRaises(ValueError, - paddle.optimizer.RMSProp, - learning_rate=0.1, - rho=None) - self.assertRaises(ValueError, - paddle.optimizer.RMSProp, - learning_rate=0.1, - epsilon=None) - self.assertRaises(ValueError, - paddle.optimizer.RMSProp, - learning_rate=0.1, - momentum=None) + self.assertRaises( + ValueError, paddle.optimizer.RMSProp, learning_rate=0.1, rho=None + ) + self.assertRaises( + ValueError, + paddle.optimizer.RMSProp, + learning_rate=0.1, + epsilon=None, + ) + self.assertRaises( + ValueError, + paddle.optimizer.RMSProp, + learning_rate=0.1, + momentum=None, + ) def test_rmsprop_op_invalid_input(self): paddle.disable_static() linear = paddle.nn.Linear(10, 10) with self.assertRaises(ValueError): - adam = paddle.optimizer.RMSProp(0.1, - epsilon=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.RMSProp( + 0.1, epsilon=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.RMSProp(0.1, - momentum=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.RMSProp( + 0.1, momentum=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.RMSProp(0.1, - rho=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.RMSProp( + 0.1, rho=-1, parameters=linear.parameters() + ) class TestRMSPropV2Group(TestRMSPropV2): - def test_rmsprop_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") @@ -302,17 +338,14 @@ def test_rmsprop_dygraph(self): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.RMSProp(learning_rate=0.01, - parameters=[{ - 'params': - linear_1.parameters() - }, { - 'params': - linear_2.parameters(), - 'weight_decay': - 0.001 - }], - weight_decay=0.01) + adam = paddle.optimizer.RMSProp( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + {'params': linear_2.parameters(), 'weight_decay': 0.001}, + ], + weight_decay=0.01, + ) out = linear_1(a) out = linear_2(out) out.backward() diff --git a/python/paddle/fluid/tests/unittests/test_rnn_cell_api.py b/python/paddle/fluid/tests/unittests/test_rnn_cell_api.py index 5cf1a79489e5e..1d04b4310539f 100644 --- a/python/paddle/fluid/tests/unittests/test_rnn_cell_api.py +++ b/python/paddle/fluid/tests/unittests/test_rnn_cell_api.py @@ -33,62 +33,68 @@ class TestLSTMCellError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_size, input_size, hidden_size = 4, 16, 16 - inputs = fluid.data(name='inputs', - shape=[None, input_size], - dtype='float32') - pre_hidden = fluid.data(name='pre_hidden', - shape=[None, hidden_size], - dtype='float32') - pre_cell = fluid.data(name='pre_cell', - shape=[None, hidden_size], - dtype='float32') + inputs = fluid.data( + name='inputs', shape=[None, input_size], dtype='float32' + ) + pre_hidden = fluid.data( + name='pre_hidden', shape=[None, hidden_size], dtype='float32' + ) + pre_cell = fluid.data( + name='pre_cell', shape=[None, hidden_size], dtype='float32' + ) cell = LSTMCell(hidden_size) def test_input_Variable(): - np_input = np.random.random( - (batch_size, input_size)).astype("float32") + np_input = np.random.random((batch_size, input_size)).astype( + "float32" + ) cell(np_input, [pre_hidden, pre_cell]) self.assertRaises(TypeError, test_input_Variable) def test_pre_hidden_Variable(): np_pre_hidden = np.random.random( - (batch_size, hidden_size)).astype("float32") + (batch_size, hidden_size) + ).astype("float32") cell(inputs, [np_pre_hidden, pre_cell]) self.assertRaises(TypeError, test_pre_hidden_Variable) def test_pre_cell_Variable(): - np_pre_cell = np.random.random( - (batch_size, input_size)).astype("float32") + np_pre_cell = np.random.random((batch_size, input_size)).astype( + "float32" + ) cell(inputs, [pre_hidden, np_pre_cell]) self.assertRaises(TypeError, test_pre_cell_Variable) def test_input_type(): - error_inputs = fluid.data(name='error_inputs', - shape=[None, input_size], - dtype='int32') + error_inputs = fluid.data( + name='error_inputs', shape=[None, input_size], dtype='int32' + ) cell(error_inputs, [pre_hidden, pre_cell]) self.assertRaises(TypeError, test_input_type) def test_pre_hidden_type(): - error_pre_hidden = fluid.data(name='error_pre_hidden', - shape=[None, hidden_size], - dtype='int32') + error_pre_hidden = fluid.data( + name='error_pre_hidden', + shape=[None, hidden_size], + dtype='int32', + ) cell(inputs, [error_pre_hidden, pre_cell]) self.assertRaises(TypeError, test_pre_hidden_type) def test_pre_cell_type(): - error_pre_cell = fluid.data(name='error_pre_cell', - shape=[None, hidden_size], - dtype='int32') + error_pre_cell = fluid.data( + name='error_pre_cell', + shape=[None, hidden_size], + dtype='int32', + ) cell(inputs, [pre_hidden, error_pre_cell]) self.assertRaises(TypeError, test_pre_cell_type) @@ -101,29 +107,35 @@ def test_dtype(): class TestLSTMCell(unittest.TestCase): - def setUp(self): self.batch_size = 4 self.input_size = 16 self.hidden_size = 16 def test_run(self): - inputs = fluid.data(name='inputs', - shape=[None, self.input_size], - dtype='float32') - pre_hidden = fluid.data(name='pre_hidden', - shape=[None, self.hidden_size], - dtype='float32') - pre_cell = fluid.data(name='pre_cell', - shape=[None, self.hidden_size], - dtype='float32') + inputs = fluid.data( + name='inputs', shape=[None, self.input_size], dtype='float32' + ) + pre_hidden = fluid.data( + name='pre_hidden', shape=[None, self.hidden_size], dtype='float32' + ) + pre_cell = fluid.data( + name='pre_cell', shape=[None, self.hidden_size], dtype='float32' + ) cell = LSTMCell(self.hidden_size) lstm_hidden_new, lstm_states_new = cell(inputs, [pre_hidden, pre_cell]) lstm_unit = contrib.layers.rnn_impl.BasicLSTMUnit( - "basicLSTM", self.hidden_size, None, None, None, None, 1.0, - "float32") + "basicLSTM", + self.hidden_size, + None, + None, + None, + None, + 1.0, + "float32", + ) lstm_hidden, lstm_cell = lstm_unit(inputs, pre_hidden, pre_cell) if core.is_compiled_with_cuda(): @@ -134,76 +146,91 @@ def test_run(self): exe.run(framework.default_startup_program()) inputs_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.input_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.input_size) + ).astype('float32') pre_hidden_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') pre_cell_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') - param_names = [[ - "LSTMCell/BasicLSTMUnit_0.w_0", "basicLSTM/BasicLSTMUnit_0.w_0" - ], ["LSTMCell/BasicLSTMUnit_0.b_0", "basicLSTM/BasicLSTMUnit_0.b_0"]] + param_names = [ + ["LSTMCell/BasicLSTMUnit_0.w_0", "basicLSTM/BasicLSTMUnit_0.w_0"], + ["LSTMCell/BasicLSTMUnit_0.b_0", "basicLSTM/BasicLSTMUnit_0.b_0"], + ] for names in param_names: - param = np.array(fluid.global_scope().find_var( - names[0]).get_tensor()) - param = np.random.uniform(-0.1, 0.1, - size=param.shape).astype('float32') + param = np.array( + fluid.global_scope().find_var(names[0]).get_tensor() + ) + param = np.random.uniform(-0.1, 0.1, size=param.shape).astype( + 'float32' + ) fluid.global_scope().find_var(names[0]).get_tensor().set( - param, place) + param, place + ) fluid.global_scope().find_var(names[1]).get_tensor().set( - param, place) - - out = exe.run(feed={ - 'inputs': inputs_np, - 'pre_hidden': pre_hidden_np, - 'pre_cell': pre_cell_np - }, - fetch_list=[lstm_hidden_new, lstm_hidden]) + param, place + ) + + out = exe.run( + feed={ + 'inputs': inputs_np, + 'pre_hidden': pre_hidden_np, + 'pre_cell': pre_cell_np, + }, + fetch_list=[lstm_hidden_new, lstm_hidden], + ) np.testing.assert_allclose(out[0], out[1], rtol=0.0001, atol=0) class TestGRUCellError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_size, input_size, hidden_size = 4, 16, 16 - inputs = fluid.data(name='inputs', - shape=[None, input_size], - dtype='float32') - pre_hidden = layers.data(name='pre_hidden', - shape=[None, hidden_size], - append_batch_size=False, - dtype='float32') + inputs = fluid.data( + name='inputs', shape=[None, input_size], dtype='float32' + ) + pre_hidden = layers.data( + name='pre_hidden', + shape=[None, hidden_size], + append_batch_size=False, + dtype='float32', + ) cell = GRUCell(hidden_size) def test_input_Variable(): - np_input = np.random.random( - (batch_size, input_size)).astype("float32") + np_input = np.random.random((batch_size, input_size)).astype( + "float32" + ) cell(np_input, pre_hidden) self.assertRaises(TypeError, test_input_Variable) def test_pre_hidden_Variable(): np_pre_hidden = np.random.random( - (batch_size, hidden_size)).astype("float32") + (batch_size, hidden_size) + ).astype("float32") cell(inputs, np_pre_hidden) self.assertRaises(TypeError, test_pre_hidden_Variable) def test_input_type(): - error_inputs = fluid.data(name='error_inputs', - shape=[None, input_size], - dtype='int32') + error_inputs = fluid.data( + name='error_inputs', shape=[None, input_size], dtype='int32' + ) cell(error_inputs, pre_hidden) self.assertRaises(TypeError, test_input_type) def test_pre_hidden_type(): - error_pre_hidden = fluid.data(name='error_pre_hidden', - shape=[None, hidden_size], - dtype='int32') + error_pre_hidden = fluid.data( + name='error_pre_hidden', + shape=[None, hidden_size], + dtype='int32', + ) cell(inputs, error_pre_hidden) self.assertRaises(TypeError, test_pre_hidden_type) @@ -216,28 +243,28 @@ def test_dtype(): class TestGRUCell(unittest.TestCase): - def setUp(self): self.batch_size = 4 self.input_size = 16 self.hidden_size = 16 def test_run(self): - inputs = fluid.data(name='inputs', - shape=[None, self.input_size], - dtype='float32') - pre_hidden = layers.data(name='pre_hidden', - shape=[None, self.hidden_size], - append_batch_size=False, - dtype='float32') + inputs = fluid.data( + name='inputs', shape=[None, self.input_size], dtype='float32' + ) + pre_hidden = layers.data( + name='pre_hidden', + shape=[None, self.hidden_size], + append_batch_size=False, + dtype='float32', + ) cell = GRUCell(self.hidden_size) gru_hidden_new, _ = cell(inputs, pre_hidden) - gru_unit = contrib.layers.rnn_impl.BasicGRUUnit("basicGRU", - self.hidden_size, None, - None, None, None, - "float32") + gru_unit = contrib.layers.rnn_impl.BasicGRUUnit( + "basicGRU", self.hidden_size, None, None, None, None, "float32" + ) gru_hidden = gru_unit(inputs, pre_hidden) if core.is_compiled_with_cuda(): @@ -248,120 +275,143 @@ def test_run(self): exe.run(framework.default_startup_program()) inputs_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.input_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.input_size) + ).astype('float32') pre_hidden_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') param_names = [ ["GRUCell/BasicGRUUnit_0.w_0", "basicGRU/BasicGRUUnit_0.w_0"], ["GRUCell/BasicGRUUnit_0.w_1", "basicGRU/BasicGRUUnit_0.w_1"], ["GRUCell/BasicGRUUnit_0.b_0", "basicGRU/BasicGRUUnit_0.b_0"], - ["GRUCell/BasicGRUUnit_0.b_1", "basicGRU/BasicGRUUnit_0.b_1"] + ["GRUCell/BasicGRUUnit_0.b_1", "basicGRU/BasicGRUUnit_0.b_1"], ] for names in param_names: - param = np.array(fluid.global_scope().find_var( - names[0]).get_tensor()) - param = np.random.uniform(-0.1, 0.1, - size=param.shape).astype('float32') + param = np.array( + fluid.global_scope().find_var(names[0]).get_tensor() + ) + param = np.random.uniform(-0.1, 0.1, size=param.shape).astype( + 'float32' + ) fluid.global_scope().find_var(names[0]).get_tensor().set( - param, place) + param, place + ) fluid.global_scope().find_var(names[1]).get_tensor().set( - param, place) + param, place + ) - out = exe.run(feed={ - 'inputs': inputs_np, - 'pre_hidden': pre_hidden_np - }, - fetch_list=[gru_hidden_new, gru_hidden]) + out = exe.run( + feed={'inputs': inputs_np, 'pre_hidden': pre_hidden_np}, + fetch_list=[gru_hidden_new, gru_hidden], + ) np.testing.assert_allclose(out[0], out[1], rtol=0.0001, atol=0) class TestRnnError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): batch_size = 4 input_size = 16 hidden_size = 16 seq_len = 4 - inputs = fluid.data(name='inputs', - shape=[None, input_size], - dtype='float32') - pre_hidden = layers.data(name='pre_hidden', - shape=[None, hidden_size], - append_batch_size=False, - dtype='float32') - inputs_basic_lstm = fluid.data(name='inputs_basic_lstm', - shape=[None, None, input_size], - dtype='float32') - sequence_length = fluid.data(name="sequence_length", - shape=[None], - dtype='int64') - - inputs_dynamic_rnn = layers.transpose(inputs_basic_lstm, - perm=[1, 0, 2]) + inputs = fluid.data( + name='inputs', shape=[None, input_size], dtype='float32' + ) + pre_hidden = layers.data( + name='pre_hidden', + shape=[None, hidden_size], + append_batch_size=False, + dtype='float32', + ) + inputs_basic_lstm = fluid.data( + name='inputs_basic_lstm', + shape=[None, None, input_size], + dtype='float32', + ) + sequence_length = fluid.data( + name="sequence_length", shape=[None], dtype='int64' + ) + + inputs_dynamic_rnn = layers.transpose( + inputs_basic_lstm, perm=[1, 0, 2] + ) cell = LSTMCell(hidden_size, name="LSTMCell_for_rnn") np_inputs_dynamic_rnn = np.random.random( - (seq_len, batch_size, input_size)).astype("float32") + (seq_len, batch_size, input_size) + ).astype("float32") def test_input_Variable(): - dynamic_rnn(cell=cell, - inputs=np_inputs_dynamic_rnn, - sequence_length=sequence_length, - is_reverse=False) + dynamic_rnn( + cell=cell, + inputs=np_inputs_dynamic_rnn, + sequence_length=sequence_length, + is_reverse=False, + ) self.assertRaises(TypeError, test_input_Variable) def test_input_list(): - dynamic_rnn(cell=cell, - inputs=[np_inputs_dynamic_rnn], - sequence_length=sequence_length, - is_reverse=False) + dynamic_rnn( + cell=cell, + inputs=[np_inputs_dynamic_rnn], + sequence_length=sequence_length, + is_reverse=False, + ) self.assertRaises(TypeError, test_input_list) def test_initial_states_type(): cell = GRUCell(hidden_size, name="GRUCell_for_rnn") error_initial_states = np.random.random( - (batch_size, hidden_size)).astype("float32") - dynamic_rnn(cell=cell, - inputs=inputs_dynamic_rnn, - initial_states=error_initial_states, - sequence_length=sequence_length, - is_reverse=False) + (batch_size, hidden_size) + ).astype("float32") + dynamic_rnn( + cell=cell, + inputs=inputs_dynamic_rnn, + initial_states=error_initial_states, + sequence_length=sequence_length, + is_reverse=False, + ) self.assertRaises(TypeError, test_initial_states_type) def test_initial_states_list(): error_initial_states = [ - np.random.random( - (batch_size, hidden_size)).astype("float32"), - np.random.random( - (batch_size, hidden_size)).astype("float32") + np.random.random((batch_size, hidden_size)).astype( + "float32" + ), + np.random.random((batch_size, hidden_size)).astype( + "float32" + ), ] - dynamic_rnn(cell=cell, - inputs=inputs_dynamic_rnn, - initial_states=error_initial_states, - sequence_length=sequence_length, - is_reverse=False) + dynamic_rnn( + cell=cell, + inputs=inputs_dynamic_rnn, + initial_states=error_initial_states, + sequence_length=sequence_length, + is_reverse=False, + ) self.assertRaises(TypeError, test_initial_states_type) def test_sequence_length_type(): - np_sequence_length = np.random.random( - (batch_size)).astype("float32") - dynamic_rnn(cell=cell, - inputs=inputs_dynamic_rnn, - sequence_length=np_sequence_length, - is_reverse=False) + np_sequence_length = np.random.random((batch_size)).astype( + "float32" + ) + dynamic_rnn( + cell=cell, + inputs=inputs_dynamic_rnn, + sequence_length=np_sequence_length, + is_reverse=False, + ) self.assertRaises(TypeError, test_sequence_length_type) class TestRnn(unittest.TestCase): - def setUp(self): self.batch_size = 4 self.input_size = 16 @@ -369,23 +419,36 @@ def setUp(self): self.seq_len = 4 def test_run(self): - inputs_basic_lstm = fluid.data(name='inputs_basic_lstm', - shape=[None, None, self.input_size], - dtype='float32') - sequence_length = fluid.data(name="sequence_length", - shape=[None], - dtype='int64') + inputs_basic_lstm = fluid.data( + name='inputs_basic_lstm', + shape=[None, None, self.input_size], + dtype='float32', + ) + sequence_length = fluid.data( + name="sequence_length", shape=[None], dtype='int64' + ) inputs_dynamic_rnn = layers.transpose(inputs_basic_lstm, perm=[1, 0, 2]) cell = LSTMCell(self.hidden_size, name="LSTMCell_for_rnn") - output, final_state = dynamic_rnn(cell=cell, - inputs=inputs_dynamic_rnn, - sequence_length=sequence_length, - is_reverse=False) + output, final_state = dynamic_rnn( + cell=cell, + inputs=inputs_dynamic_rnn, + sequence_length=sequence_length, + is_reverse=False, + ) output_new = layers.transpose(output, perm=[1, 0, 2]) - rnn_out, last_hidden, last_cell = basic_lstm(inputs_basic_lstm, None, None, self.hidden_size, num_layers=1, \ - batch_first = False, bidirectional=False, sequence_length=sequence_length, forget_bias = 1.0) + rnn_out, last_hidden, last_cell = basic_lstm( + inputs_basic_lstm, + None, + None, + self.hidden_size, + num_layers=1, + batch_first=False, + bidirectional=False, + sequence_length=sequence_length, + forget_bias=1.0, + ) if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) @@ -395,45 +458,57 @@ def test_run(self): exe.run(framework.default_startup_program()) inputs_basic_lstm_np = np.random.uniform( - -0.1, 0.1, - (self.seq_len, self.batch_size, self.input_size)).astype('float32') - sequence_length_np = np.ones(self.batch_size, - dtype='int64') * self.seq_len + -0.1, 0.1, (self.seq_len, self.batch_size, self.input_size) + ).astype('float32') + sequence_length_np = ( + np.ones(self.batch_size, dtype='int64') * self.seq_len + ) inputs_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.input_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.input_size) + ).astype('float32') pre_hidden_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') pre_cell_np = np.random.uniform( - -0.1, 0.1, (self.batch_size, self.hidden_size)).astype('float32') + -0.1, 0.1, (self.batch_size, self.hidden_size) + ).astype('float32') - param_names = [[ - "LSTMCell_for_rnn/BasicLSTMUnit_0.w_0", - "basic_lstm_layers_0/BasicLSTMUnit_0.w_0" - ], - [ - "LSTMCell_for_rnn/BasicLSTMUnit_0.b_0", - "basic_lstm_layers_0/BasicLSTMUnit_0.b_0" - ]] + param_names = [ + [ + "LSTMCell_for_rnn/BasicLSTMUnit_0.w_0", + "basic_lstm_layers_0/BasicLSTMUnit_0.w_0", + ], + [ + "LSTMCell_for_rnn/BasicLSTMUnit_0.b_0", + "basic_lstm_layers_0/BasicLSTMUnit_0.b_0", + ], + ] for names in param_names: - param = np.array(fluid.global_scope().find_var( - names[0]).get_tensor()) - param = np.random.uniform(-0.1, 0.1, - size=param.shape).astype('float32') + param = np.array( + fluid.global_scope().find_var(names[0]).get_tensor() + ) + param = np.random.uniform(-0.1, 0.1, size=param.shape).astype( + 'float32' + ) fluid.global_scope().find_var(names[0]).get_tensor().set( - param, place) + param, place + ) fluid.global_scope().find_var(names[1]).get_tensor().set( - param, place) - - out = exe.run(feed={ - 'inputs_basic_lstm': inputs_basic_lstm_np, - 'sequence_length': sequence_length_np, - 'inputs': inputs_np, - 'pre_hidden': pre_hidden_np, - 'pre_cell': pre_cell_np - }, - fetch_list=[output_new, rnn_out]) + param, place + ) + + out = exe.run( + feed={ + 'inputs_basic_lstm': inputs_basic_lstm_np, + 'sequence_length': sequence_length_np, + 'inputs': inputs_np, + 'pre_hidden': pre_hidden_np, + 'pre_cell': pre_cell_np, + }, + fetch_list=[output_new, rnn_out], + ) np.testing.assert_allclose(out[0], out[1], rtol=0.0001) @@ -462,7 +537,7 @@ def __init__( self, num_layers, hidden_size, - dropout_prob=0., + dropout_prob=0.0, init_scale=0.1, ): self.num_layers = num_layers @@ -477,10 +552,14 @@ def call(self, step_input, states): new_states = [] for i in range(self.num_layers): out, new_state = self.lstm_cells[i](step_input, states[i]) - step_input = layers.dropout( - out, - self.dropout_prob, - ) if self.dropout_prob else out + step_input = ( + layers.dropout( + out, + self.dropout_prob, + ) + if self.dropout_prob + else out + ) new_states.append(new_state) return step_input, new_states @@ -492,7 +571,7 @@ def state_shape(self): class DecoderCell(RNNCell): """Decoder Cell""" - def __init__(self, num_layers, hidden_size, dropout_prob=0.): + def __init__(self, num_layers, hidden_size, dropout_prob=0.0): self.num_layers = num_layers self.hidden_size = hidden_size self.dropout_prob = dropout_prob @@ -504,26 +583,31 @@ def call(self, step_input, states): new_lstm_states = [] for i in range(self.num_layers): out, new_lstm_state = self.lstm_cells[i](step_input, states[i]) - step_input = layers.dropout( - out, - self.dropout_prob, - ) if self.dropout_prob else out + step_input = ( + layers.dropout( + out, + self.dropout_prob, + ) + if self.dropout_prob + else out + ) new_lstm_states.append(new_lstm_state) return step_input, new_lstm_states -def def_seq2seq_model(num_layers, hidden_size, dropout_prob, src_vocab_size, - trg_vocab_size): +def def_seq2seq_model( + num_layers, hidden_size, dropout_prob, src_vocab_size, trg_vocab_size +): "vanilla seq2seq model" # data source = fluid.data(name="src", shape=[None, None], dtype="int64") - source_length = fluid.data(name="src_sequence_length", - shape=[None], - dtype="int64") + source_length = fluid.data( + name="src_sequence_length", shape=[None], dtype="int64" + ) target = fluid.data(name="trg", shape=[None, None], dtype="int64") - target_length = fluid.data(name="trg_sequence_length", - shape=[None], - dtype="int64") + target_length = fluid.data( + name="trg_sequence_length", shape=[None], dtype="int64" + ) label = fluid.data(name="label", shape=[None, None, 1], dtype="int64") # embedding @@ -532,29 +616,31 @@ def def_seq2seq_model(num_layers, hidden_size, dropout_prob, src_vocab_size, # encoder enc_cell = EncoderCell(num_layers, hidden_size, dropout_prob) - enc_output, enc_final_state = dynamic_rnn(cell=enc_cell, - inputs=src_emb, - sequence_length=source_length) + enc_output, enc_final_state = dynamic_rnn( + cell=enc_cell, inputs=src_emb, sequence_length=source_length + ) # decoder dec_cell = DecoderCell(num_layers, hidden_size, dropout_prob) - dec_output, dec_final_state = dynamic_rnn(cell=dec_cell, - inputs=tar_emb, - initial_states=enc_final_state) - logits = layers.fc(dec_output, - size=trg_vocab_size, - num_flatten_dims=len(dec_output.shape) - 1, - bias_attr=False) + dec_output, dec_final_state = dynamic_rnn( + cell=dec_cell, inputs=tar_emb, initial_states=enc_final_state + ) + logits = layers.fc( + dec_output, + size=trg_vocab_size, + num_flatten_dims=len(dec_output.shape) - 1, + bias_attr=False, + ) # loss - loss = layers.softmax_with_cross_entropy(logits=logits, - label=label, - soft_label=False) + loss = layers.softmax_with_cross_entropy( + logits=logits, label=label, soft_label=False + ) loss = layers.unsqueeze(loss, axes=[2]) max_tar_seq_len = layers.shape(target)[1] - tar_mask = layers.sequence_mask(target_length, - maxlen=max_tar_seq_len, - dtype="float32") + tar_mask = layers.sequence_mask( + target_length, maxlen=max_tar_seq_len, dtype="float32" + ) loss = loss * tar_mask loss = layers.reduce_mean(loss, dim=[0]) loss = layers.reduce_sum(loss) @@ -577,7 +663,7 @@ def setUp(self): "hidden_size": 128, "dropout_prob": 0.1, "src_vocab_size": 100, - "trg_vocab_size": 100 + "trg_vocab_size": 100, } self.iter_num = iter_num = 2 @@ -585,28 +671,34 @@ def setUp(self): src_seq_len = 10 trg_seq_len = 12 self.data = { - "src": - np.random.randint( - 2, self.model_hparams["src_vocab_size"], - (iter_num * batch_size, src_seq_len)).astype("int64"), - "src_sequence_length": - np.random.randint(1, src_seq_len, - (iter_num * batch_size, )).astype("int64"), - "trg": - np.random.randint( - 2, self.model_hparams["src_vocab_size"], - (iter_num * batch_size, trg_seq_len)).astype("int64"), - "trg_sequence_length": - np.random.randint(1, trg_seq_len, - (iter_num * batch_size, )).astype("int64"), - "label": - np.random.randint( - 2, self.model_hparams["src_vocab_size"], - (iter_num * batch_size, trg_seq_len, 1)).astype("int64"), + "src": np.random.randint( + 2, + self.model_hparams["src_vocab_size"], + (iter_num * batch_size, src_seq_len), + ).astype("int64"), + "src_sequence_length": np.random.randint( + 1, src_seq_len, (iter_num * batch_size,) + ).astype("int64"), + "trg": np.random.randint( + 2, + self.model_hparams["src_vocab_size"], + (iter_num * batch_size, trg_seq_len), + ).astype("int64"), + "trg_sequence_length": np.random.randint( + 1, trg_seq_len, (iter_num * batch_size,) + ).astype("int64"), + "label": np.random.randint( + 2, + self.model_hparams["src_vocab_size"], + (iter_num * batch_size, trg_seq_len, 1), + ).astype("int64"), } - place = core.CUDAPlace( - 0) if core.is_compiled_with_cuda() else core.CPUPlace() + place = ( + core.CUDAPlace(0) + if core.is_compiled_with_cuda() + else core.CPUPlace() + ) self.exe = Executor(place) def test_seq2seq_model(self): @@ -616,29 +708,38 @@ def test_seq2seq_model(self): cost = def_seq2seq_model(**self.model_hparams) self.exe.run(startup_program) for iter_idx in range(self.iter_num): - cost_val = self.exe.run(feed={ - "src": - self.data["src"][iter_idx * self.batch_size:(iter_idx + 1) * - self.batch_size, :], - "src_sequence_length": - self.data["src_sequence_length"][iter_idx * - self.batch_size:(iter_idx + - 1) * - self.batch_size], - "trg": - self.data["trg"][iter_idx * self.batch_size:(iter_idx + 1) * - self.batch_size, :], - "trg_sequence_length": - self.data["trg_sequence_length"][iter_idx * - self.batch_size:(iter_idx + - 1) * - self.batch_size], - "label": - self.data["label"][iter_idx * - self.batch_size:(iter_idx + 1) * - self.batch_size] - }, - fetch_list=[cost])[0] + cost_val = self.exe.run( + feed={ + "src": self.data["src"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size, + :, + ], + "src_sequence_length": self.data["src_sequence_length"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], + "trg": self.data["trg"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size, + :, + ], + "trg_sequence_length": self.data["trg_sequence_length"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], + "label": self.data["label"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], + }, + fetch_list=[cost], + )[0] print("iter_idx: %d, cost: %f" % (iter_idx, cost_val)) diff --git a/python/paddle/fluid/tests/unittests/test_rnn_decode_api.py b/python/paddle/fluid/tests/unittests/test_rnn_decode_api.py index a6f7b6ea8c2df..00bf9735fa513 100644 --- a/python/paddle/fluid/tests/unittests/test_rnn_decode_api.py +++ b/python/paddle/fluid/tests/unittests/test_rnn_decode_api.py @@ -34,8 +34,7 @@ class EncoderCell(layers.RNNCell): - - def __init__(self, num_layers, hidden_size, dropout_prob=0.): + def __init__(self, num_layers, hidden_size, dropout_prob=0.0): self.num_layers = num_layers self.hidden_size = hidden_size self.dropout_prob = dropout_prob @@ -47,8 +46,11 @@ def call(self, step_input, states): new_states = [] for i in range(self.num_layers): out, new_state = self.lstm_cells[i](step_input, states[i]) - step_input = layers.dropout( - out, self.dropout_prob) if self.dropout_prob > 0 else out + step_input = ( + layers.dropout(out, self.dropout_prob) + if self.dropout_prob > 0 + else out + ) new_states.append(new_state) return step_input, new_states @@ -58,8 +60,7 @@ def state_shape(self): class DecoderCell(layers.RNNCell): - - def __init__(self, num_layers, hidden_size, dropout_prob=0.): + def __init__(self, num_layers, hidden_size, dropout_prob=0.0): self.num_layers = num_layers self.hidden_size = hidden_size self.dropout_prob = dropout_prob @@ -68,42 +69,44 @@ def __init__(self, num_layers, hidden_size, dropout_prob=0.): ] def attention(self, hidden, encoder_output, encoder_padding_mask): - query = layers.fc(hidden, - size=encoder_output.shape[-1], - bias_attr=False) - attn_scores = layers.matmul(layers.unsqueeze(query, [1]), - encoder_output, - transpose_y=True) + query = layers.fc( + hidden, size=encoder_output.shape[-1], bias_attr=False + ) + attn_scores = layers.matmul( + layers.unsqueeze(query, [1]), encoder_output, transpose_y=True + ) if encoder_padding_mask is not None: - attn_scores = layers.elementwise_add(attn_scores, - encoder_padding_mask) + attn_scores = layers.elementwise_add( + attn_scores, encoder_padding_mask + ) attn_scores = layers.softmax(attn_scores) - attn_out = layers.squeeze(layers.matmul(attn_scores, encoder_output), - [1]) + attn_out = layers.squeeze( + layers.matmul(attn_scores, encoder_output), [1] + ) attn_out = layers.concat([attn_out, hidden], 1) attn_out = layers.fc(attn_out, size=self.hidden_size, bias_attr=False) return attn_out - def call(self, - step_input, - states, - encoder_output, - encoder_padding_mask=None): + def call( + self, step_input, states, encoder_output, encoder_padding_mask=None + ): lstm_states, input_feed = states new_lstm_states = [] step_input = layers.concat([step_input, input_feed], 1) for i in range(self.num_layers): out, new_lstm_state = self.lstm_cells[i](step_input, lstm_states[i]) - step_input = layers.dropout( - out, self.dropout_prob) if self.dropout_prob > 0 else out + step_input = ( + layers.dropout(out, self.dropout_prob) + if self.dropout_prob > 0 + else out + ) new_lstm_states.append(new_lstm_state) out = self.attention(step_input, encoder_output, encoder_padding_mask) return out, [new_lstm_states, out] class Encoder(object): - - def __init__(self, num_layers, hidden_size, dropout_prob=0.): + def __init__(self, num_layers, hidden_size, dropout_prob=0.0): self.encoder_cell = EncoderCell(num_layers, hidden_size, dropout_prob) def __call__(self, src_emb, src_sequence_length): @@ -111,25 +114,35 @@ def __call__(self, src_emb, src_sequence_length): cell=self.encoder_cell, inputs=src_emb, sequence_length=src_sequence_length, - is_reverse=False) + is_reverse=False, + ) return encoder_output, encoder_final_state class Decoder(object): - - def __init__(self, - num_layers, - hidden_size, - dropout_prob, - decoding_strategy="infer_sample", - max_decoding_length=20): + def __init__( + self, + num_layers, + hidden_size, + dropout_prob, + decoding_strategy="infer_sample", + max_decoding_length=20, + ): self.decoder_cell = DecoderCell(num_layers, hidden_size, dropout_prob) self.decoding_strategy = decoding_strategy - self.max_decoding_length = None if ( - self.decoding_strategy == "train_greedy") else max_decoding_length - - def __call__(self, decoder_initial_states, encoder_output, - encoder_padding_mask, **kwargs): + self.max_decoding_length = ( + None + if (self.decoding_strategy == "train_greedy") + else max_decoding_length + ) + + def __call__( + self, + decoder_initial_states, + encoder_output, + encoder_padding_mask, + **kwargs + ): output_layer = kwargs.pop("output_layer", None) if self.decoding_strategy == "train_greedy": # for teach-forcing MLE pre-training @@ -141,113 +154,150 @@ def __call__(self, decoder_initial_states, encoder_output, if self.decoding_strategy == "beam_search": beam_size = kwargs.get("beam_size", 4) - encoder_output = layers.BeamSearchDecoder.tile_beam_merge_with_batch( - encoder_output, beam_size) - encoder_padding_mask = layers.BeamSearchDecoder.tile_beam_merge_with_batch( - encoder_padding_mask, beam_size) - decoder = layers.BeamSearchDecoder(cell=self.decoder_cell, - output_fn=output_layer, - **kwargs) + encoder_output = ( + layers.BeamSearchDecoder.tile_beam_merge_with_batch( + encoder_output, beam_size + ) + ) + encoder_padding_mask = ( + layers.BeamSearchDecoder.tile_beam_merge_with_batch( + encoder_padding_mask, beam_size + ) + ) + decoder = layers.BeamSearchDecoder( + cell=self.decoder_cell, output_fn=output_layer, **kwargs + ) else: - decoder = layers.BasicDecoder(self.decoder_cell, - helper, - output_fn=output_layer) - - (decoder_output, decoder_final_state, - dec_seq_lengths) = layers.dynamic_decode( - decoder, - inits=decoder_initial_states, - max_step_num=self.max_decoding_length, - encoder_output=encoder_output, - encoder_padding_mask=encoder_padding_mask, - impute_finished=False # for test coverage - if self.decoding_strategy == "beam_search" else True, - is_test=True if self.decoding_strategy == "beam_search" else False, - return_length=True) + decoder = layers.BasicDecoder( + self.decoder_cell, helper, output_fn=output_layer + ) + + ( + decoder_output, + decoder_final_state, + dec_seq_lengths, + ) = layers.dynamic_decode( + decoder, + inits=decoder_initial_states, + max_step_num=self.max_decoding_length, + encoder_output=encoder_output, + encoder_padding_mask=encoder_padding_mask, + impute_finished=False # for test coverage + if self.decoding_strategy == "beam_search" + else True, + is_test=True if self.decoding_strategy == "beam_search" else False, + return_length=True, + ) return decoder_output, decoder_final_state, dec_seq_lengths class Seq2SeqModel(object): """Seq2Seq model: RNN encoder-decoder with attention""" - def __init__(self, - num_layers, - hidden_size, - dropout_prob, - src_vocab_size, - trg_vocab_size, - start_token, - end_token, - decoding_strategy="infer_sample", - max_decoding_length=20, - beam_size=4): + def __init__( + self, + num_layers, + hidden_size, + dropout_prob, + src_vocab_size, + trg_vocab_size, + start_token, + end_token, + decoding_strategy="infer_sample", + max_decoding_length=20, + beam_size=4, + ): self.start_token, self.end_token = start_token, end_token - self.max_decoding_length, self.beam_size = max_decoding_length, beam_size + self.max_decoding_length, self.beam_size = ( + max_decoding_length, + beam_size, + ) self.src_embeder = paddle.nn.Embedding( src_vocab_size, hidden_size, - weight_attr=fluid.ParamAttr(name="source_embedding")) + weight_attr=fluid.ParamAttr(name="source_embedding"), + ) self.trg_embeder = paddle.nn.Embedding( trg_vocab_size, hidden_size, - weight_attr=fluid.ParamAttr(name="target_embedding")) + weight_attr=fluid.ParamAttr(name="target_embedding"), + ) self.encoder = Encoder(num_layers, hidden_size, dropout_prob) - self.decoder = Decoder(num_layers, hidden_size, dropout_prob, - decoding_strategy, max_decoding_length) - self.output_layer = lambda x: layers.fc(x, - size=trg_vocab_size, - num_flatten_dims=len(x.shape) - - 1, - param_attr=fluid.ParamAttr(), - bias_attr=False) + self.decoder = Decoder( + num_layers, + hidden_size, + dropout_prob, + decoding_strategy, + max_decoding_length, + ) + self.output_layer = lambda x: layers.fc( + x, + size=trg_vocab_size, + num_flatten_dims=len(x.shape) - 1, + param_attr=fluid.ParamAttr(), + bias_attr=False, + ) def __call__(self, src, src_length, trg=None, trg_length=None): # encoder encoder_output, encoder_final_state = self.encoder( - self.src_embeder(src), src_length) + self.src_embeder(src), src_length + ) decoder_initial_states = [ encoder_final_state, self.decoder.decoder_cell.get_initial_states( - batch_ref=encoder_output, shape=[encoder_output.shape[-1]]) + batch_ref=encoder_output, shape=[encoder_output.shape[-1]] + ), ] - src_mask = layers.sequence_mask(src_length, - maxlen=layers.shape(src)[1], - dtype="float32") + src_mask = layers.sequence_mask( + src_length, maxlen=layers.shape(src)[1], dtype="float32" + ) encoder_padding_mask = (src_mask - 1.0) * 1e9 encoder_padding_mask = layers.unsqueeze(encoder_padding_mask, [1]) # decoder - decoder_kwargs = { - "inputs": self.trg_embeder(trg), - "sequence_length": trg_length, - } if self.decoder.decoding_strategy == "train_greedy" else ( + decoder_kwargs = ( { - "embedding_fn": self.trg_embeder, - "beam_size": self.beam_size, - "start_token": self.start_token, - "end_token": self.end_token - } if self.decoder.decoding_strategy == "beam_search" else { - "embedding_fn": - self.trg_embeder, - "start_tokens": - layers.fill_constant_batch_size_like(input=encoder_output, - shape=[-1], - dtype=src.dtype, - value=self.start_token), - "end_token": - self.end_token - }) + "inputs": self.trg_embeder(trg), + "sequence_length": trg_length, + } + if self.decoder.decoding_strategy == "train_greedy" + else ( + { + "embedding_fn": self.trg_embeder, + "beam_size": self.beam_size, + "start_token": self.start_token, + "end_token": self.end_token, + } + if self.decoder.decoding_strategy == "beam_search" + else { + "embedding_fn": self.trg_embeder, + "start_tokens": layers.fill_constant_batch_size_like( + input=encoder_output, + shape=[-1], + dtype=src.dtype, + value=self.start_token, + ), + "end_token": self.end_token, + } + ) + ) decoder_kwargs["output_layer"] = self.output_layer - (decoder_output, decoder_final_state, - dec_seq_lengths) = self.decoder(decoder_initial_states, encoder_output, - encoder_padding_mask, **decoder_kwargs) + (decoder_output, decoder_final_state, dec_seq_lengths) = self.decoder( + decoder_initial_states, + encoder_output, + encoder_padding_mask, + **decoder_kwargs + ) if self.decoder.decoding_strategy == "beam_search": # for inference return decoder_output - logits, samples, sample_length = (decoder_output.cell_outputs, - decoder_output.sample_ids, - dec_seq_lengths) + logits, samples, sample_length = ( + decoder_output.cell_outputs, + decoder_output.sample_ids, + dec_seq_lengths, + ) probs = layers.softmax(logits) return probs, samples, sample_length @@ -262,13 +312,16 @@ def learn(self, act_prob, action, reward, length=None): """ update policy model self.model with policy gradient algorithm """ - self.reward = fluid.layers.py_func(func=reward_func, - x=[action, length], - out=reward) + self.reward = fluid.layers.py_func( + func=reward_func, x=[action, length], out=reward + ) neg_log_prob = layers.cross_entropy(act_prob, action) cost = neg_log_prob * reward - cost = (layers.reduce_sum(cost) / layers.reduce_sum(length) - ) if length is not None else layers.reduce_mean(cost) + cost = ( + (layers.reduce_sum(cost) / layers.reduce_sum(length)) + if length is not None + else layers.reduce_mean(cost) + ) optimizer = fluid.optimizer.Adam(self.lr) optimizer.minimize(cost) return cost @@ -277,24 +330,26 @@ def learn(self, act_prob, action, reward, length=None): def reward_func(samples, sample_length): """toy reward""" - def discount_reward(reward, sequence_length, discount=1.): + def discount_reward(reward, sequence_length, discount=1.0): return discount_reward_1d(reward, sequence_length, discount) - def discount_reward_1d(reward, sequence_length, discount=1., dtype=None): + def discount_reward_1d(reward, sequence_length, discount=1.0, dtype=None): if sequence_length is None: raise ValueError( - 'sequence_length must not be `None` for 1D reward.') + 'sequence_length must not be `None` for 1D reward.' + ) reward = np.array(reward) sequence_length = np.array(sequence_length) batch_size = reward.shape[0] max_seq_length = np.max(sequence_length) dtype = dtype or reward.dtype - if discount == 1.: + if discount == 1.0: dmat = np.ones([batch_size, max_seq_length], dtype=dtype) else: steps = np.tile(np.arange(max_seq_length), [batch_size, 1]) - mask = np.asarray(steps < (sequence_length - 1)[:, None], - dtype=dtype) + mask = np.asarray( + steps < (sequence_length - 1)[:, None], dtype=dtype + ) # Make each row = [discount, ..., discount, 1, ..., 1] dmat = mask * discount + (1 - mask) dmat = np.cumprod(dmat[:, ::-1], axis=1)[:, ::-1] @@ -329,9 +384,15 @@ def mask_sequences(sequence, sequence_length, dtype=None, time_major=False): # repeat punishment to trapped into local minima getting all same words # beam search to get more than one sample may also can avoid this for i in range(reward.shape[0]): - reward[i] += -10 if sample_length[i] > 1 and np.all( - samples[i][:sample_length[i] - 1] == samples[i][0]) else 0 - return discount_reward(reward, sample_length, discount=1.).astype("float32") + reward[i] += ( + -10 + if sample_length[i] > 1 + and np.all(samples[i][: sample_length[i] - 1] == samples[i][0]) + else 0 + ) + return discount_reward(reward, sample_length, discount=1.0).astype( + "float32" + ) class MLE(object): @@ -353,20 +414,23 @@ def learn(self, probs, label, weight=None, length=None): class SeqPGAgent(object): - - def __init__(self, - model_cls, - alg_cls=PolicyGradient, - model_hparams={}, - alg_hparams={}, - executor=None, - main_program=None, - startup_program=None, - seed=None): - self.main_program = fluid.Program( - ) if main_program is None else main_program - self.startup_program = fluid.Program( - ) if startup_program is None else startup_program + def __init__( + self, + model_cls, + alg_cls=PolicyGradient, + model_hparams={}, + alg_hparams={}, + executor=None, + main_program=None, + startup_program=None, + seed=None, + ): + self.main_program = ( + fluid.Program() if main_program is None else main_program + ) + self.startup_program = ( + fluid.Program() if startup_program is None else startup_program + ) if seed is not None: self.main_program.random_seed = seed self.startup_program.random_seed = seed @@ -376,51 +440,55 @@ def __init__(self, def build_program(self, model_cls, alg_cls, model_hparams, alg_hparams): with fluid.program_guard(self.main_program, self.startup_program): source = fluid.data(name="src", shape=[None, None], dtype="int64") - source_length = fluid.data(name="src_sequence_length", - shape=[None], - dtype="int64") + source_length = fluid.data( + name="src_sequence_length", shape=[None], dtype="int64" + ) # only for teacher-forcing MLE training target = fluid.data(name="trg", shape=[None, None], dtype="int64") - target_length = fluid.data(name="trg_sequence_length", - shape=[None], - dtype="int64") - label = fluid.data(name="label", - shape=[None, None, 1], - dtype="int64") + target_length = fluid.data( + name="trg_sequence_length", shape=[None], dtype="int64" + ) + label = fluid.data( + name="label", shape=[None, None, 1], dtype="int64" + ) self.model = model_cls(**model_hparams) self.alg = alg_cls(**alg_hparams) self.probs, self.samples, self.sample_length = self.model( - source, source_length, target, target_length) + source, source_length, target, target_length + ) self.samples.stop_gradient = True self.reward = fluid.data( name="reward", shape=[None, None], # batch_size, seq_len - dtype=self.probs.dtype) + dtype=self.probs.dtype, + ) self.samples.stop_gradient = False - self.cost = self.alg.learn(self.probs, self.samples, self.reward, - self.sample_length) + self.cost = self.alg.learn( + self.probs, self.samples, self.reward, self.sample_length + ) # to define the same parameters between different programs self.pred_program = self.main_program._prune_with_input( [source.name, source_length.name], - [self.probs, self.samples, self.sample_length]) + [self.probs, self.samples, self.sample_length], + ) def predict(self, feed_dict): samples, sample_length = self.executor.run( self.pred_program, feed=feed_dict, - fetch_list=[self.samples, self.sample_length]) + fetch_list=[self.samples, self.sample_length], + ) return samples, sample_length def learn(self, feed_dict, fetch_list): - results = self.executor.run(self.main_program, - feed=feed_dict, - fetch_list=fetch_list) + results = self.executor.run( + self.main_program, feed=feed_dict, fetch_list=fetch_list + ) return results class TestDynamicDecode(unittest.TestCase): - def setUp(self): np.random.seed(123) self.model_hparams = { @@ -432,7 +500,7 @@ def setUp(self): "start_token": 0, "end_token": 1, "decoding_strategy": "infer_greedy", - "max_decoding_length": 10 + "max_decoding_length": 10, } self.iter_num = iter_num = 2 @@ -440,121 +508,159 @@ def setUp(self): src_seq_len = 10 trg_seq_len = 12 self.data = { - "src": - np.random.randint( - 2, self.model_hparams["src_vocab_size"], - (iter_num * batch_size, src_seq_len)).astype("int64"), - "src_sequence_length": - np.random.randint(1, src_seq_len, - (iter_num * batch_size, )).astype("int64"), - "trg": - np.random.randint( - 2, self.model_hparams["src_vocab_size"], - (iter_num * batch_size, trg_seq_len)).astype("int64"), - "trg_sequence_length": - np.random.randint(1, trg_seq_len, - (iter_num * batch_size, )).astype("int64"), - "label": - np.random.randint( - 2, self.model_hparams["src_vocab_size"], - (iter_num * batch_size, trg_seq_len, 1)).astype("int64"), + "src": np.random.randint( + 2, + self.model_hparams["src_vocab_size"], + (iter_num * batch_size, src_seq_len), + ).astype("int64"), + "src_sequence_length": np.random.randint( + 1, src_seq_len, (iter_num * batch_size,) + ).astype("int64"), + "trg": np.random.randint( + 2, + self.model_hparams["src_vocab_size"], + (iter_num * batch_size, trg_seq_len), + ).astype("int64"), + "trg_sequence_length": np.random.randint( + 1, trg_seq_len, (iter_num * batch_size,) + ).astype("int64"), + "label": np.random.randint( + 2, + self.model_hparams["src_vocab_size"], + (iter_num * batch_size, trg_seq_len, 1), + ).astype("int64"), } - place = core.CUDAPlace( - 0) if core.is_compiled_with_cuda() else core.CPUPlace() + place = ( + core.CUDAPlace(0) + if core.is_compiled_with_cuda() + else core.CPUPlace() + ) self.exe = Executor(place) def test_mle_train(self): paddle.enable_static() self.model_hparams["decoding_strategy"] = "train_greedy" - agent = SeqPGAgent(model_cls=Seq2SeqModel, - alg_cls=MLE, - model_hparams=self.model_hparams, - alg_hparams={"lr": 0.001}, - executor=self.exe, - main_program=fluid.Program(), - startup_program=fluid.Program(), - seed=123) + agent = SeqPGAgent( + model_cls=Seq2SeqModel, + alg_cls=MLE, + model_hparams=self.model_hparams, + alg_hparams={"lr": 0.001}, + executor=self.exe, + main_program=fluid.Program(), + startup_program=fluid.Program(), + seed=123, + ) self.exe.run(agent.startup_program) for iter_idx in range(self.iter_num): reward, cost = agent.learn( { - "src": - self.data["src"][iter_idx * self.batch_size:(iter_idx + 1) * - self.batch_size, :], - "src_sequence_length": - self.data["src_sequence_length"][iter_idx * self.batch_size: - (iter_idx + 1) * - self.batch_size], - "trg": - self.data["trg"][iter_idx * self.batch_size:(iter_idx + 1) * - self.batch_size, :], - "trg_sequence_length": - self.data["trg_sequence_length"][iter_idx * self.batch_size: - (iter_idx + 1) * - self.batch_size], - "label": - self.data["label"][iter_idx * - self.batch_size:(iter_idx + 1) * - self.batch_size] + "src": self.data["src"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size, + :, + ], + "src_sequence_length": self.data["src_sequence_length"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], + "trg": self.data["trg"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size, + :, + ], + "trg_sequence_length": self.data["trg_sequence_length"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], + "label": self.data["label"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], }, - fetch_list=[agent.cost, agent.cost]) - print("iter_idx: %d, reward: %f, cost: %f" % - (iter_idx, reward.mean(), cost)) + fetch_list=[agent.cost, agent.cost], + ) + print( + "iter_idx: %d, reward: %f, cost: %f" + % (iter_idx, reward.mean(), cost) + ) def test_greedy_train(self): paddle.enable_static() self.model_hparams["decoding_strategy"] = "infer_greedy" - agent = SeqPGAgent(model_cls=Seq2SeqModel, - alg_cls=PolicyGradient, - model_hparams=self.model_hparams, - alg_hparams={"lr": 0.001}, - executor=self.exe, - main_program=fluid.Program(), - startup_program=fluid.Program(), - seed=123) + agent = SeqPGAgent( + model_cls=Seq2SeqModel, + alg_cls=PolicyGradient, + model_hparams=self.model_hparams, + alg_hparams={"lr": 0.001}, + executor=self.exe, + main_program=fluid.Program(), + startup_program=fluid.Program(), + seed=123, + ) self.exe.run(agent.startup_program) for iter_idx in range(self.iter_num): reward, cost = agent.learn( { - "src": - self.data["src"][iter_idx * self.batch_size:(iter_idx + 1) * - self.batch_size, :], - "src_sequence_length": - self.data["src_sequence_length"][iter_idx * self.batch_size: - (iter_idx + 1) * - self.batch_size] + "src": self.data["src"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size, + :, + ], + "src_sequence_length": self.data["src_sequence_length"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], }, - fetch_list=[agent.reward, agent.cost]) - print("iter_idx: %d, reward: %f, cost: %f" % - (iter_idx, reward.mean(), cost)) + fetch_list=[agent.reward, agent.cost], + ) + print( + "iter_idx: %d, reward: %f, cost: %f" + % (iter_idx, reward.mean(), cost) + ) def test_sample_train(self): paddle.enable_static() self.model_hparams["decoding_strategy"] = "infer_sample" - agent = SeqPGAgent(model_cls=Seq2SeqModel, - alg_cls=PolicyGradient, - model_hparams=self.model_hparams, - alg_hparams={"lr": 0.001}, - executor=self.exe, - main_program=fluid.Program(), - startup_program=fluid.Program(), - seed=123) + agent = SeqPGAgent( + model_cls=Seq2SeqModel, + alg_cls=PolicyGradient, + model_hparams=self.model_hparams, + alg_hparams={"lr": 0.001}, + executor=self.exe, + main_program=fluid.Program(), + startup_program=fluid.Program(), + seed=123, + ) self.exe.run(agent.startup_program) for iter_idx in range(self.iter_num): reward, cost = agent.learn( { - "src": - self.data["src"][iter_idx * self.batch_size:(iter_idx + 1) * - self.batch_size, :], - "src_sequence_length": - self.data["src_sequence_length"][iter_idx * self.batch_size: - (iter_idx + 1) * - self.batch_size] + "src": self.data["src"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size, + :, + ], + "src_sequence_length": self.data["src_sequence_length"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], }, - fetch_list=[agent.reward, agent.cost]) - print("iter_idx: %d, reward: %f, cost: %f" % - (iter_idx, reward.mean(), cost)) + fetch_list=[agent.reward, agent.cost], + ) + print( + "iter_idx: %d, reward: %f, cost: %f" + % (iter_idx, reward.mean(), cost) + ) def test_beam_search_infer(self): paddle.set_default_dtype("float32") @@ -564,9 +670,9 @@ def test_beam_search_infer(self): startup_program = fluid.Program() with fluid.program_guard(main_program, startup_program): source = fluid.data(name="src", shape=[None, None], dtype="int64") - source_length = fluid.data(name="src_sequence_length", - shape=[None], - dtype="int64") + source_length = fluid.data( + name="src_sequence_length", shape=[None], dtype="int64" + ) model = Seq2SeqModel(**self.model_hparams) output = model(source, source_length) @@ -575,16 +681,20 @@ def test_beam_search_infer(self): trans_ids = self.exe.run( program=main_program, feed={ - "src": - self.data["src"][iter_idx * self.batch_size:(iter_idx + 1) * - self.batch_size, :], - "src_sequence_length": - self.data["src_sequence_length"][iter_idx * - self.batch_size:(iter_idx + - 1) * - self.batch_size] + "src": self.data["src"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size, + :, + ], + "src_sequence_length": self.data["src_sequence_length"][ + iter_idx + * self.batch_size : (iter_idx + 1) + * self.batch_size + ], }, - fetch_list=[output])[0] + fetch_list=[output], + )[0] def func_dynamic_basic_decoder(self): paddle.disable_static() @@ -601,7 +711,6 @@ def test_dynamic_basic_decoder(self): class ModuleApiTest(unittest.TestCase): - @classmethod def setUpClass(cls): cls._np_rand_state = np.random.get_state() @@ -611,10 +720,13 @@ def setUpClass(cls): random.seed(cls._random_seed) cls.model_cls = type( - cls.__name__ + "Model", (Layer, ), { + cls.__name__ + "Model", + (Layer,), + { "__init__": cls.model_init_wrapper(cls.model_init), - "forward": cls.model_forward - }) + "forward": cls.model_forward, + }, + ) @classmethod def tearDownClass(cls): @@ -623,7 +735,6 @@ def tearDownClass(cls): @staticmethod def model_init_wrapper(func): - def __impl__(self, *args, **kwargs): Layer.__init__(self) func(self, *args, **kwargs) @@ -633,7 +744,8 @@ def __impl__(self, *args, **kwargs): @staticmethod def model_init(model, *args, **kwargs): raise NotImplementedError( - "model_init acts as `Model.__init__`, thus must implement it") + "model_init acts as `Model.__init__`, thus must implement it" + ) @staticmethod def model_forward(model, *args, **kwargs): @@ -642,7 +754,8 @@ def model_forward(model, *args, **kwargs): def make_inputs(self): # TODO(guosheng): add default from `self.inputs` raise NotImplementedError( - "model_inputs makes inputs for model, thus must implement it") + "model_inputs makes inputs for model, thus must implement it" + ) def setUp(self): """ @@ -667,8 +780,11 @@ def _calc_output(self, place, mode="test", dygraph=True): paddle.framework.random._manual_program_seed(self._random_seed) scope = fluid.core.Scope() with fluid.scope_guard(scope): - layer = self.model_cls(**self.attrs) if isinstance( - self.attrs, dict) else self.model_cls(*self.attrs) + layer = ( + self.model_cls(**self.attrs) + if isinstance(self.attrs, dict) + else self.model_cls(*self.attrs) + ) model = Model(layer, inputs=self.make_inputs()) model.prepare() if self.param_states: @@ -683,10 +799,9 @@ def check_output_with_place(self, place, mode="test"): np.testing.assert_allclose(actual_t, expect_t, rtol=1e-05, atol=0) if expect_output: for actual_t, expect_t in zip(dygraph_output, expect_output): - np.testing.assert_allclose(actual_t, - expect_t, - rtol=1e-05, - atol=0) + np.testing.assert_allclose( + actual_t, expect_t, rtol=1e-05, atol=0 + ) def check_output(self): devices = ["CPU", "GPU"] if fluid.is_compiled_with_cuda() else ["CPU"] @@ -696,13 +811,12 @@ def check_output(self): class TestBeamSearch(ModuleApiTest): - def setUp(self): paddle.set_default_dtype("float64") shape = (8, 32) self.inputs = [ np.random.random(shape).astype("float64"), - np.random.random(shape).astype("float64") + np.random.random(shape).astype("float64"), ] self.outputs = None self.attrs = { @@ -713,33 +827,40 @@ def setUp(self): self.param_states = {} @staticmethod - def model_init(self, - vocab_size, - embed_dim, - hidden_size, - bos_id=0, - eos_id=1, - beam_size=4, - max_step_num=20): - embedder = paddle.fluid.dygraph.Embedding(size=[vocab_size, embed_dim], - dtype="float64") + def model_init( + self, + vocab_size, + embed_dim, + hidden_size, + bos_id=0, + eos_id=1, + beam_size=4, + max_step_num=20, + ): + embedder = paddle.fluid.dygraph.Embedding( + size=[vocab_size, embed_dim], dtype="float64" + ) output_layer = nn.Linear(hidden_size, vocab_size) cell = nn.LSTMCell(embed_dim, hidden_size) self.max_step_num = max_step_num - self.beam_search_decoder = BeamSearchDecoder(cell, - start_token=bos_id, - end_token=eos_id, - beam_size=beam_size, - embedding_fn=embedder, - output_fn=output_layer) + self.beam_search_decoder = BeamSearchDecoder( + cell, + start_token=bos_id, + end_token=eos_id, + beam_size=beam_size, + embedding_fn=embedder, + output_fn=output_layer, + ) @staticmethod def model_forward(model, init_hidden, init_cell): - return dynamic_decode(model.beam_search_decoder, - [init_hidden, init_cell], - max_step_num=model.max_step_num, - impute_finished=True, - is_test=True)[0] + return dynamic_decode( + model.beam_search_decoder, + [init_hidden, init_cell], + max_step_num=model.max_step_num, + impute_finished=True, + is_test=True, + )[0] def make_inputs(self): inputs = [ diff --git a/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py b/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py index aa5b852ff8802..f2014c29f91c7 100644 --- a/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py +++ b/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py @@ -21,59 +21,61 @@ class RNNMemoryHelperOpTest(unittest.TestCase): - def setUp(self): self.program = Program() self.place = core.CPUPlace() - self.X = self.program.global_block().create_var(name='X', - shape=[2, 3], - dtype='float32') - self.Out = self.program.global_block().create_var(name='Out', - shape=[2, 3], - dtype='float32') - self.program.global_block().append_op(type='rnn_memory_helper', - inputs={"X": self.X}, - outputs={"Out": self.Out}, - attrs={}) + self.X = self.program.global_block().create_var( + name='X', shape=[2, 3], dtype='float32' + ) + self.Out = self.program.global_block().create_var( + name='Out', shape=[2, 3], dtype='float32' + ) + self.program.global_block().append_op( + type='rnn_memory_helper', + inputs={"X": self.X}, + outputs={"Out": self.Out}, + attrs={}, + ) def test_forward(self): x_np = np.random.normal(size=(2, 3)).astype("float32") self.feed_map = {'X': x_np} self.fetch_list = [self.Out] exe = Executor(self.place) - out = exe.run(self.program, - feed=self.feed_map, - fetch_list=self.fetch_list) + out = exe.run( + self.program, feed=self.feed_map, fetch_list=self.fetch_list + ) np.testing.assert_allclose(out[0], x_np, rtol=1e-05) class RNNMemoryHelperGradOpTest(unittest.TestCase): - def setUp(self): self.program = Program() self.place = core.CPUPlace() self.input_names = ['X', 'Out', 'Out@GRAD'] self.input_vars = { - name: self.program.global_block().create_var(name=name, - shape=[2, 3], - dtype='float32') + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32' + ) for name in self.input_names } self.output_names = ['X@GRAD'] self.output_vars = { - name: self.program.global_block().create_var(name=name, - shape=[2, 3], - dtype='float32') + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32' + ) for name in self.output_names } - self.program.global_block().append_op(type='rnn_memory_helper_grad', - inputs=self.input_vars, - outputs=self.output_vars, - attrs={}) + self.program.global_block().append_op( + type='rnn_memory_helper_grad', + inputs=self.input_vars, + outputs=self.output_vars, + attrs={}, + ) def test_backward(self): self.feed_map = { @@ -83,14 +85,13 @@ def test_backward(self): self.fetch_list = [self.output_vars['X@GRAD']] exe = Executor(self.place) - out = exe.run(self.program, - feed=self.feed_map, - fetch_list=self.fetch_list) + out = exe.run( + self.program, feed=self.feed_map, fetch_list=self.fetch_list + ) np.isclose(out[0], self.feed_map['Out@GRAD'], rtol=1e-5) class RNNMemoryHelperGradOpWithoutInputTest(unittest.TestCase): - def setUp(self): self.program = Program() self.fake_program = Program() @@ -98,27 +99,31 @@ def setUp(self): self.input_names = ['X', 'Out'] self.input_vars = { - name: self.program.global_block().create_var(name=name, - shape=[2, 3], - dtype='float32') + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32' + ) for name in self.input_names } - self.input_vars["Out@GRAD"] = \ - self.fake_program.global_block().create_var( - name="Out@GRAD", shape=[2, 3], dtype='float32') + self.input_vars[ + "Out@GRAD" + ] = self.fake_program.global_block().create_var( + name="Out@GRAD", shape=[2, 3], dtype='float32' + ) self.output_names = ['X@GRAD'] self.output_vars = { - name: self.program.global_block().create_var(name=name, - shape=[2, 3], - dtype='float32') + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32' + ) for name in self.output_names } - self.program.global_block().append_op(type='rnn_memory_helper_grad', - inputs=self.input_vars, - outputs=self.output_vars, - attrs={}) + self.program.global_block().append_op( + type='rnn_memory_helper_grad', + inputs=self.input_vars, + outputs=self.output_vars, + attrs={}, + ) def test_backward(self): self.feed_map = { @@ -128,12 +133,12 @@ def test_backward(self): self.fetch_list = [self.output_vars['X@GRAD']] exe = Executor(self.place) - out = exe.run(self.program, - feed=self.feed_map, - fetch_list=self.fetch_list) - np.testing.assert_allclose(out[0], - np.zeros(shape=(2, 3)).astype('float32'), - rtol=1e-05) + out = exe.run( + self.program, feed=self.feed_map, fetch_list=self.fetch_list + ) + np.testing.assert_allclose( + out[0], np.zeros(shape=(2, 3)).astype('float32'), rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_rnn_op.py b/python/paddle/fluid/tests/unittests/test_rnn_op.py index ecf79cabab5ed..5dd255562e641 100644 --- a/python/paddle/fluid/tests/unittests/test_rnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_rnn_op.py @@ -31,7 +31,6 @@ class TestRNNOp(OpTest): - def get_weight_names(self): weight_names = [] for i in range(self.num_layers): @@ -45,8 +44,11 @@ def get_weight_names(self): def setUp(self): self.op_type = "rnn" self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.sequence_length = None if core.is_compiled_with_rocm( - ) else np.array([12, 11, 10, 9, 8], dtype=np.int32) + self.sequence_length = ( + None + if core.is_compiled_with_rocm() + else np.array([12, 11, 10, 9, 8], dtype=np.int32) + ) self.num_layers = 1 self.is_bidirec = False self.mode = "LSTM" @@ -61,27 +63,29 @@ def setUp(self): input_size = 3 hidden_size = 2 - input = np.random.uniform(low=-0.1, - high=0.1, - size=(seq_length, batch_size, - input_size)).astype(self.dtype) + input = np.random.uniform( + low=-0.1, high=0.1, size=(seq_length, batch_size, input_size) + ).astype(self.dtype) if self.sequence_length is not None: input[11][1:][:] = 0 input[10][2:][:] = 0 input[9][3:][:] = 0 input[8][4:][:] = 0 - rnn1 = LSTM(input_size, - hidden_size, - num_layers=self.num_layers, - time_major=True, - direction=direction, - dropout=self.dropout, - dtype=self.dtype) + rnn1 = LSTM( + input_size, + hidden_size, + num_layers=self.num_layers, + time_major=True, + direction=direction, + dropout=self.dropout, + dtype=self.dtype, + ) flat_w = get_params_for_net(rnn1) - output, (last_hidden, - last_cell) = rnn1(input, sequence_length=self.sequence_length) + output, (last_hidden, last_cell) = rnn1( + input, sequence_length=self.sequence_length + ) if core.is_compiled_with_rocm(): @@ -91,17 +95,19 @@ def rocm_rnn_get_place(): self._get_places = rocm_rnn_get_place - init_h = np.zeros((self.num_layers * self.direction_num, batch_size, - hidden_size)).astype(self.dtype) - init_c = np.zeros((self.num_layers * self.direction_num, batch_size, - hidden_size)).astype(self.dtype) + init_h = np.zeros( + (self.num_layers * self.direction_num, batch_size, hidden_size) + ).astype(self.dtype) + init_c = np.zeros( + (self.num_layers * self.direction_num, batch_size, hidden_size) + ).astype(self.dtype) state_out = np.ndarray((300)).astype("uint8") self.inputs = { 'Input': input, 'WeightList': flat_w, 'PreState': [('init_h', init_h), ('init_c', init_c)], - 'SequenceLength': self.sequence_length + 'SequenceLength': self.sequence_length, } if self.sequence_length is None: self.inputs = { @@ -116,13 +122,13 @@ def rocm_rnn_get_place(): 'hidden_size': hidden_size, 'num_layers': self.num_layers, 'mode': self.mode, - 'is_test': self.is_test + 'is_test': self.is_test, } self.outputs = { 'Out': output, "State": [('last_hidden', last_hidden), ('last_cell', last_cell)], 'Reserve': np.ndarray((400)).astype("uint8"), - 'DropoutState': state_out + 'DropoutState': state_out, } def test_output(self): @@ -136,56 +142,56 @@ def test_grad(self): var_name_list = self.get_weight_names() grad_check_list = ['Input', 'init_h', 'init_c'] grad_check_list.extend(var_name_list) - self.check_grad(set(grad_check_list), - ['Out', 'last_hidden', 'last_cell']) + self.check_grad( + set(grad_check_list), ['Out', 'last_hidden', 'last_cell'] + ) def test_grad_only_input(self): if not self.is_test: var_name_list = self.get_weight_names() grad_check_list = ['Input'] grad_check_list.extend(var_name_list) - self.check_grad(set(grad_check_list), - ['Out', 'last_hidden', 'last_cell']) + self.check_grad( + set(grad_check_list), ['Out', 'last_hidden', 'last_cell'] + ) def test_grad_only_h(self): if not self.is_test: var_name_list = self.get_weight_names() grad_check_list = ['init_h'] grad_check_list.extend(var_name_list) - self.check_grad(set(grad_check_list), - ['Out', 'last_hidden', 'last_cell']) + self.check_grad( + set(grad_check_list), ['Out', 'last_hidden', 'last_cell'] + ) def test_grad_only_c(self): if not self.is_test: var_name_list = self.get_weight_names() grad_check_list = ['init_c'] grad_check_list.extend(var_name_list) - self.check_grad(set(grad_check_list), - ['Out', 'last_hidden', 'last_cell']) + self.check_grad( + set(grad_check_list), ['Out', 'last_hidden', 'last_cell'] + ) class TestRNNOp1(TestRNNOp): - def set_attrs(self): self.sequence_length = None class TestRNNOp2(TestRNNOp): - def set_attrs(self): self.sequence_length = None self.is_bidirec = True class TestRNNOp3(TestRNNOp): - def set_attrs(self): self.is_test = True self.sequence_length = None class TestRNNOp4(TestRNNOp): - def set_attrs(self): self.is_test = True self.sequence_length = None @@ -193,20 +199,17 @@ def set_attrs(self): class TestRNNOp5(TestRNNOp): - def set_attrs(self): self.num_layers = 2 class TestRNNOp6(TestRNNOp): - def set_attrs(self): self.num_layers = 2 self.is_bidirec = True class TestRNNOp7(TestRNNOp): - def set_attrs(self): self.num_layers = 2 self.is_bidirec = True @@ -214,7 +217,6 @@ def set_attrs(self): class TestRNNOp8(TestRNNOp): - def set_attrs(self): self.num_layers = 2 self.is_bidirec = True @@ -222,7 +224,6 @@ def set_attrs(self): class TestRNNOp9(TestRNNOp): - def set_attrs(self): self.num_layers = 3 diff --git a/python/paddle/fluid/tests/unittests/test_roi_align_op.py b/python/paddle/fluid/tests/unittests/test_roi_align_op.py index d9111adceb66b..c26b6691f66f9 100644 --- a/python/paddle/fluid/tests/unittests/test_roi_align_op.py +++ b/python/paddle/fluid/tests/unittests/test_roi_align_op.py @@ -20,7 +20,6 @@ class TestROIAlignOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() @@ -29,7 +28,7 @@ def set_data(self): self.inputs = { 'X': self.x, 'ROIs': (self.rois[:, 1:5], self.rois_lod), - 'RoisNum': self.boxes_num + 'RoisNum': self.boxes_num, } self.attrs = { 'spatial_scale': self.spatial_scale, @@ -58,25 +57,45 @@ def init_test_case(self): self.x = np.random.random(self.x_dim).astype('float64') - def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, roi_bin_grid_w, - bin_size_h, bin_size_w): + def pre_calc( + self, + x_i, + roi_xmin, + roi_ymin, + roi_bin_grid_h, + roi_bin_grid_w, + bin_size_h, + bin_size_w, + ): count = roi_bin_grid_h * roi_bin_grid_w bilinear_pos = np.zeros( [self.channels, self.pooled_height, self.pooled_width, count, 4], - np.float64) - bilinear_w = np.zeros([self.pooled_height, self.pooled_width, count, 4], - np.float64) + np.float64, + ) + bilinear_w = np.zeros( + [self.pooled_height, self.pooled_width, count, 4], np.float64 + ) for ph in range(self.pooled_width): for pw in range(self.pooled_height): c = 0 for iy in range(roi_bin_grid_h): - y = roi_ymin + ph * bin_size_h + (iy + 0.5) * \ - bin_size_h / roi_bin_grid_h + y = ( + roi_ymin + + ph * bin_size_h + + (iy + 0.5) * bin_size_h / roi_bin_grid_h + ) for ix in range(roi_bin_grid_w): - x = roi_xmin + pw * bin_size_w + (ix + 0.5) * \ - bin_size_w / roi_bin_grid_w - if y < -1.0 or y > self.height or \ - x < -1.0 or x > self.width: + x = ( + roi_xmin + + pw * bin_size_w + + (ix + 0.5) * bin_size_w / roi_bin_grid_w + ) + if ( + y < -1.0 + or y > self.height + or x < -1.0 + or x > self.width + ): continue if y <= 0: y = 0 @@ -97,14 +116,18 @@ def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, roi_bin_grid_w, hy = 1 - ly hx = 1 - lx for ch in range(self.channels): - bilinear_pos[ch, ph, pw, c, 0] = x_i[ch, y_low, - x_low] - bilinear_pos[ch, ph, pw, c, 1] = x_i[ch, y_low, - x_high] - bilinear_pos[ch, ph, pw, c, 2] = x_i[ch, y_high, - x_low] - bilinear_pos[ch, ph, pw, c, 3] = x_i[ch, y_high, - x_high] + bilinear_pos[ch, ph, pw, c, 0] = x_i[ + ch, y_low, x_low + ] + bilinear_pos[ch, ph, pw, c, 1] = x_i[ + ch, y_low, x_high + ] + bilinear_pos[ch, ph, pw, c, 2] = x_i[ + ch, y_high, x_low + ] + bilinear_pos[ch, ph, pw, c, 3] = x_i[ + ch, y_high, x_high + ] bilinear_w[ph, pw, c, 0] = hy * hx bilinear_w[ph, pw, c, 1] = hy * lx bilinear_w[ph, pw, c, 2] = ly * hx @@ -114,10 +137,15 @@ def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, roi_bin_grid_w, def calc_roi_align(self): self.out_data = np.zeros( - (self.rois_num, self.channels, self.pooled_height, - self.pooled_width)).astype('float64') - - offset = 0.5 if self.aligned else 0. + ( + self.rois_num, + self.channels, + self.pooled_height, + self.pooled_width, + ) + ).astype('float64') + + offset = 0.5 if self.aligned else 0.0 for i in range(self.rois_num): roi = self.rois[i] roi_batch_id = int(roi[0]) @@ -135,16 +163,27 @@ def calc_roi_align(self): bin_size_h = float(roi_height) / float(self.pooled_height) bin_size_w = float(roi_width) / float(self.pooled_width) - roi_bin_grid_h = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_height / self.pooled_height) - roi_bin_grid_w = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_width / self.pooled_width) + roi_bin_grid_h = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_height / self.pooled_height) + ) + roi_bin_grid_w = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_width / self.pooled_width) + ) count = max(int(roi_bin_grid_h * roi_bin_grid_w), 1) pre_size = count * self.pooled_width * self.pooled_height - bilinear_pos, bilinear_w = self.pre_calc(x_i, roi_xmin, roi_ymin, - int(roi_bin_grid_h), - int(roi_bin_grid_w), - bin_size_h, bin_size_w) + bilinear_pos, bilinear_w = self.pre_calc( + x_i, + roi_xmin, + roi_ymin, + int(roi_bin_grid_h), + int(roi_bin_grid_w), + bin_size_h, + bin_size_w, + ) for ch in range(self.channels): align_per_bin = (bilinear_pos[ch] * bilinear_w).sum(axis=-1) output_val = align_per_bin.mean(axis=-1) @@ -157,27 +196,38 @@ def make_rois(self): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x1 = np.random.random_integers( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.random_integers( - 0, self.height // self.spatial_scale - self.pooled_height) + 0, self.height // self.spatial_scale - self.pooled_height + ) - x2 = np.random.random_integers(x1 + self.pooled_width, - self.width // self.spatial_scale) + x2 = np.random.random_integers( + x1 + self.pooled_width, self.width // self.spatial_scale + ) y2 = np.random.random_integers( - y1 + self.pooled_height, self.height // self.spatial_scale) + y1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) self.rois_num = len(rois) self.rois = np.array(rois).astype("float64") - self.boxes_num = np.array([bno + 1 for bno in range(self.batch_size) - ]).astype('int32') + self.boxes_num = np.array( + [bno + 1 for bno in range(self.batch_size)] + ).astype('int32') def setUp(self): self.op_type = "roi_align" self.python_api = lambda x, boxes, boxes_num, pooled_height, pooled_width, spatial_scale, sampling_ratio, aligned: paddle.vision.ops.roi_align( - x, boxes, boxes_num, (pooled_height, pooled_width), spatial_scale, - sampling_ratio, aligned) + x, + boxes, + boxes_num, + (pooled_height, pooled_width), + spatial_scale, + sampling_ratio, + aligned, + ) self.set_data() def test_check_output(self): @@ -188,7 +238,6 @@ def test_check_grad(self): class TestROIAlignInLodOp(TestROIAlignOp): - def set_data(self): self.init_test_case() self.make_rois() @@ -199,7 +248,7 @@ def set_data(self): self.inputs = { 'X': self.x, 'ROIs': (self.rois[:, 1:5], self.rois_lod), - 'RoisNum': np.asarray(seq_len).astype('int32') + 'RoisNum': np.asarray(seq_len).astype('int32'), } self.attrs = { @@ -207,14 +256,13 @@ def set_data(self): 'pooled_height': self.pooled_height, 'pooled_width': self.pooled_width, 'sampling_ratio': self.sampling_ratio, - 'aligned': self.aligned + 'aligned': self.aligned, } self.outputs = {'Out': self.out_data} class TestROIAlignOpWithAligned(TestROIAlignOp): - def init_test_case(self): self.batch_size = 3 self.channels = 3 diff --git a/python/paddle/fluid/tests/unittests/test_roi_perspective_transform_op.py b/python/paddle/fluid/tests/unittests/test_roi_perspective_transform_op.py index ff6805590439d..d03bdcd59202a 100644 --- a/python/paddle/fluid/tests/unittests/test_roi_perspective_transform_op.py +++ b/python/paddle/fluid/tests/unittests/test_roi_perspective_transform_op.py @@ -40,13 +40,20 @@ def in_quad(x, y, roi_x, roi_y): xe = roi_x[(i + 1) % 4] ye = roi_y[(i + 1) % 4] if abs(ys - ye) < 1e-4: - if abs(y - ys) < 1e-4 and abs(y - ye) < 1e-4 and gt_e( - x, min(xs, xe)) and lt_e(x, max(xs, xe)): + if ( + abs(y - ys) < 1e-4 + and abs(y - ye) < 1e-4 + and gt_e(x, min(xs, xe)) + and lt_e(x, max(xs, xe)) + ): return True else: intersec_x = (y - ys) * (xe - xs) / (ye - ys) + xs - if abs(intersec_x - x) < 1e-4 and gt_e(y, min(ys, ye)) and lt_e( - y, max(ys, ye)): + if ( + abs(intersec_x - x) < 1e-4 + and gt_e(y, min(ys, ye)) + and lt_e(y, max(ys, ye)) + ): return True n_cross = 0 for i in range(4): @@ -63,7 +70,7 @@ def in_quad(x, y, roi_x, roi_y): return True if gt(intersec_x, x): n_cross += 1 - return (n_cross % 2 == 1) + return n_cross % 2 == 1 def get_transform_matrix(transformed_width, transformed_height, roi_x, roi_y): @@ -84,8 +91,9 @@ def get_transform_matrix(transformed_width, transformed_height, roi_x, roi_y): estimated_width = (len1 + len3) / 2.0 normalized_height = max(2, transformed_height) - normalized_width = round(estimated_width * - (normalized_height - 1) / estimated_height) + 1 + normalized_width = ( + round(estimated_width * (normalized_height - 1) / estimated_height) + 1 + ) normalized_width = max(2, min(normalized_width, transformed_width)) dx1 = x1 - x2 @@ -95,22 +103,32 @@ def get_transform_matrix(transformed_width, transformed_height, roi_x, roi_y): dy2 = y3 - y2 dy3 = y0 - y1 + y2 - y3 matrix = np.zeros([9]) - matrix[6] = (dx3 * dy2 - dx2 * dy3) / (dx1 * dy2 - dx2 * dy1 + - 1e-5) / (normalized_width - 1) - matrix[7] = (dx1 * dy3 - dx3 * dy1) / (dx1 * dy2 - dx2 * dy1 + - 1e-5) / (normalized_height - 1) + matrix[6] = ( + (dx3 * dy2 - dx2 * dy3) + / (dx1 * dy2 - dx2 * dy1 + 1e-5) + / (normalized_width - 1) + ) + matrix[7] = ( + (dx1 * dy3 - dx3 * dy1) + / (dx1 * dy2 - dx2 * dy1 + 1e-5) + / (normalized_height - 1) + ) matrix[8] = 1 - matrix[3] = (y1 - y0 + matrix[6] * - (normalized_width - 1) * y1) / (normalized_width - 1) - matrix[4] = (y3 - y0 + matrix[7] * - (normalized_height - 1) * y3) / (normalized_height - 1) + matrix[3] = (y1 - y0 + matrix[6] * (normalized_width - 1) * y1) / ( + normalized_width - 1 + ) + matrix[4] = (y3 - y0 + matrix[7] * (normalized_height - 1) * y3) / ( + normalized_height - 1 + ) matrix[5] = y0 - matrix[0] = (x1 - x0 + matrix[6] * - (normalized_width - 1) * x1) / (normalized_width - 1) - matrix[1] = (x3 - x0 + matrix[7] * - (normalized_height - 1) * x3) / (normalized_height - 1) + matrix[0] = (x1 - x0 + matrix[6] * (normalized_width - 1) * x1) / ( + normalized_width - 1 + ) + matrix[1] = (x3 - x0 + matrix[7] * (normalized_height - 1) * x3) / ( + normalized_height - 1 + ) matrix[2] = x0 return matrix @@ -131,8 +149,12 @@ def bilinear_interpolate(in_data, in_n, in_c, in_w, in_h): height = in_data.shape[2] width = in_data.shape[3] - if gt_e(-0.5, in_w) or gt_e(in_w, width - 0.5) or gt_e(-0.5, in_h) or gt_e( - in_h, height - 0.5): + if ( + gt_e(-0.5, in_w) + or gt_e(in_w, width - 0.5) + or gt_e(-0.5, in_h) + or gt_e(in_h, height - 0.5) + ): return 0.0 if gt_e(0, in_w): @@ -180,8 +202,14 @@ def lod_convert(lod): return ret -def roi_transform(in_data, rois, rois_lod, transformed_height, - transformed_width, spatial_scale): +def roi_transform( + in_data, + rois, + rois_lod, + transformed_height, + transformed_width, + spatial_scale, +): channels = in_data.shape[1] in_height = in_data.shape[2] in_width = in_data.shape[3] @@ -194,8 +222,9 @@ def roi_transform(in_data, rois, rois_lod, transformed_height, roi2image[j] = i out = np.zeros([rois_num, channels, transformed_height, transformed_width]) - mask = np.zeros([rois_num, 1, transformed_height, - transformed_width]).astype('int') + mask = np.zeros( + [rois_num, 1, transformed_height, transformed_width] + ).astype('int') matrix = np.zeros([rois_num, 9], dtype=in_data.dtype) for n in range(rois_num): roi_x = [] @@ -204,20 +233,26 @@ def roi_transform(in_data, rois, rois_lod, transformed_height, roi_x.append(rois[n][2 * k] * spatial_scale) roi_y.append(rois[n][2 * k + 1] * spatial_scale) image_id = roi2image[n] - transform_matrix = get_transform_matrix(transformed_width, - transformed_height, roi_x, - roi_y) + transform_matrix = get_transform_matrix( + transformed_width, transformed_height, roi_x, roi_y + ) matrix[n] = transform_matrix for c in range(channels): for out_h in range(transformed_height): for out_w in range(transformed_width): - in_w, in_h = get_source_coords(transform_matrix, out_w, - out_h) - if in_quad(in_w, in_h, roi_x, roi_y) and gt( - in_w, -0.5) and gt(in_width - 0.5, in_w) and gt( - in_h, -0.5) and gt(in_height - 0.5, in_h): + in_w, in_h = get_source_coords( + transform_matrix, out_w, out_h + ) + if ( + in_quad(in_w, in_h, roi_x, roi_y) + and gt(in_w, -0.5) + and gt(in_width - 0.5, in_w) + and gt(in_h, -0.5) + and gt(in_height - 0.5, in_h) + ): out[n][c][out_h][out_w] = bilinear_interpolate( - in_data, image_id, c, in_w, in_h) + in_data, image_id, c, in_w, in_h + ) mask[n][0][out_h][out_w] = 1 else: out[n][c][out_h][out_w] = 0.0 @@ -226,7 +261,6 @@ def roi_transform(in_data, rois, rois_lod, transformed_height, class TestROIPoolOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() @@ -236,17 +270,20 @@ def set_data(self): self.attrs = { 'spatial_scale': self.spatial_scale, 'transformed_height': self.transformed_height, - 'transformed_width': self.transformed_width + 'transformed_width': self.transformed_width, } - out, mask, transform_matrix = roi_transform(self.x, self.rois, - self.rois_lod, - self.transformed_height, - self.transformed_width, - self.spatial_scale) + out, mask, transform_matrix = roi_transform( + self.x, + self.rois, + self.rois_lod, + self.transformed_height, + self.transformed_width, + self.spatial_scale, + ) self.outputs = { 'Out': out, 'Mask': mask, - 'TransformMatrix': transform_matrix + 'TransformMatrix': transform_matrix, } def init_test_case(self): @@ -271,28 +308,38 @@ def make_rois(self): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x1 = np.random.randint( - 0, - self.width // self.spatial_scale - self.transformed_width) + 0, self.width // self.spatial_scale - self.transformed_width + ) y1 = np.random.randint( 0, - self.height // self.spatial_scale - self.transformed_height) + self.height // self.spatial_scale - self.transformed_height, + ) - x2 = np.random.randint(x1 + self.transformed_width, - self.width // self.spatial_scale) + x2 = np.random.randint( + x1 + self.transformed_width, + self.width // self.spatial_scale, + ) y2 = np.random.randint( 0, - self.height // self.spatial_scale - self.transformed_height) - - x3 = np.random.randint(x1 + self.transformed_width, - self.width // self.spatial_scale) - y3 = np.random.randint(y1 + self.transformed_height, - self.height // self.spatial_scale) + self.height // self.spatial_scale - self.transformed_height, + ) + + x3 = np.random.randint( + x1 + self.transformed_width, + self.width // self.spatial_scale, + ) + y3 = np.random.randint( + y1 + self.transformed_height, + self.height // self.spatial_scale, + ) x4 = np.random.randint( - 0, - self.width // self.spatial_scale - self.transformed_width) - y4 = np.random.randint(y1 + self.transformed_height, - self.height // self.spatial_scale) + 0, self.width // self.spatial_scale - self.transformed_width + ) + y4 = np.random.randint( + y1 + self.transformed_height, + self.height // self.spatial_scale, + ) roi = [x1, y1, x2, y2, x3, y3, x4, y4] rois.append(roi) @@ -308,52 +355,70 @@ def test_check_output(self): def test_check_grad(self): self.outputs['Out2InIdx'] = np.zeros( - [np.product(self.outputs['Out'].shape), 4]).astype("int32") + [np.product(self.outputs['Out'].shape), 4] + ).astype("int32") self.outputs['Out2InWeights'] = np.zeros( - [np.product(self.outputs['Out'].shape), 4]).astype("float32") + [np.product(self.outputs['Out'].shape), 4] + ).astype("float32") self.check_grad(['X'], 'Out') def test_errors(self): x = fluid.data(name='x', shape=[100, 256, 28, 28], dtype='float32') - rois = fluid.data(name='rois', - shape=[None, 8], - lod_level=1, - dtype='float32') - - x_int = fluid.data(name='x_int', - shape=[100, 256, 28, 28], - dtype='int32') - rois_int = fluid.data(name='rois_int', - shape=[None, 8], - lod_level=1, - dtype='int32') + rois = fluid.data( + name='rois', shape=[None, 8], lod_level=1, dtype='float32' + ) + + x_int = fluid.data( + name='x_int', shape=[100, 256, 28, 28], dtype='int32' + ) + rois_int = fluid.data( + name='rois_int', shape=[None, 8], lod_level=1, dtype='int32' + ) x_tmp = [1, 2] rois_tmp = [1, 2] # type of intput and rois must be variable - self.assertRaises(TypeError, fluid.layers.roi_perspective_transform, - x_tmp, rois, 7, 7) - self.assertRaises(TypeError, fluid.layers.roi_perspective_transform, x, - rois_tmp, 7, 7) + self.assertRaises( + TypeError, fluid.layers.roi_perspective_transform, x_tmp, rois, 7, 7 + ) + self.assertRaises( + TypeError, fluid.layers.roi_perspective_transform, x, rois_tmp, 7, 7 + ) # dtype of intput and rois must be float32 - self.assertRaises(TypeError, fluid.layers.roi_perspective_transform, - x_int, rois, 7, 7) - self.assertRaises(TypeError, fluid.layers.roi_perspective_transform, x, - rois_int, 7, 7) + self.assertRaises( + TypeError, fluid.layers.roi_perspective_transform, x_int, rois, 7, 7 + ) + self.assertRaises( + TypeError, fluid.layers.roi_perspective_transform, x, rois_int, 7, 7 + ) height = 7.5 width = 7.5 # type of transformed_height and transformed_width must be int - self.assertRaises(TypeError, fluid.layers.roi_perspective_transform, x, - rois, height, 7) - self.assertRaises(TypeError, fluid.layers.roi_perspective_transform, x, - rois, 7, width) + self.assertRaises( + TypeError, + fluid.layers.roi_perspective_transform, + x, + rois, + height, + 7, + ) + self.assertRaises( + TypeError, fluid.layers.roi_perspective_transform, x, rois, 7, width + ) scale = int(2) # type of spatial_scale must be float - self.assertRaises(TypeError, fluid.layers.roi_perspective_transform, x, - rois, 7, 7, scale) + self.assertRaises( + TypeError, + fluid.layers.roi_perspective_transform, + x, + rois, + 7, + 7, + scale, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_roi_pool_op.py b/python/paddle/fluid/tests/unittests/test_roi_pool_op.py index 16ff2f57fb8f0..2fcd3eda287f7 100644 --- a/python/paddle/fluid/tests/unittests/test_roi_pool_op.py +++ b/python/paddle/fluid/tests/unittests/test_roi_pool_op.py @@ -33,7 +33,6 @@ def _round(x): class TestROIPoolOp(OpTest): - def set_data(self): self.init_test_case() self.make_rois() @@ -42,13 +41,13 @@ def set_data(self): self.inputs = { 'X': self.x, 'ROIs': (self.rois[:, 1:5], self.rois_lod), - 'RoisNum': self.boxes_num + 'RoisNum': self.boxes_num, } self.attrs = { 'spatial_scale': self.spatial_scale, 'pooled_height': self.pooled_height, - 'pooled_width': self.pooled_width + 'pooled_width': self.pooled_width, } self.outputs = {'Out': self.outs, 'Argmax': self.argmaxes} @@ -69,10 +68,22 @@ def init_test_case(self): self.x = np.random.random(self.x_dim).astype('float64') def calc_roi_pool(self): - out_data = np.zeros((self.rois_num, self.channels, self.pooled_height, - self.pooled_width)) - argmax_data = np.zeros((self.rois_num, self.channels, - self.pooled_height, self.pooled_width)) + out_data = np.zeros( + ( + self.rois_num, + self.channels, + self.pooled_height, + self.pooled_width, + ) + ) + argmax_data = np.zeros( + ( + self.rois_num, + self.channels, + self.pooled_height, + self.pooled_width, + ) + ) for i in range(self.rois_num): roi = self.rois[i] @@ -115,8 +126,9 @@ def calc_roi_pool(self): for w in range(wstart, wend): if x_i[c, h, w] > out_data[i, c, ph, pw]: out_data[i, c, ph, pw] = x_i[c, h, w] - argmax_data[i, c, ph, - pw] = h * self.width + w + argmax_data[i, c, ph, pw] = ( + h * self.width + w + ) self.outs = out_data.astype('float64') self.argmaxes = argmax_data.astype('int64') @@ -128,26 +140,32 @@ def make_rois(self): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x1 = np.random.randint( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.randint( - 0, self.height // self.spatial_scale - self.pooled_height) + 0, self.height // self.spatial_scale - self.pooled_height + ) - x2 = np.random.randint(x1 + self.pooled_width, - self.width // self.spatial_scale) - y2 = np.random.randint(y1 + self.pooled_height, - self.height // self.spatial_scale) + x2 = np.random.randint( + x1 + self.pooled_width, self.width // self.spatial_scale + ) + y2 = np.random.randint( + y1 + self.pooled_height, self.height // self.spatial_scale + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) self.rois_num = len(rois) self.rois = np.array(rois).astype("float64") - self.boxes_num = np.array([bno + 1 for bno in range(self.batch_size) - ]).astype('int32') + self.boxes_num = np.array( + [bno + 1 for bno in range(self.batch_size)] + ).astype('int32') def setUp(self): self.op_type = "roi_pool" self.python_api = lambda x, boxes, boxes_num, pooled_height, pooled_width, spatial_scale: paddle.vision.ops.roi_pool( - x, boxes, boxes_num, (pooled_height, pooled_width), spatial_scale) + x, boxes, boxes_num, (pooled_height, pooled_width), spatial_scale + ) self.python_out_sig = ["Out"] self.set_data() @@ -159,27 +177,27 @@ def test_check_grad(self): class BadInputTestRoiPool(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): def test_bad_x(): - x = fluid.layers.data(name='data1', - shape=[2, 1, 4, 4], - dtype='int64') - label = fluid.layers.data(name='label', - shape=[2, 4], - dtype='float32', - lod_level=1) + x = fluid.layers.data( + name='data1', shape=[2, 1, 4, 4], dtype='int64' + ) + label = fluid.layers.data( + name='label', shape=[2, 4], dtype='float32', lod_level=1 + ) output = fluid.layers.roi_pool(x, label, 1, 1, 1.0) self.assertRaises(TypeError, test_bad_x) def test_bad_y(): - x = fluid.layers.data(name='data2', - shape=[2, 1, 4, 4], - dtype='float32', - append_batch_size=False) + x = fluid.layers.data( + name='data2', + shape=[2, 1, 4, 4], + dtype='float32', + append_batch_size=False, + ) label = [[1, 2, 3, 4], [2, 3, 4, 5]] output = fluid.layers.roi_pool(x, label, 1, 1, 1.0) @@ -187,7 +205,6 @@ def test_bad_y(): class TestROIPoolInLodOp(TestROIPoolOp): - def set_data(self): self.init_test_case() self.make_rois() @@ -198,13 +215,13 @@ def set_data(self): self.inputs = { 'X': self.x, 'ROIs': (self.rois[:, 1:5], self.rois_lod), - 'RoisNum': np.asarray(seq_len).astype('int32') + 'RoisNum': np.asarray(seq_len).astype('int32'), } self.attrs = { 'spatial_scale': self.spatial_scale, 'pooled_height': self.pooled_height, - 'pooled_width': self.pooled_width + 'pooled_width': self.pooled_width, } self.outputs = {'Out': self.outs, 'Argmax': self.argmaxes} diff --git a/python/paddle/fluid/tests/unittests/test_roll_op.py b/python/paddle/fluid/tests/unittests/test_roll_op.py index 43e56efd6429b..bf11b0d86f668 100644 --- a/python/paddle/fluid/tests/unittests/test_roll_op.py +++ b/python/paddle/fluid/tests/unittests/test_roll_op.py @@ -21,7 +21,6 @@ class TestRollOp(OpTest): - def setUp(self): self.python_api = paddle.roll self.op_type = "roll" @@ -29,8 +28,9 @@ def setUp(self): self.inputs = {'X': np.random.random(self.x_shape).astype(self.dtype)} self.attrs = {'shifts': self.shifts, 'axis': self.axis} self.outputs = { - 'Out': - np.roll(self.inputs['X'], self.attrs['shifts'], self.attrs['axis']) + 'Out': np.roll( + self.inputs['X'], self.attrs['shifts'], self.attrs['axis'] + ) } def init_dtype_type(self): @@ -47,7 +47,6 @@ def test_check_grad_normal(self): class TestRollOpCase2(TestRollOp): - def init_dtype_type(self): self.dtype = np.float32 self.x_shape = (100, 10, 5) @@ -56,10 +55,10 @@ def init_dtype_type(self): class TestRollAPI(unittest.TestCase): - def input_data(self): - self.data_x = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], - [7.0, 8.0, 9.0]]) + self.data_x = np.array( + [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]] + ) def test_roll_op_api(self): self.input_data() @@ -70,11 +69,12 @@ def test_roll_op_api(self): x = fluid.layers.data(name='x', shape=[-1, 3]) z = paddle.roll(x, shifts=1) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': self.data_x}, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[9.0, 1.0, 2.0], [3.0, 4.0, 5.0], - [6.0, 7.0, 8.0]]) + (res,) = exe.run( + feed={'x': self.data_x}, fetch_list=[z.name], return_numpy=False + ) + expect_out = np.array( + [[9.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) # case 2: @@ -82,11 +82,12 @@ def test_roll_op_api(self): x = fluid.layers.data(name='x', shape=[-1, 3]) z = paddle.roll(x, shifts=1, axis=0) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': self.data_x}, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[7.0, 8.0, 9.0], [1.0, 2.0, 3.0], - [4.0, 5.0, 6.0]]) + (res,) = exe.run( + feed={'x': self.data_x}, fetch_list=[z.name], return_numpy=False + ) + expect_out = np.array( + [[7.0, 8.0, 9.0], [1.0, 2.0, 3.0], [4.0, 5.0, 6.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) def test_dygraph_api(self): @@ -96,8 +97,9 @@ def test_dygraph_api(self): x = fluid.dygraph.to_variable(self.data_x) z = paddle.roll(x, shifts=1) np_z = z.numpy() - expect_out = np.array([[9.0, 1.0, 2.0], [3.0, 4.0, 5.0], - [6.0, 7.0, 8.0]]) + expect_out = np.array( + [[9.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]] + ) np.testing.assert_allclose(expect_out, np_z, rtol=1e-05) # case 2: @@ -105,8 +107,9 @@ def test_dygraph_api(self): x = fluid.dygraph.to_variable(self.data_x) z = paddle.roll(x, shifts=1, axis=0) np_z = z.numpy() - expect_out = np.array([[7.0, 8.0, 9.0], [1.0, 2.0, 3.0], - [4.0, 5.0, 6.0]]) + expect_out = np.array( + [[7.0, 8.0, 9.0], [1.0, 2.0, 3.0], [4.0, 5.0, 6.0]] + ) np.testing.assert_allclose(expect_out, np_z, rtol=1e-05) def test_roll_op_false(self): @@ -117,9 +120,11 @@ def test_axis_out_range(): x = fluid.layers.data(name='x', shape=[-1, 3]) z = paddle.roll(x, shifts=1, axis=10) exe = fluid.Executor(fluid.CPUPlace()) - res, = exe.run(feed={'x': self.data_x}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': self.data_x}, + fetch_list=[z.name], + return_numpy=False, + ) self.assertRaises(ValueError, test_axis_out_range) diff --git a/python/paddle/fluid/tests/unittests/test_rot90_op.py b/python/paddle/fluid/tests/unittests/test_rot90_op.py index 6a1cbcaccda73..1cf3a21e9d021 100644 --- a/python/paddle/fluid/tests/unittests/test_rot90_op.py +++ b/python/paddle/fluid/tests/unittests/test_rot90_op.py @@ -37,15 +37,17 @@ def test_static_graph(self): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[4, 1], [5, 2], [6, 3]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_static_k_0(self): paddle.enable_static() @@ -62,15 +64,17 @@ def test_static_k_0(self): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_static_k_2(self): paddle.enable_static() @@ -87,15 +91,17 @@ def test_static_k_2(self): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[6, 5, 4], [3, 2, 1]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_static_k_3(self): paddle.enable_static() @@ -112,15 +118,17 @@ def test_static_k_3(self): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[4, 1], [5, 2], [6, 3]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_static_neg_k_1(self): paddle.enable_static() @@ -137,15 +145,17 @@ def test_static_neg_k_1(self): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[4, 1], [5, 2], [6, 3]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_static_neg_k_2(self): paddle.enable_static() @@ -162,15 +172,17 @@ def test_static_neg_k_2(self): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[6, 5, 4], [3, 2, 1]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_static_neg_k_3(self): paddle.enable_static() @@ -187,15 +199,17 @@ def test_static_neg_k_3(self): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[3, 6], [2, 5], [1, 4]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_static_neg_k_4(self): paddle.enable_static() @@ -212,15 +226,17 @@ def test_static_neg_k_4(self): exe.run(startup_program) img = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - res = exe.run(train_program, - feed={'input': img}, - fetch_list=[output]) + res = exe.run( + train_program, feed={'input': img}, fetch_list=[output] + ) out_np = np.array(res[0]) out_ref = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.float32) - self.assertTrue((out_np == out_ref).all(), - msg='rot90 output is wrong, out =' + str(out_np)) + self.assertTrue( + (out_np == out_ref).all(), + msg='rot90 output is wrong, out =' + str(out_np), + ) def test_error_api(self): paddle.enable_static() @@ -269,7 +285,8 @@ def test_dygraph(self): self.assertTrue( (ret.numpy() == out_ref).all(), - msg='rot90 output is wrong, out =' + str(ret.numpy())) + msg='rot90 output is wrong, out =' + str(ret.numpy()), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_row_conv_op.py b/python/paddle/fluid/tests/unittests/test_row_conv_op.py index 6350b25724744..6f4df64ce503e 100644 --- a/python/paddle/fluid/tests/unittests/test_row_conv_op.py +++ b/python/paddle/fluid/tests/unittests/test_row_conv_op.py @@ -43,7 +43,6 @@ def row_conv_forward(x, lod, wt): class TestRowConvOp1(OpTest): - def setUp(self): self.op_type = "row_conv" @@ -66,20 +65,17 @@ def test_check_grad_normal(self): self.check_grad(['X', 'Filter'], 'Out', check_dygraph=False) def test_check_grad_ignore_x(self): - self.check_grad(['Filter'], - 'Out', - no_grad_set=set('X'), - check_dygraph=False) + self.check_grad( + ['Filter'], 'Out', no_grad_set=set('X'), check_dygraph=False + ) def test_check_grad_ignore_wt(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Filter'), - check_dygraph=False) + self.check_grad( + ['X'], 'Out', no_grad_set=set('Filter'), check_dygraph=False + ) class TestRowConvOp2(OpTest): - def setUp(self): self.op_type = "row_conv" @@ -98,28 +94,31 @@ def setUp(self): def test_check_output(self): self.check_output(check_dygraph=False) - #max_relative_error is increased from 0.05 to 0.06 as for higher - #dimensional input, the dX on CPU for some values has max_rel_error - #slightly more than 0.05 + # max_relative_error is increased from 0.05 to 0.06 as for higher + # dimensional input, the dX on CPU for some values has max_rel_error + # slightly more than 0.05 def test_check_grad_normal(self): - self.check_grad(['X', 'Filter'], - 'Out', - max_relative_error=0.06, - check_dygraph=False) + self.check_grad( + ['X', 'Filter'], 'Out', max_relative_error=0.06, check_dygraph=False + ) def test_check_grad_ignore_x(self): - self.check_grad(['Filter'], - 'Out', - max_relative_error=0.06, - no_grad_set=set('X'), - check_dygraph=False) + self.check_grad( + ['Filter'], + 'Out', + max_relative_error=0.06, + no_grad_set=set('X'), + check_dygraph=False, + ) def test_check_grad_ignore_wt(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.06, - no_grad_set=set('Filter'), - check_dygraph=False) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.06, + no_grad_set=set('Filter'), + check_dygraph=False, + ) def row_conv_foward_Tensor(x, wt): @@ -128,8 +127,8 @@ def row_conv_foward_Tensor(x, wt): timesteps = x.shape[1] context_length = wt.shape[0] for i in range(num_sequence): - cur_in = x[i:i + 1, :][0] - cur_out = out[i:i + 1, :][0] + cur_in = x[i : i + 1, :][0] + cur_out = out[i : i + 1, :][0] for j in range(timesteps): for k in range(context_length): if j + k >= timesteps: @@ -139,7 +138,6 @@ def row_conv_foward_Tensor(x, wt): class TestRowOpWithTensorInput(OpTest): - def setUp(self): self.op_type = "row_conv" length = [1, 2, 3] @@ -159,23 +157,20 @@ def test_check_output(self): self.check_output(check_dygraph=False) def test_check_grad_ignore_x(self): - self.check_grad(['Filter'], - 'Out', - no_grad_set=set('X'), - check_dygraph=False) + self.check_grad( + ['Filter'], 'Out', no_grad_set=set('X'), check_dygraph=False + ) def test_check_grad_normal(self): self.check_grad(['X', 'Filter'], 'Out', check_dygraph=False) def test_check_grad_ignore_wt(self): - self.check_grad(['X'], - 'Out', - no_grad_set=set('Filter'), - check_dygraph=False) + self.check_grad( + ['X'], 'Out', no_grad_set=set('Filter'), check_dygraph=False + ) class TestRowConvLayer(unittest.TestCase): - def setUp(self): self.B = 2 self.T = 6 @@ -183,8 +178,9 @@ def setUp(self): self.context_length = 6 self.x = np.random.random((self.B, self.T, self.C)).astype("float32") - self.w = np.random.random( - (self.context_length, self.C)).astype("float32") + self.w = np.random.random((self.context_length, self.C)).astype( + "float32" + ) self.out = row_conv_foward_Tensor(self.x, self.w) def check_identity(self): @@ -196,11 +192,12 @@ def check_identity(self): out = fluid.layers.row_conv( x, self.context_length, - param_attr=fluid.initializer.NumpyArrayInitializer(self.w)) + param_attr=fluid.initializer.NumpyArrayInitializer(self.w), + ) place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(start) - out_np, = exe.run(main, feed={'x': self.x}, fetch_list=[out]) + (out_np,) = exe.run(main, feed={'x': self.x}, fetch_list=[out]) np.testing.assert_allclose(out_np, self.out) diff --git a/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py b/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py index de3e8e211365e..ce37fedaf59ff 100644 --- a/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py +++ b/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py @@ -22,33 +22,37 @@ from test_generate_proposal_labels_op import _bbox_overlaps, _box_to_delta -def rpn_target_assign(anchor_by_gt_overlap, - rpn_batch_size_per_im, - rpn_positive_overlap, - rpn_negative_overlap, - rpn_fg_fraction, - use_random=True): +def rpn_target_assign( + anchor_by_gt_overlap, + rpn_batch_size_per_im, + rpn_positive_overlap, + rpn_negative_overlap, + rpn_fg_fraction, + use_random=True, +): anchor_to_gt_argmax = anchor_by_gt_overlap.argmax(axis=1) anchor_to_gt_max = anchor_by_gt_overlap[ - np.arange(anchor_by_gt_overlap.shape[0]), anchor_to_gt_argmax] + np.arange(anchor_by_gt_overlap.shape[0]), anchor_to_gt_argmax + ] gt_to_anchor_argmax = anchor_by_gt_overlap.argmax(axis=0) gt_to_anchor_max = anchor_by_gt_overlap[ - gt_to_anchor_argmax, - np.arange(anchor_by_gt_overlap.shape[1])] + gt_to_anchor_argmax, np.arange(anchor_by_gt_overlap.shape[1]) + ] anchors_with_max_overlap = np.where( - anchor_by_gt_overlap == gt_to_anchor_max)[0] + anchor_by_gt_overlap == gt_to_anchor_max + )[0] - labels = np.ones((anchor_by_gt_overlap.shape[0], ), dtype=np.int32) * -1 + labels = np.ones((anchor_by_gt_overlap.shape[0],), dtype=np.int32) * -1 labels[anchors_with_max_overlap] = 1 labels[anchor_to_gt_max >= rpn_positive_overlap] = 1 num_fg = int(rpn_fg_fraction * rpn_batch_size_per_im) fg_inds = np.where(labels == 1)[0] if len(fg_inds) > num_fg and use_random: - disable_inds = np.random.choice(fg_inds, - size=(len(fg_inds) - num_fg), - replace=False) + disable_inds = np.random.choice( + fg_inds, size=(len(fg_inds) - num_fg), replace=False + ) else: disable_inds = fg_inds[num_fg:] @@ -87,26 +91,30 @@ def rpn_target_assign(anchor_by_gt_overlap, def get_anchor(n, c, h, w): input_feat = np.random.random((n, c, h, w)).astype('float32') - anchors, _ = anchor_generator_in_python(input_feat=input_feat, - anchor_sizes=[32., 64.], - aspect_ratios=[0.5, 1.0], - variances=[1.0, 1.0, 1.0, 1.0], - stride=[16.0, 16.0], - offset=0.5) + anchors, _ = anchor_generator_in_python( + input_feat=input_feat, + anchor_sizes=[32.0, 64.0], + aspect_ratios=[0.5, 1.0], + variances=[1.0, 1.0, 1.0, 1.0], + stride=[16.0, 16.0], + offset=0.5, + ) return anchors -def rpn_target_assign_in_python(all_anchors, - gt_boxes, - is_crowd, - im_info, - lod, - rpn_straddle_thresh, - rpn_batch_size_per_im, - rpn_positive_overlap, - rpn_negative_overlap, - rpn_fg_fraction, - use_random=True): +def rpn_target_assign_in_python( + all_anchors, + gt_boxes, + is_crowd, + im_info, + lod, + rpn_straddle_thresh, + rpn_batch_size_per_im, + rpn_positive_overlap, + rpn_negative_overlap, + rpn_fg_fraction, + use_random=True, +): anchor_num = all_anchors.shape[0] batch_size = len(lod) - 1 for i in range(batch_size): @@ -119,7 +127,8 @@ def rpn_target_assign_in_python(all_anchors, (all_anchors[:, 0] >= -rpn_straddle_thresh) & (all_anchors[:, 1] >= -rpn_straddle_thresh) & (all_anchors[:, 2] < im_width + rpn_straddle_thresh) - & (all_anchors[:, 3] < im_height + rpn_straddle_thresh))[0] + & (all_anchors[:, 3] < im_height + rpn_straddle_thresh) + )[0] # keep only inside anchors inside_anchors = all_anchors[inds_inside, :] else: @@ -134,19 +143,29 @@ def rpn_target_assign_in_python(all_anchors, gt_boxes_slice = gt_boxes_slice[not_crowd_inds] iou = _bbox_overlaps(inside_anchors, gt_boxes_slice) - loc_inds, score_inds, labels, gt_inds, bbox_inside_weight = \ - rpn_target_assign(iou, rpn_batch_size_per_im, - rpn_positive_overlap, - rpn_negative_overlap, - rpn_fg_fraction, - use_random) + ( + loc_inds, + score_inds, + labels, + gt_inds, + bbox_inside_weight, + ) = rpn_target_assign( + iou, + rpn_batch_size_per_im, + rpn_positive_overlap, + rpn_negative_overlap, + rpn_fg_fraction, + use_random, + ) # unmap to all anchor loc_inds = inds_inside[loc_inds] score_inds = inds_inside[score_inds] sampled_gt = gt_boxes_slice[gt_inds] sampled_anchor = all_anchors[loc_inds] - box_deltas = _box_to_delta(sampled_anchor, sampled_gt, [1., 1., 1., 1.]) + box_deltas = _box_to_delta( + sampled_anchor, sampled_gt, [1.0, 1.0, 1.0, 1.0] + ) if i == 0: loc_indexes = loc_inds @@ -156,31 +175,43 @@ def rpn_target_assign_in_python(all_anchors, bbox_inside_weights = bbox_inside_weight else: loc_indexes = np.concatenate( - [loc_indexes, loc_inds + i * anchor_num]) + [loc_indexes, loc_inds + i * anchor_num] + ) score_indexes = np.concatenate( - [score_indexes, score_inds + i * anchor_num]) + [score_indexes, score_inds + i * anchor_num] + ) tgt_labels = np.concatenate([tgt_labels, labels]) tgt_bboxes = np.vstack([tgt_bboxes, box_deltas]) - bbox_inside_weights = np.vstack([bbox_inside_weights, \ - bbox_inside_weight]) - - return loc_indexes, score_indexes, tgt_bboxes, tgt_labels, bbox_inside_weights - - -def retinanet_target_assign(anchor_by_gt_overlap, gt_labels, positive_overlap, - negative_overlap): + bbox_inside_weights = np.vstack( + [bbox_inside_weights, bbox_inside_weight] + ) + + return ( + loc_indexes, + score_indexes, + tgt_bboxes, + tgt_labels, + bbox_inside_weights, + ) + + +def retinanet_target_assign( + anchor_by_gt_overlap, gt_labels, positive_overlap, negative_overlap +): anchor_to_gt_argmax = anchor_by_gt_overlap.argmax(axis=1) anchor_to_gt_max = anchor_by_gt_overlap[ - np.arange(anchor_by_gt_overlap.shape[0]), anchor_to_gt_argmax] + np.arange(anchor_by_gt_overlap.shape[0]), anchor_to_gt_argmax + ] gt_to_anchor_argmax = anchor_by_gt_overlap.argmax(axis=0) gt_to_anchor_max = anchor_by_gt_overlap[ - gt_to_anchor_argmax, - np.arange(anchor_by_gt_overlap.shape[1])] + gt_to_anchor_argmax, np.arange(anchor_by_gt_overlap.shape[1]) + ] anchors_with_max_overlap = np.where( - anchor_by_gt_overlap == gt_to_anchor_max)[0] + anchor_by_gt_overlap == gt_to_anchor_max + )[0] - labels = np.ones((anchor_by_gt_overlap.shape[0], ), dtype=np.int32) * -1 + labels = np.ones((anchor_by_gt_overlap.shape[0],), dtype=np.int32) * -1 labels[anchors_with_max_overlap] = 1 labels[anchor_to_gt_max >= positive_overlap] = 1 @@ -209,15 +240,22 @@ def retinanet_target_assign(anchor_by_gt_overlap, gt_labels, positive_overlap, gt_inds = anchor_to_gt_argmax[loc_index] label_inds = anchor_to_gt_argmax[score_index_tmp] - labels[0:len(fg_inds)] = np.squeeze(gt_labels[label_inds]) + labels[0 : len(fg_inds)] = np.squeeze(gt_labels[label_inds]) fg_num = len(fg_fake_inds) + len(fg_inds) + 1 assert not np.any(labels == -1), "Wrong labels with -1" return loc_index, score_index, labels, gt_inds, bbox_inside_weight, fg_num -def retinanet_target_assign_in_python(all_anchors, gt_boxes, gt_labels, - is_crowd, im_info, lod, positive_overlap, - negative_overlap): +def retinanet_target_assign_in_python( + all_anchors, + gt_boxes, + gt_labels, + is_crowd, + im_info, + lod, + positive_overlap, + negative_overlap, +): anchor_num = all_anchors.shape[0] batch_size = len(lod) - 1 for i in range(batch_size): @@ -235,16 +273,25 @@ def retinanet_target_assign_in_python(all_anchors, gt_boxes, gt_labels, gt_labels_slice = gt_labels_slice[not_crowd_inds] iou = _bbox_overlaps(inside_anchors, gt_boxes_slice) - loc_inds, score_inds, labels, gt_inds, bbox_inside_weight, fg_num = \ - retinanet_target_assign(iou, gt_labels_slice, - positive_overlap, negative_overlap) + ( + loc_inds, + score_inds, + labels, + gt_inds, + bbox_inside_weight, + fg_num, + ) = retinanet_target_assign( + iou, gt_labels_slice, positive_overlap, negative_overlap + ) # unmap to all anchor loc_inds = inds_inside[loc_inds] score_inds = inds_inside[score_inds] sampled_gt = gt_boxes_slice[gt_inds] sampled_anchor = all_anchors[loc_inds] - box_deltas = _box_to_delta(sampled_anchor, sampled_gt, [1., 1., 1., 1.]) + box_deltas = _box_to_delta( + sampled_anchor, sampled_gt, [1.0, 1.0, 1.0, 1.0] + ) if i == 0: loc_indexes = loc_inds @@ -255,20 +302,29 @@ def retinanet_target_assign_in_python(all_anchors, gt_boxes, gt_labels, fg_nums = [[fg_num]] else: loc_indexes = np.concatenate( - [loc_indexes, loc_inds + i * anchor_num]) + [loc_indexes, loc_inds + i * anchor_num] + ) score_indexes = np.concatenate( - [score_indexes, score_inds + i * anchor_num]) + [score_indexes, score_inds + i * anchor_num] + ) tgt_labels = np.concatenate([tgt_labels, labels]) tgt_bboxes = np.vstack([tgt_bboxes, box_deltas]) - bbox_inside_weights = np.vstack([bbox_inside_weights, \ - bbox_inside_weight]) + bbox_inside_weights = np.vstack( + [bbox_inside_weights, bbox_inside_weight] + ) fg_nums = np.concatenate([fg_nums, [[fg_num]]]) - return loc_indexes, score_indexes, tgt_bboxes, tgt_labels, bbox_inside_weights, fg_nums + return ( + loc_indexes, + score_indexes, + tgt_bboxes, + tgt_labels, + bbox_inside_weights, + fg_nums, + ) class TestRpnTargetAssignOp(OpTest): - def setUp(self): n, c, h, w = 2, 4, 14, 14 all_anchors = get_anchor(n, c, h, w) @@ -277,16 +333,16 @@ def setUp(self): anchor_num = all_anchors.shape[0] images_shape = [[64, 64], [64, 64]] - #images_shape = [[64, 64]] + # images_shape = [[64, 64]] groundtruth, lod = _generate_groundtruth(images_shape, 3, 4) lod = [0, 4, 8] - #lod = [0, 4] + # lod = [0, 4] im_info = np.ones((len(images_shape), 3)).astype(np.float32) for i in range(len(images_shape)): im_info[i, 0] = images_shape[i][0] im_info[i, 1] = images_shape[i][1] - im_info[i, 2] = 0.8 #scale + im_info[i, 2] = 0.8 # scale gt_boxes = np.vstack([v['boxes'] for v in groundtruth]) is_crowd = np.hstack([v['is_crowd'] for v in groundtruth]) @@ -300,12 +356,25 @@ def setUp(self): rpn_fg_fraction = 0.5 use_random = False - loc_index, score_index, tgt_bbox, labels, bbox_inside_weights = \ - rpn_target_assign_in_python(all_anchors, gt_boxes, is_crowd, - im_info, lod, rpn_straddle_thresh, - rpn_batch_size_per_im, rpn_positive_overlap, - rpn_negative_overlap, - rpn_fg_fraction, use_random) + ( + loc_index, + score_index, + tgt_bbox, + labels, + bbox_inside_weights, + ) = rpn_target_assign_in_python( + all_anchors, + gt_boxes, + is_crowd, + im_info, + lod, + rpn_straddle_thresh, + rpn_batch_size_per_im, + rpn_positive_overlap, + rpn_negative_overlap, + rpn_fg_fraction, + use_random, + ) labels = labels[:, np.newaxis] self.op_type = "rpn_target_assign" @@ -313,7 +382,7 @@ def setUp(self): 'Anchor': all_anchors, 'GtBoxes': (gt_boxes, [[4, 4]]), 'IsCrowd': (is_crowd, [[4, 4]]), - 'ImInfo': (im_info, [[1, 1]]) + 'ImInfo': (im_info, [[1, 1]]), } self.attrs = { 'rpn_batch_size_per_im': rpn_batch_size_per_im, @@ -321,14 +390,14 @@ def setUp(self): 'rpn_positive_overlap': rpn_positive_overlap, 'rpn_negative_overlap': rpn_negative_overlap, 'rpn_fg_fraction': rpn_fg_fraction, - 'use_random': use_random + 'use_random': use_random, } self.outputs = { 'LocationIndex': loc_index.astype('int32'), 'ScoreIndex': score_index.astype('int32'), 'TargetBBox': tgt_bbox.astype('float32'), 'TargetLabel': labels.astype('int32'), - 'BBoxInsideWeight': bbox_inside_weights.astype('float32') + 'BBoxInsideWeight': bbox_inside_weights.astype('float32'), } def test_check_output(self): @@ -336,7 +405,6 @@ def test_check_output(self): class TestRetinanetTargetAssignOp(OpTest): - def setUp(self): n, c, h, w = 2, 4, 14, 14 all_anchors = get_anchor(n, c, h, w) @@ -352,13 +420,15 @@ def setUp(self): for i in range(len(images_shape)): im_info[i, 0] = images_shape[i][0] im_info[i, 1] = images_shape[i][1] - im_info[i, 2] = 0.8 #scale + im_info[i, 2] = 0.8 # scale gt_boxes = np.vstack([v['boxes'] for v in groundtruth]) is_crowd = np.hstack([v['is_crowd'] for v in groundtruth]) - gt_labels = np.vstack([ - v['gt_classes'].reshape(len(v['gt_classes']), 1) - for v in groundtruth - ]) + gt_labels = np.vstack( + [ + v['gt_classes'].reshape(len(v['gt_classes']), 1) + for v in groundtruth + ] + ) gt_labels = gt_labels.reshape(len(gt_labels), 1) all_anchors = all_anchors.astype('float32') gt_boxes = gt_boxes.astype('float32') @@ -367,9 +437,23 @@ def setUp(self): positive_overlap = 0.5 negative_overlap = 0.4 - loc_index, score_index, tgt_bbox, labels, bbox_inside_weights, fg_num = \ - retinanet_target_assign_in_python(all_anchors, gt_boxes, gt_labels, is_crowd, - im_info, lod, positive_overlap, negative_overlap) + ( + loc_index, + score_index, + tgt_bbox, + labels, + bbox_inside_weights, + fg_num, + ) = retinanet_target_assign_in_python( + all_anchors, + gt_boxes, + gt_labels, + is_crowd, + im_info, + lod, + positive_overlap, + negative_overlap, + ) labels = labels[:, np.newaxis] self.op_type = "retinanet_target_assign" self.inputs = { @@ -377,11 +461,11 @@ def setUp(self): 'GtBoxes': (gt_boxes, [[4, 4]]), 'GtLabels': (gt_labels, [[4, 4]]), 'IsCrowd': (is_crowd, [[4, 4]]), - 'ImInfo': (im_info, [[1, 1]]) + 'ImInfo': (im_info, [[1, 1]]), } self.attrs = { 'positive_overlap': positive_overlap, - 'negative_overlap': negative_overlap + 'negative_overlap': negative_overlap, } self.outputs = { 'LocationIndex': loc_index.astype('int32'), @@ -389,7 +473,7 @@ def setUp(self): 'TargetBBox': tgt_bbox.astype('float32'), 'TargetLabel': labels.astype('int32'), 'BBoxInsideWeight': bbox_inside_weights.astype('float32'), - 'ForegroundNumber': fg_num.astype('int32') + 'ForegroundNumber': fg_num.astype('int32'), } def test_check_output(self): @@ -397,181 +481,420 @@ def test_check_output(self): class TestRetinanetTargetAssignOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - bbox_pred1 = fluid.data(name='bbox_pred1', - shape=[1, 100, 4], - dtype='float32') - cls_logits1 = fluid.data(name='cls_logits1', - shape=[1, 100, 10], - dtype='float32') - anchor_box1 = fluid.data(name='anchor_box1', - shape=[100, 4], - dtype='float32') - anchor_var1 = fluid.data(name='anchor_var1', - shape=[100, 4], - dtype='float32') - gt_boxes1 = fluid.data(name='gt_boxes1', - shape=[10, 4], - dtype='float32') - gt_labels1 = fluid.data(name='gt_labels1', - shape=[10, 1], - dtype='int32') + bbox_pred1 = fluid.data( + name='bbox_pred1', shape=[1, 100, 4], dtype='float32' + ) + cls_logits1 = fluid.data( + name='cls_logits1', shape=[1, 100, 10], dtype='float32' + ) + anchor_box1 = fluid.data( + name='anchor_box1', shape=[100, 4], dtype='float32' + ) + anchor_var1 = fluid.data( + name='anchor_var1', shape=[100, 4], dtype='float32' + ) + gt_boxes1 = fluid.data( + name='gt_boxes1', shape=[10, 4], dtype='float32' + ) + gt_labels1 = fluid.data( + name='gt_labels1', shape=[10, 1], dtype='int32' + ) is_crowd1 = fluid.data(name='is_crowd1', shape=[1], dtype='float32') - im_info1 = fluid.data(name='im_info1', - shape=[1, 3], - dtype='float32') + im_info1 = fluid.data( + name='im_info1', shape=[1, 3], dtype='float32' + ) # The `bbox_pred` must be Variable and the data type of `bbox_pred` Tensor # one of float32 and float64. def test_bbox_pred_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign([1], cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + [1], + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_bbox_pred_type) def test_bbox_pred_tensor_dtype(): - bbox_pred2 = fluid.data(name='bbox_pred2', - shape=[1, 100, 4], - dtype='intt32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred2, cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + bbox_pred2 = fluid.data( + name='bbox_pred2', shape=[1, 100, 4], dtype='intt32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred2, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_bbox_pred_tensor_dtype) # The `cls_logits` must be Variable and the data type of `cls_logits` Tensor # one of float32 and float64. def test_cls_logits_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, 2, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + 2, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_cls_logits_type) def test_cls_logits_tensor_dtype(): - cls_logits2 = fluid.data(name='cls_logits2', - shape=[1, 100, 10], - dtype='int32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits2, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + cls_logits2 = fluid.data( + name='cls_logits2', shape=[1, 100, 10], dtype='int32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits2, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_cls_logits_tensor_dtype) # The `anchor_box` must be Variable and the data type of `anchor_box` Tensor # one of float32 and float64. def test_anchor_box_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, [5], - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + [5], + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_anchor_box_type) def test_anchor_box_tensor_dtype(): - anchor_box2 = fluid.data(name='anchor_box2', - shape=[100, 4], - dtype='int32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box2, - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + anchor_box2 = fluid.data( + name='anchor_box2', shape=[100, 4], dtype='int32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box2, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_anchor_box_tensor_dtype) # The `anchor_var` must be Variable and the data type of `anchor_var` Tensor # one of float32 and float64. def test_anchor_var_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - 5, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + 5, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_anchor_var_type) def test_anchor_var_tensor_dtype(): - anchor_var2 = fluid.data(name='anchor_var2', - shape=[100, 4], - dtype='int32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var2, gt_boxes1, gt_labels1, is_crowd1, im_info1, 10) + anchor_var2 = fluid.data( + name='anchor_var2', shape=[100, 4], dtype='int32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var2, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_anchor_var_tensor_dtype) # The `gt_boxes` must be Variable and the data type of `gt_boxes` Tensor # one of float32 and float64. def test_gt_boxes_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, [4], gt_labels1, is_crowd1, im_info1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + [4], + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_gt_boxes_type) def test_gt_boxes_tensor_dtype(): - gt_boxes2 = fluid.data(name='gt_boxes2', - shape=[10, 4], - dtype='int32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, gt_boxes2, gt_labels1, is_crowd1, im_info1, 10) + gt_boxes2 = fluid.data( + name='gt_boxes2', shape=[10, 4], dtype='int32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes2, + gt_labels1, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_gt_boxes_tensor_dtype) # The `gt_label` must be Variable and the data type of `gt_label` Tensor # int32. def test_gt_label_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, 9, is_crowd1, im_info1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + 9, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_gt_label_type) def test_gt_label_tensor_dtype(): - gt_labels2 = fluid.data(name='label2', - shape=[10, 1], - dtype='float32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, gt_labels2, is_crowd1, im_info1, 10) + gt_labels2 = fluid.data( + name='label2', shape=[10, 1], dtype='float32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels2, + is_crowd1, + im_info1, + 10, + ) self.assertRaises(TypeError, test_gt_label_tensor_dtype) # The `is_crowd` must be Variable and the data type of `is_crowd` Tensor # int32. def test_is_crowd_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, [10], im_info1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + [10], + im_info1, + 10, + ) self.assertRaises(TypeError, test_is_crowd_type) def test_is_crowd_tensor_dtype(): - is_crowd2 = fluid.data(name='is_crowd2', - shape=[10, 1], - dtype='float32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, is_crowd2, im_info1, 10) + is_crowd2 = fluid.data( + name='is_crowd2', shape=[10, 1], dtype='float32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd2, + im_info1, + 10, + ) self.assertRaises(TypeError, test_is_crowd_tensor_dtype) # The `im_info` must be Variable and the data type of `im_info` Tensor # must be one of float32 and float64. def test_im_info_type(): - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, 1, 10) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + 1, + 10, + ) self.assertRaises(TypeError, test_im_info_type) def test_im_info_tensor_dtype(): - im_info2 = fluid.data(name='im_info2', - shape=[1, 3], - dtype='int32') - score_pred, loc_pred, score_target, loc_target, bbox_inside_weight, fg_num = \ - fluid.layers.retinanet_target_assign(bbox_pred1, cls_logits1, anchor_box1, - anchor_var1, gt_boxes1, gt_labels1, is_crowd1, im_info2, 10) + im_info2 = fluid.data( + name='im_info2', shape=[1, 3], dtype='int32' + ) + ( + score_pred, + loc_pred, + score_target, + loc_target, + bbox_inside_weight, + fg_num, + ) = fluid.layers.retinanet_target_assign( + bbox_pred1, + cls_logits1, + anchor_box1, + anchor_var1, + gt_boxes1, + gt_labels1, + is_crowd1, + im_info2, + 10, + ) self.assertRaises(TypeError, test_im_info_tensor_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_rrelu_op.py b/python/paddle/fluid/tests/unittests/test_rrelu_op.py index 6701802b44329..9da00696fa393 100644 --- a/python/paddle/fluid/tests/unittests/test_rrelu_op.py +++ b/python/paddle/fluid/tests/unittests/test_rrelu_op.py @@ -42,9 +42,8 @@ def check_output(input, output, lower, upper): class TestFunctionalRReluAPI(unittest.TestCase): - def setUp(self): - self.x_np = np.random.uniform(-1., 1., [1, 2, 3, 4]).astype('float64') + self.x_np = np.random.uniform(-1.0, 1.0, [1, 2, 3, 4]).astype('float64') self.lower_0 = 0.05 self.lower_1 = 0.1 self.upper_0 = 0.25 @@ -52,36 +51,39 @@ def setUp(self): self.places = [ fluid.CUDAPlace(0) - if core.is_compiled_with_cuda() else fluid.CPUPlace() + if core.is_compiled_with_cuda() + else fluid.CPUPlace() ] def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - input = fluid.data(name="input", - shape=[2, 3, 4, 5], - dtype="float32") - res1 = F.rrelu(x=input, - lower=self.lower_0, - upper=self.upper_0, - training=False) - res2 = F.rrelu(x=input, - lower=self.lower_1, - upper=self.upper_1, - training=False) - in_np = np.random.uniform(-1., 1., [2, 3, 4, 5]).astype("float32") + input = fluid.data( + name="input", shape=[2, 3, 4, 5], dtype="float32" + ) + res1 = F.rrelu( + x=input, lower=self.lower_0, upper=self.upper_0, training=False + ) + res2 = F.rrelu( + x=input, lower=self.lower_1, upper=self.upper_1, training=False + ) + in_np = np.random.uniform(-1.0, 1.0, [2, 3, 4, 5]).astype("float32") res_np1 = ref_rrelu(in_np, self.lower_0, self.upper_0) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res1]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res1], + ) np.testing.assert_allclose(fetches[0], res_np1, rtol=1e-05) res_np2 = ref_rrelu(in_np, self.lower_1, self.upper_1) - fetches = exe.run(fluid.default_main_program(), - feed={"input": in_np}, - fetch_list=[res2]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": in_np}, + fetch_list=[res2], + ) np.testing.assert_allclose(fetches[0], res_np2, rtol=1e-05) def test_static(self): @@ -93,48 +95,55 @@ def test_static_graph_functional(self): for place in self.places: paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=self.x_np.shape, - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=self.x_np.shape, - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=self.x_np.shape, dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=self.x_np.shape, dtype="float64" + ) out_1 = F.rrelu(x_1, self.lower_0, self.upper_0, training=False) out_2 = F.rrelu(x_2, self.lower_1, self.upper_1, training=False) out_3 = F.rrelu(x_2, self.lower_1, self.upper_1, training=True) exe = paddle.static.Executor(place=place) - res_1, = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=out_1, - use_prune=True) - res_2, = exe.run(fluid.default_main_program(), - feed={"x2": self.x_np}, - fetch_list=out_2, - use_prune=True) - res_3, = exe.run(fluid.default_main_program(), - feed={"x2": self.x_np}, - fetch_list=out_3, - use_prune=True) + (res_1,) = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=out_1, + use_prune=True, + ) + (res_2,) = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_np}, + fetch_list=out_2, + use_prune=True, + ) + (res_3,) = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_np}, + fetch_list=out_3, + use_prune=True, + ) out_ref_1 = ref_rrelu(self.x_np, self.lower_0, self.upper_0) out_ref_2 = ref_rrelu(self.x_np, self.lower_1, self.upper_1) np.testing.assert_allclose(out_ref_1, res_1, rtol=1e-05) np.testing.assert_allclose(out_ref_2, res_2, rtol=1e-05) self.assertTrue( - check_output(self.x_np, res_3[0], self.lower_1, self.upper_1)) + check_output(self.x_np, res_3[0], self.lower_1, self.upper_1) + ) def test_static_graph_layer(self): '''test_static_graph_layer''' for place in self.places: paddle.enable_static() - x_1 = paddle.fluid.data(name="x", - shape=self.x_np.shape, - dtype="float64") - x_2 = paddle.fluid.data(name="x2", - shape=self.x_np.shape, - dtype="float64") + x_1 = paddle.fluid.data( + name="x", shape=self.x_np.shape, dtype="float64" + ) + x_2 = paddle.fluid.data( + name="x2", shape=self.x_np.shape, dtype="float64" + ) # init instance rrelu_1 = paddle.nn.RReLU(self.lower_0, self.upper_0) rrelu_2 = paddle.nn.RReLU(self.lower_1, self.upper_1) @@ -142,19 +151,25 @@ def test_static_graph_layer(self): out_2 = rrelu_2(x_2) exe = paddle.static.Executor(place=place) - res_1 = exe.run(fluid.default_main_program(), - feed={"x": self.x_np}, - fetch_list=out_1, - use_prune=True) - res_2 = exe.run(fluid.default_main_program(), - feed={"x2": self.x_np}, - fetch_list=out_2, - use_prune=True) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": self.x_np}, + fetch_list=out_1, + use_prune=True, + ) + res_2 = exe.run( + fluid.default_main_program(), + feed={"x2": self.x_np}, + fetch_list=out_2, + use_prune=True, + ) self.assertTrue( - check_output(self.x_np, res_1[0], self.lower_0, self.upper_0)) + check_output(self.x_np, res_1[0], self.lower_0, self.upper_0) + ) self.assertTrue( - check_output(self.x_np, res_2[0], self.lower_1, self.upper_1)) + check_output(self.x_np, res_2[0], self.lower_1, self.upper_1) + ) def dygraph_check(self, lower, upper): for place in self.places: @@ -179,8 +194,10 @@ def test_dygraph_layer(self): rrelu = paddle.nn.RReLU(self.lower_0, self.upper_0) result = rrelu(paddle.to_tensor(self.x_np)) self.assertTrue( - check_output(self.x_np, result.numpy(), self.lower_0, - self.upper_0)) + check_output( + self.x_np, result.numpy(), self.lower_0, self.upper_0 + ) + ) paddle.enable_static() def test_dygraph(self): @@ -190,67 +207,64 @@ def test_dygraph(self): rrelu = paddle.nn.RReLU(self.lower_0, self.upper_0) out_np = rrelu(paddle.to_tensor(self.x_np)) self.assertTrue( - check_output(self.x_np, out_np.numpy(), self.lower_0, - self.upper_0)) + check_output( + self.x_np, out_np.numpy(), self.lower_0, self.upper_0 + ) + ) paddle.enable_static() def test_error_functional(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): # The input type must be Variable. - self.assertRaises(TypeError, - F.rrelu, - x=1, - lower=self.lower_0, - upper=self.upper_0) + self.assertRaises( + TypeError, F.rrelu, x=1, lower=self.lower_0, upper=self.upper_0 + ) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[2, 3], - dtype='int32') - self.assertRaises(TypeError, - F.rrelu, - x=x_int32, - lower=self.lower_0, - upper=self.upper_0) - x_bool = paddle.fluid.data(name='x_bool', - shape=[2, 3], - dtype='int32') - self.assertRaises(TypeError, - F.rrelu, - x=x_bool, - lower=self.lower_0, - upper=self.upper_0) + x_int32 = paddle.fluid.data( + name='x_int32', shape=[2, 3], dtype='int32' + ) + self.assertRaises( + TypeError, + F.rrelu, + x=x_int32, + lower=self.lower_0, + upper=self.upper_0, + ) + x_bool = paddle.fluid.data( + name='x_bool', shape=[2, 3], dtype='int32' + ) + self.assertRaises( + TypeError, + F.rrelu, + x=x_bool, + lower=self.lower_0, + upper=self.upper_0, + ) # lower and upper must be float - x_fp32 = paddle.fluid.data(name='x_fp32', - shape=[2, 3], - dtype='float32') + x_fp32 = paddle.fluid.data( + name='x_fp32', shape=[2, 3], dtype='float32' + ) self.assertRaises(TypeError, F.rrelu, x=x_fp32, lower=0, upper=0.5) self.assertRaises(TypeError, F.rrelu, x=x_fp32, lower=0.5, upper=1) # lower and upper must be in (0, 1) - self.assertRaises(ValueError, - F.rrelu, - x=x_fp32, - lower=-1., - upper=0.5) - self.assertRaises(ValueError, - F.rrelu, - x=x_fp32, - lower=0.5, - upper=2.) + self.assertRaises( + ValueError, F.rrelu, x=x_fp32, lower=-1.0, upper=0.5 + ) + self.assertRaises( + ValueError, F.rrelu, x=x_fp32, lower=0.5, upper=2.0 + ) # upper should not be less than lower - self.assertRaises(ValueError, - F.rrelu, - x=x_fp32, - lower=0.5, - upper=0.2) + self.assertRaises( + ValueError, F.rrelu, x=x_fp32, lower=0.5, upper=0.2 + ) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[2, 3], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[2, 3], dtype='float16' + ) F.rrelu(x=x_fp16, lower=self.lower_0, upper=self.upper_0) def test_error_layer(self): - def error_int_dtype(): with paddle.fluid.dygraph.guard(): x = np.random.random([2, 3]).astype("float64") @@ -296,7 +310,6 @@ def error_lower_upper(): class RReluTest(OpTest): - def setUp(self): self.op_type = "rrelu" self.lower = 0.1 @@ -318,7 +331,7 @@ def init_prams(self): self.attrs = { 'lower': self.lower, "upper": self.upper, - "is_test": self.is_test + "is_test": self.is_test, } def test_check_output(self): @@ -329,7 +342,6 @@ def test_check_grad(self): class RReluTrainingTest(OpTest): - def setUp(self): self.op_type = "rrelu" self.lower = 0.3 @@ -339,7 +351,6 @@ def setUp(self): class RReluTrainingTest(OpTest): - def setUp(self): self.op_type = "rrelu" self.lower = 0.3 diff --git a/python/paddle/fluid/tests/unittests/test_run.py b/python/paddle/fluid/tests/unittests/test_run.py index 15af313b44cee..514384c0192f7 100644 --- a/python/paddle/fluid/tests/unittests/test_run.py +++ b/python/paddle/fluid/tests/unittests/test_run.py @@ -50,13 +50,13 @@ def write_file(name, ct): def get_files(pth, prefix): return [ - f for f in listdir(pth) + f + for f in listdir(pth) if isfile(join(pth, f)) and not f.endswith('gpu.log') ] class Collective_Test(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.path = os.path.join(self.temp_dir.name, pyname) @@ -87,7 +87,8 @@ def test_collective_1(self): def test_collective_2(self): log_dir = tempfile.TemporaryDirectory() args = "--job_id test2 --devices 0,1,2 --log_dir {}".format( - log_dir.name) + log_dir.name + ) p = self.pdrun(args) p.wait() self.assertTrue(p.poll() == 0) @@ -116,7 +117,6 @@ def test_collective_3(self): class PS_Test(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.path = os.path.join(self.temp_dir.name, pyname) @@ -143,8 +143,11 @@ def test_ps_1(self): def test_ps_2(self): log_dir = tempfile.TemporaryDirectory() - args = "--job_id ps2 --server_num=2 --trainer_num=2 --log_dir {}".format( - log_dir.name) + args = ( + "--job_id ps2 --server_num=2 --trainer_num=2 --log_dir {}".format( + log_dir.name + ) + ) p = self.pdrun(args) p.wait() self.assertTrue(p.poll() == 0) @@ -174,7 +177,8 @@ def test_ps_3(self): def test_ps_4(self): log_dir = tempfile.TemporaryDirectory() args = "--job_id ps4 --log_dir {} --servers 127.0.0.1:8900,127.0.0.1:8901 --trainers 127.0.0.1:8902,127.0.0.1:8903".format( - log_dir.name) + log_dir.name + ) p1 = self.pdrun(args) p1.wait() self.assertTrue(p1.poll() == 0) diff --git a/python/paddle/fluid/tests/unittests/test_run_fluid_by_module_or_command_line.py b/python/paddle/fluid/tests/unittests/test_run_fluid_by_module_or_command_line.py index d59c9db637f74..df626dc6dded7 100644 --- a/python/paddle/fluid/tests/unittests/test_run_fluid_by_module_or_command_line.py +++ b/python/paddle/fluid/tests/unittests/test_run_fluid_by_module_or_command_line.py @@ -18,7 +18,6 @@ class TestRunFluidByModule(unittest.TestCase): - def test_module(self): print(sys.executable) res = os.system(sys.executable + ' -m "paddle.fluid.reader"') @@ -26,7 +25,6 @@ def test_module(self): class TestRunFluidByCommand(unittest.TestCase): - def test_command(self): res = os.system(sys.executable + ' -c "import paddle.fluid"') self.assertEqual(res, 0) # 0 means status OK diff --git a/python/paddle/fluid/tests/unittests/test_run_program_op.py b/python/paddle/fluid/tests/unittests/test_run_program_op.py index 51538266b5daf..c2f871bd5760f 100644 --- a/python/paddle/fluid/tests/unittests/test_run_program_op.py +++ b/python/paddle/fluid/tests/unittests/test_run_program_op.py @@ -22,7 +22,10 @@ from paddle.fluid import core, framework from paddle.fluid.layers.utils import _hash_with_id from paddle.fluid.framework import _in_eager_mode_ -from paddle.fluid.executor import _is_enable_standalone_executor, _is_dy2st_enable_standalone_executor +from paddle.fluid.executor import ( + _is_enable_standalone_executor, + _is_dy2st_enable_standalone_executor, +) from paddle.fluid.dygraph.base import switch_to_static_graph paddle.enable_static() @@ -43,9 +46,11 @@ def program_scope_guard(): def _add_build_strategy_for(input_program, start_op_index, end_op_index): compiled_program = paddle.static.CompiledProgram( core.Graph(input_program.desc, start_op_index, end_op_index), - build_strategy=paddle.static.BuildStrategy()) - compiled_program._compile(core.Scope(), - paddle.framework._current_expected_place()) + build_strategy=paddle.static.BuildStrategy(), + ) + compiled_program._compile( + core.Scope(), paddle.framework._current_expected_place() + ) ir_graph = paddle.fluid.framework.IrGraph(compiled_program._graph) builded_program = ir_graph.to_program() return builded_program @@ -70,10 +75,10 @@ def _build_program_by_desc(program_desc): # when create Operator, so here compare gradients with static graph # NOTE: Here rewrite a simple unittest framework for RunProgramOp class RunProgramOpTest(unittest.TestCase): - def build_model(self): raise NotImplementedError( - "RunProgramOp test should implement build_model") + "RunProgramOp test should implement build_model" + ) def check_output(self): places = [fluid.CPUPlace()] @@ -108,9 +113,9 @@ def run_static_model(self, place, is_test=True): else: fetch_list = self.get_param_grad_names() - outs = exe.run(main_program, - feed=self.inputs['X'], - fetch_list=fetch_list) + outs = exe.run( + main_program, feed=self.inputs['X'], fetch_list=fetch_list + ) return outs def get_program_desc(self): @@ -118,21 +123,28 @@ def get_program_desc(self): fwd_op_num = self.build_model() return fluid.default_main_program().desc, fwd_op_num - def get_forward_backward_program_desc(self, whole_program_desc, - forward_op_num, output_num): + def get_forward_backward_program_desc( + self, whole_program_desc, forward_op_num, output_num + ): program = _build_program_by_desc(whole_program_desc) forward_program = _add_build_strategy_for(program, 0, forward_op_num) backward_program = _add_build_strategy_for( - program, forward_op_num + 2 * output_num, - program.desc.block(0).op_size()) + program, + forward_op_num + 2 * output_num, + program.desc.block(0).op_size(), + ) return forward_program.desc, backward_program.desc def prepare_attrs(self): return [ 'global_block', - self.program_desc.block(0), 'start_op_index', 0, 'end_op_index', - self.fwd_op_num, 'program_id', - _hash_with_id(self.program_desc, self) + self.program_desc.block(0), + 'start_op_index', + 0, + 'end_op_index', + self.fwd_op_num, + 'program_id', + _hash_with_id(self.program_desc, self), ] def get_param_grad_names(self): @@ -147,10 +159,9 @@ def check_output_with_place(self, place): # Step 2. compare output for expect_v, actual_v in zip(self.expect_outs, actual_outs): - np.testing.assert_allclose(expect_v, - actual_v.numpy(), - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + expect_v, actual_v.numpy(), rtol=1e-05, atol=1e-05 + ) def check_grad_with_place(self, place): # Step 1. calc grads @@ -159,24 +170,20 @@ def check_grad_with_place(self, place): # Step 2. compare grads for expect_v, actual_v in zip(self.expect_grads, actual_grads): np.testing.assert_array_almost_equal(expect_v, actual_v) - np.testing.assert_allclose(expect_v, - actual_v, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + expect_v, actual_v, rtol=1e-05, atol=1e-05 + ) def prepare_dygraph_input(self, place, return_param_list=False): - def create_var_base(is_input, name, np_value, stop_gradient): if _in_eager_mode_: - var = core.eager.Tensor(value=np_value, - name=name, - place=place, - zero_copy=True) + var = core.eager.Tensor( + value=np_value, name=name, place=place, zero_copy=True + ) else: - var = core.VarBase(value=np_value, - name=name, - place=place, - zero_copy=True) + var = core.VarBase( + value=np_value, name=name, place=place, zero_copy=True + ) var.stop_gradient = stop_gradient return var @@ -199,7 +206,6 @@ def create_var_base(is_input, name, np_value, stop_gradient): return inputs def prepare_dygraph_output(self): - def create_var_base(is_input, name): var = framework._varbase_creator(dtype=None, shape=None, name=name) var.stop_gradient = False @@ -217,7 +223,8 @@ def create_var_base(is_input, name): outputs['OutScope'] = framework._varbase_creator( type=core.VarDesc.VarType.STEP_SCOPES, name="program_out_scope", - persistable=True) + persistable=True, + ) inner_scope = core.Scope() outputs['OutScope'].value().set_scope(inner_scope) @@ -232,20 +239,37 @@ def calc_dygraph_output(self, place): inputs = self.prepare_dygraph_input(place) outputs = self.prepare_dygraph_output() - forward_program_desc, backward_program_desc = self.get_forward_backward_program_desc( - self.program_desc, self.fwd_op_num, len(outputs['Out'])) - - use_interpretorcore = _is_enable_standalone_executor( - ) and _is_dy2st_enable_standalone_executor() + ( + forward_program_desc, + backward_program_desc, + ) = self.get_forward_backward_program_desc( + self.program_desc, self.fwd_op_num, len(outputs['Out']) + ) + + use_interpretorcore = ( + _is_enable_standalone_executor() + and _is_dy2st_enable_standalone_executor() + ) self.attrs.extend(('use_interpretorcore', use_interpretorcore)) if use_interpretorcore: self.attrs.extend( - ('forward_global_block', forward_program_desc.block(0), - 'backward_global_block', backward_program_desc.block(0))) - - _legacy_C_ops.run_program(inputs['X'], inputs['Params'], - outputs['Out'], outputs['OutScope'], - outputs['DOut'], None, *self.attrs) + ( + 'forward_global_block', + forward_program_desc.block(0), + 'backward_global_block', + backward_program_desc.block(0), + ) + ) + + _legacy_C_ops.run_program( + inputs['X'], + inputs['Params'], + outputs['Out'], + outputs['OutScope'], + outputs['DOut'], + None, + *self.attrs + ) return outputs['Out'] @@ -258,20 +282,37 @@ def calc_dygraph_grad(self, place): inputs, input_param_list = self.prepare_dygraph_input(place, True) outputs = self.prepare_dygraph_output() - forward_program_desc, backward_program_desc = self.get_forward_backward_program_desc( - self.program_desc, self.fwd_op_num, len(outputs['Out'])) - - use_interpretorcore = _is_enable_standalone_executor( - ) and _is_dy2st_enable_standalone_executor() + ( + forward_program_desc, + backward_program_desc, + ) = self.get_forward_backward_program_desc( + self.program_desc, self.fwd_op_num, len(outputs['Out']) + ) + + use_interpretorcore = ( + _is_enable_standalone_executor() + and _is_dy2st_enable_standalone_executor() + ) self.attrs.extend(('use_interpretorcore', use_interpretorcore)) if use_interpretorcore: self.attrs.extend( - ('forward_global_block', forward_program_desc.block(0), - 'backward_global_block', backward_program_desc.block(0))) - - _legacy_C_ops.run_program(inputs['X'], inputs['Params'], - outputs['Out'], outputs['OutScope'], - outputs['DOut'], None, *self.attrs) + ( + 'forward_global_block', + forward_program_desc.block(0), + 'backward_global_block', + backward_program_desc.block(0), + ) + ) + + _legacy_C_ops.run_program( + inputs['X'], + inputs['Params'], + outputs['Out'], + outputs['OutScope'], + outputs['DOut'], + None, + *self.attrs + ) for param in input_param_list: var_type = self._get_grad_vartype(param.name) @@ -302,27 +343,29 @@ def _get_grad_vartype(self, name): class TestRunProgramOpWithFC(RunProgramOpTest): - def setUp(self): self.op_type = "run_program" self.dtype = np.float32 self.input_names = { 'X': ['img'], - 'Params': ['weight_param', 'bias_param'] + 'Params': ['weight_param', 'bias_param'], } self.output_names = {'Out': ['fc_0.tmp_2']} self.inputs = { 'X': { - self.input_names['X'][0]: - np.random.random((32, 1, 28, 28)).astype(self.dtype) + self.input_names['X'][0]: np.random.random( + (32, 1, 28, 28) + ).astype(self.dtype) }, 'Params': { - self.input_names['Params'][0]: - np.random.random((784, 10)).astype(self.dtype), - self.input_names['Params'][1]: - np.random.random((32, 10)).astype(self.dtype) - } + self.input_names['Params'][0]: np.random.random( + (784, 10) + ).astype(self.dtype), + self.input_names['Params'][1]: np.random.random( + (32, 10) + ).astype(self.dtype), + }, } def test_check_output(self): @@ -333,26 +376,34 @@ def test_check_grad(self): def build_model(self): # 1. simple model - img = fluid.data(name=self.input_names['X'][0], - shape=[None, 1, 28, 28], - dtype='float32') + img = fluid.data( + name=self.input_names['X'][0], + shape=[None, 1, 28, 28], + dtype='float32', + ) weight_attr = fluid.ParamAttr( name=self.input_names['Params'][0], learning_rate=0.5, initializer=fluid.initializer.NumpyArrayInitializer( - self.inputs['Params'][self.input_names['Params'][0]]), - trainable=True) + self.inputs['Params'][self.input_names['Params'][0]] + ), + trainable=True, + ) bias_attr = fluid.ParamAttr( name=self.input_names['Params'][1], learning_rate=0.5, initializer=fluid.initializer.NumpyArrayInitializer( - self.inputs['Params'][self.input_names['Params'][1]]), - trainable=True) - pred = fluid.layers.fc(input=img, - size=10, - param_attr=weight_attr, - bias_attr=bias_attr, - act='relu') + self.inputs['Params'][self.input_names['Params'][1]] + ), + trainable=True, + ) + pred = fluid.layers.fc( + input=img, + size=10, + param_attr=weight_attr, + bias_attr=bias_attr, + act='relu', + ) # 2. get forward op num fwd_op_num = fluid.default_main_program().global_block().desc.op_size() # 3. append backward @@ -362,7 +413,6 @@ def build_model(self): class TestRunProgramOpWithEmbedding(RunProgramOpTest): - def setUp(self): self.op_type = "run_program" self.dtype = np.float32 @@ -370,12 +420,10 @@ def setUp(self): self.output_names = {'Out': ['reduce_sum_0.tmp_0']} self.inputs = { - 'X': { - 'x': np.array([[1, 3, 0, 4, 7]]).astype("int64") - }, + 'X': {'x': np.array([[1, 3, 0, 4, 7]]).astype("int64")}, 'Params': { 'emb_weight': np.random.random(size=(10, 16)).astype("float32") - } + }, } def test_check_output(self): @@ -393,9 +441,9 @@ def test_check_grad(self): def build_model(self): # 1. simple model - x = fluid.layers.data(name=self.input_names['X'][0], - shape=[5], - dtype='int64') + x = fluid.layers.data( + name=self.input_names['X'][0], shape=[5], dtype='int64' + ) emb = fluid.input.embedding( input=x, size=[10, 16], @@ -403,8 +451,11 @@ def build_model(self): name="emb_weight", learning_rate=10, initializer=fluid.initializer.NumpyArrayInitializer( - self.inputs['Params'][self.input_names['Params'][0]])), - is_sparse=True) + self.inputs['Params'][self.input_names['Params'][0]] + ), + ), + is_sparse=True, + ) y = fluid.layers.reduce_sum(emb, dim=-1) # 2. get forward op num fwd_op_num = fluid.default_main_program().global_block().desc.op_size() @@ -415,7 +466,6 @@ def build_model(self): class Net(paddle.nn.Layer): - def __init__(self): super(Net, self).__init__() self.fc1 = paddle.nn.Linear(10, 10) @@ -429,7 +479,6 @@ def forward(self, x): class TestParametersWithStopGradient(unittest.TestCase): - def setUp(self): self.seed = 2021 self.iter = 5 diff --git a/python/paddle/fluid/tests/unittests/test_runtime_and_compiletime_exception.py b/python/paddle/fluid/tests/unittests/test_runtime_and_compiletime_exception.py index 36ccb8e0322be..d498035327ba4 100644 --- a/python/paddle/fluid/tests/unittests/test_runtime_and_compiletime_exception.py +++ b/python/paddle/fluid/tests/unittests/test_runtime_and_compiletime_exception.py @@ -19,7 +19,6 @@ class TestRunTimeException(unittest.TestCase): - def test_run_time_exception(self): place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -38,7 +37,6 @@ def _run_program(): class TestCompileTimeException(unittest.TestCase): - def test_compile_time_exception(self): self.assertRaises(ValueError, self.build_model) @@ -46,10 +44,9 @@ def build_model(self): train_program = fluid.Program() startup_program = fluid.Program() with fluid.program_guard(train_program, startup_program): - label = fluid.layers.data(name="label", - shape=[1], - dtype="int64", - append_batch_size=False) + label = fluid.layers.data( + name="label", shape=[1], dtype="int64", append_batch_size=False + ) fluid.layers.one_hot(input=label, depth=100) diff --git a/python/paddle/fluid/tests/unittests/test_sample_logits_op.py b/python/paddle/fluid/tests/unittests/test_sample_logits_op.py index 749a32978beed..8676d8ff6e5fc 100644 --- a/python/paddle/fluid/tests/unittests/test_sample_logits_op.py +++ b/python/paddle/fluid/tests/unittests/test_sample_logits_op.py @@ -19,7 +19,6 @@ class TestSampleLogitsOp(OpTest): - def setUp(self): self.op_type = "sample_logits" self.dtype = np.float64 @@ -46,33 +45,49 @@ def setUp(self): self.inputs = {"Logits": self.Logits, "Labels": self.Labels} self.fetch_list = [ - 'Samples', 'Probabilities', 'SampledLogits', 'SampledLabels' + 'Samples', + 'Probabilities', + 'SampledLogits', + 'SampledLabels', ] self.outputs = collections.OrderedDict( - (('Samples', Samples), ('Probabilities', Probabilities), - ('LogitsDim', LogitsDim), ('LabelsDim', LabelsDim), - ('SampledLogits', SampledLogits), ('SampledLabels', - SampledLabels))) + ( + ('Samples', Samples), + ('Probabilities', Probabilities), + ('LogitsDim', LogitsDim), + ('LabelsDim', LabelsDim), + ('SampledLogits', SampledLogits), + ('SampledLabels', SampledLabels), + ) + ) self.attrs = {'num_samples': self.S} def test_check_output(self): places = self._get_places() for p in places: - (Samples, Probabilities, SampledLogits, - SampledLabels) = [np.array(o) for o in self.calc_output(p)] - - assert Samples.dtype == np.int64, \ - "Samples dtype is {}, not int64".format(Samples.dtype) - assert Probabilities.dtype == np.float64, \ - "Probabilities dtype is {}, not float64".format( - Probabilities.dtype) - assert SampledLogits.dtype == np.float64, \ - "SampledLogits dtype is {}, not float64".format( - SampledLogits.dtype) - assert SampledLabels.dtype == np.int64, \ - "SampledLabels dtype is {}, not int64".format( - SampledLabels.dtype) + (Samples, Probabilities, SampledLogits, SampledLabels) = [ + np.array(o) for o in self.calc_output(p) + ] + + assert ( + Samples.dtype == np.int64 + ), "Samples dtype is {}, not int64".format(Samples.dtype) + assert ( + Probabilities.dtype == np.float64 + ), "Probabilities dtype is {}, not float64".format( + Probabilities.dtype + ) + assert ( + SampledLogits.dtype == np.float64 + ), "SampledLogits dtype is {}, not float64".format( + SampledLogits.dtype + ) + assert ( + SampledLabels.dtype == np.int64 + ), "SampledLabels dtype is {}, not int64".format( + SampledLabels.dtype + ) assert Samples.shape == (self.bs, self.NT + self.S) assert Probabilities.shape == (self.bs, self.NT + self.S) @@ -80,27 +95,26 @@ def test_check_output(self): assert SampledLabels.shape == (self.bs, self.NT) assert (SampledLabels == self.Labels).all() - sampled_logits = self.Logits[:, Samples[0][:self.NT]] - sampled_logits -= np.log(Probabilities[:, :self.NT]) - np.testing.assert_almost_equal(sampled_logits, - SampledLogits[:, :self.NT]) + sampled_logits = self.Logits[:, Samples[0][: self.NT]] + sampled_logits -= np.log(Probabilities[:, : self.NT]) + np.testing.assert_almost_equal( + sampled_logits, SampledLogits[:, : self.NT] + ) def test_check_grad(self): self._check_grad_helper() for p in self._get_places(): grads = self._get_gradient(['Logits'], p, ['SampledLogits'], []) - np.testing.assert_almost_equal(grads[0].sum(), np.array([1.])) + np.testing.assert_almost_equal(grads[0].sum(), np.array([1.0])) class TestSampleLogitsOpNoUniq(TestSampleLogitsOp): - def setUp(self): super(TestSampleLogitsOpNoUniq, self).setUp() self.attrs = {'num_samples': self.S, 'uniq': False} class TestSampleLogitsOpWithAccidentalHits(TestSampleLogitsOp): - def setUp(self): super(TestSampleLogitsOpWithAccidentalHits, self).setUp() self.attrs = {'num_samples': self.S, 'remove_accidental_hits': False} diff --git a/python/paddle/fluid/tests/unittests/test_sampling_id_op.py b/python/paddle/fluid/tests/unittests/test_sampling_id_op.py index 3481db22907c1..2e73a90abdb5b 100644 --- a/python/paddle/fluid/tests/unittests/test_sampling_id_op.py +++ b/python/paddle/fluid/tests/unittests/test_sampling_id_op.py @@ -20,7 +20,6 @@ class TestSamplingIdShape(unittest.TestCase): - def test_shape(self): paddle.enable_static() x = fluid.layers.data(name='x', shape=[3], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/test_save_inference_model_conditional_op.py b/python/paddle/fluid/tests/unittests/test_save_inference_model_conditional_op.py index 240c0c8e129de..399be5ee60de9 100644 --- a/python/paddle/fluid/tests/unittests/test_save_inference_model_conditional_op.py +++ b/python/paddle/fluid/tests/unittests/test_save_inference_model_conditional_op.py @@ -28,14 +28,13 @@ def getModelOp(model_path): result = set() for i in range(0, size): - #print(main_block.op(i).type()) + # print(main_block.op(i).type()) result.add(main_block.op(i).type()) return result class WhileNet(paddle.nn.Layer): - def __init__(self): super(WhileNet, self).__init__() @@ -53,7 +52,6 @@ def forward(self, x): class ForNet(paddle.nn.Layer): - def __init__(self): super(ForNet, self).__init__() @@ -67,7 +65,6 @@ def forward(self, x): class IfElseNet(paddle.nn.Layer): - def __init__(self): super(IfElseNet, self).__init__() @@ -81,68 +78,91 @@ def forward(self, x): class TestConditionalOp(unittest.TestCase): - def test_while_op(self): paddle.disable_static() net = WhileNet() - net = paddle.jit.to_static(net, - input_spec=[ - paddle.static.InputSpec( - shape=[1, 3, 8, 8], dtype='float32') - ]) + net = paddle.jit.to_static( + net, + input_spec=[ + paddle.static.InputSpec(shape=[1, 3, 8, 8], dtype='float32') + ], + ) root_path = tempfile.TemporaryDirectory() model_file = os.path.join(root_path.name, "while_net") paddle.jit.save(net, model_file) - right_pdmodel = set([ - "uniform_random", "shape", "slice", "not_equal", "while", - "elementwise_add" - ]) + right_pdmodel = set( + [ + "uniform_random", + "shape", + "slice", + "not_equal", + "while", + "elementwise_add", + ] + ) paddle.enable_static() pdmodel = getModelOp(model_file + ".pdmodel") self.assertTrue( len(right_pdmodel.difference(pdmodel)) == 0, - "The while op is pruned by mistake.") + "The while op is pruned by mistake.", + ) root_path.cleanup() def test_for_op(self): paddle.disable_static() net = ForNet() net = paddle.jit.to_static( - net, input_spec=[paddle.static.InputSpec(shape=[1], dtype='int32')]) + net, input_spec=[paddle.static.InputSpec(shape=[1], dtype='int32')] + ) root_path = tempfile.TemporaryDirectory() model_file = os.path.join(root_path.name, "for_net") paddle.jit.save(net, model_file) - right_pdmodel = set([ - "randint", "fill_constant", "cast", "less_than", "while", - "elementwise_add" - ]) + right_pdmodel = set( + [ + "randint", + "fill_constant", + "cast", + "less_than", + "while", + "elementwise_add", + ] + ) paddle.enable_static() pdmodel = getModelOp(model_file + ".pdmodel") self.assertTrue( len(right_pdmodel.difference(pdmodel)) == 0, - "The for op is pruned by mistake.") + "The for op is pruned by mistake.", + ) root_path.cleanup() def test_if_op(self): paddle.disable_static() net = IfElseNet() net = paddle.jit.to_static( - net, input_spec=[paddle.static.InputSpec(shape=[1], dtype='int32')]) + net, input_spec=[paddle.static.InputSpec(shape=[1], dtype='int32')] + ) root_path = tempfile.TemporaryDirectory() model_file = os.path.join(root_path.name, "if_net") paddle.jit.save(net, model_file) - right_pdmodel = set([ - "assign_value", "greater_than", "cast", "conditional_block", - "logical_not", "select_input" - ]) + right_pdmodel = set( + [ + "assign_value", + "greater_than", + "cast", + "conditional_block", + "logical_not", + "select_input", + ] + ) paddle.enable_static() pdmodel = getModelOp(model_file + ".pdmodel") self.assertTrue( len(right_pdmodel.difference(pdmodel)) == 0, - "The if op is pruned by mistake.") + "The if op is pruned by mistake.", + ) root_path.cleanup() diff --git a/python/paddle/fluid/tests/unittests/test_save_model_without_var.py b/python/paddle/fluid/tests/unittests/test_save_model_without_var.py index e8cad31514398..ac02edc42e7f8 100644 --- a/python/paddle/fluid/tests/unittests/test_save_model_without_var.py +++ b/python/paddle/fluid/tests/unittests/test_save_model_without_var.py @@ -18,12 +18,10 @@ class TestSaveModelWithoutVar(unittest.TestCase): - def test_no_var_save(self): - data = fluid.layers.data(name='data', - shape=[-1, 1], - dtype='float32', - append_batch_size=False) + data = fluid.layers.data( + name='data', shape=[-1, 1], dtype='float32', append_batch_size=False + ) data_plus = data + 1 if fluid.core.is_compiled_with_cuda(): @@ -37,12 +35,14 @@ def test_no_var_save(self): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") - fluid.io.save_inference_model(dirname='test', - feeded_var_names=['data'], - target_vars=[data_plus], - executor=exe, - model_filename='model', - params_filename='params') + fluid.io.save_inference_model( + dirname='test', + feeded_var_names=['data'], + target_vars=[data_plus], + executor=exe, + model_filename='model', + params_filename='params', + ) expected_warn = "no variable in your model, please ensure there are any variables in your model to save" self.assertTrue(len(w) > 0) self.assertTrue(expected_warn == str(w[-1].message)) diff --git a/python/paddle/fluid/tests/unittests/test_saved_tensors_hooks.py b/python/paddle/fluid/tests/unittests/test_saved_tensors_hooks.py index 02b9d146fe3c3..c945eee068a54 100644 --- a/python/paddle/fluid/tests/unittests/test_saved_tensors_hooks.py +++ b/python/paddle/fluid/tests/unittests/test_saved_tensors_hooks.py @@ -18,9 +18,7 @@ class TestSavedTensorsHooks(unittest.TestCase): - def test_save_for_multiply(self): - def pack_hook(x): return x.numpy() @@ -46,9 +44,7 @@ def unpack_hook(x): self.assertTrue(paddle.equal_all(bb.grad, b.grad)) def test_save_for_pylayer(self): - class cus_multiply(PyLayer): - @staticmethod def forward(ctx, a, b): y = paddle.multiply(a, b) diff --git a/python/paddle/fluid/tests/unittests/test_scale_op.py b/python/paddle/fluid/tests/unittests/test_scale_op.py index 6eb4d90d610ac..4fec89905373f 100644 --- a/python/paddle/fluid/tests/unittests/test_scale_op.py +++ b/python/paddle/fluid/tests/unittests/test_scale_op.py @@ -26,7 +26,6 @@ class TestScaleOp(OpTest): - def setUp(self): self.op_type = "scale" self.python_api = paddle.scale @@ -49,7 +48,6 @@ def test_check_grad(self): class TestScaleOpScaleVariable(OpTest): - def setUp(self): self.op_type = "scale" self.python_api = paddle.scale @@ -58,7 +56,7 @@ def setUp(self): self.scale = -2.3 self.inputs = { 'X': np.random.random((10, 10)).astype(self.dtype), - 'ScaleTensor': np.array([self.scale]).astype('float64') + 'ScaleTensor': np.array([self.scale]).astype('float64'), } self.attrs = {} self.outputs = {'Out': self.inputs['X'] * self.dtype(self.scale)} @@ -74,7 +72,6 @@ def test_check_grad(self): class TestScaleOpSelectedRows(unittest.TestCase): - def init_dtype_type(self): pass @@ -93,8 +90,9 @@ def check_with_place(self, place, in_name, out_name): in_selected_rows = scope.var(in_name).get_selected_rows() in_selected_rows.set_height(in_height) in_selected_rows.set_rows(in_rows) - in_array = np.random.random( - (len(in_rows), in_row_numel)).astype(self.dtype) + in_array = np.random.random((len(in_rows), in_row_numel)).astype( + self.dtype + ) in_tensor = in_selected_rows.get_tensor() in_tensor.set(in_array, place) @@ -133,9 +131,7 @@ def test_scale_selected_rows_inplace(self): class TestScaleRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.scale([10]) @@ -143,10 +139,10 @@ def test_type(): # Add FP16 test -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestScaleFp16Op(TestScaleOp): - def init_dtype_type(self): self.dtype = np.float16 @@ -158,14 +154,12 @@ def test_check_output(self): def test_check_grad(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ["X"], - "Out", - max_relative_error=0.05, - check_eager=True) + self.check_grad_with_place( + place, ["X"], "Out", max_relative_error=0.05, check_eager=True + ) class TestScaleBF16Op(OpTest): - def setUp(self): self.op_type = "scale" self.python_api = paddle.scale @@ -183,10 +177,10 @@ def test_check_grad(self): self.check_grad(['X'], 'Out', numeric_grad_delta=0.8, check_eager=True) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestScaleFp16OpSelectedRows(TestScaleOpSelectedRows): - def init_dtype_type(self): self.dtype = np.float16 @@ -202,7 +196,6 @@ def test_scale_selected_rows_inplace(self): class TestScaleApiStatic(unittest.TestCase): - def _executed_api(self, x, scale=1.0, bias=0.0): return paddle.scale(x, scale, bias) @@ -220,13 +213,11 @@ def test_api(self): class TestScaleInplaceApiStatic(TestScaleApiStatic): - def _executed_api(self, x, scale=1.0, bias=0.0): return x.scale_(scale, bias) class TestScaleApiDygraph(unittest.TestCase): - def _executed_api(self, x, scale=1.0, bias=0.0): return paddle.scale(x, scale, bias) @@ -240,13 +231,11 @@ def test_api(self): class TestScaleInplaceApiDygraph(TestScaleApiDygraph): - def _executed_api(self, x, scale=1.0, bias=0.0): return x.scale_(scale, bias) class TestScaleDoubleGradCheck(unittest.TestCase): - def scale_wrapper(self, x): return paddle.scale(x[0], scale=2.0) @@ -261,17 +250,13 @@ def func(self, place): out = paddle.scale(data, 2.0) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.scale_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.scale_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -283,7 +268,6 @@ def test_grad(self): class TestScaleTripleGradCheck(unittest.TestCase): - def scale_wrapper(self, x): return paddle.scale(x[0], scale=2.0) @@ -298,17 +282,13 @@ def func(self, place): out = paddle.scale(data, 2.0) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.scale_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.scale_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_scaled_dot_product_attention.py b/python/paddle/fluid/tests/unittests/test_scaled_dot_product_attention.py index 031b45d3dd913..7f702482440e5 100644 --- a/python/paddle/fluid/tests/unittests/test_scaled_dot_product_attention.py +++ b/python/paddle/fluid/tests/unittests/test_scaled_dot_product_attention.py @@ -19,83 +19,89 @@ class TestScaledDotProductAttentionError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - queries = fluid.data(name="queries", - shape=[3, 5, 9], - dtype="float32") + queries = fluid.data( + name="queries", shape=[3, 5, 9], dtype="float32" + ) keys = fluid.data(name="keys", shape=[3, 6, 9], dtype="float32") - values = fluid.data(name="values", - shape=[3, 6, 10], - dtype="float32") + values = fluid.data( + name="values", shape=[3, 6, 10], dtype="float32" + ) def test_queries_Variable(): queries_data = np.random.rand(3, 5, 9).astype("float32") - fluid.nets.scaled_dot_product_attention(queries_data, keys, - values) + fluid.nets.scaled_dot_product_attention( + queries_data, keys, values + ) self.assertRaises(TypeError, test_queries_Variable) def test_keys_Variable(): keys_data = np.random.rand(3, 6, 9).astype("float32") - fluid.nets.scaled_dot_product_attention(queries, keys_data, - values) + fluid.nets.scaled_dot_product_attention( + queries, keys_data, values + ) self.assertRaises(TypeError, test_keys_Variable) def test_values_Variable(): values_data = np.random.rand(3, 6, 10).astype("float32") - fluid.nets.scaled_dot_product_attention(queries, keys, - values_data) + fluid.nets.scaled_dot_product_attention( + queries, keys, values_data + ) self.assertRaises(TypeError, test_values_Variable) def test_diff_dtype(): - keys_error = fluid.data(name="keys_error", - shape=[3, 6, 9], - dtype="float64") - values_error = fluid.data(name="values_error", - shape=[3, 6, 10], - dtype="float64") - fluid.nets.scaled_dot_product_attention(queries, keys_error, - values_error) + keys_error = fluid.data( + name="keys_error", shape=[3, 6, 9], dtype="float64" + ) + values_error = fluid.data( + name="values_error", shape=[3, 6, 10], dtype="float64" + ) + fluid.nets.scaled_dot_product_attention( + queries, keys_error, values_error + ) self.assertRaises(TypeError, test_diff_dtype) def test_diff_dim(): - keys_error_dim = fluid.data(name="keys_error_dim", - shape=[3, 6], - dtype="float32") - values_error_dim = fluid.data(name="values_error_dim", - shape=[3], - dtype="float32") - fluid.nets.scaled_dot_product_attention(queries, keys_error_dim, - values_error_dim) + keys_error_dim = fluid.data( + name="keys_error_dim", shape=[3, 6], dtype="float32" + ) + values_error_dim = fluid.data( + name="values_error_dim", shape=[3], dtype="float32" + ) + fluid.nets.scaled_dot_product_attention( + queries, keys_error_dim, values_error_dim + ) self.assertRaises(ValueError, test_diff_dim) def test_diff_hidden_size(): - queries_error_hs = fluid.data(name="queries_error_hs", - shape=[3, 5, 9], - dtype="float32") - keys_error_hs = fluid.data(name="keys_error_hs", - shape=[3, 6, 10], - dtype="float32") - fluid.nets.scaled_dot_product_attention(queries_error_hs, - keys_error_hs, values) + queries_error_hs = fluid.data( + name="queries_error_hs", shape=[3, 5, 9], dtype="float32" + ) + keys_error_hs = fluid.data( + name="keys_error_hs", shape=[3, 6, 10], dtype="float32" + ) + fluid.nets.scaled_dot_product_attention( + queries_error_hs, keys_error_hs, values + ) self.assertRaises(ValueError, test_diff_hidden_size) def test_diff_max_len(): - keys_error_len = fluid.data(name="keys_error_len", - shape=[3, 7, 9], - dtype="float32") - values_error_len = fluid.data(name="values_error_len", - shape=[3, 6, 10], - dtype="float32") - fluid.nets.scaled_dot_product_attention(queries, keys_error_len, - values_error_len) + keys_error_len = fluid.data( + name="keys_error_len", shape=[3, 7, 9], dtype="float32" + ) + values_error_len = fluid.data( + name="values_error_len", shape=[3, 6, 10], dtype="float32" + ) + fluid.nets.scaled_dot_product_attention( + queries, keys_error_len, values_error_len + ) self.assertRaises(ValueError, test_diff_max_len) diff --git a/python/paddle/fluid/tests/unittests/test_scatter_nd_op.py b/python/paddle/fluid/tests/unittests/test_scatter_nd_op.py index b9b0deb3b365c..e9edd089f5d3c 100644 --- a/python/paddle/fluid/tests/unittests/test_scatter_nd_op.py +++ b/python/paddle/fluid/tests/unittests/test_scatter_nd_op.py @@ -115,8 +115,9 @@ def setUp(self): self.python_api = paddle.scatter_nd_add shape = (3, 2, 2, 1, 10) ref_np = np.random.rand(*shape).astype("float64") - index_np = np.vstack([np.random.randint(0, s, size=100) - for s in shape]).T.astype("int32") + index_np = np.vstack( + [np.random.randint(0, s, size=100) for s in shape] + ).T.astype("int32") update_shape = judge_update_shape(ref_np, index_np) updates_np = np.random.rand(*update_shape).astype("float64") expect_np = numpy_scatter_nd_add(ref_np.copy(), index_np, updates_np) @@ -157,71 +158,89 @@ def test_check_grad(self): self.check_grad(['X', 'Updates'], 'Out', check_eager=True) -#Test Python API +# Test Python API class TestScatterNdOpAPI(unittest.TestCase): """ test scatter_nd_add api and scatter_nd api """ def testcase1(self): - ref1 = fluid.layers.data(name='ref1', - shape=[10, 9, 8, 1, 3], - dtype='float32', - append_batch_size=False) - index1 = fluid.layers.data(name='index1', - shape=[5, 5, 8, 5], - dtype='int32', - append_batch_size=False) - updates1 = fluid.layers.data(name='update1', - shape=[5, 5, 8], - dtype='float32', - append_batch_size=False) + ref1 = fluid.layers.data( + name='ref1', + shape=[10, 9, 8, 1, 3], + dtype='float32', + append_batch_size=False, + ) + index1 = fluid.layers.data( + name='index1', + shape=[5, 5, 8, 5], + dtype='int32', + append_batch_size=False, + ) + updates1 = fluid.layers.data( + name='update1', + shape=[5, 5, 8], + dtype='float32', + append_batch_size=False, + ) output1 = fluid.layers.scatter_nd_add(ref1, index1, updates1) def testcase2(self): - ref2 = fluid.layers.data(name='ref2', - shape=[10, 9, 8, 1, 3], - dtype='double', - append_batch_size=False) - index2 = fluid.layers.data(name='index2', - shape=[5, 8, 5], - dtype='int32', - append_batch_size=False) - updates2 = fluid.layers.data(name='update2', - shape=[5, 8], - dtype='double', - append_batch_size=False) - output2 = fluid.layers.scatter_nd_add(ref2, - index2, - updates2, - name="scatter_nd_add") + ref2 = fluid.layers.data( + name='ref2', + shape=[10, 9, 8, 1, 3], + dtype='double', + append_batch_size=False, + ) + index2 = fluid.layers.data( + name='index2', + shape=[5, 8, 5], + dtype='int32', + append_batch_size=False, + ) + updates2 = fluid.layers.data( + name='update2', + shape=[5, 8], + dtype='double', + append_batch_size=False, + ) + output2 = fluid.layers.scatter_nd_add( + ref2, index2, updates2, name="scatter_nd_add" + ) def testcase3(self): shape3 = [10, 9, 8, 1, 3] - index3 = fluid.layers.data(name='index3', - shape=[5, 5, 8, 5], - dtype='int32', - append_batch_size=False) - updates3 = fluid.layers.data(name='update3', - shape=[5, 5, 8], - dtype='float32', - append_batch_size=False) + index3 = fluid.layers.data( + name='index3', + shape=[5, 5, 8, 5], + dtype='int32', + append_batch_size=False, + ) + updates3 = fluid.layers.data( + name='update3', + shape=[5, 5, 8], + dtype='float32', + append_batch_size=False, + ) output3 = fluid.layers.scatter_nd(index3, updates3, shape3) def testcase4(self): shape4 = [10, 9, 8, 1, 3] - index4 = fluid.layers.data(name='index4', - shape=[5, 5, 8, 5], - dtype='int32', - append_batch_size=False) - updates4 = fluid.layers.data(name='update4', - shape=[5, 5, 8], - dtype='double', - append_batch_size=False) - output4 = fluid.layers.scatter_nd(index4, - updates4, - shape4, - name='scatter_nd') + index4 = fluid.layers.data( + name='index4', + shape=[5, 5, 8, 5], + dtype='int32', + append_batch_size=False, + ) + updates4 = fluid.layers.data( + name='update4', + shape=[5, 5, 8], + dtype='double', + append_batch_size=False, + ) + output4 = fluid.layers.scatter_nd( + index4, updates4, shape4, name='scatter_nd' + ) def testcase5(self): if not fluid.core.is_compiled_with_cuda(): @@ -235,27 +254,32 @@ def testcase5(self): with fluid.dygraph.guard(): device = paddle.get_device() paddle.set_device('gpu') - gpu_value = paddle.scatter_nd_add(paddle.to_tensor(x), - paddle.to_tensor(index), - paddle.to_tensor(val)) + gpu_value = paddle.scatter_nd_add( + paddle.to_tensor(x), + paddle.to_tensor(index), + paddle.to_tensor(val), + ) paddle.set_device('cpu') - cpu_value = paddle.scatter_nd_add(paddle.to_tensor(x), - paddle.to_tensor(index), - paddle.to_tensor(val)) + cpu_value = paddle.scatter_nd_add( + paddle.to_tensor(x), + paddle.to_tensor(index), + paddle.to_tensor(val), + ) np.testing.assert_array_equal(gpu_value.numpy(), cpu_value.numpy()) paddle.set_device(device) @switch_to_static_graph def test_static_graph(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x_t = paddle.static.data(name="x", dtype=x.dtype, shape=x.shape) - index_t = paddle.static.data(name="index", - dtype=index.dtype, - shape=index.shape) - val_t = paddle.static.data(name="val", - dtype=val.dtype, - shape=val.shape) + index_t = paddle.static.data( + name="index", dtype=index.dtype, shape=index.shape + ) + val_t = paddle.static.data( + name="val", dtype=val.dtype, shape=val.shape + ) out_t = paddle.scatter_nd_add(x_t, index_t, val_t) feed = {x_t.name: x, index_t.name: index, val_t.name: val} fetch = [out_t] @@ -269,26 +293,23 @@ def test_static_graph(): test_static_graph() -#Test Raise Error +# Test Raise Error class TestScatterNdOpRaise(unittest.TestCase): - def test_check_raise(self): - def check_raise_is_test(): try: - ref5 = fluid.layers.data(name='ref5', - shape=[3, 4, 5], - dtype='float32') - index5 = fluid.layers.data(name='index5', - shape=[2, 10], - dtype='int32') - updates5 = fluid.layers.data(name='updates5', - shape=[2, 10], - dtype='float32') + ref5 = fluid.layers.data( + name='ref5', shape=[3, 4, 5], dtype='float32' + ) + index5 = fluid.layers.data( + name='index5', shape=[2, 10], dtype='int32' + ) + updates5 = fluid.layers.data( + name='updates5', shape=[2, 10], dtype='float32' + ) output5 = fluid.layers.scatter_nd_add(ref5, index5, updates5) except Exception as e: - t = \ - "The last dimension of Input(Index)'s shape should be no greater " + t = "The last dimension of Input(Index)'s shape should be no greater " if t in str(e): raise IndexError @@ -296,35 +317,39 @@ def check_raise_is_test(): def test_check_raise2(self): with self.assertRaises(ValueError): - ref6 = fluid.layers.data(name='ref6', - shape=[10, 9, 8, 1, 3], - dtype='double', - append_batch_size=False) - index6 = fluid.layers.data(name='index6', - shape=[5, 8, 5], - dtype='int32', - append_batch_size=False) - updates6 = fluid.layers.data(name='update6', - shape=[5, 8], - dtype='float32', - append_batch_size=False) + ref6 = fluid.layers.data( + name='ref6', + shape=[10, 9, 8, 1, 3], + dtype='double', + append_batch_size=False, + ) + index6 = fluid.layers.data( + name='index6', + shape=[5, 8, 5], + dtype='int32', + append_batch_size=False, + ) + updates6 = fluid.layers.data( + name='update6', + shape=[5, 8], + dtype='float32', + append_batch_size=False, + ) output6 = fluid.layers.scatter_nd_add(ref6, index6, updates6) def test_check_raise3(self): - def check_raise_is_test(): try: shape = [3, 4, 5] - index7 = fluid.layers.data(name='index7', - shape=[2, 1], - dtype='int32') - updates7 = fluid.layers.data(name='updates7', - shape=[2, 4, 5, 20], - dtype='float32') + index7 = fluid.layers.data( + name='index7', shape=[2, 1], dtype='int32' + ) + updates7 = fluid.layers.data( + name='updates7', shape=[2, 4, 5, 20], dtype='float32' + ) output7 = fluid.layers.scatter_nd(index7, updates7, shape) except Exception as e: - t = \ - "Updates has wrong shape" + t = "Updates has wrong shape" if t in str(e): raise ValueError @@ -332,7 +357,6 @@ def check_raise_is_test(): class TestDygraph(unittest.TestCase): - def test_dygraph(self): with fluid.dygraph.guard(fluid.CPUPlace()): index_data = np.array([[1, 1], [0, 1], [1, 3]]).astype(np.int64) diff --git a/python/paddle/fluid/tests/unittests/test_scatter_op.py b/python/paddle/fluid/tests/unittests/test_scatter_op.py index 05aba63fbfe83..479498bfbb451 100644 --- a/python/paddle/fluid/tests/unittests/test_scatter_op.py +++ b/python/paddle/fluid/tests/unittests/test_scatter_op.py @@ -23,7 +23,6 @@ class TestScatterOp(OpTest): - def setUp(self): self.op_type = "scatter" self.python_api = paddle.scatter @@ -43,7 +42,6 @@ def test_check_grad(self): class TestScatterOp0(OpTest): - def setUp(self): self.op_type = "scatter" self.python_api = paddle.scatter @@ -64,7 +62,6 @@ def test_check_grad(self): class TestScatterOp1(OpTest): - def setUp(self): self.op_type = "scatter" self.python_api = paddle.scatter @@ -87,10 +84,10 @@ def test_check_grad(self): self.check_grad(["X", "Updates"], "Out", check_eager=False) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestScatterOp2(OpTest): - def setUp(self): self.op_type = "scatter" self.python_api = paddle.scatter @@ -110,15 +107,15 @@ def test_check_output(self): def test_check_grad(self): if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X', 'Updates'], - 'Out', - check_eager=False) + self.check_grad_with_place( + place, ['X', 'Updates'], 'Out', check_eager=False + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestScatterOp3(OpTest): - def setUp(self): self.op_type = "scatter" self.python_api = paddle.scatter @@ -142,13 +139,12 @@ def test_check_output(self): def test_check_grad(self): if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X', 'Updates'], - 'Out', - check_eager=False) + self.check_grad_with_place( + place, ['X', 'Updates'], 'Out', check_eager=False + ) class TestScatterOp4(OpTest): - def setUp(self): self.op_type = "scatter" self.python_api = paddle.scatter @@ -167,10 +163,10 @@ def test_check_grad(self): self.check_grad(['X', 'Updates'], 'Out', check_eager=False) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestScatterOp5(OpTest): - def setUp(self): self.op_type = "scatter" self.python_api = paddle.scatter @@ -190,13 +186,12 @@ def test_check_output(self): def test_check_grad(self): if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X', 'Updates'], - 'Out', - check_eager=False) + self.check_grad_with_place( + place, ['X', 'Updates'], 'Out', check_eager=False + ) class TestScatterAPI(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -215,19 +210,26 @@ def check_static_result(self, place): input_data = np.array([[1, 1], [2, 2], [3, 3]]).astype(np.float64) index_data = np.array([2, 1, 0, 1]).astype(np.int64) - updates_data = np.array([[1, 1], [2, 2], [3, 3], - [4, 4]]).astype(np.float64) + updates_data = np.array([[1, 1], [2, 2], [3, 3], [4, 4]]).astype( + np.float64 + ) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input": input_data, - "index": index_data, - "updates": updates_data - }, - fetch_list=[result]) - self.assertEqual((fetches[0] == \ - np.array([[3., 3.],[6., 6.],[1., 1.]])).all(), True) + fetches = exe.run( + fluid.default_main_program(), + feed={ + "input": input_data, + "index": index_data, + "updates": updates_data, + }, + fetch_list=[result], + ) + self.assertEqual( + ( + fetches[0] == np.array([[3.0, 3.0], [6.0, 6.0], [1.0, 1.0]]) + ).all(), + True, + ) def test_static(self): for place in self.places: @@ -238,16 +240,22 @@ def test_dygraph(self): with fluid.dygraph.guard(place): x_data = np.array([[1, 1], [2, 2], [3, 3]]).astype(np.float64) index_data = np.array([2, 1, 0, 1]).astype(np.int64) - updates_data = np.array([[1, 1], [2, 2], [3, 3], - [4, 4]]).astype(np.float64) + updates_data = np.array( + [[1, 1], [2, 2], [3, 3], [4, 4]] + ).astype(np.float64) x = fluid.dygraph.to_variable(x_data) index = fluid.dygraph.to_variable(index_data) updates = fluid.dygraph.to_variable(updates_data) output1 = self.scatter(x, index, updates, overwrite=False) - self.assertEqual((output1.numpy() == \ - np.array([[3., 3.],[6., 6.],[1., 1.]])).all(), True) + self.assertEqual( + ( + output1.numpy() + == np.array([[3.0, 3.0], [6.0, 6.0], [1.0, 1.0]]) + ).all(), + True, + ) def test_large_data(self): if os.name == "nt" or not paddle.is_compiled_with_cuda(): @@ -259,27 +267,30 @@ def test_large_data(self): def test_dygraph(): with fluid.dygraph.guard(): - gpu_out = paddle.scatter(paddle.to_tensor(x), - paddle.to_tensor(index), - paddle.to_tensor(updates)) + gpu_out = paddle.scatter( + paddle.to_tensor(x), + paddle.to_tensor(index), + paddle.to_tensor(updates), + ) return gpu_out.numpy() @switch_to_static_graph def test_static_graph(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x_t = paddle.static.data(name="x", dtype=x.dtype, shape=x.shape) - index_t = paddle.static.data(name="index", - dtype=index.dtype, - shape=index.shape) - updates_t = paddle.static.data(name="updates", - dtype=updates.dtype, - shape=updates.shape) + index_t = paddle.static.data( + name="index", dtype=index.dtype, shape=index.shape + ) + updates_t = paddle.static.data( + name="updates", dtype=updates.dtype, shape=updates.shape + ) out_t = paddle.scatter(x_t, index_t, updates_t) feed = { x_t.name: x, index_t.name: index, - updates_t.name: updates + updates_t.name: updates, } fetch = [out_t] @@ -290,10 +301,10 @@ def test_static_graph(): np.testing.assert_array_equal(test_dygraph(), test_static_graph()) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestScatterOpFp16(OpTest): - def setUp(self): self.__class__.op_type = "scatter" self.python_api = paddle.scatter @@ -313,8 +324,9 @@ def setUp(self): self.ref_dx[self.index_np] = zero_np def compute_ref_grad_updates(self): - ref_grad_updates = paddle.gather(paddle.to_tensor(self.dout_np), - paddle.to_tensor(self.index_np)) + ref_grad_updates = paddle.gather( + paddle.to_tensor(self.dout_np), paddle.to_tensor(self.index_np) + ) return ref_grad_updates def test_scatter_fp16(self): @@ -323,21 +335,22 @@ def test_scatter_fp16(self): index_tensor = paddle.to_tensor(self.index_np) updates_tensor = paddle.to_tensor(self.updates_np, stop_gradient=False) out_tensor = paddle.scatter(x_tensor, index_tensor, updates_tensor) - paddle.autograd.backward([out_tensor], [paddle.to_tensor(self.dout_np)], - retain_graph=True) + paddle.autograd.backward( + [out_tensor], [paddle.to_tensor(self.dout_np)], retain_graph=True + ) ref_grad_updates = self.compute_ref_grad_updates() - np.testing.assert_allclose(ref_grad_updates.numpy(), - updates_tensor.grad.numpy(), - rtol=1e-5, - atol=1e-5) - np.testing.assert_allclose(self.ref_dx, - x_tensor.grad.numpy(), - rtol=1e-5, - atol=1e-5) + np.testing.assert_allclose( + ref_grad_updates.numpy(), + updates_tensor.grad.numpy(), + rtol=1e-5, + atol=1e-5, + ) + np.testing.assert_allclose( + self.ref_dx, x_tensor.grad.numpy(), rtol=1e-5, atol=1e-5 + ) class TestScatterInplaceAPI(TestScatterAPI): - def executed_api(self): self.scatter = paddle.scatter_ diff --git a/python/paddle/fluid/tests/unittests/test_scope.py b/python/paddle/fluid/tests/unittests/test_scope.py index ddd26ad1f3b9c..6eaab539fc333 100644 --- a/python/paddle/fluid/tests/unittests/test_scope.py +++ b/python/paddle/fluid/tests/unittests/test_scope.py @@ -17,7 +17,6 @@ class TestScope(unittest.TestCase): - def test_create_destroy(self): paddle_c = paddle.fluid.core scope = paddle_c.Scope() @@ -53,7 +52,8 @@ def test_scope_pool(self): # Delete the scope. scope._remove_from_pool() with self.assertRaisesRegexp( - Exception, "Deleting a nonexistent scope is not allowed*"): + Exception, "Deleting a nonexistent scope is not allowed*" + ): # It is not allowed to delete a nonexistent scope. scope._remove_from_pool() diff --git a/python/paddle/fluid/tests/unittests/test_searchsorted_op.py b/python/paddle/fluid/tests/unittests/test_searchsorted_op.py index e63107026e680..d185fbaa110d0 100644 --- a/python/paddle/fluid/tests/unittests/test_searchsorted_op.py +++ b/python/paddle/fluid/tests/unittests/test_searchsorted_op.py @@ -23,7 +23,6 @@ class TestSearchSorted(OpTest): - def setUp(self): self.python_api = paddle.searchsorted self.op_type = "searchsorted" @@ -31,13 +30,14 @@ def setUp(self): self.inputs = { 'SortedSequence': self.sorted_sequence, - 'Values': self.values + 'Values': self.values, } self.attrs = {"out_int32": False, "right": False} self.attrs["right"] = True if self.side == 'right' else False self.outputs = { - 'Out': - np.searchsorted(self.sorted_sequence, self.values, side=self.side) + 'Out': np.searchsorted( + self.sorted_sequence, self.values, side=self.side + ) } def test_check_output(self): @@ -50,7 +50,6 @@ def init_test_case(self): class TestSearchSortedOp1(TestSearchSorted): - def init_test_case(self): self.sorted_sequence = np.array([1, 3, 5, 7, 9]).astype("int32") self.values = np.array([[3, 6, 9], [3, 6, 9]]).astype("int32") @@ -58,7 +57,6 @@ def init_test_case(self): class TestSearchSortedOp2(TestSearchSorted): - def init_test_case(self): self.sorted_sequence = np.array([1, 3, 5, 7, 9]).astype("int64") self.values = np.array([[3, 6, 9], [3, 6, 9]]).astype("int64") @@ -66,34 +64,33 @@ def init_test_case(self): class TestSearchSortedOp3(TestSearchSorted): - def init_test_case(self): self.sorted_sequence = np.array([1, 3, 5, 7, 9]).astype("float64") - self.values = np.array([[np.nan, np.nan, np.nan], - [3, 6, 9]]).astype("float64") + self.values = np.array([[np.nan, np.nan, np.nan], [3, 6, 9]]).astype( + "float64" + ) self.side = "left" class TestSearchSortedOp4(TestSearchSorted): - def init_test_case(self): self.sorted_sequence = np.array([1, 3, 5, 7, 9]).astype("float64") - self.values = np.array([[np.inf, np.inf, np.inf], - [3, 6, 9]]).astype("float64") + self.values = np.array([[np.inf, np.inf, np.inf], [3, 6, 9]]).astype( + "float64" + ) self.side = "right" class TestSearchSortedOp5(TestSearchSorted): - def init_test_case(self): self.sorted_sequence = np.array([1, 3, 5, 7, 9]).astype("float64") - self.values = np.array([[np.inf, np.inf, np.inf], - [np.nan, np.nan, np.nan]]).astype("float64") + self.values = np.array( + [[np.inf, np.inf, np.inf], [np.nan, np.nan, np.nan]] + ).astype("float64") self.side = "right" class TestSearchSortedAPI(unittest.TestCase): - def init_test_case(self): self.sorted_sequence = np.array([2, 4, 6, 8, 10]).astype("float64") self.values = np.array([[3, 6, 9], [3, 6, 9]]).astype("float64") @@ -112,17 +109,20 @@ def run(place): sorted_sequence = paddle.static.data( 'SortedSequence', shape=self.sorted_sequence.shape, - dtype="float64") - values = paddle.static.data('Values', - shape=self.values.shape, - dtype="float64") + dtype="float64", + ) + values = paddle.static.data( + 'Values', shape=self.values.shape, dtype="float64" + ) out = paddle.searchsorted(sorted_sequence, values) exe = paddle.static.Executor(place) - res, = exe.run(feed={ - 'SortedSequence': self.sorted_sequence, - 'Values': self.values - }, - fetch_list=out) + (res,) = exe.run( + feed={ + 'SortedSequence': self.sorted_sequence, + 'Values': self.values, + }, + fetch_list=out, + ) out_ref = np.searchsorted(self.sorted_sequence, self.values) np.testing.assert_allclose(out_ref, res, rtol=1e-05) @@ -130,16 +130,15 @@ def run(place): run(place) def test_dygraph_api(self): - def run(place): paddle.disable_static(place) sorted_sequence = paddle.to_tensor(self.sorted_sequence) values = paddle.to_tensor(self.values) out = paddle.searchsorted(sorted_sequence, values, right=True) - out_ref = np.searchsorted(self.sorted_sequence, - self.values, - side='right') + out_ref = np.searchsorted( + self.sorted_sequence, self.values, side='right' + ) np.testing.assert_allclose(out_ref, out.numpy(), rtol=1e-05) paddle.enable_static() @@ -155,60 +154,61 @@ def test_out_int32(self): class TestSearchSortedError(unittest.TestCase): - def test_error_api(self): paddle.enable_static() def test_searchsorted_dims_matched_before_lastdim_error1(): with paddle.static.program_guard(paddle.static.Program()): - sorted_sequence = paddle.static.data('SortedSequence', - shape=[2, 2, 3], - dtype="float64") - values = paddle.static.data('Values', - shape=[2, 5], - dtype="float64") + sorted_sequence = paddle.static.data( + 'SortedSequence', shape=[2, 2, 3], dtype="float64" + ) + values = paddle.static.data( + 'Values', shape=[2, 5], dtype="float64" + ) out = paddle.searchsorted(sorted_sequence, values) - self.assertRaises(RuntimeError, - test_searchsorted_dims_matched_before_lastdim_error1) + self.assertRaises( + RuntimeError, test_searchsorted_dims_matched_before_lastdim_error1 + ) def test_searchsorted_dims_matched_before_lastdim_error2(): with paddle.static.program_guard(paddle.static.Program()): - sorted_sequence = paddle.static.data('SortedSequence', - shape=[2, 2, 3], - dtype="float64") - values = paddle.static.data('Values', - shape=[2, 3, 5], - dtype="float64") + sorted_sequence = paddle.static.data( + 'SortedSequence', shape=[2, 2, 3], dtype="float64" + ) + values = paddle.static.data( + 'Values', shape=[2, 3, 5], dtype="float64" + ) out = paddle.searchsorted(sorted_sequence, values) - self.assertRaises(RuntimeError, - test_searchsorted_dims_matched_before_lastdim_error2) + self.assertRaises( + RuntimeError, test_searchsorted_dims_matched_before_lastdim_error2 + ) def test_searchsorted_sortedsequence_size_error(): with paddle.static.program_guard(paddle.static.Program()): - sorted_sequence = paddle.static.data('SortedSequence', - shape=[2, 2, - pow(2, 34)], - dtype="float64") - values = paddle.static.data('Values', - shape=[2, 2, 5], - dtype="float64") - out = paddle.searchsorted(sorted_sequence, - values, - out_int32=True) - - self.assertRaises(RuntimeError, - test_searchsorted_sortedsequence_size_error) + sorted_sequence = paddle.static.data( + 'SortedSequence', shape=[2, 2, pow(2, 34)], dtype="float64" + ) + values = paddle.static.data( + 'Values', shape=[2, 2, 5], dtype="float64" + ) + out = paddle.searchsorted( + sorted_sequence, values, out_int32=True + ) + + self.assertRaises( + RuntimeError, test_searchsorted_sortedsequence_size_error + ) def test_sortedsequence_values_type_error(): with paddle.static.program_guard(paddle.static.Program()): - sorted_sequence = paddle.static.data('SortedSequence', - shape=[2, 3], - dtype="int16") - values = paddle.static.data('Values', - shape=[2, 5], - dtype="int16") + sorted_sequence = paddle.static.data( + 'SortedSequence', shape=[2, 3], dtype="int16" + ) + values = paddle.static.data( + 'Values', shape=[2, 5], dtype="int16" + ) out = paddle.searchsorted(sorted_sequence, values) self.assertRaises(TypeError, test_sortedsequence_values_type_error) diff --git a/python/paddle/fluid/tests/unittests/test_seed_op.py b/python/paddle/fluid/tests/unittests/test_seed_op.py index 8a1933fbe6e86..14aa2c4f4dd75 100644 --- a/python/paddle/fluid/tests/unittests/test_seed_op.py +++ b/python/paddle/fluid/tests/unittests/test_seed_op.py @@ -22,7 +22,6 @@ class TestSeedOpFixSeed(OpTest): - def setUp(self): self.op_type = "seed" self.inputs = {} @@ -34,7 +33,6 @@ def test_check_output(self): class TestSeedOpDiffSeed(OpTest): - def setUp(self): self.op_type = "seed" self.inputs = {} @@ -46,7 +44,6 @@ def test_check_output(self): class TestDropoutWithRandomSeedGenerator(unittest.TestCase): - def setUp(self): paddle.framework.random.set_random_seed_generator('seed0', 123) paddle.framework.random.set_random_seed_generator('seed1', 123) @@ -58,14 +55,16 @@ def setUp(self): def check_static_result(self, place): import paddle.distributed.fleet.meta_parallel.parallel_layers.random as random + with static.program_guard(static.Program(), static.Program()): res1 = random.determinate_seed('seed0') exe = static.Executor(place) res_list = [res1] for i in range(2): - out1, = exe.run(static.default_main_program(), - fetch_list=res_list) + (out1,) = exe.run( + static.default_main_program(), fetch_list=res_list + ) self.assertEqual(out1, np.cast['int32'](self.rng1.random())) def test_static(self): diff --git a/python/paddle/fluid/tests/unittests/test_segment_ops.py b/python/paddle/fluid/tests/unittests/test_segment_ops.py index 71672723e82e9..1c6fe88ca77a1 100644 --- a/python/paddle/fluid/tests/unittests/test_segment_ops.py +++ b/python/paddle/fluid/tests/unittests/test_segment_ops.py @@ -83,7 +83,6 @@ def segment_pool_split(X, SegmentIds, pooltype): class TestSegmentOps(OpTest): - def set_data(self): x = np.random.uniform(-1, 1, self.shape).astype(self.dtype) segment_ids = self.set_segment(len(x), len(x) // 5 + 1) @@ -112,7 +111,7 @@ def setUp(self): result = self.compute(x, segment_ids) self.inputs = { 'X': x.astype(self.dtype), - 'SegmentIds': segment_ids.astype(np.int64) + 'SegmentIds': segment_ids.astype(np.int64), } self.outputs = {'Out': result.astype(self.dtype)} @@ -124,7 +123,6 @@ def test_check_grad(self): class TestSegmentSum2(TestSegmentOps): - def prepare(self): super(TestSegmentSum2, self).prepare() self.shape = [40, 20] @@ -136,13 +134,12 @@ def setUp(self): result = self.compute(x, segment_ids) self.inputs = { 'X': x.astype(self.dtype), - 'SegmentIds': segment_ids.astype(np.int32) + 'SegmentIds': segment_ids.astype(np.int32), } self.outputs = {'Out': result.astype(self.dtype)} class TestSegmentMax(TestSegmentOps): - def compute(self, x, segment_ids): return compute_segment_min_max(x, segment_ids, pooltype="MAX") @@ -157,7 +154,7 @@ def setUp(self): result, self.gradient = self.compute(x, segment_ids) self.inputs = { 'X': x.astype(self.dtype), - 'SegmentIds': segment_ids.astype(np.int32) + 'SegmentIds': segment_ids.astype(np.int32), } self.outputs = {'Out': result.astype(self.dtype)} @@ -166,14 +163,12 @@ def test_check_grad(self): class TestSegmentMax2(TestSegmentMax): - def prepare(self): super(TestSegmentMax2, self).prepare() self.dtype = np.float32 class TestSegmentMin(TestSegmentMax): - def compute(self, x, segment_ids): return compute_segment_min_max(x, segment_ids, pooltype="MIN") @@ -183,14 +178,12 @@ def prepare(self): class TestSegmentMin2(TestSegmentMin): - def prepare(self): super(TestSegmentMin2, self).prepare() self.dtype = np.float32 class TestSegmentMean(TestSegmentOps): - def compute(self, x, segment_ids): return compute_segment_mean(x, segment_ids) @@ -205,16 +198,14 @@ def setUp(self): result = self.compute(x, segment_ids) self.inputs = {'X': x, 'SegmentIds': segment_ids} self.outputs = { - 'Out': - result, - 'SummedIds': - compute_segment_sum( - np.ones([len(x), 1]).astype(self.dtype), segment_ids) + 'Out': result, + 'SummedIds': compute_segment_sum( + np.ones([len(x), 1]).astype(self.dtype), segment_ids + ), } class TestSegmentMean2(TestSegmentMean): - def prepare(self): super(TestSegmentMean2, self).prepare() self.dtype = np.float32 @@ -223,7 +214,6 @@ def prepare(self): class API_SegmentOpsTest(unittest.TestCase): - def test_static(self): with paddle.static.program_guard(paddle.static.Program()): x = paddle.static.data(name="x", shape=[3, 3], dtype="float32") @@ -243,11 +233,10 @@ def test_static(self): np_max = np.array([[3, 2, 3], [4, 5, 6]], dtype="float32") np_min = np.array([[1, 2, 1], [4, 5, 6]], dtype="float32") - ret = exe.run(feed={ - 'x': data1, - 'y': data2 - }, - fetch_list=[res_sum, res_mean, res_max, res_min]) + ret = exe.run( + feed={'x': data1, 'y': data2}, + fetch_list=[res_sum, res_mean, res_max, res_min], + ) for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): np.testing.assert_allclose(np_res, ret_res, rtol=1e-05, atol=1e-06) @@ -255,8 +244,9 @@ def test_static(self): def test_dygraph(self): device = paddle.CPUPlace() with paddle.fluid.dygraph.guard(device): - x = paddle.to_tensor([[1, 2, 3], [3, 2, 1], [4, 5, 6]], - dtype='float32') + x = paddle.to_tensor( + [[1, 2, 3], [3, 2, 1], [4, 5, 6]], dtype='float32' + ) y = paddle.to_tensor([0, 0, 1], dtype="int32") res_sum = paddle.incubate.segment_sum(x, y) res_mean = paddle.incubate.segment_mean(x, y) @@ -271,14 +261,12 @@ def test_dygraph(self): ret = [res_sum, res_mean, res_max, res_min] for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): - np.testing.assert_allclose(np_res, - ret_res.numpy(), - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np_res, ret_res.numpy(), rtol=1e-05, atol=1e-06 + ) class API_GeometricSegmentOpsTest(unittest.TestCase): - def test_static(self): with paddle.static.program_guard(paddle.static.Program()): x = paddle.static.data(name="x", shape=[3, 3], dtype="float32") @@ -298,11 +286,10 @@ def test_static(self): np_max = np.array([[3, 2, 3], [4, 5, 6]], dtype="float32") np_min = np.array([[1, 2, 1], [4, 5, 6]], dtype="float32") - ret = exe.run(feed={ - 'x': data1, - 'y': data2 - }, - fetch_list=[res_sum, res_mean, res_max, res_min]) + ret = exe.run( + feed={'x': data1, 'y': data2}, + fetch_list=[res_sum, res_mean, res_max, res_min], + ) for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): np.testing.assert_allclose(np_res, ret_res, rtol=1e-05, atol=1e-06) @@ -310,8 +297,9 @@ def test_static(self): def test_dygraph(self): device = paddle.CPUPlace() with paddle.fluid.dygraph.guard(device): - x = paddle.to_tensor([[1, 2, 3], [3, 2, 1], [4, 5, 6]], - dtype='float32') + x = paddle.to_tensor( + [[1, 2, 3], [3, 2, 1], [4, 5, 6]], dtype='float32' + ) y = paddle.to_tensor([0, 0, 1], dtype="int32") res_sum = paddle.geometric.segment_sum(x, y) res_mean = paddle.geometric.segment_mean(x, y) @@ -326,16 +314,16 @@ def test_dygraph(self): ret = [res_sum, res_mean, res_max, res_min] for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): - np.testing.assert_allclose(np_res, - ret_res.numpy(), - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np_res, ret_res.numpy(), rtol=1e-05, atol=1e-06 + ) def test_dygraph_cpu_float16(self): device = paddle.CPUPlace() with paddle.fluid.dygraph.guard(device): - x = paddle.to_tensor([[1, 2, 3], [3, 2, 1], [4, 5, 6]], - dtype='float16') + x = paddle.to_tensor( + [[1, 2, 3], [3, 2, 1], [4, 5, 6]], dtype='float16' + ) y = paddle.to_tensor([0, 0, 1], dtype="int32") res_sum = paddle.geometric.segment_sum(x, y) res_mean = paddle.geometric.segment_mean(x, y) @@ -349,17 +337,17 @@ def test_dygraph_cpu_float16(self): ret = [res_sum, res_mean, res_max, res_min] for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): - np.testing.assert_allclose(np_res, - ret_res.numpy(), - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np_res, ret_res.numpy(), rtol=1e-05, atol=1e-06 + ) def test_dygraph_cuda_float16(self): if core.is_compiled_with_cuda(): device = paddle.CUDAPlace(0) with paddle.fluid.dygraph.guard(device): - x = paddle.to_tensor([[1, 2, 3], [3, 2, 1], [4, 5, 6]], - dtype='float16') + x = paddle.to_tensor( + [[1, 2, 3], [3, 2, 1], [4, 5, 6]], dtype='float16' + ) y = paddle.to_tensor([0, 0, 1], dtype="int32") res_sum = paddle.geometric.segment_sum(x, y) res_mean = paddle.geometric.segment_mean(x, y) @@ -374,10 +362,9 @@ def test_dygraph_cuda_float16(self): ret = [res_sum, res_mean, res_max, res_min] for np_res, ret_res in zip([np_sum, np_mean, np_max, np_min], ret): - np.testing.assert_allclose(np_res, - ret_res.numpy(), - rtol=1e-05, - atol=1e-06) + np.testing.assert_allclose( + np_res, ret_res.numpy(), rtol=1e-05, atol=1e-06 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_select_input_output_op.py b/python/paddle/fluid/tests/unittests/test_select_input_output_op.py index 119bcc0ee5a3e..eeab0ca874d96 100644 --- a/python/paddle/fluid/tests/unittests/test_select_input_output_op.py +++ b/python/paddle/fluid/tests/unittests/test_select_input_output_op.py @@ -27,7 +27,6 @@ class TestSplitMergeSelectedVarOps(unittest.TestCase): - def test_forward_backward_list_output(self): for branch_num in range(2, 10): program = Program() @@ -41,7 +40,8 @@ def test_forward_backward_list_output(self): out = program.current_block().create_var( dtype='float32', shape=[2], - type=core.VarDesc.VarType.LOD_TENSOR) + type=core.VarDesc.VarType.LOD_TENSOR, + ) outputs.append(out) select_output(x, outputs, mask) @@ -49,30 +49,31 @@ def test_forward_backward_list_output(self): mean = paddle.mean(y) append_backward(mean) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = Executor(place) feed_x = np.asarray([1.3, -1.4]).astype(np.float32) for i in range(branch_num): feed_mask = np.asarray([i]).astype(np.int32) - ret = exe.run(program, - feed={ - 'x': feed_x, - 'mask': feed_mask - }, - fetch_list=[y.name, x.grad_name]) + ret = exe.run( + program, + feed={'x': feed_x, 'mask': feed_mask}, + fetch_list=[y.name, x.grad_name], + ) x_grad = np.asarray([0.5, 0.5]).astype(np.float32) - np.testing.assert_allclose(np.asarray(ret[0]), - feed_x, - rtol=1e-05) - np.testing.assert_allclose(np.asarray(ret[1]), - x_grad, - rtol=1e-05) + np.testing.assert_allclose( + np.asarray(ret[0]), feed_x, rtol=1e-05 + ) + np.testing.assert_allclose( + np.asarray(ret[1]), x_grad, rtol=1e-05 + ) class TestSelectInputOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): mask = layers.data(name='mask', shape=[1], dtype='int32') @@ -99,17 +100,16 @@ def test_mask_dtype(): class TestSelectOutput_Error(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): in1 = layers.data(name='in1', shape=[1], dtype='int32') - mask_int32 = layers.data(name='mask_int32', - shape=[1], - dtype='int32') - mask_float32 = layers.data(name='mask_float32', - shape=[1], - dtype='float32') + mask_int32 = layers.data( + name='mask_int32', shape=[1], dtype='int32' + ) + mask_float32 = layers.data( + name='mask_float32', shape=[1], dtype='float32' + ) out1 = layers.data(name='out1', shape=[1], dtype='int32') # 1. The type of input in select_output must Variable. diff --git a/python/paddle/fluid/tests/unittests/test_selected_rows.py b/python/paddle/fluid/tests/unittests/test_selected_rows.py index 4eac57f834bcf..a66dc40dfb1f7 100644 --- a/python/paddle/fluid/tests/unittests/test_selected_rows.py +++ b/python/paddle/fluid/tests/unittests/test_selected_rows.py @@ -18,7 +18,6 @@ class TestSelectedRows(unittest.TestCase): - def test_selected_rows(self): place = core.CPUPlace() height = 10 @@ -40,13 +39,16 @@ def test_selected_rows(self): self.assertEqual(10, selected_rows.height()) # compare tensor - self.assertAlmostEqual(2.0, - selected_rows.get_tensor()._get_float_element(0)) - self.assertAlmostEqual(1.0, - selected_rows.get_tensor()._get_float_element(1)) + self.assertAlmostEqual( + 2.0, selected_rows.get_tensor()._get_float_element(0) + ) + self.assertAlmostEqual( + 1.0, selected_rows.get_tensor()._get_float_element(1) + ) self.assertAlmostEqual( 4.0, - selected_rows.get_tensor()._get_float_element(2 * row_numel + 8)) + selected_rows.get_tensor()._get_float_element(2 * row_numel + 8), + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_selu_op.py b/python/paddle/fluid/tests/unittests/test_selu_op.py index 5de31d26b2b08..1390ddb5c9606 100644 --- a/python/paddle/fluid/tests/unittests/test_selu_op.py +++ b/python/paddle/fluid/tests/unittests/test_selu_op.py @@ -21,9 +21,11 @@ import paddle.nn.functional as F -def ref_selu(x, - scale=1.0507009873554804934193349852946, - alpha=1.6732632423543772848170429916717): +def ref_selu( + x, + scale=1.0507009873554804934193349852946, + alpha=1.6732632423543772848170429916717, +): out = np.copy(x) out_flat = out.flatten() for i in range(out_flat.size): @@ -35,7 +37,6 @@ def ref_selu(x, class SeluTest(OpTest): - def setUp(self): self.op_type = "selu" self.python_api = paddle.nn.functional.selu @@ -85,8 +86,11 @@ def setUp(self): # Since zero point in selu is not differentiable, avoid randomize # zero. self.x_np[np.abs(self.x_np) < 0.005] = 0.02 - self.place=paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): with paddle.static.program_guard(paddle.static.Program()): @@ -125,21 +129,21 @@ def test_errors(self): # The input type must be Variable. self.assertRaises(TypeError, F.selu, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[12, 10], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[12, 10], dtype='int32' + ) self.assertRaises(TypeError, F.selu, x_int32) # The scale must be greater than 1.0 - x_fp32 = paddle.fluid.data(name='x_fp32', - shape=[12, 10], - dtype='float32') + x_fp32 = paddle.fluid.data( + name='x_fp32', shape=[12, 10], dtype='float32' + ) self.assertRaises(ValueError, F.selu, x_fp32, -1.0) # The alpha must be no less than 0 self.assertRaises(ValueError, F.selu, x_fp32, 1.6, -1.0) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[12, 10], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[12, 10], dtype='float16' + ) F.selu(x_fp16) diff --git a/python/paddle/fluid/tests/unittests/test_set_bool_attr.py b/python/paddle/fluid/tests/unittests/test_set_bool_attr.py index a089563b6ecd0..2c233141d095e 100644 --- a/python/paddle/fluid/tests/unittests/test_set_bool_attr.py +++ b/python/paddle/fluid/tests/unittests/test_set_bool_attr.py @@ -17,18 +17,19 @@ class TestAttrSet(unittest.TestCase): - def test_set_bool_attr(self): x = fluid.layers.data(name='x', shape=[3, 7, 3, 7], dtype='float32') param_attr = fluid.ParamAttr( name='batch_norm_w', - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0), + ) bias_attr = fluid.ParamAttr( name='batch_norm_b', - initializer=fluid.initializer.Constant(value=0.0)) - bn = fluid.layers.batch_norm(input=x, - param_attr=param_attr, - bias_attr=bias_attr) + initializer=fluid.initializer.Constant(value=0.0), + ) + bn = fluid.layers.batch_norm( + input=x, param_attr=param_attr, bias_attr=bias_attr + ) block = fluid.default_main_program().desc.block(0) op = block.op(0) before_type = op.attr_type('is_test') diff --git a/python/paddle/fluid/tests/unittests/test_set_value_op.py b/python/paddle/fluid/tests/unittests/test_set_value_op.py index 7f6b9831aaee6..92dc8b8fadcb7 100644 --- a/python/paddle/fluid/tests/unittests/test_set_value_op.py +++ b/python/paddle/fluid/tests/unittests/test_set_value_op.py @@ -23,8 +23,8 @@ from functools import reduce from paddle.fluid.framework import _test_eager_guard -class TestSetValueBase(unittest.TestCase): +class TestSetValueBase(unittest.TestCase): def setUp(self): paddle.enable_static() self.set_dtype() @@ -50,7 +50,6 @@ def _get_answer(self): class TestSetValueApi(TestSetValueBase): - def _run_static(self): paddle.enable_static() with paddle.static.program_guard(self.program): @@ -75,11 +74,17 @@ def func_test_api(self): dynamic_out = self._run_dynamic() self._get_answer() - error_msg = "\nIn {} mode: \nExpected res = \n{}, \n\nbut received : \n{}" - self.assertTrue((self.data == static_out).all(), - msg=error_msg.format("static", self.data, static_out)) - self.assertTrue((self.data == dynamic_out).all(), - msg=error_msg.format("dynamic", self.data, dynamic_out)) + error_msg = ( + "\nIn {} mode: \nExpected res = \n{}, \n\nbut received : \n{}" + ) + self.assertTrue( + (self.data == static_out).all(), + msg=error_msg.format("static", self.data, static_out), + ) + self.assertTrue( + (self.data == dynamic_out).all(), + msg=error_msg.format("dynamic", self.data, dynamic_out), + ) def test_api(self): with _test_eager_guard(): @@ -90,7 +95,6 @@ def test_api(self): # 1. Test different type of item: int, Python slice, Paddle Tensor # 1.1 item is int class TestSetValueItemInt(TestSetValueApi): - def _call_setitem(self, x): x[0] = self.value @@ -101,7 +105,6 @@ def _get_answer(self): # 1.2 item is slice # 1.2.1 step is 1 class TestSetValueItemSlice(TestSetValueApi): - def _call_setitem(self, x): x[0:2] = self.value @@ -110,7 +113,6 @@ def _get_answer(self): class TestSetValueItemSlice2(TestSetValueApi): - def _call_setitem(self, x): x[0:-1] = self.value @@ -119,7 +121,6 @@ def _get_answer(self): class TestSetValueItemSlice3(TestSetValueApi): - def _call_setitem(self, x): x[0:-1, 0:2] = self.value @@ -128,7 +129,6 @@ def _get_answer(self): class TestSetValueItemSlice4(TestSetValueApi): - def _call_setitem(self, x): x[0:, 1:2, :] = self.value @@ -137,7 +137,6 @@ def _get_answer(self): class TestSetValueItemSlice5(TestSetValueApi): - def _call_setitem(self, x): x[0:, 1:1, :] = self.value @@ -146,9 +145,7 @@ def _get_answer(self): class TestSetValueItemSliceInWhile(TestSetValueApi): - def _call_setitem(self, x): - def cond(i, x): return i < 1 @@ -157,7 +154,7 @@ def body(i, x): i = i + 1 return i, x - i = paddle.zeros(shape=(1, ), dtype='int32') + i = paddle.zeros(shape=(1,), dtype='int32') i, x = paddle.fluid.layers.while_loop(cond, body, [i, x]) def _get_answer(self): @@ -166,7 +163,6 @@ def _get_answer(self): # 1.2.2 step > 1 class TestSetValueItemSliceStep(TestSetValueApi): - def set_shape(self): self.shape = [5, 5, 5] @@ -178,7 +174,6 @@ def _get_answer(self): class TestSetValueItemSliceStep2(TestSetValueApi): - def set_shape(self): self.shape = [7, 5, 5] @@ -190,7 +185,6 @@ def _get_answer(self): class TestSetValueItemSliceStep3(TestSetValueApi): - def _call_setitem(self, x): x[0:-1, 0:2, ::2] = self.value @@ -199,7 +193,6 @@ def _get_answer(self): class TestSetValueItemSliceStep4(TestSetValueApi): - def _call_setitem(self, x): x[0:, 1:2:2, :] = self.value @@ -209,7 +202,6 @@ def _get_answer(self): # 1.2.3 step < 0 class TestSetValueItemSliceNegetiveStep(TestSetValueApi): - def set_shape(self): self.shape = [5, 2] @@ -224,7 +216,6 @@ def _get_answer(self): class TestSetValueItemSliceNegetiveStep2(TestSetValueApi): - def set_shape(self): self.shape = [5] @@ -239,7 +230,6 @@ def _get_answer(self): class TestSetValueItemSliceNegetiveStep3(TestSetValueApi): - def set_shape(self): self.shape = [3] @@ -254,7 +244,6 @@ def _get_answer(self): class TestSetValueItemSliceNegetiveStep4(TestSetValueApi): - def set_shape(self): self.shape = [3, 4, 5] @@ -269,7 +258,6 @@ def _get_answer(self): class TestSetValueItemEllipsis1(TestSetValueApi): - def _call_setitem(self, x): x[0:, ..., 1:] = self.value @@ -278,7 +266,6 @@ def _get_answer(self): class TestSetValueItemEllipsis2(TestSetValueApi): - def _call_setitem(self, x): x[0:, ...] = self.value @@ -287,7 +274,6 @@ def _get_answer(self): class TestSetValueItemEllipsis3(TestSetValueApi): - def _call_setitem(self, x): x[..., 1:] = self.value @@ -296,7 +282,6 @@ def _get_answer(self): class TestSetValueItemEllipsis4(TestSetValueApi): - def _call_setitem(self, x): x[...] = self.value @@ -306,7 +291,6 @@ def _get_answer(self): # 1.4 item is Paddle Tensor class TestSetValueItemTensor(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") x[zero] = self.value @@ -316,7 +300,6 @@ def _get_answer(self): class TestSetValueItemTensor2(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -327,7 +310,6 @@ def _get_answer(self): class TestSetValueItemTensor3(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -338,7 +320,6 @@ def _get_answer(self): class TestSetValueItemTensor4(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -349,7 +330,6 @@ def _get_answer(self): class TestSetValueItemTensor5(TestSetValueApi): - def _call_setitem(self, x): zero = paddle.full([1], 0, dtype="int32") two = paddle.full([1], 2, dtype="int64") @@ -360,7 +340,6 @@ def _get_answer(self): class TestSetValueItemTensor6(TestSetValueApi): - def set_shape(self): self.shape = [3, 4, 5] @@ -375,7 +354,6 @@ def _get_answer(self): # 1.5 item is None class TestSetValueItemNone1(TestSetValueApi): - def _call_setitem(self, x): x[None] = self.value @@ -384,7 +362,6 @@ def _get_answer(self): class TestSetValueItemNone2(TestSetValueApi): - def _call_setitem(self, x): x[0, None, 1] = self.value @@ -393,7 +370,6 @@ def _get_answer(self): class TestSetValueItemNone3(TestSetValueApi): - def _call_setitem(self, x): x[:, None, None, 1] = self.value @@ -402,7 +378,6 @@ def _get_answer(self): class TestSetValueItemNone4(TestSetValueApi): - def _call_setitem(self, x): x[0, 0, None, 1] = self.value @@ -411,7 +386,6 @@ def _get_answer(self): class TestSetValueItemNone5(TestSetValueApi): - def _call_setitem(self, x): x[0, None, 0, None, 1] = self.value @@ -420,7 +394,6 @@ def _get_answer(self): class TestSetValueItemNone6(TestSetValueApi): - def _call_setitem(self, x): x[None, 0, 0, None, 0] = self.value @@ -429,7 +402,6 @@ def _get_answer(self): class TestSetValueItemNone7(TestSetValueApi): - def _call_setitem(self, x): x[:, None, 1] = np.zeros(self.shape)[:, None, 0] @@ -438,7 +410,6 @@ def _get_answer(self): class TestSetValueItemNone8(TestSetValueApi): - def _call_setitem(self, x): x[:, 1, None] = np.zeros(self.shape)[:, 0, None] @@ -447,7 +418,6 @@ def _get_answer(self): class TestSetValueItemNone9(TestSetValueApi): - def _call_setitem(self, x): x[None, :, 1, ..., None] = np.zeros(self.shape)[0, 0, :, None] @@ -456,7 +426,6 @@ def _get_answer(self): class TestSetValueItemNone10(TestSetValueApi): - def _call_setitem(self, x): x[..., None, :, None] = np.zeros(self.shape)[..., None, :, None] @@ -466,7 +435,6 @@ def _get_answer(self): # 1.5 item is list or Tensor of bol class TestSetValueItemBool1(TestSetValueApi): - def _call_setitem(self, x): x[[True, False]] = self.value @@ -475,7 +443,6 @@ def _get_answer(self): class TestSetValueItemBool2(TestSetValueApi): - def _call_setitem(self, x): x[[False, False]] = self.value @@ -484,7 +451,6 @@ def _get_answer(self): class TestSetValueItemBool3(TestSetValueApi): - def _call_setitem(self, x): x[[False, True]] = np.zeros(self.shape[2]) @@ -493,7 +459,6 @@ def _get_answer(self): class TestSetValueItemBool4(TestSetValueApi): - def _call_setitem(self, x): idx = paddle.assign(np.array([False, True])) x[idx] = np.zeros(self.shape[2]) @@ -503,19 +468,19 @@ def _get_answer(self): class TestSetValueItemBool5(TestSetValueApi): - def _call_setitem(self, x): idx = paddle.assign( - np.array([[False, True, False], [True, True, False]])) + np.array([[False, True, False], [True, True, False]]) + ) x[idx] = self.value def _get_answer(self): - self.data[np.array([[False, True, False], [True, True, - False]])] = self.value + self.data[ + np.array([[False, True, False], [True, True, False]]) + ] = self.value class TestSetValueItemBool6(TestSetValueApi): - def _call_setitem(self, x): x[0, ...] = 0 x[x > 0] = self.value @@ -530,9 +495,7 @@ def _get_answer(self): def create_test_value_int32(parent): - class TestValueInt(parent): - def set_value(self): self.value = 7 @@ -552,9 +515,7 @@ def set_dtype(self): def create_test_value_int64(parent): - class TestValueInt(parent): - def set_value(self): self.value = 7 @@ -574,9 +535,7 @@ def set_dtype(self): def create_test_value_fp16(parent): - class TestValueInt(parent): - def set_value(self): self.value = 3.7 @@ -596,9 +555,7 @@ def set_dtype(self): def create_test_value_fp32(parent): - class TestValueInt(parent): - def set_value(self): self.value = 3.3 @@ -618,9 +575,7 @@ def set_dtype(self): def create_test_value_fp64(parent): - class TestValueInt(parent): - def set_value(self): self.value = 2.0**127 # float32:[-2^128, 2^128) @@ -640,9 +595,7 @@ def set_dtype(self): def create_test_value_bool(parent): - class TestValueInt(parent): - def set_value(self): self.value = 0 @@ -663,9 +616,7 @@ def set_dtype(self): # 2.2 value is numpy.array (int32, int64, float32, float64, bool) def create_test_value_numpy_int32(parent): - class TestValueInt(parent): - def set_value(self): self.value = np.array([5]) @@ -685,9 +636,7 @@ def set_dtype(self): def create_test_value_numpy_int64(parent): - class TestValueInt(parent): - def set_value(self): self.value = np.array([1]) @@ -707,9 +656,7 @@ def set_dtype(self): def create_test_value_numpy_fp32(parent): - class TestValueInt(parent): - def set_value(self): self.value = np.array([1]) @@ -729,9 +676,7 @@ def set_dtype(self): def create_test_value_numpy_fp64(parent): - class TestValueInt(parent): - def set_value(self): self.value = np.array([2**127]).astype("float64") @@ -751,9 +696,7 @@ def set_dtype(self): def create_test_value_numpy_bool(parent): - class TestValueInt(parent): - def set_value(self): self.value = np.array([0]) @@ -774,9 +717,7 @@ def set_dtype(self): # 2.3 value is a Paddle Tensor (int32, int64, float32, float64, bool) def create_test_value_tensor_int32(parent): - class TestValueInt(parent): - def set_dtype(self): self.dtype = "int32" @@ -800,9 +741,7 @@ def _get_answer(self): def create_test_value_tensor_int64(parent): - class TestValueInt(parent): - def set_dtype(self): self.dtype = "int64" @@ -826,9 +765,7 @@ def _get_answer(self): def create_test_value_tensor_fp32(parent): - class TestValueInt(parent): - def set_dtype(self): self.dtype = "float32" @@ -852,9 +789,7 @@ def _get_answer(self): def create_test_value_tensor_fp64(parent): - class TestValueInt(parent): - def set_dtype(self): self.dtype = "float64" @@ -878,9 +813,7 @@ def _get_answer(self): def create_test_value_tensor_bool(parent): - class TestValueInt(parent): - def set_dtype(self): self.dtype = "bool" @@ -905,7 +838,6 @@ def _get_answer(self): # 3. Test different shape of value class TestSetValueValueShape1(TestSetValueApi): - def set_value(self): self.value = np.array([3, 4, 5, 6]) # shape is (4,) @@ -917,7 +849,6 @@ def _get_answer(self): class TestSetValueValueShape2(TestSetValueApi): - def set_value(self): self.value = np.array([[3, 4, 5, 6]]) # shape is (1,4) @@ -929,10 +860,10 @@ def _get_answer(self): class TestSetValueValueShape3(TestSetValueApi): - def set_value(self): - self.value = np.array([[1, 1, 1, 1], [2, 2, 2, 2], - [3, 3, 3, 3]]) # shape is (3,4) + self.value = np.array( + [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]] + ) # shape is (3,4) def _call_setitem(self, x): x[0] = self.value @@ -942,11 +873,12 @@ def _get_answer(self): class TestSetValueValueShape4(TestSetValueApi): - def set_value(self): - self.value = np.array([[1, 1, 1, 1], [2, 2, 2, 2], - [3, 3, 3, - 3]]).astype(self.dtype) # shape is (3,4) + self.value = np.array( + [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]] + ).astype( + self.dtype + ) # shape is (3,4) def _call_setitem(self, x): x[0] = paddle.assign(self.value) # x is Paddle.Tensor @@ -956,7 +888,6 @@ def _get_answer(self): class TestSetValueValueShape5(TestSetValueApi): - def set_value(self): self.value = np.array([3, 3, 3]).astype(self.dtype) @@ -972,11 +903,10 @@ def _get_answer(self): # 4. Test error class TestError(TestSetValueBase): - def _value_type_error(self): with self.assertRaisesRegexp( - TypeError, - "Only support to assign an integer, float, numpy.ndarray or paddle.Tensor" + TypeError, + "Only support to assign an integer, float, numpy.ndarray or paddle.Tensor", ): x = paddle.ones(shape=self.shape, dtype=self.dtype) value = [1] @@ -984,8 +914,8 @@ def _value_type_error(self): def _dtype_error(self): with self.assertRaisesRegexp( - TypeError, - "When assign a numpy.ndarray, integer or float to a paddle.Tensor, " + TypeError, + "When assign a numpy.ndarray, integer or float to a paddle.Tensor, ", ): y = paddle.ones(shape=self.shape, dtype="float16") y[0] = 1 @@ -997,7 +927,8 @@ def _step_error(self): def _ellipsis_error(self): with self.assertRaisesRegexp( - IndexError, "An index can only have a single ellipsis"): + IndexError, "An index can only have a single ellipsis" + ): x = paddle.ones(shape=self.shape, dtype=self.dtype) x[..., ...] = self.value with self.assertRaisesRegexp(ValueError, "the start or end is None"): @@ -1044,7 +975,6 @@ def test_error(self): class Model(paddle.nn.Layer): - def __init__(self): super(Model, self).__init__() self.conv = paddle.nn.Conv2D(12, 12, 3) @@ -1060,7 +990,6 @@ def forward(self, x, y): class TestBackward(unittest.TestCase): - def test_static(self): paddle.enable_static() main_program = paddle.static.Program() @@ -1074,9 +1003,9 @@ def test_static(self): x = paddle.static.data(name="x", shape=[4, 4], dtype='float32') y = paddle.static.data(name="y", shape=[4, 4], dtype='float32') - label = paddle.static.data(name="label", - shape=[4, 1], - dtype='int64') + label = paddle.static.data( + name="label", shape=[4, 1], dtype='int64' + ) z = paddle.add(x, y) var = y[0, :] @@ -1084,8 +1013,9 @@ def test_static(self): prediction = paddle.static.nn.fc(x=z, size=2, activation='softmax') - cost = paddle.nn.functional.cross_entropy(input=prediction, - label=label) + cost = paddle.nn.functional.cross_entropy( + input=prediction, label=label + ) loss = paddle.mean(cost) sgd = paddle.optimizer.SGD(learning_rate=0.01) sgd.minimize(loss) @@ -1095,12 +1025,9 @@ def test_static(self): var_grad, z_grad = exe.run( main_program, - feed={ - "x": x_np, - "y": y_np, - "label": label_np - }, - fetch_list=[var.name + "@GRAD", z.name + "@GRAD"]) + feed={"x": x_np, "y": y_np, "label": label_np}, + fetch_list=[var.name + "@GRAD", z.name + "@GRAD"], + ) self.assertTrue((var_grad == z_grad[0, :]).all()) paddle.disable_static() @@ -1124,7 +1051,6 @@ def test_dynamic(self): class TestGradientTruncated(unittest.TestCase): - def func_test_consistent_with_competitor(self): paddle.disable_static() @@ -1135,8 +1061,9 @@ def set_value(t, value): return y.sum() # case 1 - array = np.arange(1, 1 + 2 * 3 * 4, - dtype="float32").reshape([1, 2, 1, 3, 1, 4]) + array = np.arange(1, 1 + 2 * 3 * 4, dtype="float32").reshape( + [1, 2, 1, 3, 1, 4] + ) value = np.arange(100, 104, dtype="float32").reshape(1, 4) inps = paddle.to_tensor(array, stop_gradient=False) @@ -1145,22 +1072,41 @@ def set_value(t, value): loss = set_value(inps, value) loss.backward() - value_grad = np.array([[600., 606., 612., 618.]]) - input_grad = np.array([[[[[[4., 32., 108., 256.]], - [[500., 864., 1372., 2048.]], - [[2916., 4000., 5324., 6912.]]]], - [[[[0., 0., 0., 0.]], [[0., 0., 0., 0.]], - [[0., 0., 0., 0.]]]]]]) + value_grad = np.array([[600.0, 606.0, 612.0, 618.0]]) + input_grad = np.array( + [ + [ + [ + [ + [[4.0, 32.0, 108.0, 256.0]], + [[500.0, 864.0, 1372.0, 2048.0]], + [[2916.0, 4000.0, 5324.0, 6912.0]], + ] + ], + [ + [ + [[0.0, 0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0, 0.0]], + [[0.0, 0.0, 0.0, 0.0]], + ] + ], + ] + ] + ) np.testing.assert_array_equal( inps.grad.numpy(), input_grad, - err_msg='The gradient of value should be \n{},\n but reveived {}'. - format(input_grad, inps.grad.numpy())) + err_msg='The gradient of value should be \n{},\n but reveived {}'.format( + input_grad, inps.grad.numpy() + ), + ) np.testing.assert_array_equal( value.grad.numpy(), value_grad, - err_msg='The gradient of input should be \n{},\n but reveived {}'. - format(value_grad, value.grad.numpy())) + err_msg='The gradient of input should be \n{},\n but reveived {}'.format( + value_grad, value.grad.numpy() + ), + ) # case 2 array = np.arange(1, 2 * 3 * 4 + 1, dtype="float32").reshape([4, 2, 3]) @@ -1172,23 +1118,29 @@ def set_value(t, value): loss = set_value(inps2, value2) loss.backward() - value_grad2 = np.array([600.]) - input_grad2 = np.array([[[4., 32., 108.], [0., 0., 0.]], - [[1372., 2048., 2916.], [4000., 5324., 6912.]], - [[8788., 10976., 13500.], - [16384., 19652., 23328.]], - [[27436., 32000., 37044.], - [42592., 48668., 55296.]]]) + value_grad2 = np.array([600.0]) + input_grad2 = np.array( + [ + [[4.0, 32.0, 108.0], [0.0, 0.0, 0.0]], + [[1372.0, 2048.0, 2916.0], [4000.0, 5324.0, 6912.0]], + [[8788.0, 10976.0, 13500.0], [16384.0, 19652.0, 23328.0]], + [[27436.0, 32000.0, 37044.0], [42592.0, 48668.0, 55296.0]], + ] + ) np.testing.assert_array_equal( inps2.grad.numpy(), input_grad2, - err_msg='The gradient of value should be \n{},\n but reveived {}'. - format(input_grad, inps2.grad.numpy())) + err_msg='The gradient of value should be \n{},\n but reveived {}'.format( + input_grad, inps2.grad.numpy() + ), + ) np.testing.assert_array_equal( value2.grad.numpy(), value_grad2, - err_msg='The gradient of input should be \n{},\n but reveived {}'. - format(value_grad, value2.grad.numpy())) + err_msg='The gradient of input should be \n{},\n but reveived {}'.format( + value_grad, value2.grad.numpy() + ), + ) # case 3 def set_value3(t, value): @@ -1197,8 +1149,9 @@ def set_value3(t, value): y = a * a return y.sum() - array = np.arange(1, 1 + 2 * 3 * 4, - dtype="float32").reshape([4, 3, 1, 1, 2, 1]) + array = np.arange(1, 1 + 2 * 3 * 4, dtype="float32").reshape( + [4, 3, 1, 1, 2, 1] + ) value = np.arange(100, 100 + 2, dtype="float32").reshape(1, 2, 1) inps = paddle.to_tensor(array, stop_gradient=False) @@ -1207,37 +1160,52 @@ def set_value3(t, value): loss = set_value3(inps, value) loss.backward() - value_grad = np.array([[[600.], [606.]]]) - input_grad = np.array([[[[[[0.], [0.]]]], [[[[0.], [0.]]]], - [[[[0.], [0.]]]]], - [[[[[1372.], [2048.]]]], [[[[2916.], [4000.]]]], - [[[[5324.], [6912.]]]]], - [[[[[8788.], [10976.]]]], [[[[13500.], - [16384.]]]], - [[[[19652.], [23328.]]]]], - [[[[[27436.], [32000.]]]], - [[[[37044.], [42592.]]]], - [[[[48668.], [55296.]]]]]]) + value_grad = np.array([[[600.0], [606.0]]]) + input_grad = np.array( + [ + [[[[[0.0], [0.0]]]], [[[[0.0], [0.0]]]], [[[[0.0], [0.0]]]]], + [ + [[[[1372.0], [2048.0]]]], + [[[[2916.0], [4000.0]]]], + [[[[5324.0], [6912.0]]]], + ], + [ + [[[[8788.0], [10976.0]]]], + [[[[13500.0], [16384.0]]]], + [[[[19652.0], [23328.0]]]], + ], + [ + [[[[27436.0], [32000.0]]]], + [[[[37044.0], [42592.0]]]], + [[[[48668.0], [55296.0]]]], + ], + ] + ) np.testing.assert_array_equal( inps.grad.numpy(), input_grad, - err_msg='The gradient of value should be \n{},\n but reveived {}'. - format(input_grad, inps.grad.numpy())) + err_msg='The gradient of value should be \n{},\n but reveived {}'.format( + input_grad, inps.grad.numpy() + ), + ) np.testing.assert_array_equal( value.grad.numpy(), value_grad, - err_msg='The gradient of input should be \n{},\n but reveived {}'. - format(value_grad, value.grad.numpy())) + err_msg='The gradient of input should be \n{},\n but reveived {}'.format( + value_grad, value.grad.numpy() + ), + ) - #case 4: step >0 + # case 4: step >0 def set_value4(t, value): a = t * t a[0, :, 0, ::3] = value y = a * a return y.sum() - array = np.arange(1, 1 + 2 * 3 * 4, - dtype="float32").reshape([2, 3, 1, 4, 1]) + array = np.arange(1, 1 + 2 * 3 * 4, dtype="float32").reshape( + [2, 3, 1, 4, 1] + ) value = np.arange(100, 100 + 2, dtype="float32").reshape(1, 2, 1) inps = paddle.to_tensor(array, stop_gradient=False) @@ -1246,23 +1214,35 @@ def set_value4(t, value): loss = set_value4(inps, value) loss.backward() - value_grad = np.array([[[600.], [606.]]]) - input_grad = np.array([[[[[0.], [32.], [108.], [0.]]], - [[[0.], [864.], [1372.], [0.]]], - [[[0.], [4000.], [5324.], [0.]]]], - [[[[8788.], [10976.], [13500.], [16384.]]], - [[[19652.], [23328.], [27436.], [32000.]]], - [[[37044.], [42592.], [48668.], [55296.]]]]]) + value_grad = np.array([[[600.0], [606.0]]]) + input_grad = np.array( + [ + [ + [[[0.0], [32.0], [108.0], [0.0]]], + [[[0.0], [864.0], [1372.0], [0.0]]], + [[[0.0], [4000.0], [5324.0], [0.0]]], + ], + [ + [[[8788.0], [10976.0], [13500.0], [16384.0]]], + [[[19652.0], [23328.0], [27436.0], [32000.0]]], + [[[37044.0], [42592.0], [48668.0], [55296.0]]], + ], + ] + ) np.testing.assert_array_equal( inps.grad.numpy(), input_grad, - err_msg='The gradient of value should be \n{},\n but reveived {}'. - format(input_grad, inps.grad.numpy())) + err_msg='The gradient of value should be \n{},\n but reveived {}'.format( + input_grad, inps.grad.numpy() + ), + ) np.testing.assert_array_equal( value.grad.numpy(), value_grad, - err_msg='The gradient of input should be \n{},\n but reveived {}'. - format(value_grad, value.grad.numpy())) + err_msg='The gradient of input should be \n{},\n but reveived {}'.format( + value_grad, value.grad.numpy() + ), + ) # case 5:a[0].shape==value.shape def set_value5(t, value): @@ -1280,24 +1260,41 @@ def set_value5(t, value): loss = set_value5(inps, value) loss.backward() - value_grad = np.array([[200., 202., 204., - 206.], [208., 210., 212., 214.], - [216., 218., 220., 222.]]) - input_grad = np.array([[[0., 0., 0., 0.], [0., 0., 0., 0.], - [0., 0., 0., 0.]], - [[8788., 10976., 13500., 16384.], - [19652., 23328., 27436., 32000.], - [37044., 42592., 48668., 55296.]]]) + value_grad = np.array( + [ + [200.0, 202.0, 204.0, 206.0], + [208.0, 210.0, 212.0, 214.0], + [216.0, 218.0, 220.0, 222.0], + ] + ) + input_grad = np.array( + [ + [ + [0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0], + ], + [ + [8788.0, 10976.0, 13500.0, 16384.0], + [19652.0, 23328.0, 27436.0, 32000.0], + [37044.0, 42592.0, 48668.0, 55296.0], + ], + ] + ) np.testing.assert_array_equal( inps.grad.numpy(), input_grad, - err_msg='The gradient of value should be \n{},\n but reveived {}'. - format(input_grad, inps.grad.numpy())) + err_msg='The gradient of value should be \n{},\n but reveived {}'.format( + input_grad, inps.grad.numpy() + ), + ) np.testing.assert_array_equal( value.grad.numpy(), value_grad, - err_msg='The gradient of input should be \n{},\n but reveived {}'. - format(value_grad, value.grad.numpy())) + err_msg='The gradient of input should be \n{},\n but reveived {}'.format( + value_grad, value.grad.numpy() + ), + ) # case 6: pass stop_gradient from value to x x = paddle.zeros([8, 8], dtype='float32') @@ -1319,7 +1316,7 @@ def test_consistent_with_competitor(self): def test_static_graph(self): paddle.enable_static() - to_string = lambda x, i, : x + '_' + str(i) + to_string = lambda x, i,: x + '_' + str(i) numel = lambda input_shape: reduce(lambda x, y: x * y, input_shape) def op1(x): @@ -1327,18 +1324,15 @@ def op1(x): # test stop_gradient value.stop_gradient = True x.stop_gradient = False - start = paddle.fluid.layers.fill_constant([1], - "int32", - 5, - force_cpu=True) - end = paddle.fluid.layers.fill_constant([1], - "int32", - 0, - force_cpu=True) - step = paddle.fluid.layers.fill_constant([1], - "int32", - -2, - force_cpu=True) + start = paddle.fluid.layers.fill_constant( + [1], "int32", 5, force_cpu=True + ) + end = paddle.fluid.layers.fill_constant( + [1], "int32", 0, force_cpu=True + ) + step = paddle.fluid.layers.fill_constant( + [1], "int32", -2, force_cpu=True + ) inputs = { 'Input': x, @@ -1351,16 +1345,18 @@ def op1(x): ], 'StepsTensorList': [ step, - ] + ], } helper = LayerHelper("set_value") y = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="set_value", - inputs=inputs, - outputs={'Out': y}, - attrs={'axes': [0]}) + helper.append_op( + type="set_value", + inputs=inputs, + outputs={'Out': y}, + attrs={'axes': [0]}, + ) return y, value @@ -1376,17 +1372,16 @@ def op2(x): 'steps': [-4], 'decrease_axes': [], 'none_axes': [], - 'dtype': paddle.float32 + 'dtype': paddle.float32, } inputs = {'Input': x, 'ValueTensor': value} helper = LayerHelper("set_value") y = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="set_value", - inputs=inputs, - outputs={'Out': y}, - attrs=attrs) + helper.append_op( + type="set_value", inputs=inputs, outputs={'Out': y}, attrs=attrs + ) return y, value @@ -1394,18 +1389,15 @@ def op3(x): value = paddle.fluid.layers.fill_constant([1], "float32", 1) x.stop_gradient = True value.stop_gradient = False - start = paddle.fluid.layers.fill_constant([1], - "int32", - 0, - force_cpu=True) - end = paddle.fluid.layers.fill_constant([1], - "int32", - 5, - force_cpu=True) - step = paddle.fluid.layers.fill_constant([1], - "int32", - 3, - force_cpu=True) + start = paddle.fluid.layers.fill_constant( + [1], "int32", 0, force_cpu=True + ) + end = paddle.fluid.layers.fill_constant( + [1], "int32", 5, force_cpu=True + ) + step = paddle.fluid.layers.fill_constant( + [1], "int32", 3, force_cpu=True + ) inputs = { 'Input': x, @@ -1418,24 +1410,26 @@ def op3(x): ], 'StepsTensorList': [ step, - ] + ], } helper = LayerHelper("set_value") y = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="set_value", - inputs=inputs, - outputs={'Out': y}, - attrs={'axes': [0]}) + helper.append_op( + type="set_value", + inputs=inputs, + outputs={'Out': y}, + attrs={'axes': [0]}, + ) return y, value def set_value(array, i, op): name_x = to_string('x', i) - x = paddle.static.data(name=name_x, - shape=array.shape, - dtype='float32') + x = paddle.static.data( + name=name_x, shape=array.shape, dtype='float32' + ) # set_value_op in __get/setitem__ is an inplace operation. # When `input.stop_gradient = True` and `value.stop_gradient = False`, @@ -1445,9 +1439,11 @@ def set_value(array, i, op): loss = paddle.fluid.layers.reduce_sum(y2) sgd = paddle.optimizer.Adam() sgd.minimize(loss) - place = paddle.fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else paddle.fluid.CUDAPlace(0) + place = ( + paddle.fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else paddle.fluid.CUDAPlace(0) + ) prog = paddle.static.default_main_program() exe = paddle.static.Executor(place) @@ -1462,8 +1458,9 @@ def set_value(array, i, op): input_shape = [7, 6, 5, 4, 3, 2] - array = np.arange(0, numel(input_shape), - dtype="float32").reshape(input_shape) + array = np.arange(0, numel(input_shape), dtype="float32").reshape( + input_shape + ) for i in range(len(input_shape)): program = paddle.static.Program() @@ -1487,7 +1484,6 @@ def set_value(array, i, op): class TestSetValueInplace(unittest.TestCase): - def test_inplace(self): paddle.disable_static() with paddle.fluid.dygraph.guard(): @@ -1506,7 +1502,6 @@ def test_inplace(self): class TestSetValueInplaceLeafVar(unittest.TestCase): - def test_inplace_var_become_leaf_var(self): paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_sgd_op.py b/python/paddle/fluid/tests/unittests/test_sgd_op.py index 2643dfe272cba..34420ce5a9c56 100644 --- a/python/paddle/fluid/tests/unittests/test_sgd_op.py +++ b/python/paddle/fluid/tests/unittests/test_sgd_op.py @@ -25,7 +25,6 @@ class TestSGDOp(OpTest): - def setUp(self): self.op_type = "sgd" self.conf() @@ -45,14 +44,12 @@ def test_check_output(self): class TestSGDOpCase8X(TestSGDOp): - def conf(self): self.h = 10 self.w = 64 class TestSparseSGDOp(unittest.TestCase): - def check_with_place(self, place): scope = core.Scope() @@ -82,11 +79,13 @@ def check_with_place(self, place): lr.set(lr_array, place) # create and run sgd operator - sgd_op = Operator("sgd", - Param='Param', - Grad='Grad', - ParamOut='Param', - LearningRate='LearningRate') + sgd_op = Operator( + "sgd", + Param='Param', + Grad='Grad', + ParamOut='Param', + LearningRate='LearningRate', + ) sgd_op.run(scope, place) # get and compare result @@ -119,13 +118,11 @@ def conf(self): class TestSparseSGDOpCase8X(TestSparseSGDOp): - def conf(self): self.row_numel = 16 class TestSGDOpOptimizeSelectedRows(unittest.TestCase): - def check_with_place(self, place): scope = core.Scope() @@ -170,15 +167,18 @@ def check_with_place(self, place): # optimize with Python w_after_optimize = np.copy(w_before_optimize) for index, id in enumerate(grad_rows): - w_after_optimize[ - id] = w_before_optimize[id] - lr_value * grad_array[index] + w_after_optimize[id] = ( + w_before_optimize[id] - lr_value * grad_array[index] + ) # create and run sgd operator - sgd_op = Operator("sgd", - Param='Param', - Grad='Grad', - ParamOut='Param', - LearningRate='LearningRate') + sgd_op = Operator( + "sgd", + Param='Param', + Grad='Grad', + ParamOut='Param', + LearningRate='LearningRate', + ) sgd_op.run(scope, place) # get and compare result @@ -193,13 +193,12 @@ def test_sparse_parameter_sgd(self): class TestSGDOpWithLargeInput(unittest.TestCase): - def runTest(self): paddle.enable_static() data = fluid.layers.fill_constant(shape=[1], value=128, dtype='int64') - label = fluid.layers.fill_constant(shape=[1, 150], - value=0.5, - dtype='float32') + label = fluid.layers.fill_constant( + shape=[1, 150], value=0.5, dtype='float32' + ) emb = fluid.embedding(input=data, size=(10000000, 150), dtype='float32') out = fluid.layers.l2_normalize(x=emb, axis=-1) @@ -212,21 +211,23 @@ def runTest(self): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) compiled_prog = fluid.compiler.CompiledProgram( - fluid.default_main_program()) + fluid.default_main_program() + ) result = exe.run(compiled_prog, fetch_list=[avg_cost]) class TestSGDV2(unittest.TestCase): - def test_sgd_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype("float32") a = paddle.to_tensor(value) linear = paddle.nn.Linear(13, 5) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.SGD(learning_rate=0.01, - parameters=linear.parameters(), - weight_decay=0.01) + adam = paddle.optimizer.SGD( + learning_rate=0.01, + parameters=linear.parameters(), + weight_decay=0.01, + ) out = linear(a) out.backward() adam.step() @@ -239,33 +240,31 @@ def check_sgd_optimizer(optimizer_attr): init_program = paddle.static.Program() program = paddle.static.Program() block = program.global_block() - mul_x = block.create_parameter(dtype="float32", - shape=[5, 10], - lod_level=0, - name="mul.x", - optimize_attr=optimizer_attr) - mul_y = block.create_var(dtype="float32", - shape=[10, 8], - lod_level=0, - name="mul.y") - mul_out = block.create_var(dtype="float32", - shape=[5, 8], - lod_level=0, - name="mul.out") - mean_out = block.create_var(dtype="float32", - shape=[1], - lod_level=0, - name="mean.out") - block.append_op(type="mul", - inputs={ - "X": mul_x, - "Y": mul_y - }, - outputs={"Out": mul_out}, - attrs={"x_num_col_dims": 1}) - block.append_op(type="mean", - inputs={"X": mul_out}, - outputs={"Out": mean_out}) + mul_x = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="mul.x", + optimize_attr=optimizer_attr, + ) + mul_y = block.create_var( + dtype="float32", shape=[10, 8], lod_level=0, name="mul.y" + ) + mul_out = block.create_var( + dtype="float32", shape=[5, 8], lod_level=0, name="mul.out" + ) + mean_out = block.create_var( + dtype="float32", shape=[1], lod_level=0, name="mean.out" + ) + block.append_op( + type="mul", + inputs={"X": mul_x, "Y": mul_y}, + outputs={"Out": mul_out}, + attrs={"x_num_col_dims": 1}, + ) + block.append_op( + type="mean", inputs={"X": mul_out}, outputs={"Out": mean_out} + ) sgd_optimizer = paddle.optimizer.SGD(learning_rate=0.01) opts, _ = sgd_optimizer.minimize(mean_out, init_program) return opts @@ -288,15 +287,18 @@ def test_sgd_group_dygraph(self): linear_1 = paddle.nn.Linear(13, 5) linear_2 = paddle.nn.Linear(5, 3) # This can be any optimizer supported by dygraph. - adam = paddle.optimizer.SGD(learning_rate=0.01, - parameters=[{ - 'params': linear_1.parameters() - }, { - 'params': linear_2.parameters(), - 'weight_decay': 0.001, - 'learning_rate': 0.1 - }], - weight_decay=0.01) + adam = paddle.optimizer.SGD( + learning_rate=0.01, + parameters=[ + {'params': linear_1.parameters()}, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + 'learning_rate': 0.1, + }, + ], + weight_decay=0.01, + ) out = linear_1(a) out = linear_2(out) out.backward() @@ -310,15 +312,15 @@ def test_eager(self): class TestSGDMultiPrecision2_0(unittest.TestCase): - def dygraph_sgd_mp(self, mp): paddle.disable_static() paddle.seed(10) paddle.set_device('gpu') input = paddle.randn((2, 2)) model = paddle.nn.Linear(2, 2) - optimizer = paddle.optimizer.SGD(parameters=model.parameters(), - multi_precision=mp) + optimizer = paddle.optimizer.SGD( + parameters=model.parameters(), multi_precision=mp + ) if mp == True: model = paddle.amp.decorate(models=model, level='O2') scaler = paddle.amp.GradScaler(init_loss_scaling=1024) @@ -355,16 +357,17 @@ def static_sgd_mp(self, mp): init_loss_scaling=128.0, use_dynamic_loss_scaling=True, use_pure_fp16=True, - use_fp16_guard=False) + use_fp16_guard=False, + ) with paddle.static.program_guard(train_program, startup_program): if mp: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float16') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float16' + ) else: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float32') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float32' + ) hidden = paddle.static.nn.fc(x=data, size=10) loss = paddle.mean(hidden) optimizer.minimize(loss) @@ -377,9 +380,9 @@ def static_sgd_mp(self, mp): x = np.random.random(size=(2, 2)).astype('float32') out = [] for idx in range(5): - loss_data, = exe.run(train_program, - feed={"X": x}, - fetch_list=[loss.name]) + (loss_data,) = exe.run( + train_program, feed={"X": x}, fetch_list=[loss.name] + ) out.append(loss_data) return out @@ -389,28 +392,32 @@ def test_main(self): "Test dygraph mode" output1_dy, params1_dy = self.dygraph_sgd_mp(mp=True) output2_dy, params2_dy = self.dygraph_sgd_mp(mp=False) - np.testing.assert_allclose(output1_dy.astype('float32').numpy(), - output2_dy.astype('float32').numpy(), - rtol=1e-05, - atol=0.1) + np.testing.assert_allclose( + output1_dy.astype('float32').numpy(), + output2_dy.astype('float32').numpy(), + rtol=1e-05, + atol=0.1, + ) for idx in range(len(params1_dy)): np.testing.assert_allclose( params1_dy[idx].astype('float32').numpy(), params2_dy[idx].astype('float32').numpy(), rtol=1e-05, - atol=0.1) + atol=0.1, + ) "Test static mode" output1_st = self.static_sgd_mp(mp=True) output2_st = self.static_sgd_mp(mp=False) for idx in range(len(output1_st)): - np.testing.assert_allclose(output1_st[idx].astype('float32'), - output2_st[idx].astype('float32'), - rtol=1e-05, - atol=0.1) + np.testing.assert_allclose( + output1_st[idx].astype('float32'), + output2_st[idx].astype('float32'), + rtol=1e-05, + atol=0.1, + ) class TestSGDMultiPrecision1_0(unittest.TestCase): - def dygraph_sgd_mp(self, mp): paddle.disable_static() paddle.seed(10) @@ -420,7 +427,8 @@ def dygraph_sgd_mp(self, mp): optimizer = paddle.fluid.optimizer.SGD( learning_rate=0.001, parameter_list=model.parameters(), - multi_precision=mp) + multi_precision=mp, + ) if mp == True: model = paddle.amp.decorate(models=model, level='O2') scaler = paddle.amp.GradScaler(init_loss_scaling=1024) @@ -449,8 +457,9 @@ def static_sgd_mp(self, mp): exe = paddle.static.Executor('gpu') train_program = paddle.static.Program() startup_program = paddle.static.Program() - optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.001, - multi_precision=mp) + optimizer = paddle.fluid.optimizer.SGD( + learning_rate=0.001, multi_precision=mp + ) if mp: optimizer = paddle.static.amp.decorate( @@ -458,16 +467,17 @@ def static_sgd_mp(self, mp): init_loss_scaling=128.0, use_dynamic_loss_scaling=True, use_pure_fp16=True, - use_fp16_guard=False) + use_fp16_guard=False, + ) with paddle.static.program_guard(train_program, startup_program): if mp: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float16') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float16' + ) else: - data = paddle.static.data(shape=[2, 2], - name='X', - dtype='float32') + data = paddle.static.data( + shape=[2, 2], name='X', dtype='float32' + ) hidden = paddle.static.nn.fc(x=data, size=10) loss = paddle.mean(hidden) optimizer.minimize(loss) @@ -480,9 +490,9 @@ def static_sgd_mp(self, mp): x = np.random.random(size=(2, 2)).astype('float32') out = [] for idx in range(5): - loss_data, = exe.run(train_program, - feed={"X": x}, - fetch_list=[loss.name]) + (loss_data,) = exe.run( + train_program, feed={"X": x}, fetch_list=[loss.name] + ) out.append(loss_data) return out @@ -492,24 +502,29 @@ def test_main(self): "Test dygraph mode" output1_dy, params1_dy = self.dygraph_sgd_mp(mp=True) output2_dy, params2_dy = self.dygraph_sgd_mp(mp=False) - np.testing.assert_allclose(output1_dy.astype('float32').numpy(), - output2_dy.astype('float32').numpy(), - rtol=1e-05, - atol=0.1) + np.testing.assert_allclose( + output1_dy.astype('float32').numpy(), + output2_dy.astype('float32').numpy(), + rtol=1e-05, + atol=0.1, + ) for idx in range(len(params1_dy)): np.testing.assert_allclose( params1_dy[idx].astype('float32').numpy(), params2_dy[idx].astype('float32').numpy(), rtol=1e-05, - atol=0.1) + atol=0.1, + ) "Test static mode" output1_st = self.static_sgd_mp(mp=True) output2_st = self.static_sgd_mp(mp=False) for idx in range(len(output1_st)): - np.testing.assert_allclose(output1_st[idx].astype('float32'), - output2_st[idx].astype('float32'), - rtol=1e-05, - atol=0.1) + np.testing.assert_allclose( + output1_st[idx].astype('float32'), + output2_st[idx].astype('float32'), + rtol=1e-05, + atol=0.1, + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_sgd_op_bf16.py b/python/paddle/fluid/tests/unittests/test_sgd_op_bf16.py index 3eb75a9b14052..ae7e87d1b3f71 100644 --- a/python/paddle/fluid/tests/unittests/test_sgd_op_bf16.py +++ b/python/paddle/fluid/tests/unittests/test_sgd_op_bf16.py @@ -17,18 +17,21 @@ import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.op import Operator -from paddle.fluid.tests.unittests.op_test import (convert_float_to_uint16, - convert_uint16_to_float, - OpTest, OpTestTool) +from paddle.fluid.tests.unittests.op_test import ( + convert_float_to_uint16, + convert_uint16_to_float, + OpTest, + OpTestTool, +) import paddle import paddle.static.amp as amp import struct -@unittest.skipIf(not core.supports_bfloat16(), - 'place does not support BF16 evaluation') +@unittest.skipIf( + not core.supports_bfloat16(), 'place does not support BF16 evaluation' +) class TestSGDOpBF16(OpTest): - def setUp(self): self.op_type = 'sgd' self.dtype = np.uint16 @@ -53,17 +56,16 @@ def test_check_output(self): self.check_output_with_place(core.CPUPlace(), check_dygraph=False) -@unittest.skipIf(not core.supports_bfloat16(), - 'place does not support BF16 evaluation') +@unittest.skipIf( + not core.supports_bfloat16(), 'place does not support BF16 evaluation' +) class TestSGDOpBF16Case2(TestSGDOpBF16): - def conf(self): self.h = 10 self.w = 64 class TestSparseSGDOpBF16(unittest.TestCase): - @classmethod def setUpClass(cls): np.random.seed(12345) @@ -121,10 +123,10 @@ def create_dense_lr_var(self, scope, place): return lr_tensor, lr_value -@unittest.skipIf(not core.supports_bfloat16(), - 'place does not support BF16 evaluation') +@unittest.skipIf( + not core.supports_bfloat16(), 'place does not support BF16 evaluation' +) class TestSparseGradSGDOpBF16(TestSparseSGDOpBF16): - def setUp(self): self.setup_params() @@ -136,32 +138,35 @@ def setup_params(self): def test_sparse_grad_sgd(self): scope = core.Scope() place = core.CPUPlace() - _, grad_array = self.create_sparse_grad_var(scope, place, - self.grad_height, - self.grad_rows, - self.grad_row_numel) + _, grad_array = self.create_sparse_grad_var( + scope, place, self.grad_height, self.grad_rows, self.grad_row_numel + ) param_tensor, param_array = self.create_dense_param_var( - scope, place, self.grad_height, self.grad_row_numel) + scope, place, self.grad_height, self.grad_row_numel + ) _, lr_value = self.create_dense_lr_var(scope, place) - sgd_op = Operator('sgd', - Param='Param', - Grad='Grad', - ParamOut='Param', - LearningRate='LearningRate', - use_mkldnn=True) + sgd_op = Operator( + 'sgd', + Param='Param', + Grad='Grad', + ParamOut='Param', + LearningRate='LearningRate', + use_mkldnn=True, + ) sgd_op.run(scope, place) - reference = self.ref_optimize(param_array, self.grad_rows, grad_array, - lr_value) + reference = self.ref_optimize( + param_array, self.grad_rows, grad_array, lr_value + ) output = np.array(param_tensor) self.check_output(output, reference, atol=5e-3, rtol=1e-1) -@unittest.skipIf(not core.supports_bfloat16(), - 'place does not support BF16 evaluation') +@unittest.skipIf( + not core.supports_bfloat16(), 'place does not support BF16 evaluation' +) class TestSparseGradSGDOpBF16Case2(TestSparseGradSGDOpBF16): - def setup_params(self): self.grad_height = 14 self.grad_rows = [1, 4, 12, 7, 8] @@ -169,17 +174,16 @@ def setup_params(self): class TestSparseGradSGDOpBF16Case3(TestSparseGradSGDOpBF16): - def setup_params(self): self.grad_height = 10 self.grad_rows = [0, 4, 7] self.grad_row_numel = 120 -@unittest.skipIf(not core.supports_bfloat16(), - 'place does not support BF16 evaluation') +@unittest.skipIf( + not core.supports_bfloat16(), 'place does not support BF16 evaluation' +) class TestSparseGradParamSGDOpBF16(TestSparseSGDOpBF16): - def setUp(self): self.setup_params() @@ -192,31 +196,32 @@ def setup_params(self): def test_sparse_param_grad_sgd(self): scope = core.Scope() place = core.CPUPlace() - _, grad_array = self.create_sparse_grad_var(scope, place, - self.grad_height, - self.grad_rows, - self.grad_row_numel) + _, grad_array = self.create_sparse_grad_var( + scope, place, self.grad_height, self.grad_rows, self.grad_row_numel + ) param_tensor, param_array = self.create_sparse_param_var( - scope, place, self.grad_height, self.param_rows, - self.grad_row_numel) + scope, place, self.grad_height, self.param_rows, self.grad_row_numel + ) _, lr_value = self.create_dense_lr_var(scope, place) - sgd_op = Operator('sgd', - Param='Param', - Grad='Grad', - ParamOut='Param', - LearningRate='LearningRate', - use_mkldnn=True) + sgd_op = Operator( + 'sgd', + Param='Param', + Grad='Grad', + ParamOut='Param', + LearningRate='LearningRate', + use_mkldnn=True, + ) sgd_op.run(scope, place) - reference = self.ref_optimize(param_array, self.grad_rows, grad_array, - lr_value) + reference = self.ref_optimize( + param_array, self.grad_rows, grad_array, lr_value + ) output = np.array(param_tensor) self.check_output(output, reference, atol=5e-3, rtol=1e-1) class TestSparseGradParamSGDOpBF16Case2(TestSparseGradParamSGDOpBF16): - def setup_params(self): self.grad_height = 14 self.grad_rows = [1, 4, 12, 7, 8] @@ -226,7 +231,6 @@ def setup_params(self): @OpTestTool.skip_if_not_cpu_bf16() class TestSGDOpBF16API(unittest.TestCase): - @classmethod def setUpClass(cls): np.random.seed(12345) @@ -259,18 +263,20 @@ def _mul_bf16(self, lhs: np.uint16, rhs: np.uint16): return self._fp322bf16(self._bf162fp32(lhs) * self._bf162fp32(rhs)) def _reference(self, data, emb_weight, bf16=False): - emb_out_shape = np.array([self.ids_shape[0], self.w_shape[1]], - dtype=np.int64) - mean_grad_value = np.float32(1.0) / np.prod(emb_out_shape, - dtype=np.float32) + emb_out_shape = np.array( + [self.ids_shape[0], self.w_shape[1]], dtype=np.int64 + ) + mean_grad_value = np.float32(1.0) / np.prod( + emb_out_shape, dtype=np.float32 + ) if bf16: - mean_grad = np.full(emb_out_shape, - self._fp322bf16(mean_grad_value), - dtype=np.uint16) + mean_grad = np.full( + emb_out_shape, self._fp322bf16(mean_grad_value), dtype=np.uint16 + ) else: - mean_grad = np.full(emb_out_shape, - mean_grad_value, - dtype=np.float32) + mean_grad = np.full( + emb_out_shape, mean_grad_value, dtype=np.float32 + ) # add_grad = 1 * mean_grad out_dtype = np.uint16 if bf16 else np.float32 lookup_table_grad = np.zeros(self.w_shape, dtype=out_dtype) @@ -281,7 +287,8 @@ def _reference(self, data, emb_weight, bf16=False): idxv = idx[0] for j in range(self.w_shape[1]): lookup_table_grad[idxv, j] = self._add_bf16( - lookup_table_grad[idxv, j], mean_grad[i, j]) + lookup_table_grad[idxv, j], mean_grad[i, j] + ) ref_grad = np.ndarray(shape=emb_weight.shape, dtype=np.uint16) lr_bf16 = self._fp322bf16(self.learning_rate) @@ -289,29 +296,26 @@ def _reference(self, data, emb_weight, bf16=False): for i, row in enumerate(emb_weight): for j, val in enumerate(row): ref_grad[i, j] = self._sub_bf16( - val, self._mul_bf16(lr_bf16, lookup_table_grad[i, j])) + val, self._mul_bf16(lr_bf16, lookup_table_grad[i, j]) + ) else: for i, idx in enumerate(data): lookup_table_grad[idx, :] += mean_grad[i] ref_grad = emb_weight - self.learning_rate * lookup_table_grad return ref_grad - def _check_output(self, - actual, - reference, - bf16=False, - atol=0, - rtol=0.15e-2): + def _check_output( + self, actual, reference, bf16=False, atol=0, rtol=0.15e-2 + ): output = actual if bf16 else convert_uint16_to_float(actual) if bf16: np.testing.assert_allclose(output, reference, atol=atol, rtol=rtol) else: try: print('Compare with FP32 values:') - np.testing.assert_allclose(output, - reference, - atol=atol, - rtol=rtol) + np.testing.assert_allclose( + output, reference, atol=atol, rtol=rtol + ) except AssertionError as e: print(e) @@ -329,56 +333,61 @@ def test_sgd(self): main = fluid.Program() with fluid.program_guard(main): x = fluid.layers.data(name='X', shape=self.ids_shape, dtype='int64') - label = fluid.layers.data(name='Y', - shape=self.y_shape, - dtype='uint16') - emb = fluid.layers.embedding(input=x, - size=self.w_shape, - param_attr=fluid.ParamAttr( - name="emb_weight", - initializer=self.initializer), - is_sparse=False, - dtype="uint16") # bfloat16 + label = fluid.layers.data( + name='Y', shape=self.y_shape, dtype='uint16' + ) + emb = fluid.layers.embedding( + input=x, + size=self.w_shape, + param_attr=fluid.ParamAttr( + name="emb_weight", initializer=self.initializer + ), + is_sparse=False, + dtype="uint16", + ) # bfloat16 cost = fluid.layers.elementwise_add(emb, label) avg_cost = paddle.mean(cost) sgd_optimizer = paddle.optimizer.SGD( - learning_rate=self.learning_rate) + learning_rate=self.learning_rate + ) sgd_optimizer = amp.bf16.decorate_bf16( sgd_optimizer, amp_lists=amp.bf16.AutoMixedPrecisionListsBF16( custom_bf16_list={ 'lookup_table', - }), + } + ), use_bf16_guard=False, - use_pure_bf16=True) + use_pure_bf16=True, + ) sgd_optimizer.minimize( - avg_cost, startup_program=fluid.default_startup_program()) + avg_cost, startup_program=fluid.default_startup_program() + ) train_reader = paddle.batch(self._data_reader, batch_size=1) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) test_prog = main.clone(for_test=True) - sgd_optimizer.amp_init(place, - test_program=test_prog, - use_bf16_test=True) + sgd_optimizer.amp_init( + place, test_program=test_prog, use_bf16_test=True + ) ref_emb = np.full(self.w_shape, self.value, dtype=np.float32) - ref_emb_bf16 = np.full(self.w_shape, - self._fp322bf16(self.value), - dtype=np.uint16) + ref_emb_bf16 = np.full( + self.w_shape, self._fp322bf16(self.value), dtype=np.uint16 + ) emb_weight = [] for sample in train_reader(): data = sample[0][0] label = sample[0][1] y_bf16 = convert_float_to_uint16(label) - emb_weight = exe.run(main, - feed={ - 'X': data, - 'Y': y_bf16 - }, - fetch_list=['emb_weight']) + emb_weight = exe.run( + main, + feed={'X': data, 'Y': y_bf16}, + fetch_list=['emb_weight'], + ) ref_emb = self._reference(data, ref_emb) ref_emb_bf16 = self._reference(data, ref_emb_bf16, True) diff --git a/python/paddle/fluid/tests/unittests/test_sgn.py b/python/paddle/fluid/tests/unittests/test_sgn.py index dacfe7aa1d21d..75d5d1b7847c4 100644 --- a/python/paddle/fluid/tests/unittests/test_sgn.py +++ b/python/paddle/fluid/tests/unittests/test_sgn.py @@ -29,40 +29,49 @@ def np_sgn(x: np.ndarray): class TestSgnError(unittest.TestCase): - def test_errors(self): # The input dtype of sgn must be float16, float32, float64,complex64,complex128. input2 = paddle.to_tensor( - np.random.randint(-10, 10, size=[12, 20]).astype('int32')) + np.random.randint(-10, 10, size=[12, 20]).astype('int32') + ) input3 = paddle.to_tensor( - np.random.randint(-10, 10, size=[12, 20]).astype('int64')) + np.random.randint(-10, 10, size=[12, 20]).astype('int64') + ) self.assertRaises(TypeError, paddle.sgn, input2) self.assertRaises(TypeError, paddle.sgn, input3) class TestSignAPI(unittest.TestCase): - def setUp(self) -> None: self.support_dtypes = [ - 'float16', 'float32', 'float64', 'complex64', 'complex128' + 'float16', + 'float32', + 'float64', + 'complex64', + 'complex128', ] if paddle.device.get_device() == 'cpu': self.support_dtypes = [ - 'float32', 'float64', 'complex64', 'complex128' + 'float32', + 'float64', + 'complex64', + 'complex128', ] def test_dtype(self): for dtype in self.support_dtypes: x = paddle.to_tensor( - np.random.randint(-10, 10, size=[12, 20, 2]).astype(dtype)) + np.random.randint(-10, 10, size=[12, 20, 2]).astype(dtype) + ) paddle.sgn(x) def test_complex(self): for dtype in ['complex64', 'complex128']: - np_x = np.array([[3 + 4j, 7 - 24j, 0, 1 + 2j], [6 + 8j, 3, 0, -2]], - dtype=dtype) + np_x = np.array( + [[3 + 4j, 7 - 24j, 0, 1 + 2j], [6 + 8j, 3, 0, -2]], dtype=dtype + ) x = paddle.to_tensor(np_x) z = paddle.sgn(x) np_z = z.numpy() diff --git a/python/paddle/fluid/tests/unittests/test_shape_op.py b/python/paddle/fluid/tests/unittests/test_shape_op.py index 8bee609623ea1..f4de948d229a6 100644 --- a/python/paddle/fluid/tests/unittests/test_shape_op.py +++ b/python/paddle/fluid/tests/unittests/test_shape_op.py @@ -21,7 +21,6 @@ class TestShapeOp(OpTest): - def setUp(self): self.op_type = "shape" self.python_api = paddle.shape @@ -39,19 +38,16 @@ def test_check_output(self): class case1(TestShapeOp): - def config(self): self.shape = [2] class case2(TestShapeOp): - def config(self): self.shape = [1, 2, 3] class TestShapeWithSelectedRows(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_shard_index_op.py b/python/paddle/fluid/tests/unittests/test_shard_index_op.py index 5c51b3cf58e8b..8c13e2b44c3c5 100644 --- a/python/paddle/fluid/tests/unittests/test_shard_index_op.py +++ b/python/paddle/fluid/tests/unittests/test_shard_index_op.py @@ -37,13 +37,12 @@ def common_setup(self, index_num, nshards, shard_id, ignore_value): 'index_num': index_num, 'nshards': nshards, 'shard_id': shard_id, - 'ignore_value': ignore_value + 'ignore_value': ignore_value, } self.outputs = {'Out': (out, x_lod)} class TestShardIndexShardId0Op(OpTest): - def setUp(self): common_setup(self, 20, 2, 0, -1) @@ -52,7 +51,6 @@ def test_check_output(self): class TestShardIndexShardId1Op(OpTest): - def setUp(self): common_setup(self, 20, 2, 1, -1) @@ -61,7 +59,6 @@ def test_check_output(self): class TestShardIndexIgnoreValueOp(OpTest): - def setUp(self): common_setup(self, 20, 2, 0, -2) @@ -70,7 +67,6 @@ def test_check_output(self): class TestShardIndexNotEvenlyDividedOp(OpTest): - def setUp(self): common_setup(self, 15, 2, 1, -1) diff --git a/python/paddle/fluid/tests/unittests/test_share_data_op.py b/python/paddle/fluid/tests/unittests/test_share_data_op.py index 56ac3af6d8538..b6f2d1ecbec46 100644 --- a/python/paddle/fluid/tests/unittests/test_share_data_op.py +++ b/python/paddle/fluid/tests/unittests/test_share_data_op.py @@ -20,7 +20,6 @@ class TestShareDataOp(OpTest): - def setUp(self): self.op_type = "share_data" input = np.random.rand(2, 3, 5).astype("float32") @@ -32,7 +31,6 @@ def test_check_output(self): class TestShareDataOpOnDifferentPlaces(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py b/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py index 9e765859127c7..c30c2095c9bac 100644 --- a/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py +++ b/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py @@ -27,16 +27,14 @@ class TestShrinkRNNMemoryBase(unittest.TestCase): - def setUp(self): self.main_program = Program() switch_main_program(self.main_program) x = layers.data('x', shape=[100], dtype='float32') x.stop_gradient = False - rank_table_tensor = layers.data('rank_table_tensor', - shape=[1], - dtype='float32', - lod_level=1) + rank_table_tensor = layers.data( + 'rank_table_tensor', shape=[1], dtype='float32', lod_level=1 + ) table = lod_rank_table(x=rank_table_tensor) i = layers.zeros(dtype='int64', shape=[1]) self.mem1 = shrink_memory(x=x, i=i, table=table) @@ -58,7 +56,6 @@ def sum_lodtensor(self, tensor): class TestShrinkRNNMemoryReferLoD(TestShrinkRNNMemoryBase): - def test_refer_lod(self): cpu = core.CPUPlace() x_tensor = core.LoDTensor() @@ -69,16 +66,15 @@ def test_refer_lod(self): rank_table_tensor = core.LoDTensor() rank_table_tensor.set_recursive_sequence_lengths([[1, 2, 3]]) rank_table_tensor.set( - np.random.random(size=(6, 1)).astype('float32'), cpu) + np.random.random(size=(6, 1)).astype('float32'), cpu + ) exe = Executor(cpu) outs = exe.run( - feed={ - 'x': x_tensor, - 'rank_table_tensor': rank_table_tensor - }, + feed={'x': x_tensor, 'rank_table_tensor': rank_table_tensor}, fetch_list=[self.mem1, self.mem2, self.mem3, self.x_grad], - return_numpy=False) + return_numpy=False, + ) np.testing.assert_allclose(tensor_np[0:6], outs[0], rtol=1e-05) np.testing.assert_allclose(tensor_np[0:5], outs[1], rtol=1e-05) np.testing.assert_allclose(tensor_np[0:2], outs[2], rtol=1e-05) @@ -86,7 +82,6 @@ def test_refer_lod(self): class TestShrinkRNNMemoryNoLoD(TestShrinkRNNMemoryBase): - def test_no_lod(self): cpu = core.CPUPlace() x_tensor = core.LoDTensor() @@ -96,16 +91,15 @@ def test_no_lod(self): rank_table_tensor = core.LoDTensor() rank_table_tensor.set_recursive_sequence_lengths([[1, 2, 3]]) rank_table_tensor.set( - np.random.random(size=(6, 1)).astype('float32'), cpu) + np.random.random(size=(6, 1)).astype('float32'), cpu + ) exe = Executor(cpu) outs = exe.run( - feed={ - 'x': x_tensor, - 'rank_table_tensor': rank_table_tensor - }, + feed={'x': x_tensor, 'rank_table_tensor': rank_table_tensor}, fetch_list=[self.mem1, self.mem2, self.mem3, self.x_grad], - return_numpy=False) + return_numpy=False, + ) np.testing.assert_allclose(tensor_np[0:3], outs[0], rtol=1e-05) np.testing.assert_allclose(tensor_np[0:2], outs[1], rtol=1e-05) np.testing.assert_allclose(tensor_np[0:1], outs[2], rtol=1e-05) @@ -113,7 +107,6 @@ def test_no_lod(self): class TestShrinkRNNMemoryOpError(unittest.TestCase): - def test_erroes(self): with program_guard(Program(), Program()): x = layers.zeros(dtype='int64', shape=[3, 100]) @@ -121,8 +114,8 @@ def test_erroes(self): rank_table_tensor = core.LoDTensor() rank_table_tensor.set_recursive_sequence_lengths([[1, 2, 3]]) rank_table_tensor.set( - np.random.random(size=(6, 1)).astype('float32'), - core.CPUPlace()) + np.random.random(size=(6, 1)).astype('float32'), core.CPUPlace() + ) rank_table = np.random.random(size=(6, 1)).astype('float32') # The type of x in shrink_rnn_memory must be Variable. diff --git a/python/paddle/fluid/tests/unittests/test_shuffle_batch_op.py b/python/paddle/fluid/tests/unittests/test_shuffle_batch_op.py index 7752cfa3a65ae..95fc233239a11 100644 --- a/python/paddle/fluid/tests/unittests/test_shuffle_batch_op.py +++ b/python/paddle/fluid/tests/unittests/test_shuffle_batch_op.py @@ -21,7 +21,6 @@ class TestShuffleBatchOpBase(OpTest): - def gen_random_array(self, shape, low=0, high=1): rnd = (high - low) * np.random.random(shape) + low return rnd.astype(self.dtype) @@ -40,8 +39,9 @@ def setUp(self): self.dtype = np.float64 self.shape = self.get_shape() x = self.gen_random_array(self.shape) - seed = np.random.random_integers(low=10, high=100, - size=(1, )).astype('int64') + seed = np.random.random_integers(low=10, high=100, size=(1,)).astype( + 'int64' + ) self.inputs = {'X': x, 'Seed': seed} self.outputs = { 'Out': np.array([]).astype(x.dtype), @@ -78,7 +78,6 @@ def test_check_grad(self): class TestShuffleBatchOp2(TestShuffleBatchOpBase): - def get_shape(self): return (4, 30) diff --git a/python/paddle/fluid/tests/unittests/test_shuffle_channel_op.py b/python/paddle/fluid/tests/unittests/test_shuffle_channel_op.py index 594df3f6c091d..01bdd48ff24ee 100644 --- a/python/paddle/fluid/tests/unittests/test_shuffle_channel_op.py +++ b/python/paddle/fluid/tests/unittests/test_shuffle_channel_op.py @@ -18,7 +18,6 @@ class TestShuffleChannelOp(OpTest): - def setUp(self): self.op_type = "shuffle_channel" self.batch_size = 10 @@ -27,13 +26,14 @@ def setUp(self): self.layer_w = 4 self.group = 4 self.x = np.random.random( - (self.batch_size, self.input_channels, self.layer_h, - self.layer_w)).astype('float32') + (self.batch_size, self.input_channels, self.layer_h, self.layer_w) + ).astype('float32') self.inputs = {'X': self.x} self.attrs = {'group': self.group} n, c, h, w = self.x.shape - input_reshaped = np.reshape(self.x, - (-1, self.group, c // self.group, h, w)) + input_reshaped = np.reshape( + self.x, (-1, self.group, c // self.group, h, w) + ) input_transposed = np.transpose(input_reshaped, (0, 2, 1, 3, 4)) self.outputs = {'Out': np.reshape(input_transposed, (-1, c, h, w))} diff --git a/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py b/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py index f6ea5018ea986..1d09aefd2d8a8 100644 --- a/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py +++ b/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py @@ -24,12 +24,12 @@ def test_fluid_sigmoid(x, label, normalize=False, ignore_index=-100): return paddle.fluid.layers.sigmoid_cross_entropy_with_logits( - x, label, int(ignore_index), normalize=normalize) + x, label, int(ignore_index), normalize=normalize + ) class TestSigmoidCrossEntropyWithLogitsOp1(OpTest): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -37,12 +37,14 @@ def setUp(self): batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - (batch_size, num_classes)).astype("float64")), - 'Label': - np.random.randint(0, 2, (batch_size, num_classes)).astype("float64") + 'X': logit( + np.random.uniform(0, 1, (batch_size, num_classes)).astype( + "float64" + ) + ), + 'Label': np.random.randint(0, 2, (batch_size, num_classes)).astype( + "float64" + ), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -61,8 +63,7 @@ def test_check_grad(self): class TestSigmoidCrossEntropyWithLogitsOp2(OpTest): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -71,13 +72,14 @@ def setUp(self): num_classes = 20 ignore_index = -1 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - (batch_size, num_classes)).astype("float64")), - 'Label': - np.random.randint(-1, 2, - (batch_size, num_classes)).astype("float64") + 'X': logit( + np.random.uniform(0, 1, (batch_size, num_classes)).astype( + "float64" + ) + ), + 'Label': np.random.randint(-1, 2, (batch_size, num_classes)).astype( + "float64" + ), } self.attrs = { 'ignore_index': ignore_index, @@ -100,8 +102,7 @@ def test_check_grad(self): class TestSigmoidCrossEntropyWithLogitsOp3(OpTest): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -109,12 +110,14 @@ def setUp(self): batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - (batch_size, num_classes)).astype("float64")), - 'Label': - np.random.uniform(0, 1, (batch_size, num_classes)).astype("float64") + 'X': logit( + np.random.uniform(0, 1, (batch_size, num_classes)).astype( + "float64" + ) + ), + 'Label': np.random.uniform(0, 1, (batch_size, num_classes)).astype( + "float64" + ), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -133,7 +136,6 @@ def test_check_grad(self): class TestSigmoidCrossEntropyWithNorm(OpTest): - def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" self.python_api = test_fluid_sigmoid @@ -141,13 +143,14 @@ def setUp(self): num_classes = 20 ignore_index = -1 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - (batch_size, num_classes)).astype("float64")), - 'Label': - np.random.randint(-1, 2, - (batch_size, num_classes)).astype("float64") + 'X': logit( + np.random.uniform(0, 1, (batch_size, num_classes)).astype( + "float64" + ) + ), + 'Label': np.random.randint(-1, 2, (batch_size, num_classes)).astype( + "float64" + ), } self.attrs = {'ignore_index': ignore_index, 'normalize': True} sigmoid_X = expit(self.inputs['X']) @@ -157,7 +160,8 @@ def setUp(self): out[np.where(self.inputs['Label'] == ignore_index)] = 0 if self.attrs['normalize']: out = out / float( - np.where(self.inputs['Label'] != ignore_index)[0].size) + np.where(self.inputs['Label'] != ignore_index)[0].size + ) self.outputs = {'Out': out} def test_check_output(self): @@ -168,8 +172,7 @@ def test_check_grad(self): class TestSigmoidCrossEntropyWithLogitsOp5(OpTest): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -177,13 +180,14 @@ def setUp(self): batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform( - 0, 1, tuple(batch_size + [num_classes])).astype("float64")), - 'Label': - np.random.uniform(0, 1, tuple(batch_size + - [num_classes])).astype("float64") + 0, 1, tuple(batch_size + [num_classes]) + ).astype("float64") + ), + 'Label': np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype("float64"), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -202,7 +206,6 @@ def test_check_grad(self): class TestSigmoidCrossEntropyWithNorm2(OpTest): - def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" self.python_api = test_fluid_sigmoid @@ -210,13 +213,14 @@ def setUp(self): num_classes = 20 ignore_index = -1 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform( - 0, 1, tuple(batch_size + [num_classes])).astype("float64")), - 'Label': - np.random.randint(-1, 2, tuple(batch_size + - [num_classes])).astype("float64") + 0, 1, tuple(batch_size + [num_classes]) + ).astype("float64") + ), + 'Label': np.random.randint( + -1, 2, tuple(batch_size + [num_classes]) + ).astype("float64"), } self.attrs = {'ignore_index': ignore_index, 'normalize': True} sigmoid_X = expit(self.inputs['X']) @@ -226,7 +230,8 @@ def setUp(self): out[np.where(self.inputs['Label'] == ignore_index)] = 0 if self.attrs['normalize']: out = out / float( - np.where(self.inputs['Label'] != ignore_index)[0].size) + np.where(self.inputs['Label'] != ignore_index)[0].size + ) self.outputs = {'Out': out} def test_check_output(self): @@ -236,8 +241,7 @@ def test_check_grad(self): self.check_grad(['X'], 'Out', check_eager=True) class TestSigmoidCrossEntropyWithLogitsOp6(OpTest): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def setUp(self): self.op_type = "sigmoid_cross_entropy_with_logits" @@ -245,14 +249,14 @@ def setUp(self): batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + - [num_classes])).astype("float64")), - 'Label': - np.random.randint(0, 2, tuple(batch_size + - [num_classes])).astype("float64") + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype("float64") + ), + 'Label': np.random.randint( + 0, 2, tuple(batch_size + [num_classes]) + ).astype("float64"), } # Fw Pass is implemented as elementwise sigmoid followed by @@ -270,18 +274,21 @@ def test_check_grad(self): self.check_grad(['X'], 'Out', check_eager=True) class TestSigmoidCrossEntropyWithLogitsOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of sigmoid_cross_entropy_with_logits must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, - 5]), [[1, 1, 1, 1]], - fluid.CPUPlace()) - lab1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), + [[1, 1, 1, 1]], + fluid.CPUPlace(), + ) + lab1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), + [[1, 1, 1, 1]], + fluid.CPUPlace(), + ) fluid.layers.sigmoid_cross_entropy_with_logits(x1, lab1) self.assertRaises(TypeError, test_Variable) @@ -289,12 +296,12 @@ def test_Variable(): def test_dtype(): # the input dtype of sigmoid_cross_entropy_with_logits must be float16 or float32 or float64 # float16 only can be set on GPU place - x2 = fluid.layers.data(name='x2', - shape=[3, 4, 5, 6], - dtype="int32") - lab2 = fluid.layers.data(name='lab2', - shape=[3, 4, 5, 6], - dtype="int32") + x2 = fluid.layers.data( + name='x2', shape=[3, 4, 5, 6], dtype="int32" + ) + lab2 = fluid.layers.data( + name='lab2', shape=[3, 4, 5, 6], dtype="int32" + ) fluid.layers.sigmoid_cross_entropy_with_logits(x2, lab2) self.assertRaises(TypeError, test_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss.py b/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss.py index a3b614cb51f3e..e36ba383c07ef 100644 --- a/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss.py +++ b/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss.py @@ -19,84 +19,83 @@ from paddle.fluid.framework import _test_eager_guard -def call_sfl_functional(logit, - label, - normalizer, - alpha=0.25, - gamma=2.0, - reduction='sum'): - res = paddle.nn.functional.sigmoid_focal_loss(logit, - label, - normalizer, - alpha=alpha, - gamma=gamma, - reduction=reduction) +def call_sfl_functional( + logit, label, normalizer, alpha=0.25, gamma=2.0, reduction='sum' +): + res = paddle.nn.functional.sigmoid_focal_loss( + logit, label, normalizer, alpha=alpha, gamma=gamma, reduction=reduction + ) return res -def test_static(place, - logit_np, - label_np, - normalizer_np, - alpha=0.25, - gamma=2.0, - reduction='sum'): +def test_static( + place, + logit_np, + label_np, + normalizer_np, + alpha=0.25, + gamma=2.0, + reduction='sum', +): paddle.enable_static() prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - logit = paddle.fluid.data(name='logit', - shape=logit_np.shape, - dtype='float64') - label = paddle.fluid.data(name='label', - shape=label_np.shape, - dtype='float64') + logit = paddle.fluid.data( + name='logit', shape=logit_np.shape, dtype='float64' + ) + label = paddle.fluid.data( + name='label', shape=label_np.shape, dtype='float64' + ) feed_dict = {"logit": logit_np, "label": label_np} normalizer = None if normalizer_np is not None: - normalizer = paddle.fluid.data(name='normalizer', - shape=normalizer_np.shape, - dtype='float64') + normalizer = paddle.fluid.data( + name='normalizer', shape=normalizer_np.shape, dtype='float64' + ) feed_dict["normalizer"] = normalizer_np - res = call_sfl_functional(logit, label, normalizer, alpha, gamma, - reduction) + res = call_sfl_functional( + logit, label, normalizer, alpha, gamma, reduction + ) exe = paddle.static.Executor(place) static_result = exe.run(prog, feed=feed_dict, fetch_list=[res]) return static_result -def test_dygraph(place, - logit_np, - label_np, - normalizer_np, - alpha=0.25, - gamma=2.0, - reduction='sum'): +def test_dygraph( + place, + logit_np, + label_np, + normalizer_np, + alpha=0.25, + gamma=2.0, + reduction='sum', +): paddle.disable_static() logit = paddle.to_tensor(logit_np) label = paddle.to_tensor(label_np) normalizer = None if normalizer_np is not None: normalizer = paddle.to_tensor(normalizer_np) - dy_res = call_sfl_functional(logit, label, normalizer, alpha, gamma, - reduction) + dy_res = call_sfl_functional( + logit, label, normalizer, alpha, gamma, reduction + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result -def calc_sigmoid_focal_loss(logit_np, - label_np, - normalizer_np, - alpha=0.25, - gamma=2.0, - reduction='sum'): +def calc_sigmoid_focal_loss( + logit_np, label_np, normalizer_np, alpha=0.25, gamma=2.0, reduction='sum' +): - loss = np.maximum( - logit_np, - 0) - logit_np * label_np + np.log(1 + np.exp(-np.abs(logit_np))) + loss = ( + np.maximum(logit_np, 0) + - logit_np * label_np + + np.log(1 + np.exp(-np.abs(logit_np))) + ) pred = 1 / (1 + np.exp(-logit_np)) p_t = pred * label_np + (1 - pred) * (1 - label_np) @@ -106,7 +105,7 @@ def calc_sigmoid_focal_loss(logit_np, loss = alpha_t * loss if gamma is not None: - loss = loss * ((1 - p_t)**gamma) + loss = loss * ((1 - p_t) ** gamma) if normalizer_np is not None: loss = loss / normalizer_np @@ -120,62 +119,89 @@ def calc_sigmoid_focal_loss(logit_np, class TestSigmoidFocalLoss(unittest.TestCase): - def test_SigmoidFocalLoss(self): - logit_np = np.random.uniform(0.1, 0.8, - size=(2, 3, 4, 10)).astype(np.float64) - label_np = np.random.randint(0, 2, - size=(2, 3, 4, 10)).astype(np.float64) + logit_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( + np.float64 + ) + label_np = np.random.randint(0, 2, size=(2, 3, 4, 10)).astype( + np.float64 + ) normalizer_nps = [ - np.asarray([np.sum(label_np > 0)], dtype=label_np.dtype), None + np.asarray([np.sum(label_np > 0)], dtype=label_np.dtype), + None, ] places = [fluid.CPUPlace()] if fluid.core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) reductions = ['sum', 'mean', 'none'] alphas = [0.25, 0.5] - gammas = [3, 0.] + gammas = [3, 0.0] for place in places: for reduction in reductions: for alpha in alphas: for gamma in gammas: for normalizer_np in normalizer_nps: - static_result, = test_static( - place, logit_np, label_np, normalizer_np, alpha, - gamma, reduction) - dy_result = test_dygraph(place, logit_np, label_np, - normalizer_np, alpha, - gamma, reduction) + (static_result,) = test_static( + place, + logit_np, + label_np, + normalizer_np, + alpha, + gamma, + reduction, + ) + dy_result = test_dygraph( + place, + logit_np, + label_np, + normalizer_np, + alpha, + gamma, + reduction, + ) with _test_eager_guard(): eager_result = test_dygraph( - place, logit_np, label_np, normalizer_np, - alpha, gamma, reduction) + place, + logit_np, + label_np, + normalizer_np, + alpha, + gamma, + reduction, + ) expected = calc_sigmoid_focal_loss( - logit_np, label_np, normalizer_np, alpha, gamma, - reduction) - np.testing.assert_allclose(static_result, - expected, - rtol=1e-05) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-05) - np.testing.assert_allclose(dy_result, - expected, - rtol=1e-05) - np.testing.assert_allclose(eager_result, - expected, - rtol=1e-05) + logit_np, + label_np, + normalizer_np, + alpha, + gamma, + reduction, + ) + np.testing.assert_allclose( + static_result, expected, rtol=1e-05 + ) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-05 + ) + np.testing.assert_allclose( + dy_result, expected, rtol=1e-05 + ) + np.testing.assert_allclose( + eager_result, expected, rtol=1e-05 + ) def test_SigmoidFocalLoss_error(self): paddle.disable_static() logit = paddle.to_tensor([[0.97], [0.91], [0.03]], dtype='float32') label = paddle.to_tensor([[1.0], [1.0], [0.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.sigmoid_focal_loss, - logit=logit, - label=label, - normalizer=None, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.sigmoid_focal_loss, + logit=logit, + label=label, + normalizer=None, + reduction="unsupport reduction", + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss_op.py b/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss_op.py index 91b2981acd53a..4b9831f8e8d0a 100644 --- a/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_sigmoid_focal_loss_op.py @@ -22,8 +22,9 @@ from paddle.fluid import Program, program_guard -def sigmoid_focal_loss_forward(x_data, label_data, fg_num_data, gamma, alpha, - num_classes): +def sigmoid_focal_loss_forward( + x_data, label_data, fg_num_data, gamma, alpha, num_classes +): x_data_t = copy.deepcopy(x_data) out_data = copy.deepcopy(x_data) x_width = len(x_data) @@ -41,12 +42,13 @@ def sigmoid_focal_loss_forward(x_data, label_data, fg_num_data, gamma, alpha, z_neg = (1.0 - alpha) / fg_num z_pos = alpha / fg_num - p = 1. / (1. + math.exp(-x)) + p = 1.0 / (1.0 + math.exp(-x)) FLT_MIN = 1.175494351e-38 - term_pos = math.pow((1. - p), gamma) * math.log(max(FLT_MIN, p)) - term_neg = math.pow(p, gamma) * (-1. * x * (x >= 0) - - math.log(1. + math.exp(x - 2. * x * - (x >= 0)))) + term_pos = math.pow((1.0 - p), gamma) * math.log(max(FLT_MIN, p)) + term_neg = math.pow(p, gamma) * ( + -1.0 * x * (x >= 0) + - math.log(1.0 + math.exp(x - 2.0 * x * (x >= 0))) + ) out_data[idx] = 0.0 out_data[idx] += -c_pos * term_pos * z_pos out_data[idx] += -c_neg * term_neg * z_neg @@ -56,7 +58,6 @@ def sigmoid_focal_loss_forward(x_data, label_data, fg_num_data, gamma, alpha, class TestSigmoidFocalLossOp1(OpTest): - def set_argument(self): self.num_anchors = 10 self.num_classes = 10 @@ -68,8 +69,9 @@ def setUp(self): dims = (self.num_anchors, self.num_classes) X = np.random.standard_normal(dims).astype("float64") - L = np.random.randint(0, self.num_classes + 1, - (dims[0], 1)).astype("int32") + L = np.random.randint(0, self.num_classes + 1, (dims[0], 1)).astype( + "int32" + ) F = np.zeros(1) F[0] = len(np.where(L > 0)[0]) F = F.astype("int32") @@ -84,10 +86,14 @@ def setUp(self): 'gamma': self.gamma, 'alpha': self.alpha, } - loss = sigmoid_focal_loss_forward(self.inputs['X'], - self.inputs['Label'], - self.inputs['FgNum'], self.gamma, - self.alpha, self.num_classes) + loss = sigmoid_focal_loss_forward( + self.inputs['X'], + self.inputs['Label'], + self.inputs['FgNum'], + self.gamma, + self.alpha, + self.num_classes, + ) self.outputs = {'Out': loss.astype('float64')} def test_check_output(self): @@ -97,23 +103,22 @@ def test_check_grad(self): self.check_grad(['X'], 'Out') -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSigmoidFocalLossOp2(TestSigmoidFocalLossOp1): - def test_check_output(self): place = core.CUDAPlace(0) self.check_output_with_place(place, atol=2e-3) def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.002) + self.check_grad_with_place( + place, ['X'], 'Out', max_relative_error=0.002 + ) class TestSigmoidFocalLossOp3(TestSigmoidFocalLossOp1): - def set_argument(self): self.num_anchors = 200 self.num_classes = 10 @@ -121,50 +126,45 @@ def set_argument(self): self.alpha = 0.5 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSigmoidFocalLossOp4(TestSigmoidFocalLossOp3): - def test_check_output(self): place = core.CUDAPlace(0) self.check_output_with_place(place, atol=2e-3) def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.002) + self.check_grad_with_place( + place, ['X'], 'Out', max_relative_error=0.002 + ) class TestSigmoidFocalLossOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - label1 = fluid.layers.fill_constant(shape=[10, 1], - dtype="int32", - value=1) - fg_num1 = fluid.layers.fill_constant(shape=[1], - dtype="int32", - value=5) + label1 = fluid.layers.fill_constant( + shape=[10, 1], dtype="int32", value=1 + ) + fg_num1 = fluid.layers.fill_constant( + shape=[1], dtype="int32", value=5 + ) # The `x` must be Variable and the data type of `x` Tensor must be one of float32 and float64. def test_x_type(): x1 = [2] - fluid.layers.sigmoid_focal_loss(x=x1, - label=label1, - fg_num=fg_num1, - gamma=2., - alpha=0.25) + fluid.layers.sigmoid_focal_loss( + x=x1, label=label1, fg_num=fg_num1, gamma=2.0, alpha=0.25 + ) self.assertRaises(TypeError, test_x_type) def test_x_tensor_dtype(): x2 = fluid.layers.data(name='x2', shape=[10, 10], dtype="int16") - fluid.layers.sigmoid_focal_loss(x=x2, - label=label1, - fg_num=fg_num1, - gamma=2., - alpha=0.25) + fluid.layers.sigmoid_focal_loss( + x=x2, label=label1, fg_num=fg_num1, gamma=2.0, alpha=0.25 + ) self.assertRaises(TypeError, test_x_tensor_dtype) @@ -173,46 +173,38 @@ def test_x_tensor_dtype(): # The `label` must be Variable and the data type of `label` Tensor must be int32. def test_label_type(): label2 = [2] - fluid.layers.sigmoid_focal_loss(x=x3, - label=label2, - fg_num=fg_num1, - gamma=2., - alpha=0.25) + fluid.layers.sigmoid_focal_loss( + x=x3, label=label2, fg_num=fg_num1, gamma=2.0, alpha=0.25 + ) self.assertRaises(TypeError, test_label_type) def test_label_tensor_dtype(): - label3 = fluid.layers.fill_constant(shape=[10, 1], - dtype="float32", - value=1.) - fluid.layers.sigmoid_focal_loss(x=x3, - label=label3, - fg_num=fg_num1, - gamma=2., - alpha=0.25) + label3 = fluid.layers.fill_constant( + shape=[10, 1], dtype="float32", value=1.0 + ) + fluid.layers.sigmoid_focal_loss( + x=x3, label=label3, fg_num=fg_num1, gamma=2.0, alpha=0.25 + ) self.assertRaises(TypeError, test_label_tensor_dtype) # The `fg_num` must be Variable and the data type of `fg_num` Tensor must be int32. def test_fgnum_type(): fg_num2 = [2] - fluid.layers.sigmoid_focal_loss(x=x3, - label=label1, - fg_num=fg_num2, - gamma=2., - alpha=0.25) + fluid.layers.sigmoid_focal_loss( + x=x3, label=label1, fg_num=fg_num2, gamma=2.0, alpha=0.25 + ) self.assertRaises(TypeError, test_fgnum_type) def test_fgnum_tensor_dtype(): - fg_num3 = fluid.layers.fill_constant(shape=[1], - dtype="float32", - value=5.) - fluid.layers.sigmoid_focal_loss(x=x3, - label=label1, - fg_num=fg_num3, - gamma=2., - alpha=0.25) + fg_num3 = fluid.layers.fill_constant( + shape=[1], dtype="float32", value=5.0 + ) + fluid.layers.sigmoid_focal_loss( + x=x3, label=label1, fg_num=fg_num3, gamma=2.0, alpha=0.25 + ) self.assertRaises(TypeError, test_fgnum_tensor_dtype) diff --git a/python/paddle/fluid/tests/unittests/test_sign_op.py b/python/paddle/fluid/tests/unittests/test_sign_op.py index 1a55cfd3fd973..2db25d0eaee50 100644 --- a/python/paddle/fluid/tests/unittests/test_sign_op.py +++ b/python/paddle/fluid/tests/unittests/test_sign_op.py @@ -25,7 +25,6 @@ class TestSignOp(OpTest): - def setUp(self): self.op_type = "sign" self.inputs = { @@ -41,32 +40,30 @@ def test_check_grad(self): class TestSignOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of sign_op must be Variable or numpy.ndarray. input1 = 12 self.assertRaises(TypeError, fluid.layers.sign, input1) # The input dtype of sign_op must be float16, float32, float64. - input2 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") - input3 = fluid.layers.data(name='input3', - shape=[12, 10], - dtype="int64") + input2 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) + input3 = fluid.layers.data( + name='input3', shape=[12, 10], dtype="int64" + ) self.assertRaises(TypeError, fluid.layers.sign, input2) self.assertRaises(TypeError, fluid.layers.sign, input3) - input4 = fluid.layers.data(name='input4', - shape=[4], - dtype="float16") + input4 = fluid.layers.data( + name='input4', shape=[4], dtype="float16" + ) fluid.layers.sign(input4) class TestSignAPI(unittest.TestCase): - def test_dygraph(self): with fluid.dygraph.guard(): - np_x = np.array([-1., 0., -0., 1.2, 1.5], dtype='float64') + np_x = np.array([-1.0, 0.0, -0.0, 1.2, 1.5], dtype='float64') x = paddle.to_tensor(np_x) z = paddle.sign(x) np_z = z.numpy() @@ -79,22 +76,21 @@ def test_static(self): input1 = 12 self.assertRaises(TypeError, paddle.tensor.math.sign, input1) # The input dtype of sign_op must be float16, float32, float64. - input2 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") - input3 = fluid.layers.data(name='input3', - shape=[12, 10], - dtype="int64") + input2 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) + input3 = fluid.layers.data( + name='input3', shape=[12, 10], dtype="int64" + ) self.assertRaises(TypeError, paddle.tensor.math.sign, input2) self.assertRaises(TypeError, paddle.tensor.math.sign, input3) - input4 = fluid.layers.data(name='input4', - shape=[4], - dtype="float16") + input4 = fluid.layers.data( + name='input4', shape=[4], dtype="float16" + ) paddle.sign(input4) class TestSignDoubleGradCheck(unittest.TestCase): - def sign_wrapper(self, x): return paddle.sign(x[0]) @@ -109,17 +105,13 @@ def func(self, place): out = paddle.sign(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.sign_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.sign_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -131,7 +123,6 @@ def test_grad(self): class TestSignTripleGradCheck(unittest.TestCase): - def sign_wrapper(self, x): return paddle.sign(x[0]) @@ -146,17 +137,13 @@ def func(self, place): out = paddle.sign(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.sign_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.sign_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_signal.py b/python/paddle/fluid/tests/unittests/test_signal.py index d07e054bd4827..5e363dd39513b 100644 --- a/python/paddle/fluid/tests/unittests/test_signal.py +++ b/python/paddle/fluid/tests/unittests/test_signal.py @@ -57,7 +57,8 @@ def tiny(x): # Only floating types generate a tiny if np.issubdtype(x.dtype, np.floating) or np.issubdtype( - x.dtype, np.complexfloating): + x.dtype, np.complexfloating + ): dtype = x.dtype else: dtype = np.float32 @@ -71,8 +72,9 @@ def normalize(S, norm=np.inf, axis=0, threshold=None, fill=None): threshold = tiny(S) elif threshold <= 0: - raise Exception("threshold={} must be strictly " - "positive".format(threshold)) + raise Exception( + "threshold={} must be strictly " "positive".format(threshold) + ) if fill not in [None, False, True]: raise Exception("fill={} must be None or boolean".format(fill)) @@ -99,12 +101,12 @@ def normalize(S, norm=np.inf, axis=0, threshold=None, fill=None): length = np.sum(mag > 0, axis=axis, keepdims=True, dtype=mag.dtype) elif np.issubdtype(type(norm), np.number) and norm > 0: - length = np.sum(mag**norm, axis=axis, keepdims=True)**(1.0 / norm) + length = np.sum(mag**norm, axis=axis, keepdims=True) ** (1.0 / norm) if axis is None: - fill_norm = mag.size**(-1.0 / norm) + fill_norm = mag.size ** (-1.0 / norm) else: - fill_norm = mag.shape[axis]**(-1.0 / norm) + fill_norm = mag.shape[axis] ** (-1.0 / norm) elif norm is None: return S @@ -144,8 +146,9 @@ def __window_ss_fill(x, win_sq, n_frames, hop_length): # pragma: no cover n_fft = len(win_sq) for i in range(n_frames): sample = i * hop_length - x[sample:min(n, sample + - n_fft)] += win_sq[:max(0, min(n_fft, n - sample))] + x[sample : min(n, sample + n_fft)] += win_sq[ + : max(0, min(n_fft, n - sample)) + ] def window_sumsquare( @@ -165,7 +168,7 @@ def window_sumsquare( # Compute the squared window at the desired length win_sq = get_window(window, win_length) - win_sq = normalize(win_sq, norm=norm)**2 + win_sq = normalize(win_sq, norm=norm) ** 2 win_sq = pad_center(win_sq, n_fft) # Fill the envelope @@ -208,26 +211,35 @@ def dtype_r2c(d, default=np.complex64): def frame(x, frame_length, hop_length, axis=-1): if not isinstance(x, np.ndarray): - raise Exception("Input must be of type numpy.ndarray, " - "given type(x)={}".format(type(x))) + raise Exception( + "Input must be of type numpy.ndarray, " + "given type(x)={}".format(type(x)) + ) if x.shape[axis] < frame_length: - raise Exception("Input is too short (n={:d})" - " for frame_length={:d}".format(x.shape[axis], - frame_length)) + raise Exception( + "Input is too short (n={:d})" + " for frame_length={:d}".format(x.shape[axis], frame_length) + ) if hop_length < 1: raise Exception("Invalid hop_length: {:d}".format(hop_length)) if axis == -1 and not x.flags["F_CONTIGUOUS"]: - print("librosa.util.frame called with axis={} " - "on a non-contiguous input. This will result in a copy.".format( - axis)) + print( + "librosa.util.frame called with axis={} " + "on a non-contiguous input. This will result in a copy.".format( + axis + ) + ) x = np.asfortranarray(x) elif axis == 0 and not x.flags["C_CONTIGUOUS"]: - print("librosa.util.frame called with axis={} " - "on a non-contiguous input. This will result in a copy.".format( - axis)) + print( + "librosa.util.frame called with axis={} " + "on a non-contiguous input. This will result in a copy.".format( + axis + ) + ) x = np.ascontiguousarray(x) n_frames = 1 + (x.shape[axis] - frame_length) // hop_length @@ -260,8 +272,11 @@ def pad_center(data, size, axis=-1, **kwargs): lengths[axis] = (lpad, int(size - n - lpad)) if lpad < 0: - raise Exception(("Target size ({:d}) must be " - "at least input size ({:d})").format(size, n)) + raise Exception( + ("Target size ({:d}) must be " "at least input size ({:d})").format( + size, n + ) + ) return np.pad(data, lengths, **kwargs) @@ -279,8 +294,9 @@ def get_window(window, Nx, fftbins=True): if len(window) == Nx: return np.asarray(window) - raise Exception("Window size mismatch: " - "{:d} != {:d}".format(len(window), Nx)) + raise Exception( + "Window size mismatch: " "{:d} != {:d}".format(len(window), Nx) + ) else: raise Exception("Invalid window specification: {}".format(window)) @@ -294,16 +310,18 @@ def __overlap_add(y, ytmp, hop_length): n_fft = ytmp.shape[0] for frame in range(ytmp.shape[1]): sample = frame * hop_length - y[sample:(sample + n_fft)] += ytmp[:, frame] + y[sample : (sample + n_fft)] += ytmp[:, frame] -def stft(x, - n_fft=2048, - hop_length=None, - win_length=None, - window="hann", - center=True, - pad_mode="reflect"): +def stft( + x, + n_fft=2048, + hop_length=None, + win_length=None, + window="hann", + center=True, + pad_mode="reflect", +): y = x input_rank = len(y.shape) if input_rank == 2: @@ -330,15 +348,20 @@ def stft(x, # Pad the time series so that frames are centered if center: if n_fft > y.shape[-1]: - print("n_fft={} is too small for input signal of length={}".format( - n_fft, y.shape[-1])) + print( + "n_fft={} is too small for input signal of length={}".format( + n_fft, y.shape[-1] + ) + ) y = np.pad(y, int(n_fft // 2), mode=pad_mode) elif n_fft > y.shape[-1]: raise Exception( "n_fft={} is too large for input signal of length={}".format( - n_fft, y.shape[-1])) + n_fft, y.shape[-1] + ) + ) # Window the time series. y_frames = frame(y, frame_length=n_fft, hop_length=hop_length) @@ -347,9 +370,9 @@ def stft(x, dtype = dtype_r2c(y.dtype) # Pre-allocate the STFT matrix - stft_matrix = np.empty((int(1 + n_fft // 2), y_frames.shape[1]), - dtype=dtype, - order="F") + stft_matrix = np.empty( + (int(1 + n_fft // 2), y_frames.shape[1]), dtype=dtype, order="F" + ) # how many columns can we fit within MAX_MEM_BLOCK? n_columns = MAX_MEM_BLOCK // (stft_matrix.shape[0] * stft_matrix.itemsize) @@ -358,9 +381,9 @@ def stft(x, for bl_s in range(0, stft_matrix.shape[1], n_columns): bl_t = min(bl_s + n_columns, stft_matrix.shape[1]) - stft_matrix[:, - bl_s:bl_t] = fft.rfft(fft_window * y_frames[:, bl_s:bl_t], - axis=0) + stft_matrix[:, bl_s:bl_t] = fft.rfft( + fft_window * y_frames[:, bl_s:bl_t], axis=0 + ) if input_rank == 2: stft_matrix = np.expand_dims(stft_matrix, 0) @@ -405,8 +428,9 @@ def istft( padded_length = length + int(n_fft) else: padded_length = length - n_frames = min(stft_matrix.shape[1], - int(np.ceil(padded_length / hop_length))) + n_frames = min( + stft_matrix.shape[1], int(np.ceil(padded_length / hop_length)) + ) else: n_frames = stft_matrix.shape[1] @@ -428,7 +452,7 @@ def istft( ytmp = ifft_window * fft.irfft(stft_matrix[:, bl_s:bl_t], axis=0) # Overlap-add the istft block starting at the i'th frame - __overlap_add(y[frame * hop_length:], ytmp, hop_length) + __overlap_add(y[frame * hop_length :], ytmp, hop_length) frame += bl_t - bl_s @@ -449,7 +473,7 @@ def istft( # If we don't need to control length, just do the usual center trimming # to eliminate padded data if center: - y = y[int(n_fft // 2):-int(n_fft // 2)] + y = y[int(n_fft // 2) : -int(n_fft // 2)] else: if center: # If we're centering, crop off the first n_fft//2 samples @@ -506,8 +530,9 @@ def overlap_add_for_api_test(x, hop_length, axis=-1): frame_length = x.shape[1] if axis == 0 else x.shape[-2] # Assure no gaps between frames. - assert 0 < hop_length <= frame_length, \ - f'hop_length should be in (0, frame_length({frame_length})], but got {hop_length}.' + assert ( + 0 < hop_length <= frame_length + ), f'hop_length should be in (0, frame_length({frame_length})], but got {hop_length}.' seq_length = (n_frames - 1) * hop_length + frame_length @@ -528,7 +553,7 @@ def overlap_add_for_api_test(x, hop_length, axis=-1): for i in range(x.shape[0]): for frame in range(x.shape[-1]): sample = frame * hop_length - y[i, sample:sample + frame_length] += x[i, :, frame] + y[i, sample : sample + frame_length] += x[i, :, frame] if axis == 0: y = y.transpose((1, 0)) @@ -543,12 +568,10 @@ def overlap_add_for_api_test(x, hop_length, axis=-1): def place(devices, key='place'): - def decorate(cls): module = sys.modules[cls.__module__].__dict__ raw_classes = { - k: v - for k, v in module.items() if k.startswith(cls.__name__) + k: v for k, v in module.items() if k.startswith(cls.__name__) } for raw_name, raw_cls in raw_classes.items(): @@ -556,7 +579,7 @@ def decorate(cls): test_cls = dict(raw_cls.__dict__) test_cls.update({key: d}) new_name = raw_name + '.' + d.__class__.__name__ - module[new_name] = type(new_name, (raw_cls, ), test_cls) + module[new_name] = type(new_name, (raw_cls,), test_cls) del module[raw_name] return cls @@ -586,20 +609,23 @@ def tearDownModule(): pass -def rand_x(dims=1, - dtype='float64', - min_dim_len=1, - max_dim_len=10, - shape=None, - complex=False): +def rand_x( + dims=1, + dtype='float64', + min_dim_len=1, + max_dim_len=10, + shape=None, + complex=False, +): if shape is None: shape = [ np.random.randint(min_dim_len, max_dim_len) for i in range(dims) ] if complex: - return np.random.randn( - *shape).astype(dtype) + 1.j * np.random.randn(*shape).astype(dtype) + return np.random.randn(*shape).astype(dtype) + 1.0j * np.random.randn( + *shape + ).astype(dtype) else: return np.random.randn(*shape).astype(dtype) @@ -608,8 +634,11 @@ def parameterize(attrs, input_values=None): if isinstance(attrs, str): attrs = [attrs] - input_dicts = (attrs if input_values is None else - [dict(zip(attrs, vals)) for vals in input_values]) + input_dicts = ( + attrs + if input_values is None + else [dict(zip(attrs, vals)) for vals in input_values] + ) def decorator(base_class): test_class_module = sys.modules[base_class.__module__].__dict__ @@ -619,8 +648,7 @@ def decorator(base_class): name = class_name(base_class, idx, input_dict) - test_class_module[name] = type(name, (base_class, ), - test_class_dict) + test_class_module[name] = type(name, (base_class,), test_class_dict) for method_name in list(base_class.__dict__): if method_name.startswith("test"): @@ -632,7 +660,8 @@ def decorator(base_class): def class_name(cls, num, params_dict): suffix = to_safe_name( - next((v for v in params_dict.values() if isinstance(v, str)), "")) + next((v for v in params_dict.values() if isinstance(v, str)), "") + ) if TEST_CASE_NAME in params_dict: suffix = to_safe_name(params_dict["test_case"]) return "{}_{}{}".format(cls.__name__, num, suffix and "_" + suffix) @@ -652,19 +681,22 @@ def to_safe_name(s): ('test_2d_input2', rand_x(2, np.float64, shape=[8, 150]), 50, 15, -1), ('test_3d_input1', rand_x(3, np.float64, shape=[150, 4, 2]), 50, 15, 0), ('test_3d_input2', rand_x(3, np.float64, shape=[4, 2, 150]), 50, 15, -1), - ]) # yapf: disable + ]) # fmt: skip class TestFrame(unittest.TestCase): - def test_frame(self): - np.testing.assert_allclose(frame_for_api_test(self.x, self.frame_length, - self.hop_length, - self.axis), - paddle.signal.frame(paddle.to_tensor(self.x), - self.frame_length, - self.hop_length, - self.axis), - rtol=rtol.get(str(self.x.dtype)), - atol=atol.get(str(self.x.dtype))) + np.testing.assert_allclose( + frame_for_api_test( + self.x, self.frame_length, self.hop_length, self.axis + ), + paddle.signal.frame( + paddle.to_tensor(self.x), + self.frame_length, + self.hop_length, + self.axis, + ), + rtol=rtol.get(str(self.x.dtype)), + atol=atol.get(str(self.x.dtype)), + ) @place(DEVICES) @@ -677,29 +709,33 @@ def test_frame(self): ('test_2d_input2', rand_x(2, np.float64, shape=[8, 150]), 50, 15, -1), ('test_3d_input1', rand_x(3, np.float64, shape=[150, 4, 2]), 50, 15, 0), ('test_3d_input2', rand_x(3, np.float64, shape=[4, 2, 150]), 50, 15, -1), - ]) # yapf: disable + ]) # fmt: skip class TestFrameStatic(unittest.TestCase): - def test_frame_static(self): paddle.enable_static() mp, sp = paddle.static.Program(), paddle.static.Program() with paddle.static.program_guard(mp, sp): - input = paddle.static.data('input', - self.x.shape, - dtype=self.x.dtype) - output = paddle.signal.frame(input, self.frame_length, - self.hop_length, self.axis), + input = paddle.static.data( + 'input', self.x.shape, dtype=self.x.dtype + ) + output = ( + paddle.signal.frame( + input, self.frame_length, self.hop_length, self.axis + ), + ) exe = paddle.static.Executor(self.place) exe.run(sp) [output] = exe.run(mp, feed={'input': self.x}, fetch_list=[output]) paddle.disable_static() - np.testing.assert_allclose(frame_for_api_test(self.x, self.frame_length, - self.hop_length, - self.axis), - output, - rtol=rtol.get(str(self.x.dtype)), - atol=atol.get(str(self.x.dtype))) + np.testing.assert_allclose( + frame_for_api_test( + self.x, self.frame_length, self.hop_length, self.axis + ), + output, + rtol=rtol.get(str(self.x.dtype)), + atol=atol.get(str(self.x.dtype)), + ) @place(DEVICES) @@ -710,13 +746,16 @@ def test_frame_static(self): ('test_hop_length', rand_x(1, np.float64, shape=[150]), 50, 0, -1, ValueError), ('test_frame_length1', rand_x(2, np.float64, shape=[150, 8]), 0, 15, 0, ValueError), ('test_frame_length2', rand_x(2, np.float64, shape=[150, 8]), 151, 15, 0, ValueError), - ]) # yapf: disable + ]) # fmt: skip class TestFrameException(unittest.TestCase): - def test_frame(self): with self.assertRaises(self.expect_exception): - paddle.signal.frame(paddle.to_tensor(self.x), self.frame_length, - self.hop_length, self.axis) + paddle.signal.frame( + paddle.to_tensor(self.x), + self.frame_length, + self.hop_length, + self.axis, + ) @place(DEVICES) @@ -729,16 +768,17 @@ def test_frame(self): ('test_3d_input2', rand_x(3, np.float64, shape=[2, 40, 5]), 10, -1), ('test_4d_input1', rand_x(4, np.float64, shape=[8, 12, 5, 3]), 5, 0), ('test_4d_input2', rand_x(4, np.float64, shape=[3, 5, 12, 8]), 5, -1), - ]) # yapf: disable + ]) # fmt: skip class TestOverlapAdd(unittest.TestCase): - def test_overlap_add(self): np.testing.assert_allclose( overlap_add_for_api_test(self.x, self.hop_length, self.axis), - paddle.signal.overlap_add(paddle.to_tensor(self.x), self.hop_length, - self.axis), + paddle.signal.overlap_add( + paddle.to_tensor(self.x), self.hop_length, self.axis + ), rtol=rtol.get(str(self.x.dtype)), - atol=atol.get(str(self.x.dtype))) + atol=atol.get(str(self.x.dtype)), + ) @place(DEVICES) @@ -751,28 +791,29 @@ def test_overlap_add(self): ('test_3d_input2', rand_x(3, np.float64, shape=[2, 40, 5]), 10, -1), ('test_4d_input1', rand_x(4, np.float64, shape=[8, 12, 5, 3]), 5, 0), ('test_4d_input2', rand_x(4, np.float64, shape=[3, 5, 12, 8]), 5, -1), - ]) # yapf: disable + ]) # fmt: skip class TestOverlapAddStatic(unittest.TestCase): - def test_overlap_add_static(self): paddle.enable_static() mp, sp = paddle.static.Program(), paddle.static.Program() with paddle.static.program_guard(mp, sp): - input = paddle.static.data('input', - self.x.shape, - dtype=self.x.dtype) - output = paddle.signal.overlap_add(input, self.hop_length, - self.axis), + input = paddle.static.data( + 'input', self.x.shape, dtype=self.x.dtype + ) + output = ( + paddle.signal.overlap_add(input, self.hop_length, self.axis), + ) exe = paddle.static.Executor(self.place) exe.run(sp) [output] = exe.run(mp, feed={'input': self.x}, fetch_list=[output]) paddle.disable_static() - np.testing.assert_allclose(overlap_add_for_api_test( - self.x, self.hop_length, self.axis), - output, - rtol=rtol.get(str(self.x.dtype)), - atol=atol.get(str(self.x.dtype))) + np.testing.assert_allclose( + overlap_add_for_api_test(self.x, self.hop_length, self.axis), + output, + rtol=rtol.get(str(self.x.dtype)), + atol=atol.get(str(self.x.dtype)), + ) @place(DEVICES) @@ -781,13 +822,13 @@ def test_overlap_add_static(self): [ ('test_axis', rand_x(2, np.float64, shape=[3, 50]), 4, 2, ValueError), ('test_hop_length', rand_x(2, np.float64, shape=[50, 3]), -1, -1, ValueError), - ]) # yapf: disable + ]) # fmt: skip class TestOverlapAddException(unittest.TestCase): - def test_overlap_add(self): with self.assertRaises(self.expect_exception): - paddle.signal.overlap_add(paddle.to_tensor(self.x), self.hop_length, - self.axis) + paddle.signal.overlap_add( + paddle.to_tensor(self.x), self.hop_length, self.axis + ) # ================= STFT @@ -836,9 +877,8 @@ def test_overlap_add(self): 512, None, None, None, True, 'reflect', False, True), ('test_center', rand_x(2, np.float64, shape=[1, 160000]), 512, None, None, None, False, 'reflect', False, True), - ])# yapf: disable + ]) # fmt: skip class TestStft(unittest.TestCase): - def test_stft(self): if self.window is None: win_p = None @@ -848,14 +888,29 @@ def test_stft(self): win_l = self.window np.testing.assert_allclose( - stft(self.x, self.n_fft, self.hop_length, self.win_length, win_l, - self.center, self.pad_mode), - paddle.signal.stft(paddle.to_tensor(self.x), self.n_fft, - self.hop_length, self.win_length, win_p, - self.center, self.pad_mode, self.normalized, - self.onesided), + stft( + self.x, + self.n_fft, + self.hop_length, + self.win_length, + win_l, + self.center, + self.pad_mode, + ), + paddle.signal.stft( + paddle.to_tensor(self.x), + self.n_fft, + self.hop_length, + self.win_length, + win_p, + self.center, + self.pad_mode, + self.normalized, + self.onesided, + ), rtol=rtol.get(str(self.x.dtype)), - atol=atol.get(str(self.x.dtype))) + atol=atol.get(str(self.x.dtype)), + ) @place(DEVICES) @@ -878,9 +933,8 @@ def test_stft(self): 512, None, None, None, True, 'nonsense', False, True, AssertionError), ('test_complex_onesided', rand_x(1, np.float64, shape=[16000], complex=True), 512, None, None, None, False, 'reflect', False, True, AssertionError), - ]) # yapf: disable + ]) # fmt: skip class TestStftException(unittest.TestCase): - def test_stft(self): if self.window is None: win_p = None @@ -888,10 +942,17 @@ def test_stft(self): win_p = paddle.to_tensor(self.window) with self.assertRaises(self.expect_exception): - paddle.signal.stft(paddle.to_tensor(self.x), self.n_fft, - self.hop_length, self.win_length, win_p, - self.center, self.pad_mode, self.normalized, - self.onesided), + paddle.signal.stft( + paddle.to_tensor(self.x), + self.n_fft, + self.hop_length, + self.win_length, + win_p, + self.center, + self.pad_mode, + self.normalized, + self.onesided, + ), @place(DEVICES) @@ -912,9 +973,8 @@ def test_stft(self): 512, None, None, None, False, False, True, None, False), ('test_length', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, None, None, None, False, False, True, 1888, False), - ]) # yapf: disable + ]) # fmt: skip class TestIstft(unittest.TestCase): - def test_istft(self): if self.window is None: win_p = None @@ -924,14 +984,29 @@ def test_istft(self): win_l = self.window np.testing.assert_allclose( - istft(self.x, self.hop_length, self.win_length, win_l, self.center, - self.length), - paddle.signal.istft(paddle.to_tensor(self.x), self.n_fft, - self.hop_length, self.win_length, win_p, - self.center, self.normalized, self.onesided, - self.length, self.return_complex), + istft( + self.x, + self.hop_length, + self.win_length, + win_l, + self.center, + self.length, + ), + paddle.signal.istft( + paddle.to_tensor(self.x), + self.n_fft, + self.hop_length, + self.win_length, + win_p, + self.center, + self.normalized, + self.onesided, + self.length, + self.return_complex, + ), rtol=rtol.get(str(self.x.dtype)), - atol=atol.get(str(self.x.dtype))) + atol=atol.get(str(self.x.dtype)), + ) @place(DEVICES) @@ -962,9 +1037,8 @@ def test_istft(self): 512, None, None, rand_x(1, np.float64, shape=[512], complex=True), True, False, True, None, False, AssertionError), ('test_NOLA', rand_x(3, np.float64, shape=[1, 257, 471], complex=True), 512, 512, None, get_window('hann', 512), True, False, True, None, False, ValueError), - ]) # yapf: disable + ]) # fmt: skip class TestIstftException(unittest.TestCase): - def test_istft(self): if self.window is None: win_p = None @@ -972,10 +1046,18 @@ def test_istft(self): win_p = paddle.to_tensor(self.window) with self.assertRaises(self.expect_exception): - paddle.signal.istft(paddle.to_tensor(self.x), self.n_fft, - self.hop_length, self.win_length, win_p, - self.center, self.normalized, self.onesided, - self.length, self.return_complex), + paddle.signal.istft( + paddle.to_tensor(self.x), + self.n_fft, + self.hop_length, + self.win_length, + win_p, + self.center, + self.normalized, + self.onesided, + self.length, + self.return_complex, + ), if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_similarity_focus_op.py b/python/paddle/fluid/tests/unittests/test_similarity_focus_op.py index d7d13bdd39e8f..15a050211a15d 100755 --- a/python/paddle/fluid/tests/unittests/test_similarity_focus_op.py +++ b/python/paddle/fluid/tests/unittests/test_similarity_focus_op.py @@ -20,17 +20,25 @@ class TestSimilarityFocusOp(OpTest): - def setUp(self): self.op_type = "similarity_focus" batch_size = 2 x_dim, y_dim, z_dim = 3, 2, 2 self.inputs = { - 'X': - np.array([[[[0.8, 0.1], [0.4, 0.5]], [[0.9, 0.7], [0.9, 0.9]], - [[0.8, 0.9], [0.1, 0.2]]], - [[[0.2, 0.5], [0.3, 0.4]], [[0.9, 0.7], [0.8, 0.4]], - [[0.0, 0.2], [0.4, 0.7]]]]), + 'X': np.array( + [ + [ + [[0.8, 0.1], [0.4, 0.5]], + [[0.9, 0.7], [0.9, 0.9]], + [[0.8, 0.9], [0.1, 0.2]], + ], + [ + [[0.2, 0.5], [0.3, 0.4]], + [[0.9, 0.7], [0.8, 0.4]], + [[0.0, 0.2], [0.4, 0.7]], + ], + ] + ), } self.attrs = { 'axis': 1, @@ -41,8 +49,9 @@ def setUp(self): for batch in range(batch_size): res = np.zeros((1, y_dim, z_dim)).astype("float32").reshape(-1) for index in self.attrs['indexes']: - channel = self.inputs['X'][batch, - index, :, :].reshape(-1).copy() + channel = ( + self.inputs['X'][batch, index, :, :].reshape(-1).copy() + ) tag1 = [0 for i in range(y_dim)] tag2 = [0 for i in range(z_dim)] cnt = 0 @@ -71,15 +80,14 @@ def test_check_output(self): class TestSimilarityFocusOp_axis1(OpTest): - def setUp(self): self.op_type = "similarity_focus" batch_size = 3 x_dim, y_dim, z_dim = 4, 5, 6 self.inputs = { - 'X': - np.random.random( - (batch_size, x_dim, y_dim, z_dim)).astype("float32"), + 'X': np.random.random((batch_size, x_dim, y_dim, z_dim)).astype( + "float32" + ), } self.attrs = { 'axis': 1, @@ -90,8 +98,9 @@ def setUp(self): for batch in range(batch_size): res = np.zeros((1, y_dim, z_dim)).astype("float32").reshape(-1) for index in self.attrs['indexes']: - channel = self.inputs['X'][batch, - index, :, :].reshape(-1).copy() + channel = ( + self.inputs['X'][batch, index, :, :].reshape(-1).copy() + ) tag1 = [0 for i in range(y_dim)] tag2 = [0 for i in range(z_dim)] cnt = 0 @@ -121,15 +130,14 @@ def test_check_output(self): class TestSimilarityFocusOp_axis2(OpTest): - def setUp(self): self.op_type = "similarity_focus" batch_size = 6 x_dim, y_dim, z_dim = 7, 8, 9 self.inputs = { - 'X': - np.random.random( - (batch_size, x_dim, y_dim, z_dim)).astype("float32"), + 'X': np.random.random((batch_size, x_dim, y_dim, z_dim)).astype( + "float32" + ), } self.attrs = { 'axis': 2, @@ -140,8 +148,9 @@ def setUp(self): for batch in range(batch_size): res = np.zeros((x_dim, 1, z_dim)).astype("float32").reshape(-1) for index in self.attrs['indexes']: - channel = self.inputs['X'][batch, :, - index, :].reshape(-1).copy() + channel = ( + self.inputs['X'][batch, :, index, :].reshape(-1).copy() + ) tag1 = [0 for i in range(x_dim)] tag2 = [0 for i in range(z_dim)] cnt = 0 @@ -171,15 +180,14 @@ def test_check_output(self): class TestSimilarityFocusOp_axis3(OpTest): - def setUp(self): self.op_type = "similarity_focus" batch_size = 64 x_dim, y_dim, z_dim = 48, 48, 13 self.inputs = { - 'X': - np.random.random( - (batch_size, x_dim, y_dim, z_dim)).astype("float32"), + 'X': np.random.random((batch_size, x_dim, y_dim, z_dim)).astype( + "float32" + ), } self.attrs = { 'axis': 3, @@ -190,8 +198,9 @@ def setUp(self): for batch in range(batch_size): res = np.zeros((x_dim, y_dim, 1)).astype("float32").reshape(-1) for index in self.attrs['indexes']: - channel = self.inputs['X'][batch, :, :, - index].reshape(-1).copy() + channel = ( + self.inputs['X'][batch, :, :, index].reshape(-1).copy() + ) tag1 = [0 for i in range(x_dim)] tag2 = [0 for i in range(y_dim)] cnt = 0 @@ -221,32 +230,31 @@ def test_check_output(self): class TestSimilarityFocusOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): data = fluid.data(name='data', shape=[16, 3, 2, 2], dtype='float32') def test_input_Variable(): input = np.random.rand(16, 3, 2, 2).astype("float32") - out = fluid.layers.similarity_focus(input=input, - axis=1, - indexes=[0]) + out = fluid.layers.similarity_focus( + input=input, axis=1, indexes=[0] + ) self.assertRaises(TypeError, test_input_Variable) def test_axis_Int(): axis = 1.0 - out = fluid.layers.similarity_focus(input=data, - axis=axis, - indexes=[0]) + out = fluid.layers.similarity_focus( + input=data, axis=axis, indexes=[0] + ) self.assertRaises(TypeError, test_axis_Int) def test_indexes_List(): indexes = 0 - out = fluid.layers.similarity_focus(input=data, - axis=1, - indexes=indexes) + out = fluid.layers.similarity_focus( + input=data, axis=1, indexes=indexes + ) self.assertRaises(TypeError, test_indexes_List) diff --git a/python/paddle/fluid/tests/unittests/test_simple_rnn_op.py b/python/paddle/fluid/tests/unittests/test_simple_rnn_op.py index 4a0f02204c7de..fa129cda06627 100644 --- a/python/paddle/fluid/tests/unittests/test_simple_rnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_simple_rnn_op.py @@ -31,7 +31,6 @@ class TestSimpleRNNOp(OpTest): - def get_weight_names(self): weight_names = [] for i in range(self.num_layers): @@ -45,13 +44,16 @@ def get_weight_names(self): def setUp(self): self.op_type = "rnn" self.dtype = "float32" if core.is_compiled_with_rocm() else "float64" - self.sequence_length = None if core.is_compiled_with_rocm( - ) else np.array([12, 11, 10, 9, 8], dtype=np.int32) + self.sequence_length = ( + None + if core.is_compiled_with_rocm() + else np.array([12, 11, 10, 9, 8], dtype=np.int32) + ) self.num_layers = 1 self.is_bidirec = False self.is_test = False self.mode = "RNN_TANH" - self.dropout = 0. + self.dropout = 0.0 self.set_attrs() self.direction_num = 2 if self.is_bidirec else 1 @@ -61,31 +63,33 @@ def setUp(self): input_size = 3 hidden_size = 2 - input = np.random.uniform(low=-0.1, - high=0.1, - size=(seq_length, batch_size, - input_size)).astype(self.dtype) + input = np.random.uniform( + low=-0.1, high=0.1, size=(seq_length, batch_size, input_size) + ).astype(self.dtype) if self.sequence_length is not None: input[11][1:][:] = 0 input[10][2:][:] = 0 input[9][3:][:] = 0 input[8][4:][:] = 0 - rnn1 = SimpleRNN(input_size, - hidden_size, - num_layers=self.num_layers, - time_major=True, - direction=direction, - dropout=self.dropout, - nonlinearity=self.mode, - dtype=self.dtype) + rnn1 = SimpleRNN( + input_size, + hidden_size, + num_layers=self.num_layers, + time_major=True, + direction=direction, + dropout=self.dropout, + nonlinearity=self.mode, + dtype=self.dtype, + ) flat_w = get_params_for_net(rnn1) output, last_hidden = rnn1(input, sequence_length=self.sequence_length) - init_h = np.zeros((self.num_layers * self.direction_num, batch_size, - hidden_size)).astype(self.dtype) + init_h = np.zeros( + (self.num_layers * self.direction_num, batch_size, hidden_size) + ).astype(self.dtype) state_out = np.ndarray((300)).astype("uint8") @@ -93,13 +97,13 @@ def setUp(self): 'Input': input, 'WeightList': flat_w, 'PreState': [('init_h', init_h)], - 'SequenceLength': self.sequence_length + 'SequenceLength': self.sequence_length, } if self.sequence_length is None: self.inputs = { 'Input': input, 'WeightList': flat_w, - 'PreState': [('init_h', init_h)] + 'PreState': [('init_h', init_h)], } self.attrs = { 'dropout_prob': self.dropout, @@ -108,13 +112,13 @@ def setUp(self): 'hidden_size': hidden_size, 'num_layers': self.num_layers, 'is_test': self.is_test, - 'mode': self.mode + 'mode': self.mode, } self.outputs = { 'Out': output, 'State': [('last_hidden', last_hidden)], 'Reserve': np.ndarray((400)).astype("uint8"), - 'DropoutState': state_out + 'DropoutState': state_out, } def set_attrs(self): @@ -132,27 +136,23 @@ def test_grad(self): class TestSimpleRNNOp1(TestSimpleRNNOp): - def set_attrs(self): self.sequence_length = None class TestSimpleRNNOp2(TestSimpleRNNOp): - def set_attrs(self): self.sequence_length = None self.is_bidirec = True class TestSimpleRNNOp3(TestSimpleRNNOp): - def set_attrs(self): self.sequence_length = None self.is_test = True class TestSimpleRNNOp4(TestSimpleRNNOp): - def set_attrs(self): self.sequence_length = None self.is_bidirec = True @@ -160,7 +160,6 @@ def set_attrs(self): class TestSimpleRNNOp5(TestSimpleRNNOp): - def set_attrs(self): self.mode = "RNN_RELU" diff --git a/python/paddle/fluid/tests/unittests/test_size_op.py b/python/paddle/fluid/tests/unittests/test_size_op.py index bb64e3e66b24b..6f9898ade4fb0 100644 --- a/python/paddle/fluid/tests/unittests/test_size_op.py +++ b/python/paddle/fluid/tests/unittests/test_size_op.py @@ -20,7 +20,6 @@ class TestSizeOp(OpTest): - def setUp(self): self.op_type = "size" self.shape = [] @@ -37,31 +36,26 @@ def test_check_output(self): class TestRank1Tensor(TestSizeOp): - def config(self): self.shape = [2] class TestRank2Tensor(TestSizeOp): - def config(self): self.shape = [2, 3] class TestRank3Tensor(TestSizeOp): - def config(self): self.shape = [2, 3, 100] class TestLargeTensor(TestSizeOp): - def config(self): self.shape = [2**10] class TestSizeAPI(unittest.TestCase): - def test_size_static(self): main_program = fluid.Program() startup_program = fluid.Program() @@ -75,17 +69,19 @@ def test_size_static(self): out_1 = paddle.fluid.layers.size(x_1) out_2 = paddle.fluid.layers.size(x_2) exe = paddle.static.Executor(place=paddle.CPUPlace()) - res_1, res_2 = exe.run(feed={ - "x_1": input_1, - "x_2": input_2, - }, - fetch_list=[out_1, out_2]) - assert (np.array_equal(res_1, - np.array([np.size(input_1) - ]).astype("int64"))) - assert (np.array_equal(res_2, - np.array([np.size(input_2) - ]).astype("int64"))) + res_1, res_2 = exe.run( + feed={ + "x_1": input_1, + "x_2": input_2, + }, + fetch_list=[out_1, out_2], + ) + assert np.array_equal( + res_1, np.array([np.size(input_1)]).astype("int64") + ) + assert np.array_equal( + res_2, np.array([np.size(input_2)]).astype("int64") + ) def test_size_imperative(self): paddle.disable_static(paddle.CPUPlace()) @@ -95,8 +91,8 @@ def test_size_imperative(self): x_2 = paddle.to_tensor(input_2) out_1 = paddle.fluid.layers.size(x_1) out_2 = paddle.fluid.layers.size(x_2) - assert (np.array_equal(out_1.numpy().item(0), np.size(input_1))) - assert (np.array_equal(out_2.numpy().item(0), np.size(input_2))) + assert np.array_equal(out_1.numpy().item(0), np.size(input_1)) + assert np.array_equal(out_2.numpy().item(0), np.size(input_2)) paddle.enable_static() def test_error(self): diff --git a/python/paddle/fluid/tests/unittests/test_slice_op.py b/python/paddle/fluid/tests/unittests/test_slice_op.py index bf8b9e92209d5..59cd41ae9de21 100644 --- a/python/paddle/fluid/tests/unittests/test_slice_op.py +++ b/python/paddle/fluid/tests/unittests/test_slice_op.py @@ -30,7 +30,6 @@ # Situation 1: starts(list, no tensor), ends(list, no tensor) # 1.1 without attr(decrease) class TestSliceOp(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -40,7 +39,7 @@ def setUp(self): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -59,7 +58,6 @@ def test_check_grad_normal(self): class TestCase1(TestSliceOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float64") self.starts = [-3, 0, 2] @@ -70,7 +68,6 @@ def config(self): class TestCase2(TestSliceOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float64") self.starts = [-3, 0, 2] @@ -81,7 +78,6 @@ def config(self): class TestSliceZerosShapeTensor(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -92,7 +88,7 @@ def setUp(self): 'starts': self.starts, 'ends': self.ends, 'infer_flags': self.infer_flags, - 'use_mkldnn': True + 'use_mkldnn': True, } def config(self): @@ -109,7 +105,6 @@ def test_check_output(self): # 1.2 with attr(decrease) class TestSliceOp_decs_dim(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -140,7 +135,6 @@ def test_check_grad_normal(self): class TestSliceOp_decs_dim_2(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float64") self.starts = [1, 0, 2] @@ -152,7 +146,6 @@ def config(self): class TestSliceOp_decs_dim_3(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float64") self.starts = [-1, 0, 2] @@ -164,7 +157,6 @@ def config(self): class TestSliceOp_decs_dim_4(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 7]).astype("float64") self.starts = [0, 1, 2, 3] @@ -176,7 +168,6 @@ def config(self): class TestSliceOp_decs_dim_5(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float64") self.starts = [-1] @@ -188,7 +179,6 @@ def config(self): class TestSliceOp_decs_dim_6(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float64") self.starts = [0, 1, 2, 3] @@ -202,15 +192,15 @@ def config(self): # Situation 2: starts(list, have tensor), ends(list, no tensor) # without attr(decrease) class TestSliceOp_starts_ListTensor(OpTest): - def setUp(self): self.op_type = "slice" self.config() starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int64') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int64') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} self.outputs = {'Out': self.out} @@ -218,7 +208,7 @@ def setUp(self): 'axes': self.axes, 'starts': self.starts_infer, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -241,15 +231,15 @@ def test_check_grad_normal(self): # Situation 2: starts(list, have tensor), ends(list, no tensor) # with attr(decrease) class TestSliceOp_decs_dim_starts_ListTensor(OpTest): - def setUp(self): self.op_type = "slice" self.config() starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} @@ -281,8 +271,8 @@ def test_check_grad_normal(self): class TestSliceOp_decs_dim_5_starts_ListTensor( - TestSliceOp_decs_dim_starts_ListTensor): - + TestSliceOp_decs_dim_starts_ListTensor +): def config(self): self.input = np.random.random([3, 4, 5, 6]).astype("float64") self.starts = [-1] @@ -298,13 +288,12 @@ def config(self): # Situation 3: starts(tensor), ends(list, no tensor) # with attr(decrease) class TestSliceOp_decs_dim_starts_OneTensor(OpTest): - def setUp(self): self.op_type = "slice" self.config() self.inputs = { 'Input': self.input, - "StartsTensor": np.array(self.starts, dtype="int32") + "StartsTensor": np.array(self.starts, dtype="int32"), } self.outputs = {'Out': self.out} self.attrs = { @@ -334,7 +323,6 @@ def test_check_grad_normal(self): # Situation 4: starts(tensor), ends(tensor) # without attr(decrease) class TestSliceOp_starts_OneTensor_ends_OneTensor(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -342,14 +330,14 @@ def setUp(self): self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int64"), - "EndsTensor": np.array(self.ends, dtype="int32") + "EndsTensor": np.array(self.ends, dtype="int32"), } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, #'starts': self.starts, #'ends': self.ends_infer, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -370,14 +358,13 @@ def test_check_grad_normal(self): # Situation 5: starts(tensor), ends(tensor) # with attr(decrease) class TestSliceOp_decs_dim_starts_and_ends_OneTensor(OpTest): - def setUp(self): self.op_type = "slice" self.config() self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int32"), - "EndsTensor": np.array(self.ends, dtype="int32") + "EndsTensor": np.array(self.ends, dtype="int32"), } self.outputs = {'Out': self.out} self.attrs = { @@ -407,27 +394,27 @@ def test_check_grad_normal(self): # Situation 6: starts(tensor), ends(list, have tensor) # without attr(decrease) class TestSliceOp_starts_OneTensor_ends_ListTensor(OpTest): - def setUp(self): self.op_type = "slice" self.config() ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("y" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("y" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int32"), - 'EndsTensorList': ends_tensor + 'EndsTensorList': ends_tensor, } self.outputs = {'Out': self.out} self.attrs = { 'axes': self.axes, #'starts': self.starts, 'ends': self.ends_infer, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -448,10 +435,10 @@ def test_check_grad_normal(self): # Test CUDA float16 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -461,7 +448,7 @@ def setUp(self): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -481,15 +468,15 @@ def test_check_output(self): def test_check_grad_normal(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['Input'], - 'Out', - max_relative_error=0.006) + self.check_grad_with_place( + place, ['Input'], 'Out', max_relative_error=0.006 + ) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16_2(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -499,7 +486,7 @@ def setUp(self): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -519,14 +506,16 @@ def test_check_output(self): def test_check_grad_normal(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): - self.check_grad_with_place(place, ['Input'], - 'Out', - max_relative_error=0.006, - numeric_grad_delta=0.5) + self.check_grad_with_place( + place, + ['Input'], + 'Out', + max_relative_error=0.006, + numeric_grad_delta=0.5, + ) class TestBF16(OpTest): - def setUp(self): self.op_type = "slice" self.config() @@ -536,7 +525,7 @@ def setUp(self): 'axes': self.axes, 'starts': self.starts, 'ends': self.ends, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -557,38 +546,36 @@ def test_check_grad_normal(self): # Test python API class TestSliceAPI(unittest.TestCase): - def test_1(self): input = np.random.random([3, 4, 5, 6]).astype("float64") minus_1 = fluid.layers.fill_constant([1], "int32", -1) minus_3 = fluid.layers.fill_constant([1], "int64", -3) - starts = fluid.layers.data(name='starts', - shape=[1, 3], - append_batch_size=False) - ends = fluid.layers.data(name='ends', - shape=[3], - append_batch_size=False) - - x = fluid.layers.data(name="x", - shape=[3, 4, 5, 6], - append_batch_size=False, - dtype="float64") + starts = fluid.layers.data( + name='starts', shape=[1, 3], append_batch_size=False + ) + ends = fluid.layers.data( + name='ends', shape=[3], append_batch_size=False + ) + + x = fluid.layers.data( + name="x", + shape=[3, 4, 5, 6], + append_batch_size=False, + dtype="float64", + ) # value_int64 is greater than 2147483647 which is the max of int32 value_int64 = fluid.layers.fill_constant([1], "int64", 2147483648) - out_1 = paddle.slice(x, - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[value_int64, 100, -1]) - out_2 = paddle.slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, -1]) - out_3 = paddle.slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, minus_1]) + out_1 = paddle.slice( + x, axes=[0, 1, 2], starts=[-3, 0, 2], ends=[value_int64, 100, -1] + ) + out_2 = paddle.slice( + x, axes=[0, 1, 3], starts=[minus_3, 0, 2], ends=[3, 100, -1] + ) + out_3 = paddle.slice( + x, axes=[0, 1, 3], starts=[minus_3, 0, 2], ends=[3, 100, minus_1] + ) out_4 = paddle.slice(x, axes=[0, 1, 2], starts=starts, ends=ends) out_5 = x[-3:3, 0:100, 2:-1] @@ -601,9 +588,10 @@ def test_1(self): feed={ "x": input, 'starts': np.array([-3, 0, 2]).astype("int32"), - 'ends': np.array([3, 100, -1]).astype("int32") + 'ends': np.array([3, 100, -1]).astype("int32"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7], + ) assert np.array_equal(res_1, input[-3:3, 0:100, 2:-1, :]) assert np.array_equal(res_2, input[-3:3, 0:100, :, 2:-1]) @@ -615,17 +603,18 @@ def test_1(self): class TestSliceApiWithTensor(unittest.TestCase): - def test_starts_ends_is_tensor(self): with paddle.fluid.dygraph.guard(): a = paddle.rand(shape=[4, 5, 6], dtype='float32') axes = [0, 1, 2] starts = [-3, 0, 2] ends = [3, 2, 4] - a_1 = paddle.slice(a, - axes=axes, - starts=paddle.to_tensor(starts, dtype='int32'), - ends=paddle.to_tensor(ends, dtype='int32')) + a_1 = paddle.slice( + a, + axes=axes, + starts=paddle.to_tensor(starts, dtype='int32'), + ends=paddle.to_tensor(ends, dtype='int32'), + ) a_2 = paddle.slice(a, axes=axes, starts=starts, ends=ends) np.testing.assert_array_equal(a_1.numpy(), a_2.numpy()) @@ -648,7 +637,6 @@ def test_bool_tensor(self): class TestSliceApiEager(unittest.TestCase): - def test_slice_api(self): with paddle.fluid.dygraph.guard(): with _test_eager_guard(): @@ -659,23 +647,24 @@ def test_slice_api(self): ends = [3, 2, 4] a_1 = paddle.slice(a, axes=axes, starts=starts, ends=ends) - a_2 = paddle.slice(a, - axes=axes, - starts=paddle.to_tensor(starts), - ends=paddle.to_tensor(ends)) + a_2 = paddle.slice( + a, + axes=axes, + starts=paddle.to_tensor(starts), + ends=paddle.to_tensor(ends), + ) np.testing.assert_array_equal(a_1.numpy(), a_2.numpy()) a_1.backward() grad_truth = paddle.zeros_like(a) grad_truth[-3:3, 0:2, 2:4] = 1 np.testing.assert_array_equal(grad_truth, a.gradient()) - np.testing.assert_allclose(a_1.numpy(), - a[-3:3, 0:2, 2:4], - rtol=1e-05) + np.testing.assert_allclose( + a_1.numpy(), a[-3:3, 0:2, 2:4], rtol=1e-05 + ) class TestSliceApiWithLoDTensorArray(unittest.TestCase): - def setUp(self): self.shape = (3, 4) self.data = np.random.random(size=self.shape).astype('float32') @@ -684,8 +673,11 @@ def setUp(self): self.end = 2 self.axis = 1 - self.place = fluid.CUDAPlace( - 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.exe = fluid.Executor(self.place) def set_program_and_run(self, main_program, case_num): @@ -693,7 +685,7 @@ def set_program_and_run(self, main_program, case_num): x = [ fluid.data(name='x0', shape=self.shape, dtype="float32"), fluid.data(name='x1', shape=self.shape, dtype="float32"), - fluid.data(name='x2', shape=self.shape, dtype="float32") + fluid.data(name='x2', shape=self.shape, dtype="float32"), ] for each_x in x: @@ -708,31 +700,35 @@ def set_program_and_run(self, main_program, case_num): self.sliced_arr = output = arr[0] elif case_num == 2: - end = fluid.layers.array_length( - arr) - 1 # dtype of end is int64 - self.sliced_arr = slice_arr = arr[self.start:end] - output, _ = fluid.layers.tensor_array_to_tensor(slice_arr, - axis=self.axis, - use_stack=True) + end = ( + fluid.layers.array_length(arr) - 1 + ) # dtype of end is int64 + self.sliced_arr = slice_arr = arr[self.start : end] + output, _ = fluid.layers.tensor_array_to_tensor( + slice_arr, axis=self.axis, use_stack=True + ) elif case_num == 3: - value_int64 = fluid.layers.fill_constant([1], "int64", - 2147483648) - self.sliced_arr = slice_arr = arr[self.start:value_int64] - output, _ = fluid.layers.tensor_array_to_tensor(slice_arr, - axis=self.axis, - use_stack=True) + value_int64 = fluid.layers.fill_constant( + [1], "int64", 2147483648 + ) + self.sliced_arr = slice_arr = arr[self.start : value_int64] + output, _ = fluid.layers.tensor_array_to_tensor( + slice_arr, axis=self.axis, use_stack=True + ) loss = fluid.layers.reduce_sum(output) fluid.backward.append_backward(loss) g_vars = list( - map(main_program.global_block().var, - [each_x.name + "@GRAD" for each_x in x])) - self.out, self.g_x0, self.g_x1, self.g_x2 = \ - self.exe.run(main_program, - feed = {'x0': self.data, - 'x1': self.data, - 'x2': self.data}, - fetch_list=[output] + g_vars) + map( + main_program.global_block().var, + [each_x.name + "@GRAD" for each_x in x], + ) + ) + self.out, self.g_x0, self.g_x1, self.g_x2 = self.exe.run( + main_program, + feed={'x0': self.data, 'x1': self.data, 'x2': self.data}, + fetch_list=[output] + g_vars, + ) def test_case_1(self): main_program = fluid.Program() @@ -750,10 +746,12 @@ def test_case_2(self): self.set_program_and_run(main_program, 2) self.assertTrue( - self.sliced_arr.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY) + self.sliced_arr.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + ) self.assertEqual(self.sliced_arr.shape, self.shape) np.testing.assert_array_equal( - self.out, np.stack([self.data, self.data], axis=self.axis)) + self.out, np.stack([self.data, self.data], axis=self.axis) + ) np.testing.assert_array_equal(self.g_x0, np.ones_like(self.data)) np.testing.assert_array_equal(self.g_x1, np.ones_like(self.data)) np.testing.assert_array_equal(self.g_x2, np.zeros_like(self.data)) @@ -763,35 +761,35 @@ def test_case_3(self): self.set_program_and_run(main_program, 3) self.assertTrue( - self.sliced_arr.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY) + self.sliced_arr.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY + ) self.assertEqual(self.sliced_arr.shape, self.shape) np.testing.assert_array_equal( - self.out, np.stack([self.data, self.data, self.data], - axis=self.axis)) + self.out, + np.stack([self.data, self.data, self.data], axis=self.axis), + ) np.testing.assert_array_equal(self.g_x0, np.ones_like(self.data)) np.testing.assert_array_equal(self.g_x1, np.ones_like(self.data)) np.testing.assert_array_equal(self.g_x2, np.ones_like(self.data)) class TestImperativeVarBaseGetItem(unittest.TestCase): - def test_getitem_with_long(self): with fluid.dygraph.guard(): data = np.random.random((2, 80, 16128)).astype('float32') var = fluid.dygraph.to_variable(data) - sliced = var[:, 10:, :var.shape[1]] # var.shape[1] is 80L here + sliced = var[:, 10:, : var.shape[1]] # var.shape[1] is 80L here self.assertEqual(sliced.shape, [2, 70, 80]) - sliced = var[:, var.shape[0]:, var.shape[0]:var.shape[1]] + sliced = var[:, var.shape[0] :, var.shape[0] : var.shape[1]] self.assertEqual(sliced.shape, [2, 78, 78]) def test_getitem_with_float(self): - def test_float_in_slice_item(): with fluid.dygraph.guard(): data = np.random.random((2, 80, 16128)).astype('float32') var = fluid.dygraph.to_variable(data) - sliced = var[:, 1.1:, :var.shape[1]] + sliced = var[:, 1.1:, : var.shape[1]] self.assertRaises(Exception, test_float_in_slice_item) @@ -805,7 +803,6 @@ def test_float_in_index(): class TestInferShape(unittest.TestCase): - def test(self): x = paddle.ones(shape=[3, 4, 5]) x.desc.set_shape([3, -1, 5]) @@ -820,13 +817,18 @@ def test_axis_less_than_zero(self): x_arr = np.arange(0, 24, dtype=np.float32).reshape([2, 3, 4]) x = paddle.to_tensor(x_arr) - pp_slice = paddle.slice(x, [ - 100, - ], [0], [1]) + pp_slice = paddle.slice( + x, + [ + 100, + ], + [0], + [1], + ) np_slice = x_arr[:, :, 0:1] np.testing.assert_array_equal(pp_slice, np_slice) - pp_slice = paddle.slice(x, (-100, ), [0], [1]) + pp_slice = paddle.slice(x, (-100,), [0], [1]) np_slice = x_arr[0:1] np.testing.assert_array_equal(pp_slice, np_slice) @@ -834,9 +836,11 @@ def test_axis_less_than_zero(self): x = paddle.to_tensor(np.reshape(x_arr, (0, 0, 0))) starts = paddle.to_tensor( - np.reshape(np.array([], dtype=np.int32), (0, ))) + np.reshape(np.array([], dtype=np.int32), (0,)) + ) ends = paddle.to_tensor( - np.reshape(np.array([], dtype=np.int32), (0, ))) + np.reshape(np.array([], dtype=np.int32), (0,)) + ) with self.assertRaises(ValueError): paddle.slice(x, [-1000000], starts, ends) @@ -851,30 +855,30 @@ def test_axis_less_than_zero(self): paddle.slice(x, 0, starts, ends) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestImperativeCUDAPinnedInput(unittest.TestCase): - def test_input_cuda_pinned_var(self): _enable_legacy_dygraph() with fluid.dygraph.guard(): data = np.random.random((2, 80, 16128)).astype('float32') - var = core.VarBase(value=data, - name='', - persistable=False, - place=fluid.CUDAPinnedPlace(), - zero_copy=False) - sliced = var[:, 10:, :var.shape[1]] + var = core.VarBase( + value=data, + name='', + persistable=False, + place=fluid.CUDAPinnedPlace(), + zero_copy=False, + ) + sliced = var[:, 10:, : var.shape[1]] self.assertEqual(sliced.shape, [2, 70, 80]) class TestSliceDoubleGradCheck(unittest.TestCase): - def slice_wrapper(self, x): - return paddle.slice(x[0], - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[3, 2, 4]) + return paddle.slice( + x[0], axes=[0, 1, 2], starts=[-3, 0, 2], ends=[3, 2, 4] + ) @prog_scope() def func(self, place): @@ -884,23 +888,18 @@ def func(self, place): data = layers.data('data', [4, 5, 6], False, dtype) data.persistable = True - out = paddle.slice(data, - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[3, 2, 4]) + out = paddle.slice( + data, axes=[0, 1, 2], starts=[-3, 0, 2], ends=[3, 2, 4] + ) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.slice_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.slice_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -912,12 +911,10 @@ def test_grad(self): class TestSliceTripleGradCheck(unittest.TestCase): - def slice_wrapper(self, x): - return paddle.slice(x[0], - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[3, 2, 4]) + return paddle.slice( + x[0], axes=[0, 1, 2], starts=[-3, 0, 2], ends=[3, 2, 4] + ) @prog_scope() def func(self, place): @@ -927,23 +924,18 @@ def func(self, place): data = layers.data('data', [4, 5, 6], False, dtype) data.persistable = True - out = paddle.slice(data, - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[3, 2, 4]) + out = paddle.slice( + data, axes=[0, 1, 2], starts=[-3, 0, 2], ends=[3, 2, 4] + ) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.slice_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.slice_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_slice_var.py b/python/paddle/fluid/tests/unittests/test_slice_var.py index a48b6de556617..b97e5243c3299 100644 --- a/python/paddle/fluid/tests/unittests/test_slice_var.py +++ b/python/paddle/fluid/tests/unittests/test_slice_var.py @@ -19,15 +19,15 @@ class TestSliceVar(unittest.TestCase): - def check_slice_output(self, shapes, expected_sizes, min_size): var_list = [] program = fluid.Program() for shape in shapes: - var = program.global_block().create_var(name=str( - random.randint(10000, 99999)), - persistable=True, - shape=shape) + var = program.global_block().create_var( + name=str(random.randint(10000, 99999)), + persistable=True, + shape=shape, + ) var_list.append(var) blocks = slice_variable(var_list, 10, min_size) all_sizes = [] @@ -40,20 +40,33 @@ def check_slice_output(self, shapes, expected_sizes, min_size): def test_1k(self): shapes = [[3, 5], [1024], [28, 784], [8, 1020], [800, 10]] - expected_sizes = [[15], [1024], - [ - 2352, 2352, 2352, 2352, 2352, 2352, 2352, 2352, - 2352, 784 - ], [2040, 2040, 2040, 2040], - [1150, 1150, 1150, 1150, 1150, 1150, 1100]] + expected_sizes = [ + [15], + [1024], + [2352, 2352, 2352, 2352, 2352, 2352, 2352, 2352, 2352, 784], + [2040, 2040, 2040, 2040], + [1150, 1150, 1150, 1150, 1150, 1150, 1100], + ] self.check_slice_output(shapes, expected_sizes, 1024) def test_check_output_8k(self): - shapes = [[3, 5], [1024], [28, 784], [8, 1020], [800, 10], - [6, 33, 33, 33]] - expected_sizes = [[15], [1024], [10976, 10976], [8160], [8000], - [35937, 35937, 35937, 35937, 35937, 35937]] + shapes = [ + [3, 5], + [1024], + [28, 784], + [8, 1020], + [800, 10], + [6, 33, 33, 33], + ] + expected_sizes = [ + [15], + [1024], + [10976, 10976], + [8160], + [8000], + [35937, 35937, 35937, 35937, 35937, 35937], + ] self.check_slice_output(shapes, expected_sizes, 8192) diff --git a/python/paddle/fluid/tests/unittests/test_smooth_l1_loss.py b/python/paddle/fluid/tests/unittests/test_smooth_l1_loss.py index f9df4160ff619..4e078e506c7d6 100644 --- a/python/paddle/fluid/tests/unittests/test_smooth_l1_loss.py +++ b/python/paddle/fluid/tests/unittests/test_smooth_l1_loss.py @@ -38,7 +38,6 @@ def smooth_l1_loss_np(input, label, reduction='mean', delta=1.0): class SmoothL1Loss(unittest.TestCase): - def setUp(self): np.random.seed(123) @@ -47,8 +46,11 @@ def test_smooth_l1_loss_mean(self): label_np = np.random.random([100, 200]).astype(np.float32) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype='float32') label = fluid.data(name='label', shape=[100, 200], dtype='float32') @@ -56,17 +58,21 @@ def test_smooth_l1_loss_mean(self): ret = smooth_l1_loss(input, label) exe = fluid.Executor(place) - static_ret, = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + (static_ret,) = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): smooth_l1_loss = paddle.nn.loss.SmoothL1Loss() - dy_ret = smooth_l1_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = smooth_l1_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = smooth_l1_loss_np(input_np, label_np, reduction='mean') @@ -79,8 +85,11 @@ def test_smooth_l1_loss_sum(self): label_np = np.random.random([100, 200]).astype(np.float32) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype='float32') label = fluid.data(name='label', shape=[100, 200], dtype='float32') @@ -88,17 +97,21 @@ def test_smooth_l1_loss_sum(self): ret = smooth_l1_loss(input, label) exe = fluid.Executor(place) - static_ret, = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + (static_ret,) = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): smooth_l1_loss = paddle.nn.loss.SmoothL1Loss(reduction='sum') - dy_ret = smooth_l1_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = smooth_l1_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = smooth_l1_loss_np(input_np, label_np, reduction='sum') @@ -111,8 +124,11 @@ def test_smooth_l1_loss_none(self): label_np = np.random.random([100, 200]).astype(np.float32) prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype='float32') label = fluid.data(name='label', shape=[100, 200], dtype='float32') @@ -120,17 +136,21 @@ def test_smooth_l1_loss_none(self): ret = smooth_l1_loss(input, label) exe = fluid.Executor(place) - static_ret, = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + (static_ret,) = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): smooth_l1_loss = paddle.nn.loss.SmoothL1Loss(reduction='none') - dy_ret = smooth_l1_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = smooth_l1_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = smooth_l1_loss_np(input_np, label_np, reduction='none') @@ -144,8 +164,11 @@ def test_smooth_l1_loss_delta(self): delta = np.random.rand() prog = fluid.Program() startup_prog = fluid.Program() - place = fluid.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) with fluid.program_guard(prog, startup_prog): input = fluid.data(name='input', shape=[100, 200], dtype='float32') label = fluid.data(name='label', shape=[100, 200], dtype='float32') @@ -153,17 +176,21 @@ def test_smooth_l1_loss_delta(self): ret = smooth_l1_loss(input, label) exe = fluid.Executor(place) - static_ret, = exe.run(prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret]) + (static_ret,) = exe.run( + prog, + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) self.assertIsNotNone(static_ret) with fluid.dygraph.guard(): smooth_l1_loss = paddle.nn.loss.SmoothL1Loss(delta=delta) - dy_ret = smooth_l1_loss(fluid.dygraph.to_variable(input_np), - fluid.dygraph.to_variable(label_np)) + dy_ret = smooth_l1_loss( + fluid.dygraph.to_variable(input_np), + fluid.dygraph.to_variable(label_np), + ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) expected = smooth_l1_loss_np(input_np, label_np, delta=delta) diff --git a/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py b/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py index ffabe17915a26..4b31e7ead6b94 100644 --- a/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py @@ -27,13 +27,12 @@ def smooth_l1_loss_forward(val, sigma2): class TestSmoothL1LossOp1(OpTest): - def setUp(self): self.op_type = "smooth_l1_loss" dims = (5, 20) self.inputs = { 'X': np.random.random(dims).astype("float32"), - 'Y': np.random.random(dims).astype("float32") + 'Y': np.random.random(dims).astype("float32"), } sigma = 3.0 self.attrs = {'sigma': sigma} @@ -43,35 +42,37 @@ def setUp(self): loss = loss.reshape((dims[0], 1)) self.outputs = { 'Diff': diff.astype('float32'), - 'Out': loss.astype('float32') + 'Out': loss.astype('float32'), } def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=0.02, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=0.02, check_eager=True + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.03, - no_grad_set=set("X"), - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + max_relative_error=0.03, + no_grad_set=set("X"), + check_eager=True, + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.03, - no_grad_set=set('Y'), - check_eager=True) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.03, + no_grad_set=set('Y'), + check_eager=True, + ) class TestSmoothL1LossOp2(OpTest): - def setUp(self): self.op_type = "smooth_l1_loss" dims = (5, 20) @@ -79,7 +80,7 @@ def setUp(self): 'X': np.random.random(dims).astype("float32"), 'Y': np.random.random(dims).astype("float32"), 'InsideWeight': np.random.random(dims).astype("float32"), - 'OutsideWeight': np.random.random(dims).astype("float32") + 'OutsideWeight': np.random.random(dims).astype("float32"), } sigma = 3.0 self.attrs = {'sigma': sigma} @@ -91,42 +92,46 @@ def setUp(self): loss = loss.sum(1).reshape((dims[0], 1)) self.outputs = { 'Diff': diff.astype('float32'), - 'Out': loss.astype('float32') + 'Out': loss.astype('float32'), } def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=0.03, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=0.03, check_eager=True + ) def test_check_grad_ingore_x(self): - self.check_grad(['Y'], - 'Out', - max_relative_error=0.03, - no_grad_set=set(['X', 'InsideWeight', 'OutsideWeight']), - check_eager=True) + self.check_grad( + ['Y'], + 'Out', + max_relative_error=0.03, + no_grad_set=set(['X', 'InsideWeight', 'OutsideWeight']), + check_eager=True, + ) def test_check_grad_ingore_y(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.03, - no_grad_set=set(['Y', 'InsideWeight', 'OutsideWeight']), - check_eager=True) + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.03, + no_grad_set=set(['Y', 'InsideWeight', 'OutsideWeight']), + check_eager=True, + ) class TestSmoothL1LossOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program(), fluid.Program()): # The input type of accuracy_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, fluid.layers.smooth_l1, x1, y1) # The input dtype of accuracy_op must be float32 or float64. x2 = fluid.layers.data(name='x2', shape=[4], dtype="int32") diff --git a/python/paddle/fluid/tests/unittests/test_soft_margin_loss.py b/python/paddle/fluid/tests/unittests/test_soft_margin_loss.py index b69a2a5c18740..98918fb4b0bab 100644 --- a/python/paddle/fluid/tests/unittests/test_soft_margin_loss.py +++ b/python/paddle/fluid/tests/unittests/test_soft_margin_loss.py @@ -27,21 +27,18 @@ def test_static_layer( prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.static.data(name='input', - shape=input_np.shape, - dtype=input_np.dtype) - label = paddle.static.data(name='label', - shape=label_np.shape, - dtype=label_np.dtype) + input = paddle.static.data( + name='input', shape=input_np.shape, dtype=input_np.dtype + ) + label = paddle.static.data( + name='label', shape=label_np.shape, dtype=label_np.dtype + ) sm_loss = paddle.nn.loss.SoftMarginLoss(reduction=reduction) res = sm_loss(input, label) exe = paddle.static.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, feed={"input": input_np, "label": label_np}, fetch_list=[res] + ) return static_result @@ -55,23 +52,20 @@ def test_static_functional( prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.static.data(name='input', - shape=input_np.shape, - dtype=input_np.dtype) - label = paddle.static.data(name='label', - shape=label_np.shape, - dtype=label_np.dtype) - - res = paddle.nn.functional.soft_margin_loss(input, - label, - reduction=reduction) + input = paddle.static.data( + name='input', shape=input_np.shape, dtype=input_np.dtype + ) + label = paddle.static.data( + name='label', shape=label_np.shape, dtype=label_np.dtype + ) + + res = paddle.nn.functional.soft_margin_loss( + input, label, reduction=reduction + ) exe = paddle.static.Executor(place) - static_result, = exe.run(prog, - feed={ - "input": input_np, - "label": label_np - }, - fetch_list=[res]) + (static_result,) = exe.run( + prog, feed={"input": input_np, "label": label_np}, fetch_list=[res] + ) return static_result @@ -99,9 +93,9 @@ def test_dygraph_functional( input = paddle.to_tensor(input_np) label = paddle.to_tensor(label_np) - dy_res = paddle.nn.functional.soft_margin_loss(input, - label, - reduction=reduction) + dy_res = paddle.nn.functional.soft_margin_loss( + input, label, reduction=reduction + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result @@ -126,7 +120,6 @@ def calc_softmarginloss( class TestSoftMarginLoss(unittest.TestCase): - def test_SoftMarginLoss(self): input_np = np.random.uniform(0.1, 0.8, size=(5, 5)).astype(np.float64) types = [np.int32, np.int64, np.float32, np.float64] @@ -137,48 +130,58 @@ def test_SoftMarginLoss(self): for place in places: for reduction in reductions: for _type in types: - label_np = np.random.randint(0, 2, - size=(5, 5)).astype(_type) + label_np = np.random.randint(0, 2, size=(5, 5)).astype( + _type + ) label_np[label_np == 0] = -1 - static_result = test_static_layer(place, input_np, label_np, - reduction) - dy_result = test_dygraph_layer(place, input_np, label_np, - reduction) - expected = calc_softmarginloss(input_np, label_np, - reduction) - np.testing.assert_allclose(static_result, - expected, - rtol=1e-05) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-05) + static_result = test_static_layer( + place, input_np, label_np, reduction + ) + dy_result = test_dygraph_layer( + place, input_np, label_np, reduction + ) + expected = calc_softmarginloss( + input_np, label_np, reduction + ) + np.testing.assert_allclose( + static_result, expected, rtol=1e-05 + ) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-05 + ) np.testing.assert_allclose(dy_result, expected, rtol=1e-05) static_functional = test_static_functional( - place, input_np, label_np, reduction) + place, input_np, label_np, reduction + ) dy_functional = test_dygraph_functional( - place, input_np, label_np, reduction) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-05) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-05) - np.testing.assert_allclose(dy_functional, - expected, - rtol=1e-05) + place, input_np, label_np, reduction + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-05 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-05 + ) + np.testing.assert_allclose( + dy_functional, expected, rtol=1e-05 + ) def test_SoftMarginLoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.loss.SoftMarginLoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.loss.SoftMarginLoss, + reduction="unsupport reduction", + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') label = paddle.to_tensor([[-1.0, 1.0]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.soft_margin_loss, - input=input, - label=label, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.soft_margin_loss, + input=input, + label=label, + reduction="unsupport reduction", + ) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_softmax2d.py b/python/paddle/fluid/tests/unittests/test_softmax2d.py index c5e45dec22801..8297ca9c90814 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax2d.py +++ b/python/paddle/fluid/tests/unittests/test_softmax2d.py @@ -20,13 +20,15 @@ class TestSoftmax2DAPI(unittest.TestCase): - def setUp(self): self.shape = [2, 6, 5, 4] self.x_np = np.random.uniform(-1, 1, self.shape).astype('float64') self.axis = -3 - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -35,7 +37,7 @@ def test_static_api(self): m = paddle.nn.Softmax2D() out = m(x) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={'X': self.x_np}, fetch_list=[out]) + (res,) = exe.run(feed={'X': self.x_np}, fetch_list=[out]) out_ref = ref_softmax(self.x_np, self.axis) np.testing.assert_allclose(out_ref, res, rtol=1e-05) @@ -50,27 +52,30 @@ def test_dygraph_api(self): class TestSoftmax2DShape(TestSoftmax2DAPI): - def setUp(self): self.shape = [2, 6, 4] self.x_np = np.random.uniform(-1, 1, self.shape).astype('float64') self.axis = -3 - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) class TestSoftmax2DFloat32(TestSoftmax2DAPI): - def setUp(self): self.shape = [2, 3, 4] self.x_np = np.random.uniform(-1, 1, self.shape).astype('float32') self.axis = -3 - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) class TestSoftmax2DCPU(TestSoftmax2DAPI): - def setUp(self): self.shape = [2, 6, 4] self.x_np = np.random.uniform(-1, 1, self.shape).astype('float64') @@ -79,10 +84,12 @@ def setUp(self): class TestSoftmax2DRepr(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_extra_repr(self): paddle.disable_static(self.place) @@ -92,10 +99,12 @@ def test_extra_repr(self): class TestSoftmax2DError(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_error(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_op.py b/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_op.py index 2bdede66f4ab5..c36b8362c84bc 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_op.py +++ b/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_op.py @@ -35,10 +35,10 @@ def _get_softmax(x, mask, fp16=True): return rst -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxMaskFuseOp(OpTest): - def setUp(self): self.op_type = "fused_softmax_mask" x = np.random.random((1, 1, 8, 32)) @@ -61,10 +61,10 @@ def test_check_grad(self): pass -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxMaskFuseOp0(OpTest): - def setUp(self): self.op_type = "fused_softmax_mask" x = np.random.random((1, 1, 8, 32)).astype("float16") @@ -81,16 +81,16 @@ def test_check_grad(self): self.check_grad_with_place(core.CUDAPlace(0), ["X"], "Out") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestDropoutBiasFuseOp3(unittest.TestCase): - def test_static_result(self): with fluid.program_guard(fluid.Program(), fluid.Program()): input_x = fluid.data(name="x", shape=[1, 1, 8, 32], dtype="float32") - input_mask = fluid.data(name="mask", - shape=[1, 1, 8, 32], - dtype="float32") + input_mask = fluid.data( + name="mask", shape=[1, 1, 8, 32], dtype="float32" + ) rst = incubate.softmax_mask_fuse(input_x, input_mask) x_in_np = np.random.random((1, 1, 8, 32)).astype("float32") @@ -99,12 +99,11 @@ def test_static_result(self): rst_np = _get_softmax(x_in_np, mask_in_np, False) exe = fluid.Executor(fluid.CUDAPlace(0)) - fetches = exe.run(fluid.default_main_program(), - feed={ - "x": x_in_np, - "mask": mask_in_np - }, - fetch_list=[rst]) + fetches = exe.run( + fluid.default_main_program(), + feed={"x": x_in_np, "mask": mask_in_np}, + fetch_list=[rst], + ) np.testing.assert_allclose(fetches[0], rst_np, rtol=1e-05) def test_dygraph(self): diff --git a/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_upper_triangle_op.py b/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_upper_triangle_op.py index 62a2e729051e5..84032336402c4 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_upper_triangle_op.py +++ b/python/paddle/fluid/tests/unittests/test_softmax_mask_fuse_upper_triangle_op.py @@ -36,10 +36,10 @@ def _get_softmax_upper(x, fp16=True): return rst -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxMaskFuseOp(OpTest): - def setUp(self): self.op_type = "fused_softmax_mask_upper_triangle" x = np.random.random((1, 4, 32, 32)).astype("float16") @@ -54,10 +54,10 @@ def test_check_grad(self): self.check_grad_with_place(core.CUDAPlace(0), ["X"], "Out") -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxMaskFuseOp1(OpTest): - def setUp(self): self.op_type = "fused_softmax_mask_upper_triangle" x = np.random.random((1, 4, 32, 32)) @@ -78,8 +78,9 @@ def test_check_grad(self): pass -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestDropoutBiasFuseOp2(unittest.TestCase): # test the python side API for softmax_mask_fuse op def setUp(self): @@ -89,18 +90,20 @@ def setUp(self): def test_static(self): for dtype in self.dtypes: with fluid.program_guard(fluid.Program(), fluid.Program()): - input_x = fluid.data(name="x", - shape=[1, 4, 32, 32], - dtype=dtype) + input_x = fluid.data( + name="x", shape=[1, 4, 32, 32], dtype=dtype + ) rst = incubate.softmax_mask_fuse_upper_triangle(input_x) x_in_np = np.random.random((1, 4, 32, 32)).astype(dtype) rst_np = _get_softmax_upper(x_in_np, dtype == 'float16') exe = fluid.Executor(fluid.CUDAPlace(0)) - fetches = exe.run(fluid.default_main_program(), - feed={"x": x_in_np}, - fetch_list=[rst]) + fetches = exe.run( + fluid.default_main_program(), + feed={"x": x_in_np}, + fetch_list=[rst], + ) np.testing.assert_allclose(fetches[0], rst_np, rtol=1e-05) def test_dygraph(self): diff --git a/python/paddle/fluid/tests/unittests/test_softmax_op.py b/python/paddle/fluid/tests/unittests/test_softmax_op.py index 13a43e4d3e900..75324475f46de 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/test_softmax_op.py @@ -26,7 +26,7 @@ def stable_softmax(x): """Compute the softmax of vector x in a numerically stable way.""" # clip to shiftx, otherwise, when calc loss with # log(exp(shiftx)), may get log(0)=INF - shiftx = (x - np.max(x)).clip(-64.) + shiftx = (x - np.max(x)).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) @@ -41,7 +41,6 @@ def ref_softmax(x, axis=None, dtype=None): class TestSoftmaxOp(OpTest): - def get_x_shape(self): return [10, 10] @@ -67,7 +66,7 @@ def setUp(self): self.attrs = { 'axis': self.axis, 'use_cudnn': self.use_cudnn, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } def init_kernel_type(self): @@ -78,7 +77,8 @@ def test_check_output(self): if self.use_cudnn: place = core.CUDAPlace(0) self.check_output_with_place( - place, atol=1e-5, check_dygraph=(self.use_mkldnn == False)) + place, atol=1e-5, check_dygraph=(self.use_mkldnn == False) + ) else: self.check_output(check_dygraph=(self.use_mkldnn == False)) @@ -88,25 +88,27 @@ def test_check_grad(self): place = core.CUDAPlace(0) if core.is_float16_supported(place): self.check_grad_with_place( - place, ["X"], + place, + ["X"], "Out", max_relative_error=0.01, - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) else: - self.check_grad(["X"], - "Out", - max_relative_error=0.01, - check_dygraph=(self.use_mkldnn == False)) + self.check_grad( + ["X"], + "Out", + max_relative_error=0.01, + check_dygraph=(self.use_mkldnn == False), + ) class TestSoftmaxOp2(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] class TestSoftmaxOp3(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -115,7 +117,6 @@ def get_axis(self): class TestSoftmaxOp4(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -124,7 +125,6 @@ def get_axis(self): class TestSoftmaxOp5(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -133,7 +133,6 @@ def get_axis(self): class TestSoftmaxOp6(TestSoftmaxOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -141,26 +140,26 @@ def get_axis(self): return 3 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp(TestSoftmaxOp): - def init_kernel_type(self): self.use_cudnn = True -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp2(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp3(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -168,10 +167,10 @@ def get_axis(self): return 0 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp4(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -179,10 +178,10 @@ def get_axis(self): return 1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp5(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -190,10 +189,10 @@ def get_axis(self): return 2 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp6(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5] @@ -201,18 +200,18 @@ def get_axis(self): return 3 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp7(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5, 6] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp8(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5, 6] @@ -220,10 +219,10 @@ def get_axis(self): return 0 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp9(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5, 6] @@ -231,10 +230,10 @@ def get_axis(self): return 1 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp10(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5, 6] @@ -242,10 +241,10 @@ def get_axis(self): return 2 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp11(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5, 6] @@ -253,10 +252,10 @@ def get_axis(self): return 3 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxCUDNNOp12(TestSoftmaxCUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5, 6] @@ -264,10 +263,10 @@ def get_axis(self): return 4 -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxFP16Op(TestSoftmaxOp): - def init_kernel_type(self): self.dtype = np.float16 @@ -282,18 +281,18 @@ def test_check_grad(self): pass -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxFP16Op2(TestSoftmaxFP16Op): - def get_x_shape(self): return [2, 3, 4, 10] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxFP16CUDNNOp(TestSoftmaxOp): - def init_kernel_type(self): self.use_cudnn = True self.dtype = np.float16 @@ -305,18 +304,18 @@ def test_check_output(self): self.check_output_with_place(place, atol=1e-3) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxFP16CUDNNOp2(TestSoftmaxFP16CUDNNOp): - def get_x_shape(self): return [2, 3, 4, 5] -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxBF16Op(OpTest): - def setUp(self): self.op_type = "softmax" self.use_cudnn = self.init_cudnn() @@ -336,7 +335,7 @@ def setUp(self): self.attrs = { 'axis': self.axis, 'use_cudnn': self.use_cudnn, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } def init_cudnn(self): @@ -344,34 +343,40 @@ def init_cudnn(self): def test_check_output(self): place = core.CUDAPlace(0) - self.check_output_with_place(place, - check_dygraph=(self.use_mkldnn == False)) + self.check_output_with_place( + place, check_dygraph=(self.use_mkldnn == False) + ) def test_check_grad(self): place = core.CUDAPlace(0) - self.check_grad_with_place(place, ["X"], - "Out", - numeric_grad_delta=0.05, - check_dygraph=(self.use_mkldnn == False)) + self.check_grad_with_place( + place, + ["X"], + "Out", + numeric_grad_delta=0.05, + check_dygraph=(self.use_mkldnn == False), + ) @unittest.skipIf( - not core.is_compiled_with_cuda() or core.cudnn_version() < 8100 + not core.is_compiled_with_cuda() + or core.cudnn_version() < 8100 or paddle.device.cuda.get_device_capability()[0] < 8, - "only support compiled with CUDA and cudnn version need larger than 8.1.0 and device's compute capability is at least 8.0" + "only support compiled with CUDA and cudnn version need larger than 8.1.0 and device's compute capability is at least 8.0", ) class TestSoftmaxBF16CUDNNOp(TestSoftmaxBF16Op): - def init_cudnn(self): return True class TestSoftmaxAPI(unittest.TestCase): - def setUp(self): - self.place = paddle.CUDAPlace( - 0) if core.is_compiled_with_cuda() else paddle.CPUPlace() - self.x_np = np.random.uniform(-1., 1., [2, 3, 4, 5]).astype('float32') + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) + self.x_np = np.random.uniform(-1.0, 1.0, [2, 3, 4, 5]).astype('float32') self.out_ref = np.apply_along_axis(stable_softmax, -1, self.x_np) self.executed_api() @@ -426,19 +431,18 @@ def test_error(self): # The input type must be Variable. self.assertRaises(TypeError, self.softmax, 1) # The input dtype must be float16, float32, float64. - x_int32 = paddle.fluid.data(name='x_int32', - shape=[2, 3], - dtype='int32') + x_int32 = paddle.fluid.data( + name='x_int32', shape=[2, 3], dtype='int32' + ) self.assertRaises(TypeError, self.softmax, x_int32) # support the input dtype is float16 - x_fp16 = paddle.fluid.data(name='x_fp16', - shape=[2, 3], - dtype='float16') + x_fp16 = paddle.fluid.data( + name='x_fp16', shape=[2, 3], dtype='float16' + ) self.softmax(x_fp16) class TestSoftmaxInplaceAPI(TestSoftmaxAPI): - def executed_api(self): self.softmax = F.softmax_ diff --git a/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py b/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py index 6b4a733853b4c..fb0e46b6740b8 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py +++ b/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py @@ -28,7 +28,7 @@ def cross_entropy(softmax, label, soft_label, axis, ignore_index=-1): axis %= len(shape) n = int(np.prod(shape[:axis])) axis_dim = shape[axis] - remain = int(np.prod(shape[axis + 1:])) + remain = int(np.prod(shape[axis + 1 :])) softmax_reshape = softmax.reshape((n, axis_dim, remain)) label_reshape = label.reshape((n, 1, remain)) result = np.zeros_like(label_reshape, dtype=softmax.dtype) @@ -40,13 +40,15 @@ def cross_entropy(softmax, label, soft_label, axis, ignore_index=-1): return result.reshape(label.shape) -def python_api(logits, - label, - soft_label=False, - use_softmax=True, - numeric_stable_mode=True, - ignore_index=-100, - axis=-1): +def python_api( + logits, + label, + soft_label=False, + use_softmax=True, + numeric_stable_mode=True, + ignore_index=-100, + axis=-1, +): # here only can test paddle.nn.functional.softmax_with_cross_entropy, # the paddle.nn.functional.cross_entropy contains other math ops return paddle.nn.functional.softmax_with_cross_entropy( @@ -56,23 +58,31 @@ def python_api(logits, ignore_index=ignore_index, numeric_stable_mode=numeric_stable_mode, return_softmax=use_softmax, - axis=axis) - - -def python_core_api_without_softmax(logits, - label, - soft_label=False, - use_softmax=False, - numeric_stable_mode=True, - ignore_index=-100, - axis=-1): + axis=axis, + ) + + +def python_core_api_without_softmax( + logits, + label, + soft_label=False, + use_softmax=False, + numeric_stable_mode=True, + ignore_index=-100, + axis=-1, +): # the API paddle.nn.functional.softmax_with_cross_entropy cannot # set use_softmax=False, so add a core api manually assert use_softmax is False - _, loss = paddle._C_ops.cross_entropy_with_softmax(logits, label, - soft_label, use_softmax, - numeric_stable_mode, - ignore_index, axis) + _, loss = paddle._C_ops.cross_entropy_with_softmax( + logits, + label, + soft_label, + use_softmax, + numeric_stable_mode, + ignore_index, + axis, + ) return loss @@ -101,8 +111,10 @@ def setUp(self): self.initParams() logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, logits) if self.soft_label: @@ -111,13 +123,13 @@ def setUp(self): else: axis_dim = self.shape[self.axis] self.shape[self.axis] = 1 - labels = np.random.randint(0, - axis_dim, - self.shape, - dtype=self.hard_label_dtype()) + labels = np.random.randint( + 0, axis_dim, self.shape, dtype=self.hard_label_dtype() + ) - loss = cross_entropy(softmax, labels, self.soft_label, self.axis, - self.ignore_index) + loss = cross_entropy( + softmax, labels, self.soft_label, self.axis, self.ignore_index + ) if self.use_softmax == False: self.inputs = {"Logits": softmax, "Label": labels} @@ -126,7 +138,7 @@ def setUp(self): self.outputs = { "Softmax": softmax.astype(self.dtype), - "Loss": loss.astype(self.dtype) + "Loss": loss.astype(self.dtype), } self.attrs = { "numeric_stable_mode": self.numeric_stable_mode, @@ -146,48 +158,48 @@ def test_check_output(self): def test_check_grad(self): if core.is_compiled_with_rocm(): if self.python_api is not None: - self.check_grad(["Logits"], - "Loss", - max_relative_error=5e-1, - check_eager=True) + self.check_grad( + ["Logits"], + "Loss", + max_relative_error=5e-1, + check_eager=True, + ) # HIP will have accuracy fail when using float32 in CPU place self.check_grad(["Logits"], "Loss", max_relative_error=5e-1) else: if self.python_api is not None: - self.check_grad(["Logits"], - "Loss", - numeric_grad_delta=0.001, - check_eager=True) + self.check_grad( + ["Logits"], + "Loss", + numeric_grad_delta=0.001, + check_eager=True, + ) self.check_grad(["Logits"], "Loss", numeric_grad_delta=0.001) class TestSoftmaxWithCrossEntropyOpInt32(TestSoftmaxWithCrossEntropyOp): - def hard_label_dtype(self): return "int32" class TestSoftmaxWithCrossEntropyOpInt16(TestSoftmaxWithCrossEntropyOp): - def hard_label_dtype(self): return "int16" class TestSoftmaxWithCrossEntropyOpInt8(TestSoftmaxWithCrossEntropyOp): - def hard_label_dtype(self): return "int8" class TestSoftmaxWithCrossEntropyOpUInt8(TestSoftmaxWithCrossEntropyOp): - def hard_label_dtype(self): return "uint8" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_1D( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -198,12 +210,12 @@ def initParams(self): self.axis = -1 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_1D( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -214,15 +226,15 @@ def initParams(self): self.axis = -1 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" ############################################################################## -#NotWithSoftmax_SoftLabel_2D start +# NotWithSoftmax_SoftLabel_2D start ############################################################################## class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_2D( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -233,12 +245,12 @@ def initParams(self): self.axis = -1 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_2D_Axis2( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -249,12 +261,12 @@ def initParams(self): self.axis = 1 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_2D_Axis3( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -265,12 +277,12 @@ def initParams(self): self.axis = 2 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_SoftLabel_2D_Axis4( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -281,21 +293,21 @@ def initParams(self): self.axis = 3 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" ############################################################################## -#NotWithSoftmax_SoftLabel_2D end +# NotWithSoftmax_SoftLabel_2D end ############################################################################## ############################################################################## -#NotWithSoftmax_HardLabel_2D start +# NotWithSoftmax_HardLabel_2D start ############################################################################## class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -306,12 +318,12 @@ def initParams(self): self.axis = -1 self.ignore_index = -1 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Axis2( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -322,12 +334,12 @@ def initParams(self): self.axis = 1 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Axis3( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -338,12 +350,12 @@ def initParams(self): self.axis = 2 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Axis4( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -354,21 +366,21 @@ def initParams(self): self.axis = 3 self.ignore_index = -1 self.shape = [3, 5, 7, 11] - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" ############################################################################## -#NotWithSoftmax_HardLabel_2D end +# NotWithSoftmax_HardLabel_2D end ############################################################################## ############################################################################## -#NotWithSoftmax_HardLabel_2D_Ignore start +# NotWithSoftmax_HardLabel_2D_Ignore start ############################################################################## class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_Ignore( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -379,12 +391,12 @@ def initParams(self): self.axis = -1 self.ignore_index = 2 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_Ignore_Axis( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -395,12 +407,12 @@ def initParams(self): self.axis = 1 self.ignore_index = 2 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Ignore( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -411,12 +423,12 @@ def initParams(self): self.axis = -1 self.ignore_index = 2 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" class TestSoftmaxWithCrossEntropyOp_NotWithSoftmax_HardLabel_2D_Ignore_Axis3( - TestSoftmaxWithCrossEntropyOp): - + TestSoftmaxWithCrossEntropyOp +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_core_api_without_softmax @@ -427,16 +439,15 @@ def initParams(self): self.axis = 2 self.ignore_index = 2 self.shape = [3, 5, 7, 11] - self.use_softmax = False #default is true, means "with softmax" + self.use_softmax = False # default is true, means "with softmax" ############################################################################## -#NotWithSoftmax_HardLabel_2D_Ignore end +# NotWithSoftmax_HardLabel_2D_Ignore end ############################################################################## class TestSoftmaxWithCrossEntropyOpNoCudnn(TestSoftmaxWithCrossEntropyOp): - def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -450,10 +461,10 @@ def initParams(self): self.use_softmax = True -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSoftmaxWithCrossEntropyOpFp16(TestSoftmaxWithCrossEntropyOp): - def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -472,8 +483,10 @@ def setUp(self): # NOTE: numpy float16 have very low accuracy, use float32 for numpy check. date_type = np.float32 if core.is_compiled_with_rocm() else np.float64 logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(date_type)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(date_type), + ) softmax = np.apply_along_axis(stable_softmax, self.axis, logits) axis_dim = self.shape[self.axis] @@ -485,7 +498,7 @@ def setUp(self): self.inputs = {"Logits": logits.astype(self.dtype), "Label": labels} self.outputs = { "Softmax": softmax.astype(self.dtype), - "Loss": loss.astype(self.dtype) + "Loss": loss.astype(self.dtype), } self.attrs = { "numeric_stable_mode": self.numeric_stable_mode, @@ -501,16 +514,15 @@ def test_check_output(self): def test_check_grad(self): if self.python_api is not None: - self.check_grad(["Logits"], - "Loss", - max_relative_error=0.1, - check_eager=True) + self.check_grad( + ["Logits"], "Loss", max_relative_error=0.1, check_eager=True + ) self.check_grad(["Logits"], "Loss", max_relative_error=0.1) -class TestSoftmaxWithCrossEntropyOpNoCudnnFp16(TestSoftmaxWithCrossEntropyOpFp16 - ): - +class TestSoftmaxWithCrossEntropyOpNoCudnnFp16( + TestSoftmaxWithCrossEntropyOpFp16 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -524,10 +536,9 @@ def initParams(self): def test_check_grad(self): if self.python_api is not None: - self.check_grad(["Logits"], - "Loss", - max_relative_error=0.1, - check_eager=True) + self.check_grad( + ["Logits"], "Loss", max_relative_error=0.1, check_eager=True + ) self.check_grad(["Logits"], "Loss", max_relative_error=0.1) @@ -557,10 +568,9 @@ def test_check_grad(self): if core.is_compiled_with_rocm(): # HIP will have accuracy fail when using float32 in CPU place if self.python_api is not None: - self.check_grad(["Logits"], - "Loss", - max_relative_error=0.1, - check_eager=True) + self.check_grad( + ["Logits"], "Loss", max_relative_error=0.1, check_eager=True + ) self.check_grad(["Logits"], "Loss", max_relative_error=0.1) else: if self.python_api is not None: @@ -587,7 +597,6 @@ def initParams(self): class TestSoftmaxWithCrossEntropyOp3NoCudnn(TestSoftmaxWithCrossEntropyOp3): - def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -677,8 +686,9 @@ def initParams(self): self.use_softmax = True -class TestSoftmaxWithCrossEntropyOpAxisDimEqualOne(TestSoftmaxWithCrossEntropyOp - ): +class TestSoftmaxWithCrossEntropyOpAxisDimEqualOne( + TestSoftmaxWithCrossEntropyOp +): """ Test softmax with cross entropy operator with discreate one-hot labels. Given axis != -1 @@ -698,8 +708,8 @@ def initParams(self): class TestSoftmaxWithCrossEntropyOpNoCudnnFp16Axis1( - TestSoftmaxWithCrossEntropyOpNoCudnnFp16): - + TestSoftmaxWithCrossEntropyOpNoCudnnFp16 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -714,8 +724,8 @@ def initParams(self): class TestSoftmaxWithCrossEntropyOpNoCudnnFp16Axis2( - TestSoftmaxWithCrossEntropyOpNoCudnnFp16): - + TestSoftmaxWithCrossEntropyOpNoCudnnFp16 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -730,8 +740,8 @@ def initParams(self): class TestSoftmaxWithCrossEntropyOpNoCudnnFp16Axis3( - TestSoftmaxWithCrossEntropyOpNoCudnnFp16): - + TestSoftmaxWithCrossEntropyOpNoCudnnFp16 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -745,9 +755,9 @@ def initParams(self): self.use_softmax = True -class TestSoftmaxWithCrossEntropyOpSoftLabelAxis1(TestSoftmaxWithCrossEntropyOp2 - ): - +class TestSoftmaxWithCrossEntropyOpSoftLabelAxis1( + TestSoftmaxWithCrossEntropyOp2 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -761,9 +771,9 @@ def initParams(self): self.use_softmax = True -class TestSoftmaxWithCrossEntropyOpSoftLabelAxis2(TestSoftmaxWithCrossEntropyOp2 - ): - +class TestSoftmaxWithCrossEntropyOpSoftLabelAxis2( + TestSoftmaxWithCrossEntropyOp2 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -777,9 +787,9 @@ def initParams(self): self.use_softmax = True -class TestSoftmaxWithCrossEntropyOpSoftLabelAxis3(TestSoftmaxWithCrossEntropyOp2 - ): - +class TestSoftmaxWithCrossEntropyOpSoftLabelAxis3( + TestSoftmaxWithCrossEntropyOp2 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -793,9 +803,9 @@ def initParams(self): self.use_softmax = True -class TestSoftmaxWithCrossEntropyOpSoftLabelAxis4(TestSoftmaxWithCrossEntropyOp2 - ): - +class TestSoftmaxWithCrossEntropyOpSoftLabelAxis4( + TestSoftmaxWithCrossEntropyOp2 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -810,8 +820,8 @@ def initParams(self): class TestSoftmaxWithCrossEntropyOpIgnoreIndexNoCudnnAxis1( - TestSoftmaxWithCrossEntropyOp3): - + TestSoftmaxWithCrossEntropyOp3 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -826,8 +836,8 @@ def initParams(self): class TestSoftmaxWithCrossEntropyOpIgnoreIndexNoCudnnAxis2( - TestSoftmaxWithCrossEntropyOp3): - + TestSoftmaxWithCrossEntropyOp3 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -842,8 +852,8 @@ def initParams(self): class TestSoftmaxWithCrossEntropyOpIgnoreIndexNoCudnnAxis3( - TestSoftmaxWithCrossEntropyOp3): - + TestSoftmaxWithCrossEntropyOp3 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api @@ -858,8 +868,8 @@ def initParams(self): class TestSoftmaxWithCrossEntropyOpIgnoreIndexNoCudnnAxis4( - TestSoftmaxWithCrossEntropyOp3): - + TestSoftmaxWithCrossEntropyOp3 +): def initParams(self): self.op_type = "softmax_with_cross_entropy" self.python_api = python_api diff --git a/python/paddle/fluid/tests/unittests/test_solve_op.py b/python/paddle/fluid/tests/unittests/test_solve_op.py index b07c3d5d080e9..4fcea5e67d590 100644 --- a/python/paddle/fluid/tests/unittests/test_solve_op.py +++ b/python/paddle/fluid/tests/unittests/test_solve_op.py @@ -27,7 +27,6 @@ # 2D normal case class TestSolveOp(OpTest): - def config(self): self.python_api = paddle.linalg.solve self.input_x_matrix_shape = [15, 15] @@ -42,7 +41,7 @@ def setUp(self): np.random.seed(2021) self.inputs = { 'X': np.random.random(self.input_x_matrix_shape).astype(self.dtype), - 'Y': np.random.random(self.input_y_matrix_shape).astype(self.dtype) + 'Y': np.random.random(self.input_y_matrix_shape).astype(self.dtype), } self.outputs = { 'Out': np.linalg.solve(self.inputs['X'], self.inputs['Y']) @@ -57,7 +56,6 @@ def test_check_grad_normal(self): # x broadcast + 3D batch case class TestSolveOpBatched_case0(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -65,7 +63,7 @@ def setUp(self): np.random.seed(2021) self.inputs = { 'X': np.random.random((11, 11)).astype(self.dtype), - 'Y': np.random.random((2, 11, 7)).astype(self.dtype) + 'Y': np.random.random((2, 11, 7)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -74,15 +72,13 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=1e-1, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=1e-1, check_eager=True + ) # 3D batch + y vector case class TestSolveOpBatched_case1(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -90,7 +86,7 @@ def setUp(self): np.random.seed(2021) self.inputs = { 'X': np.random.random((20, 6, 6)).astype(self.dtype), - 'Y': np.random.random((20, 6)).astype(self.dtype) + 'Y': np.random.random((20, 6)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -99,15 +95,13 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=0.04, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=0.04, check_eager=True + ) # 3D batch + y broadcast case class TestSolveOpBatched_case2(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -115,7 +109,7 @@ def setUp(self): np.random.seed(2021) self.inputs = { 'X': np.random.random((2, 10, 10)).astype(self.dtype), - 'Y': np.random.random((1, 10, 10)).astype(self.dtype) + 'Y': np.random.random((1, 10, 10)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -124,15 +118,13 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=0.02, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=0.02, check_eager=True + ) # x broadcast + 3D batch case class TestSolveOpBatched_case3(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -140,7 +132,7 @@ def setUp(self): np.random.seed(2021) self.inputs = { 'X': np.random.random((1, 10, 10)).astype(self.dtype), - 'Y': np.random.random((2, 10, 10)).astype(self.dtype) + 'Y': np.random.random((2, 10, 10)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -149,15 +141,13 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=0.02, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=0.02, check_eager=True + ) # 3D normal batch case class TestSolveOpBatched_case4(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -165,7 +155,7 @@ def setUp(self): np.random.seed(2021) self.inputs = { 'X': np.random.random((3, 6, 6)).astype(self.dtype), - 'Y': np.random.random((3, 6, 7)).astype(self.dtype) + 'Y': np.random.random((3, 6, 7)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -179,7 +169,6 @@ def test_check_grad_normal(self): # 4D normal batch case class TestSolveOpBatched_case5(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -187,7 +176,7 @@ def setUp(self): np.random.seed(2021) self.inputs = { 'X': np.random.random((2, 2, 6, 6)).astype(self.dtype), - 'Y': np.random.random((2, 2, 6, 6)).astype(self.dtype) + 'Y': np.random.random((2, 2, 6, 6)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -201,7 +190,6 @@ def test_check_grad_normal(self): # 4D batch + y broadcast case class TestSolveOpBatched_case6(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -209,7 +197,7 @@ def setUp(self): np.random.seed(2021) self.inputs = { 'X': np.random.random((2, 2, 6, 6)).astype(self.dtype), - 'Y': np.random.random((1, 2, 6, 9)).astype(self.dtype) + 'Y': np.random.random((1, 2, 6, 9)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -223,7 +211,6 @@ def test_check_grad_normal(self): # 5D normal batch case class TestSolveOpBatched_case7(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -231,7 +218,7 @@ def setUp(self): np.random.seed(2021) self.inputs = { 'X': np.random.random((2, 2, 2, 4, 4)).astype(self.dtype), - 'Y': np.random.random((2, 2, 2, 4, 4)).astype(self.dtype) + 'Y': np.random.random((2, 2, 2, 4, 4)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -240,15 +227,13 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=0.04, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=0.04, check_eager=True + ) # 5D batch + y broadcast case class TestSolveOpBatched_case8(OpTest): - def setUp(self): self.python_api = paddle.linalg.solve self.op_type = "solve" @@ -256,7 +241,7 @@ def setUp(self): np.random.seed(2021) self.inputs = { 'X': np.random.random((2, 2, 2, 4, 4)).astype(self.dtype), - 'Y': np.random.random((1, 2, 2, 4, 7)).astype(self.dtype) + 'Y': np.random.random((1, 2, 2, 4, 7)).astype(self.dtype), } result = np.linalg.solve(self.inputs['X'], self.inputs['Y']) self.outputs = {'Out': result} @@ -265,21 +250,21 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], - 'Out', - max_relative_error=0.04, - check_eager=True) + self.check_grad( + ['X', 'Y'], 'Out', max_relative_error=0.04, check_eager=True + ) class TestSolveOpError(unittest.TestCase): - def func_errors(self): with program_guard(Program(), Program()): # The input type of solve_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, paddle.linalg.solve, x1, y1) # The data type of input must be float32 or float64. @@ -317,7 +302,6 @@ def test_dygraph(self): # 2D + vector case, FP64 class TestSolveOpAPI_1(unittest.TestCase): - def setUp(self): np.random.seed(2021) self.place = [paddle.CPUPlace()] @@ -327,12 +311,12 @@ def setUp(self): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - paddle_input_x = fluid.data(name="input_x", - shape=[3, 3], - dtype=self.dtype) - paddle_input_y = fluid.data(name="input_y", - shape=[3], - dtype=self.dtype) + paddle_input_x = fluid.data( + name="input_x", shape=[3, 3], dtype=self.dtype + ) + paddle_input_y = fluid.data( + name="input_y", shape=[3], dtype=self.dtype + ) paddle_result = paddle.linalg.solve(paddle_input_x, paddle_input_y) np_input_x = np.random.random([3, 3]).astype(self.dtype) @@ -341,22 +325,20 @@ def check_static_result(self, place): np_result = np.linalg.solve(np_input_x, np_input_y) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input_x": np_input_x, - "input_y": np_input_y - }, - fetch_list=[paddle_result]) - np.testing.assert_allclose(fetches[0], - np.linalg.solve(np_input_x, np_input_y), - rtol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": np_input_x, "input_y": np_input_y}, + fetch_list=[paddle_result], + ) + np.testing.assert_allclose( + fetches[0], np.linalg.solve(np_input_x, np_input_y), rtol=1e-05 + ) def test_static(self): for place in self.place: self.check_static_result(place=place) def func_dygraph(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -368,9 +350,9 @@ def run(place): numpy_output = np.linalg.solve(input_x_np, input_y_np) paddle_output = paddle.linalg.solve(tensor_input_x, tensor_input_y) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() @@ -385,7 +367,6 @@ def test_dygraph(self): # 2D normal case, FP64 class TestSolveOpAPI_2(unittest.TestCase): - def setUp(self): np.random.seed(2021) self.place = [paddle.CPUPlace()] @@ -396,12 +377,12 @@ def setUp(self): def check_static_result(self, place): paddle.enable_static() with fluid.program_guard(fluid.Program(), fluid.Program()): - paddle_input_x = fluid.data(name="input_x", - shape=[10, 10], - dtype=self.dtype) - paddle_input_y = fluid.data(name="input_y", - shape=[10, 4], - dtype=self.dtype) + paddle_input_x = fluid.data( + name="input_x", shape=[10, 10], dtype=self.dtype + ) + paddle_input_y = fluid.data( + name="input_y", shape=[10, 4], dtype=self.dtype + ) paddle_result = paddle.linalg.solve(paddle_input_x, paddle_input_y) np_input_x = np.random.random([10, 10]).astype(self.dtype) @@ -410,22 +391,20 @@ def check_static_result(self, place): np_result = np.linalg.solve(np_input_x, np_input_y) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input_x": np_input_x, - "input_y": np_input_y - }, - fetch_list=[paddle_result]) - np.testing.assert_allclose(fetches[0], - np.linalg.solve(np_input_x, np_input_y), - rtol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": np_input_x, "input_y": np_input_y}, + fetch_list=[paddle_result], + ) + np.testing.assert_allclose( + fetches[0], np.linalg.solve(np_input_x, np_input_y), rtol=1e-05 + ) def test_static(self): for place in self.place: self.check_static_result(place=place) def func_dygraph(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -436,9 +415,9 @@ def run(place): numpy_output = np.linalg.solve(input_x_np, input_y_np) paddle_output = paddle.linalg.solve(tensor_input_x, tensor_input_y) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() @@ -453,7 +432,6 @@ def test_dygraph(self): # 2D normal case, FP32 class TestSolveOpAPI_3(unittest.TestCase): - def setUp(self): np.random.seed(2021) self.place = [paddle.CPUPlace()] @@ -464,12 +442,12 @@ def setUp(self): def check_static_result(self, place): paddle.enable_static() with fluid.program_guard(fluid.Program(), fluid.Program()): - paddle_input_x = fluid.data(name="input_x", - shape=[10, 10], - dtype=self.dtype) - paddle_input_y = fluid.data(name="input_y", - shape=[10, 4], - dtype=self.dtype) + paddle_input_x = fluid.data( + name="input_x", shape=[10, 10], dtype=self.dtype + ) + paddle_input_y = fluid.data( + name="input_y", shape=[10, 4], dtype=self.dtype + ) paddle_result = paddle.linalg.solve(paddle_input_x, paddle_input_y) np_input_x = np.random.random([10, 10]).astype(self.dtype) @@ -478,22 +456,20 @@ def check_static_result(self, place): np_result = np.linalg.solve(np_input_x, np_input_y) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input_x": np_input_x, - "input_y": np_input_y - }, - fetch_list=[paddle_result]) - np.testing.assert_allclose(fetches[0], - np.linalg.solve(np_input_x, np_input_y), - rtol=0.0001) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": np_input_x, "input_y": np_input_y}, + fetch_list=[paddle_result], + ) + np.testing.assert_allclose( + fetches[0], np.linalg.solve(np_input_x, np_input_y), rtol=0.0001 + ) def test_static(self): for place in self.place: self.check_static_result(place=place) def func_dygraph(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -505,9 +481,9 @@ def run(place): numpy_output = np.linalg.solve(input_x_np, input_y_np) paddle_output = paddle.linalg.solve(tensor_input_x, tensor_input_y) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=0.0001) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=0.0001 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() @@ -522,7 +498,6 @@ def test_dygraph(self): # 3D + y broadcast case, FP64 class TestSolveOpAPI_4(unittest.TestCase): - def setUp(self): np.random.seed(2021) self.place = [paddle.CPUPlace()] @@ -532,12 +507,12 @@ def setUp(self): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): - paddle_input_x = fluid.data(name="input_x", - shape=[2, 3, 3], - dtype=self.dtype) - paddle_input_y = fluid.data(name="input_y", - shape=[1, 3, 3], - dtype=self.dtype) + paddle_input_x = fluid.data( + name="input_x", shape=[2, 3, 3], dtype=self.dtype + ) + paddle_input_y = fluid.data( + name="input_y", shape=[1, 3, 3], dtype=self.dtype + ) paddle_result = paddle.linalg.solve(paddle_input_x, paddle_input_y) np_input_x = np.random.random([2, 3, 3]).astype(self.dtype) @@ -546,22 +521,20 @@ def check_static_result(self, place): np_result = np.linalg.solve(np_input_x, np_input_y) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "input_x": np_input_x, - "input_y": np_input_y - }, - fetch_list=[paddle_result]) - np.testing.assert_allclose(fetches[0], - np.linalg.solve(np_input_x, np_input_y), - rtol=1e-05) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": np_input_x, "input_y": np_input_y}, + fetch_list=[paddle_result], + ) + np.testing.assert_allclose( + fetches[0], np.linalg.solve(np_input_x, np_input_y), rtol=1e-05 + ) def test_static(self): for place in self.place: self.check_static_result(place=place) def func_dygraph(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -573,9 +546,9 @@ def run(place): numpy_output = np.linalg.solve(input_x_np, input_y_np) paddle_output = paddle.linalg.solve(tensor_input_x, tensor_input_y) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() @@ -608,12 +581,11 @@ def check_static_result(self, place): exe = fluid.Executor(place) try: - fetches = exe.run(fluid.default_main_program(), - feed={ - "x": input_x_np, - "y": input_y_np - }, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"x": input_x_np, "y": input_y_np}, + fetch_list=[result], + ) except RuntimeError as ex: print("The mat is singular") except ValueError as ex: diff --git a/python/paddle/fluid/tests/unittests/test_sort_op.py b/python/paddle/fluid/tests/unittests/test_sort_op.py index d28af8f684beb..387f5474dd4b9 100644 --- a/python/paddle/fluid/tests/unittests/test_sort_op.py +++ b/python/paddle/fluid/tests/unittests/test_sort_op.py @@ -21,7 +21,6 @@ class TestSortOnCPU(unittest.TestCase): - def setUp(self): self.place = core.CPUPlace() @@ -30,10 +29,14 @@ def test_api_0(self): input = fluid.data(name="input", shape=[2, 3, 4], dtype="float32") output = paddle.sort(x=input) exe = fluid.Executor(self.place) - data = np.array([[[5, 8, 9, 5], [0, 0, 1, 7], [6, 9, 2, 4]], - [[5, 2, 4, 2], [4, 7, 7, 9], [1, 7, 0, 6]]], - dtype='float32') - result, = exe.run(feed={'input': data}, fetch_list=[output]) + data = np.array( + [ + [[5, 8, 9, 5], [0, 0, 1, 7], [6, 9, 2, 4]], + [[5, 2, 4, 2], [4, 7, 7, 9], [1, 7, 0, 6]], + ], + dtype='float32', + ) + (result,) = exe.run(feed={'input': data}, fetch_list=[output]) np_result = np.sort(result) self.assertEqual((result == np_result).all(), True) @@ -42,16 +45,19 @@ def test_api_1(self): input = fluid.data(name="input", shape=[2, 3, 4], dtype="float32") output = paddle.sort(x=input, axis=1) exe = fluid.Executor(self.place) - data = np.array([[[5, 8, 9, 5], [0, 0, 1, 7], [6, 9, 2, 4]], - [[5, 2, 4, 2], [4, 7, 7, 9], [1, 7, 0, 6]]], - dtype='float32') - result, = exe.run(feed={'input': data}, fetch_list=[output]) + data = np.array( + [ + [[5, 8, 9, 5], [0, 0, 1, 7], [6, 9, 2, 4]], + [[5, 2, 4, 2], [4, 7, 7, 9], [1, 7, 0, 6]], + ], + dtype='float32', + ) + (result,) = exe.run(feed={'input': data}, fetch_list=[output]) np_result = np.sort(result, axis=1) self.assertEqual((result == np_result).all(), True) class TestSortOnGPU(TestSortOnCPU): - def init_place(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -60,7 +66,6 @@ def init_place(self): class TestSortDygraph(unittest.TestCase): - def setUp(self): self.input_data = np.random.rand(10, 10) if core.is_compiled_with_cuda(): @@ -84,8 +89,9 @@ def func_api_1(self): paddle.disable_static(self.place) var_x = paddle.to_tensor(self.input_data) out = paddle.sort(var_x, axis=-1) - self.assertEqual((np.sort(self.input_data, - axis=-1) == out.numpy()).all(), True) + self.assertEqual( + (np.sort(self.input_data, axis=-1) == out.numpy()).all(), True + ) paddle.enable_static() def test_api_1(self): diff --git a/python/paddle/fluid/tests/unittests/test_space_to_depth_op.py b/python/paddle/fluid/tests/unittests/test_space_to_depth_op.py index b80f511716c95..9c43836e2949f 100644 --- a/python/paddle/fluid/tests/unittests/test_space_to_depth_op.py +++ b/python/paddle/fluid/tests/unittests/test_space_to_depth_op.py @@ -19,7 +19,6 @@ class TestSpaceToDepthOp(OpTest): - @staticmethod def helper(in_, width, height, channel, batch, blocksize, forward, out_): channel_out = channel // (blocksize * blocksize) @@ -33,8 +32,9 @@ def helper(in_, width, height, channel, batch, blocksize, forward, out_): width2 = i * blocksize + offset % blocksize height2 = j * blocksize + offset // blocksize out_index = width2 + width * blocksize * ( - height2 + height * blocksize * - (channel2 + channel_out * b)) + height2 + + height * blocksize * (channel2 + channel_out * b) + ) if forward: out_[out_index] = in_[in_index] else: @@ -45,9 +45,16 @@ def setUp(self): self.op_type = "space_to_depth" self.inputs = {"X": self.x} - self.helper(self.x_1d, self.x.shape[3], self.x.shape[2], - self.x.shape[1], self.x.shape[0], self.blocksize, - self.forward, self.out_1d) + self.helper( + self.x_1d, + self.x.shape[3], + self.x.shape[2], + self.x.shape[1], + self.x.shape[0], + self.blocksize, + self.forward, + self.out_1d, + ) self.out = np.reshape(self.out_1d, self.infered_shape) self.attrs = {"blocksize": self.blocksize} self.outputs = {"Out": self.out} @@ -65,18 +72,23 @@ def init_data(self): self.forward = 1 def test_check_output(self): - place = fluid.core.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.core.CPUPlace() + place = ( + fluid.core.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.core.CPUPlace() + ) self.check_output_with_place(place, 1e-5, None, False) def test_check_grad(self): - place = fluid.core.CUDAPlace( - 0) if fluid.core.is_compiled_with_cuda() else fluid.core.CPUPlace() + place = ( + fluid.core.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.core.CPUPlace() + ) self.check_grad_with_place(place, ['X'], 'Out') class TestSpaceToDepthOpBasic(TestSpaceToDepthOp): - def init_data(self): self.ori_shape = (32, 8, 6, 6) self.infered_shape = (32, 32, 3, 3) @@ -91,7 +103,6 @@ def init_data(self): class TestSpaceToDepthOpDoubleBasic(TestSpaceToDepthOp): - def init_data(self): self.ori_shape = (32, 8, 6, 6) self.infered_shape = (32, 32, 3, 3) @@ -106,7 +117,6 @@ def init_data(self): class TestSpaceToDepthOpWithStride3(TestSpaceToDepthOp): - def init_data(self): self.ori_shape = (32, 9, 6, 6) self.infered_shape = (32, 81, 2, 2) @@ -121,7 +131,6 @@ def init_data(self): class TestSpaceToDepthOpWithNotSquare(TestSpaceToDepthOp): - def init_data(self): self.ori_shape = (32, 9, 9, 6) self.infered_shape = (32, 81, 3, 2) diff --git a/python/paddle/fluid/tests/unittests/test_sparse_addmm_op.py b/python/paddle/fluid/tests/unittests/test_sparse_addmm_op.py index d93dba5c77045..ca3dbe4a19a8a 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_addmm_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_addmm_op.py @@ -51,7 +51,7 @@ def check_result(self, input_shape, x_shape, y_shape, format): dense_x.stop_gradient = False dense_y = origin_y.detach() dense_y.stop_gradient = False - dense_out = 2. * paddle.matmul(dense_x, dense_y) + 3. * dense_input + dense_out = 2.0 * paddle.matmul(dense_x, dense_y) + 3.0 * dense_input sp_input = dense_input.detach() sp_input.stop_gradient = False @@ -64,30 +64,36 @@ def check_result(self, input_shape, x_shape, y_shape, format): sp_y.stop_gradient = False sp_out = paddle.sparse.addmm(sp_input, sp_x, sp_y, 3.0, 2.0) - np.testing.assert_allclose(sp_out.numpy(), - dense_out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.numpy(), dense_out.numpy(), rtol=1e-05 + ) if get_cuda_version() >= 11030: dense_out.backward() sp_out.backward() - np.testing.assert_allclose(sp_input.grad.numpy(), - dense_input.grad.numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - (dense_x.grad * mask).numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_y.grad.numpy(), - dense_y.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_input.grad.numpy(), dense_input.grad.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + (dense_x.grad * mask).numpy(), + rtol=1e-05, + ) + np.testing.assert_allclose( + sp_y.grad.numpy(), dense_y.grad.numpy(), rtol=1e-05 + ) - @unittest.skipIf(not paddle.is_compiled_with_cuda() - or get_cuda_version() < 11000, "only support cuda>=11.0") + @unittest.skipIf( + not paddle.is_compiled_with_cuda() or get_cuda_version() < 11000, + "only support cuda>=11.0", + ) def test_addmm_2d(self): self.check_result([16, 10], [16, 12], [12, 10], 'coo') self.check_result([16, 10], [16, 12], [12, 10], 'csr') - @unittest.skipIf(not paddle.is_compiled_with_cuda() - or get_cuda_version() < 11070, "only support cuda>=11.7") + @unittest.skipIf( + not paddle.is_compiled_with_cuda() or get_cuda_version() < 11070, + "only support cuda>=11.7", + ) def test_addmm_3d(self): self.check_result([8, 16, 10], [8, 16, 12], [8, 12, 10], 'coo') self.check_result([8, 16, 10], [8, 16, 12], [8, 12, 10], 'csr') diff --git a/python/paddle/fluid/tests/unittests/test_sparse_attention_op.py b/python/paddle/fluid/tests/unittests/test_sparse_attention_op.py index 6e077502c018b..3b29d335da42b 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_attention_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_attention_op.py @@ -91,14 +91,9 @@ def get_csr_value(mat, layout, nnz): return value -def ref_sparse_attention(q, - k, - v, - offset, - columns, - kp_mask=None, - attn_mask=None, - bsz=None): +def ref_sparse_attention( + q, k, v, offset, columns, kp_mask=None, attn_mask=None, bsz=None +): row, col, nnz = q.shape[0], q.shape[1], columns.shape[0] mat = np.zeros((row, row)) for cur_row in range(row): @@ -109,7 +104,7 @@ def ref_sparse_attention(q, mat[cur_row][cur_col] = 1 a = np.dot(q, k.T) * mat a_value = get_csr_value(a, mat, nnz) - scaling = float(col)**-0.5 + scaling = float(col) ** -0.5 a = scaling * a for i in range(row): for j in range(row): @@ -125,13 +120,9 @@ def ref_sparse_attention(q, return result, a_value, b_value -def ref_batch_sparse_attention(q, - k, - v, - offset, - columns, - kp_mask=None, - attn_mask=None): +def ref_batch_sparse_attention( + q, k, v, offset, columns, kp_mask=None, attn_mask=None +): batch_size, num_heads, row, col = q.shape nnz = columns.shape[2] result = np.zeros((batch_size, num_heads, row, col)) @@ -139,11 +130,16 @@ def ref_batch_sparse_attention(q, result_softmax = np.zeros((batch_size, num_heads, nnz)) for i in range(batch_size): for j in range(num_heads): - cur_q, cur_k, cur_v, = q[i][j], k[i][j], v[i][j] + cur_q, cur_k, cur_v, = ( + q[i][j], + k[i][j], + v[i][j], + ) cur_offset, cur_columns = offset[i][j], columns[i][j] if kp_mask is None and attn_mask is None: cur_result, cur_sdd, cur_softmax = ref_sparse_attention( - cur_q, cur_k, cur_v, cur_offset, cur_columns) + cur_q, cur_k, cur_v, cur_offset, cur_columns + ) else: cur_result, cur_sdd, cur_softmax = ref_sparse_attention( cur_q, @@ -153,7 +149,8 @@ def ref_batch_sparse_attention(q, cur_columns, kp_mask=kp_mask, attn_mask=attn_mask, - bsz=i) + bsz=i, + ) result[i][j] = cur_result result_sdd[i][j], result_softmax[i][j] = cur_sdd, cur_softmax return result, result_sdd, result_softmax @@ -191,10 +188,9 @@ def init_csr_format(batch_size, num_heads, rows, blocksize): @unittest.skipIf( not core.is_compiled_with_cuda() or get_cuda_version() < 11030, - "core is not compiled with CUDA and cuda version need larger than or equal to 11.3" + "core is not compiled with CUDA and cuda version need larger than or equal to 11.3", ) class TestSparseAttentionOp(OpTest): - def config(self): self.shape = (1, 1, 16, 16) self.blocksize = 4 @@ -210,8 +206,9 @@ def setUp(self): self.k = np.random.random(self.shape).astype(self.dtype) self.v = np.random.random(self.shape).astype(self.dtype) # init CSR tensor - offset, columns = init_csr_format(self.shape[0], self.shape[1], - self.shape[2], self.blocksize) + offset, columns = init_csr_format( + self.shape[0], self.shape[1], self.shape[2], self.blocksize + ) self.offset = offset.astype('int32') self.columns = columns.astype('int32') # init mask tensor @@ -232,10 +229,12 @@ def setUp(self): self.offset, self.columns, kp_mask=self.key_padding_mask, - attn_mask=self.attn_mask) + attn_mask=self.attn_mask, + ) else: result, result_sdd, result_softmax = ref_batch_sparse_attention( - self.q, self.k, self.v, self.offset, self.columns) + self.q, self.k, self.v, self.offset, self.columns + ) if self.use_mask == True: self.inputs = { @@ -258,7 +257,7 @@ def setUp(self): self.outputs = { 'Out': result.astype(self.dtype), 'SparseDotSdd': result_sdd.astype(self.dtype), - 'Softmax': result_softmax.astype(self.dtype) + 'Softmax': result_softmax.astype(self.dtype), } def test_check_output(self): @@ -271,7 +270,6 @@ def test_check_grad(self): class TestSparseAttentionOpFp32Test(TestSparseAttentionOp): - def config(self): self.shape = (1, 1, 8, 16) self.blocksize = 2 @@ -280,7 +278,6 @@ def config(self): class TestSparseAttentionOpShapeTest(TestSparseAttentionOp): - def config(self): self.shape = (2, 2, 32, 8) self.blocksize = 8 @@ -290,10 +287,9 @@ def config(self): @unittest.skipIf( not core.is_compiled_with_cuda() or get_cuda_version() < 11030, - "core is not compiled with CUDA and cuda version need larger than or equal to 11.3" + "core is not compiled with CUDA and cuda version need larger than or equal to 11.3", ) class TestSparseAttentionAPI(unittest.TestCase): - def setUp(self): self.place = paddle.CUDAPlace(0) self.shape = (1, 1, 8, 4) @@ -308,54 +304,62 @@ def test_static_graph(self): K = paddle.static.data(name="K", shape=self.shape, dtype=self.dtype) V = paddle.static.data(name="V", shape=self.shape, dtype=self.dtype) - batch_size, num_heads, rows = self.shape[0], self.shape[ - 1], self.shape[2] + batch_size, num_heads, rows = ( + self.shape[0], + self.shape[1], + self.shape[2], + ) block_num = rows / self.blocksize block_last = rows % self.blocksize - sparse_nnz_num = block_num * self.blocksize * self.blocksize + block_last * block_last + sparse_nnz_num = ( + block_num * self.blocksize * self.blocksize + + block_last * block_last + ) offset_shape = (batch_size, num_heads, rows + 1) columns_shape = (batch_size, num_heads, int(sparse_nnz_num)) - offset = paddle.static.data(name="Offset", - shape=offset_shape, - dtype="int32") - columns = paddle.static.data(name="Columns", - shape=columns_shape, - dtype="int32") + offset = paddle.static.data( + name="Offset", shape=offset_shape, dtype="int32" + ) + columns = paddle.static.data( + name="Columns", shape=columns_shape, dtype="int32" + ) key_padding_mask_shape = (self.shape[0], self.shape[2]) attn_mask_shape = (self.shape[2], self.shape[2]) if self.use_mask == True: key_padding_mask = paddle.static.data( name="KeyPaddingMask", shape=key_padding_mask_shape, - dtype=self.dtype) - attn_mask = paddle.static.data(name="AttnMask", - shape=attn_mask_shape, - dtype=self.dtype) - Out = F.sparse_attention(Q, - K, - V, - offset, - columns, - key_padding_mask=key_padding_mask, - attn_mask=attn_mask) + dtype=self.dtype, + ) + attn_mask = paddle.static.data( + name="AttnMask", shape=attn_mask_shape, dtype=self.dtype + ) + Out = F.sparse_attention( + Q, + K, + V, + offset, + columns, + key_padding_mask=key_padding_mask, + attn_mask=attn_mask, + ) else: Out = F.sparse_attention(Q, K, V, offset, columns) Q_np = np.random.random(self.shape).astype(self.dtype) K_np = np.random.random(self.shape).astype(self.dtype) V_np = np.random.random(self.shape).astype(self.dtype) - offset_np, columns_np = init_csr_format(self.shape[0], - self.shape[1], - self.shape[2], - self.blocksize) + offset_np, columns_np = init_csr_format( + self.shape[0], self.shape[1], self.shape[2], self.blocksize + ) offset_np = offset_np.astype('int32') columns_np = columns_np.astype('int32') # init mask tensor - key_padding_mask_np = np.random.randint(0, - 2, - size=key_padding_mask_shape) + key_padding_mask_np = np.random.randint( + 0, 2, size=key_padding_mask_shape + ) attn_mask_np = np.random.randint(0, 2, size=attn_mask_shape) key_padding_mask_np = init_mask(key_padding_mask_np) attn_mask_np = init_mask(attn_mask_np) @@ -364,16 +368,18 @@ def test_static_graph(self): exe = fluid.Executor(self.place) if self.use_mask == True: - fetches_result = exe.run(feed={ - "Q": Q_np, - "K": K_np, - "V": V_np, - "Offset": offset_np, - "Columns": columns_np, - 'KeyPaddingMask': key_padding_mask_np, - 'AttnMask': attn_mask_np - }, - fetch_list=[Out]) + fetches_result = exe.run( + feed={ + "Q": Q_np, + "K": K_np, + "V": V_np, + "Offset": offset_np, + "Columns": columns_np, + 'KeyPaddingMask': key_padding_mask_np, + 'AttnMask': attn_mask_np, + }, + fetch_list=[Out], + ) expected_result, __, __ = ref_batch_sparse_attention( Q_np, K_np, @@ -381,28 +387,32 @@ def test_static_graph(self): offset_np, columns_np, kp_mask=key_padding_mask_np, - attn_mask=attn_mask_np) + attn_mask=attn_mask_np, + ) else: - fetches_result = exe.run(feed={ - "Q": Q_np, - "K": K_np, - "V": V_np, - "Offset": offset_np, - "Columns": columns_np - }, - fetch_list=[Out]) + fetches_result = exe.run( + feed={ + "Q": Q_np, + "K": K_np, + "V": V_np, + "Offset": offset_np, + "Columns": columns_np, + }, + fetch_list=[Out], + ) expected_result, __, __ = ref_batch_sparse_attention( - Q_np, K_np, V_np, offset_np, columns_np) + Q_np, K_np, V_np, offset_np, columns_np + ) - np.testing.assert_allclose(fetches_result, - expected_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + fetches_result, expected_result, rtol=1e-05, atol=1e-05 + ) def test_dygraph(self): paddle.disable_static() - offset, columns = init_csr_format(self.shape[0], self.shape[1], - self.shape[2], self.blocksize) + offset, columns = init_csr_format( + self.shape[0], self.shape[1], self.shape[2], self.blocksize + ) offset = offset.astype('int32') columns = columns.astype('int32') query = np.random.random(self.shape).astype(self.dtype) @@ -427,13 +437,15 @@ def test_dygraph(self): paddle_attn_mask = paddle.to_tensor(attn_mask, place=self.place) if self.use_mask == True: - paddle_result = F.sparse_attention(paddle_query, - paddle_key, - paddle_value, - paddle_offset, - paddle_colunmns, - key_padding_mask=paddle_kp_mask, - attn_mask=paddle_attn_mask) + paddle_result = F.sparse_attention( + paddle_query, + paddle_key, + paddle_value, + paddle_offset, + paddle_colunmns, + key_padding_mask=paddle_kp_mask, + attn_mask=paddle_attn_mask, + ) numpy_result, __, __ = ref_batch_sparse_attention( query, @@ -442,25 +454,29 @@ def test_dygraph(self): offset, columns, kp_mask=key_padding_mask, - attn_mask=attn_mask) + attn_mask=attn_mask, + ) numpy_result = numpy_result.astype(self.dtype) else: - paddle_result = F.sparse_attention(paddle_query, paddle_key, - paddle_value, paddle_offset, - paddle_colunmns) + paddle_result = F.sparse_attention( + paddle_query, + paddle_key, + paddle_value, + paddle_offset, + paddle_colunmns, + ) numpy_result, __, __ = ref_batch_sparse_attention( - query, key, value, offset, columns) + query, key, value, offset, columns + ) numpy_result = numpy_result.astype(self.dtype) - np.testing.assert_allclose(paddle_result.numpy(), - numpy_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + paddle_result.numpy(), numpy_result, rtol=1e-05, atol=1e-05 + ) class TestSparseAttentionAPITestFloat(TestSparseAttentionAPI): - def setUp(self): self.place = paddle.CUDAPlace(0) self.shape = (2, 2, 8, 4) @@ -470,7 +486,6 @@ def setUp(self): class TestSparseAttentionAPITestShape1(TestSparseAttentionAPI): - def setUp(self): self.place = paddle.CUDAPlace(0) self.shape = (2, 2, 64, 32) @@ -480,7 +495,6 @@ def setUp(self): class TestSparseAttentionAPITestShape2(TestSparseAttentionAPI): - def setUp(self): self.place = paddle.CUDAPlace(0) self.shape = (2, 1, 64, 32) @@ -490,7 +504,6 @@ def setUp(self): class TestSparseAttentionAPITestShape3(TestSparseAttentionAPI): - def setUp(self): self.place = paddle.CUDAPlace(0) self.shape = (4, 4, 128, 32) @@ -500,7 +513,6 @@ def setUp(self): class TestSparseAttentionAPITestShape4(TestSparseAttentionAPI): - def setUp(self): self.place = paddle.CUDAPlace(0) self.shape = (3, 3, 35, 15) diff --git a/python/paddle/fluid/tests/unittests/test_sparse_conv_op.py b/python/paddle/fluid/tests/unittests/test_sparse_conv_op.py index dfd7d23f8a44c..0b9c88aad9de0 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_conv_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_conv_op.py @@ -21,13 +21,12 @@ class TestSparseConv(unittest.TestCase): - def test_conv3d(self): with _test_eager_guard(): kernel = [[[[[1], [1], [1]], [[1], [1], [1]], [[1], [1], [1]]]]] - dense_kernel = paddle.to_tensor(kernel, - dtype='float32', - stop_gradient=False) + dense_kernel = paddle.to_tensor( + kernel, dtype='float32', stop_gradient=False + ) dense_kernel = paddle.reshape(dense_kernel, [1, 3, 3, 1, 1]) paddings = [0, 0, 0] strides = [1, 1, 1] @@ -40,17 +39,19 @@ def test_conv3d(self): values = paddle.to_tensor(values, dtype='float32') dense_shape = [1, 1, 3, 4, 1] correct_out_values = [[5], [11]] - sparse_input = core.eager.sparse_coo_tensor(indices, values, - dense_shape, False) - out = paddle.sparse.nn.functional.conv3d(sparse_input, - dense_kernel, - bias=paddle.to_tensor( - bias, dtype='float32'), - stride=strides, - padding=paddings, - dilation=dilations, - groups=1, - data_format="NDHWC") + sparse_input = core.eager.sparse_coo_tensor( + indices, values, dense_shape, False + ) + out = paddle.sparse.nn.functional.conv3d( + sparse_input, + dense_kernel, + bias=paddle.to_tensor(bias, dtype='float32'), + stride=strides, + padding=paddings, + dilation=dilations, + groups=1, + data_format="NDHWC", + ) out.backward(out) out = paddle.sparse.coalesce(out) assert np.array_equal(correct_out_values, out.values().numpy()) @@ -62,41 +63,41 @@ def test_subm_conv3d(self): indices = paddle.to_tensor(indices, dtype='int32') values = paddle.to_tensor(values, dtype='float32') dense_shape = [1, 1, 3, 4, 1] - sparse_x = paddle.sparse.sparse_coo_tensor(indices, - values, - dense_shape, - stop_gradient=True) + sparse_x = paddle.sparse.sparse_coo_tensor( + indices, values, dense_shape, stop_gradient=True + ) weight = paddle.randn((1, 3, 3, 1, 1), dtype='float32') - y = paddle.sparse.nn.functional.subm_conv3d(sparse_x, - weight, - key='subm_conv') - assert np.array_equal(sparse_x.indices().numpy(), - y.indices().numpy()) + y = paddle.sparse.nn.functional.subm_conv3d( + sparse_x, weight, key='subm_conv' + ) + assert np.array_equal( + sparse_x.indices().numpy(), y.indices().numpy() + ) def test_Conv3D(self): with _test_eager_guard(): - #(4, non_zero_num), 4-D:(N, D, H, W) + # (4, non_zero_num), 4-D:(N, D, H, W) indices = [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 1, 2], [1, 3, 2, 3]] - #(non_zero_num, C) + # (non_zero_num, C) values = [[1], [2], [3], [4]] indices = paddle.to_tensor(indices, dtype='int32') values = paddle.to_tensor(values, dtype='float32') dense_shape = [1, 1, 3, 4, 1] correct_out_values = [[4], [10]] sparse_input = paddle.sparse.sparse_coo_tensor( - indices, values, dense_shape, False) + indices, values, dense_shape, False + ) - sparse_conv3d = paddle.sparse.nn.Conv3D(1, - 1, (1, 3, 3), - data_format='NDHWC') + sparse_conv3d = paddle.sparse.nn.Conv3D( + 1, 1, (1, 3, 3), data_format='NDHWC' + ) sparse_out = sparse_conv3d(sparse_input) - #test errors + # test errors with self.assertRaises(ValueError): - #Currently, only support data_format='NDHWC' - conv3d = paddle.sparse.nn.SubmConv3D(1, - 1, (1, 3, 3), - data_format='NCDHW', - key='subm_conv') + # Currently, only support data_format='NDHWC' + conv3d = paddle.sparse.nn.SubmConv3D( + 1, 1, (1, 3, 3), data_format='NCDHW', key='subm_conv' + ) def test_SubmConv3D(self): with _test_eager_guard(): @@ -107,12 +108,12 @@ def test_SubmConv3D(self): dense_shape = [1, 1, 3, 4, 1] correct_out_values = [[4], [10]] sparse_input = paddle.sparse.sparse_coo_tensor( - indices, values, dense_shape, False) + indices, values, dense_shape, False + ) - subm_conv3d = paddle.sparse.nn.SubmConv3D(1, - 1, (1, 3, 3), - data_format='NDHWC', - key='subm_conv') + subm_conv3d = paddle.sparse.nn.SubmConv3D( + 1, 1, (1, 3, 3), data_format='NDHWC', key='subm_conv' + ) # test extra_repr print(subm_conv3d.extra_repr()) @@ -120,13 +121,12 @@ def test_SubmConv3D(self): # the output shape of subm_conv is same as input shape assert np.array_equal(indices, sparse_out.indices().numpy()) - #test errors + # test errors with self.assertRaises(ValueError): - #Currently, only support data_format='NDHWC' - conv3d = paddle.sparse.nn.SubmConv3D(1, - 1, (1, 3, 3), - data_format='NCDHW', - key='subm_conv') + # Currently, only support data_format='NDHWC' + conv3d = paddle.sparse.nn.SubmConv3D( + 1, 1, (1, 3, 3), data_format='NCDHW', key='subm_conv' + ) def test_Conv3D_bias(self): with _test_eager_guard(): @@ -138,8 +138,8 @@ def test_Conv3D_bias(self): sp_conv3d = paddle.sparse.nn.Conv3D(3, 2, 3, data_format='NDHWC') sp_conv3d.weight.set_value( - paddle.to_tensor(conv3d.weight.numpy().transpose(2, 3, 4, 1, - 0))) + paddle.to_tensor(conv3d.weight.numpy().transpose(2, 3, 4, 1, 0)) + ) sp_conv3d.bias.set_value(paddle.to_tensor(conv3d.bias.numpy())) x.stop_gradient = False @@ -152,50 +152,53 @@ def test_Conv3D_bias(self): dense_out = sp_out.to_dense() sp_loss = dense_out.mean() sp_loss.backward() - assert np.allclose(out.numpy(), - dense_out.numpy(), - atol=1e-3, - rtol=1e-3) - assert np.allclose(conv3d.weight.grad.numpy().transpose( - 2, 3, 4, 1, 0), - sp_conv3d.weight.grad.numpy(), - atol=1e-3, - rtol=1e-3) - assert np.allclose(conv3d.bias.grad.numpy(), - sp_conv3d.bias.grad.numpy(), - atol=1e-5, - rtol=1e-5) + assert np.allclose( + out.numpy(), dense_out.numpy(), atol=1e-3, rtol=1e-3 + ) + assert np.allclose( + conv3d.weight.grad.numpy().transpose(2, 3, 4, 1, 0), + sp_conv3d.weight.grad.numpy(), + atol=1e-3, + rtol=1e-3, + ) + assert np.allclose( + conv3d.bias.grad.numpy(), + sp_conv3d.bias.grad.numpy(), + atol=1e-5, + rtol=1e-5, + ) class TestStatic(unittest.TestCase): - def test(self): paddle.enable_static() - indices = paddle.static.data(name='indices', - shape=[4, 4], - dtype='int32') - values = paddle.static.data(name='values', - shape=[4, 1], - dtype='float32') + indices = paddle.static.data( + name='indices', shape=[4, 4], dtype='int32' + ) + values = paddle.static.data( + name='values', shape=[4, 1], dtype='float32' + ) dense_shape = [1, 1, 3, 4, 1] sp_x = sparse.sparse_coo_tensor(indices, values, dense_shape) weight_shape = [1, 3, 3, 1, 1] - weight = paddle.static.data(name='weight', - shape=weight_shape, - dtype='float32') + weight = paddle.static.data( + name='weight', shape=weight_shape, dtype='float32' + ) bias_shape = [1] - bias = paddle.static.data(name='bias', - shape=bias_shape, - dtype='float32') - out = sparse.nn.functional.conv3d(sp_x, - weight, - bias, - stride=1, - padding=0, - dilation=1, - groups=1, - data_format="NDHWC") + bias = paddle.static.data( + name='bias', shape=bias_shape, dtype='float32' + ) + out = sparse.nn.functional.conv3d( + sp_x, + weight, + bias, + stride=1, + padding=0, + dilation=1, + groups=1, + data_format="NDHWC", + ) sp_out = sparse.nn.functional.relu(out) out_indices = sp_out.indices() out_values = sp_out.values() @@ -205,19 +208,22 @@ def test(self): indices_data = [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 1, 2], [1, 3, 2, 3]] values_data = [[1.0], [2.0], [3.0], [4.0]] - weight_data = np.array([[[[[1], [1], [1]], [[1], [1], [1]], - [[1], [1], [1]]]]]).astype('float32') + weight_data = np.array( + [[[[[1], [1], [1]], [[1], [1], [1]], [[1], [1], [1]]]]] + ).astype('float32') weight_data = weight_data.reshape(weight_shape) bias_data = np.array([1]).astype('float32') - fetch = exe.run(feed={ - 'indices': indices_data, - 'values': values_data, - 'weight': weight_data, - 'bias': bias_data - }, - fetch_list=[out, out_indices, out_values], - return_numpy=True) + fetch = exe.run( + feed={ + 'indices': indices_data, + 'values': values_data, + 'weight': weight_data, + 'bias': bias_data, + }, + fetch_list=[out, out_indices, out_values], + return_numpy=True, + ) correct_out = np.array([[[[[5.0], [11.0]]]]]).astype('float64') correct_out_values = [[5.0], [11.0]] assert np.array_equal(correct_out, fetch[0]) diff --git a/python/paddle/fluid/tests/unittests/test_sparse_copy_op.py b/python/paddle/fluid/tests/unittests/test_sparse_copy_op.py index 587e4b6d68a7b..c31dc4e159fce 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_copy_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_copy_op.py @@ -19,7 +19,6 @@ class TestSparseCopy(unittest.TestCase): - def test_copy_sparse_coo(self): with _test_eager_guard(): np_x = [[0, 1.0, 0], [2.0, 0, 0], [0, 3.0, 0]] diff --git a/python/paddle/fluid/tests/unittests/test_sparse_elementwise_op.py b/python/paddle/fluid/tests/unittests/test_sparse_elementwise_op.py index 172bdaee2abb3..f762584801ea7 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_elementwise_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_elementwise_op.py @@ -68,37 +68,45 @@ def func_test_csr(self, op): expect_res = op(dense_x, dense_y) expect_res.backward(expect_res) - np.testing.assert_allclose(expect_res.numpy(), - actual_res.to_dense().numpy(), - rtol=1e-05, - equal_nan=True) + np.testing.assert_allclose( + expect_res.numpy(), + actual_res.to_dense().numpy(), + rtol=1e-05, + equal_nan=True, + ) if not (op == __truediv__ and dtype in ['int32', 'int64']): - np.testing.assert_allclose(dense_x.grad.numpy(), - csr_x.grad.to_dense().numpy(), - rtol=1e-05, - equal_nan=True) - np.testing.assert_allclose(dense_y.grad.numpy(), - csr_y.grad.to_dense().numpy(), - rtol=1e-05, - equal_nan=True) + np.testing.assert_allclose( + dense_x.grad.numpy(), + csr_x.grad.to_dense().numpy(), + rtol=1e-05, + equal_nan=True, + ) + np.testing.assert_allclose( + dense_y.grad.numpy(), + csr_y.grad.to_dense().numpy(), + rtol=1e-05, + equal_nan=True, + ) def func_test_coo(self, op): for sparse_dim in range(len(self.coo_shape) - 1, len(self.coo_shape)): for dtype in self.support_dtypes: - x = np.random.randint(-255, 255, - size=self.coo_shape).astype(dtype) - y = np.random.randint(-255, 255, - size=self.coo_shape).astype(dtype) + x = np.random.randint(-255, 255, size=self.coo_shape).astype( + dtype + ) + y = np.random.randint(-255, 255, size=self.coo_shape).astype( + dtype + ) dense_x = paddle.to_tensor(x, dtype=dtype, stop_gradient=False) dense_y = paddle.to_tensor(y, dtype=dtype, stop_gradient=False) - s_dense_x = paddle.to_tensor(x, - dtype=dtype, - stop_gradient=False) - s_dense_y = paddle.to_tensor(y, - dtype=dtype, - stop_gradient=False) + s_dense_x = paddle.to_tensor( + x, dtype=dtype, stop_gradient=False + ) + s_dense_y = paddle.to_tensor( + y, dtype=dtype, stop_gradient=False + ) coo_x = s_dense_x.to_sparse_coo(sparse_dim) coo_y = s_dense_y.to_sparse_coo(sparse_dim) @@ -108,18 +116,24 @@ def func_test_coo(self, op): expect_res = op(dense_x, dense_y) expect_res.backward(expect_res) - np.testing.assert_allclose(expect_res.numpy(), - actual_res.to_dense().numpy(), - rtol=1e-05, - equal_nan=True) - np.testing.assert_allclose(dense_x.grad.numpy(), - coo_x.grad.to_dense().numpy(), - rtol=1e-05, - equal_nan=True) - np.testing.assert_allclose(dense_y.grad.numpy(), - coo_y.grad.to_dense().numpy(), - rtol=1e-05, - equal_nan=True) + np.testing.assert_allclose( + expect_res.numpy(), + actual_res.to_dense().numpy(), + rtol=1e-05, + equal_nan=True, + ) + np.testing.assert_allclose( + dense_x.grad.numpy(), + coo_x.grad.to_dense().numpy(), + rtol=1e-05, + equal_nan=True, + ) + np.testing.assert_allclose( + dense_y.grad.numpy(), + coo_y.grad.to_dense().numpy(), + rtol=1e-05, + equal_nan=True, + ) def test_support_dtypes_csr(self): paddle.device.set_device('cpu') @@ -139,38 +153,37 @@ def test_add_same_indices(self): values2_data = [[1.0], [2.0]] shape = [2, 4, 2] - sp_a = sparse.sparse_coo_tensor(indices_data, - values1_data, - shape, - stop_gradient=False) - sp_b = sparse.sparse_coo_tensor(indices_data, - values2_data, - shape, - stop_gradient=False) + sp_a = sparse.sparse_coo_tensor( + indices_data, values1_data, shape, stop_gradient=False + ) + sp_b = sparse.sparse_coo_tensor( + indices_data, values2_data, shape, stop_gradient=False + ) values1 = paddle.to_tensor(values1_data, stop_gradient=False) values2 = paddle.to_tensor(values2_data, stop_gradient=False) - #c.values() = a.values() + b.values() + # c.values() = a.values() + b.values() sp_c = sparse.add(sp_a, sp_b) sp_c.backward() ref_c = values1 + values2 ref_c.backward() np.testing.assert_allclose(sp_c.values().numpy(), ref_c.numpy()) - np.testing.assert_allclose(sp_a.grad.values().numpy(), - values1.grad.numpy()) - np.testing.assert_allclose(sp_b.grad.values().numpy(), - values2.grad.numpy()) + np.testing.assert_allclose( + sp_a.grad.values().numpy(), values1.grad.numpy() + ) + np.testing.assert_allclose( + sp_b.grad.values().numpy(), values2.grad.numpy() + ) def test_add_bias(self): indices_data = [[0, 1], [0, 3]] values_data = [[1.0, 1.0], [2.0, 2.0]] shape = [2, 4, 2] - sp_a = sparse.sparse_coo_tensor(indices_data, - values_data, - shape, - stop_gradient=False) + sp_a = sparse.sparse_coo_tensor( + indices_data, values_data, shape, stop_gradient=False + ) bias_values = [1.0, 2.0] @@ -178,14 +191,15 @@ def test_add_bias(self): values2 = paddle.to_tensor(bias_values, stop_gradient=False) values3 = paddle.to_tensor(bias_values, stop_gradient=False) - #c.values() = a.values() + b + # c.values() = a.values() + b sp_c = sparse.add(sp_a, values2) sp_c.backward() ref_c = values1 + values3 ref_c.backward() np.testing.assert_allclose(sp_c.values().numpy(), ref_c.numpy()) - np.testing.assert_allclose(sp_a.grad.values().numpy(), - values1.grad.numpy()) + np.testing.assert_allclose( + sp_a.grad.values().numpy(), values1.grad.numpy() + ) np.testing.assert_allclose(values2.grad.numpy(), values3.grad.numpy()) diff --git a/python/paddle/fluid/tests/unittests/test_sparse_fused_attention_op.py b/python/paddle/fluid/tests/unittests/test_sparse_fused_attention_op.py index 50e9218a27d35..dfc8806fd02ad 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_fused_attention_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_fused_attention_op.py @@ -37,10 +37,9 @@ def get_cuda_version(): @unittest.skipIf( not core.is_compiled_with_cuda() or get_cuda_version() < 11070, - "core is not compiled with CUDA and cuda version need larger than or equal to 11.7" + "core is not compiled with CUDA and cuda version need larger than or equal to 11.7", ) class TestSparseAttentionAPI1(unittest.TestCase): - def setUp(self): self.batch_size = 16 self.num_heads = 16 @@ -52,7 +51,10 @@ def setUp(self): def test_dygraph(self): with _test_eager_guard(): self.shape = [ - self.batch_size, self.num_heads, self.seq_len, self.head_dim + self.batch_size, + self.num_heads, + self.seq_len, + self.head_dim, ] query = paddle.rand(self.shape, self.dtype) key = paddle.rand(self.shape, self.dtype) @@ -62,13 +64,16 @@ def test_dygraph(self): key.stop_gradient = False value.stop_gradient = False - mask = paddle.nn.functional.dropout(paddle.ones( - [self.seq_len, self.seq_len]), - mode='downscale_in_infer') + mask = paddle.nn.functional.dropout( + paddle.ones([self.seq_len, self.seq_len]), + mode='downscale_in_infer', + ) mask = mask.expand( - [self.batch_size, self.num_heads, self.seq_len, self.seq_len]) - sp_mask = mask.reshape([-1, self.seq_len, - self.seq_len]).to_sparse_csr() + [self.batch_size, self.num_heads, self.seq_len, self.seq_len] + ) + sp_mask = mask.reshape( + [-1, self.seq_len, self.seq_len] + ).to_sparse_csr() query_sp = copy.deepcopy(query) key_sp = copy.deepcopy(key) @@ -80,49 +85,57 @@ def test_dygraph(self): if self.use_mask: kp_mask = paddle.randint( - 0, 2, [self.batch_size, self.seq_len]).astype(self.dtype) + 0, 2, [self.batch_size, self.seq_len] + ).astype(self.dtype) attn_mask = paddle.randint( - 0, 2, [self.seq_len, self.seq_len]).astype(self.dtype) + 0, 2, [self.seq_len, self.seq_len] + ).astype(self.dtype) sdd = paddle.matmul(query, key, False, True) / math.sqrt( - float(self.head_dim)) - sdd = sdd + ( - (mask * kp_mask.unsqueeze([1, 2]) * attn_mask) - 1.0) * 1e9 + float(self.head_dim) + ) + sdd = ( + sdd + + ((mask * kp_mask.unsqueeze([1, 2]) * attn_mask) - 1.0) + * 1e9 + ) softmax = paddle.nn.functional.softmax(sdd) output = paddle.matmul(softmax, value) output.backward() output_sp = paddle.sparse.nn.functional.attention( - query_sp, key_sp, value_sp, sp_mask, kp_mask, attn_mask) + query_sp, key_sp, value_sp, sp_mask, kp_mask, attn_mask + ) output_sp.backward() else: sdd = paddle.matmul(query, key, False, True) / math.sqrt( - float(self.head_dim)) + float(self.head_dim) + ) sdd = sdd + (mask - 1.0) * 1e9 softmax = paddle.nn.functional.softmax(sdd) output = paddle.matmul(softmax, value) output.backward() output_sp = paddle.sparse.nn.functional.attention( - query_sp, key_sp, value_sp, sp_mask) + query_sp, key_sp, value_sp, sp_mask + ) output_sp.backward() - np.testing.assert_allclose(output_sp.numpy(), - output.numpy(), - rtol=1e-05) - np.testing.assert_allclose(query_sp.grad.numpy(), - query.grad.numpy(), - rtol=1e-05) - np.testing.assert_allclose(key_sp.grad.numpy(), - key.grad.numpy(), - rtol=1e-05) - np.testing.assert_allclose(value_sp.grad.numpy(), - value.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + output_sp.numpy(), output.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + query_sp.grad.numpy(), query.grad.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + key_sp.grad.numpy(), key.grad.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + value_sp.grad.numpy(), value.grad.numpy(), rtol=1e-05 + ) class TestSparseAttentionAPI2(TestSparseAttentionAPI1): - def setUp(self): self.batch_size = 16 self.num_heads = 16 @@ -133,7 +146,6 @@ def setUp(self): class TestSparseAttentionAPI3(TestSparseAttentionAPI1): - def setUp(self): self.batch_size = 16 self.num_heads = 16 @@ -144,7 +156,6 @@ def setUp(self): class TestSparseAttentionAPI4(TestSparseAttentionAPI1): - def setUp(self): self.batch_size = 16 self.num_heads = 16 @@ -155,7 +166,6 @@ def setUp(self): class TestSparseAttentionAPI5(TestSparseAttentionAPI1): - def setUp(self): self.batch_size = 16 self.num_heads = 16 diff --git a/python/paddle/fluid/tests/unittests/test_sparse_is_same_shape.py b/python/paddle/fluid/tests/unittests/test_sparse_is_same_shape.py index 8715b55d522ad..f1c2ded049d6d 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_is_same_shape.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_is_same_shape.py @@ -28,7 +28,7 @@ def setUp(self): self.tensors = [ paddle.rand(self.shapes[0]), paddle.rand(self.shapes[0]), - paddle.rand(self.shapes[1]) + paddle.rand(self.shapes[1]), ] self.sparse_dim = 2 @@ -39,85 +39,136 @@ def test_dense_dense(self): def test_dense_csr(self): self.assertTrue( - is_same_shape(self.tensors[0], self.tensors[1].to_sparse_csr())) + is_same_shape(self.tensors[0], self.tensors[1].to_sparse_csr()) + ) self.assertFalse( - is_same_shape(self.tensors[0], self.tensors[2].to_sparse_csr())) + is_same_shape(self.tensors[0], self.tensors[2].to_sparse_csr()) + ) self.assertFalse( - is_same_shape(self.tensors[1], self.tensors[2].to_sparse_csr())) + is_same_shape(self.tensors[1], self.tensors[2].to_sparse_csr()) + ) def test_dense_coo(self): self.assertTrue( - is_same_shape(self.tensors[0], - self.tensors[1].to_sparse_coo(self.sparse_dim))) + is_same_shape( + self.tensors[0], self.tensors[1].to_sparse_coo(self.sparse_dim) + ) + ) self.assertFalse( - is_same_shape(self.tensors[0], - self.tensors[2].to_sparse_coo(self.sparse_dim))) + is_same_shape( + self.tensors[0], self.tensors[2].to_sparse_coo(self.sparse_dim) + ) + ) self.assertFalse( - is_same_shape(self.tensors[1], - self.tensors[2].to_sparse_coo(self.sparse_dim))) + is_same_shape( + self.tensors[1], self.tensors[2].to_sparse_coo(self.sparse_dim) + ) + ) def test_csr_dense(self): self.assertTrue( - is_same_shape(self.tensors[0].to_sparse_csr(), self.tensors[1])) + is_same_shape(self.tensors[0].to_sparse_csr(), self.tensors[1]) + ) self.assertFalse( - is_same_shape(self.tensors[0].to_sparse_csr(), self.tensors[2])) + is_same_shape(self.tensors[0].to_sparse_csr(), self.tensors[2]) + ) self.assertFalse( - is_same_shape(self.tensors[1].to_sparse_csr(), self.tensors[2])) + is_same_shape(self.tensors[1].to_sparse_csr(), self.tensors[2]) + ) def test_csr_csr(self): self.assertTrue( - is_same_shape(self.tensors[0].to_sparse_csr(), - self.tensors[1].to_sparse_csr())) + is_same_shape( + self.tensors[0].to_sparse_csr(), self.tensors[1].to_sparse_csr() + ) + ) self.assertFalse( - is_same_shape(self.tensors[0].to_sparse_csr(), - self.tensors[2].to_sparse_csr())) + is_same_shape( + self.tensors[0].to_sparse_csr(), self.tensors[2].to_sparse_csr() + ) + ) self.assertFalse( - is_same_shape(self.tensors[1].to_sparse_csr(), - self.tensors[2].to_sparse_csr())) + is_same_shape( + self.tensors[1].to_sparse_csr(), self.tensors[2].to_sparse_csr() + ) + ) def test_csr_coo(self): self.assertTrue( - is_same_shape(self.tensors[0].to_sparse_csr(), - self.tensors[1].to_sparse_coo(self.sparse_dim))) - self.assertFalse( - is_same_shape(self.tensors[0].to_sparse_csr(), - self.tensors[2].to_sparse_coo(self.sparse_dim))) - self.assertFalse( - is_same_shape(self.tensors[1].to_sparse_csr(), - self.tensors[2].to_sparse_coo(self.sparse_dim))) + is_same_shape( + self.tensors[0].to_sparse_csr(), + self.tensors[1].to_sparse_coo(self.sparse_dim), + ) + ) + self.assertFalse( + is_same_shape( + self.tensors[0].to_sparse_csr(), + self.tensors[2].to_sparse_coo(self.sparse_dim), + ) + ) + self.assertFalse( + is_same_shape( + self.tensors[1].to_sparse_csr(), + self.tensors[2].to_sparse_coo(self.sparse_dim), + ) + ) def test_coo_dense(self): self.assertTrue( - is_same_shape(self.tensors[0].to_sparse_coo(self.sparse_dim), - self.tensors[1])) + is_same_shape( + self.tensors[0].to_sparse_coo(self.sparse_dim), self.tensors[1] + ) + ) self.assertFalse( - is_same_shape(self.tensors[0].to_sparse_coo(self.sparse_dim), - self.tensors[2])) + is_same_shape( + self.tensors[0].to_sparse_coo(self.sparse_dim), self.tensors[2] + ) + ) self.assertFalse( - is_same_shape(self.tensors[1].to_sparse_coo(self.sparse_dim), - self.tensors[2])) + is_same_shape( + self.tensors[1].to_sparse_coo(self.sparse_dim), self.tensors[2] + ) + ) def test_coo_csr(self): self.assertTrue( - is_same_shape(self.tensors[0].to_sparse_coo(self.sparse_dim), - self.tensors[1].to_sparse_csr())) - self.assertFalse( - is_same_shape(self.tensors[0].to_sparse_coo(self.sparse_dim), - self.tensors[2].to_sparse_csr())) - self.assertFalse( - is_same_shape(self.tensors[1].to_sparse_coo(self.sparse_dim), - self.tensors[2].to_sparse_csr())) + is_same_shape( + self.tensors[0].to_sparse_coo(self.sparse_dim), + self.tensors[1].to_sparse_csr(), + ) + ) + self.assertFalse( + is_same_shape( + self.tensors[0].to_sparse_coo(self.sparse_dim), + self.tensors[2].to_sparse_csr(), + ) + ) + self.assertFalse( + is_same_shape( + self.tensors[1].to_sparse_coo(self.sparse_dim), + self.tensors[2].to_sparse_csr(), + ) + ) def test_coo_coo(self): self.assertTrue( - is_same_shape(self.tensors[0].to_sparse_coo(self.sparse_dim), - self.tensors[1].to_sparse_coo(self.sparse_dim))) - self.assertFalse( - is_same_shape(self.tensors[0].to_sparse_coo(self.sparse_dim), - self.tensors[2].to_sparse_coo(self.sparse_dim))) - self.assertFalse( - is_same_shape(self.tensors[1].to_sparse_coo(self.sparse_dim), - self.tensors[2].to_sparse_coo(self.sparse_dim))) + is_same_shape( + self.tensors[0].to_sparse_coo(self.sparse_dim), + self.tensors[1].to_sparse_coo(self.sparse_dim), + ) + ) + self.assertFalse( + is_same_shape( + self.tensors[0].to_sparse_coo(self.sparse_dim), + self.tensors[2].to_sparse_coo(self.sparse_dim), + ) + ) + self.assertFalse( + is_same_shape( + self.tensors[1].to_sparse_coo(self.sparse_dim), + self.tensors[2].to_sparse_coo(self.sparse_dim), + ) + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_sparse_matmul_op.py b/python/paddle/fluid/tests/unittests/test_sparse_matmul_op.py index 34a270b0fe504..368e9cbbd2ff2 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_matmul_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_matmul_op.py @@ -59,27 +59,33 @@ def check_result(self, x_shape, y_shape, format): sp_y.stop_gradient = False sp_out = paddle.sparse.matmul(sp_x, sp_y) - np.testing.assert_allclose(sp_out.numpy(), - dense_out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.numpy(), dense_out.numpy(), rtol=1e-05 + ) if get_cuda_version() >= 11030: dense_out.backward() sp_out.backward() - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - (dense_x.grad * mask).numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_y.grad.numpy(), - dense_y.grad.numpy(), - rtol=1e-05) - - @unittest.skipIf(not paddle.is_compiled_with_cuda() - or get_cuda_version() < 11000, "only support cuda>=11.0") + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + (dense_x.grad * mask).numpy(), + rtol=1e-05, + ) + np.testing.assert_allclose( + sp_y.grad.numpy(), dense_y.grad.numpy(), rtol=1e-05 + ) + + @unittest.skipIf( + not paddle.is_compiled_with_cuda() or get_cuda_version() < 11000, + "only support cuda>=11.0", + ) def test_matmul_2d(self): self.check_result([16, 12], [12, 10], 'coo') self.check_result([16, 12], [12, 10], 'csr') - @unittest.skipIf(not paddle.is_compiled_with_cuda() - or get_cuda_version() < 11070, "only support cuda>=11.7") + @unittest.skipIf( + not paddle.is_compiled_with_cuda() or get_cuda_version() < 11070, + "only support cuda>=11.7", + ) def test_matmul_3d(self): self.check_result([8, 16, 12], [8, 12, 10], 'coo') self.check_result([8, 16, 12], [8, 12, 10], 'csr') @@ -87,9 +93,10 @@ def test_matmul_3d(self): class TestMaskedMatmul(unittest.TestCase): # x: dense, y: dense, out: sparse_`csr - @unittest.skipIf(not paddle.is_compiled_with_cuda() - or get_cuda_version() < 11030, - "only support on cuda>=11.3") + @unittest.skipIf( + not paddle.is_compiled_with_cuda() or get_cuda_version() < 11030, + "only support on cuda>=11.3", + ) def test_masked_matmul_2d(self): np_mask = np.random.rand(10, 6) < 0.2 @@ -108,24 +115,25 @@ def test_masked_matmul_2d(self): mask = paddle.to_tensor(np.ones([10, 6]) * np_mask).to_sparse_csr() out = paddle.sparse.masked_matmul(x, y, mask) - np.testing.assert_allclose(np_out.indptr, - out.crows().numpy(), - rtol=1e-05) - np.testing.assert_allclose(np_out.indices, - out.cols().numpy(), - rtol=1e-05) - np.testing.assert_allclose(np_out.data, - out.values().numpy(), - rtol=1e-05) + np.testing.assert_allclose( + np_out.indptr, out.crows().numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + np_out.indices, out.cols().numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + np_out.data, out.values().numpy(), rtol=1e-05 + ) out.backward() np.testing.assert_allclose(out.is_sparse_csr(), True, rtol=1e-05) np.testing.assert_allclose(np_x_grad, x.grad.numpy(), rtol=1e-05) np.testing.assert_allclose(np_y_grad, y.grad.numpy(), rtol=1e-05) - @unittest.skipIf(not paddle.is_compiled_with_cuda() - or get_cuda_version() < 11070, - "only support on cuda>=11.7") + @unittest.skipIf( + not paddle.is_compiled_with_cuda() or get_cuda_version() < 11070, + "only support on cuda>=11.7", + ) def test_masked_matmul_3d(self): paddle.set_default_dtype('float32') origin_x = paddle.rand([16, 16, 12]) @@ -147,15 +155,17 @@ def test_masked_matmul_3d(self): sp_out = paddle.sparse.matmul(sp_x, sp_y) sp_out.backward() - np.testing.assert_allclose(sp_out.numpy(), - dense_out.numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - (dense_x.grad * mask).numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_y.grad.numpy(), - dense_y.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.numpy(), dense_out.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + (dense_x.grad * mask).numpy(), + rtol=1e-05, + ) + np.testing.assert_allclose( + sp_y.grad.numpy(), dense_y.grad.numpy(), rtol=1e-05 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_sparse_model.py b/python/paddle/fluid/tests/unittests/test_sparse_model.py index aa3d97919c4de..52f37e60dfbc9 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_model.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_model.py @@ -20,15 +20,16 @@ class TestGradientAdd(unittest.TestCase): - def sparse(self, sp_x): indentity = sp_x out = nn.functional.relu(sp_x) values = out.values() + indentity.values() - out = paddle.sparse.sparse_coo_tensor(out.indices(), - values, - shape=out.shape, - stop_gradient=out.stop_gradient) + out = paddle.sparse.sparse_coo_tensor( + out.indices(), + values, + shape=out.shape, + stop_gradient=out.stop_gradient, + ) return out def dense(self, x): diff --git a/python/paddle/fluid/tests/unittests/test_sparse_momentum_op.py b/python/paddle/fluid/tests/unittests/test_sparse_momentum_op.py index bcc555b9a3e8f..fefb31255247a 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_momentum_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_momentum_op.py @@ -17,16 +17,18 @@ from op_test import OpTest -def calculate_sparse_momentum_by_numpy(param, - grad, - mu, - velocity, - use_nesterov, - learning_rate, - index, - axis, - regularization_method=None, - regularization_coeff=1.0): +def calculate_sparse_momentum_by_numpy( + param, + grad, + mu, + velocity, + use_nesterov, + learning_rate, + index, + axis, + regularization_method=None, + regularization_coeff=1.0, +): sub_grad = grad.copy() grad = np.zeros_like(param) if axis == 0: @@ -48,8 +50,9 @@ def calculate_sparse_momentum_by_numpy(param, else: velocity_out = mu * velocity + grad if use_nesterov: - param_out = param - grad * learning_rate - \ - velocity_out * mu * learning_rate + param_out = ( + param - grad * learning_rate - velocity_out * mu * learning_rate + ) else: param_out = param - learning_rate * velocity_out @@ -57,7 +60,6 @@ def calculate_sparse_momentum_by_numpy(param, class TestSparseMomentumOp(OpTest): - def setUp(self): self.op_type = "sparse_momentum" self.dtype = np.float32 @@ -75,24 +77,31 @@ def setUp(self): if self.multi_precision: assert self.dtype == np.float16 - param = np.random.random( - (self.batch_size, self.num_classes)).astype(self.dtype) - grad = np.random.random( - (self.batch_size, self.num_classes)).astype(self.dtype) + param = np.random.random((self.batch_size, self.num_classes)).astype( + self.dtype + ) + grad = np.random.random((self.batch_size, self.num_classes)).astype( + self.dtype + ) if self.axis == 0: - index = np.random.randint(0, - self.batch_size, - size=(self.batch_size // 2, ), - dtype=self.index_dtype) + index = np.random.randint( + 0, + self.batch_size, + size=(self.batch_size // 2,), + dtype=self.index_dtype, + ) grad = grad[index] else: - index = np.random.randint(0, - self.num_classes, - size=(self.num_classes // 2, ), - dtype=self.index_dtype) + index = np.random.randint( + 0, + self.num_classes, + size=(self.num_classes // 2,), + dtype=self.index_dtype, + ) grad = grad[:, index] - velocity = np.random.random( - (self.batch_size, self.num_classes)).astype(self.dtype) + velocity = np.random.random((self.batch_size, self.num_classes)).astype( + self.dtype + ) learning_rate = np.array([0.001]).astype(self.dtype) mu = 0.9 @@ -109,7 +118,8 @@ def setUp(self): regularization_method=regularization_method, regularization_coeff=regularization_coeff, index=index, - axis=self.axis) + axis=self.axis, + ) self.attrs = { 'mu': mu, @@ -121,33 +131,35 @@ def setUp(self): } self.inputs = { - 'Param': - param.astype("float16") if self.multi_precision else param, - 'Velocity': - velocity.astype("float32") if self.multi_precision else velocity, - 'LearningRate': - learning_rate.astype("float32") - if self.multi_precision else learning_rate, - 'Grad': - grad.astype("float16") if self.multi_precision else grad, - 'Index': - index, - 'Axis': - np.array(self.axis).astype(np.int32), + 'Param': param.astype("float16") if self.multi_precision else param, + 'Velocity': velocity.astype("float32") + if self.multi_precision + else velocity, + 'LearningRate': learning_rate.astype("float32") + if self.multi_precision + else learning_rate, + 'Grad': grad.astype("float16") if self.multi_precision else grad, + 'Index': index, + 'Axis': np.array(self.axis).astype(np.int32), } self.outputs = { - 'ParamOut': - param_out.astype("float16") if self.multi_precision else param_out, - 'VelocityOut': - velocity_out.astype("float32") - if self.multi_precision else velocity_out, + 'ParamOut': param_out.astype("float16") + if self.multi_precision + else param_out, + 'VelocityOut': velocity_out.astype("float32") + if self.multi_precision + else velocity_out, } if self.multi_precision: - self.inputs['MasterParam'] = param.astype( - "float32") if self.multi_precision else param - self.outputs['MasterParamOut'] = param_out.astype( - "float32") if self.multi_precision else param_out + self.inputs['MasterParam'] = ( + param.astype("float32") if self.multi_precision else param + ) + self.outputs['MasterParamOut'] = ( + param_out.astype("float32") + if self.multi_precision + else param_out + ) def init_dtype(self): pass @@ -162,45 +174,40 @@ def init_use_nesterov(self): pass def test_check_output(self): - self.check_output(atol=5e-3 if self.multi_precision else 1e-5, - check_eager=True) + self.check_output( + atol=5e-3 if self.multi_precision else 1e-5, check_eager=True + ) class TestSparseMomentumOpDtype1(TestSparseMomentumOp): - def init_dtype(self): self.dtype = np.float32 self.index_dtype = np.int64 class TestSparseMomentumOpDtype2(TestSparseMomentumOp): - def init_dtype(self): self.dtype = np.float64 self.index_dtype = np.int32 class TestSparseMomentumOpDtype3(TestSparseMomentumOp): - def init_dtype(self): self.dtype = np.float64 self.index_dtype = np.int64 class TestSparseMomentumOpAxis(TestSparseMomentumOp): - def init_axis(self): self.axis = 1 class TestSparseMomentumOpNesterov(TestSparseMomentumOp): - def init_use_nesterov(self): self.use_nesterov = True class TestSparseMomentumOpMultiPrecision(TestSparseMomentumOp): - def init_dtype(self): self.dtype = np.float16 self.index_dtype = np.int32 @@ -213,7 +220,6 @@ def init_use_nesterov(self): class TestSparseMomentumOpMultiPrecision1(TestSparseMomentumOp): - def init_dtype(self): self.dtype = np.float16 self.index_dtype = np.int64 @@ -226,7 +232,6 @@ def init_use_nesterov(self): class TestSparseMomentumOpMultiPrecision2(TestSparseMomentumOp): - def init_dtype(self): self.dtype = np.float16 self.index_dtype = np.int32 @@ -239,7 +244,6 @@ def init_use_nesterov(self): class TestSparseMomentumOpMultiPrecision3(TestSparseMomentumOp): - def init_dtype(self): self.dtype = np.float16 self.index_dtype = np.int64 diff --git a/python/paddle/fluid/tests/unittests/test_sparse_mv_op.py b/python/paddle/fluid/tests/unittests/test_sparse_mv_op.py index e532063ad8faf..ae04ddc7a487a 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_mv_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_mv_op.py @@ -37,7 +37,8 @@ def get_cuda_version(): @unittest.skipIf( not paddle.is_compiled_with_cuda() or get_cuda_version() < 11000, - "paddle is not compiled with CUDA and cuda version need to >= 11.0") + "paddle is not compiled with CUDA and cuda version need to >= 11.0", +) class TestCsrMv(unittest.TestCase): # x: csr-matrix, y: dense-vec, out: dense-vec def test_mv(self): @@ -62,20 +63,23 @@ def test_mv(self): sp_out = paddle.sparse.mv(sp_x, sp_vec) sp_out.backward() - np.testing.assert_allclose(sp_out.numpy(), - dense_out.numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - (dense_x.grad * mask).numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_vec.grad.numpy(), - dense_vec.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.numpy(), dense_out.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + (dense_x.grad * mask).numpy(), + rtol=1e-05, + ) + np.testing.assert_allclose( + sp_vec.grad.numpy(), dense_vec.grad.numpy(), rtol=1e-05 + ) @unittest.skipIf( not paddle.is_compiled_with_cuda() or get_cuda_version() < 11000, - "paddle is not compiled with CUDA and cuda version need to >= 11.0") + "paddle is not compiled with CUDA and cuda version need to >= 11.0", +) class TestCooMv(unittest.TestCase): # x: csr-matrix, y: dense-vec, out: dense-vec def test_mv(self): @@ -100,15 +104,17 @@ def test_mv(self): sp_out = paddle.sparse.mv(sp_x, sp_vec) sp_out.backward() - np.testing.assert_allclose(sp_out.numpy(), - dense_out.numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - (dense_x.grad * mask).numpy(), - rtol=1e-05) - np.testing.assert_allclose(sp_vec.grad.numpy(), - dense_vec.grad.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.numpy(), dense_out.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + (dense_x.grad * mask).numpy(), + rtol=1e-05, + ) + np.testing.assert_allclose( + sp_vec.grad.numpy(), dense_vec.grad.numpy(), rtol=1e-05 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_sparse_norm_op.py b/python/paddle/fluid/tests/unittests/test_sparse_norm_op.py index 503f171bdc3e9..4f48b08e9f8cb 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_norm_op.py @@ -22,13 +22,12 @@ class TestSparseBatchNorm(unittest.TestCase): - def test(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) paddle.seed(0) channels = 4 shape = [2, 3, 6, 6, channels] - #there is no zero in dense_x + # there is no zero in dense_x dense_x = paddle.randn(shape) dense_x.stop_gradient = False @@ -48,17 +47,21 @@ def test(self): sparse_y = sparse_batch_norm(sparse_x) # compare the result with dense batch_norm - assert np.allclose(dense_y.flatten().numpy(), - sparse_y.values().flatten().numpy(), - atol=1e-5, - rtol=1e-5) + assert np.allclose( + dense_y.flatten().numpy(), + sparse_y.values().flatten().numpy(), + atol=1e-5, + rtol=1e-5, + ) # test backward sparse_y.backward(sparse_y) - assert np.allclose(dense_x.grad.flatten().numpy(), - sparse_x.grad.values().flatten().numpy(), - atol=1e-5, - rtol=1e-5) + assert np.allclose( + dense_x.grad.flatten().numpy(), + sparse_x.grad.values().flatten().numpy(), + atol=1e-5, + rtol=1e-5, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_error_layout(self): @@ -66,8 +69,9 @@ def test_error_layout(self): shape = [2, 3, 6, 6, 3] x = paddle.randn(shape) sparse_x = x.to_sparse_coo(4) - sparse_batch_norm = paddle.sparse.nn.BatchNorm(3, - data_format='NCDHW') + sparse_batch_norm = paddle.sparse.nn.BatchNorm( + 3, data_format='NCDHW' + ) sparse_batch_norm(sparse_x) def test2(self): @@ -86,10 +90,10 @@ def test2(self): class TestSyncBatchNorm(unittest.TestCase): - def test_sync_batch_norm(self): - x = np.array([[[[0.3, 0.4], [0.3, 0.07]], - [[0.83, 0.37], [0.18, 0.93]]]]).astype('float32') + x = np.array( + [[[[0.3, 0.4], [0.3, 0.07]], [[0.83, 0.37], [0.18, 0.93]]]] + ).astype('float32') x = paddle.to_tensor(x) sparse_x = x.to_sparse_coo(len(x.shape) - 1) @@ -100,18 +104,24 @@ def test_sync_batch_norm(self): dense_sync_bn = paddle.nn.SyncBatchNorm(2) x = x.reshape((-1, x.shape[-1])) dense_hidden = dense_sync_bn(x) - assert np.allclose(sparse_hidden.values().numpy(), - dense_hidden.numpy()) + assert np.allclose( + sparse_hidden.values().numpy(), dense_hidden.numpy() + ) def test_convert(self): - base_model = paddle.nn.Sequential(nn.Conv3D(3, 5, 3), nn.BatchNorm(5), - nn.BatchNorm(5)) + base_model = paddle.nn.Sequential( + nn.Conv3D(3, 5, 3), nn.BatchNorm(5), nn.BatchNorm(5) + ) model = paddle.nn.Sequential( - nn.Conv3D(3, 5, 3), nn.BatchNorm(5), - nn.BatchNorm(5, - weight_attr=fluid.ParamAttr(name='bn.scale'), - bias_attr=fluid.ParamAttr(name='bn.bias'))) + nn.Conv3D(3, 5, 3), + nn.BatchNorm(5), + nn.BatchNorm( + 5, + weight_attr=fluid.ParamAttr(name='bn.scale'), + bias_attr=fluid.ParamAttr(name='bn.bias'), + ), + ) model = nn.SyncBatchNorm.convert_sync_batchnorm(model) for idx, sublayer in enumerate(base_model.sublayers()): if isinstance(sublayer, nn.BatchNorm): @@ -119,15 +129,14 @@ def test_convert(self): class TestStatic(unittest.TestCase): - def test(self): paddle.enable_static() - indices = paddle.static.data(name='indices', - shape=[4, 4], - dtype='int32') - values = paddle.static.data(name='values', - shape=[4, 1], - dtype='float32') + indices = paddle.static.data( + name='indices', shape=[4, 4], dtype='int32' + ) + values = paddle.static.data( + name='values', shape=[4, 1], dtype='float32' + ) channels = 1 dense_shape = [1, 1, 3, 4, channels] sp_x = sparse.sparse_coo_tensor(indices, values, dense_shape) @@ -144,20 +153,29 @@ def test(self): mean_data = np.array([1.0]).astype('float32') variance_data = np.array([2.0]).astype('float32') - fetch = exe.run(feed={ - 'indices': indices_data, - 'values': values_data, - 'batch_norm_0.b_0': bias_data, - 'batch_norm_0.w_0': weight_data, - 'batch_norm_0.w_1': mean_data, - 'batch_norm_0.w_2': variance_data - }, - fetch_list=[out], - return_numpy=True) - correct_out = np.array([[[[[0.0], [-1.6832708], [0.0], [0.1055764]], - [[0.0], [0.0], [1.8944236], [0.0]], - [[0.0], [0.0], [0.0], - [3.683271]]]]]).astype('float32') + fetch = exe.run( + feed={ + 'indices': indices_data, + 'values': values_data, + 'batch_norm_0.b_0': bias_data, + 'batch_norm_0.w_0': weight_data, + 'batch_norm_0.w_1': mean_data, + 'batch_norm_0.w_2': variance_data, + }, + fetch_list=[out], + return_numpy=True, + ) + correct_out = np.array( + [ + [ + [ + [[0.0], [-1.6832708], [0.0], [0.1055764]], + [[0.0], [0.0], [1.8944236], [0.0]], + [[0.0], [0.0], [0.0], [3.683271]], + ] + ] + ] + ).astype('float32') np.testing.assert_allclose(correct_out, fetch[0], rtol=1e-5) paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_sparse_pooling_op.py b/python/paddle/fluid/tests/unittests/test_sparse_pooling_op.py index 998b5be9bb180..5425ebb6ea98d 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_pooling_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_pooling_op.py @@ -20,7 +20,6 @@ class TestMaxPool3DFunc(unittest.TestCase): - def setInput(self): paddle.seed(0) self.dense_x = paddle.randn((1, 4, 4, 4, 4)) @@ -49,31 +48,32 @@ def test(self): sparse_x, self.kernel_sizes, stride=self.strides, - padding=self.paddings) + padding=self.paddings, + ) out = sparse_out.to_dense() out.backward(out) dense_x = copy.deepcopy(self.dense_x) - dense_out = paddle.nn.functional.max_pool3d(dense_x, - self.kernel_sizes, - stride=self.strides, - padding=self.paddings, - data_format='NDHWC') + dense_out = paddle.nn.functional.max_pool3d( + dense_x, + self.kernel_sizes, + stride=self.strides, + padding=self.paddings, + data_format='NDHWC', + ) dense_out.backward(dense_out) - #compare with dense + # compare with dense assert np.allclose(dense_out.numpy(), out.numpy()) assert np.allclose(dense_x.grad.numpy(), self.dense_x.grad.numpy()) class TestStride(TestMaxPool3DFunc): - def setStride(self): self.strides = 1 class TestPadding(TestMaxPool3DFunc): - def setPadding(self): self.paddings = 1 @@ -82,7 +82,6 @@ def setInput(self): class TestKernelSize(TestMaxPool3DFunc): - def setKernelSize(self): self.kernel_sizes = [5, 5, 5] @@ -92,7 +91,6 @@ def setInput(self): class TestInput(TestMaxPool3DFunc): - def setInput(self): paddle.seed(0) self.dense_x = paddle.randn((2, 6, 7, 9, 3)) @@ -101,19 +99,19 @@ def setInput(self): class TestMaxPool3DAPI(unittest.TestCase): - def test(self): with _test_eager_guard(): dense_x = paddle.randn((2, 3, 6, 6, 3)) sparse_x = dense_x.to_sparse_coo(4) - max_pool3d = paddle.sparse.nn.MaxPool3D(kernel_size=3, - data_format='NDHWC') + max_pool3d = paddle.sparse.nn.MaxPool3D( + kernel_size=3, data_format='NDHWC' + ) out = max_pool3d(sparse_x) out = out.to_dense() - dense_out = paddle.nn.functional.max_pool3d(dense_x, - 3, - data_format='NDHWC') + dense_out = paddle.nn.functional.max_pool3d( + dense_x, 3, data_format='NDHWC' + ) assert np.allclose(dense_out.numpy(), out.numpy()) diff --git a/python/paddle/fluid/tests/unittests/test_sparse_reshape_op.py b/python/paddle/fluid/tests/unittests/test_sparse_reshape_op.py index 01619f76dc948..fd748c4dcb2c9 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_reshape_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_reshape_op.py @@ -42,25 +42,27 @@ def check_result(self, x_shape, new_shape, format): dense_out = paddle.reshape(dense_x, new_shape) if format == "coo": - sp_x = paddle.to_tensor(np_x, - place=paddle.CPUPlace()).to_sparse_coo( - len(x_shape)) + sp_x = paddle.to_tensor( + np_x, place=paddle.CPUPlace() + ).to_sparse_coo(len(x_shape)) else: - sp_x = paddle.to_tensor(np_x, - place=paddle.CPUPlace()).to_sparse_csr() + sp_x = paddle.to_tensor( + np_x, place=paddle.CPUPlace() + ).to_sparse_csr() sp_x.stop_gradient = False sp_out = paddle.sparse.reshape(sp_x, new_shape) - np.testing.assert_allclose(sp_out.to_dense().numpy(), - dense_out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.to_dense().numpy(), dense_out.numpy(), rtol=1e-05 + ) dense_out.backward() sp_out.backward() - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - dense_x.grad.numpy() * - np_x.astype('bool').astype('int'), - rtol=1e-05) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + dense_x.grad.numpy() * np_x.astype('bool').astype('int'), + rtol=1e-05, + ) # check gpu kernel if paddle.device.is_compiled_with_cuda(): @@ -70,28 +72,35 @@ def check_result(self, x_shape, new_shape, format): if format == "coo": sp_x = paddle.to_tensor( - np_x, place=paddle.CUDAPlace(0)).to_sparse_coo(len(x_shape)) + np_x, place=paddle.CUDAPlace(0) + ).to_sparse_coo(len(x_shape)) else: sp_x = paddle.to_tensor( - np_x, place=paddle.CUDAPlace(0)).to_sparse_csr() + np_x, place=paddle.CUDAPlace(0) + ).to_sparse_csr() sp_x.stop_gradient = False sp_out = paddle.sparse.reshape(sp_x, new_shape) - np.testing.assert_allclose(sp_out.to_dense().numpy(), - dense_out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.to_dense().numpy(), dense_out.numpy(), rtol=1e-05 + ) dense_out.backward() sp_out.backward() - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - dense_x.grad.numpy() * - np_x.astype('bool').astype('int'), - rtol=1e-05) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + dense_x.grad.numpy() * np_x.astype('bool').astype('int'), + rtol=1e-05, + ) def test_reshape_2d(self): - self.check_result([2, 5], [ - 10, - ], 'coo') + self.check_result( + [2, 5], + [ + 10, + ], + 'coo', + ) self.check_result([12, 5], [15, 4], 'coo') self.check_result([10, 5], [2, 25], 'csr') diff --git a/python/paddle/fluid/tests/unittests/test_sparse_softmax_op.py b/python/paddle/fluid/tests/unittests/test_sparse_softmax_op.py index 251f6cce1cec6..ef1f672047fb3 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_softmax_op.py @@ -23,7 +23,6 @@ class TestCsrSoftmax(unittest.TestCase): - def test_softmax2d(self): with _test_eager_guard(): mask = np.random.rand(16, 128) < 0.5 @@ -46,12 +45,12 @@ def test_softmax2d(self): csr = paddle.to_tensor(np_x, stop_gradient=False).to_sparse_csr() m = paddle.sparse.nn.Softmax() out = m(csr) - np.testing.assert_allclose(out.crows().numpy(), - np_csr.indptr, - rtol=1e-05) - np.testing.assert_allclose(out.cols().numpy(), - np_csr.indices, - rtol=1e-05) + np.testing.assert_allclose( + out.crows().numpy(), np_csr.indptr, rtol=1e-05 + ) + np.testing.assert_allclose( + out.cols().numpy(), np_csr.indices, rtol=1e-05 + ) np.testing.assert_allclose(out.values().numpy(), np_out, rtol=1e-05) # dx = (dout - sum(dout * out)) * out, dout=rand_x @@ -67,15 +66,15 @@ def test_softmax2d(self): sum = np.sum(dout * out, keepdims=True) dx = np.concatenate([dx, (dout - sum) * out]) - np.testing.assert_allclose(csr.grad.crows().numpy(), - np_csr.indptr, - rtol=1e-05) - np.testing.assert_allclose(csr.grad.cols().numpy(), - np_csr.indices, - rtol=1e-05) - np.testing.assert_allclose(csr.grad.values().numpy(), - dx, - rtol=1e-05) + np.testing.assert_allclose( + csr.grad.crows().numpy(), np_csr.indptr, rtol=1e-05 + ) + np.testing.assert_allclose( + csr.grad.cols().numpy(), np_csr.indices, rtol=1e-05 + ) + np.testing.assert_allclose( + csr.grad.values().numpy(), dx, rtol=1e-05 + ) def test_softmax3d(self): with _test_eager_guard(): @@ -88,7 +87,9 @@ def test_softmax3d(self): for i in range(batchNum): np_csr = sp.csr_matrix(np_x[i, :, :]) row_number = np_csr.shape[0] - for j in range(row_number, ): + for j in range( + row_number, + ): start = np_csr.indptr[j] end = np_csr.indptr[j + 1] if start == end: @@ -118,15 +119,15 @@ def test_softmax3d(self): if start == end: continue dout = np_csr.data[start:end] - out = np_out[batch_offset + start:batch_offset + end] + out = np_out[batch_offset + start : batch_offset + end] sum = np.sum(dout * out, keepdims=True) dx = np.concatenate([dx, (dout - sum) * out]) batch_offset += np_csr.nnz - np.testing.assert_allclose(csr.grad.values().numpy(), - dx, - rtol=1e-05) + np.testing.assert_allclose( + csr.grad.values().numpy(), dx, rtol=1e-05 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_sparse_transpose_op.py b/python/paddle/fluid/tests/unittests/test_sparse_transpose_op.py index 12d1fd9a8b770..06f221bfe6c81 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_transpose_op.py @@ -37,14 +37,16 @@ def check_result(self, x_shape, dims, format): sp_x.stop_gradient = False sp_out = paddle.sparse.transpose(sp_x, dims) - np.testing.assert_allclose(sp_out.to_dense().numpy(), - dense_out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.to_dense().numpy(), dense_out.numpy(), rtol=1e-05 + ) dense_out.backward() sp_out.backward() - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - (dense_x.grad * mask).numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), + (dense_x.grad * mask).numpy(), + rtol=1e-05, + ) def test_transpose_2d(self): self.check_result([2, 5], [0, 1], 'coo') @@ -69,8 +71,9 @@ def test_transpose_3d(self): def test_transpose_nd(self): self.check_result([8, 3, 4, 4, 5, 3], [5, 3, 4, 1, 0, 2], 'coo') # Randint now only supports access to dimension 0 to 9. - self.check_result([2, 3, 4, 2, 3, 4, 2, 3, 4], - [2, 3, 4, 5, 6, 7, 8, 0, 1], 'coo') + self.check_result( + [2, 3, 4, 2, 3, 4, 2, 3, 4], [2, 3, 4, 5, 6, 7, 8, 0, 1], 'coo' + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_sparse_unary_op.py b/python/paddle/fluid/tests/unittests/test_sparse_unary_op.py index fc39ab973c009..dd50608050e77 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_unary_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_unary_op.py @@ -19,7 +19,6 @@ class TestSparseUnary(unittest.TestCase): - def to_sparse(self, x, format): if format == 'coo': return x.detach().to_sparse_coo(sparse_dim=x.ndim) @@ -63,18 +62,18 @@ def check_result(self, dense_func, sparse_func, format, *args): dense_out.backward() # compare forward - np.testing.assert_allclose(sp_out.to_dense().numpy(), - dense_out.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + sp_out.to_dense().numpy(), dense_out.numpy(), rtol=1e-05 + ) # compare backward if dense_func == paddle.sqrt: - expect_grad = np.nan_to_num(dense_x.grad.numpy(), 0., 0., 0.) + expect_grad = np.nan_to_num(dense_x.grad.numpy(), 0.0, 0.0, 0.0) else: expect_grad = (dense_x.grad * mask).numpy() - np.testing.assert_allclose(sp_x.grad.to_dense().numpy(), - expect_grad, - rtol=1e-05) + np.testing.assert_allclose( + sp_x.grad.to_dense().numpy(), expect_grad, rtol=1e-05 + ) def compare_with_dense(self, dense_func, sparse_func): self.check_result(dense_func, sparse_func, 'coo') @@ -84,8 +83,9 @@ def compare_with_dense_one_attr(self, dense_func, sparse_func, attr1): self.check_result(dense_func, sparse_func, 'coo', attr1) self.check_result(dense_func, sparse_func, 'csr', attr1) - def compare_with_dense_two_attr(self, dense_func, sparse_func, attr1, - attr2): + def compare_with_dense_two_attr( + self, dense_func, sparse_func, attr1, attr2 + ): self.check_result(dense_func, sparse_func, 'coo', attr1, attr2) self.check_result(dense_func, sparse_func, 'csr', attr1, attr2) @@ -129,8 +129,9 @@ def test_sparse_relu6(self): self.compare_with_dense(paddle.nn.ReLU6(), paddle.sparse.nn.ReLU6()) def test_sparse_leaky_relu(self): - self.compare_with_dense(paddle.nn.LeakyReLU(0.1), - paddle.sparse.nn.LeakyReLU(0.1)) + self.compare_with_dense( + paddle.nn.LeakyReLU(0.1), paddle.sparse.nn.LeakyReLU(0.1) + ) def test_sparse_abs(self): self.compare_with_dense(paddle.abs, paddle.sparse.abs) @@ -151,18 +152,22 @@ def test_sparse_pow(self): self.compare_with_dense_one_attr(paddle.pow, paddle.sparse.pow, 3) def test_sparse_mul_scalar(self): - self.compare_with_dense_one_attr(paddle.Tensor.__mul__, - paddle.sparse.multiply, 3) + self.compare_with_dense_one_attr( + paddle.Tensor.__mul__, paddle.sparse.multiply, 3 + ) def test_sparse_div_scalar(self): - self.compare_with_dense_one_attr(paddle.Tensor.__div__, - paddle.sparse.divide, 2) + self.compare_with_dense_one_attr( + paddle.Tensor.__div__, paddle.sparse.divide, 2 + ) def test_sparse_cast(self): - self.compare_with_dense_two_attr(paddle.cast, paddle.sparse.cast, - 'int32', 'float32') - self.compare_with_dense_two_attr(paddle.cast, paddle.sparse.cast, - 'int32', 'float64') + self.compare_with_dense_two_attr( + paddle.cast, paddle.sparse.cast, 'int32', 'float32' + ) + self.compare_with_dense_two_attr( + paddle.cast, paddle.sparse.cast, 'int32', 'float64' + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py b/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py index 700c0795a0d1f..8659491da2e9a 100644 --- a/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py +++ b/python/paddle/fluid/tests/unittests/test_sparse_utils_op.py @@ -23,7 +23,6 @@ class TestSparseCreate(unittest.TestCase): - def test_create_coo_by_tensor(self): with _test_eager_guard(): indices = [[0, 0, 1, 2, 2], [1, 3, 2, 0, 1]] @@ -31,10 +30,9 @@ def test_create_coo_by_tensor(self): dense_shape = [3, 4] dense_indices = paddle.to_tensor(indices) dense_elements = paddle.to_tensor(values, dtype='float32') - coo = paddle.sparse.sparse_coo_tensor(dense_indices, - dense_elements, - dense_shape, - stop_gradient=False) + coo = paddle.sparse.sparse_coo_tensor( + dense_indices, dense_elements, dense_shape, stop_gradient=False + ) # test the to_string.py assert np.array_equal(indices, coo.indices().numpy()) assert np.array_equal(values, coo.values().numpy()) @@ -59,11 +57,13 @@ def test_create_csr_by_tensor(self): dense_cols = paddle.to_tensor(cols) dense_elements = paddle.to_tensor(values, dtype='float32') stop_gradient = False - csr = paddle.sparse.sparse_csr_tensor(dense_crows, - dense_cols, - dense_elements, - dense_shape, - stop_gradient=stop_gradient) + csr = paddle.sparse.sparse_csr_tensor( + dense_crows, + dense_cols, + dense_elements, + dense_shape, + stop_gradient=stop_gradient, + ) def test_create_csr_by_np(self): with _test_eager_guard(): @@ -71,8 +71,9 @@ def test_create_csr_by_np(self): cols = [1, 3, 2, 0, 1] values = [1, 2, 3, 4, 5] dense_shape = [3, 4] - csr = paddle.sparse.sparse_csr_tensor(crows, cols, values, - dense_shape) + csr = paddle.sparse.sparse_csr_tensor( + crows, cols, values, dense_shape + ) # test the to_string.py assert np.array_equal(5, csr.nnz()) assert np.array_equal(crows, csr.crows().numpy()) @@ -85,10 +86,9 @@ def test_place(self): indices = [[0, 1], [0, 1]] values = [1.0, 2.0] dense_shape = [2, 2] - coo = paddle.sparse.sparse_coo_tensor(indices, - values, - dense_shape, - place=place) + coo = paddle.sparse.sparse_coo_tensor( + indices, values, dense_shape, place=place + ) assert coo.place.is_cpu_place() assert coo.values().place.is_cpu_place() assert coo.indices().place.is_cpu_place() @@ -96,10 +96,9 @@ def test_place(self): crows = [0, 2, 3, 5] cols = [1, 3, 2, 0, 1] values = [1.0, 2.0, 3.0, 4.0, 5.0] - csr = paddle.sparse.sparse_csr_tensor(crows, - cols, - values, [3, 5], - place=place) + csr = paddle.sparse.sparse_csr_tensor( + crows, cols, values, [3, 5], place=place + ) assert csr.place.is_cpu_place() assert csr.crows().place.is_cpu_place() assert csr.cols().place.is_cpu_place() @@ -112,19 +111,17 @@ def test_dtype(self): dense_shape = [2, 2] indices = paddle.to_tensor(indices, dtype='int32') values = paddle.to_tensor(values, dtype='float32') - coo = paddle.sparse.sparse_coo_tensor(indices, - values, - dense_shape, - dtype='float64') + coo = paddle.sparse.sparse_coo_tensor( + indices, values, dense_shape, dtype='float64' + ) assert coo.dtype == paddle.float64 crows = [0, 2, 3, 5] cols = [1, 3, 2, 0, 1] values = [1.0, 2.0, 3.0, 4.0, 5.0] - csr = paddle.sparse.sparse_csr_tensor(crows, - cols, - values, [3, 5], - dtype='float16') + csr = paddle.sparse.sparse_csr_tensor( + crows, cols, values, [3, 5], dtype='float16' + ) assert csr.dtype == paddle.float16 def test_create_coo_no_shape(self): @@ -138,7 +135,6 @@ def test_create_coo_no_shape(self): class TestSparseConvert(unittest.TestCase): - def test_to_sparse_coo(self): with _test_eager_guard(): x = [[0, 1, 0, 2], [0, 0, 3, 0], [4, 5, 0, 0]] @@ -148,17 +144,19 @@ def test_to_sparse_coo(self): out = dense_x.to_sparse_coo(2) assert np.array_equal(out.indices().numpy(), indices) assert np.array_equal(out.values().numpy(), values) - #test to_sparse_coo_grad backward + # test to_sparse_coo_grad backward out_grad_indices = [[0, 1], [0, 1]] out_grad_values = [2.0, 3.0] out_grad = paddle.sparse.sparse_coo_tensor( paddle.to_tensor(out_grad_indices), paddle.to_tensor(out_grad_values), shape=out.shape, - stop_gradient=True) + stop_gradient=True, + ) out.backward(out_grad) - assert np.array_equal(dense_x.grad.numpy(), - out_grad.to_dense().numpy()) + assert np.array_equal( + dense_x.grad.numpy(), out_grad.to_dense().numpy() + ) def test_coo_to_dense(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -171,27 +169,34 @@ def test_coo_to_dense(self): paddle.to_tensor(indices, dtype=indices_dtype), paddle.to_tensor(values), shape=[3, 4], - stop_gradient=False) + stop_gradient=False, + ) dense_tensor = sparse_x.to_dense() - #test to_dense_grad backward - out_grad = [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [9.0, 10.0, 11.0, 12.0]] + # test to_dense_grad backward + out_grad = [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ] dense_tensor.backward(paddle.to_tensor(out_grad)) - #mask the out_grad by sparse_x.indices() + # mask the out_grad by sparse_x.indices() correct_x_grad = [2.0, 4.0, 7.0, 9.0, 10.0] - assert np.array_equal(correct_x_grad, - sparse_x.grad.values().numpy()) + assert np.array_equal( + correct_x_grad, sparse_x.grad.values().numpy() + ) paddle.device.set_device("cpu") sparse_x_cpu = paddle.sparse.sparse_coo_tensor( paddle.to_tensor(indices, dtype=indices_dtype), paddle.to_tensor(values), shape=[3, 4], - stop_gradient=False) + stop_gradient=False, + ) dense_tensor_cpu = sparse_x_cpu.to_dense() dense_tensor_cpu.backward(paddle.to_tensor(out_grad)) - assert np.array_equal(correct_x_grad, - sparse_x_cpu.grad.values().numpy()) + assert np.array_equal( + correct_x_grad, sparse_x_cpu.grad.values().numpy() + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_to_sparse_csr(self): @@ -218,23 +223,35 @@ def test_coo_values_grad(self): paddle.to_tensor(indices), paddle.to_tensor(values), shape=[3, 4], - stop_gradient=False) + stop_gradient=False, + ) values_tensor = sparse_x.values() out_grad = [2.0, 3.0, 5.0, 8.0, 9.0] # test coo_values_grad values_tensor.backward(paddle.to_tensor(out_grad)) assert np.array_equal(out_grad, sparse_x.grad.values().numpy()) indices = [[0, 0, 1, 2, 2], [1, 3, 2, 0, 1]] - values = [[1.0, 1.0], [2.0, 2.0], [3.0, 3.0], [4.0, 4.0], - [5.0, 5.0]] + values = [ + [1.0, 1.0], + [2.0, 2.0], + [3.0, 3.0], + [4.0, 4.0], + [5.0, 5.0], + ] sparse_x = paddle.sparse.sparse_coo_tensor( paddle.to_tensor(indices), paddle.to_tensor(values), shape=[3, 4, 2], - stop_gradient=False) + stop_gradient=False, + ) values_tensor = sparse_x.values() - out_grad = [[2.0, 2.0], [3.0, 3.0], [5.0, 5.0], [8.0, 8.0], - [9.0, 9.0]] + out_grad = [ + [2.0, 2.0], + [3.0, 3.0], + [5.0, 5.0], + [8.0, 8.0], + [9.0, 9.0], + ] # test coo_values_grad values_tensor.backward(paddle.to_tensor(out_grad)) assert np.array_equal(out_grad, sparse_x.grad.values().numpy()) @@ -243,51 +260,59 @@ def test_coo_values_grad(self): def test_sparse_coo_tensor_grad(self): with _test_eager_guard(): for device in devices: - if device == 'cpu' or (device == 'gpu' - and paddle.is_compiled_with_cuda()): + if device == 'cpu' or ( + device == 'gpu' and paddle.is_compiled_with_cuda() + ): paddle.device.set_device(device) indices = [[0, 1], [0, 1]] values = [1, 2] indices = paddle.to_tensor(indices, dtype='int32') - values = paddle.to_tensor(values, - dtype='float32', - stop_gradient=False) + values = paddle.to_tensor( + values, dtype='float32', stop_gradient=False + ) sparse_x = paddle.sparse.sparse_coo_tensor( - indices, values, shape=[2, 2], stop_gradient=False) + indices, values, shape=[2, 2], stop_gradient=False + ) grad_indices = [[0, 1], [1, 1]] grad_values = [2, 3] grad_indices = paddle.to_tensor(grad_indices, dtype='int32') grad_values = paddle.to_tensor(grad_values, dtype='float32') sparse_out_grad = paddle.sparse.sparse_coo_tensor( - grad_indices, grad_values, shape=[2, 2]) + grad_indices, grad_values, shape=[2, 2] + ) sparse_x.backward(sparse_out_grad) correct_values_grad = [0, 3] - assert np.array_equal(correct_values_grad, - values.grad.numpy()) + assert np.array_equal( + correct_values_grad, values.grad.numpy() + ) # test the non-zero values is a vector values = [[1, 1], [2, 2]] - values = paddle.to_tensor(values, - dtype='float32', - stop_gradient=False) + values = paddle.to_tensor( + values, dtype='float32', stop_gradient=False + ) sparse_x = paddle.sparse.sparse_coo_tensor( - indices, values, shape=[2, 2, 2], stop_gradient=False) + indices, values, shape=[2, 2, 2], stop_gradient=False + ) grad_values = [[2, 2], [3, 3]] grad_values = paddle.to_tensor(grad_values, dtype='float32') sparse_out_grad = paddle.sparse.sparse_coo_tensor( - grad_indices, grad_values, shape=[2, 2, 2]) + grad_indices, grad_values, shape=[2, 2, 2] + ) sparse_x.backward(sparse_out_grad) correct_values_grad = [[0, 0], [3, 3]] - assert np.array_equal(correct_values_grad, - values.grad.numpy()) + assert np.array_equal( + correct_values_grad, values.grad.numpy() + ) def test_sparse_coo_tensor_sorted(self): with _test_eager_guard(): for device in devices: - if device == 'cpu' or (device == 'gpu' - and paddle.is_compiled_with_cuda()): + if device == 'cpu' or ( + device == 'gpu' and paddle.is_compiled_with_cuda() + ): paddle.device.set_device(device) - #test unsorted and duplicate indices + # test unsorted and duplicate indices indices = [[1, 0, 0], [0, 1, 1]] values = [1.0, 2.0, 3.0] indices = paddle.to_tensor(indices, dtype='int32') @@ -296,10 +321,12 @@ def test_sparse_coo_tensor_sorted(self): sparse_x = paddle.sparse.coalesce(sparse_x) indices_sorted = [[0, 1], [1, 0]] values_sorted = [5.0, 1.0] - assert np.array_equal(indices_sorted, - sparse_x.indices().numpy()) - assert np.array_equal(values_sorted, - sparse_x.values().numpy()) + assert np.array_equal( + indices_sorted, sparse_x.indices().numpy() + ) + assert np.array_equal( + values_sorted, sparse_x.values().numpy() + ) # test the non-zero values is a vector values = [[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]] @@ -307,10 +334,12 @@ def test_sparse_coo_tensor_sorted(self): sparse_x = paddle.sparse.sparse_coo_tensor(indices, values) sparse_x = paddle.sparse.coalesce(sparse_x) values_sorted = [[5.0, 5.0], [1.0, 1.0]] - assert np.array_equal(indices_sorted, - sparse_x.indices().numpy()) - assert np.array_equal(values_sorted, - sparse_x.values().numpy()) + assert np.array_equal( + indices_sorted, sparse_x.indices().numpy() + ) + assert np.array_equal( + values_sorted, sparse_x.values().numpy() + ) def test_batch_csr(self): with _test_eager_guard(): @@ -326,7 +355,7 @@ def verify(dense_x): dense_x = paddle.nn.functional.dropout(dense_x, p=0.5) verify(dense_x) - #test batchs=1 + # test batchs=1 shape[0] = 1 dense_x = paddle.randn(shape) dense_x = paddle.nn.functional.dropout(dense_x, p=0.5) @@ -335,23 +364,22 @@ def verify(dense_x): shape = np.random.randint(low=3, high=10, size=3) shape = list(shape) dense_x = paddle.randn(shape) - #set the 0th batch to zero + # set the 0th batch to zero dense_x[0] = 0 verify(dense_x) dense_x = paddle.randn(shape) - #set the 1th batch to zero + # set the 1th batch to zero dense_x[1] = 0 verify(dense_x) dense_x = paddle.randn(shape) - #set the 2th batch to zero + # set the 2th batch to zero dense_x[2] = 0 verify(dense_x) class TestCooError(unittest.TestCase): - def test_small_shape(self): with _test_eager_guard(): with self.assertRaises(ValueError): @@ -359,9 +387,9 @@ def test_small_shape(self): values = [1, 2] # 1. the shape too small dense_shape = [2, 2] - sparse_x = paddle.sparse.sparse_coo_tensor(indices, - values, - shape=dense_shape) + sparse_x = paddle.sparse.sparse_coo_tensor( + indices, values, shape=dense_shape + ) def test_same_nnz(self): with _test_eager_guard(): @@ -377,9 +405,9 @@ def test_same_dimensions(self): indices = [[1, 2], [1, 0]] values = [1, 2, 3] shape = [2, 3, 4] - sparse_x = paddle.sparse.sparse_coo_tensor(indices, - values, - shape=shape) + sparse_x = paddle.sparse.sparse_coo_tensor( + indices, values, shape=shape + ) def test_indices_dtype(self): with _test_eager_guard(): @@ -390,7 +418,6 @@ def test_indices_dtype(self): class TestCsrError(unittest.TestCase): - def test_dimension1(self): with _test_eager_guard(): with self.assertRaises(ValueError): @@ -399,7 +426,8 @@ def test_dimension1(self): values = [1, 2, 3] shape = [3] sparse_x = paddle.sparse.sparse_csr_tensor( - crows, cols, values, shape) + crows, cols, values, shape + ) def test_dimension2(self): with _test_eager_guard(): @@ -409,7 +437,8 @@ def test_dimension2(self): values = [1, 2, 3] shape = [3, 3, 3, 3] sparse_x = paddle.sparse.sparse_csr_tensor( - crows, cols, values, shape) + crows, cols, values, shape + ) def test_same_shape1(self): with _test_eager_guard(): @@ -419,7 +448,8 @@ def test_same_shape1(self): values = [1, 2, 3] shape = [3, 4] sparse_x = paddle.sparse.sparse_csr_tensor( - crows, cols, values, shape) + crows, cols, values, shape + ) def test_same_shape2(self): with _test_eager_guard(): @@ -429,7 +459,8 @@ def test_same_shape2(self): values = [1, 2, 3, 4] shape = [3, 4] sparse_x = paddle.sparse.sparse_csr_tensor( - crows, cols, values, shape) + crows, cols, values, shape + ) def test_same_shape3(self): with _test_eager_guard(): @@ -439,7 +470,8 @@ def test_same_shape3(self): values = [1, 2, 3, 4, 0, 1, 2] shape = [2, 3, 4] sparse_x = paddle.sparse.sparse_csr_tensor( - crows, cols, values, shape) + crows, cols, values, shape + ) def test_crows_first_value(self): with _test_eager_guard(): @@ -449,7 +481,8 @@ def test_crows_first_value(self): values = [1, 2, 3] shape = [3, 4] sparse_x = paddle.sparse.sparse_csr_tensor( - crows, cols, values, shape) + crows, cols, values, shape + ) def test_dtype(self): with _test_eager_guard(): @@ -459,7 +492,8 @@ def test_dtype(self): values = [1, 2, 3] shape = [3] sparse_x = paddle.sparse.sparse_csr_tensor( - crows, cols, values, shape) + crows, cols, values, shape + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_spawn_and_init_parallel_env.py b/python/paddle/fluid/tests/unittests/test_spawn_and_init_parallel_env.py index 1b230df912879..e05b1cdaaf378 100644 --- a/python/paddle/fluid/tests/unittests/test_spawn_and_init_parallel_env.py +++ b/python/paddle/fluid/tests/unittests/test_spawn_and_init_parallel_env.py @@ -17,7 +17,11 @@ import paddle import paddle.distributed as dist -from paddle.distributed.spawn import _get_subprocess_env_list, _options_valid_check, _get_default_nprocs +from paddle.distributed.spawn import ( + _get_subprocess_env_list, + _options_valid_check, + _get_default_nprocs, +) from paddle.fluid import core from paddle.fluid.dygraph import parallel_helper @@ -28,10 +32,10 @@ # executed in the python3 sub-process. -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestInitParallelEnv(unittest.TestCase): - def test_check_env_failed(self): os.environ['FLAGS_selected_gpus'] = '0' os.environ['PADDLE_TRAINER_ID'] = '0' @@ -51,10 +55,10 @@ def test_init_parallel_env_break(self): self.assertFalse(parallel_helper._is_parallel_ctx_initialized()) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestSpawnAssistMethod(unittest.TestCase): - def test_nprocs_greater_than_device_num_error(self): with self.assertRaises(RuntimeError): _get_subprocess_env_list(nprocs=100, options=dict()) diff --git a/python/paddle/fluid/tests/unittests/test_spectral_norm_op.py b/python/paddle/fluid/tests/unittests/test_spectral_norm_op.py index c310819bf1851..5cff9628d8dd0 100644 --- a/python/paddle/fluid/tests/unittests/test_spectral_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_spectral_norm_op.py @@ -47,15 +47,15 @@ def spectral_norm(weight, u, v, dim, power_iters, eps): @skip_check_grad_ci( reason="Spectral norm do not check grad when power_iters > 0 " "because grad is not calculated in power iterations, " - "which cannot be checked by python grad unittests") + "which cannot be checked by python grad unittests" +) class TestSpectralNormOpNoGrad(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'spectral_norm' weight = np.random.random(self.weight_shape).astype('float64') - u = np.random.normal(0., 1., self.u_shape).astype('float64') - v = np.random.normal(0., 1., self.v_shape).astype('float64') + u = np.random.normal(0.0, 1.0, self.u_shape).astype('float64') + v = np.random.normal(0.0, 1.0, self.v_shape).astype('float64') self.attrs = { "dim": self.dim, @@ -69,8 +69,9 @@ def setUp(self): "V": v, } - output = spectral_norm(weight, u, v, self.dim, self.power_iters, - self.eps) + output = spectral_norm( + weight, u, v, self.dim, self.power_iters, self.eps + ) self.outputs = {"Out": output} def test_check_output(self): @@ -78,8 +79,8 @@ def test_check_output(self): def initTestCase(self): self.weight_shape = (10, 12) - self.u_shape = (10, ) - self.v_shape = (12, ) + self.u_shape = (10,) + self.v_shape = (12,) self.dim = 0 self.power_iters = 5 self.eps = 1e-12 @@ -88,20 +89,19 @@ def initTestCase(self): @skip_check_grad_ci( reason="Spectral norm do not check grad when power_iters > 0 " "because grad is not calculated in power iterations, " - "which cannot be checked by python grad unittests") + "which cannot be checked by python grad unittests" +) class TestSpectralNormOpNoGrad2(TestSpectralNormOpNoGrad): - def initTestCase(self): self.weight_shape = (2, 3, 3, 3) - self.u_shape = (3, ) - self.v_shape = (18, ) + self.u_shape = (3,) + self.v_shape = (18,) self.dim = 1 self.power_iters = 10 self.eps = 1e-12 class TestSpectralNormOp(TestSpectralNormOpNoGrad): - def test_check_grad_ignore_uv(self): self.check_grad( ['Weight'], @@ -111,26 +111,24 @@ def test_check_grad_ignore_uv(self): def initTestCase(self): self.weight_shape = (10, 12) - self.u_shape = (10, ) - self.v_shape = (12, ) + self.u_shape = (10,) + self.v_shape = (12,) self.dim = 0 self.power_iters = 0 self.eps = 1e-12 class TestSpectralNormOp2(TestSpectralNormOp): - def initTestCase(self): self.weight_shape = (2, 6, 3, 3) - self.u_shape = (6, ) - self.v_shape = (18, ) + self.u_shape = (6,) + self.v_shape = (18,) self.dim = 1 self.power_iters = 0 self.eps = 1e-12 class TestSpectralNormOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): @@ -150,13 +148,12 @@ def test_weight_dtype(): class TestDygraphSpectralNormOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): shape = (2, 4, 3, 3) - spectralNorm = fluid.dygraph.nn.SpectralNorm(shape, - dim=1, - power_iters=2) + spectralNorm = fluid.dygraph.nn.SpectralNorm( + shape, dim=1, power_iters=2 + ) def test_Variable(): weight_1 = np.random.random((2, 4)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py b/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py index 0afd3ec56bb26..fefa11be9aa58 100644 --- a/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py +++ b/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py @@ -27,7 +27,6 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): - def place(self): return core.CPUPlace() @@ -52,11 +51,13 @@ def test_split_and_merge_lod_tensor_no_lod(self): expect_false = core.LoDTensor() expect_false.set(expect_false_tensor, self.place()) - self.main(tensor=tensor, - mask=mask, - expect_true=expect_true, - expect_false=expect_false, - expect_out=tensor) + self.main( + tensor=tensor, + mask=mask, + expect_true=expect_true, + expect_false=expect_false, + expect_out=tensor, + ) def split_and_merge_lod_tensor_level_0(self, use_merge_lod_infer=False): tensor = core.LoDTensor() @@ -83,12 +84,14 @@ def split_and_merge_lod_tensor_level_0(self, use_merge_lod_infer=False): expect_false.set(expect_false_tensor, self.place()) expect_false.set_recursive_sequence_lengths(expect_false_lod) - self.main(tensor=tensor, - mask=mask, - expect_true=expect_true, - expect_false=expect_false, - expect_out=tensor, - use_merge_lod_infer=use_merge_lod_infer) + self.main( + tensor=tensor, + mask=mask, + expect_true=expect_true, + expect_false=expect_false, + expect_out=tensor, + use_merge_lod_infer=use_merge_lod_infer, + ) def test_split_and_merge_lod_tensor_1(self): self.split_and_merge_lod_tensor_level_0() @@ -96,14 +99,16 @@ def test_split_and_merge_lod_tensor_1(self): def test_split_and_merge_lod_tensor_2(self): self.split_and_merge_lod_tensor_level_0(True) - def main(self, - tensor, - mask, - expect_true, - expect_false, - expect_out, - level=0, - use_merge_lod_infer=False): + def main( + self, + tensor, + mask, + expect_true, + expect_false, + expect_out, + level=0, + use_merge_lod_infer=False, + ): place = self.place() program = Program() with program_guard(program): @@ -122,38 +127,42 @@ def main(self, 'Mask': mask, 'InTrue': out_true, 'InFalse': out_false, - 'level': level + 'level': level, } helper = LayerHelper('merge_lod_tensor_infer') out = helper.create_variable_for_type_inference( - dtype=out_true.dtype) - helper.append_op(type='merge_lod_tensor_infer', - inputs={ - 'X': x, - 'Mask': y, - 'InTrue': out_true, - 'InFalse': out_false - }, - outputs={'Out': out}, - attrs={'level': level}) + dtype=out_true.dtype + ) + helper.append_op( + type='merge_lod_tensor_infer', + inputs={ + 'X': x, + 'Mask': y, + 'InTrue': out_true, + 'InFalse': out_false, + }, + outputs={'Out': out}, + attrs={'level': level}, + ) out.persistable = True else: - out = merge_lod_tensor(in_true=out_true, - in_false=out_false, - mask=y, - x=x, - level=level) + out = merge_lod_tensor( + in_true=out_true, + in_false=out_false, + mask=y, + x=x, + level=level, + ) out.persistable = True exe = Executor(place) scope = core.Scope() - exe.run(program, - feed={ - 'x': tensor, - 'y': mask - }, - scope=scope, - return_numpy=False) + exe.run( + program, + feed={'x': tensor, 'y': mask}, + scope=scope, + return_numpy=False, + ) var_true = scope.find_var(out_true.name).get_tensor() @@ -166,36 +175,33 @@ def main(self, self.check_tensor_same(var_out, expect_out) def check_tensor_same(self, actual, expect): - np.testing.assert_allclose(np.array(actual), - np.array(expect), - rtol=1e-05) - self.assertEqual(actual.recursive_sequence_lengths(), - expect.recursive_sequence_lengths()) + np.testing.assert_allclose( + np.array(actual), np.array(expect), rtol=1e-05 + ) + self.assertEqual( + actual.recursive_sequence_lengths(), + expect.recursive_sequence_lengths(), + ) class TestCPUSplitMergeLoDTensorGrad(unittest.TestCase): - def test_grad(self): place = core.CPUPlace() program = Program() with program_guard(program): - x = layers.data(name='x', - shape=[1], - dtype='float32', - stop_gradient=False) - y = layers.data(name='y', - shape=[1], - dtype='bool', - stop_gradient=False) + x = layers.data( + name='x', shape=[1], dtype='float32', stop_gradient=False + ) + y = layers.data( + name='y', shape=[1], dtype='bool', stop_gradient=False + ) level = 0 out_true, out_false = split_lod_tensor(input=x, mask=y, level=level) - out = merge_lod_tensor(in_true=out_true, - in_false=out_false, - mask=y, - x=x, - level=level) + out = merge_lod_tensor( + in_true=out_true, in_false=out_false, mask=y, x=x, level=level + ) mean = paddle.mean(out) append_backward(mean) @@ -215,16 +221,17 @@ def test_grad(self): g_vars = program.global_block().var(x.name + "@GRAD") g_out = [ - item.sum() for item in map( + item.sum() + for item in map( np.array, - exe.run(program, - feed={ - 'x': tensor, - 'y': mask - }, - fetch_list=[g_vars], - scope=scope, - return_numpy=False)) + exe.run( + program, + feed={'x': tensor, 'y': mask}, + fetch_list=[g_vars], + scope=scope, + return_numpy=False, + ), + ) ] g_out_sum = np.array(g_out).sum() @@ -233,78 +240,78 @@ def test_grad(self): class TestMergeLodTensorOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - input_data = layers.data(name='x', - shape=[1], - dtype='float32', - stop_gradient=False) - y = layers.data(name='y', - shape=[1], - dtype='bool', - stop_gradient=False) - x_true = layers.data(name='x_true', - shape=[1], - dtype='float32', - stop_gradient=False) - x_false = layers.data(name='x_false', - shape=[1], - dtype='float32', - stop_gradient=False) + input_data = layers.data( + name='x', shape=[1], dtype='float32', stop_gradient=False + ) + y = layers.data( + name='y', shape=[1], dtype='bool', stop_gradient=False + ) + x_true = layers.data( + name='x_true', shape=[1], dtype='float32', stop_gradient=False + ) + x_false = layers.data( + name='x_false', shape=[1], dtype='float32', stop_gradient=False + ) level = 0 def test_x(): - out = merge_lod_tensor(int_true=x_true, - in_false=x_false, - x=set(), - mask=y, - level=level) + out = merge_lod_tensor( + int_true=x_true, + in_false=x_false, + x=set(), + mask=y, + level=level, + ) self.assertRaises(TypeError, test_x) def test_mask(): - out = merge_lod_tensor(int_true=x_true, - in_false=x_false, - x=input_data, - mask=set(), - level=level) + out = merge_lod_tensor( + int_true=x_true, + in_false=x_false, + x=input_data, + mask=set(), + level=level, + ) self.assertRaises(TypeError, test_mask) def test_xtrue(): - out = merge_lod_tensor(int_true=set(), - in_false=x_false, - x=input_data, - mask=y, - level=level) + out = merge_lod_tensor( + int_true=set(), + in_false=x_false, + x=input_data, + mask=y, + level=level, + ) self.assertRaises(TypeError, test_xtrue) def test_xfalse(): - out = merge_lod_tensor(int_true=x_true, - in_false=set(), - x=input_data, - mask=y, - level=level) + out = merge_lod_tensor( + int_true=x_true, + in_false=set(), + x=input_data, + mask=y, + level=level, + ) self.assertRaises(TypeError, test_xfalse) class TestSplitLodTensorWithError(unittest.TestCase): - def test_error(self): main_program = Program() startup_program = Program() with program_guard(main_program, startup_program): - x = layers.data(name='x', - shape=[1], - dtype='float32', - stop_gradient=False) - y = layers.data(name='y', - shape=[1], - dtype='bool', - stop_gradient=False) + x = layers.data( + name='x', shape=[1], dtype='float32', stop_gradient=False + ) + y = layers.data( + name='y', shape=[1], dtype='bool', stop_gradient=False + ) level = 0 with self.assertRaises(TypeError): diff --git a/python/paddle/fluid/tests/unittests/test_split_op.py b/python/paddle/fluid/tests/unittests/test_split_op.py index 14ba9eb45a065..e087cc8b1bb84 100644 --- a/python/paddle/fluid/tests/unittests/test_split_op.py +++ b/python/paddle/fluid/tests/unittests/test_split_op.py @@ -22,7 +22,6 @@ class TestSplitOp(OpTest): - def setUp(self): self._set_op_type() self.dtype = self.get_dtype() @@ -31,14 +30,19 @@ def setUp(self): x = np.random.random((4, 5, 6)).astype(np.float32) out = np.split(x, [2, 3], axis) self.inputs = {'X': convert_float_to_uint16(x)} - self.outputs = {'Out': [('out%d' % i, convert_float_to_uint16(out[i])) \ - for i in range(len(out))]} + self.outputs = { + 'Out': [ + ('out%d' % i, convert_float_to_uint16(out[i])) + for i in range(len(out)) + ] + } else: x = np.random.random((4, 5, 6)).astype(self.dtype) out = np.split(x, [2, 3], axis) self.inputs = {'X': x} - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = { + 'Out': [('out%d' % i, out[i]) for i in range(len(out))] + } self.attrs = {'axis': axis, 'sections': [2, 1, 2]} def get_dtype(self): @@ -56,7 +60,6 @@ def test_check_grad(self): # test with attr(num) class TestSplitOp_2(OpTest): - def setUp(self): self._set_op_type() self.dtype = self.get_dtype() @@ -65,12 +68,11 @@ def setUp(self): self.attrs = { 'axis': self.axis, 'sections': self.sections, - 'num': self.num + 'num': self.num, } out = np.split(self.x, self.indices_or_sections, self.axis) - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = {'Out': [('out%d' % i, out[i]) for i in range(len(out))]} def init_data(self): self.x = np.random.random((4, 5, 6)).astype(self.dtype) @@ -94,20 +96,18 @@ def test_check_grad(self): # attr(axis) is Tensor class TestSplitOp_AxisTensor(OpTest): - def setUp(self): self._set_op_type() self.dtype = self.get_dtype() self.init_data() self.inputs = { 'X': self.x, - 'AxisTensor': np.array([self.axis]).astype("int32") + 'AxisTensor': np.array([self.axis]).astype("int32"), } self.attrs = {'sections': self.sections, 'num': self.num} out = np.split(self.x, self.indices_or_sections, self.axis) - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = {'Out': [('out%d' % i, out[i]) for i in range(len(out))]} def init_data(self): self.x = np.random.random((4, 5, 6)).astype(self.dtype) @@ -131,7 +131,6 @@ def test_check_grad(self): # attr(sections) is list containing Tensor class TestSplitOp_SectionsTensor(OpTest): - def setUp(self): self._set_op_type() self.dtype = self.get_dtype() @@ -140,20 +139,20 @@ def setUp(self): sections_tensor = [] for index, ele in enumerate(self.sections): - sections_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + sections_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SectionsTensorList'] = sections_tensor self.attrs = { 'axis': self.axis, 'sections': self.sections_infer, - 'num': self.num + 'num': self.num, } out = np.split(self.x, self.indices_or_sections, self.axis) - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = {'Out': [('out%d' % i, out[i]) for i in range(len(out))]} def init_data(self): self.x = np.random.random((4, 5, 6)).astype(self.dtype) @@ -177,7 +176,6 @@ def test_check_grad(self): class TestSplitOp_unk_section(OpTest): - def setUp(self): self._set_op_type() self.dtype = self.get_dtype() @@ -186,12 +184,11 @@ def setUp(self): self.attrs = { 'axis': self.axis, 'sections': self.sections, - 'num': self.num + 'num': self.num, } out = np.split(self.x, self.indices_or_sections, self.axis) - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = {'Out': [('out%d' % i, out[i]) for i in range(len(out))]} def init_data(self): self.x = np.random.random((4, 5, 6)).astype(self.dtype) @@ -214,20 +211,18 @@ def test_check_grad(self): class TestSplitByrefOp(OpTest): - def _set_op_type(self): self.op_type = "split_byref" -#----------------Split Fp16---------------- +# ----------------Split Fp16---------------- def create_test_fp16(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestSplitFp16(parent): - def get_dtype(self): return np.float16 @@ -241,15 +236,14 @@ def test_check_grad(self): create_test_fp16(TestSplitOp) -#----------------Split Bf16---------------- +# ----------------Split Bf16---------------- def create_test_bf16(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestSplitBf16(parent): - def get_dtype(self): return np.uint16 @@ -269,7 +263,6 @@ def test_check_grad(self): class TestSplitAPI(unittest.TestCase): - def test_api(self): input_1 = np.random.random([4, 5, 6]).astype("int32") positive_1_int32 = fluid.layers.fill_constant([1], "int32", 1) @@ -281,21 +274,20 @@ def test_api(self): out_0, out_1, out_2 = fluid.layers.split( input=x_1, num_or_sections=[positive_2_int64, positive_1_int32, -1], - dim=positive_1_int64) + dim=positive_1_int64, + ) - out_3, out_4, out_5 = fluid.layers.split(input=x_1, - num_or_sections=[2, 1, 2], - dim=positive_1_int32) + out_3, out_4, out_5 = fluid.layers.split( + input=x_1, num_or_sections=[2, 1, 2], dim=positive_1_int32 + ) fluid.layers.split(input=x_2, num_or_sections=2, dim=2) exe = fluid.Executor(place=fluid.CPUPlace()) - [res_0, res_1, res_2, res_3, res_4, - res_5] = exe.run(fluid.default_main_program(), - feed={ - "x_1": input_1, - "x_2": input_1 - }, - fetch_list=[out_0, out_1, out_2, out_3, out_4, out_5]) + [res_0, res_1, res_2, res_3, res_4, res_5] = exe.run( + fluid.default_main_program(), + feed={"x_1": input_1, "x_2": input_1}, + fetch_list=[out_0, out_1, out_2, out_3, out_4, out_5], + ) out = np.split(input_1, [2, 3], 1) assert np.array_equal(res_0, out[0]) @@ -307,7 +299,6 @@ def test_api(self): class TestSplitOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The type of axis in split_op should be int or Variable. @@ -346,7 +337,6 @@ def test_axis_type_tensor(): class API_TestSplit(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[4, 6, 6], dtype='float64') @@ -356,11 +346,9 @@ def test_out(self): exe = fluid.Executor(place) input1 = np.random.random([4, 6, 6]).astype('float64') input2 = np.array([2]).astype('int32') - r0, r1, r2, = exe.run(feed={ - "data1": input1, - "data2": input2 - }, - fetch_list=[x0, x1, x2]) + r0, r1, r2, = exe.run( + feed={"data1": input1, "data2": input2}, fetch_list=[x0, x1, x2] + ) ex_x0, ex_x1, ex_x2 = np.split(input1, 3, axis=2) np.testing.assert_allclose(ex_x0, r0, rtol=1e-05) np.testing.assert_allclose(ex_x1, r1, rtol=1e-05) @@ -368,7 +356,6 @@ def test_out(self): class API_TestSplit2(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[4, 6, 6], dtype='float64') @@ -376,8 +363,11 @@ def test_out(self): place = fluid.CPUPlace() exe = fluid.Executor(place) input1 = np.random.random([4, 6, 6]).astype('float64') - r0, r1, r2, = exe.run(feed={"data1": input1}, - fetch_list=[x0, x1, x2]) + ( + r0, + r1, + r2, + ) = exe.run(feed={"data1": input1}, fetch_list=[x0, x1, x2]) ex_x0, ex_x1, ex_x2 = np.split(input1, 3, axis=2) np.testing.assert_allclose(ex_x0, r0, rtol=1e-05) np.testing.assert_allclose(ex_x1, r1, rtol=1e-05) @@ -385,7 +375,6 @@ def test_out(self): class API_TestSplit3(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.layers.data('data', shape=[-1, 10], dtype='float64') @@ -394,13 +383,12 @@ def test_out(self): exe = fluid.Executor(place) input1 = np.random.random([1, 10]).astype('float64') r0, r1 = exe.run(feed={"data": input1}, fetch_list=[x0, x1]) - ex_x0, ex_x1 = np.split(input1, (3, ), axis=1) + ex_x0, ex_x1 = np.split(input1, (3,), axis=1) np.testing.assert_allclose(ex_x0, r0, rtol=1e-05) np.testing.assert_allclose(ex_x1, r1, rtol=1e-05) class API_TestSplit4(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.layers.data('data', shape=[-1, 10], dtype='float64') @@ -410,21 +398,19 @@ def test_out(self): exe = fluid.Executor(place) input1 = np.random.random([1, 10]).astype('float64') input2 = np.array([7]).astype('int32') - r0, r1 = exe.run(feed={ - "data": input1, - "index": input2 - }, - fetch_list=[x0, x1]) - ex_x0, ex_x1 = np.split(input1, (3, ), axis=1) + r0, r1 = exe.run( + feed={"data": input1, "index": input2}, fetch_list=[x0, x1] + ) + ex_x0, ex_x1 = np.split(input1, (3,), axis=1) np.testing.assert_allclose(ex_x0, r0, rtol=1e-05) np.testing.assert_allclose(ex_x1, r1, rtol=1e-05) class API_TestSplit5(unittest.TestCase): - def test_out(self): - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() with fluid.program_guard(fluid.Program(), fluid.Program()): input_1 = np.random.random([5, 4]).astype("int32") @@ -441,7 +427,6 @@ def test_out(self): class API_TestSplit6(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data = fluid.layers.data('data', shape=[-1, 10], dtype='float64') @@ -450,13 +435,12 @@ def test_out(self): exe = fluid.Executor(place) input1 = np.random.random([2, 10]).astype('float64') r0, r1 = exe.run(feed={"data": input1}, fetch_list=[x0, x1]) - ex_x0, ex_x1 = np.split(input1, (1, ), axis=0) + ex_x0, ex_x1 = np.split(input1, (1,), axis=0) np.testing.assert_allclose(ex_x0, r0, rtol=1e-05) np.testing.assert_allclose(ex_x1, r1, rtol=1e-05) class API_TestDygraphFluidSplit(unittest.TestCase): - def test_out1(self): with fluid.dygraph.guard(): input_1 = np.random.random([4, 6, 6]).astype("int32") @@ -479,9 +463,9 @@ def test_out1(self): loss.backward() manul_grad = np.zeros_like(input_1) manul_grad[:, :2, :] = 1 - np.testing.assert_allclose(input.gradient(), - manul_grad, - rtol=1e-05) + np.testing.assert_allclose( + input.gradient(), manul_grad, rtol=1e-05 + ) np.testing.assert_allclose(ex_x0, eager_x0_out, rtol=1e-05) np.testing.assert_allclose(ex_x1, eager_x1_out, rtol=1e-05) np.testing.assert_allclose(ex_x2, eager_x2_out, rtol=1e-05) @@ -512,9 +496,9 @@ def test_out2(self): loss.backward() manul_grad = np.zeros_like(input_1) manul_grad[:, :2, :] = 1 - np.testing.assert_allclose(input.gradient(), - manul_grad, - rtol=1e-05) + np.testing.assert_allclose( + input.gradient(), manul_grad, rtol=1e-05 + ) np.testing.assert_allclose(ex_x0, eager_x0_out, rtol=1e-05) np.testing.assert_allclose(ex_x1, eager_x1_out, rtol=1e-05) np.testing.assert_allclose(ex_x2, eager_x2_out, rtol=1e-05) @@ -525,7 +509,6 @@ def test_out2(self): class API_TestDygraphSplit(unittest.TestCase): - def test_out1(self): with fluid.dygraph.guard(): input_1 = np.random.random([4, 6, 6]).astype("int32") @@ -549,9 +532,9 @@ def test_out1(self): loss.backward() manul_grad = np.zeros_like(input_1) manul_grad[:, :2, :] = 1 - np.testing.assert_allclose(input.gradient(), - manul_grad, - rtol=1e-05) + np.testing.assert_allclose( + input.gradient(), manul_grad, rtol=1e-05 + ) np.testing.assert_allclose(ex_x0, eager_x0_out, rtol=1e-05) np.testing.assert_allclose(ex_x1, eager_x1_out, rtol=1e-05) np.testing.assert_allclose(ex_x2, eager_x2_out, rtol=1e-05) @@ -599,9 +582,9 @@ def test_out_tensor_input(self): # input is a variable which shape is [4, 6, 6] input = paddle.to_tensor(input_1) num1 = paddle.full(shape=[1], fill_value=2, dtype='int32') - x0, x1, x2 = paddle.split(input, - num_or_sections=[num1, 2, 2], - axis=1) + x0, x1, x2 = paddle.split( + input, num_or_sections=[num1, 2, 2], axis=1 + ) x0_out = x0.numpy() x1_out = x1.numpy() x2_out = x2.numpy() @@ -616,9 +599,9 @@ def test_axis_tensor_input(self): # input is a variable which shape is [4, 6, 6] input = paddle.to_tensor(input_1) num1 = paddle.full(shape=[1], fill_value=1, dtype='int32') - x0, x1, x2 = paddle.split(input, - num_or_sections=[2, 2, 2], - axis=num1) + x0, x1, x2 = paddle.split( + input, num_or_sections=[2, 2, 2], axis=num1 + ) x0_out = x0.numpy() x1_out = x1.numpy() x2_out = x2.numpy() @@ -644,7 +627,6 @@ def test_negative_one_section(self): class API_TestEmptySplit(unittest.TestCase): - def test_axis_input_empty_section(self): with fluid.dygraph.guard(): input_1 = np.random.random([8, 6, 6]).astype("float32") @@ -654,10 +636,13 @@ def test_axis_input_empty_section(self): x0_out = x0.numpy() x1_out = x1.numpy() x2_out = x2.numpy() - ex_x0, ex_x1, ex_x2 = np.split(input_1, [ - 5, - 5, - ]) + ex_x0, ex_x1, ex_x2 = np.split( + input_1, + [ + 5, + 5, + ], + ) np.testing.assert_allclose(ex_x0, x0_out, rtol=1e-05) np.testing.assert_allclose(ex_x1, x1_out, rtol=1e-05) np.testing.assert_allclose(ex_x2, x2_out, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_split_program.py b/python/paddle/fluid/tests/unittests/test_split_program.py index 761c5ed5587fa..bab36fd88ce4a 100644 --- a/python/paddle/fluid/tests/unittests/test_split_program.py +++ b/python/paddle/fluid/tests/unittests/test_split_program.py @@ -21,7 +21,6 @@ class TestSplitProgram(unittest.TestCase): - def setUp(self): paddle.enable_static() if paddle.is_compiled_with_cuda(): @@ -31,12 +30,12 @@ def get_model(self, batch_size): main = paddle.static.Program() startup = paddle.static.Program() with paddle.static.program_guard(main, startup): - image = paddle.static.data(shape=[batch_size, 3, 224, 224], - dtype='float32', - name='image') - label = paddle.static.data(shape=[batch_size, 1], - dtype='int64', - name='label') + image = paddle.static.data( + shape=[batch_size, 3, 224, 224], dtype='float32', name='image' + ) + label = paddle.static.data( + shape=[batch_size, 1], dtype='int64', name='label' + ) model = resnet(pretrained=False) loss_fn = nn.loss.CrossEntropyLoss() @@ -65,10 +64,11 @@ def test_split_program(self): self.assertEqual(len(vars_actual), len(vars_expected)) for actual, expected in zip(vars_actual, vars_expected): self.assertEqual(actual.shape, expected.shape) - np.testing.assert_array_equal(actual, - expected, - err_msg='{}\n{}\n'.format( - actual, expected)) + np.testing.assert_array_equal( + actual, + expected, + err_msg='{}\n{}\n'.format(actual, expected), + ) def get_places(self): places = [paddle.CPUPlace()] @@ -93,27 +93,26 @@ def check_split_program(self, place, use_split=True, seed=100, batch_num=5): exe = paddle.static.Executor(place) image_np = np.random.random(size=image.shape).astype('float32') - label_np = np.random.randint(low=0, - high=1000, - dtype='int64', - size=label.shape) + label_np = np.random.randint( + low=0, high=1000, dtype='int64', size=label.shape + ) scope = paddle.static.Scope() if not use_split: with paddle.static.scope_guard(scope): exe.run(startup_prog) for _ in range(batch_num): - exe.run(main_prog, - feed={ - image.name: image_np, - label.name: label_np - }) + exe.run( + main_prog, + feed={image.name: image_np, label.name: label_np}, + ) return self.get_var_values(scope, startup_vars) op_num = len(main_prog.global_block().ops) split_op_indices = [int(op_num / 3.0), int(op_num * 3 / 4.0)] programs, input_vars, output_vars = split_program( - main_prog, split_op_indices) + main_prog, split_op_indices + ) op_nums = [0] + split_op_indices + [op_num] op_nums = [op_nums[i + 1] - op_nums[i] for i in range(len(op_nums) - 1)] num_split = len(split_op_indices) + 1 @@ -137,15 +136,19 @@ def check_split_program(self, place, use_split=True, seed=100, batch_num=5): if tmp_vars[in_name] is not None: feed_dict[in_name] = tmp_vars[in_name] - output_var_values = exe.run(program, - feed=feed_dict, - fetch_list=output_vars[i], - return_numpy=False) - for out_name, out_value in zip(output_vars[i], - output_var_values): + output_var_values = exe.run( + program, + feed=feed_dict, + fetch_list=output_vars[i], + return_numpy=False, + ) + for out_name, out_value in zip( + output_vars[i], output_var_values + ): if not out_value._is_initialized(): tmp_vars[out_name] = np.ndarray( - out_value._get_dims()).astype('float32') + out_value._get_dims() + ).astype('float32') else: tmp_vars[out_name] = np.array(out_value) diff --git a/python/paddle/fluid/tests/unittests/test_splits_api.py b/python/paddle/fluid/tests/unittests/test_splits_api.py index 3613c7be3b621..491fed74f775c 100644 --- a/python/paddle/fluid/tests/unittests/test_splits_api.py +++ b/python/paddle/fluid/tests/unittests/test_splits_api.py @@ -35,7 +35,6 @@ def func_ref(func, x, num_or_sections): class TestSplitsAPI(unittest.TestCase): - def setUp(self): self.rtol = 1e-5 self.atol = 1e-8 @@ -45,8 +44,11 @@ def set_input(self): self.shape = [4, 5, 2] self.num_or_sections = 2 self.x_np = np.random.uniform(-1, 1, self.shape).astype('float64') - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_api(self): paddle.enable_static() @@ -67,68 +69,79 @@ def test_dygraph_api(self): out = func(x, self.num_or_sections) out_ref = func_ref(func_type, self.x_np, self.num_or_sections) for n, p in zip(out_ref, out): - np.testing.assert_allclose(n, - p.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + n, p.numpy(), rtol=self.rtol, atol=self.atol + ) paddle.enable_static() class TestSplitsSections(TestSplitsAPI): """ - Test num_or_sections which is a list and date type is float64. + Test num_or_sections which is a list and date type is float64. """ def set_input(self): self.shape = [6, 2, 4] self.num_or_sections = [2, 1, 3] self.x_np = np.random.uniform(-1, 1, self.shape).astype('float64') - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) class TestSplitsFloat32(TestSplitsAPI): """ - Test num_or_sections which is an integer and data type is float32. + Test num_or_sections which is an integer and data type is float32. """ def set_input(self): self.shape = [2, 3, 4] self.num_or_sections = 2 self.x_np = np.random.uniform(-1, 1, self.shape).astype('float32') - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) class TestSplitsInt32(TestSplitsAPI): """ - Test data type int32. + Test data type int32. """ def set_input(self): self.shape = [5, 1, 2] self.num_or_sections = 5 self.x_np = np.random.uniform(-1, 1, self.shape).astype('int32') - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) class TestSplitsInt64(TestSplitsAPI): """ - Test data type int64. + Test data type int64. """ def set_input(self): self.shape = [4, 3, 2] self.num_or_sections = 2 self.x_np = np.random.uniform(-1, 1, self.shape).astype('int64') - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) class TestSplitsCPU(TestSplitsAPI): """ - Test cpu place and num_or_sections which is a tuple. + Test cpu place and num_or_sections which is a tuple. """ def set_input(self): @@ -140,13 +153,16 @@ def set_input(self): class TestSplitsError(unittest.TestCase): """ - Test the situation that input shape less than 2. + Test the situation that input shape less than 2. """ def setUp(self): self.num_or_sections = 1 - self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def test_static_error(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_spp_op.py b/python/paddle/fluid/tests/unittests/test_spp_op.py index 460abd59c421b..776cde9b55398 100644 --- a/python/paddle/fluid/tests/unittests/test_spp_op.py +++ b/python/paddle/fluid/tests/unittests/test_spp_op.py @@ -20,7 +20,6 @@ class TestSppOp(OpTest): - def setUp(self): self.op_type = "spp" self.init_test_case() @@ -34,19 +33,25 @@ def setUp(self): bins = np.power(2, i) kernel_size = [0, 0] padding = [0, 0] - kernel_size[0] = np.ceil(hsize / - bins.astype("double")).astype("int32") - padding[0] = ((kernel_size[0] * bins - hsize + 1) / - 2).astype("int32") + kernel_size[0] = np.ceil(hsize / bins.astype("double")).astype( + "int32" + ) + padding[0] = ((kernel_size[0] * bins - hsize + 1) / 2).astype( + "int32" + ) - kernel_size[1] = np.ceil(wsize / - bins.astype("double")).astype("int32") - padding[1] = ((kernel_size[1] * bins - wsize + 1) / - 2).astype("int32") - out_level = self.pool2D_forward_naive(input, kernel_size, - kernel_size, padding) + kernel_size[1] = np.ceil(wsize / bins.astype("double")).astype( + "int32" + ) + padding[1] = ((kernel_size[1] * bins - wsize + 1) / 2).astype( + "int32" + ) + out_level = self.pool2D_forward_naive( + input, kernel_size, kernel_size, padding + ) out_level_flatten.append( - out_level.reshape(nsize, bins * bins * csize)) + out_level.reshape(nsize, bins * bins * csize) + ) if i == 0: output = out_level_flatten[i] else: @@ -57,7 +62,7 @@ def setUp(self): } self.attrs = { 'pyramid_height': self.pyramid_height, - 'pooling_type': self.pool_type + 'pooling_type': self.pool_type, } self.outputs = {'Out': output.astype('float64')} @@ -75,7 +80,6 @@ def init_test_case(self): class TestCase2(TestSppOp): - def init_test_case(self): self.shape = [3, 2, 16, 16] self.pyramid_height = 3 diff --git a/python/paddle/fluid/tests/unittests/test_square_error_cost.py b/python/paddle/fluid/tests/unittests/test_square_error_cost.py index 89001fe78b021..498eee8051e9c 100644 --- a/python/paddle/fluid/tests/unittests/test_square_error_cost.py +++ b/python/paddle/fluid/tests/unittests/test_square_error_cost.py @@ -21,7 +21,6 @@ class TestSquareErrorCost(unittest.TestCase): - def test_square_error_cost(self): input_val = np.random.uniform(0.1, 0.5, (2, 3)).astype("float32") label_val = np.random.uniform(0.1, 0.5, (2, 3)).astype("float32") @@ -33,25 +32,23 @@ def test_square_error_cost(self): label_var = layers.create_tensor(dtype="float32", name="label") output = layers.square_error_cost(input=input_var, label=label_var) - for use_cuda in ([False, True] - if core.is_compiled_with_cuda() else [False]): + for use_cuda in ( + [False, True] if core.is_compiled_with_cuda() else [False] + ): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = Executor(place) - result, = exe.run(fluid.default_main_program(), - feed={ - "input": input_val, - "label": label_val - }, - fetch_list=[output]) + (result,) = exe.run( + fluid.default_main_program(), + feed={"input": input_val, "label": label_val}, + fetch_list=[output], + ) np.testing.assert_allclose(np_result, result, rtol=1e-05) class TestSquareErrorInvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_input(): input = [256, 3] label = fluid.data(name='label1', shape=[None, 3], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py b/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py index 6804d8a771e99..21fdb592402f5 100644 --- a/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py +++ b/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py @@ -18,18 +18,17 @@ class TestSquaredL2DistanceOp_f0(OpTest): - def setUp(self): self.op_type = "squared_l2_distance" self.inputs = { 'X': np.random.uniform(0.1, 0.6, (5, 20)).astype("float32"), - 'Y': np.random.uniform(0.1, 0.6, (5, 20)).astype("float32") + 'Y': np.random.uniform(0.1, 0.6, (5, 20)).astype("float32"), } sub_res = self.inputs['X'] - self.inputs['Y'] output = sub_res * sub_res self.outputs = { 'sub_result': sub_res, - 'Out': np.expand_dims(output.sum(1), 1) + 'Out': np.expand_dims(output.sum(1), 1), } def test_check_output(self): @@ -40,18 +39,17 @@ def test_check_grad(self): class TestSquaredL2DistanceOp_f1(OpTest): - def setUp(self): self.op_type = "squared_l2_distance" self.inputs = { 'X': np.random.uniform(0.1, 0.6, (2, 3)).astype("float32"), - 'Y': np.random.uniform(0.1, 0.6, (1, 3)).astype("float32") + 'Y': np.random.uniform(0.1, 0.6, (1, 3)).astype("float32"), } sub_res = self.inputs['X'] - self.inputs['Y'] output = sub_res * sub_res self.outputs = { 'sub_result': sub_res, - 'Out': np.expand_dims(output.sum(1), 1) + 'Out': np.expand_dims(output.sum(1), 1), } def test_check_output(self): @@ -62,19 +60,18 @@ def test_check_grad(self): class TestSquaredL2DistanceOp_f2(OpTest): - def setUp(self): self.op_type = "squared_l2_distance" self.inputs = { 'X': np.random.uniform(0.1, 0.6, (2, 3, 4)).astype("float32"), - 'Y': np.random.uniform(0.1, 0.6, (1, 3, 4)).astype("float32") + 'Y': np.random.uniform(0.1, 0.6, (1, 3, 4)).astype("float32"), } sub_res = self.inputs['X'] - self.inputs['Y'] sub_res = sub_res.reshape((2, 3 * 4)) output = sub_res * sub_res self.outputs = { 'sub_result': sub_res, - 'Out': np.expand_dims(output.sum(1), 1) + 'Out': np.expand_dims(output.sum(1), 1), } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py b/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py index bc4dcfd1e28a0..8e09d83975a60 100644 --- a/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py @@ -29,8 +29,7 @@ def test_squared_l2_norm(x): class TestL2LossOp(OpTest): - """Test squared_l2_norm - """ + """Test squared_l2_norm""" def setUp(self): self.python_api = test_squared_l2_norm @@ -46,14 +45,15 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - max_relative_error=self.max_relative_error, - check_eager=True) + self.check_grad( + ['X'], + 'Out', + max_relative_error=self.max_relative_error, + check_eager=True, + ) class TestL2LossDeterministic(unittest.TestCase): - def check_place(self, place): with paddle.fluid.dygraph.guard(place): x_np = np.random.rand(5, 11, 13).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_squeeze2_op.py b/python/paddle/fluid/tests/unittests/test_squeeze2_op.py index 798afe703243b..c64b58efea141 100755 --- a/python/paddle/fluid/tests/unittests/test_squeeze2_op.py +++ b/python/paddle/fluid/tests/unittests/test_squeeze2_op.py @@ -27,7 +27,6 @@ # Correct: General. class TestSqueezeOp(OpTest): - def setUp(self): self.op_type = "squeeze2" self.python_api = paddle.squeeze @@ -39,7 +38,7 @@ def setUp(self): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float64") + "XShape": np.random.random(self.ori_shape).astype("float64"), } def test_check_output(self): @@ -59,7 +58,6 @@ def init_attrs(self): # Correct: There is mins axis. class TestSqueezeOp1(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = (0, -2) @@ -68,7 +66,6 @@ def init_test_case(self): # Correct: No axes input. class TestSqueezeOp2(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -77,7 +74,6 @@ def init_test_case(self): # Correct: Just part of axes be squeezed. class TestSqueezeOp3(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) @@ -85,7 +81,6 @@ def init_test_case(self): class TestSqueeze2AxesTensor(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.save_path = os.path.join(self.temp_dir.name, 'squeeze_tensor') @@ -122,7 +117,6 @@ def test_static(self): class TestSqueeze2AxesTensorList(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.save_path = os.path.join(self.temp_dir.name, 'squeeze_tensor') @@ -139,7 +133,7 @@ def test_static(self): # axes is a list[Variable] axes = [ paddle.full([1], 0, dtype='int32'), - paddle.full([1], 2, dtype='int32') + paddle.full([1], 2, dtype='int32'), ] out = paddle.squeeze(feat, axes) out2 = paddle.fluid.layers.squeeze(feat, axes) diff --git a/python/paddle/fluid/tests/unittests/test_squeeze_op.py b/python/paddle/fluid/tests/unittests/test_squeeze_op.py index 53b5cb272534f..e4ea13844b705 100755 --- a/python/paddle/fluid/tests/unittests/test_squeeze_op.py +++ b/python/paddle/fluid/tests/unittests/test_squeeze_op.py @@ -30,7 +30,6 @@ # Correct: General. class TestSqueezeOp(OpTest): - def setUp(self): self.op_type = "squeeze" self.init_test_case() @@ -56,7 +55,6 @@ def init_attrs(self): class TestSqueezeBF16Op(OpTest): - def setUp(self): self.op_type = "squeeze" self.dtype = np.uint16 @@ -84,7 +82,6 @@ def init_attrs(self): # Correct: There is mins axis. class TestSqueezeOp1(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 3, 1, 40) self.axes = (0, -2) @@ -93,7 +90,6 @@ def init_test_case(self): # Correct: No axes input. class TestSqueezeOp2(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -102,7 +98,6 @@ def init_test_case(self): # Correct: Just part of axes be squeezed. class TestSqueezeOp3(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) @@ -111,7 +106,6 @@ def init_test_case(self): # Correct: The demension of axis is not of size 1 remains unchanged. class TestSqueezeOp4(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, 2) @@ -119,13 +113,13 @@ def init_test_case(self): class TestSqueezeOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): # The input type of softmax_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - paddle.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], paddle.CPUPlace() + ) self.assertRaises(TypeError, paddle.squeeze, x1) # The input axes of squeeze must be list. x2 = paddle.static.data(name='x2', shape=[4], dtype="int32") @@ -136,7 +130,6 @@ def test_errors(self): class API_TestSqueeze(unittest.TestCase): - def setUp(self): self.executed_api() @@ -145,29 +138,29 @@ def executed_api(self): def test_out(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - data1 = paddle.static.data('data1', - shape=[-1, 1, 10], - dtype='float64') + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + data1 = paddle.static.data( + 'data1', shape=[-1, 1, 10], dtype='float64' + ) result_squeeze = self.squeeze(data1, axis=[1]) place = paddle.CPUPlace() exe = paddle.static.Executor(place) input1 = np.random.random([5, 1, 10]).astype('float64') - result, = exe.run(feed={"data1": input1}, - fetch_list=[result_squeeze]) + (result,) = exe.run( + feed={"data1": input1}, fetch_list=[result_squeeze] + ) expected_result = np.squeeze(input1, axis=1) np.testing.assert_allclose(expected_result, result, rtol=1e-05) class API_TestStaticSqueeze_(API_TestSqueeze): - def executed_api(self): self.squeeze = paddle.squeeze_ class API_TestDygraphSqueeze(unittest.TestCase): - def setUp(self): self.executed_api() @@ -221,13 +214,11 @@ def test_dimension_not_1(self): class API_TestDygraphSqueezeInplace(API_TestDygraphSqueeze): - def executed_api(self): self.squeeze = paddle.squeeze_ class TestSqueezeDoubleGradCheck(unittest.TestCase): - def squeeze_wrapper(self, x): return paddle.squeeze(x[0]) @@ -242,17 +233,13 @@ def func(self, place): out = paddle.squeeze(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.squeeze_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.squeeze_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -264,7 +251,6 @@ def test_grad(self): class TestSqueezeTripleGradCheck(unittest.TestCase): - def squeeze_wrapper(self, x): return paddle.squeeze(x[0]) @@ -279,17 +265,13 @@ def func(self, place): out = paddle.squeeze(data) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.squeeze_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.squeeze_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_stack_op.py b/python/paddle/fluid/tests/unittests/test_stack_op.py index 0677211aed948..0f71ebeac5b21 100644 --- a/python/paddle/fluid/tests/unittests/test_stack_op.py +++ b/python/paddle/fluid/tests/unittests/test_stack_op.py @@ -23,7 +23,6 @@ class TestStackOpBase(OpTest): - def initDefaultParameters(self): self.num_inputs = 4 self.input_dim = (5, 6, 7) @@ -47,7 +46,8 @@ def setUp(self): self.x = [] for i in range(self.num_inputs): self.x.append( - np.random.random(size=self.input_dim).astype(self.dtype)) + np.random.random(size=self.input_dim).astype(self.dtype) + ) tmp = [] x_names = self.get_x_names() @@ -66,49 +66,41 @@ def test_check_grad(self): class TestStackOp1(TestStackOpBase): - def initParameters(self): self.num_inputs = 8 class TestStackOp2(TestStackOpBase): - def initParameters(self): self.num_inputs = 10 class TestStackOp3(TestStackOpBase): - def initParameters(self): self.axis = -1 class TestStackOp4(TestStackOpBase): - def initParameters(self): self.axis = -4 class TestStackOp5(TestStackOpBase): - def initParameters(self): self.axis = 1 class TestStackOp6(TestStackOpBase): - def initParameters(self): self.axis = 3 class TestStackOp_ZeroDim(TestStackOpBase): - def initParameters(self): self.input_dim = () class TestStackBF16Op(OpTest): - def initDefaultParameters(self): self.num_inputs = 4 self.input_dim = (5, 6, 7) @@ -132,7 +124,8 @@ def setUp(self): self.x = [] for i in range(self.num_inputs): self.x.append( - np.random.random(size=self.input_dim).astype(np.float32)) + np.random.random(size=self.input_dim).astype(np.float32) + ) out = np.stack(self.x, axis=self.axis) @@ -162,8 +155,11 @@ def setUp(self): self.iter_num = 3 self.input_shape = [2, 3] self.x = np.random.random(self.input_shape).astype("float32") - self.place = fluid.CUDAPlace(0) \ - if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.set_program() def set_program(self): @@ -183,7 +179,8 @@ def test_case(self): exe = fluid.Executor(self.place) res = exe.run(self.program, fetch_list=self.out_var) np.testing.assert_array_equal( - res[0], np.stack([self.x] * self.iter_num, axis=self.axis)) + res[0], np.stack([self.x] * self.iter_num, axis=self.axis) + ) class TestTensorStackAPIWithLoDTensorArray(unittest.TestCase): @@ -196,8 +193,11 @@ def setUp(self): self.iter_num = 3 self.input_shape = [2, 3] self.x = np.random.random(self.input_shape).astype("float32") - self.place = fluid.CUDAPlace(0) \ - if fluid.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if fluid.is_compiled_with_cuda() + else fluid.CPUPlace() + ) self.set_program() def set_program(self): @@ -217,11 +217,11 @@ def test_case(self): exe = fluid.Executor(self.place) res = exe.run(self.program, fetch_list=self.out_var) np.testing.assert_array_equal( - res[0], np.stack([self.x] * self.iter_num, axis=self.axis)) + res[0], np.stack([self.x] * self.iter_num, axis=self.axis) + ) class API_test(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program(), fluid.Program()): data1 = fluid.layers.data('data1', shape=[1, 2], dtype='float64') @@ -233,12 +233,10 @@ def test_out(self): input1 = np.random.random([1, 2]).astype('float64') input2 = np.random.random([1, 2]).astype('float64') input3 = np.random.random([1, 2]).astype('float64') - result, = exe.run(feed={ - "data1": input1, - "data2": input2, - "data3": input3 - }, - fetch_list=[result_stack]) + (result,) = exe.run( + feed={"data1": input1, "data2": input2, "data3": input3}, + fetch_list=[result_stack], + ) expected_result = np.stack([input1, input2, input3], axis=0) np.testing.assert_allclose(expected_result, result, rtol=1e-05) @@ -249,7 +247,6 @@ def test_single_tensor_error(self): class API_DygraphTest(unittest.TestCase): - def test_out(self): data1 = np.array([[1.0, 2.0]]) data2 = np.array([[3.0, 4.0]]) @@ -277,7 +274,6 @@ def test_single_tensor_error(self): class TestStackOpWithNegativeShape(unittest.TestCase): - def test_out(self): main_prg, startup_prg = Program(), Program() with program_guard(main_prg, startup_prg): @@ -286,23 +282,28 @@ def test_out(self): k = paddle.stack([b, e], axis=0) exe = paddle.static.Executor() exe.run(startup_prg) - out = exe.run(main_prg, - feed={ - 'b': np.ones([ - 3, - ]).astype("int64"), - 'e': np.zeros([ - 3, - ]).astype("int64") - }, - fetch_list=[k]) - np.testing.assert_allclose(out[0], - np.array([[1, 1, 1], [0, 0, 0]]), - rtol=1e-05) + out = exe.run( + main_prg, + feed={ + 'b': np.ones( + [ + 3, + ] + ).astype("int64"), + 'e': np.zeros( + [ + 3, + ] + ).astype("int64"), + }, + fetch_list=[k], + ) + np.testing.assert_allclose( + out[0], np.array([[1, 1, 1], [0, 0, 0]]), rtol=1e-05 + ) class TestStackAPI_ZeroDim(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) diff --git a/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_attention.py b/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_attention.py index 85b9edd2e3310..24ee5a3677556 100644 --- a/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_attention.py +++ b/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_attention.py @@ -23,7 +23,6 @@ class TestStaticModelParallel(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -33,11 +32,14 @@ def _setup_config(self): def test_dist_static_model_parallel_fused_feedforward(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("static_model_parallel_fused_attention.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "static_model_parallel_fused_attention.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_feedforward.py b/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_feedforward.py index d2779eea74923..c6153f2be0b93 100644 --- a/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_feedforward.py +++ b/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_feedforward.py @@ -23,7 +23,6 @@ class TestStaticModelParallel(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -33,11 +32,14 @@ def _setup_config(self): def test_dist_static_model_parallel_fused_feedforward(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): - self.check_with_place("static_model_parallel_fused_feedforward.py", - delta=1e-5, - check_error_log=True, - log_name=flag_name) + self.check_with_place( + "static_model_parallel_fused_feedforward.py", + delta=1e-5, + check_error_log=True, + log_name=flag_name, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_multi_transformer.py b/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_multi_transformer.py index 211920cfebce1..d34ced7b4f416 100644 --- a/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_multi_transformer.py +++ b/python/paddle/fluid/tests/unittests/test_static_model_parallel_fused_multi_transformer.py @@ -23,7 +23,6 @@ class TestStaticModelParallel(TestDistBase): - def _setup_config(self): self._sync_mode = True self._use_reduce = False @@ -33,12 +32,14 @@ def _setup_config(self): def test_dist_static_model_parallel_fused_multi_transformer(self): import paddle.fluid as fluid + if fluid.core.is_compiled_with_cuda(): self.check_with_place( "static_model_parallel_fused_multi_transformer.py", delta=1e-5, check_error_log=True, - log_name=flag_name) + log_name=flag_name, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load.py b/python/paddle/fluid/tests/unittests/test_static_save_load.py index 562477a055275..a0cd268f4846a 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load.py @@ -31,14 +31,15 @@ class SimpleLSTMRNN(fluid.Layer): - - def __init__(self, - name_scope, - hidden_size, - num_steps, - num_layers=2, - init_scale=0.1, - dropout=None): + def __init__( + self, + name_scope, + hidden_size, + num_steps, + num_layers=2, + init_scale=0.1, + dropout=None, + ): super(SimpleLSTMRNN, self).__init__() self._hidden_size = hidden_size self._num_layers = num_layers @@ -58,19 +59,26 @@ def __init__(self, weight_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 2, self._hidden_size * 4], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)) + low=-self._init_scale, high=self._init_scale + ), + ) self.weight_1_arr.append(self.add_parameter('w_%d' % i, weight_1)) bias_1 = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.UniformInitializer( - low=-self._init_scale, high=self._init_scale)), + low=-self._init_scale, high=self._init_scale + ) + ), shape=[self._hidden_size * 4], dtype="float32", - default_initializer=fluid.initializer.Constant(0.0)) + default_initializer=fluid.initializer.Constant(0.0), + ) self.bias_arr.append(self.add_parameter('b_%d' % i, bias_1)) def forward(self, input_embedding, init_hidden=None, init_cell=None): @@ -78,29 +86,29 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): self.hidden_array = [] for i in range(self._num_layers): - pre_hidden = fluid.layers.slice(init_hidden, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_cell = fluid.layers.slice(init_cell, - axes=[0], - starts=[i], - ends=[i + 1]) - pre_hidden = fluid.layers.reshape(pre_hidden, - shape=[-1, self._hidden_size]) - pre_cell = fluid.layers.reshape(pre_cell, - shape=[-1, self._hidden_size]) + pre_hidden = fluid.layers.slice( + init_hidden, axes=[0], starts=[i], ends=[i + 1] + ) + pre_cell = fluid.layers.slice( + init_cell, axes=[0], starts=[i], ends=[i + 1] + ) + pre_hidden = fluid.layers.reshape( + pre_hidden, shape=[-1, self._hidden_size] + ) + pre_cell = fluid.layers.reshape( + pre_cell, shape=[-1, self._hidden_size] + ) self.hidden_array.append(pre_hidden) self.cell_array.append(pre_cell) res = [] for index in range(self._num_steps): - self._input = fluid.layers.slice(input_embedding, - axes=[1], - starts=[index], - ends=[index + 1]) - self._input = fluid.layers.reshape(self._input, - shape=[-1, self._hidden_size]) + self._input = fluid.layers.slice( + input_embedding, axes=[1], starts=[index], ends=[index + 1] + ) + self._input = fluid.layers.reshape( + self._input, shape=[-1, self._hidden_size] + ) for k in range(self._num_layers): pre_hidden = self.hidden_array[k] pre_cell = self.cell_array[k] @@ -111,11 +119,12 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): gate_input = fluid.layers.matmul(x=nn, y=weight_1) gate_input = fluid.layers.elementwise_add(gate_input, bias) - i, j, f, o = fluid.layers.split(gate_input, - num_or_sections=4, - dim=-1) + i, j, f, o = fluid.layers.split( + gate_input, num_or_sections=4, dim=-1 + ) c = pre_cell * fluid.layers.sigmoid(f) + fluid.layers.sigmoid( - i) * fluid.layers.tanh(j) + i + ) * fluid.layers.tanh(j) m = fluid.layers.tanh(c) * fluid.layers.sigmoid(o) self.hidden_array[k] = m self.cell_array[k] = c @@ -125,33 +134,39 @@ def forward(self, input_embedding, init_hidden=None, init_cell=None): self._input = fluid.layers.dropout( self._input, dropout_prob=self._dropout, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) res.append( - fluid.layers.reshape(self._input, - shape=[1, -1, self._hidden_size])) + fluid.layers.reshape( + self._input, shape=[1, -1, self._hidden_size] + ) + ) real_res = fluid.layers.concat(res, 0) real_res = fluid.layers.transpose(x=real_res, perm=[1, 0, 2]) last_hidden = fluid.layers.concat(self.hidden_array, 1) last_hidden = fluid.layers.reshape( - last_hidden, shape=[-1, self._num_layers, self._hidden_size]) + last_hidden, shape=[-1, self._num_layers, self._hidden_size] + ) last_hidden = fluid.layers.transpose(x=last_hidden, perm=[1, 0, 2]) last_cell = fluid.layers.concat(self.cell_array, 1) last_cell = fluid.layers.reshape( - last_cell, shape=[-1, self._num_layers, self._hidden_size]) + last_cell, shape=[-1, self._num_layers, self._hidden_size] + ) last_cell = fluid.layers.transpose(x=last_cell, perm=[1, 0, 2]) return real_res, last_hidden, last_cell class PtbModel(fluid.Layer): - - def __init__(self, - name_scope, - hidden_size, - vocab_size, - num_layers=2, - num_steps=20, - init_scale=0.1, - dropout=None): + def __init__( + self, + name_scope, + hidden_size, + vocab_size, + num_layers=2, + num_steps=20, + init_scale=0.1, + dropout=None, + ): super(PtbModel, self).__init__() self.hidden_size = hidden_size self.vocab_size = vocab_size @@ -159,61 +174,77 @@ def __init__(self, self.num_layers = num_layers self.num_steps = num_steps self.dropout = dropout - self.simple_lstm_rnn = SimpleLSTMRNN(self.full_name(), - hidden_size, - num_steps, - num_layers=num_layers, - init_scale=init_scale, - dropout=dropout) + self.simple_lstm_rnn = SimpleLSTMRNN( + self.full_name(), + hidden_size, + num_steps, + num_layers=num_layers, + init_scale=init_scale, + dropout=dropout, + ) self.embedding = paddle.nn.Embedding( num_embeddings=vocab_size, embedding_dim=hidden_size, weight_attr=fluid.ParamAttr( name='embedding_para', initializer=fluid.initializer.UniformInitializer( - low=-init_scale, high=init_scale))) + low=-init_scale, high=init_scale + ), + ), + ) self.softmax_weight = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.hidden_size, self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) self.softmax_bias = self.create_parameter( attr=fluid.ParamAttr(), shape=[self.vocab_size], dtype="float32", default_initializer=fluid.initializer.UniformInitializer( - low=-self.init_scale, high=self.init_scale)) + low=-self.init_scale, high=self.init_scale + ), + ) def forward(self, input, label, init_hidden, init_cell): init_h = fluid.layers.reshape( - init_hidden, shape=[self.num_layers, -1, self.hidden_size]) + init_hidden, shape=[self.num_layers, -1, self.hidden_size] + ) init_c = fluid.layers.reshape( - init_cell, shape=[self.num_layers, -1, self.hidden_size]) + init_cell, shape=[self.num_layers, -1, self.hidden_size] + ) # NPU 'tok_k' kernel only support `int32` dtype, so cast `input` from `int64` to `int32`. input = fluid.layers.cast(input, "int32") x_emb = self.embedding(input) x_emb = fluid.layers.reshape( - x_emb, shape=[-1, self.num_steps, self.hidden_size]) + x_emb, shape=[-1, self.num_steps, self.hidden_size] + ) if self.dropout is not None and self.dropout > 0.0: x_emb = fluid.layers.dropout( x_emb, dropout_prob=self.drop_out, - dropout_implementation='upscale_in_train') + dropout_implementation='upscale_in_train', + ) rnn_out, last_hidden, last_cell = self.simple_lstm_rnn( - x_emb, init_h, init_c) + x_emb, init_h, init_c + ) rnn_out = fluid.layers.reshape( - rnn_out, shape=[-1, self.num_steps, self.hidden_size]) + rnn_out, shape=[-1, self.num_steps, self.hidden_size] + ) projection = fluid.layers.matmul(rnn_out, self.softmax_weight) projection = fluid.layers.elementwise_add(projection, self.softmax_bias) - projection = fluid.layers.reshape(projection, - shape=[-1, self.vocab_size]) - loss = fluid.layers.softmax_with_cross_entropy(logits=projection, - label=label, - soft_label=False) + projection = fluid.layers.reshape( + projection, shape=[-1, self.vocab_size] + ) + loss = fluid.layers.softmax_with_cross_entropy( + logits=projection, label=label, soft_label=False + ) loss = fluid.layers.reshape(loss, shape=[-1, self.num_steps]) loss = fluid.layers.reduce_mean(loss, dim=[0]) loss = fluid.layers.reduce_sum(loss) @@ -222,10 +253,12 @@ def forward(self, input, label, init_hidden, init_cell): class TestSaveLoadBase(unittest.TestCase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_ptb_rnn_cpu_float32(self): seed = 90 @@ -241,29 +274,32 @@ def test_ptb_rnn_cpu_float32(self): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) sgd.minimize(static_loss) static_param_updated = dict() static_param_init = dict() @@ -279,18 +315,22 @@ def test_ptb_rnn_cpu_float32(self): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -300,8 +340,9 @@ def test_ptb_rnn_cpu_float32(self): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -314,28 +355,35 @@ def test_ptb_rnn_cpu_float32(self): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, - os.path.join(temp_dir.name, "test_1.pdparams"), exe) + fluid.load( + main_program, + os.path.join(temp_dir.name, "test_1.pdparams"), + exe, + ) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) temp_dir.cleanup() class TestSaveLoadPartial(unittest.TestCase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_ptb_rnn_cpu_float32(self): seed = 90 @@ -351,36 +399,41 @@ def test_ptb_rnn_cpu_float32(self): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) test_program = fluid.default_main_program().clone(for_test=True) - add_1 = fluid.layers.fc(static_last_hidden, - size=hidden_size, - num_flatten_dims=2, - bias_attr=False) + add_1 = fluid.layers.fc( + static_last_hidden, + size=hidden_size, + num_flatten_dims=2, + bias_attr=False, + ) sgd.minimize(static_loss) static_param_updated = dict() @@ -397,18 +450,22 @@ def test_ptb_rnn_cpu_float32(self): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -418,8 +475,9 @@ def test_ptb_rnn_cpu_float32(self): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -432,30 +490,38 @@ def test_ptb_rnn_cpu_float32(self): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(test_program, os.path.join(temp_dir.name, - "test_1.pdopt"), None) + fluid.load( + test_program, os.path.join(temp_dir.name, "test_1.pdopt"), None + ) for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) - fluid.load(test_program, - os.path.join(temp_dir.name, "test_1.pdmodel"), None) + fluid.load( + test_program, + os.path.join(temp_dir.name, "test_1.pdmodel"), + None, + ) temp_dir.cleanup() class TestSaveLoadSetStateDict(unittest.TestCase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_ptb_rnn_cpu_float32(self): seed = 90 @@ -471,29 +537,32 @@ def test_ptb_rnn_cpu_float32(self): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) sgd.minimize(static_loss) static_param_updated = dict() static_param_init = dict() @@ -509,18 +578,22 @@ def test_ptb_rnn_cpu_float32(self): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -530,8 +603,9 @@ def test_ptb_rnn_cpu_float32(self): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -544,8 +618,9 @@ def test_ptb_rnn_cpu_float32(self): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) @@ -553,18 +628,21 @@ def test_ptb_rnn_cpu_float32(self): for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) temp_dir.cleanup() class TestProgramStatePartial(unittest.TestCase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_ptb_rnn_cpu_float32(self): seed = 90 @@ -580,36 +658,41 @@ def test_ptb_rnn_cpu_float32(self): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) test_program = fluid.default_main_program().clone(for_test=True) - add_1 = fluid.layers.fc(static_last_hidden, - size=hidden_size, - num_flatten_dims=2, - bias_attr=False) + add_1 = fluid.layers.fc( + static_last_hidden, + size=hidden_size, + num_flatten_dims=2, + bias_attr=False, + ) sgd.minimize(static_loss) static_param_updated = dict() @@ -626,18 +709,22 @@ def test_ptb_rnn_cpu_float32(self): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -647,8 +734,9 @@ def test_ptb_rnn_cpu_float32(self): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -661,30 +749,36 @@ def test_ptb_rnn_cpu_float32(self): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - #fluid.load(test_program, "./test_1", None ) + # fluid.load(test_program, "./test_1", None ) program_state = fluid.load_program_state( - os.path.join(temp_dir.name, 'test_1')) + os.path.join(temp_dir.name, 'test_1') + ) program_state_1 = fluid.load_program_state( - os.path.join(temp_dir.name, 'test_1.pdparams')) + os.path.join(temp_dir.name, 'test_1.pdparams') + ) program_state_2 = fluid.load_program_state( - os.path.join(temp_dir.name, 'test_1.pdopt')) + os.path.join(temp_dir.name, 'test_1.pdopt') + ) program_state_3 = fluid.load_program_state( - os.path.join(temp_dir.name, 'test_1.pdmodel')) + os.path.join(temp_dir.name, 'test_1.pdmodel') + ) fluid.set_program_state(test_program, program_state) for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -694,8 +788,9 @@ def test_ptb_rnn_cpu_float32(self): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) @@ -703,8 +798,9 @@ def test_ptb_rnn_cpu_float32(self): for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -714,8 +810,9 @@ def test_ptb_rnn_cpu_float32(self): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) @@ -723,8 +820,9 @@ def test_ptb_rnn_cpu_float32(self): for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -734,8 +832,9 @@ def test_ptb_rnn_cpu_float32(self): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) @@ -743,18 +842,21 @@ def test_ptb_rnn_cpu_float32(self): for var in test_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) temp_dir.cleanup() class TestVariableInit(unittest.TestCase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_variable_init(self): @@ -767,8 +869,10 @@ def test_variable_init(self): exe.run(fluid.default_startup_program()) temp_dir = tempfile.TemporaryDirectory() - fluid.save(fluid.default_main_program(), - os.path.join(temp_dir.name, "test_path")) + fluid.save( + fluid.default_main_program(), + os.path.join(temp_dir.name, "test_path"), + ) def set_var(var, ndarray): t = var.get_tensor() @@ -789,35 +893,43 @@ def set_var(var, ndarray): place = self.set_place() exe = fluid.Executor(place) - parameter_list = list(filter(fluid.io.is_parameter, - program.list_vars())) + parameter_list = list( + filter(fluid.io.is_parameter, program.list_vars()) + ) - fluid.core._create_loaded_parameter(parameter_list, new_scope, - exe._default_executor) + fluid.core._create_loaded_parameter( + parameter_list, new_scope, exe._default_executor + ) parameter_file_name = os.path.join(temp_dir.name, "test_path.pdparams") with open(parameter_file_name, 'rb') as f: load_dict = pickle.load(f) for v in parameter_list: - assert v.name in load_dict, \ - "Can not find [{}] in model file [{}]".format( - v.name, parameter_file_name) + assert ( + v.name in load_dict + ), "Can not find [{}] in model file [{}]".format( + v.name, parameter_file_name + ) new_v = new_scope.find_var(v.name) set_var(new_v, load_dict[v.name]) opt_list = list( - filter(fluid.io.is_belong_to_optimizer, program.list_vars())) + filter(fluid.io.is_belong_to_optimizer, program.list_vars()) + ) - fluid.core._create_loaded_parameter(opt_list, new_scope, - exe._default_executor) + fluid.core._create_loaded_parameter( + opt_list, new_scope, exe._default_executor + ) opt_file_name = os.path.join(temp_dir.name, "test_path.pdopt") with open(opt_file_name, 'rb') as f: load_dict = pickle.load(f) for v in opt_list: - assert v.name in load_dict, \ - "Can not find [{}] in model file [{}]".format( - v.name, opt_file_name) + assert ( + v.name in load_dict + ), "Can not find [{}] in model file [{}]".format( + v.name, opt_file_name + ) new_v = new_scope.find_var(v.name) set_var(new_v, load_dict[v.name]) @@ -825,8 +937,9 @@ def set_var(var, ndarray): base_map = {} for var in program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update base_map[var.name] = t @@ -840,7 +953,6 @@ def set_var(var, ndarray): class TestLoadFromOldInterface(unittest.TestCase): - def setUp(self): if os.path.exists("test_path.pdparams"): os.remove("test_path.pdparams") @@ -851,8 +963,11 @@ def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def tearDown(self): self.temp_dir.cleanup() @@ -870,29 +985,32 @@ def test_load_from_old_interface(self): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) test_clone_program = fluid.default_main_program().clone() sgd.minimize(static_loss) @@ -910,18 +1028,22 @@ def test_load_from_old_interface(self): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -931,16 +1053,17 @@ def test_load_from_old_interface(self): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - #fluid.save(main_program, "./test_1") + # fluid.save(main_program, "./test_1") fluid.io.save_persistables( - exe, os.path.join(self.temp_dir.name, "test_path"), - main_program) + exe, os.path.join(self.temp_dir.name, "test_path"), main_program + ) # set var to zero for var in main_program.list_vars(): @@ -948,18 +1071,21 @@ def test_load_from_old_interface(self): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, - os.path.join(self.temp_dir.name, "test_path"), exe) + fluid.load( + main_program, os.path.join(self.temp_dir.name, "test_path"), exe + ) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -971,13 +1097,19 @@ def test_load_from_old_interface(self): var.desc.set_shape(new_shape) with self.assertRaises(RuntimeError): - fluid.load(main_program, - os.path.join(self.temp_dir.name, "test_path"), exe) + fluid.load( + main_program, + os.path.join(self.temp_dir.name, "test_path"), + exe, + ) # check unused parameter - fluid.load(test_clone_program, - os.path.join(self.temp_dir.name, "test_path"), exe) + fluid.load( + test_clone_program, + os.path.join(self.temp_dir.name, "test_path"), + exe, + ) def test_load_from_old_interface_var_list(self): seed = 90 @@ -992,29 +1124,32 @@ def test_load_from_old_interface_var_list(self): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) test_clone_program = fluid.default_main_program().clone() sgd.minimize(static_loss) @@ -1032,18 +1167,22 @@ def test_load_from_old_interface_var_list(self): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -1053,17 +1192,19 @@ def test_load_from_old_interface_var_list(self): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t - #fluid.save(main_program, "./test_1") + # fluid.save(main_program, "./test_1") fluid.io.save_persistables( exe, os.path.join(self.temp_dir.name, "test_static_load_var_list"), - main_program) + main_program, + ) # set var to zero var_list = [] @@ -1074,34 +1215,40 @@ def test_load_from_old_interface_var_list(self): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) fluid.load( main_program, os.path.join(self.temp_dir.name, "test_static_load_var_list"), - exe, var_list) + exe, + var_list, + ) var_list_names = [var.name for var in var_list] for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) if var.name in var_list_names: # loaded vars base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) else: - #not loaded vars + # not loaded vars self.assertTrue(np.sum(np.abs(new_t)) == 0) class TestLoadFromOldInterfaceSingleFile(unittest.TestCase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_load_from_old_interface(self): seed = 90 @@ -1117,29 +1264,32 @@ def test_load_from_old_interface(self): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) sgd.minimize(static_loss) static_param_updated = dict() static_param_init = dict() @@ -1155,18 +1305,22 @@ def test_load_from_old_interface(self): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -1176,17 +1330,17 @@ def test_load_from_old_interface(self): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t save_dir = os.path.join(temp_dir.name, "test_path") - #fluid.save(main_program, "./test_1") - fluid.io.save_persistables(exe, - save_dir, - main_program, - filename="model_single") + # fluid.save(main_program, "./test_1") + fluid.io.save_persistables( + exe, save_dir, main_program, filename="model_single" + ) # set var to zero for var in main_program.list_vars(): @@ -1194,19 +1348,25 @@ def test_load_from_old_interface(self): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) file_model_path = os.path.join(save_dir, "model_single") - fluid.load(main_program, file_model_path, exe, - fluid.io.get_program_persistable_vars(main_program)) + fluid.load( + main_program, + file_model_path, + exe, + fluid.io.get_program_persistable_vars(main_program), + ) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -1221,21 +1381,32 @@ def test_load_from_old_interface(self): var.desc.set_shape(new_shape) with self.assertRaises(RuntimeError): - fluid.load(main_program, file_model_path, exe, - fluid.io.get_program_persistable_vars(main_program)) - - fluid.io.save_params(exe, - "test_path", - main_program, - filename="model_single") + fluid.load( + main_program, + file_model_path, + exe, + fluid.io.get_program_persistable_vars(main_program), + ) + + fluid.io.save_params( + exe, "test_path", main_program, filename="model_single" + ) with self.assertRaises(RuntimeError): - fluid.load(main_program, file_model_path, exe, - fluid.io.get_program_persistable_vars(main_program)) + fluid.load( + main_program, + file_model_path, + exe, + fluid.io.get_program_persistable_vars(main_program), + ) # check when executor is None with self.assertRaises(ValueError): - fluid.load(main_program, file_model_path, None, - fluid.io.get_program_persistable_vars(main_program)) + fluid.load( + main_program, + file_model_path, + None, + fluid.io.get_program_persistable_vars(main_program), + ) # check when var list is None with self.assertRaises(ValueError): @@ -1243,17 +1414,20 @@ def test_load_from_old_interface(self): # check save params, load var_list = get_program_persistable_vars with self.assertRaises(RuntimeError): - temp_var = framework.Variable(main_program.global_block(), - shape=[1], - name="test_temp_var") + temp_var = framework.Variable( + main_program.global_block(), shape=[1], name="test_temp_var" + ) all_var_list = list(main_program.list_vars()) - fluid.load(main_program, file_model_path, exe, - all_var_list + [temp_var]) + fluid.load( + main_program, + file_model_path, + exe, + all_var_list + [temp_var], + ) temp_dir.cleanup() class TestProgramStateOldSave(unittest.TestCase): - def setUp(self): self.test_dygraph = True self.temp_dir = tempfile.TemporaryDirectory() @@ -1262,8 +1436,11 @@ def tearDown(self): self.temp_dir.cleanup() def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_ptb_rnn_cpu_float32(self): seed = 90 @@ -1278,36 +1455,41 @@ def test_ptb_rnn_cpu_float32(self): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) test_program = fluid.default_main_program().clone(for_test=True) - add_1 = fluid.layers.fc(static_last_hidden, - size=hidden_size, - num_flatten_dims=2, - bias_attr=False) + add_1 = fluid.layers.fc( + static_last_hidden, + size=hidden_size, + num_flatten_dims=2, + bias_attr=False, + ) sgd.minimize(static_loss) static_param_updated = dict() @@ -1324,18 +1506,22 @@ def test_ptb_rnn_cpu_float32(self): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -1345,8 +1531,9 @@ def test_ptb_rnn_cpu_float32(self): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -1359,8 +1546,9 @@ def test_ptb_rnn_cpu_float32(self): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) @@ -1386,7 +1574,8 @@ def symlink_force(target, link_name): # case 3: load with var_list program_state = fluid.load_program_state( - save_dir, main_program.all_parameters()) + save_dir, main_program.all_parameters() + ) fluid.set_program_state(main_program, program_state) self.check_in_static(main_program, base_map) @@ -1402,23 +1591,27 @@ def create_symlink(self, target, link_name): os.symlink(target, link_name) except AttributeError: import ctypes + kernel_dll = ctypes.windll.LoadLibrary("kernel32.dll") kernel_dll.CreateSymbolicLinkA(target, link_name, 0) def check_in_static(self, main_program, base_map): for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) class TestProgramStateOldSaveSingleModel(unittest.TestCase): - def set_place(self): - return fluid.CPUPlace( - ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) + return ( + fluid.CPUPlace() + if not core.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) def test_ptb_rnn_cpu_float32(self): seed = 90 @@ -1434,36 +1627,41 @@ def test_ptb_rnn_cpu_float32(self): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = Adam(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) test_program = fluid.default_main_program().clone(for_test=True) - add_1 = fluid.layers.fc(static_last_hidden, - size=hidden_size, - num_flatten_dims=2, - bias_attr=False) + add_1 = fluid.layers.fc( + static_last_hidden, + size=hidden_size, + num_flatten_dims=2, + bias_attr=False, + ) sgd.minimize(static_loss) static_param_updated = dict() @@ -1480,18 +1678,22 @@ def test_ptb_rnn_cpu_float32(self): x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='float32') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='float32') + (num_layers, batch_size, hidden_size), dtype='float32' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='float32' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) static_loss_value = out[0] static_last_hidden_value = out[1] static_last_cell_value = out[2] @@ -1501,17 +1703,17 @@ def test_ptb_rnn_cpu_float32(self): base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t save_dir = os.path.join(temp_dir.name, "test_program_2") - fluid.io.save_persistables(exe, - save_dir, - main_program, - filename="model_1") + fluid.io.save_persistables( + exe, save_dir, main_program, filename="model_1" + ) # set var to zero for var in main_program.list_vars(): @@ -1519,21 +1721,24 @@ def test_ptb_rnn_cpu_float32(self): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - #fluid.load(test_program, "./test_1", None ) + # fluid.load(test_program, "./test_1", None ) program_state = fluid.load_program_state( os.path.join(save_dir, "model_1"), - var_list=fluid.io.get_program_persistable_vars(main_program)) + var_list=fluid.io.get_program_persistable_vars(main_program), + ) fluid.set_program_state(main_program, program_state) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -1541,30 +1746,34 @@ def test_ptb_rnn_cpu_float32(self): fluid.load_program_state(os.path.join(save_dir, "model_1")) with self.assertRaises(TypeError): - fluid.load_program_state(os.path.join(save_dir, "model_1"), - var_list=["str"]) + fluid.load_program_state( + os.path.join(save_dir, "model_1"), var_list=["str"] + ) with self.assertRaises(RuntimeError): fluid.load_program_state( os.path.join(save_dir, "model_1"), var_list=[ main_program.global_block().create_var( - name="fake_var_name", persistable=True) - ]) + name="fake_var_name", persistable=True + ) + ], + ) temp_dir.cleanup() class TestStaticSaveLoadPickle(unittest.TestCase): - def test_pickle_protocol(self): # enable static mode paddle.enable_static() with new_program_scope(): # create network - x = paddle.static.data(name="static_save_load_large_x", - shape=[None, 10], - dtype='float32') + x = paddle.static.data( + name="static_save_load_large_x", + shape=[None, 10], + dtype='float32', + ) z = paddle.static.nn.fc(x, 10, bias_attr=False) place = paddle.CPUPlace() exe = paddle.static.Executor(place) @@ -1574,15 +1783,17 @@ def test_pickle_protocol(self): base_map = {} for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t temp_dir = tempfile.TemporaryDirectory() - path = os.path.join(temp_dir.name, "test_static_save_load_pickle", - "pickle_protocol") + path = os.path.join( + temp_dir.name, "test_static_save_load_pickle", "pickle_protocol" + ) with self.assertRaises(ValueError): paddle.fluid.save(prog, path, 2.0) @@ -1603,26 +1814,28 @@ def test_pickle_protocol(self): # set var to zero for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - ten = fluid.global_scope().find_var( - var.name).get_tensor() + ten = ( + fluid.global_scope().find_var(var.name).get_tensor() + ) ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) self.assertTrue(np.sum(np.abs(new_t)) == 0) paddle.fluid.load(prog, path) for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) class TestSaveLoadInferenceModel(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.model_path = os.path.join(self.temp_dir.name, 'no_params') @@ -1641,8 +1854,11 @@ def test_no_params(self): paddle.static.save_inference_model(self.model_path, [x], [y], exe) - [inference_program, feed_target_names, fetch_targets - ] = (paddle.static.load_inference_model(self.model_path, exe)) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(self.model_path, exe) self.assertEqual(feed_target_names, ['x']) self.assertEqual(fetch_targets[0].shape, (10, 10)) diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py b/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py index 3e3f41e58d2e4..fe578907b866e 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load_bf16.py @@ -25,10 +25,10 @@ import os -@unittest.skipIf(not core.supports_bfloat16(), - "place does not support BF16 evaluation") +@unittest.skipIf( + not core.supports_bfloat16(), "place does not support BF16 evaluation" +) class TestSaveLoadBF16(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -51,36 +51,41 @@ def test_ptb_rnn_cpu_bfloat16(self): with new_program_scope(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed - ptb_model = PtbModel("ptb_model", - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale) + ptb_model = PtbModel( + "ptb_model", + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + ) place = self.set_place() exe = fluid.Executor(place) sgd = SGDOptimizer(learning_rate=1e-3) - x = fluid.layers.data(name="x", - shape=[-1, num_steps], - dtype='int64') + x = fluid.layers.data( + name="x", shape=[-1, num_steps], dtype='int64' + ) y = fluid.layers.data(name="y", shape=[-1, 1], dtype='float32') - init_hidden = fluid.layers.data(name="init_hidden", - shape=[1], - dtype='float32') - init_cell = fluid.layers.data(name="init_cell", - shape=[1], - dtype='float32') + init_hidden = fluid.layers.data( + name="init_hidden", shape=[1], dtype='float32' + ) + init_cell = fluid.layers.data( + name="init_cell", shape=[1], dtype='float32' + ) static_loss, static_last_hidden, static_last_cell = ptb_model( - x, y, init_hidden, init_cell) + x, y, init_hidden, init_cell + ) sgd = paddle.static.amp.bf16.decorate_bf16( sgd, amp_lists=paddle.static.amp.bf16.AutoMixedPrecisionListsBF16( - custom_fp32_list={'transpose2', 'concat'}), + custom_fp32_list={'transpose2', 'concat'} + ), use_bf16_guard=False, - use_pure_bf16=True) + use_pure_bf16=True, + ) sgd.minimize(static_loss, framework.default_startup_program()) out = exe.run(framework.default_startup_program()) @@ -90,30 +95,35 @@ def test_ptb_rnn_cpu_bfloat16(self): y_data = np.arange(1, 13).reshape(4, 3).astype('int64') x_data = x_data.reshape((-1, num_steps, 1)) y_data = y_data.reshape((-1, 1)) - #TODO investigate initializing model with "float32" instead of "uint16" as it was before + # TODO investigate initializing model with "float32" instead of "uint16" as it was before # slice_op PR(datatypes in model graph are different than datatypes during runtime because of that) init_hidden_data = np.zeros( - (num_layers, batch_size, hidden_size), dtype='uint16') - init_cell_data = np.zeros((num_layers, batch_size, hidden_size), - dtype='uint16') + (num_layers, batch_size, hidden_size), dtype='uint16' + ) + init_cell_data = np.zeros( + (num_layers, batch_size, hidden_size), dtype='uint16' + ) fetch_list = [static_loss, static_last_hidden, static_last_cell] - out = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": y_data, - "init_hidden": init_hidden_data, - "init_cell": init_cell_data - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": y_data, + "init_hidden": init_hidden_data, + "init_cell": init_cell_data, + }, + fetch_list=fetch_list, + ) # get value before save main_program = framework.default_main_program() base_map = {} for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t @@ -126,18 +136,23 @@ def test_ptb_rnn_cpu_bfloat16(self): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been set to zero self.assertTrue(np.sum(np.abs(new_t)) == 0) - fluid.load(main_program, - os.path.join(self.temp_dir.name, "test_1.pdparams"), exe) + fluid.load( + main_program, + os.path.join(self.temp_dir.name, "test_1.pdparams"), + exe, + ) for var in main_program.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) diff --git a/python/paddle/fluid/tests/unittests/test_static_save_load_large.py b/python/paddle/fluid/tests/unittests/test_static_save_load_large.py index 61568b50c1ba1..ddae3373b5a8a 100644 --- a/python/paddle/fluid/tests/unittests/test_static_save_load_large.py +++ b/python/paddle/fluid/tests/unittests/test_static_save_load_large.py @@ -26,15 +26,16 @@ class TestStaticSaveLoadLargeParameters(unittest.TestCase): - def test_large_parameters_static_save(self): # enable static mode paddle.enable_static() with new_program_scope(): # create network - x = paddle.static.data(name="static_save_load_large_x", - shape=[None, 10], - dtype='float32') + x = paddle.static.data( + name="static_save_load_large_x", + shape=[None, 10], + dtype='float32', + ) z = paddle.static.nn.fc(x, LARGE_PARAM, bias_attr=False) place = paddle.CPUPlace() exe = paddle.static.Executor(place) @@ -44,14 +45,16 @@ def test_large_parameters_static_save(self): base_map = {} for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) # make sure all the paramerter or optimizer var have been update self.assertTrue(np.sum(np.abs(t)) != 0) base_map[var.name] = t temp_dir = tempfile.TemporaryDirectory() - path = os.path.join(temp_dir.name, - "test_static_save_load_large_param") + path = os.path.join( + temp_dir.name, "test_static_save_load_large_param" + ) path = os.path.join(path, "static_save") protocol = 4 paddle.fluid.save(prog, path, pickle_protocol=protocol) @@ -61,16 +64,18 @@ def test_large_parameters_static_save(self): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) self.assertTrue(np.sum(np.abs(new_t)) == 0) paddle.fluid.load(prog, path) for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) @@ -80,16 +85,18 @@ def test_large_parameters_static_save(self): ten = fluid.global_scope().find_var(var.name).get_tensor() ten.set(np.zeros_like(np.array(ten)), place) - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) self.assertTrue(np.sum(np.abs(new_t)) == 0) program_state = fluid.load_program_state(path) fluid.set_program_state(prog, program_state) for var in prog.list_vars(): if isinstance(var, framework.Parameter) or var.persistable: - new_t = np.array(fluid.global_scope().find_var( - var.name).get_tensor()) + new_t = np.array( + fluid.global_scope().find_var(var.name).get_tensor() + ) base_t = base_map[var.name] np.testing.assert_array_equal(new_t, base_t) temp_dir.cleanup() diff --git a/python/paddle/fluid/tests/unittests/test_static_shape_inferrence_for_shape_tensor.py b/python/paddle/fluid/tests/unittests/test_static_shape_inferrence_for_shape_tensor.py index 0e22905e81d69..c6fd490cb60dd 100644 --- a/python/paddle/fluid/tests/unittests/test_static_shape_inferrence_for_shape_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_static_shape_inferrence_for_shape_tensor.py @@ -17,12 +17,11 @@ class StaticShapeInferrenceTest(unittest.TestCase): - def test_static_graph(self): paddle.enable_static() - data = paddle.fluid.layers.data(name="x", - shape=[-1, 2], - dtype='float32') + data = paddle.fluid.layers.data( + name="x", shape=[-1, 2], dtype='float32' + ) shape = paddle.fluid.layers.shape(data) # shape should be [-1, 2] x = paddle.fluid.layers.uniform_random(shape) self.assertEqual(x.shape, data.shape) diff --git a/python/paddle/fluid/tests/unittests/test_std_layer.py b/python/paddle/fluid/tests/unittests/test_std_layer.py index a724e320c9545..cc3e7740f4593 100644 --- a/python/paddle/fluid/tests/unittests/test_std_layer.py +++ b/python/paddle/fluid/tests/unittests/test_std_layer.py @@ -20,14 +20,13 @@ def ref_std(x, axis=None, unbiased=True, keepdim=False): ddof = 1 if unbiased else 0 if isinstance(axis, int): - axis = (axis, ) + axis = (axis,) if axis is not None: axis = tuple(axis) return np.std(x, axis=axis, ddof=ddof, keepdims=keepdim) class TestStdAPI(unittest.TestCase): - def setUp(self): self.dtype = 'float64' self.shape = [1, 3, 4, 10] @@ -36,9 +35,11 @@ def setUp(self): self.unbiased = True self.set_attrs() self.x = np.random.uniform(-1, 1, self.shape).astype(self.dtype) - self.place=paddle.CUDAPlace(0) \ - if paddle.fluid.core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def set_attrs(self): pass @@ -68,43 +69,36 @@ def test_api(self): class TestStdAPI_dtype(TestStdAPI): - def set_attrs(self): self.dtype = 'float32' class TestStdAPI_axis_int(TestStdAPI): - def set_attrs(self): self.axis = 2 class TestStdAPI_axis_list(TestStdAPI): - def set_attrs(self): self.axis = [1, 2] class TestStdAPI_axis_tuple(TestStdAPI): - def set_attrs(self): self.axis = (1, 3) class TestStdAPI_keepdim(TestStdAPI): - def set_attrs(self): self.keepdim = False class TestStdAPI_unbiased(TestStdAPI): - def set_attrs(self): self.unbiased = False class TestStdAPI_alias(unittest.TestCase): - def test_alias(self): paddle.disable_static() x = paddle.to_tensor(np.array([10, 12], 'float32')) @@ -117,7 +111,6 @@ def test_alias(self): class TestStdError(unittest.TestCase): - def test_error(self): with paddle.static.program_guard(paddle.static.Program()): x = paddle.fluid.data('X', [2, 3, 4], 'int32') diff --git a/python/paddle/fluid/tests/unittests/test_stft_op.py b/python/paddle/fluid/tests/unittests/test_stft_op.py index 8110f1d805fbb..91206c3da1261 100644 --- a/python/paddle/fluid/tests/unittests/test_stft_op.py +++ b/python/paddle/fluid/tests/unittests/test_stft_op.py @@ -45,14 +45,14 @@ def frame_from_librosa(x, frame_length, hop_length, axis=-1): def stft_np(x, window, n_fft, hop_length, **kwargs): frames = frame_from_librosa(x, n_fft, hop_length) - frames = np.multiply(frames.transpose([0, 2, 1]), - window).transpose([0, 2, 1]) + frames = np.multiply(frames.transpose([0, 2, 1]), window).transpose( + [0, 2, 1] + ) res = np.fft.rfft(frames, axis=1) return res class TestStftOp(OpTest): - def setUp(self): self.op_type = "stft" self.shape, self.type, self.attrs = self.initTestCase() @@ -61,10 +61,9 @@ def setUp(self): 'Window': np.hamming(self.attrs['n_fft']).astype(self.type), } self.outputs = { - 'Out': - stft_np(x=self.inputs['X'], - window=self.inputs['Window'], - **self.attrs) + 'Out': stft_np( + x=self.inputs['X'], window=self.inputs['Window'], **self.attrs + ) } def initTestCase(self): diff --git a/python/paddle/fluid/tests/unittests/test_strided_slice_op.py b/python/paddle/fluid/tests/unittests/test_strided_slice_op.py index 05f4e4a994168..7ce7db1ba7850 100644 --- a/python/paddle/fluid/tests/unittests/test_strided_slice_op.py +++ b/python/paddle/fluid/tests/unittests/test_strided_slice_op.py @@ -37,32 +37,51 @@ def strided_slice_native_forward(input, axes, starts, ends, strides): stride[axes[i]] = strides[i] result = { - 1: lambda input, start, end, stride: input[start[0]:end[0]:stride[0]], - 2: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1]], - 3: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2]], - 4: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2], start[3]:end[3]:stride[3]], - 5: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2], start[3]:end[3]:stride[3], start[4]:end[4]:stride[4]], - 6: lambda input, start, end, stride: input[start[0]:end[0]:stride[0], \ - start[1]:end[1]:stride[1], start[2]:end[2]:stride[2], start[3]:end[3]:stride[3], \ - start[4]:end[4]:stride[4], start[5]:end[5]:stride[5]] + 1: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0] + ], + 2: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], start[1] : end[1] : stride[1] + ], + 3: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + ], + 4: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + start[3] : end[3] : stride[3], + ], + 5: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + start[3] : end[3] : stride[3], + start[4] : end[4] : stride[4], + ], + 6: lambda input, start, end, stride: input[ + start[0] : end[0] : stride[0], + start[1] : end[1] : stride[1], + start[2] : end[2] : stride[2], + start[3] : end[3] : stride[3], + start[4] : end[4] : stride[4], + start[5] : end[5] : stride[5], + ], }[dim](input, start, end, stride) return result class TestStrideSliceOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'strided_slice' self.python_api = paddle.strided_slice - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.inputs = {'Input': self.input} self.outputs = {'Out': self.output} @@ -71,7 +90,7 @@ def setUp(self): 'starts': self.starts, 'ends': self.ends, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def test_check_output(self): @@ -90,7 +109,6 @@ def initTestCase(self): class TestStrideSliceOp1(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(100) self.axes = [0] @@ -101,7 +119,6 @@ def initTestCase(self): class TestStrideSliceOp2(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(100) self.axes = [0] @@ -112,7 +129,6 @@ def initTestCase(self): class TestStrideSliceOp3(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(100) self.axes = [0] @@ -123,7 +139,6 @@ def initTestCase(self): class TestStrideSliceOp4(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 4, 10) self.axes = [0, 1, 2] @@ -134,7 +149,6 @@ def initTestCase(self): class TestStrideSliceOp5(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5) self.axes = [0, 1, 2] @@ -145,7 +159,6 @@ def initTestCase(self): class TestStrideSliceOp6(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5) self.axes = [0, 1, 2] @@ -156,7 +169,6 @@ def initTestCase(self): class TestStrideSliceOp7(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(5, 5, 5) self.axes = [0, 1, 2] @@ -167,7 +179,6 @@ def initTestCase(self): class TestStrideSliceOp8(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(1, 100, 1) self.axes = [1] @@ -178,7 +189,6 @@ def initTestCase(self): class TestStrideSliceOp9(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(1, 100, 1) self.axes = [1] @@ -189,7 +199,6 @@ def initTestCase(self): class TestStrideSliceOp10(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(10, 10) self.axes = [0, 1] @@ -200,7 +209,6 @@ def initTestCase(self): class TestStrideSliceOp11(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4) self.axes = [0, 1, 2, 3] @@ -211,7 +219,6 @@ def initTestCase(self): class TestStrideSliceOp12(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4, 5) self.axes = [0, 1, 2, 3, 4] @@ -222,7 +229,6 @@ def initTestCase(self): class TestStrideSliceOp13(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 6, 7, 8) self.axes = [0, 1, 2, 3, 4, 5] @@ -233,7 +239,6 @@ def initTestCase(self): class TestStrideSliceOp14(TestStrideSliceOp): - def initTestCase(self): self.input = np.random.rand(4, 4, 4, 4) self.axes = [1, 2, 3] @@ -244,13 +249,11 @@ def initTestCase(self): class TestStrideSliceOpBool(TestStrideSliceOp): - def test_check_grad(self): pass class TestStrideSliceOpBool1D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(100).astype("bool") self.axes = [0] @@ -261,7 +264,6 @@ def initTestCase(self): class TestStrideSliceOpBool2D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(10, 10).astype("bool") self.axes = [0, 1] @@ -272,7 +274,6 @@ def initTestCase(self): class TestStrideSliceOpBool3D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 4, 10).astype("bool") self.axes = [0, 1, 2] @@ -283,7 +284,6 @@ def initTestCase(self): class TestStrideSliceOpBool4D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4).astype("bool") self.axes = [0, 1, 2, 3] @@ -294,7 +294,6 @@ def initTestCase(self): class TestStrideSliceOpBool5D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 4, 5).astype("bool") self.axes = [0, 1, 2, 3, 4] @@ -305,7 +304,6 @@ def initTestCase(self): class TestStrideSliceOpBool6D(TestStrideSliceOpBool): - def initTestCase(self): self.input = np.random.rand(3, 3, 3, 6, 7, 8).astype("bool") self.axes = [0, 1, 2, 3, 4, 5] @@ -316,15 +314,15 @@ def initTestCase(self): class TestStridedSliceOp_starts_ListTensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.config() starts_tensor = [] for index, ele in enumerate(self.starts): - starts_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + starts_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'StartsTensorList': starts_tensor} self.outputs = {'Out': self.output} @@ -333,7 +331,7 @@ def setUp(self): 'starts': self.starts_infer, 'ends': self.ends, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -343,9 +341,9 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [1, -1, 1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.starts_infer = [1, 10, 2] @@ -357,15 +355,15 @@ def test_check_grad_normal(self): class TestStridedSliceOp_ends_ListTensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.config() ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {'Input': self.input, 'EndsTensorList': ends_tensor} self.outputs = {'Out': self.output} @@ -374,7 +372,7 @@ def setUp(self): 'starts': self.starts, 'ends': self.ends_infer, 'strides': self.strides, - 'infer_flags': self.infer_flags + 'infer_flags': self.infer_flags, } def config(self): @@ -384,9 +382,9 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, 1, 2] self.infer_flags = [1, -1, 1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) self.ends_infer = [3, 1, 4] @@ -398,13 +396,12 @@ def test_check_grad_normal(self): class TestStridedSliceOp_starts_Tensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.config() self.inputs = { 'Input': self.input, - "StartsTensor": np.array(self.starts, dtype="int32") + "StartsTensor": np.array(self.starts, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -422,9 +419,9 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output() @@ -434,13 +431,12 @@ def test_check_grad_normal(self): class TestStridedSliceOp_ends_Tensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.config() self.inputs = { 'Input': self.input, - "EndsTensor": np.array(self.ends, dtype="int32") + "EndsTensor": np.array(self.ends, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -458,9 +454,9 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output() @@ -470,19 +466,19 @@ def test_check_grad_normal(self): class TestStridedSliceOp_listTensor_Tensor(OpTest): - def setUp(self): self.config() ends_tensor = [] for index, ele in enumerate(self.ends): - ends_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + ends_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.op_type = "strided_slice" self.inputs = { 'Input': self.input, "StartsTensor": np.array(self.starts, dtype="int32"), - "EndsTensorList": ends_tensor + "EndsTensorList": ends_tensor, } self.outputs = {'Out': self.output} self.attrs = { @@ -500,9 +496,9 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, 1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output() @@ -512,13 +508,12 @@ def test_check_grad_normal(self): class TestStridedSliceOp_strides_Tensor(OpTest): - def setUp(self): self.op_type = "strided_slice" self.config() self.inputs = { 'Input': self.input, - "StridesTensor": np.array(self.strides, dtype="int32") + "StridesTensor": np.array(self.strides, dtype="int32"), } self.outputs = {'Out': self.output} self.attrs = { @@ -536,9 +531,9 @@ def config(self): self.axes = [0, 1, 2] self.strides = [1, -1, 1] self.infer_flags = [-1, -1, -1] - self.output = strided_slice_native_forward(self.input, self.axes, - self.starts, self.ends, - self.strides) + self.output = strided_slice_native_forward( + self.input, self.axes, self.starts, self.ends, self.strides + ) def test_check_output(self): self.check_output() @@ -549,48 +544,50 @@ def test_check_grad_normal(self): # Test python API class TestStridedSliceAPI(unittest.TestCase): - def test_1(self): input = np.random.random([3, 4, 5, 6]).astype("float64") minus_1 = fluid.layers.fill_constant([1], "int32", -1) minus_3 = fluid.layers.fill_constant([1], "int32", -3) - starts = fluid.layers.data(name='starts', - shape=[3], - dtype='int32', - append_batch_size=False) - ends = fluid.layers.data(name='ends', - shape=[3], - dtype='int32', - append_batch_size=False) - strides = fluid.layers.data(name='strides', - shape=[3], - dtype='int32', - append_batch_size=False) - - x = fluid.layers.data(name="x", - shape=[3, 4, 5, 6], - append_batch_size=False, - dtype="float64") - out_1 = paddle.strided_slice(x, - axes=[0, 1, 2], - starts=[-3, 0, 2], - ends=[3, 100, -1], - strides=[1, 1, 1]) - out_2 = paddle.strided_slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, -1], - strides=[1, 1, 1]) - out_3 = paddle.strided_slice(x, - axes=[0, 1, 3], - starts=[minus_3, 0, 2], - ends=[3, 100, minus_1], - strides=[1, 1, 1]) - out_4 = paddle.strided_slice(x, - axes=[0, 1, 2], - starts=starts, - ends=ends, - strides=strides) + starts = fluid.layers.data( + name='starts', shape=[3], dtype='int32', append_batch_size=False + ) + ends = fluid.layers.data( + name='ends', shape=[3], dtype='int32', append_batch_size=False + ) + strides = fluid.layers.data( + name='strides', shape=[3], dtype='int32', append_batch_size=False + ) + + x = fluid.layers.data( + name="x", + shape=[3, 4, 5, 6], + append_batch_size=False, + dtype="float64", + ) + out_1 = paddle.strided_slice( + x, + axes=[0, 1, 2], + starts=[-3, 0, 2], + ends=[3, 100, -1], + strides=[1, 1, 1], + ) + out_2 = paddle.strided_slice( + x, + axes=[0, 1, 3], + starts=[minus_3, 0, 2], + ends=[3, 100, -1], + strides=[1, 1, 1], + ) + out_3 = paddle.strided_slice( + x, + axes=[0, 1, 3], + starts=[minus_3, 0, 2], + ends=[3, 100, minus_1], + strides=[1, 1, 1], + ) + out_4 = paddle.strided_slice( + x, axes=[0, 1, 2], starts=starts, ends=ends, strides=strides + ) out_5 = x[-3:3, 0:100:2, -1:2:-1] out_6 = x[minus_3:3:1, 0:100:2, :, minus_1:2:minus_1] @@ -603,9 +600,10 @@ def test_1(self): "x": input, 'starts': np.array([-3, 0, 2]).astype("int32"), 'ends': np.array([3, 2147483648, -1]).astype("int64"), - 'strides': np.array([1, 1, 1]).astype("int32") + 'strides': np.array([1, 1, 1]).astype("int32"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7], + ) assert np.array_equal(res_1, input[-3:3, 0:100, 2:-1, :]) assert np.array_equal(res_2, input[-3:3, 0:100, :, 2:-1]) assert np.array_equal(res_3, input[-3:3, 0:100, :, 2:-1]) @@ -620,19 +618,20 @@ def test_dygraph_op(self): starts = [-3, 0, 2] ends = [3, 2, 4] strides_1 = [1, 1, 1] - sliced_1 = paddle.strided_slice(x, - axes=axes, - starts=starts, - ends=ends, - strides=strides_1) + sliced_1 = paddle.strided_slice( + x, axes=axes, starts=starts, ends=ends, strides=strides_1 + ) assert sliced_1.shape == (3, 2, 2, 2) - @unittest.skipIf(not paddle.is_compiled_with_cuda(), - "Cannot use CUDAPinnedPlace in CPU only version") + @unittest.skipIf( + not paddle.is_compiled_with_cuda(), + "Cannot use CUDAPinnedPlace in CPU only version", + ) def test_cuda_pinned_place(self): with paddle.fluid.dygraph.guard(): - x = paddle.to_tensor(np.random.randn(2, 10), - place=paddle.CUDAPinnedPlace()) + x = paddle.to_tensor( + np.random.randn(2, 10), place=paddle.CUDAPinnedPlace() + ) self.assertTrue(x.place.is_cuda_pinned_place()) y = x[:, ::2] self.assertFalse(x.place.is_cuda_pinned_place()) @@ -640,15 +639,17 @@ def test_cuda_pinned_place(self): class ArrayLayer(paddle.nn.Layer): - def __init__(self, input_size=224, output_size=10, array_size=1): super(ArrayLayer, self).__init__() self.input_size = input_size self.output_size = output_size self.array_size = array_size for i in range(self.array_size): - setattr(self, self.create_name(i), - paddle.nn.Linear(input_size, output_size)) + setattr( + self, + self.create_name(i), + paddle.nn.Linear(input_size, output_size), + ) def create_name(self, index): return 'linear_' + str(index) @@ -704,7 +705,6 @@ def create_tensor_array(self, tensors): class TestStridedSliceTensorArray(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -718,15 +718,17 @@ def grad_equal(self, g1, g2): def is_grads_equal(self, g1, g2): for i, g in enumerate(g1): - self.assertTrue(self.grad_equal(g, g2[i]), - msg="gradient_1:\n{} \ngradient_2:\n{}".format( - g, g2)) + self.assertTrue( + self.grad_equal(g, g2[i]), + msg="gradient_1:\n{} \ngradient_2:\n{}".format(g, g2), + ) def is_grads_equal_zeros(self, grads): for g in grads: self.assertTrue( self.grad_equal(np.zeros_like(g), g), - msg="The gradient should be zeros, but received \n{}".format(g)) + msg="The gradient should be zeros, but received \n{}".format(g), + ) def create_case(self, net): inps1 = paddle.randn([1, net.input_size], dtype='float32') @@ -751,31 +753,33 @@ def create_case(self, net): np.testing.assert_array_equal( s1, s2, - err_msg='dygraph graph result:\n{} \nstatic dygraph result:\n{}'. - format(l1.numpy(), l2.numpy())) + err_msg='dygraph graph result:\n{} \nstatic dygraph result:\n{}'.format( + l1.numpy(), l2.numpy() + ), + ) def test_strided_slice_tensor_array_cuda_pinned_place(self): if paddle.device.is_compiled_with_cuda(): with paddle.fluid.dygraph.guard(): class Simple(paddle.nn.Layer): - def __init__(self): super(Simple, self).__init__() def forward(self, inps): tensor_array = None for i, tensor in enumerate(inps): - index = paddle.full(shape=[1], - dtype='int64', - fill_value=i) + index = paddle.full( + shape=[1], dtype='int64', fill_value=i + ) if tensor_array is None: tensor_array = paddle.tensor.array_write( - tensor, i=index) + tensor, i=index + ) else: - paddle.tensor.array_write(tensor, - i=index, - array=tensor_array) + paddle.tensor.array_write( + tensor, i=index, array=tensor_array + ) array1 = paddle.concat(tensor_array) array2 = paddle.concat(tensor_array[::-1]) @@ -784,12 +788,16 @@ def forward(self, inps): net = Simple() func = paddle.jit.to_static(net.forward) - inps1 = paddle.to_tensor(np.random.randn(2, 10), - place=paddle.CUDAPinnedPlace(), - stop_gradient=False) - inps2 = paddle.to_tensor(np.random.randn(2, 10), - place=paddle.CUDAPinnedPlace(), - stop_gradient=False) + inps1 = paddle.to_tensor( + np.random.randn(2, 10), + place=paddle.CUDAPinnedPlace(), + stop_gradient=False, + ) + inps2 = paddle.to_tensor( + np.random.randn(2, 10), + place=paddle.CUDAPinnedPlace(), + stop_gradient=False, + ) self.assertTrue(inps1.place.is_cuda_pinned_place()) self.assertTrue(inps2.place.is_cuda_pinned_place()) @@ -799,201 +807,173 @@ def forward(self, inps): self.assertFalse(result.place.is_cuda_pinned_place()) def test_strided_slice_tensor_array(self): - class Net01(ArrayLayer): - def array_slice(self, tensors): return tensors[::-1] self.create_case(Net01(array_size=10)) class Net02(ArrayLayer): - def array_slice(self, tensors): return tensors[::-2] self.create_case(Net02(input_size=112, array_size=11)) class Net03(ArrayLayer): - def array_slice(self, tensors): return tensors[::-3] self.create_case(Net03(input_size=112, array_size=9)) class Net04(ArrayLayer): - def array_slice(self, tensors): return tensors[1::-4] self.create_case(Net04(input_size=112, array_size=9)) class Net05(ArrayLayer): - def array_slice(self, tensors): return tensors[:7:-4] self.create_case(Net05(input_size=112, array_size=9)) class Net06(ArrayLayer): - def array_slice(self, tensors): return tensors[8:0:-4] self.create_case(Net06(input_size=112, array_size=9)) class Net07(ArrayLayer): - def array_slice(self, tensors): return tensors[8:1:-4] self.create_case(Net07(input_size=112, array_size=9)) class Net08(ArrayLayer): - def array_slice(self, tensors): return tensors[::2] self.create_case(Net08(input_size=112, array_size=11)) class Net09(ArrayLayer): - def array_slice(self, tensors): return tensors[::3] self.create_case(Net09(input_size=112, array_size=9)) class Net10(ArrayLayer): - def array_slice(self, tensors): return tensors[1::4] self.create_case(Net10(input_size=112, array_size=9)) class Net11(ArrayLayer): - def array_slice(self, tensors): return tensors[:8:4] self.create_case(Net11(input_size=112, array_size=9)) class Net12(ArrayLayer): - def array_slice(self, tensors): return tensors[1:8:4] self.create_case(Net12(input_size=112, array_size=9)) class Net13(ArrayLayer): - def array_slice(self, tensors): return tensors[8:10:4] self.create_case(Net13(input_size=112, array_size=13)) class Net14(ArrayLayer): - def array_slice(self, tensors): return tensors[3:10:4] self.create_case(Net14(input_size=112, array_size=13)) class Net15(ArrayLayer): - def array_slice(self, tensors): return tensors[2:10:4] self.create_case(Net15(input_size=112, array_size=13)) class Net16(ArrayLayer): - def array_slice(self, tensors): return tensors[3:10:3] self.create_case(Net16(input_size=112, array_size=13)) class Net17(ArrayLayer): - def array_slice(self, tensors): return tensors[3:15:3] self.create_case(Net17(input_size=112, array_size=13)) class Net18(ArrayLayer): - def array_slice(self, tensors): return tensors[0:15:3] self.create_case(Net18(input_size=112, array_size=13)) class Net19(ArrayLayer): - def array_slice(self, tensors): return tensors[-1:-5:-3] self.create_case(Net19(input_size=112, array_size=13)) class Net20(ArrayLayer): - def array_slice(self, tensors): return tensors[-1:-6:-3] self.create_case(Net20(input_size=112, array_size=13)) class Net21(ArrayLayer): - def array_slice(self, tensors): return tensors[-3:-6:-3] self.create_case(Net21(input_size=112, array_size=13)) class Net22(ArrayLayer): - def array_slice(self, tensors): return tensors[-5:-1:3] self.create_case(Net22(input_size=112, array_size=13)) class Net23(ArrayLayer): - def array_slice(self, tensors): return tensors[-6:-1:3] self.create_case(Net23(input_size=112, array_size=13)) class Net24(ArrayLayer): - def array_slice(self, tensors): return tensors[-6:-3:3] self.create_case(Net24(input_size=112, array_size=13)) class Net25(ArrayLayer): - def array_slice(self, tensors): return tensors[0::3] self.create_case(Net25(input_size=112, array_size=13)) class Net26(ArrayLayer): - def array_slice(self, tensors): return tensors[-60:20:3] self.create_case(Net26(input_size=112, array_size=13)) class Net27(ArrayLayer): - def array_slice(self, tensors): return tensors[-3:-60:-3] self.create_case(Net27(input_size=112, array_size=13)) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestStridedSliceFloat16(unittest.TestCase): - def init_test_case(self): self.op_type = 'strided_slice' self.input_shape = [3, 3, 3, 6, 7, 8] @@ -1008,8 +988,9 @@ def check_main(self, x_np, dtype): x_np = x_np.astype(dtype) x = paddle.to_tensor(x_np) x.stop_gradient = False - output = strided_slice_native_forward(x, self.axes, self.starts, - self.ends, self.strides) + output = strided_slice_native_forward( + x, self.axes, self.starts, self.ends, self.strides + ) x_grad = paddle.grad(output, x) output_np = output[0].numpy().astype('float32') x_grad_np = x_grad[0].numpy().astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_subtract_op.py b/python/paddle/fluid/tests/unittests/test_subtract_op.py index 5eda6e367d4e6..ce58725d05017 100644 --- a/python/paddle/fluid/tests/unittests/test_subtract_op.py +++ b/python/paddle/fluid/tests/unittests/test_subtract_op.py @@ -19,7 +19,6 @@ class ApiSubtractTest(unittest.TestCase): - def setUp(self): if core.is_compiled_with_cuda(): self.place = core.CUDAPlace(0) @@ -40,56 +39,56 @@ def setUp(self): def test_static_api(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_y = paddle.static.data("y", shape=[10, 15], dtype="float32") result_max = paddle.subtract(data_x, data_y) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "y": self.input_y - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"x": self.input_x, "y": self.input_y}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected1, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_x = paddle.static.data("x", shape=[10, 15], dtype="float32") data_z = paddle.static.data("z", shape=[15], dtype="float32") result_max = paddle.subtract(data_x, data_z) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "x": self.input_x, - "z": self.input_z - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"x": self.input_x, "z": self.input_z}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected2, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_a = paddle.static.data("a", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_max = paddle.subtract(data_a, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "a": self.input_a, - "c": self.input_c - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"a": self.input_a, "c": self.input_c}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected3, rtol=1e-05) - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data_b = paddle.static.data("b", shape=[3], dtype="int64") data_c = paddle.static.data("c", shape=[3], dtype="int64") result_max = paddle.subtract(data_b, data_c) exe = paddle.static.Executor(self.place) - res, = exe.run(feed={ - "b": self.input_b, - "c": self.input_c - }, - fetch_list=[result_max]) + (res,) = exe.run( + feed={"b": self.input_b, "c": self.input_c}, + fetch_list=[result_max], + ) np.testing.assert_allclose(res, self.np_expected4, rtol=1e-05) def test_dynamic_api(self): diff --git a/python/paddle/fluid/tests/unittests/test_sum_op.py b/python/paddle/fluid/tests/unittests/test_sum_op.py index 7d99391e5b1b2..6c8cc00ea8fd9 100644 --- a/python/paddle/fluid/tests/unittests/test_sum_op.py +++ b/python/paddle/fluid/tests/unittests/test_sum_op.py @@ -22,9 +22,11 @@ import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.op import Operator -from paddle.fluid.tests.unittests.op_test import (OpTest, - convert_float_to_uint16, - convert_uint16_to_float) +from paddle.fluid.tests.unittests.op_test import ( + OpTest, + convert_float_to_uint16, + convert_uint16_to_float, +) from paddle.fluid.framework import _test_eager_guard import paddle.inference as paddle_infer import gradient_checker @@ -33,7 +35,6 @@ class TestSumOp(OpTest): - def setUp(self): self.op_type = "sum" self.init_kernel_type() @@ -58,7 +59,6 @@ def test_check_grad(self): class TestSelectedRowsSumOp(unittest.TestCase): - def setUp(self): self.height = 10 self.row_numel = 12 @@ -67,14 +67,18 @@ def setUp(self): self.init_kernel_type() def check_with_place(self, place, inplace): - self.check_input_and_optput(core.Scope(), place, inplace, True, True, - True) - self.check_input_and_optput(core.Scope(), place, inplace, False, True, - True) - self.check_input_and_optput(core.Scope(), place, inplace, False, False, - True) - self.check_input_and_optput(core.Scope(), place, inplace, False, False, - False) + self.check_input_and_optput( + core.Scope(), place, inplace, True, True, True + ) + self.check_input_and_optput( + core.Scope(), place, inplace, False, True, True + ) + self.check_input_and_optput( + core.Scope(), place, inplace, False, False, True + ) + self.check_input_and_optput( + core.Scope(), place, inplace, False, False, False + ) def init_kernel_type(self): pass @@ -85,13 +89,15 @@ def _get_array(self, rows, row_numel): array[i] *= rows[i] return array - def check_input_and_optput(self, - scope, - place, - inplace, - w1_has_data=False, - w2_has_data=False, - w3_has_data=False): + def check_input_and_optput( + self, + scope, + place, + inplace, + w1_has_data=False, + w2_has_data=False, + w3_has_data=False, + ): self.create_selected_rows(scope, place, "W1", w1_has_data) self.create_selected_rows(scope, place, "W2", w2_has_data) @@ -117,7 +123,8 @@ def check_input_and_optput(self, self.assertEqual(len(out.rows()), 7) np.testing.assert_array_equal( np.array(out.get_tensor()), - self._get_array(self.rows, self.row_numel) * has_data_w_num) + self._get_array(self.rows, self.row_numel) * has_data_w_num, + ) else: self.assertEqual(len(out.rows()), 0) @@ -148,15 +155,14 @@ def test_w_is_selected_rows(self): class TestSelectedRowsSumOpInt(TestSelectedRowsSumOp): - def init_kernel_type(self): self.dtype = np.int32 -@unittest.skipIf(not core.supports_bfloat16(), - 'place does not support BF16 evaluation') +@unittest.skipIf( + not core.supports_bfloat16(), 'place does not support BF16 evaluation' +) class TestSelectedRowsSumBF16Op(TestSelectedRowsSumOp): - def setUp(self): self.height = 10 self.row_numel = 12 @@ -164,8 +170,9 @@ def setUp(self): self.dtype = np.uint16 self.init_kernel_type() np.random.seed(12345) - self.data = np.random.random( - (len(self.rows), self.row_numel)).astype(np.float32) + self.data = np.random.random((len(self.rows), self.row_numel)).astype( + np.float32 + ) def _get_array(self, rows, row_numel): if len(rows) > 0: @@ -173,13 +180,15 @@ def _get_array(self, rows, row_numel): else: return np.ndarray((0, row_numel), dtype=self.dtype) - def check_input_and_optput(self, - scope, - place, - inplace, - w1_has_data=False, - w2_has_data=False, - w3_has_data=False): + def check_input_and_optput( + self, + scope, + place, + inplace, + w1_has_data=False, + w2_has_data=False, + w3_has_data=False, + ): self.create_selected_rows(scope, place, "W1", w1_has_data) self.create_selected_rows(scope, place, "W2", w2_has_data) @@ -205,8 +214,12 @@ def check_input_and_optput(self, self.assertEqual(len(out.rows()), 7) out_bf16 = np.array(out.get_tensor()) out_fp32 = convert_uint16_to_float(out_bf16) - ref_fp32 = convert_uint16_to_float( - self._get_array(self.rows, self.row_numel)) * has_data_w_num + ref_fp32 = ( + convert_uint16_to_float( + self._get_array(self.rows, self.row_numel) + ) + * has_data_w_num + ) np.testing.assert_allclose(out_fp32, ref_fp32, atol=0, rtol=0.95e-2) else: self.assertEqual(len(out.rows()), 0) @@ -217,13 +230,11 @@ def test_w_is_selected_rows(self): class TestSelectedRowsSumBF16OpBigRow(TestSelectedRowsSumBF16Op): - def init_kernel_type(self): self.row_numel = 102 class TestLoDTensorAndSelectedRowsOp(TestSelectedRowsSumOp): - def setUp(self): self.height = 10 self.row_numel = 12 @@ -255,23 +266,25 @@ def check_with_place(self, place, inplace): self.assertEqual(out_t.shape[0], self.height) np.testing.assert_array_equal( out_t, - self._get_array([i for i in range(self.height)], self.row_numel) * - np.tile(np.array(result).reshape(self.height, 1), self.row_numel)) + self._get_array([i for i in range(self.height)], self.row_numel) + * np.tile(np.array(result).reshape(self.height, 1), self.row_numel), + ) def create_lod_tensor(self, scope, place, var_name): var = scope.var(var_name) w_tensor = var.get_tensor() - w_array = self._get_array([i for i in range(self.height)], - self.row_numel) + w_array = self._get_array( + [i for i in range(self.height)], self.row_numel + ) w_tensor.set(w_array, place) return var -#----------- test fp16 ----------- -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +# ----------- test fp16 ----------- +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestFP16SumOp(TestSumOp): - def init_kernel_type(self): self.dtype = np.float16 @@ -289,11 +302,10 @@ def test_check_grad(self): def create_test_sum_fp16_class(parent): - - @unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") + @unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" + ) class TestSumFp16Case(parent): - def init_kernel_type(self): self.dtype = np.float16 @@ -308,9 +320,8 @@ def test_w_is_selected_rows(self): globals()[cls_name] = TestSumFp16Case -#----------- test bf16 ----------- +# ----------- test bf16 ----------- class TestSumBF16Op(OpTest): - def setUp(self): self.op_type = "sum" self.init_kernel_type() @@ -319,9 +330,11 @@ def setUp(self): x2 = np.random.random((3, 40)).astype(np.float32) y = x0 + x1 + x2 self.inputs = { - "X": [("x0", convert_float_to_uint16(x0)), - ("x1", convert_float_to_uint16(x1)), - ("x2", convert_float_to_uint16(x2))] + "X": [ + ("x0", convert_float_to_uint16(x0)), + ("x1", convert_float_to_uint16(x1)), + ("x2", convert_float_to_uint16(x2)), + ] } self.outputs = {'Out': convert_float_to_uint16(y)} @@ -336,15 +349,14 @@ def test_check_grad(self): class API_Test_Add_n(unittest.TestCase): - def test_api(self): with fluid.program_guard(fluid.Program(), fluid.Program()): - input0 = fluid.layers.fill_constant(shape=[2, 3], - dtype='int64', - value=5) - input1 = fluid.layers.fill_constant(shape=[2, 3], - dtype='int64', - value=3) + input0 = fluid.layers.fill_constant( + shape=[2, 3], dtype='int64', value=5 + ) + input1 = fluid.layers.fill_constant( + shape=[2, 3], dtype='int64', value=3 + ) expected_result = np.empty((2, 3)) expected_result.fill(8) sum_value = paddle.add_n([input0, input1]) @@ -371,16 +383,19 @@ def test_dygraph_api(self): expected_result = np.empty((2, 3)) expected_result.fill(2) sum_value = paddle.add_n([input0, input1]) - self.assertEqual((sum_value.numpy() == expected_result).all(), - True) + self.assertEqual( + (sum_value.numpy() == expected_result).all(), True + ) expected_grad_result = np.empty((2, 3)) expected_grad_result.fill(1) sum_value.backward() self.assertEqual( - (input0.grad.numpy() == expected_grad_result).all(), True) + (input0.grad.numpy() == expected_grad_result).all(), True + ) self.assertEqual( - (input1.grad.numpy() == expected_grad_result).all(), True) + (input1.grad.numpy() == expected_grad_result).all(), True + ) def test_add_n_and_add_and_grad(self): with fluid.dygraph.guard(): @@ -397,7 +412,7 @@ def test_add_n_and_add_and_grad(self): dx, dy = paddle.grad([out], [x, y], create_graph=True) - expected_out = np.array([[10., 12., 14.], [16., 18., 20.]]) + expected_out = np.array([[10.0, 12.0, 14.0], [16.0, 18.0, 20.0]]) expected_dx = np.array([[1, 1, 1], [1, 1, 1]]) expected_dy = np.array([[1, 1, 1], [1, 1, 1]]) @@ -407,9 +422,7 @@ def test_add_n_and_add_and_grad(self): class TestRaiseSumError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.sum([11, 22]) @@ -430,9 +443,7 @@ def test_dtype1(): class TestRaiseSumsError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.sums([11, 22]) @@ -468,9 +479,7 @@ def test_out_dtype(): class TestSumOpError(unittest.TestCase): - def test_errors(self): - def test_empty_list_input(): with fluid.dygraph.guard(): fluid._legacy_C_ops.sum([]) @@ -488,14 +497,16 @@ def test_list_of_none_input(): class TestReduceOPTensorAxisBase(unittest.TestCase): - def setUp(self): paddle.disable_static() paddle.seed(2022) self.temp_dir = tempfile.TemporaryDirectory() self.save_path = os.path.join(self.temp_dir.name, 'reduce_tensor_axis') - self.place = paddle.CUDAPlace( - 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() + self.place = ( + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() + ) self.keepdim = False self.init_data() @@ -514,7 +525,8 @@ def test_dygraph(self): pd_out = self.pd_api(self.x, self.tensor_axis) np_out = self.np_api(self.x.numpy(), tuple(self.np_axis)) np.testing.assert_allclose( - pd_out.numpy() if pd_out.size > 1 else pd_out.item(), np_out) + pd_out.numpy() if pd_out.size > 1 else pd_out.item(), np_out + ) pd_out.backward() self.assertEqual(self.x.gradient().shape, tuple(self.x.shape)) @@ -524,9 +536,9 @@ def test_static_and_infer(self): starup_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, starup_prog): # run static - x = paddle.static.data(shape=self.x.shape, - name='x', - dtype='float32') + x = paddle.static.data( + shape=self.x.shape, name='x', dtype='float32' + ) if isinstance(self.tensor_axis, paddle.Tensor): axis = paddle.assign(self.np_axis) else: @@ -541,17 +553,19 @@ def test_static_and_infer(self): linear_out = linear(x) out = self.pd_api(linear_out, axis, keepdim=self.keepdim) - sgd = paddle.optimizer.SGD(learning_rate=0.) + sgd = paddle.optimizer.SGD(learning_rate=0.0) sgd.minimize(paddle.mean(out)) exe = paddle.static.Executor(self.place) exe.run(starup_prog) - static_out = exe.run(feed={'x': self.x.numpy().astype('float32')}, - fetch_list=[out]) + static_out = exe.run( + feed={'x': self.x.numpy().astype('float32')}, fetch_list=[out] + ) # run infer paddle.static.save_inference_model(self.save_path, [x], [out], exe) - config = paddle_infer.Config(self.save_path + '.pdmodel', - self.save_path + '.pdiparams') + config = paddle_infer.Config( + self.save_path + '.pdmodel', self.save_path + '.pdiparams' + ) if paddle.is_compiled_with_cuda(): config.enable_use_gpu(100, 0) else: @@ -570,7 +584,6 @@ def test_static_and_infer(self): class TestSumWithTensorAxis1(TestReduceOPTensorAxisBase): - def init_data(self): self.pd_api = paddle.sum self.np_api = np.sum @@ -579,12 +592,11 @@ def init_data(self): self.tensor_axis = [ 0, paddle.to_tensor([1], 'int64'), - paddle.to_tensor([2], 'int64') + paddle.to_tensor([2], 'int64'), ] class TestAddNDoubleGradCheck(unittest.TestCase): - def add_n_wrapper(self, x): return paddle.add_n(x) @@ -602,17 +614,21 @@ def func(self, place): data1_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) data2_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) - gradient_checker.double_grad_check([data1, data2], - out, - x_init=[data1_arr, data2_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data1, data2], + out, + x_init=[data1_arr, data2_arr], + place=place, + eps=eps, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) gradient_checker.double_grad_check_for_dygraph( - self.add_n_wrapper, [data1, data2], + self.add_n_wrapper, + [data1, data2], out, x_init=[data1_arr, data2_arr], - place=place) + place=place, + ) def test_grad(self): paddle.enable_static() @@ -624,7 +640,6 @@ def test_grad(self): class TestAddNTripleGradCheck(unittest.TestCase): - def add_n_wrapper(self, x): return paddle.add_n(x) @@ -642,17 +657,21 @@ def func(self, place): data1_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) data2_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) - gradient_checker.triple_grad_check([data1, data2], - out, - x_init=[data1_arr, data2_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data1, data2], + out, + x_init=[data1_arr, data2_arr], + place=place, + eps=eps, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) gradient_checker.triple_grad_check_for_dygraph( - self.add_n_wrapper, [data1, data2], + self.add_n_wrapper, + [data1, data2], out, x_init=[data1_arr, data2_arr], - place=place) + place=place, + ) def test_grad(self): paddle.enable_static() @@ -664,7 +683,6 @@ def test_grad(self): class TestSumDoubleGradCheck(unittest.TestCase): - def sum_wrapper(self, x): return paddle.sum(x[0], axis=1, keepdim=True) @@ -679,16 +697,13 @@ def func(self, place): out = paddle.sum(data, axis=1, keepdim=True) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.sum_wrapper, [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.sum_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -700,7 +715,6 @@ def test_grad(self): class TestSumTripleGradCheck(unittest.TestCase): - def sum_wrapper(self, x): return paddle.sum(x[0], axis=1, keepdim=True) @@ -715,16 +729,13 @@ def func(self, place): out = paddle.sum(data, axis=1, keepdim=True) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.sum_wrapper, [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.sum_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_svd_op.py b/python/paddle/fluid/tests/unittests/test_svd_op.py index aca403dac074c..146b18ae20cd8 100644 --- a/python/paddle/fluid/tests/unittests/test_svd_op.py +++ b/python/paddle/fluid/tests/unittests/test_svd_op.py @@ -21,25 +21,23 @@ class TestSvdOp(OpTest): - def setUp(self): paddle.enable_static() self.python_api = paddle.linalg.svd self.generate_input() self.generate_output() self.op_type = "svd" - assert (hasattr(self, "_output_data")) + assert hasattr(self, "_output_data") self.inputs = {"X": self._input_data} self.attrs = {'full_matrices': self.get_full_matrices_option()} self.outputs = { "U": self._output_data[0], "S": self._output_data[1], - "VH": self._output_data[2] + "VH": self._output_data[2], } def generate_input(self): - """ return a input_data and input_shape - """ + """return a input_data and input_shape""" self._input_shape = (100, 1) self._input_data = np.random.random(self._input_shape).astype("float64") @@ -47,17 +45,17 @@ def get_full_matrices_option(self): return False def generate_output(self): - assert (hasattr(self, "_input_data")) + assert hasattr(self, "_input_data") self._output_data = np.linalg.svd(self._input_data) def test_check_output(self): self.check_output(no_check_set=['U', 'VH'], check_eager=True) def test_svd_forward(self): - """ u matmul diag(s) matmul vt must become X - """ + """u matmul diag(s) matmul vt must become X""" single_input = self._input_data.reshape( - [-1, self._input_shape[-2], self._input_shape[-1]])[0] + [-1, self._input_shape[-2], self._input_shape[-1]] + )[0] paddle.disable_static() dy_x = paddle.to_tensor(single_input) dy_u, dy_s, dy_vt = paddle.linalg.svd(dy_x) @@ -71,19 +69,19 @@ def test_svd_forward(self): paddle.enable_static() def check_S_grad(self): - self.check_grad(['X'], ['S'], - numeric_grad_delta=0.001, - check_eager=True) + self.check_grad( + ['X'], ['S'], numeric_grad_delta=0.001, check_eager=True + ) def check_U_grad(self): - self.check_grad(['X'], ['U'], - numeric_grad_delta=0.001, - check_eager=True) + self.check_grad( + ['X'], ['U'], numeric_grad_delta=0.001, check_eager=True + ) def check_V_grad(self): - self.check_grad(['X'], ['VH'], - numeric_grad_delta=0.001, - check_eager=True) + self.check_grad( + ['X'], ['VH'], numeric_grad_delta=0.001, check_eager=True + ) def test_check_grad(self): """ @@ -101,123 +99,146 @@ class TestSvdCheckGrad2(TestSvdOp): no_need_check_grad = True def generate_input(self): - """ return a deterministic matrix, the range matrix; - vander matrix must be a full rank matrix. + """return a deterministic matrix, the range matrix; + vander matrix must be a full rank matrix. """ self._input_shape = (5, 5) - self._input_data = np.vander([2, 3, 4, 5, 6]).astype("float64").reshape( - self._input_shape) + self._input_data = ( + np.vander([2, 3, 4, 5, 6]) + .astype("float64") + .reshape(self._input_shape) + ) class TestSvdNormalMatrixSmall(TestSvdCheckGrad2): - def generate_input(self): - """ small matrix SVD. - """ + """small matrix SVD.""" self._input_shape = (1, 1) self._input_data = np.random.random(self._input_shape).astype("float64") class TestSvdNormalMatrix6x3(TestSvdCheckGrad2): - def generate_input(self): - """ return a deterministic matrix, the range matrix; - vander matrix must be a full rank matrix. + """return a deterministic matrix, the range matrix; + vander matrix must be a full rank matrix. """ self._input_shape = (6, 3) - self._input_data = np.array([[1.0, 2.0, 3.0], [0.0, 1.0, 5.0], - [0.0, 0.0, 6.0], [2.0, 4.0, 9.0], - [3.0, 6.0, 8.0], [3.0, 1.0, - 0.0]]).astype("float64") + self._input_data = np.array( + [ + [1.0, 2.0, 3.0], + [0.0, 1.0, 5.0], + [0.0, 0.0, 6.0], + [2.0, 4.0, 9.0], + [3.0, 6.0, 8.0], + [3.0, 1.0, 0.0], + ] + ).astype("float64") class TestSvdNormalMatrix3x6(TestSvdCheckGrad2): - def generate_input(self): - """ return a deterministic matrix, the range matrix; - vander matrix must be a full rank matrix. + """return a deterministic matrix, the range matrix; + vander matrix must be a full rank matrix. """ self._input_shape = (3, 6) - self._input_data = np.array([[1.0, 2.0, 3.0], [0.0, 1.0, 5.0], - [0.0, 0.0, 6.0], [2.0, 4.0, 9.0], - [3.0, 6.0, 8.0], [3.0, 1.0, - 0.0]]).astype("float64") + self._input_data = np.array( + [ + [1.0, 2.0, 3.0], + [0.0, 1.0, 5.0], + [0.0, 0.0, 6.0], + [2.0, 4.0, 9.0], + [3.0, 6.0, 8.0], + [3.0, 1.0, 0.0], + ] + ).astype("float64") self._input_data = self._input_data.transpose((-1, -2)) class TestSvdNormalMatrix6x3Batched(TestSvdOp): - def generate_input(self): self._input_shape = (10, 6, 3) - self._input_data = np.array([[1.0, 2.0, 3.0], [0.0, 1.0, 5.0], - [0.0, 0.0, 6.0], [2.0, 4.0, 9.0], - [3.0, 6.0, 8.0], [3.0, 1.0, - 0.0]]).astype("float64") + self._input_data = np.array( + [ + [1.0, 2.0, 3.0], + [0.0, 1.0, 5.0], + [0.0, 0.0, 6.0], + [2.0, 4.0, 9.0], + [3.0, 6.0, 8.0], + [3.0, 1.0, 0.0], + ] + ).astype("float64") self._input_data = np.stack([self._input_data] * 10, axis=0) def test_svd_forward(self): - """ test_svd_forward not support batched input, so disable this test. - """ + """test_svd_forward not support batched input, so disable this test.""" pass class TestSvdNormalMatrix3x6Batched(TestSvdOp): - def generate_input(self): - """ return a deterministic matrix, the range matrix; - vander matrix must be a full rank matrix. + """return a deterministic matrix, the range matrix; + vander matrix must be a full rank matrix. """ self._input_shape = (10, 3, 6) - self._input_data = np.array([[1.0, 2.0, 3.0], [0.0, 1.0, 5.0], - [0.0, 0.0, 6.0], [2.0, 4.0, 9.0], - [3.0, 6.0, 8.0], [3.0, 1.0, - 0.0]]).astype("float64") + self._input_data = np.array( + [ + [1.0, 2.0, 3.0], + [0.0, 1.0, 5.0], + [0.0, 0.0, 6.0], + [2.0, 4.0, 9.0], + [3.0, 6.0, 8.0], + [3.0, 1.0, 0.0], + ] + ).astype("float64") self._input_data = self._input_data.transpose((-1, -2)) self._input_data = np.stack([self._input_data] * 10, axis=0) def test_svd_forward(self): - """ test_svd_forward not support batched input, so disable this test. - """ + """test_svd_forward not support batched input, so disable this test.""" pass class TestSvdNormalMatrix3x3x3x6Batched(TestSvdOp): - def generate_input(self): - """ return a deterministic matrix, the range matrix; - vander matrix must be a full rank matrix. + """return a deterministic matrix, the range matrix; + vander matrix must be a full rank matrix. """ self._input_shape = (3, 3, 3, 6) - self._input_data = np.array([[1.0, 2.0, 3.0], [0.0, 1.0, 5.0], - [0.0, 0.0, 6.0], [2.0, 4.0, 9.0], - [3.0, 6.0, 8.0], [3.0, 1.0, - 0.0]]).astype("float64") + self._input_data = np.array( + [ + [1.0, 2.0, 3.0], + [0.0, 1.0, 5.0], + [0.0, 0.0, 6.0], + [2.0, 4.0, 9.0], + [3.0, 6.0, 8.0], + [3.0, 1.0, 0.0], + ] + ).astype("float64") self._input_data = self._input_data.transpose((-1, -2)) self._input_data = np.stack( - [self._input_data, self._input_data, self._input_data], axis=0) + [self._input_data, self._input_data, self._input_data], axis=0 + ) self._input_data = np.stack( - [self._input_data, self._input_data, self._input_data], axis=0) + [self._input_data, self._input_data, self._input_data], axis=0 + ) def test_svd_forward(self): - """ test_svd_forward not support batched input, so disable this test. - """ + """test_svd_forward not support batched input, so disable this test.""" pass -@skip_check_grad_ci(reason="'check_grad' on large inputs is too slow, " + - "however it is desirable to cover the forward pass") +@skip_check_grad_ci( + reason="'check_grad' on large inputs is too slow, " + + "however it is desirable to cover the forward pass" +) class TestSvdNormalMatrixBig(TestSvdOp): - def generate_input(self): - """ big matrix SVD. - - """ + """big matrix SVD.""" self._input_shape = (2, 200, 300) self._input_data = np.random.random(self._input_shape).astype("float64") def test_svd_forward(self): - """ test_svd_forward not support batched input, so disable this test. - """ + """test_svd_forward not support batched input, so disable this test.""" pass def test_check_grad(self): @@ -225,16 +246,13 @@ def test_check_grad(self): class TestSvdNormalMatrixBig2(TestSvdOp): - def generate_input(self): - """ big matrix SVD. - """ + """big matrix SVD.""" self._input_shape = (1, 100) self._input_data = np.random.random(self._input_shape).astype("float64") class TestSvdNormalMatrixFullMatrices(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -246,21 +264,19 @@ def test_full_matrices(self): mat = np.random.random(mat_shape).astype("float64") x = paddle.to_tensor(mat) u, s, vh = paddle.linalg.svd(x, full_matrices=True) - assert (u.shape == [2, 2]) - assert (vh.shape == [3, 3]) + assert u.shape == [2, 2] + assert vh.shape == [3, 3] x_recover = u.matmul(paddle.diag(s)).matmul(vh[0:2]) - if ((paddle.abs(x_recover - x) > 1e-4).any()): + if (paddle.abs(x_recover - x) > 1e-4).any(): raise RuntimeError("mat can't be recovered\n") class TestSvdFullMatriceGrad(TestSvdNormalMatrix6x3): - def get_full_matrices_option(self): return True def test_svd_forward(self): - """ test_svd_forward not support full matrices, so disable this test. - """ + """test_svd_forward not support full matrices, so disable this test.""" pass def test_check_grad(self): @@ -268,12 +284,11 @@ def test_check_grad(self): remember the input matrix must be the full rank matrix, otherwise the gradient will stochatic because the u / v 's (n-k) freedom vectors """ self.check_S_grad() - #self.check_U_grad() // don't check U grad, because U have freedom vector + # self.check_U_grad() // don't check U grad, because U have freedom vector self.check_V_grad() class TestSvdAPI(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() a = np.random.rand(5, 5) @@ -290,15 +305,17 @@ def test_static(self): for place in places: with fluid.program_guard(fluid.Program(), fluid.Program()): a = np.random.rand(5, 5) - x = paddle.fluid.data(name="input", - shape=[5, 5], - dtype='float64') + x = paddle.fluid.data( + name="input", shape=[5, 5], dtype='float64' + ) u, s, vh = paddle.linalg.svd(x) exe = fluid.Executor(place) gt_u, gt_s, gt_vh = np.linalg.svd(a, full_matrices=False) - fetches = exe.run(fluid.default_main_program(), - feed={"input": a}, - fetch_list=[s]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input": a}, + fetch_list=[s], + ) np.testing.assert_allclose(fetches[0], gt_s, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_switch.py b/python/paddle/fluid/tests/unittests/test_switch.py index cdc2162a59961..31fe519593fca 100644 --- a/python/paddle/fluid/tests/unittests/test_switch.py +++ b/python/paddle/fluid/tests/unittests/test_switch.py @@ -22,7 +22,6 @@ class TestSwitch(unittest.TestCase): - def check_switch(self, value): x = layers.fill_constant(shape=[1], dtype='float32', value=value) zero_var = layers.fill_constant(shape=[1], dtype='float32', value=0.0) @@ -30,10 +29,9 @@ def check_switch(self, value): two_var = layers.fill_constant(shape=[1], dtype='float32', value=2.0) three_var = layers.fill_constant(shape=[1], dtype='float32', value=3.0) - result = layers.create_global_var(shape=[1], - value=-1.0, - dtype='float32', - persistable=True) + result = layers.create_global_var( + shape=[1], value=-1.0, dtype='float32', persistable=True + ) with layers.Switch() as switch: with switch.case(layers.less_than(x, zero_var)): @@ -63,20 +61,18 @@ def test_switch(self): class TestSwitchCaseError(unittest.TestCase): - def test_error(self): main_program = framework.Program() startup_program = framework.Program() with framework.program_guard(main_program, startup_program): cond = layers.fill_constant(shape=[1], dtype='float32', value=0.0) - zero_var = layers.fill_constant(shape=[1], - dtype='float32', - value=0.0) - - result = layers.create_global_var(shape=[1], - value=-1.0, - dtype='float32', - persistable=True) + zero_var = layers.fill_constant( + shape=[1], dtype='float32', value=0.0 + ) + + result = layers.create_global_var( + shape=[1], value=-1.0, dtype='float32', persistable=True + ) # 1. The type of 'condition' in case must be Variable. def test_condition_type(): diff --git a/python/paddle/fluid/tests/unittests/test_switch_autotune.py b/python/paddle/fluid/tests/unittests/test_switch_autotune.py index 93dd06ec2916a..52b39fcf617c5 100644 --- a/python/paddle/fluid/tests/unittests/test_switch_autotune.py +++ b/python/paddle/fluid/tests/unittests/test_switch_autotune.py @@ -22,7 +22,6 @@ class SimpleNet(paddle.nn.Layer): - def __init__(self): super(SimpleNet, self).__init__() self.conv = paddle.nn.Conv2D(1, 2, (3, 3)) @@ -51,7 +50,6 @@ def static_program(net, data): class TestAutoTune(unittest.TestCase): - def set_flags(self, enable_autotune): if paddle.is_compiled_with_cuda(): if enable_autotune: @@ -67,7 +65,7 @@ def get_expected_res(self, step_id, enable_autotune): expected_res = { "step_id": step_id, "cache_size": 0, - "cache_hit_rate": 0 + "cache_hit_rate": 0, } if paddle.is_compiled_with_cuda(): # Total 3 * num_iters cache accesses, only iter 2 hits the cache. @@ -77,9 +75,8 @@ def get_expected_res(self, step_id, enable_autotune): def test_autotune(self): paddle.incubate.autotune.set_config( - config={"kernel": { - "enable": False - }}) + config={"kernel": {"enable": False}} + ) self.assertEqual(self.get_flags("FLAGS_use_autotune"), False) paddle.incubate.autotune.set_config(config={"kernel": {"enable": True}}) @@ -96,21 +93,17 @@ def check_status(self, expected_res): class TestDygraphAutoTuneStatus(TestAutoTune): - def run_program(self, enable_autotune): self.set_flags(enable_autotune) if enable_autotune: paddle.incubate.autotune.set_config( - config={"kernel": { - "enable": True, - "tuning_range": [1, 2] - }}) + config={"kernel": {"enable": True, "tuning_range": [1, 2]}} + ) else: paddle.incubate.autotune.set_config( - config={"kernel": { - "enable": False - }}) - x_var = paddle.uniform((1, 1, 8, 8), dtype='float32', min=-1., max=1.) + config={"kernel": {"enable": False}} + ) + x_var = paddle.uniform((1, 1, 8, 8), dtype='float32', min=-1.0, max=1.0) net = SimpleNet() for i in range(3): train_dygraph(net, x_var) @@ -135,7 +128,6 @@ def test_disable_autotune(self): class TestStaticAutoTuneStatus(TestAutoTune): - def run_program(self, enable_autotune): paddle.enable_static() @@ -143,13 +135,16 @@ def run_program(self, enable_autotune): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - data = paddle.static.data(name='X', - shape=data_shape, - dtype='float32') + data = paddle.static.data( + name='X', shape=data_shape, dtype='float32' + ) net = SimpleNet() loss = static_program(net, data) - place = paddle.CUDAPlace(0) if paddle.fluid.core.is_compiled_with_cuda( - ) else paddle.CPUPlace() + place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) exe = paddle.static.Executor(place) exe.run(startup_program) x = np.random.random(size=data_shape).astype('float32') @@ -164,10 +159,8 @@ def run_program(self, enable_autotune): os.remove(tfile.name) else: paddle.incubate.autotune.set_config( - config={"kernel": { - "enable": False, - "tuning_range": [1, 2] - }}) + config={"kernel": {"enable": False, "tuning_range": [1, 2]}} + ) for i in range(3): exe.run(program=main_program, feed={'X': x}, fetch_list=[loss]) @@ -190,7 +183,6 @@ def test_disable_autotune(self): class TestAutoTuneAPI(unittest.TestCase): - def test_set_config_warnings(self): with warnings.catch_warnings(record=True) as w: config = {"kernel": {"enable": 1, "tuning_range": 1}} @@ -204,7 +196,8 @@ def test_set_config_warnings(self): def test_set_config_attr(self): paddle.incubate.autotune.set_config(config=None) self.assertEqual( - paddle.get_flags("FLAGS_use_autotune")["FLAGS_use_autotune"], True) + paddle.get_flags("FLAGS_use_autotune")["FLAGS_use_autotune"], True + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_switch_case.py b/python/paddle/fluid/tests/unittests/test_switch_case.py index 3c529d789beb9..aad70fe789bfe 100644 --- a/python/paddle/fluid/tests/unittests/test_switch_case.py +++ b/python/paddle/fluid/tests/unittests/test_switch_case.py @@ -23,9 +23,7 @@ class TestAPISwitchCase(unittest.TestCase): - def test_return_single_var(self): - def fn_1(): return layers.fill_constant(shape=[4, 2], dtype='int32', value=1) @@ -43,86 +41,90 @@ def fn_3(): index_5 = layers.fill_constant(shape=[1], dtype='int32', value=5) # call fn_1 - out_0 = layers.switch_case(branch_index=index_1, - branch_fns={ - 1: fn_1, - 2: fn_2, - 3: fn_3 - }) + out_0 = layers.switch_case( + branch_index=index_1, branch_fns={1: fn_1, 2: fn_2, 3: fn_3} + ) # call fn_2 : branch_fns={0: fn_1, 1:fn_2, 2:fn_3} - out_1 = layers.switch_case(branch_index=index_1, - branch_fns=(fn_1, fn_2, fn_3)) + out_1 = layers.switch_case( + branch_index=index_1, branch_fns=(fn_1, fn_2, fn_3) + ) # call default fn_3 - out_2 = layers.switch_case(branch_index=index_5, - branch_fns=((1, fn_1), (2, fn_2)), - default=fn_3) + out_2 = layers.switch_case( + branch_index=index_5, + branch_fns=((1, fn_1), (2, fn_2)), + default=fn_3, + ) # no default, call fn_2 - out_3 = layers.switch_case(branch_index=index_2, - branch_fns=[(1, fn_1), (2, fn_2)]) + out_3 = layers.switch_case( + branch_index=index_2, branch_fns=[(1, fn_1), (2, fn_2)] + ) # no default, call fn_2 but branch_index is 5 - out_4 = layers.switch_case(branch_index=index_5, - branch_fns=[(1, fn_1), (3, fn_2), - (2, fn_3)]) - - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + out_4 = layers.switch_case( + branch_index=index_5, + branch_fns=[(1, fn_1), (3, fn_2), (2, fn_3)], + ) + + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(main_program, - fetch_list=[out_0, out_1, out_2, out_3, out_4]) + res = exe.run( + main_program, fetch_list=[out_0, out_1, out_2, out_3, out_4] + ) np.testing.assert_allclose( res[0], 1, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[0], 1)) + err_msg='result is {} but answer is {}'.format(res[0], 1), + ) np.testing.assert_allclose( res[1], 2, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[0], 2)) + err_msg='result is {} but answer is {}'.format(res[0], 2), + ) np.testing.assert_allclose( res[2], 3, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[0], 3)) + err_msg='result is {} but answer is {}'.format(res[0], 3), + ) np.testing.assert_allclose( res[3], 2, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[0], 2)) + err_msg='result is {} but answer is {}'.format(res[0], 2), + ) np.testing.assert_allclose( res[4], 2, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[0], 2)) + err_msg='result is {} but answer is {}'.format(res[0], 2), + ) def test_return_var_tuple(self): - def fn_1(): - return layers.fill_constant(shape=[1, 2], dtype='int32', - value=1), layers.fill_constant( - shape=[2, 3], - dtype='float32', - value=2) + return layers.fill_constant( + shape=[1, 2], dtype='int32', value=1 + ), layers.fill_constant(shape=[2, 3], dtype='float32', value=2) def fn_2(): - return layers.fill_constant(shape=[3, 4], dtype='int32', - value=3), layers.fill_constant( - shape=[4, 5], - dtype='float32', - value=4) + return layers.fill_constant( + shape=[3, 4], dtype='int32', value=3 + ), layers.fill_constant(shape=[4, 5], dtype='float32', value=4) def fn_3(): - return layers.fill_constant(shape=[5], dtype='int32', - value=5), layers.fill_constant( - shape=[5, 6], - dtype='float32', - value=6) + return layers.fill_constant( + shape=[5], dtype='int32', value=5 + ), layers.fill_constant(shape=[5, 6], dtype='float32', value=6) main_program = Program() startup_program = Program() @@ -131,66 +133,72 @@ def fn_3(): out = layers.switch_case(index_1, ((1, fn_1), (2, fn_2)), fn_3) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) ret = exe.run(main_program, fetch_list=out) - np.testing.assert_allclose(np.asarray(ret[0]), - np.full((1, 2), 1, np.int32), - rtol=1e-05) - np.testing.assert_allclose(np.asarray(ret[1]), - np.full((2, 3), 2, np.float32), - rtol=1e-05) + np.testing.assert_allclose( + np.asarray(ret[0]), np.full((1, 2), 1, np.int32), rtol=1e-05 + ) + np.testing.assert_allclose( + np.asarray(ret[1]), np.full((2, 3), 2, np.float32), rtol=1e-05 + ) class TestAPISwitchCase_Nested(unittest.TestCase): - def test_nested_switch_case(self): - def fn_1(x=1): - out = layers.switch_case(branch_index=layers.fill_constant( - shape=[1], dtype='int32', value=x), - branch_fns={ - 1: - partial(layers.fill_constant, - shape=[1], - dtype='int32', - value=1), - x: - partial(layers.fill_constant, - shape=[2], - dtype='int32', - value=x) - }) + out = layers.switch_case( + branch_index=layers.fill_constant( + shape=[1], dtype='int32', value=x + ), + branch_fns={ + 1: partial( + layers.fill_constant, shape=[1], dtype='int32', value=1 + ), + x: partial( + layers.fill_constant, shape=[2], dtype='int32', value=x + ), + }, + ) return out def fn_2(x=2): - out = layers.switch_case(branch_index=layers.fill_constant( - shape=[1], dtype='int32', value=2), - branch_fns={ - 1: - partial(layers.fill_constant, - shape=[4, 3], - dtype='int32', - value=1), - 2: - partial(fn_1, x=x) - }) + out = layers.switch_case( + branch_index=layers.fill_constant( + shape=[1], dtype='int32', value=2 + ), + branch_fns={ + 1: partial( + layers.fill_constant, + shape=[4, 3], + dtype='int32', + value=1, + ), + 2: partial(fn_1, x=x), + }, + ) return out def fn_3(): - out = layers.switch_case(branch_index=layers.fill_constant( - shape=[1], dtype='int32', value=3), - branch_fns={ - 1: - partial(layers.fill_constant, - shape=[4, 3], - dtype='int32', - value=1), - 3: - partial(fn_2, x=3) - }) + out = layers.switch_case( + branch_index=layers.fill_constant( + shape=[1], dtype='int32', value=3 + ), + branch_fns={ + 1: partial( + layers.fill_constant, + shape=[4, 3], + dtype='int32', + value=1, + ), + 3: partial(fn_2, x=3), + }, + ) return out main_program = Program() @@ -200,56 +208,53 @@ def fn_3(): index_2 = layers.fill_constant(shape=[1], dtype='int32', value=2) index_3 = layers.fill_constant(shape=[1], dtype='int64', value=3) - out_1 = layers.switch_case(branch_index=index_1, - branch_fns={ - 1: fn_1, - 2: fn_2, - 3: fn_3 - }) - out_2 = layers.switch_case(branch_index=index_2, - branch_fns={ - 1: fn_1, - 2: fn_2, - 3: fn_3 - }) - - out_3 = layers.switch_case(branch_index=index_3, - branch_fns={ - 1: fn_1, - 2: fn_2, - 3: fn_3 - }) - - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + out_1 = layers.switch_case( + branch_index=index_1, branch_fns={1: fn_1, 2: fn_2, 3: fn_3} + ) + out_2 = layers.switch_case( + branch_index=index_2, branch_fns={1: fn_1, 2: fn_2, 3: fn_3} + ) + + out_3 = layers.switch_case( + branch_index=index_3, branch_fns={1: fn_1, 2: fn_2, 3: fn_3} + ) + + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(main_program, - feed={"index_1": np.array([1], dtype="uint8")}, - fetch_list=[out_1, out_2, out_3]) + res = exe.run( + main_program, + feed={"index_1": np.array([1], dtype="uint8")}, + fetch_list=[out_1, out_2, out_3], + ) np.testing.assert_allclose( res[0], 1, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[0], 1)) + err_msg='result is {} but answer is {}'.format(res[0], 1), + ) np.testing.assert_allclose( res[1], 2, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[1], 2)) + err_msg='result is {} but answer is {}'.format(res[1], 2), + ) np.testing.assert_allclose( res[2], 3, rtol=1e-05, - err_msg='result is {} but answer is {}'.format(res[2], 3)) + err_msg='result is {} but answer is {}'.format(res[2], 3), + ) # test TypeError and ValueError of api switch_case class TestAPISwitchCase_Error(unittest.TestCase): - def test_error(self): - def fn_1(): return layers.fill_constant(shape=[4, 2], dtype='int32', value=1) @@ -262,82 +267,90 @@ def fn_3(): main_program = Program() startup_program = Program() with program_guard(main_program, startup_program): - key_float32 = layers.fill_constant(shape=[1], - dtype='float32', - value=0.23) - key_int32 = layers.fill_constant(shape=[1], - dtype='int32', - value=0.23) + key_float32 = layers.fill_constant( + shape=[1], dtype='float32', value=0.23 + ) + key_int32 = layers.fill_constant( + shape=[1], dtype='int32', value=0.23 + ) # The type of 'branch_index' in Op(switch_case) must be Variable def type_error_branch_index(): - layers.switch_case(branch_index=1, - branch_fns=[(1, fn_1)], - default=fn_3) + layers.switch_case( + branch_index=1, branch_fns=[(1, fn_1)], default=fn_3 + ) self.assertRaises(TypeError, type_error_branch_index) # The data type of 'branch_index' in Op(switch_case) must be int32, int64 or uint8 def dtype_error_branch_index(): - layers.switch_case(branch_index=key_float32, - branch_fns=[(1, fn_1)], - default=fn_3) + layers.switch_case( + branch_index=key_float32, + branch_fns=[(1, fn_1)], + default=fn_3, + ) self.assertRaises(TypeError, dtype_error_branch_index) # The type of 'branch_fns' in Op(switch_case) must be list, tuple or dict def type_error_branch_fns(): - layers.switch_case(branch_index=key_int32, - branch_fns=1, - default=fn_3) + layers.switch_case( + branch_index=key_int32, branch_fns=1, default=fn_3 + ) self.assertRaises(TypeError, type_error_branch_fns) # The elements' type of 'branch_fns' in Op(switch_case) must be tuple def type_error_index_fn_pair_1(): - layers.switch_case(branch_index=key_int32, - branch_fns=[1], - default=fn_3) + layers.switch_case( + branch_index=key_int32, branch_fns=[1], default=fn_3 + ) self.assertRaises(TypeError, type_error_index_fn_pair_1) # The tuple's size of 'branch_fns' in Op(switch_case) must be 2 def type_error_index_fn_pair_2(): - layers.switch_case(branch_index=key_int32, - branch_fns=[(1, 2, 3)], - default=fn_3) + layers.switch_case( + branch_index=key_int32, branch_fns=[(1, 2, 3)], default=fn_3 + ) self.assertRaises(TypeError, type_error_index_fn_pair_2) # The key's type of 'branch_fns' in Op(switch_case) must be int def type_error_key(): - layers.switch_case(branch_index=key_int32, - branch_fns=[(2.3, 2)], - default=fn_3) + layers.switch_case( + branch_index=key_int32, branch_fns=[(2.3, 2)], default=fn_3 + ) self.assertRaises(TypeError, type_error_key) # The key in 'branch_fns' must be unique def value_error_key(): - layers.switch_case(branch_index=key_int32, - branch_fns=[(2, fn_1), (2, fn_2)], - default=fn_3) + layers.switch_case( + branch_index=key_int32, + branch_fns=[(2, fn_1), (2, fn_2)], + default=fn_3, + ) self.assertRaises(ValueError, value_error_key) # The type of function in 'branch_fns' must be callable def type_error_fn(): - layers.switch_case(branch_index=key_int32, - branch_fns=[(1, 1), (2, fn_2)], - default=fn_3) + layers.switch_case( + branch_index=key_int32, + branch_fns=[(1, 1), (2, fn_2)], + default=fn_3, + ) self.assertRaises(TypeError, type_error_fn) # The default in Op(case) must be callable def type_error_default(): - layers.switch_case(branch_index=key_int32, - branch_fns=[(1, fn_1), (2, fn_2)], - default=1) + layers.switch_case( + branch_index=key_int32, + branch_fns=[(1, fn_1), (2, fn_2)], + default=1, + ) self.assertRaises(TypeError, type_error_default) diff --git a/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py b/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py index 055187d8aabc3..e2a53465d44cc 100644 --- a/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py @@ -47,7 +47,7 @@ class TestSyncBatchNormOpTraining(unittest.TestCase): def setUp(self): """Setup.""" - #self.dtype = np.float32 + # self.dtype = np.float32 self.dtype = np.float32 if core.is_compiled_with_rocm() else np.float64 self.N = 8 self.C = 16 @@ -56,12 +56,9 @@ def setUp(self): self.dshape = [self.N, self.C, self.H, self.W] self.atol = 1e-3 - def _build_program(self, - place, - layout, - seed, - sync_bn=False, - only_forward=False): + def _build_program( + self, place, layout, seed, sync_bn=False, only_forward=False + ): """Build program.""" main = fluid.Program() startup = fluid.Program() @@ -70,17 +67,20 @@ def _build_program(self, use_cudnn = self.dtype == np.float16 with fluid.unique_name.guard(): with fluid.program_guard(main, startup): - data = fluid.layers.data(name='input', - shape=self.dshape, - dtype=self.dtype, - append_batch_size=False) + data = fluid.layers.data( + name='input', + shape=self.dshape, + dtype=self.dtype, + append_batch_size=False, + ) conv = fluid.layers.conv2d( input=data, num_filters=32, filter_size=1, param_attr=fluid.ParamAttr(name='conv2d_weight'), bias_attr=False, - use_cudnn=use_cudnn) + use_cudnn=use_cudnn, + ) bn = fluid.layers.batch_norm( conv, param_attr=fluid.ParamAttr(name='bn_scale'), @@ -88,7 +88,8 @@ def _build_program(self, moving_mean_name='bn_moving_mean', moving_variance_name='bn_moving_variance', data_layout=layout, - is_test=only_forward) + is_test=only_forward, + ) if core.is_compiled_with_rocm(): bn = fluid.layers.cast(bn, 'float32') else: @@ -108,42 +109,59 @@ def _compare(self, place, layout, only_forward): seed = 10 os.environ['FLAGS_cudnn_deterministic'] = "1" scope = core.Scope() - data = np.random.random(size=self.dshape).astype(self.dtype) * 4. - 2 - data = create_or_get_tensor(scope, "input", - OpTest.np_dtype_to_fluid_dtype(data), place) + data = np.random.random(size=self.dshape).astype(self.dtype) * 4.0 - 2 + data = create_or_get_tensor( + scope, "input", OpTest.np_dtype_to_fluid_dtype(data), place + ) # Single-GPU, N = 32 per GPU - main, startup, outs = self._build_program(place, layout, seed, False, - only_forward) + main, startup, outs = self._build_program( + place, layout, seed, False, only_forward + ) exe = fluid.Executor(place) exe.run(startup) fetch_names = [v.name for v in outs] + [ - 'bn_moving_mean', 'bn_moving_variance', 'bn_scale', 'bn_bias' + 'bn_moving_mean', + 'bn_moving_variance', + 'bn_scale', + 'bn_bias', ] if not only_forward: others = [ - 'batch_norm_0.tmp_0', 'batch_norm_0.tmp_1', 'bn_scale@GRAD', - 'bn_bias@GRAD', 'batch_norm_0.tmp_3@GRAD', 'conv2d_0.tmp_0@GRAD' + 'batch_norm_0.tmp_0', + 'batch_norm_0.tmp_1', + 'bn_scale@GRAD', + 'bn_bias@GRAD', + 'batch_norm_0.tmp_3@GRAD', + 'conv2d_0.tmp_0@GRAD', ] fetch_names += others - bn_fetches = exe.run(program=main, - feed={'input': data}, - fetch_list=fetch_names) + bn_fetches = exe.run( + program=main, feed={'input': data}, fetch_list=fetch_names + ) ##################################################################### # Multi-GPUs, self.N / core.get_cuda_device_count() per GPU assert core.get_cuda_device_count() > 1 - main, startup, outs = self._build_program(place, layout, seed, True, - only_forward) + main, startup, outs = self._build_program( + place, layout, seed, True, only_forward + ) exe = fluid.Executor(place) exe.run(startup) fetch_names = [v.name for v in outs] + [ - 'bn_moving_mean', 'bn_moving_variance', 'bn_scale', 'bn_bias' + 'bn_moving_mean', + 'bn_moving_variance', + 'bn_scale', + 'bn_bias', ] if not only_forward: others = [ - 'batch_norm_0.tmp_0', 'batch_norm_0.tmp_1', 'bn_scale@GRAD', - 'bn_bias@GRAD', 'batch_norm_0.tmp_3@GRAD', 'conv2d_0.tmp_0@GRAD' + 'batch_norm_0.tmp_0', + 'batch_norm_0.tmp_1', + 'bn_scale@GRAD', + 'bn_bias@GRAD', + 'batch_norm_0.tmp_3@GRAD', + 'conv2d_0.tmp_0@GRAD', ] fetch_names += others for nm in fetch_names: @@ -155,24 +173,31 @@ def _compare(self, place, layout, only_forward): build_strategy.memory_optimize = False comp_prog = compiler.CompiledProgram(main).with_data_parallel( outs[0].name if not only_forward else None, - build_strategy=build_strategy) - sync_bn_fetches = exe.run(program=comp_prog, - feed={'input': data}, - fetch_list=fetch_names) + build_strategy=build_strategy, + ) + sync_bn_fetches = exe.run( + program=comp_prog, feed={'input': data}, fetch_list=fetch_names + ) for i in range(1, len(sync_bn_fetches)): bn_val = bn_fetches[i] sync_bn_val = sync_bn_fetches[i] if sync_bn_val.shape != bn_val.shape: - sync_bn_val = sync_bn_val[:bn_val.shape[0]] - np.testing.assert_allclose(bn_val, - sync_bn_val, - rtol=1e-05, - atol=self.atol, - err_msg='Output (' + fetch_names[i] + - ') has diff. \n' + '\nBN ' + - str(bn_val) + '\n' + 'Sync BN ' + - str(sync_bn_val)) + sync_bn_val = sync_bn_val[: bn_val.shape[0]] + np.testing.assert_allclose( + bn_val, + sync_bn_val, + rtol=1e-05, + atol=self.atol, + err_msg='Output (' + + fetch_names[i] + + ') has diff. \n' + + '\nBN ' + + str(bn_val) + + '\n' + + 'Sync BN ' + + str(sync_bn_val), + ) def test_train(self): """Test training.""" @@ -210,15 +235,15 @@ def setUp(self): class TestDygraphSyncBatchNormAPIError(unittest.TestCase): - def test_errors(self): if not core.is_compiled_with_cuda(): return with program_guard(Program(), Program()): my_sync_batch_norm = paddle.nn.SyncBatchNorm(10) - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.CUDAPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CUDAPlace(0) + ) self.assertRaises(TypeError, my_sync_batch_norm, x1) # the input dtype of SyncBatchNorm must be float16 or float32 or float64 @@ -228,36 +253,39 @@ def test_errors(self): class TestConvertSyncBatchNorm(unittest.TestCase): - def test_convert(self): if not core.is_compiled_with_cuda(): return with program_guard(Program(), Program()): - compare_model = paddle.nn.Sequential(paddle.nn.Conv2D(3, 5, 3), - paddle.nn.BatchNorm2D(5), - paddle.nn.BatchNorm2D(5)) + compare_model = paddle.nn.Sequential( + paddle.nn.Conv2D(3, 5, 3), + paddle.nn.BatchNorm2D(5), + paddle.nn.BatchNorm2D(5), + ) model = paddle.nn.Sequential( - paddle.nn.Conv2D(3, 5, 3), paddle.nn.BatchNorm2D(5), + paddle.nn.Conv2D(3, 5, 3), + paddle.nn.BatchNorm2D(5), paddle.nn.BatchNorm2D( 5, weight_attr=fluid.ParamAttr(name='bn.scale'), - bias_attr=fluid.ParamAttr(name='bn.bias'))) + bias_attr=fluid.ParamAttr(name='bn.bias'), + ), + ) model = paddle.nn.SyncBatchNorm.convert_sync_batchnorm(model) for idx, sublayer in enumerate(compare_model.sublayers()): if isinstance(sublayer, paddle.nn.BatchNorm2D): self.assertEqual( - isinstance(model[idx], paddle.nn.SyncBatchNorm), True) + isinstance(model[idx], paddle.nn.SyncBatchNorm), True + ) class TestConvertSyncBatchNormCast1(unittest.TestCase): - def test_convert(self): if not core.is_compiled_with_cuda(): return class Net(nn.Layer): - def __init__(self): super(Net, self).__init__() self.conv1 = nn.Conv2D(3, 5, 3) @@ -282,7 +310,6 @@ def forward(self, x): class TestConvertSyncBatchNormCase2(unittest.TestCase): - def test_convert(self): if not core.is_compiled_with_cuda(): return @@ -290,16 +317,19 @@ def test_convert(self): with fluid.dygraph.guard(fluid.CUDAPlace(0)): class SyBNNet(paddle.nn.Layer): - def __init__(self, in_ch=3, out_ch=3, dirate=1): super(SyBNNet, self).__init__() self.bn_s1 = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( paddle.nn.BatchNorm3D( out_ch, weight_attr=paddle.ParamAttr( - regularizer=paddle.regularizer.L2Decay(0.)))) + regularizer=paddle.regularizer.L2Decay(0.0) + ), + ) + ) self.bn_s2 = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( - paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC')) + paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC') + ) def forward(self, x): x = self.bn_s1(x) @@ -307,15 +337,17 @@ def forward(self, x): return out class BNNet(paddle.nn.Layer): - def __init__(self, in_ch=3, out_ch=3, dirate=1): super(BNNet, self).__init__() self.bn_s1 = paddle.nn.BatchNorm3D( out_ch, weight_attr=paddle.ParamAttr( - regularizer=paddle.regularizer.L2Decay(0.))) + regularizer=paddle.regularizer.L2Decay(0.0) + ), + ) self.bn_s2 = paddle.nn.SyncBatchNorm.convert_sync_batchnorm( - paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC')) + paddle.nn.BatchNorm3D(out_ch, data_format='NDHWC') + ) def forward(self, x): x = self.bn_s1(x) @@ -333,12 +365,16 @@ def forward(self, x): bn_out.numpy(), sybn_out.numpy(), rtol=1e-05, - err_msg='Output has diff. \n' + '\nBN ' + - str(bn_out.numpy()) + '\n' + 'Sync BN ' + str(sybn_out.numpy())) + err_msg='Output has diff. \n' + + '\nBN ' + + str(bn_out.numpy()) + + '\n' + + 'Sync BN ' + + str(sybn_out.numpy()), + ) class TestDygraphSyncBatchNormDataFormatError(unittest.TestCase): - def test_errors(self): if not core.is_compiled_with_cuda(): return diff --git a/python/paddle/fluid/tests/unittests/test_take.py b/python/paddle/fluid/tests/unittests/test_take.py index 27ccee81d764d..f713d777c1683 100644 --- a/python/paddle/fluid/tests/unittests/test_take.py +++ b/python/paddle/fluid/tests/unittests/test_take.py @@ -21,7 +21,6 @@ class TestTakeAPI(unittest.TestCase): - def set_mode(self): self.mode = 'raise' @@ -32,41 +31,46 @@ def set_dtype(self): def set_input(self): self.input_shape = [3, 4] self.index_shape = [2, 3] - self.input_np = np.arange(0, 12).reshape(self.input_shape).astype( - self.input_dtype) - self.index_np = np.arange(-4, 2).reshape(self.index_shape).astype( - self.index_dtype) + self.input_np = ( + np.arange(0, 12).reshape(self.input_shape).astype(self.input_dtype) + ) + self.index_np = ( + np.arange(-4, 2).reshape(self.index_shape).astype(self.index_dtype) + ) def setUp(self): self.set_mode() self.set_dtype() self.set_input() - self.place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def test_static_graph(self): paddle.enable_static() startup_program = Program() train_program = Program() with program_guard(startup_program, train_program): - x = fluid.data(name='input', - dtype=self.input_dtype, - shape=self.input_shape) - index = fluid.data(name='index', - dtype=self.index_dtype, - shape=self.index_shape) + x = fluid.data( + name='input', dtype=self.input_dtype, shape=self.input_shape + ) + index = fluid.data( + name='index', dtype=self.index_dtype, shape=self.index_shape + ) out = paddle.take(x, index, mode=self.mode) exe = fluid.Executor(self.place) - st_result = exe.run(fluid.default_main_program(), - feed={ - 'input': self.input_np, - 'index': self.index_np - }, - fetch_list=out) + st_result = exe.run( + fluid.default_main_program(), + feed={'input': self.input_np, 'index': self.index_np}, + fetch_list=out, + ) np.testing.assert_allclose( st_result[0], - np.take(self.input_np, self.index_np, mode=self.mode)) + np.take(self.input_np, self.index_np, mode=self.mode), + ) def test_dygraph(self): paddle.disable_static(self.place) @@ -75,7 +79,8 @@ def test_dygraph(self): dy_result = paddle.take(x, index, mode=self.mode) np.testing.assert_allclose( np.take(self.input_np, self.index_np, mode=self.mode), - dy_result.numpy()) + dy_result.numpy(), + ) class TestTakeInt32(TestTakeAPI): @@ -109,11 +114,12 @@ def test_static_type_error(self): """Argument 'index' must be Tensor""" paddle.enable_static() with program_guard(Program()): - x = fluid.data(name='input', - dtype=self.input_dtype, - shape=self.input_shape) - self.assertRaises(TypeError, paddle.take, x, self.index_np, - self.mode) + x = fluid.data( + name='input', dtype=self.input_dtype, shape=self.input_shape + ) + self.assertRaises( + TypeError, paddle.take, x, self.index_np, self.mode + ) def test_dygraph_type_error(self): paddle.disable_static(self.place) @@ -124,12 +130,12 @@ def test_static_dtype_error(self): """Data type of argument 'index' must be in [paddle.int32, paddle.int64]""" paddle.enable_static() with program_guard(Program()): - x = fluid.data(name='input', - dtype='float64', - shape=self.input_shape) - index = fluid.data(name='index', - dtype='float32', - shape=self.index_shape) + x = fluid.data( + name='input', dtype='float64', shape=self.input_shape + ) + index = fluid.data( + name='index', dtype='float32', shape=self.index_shape + ) self.assertRaises(TypeError, paddle.take, x, index, self.mode) def test_dygraph_dtype_error(self): @@ -152,29 +158,36 @@ def set_dtype(self): def set_input(self): self.input_shape = [3, 4] self.index_shape = [5, 6] - self.input_np = np.arange(0, 12).reshape(self.input_shape).astype( - self.input_dtype) - self.index_np = np.arange(-10, 20).reshape(self.index_shape).astype( - self.index_dtype) # positive indices are out of range + self.input_np = ( + np.arange(0, 12).reshape(self.input_shape).astype(self.input_dtype) + ) + self.index_np = ( + np.arange(-10, 20) + .reshape(self.index_shape) + .astype(self.index_dtype) + ) # positive indices are out of range def setUp(self): self.set_mode() self.set_dtype() self.set_input() - self.place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) def test_static_index_error(self): """When the index is out of range, an error is reported directly through `paddle.index_select`""" paddle.enable_static() with program_guard(Program()): - x = fluid.data(name='input', - dtype=self.input_dtype, - shape=self.input_shape) - index = fluid.data(name='index', - dtype=self.index_dtype, - shape=self.index_shape) + x = fluid.data( + name='input', dtype=self.input_dtype, shape=self.input_shape + ) + index = fluid.data( + name='index', dtype=self.index_dtype, shape=self.index_shape + ) self.assertRaises(ValueError, paddle.index_select, x, index) def test_dygraph_index_error(self): @@ -197,17 +210,24 @@ def set_dtype(self): def set_input(self): self.input_shape = [3, 4] self.index_shape = [5, 6] - self.input_np = np.arange(0, 12).reshape(self.input_shape).astype( - self.input_dtype) - self.index_np = np.arange(-20, 10).reshape(self.index_shape).astype( - self.index_dtype) # negative indices are out of range + self.input_np = ( + np.arange(0, 12).reshape(self.input_shape).astype(self.input_dtype) + ) + self.index_np = ( + np.arange(-20, 10) + .reshape(self.index_shape) + .astype(self.index_dtype) + ) # negative indices are out of range def setUp(self): self.set_mode() self.set_dtype() self.set_input() - self.place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + self.place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) class TestTakeModeWrap(TestTakeAPI): @@ -219,10 +239,14 @@ def set_mode(self): def set_input(self): self.input_shape = [3, 4] self.index_shape = [5, 8] - self.input_np = np.arange(0, 12).reshape(self.input_shape).astype( - self.input_dtype) - self.index_np = np.arange(-20, 20).reshape(self.index_shape).astype( - self.index_dtype) # Both ends of the index are out of bounds + self.input_np = ( + np.arange(0, 12).reshape(self.input_shape).astype(self.input_dtype) + ) + self.index_np = ( + np.arange(-20, 20) + .reshape(self.index_shape) + .astype(self.index_dtype) + ) # Both ends of the index are out of bounds class TestTakeModeClip(TestTakeAPI): @@ -234,10 +258,14 @@ def set_mode(self): def set_input(self): self.input_shape = [3, 4] self.index_shape = [5, 8] - self.input_np = np.arange(0, 12).reshape(self.input_shape).astype( - self.input_dtype) - self.index_np = np.arange(-20, 20).reshape(self.index_shape).astype( - self.index_dtype) # Both ends of the index are out of bounds + self.input_np = ( + np.arange(0, 12).reshape(self.input_shape).astype(self.input_dtype) + ) + self.index_np = ( + np.arange(-20, 20) + .reshape(self.index_shape) + .astype(self.index_dtype) + ) # Both ends of the index are out of bounds if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_take_along_axis_op.py b/python/paddle/fluid/tests/unittests/test_take_along_axis_op.py index 947ea1fb6f2c2..a7a65aec32611 100644 --- a/python/paddle/fluid/tests/unittests/test_take_along_axis_op.py +++ b/python/paddle/fluid/tests/unittests/test_take_along_axis_op.py @@ -22,7 +22,6 @@ class TestTakeAlongAxisOp(OpTest): - def setUp(self): self.init_data() self.op_type = "take_along_axis" @@ -49,14 +48,14 @@ def init_data(self): self.x_type = "float64" self.x_shape = (5, 5, 5) self.index_type = "int32" - self.index = np.array([[[1]], [[1]], [[2]], [[4]], - [[3]]]).astype(self.index_type) + self.index = np.array([[[1]], [[1]], [[2]], [[4]], [[3]]]).astype( + self.index_type + ) self.axis = 2 self.axis_type = "int64" class TestCase1(TestTakeAlongAxisOp): - def init_data(self): self.x_type = "float64" self.x_shape = (5, 5, 5) @@ -67,7 +66,6 @@ def init_data(self): class TestTakeAlongAxisAPI(unittest.TestCase): - def setUp(self): np.random.seed(0) self.shape = [3, 3] @@ -86,13 +84,12 @@ def test_api_static(self): index = paddle.fluid.data('Index', self.index_shape, "int64") out = paddle.take_along_axis(x, index, self.axis) exe = paddle.static.Executor(self.place[0]) - res = exe.run(feed={ - 'X': self.x_np, - 'Index': self.index_np - }, - fetch_list=[out]) + res = exe.run( + feed={'X': self.x_np, 'Index': self.index_np}, fetch_list=[out] + ) out_ref = np.array( - np.take_along_axis(self.x_np, self.index_np, self.axis)) + np.take_along_axis(self.x_np, self.index_np, self.axis) + ) for out in res: np.testing.assert_allclose(out, out_ref, rtol=0.001) @@ -102,19 +99,20 @@ def test_api_dygraph(self): self.index = paddle.to_tensor(self.index_np) out = paddle.take_along_axis(x_tensor, self.index, self.axis) out_ref = np.array( - np.take_along_axis(self.x_np, self.index_np, self.axis)) + np.take_along_axis(self.x_np, self.index_np, self.axis) + ) np.testing.assert_allclose(out.numpy(), out_ref, rtol=0.001) paddle.enable_static() class TestTakeAlongAxisAPICase1(TestTakeAlongAxisAPI): - def setUp(self): np.random.seed(0) self.shape = [2, 2] self.index_shape = [4, 2] - self.index_np = np.array([[0, 0], [1, 0], [0, 0], [1, - 0]]).astype('int64') + self.index_np = np.array([[0, 0], [1, 0], [0, 0], [1, 0]]).astype( + 'int64' + ) self.x_np = np.random.random(self.shape).astype(np.float32) self.place = [paddle.CPUPlace()] self.axis = 0 diff --git a/python/paddle/fluid/tests/unittests/test_target_assign_op.py b/python/paddle/fluid/tests/unittests/test_target_assign_op.py index d51444da406a7..103f28a202415 100644 --- a/python/paddle/fluid/tests/unittests/test_target_assign_op.py +++ b/python/paddle/fluid/tests/unittests/test_target_assign_op.py @@ -36,16 +36,23 @@ def gen_match_and_neg_indices(num_prior, gt_lod, neg_lod): ret_ids = set([i for i in range(num_prior)]) - set(ids) l = neg_lod[n] neg_ids = random.sample(ret_ids, l) - neg_indices[offset:offset + - neg_lod[n], :] = np.array(neg_ids).astype('int32').reshape( - l, 1) + neg_indices[offset : offset + neg_lod[n], :] = ( + np.array(neg_ids).astype('int32').reshape(l, 1) + ) offset += neg_lod[n] return match_indices, neg_indices -def target_assign(encoded_box, gt_label, match_indices, neg_indices, gt_lod, - neg_lod, mismatch_value): +def target_assign( + encoded_box, + gt_label, + match_indices, + neg_indices, + gt_lod, + neg_lod, + mismatch_value, +): batch_size, num_prior = match_indices.shape # init target bbox @@ -75,7 +82,7 @@ def target_assign(encoded_box, gt_label, match_indices, neg_indices, gt_lod, trg_label_wt[i][col_ids] = 1.0 # set target label weight to 1.0 for the negative samples if neg_indices is not None: - neg_ids = neg_indices[neg_offset:neg_offset + neg_lod[i]] + neg_ids = neg_indices[neg_offset : neg_offset + neg_lod[i]] trg_label_wt[i][neg_ids] = 1.0 # update offset gt_offset += gt_lod[i] @@ -85,7 +92,6 @@ def target_assign(encoded_box, gt_label, match_indices, neg_indices, gt_lod, class TestTargetAssginFloatType(OpTest): - def setUp(self): self.op_type = "target_assign" num_prior = 120 @@ -97,15 +103,23 @@ def setUp(self): num_gt = sum(gt_lod) encoded_box = np.random.random((num_gt, num_prior, 4)).astype('float32') - gt_label = np.random.randint(num_class, - size=(num_gt, 1)).astype('int32') + gt_label = np.random.randint(num_class, size=(num_gt, 1)).astype( + 'int32' + ) match_indices, neg_indices = gen_match_and_neg_indices( - num_prior, gt_lod, neg_lod) - - out, out_wt, _, _ = target_assign(encoded_box, gt_label, match_indices, - neg_indices, gt_lod, neg_lod, - mismatch_value) + num_prior, gt_lod, neg_lod + ) + + out, out_wt, _, _ = target_assign( + encoded_box, + gt_label, + match_indices, + neg_indices, + gt_lod, + neg_lod, + mismatch_value, + ) # assign regression targets x = encoded_box @@ -124,7 +138,6 @@ def test_check_output(self): class TestTargetAssginIntType(OpTest): - def setUp(self): self.op_type = "target_assign" num_prior = 120 @@ -136,15 +149,23 @@ def setUp(self): num_gt = sum(gt_lod) encoded_box = np.random.random((num_gt, num_prior, 4)).astype('float32') - gt_label = np.random.randint(num_class, - size=(num_gt, 1)).astype('int32') + gt_label = np.random.randint(num_class, size=(num_gt, 1)).astype( + 'int32' + ) match_indices, neg_indices = gen_match_and_neg_indices( - num_prior, gt_lod, neg_lod) - - _, _, out, out_wt, = target_assign(encoded_box, gt_label, match_indices, - neg_indices, gt_lod, neg_lod, - mismatch_value) + num_prior, gt_lod, neg_lod + ) + + _, _, out, out_wt, = target_assign( + encoded_box, + gt_label, + match_indices, + neg_indices, + gt_lod, + neg_lod, + mismatch_value, + ) # assign cassification argets x = np.reshape(gt_label, (num_gt, 1, 1)) diff --git a/python/paddle/fluid/tests/unittests/test_tdm_child_op.py b/python/paddle/fluid/tests/unittests/test_tdm_child_op.py index 5c17b737eef5d..95c49c904db5b 100644 --- a/python/paddle/fluid/tests/unittests/test_tdm_child_op.py +++ b/python/paddle/fluid/tests/unittests/test_tdm_child_op.py @@ -52,15 +52,15 @@ def create_tdm_tree(): class TestTDMChildOp(OpTest): - def setUp(self): self.__class__.op_type = "tdm_child" self.config() tree_info = create_tdm_tree() tree_info_np = np.array(tree_info).astype(self.info_type) - x_np = np.random.randint(low=0, high=26, - size=self.x_shape).astype(self.x_type) + x_np = np.random.randint(low=0, high=26, size=self.x_shape).astype( + self.x_type + ) children_res = [] leaf_mask_res = [] for batch in x_np: @@ -100,9 +100,8 @@ def test_check_output(self): class TestCase1(TestTDMChildOp): - def config(self): - """check int int64_t """ + """check int int64_t""" self.x_shape = (10, 20) self.child_shape = (10, 20, 2) self.x_type = 'int32' @@ -110,9 +109,8 @@ def config(self): class TestCase2(TestTDMChildOp): - def config(self): - """check int64_t int64_t """ + """check int64_t int64_t""" self.x_shape = (10, 20) self.child_shape = (10, 20, 2) self.x_type = 'int64' @@ -120,9 +118,8 @@ def config(self): class TestCase3(TestTDMChildOp): - def config(self): - """check int64 int32 """ + """check int64 int32""" self.x_shape = (10, 20) self.child_shape = (10, 20, 2) self.x_type = 'int64' @@ -130,9 +127,8 @@ def config(self): class TestCase4(TestTDMChildOp): - def config(self): - """check large shape """ + """check large shape""" self.x_shape = (100, 20) self.child_shape = (100, 20, 2) self.x_type = 'int32' @@ -140,7 +136,6 @@ def config(self): class TestTDMChildShape(unittest.TestCase): - def test_shape(self): x = fluid.layers.data(name='x', shape=[1], dtype='int32', lod_level=1) tdm_tree_info = create_tdm_tree() @@ -150,17 +145,21 @@ def test_shape(self): x=x, node_nums=26, child_nums=2, - param_attr=fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(tree_info_np))) + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer( + tree_info_np + ) + ), + ) place = fluid.CPUPlace() exe = fluid.Executor(place=place) exe.run(fluid.default_startup_program()) feed = { - 'x': - np.array([[1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], - [12]]).astype('int32') + 'x': np.array( + [[1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12]] + ).astype('int32') } exe.run(feed=feed) diff --git a/python/paddle/fluid/tests/unittests/test_tdm_sampler_op.py b/python/paddle/fluid/tests/unittests/test_tdm_sampler_op.py index 1d576038e706a..9c9ce12078008 100644 --- a/python/paddle/fluid/tests/unittests/test_tdm_sampler_op.py +++ b/python/paddle/fluid/tests/unittests/test_tdm_sampler_op.py @@ -21,27 +21,41 @@ def create_tdm_travel(): - tree_travel = [[1, 3, 7, 14], [1, 3, 7, 15], [1, 3, 8, 16], [1, 3, 8, 17], - [1, 4, 9, 18], [1, 4, 9, 19], [1, 4, 10, 20], [1, 4, 10, 21], - [2, 5, 11, 22], [2, 5, 11, 23], [2, 5, 12, 24], - [2, 5, 12, 25], [2, 6, 13, 0]] + tree_travel = [ + [1, 3, 7, 14], + [1, 3, 7, 15], + [1, 3, 8, 16], + [1, 3, 8, 17], + [1, 4, 9, 18], + [1, 4, 9, 19], + [1, 4, 10, 20], + [1, 4, 10, 21], + [2, 5, 11, 22], + [2, 5, 11, 23], + [2, 5, 12, 24], + [2, 5, 12, 25], + [2, 6, 13, 0], + ] return tree_travel def create_tdm_layer(): - tree_layer = [[1, 2], [3, 4, 5, 6], [7, 8, 9, 10, 11, 12, 13], - [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]] + tree_layer = [ + [1, 2], + [3, 4, 5, 6], + [7, 8, 9, 10, 11, 12, 13], + [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25], + ] return tree_layer type_dict = { "int32": int(core.VarDesc.VarType.INT32), - "int64": int(core.VarDesc.VarType.INT64) + "int64": int(core.VarDesc.VarType.INT64), } class TestTDMSamplerOp(OpTest): - def setUp(self): self.__class__.op_type = "tdm_sampler" self.config() @@ -50,8 +64,9 @@ def setUp(self): self.tree_layer = create_tdm_layer() output_0 = self.x_shape[0] - output_1 = len(self.neg_samples_num_list) + \ - np.sum(self.neg_samples_num_list) + output_1 = len(self.neg_samples_num_list) + np.sum( + self.neg_samples_num_list + ) self.output_shape = (output_0, output_1) self.layer_sample_nums = [1 + i for i in self.neg_samples_num_list] @@ -68,8 +83,9 @@ def setUp(self): layer_np = np.array(tree_layer_flat).astype(self.tree_dtype) layer_np = layer_np.reshape([-1, 1]) - self.x_np = np.random.randint(low=0, high=13, - size=self.x_shape).astype(self.x_type) + self.x_np = np.random.randint(low=0, high=13, size=self.x_shape).astype( + self.x_type + ) out = np.random.random(self.output_shape).astype(self.out_dtype) label = np.random.random(self.output_shape).astype(self.out_dtype) @@ -80,7 +96,7 @@ def setUp(self): 'output_positive': True, 'layer_offset_lod': tree_layer_offset_lod, 'seed': 0, - 'dtype': type_dict[self.out_dtype] + 'dtype': type_dict[self.out_dtype], } self.inputs = {'X': self.x_np, 'Travel': travel_np, 'Layer': layer_np} self.outputs = {'Out': out, 'Labels': label, 'Mask': mask} @@ -128,7 +144,8 @@ def test_check_output(self): positive_travel.append(sampling_res_list[0]) label_sampling_res = label_res[batch_ids][ - start_offset:end_offset] + start_offset:end_offset + ] mask_sampling_res = mask_res[batch_ids][start_offset:end_offset] # check unique @@ -136,8 +153,12 @@ def test_check_output(self): assert len(set(sampling_res_list)) == len( sampling_res_list ), "len(set(sampling_res_list)): {}, len(sampling_res_list): {} , sample_res: {}, label_res:{}, mask_res: {}".format( - len(set(sampling_res_list)), len(sampling_res_list), - sampling_res, label_sampling_res, mask_sampling_res) + len(set(sampling_res_list)), + len(sampling_res_list), + sampling_res, + label_sampling_res, + mask_sampling_res, + ) # check legal layer_node = self.tree_layer[layer_idx] layer_node.append(0) @@ -145,8 +166,12 @@ def test_check_output(self): assert ( sample in layer_node ), "sample: {}, layer_node: {} , sample_res: {}, label_res: {}, mask_res:{}".format( - sample, layer_node, sampling_res, label_sampling_res, - mask_sampling_res) + sample, + layer_node, + sampling_res, + label_sampling_res, + mask_sampling_res, + ) # check label label_flag = 1 @@ -158,15 +183,16 @@ def test_check_output(self): assert not np.sum( mask_sampling_res[padding_index] ), "np.sum(mask_sampling_res[padding_index]): {} ".format( - np.sum(mask_sampling_res[padding_index])) + np.sum(mask_sampling_res[padding_index]) + ) start_offset = end_offset # check travel legal - assert self.tree_travel[int( - self.x_np[batch_ids])] == positive_travel + assert ( + self.tree_travel[int(self.x_np[batch_ids])] == positive_travel + ) class TestCase1(TestTDMSamplerOp): - def config(self): """test input int64""" self.neg_samples_num_list = [0, 0, 0, 0] @@ -177,7 +203,6 @@ def config(self): class TestCase2(TestTDMSamplerOp): - def config(self): """test dtype int64""" self.neg_samples_num_list = [0, 0, 0, 0] @@ -188,7 +213,6 @@ def config(self): class TestCase3(TestTDMSamplerOp): - def config(self): """test all dtype int64""" self.neg_samples_num_list = [0, 0, 0, 0] @@ -199,7 +223,6 @@ def config(self): class TestCase4(TestTDMSamplerOp): - def config(self): """test one neg""" self.neg_samples_num_list = [1, 1, 1, 1] @@ -210,7 +233,6 @@ def config(self): class TestCase5(TestTDMSamplerOp): - def config(self): """test normal neg""" self.neg_samples_num_list = [1, 2, 3, 4] @@ -221,7 +243,6 @@ def config(self): class TestCase6(TestTDMSamplerOp): - def config(self): """test huge batchsize""" self.neg_samples_num_list = [1, 2, 3, 4] @@ -232,7 +253,6 @@ def config(self): class TestCase7(TestTDMSamplerOp): - def config(self): """test full neg""" self.neg_samples_num_list = [1, 3, 6, 11] @@ -243,7 +263,6 @@ def config(self): class TestTDMSamplerShape(unittest.TestCase): - def test_shape(self): x = fluid.layers.data(name='x', shape=[1], dtype='int32', lod_level=1) tdm_tree_travel = create_tdm_travel() @@ -267,23 +286,41 @@ def test_shape(self): leaf_node_num, tree_travel_attr=fluid.ParamAttr( initializer=fluid.initializer.NumpyArrayInitializer( - travel_array)), - tree_layer_attr=fluid.ParamAttr(initializer=fluid.initializer. - NumpyArrayInitializer(layer_array)), + travel_array + ) + ), + tree_layer_attr=fluid.ParamAttr( + initializer=fluid.initializer.NumpyArrayInitializer(layer_array) + ), output_positive=True, output_list=True, seed=0, tree_dtype='int32', - dtype='int32') + dtype='int32', + ) place = fluid.CPUPlace() exe = fluid.Executor(place=place) exe.run(fluid.default_startup_program()) feed = { - 'x': - np.array([[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], - [11], [12]]).astype('int32') + 'x': np.array( + [ + [0], + [1], + [2], + [3], + [4], + [5], + [6], + [7], + [8], + [9], + [10], + [11], + [12], + ] + ).astype('int32') } exe.run(feed=feed) diff --git a/python/paddle/fluid/tests/unittests/test_teacher_student_sigmoid_loss_op.py b/python/paddle/fluid/tests/unittests/test_teacher_student_sigmoid_loss_op.py index 228ecbb06ead3..88ea8f647c4bf 100644 --- a/python/paddle/fluid/tests/unittests/test_teacher_student_sigmoid_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_teacher_student_sigmoid_loss_op.py @@ -23,7 +23,7 @@ class TestTeacherStudentSigmoidLossOp(OpTest): """ - Test teacher_student_sigmoid_loss with discrete one-hot labels. + Test teacher_student_sigmoid_loss with discrete one-hot labels. """ def setUp(self): @@ -31,12 +31,14 @@ def setUp(self): batch_size = 100 num_classes = 1 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - (batch_size, num_classes)).astype("float64")), - 'Label': - np.random.uniform(0, 2, (batch_size, num_classes)).astype("float64") + 'X': logit( + np.random.uniform(0, 1, (batch_size, num_classes)).astype( + "float64" + ) + ), + 'Label': np.random.uniform(0, 2, (batch_size, num_classes)).astype( + "float64" + ), } outs = [] for index, label in enumerate(self.inputs["Label"]): @@ -46,11 +48,22 @@ def setUp(self): elif label < 0.0: outs.append(max(x, 0.0) - x + log(1.0 + exp(-abs(x)))) elif label < 1.0: - outs.append(max(x, 0.0) + log(1.0 + exp(-abs(x))) + \ - max(x, 0.0) - x * label + log(1.0 + exp(-abs(x)))) + outs.append( + max(x, 0.0) + + log(1.0 + exp(-abs(x))) + + max(x, 0.0) + - x * label + + log(1.0 + exp(-abs(x))) + ) else: - outs.append(max(x, 0.0) - x + log(1.0 + exp(-abs(x))) + \ - max(x, 0.0) - x * (label - 1.0) + log(1.0 + exp(-abs(x)))) + outs.append( + max(x, 0.0) + - x + + log(1.0 + exp(-abs(x))) + + max(x, 0.0) + - x * (label - 1.0) + + log(1.0 + exp(-abs(x))) + ) self.outputs = {'Y': np.array(outs)} def test_check_output(self): @@ -61,9 +74,7 @@ def test_check_grad(self): class TestTeacherStudentSigmoidLossInvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_input(): input = [512, 1] label = fluid.data(name='label', shape=[None, 1], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/test_temporal_shift_op.py b/python/paddle/fluid/tests/unittests/test_temporal_shift_op.py index ea6d9abc37284..13dc115f6bc4d 100644 --- a/python/paddle/fluid/tests/unittests/test_temporal_shift_op.py +++ b/python/paddle/fluid/tests/unittests/test_temporal_shift_op.py @@ -25,13 +25,14 @@ def temporal_shift(x, seg_num, shift_ratio, data_format): x = np.transpose(x, (0, 3, 1, 2)) shape = x.shape reshape_x = x.reshape((-1, seg_num, shape[1], shape[2], shape[3])) - pad_x = np.pad(reshape_x, ((0, 0), (1, 1), (0, 0), (0, 0), (0, 0)), - 'constant') + pad_x = np.pad( + reshape_x, ((0, 0), (1, 1), (0, 0), (0, 0), (0, 0)), 'constant' + ) c1 = int(shape[1] * shift_ratio) c2 = int(shape[1] * 2 * shift_ratio) slice1 = pad_x[:, :seg_num, :c1, :, :] - slice2 = pad_x[:, 2:seg_num + 2, c1:c2, :, :] - slice3 = pad_x[:, 1:seg_num + 1, c2:, :, :] + slice2 = pad_x[:, 2 : seg_num + 2, c1:c2, :, :] + slice3 = pad_x[:, 1 : seg_num + 1, c2:, :, :] concat_x = np.concatenate([slice1, slice2, slice3], axis=2) out = concat_x.reshape(shape) if data_format == "NHWC": @@ -40,7 +41,6 @@ def temporal_shift(x, seg_num, shift_ratio, data_format): class TestTemporalShift(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'temporal_shift' @@ -50,15 +50,16 @@ def setUp(self): self.attrs = { "seg_num": self.seg_num, "shift_ratio": self.shift_ratio, - "data_format": self.data_format + "data_format": self.data_format, } self.inputs = { "X": x, } - output = temporal_shift(x, self.seg_num, self.shift_ratio, - self.data_format) + output = temporal_shift( + x, self.seg_num, self.shift_ratio, self.data_format + ) self.outputs = {"Out": output} self.python_out_sig = ["Out"] @@ -77,7 +78,6 @@ def initTestCase(self): class TestTemporalShift2(TestTemporalShift): - def initTestCase(self): self.x_shape = (4, 9, 7, 7) self.seg_num = 2 @@ -86,7 +86,6 @@ def initTestCase(self): class TestTemporalShift3(TestTemporalShift): - def initTestCase(self): self.x_shape = (3, 10, 5, 5) self.seg_num = 1 @@ -95,7 +94,6 @@ def initTestCase(self): class TestTemporalShift4(TestTemporalShift): - def initTestCase(self): self.x_shape = (6, 5, 5, 4) self.seg_num = 3 @@ -103,10 +101,10 @@ def initTestCase(self): self.data_format = 'NHWC' -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestTemporalShiftFP16(TestTemporalShift): - def initTestCase(self): self.x_shape = (3, 10, 5, 5) self.seg_num = 1 @@ -126,32 +124,29 @@ def test_check_grad_ignore_uv(self): class TestTemporalShiftAPI(unittest.TestCase): - def test_api(self): input = paddle.randn([6, 4, 2, 2]) - out = paddle.fluid.layers.temporal_shift(x=input, - seg_num=2, - shift_ratio=0.2) + out = paddle.fluid.layers.temporal_shift( + x=input, seg_num=2, shift_ratio=0.2 + ) - out_from_function = paddle.nn.functional.temporal_shift(x=input, - seg_num=2, - shift_ratio=0.2) + out_from_function = paddle.nn.functional.temporal_shift( + x=input, seg_num=2, shift_ratio=0.2 + ) # dygraph with paddle.fluid.dygraph.guard(): input = paddle.randn([6, 4, 2, 2]) - out = paddle.nn.functional.temporal_shift(x=input, - seg_num=2, - shift_ratio=0.2) + out = paddle.nn.functional.temporal_shift( + x=input, seg_num=2, shift_ratio=0.2 + ) def test_error(self): - def attr_data_format(): input = paddle.randn([6, 4, 2, 2]) - out = paddle.nn.functional.temporal_shift(x=input, - seg_num=2, - shift_ratio=0.2, - data_format="HWC") + out = paddle.nn.functional.temporal_shift( + x=input, seg_num=2, shift_ratio=0.2, data_format="HWC" + ) self.assertRaises(ValueError, attr_data_format) diff --git a/python/paddle/fluid/tests/unittests/test_tensor.py b/python/paddle/fluid/tests/unittests/test_tensor.py index 2226805b1253f..ef8e60df9d160 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_tensor.py @@ -20,7 +20,6 @@ class TestTensorPtr(unittest.TestCase): - def test_tensor_ptr(self): t = core.Tensor() np_arr = np.zeros([2, 3]) @@ -29,11 +28,17 @@ def test_tensor_ptr(self): class TestTensor(unittest.TestCase): - def setUp(self): self.support_dtypes = [ - 'bool', 'uint8', 'int8', 'int16', 'int32', 'int64', 'float16', - 'float32', 'float64' + 'bool', + 'uint8', + 'int8', + 'int16', + 'int32', + 'int64', + 'float16', + 'float32', + 'float64', ] def test_int_tensor(self): @@ -79,10 +84,9 @@ def test_int8_tensor(self): scope = core.Scope() var = scope.var("int8_tensor") cpu_tensor = var.get_tensor() - tensor_array = np.random.randint(-127, - high=128, - size=[100, 200], - dtype=np.int8) + tensor_array = np.random.randint( + -127, high=128, size=[100, 200], dtype=np.int8 + ) place = core.CPUPlace() cpu_tensor.set(tensor_array, place) cpu_tensor_array_2 = np.array(cpu_tensor) @@ -90,15 +94,15 @@ def test_int8_tensor(self): if core.is_compiled_with_cuda(): cuda_tensor = var.get_tensor() - tensor_array = np.random.randint(-127, - high=128, - size=[100, 200], - dtype=np.int8) + tensor_array = np.random.randint( + -127, high=128, size=[100, 200], dtype=np.int8 + ) place = core.CUDAPlace(0) cuda_tensor.set(tensor_array, place) cuda_tensor_array_2 = np.array(cuda_tensor) - self.assertAlmostEqual(cuda_tensor_array_2.all(), - tensor_array.all()) + self.assertAlmostEqual( + cuda_tensor_array_2.all(), tensor_array.all() + ) def test_int_lod_tensor(self): place = core.CPUPlace() @@ -207,10 +211,13 @@ def run_slice_tensor(self, place, dtype): shape = [3, 3, 3] tensor._set_dims(shape) - tensor_array = np.array([[[1, 2, 3], [4, 5, 6], [7, 8, 9]], - [[10, 11, 12], [13, 14, 15], [16, 17, 18]], - [[19, 20, 21], [22, 23, 24], - [25, 26, 27]]]).astype(dtype) + tensor_array = np.array( + [ + [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + [[10, 11, 12], [13, 14, 15], [16, 17, 18]], + [[19, 20, 21], [22, 23, 24], [25, 26, 27]], + ] + ).astype(dtype) tensor.set(tensor_array, place) n1 = tensor[1] @@ -283,21 +290,24 @@ def test_tensor_poiter(self): tensor = var.get_tensor() dtype = core.VarDesc.VarType.FP32 self.assertTrue( - isinstance(tensor._mutable_data(place, dtype), numbers.Integral)) + isinstance(tensor._mutable_data(place, dtype), numbers.Integral) + ) if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) self.assertTrue( - isinstance(tensor._mutable_data(place, dtype), - numbers.Integral)) + isinstance(tensor._mutable_data(place, dtype), numbers.Integral) + ) place = core.CUDAPinnedPlace() self.assertTrue( - isinstance(tensor._mutable_data(place, dtype), - numbers.Integral)) + isinstance(tensor._mutable_data(place, dtype), numbers.Integral) + ) places = fluid.cuda_pinned_places() self.assertTrue( - isinstance(tensor._mutable_data(places[0], dtype), - numbers.Integral)) + isinstance( + tensor._mutable_data(places[0], dtype), numbers.Integral + ) + ) def test_tensor_set_fp16(self): array = np.random.random((300, 500)).astype("float16") diff --git a/python/paddle/fluid/tests/unittests/test_tensor_array_to_tensor.py b/python/paddle/fluid/tests/unittests/test_tensor_array_to_tensor.py index 0291052a7e08a..21f90556f7a5b 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_array_to_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_array_to_tensor.py @@ -50,8 +50,10 @@ def test_get_set(self): program = fluid.Program() block = program.global_block() - input_arr = block.create_var(name="tmp_lod_tensor_array", - type=core.VarDesc.VarType.LOD_TENSOR_ARRAY) + input_arr = block.create_var( + name="tmp_lod_tensor_array", + type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, + ) input_arr.persistable = True input_arr_var = scope.var('tmp_lod_tensor_array') input_tensor_array = input_arr_var.get_lod_tensor_array() @@ -75,28 +77,30 @@ def test_get_set(self): y_out_index = block.create_var(name="OutIndex") y_out_index.persistable = True - y_grad_arr = block.create_var(name='Out@GRAD', - dtype='float32', - shape=[11]) + y_grad_arr = block.create_var( + name='Out@GRAD', dtype='float32', shape=[11] + ) y_grad_arr.persistable = True y_grad = scope.var('Out@GRAD') y_grad_tensor = y_grad.get_tensor() y_grad_tensor.set(random_grad, cpu) - op = block.append_op(type=self.op_type, - inputs={"X": input_arr}, - outputs={ - "Out": y_out, - "OutIndex": y_out_index - }, - attrs=self.attrs) - - out_grad = block.create_var(name="tmp_lod_tensor_array@GRAD", - type=core.VarDesc.VarType.LOD_TENSOR_ARRAY) + op = block.append_op( + type=self.op_type, + inputs={"X": input_arr}, + outputs={"Out": y_out, "OutIndex": y_out_index}, + attrs=self.attrs, + ) + + out_grad = block.create_var( + name="tmp_lod_tensor_array@GRAD", + type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, + ) out_grad.persistable = True grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - op.desc, set(), []) + op.desc, set(), [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -115,13 +119,14 @@ def test_get_set(self): exe = fluid.Executor(fluid.CPUPlace()) out = exe.run(program, fetch_list=fetch_list, scope=scope) - #print ("index: ", np.array(out[1])) + # print ("index: ", np.array(out[1])) # test forward tensor_res = np.array(out[0]) tensor_res_out_idx = np.array(out[1]) - tensor_gt = np.array([0] + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - dtype='float32') + tensor_gt = np.array( + [0] + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype='float32' + ) self.assertEqual(len(tensor_res), len(tensor_gt)) self.assertEqual(len(tensor_res_out_idx), 10) @@ -144,13 +149,16 @@ def test_get_set(self): for i in range(len(grad_tensor_array)): if i == 0: self.assertEqual( - np.array(grad_tensor_array[i])[0], np.array(random_grad[i])) + np.array(grad_tensor_array[i])[0], np.array(random_grad[i]) + ) self.assertEqual( np.array(grad_tensor_array[i])[1], - np.array(random_grad[i + 1])) + np.array(random_grad[i + 1]), + ) if i == 1: - self.assertEqual(np.array(grad_tensor_array[i]), - np.array(random_grad[i + 1])) + self.assertEqual( + np.array(grad_tensor_array[i]), np.array(random_grad[i + 1]) + ) class TestLoDTensorArrayStack(unittest.TestCase): @@ -162,12 +170,14 @@ def setUp(self): self.inputs = [ np.random.rand(2, 3, 4).astype("float32"), np.random.rand(2, 3, 4).astype("float32"), - np.random.rand(2, 3, 4).astype("float32") + np.random.rand(2, 3, 4).astype("float32"), ] self.outputs = [ np.stack(self.inputs, axis=self.attrs["axis"]), - np.array([x.shape[self.attrs["axis"]] for x in self.inputs], - dtype="int32") + np.array( + [x.shape[self.attrs["axis"]] for x in self.inputs], + dtype="int32", + ), ] self.input_grads = [np.ones_like(x) for x in self.inputs] self.set_program() @@ -184,7 +194,8 @@ def set_program(self): x = fluid.layers.assign(x) fluid.layers.array_write(x, idx + i, array) output, output_index = fluid.layers.tensor_array_to_tensor( - input=array, **self.attrs) + input=array, **self.attrs + ) loss = fluid.layers.reduce_sum(output) fluid.backward.append_backward(loss) self.output_vars = [output, output_index] @@ -192,10 +203,11 @@ def set_program(self): def run_check(self, executor, scope): executor.run(self.program, scope=scope) for i, output in enumerate(self.outputs): - np.allclose(np.array( - scope.var(self.output_vars[i].name).get_tensor()), - output, - atol=0) + np.allclose( + np.array(scope.var(self.output_vars[i].name).get_tensor()), + output, + atol=0, + ) tensor_array_grad = scope.var(self.array.name).get_lod_tensor_array() for i, input_grad in enumerate(self.input_grads): np.allclose(np.array(tensor_array_grad[i]), input_grad, atol=0) @@ -215,7 +227,6 @@ def test_gpu(self): class TestTensorArrayToTensorAPI(unittest.TestCase): - def _test_case(self, inp1, inp2): x0 = fluid.layers.assign(inp1) x0.stop_gradient = False @@ -226,10 +237,20 @@ def _test_case(self, inp1, inp2): fluid.layers.array_write(x0, i, array) fluid.layers.array_write(x1, i + 1, array) output_stack, output_index_stack = fluid.layers.tensor_array_to_tensor( - input=array, axis=1, use_stack=True) - output_concat, output_index_concat = fluid.layers.tensor_array_to_tensor( - input=array, axis=1, use_stack=False) - return output_stack, output_index_stack, output_concat, output_index_concat + input=array, axis=1, use_stack=True + ) + ( + output_concat, + output_index_concat, + ) = fluid.layers.tensor_array_to_tensor( + input=array, axis=1, use_stack=False + ) + return ( + output_stack, + output_index_stack, + output_concat, + output_index_concat, + ) def test_case(self): inp0 = np.random.rand(2, 3, 4).astype("float32") @@ -269,7 +290,8 @@ def body(i, end, array): self.assertTrue(fluid.layers.array_length(array), 10) last = fluid.layers.fill_constant(shape=[1], dtype='int64', value=9) np.testing.assert_array_equal( - fluid.layers.array_read(array, last).numpy(), inp0) + fluid.layers.array_read(array, last).numpy(), inp0 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_tensor_copy_from.py b/python/paddle/fluid/tests/unittests/test_tensor_copy_from.py index 6c38d2349905b..3f2b00e8e7337 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_copy_from.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_copy_from.py @@ -19,7 +19,6 @@ class TestTensorCopyFrom(unittest.TestCase): - def test_main(self): place = paddle.CPUPlace() np_value = np.random.random(size=[10, 30]).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_tensor_fill_.py b/python/paddle/fluid/tests/unittests/test_tensor_fill_.py index e52c761fd4ef5..7998cbcc19e48 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_fill_.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_fill_.py @@ -20,7 +20,6 @@ class TensorFill_Test(unittest.TestCase): - def setUp(self): self.shape = [32, 32] @@ -36,15 +35,16 @@ def func_test_tensor_fill_true(self): paddle.set_device('cpu') else: paddle.set_device('gpu') - np_arr = np.reshape(np.array(range(np.prod(self.shape))), - self.shape) + np_arr = np.reshape( + np.array(range(np.prod(self.shape))), self.shape + ) for dtype in typelist: - var = 1. + var = 1.0 tensor = paddle.to_tensor(np_arr, place=p, dtype=dtype) target = tensor.numpy() target[...] = var - tensor.fill_(var) #var type is basic type in typelist + tensor.fill_(var) # var type is basic type in typelist self.assertEqual((tensor.numpy() == target).all(), True) def test_tensor_fill_true(self): @@ -64,8 +64,9 @@ def func_test_tensor_fill_backward(self): paddle.set_device('cpu') else: paddle.set_device('gpu') - np_arr = np.reshape(np.array(range(np.prod(self.shape))), - self.shape) + np_arr = np.reshape( + np.array(range(np.prod(self.shape))), self.shape + ) for dtype in typelist: var = int(1) tensor = paddle.to_tensor(np_arr, place=p, dtype=dtype) @@ -85,7 +86,6 @@ def test_tensor_fill_backward(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_test_errors(self): - def test_list(): x = paddle.to_tensor([2, 3, 4]) x.fill_([1]) diff --git a/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_.py b/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_.py index 3eaf01642a6e5..b3c68f9320439 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_.py @@ -20,12 +20,13 @@ class TensorFillDiagonal_Test(unittest.TestCase): - def func_dim2_normal(self): - expected_np = np.array([[1, 2, 2], [2, 1, 2], [2, 2, - 1]]).astype('float32') - expected_grad = np.array([[0, 1, 1], [1, 0, 1], [1, 1, - 0]]).astype('float32') + expected_np = np.array([[1, 2, 2], [2, 1, 2], [2, 2, 1]]).astype( + 'float32' + ) + expected_grad = np.array([[0, 1, 1], [1, 0, 1], [1, 1, 0]]).astype( + 'float32' + ) typelist = ['float32', 'float64', 'int32', 'int64'] places = [fluid.CPUPlace()] @@ -46,10 +47,12 @@ def func_dim2_normal(self): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_dim2_normal(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -59,10 +62,12 @@ def test_dim2_normal(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_offset(self): - expected_np = np.array([[2, 2, 1], [2, 2, 2], [2, 2, - 2]]).astype('float32') - expected_grad = np.array([[1, 1, 0], [1, 1, 1], [1, 1, - 1]]).astype('float32') + expected_np = np.array([[2, 2, 1], [2, 2, 2], [2, 2, 2]]).astype( + 'float32' + ) + expected_grad = np.array([[1, 1, 0], [1, 1, 1], [1, 1, 1]]).astype( + 'float32' + ) typelist = ['float32', 'float64', 'int32', 'int64'] places = [fluid.CPUPlace()] @@ -83,10 +88,12 @@ def func_offset(self): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_offset(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -96,8 +103,9 @@ def test_offset(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_bool(self): - expected_np = np.array([[False, True, True], [True, False, True], - [True, True, False]]) + expected_np = np.array( + [[False, True, True], [True, False, True], [True, True, False]] + ) typelist = ['bool'] places = [fluid.CPUPlace()] @@ -122,12 +130,28 @@ def test_bool(self): self.func_bool() def func_dim2_unnormal_wrap(self): - expected_np = np.array([[1, 2, 2], [2, 1, 2], [2, 2, 1], [2, 2, 2], - [1, 2, 2], [2, 1, 2], [2, 2, - 1]]).astype('float32') - expected_grad = np.array([[0, 1, 1], [1, 0, 1], [1, 1, 0], [1, 1, 1], - [0, 1, 1], [1, 0, 1], [1, 1, - 0]]).astype('float32') + expected_np = np.array( + [ + [1, 2, 2], + [2, 1, 2], + [2, 2, 1], + [2, 2, 2], + [1, 2, 2], + [2, 1, 2], + [2, 2, 1], + ] + ).astype('float32') + expected_grad = np.array( + [ + [0, 1, 1], + [1, 0, 1], + [1, 1, 0], + [1, 1, 1], + [0, 1, 1], + [1, 0, 1], + [1, 1, 0], + ] + ).astype('float32') typelist = ['float32', 'float64', 'int32', 'int64'] places = [fluid.CPUPlace()] @@ -148,10 +172,12 @@ def func_dim2_unnormal_wrap(self): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_dim2_unnormal_wrap(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -161,12 +187,28 @@ def test_dim2_unnormal_wrap(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_dim2_unnormal_unwrap(self): - expected_np = np.array([[1, 2, 2], [2, 1, 2], [2, 2, 1], [2, 2, 2], - [2, 2, 2], [2, 2, 2], [2, 2, - 2]]).astype('float32') - expected_grad = np.array([[0, 1, 1], [1, 0, 1], [1, 1, 0], [1, 1, 1], - [1, 1, 1], [1, 1, 1], [1, 1, - 1]]).astype('float32') + expected_np = np.array( + [ + [1, 2, 2], + [2, 1, 2], + [2, 2, 1], + [2, 2, 2], + [2, 2, 2], + [2, 2, 2], + [2, 2, 2], + ] + ).astype('float32') + expected_grad = np.array( + [ + [0, 1, 1], + [1, 0, 1], + [1, 1, 0], + [1, 1, 1], + [1, 1, 1], + [1, 1, 1], + [1, 1, 1], + ] + ).astype('float32') typelist = ['float32', 'float64', 'int32', 'int64'] places = [fluid.CPUPlace()] @@ -187,10 +229,12 @@ def func_dim2_unnormal_unwrap(self): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_dim2_unnormal_unwrap(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -200,14 +244,20 @@ def test_dim2_unnormal_unwrap(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_dim_larger2_normal(self): - expected_np = np.array([[[1, 2, 2], [2, 2, 2], [2, 2, 2]], - [[2, 2, 2], [2, 1, 2], [2, 2, 2]], - [[2, 2, 2], [2, 2, 2], [2, 2, - 1]]]).astype('float32') - expected_grad = np.array([[[0, 1, 1], [1, 1, 1], [1, 1, 1]], - [[1, 1, 1], [1, 0, 1], [1, 1, 1]], - [[1, 1, 1], [1, 1, 1], - [1, 1, 0]]]).astype('float32') + expected_np = np.array( + [ + [[1, 2, 2], [2, 2, 2], [2, 2, 2]], + [[2, 2, 2], [2, 1, 2], [2, 2, 2]], + [[2, 2, 2], [2, 2, 2], [2, 2, 1]], + ] + ).astype('float32') + expected_grad = np.array( + [ + [[0, 1, 1], [1, 1, 1], [1, 1, 1]], + [[1, 1, 1], [1, 0, 1], [1, 1, 1]], + [[1, 1, 1], [1, 1, 1], [1, 1, 0]], + ] + ).astype('float32') typelist = ['float32', 'float64', 'int32', 'int64'] places = [fluid.CPUPlace()] @@ -228,10 +278,12 @@ def func_dim_larger2_normal(self): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_dim_larger2_normal(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) diff --git a/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor.py b/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor.py index 3aa2ac9612395..8dd9b327f784d 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor.py @@ -21,7 +21,6 @@ class TensorFillDiagTensor_Test(unittest.TestCase): - def setUp(self): self.typelist = ['float32', 'float64', 'int32', 'int64'] self.places = [fluid.CPUPlace()] @@ -30,10 +29,12 @@ def setUp(self): def test_dim2(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - expected_np = np.array([[1, 2, 2], [2, 1, 2], [2, 2, 1], - [2, 2, 2]]).astype('float32') - expected_grad = np.array([[0, 1, 1], [1, 0, 1], [1, 1, 0], - [1, 1, 1]]).astype('float32') + expected_np = np.array( + [[1, 2, 2], [2, 1, 2], [2, 2, 1], [2, 2, 2]] + ).astype('float32') + expected_grad = np.array( + [[0, 1, 1], [1, 0, 1], [1, 1, 0], [1, 1, 1]] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -41,8 +42,8 @@ def test_dim2(self): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.ones((3, ), dtype=dtype) - var = (np.random.random() + 1) + v = paddle.ones((3,), dtype=dtype) + var = np.random.random() + 1 x = paddle.ones((4, 3), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -51,18 +52,22 @@ def test_dim2(self): loss.backward() self.assertEqual( - (ny.numpy().astype('float32') == expected_np).all(), True) + (ny.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_dim2_offset_1(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - expected_np = np.array([[2, 2, 2], [1, 2, 2], [2, 1, 2], - [2, 2, 1]]).astype('float32') - expected_grad = np.array([[1, 1, 1], [0, 1, 1], [1, 0, 1], - [1, 1, 0]]).astype('float32') + expected_np = np.array( + [[2, 2, 2], [1, 2, 2], [2, 1, 2], [2, 2, 1]] + ).astype('float32') + expected_grad = np.array( + [[1, 1, 1], [0, 1, 1], [1, 0, 1], [1, 1, 0]] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -70,8 +75,8 @@ def test_dim2_offset_1(self): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.ones((3, ), dtype=dtype) - var = (np.random.random() + 1) + v = paddle.ones((3,), dtype=dtype) + var = np.random.random() + 1 x = paddle.ones((4, 3), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -80,18 +85,22 @@ def test_dim2_offset_1(self): loss.backward() self.assertEqual( - (ny.numpy().astype('float32') == expected_np).all(), True) + (ny.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_dim2_offset1(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - expected_np = np.array([[2, 1, 2], [2, 2, 1], [2, 2, 2], - [2, 2, 2]]).astype('float32') - expected_grad = np.array([[1, 0, 1], [1, 1, 0], [1, 1, 1], - [1, 1, 1]]).astype('float32') + expected_np = np.array( + [[2, 1, 2], [2, 2, 1], [2, 2, 2], [2, 2, 2]] + ).astype('float32') + expected_grad = np.array( + [[1, 0, 1], [1, 1, 0], [1, 1, 1], [1, 1, 1]] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -99,8 +108,8 @@ def test_dim2_offset1(self): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.ones((2, ), dtype=dtype) - var = (np.random.random() + 1) + v = paddle.ones((2,), dtype=dtype) + var = np.random.random() + 1 x = paddle.ones((4, 3), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -109,30 +118,48 @@ def test_dim2_offset1(self): loss.backward() self.assertEqual( - (ny.numpy().astype('float32') == expected_np).all(), True) + (ny.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_dim4(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - expected_np = np.array([[[[0, 3], [2, 2], [2, 2]], - [[2, 2], [1, 4], [2, 2]], - [[2, 2], [2, 2], [2, 5]], - [[2, 2], [2, 2], [2, 2]]], - [[[6, 9], [2, 2], [2, 2]], - [[2, 2], [7, 10], [2, 2]], - [[2, 2], [2, 2], [8, 11]], - [[2, 2], [2, 2], [2, 2]]]]).astype('float32') - expected_grad = np.array([[[[0, 0], [1, 1], [1, 1]], - [[1, 1], [0, 0], [1, 1]], - [[1, 1], [1, 1], [0, 0]], - [[1, 1], [1, 1], [1, 1]]], - [[[0, 0], [1, 1], [1, 1]], - [[1, 1], [0, 0], [1, 1]], - [[1, 1], [1, 1], [0, 0]], - [[1, 1], [1, 1], [1, 1]]]]).astype('float32') + expected_np = np.array( + [ + [ + [[0, 3], [2, 2], [2, 2]], + [[2, 2], [1, 4], [2, 2]], + [[2, 2], [2, 2], [2, 5]], + [[2, 2], [2, 2], [2, 2]], + ], + [ + [[6, 9], [2, 2], [2, 2]], + [[2, 2], [7, 10], [2, 2]], + [[2, 2], [2, 2], [8, 11]], + [[2, 2], [2, 2], [2, 2]], + ], + ] + ).astype('float32') + expected_grad = np.array( + [ + [ + [[0, 0], [1, 1], [1, 1]], + [[1, 1], [0, 0], [1, 1]], + [[1, 1], [1, 1], [0, 0]], + [[1, 1], [1, 1], [1, 1]], + ], + [ + [[0, 0], [1, 1], [1, 1]], + [[1, 1], [0, 0], [1, 1]], + [[1, 1], [1, 1], [0, 0]], + [[1, 1], [1, 1], [1, 1]], + ], + ] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -140,9 +167,10 @@ def test_dim4(self): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.to_tensor(np.arange(12).reshape(2, 2, 3), - dtype=dtype) - var = (np.random.random() + 1) + v = paddle.to_tensor( + np.arange(12).reshape(2, 2, 3), dtype=dtype + ) + var = np.random.random() + 1 x = paddle.ones((2, 4, 3, 2), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -151,10 +179,12 @@ def test_dim4(self): loss.backward() self.assertEqual( - (ny.numpy().astype('float32') == expected_np).all(), True) + (ny.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def test_largedim(self): @@ -165,7 +195,8 @@ def test_largedim(self): paddle.set_device('gpu') for dtype in self.typelist: v = paddle.arange(bsdim * fsdim, dtype=dtype).reshape( - (bsdim, fsdim)) + (bsdim, fsdim) + ) y = paddle.ones((bsdim, fsdim, fsdim), dtype=dtype) y.stop_gradient = False y = y * 2 diff --git a/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor_.py b/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor_.py index eb223dc6317e0..17be54720c97a 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor_.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_fill_diagonal_tensor_.py @@ -21,7 +21,6 @@ class TensorFillDiagTensor_Test(unittest.TestCase): - def setUp(self): self.typelist = ['float32', 'float64', 'int32', 'int64'] self.places = [fluid.CPUPlace()] @@ -29,10 +28,12 @@ def setUp(self): self.places.append(fluid.CUDAPlace(0)) def func_dim2(self): - expected_np = np.array([[1, 2, 2], [2, 1, 2], [2, 2, 1], - [2, 2, 2]]).astype('float32') - expected_grad = np.array([[0, 1, 1], [1, 0, 1], [1, 1, 0], - [1, 1, 1]]).astype('float32') + expected_np = np.array( + [[1, 2, 2], [2, 1, 2], [2, 2, 1], [2, 2, 2]] + ).astype('float32') + expected_grad = np.array( + [[0, 1, 1], [1, 0, 1], [1, 1, 0], [1, 1, 1]] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -40,8 +41,8 @@ def func_dim2(self): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.ones((3, ), dtype=dtype) - var = (np.random.random() + 1) + v = paddle.ones((3,), dtype=dtype) + var = np.random.random() + 1 x = paddle.ones((4, 3), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -50,10 +51,12 @@ def func_dim2(self): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_dim2(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -63,10 +66,12 @@ def test_dim2(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_dim2_offset_1(self): - expected_np = np.array([[2, 2, 2], [1, 2, 2], [2, 1, 2], - [2, 2, 1]]).astype('float32') - expected_grad = np.array([[1, 1, 1], [0, 1, 1], [1, 0, 1], - [1, 1, 0]]).astype('float32') + expected_np = np.array( + [[2, 2, 2], [1, 2, 2], [2, 1, 2], [2, 2, 1]] + ).astype('float32') + expected_grad = np.array( + [[1, 1, 1], [0, 1, 1], [1, 0, 1], [1, 1, 0]] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -74,8 +79,8 @@ def func_dim2_offset_1(self): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.ones((3, ), dtype=dtype) - var = (np.random.random() + 1) + v = paddle.ones((3,), dtype=dtype) + var = np.random.random() + 1 x = paddle.ones((4, 3), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -84,10 +89,12 @@ def func_dim2_offset_1(self): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_dim2_offset_1(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -97,10 +104,12 @@ def test_dim2_offset_1(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_dim2_offset1(self): - expected_np = np.array([[2, 1, 2], [2, 2, 1], [2, 2, 2], - [2, 2, 2]]).astype('float32') - expected_grad = np.array([[1, 0, 1], [1, 1, 0], [1, 1, 1], - [1, 1, 1]]).astype('float32') + expected_np = np.array( + [[2, 1, 2], [2, 2, 1], [2, 2, 2], [2, 2, 2]] + ).astype('float32') + expected_grad = np.array( + [[1, 0, 1], [1, 1, 0], [1, 1, 1], [1, 1, 1]] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -108,8 +117,8 @@ def func_dim2_offset1(self): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.ones((2, ), dtype=dtype) - var = (np.random.random() + 1) + v = paddle.ones((2,), dtype=dtype) + var = np.random.random() + 1 x = paddle.ones((4, 3), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -118,10 +127,12 @@ def func_dim2_offset1(self): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_dim2_offset1(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -131,22 +142,38 @@ def test_dim2_offset1(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_dim4(self): - expected_np = np.array([[[[0, 3], [2, 2], [2, 2]], - [[2, 2], [1, 4], [2, 2]], - [[2, 2], [2, 2], [2, 5]], - [[2, 2], [2, 2], [2, 2]]], - [[[6, 9], [2, 2], [2, 2]], - [[2, 2], [7, 10], [2, 2]], - [[2, 2], [2, 2], [8, 11]], - [[2, 2], [2, 2], [2, 2]]]]).astype('float32') - expected_grad = np.array([[[[0, 0], [1, 1], [1, 1]], - [[1, 1], [0, 0], [1, 1]], - [[1, 1], [1, 1], [0, 0]], - [[1, 1], [1, 1], [1, 1]]], - [[[0, 0], [1, 1], [1, 1]], - [[1, 1], [0, 0], [1, 1]], - [[1, 1], [1, 1], [0, 0]], - [[1, 1], [1, 1], [1, 1]]]]).astype('float32') + expected_np = np.array( + [ + [ + [[0, 3], [2, 2], [2, 2]], + [[2, 2], [1, 4], [2, 2]], + [[2, 2], [2, 2], [2, 5]], + [[2, 2], [2, 2], [2, 2]], + ], + [ + [[6, 9], [2, 2], [2, 2]], + [[2, 2], [7, 10], [2, 2]], + [[2, 2], [2, 2], [8, 11]], + [[2, 2], [2, 2], [2, 2]], + ], + ] + ).astype('float32') + expected_grad = np.array( + [ + [ + [[0, 0], [1, 1], [1, 1]], + [[1, 1], [0, 0], [1, 1]], + [[1, 1], [1, 1], [0, 0]], + [[1, 1], [1, 1], [1, 1]], + ], + [ + [[0, 0], [1, 1], [1, 1]], + [[1, 1], [0, 0], [1, 1]], + [[1, 1], [1, 1], [0, 0]], + [[1, 1], [1, 1], [1, 1]], + ], + ] + ).astype('float32') for idx, p in enumerate(self.places): if idx == 0: @@ -154,9 +181,10 @@ def func_dim4(self): else: paddle.set_device('gpu') for dtype in self.typelist: - v = paddle.to_tensor(np.arange(12).reshape(2, 2, 3), - dtype=dtype) - var = (np.random.random() + 1) + v = paddle.to_tensor( + np.arange(12).reshape(2, 2, 3), dtype=dtype + ) + var = np.random.random() + 1 x = paddle.ones((2, 4, 3, 2), dtype=dtype) x.stop_gradient = False y = x * 2 @@ -165,10 +193,12 @@ def func_dim4(self): loss.backward() self.assertEqual( - (y.numpy().astype('float32') == expected_np).all(), True) + (y.numpy().astype('float32') == expected_np).all(), True + ) self.assertEqual( (y.grad.numpy().astype('float32') == expected_grad).all(), - True) + True, + ) def test_func_dim4(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -178,14 +208,15 @@ def test_func_dim4(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_largedim(self): - #large dim only test on gpu because the cpu version is too slow for ci test, and the memory is limited + # large dim only test on gpu because the cpu version is too slow for ci test, and the memory is limited if len(self.places) > 1: bsdim = 1024 fsdim = 128 paddle.set_device('gpu') for dtype in self.typelist: v = paddle.arange(bsdim * fsdim, dtype=dtype).reshape( - (bsdim, fsdim)) + (bsdim, fsdim) + ) y = paddle.ones((bsdim, fsdim, fsdim), dtype=dtype) y.stop_gradient = False y = y * 2 diff --git a/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py b/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py index 0a249d002836d..8e7de65896967 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_register_hook.py @@ -23,7 +23,6 @@ class SimpleNet(nn.Layer): - def __init__(self, in_size, out_size): super(SimpleNet, self).__init__() self.linear1 = nn.Linear(in_size, in_size) @@ -42,7 +41,6 @@ def forward(self, x, hook=None, register=False, remove=False): class SimpleNetForStatic(nn.Layer): - def __init__(self, in_size, out_size): super(SimpleNetForStatic, self).__init__() self.linear1 = nn.Linear(in_size, in_size) @@ -58,7 +56,6 @@ def forward(self, x): class TestTensorRegisterHook(unittest.TestCase): - def setUp(self): self.seed = 2021 self.in_size = 10 @@ -69,13 +66,12 @@ def setUp(self): self.devices.append("gpu") def func_hook_for_interior_var(self): - def run_double_hook_for_interior_var(double_hook, removed=False): for device in self.devices: paddle.set_device(device) - x = paddle.to_tensor([0., 1., 2., 3.]) - y = paddle.to_tensor([4., 5., 6., 7.]) + x = paddle.to_tensor([0.0, 1.0, 2.0, 3.0]) + y = paddle.to_tensor([4.0, 5.0, 6.0, 7.0]) x.stop_gradient = False y.stop_gradient = False @@ -83,7 +79,7 @@ def run_double_hook_for_interior_var(double_hook, removed=False): w.stop_gradient = False helper = w.register_hook(double_hook) - z = paddle.to_tensor([1., 2., 3., 4.]) + z = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) z.stop_gradient = False o = z.matmul(w) @@ -100,18 +96,18 @@ def run_double_hook_for_interior_var(double_hook, removed=False): np.testing.assert_array_equal(w.grad.numpy(), z.numpy()) # x.grad and y.grad are changed if run hook np.testing.assert_array_equal( - x.grad.numpy(), - z.numpy() * 2 if not removed else z.numpy()) + x.grad.numpy(), z.numpy() * 2 if not removed else z.numpy() + ) np.testing.assert_array_equal( - y.grad.numpy(), - z.numpy() * 2 if not removed else z.numpy()) + y.grad.numpy(), z.numpy() * 2 if not removed else z.numpy() + ) def run_print_hook_for_interior_var(print_hook, removed=False): for device in self.devices: paddle.set_device(device) - x = paddle.to_tensor([0., 1., 2., 3.]) - y = paddle.to_tensor([4., 5., 6., 7.]) + x = paddle.to_tensor([0.0, 1.0, 2.0, 3.0]) + y = paddle.to_tensor([4.0, 5.0, 6.0, 7.0]) x.stop_gradient = False y.stop_gradient = False @@ -119,7 +115,7 @@ def run_print_hook_for_interior_var(print_hook, removed=False): w.stop_gradient = False helper = w.register_hook(print_hook) - z = paddle.to_tensor([1., 2., 3., 4.]) + z = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) z.stop_gradient = False o = z.matmul(w) @@ -167,13 +163,12 @@ def test_hook_for_interior_var(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_hook_for_leaf_var(self): - def run_double_hook_for_leaf_var(double_hook, removed=False): for device in self.devices: paddle.set_device(device) - x = paddle.to_tensor([0., 1., 2., 3.]) - y = paddle.to_tensor([4., 5., 6., 7.]) + x = paddle.to_tensor([0.0, 1.0, 2.0, 3.0]) + y = paddle.to_tensor([4.0, 5.0, 6.0, 7.0]) x.stop_gradient = False y.stop_gradient = False helper = y.register_hook(double_hook) @@ -181,7 +176,7 @@ def run_double_hook_for_leaf_var(double_hook, removed=False): w = x + y w.stop_gradient = False - z = paddle.to_tensor([1., 2., 3., 4.]) + z = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) z.stop_gradient = False o = z.matmul(w) @@ -198,8 +193,8 @@ def run_double_hook_for_leaf_var(double_hook, removed=False): np.testing.assert_array_equal(x.grad.numpy(), z.numpy()) # y.grad are changed if run hook np.testing.assert_array_equal( - y.grad.numpy(), - z.numpy() * 2 if not removed else z.numpy()) + y.grad.numpy(), z.numpy() * 2 if not removed else z.numpy() + ) # register hook run_double_hook_for_leaf_var(lambda grad: grad * 2) @@ -214,14 +209,14 @@ def test_hook_for_leaf_var(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_hook_for_accumulated_grad_interior_var(self): - def run_double_hook_for_accumulated_grad_interior_var( - double_hook, removed=False): + double_hook, removed=False + ): for device in self.devices: paddle.set_device(device) - a = paddle.to_tensor([0., 1., 1., 2.]) - b = paddle.to_tensor([0., 0., 1., 2.]) + a = paddle.to_tensor([0.0, 1.0, 1.0, 2.0]) + b = paddle.to_tensor([0.0, 0.0, 1.0, 2.0]) a.stop_gradient = False b.stop_gradient = False @@ -232,8 +227,8 @@ def run_double_hook_for_accumulated_grad_interior_var( helper2 = x.register_hook(double_hook) - y = paddle.to_tensor([4., 5., 6., 7.]) - z = paddle.to_tensor([1., 2., 3., 4.]) + y = paddle.to_tensor([4.0, 5.0, 6.0, 7.0]) + z = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) y.stop_gradient = False z.stop_gradient = False @@ -251,21 +246,24 @@ def run_double_hook_for_accumulated_grad_interior_var( o.backward() - base_grad = np.array([5., 9., 13., 19.]) + base_grad = np.array([5.0, 9.0, 13.0, 19.0]) # x.grad is not changed np.testing.assert_array_equal(x.grad.numpy(), base_grad) # b.grad is changed by x.hook np.testing.assert_array_equal( - b.grad.numpy(), base_grad * 2 if not removed else base_grad) + b.grad.numpy(), base_grad * 2 if not removed else base_grad + ) # a.grad is changed by x.hook and a.hook np.testing.assert_array_equal( - a.grad.numpy(), base_grad * 4 if not removed else base_grad) + a.grad.numpy(), base_grad * 4 if not removed else base_grad + ) # register hook run_double_hook_for_accumulated_grad_interior_var(lambda grad: grad * 2) # register hook and removed - run_double_hook_for_accumulated_grad_interior_var(lambda grad: grad * 2, - removed=True) + run_double_hook_for_accumulated_grad_interior_var( + lambda grad: grad * 2, removed=True + ) def test_hook_for_accumulated_grad_interior_var(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -275,19 +273,19 @@ def test_hook_for_accumulated_grad_interior_var(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_hook_for_accumulated_grad_leaf_var(self): - def run_double_hook_for_accumulated_grad_leaf_var( - double_hook, removed=False): + double_hook, removed=False + ): for device in self.devices: paddle.set_device(device) - x = paddle.to_tensor([0., 1., 2., 4.]) + x = paddle.to_tensor([0.0, 1.0, 2.0, 4.0]) x.stop_gradient = False helper = x.register_hook(double_hook) - y = paddle.to_tensor([4., 5., 6., 7.]) - z = paddle.to_tensor([1., 2., 3., 4.]) + y = paddle.to_tensor([4.0, 5.0, 6.0, 7.0]) + z = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) y.stop_gradient = False z.stop_gradient = False @@ -304,16 +302,18 @@ def run_double_hook_for_accumulated_grad_leaf_var( o.backward() - base_grad = np.array([5., 9., 13., 19.]) + base_grad = np.array([5.0, 9.0, 13.0, 19.0]) # x.grad is changed by x.hook np.testing.assert_array_equal( - x.grad.numpy(), base_grad * 2 if not removed else base_grad) + x.grad.numpy(), base_grad * 2 if not removed else base_grad + ) # register hook run_double_hook_for_accumulated_grad_leaf_var(lambda grad: grad * 2) # register hook and removed - run_double_hook_for_accumulated_grad_leaf_var(lambda grad: grad * 2, - removed=True) + run_double_hook_for_accumulated_grad_leaf_var( + lambda grad: grad * 2, removed=True + ) def test_hook_for_accumulated_grad_leaf_var(self): with _test_eager_guard(): @@ -321,12 +321,9 @@ def test_hook_for_accumulated_grad_leaf_var(self): self.func_hook_for_accumulated_grad_leaf_var() def func_hook_in_model(self): - - def run_double_hook_in_model(data, - label, - hook=None, - register=False, - remove=False): + def run_double_hook_in_model( + data, label, hook=None, register=False, remove=False + ): for device in self.devices: paddle.seed(self.seed) paddle.set_device(device) @@ -341,22 +338,35 @@ def run_double_hook_in_model(data, loss = loss_fn(out, label) loss.backward() - return (ret1.grad.numpy(), net.linear1.weight.grad.numpy(), - net.linear1.bias.grad.numpy()) + return ( + ret1.grad.numpy(), + net.linear1.weight.grad.numpy(), + net.linear1.bias.grad.numpy(), + ) - data = np.random.uniform( - size=[self.batch_size, self.in_size]).astype('float32') + data = np.random.uniform(size=[self.batch_size, self.in_size]).astype( + 'float32' + ) label = np.random.uniform(size=[self.batch_size, 1]).astype('float32') # get original value ret1_grad, linear1_w_grad, linear1_b_grad = run_double_hook_in_model( - data, label) + data, label + ) # get value changed by hook - ret1_grad_hook, linear1_w_grad_hook, linear1_b_grad_hook = run_double_hook_in_model( - data, label, lambda grad: grad * 2, True) + ( + ret1_grad_hook, + linear1_w_grad_hook, + linear1_b_grad_hook, + ) = run_double_hook_in_model(data, label, lambda grad: grad * 2, True) # get value after removing hook - ret1_grad_rm, linear1_w_grad_rm, linear1_b_grad_rm = run_double_hook_in_model( - data, label, lambda grad: grad * 2, True, True) + ( + ret1_grad_rm, + linear1_w_grad_rm, + linear1_b_grad_rm, + ) = run_double_hook_in_model( + data, label, lambda grad: grad * 2, True, True + ) # compare original value and with hook np.testing.assert_array_equal(ret1_grad, ret1_grad_hook) @@ -376,16 +386,13 @@ def test_func_hook_in_model(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_multiple_hooks_for_interior_var(self): - - def run_multiple_hooks_for_interior_var(device, - hooks, - remove1=False, - remove2=False, - remove3=False): + def run_multiple_hooks_for_interior_var( + device, hooks, remove1=False, remove2=False, remove3=False + ): paddle.set_device(device) - x = paddle.to_tensor([0., 1., 2., 3.]) - y = paddle.to_tensor([4., 5., 6., 7.]) + x = paddle.to_tensor([0.0, 1.0, 2.0, 3.0]) + y = paddle.to_tensor([4.0, 5.0, 6.0, 7.0]) x.stop_gradient = False y.stop_gradient = False @@ -397,7 +404,7 @@ def run_multiple_hooks_for_interior_var(device, helper = w.register_hook(hook) helpers.append(helper) - z = paddle.to_tensor([1., 2., 3., 4.]) + z = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) z.stop_gradient = False o = z.matmul(w) @@ -420,35 +427,40 @@ def double_hook(grad): for device in self.devices: z, w_grad, x_grad, y_grad = run_multiple_hooks_for_interior_var( - device, hooks) + device, hooks + ) np.testing.assert_array_equal(w_grad, z) np.testing.assert_array_equal(x_grad, z * 8) np.testing.assert_array_equal(y_grad, z * 8) z, w_grad, x_grad, y_grad = run_multiple_hooks_for_interior_var( - device, hooks, remove1=True) + device, hooks, remove1=True + ) np.testing.assert_array_equal(w_grad, z) np.testing.assert_array_equal(x_grad, z * 4) np.testing.assert_array_equal(y_grad, z * 4) z, w_grad, x_grad, y_grad = run_multiple_hooks_for_interior_var( - device, hooks, remove2=True) + device, hooks, remove2=True + ) np.testing.assert_array_equal(w_grad, z) np.testing.assert_array_equal(x_grad, z * 4) np.testing.assert_array_equal(y_grad, z * 4) z, w_grad, x_grad, y_grad = run_multiple_hooks_for_interior_var( - device, hooks, remove3=True) + device, hooks, remove3=True + ) np.testing.assert_array_equal(w_grad, z) np.testing.assert_array_equal(x_grad, z * 4) np.testing.assert_array_equal(y_grad, z * 4) z, w_grad, x_grad, y_grad = run_multiple_hooks_for_interior_var( - device, hooks, remove1=True, remove2=True, remove3=True) + device, hooks, remove1=True, remove2=True, remove3=True + ) np.testing.assert_array_equal(w_grad, z) np.testing.assert_array_equal(x_grad, z) @@ -462,7 +474,6 @@ def test_multiple_hooks_for_interior_var(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_hook_in_double_grad(self): - def double_print_hook(grad): grad = grad * 2 print(grad) @@ -478,10 +489,9 @@ def double_print_hook(grad): y = x * x # Since y = x * x, dx = 2 * x - dx = paddle.grad(outputs=[y], - inputs=[x], - create_graph=True, - retain_graph=True)[0] + dx = paddle.grad( + outputs=[y], inputs=[x], create_graph=True, retain_graph=True + )[0] z = y + dx self.assertTrue(x.grad is None) @@ -508,7 +518,7 @@ def func_remove_one_hook_multiple_times(self): for device in self.devices: paddle.set_device(device) - x = paddle.to_tensor([1., 2., 3., 4.]) + x = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) x.stop_gradient = False h = x.register_hook(lambda grad: grad * 2) @@ -524,7 +534,7 @@ def func_register_hook_for_stop_gradient_var(self): for device in self.devices: paddle.set_device(device) - x = paddle.to_tensor([1., 2., 3., 4.]) + x = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) with self.assertRaises(RuntimeError): x.register_hook(lambda grad: grad * 2) @@ -541,9 +551,9 @@ def test_register_hook_in_static_mode(self): main_program = paddle.static.Program() with paddle.static.scope_guard(paddle.static.Scope()): with paddle.static.program_guard(main_program, startup_program): - x = paddle.static.data(name='x', - shape=[None, self.in_size], - dtype='float32') + x = paddle.static.data( + name='x', shape=[None, self.in_size], dtype='float32' + ) net = SimpleNetForStatic(self.in_size, self.out_size) with self.assertRaises(AssertionError): @@ -554,10 +564,12 @@ def test_register_hook_in_static_mode(self): def func_register_hook_in_dy2static_mode(self): net = SimpleNetForStatic(self.in_size, self.out_size) jit_net = paddle.jit.to_static( - net, input_spec=[paddle.static.InputSpec([None, self.in_size])]) + net, input_spec=[paddle.static.InputSpec([None, self.in_size])] + ) - data = np.random.uniform( - size=[self.batch_size, self.in_size]).astype('float32') + data = np.random.uniform(size=[self.batch_size, self.in_size]).astype( + 'float32' + ) data_t = paddle.to_tensor(data) with self.assertRaises(AssertionError): @@ -581,7 +593,6 @@ def global_void_hook(): class TestTensorRegisterBackwardHook(unittest.TestCase): - def setUp(self): self.devices = ["cpu"] if paddle.is_compiled_with_cuda(): @@ -591,7 +602,7 @@ def func_register_backward_hook(self): global HOOK_INIT_VALUE global HOOK_IS_CALLED for device in self.devices: - x = paddle.to_tensor(5., stop_gradient=False) + x = paddle.to_tensor(5.0, stop_gradient=False) x._register_backward_hook(global_void_hook) for i in range(5): y = paddle.pow(x, 4.0) @@ -610,7 +621,7 @@ def test_register_backward_hook(self): self.func_register_backward_hook() def func_register_backward_hook_for_interior_var(self): - x = paddle.to_tensor(5., stop_gradient=False) + x = paddle.to_tensor(5.0, stop_gradient=False) y = paddle.pow(x, 4.0) with self.assertRaises(ValueError): @@ -622,7 +633,7 @@ def test_register_backward_hook_for_interior_var(self): self.func_register_backward_hook_for_interior_var() def func_register_backward_hook_for_var_without_gradient(self): - x = paddle.to_tensor(5.) + x = paddle.to_tensor(5.0) y = paddle.pow(x, 4.0) with self.assertRaises(ValueError): @@ -635,7 +646,6 @@ def test_register_backward_hook_for_var_without_gradient(self): class TestRegsiterBackwardFinalHook(unittest.TestCase): - def setUp(self): self.devices = ["cpu"] if paddle.is_compiled_with_cuda(): diff --git a/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py b/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py index 594731fb08f19..0ab01f4155171 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py @@ -28,7 +28,6 @@ class TestTensorScalarTypePromotionDynamic(unittest.TestCase): - def check_operation(self, a, b, c, op): if op == '+': c_rlt = a + b diff --git a/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_static.py b/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_static.py index 127a36d70703a..7949d2457e928 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_static.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_static.py @@ -29,7 +29,6 @@ class TestTensorScalarTypePromotionStatic(unittest.TestCase): - def setUp(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_tensor_to_list.py b/python/paddle/fluid/tests/unittests/test_tensor_to_list.py index 899be8c996cf2..709ff84179f24 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_to_list.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_to_list.py @@ -20,7 +20,6 @@ class TensorToListTest(unittest.TestCase): - def setUp(self): self.shape = [11, 25, 32, 43] @@ -31,8 +30,9 @@ def func_tensor_tolist(self): places.append(fluid.CUDAPinnedPlace()) for p in places: - np_arr = np.reshape(np.array(range(np.prod(self.shape))), - self.shape) + np_arr = np.reshape( + np.array(range(np.prod(self.shape))), self.shape + ) expectlist = np_arr.tolist() t = paddle.to_tensor(np_arr, place=p) diff --git a/python/paddle/fluid/tests/unittests/test_tensor_to_numpy.py b/python/paddle/fluid/tests/unittests/test_tensor_to_numpy.py index 1bd2f104854e2..30843455ca5e2 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_to_numpy.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_to_numpy.py @@ -18,13 +18,18 @@ class TensorToNumpyTest(unittest.TestCase): - def setUp(self): self.shape = [11, 25, 32, 43] def test_main(self): dtypes = [ - 'float32', 'float64', 'int32', 'int64', 'uint8', 'int8', 'bool' + 'float32', + 'float64', + 'int32', + 'int64', + 'uint8', + 'int8', + 'bool', ] places = [fluid.CPUPlace()] @@ -36,7 +41,8 @@ def test_main(self): for dtype in dtypes: np_arr = np.reshape( np.array(range(np.prod(self.shape))).astype(dtype), - self.shape) + self.shape, + ) t = fluid.LoDTensor() t.set(np_arr, p) diff --git a/python/paddle/fluid/tests/unittests/test_tensor_type_promotion.py b/python/paddle/fluid/tests/unittests/test_tensor_type_promotion.py index 4e31fec3543e6..f6a1f7e5def41 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_type_promotion.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_type_promotion.py @@ -19,7 +19,6 @@ class TestTensorTypePromotion(unittest.TestCase): - def setUp(self): self.x = paddle.to_tensor([2, 3]) self.y = paddle.to_tensor([1.0, 2.0]) @@ -29,32 +28,36 @@ def add_operator(self): warnings.simplefilter("always") self.x + self.y self.assertTrue( - "The dtype of left and right variables are not the same" in str( - context[-1].message)) + "The dtype of left and right variables are not the same" + in str(context[-1].message) + ) def sub_operator(self): with warnings.catch_warnings(record=True) as context: warnings.simplefilter("always") self.x - self.y self.assertTrue( - "The dtype of left and right variables are not the same" in str( - context[-1].message)) + "The dtype of left and right variables are not the same" + in str(context[-1].message) + ) def mul_operator(self): with warnings.catch_warnings(record=True) as context: warnings.simplefilter("always") self.x * self.y self.assertTrue( - "The dtype of left and right variables are not the same" in str( - context[-1].message)) + "The dtype of left and right variables are not the same" + in str(context[-1].message) + ) def div_operator(self): with warnings.catch_warnings(record=True) as context: warnings.simplefilter("always") self.x / self.y self.assertTrue( - "The dtype of left and right variables are not the same" in str( - context[-1].message)) + "The dtype of left and right variables are not the same" + in str(context[-1].message) + ) def test_operator(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_tensor_uva.py b/python/paddle/fluid/tests/unittests/test_tensor_uva.py index 1261cf41b79ed..8fe03148f4921 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_uva.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_uva.py @@ -20,7 +20,6 @@ class TestTensorCopyFrom(unittest.TestCase): - def func_main(self): if paddle.fluid.core.is_compiled_with_cuda(): place = paddle.CPUPlace() @@ -36,12 +35,17 @@ def test_main(self): class TestUVATensorFromNumpy(unittest.TestCase): - def func_uva_tensor_creation(self): if paddle.fluid.core.is_compiled_with_cuda(): dtype_list = [ - "int32", "int64", "float32", "float64", "float16", "int8", - "int16", "bool" + "int32", + "int64", + "float32", + "float64", + "float16", + "int8", + "int16", + "bool", ] for dtype in dtype_list: data = np.random.randint(10, size=[4, 5]).astype(dtype) @@ -64,9 +68,9 @@ def test_uva_tensor_corectness(self): slice_a = a[:, 5] tensor1 = paddle.to_tensor(slice_a) tensor2 = core.eager.to_uva_tensor(slice_a) - np.testing.assert_allclose(tensor1.numpy(), - tensor2.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + tensor1.numpy(), tensor2.numpy(), rtol=1e-05 + ) def test_uva_tensor_creation(self): with _test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_tensor_zero_.py b/python/paddle/fluid/tests/unittests/test_tensor_zero_.py index b8d55028ed24c..2870ad8e75a19 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_zero_.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_zero_.py @@ -20,7 +20,6 @@ class TensorFill_Test(unittest.TestCase): - def setUp(self): self.shape = [32, 32] @@ -32,8 +31,9 @@ def func_test_tensor_fill_true(self): places.append(fluid.CUDAPinnedPlace()) for p in places: - np_arr = np.reshape(np.array(range(np.prod(self.shape))), - self.shape) + np_arr = np.reshape( + np.array(range(np.prod(self.shape))), self.shape + ) for dtype in typelist: tensor = paddle.to_tensor(np_arr, place=p, dtype=dtype) target = tensor.numpy() diff --git a/python/paddle/fluid/tests/unittests/test_tensordot.py b/python/paddle/fluid/tests/unittests/test_tensordot.py index c0fb398d830c4..8709772c18e14 100644 --- a/python/paddle/fluid/tests/unittests/test_tensordot.py +++ b/python/paddle/fluid/tests/unittests/test_tensordot.py @@ -26,7 +26,7 @@ def tensordot_np(x, y, axes): # np.tensordot does not support empty axes if not axes: axes = 0 - if (isinstance(axes, (tuple, list))): + if isinstance(axes, (tuple, list)): if all(np.issubdtype(type(i), np.integer) for i in axes): axes = [axes, axes] else: @@ -43,7 +43,7 @@ def tensordot_np(x, y, axes): axes = [axes_x, axes_y] # np.tensordot does not support broadcast - if (isinstance(axes, (tuple, list))): + if isinstance(axes, (tuple, list)): axes_x, axes_y = axes else: axes_x = list(range(x.ndim - axes, x.ndim)) @@ -65,7 +65,6 @@ def tensordot_np(x, y, axes): class TestTensordotAPI(unittest.TestCase): - def setUp(self): self.set_place() self.set_dtype() @@ -90,60 +89,108 @@ def set_input_data(self): self.y = np.random.random(self.y_shape).astype(self.dtype) def set_test_axes(self): - self.all_axes = [[[3, 2], [3]], [[2, 1, 0], [2, 1]], - [[1, 2, 0], [1, 3, 2]], [3, 0], [[], [0, 3, 1]], - [[2, 1, 0, 3], [2, 0, 1, 3]], [[3, 1, 2], [1, 3, 2, - 0]], - [[2, 1], [0, 2]], [[2, 0, 1, 3], [2]], - [[1, 2, 0, 3], [0, 2, 1]], [[2, 1, 3, 0], [1, 2, 3]], - [[2, 0, 1, 3], [3, 1, 0, 2]], [[0, 3], [0, 3, 2, 1]], - [[1, 3, 2, 0], [2, 1, 0, 3]], - [[1, 3, 2, 0], [1, 3, 2, 0]], [[1, 0, 2], [0, 1]], - [[2, 3, 0], [3, 1]], [[1, 3, 2, 0], [3, 0, 1, 2]], - [[3, 2, 1], [2, 0, 1]], [[0], []], - [[2, 3, 0], [1, 2, 0]], [[3, 0, 2, 1], [2, 1, 0, 3]], - [[3, 1, 2], [2, 3, 1]], [[1, 0, 2, 3], []], - [[1, 2], [1, 2, 3]], [[2, 0, 1, 3], [2, 0, 1]], - [[3, 1, 2], [1, 3, 2]], [[3, 1, 2, 0], [1, 2, 3, 0]], - [[0, 2, 3], [0, 1, 2]], [[3, 2, 0], [2, 0, 3, 1]], - [[2, 1, 0, 3], [3, 1, 2, 0]], - [[1, 2, 3, 0], [1, 3, 0, 2]], [[3, 0], [2, 1]], - [[0, 1, 3, 2], [0, 2, 1, 3]], [[1, 0], [2, 1, 3]], - [[1, 0, 3, 2], [2, 3, 0, 1]], [[1, 2], [3]], - [[1, 2, 3, 0], [3, 2, 1, 0]], - [[0, 3, 2, 1], [2, 1, 3, 0]], [0], - [[0, 2, 3], [3, 2, 0, 1]], [[1, 2, 3, 0], [3, 2, 1, - 0]], - [[3, 1], [3]], [[3, 2, 0, 1], [3, 2, 0]], - [[2, 3, 0, 1], [0, 3, 2]], [[1], [1, 3]], - [[1, 2], [2, 1, 0]], [[3, 1, 2], [3, 1, 0]], - [[1, 3], [3, 1, 2]], [[2, 0, 1, 3], [3, 1, 0, 2]], - [[1, 3, 0], [1, 3]], [[2, 3, 1], [1, 0, 2]], - [[1, 2, 0, 3], [0, 2, 1, 3]], [[2], [0, 1, 3]], - [[1], [1, 2]], [[1, 0, 2, 3], [3, 0, 1, 2]], - [[0, 1, 3, 2], [1, 3, 0, 2]], [[3, 0, 2, 1], [0, 2, - 3]], - [[1, 2, 0], [1, 2, 3]], [[1, 0, 3], [2, 3, 0]], - [[2, 3, 0], [3, 1, 0]], [[1, 3], [1, 0]], - [[2, 1, 0, 3], [2, 0, 3, 1]], [[3, 2, 0], [2, 1, 0]], - [[0, 1, 3], [0, 3, 1]], [[3, 1, 0], [3, 2, 1]], - [[3, 2], [3, 1]], [[3], [2, 1, 0]], [[1, 2, 3, 0], []], - [[1, 3, 2, 0], [3, 1, 2]], [[1], [0, 2]], - [[3, 2, 0], [3, 2, 0]], [[3], []], [[1, 0, 3], [2, 1]], - [[3, 1, 0, 2], [2, 3, 1, 0]], [[0, 1], [0, 3, 2]], - [[0, 2, 3], [0, 2, 1]], [[1, 3, 0], [3, 0, 2]], - [[3, 1, 2], [1, 2, 3]], [[3, 1, 2], [3, 1, 0]], - [[0, 3, 1, 2], [3, 2, 1, 0]], [[0, 3], [3, 2, 1]], - [[2, 3], [1, 3, 0]], [[0, 3, 2], [2, 0, 3, 1]], - [[2, 3], [1, 3]], [[3, 1, 2, 0], [2, 3, 1, 0]], - [[1, 0, 3, 2], [3, 0, 1, 2]], - [[3, 2, 1, 0], [0, 1, 3, 2]], [[3, 1, 2], [3]], - [[0, 1, 3, 2], [2, 3, 0, 1]], - [[1, 2, 3, 0], [1, 3, 0, 2]], [3, 1, 2], - [[3, 1, 2], [0, 3, 2]], [[2, 3, 0], [1, 2, 0]], - [[2, 0, 3], [2, 0]], [[3, 1, 0, 2], [3, 1, 0, 2]], - [[0, 1, 2], [2, 0, 1]], [[1, 0, 3], [2, 3, 0]], - [[2, 0, 1], [0, 1, 3]], [[2, 1], [0, 1, 3]]] + self.all_axes = [ + [[3, 2], [3]], + [[2, 1, 0], [2, 1]], + [[1, 2, 0], [1, 3, 2]], + [3, 0], + [[], [0, 3, 1]], + [[2, 1, 0, 3], [2, 0, 1, 3]], + [[3, 1, 2], [1, 3, 2, 0]], + [[2, 1], [0, 2]], + [[2, 0, 1, 3], [2]], + [[1, 2, 0, 3], [0, 2, 1]], + [[2, 1, 3, 0], [1, 2, 3]], + [[2, 0, 1, 3], [3, 1, 0, 2]], + [[0, 3], [0, 3, 2, 1]], + [[1, 3, 2, 0], [2, 1, 0, 3]], + [[1, 3, 2, 0], [1, 3, 2, 0]], + [[1, 0, 2], [0, 1]], + [[2, 3, 0], [3, 1]], + [[1, 3, 2, 0], [3, 0, 1, 2]], + [[3, 2, 1], [2, 0, 1]], + [[0], []], + [[2, 3, 0], [1, 2, 0]], + [[3, 0, 2, 1], [2, 1, 0, 3]], + [[3, 1, 2], [2, 3, 1]], + [[1, 0, 2, 3], []], + [[1, 2], [1, 2, 3]], + [[2, 0, 1, 3], [2, 0, 1]], + [[3, 1, 2], [1, 3, 2]], + [[3, 1, 2, 0], [1, 2, 3, 0]], + [[0, 2, 3], [0, 1, 2]], + [[3, 2, 0], [2, 0, 3, 1]], + [[2, 1, 0, 3], [3, 1, 2, 0]], + [[1, 2, 3, 0], [1, 3, 0, 2]], + [[3, 0], [2, 1]], + [[0, 1, 3, 2], [0, 2, 1, 3]], + [[1, 0], [2, 1, 3]], + [[1, 0, 3, 2], [2, 3, 0, 1]], + [[1, 2], [3]], + [[1, 2, 3, 0], [3, 2, 1, 0]], + [[0, 3, 2, 1], [2, 1, 3, 0]], + [0], + [[0, 2, 3], [3, 2, 0, 1]], + [[1, 2, 3, 0], [3, 2, 1, 0]], + [[3, 1], [3]], + [[3, 2, 0, 1], [3, 2, 0]], + [[2, 3, 0, 1], [0, 3, 2]], + [[1], [1, 3]], + [[1, 2], [2, 1, 0]], + [[3, 1, 2], [3, 1, 0]], + [[1, 3], [3, 1, 2]], + [[2, 0, 1, 3], [3, 1, 0, 2]], + [[1, 3, 0], [1, 3]], + [[2, 3, 1], [1, 0, 2]], + [[1, 2, 0, 3], [0, 2, 1, 3]], + [[2], [0, 1, 3]], + [[1], [1, 2]], + [[1, 0, 2, 3], [3, 0, 1, 2]], + [[0, 1, 3, 2], [1, 3, 0, 2]], + [[3, 0, 2, 1], [0, 2, 3]], + [[1, 2, 0], [1, 2, 3]], + [[1, 0, 3], [2, 3, 0]], + [[2, 3, 0], [3, 1, 0]], + [[1, 3], [1, 0]], + [[2, 1, 0, 3], [2, 0, 3, 1]], + [[3, 2, 0], [2, 1, 0]], + [[0, 1, 3], [0, 3, 1]], + [[3, 1, 0], [3, 2, 1]], + [[3, 2], [3, 1]], + [[3], [2, 1, 0]], + [[1, 2, 3, 0], []], + [[1, 3, 2, 0], [3, 1, 2]], + [[1], [0, 2]], + [[3, 2, 0], [3, 2, 0]], + [[3], []], + [[1, 0, 3], [2, 1]], + [[3, 1, 0, 2], [2, 3, 1, 0]], + [[0, 1], [0, 3, 2]], + [[0, 2, 3], [0, 2, 1]], + [[1, 3, 0], [3, 0, 2]], + [[3, 1, 2], [1, 2, 3]], + [[3, 1, 2], [3, 1, 0]], + [[0, 3, 1, 2], [3, 2, 1, 0]], + [[0, 3], [3, 2, 1]], + [[2, 3], [1, 3, 0]], + [[0, 3, 2], [2, 0, 3, 1]], + [[2, 3], [1, 3]], + [[3, 1, 2, 0], [2, 3, 1, 0]], + [[1, 0, 3, 2], [3, 0, 1, 2]], + [[3, 2, 1, 0], [0, 1, 3, 2]], + [[3, 1, 2], [3]], + [[0, 1, 3, 2], [2, 3, 0, 1]], + [[1, 2, 3, 0], [1, 3, 0, 2]], + [3, 1, 2], + [[3, 1, 2], [0, 3, 2]], + [[2, 3, 0], [1, 2, 0]], + [[2, 0, 3], [2, 0]], + [[3, 1, 0, 2], [3, 1, 0, 2]], + [[0, 1, 2], [2, 0, 1]], + [[1, 0, 3], [2, 3, 0]], + [[2, 0, 1], [0, 1, 3]], + [[2, 1], [0, 1, 3]], + ] def test_dygraph(self): paddle.disable_static() @@ -159,88 +206,89 @@ def test_static(self): paddle.enable_static() for axes in self.all_axes: for place in self.places: - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - x = paddle.static.data(name='x', - shape=self.x_shape, - dtype=self.dtype) - y = paddle.static.data(name='y', - shape=self.y_shape, - dtype=self.dtype) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + x = paddle.static.data( + name='x', shape=self.x_shape, dtype=self.dtype + ) + y = paddle.static.data( + name='y', shape=self.y_shape, dtype=self.dtype + ) z = paddle.tensordot(x, y, axes) exe = paddle.static.Executor(place) - paddle_res = exe.run(feed={ - 'x': self.x, - 'y': self.y - }, - fetch_list=[z]) + paddle_res = exe.run( + feed={'x': self.x, 'y': self.y}, fetch_list=[z] + ) np_res = tensordot_np(self.x, self.y, axes) np.testing.assert_allclose(paddle_res[0], np_res, rtol=1e-6) class TestTensordotAPIFloat64(TestTensordotAPI): - def set_dtype(self): self.dtype = np.float64 class TestTensordotAPIBroadcastCase1(TestTensordotAPIFloat64): - def set_input_shape(self): self.x_shape = [1, 1, 1, 5] self.y_shape = [1, 5, 1, 1] class TestTensordotAPIBroadcastCase2(TestTensordotAPIFloat64): - def set_input_shape(self): self.x_shape = [1, 5, 5, 5] self.y_shape = [1, 1, 1, 5] class TestTensordotAPIBroadcastCase3(TestTensordotAPIFloat64): - def set_input_shape(self): self.x_shape = [5, 5, 5, 1] self.y_shape = [5, 5, 1, 5] class TestTensordotAPIBroadcastCase4(TestTensordotAPIFloat64): - def set_input_shape(self): self.x_shape = [5, 5, 5, 1] self.y_shape = [1, 1, 1, 1] class TestTensordotAPIBroadcastCase5(TestTensordotAPIFloat64): - def set_input_shape(self): self.x_shape = [1, 1, 5, 5] self.y_shape = [5, 5, 1, 5] class TestTensordotAPIAxesType(TestTensordotAPI): - def set_input_shape(self): self.x_shape = [3, 4, 4] self.y_shape = [4, 4, 5] def set_test_axes(self): self.all_axes = [ - 0, 1, 2, (1, ), [1], ((1, ), ), ([1], ), ((2, 1), (0, )), - ((1, 2), (0, 1)), ([1, 2], [0, 1]), ([1, 2], [0, 1]), - [[1, 2], [0, 1]] + 0, + 1, + 2, + (1,), + [1], + ((1,),), + ([1],), + ((2, 1), (0,)), + ((1, 2), (0, 1)), + ([1, 2], [0, 1]), + ([1, 2], [0, 1]), + [[1, 2], [0, 1]], ] def test_tensor_axes(self): # The 'axes' with type 'Tensor' in tensordot is not available in static mode paddle.disable_static() tensor_axes = [ - paddle.to_tensor([1]), (paddle.to_tensor([1])), + paddle.to_tensor([1]), + (paddle.to_tensor([1])), (paddle.to_tensor([1, 2]), paddle.to_tensor([0, 1])), - [paddle.to_tensor([1, 2]), - paddle.to_tensor([0, 1])], - paddle.to_tensor([[1, 2], [0, 1]]) + [paddle.to_tensor([1, 2]), paddle.to_tensor([0, 1])], + paddle.to_tensor([[1, 2], [0, 1]]), ] for place in self.places: @@ -252,8 +300,15 @@ def test_tensor_axes(self): np.testing.assert_allclose(paddle_res, np_res, rtol=1e-6) def test_error(self): - self.all_axes = [[[[0], [1]]], 0.1, -1, 100, [[1, 2], [0, 0]], - [[1, 2], [0, -1]], [0, 1, 2, 3]] + self.all_axes = [ + [[[0], [1]]], + 0.1, + -1, + 100, + [[1, 2], [0, 0]], + [[1, 2], [0, -1]], + [0, 1, 2, 3], + ] paddle.disable_static() x = paddle.to_tensor(self.x) y = paddle.to_tensor(self.y) @@ -263,7 +318,6 @@ def test_error(self): class TestTensordotAPIAxesTypeFloat64(TestTensordotAPIAxesType): - def set_dtype(self): self.dtype = np.float64 diff --git a/python/paddle/fluid/tests/unittests/test_tf32_cublas.py b/python/paddle/fluid/tests/unittests/test_tf32_cublas.py index 91dbb9938bb5b..ce97a7ad173db 100644 --- a/python/paddle/fluid/tests/unittests/test_tf32_cublas.py +++ b/python/paddle/fluid/tests/unittests/test_tf32_cublas.py @@ -20,7 +20,6 @@ class TestTF32Switch(unittest.TestCase): - def test_on_off(self): if core.is_compiled_with_cuda(): place = fluid.CUDAPlace(0) @@ -36,7 +35,6 @@ def test_on_off(self): class TestTF32OnMatmul(unittest.TestCase): - def test_dygraph_without_out(self): if core.is_compiled_with_cuda(): place = fluid.CUDAPlace(0) diff --git a/python/paddle/fluid/tests/unittests/test_tf32_cudnn.py b/python/paddle/fluid/tests/unittests/test_tf32_cudnn.py index e50bc28133332..cb57c93b71cb5 100644 --- a/python/paddle/fluid/tests/unittests/test_tf32_cudnn.py +++ b/python/paddle/fluid/tests/unittests/test_tf32_cudnn.py @@ -17,7 +17,6 @@ class TestTF32Switch(unittest.TestCase): - def test_on_off(self): if core.is_compiled_with_cuda(): self.assertTrue(core.get_cudnn_switch()) # default diff --git a/python/paddle/fluid/tests/unittests/test_tile_op.py b/python/paddle/fluid/tests/unittests/test_tile_op.py index 362464a1ba367..b6ca4b5711ac5 100644 --- a/python/paddle/fluid/tests/unittests/test_tile_op.py +++ b/python/paddle/fluid/tests/unittests/test_tile_op.py @@ -23,9 +23,8 @@ import paddle.fluid.layers as layers -#Situation 1: repeat_times is a list (without tensor) +# Situation 1: repeat_times is a list (without tensor) class TestTileOpRank1(OpTest): - def setUp(self): self.op_type = "tile" self.init_data() @@ -47,21 +46,18 @@ def test_check_grad(self): class TestTileOpRank_ZeroDim1(TestTileOpRank1): - def init_data(self): self.ori_shape = [] self.repeat_times = [] class TestTileOpRank_ZeroDim2(TestTileOpRank1): - def init_data(self): self.ori_shape = [] self.repeat_times = [2] class TestTileOpRank_ZeroDim3(TestTileOpRank1): - def init_data(self): self.ori_shape = [] self.repeat_times = [2, 3] @@ -69,42 +65,36 @@ def init_data(self): # with dimension expanding class TestTileOpRank2Expanding(TestTileOpRank1): - def init_data(self): self.ori_shape = [120] self.repeat_times = [2, 2] class TestTileOpRank2(TestTileOpRank1): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] class TestTileOpRank3_Corner(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (1, 1, 1) class TestTileOpRank3_Corner2(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (2, 2) class TestTileOpRank3(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 15) self.repeat_times = (2, 1, 4) class TestTileOpRank4(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.repeat_times = (3, 2, 1, 2) @@ -112,14 +102,14 @@ def init_data(self): # Situation 2: repeat_times is a list (with tensor) class TestTileOpRank1_tensor_attr(OpTest): - def setUp(self): self.op_type = "tile" self.init_data() repeat_times_tensor = [] for index, ele in enumerate(self.repeat_times): - repeat_times_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + repeat_times_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype("float64"), @@ -142,7 +132,6 @@ def test_check_grad(self): class TestTileOpRank2_Corner_tensor_attr(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [1, 1] @@ -150,7 +139,6 @@ def init_data(self): class TestTileOpRank2_attr_tensor(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -159,7 +147,6 @@ def init_data(self): # Situation 3: repeat_times is a tensor class TestTileOpRank1_tensor(OpTest): - def setUp(self): self.op_type = "tile" self.init_data() @@ -184,7 +171,6 @@ def test_check_grad(self): class TestTileOpRank2_tensor(TestTileOpRank1_tensor): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -192,7 +178,6 @@ def init_data(self): # Situation 4: input x is Integer class TestTileOpInteger(OpTest): - def setUp(self): self.op_type = "tile" self.inputs = { @@ -208,7 +193,6 @@ def test_check_output(self): # Situation 5: input x is Bool class TestTileOpBoolean(OpTest): - def setUp(self): self.op_type = "tile" self.inputs = {'X': np.random.randint(2, size=(2, 4, 5)).astype("bool")} @@ -222,7 +206,6 @@ def test_check_output(self): # Situation 56: input x is Integer class TestTileOpInt64_t(OpTest): - def setUp(self): self.op_type = "tile" self.inputs = { @@ -237,11 +220,11 @@ def test_check_output(self): class TestTileError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) repeat_times = [2, 2] self.assertRaises(TypeError, paddle.tile, x1, repeat_times) x2 = fluid.layers.data(name='x2', shape=[4], dtype="uint8") @@ -252,7 +235,6 @@ def test_errors(self): class TestTileAPIStatic(unittest.TestCase): - def test_api(self): with program_guard(Program(), Program()): repeat_times = [2, 2] @@ -264,7 +246,6 @@ def test_api(self): # Test python API class TestTileAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): np_x = np.random.random([12, 14]).astype("float32") @@ -286,7 +267,6 @@ def test_api(self): class TestTileDoubleGradCheck(unittest.TestCase): - def tile_wrapper(self, x): return paddle.tile(x[0], [2, 1]) @@ -301,17 +281,13 @@ def func(self, place): out = paddle.tile(data, [2, 1]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.tile_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.tile_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -323,7 +299,6 @@ def test_grad(self): class TestTileTripleGradCheck(unittest.TestCase): - def tile_wrapper(self, x): return paddle.tile(x[0], [2, 1]) @@ -338,17 +313,13 @@ def func(self, place): out = paddle.tile(data, [2, 1]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.tile_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.tile_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -360,7 +331,6 @@ def test_grad(self): class TestTileAPI_ZeroDim(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) diff --git a/python/paddle/fluid/tests/unittests/test_top_k_op.py b/python/paddle/fluid/tests/unittests/test_top_k_op.py index a4e54af05bd13..d722d3f622fe5 100644 --- a/python/paddle/fluid/tests/unittests/test_top_k_op.py +++ b/python/paddle/fluid/tests/unittests/test_top_k_op.py @@ -19,7 +19,6 @@ class TestTopkOp(OpTest): - def setUp(self): self.variable_k = False self.set_args() diff --git a/python/paddle/fluid/tests/unittests/test_top_k_v2_op.py b/python/paddle/fluid/tests/unittests/test_top_k_v2_op.py index c6254801b6eca..60d2502a56b2a 100644 --- a/python/paddle/fluid/tests/unittests/test_top_k_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_top_k_v2_op.py @@ -36,7 +36,6 @@ def numpy_topk(x, k=1, axis=-1, largest=True): class TestTopkOp(OpTest): - def init_args(self): self.k = 3 self.axis = 1 @@ -50,10 +49,9 @@ def setUp(self): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} def test_check_output(self): @@ -64,7 +62,6 @@ def test_check_grad(self): class TestTopkOp1(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 0 @@ -72,7 +69,6 @@ def init_args(self): class TestTopkOp2(TestTopkOp): - def init_args(self): self.k = 4 self.axis = 0 @@ -80,7 +76,6 @@ def init_args(self): class TestTopkOp3(OpTest): - def init_args(self): self.k = 6 self.axis = 1 @@ -94,15 +89,13 @@ def setUp(self): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopkOp4(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -116,15 +109,13 @@ def setUp(self): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopkOp5(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -138,15 +129,13 @@ def setUp(self): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopkOp6(OpTest): - def init_args(self): self.k = 100 self.axis = 1 @@ -160,15 +149,13 @@ def setUp(self): self.init_args() self.inputs = {'X': self.input_data} self.attrs = {'k': self.k, 'axis': self.axis, 'largest': self.largest} - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} class TestTopKAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.input_data = np.random.rand(6, 7, 8) @@ -181,92 +168,89 @@ def run_dygraph(self, place): # test case for basic test case 1 paddle_result = paddle.topk(input_tensor, k=2) numpy_result = numpy_topk(self.input_data, k=2) - np.testing.assert_allclose(paddle_result[0].numpy(), - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[1].numpy(), - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[0].numpy(), numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[1].numpy(), numpy_result[1], rtol=1e-05 + ) # test case for basic test case 2 with axis paddle_result = paddle.topk(input_tensor, k=2, axis=1) numpy_result = numpy_topk(self.input_data, k=2, axis=1) - np.testing.assert_allclose(paddle_result[0].numpy(), - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[1].numpy(), - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[0].numpy(), numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[1].numpy(), numpy_result[1], rtol=1e-05 + ) # test case for basic test case 3 with tensor K k_tensor = paddle.to_tensor(np.array([2])) paddle_result = paddle.topk(input_tensor, k=k_tensor, axis=1) numpy_result = numpy_topk(self.input_data, k=2, axis=1) - np.testing.assert_allclose(paddle_result[0].numpy(), - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[1].numpy(), - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[0].numpy(), numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[1].numpy(), numpy_result[1], rtol=1e-05 + ) # test case for basic test case 4 with tensor largest k_tensor = paddle.to_tensor(np.array([2])) - paddle_result = paddle.topk(input_tensor, - k=2, - axis=1, - largest=False) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=1, - largest=False) - np.testing.assert_allclose(paddle_result[0].numpy(), - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[1].numpy(), - numpy_result[1], - rtol=1e-05) + paddle_result = paddle.topk( + input_tensor, k=2, axis=1, largest=False + ) + numpy_result = numpy_topk( + self.input_data, k=2, axis=1, largest=False + ) + np.testing.assert_allclose( + paddle_result[0].numpy(), numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[1].numpy(), numpy_result[1], rtol=1e-05 + ) # test case for basic test case 5 with axis -1 k_tensor = paddle.to_tensor(np.array([2])) - paddle_result = paddle.topk(input_tensor, - k=2, - axis=-1, - largest=False) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=-1, - largest=False) - np.testing.assert_allclose(paddle_result[0].numpy(), - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[1].numpy(), - numpy_result[1], - rtol=1e-05) + paddle_result = paddle.topk( + input_tensor, k=2, axis=-1, largest=False + ) + numpy_result = numpy_topk( + self.input_data, k=2, axis=-1, largest=False + ) + np.testing.assert_allclose( + paddle_result[0].numpy(), numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[1].numpy(), numpy_result[1], rtol=1e-05 + ) # test case for basic test case 6 for the partial sort paddle_result = paddle.topk(large_input_tensor, k=1, axis=-1) numpy_result = numpy_topk(self.large_input_data, k=1, axis=-1) - np.testing.assert_allclose(paddle_result[0].numpy(), - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[1].numpy(), - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[0].numpy(), numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[1].numpy(), numpy_result[1], rtol=1e-05 + ) # test case for basic test case 7 for the unsorted paddle_result = paddle.topk(input_tensor, k=2, axis=1, sorted=False) - sort_paddle = numpy_topk(np.array(paddle_result[0].numpy()), - axis=1, - k=2) + sort_paddle = numpy_topk( + np.array(paddle_result[0].numpy()), axis=1, k=2 + ) numpy_result = numpy_topk(self.input_data, k=2, axis=1) - np.testing.assert_allclose(sort_paddle[0], - numpy_result[0], - rtol=1e-05) + np.testing.assert_allclose( + sort_paddle[0], numpy_result[0], rtol=1e-05 + ) def run_static(self, place): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - input_tensor = paddle.static.data(name="x", - shape=[6, 7, 8], - dtype="float64") - large_input_tensor = paddle.static.data(name="large_x", - shape=[2, 1030], - dtype="float64") + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + input_tensor = paddle.static.data( + name="x", shape=[6, 7, 8], dtype="float64" + ) + large_input_tensor = paddle.static.data( + name="large_x", shape=[2, 1030], dtype="float64" + ) k_tensor = paddle.static.data(name="k", shape=[1], dtype="int32") result1 = paddle.topk(input_tensor, k=2) result2 = paddle.topk(input_tensor, k=2, axis=-1) @@ -280,71 +264,80 @@ def run_static(self, place): exe = paddle.static.Executor(place) input_data = np.random.rand(10, 20).astype("float64") large_input_data = np.random.rand(2, 100).astype("float64") - paddle_result = exe.run(feed={ - "x": self.input_data, - "large_x": self.large_input_data, - "k": np.array([2]).astype("int32") - }, - fetch_list=[ - result1[0], result1[1], result2[0], - result2[1], result3[0], result3[1], - result4[0], result4[1], result5[0], - result5[1], result6[0], result6[1], - result7[0], result7[1] - ]) + paddle_result = exe.run( + feed={ + "x": self.input_data, + "large_x": self.large_input_data, + "k": np.array([2]).astype("int32"), + }, + fetch_list=[ + result1[0], + result1[1], + result2[0], + result2[1], + result3[0], + result3[1], + result4[0], + result4[1], + result5[0], + result5[1], + result6[0], + result6[1], + result7[0], + result7[1], + ], + ) numpy_result = numpy_topk(self.input_data, k=2) - np.testing.assert_allclose(paddle_result[0], - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[1], - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[0], numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[1], numpy_result[1], rtol=1e-05 + ) numpy_result = numpy_topk(self.input_data, k=2, axis=-1) - np.testing.assert_allclose(paddle_result[2], - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[3], - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[2], numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[3], numpy_result[1], rtol=1e-05 + ) numpy_result = numpy_topk(self.input_data, k=2, axis=1) - np.testing.assert_allclose(paddle_result[4], - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[5], - numpy_result[1], - rtol=1e-05) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=1, - largest=False) - np.testing.assert_allclose(paddle_result[6], - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[7], - numpy_result[1], - rtol=1e-05) - numpy_result = numpy_topk(self.input_data, - k=2, - axis=-1, - largest=False) - np.testing.assert_allclose(paddle_result[8], - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[9], - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[4], numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[5], numpy_result[1], rtol=1e-05 + ) + numpy_result = numpy_topk( + self.input_data, k=2, axis=1, largest=False + ) + np.testing.assert_allclose( + paddle_result[6], numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[7], numpy_result[1], rtol=1e-05 + ) + numpy_result = numpy_topk( + self.input_data, k=2, axis=-1, largest=False + ) + np.testing.assert_allclose( + paddle_result[8], numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[9], numpy_result[1], rtol=1e-05 + ) numpy_result = numpy_topk(self.large_input_data, k=1, axis=-1) - np.testing.assert_allclose(paddle_result[10], - numpy_result[0], - rtol=1e-05) - np.testing.assert_allclose(paddle_result[11], - numpy_result[1], - rtol=1e-05) + np.testing.assert_allclose( + paddle_result[10], numpy_result[0], rtol=1e-05 + ) + np.testing.assert_allclose( + paddle_result[11], numpy_result[1], rtol=1e-05 + ) sort_paddle = numpy_topk(paddle_result[12], axis=1, k=2) numpy_result = numpy_topk(self.input_data, k=2, axis=1) - np.testing.assert_allclose(sort_paddle[0], - numpy_result[0], - rtol=1e-05) + np.testing.assert_allclose( + sort_paddle[0], numpy_result[0], rtol=1e-05 + ) def test_cases(self): places = [core.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_trace_op.py b/python/paddle/fluid/tests/unittests/test_trace_op.py index 83aca5654c20c..8bceee2fdfdf6 100644 --- a/python/paddle/fluid/tests/unittests/test_trace_op.py +++ b/python/paddle/fluid/tests/unittests/test_trace_op.py @@ -22,7 +22,6 @@ class TestTraceOp(OpTest): - def setUp(self): self.op_type = "trace" self.python_api = paddle.trace @@ -43,31 +42,32 @@ def init_config(self): class TestTraceOpCase1(TestTraceOp): - def init_config(self): self.case = np.random.randn(2, 20, 2, 3).astype('float32') self.inputs = {'Input': self.case} self.attrs = {'offset': 1, 'axis1': 0, 'axis2': 2} - self.target = np.trace(self.inputs['Input'], - offset=self.attrs['offset'], - axis1=self.attrs['axis1'], - axis2=self.attrs['axis2']) + self.target = np.trace( + self.inputs['Input'], + offset=self.attrs['offset'], + axis1=self.attrs['axis1'], + axis2=self.attrs['axis2'], + ) class TestTraceOpCase2(TestTraceOp): - def init_config(self): self.case = np.random.randn(2, 20, 2, 3).astype('float32') self.inputs = {'Input': self.case} self.attrs = {'offset': -5, 'axis1': 1, 'axis2': -1} - self.target = np.trace(self.inputs['Input'], - offset=self.attrs['offset'], - axis1=self.attrs['axis1'], - axis2=self.attrs['axis2']) + self.target = np.trace( + self.inputs['Input'], + offset=self.attrs['offset'], + axis1=self.attrs['axis1'], + axis2=self.attrs['axis2'], + ) class TestTraceAPICase(unittest.TestCase): - def test_case1(self): case = np.random.randn(2, 20, 2, 3).astype('float32') data1 = fluid.data(name='data1', shape=[2, 20, 2, 3], dtype='float32') @@ -76,10 +76,12 @@ def test_case1(self): place = core.CPUPlace() exe = fluid.Executor(place) - results = exe.run(fluid.default_main_program(), - feed={"data1": case}, - fetch_list=[out1, out2], - return_numpy=True) + results = exe.run( + fluid.default_main_program(), + feed={"data1": case}, + fetch_list=[out1, out2], + return_numpy=True, + ) target1 = np.trace(case) target2 = np.trace(case, offset=-5, axis1=1, axis2=-1) np.testing.assert_allclose(results[0], target1, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py b/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py index 59b652b1d3d67..b160debe41c9a 100644 --- a/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py +++ b/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py @@ -24,12 +24,12 @@ class SimpleFCLayer(nn.Layer): - def __init__(self, feature_size, batch_size, fc_size): super(SimpleFCLayer, self).__init__() self._linear = nn.Linear(feature_size, fc_size) self._offset = paddle.to_tensor( - np.random.random((batch_size, fc_size)).astype('float32')) + np.random.random((batch_size, fc_size)).astype('float32') + ) def forward(self, x): fc = self._linear(x) @@ -37,7 +37,6 @@ def forward(self, x): class LinearNetWithNone(nn.Layer): - def __init__(self, feature_size, fc_size): super(LinearNetWithNone, self).__init__() self._linear = nn.Linear(feature_size, fc_size) @@ -49,7 +48,6 @@ def forward(self, x): class TestTracedLayerErrMsg(unittest.TestCase): - def setUp(self): self.batch_size = 4 self.feature_size = 3 @@ -66,116 +64,163 @@ def test_trace_err(self): return with fluid.dygraph.guard(): in_x = fluid.dygraph.to_variable( - np.random.random( - (self.batch_size, self.feature_size)).astype('float32')) + np.random.random((self.batch_size, self.feature_size)).astype( + 'float32' + ) + ) with self.assertRaises(AssertionError) as e: dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - None, [in_x]) + None, [in_x] + ) self.assertEqual( - "The type of 'layer' in fluid.dygraph.jit.TracedLayer.trace must be fluid.dygraph.Layer, but received <{} 'NoneType'>." - .format(self.type_str), str(e.exception)) + "The type of 'layer' in fluid.dygraph.jit.TracedLayer.trace must be fluid.dygraph.Layer, but received <{} 'NoneType'>.".format( + self.type_str + ), + str(e.exception), + ) with self.assertRaises(TypeError) as e: dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - self.layer, 3) + self.layer, 3 + ) self.assertEqual( - "The type of 'each element of inputs' in fluid.dygraph.jit.TracedLayer.trace must be fluid.Variable, but received <{} 'int'>." - .format(self.type_str), str(e.exception)) + "The type of 'each element of inputs' in fluid.dygraph.jit.TracedLayer.trace must be fluid.Variable, but received <{} 'int'>.".format( + self.type_str + ), + str(e.exception), + ) with self.assertRaises(TypeError) as e: dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - self.layer, [True, 1]) + self.layer, [True, 1] + ) self.assertEqual( - "The type of 'each element of inputs' in fluid.dygraph.jit.TracedLayer.trace must be fluid.Variable, but received <{} 'bool'>." - .format(self.type_str), str(e.exception)) + "The type of 'each element of inputs' in fluid.dygraph.jit.TracedLayer.trace must be fluid.Variable, but received <{} 'bool'>.".format( + self.type_str + ), + str(e.exception), + ) dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - self.layer, [in_x]) + self.layer, [in_x] + ) def test_set_strategy_err(self): if fluid.framework.in_dygraph_mode(): return with fluid.dygraph.guard(): in_x = fluid.dygraph.to_variable( - np.random.random( - (self.batch_size, self.feature_size)).astype('float32')) + np.random.random((self.batch_size, self.feature_size)).astype( + 'float32' + ) + ) dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - self.layer, [in_x]) + self.layer, [in_x] + ) with self.assertRaises(AssertionError) as e: traced_layer.set_strategy(1, fluid.ExecutionStrategy()) self.assertEqual( - "The type of 'build_strategy' in fluid.dygraph.jit.TracedLayer.set_strategy must be fluid.BuildStrategy, but received <{} 'int'>." - .format(self.type_str), str(e.exception)) + "The type of 'build_strategy' in fluid.dygraph.jit.TracedLayer.set_strategy must be fluid.BuildStrategy, but received <{} 'int'>.".format( + self.type_str + ), + str(e.exception), + ) with self.assertRaises(AssertionError) as e: traced_layer.set_strategy(fluid.BuildStrategy(), False) self.assertEqual( - "The type of 'exec_strategy' in fluid.dygraph.jit.TracedLayer.set_strategy must be fluid.ExecutionStrategy, but received <{} 'bool'>." - .format(self.type_str), str(e.exception)) + "The type of 'exec_strategy' in fluid.dygraph.jit.TracedLayer.set_strategy must be fluid.ExecutionStrategy, but received <{} 'bool'>.".format( + self.type_str + ), + str(e.exception), + ) traced_layer.set_strategy(build_strategy=fluid.BuildStrategy()) traced_layer.set_strategy(exec_strategy=fluid.ExecutionStrategy()) - traced_layer.set_strategy(fluid.BuildStrategy(), - fluid.ExecutionStrategy()) + traced_layer.set_strategy( + fluid.BuildStrategy(), fluid.ExecutionStrategy() + ) def test_save_inference_model_err(self): if fluid.framework.in_dygraph_mode(): return with fluid.dygraph.guard(): in_x = fluid.dygraph.to_variable( - np.random.random( - (self.batch_size, self.feature_size)).astype('float32')) + np.random.random((self.batch_size, self.feature_size)).astype( + 'float32' + ) + ) dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - self.layer, [in_x]) + self.layer, [in_x] + ) path = os.path.join(self.temp_dir.name, './traced_layer_err_msg') with self.assertRaises(TypeError) as e: traced_layer.save_inference_model([0]) self.assertEqual( - "The type of 'path' in fluid.dygraph.jit.TracedLayer.save_inference_model must be <{} 'str'>, but received <{} 'list'>. " - .format(self.type_str, self.type_str), str(e.exception)) + "The type of 'path' in fluid.dygraph.jit.TracedLayer.save_inference_model must be <{} 'str'>, but received <{} 'list'>. ".format( + self.type_str, self.type_str + ), + str(e.exception), + ) with self.assertRaises(TypeError) as e: traced_layer.save_inference_model(path, [0], [None]) self.assertEqual( - "The type of 'each element of fetch' in fluid.dygraph.jit.TracedLayer.save_inference_model must be <{} 'int'>, but received <{} 'NoneType'>. " - .format(self.type_str, self.type_str), str(e.exception)) + "The type of 'each element of fetch' in fluid.dygraph.jit.TracedLayer.save_inference_model must be <{} 'int'>, but received <{} 'NoneType'>. ".format( + self.type_str, self.type_str + ), + str(e.exception), + ) with self.assertRaises(TypeError) as e: traced_layer.save_inference_model(path, [0], False) self.assertEqual( - "The type of 'fetch' in fluid.dygraph.jit.TracedLayer.save_inference_model must be (<{} 'NoneType'>, <{} 'list'>), but received <{} 'bool'>. " - .format(self.type_str, self.type_str, self.type_str), - str(e.exception)) + "The type of 'fetch' in fluid.dygraph.jit.TracedLayer.save_inference_model must be (<{} 'NoneType'>, <{} 'list'>), but received <{} 'bool'>. ".format( + self.type_str, self.type_str, self.type_str + ), + str(e.exception), + ) with self.assertRaises(TypeError) as e: traced_layer.save_inference_model(path, [None], [0]) self.assertEqual( - "The type of 'each element of feed' in fluid.dygraph.jit.TracedLayer.save_inference_model must be <{} 'int'>, but received <{} 'NoneType'>. " - .format(self.type_str, self.type_str), str(e.exception)) + "The type of 'each element of feed' in fluid.dygraph.jit.TracedLayer.save_inference_model must be <{} 'int'>, but received <{} 'NoneType'>. ".format( + self.type_str, self.type_str + ), + str(e.exception), + ) with self.assertRaises(TypeError) as e: traced_layer.save_inference_model(path, True, [0]) self.assertEqual( - "The type of 'feed' in fluid.dygraph.jit.TracedLayer.save_inference_model must be (<{} 'NoneType'>, <{} 'list'>), but received <{} 'bool'>. " - .format(self.type_str, self.type_str, self.type_str), - str(e.exception)) + "The type of 'feed' in fluid.dygraph.jit.TracedLayer.save_inference_model must be (<{} 'NoneType'>, <{} 'list'>), but received <{} 'bool'>. ".format( + self.type_str, self.type_str, self.type_str + ), + str(e.exception), + ) with self.assertRaises(ValueError) as e: traced_layer.save_inference_model("") self.assertEqual( "The input path MUST be format of dirname/file_prefix [dirname\\file_prefix in Windows system], " - "but received file_prefix is empty string.", str(e.exception)) + "but received file_prefix is empty string.", + str(e.exception), + ) traced_layer.save_inference_model(path) def _train_simple_net(self): layer = None with fluid.dygraph.guard(): - layer = SimpleFCLayer(self.feature_size, self.batch_size, - self.fc_size) - optimizer = fluid.optimizer.SGD(learning_rate=1e-3, - parameter_list=layer.parameters()) + layer = SimpleFCLayer( + self.feature_size, self.batch_size, self.fc_size + ) + optimizer = fluid.optimizer.SGD( + learning_rate=1e-3, parameter_list=layer.parameters() + ) for i in range(5): in_x = fluid.dygraph.to_variable( np.random.random( - (self.batch_size, self.feature_size)).astype('float32')) + (self.batch_size, self.feature_size) + ).astype('float32') + ) dygraph_out = layer(in_x) loss = fluid.layers.reduce_mean(dygraph_out) loss.backward() @@ -184,7 +229,6 @@ def _train_simple_net(self): class TestOutVarWithNoneErrMsg(unittest.TestCase): - def test_linear_net_with_none(self): if fluid.framework.in_dygraph_mode(): return @@ -192,7 +236,8 @@ def test_linear_net_with_none(self): in_x = paddle.to_tensor(np.random.random((4, 100)).astype('float32')) with self.assertRaises(TypeError): dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - model, [in_x]) + model, [in_x] + ) class TestTracedLayerSaveInferenceModel(unittest.TestCase): @@ -202,6 +247,7 @@ def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() self.save_path = os.path.join(self.temp_dir.name, "./nonexist_dir/fc") import shutil + if os.path.exists(os.path.dirname(self.save_path)): shutil.rmtree(os.path.dirname(self.save_path)) @@ -215,7 +261,8 @@ def test_mkdir_when_input_path_non_exist(self): input_var = paddle.to_tensor(np.random.random([4, 3]).astype('float32')) with fluid.dygraph.guard(): dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( - fc_layer, inputs=[input_var]) + fc_layer, inputs=[input_var] + ) self.assertFalse(os.path.exists(os.path.dirname(self.save_path))) traced_layer.save_inference_model(self.save_path) self.assertTrue(os.path.exists(os.path.dirname(self.save_path))) diff --git a/python/paddle/fluid/tests/unittests/test_trainable.py b/python/paddle/fluid/tests/unittests/test_trainable.py index d9d5121ff9b52..bbd7ae55d30df 100644 --- a/python/paddle/fluid/tests/unittests/test_trainable.py +++ b/python/paddle/fluid/tests/unittests/test_trainable.py @@ -22,21 +22,18 @@ def test_trainable(): x = fluid.layers.data(name='image', shape=[784], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') - feature = fluid.layers.fc(input=x, - size=10, - param_attr=fluid.ParamAttr(trainable=False)) + feature = fluid.layers.fc( + input=x, size=10, param_attr=fluid.ParamAttr(trainable=False) + ) loss = fluid.layers.cross_entropy(input=feature, label=label) loss = paddle.mean(loss) return loss class TestTrainable(unittest.TestCase): - - def check_trainable(self, - model, - feed_dict, - op_count, - optimizer=fluid.optimizer.Adam()): + def check_trainable( + self, model, feed_dict, op_count, optimizer=fluid.optimizer.Adam() + ): place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -64,22 +61,17 @@ def test_trainable(self): feed_dict = {'image': img, 'label': label} # Note that, because the Weight of FC is not trainable and the x is stop_gradient, # so the 'mul_grad' should not be appended. - self.check_trainable(test_trainable, - feed_dict, - op_count={ - 'adam': 1, - 'scale': 0, - 'mul_grad': 0 - }) self.check_trainable( test_trainable, feed_dict, - op_count={ - 'adamax': 1, - 'scale': 1, - 'mul_grad': 0 - }, - optimizer=fluid.optimizer.Adamax(learning_rate=0.2)) + op_count={'adam': 1, 'scale': 0, 'mul_grad': 0}, + ) + self.check_trainable( + test_trainable, + feed_dict, + op_count={'adamax': 1, 'scale': 1, 'mul_grad': 0}, + optimizer=fluid.optimizer.Adamax(learning_rate=0.2), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_trainer_desc.py b/python/paddle/fluid/tests/unittests/test_trainer_desc.py index 390a650fd1395..bc584439fdfd8 100644 --- a/python/paddle/fluid/tests/unittests/test_trainer_desc.py +++ b/python/paddle/fluid/tests/unittests/test_trainer_desc.py @@ -21,7 +21,7 @@ class TestTrainerDesc(unittest.TestCase): - """ TestCases for TrainerDesc. """ + """TestCases for TrainerDesc.""" def test_config(self): """ diff --git a/python/paddle/fluid/tests/unittests/test_transfer_dtype_op.py b/python/paddle/fluid/tests/unittests/test_transfer_dtype_op.py index 9d3edc0e06214..364e6ff167ca1 100644 --- a/python/paddle/fluid/tests/unittests/test_transfer_dtype_op.py +++ b/python/paddle/fluid/tests/unittests/test_transfer_dtype_op.py @@ -21,14 +21,13 @@ class TestTransferDtypeOpFp32ToFp64(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float32')} self.outputs = {'Out': ipt.astype('float64')} self.attrs = { 'out_dtype': int(core.VarDesc.VarType.FP64), - 'in_dtype': int(core.VarDesc.VarType.FP32) + 'in_dtype': int(core.VarDesc.VarType.FP32), } self.op_type = 'transfer_dtype' @@ -37,14 +36,13 @@ def test_check_output(self): class TestTransferDtypeOpFp16ToFp32(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float16')} self.outputs = {'Out': ipt.astype('float32')} self.attrs = { 'out_dtype': int(core.VarDesc.VarType.FP32), - 'in_dtype': int(core.VarDesc.VarType.FP16) + 'in_dtype': int(core.VarDesc.VarType.FP16), } self.op_type = 'transfer_dtype' @@ -53,14 +51,13 @@ def test_check_output(self): class TestTransferDtypeOpFp32ToFp16(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) self.inputs = {'X': ipt.astype('float32')} self.outputs = {'Out': ipt.astype('float16')} self.attrs = { 'out_dtype': int(core.VarDesc.VarType.FP16), - 'in_dtype': int(core.VarDesc.VarType.FP32) + 'in_dtype': int(core.VarDesc.VarType.FP32), } self.op_type = 'transfer_dtype' @@ -69,14 +66,13 @@ def test_check_output(self): class TestTransferDtypeOpBf16ToFp32(OpTest): - def setUp(self): ipt = np.array(np.random.randint(10, size=[10, 10])).astype('uint16') self.inputs = {'X': ipt} self.outputs = {'Out': convert_uint16_to_float(ipt)} self.attrs = { 'out_dtype': int(core.VarDesc.VarType.FP32), - 'in_dtype': int(core.VarDesc.VarType.BF16) + 'in_dtype': int(core.VarDesc.VarType.BF16), } self.op_type = 'transfer_dtype' @@ -85,14 +81,13 @@ def test_check_output(self): class TestTransferDtypeFp32ToBf16(OpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]).astype('float32') self.inputs = {'X': ipt} self.outputs = {'Out': convert_float_to_uint16(ipt)} self.attrs = { 'out_dtype': int(core.VarDesc.VarType.BF16), - 'in_dtype': int(core.VarDesc.VarType.FP32) + 'in_dtype': int(core.VarDesc.VarType.FP32), } self.op_type = 'transfer_dtype' diff --git a/python/paddle/fluid/tests/unittests/test_transfer_layout_op.py b/python/paddle/fluid/tests/unittests/test_transfer_layout_op.py index a24fb767d10d3..48c2897b00549 100644 --- a/python/paddle/fluid/tests/unittests/test_transfer_layout_op.py +++ b/python/paddle/fluid/tests/unittests/test_transfer_layout_op.py @@ -25,15 +25,11 @@ # default kNCHW class TestTransferLayoutOpkNCHWTokNHWC(OpTest): - def setUp(self): ipt = np.random.random(size=[2, 3, 10, 10]) self.inputs = {'X': ipt.astype('float32')} self.outputs = {'Out': ipt.transpose([0, 2, 3, 1])} - self.attrs = { - 'src_layout': 0, - 'dst_layout': 1 # kNHWC - } + self.attrs = {'src_layout': 0, 'dst_layout': 1} # kNHWC self.op_type = 'transfer_layout' def test_check_output(self): @@ -45,20 +41,17 @@ def softmax_with_data_format(x, data_format, axis=-1, dtype=None, name=None): outs_cast = x outs_softmax = helper.create_variable_for_type_inference(outs_cast.dtype) - helper.append_op(type='softmax', - inputs={'X': outs_cast}, - outputs={'Out': outs_softmax}, - attrs={ - 'axis': axis, - 'use_cudnn': True, - 'data_format': data_format - }) + helper.append_op( + type='softmax', + inputs={'X': outs_cast}, + outputs={'Out': outs_softmax}, + attrs={'axis': axis, 'use_cudnn': True, 'data_format': data_format}, + ) return outs_softmax class TestTransferLayoutOpGpu(unittest.TestCase): - def test_layout_transfer(self): if not core.is_compiled_with_cuda(): return @@ -69,19 +62,24 @@ def test_layout_transfer(self): startup_program = Program() n, c, h, w = 2, 3, 4, 5 with program_guard(main_program, startup_program): - x = paddle.static.data(shape=[n, c, h, w], - dtype='float32', - name='x') + x = paddle.static.data( + shape=[n, c, h, w], dtype='float32', name='x' + ) y = softmax_with_data_format(x, data_format='NCHW') z = softmax_with_data_format(y, data_format='NHWC') - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) exe.run(startup_program) - ret = exe.run(main_program, - feed={'x': np.full((n, c, h, w), 1, np.float32)}, - fetch_list=[z.name]) + ret = exe.run( + main_program, + feed={'x': np.full((n, c, h, w), 1, np.float32)}, + fetch_list=[z.name], + ) assert len(ret) == 1 assert ret[0].shape == (n, h, w, c) diff --git a/python/paddle/fluid/tests/unittests/test_transformer_api.py b/python/paddle/fluid/tests/unittests/test_transformer_api.py index 6b254ac3115d4..3f2e2185f9ee2 100644 --- a/python/paddle/fluid/tests/unittests/test_transformer_api.py +++ b/python/paddle/fluid/tests/unittests/test_transformer_api.py @@ -15,7 +15,14 @@ import numpy as np import paddle import paddle.fluid as fluid -from paddle.nn.layer.transformer import MultiHeadAttention, TransformerEncoderLayer, TransformerDecoderLayer, TransformerEncoder, TransformerDecoder, Transformer +from paddle.nn.layer.transformer import ( + MultiHeadAttention, + TransformerEncoderLayer, + TransformerDecoderLayer, + TransformerEncoder, + TransformerDecoder, + Transformer, +) import unittest @@ -33,34 +40,67 @@ def generate_basic_params(mode="attn", self_attention=True): kdim, vdim = [np.random.randint(5, 20) for _ in range(2)] key_length = np.random.randint(2, 10) value_length = key_length - return batch_size, query_length, key_length, value_length, embed_dim, kdim, vdim, num_heads, attn_dropout + return ( + batch_size, + query_length, + key_length, + value_length, + embed_dim, + kdim, + vdim, + num_heads, + attn_dropout, + ) else: dropout, act_dropout = 0.0, 0.0 dim_feedforward = np.random.randint(128, 1024) sequence_length = np.random.randint(2, 10) if mode == "encoder_layer": - return batch_size, embed_dim, num_heads, dim_feedforward, dropout, attn_dropout, act_dropout, sequence_length + return ( + batch_size, + embed_dim, + num_heads, + dim_feedforward, + dropout, + attn_dropout, + act_dropout, + sequence_length, + ) elif mode == "decoder_layer": target_length = np.random.randint(2, 10) - return batch_size, embed_dim, num_heads, dim_feedforward, dropout, attn_dropout, act_dropout, sequence_length, target_length - - -def generate_query_key_value_cache(self_attention, - batch_size, - num_heads, - query_length, - embed_dim, - attn_mask_type, - key_length=None, - value_length=None, - kdim=None, - vdim=None, - cache=None): - query = np.random.rand(batch_size, query_length, - embed_dim).astype("float32") - attn_mask = np.ones((batch_size, num_heads, query_length, key_length), - dtype=attn_mask_type) + return ( + batch_size, + embed_dim, + num_heads, + dim_feedforward, + dropout, + attn_dropout, + act_dropout, + sequence_length, + target_length, + ) + + +def generate_query_key_value_cache( + self_attention, + batch_size, + num_heads, + query_length, + embed_dim, + attn_mask_type, + key_length=None, + value_length=None, + kdim=None, + vdim=None, + cache=None, +): + query = np.random.rand(batch_size, query_length, embed_dim).astype( + "float32" + ) + attn_mask = np.ones( + (batch_size, num_heads, query_length, key_length), dtype=attn_mask_type + ) if attn_mask_type == 'int64': attn_mask = np.tril(attn_mask) elif attn_mask_type == 'float64': @@ -77,18 +117,19 @@ def generate_query_key_value_cache(self_attention, cache_dict = {} if cache: if not self_attention: - cache_dict["static_k"] = np.random.rand(batch_size, num_heads, - key_length, - head_dim).astype("float32") - cache_dict["static_v"] = np.random.rand(batch_size, num_heads, - value_length, - head_dim).astype("float32") + cache_dict["static_k"] = np.random.rand( + batch_size, num_heads, key_length, head_dim + ).astype("float32") + cache_dict["static_v"] = np.random.rand( + batch_size, num_heads, value_length, head_dim + ).astype("float32") else: - cache_dict["k"] = np.random.rand(batch_size, num_heads, key_length, - head_dim).astype("float32") - cache_dict["v"] = np.random.rand(batch_size, num_heads, - value_length, - head_dim).astype("float32") + cache_dict["k"] = np.random.rand( + batch_size, num_heads, key_length, head_dim + ).astype("float32") + cache_dict["v"] = np.random.rand( + batch_size, num_heads, value_length, head_dim + ).astype("float32") else: cache_dict = None return query, key, value, attn_mask, cache_dict @@ -113,8 +154,9 @@ def softmax(x): def batch_matmul(x, y): assert x.shape[0] == y.shape[0] assert x.shape[1] == y.shape[1] - retval = np.zeros((x.shape[0], x.shape[1], x.shape[2], y.shape[3]), - dtype=np.float64) + retval = np.zeros( + (x.shape[0], x.shape[1], x.shape[2], y.shape[3]), dtype=np.float64 + ) for i in range(x.shape[0]): for j in range(x.shape[1]): retval[i, j, :, :] = np.matmul(x[i, j, :, :], y[i, j, :, :]) @@ -133,8 +175,13 @@ def scaled_dot_product_attention(q, k, v, d_key, attn_mask, multi_head_attn): weight = softmax(qkt) attn_heads = batch_matmul(weight, v) attn_heads = attn_heads.transpose((0, 2, 1, 3)) - attn_heads = attn_heads.reshape((attn_heads.shape[0], attn_heads.shape[1], - attn_heads.shape[2] * attn_heads.shape[3])) + attn_heads = attn_heads.reshape( + ( + attn_heads.shape[0], + attn_heads.shape[1], + attn_heads.shape[2] * attn_heads.shape[3], + ) + ) return attn_heads @@ -152,8 +199,16 @@ def cal_qkv(key, value, num_heads, embed_dim, multi_head_attn): return k, v -def prepare_qkv(query, key, value, num_heads, embed_dim, self_attention, - multi_head_attn, cache_dict): +def prepare_qkv( + query, + key, + value, + num_heads, + embed_dim, + self_attention, + multi_head_attn, + cache_dict, +): q_weight = multi_head_attn.q_proj.weight.numpy() q = fc(query, q_weight) q = q.reshape((q.shape[0], q.shape[1], num_heads, embed_dim // num_heads)) @@ -196,7 +251,7 @@ def layer_norm(x, normalized_shape, norm, epsilon=1e-05, act=None): x = x.reshape((batch_size * src_len, d_model)) mu = np.mean(x, axis=1, keepdims=True) sigma_squar = np.sum(np.square(x - mu), axis=1) / d_model - x1_up = (x - mu) + x1_up = x - mu x1_down_1 = sigma_squar + epsilon x1_down = np.sqrt(x1_down_1) x1_down = x1_down.reshape((x1_down.shape[0], 1)) @@ -223,9 +278,7 @@ def ffn(src, encoder_layer, ffn_fc1_act="relu"): class TestTransformer(unittest.TestCase): - def test_multi_head_attention(self): - def multihead_attention_test_helper(self_attention, cache): paddle.seed(2020) paddle.framework.random._manual_program_seed(2020) @@ -233,59 +286,111 @@ def multihead_attention_test_helper(self_attention, cache): with fluid.dygraph.guard(fluid.CPUPlace()): # generate params for multi_head_attention - batch_size, query_length, key_length, value_length, embed_dim, kdim, vdim, num_heads, attn_dropout = generate_basic_params( - "attn", self_attention) + ( + batch_size, + query_length, + key_length, + value_length, + embed_dim, + kdim, + vdim, + num_heads, + attn_dropout, + ) = generate_basic_params("attn", self_attention) for attn_mask_type in ['int64', 'float64']: - query, key, value, attn_mask, cache_dict = generate_query_key_value_cache( - self_attention, batch_size, num_heads, query_length, - embed_dim, attn_mask_type, key_length, value_length, - kdim, vdim, cache) + ( + query, + key, + value, + attn_mask, + cache_dict, + ) = generate_query_key_value_cache( + self_attention, + batch_size, + num_heads, + query_length, + embed_dim, + attn_mask_type, + key_length, + value_length, + kdim, + vdim, + cache, + ) if cache and self_attention: - attn_mask = np.concatenate((attn_mask, attn_mask), - axis=3) + attn_mask = np.concatenate( + (attn_mask, attn_mask), axis=3 + ) need_weight, param_attr, bias_attr = False, None, None # call paddle's function multi_head_attn = MultiHeadAttention( - embed_dim, num_heads, attn_dropout, kdim, vdim, - need_weight, param_attr, bias_attr) + embed_dim, + num_heads, + attn_dropout, + kdim, + vdim, + need_weight, + param_attr, + bias_attr, + ) # construct cache object cache_obj = None if cache_dict: if 'k' and 'v' in cache_dict: cache_obj = multi_head_attn.Cache( paddle.to_tensor(cache_dict['k']), - paddle.to_tensor(cache_dict['v'])) + paddle.to_tensor(cache_dict['v']), + ) elif 'static_k' and 'static_v' in cache_dict: cache_obj = multi_head_attn.StaticCache( paddle.to_tensor(cache_dict['static_k']), - paddle.to_tensor(cache_dict['static_v'])) + paddle.to_tensor(cache_dict['static_v']), + ) if attn_mask is not None: attn_output = multi_head_attn( - paddle.to_tensor(query), paddle.to_tensor(key), + paddle.to_tensor(query), + paddle.to_tensor(key), paddle.to_tensor(value), - paddle.to_tensor(attn_mask), cache_obj) + paddle.to_tensor(attn_mask), + cache_obj, + ) else: - attn_output = multi_head_attn(paddle.to_tensor(query), - paddle.to_tensor(key), - paddle.to_tensor(value), - attn_mask, cache_obj) + attn_output = multi_head_attn( + paddle.to_tensor(query), + paddle.to_tensor(key), + paddle.to_tensor(value), + attn_mask, + cache_obj, + ) attn_output = attn_output[0] if cache_dict else attn_output # implementation by numpy # compute q, k, v - q, k, v, _ = prepare_qkv(query, key, value, num_heads, - embed_dim, self_attention, - multi_head_attn, cache_dict) + q, k, v, _ = prepare_qkv( + query, + key, + value, + num_heads, + embed_dim, + self_attention, + multi_head_attn, + cache_dict, + ) # scale dot product attention attn_heads = scaled_dot_product_attention( - q, k, v, embed_dim // num_heads, attn_mask, - multi_head_attn) + q, + k, + v, + embed_dim // num_heads, + attn_mask, + multi_head_attn, + ) out_proj_weight = multi_head_attn.out_proj.weight.numpy() reference = fc(attn_heads, out_proj_weight) - np.testing.assert_allclose(attn_output.numpy(), - reference, - atol=1e-6) + np.testing.assert_allclose( + attn_output.numpy(), reference, atol=1e-6 + ) multihead_attention_test_helper(True, True) multihead_attention_test_helper(True, False) @@ -300,34 +405,51 @@ def test_transformer_encoder_layer(self): ffn_fc1_act = "relu" # 1.generate basic params - batch_size, d_model, n_head, dim_feedforward, dropout, attn_dropout, act_dropout, sequence_length = generate_basic_params( - mode="encoder_layer") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + attn_dropout, + act_dropout, + sequence_length, + ) = generate_basic_params(mode="encoder_layer") # 2.generate input for encoder - src = np.random.rand(batch_size, sequence_length, - d_model).astype("float32") + src = np.random.rand(batch_size, sequence_length, d_model).astype( + "float32" + ) residual = src - src_mask = np.zeros((batch_size, n_head, sequence_length, - sequence_length)).astype("float32") + src_mask = np.zeros( + (batch_size, n_head, sequence_length, sequence_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf # paddle - encoder_layer = TransformerEncoderLayer(d_model, n_head, - dim_feedforward, dropout, - ffn_fc1_act, attn_dropout, - act_dropout) + encoder_layer = TransformerEncoderLayer( + d_model, + n_head, + dim_feedforward, + dropout, + ffn_fc1_act, + attn_dropout, + act_dropout, + ) encoder_output = encoder_layer( - paddle.to_tensor(src), - paddle.to_tensor(src_mask)) # paddle.to_tensor(src_mask)) + paddle.to_tensor(src), paddle.to_tensor(src_mask) + ) # paddle.to_tensor(src_mask)) # 4.numpy: # paddle self attention - self_attn = MultiHeadAttention(d_model, - n_head, - dropout=attn_dropout) - attn_output = self_attn(paddle.to_tensor(src), - paddle.to_tensor(src), - paddle.to_tensor(src), - paddle.to_tensor(src_mask)).numpy() + self_attn = MultiHeadAttention( + d_model, n_head, dropout=attn_dropout + ) + attn_output = self_attn( + paddle.to_tensor(src), + paddle.to_tensor(src), + paddle.to_tensor(src), + paddle.to_tensor(src_mask), + ).numpy() src = attn_output + residual src_norm = layer_norm(src, d_model, encoder_layer.norm1) @@ -337,10 +459,9 @@ def test_transformer_encoder_layer(self): src = residual + ffn_output src = layer_norm(src, d_model, encoder_layer.norm2) - np.testing.assert_allclose(encoder_output.numpy(), - src, - rtol=1e-5, - atol=1e-6) + np.testing.assert_allclose( + encoder_output.numpy(), src, rtol=1e-5, atol=1e-6 + ) def test_transformer_encoder_layer_attr_1(self): with fluid.dygraph.guard(fluid.CPUPlace()): @@ -349,44 +470,67 @@ def test_transformer_encoder_layer_attr_1(self): ffn_fc1_act = "relu" # 1.generate basic params - batch_size, d_model, n_head, dim_feedforward, dropout, attn_dropout, act_dropout, sequence_length = generate_basic_params( - mode="encoder_layer") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + attn_dropout, + act_dropout, + sequence_length, + ) = generate_basic_params(mode="encoder_layer") # 2.generate input for encoder - src = np.random.rand(batch_size, sequence_length, - d_model).astype("float32") - src_mask = np.zeros((batch_size, n_head, sequence_length, - sequence_length)).astype("float32") + src = np.random.rand(batch_size, sequence_length, d_model).astype( + "float32" + ) + src_mask = np.zeros( + (batch_size, n_head, sequence_length, sequence_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf for cache in [True, False]: # paddle - encoder_layer = TransformerEncoderLayer(d_model, n_head, - dim_feedforward, - dropout, ffn_fc1_act, - attn_dropout, - act_dropout) + encoder_layer = TransformerEncoderLayer( + d_model, + n_head, + dim_feedforward, + dropout, + ffn_fc1_act, + attn_dropout, + act_dropout, + ) cache_objs = None if cache: cache_objs = encoder_layer.gen_cache(paddle.to_tensor(src)) - encoder_output = encoder_layer(paddle.to_tensor(src), - paddle.to_tensor(src_mask), - cache_objs) - encoder_output = encoder_output[0].numpy( - ) if cache else encoder_output.numpy() + encoder_output = encoder_layer( + paddle.to_tensor(src), + paddle.to_tensor(src_mask), + cache_objs, + ) + encoder_output = ( + encoder_output[0].numpy() + if cache + else encoder_output.numpy() + ) # 4.numpy: residual = src # paddle self attention - self_attn = MultiHeadAttention(d_model, - n_head, - dropout=attn_dropout) - attn_output = self_attn(paddle.to_tensor(src), - paddle.to_tensor(src), - paddle.to_tensor(src), - paddle.to_tensor(src_mask), cache_objs) - attn_output = attn_output[0].numpy( - ) if cache else attn_output.numpy() + self_attn = MultiHeadAttention( + d_model, n_head, dropout=attn_dropout + ) + attn_output = self_attn( + paddle.to_tensor(src), + paddle.to_tensor(src), + paddle.to_tensor(src), + paddle.to_tensor(src_mask), + cache_objs, + ) + attn_output = ( + attn_output[0].numpy() if cache else attn_output.numpy() + ) src = attn_output + residual src_norm = layer_norm(src, d_model, encoder_layer.norm1) @@ -396,62 +540,92 @@ def test_transformer_encoder_layer_attr_1(self): src = residual + ffn_output src = layer_norm(src, d_model, encoder_layer.norm2) - np.testing.assert_allclose(encoder_output, - src, - rtol=1e-5, - atol=1e-6) + np.testing.assert_allclose( + encoder_output, src, rtol=1e-5, atol=1e-6 + ) def test_transformer_decoder_layer(self): with fluid.dygraph.guard(fluid.CPUPlace()): paddle.framework.seed(2020) activation = "relu" normalize_before = False - batch_size, d_model, n_head, dim_feedforward, dropout, attn_dropout, act_dropout, source_length, target_length = generate_basic_params( - mode="decoder_layer") - tgt = np.random.rand(batch_size, target_length, - d_model).astype("float32") - memory = np.random.rand(batch_size, source_length, - d_model).astype("float32") - tgt_mask = np.zeros((batch_size, n_head, target_length, - target_length)).astype("float32") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + attn_dropout, + act_dropout, + source_length, + target_length, + ) = generate_basic_params(mode="decoder_layer") + tgt = np.random.rand(batch_size, target_length, d_model).astype( + "float32" + ) + memory = np.random.rand(batch_size, source_length, d_model).astype( + "float32" + ) + tgt_mask = np.zeros( + (batch_size, n_head, target_length, target_length) + ).astype("float32") tgt_mask[0][0][0][0] = -1e9 - memory_mask = np.zeros((batch_size, n_head, target_length, - source_length)).astype("float32") + memory_mask = np.zeros( + (batch_size, n_head, target_length, source_length) + ).astype("float32") memory_mask[0][0][0][0] = -1e9 for cache in [True, False]: - self_attn = MultiHeadAttention(d_model, - n_head, - dropout=attn_dropout) - cross_attn = MultiHeadAttention(d_model, - n_head, - dropout=attn_dropout) + self_attn = MultiHeadAttention( + d_model, n_head, dropout=attn_dropout + ) + cross_attn = MultiHeadAttention( + d_model, n_head, dropout=attn_dropout + ) # paddle decoderlayer: decoder_layer = TransformerDecoderLayer( - d_model, n_head, dim_feedforward, dropout, activation, - attn_dropout, act_dropout, normalize_before) + d_model, + n_head, + dim_feedforward, + dropout, + activation, + attn_dropout, + act_dropout, + normalize_before, + ) cache_objs = None if cache: cache_objs = decoder_layer.gen_cache( - paddle.to_tensor(memory)) - - decoder_output = decoder_layer(paddle.to_tensor(tgt), - paddle.to_tensor(memory), - paddle.to_tensor(tgt_mask), - paddle.to_tensor(memory_mask), - cache_objs) - - decoder_output = decoder_output[0].numpy( - ) if cache else decoder_output.numpy() + paddle.to_tensor(memory) + ) + + decoder_output = decoder_layer( + paddle.to_tensor(tgt), + paddle.to_tensor(memory), + paddle.to_tensor(tgt_mask), + paddle.to_tensor(memory_mask), + cache_objs, + ) + + decoder_output = ( + decoder_output[0].numpy() + if cache + else decoder_output.numpy() + ) # numpy: residual = tgt # self-attn - self_attn_cache = cache_objs[ - 0] if cache_objs is not None else None - tgt = self_attn(paddle.to_tensor(tgt), paddle.to_tensor(tgt), - paddle.to_tensor(tgt), - paddle.to_tensor(tgt_mask), self_attn_cache) + self_attn_cache = ( + cache_objs[0] if cache_objs is not None else None + ) + tgt = self_attn( + paddle.to_tensor(tgt), + paddle.to_tensor(tgt), + paddle.to_tensor(tgt), + paddle.to_tensor(tgt_mask), + self_attn_cache, + ) tgt = tgt[0].numpy() if cache else tgt.numpy() @@ -460,13 +634,16 @@ def test_transformer_decoder_layer(self): tgt_norm = layer_norm(tgt, d_model, decoder_layer.norm1) residual = tgt_norm # cross-attn - cross_attn_cache = cache_objs[ - 1] if cache_objs is not None else None - tgt = cross_attn(paddle.to_tensor(tgt_norm), - paddle.to_tensor(memory), - paddle.to_tensor(memory), - paddle.to_tensor(memory_mask), - cross_attn_cache) + cross_attn_cache = ( + cache_objs[1] if cache_objs is not None else None + ) + tgt = cross_attn( + paddle.to_tensor(tgt_norm), + paddle.to_tensor(memory), + paddle.to_tensor(memory), + paddle.to_tensor(memory_mask), + cross_attn_cache, + ) tgt = tgt[0].numpy() if cache else tgt.numpy() # postprocess @@ -479,46 +656,67 @@ def test_transformer_decoder_layer(self): tgt = residual + ffn_output tgt_norm = layer_norm(tgt, d_model, decoder_layer.norm3) - np.testing.assert_allclose(decoder_output, - tgt_norm, - rtol=1e-5, - atol=1e-6) + np.testing.assert_allclose( + decoder_output, tgt_norm, rtol=1e-5, atol=1e-6 + ) def test_encoder(self): - batch_size, d_model, n_head, dim_feedforward, dropout, attn_dropout, act_dropout, sequence_length = generate_basic_params( - mode="encoder_layer") - - src = np.random.rand(batch_size, sequence_length, - d_model).astype("float32") - - src_mask = np.zeros((batch_size, n_head, sequence_length, - sequence_length)).astype("float32") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + attn_dropout, + act_dropout, + sequence_length, + ) = generate_basic_params(mode="encoder_layer") + + src = np.random.rand(batch_size, sequence_length, d_model).astype( + "float32" + ) + + src_mask = np.zeros( + (batch_size, n_head, sequence_length, sequence_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf with fluid.dygraph.guard(fluid.CPUPlace()): - encoder_layer = TransformerEncoderLayer(d_model, n_head, - dim_feedforward, dropout) + encoder_layer = TransformerEncoderLayer( + d_model, n_head, dim_feedforward, dropout + ) num_layers = 6 encoder = TransformerEncoder(encoder_layer, num_layers) # src, src_mask - enc_output = encoder(paddle.to_tensor(src), - paddle.to_tensor(src_mask)) + enc_output = encoder( + paddle.to_tensor(src), paddle.to_tensor(src_mask) + ) def test_encoder_attr_1(self): - batch_size, d_model, n_head, dim_feedforward, dropout, attn_dropout, act_dropout, sequence_length = generate_basic_params( - mode="encoder_layer") - - src = np.random.rand(batch_size, sequence_length, - d_model).astype("float32") - - src_mask = np.zeros((batch_size, n_head, sequence_length, - sequence_length)).astype("float32") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + attn_dropout, + act_dropout, + sequence_length, + ) = generate_basic_params(mode="encoder_layer") + + src = np.random.rand(batch_size, sequence_length, d_model).astype( + "float32" + ) + + src_mask = np.zeros( + (batch_size, n_head, sequence_length, sequence_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf with fluid.dygraph.guard(fluid.CPUPlace()): for cache in [True, False]: # paddle - encoder_layer = TransformerEncoderLayer(d_model, n_head, - dim_feedforward, - dropout) + encoder_layer = TransformerEncoderLayer( + d_model, n_head, dim_feedforward, dropout + ) num_layers = 6 encoder = TransformerEncoder(encoder_layer, num_layers) cache_objs = None @@ -526,200 +724,320 @@ def test_encoder_attr_1(self): cache_objs = encoder.gen_cache(paddle.to_tensor(src)) # src, src_mask - enc_output = encoder(paddle.to_tensor(src), - paddle.to_tensor(src_mask), cache_objs) + enc_output = encoder( + paddle.to_tensor(src), + paddle.to_tensor(src_mask), + cache_objs, + ) def test_decoder(self): - batch_size, d_model, n_head, dim_feedforward, dropout, _, _, source_length, target_length = generate_basic_params( - mode="decoder_layer") - tgt = np.random.rand(batch_size, target_length, - d_model).astype("float32") - memory = np.random.rand(batch_size, source_length, - d_model).astype("float32") - tgt_mask = np.zeros((batch_size, n_head, target_length, - target_length)).astype("float32") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + _, + _, + source_length, + target_length, + ) = generate_basic_params(mode="decoder_layer") + tgt = np.random.rand(batch_size, target_length, d_model).astype( + "float32" + ) + memory = np.random.rand(batch_size, source_length, d_model).astype( + "float32" + ) + tgt_mask = np.zeros( + (batch_size, n_head, target_length, target_length) + ).astype("float32") tgt_mask[0][0][0][0] = -1e9 - memory_mask = np.zeros((batch_size, n_head, target_length, - source_length)).astype("float32") + memory_mask = np.zeros( + (batch_size, n_head, target_length, source_length) + ).astype("float32") memory_mask[0][0][0][0] = -1e9 with fluid.dygraph.guard(fluid.CPUPlace()): - decoder_layer = TransformerDecoderLayer(d_model, n_head, - dim_feedforward, dropout) + decoder_layer = TransformerDecoderLayer( + d_model, n_head, dim_feedforward, dropout + ) num_layers = 6 decoder = TransformerDecoder(decoder_layer, num_layers) - output = decoder(paddle.to_tensor(tgt), paddle.to_tensor(memory), - paddle.to_tensor(tgt_mask), - paddle.to_tensor(memory_mask)) + output = decoder( + paddle.to_tensor(tgt), + paddle.to_tensor(memory), + paddle.to_tensor(tgt_mask), + paddle.to_tensor(memory_mask), + ) def test_transformer(self): - batch_size, d_model, n_head, dim_feedforward, dropout, _, _, source_length, target_length = generate_basic_params( - mode="decoder_layer") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + _, + _, + source_length, + target_length, + ) = generate_basic_params(mode="decoder_layer") # batch_size, source_length, target_length, d_model, n_head = 4, 8, 8, 64, 8 with fluid.dygraph.guard(fluid.CPUPlace()): - transformer = Transformer(d_model, - n_head, - dim_feedforward=dim_feedforward, - dropout=dropout) + transformer = Transformer( + d_model, + n_head, + dim_feedforward=dim_feedforward, + dropout=dropout, + ) src = paddle.to_tensor( - np.random.rand(batch_size, source_length, - d_model).astype("float32")) + np.random.rand(batch_size, source_length, d_model).astype( + "float32" + ) + ) tgt = paddle.to_tensor( - np.random.rand(batch_size, target_length, - d_model).astype("float32")) - src_mask = np.zeros((batch_size, n_head, source_length, - source_length)).astype("float32") + np.random.rand(batch_size, target_length, d_model).astype( + "float32" + ) + ) + src_mask = np.zeros( + (batch_size, n_head, source_length, source_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf src_mask = paddle.to_tensor(src_mask) - tgt_mask = np.zeros((batch_size, n_head, target_length, - target_length)).astype("float32") + tgt_mask = np.zeros( + (batch_size, n_head, target_length, target_length) + ).astype("float32") tgt_mask[0][0][0][0] = -1e9 - memory_mask = np.zeros((batch_size, n_head, target_length, - source_length)).astype("float32") + memory_mask = np.zeros( + (batch_size, n_head, target_length, source_length) + ).astype("float32") memory_mask[0][0][0][0] = -1e9 tgt_mask, memory_mask = paddle.to_tensor( - tgt_mask), paddle.to_tensor(memory_mask) - trans_output = transformer(src, tgt, src_mask, tgt_mask, - memory_mask) + tgt_mask + ), paddle.to_tensor(memory_mask) + trans_output = transformer( + src, tgt, src_mask, tgt_mask, memory_mask + ) def test_transformer_attr_1(self): - batch_size, d_model, n_head, dim_feedforward, dropout, _, _, source_length, target_length = generate_basic_params( - mode="decoder_layer") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + _, + _, + source_length, + target_length, + ) = generate_basic_params(mode="decoder_layer") # batch_size, source_length, target_length, d_model, n_head = 4, 8, 8, 64, 8 with fluid.dygraph.guard(fluid.CPUPlace()): - transformer = Transformer(d_model, - n_head, - dim_feedforward=dim_feedforward, - dropout=dropout, - weight_attr=[None], - bias_attr=[False]) + transformer = Transformer( + d_model, + n_head, + dim_feedforward=dim_feedforward, + dropout=dropout, + weight_attr=[None], + bias_attr=[False], + ) src = paddle.to_tensor( - np.random.rand(batch_size, source_length, - d_model).astype("float32")) + np.random.rand(batch_size, source_length, d_model).astype( + "float32" + ) + ) tgt = paddle.to_tensor( - np.random.rand(batch_size, target_length, - d_model).astype("float32")) - src_mask = np.zeros((batch_size, n_head, source_length, - source_length)).astype("float32") + np.random.rand(batch_size, target_length, d_model).astype( + "float32" + ) + ) + src_mask = np.zeros( + (batch_size, n_head, source_length, source_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf src_mask = paddle.to_tensor(src_mask) - tgt_mask = np.zeros((batch_size, n_head, target_length, - target_length)).astype("float32") + tgt_mask = np.zeros( + (batch_size, n_head, target_length, target_length) + ).astype("float32") tgt_mask[0][0][0][0] = -1e9 - memory_mask = np.zeros((batch_size, n_head, target_length, - source_length)).astype("float32") + memory_mask = np.zeros( + (batch_size, n_head, target_length, source_length) + ).astype("float32") memory_mask[0][0][0][0] = -1e9 tgt_mask, memory_mask = paddle.to_tensor( - tgt_mask), paddle.to_tensor(memory_mask) - trans_output = transformer(src, tgt, src_mask, tgt_mask, - memory_mask) + tgt_mask + ), paddle.to_tensor(memory_mask) + trans_output = transformer( + src, tgt, src_mask, tgt_mask, memory_mask + ) def test_transformer_attr_2(self): - batch_size, d_model, n_head, dim_feedforward, dropout, _, _, source_length, target_length = generate_basic_params( - mode="decoder_layer") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + _, + _, + source_length, + target_length, + ) = generate_basic_params(mode="decoder_layer") # batch_size, source_length, target_length, d_model, n_head = 4, 8, 8, 64, 8 with fluid.dygraph.guard(fluid.CPUPlace()): - transformer = Transformer(d_model, - n_head, - dim_feedforward=dim_feedforward, - dropout=dropout, - weight_attr=[None, None], - bias_attr=[False, False]) + transformer = Transformer( + d_model, + n_head, + dim_feedforward=dim_feedforward, + dropout=dropout, + weight_attr=[None, None], + bias_attr=[False, False], + ) src = paddle.to_tensor( - np.random.rand(batch_size, source_length, - d_model).astype("float32")) + np.random.rand(batch_size, source_length, d_model).astype( + "float32" + ) + ) tgt = paddle.to_tensor( - np.random.rand(batch_size, target_length, - d_model).astype("float32")) - src_mask = np.zeros((batch_size, n_head, source_length, - source_length)).astype("float32") + np.random.rand(batch_size, target_length, d_model).astype( + "float32" + ) + ) + src_mask = np.zeros( + (batch_size, n_head, source_length, source_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf src_mask = paddle.to_tensor(src_mask) - tgt_mask = np.zeros((batch_size, n_head, target_length, - target_length)).astype("float32") + tgt_mask = np.zeros( + (batch_size, n_head, target_length, target_length) + ).astype("float32") tgt_mask[0][0][0][0] = -1e9 - memory_mask = np.zeros((batch_size, n_head, target_length, - source_length)).astype("float32") + memory_mask = np.zeros( + (batch_size, n_head, target_length, source_length) + ).astype("float32") memory_mask[0][0][0][0] = -1e9 tgt_mask, memory_mask = paddle.to_tensor( - tgt_mask), paddle.to_tensor(memory_mask) - trans_output = transformer(src, tgt, src_mask, tgt_mask, - memory_mask) + tgt_mask + ), paddle.to_tensor(memory_mask) + trans_output = transformer( + src, tgt, src_mask, tgt_mask, memory_mask + ) def test_transformer_attr_3(self): - batch_size, d_model, n_head, dim_feedforward, dropout, _, _, source_length, target_length = generate_basic_params( - mode="decoder_layer") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + _, + _, + source_length, + target_length, + ) = generate_basic_params(mode="decoder_layer") # batch_size, source_length, target_length, d_model, n_head = 4, 8, 8, 64, 8 with fluid.dygraph.guard(fluid.CPUPlace()): - transformer = Transformer(d_model, - n_head, - dim_feedforward=dim_feedforward, - dropout=dropout, - weight_attr=[None, None, None], - bias_attr=[False, False, True]) + transformer = Transformer( + d_model, + n_head, + dim_feedforward=dim_feedforward, + dropout=dropout, + weight_attr=[None, None, None], + bias_attr=[False, False, True], + ) src = paddle.to_tensor( - np.random.rand(batch_size, source_length, - d_model).astype("float32")) + np.random.rand(batch_size, source_length, d_model).astype( + "float32" + ) + ) tgt = paddle.to_tensor( - np.random.rand(batch_size, target_length, - d_model).astype("float32")) - src_mask = np.zeros((batch_size, n_head, source_length, - source_length)).astype("float32") + np.random.rand(batch_size, target_length, d_model).astype( + "float32" + ) + ) + src_mask = np.zeros( + (batch_size, n_head, source_length, source_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf src_mask = paddle.to_tensor(src_mask) - tgt_mask = np.zeros((batch_size, n_head, target_length, - target_length)).astype("float32") + tgt_mask = np.zeros( + (batch_size, n_head, target_length, target_length) + ).astype("float32") tgt_mask[0][0][0][0] = -1e9 - memory_mask = np.zeros((batch_size, n_head, target_length, - source_length)).astype("float32") + memory_mask = np.zeros( + (batch_size, n_head, target_length, source_length) + ).astype("float32") memory_mask[0][0][0][0] = -1e9 tgt_mask, memory_mask = paddle.to_tensor( - tgt_mask), paddle.to_tensor(memory_mask) - trans_output = transformer(src, tgt, src_mask, tgt_mask, - memory_mask) + tgt_mask + ), paddle.to_tensor(memory_mask) + trans_output = transformer( + src, tgt, src_mask, tgt_mask, memory_mask + ) def test_transformer_attr_boolean(self): - batch_size, d_model, n_head, dim_feedforward, dropout, _, _, source_length, target_length = generate_basic_params( - mode="decoder_layer") + ( + batch_size, + d_model, + n_head, + dim_feedforward, + dropout, + _, + _, + source_length, + target_length, + ) = generate_basic_params(mode="decoder_layer") # batch_size, source_length, target_length, d_model, n_head = 4, 8, 8, 64, 8 with fluid.dygraph.guard(fluid.CPUPlace()): - transformer = Transformer(d_model, - n_head, - dim_feedforward=dim_feedforward, - dropout=dropout, - bias_attr=False) + transformer = Transformer( + d_model, + n_head, + dim_feedforward=dim_feedforward, + dropout=dropout, + bias_attr=False, + ) src = paddle.to_tensor( - np.random.rand(batch_size, source_length, - d_model).astype("float32")) + np.random.rand(batch_size, source_length, d_model).astype( + "float32" + ) + ) tgt = paddle.to_tensor( - np.random.rand(batch_size, target_length, - d_model).astype("float32")) - src_mask = np.zeros((batch_size, n_head, source_length, - source_length)).astype("float32") + np.random.rand(batch_size, target_length, d_model).astype( + "float32" + ) + ) + src_mask = np.zeros( + (batch_size, n_head, source_length, source_length) + ).astype("float32") src_mask[0][0][0][0] = -np.inf src_mask = paddle.to_tensor(src_mask) - tgt_mask = np.zeros((batch_size, n_head, target_length, - target_length)).astype("float32") + tgt_mask = np.zeros( + (batch_size, n_head, target_length, target_length) + ).astype("float32") tgt_mask[0][0][0][0] = -1e9 - memory_mask = np.zeros((batch_size, n_head, target_length, - source_length)).astype("float32") + memory_mask = np.zeros( + (batch_size, n_head, target_length, source_length) + ).astype("float32") memory_mask[0][0][0][0] = -1e9 tgt_mask, memory_mask = paddle.to_tensor( - tgt_mask), paddle.to_tensor(memory_mask) - trans_output = transformer(src, tgt, src_mask, tgt_mask, - memory_mask) + tgt_mask + ), paddle.to_tensor(memory_mask) + trans_output = transformer( + src, tgt, src_mask, tgt_mask, memory_mask + ) def test_generate_square_subsequent_mask(self): length = 5 d_model, n_head, dim_feedforward = 8, 4, 64 - transformer = Transformer(d_model, - n_head, - dim_feedforward=dim_feedforward) + transformer = Transformer( + d_model, n_head, dim_feedforward=dim_feedforward + ) mask = transformer.generate_square_subsequent_mask(length) diff --git a/python/paddle/fluid/tests/unittests/test_translated_layer.py b/python/paddle/fluid/tests/unittests/test_translated_layer.py index ef75af2fb6171..2c249b0e5581a 100644 --- a/python/paddle/fluid/tests/unittests/test_translated_layer.py +++ b/python/paddle/fluid/tests/unittests/test_translated_layer.py @@ -31,14 +31,13 @@ # define a random dataset class RandomDataset(paddle.io.Dataset): - def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, idx): np.random.seed(SEED) image = np.random.random([IMAGE_SIZE]).astype('float32') - label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64') + label = np.random.randint(0, CLASS_NUM - 1, (1,)).astype('int64') return image, label def __len__(self): @@ -46,17 +45,18 @@ def __len__(self): class LinearNet(nn.Layer): - def __init__(self): super(LinearNet, self).__init__() self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) self._dropout = paddle.nn.Dropout(p=0.5) - @paddle.jit.to_static(input_spec=[ - paddle.static.InputSpec(shape=[None, IMAGE_SIZE], - dtype='float32', - name='x') - ]) + @paddle.jit.to_static( + input_spec=[ + paddle.static.InputSpec( + shape=[None, IMAGE_SIZE], dtype='float32', name='x' + ) + ] + ) def forward(self, x): return self._linear(x) @@ -69,13 +69,15 @@ def train(layer, loader, loss_fn, opt): loss.backward() opt.step() opt.clear_grad() - print("Epoch {} batch {}: loss = {}".format(epoch_id, batch_id, - np.mean(loss.numpy()))) + print( + "Epoch {} batch {}: loss = {}".format( + epoch_id, batch_id, np.mean(loss.numpy()) + ) + ) return loss class TestTranslatedLayer(unittest.TestCase): - def tearDown(self): self.temp_dir.cleanup() @@ -91,17 +93,20 @@ def setUp(self): # create network self.layer = LinearNet() self.loss_fn = nn.CrossEntropyLoss() - self.sgd = opt.SGD(learning_rate=0.001, - parameters=self.layer.parameters()) + self.sgd = opt.SGD( + learning_rate=0.001, parameters=self.layer.parameters() + ) # create data loader dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) - self.loader = paddle.io.DataLoader(dataset, - places=place, - batch_size=BATCH_SIZE, - shuffle=True, - drop_last=True, - num_workers=0) + self.loader = paddle.io.DataLoader( + dataset, + places=place, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=0, + ) self.temp_dir = tempfile.TemporaryDirectory() @@ -109,8 +114,9 @@ def setUp(self): train(self.layer, self.loader, self.loss_fn, self.sgd) # save - self.model_path = os.path.join(self.temp_dir.name, - './linear.example.model') + self.model_path = os.path.join( + self.temp_dir.name, './linear.example.model' + ) paddle.jit.save(self.layer, self.model_path) def test_inference_and_fine_tuning(self): @@ -142,15 +148,18 @@ def load_and_fine_tuning(self): # fine-tuning translated_layer.train() - sgd = opt.SGD(learning_rate=0.001, - parameters=translated_layer.parameters()) + sgd = opt.SGD( + learning_rate=0.001, parameters=translated_layer.parameters() + ) loss = train(translated_layer, self.loader, self.loss_fn, sgd) np.testing.assert_array_equal( orig_loss.numpy(), loss.numpy(), err_msg='original loss:\n{}\nnew loss:\n{}\n'.format( - orig_loss.numpy(), loss.numpy())) + orig_loss.numpy(), loss.numpy() + ), + ) def test_get_program(self): # load @@ -171,9 +180,9 @@ def test_get_input_spec(self): translated_layer = paddle.jit.load(self.model_path) expect_spec = [ - paddle.static.InputSpec(shape=[None, IMAGE_SIZE], - dtype='float32', - name='x') + paddle.static.InputSpec( + shape=[None, IMAGE_SIZE], dtype='float32', name='x' + ) ] actual_spec = translated_layer._input_spec() @@ -185,9 +194,11 @@ def test_get_output_spec(self): translated_layer = paddle.jit.load(self.model_path) expect_spec = [ - paddle.static.InputSpec(shape=[None, CLASS_NUM], - dtype='float32', - name='translated_layer/scale_0.tmp_1') + paddle.static.InputSpec( + shape=[None, CLASS_NUM], + dtype='float32', + name='translated_layer/scale_0.tmp_1', + ) ] actual_spec = translated_layer._output_spec() diff --git a/python/paddle/fluid/tests/unittests/test_transpose_op.py b/python/paddle/fluid/tests/unittests/test_transpose_op.py index 390208c0cddc6..67c2b8772c5dd 100644 --- a/python/paddle/fluid/tests/unittests/test_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_transpose_op.py @@ -27,7 +27,6 @@ class TestTransposeOp(OpTest): - def setUp(self): self.init_op_type() self.initTestCase() @@ -39,7 +38,7 @@ def setUp(self): } self.outputs = { 'XShape': np.random.random(self.shape).astype("float64"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } def init_op_type(self): @@ -58,84 +57,72 @@ def initTestCase(self): class TestCase0(TestTransposeOp): - def initTestCase(self): - self.shape = (100, ) - self.axis = (0, ) + self.shape = (100,) + self.axis = (0,) class TestCase1(TestTransposeOp): - def initTestCase(self): self.shape = (3, 4, 10) self.axis = (0, 2, 1) class TestCase2(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.axis = (0, 2, 3, 1) class TestCase3(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.axis = (4, 2, 3, 1, 0) class TestCase4(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6, 1) self.axis = (4, 2, 3, 1, 0, 5) class TestCase5(TestTransposeOp): - def initTestCase(self): self.shape = (2, 16, 96) self.axis = (0, 2, 1) class TestCase6(TestTransposeOp): - def initTestCase(self): self.shape = (2, 10, 12, 16) self.axis = (3, 1, 2, 0) class TestCase7(TestTransposeOp): - def initTestCase(self): self.shape = (2, 10, 2, 16) self.axis = (0, 1, 3, 2) class TestCase8(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (0, 1, 3, 2, 4, 5, 6, 7) class TestCase9(TestTransposeOp): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (6, 1, 3, 5, 0, 2, 4, 7) class TestCase_ZeroDim(TestTransposeOp): - def initTestCase(self): self.shape = () self.axis = () class TestAutoTuneTransposeOp(OpTest): - def setUp(self): self.init_op_type() self.initTestCase() @@ -147,7 +134,7 @@ def setUp(self): } self.outputs = { 'XShape': np.random.random(self.shape).astype("float64"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } def initTestCase(self): @@ -170,7 +157,6 @@ def test_check_grad(self): class TestTransposeBF16Op(OpTest): - def setUp(self): self.init_op_type() self.initTestCase() @@ -183,11 +169,10 @@ def setUp(self): 'use_mkldnn': self.use_mkldnn, } self.outputs = { - 'XShape': - convert_float_to_uint16( - np.random.random(self.shape).astype("float32")), - 'Out': - self.inputs['X'].transpose(self.axis) + 'XShape': convert_float_to_uint16( + np.random.random(self.shape).astype("float32") + ), + 'Out': self.inputs['X'].transpose(self.axis), } def init_op_type(self): @@ -206,109 +191,99 @@ def initTestCase(self): class TestTransposeOpBool(TestTransposeOp): - def test_check_grad(self): pass class TestTransposeOpBool1D(TestTransposeOpBool): - def initTestCase(self): - self.shape = (100, ) - self.axis = (0, ) + self.shape = (100,) + self.axis = (0,) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpBool2D(TestTransposeOpBool): - def initTestCase(self): self.shape = (3, 40) self.axis = (1, 0) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpBool3D(TestTransposeOpBool): - def initTestCase(self): self.shape = (3, 4, 10) self.axis = (0, 2, 1) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpBool4D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.axis = (0, 2, 3, 1) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpBool5D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.axis = (4, 2, 3, 1, 0) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpBool6D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6, 1) self.axis = (4, 2, 3, 1, 0, 5) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpBool7D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3) self.axis = (0, 1, 3, 2, 4, 5, 6) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpBool8D(TestTransposeOpBool): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (6, 1, 3, 5, 0, 2, 4, 7) self.inputs = {'X': np.random.random(self.shape).astype("bool")} self.outputs = { 'XShape': np.random.random(self.shape).astype("bool"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } class TestTransposeOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -322,9 +297,9 @@ def test_x_Variable_check(): def test_x_dtype_check(): # the Input(x)'s dtype must be one of [bool, float16, float32, float64, int32, int64] - x1 = fluid.layers.data(name='x1', - shape=[10, 5, 3], - dtype='int8') + x1 = fluid.layers.data( + name='x1', shape=[10, 5, 3], dtype='int8' + ) fluid.layers.transpose(x1, perm=[1, 0, 2]) self.assertRaises(TypeError, test_x_dtype_check) @@ -350,7 +325,6 @@ def test_each_elem_value_check(): class TestTransposeApi(unittest.TestCase): - def test_static_out(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): @@ -360,8 +334,9 @@ def test_static_out(self): place = paddle.CPUPlace() exe = paddle.static.Executor(place) x_np = np.random.random([2, 3, 4]).astype("float32") - result1, result2 = exe.run(feed={"x": x_np}, - fetch_list=[x_trans1, x_trans2]) + result1, result2 = exe.run( + feed={"x": x_np}, fetch_list=[x_trans1, x_trans2] + ) expected_result1 = np.transpose(x_np, [1, 0, 2]) expected_result2 = np.transpose(x_np, (2, 1, 0)) @@ -387,7 +362,6 @@ def test_dygraph_out(self): class TestTAPI(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program()): data = fluid.data(shape=[10], dtype="float64", name="data") @@ -395,7 +369,7 @@ def test_out(self): place = fluid.CPUPlace() exe = fluid.Executor(place) data_np = np.random.random([10]).astype("float64") - result, = exe.run(feed={"data": data_np}, fetch_list=[data_t]) + (result,) = exe.run(feed={"data": data_np}, fetch_list=[data_t]) expected_result = np.transpose(data_np) self.assertEqual((result == expected_result).all(), True) @@ -405,7 +379,7 @@ def test_out(self): place = fluid.CPUPlace() exe = fluid.Executor(place) data_np = np.random.random([10, 5]).astype("float64") - result, = exe.run(feed={"data": data_np}, fetch_list=[data_t]) + (result,) = exe.run(feed={"data": data_np}, fetch_list=[data_t]) expected_result = np.transpose(data_np) self.assertEqual((result == expected_result).all(), True) @@ -415,7 +389,7 @@ def test_out(self): place = fluid.CPUPlace() exe = fluid.Executor(place) data_np = np.random.random([1, 5]).astype("float64") - result, = exe.run(feed={"data": data_np}, fetch_list=[data_t]) + (result,) = exe.run(feed={"data": data_np}, fetch_list=[data_t]) expected_result = np.transpose(data_np) self.assertEqual((result == expected_result).all(), True) @@ -454,7 +428,6 @@ def test_x_dimension_check(): class TestMoveAxis(unittest.TestCase): - def test_moveaxis1(self): x_np = np.random.randn(2, 3, 4, 5, 7) expected = np.moveaxis(x_np, [0, 4, 3, 2], [1, 3, 2, 0]) @@ -497,8 +470,9 @@ def test_moveaxis2(self): def test_moveaxis3(self): paddle.disable_static() - x = paddle.to_tensor([[1 + 1j, -1 - 1j], [1 + 1j, -1 - 1j], - [1 + 1j, -1 - 1j]]) + x = paddle.to_tensor( + [[1 + 1j, -1 - 1j], [1 + 1j, -1 - 1j], [1 + 1j, -1 - 1j]] + ) out = x.moveaxis(0, 1) self.assertEqual(out.shape, [2, 3]) paddle.enable_static() @@ -535,7 +509,6 @@ def test_error(self): class TestTransposeDoubleGradCheck(unittest.TestCase): - def transpose_wrapper(self, x): return paddle.transpose(x[0], [1, 0, 2]) @@ -550,17 +523,13 @@ def func(self, place): out = paddle.transpose(data, [1, 0, 2]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.transpose_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.transpose_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -572,7 +541,6 @@ def test_grad(self): class TestTransposeTripleGradCheck(unittest.TestCase): - def transpose_wrapper(self, x): return paddle.transpose(x[0], [1, 0, 2]) @@ -587,17 +555,13 @@ def func(self, place): out = paddle.transpose(data, [1, 0, 2]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.transpose_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.transpose_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -609,7 +573,6 @@ def test_grad(self): class TestTransposeAPI_ZeroDim(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) diff --git a/python/paddle/fluid/tests/unittests/test_tree_conv_op.py b/python/paddle/fluid/tests/unittests/test_tree_conv_op.py index 114d713b092ac..d4582c3ad2e50 100644 --- a/python/paddle/fluid/tests/unittests/test_tree_conv_op.py +++ b/python/paddle/fluid/tests/unittests/test_tree_conv_op.py @@ -51,7 +51,6 @@ def recurse_helper(node, depth): class TestTreeConvOp(OpTest): - def setUp(self): self.n = 17 self.fea_size = 3 @@ -60,22 +59,51 @@ def setUp(self): self.batch_size = 2 self.num_filters = 1 adj_array = [ - 1, 2, 1, 3, 1, 4, 1, 5, 2, 6, 2, 7, 2, 8, 4, 9, 4, 10, 5, 11, 6, 12, - 6, 13, 9, 14, 9, 15, 9, 16, 9, 17 + 1, + 2, + 1, + 3, + 1, + 4, + 1, + 5, + 2, + 6, + 2, + 7, + 2, + 8, + 4, + 9, + 4, + 10, + 5, + 11, + 6, + 12, + 6, + 13, + 9, + 14, + 9, + 15, + 9, + 16, + 9, + 17, ] adj = np.array(adj_array).reshape((1, self.n - 1, 2)).astype('int32') adj = np.tile(adj, (self.batch_size, 1, 1)) self.op_type = 'tree_conv' vectors = np.random.random( - (self.batch_size, self.n, self.fea_size)).astype('float64') + (self.batch_size, self.n, self.fea_size) + ).astype('float64') self.inputs = { - 'EdgeSet': - adj, - 'NodesVector': - vectors, - 'Filter': - np.random.random((self.fea_size, 3, self.output_size, - self.num_filters)).astype('float64') + 'EdgeSet': adj, + 'NodesVector': vectors, + 'Filter': np.random.random( + (self.fea_size, 3, self.output_size, self.num_filters) + ).astype('float64'), } self.attrs = {'max_depth': self.max_depth} vectors = [] @@ -90,9 +118,9 @@ def test_check_output(self): self.check_output() def test_check_grad(self): - self.check_grad(['NodesVector', 'Filter'], - 'Out', - max_relative_error=0.5) + self.check_grad( + ['NodesVector', 'Filter'], 'Out', max_relative_error=0.5 + ) def get_output_naive(self, batch_id): og = [[] for i in range(1, self.n + 2)] @@ -107,71 +135,88 @@ def get_output_naive(self, batch_id): result = np.zeros((1, W.shape[2], W.shape[3])) for v in patch: eta_t = float(v[4] - v[3]) / float(v[4]) - eta_l = (1.0 - eta_t) * (0.5 if v[2] == 1 else - float(v[1] - 1.0) / float(v[2] - 1.0)) + eta_l = (1.0 - eta_t) * ( + 0.5 if v[2] == 1 else float(v[1] - 1.0) / float(v[2] - 1.0) + ) eta_r = (1.0 - eta_t) * (1.0 - eta_l) x = self.inputs['NodesVector'][batch_id][v[0] - 1] - eta = np.array([eta_l, eta_r, eta_t]).reshape( - (3, 1)).astype('float64') + eta = ( + np.array([eta_l, eta_r, eta_t]) + .reshape((3, 1)) + .astype('float64') + ) Wconvi = np.tensordot(eta, W, axes=([0], [0])) x = np.array(x).reshape((1, 1, self.fea_size)) res = np.tensordot(x, Wconvi, axes=2) result = result + res vec.append(result) vec = np.concatenate(vec, axis=0) - vec = np.concatenate([ - vec, - np.zeros((self.n - vec.shape[0], W.shape[2], W.shape[3]), - dtype='float64') - ], - axis=0) + vec = np.concatenate( + [ + vec, + np.zeros( + (self.n - vec.shape[0], W.shape[2], W.shape[3]), + dtype='float64', + ), + ], + axis=0, + ) return vec class TestTreeConv_OpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): nodes_vector_1 = np.random.random((10, 5)).astype("float32") - edge_set_1 = fluid.layers.data(name='edge_set_1', - shape=[10, 2], - dtype='float32') + edge_set_1 = fluid.layers.data( + name='edge_set_1', shape=[10, 2], dtype='float32' + ) # the nodes_vector of tree_conv must be Variable. - self.assertRaises(TypeError, fluid.contrib.layers.tree_conv, - nodes_vector_1, edge_set_1, 3) - - nodes_vector_2 = fluid.layers.data(name='vectors2', - shape=[10, 5], - dtype='float32') + self.assertRaises( + TypeError, + fluid.contrib.layers.tree_conv, + nodes_vector_1, + edge_set_1, + 3, + ) + + nodes_vector_2 = fluid.layers.data( + name='vectors2', shape=[10, 5], dtype='float32' + ) edge_set_2 = np.random.random((10, 2)).astype("float32") # the edge_set of tree_conv must be Variable. - self.assertRaises(TypeError, fluid.contrib.layers.tree_conv, - nodes_vector_2, edge_set_2, 3) + self.assertRaises( + TypeError, + fluid.contrib.layers.tree_conv, + nodes_vector_2, + edge_set_2, + 3, + ) class TestDygraphTreeConv_OpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - TreeConv = fluid.dygraph.nn.TreeConv(feature_size=5, - output_size=6, - num_filters=1, - max_depth=2) + TreeConv = fluid.dygraph.nn.TreeConv( + feature_size=5, output_size=6, num_filters=1, max_depth=2 + ) nodes_vector_1 = np.random.random((10, 5)).astype("float32") - edge_set_1 = fluid.layers.data(name='edge_set_1', - shape=[10, 2], - dtype='float32') + edge_set_1 = fluid.layers.data( + name='edge_set_1', shape=[10, 2], dtype='float32' + ) # the nodes_vector of TreeConv must be Variable. - self.assertRaises(TypeError, TreeConv, nodes_vector_1, edge_set_1, - 3) + self.assertRaises( + TypeError, TreeConv, nodes_vector_1, edge_set_1, 3 + ) - nodes_vector_2 = fluid.layers.data(name='vectors2', - shape=[10, 5], - dtype='float32') + nodes_vector_2 = fluid.layers.data( + name='vectors2', shape=[10, 5], dtype='float32' + ) edge_set_2 = np.random.random((10, 2)).astype("float32") # the edge_set of TreeConv must be Variable. - self.assertRaises(TypeError, TreeConv, nodes_vector_2, edge_set_2, - 3) + self.assertRaises( + TypeError, TreeConv, nodes_vector_2, edge_set_2, 3 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_triangular_solve_op.py b/python/paddle/fluid/tests/unittests/test_triangular_solve_op.py index fabef08b1891e..c71619e313742 100644 --- a/python/paddle/fluid/tests/unittests/test_triangular_solve_op.py +++ b/python/paddle/fluid/tests/unittests/test_triangular_solve_op.py @@ -41,8 +41,9 @@ def config(self): self.dtype = "float64" def set_output(self): - self.output = np.linalg.solve(np.triu(self.inputs['X']), - self.inputs['Y']) + self.output = np.linalg.solve( + np.triu(self.inputs['X']), self.inputs['Y'] + ) def setUp(self): self.op_type = "triangular_solve" @@ -51,7 +52,7 @@ def setUp(self): self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype), - 'Y': np.random.random(self.y_shape).astype(self.dtype) + 'Y': np.random.random(self.y_shape).astype(self.dtype), } self.attrs = { 'upper': self.upper, @@ -144,24 +145,26 @@ def config(self): def set_output(self): x = np.triu(self.inputs['X']) - np.fill_diagonal(x, 1.) + np.fill_diagonal(x, 1.0) y = self.inputs['Y'] self.output = np.linalg.solve(x, y) def test_check_grad_normal(self): x = np.triu(self.inputs['X']) - np.fill_diagonal(x, 1.) + np.fill_diagonal(x, 1.0) grad_out = np.ones([10, 10]).astype('float64') grad_y = np.linalg.solve(x.transpose(1, 0), grad_out) grad_x = -np.matmul(grad_y, self.output.transpose(1, 0)) grad_x = np.triu(grad_x) - np.fill_diagonal(grad_x, 0.) + np.fill_diagonal(grad_x, 0.0) - self.check_grad(['X', 'Y'], - 'Out', - user_defined_grads=[grad_x, grad_y], - user_defined_grad_outputs=[grad_out]) + self.check_grad( + ['X', 'Y'], + 'Out', + user_defined_grads=[grad_x, grad_y], + user_defined_grad_outputs=[grad_out], + ) # 4D(broadcast) + 4D(broadcast) @@ -245,7 +248,6 @@ def set_output(self): class TestTriangularSolveAPI(unittest.TestCase): - def setUp(self): np.random.seed(2021) self.place = [paddle.CPUPlace()] @@ -264,12 +266,11 @@ def check_static_result(self, place): z_np = np.linalg.solve(np.triu(x_np), y_np) exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={ - "x": x_np, - "y": y_np - }, - fetch_list=[z]) + fetches = exe.run( + fluid.default_main_program(), + feed={"x": x_np, "y": y_np}, + fetch_list=[z], + ) np.testing.assert_allclose(fetches[0], z_np, rtol=1e-05) def test_static(self): @@ -277,7 +278,6 @@ def test_static(self): self.check_static_result(place=place) def test_dygraph(self): - def run(place): paddle.disable_static(place) x_np = np.random.random([3, 3]).astype(self.dtype) @@ -297,14 +297,15 @@ def run(place): class TestTriangularSolveOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of solve_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) - y1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) + y1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.CPUPlace() + ) self.assertRaises(TypeError, paddle.linalg.triangular_solve, x1, y1) # The data type of input must be float32 or float64. @@ -323,20 +324,23 @@ def test_errors(self): # The number of dimensions of input'X must be >= 2. x5 = fluid.data(name="x5", shape=[30], dtype="float64") y5 = fluid.data(name="y5", shape=[30, 30], dtype="float64") - self.assertRaises(ValueError, paddle.linalg.triangular_solve, x5, - y5) + self.assertRaises( + ValueError, paddle.linalg.triangular_solve, x5, y5 + ) # The number of dimensions of input'Y must be >= 2. x6 = fluid.data(name="x6", shape=[30, 30], dtype="float64") y6 = fluid.data(name="y6", shape=[30], dtype="float64") - self.assertRaises(ValueError, paddle.linalg.triangular_solve, x6, - y6) + self.assertRaises( + ValueError, paddle.linalg.triangular_solve, x6, y6 + ) # The inner-most 2 dimensions of input'X should be equal to each other x7 = fluid.data(name="x7", shape=[2, 3, 4], dtype="float64") y7 = fluid.data(name="y7", shape=[2, 4, 3], dtype="float64") - self.assertRaises(ValueError, paddle.linalg.triangular_solve, x7, - y7) + self.assertRaises( + ValueError, paddle.linalg.triangular_solve, x7, y7 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_tril_indices_op.py b/python/paddle/fluid/tests/unittests/test_tril_indices_op.py index fae0221240c19..0d18b32304ead 100644 --- a/python/paddle/fluid/tests/unittests/test_tril_indices_op.py +++ b/python/paddle/fluid/tests/unittests/test_tril_indices_op.py @@ -21,7 +21,6 @@ class TestTrilIndicesOp(OpTest): - def setUp(self): self.op_type = "tril_indices" self.inputs = {} @@ -34,13 +33,13 @@ def test_check_output(self): def init_config(self): self.attrs = {'rows': 4, 'cols': 4, 'offset': -1} - self.target = np.tril_indices(self.attrs['rows'], self.attrs['offset'], - self.attrs['cols']) + self.target = np.tril_indices( + self.attrs['rows'], self.attrs['offset'], self.attrs['cols'] + ) self.target = np.array(self.target) class TestTrilIndicesOpCase1(TestTrilIndicesOp): - def init_config(self): self.attrs = {'rows': 0, 'cols': 0, 'offset': 0} self.target = np.tril_indices(0, 0, 0) @@ -48,37 +47,40 @@ def init_config(self): class TestTrilIndicesOpCase2(TestTrilIndicesOp): - def init_config(self): self.attrs = {'rows': 4, 'cols': 4, 'offset': 2} - self.target = np.tril_indices(self.attrs['rows'], self.attrs['offset'], - self.attrs['cols']) + self.target = np.tril_indices( + self.attrs['rows'], self.attrs['offset'], self.attrs['cols'] + ) self.target = np.array(self.target) class TestTrilIndicesAPICaseStatic(unittest.TestCase): - def test_static(self): - places = [ - paddle.CPUPlace(), paddle.fluid.CUDAPlace(0) - ] if fluid.core.is_compiled_with_cuda() else [paddle.CPUPlace()] + places = ( + [paddle.CPUPlace(), paddle.fluid.CUDAPlace(0)] + if fluid.core.is_compiled_with_cuda() + else [paddle.CPUPlace()] + ) paddle.enable_static() for place in places: - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data1 = paddle.tril_indices(4, 4, -1) exe1 = paddle.static.Executor(place) - result1, = exe1.run(feed={}, fetch_list=[data1]) + (result1,) = exe1.run(feed={}, fetch_list=[data1]) expected_result1 = np.tril_indices(4, -1, 4) np.testing.assert_allclose(result1, expected_result1, rtol=1e-05) class TestTrilIndicesAPICaseDygraph(unittest.TestCase): - def test_dygraph(self): - places = [ - paddle.CPUPlace(), paddle.fluid.CUDAPlace(0) - ] if fluid.core.is_compiled_with_cuda() else [paddle.CPUPlace()] + places = ( + [paddle.CPUPlace(), paddle.fluid.CUDAPlace(0)] + if fluid.core.is_compiled_with_cuda() + else [paddle.CPUPlace()] + ) for place in places: with fluid.dygraph.base.guard(place=place): out1 = paddle.tril_indices(4, 4, 2) @@ -91,9 +93,7 @@ def test_dygraph_eager(self): class TestTrilIndicesAPICaseError(unittest.TestCase): - def test_case_error(self): - def test_num_rows_type_check(): out1 = paddle.tril_indices(1.0, 1, 2) @@ -111,14 +111,14 @@ def test_num_offset_type_check(): class TestTrilIndicesAPICaseDefault(unittest.TestCase): - def test_default_CPU(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.tril_indices(4, None, 2) exe = paddle.static.Executor(paddle.CPUPlace()) - result, = exe.run(feed={}, fetch_list=[data]) + (result,) = exe.run(feed={}, fetch_list=[data]) expected_result = np.tril_indices(4, 2) np.testing.assert_allclose(result, expected_result, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/test_tril_triu_op.py b/python/paddle/fluid/tests/unittests/test_tril_triu_op.py index 07cb0aca214f0..a17a1f24928ed 100644 --- a/python/paddle/fluid/tests/unittests/test_tril_triu_op.py +++ b/python/paddle/fluid/tests/unittests/test_tril_triu_op.py @@ -21,12 +21,13 @@ class TrilTriuOpDefaultTest(OpTest): - """ the base class of other op testcases - """ + """the base class of other op testcases""" def setUp(self): self.initTestCase() - self.python_api = paddle.tril if self.real_op_type == 'tril' else paddle.triu + self.python_api = ( + paddle.tril if self.real_op_type == 'tril' else paddle.triu + ) self.real_np_op = getattr(np, self.real_op_type) self.op_type = "tril_triu" @@ -36,9 +37,9 @@ def setUp(self): 'lower': True if self.real_op_type == 'tril' else False, } self.outputs = { - 'Out': - self.real_np_op(self.X, self.diagonal) - if self.diagonal else self.real_np_op(self.X) + 'Out': self.real_np_op(self.X, self.diagonal) + if self.diagonal + else self.real_np_op(self.X) } def test_check_output(self): @@ -59,27 +60,29 @@ def case_generator(op_type, Xshape, diagonal, expected): If arg`expercted` is 'success', it will register an Optest case and expect to pass. Otherwise, it will register an API case and check the expect failure. """ - cls_name = "{0}_{1}_shape_{2}_diag_{3}".format(expected, op_type, Xshape, - diagonal) + cls_name = "{0}_{1}_shape_{2}_diag_{3}".format( + expected, op_type, Xshape, diagonal + ) errmsg = { - "diagonal: TypeError": - "diagonal in {} must be a python Int".format(op_type), - "input: ValueError": - "x shape in {} must be at least 2-D".format(op_type), + "diagonal: TypeError": "diagonal in {} must be a python Int".format( + op_type + ), + "input: ValueError": "x shape in {} must be at least 2-D".format( + op_type + ), } class FailureCase(unittest.TestCase): - def test_failure(self): paddle.enable_static() data = fluid.data(shape=Xshape, dtype='float64', name=cls_name) - with self.assertRaisesRegexp(eval(expected.split(':')[-1]), - errmsg[expected]): + with self.assertRaisesRegexp( + eval(expected.split(':')[-1]), errmsg[expected] + ): getattr(tensor, op_type)(x=data, diagonal=diagonal) class SuccessCase(TrilTriuOpDefaultTest): - def initTestCase(self): paddle.enable_static() @@ -104,15 +107,13 @@ def initTestCase(self): (20, 20): [ '2020', [20], - { - 20: 20 - }, + {20: 20}, (20, 20), 20.20, ], # str, list, dict, tuple, float }, 'input: ValueError': { - (2020, ): [None], + (2020,): [None], }, } for _op_type in ['tril', 'triu']: @@ -121,12 +122,15 @@ def initTestCase(self): list( map( lambda _diagonal: case_generator( - _op_type, _Xshape, _diagonal, _expected), _diaglist)) + _op_type, _Xshape, _diagonal, _expected + ), + _diaglist, + ) + ) class TestTrilTriuOpAPI(unittest.TestCase): - """ test case by using API and has -1 dimension - """ + """test case by using API and has -1 dimension""" def test_api(self): paddle.enable_static() @@ -140,8 +144,11 @@ def test_api(self): x = fluid.data(shape=[1, 9, -1, 4], dtype=dtype, name='x') tril_out, triu_out = tensor.tril(x), tensor.triu(x) - place = fluid.CUDAPlace(0) if fluid.core.is_compiled_with_cuda( - ) else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) tril_out, triu_out = exe.run( fluid.default_main_program(), @@ -159,8 +166,10 @@ def test_api_with_dygraph(self): with fluid.dygraph.guard(): data = np.random.random([1, 9, 9, 4]).astype(dtype) x = fluid.dygraph.to_variable(data) - tril_out, triu_out = tensor.tril(x).numpy(), tensor.triu( - x).numpy() + tril_out, triu_out = ( + tensor.tril(x).numpy(), + tensor.triu(x).numpy(), + ) np.testing.assert_allclose(tril_out, np.tril(data), rtol=1e-05) np.testing.assert_allclose(triu_out, np.triu(data), rtol=1e-05) @@ -176,12 +185,17 @@ def test_fluid_api(self): x = fluid.data(shape=[1, 9, -1, 4], dtype=dtype, name='x') triu_out = fluid.layers.triu(x) - place = fluid.CUDAPlace(0) if fluid.core.is_compiled_with_cuda( - ) else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if fluid.core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - triu_out = exe.run(fluid.default_main_program(), - feed={"x": data}, - fetch_list=[triu_out]) + triu_out = exe.run( + fluid.default_main_program(), + feed={"x": data}, + fetch_list=[triu_out], + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_trilinear_interp_op.py b/python/paddle/fluid/tests/unittests/test_trilinear_interp_op.py index 2548c0aa2db7d..901d25846613f 100755 --- a/python/paddle/fluid/tests/unittests/test_trilinear_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_trilinear_interp_op.py @@ -20,15 +20,17 @@ from paddle.nn.functional import interpolate -def trilinear_interp_np(input, - out_d, - out_h, - out_w, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCDHW'): +def trilinear_interp_np( + input, + out_d, + out_h, + out_w, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCDHW', +): """trilinear interpolation implement in shape [N, C, D, H, W]""" if data_layout == "NDHWC": input = np.transpose(input, (0, 4, 1, 2, 3)) # NDHWC => NCDHW @@ -44,17 +46,17 @@ def trilinear_interp_np(input, ratio_d = ratio_h = ratio_w = 0.0 if out_d > 1: - if (align_corners): + if align_corners: ratio_d = (in_d - 1.0) / (out_d - 1.0) else: ratio_d = 1.0 * in_d / out_d if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: ratio_w = 1.0 * in_w / out_w @@ -62,14 +64,14 @@ def trilinear_interp_np(input, out = np.zeros((batch_size, channel, out_d, out_h, out_w)) for i in range(out_d): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: d = int(ratio_d * (i + 0.5) - 0.5) else: d = int(ratio_d * i) d = max(0, d) did = 1 if d < in_d - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_d = max(ratio_d * (i + 0.5) - 0.5, 0) d1lambda = idx_src_d - d else: @@ -77,14 +79,14 @@ def trilinear_interp_np(input, d2lambda = 1.0 - d1lambda for j in range(out_h): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: h = int(ratio_h * (j + 0.5) - 0.5) else: h = int(ratio_h * j) h = max(0, h) hid = 1 if h < in_h - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_h = max(ratio_h * (j + 0.5) - 0.5, 0) h1lambda = idx_src_h - h else: @@ -92,30 +94,42 @@ def trilinear_interp_np(input, h2lambda = 1.0 - h1lambda for k in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (k + 0.5) - 0.5) else: w = int(ratio_w * k) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (k + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * k - w w2lambda = 1.0 - w1lambda - out[:, :, i, j, k] = \ - d2lambda * \ - (h2lambda * (w2lambda * input[:, :, d, h, w] + \ - w1lambda * input[:, :, d, h, w+wid]) + \ - h1lambda * (w2lambda * input[:, :, d, h+hid, w] + \ - w1lambda * input[:, :, d, h+hid, w+wid])) + \ - d1lambda * \ - (h2lambda * (w2lambda * input[:, :, d+did, h, w] + \ - w1lambda * input[:, :, d+did, h, w+wid]) + \ - h1lambda * (w2lambda * input[:, :, d+did, h+hid, w] + \ - w1lambda * input[:, :, d+did, h+hid, w+wid])) + out[:, :, i, j, k] = d2lambda * ( + h2lambda + * ( + w2lambda * input[:, :, d, h, w] + + w1lambda * input[:, :, d, h, w + wid] + ) + + h1lambda + * ( + w2lambda * input[:, :, d, h + hid, w] + + w1lambda * input[:, :, d, h + hid, w + wid] + ) + ) + d1lambda * ( + h2lambda + * ( + w2lambda * input[:, :, d + did, h, w] + + w1lambda * input[:, :, d + did, h, w + wid] + ) + + h1lambda + * ( + w2lambda * input[:, :, d + did, h + hid, w] + + w1lambda * input[:, :, d + did, h + hid, w + wid] + ) + ) if data_layout == "NDHWC": out = np.transpose(out, (0, 2, 3, 4, 1)) # NCDHW => NDHWC @@ -123,7 +137,6 @@ def trilinear_interp_np(input, class TestTrilinearInterpOp(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -153,10 +166,17 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode, - self.data_layout) + output_np = trilinear_interp_np( + input_np, + out_d, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -177,7 +197,7 @@ def setUp(self): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': data_layout + 'data_layout': data_layout, } self.outputs = {'Out': output_np} @@ -185,10 +205,9 @@ def test_check_output(self): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'trilinear' @@ -196,142 +215,132 @@ def init_test_case(self): self.out_d = 2 self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3, 3]).astype("int32") self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase1(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 1, 7, 8, 9] self.out_d = 1 self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase2(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 9, 6, 8] self.out_d = 12 self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase3(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 2, 16, 8, 4] self.out_d = 32 self.out_h = 16 self.out_w = 8 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase4(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [4, 1, 7, 8, 9] self.out_d = 1 self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2, 2]).astype("int32") self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase5(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 3, 9, 6, 8] self.out_d = 12 self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11, 11]).astype("int32") self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase6(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [1, 1, 16, 8, 4] self.out_d = 8 self.out_h = 32 self.out_w = 16 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([17, 9, 5]).astype("int32") self.align_corners = True self.align_mode = 1 class TestTrilinearInterpSame(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [1, 1, 16, 8, 4] self.out_d = 16 self.out_h = 8 self.out_w = 4 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpSameHW(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [1, 1, 16, 8, 4] self.out_d = 8 self.out_h = 8 self.out_w = 4 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpActualShape(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 2, 16, 8, 4] self.out_d = 64 self.out_h = 32 self.out_w = 16 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([33, 19, 7]).astype("int32") self.align_corners = True self.align_mode = 1 class TestTrilinearInterpDatalayout(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 4, 4, 4, 3] self.out_d = 2 self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3, 3]).astype("int32") self.align_corners = True self.align_mode = 1 @@ -339,15 +348,15 @@ def init_test_case(self): class TestTrilinearInterpOpUint8(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None self.init_test_case() self.op_type = "trilinear_interp" self.check_eager = True - input_np = np.random.randint(low=0, high=256, - size=self.input_shape).astype("uint8") + input_np = np.random.randint( + low=0, high=256, size=self.input_shape + ).astype("uint8") if self.scale > 0: out_d = int(self.input_shape[2] * self.scale) @@ -358,9 +367,16 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode) + output_np = trilinear_interp_np( + input_np, + out_d, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -373,14 +389,14 @@ def setUp(self): 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } self.outputs = {'Out': output_np} def test_check_output(self): - self.check_output_with_place(place=core.CPUPlace(), - atol=1, - check_eager=self.check_eager) + self.check_output_with_place( + place=core.CPUPlace(), atol=1, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'trilinear' @@ -388,87 +404,79 @@ def init_test_case(self): self.out_d = 13 self.out_h = 10 self.out_w = 9 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase1Uint8(TestTrilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 16, 8, 4] self.out_d = 13 self.out_h = 7 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpCase2Uint8(TestTrilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [4, 1, 7, 8, 9] self.out_d = 3 self.out_h = 5 self.out_w = 13 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([6, 15, 21]).astype("int32") self.align_corners = True self.align_mode = 1 class TestTrilinearInterpOtherMethod1(TestTrilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 1 class TestTrilinearInterpWithMethod2(TestTrilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 0 class TestTrilinearInterpWithMethod3(TestTrilinearInterpOp): - def set_align_mode(self): self.align_corners = True self.align_mode = 0 class TestTrilinearInterpScale1(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 9] self.out_d = 82 self.out_h = 60 self.out_w = 25 - self.scale = 2. + self.scale = 2.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpScale2(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 9] self.out_d = 60 self.out_h = 40 self.out_w = 25 - self.scale = 1. + self.scale = 1.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpScale3(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 9] @@ -481,7 +489,6 @@ def init_test_case(self): class TestTrilinearInterpZero(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 11] @@ -494,7 +501,6 @@ def init_test_case(self): class TestTrilinearInterpOp_attr_tensor(OpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -506,7 +512,7 @@ def setUp(self): self.attrs = { 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } input_np = np.random.random(self.input_shape).astype("float32") @@ -530,27 +536,34 @@ def setUp(self): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.check_eager = False self.attrs['out_d'] = self.out_d self.attrs['out_h'] = self.out_h self.attrs['out_w'] = self.out_w - output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode) + output_np = trilinear_interp_np( + input_np, + out_d, + out_h, + out_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.outputs = {'Out': output_np} def test_check_output(self): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'trilinear' @@ -558,7 +571,7 @@ def init_test_case(self): self.out_d = 2 self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [2, 3, 3] self.align_corners = True self.align_mode = 1 @@ -566,7 +579,6 @@ def init_test_case(self): # out_size is a 1-D tensor class TestTrilinearInterp_attr_tensor_Case1(TestTrilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 2, 9, 6, 8] @@ -581,14 +593,13 @@ def init_test_case(self): # scale is a 1-D tensor class TestTrilinearInterp_attr_tensor_Case2(TestTrilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 8, 8, 4] self.out_d = 16 self.out_h = 12 self.out_w = 4 - self.scale = 0. + self.scale = 0.0 self.out_size = [16, 4, 10] self.align_corners = True self.align_mode = 1 @@ -597,7 +608,6 @@ def init_test_case(self): # scale is a 1-D tensor class TestTrilinearInterp_attr_tensor_Case3(TestTrilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 8, 8, 4] @@ -612,7 +622,6 @@ def init_test_case(self): class TestTrilinearInterpAPI(unittest.TestCase): - def test_case(self): x = fluid.data(name="x", shape=[2, 3, 6, 9, 4], dtype="float32") y = fluid.data(name="y", shape=[2, 6, 9, 4, 3], dtype="float32") @@ -620,31 +629,28 @@ def test_case(self): dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[3], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[3], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) - out1 = fluid.layers.resize_trilinear(y, - out_shape=[12, 18, 8], - data_format='NDHWC') + out1 = fluid.layers.resize_trilinear( + y, out_shape=[12, 18, 8], data_format='NDHWC' + ) out2 = fluid.layers.resize_trilinear(x, out_shape=[12, dim, 8]) out3 = fluid.layers.resize_trilinear(x, out_shape=shape_tensor) - out4 = fluid.layers.resize_trilinear(x, - out_shape=[4, 4, 8], - actual_shape=actual_size) + out4 = fluid.layers.resize_trilinear( + x, out_shape=[4, 4, 8], actual_shape=actual_size + ) out5 = fluid.layers.resize_trilinear(x, scale=scale_tensor) - out6 = interpolate(x, - scale_factor=scale_tensor, - mode='trilinear', - data_format="NCDHW") - out7 = interpolate(x, - size=[4, 4, 8], - mode='trilinear', - data_format="NCDHW") - out8 = interpolate(x, - size=shape_tensor, - mode='trilinear', - data_format="NCDHW") + out6 = interpolate( + x, scale_factor=scale_tensor, mode='trilinear', data_format="NCDHW" + ) + out7 = interpolate( + x, size=[4, 4, 8], mode='trilinear', data_format="NCDHW" + ) + out8 = interpolate( + x, size=shape_tensor, mode='trilinear', data_format="NCDHW" + ) x_data = np.random.random((2, 3, 6, 9, 4)).astype("float32") dim_data = np.array([18]).astype("int32") @@ -658,40 +664,39 @@ def test_case(self): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": np.transpose(x_data, (0, 2, 3, 4, 1)), - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = trilinear_interp_np(x_data, - out_d=12, - out_h=18, - out_w=8, - align_mode=1) - np.testing.assert_allclose(results[0], - np.transpose(expect_res, (0, 2, 3, 4, 1)), - rtol=1e-05) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": np.transpose(x_data, (0, 2, 3, 4, 1)), + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = trilinear_interp_np( + x_data, out_d=12, out_h=18, out_w=8, align_mode=1 + ) + np.testing.assert_allclose( + results[0], np.transpose(expect_res, (0, 2, 3, 4, 1)), rtol=1e-05 + ) for i in range(len(results) - 1): np.testing.assert_allclose(results[i + 1], expect_res, rtol=1e-05) class TestTrilinearInterpOpException(unittest.TestCase): - def test_exception(self): input = fluid.data(name="input", shape=[2, 3, 6, 9, 4], dtype="float32") def attr_data_format(): # for 5-D input, data_format only can be NCDHW or NDHWC - out = fluid.layers.resize_trilinear(input, - out_shape=[4, 8, 4], - data_format='NHWC') + out = fluid.layers.resize_trilinear( + input, out_shape=[4, 8, 4], data_format='NHWC' + ) self.assertRaises(ValueError, attr_data_format) diff --git a/python/paddle/fluid/tests/unittests/test_trilinear_interp_v2_op.py b/python/paddle/fluid/tests/unittests/test_trilinear_interp_v2_op.py index 6b5391aa195df..67ab8805dc190 100755 --- a/python/paddle/fluid/tests/unittests/test_trilinear_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_trilinear_interp_v2_op.py @@ -24,18 +24,20 @@ np.random.seed(123) -def trilinear_interp_test(x, - OutSize=None, - SizeTensor=None, - Scale=None, - data_layout='NCHW', - out_d=-1, - out_h=-1, - out_w=-1, - scale=[], - interp_method='trilinear', - align_corners=True, - align_mode=0): +def trilinear_interp_test( + x, + OutSize=None, + SizeTensor=None, + Scale=None, + data_layout='NCHW', + out_d=-1, + out_h=-1, + out_w=-1, + scale=[], + interp_method='trilinear', + align_corners=True, + align_mode=0, +): if isinstance(scale, float) or isinstance(scale, int): scale_list = [] for _ in range(len(x.shape) - 2): @@ -45,26 +47,39 @@ def trilinear_interp_test(x, scale = list(map(float, scale)) if SizeTensor is not None: if not isinstance(SizeTensor, list) and not isinstance( - SizeTensor, tuple): + SizeTensor, tuple + ): SizeTensor = [SizeTensor] - return paddle._C_ops.trilinear_interp(x, OutSize, SizeTensor, Scale, - data_layout, out_d, out_h, out_w, - scale, interp_method, align_corners, - align_mode) - - -def trilinear_interp_np(input, - out_d, - out_h, - out_w, - scale_d=0, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCDHW'): + return paddle._C_ops.trilinear_interp( + x, + OutSize, + SizeTensor, + Scale, + data_layout, + out_d, + out_h, + out_w, + scale, + interp_method, + align_corners, + align_mode, + ) + + +def trilinear_interp_np( + input, + out_d, + out_h, + out_w, + scale_d=0, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCDHW', +): """trilinear interpolation implement in shape [N, C, D, H, W]""" if data_layout == "NDHWC": input = np.transpose(input, (0, 4, 1, 2, 3)) # NDHWC => NCDHW @@ -80,7 +95,7 @@ def trilinear_interp_np(input, ratio_d = ratio_h = ratio_w = 0.0 if out_d > 1: - if (align_corners): + if align_corners: ratio_d = (in_d - 1.0) / (out_d - 1.0) else: if scale_d > 0: @@ -88,7 +103,7 @@ def trilinear_interp_np(input, else: ratio_d = 1.0 * in_d / out_d if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: @@ -96,7 +111,7 @@ def trilinear_interp_np(input, else: ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -107,14 +122,14 @@ def trilinear_interp_np(input, out = np.zeros((batch_size, channel, out_d, out_h, out_w)) for i in range(out_d): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: d = int(ratio_d * (i + 0.5) - 0.5) else: d = int(ratio_d * i) d = max(0, d) did = 1 if d < in_d - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_d = max(ratio_d * (i + 0.5) - 0.5, 0) d1lambda = idx_src_d - d else: @@ -122,14 +137,14 @@ def trilinear_interp_np(input, d2lambda = 1.0 - d1lambda for j in range(out_h): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: h = int(ratio_h * (j + 0.5) - 0.5) else: h = int(ratio_h * j) h = max(0, h) hid = 1 if h < in_h - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_h = max(ratio_h * (j + 0.5) - 0.5, 0) h1lambda = idx_src_h - h else: @@ -137,30 +152,42 @@ def trilinear_interp_np(input, h2lambda = 1.0 - h1lambda for k in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (k + 0.5) - 0.5) else: w = int(ratio_w * k) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (k + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * k - w w2lambda = 1.0 - w1lambda - out[:, :, i, j, k] = \ - d2lambda * \ - (h2lambda * (w2lambda * input[:, :, d, h, w] + - w1lambda * input[:, :, d, h, w+wid]) + - h1lambda * (w2lambda * input[:, :, d, h+hid, w] + - w1lambda * input[:, :, d, h+hid, w+wid])) + \ - d1lambda * \ - (h2lambda * (w2lambda * input[:, :, d+did, h, w] + - w1lambda * input[:, :, d+did, h, w+wid]) + - h1lambda * (w2lambda * input[:, :, d+did, h+hid, w] + - w1lambda * input[:, :, d+did, h+hid, w+wid])) + out[:, :, i, j, k] = d2lambda * ( + h2lambda + * ( + w2lambda * input[:, :, d, h, w] + + w1lambda * input[:, :, d, h, w + wid] + ) + + h1lambda + * ( + w2lambda * input[:, :, d, h + hid, w] + + w1lambda * input[:, :, d, h + hid, w + wid] + ) + ) + d1lambda * ( + h2lambda + * ( + w2lambda * input[:, :, d + did, h, w] + + w1lambda * input[:, :, d + did, h, w + wid] + ) + + h1lambda + * ( + w2lambda * input[:, :, d + did, h + hid, w] + + w1lambda * input[:, :, d + did, h + hid, w + wid] + ) + ) if data_layout == "NDHWC": out = np.transpose(out, (0, 2, 3, 4, 1)) # NCDHW => NDHWC @@ -168,7 +195,6 @@ def trilinear_interp_np(input, class TestTrilinearInterpOp(OpTest): - def setUp(self): self.python_api = trilinear_interp_test self.out_size = None @@ -211,10 +237,20 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, scale_d, - scale_h, scale_w, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode, self.data_layout) + output_np = trilinear_interp_np( + input_np, + out_d, + out_h, + out_w, + scale_d, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -234,7 +270,7 @@ def setUp(self): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': data_layout + 'data_layout': data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -249,10 +285,9 @@ def test_check_output(self): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'trilinear' @@ -267,7 +302,6 @@ def init_test_case(self): class TestTrilinearInterpCase1(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 1, 7, 8, 9] @@ -280,7 +314,6 @@ def init_test_case(self): class TestTrilinearInterpCase2(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 9, 6, 8] @@ -293,7 +326,6 @@ def init_test_case(self): class TestTrilinearInterpCase3(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 2, 16, 8, 4] @@ -306,7 +338,6 @@ def init_test_case(self): class TestTrilinearInterpCase4(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [4, 1, 7, 8, 9] @@ -320,7 +351,6 @@ def init_test_case(self): class TestTrilinearInterpCase5(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 3, 9, 6, 8] @@ -334,7 +364,6 @@ def init_test_case(self): class TestTrilinearInterpCase6(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [1, 1, 16, 8, 4] @@ -348,7 +377,6 @@ def init_test_case(self): class TestTrilinearInterpSame(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [1, 1, 16, 8, 4] @@ -361,7 +389,6 @@ def init_test_case(self): class TestTrilinearInterpSameHW(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [1, 1, 16, 8, 4] @@ -374,7 +401,6 @@ def init_test_case(self): class TestTrilinearInterpActualShape(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 2, 16, 8, 4] @@ -388,7 +414,6 @@ def init_test_case(self): class TestTrilinearInterpDatalayout(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 4, 4, 4, 3] @@ -403,7 +428,6 @@ def init_test_case(self): class TestTrilinearInterpOpUint8(OpTest): - def setUp(self): self.python_api = trilinear_interp_test self.out_size = None @@ -411,8 +435,9 @@ def setUp(self): self.init_test_case() self.op_type = "trilinear_interp_v2" self.check_eager = True - input_np = np.random.randint(low=0, high=256, - size=self.input_shape).astype("uint8") + input_np = np.random.randint( + low=0, high=256, size=self.input_shape + ).astype("uint8") if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -432,9 +457,19 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, 0, 0, 0, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode) + output_np = trilinear_interp_np( + input_np, + out_d, + out_h, + out_w, + 0, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -446,7 +481,7 @@ def setUp(self): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): @@ -458,9 +493,9 @@ def setUp(self): self.outputs = {'Out': output_np} def test_check_output(self): - self.check_output_with_place(place=core.CPUPlace(), - atol=1, - check_eager=self.check_eager) + self.check_output_with_place( + place=core.CPUPlace(), atol=1, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'trilinear' @@ -474,7 +509,6 @@ def init_test_case(self): class TestTrilinearInterpCase1Uint8(TestTrilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 16, 8, 4] @@ -487,7 +521,6 @@ def init_test_case(self): class TestTrilinearInterpCase2Uint8(TestTrilinearInterpOpUint8): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [4, 1, 7, 8, 9] @@ -501,54 +534,48 @@ def init_test_case(self): class TestTrilinearInterpOtherMethod1(TestTrilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 1 class TestTrilinearInterpWithMethod2(TestTrilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 0 class TestTrilinearInterpWithMethod3(TestTrilinearInterpOp): - def set_align_mode(self): self.align_corners = True self.align_mode = 0 class TestTrilinearInterpScale1(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 9] self.out_d = 19 self.out_h = 15 self.out_w = 8 - self.scale = 2. + self.scale = 2.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpScale2(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 9] self.out_d = 30 self.out_h = 20 self.out_w = 25 - self.scale = 1. + self.scale = 1.0 self.align_corners = True self.align_mode = 1 class TestTrilinearInterpScale3(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 9] @@ -561,7 +588,6 @@ def init_test_case(self): class TestTrilinearInterpZero(TestTrilinearInterpOp): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 5, 7, 11] @@ -574,7 +600,6 @@ def init_test_case(self): class TestTrilinearInterpOp_attr_tensor(OpTest): - def setUp(self): self.python_api = trilinear_interp_test self.out_size = None @@ -587,7 +612,7 @@ def setUp(self): self.attrs = { 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'align_mode': self.align_mode + 'align_mode': self.align_mode, } input_np = np.random.random(self.input_shape).astype("float32") @@ -619,8 +644,9 @@ def setUp(self): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.check_eager = False @@ -634,19 +660,28 @@ def setUp(self): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, 0, 0, 0, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode) + output_np = trilinear_interp_np( + input_np, + out_d, + out_h, + out_w, + 0, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + ) self.outputs = {'Out': output_np} def test_check_output(self): self.check_output(check_eager=self.check_eager) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - in_place=True, - check_eager=self.check_eager) + self.check_grad( + ['X'], 'Out', in_place=True, check_eager=self.check_eager + ) def init_test_case(self): self.interp_method = 'trilinear' @@ -662,7 +697,6 @@ def init_test_case(self): # out_size is a 1-D tensor class TestTrilinearInterp_attr_tensor_Case1(TestTrilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [3, 2, 9, 6, 8] @@ -677,7 +711,6 @@ def init_test_case(self): # scale is a 1-D tensor class TestTrilinearInterp_attr_tensor_Case2(TestTrilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 8, 8, 4] @@ -693,7 +726,6 @@ def init_test_case(self): # scale is a 1-D tensor class TestTrilinearInterp_attr_tensor_Case3(TestTrilinearInterpOp_attr_tensor): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 8, 8, 4] @@ -708,7 +740,6 @@ def init_test_case(self): class TestTrilinearInterpAPI(unittest.TestCase): - def test_imperative_case(self): with _test_eager_guard(): self.func_case() @@ -721,31 +752,28 @@ def func_case(self): dim = fluid.data(name="dim", shape=[1], dtype="int32") shape_tensor = fluid.data(name="shape_tensor", shape=[3], dtype="int32") actual_size = fluid.data(name="actual_size", shape=[3], dtype="int32") - scale_tensor = fluid.data(name="scale_tensor", - shape=[1], - dtype="float32") + scale_tensor = fluid.data( + name="scale_tensor", shape=[1], dtype="float32" + ) - out1 = fluid.layers.resize_trilinear(y, - out_shape=[12, 18, 8], - data_format='NDHWC') + out1 = fluid.layers.resize_trilinear( + y, out_shape=[12, 18, 8], data_format='NDHWC' + ) out2 = fluid.layers.resize_trilinear(x, out_shape=[12, dim, 8]) out3 = fluid.layers.resize_trilinear(x, out_shape=shape_tensor) - out4 = fluid.layers.resize_trilinear(x, - out_shape=[4, 4, 8], - actual_shape=actual_size) + out4 = fluid.layers.resize_trilinear( + x, out_shape=[4, 4, 8], actual_shape=actual_size + ) out5 = fluid.layers.resize_trilinear(x, scale=scale_tensor) - out6 = interpolate(x, - scale_factor=scale_tensor, - mode='trilinear', - data_format="NCDHW") - out7 = interpolate(x, - size=[4, 4, 8], - mode='trilinear', - data_format="NCDHW") - out8 = interpolate(x, - size=shape_tensor, - mode='trilinear', - data_format="NCDHW") + out6 = interpolate( + x, scale_factor=scale_tensor, mode='trilinear', data_format="NCDHW" + ) + out7 = interpolate( + x, size=[4, 4, 8], mode='trilinear', data_format="NCDHW" + ) + out8 = interpolate( + x, size=shape_tensor, mode='trilinear', data_format="NCDHW" + ) x_data = np.random.random((2, 3, 6, 9, 4)).astype("float32") dim_data = np.array([18]).astype("int32") @@ -759,26 +787,26 @@ def func_case(self): place = core.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(fluid.default_main_program(), - feed={ - "x": x_data, - "y": np.transpose(x_data, (0, 2, 3, 4, 1)), - "dim": dim_data, - "shape_tensor": shape_data, - "actual_size": actual_size_data, - "scale_tensor": scale_data - }, - fetch_list=[out1, out2, out3, out4, out5], - return_numpy=True) - - expect_res = trilinear_interp_np(x_data, - out_d=12, - out_h=18, - out_w=8, - align_mode=1) - np.testing.assert_allclose(results[0], - np.transpose(expect_res, (0, 2, 3, 4, 1)), - rtol=1e-05) + results = exe.run( + fluid.default_main_program(), + feed={ + "x": x_data, + "y": np.transpose(x_data, (0, 2, 3, 4, 1)), + "dim": dim_data, + "shape_tensor": shape_data, + "actual_size": actual_size_data, + "scale_tensor": scale_data, + }, + fetch_list=[out1, out2, out3, out4, out5], + return_numpy=True, + ) + + expect_res = trilinear_interp_np( + x_data, out_d=12, out_h=18, out_w=8, align_mode=1 + ) + np.testing.assert_allclose( + results[0], np.transpose(expect_res, (0, 2, 3, 4, 1)), rtol=1e-05 + ) for i in range(len(results) - 1): np.testing.assert_allclose(results[i + 1], expect_res, rtol=1e-05) @@ -787,30 +815,28 @@ def func_case(self): # Preceding PR link: https://github.com/PaddlePaddle/Paddle/pull/26520/files#diff-ee0c2b73d08659e90a8f3ac48451a6588d35e1613742f864f9aad4394e12c290 with fluid.dygraph.guard(): x = fluid.dygraph.to_variable(x_data) - out9 = interpolate(x, - size=[12, 18, 8], - mode='trilinear', - data_format="NCDHW") + out9 = interpolate( + x, size=[12, 18, 8], mode='trilinear', data_format="NCDHW" + ) class TestTrilinearInterpOpException(unittest.TestCase): - def test_exception(self): input = fluid.data(name="input", shape=[2, 3, 6, 9, 4], dtype="float32") def attr_data_format(): # for 5-D input, data_format only can be NCDHW or NDHWC - out = fluid.layers.resize_trilinear(input, - out_shape=[4, 8, 4], - data_format='NHWC') + out = fluid.layers.resize_trilinear( + input, out_shape=[4, 8, 4], data_format='NHWC' + ) self.assertRaises(ValueError, attr_data_format) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestTrilinearInterpOpForFloat16(unittest.TestCase): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 3, 4, 4, 4] @@ -824,12 +850,14 @@ def check_main(self, x_np, dtype): x_np = x_np.astype(dtype) x = paddle.to_tensor(x_np) x.stop_gradient = False - y = interpolate(x, - size=self.out_size.tolist(), - mode=self.interp_method, - align_corners=self.align_corners, - align_mode=self.align_mode, - data_format=self.data_layout) + y = interpolate( + x, + size=self.out_size.tolist(), + mode=self.interp_method, + align_corners=self.align_corners, + align_mode=self.align_mode, + data_format=self.data_layout, + ) x_g = paddle.grad(y, x) y_np = y[0].numpy().astype('float32') x_g_np = x_g[0].numpy().astype('float32') @@ -848,10 +876,10 @@ def test_main(self): np.testing.assert_allclose(x_g_np_1, x_g_np_2, rtol=1e-05) -@unittest.skipIf(not fluid.core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not fluid.core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestTrilinearInterpDatalayoutForFloat16(TestTrilinearInterpOpForFloat16): - def init_test_case(self): self.interp_method = 'trilinear' self.input_shape = [2, 4, 4, 4, 3] diff --git a/python/paddle/fluid/tests/unittests/test_triplet_margin_loss.py b/python/paddle/fluid/tests/unittests/test_triplet_margin_loss.py index bf6faeca556d2..59f63a699d569 100644 --- a/python/paddle/fluid/tests/unittests/test_triplet_margin_loss.py +++ b/python/paddle/fluid/tests/unittests/test_triplet_margin_loss.py @@ -27,11 +27,9 @@ def call_TripletMarginLoss_layer( eps=1e-6, reduction='mean', ): - triplet_margin_loss = paddle.nn.TripletMarginLoss(p=p, - epsilon=eps, - margin=margin, - swap=swap, - reduction=reduction) + triplet_margin_loss = paddle.nn.TripletMarginLoss( + p=p, epsilon=eps, margin=margin, swap=swap, reduction=reduction + ) res = triplet_margin_loss( input=input, positive=positive, @@ -50,102 +48,116 @@ def call_TripletMarginLoss_functional( eps=1e-6, reduction='mean', ): - res = paddle.nn.functional.triplet_margin_loss(input=input, - positive=positive, - negative=negative, - p=p, - epsilon=eps, - margin=margin, - swap=swap, - reduction=reduction) + res = paddle.nn.functional.triplet_margin_loss( + input=input, + positive=positive, + negative=negative, + p=p, + epsilon=eps, + margin=margin, + swap=swap, + reduction=reduction, + ) return res -def test_static(place, - input_np, - positive_np, - negative_np, - p=2, - margin=0.3, - swap=False, - eps=1e-6, - reduction='mean', - functional=False): +def test_static( + place, + input_np, + positive_np, + negative_np, + p=2, + margin=0.3, + swap=False, + eps=1e-6, + reduction='mean', + functional=False, +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.static.data(name='input', - shape=input_np.shape, - dtype='float64') - positive = paddle.static.data(name='positive', - shape=positive_np.shape, - dtype='float64') - negative = paddle.static.data(name='negative', - shape=negative_np.shape, - dtype='float64') + input = paddle.static.data( + name='input', shape=input_np.shape, dtype='float64' + ) + positive = paddle.static.data( + name='positive', shape=positive_np.shape, dtype='float64' + ) + negative = paddle.static.data( + name='negative', shape=negative_np.shape, dtype='float64' + ) feed_dict = { "input": input_np, "positive": positive_np, - "negative": negative_np + "negative": negative_np, } if functional: - res = call_TripletMarginLoss_functional(input=input, - positive=positive, - negative=negative, - p=p, - eps=eps, - margin=margin, - swap=swap, - reduction=reduction) + res = call_TripletMarginLoss_functional( + input=input, + positive=positive, + negative=negative, + p=p, + eps=eps, + margin=margin, + swap=swap, + reduction=reduction, + ) else: - res = call_TripletMarginLoss_layer(input=input, - positive=positive, - negative=negative, - p=p, - eps=eps, - margin=margin, - swap=swap, - reduction=reduction) + res = call_TripletMarginLoss_layer( + input=input, + positive=positive, + negative=negative, + p=p, + eps=eps, + margin=margin, + swap=swap, + reduction=reduction, + ) exe = paddle.static.Executor(place) static_result = exe.run(prog, feed=feed_dict, fetch_list=[res])[0] return static_result -def test_dygraph(place, - input, - positive, - negative, - p=2, - margin=0.3, - swap=False, - eps=1e-6, - reduction='mean', - functional=False): +def test_dygraph( + place, + input, + positive, + negative, + p=2, + margin=0.3, + swap=False, + eps=1e-6, + reduction='mean', + functional=False, +): paddle.disable_static() input = paddle.to_tensor(input) positive = paddle.to_tensor(positive) negative = paddle.to_tensor(negative) if functional: - dy_res = call_TripletMarginLoss_functional(input=input, - positive=positive, - negative=negative, - p=p, - eps=eps, - margin=margin, - swap=swap, - reduction=reduction) + dy_res = call_TripletMarginLoss_functional( + input=input, + positive=positive, + negative=negative, + p=p, + eps=eps, + margin=margin, + swap=swap, + reduction=reduction, + ) else: - dy_res = call_TripletMarginLoss_layer(input=input, - positive=positive, - negative=negative, - p=p, - eps=eps, - margin=margin, - swap=swap, - reduction=reduction) + dy_res = call_TripletMarginLoss_layer( + input=input, + positive=positive, + negative=negative, + p=p, + eps=eps, + margin=margin, + swap=swap, + reduction=reduction, + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result @@ -179,7 +191,6 @@ def calc_triplet_margin_loss( class TestTripletMarginLoss(unittest.TestCase): - def test_TripletMarginLoss(self): shape = (2, 2) input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) @@ -192,10 +203,12 @@ def test_TripletMarginLoss(self): reductions = ['sum', 'mean', 'none'] for place in places: for reduction in reductions: - expected = calc_triplet_margin_loss(input=input, - positive=positive, - negative=negative, - reduction=reduction) + expected = calc_triplet_margin_loss( + input=input, + positive=positive, + negative=negative, + reduction=reduction, + ) dy_result = test_dygraph( place=place, @@ -212,57 +225,59 @@ def test_TripletMarginLoss(self): negative_np=negative, reduction=reduction, ) - np.testing.assert_allclose(static_result, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(dy_result, - expected, - rtol=1e-5, - atol=1e-8) - static_functional = test_static(place=place, - input_np=input, - positive_np=positive, - negative_np=negative, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - input=input, - positive=positive, - negative=negative, - reduction=reduction, - functional=True) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(dy_functional, - expected, - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + static_result, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + dy_result, expected, rtol=1e-5, atol=1e-8 + ) + static_functional = test_static( + place=place, + input_np=input, + positive_np=positive, + negative_np=negative, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + input=input, + positive=positive, + negative=negative, + reduction=reduction, + functional=True, + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + dy_functional, expected, rtol=1e-5, atol=1e-8 + ) def test_TripletMarginLoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.loss.TripletMarginLoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.loss.TripletMarginLoss, + reduction="unsupport reduction", + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') positive = paddle.to_tensor([[0.0, 1.0]], dtype='float32') negative = paddle.to_tensor([[0.2, 0.1]], dtype='float32') - self.assertRaises(ValueError, - paddle.nn.functional.triplet_margin_loss, - input=input, - positive=positive, - negative=negative, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.functional.triplet_margin_loss, + input=input, + positive=positive, + negative=negative, + reduction="unsupport reduction", + ) paddle.enable_static() def test_TripletMarginLoss_dimension(self): @@ -295,11 +310,13 @@ def test_TripletMarginLoss_swap(self): input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) positive = np.random.uniform(0, 2, size=shape).astype(np.float64) negative = np.random.uniform(0, 2, size=shape).astype(np.float64) - expected = calc_triplet_margin_loss(input=input, - swap=True, - positive=positive, - negative=negative, - reduction=reduction) + expected = calc_triplet_margin_loss( + input=input, + swap=True, + positive=positive, + negative=negative, + reduction=reduction, + ) dy_result = test_dygraph( place=place, @@ -318,41 +335,40 @@ def test_TripletMarginLoss_swap(self): negative_np=negative, reduction=reduction, ) - np.testing.assert_allclose(static_result, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + static_result, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-5, atol=1e-8 + ) np.testing.assert_allclose(dy_result, expected, rtol=1e-5, atol=1e-8) - static_functional = test_static(place=place, - swap=True, - input_np=input, - positive_np=positive, - negative_np=negative, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - swap=True, - input=input, - positive=positive, - negative=negative, - reduction=reduction, - functional=True) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(dy_functional, - expected, - rtol=1e-5, - atol=1e-8) + static_functional = test_static( + place=place, + swap=True, + input_np=input, + positive_np=positive, + negative_np=negative, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + swap=True, + input=input, + positive=positive, + negative=negative, + reduction=reduction, + functional=True, + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + dy_functional, expected, rtol=1e-5, atol=1e-8 + ) def test_TripletMarginLoss_margin(self): paddle.disable_static() @@ -379,11 +395,13 @@ def test_TripletMarginLoss_p(self): input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) positive = np.random.uniform(0, 2, size=shape).astype(np.float64) negative = np.random.uniform(0, 2, size=shape).astype(np.float64) - expected = calc_triplet_margin_loss(input=input, - p=p, - positive=positive, - negative=negative, - reduction=reduction) + expected = calc_triplet_margin_loss( + input=input, + p=p, + positive=positive, + negative=negative, + reduction=reduction, + ) dy_result = test_dygraph( place=place, @@ -402,41 +420,40 @@ def test_TripletMarginLoss_p(self): negative_np=negative, reduction=reduction, ) - np.testing.assert_allclose(static_result, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + static_result, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-5, atol=1e-8 + ) np.testing.assert_allclose(dy_result, expected, rtol=1e-5, atol=1e-8) - static_functional = test_static(place=place, - p=p, - input_np=input, - positive_np=positive, - negative_np=negative, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - p=p, - input=input, - positive=positive, - negative=negative, - reduction=reduction, - functional=True) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(dy_functional, - expected, - rtol=1e-5, - atol=1e-8) + static_functional = test_static( + place=place, + p=p, + input_np=input, + positive_np=positive, + negative_np=negative, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + p=p, + input=input, + positive=positive, + negative=negative, + reduction=reduction, + functional=True, + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + dy_functional, expected, rtol=1e-5, atol=1e-8 + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_triplet_margin_with_distance_loss.py b/python/paddle/fluid/tests/unittests/test_triplet_margin_with_distance_loss.py index 8b7f79b77abd9..c8d4da36a1cb5 100644 --- a/python/paddle/fluid/tests/unittests/test_triplet_margin_with_distance_loss.py +++ b/python/paddle/fluid/tests/unittests/test_triplet_margin_with_distance_loss.py @@ -30,7 +30,8 @@ def call_TripletMarginDistanceLoss_layer( distance_function=distance_function, margin=margin, swap=swap, - reduction=reduction) + reduction=reduction, + ) res = triplet_margin_with_distance_loss( input=input, positive=positive, @@ -55,35 +56,38 @@ def call_TripletMaginDistanceLoss_functional( distance_function=distance_function, margin=margin, swap=swap, - reduction=reduction) + reduction=reduction, + ) return res -def test_static(place, - input_np, - positive_np, - negative_np, - distance_function=None, - margin=0.3, - swap=False, - reduction='mean', - functional=False): +def test_static( + place, + input_np, + positive_np, + negative_np, + distance_function=None, + margin=0.3, + swap=False, + reduction='mean', + functional=False, +): prog = paddle.static.Program() startup_prog = paddle.static.Program() with paddle.static.program_guard(prog, startup_prog): - input = paddle.static.data(name='input', - shape=input_np.shape, - dtype='float64') - positive = paddle.static.data(name='positive', - shape=positive_np.shape, - dtype='float64') - negative = paddle.static.data(name='negative', - shape=negative_np.shape, - dtype='float64') + input = paddle.static.data( + name='input', shape=input_np.shape, dtype='float64' + ) + positive = paddle.static.data( + name='positive', shape=positive_np.shape, dtype='float64' + ) + negative = paddle.static.data( + name='negative', shape=negative_np.shape, dtype='float64' + ) feed_dict = { "input": input_np, "positive": positive_np, - "negative": negative_np + "negative": negative_np, } if functional: @@ -94,7 +98,8 @@ def test_static(place, distance_function=distance_function, margin=margin, swap=swap, - reduction=reduction) + reduction=reduction, + ) else: res = call_TripletMarginDistanceLoss_layer( input=input, @@ -103,7 +108,8 @@ def test_static(place, distance_function=distance_function, margin=margin, swap=swap, - reduction=reduction) + reduction=reduction, + ) exe = paddle.static.Executor(place) static_result = exe.run(prog, feed=feed_dict, fetch_list=[res])[0] @@ -111,15 +117,17 @@ def test_static(place, return static_result -def test_dygraph(place, - input, - positive, - negative, - distance_function=None, - margin=0.3, - swap=False, - reduction='mean', - functional=False): +def test_dygraph( + place, + input, + positive, + negative, + distance_function=None, + margin=0.3, + swap=False, + reduction='mean', + functional=False, +): paddle.disable_static() input = paddle.to_tensor(input) positive = paddle.to_tensor(positive) @@ -133,7 +141,8 @@ def test_dygraph(place, distance_function=distance_function, margin=margin, swap=swap, - reduction=reduction) + reduction=reduction, + ) else: dy_res = call_TripletMarginDistanceLoss_layer( input=input, @@ -142,7 +151,8 @@ def test_dygraph(place, distance_function=distance_function, margin=margin, swap=swap, - reduction=reduction) + reduction=reduction, + ) dy_result = dy_res.numpy() paddle.enable_static() return dy_result @@ -177,7 +187,6 @@ def calc_triplet_margin_distance_loss( class TestTripletMarginWithDistanceLoss(unittest.TestCase): - def test_TripletMarginDistanceLoss(self): shape = (5, 5) input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) @@ -194,7 +203,8 @@ def test_TripletMarginDistanceLoss(self): input=input, positive=positive, negative=negative, - reduction=reduction) + reduction=reduction, + ) dy_result = test_dygraph( place=place, @@ -211,48 +221,48 @@ def test_TripletMarginDistanceLoss(self): negative_np=negative, reduction=reduction, ) - np.testing.assert_allclose(static_result, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(dy_result, - expected, - rtol=1e-5, - atol=1e-8) - static_functional = test_static(place=place, - input_np=input, - positive_np=positive, - negative_np=negative, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - input=input, - positive=positive, - negative=negative, - reduction=reduction, - functional=True) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(dy_functional, - expected, - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + static_result, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + dy_result, expected, rtol=1e-5, atol=1e-8 + ) + static_functional = test_static( + place=place, + input_np=input, + positive_np=positive, + negative_np=negative, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + input=input, + positive=positive, + negative=negative, + reduction=reduction, + functional=True, + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + dy_functional, expected, rtol=1e-5, atol=1e-8 + ) def test_TripletMarginDistanceLoss_error(self): paddle.disable_static() - self.assertRaises(ValueError, - paddle.nn.TripletMarginWithDistanceLoss, - reduction="unsupport reduction") + self.assertRaises( + ValueError, + paddle.nn.TripletMarginWithDistanceLoss, + reduction="unsupport reduction", + ) input = paddle.to_tensor([[0.1, 0.3]], dtype='float32') positive = paddle.to_tensor([[0.0, 1.0]], dtype='float32') negative = paddle.to_tensor([[0.2, 0.1]], dtype='float32') @@ -262,11 +272,11 @@ def test_TripletMarginDistanceLoss_error(self): input=input, positive=positive, negative=negative, - reduction="unsupport reduction") + reduction="unsupport reduction", + ) paddle.enable_static() def test_TripletMarginDistanceLoss_distance_function(self): - def distance_function_1(x1, x2): return 1.0 - paddle.nn.functional.cosine_similarity(x1, x2) @@ -299,28 +309,30 @@ def distance_function_2(x1, x2): distance_function=distance_function, reduction=reduction, ) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-5, - atol=1e-8) - static_functional = test_static(place=place, - input_np=input, - positive_np=positive, - negative_np=negative, - distance_function=distance_function, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - input=input, - positive=positive, - negative=negative, - distance_function=distance_function, - reduction=reduction, - functional=True) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-5, atol=1e-8 + ) + static_functional = test_static( + place=place, + input_np=input, + positive_np=positive, + negative_np=negative, + distance_function=distance_function, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + input=input, + positive=positive, + negative=negative, + distance_function=distance_function, + reduction=reduction, + functional=True, + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-5, atol=1e-8 + ) def test_TripletMarginWithDistanceLoss_distance_funtion_error(self): paddle.disable_static() @@ -357,7 +369,8 @@ def test_TripletMarginDistanceLoss_dimension(self): positive=positive, negative=negative, ) - triplet_margin_with_distance_loss = paddle.nn.loss.TripletMarginWithDistanceLoss( + triplet_margin_with_distance_loss = ( + paddle.nn.loss.TripletMarginWithDistanceLoss() ) self.assertRaises( ValueError, @@ -375,11 +388,13 @@ def test_TripletMarginWithDistanceLoss_swap(self): input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) positive = np.random.uniform(0, 2, size=shape).astype(np.float64) negative = np.random.uniform(0, 2, size=shape).astype(np.float64) - expected = calc_triplet_margin_distance_loss(input=input, - swap=True, - positive=positive, - negative=negative, - reduction=reduction) + expected = calc_triplet_margin_distance_loss( + input=input, + swap=True, + positive=positive, + negative=negative, + reduction=reduction, + ) dy_result = test_dygraph( place=place, @@ -398,41 +413,40 @@ def test_TripletMarginWithDistanceLoss_swap(self): negative_np=negative, reduction=reduction, ) - np.testing.assert_allclose(static_result, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_result, - dy_result, - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + static_result, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_result, dy_result, rtol=1e-5, atol=1e-8 + ) np.testing.assert_allclose(dy_result, expected, rtol=1e-5, atol=1e-8) - static_functional = test_static(place=place, - swap=True, - input_np=input, - positive_np=positive, - negative_np=negative, - reduction=reduction, - functional=True) - dy_functional = test_dygraph(place=place, - swap=True, - input=input, - positive=positive, - negative=negative, - reduction=reduction, - functional=True) - np.testing.assert_allclose(static_functional, - expected, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(static_functional, - dy_functional, - rtol=1e-5, - atol=1e-8) - np.testing.assert_allclose(dy_functional, - expected, - rtol=1e-5, - atol=1e-8) + static_functional = test_static( + place=place, + swap=True, + input_np=input, + positive_np=positive, + negative_np=negative, + reduction=reduction, + functional=True, + ) + dy_functional = test_dygraph( + place=place, + swap=True, + input=input, + positive=positive, + negative=negative, + reduction=reduction, + functional=True, + ) + np.testing.assert_allclose( + static_functional, expected, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + static_functional, dy_functional, rtol=1e-5, atol=1e-8 + ) + np.testing.assert_allclose( + dy_functional, expected, rtol=1e-5, atol=1e-8 + ) def test_TripletMarginWithDistanceLoss_margin(self): paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_triu_indices_op.py b/python/paddle/fluid/tests/unittests/test_triu_indices_op.py index 6863517874e0e..1ad6b7bbd458f 100644 --- a/python/paddle/fluid/tests/unittests/test_triu_indices_op.py +++ b/python/paddle/fluid/tests/unittests/test_triu_indices_op.py @@ -21,7 +21,6 @@ class TestTriuIndicesOp(OpTest): - def setUp(self): self.op_type = "triu_indices" self.inputs = {} @@ -34,13 +33,13 @@ def test_check_output(self): def init_config(self): self.attrs = {'row': 4, 'col': 4, 'offset': -1} - self.target = np.triu_indices(self.attrs['row'], self.attrs['offset'], - self.attrs['col']) + self.target = np.triu_indices( + self.attrs['row'], self.attrs['offset'], self.attrs['col'] + ) self.target = np.array(self.target) class TestTriuIndicesOpCase1(TestTriuIndicesOp): - def init_config(self): self.attrs = {'row': 0, 'col': 0, 'offset': 0} self.target = np.triu_indices(0, 0, 0) @@ -48,23 +47,23 @@ def init_config(self): class TestTriuIndicesOpCase2(TestTriuIndicesOp): - def init_config(self): self.attrs = {'row': 4, 'col': 4, 'offset': 2} - self.target = np.triu_indices(self.attrs['row'], self.attrs['offset'], - self.attrs['col']) + self.target = np.triu_indices( + self.attrs['row'], self.attrs['offset'], self.attrs['col'] + ) self.target = np.array(self.target) class TestTriuIndicesAPICaseStatic(unittest.TestCase): - def test_static(self): if fluid.core.is_compiled_with_cuda(): place = paddle.fluid.CUDAPlace(0) else: place = paddle.CPUPlace() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.triu_indices(4, 4, -1) exe = paddle.static.Executor(place) result = exe.run(feed={}, fetch_list=[data]) @@ -73,7 +72,6 @@ def test_static(self): class TestTriuIndicesAPICaseDygraph(unittest.TestCase): - def test_dygraph(self): if fluid.core.is_compiled_with_cuda(): place = paddle.fluid.CUDAPlace(0) @@ -90,9 +88,7 @@ def test_dygraph_eager(self): class TestTriuIndicesAPICaseError(unittest.TestCase): - def test_case_error(self): - def test_num_rows_type_check(): out1 = paddle.triu_indices(1.0, 1, 2) @@ -110,11 +106,11 @@ def test_num_offset_type_check(): class TestTriuIndicesAPICaseDefault(unittest.TestCase): - def test_default_CPU(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data = paddle.triu_indices(4, None, 2) exe = paddle.static.Executor(paddle.CPUPlace()) result = exe.run(feed={}, fetch_list=[data]) diff --git a/python/paddle/fluid/tests/unittests/test_trunc_op.py b/python/paddle/fluid/tests/unittests/test_trunc_op.py index 0a410e5380541..f8e23bd807488 100644 --- a/python/paddle/fluid/tests/unittests/test_trunc_op.py +++ b/python/paddle/fluid/tests/unittests/test_trunc_op.py @@ -22,7 +22,6 @@ class TestTruncOp(OpTest): - def setUp(self): self.op_type = "trunc" self.python_api = paddle.trunc @@ -42,7 +41,6 @@ def test_check_grad(self): class TestFloatTruncOp(TestTruncOp): - def init_dtype_type(self): self.dtype = np.float32 self.__class__.exist_fp64_check_grad = True @@ -52,7 +50,6 @@ def test_check_grad(self): class TestIntTruncOp(TestTruncOp): - def init_dtype_type(self): self.dtype = np.int32 self.__class__.exist_fp64_check_grad = True @@ -62,7 +59,6 @@ def test_check_grad(self): class TestTruncAPI(unittest.TestCase): - def setUp(self): self.shape = [20, 20] self.x = np.random.random((20, 20)).astype(np.float32) diff --git a/python/paddle/fluid/tests/unittests/test_truncated_gaussian_random_op.py b/python/paddle/fluid/tests/unittests/test_truncated_gaussian_random_op.py index 780caa74333d1..70e79b4898af3 100644 --- a/python/paddle/fluid/tests/unittests/test_truncated_gaussian_random_op.py +++ b/python/paddle/fluid/tests/unittests/test_truncated_gaussian_random_op.py @@ -23,14 +23,13 @@ class TestTrunctedGaussianRandomOp(unittest.TestCase): - def setUp(self): self.op_type = "truncated_gaussian_random" self.inputs = {} self.attrs = { "shape": [10000], - "mean": .0, - "std": 1., + "mean": 0.0, + "std": 1.0, "seed": 10, } self.outputs = ["Out"] @@ -49,9 +48,9 @@ def gaussian_random_test(self, place): program = fluid.Program() block = program.global_block() vout = block.create_var(name="Out") - op = block.append_op(type=self.op_type, - outputs={"Out": vout}, - attrs=self.attrs) + op = block.append_op( + type=self.op_type, outputs={"Out": vout}, attrs=self.attrs + ) op.desc.infer_var_type(block.desc) op.desc.infer_shape(block.desc) @@ -63,7 +62,7 @@ def gaussian_random_test(self, place): exe = Executor(place) outs = exe.run(program, fetch_list=fetch_list) tensor = outs[0] - self.assertAlmostEqual(numpy.mean(tensor), .0, delta=0.1) + self.assertAlmostEqual(numpy.mean(tensor), 0.0, delta=0.1) self.assertAlmostEqual(numpy.var(tensor), 0.773, delta=0.1) # TruncatedNormal.__call__ has no return value, so here call _C_ops api @@ -72,9 +71,14 @@ def gaussian_random_test_eager(self, place): with fluid.dygraph.guard(place): with _test_eager_guard(): out = paddle._C_ops.truncated_gaussian_random( - self.attrs["shape"], self.attrs["mean"], self.attrs["std"], - self.attrs["seed"], core.VarDesc.VarType.FP32, place) - self.assertAlmostEqual(numpy.mean(out.numpy()), .0, delta=0.1) + self.attrs["shape"], + self.attrs["mean"], + self.attrs["std"], + self.attrs["seed"], + core.VarDesc.VarType.FP32, + place, + ) + self.assertAlmostEqual(numpy.mean(out.numpy()), 0.0, delta=0.1) self.assertAlmostEqual(numpy.var(out.numpy()), 0.773, delta=0.1) diff --git a/python/paddle/fluid/tests/unittests/test_unbind_op.py b/python/paddle/fluid/tests/unittests/test_unbind_op.py index e030792ce6014..6f719ae7d4d5c 100644 --- a/python/paddle/fluid/tests/unittests/test_unbind_op.py +++ b/python/paddle/fluid/tests/unittests/test_unbind_op.py @@ -23,7 +23,6 @@ class TestUnbind(unittest.TestCase): - def test_unbind(self): x_1 = fluid.data(shape=[2, 3], dtype='float32', name='x_1') @@ -32,12 +31,11 @@ def test_unbind(self): axis = fluid.data(shape=[1], dtype='int32', name='axis') exe = fluid.Executor(place=fluid.CPUPlace()) - [res_1, res_2] = exe.run(fluid.default_main_program(), - feed={ - "x_1": input_1, - "axis": 0 - }, - fetch_list=[out_0, out_1]) + [res_1, res_2] = exe.run( + fluid.default_main_program(), + feed={"x_1": input_1, "axis": 0}, + fetch_list=[out_0, out_1], + ) assert np.array_equal(res_1, input_1[0, 0:100]) assert np.array_equal(res_2, input_1[1, 0:100]) @@ -63,7 +61,6 @@ def test_unbind_dygraph_final_state(self): class TestLayersUnbind(unittest.TestCase): - def test_layers_unbind(self): x_1 = fluid.data(shape=[2, 3], dtype='float32', name='x_1') @@ -72,19 +69,17 @@ def test_layers_unbind(self): axis = fluid.data(shape=[1], dtype='int32', name='axis') exe = fluid.Executor(place=fluid.CPUPlace()) - [res_1, res_2] = exe.run(fluid.default_main_program(), - feed={ - "x_1": input_1, - "axis": 0 - }, - fetch_list=[out_0, out_1]) + [res_1, res_2] = exe.run( + fluid.default_main_program(), + feed={"x_1": input_1, "axis": 0}, + fetch_list=[out_0, out_1], + ) assert np.array_equal(res_1, input_1[0, 0:100]) assert np.array_equal(res_2, input_1[1, 0:100]) class TestUnbindOp(OpTest): - def initParameters(self): pass @@ -106,8 +101,9 @@ def setUp(self): self.inputs = {'X': x} self.attrs = {'axis': self.axis} self.setAxis() - self.outputs = {'Out': [('out%d' % i, self.out[i]) \ - for i in range(len(self.out))]} + self.outputs = { + 'Out': [('out%d' % i, self.out[i]) for i in range(len(self.out))] + } def get_dtype(self): return "float64" @@ -123,7 +119,6 @@ def test_check_grad(self): class TestUnbindOp1(TestUnbindOp): - def initParameters(self): self.axis = 1 self.num = 2 @@ -137,7 +132,6 @@ def outReshape(self): class TestUnbindOp2(TestUnbindOp): - def initParameters(self): self.axis = 2 self.num = 2 @@ -151,7 +145,6 @@ def outReshape(self): class TestUnbindOp3(TestUnbindOp): - def initParameters(self): self.axis = 2 self.num = 2 @@ -168,7 +161,6 @@ def outReshape(self): class TestUnbindOp4(TestUnbindOp): - def initParameters(self): self.axis = 1 self.num = 2 @@ -185,7 +177,6 @@ def outReshape(self): class TestUnbindBF16Op(OpTest): - def setUp(self): self._set_op_type() self.python_api = paddle.unbind @@ -196,8 +187,12 @@ def setUp(self): self.out = np.split(x, self.num, self.axis) self.inputs = {'X': convert_float_to_uint16(x)} self.attrs = {'axis': self.axis} - self.outputs = {'Out': [('out%d' % i, convert_float_to_uint16(self.out[i])) \ - for i in range(len(self.out))]} + self.outputs = { + 'Out': [ + ('out%d' % i, convert_float_to_uint16(self.out[i])) + for i in range(len(self.out)) + ] + } def get_dtype(self): return np.uint16 @@ -213,7 +208,6 @@ def test_check_grad(self): class TestUnbindAxisError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x = fluid.data(shape=[2, 3], dtype='float32', name='x') diff --git a/python/paddle/fluid/tests/unittests/test_unfold_op.py b/python/paddle/fluid/tests/unittests/test_unfold_op.py index ea38ae2be1cb2..0bf5f3d2932e2 100644 --- a/python/paddle/fluid/tests/unittests/test_unfold_op.py +++ b/python/paddle/fluid/tests/unittests/test_unfold_op.py @@ -35,23 +35,45 @@ def init_data(self): self.paddings = [1, 1, 1, 1] self.dilations = [1, 1] input_shape = [ - self.batch_size, self.input_channels, self.input_height, - self.input_width + self.batch_size, + self.input_channels, + self.input_height, + self.input_width, ] self.x = np.random.rand(*input_shape).astype(np.float64) def calc_unfold(self): output_shape = [0] * 3 output_shape[0] = self.batch_size - output_shape[1] = self.input_channels * self.kernel_sizes[ - 0] * self.kernel_sizes[1] + output_shape[1] = ( + self.input_channels * self.kernel_sizes[0] * self.kernel_sizes[1] + ) dkernel_h = self.dilations[0] * (self.kernel_sizes[0] - 1) + 1 dkernel_w = self.dilations[1] * (self.kernel_sizes[1] - 1) + 1 - out_height = int((self.input_height + self.paddings[0] + - self.paddings[2] - dkernel_h) / self.strides[0]) + 1 - out_width = int( - (self.input_width + self.paddings[1] + self.paddings[3] - dkernel_w) - / self.strides[1]) + 1 + out_height = ( + int( + ( + self.input_height + + self.paddings[0] + + self.paddings[2] + - dkernel_h + ) + / self.strides[0] + ) + + 1 + ) + out_width = ( + int( + ( + self.input_width + + self.paddings[1] + + self.paddings[3] + - dkernel_w + ) + / self.strides[1] + ) + + 1 + ) output_shape[2] = out_height * out_width output = np.zeros(output_shape).astype(np.float64) ############ calculate output ############## @@ -61,16 +83,25 @@ def calc_unfold(self): h_out = int(k / out_width) w_out = k % out_width w_offset = j % self.kernel_sizes[1] - h_offset = int( - j / self.kernel_sizes[1]) % self.kernel_sizes[0] - c_in = int(j / - (self.kernel_sizes[0] * self.kernel_sizes[1])) - h_in = h_offset * self.dilations[0] + h_out * self.strides[ - 0] - self.paddings[0] - w_in = w_offset * self.dilations[1] + w_out * self.strides[ - 1] - self.paddings[1] - if (h_in>=0 and h_in=0 and w_in= 0 and h_in < self.input_height) and ( + w_in >= 0 and w_in < self.input_width + ): output[i, j, k] = self.x[i, c_in, h_in, w_in] self.outputs = output @@ -84,7 +115,7 @@ def set_data(self): 'kernel_sizes': self.kernel_sizes, 'paddings': self.paddings, 'dilations': self.dilations, - 'strides': self.strides + 'strides': self.strides, } self.outputs = {'Y': self.outputs} @@ -118,9 +149,9 @@ def test_dygraph(self): m = paddle.nn.Unfold(**self.attrs) m.eval() result = m(input) - np.testing.assert_allclose(result.numpy(), - self.outputs['Y'], - rtol=1e-05) + np.testing.assert_allclose( + result.numpy(), self.outputs['Y'], rtol=1e-05 + ) def test_info(self): str(paddle.nn.Unfold(**self.attrs)) diff --git a/python/paddle/fluid/tests/unittests/test_uniform_random_bf16_op.py b/python/paddle/fluid/tests/unittests/test_uniform_random_bf16_op.py index 77d9903cefff2..0977ec69ac3ae 100644 --- a/python/paddle/fluid/tests/unittests/test_uniform_random_bf16_op.py +++ b/python/paddle/fluid/tests/unittests/test_uniform_random_bf16_op.py @@ -19,11 +19,13 @@ import paddle.fluid.core as core from paddle.fluid.op import Operator import paddle.fluid as fluid -from paddle.fluid.tests.unittests.test_uniform_random_op import output_hist, output_hist_diag +from paddle.fluid.tests.unittests.test_uniform_random_op import ( + output_hist, + output_hist_diag, +) class TestUniformRandomOpBF16(OpTest): - def setUp(self): self.op_type = "uniform_random" self.dtype = "uint16" @@ -37,7 +39,7 @@ def init_attrs(self): "min": -5.0, "max": 10.0, "seed": 10, - 'dtype': int(core.VarDesc.VarType.BF16) + 'dtype': int(core.VarDesc.VarType.BF16), } self.output_hist = output_hist @@ -58,15 +60,15 @@ def test_check_output(self): class TestUniformRandomOpBF16AttrTensorList(TestUniformRandomOpBF16): - def setUp(self): self.op_type = "uniform_random" self.new_shape = (1000, 784) self.dtype = "uint16" shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype("int64") * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype("int64") * ele) + ) self.inputs = {'ShapeTensorList': shape_tensor} self.init_attrs() self.outputs = {"Out": np.zeros((1000, 784)).astype("uint16")} @@ -76,14 +78,14 @@ def init_attrs(self): "min": -5.0, "max": 10.0, "seed": 10, - 'dtype': int(core.VarDesc.VarType.BF16) + 'dtype': int(core.VarDesc.VarType.BF16), } self.output_hist = output_hist class TestUniformRandomOpBF16AttrTensorInt32( - TestUniformRandomOpBF16AttrTensorList): - + TestUniformRandomOpBF16AttrTensorList +): def setUp(self): self.op_type = "uniform_random" self.dtype = "uint16" @@ -93,7 +95,6 @@ def setUp(self): class TestUniformRandomOpBF16WithDiagInit(TestUniformRandomOpBF16): - def init_attrs(self): self.attrs = { "shape": [1000, 784], @@ -103,13 +104,12 @@ def init_attrs(self): "diag_num": 784, "diag_step": 784, "diag_val": 1.0, - 'dtype': int(core.VarDesc.VarType.BF16) + 'dtype': int(core.VarDesc.VarType.BF16), } self.output_hist = output_hist_diag class TestUniformRandomOpBF16SelectedRows(unittest.TestCase): - def test_check_output(self): self.check_with_place(core.CPUPlace()) @@ -117,13 +117,15 @@ def check_with_place(self, place): scope = core.Scope() out = scope.var("X").get_selected_rows() paddle.seed(10) - op = Operator("uniform_random", - Out="X", - shape=[1000, 784], - min=-5.0, - max=10.0, - seed=10, - dtype=int(core.VarDesc.VarType.BF16)) + op = Operator( + "uniform_random", + Out="X", + shape=[1000, 784], + min=-5.0, + max=10.0, + seed=10, + dtype=int(core.VarDesc.VarType.BF16), + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) result = convert_uint16_to_float(np.array(out.get_tensor())) @@ -132,22 +134,24 @@ def check_with_place(self, place): class TestUniformRandomOpBF16SelectedRowsWithDiagInit( - TestUniformRandomOpBF16SelectedRows): - + TestUniformRandomOpBF16SelectedRows +): def check_with_place(self, place): scope = core.Scope() out = scope.var("X").get_selected_rows() paddle.seed(10) - op = Operator("uniform_random", - Out="X", - shape=[500, 784], - min=-5.0, - max=10.0, - seed=10, - diag_num=500, - diag_step=784, - diag_val=1.0, - dtype=int(core.VarDesc.VarType.BF16)) + op = Operator( + "uniform_random", + Out="X", + shape=[500, 784], + min=-5.0, + max=10.0, + seed=10, + diag_num=500, + diag_step=784, + diag_val=1.0, + dtype=int(core.VarDesc.VarType.BF16), + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [500, 784]) result = convert_uint16_to_float(np.array(out.get_tensor())) @@ -156,14 +160,14 @@ def check_with_place(self, place): class TestUniformRandomOpBF16AttrTensorAPI(unittest.TestCase): - def test_attr_tensor_API(self): startup_program = fluid.Program() train_program = fluid.Program() with fluid.program_guard(train_program, startup_program): dim_tensor = fluid.layers.fill_constant([1], "int64", 3) - ret = fluid.layers.nn.uniform_random([1, dim_tensor, 2], - dtype=np.uint16) + ret = fluid.layers.nn.uniform_random( + [1, dim_tensor, 2], dtype=np.uint16 + ) place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -173,7 +177,6 @@ def test_attr_tensor_API(self): class TestUniformRandomOpAPISeed(unittest.TestCase): - def test_attr_tensor_API(self): _seed = 10 gen = paddle.seed(_seed) @@ -183,14 +186,12 @@ def test_attr_tensor_API(self): _min = 5 _max = 10 - ret = fluid.layers.nn.uniform_random([2, 3, 2], - min=_min, - max=_max, - seed=_seed) - ret_2 = fluid.layers.nn.uniform_random([2, 3, 2], - min=_min, - max=_max, - seed=_seed) + ret = fluid.layers.nn.uniform_random( + [2, 3, 2], min=_min, max=_max, seed=_seed + ) + ret_2 = fluid.layers.nn.uniform_random( + [2, 3, 2], min=_min, max=_max, seed=_seed + ) res = fluid.layers.equal(ret, ret_2) place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -204,7 +205,6 @@ def test_attr_tensor_API(self): class TestUniformRandomOpBF16SelectedRowsShapeTensor(unittest.TestCase): - def test_check_output(self): place = core.CPUPlace() scope = core.Scope() @@ -212,13 +212,15 @@ def test_check_output(self): shape_tensor = scope.var("Shape").get_tensor() shape_tensor.set(np.array([1000, 784]).astype("int64"), place) paddle.seed(10) - op = Operator("uniform_random", - ShapeTensor="Shape", - Out="X", - min=-5.0, - max=10.0, - seed=10, - dtype=int(core.VarDesc.VarType.BF16)) + op = Operator( + "uniform_random", + ShapeTensor="Shape", + Out="X", + min=-5.0, + max=10.0, + seed=10, + dtype=int(core.VarDesc.VarType.BF16), + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) result = convert_uint16_to_float(np.array(out.get_tensor())) @@ -227,8 +229,8 @@ def test_check_output(self): class TestUniformRandomOpBF16SelectedRowsShapeTensorList( - TestUniformRandomOpBF16SelectedRowsShapeTensor): - + TestUniformRandomOpBF16SelectedRowsShapeTensor +): def test_check_output(self): place = core.CPUPlace() scope = core.Scope() @@ -238,13 +240,15 @@ def test_check_output(self): shape_2 = scope.var("shape2").get_tensor() shape_2.set(np.array([784]).astype("int64"), place) paddle.seed(10) - op = Operator("uniform_random", - ShapeTensorList=["shape1", "shape2"], - Out="X", - min=-5.0, - max=10.0, - seed=10, - dtype=int(core.VarDesc.VarType.BF16)) + op = Operator( + "uniform_random", + ShapeTensorList=["shape1", "shape2"], + Out="X", + min=-5.0, + max=10.0, + seed=10, + dtype=int(core.VarDesc.VarType.BF16), + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) result = convert_uint16_to_float(np.array(out.get_tensor())) @@ -253,14 +257,14 @@ def test_check_output(self): class TestUniformRandomBatchSizeLikeOpBF16API(unittest.TestCase): - def test_attr_tensorlist_int32_API(self): startup_program = fluid.Program() train_program = fluid.Program() with fluid.program_guard(train_program, startup_program): input = fluid.data(name="input", shape=[1, 3], dtype='uint16') out_1 = fluid.layers.uniform_random_batch_size_like( - input, [2, 4], dtype=np.uint16) # out_1.shape=[1, 4] + input, [2, 4], dtype=np.uint16 + ) # out_1.shape=[1, 4] place = fluid.CPUPlace() exe = fluid.Executor(place) @@ -271,5 +275,6 @@ def test_attr_tensorlist_int32_API(self): if __name__ == "__main__": from paddle import enable_static + enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_uniform_random_inplace_op.py b/python/paddle/fluid/tests/unittests/test_uniform_random_inplace_op.py index 2b3881535c955..ea30b0da0ab46 100644 --- a/python/paddle/fluid/tests/unittests/test_uniform_random_inplace_op.py +++ b/python/paddle/fluid/tests/unittests/test_uniform_random_inplace_op.py @@ -16,16 +16,17 @@ import paddle import paddle.fluid as fluid import numpy as np -from paddle.fluid.framework import _enable_legacy_dygraph, _disable_legacy_dygraph +from paddle.fluid.framework import ( + _enable_legacy_dygraph, + _disable_legacy_dygraph, +) class TestUniformRandomInplaceOpDtype(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) def test_uniform_random_inplace_op_dtype(self): - def test_fp32(): tensor_fp32 = paddle.ones(self.shape, dtype=paddle.float32) tensor_fp32.uniform_() @@ -46,7 +47,6 @@ def test_fp64(): class TestUniformRandomInplaceOpIsInplace(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) @@ -57,7 +57,6 @@ def test_uniform_random_inplace_op_is_inplace(self): class TestUniformRandomInplaceOpSeedIsZero(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) self.seed = 0 @@ -72,7 +71,6 @@ def test_uniform_random_inplace_op_seed_is_zero(self): class TestUniformRandomInplaceOpSeedIsNotZero(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) self.seed = 10 @@ -87,7 +85,6 @@ def test_uniform_random_inplace_op_seed_is_not_zero(self): class TestUniformRandomInplaceOpWithinRange(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) self.min = -2 @@ -98,12 +95,12 @@ def test_uniform_random_inplace_op_within_range(self): tensor = paddle.ones(self.shape) tensor.uniform_(min=self.min, max=self.max, seed=self.seed) tensor_data = tensor.numpy() - self.assertTrue((tensor_data > self.min).all() - and (tensor_data < self.max).all()) + self.assertTrue( + (tensor_data > self.min).all() and (tensor_data < self.max).all() + ) class TestUniformRandomInplaceOpShape(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) @@ -116,7 +113,6 @@ def test_uniform_random_inplace_op_shape(self): class TestUniformRandomInplaceOpDistribution(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) self.min = -3 @@ -130,17 +126,15 @@ def test_uniform_random_inplace_op_distribution(self): hist, _ = np.histogram(tensor.numpy()[0], bins=self.bins) prob = hist / float(self.shape[0]) - prob_expect = np.ones((self.bins, )) / float(self.bins) + prob_expect = np.ones((self.bins,)) / float(self.bins) np.testing.assert_allclose(prob, prob_expect, rtol=0, atol=0.01) class TestUniformRandomInplaceOpError(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) def test_uniform_random_inplace_op_error(self): - def test_attr_error(): tensor = paddle.ones(self.shape) tensor.uniform_(shape=self.shape, min=-2, max=2) @@ -149,7 +143,6 @@ def test_attr_error(): class TestUniformRandomInplaceOpEmptyTensor(unittest.TestCase): - def test_uniform_random_inplace_op_empty_tensor(self): places = ['cpu'] if fluid.core.is_compiled_with_cuda(): @@ -166,7 +159,6 @@ def test_uniform_random_inplace_op_empty_tensor(self): class TestUniformRandomInplaceGrad(unittest.TestCase): - def setUp(self): self.shape = (1000, 784) diff --git a/python/paddle/fluid/tests/unittests/test_uniform_random_op.py b/python/paddle/fluid/tests/unittests/test_uniform_random_op.py index 285e4f4f20b5c..efcbf075bf3fc 100644 --- a/python/paddle/fluid/tests/unittests/test_uniform_random_op.py +++ b/python/paddle/fluid/tests/unittests/test_uniform_random_op.py @@ -49,15 +49,15 @@ def output_hist_diag(out): class TestUniformRandomOp_attr_tensorlist(OpTest): - def setUp(self): self.op_type = "uniform_random" self.python_api = paddle.uniform self.new_shape = (1000, 784) shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype("int64") * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype("int64") * ele) + ) self.inputs = {'ShapeTensorList': shape_tensor} self.init_attrs() self.outputs = {"Out": np.zeros((1000, 784)).astype("float32")} @@ -75,22 +75,21 @@ def verify_output(self, outs): class TestMaxMinAreInt(TestUniformRandomOp_attr_tensorlist): - def init_attrs(self): self.attrs = {"min": -5, "max": 10, "seed": 10} self.output_hist = output_hist class TestUniformRandomOp_attr_tensorlist_int32(OpTest): - def setUp(self): self.op_type = "uniform_random" self.python_api = paddle.uniform self.new_shape = (1000, 784) shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype("int32") * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype("int32") * ele) + ) self.inputs = {'ShapeTensorList': shape_tensor} self.init_attrs() self.outputs = {"Out": np.zeros((1000, 784)).astype("float32")} @@ -108,7 +107,6 @@ def verify_output(self, outs): class TestUniformRandomOp_attr_tensor(OpTest): - def setUp(self): self.op_type = "uniform_random" self.python_api = paddle.uniform @@ -129,7 +127,6 @@ def verify_output(self, outs): class TestUniformRandomOp_attr_tensor_int32(OpTest): - def setUp(self): self.op_type = "uniform_random" self.python_api = paddle.uniform @@ -150,7 +147,6 @@ def verify_output(self, outs): class TestUniformRandomOp(OpTest): - def setUp(self): self.op_type = "uniform_random" self.python_api = paddle.uniform @@ -163,7 +159,7 @@ def init_attrs(self): "shape": [1000, 784], "min": -5.0, "max": 10.0, - "seed": 10 + "seed": 10, } self.output_hist = output_hist @@ -178,9 +174,13 @@ def func_test_check_api(self): places = self._get_places() for place in places: with fluid.dygraph.base.guard(place=place): - out = self.python_api(self.attrs['shape'], 'float32', - self.attrs['min'], self.attrs['max'], - self.attrs['seed']) + out = self.python_api( + self.attrs['shape'], + 'float32', + self.attrs['min'], + self.attrs['max'], + self.attrs['seed'], + ) def test_check_api_eager(self): with _test_eager_guard(): @@ -189,15 +189,15 @@ def test_check_api_eager(self): class TestUniformRandomOpError(unittest.TestCase): - def test_errors(self): main_prog = Program() start_prog = Program() with program_guard(main_prog, start_prog): def test_Variable(): - x1 = fluid.create_lod_tensor(np.zeros((4, 784)), [[1, 1, 1, 1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.zeros((4, 784)), [[1, 1, 1, 1]], fluid.CPUPlace() + ) fluid.layers.uniform_random(x1) self.assertRaises(TypeError, test_Variable) @@ -209,9 +209,9 @@ def test_Variable2(): self.assertRaises(TypeError, test_Variable2) def test_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[4, 784], - dtype='float32') + x2 = fluid.layers.data( + name='x2', shape=[4, 784], dtype='float32' + ) fluid.layers.uniform_random(x2, 'int32') self.assertRaises(TypeError, test_dtype) @@ -224,7 +224,6 @@ def test_out_dtype(): class TestUniformRandomOpWithDiagInit(TestUniformRandomOp): - def init_attrs(self): self.attrs = { "shape": [1000, 784], @@ -233,13 +232,12 @@ def init_attrs(self): "seed": 10, "diag_num": 784, "diag_step": 784, - "diag_val": 1.0 + "diag_val": 1.0, } self.output_hist = output_hist_diag class TestUniformRandomOpSelectedRows(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_cuda(): @@ -254,12 +252,14 @@ def check_with_place(self, place): scope = core.Scope() out = scope.var("X").get_selected_rows() paddle.seed(10) - op = Operator("uniform_random", - Out="X", - shape=[1000, 784], - min=-5.0, - max=10.0, - seed=10) + op = Operator( + "uniform_random", + Out="X", + shape=[1000, 784], + min=-5.0, + max=10.0, + seed=10, + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) hist, prob = output_hist(np.array(out.get_tensor())) @@ -267,21 +267,23 @@ def check_with_place(self, place): class TestUniformRandomOpSelectedRowsWithDiagInit( - TestUniformRandomOpSelectedRows): - + TestUniformRandomOpSelectedRows +): def check_with_place(self, place): scope = core.Scope() out = scope.var("X").get_selected_rows() paddle.seed(10) - op = Operator("uniform_random", - Out="X", - shape=[500, 784], - min=-5.0, - max=10.0, - seed=10, - diag_num=500, - diag_step=784, - diag_val=1.0) + op = Operator( + "uniform_random", + Out="X", + shape=[500, 784], + min=-5.0, + max=10.0, + seed=10, + diag_num=500, + diag_step=784, + diag_val=1.0, + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [500, 784]) hist, prob = output_hist_diag(np.array(out.get_tensor())) @@ -289,29 +291,32 @@ def check_with_place(self, place): class TestUniformRandomOpApi(unittest.TestCase): - def test_api(self): paddle.seed(10) x = fluid.layers.data('x', shape=[16], dtype='float32', lod_level=1) - y = fluid.layers.fc(x, - size=16, - param_attr=fluid.initializer.Uniform(low=-0.5, - high=0.5, - seed=10, - diag_num=16, - diag_step=16, - diag_val=1.0)) + y = fluid.layers.fc( + x, + size=16, + param_attr=fluid.initializer.Uniform( + low=-0.5, + high=0.5, + seed=10, + diag_num=16, + diag_step=16, + diag_val=1.0, + ), + ) place = fluid.CPUPlace() x_tensor = fluid.create_lod_tensor( - np.random.rand(3, 16).astype("float32"), [[1, 2]], place) + np.random.rand(3, 16).astype("float32"), [[1, 2]], place + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) ret = exe.run(feed={'x': x_tensor}, fetch_list=[y], return_numpy=False) class TestUniformRandomOp_attr_tensor_API(unittest.TestCase): - def test_attr_tensor_API(self): startup_program = fluid.Program() train_program = fluid.Program() @@ -356,13 +361,12 @@ def test_attr_tensor_int32_API(self): exe = fluid.Executor(place) Shape = np.array([2, 3]).astype('int32') exe.run(startup_program) - outs = exe.run(train_program, - feed={'shape_tensor': Shape}, - fetch_list=[ret]) + outs = exe.run( + train_program, feed={'shape_tensor': Shape}, fetch_list=[ret] + ) class TestUniformRandomOp_API_seed(unittest.TestCase): - def test_attr_tensor_API(self): _seed = 10 gen = paddle.seed(_seed) @@ -372,14 +376,12 @@ def test_attr_tensor_API(self): _min = 5 _max = 10 - ret = fluid.layers.nn.uniform_random([2, 3, 2], - min=_min, - max=_max, - seed=_seed) - ret_2 = fluid.layers.nn.uniform_random([2, 3, 2], - min=_min, - max=_max, - seed=_seed) + ret = fluid.layers.nn.uniform_random( + [2, 3, 2], min=_min, max=_max, seed=_seed + ) + ret_2 = fluid.layers.nn.uniform_random( + [2, 3, 2], min=_min, max=_max, seed=_seed + ) res = fluid.layers.equal(ret, ret_2) place = fluid.CPUPlace() if fluid.core.is_compiled_with_cuda(): @@ -395,7 +397,6 @@ def test_attr_tensor_API(self): class TestUniformRandomOpSelectedRowsShapeTensor(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_cuda(): @@ -412,12 +413,14 @@ def check_with_place(self, place): shape_tensor = scope.var("Shape").get_tensor() shape_tensor.set(np.array([1000, 784]).astype("int64"), place) paddle.seed(10) - op = Operator("uniform_random", - ShapeTensor="Shape", - Out="X", - min=-5.0, - max=10.0, - seed=10) + op = Operator( + "uniform_random", + ShapeTensor="Shape", + Out="X", + min=-5.0, + max=10.0, + seed=10, + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) hist, prob = output_hist(np.array(out.get_tensor())) @@ -425,7 +428,6 @@ def check_with_place(self, place): class TestUniformRandomOpSelectedRowsShapeTensorList(unittest.TestCase): - def get_places(self): places = [core.CPUPlace()] if core.is_compiled_with_cuda(): @@ -444,12 +446,14 @@ def check_with_place(self, place): shape_2 = scope.var("shape2").get_tensor() shape_2.set(np.array([784]).astype("int64"), place) paddle.seed(10) - op = Operator("uniform_random", - ShapeTensorList=["shape1", "shape2"], - Out="X", - min=-5.0, - max=10.0, - seed=10) + op = Operator( + "uniform_random", + ShapeTensorList=["shape1", "shape2"], + Out="X", + min=-5.0, + max=10.0, + seed=10, + ) op.run(scope, place) self.assertEqual(out.get_tensor().shape(), [1000, 784]) hist, prob = output_hist(np.array(out.get_tensor())) @@ -457,51 +461,48 @@ def check_with_place(self, place): class TestUniformRandomDygraphMode(unittest.TestCase): - def test_check_output(self): with fluid.dygraph.guard(): - x = fluid.layers.uniform_random([10], - dtype="float32", - min=0.0, - max=1.0) + x = fluid.layers.uniform_random( + [10], dtype="float32", min=0.0, max=1.0 + ) x_np = x.numpy() for i in range(10): self.assertTrue((x_np[i] > 0 and x_np[i] < 1.0)) class TestUniformRandomBatchSizeLikeOpError(unittest.TestCase): - def test_errors(self): main_prog = Program() start_prog = Program() with program_guard(main_prog, start_prog): def test_Variable(): - x1 = fluid.create_lod_tensor(np.zeros( - (100, 784)), [[10, 10, 10, 70]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.zeros((100, 784)), [[10, 10, 10, 70]], fluid.CPUPlace() + ) fluid.layers.uniform_random_batch_size_like(x1) self.assertRaises(TypeError, test_Variable) def test_shape(): - x1 = fluid.layers.data(name='x2', - shape=[100, 784], - dtype='float32') + x1 = fluid.layers.data( + name='x2', shape=[100, 784], dtype='float32' + ) fluid.layers.uniform_random_batch_size_like(x1, shape="shape") self.assertRaises(TypeError, test_shape) def test_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[100, 784], - dtype='float32') + x2 = fluid.layers.data( + name='x2', shape=[100, 784], dtype='float32' + ) fluid.layers.uniform_random_batch_size_like(x2, 'int32') self.assertRaises(TypeError, test_dtype) class TestUniformAlias(unittest.TestCase): - def test_alias(self): paddle.uniform([2, 3], min=-5.0, max=5.0) paddle.tensor.uniform([2, 3], min=-5.0, max=5.0) @@ -514,15 +515,15 @@ def test_uniform_random(): class TestUniformOpError(unittest.TestCase): - def test_errors(self): main_prog = Program() start_prog = Program() with program_guard(main_prog, start_prog): def test_Variable(): - x1 = fluid.create_lod_tensor(np.zeros( - (100, 784)), [[10, 10, 10, 70]], fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.zeros((100, 784)), [[10, 10, 10, 70]], fluid.CPUPlace() + ) paddle.tensor.random.uniform(x1) self.assertRaises(TypeError, test_Variable) @@ -534,36 +535,34 @@ def test_Variable2(): self.assertRaises(TypeError, test_Variable2) def test_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[100, 784], - dtype='float32') + x2 = fluid.layers.data( + name='x2', shape=[100, 784], dtype='float32' + ) paddle.tensor.random.uniform(x2, 'int32') self.assertRaises(TypeError, test_dtype) def test_out_dtype(): - out = paddle.tensor.random.uniform(shape=[3, 4], - dtype='float64') + out = paddle.tensor.random.uniform( + shape=[3, 4], dtype='float64' + ) self.assertEqual(out.dtype, fluid.core.VarDesc.VarType.FP64) test_out_dtype() class TestUniformDygraphMode(unittest.TestCase): - def test_check_output(self): with fluid.dygraph.guard(): - x = paddle.tensor.random.uniform([10], - dtype="float32", - min=0.0, - max=1.0) + x = paddle.tensor.random.uniform( + [10], dtype="float32", min=0.0, max=1.0 + ) x_np = x.numpy() for i in range(10): self.assertTrue((x_np[i] > 0 and x_np[i] < 1.0)) class TestUniformDtype(unittest.TestCase): - def test_default_dtype(self): paddle.disable_static() @@ -599,7 +598,6 @@ def test_dygraph_fp16(): class TestRandomValue(unittest.TestCase): - def test_fixed_random_number(self): # Test GPU Fixed random number, which is generated by 'curandStatePhilox4_32_10_t' if not paddle.is_compiled_with_cuda(): @@ -618,37 +616,64 @@ def test_fixed_random_number(self): expect_mean = 0.50000454338820143895816272561205551028251647949218750 expect_std = 0.28867379167297479991560749112977646291255950927734375 expect = [ - 0.55298901, 0.65184678, 0.49375412, 0.57943639, 0.16459608, - 0.67181056, 0.03021481, 0.0238559, 0.07742096, 0.55972187 + 0.55298901, + 0.65184678, + 0.49375412, + 0.57943639, + 0.16459608, + 0.67181056, + 0.03021481, + 0.0238559, + 0.07742096, + 0.55972187, ] out = paddle.rand([32, 3, 1024, 1024], dtype='float64').numpy() self.assertEqual(np.mean(out), expect_mean) self.assertEqual(np.std(out), expect_std) - np.testing.assert_allclose(out[2, 1, 512, 1000:1010], - expect, - rtol=1e-05) + np.testing.assert_allclose( + out[2, 1, 512, 1000:1010], expect, rtol=1e-05 + ) expect_mean = 0.50002604722976684570312500 expect_std = 0.2886914908885955810546875 expect = [ - 0.45320973, 0.17582087, 0.725341, 0.30849215, 0.622257, 0.46352342, - 0.97228295, 0.12771158, 0.286525, 0.9810645 + 0.45320973, + 0.17582087, + 0.725341, + 0.30849215, + 0.622257, + 0.46352342, + 0.97228295, + 0.12771158, + 0.286525, + 0.9810645, ] out = paddle.rand([32, 3, 1024, 1024], dtype='float32').numpy() self.assertEqual(np.mean(out), expect_mean) self.assertEqual(np.std(out), expect_std) - np.testing.assert_allclose(out[2, 1, 512, 1000:1010], - expect, - rtol=1e-05) + np.testing.assert_allclose( + out[2, 1, 512, 1000:1010], expect, rtol=1e-05 + ) expect_mean = 25.11843109130859375 expect_std = 43.370647430419921875 expect = [ - 30.089634, 77.05225, 3.1201615, 68.34072, 59.266724, -25.33281, - 12.973292, 27.41127, -17.412298, 27.931019 + 30.089634, + 77.05225, + 3.1201615, + 68.34072, + 59.266724, + -25.33281, + 12.973292, + 27.41127, + -17.412298, + 27.931019, ] - out = paddle.empty([16, 16, 16, 16], - dtype='float32').uniform_(-50, 100).numpy() + out = ( + paddle.empty([16, 16, 16, 16], dtype='float32') + .uniform_(-50, 100) + .numpy() + ) self.assertEqual(np.mean(out), expect_mean) self.assertEqual(np.std(out), expect_std) np.testing.assert_allclose(out[10, 10, 10, 0:10], expect, rtol=1e-05) @@ -657,7 +682,6 @@ def test_fixed_random_number(self): class TestUniformMinMaxTensor(UnittestBase): - def init_info(self): self.shapes = [[2, 3, 4]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -673,9 +697,9 @@ def test_static(self): min_v = paddle.to_tensor([0.1]) max_v = paddle.to_tensor([0.9]) y = paddle.uniform([2, 3, 10], min=min_v, max=max_v) - z = paddle.fluid.layers.uniform_random([2, 3, 10], - min=min_v, - max=max_v) + z = paddle.fluid.layers.uniform_random( + [2, 3, 10], min=min_v, max=max_v + ) out = feat + y + z diff --git a/python/paddle/fluid/tests/unittests/test_unique.py b/python/paddle/fluid/tests/unittests/test_unique.py index a62f5e2808d9f..6e739bb385950 100644 --- a/python/paddle/fluid/tests/unittests/test_unique.py +++ b/python/paddle/fluid/tests/unittests/test_unique.py @@ -22,7 +22,6 @@ class TestUniqueOp(OpTest): - def setUp(self): self.op_type = "unique" self.init_config() @@ -37,12 +36,11 @@ def init_config(self): self.attrs = {'dtype': int(core.VarDesc.VarType.INT32)} self.outputs = { 'Out': np.array([2, 3, 1, 5], dtype='int64'), - 'Index': np.array([0, 1, 1, 2, 3, 1], dtype='int32') + 'Index': np.array([0, 1, 1, 2, 3, 1], dtype='int32'), } class TestOne(TestUniqueOp): - def init_config(self): self.inputs = { 'X': np.array([2], dtype='int64'), @@ -50,31 +48,29 @@ def init_config(self): self.attrs = {'dtype': int(core.VarDesc.VarType.INT32)} self.outputs = { 'Out': np.array([2], dtype='int64'), - 'Index': np.array([0], dtype='int32') + 'Index': np.array([0], dtype='int32'), } class TestRandom(TestUniqueOp): - def init_config(self): - self.inputs = {'X': np.random.randint(0, 100, (150, ), dtype='int64')} + self.inputs = {'X': np.random.randint(0, 100, (150,), dtype='int64')} self.attrs = {'dtype': int(core.VarDesc.VarType.INT64)} - np_unique, np_index, reverse_index = np.unique(self.inputs['X'], True, - True) + np_unique, np_index, reverse_index = np.unique( + self.inputs['X'], True, True + ) np_tuple = [(np_unique[i], np_index[i]) for i in range(len(np_unique))] np_tuple.sort(key=lambda x: x[1]) target_out = np.array([i[0] for i in np_tuple], dtype='int64') target_index = np.array( - [list(target_out).index(i) for i in self.inputs['X']], - dtype='int64') + [list(target_out).index(i) for i in self.inputs['X']], dtype='int64' + ) self.outputs = {'Out': target_out, 'Index': target_index} class TestUniqueRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.unique([10]) @@ -87,10 +83,10 @@ def test_dtype(): self.assertRaises(TypeError, test_dtype) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestOneGPU(TestUniqueOp): - def init_config(self): self.inputs = { 'X': np.array([2], dtype='int64'), @@ -98,7 +94,7 @@ def init_config(self): self.attrs = {'dtype': int(core.VarDesc.VarType.INT32)} self.outputs = { 'Out': np.array([2], dtype='int64'), - 'Index': np.array([0], dtype='int32') + 'Index': np.array([0], dtype='int32'), } def test_check_output(self): @@ -107,21 +103,22 @@ def test_check_output(self): self.check_output_with_place(place, atol=1e-5) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestRandomGPU(TestUniqueOp): - def init_config(self): - self.inputs = {'X': np.random.randint(0, 100, (150, ), dtype='int64')} + self.inputs = {'X': np.random.randint(0, 100, (150,), dtype='int64')} self.attrs = {'dtype': int(core.VarDesc.VarType.INT64)} - np_unique, np_index, reverse_index = np.unique(self.inputs['X'], True, - True) + np_unique, np_index, reverse_index = np.unique( + self.inputs['X'], True, True + ) np_tuple = [(np_unique[i], np_index[i]) for i in range(len(np_unique))] np_tuple.sort(key=lambda x: x[1]) target_out = np.array([i[0] for i in np_tuple], dtype='int64') target_index = np.array( - [list(target_out).index(i) for i in self.inputs['X']], - dtype='int64') + [list(target_out).index(i) for i in self.inputs['X']], dtype='int64' + ) self.outputs = {'Out': target_out, 'Index': target_index} @@ -132,21 +129,22 @@ def test_check_output(self): class TestSortedUniqueOp(TestUniqueOp): - def init_config(self): self.inputs = {'X': np.array([2, 3, 3, 1, 5, 3], dtype='int64')} - unique, indices, inverse, count = np.unique(self.inputs['X'], - return_index=True, - return_inverse=True, - return_counts=True, - axis=None) + unique, indices, inverse, count = np.unique( + self.inputs['X'], + return_index=True, + return_inverse=True, + return_counts=True, + axis=None, + ) self.attrs = { 'dtype': int(core.VarDesc.VarType.INT32), "return_index": True, "return_inverse": True, "return_counts": True, "axis": None, - "is_sorted": True + "is_sorted": True, } self.outputs = { 'Out': unique, @@ -157,21 +155,22 @@ def init_config(self): class TestUniqueOpAxisNone(TestUniqueOp): - def init_config(self): self.inputs = {'X': np.random.random((4, 7, 10)).astype('float64')} - unique, indices, inverse, counts = np.unique(self.inputs['X'], - return_index=True, - return_inverse=True, - return_counts=True, - axis=None) + unique, indices, inverse, counts = np.unique( + self.inputs['X'], + return_index=True, + return_inverse=True, + return_counts=True, + axis=None, + ) self.attrs = { 'dtype': int(core.VarDesc.VarType.INT32), "return_index": True, "return_inverse": True, "return_counts": True, "axis": None, - "is_sorted": True + "is_sorted": True, } self.outputs = { 'Out': unique, @@ -182,21 +181,22 @@ def init_config(self): class TestUniqueOpAxis1(TestUniqueOp): - def init_config(self): self.inputs = {'X': np.random.random((3, 8, 8)).astype('float64')} - unique, indices, inverse, counts = np.unique(self.inputs['X'], - return_index=True, - return_inverse=True, - return_counts=True, - axis=1) + unique, indices, inverse, counts = np.unique( + self.inputs['X'], + return_index=True, + return_inverse=True, + return_counts=True, + axis=1, + ) self.attrs = { 'dtype': int(core.VarDesc.VarType.INT32), "return_index": True, "return_inverse": True, "return_counts": True, "axis": [1], - "is_sorted": True + "is_sorted": True, } self.outputs = { 'Out': unique, @@ -207,7 +207,6 @@ def init_config(self): class TestUniqueAPI(unittest.TestCase): - def test_dygraph_api_out(self): paddle.disable_static() x_data = x_data = np.random.randint(0, 10, (120)) @@ -221,16 +220,20 @@ def test_dygraph_api_attr(self): paddle.disable_static() x_data = np.random.random((3, 5, 5)).astype("float32") x = paddle.to_tensor(x_data) - out, index, inverse, counts = paddle.unique(x, - return_index=True, - return_inverse=True, - return_counts=True, - axis=0) - np_out, np_index, np_inverse, np_counts = np.unique(x_data, - return_index=True, - return_inverse=True, - return_counts=True, - axis=0) + out, index, inverse, counts = paddle.unique( + x, + return_index=True, + return_inverse=True, + return_counts=True, + axis=0, + ) + np_out, np_index, np_inverse, np_counts = np.unique( + x_data, + return_index=True, + return_inverse=True, + return_counts=True, + axis=0, + ) self.assertTrue((out.numpy() == np_out).all(), True) self.assertTrue((index.numpy() == np_index).all(), True) self.assertTrue((inverse.numpy() == np_inverse).all(), True) @@ -241,13 +244,16 @@ def test_dygraph_attr_dtype(self): paddle.disable_static() x_data = x_data = np.random.randint(0, 10, (120)) x = paddle.to_tensor(x_data) - out, indices, inverse, counts = paddle.unique(x, - return_index=True, - return_inverse=True, - return_counts=True, - dtype="int32") + out, indices, inverse, counts = paddle.unique( + x, + return_index=True, + return_inverse=True, + return_counts=True, + dtype="int32", + ) expected_out, np_indices, np_inverse, np_counts = np.unique( - x_data, return_index=True, return_inverse=True, return_counts=True) + x_data, return_index=True, return_inverse=True, return_counts=True + ) self.assertTrue((out.numpy() == expected_out).all(), True) self.assertTrue((indices.numpy() == np_indices).all(), True) self.assertTrue((inverse.numpy() == np_inverse).all(), True) @@ -261,34 +267,33 @@ def test_dygraph_api(self): self.test_dygraph_attr_dtype() def test_static_graph(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[3, 2], dtype='float64') - unique, inverse, counts = paddle.unique(x, - return_inverse=True, - return_counts=True, - axis=0) + unique, inverse, counts = paddle.unique( + x, return_inverse=True, return_counts=True, axis=0 + ) place = paddle.CPUPlace() exe = paddle.static.Executor(place) x_np = np.array([[1, 2], [3, 4], [1, 2]]).astype('float64') - result = exe.run(feed={"x": x_np}, - fetch_list=[unique, inverse, counts]) - np_unique, np_inverse, np_counts = np.unique(x_np, - return_inverse=True, - return_counts=True, - axis=0) + result = exe.run( + feed={"x": x_np}, fetch_list=[unique, inverse, counts] + ) + np_unique, np_inverse, np_counts = np.unique( + x_np, return_inverse=True, return_counts=True, axis=0 + ) np.testing.assert_allclose(result[0], np_unique, rtol=1e-05) np.testing.assert_allclose(result[1], np_inverse, rtol=1e-05) np.testing.assert_allclose(result[2], np_counts, rtol=1e-05) class TestUniqueError(unittest.TestCase): - def test_input_dtype(self): - def test_x_dtype(): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.fluid.data(name='x', shape=[10, 10], dtype='float16') result = paddle.unique(x) diff --git a/python/paddle/fluid/tests/unittests/test_unique_consecutive_op.py b/python/paddle/fluid/tests/unittests/test_unique_consecutive_op.py index 91a8c6d6a4bf3..b8a3096dbd62f 100644 --- a/python/paddle/fluid/tests/unittests/test_unique_consecutive_op.py +++ b/python/paddle/fluid/tests/unittests/test_unique_consecutive_op.py @@ -50,7 +50,7 @@ def reference_unique_consecutive(X, return_inverse=False, return_counts=False): last += 1 inverse_vec[last] = i if return_counts: - counts_vec = counts_vec[:len(X)] + counts_vec = counts_vec[: len(X)] if return_inverse and return_counts: return X, np.array(inverse_vec), np.array(counts_vec) elif return_counts: @@ -79,8 +79,9 @@ def setUp(self): self.config() self.op_type = "unique_consecutive" x = np.random.randint(self.x_range, size=self.x_size).astype(self.dtype) - result = reference_unique_consecutive(x, self.return_inverse, - self.return_counts) + result = reference_unique_consecutive( + x, self.return_inverse, self.return_counts + ) out = reference_unique_consecutive(x) out = np.array(out).astype(self.dtype) self.inputs = { @@ -111,8 +112,9 @@ def setUp(self): self.config() self.op_type = "unique_consecutive" x = np.random.randint(self.x_range, size=self.x_size).astype(self.dtype) - result, inverse = reference_unique_consecutive(x, self.return_inverse, - self.return_counts) + result, inverse = reference_unique_consecutive( + x, self.return_inverse, self.return_counts + ) result = np.array(result).astype(self.dtype) inverse = inverse.astype(self.dtype) self.inputs = { @@ -120,7 +122,7 @@ def setUp(self): } self.attrs = { 'return_inverse': self.return_inverse, - 'dtype': int(core.VarDesc.VarType.INT32) + 'dtype': int(core.VarDesc.VarType.INT32), } self.python_out_sig = ["Out"] self.outputs = {'Out': result, 'Index': inverse} @@ -141,8 +143,9 @@ def setUp(self): self.config() self.op_type = "unique_consecutive" x = np.random.randint(self.x_range, size=self.x_size).astype(self.dtype) - result, counts = reference_unique_consecutive(x, self.return_inverse, - self.return_counts) + result, counts = reference_unique_consecutive( + x, self.return_inverse, self.return_counts + ) result = np.array(result).astype(self.dtype) counts = counts.astype(self.dtype) self.inputs = { @@ -150,7 +153,7 @@ def setUp(self): } self.attrs = { 'return_counts': self.return_counts, - 'dtype': int(core.VarDesc.VarType.INT32) + 'dtype': int(core.VarDesc.VarType.INT32), } self.python_out_sig = ["Out"] self.outputs = {'Out': result, 'Counts': counts} @@ -172,7 +175,8 @@ def setUp(self): self.op_type = "unique_consecutive" x = np.random.randint(self.x_range, size=self.x_size).astype(self.dtype) result, inverse, counts = reference_unique_consecutive( - x, self.return_inverse, self.return_counts) + x, self.return_inverse, self.return_counts + ) result = np.array(result).astype(self.dtype) inverse = inverse.astype(self.dtype) counts = counts.astype(self.dtype) @@ -182,14 +186,13 @@ def setUp(self): self.attrs = { 'return_inverse': self.return_inverse, 'return_counts': self.return_counts, - 'dtype': int(core.VarDesc.VarType.INT32) + 'dtype': int(core.VarDesc.VarType.INT32), } self.python_out_sig = ["Out"] self.outputs = {'Out': result, 'Index': inverse, 'Counts': counts} class TestUniqueConsecutiveAPI(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -198,15 +201,21 @@ def setUp(self): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): paddle.enable_static() - input_x = fluid.data(name="input_x", shape=[ - 100, - ], dtype="float32") + input_x = fluid.data( + name="input_x", + shape=[ + 100, + ], + dtype="float32", + ) result = paddle.unique_consecutive(input_x) x_np = np.random.randint(20, size=100).astype("float32") exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input_x": x_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": x_np}, + fetch_list=[result], + ) def test_static(self): for place in self.places: @@ -221,7 +230,6 @@ def test_dygraph(self): class TestUniqueConsecutiveCase2API(unittest.TestCase): - def setUp(self): self.places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): @@ -230,16 +238,23 @@ def setUp(self): def check_static_result(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): paddle.enable_static() - input_x = fluid.data(name="input_x", shape=[ - 100, - ], dtype="float32") + input_x = fluid.data( + name="input_x", + shape=[ + 100, + ], + dtype="float32", + ) result, inverse, counts = paddle.unique_consecutive( - input_x, return_inverse=True, return_counts=True) + input_x, return_inverse=True, return_counts=True + ) x_np = np.random.randint(20, size=100).astype("float32") exe = fluid.Executor(place) - fetches = exe.run(fluid.default_main_program(), - feed={"input_x": x_np}, - fetch_list=[result]) + fetches = exe.run( + fluid.default_main_program(), + feed={"input_x": x_np}, + fetch_list=[result], + ) def test_static(self): for place in self.places: @@ -251,7 +266,8 @@ def test_dygraph(self): input_x = np.random.randint(20, size=100).astype("float64") x = paddle.to_tensor(input_x) result, inverse, counts = paddle.unique_consecutive( - x, return_inverse=True, return_counts=True) + x, return_inverse=True, return_counts=True + ) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_unique_name.py b/python/paddle/fluid/tests/unittests/test_unique_name.py index 4393089bab8e4..a423247370d14 100644 --- a/python/paddle/fluid/tests/unittests/test_unique_name.py +++ b/python/paddle/fluid/tests/unittests/test_unique_name.py @@ -17,7 +17,6 @@ class TestUniqueName(unittest.TestCase): - def test_guard(self): with fluid.unique_name.guard(): name_1 = fluid.unique_name.generate('') @@ -45,7 +44,6 @@ def test_generate(self): class TestImperativeUniqueName(unittest.TestCase): - def test_name_generator(self): with fluid.dygraph.guard(): tracer = fluid.framework._dygraph_tracer() diff --git a/python/paddle/fluid/tests/unittests/test_unique_with_counts.py b/python/paddle/fluid/tests/unittests/test_unique_with_counts.py index 30fc203d933fb..6a5b58454b56c 100644 --- a/python/paddle/fluid/tests/unittests/test_unique_with_counts.py +++ b/python/paddle/fluid/tests/unittests/test_unique_with_counts.py @@ -20,7 +20,6 @@ class TestUniqueWithCountsOp(OpTest): - def setUp(self): self.op_type = "unique_with_counts" self.init_config() @@ -36,12 +35,11 @@ def init_config(self): self.outputs = { 'Out': np.array([2, 3, 1, 5], dtype='int64'), 'Index': np.array([0, 1, 1, 2, 3, 1], dtype='int32'), - 'Count': np.array([1, 3, 1, 1], dtype='int32') + 'Count': np.array([1, 3, 1, 1], dtype='int32'), } class TestOne(TestUniqueWithCountsOp): - def init_config(self): self.inputs = { 'X': np.array([2], dtype='int64'), @@ -50,24 +48,24 @@ def init_config(self): self.outputs = { 'Out': np.array([2], dtype='int64'), 'Index': np.array([0], dtype='int32'), - 'Count': np.array([1], dtype='int32') + 'Count': np.array([1], dtype='int32'), } class TestRandom(TestUniqueWithCountsOp): - def init_config(self): - input_data = np.random.randint(0, 100, (2000, ), dtype='int64') + input_data = np.random.randint(0, 100, (2000,), dtype='int64') self.inputs = {'X': input_data} self.attrs = {'dtype': int(core.VarDesc.VarType.INT64)} - np_unique, np_index, reverse_index = np.unique(self.inputs['X'], True, - True) + np_unique, np_index, reverse_index = np.unique( + self.inputs['X'], True, True + ) np_tuple = [(np_unique[i], np_index[i]) for i in range(len(np_unique))] np_tuple.sort(key=lambda x: x[1]) target_out = np.array([i[0] for i in np_tuple], dtype='int64') target_index = np.array( - [list(target_out).index(i) for i in self.inputs['X']], - dtype='int64') + [list(target_out).index(i) for i in self.inputs['X']], dtype='int64' + ) count = [0 for i in range(len(np_unique))] for i in range(target_index.shape[0]): count[target_index[i]] += 1 @@ -75,14 +73,12 @@ def init_config(self): self.outputs = { 'Out': target_out, 'Index': target_index, - 'Count': target_count + 'Count': target_count, } class TestUniqueWithCountsRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.unique_with_counts([10]) @@ -95,10 +91,10 @@ def test_dtype(): self.assertRaises(TypeError, test_dtype) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestOneGPU(TestUniqueWithCountsOp): - def init_config(self): self.inputs = { 'X': np.array([2], dtype='int64'), @@ -107,7 +103,7 @@ def init_config(self): self.outputs = { 'Out': np.array([2], dtype='int64'), 'Index': np.array([0], dtype='int32'), - 'Count': np.array([1], dtype='int32') + 'Count': np.array([1], dtype='int32'), } def test_check_output(self): @@ -116,22 +112,23 @@ def test_check_output(self): self.check_output_with_place(place, atol=1e-5) -@unittest.skipIf(not core.is_compiled_with_cuda(), - "core is not compiled with CUDA") +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestRandomGPU(TestUniqueWithCountsOp): - def init_config(self): - input_data = np.random.randint(0, 100, (2000, ), dtype='int64') + input_data = np.random.randint(0, 100, (2000,), dtype='int64') self.inputs = {'X': input_data} self.attrs = {'dtype': int(core.VarDesc.VarType.INT64)} - np_unique, np_index, reverse_index = np.unique(self.inputs['X'], True, - True) + np_unique, np_index, reverse_index = np.unique( + self.inputs['X'], True, True + ) np_tuple = [(np_unique[i], np_index[i]) for i in range(len(np_unique))] np_tuple.sort(key=lambda x: x[1]) target_out = np.array([i[0] for i in np_tuple], dtype='int64') target_index = np.array( - [list(target_out).index(i) for i in self.inputs['X']], - dtype='int64') + [list(target_out).index(i) for i in self.inputs['X']], dtype='int64' + ) count = [0 for i in range(len(np_unique))] for i in range(target_index.shape[0]): count[target_index[i]] += 1 @@ -139,7 +136,7 @@ def init_config(self): self.outputs = { 'Out': target_out, 'Index': target_index, - 'Count': target_count + 'Count': target_count, } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/test_unpool1d_op.py b/python/paddle/fluid/tests/unittests/test_unpool1d_op.py index f15d295ce9ebc..787d9367f5890 100644 --- a/python/paddle/fluid/tests/unittests/test_unpool1d_op.py +++ b/python/paddle/fluid/tests/unittests/test_unpool1d_op.py @@ -25,8 +25,11 @@ def _unpool_output_size(x, kernel_size, stride, padding, output_size): input_size = x.shape default_size = [] for d in range(len(kernel_size)): - default_size.append((input_size[-len(kernel_size) + d] - 1) * - stride[d] + kernel_size[d] - 2 * padding[d]) + default_size.append( + (input_size[-len(kernel_size) + d] - 1) * stride[d] + + kernel_size[d] + - 2 * padding[d] + ) if output_size is None: ret = default_size else: @@ -34,11 +37,13 @@ def _unpool_output_size(x, kernel_size, stride, padding, output_size): return ret -def unpool1dmax_forward_naive(input, indices, ksize, strides, paddings, - output_size): +def unpool1dmax_forward_naive( + input, indices, ksize, strides, paddings, output_size +): s0, s1, s2 = input.shape - output_size = _unpool_output_size(input, ksize, strides, paddings, - output_size) + output_size = _unpool_output_size( + input, ksize, strides, paddings, output_size + ) out_lsize = output_size[0] out = np.zeros((s0, s1, out_lsize)) for nidx in range(s0): @@ -52,7 +57,6 @@ def unpool1dmax_forward_naive(input, indices, ksize, strides, paddings, class TestUnpool1DOpAPI_dygraph(unittest.TestCase): - def test_case(self): places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): @@ -61,25 +65,23 @@ def test_case(self): paddle.disable_static() input_data = np.random.rand(1, 3, 16) input_x = paddle.to_tensor(input_data) - output, indices = F.max_pool1d(input_x, - kernel_size=2, - stride=2, - return_mask=True) - output_unpool = F.max_unpool1d(output, - indices, - kernel_size=2, - stride=2) + output, indices = F.max_pool1d( + input_x, kernel_size=2, stride=2, return_mask=True + ) + output_unpool = F.max_unpool1d( + output, indices, kernel_size=2, stride=2 + ) expected_output_unpool = unpool1dmax_forward_naive( - output.numpy(), indices.numpy(), [2], [2], [0], [16]) - np.testing.assert_allclose(output_unpool.numpy(), - expected_output_unpool, - rtol=1e-05) + output.numpy(), indices.numpy(), [2], [2], [0], [16] + ) + np.testing.assert_allclose( + output_unpool.numpy(), expected_output_unpool, rtol=1e-05 + ) paddle.enable_static() class TestUnpool1DOpAPI_dygraph2(unittest.TestCase): - def test_case(self): places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): @@ -88,25 +90,23 @@ def test_case(self): paddle.disable_static() input_data = np.random.rand(1, 3, 16) input_x = paddle.to_tensor(input_data) - output, indices = F.max_pool1d(input_x, - kernel_size=2, - stride=2, - return_mask=True) - output_unpool = F.max_unpool1d(output, - indices, - kernel_size=2, - stride=None) + output, indices = F.max_pool1d( + input_x, kernel_size=2, stride=2, return_mask=True + ) + output_unpool = F.max_unpool1d( + output, indices, kernel_size=2, stride=None + ) expected_output_unpool = unpool1dmax_forward_naive( - output.numpy(), indices.numpy(), [2], [2], [0], [16]) - np.testing.assert_allclose(output_unpool.numpy(), - expected_output_unpool, - rtol=1e-05) + output.numpy(), indices.numpy(), [2], [2], [0], [16] + ) + np.testing.assert_allclose( + output_unpool.numpy(), expected_output_unpool, rtol=1e-05 + ) paddle.enable_static() class TestUnpool1DOpAPI_dygraph3(unittest.TestCase): - def test_case(self): places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): @@ -115,61 +115,66 @@ def test_case(self): paddle.disable_static() input_data = np.random.rand(1, 3, 16) input_x = paddle.to_tensor(input_data) - Pool1d = paddle.nn.MaxPool1D(kernel_size=2, - stride=2, - return_mask=True) + Pool1d = paddle.nn.MaxPool1D( + kernel_size=2, stride=2, return_mask=True + ) UnPool1d = paddle.nn.MaxUnPool1D(kernel_size=2, stride=2) output, indices = Pool1d(input_x) output_unpool = UnPool1d(output, indices) expected_output_unpool = unpool1dmax_forward_naive( - output.numpy(), indices.numpy(), [2], [2], [0], [16]) - np.testing.assert_allclose(output_unpool.numpy(), - expected_output_unpool, - rtol=1e-05) + output.numpy(), indices.numpy(), [2], [2], [0], [16] + ) + np.testing.assert_allclose( + output_unpool.numpy(), expected_output_unpool, rtol=1e-05 + ) paddle.enable_static() class TestUnpool1DOpAPI_static(unittest.TestCase): - def test_case(self): paddle.enable_static() places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): places.append(paddle.CUDAPlace(0)) for place in places: - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - - input_data = np.array([[[1, 2, 3, 4], [5, 6, 7, 8], - [9, 10, 11, 12]]]).astype("float32") - x = paddle.fluid.data(name='x', - shape=[1, 3, 4], - dtype='float32') - output, indices = F.max_pool1d(x, - kernel_size=2, - stride=2, - return_mask=True) - output_unpool = F.max_unpool1d(output, - indices, - kernel_size=2, - stride=None) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + + input_data = np.array( + [[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]] + ).astype("float32") + x = paddle.fluid.data( + name='x', shape=[1, 3, 4], dtype='float32' + ) + output, indices = F.max_pool1d( + x, kernel_size=2, stride=2, return_mask=True + ) + output_unpool = F.max_unpool1d( + output, indices, kernel_size=2, stride=None + ) exe = paddle.fluid.Executor(place) - fetches = exe.run(paddle.fluid.default_main_program(), - feed={"x": input_data}, - fetch_list=[output_unpool], - return_numpy=True) - pool1d_out_np = np.array([[[2., 4.], [6., 8.], - [10., 12.]]]).astype("float32") - indices_np = np.array([[[1, 3], [1, 3], [1, - 3]]]).astype("int32") + fetches = exe.run( + paddle.fluid.default_main_program(), + feed={"x": input_data}, + fetch_list=[output_unpool], + return_numpy=True, + ) + pool1d_out_np = np.array( + [[[2.0, 4.0], [6.0, 8.0], [10.0, 12.0]]] + ).astype("float32") + indices_np = np.array([[[1, 3], [1, 3], [1, 3]]]).astype( + "int32" + ) expected_output_unpool = unpool1dmax_forward_naive( - pool1d_out_np, indices_np, [2], [2], [0], [4]) - np.testing.assert_allclose(fetches[0], - expected_output_unpool, - rtol=1e-05) + pool1d_out_np, indices_np, [2], [2], [0], [4] + ) + np.testing.assert_allclose( + fetches[0], expected_output_unpool, rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_unpool3d_op.py b/python/paddle/fluid/tests/unittests/test_unpool3d_op.py index 5c3411d6899aa..3db2d1514caaf 100644 --- a/python/paddle/fluid/tests/unittests/test_unpool3d_op.py +++ b/python/paddle/fluid/tests/unittests/test_unpool3d_op.py @@ -26,8 +26,11 @@ def _unpool_output_size(x, kernel_size, stride, padding, output_size): input_size = x.shape default_size = [] for d in range(len(kernel_size)): - default_size.append((input_size[-len(kernel_size) + d] - 1) * - stride[d] + kernel_size[d] - 2 * padding[d]) + default_size.append( + (input_size[-len(kernel_size) + d] - 1) * stride[d] + + kernel_size[d] + - 2 * padding[d] + ) if output_size is None: ret = default_size else: @@ -35,11 +38,13 @@ def _unpool_output_size(x, kernel_size, stride, padding, output_size): return ret -def unpool3dmax_forward_naive(input, indices, ksize, strides, paddings, - output_size): +def unpool3dmax_forward_naive( + input, indices, ksize, strides, paddings, output_size +): s0, s1, s2, s3, s4 = input.shape - output_size = _unpool_output_size(input, ksize, strides, paddings, - output_size) + output_size = _unpool_output_size( + input, ksize, strides, paddings, output_size + ) out_dsize = output_size[0] out_hsize = output_size[1] out_wsize = output_size[2] @@ -51,49 +56,58 @@ def unpool3dmax_forward_naive(input, indices, ksize, strides, paddings, for w in range(s4): index = indices[nidx, cidx, d, h, w] didx = index // (out_wsize * out_hsize) - hidx = (index - - didx * out_hsize * out_wsize) // out_wsize - widx = (index - - didx * out_hsize * out_wsize) % out_wsize - out[nidx, cidx, didx, hidx, widx] = \ - input[nidx, cidx, d, h, w] + hidx = ( + index - didx * out_hsize * out_wsize + ) // out_wsize + widx = ( + index - didx * out_hsize * out_wsize + ) % out_wsize + out[nidx, cidx, didx, hidx, widx] = input[ + nidx, cidx, d, h, w + ] return out -def max_unpool3d_wrapper(x, - indices, - kernel_size, - stride=None, - padding=0, - output_size=None, - data_format="NCDHW", - name=None): - out = paddle.nn.functional.max_unpool3d(x, - indices, - kernel_size, - stride=stride, - padding=padding, - data_format=data_format, - output_size=output_size, - name=name) +def max_unpool3d_wrapper( + x, + indices, + kernel_size, + stride=None, + padding=0, + output_size=None, + data_format="NCDHW", + name=None, +): + out = paddle.nn.functional.max_unpool3d( + x, + indices, + kernel_size, + stride=stride, + padding=padding, + data_format=data_format, + output_size=output_size, + name=name, + ) return out class TestUnpool3DOp(OpTest): - def setUp(self): self.op_type = "unpool3d" self.python_api = max_unpool3d_wrapper self.init_test_case() inputs = np.random.randint(0, 100, self.shape) nsize, csize, dsize, hsize, wsize = inputs.shape - self.output_size = _unpool_output_size(inputs, self.ksize, self.strides, - self.paddings, self.output_size) + self.output_size = _unpool_output_size( + inputs, self.ksize, self.strides, self.paddings, self.output_size + ) indices = np.random.permutation( np.arange( - 0, self.output_size[0] * self.output_size[1] * - self.output_size[2]))[:dsize * hsize * wsize] + 0, + self.output_size[0] * self.output_size[1] * self.output_size[2], + ) + )[: dsize * hsize * wsize] indices = np.reshape(indices, [dsize, hsize, wsize]) idx_list = [] for n in range(nsize): @@ -103,12 +117,18 @@ def setUp(self): idx_list.append(c_list) indices = np.array(idx_list) - output = self.unpool3d_forward_naive(inputs, indices, self.ksize, \ - self.strides, self.paddings, self.output_size).astype("float64") + output = self.unpool3d_forward_naive( + inputs, + indices, + self.ksize, + self.strides, + self.paddings, + self.output_size, + ).astype("float64") self.inputs = { 'X': inputs.astype('float64'), - 'Indices': indices.astype('int32') + 'Indices': indices.astype('int32'), } self.attrs = { 'strides': self.strides, @@ -136,7 +156,6 @@ def init_test_case(self): class TestUnpool3DOpcase1(TestUnpool3DOp): - def init_test_case(self): self.unpool3d_forward_naive = unpool3dmax_forward_naive self.unpooling_type = "max" @@ -148,7 +167,6 @@ def init_test_case(self): class TestUnpool3DOpOutput(TestUnpool3DOp): - def init_test_case(self): self.unpool3d_forward_naive = unpool3dmax_forward_naive self.unpooling_type = "max" @@ -160,50 +178,51 @@ def init_test_case(self): class TestUnpool3DOpException(unittest.TestCase): - def test_exception(self): - def indices_size_error(): data = paddle.randint(shape=[1, 1, 3, 3, 3]) - indices = paddle.reshape(paddle.arange(0, 36), - shape=[1, 1, 3, 3, 4]) + indices = paddle.reshape( + paddle.arange(0, 36), shape=[1, 1, 3, 3, 4] + ) MaxUnPool3D = F.maxunpool3d(data, indices, kernel_size=2, stride=2) def indices_value_error(): data = paddle.randint(shape=[1, 1, 3, 3, 3]) - indices = paddle.reshape(paddle.arange(4, 40), - shape=[1, 1, 3, 3, 3]) + indices = paddle.reshape( + paddle.arange(4, 40), shape=[1, 1, 3, 3, 3] + ) MaxUnPool3D = F.maxunpool3d(data, indices, kernel_size=2, stride=2) def data_format_error(): data = paddle.randint(shape=[1, 1, 3, 3, 3]) - indices = paddle.reshape(paddle.arange(0, 27), - shape=[1, 1, 3, 3, 3]) - MaxUnPool3D = F.maxunpool3d(data, - indices, - kernel_size=2, - stride=2, - data_format="NDHWC") + indices = paddle.reshape( + paddle.arange(0, 27), shape=[1, 1, 3, 3, 3] + ) + MaxUnPool3D = F.maxunpool3d( + data, indices, kernel_size=2, stride=2, data_format="NDHWC" + ) def data_outputsize_error(): data = paddle.randint(shape=[1, 1, 3, 3, 3]) - indices = paddle.reshape(paddle.arange(0, 27), - shape=[1, 1, 3, 3, 3]) - MaxUnPool3D = F.maxunpool3d(data, - indices, - kernel_size=2, - stride=2, - output_size=[2, 2, 3, 4, 5]) + indices = paddle.reshape( + paddle.arange(0, 27), shape=[1, 1, 3, 3, 3] + ) + MaxUnPool3D = F.maxunpool3d( + data, + indices, + kernel_size=2, + stride=2, + output_size=[2, 2, 3, 4, 5], + ) def data_outputsize_error2(): data = paddle.randint(shape=[1, 1, 3, 3, 3]) - indices = paddle.reshape(paddle.arange(0, 27), - shape=[1, 1, 3, 3, 3]) - MaxUnPool3D = F.maxunpool3d(data, - indices, - kernel_size=2, - stride=2, - output_size=[10, 10, 10]) + indices = paddle.reshape( + paddle.arange(0, 27), shape=[1, 1, 3, 3, 3] + ) + MaxUnPool3D = F.maxunpool3d( + data, indices, kernel_size=2, stride=2, output_size=[10, 10, 10] + ) self.assertRaises(ValueError, indices_size_error) self.assertRaises(ValueError, indices_value_error) @@ -213,7 +232,6 @@ def data_outputsize_error2(): class TestUnpool3DOpAPI_dygraph(unittest.TestCase): - def test_case(self): places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): @@ -222,26 +240,28 @@ def test_case(self): paddle.disable_static() input_data = np.random.rand(1, 3, 4, 4, 6) input_x = paddle.to_tensor(input_data) - output, indices = F.max_pool3d(input_x, - kernel_size=2, - stride=2, - return_mask=True) - output_unpool = F.max_unpool3d(output, - indices, - kernel_size=2, - stride=2) + output, indices = F.max_pool3d( + input_x, kernel_size=2, stride=2, return_mask=True + ) + output_unpool = F.max_unpool3d( + output, indices, kernel_size=2, stride=2 + ) expected_output_unpool = unpool3dmax_forward_naive( - output.numpy(), indices.numpy(), [2, 2, 2], [2, 2, 2], - [0, 0, 0], [4, 4, 6]) - np.testing.assert_allclose(output_unpool.numpy(), - expected_output_unpool, - rtol=1e-05) + output.numpy(), + indices.numpy(), + [2, 2, 2], + [2, 2, 2], + [0, 0, 0], + [4, 4, 6], + ) + np.testing.assert_allclose( + output_unpool.numpy(), expected_output_unpool, rtol=1e-05 + ) paddle.enable_static() class TestUnpool3DOpAPI_dygraph2(unittest.TestCase): - def test_case(self): places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): @@ -250,26 +270,28 @@ def test_case(self): paddle.disable_static() input_data = np.random.rand(1, 3, 4, 4, 6) input_x = paddle.to_tensor(input_data) - output, indices = F.max_pool3d(input_x, - kernel_size=2, - stride=2, - return_mask=True) - output_unpool = F.max_unpool3d(output, - indices, - kernel_size=2, - stride=None) + output, indices = F.max_pool3d( + input_x, kernel_size=2, stride=2, return_mask=True + ) + output_unpool = F.max_unpool3d( + output, indices, kernel_size=2, stride=None + ) expected_output_unpool = unpool3dmax_forward_naive( - output.numpy(), indices.numpy(), [2, 2, 2], [2, 2, 2], - [0, 0, 0], [4, 4, 6]) - np.testing.assert_allclose(output_unpool.numpy(), - expected_output_unpool, - rtol=1e-05) + output.numpy(), + indices.numpy(), + [2, 2, 2], + [2, 2, 2], + [0, 0, 0], + [4, 4, 6], + ) + np.testing.assert_allclose( + output_unpool.numpy(), expected_output_unpool, rtol=1e-05 + ) paddle.enable_static() class TestUnpool3DOpAPI_dygraph3(unittest.TestCase): - def test_case(self): places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): @@ -278,63 +300,91 @@ def test_case(self): paddle.disable_static() input_data = np.random.rand(1, 3, 4, 4, 6) input_x = paddle.to_tensor(input_data) - Pool3d = paddle.nn.MaxPool3D(kernel_size=2, - stride=2, - return_mask=True) + Pool3d = paddle.nn.MaxPool3D( + kernel_size=2, stride=2, return_mask=True + ) UnPool3d = paddle.nn.MaxUnPool3D(kernel_size=2, stride=2) output, indices = Pool3d(input_x) output_unpool = UnPool3d(output, indices) expected_output_unpool = unpool3dmax_forward_naive( - output.numpy(), indices.numpy(), [2, 2, 2], [2, 2, 2], - [0, 0, 0], [4, 4, 6]) - np.testing.assert_allclose(output_unpool.numpy(), - expected_output_unpool, - rtol=1e-05) + output.numpy(), + indices.numpy(), + [2, 2, 2], + [2, 2, 2], + [0, 0, 0], + [4, 4, 6], + ) + np.testing.assert_allclose( + output_unpool.numpy(), expected_output_unpool, rtol=1e-05 + ) paddle.enable_static() class TestUnpool3DOpAPI_static(unittest.TestCase): - def test_case(self): paddle.enable_static() places = [paddle.CPUPlace()] if paddle.fluid.core.is_compiled_with_cuda(): places.append(paddle.CUDAPlace(0)) for place in places: - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): - - input_data = np.array([[[[[1, 2, 3, 4], [5, 6, 7, 8], \ - [9, 10, 11, 12], [13, 14, 15, 16]], [[1, 2, 3, 4], [5, 6, 7, 8], \ - [9, 10, 11, 12], [13, 14, 15, 16]]]]]).astype("float32") - x = paddle.fluid.data(name='x', - shape=[1, 1, 2, 4, 4], - dtype='float32') - output, indices = F.max_pool3d(x, - kernel_size=2, - stride=2, - return_mask=True) - output_unpool = F.max_unpool3d(output, - indices, - kernel_size=2, - stride=None) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + + input_data = np.array( + [ + [ + [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + [13, 14, 15, 16], + ], + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + [13, 14, 15, 16], + ], + ] + ] + ] + ).astype("float32") + x = paddle.fluid.data( + name='x', shape=[1, 1, 2, 4, 4], dtype='float32' + ) + output, indices = F.max_pool3d( + x, kernel_size=2, stride=2, return_mask=True + ) + output_unpool = F.max_unpool3d( + output, indices, kernel_size=2, stride=None + ) exe = paddle.fluid.Executor(place) - fetches = exe.run(paddle.fluid.default_main_program(), - feed={"x": input_data}, - fetch_list=[output_unpool], - return_numpy=True) - pool3d_out_np = np.array([[[[[6., 8.], - [14., 16.]]]]]).astype("float32") + fetches = exe.run( + paddle.fluid.default_main_program(), + feed={"x": input_data}, + fetch_list=[output_unpool], + return_numpy=True, + ) + pool3d_out_np = np.array( + [[[[[6.0, 8.0], [14.0, 16.0]]]]] + ).astype("float32") indices_np = np.array([[[[[5, 7], [13, 15]]]]]).astype("int32") expected_output_unpool = unpool3dmax_forward_naive( - pool3d_out_np, indices_np, [2, 2, 2], [2, 2, 2], [0, 0, 0], - [2, 4, 4]) - np.testing.assert_allclose(fetches[0], - expected_output_unpool, - rtol=1e-05) + pool3d_out_np, + indices_np, + [2, 2, 2], + [2, 2, 2], + [0, 0, 0], + [2, 4, 4], + ) + np.testing.assert_allclose( + fetches[0], expected_output_unpool, rtol=1e-05 + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_unpool_op.py b/python/paddle/fluid/tests/unittests/test_unpool_op.py index 7c59d628e81b2..00c48e0872df8 100644 --- a/python/paddle/fluid/tests/unittests/test_unpool_op.py +++ b/python/paddle/fluid/tests/unittests/test_unpool_op.py @@ -27,8 +27,11 @@ def _unpool_output_size(x, kernel_size, stride, padding, output_size): input_size = x.shape default_size = [] for d in range(len(kernel_size)): - default_size.append((input_size[-len(kernel_size) + d] - 1) * - stride[d] + kernel_size[d] - 2 * padding[d]) + default_size.append( + (input_size[-len(kernel_size) + d] - 1) * stride[d] + + kernel_size[d] + - 2 * padding[d] + ) if output_size is None: ret = default_size else: @@ -36,11 +39,13 @@ def _unpool_output_size(x, kernel_size, stride, padding, output_size): return ret -def unpool2dmax_forward_naive(input, indices, ksize, strides, paddings, - output_size): +def unpool2dmax_forward_naive( + input, indices, ksize, strides, paddings, output_size +): s0, s1, s2, s3 = input.shape - output_size = _unpool_output_size(input, ksize, strides, paddings, - output_size) + output_size = _unpool_output_size( + input, ksize, strides, paddings, output_size + ) out_hsize = output_size[0] out_wsize = output_size[1] out = np.zeros((s0, s1, out_hsize, out_wsize)) @@ -51,44 +56,47 @@ def unpool2dmax_forward_naive(input, indices, ksize, strides, paddings, index = indices[nidx, cidx, h, w] hidx = (index - index % out_wsize) // out_wsize widx = index % out_wsize - out[nidx, cidx, hidx, widx] = \ - input[nidx, cidx, h, w] + out[nidx, cidx, hidx, widx] = input[nidx, cidx, h, w] return out -def max_unpool2d_wrapper(x, - indices, - kernel_size, - stride=None, - padding=0, - output_size=None, - data_format="NCHW", - name=None): - out = paddle.nn.functional.max_unpool2d(x, - indices, - kernel_size, - stride=stride, - padding=padding, - data_format=data_format, - output_size=output_size, - name=name) +def max_unpool2d_wrapper( + x, + indices, + kernel_size, + stride=None, + padding=0, + output_size=None, + data_format="NCHW", + name=None, +): + out = paddle.nn.functional.max_unpool2d( + x, + indices, + kernel_size, + stride=stride, + padding=padding, + data_format=data_format, + output_size=output_size, + name=name, + ) return out class TestUnpoolOp(OpTest): - def setUp(self): self.op_type = "unpool" self.python_api = max_unpool2d_wrapper self.init_test_case() input = np.random.randint(0, 100, self.shape) nsize, csize, hsize, wsize = input.shape - self.output_size = _unpool_output_size(input, self.ksize, self.strides, - self.paddings, self.output_size) + self.output_size = _unpool_output_size( + input, self.ksize, self.strides, self.paddings, self.output_size + ) indices = np.random.permutation( - np.arange(0, self.output_size[0] * self.output_size[1]))[:hsize * - wsize] + np.arange(0, self.output_size[0] * self.output_size[1]) + )[: hsize * wsize] indices = np.reshape(indices, [hsize, wsize]) idx_list = [] for n in range(nsize): @@ -98,12 +106,18 @@ def setUp(self): idx_list.append(c_list) indices = np.array(idx_list) - output = self.unpool2d_forward_naive(input, indices, self.ksize, \ - self.strides, self.paddings, self.output_size).astype("float64") + output = self.unpool2d_forward_naive( + input, + indices, + self.ksize, + self.strides, + self.paddings, + self.output_size, + ).astype("float64") self.inputs = { 'X': input.astype('float64'), - 'Indices': indices.astype('int32') + 'Indices': indices.astype('int32'), } self.attrs = { 'strides': self.strides, @@ -131,7 +145,6 @@ def init_test_case(self): class TestUnpoolOpcase1(TestUnpoolOp): - def init_test_case(self): self.unpool2d_forward_naive = unpool2dmax_forward_naive self.unpooling_type = "max" @@ -143,7 +156,6 @@ def init_test_case(self): class TestUnpoolOpOutputsize(TestUnpoolOp): - def init_test_case(self): self.unpool2d_forward_naive = unpool2dmax_forward_naive self.unpooling_type = "max" @@ -155,7 +167,6 @@ def init_test_case(self): class TestUnpoolOpOutput(TestUnpoolOp): - def init_test_case(self): self.unpool2d_forward_naive = unpool2dmax_forward_naive self.unpooling_type = "max" @@ -167,7 +178,6 @@ def init_test_case(self): class TestUnpoolOpException(unittest.TestCase): - def test_exception(self): import paddle.nn.functional as F import paddle @@ -185,29 +195,23 @@ def indices_value_error(): def data_format_error(): data = paddle.randint(shape=[1, 1, 3, 3]) indices = paddle.reshape(paddle.arange(4, 40), shape[1, 1, 3, 4]) - MaxPool2D = F.maxunpool2d(data, - indices, - kernel_size=2, - stride=2, - data_format="NHWC") + MaxPool2D = F.maxunpool2d( + data, indices, kernel_size=2, stride=2, data_format="NHWC" + ) def data_outputsize_error(): data = paddle.randint(shape=[1, 1, 3, 3]) indices = paddle.reshape(paddle.arange(4, 40), shape[1, 1, 3, 4]) - MaxPool2D = F.maxunpool2d(data, - indices, - kernel_size=2, - stride=2, - output_size=[5, 6, 7, 8]) + MaxPool2D = F.maxunpool2d( + data, indices, kernel_size=2, stride=2, output_size=[5, 6, 7, 8] + ) def data_outputsize_error2(): data = paddle.randint(shape=[1, 1, 3, 3]) indices = paddle.reshape(paddle.arange(4, 40), shape[1, 1, 3, 4]) - MaxPool2D = F.maxunpool2d(data, - indices, - kernel_size=2, - stride=2, - output_size=[100, 100]) + MaxPool2D = F.maxunpool2d( + data, indices, kernel_size=2, stride=2, output_size=[100, 100] + ) self.assertRaises(ValueError, indices_size_error) self.assertRaises(ValueError, indices_value_error) @@ -217,7 +221,6 @@ def data_outputsize_error2(): class TestUnpoolOpAPI_dy(unittest.TestCase): - def test_case(self): import paddle import paddle.nn.functional as F @@ -230,28 +233,34 @@ def test_case(self): else: place = core.CPUPlace() with fluid.dygraph.guard(place): - input_data = np.array([[[[1, 2, 3, 4], [5, 6, 7, - 8], [9, 10, 11, 12], - [13, 14, 15, 16]]]]).astype("float32") + input_data = np.array( + [ + [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + [13, 14, 15, 16], + ] + ] + ] + ).astype("float32") input_x = paddle.to_tensor(input_data) - output, indices = F.max_pool2d(input_x, - kernel_size=2, - stride=2, - return_mask=True) - out_pp = F.max_unpool2d(output, - indices, - kernel_size=2, - stride=2, - output_size=(5, 5)) + output, indices = F.max_pool2d( + input_x, kernel_size=2, stride=2, return_mask=True + ) + out_pp = F.max_unpool2d( + output, indices, kernel_size=2, stride=2, output_size=(5, 5) + ) output_np = output.numpy() indices_np = indices.numpy() - expect_res =unpool2dmax_forward_naive(output_np, indices_np, [2,2], \ - [2,2], [0,0], [5,5]).astype("float64") + expect_res = unpool2dmax_forward_naive( + output_np, indices_np, [2, 2], [2, 2], [0, 0], [5, 5] + ).astype("float64") np.testing.assert_allclose(out_pp.numpy(), expect_res, rtol=1e-05) class TestUnpoolOpAPI_dy2(unittest.TestCase): - def test_case(self): import paddle import paddle.nn.functional as F @@ -264,28 +273,34 @@ def test_case(self): else: place = core.CPUPlace() with fluid.dygraph.guard(place): - input_data = np.array([[[[1, 2, 3, 4], [5, 6, 7, - 8], [9, 10, 11, 12], - [13, 14, 15, 16]]]]).astype("float32") + input_data = np.array( + [ + [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + [13, 14, 15, 16], + ] + ] + ] + ).astype("float32") input_x = paddle.to_tensor(input_data) - output, indices = F.max_pool2d(input_x, - kernel_size=2, - stride=2, - return_mask=True) - out_pp = F.max_unpool2d(output, - indices, - kernel_size=2, - stride=None, - output_size=(5, 5)) + output, indices = F.max_pool2d( + input_x, kernel_size=2, stride=2, return_mask=True + ) + out_pp = F.max_unpool2d( + output, indices, kernel_size=2, stride=None, output_size=(5, 5) + ) output_np = output.numpy() indices_np = indices.numpy() - expect_res =unpool2dmax_forward_naive(output_np, indices_np, [2,2], \ - [2,2], [0,0], [5,5]).astype("float64") + expect_res = unpool2dmax_forward_naive( + output_np, indices_np, [2, 2], [2, 2], [0, 0], [5, 5] + ).astype("float64") np.testing.assert_allclose(out_pp.numpy(), expect_res, rtol=1e-05) class TestUnpoolOpAPI_dy3(unittest.TestCase): - def test_case(self): import paddle import paddle.fluid.core as core @@ -297,46 +312,54 @@ def test_case(self): else: place = core.CPUPlace() with fluid.dygraph.guard(place): - input_data = np.array([[[[1, 2, 3, 4], [5, 6, 7, - 8], [9, 10, 11, 12], - [13, 14, 15, 16]]]]).astype("float32") + input_data = np.array( + [ + [ + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + [13, 14, 15, 16], + ] + ] + ] + ).astype("float32") input_x = paddle.to_tensor(input_data) - Pool2d = paddle.nn.MaxPool2D(kernel_size=2, - stride=2, - return_mask=True) + Pool2d = paddle.nn.MaxPool2D( + kernel_size=2, stride=2, return_mask=True + ) UnPool = paddle.nn.MaxUnPool2D(kernel_size=2, stride=2) output, indices = Pool2d(input_x) out_pp = UnPool(output, indices) output_np = output.numpy() indices_np = indices.numpy() - expect_res =unpool2dmax_forward_naive(output_np, indices_np, [2,2], \ - [2,2], [0,0], [4,4]).astype("float64") + expect_res = unpool2dmax_forward_naive( + output_np, indices_np, [2, 2], [2, 2], [0, 0], [4, 4] + ).astype("float64") np.testing.assert_allclose(out_pp.numpy(), expect_res, rtol=1e-05) class TestUnpoolOpAPI_st(unittest.TestCase): - def test_case(self): import paddle import paddle.nn.functional as F import paddle.fluid.core as core import paddle.fluid as fluid + paddle.enable_static() - input_data = np.array([[[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], - [13, 14, 15, 16]]]]).astype("float32") + input_data = np.array( + [[[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]]] + ).astype("float32") x = fluid.data(name="x", shape=[1, 1, 4, 4], dtype="float32") - output, indices = F.max_pool2d(x, - kernel_size=2, - stride=2, - return_mask=True) - unpool_out = F.max_unpool2d(output, - indices, - kernel_size=2, - stride=None, - output_size=(5, 5)) + output, indices = F.max_pool2d( + x, kernel_size=2, stride=2, return_mask=True + ) + unpool_out = F.max_unpool2d( + output, indices, kernel_size=2, stride=None, output_size=(5, 5) + ) if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) else: @@ -344,20 +367,22 @@ def test_case(self): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - results = exe.run(paddle.fluid.default_main_program(),\ - feed={"x":input_data}, - fetch_list=[unpool_out], - return_numpy=True) + results = exe.run( + paddle.fluid.default_main_program(), + feed={"x": input_data}, + fetch_list=[unpool_out], + return_numpy=True, + ) - pool_out_np = np.array([[[[6., 8.], [14., 16.]]]]).astype("float32") + pool_out_np = np.array([[[[6.0, 8.0], [14.0, 16.0]]]]).astype("float32") indices_np = np.array([[[[5, 7], [13, 15]]]]).astype("int32") - expect_res =unpool2dmax_forward_naive(pool_out_np, indices_np, [2,2], \ - [2,2], [0,0], [5,5]).astype("float64") + expect_res = unpool2dmax_forward_naive( + pool_out_np, indices_np, [2, 2], [2, 2], [0, 0], [5, 5] + ).astype("float64") np.testing.assert_allclose(results[0], expect_res, rtol=1e-05) class TestOutputSizeTensor(UnittestBase): - def init_info(self): self.shapes = [[1, 3, 6, 6]] self.save_path = os.path.join(self.temp_dir.name, self.path_prefix()) @@ -394,23 +419,18 @@ def var_prefix(self): def call_func(self, x): output_size = [paddle.assign([7]), paddle.assign([7])] - pool_out, indices = F.max_pool2d(x, - kernel_size=2, - stride=2, - padding=0, - return_mask=True) + pool_out, indices = F.max_pool2d( + x, kernel_size=2, stride=2, padding=0, return_mask=True + ) # pool_out shape: [1, 1, 6, 6], indices shape: [1, 1, 6, 6] - unpool_out = F.max_unpool2d(pool_out, - indices, - kernel_size=2, - padding=0, - output_size=output_size) + unpool_out = F.max_unpool2d( + pool_out, indices, kernel_size=2, padding=0, output_size=output_size + ) # unpool_out shape: [1, 1, 7, 7] return unpool_out class TestZOutputSizeTensor2(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -419,22 +439,17 @@ def tearDown(self): def test_dygraph(self): x = paddle.randn([1, 3, 6, 6]) - pool_out, indices = F.max_pool2d(x, - kernel_size=2, - stride=2, - padding=0, - return_mask=True) + pool_out, indices = F.max_pool2d( + x, kernel_size=2, stride=2, padding=0, return_mask=True + ) output_size = [paddle.assign([7]), paddle.assign([7])] - unpool_out = F.max_unpool2d(pool_out, - indices, - kernel_size=2, - padding=0, - output_size=output_size) + unpool_out = F.max_unpool2d( + pool_out, indices, kernel_size=2, padding=0, output_size=output_size + ) np.testing.assert_array_equal(unpool_out.shape, [1, 3, 7, 7]) class TestZOutputSizeTensor3(unittest.TestCase): - def setUp(self): paddle.disable_static() @@ -443,22 +458,18 @@ def tearDown(self): def test_dygraph(self): x = paddle.randn([1, 3, 6, 6]) - pool_out, indices = F.max_pool2d(x, - kernel_size=2, - stride=2, - padding=0, - return_mask=True) + pool_out, indices = F.max_pool2d( + x, kernel_size=2, stride=2, padding=0, return_mask=True + ) output_size = [ paddle.assign([1]), paddle.assign([1]), paddle.assign([7]), - paddle.assign([7]) + paddle.assign([7]), ] - unpool_out = F.max_unpool2d(pool_out, - indices, - kernel_size=2, - padding=0, - output_size=output_size) + unpool_out = F.max_unpool2d( + pool_out, indices, kernel_size=2, padding=0, output_size=output_size + ) np.testing.assert_array_equal(unpool_out.shape, [1, 3, 7, 7]) diff --git a/python/paddle/fluid/tests/unittests/test_unsqueeze2_op.py b/python/paddle/fluid/tests/unittests/test_unsqueeze2_op.py index d585c8a3d0f78..8de9185162a2b 100755 --- a/python/paddle/fluid/tests/unittests/test_unsqueeze2_op.py +++ b/python/paddle/fluid/tests/unittests/test_unsqueeze2_op.py @@ -25,7 +25,6 @@ # Correct: General. class TestUnsqueezeOp(OpTest): - def setUp(self): self.init_test_case() self.op_type = "unsqueeze2" @@ -35,7 +34,7 @@ def setUp(self): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float64") + "XShape": np.random.random(self.ori_shape).astype("float64"), } def test_check_output(self): @@ -55,16 +54,14 @@ def init_attrs(self): # Correct: Single input index. class TestUnsqueezeOp1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) # Correct: Mixed input axis. class TestUnsqueezeOp2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -73,7 +70,6 @@ def init_test_case(self): # Correct: There is duplicated axis. class TestUnsqueezeOp3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -82,7 +78,6 @@ def init_test_case(self): # Correct: Reversed axes. class TestUnsqueezeOp4(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -90,15 +85,13 @@ def init_test_case(self): class TestUnsqueezeOp_ZeroDim1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = () - self.axes = (-1, ) - self.new_shape = (1) + self.axes = (-1,) + self.new_shape = 1 class TestUnsqueezeOp_ZeroDim2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = () self.axes = (-1, 1) @@ -106,7 +99,6 @@ def init_test_case(self): class TestUnsqueezeOp_ZeroDim3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = () self.axes = (0, 1, 2) @@ -115,7 +107,6 @@ def init_test_case(self): # axes is a list(with tensor) class TestUnsqueezeOp_AxesTensorList(OpTest): - def setUp(self): self.init_test_case() self.op_type = "unsqueeze2" @@ -124,17 +115,18 @@ def setUp(self): axes_tensor_list = [] for index, ele in enumerate(self.axes): - axes_tensor_list.append(("axes" + str(index), np.ones( - (1)).astype('int32') * ele)) + axes_tensor_list.append( + ("axes" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { "X": np.random.random(self.ori_shape).astype("float64"), - "AxesTensorList": axes_tensor_list + "AxesTensorList": axes_tensor_list, } self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float64") + "XShape": np.random.random(self.ori_shape).astype("float64"), } def test_check_output(self): @@ -153,15 +145,13 @@ def init_attrs(self): class TestUnsqueezeOp1_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) class TestUnsqueezeOp2_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -169,7 +159,6 @@ def init_test_case(self): class TestUnsqueezeOp3_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -177,7 +166,6 @@ def init_test_case(self): class TestUnsqueezeOp4_AxesTensorList(TestUnsqueezeOp_AxesTensorList): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -186,7 +174,6 @@ def init_test_case(self): # axes is a Tensor class TestUnsqueezeOp_AxesTensor(OpTest): - def setUp(self): self.init_test_case() self.op_type = "unsqueeze2" @@ -195,12 +182,12 @@ def setUp(self): self.inputs = { "X": np.random.random(self.ori_shape).astype("float64"), - "AxesTensor": np.array(self.axes).astype("int32") + "AxesTensor": np.array(self.axes).astype("int32"), } self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype("float64") + "XShape": np.random.random(self.ori_shape).astype("float64"), } def test_check_output(self): @@ -219,15 +206,13 @@ def init_attrs(self): class TestUnsqueezeOp1_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) class TestUnsqueezeOp2_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -235,7 +220,6 @@ def init_test_case(self): class TestUnsqueezeOp3_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -243,7 +227,6 @@ def init_test_case(self): class TestUnsqueezeOp4_AxesTensor(TestUnsqueezeOp_AxesTensor): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -252,7 +235,6 @@ def init_test_case(self): # test api class TestUnsqueezeAPI(unittest.TestCase): - def setUp(self): self.executed_api() @@ -264,12 +246,12 @@ def test_api(self): x = paddle.static.data(name='x', shape=[3, 2, 5], dtype="float64") positive_3_int32 = fluid.layers.fill_constant([1], "int32", 3) positive_1_int64 = fluid.layers.fill_constant([1], "int64", 1) - axes_tensor_int32 = paddle.static.data(name='axes_tensor_int32', - shape=[3], - dtype="int32") - axes_tensor_int64 = paddle.static.data(name='axes_tensor_int64', - shape=[3], - dtype="int64") + axes_tensor_int32 = paddle.static.data( + name='axes_tensor_int32', shape=[3], dtype="int32" + ) + axes_tensor_int64 = paddle.static.data( + name='axes_tensor_int64', shape=[3], dtype="int64" + ) out_1 = self.unsqueeze(x, axis=[3, 1, 1]) out_2 = self.unsqueeze(x, axis=[positive_3_int32, positive_1_int64, 1]) @@ -283,9 +265,10 @@ def test_api(self): feed={ "x": input, "axes_tensor_int32": np.array([3, 1, 1]).astype("int32"), - "axes_tensor_int64": np.array([3, 1, 1]).astype("int64") + "axes_tensor_int64": np.array([3, 1, 1]).astype("int64"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5]) + fetch_list=[out_1, out_2, out_3, out_4, out_5], + ) assert np.array_equal(res_1, input.reshape([3, 1, 1, 2, 5, 1])) assert np.array_equal(res_2, input.reshape([3, 1, 1, 2, 5, 1])) @@ -294,7 +277,6 @@ def test_api(self): assert np.array_equal(res_5, input.reshape([3, 1, 1, 2, 5, 1])) def test_error(self): - def test_axes_type(): x2 = paddle.static.data(name="x2", shape=[2, 25], dtype="int32") self.unsqueeze(x2, axis=2.1) @@ -303,13 +285,11 @@ def test_axes_type(): class TestUnsqueezeInplaceAPI(TestUnsqueezeAPI): - def executed_api(self): self.unsqueeze = paddle.unsqueeze_ class TestUnsqueezeAPI_ZeroDim(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) diff --git a/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py b/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py index c1261d0485799..b73c2a3906ff4 100755 --- a/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py +++ b/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py @@ -29,7 +29,6 @@ # Correct: General. class TestUnsqueezeOp(OpTest): - def setUp(self): self.init_test_case() self.op_type = "unsqueeze" @@ -53,7 +52,6 @@ def init_attrs(self): class TestUnsqueezeBF16Op(OpTest): - def setUp(self): self.init_test_case() self.op_type = "unsqueeze" @@ -81,16 +79,14 @@ def init_attrs(self): # Correct: Single input index. class TestUnsqueezeOp1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) # Correct: Mixed input axis. class TestUnsqueezeOp2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -99,7 +95,6 @@ def init_test_case(self): # Correct: There is duplicated axis. class TestUnsqueezeOp3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -108,7 +103,6 @@ def init_test_case(self): # Correct: Reversed axes. class TestUnsqueezeOp4(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -116,15 +110,13 @@ def init_test_case(self): class TestUnsqueezeOp_ZeroDim1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = () - self.axes = (-1, ) - self.new_shape = (1) + self.axes = (-1,) + self.new_shape = 1 class TestUnsqueezeOp_ZeroDim2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = () self.axes = (-1, 1) @@ -132,7 +124,6 @@ def init_test_case(self): class TestUnsqueezeOp_ZeroDim3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = () self.axes = (0, 1, 2) @@ -140,44 +131,45 @@ def init_test_case(self): class API_TestUnsqueeze(unittest.TestCase): - def test_out(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data1 = paddle.static.data('data1', shape=[-1, 10], dtype='float64') result_squeeze = paddle.unsqueeze(data1, axis=[1]) place = paddle.CPUPlace() exe = paddle.static.Executor(place) input1 = np.random.random([5, 1, 10]).astype('float64') input = np.squeeze(input1, axis=1) - result, = exe.run(feed={"data1": input}, - fetch_list=[result_squeeze]) + (result,) = exe.run( + feed={"data1": input}, fetch_list=[result_squeeze] + ) np.testing.assert_allclose(input1, result, rtol=1e-05) class TestUnsqueezeOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): # The type of axis in split_op should be int or Variable. def test_axes_type(): - x6 = paddle.static.data(shape=[-1, 10], - dtype='float16', - name='x3') + x6 = paddle.static.data( + shape=[-1, 10], dtype='float16', name='x3' + ) paddle.unsqueeze(x6, axis=3.2) self.assertRaises(TypeError, test_axes_type) class API_TestUnsqueeze2(unittest.TestCase): - def test_out(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data1 = paddle.static.data('data1', shape=[-1, 10], dtype='float64') data2 = paddle.static.data('data2', shape=[1], dtype='int32') result_squeeze = paddle.unsqueeze(data1, axis=data2) @@ -186,20 +178,19 @@ def test_out(self): input1 = np.random.random([5, 1, 10]).astype('float64') input2 = np.array([1]).astype('int32') input = np.squeeze(input1, axis=1) - result1, = exe.run(feed={ - "data1": input, - "data2": input2 - }, - fetch_list=[result_squeeze]) + (result1,) = exe.run( + feed={"data1": input, "data2": input2}, + fetch_list=[result_squeeze], + ) np.testing.assert_allclose(input1, result1, rtol=1e-05) class API_TestUnsqueeze3(unittest.TestCase): - def test_out(self): paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): data1 = paddle.static.data('data1', shape=[-1, 10], dtype='float64') data2 = paddle.static.data('data2', shape=[1], dtype='int32') result_squeeze = paddle.unsqueeze(data1, axis=[data2, 3]) @@ -208,17 +199,15 @@ def test_out(self): input1 = np.random.random([5, 1, 10, 1]).astype('float64') input2 = np.array([1]).astype('int32') input = np.squeeze(input1) - result1, = exe.run(feed={ - "data1": input, - "data2": input2 - }, - fetch_list=[result_squeeze]) + (result1,) = exe.run( + feed={"data1": input, "data2": input2}, + fetch_list=[result_squeeze], + ) np.testing.assert_array_equal(input1, result1) self.assertEqual(input1.shape, result1.shape) class API_TestDyUnsqueeze(unittest.TestCase): - def test_out(self): paddle.disable_static() input_1 = np.random.random([5, 1, 10]).astype("int32") @@ -231,7 +220,6 @@ def test_out(self): class API_TestDyUnsqueeze2(unittest.TestCase): - def test_out(self): paddle.disable_static() input1 = np.random.random([5, 10]).astype("int32") @@ -244,7 +232,6 @@ def test_out(self): class API_TestDyUnsqueezeAxisTensor(unittest.TestCase): - def test_out(self): paddle.disable_static() input1 = np.random.random([5, 10]).astype("int32") @@ -258,7 +245,6 @@ def test_out(self): class API_TestDyUnsqueezeAxisTensorList(unittest.TestCase): - def test_out(self): paddle.disable_static() input1 = np.random.random([5, 10]).astype("int32") @@ -268,15 +254,14 @@ def test_out(self): input = paddle.to_tensor(input1) output = paddle.unsqueeze( paddle.to_tensor(input1), - axis=[paddle.to_tensor([1]), - paddle.to_tensor([2])]) + axis=[paddle.to_tensor([1]), paddle.to_tensor([2])], + ) out_np = output.numpy() np.testing.assert_array_equal(out1, out_np) self.assertEqual(out1.shape, out_np.shape) class API_TestDygraphUnSqueeze(unittest.TestCase): - def setUp(self): self.executed_api() @@ -330,13 +315,11 @@ def test_dimension_not_1(self): class API_TestDygraphUnSqueezeInplace(API_TestDygraphUnSqueeze): - def executed_api(self): self.unsqueeze = paddle.unsqueeze_ class TestUnsqueezeDoubleGradCheck(unittest.TestCase): - def unsqueeze_wrapper(self, x): return paddle.unsqueeze(x[0], [0, 2]) @@ -351,17 +334,13 @@ def func(self, place): out = paddle.unsqueeze(data, [0, 2]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.double_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.double_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.double_grad_check_for_dygraph(self.unsqueeze_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.double_grad_check_for_dygraph( + self.unsqueeze_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() @@ -373,7 +352,6 @@ def test_grad(self): class TestUnsqueezeTripleGradCheck(unittest.TestCase): - def unsqueeze_wrapper(self, x): return paddle.unsqueeze(x[0], [0, 2]) @@ -388,17 +366,13 @@ def func(self, place): out = paddle.unsqueeze(data, [0, 2]) data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) - gradient_checker.triple_grad_check([data], - out, - x_init=[data_arr], - place=place, - eps=eps) + gradient_checker.triple_grad_check( + [data], out, x_init=[data_arr], place=place, eps=eps + ) fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) - gradient_checker.triple_grad_check_for_dygraph(self.unsqueeze_wrapper, - [data], - out, - x_init=[data_arr], - place=place) + gradient_checker.triple_grad_check_for_dygraph( + self.unsqueeze_wrapper, [data], out, x_init=[data_arr], place=place + ) def test_grad(self): paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_unstack_op.py b/python/paddle/fluid/tests/unittests/test_unstack_op.py index bb28bdeba79d3..87095d8eeac24 100755 --- a/python/paddle/fluid/tests/unittests/test_unstack_op.py +++ b/python/paddle/fluid/tests/unittests/test_unstack_op.py @@ -19,7 +19,6 @@ class TestUnStackOpBase(OpTest): - def initDefaultParameters(self): self.input_dim = (5, 6, 7) self.axis = 0 @@ -64,25 +63,21 @@ def test_check_grad(self): class TestStackOp3(TestUnStackOpBase): - def initParameters(self): self.axis = -1 class TestStackOp4(TestUnStackOpBase): - def initParameters(self): self.axis = -3 class TestStackOp5(TestUnStackOpBase): - def initParameters(self): self.axis = 1 class TestStackOp6(TestUnStackOpBase): - def initParameters(self): self.axis = 2 diff --git a/python/paddle/fluid/tests/unittests/test_update_loss_scaling_op.py b/python/paddle/fluid/tests/unittests/test_update_loss_scaling_op.py index 9032824950196..7f1e6c8614f88 100644 --- a/python/paddle/fluid/tests/unittests/test_update_loss_scaling_op.py +++ b/python/paddle/fluid/tests/unittests/test_update_loss_scaling_op.py @@ -19,31 +19,43 @@ import paddle.fluid.contrib.mixed_precision.amp_nn as amp_nn -def update_loss_scaling_wrapper(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - stop_update=False): - amp_nn.update_loss_scaling([x], found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps, - incr_every_n_steps, decr_every_n_nan_or_inf, - incr_ratio, decr_ratio, stop_update) +def update_loss_scaling_wrapper( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + stop_update=False, +): + amp_nn.update_loss_scaling( + [x], + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + stop_update, + ) return x, prev_loss_scaling, num_good_steps, num_bad_steps class TestUpdateLossScalingOp(OpTest): - def setUp(self): self.op_type = "update_loss_scaling" self.init() self.python_api = update_loss_scaling_wrapper self.python_out_sig = [ - "out0", "LossScaling", "OutGoodSteps", "OutBadSteps" + "out0", + "LossScaling", + "OutGoodSteps", + "OutBadSteps", ] found_inf = np.array([False], dtype=np.bool_) x = np.random.random((1024, 1024)).astype(self.dtype) @@ -53,14 +65,14 @@ def setUp(self): 'FoundInfinite': found_inf, 'PrevLossScaling': self.prev_loss_scaling, 'InGoodSteps': self.num_good_steps, - 'InBadSteps': self.num_bad_steps + 'InBadSteps': self.num_bad_steps, } self.outputs = { 'Out': [('out0', x)], 'LossScaling': self.prev_loss_scaling * self.incr_ratio, 'OutGoodSteps': self.zero_steps, - 'OutBadSteps': self.zero_steps + 'OutBadSteps': self.zero_steps, } def init(self): @@ -84,13 +96,15 @@ def test_check_output(self): class TestUpdateLossScalingOpBad(TestUpdateLossScalingOp): - def setUp(self): self.op_type = "update_loss_scaling" self.init() self.python_api = update_loss_scaling_wrapper self.python_out_sig = [ - "out0", "LossScaling", "OutGoodSteps", "OutBadSteps" + "out0", + "LossScaling", + "OutGoodSteps", + "OutBadSteps", ] found_inf = np.array([True], dtype=np.bool_) x = np.random.random((1024, 1024)).astype(self.dtype) @@ -104,14 +118,14 @@ def setUp(self): 'PrevLossScaling': self.prev_loss_scaling, 'InGoodSteps': self.num_good_steps, 'InBadSteps': self.num_bad_steps, - 'StopUpdate': self.stop_update + 'StopUpdate': self.stop_update, } self.outputs = { 'Out': [('out0', np.zeros_like(x))], 'LossScaling': self.prev_loss_scaling * self.decr_ratio, 'OutGoodSteps': self.zero_steps, - 'OutBadSteps': self.zero_steps + 'OutBadSteps': self.zero_steps, } def test_check_output(self): @@ -119,21 +133,20 @@ def test_check_output(self): class TestUpdateLossScalingLayer(unittest.TestCase): - def loss_scaling_check(self, use_cuda=True, scope=fluid.Scope()): a = fluid.data(name="a", shape=[1024, 1024], dtype='float32') b = fluid.data(name="b", shape=[512, 128], dtype='float32') x = [a, b] found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') - prev_loss_scaling = fluid.data(name="prev_loss_scaling", - shape=[1], - dtype='float32') - num_good_steps = fluid.data(name="num_good_steps", - shape=[1], - dtype='int32') - num_bad_steps = fluid.data(name="num_bad_steps", - shape=[1], - dtype='int32') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32' + ) + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32' + ) + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32' + ) a_v = np.random.random([1024, 1024]).astype('float32') b_v = np.random.random([512, 128]).astype('float32') @@ -147,33 +160,41 @@ def loss_scaling_check(self, use_cuda=True, scope=fluid.Scope()): incr_ratio = 2 decr_ratio = 0.8 - result = amp_nn.update_loss_scaling(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - name="update_loss_scaling") + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling", + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) with fluid.scope_guard(scope): exe.run(fluid.default_startup_program()) - result_v = exe.run(feed={ - 'a': a_v, - 'b': b_v, - 'found_inf': found_inf_v, - 'prev_loss_scaling': prev_loss_scaling_v, - 'num_good_steps': num_good_steps_v, - 'num_bad_steps': num_bad_steps_v - }, - fetch_list=[ - result, x, found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps - ]) + result_v = exe.run( + feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v, + }, + fetch_list=[ + result, + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + ], + ) assert np.array_equal(result_v[0], a_v) assert np.array_equal(result_v[1], b_v) assert np.array_equal(result_v[0], result_v[2]) @@ -188,15 +209,15 @@ def loss_scaling_check_inf(self, use_cuda=True, scope=fluid.Scope()): b = fluid.data(name="b", shape=[512, 128], dtype='float32') x = [a, b] found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') - prev_loss_scaling = fluid.data(name="prev_loss_scaling", - shape=[1], - dtype='float32') - num_good_steps = fluid.data(name="num_good_steps", - shape=[1], - dtype='int32') - num_bad_steps = fluid.data(name="num_bad_steps", - shape=[1], - dtype='int32') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32' + ) + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32' + ) + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32' + ) a_v = np.random.random([1024, 1024]).astype('float32') b_v = np.random.random([512, 128]).astype('float32') @@ -213,33 +234,41 @@ def loss_scaling_check_inf(self, use_cuda=True, scope=fluid.Scope()): incr_ratio = 2 decr_ratio = 0.8 - result = amp_nn.update_loss_scaling(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - name="update_loss_scaling") + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling", + ) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) with fluid.scope_guard(scope): exe.run(fluid.default_startup_program()) - result_v = exe.run(feed={ - 'a': a_v, - 'b': b_v, - 'found_inf': found_inf_v, - 'prev_loss_scaling': prev_loss_scaling_v, - 'num_good_steps': num_good_steps_v, - 'num_bad_steps': num_bad_steps_v - }, - fetch_list=[ - result, x, found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps - ]) + result_v = exe.run( + feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v, + }, + fetch_list=[ + result, + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + ], + ) assert np.array_equal(result_v[0], np.zeros_like(a_v)) assert np.array_equal(result_v[1], np.zeros_like(b_v)) assert np.array_equal(result_v[2], np.zeros_like(a_v)) diff --git a/python/paddle/fluid/tests/unittests/test_var_base.py b/python/paddle/fluid/tests/unittests/test_var_base.py index 7804aec60c995..d1b820bd74c1d 100644 --- a/python/paddle/fluid/tests/unittests/test_var_base.py +++ b/python/paddle/fluid/tests/unittests/test_var_base.py @@ -23,14 +23,12 @@ class TestVarBase(unittest.TestCase): - def setUp(self): self.shape = [512, 1234] self.dtype = np.float32 self.array = np.random.uniform(0.1, 1, self.shape).astype(self.dtype) def func_test_to_tensor(self): - def check_with_place(place): with fluid.dygraph.guard(): paddle.set_default_dtype('float32') @@ -43,11 +41,14 @@ def check_with_place(place): self.assertEqual(str(x.place), str(y.place)) # set_default_dtype should not take effect on numpy - x = paddle.to_tensor(np.array([1.2]).astype('float16'), - place=place, - stop_gradient=False) - np.testing.assert_array_equal(x.numpy(), - np.array([1.2], 'float16')) + x = paddle.to_tensor( + np.array([1.2]).astype('float16'), + place=place, + stop_gradient=False, + ) + np.testing.assert_array_equal( + x.numpy(), np.array([1.2], 'float16') + ) self.assertEqual(x.dtype, core.VarDesc.VarType.FP16) # set_default_dtype take effect on int @@ -56,17 +57,20 @@ def check_with_place(place): # set_default_dtype take effect on float x = paddle.to_tensor(1.2, place=place, stop_gradient=False) - np.testing.assert_array_equal(x.numpy(), - np.array([1.2]).astype('float32')) + np.testing.assert_array_equal( + x.numpy(), np.array([1.2]).astype('float32') + ) self.assertEqual(x.dtype, core.VarDesc.VarType.FP32) clone_x = x.clone() - np.testing.assert_array_equal(clone_x.numpy(), - np.array([1.2]).astype('float32')) + np.testing.assert_array_equal( + clone_x.numpy(), np.array([1.2]).astype('float32') + ) self.assertEqual(clone_x.dtype, core.VarDesc.VarType.FP32) y = clone_x**2 y.backward() - np.testing.assert_array_equal(x.grad.numpy(), - np.array([2.4]).astype('float32')) + np.testing.assert_array_equal( + x.grad.numpy(), np.array([2.4]).astype('float32') + ) y = x.cpu() self.assertEqual(y.place.__repr__(), "Place(cpu)") if core.is_compiled_with_cuda(): @@ -104,24 +108,21 @@ def check_with_place(place): np.testing.assert_array_equal(x.numpy(), [1 + 2j]) self.assertEqual(x.dtype, core.VarDesc.VarType.COMPLEX128) - x = paddle.to_tensor(1, - dtype='float32', - place=place, - stop_gradient=False) + x = paddle.to_tensor( + 1, dtype='float32', place=place, stop_gradient=False + ) np.testing.assert_array_equal(x.numpy(), [1.0]) self.assertEqual(x.dtype, core.VarDesc.VarType.FP32) self.assertEqual(x.shape, [1]) self.assertEqual(x.stop_gradient, False) self.assertEqual(x.type, core.VarDesc.VarType.LOD_TENSOR) - x = paddle.to_tensor((1, 2), - dtype='float32', - place=place, - stop_gradient=False) - x = paddle.to_tensor([1, 2], - dtype='float32', - place=place, - stop_gradient=False) + x = paddle.to_tensor( + (1, 2), dtype='float32', place=place, stop_gradient=False + ) + x = paddle.to_tensor( + [1, 2], dtype='float32', place=place, stop_gradient=False + ) np.testing.assert_array_equal(x.numpy(), [1.0, 2.0]) self.assertEqual(x.dtype, core.VarDesc.VarType.FP32) self.assertEqual(x.grad, None) @@ -129,10 +130,12 @@ def check_with_place(place): self.assertEqual(x.stop_gradient, False) self.assertEqual(x.type, core.VarDesc.VarType.LOD_TENSOR) - x = paddle.to_tensor(self.array, - dtype='float32', - place=place, - stop_gradient=False) + x = paddle.to_tensor( + self.array, + dtype='float32', + place=place, + stop_gradient=False, + ) np.testing.assert_array_equal(x.numpy(), self.array) self.assertEqual(x.dtype, core.VarDesc.VarType.FP32) self.assertEqual(x.shape, self.shape) @@ -149,9 +152,9 @@ def check_with_place(place): z = x + y np.testing.assert_array_equal(z.numpy(), 2 * self.array) - x = paddle.to_tensor([1 + 2j, 1 - 2j], - dtype='complex64', - place=place) + x = paddle.to_tensor( + [1 + 2j, 1 - 2j], dtype='complex64', place=place + ) y = paddle.to_tensor(x) np.testing.assert_array_equal(x.numpy(), [1 + 2j, 1 - 2j]) self.assertEqual(y.dtype, core.VarDesc.VarType.COMPLEX64) @@ -172,8 +175,9 @@ def check_with_place(place): self.assertTrue(isinstance(x.item(5), float)) self.assertTrue(isinstance(x.item(1, 0, 1), float)) self.assertEqual(x.item(5), x.item(1, 0, 1)) - np.testing.assert_array_equal(x.item(1, 0, 1), - x.numpy().item(1, 0, 1)) + np.testing.assert_array_equal( + x.item(1, 0, 1), x.numpy().item(1, 0, 1) + ) x = paddle.to_tensor([[1.111111, 2.222222, 3.333333]]) self.assertEqual(x.item(0, 2), x.item(2)) @@ -404,16 +408,18 @@ def func_test_leaf_tensor(self): y = x + 1 self.assertTrue(y.is_leaf) - x = paddle.to_tensor(np.random.uniform(-1, 1, size=[10, 10]), - stop_gradient=False) + x = paddle.to_tensor( + np.random.uniform(-1, 1, size=[10, 10]), stop_gradient=False + ) self.assertTrue(x.is_leaf) y = x + 1 self.assertFalse(y.is_leaf) linear = paddle.nn.Linear(10, 10) - input = paddle.to_tensor(np.random.uniform( - -1, 1, size=[10, 10]).astype('float32'), - stop_gradient=False) + input = paddle.to_tensor( + np.random.uniform(-1, 1, size=[10, 10]).astype('float32'), + stop_gradient=False, + ) self.assertTrue(input.is_leaf) out = linear(input) @@ -432,8 +438,9 @@ def func_test_detach(self): detach_x = x.detach() self.assertTrue(detach_x.stop_gradient, True) - cmp_float = np.allclose if core.is_compiled_with_rocm( - ) else np.array_equal + cmp_float = ( + np.allclose if core.is_compiled_with_rocm() else np.array_equal + ) detach_x[:] = 10.0 self.assertTrue(cmp_float(x.numpy(), [10.0])) @@ -442,7 +449,9 @@ def func_test_detach(self): self.assertTrue(cmp_float(x.grad.numpy(), [20.0])) self.assertEqual(detach_x.grad, None) - detach_x.stop_gradient = False # Set stop_gradient to be False, supported auto-grad + detach_x.stop_gradient = ( + False # Set stop_gradient to be False, supported auto-grad + ) z = 3 * detach_x**2 z.backward() self.assertTrue(cmp_float(x.grad.numpy(), [20.0])) @@ -492,14 +501,15 @@ def func_test_deep_copy(self): else: empty_var = core.eager.Tensor() empty_var_copy = copy.deepcopy(empty_var) - self.assertEqual(empty_var.stop_gradient, - empty_var_copy.stop_gradient) + self.assertEqual( + empty_var.stop_gradient, empty_var_copy.stop_gradient + ) self.assertEqual(empty_var.persistable, empty_var_copy.persistable) self.assertEqual(empty_var.type, empty_var_copy.type) self.assertEqual(empty_var.dtype, empty_var_copy.dtype) - x = paddle.to_tensor([2.], stop_gradient=False) - y = paddle.to_tensor([3.], stop_gradient=False) + x = paddle.to_tensor([2.0], stop_gradient=False) + y = paddle.to_tensor([3.0], stop_gradient=False) z = x * y memo = {} x_copy = copy.deepcopy(x, memo) @@ -516,7 +526,7 @@ def func_test_deep_copy(self): np.testing.assert_array_equal(x.numpy(), [2.0]) with self.assertRaises(ValueError): - x_copy[:] = 5. + x_copy[:] = 5.0 with self.assertRaises(RuntimeError): copy.deepcopy(z) @@ -528,17 +538,26 @@ def func_test_deep_copy(self): # test copy selected rows if _in_legacy_dygraph(): - x = core.VarBase(core.VarDesc.VarType.FP32, [3, 100], - "selected_rows", - core.VarDesc.VarType.SELECTED_ROWS, True) + x = core.VarBase( + core.VarDesc.VarType.FP32, + [3, 100], + "selected_rows", + core.VarDesc.VarType.SELECTED_ROWS, + True, + ) else: - x = core.eager.Tensor(core.VarDesc.VarType.FP32, [3, 100], - "selected_rows", - core.VarDesc.VarType.SELECTED_ROWS, True) + x = core.eager.Tensor( + core.VarDesc.VarType.FP32, + [3, 100], + "selected_rows", + core.VarDesc.VarType.SELECTED_ROWS, + True, + ) selected_rows = x.value().get_selected_rows() - selected_rows.get_tensor().set(np.random.rand(3, 100), - core.CPUPlace()) + selected_rows.get_tensor().set( + np.random.rand(3, 100), core.CPUPlace() + ) selected_rows.set_height(10) selected_rows.set_rows([3, 5, 7]) x_copy = copy.deepcopy(x) @@ -549,12 +568,14 @@ def func_test_deep_copy(self): self.assertEqual(x_copy.dtype, x.dtype) copy_selected_rows = x_copy.value().get_selected_rows() - self.assertEqual(copy_selected_rows.height(), - selected_rows.height()) + self.assertEqual( + copy_selected_rows.height(), selected_rows.height() + ) self.assertEqual(copy_selected_rows.rows(), selected_rows.rows()) np.testing.assert_array_equal( np.array(copy_selected_rows.get_tensor()), - np.array(selected_rows.get_tensor())) + np.array(selected_rows.get_tensor()), + ) def test_deep_copy(self): with _test_eager_guard(): @@ -658,8 +679,9 @@ def test_gradient(self): def func_test_block(self): with fluid.dygraph.guard(): var = fluid.dygraph.to_variable(self.array) - self.assertEqual(var.block, - fluid.default_main_program().global_block()) + self.assertEqual( + var.block, fluid.default_main_program().global_block() + ) def test_block(self): with _test_eager_guard(): @@ -668,7 +690,8 @@ def test_block(self): def _test_slice(self): w = fluid.dygraph.to_variable( - np.random.random((784, 100, 100)).astype('float64')) + np.random.random((784, 100, 100)).astype('float64') + ) for i in range(3): nw = w[i] @@ -691,10 +714,13 @@ def _test_slice(self): nw = w[:, :, :-1] self.assertEqual((784, 100, 99), tuple(nw.shape)) - tensor_array = np.array([[[1, 2, 3], [4, 5, 6], [7, 8, 9]], - [[10, 11, 12], [13, 14, 15], [16, 17, 18]], - [[19, 20, 21], [22, 23, 24], - [25, 26, 27]]]).astype('float32') + tensor_array = np.array( + [ + [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + [[10, 11, 12], [13, 14, 15], [16, 17, 18]], + [[19, 20, 21], [22, 23, 24], [25, 26, 27]], + ] + ).astype('float32') var = fluid.dygraph.to_variable(tensor_array) var1 = var[0, 1, 1] var2 = var[1:] @@ -717,8 +743,25 @@ def _test_slice(self): var18 = var[:, 1:1:2] vars = [ - var, var1, var2, var3, var4, var5, var6, var7, var8, var9, var10, - var11, var12, var13, var14, var15, var16, var17, var18 + var, + var1, + var2, + var3, + var4, + var5, + var6, + var7, + var8, + var9, + var10, + var11, + var12, + var13, + var14, + var15, + var16, + var17, + var18, ] local_out = [var.numpy() for var in vars] @@ -728,32 +771,41 @@ def _test_slice(self): np.testing.assert_array_equal(local_out[4], tensor_array[::-1]) np.testing.assert_array_equal(local_out[5], tensor_array[1, 1:, 1:]) np.testing.assert_array_equal( - local_out[6], - tensor_array.reshape((3, -1, 3))[:, :, -1]) + local_out[6], tensor_array.reshape((3, -1, 3))[:, :, -1] + ) np.testing.assert_array_equal(local_out[7], tensor_array[:, :, :-1]) np.testing.assert_array_equal(local_out[8], tensor_array[:1, :1, :1]) np.testing.assert_array_equal(local_out[9], tensor_array[:-1, :-1, :-1]) - np.testing.assert_array_equal(local_out[10], - tensor_array[::-1, :1, :-1]) - np.testing.assert_array_equal(local_out[11], tensor_array[:-1, ::-1, - -1:]) - np.testing.assert_array_equal(local_out[12], tensor_array[1:2, - 2:, ::-1]) - np.testing.assert_array_equal(local_out[13], tensor_array[2:10, 2:, - -2:-1]) - np.testing.assert_array_equal(local_out[14], tensor_array[1:-1, - 0:2, ::-1]) - np.testing.assert_array_equal(local_out[15], - tensor_array[::-1, ::-1, ::-1]) + np.testing.assert_array_equal( + local_out[10], tensor_array[::-1, :1, :-1] + ) + np.testing.assert_array_equal( + local_out[11], tensor_array[:-1, ::-1, -1:] + ) + np.testing.assert_array_equal( + local_out[12], tensor_array[1:2, 2:, ::-1] + ) + np.testing.assert_array_equal( + local_out[13], tensor_array[2:10, 2:, -2:-1] + ) + np.testing.assert_array_equal( + local_out[14], tensor_array[1:-1, 0:2, ::-1] + ) + np.testing.assert_array_equal( + local_out[15], tensor_array[::-1, ::-1, ::-1] + ) np.testing.assert_array_equal(local_out[16], tensor_array[-4:4]) np.testing.assert_array_equal(local_out[17], tensor_array[:, 0, 0:0]) np.testing.assert_array_equal(local_out[18], tensor_array[:, 1:1:2]) def _test_slice_for_tensor_attr(self): - tensor_array = np.array([[[1, 2, 3], [4, 5, 6], [7, 8, 9]], - [[10, 11, 12], [13, 14, 15], [16, 17, 18]], - [[19, 20, 21], [22, 23, 24], - [25, 26, 27]]]).astype('float32') + tensor_array = np.array( + [ + [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + [[10, 11, 12], [13, 14, 15], [16, 17, 18]], + [[19, 20, 21], [22, 23, 24], [25, 26, 27]], + ] + ).astype('float32') var = paddle.to_tensor(tensor_array) @@ -782,8 +834,23 @@ def _test_slice_for_tensor_attr(self): var16 = var[-4:4] vars = [ - var, var1, var2, var3, var4, var5, var6, var7, var8, var9, var10, - var11, var12, var13, var14, var15, var16 + var, + var1, + var2, + var3, + var4, + var5, + var6, + var7, + var8, + var9, + var10, + var11, + var12, + var13, + var14, + var15, + var16, ] local_out = [var.numpy() for var in vars] @@ -793,23 +860,29 @@ def _test_slice_for_tensor_attr(self): np.testing.assert_array_equal(local_out[4], tensor_array[::-1]) np.testing.assert_array_equal(local_out[5], tensor_array[1, 1:, 1:]) np.testing.assert_array_equal( - local_out[6], - tensor_array.reshape((3, -1, 3))[:, :, -1]) + local_out[6], tensor_array.reshape((3, -1, 3))[:, :, -1] + ) np.testing.assert_array_equal(local_out[7], tensor_array[:, :, :-1]) np.testing.assert_array_equal(local_out[8], tensor_array[:1, :1, :1]) np.testing.assert_array_equal(local_out[9], tensor_array[:-1, :-1, :-1]) - np.testing.assert_array_equal(local_out[10], - tensor_array[::-1, :1, :-1]) - np.testing.assert_array_equal(local_out[11], tensor_array[:-1, ::-1, - -1:]) - np.testing.assert_array_equal(local_out[12], tensor_array[1:2, - 2:, ::-1]) - np.testing.assert_array_equal(local_out[13], tensor_array[2:10, 2:, - -2:-1]) - np.testing.assert_array_equal(local_out[14], tensor_array[1:-1, - 0:2, ::-1]) - np.testing.assert_array_equal(local_out[15], - tensor_array[::-1, ::-1, ::-1]) + np.testing.assert_array_equal( + local_out[10], tensor_array[::-1, :1, :-1] + ) + np.testing.assert_array_equal( + local_out[11], tensor_array[:-1, ::-1, -1:] + ) + np.testing.assert_array_equal( + local_out[12], tensor_array[1:2, 2:, ::-1] + ) + np.testing.assert_array_equal( + local_out[13], tensor_array[2:10, 2:, -2:-1] + ) + np.testing.assert_array_equal( + local_out[14], tensor_array[1:-1, 0:2, ::-1] + ) + np.testing.assert_array_equal( + local_out[15], tensor_array[::-1, ::-1, ::-1] + ) np.testing.assert_array_equal(local_out[16], tensor_array[-4:4]) def _test_for_getitem_ellipsis_index(self): @@ -851,8 +924,9 @@ def assert_getitem_ellipsis_index(var_tensor, var_np): # test 1 dim tensor var_one_dim = paddle.to_tensor([1, 2, 3, 4]) - np.testing.assert_array_equal(var_one_dim[..., 0].numpy(), - np.array([1])) + np.testing.assert_array_equal( + var_one_dim[..., 0].numpy(), np.array([1]) + ) def _test_none_index(self): shape = (8, 64, 5, 256) @@ -883,8 +957,9 @@ def _test_none_index(self): np.testing.assert_array_equal(var[6], np_value[None, 2, None, 1]) np.testing.assert_array_equal(var[7], np_value[None]) np.testing.assert_array_equal(var[8], np_value[0, 0, None, 0, 0, None]) - np.testing.assert_array_equal(var[9], np_value[None, None, 0, ..., - None]) + np.testing.assert_array_equal( + var[9], np_value[None, None, 0, ..., None] + ) np.testing.assert_array_equal(var[10], np_value[..., None, :, None]) # TODO(zyfncg) there is a bug of dimensions when slice step > 1 and @@ -896,18 +971,25 @@ def _test_bool_index(self): shape = (4, 2, 5, 64) np_value = np.random.random(shape).astype('float32') var_tensor = paddle.to_tensor(np_value) - index = [[True, True, True, True], [True, False, True, True], - [True, False, False, True], [False, 0, 1, True, True], - [False, False, False, False]] - index2d = np.array([[True, True], [False, False], [True, False], - [True, True]]) + index = [ + [True, True, True, True], + [True, False, True, True], + [True, False, False, True], + [False, 0, 1, True, True], + [False, False, False, False], + ] + index2d = np.array( + [[True, True], [False, False], [True, False], [True, True]] + ) tensor_index = paddle.to_tensor(index2d) var = [ - var_tensor[index[0]].numpy(), var_tensor[index[1]].numpy(), - var_tensor[index[2]].numpy(), var_tensor[index[3]].numpy(), + var_tensor[index[0]].numpy(), + var_tensor[index[1]].numpy(), + var_tensor[index[2]].numpy(), + var_tensor[index[3]].numpy(), var_tensor[paddle.to_tensor(index[0])].numpy(), var_tensor[tensor_index].numpy(), - var_tensor[paddle.to_tensor(index[4])].numpy() + var_tensor[paddle.to_tensor(index[4])].numpy(), ] np.testing.assert_array_equal(var[0], np_value[index[0]]) np.testing.assert_array_equal(var[1], np_value[index[1]]) @@ -916,10 +998,12 @@ def _test_bool_index(self): np.testing.assert_array_equal(var[4], np_value[index[0]]) np.testing.assert_array_equal(var[5], np_value[index2d]) np.testing.assert_array_equal(var[6], np_value[index[4]]) - np.testing.assert_array_equal(var_tensor[var_tensor > 0.67], - np_value[np_value > 0.67]) - np.testing.assert_array_equal(var_tensor[var_tensor < 0.55], - np_value[np_value < 0.55]) + np.testing.assert_array_equal( + var_tensor[var_tensor > 0.67], np_value[np_value > 0.67] + ) + np.testing.assert_array_equal( + var_tensor[var_tensor < 0.55], np_value[np_value < 0.55] + ) with self.assertRaises(ValueError): var_tensor[[False, False, False, False]] @@ -953,13 +1037,16 @@ def _test_numpy_index(self): t = paddle.to_tensor(array) np.testing.assert_array_equal(t[np.longlong(0)].numpy(), array[0]) np.testing.assert_array_equal( - t[np.longlong(0):np.longlong(4):np.longlong(2)].numpy(), - array[0:4:2]) + t[np.longlong(0) : np.longlong(4) : np.longlong(2)].numpy(), + array[0:4:2], + ) np.testing.assert_array_equal(t[np.int64(0)].numpy(), array[0]) np.testing.assert_array_equal( - t[np.int32(1):np.int32(4):np.int32(2)].numpy(), array[1:4:2]) + t[np.int32(1) : np.int32(4) : np.int32(2)].numpy(), array[1:4:2] + ) np.testing.assert_array_equal( - t[np.int16(0):np.int16(4):np.int16(2)].numpy(), array[0:4:2]) + t[np.int16(0) : np.int16(4) : np.int16(2)].numpy(), array[0:4:2] + ) def _test_list_index(self): # case1: @@ -971,13 +1058,18 @@ def _test_list_index(self): np.testing.assert_array_equal(x[py_idx].numpy(), array[py_idx]) # case2: tensor_x = paddle.to_tensor( - np.zeros(12).reshape(2, 6).astype(np.float32)) + np.zeros(12).reshape(2, 6).astype(np.float32) + ) tensor_y1 = paddle.zeros([1], dtype='int32') + 2 tensor_y2 = paddle.zeros([1], dtype='int32') + 5 tensor_x[:, tensor_y1:tensor_y2] = 42 res = tensor_x.numpy() - exp = np.array([[0., 0., 42., 42., 42., 0.], - [0., 0., 42., 42., 42., 0.]]) + exp = np.array( + [ + [0.0, 0.0, 42.0, 42.0, 42.0, 0.0], + [0.0, 0.0, 42.0, 42.0, 42.0, 0.0], + ] + ) np.testing.assert_array_equal(res, exp) # case3: @@ -1019,8 +1111,9 @@ def test_slice(self): def func_test_var_base_to_np(self): with fluid.dygraph.guard(): var = fluid.dygraph.to_variable(self.array) - np.testing.assert_array_equal(var.numpy(), - fluid.framework._var_base_to_np(var)) + np.testing.assert_array_equal( + var.numpy(), fluid.framework._var_base_to_np(var) + ) def test_var_base_to_np(self): with _test_eager_guard(): @@ -1031,8 +1124,9 @@ def func_test_var_base_as_np(self): with fluid.dygraph.guard(): var = fluid.dygraph.to_variable(self.array) np.testing.assert_array_equal(var.numpy(), np.array(var)) - np.testing.assert_array_equal(var.numpy(), - np.array(var, dtype=np.float32)) + np.testing.assert_array_equal( + var.numpy(), np.array(var, dtype=np.float32) + ) def test_var_base_as_np(self): with _test_eager_guard(): @@ -1081,7 +1175,9 @@ def func_test_to_static_var(self): param_attr=fluid.ParamAttr( learning_rate=0.001, do_model_average=True, - regularizer=fluid.regularizer.L1Decay())) + regularizer=fluid.regularizer.L1Decay(), + ), + ) weight = fc.parameters()[0] static_param = weight._to_static_var() self._assert_to_static(weight, static_param, True) @@ -1097,14 +1193,18 @@ def _assert_to_static(self, var_base, static_var, is_param=False): self.assertTrue(static_var.persistable, True) if isinstance(var_base, fluid.framework.ParamBase): for attr in ['trainable', 'is_distributed', 'do_model_average']: - self.assertEqual(getattr(var_base, attr), - getattr(static_var, attr)) + self.assertEqual( + getattr(var_base, attr), getattr(static_var, attr) + ) - self.assertEqual(static_var.optimize_attr['learning_rate'], - 0.001) + self.assertEqual( + static_var.optimize_attr['learning_rate'], 0.001 + ) self.assertTrue( - isinstance(static_var.regularizer, - fluid.regularizer.L1Decay)) + isinstance( + static_var.regularizer, fluid.regularizer.L1Decay + ) + ) else: self.assertTrue(isinstance(static_var, fluid.framework.Variable)) @@ -1205,10 +1305,9 @@ def func_test_tensor_str_linewidth(self): paddle.disable_static(paddle.CPUPlace()) paddle.seed(2021) x = paddle.rand([128]) - paddle.set_printoptions(precision=4, - threshold=1000, - edgeitems=3, - linewidth=80) + paddle.set_printoptions( + precision=4, threshold=1000, edgeitems=3, linewidth=80 + ) a_str = str(x) expected = '''Tensor(shape=[128], dtype=float32, place=Place(cpu), stop_gradient=True, @@ -1301,7 +1400,6 @@ def test_print_tensor_dtype(self): class TestVarBaseSetitem(unittest.TestCase): - def func_setUp(self): self.set_dtype() self.tensor_x = paddle.to_tensor(np.ones((4, 2, 3)).astype(self.dtype)) @@ -1373,13 +1471,11 @@ def test_value_int(self): class TestVarBaseSetitemInt64(TestVarBaseSetitem): - def set_dtype(self): self.dtype = "int64" class TestVarBaseSetitemFp32(TestVarBaseSetitem): - def set_dtype(self): self.dtype = "float32" @@ -1396,13 +1492,11 @@ def test_value_float(self): class TestVarBaseSetitemFp64(TestVarBaseSetitem): - def set_dtype(self): self.dtype = "float64" class TestVarBaseSetitemBoolIndex(unittest.TestCase): - def func_setUp(self): paddle.disable_static() self.set_dtype() @@ -1481,7 +1575,6 @@ def test_value_int(self): class TestVarBaseSetitemBoolScalarIndex(unittest.TestCase): - def set_input(self): self.tensor_x = paddle.to_tensor(np.ones((1, 2, 3)).astype(self.dtype)) self.np_value = np.random.random((2, 3)).astype(self.dtype) @@ -1507,7 +1600,6 @@ def _test(self, value): class TestVarBaseInplaceVersion(unittest.TestCase): - def func_test_setitem(self): paddle.disable_static() @@ -1543,7 +1635,6 @@ def test_bump_inplace_version(self): class TestVarBaseSlice(unittest.TestCase): - def func_test_slice(self): paddle.disable_static() np_x = np.random.random((3, 8, 8)) @@ -1559,7 +1650,6 @@ def test_slice(self): class TestVarBaseClear(unittest.TestCase): - def func_test_clear(self): paddle.disable_static() np_x = np.random.random((3, 8, 8)) @@ -1574,7 +1664,6 @@ def test_clear(self): class TestVarBaseOffset(unittest.TestCase): - def func_offset(self): paddle.disable_static() np_x = np.random.random((3, 8, 8)) @@ -1591,7 +1680,6 @@ def test_offset(self): class TestVarBaseShareBufferTo(unittest.TestCase): - def func_test_share_buffer_To(self): paddle.disable_static() np_src = np.random.random((3, 8, 8)) @@ -1611,7 +1699,6 @@ def test_share_buffer_To(self): class TestVarBaseTo(unittest.TestCase): - def func_setUp(self): paddle.disable_static() self.np_x = np.random.random((3, 8, 8)) @@ -1638,14 +1725,16 @@ def func_test_to_api(self): x_gpu1 = self.x._to(device='gpu:0', dtype="float64") self.assertTrue(x_gpu1.place.is_gpu_place()) self.assertEqual(x_gpu1.place.gpu_device_id(), 0) - self.assertEqual(x_gpu1.dtype, - paddle.fluid.core.VarDesc.VarType.FP64) + self.assertEqual( + x_gpu1.dtype, paddle.fluid.core.VarDesc.VarType.FP64 + ) x_gpu2 = self.x._to(device='gpu:0', dtype="float16") self.assertTrue(x_gpu2.place.is_gpu_place()) self.assertEqual(x_gpu2.place.gpu_device_id(), 0) - self.assertEqual(x_gpu2.dtype, - paddle.fluid.core.VarDesc.VarType.FP16) + self.assertEqual( + x_gpu2.dtype, paddle.fluid.core.VarDesc.VarType.FP16 + ) x_cpu = self.x._to(device=paddle.CPUPlace()) self.assertTrue(x_cpu.place.is_cpu_place()) @@ -1673,7 +1762,6 @@ def test_to_api(self): class TestVarBaseInitVarBaseFromTensorWithDevice(unittest.TestCase): - def func_test_varbase_init(self): paddle.disable_static() t = fluid.Tensor() @@ -1703,7 +1791,6 @@ def test_varbase_init(self): class TestVarBaseNumel(unittest.TestCase): - def func_test_numel_normal(self): paddle.disable_static() np_x = np.random.random((3, 8, 8)) @@ -1733,7 +1820,6 @@ def ttest_numel_without_holder(self): class TestVarBaseCopyGradientFrom(unittest.TestCase): - def func_test_copy_gradient_from(self): paddle.disable_static() np_x = np.random.random((2, 2)) @@ -1752,7 +1838,6 @@ def test_copy_gradient_from(self): class TestEagerTensorGradNameValue(unittest.TestCase): - def test_eager_tensor_grad_name_value(self): with _test_eager_guard(): a_np = np.array([2, 3]).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_var_conv_2d.py b/python/paddle/fluid/tests/unittests/test_var_conv_2d.py index 9836ea72aab5e..ef313a13944af 100644 --- a/python/paddle/fluid/tests/unittests/test_var_conv_2d.py +++ b/python/paddle/fluid/tests/unittests/test_var_conv_2d.py @@ -18,7 +18,6 @@ class TestVarConv2DOp(OpTest): - def setUp(self): self.init_op_type() self.set_data() @@ -34,11 +33,13 @@ def set_data(self): stride = [1, 1] row = [2, 4] col = [3, 2] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) - def init_data(self, input_channel, output_channel, filter_size, stride, row, - col): + def init_data( + self, input_channel, output_channel, filter_size, stride, row, col + ): feature = [row[i] * col[i] for i in range(len(row))] numel = sum(feature) * input_channel @@ -46,14 +47,16 @@ def init_data(self, input_channel, output_channel, filter_size, stride, row, x_lod = [[x * input_channel for x in feature]] row_data = np.random.random((sum(row), 10)).astype('float32') col_data = np.random.random((sum(col), 10)).astype('float32') - w_shape = (output_channel, - input_channel * filter_size[0] * filter_size[1]) + w_shape = ( + output_channel, + input_channel * filter_size[0] * filter_size[1], + ) w_data = np.random.random(w_shape).astype('float32') self.inputs = { 'X': (x_data, x_lod), 'ROW': (row_data, [row]), 'COLUMN': (col_data, [col]), - 'W': w_data + 'W': w_data, } self.attrs = { 'InputChannel': input_channel, @@ -95,8 +98,9 @@ def compute(self): if top_im_size == 0: out_tmp = np.zeros((out_ch * top_im_size, 1)).astype('float32') else: - col_batch_data = col_res_data[col_data_offset:col_data_offset + - col_res_lod[0][idx]] + col_batch_data = col_res_data[ + col_data_offset : col_data_offset + col_res_lod[0][idx] + ] gemm_shape = (in_ch * kernel_h * kernel_w, top_im_size) col_batch_data = col_batch_data.reshape(gemm_shape) out_tmp = np.dot(w_data, col_batch_data).reshape(-1, 1) @@ -106,7 +110,7 @@ def compute(self): self.outputs = { 'Out': (out_data.astype('float32'), out_lod), - 'Col': (col_res_data, col_res_lod) + 'Col': (col_res_data, col_res_lod), } def Im2Col(self): @@ -159,11 +163,23 @@ def Im2Col(self): for kx in range(kernel_w): im_y = y + ky - half_kernel_h im_x = x + kx - half_kernel_w - if im_x >= 0 and im_x < width and im_y >= 0 and im_y < height: - col_res[t_offset + - (row_offset + ky * kernel_w + kx) * top_x + - col_offset] = \ - x_data[b_offset + im_offset + im_y * width + im_x] + if ( + im_x >= 0 + and im_x < width + and im_y >= 0 + and im_y < height + ): + col_res[ + t_offset + + (row_offset + ky * kernel_w + kx) + * top_x + + col_offset + ] = x_data[ + b_offset + + im_offset + + im_y * width + + im_x + ] t_offset += col_res_lod[0][idx] b_offset += x_lod[0][idx] @@ -174,14 +190,12 @@ def test_check_output(self): self.check_output(check_dygraph=False) def test_check_grad(self): - self.check_grad(['X'], - 'Out', - max_relative_error=0.005, - check_dygraph=False) + self.check_grad( + ['X'], 'Out', max_relative_error=0.005, check_dygraph=False + ) class TestVarConv2DOpCase1(TestVarConv2DOp): - def set_data(self): # set in_ch 1 input_channel = 1 @@ -190,12 +204,12 @@ def set_data(self): stride = [1, 1] row = [1, 10] col = [40, 6] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) class TestVarConv2DOpCase2(TestVarConv2DOp): - def set_data(self): # set out_ch 1 input_channel = 2 @@ -204,12 +218,12 @@ def set_data(self): stride = [2, 2] row = [6, 7] col = [8, 2] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) class TestVarConv2DOpCase3(TestVarConv2DOp): - def set_data(self): # set batch 1 input_channel = 2 @@ -218,12 +232,12 @@ def set_data(self): stride = [2, 2] row = [14] col = [4] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) class TestVarConv2DOpCase4(TestVarConv2DOp): - def set_data(self): # set filter size very large input_channel = 3 @@ -232,12 +246,12 @@ def set_data(self): stride = [2, 2] row = [4, 7] col = [5, 2] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) class TestVarConv2DOpCase5(TestVarConv2DOp): - def set_data(self): # set input very small input_channel = 50 @@ -246,16 +260,15 @@ def set_data(self): stride = [1, 1] row = [1, 1] col = [1, 1] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) @skip_check_grad_ci( - reason= - "[skip shape check] Use shape of input_channel, row and col all is 1 to test special LoDTensor." + reason="[skip shape check] Use shape of input_channel, row and col all is 1 to test special LoDTensor." ) class TestVarConv2DOpCase6(TestVarConv2DOp): - def set_data(self): input_channel = 1 output_channel = 3 @@ -263,12 +276,12 @@ def set_data(self): stride = [1, 1] row = [1, 1] col = [1, 1] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) class TestVarConv2DOpCase7(TestVarConv2DOp): - def set_data(self): input_channel = 2 output_channel = 3 @@ -276,43 +289,46 @@ def set_data(self): stride = [1, 1] row = [5, 4] col = [6, 7] - self.init_data(input_channel, output_channel, filter_size, stride, row, - col) + self.init_data( + input_channel, output_channel, filter_size, stride, row, col + ) class TestVarConv2DApi(unittest.TestCase): - def test_api(self): import paddle.fluid as fluid x = fluid.layers.data(name='x', shape=[1], lod_level=1) row = fluid.layers.data(name='row', shape=[6], lod_level=1) col = fluid.layers.data(name='col', shape=[6], lod_level=1) - out = fluid.contrib.var_conv_2d(input=x, - row=row, - col=col, - input_channel=3, - output_channel=5, - filter_size=[3, 3], - stride=1) + out = fluid.contrib.var_conv_2d( + input=x, + row=row, + col=col, + input_channel=3, + output_channel=5, + filter_size=[3, 3], + stride=1, + ) place = fluid.CPUPlace() x_tensor = fluid.create_lod_tensor( - np.random.rand(116, 1).astype('float32'), [[60, 56]], place) + np.random.rand(116, 1).astype('float32'), [[60, 56]], place + ) row_tensor = fluid.create_lod_tensor( - np.random.rand(9, 6).astype('float32'), [[5, 4]], place) + np.random.rand(9, 6).astype('float32'), [[5, 4]], place + ) col_tensor = fluid.create_lod_tensor( - np.random.rand(13, 6).astype('float32'), [[6, 7]], place) + np.random.rand(13, 6).astype('float32'), [[6, 7]], place + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'x': x_tensor, - 'row': row_tensor, - 'col': col_tensor - }, - fetch_list=[out], - return_numpy=False) + ret = exe.run( + feed={'x': x_tensor, 'row': row_tensor, 'col': col_tensor}, + fetch_list=[out], + return_numpy=False, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_var_info.py b/python/paddle/fluid/tests/unittests/test_var_info.py index 7af77ae96b551..4bb6648488ee6 100644 --- a/python/paddle/fluid/tests/unittests/test_var_info.py +++ b/python/paddle/fluid/tests/unittests/test_var_info.py @@ -22,10 +22,10 @@ class TestVarInfo(unittest.TestCase): - """ TestCases for Dataset. """ + """TestCases for Dataset.""" def test_var_info(self): - """ Testcase for get and set info for variable. """ + """Testcase for get and set info for variable.""" value = np.random.randn(1) var = fluid.layers.create_global_var([1], value, "float32") var._set_info("name", "test") diff --git a/python/paddle/fluid/tests/unittests/test_variable.py b/python/paddle/fluid/tests/unittests/test_variable.py index 8ab9d14c924db..bf9ee54afc767 100644 --- a/python/paddle/fluid/tests/unittests/test_variable.py +++ b/python/paddle/fluid/tests/unittests/test_variable.py @@ -16,7 +16,11 @@ from functools import reduce import paddle -from paddle.fluid.framework import Program, convert_np_dtype_to_dtype_, default_main_program +from paddle.fluid.framework import ( + Program, + convert_np_dtype_to_dtype_, + default_main_program, +) import paddle import paddle.fluid as fluid import paddle.fluid.core as core @@ -26,7 +30,6 @@ class TestVariable(unittest.TestCase): - def setUp(self): np.random.seed(2022) @@ -45,10 +48,9 @@ def test_np_dtype_convert(self): def test_var(self): b = default_main_program().current_block() - w = b.create_var(dtype="float64", - shape=[784, 100], - lod_level=0, - name="fc.w") + w = b.create_var( + dtype="float64", shape=[784, 100], lod_level=0, name="fc.w" + ) self.assertNotEqual(str(w), "") self.assertEqual(core.VarDesc.VarType.FP64, w.dtype) self.assertEqual((784, 100), w.shape) @@ -63,12 +65,15 @@ def test_var(self): self.assertEqual("fc.w@GRAD", w.grad_name) self.assertEqual(0, w.lod_level) - self.assertRaises(ValueError, - lambda: b.create_var(name="fc.w", shape=(24, 100))) + self.assertRaises( + ValueError, lambda: b.create_var(name="fc.w", shape=(24, 100)) + ) - w = b.create_var(dtype=paddle.fluid.core.VarDesc.VarType.STRINGS, - shape=[1], - name="str_var") + w = b.create_var( + dtype=paddle.fluid.core.VarDesc.VarType.STRINGS, + shape=[1], + name="str_var", + ) self.assertEqual(None, w.lod_level) def test_element_size(self): @@ -103,8 +108,9 @@ def test_element_size(self): def test_step_scopes(self): prog = Program() b = prog.current_block() - var = b.create_var(name='step_scopes', - type=core.VarDesc.VarType.STEP_SCOPES) + var = b.create_var( + name='step_scopes', type=core.VarDesc.VarType.STEP_SCOPES + ) self.assertEqual(core.VarDesc.VarType.STEP_SCOPES, var.type) def _test_slice(self, place): @@ -137,10 +143,13 @@ def _test_slice(self, place): main = fluid.Program() with fluid.program_guard(main): exe = fluid.Executor(place) - tensor_array = np.array([[[1, 2, 3], [4, 5, 6], [7, 8, 9]], - [[10, 11, 12], [13, 14, 15], [16, 17, 18]], - [[19, 20, 21], [22, 23, 24], - [25, 26, 27]]]).astype('float32') + tensor_array = np.array( + [ + [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + [[10, 11, 12], [13, 14, 15], [16, 17, 18]], + [[19, 20, 21], [22, 23, 24], [25, 26, 27]], + ] + ).astype('float32') var = fluid.layers.assign(tensor_array) var1 = var[0, 1, 1] var2 = var[1:] @@ -167,13 +176,28 @@ def _test_slice(self, place): data.append((np.random.randint(10, size=[13]).astype('float32'))) exe.run(fluid.default_startup_program()) - local_out = exe.run(main, - feed=feeder.feed([data]), - fetch_list=[ - var, var1, var2, var3, var4, var5, var6, - var7, var8, var9, var10, var11, var12, - var13, var14, var15 - ]) + local_out = exe.run( + main, + feed=feeder.feed([data]), + fetch_list=[ + var, + var1, + var2, + var3, + var4, + var5, + var6, + var7, + var8, + var9, + var10, + var11, + var12, + var13, + var14, + var15, + ], + ) np.testing.assert_array_equal(local_out[1], tensor_array[0, 1, 1:2]) np.testing.assert_array_equal(local_out[2], tensor_array[1:]) @@ -181,25 +205,33 @@ def _test_slice(self, place): np.testing.assert_array_equal(local_out[4], tensor_array[::-1]) np.testing.assert_array_equal(local_out[5], tensor_array[1, 1:, 1:]) np.testing.assert_array_equal( - local_out[6], - tensor_array.reshape((3, -1, 3))[:, :, -1]) + local_out[6], tensor_array.reshape((3, -1, 3))[:, :, -1] + ) np.testing.assert_array_equal(local_out[7], tensor_array[:, :, :-1]) - np.testing.assert_array_equal(local_out[8], - tensor_array[:1, :1, :1]) - np.testing.assert_array_equal(local_out[9], - tensor_array[:-1, :-1, :-1]) - np.testing.assert_array_equal(local_out[10], - tensor_array[::-1, :1, :-1]) - np.testing.assert_array_equal(local_out[11], tensor_array[:-1, ::-1, - -1:]) - np.testing.assert_array_equal(local_out[12], tensor_array[1:2, - 2:, ::-1]) - np.testing.assert_array_equal(local_out[13], tensor_array[2:10, 2:, - -2:-1]) - np.testing.assert_array_equal(local_out[14], - tensor_array[1:-1, 0:2, ::-1]) - np.testing.assert_array_equal(local_out[15], - tensor_array[::-1, ::-1, ::-1]) + np.testing.assert_array_equal( + local_out[8], tensor_array[:1, :1, :1] + ) + np.testing.assert_array_equal( + local_out[9], tensor_array[:-1, :-1, :-1] + ) + np.testing.assert_array_equal( + local_out[10], tensor_array[::-1, :1, :-1] + ) + np.testing.assert_array_equal( + local_out[11], tensor_array[:-1, ::-1, -1:] + ) + np.testing.assert_array_equal( + local_out[12], tensor_array[1:2, 2:, ::-1] + ) + np.testing.assert_array_equal( + local_out[13], tensor_array[2:10, 2:, -2:-1] + ) + np.testing.assert_array_equal( + local_out[14], tensor_array[1:-1, 0:2, ::-1] + ) + np.testing.assert_array_equal( + local_out[15], tensor_array[::-1, ::-1, ::-1] + ) def _test_slice_index_tensor(self, place): data = np.random.rand(2, 3).astype("float32") @@ -270,13 +302,18 @@ def _test_slice_index_ellipsis(self, place): out7 = y[..., 0] exe = paddle.static.Executor(place) - result = exe.run(prog, - fetch_list=[out1, out2, out3, out4, out5, out6, out7]) + result = exe.run( + prog, fetch_list=[out1, out2, out3, out4, out5, out6, out7] + ) expected = [ - data[0:, ..., 1:], data[0:, ...], data[..., 1:], data[...], - data[[1, 0], [0, 0]], data[([1, 0], [0, 0])], - np.array([1]) + data[0:, ..., 1:], + data[0:, ...], + data[..., 1:], + data[...], + data[[1, 0], [0, 0]], + data[([1, 0], [0, 0])], + np.array([1]), ] self.assertTrue((result[0] == expected[0]).all()) @@ -314,11 +351,18 @@ def _test_slice_index_list_bool(self, place): exe = paddle.static.Executor(place) result = exe.run( - prog, fetch_list=[out0, out1, out2, out3, out4, out5, out6, out7]) + prog, fetch_list=[out0, out1, out2, out3, out4, out5, out6, out7] + ) expected = [ - data[idx0], data[idx1], data[idx2], data[idx3], data[idx4], - data[np_idx], data[data < 0.36], data[data > 0.6] + data[idx0], + data[idx1], + data[idx2], + data[idx3], + data[idx4], + data[np_idx], + data[data < 0.36], + data[data > 0.6], ] self.assertTrue((result[0] == expected[0]).all()) @@ -413,11 +457,13 @@ def test_variable_in_dygraph_mode(self): def test_create_selected_rows(self): b = default_main_program().current_block() - var = b.create_var(name="var", - shape=[1, 1], - dtype="float32", - type=fluid.core.VarDesc.VarType.SELECTED_ROWS, - persistable=True) + var = b.create_var( + name="var", + shape=[1, 1], + dtype="float32", + type=fluid.core.VarDesc.VarType.SELECTED_ROWS, + persistable=True, + ) def _test(): var.lod_level() @@ -452,17 +498,17 @@ def test_detach(self): scope = fluid.core.Scope() with paddle.static.scope_guard(scope): with paddle.static.program_guard(main, startup): - x = paddle.static.data(name='x', - shape=[3, 2, 1], - dtype='float32') + x = paddle.static.data( + name='x', shape=[3, 2, 1], dtype='float32' + ) x.persistable = True feed_data = np.ones(shape=[3, 2, 1], dtype=np.float32) detach_x = x.detach() exe = paddle.static.Executor(paddle.CPUPlace()) exe.run(startup) - result = exe.run(main, - feed={'x': feed_data}, - fetch_list=[x, detach_x]) + result = exe.run( + main, feed={'x': feed_data}, fetch_list=[x, detach_x] + ) self.assertTrue((result[1] == feed_data).all()) self.assertTrue((result[0] == result[1]).all()) @@ -472,9 +518,9 @@ def test_detach(self): self.assertTrue((result[1] == modified_value).all()) self.assertTrue((result[0] == result[1]).all()) - modified_value = np.random.uniform(-1, 1, - size=[3, 2, - 1]).astype('float32') + modified_value = np.random.uniform( + -1, 1, size=[3, 2, 1] + ).astype('float32') x.set_value(modified_value, scope) result = exe.run(main, fetch_list=[x, detach_x]) self.assertTrue((result[1] == modified_value).all()) @@ -482,7 +528,6 @@ def test_detach(self): class TestVariableSlice(unittest.TestCase): - def setUp(self): np.random.seed(2022) @@ -502,8 +547,11 @@ def _test_item_none(self, place): result = exe.run(prog, fetch_list=outs) expected = [ - data[0:, None, 1:], data[0:, None], data[None, 1:], data[None], - data[..., None, :, None] + data[0:, None, 1:], + data[0:, None], + data[None, 1:], + data[None], + data[..., None, :, None], ] for i in range(len(outs)): self.assertEqual(outs[i].shape, expected[i].shape) @@ -525,8 +573,12 @@ def _test_item_none_and_decrease(self, place): exe = paddle.static.Executor(place) result = exe.run(prog, fetch_list=outs) expected = [ - data[0, 1:, None], data[0, None], data[None, 1], data[None], - data[0, 0, 0, None], data[None, 0, 0, 0, None] + data[0, 1:, None], + data[0, None], + data[None, 1], + data[None], + data[0, 0, 0, None], + data[None, 0, 0, 0, None], ] for i in range(len(outs)): @@ -544,7 +596,6 @@ def test_slice(self): class TestListIndex(unittest.TestCase): - def setUp(self): np.random.seed(2022) @@ -555,8 +606,9 @@ def test_static_graph_list_index(self): paddle.enable_static() inps_shape = [3, 4, 5, 2] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [3, 3, 2, 1] index = np.arange(self.numel(index_shape)).reshape(index_shape) @@ -567,15 +619,17 @@ def test_static_graph_list_index(self): index_mod = (index % (array.shape[0])).tolist() with paddle.static.program_guard(program): - x = paddle.static.data(name='x', - shape=array.shape, - dtype='float32') + x = paddle.static.data( + name='x', shape=array.shape, dtype='float32' + ) y = x[index_mod] - place = paddle.fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else paddle.fluid.CUDAPlace(0) + place = ( + paddle.fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else paddle.fluid.CUDAPlace(0) + ) prog = paddle.static.default_main_program() exe = paddle.static.Executor(place) @@ -584,9 +638,9 @@ def test_static_graph_list_index(self): fetch_list = [y.name] getitem_np = array[index_mod] - getitem_pp = exe.run(prog, - feed={x.name: array}, - fetch_list=fetch_list) + getitem_pp = exe.run( + prog, feed={x.name: array}, fetch_list=fetch_list + ) np.testing.assert_array_equal(getitem_np, getitem_pp[0]) array = array[0] @@ -622,16 +676,21 @@ def test_dygraph_list_index(self): def test_static_graph_list_index_muti_dim(self): paddle.enable_static() inps_shape = [3, 4, 5] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [2, 2] index1 = np.arange(self.numel(index_shape)).reshape(index_shape) index2 = np.arange(self.numel(index_shape)).reshape(index_shape) + 2 value_shape = [3, 2, 2, 3] - value_np = np.arange(self.numel(value_shape), - dtype='float32').reshape(value_shape) + 100 + value_np = ( + np.arange(self.numel(value_shape), dtype='float32').reshape( + value_shape + ) + + 100 + ) index_mod1 = (index1 % (min(array.shape))).tolist() index_mod2 = (index2 % (min(array.shape))).tolist() @@ -641,21 +700,23 @@ def test_static_graph_list_index_muti_dim(self): x = paddle.static.data(name='x', shape=array.shape, dtype='float32') - value = paddle.static.data(name='value', - shape=value_np.shape, - dtype='float32') - index1 = paddle.static.data(name='index1', - shape=index1.shape, - dtype='int32') - index2 = paddle.static.data(name='index2', - shape=index2.shape, - dtype='int32') + value = paddle.static.data( + name='value', shape=value_np.shape, dtype='float32' + ) + index1 = paddle.static.data( + name='index1', shape=index1.shape, dtype='int32' + ) + index2 = paddle.static.data( + name='index2', shape=index2.shape, dtype='int32' + ) y = x[index1, index2] - place = paddle.fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else paddle.fluid.CUDAPlace(0) + place = ( + paddle.fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else paddle.fluid.CUDAPlace(0) + ) prog = paddle.static.default_main_program() exe = paddle.static.Executor(place) @@ -666,32 +727,40 @@ def test_static_graph_list_index_muti_dim(self): y2 = array2[index_mod1, index_mod2] - getitem_pp = exe.run(prog, - feed={ - x.name: array, - index1.name: index_mod1, - index2.name: index_mod2 - }, - fetch_list=fetch_list) + getitem_pp = exe.run( + prog, + feed={ + x.name: array, + index1.name: index_mod1, + index2.name: index_mod2, + }, + fetch_list=fetch_list, + ) np.testing.assert_array_equal( y2, getitem_pp[0], - err_msg='\n numpy:{},\n paddle:{}'.format(y2, getitem_pp[0])) + err_msg='\n numpy:{},\n paddle:{}'.format(y2, getitem_pp[0]), + ) def test_dygraph_list_index_muti_dim(self): paddle.disable_static() inps_shape = [3, 4, 5] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [2, 2] index1 = np.arange(self.numel(index_shape)).reshape(index_shape) index2 = np.arange(self.numel(index_shape)).reshape(index_shape) + 2 value_shape = [3, 2, 2, 3] - value_np = np.arange(self.numel(value_shape), - dtype='float32').reshape(value_shape) + 100 + value_np = ( + np.arange(self.numel(value_shape), dtype='float32').reshape( + value_shape + ) + + 100 + ) index_mod1 = (index1 % (min(array.shape))).tolist() index_mod2 = (index2 % (min(array.shape))).tolist() @@ -721,16 +790,15 @@ def run_getitem_list_index(self, array, index): value_np = array2[index] except: with self.assertRaises(ValueError): - getitem_pp = exe.run(prog, - feed={x.name: array}, - fetch_list=fetch_list) + getitem_pp = exe.run( + prog, feed={x.name: array}, fetch_list=fetch_list + ) return getitem_pp = exe.run(prog, feed={x.name: array}, fetch_list=fetch_list) - np.testing.assert_allclose(value_np, - getitem_pp[0], - rtol=1e-5, - atol=1e-8) + np.testing.assert_allclose( + value_np, getitem_pp[0], rtol=1e-5, atol=1e-8 + ) def test_static_graph_getitem_bool_index(self): paddle.enable_static() @@ -762,9 +830,9 @@ def test_static_graph_getitem_bool_index(self): def run_setitem_list_index(self, array, index, value_np): x = paddle.static.data(name='x', shape=array.shape, dtype='float32') - value = paddle.static.data(name='value', - shape=value_np.shape, - dtype='float32') + value = paddle.static.data( + name='value', shape=value_np.shape, dtype='float32' + ) x[index] = value y = x @@ -781,19 +849,17 @@ def run_setitem_list_index(self, array, index, value_np): array2[index] = value_np except: with self.assertRaises(ValueError): - setitem_pp = exe.run(prog, - feed={ - x.name: array, - value.name: value_np - }, - fetch_list=fetch_list) + setitem_pp = exe.run( + prog, + feed={x.name: array, value.name: value_np}, + fetch_list=fetch_list, + ) return - setitem_pp = exe.run(prog, - feed={ - x.name: array, - value.name: value_np - }, - fetch_list=fetch_list) + setitem_pp = exe.run( + prog, + feed={x.name: array, value.name: value_np}, + fetch_list=fetch_list, + ) np.testing.assert_allclose(array2, setitem_pp[0], rtol=1e-5, atol=1e-8) @@ -801,15 +867,20 @@ def test_static_graph_setitem_list_index(self): paddle.enable_static() # case 1: inps_shape = [3, 4, 5, 2, 3] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [3, 3, 1, 2] index = np.arange(self.numel(index_shape)).reshape(index_shape) value_shape = inps_shape[3:] - value_np = np.arange(self.numel(value_shape), - dtype='float32').reshape(value_shape) + 100 + value_np = ( + np.arange(self.numel(value_shape), dtype='float32').reshape( + value_shape + ) + + 100 + ) for _ in range(3): program = paddle.static.Program() @@ -824,15 +895,20 @@ def test_static_graph_setitem_list_index(self): # case 2: inps_shape = [3, 4, 5, 4, 3] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [4, 3, 2, 2] index = np.arange(self.numel(index_shape)).reshape(index_shape) value_shape = [3] - value_np = np.arange(self.numel(value_shape), - dtype='float32').reshape(value_shape) + 100 + value_np = ( + np.arange(self.numel(value_shape), dtype='float32').reshape( + value_shape + ) + + 100 + ) for _ in range(4): program = paddle.static.Program() @@ -846,15 +922,20 @@ def test_static_graph_setitem_list_index(self): # case 3: inps_shape = [3, 4, 5, 3, 3] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [4, 3, 2, 2] index = np.arange(self.numel(index_shape)).reshape(index_shape) value_shape = [3, 2, 2, 3] - value_np = np.arange(self.numel(value_shape), - dtype='float32').reshape(value_shape) + 100 + value_np = ( + np.arange(self.numel(value_shape), dtype='float32').reshape( + value_shape + ) + + 100 + ) index_mod = (index % (min(array.shape))).tolist() self.run_setitem_list_index(array, index_mod, value_np) @@ -897,18 +978,28 @@ def test_static_graph_setitem_bool_scalar_index(self): def test_static_graph_tensor_index_setitem_muti_dim(self): paddle.enable_static() inps_shape = [3, 4, 5, 4] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [2, 3, 4] - index1 = np.arange(self.numel(index_shape), - dtype='int32').reshape(index_shape) - index2 = np.arange(self.numel(index_shape), - dtype='int32').reshape(index_shape) + 2 + index1 = np.arange(self.numel(index_shape), dtype='int32').reshape( + index_shape + ) + index2 = ( + np.arange(self.numel(index_shape), dtype='int32').reshape( + index_shape + ) + + 2 + ) value_shape = [4] - value_np = np.arange(self.numel(value_shape), - dtype='float32').reshape(value_shape) + 100 + value_np = ( + np.arange(self.numel(value_shape), dtype='float32').reshape( + value_shape + ) + + 100 + ) for _ in range(3): index_mod1 = index1 % (min(array.shape)) @@ -922,29 +1013,31 @@ def test_static_graph_tensor_index_setitem_muti_dim(self): program = paddle.static.Program() with paddle.static.program_guard(program): - x1 = paddle.static.data(name='x1', - shape=array.shape, - dtype='float32') - x2 = paddle.static.data(name='x2', - shape=array.shape, - dtype='float32') - - value = paddle.static.data(name='value', - shape=value_np.shape, - dtype='float32') - index_1 = paddle.static.data(name='index_1', - shape=index1.shape, - dtype='int32') - index_2 = paddle.static.data(name='index_2', - shape=index2.shape, - dtype='int32') + x1 = paddle.static.data( + name='x1', shape=array.shape, dtype='float32' + ) + x2 = paddle.static.data( + name='x2', shape=array.shape, dtype='float32' + ) + + value = paddle.static.data( + name='value', shape=value_np.shape, dtype='float32' + ) + index_1 = paddle.static.data( + name='index_1', shape=index1.shape, dtype='int32' + ) + index_2 = paddle.static.data( + name='index_2', shape=index2.shape, dtype='int32' + ) x1[index_1, index_2] = value x2[index_1] = value - place = paddle.fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else paddle.fluid.CUDAPlace(0) + place = ( + paddle.fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else paddle.fluid.CUDAPlace(0) + ) prog = paddle.static.default_main_program() exe = paddle.static.Executor(place) @@ -952,25 +1045,31 @@ def test_static_graph_tensor_index_setitem_muti_dim(self): exe.run(paddle.static.default_startup_program()) fetch_list = [x1.name, x2.name] - setitem_pp = exe.run(prog, - feed={ - x1.name: array, - x2.name: array, - value.name: value_np, - index_1.name: index_mod1, - index_2.name: index_mod2 - }, - fetch_list=fetch_list) + setitem_pp = exe.run( + prog, + feed={ + x1.name: array, + x2.name: array, + value.name: value_np, + index_1.name: index_mod1, + index_2.name: index_mod2, + }, + fetch_list=fetch_list, + ) np.testing.assert_array_equal( array2, setitem_pp[0], err_msg='\n numpy:{},\n paddle:{}'.format( - array2, setitem_pp[0])) + array2, setitem_pp[0] + ), + ) np.testing.assert_array_equal( array3, setitem_pp[1], err_msg='\n numpy:{},\n paddle:{}'.format( - array3, setitem_pp[1])) + array3, setitem_pp[1] + ), + ) array = array[0] index1 = index1[0] index2 = index2[0] @@ -978,14 +1077,20 @@ def test_static_graph_tensor_index_setitem_muti_dim(self): def test_static_graph_array_index_muti_dim(self): paddle.enable_static() inps_shape = [3, 4, 5, 4] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [2, 3, 4] - index1 = np.arange(self.numel(index_shape), - dtype='int32').reshape(index_shape) - index2 = np.arange(self.numel(index_shape), - dtype='int32').reshape(index_shape) + 2 + index1 = np.arange(self.numel(index_shape), dtype='int32').reshape( + index_shape + ) + index2 = ( + np.arange(self.numel(index_shape), dtype='int32').reshape( + index_shape + ) + + 2 + ) for _ in range(3): index_mod1 = index1 % (min(array.shape)) @@ -1001,53 +1106,62 @@ def test_static_graph_array_index_muti_dim(self): program = paddle.static.Program() with paddle.static.program_guard(program): - x1 = paddle.static.data(name='x1', - shape=array.shape, - dtype='float32') - x2 = paddle.static.data(name='x2', - shape=array.shape, - dtype='float32') + x1 = paddle.static.data( + name='x1', shape=array.shape, dtype='float32' + ) + x2 = paddle.static.data( + name='x2', shape=array.shape, dtype='float32' + ) x1[index_mod1, index_mod2] = 1 x2[index_mod1] = 2.5 y1 = x1[index_mod2, index_mod1] y2 = x2[index_mod2] - place = paddle.fluid.CPUPlace( - ) if not paddle.fluid.core.is_compiled_with_cuda( - ) else paddle.fluid.CUDAPlace(0) + place = ( + paddle.fluid.CPUPlace() + if not paddle.fluid.core.is_compiled_with_cuda() + else paddle.fluid.CUDAPlace(0) + ) prog = paddle.static.default_main_program() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) fetch_list = [x1.name, x2.name, y1.name, y2.name] - setitem_pp = exe.run(prog, - feed={ - x1.name: array, - x2.name: array - }, - fetch_list=fetch_list) + setitem_pp = exe.run( + prog, + feed={x1.name: array, x2.name: array}, + fetch_list=fetch_list, + ) np.testing.assert_array_equal( array2, setitem_pp[0], err_msg='\n numpy:{},\n paddle:{}'.format( - array2, setitem_pp[0])) + array2, setitem_pp[0] + ), + ) np.testing.assert_array_equal( array3, setitem_pp[1], err_msg='\n numpy:{},\n paddle:{}'.format( - array3, setitem_pp[1])) + array3, setitem_pp[1] + ), + ) np.testing.assert_array_equal( y_np1, setitem_pp[2], err_msg='\n numpy:{},\n paddle:{}'.format( - y_np1, setitem_pp[2])) + y_np1, setitem_pp[2] + ), + ) np.testing.assert_array_equal( y_np2, setitem_pp[3], err_msg='\n numpy:{},\n paddle:{}'.format( - y_np2, setitem_pp[3])) + y_np2, setitem_pp[3] + ), + ) array = array[0] index1 = index1[0] index2 = index2[0] @@ -1055,13 +1169,19 @@ def test_static_graph_array_index_muti_dim(self): def test_dygraph_array_index_muti_dim(self): paddle.disable_static() inps_shape = [3, 4, 5, 4] - array = np.arange(self.numel(inps_shape), - dtype='float32').reshape(inps_shape) + array = np.arange(self.numel(inps_shape), dtype='float32').reshape( + inps_shape + ) index_shape = [2, 3, 4] - index1 = np.arange(self.numel(index_shape), - dtype='int32').reshape(index_shape) - index2 = np.arange(self.numel(index_shape), - dtype='int32').reshape(index_shape) + 2 + index1 = np.arange(self.numel(index_shape), dtype='int32').reshape( + index_shape + ) + index2 = ( + np.arange(self.numel(index_shape), dtype='int32').reshape( + index_shape + ) + + 2 + ) for _ in range(3): @@ -1079,7 +1199,8 @@ def test_dygraph_array_index_muti_dim(self): np.testing.assert_array_equal( y_t1.numpy(), y_np1, - err_msg='\n numpy:{},\n paddle:{}'.format(y_np1, y_t1.numpy())) + err_msg='\n numpy:{},\n paddle:{}'.format(y_np1, y_t1.numpy()), + ) # 1 dim getitem array2 = array.copy() y_np2 = array2[index_mod2] @@ -1089,7 +1210,8 @@ def test_dygraph_array_index_muti_dim(self): np.testing.assert_array_equal( y_t2.numpy(), y_np2, - err_msg='\n numpy:{},\n paddle:{}'.format(y_np2, y_t2.numpy())) + err_msg='\n numpy:{},\n paddle:{}'.format(y_np2, y_t2.numpy()), + ) # 2 dim setitem array1 = array.copy() @@ -1099,7 +1221,9 @@ def test_dygraph_array_index_muti_dim(self): tensor1.numpy(), array1, err_msg='\n numpy:{},\n paddle:{}'.format( - array1, tensor1.numpy())) + array1, tensor1.numpy() + ), + ) # 1 dim setitem array2 = array.copy() @@ -1111,7 +1235,9 @@ def test_dygraph_array_index_muti_dim(self): tensor2.numpy(), array2, err_msg='\n numpy:{},\n paddle:{}'.format( - array2, tensor2.numpy())) + array2, tensor2.numpy() + ), + ) array = array[0] index1 = index1[0] diff --git a/python/paddle/fluid/tests/unittests/test_variance_layer.py b/python/paddle/fluid/tests/unittests/test_variance_layer.py index ca0ad7e9fd983..fc97a20533bcb 100644 --- a/python/paddle/fluid/tests/unittests/test_variance_layer.py +++ b/python/paddle/fluid/tests/unittests/test_variance_layer.py @@ -20,14 +20,13 @@ def ref_var(x, axis=None, unbiased=True, keepdim=False): ddof = 1 if unbiased else 0 if isinstance(axis, int): - axis = (axis, ) + axis = (axis,) if axis is not None: axis = tuple(axis) return np.var(x, axis=axis, ddof=ddof, keepdims=keepdim) class TestVarAPI(unittest.TestCase): - def setUp(self): self.dtype = 'float64' self.shape = [1, 3, 4, 10] @@ -36,9 +35,11 @@ def setUp(self): self.unbiased = True self.set_attrs() self.x = np.random.uniform(-1, 1, self.shape).astype(self.dtype) - self.place=paddle.CUDAPlace(0) \ - if paddle.fluid.core.is_compiled_with_cuda() \ + self.place = ( + paddle.CUDAPlace(0) + if paddle.fluid.core.is_compiled_with_cuda() else paddle.CPUPlace() + ) def set_attrs(self): pass @@ -68,43 +69,36 @@ def test_api(self): class TestVarAPI_dtype(TestVarAPI): - def set_attrs(self): self.dtype = 'float32' class TestVarAPI_axis_int(TestVarAPI): - def set_attrs(self): self.axis = 2 class TestVarAPI_axis_list(TestVarAPI): - def set_attrs(self): self.axis = [1, 2] class TestVarAPI_axis_tuple(TestVarAPI): - def set_attrs(self): self.axis = (1, 3) class TestVarAPI_keepdim(TestVarAPI): - def set_attrs(self): self.keepdim = False class TestVarAPI_unbiased(TestVarAPI): - def set_attrs(self): self.unbiased = False class TestVarAPI_alias(unittest.TestCase): - def test_alias(self): paddle.disable_static() x = paddle.to_tensor(np.array([10, 12], 'float32')) @@ -117,7 +111,6 @@ def test_alias(self): class TestVarError(unittest.TestCase): - def test_error(self): with paddle.static.program_guard(paddle.static.Program()): x = paddle.fluid.data('X', [2, 3, 4], 'int32') diff --git a/python/paddle/fluid/tests/unittests/test_version.py b/python/paddle/fluid/tests/unittests/test_version.py index 0fda61f7bbeb0..507ec953764df 100644 --- a/python/paddle/fluid/tests/unittests/test_version.py +++ b/python/paddle/fluid/tests/unittests/test_version.py @@ -19,7 +19,6 @@ class VersionTest(unittest.TestCase): - def setUp(self): self._major_regex = "[0-9]+" self._minor_regex = "[0-9]+" @@ -46,4 +45,5 @@ def test_check_output(self): self.assertTrue(re.match(self._patch_regex, fluid_version.patch)) self.assertTrue(re.match(self._rc_regex, fluid_version.rc)) self.assertTrue( - re.match(self._version_regex, fluid_version.full_version)) + re.match(self._version_regex, fluid_version.full_version) + ) diff --git a/python/paddle/fluid/tests/unittests/test_view_op_reuse_allocation.py b/python/paddle/fluid/tests/unittests/test_view_op_reuse_allocation.py index 1f4725568a810..ab2c091cc0cb4 100644 --- a/python/paddle/fluid/tests/unittests/test_view_op_reuse_allocation.py +++ b/python/paddle/fluid/tests/unittests/test_view_op_reuse_allocation.py @@ -26,7 +26,6 @@ # reuse the input varbase's allocation. # View APIs include: `squeeze`, `unsqueeze`, `reshape`, `flatten`, `detach` class TestDygraphViewReuseAllocation(unittest.TestCase): - def setUp(self): self.init_shape() @@ -40,7 +39,7 @@ def view_api_processing(self, var): def func_test_view_api(self): var = paddle.rand(self.input_shape) view_var = self.view_api_processing(var) - view_var[0] = 2. + view_var[0] = 2.0 self.assertEqual(var.shape, self.input_shape) self.assertEqual(view_var.shape, self.output_shape) @@ -59,14 +58,14 @@ def func_test_forward_version(self): view_var = self.view_api_processing(var) self.assertEqual(view_var.inplace_version, 0) - var[0] = 2. + var[0] = 2.0 self.assertEqual(var.inplace_version, 1) self.assertEqual(view_var.inplace_version, 1) view_var_2 = self.view_api_processing(var) self.assertEqual(view_var_2.inplace_version, 1) - var[0] = 3. + var[0] = 3.0 self.assertEqual(view_var.inplace_version, 2) self.assertEqual(view_var_2.inplace_version, 2) @@ -87,13 +86,15 @@ def func_test_backward_error(self): # Here, the gradient computation will use the value of var_b var_c = var_b**2 view_var_b = self.view_api_processing(var_b) - view_var_b[0] = 2. # var_b is modified inplace + view_var_b[0] = 2.0 # var_b is modified inplace loss = paddle.nn.functional.relu(var_c) with self.assertRaisesRegexp( - RuntimeError, - "received tensor_version:{} != wrapper_version_snapshot:{}". - format(1, 0)): + RuntimeError, + "received tensor_version:{} != wrapper_version_snapshot:{}".format( + 1, 0 + ), + ): loss.backward() def test_backward_error(self): @@ -103,7 +104,6 @@ def test_backward_error(self): class TestUnsqueezeDygraphViewReuseAllocation(TestDygraphViewReuseAllocation): - def init_shape(self): self.input_shape = [2, 3] self.output_shape = [2, 3, 1] @@ -113,7 +113,6 @@ def view_api_processing(self, var): class TestReshapeDygraphViewReuseAllocation(TestDygraphViewReuseAllocation): - def init_shape(self): self.input_shape = [3, 4] self.output_shape = [2, 2, 3] @@ -123,7 +122,6 @@ def view_api_processing(self, var): class TestFlattenDygraphViewReuseAllocation(TestDygraphViewReuseAllocation): - def init_shape(self): self.input_shape = [3, 4] self.output_shape = [12] diff --git a/python/paddle/fluid/tests/unittests/test_viterbi_decode_op.py b/python/paddle/fluid/tests/unittests/test_viterbi_decode_op.py index 7b78957777154..624f2f5e6165b 100644 --- a/python/paddle/fluid/tests/unittests/test_viterbi_decode_op.py +++ b/python/paddle/fluid/tests/unittests/test_viterbi_decode_op.py @@ -19,7 +19,6 @@ class Decoder(object): - def __init__(self, transitions, use_tag=True): self.transitions = transitions self.use_tag = use_tag @@ -33,8 +32,11 @@ def __call__(self, inputs, length): left_length = np.array(length) max_seq_len = np.amax(left_length) left_length = np.expand_dims(left_length, 1) - alpha = np.full((bs, n_label), -1e4, dtype='float32') if self.use_tag \ + alpha = ( + np.full((bs, n_label), -1e4, dtype='float32') + if self.use_tag else np.zeros((bs, n_label), dtype='float32') + ) alpha[:, -1] = 0 for i, logit in enumerate(inputs_t[:max_seq_len]): if i == 0 and not self.use_tag: @@ -46,7 +48,7 @@ def __call__(self, inputs, length): max_res = np.amax(alpha_trn_sum, 1), np.argmax(alpha_trn_sum, 1) historys = historys + [max_res[1]] if i >= 1 else [] alpha_nxt = max_res[0] + logit - mask = (left_length > 0) + mask = left_length > 0 alpha = mask * alpha_nxt + (1 - mask) * alpha if self.use_tag: alpha += (left_length == 1) * trans_exp[:, self.stop_idx] @@ -60,7 +62,7 @@ def __call__(self, inputs, length): left_length = left_length + 1 gather_idx = batch_offset + last_ids last_ids_update = np.take(hist, gather_idx) * (left_length > 0) - mask = (left_length == 0) + mask = left_length == 0 last_ids_update = last_ids_update * (1 - mask) + last_ids * mask batch_path.insert(0, last_ids_update) last_ids = last_ids_update + (left_length < 0) * last_ids @@ -69,7 +71,6 @@ def __call__(self, inputs, length): class TestViterbiOp(OpTest): - def set_attr(self): self.dtype = "float32" if core.is_compiled_with_rocm() else "float64" self.use_tag = True @@ -88,7 +89,7 @@ def setUp(self): self.inputs = { 'Input': self.input, 'Transition': self.trans, - 'Length': self.length + 'Length': self.length, } self.attrs = { 'include_bos_eos_tag': self.use_tag, @@ -100,12 +101,14 @@ def test_output(self): class TestViterbiAPI(unittest.TestCase): - def set_attr(self): self.use_tag = True self.bz, self.len, self.ntags = 4, 8, 10 - self.places = [fluid.CPUPlace(), fluid.CUDAPlace(0)] \ - if core.is_compiled_with_cuda() else [fluid.CPUPlace()] + self.places = ( + [fluid.CPUPlace(), fluid.CUDAPlace(0)] + if core.is_compiled_with_cuda() + else [fluid.CPUPlace()] + ) def setUp(self): self.set_attr() @@ -119,12 +122,12 @@ def setUp(self): def check_static_result(self, place): bz, length, ntags = self.bz, self.len, self.ntags with fluid.program_guard(fluid.Program(), fluid.Program()): - Input = fluid.data(name="Input", - shape=[bz, length, ntags], - dtype="float32") - Transition = fluid.data(name="Transition", - shape=[ntags, ntags], - dtype="float32") + Input = fluid.data( + name="Input", shape=[bz, length, ntags], dtype="float32" + ) + Transition = fluid.data( + name="Transition", shape=[ntags, ntags], dtype="float32" + ) Length = fluid.data(name="Length", shape=[bz], dtype="int64") decoder = paddle.text.ViterbiDecoder(Transition, self.use_tag) score, path = decoder(Input, Length) @@ -132,7 +135,7 @@ def check_static_result(self, place): feed_list = { "Input": self.input, "Transition": self.transitions, - "Length": self.length + "Length": self.length, } fetches = exe.run(feed=feed_list, fetch_list=[score, path]) np.testing.assert_allclose(fetches[0], self.scores, rtol=1e-5) diff --git a/python/paddle/fluid/tests/unittests/test_warpctc_op.py b/python/paddle/fluid/tests/unittests/test_warpctc_op.py index 81dce5b1a8692..91bebab2f6c34 100644 --- a/python/paddle/fluid/tests/unittests/test_warpctc_op.py +++ b/python/paddle/fluid/tests/unittests/test_warpctc_op.py @@ -29,9 +29,17 @@ class CTCForward(object): - - def __init__(self, softmax, softmax_lod, labels, labels_lod, num_classes, - batch_size, blank, norm_by_times): + def __init__( + self, + softmax, + softmax_lod, + labels, + labels_lod, + num_classes, + batch_size, + blank, + norm_by_times, + ): self.softmax = softmax self.softmax_lod = softmax_lod self.labels = labels @@ -114,15 +122,17 @@ def forward_a_sequence(self, softmax_a_sequence, labels_a_sequence): # calculate the forward and backward variables, # reference Chapter 7.3 of "Alex Grave, Supervised Sequence # Labelling with Recurrent Neural Networks" - log_acts = np.zeros([total_times, self.num_classes], - dtype=softmax_a_sequence.dtype) + log_acts = np.zeros( + [total_times, self.num_classes], dtype=softmax_a_sequence.dtype + ) for i in range(total_times): for j in range(self.num_classes): log_acts[i, j] = self.safe_log(softmax_a_sequence[i, j]) # calculate the forward variables - forward_vars = np.zeros([total_times, total_segments], - dtype=softmax_a_sequence.dtype) + forward_vars = np.zeros( + [total_times, total_segments], dtype=softmax_a_sequence.dtype + ) for i in range(total_times): for j in range(total_segments): forward_vars[i, j] = self.LOG_ZERO @@ -142,10 +152,13 @@ def forward_a_sequence(self, softmax_a_sequence, labels_a_sequence): if j & 1 == 1: label_idx = j // 2 label_val = labels_a_sequence[label_idx, 0] - fv = self.log_add(forward_vars[i - 1, j], - forward_vars[i - 1, j - 1]) - if j > 1 and label_val != labels_a_sequence[label_idx - 1, - 0]: + fv = self.log_add( + forward_vars[i - 1, j], forward_vars[i - 1, j - 1] + ) + if ( + j > 1 + and label_val != labels_a_sequence[label_idx - 1, 0] + ): fv = self.log_add(fv, forward_vars[i - 1, j - 2]) fv = self.log_mul(fv, log_acts[i, label_val]) else: @@ -159,7 +172,8 @@ def forward_a_sequence(self, softmax_a_sequence, labels_a_sequence): log_prob = forward_vars[total_times - 1, total_segments - 1] if total_segments > 1: log_prob = self.log_add( - log_prob, forward_vars[total_times - 1, total_segments - 2]) + log_prob, forward_vars[total_times - 1, total_segments - 2] + ) return -log_prob @@ -174,33 +188,38 @@ def forward(self): labels_end_i = labels_offset + self.labels_lod[self.level][i] softmax_a_sequence = self.softmax[ - softmax_start_i:softmax_end_i, :] + softmax_start_i:softmax_end_i, : + ] labels_a_sequence = self.labels[labels_start_i:labels_end_i, :] - self.loss[i] = self.forward_a_sequence(softmax_a_sequence, - labels_a_sequence) + self.loss[i] = self.forward_a_sequence( + softmax_a_sequence, labels_a_sequence + ) softmax_offset += self.softmax_lod[self.level][i] labels_offset += self.labels_lod[self.level][i] else: - softmax_a_sequence = self.softmax[:self.softmax_lod[i], i, :] - labels_a_sequence = self.labels[:self.labels_lod[i], :] - self.loss[i] = self.forward_a_sequence(softmax_a_sequence, - labels_a_sequence) + softmax_a_sequence = self.softmax[: self.softmax_lod[i], i, :] + labels_a_sequence = self.labels[: self.labels_lod[i], :] + self.loss[i] = self.forward_a_sequence( + softmax_a_sequence, labels_a_sequence + ) return self.loss -def python_api(logits, - label, - logits_length=None, - labels_length=None, - blank=0, - norm_by_times=False): - return paddle.fluid.layers.warpctc(logits, label, blank, norm_by_times, - logits_length, labels_length) +def python_api( + logits, + label, + logits_length=None, + labels_length=None, + blank=0, + norm_by_times=False, +): + return paddle.fluid.layers.warpctc( + logits, label, blank, norm_by_times, logits_length, labels_length + ) class TestWarpCTCOp(OpTest): - def config(self): self.batch_size = 4 self.num_classes = 12 @@ -214,31 +233,39 @@ def setUp(self): self.config() logits = np.random.uniform( - 0.1, 1.0, - [sum(self.logits_lod[0]), self.num_classes]).astype("float32") + 0.1, 1.0, [sum(self.logits_lod[0]), self.num_classes] + ).astype("float32") softmax = np.apply_along_axis(stable_softmax, 1, logits) # labels should not be blank - labels = np.random.randint(0, - self.num_classes - 1, - [sum(self.labels_lod[0]), 1], - dtype="int32") - - ctc = CTCForward(softmax, self.logits_lod, labels, self.labels_lod, - self.num_classes, self.batch_size, self.blank, - self.norm_by_times) + labels = np.random.randint( + 0, self.num_classes - 1, [sum(self.labels_lod[0]), 1], dtype="int32" + ) + + ctc = CTCForward( + softmax, + self.logits_lod, + labels, + self.labels_lod, + self.num_classes, + self.batch_size, + self.blank, + self.norm_by_times, + ) loss = ctc.forward() max_sequence_length = 0 for i in range(self.batch_size): - max_sequence_length = max(max_sequence_length, - self.logits_lod[0][i]) + max_sequence_length = max( + max_sequence_length, self.logits_lod[0][i] + ) self.gradient = np.zeros( [max_sequence_length, self.batch_size, self.num_classes], - dtype=logits.dtype) + dtype=logits.dtype, + ) self.inputs = { "Logits": (logits, self.logits_lod), - "Label": (labels, self.labels_lod) + "Label": (labels, self.labels_lod), } self.outputs = {"Loss": loss} self.attrs = { @@ -252,19 +279,22 @@ def test_check_output(self): def test_check_grad(self): self.outputs['WarpCTCGrad'] = self.gradient if core.is_compiled_with_rocm(): - self.check_grad(["Logits"], - "Loss", - max_relative_error=0.009, - check_dygraph=False) + self.check_grad( + ["Logits"], + "Loss", + max_relative_error=0.009, + check_dygraph=False, + ) else: - self.check_grad(["Logits"], - "Loss", - max_relative_error=0.007, - check_dygraph=False) + self.check_grad( + ["Logits"], + "Loss", + max_relative_error=0.007, + check_dygraph=False, + ) class TestWarpCTCOpCase1(TestWarpCTCOp): - def config(self): self.batch_size = 4 self.num_classes = CUDA_BLOCK_SIZE + 2 @@ -275,7 +305,6 @@ def config(self): class TestWarpCTCOpWithPadding(OpTest): - def config(self): self.batch_size = 4 self.num_classes = 8 @@ -293,28 +322,36 @@ def setUp(self): self.config() logits = np.random.uniform( - 0.1, 1.0, - [sum(self.logits_length), self.num_classes]).astype("float32") + 0.1, 1.0, [sum(self.logits_length), self.num_classes] + ).astype("float32") softmax = np.apply_along_axis(stable_softmax, 1, logits) # labels should not be blank - labels = np.random.randint(0, - self.num_classes - 1, - [sum(self.labels_length), 1], - dtype="int32") - - ctc = CTCForward(softmax, self.logits_lod, labels, self.labels_lod, - self.num_classes, self.batch_size, self.blank, - self.norm_by_times) + labels = np.random.randint( + 0, self.num_classes - 1, [sum(self.labels_length), 1], dtype="int32" + ) + + ctc = CTCForward( + softmax, + self.logits_lod, + labels, + self.labels_lod, + self.num_classes, + self.batch_size, + self.blank, + self.norm_by_times, + ) loss = ctc.forward() max_sequence_length = 0 for i in range(self.batch_size): - max_sequence_length = max(max_sequence_length, - self.logits_length[i]) + max_sequence_length = max( + max_sequence_length, self.logits_length[i] + ) # reshape logits to T*N*S new_logits = np.zeros( [max_sequence_length, self.batch_size, self.num_classes], - dtype=logits.dtype) + dtype=logits.dtype, + ) cur = 0 for batch_id in range(self.batch_size): @@ -326,10 +363,12 @@ def setUp(self): # reshape labels to N*S max_target_seq_length = 0 for i in range(self.batch_size): - max_target_seq_length = max(max_target_seq_length, - self.labels_length[i]) - new_labels = np.zeros([self.batch_size, max_target_seq_length], - dtype="int32") + max_target_seq_length = max( + max_target_seq_length, self.labels_length[i] + ) + new_labels = np.zeros( + [self.batch_size, max_target_seq_length], dtype="int32" + ) cur = 0 for batch_id in range(self.batch_size): @@ -339,13 +378,14 @@ def setUp(self): self.gradient = np.zeros( [max_sequence_length, self.batch_size, self.num_classes], - dtype=logits.dtype) + dtype=logits.dtype, + ) self.inputs = { "Logits": new_logits, "Label": new_labels, "LogitsLength": self.logits_length, - "LabelLength": self.labels_length + "LabelLength": self.labels_length, } self.outputs = {"Loss": loss} self.attrs = { @@ -359,19 +399,22 @@ def test_check_output(self): def test_check_grad(self): self.outputs['WarpCTCGrad'] = self.gradient if core.is_compiled_with_rocm(): - self.check_grad(["Logits"], - "Loss", - max_relative_error=0.009, - check_dygraph=False) + self.check_grad( + ["Logits"], + "Loss", + max_relative_error=0.009, + check_dygraph=False, + ) else: - self.check_grad(["Logits"], - "Loss", - max_relative_error=0.007, - check_dygraph=False) + self.check_grad( + ["Logits"], + "Loss", + max_relative_error=0.007, + check_dygraph=False, + ) class TestWarpCTCOpWithPaddingCase1(TestWarpCTCOpWithPadding): - def config(self): self.batch_size = 4 self.num_classes = CUDA_BLOCK_SIZE + 2 @@ -384,7 +427,6 @@ def config(self): class TestWarpCTCOpFp64(OpTest): - def config(self): self.batch_size = 4 self.num_classes = 8 @@ -402,28 +444,36 @@ def setUp(self): self.config() logits = np.random.uniform( - 0.1, 1.0, - [sum(self.logits_length), self.num_classes]).astype("float64") + 0.1, 1.0, [sum(self.logits_length), self.num_classes] + ).astype("float64") softmax = np.apply_along_axis(stable_softmax, 1, logits) # labels should not be blank - labels = np.random.randint(0, - self.num_classes - 1, - [sum(self.labels_length), 1], - dtype="int32") - - ctc = CTCForward(softmax, self.logits_lod, labels, self.labels_lod, - self.num_classes, self.batch_size, self.blank, - self.norm_by_times) + labels = np.random.randint( + 0, self.num_classes - 1, [sum(self.labels_length), 1], dtype="int32" + ) + + ctc = CTCForward( + softmax, + self.logits_lod, + labels, + self.labels_lod, + self.num_classes, + self.batch_size, + self.blank, + self.norm_by_times, + ) loss = ctc.forward() max_sequence_length = 0 for i in range(self.batch_size): - max_sequence_length = max(max_sequence_length, - self.logits_length[i]) + max_sequence_length = max( + max_sequence_length, self.logits_length[i] + ) # reshape logits to T*N*S new_logits = np.zeros( [max_sequence_length, self.batch_size, self.num_classes], - dtype=logits.dtype) + dtype=logits.dtype, + ) cur = 0 for batch_id in range(self.batch_size): @@ -435,10 +485,12 @@ def setUp(self): # reshape labels to N*S max_target_seq_length = 0 for i in range(self.batch_size): - max_target_seq_length = max(max_target_seq_length, - self.labels_length[i]) - new_labels = np.zeros([self.batch_size, max_target_seq_length], - dtype="int32") + max_target_seq_length = max( + max_target_seq_length, self.labels_length[i] + ) + new_labels = np.zeros( + [self.batch_size, max_target_seq_length], dtype="int32" + ) cur = 0 for batch_id in range(self.batch_size): @@ -448,13 +500,14 @@ def setUp(self): self.gradient = np.zeros( [max_sequence_length, self.batch_size, self.num_classes], - dtype=logits.dtype) + dtype=logits.dtype, + ) self.inputs = { "Logits": new_logits, "Label": new_labels, "LogitsLength": self.logits_length, - "LabelLength": self.labels_length + "LabelLength": self.labels_length, } self.outputs = {"Loss": loss} self.attrs = { @@ -471,58 +524,64 @@ def test_check_grad(self): class TestWarpCTCOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): - logits = fluid.data(name='logits', - shape=[5, 16, 6], - dtype='float32') - logits_length = fluid.data(name='logits_length', - shape=[None], - dtype='int64') + logits = fluid.data( + name='logits', shape=[5, 16, 6], dtype='float32' + ) + logits_length = fluid.data( + name='logits_length', shape=[None], dtype='int64' + ) label = fluid.data(name='label', shape=[16, 3], dtype='int32') - label_length = fluid.data(name='labels_length', - shape=[None], - dtype='int64') + label_length = fluid.data( + name='labels_length', shape=[None], dtype='int64' + ) def test_logits_Variable(): logits_data = np.random.rand(5, 16, 6).astype(logits.dtype) - fluid.layers.warpctc(input=logits_data, - label=label, - input_length=logits_length, - label_length=label_length) + fluid.layers.warpctc( + input=logits_data, + label=label, + input_length=logits_length, + label_length=label_length, + ) self.assertRaises(TypeError, test_logits_Variable) def test_label_Variable(): label_data = np.random.randint(0, 5, [5, 1]).astype("int32") - fluid.layers.warpctc(input=logits, - label=label_data, - input_length=logits_length, - label_length=label_length) + fluid.layers.warpctc( + input=logits, + label=label_data, + input_length=logits_length, + label_length=label_length, + ) self.assertRaises(TypeError, test_label_Variable) def test_logits_len_Variable(): logits_length_data = np.array([5] * 16).astype("int64") - fluid.layers.warpctc(input=logits, - label=label, - input_length=logits_length_data, - label_length=label_length) + fluid.layers.warpctc( + input=logits, + label=label, + input_length=logits_length_data, + label_length=label_length, + ) self.assertRaises(TypeError, test_logits_len_Variable) def test_label_len_Variable(): label_length_data = np.array([3] * 16).astype("int64") - fluid.layers.warpctc(input=logits, - label=label, - input_length=logits_length, - label_length=label_length_data) + fluid.layers.warpctc( + input=logits, + label=label, + input_length=logits_length, + label_length=label_length_data, + ) self.assertRaises(TypeError, test_label_len_Variable) def test_dygraph_errors(self): - def test_dygraph_with_lod(): logits = np.random.uniform(0.1, 1.0, [20, 15]).astype("float32") @@ -539,7 +598,6 @@ def test_dygraph_with_lod(): class TestCTCLossAPICase(unittest.TestCase): - def test_functinal_api(self): self.batch_size = 4 self.num_classes = CUDA_BLOCK_SIZE + 2 @@ -549,20 +607,29 @@ def test_functinal_api(self): self.norm_by_times = False logits = np.random.uniform( - 0.1, 1.0, - [max(self.logits_length), self.batch_size, self.num_classes - ]).astype("float32") + 0.1, + 1.0, + [max(self.logits_length), self.batch_size, self.num_classes], + ).astype("float32") softmax = np.apply_along_axis(stable_softmax, -1, logits) # labels should not be blank labels = np.random.randint( 0, self.num_classes - 1, [self.batch_size, max(self.labels_length)], - dtype="int32") - - ctc = CTCForward(softmax, self.logits_length, labels, - self.labels_length, self.num_classes, self.batch_size, - self.blank, self.norm_by_times) + dtype="int32", + ) + + ctc = CTCForward( + softmax, + self.logits_length, + labels, + self.labels_length, + self.num_classes, + self.batch_size, + self.blank, + self.norm_by_times, + ) loss_np = ctc.forward() paddle.disable_static() @@ -570,30 +637,33 @@ def test_functinal_api(self): labels = paddle.to_tensor(labels) logits_length = paddle.to_tensor(self.logits_length) labels_length = paddle.to_tensor(self.labels_length) - loss_pd_mean = F.ctc_loss(softmax, - labels, - logits_length, - labels_length, - blank=self.blank, - reduction='mean') + loss_pd_mean = F.ctc_loss( + softmax, + labels, + logits_length, + labels_length, + blank=self.blank, + reduction='mean', + ) loss_pd_mean = loss_pd_mean.numpy() - loss_pd_sum = F.ctc_loss(softmax, - labels, - logits_length, - labels_length, - blank=self.blank, - reduction='sum') + loss_pd_sum = F.ctc_loss( + softmax, + labels, + logits_length, + labels_length, + blank=self.blank, + reduction='sum', + ) loss_pd_sum = loss_pd_sum.numpy() paddle.enable_static() loss_np = np.squeeze(loss_np, axis=-1) loss_np_mean = (loss_np / labels_length.numpy()).mean() loss_np_sum = loss_np.sum() - np.testing.assert_allclose(loss_pd_mean, - loss_np_mean, - rtol=1e-05, - atol=1) + np.testing.assert_allclose( + loss_pd_mean, loss_np_mean, rtol=1e-05, atol=1 + ) np.testing.assert_allclose(loss_pd_sum, loss_np_sum, rtol=1e-05, atol=1) def test_class_api(self): @@ -605,20 +675,29 @@ def test_class_api(self): self.norm_by_times = False logits = np.random.uniform( - 0.1, 1.0, - [max(self.logits_length), self.batch_size, self.num_classes - ]).astype("float32") + 0.1, + 1.0, + [max(self.logits_length), self.batch_size, self.num_classes], + ).astype("float32") softmax = np.apply_along_axis(stable_softmax, -1, logits) # labels should not be blank labels = np.random.randint( 1, self.num_classes, [self.batch_size, max(self.labels_length)], - dtype="int32") - - ctc = CTCForward(softmax, self.logits_length, labels, - self.labels_length, self.num_classes, self.batch_size, - self.blank, self.norm_by_times) + dtype="int32", + ) + + ctc = CTCForward( + softmax, + self.logits_length, + labels, + self.labels_length, + self.num_classes, + self.batch_size, + self.blank, + self.norm_by_times, + ) loss_np = ctc.forward() paddle.disable_static() @@ -627,9 +706,9 @@ def test_class_api(self): logits_length = paddle.to_tensor(self.logits_length) labels_length = paddle.to_tensor(self.labels_length) - loss_pd = paddle.nn.CTCLoss(self.blank, - 'none')(softmax, labels, logits_length, - labels_length) + loss_pd = paddle.nn.CTCLoss(self.blank, 'none')( + softmax, labels, logits_length, labels_length + ) loss_pd = loss_pd.numpy() paddle.enable_static() loss_np = np.squeeze(loss_np, axis=-1) diff --git a/python/paddle/fluid/tests/unittests/test_weight_decay.py b/python/paddle/fluid/tests/unittests/test_weight_decay.py index 1b099ab25a347..5133a20375593 100644 --- a/python/paddle/fluid/tests/unittests/test_weight_decay.py +++ b/python/paddle/fluid/tests/unittests/test_weight_decay.py @@ -40,22 +40,24 @@ def prog_scope_guard(main_prog, startup_prog): yield -def bow_net(data, - label, - dict_dim, - is_sparse=False, - emb_dim=128, - hid_dim=128, - hid_dim2=96, - class_dim=2): +def bow_net( + data, + label, + dict_dim, + is_sparse=False, + emb_dim=128, + hid_dim=128, + hid_dim2=96, + class_dim=2, +): """ BOW net This model is from https://github.com/PaddlePaddle/models: fluid/PaddleNLP/text_classification/nets.py """ - emb = fluid.layers.embedding(input=data, - is_sparse=is_sparse, - size=[dict_dim, emb_dim]) + emb = fluid.layers.embedding( + input=data, is_sparse=is_sparse, size=[dict_dim, emb_dim] + ) bow = fluid.layers.sequence_pool(input=emb, pool_type='sum') bow_tanh = fluid.layers.tanh(bow) fc_1 = fluid.layers.fc(input=bow_tanh, size=hid_dim, act="tanh") @@ -68,13 +70,13 @@ def bow_net(data, class TestWeightDecay(unittest.TestCase): - def setUp(self): self.word_dict = paddle.dataset.imdb.word_dict() - reader = paddle.batch(paddle.dataset.imdb.train(self.word_dict), - batch_size=4)() + reader = paddle.batch( + paddle.dataset.imdb.train(self.word_dict), batch_size=4 + )() self.train_data = [next(reader) for _ in range(5)] - self.learning_rate = .5 + self.learning_rate = 0.5 def run_executor(self, place, feed_list, loss): exe = fluid.Executor(place) @@ -83,22 +85,24 @@ def run_executor(self, place, feed_list, loss): main_prog = fluid.default_main_program() loss_set = [] for data in self.train_data: - out = exe.run(main_prog, - feed=feeder.feed(data), - fetch_list=[loss.name]) + out = exe.run( + main_prog, feed=feeder.feed(data), fetch_list=[loss.name] + ) print("loss %s" % (np.average(out))) loss_set.append(np.average(out)) return loss_set - def run_parallel_exe(self, - place, - feed_list, - loss, - use_reduce=False, - use_fast_executor=False, - use_ir_memory_optimize=False): + def run_parallel_exe( + self, + place, + feed_list, + loss, + use_reduce=False, + use_fast_executor=False, + use_ir_memory_optimize=False, + ): exe = fluid.Executor(place) feeder = fluid.DataFeeder(feed_list=feed_list, place=place) exe.run(fluid.default_startup_program()) @@ -108,57 +112,63 @@ def run_parallel_exe(self, exec_strategy.use_experimental_executor = True build_strategy = fluid.BuildStrategy() - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce \ - if use_reduce else fluid.BuildStrategy.ReduceStrategy.AllReduce + build_strategy.reduce_strategy = ( + fluid.BuildStrategy.ReduceStrategy.Reduce + if use_reduce + else fluid.BuildStrategy.ReduceStrategy.AllReduce + ) build_strategy.memory_optimize = use_ir_memory_optimize train_cp = compiler.CompiledProgram( - fluid.default_main_program()).with_data_parallel( - loss_name=loss.name, - exec_strategy=exec_strategy, - build_strategy=build_strategy) + fluid.default_main_program() + ).with_data_parallel( + loss_name=loss.name, + exec_strategy=exec_strategy, + build_strategy=build_strategy, + ) loss_set = [] for data in self.train_data: - out = exe.run(train_cp, - feed=feeder.feed(data), - fetch_list=[loss.name]) + out = exe.run( + train_cp, feed=feeder.feed(data), fetch_list=[loss.name] + ) loss_set.append(np.average(out)) return loss_set - def check_weight_decay(self, - place, - model, - use_parallel_exe=False, - use_reduce=False): + def check_weight_decay( + self, place, model, use_parallel_exe=False, use_reduce=False + ): main_prog = fluid.framework.Program() startup_prog = fluid.framework.Program() startup_prog.random_seed = 1 with prog_scope_guard(main_prog=main_prog, startup_prog=startup_prog): - data = fluid.layers.data(name="words", - shape=[1], - dtype="int64", - lod_level=1) + data = fluid.layers.data( + name="words", shape=[1], dtype="int64", lod_level=1 + ) label = fluid.layers.data(name="label", shape=[1], dtype="int64") avg_cost = model(data, label, len(self.word_dict)) - param_list = [(var, var * self.learning_rate) - for var in main_prog.block(0).all_parameters()] + param_list = [ + (var, var * self.learning_rate) + for var in main_prog.block(0).all_parameters() + ] optimizer = fluid.optimizer.Adagrad( - learning_rate=self.learning_rate) + learning_rate=self.learning_rate + ) optimizer.minimize(avg_cost) for params in param_list: - updated_p = fluid.layers.elementwise_sub(x=params[0], - y=params[1]) + updated_p = fluid.layers.elementwise_sub( + x=params[0], y=params[1] + ) fluid.layers.assign(input=updated_p, output=params[0]) if use_parallel_exe: - loss = self.run_parallel_exe(place, [data, label], - loss=avg_cost, - use_reduce=use_reduce) + loss = self.run_parallel_exe( + place, [data, label], loss=avg_cost, use_reduce=use_reduce + ) else: loss = self.run_executor(place, [data, label], loss=avg_cost) @@ -170,16 +180,21 @@ def test_weight_decay(self): loss = self.check_weight_decay(place, model, use_parallel_exe=False) # TODO(zcd): should test use_reduce=True - loss2 = self.check_weight_decay(place, - model, - use_parallel_exe=True, - use_reduce=False) + loss2 = self.check_weight_decay( + place, model, use_parallel_exe=True, use_reduce=False + ) for i in range(len(loss)): self.assertTrue( np.isclose(a=loss[i], b=loss2[i], rtol=5e-5), - "Expect " + str(loss[i]) + "\n" + "But Got" + - str(loss2[i]) + " in class " + self.__class__.__name__) + "Expect " + + str(loss[i]) + + "\n" + + "But Got" + + str(loss2[i]) + + " in class " + + self.__class__.__name__, + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_weight_normalization.py b/python/paddle/fluid/tests/unittests/test_weight_normalization.py index d7a0fa3cd185a..9d77dadf8dc09 100644 --- a/python/paddle/fluid/tests/unittests/test_weight_normalization.py +++ b/python/paddle/fluid/tests/unittests/test_weight_normalization.py @@ -24,7 +24,7 @@ class TestWeightNormalization(unittest.TestCase): batch_size = 3 hidden_size = 5 - data_desc = (['x', [10], 0], ) + data_desc = (['x', [10], 0],) @classmethod def setUpClass(cls): @@ -32,21 +32,26 @@ def setUpClass(cls): @classmethod def set_program(cls): - data = fluid.layers.data(name=cls.data_desc[0][0], - shape=cls.data_desc[0][1]) - out = fluid.layers.fc(input=data, - size=cls.hidden_size, - param_attr=WeightNormParamAttr( - dim=None, - name='weight_norm_param', - initializer=ConstantInitializer(1.0)), - bias_attr=False, - act=None) + data = fluid.layers.data( + name=cls.data_desc[0][0], shape=cls.data_desc[0][1] + ) + out = fluid.layers.fc( + input=data, + size=cls.hidden_size, + param_attr=WeightNormParamAttr( + dim=None, + name='weight_norm_param', + initializer=ConstantInitializer(1.0), + ), + bias_attr=False, + act=None, + ) loss = fluid.layers.reduce_sum(out) fluid.backward.append_backward(loss=loss) cls.fetch_list = [ - 'weight_norm_param_g', 'weight_norm_param_v', - 'weight_norm_param_g@GRAD' + 'weight_norm_param_g', + 'weight_norm_param_v', + 'weight_norm_param_g@GRAD', ] def run_program(self): @@ -58,10 +63,12 @@ def run_program(self): self.set_inputs(place) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - output = exe.run(fluid.default_main_program(), - feed=self.inputs, - fetch_list=self.fetch_list, - return_numpy=False) + output = exe.run( + fluid.default_main_program(), + feed=self.inputs, + fetch_list=self.fetch_list, + return_numpy=False, + ) outputs.append(output) self.actual_outputs = outputs @@ -73,14 +80,16 @@ def set_data(self): data_lod_level = desc[2] data_lod = [] for i in range(data_lod_level): - lod_level_i = np.random.randint(low=1, - high=5, - size=self.batch_size if i == 0 - else sum(lod_level_i)).tolist() + lod_level_i = np.random.randint( + low=1, + high=5, + size=self.batch_size if i == 0 else sum(lod_level_i), + ).tolist() data_lod.append(lod_level_i) data_value = np.random.random( - size=[sum(data_lod[-1]) if data_lod else self.batch_size] + - data_shape).astype('float32') + size=[sum(data_lod[-1]) if data_lod else self.batch_size] + + data_shape + ).astype('float32') self.data[data_name] = (data_value, data_lod) def set_inputs(self, place): @@ -94,14 +103,16 @@ def set_inputs(self, place): def weight_normalize(self): v = np.ones( - (self.data[self.data_desc[0][0]][0].shape[-1], self.hidden_size)) + (self.data[self.data_desc[0][0]][0].shape[-1], self.hidden_size) + ) g = np.linalg.norm(v, axis=None, keepdims=True) w = g * v / np.linalg.norm(v, axis=None, keepdims=True) x = self.data[self.data_desc[0][0]][0] out = np.dot(x, w) - g_grad = (np.dot(x.T, np.ones_like(out)) * - (v / np.linalg.norm(v, axis=None, keepdims=True))).sum( - axis=None, keepdims=True) + g_grad = ( + np.dot(x.T, np.ones_like(out)) + * (v / np.linalg.norm(v, axis=None, keepdims=True)) + ).sum(axis=None, keepdims=True) return g, v, g_grad def test_weight_normalization(self): @@ -110,10 +121,9 @@ def test_weight_normalization(self): expect_output = self.weight_normalize() for actual_output in self.actual_outputs: [ - np.testing.assert_allclose(np.array(actual), - expect, - rtol=1e-05, - atol=0.001) + np.testing.assert_allclose( + np.array(actual), expect, rtol=1e-05, atol=0.001 + ) for expect, actual in zip(expect_output, actual_output) ] diff --git a/python/paddle/fluid/tests/unittests/test_where_index.py b/python/paddle/fluid/tests/unittests/test_where_index.py index ad561c4a51268..96b26dd6cae88 100644 --- a/python/paddle/fluid/tests/unittests/test_where_index.py +++ b/python/paddle/fluid/tests/unittests/test_where_index.py @@ -22,7 +22,6 @@ class TestWhereIndexOp(OpTest): - def setUp(self): self.op_type = "where_index" self.init_config() @@ -39,7 +38,6 @@ def init_config(self): class TestAllFalse(unittest.TestCase): - def setUp(self): self.op_type = "where_index" self.init_config() @@ -71,7 +69,6 @@ def test_all_false(self): class TestRank2(TestWhereIndexOp): - def init_config(self): self.inputs = { 'Condition': np.array([[True, False], [False, True]]), @@ -81,24 +78,26 @@ def init_config(self): class TestRank3(TestWhereIndexOp): - def init_config(self): self.inputs = { - 'Condition': - np.array([[[True, False], [False, True]], - [[False, True], [True, False]], - [[False, False], [False, True]]]), + 'Condition': np.array( + [ + [[True, False], [False, True]], + [[False, True], [True, False]], + [[False, False], [False, True]], + ] + ), } self.outputs = { - 'Out': - np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], [2, 1, 1]], - dtype='int64') + 'Out': np.array( + [[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], [2, 1, 1]], + dtype='int64', + ) } class TestWhereOpError(unittest.TestCase): - def test_api(self): with program_guard(Program(), Program()): cond = fluid.layers.data(name='cond', shape=[4], dtype='bool') @@ -111,9 +110,7 @@ def test_api(self): class TestWhereRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.where([10]) diff --git a/python/paddle/fluid/tests/unittests/test_where_op.py b/python/paddle/fluid/tests/unittests/test_where_op.py index fafa4e5279142..9ae7d9a48331b 100644 --- a/python/paddle/fluid/tests/unittests/test_where_op.py +++ b/python/paddle/fluid/tests/unittests/test_where_op.py @@ -23,7 +23,6 @@ class TestWhereOp(OpTest): - def setUp(self): self.op_type = 'where' self.python_api = paddle.where @@ -44,7 +43,6 @@ def init_config(self): class TestWhereOp2(TestWhereOp): - def init_config(self): self.x = np.random.uniform((-5), 5, (60, 2)).astype('float64') self.y = np.random.uniform((-5), 5, (60, 2)).astype('float64') @@ -52,7 +50,6 @@ def init_config(self): class TestWhereOp3(TestWhereOp): - def init_config(self): self.x = np.random.uniform((-3), 5, (20, 2, 4)).astype('float64') self.y = np.random.uniform((-3), 5, (20, 2, 4)).astype('float64') @@ -60,7 +57,6 @@ def init_config(self): class TestWhereAPI(unittest.TestCase): - def setUp(self): self.init_data() @@ -81,48 +77,51 @@ def test_api(self, use_cuda=False): for x_stop_gradient in [False, True]: for y_stop_gradient in [False, True]: with fluid.program_guard(Program(), Program()): - cond = fluid.layers.data(name='cond', - shape=self.shape, - dtype='bool') - x = fluid.layers.data(name='x', - shape=self.shape, - dtype='float32') - y = fluid.layers.data(name='y', - shape=self.shape, - dtype='float32') + cond = fluid.layers.data( + name='cond', shape=self.shape, dtype='bool' + ) + x = fluid.layers.data( + name='x', shape=self.shape, dtype='float32' + ) + y = fluid.layers.data( + name='y', shape=self.shape, dtype='float32' + ) x.stop_gradient = x_stop_gradient y.stop_gradient = y_stop_gradient result = paddle.where(cond, x, y) append_backward(paddle.mean(result)) for use_cuda in [False, True]: - if (use_cuda - and (not fluid.core.is_compiled_with_cuda())): + if use_cuda and ( + not fluid.core.is_compiled_with_cuda() + ): break - place = (fluid.CUDAPlace(0) - if use_cuda else fluid.CPUPlace()) + place = ( + fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() + ) exe = fluid.Executor(place) fetch_list = [result, result.grad_name] - if (x_stop_gradient is False): + if x_stop_gradient is False: fetch_list.append(x.grad_name) - if (y_stop_gradient is False): + if y_stop_gradient is False: fetch_list.append(y.grad_name) - out = exe.run(fluid.default_main_program(), - feed={ - 'cond': self.cond, - 'x': self.x, - 'y': self.y - }, - fetch_list=fetch_list) + out = exe.run( + fluid.default_main_program(), + feed={'cond': self.cond, 'x': self.x, 'y': self.y}, + fetch_list=fetch_list, + ) assert np.array_equal(out[0], self.out) - if (x_stop_gradient is False): - assert np.array_equal(out[2], - self.ref_x_backward(out[1])) - if (y.stop_gradient is False): + if x_stop_gradient is False: + assert np.array_equal( + out[2], self.ref_x_backward(out[1]) + ) + if y.stop_gradient is False: assert np.array_equal( - out[3], self.ref_y_backward(out[1])) - elif (y.stop_gradient is False): - assert np.array_equal(out[2], - self.ref_y_backward(out[1])) + out[3], self.ref_y_backward(out[1]) + ) + elif y.stop_gradient is False: + assert np.array_equal( + out[2], self.ref_y_backward(out[1]) + ) def test_api_broadcast(self, use_cuda=False): main_program = Program() @@ -130,20 +129,20 @@ def test_api_broadcast(self, use_cuda=False): x = fluid.layers.data(name='x', shape=[4, 1], dtype='float32') y = fluid.layers.data(name='y', shape=[4, 2], dtype='float32') x_i = np.array([[0.9383, 0.1983, 3.2, 1.2]]).astype('float32') - y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, - 1.0]]).astype('float32') + y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]).astype( + 'float32' + ) result = paddle.where((x > 1), x=x, y=y) for use_cuda in [False, True]: - if (use_cuda and (not fluid.core.is_compiled_with_cuda())): + if use_cuda and (not fluid.core.is_compiled_with_cuda()): return - place = (fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()) + place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) - out = exe.run(fluid.default_main_program(), - feed={ - 'x': x_i, - 'y': y_i - }, - fetch_list=[result]) + out = exe.run( + fluid.default_main_program(), + feed={'x': x_i, 'y': y_i}, + fetch_list=[result], + ) assert np.array_equal(out[0], np.where((x_i > 1), x_i, y_i)) def test_scalar(self): @@ -151,21 +150,23 @@ def test_scalar(self): main_program = Program() with fluid.program_guard(main_program): cond_shape = [2, 4] - cond = fluid.layers.data(name='cond', - shape=cond_shape, - dtype='bool') + cond = fluid.layers.data( + name='cond', shape=cond_shape, dtype='bool' + ) x_data = 1.0 y_data = 2.0 cond_data = np.array([False, False, True, True]).astype('bool') result = paddle.where(condition=cond, x=x_data, y=y_data) for use_cuda in [False, True]: - if (use_cuda and (not fluid.core.is_compiled_with_cuda())): + if use_cuda and (not fluid.core.is_compiled_with_cuda()): return - place = (fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()) + place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) - out = exe.run(fluid.default_main_program(), - feed={'cond': cond_data}, - fetch_list=[result]) + out = exe.run( + fluid.default_main_program(), + feed={'cond': cond_data}, + fetch_list=[result], + ) expect = np.where(cond_data, x_data, y_data) assert np.array_equal(out[0], expect) @@ -173,28 +174,26 @@ def __test_where_with_broadcast_static(self, cond_shape, x_shape, y_shape): paddle.enable_static() main_program = Program() with fluid.program_guard(main_program): - cond = fluid.layers.data(name='cond', - shape=cond_shape, - dtype='bool') + cond = fluid.layers.data( + name='cond', shape=cond_shape, dtype='bool' + ) x = fluid.layers.data(name='x', shape=x_shape, dtype='float32') y = fluid.layers.data(name='y', shape=y_shape, dtype='float32') cond_data_tmp = np.random.random(size=cond_shape).astype('float32') - cond_data = (cond_data_tmp < 0.3) + cond_data = cond_data_tmp < 0.3 x_data = np.random.random(size=x_shape).astype('float32') y_data = np.random.random(size=y_shape).astype('float32') result = paddle.where(condition=cond, x=x, y=y) for use_cuda in [False, True]: - if (use_cuda and (not fluid.core.is_compiled_with_cuda())): + if use_cuda and (not fluid.core.is_compiled_with_cuda()): return - place = (fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()) + place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) - out = exe.run(fluid.default_main_program(), - feed={ - 'cond': cond_data, - 'x': x_data, - 'y': y_data - }, - fetch_list=[result]) + out = exe.run( + fluid.default_main_program(), + feed={'cond': cond_data, 'x': x_data, 'y': y_data}, + fetch_list=[result], + ) expect = np.where(cond_data, x_data, y_data) assert np.array_equal(out[0], expect) @@ -248,7 +247,6 @@ def test_static_api_broadcast_8(self): class TestWhereDygraphAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(): x_i = np.array([0.9383, 0.1983, 3.2, 1.2]).astype('float64') @@ -272,7 +270,7 @@ def test_scalar(self): def __test_where_with_broadcast_dygraph(self, cond_shape, a_shape, b_shape): with fluid.dygraph.guard(): cond_tmp = paddle.rand(cond_shape) - cond = (cond_tmp < 0.3) + cond = cond_tmp < 0.3 a = paddle.rand(a_shape) b = paddle.rand(b_shape) result = paddle.where(cond, a, b) @@ -337,9 +335,9 @@ def test_where_condition(self): self.assertEqual(len(y), 2) z = fluid.layers.concat(list(y), axis=1) exe = fluid.Executor(fluid.CPUPlace()) - (res, ) = exe.run(feed={'x': data}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[z.name], return_numpy=False + ) expect_out = np.array([[0, 0], [1, 1]]) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) data = np.array([True, True, False]) @@ -350,9 +348,9 @@ def test_where_condition(self): self.assertEqual(len(y), 1) z = fluid.layers.concat(list(y), axis=1) exe = fluid.Executor(fluid.CPUPlace()) - (res, ) = exe.run(feed={'x': data}, - fetch_list=[z.name], - return_numpy=False) + (res,) = exe.run( + feed={'x': data}, fetch_list=[z.name], return_numpy=False + ) expect_out = np.array([[0], [1]]) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) @@ -370,7 +368,6 @@ def test_eager(self): class TestWhereOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x_i = np.array([0.9383, 0.1983, 3.2, 1.2]).astype('float64') @@ -394,7 +391,7 @@ def test_value_error(self): with fluid.dygraph.guard(): cond_shape = [2, 2, 4] cond_tmp = paddle.rand(cond_shape) - cond = (cond_tmp < 0.3) + cond = cond_tmp < 0.3 a = paddle.rand(cond_shape) self.assertRaises(ValueError, paddle.where, cond, a) diff --git a/python/paddle/fluid/tests/unittests/test_while_loop_op.py b/python/paddle/fluid/tests/unittests/test_while_loop_op.py index 7d5fc61f28bad..3c91b8c1e235c 100644 --- a/python/paddle/fluid/tests/unittests/test_while_loop_op.py +++ b/python/paddle/fluid/tests/unittests/test_while_loop_op.py @@ -26,9 +26,7 @@ class TestApiWhileLoop(unittest.TestCase): - def test_var_tuple(self): - def cond(i): return layers.less_than(i, ten) @@ -41,18 +39,20 @@ def body(i): i = layers.fill_constant(shape=[1], dtype='int64', value=0) one = layers.fill_constant(shape=[1], dtype='int64', value=1) ten = layers.fill_constant(shape=[1], dtype='int64', value=10) - out = layers.while_loop(cond, body, (i, )) + out = layers.while_loop(cond, body, (i,)) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) res = exe.run(main_program, fetch_list=out) - np.testing.assert_allclose(np.asarray(res[0]), - np.full(1, 10, np.int64), - rtol=1e-05) + np.testing.assert_allclose( + np.asarray(res[0]), np.full(1, 10, np.int64), rtol=1e-05 + ) def test_var_list(self): - def cond(i, mem): return layers.less_than(i, ten) @@ -73,8 +73,11 @@ def body(i, mem): data = np.random.rand(10).astype('float32') data_one = np.ones(10).astype('float32') - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) res = exe.run(main_program, feed={'mem': data}, fetch_list=out) for i in range(10): @@ -82,7 +85,6 @@ def body(i, mem): np.testing.assert_allclose(np.asarray(res[1]), data, rtol=1e-05) def test_var_dict(self): - def cond(i, ten, test_dict, test_list, test_list_dict): return layers.less_than(i, ten) @@ -94,7 +96,8 @@ def body(i, ten, test_dict, test_list, test_list_dict): test_list_dict[0]["test_key"] += 1 test_list_dict[0]["test_key"] = fluid.layers.relu( - test_list_dict[0]["test_key"]) + test_list_dict[0]["test_key"] + ) i = layers.increment(i) return [i, ten, test_dict, test_list, test_list_dict] @@ -110,47 +113,54 @@ def body(i, ten, test_dict, test_list, test_list_dict): test_list = [ layers.fill_constant(shape=[1, 2], dtype='int64', value=0) ] - test_list_dict = [{ - "test_key": - layers.fill_constant(shape=[1], dtype='float32', value=0) - }] + test_list_dict = [ + { + "test_key": layers.fill_constant( + shape=[1], dtype='float32', value=0 + ) + } + ] i, ten, test_dict, test_list, test_list_dict = layers.while_loop( - cond, body, [i, ten, test_dict, test_list, test_list_dict]) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + cond, body, [i, ten, test_dict, test_list, test_list_dict] + ) + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(main_program, - fetch_list=[ - test_dict["test_key"], test_list[0], - test_list_dict[0]["test_key"] - ]) - np.testing.assert_allclose(np.asarray(res[0]), - np.full(shape=1, - fill_value=10, - dtype=np.int64), - rtol=1e-05) - np.testing.assert_allclose(np.asarray(res[1]), - np.full(shape=(2, 1), - fill_value=10, - dtype=np.int64), - rtol=1e-05) - np.testing.assert_allclose(np.asarray(res[2]), - np.full(shape=1, - fill_value=10, - dtype=np.float32), - rtol=1e-05) + res = exe.run( + main_program, + fetch_list=[ + test_dict["test_key"], + test_list[0], + test_list_dict[0]["test_key"], + ], + ) + np.testing.assert_allclose( + np.asarray(res[0]), + np.full(shape=1, fill_value=10, dtype=np.int64), + rtol=1e-05, + ) + np.testing.assert_allclose( + np.asarray(res[1]), + np.full(shape=(2, 1), fill_value=10, dtype=np.int64), + rtol=1e-05, + ) + np.testing.assert_allclose( + np.asarray(res[2]), + np.full(shape=1, fill_value=10, dtype=np.float32), + rtol=1e-05, + ) class TestApiWhileLoop_Nested(unittest.TestCase): - def test_nested_net(self): - def external_cond(i, j, init, sums): return layers.less_than(i, loop_len1) def external_body(i, j, init, sums): - def internal_cond(j, init, sums): return layers.less_than(j, loop_len2) @@ -160,8 +170,9 @@ def internal_body(j, init, sums): j = layers.increment(j) return [j, init, sums] - result = layers.while_loop(internal_cond, internal_body, - [j, init, sums]) + result = layers.while_loop( + internal_cond, internal_body, [j, init, sums] + ) j = result[0] init = result[1] sums = result[2] @@ -180,21 +191,22 @@ def internal_body(j, init, sums): loop_len2 = layers.fill_constant(shape=[1], dtype='int64', value=3) ones = layers.fill_constant(shape=[3, 3], dtype='float32', value=1) - out = layers.while_loop(external_cond, external_body, - [i, j, init, sums]) + out = layers.while_loop( + external_cond, external_body, [i, j, init, sums] + ) data = np.random.rand(3, 3).astype('float32') data_sums = np.zeros([3, 3]).astype('float32') - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - res = exe.run(main_program, - feed={ - 'init': data, - 'sums': data_sums - }, - fetch_list=out) + res = exe.run( + main_program, feed={'init': data, 'sums': data_sums}, fetch_list=out + ) for i in range(3): data = np.add(data, 1) data_sums = np.add(data, data_sums) @@ -204,9 +216,7 @@ def internal_body(j, init, sums): class TestApiWhileLoop_Backward(unittest.TestCase): - def test_while_loop_backward(self): - def cond(i, x): return layers.less_than(i, eleven) @@ -229,8 +239,11 @@ def body(i, x): mean = paddle.mean(out[1]) append_backward(mean) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feed_i = np.ones(1).astype('float32') @@ -238,17 +251,15 @@ def body(i, x): data = np.asarray([100]).astype('float32') i_grad = np.asarray([110]).astype('float32') - res = exe.run(main_program, - feed={ - 'i': feed_i, - 'x': feed_x - }, - fetch_list=[mean.name, i.grad_name]) + res = exe.run( + main_program, + feed={'i': feed_i, 'x': feed_x}, + fetch_list=[mean.name, i.grad_name], + ) np.testing.assert_allclose(np.asarray(res[0]), data, rtol=1e-05) np.testing.assert_allclose(np.asarray(res[1]), i_grad, rtol=1e-05) def test_while_loop_backward2(self): - def cond(i, x): return i < 3 @@ -269,8 +280,11 @@ def body(i, x): mean = paddle.mean(out[1]) append_backward(mean) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) feed_i = np.ones(1).astype('float32') @@ -279,26 +293,22 @@ def body(i, x): i_grad = np.asarray([3]).astype('float32') x_grad = np.asarray([2]).astype('float32') - res = exe.run(main_program, - feed={ - 'i': feed_i, - 'x': feed_x - }, - fetch_list=[mean.name, i.grad_name, x.grad_name]) + res = exe.run( + main_program, + feed={'i': feed_i, 'x': feed_x}, + fetch_list=[mean.name, i.grad_name, x.grad_name], + ) np.testing.assert_allclose(np.asarray(res[0]), data, rtol=1e-05) np.testing.assert_allclose(np.asarray(res[1]), i_grad, rtol=1e-05) np.testing.assert_allclose(np.asarray(res[2]), x_grad, rtol=1e-05) class TestApiWhileLoop_NestedWithBackwardAndLoDTensorArray(unittest.TestCase): - def test_nested_net_with_backward_and_lodtensor(self): - def external_cond(i, j, x, mem_array): return layers.less_than(i, array_len) def external_body(i, j, x, mem_array): - def internal_cond(j, x, mem_array): return layers.less_than(j, array_len2) @@ -317,8 +327,9 @@ def internal_body(j, x, mem_array): outer_sum_1 = layers.elementwise_add(x=x, y=outer_sum_0) i = layers.increment(x=i, in_place=True) layers.array_write(outer_sum_1, i=i, array=mem_array) - j, x, mem_array = layers.while_loop(internal_cond, internal_body, - [j, x, mem_array]) + j, x, mem_array = layers.while_loop( + internal_cond, internal_body, [j, x, mem_array] + ) return [i, j, x, mem_array] main_program = Program() @@ -345,15 +356,19 @@ def internal_body(j, x, mem_array): j.stop_gradient = True array_len2 = layers.fill_constant(shape=[1], dtype='int64', value=3) - out = layers.while_loop(external_cond, external_body, - [i, j, x, mem_array]) + out = layers.while_loop( + external_cond, external_body, [i, j, x, mem_array] + ) sum_result = layers.array_read(array=mem_array, i=j) mean = paddle.mean(sum_result) append_backward(mean) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) d = [] @@ -362,27 +377,21 @@ def internal_body(j, x, mem_array): feed_x = np.ones(10).astype('float32') data_sum = d[0] + d[1] + d[2] + 3 * feed_x x_grad = [0.3] * 10 - res = exe.run(main_program, - feed={ - 'd0': d[0], - 'd1': d[1], - 'd2': d[2], - 'x': feed_x - }, - fetch_list=[sum_result.name, x.grad_name]) + res = exe.run( + main_program, + feed={'d0': d[0], 'd1': d[1], 'd2': d[2], 'x': feed_x}, + fetch_list=[sum_result.name, x.grad_name], + ) np.testing.assert_allclose(res[0], data_sum, rtol=1e-05) np.testing.assert_allclose(res[1], x_grad, rtol=1e-05) class TestApiWhileLoopWithSwitchCase(unittest.TestCase): - def test_with_switch_case(self): - def cond(i): return layers.less_than(i, ten) def body(i): - def fn_add_three(): data_add_three = layers.elementwise_add(x=i, y=three) return data_add_three @@ -395,12 +404,11 @@ def fn_add_one(): data_add_one = layers.elementwise_add(x=i, y=one) return data_add_one - return layers.switch_case(branch_index=i, - branch_fns={ - 2: fn_add_three, - 5: fn_square - }, - default=fn_add_one) + return layers.switch_case( + branch_index=i, + branch_fns={2: fn_add_three, 5: fn_square}, + default=fn_add_one, + ) main_program = Program() startup_program = Program() @@ -411,8 +419,11 @@ def fn_add_one(): one = layers.fill_constant(shape=[1], dtype='int64', value=1) out = layers.while_loop(cond, body, [i]) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) res = exe.run(main_program, fetch_list=out) @@ -421,9 +432,7 @@ def fn_add_one(): class TestApiWhileLoop_Error(unittest.TestCase): - def test_error(self): - def cond_returns_constant(i): return 1 @@ -453,9 +462,9 @@ def cond_returns_with_mutable_dict(i, test_dict): return i > 0 def body_returns_with_mutable_dict(i, test_dict): - test_dict['new_key'] = layers.fill_constant(shape=[1], - dtype='int64', - value=1) + test_dict['new_key'] = layers.fill_constant( + shape=[1], dtype='int64', value=1 + ) return layers.increment(i), test_dict def cond_returns_with_mutable_list(i, test_list): @@ -463,7 +472,8 @@ def cond_returns_with_mutable_list(i, test_list): def body_returns_with_mutable_list(i, test_list): test_list.append( - layers.fill_constant(shape=[1], dtype='int64', value=1)) + layers.fill_constant(shape=[1], dtype='int64', value=1) + ) return layers.increment(i), test_list main_program = Program() @@ -483,8 +493,9 @@ def type_error_cond(): # The type of `body` in Op(while_loop) must be callable def type_error_body(): - out = layers.while_loop(cond_returns_bool_tensor, data, - [data_1d]) + out = layers.while_loop( + cond_returns_bool_tensor, data, [data_1d] + ) self.assertRaises(TypeError, type_error_body) @@ -508,8 +519,9 @@ def type_error_cond_returns_not_variable(): # The type of `cond` returns in Op(while_loop) must be a bollean variable def type_error_cond_returns_not_boolean(): - out = layers.while_loop(cond_returns_not_bool_tensor, body, - [data_1d]) + out = layers.while_loop( + cond_returns_not_bool_tensor, body, [data_1d] + ) self.assertRaises(TypeError, type_error_cond_returns_not_boolean) @@ -521,47 +533,54 @@ def type_error_shape_cond_returns_2d(): # The length of `body` returns in Op(while_loop) must be same as `loop_vars` def value_error_body_returns_error_length(): - out = layers.while_loop(cond_returns_bool_tensor, - body_returns_error_length, [data]) + out = layers.while_loop( + cond_returns_bool_tensor, body_returns_error_length, [data] + ) self.assertRaises(ValueError, value_error_body_returns_error_length) # The type of `body` returns in Op(while_loop) must be same as `loop_vars` def value_error_body_returns_error_type(): - out = layers.while_loop(cond_receives_two_args, - body_returns_error_type, [data, ten]) + out = layers.while_loop( + cond_receives_two_args, body_returns_error_type, [data, ten] + ) self.assertRaises(ValueError, value_error_body_returns_error_type) # The length of `output_vars` with mutable value should keep same with `loop_vars` def value_error_body_returns_with_mutable_dict(): test_dict = { - "int_constant": - layers.fill_constant(shape=[2, 2], dtype='int64', value=1) + "int_constant": layers.fill_constant( + shape=[2, 2], dtype='int64', value=1 + ) } - out = layers.while_loop(cond_returns_with_mutable_dict, - body_returns_with_mutable_dict, - [data, test_dict]) + out = layers.while_loop( + cond_returns_with_mutable_dict, + body_returns_with_mutable_dict, + [data, test_dict], + ) - self.assertRaises(ValueError, - value_error_body_returns_with_mutable_dict) + self.assertRaises( + ValueError, value_error_body_returns_with_mutable_dict + ) def value_error_body_returns_with_mutable_list(): test_list = [ layers.fill_constant(shape=[2, 2], dtype='int64', value=1) ] - out = layers.while_loop(cond_returns_with_mutable_list, - body_returns_with_mutable_list, - [data, test_list]) + out = layers.while_loop( + cond_returns_with_mutable_list, + body_returns_with_mutable_list, + [data, test_list], + ) - self.assertRaises(ValueError, - value_error_body_returns_with_mutable_list) + self.assertRaises( + ValueError, value_error_body_returns_with_mutable_list + ) class TestApiWhileLoopSliceInBody(unittest.TestCase): - def test_var_slice(self): - def cond(z, i): return i + 1 <= x_shape[0] @@ -579,8 +598,11 @@ def body(z, i): i = fluid.layers.fill_constant([1], 'int32', 0) z, _ = fluid.layers.while_loop(cond, body, [z, i]) - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) np_x = np.array([1, 2, 3, 4, 5], dtype='int32') diff --git a/python/paddle/fluid/tests/unittests/test_while_op.py b/python/paddle/fluid/tests/unittests/test_while_op.py index 648e93dede8d0..341e6a8ef5644 100644 --- a/python/paddle/fluid/tests/unittests/test_while_op.py +++ b/python/paddle/fluid/tests/unittests/test_while_op.py @@ -25,20 +25,16 @@ class TestWhileOp(unittest.TestCase): - def simple_net(self): - d0 = layers.data("d0", - shape=[10], - append_batch_size=False, - dtype='float32') - d1 = layers.data("d1", - shape=[10], - append_batch_size=False, - dtype='float32') - d2 = layers.data("d2", - shape=[10], - append_batch_size=False, - dtype='float32') + d0 = layers.data( + "d0", shape=[10], append_batch_size=False, dtype='float32' + ) + d1 = layers.data( + "d1", shape=[10], append_batch_size=False, dtype='float32' + ) + d2 = layers.data( + "d2", shape=[10], append_batch_size=False, dtype='float32' + ) i = layers.zeros(shape=[1], dtype='int64') i.stop_gradient = True init = layers.zeros(shape=[10], dtype='float32') @@ -96,12 +92,10 @@ def test_simple_net(self): for i in range(3): d.append(numpy.random.random(size=[10]).astype('float32')) - outs = exe.run(feed={ - 'd0': d[0], - 'd1': d[1], - 'd2': d[2] - }, - fetch_list=[sum_result]) + outs = exe.run( + feed={'d0': d[0], 'd1': d[1], 'd2': d[2]}, + fetch_list=[sum_result], + ) self.assertAlmostEqual(numpy.sum(d), numpy.sum(outs[0]), delta=0.01) def test_simple_net_forward(self): @@ -133,7 +127,6 @@ def test_exceptions(self): class BadInputTest(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): @@ -145,9 +138,7 @@ def test_bad_x(): class TestIgnoreVarNameInWhile(unittest.TestCase): - def test_ignore_var(self): - def cond(i, ten, temp, y): return i < ten @@ -164,8 +155,9 @@ def body_func(i, ten, batch_info, origin_seq): i = layers.fill_constant(shape=[1], value=0, dtype='int32') num = layers.fill_constant(shape=[1], value=5, dtype='int32') - i, ten, shuffle_temp, y = layers.while_loop(cond, body_func, - [i, num, temp, y]) + i, ten, shuffle_temp, y = layers.while_loop( + cond, body_func, [i, num, temp, y] + ) output = shuffle_temp @@ -177,18 +169,16 @@ def body_func(i, ten, batch_info, origin_seq): input_y = numpy.array([[10], [12], [33]]) input_y = input_y.reshape(3, 1, 1) - res, = exe.run(fluid.default_main_program(), - feed={ - 'x': input_x, - 'y': input_y - }, - fetch_list=[output]) + (res,) = exe.run( + fluid.default_main_program(), + feed={'x': input_x, 'y': input_y}, + fetch_list=[output], + ) self.assertListEqual(list(res.shape), [3, 1, 5]) class TestOutputsMustExistsInputs(unittest.TestCase): - def test_outputs_exists_inputs(self): """ We guarantee that the output tensor must be in the input tensor, so that the output and input can correspond to each other, but the input can be greater than the number of outputs. It's required in paddle2onnx. @@ -220,11 +210,14 @@ def body(i, s, x): for op in main_program.block(0).ops: if op.type == "while": for out_name in op.output("Out"): - if out_name in op.input("Condition"): continue + if out_name in op.input("Condition"): + continue self.assertTrue( out_name in op.input("X"), - "In while op, the variable in output(`Out`) must exists in inputs(`X`), but the variable with name `{}` not meet the precondition." - .format(out_name)) + "In while op, the variable in output(`Out`) must exists in inputs(`X`), but the variable with name `{}` not meet the precondition.".format( + out_name + ), + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_yolo_box_op.py b/python/paddle/fluid/tests/unittests/test_yolo_box_op.py index 0a4897b821d66..8584332f02c07 100644 --- a/python/paddle/fluid/tests/unittests/test_yolo_box_op.py +++ b/python/paddle/fluid/tests/unittests/test_yolo_box_op.py @@ -20,7 +20,7 @@ def sigmoid(x): - return (1.0 / (1.0 + np.exp(((-1.0) * x)))) + return 1.0 / (1.0 + np.exp(((-1.0) * x))) def YoloBox(x, img_size, attrs): @@ -34,9 +34,9 @@ def YoloBox(x, img_size, attrs): scale_x_y = attrs['scale_x_y'] iou_aware = attrs['iou_aware'] iou_aware_factor = attrs['iou_aware_factor'] - bias_x_y = ((-0.5) * (scale_x_y - 1.0)) - input_h = (downsample * h) - input_w = (downsample * w) + bias_x_y = (-0.5) * (scale_x_y - 1.0) + input_h = downsample * h + input_w = downsample * w if iou_aware: ioup = x[:, :an_num, :, :] ioup = np.expand_dims(ioup, axis=(-1)) @@ -45,49 +45,54 @@ def YoloBox(x, img_size, attrs): pred_box = x[:, :, :, :, :4].copy() grid_x = np.tile(np.arange(w).reshape((1, w)), (h, 1)) grid_y = np.tile(np.arange(h).reshape((h, 1)), (1, w)) - pred_box[:, :, :, :, 0] = (( - (grid_x + (sigmoid(pred_box[:, :, :, :, 0]) * scale_x_y)) + bias_x_y) / - w) - pred_box[:, :, :, :, 1] = (( - (grid_y + (sigmoid(pred_box[:, :, :, :, 1]) * scale_x_y)) + bias_x_y) / - h) - anchors = [(anchors[i], anchors[(i + 1)]) - for i in range(0, len(anchors), 2)] - anchors_s = np.array([((an_w / input_w), (an_h / input_h)) - for (an_w, an_h) in anchors]) + pred_box[:, :, :, :, 0] = ( + (grid_x + (sigmoid(pred_box[:, :, :, :, 0]) * scale_x_y)) + bias_x_y + ) / w + pred_box[:, :, :, :, 1] = ( + (grid_y + (sigmoid(pred_box[:, :, :, :, 1]) * scale_x_y)) + bias_x_y + ) / h + anchors = [ + (anchors[i], anchors[(i + 1)]) for i in range(0, len(anchors), 2) + ] + anchors_s = np.array( + [((an_w / input_w), (an_h / input_h)) for (an_w, an_h) in anchors] + ) anchor_w = anchors_s[:, 0:1].reshape((1, an_num, 1, 1)) anchor_h = anchors_s[:, 1:2].reshape((1, an_num, 1, 1)) - pred_box[:, :, :, :, 2] = (np.exp(pred_box[:, :, :, :, 2]) * anchor_w) - pred_box[:, :, :, :, 3] = (np.exp(pred_box[:, :, :, :, 3]) * anchor_h) + pred_box[:, :, :, :, 2] = np.exp(pred_box[:, :, :, :, 2]) * anchor_w + pred_box[:, :, :, :, 3] = np.exp(pred_box[:, :, :, :, 3]) * anchor_h if iou_aware: - pred_conf = ((sigmoid(x[:, :, :, :, 4:5])**(1 - iou_aware_factor)) * - (sigmoid(ioup)**iou_aware_factor)) + pred_conf = (sigmoid(x[:, :, :, :, 4:5]) ** (1 - iou_aware_factor)) * ( + sigmoid(ioup) ** iou_aware_factor + ) else: pred_conf = sigmoid(x[:, :, :, :, 4:5]) pred_conf[(pred_conf < conf_thresh)] = 0.0 - pred_score = (sigmoid(x[:, :, :, :, 5:]) * pred_conf) - pred_box = (pred_box * (pred_conf > 0.0).astype('float32')) + pred_score = sigmoid(x[:, :, :, :, 5:]) * pred_conf + pred_box = pred_box * (pred_conf > 0.0).astype('float32') pred_box = pred_box.reshape((n, (-1), 4)) - (pred_box[:, :, :2], - pred_box[:, :, 2:4]) = ((pred_box[:, :, :2] - (pred_box[:, :, 2:4] / 2.0)), - (pred_box[:, :, :2] + (pred_box[:, :, 2:4] / 2.0))) - pred_box[:, :, 0] = (pred_box[:, :, 0] * img_size[:, 1][:, np.newaxis]) - pred_box[:, :, 1] = (pred_box[:, :, 1] * img_size[:, 0][:, np.newaxis]) - pred_box[:, :, 2] = (pred_box[:, :, 2] * img_size[:, 1][:, np.newaxis]) - pred_box[:, :, 3] = (pred_box[:, :, 3] * img_size[:, 0][:, np.newaxis]) + (pred_box[:, :, :2], pred_box[:, :, 2:4]) = ( + (pred_box[:, :, :2] - (pred_box[:, :, 2:4] / 2.0)), + (pred_box[:, :, :2] + (pred_box[:, :, 2:4] / 2.0)), + ) + pred_box[:, :, 0] = pred_box[:, :, 0] * img_size[:, 1][:, np.newaxis] + pred_box[:, :, 1] = pred_box[:, :, 1] * img_size[:, 0][:, np.newaxis] + pred_box[:, :, 2] = pred_box[:, :, 2] * img_size[:, 1][:, np.newaxis] + pred_box[:, :, 3] = pred_box[:, :, 3] * img_size[:, 0][:, np.newaxis] if clip_bbox: for i in range(len(pred_box)): pred_box[i, :, 0] = np.clip(pred_box[i, :, 0], 0, np.inf) pred_box[i, :, 1] = np.clip(pred_box[i, :, 1], 0, np.inf) - pred_box[i, :, 2] = np.clip(pred_box[i, :, 2], (-np.inf), - (img_size[(i, 1)] - 1)) - pred_box[i, :, 3] = np.clip(pred_box[i, :, 3], (-np.inf), - (img_size[(i, 0)] - 1)) + pred_box[i, :, 2] = np.clip( + pred_box[i, :, 2], (-np.inf), (img_size[(i, 1)] - 1) + ) + pred_box[i, :, 3] = np.clip( + pred_box[i, :, 3], (-np.inf), (img_size[(i, 0)] - 1) + ) return (pred_box, pred_score.reshape((n, (-1), class_num))) class TestYoloBoxOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'yolo_box' @@ -102,7 +107,7 @@ def setUp(self): 'clip_bbox': self.clip_bbox, 'scale_x_y': self.scale_x_y, 'iou_aware': self.iou_aware, - 'iou_aware_factor': self.iou_aware_factor + 'iou_aware_factor': self.iou_aware_factor, } self.inputs = {'X': x, 'ImgSize': img_size} (boxes, scores) = YoloBox(x, img_size, self.attrs) @@ -119,8 +124,12 @@ def initTestCase(self): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = True - self.x_shape = (self.batch_size, (an_num * (5 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (5 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.0 self.iou_aware = False @@ -128,7 +137,6 @@ def initTestCase(self): class TestYoloBoxOpNoClipBbox(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) @@ -137,8 +145,12 @@ def initTestCase(self): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = False - self.x_shape = (self.batch_size, (an_num * (5 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (5 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.0 self.iou_aware = False @@ -146,7 +158,6 @@ def initTestCase(self): class TestYoloBoxOpScaleXY(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) @@ -155,8 +166,12 @@ def initTestCase(self): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = True - self.x_shape = (self.batch_size, (an_num * (5 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (5 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.2 self.iou_aware = False @@ -164,7 +179,6 @@ def initTestCase(self): class TestYoloBoxOpIoUAware(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) @@ -173,8 +187,12 @@ def initTestCase(self): self.conf_thresh = 0.5 self.downsample = 32 self.clip_bbox = True - self.x_shape = (self.batch_size, (an_num * (6 + self.class_num)), 13, - 13) + self.x_shape = ( + self.batch_size, + (an_num * (6 + self.class_num)), + 13, + 13, + ) self.imgsize_shape = (self.batch_size, 2) self.scale_x_y = 1.0 self.iou_aware = True @@ -182,34 +200,37 @@ def initTestCase(self): class TestYoloBoxDygraph(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() img_size = np.ones((2, 2)).astype('int32') img_size = paddle.to_tensor(img_size) x1 = np.random.random([2, 14, 8, 8]).astype('float32') x1 = paddle.to_tensor(x1) - (boxes, scores) = paddle.vision.ops.yolo_box(x1, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0) - assert ((boxes is not None) and (scores is not None)) + (boxes, scores) = paddle.vision.ops.yolo_box( + x1, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + ) + assert (boxes is not None) and (scores is not None) x2 = np.random.random([2, 16, 8, 8]).astype('float32') x2 = paddle.to_tensor(x2) - (boxes, scores) = paddle.vision.ops.yolo_box(x2, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0, - iou_aware=True, - iou_aware_factor=0.5) + (boxes, scores) = paddle.vision.ops.yolo_box( + x2, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + iou_aware=True, + iou_aware_factor=0.5, + ) paddle.enable_static() def test_eager(self): @@ -218,35 +239,37 @@ def test_eager(self): class TestYoloBoxStatic(unittest.TestCase): - def test_static(self): x1 = paddle.static.data('x1', [2, 14, 8, 8], 'float32') img_size = paddle.static.data('img_size', [2, 2], 'int32') - (boxes, scores) = paddle.vision.ops.yolo_box(x1, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0) - assert ((boxes is not None) and (scores is not None)) + (boxes, scores) = paddle.vision.ops.yolo_box( + x1, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + ) + assert (boxes is not None) and (scores is not None) x2 = paddle.static.data('x2', [2, 16, 8, 8], 'float32') - (boxes, scores) = paddle.vision.ops.yolo_box(x2, - img_size=img_size, - anchors=[10, 13, 16, 30], - class_num=2, - conf_thresh=0.01, - downsample_ratio=8, - clip_bbox=True, - scale_x_y=1.0, - iou_aware=True, - iou_aware_factor=0.5) - assert ((boxes is not None) and (scores is not None)) + (boxes, scores) = paddle.vision.ops.yolo_box( + x2, + img_size=img_size, + anchors=[10, 13, 16, 30], + class_num=2, + conf_thresh=0.01, + downsample_ratio=8, + clip_bbox=True, + scale_x_y=1.0, + iou_aware=True, + iou_aware_factor=0.5, + ) + assert (boxes is not None) and (scores is not None) class TestYoloBoxOpHW(TestYoloBoxOp): - def initTestCase(self): self.anchors = [10, 13, 16, 30, 33, 23] an_num = int((len(self.anchors) // 2)) diff --git a/python/paddle/fluid/tests/unittests/test_yolov3_loss_op.py b/python/paddle/fluid/tests/unittests/test_yolov3_loss_op.py index c2c747836eef9..9105f27e96d73 100644 --- a/python/paddle/fluid/tests/unittests/test_yolov3_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_yolov3_loss_op.py @@ -51,11 +51,12 @@ def batch_xywh_box_iou(box1, box2): left = np.maximum(b1_left[:, :, np.newaxis], b2_left[:, np.newaxis, :]) right = np.minimum(b1_right[:, :, np.newaxis], b2_right[:, np.newaxis, :]) top = np.maximum(b1_top[:, :, np.newaxis], b2_top[:, np.newaxis, :]) - bottom = np.minimum(b1_bottom[:, :, np.newaxis], b2_bottom[:, - np.newaxis, :]) + bottom = np.minimum( + b1_bottom[:, :, np.newaxis], b2_bottom[:, np.newaxis, :] + ) - inter_w = np.clip(right - left, 0., 1.) - inter_h = np.clip(bottom - top, 0., 1.) + inter_w = np.clip(right - left, 0.0, 1.0) + inter_h = np.clip(bottom - top, 0.0, 1.0) inter_area = inter_w * inter_h b1_area = (b1_right - b1_left) * (b1_bottom - b1_top) @@ -77,7 +78,7 @@ def YOLOv3Loss(x, gtbox, gtlabel, gtscore, attrs): downsample_ratio = attrs['downsample_ratio'] use_label_smooth = attrs['use_label_smooth'] scale_x_y = attrs['scale_x_y'] - bias_x_y = -0.5 * (scale_x_y - 1.) + bias_x_y = -0.5 * (scale_x_y - 1.0) input_size = downsample_ratio * h x = x.reshape((n, mask_num, 5 + class_num, h, w)).transpose((0, 1, 3, 4, 2)) loss = np.zeros((n)).astype('float64') @@ -89,18 +90,19 @@ def YOLOv3Loss(x, gtbox, gtlabel, gtscore, attrs): pred_box = x[:, :, :, :, :4].copy() grid_x = np.tile(np.arange(w).reshape((1, w)), (h, 1)) grid_y = np.tile(np.arange(h).reshape((h, 1)), (1, w)) - pred_box[:, :, :, :, - 0] = (grid_x + sigmoid(pred_box[:, :, :, :, 0]) * scale_x_y + - bias_x_y) / w - pred_box[:, :, :, :, - 1] = (grid_y + sigmoid(pred_box[:, :, :, :, 1]) * scale_x_y + - bias_x_y) / h + pred_box[:, :, :, :, 0] = ( + grid_x + sigmoid(pred_box[:, :, :, :, 0]) * scale_x_y + bias_x_y + ) / w + pred_box[:, :, :, :, 1] = ( + grid_y + sigmoid(pred_box[:, :, :, :, 1]) * scale_x_y + bias_x_y + ) / h mask_anchors = [] for m in anchor_mask: mask_anchors.append((anchors[2 * m], anchors[2 * m + 1])) - anchors_s = np.array([(an_w / input_size, an_h / input_size) - for an_w, an_h in mask_anchors]) + anchors_s = np.array( + [(an_w / input_size, an_h / input_size) for an_w, an_h in mask_anchors] + ) anchor_w = anchors_s[:, 0:1].reshape((1, mask_num, 1, 1)) anchor_h = anchors_s[:, 1:2].reshape((1, mask_num, 1, 1)) pred_box[:, :, :, :, 2] = np.exp(pred_box[:, :, :, :, 2]) * anchor_w @@ -111,18 +113,21 @@ def YOLOv3Loss(x, gtbox, gtlabel, gtscore, attrs): objness = np.zeros(pred_box.shape[:2]).astype('float64') ious = batch_xywh_box_iou(pred_box, gtbox) ious_max = np.max(ious, axis=-1) - objness = np.where(ious_max > ignore_thresh, -np.ones_like(objness), - objness) + objness = np.where( + ious_max > ignore_thresh, -np.ones_like(objness), objness + ) gtbox_shift = gtbox.copy() gtbox_shift[:, :, 0] = 0 gtbox_shift[:, :, 1] = 0 anchors = [(anchors[2 * i], anchors[2 * i + 1]) for i in range(0, an_num)] - anchors_s = np.array([(an_w / input_size, an_h / input_size) - for an_w, an_h in anchors]) - anchor_boxes = np.concatenate([np.zeros_like(anchors_s), anchors_s], - axis=-1) + anchors_s = np.array( + [(an_w / input_size, an_h / input_size) for an_w, an_h in anchors] + ) + anchor_boxes = np.concatenate( + [np.zeros_like(anchors_s), anchors_s], axis=-1 + ) anchor_boxes = np.tile(anchor_boxes[np.newaxis, :, :], (n, 1, 1)) ious = batch_xywh_box_iou(gtbox_shift, anchor_boxes) iou_matches = np.argmax(ious, axis=-1) @@ -153,9 +158,13 @@ def YOLOv3Loss(x, gtbox, gtlabel, gtscore, attrs): objness[i, an_idx * h * w + gj * w + gi] = gtscore[i, j] for label_idx in range(class_num): - loss[i] += sce( - x[i, an_idx, gj, gi, 5 + label_idx], label_pos if label_idx - == gtlabel[i, j] else label_neg) * gtscore[i, j] + loss[i] += ( + sce( + x[i, an_idx, gj, gi, 5 + label_idx], + label_pos if label_idx == gtlabel[i, j] else label_neg, + ) + * gtscore[i, j] + ) for j in range(mask_num * h * w): if objness[i, j] > 0: @@ -163,40 +172,62 @@ def YOLOv3Loss(x, gtbox, gtlabel, gtscore, attrs): elif objness[i, j] == 0: loss[i] += sce(pred_obj[i, j], 0.0) - return (loss, objness.reshape((n, mask_num, h, w)).astype('float64'), \ - gt_matches.astype('int32')) - - -def yolo_loss_wrapper(x, - gt_box, - gt_label, - gt_score=None, - anchors=[ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, - 90, 156, 198, 373, 326 - ], - anchor_mask=[0, 1, 2], - class_num=5, - ignore_thresh=0.7, - downsample_ratio=32, - use_label_smooth=True, - scale_x_y=1.): - loss = paddle.vision.ops.yolo_loss(x, - gt_box=gt_box, - gt_label=gt_label, - anchors=anchors, - anchor_mask=anchor_mask, - class_num=class_num, - ignore_thresh=ignore_thresh, - downsample_ratio=downsample_ratio, - gt_score=gt_score, - use_label_smooth=use_label_smooth, - scale_x_y=scale_x_y) + return ( + loss, + objness.reshape((n, mask_num, h, w)).astype('float64'), + gt_matches.astype('int32'), + ) + + +def yolo_loss_wrapper( + x, + gt_box, + gt_label, + gt_score=None, + anchors=[ + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, + ], + anchor_mask=[0, 1, 2], + class_num=5, + ignore_thresh=0.7, + downsample_ratio=32, + use_label_smooth=True, + scale_x_y=1.0, +): + loss = paddle.vision.ops.yolo_loss( + x, + gt_box=gt_box, + gt_label=gt_label, + anchors=anchors, + anchor_mask=anchor_mask, + class_num=class_num, + ignore_thresh=ignore_thresh, + downsample_ratio=downsample_ratio, + gt_score=gt_score, + use_label_smooth=use_label_smooth, + scale_x_y=scale_x_y, + ) return loss class TestYolov3LossOp(OpTest): - def setUp(self): self.initTestCase() self.op_type = 'yolov3_loss' @@ -230,12 +261,13 @@ def setUp(self): gtscore = np.random.random(self.gtbox_shape[:2]).astype('float64') self.inputs['GTScore'] = gtscore - loss, objness, gt_matches = YOLOv3Loss(x, gtbox, gtlabel, gtscore, - self.attrs) + loss, objness, gt_matches = YOLOv3Loss( + x, gtbox, gtlabel, gtscore, self.attrs + ) self.outputs = { 'Loss': loss, 'ObjectnessMask': objness, - "GTMatchMask": gt_matches + "GTMatchMask": gt_matches, } def test_check_output(self): @@ -244,15 +276,30 @@ def test_check_output(self): def test_check_grad_ignore_gtbox(self): place = core.CPUPlace() - self.check_grad_with_place(place, ['X'], - 'Loss', - max_relative_error=0.2, - check_eager=True) + self.check_grad_with_place( + place, ['X'], 'Loss', max_relative_error=0.2, check_eager=True + ) def initTestCase(self): self.anchors = [ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, - 373, 326 + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, ] self.anchor_mask = [0, 1, 2] self.class_num = 5 @@ -262,15 +309,30 @@ def initTestCase(self): self.gtbox_shape = (3, 5, 4) self.gtscore = True self.use_label_smooth = True - self.scale_x_y = 1. + self.scale_x_y = 1.0 class TestYolov3LossWithoutLabelSmooth(TestYolov3LossOp): - def initTestCase(self): self.anchors = [ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, - 373, 326 + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, ] self.anchor_mask = [0, 1, 2] self.class_num = 5 @@ -280,15 +342,30 @@ def initTestCase(self): self.gtbox_shape = (3, 5, 4) self.gtscore = True self.use_label_smooth = False - self.scale_x_y = 1. + self.scale_x_y = 1.0 class TestYolov3LossNoGTScore(TestYolov3LossOp): - def initTestCase(self): self.anchors = [ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, - 373, 326 + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, ] self.anchor_mask = [0, 1, 2] self.class_num = 5 @@ -298,15 +375,30 @@ def initTestCase(self): self.gtbox_shape = (3, 5, 4) self.gtscore = False self.use_label_smooth = True - self.scale_x_y = 1. + self.scale_x_y = 1.0 class TestYolov3LossWithScaleXY(TestYolov3LossOp): - def initTestCase(self): self.anchors = [ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, - 373, 326 + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, ] self.anchor_mask = [0, 1, 2] self.class_num = 5 @@ -320,7 +412,6 @@ def initTestCase(self): class TestYolov3LossDygraph(unittest.TestCase): - def test_dygraph(self): paddle.disable_static() x = np.random.random([2, 14, 8, 8]).astype('float32') @@ -331,52 +422,57 @@ def test_dygraph(self): gt_box = paddle.to_tensor(gt_box) gt_label = paddle.to_tensor(gt_label) - loss = paddle.vision.ops.yolo_loss(x, - gt_box=gt_box, - gt_label=gt_label, - anchors=[10, 13, 16, 30], - anchor_mask=[0, 1], - class_num=2, - ignore_thresh=0.7, - downsample_ratio=8, - use_label_smooth=True, - scale_x_y=1.) + loss = paddle.vision.ops.yolo_loss( + x, + gt_box=gt_box, + gt_label=gt_label, + anchors=[10, 13, 16, 30], + anchor_mask=[0, 1], + class_num=2, + ignore_thresh=0.7, + downsample_ratio=8, + use_label_smooth=True, + scale_x_y=1.0, + ) assert loss is not None assert loss.shape == [2] paddle.enable_static() class TestYolov3LossStatic(unittest.TestCase): - def test_static(self): x = paddle.static.data('x', [2, 14, 8, 8], 'float32') gt_box = paddle.static.data('gt_box', [2, 10, 4], 'float32') gt_label = paddle.static.data('gt_label', [2, 10], 'int32') gt_score = paddle.static.data('gt_score', [2, 10], 'float32') - loss = paddle.vision.ops.yolo_loss(x, - gt_box=gt_box, - gt_label=gt_label, - anchors=[10, 13, 16, 30], - anchor_mask=[0, 1], - class_num=2, - ignore_thresh=0.7, - downsample_ratio=8, - gt_score=gt_score, - use_label_smooth=True, - scale_x_y=1.) + loss = paddle.vision.ops.yolo_loss( + x, + gt_box=gt_box, + gt_label=gt_label, + anchors=[10, 13, 16, 30], + anchor_mask=[0, 1], + class_num=2, + ignore_thresh=0.7, + downsample_ratio=8, + gt_score=gt_score, + use_label_smooth=True, + scale_x_y=1.0, + ) assert loss is not None - loss = paddle.vision.ops.yolo_loss(x, - gt_box=gt_box, - gt_label=gt_label, - anchors=[10, 13, 16, 30], - anchor_mask=[0, 1], - class_num=2, - ignore_thresh=0.7, - downsample_ratio=8, - use_label_smooth=True, - scale_x_y=1.) + loss = paddle.vision.ops.yolo_loss( + x, + gt_box=gt_box, + gt_label=gt_label, + anchors=[10, 13, 16, 30], + anchor_mask=[0, 1], + class_num=2, + ignore_thresh=0.7, + downsample_ratio=8, + use_label_smooth=True, + scale_x_y=1.0, + ) assert loss is not None diff --git a/python/paddle/fluid/tests/unittests/test_zero_dim_shape.py b/python/paddle/fluid/tests/unittests/test_zero_dim_shape.py index 089a076ad80dc..df4fa96d4a36c 100644 --- a/python/paddle/fluid/tests/unittests/test_zero_dim_shape.py +++ b/python/paddle/fluid/tests/unittests/test_zero_dim_shape.py @@ -68,7 +68,6 @@ # Use to test zero-dim in the whole API class TestUnaryAPI(unittest.TestCase): - def test_dygraph_unary(self): paddle.disable_static() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) @@ -108,13 +107,14 @@ def test_static_unary(self): self.assertEqual(out.shape, ()) exe = fluid.Executor() - result = exe.run(main_prog, - fetch_list=[x, out, x_grad, out_grad]) + result = exe.run( + main_prog, fetch_list=[x, out, x_grad, out_grad] + ) # Test runtime shape self.assertEqual(result[0].shape, ()) self.assertEqual(result[1].shape, ()) - self.assertEqual(result[3].shape, (1, )) + self.assertEqual(result[3].shape, (1,)) # 0D will be stacked when 1+ place, due to it cannot be concated # for 1 place: [ x-place1 ] @@ -126,28 +126,34 @@ def test_static_unary(self): else: places = [paddle.CPUPlace()] * 4 device_num = 4 - expect_shape = (device_num, ) + expect_shape = (device_num,) compiled_program = fluid.CompiledProgram( - main_prog).with_data_parallel(out.name, places=places) - result = exe.run(compiled_program, - fetch_list=[x, out, x_grad, out_grad], - return_merged=True) + main_prog + ).with_data_parallel(out.name, places=places) + result = exe.run( + compiled_program, + fetch_list=[x, out, x_grad, out_grad], + return_merged=True, + ) # Test runtime parallel shape self.assertEqual(result[0].shape, expect_shape) self.assertEqual(result[1].shape, expect_shape) - self.assertEqual(result[3].shape, (device_num, )) + self.assertEqual(result[3].shape, (device_num,)) compiled_program = fluid.CompiledProgram( - main_prog).with_data_parallel(out.name, places=places) - result = exe.run(compiled_program, - fetch_list=[x, out, x_grad, out_grad], - return_merged=False) + main_prog + ).with_data_parallel(out.name, places=places) + result = exe.run( + compiled_program, + fetch_list=[x, out, x_grad, out_grad], + return_merged=False, + ) # [[x-place1, x-place2, ...], [], [], ...] - self.assertEqual(np.array(result[0]).shape, (device_num, )) - self.assertEqual(np.array(result[1]).shape, (device_num, )) + self.assertEqual(np.array(result[0]).shape, (device_num,)) + self.assertEqual(np.array(result[1]).shape, (device_num,)) self.assertEqual(np.array(result[3]).shape, (device_num, 1)) paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_zeropad2d.py b/python/paddle/fluid/tests/unittests/test_zeropad2d.py index f15f3d277d58f..99a6b9143a911 100644 --- a/python/paddle/fluid/tests/unittests/test_zeropad2d.py +++ b/python/paddle/fluid/tests/unittests/test_zeropad2d.py @@ -85,7 +85,8 @@ def func_support_pad2(self): pad = [1, 2, 3, 4] x = np.random.randint(-255, 255, size=self.shape) expect_res = np.pad( - x, [[0, 0], [0, 0], [pad[2], pad[3]], [pad[0], pad[1]]]) + x, [[0, 0], [0, 0], [pad[2], pad[3]], [pad[0], pad[1]]] + ) x_tensor = to_tensor(x) ret_res = zeropad2d(x_tensor, pad).numpy() @@ -103,7 +104,8 @@ def func_support_pad3(self): pad = (1, 2, 3, 4) x = np.random.randint(-255, 255, size=self.shape) expect_res = np.pad( - x, [[0, 0], [0, 0], [pad[2], pad[3]], [pad[0], pad[1]]]) + x, [[0, 0], [0, 0], [pad[2], pad[3]], [pad[0], pad[1]]] + ) x_tensor = to_tensor(x) ret_res = zeropad2d(x_tensor, pad).numpy() @@ -121,7 +123,8 @@ def func_support_pad4(self): pad = [1, 2, 3, 4] x = np.random.randint(-255, 255, size=self.shape) expect_res = np.pad( - x, [[0, 0], [0, 0], [pad[2], pad[3]], [pad[0], pad[1]]]) + x, [[0, 0], [0, 0], [pad[2], pad[3]], [pad[0], pad[1]]] + ) x_tensor = to_tensor(x) pad_tensor = to_tensor(pad, dtype='int32') @@ -144,15 +147,22 @@ def setUp(self): self.pad = [2, 2, 4, 1] self.padLayer = ZeroPad2D(padding=self.pad) self.x = np.random.randint(-255, 255, size=self.shape) - self.expect_res = np.pad(self.x, - [[0, 0], [0, 0], [self.pad[2], self.pad[3]], - [self.pad[0], self.pad[1]]]) + self.expect_res = np.pad( + self.x, + [ + [0, 0], + [0, 0], + [self.pad[2], self.pad[3]], + [self.pad[0], self.pad[1]], + ], + ) def func_layer(self): - np.testing.assert_allclose(zeropad2d(to_tensor(self.x), - self.pad).numpy(), - self.padLayer(to_tensor(self.x)), - rtol=1e-05) + np.testing.assert_allclose( + zeropad2d(to_tensor(self.x), self.pad).numpy(), + self.padLayer(to_tensor(self.x)), + rtol=1e-05, + ) def test_layer(self): with paddle.fluid.framework._test_eager_guard(): diff --git a/python/paddle/fluid/tests/unittests/test_zeros_like_op.py b/python/paddle/fluid/tests/unittests/test_zeros_like_op.py index 0a86123ea2c16..5712fc4df6e5f 100644 --- a/python/paddle/fluid/tests/unittests/test_zeros_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_zeros_like_op.py @@ -24,7 +24,6 @@ class TestZerosLikeAPIError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): x = paddle.fluid.data('x', [3, 4]) @@ -36,7 +35,6 @@ def test_eager(self): class TestZerosLikeAPI(unittest.TestCase): - def test_api(self): shape = [3, 4] startup_program = Program() @@ -48,14 +46,20 @@ def test_api(self): out3 = zeros_like(x, 'float64') out4 = zeros_like(x, 'int32') out5 = zeros_like(x, 'int64') - place = (fluid.CUDAPlace(0) - if core.is_compiled_with_cuda() else fluid.CPUPlace()) + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) - outs = exe.run(train_program, - feed={'X': np.ones(shape).astype('float32')}, - fetch_list=[out1, out2, out3, out4, out5]) + outs = exe.run( + train_program, + feed={'X': np.ones(shape).astype('float32')}, + fetch_list=[out1, out2, out3, out4, out5], + ) for (i, dtype) in enumerate( - [np.float32, np.bool_, np.float64, np.int32, np.int64]): + [np.float32, np.bool_, np.float64, np.int32, np.int64] + ): self.assertEqual(outs[i].dtype, dtype) self.assertEqual((outs[i] == np.zeros(shape, dtype)).all(), True) @@ -65,17 +69,20 @@ def test_eager(self): class TestZerosLikeImpeartive(unittest.TestCase): - def test_out(self): shape = [3, 4] - place = (fluid.CUDAPlace(0) - if core.is_compiled_with_cuda() else fluid.CPUPlace()) + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) paddle.disable_static(place) x = paddle.to_tensor(np.ones(shape)) for dtype in [np.bool_, np.float32, np.float64, np.int32, np.int64]: out = zeros_like(x, dtype) - self.assertEqual((out.numpy() == np.zeros(shape, dtype)).all(), - True) + self.assertEqual( + (out.numpy() == np.zeros(shape, dtype)).all(), True + ) out = paddle.tensor.zeros_like(x) self.assertEqual((out.numpy() == np.zeros(shape, dtype)).all(), True) out = paddle.tensor.creation.zeros_like(x) @@ -88,20 +95,23 @@ def test_eager(self): class TestZerosAPI(unittest.TestCase): - def test_api(self): shape = [3, 4] - place = fluid.CUDAPlace( - 0) if core.is_compiled_with_cuda() else fluid.CPUPlace() + place = ( + fluid.CUDAPlace(0) + if core.is_compiled_with_cuda() + else fluid.CPUPlace() + ) paddle.disable_static(place) for dtype in [np.float32, np.float64, np.int32, np.int64]: out = _C_ops.zeros(shape, convert_np_dtype_to_dtype_(dtype), place) - self.assertEqual((out.numpy() == np.zeros(shape, dtype)).all(), - True) + self.assertEqual( + (out.numpy() == np.zeros(shape, dtype)).all(), True + ) paddle.enable_static() -if (__name__ == '__main__'): +if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_zeros_op.py b/python/paddle/fluid/tests/unittests/test_zeros_op.py index d91230a48f258..59a87dba56d86 100644 --- a/python/paddle/fluid/tests/unittests/test_zeros_op.py +++ b/python/paddle/fluid/tests/unittests/test_zeros_op.py @@ -21,7 +21,6 @@ class TestZerosOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): shape = [4] @@ -34,27 +33,26 @@ def test_eager(self): class ApiZerosTest(unittest.TestCase): - def test_out(self): with program_guard(Program()): zeros = paddle.zeros(shape=[10], dtype='float64') place = paddle.CPUPlace() exe = paddle.static.Executor(place) - (result, ) = exe.run(fetch_list=[zeros]) + (result,) = exe.run(fetch_list=[zeros]) expected_result = np.zeros(10, dtype='float64') self.assertEqual((result == expected_result).all(), True) with paddle.static.program_guard(Program()): zeros = paddle.zeros(shape=[10], dtype='int64') place = paddle.CPUPlace() exe = paddle.static.Executor(place) - (result, ) = exe.run(fetch_list=[zeros]) + (result,) = exe.run(fetch_list=[zeros]) expected_result = np.zeros(10, dtype='int64') self.assertEqual((result == expected_result).all(), True) with program_guard(Program()): zeros = paddle.zeros(shape=[10], dtype='int64') place = paddle.CPUPlace() exe = paddle.static.Executor(place) - (result, ) = exe.run(fetch_list=[zeros]) + (result,) = exe.run(fetch_list=[zeros]) expected_result = np.zeros(10, dtype='int64') self.assertEqual((result == expected_result).all(), True) with program_guard(Program()): @@ -70,7 +68,7 @@ def test_fluid_out(self): zeros = fluid.layers.zeros(shape=[10], dtype='int64') place = paddle.CPUPlace() exe = paddle.static.Executor(place) - (result, ) = exe.run(fetch_list=[zeros]) + (result,) = exe.run(fetch_list=[zeros]) expected_result = np.zeros(10, dtype='int64') self.assertEqual((result == expected_result).all(), True) @@ -81,9 +79,7 @@ def test_eager(self): class ApiZerosError(unittest.TestCase): - def test_errors(self): - def test_error1(): with paddle.static.program_guard(fluid.Program()): ones = fluid.layers.zeros(shape=10, dtype='int64') @@ -111,5 +107,5 @@ def test_eager(self): self.test_shape_errors() -if (__name__ == '__main__'): +if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/testsuite.py b/python/paddle/fluid/tests/unittests/testsuite.py index 619d823281166..a13b73f501bd6 100644 --- a/python/paddle/fluid/tests/unittests/testsuite.py +++ b/python/paddle/fluid/tests/unittests/testsuite.py @@ -70,7 +70,6 @@ def __create_var__(name, var_name): def set_input(scope, op, inputs, place): - def __set_input__(var_name, var): if isinstance(var, tuple) or isinstance(var, np.ndarray): tensor = scope.find_var(var_name).get_tensor() @@ -119,10 +118,9 @@ def create_var(block, name, np_list, var_proto): if is_input: shape = list(np_value.shape) lod_level = 0 - return block.create_var(dtype=dtype, - shape=shape, - lod_level=lod_level, - name=name) + return block.create_var( + dtype=dtype, shape=shape, lod_level=lod_level, name=name + ) var_dict = {} for var_proto in proto_list: @@ -130,15 +128,18 @@ def create_var(block, name, np_list, var_proto): if (var_name not in np_list) and var_proto.dispensable: continue if is_input: - assert (var_name in np_list) or (var_proto.dispensable), \ - "Missing {} as input".format(var_name) + assert (var_name in np_list) or ( + var_proto.dispensable + ), "Missing {} as input".format(var_name) if var_proto.duplicable: - assert isinstance(np_list[var_name], list), \ - "Duplicable {} should be set as list".format(var_name) + assert isinstance( + np_list[var_name], list + ), "Duplicable {} should be set as list".format(var_name) var_list = [] for (name, np_value) in np_list[var_name]: var_list.append( - create_var(block, name, {name: np_value}, var_proto)) + create_var(block, name, {name: np_value}, var_proto) + ) var_dict[var_name] = var_list else: var_dict[var_name] = create_var(block, var_name, np_list, var_proto) @@ -151,34 +152,38 @@ def append_loss_ops(block, output_names): if len(mean_inputs) == 1: loss = block.create_var(dtype=mean_inputs[0].dtype, shape=[1]) - op = block.append_op(inputs={"X": mean_inputs}, - outputs={"Out": loss}, - type='mean') + op = block.append_op( + inputs={"X": mean_inputs}, outputs={"Out": loss}, type='mean' + ) op.desc.infer_var_type(block.desc) op.desc.infer_shape(block.desc) else: avg_sum = [] for cur_loss in mean_inputs: cur_avg_loss = block.create_var(dtype=cur_loss.dtype, shape=[1]) - op = block.append_op(inputs={"X": [cur_loss]}, - outputs={"Out": [cur_avg_loss]}, - type="mean") + op = block.append_op( + inputs={"X": [cur_loss]}, + outputs={"Out": [cur_avg_loss]}, + type="mean", + ) op.desc.infer_var_type(block.desc) op.desc.infer_shape(block.desc) avg_sum.append(cur_avg_loss) loss_sum = block.create_var(dtype=avg_sum[0].dtype, shape=[1]) - op_sum = block.append_op(inputs={"X": avg_sum}, - outputs={"Out": loss_sum}, - type='sum') + op_sum = block.append_op( + inputs={"X": avg_sum}, outputs={"Out": loss_sum}, type='sum' + ) op_sum.desc.infer_var_type(block.desc) op_sum.desc.infer_shape(block.desc) loss = block.create_var(dtype=loss_sum.dtype, shape=[1]) - op_loss = block.append_op(inputs={"X": loss_sum}, - outputs={"Out": loss}, - type='scale', - attrs={'scale': 1.0 / float(len(avg_sum))}) + op_loss = block.append_op( + inputs={"X": loss_sum}, + outputs={"Out": loss}, + type='scale', + attrs={'scale': 1.0 / float(len(avg_sum))}, + ) op_loss.desc.infer_var_type(block.desc) op_loss.desc.infer_shape(block.desc) return loss diff --git a/python/paddle/fluid/tests/unittests/tokenizer/bert_tokenizer.py b/python/paddle/fluid/tests/unittests/tokenizer/bert_tokenizer.py index 24f2752854f38..cd3546ebe3cd0 100755 --- a/python/paddle/fluid/tests/unittests/tokenizer/bert_tokenizer.py +++ b/python/paddle/fluid/tests/unittests/tokenizer/bert_tokenizer.py @@ -17,7 +17,13 @@ import unicodedata from tokenizer_utils import PretrainedTokenizer -from tokenizer_utils import convert_to_unicode, whitespace_tokenize, _is_whitespace, _is_control, _is_punctuation +from tokenizer_utils import ( + convert_to_unicode, + whitespace_tokenize, + _is_whitespace, + _is_control, + _is_punctuation, +) class BasicTokenizer(object): @@ -129,14 +135,16 @@ def _is_chinese_char(self, cp): # as is Japanese Hiragana and Katakana. Those alphabets are used to write # space-separated words, so they are not treated specially and handled # like the all of the other languages. - if ((cp >= 0x4E00 and cp <= 0x9FFF) or # - (cp >= 0x3400 and cp <= 0x4DBF) or # - (cp >= 0x20000 and cp <= 0x2A6DF) or # - (cp >= 0x2A700 and cp <= 0x2B73F) or # - (cp >= 0x2B740 and cp <= 0x2B81F) or # - (cp >= 0x2B820 and cp <= 0x2CEAF) or - (cp >= 0xF900 and cp <= 0xFAFF) or # - (cp >= 0x2F800 and cp <= 0x2FA1F)): # + if ( + (cp >= 0x4E00 and cp <= 0x9FFF) + or (cp >= 0x3400 and cp <= 0x4DBF) # + or (cp >= 0x20000 and cp <= 0x2A6DF) # + or (cp >= 0x2A700 and cp <= 0x2B73F) # + or (cp >= 0x2B740 and cp <= 0x2B81F) # + or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0xF900 and cp <= 0xFAFF) + or (cp >= 0x2F800 and cp <= 0x2FA1F) # + ): # return True return False @@ -148,7 +156,7 @@ def _clean_text(self, text): output = [] for char in text: cp = ord(char) - if cp == 0 or cp == 0xfffd or _is_control(char): + if cp == 0 or cp == 0xFFFD or _is_control(char): continue if _is_whitespace(char): output.append(" ") @@ -273,101 +281,71 @@ class BertTokenizer(PretrainedTokenizer): {'input_ids': [101, 2002, 2001, 1037, 13997, 11510, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0]} ''' """ + resource_files_names = {"vocab_file": "vocab.txt"} # for save_pretrained pretrained_resource_files_map = { "vocab_file": { - "bert-base-uncased": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-uncased-vocab.txt", - "bert-large-uncased": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-large-uncased-vocab.txt", - "bert-base-cased": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-cased-vocab.txt", - "bert-large-cased": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-large-cased-vocab.txt", - "bert-base-multilingual-uncased": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-multilingual-uncased-vocab.txt", - "bert-base-multilingual-cased": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-multilingual-cased-vocab.txt", - "bert-base-chinese": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-chinese-vocab.txt", - "bert-wwm-chinese": - "http://paddlenlp.bj.bcebos.com/models/transformers/bert/bert-wwm-chinese-vocab.txt", - "bert-wwm-ext-chinese": - "http://paddlenlp.bj.bcebos.com/models/transformers/bert/bert-wwm-ext-chinese-vocab.txt", - "macbert-large-chinese": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-chinese-vocab.txt", - "macbert-base-chinese": - "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-chinese-vocab.txt", - "simbert-base-chinese": - "https://paddlenlp.bj.bcebos.com/models/transformers/simbert/vocab.txt", + "bert-base-uncased": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-uncased-vocab.txt", + "bert-large-uncased": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-large-uncased-vocab.txt", + "bert-base-cased": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-cased-vocab.txt", + "bert-large-cased": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-large-cased-vocab.txt", + "bert-base-multilingual-uncased": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-multilingual-uncased-vocab.txt", + "bert-base-multilingual-cased": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-multilingual-cased-vocab.txt", + "bert-base-chinese": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-chinese-vocab.txt", + "bert-wwm-chinese": "http://paddlenlp.bj.bcebos.com/models/transformers/bert/bert-wwm-chinese-vocab.txt", + "bert-wwm-ext-chinese": "http://paddlenlp.bj.bcebos.com/models/transformers/bert/bert-wwm-ext-chinese-vocab.txt", + "macbert-large-chinese": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-chinese-vocab.txt", + "macbert-base-chinese": "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-chinese-vocab.txt", + "simbert-base-chinese": "https://paddlenlp.bj.bcebos.com/models/transformers/simbert/vocab.txt", } } pretrained_init_configuration = { - "bert-base-uncased": { - "do_lower_case": True - }, - "bert-large-uncased": { - "do_lower_case": True - }, - "bert-base-cased": { - "do_lower_case": False - }, - "bert-large-cased": { - "do_lower_case": False - }, - "bert-base-multilingual-uncased": { - "do_lower_case": True - }, - "bert-base-multilingual-cased": { - "do_lower_case": False - }, - "bert-base-chinese": { - "do_lower_case": False - }, - "bert-wwm-chinese": { - "do_lower_case": False - }, - "bert-wwm-ext-chinese": { - "do_lower_case": False - }, - "macbert-large-chinese": { - "do_lower_case": False - }, - "macbert-base-chinese": { - "do_lower_case": False - }, - "simbert-base-chinese": { - "do_lower_case": True - }, + "bert-base-uncased": {"do_lower_case": True}, + "bert-large-uncased": {"do_lower_case": True}, + "bert-base-cased": {"do_lower_case": False}, + "bert-large-cased": {"do_lower_case": False}, + "bert-base-multilingual-uncased": {"do_lower_case": True}, + "bert-base-multilingual-cased": {"do_lower_case": False}, + "bert-base-chinese": {"do_lower_case": False}, + "bert-wwm-chinese": {"do_lower_case": False}, + "bert-wwm-ext-chinese": {"do_lower_case": False}, + "macbert-large-chinese": {"do_lower_case": False}, + "macbert-base-chinese": {"do_lower_case": False}, + "simbert-base-chinese": {"do_lower_case": True}, } padding_side = 'right' - def __init__(self, - vocab_file, - do_lower_case=True, - unk_token="[UNK]", - sep_token="[SEP]", - pad_token="[PAD]", - cls_token="[CLS]", - mask_token="[MASK]"): + def __init__( + self, + vocab_file, + do_lower_case=True, + unk_token="[UNK]", + sep_token="[SEP]", + pad_token="[PAD]", + cls_token="[CLS]", + mask_token="[MASK]", + ): if not os.path.isfile(vocab_file): raise ValueError( "Can't find a vocabulary file at path '{}'. To load the " "vocabulary from a pretrained model please use " - "`tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`" - .format(vocab_file)) + "`tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format( + vocab_file + ) + ) self.vocab = self.load_vocabulary(vocab_file, unk_token=unk_token) self.do_lower_case = do_lower_case self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) - self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, - unk_token=unk_token) + self.wordpiece_tokenizer = WordpieceTokenizer( + vocab=self.vocab, unk_token=unk_token + ) self.special_tokens_map = { 'unk_token': unk_token, 'sep_token': sep_token, 'pad_token': pad_token, 'cls_token': cls_token, - 'mask_token': mask_token + 'mask_token': mask_token, } @property @@ -430,7 +408,9 @@ def num_special_tokens_to_add(self, pair=False): token_ids_1 = [] return len( self.build_inputs_with_special_tokens( - token_ids_0, token_ids_1 if pair else None)) + token_ids_0, token_ids_1 if pair else None + ) + ) def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): """ @@ -454,9 +434,9 @@ def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): _sep = [self.sep_token_id] return _cls + token_ids_0 + _sep + token_ids_1 + _sep - def create_token_type_ids_from_sequences(self, - token_ids_0, - token_ids_1=None): + def create_token_type_ids_from_sequences( + self, token_ids_0, token_ids_1=None + ): """ Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence pair mask has the following format: @@ -476,13 +456,13 @@ def create_token_type_ids_from_sequences(self, _cls = [self.cls_token_id] if token_ids_1 is None: return len(_cls + token_ids_0 + _sep) * [0] - return len(_cls + token_ids_0 + _sep) * [0] + len(token_ids_1 + - _sep) * [1] + return len(_cls + token_ids_0 + _sep) * [0] + len( + token_ids_1 + _sep + ) * [1] - def get_special_tokens_mask(self, - token_ids_0, - token_ids_1=None, - already_has_special_tokens=False): + def get_special_tokens_mask( + self, token_ids_0, token_ids_1=None, already_has_special_tokens=False + ): """ Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer ``encode`` methods. @@ -506,10 +486,18 @@ def get_special_tokens_mask(self, return list( map( lambda x: 1 - if x in [self.sep_token_id, self.cls_token_id] else 0, - token_ids_0)) + if x in [self.sep_token_id, self.cls_token_id] + else 0, + token_ids_0, + ) + ) if token_ids_1 is not None: - return [1] + ([0] * len(token_ids_0)) + [1] + ( - [0] * len(token_ids_1)) + [1] + return ( + [1] + + ([0] * len(token_ids_0)) + + [1] + + ([0] * len(token_ids_1)) + + [1] + ) return [1] + ([0] * len(token_ids_0)) + [1] diff --git a/python/paddle/fluid/tests/unittests/tokenizer/tokenizer_utils.py b/python/paddle/fluid/tests/unittests/tokenizer/tokenizer_utils.py index 0524854834b52..9e60b29ffb7ab 100644 --- a/python/paddle/fluid/tests/unittests/tokenizer/tokenizer_utils.py +++ b/python/paddle/fluid/tests/unittests/tokenizer/tokenizer_utils.py @@ -89,8 +89,12 @@ def _is_punctuation(char): # Characters such as "^", "$", and "`" are not in the Unicode # Punctuation class but we treat them as punctuation anyways, for # consistency. - if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) - or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + if ( + (cp >= 33 and cp <= 47) + or (cp >= 58 and cp <= 64) + or (cp >= 91 and cp <= 96) + or (cp >= 123 and cp <= 126) + ): return True cat = unicodedata.category(char) if cat.startswith("P"): @@ -108,14 +112,16 @@ def is_chinese_char(cp): # as is Japanese Hiragana and Katakana. Those alphabets are used to write # space-separated words, so they are not treated specially and handled # like the all of the other languages. - if ((cp >= 0x4E00 and cp <= 0x9FFF) or # - (cp >= 0x3400 and cp <= 0x4DBF) or # - (cp >= 0x20000 and cp <= 0x2A6DF) or # - (cp >= 0x2A700 and cp <= 0x2B73F) or # - (cp >= 0x2B740 and cp <= 0x2B81F) or # - (cp >= 0x2B820 and cp <= 0x2CEAF) or - (cp >= 0xF900 and cp <= 0xFAFF) or # - (cp >= 0x2F800 and cp <= 0x2FA1F)): # + if ( + (cp >= 0x4E00 and cp <= 0x9FFF) + or (cp >= 0x3400 and cp <= 0x4DBF) # + or (cp >= 0x20000 and cp <= 0x2A6DF) # + or (cp >= 0x2A700 and cp <= 0x2B73F) # + or (cp >= 0x2B740 and cp <= 0x2B81F) # + or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0xF900 and cp <= 0xFAFF) + or (cp >= 0x2F800 and cp <= 0x2FA1F) # + ): # return True return False @@ -172,6 +178,7 @@ class PretrainedTokenizer(object): by which subclasses can track arguments for initialization automatically and expose special tokens initialization used as attributes. """ + tokenizer_config_file = "tokenizer_config.json" pretrained_init_configuration = {} resource_files_names = {} # keys are arguments of __init__ @@ -179,20 +186,22 @@ class PretrainedTokenizer(object): padding_side = 'right' pad_token_type_id = 0 - def __call__(self, - text, - text_pair=None, - max_seq_len: Optional[int] = None, - stride=0, - is_split_into_words=False, - pad_to_max_seq_len=False, - truncation_strategy="longest_first", - return_position_ids=False, - return_token_type_ids=True, - return_attention_mask=False, - return_length=False, - return_overflowing_tokens=False, - return_special_tokens_mask=False): + def __call__( + self, + text, + text_pair=None, + max_seq_len: Optional[int] = None, + stride=0, + is_split_into_words=False, + pad_to_max_seq_len=False, + truncation_strategy="longest_first", + return_position_ids=False, + return_token_type_ids=True, + return_attention_mask=False, + return_length=False, + return_overflowing_tokens=False, + return_special_tokens_mask=False, + ): """ Performs tokenization and uses the tokenized tokens to prepare model inputs. It supports sequence or sequence pair as input, and batch input @@ -287,32 +296,60 @@ def __call__(self, """ # Input type checking for clearer error assert isinstance(text, str) or ( - isinstance(text, (list, tuple)) and - (len(text) == 0 or - (isinstance(text[0], str) or - (isinstance(text[0], (list, tuple)) and - (len(text[0]) == 0 or isinstance(text[0][0], str))))) - ), ("text input must of type `str` (single example), `List[str]` (batch or single pretokenized example) " - "or `List[List[str]]` (batch of pretokenized examples).") + isinstance(text, (list, tuple)) + and ( + len(text) == 0 + or ( + isinstance(text[0], str) + or ( + isinstance(text[0], (list, tuple)) + and (len(text[0]) == 0 or isinstance(text[0][0], str)) + ) + ) + ) + ), ( + "text input must of type `str` (single example), `List[str]` (batch or single pretokenized example) " + "or `List[List[str]]` (batch of pretokenized examples)." + ) assert ( - text_pair is None or isinstance(text_pair, str) or - (isinstance(text_pair, (list, tuple)) and - (len(text_pair) == 0 or - (isinstance(text_pair[0], str) or - (isinstance(text_pair[0], (list, tuple)) and - (len(text_pair[0]) == 0 or isinstance(text_pair[0][0], str)))))) - ), ("text_pair input must of type `str` (single example), `List[str]` (batch or single pretokenized example) " - "or `List[List[str]]` (batch of pretokenized examples).") + text_pair is None + or isinstance(text_pair, str) + or ( + isinstance(text_pair, (list, tuple)) + and ( + len(text_pair) == 0 + or ( + isinstance(text_pair[0], str) + or ( + isinstance(text_pair[0], (list, tuple)) + and ( + len(text_pair[0]) == 0 + or isinstance(text_pair[0][0], str) + ) + ) + ) + ) + ) + ), ( + "text_pair input must of type `str` (single example), `List[str]` (batch or single pretokenized example) " + "or `List[List[str]]` (batch of pretokenized examples)." + ) is_batched = bool( (not is_split_into_words and isinstance(text, (list, tuple))) - or (is_split_into_words and isinstance(text, (list, tuple)) and text - and isinstance(text[0], (list, tuple)))) + or ( + is_split_into_words + and isinstance(text, (list, tuple)) + and text + and isinstance(text[0], (list, tuple)) + ) + ) if is_batched: - batch_text_or_text_pairs = list(zip( - text, text_pair)) if text_pair is not None else text + batch_text_or_text_pairs = ( + list(zip(text, text_pair)) if text_pair is not None else text + ) return self.batch_encode( batch_text_or_text_pairs=batch_text_or_text_pairs, max_seq_len=max_seq_len, @@ -325,7 +362,8 @@ def __call__(self, return_attention_mask=return_attention_mask, return_length=return_length, return_overflowing_tokens=return_overflowing_tokens, - return_special_tokens_mask=return_special_tokens_mask) + return_special_tokens_mask=return_special_tokens_mask, + ) else: return self.encode( text=text, @@ -338,7 +376,8 @@ def __call__(self, return_attention_mask=return_attention_mask, return_length=return_length, return_overflowing_tokens=return_overflowing_tokens, - return_special_tokens_mask=return_special_tokens_mask) + return_special_tokens_mask=return_special_tokens_mask, + ) @property def all_special_tokens(self): @@ -349,8 +388,11 @@ def all_special_tokens(self): all_toks = [] set_attr = self.special_tokens_map for attr_value in set_attr.values(): - all_toks = all_toks + (list(attr_value) if isinstance( - attr_value, (list, tuple)) else [attr_value]) + all_toks = all_toks + ( + list(attr_value) + if isinstance(attr_value, (list, tuple)) + else [attr_value] + ) all_toks = list(set(all_toks)) return all_toks @@ -426,11 +468,13 @@ def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs): # From local dir path elif os.path.isdir(pretrained_model_name_or_path): for file_id, file_name in cls.resource_files_names.items(): - full_file_name = os.path.join(pretrained_model_name_or_path, - file_name) + full_file_name = os.path.join( + pretrained_model_name_or_path, file_name + ) vocab_files[file_id] = full_file_name vocab_files["tokenizer_config_file"] = os.path.join( - pretrained_model_name_or_path, cls.tokenizer_config_file) + pretrained_model_name_or_path, cls.tokenizer_config_file + ) default_root = os.path.join(DATA_HOME, pretrained_model_name_or_path) resolved_vocab_files = {} @@ -443,11 +487,13 @@ def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs): print("Already cached %s" % path) resolved_vocab_files[file_id] = path else: - print("Downloading %s and saved to %s" % - (file_path, default_root)) + print( + "Downloading %s and saved to %s" % (file_path, default_root) + ) try: resolved_vocab_files[file_id] = get_path_from_url( - file_path, default_root) + file_path, default_root + ) except RuntimeError as err: print(err) raise RuntimeError( @@ -461,7 +507,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs): # Prepare tokenizer initialization kwargs # Did we saved some inputs and kwargs to reload ? tokenizer_config_file = resolved_vocab_files.pop( - "tokenizer_config_file", None) + "tokenizer_config_file", None + ) if tokenizer_config_file is not None: with open(tokenizer_config_file, 'r', encoding="utf-8") as f: init_kwargs = json.load(f) @@ -489,8 +536,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs): # does include a vocab file path in it. However, if the vocab file # path included in json does not exist, such as was deleted, to make # it still work, use the vocab file under this dir. - elif not os.path.isfile( - init_kwargs[args_name]) and os.path.isfile(file_path): + elif not os.path.isfile(init_kwargs[args_name]) and os.path.isfile( + file_path + ): init_kwargs[args_name] = file_path # TODO(guosheng): avoid reduplication of position args and key word args tokenizer = cls(*init_args, **init_kwargs) @@ -519,11 +567,13 @@ def save_pretrained(self, save_directory): assert not os.path.isfile( save_directory ), "Saving directory ({}) should be a directory, not a file".format( - save_directory) + save_directory + ) os.makedirs(save_directory, exist_ok=True) - tokenizer_config_file = os.path.join(save_directory, - self.tokenizer_config_file) + tokenizer_config_file = os.path.join( + save_directory, self.tokenizer_config_file + ) # init_config is set in metaclass created `__init__`, tokenizer_config = self.init_config with open(tokenizer_config_file, "w", encoding="utf-8") as f: @@ -545,12 +595,14 @@ def save_resources(self, save_directory): copyfile(src_path, dst_path) @staticmethod - def load_vocabulary(filepath, - unk_token=None, - pad_token=None, - bos_token=None, - eos_token=None, - **kwargs): + def load_vocabulary( + filepath, + unk_token=None, + pad_token=None, + bos_token=None, + eos_token=None, + **kwargs, + ): """ Instantiate an instance of `Vocab` from a file reserving all tokens by using `Vocab.from_dict`. The file contains a token per line, and the @@ -581,15 +633,20 @@ def __getattr__(self, name): return self.special_tokens_map[name] elif name.endswith('_token_id'): return self.vocab[self.special_tokens_map[name[:-3]]] - raise AttributeError("'{}' object has no attribute '{}'".format( - type(self).__name__, name)) - - def truncate_sequences(self, - ids, - pair_ids=None, - num_tokens_to_remove=0, - truncation_strategy='longest_first', - stride=0): + raise AttributeError( + "'{}' object has no attribute '{}'".format( + type(self).__name__, name + ) + ) + + def truncate_sequences( + self, + ids, + pair_ids=None, + num_tokens_to_remove=0, + truncation_strategy='longest_first', + stride=0, + ): """ Truncates a sequence pair in place to the maximum length. Args: @@ -662,9 +719,9 @@ def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): return token_ids_0 + token_ids_1 - def build_offset_mapping_with_special_tokens(self, - offset_mapping_0, - offset_mapping_1=None): + def build_offset_mapping_with_special_tokens( + self, offset_mapping_0, offset_mapping_1=None + ): """ Build offset map from a pair of offset map by concatenating and adding offsets of special tokens. Should be overridden in a subclass if the model has a special way of building those. @@ -681,10 +738,9 @@ def build_offset_mapping_with_special_tokens(self, return offset_mapping_0 + offset_mapping_1 - def get_special_tokens_mask(self, - token_ids_0, - token_ids_1=None, - already_has_special_tokens=False): + def get_special_tokens_mask( + self, token_ids_0, token_ids_1=None, already_has_special_tokens=False + ): """ Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer ``encode`` methods. @@ -698,11 +754,12 @@ def get_special_tokens_mask(self, 1 for a special token, 0 for a sequence token. """ return [0] * ( - (len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0)) + (len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0) + ) - def create_token_type_ids_from_sequences(self, - token_ids_0, - token_ids_1=None): + def create_token_type_ids_from_sequences( + self, token_ids_0, token_ids_1=None + ): """ Create a mask from the two sequences passed to be used in a sequence-pair classification task. Should be overridden in a subclass if the model has a special way of building those. @@ -733,20 +790,24 @@ def num_special_tokens_to_add(self, pair): token_ids_1 = [] return len( self.build_inputs_with_special_tokens( - token_ids_0, token_ids_1 if pair else None)) - - def encode(self, - text, - text_pair=None, - max_seq_len=512, - pad_to_max_seq_len=False, - truncation_strategy="longest_first", - return_position_ids=False, - return_token_type_ids=True, - return_attention_mask=False, - return_length=False, - return_overflowing_tokens=False, - return_special_tokens_mask=False): + token_ids_0, token_ids_1 if pair else None + ) + ) + + def encode( + self, + text, + text_pair=None, + max_seq_len=512, + pad_to_max_seq_len=False, + truncation_strategy="longest_first", + return_position_ids=False, + return_token_type_ids=True, + return_attention_mask=False, + return_length=False, + return_overflowing_tokens=False, + return_special_tokens_mask=False, + ): """ Performs tokenization and uses the tokenized tokens to prepare model inputs. It supports sequence or sequence pair as input, and batch input @@ -834,13 +895,17 @@ def get_input_ids(text): if isinstance(text, str): tokens = self._tokenize(text) return self.convert_tokens_to_ids(tokens) - elif isinstance(text, - (list, tuple)) and len(text) > 0 and isinstance( - text[0], str): + elif ( + isinstance(text, (list, tuple)) + and len(text) > 0 + and isinstance(text[0], str) + ): return self.convert_tokens_to_ids(text) - elif isinstance(text, - (list, tuple)) and len(text) > 0 and isinstance( - text[0], int): + elif ( + isinstance(text, (list, tuple)) + and len(text) > 0 + and isinstance(text[0], int) + ): return text else: raise ValueError( @@ -857,8 +922,9 @@ def get_input_ids(text): encoded_inputs = {} # Truncation: Handle max sequence length - total_len = len_ids + len_pair_ids + (self.num_special_tokens_to_add( - pair=pair)) + total_len = ( + len_ids + len_pair_ids + (self.num_special_tokens_to_add(pair=pair)) + ) if max_seq_len and total_len > max_seq_len: ids, pair_ids, overflowing_tokens = self.truncate_sequences( @@ -875,7 +941,8 @@ def get_input_ids(text): sequence = self.build_inputs_with_special_tokens(ids, pair_ids) token_type_ids = self.create_token_type_ids_from_sequences( - ids, pair_ids) + ids, pair_ids + ) # Build output dictionnary encoded_inputs["input_ids"] = sequence @@ -883,44 +950,53 @@ def get_input_ids(text): encoded_inputs["token_type_ids"] = token_type_ids if return_special_tokens_mask: encoded_inputs[ - "special_tokens_mask"] = self.get_special_tokens_mask( - ids, pair_ids) + "special_tokens_mask" + ] = self.get_special_tokens_mask(ids, pair_ids) if return_length: encoded_inputs["seq_len"] = len(encoded_inputs["input_ids"]) # Check lengths - assert max_seq_len is None or len( - encoded_inputs["input_ids"]) <= max_seq_len + assert ( + max_seq_len is None + or len(encoded_inputs["input_ids"]) <= max_seq_len + ) # Padding - needs_to_be_padded = pad_to_max_seq_len and \ - max_seq_len and len(encoded_inputs["input_ids"]) < max_seq_len + needs_to_be_padded = ( + pad_to_max_seq_len + and max_seq_len + and len(encoded_inputs["input_ids"]) < max_seq_len + ) if needs_to_be_padded: difference = max_seq_len - len(encoded_inputs["input_ids"]) if self.padding_side == 'right': if return_attention_mask: encoded_inputs["attention_mask"] = [1] * len( - encoded_inputs["input_ids"]) + [0] * difference + encoded_inputs["input_ids"] + ) + [0] * difference if return_token_type_ids: encoded_inputs["token_type_ids"] = ( - encoded_inputs["token_type_ids"] + - [self.pad_token_type_id] * difference) + encoded_inputs["token_type_ids"] + + [self.pad_token_type_id] * difference + ) if return_special_tokens_mask: - encoded_inputs["special_tokens_mask"] = encoded_inputs[ - "special_tokens_mask"] + [1] * difference - encoded_inputs["input_ids"] = encoded_inputs["input_ids"] + [ - self.pad_token_id - ] * difference + encoded_inputs["special_tokens_mask"] = ( + encoded_inputs["special_tokens_mask"] + [1] * difference + ) + encoded_inputs["input_ids"] = ( + encoded_inputs["input_ids"] + + [self.pad_token_id] * difference + ) elif self.padding_side == 'left': if return_attention_mask: encoded_inputs["attention_mask"] = [0] * difference + [ 1 ] * len(encoded_inputs["input_ids"]) if return_token_type_ids: - encoded_inputs["token_type_ids"] = ( - [self.pad_token_type_id] * difference + - encoded_inputs["token_type_ids"]) + encoded_inputs["token_type_ids"] = [ + self.pad_token_type_id + ] * difference + encoded_inputs["token_type_ids"] if return_special_tokens_mask: encoded_inputs["special_tokens_mask"] = [ 1 @@ -931,27 +1007,31 @@ def get_input_ids(text): else: if return_attention_mask: encoded_inputs["attention_mask"] = [1] * len( - encoded_inputs["input_ids"]) + encoded_inputs["input_ids"] + ) if return_position_ids: encoded_inputs["position_ids"] = list( - range(len(encoded_inputs["input_ids"]))) + range(len(encoded_inputs["input_ids"])) + ) return encoded_inputs - def batch_encode(self, - batch_text_or_text_pairs, - max_seq_len=512, - pad_to_max_seq_len=False, - stride=0, - is_split_into_words=False, - truncation_strategy="longest_first", - return_position_ids=False, - return_token_type_ids=True, - return_attention_mask=False, - return_length=False, - return_overflowing_tokens=False, - return_special_tokens_mask=False): + def batch_encode( + self, + batch_text_or_text_pairs, + max_seq_len=512, + pad_to_max_seq_len=False, + stride=0, + is_split_into_words=False, + truncation_strategy="longest_first", + return_position_ids=False, + return_token_type_ids=True, + return_attention_mask=False, + return_length=False, + return_overflowing_tokens=False, + return_special_tokens_mask=False, + ): """ Performs tokenization and uses the tokenized tokens to prepare model inputs. It supports batch inputs of sequence or sequence pair. @@ -1043,13 +1123,17 @@ def get_input_ids(text): if isinstance(text, str): tokens = self._tokenize(text) return self.convert_tokens_to_ids(tokens) - elif isinstance(text, - (list, tuple)) and len(text) > 0 and isinstance( - text[0], str): + elif ( + isinstance(text, (list, tuple)) + and len(text) > 0 + and isinstance(text[0], str) + ): return self.convert_tokens_to_ids(text) - elif isinstance(text, - (list, tuple)) and len(text) > 0 and isinstance( - text[0], int): + elif ( + isinstance(text, (list, tuple)) + and len(text) > 0 + and isinstance(text[0], int) + ): return text else: raise ValueError( @@ -1058,23 +1142,29 @@ def get_input_ids(text): batch_encode_inputs = [] for example_id, tokens_or_pair_tokens in enumerate( - batch_text_or_text_pairs): + batch_text_or_text_pairs + ): if not isinstance(tokens_or_pair_tokens, (list, tuple)): text, text_pair = tokens_or_pair_tokens, None elif is_split_into_words and not isinstance( - tokens_or_pair_tokens[0], (list, tuple)): + tokens_or_pair_tokens[0], (list, tuple) + ): text, text_pair = tokens_or_pair_tokens, None else: text, text_pair = tokens_or_pair_tokens first_ids = get_input_ids(text) - second_ids = get_input_ids( - text_pair) if text_pair is not None else None + second_ids = ( + get_input_ids(text_pair) if text_pair is not None else None + ) if stride > 0 and second_ids is not None: - max_len_for_pair = max_seq_len - len( - first_ids) - self.num_special_tokens_to_add(pair=True) + max_len_for_pair = ( + max_seq_len + - len(first_ids) + - self.num_special_tokens_to_add(pair=True) + ) token_offset_mapping = self.get_offset_mapping(text) token_pair_offset_mapping = self.get_offset_mapping(text_pair) @@ -1087,18 +1177,24 @@ def get_input_ids(text): length = max_len_for_pair ids = first_ids - pair_ids = second_ids[offset:offset + length] + pair_ids = second_ids[offset : offset + length] mapping = token_offset_mapping - pair_mapping = token_pair_offset_mapping[offset:offset + - length] - - offset_mapping = self.build_offset_mapping_with_special_tokens( - mapping, pair_mapping) + pair_mapping = token_pair_offset_mapping[ + offset : offset + length + ] + + offset_mapping = ( + self.build_offset_mapping_with_special_tokens( + mapping, pair_mapping + ) + ) sequence = self.build_inputs_with_special_tokens( - ids, pair_ids) + ids, pair_ids + ) token_type_ids = self.create_token_type_ids_from_sequences( - ids, pair_ids) + ids, pair_ids + ) # Build output dictionnary encoded_inputs["input_ids"] = sequence @@ -1106,25 +1202,32 @@ def get_input_ids(text): encoded_inputs["token_type_ids"] = token_type_ids if return_special_tokens_mask: encoded_inputs[ - "special_tokens_mask"] = self.get_special_tokens_mask( - ids, pair_ids) + "special_tokens_mask" + ] = self.get_special_tokens_mask(ids, pair_ids) if return_length: encoded_inputs["seq_len"] = len( - encoded_inputs["input_ids"]) + encoded_inputs["input_ids"] + ) # Check lengths - assert max_seq_len is None or len( - encoded_inputs["input_ids"]) <= max_seq_len + assert ( + max_seq_len is None + or len(encoded_inputs["input_ids"]) <= max_seq_len + ) # Padding - needs_to_be_padded = pad_to_max_seq_len and \ - max_seq_len and len(encoded_inputs["input_ids"]) < max_seq_len + needs_to_be_padded = ( + pad_to_max_seq_len + and max_seq_len + and len(encoded_inputs["input_ids"]) < max_seq_len + ) encoded_inputs['offset_mapping'] = offset_mapping if needs_to_be_padded: difference = max_seq_len - len( - encoded_inputs["input_ids"]) + encoded_inputs["input_ids"] + ) if self.padding_side == 'right': if return_attention_mask: encoded_inputs["attention_mask"] = [1] * len( @@ -1133,33 +1236,42 @@ def get_input_ids(text): if return_token_type_ids: # 0 for padding token mask encoded_inputs["token_type_ids"] = ( - encoded_inputs["token_type_ids"] + - [self.pad_token_type_id] * difference) + encoded_inputs["token_type_ids"] + + [self.pad_token_type_id] * difference + ) if return_special_tokens_mask: - encoded_inputs[ - "special_tokens_mask"] = encoded_inputs[ - "special_tokens_mask"] + [1 - ] * difference - encoded_inputs["input_ids"] = encoded_inputs[ - "input_ids"] + [self.pad_token_id] * difference - encoded_inputs['offset_mapping'] = encoded_inputs[ - 'offset_mapping'] + [(0, 0)] * difference + encoded_inputs["special_tokens_mask"] = ( + encoded_inputs["special_tokens_mask"] + + [1] * difference + ) + encoded_inputs["input_ids"] = ( + encoded_inputs["input_ids"] + + [self.pad_token_id] * difference + ) + encoded_inputs['offset_mapping'] = ( + encoded_inputs['offset_mapping'] + + [(0, 0)] * difference + ) elif self.padding_side == 'left': if return_attention_mask: encoded_inputs["attention_mask"] = [ 0 ] * difference + [1] * len( - encoded_inputs["input_ids"]) + encoded_inputs["input_ids"] + ) if return_token_type_ids: # 0 for padding token mask - encoded_inputs["token_type_ids"] = ( - [self.pad_token_type_id] * difference + - encoded_inputs["token_type_ids"]) + encoded_inputs["token_type_ids"] = [ + self.pad_token_type_id + ] * difference + encoded_inputs[ + "token_type_ids" + ] if return_special_tokens_mask: encoded_inputs["special_tokens_mask"] = [ 1 ] * difference + encoded_inputs[ - "special_tokens_mask"] + "special_tokens_mask" + ] encoded_inputs["input_ids"] = [ self.pad_token_id ] * difference + encoded_inputs["input_ids"] @@ -1169,11 +1281,13 @@ def get_input_ids(text): else: if return_attention_mask: encoded_inputs["attention_mask"] = [1] * len( - encoded_inputs["input_ids"]) + encoded_inputs["input_ids"] + ) if return_position_ids: encoded_inputs["position_ids"] = list( - range(len(encoded_inputs["input_ids"]))) + range(len(encoded_inputs["input_ids"])) + ) encoded_inputs['overflow_to_sample'] = example_id batch_encode_inputs.append(encoded_inputs) @@ -1194,7 +1308,9 @@ def get_input_ids(text): return_attention_mask=return_attention_mask, return_length=return_length, return_overflowing_tokens=return_overflowing_tokens, - return_special_tokens_mask=return_special_tokens_mask)) + return_special_tokens_mask=return_special_tokens_mask, + ) + ) return batch_encode_inputs @@ -1213,7 +1329,8 @@ def get_offset_mapping(self, text): for token in self.basic_tokenizer.tokenize(text): for sub_token in self.wordpiece_tokenizer.tokenize(token): split_tokens.append( - sub_token if sub_token != self.unk_token else token) + sub_token if sub_token != self.unk_token else token + ) normalized_text, char_mapping = '', [] @@ -1223,10 +1340,13 @@ def get_offset_mapping(self, text): ch = unicodedata.normalize('NFD', ch) ch = ''.join([c for c in ch if unicodedata.category(c) != 'Mn']) - ch = ''.join([ - c for c in ch - if not (ord(c) == 0 or ord(c) == 0xfffd or _is_control(c)) - ]) + ch = ''.join( + [ + c + for c in ch + if not (ord(c) == 0 or ord(c) == 0xFFFD or _is_control(c)) + ] + ) normalized_text += ch char_mapping.extend([i] * len(ch)) @@ -1241,7 +1361,8 @@ def get_offset_mapping(self, text): end = start + len(token) token_mapping.append( - (char_mapping[start], char_mapping[end - 1] + 1)) + (char_mapping[start], char_mapping[end - 1] + 1) + ) offset = end return token_mapping diff --git a/python/paddle/fluid/tests/unittests/transformer_model.py b/python/paddle/fluid/tests/unittests/transformer_model.py index 7fd02339ec910..2baf56caaf391 100644 --- a/python/paddle/fluid/tests/unittests/transformer_model.py +++ b/python/paddle/fluid/tests/unittests/transformer_model.py @@ -30,24 +30,33 @@ def position_encoding_init(n_position, d_pos_vec): """ Generate the initial values for the sinusoid position encoding table. """ - position_enc = np.array([[ - pos / np.power(10000, 2 * (j // 2) / d_pos_vec) - for j in range(d_pos_vec) - ] if pos != 0 else np.zeros(d_pos_vec) for pos in range(n_position)]) + position_enc = np.array( + [ + [ + pos / np.power(10000, 2 * (j // 2) / d_pos_vec) + for j in range(d_pos_vec) + ] + if pos != 0 + else np.zeros(d_pos_vec) + for pos in range(n_position) + ] + ) position_enc[1:, 0::2] = np.sin(position_enc[1:, 0::2]) # dim 2i position_enc[1:, 1::2] = np.cos(position_enc[1:, 1::2]) # dim 2i+1 return position_enc.astype("float32") -def multi_head_attention(queries, - keys, - values, - attn_bias, - d_key, - d_value, - d_model, - n_head=1, - dropout_rate=0.): +def multi_head_attention( + queries, + keys, + values, + attn_bias, + d_key, + d_value, + d_model, + n_head=1, + dropout_rate=0.0, +): """ Multi-Head Attention. Note that attn_bias is added to the logit before computing softmax activiation to mask certain selected positions so that @@ -55,36 +64,42 @@ def multi_head_attention(queries, """ if not (len(queries.shape) == len(keys.shape) == len(values.shape) == 3): raise ValueError( - "Inputs: queries, keys and values should all be 3-D tensors.") + "Inputs: queries, keys and values should all be 3-D tensors." + ) def __compute_qkv(queries, keys, values, n_head, d_key, d_value): """ Add linear projection to queries, keys, and values. """ - q = layers.fc(input=queries, - size=d_key * n_head, - param_attr=fluid.initializer.Xavier( - uniform=False, - fan_in=d_model * d_key, - fan_out=n_head * d_key), - bias_attr=False, - num_flatten_dims=2) - k = layers.fc(input=keys, - size=d_key * n_head, - param_attr=fluid.initializer.Xavier( - uniform=False, - fan_in=d_model * d_key, - fan_out=n_head * d_key), - bias_attr=False, - num_flatten_dims=2) - v = layers.fc(input=values, - size=d_value * n_head, - param_attr=fluid.initializer.Xavier( - uniform=False, - fan_in=d_model * d_value, - fan_out=n_head * d_value), - bias_attr=False, - num_flatten_dims=2) + q = layers.fc( + input=queries, + size=d_key * n_head, + param_attr=fluid.initializer.Xavier( + uniform=False, fan_in=d_model * d_key, fan_out=n_head * d_key + ), + bias_attr=False, + num_flatten_dims=2, + ) + k = layers.fc( + input=keys, + size=d_key * n_head, + param_attr=fluid.initializer.Xavier( + uniform=False, fan_in=d_model * d_key, fan_out=n_head * d_key + ), + bias_attr=False, + num_flatten_dims=2, + ) + v = layers.fc( + input=values, + size=d_value * n_head, + param_attr=fluid.initializer.Xavier( + uniform=False, + fan_in=d_model * d_value, + fan_out=n_head * d_value, + ), + bias_attr=False, + num_flatten_dims=2, + ) return q, k, v def __split_heads(x, n_head): @@ -100,7 +115,8 @@ def __split_heads(x, n_head): hidden_size = x.shape[-1] # FIXME(guosheng): Decouple the program desc with batch_size. reshaped = layers.reshape( - x=x, shape=[batch_size, -1, n_head, hidden_size // n_head]) + x=x, shape=[batch_size, -1, n_head, hidden_size // n_head] + ) # permute the dimensions into: # [batch_size, n_head, max_sequence_len, hidden_size_per_head] @@ -111,7 +127,8 @@ def __combine_heads(x): Transpose and then reshape the last two dimensions of input tensor x so that it becomes one dimension, which is reverse to __split_heads. """ - if len(x.shape) == 3: return x + if len(x.shape) == 3: + return x if len(x.shape) != 4: raise ValueError("Input(x) should be a 4-D Tensor.") @@ -120,8 +137,9 @@ def __combine_heads(x): return layers.reshape( x=trans_x, shape=list( - map(int, - [batch_size, -1, trans_x.shape[2] * trans_x.shape[3]]))) + map(int, [batch_size, -1, trans_x.shape[2] * trans_x.shape[3]]) + ), + ) def scaled_dot_product_attention(q, k, v, attn_bias, d_model, dropout_rate): """ @@ -146,9 +164,9 @@ def __softmax(x, eps=1e-9): product = layers.matmul(x=scaled_q, y=k, transpose_y=True) weights = __softmax(layers.elementwise_add(x=product, y=attn_bias)) if dropout_rate: - weights = layers.dropout(weights, - dropout_prob=dropout_rate, - is_test=False) + weights = layers.dropout( + weights, dropout_prob=dropout_rate, is_test=False + ) out = layers.matmul(weights, v) return out @@ -158,17 +176,20 @@ def __softmax(x, eps=1e-9): k = __split_heads(k, n_head) v = __split_heads(v, n_head) - ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_model, - dropout_rate) + ctx_multiheads = scaled_dot_product_attention( + q, k, v, attn_bias, d_model, dropout_rate + ) out = __combine_heads(ctx_multiheads) # Project back to the model size. - proj_out = layers.fc(input=out, - size=d_model, - param_attr=fluid.initializer.Xavier(uniform=False), - bias_attr=False, - num_flatten_dims=2) + proj_out = layers.fc( + input=out, + size=d_model, + param_attr=fluid.initializer.Xavier(uniform=False), + bias_attr=False, + num_flatten_dims=2, + ) return proj_out @@ -178,21 +199,27 @@ def positionwise_feed_forward(x, d_inner_hid, d_hid): This module consists of two linear transformations with a ReLU activation in between, which is applied to each position separately and identically. """ - hidden = layers.fc(input=x, - size=d_inner_hid, - num_flatten_dims=2, - param_attr=fluid.initializer.Uniform(low=-(d_hid**-0.5), - high=(d_hid**-0.5)), - act="relu") - out = layers.fc(input=hidden, - size=d_hid, - num_flatten_dims=2, - param_attr=fluid.initializer.Uniform( - low=-(d_inner_hid**-0.5), high=(d_inner_hid**-0.5))) + hidden = layers.fc( + input=x, + size=d_inner_hid, + num_flatten_dims=2, + param_attr=fluid.initializer.Uniform( + low=-(d_hid**-0.5), high=(d_hid**-0.5) + ), + act="relu", + ) + out = layers.fc( + input=hidden, + size=d_hid, + num_flatten_dims=2, + param_attr=fluid.initializer.Uniform( + low=-(d_inner_hid**-0.5), high=(d_inner_hid**-0.5) + ), + ) return out -def pre_post_process_layer(prev_out, out, process_cmd, dropout=0.): +def pre_post_process_layer(prev_out, out, process_cmd, dropout=0.0): """ Add residual connection, layer normalization and droput to the out tensor optionally according to the value of process_cmd. @@ -204,10 +231,12 @@ def pre_post_process_layer(prev_out, out, process_cmd, dropout=0.): if cmd == "a": # add residual connection out = out + prev_out if prev_out else out elif cmd == "n": # add layer normalization - out = layers.layer_norm(out, - begin_norm_axis=len(out.shape) - 1, - param_attr=fluid.initializer.Constant(1.), - bias_attr=fluid.initializer.Constant(0.)) + out = layers.layer_norm( + out, + begin_norm_axis=len(out.shape) - 1, + param_attr=fluid.initializer.Constant(1.0), + bias_attr=fluid.initializer.Constant(0.0), + ) elif cmd == "d": # add dropout if dropout: out = layers.dropout(out, dropout_prob=dropout, is_test=False) @@ -218,53 +247,65 @@ def pre_post_process_layer(prev_out, out, process_cmd, dropout=0.): post_process_layer = pre_post_process_layer -def prepare_encoder(src_word, - src_pos, - src_vocab_size, - src_emb_dim, - src_pad_idx, - src_max_len, - dropout=0., - pos_pad_idx=0, - pos_enc_param_name=None): +def prepare_encoder( + src_word, + src_pos, + src_vocab_size, + src_emb_dim, + src_pad_idx, + src_max_len, + dropout=0.0, + pos_pad_idx=0, + pos_enc_param_name=None, +): """Add word embeddings and position encodings. The output tensor has a shape of: [batch_size, max_src_length_in_batch, d_model]. This module is used at the bottom of the encoder stacks. """ - src_word_emb = layers.embedding(src_word, - size=[src_vocab_size, src_emb_dim], - padding_idx=src_pad_idx, - param_attr=fluid.initializer.Normal(0., 1.)) + src_word_emb = layers.embedding( + src_word, + size=[src_vocab_size, src_emb_dim], + padding_idx=src_pad_idx, + param_attr=fluid.initializer.Normal(0.0, 1.0), + ) src_pos_enc = layers.embedding( src_pos, size=[src_max_len, src_emb_dim], padding_idx=pos_pad_idx, - param_attr=fluid.ParamAttr(name=pos_enc_param_name, trainable=False)) + param_attr=fluid.ParamAttr(name=pos_enc_param_name, trainable=False), + ) src_pos_enc.stop_gradient = True enc_input = src_word_emb + src_pos_enc # FIXME(guosheng): Decouple the program desc with batch_size. enc_input = layers.reshape(x=enc_input, shape=[batch_size, -1, src_emb_dim]) - return layers.dropout(enc_input, dropout_prob=dropout, - is_test=False) if dropout else enc_input + return ( + layers.dropout(enc_input, dropout_prob=dropout, is_test=False) + if dropout + else enc_input + ) -prepare_encoder = partial(prepare_encoder, - pos_enc_param_name=pos_enc_param_names[0]) -prepare_decoder = partial(prepare_encoder, - pos_enc_param_name=pos_enc_param_names[1]) +prepare_encoder = partial( + prepare_encoder, pos_enc_param_name=pos_enc_param_names[0] +) +prepare_decoder = partial( + prepare_encoder, pos_enc_param_name=pos_enc_param_names[1] +) -def encoder_layer(enc_input, - attn_bias, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate=0.): +def encoder_layer( + enc_input, + attn_bias, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, +): """The encoder layers that can be stacked to form a deep encoder. This module consits of a multi-head (self) attention followed by @@ -272,46 +313,67 @@ def encoder_layer(enc_input, with the post_process_layer to add residual connection, layer normalization and droput. """ - attn_output = multi_head_attention(enc_input, enc_input, enc_input, - attn_bias, d_key, d_value, d_model, - n_head, dropout_rate) - attn_output = post_process_layer(enc_input, attn_output, "dan", - dropout_rate) + attn_output = multi_head_attention( + enc_input, + enc_input, + enc_input, + attn_bias, + d_key, + d_value, + d_model, + n_head, + dropout_rate, + ) + attn_output = post_process_layer( + enc_input, attn_output, "dan", dropout_rate + ) ffd_output = positionwise_feed_forward(attn_output, d_inner_hid, d_model) return post_process_layer(attn_output, ffd_output, "dan", dropout_rate) -def encoder(enc_input, +def encoder( + enc_input, + attn_bias, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, +): + """ + The encoder is composed of a stack of identical layers returned by calling + encoder_layer. + """ + for i in range(n_layer): + enc_output = encoder_layer( + enc_input, attn_bias, - n_layer, n_head, d_key, d_value, d_model, d_inner_hid, - dropout_rate=0.): - """ - The encoder is composed of a stack of identical layers returned by calling - encoder_layer. - """ - for i in range(n_layer): - enc_output = encoder_layer(enc_input, attn_bias, n_head, d_key, d_value, - d_model, d_inner_hid, dropout_rate) + dropout_rate, + ) enc_input = enc_output return enc_output -def decoder_layer(dec_input, - enc_output, - slf_attn_bias, - dec_enc_attn_bias, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate=0.): - """ The layer to be stacked in decoder part. +def decoder_layer( + dec_input, + enc_output, + slf_attn_bias, + dec_enc_attn_bias, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, +): + """The layer to be stacked in decoder part. The structure of this module is similar to that in the encoder part except a multi-head attention is added to implement encoder-decoder attention. @@ -364,17 +426,19 @@ def decoder_layer(dec_input, return dec_output -def decoder(dec_input, - enc_output, - dec_slf_attn_bias, - dec_enc_attn_bias, - n_layer, - n_head, - d_key, - d_value, - d_model, - d_inner_hid, - dropout_rate=0.): +def decoder( + dec_input, + enc_output, + dec_slf_attn_bias, + dec_enc_attn_bias, + n_layer, + n_head, + d_key, + d_value, + d_model, + d_inner_hid, + dropout_rate=0.0, +): """ The decoder is composed of a stack of identical decoder_layer layers. """ @@ -435,10 +499,10 @@ def build_inputs(max_length, n_head): all_inputs = [] for name, shape, dtype in zip(names, shapes, dtypes): all_inputs.append( - fluid.layers.data(name=name, - shape=shape, - dtype=dtype, - append_batch_size=False)) + fluid.layers.data( + name=name, shape=shape, dtype=dtype, append_batch_size=False + ) + ) return all_inputs @@ -458,8 +522,17 @@ def transformer( pos_pad_idx, ): - src_word, src_pos, trg_word, trg_pos, src_slf_attn_bias, trg_slf_attn_bias, trg_src_attn_bias, gold, weights = build_inputs( - max_length, n_head) + ( + src_word, + src_pos, + trg_word, + trg_pos, + src_slf_attn_bias, + trg_slf_attn_bias, + trg_src_attn_bias, + gold, + weights, + ) = build_inputs(max_length, n_head) enc_input = prepare_encoder( src_word, @@ -507,14 +580,17 @@ def transformer( # TODO(guosheng): Share the weight matrix between the embedding layers and # the pre-softmax linear transformation. - predict = layers.reshape(x=layers.fc( - input=dec_output, - size=trg_vocab_size, - param_attr=fluid.initializer.Xavier(uniform=False), - bias_attr=False, - num_flatten_dims=2), - shape=[-1, trg_vocab_size], - act="softmax") + predict = layers.reshape( + x=layers.fc( + input=dec_output, + size=trg_vocab_size, + param_attr=fluid.initializer.Xavier(uniform=False), + bias_attr=False, + num_flatten_dims=2, + ), + shape=[-1, trg_vocab_size], + act="softmax", + ) cost = layers.cross_entropy(input=predict, label=gold) weighted_cost = cost * weights diff --git a/python/paddle/fluid/tests/unittests/utils.py b/python/paddle/fluid/tests/unittests/utils.py index 346d40025837b..03993c2355eea 100644 --- a/python/paddle/fluid/tests/unittests/utils.py +++ b/python/paddle/fluid/tests/unittests/utils.py @@ -83,7 +83,6 @@ def _is_equal_program(prog1, prog2): def load_dygraph_vars_to_scope(model_path, scope, place): - def load_dict_to_scope(scope, dictionary): if scope is None: scope = fluid.global_scope() @@ -103,7 +102,6 @@ def load_dict_to_scope(scope, dictionary): class DyGraphProgramDescTracerTestHelper(object): - def __init__(self, unittest_obj): self.unittest_obj = unittest_obj diff --git a/python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py b/python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py index a3a3eca128cb0..730d77916216c 100644 --- a/python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py +++ b/python/paddle/fluid/tests/unittests/white_list/check_shape_white_list.py @@ -13,8 +13,21 @@ # limitations under the License. NEED_TO_FIX_OP_LIST = [ - 'fused_elemwise_activation', 'bilinear_tensor_product', 'conv2d_transpose', - 'depthwise_conv2d_transpose', 'grid_sampler', 'lstmp', 'margin_rank_loss', - 'matmul', 'scatter', 'soft_relu', 'squared_l2_distance', 'tree_conv', 'cvm', - 'cudnn_lstm', 'rnn', 'multi_dot', 'index_add' + 'fused_elemwise_activation', + 'bilinear_tensor_product', + 'conv2d_transpose', + 'depthwise_conv2d_transpose', + 'grid_sampler', + 'lstmp', + 'margin_rank_loss', + 'matmul', + 'scatter', + 'soft_relu', + 'squared_l2_distance', + 'tree_conv', + 'cvm', + 'cudnn_lstm', + 'rnn', + 'multi_dot', + 'index_add', ] diff --git a/python/paddle/fluid/tests/unittests/white_list/compile_vs_runtime_white_list.py b/python/paddle/fluid/tests/unittests/white_list/compile_vs_runtime_white_list.py index ee8202aa9f33e..a00c1a720aa5c 100644 --- a/python/paddle/fluid/tests/unittests/white_list/compile_vs_runtime_white_list.py +++ b/python/paddle/fluid/tests/unittests/white_list/compile_vs_runtime_white_list.py @@ -19,20 +19,20 @@ # reasons for skipping compile_vs_runtime test or be fixed later. COMPILE_RUN_OP_WHITE_LIST = [ - 'sequence_pool', \ - 'sequence_slice', \ - 'generate_proposals', \ - 'mine_hard_examples', \ - 'retinanet_detection_output', \ - 'ctc_align', \ - 'fusion_seqpool_cvm_concat', \ - 'gru', \ - 'rpn_target_assign', \ - 'retinanet_target_assign', \ - 'filter_by_instag', \ - 'im2sequence', \ - 'generate_proposal_labels', \ - 'detection_map', \ - 'locality_aware_nms', \ - 'var_conv_2d' + 'sequence_pool', + 'sequence_slice', + 'generate_proposals', + 'mine_hard_examples', + 'retinanet_detection_output', + 'ctc_align', + 'fusion_seqpool_cvm_concat', + 'gru', + 'rpn_target_assign', + 'retinanet_target_assign', + 'filter_by_instag', + 'im2sequence', + 'generate_proposal_labels', + 'detection_map', + 'locality_aware_nms', + 'var_conv_2d', ] diff --git a/python/paddle/fluid/tests/unittests/white_list/no_grad_set_white_list.py b/python/paddle/fluid/tests/unittests/white_list/no_grad_set_white_list.py index fb1cd35c45380..33960cf4c64d3 100644 --- a/python/paddle/fluid/tests/unittests/white_list/no_grad_set_white_list.py +++ b/python/paddle/fluid/tests/unittests/white_list/no_grad_set_white_list.py @@ -17,7 +17,6 @@ # TODO(Shixiaowei02): Check if the items do not need fix. # no_grad_set has value in NEED_TO_FIX_OP_LIST -# yapf: disable NEED_TO_FIX_OP_LIST = [ 'affine_channel', 'affine_grid', @@ -72,4 +71,3 @@ 'spectral_norm', 'complex', ] -# yapf: enable diff --git a/python/paddle/fluid/tests/unittests/white_list/op_accuracy_white_list.py b/python/paddle/fluid/tests/unittests/white_list/op_accuracy_white_list.py index 9b57b0d82471c..ba93c5bb874a3 100644 --- a/python/paddle/fluid/tests/unittests/white_list/op_accuracy_white_list.py +++ b/python/paddle/fluid/tests/unittests/white_list/op_accuracy_white_list.py @@ -14,76 +14,76 @@ # For op in NO_FP64_CHECK_GRAD_OP_LIST, the op test requires check_grad with fp64 precision NO_FP64_CHECK_GRAD_OP_LIST = [ - 'affine_grid', \ - 'clip', \ - 'conv2d', \ - 'conv2d_transpose', \ - 'conv3d', \ - 'conv3d_transpose', \ - 'conv_shift', \ - 'cos_sim', \ - 'cudnn_lstm', \ - 'cvm', \ - 'data_norm', \ - 'deformable_conv', \ - 'deformable_conv_v1', \ - 'deformable_psroi_pooling', \ - 'depthwise_conv2d', \ - 'depthwise_conv2d_transpose', \ - 'dropout', \ - 'fused_elemwise_activation', \ - 'hinge_loss', \ - 'huber_loss', \ - 'im2sequence', \ - 'increment', \ - 'l1_norm', \ - 'log_loss', \ - 'lrn', \ - 'margin_rank_loss', \ - 'match_matrix_tensor', \ - 'matmul', \ - 'max_pool2d_with_index', \ - 'max_pool3d_with_index', \ - 'minus', \ - 'modified_huber_loss', \ - 'nce', \ - 'pool2d', \ - 'pool3d', \ - 'prroi_pool', \ - 'rank_loss', \ - 'reduce_max', \ - 'reduce_min', \ - 'reshape2', \ - 'roi_perspective_transform', \ - 'row_conv', \ - 'scatter', \ - 'sequence_conv', \ - 'sequence_pool', \ - 'sequence_reverse', \ - 'sequence_slice', \ - 'sequence_topk_avg_pooling', \ - 'shuffle_channel', \ - 'sigmoid', \ - 'smooth_l1_loss', \ - 'softmax', \ - 'spectral_norm', \ - 'squared_l2_distance', \ - 'squared_l2_norm', \ - 'tanh', \ - 'mish', \ - 'transpose2', \ - 'trilinear_interp', \ - 'trilinear_interp_v2', \ - 'var_conv_2d', \ - 'warpctc', \ - 'bilateral_slice', \ - 'cast' + 'affine_grid', + 'clip', + 'conv2d', + 'conv2d_transpose', + 'conv3d', + 'conv3d_transpose', + 'conv_shift', + 'cos_sim', + 'cudnn_lstm', + 'cvm', + 'data_norm', + 'deformable_conv', + 'deformable_conv_v1', + 'deformable_psroi_pooling', + 'depthwise_conv2d', + 'depthwise_conv2d_transpose', + 'dropout', + 'fused_elemwise_activation', + 'hinge_loss', + 'huber_loss', + 'im2sequence', + 'increment', + 'l1_norm', + 'log_loss', + 'lrn', + 'margin_rank_loss', + 'match_matrix_tensor', + 'matmul', + 'max_pool2d_with_index', + 'max_pool3d_with_index', + 'minus', + 'modified_huber_loss', + 'nce', + 'pool2d', + 'pool3d', + 'prroi_pool', + 'rank_loss', + 'reduce_max', + 'reduce_min', + 'reshape2', + 'roi_perspective_transform', + 'row_conv', + 'scatter', + 'sequence_conv', + 'sequence_pool', + 'sequence_reverse', + 'sequence_slice', + 'sequence_topk_avg_pooling', + 'shuffle_channel', + 'sigmoid', + 'smooth_l1_loss', + 'softmax', + 'spectral_norm', + 'squared_l2_distance', + 'squared_l2_norm', + 'tanh', + 'mish', + 'transpose2', + 'trilinear_interp', + 'trilinear_interp_v2', + 'var_conv_2d', + 'warpctc', + 'bilateral_slice', + 'cast', ] NO_FP16_CHECK_GRAD_OP_LIST = [ - 'fused_elemwise_activation', \ - 'pool2d', \ - 'pool3d', \ - 'softmax',\ - 'conv2d_transpose' + 'fused_elemwise_activation', + 'pool2d', + 'pool3d', + 'softmax', + 'conv2d_transpose', ] diff --git a/python/paddle/fluid/tests/unittests/white_list/op_threshold_white_list.py b/python/paddle/fluid/tests/unittests/white_list/op_threshold_white_list.py index 91731c1dd0b21..22bc42d969468 100644 --- a/python/paddle/fluid/tests/unittests/white_list/op_threshold_white_list.py +++ b/python/paddle/fluid/tests/unittests/white_list/op_threshold_white_list.py @@ -13,47 +13,48 @@ # limitations under the License. NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST = [ - 'affine_channel', \ - 'bilinear_interp', \ - 'bilinear_interp_v2',\ - 'bilinear_tensor_product', \ - 'conv2d', \ - 'conv3d', \ - 'cross_entropy', \ - 'depthwise_conv2d_transpose', \ - 'grid_sampler', \ - 'group_norm', \ - 'gru', \ - 'gru_unit', \ - 'kldiv_loss', \ - 'lstm', \ - 'lstmp', \ - 'max_pool2d_with_index', \ - 'max_pool3d_with_index', \ - 'norm', \ - 'pool3d', \ - 'reduce_prod', \ - 'selu', \ - 'sigmoid_cross_entropy_with_logits', \ - 'soft_relu', \ - 'softmax_with_cross_entropy', \ - 'spp', \ - 'teacher_student_sigmoid_loss', \ - 'unpool', \ - 'yolov3_loss', \ - 'inverse', \ - 'bilateral_slice',\ - 'cudnn_lstm', \ - 'rnn', \ - 'lgamma', \ - 'sparse_attention', \ - 'svd', \ - 'matrix_power', \ - 'cholesky_solve', \ - 'solve', \ - 'qr', \ + 'affine_channel', + 'bilinear_interp', + 'bilinear_interp_v2', + 'bilinear_tensor_product', + 'conv2d', + 'conv3d', + 'cross_entropy', + 'depthwise_conv2d_transpose', + 'grid_sampler', + 'group_norm', + 'gru', + 'gru_unit', + 'kldiv_loss', + 'lstm', + 'lstmp', + 'max_pool2d_with_index', + 'max_pool3d_with_index', + 'norm', + 'pool3d', + 'reduce_prod', + 'selu', + 'sigmoid_cross_entropy_with_logits', + 'soft_relu', + 'softmax_with_cross_entropy', + 'spp', + 'teacher_student_sigmoid_loss', + 'unpool', + 'yolov3_loss', + 'inverse', + 'bilateral_slice', + 'cudnn_lstm', + 'rnn', + 'lgamma', + 'sparse_attention', + 'svd', + 'matrix_power', + 'cholesky_solve', + 'solve', + 'qr', ] -NEED_FIX_FP64_CHECK_OUTPUT_THRESHOLD_OP_LIST = ['bilinear_interp',\ - 'bilinear_interp_v2' - ] +NEED_FIX_FP64_CHECK_OUTPUT_THRESHOLD_OP_LIST = [ + 'bilinear_interp', + 'bilinear_interp_v2', +] diff --git a/python/paddle/fluid/tests/unittests/xpu/collective_allgather_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/collective_allgather_op_xpu.py index c2c2009a1f21f..e42c5144a7b7c 100644 --- a/python/paddle/fluid/tests/unittests/xpu/collective_allgather_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/collective_allgather_op_xpu.py @@ -22,7 +22,6 @@ class TestCollectiveAllGather(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,26 +29,28 @@ def get_model(self, main_prog, startup_program): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofgather", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_allgather", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_allgather", + inputs={'X': tindata}, + attrs={'ring_id': ring_id, 'nranks': nranks}, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/xpu/collective_allreduce_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/collective_allreduce_op_xpu.py index 07ea05d0ece8e..889eecf6327de 100644 --- a/python/paddle/fluid/tests/unittests/xpu/collective_allreduce_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/collective_allreduce_op_xpu.py @@ -23,32 +23,36 @@ class TestCollectiveAllReduce(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 def get_model(self, main_prog, startup_program): ring_id = 0 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofreduce", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_allreduce_sum", - inputs={'X': tindata}, - attrs={ - 'ring_id': ring_id, - }, - outputs={'Out': toutdata}) - main_prog.global_block().append_op(type="c_sync_comm_stream", - inputs={'X': toutdata}, - outputs={'Out': toutdata}, - attrs={'ring_id': ring_id}) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_allreduce_sum", + inputs={'X': tindata}, + attrs={ + 'ring_id': ring_id, + }, + outputs={'Out': toutdata}, + ) + main_prog.global_block().append_op( + type="c_sync_comm_stream", + inputs={'X': toutdata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/xpu/collective_identity_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/collective_identity_op_xpu.py index 6737ab772d987..967d4fcc70a8f 100644 --- a/python/paddle/fluid/tests/unittests/xpu/collective_identity_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/collective_identity_op_xpu.py @@ -22,7 +22,6 @@ class TestCollectiveIdentity(TestCollectiveRunnerBase): - def __init__(self): self.global_ring_id = 0 @@ -30,22 +29,22 @@ def get_model(self, main_prog, startup_program): ring_id = 0 nranks = 2 with fluid.program_guard(main_prog, startup_program): - tindata = layers.data(name="tindata", - shape=[10, 1000], - dtype='float32') + tindata = layers.data( + name="tindata", shape=[10, 1000], dtype='float32' + ) toutdata = main_prog.current_block().create_var( name="outofgather", dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) - main_prog.global_block().append_op(type="c_identity", - inputs={'X': tindata}, - outputs={'Out': toutdata}, - attrs={ - 'ring_id': ring_id, - 'nranks': nranks - }) + stop_gradient=False, + ) + main_prog.global_block().append_op( + type="c_identity", + inputs={'X': tindata}, + outputs={'Out': toutdata}, + attrs={'ring_id': ring_id, 'nranks': nranks}, + ) return toutdata diff --git a/python/paddle/fluid/tests/unittests/xpu/get_test_cover_info.py b/python/paddle/fluid/tests/unittests/xpu/get_test_cover_info.py index c0129fabe28b5..f1276f765a87e 100644 --- a/python/paddle/fluid/tests/unittests/xpu/get_test_cover_info.py +++ b/python/paddle/fluid/tests/unittests/xpu/get_test_cover_info.py @@ -99,7 +99,6 @@ class XPUOpTestWrapper(object): - def create_classes(self): base_class = None classes = [] @@ -110,7 +109,8 @@ def get_op_white_list(): op_white_list = xpu_test_op_white_list if os.getenv('XPU_TEST_OP_WHITE_LIST') is not None: op_white_list.extend( - os.getenv('XPU_TEST_OP_WHITE_LIST').strip().split(',')) + os.getenv('XPU_TEST_OP_WHITE_LIST').strip().split(',') + ) return list(set(op_white_list)) @@ -126,10 +126,13 @@ def get_type_white_list(): else: xpu2_type_white_list.append(t_type) - type_white_list = xpu1_type_white_list if version_str == "xpu1" else xpu2_type_white_list + type_white_list = ( + xpu1_type_white_list if version_str == "xpu1" else xpu2_type_white_list + ) if os.getenv('XPU_TEST_TYPE_WHITE_LIST') is not None: type_white_list.extend( - os.getenv('XPU_TEST_TYPE_WHITE_LIST').strip().split(',')) + os.getenv('XPU_TEST_TYPE_WHITE_LIST').strip().split(',') + ) return list(set(type_white_list)) @@ -137,7 +140,8 @@ def get_op_type_white_list(): op_type_white_list = xpu_test_op_type_white_list if os.getenv('XPU_TEST_OP_TYPE_WHITE_LIST') is not None: op_type_white_list.extend( - os.getenv('XPU_TEST_OP_TYPE_WHITE_LIST').strip().split(',')) + os.getenv('XPU_TEST_OP_TYPE_WHITE_LIST').strip().split(',') + ) return list(set(op_type_white_list)) @@ -145,7 +149,8 @@ def get_device_op_white_list(): device_op_white_list = xpu_test_device_op_white_list if os.getenv('XPU_TEST_DEVICE_OP_WHITE_LIST') is not None: device_op_white_list.extend( - os.getenv('XPU_TEST_DEVICE_OP_WHITE_LIST').strip().split(',')) + os.getenv('XPU_TEST_DEVICE_OP_WHITE_LIST').strip().split(',') + ) return list(set(device_op_white_list)) @@ -153,7 +158,8 @@ def get_device_op_type_white_list(): device_op_type_white_list = xpu_test_device_op_type_white_list if os.getenv('XPU_TEST_DEVICE_OP_TYPE_WHITE_LIST') is not None: device_op_type_white_list.extend( - os.getenv('XPU_TEST_DEVICE_OP_TYPE_WHITE_LIST').strip().split(',')) + os.getenv('XPU_TEST_DEVICE_OP_TYPE_WHITE_LIST').strip().split(',') + ) return list(set(device_op_type_white_list)) @@ -180,13 +186,15 @@ def make_xpu_op_list(xpu_version): if op_type == paddle.bfloat16: op_type = paddle.bfloat16 - if type_dict_paddle_to_str[ - op_type] in type_white_list or op_type not in type_dict_paddle_to_str.keys( - ): + if ( + type_dict_paddle_to_str[op_type] in type_white_list + or op_type not in type_dict_paddle_to_str.keys() + ): continue - device_op_type_name = device_op_name + '_' + type_dict_paddle_to_str[ - op_type] + device_op_type_name = ( + device_op_name + '_' + type_dict_paddle_to_str[op_type] + ) if device_op_type_name in device_op_type_white_list: continue @@ -201,12 +209,14 @@ def make_xpu_op_list(xpu_version): def get_xpu_op_support_types(op_name, dev_id=0): xpu_version = core.get_xpu_device_version(dev_id) support_type_list = core.get_xpu_device_op_support_types( - op_name, xpu_version) + op_name, xpu_version + ) support_type_str_list = [] for stype in support_type_list: if stype == paddle.bfloat16: support_type_str_list.append( - type_dict_paddle_to_str[paddle.bfloat16]) + type_dict_paddle_to_str[paddle.bfloat16] + ) else: support_type_str_list.append(type_dict_paddle_to_str[stype]) ops = make_xpu_op_list(xpu_version) @@ -243,12 +253,14 @@ def is_empty_grad_op_type(xpu_version, op, test_type): return False -def create_test_class(func_globals, - test_class, - test_type, - test_grad=True, - ignore_device_version=[], - test_device_version=[]): +def create_test_class( + func_globals, + test_class, + test_type, + test_grad=True, + ignore_device_version=[], + test_device_version=[], +): xpu_version = core.get_xpu_device_version(0) if xpu_version in ignore_device_version: return @@ -267,14 +279,19 @@ def create_test_class(func_globals, class_obj = test_class[1] cls_name = "{0}_{1}".format(test_class[0], str(test_type)) func_globals[cls_name] = type( - cls_name, (class_obj, ), { + cls_name, + (class_obj,), + { 'in_type': type_dict_str_to_numpy[test_type], 'in_type_str': test_type, - 'op_type_need_check_grad': True - }) - - if hasattr(test_class_obj, 'use_dynamic_create_class' - ) and test_class_obj.use_dynamic_create_class: + 'op_type_need_check_grad': True, + }, + ) + + if ( + hasattr(test_class_obj, 'use_dynamic_create_class') + and test_class_obj.use_dynamic_create_class + ): base_class, dynamic_classes = test_class_obj.dynamic_create_class() for dy_class in dynamic_classes: cls_name = "{0}_{1}".format(dy_class[0], str(test_type)) @@ -282,7 +299,7 @@ def create_test_class(func_globals, attr_dict['in_type'] = type_dict_str_to_numpy[test_type] attr_dict['in_type_str'] = test_type attr_dict['op_type_need_check_grad'] = True - func_globals[cls_name] = type(cls_name, (base_class, ), attr_dict) + func_globals[cls_name] = type(cls_name, (base_class,), attr_dict) record_op_test(op_name, test_type) if not no_grad: @@ -309,9 +326,12 @@ def get_test_cover_info(): total_len = len(set(xpu_op_list)) covered_len = len(set(xpu_op_covered)) print('{} test: {}/{}'.format(version_str, covered_len, total_len)) - if (len(diff_list) != 0): - print("These ops need to be tested on {0}! ops:{1}".format( - version_str, ','.join(diff_list))) + if len(diff_list) != 0: + print( + "These ops need to be tested on {0}! ops:{1}".format( + version_str, ','.join(diff_list) + ) + ) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/xpu/test_accuracy_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_accuracy_op_xpu.py index 495f1d4fbb606..760a45055ad38 100755 --- a/python/paddle/fluid/tests/unittests/xpu/test_accuracy_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_accuracy_op_xpu.py @@ -20,19 +20,21 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestAccuracyOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'accuracy' self.use_dynamic_create_class = False class TestXPUAccuracyOp(XPUOpTest): - def setUp(self): self.op_type = "accuracy" self.init_dtype() @@ -48,10 +50,11 @@ def setUp(self): num_correct += 1 break self.outputs = { - 'Accuracy': - np.array([num_correct / float(n)]).astype(self.dtype), + 'Accuracy': np.array([num_correct / float(n)]).astype( + self.dtype + ), 'Correct': np.array([num_correct]).astype("int32"), - 'Total': np.array([n]).astype("int32") + 'Total': np.array([n]).astype("int32"), } self.attrs = {'use_xpu': True} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_activation_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_activation_op_xpu.py index aac695e74c911..49b673133fcaf 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_activation_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_activation_op_xpu.py @@ -22,13 +22,16 @@ from op_test import OpTest from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class TestActivationOPBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -57,13 +60,11 @@ def test_check_grad(self): class XPUTestExpOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'exp' self.use_dynamic_create_class = False class XPUTestExp(TestActivationOPBase): - def set_case(self): self.op_type = 'exp' self.dtype = self.in_type @@ -81,13 +82,11 @@ def set_case(self): class XPUTestSigmoidOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'sigmoid' self.use_dynamic_create_class = False class XPUTestSigmoid(TestActivationOPBase): - def set_case(self): self.op_type = "sigmoid" self.dtype = self.in_type @@ -102,25 +101,22 @@ def init_config(self): self.x = np.random.uniform(-1, 1, [11, 17]).astype(self.dtype) class XPUTestSigmoid2(XPUTestSigmoid): - def init_config(self): self.x = np.random.uniform(-2, 2, [100]).astype(self.dtype) class XPUTestSigmoid3(XPUTestSigmoid): - def init_config(self): self.x = np.random.uniform(-2, 2, [10, 12, 15]).astype(self.dtype) class XPUTestSigmoid4(XPUTestSigmoid): - def init_config(self): self.x = np.random.uniform(-2, 2, [19, 19]).astype(self.dtype) class XPUTestSigmoid5(XPUTestSigmoid): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [10, 20, 30, 40]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [10, 20, 30, 40]).astype( + self.dtype + ) support_types = get_xpu_op_support_types('sigmoid') @@ -129,13 +125,11 @@ def init_config(self): class XPUTestTanhOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'tanh' self.use_dynamic_create_class = False class XPUTestTanh(TestActivationOPBase): - def set_case(self): self.op_type = "tanh" self.dtype = self.in_type @@ -153,13 +147,11 @@ def set_case(self): class XPUTestSqrtOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'sqrt' self.use_dynamic_create_class = False class XPUTestSqrt(TestActivationOPBase): - def set_case(self): self.op_type = "sqrt" self.dtype = self.in_type @@ -178,13 +170,11 @@ def set_case(self): class XPUTestAbsOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'abs' self.use_dynamic_create_class = False class XPUTestAbs(TestActivationOPBase): - def set_case(self): self.op_type = "abs" self.dtype = self.in_type @@ -208,13 +198,11 @@ def set_case(self): class XPUTestReluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'relu' self.use_dynamic_create_class = False class XPUTestRelu(TestActivationOPBase): - def set_case(self): self.op_type = "relu" self.dtype = self.in_type @@ -235,13 +223,11 @@ def set_case(self): class XPUTestGeluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'gelu' self.use_dynamic_create_class = False class XPUTestGelu(TestActivationOPBase): - def set_case(self): self.op_type = "gelu" self.dtype = self.in_type @@ -262,22 +248,27 @@ def set_case(self): def gelu(x, approximate): from scipy.special import erf + if approximate: - y_ref = 0.5 * x * ( - 1.0 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)))) + y_ref = ( + 0.5 + * x + * ( + 1.0 + + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3))) + ) + ) else: y_ref = 0.5 * x * (1 + erf(x / np.sqrt(2))) return y_ref.astype(x.dtype) class XPUTestHardSwishOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'hard_swish' self.use_dynamic_create_class = False class XPUTestHardSwish(TestActivationOPBase): - def set_case(self): self.op_type = "hard_swish" self.dtype = self.in_type @@ -304,13 +295,11 @@ def hard_swish(x, offset, threshold, scale): class XPUTestLogOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'log' self.use_dynamic_create_class = False class XPUTestLog(TestActivationOPBase): - def set_case(self): self.op_type = "log" self.dtype = self.in_type @@ -322,22 +311,18 @@ def set_case(self): self.outputs = {'Out': out} class TestLogCase1(XPUTestLog): - def set_shape(self): self.shape = [1, 11, 17] class TestLogCase2(XPUTestLog): - def set_shape(self): self.shape = [2, 2, 2] class TestLogCase3(XPUTestLog): - def set_shape(self): self.shape = [2] class TestLogCase4(XPUTestLog): - def set_shape(self): self.shape = [1, 2, 3, 4] @@ -348,13 +333,11 @@ def set_shape(self): class XPUTestSquareOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'square' self.use_dynamic_create_class = False class XPUTestSquare(TestActivationOPBase): - def set_case(self): self.op_type = "square" self.dtype = self.in_type @@ -369,22 +352,18 @@ def init_config(self): self.x = np.random.uniform(-1, 1, [11, 17]).astype(self.dtype) class XPUTestSquare2(XPUTestSquare): - def init_config(self): self.x = np.random.uniform(-2, 2, [100]).astype(self.dtype) class XPUTestSquare3(XPUTestSquare): - def init_config(self): self.x = np.random.uniform(-2, 2, [1, 15, 19]).astype(self.dtype) class XPUTestSquare4(XPUTestSquare): - def init_config(self): self.x = np.random.uniform(-2, 2, [100, 10]).astype(self.dtype) class XPUTestSquare5(XPUTestSquare): - def init_config(self): self.x = np.random.uniform(-2, 2, [1, 2, 5, 17]).astype(self.dtype) @@ -395,13 +374,11 @@ def init_config(self): class XPUTestPowOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'pow' self.use_dynamic_create_class = False class XPUTestPowBase(TestActivationOPBase): - def set_case(self): self.op_type = "pow" self.dtype = self.in_type @@ -418,40 +395,37 @@ def init_config(self): self.factor = 3.0 class XPUTestPow1(XPUTestPowBase): - def init_config(self): self.x = np.random.uniform(-1, 1, [1024, 8]).astype(self.dtype) self.factor = 1 class XPUTestPow2(XPUTestPowBase): - def init_config(self): self.x = np.random.uniform(-1, 1, [1024, 8]).astype(self.dtype) self.factor = 2 class XPUTestPow3(XPUTestPowBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 512, 15, 15]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 512, 15, 15]).astype( + self.dtype + ) self.factor = 3 class XPUTestPow4(XPUTestPowBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 256, 22, 22]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 256, 22, 22]).astype( + self.dtype + ) self.factor = 4 class XPUTestPow5(XPUTestPowBase): - def init_config(self): - self.x = np.random.uniform(0, 1, - [4, 256, 22, 22]).astype(self.dtype) + self.x = np.random.uniform(0, 1, [4, 256, 22, 22]).astype( + self.dtype + ) self.factor = 1.2 class XPUTestPow6(XPUTestPowBase): - def init_config(self): self.x = np.random.uniform(0, 1, [1024, 8]).astype(self.dtype) self.factor = 3.2 @@ -463,13 +437,11 @@ def init_config(self): class XPUTestLeakyReluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'leaky_relu' self.use_dynamic_create_class = False class XPUTestLeakyRelu(TestActivationOPBase): - def set_case(self): self.op_type = "leaky_relu" self.dtype = self.in_type @@ -492,7 +464,7 @@ def set_case(self): def leaky_relu(x, alpha): - if (alpha < 1): + if alpha < 1: y_ref = np.maximum(x, alpha * x) else: y_ref = np.minimum(x, alpha * x) @@ -500,13 +472,11 @@ def leaky_relu(x, alpha): class XPUTestReciprocalOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reciprocal' self.use_dynamic_create_class = False class XPUTestRecipocal(TestActivationOPBase): - def set_case(self): self.op_type = "reciprocal" self.dtype = self.in_type @@ -526,13 +496,11 @@ def set_case(self): class XPUTestSoftPlusOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'softplus' self.use_dynamic_create_class = False class XPUTestSoftPlusBase(TestActivationOPBase): - def set_case(self): self.op_type = "softplus" self.dtype = self.in_type @@ -550,21 +518,20 @@ def init_config(self): self.x = np.random.uniform(-1, 1, [11, 17]).astype(self.dtype) class XPUTestSoftPlus2(XPUTestSoftPlusBase): - def init_config(self): self.x = np.random.uniform(-2, 2, [1024, 8]).astype(self.dtype) class XPUTestSoftPlus3(XPUTestSoftPlusBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 512, 15, 15]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 512, 15, 15]).astype( + self.dtype + ) class XPUTestSoftPlus4(XPUTestSoftPlusBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 256, 22, 22]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 256, 22, 22]).astype( + self.dtype + ) support_types = get_xpu_op_support_types('softplus') @@ -574,20 +541,20 @@ def init_config(self): def ref_softplus(x, beta=1, threshold=20): x_beta = beta * x - out = np.select([x_beta <= threshold, x_beta > threshold], - [np.log(1 + np.exp(x_beta)) / beta, x]) + out = np.select( + [x_beta <= threshold, x_beta > threshold], + [np.log(1 + np.exp(x_beta)) / beta, x], + ) return out # XPU_KP unittests, these ops can be found from xpu_op_kpfirst_list.h class XPUTestBReluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'brelu' self.use_dynamic_create_class = False class XPUTestBRelu(TestActivationOPBase): - def set_case(self): self.op_type = "brelu" self.dtype = self.in_type @@ -614,13 +581,11 @@ def set_case(self): class XPUTestCeilOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'ceil' self.use_dynamic_create_class = False class XPUTestCeil(TestActivationOPBase): - def set_case(self): self.op_type = "ceil" self.dtype = self.in_type @@ -640,13 +605,11 @@ def set_case(self): class XPUTestCeluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'celu' self.use_dynamic_create_class = False class XPUTestCelu(TestActivationOPBase): - def set_case(self): self.op_type = "celu" self.dtype = self.in_type @@ -671,18 +634,16 @@ def ref_celu(x, alpha): class XPUTestEluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elu' self.use_dynamic_create_class = False class XPUTestElu(TestActivationOPBase): - def set_case(self): self.op_type = "elu" self.dtype = self.in_type - alpha = 1. + alpha = 1.0 x = np.random.uniform(-3, 3, [10, 12]).astype(self.dtype) out = ref_elu(x, alpha) @@ -702,13 +663,11 @@ def ref_elu(x, alpha): class XPUTestFloorOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'floor' self.use_dynamic_create_class = False class XPUTestFloor(TestActivationOPBase): - def set_case(self): self.op_type = "floor" self.dtype = self.in_type @@ -728,13 +687,11 @@ def set_case(self): class XPUTestHardShrinkOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'hard_shrink' self.use_dynamic_create_class = False class XPUTestHardShrink(TestActivationOPBase): - def set_case(self): self.op_type = "hard_shrink" self.dtype = self.in_type @@ -762,13 +719,11 @@ def ref_hardshrink(x, threshold): class XPUTestHardSigmoidOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'hard_sigmoid' self.use_dynamic_create_class = False class XPUTestHardSigmoid(TestActivationOPBase): - def set_case(self): self.op_type = "hard_sigmoid" self.dtype = self.in_type @@ -777,7 +732,7 @@ def set_case(self): x = np.random.uniform(-5, 5, [10, 12]).astype(self.dtype) lower_threshold = -self.offset / self.slope - upper_threshold = (1. - self.offset) / self.slope + upper_threshold = (1.0 - self.offset) / self.slope # Same reason as TestAbs delta = 0.005 @@ -789,7 +744,7 @@ def set_case(self): self.attrs = { 'use_xpu': True, 'slope': self.slope, - 'offset': self.offset + 'offset': self.offset, } self.inputs = {'X': x} self.outputs = {'Out': out} @@ -801,17 +756,15 @@ def set_case(self): def ref_hardsigmoid(x, slope=0.166666666666667, offset=0.5): - return np.maximum(np.minimum(x * slope + offset, 1.), 0.).astype(x.dtype) + return np.maximum(np.minimum(x * slope + offset, 1.0), 0.0).astype(x.dtype) class XPUTestLog1pOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'log1p' self.use_dynamic_create_class = False class XPUTestLog1p(TestActivationOPBase): - def set_case(self): self.op_type = "log1p" self.dtype = self.in_type @@ -831,13 +784,11 @@ def set_case(self): class XPUTestLogsigmoidOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'logsigmoid' self.use_dynamic_create_class = False class XPUTestLogsigmoid(TestActivationOPBase): - def set_case(self): self.op_type = "logsigmoid" self.dtype = self.in_type @@ -857,13 +808,11 @@ def set_case(self): class XPUTestRelu6OP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'relu6' self.use_dynamic_create_class = False class XPUTestRelu6(TestActivationOPBase): - def set_case(self): self.op_type = "relu6" self.dtype = self.in_type @@ -891,13 +840,11 @@ def ref_relu6(x, threshold=6.0): class XPUTestSiluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'silu' self.use_dynamic_create_class = False class XPUTestSilu(TestActivationOPBase): - def set_case(self): self.op_type = "silu" self.dtype = self.in_type @@ -917,13 +864,11 @@ def set_case(self): class XPUTestSoftReluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'soft_relu' self.use_dynamic_create_class = False class XPUTestSoftRelu(TestActivationOPBase): - def set_case(self): self.op_type = "soft_relu" self.dtype = self.in_type @@ -950,13 +895,11 @@ def set_case(self): class XPUTestSoftSignOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'softsign' self.use_dynamic_create_class = False class XPUTestSoftSign(TestActivationOPBase): - def set_case(self): self.op_type = "softsign" self.dtype = self.in_type @@ -981,13 +924,11 @@ def ref_softsign(x): class XPUTestSoftshrinkOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'softshrink' self.use_dynamic_create_class = False class XPUTestSoftshrink(TestActivationOPBase): - def set_case(self): self.op_type = "softshrink" self.dtype = self.in_type @@ -1010,18 +951,17 @@ def set_case(self): def ref_softshrink(x, threshold=0.5): out = np.copy(x) out = (out < -threshold) * (out + threshold) + (out > threshold) * ( - out - threshold) + out - threshold + ) return out class XPUTestSwishOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'swish' self.use_dynamic_create_class = False class XPUTestSwishBase(TestActivationOPBase): - def set_case(self): self.op_type = "swish" self.dtype = self.in_type @@ -1037,21 +977,20 @@ def init_config(self): self.x = np.random.uniform(-1, 1, [11, 17]).astype(self.dtype) class XPUTestSwish2(XPUTestSwishBase): - def init_config(self): self.x = np.random.uniform(-2, 2, [1024, 8]).astype(self.dtype) class XPUTestSwish3(XPUTestSwishBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 512, 15, 15]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 512, 15, 15]).astype( + self.dtype + ) class XPUTestSwish4(XPUTestSwishBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 256, 22, 22]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 256, 22, 22]).astype( + self.dtype + ) support_types = get_xpu_op_support_types('swish') @@ -1061,18 +1000,17 @@ def init_config(self): def ref_swish(x): from scipy.special import expit + out = x * expit(x) return out class XPUTestThresholdedReluOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'thresholded_relu' self.use_dynamic_create_class = False class XPUTestThresholdedRelu(TestActivationOPBase): - def set_case(self): self.op_type = "thresholded_relu" self.dtype = self.in_type @@ -1099,13 +1037,11 @@ def ref_thresholded_relu(x, threshold=1.0): class XPUTestMishOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'mish' self.use_dynamic_create_class = False class XPUTestMishBase(TestActivationOPBase): - def set_case(self): self.op_type = "mish" self.dtype = self.in_type @@ -1122,21 +1058,20 @@ def init_config(self): self.x = np.random.uniform(-1, 1, [11, 17]).astype(self.dtype) class XPUTestMish2(XPUTestMishBase): - def init_config(self): self.x = np.random.uniform(-2, 2, [1024, 8]).astype(self.dtype) class XPUTestMish3(XPUTestMishBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 512, 15, 15]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 512, 15, 15]).astype( + self.dtype + ) class XPUTestMish4(XPUTestMishBase): - def init_config(self): - self.x = np.random.uniform(-2, 2, - [4, 256, 22, 22]).astype(self.dtype) + self.x = np.random.uniform(-2, 2, [4, 256, 22, 22]).astype( + self.dtype + ) support_types = get_xpu_op_support_types('mish') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_adam_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_adam_op_xpu.py index 332fc3fd489c4..fb42d56457746 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_adam_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_adam_op_xpu.py @@ -22,18 +22,20 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class XPUTestAdamOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'adam' self.use_dynamic_create_class = False class TestAdamOp(XPUOpTest): - '''Test Adam Op with supplied attributes - ''' + '''Test Adam Op with supplied attributes''' def setUp(self): self.init_dtype() @@ -45,20 +47,18 @@ def setUp(self): self.set_shape() self.set_inputs() self.set_steps() - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step( + self.inputs, self.attrs + ) self.outputs = { - 'Moment1Out': - moment1_out, - 'Moment2Out': - moment2_out, - 'ParamOut': - param_out, - 'Beta1PowOut': - np.array([self.beta1_pow]).astype("float32") * self.beta1, - 'Beta2PowOut': - np.array([self.beta2_pow]).astype("float32") * self.beta2 + 'Moment1Out': moment1_out, + 'Moment2Out': moment2_out, + 'ParamOut': param_out, + 'Beta1PowOut': np.array([self.beta1_pow]).astype("float32") + * self.beta1, + 'Beta2PowOut': np.array([self.beta2_pow]).astype("float32") + * self.beta2, } def set_xpu(self): @@ -73,7 +73,7 @@ def set_attrs(self): self.attrs = { 'epsilon': self.epsilon, 'beta1': self.beta1, - 'beta2': self.beta2 + 'beta2': self.beta2, } def set_data(self): @@ -103,18 +103,18 @@ def set_inputs(self): 'Grad': grad, 'Moment1': moment1, 'Moment2': moment2, - 'LearningRate': - np.array([self.learning_rate]).astype("float32"), + 'LearningRate': np.array([self.learning_rate]).astype( + "float32" + ), 'Beta1Pow': np.array([self.beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([self.beta2_pow]).astype("float32") + 'Beta2Pow': np.array([self.beta2_pow]).astype("float32"), } def test_check_output(self): self.check_output_with_place(place=paddle.XPUPlace(0), atol=1e-2) class TestAdamOp2(TestAdamOp): - '''Test Adam Op with supplied attributes - ''' + '''Test Adam Op with supplied attributes''' def set_data(self): self.beta1 = 0.9 @@ -123,29 +123,25 @@ def set_data(self): self.epsilon = 1e-8 class TestAdamOp3(TestAdamOp2): - '''Test Adam Op with supplied attributes - ''' + '''Test Adam Op with supplied attributes''' def set_shape(self): self.shape = (101, 47) class TestAdamOp4(TestAdamOp2): - '''Test Adam Op with supplied attributes - ''' + '''Test Adam Op with supplied attributes''' def set_shape(self): self.shape = (512, 26) class TestAdamOp5(TestAdamOp2): - '''Test Adam Op with supplied attributes - ''' + '''Test Adam Op with supplied attributes''' def set_shape(self): self.shape = (11, 1) class TestAdamOp6(TestAdamOp2): - '''Test Adam Op with beta as Variable - ''' + '''Test Adam Op with beta as Variable''' def set_shape(self): self.shape = (10, 10) @@ -157,8 +153,7 @@ def set_data(self): self.epsilon = 1e-8 class TestAdamOp7(TestAdamOp): - '''Test Adam Op with float16 accuracy - ''' + '''Test Adam Op with float16 accuracy''' def setUp(self): self.init_dtype() @@ -170,20 +165,18 @@ def setUp(self): self.set_shape() self.set_inputs() self.set_steps() - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step( + self.inputs, self.attrs + ) self.outputs = { - 'Moment1Out': - moment1_out, - 'Moment2Out': - moment2_out, - 'ParamOut': - param_out, - 'Beta1PowOut': - np.array([self.beta1_pow]).astype("float16") * self.beta1, - 'Beta2PowOut': - np.array([self.beta2_pow]).astype("float16") * self.beta2 + 'Moment1Out': moment1_out, + 'Moment2Out': moment2_out, + 'ParamOut': param_out, + 'Beta1PowOut': np.array([self.beta1_pow]).astype("float16") + * self.beta1, + 'Beta2PowOut': np.array([self.beta2_pow]).astype("float16") + * self.beta2, } def set_inputs(self): @@ -201,23 +194,24 @@ def set_inputs(self): 'Grad': grad, 'Moment1': moment1, 'Moment2': moment2, - 'LearningRate': - np.array([self.learning_rate]).astype("float16"), + 'LearningRate': np.array([self.learning_rate]).astype( + "float16" + ), 'Beta1Pow': np.array([self.beta1_pow]).astype("float16"), - 'Beta2Pow': np.array([self.beta2_pow]).astype("float16") + 'Beta2Pow': np.array([self.beta2_pow]).astype("float16"), } class TestAdamOpMultipleSteps(TestAdamOp2): - '''Test Adam Operator with supplied attributes - ''' + '''Test Adam Operator with supplied attributes''' def set_steps(self): self.num_steps = 10 def test_check_output(self): for _ in range(self.num_steps): - param_out, moment1_out, \ - moment2_out = adam_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adam_step( + self.inputs, self.attrs + ) beta1_pow_out = self.inputs['Beta1Pow'] * self.beta1 beta2_pow_out = self.inputs['Beta2Pow'] * self.beta2 @@ -226,12 +220,13 @@ def test_check_output(self): 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out + 'Beta2PowOut': beta2_pow_out, } # Verify output for this step - self.check_output_with_place(place=paddle.XPUPlace(0), - atol=1e-2) + self.check_output_with_place( + place=paddle.XPUPlace(0), atol=1e-2 + ) # Output of this step becomes input for next step self.inputs['Param'] = param_out @@ -244,7 +239,8 @@ def test_check_output(self): # Randomize gradient for next step self.inputs['Grad'] = np.random.uniform( - -1, 1, (102, 105)).astype("float32") + -1, 1, (102, 105) + ).astype("float32") def adam_step(inputs, attributes): @@ -281,8 +277,9 @@ def adam_step(inputs, attributes): return param_out, moment1_out, moment2_out -def adam_step_sparse(inputs, attributes, height, rows, row_numel, np_grad, - lazy_mode): +def adam_step_sparse( + inputs, attributes, height, rows, row_numel, np_grad, lazy_mode +): ''' Simulate one step of the adam optimizer :param inputs: dict of inputs @@ -307,13 +304,16 @@ def adam_step_sparse(inputs, attributes, height, rows, row_numel, np_grad, param_out = np.zeros(shape=[height, row_numel]) def update_row(row_id, update_value): - moment1_out[row_id] = beta1 * moment1[row_id] + (1 - - beta1) * update_value - moment2_out[row_id] = beta2 * moment2[row_id] + ( - 1 - beta2) * np.square(update_value) + moment1_out[row_id] = ( + beta1 * moment1[row_id] + (1 - beta1) * update_value + ) + moment2_out[row_id] = beta2 * moment2[row_id] + (1 - beta2) * np.square( + update_value + ) lr_t = lr * np.sqrt(1 - beta2_pow) / (1 - beta1_pow) param_out[row_id] = param[row_id] - lr_t * ( - moment1_out[row_id] / (np.sqrt(moment2_out[row_id]) + epsilon)) + moment1_out[row_id] / (np.sqrt(moment2_out[row_id]) + epsilon) + ) if lazy_mode: for idx, row_id in enumerate(rows): @@ -329,7 +329,6 @@ def update_row(row_id, update_value): class TestSparseAdamOp(unittest.TestCase): - def setup(self, scope, place, lazy_mode): beta1 = 0.78 beta2 = 0.836 @@ -348,14 +347,14 @@ def setup(self, scope, place, lazy_mode): "Moment2": np.full((height, row_numel), 5.0).astype("float32"), 'Beta1Pow': beta1_pow, 'Beta2Pow': beta2_pow, - "LearningRate": np.full((1), 2.0).astype("float32") + "LearningRate": np.full((1), 2.0).astype("float32"), } self.init_output = np.full((height, row_numel), 0.0).astype("float32") self.attrs = { 'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2, - 'min_row_size_to_use_multithread': 2 + 'min_row_size_to_use_multithread': 2, } grad_selected_rows = scope.var('Grad').get_selected_rows() @@ -370,15 +369,21 @@ def setup(self, scope, place, lazy_mode): self.sparse_inputs = ["Grad"] - param_out, mom1, mom2 = adam_step_sparse(self.dense_inputs, self.attrs, - height, rows, row_numel, - np_array, lazy_mode) + param_out, mom1, mom2 = adam_step_sparse( + self.dense_inputs, + self.attrs, + height, + rows, + row_numel, + np_array, + lazy_mode, + ) self.outputs = { "ParamOut": param_out, "Moment1Out": mom1, "Moment2Out": mom2, 'Beta1PowOut': beta1_pow * beta1, - 'Beta2PowOut': beta2_pow * beta2 + 'Beta2PowOut': beta2_pow * beta2, } def check_with_place(self, place, lazy_mode): @@ -421,7 +426,6 @@ def test_sparse_adam(self): class TestSparseAdamOp1(TestSparseAdamOp): - def setup(self, scope, place, lazy_mode): beta1 = 0.78 beta2 = 0.836 @@ -440,14 +444,14 @@ def setup(self, scope, place, lazy_mode): "Moment2": np.full((height, row_numel), 5.0).astype("float16"), 'Beta1Pow': beta1_pow, 'Beta2Pow': beta2_pow, - "LearningRate": np.full((1), 2.0).astype("float16") + "LearningRate": np.full((1), 2.0).astype("float16"), } self.init_output = np.full((height, row_numel), 0.0).astype("float16") self.attrs = { 'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2, - 'min_row_size_to_use_multithread': 2 + 'min_row_size_to_use_multithread': 2, } grad_selected_rows = scope.var('Grad').get_selected_rows() @@ -462,15 +466,21 @@ def setup(self, scope, place, lazy_mode): self.sparse_inputs = ["Grad"] - param_out, mom1, mom2 = adam_step_sparse(self.dense_inputs, self.attrs, - height, rows, row_numel, - np_array, lazy_mode) + param_out, mom1, mom2 = adam_step_sparse( + self.dense_inputs, + self.attrs, + height, + rows, + row_numel, + np_array, + lazy_mode, + ) self.outputs = { "ParamOut": param_out, "Moment1Out": mom1, "Moment2Out": mom2, 'Beta1PowOut': beta1_pow * beta1, - 'Beta2PowOut': beta2_pow * beta2 + 'Beta2PowOut': beta2_pow * beta2, } diff --git a/python/paddle/fluid/tests/unittests/xpu/test_adamw_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_adamw_op_xpu.py index bc4c9095e8fa5..7280258272b40 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_adamw_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_adamw_op_xpu.py @@ -23,7 +23,11 @@ from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def adamw_step(inputs, attributes): @@ -70,19 +74,17 @@ def simple_lr_setting(param, decay_rate, n_layers): else: depth = 0 - return decay_rate**(n_layers + 2 - depth) + return decay_rate ** (n_layers + 2 - depth) class XPUTestAdamwOp1(XPUOpTestWrapper): - def __init__(self): self.op_name = 'adamw' self.use_dynamic_create_class = False class TestAdamW(XPUOpTest): - def setUp(self): - #Test AdamW Op with supplied attributes + # Test AdamW Op with supplied attributes self.op_type = "adamw" self.init_shape() self.dtype = self.in_type_str @@ -106,7 +108,7 @@ def setUp(self): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype(self.dtype), 'Beta1Pow': np.array([beta1_pow]).astype(self.dtype), - 'Beta2Pow': np.array([beta2_pow]).astype(self.dtype) + 'Beta2Pow': np.array([beta2_pow]).astype(self.dtype), } self.attrs = { @@ -114,18 +116,19 @@ def setUp(self): 'beta1': beta1, 'beta2': beta2, "coeff": 0.5, - "with_decay": True + "with_decay": True, } - param_out, moment1_out, \ - moment2_out = adamw_step(self.inputs, self.attrs) + param_out, moment1_out, moment2_out = adamw_step( + self.inputs, self.attrs + ) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': np.array([beta1_pow]).astype(self.dtype) * beta1, - 'Beta2PowOut': np.array([beta2_pow]).astype(self.dtype) * beta2 + 'Beta2PowOut': np.array([beta2_pow]).astype(self.dtype) * beta2, } def init_shape(self): @@ -136,26 +139,22 @@ def test_check_output(self): self.check_output_with_place(place=paddle.XPUPlace(0)) class TestAdamW2(TestAdamW): - def init_shape(self): self.shape = [ 1000, ] class TestAdamW3(TestAdamW): - def init_shape(self): self.shape = [200, 3000] class XPUTestAdamwOp2(XPUOpTestWrapper): - def __init__(self): self.op_name = 'adamw' self.use_dynamic_create_class = False class TestAdamWOp(unittest.TestCase): - def test_adamw_op_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype(self.in_type_str) @@ -165,7 +164,8 @@ def test_adamw_op_dygraph(self): learning_rate=0.01, parameters=linear.parameters(), apply_decay_param_fun=lambda name: True, - weight_decay=0.01) + weight_decay=0.01, + ) for _ in range(2): out = linear(a) @@ -182,8 +182,9 @@ def test_adamw_op_coverage(self): learning_rate=0.0, parameters=linear.parameters(), apply_decay_param_fun=lambda name: True, - weight_decay=0.01) - assert (adam.__str__() is not None) + weight_decay=0.01, + ) + assert adam.__str__() is not None def test_adamw_op(self): paddle.enable_static() @@ -202,25 +203,29 @@ def test_adamw_op(self): shape=[1], value=0.85, dtype=self.in_type_str, - persistable=True) + persistable=True, + ) beta2 = fluid.layers.create_global_var( shape=[1], value=0.95, dtype=self.in_type_str, - persistable=True) + persistable=True, + ) betas = [beta1, beta2] - opt = paddle.optimizer.AdamW(learning_rate=1e-5, - beta1=beta1, - beta2=beta2, - weight_decay=0.01, - epsilon=1e-8) + opt = paddle.optimizer.AdamW( + learning_rate=1e-5, + beta1=beta1, + beta2=beta2, + weight_decay=0.01, + epsilon=1e-8, + ) opt.minimize(loss) exe.run(startup) data_np = np.random.random(shape).astype(self.in_type_str) - rets = exe.run(train_prog, - feed={"data": data_np}, - fetch_list=[loss]) + rets = exe.run( + train_prog, feed={"data": data_np}, fetch_list=[loss] + ) assert rets[0] is not None paddle.disable_static() @@ -228,20 +233,19 @@ def test_adamw_op_invalid_input(self): paddle.disable_static() linear = paddle.nn.Linear(10, 10) with self.assertRaises(ValueError): - adam = paddle.optimizer.AdamW(0.1, - beta1=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.AdamW( + 0.1, beta1=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.AdamW(0.1, - beta2=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.AdamW( + 0.1, beta2=-1, parameters=linear.parameters() + ) with self.assertRaises(ValueError): - adam = paddle.optimizer.AdamW(0.1, - epsilon=-1, - parameters=linear.parameters()) + adam = paddle.optimizer.AdamW( + 0.1, epsilon=-1, parameters=linear.parameters() + ) class TestAdamWOpGroup(TestAdamWOp): - def test_adamw_op_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype(self.in_type_str) @@ -250,14 +254,13 @@ def test_adamw_op_dygraph(self): linear_2 = paddle.nn.Linear(5, 3) adam = paddle.optimizer.AdamW( learning_rate=0.01, - parameters=[{ - 'params': linear_1.parameters() - }, { - 'params': linear_2.parameters(), - 'weight_decay': 0.001 - }], + parameters=[ + {'params': linear_1.parameters()}, + {'params': linear_2.parameters(), 'weight_decay': 0.001}, + ], apply_decay_param_fun=lambda name: True, - weight_decay=0.01) + weight_decay=0.01, + ) for _ in range(2): out = linear_1(a) @@ -267,7 +270,6 @@ def test_adamw_op_dygraph(self): adam.clear_gradients() class TestAdamWOpGroupWithLR(TestAdamWOp): - def test_adamw_op_dygraph(self): paddle.disable_static() value = np.arange(26).reshape(2, 13).astype(self.in_type_str) @@ -276,16 +278,21 @@ def test_adamw_op_dygraph(self): linear_2 = paddle.nn.Linear(5, 3) adam = paddle.optimizer.AdamW( learning_rate=paddle.optimizer.lr.PiecewiseDecay( - boundaries=[3, 6], values=[0.1, 0.2, 0.3]), - parameters=[{ - 'params': linear_1.parameters(), - 'learning_rate': 0.1, - }, { - 'params': linear_2.parameters(), - 'weight_decay': 0.001, - }], + boundaries=[3, 6], values=[0.1, 0.2, 0.3] + ), + parameters=[ + { + 'params': linear_1.parameters(), + 'learning_rate': 0.1, + }, + { + 'params': linear_2.parameters(), + 'weight_decay': 0.001, + }, + ], apply_decay_param_fun=lambda name: True, - weight_decay=0.01) + weight_decay=0.01, + ) for _ in range(2): out = linear_1(a) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_affine_channel_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_affine_channel_op_xpu.py index 447f18bd3c2af..0718c040bcaca 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_affine_channel_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_affine_channel_op_xpu.py @@ -39,7 +39,6 @@ def affine_channel(x, scale, bias, layout): class TestAffineChannelOp(XPUOpTest): - def setUp(self): self.op_type = "affine_channel" self.init_test_case() @@ -70,17 +69,17 @@ def test_check_grad_stopgrad_dx(self): if core.is_compiled_with_xpu(): paddle.enable_static() place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['Scale', 'Bias'], - 'Out', - no_grad_set=set('X')) + self.check_grad_with_place( + place, ['Scale', 'Bias'], 'Out', no_grad_set=set('X') + ) def test_check_grad_stopgrad_dscale_dbias(self): if core.is_compiled_with_xpu(): paddle.enable_static() place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - no_grad_set=set(['Scale', 'Bias'])) + self.check_grad_with_place( + place, ['X'], 'Out', no_grad_set=set(['Scale', 'Bias']) + ) def init_test_case(self): self.shape = [2, 100, 3, 3] @@ -89,7 +88,6 @@ def init_test_case(self): class TestAffineChannelOpError(unittest.TestCase): - def test_errors(self): with fluid.program_guard(fluid.Program()): @@ -100,32 +98,31 @@ def test_x_type(): self.assertRaises(TypeError, test_x_type) def test_x_dtype(): - x2 = fluid.layers.data(name='x2', - shape=[None, 1, 2, 2], - dtype='int32') + x2 = fluid.layers.data( + name='x2', shape=[None, 1, 2, 2], dtype='int32' + ) fluid.layers.affine_channel(x2) self.assertRaises(TypeError, test_x_dtype) def test_scale_type(): - x3 = fluid.layers.data(name='x3', - shape=[None, 1, 2, 2], - dtype='float32') + x3 = fluid.layers.data( + name='x3', shape=[None, 1, 2, 2], dtype='float32' + ) fluid.layers.affine_channel(x3, scale=1) self.assertRaises(TypeError, test_scale_type) def test_bias_type(): - x4 = fluid.layers.data(name='x4', - shape=[None, 1, 2, 2], - dtype='float32') + x4 = fluid.layers.data( + name='x4', shape=[None, 1, 2, 2], dtype='float32' + ) fluid.layers.affine_channel(x4, bias=1) self.assertRaises(TypeError, test_bias_type) class TestAffineChannelNHWC(TestAffineChannelOp): - def init_test_case(self): self.shape = [2, 3, 3, 100] self.C = 100 @@ -139,7 +136,6 @@ def test_check_grad_stopgrad_dscale_dbias(self): class TestAffineChannel2D(TestAffineChannelOp): - def init_test_case(self): self.shape = [2, 100] self.C = 100 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_amp_check_finite_and_scale_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_amp_check_finite_and_scale_op_xpu.py index e6bc61b895abb..b6e95e256901a 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_amp_check_finite_and_scale_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_amp_check_finite_and_scale_op_xpu.py @@ -19,19 +19,21 @@ import unittest import numpy as np from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestCheckFiniteAndUnscaleOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'check_finite_and_unscale' self.use_dynamic_create_class = False class TestCheckFiniteAndUnscaleOpNormal(XPUOpTest): - def setUp(self): self.op_type = "check_finite_and_unscale" self.init_dtype() @@ -52,7 +54,6 @@ def test_check_output(self): self.check_output_with_place(place) class TestCheckFiniteAndUnscaleOpWithNan(XPUOpTest): - def setUp(self): self.op_type = "check_finite_and_unscale" self.init_dtype() @@ -80,7 +81,6 @@ def test_check_output(self): self.check_output_with_place(place, no_check_set=['Out']) class TestCheckFiniteAndUnscaleOpWithInf(XPUOpTest): - def setUp(self): self.op_type = "check_finite_and_unscale" self.init_dtype() @@ -108,7 +108,6 @@ def test_check_output(self): self.check_output_with_place(place, no_check_set=['Out']) class TestCheckFiniteAndUnscaleOpWithInfAndNan(XPUOpTest): - def setUp(self): self.op_type = "check_finite_and_unscale" self.init_dtype() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_arg_max_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_arg_max_op_xpu.py index cdc16d4507267..d041e859d9862 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_arg_max_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_arg_max_op_xpu.py @@ -20,18 +20,20 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestArgMax(XPUOpTestWrapper): - def __init__(self): self.op_name = 'arg_max' class XPUBaseTestCase(XPUOpTest): - def initTestCase(self): self.dims = (3, 4) self.axis = 1 @@ -52,63 +54,53 @@ def test_check_output(self): self.check_output_with_place(place) class TestArgMaxCase1(XPUBaseTestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.axis = -1 class TestArgMaxCase2(XPUBaseTestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.axis = 0 class TestArgMaxCase3(XPUBaseTestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.axis = 1 class TestArgMaxCase4(XPUBaseTestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.axis = 2 class TestArgMaxCase5(XPUBaseTestCase): - def initTestCase(self): self.dims = (3, 4) self.axis = -1 class TestArgMaxCase6(XPUBaseTestCase): - def initTestCase(self): self.dims = (3, 4) self.axis = 0 class TestArgMaxCase7(XPUBaseTestCase): - def initTestCase(self): self.dims = (3, 4) self.axis = 1 class TestArgMaxCase8(XPUBaseTestCase): - def initTestCase(self): - self.dims = (1, ) + self.dims = (1,) self.axis = 0 class TestArgMaxCase9(XPUBaseTestCase): - def initTestCase(self): - self.dims = (2, ) + self.dims = (2,) self.axis = 0 class TestArgMaxCase10(XPUBaseTestCase): - def initTestCase(self): - self.dims = (3, ) + self.dims = (3,) self.axis = 0 @@ -118,7 +110,6 @@ def initTestCase(self): class TestArgMaxAPI(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -130,7 +121,6 @@ def setUp(self): self.place = [paddle.XPUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) @@ -138,9 +128,9 @@ def run(place): tensor_input = paddle.to_tensor(numpy_input) numpy_output = np.argmax(numpy_input, axis=self.axis) paddle_output = paddle.argmax(tensor_input, axis=self.axis) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) paddle.enable_static() for place in self.place: @@ -148,7 +138,6 @@ def run(place): class TestArgMaxAPI_2(unittest.TestCase): - def initTestCase(self): self.dims = (3, 4, 5) self.dtype = 'float32' @@ -161,20 +150,20 @@ def setUp(self): self.place = [paddle.XPUPlace(0)] def test_dygraph_api(self): - def run(place): paddle.disable_static(place) np.random.seed(2021) numpy_input = (np.random.random(self.dims)).astype(self.dtype) tensor_input = paddle.to_tensor(numpy_input) - numpy_output = np.argmax(numpy_input, - axis=self.axis).reshape(1, 4, 5) - paddle_output = paddle.argmax(tensor_input, - axis=self.axis, - keepdim=self.keep_dims) - np.testing.assert_allclose(numpy_output, - paddle_output.numpy(), - rtol=1e-05) + numpy_output = np.argmax(numpy_input, axis=self.axis).reshape( + 1, 4, 5 + ) + paddle_output = paddle.argmax( + tensor_input, axis=self.axis, keepdim=self.keep_dims + ) + np.testing.assert_allclose( + numpy_output, paddle_output.numpy(), rtol=1e-05 + ) self.assertEqual(numpy_output.shape, paddle_output.numpy().shape) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_argsort_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_argsort_op_xpu.py index 039d3c2051e49..8ee7716447f87 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_argsort_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_argsort_op_xpu.py @@ -20,13 +20,16 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestArgsortOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'argsort' self.use_dynamic_create_class = True @@ -36,14 +39,14 @@ def dynamic_create_class(self): classes = [] for descending in [True, False]: for axis in [0, 1, 2, -1, -2]: - class_name = 'XPUTestArgsortOp_axis_' + str(axis) + '_' + str( - descending) + class_name = ( + 'XPUTestArgsortOp_axis_' + str(axis) + '_' + str(descending) + ) attr_dict = {'init_axis': axis, 'init_descending': descending} classes.append([class_name, attr_dict]) return base_class, classes class TestArgsortOp(XPUOpTest): - def setUp(self): self.set_xpu() self.op_type = "argsort" @@ -51,16 +54,18 @@ def setUp(self): self.dtype = self.in_type self.input_shape = (2, 2, 2, 3, 3) self.axis = -1 if not hasattr(self, 'init_axis') else self.init_axis - self.descending = False if not hasattr( - self, 'init_descending') else self.init_descending + self.descending = ( + False + if not hasattr(self, 'init_descending') + else self.init_descending + ) if self.dtype == np.float32: self.x = np.random.random(self.input_shape).astype(self.dtype) else: - self.x = np.random.randint(low=-1000, - high=1000, - size=self.input_shape).astype( - self.dtype) + self.x = np.random.randint( + low=-1000, high=1000, size=self.input_shape + ).astype(self.dtype) self.inputs = {"X": self.x} self.attrs = {"axis": self.axis, "descending": self.descending} @@ -71,13 +76,15 @@ def get_output(self): if self.descending: self.indices = np.flip( np.argsort(self.x, kind='heapsort', axis=self.axis), - self.axis) + self.axis, + ) self.sorted_x = np.flip( - np.sort(self.x, kind='heapsort', axis=self.axis), self.axis) + np.sort(self.x, kind='heapsort', axis=self.axis), self.axis + ) else: - self.indices = np.argsort(self.x, - kind='heapsort', - axis=self.axis) + self.indices = np.argsort( + self.x, kind='heapsort', axis=self.axis + ) self.sorted_x = np.sort(self.x, kind='heapsort', axis=self.axis) def set_xpu(self): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_assign_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_assign_op_xpu.py index 306ebf957c385..74f2a41a9eee2 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_assign_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_assign_op_xpu.py @@ -17,6 +17,7 @@ sys.path.append("..") import unittest import paddle + ''' class TestAssignOp(op_test.OpTest): def setUp(self): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_assign_value_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_assign_value_op_xpu.py index c06ef86b4fee9..abb8dff9ecb76 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_assign_value_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_assign_value_op_xpu.py @@ -21,20 +21,22 @@ import paddle.fluid.framework as framework import paddle.fluid.layers as layers from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle paddle.enable_static() class XPUTestAssignValueOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'assign_value' self.use_dynamic_create_class = False class TestAssignValueOp(XPUOpTest): - def init(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -47,7 +49,8 @@ def setUp(self): self.init_data() self.attrs["shape"] = self.value.shape self.attrs["dtype"] = framework.convert_np_dtype_to_dtype_( - self.value.dtype) + self.value.dtype + ) self.outputs = {"Out": self.value} def init_data(self): @@ -58,31 +61,29 @@ def test_forward(self): self.check_output_with_place(self.place) class TestAssignValueOp2(TestAssignValueOp): - def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int32) self.attrs["int32_values"] = [int(v) for v in self.value.flat] class TestAssignValueOp3(TestAssignValueOp): - def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int64) self.attrs["int64_values"] = [int(v) for v in self.value.flat] class TestAssignValueOp4(TestAssignValueOp): - def init_data(self): - self.value = np.random.choice(a=[False, True], - size=(2, 5)).astype(np.bool) + self.value = np.random.choice(a=[False, True], size=(2, 5)).astype( + np.bool + ) self.attrs["bool_values"] = [int(v) for v in self.value.flat] class TestAssignApi(unittest.TestCase): - def setUp(self): self.init_dtype() self.value = (-100 + 200 * np.random.random(size=(2, 5))).astype( - self.dtype) + self.dtype + ) self.place = fluid.XPUPlace(0) def init_dtype(self): @@ -101,23 +102,21 @@ def test_assign(self): class TestAssignApi2(TestAssignApi): - def init_dtype(self): self.dtype = "int32" class TestAssignApi3(TestAssignApi): - def init_dtype(self): self.dtype = "int64" class TestAssignApi4(TestAssignApi): - def setUp(self): self.init_dtype() - self.value = np.random.choice(a=[False, True], - size=(2, 5)).astype(np.bool) + self.value = np.random.choice(a=[False, True], size=(2, 5)).astype( + np.bool + ) self.place = fluid.XPUPlace(0) def init_dtype(self): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_batch_norm_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_batch_norm_op_xpu.py index 09d813d23769d..7d818cc02c868 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_batch_norm_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_batch_norm_op_xpu.py @@ -21,13 +21,18 @@ import paddle import paddle.fluid as fluid import paddle.nn.functional as F -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() -def ref_batch_norm_infer(x, scale, bias, mean, variance, momentum, epsilon, - data_layout): +def ref_batch_norm_infer( + x, scale, bias, mean, variance, momentum, epsilon, data_layout +): if data_layout == "NCHW": n, c, h, w = x.shape mean_tile = np.reshape(mean, (1, c, 1, 1)) @@ -46,12 +51,14 @@ def ref_batch_norm_infer(x, scale, bias, mean, variance, momentum, epsilon, else: raise ValueError( "Unsupported data layout! Only NCHW and NHWC is supported, but received " - + data_layout) + + data_layout + ) return y -def ref_batch_norm_train(x, y_grad, scale, bias, mean, variance, momentum, - epsilon, data_layout): +def ref_batch_norm_train( + x, y_grad, scale, bias, mean, variance, momentum, epsilon, data_layout +): # Forward if data_layout == "NCHW": n, c, h, w = x.shape @@ -65,8 +72,9 @@ def ref_batch_norm_train(x, y_grad, scale, bias, mean, variance, momentum, saved_mean_tile = np.tile(saved_mean_tile, (n, 1, h, w)) saved_variance_tile = np.reshape(saved_variance, (1, c, 1, 1)) saved_variance_tile = np.tile(saved_variance_tile, (n, 1, h, w)) - normalized_x = (x - saved_mean_tile) / np.sqrt(saved_variance_tile + - epsilon) + normalized_x = (x - saved_mean_tile) / np.sqrt( + saved_variance_tile + epsilon + ) scale_tile = np.reshape(scale, (1, c, 1, 1)) scale_tile = np.tile(scale_tile, (n, 1, h, w)) bias_tile = np.reshape(bias, (1, c, 1, 1)) @@ -84,10 +92,11 @@ def ref_batch_norm_train(x, y_grad, scale, bias, mean, variance, momentum, else: raise ValueError( "Unsupported data layout! Only NCHW and NHWC is supported, but received " - + data_layout) - mean_out = saved_mean * (1. - momentum) + momentum * mean - variance_out = saved_variance * (1. - momentum) + momentum * variance - saved_inv_std = 1. / np.sqrt(saved_variance + epsilon) + + data_layout + ) + mean_out = saved_mean * (1.0 - momentum) + momentum * mean + variance_out = saved_variance * (1.0 - momentum) + momentum * variance + saved_inv_std = 1.0 / np.sqrt(saved_variance + epsilon) # Backward # Use the following formulas to calculate gradients: # grad_scale = @@ -102,32 +111,48 @@ def ref_batch_norm_train(x, y_grad, scale, bias, mean, variance, momentum, if data_layout == "NCHW": x = np.transpose(x, (0, 2, 3, 1)) y_grad = np.transpose(y_grad, (0, 2, 3, 1)) - x_grad = scale * ( - y_grad - np.mean(y_grad, axis=(0, 1, 2)) - - (x - saved_mean) * np.mean(y_grad * (x - saved_mean), axis=(0, 1, 2)) / - (saved_variance + epsilon)) / np.sqrt(saved_variance + epsilon) - scale_grad = np.sum(y_grad * (x - saved_mean) / - np.sqrt(saved_variance + epsilon), - axis=(0, 1, 2)) + x_grad = ( + scale + * ( + y_grad + - np.mean(y_grad, axis=(0, 1, 2)) + - (x - saved_mean) + * np.mean(y_grad * (x - saved_mean), axis=(0, 1, 2)) + / (saved_variance + epsilon) + ) + / np.sqrt(saved_variance + epsilon) + ) + scale_grad = np.sum( + y_grad * (x - saved_mean) / np.sqrt(saved_variance + epsilon), + axis=(0, 1, 2), + ) bias_grad = np.sum(y_grad, axis=(0, 1, 2)) # Transfer back to N, C, H, W if data_layout == "NCHW": x_grad = np.transpose(x_grad, (0, 3, 1, 2)) x = np.transpose(x, (0, 3, 1, 2)) y_grad = np.transpose(y_grad, (0, 3, 1, 2)) - return y, mean_out, variance_out, saved_mean, saved_inv_std, x_grad, scale_grad, bias_grad + return ( + y, + mean_out, + variance_out, + saved_mean, + saved_inv_std, + x_grad, + scale_grad, + bias_grad, + ) class XPUTestBatchNormOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'batch_norm' self.use_dynamic_create_class = False - @unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") + @unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" + ) class TestBatchNormOp(unittest.TestCase): - def setUp(self): self.op_type = "batch_norm" self.dtype = np.float32 @@ -146,13 +171,16 @@ def setUp(self): else: raise ValueError( "Unsupported data layout! Only NCHW and NHWC is supported, but received " - + self.data_layout) + + self.data_layout + ) np.random.seed(1024) self.x_np = np.random.random_sample(self.shape).astype(self.dtype) - self.scale_np = np.random.random_sample([channel_size - ]).astype(self.dtype) - self.bias_np = np.random.random_sample([channel_size - ]).astype(self.dtype) + self.scale_np = np.random.random_sample([channel_size]).astype( + self.dtype + ) + self.bias_np = np.random.random_sample([channel_size]).astype( + self.dtype + ) self.mean_np = np.zeros([channel_size]).astype(self.dtype) self.variance_np = np.ones([channel_size]).astype(self.dtype) self.saved_mean_np = np.zeros([channel_size]).astype(self.dtype) @@ -173,43 +201,81 @@ def test_infer(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): x = paddle.fluid.data('X', self.x_np.shape, self.x_np.dtype) - scale = paddle.fluid.data('Scale', self.scale_np.shape, - self.scale_np.dtype) - bias = paddle.fluid.data('Bias', self.bias_np.shape, - self.bias_np.dtype) - mean = paddle.fluid.data('Mean', self.mean_np.shape, - self.mean_np.dtype) - variance = paddle.fluid.data('Variance', self.variance_np.shape, - self.variance_np.dtype) - y = F.batch_norm(x, mean, variance, scale, bias, False, - self.momentum, self.epsilon, self.data_layout) + scale = paddle.fluid.data( + 'Scale', self.scale_np.shape, self.scale_np.dtype + ) + bias = paddle.fluid.data( + 'Bias', self.bias_np.shape, self.bias_np.dtype + ) + mean = paddle.fluid.data( + 'Mean', self.mean_np.shape, self.mean_np.dtype + ) + variance = paddle.fluid.data( + 'Variance', self.variance_np.shape, self.variance_np.dtype + ) + y = F.batch_norm( + x, + mean, + variance, + scale, + bias, + False, + self.momentum, + self.epsilon, + self.data_layout, + ) exe = paddle.static.Executor(self.place) - [y_np] = exe.run(feed={ - 'X': self.x_np, - 'Scale': self.scale_np, - 'Bias': self.bias_np, - 'Mean': self.mean_np, - 'Variance': self.variance_np - }, - fetch_list=[y]) - y_np_ref = ref_batch_norm_infer(self.x_np, self.scale_np, - self.bias_np, self.mean_np, - self.variance_np, self.momentum, - self.epsilon, self.data_layout) + [y_np] = exe.run( + feed={ + 'X': self.x_np, + 'Scale': self.scale_np, + 'Bias': self.bias_np, + 'Mean': self.mean_np, + 'Variance': self.variance_np, + }, + fetch_list=[y], + ) + y_np_ref = ref_batch_norm_infer( + self.x_np, + self.scale_np, + self.bias_np, + self.mean_np, + self.variance_np, + self.momentum, + self.epsilon, + self.data_layout, + ) np.testing.assert_allclose(y_np_ref, y_np, rtol=1e-05) def test_train(self): y_grad_np = np.random.random_sample(self.shape).astype(self.dtype) - y_np, mean_out_np, variance_out_np, saved_mean_np, saved_variance_np, x_grad_np, scale_grad_np, bias_grad_np = ref_batch_norm_train( - self.x_np, y_grad_np, self.scale_np, self.bias_np, self.mean_np, - self.variance_np, self.momentum, self.epsilon, self.data_layout) + ( + y_np, + mean_out_np, + variance_out_np, + saved_mean_np, + saved_variance_np, + x_grad_np, + scale_grad_np, + bias_grad_np, + ) = ref_batch_norm_train( + self.x_np, + y_grad_np, + self.scale_np, + self.bias_np, + self.mean_np, + self.variance_np, + self.momentum, + self.epsilon, + self.data_layout, + ) inputs = { 'X': self.x_np, 'Scale': self.scale_np, 'Bias': self.bias_np, 'Mean': self.mean_np, 'Variance': self.variance_np, - 'Y@GRAD': y_grad_np + 'Y@GRAD': y_grad_np, } outputs = { 'Y': y_np, @@ -219,7 +285,7 @@ def test_train(self): 'SavedVariance': saved_variance_np, 'X@GRAD': x_grad_np, 'Scale@GRAD': scale_grad_np, - 'Bias@GRAD': bias_grad_np + 'Bias@GRAD': bias_grad_np, } attrs = { 'momentum': self.momentum, @@ -240,9 +306,11 @@ def test_train(self): arg_name = var_name np_value = inputs[var_name] if not block.has_var(var_name): - block.create_var(name=var_name, - shape=np_value.shape, - dtype=np_value.dtype) + block.create_var( + name=var_name, + shape=np_value.shape, + dtype=np_value.dtype, + ) input_vars[arg_name] = block.var(var_name) fetch_list = [] output_vars = {} @@ -250,22 +318,27 @@ def test_train(self): arg_name = var_name np_value = outputs[var_name] if not block.has_var(var_name): - block.create_var(name=var_name, - shape=np_value.shape, - dtype=np_value.dtype) + block.create_var( + name=var_name, + shape=np_value.shape, + dtype=np_value.dtype, + ) if var_name == 'Mean': arg_name = 'MeanOut' # Share memory if var_name == 'Variance': arg_name = 'VarianceOut' # Share memory output_vars[arg_name] = block.var(var_name) fetch_list.append(var_name) - batch_norm_op = block.append_op(type="batch_norm", - inputs=input_vars, - outputs=output_vars, - attrs=attrs) + batch_norm_op = block.append_op( + type="batch_norm", + inputs=input_vars, + outputs=output_vars, + attrs=attrs, + ) # Generate the backward op_desc of batch_norm grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( - batch_norm_op.desc, set(), []) + batch_norm_op.desc, set(), [] + ) grad_op_desc = grad_op_desc_list[0] new_op_desc = block.desc.append_op() new_op_desc.copy_from(grad_op_desc) @@ -273,13 +346,11 @@ def test_train(self): exe = paddle.static.Executor(self.place) outs = exe.run(program, feed=inputs, fetch_list=fetch_list) for id, name in enumerate(fetch_list): - np.testing.assert_allclose(outputs[name], - outs[id], - rtol=1e-05, - atol=1e-4) + np.testing.assert_allclose( + outputs[name], outs[id], rtol=1e-05, atol=1e-4 + ) class TestBatchNormOpUseGlobalStats(unittest.TestCase): - def setUp(self): self.places = [paddle.XPUPlace(0)] self.init_test() @@ -296,11 +367,14 @@ def test_global_stats(self): net1 = paddle.fluid.dygraph.BatchNorm( 6, param_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(1.0)), + initializer=fluid.initializer.Constant(1.0) + ), use_global_stats=self.use_global_stats, - trainable_statistics=self.trainable_statistics) + trainable_statistics=self.trainable_statistics, + ) net2 = paddle.nn.BatchNorm2D( - 6, use_global_stats=self.use_global_stats) + 6, use_global_stats=self.use_global_stats + ) net2.weight = net1.weight net2.bias = net1.bias if self.trainable_statistics == True: @@ -308,9 +382,9 @@ def test_global_stats(self): net2.training = False y1 = net1(x) y2 = net2(x) - np.testing.assert_allclose(y1.numpy(), - y2.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + y1.numpy(), y2.numpy(), rtol=1e-05 + ) class TestBatchNormOpUseGlobalStats1(TestBatchNormOpUseGlobalStats): ### test mode diff --git a/python/paddle/fluid/tests/unittests/xpu/test_bce_loss_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_bce_loss_op_xpu.py index e5fa996f16cd8..01256f1d9cc55 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_bce_loss_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_bce_loss_op_xpu.py @@ -19,30 +19,33 @@ import numpy as np import unittest from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() def bce_loss(input, label): - return -1 * (label * np.log(input) + (1. - label) * np.log(1. - input)) + return -1 * (label * np.log(input) + (1.0 - label) * np.log(1.0 - input)) class XPUTestBceLossOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'bce_loss' self.use_dynamic_create_class = False class TestBceLossOp(XPUOpTest): - def setUp(self): self.op_type = "bce_loss" self.dtype = self.in_type self.place = paddle.XPUPlace(0) self.init_test_case() - input_np = np.random.uniform(0.1, 0.8, - self.shape).astype(self.dtype) + input_np = np.random.uniform(0.1, 0.8, self.shape).astype( + self.dtype + ) label_np = np.random.randint(0, 2, self.shape).astype(self.dtype) output_np = bce_loss(input_np, label_np) @@ -59,12 +62,10 @@ def init_test_case(self): self.shape = [10, 10] class TestBceLossOpCase1(TestBceLossOp): - def init_test_cast(self): self.shape = [2, 3, 4, 5] class TestBceLossOpCase2(TestBceLossOp): - def init_test_cast(self): self.shape = [2, 3, 20] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_bilinear_interp_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_bilinear_interp_v2_op_xpu.py index 811a6f6cb7c80..842018ba8f4e9 100755 --- a/python/paddle/fluid/tests/unittests/xpu/test_bilinear_interp_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_bilinear_interp_v2_op_xpu.py @@ -20,21 +20,27 @@ import paddle from op_test_xpu import XPUOpTest import unittest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() -def bilinear_interp_np(input, - out_h, - out_w, - scale_w=0, - scale_h=0, - out_size=None, - actual_shape=None, - align_corners=True, - align_mode=0, - data_layout='NCHW'): +def bilinear_interp_np( + input, + out_h, + out_w, + scale_w=0, + scale_h=0, + out_size=None, + actual_shape=None, + align_corners=True, + align_mode=0, + data_layout='NCHW', +): """bilinear interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": input = np.transpose(input, (0, 3, 1, 2)) # NHWC => NCHW @@ -48,7 +54,7 @@ def bilinear_interp_np(input, ratio_h = ratio_w = 0.0 if out_h > 1: - if (align_corners): + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: @@ -56,7 +62,7 @@ def bilinear_interp_np(input, else: ratio_h = 1.0 * in_h / out_h if out_w > 1: - if (align_corners): + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -67,37 +73,40 @@ def bilinear_interp_np(input, out = np.zeros((batch_size, channel, out_h, out_w)) for i in range(out_h): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: h = int(ratio_h * (i + 0.5) - 0.5) else: h = int(ratio_h * i) h = max(0, h) hid = 1 if h < in_h - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_h = max(ratio_h * (i + 0.5) - 0.5, 0) h1lambda = idx_src_h - h else: h1lambda = ratio_h * i - h h2lambda = 1.0 - h1lambda for j in range(out_w): - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: w = int(ratio_w * (j + 0.5) - 0.5) else: w = int(ratio_w * j) w = max(0, w) wid = 1 if w < in_w - 1 else 0 - if (align_mode == 0 and not align_corners): + if align_mode == 0 and not align_corners: idx_src_w = max(ratio_w * (j + 0.5) - 0.5, 0) w1lambda = idx_src_w - w else: w1lambda = ratio_w * j - w w2lambda = 1.0 - w1lambda - out[:, :, i, j] = h2lambda*(w2lambda*input[:, :, h, w] + - w1lambda*input[:, :, h, w+wid]) + \ - h1lambda*(w2lambda*input[:, :, h+hid, w] + - w1lambda*input[:, :, h+hid, w+wid]) + out[:, :, i, j] = h2lambda * ( + w2lambda * input[:, :, h, w] + + w1lambda * input[:, :, h, w + wid] + ) + h1lambda * ( + w2lambda * input[:, :, h + hid, w] + + w1lambda * input[:, :, h + hid, w + wid] + ) if data_layout == "NHWC": out = np.transpose(out, (0, 2, 3, 1)) # NCHW => NHWC @@ -106,13 +115,11 @@ def bilinear_interp_np(input, class XPUTestBilinearInterpV2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = 'bilinear_interp_v2' self.use_dynamic_create_class = False class TestBilinearInterpOp(XPUOpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -133,7 +140,7 @@ def setUp(self): scale_w = 0 if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: scale_h = scale_w = float(self.scale) if isinstance(self.scale, list) and len(self.scale) == 1: scale_w = scale_h = self.scale[0] @@ -146,10 +153,18 @@ def setUp(self): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode, - self.data_layout) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + self.align_mode, + self.data_layout, + ) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -162,11 +177,11 @@ def setUp(self): 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): - if self.scale > 0.: + if self.scale > 0.0: self.scale = [self.scale] if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] @@ -184,7 +199,7 @@ def init_test_case(self): self.input_shape = [2, 3, 5, 5] self.out_h = 2 self.out_w = 2 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True self.align_mode = 1 @@ -193,76 +208,69 @@ def init_place(self): self.place = paddle.XPUPlace(0) class TestBilinearInterpCase1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase4(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([2, 2]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase5(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([11, 11]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase6(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([65, 33]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpCase7(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [1, 1, 32, 64] @@ -273,70 +281,62 @@ def init_test_case(self): self.align_mode = 1 class TestBilinearInterpSame(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpActualShape(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.align_mode = 1 class TestBilinearInterpOtherMethod1(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 1 class TestBilinearInterpWithMethod2(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = False self.align_mode = 0 class TestBilinearInterpWithMethod3(TestBilinearInterpOp): - def set_align_mode(self): self.align_corners = True self.align_mode = 0 class TestBilinearInterpScale1(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 2. + self.scale = 2.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale2(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] self.out_h = 60 self.out_w = 25 - self.scale = 1. + self.scale = 1.0 self.align_corners = True self.align_mode = 1 class TestBilinearInterpScale3(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -347,7 +347,6 @@ def init_test_case(self): self.align_mode = 1 class TestBilinearInterpScale4(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -358,7 +357,6 @@ def init_test_case(self): self.align_mode = 1 class TestBilinearInterpZero(TestBilinearInterpOp): - def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [2, 3, 5, 7] @@ -369,7 +367,6 @@ def init_test_case(self): self.align_mode = 0 class TestBilinearInterpOp_attr_tensor(XPUOpTest): - def setUp(self): self.out_size = None self.actual_shape = None @@ -409,8 +406,9 @@ def setUp(self): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h @@ -422,9 +420,16 @@ def setUp(self): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, self.actual_shape, - self.align_corners) + output_np = bilinear_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def test_check_output(self): @@ -438,7 +443,7 @@ def init_test_case(self): self.input_shape = [2, 3, 5, 5] self.out_h = 3 self.out_w = 3 - self.scale = 0. + self.scale = 0.0 self.out_size = [3, 3] self.align_corners = True @@ -446,36 +451,36 @@ def init_place(self): self.place = paddle.XPUPlace(0) # out_size is a 1-D tensor - class TestBilinearInterp_attr_tensor_Case1(TestBilinearInterpOp_attr_tensor - ): - + class TestBilinearInterp_attr_tensor_Case1( + TestBilinearInterpOp_attr_tensor + ): def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 - self.scale = 0. + self.scale = 0.0 self.out_size = [8, 12] self.align_corners = True # scale is a 1-D tensor - class TestBilinearInterp_attr_tensor_Case2(TestBilinearInterpOp_attr_tensor - ): - + class TestBilinearInterp_attr_tensor_Case2( + TestBilinearInterpOp_attr_tensor + ): def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 - self.scale = 0. + self.scale = 0.0 self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.shape_by_1Dtensor = True # scale is a 1-D tensor - class TestBilinearInterp_attr_tensor_Case3(TestBilinearInterpOp_attr_tensor - ): - + class TestBilinearInterp_attr_tensor_Case3( + TestBilinearInterpOp_attr_tensor + ): def init_test_case(self): self.interp_method = 'bilinear' self.input_shape = [3, 2, 32, 16] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_bitwise_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_bitwise_op_xpu.py index e7e81707376a8..2482709f3452c 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_bitwise_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_bitwise_op_xpu.py @@ -21,19 +21,21 @@ import paddle from op_test import OpTest from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() ################## TEST OP: BitwiseAnd ################## class XPUTestBitwiseAnd(XPUOpTestWrapper): - def __init__(self): self.op_name = 'bitwise_and' class XPUTestBitwiseAndBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -42,20 +44,18 @@ def setUp(self): def set_case(self): self.op_type = 'bitwise_and' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.bitwise_and(x, y) self.attrs = {'use_xpu': True} self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.outputs = {'Out': out} @@ -73,7 +73,6 @@ def test_check_grad(self): pass class XPUTestBitwiseAndCase1(XPUTestBitwiseAndBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [4, 5] @@ -82,7 +81,6 @@ def init_case(self): self.high = 100 class XPUTestBitwiseAndCase2(XPUTestBitwiseAndBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [2, 3, 4, 5] @@ -91,7 +89,6 @@ def init_case(self): self.high = 100 class XPUTestBitwiseAndCase3(XPUTestBitwiseAndBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [2, 3, 4, 5] @@ -107,12 +104,10 @@ def init_case(self): ################## TEST OP: BitwiseOr ################## class XPUTestBitwiseOr(XPUOpTestWrapper): - def __init__(self): self.op_name = 'bitwise_or' class XPUTestBitwiseOrBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -121,20 +116,18 @@ def setUp(self): def set_case(self): self.op_type = 'bitwise_or' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.bitwise_or(x, y) self.attrs = {'use_xpu': True} self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.outputs = {'Out': out} @@ -152,7 +145,6 @@ def test_check_grad(self): pass class XPUTestBitwiseOrCase1(XPUTestBitwiseOrBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [4, 5] @@ -161,7 +153,6 @@ def init_case(self): self.high = 100 class XPUTestBitwiseOrCase2(XPUTestBitwiseOrBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [2, 3, 4, 5] @@ -170,7 +161,6 @@ def init_case(self): self.high = 100 class XPUTestBitwiseOrCase3(XPUTestBitwiseOrBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [2, 3, 4, 5] @@ -186,12 +176,10 @@ def init_case(self): ################## TEST OP: BitwiseXor ################## class XPUTestBitwiseXor(XPUOpTestWrapper): - def __init__(self): self.op_name = 'bitwise_xor' class XPUTestBitwiseXorBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -200,20 +188,18 @@ def setUp(self): def set_case(self): self.op_type = 'bitwise_xor' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.bitwise_xor(x, y) self.attrs = {'use_xpu': True} self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.outputs = {'Out': out} @@ -231,7 +217,6 @@ def test_check_grad(self): pass class XPUTestBitwiseXorCase1(XPUTestBitwiseXorBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [4, 5] @@ -240,7 +225,6 @@ def init_case(self): self.high = 100 class XPUTestBitwiseXorCase2(XPUTestBitwiseXorBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [2, 3, 4, 5] @@ -249,7 +233,6 @@ def init_case(self): self.high = 100 class XPUTestBitwiseXorCase3(XPUTestBitwiseXorBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [2, 3, 4, 5] @@ -265,12 +248,10 @@ def init_case(self): ################## TEST OP: BitwiseNot ################## class XPUTestBitwiseNot(XPUOpTestWrapper): - def __init__(self): self.op_name = 'bitwise_not' class XPUTestBitwiseNotBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -279,10 +260,9 @@ def setUp(self): def set_case(self): self.op_type = 'bitwise_not' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) out = np.bitwise_not(x) self.attrs = {'use_xpu': True} @@ -302,7 +282,6 @@ def test_check_grad(self): pass class XPUTestBitwiseNotBool(XPUTestBitwiseNotBase): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_bmm_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_bmm_op_xpu.py index 840f88198cbb5..fbc0ca6692140 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_bmm_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_bmm_op_xpu.py @@ -18,7 +18,11 @@ import unittest import numpy as np from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -33,7 +37,6 @@ def __init__(self): self.use_dynamic_create_class = False class TestBmmOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -66,31 +69,26 @@ def test_check_grad_normal(self): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') class TestBmmOp1(TestBmmOp): - def set_shape(self): self.Xshape = (3, 3, 3) self.Yshape = (3, 3, 3) class TestBmmOp2(TestBmmOp): - def set_shape(self): self.Xshape = (128, 3, 16) self.Yshape = (128, 16, 3) class TestBmmOp3(TestBmmOp): - def set_shape(self): self.Xshape = (2048, 16, 27) self.Yshape = (2048, 27, 16) class TestBmmOp4(TestBmmOp): - def set_shape(self): self.Xshape = (2, 27, 27) self.Yshape = (2, 27, 27) class TestBmmOp5(TestBmmOp): - def set_shape(self): self.Xshape = (2, 1, 1) self.Yshape = (2, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_c_embedding_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_c_embedding_op_xpu.py index 7324fc100d5b8..854ec71cf7374 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_c_embedding_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_c_embedding_op_xpu.py @@ -17,7 +17,11 @@ sys.path.append("..") import paddle -from paddle.fluid.tests.unittests.c_embedding_op_base import TestCEmbeddingCPU, TestCEmbeddingOpBase, TestCEmbeddingOpFP32 +from paddle.fluid.tests.unittests.c_embedding_op_base import ( + TestCEmbeddingCPU, + TestCEmbeddingOpBase, + TestCEmbeddingOpFP32, +) paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_cast_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_cast_op_xpu.py index 908be556fd102..a2e136dccaab0 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_cast_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_cast_op_xpu.py @@ -23,7 +23,11 @@ from paddle.fluid import Program, program_guard from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) typeid_dict = { 'int32': int(core.VarDesc.VarType.INT32), @@ -36,7 +40,6 @@ class XPUTestCastOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'cast' self.use_dynamic_create_class = True @@ -51,12 +54,14 @@ def dynamic_create_class(self): return base_class, classes class TestCastOp(XPUOpTest): - def setUp(self): ipt = np.random.random(size=[10, 10]) in_typename = self.in_type_str - out_typename = 'float32' if not hasattr( - self, 'out_typename') else self.out_typename + out_typename = ( + 'float32' + if not hasattr(self, 'out_typename') + else self.out_typename + ) self.inputs = {'X': ipt.astype(in_typename)} self.outputs = {'Out': ipt.astype(in_typename).astype(out_typename)} @@ -77,12 +82,12 @@ def test_check_output(self): class TestCastOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of cast_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.XPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.XPUPlace(0) + ) self.assertRaises(TypeError, fluid.layers.cast, x1, 'int32') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_clip_by_norm_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_clip_by_norm_op_xpu.py index 32838a21a5afc..1bfa96b5975de 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_clip_by_norm_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_clip_by_norm_op_xpu.py @@ -22,7 +22,6 @@ class TestXPUClipByNormOp(XPUOpTest): - def setUp(self): self.op_type = "clip_by_norm" self.dtype = np.float32 @@ -50,26 +49,23 @@ def test_check_output(self): self.check_output_with_place(place) def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1.0 class TestCase1(TestXPUClipByNormOp): - def initTestCase(self): - self.shape = (100, ) + self.shape = (100,) self.max_norm = 1e20 class TestCase2(TestXPUClipByNormOp): - def initTestCase(self): self.shape = (16, 16) self.max_norm = 0.1 class TestCase3(TestXPUClipByNormOp): - def initTestCase(self): self.shape = (4, 8, 16) self.max_norm = 1.0 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_clip_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_clip_op_xpu.py index 5b61bfa8d1be9..cd6dd33b6facf 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_clip_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_clip_op_xpu.py @@ -23,17 +23,19 @@ from paddle.fluid import Program, program_guard from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class XPUTestClipOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'clip' self.use_dynamic_create_class = False class TestClipOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -90,28 +92,24 @@ def test_check_output(self): paddle.disable_static() class TestClipOp1(TestClipOp): - def init_data(self): self.shape = (8, 16, 8) self.max = 0.7 self.min = 0.0 class TestClipOp2(TestClipOp): - def init_data(self): self.shape = (8, 16) self.max = 1.0 self.min = 0.0 class TestClipOp3(TestClipOp): - def init_data(self): self.shape = (4, 8, 16) self.max = 0.7 self.min = 0.2 class TestClipOp4(TestClipOp): - def init_data(self): self.shape = (4, 8, 8) self.max = 0.7 @@ -120,7 +118,6 @@ def init_data(self): self.inputs['Min'] = np.array([0.3]).astype('float32') class TestClipOp5(TestClipOp): - def init_data(self): self.shape = (4, 8, 16) self.max = 0.5 @@ -128,7 +125,6 @@ def init_data(self): class TestClipOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): @@ -148,7 +144,6 @@ def test_dtype(): class TestClipAPI(unittest.TestCase): - def _executed_api(self, x, min=None, max=None): return paddle.clip(x, min, max) @@ -160,8 +155,11 @@ def test_clip(self): min = fluid.data(name='min', shape=[1], dtype='float32') max = fluid.data(name='max', shape=[1], dtype='float32') - place = fluid.XPUPlace( - 0) if fluid.core.is_compiled_with_xpu() else fluid.CPUPlace() + place = ( + fluid.XPUPlace(0) + if fluid.core.is_compiled_with_xpu() + else fluid.CPUPlace() + ) exe = fluid.Executor(place) out_1 = self._executed_api(images, min=min, max=max) @@ -170,7 +168,7 @@ def test_clip(self): out_4 = self._executed_api(images, max=0.7) out_5 = self._executed_api(images, min=min) out_6 = self._executed_api(images, max=max) - out_7 = self._executed_api(images, max=-1.) + out_7 = self._executed_api(images, max=-1.0) out_8 = self._executed_api(images) res1, res2, res3, res4, res5, res6, res7, res8 = exe.run( @@ -178,9 +176,10 @@ def test_clip(self): feed={ "image": data, "min": np.array([0.2]).astype('float32'), - "max": np.array([0.8]).astype('float32') + "max": np.array([0.8]).astype('float32'), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6, out_7, out_8], + ) np.testing.assert_allclose(res1, data.clip(0.2, 0.8)) np.testing.assert_allclose(res2, data.clip(0.2, 0.9)) @@ -194,8 +193,11 @@ def test_clip(self): def test_clip_dygraph(self): paddle.disable_static() - place = fluid.XPUPlace( - 0) if fluid.core.is_compiled_with_xpu() else fluid.CPUPlace() + place = ( + fluid.XPUPlace(0) + if fluid.core.is_compiled_with_xpu() + else fluid.CPUPlace() + ) paddle.disable_static(place) data_shape = [1, 9, 9, 4] data = np.random.random(data_shape).astype('float32') @@ -223,7 +225,6 @@ def test_errors(self): class TestInplaceClipAPI(TestClipAPI): - def _executed_api(self, x, min=None, max=None): return x.clip_(min, max) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_coalesce_tensor_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_coalesce_tensor_op_xpu.py index 9b295aa12cbb8..5370c1947a8de 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_coalesce_tensor_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_coalesce_tensor_op_xpu.py @@ -22,19 +22,21 @@ alignment = 256 import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestCoalesceTensorOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'coalesce_tensor' self.use_dynamic_create_class = False class TestAllocContinuousSpace(XPUOpTest): - def setUp(self): self.op_type = "coalesce_tensor" self.use_xpu = True @@ -45,12 +47,13 @@ def setUp(self): self.set_constant = attrs["set_constant"] self.Inputs = self.init_input() self.Outputs, self.FusedOutput = self.init_output( - self.Inputs, self.set_constant, self.constant) + self.Inputs, self.set_constant, self.constant + ) self.inputs = {'Input': self.Inputs} self.attrs = attrs self.outputs = { 'Output': self.Outputs, - 'FusedOutput': self.FusedOutput + 'FusedOutput': self.FusedOutput, } def init_dtype(self): @@ -61,8 +64,9 @@ def init_input(self): inputs.append(("x1", np.random.random([20, 3]).astype(self.dtype))) inputs.append(("x2", np.random.random([20]).astype(self.dtype))) inputs.append(("x3", np.random.random([1]).astype(self.dtype))) - inputs.append(("x4", np.random.random([200, - 30]).astype(self.dtype))) + inputs.append( + ("x4", np.random.random([200, 30]).astype(self.dtype)) + ) inputs.append(("x5", np.random.random([30]).astype(self.dtype))) inputs.append(("x6", np.random.random([1]).astype(self.dtype))) return inputs @@ -72,7 +76,7 @@ def init_attr(self): "copy_data": True, "set_constant": False, "constant": 0.0, - "dtype": self.fluid_dtype + "dtype": self.fluid_dtype, } def init_output(self, input_list, set_constant, constant): @@ -88,33 +92,37 @@ def init_output(self, input_list, set_constant, constant): coalesce_tensor_var = np.concatenate([input for input in inputs]) if set_constant: - coalesce_tensor_var = np.ones( - (len(coalesce_tensor_var))) * constant - outputs = [(out[0], - np.ones(out[1].shape).astype(self.dtype) * constant) - for out in outputs] + coalesce_tensor_var = ( + np.ones((len(coalesce_tensor_var))) * constant + ) + outputs = [ + ( + out[0], + np.ones(out[1].shape).astype(self.dtype) * constant, + ) + for out in outputs + ] return outputs, coalesce_tensor_var def test_check_output(self): - self.check_output_with_place(place=core.XPUPlace(0), - no_check_set=["FusedOutput"], - atol=1e-5) + self.check_output_with_place( + place=core.XPUPlace(0), no_check_set=["FusedOutput"], atol=1e-5 + ) class TestAllocContinuousSpace2(TestAllocContinuousSpace): - def init_attr(self): return { "copy_data": False, "set_constant": True, "constant": 0.5, "dtype": self.fluid_dtype, - "user_defined_size_of_dtype": 2 + "user_defined_size_of_dtype": 2, } def test_check_output(self): - self.check_output_with_place(place=core.XPUPlace(0), - no_check_set=["FusedOutput"], - atol=1e-5) + self.check_output_with_place( + place=core.XPUPlace(0), no_check_set=["FusedOutput"], atol=1e-5 + ) support_types = get_xpu_op_support_types('coalesce_tensor') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_collective_allgather_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_collective_allgather_xpu.py index eed540811159f..568b3039a955e 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_collective_allgather_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_collective_allgather_xpu.py @@ -22,33 +22,38 @@ sys.path.append("..") -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestCAllgatherOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'c_allgather' self.use_dynamic_create_class = False class TestCAllgatherOp(TestDistBase): - def _setup_config(self): pass def test_allgather(self): - self.check_with_place("collective_allgather_op_xpu.py", "allgather", - self.in_type_str) + self.check_with_place( + "collective_allgather_op_xpu.py", "allgather", self.in_type_str + ) support_types = get_xpu_op_support_types('c_allgather') for stype in support_types: - create_test_class(globals(), - XPUTestCAllgatherOP, - stype, - ignore_device_version=[core.XPUVersion.XPU1]) + create_test_class( + globals(), + XPUTestCAllgatherOP, + stype, + ignore_device_version=[core.XPUVersion.XPU1], + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_collective_allreduce_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_collective_allreduce_xpu.py index 57a96530c5592..3fa7405d54890 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_collective_allreduce_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_collective_allreduce_xpu.py @@ -22,33 +22,38 @@ sys.path.append("..") -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestCAllreduceOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'c_allreduce_sum' self.use_dynamic_create_class = False class TestCAllreduceOp(TestDistBase): - def _setup_config(self): pass def test_allreduce(self): - self.check_with_place("collective_allreduce_op_xpu.py", "allreduce", - self.in_type_str) + self.check_with_place( + "collective_allreduce_op_xpu.py", "allreduce", self.in_type_str + ) support_types = get_xpu_op_support_types('c_allreduce_sum') for stype in support_types: - create_test_class(globals(), - XPUTestCAllreduceOP, - stype, - ignore_device_version=[core.XPUVersion.XPU1]) + create_test_class( + globals(), + XPUTestCAllreduceOP, + stype, + ignore_device_version=[core.XPUVersion.XPU1], + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_collective_base_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_collective_base_xpu.py index 74e7267236927..1c00f3b94b860 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_collective_base_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_collective_base_xpu.py @@ -50,10 +50,10 @@ def DataTypeCast(date_type): class TestCollectiveRunnerBase(object): - def get_model(self, train_prog, startup_prog): raise NotImplementedError( - "get model should be implemented by child class.") + "get model should be implemented by child class." + ) def wait_server_ready(self, endpoints): while True: @@ -61,13 +61,15 @@ def wait_server_ready(self, endpoints): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, - 1) + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT, 1 + ) result = sock.connect_ex((ip_port[0], int(ip_port[1]))) if result != 0: @@ -75,44 +77,51 @@ def wait_server_ready(self, endpoints): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + - str(not_ready_endpoints) + "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: break + # endpoints should be ["ip1:port1","ip2:port2"] -#endpoints should be ["ip1:port1","ip2:port2"] - - def initCommunicator(self, program, rank, nranks, wait_port, - current_endpoint, endpoints): + def initCommunicator( + self, program, rank, nranks, wait_port, current_endpoint, endpoints + ): other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) if rank == 0 and wait_port: self.wait_server_ready(other_endpoints) block = program.global_block() - bkcl_id_var = block.create_var(name=nameGen.generate('bkcl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) - - block.append_op(type='c_gen_bkcl_id', - inputs={}, - outputs={'Out': bkcl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) - - block.append_op(type='c_comm_init', - inputs={'X': bkcl_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': self.global_ring_id - }) + bkcl_id_var = block.create_var( + name=nameGen.generate('bkcl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) + + block.append_op( + type='c_gen_bkcl_id', + inputs={}, + outputs={'Out': bkcl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) + + block.append_op( + type='c_comm_init', + inputs={'X': bkcl_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': self.global_ring_id, + }, + ) def run_trainer(self, args): train_prog = fluid.Program() @@ -121,8 +130,9 @@ def run_trainer(self, args): rank = args["trainerid"] current_endpoint = args["currentendpoint"] nranks = 2 - self.initCommunicator(startup_prog, rank, nranks, True, - current_endpoint, endpoints) + self.initCommunicator( + startup_prog, rank, nranks, True, current_endpoint, endpoints + ) self.rank = rank result = self.get_model(train_prog, startup_prog) device_id = int(os.getenv("FLAGS_selected_xpus", "0")) @@ -131,11 +141,12 @@ def run_trainer(self, args): exe.run(startup_prog) np.random.seed(os.getpid()) np_data_type = DataTypeCast(args["data_type"]) - indata = np.random.uniform(low=-10.0, high=10.0, - size=(10, 1000)).astype(np_data_type) - out = exe.run(train_prog, - feed={'tindata': indata}, - fetch_list=[result.name]) + indata = np.random.uniform( + low=-10.0, high=10.0, size=(10, 1000) + ).astype(np_data_type) + out = exe.run( + train_prog, feed={'tindata': indata}, fetch_list=[result.name] + ) sys.stdout.buffer.write(pickle.dumps(out[0])) @@ -157,12 +168,13 @@ def runtime_main(test_class, col_type, sub_type): class TestDistBase(unittest.TestCase): - def setUp(self): self._port_set = set() self._trainers = 2 self._ps_endpoints = "127.0.0.1:%s,127.0.0.1:%s" % ( - self._find_free_port(), self._find_free_port()) + self._find_free_port(), + self._find_free_port(), + ) self._python_interp = sys.executable self.temp_dir = tempfile.TemporaryDirectory() @@ -171,10 +183,10 @@ def tearDown(self): self.temp_dir.cleanup() def _find_free_port(self): - def __free_port(): - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.bind(('', 0)) return s.getsockname()[1] @@ -192,7 +204,7 @@ def _run_cluster(self, model_file, envs): "PADDLE_TRAINER_ID": "0", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w0_ep + "PADDLE_CURRENT_ENDPOINT": w0_ep, } env1 = { @@ -200,9 +212,9 @@ def _run_cluster(self, model_file, envs): "PADDLE_TRAINER_ID": "1", "PADDLE_TRAINERS_NUM": "2", "PADDLE_TRAINER_ENDPOINTS": self._ps_endpoints, - "PADDLE_CURRENT_ENDPOINT": w1_ep + "PADDLE_CURRENT_ENDPOINT": w1_ep, } - #update environment + # update environment env0.update(envs) env1.update(envs) tr_cmd = "%s %s" @@ -215,14 +227,16 @@ def _run_cluster(self, model_file, envs): tr0_proc = subprocess.Popen( tr0_cmd.strip().split(), stdout=subprocess.PIPE, - #stderr=tr0_pipe, - env=env0) + # stderr=tr0_pipe, + env=env0, + ) tr1_proc = subprocess.Popen( tr0_cmd.strip().split(), stdout=subprocess.PIPE, - #stderr=tr1_pipe, - env=env1) + # stderr=tr1_pipe, + env=env1, + ) tr0_out, tr0_err = tr0_proc.communicate() tr1_out, tr1_err = tr1_proc.communicate() @@ -231,15 +245,21 @@ def _run_cluster(self, model_file, envs): # close trainer file tr0_pipe.close() tr1_pipe.close() - return pickle.loads(tr0_out), pickle.loads( - tr1_out), tr0_proc.pid, tr1_proc.pid - - def check_with_place(self, - model_file, - col_type, - data_type, - check_error_log=False, - need_envs={}): + return ( + pickle.loads(tr0_out), + pickle.loads(tr1_out), + tr0_proc.pid, + tr1_proc.pid, + ) + + def check_with_place( + self, + model_file, + col_type, + data_type, + check_error_log=False, + need_envs={}, + ): required_envs = { "FLAGS_eager_delete_tensor_gb": "0.0", "PATH": os.getenv("PATH"), @@ -254,14 +274,17 @@ def check_with_place(self, required_envs["GLOG_v"] = "3" required_envs["GLOG_logtostderr"] = "1" tr0_out, tr1_out, pid0, pid1 = self._run_cluster( - model_file, required_envs) + model_file, required_envs + ) np_data_type = DataTypeCast(data_type) np.random.seed(pid0) - input1 = np.random.uniform(low=-10.0, high=10.0, - size=(10, 1000)).astype(np_data_type) + input1 = np.random.uniform( + low=-10.0, high=10.0, size=(10, 1000) + ).astype(np_data_type) np.random.seed(pid1) - input2 = np.random.uniform(low=-10.0, high=10.0, - size=(10, 1000)).astype(np_data_type) + input2 = np.random.uniform( + low=-10.0, high=10.0, size=(10, 1000) + ).astype(np_data_type) if col_type == "allgather": need_result = np.vstack((input1, input2)) np.testing.assert_allclose(tr0_out, need_result) @@ -275,38 +298,33 @@ def check_with_place(self, np.testing.assert_allclose(tr1_out, need_result) elif col_type == "scatter": need_result = input2 - need_result1 = need_result[0:need_result.shape[0] // 2] - need_result2 = need_result[need_result.shape[0] // 2:] + need_result1 = need_result[0 : need_result.shape[0] // 2] + need_result2 = need_result[need_result.shape[0] // 2 :] np.testing.assert_allclose(tr0_out, need_result1) np.testing.assert_allclose(tr1_out, need_result2) elif col_type == "allreduce": need_result = input1 + input2 - np.testing.assert_allclose(tr0_out, - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out, - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out, need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out, need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "reduce_scatter": tmp = input1 + input2 - need_result1 = tmp[0:tmp.shape[0] // 2] - need_result2 = tmp[tmp.shape[0] // 2:] - np.testing.assert_allclose(tr0_out, - need_result1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out, - need_result2, - rtol=1e-05, - atol=1e-05) + need_result1 = tmp[0 : tmp.shape[0] // 2] + need_result2 = tmp[tmp.shape[0] // 2 :] + np.testing.assert_allclose( + tr0_out, need_result1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out, need_result2, rtol=1e-05, atol=1e-05 + ) elif col_type == "sendrecv": need_result = input1 - np.testing.assert_allclose(tr1_out, - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr1_out, need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "identity": need_result1 = input1 need_result2 = input2 @@ -324,35 +342,29 @@ def check_with_place(self, np.testing.assert_allclose(tr1_out, need_result2) elif col_type == "concat": need_result = np.concatenate((input1, input2), axis=1) - np.testing.assert_allclose(tr0_out, - need_result, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out, - need_result, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out, need_result, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out, need_result, rtol=1e-05, atol=1e-05 + ) elif col_type == "split": need_result1 = np.split(input1, 2, axis=1)[0] need_result2 = np.split(input2, 2, axis=1)[1] - np.testing.assert_allclose(tr0_out, - need_result1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out, - need_result2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr0_out, need_result1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out, need_result2, rtol=1e-05, atol=1e-05 + ) elif col_type == "sendrecv_array": need_result1 = np.array([[0, 1, 2]]) need_result2 = np.array([[3, 4, 5]]) - np.testing.assert_allclose(tr1_out[0][0], - need_result1, - rtol=1e-05, - atol=1e-05) - np.testing.assert_allclose(tr1_out[0][1], - need_result2, - rtol=1e-05, - atol=1e-05) + np.testing.assert_allclose( + tr1_out[0][0], need_result1, rtol=1e-05, atol=1e-05 + ) + np.testing.assert_allclose( + tr1_out[0][1], need_result2, rtol=1e-05, atol=1e-05 + ) else: pass diff --git a/python/paddle/fluid/tests/unittests/xpu/test_collective_identity_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_collective_identity_xpu.py index 07c523629b600..4d84efebbe2f2 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_collective_identity_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_collective_identity_xpu.py @@ -22,33 +22,38 @@ sys.path.append("..") -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestCIdentityOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'c_identity' self.use_dynamic_create_class = False class TestCIdentityOp(TestDistBase): - def _setup_config(self): pass def test_identity(self): - self.check_with_place("collective_identity_op_xpu.py", "identity", - self.in_type_str) + self.check_with_place( + "collective_identity_op_xpu.py", "identity", self.in_type_str + ) support_types = get_xpu_op_support_types('c_identity') for stype in support_types: - create_test_class(globals(), - XPUTestCIdentityOP, - stype, - ignore_device_version=[core.XPUVersion.XPU1]) + create_test_class( + globals(), + XPUTestCIdentityOP, + stype, + ignore_device_version=[core.XPUVersion.XPU1], + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_compare_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_compare_op_xpu.py index cdaf767a1de68..8e7fb2eb3421d 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_compare_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_compare_op_xpu.py @@ -24,7 +24,6 @@ class TestCompareOpBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.config() @@ -33,10 +32,12 @@ def setUp(self): self.outputs = {'Out': self.result} def set_case(self): - self.x = np.random.uniform(self.lbound, self.hbound, - self.x_shape).astype(self.dtype) - self.y = np.random.uniform(self.lbound, self.hbound, - self.y_shape).astype(self.dtype) + self.x = np.random.uniform( + self.lbound, self.hbound, self.x_shape + ).astype(self.dtype) + self.y = np.random.uniform( + self.lbound, self.hbound, self.y_shape + ).astype(self.dtype) self.result = self.compute(self.x, self.y) def config(self): @@ -54,13 +55,11 @@ def test_check_output(self): class XPUTestLessThanOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'less_than' self.use_dynamic_create_class = False class LessThanOpTestCase1(TestCompareOpBase): - def config(self): self.dtype = self.in_type self.op_type = 'less_than' @@ -74,7 +73,6 @@ def set_data(self): self.y_shape = [11, 17] class LessThanOpTestCase2(LessThanOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -82,7 +80,6 @@ def set_data(self): self.y_shape = [1] class LessThanOpTestCase3(LessThanOpTestCase1): - def set_data(self): self.lbound = -300 self.hbound = 300 @@ -90,7 +87,6 @@ def set_data(self): self.y_shape = [1] class LessThanOpTestCase4(LessThanOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -98,7 +94,6 @@ def set_data(self): self.y_shape = [1] class LessThanOpTestCase5(LessThanOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -112,13 +107,11 @@ def set_data(self): class XPUTestLessEqualOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'less_equal' self.use_dynamic_create_class = False class LessEqualOpTestCase1(TestCompareOpBase): - def config(self): self.dtype = self.in_type self.op_type = 'less_equal' @@ -132,7 +125,6 @@ def set_data(self): self.y_shape = [11, 17] class LessEqualOpTestCase2(LessEqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -140,7 +132,6 @@ def set_data(self): self.y_shape = [11, 17, 255] class LessEqualOpTestCase3(LessEqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -148,7 +139,6 @@ def set_data(self): self.y_shape = [1] class LessEqualOpTestCase4(LessEqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -156,7 +146,6 @@ def set_data(self): self.y_shape = [1] class LessEqualOpTestCase5(LessEqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -170,13 +159,11 @@ def set_data(self): class XPUTestGreaterThanOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'greater_than' self.use_dynamic_create_class = False class GreaterThanOpTestCase1(TestCompareOpBase): - def config(self): self.dtype = self.in_type self.op_type = 'greater_than' @@ -190,7 +177,6 @@ def set_data(self): self.y_shape = [128, 128, 512] class GreaterThanOpTestCase2(GreaterThanOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -198,7 +184,6 @@ def set_data(self): self.y_shape = [1] class GreaterThanOpTestCase3(GreaterThanOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -206,7 +191,6 @@ def set_data(self): self.y_shape = [1] class GreaterThanOpTestCase4(GreaterThanOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -214,7 +198,6 @@ def set_data(self): self.y_shape = [11, 17] class GreaterThanOpTestCase5(GreaterThanOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -228,13 +211,11 @@ def set_data(self): class XPUTestGreaterEqualOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'greater_equal' self.use_dynamic_create_class = False class GreaterEqualOpTestCase1(TestCompareOpBase): - def config(self): self.dtype = self.in_type self.op_type = 'greater_equal' @@ -248,7 +229,6 @@ def set_data(self): self.y_shape = [10, 10, 20, 20] class GreaterEqualOpTestCase2(GreaterEqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -256,7 +236,6 @@ def set_data(self): self.y_shape = [10, 10] class GreaterEqualOpTestCase3(GreaterEqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -264,7 +243,6 @@ def set_data(self): self.y_shape = [1] class GreaterEqualOpTestCase4(GreaterEqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -272,7 +250,6 @@ def set_data(self): self.y_shape = [1] class GreaterEqualOpTestCase5(GreaterEqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -286,13 +263,11 @@ def set_data(self): class XPUTestEqualOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'equal' self.use_dynamic_create_class = False class EqualOpTestCase1(TestCompareOpBase): - def config(self): self.dtype = self.in_type self.op_type = 'equal' @@ -306,7 +281,6 @@ def set_data(self): self.y_shape = [10, 30, 15] class EqualOpTestCase2(EqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -314,7 +288,6 @@ def set_data(self): self.y_shape = [1] class EqualOpTestCase3(EqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -322,7 +295,6 @@ def set_data(self): self.y_shape = [10, 30] class EqualOpTestCase4(EqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -330,7 +302,6 @@ def set_data(self): self.y_shape = [256, 256, 10] class EqualOpTestCase5(EqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 @@ -344,13 +315,11 @@ def set_data(self): class XPUTestNotEqualOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'not_equal' self.use_dynamic_create_class = False class NotEqualOpTestCase1(TestCompareOpBase): - def config(self): self.dtype = self.in_type self.op_type = 'not_equal' @@ -364,7 +333,6 @@ def set_data(self): self.y_shape = [1] class NotEqualOpTestCase2(NotEqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -372,7 +340,6 @@ def set_data(self): self.y_shape = [11, 17] class NotEqualOpTestCase3(NotEqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -380,7 +347,6 @@ def set_data(self): self.y_shape = [1] class NotEqualOpTestCase4(NotEqualOpTestCase1): - def set_data(self): self.lbound = -200 self.hbound = 200 @@ -388,7 +354,6 @@ def set_data(self): self.y_shape = [256, 256, 10] class NotEqualOpTestCase5(NotEqualOpTestCase1): - def set_data(self): self.lbound = -100 self.hbound = 100 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_concat_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_concat_op_xpu.py index 951eec301c095..e7e4ca073c05e 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_concat_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_concat_op_xpu.py @@ -21,19 +21,21 @@ import paddle from op_test import skip_check_grad_ci from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestConcatOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'concat' self.use_dynamic_create_class = False class TestConcatOp(XPUOpTest): - def setUp(self): self.set_xpu() self.op_type = "concat" @@ -47,14 +49,16 @@ def setUp(self): self.attrs = {'axis': self.axis} if self.axis < 0: self.actual_axis = self.axis + len(self.x0.shape) - self.actual_axis = self.actual_axis if self.actual_axis > 0 else 0 + self.actual_axis = ( + self.actual_axis if self.actual_axis > 0 else 0 + ) else: self.actual_axis = self.axis self.outputs = { - 'Out': - np.concatenate((self.x0, self.x1, self.x2), - axis=self.actual_axis) + 'Out': np.concatenate( + (self.x0, self.x1, self.x2), axis=self.actual_axis + ) } def set_inputs(self): @@ -83,12 +87,10 @@ def test_check_grad(self): self.check_grad_with_place(place, ['x2'], 'Out') class TestConcatOpAxis0XPU(TestConcatOp): - def init_axis(self): self.axis = 0 class TestConcatOpAxis1XPU(TestConcatOp): - def set_inputs(self): self.x0 = np.random.random((5, 1, 4, 5)).astype(self.dtype) self.x1 = np.random.random((5, 2, 4, 5)).astype(self.dtype) @@ -98,34 +100,29 @@ def init_axis(self): self.axis = 1 class TestConcatOpAxis2XPU(TestConcatOp): - def init_axis(self): self.axis = 2 class TestConcatOpAxis3XPU(TestConcatOp): - def init_axis(self): self.axis = 3 class TestConcatOpAxisNeg1XPU(TestConcatOp): - def init_axis(self): self.axis = -1 class TestConcatOpAxisNeg2XPU(TestConcatOp): - def init_axis(self): self.axis = -2 class TestConcatOpAxisNeg3XPU(TestConcatOp): - def init_axis(self): self.axis = -3 @skip_check_grad_ci( - reason="The function 'check_grad' for large inputs is too slow.") + reason="The function 'check_grad' for large inputs is too slow." + ) class TestConcatOp3(TestConcatOp): - def set_inputs(self): self.x0 = np.random.random((1, 256, 170, 256)).astype(self.dtype) self.x1 = np.random.random((1, 128, 170, 256)).astype(self.dtype) @@ -136,11 +133,9 @@ def test_check_grad(self): pass @skip_check_grad_ci( - reason= - "This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." + reason="This test will meet fetch error when there is a null grad. The detailed information is in PR#17015." ) class TestConcatOp4(TestConcatOp): - def set_inputs(self): self.x0 = np.random.random((2, 3, 4, 5)).astype(self.dtype) self.x1 = np.random.random((2, 3, 4, 5)).astype(self.dtype) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_conv2d_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_conv2d_op_xpu.py index b7924818177e7..973e2908c4ecc 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_conv2d_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_conv2d_op_xpu.py @@ -21,25 +21,34 @@ import paddle.fluid.core as core from op_test_xpu import XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper - - -def conv2d_forward_naive(input, - filter, - group, - conv_param, - padding_algorithm='EXPLICIT', - data_format='NCHW'): +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) + + +def conv2d_forward_naive( + input, + filter, + group, + conv_param, + padding_algorithm='EXPLICIT', + data_format='NCHW', +): if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if data_format not in ["NCHW", "NHWC"]: - raise ValueError("Unknown Attr(data_format): '%s' ." - "It can only be 'NCHW' or 'NHWC'." % str(data_format)) + raise ValueError( + "Unknown Attr(data_format): '%s' ." + "It can only be 'NCHW' or 'NHWC'." % str(data_format) + ) - channel_last = (data_format == "NHWC") + channel_last = data_format == "NHWC" if channel_last: input = np.transpose(input, [0, 3, 1, 2]) @@ -52,17 +61,22 @@ def conv2d_forward_naive(input, sub_out_c = out_c // group sub_f_n = f_n // group - stride, pad, dilation = conv_param['stride'], conv_param['pad'], conv_param[ - 'dilation'] + stride, pad, dilation = ( + conv_param['stride'], + conv_param['pad'], + conv_param['dilation'], + ) # update pad and dilation def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -82,39 +96,52 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): if len(pad) == 4: pad_h_0, pad_h_1 = pad[0], pad[1] pad_w_0, pad_w_1 = pad[2], pad[3] - out_h = 1 + (in_h + pad_h_0 + pad_h_1 - (dilation[0] * - (f_h - 1) + 1)) // stride[0] - out_w = 1 + (in_w + pad_w_0 + pad_w_1 - (dilation[1] * - (f_w - 1) + 1)) // stride[1] + out_h = ( + 1 + + (in_h + pad_h_0 + pad_h_1 - (dilation[0] * (f_h - 1) + 1)) + // stride[0] + ) + out_w = ( + 1 + + (in_w + pad_w_0 + pad_w_1 - (dilation[1] * (f_w - 1) + 1)) + // stride[1] + ) out = np.zeros((out_n, out_c, out_h, out_w)) - d_bolck_h = (dilation[0] * (f_h - 1) + 1) - d_bolck_w = (dilation[1] * (f_w - 1) + 1) + d_bolck_h = dilation[0] * (f_h - 1) + 1 + d_bolck_w = dilation[1] * (f_w - 1) + 1 - input_pad = np.pad(input, - ((0, 0), (0, 0), (pad_h_0, pad_h_1), (pad_w_0, pad_w_1)), - mode='constant', - constant_values=0) + input_pad = np.pad( + input, + ((0, 0), (0, 0), (pad_h_0, pad_h_1), (pad_w_0, pad_w_1)), + mode='constant', + constant_values=0, + ) filter_dilation = np.zeros((f_n, f_c, d_bolck_h, d_bolck_w)) - filter_dilation[:, :, 0:d_bolck_h:dilation[0], - 0:d_bolck_w:dilation[1]] = filter + filter_dilation[ + :, :, 0 : d_bolck_h : dilation[0], 0 : d_bolck_w : dilation[1] + ] = filter for i in range(out_h): for j in range(out_w): for g in range(group): - input_pad_masked = \ - input_pad[:, g * f_c:(g + 1) * f_c, - i * stride[0]:i * stride[0] + d_bolck_h, - j * stride[1]:j * stride[1] + d_bolck_w] - - f_sub = filter_dilation[g * sub_f_n:(g + 1) * sub_f_n, :, :, :] + input_pad_masked = input_pad[ + :, + g * f_c : (g + 1) * f_c, + i * stride[0] : i * stride[0] + d_bolck_h, + j * stride[1] : j * stride[1] + d_bolck_w, + ] + + f_sub = filter_dilation[ + g * sub_f_n : (g + 1) * sub_f_n, :, :, : + ] # sub_f_n == sub_out_c for k in range(sub_out_c): # Multiplication of Corresponding Elements, then sum all - out[:, g * sub_out_c + k, i, j] = \ - np.sum(input_pad_masked * f_sub[k, :, :, :], - axis=(1, 2, 3)) + out[:, g * sub_out_c + k, i, j] = np.sum( + input_pad_masked * f_sub[k, :, :, :], axis=(1, 2, 3) + ) if channel_last: out = np.transpose(out, [0, 2, 3, 1]) @@ -123,9 +150,7 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): def create_test_channel_last_class(parent): - class TestChannelLastCase(parent): - def init_data_format(self): self.data_format = "NHWC" @@ -139,9 +164,7 @@ def init_test_case_2(self): def create_test_padding_SAME_class(parent): - class TestPaddingSMAECase(parent): - def init_paddings(self): self.pad = [0, 0] self.padding_algorithm = "SAME" @@ -152,9 +175,7 @@ def init_paddings(self): def create_test_padding_VALID_class(parent): - class TestPaddingVALIDCase(parent): - def init_paddings(self): self.pad = [1, 1] self.padding_algorithm = "VALID" @@ -165,13 +186,11 @@ def init_paddings(self): class XPUTestConv2DOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'conv2d' self.use_dynamic_create_class = False class TestConv2DOp(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -190,7 +209,7 @@ def setUp(self): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } np.random.seed(100) @@ -205,16 +224,18 @@ def setUp(self): else: input2 = input np.random.seed(1) - filter = np.random.uniform(-1, 1, - self.filter_size).astype(self.dtype) + filter = np.random.uniform(-1, 1, self.filter_size).astype( + self.dtype + ) - output, _, _, _, _ = conv2d_forward_naive(input2, filter, - self.groups, conv2d_param) + output, _, _, _, _ = conv2d_forward_naive( + input2, filter, self.groups, conv2d_param + ) output = output.astype(self.dtype) self.inputs = { 'Input': XPUOpTest.np_dtype_to_fluid_dtype(input), - 'Filter': XPUOpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': XPUOpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -224,15 +245,15 @@ def setUp(self): 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, 'data_format': self.data_format, - 'fuse_relu_before_depthwise_conv': - self.fuse_relu_before_depthwise_conv, - 'exhaustive_search': self.exhaustive_search + 'fuse_relu_before_depthwise_conv': self.fuse_relu_before_depthwise_conv, + 'exhaustive_search': self.exhaustive_search, } self.outputs = {'Output': output} def has_cuda(self): - return core.is_compiled_with_cuda() and (self.use_cudnn - or self.use_cuda) + return core.is_compiled_with_cuda() and ( + self.use_cudnn or self.use_cuda + ) def test_check_output(self): if core.is_compiled_with_xpu(): @@ -240,33 +261,40 @@ def test_check_output(self): self.check_output_with_place(self.place) def test_check_grad(self): - if (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return if core.is_compiled_with_xpu(): paddle.enable_static() - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output') + self.check_grad_with_place( + self.place, {'Input', 'Filter'}, 'Output' + ) def test_check_grad_no_filter(self): - if (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return if core.is_compiled_with_xpu(): paddle.enable_static() - self.check_grad_with_place(self.place, ['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + self.place, ['Input'], 'Output', no_grad_set=set(['Filter']) + ) def test_check_grad_no_input(self): - if (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return if core.is_compiled_with_xpu(): paddle.enable_static() - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - no_grad_set=set(['Input'])) + self.check_grad_with_place( + self.place, ['Filter'], 'Output', no_grad_set=set(['Input']) + ) def init_test_case(self): self.pad = [0, 0] @@ -289,7 +317,6 @@ def init_kernel_type(self): pass class TestWithPad(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -299,7 +326,6 @@ def init_test_case(self): self.filter_size = [6, f_c, 3, 3] class TestWithStride(TestConv2DOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] @@ -309,7 +335,6 @@ def init_test_case(self): self.filter_size = [6, f_c, 3, 3] class TestWith1x1(TestConv2DOp): - def init_test_case(self): self.pad = [0, 0] self.stride = [1, 1] @@ -324,13 +349,11 @@ def init_group(self): # ---- test asymmetric padding ---- class XPUTestConv2DOp_v2(XPUOpTestWrapper): - def __init__(self): self.op_name = 'conv2d' self.use_dynamic_create_class = False class TestConv2DOp_v2(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -351,7 +374,7 @@ def setUp(self): conv2d_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } np.random.seed(100) @@ -366,17 +389,22 @@ def setUp(self): else: input2 = input np.random.seed(8) - filter = np.random.uniform(-1, 1, - self.filter_size).astype(self.dtype) - output, _, _, _, _ = conv2d_forward_naive(input2, filter, - self.groups, conv2d_param, - self.padding_algorithm, - self.data_format) + filter = np.random.uniform(-1, 1, self.filter_size).astype( + self.dtype + ) + output, _, _, _, _ = conv2d_forward_naive( + input2, + filter, + self.groups, + conv2d_param, + self.padding_algorithm, + self.data_format, + ) output = output.astype(self.dtype) self.inputs = { 'Input': XPUOpTest.np_dtype_to_fluid_dtype(input), - 'Filter': XPUOpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': XPUOpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -387,15 +415,15 @@ def setUp(self): 'use_cudnn': self.use_cudnn, 'use_mkldnn': self.use_mkldnn, 'data_format': self.data_format, - 'fuse_relu_before_depthwise_conv': - self.fuse_relu_before_depthwise_conv, - 'exhaustive_search': self.exhaustive_search + 'fuse_relu_before_depthwise_conv': self.fuse_relu_before_depthwise_conv, + 'exhaustive_search': self.exhaustive_search, } self.outputs = {'Output': output} def has_cuda(self): - return core.is_compiled_with_cuda() and (self.use_cudnn - or self.use_cuda) + return core.is_compiled_with_cuda() and ( + self.use_cudnn or self.use_cuda + ) def test_check_output(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode @@ -405,35 +433,42 @@ def test_check_output(self): def test_check_grad(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - if (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return if core.is_compiled_with_xpu(): paddle.enable_static() - self.check_grad_with_place(self.place, {'Input', 'Filter'}, - 'Output') + self.check_grad_with_place( + self.place, {'Input', 'Filter'}, 'Output' + ) def test_check_grad_no_filter(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - if (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return if core.is_compiled_with_xpu(): paddle.enable_static() - self.check_grad_with_place(self.place, ['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + self.place, ['Input'], 'Output', no_grad_set=set(['Filter']) + ) def test_check_grad_no_input(self): # TODO(wangzhongpu): support mkldnn op in dygraph mode - if (hasattr(self, "no_need_check_grad") - and self.no_need_check_grad == True): + if ( + hasattr(self, "no_need_check_grad") + and self.no_need_check_grad == True + ): return if core.is_compiled_with_xpu(): paddle.enable_static() - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - no_grad_set=set(['Input'])) + self.check_grad_with_place( + self.place, ['Filter'], 'Output', no_grad_set=set(['Input']) + ) def init_test_case(self): self.pad = [0, 0] @@ -463,13 +498,11 @@ def init_test_case_2(self): pass class TestConv2DOp_AsyPadding(TestConv2DOp_v2): - def init_paddings(self): self.pad = [0, 0, 0, 0] self.padding_algorithm = "EXPLICIT" class TestWithPad_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW @@ -482,7 +515,6 @@ def init_paddings(self): self.padding_algorithm = "EXPLICIT" class TestWithStride_AsyPadding(TestConv2DOp_v2): - def init_test_case(self): self.stride = [2, 2] self.input_size = [2, 3, 6, 6] # NCHW @@ -496,14 +528,13 @@ def init_paddings(self): class XPUTestConv2DOp_NHWC(XPUOpTestWrapper): - def __init__(self): self.op_name = 'conv2d' self.use_dynamic_create_class = False class TestConv2DOp_AsyPadding_NHWC( - XPUTestConv2DOp_v2.TestConv2DOp_AsyPadding): - + XPUTestConv2DOp_v2.TestConv2DOp_AsyPadding + ): def init_data_format(self): self.data_format = "NHWC" @@ -511,9 +542,9 @@ def init_test_case_2(self): N, C, H, W = self.input_size self.input_size = [N, H, W, C] - class TestWithPad_AsyPadding_NHWC(XPUTestConv2DOp_v2.TestWithPad_AsyPadding - ): - + class TestWithPad_AsyPadding_NHWC( + XPUTestConv2DOp_v2.TestWithPad_AsyPadding + ): def init_data_format(self): self.data_format = "NHWC" @@ -526,19 +557,21 @@ def init_test_case_2(self): for stype in ['float32']: create_test_class(globals(), XPUTestConv2DOp, stype) create_test_class(globals(), XPUTestConv2DOp_v2, stype) - create_test_class(globals(), - XPUTestConv2DOp_NHWC, - stype, - ignore_device_version=[core.XPUVersion.XPU1]) - -#---------- test SAME VALID ----------- -#create_test_padding_SAME_class(TestConv2DOp_AsyPadding) -#create_test_padding_SAME_class(TestWithPad_AsyPadding) -#create_test_padding_SAME_class(TestWithStride_AsyPadding) - -#create_test_padding_VALID_class(TestConv2DOp_AsyPadding) -#create_test_padding_VALID_class(TestWithPad_AsyPadding) -#create_test_padding_VALID_class(TestWithStride_AsyPadding) + create_test_class( + globals(), + XPUTestConv2DOp_NHWC, + stype, + ignore_device_version=[core.XPUVersion.XPU1], + ) + +# ---------- test SAME VALID ----------- +# create_test_padding_SAME_class(TestConv2DOp_AsyPadding) +# create_test_padding_SAME_class(TestWithPad_AsyPadding) +# create_test_padding_SAME_class(TestWithStride_AsyPadding) + +# create_test_padding_VALID_class(TestConv2DOp_AsyPadding) +# create_test_padding_VALID_class(TestWithPad_AsyPadding) +# create_test_padding_VALID_class(TestWithStride_AsyPadding) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_conv2d_transpose_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_conv2d_transpose_op_xpu.py index 49011dd36187d..47d13d04b3a1a 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_conv2d_transpose_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_conv2d_transpose_op_xpu.py @@ -19,7 +19,11 @@ import numpy as np from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle paddle.enable_static() @@ -28,9 +32,10 @@ def conv2dtranspose_forward_naive(input_, filter_, attrs): padding_algorithm = attrs['padding_algorithm'] if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if attrs['data_format'] == 'NHWC': input_ = np.transpose(input_, [0, 3, 1, 2]) @@ -41,18 +46,22 @@ def conv2dtranspose_forward_naive(input_, filter_, attrs): out_c = f_out_c * groups sub_in_c = in_c // groups - stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ - 'dilations'] + stride, pad, dilations = ( + attrs['strides'], + attrs['paddings'], + attrs['dilations'], + ) # update pad and dilation def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, - kernel_size, - kernel_stride): + for input_size, filter_size, stride_size in zip( + input_shape, kernel_size, kernel_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -86,43 +95,54 @@ def _get_padding_with_SAME(input_shape, kernel_size, kernel_stride): if 'output_padding' in attrs: out_pad_h = attrs['output_padding'][0] out_pad_w = attrs['output_padding'][1] - out = np.zeros((in_n, out_c, out_h + out_pad_h, out_w + out_pad_w), - dtype=input_.dtype) + out = np.zeros( + (in_n, out_c, out_h + out_pad_h, out_w + out_pad_w), dtype=input_.dtype + ) for n in range(in_n): for i in range(in_h): for j in range(in_w): for g in range(groups): - input_masked = input_[n, g * sub_in_c:(g + 1) * sub_in_c, i, - j] # (c) + input_masked = input_[ + n, g * sub_in_c : (g + 1) * sub_in_c, i, j + ] # (c) input_masked = np.reshape(input_masked, (sub_in_c, 1, 1)) input_masked = np.tile(input_masked, (1, f_h, f_w)) for k in range(f_out_c): tmp_out = np.sum( - input_masked * - filter_[g * sub_in_c:(g + 1) * sub_in_c, k, :, :], - axis=0) + input_masked + * filter_[ + g * sub_in_c : (g + 1) * sub_in_c, k, :, : + ], + axis=0, + ) i1, i2 = i * stride[0], i * stride[0] + d_bolck_h j1, j2 = j * stride[1], j * stride[1] + d_bolck_w - out[n, g * f_out_c + k, i1:i2:dilations[0], - j1:j2:dilations[1]] += tmp_out - - out = out[:, :, pad_h_0:out_h - pad_h_1 + out_pad_h, - pad_w_0:out_w - pad_w_1 + out_pad_w] + out[ + n, + g * f_out_c + k, + i1 : i2 : dilations[0], + j1 : j2 : dilations[1], + ] += tmp_out + + out = out[ + :, + :, + pad_h_0 : out_h - pad_h_1 + out_pad_h, + pad_w_0 : out_w - pad_w_1 + out_pad_w, + ] if attrs['data_format'] == 'NHWC': out = np.transpose(out, [0, 2, 3, 1]) return out class XPUTestConv2DTransposeOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'conv2d_transpose' self.use_dynamic_create_class = False class TestConv2DTransposeOp(XPUOpTest): - def setUp(self): # init as conv transpose self.need_check_grad = True @@ -151,7 +171,7 @@ def setUp(self): 'use_cudnn': self.use_cudnn, 'is_test': self.is_test, 'use_mkldnn': self.use_mkldnn, - 'data_format': self.data_format + 'data_format': self.data_format, } if self.output_size is not None: self.attrs['output_size'] = self.output_size @@ -160,7 +180,8 @@ def setUp(self): self.attrs['output_padding'] = self.output_padding output = conv2dtranspose_forward_naive( - input_, filter_, self.attrs).astype(self.dtype) + input_, filter_, self.attrs + ).astype(self.dtype) self.outputs = {'Output': output} @@ -169,20 +190,21 @@ def test_check_output(self): def test_check_grad_no_input(self): if self.need_check_grad: - self.check_grad_with_place(self.place, ['Filter'], - 'Output', - no_grad_set=set(['Input'])) + self.check_grad_with_place( + self.place, ['Filter'], 'Output', no_grad_set=set(['Input']) + ) def test_check_grad_no_filter(self): if self.need_check_grad: - self.check_grad_with_place(self.place, ['Input'], - 'Output', - no_grad_set=set(['Filter'])) + self.check_grad_with_place( + self.place, ['Input'], 'Output', no_grad_set=set(['Filter']) + ) def test_check_grad(self): if self.need_check_grad: - self.check_grad_with_place(self.place, set(['Input', 'Filter']), - 'Output') + self.check_grad_with_place( + self.place, set(['Input', 'Filter']), 'Output' + ) def init_test_case(self): self.pad = [0, 0] @@ -199,7 +221,6 @@ def init_op_type(self): self.op_type = "conv2d_transpose" class TestWithSymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -210,7 +231,6 @@ def init_test_case(self): self.filter_size = [f_c, 6, 3, 3] class TestWithAsymmetricPad(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 0, 1, 2] self.stride = [1, 1] @@ -221,7 +241,6 @@ def init_test_case(self): self.filter_size = [f_c, 6, 3, 3] class TestWithSAMEPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [2, 1] self.dilations = [1, 2] @@ -232,7 +251,6 @@ def init_test_case(self): self.padding_algorithm = 'SAME' class TestWithVALIDPad(TestConv2DTransposeOp): - def init_test_case(self): self.stride = [1, 1] self.dilations = [1, 1] @@ -243,7 +261,6 @@ def init_test_case(self): self.padding_algorithm = 'VALID' class TestWithGroups(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -254,7 +271,6 @@ def init_test_case(self): self.filter_size = [f_c, 3, 3, 3] class TestWithStride(TestConv2DTransposeOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [2, 2] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_deformable_conv_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_deformable_conv_op_xpu.py index 85b05e8990ba9..678afafc08c4b 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_deformable_conv_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_deformable_conv_op_xpu.py @@ -22,7 +22,11 @@ import paddle.fluid as fluid from op_test_xpu import OpTest, XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def dmc_bilinear(data_im, height, width, h, w): @@ -64,8 +68,11 @@ def dconv_im2col_gemm(input, offset, mask, filter, group, conv_param): assert f_c * group == in_c assert np.mod(out_c, group) == 0 - stride, pad, dilation = conv_param['stride'], conv_param['pad'],\ - conv_param['dilation'] + stride, pad, dilation = ( + conv_param['stride'], + conv_param['pad'], + conv_param['dilation'], + ) out_h = 1 + (in_h + 2 * pad[0] - (dilation[0] * (f_h - 1) + 1)) // stride[0] out_w = 1 + (in_w + 2 * pad[1] - (dilation[1] * (f_w - 1) + 1)) // stride[1] assert out_h == in_h @@ -78,31 +85,47 @@ def dconv_im2col_gemm(input, offset, mask, filter, group, conv_param): for w in range(out_w): for kh in range(f_h): for kw in range(f_w): - offset_h_table = \ - offset[n, ::2, h, w].reshape(f_h, f_w) - offset_w_table = \ - offset[n, 1::2, h, w].reshape(f_h, f_w) - mask_table = \ - mask[n, :, h, w].reshape(f_h, f_w) + offset_h_table = offset[n, ::2, h, w].reshape( + f_h, f_w + ) + offset_w_table = offset[n, 1::2, h, w].reshape( + f_h, f_w + ) + mask_table = mask[n, :, h, w].reshape(f_h, f_w) offset_h = offset_h_table[kh, kw] offset_w = offset_w_table[kh, kw] val = 0 - im_h = h * stride[0] + kh * dilation[0] \ - + offset_h - pad[0] - im_w = w * stride[0] + kw * dilation[0] \ - + offset_w - pad[1] - if im_h > -1 and im_w > -1 and \ - im_h < in_h and im_w < in_h: - val = dmc_bilinear(input[n, c], in_h, in_w, - im_h, im_w) + im_h = ( + h * stride[0] + + kh * dilation[0] + + offset_h + - pad[0] + ) + im_w = ( + w * stride[0] + + kw * dilation[0] + + offset_w + - pad[1] + ) + if ( + im_h > -1 + and im_w > -1 + and im_h < in_h + and im_w < in_h + ): + val = dmc_bilinear( + input[n, c], in_h, in_w, im_h, im_w + ) val_out = val * mask_table[kh, kw] - col_buffer[n, c * f_h * f_w + kh * f_w + kw, - h * in_w + w] = val_out + col_buffer[ + n, c * f_h * f_w + kh * f_w + kw, h * in_w + w + ] = val_out out = np.zeros((in_n, group, int(out_c // group), out_h * out_w)) weight = filter.reshape(group, int(out_c // group), f_c * f_h * f_w) col_buffer = col_buffer.reshape( - (in_n, group, int(in_c // group * f_h * f_w), in_h * in_w)) + (in_n, group, int(in_c // group * f_h * f_w), in_h * in_w) + ) for n in range(in_n): for g in range(group): out[n, g] = np.matmul(weight[g], col_buffer[n, g]) @@ -111,13 +134,11 @@ def dconv_im2col_gemm(input, offset, mask, filter, group, conv_param): class XPUTestModulatedDeformableConvOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'deformable_conv' self.use_dynamic_create_class = False class TestModulatedDeformableConvOp(XPUOpTest): - def setUp(self): self.op_type = "deformable_conv" self.dtype = self.in_type @@ -129,22 +150,23 @@ def setUp(self): conv_param = { 'stride': self.stride, 'pad': self.pad, - 'dilation': self.dilations + 'dilation': self.dilations, } input = np.random.random(self.input_size).astype(self.dtype) offset = 10 * np.random.random(self.offset_size).astype(self.dtype) mask = 10 * np.random.random(self.mask_size).astype(self.dtype) filter = np.random.random(self.filter_size).astype(self.dtype) - output = dconv_im2col_gemm(input, offset, mask, filter, self.groups, - conv_param) + output = dconv_im2col_gemm( + input, offset, mask, filter, self.groups, conv_param + ) output = output.astype(self.dtype) self.inputs = { 'Input': OpTest.np_dtype_to_fluid_dtype(input), 'Offset': OpTest.np_dtype_to_fluid_dtype(offset), 'Mask': OpTest.np_dtype_to_fluid_dtype(mask), - 'Filter': OpTest.np_dtype_to_fluid_dtype(filter) + 'Filter': OpTest.np_dtype_to_fluid_dtype(filter), } self.attrs = { 'strides': self.stride, @@ -165,9 +187,11 @@ def test_check_grad(self): if core.is_compiled_with_xpu(): paddle.enable_static() self.check_grad_with_place( - self.place, {'Input', 'Offset', 'Mask', 'Filter'}, + self.place, + {'Input', 'Offset', 'Mask', 'Filter'}, 'Output', - max_relative_error=0.06) + max_relative_error=0.06, + ) def init_test_case(self): self.pad = [1, 1] @@ -179,17 +203,28 @@ def init_test_case(self): self.filter_size = [8, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], - self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], - self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): @@ -199,7 +234,6 @@ def init_group(self): self.groups = 1 class TestWithDilation(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [2, 2] self.stride = [1, 1] @@ -209,24 +243,34 @@ def init_test_case(self): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], - self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], - self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] def init_dilation(self): self.dilations = [2, 2] class TestWith3x3(TestModulatedDeformableConvOp): - def init_test_case(self): self.pad = [1, 1] self.stride = [1, 1] @@ -236,56 +280,61 @@ def init_test_case(self): self.filter_size = [6, f_c, 3, 3] self.im2col_step = 1 self.deformable_groups = 1 - offset_c = 2 * self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] - mask_c = self.deformable_groups * self.filter_size[ - 2] * self.filter_size[3] + offset_c = ( + 2 + * self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) + mask_c = ( + self.deformable_groups + * self.filter_size[2] + * self.filter_size[3] + ) self.offset_size = [ - self.input_size[0], offset_c, self.input_size[2], - self.input_size[3] + self.input_size[0], + offset_c, + self.input_size[2], + self.input_size[3], ] self.mask_size = [ - self.input_size[0], mask_c, self.input_size[2], - self.input_size[3] + self.input_size[0], + mask_c, + self.input_size[2], + self.input_size[3], ] class TestModulatedDeformableConvInvalidInput(unittest.TestCase): - def test_error(self): - def test_invalid_input(): paddle.enable_static() input = [1, 3, 32, 32] - offset = fluid.data(name='offset', - shape=[None, 3, 32, 32], - dtype='float32') - mask = fluid.data(name='mask', - shape=[None, 3, 32, 32], - dtype='float32') - loss = fluid.layers.deformable_conv(input, - offset, - mask, - num_filters=4, - filter_size=1) + offset = fluid.data( + name='offset', shape=[None, 3, 32, 32], dtype='float32' + ) + mask = fluid.data( + name='mask', shape=[None, 3, 32, 32], dtype='float32' + ) + loss = fluid.layers.deformable_conv( + input, offset, mask, num_filters=4, filter_size=1 + ) self.assertRaises(TypeError, test_invalid_input) def test_invalid_offset(): paddle.enable_static() - input = fluid.data(name='input', - shape=[None, 3, 32, 32], - dtype='int32') - offset = fluid.data(name='offset', - shape=[None, 3, 32, 32], - dtype='float32') - mask = fluid.data(name='mask', - shape=[None, 3, 32, 32], - dtype='float32') - loss = fluid.layers.deformable_conv(input, - offset, - mask, - num_filters=4, - filter_size=1) + input = fluid.data( + name='input', shape=[None, 3, 32, 32], dtype='int32' + ) + offset = fluid.data( + name='offset', shape=[None, 3, 32, 32], dtype='float32' + ) + mask = fluid.data( + name='mask', shape=[None, 3, 32, 32], dtype='float32' + ) + loss = fluid.layers.deformable_conv( + input, offset, mask, num_filters=4, filter_size=1 + ) self.assertRaises(TypeError, test_invalid_offset) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_depthwise_conv2d_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_depthwise_conv2d_op_xpu.py index a8bfd68163420..75c6bd7b524c4 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_depthwise_conv2d_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_depthwise_conv2d_op_xpu.py @@ -22,17 +22,19 @@ paddle.enable_static() from test_conv2d_op_xpu import XPUTestConv2DOp, XPUTestConv2DOp_v2 -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class XPUTestDepthwiseConv2DOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'depthwise_conv2d' self.use_dynamic_create_class = False class TestDepthwiseConv(XPUTestConv2DOp.TestConv2DOp): - def init_test_case(self): self.use_cuda = False self.pad = [1, 1] @@ -45,7 +47,6 @@ def init_test_case(self): self.op_type = "depthwise_conv2d" class TestDepthwiseConv2(XPUTestConv2DOp.TestConv2DOp): - def init_test_case(self): self.use_cuda = False self.pad = [1, 1] @@ -58,7 +59,6 @@ def init_test_case(self): self.op_type = "depthwise_conv2d" class TestDepthwiseConv3(XPUTestConv2DOp.TestConv2DOp): - def init_test_case(self): self.use_cuda = False self.pad = [1, 1] @@ -71,7 +71,6 @@ def init_test_case(self): self.op_type = "depthwise_conv2d" class TestDepthwiseConvWithDilation(XPUTestConv2DOp.TestConv2DOp): - def init_test_case(self): self.use_cuda = False self.pad = [1, 1] @@ -85,7 +84,6 @@ def init_test_case(self): self.op_type = "depthwise_conv2d" class TestDepthwiseConvWithDilation2(XPUTestConv2DOp.TestConv2DOp): - def init_test_case(self): self.use_cuda = False self.pad = [1, 1] @@ -100,13 +98,11 @@ def init_test_case(self): class XPUTestDepthwiseConv2DOp_v2(XPUOpTestWrapper): - def __init__(self): self.op_name = 'depthwise_conv2d' self.use_dynamic_create_class = False class TestDepthwiseConv_AsyPadding(XPUTestConv2DOp_v2.TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = False self.stride = [2, 2] @@ -122,7 +118,6 @@ def init_paddings(self): self.padding_algorithm = "EXPLICIT" class TestDepthwiseConv2_AsyPadding(XPUTestConv2DOp_v2.TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = False self.stride = [1, 1] @@ -138,7 +133,6 @@ def init_paddings(self): self.padding_algorithm = "EXPLICIT" class TestDepthwiseConv3_AsyPadding(XPUTestConv2DOp_v2.TestConv2DOp_v2): - def init_test_case(self): self.use_cuda = False self.stride = [1, 1] @@ -154,8 +148,8 @@ def init_paddings(self): self.padding_algorithm = "EXPLICIT" class TestDepthwiseConvWithDilation_AsyPadding( - XPUTestConv2DOp_v2.TestConv2DOp_v2): - + XPUTestConv2DOp_v2.TestConv2DOp_v2 + ): def init_test_case(self): self.use_cuda = False self.pad = [1, 1] @@ -173,8 +167,8 @@ def init_paddings(self): self.padding_algorithm = "EXPLICIT" class TestDepthwiseConvWithDilation2_AsyPadding( - XPUTestConv2DOp_v2.TestConv2DOp_v2): - + XPUTestConv2DOp_v2.TestConv2DOp_v2 + ): def init_test_case(self): self.use_cuda = True self.pad = [1, 1] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_device_guard_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_device_guard_xpu.py index 3fbd3c6c2fef3..8c41cb8675ccb 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_device_guard_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_device_guard_xpu.py @@ -44,17 +44,16 @@ def get_vaild_warning_num(warning, w): class TestDeviceGuard(unittest.TestCase): - def test_device_guard(self): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - data1 = paddle.full(shape=[1, 3, 8, 8], - fill_value=0.5, - dtype='float32') - data2 = paddle.full(shape=[1, 3, 5, 5], - fill_value=0.5, - dtype='float32') + data1 = paddle.full( + shape=[1, 3, 8, 8], fill_value=0.5, dtype='float32' + ) + data2 = paddle.full( + shape=[1, 3, 5, 5], fill_value=0.5, dtype='float32' + ) shape = paddle.shape(data2) with paddle.static.device_guard("cpu"): shape = paddle.slice(shape, axes=[0], starts=[0], ends=[4]) @@ -75,12 +74,12 @@ def test_device_guard_with_id(self): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - data1 = paddle.full(shape=[1, 3, 8, 8], - fill_value=0.5, - dtype='float32') - data2 = paddle.full(shape=[1, 3, 5, 5], - fill_value=0.5, - dtype='float32') + data1 = paddle.full( + shape=[1, 3, 8, 8], fill_value=0.5, dtype='float32' + ) + data2 = paddle.full( + shape=[1, 3, 5, 5], fill_value=0.5, dtype='float32' + ) shape = paddle.shape(data2) with paddle.static.device_guard("cpu"): shape = paddle.slice(shape, axes=[0], starts=[0], ends=[4]) @@ -101,32 +100,50 @@ def test_cpu_only_op(self): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - x = paddle.full(shape=[2, 255, 13, 13], - fill_value=0.3, - dtype='float32') - gt_box = paddle.full(shape=[2, 6, 4], - fill_value=0.5, - dtype='float32') + x = paddle.full( + shape=[2, 255, 13, 13], fill_value=0.3, dtype='float32' + ) + gt_box = paddle.full( + shape=[2, 6, 4], fill_value=0.5, dtype='float32' + ) gt_label = paddle.full(shape=[2, 6], fill_value=1.0, dtype='int32') - gt_score = paddle.full(shape=[2, 6], - fill_value=0.5, - dtype='float32') + gt_score = paddle.full( + shape=[2, 6], fill_value=0.5, dtype='float32' + ) anchors = [ - 10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, - 198, 373, 326 + 10, + 13, + 16, + 30, + 33, + 23, + 30, + 61, + 62, + 45, + 59, + 119, + 116, + 90, + 156, + 198, + 373, + 326, ] anchor_mask = [0, 1, 2] with paddle.static.device_guard("xpu"): # yolov3_loss only has cpu kernel, so its cpu kernel will be executed - loss = fluid.layers.yolov3_loss(x=x, - gt_box=gt_box, - gt_label=gt_label, - gt_score=gt_score, - anchors=anchors, - anchor_mask=anchor_mask, - class_num=80, - ignore_thresh=0.7, - downsample_ratio=32) + loss = fluid.layers.yolov3_loss( + x=x, + gt_box=gt_box, + gt_label=gt_label, + gt_score=gt_score, + anchors=anchors, + anchor_mask=anchor_mask, + class_num=80, + ignore_thresh=0.7, + downsample_ratio=32, + ) execute(main_program, startup_program) @@ -159,7 +176,6 @@ def test_without_kernel_op(self): execute(main_program, startup_program) def test_error(self): - def device_attr(): with paddle.static.device_guard("cpu1"): out = paddle.full(shape=[1], fill_value=0.2, dtype='float32') @@ -176,17 +192,18 @@ def test_op_descs_device_attr(self): main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): - data1 = paddle.static.data(name="data_1", - shape=[4, 2], - dtype="float32") - label = paddle.static.data(name="label", - shape=[4, 1], - dtype="int64") + data1 = paddle.static.data( + name="data_1", shape=[4, 2], dtype="float32" + ) + label = paddle.static.data( + name="label", shape=[4, 1], dtype="int64" + ) fc1 = paddle.static.nn.fc(x=data1, size=10) fc2 = paddle.static.nn.fc(x=fc1, size=10) with paddle.static.device_guard("xpu"): out = paddle.nn.functional.softmax_with_cross_entropy( - logits=fc1 + fc2, label=label) + logits=fc1 + fc2, label=label + ) loss = paddle.mean(out) opt = paddle.optimizer.SGD(0.1) opt.minimize(loss) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_dropout_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_dropout_op_xpu.py index 7aff1acfdf7ff..794ff490d7ec9 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_dropout_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_dropout_op_xpu.py @@ -25,17 +25,19 @@ paddle.enable_static() -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class XPUTestDropoutOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'dropout' self.use_dynamic_create_class = False class TestDropoutOp(XPUOpTest): - def setUp(self): self.init_inputs_shape() self.init_attrs() @@ -46,7 +48,7 @@ def setUp(self): 'dropout_prob': self.dropout_prob, 'fix_seed': self.fix_seed, 'is_test': self.is_test, - 'dropout_implementation': self.dropout_implementation + 'dropout_implementation': self.dropout_implementation, } out = self.inputs['X'] * (1.0 - self.dropout_prob) @@ -74,19 +76,19 @@ def test_check_output(self): self.check_output() def test_check_grad_normal(self): - if hasattr(self.__class__, "no_need_check_grad" - ) and self.__class__.no_need_check_grad == True: + if ( + hasattr(self.__class__, "no_need_check_grad") + and self.__class__.no_need_check_grad == True + ): return self.check_grad(['X'], 'Out') class TestDropoutOpInput1d(TestDropoutOp): - def init_inputs_shape(self): self.shape = [2000] class TestDropoutOp2(TestDropoutOp): - def init_inputs_shape(self): self.shape = [32, 64] @@ -97,12 +99,10 @@ def init_attrs(self): self.dropout_implementation = "upscale_in_train" class TestDropoutOp3(TestDropoutOp): - def init_inputs_shape(self): self.shape = [32, 64, 2] class TestDropoutOp4(TestDropoutOp): - def init_attrs(self): self.__class__.no_need_check_grad = True self.dropout_prob = 0.35 @@ -111,7 +111,6 @@ def init_attrs(self): self.dropout_implementation = "downgrade_in_infer" class TestDropoutOp5(TestDropoutOp): - def init_inputs_shape(self): self.shape = [32, 64, 3] @@ -123,15 +122,16 @@ def init_attrs(self): self.dropout_implementation = "downgrade_in_infer" class TestDropoutOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): def test_Variable(): # the input of dropout must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, - 5]), [[1, 1, 1, 1]], - fluid.CPUPlace()) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), + [[1, 1, 1, 1]], + fluid.CPUPlace(), + ) fluid.layers.dropout(x1, dropout_prob=0.5) self.assertRaises(TypeError, test_Variable) @@ -139,15 +139,14 @@ def test_Variable(): def test_dtype(): # the input dtype of dropout must be float16 or float32 or float64 # float16 only can be set on GPU place - x2 = fluid.layers.data(name='x2', - shape=[3, 4, 5, 6], - dtype="int32") + x2 = fluid.layers.data( + name='x2', shape=[3, 4, 5, 6], dtype="int32" + ) fluid.layers.dropout(x2, dropout_prob=0.5) self.assertRaises(TypeError, test_dtype) class TestDropoutCAPI(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -159,13 +158,12 @@ def test_dygraph(self): input_np = np.random.random([40, 40]).astype(self.in_type) result_np = input_np input = fluid.dygraph.to_variable(input_np) - m = paddle.nn.Dropout(p=0.) + m = paddle.nn.Dropout(p=0.0) m.eval() result = m(input) np.testing.assert_allclose(result.numpy(), result_np) class TestDropoutBackward(unittest.TestCase): - def setUp(self): np.random.seed(123) self.places = [fluid.CPUPlace()] @@ -183,13 +181,15 @@ def test_backward_downscale_in_infer(self): input = paddle.uniform([40, 40], dtype=self.in_type) input.stop_gradient = False - out, mask = _legacy_C_ops.dropout(input, 'dropout_prob', - 0.5) + out, mask = _legacy_C_ops.dropout( + input, 'dropout_prob', 0.5 + ) out.backward() np.testing.assert_allclose( input.gradient(), - self.cal_grad_downscale_in_infer(mask.numpy())) + self.cal_grad_downscale_in_infer(mask.numpy()), + ) def test_backward_upscale_train(self): for place in self.places: @@ -198,15 +198,19 @@ def test_backward_upscale_train(self): prob = 0.5 input = paddle.uniform([40, 40], dtype=self.in_type) input.stop_gradient = False - out, mask = _legacy_C_ops.dropout(input, 'dropout_prob', - prob, - "dropout_implementation", - "upscale_in_train") + out, mask = _legacy_C_ops.dropout( + input, + 'dropout_prob', + prob, + "dropout_implementation", + "upscale_in_train", + ) out.backward() np.testing.assert_allclose( input.gradient(), - self.cal_grad_upscale_train(mask.numpy(), prob)) + self.cal_grad_upscale_train(mask.numpy(), prob), + ) def test_backward_upscale_train_2(self): for place in self.places: @@ -215,15 +219,19 @@ def test_backward_upscale_train_2(self): prob = 0.3 input = paddle.uniform([40, 40], dtype=self.in_type) input.stop_gradient = False - out, mask = _legacy_C_ops.dropout(input, 'dropout_prob', - prob, - "dropout_implementation", - "upscale_in_train") + out, mask = _legacy_C_ops.dropout( + input, + 'dropout_prob', + prob, + "dropout_implementation", + "upscale_in_train", + ) out.backward() np.testing.assert_allclose( input.gradient(), - self.cal_grad_upscale_train(mask.numpy(), prob)) + self.cal_grad_upscale_train(mask.numpy(), prob), + ) support_types = get_xpu_op_support_types('dropout') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu.py index 6ab52fcc836b8..2d04b3d7549e4 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu.py @@ -22,19 +22,21 @@ import unittest import paddle.fluid as fluid from paddle.fluid import Program, program_guard -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestElementwiseAddOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_add' self.use_dynamic_create_class = False class TestElementwiseAddOp(XPUOpTest): - def setUp(self): self.op_type = "elementwise_add" self.init_dtype() @@ -43,7 +45,7 @@ def setUp(self): self.init_max_relative_error() self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -57,27 +59,33 @@ def test_check_grad_normal(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['X', 'Y'], + place, + ['X', 'Y'], 'Out', - max_relative_error=self.max_relative_error) + max_relative_error=self.max_relative_error, + ) def test_check_grad_ingore_x(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['Y'], + place, + ['Y'], 'Out', no_grad_set=set("X"), - max_relative_error=self.max_relative_error) + max_relative_error=self.max_relative_error, + ) def test_check_grad_ingore_y(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['X'], + place, + ['X'], 'Out', no_grad_set=set("Y"), - max_relative_error=self.max_relative_error) + max_relative_error=self.max_relative_error, + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -94,32 +102,30 @@ def init_max_relative_error(self): self.max_relative_error = 0.006 @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwiseAddOp_scalar(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) self.out = self.x + self.y @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." + ) class TestElementwiseAddOp_scalar2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) self.out = self.x + self.y class TestElementwiseAddOp_Vector(TestElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) class TestElementwiseAddOp_broadcast_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -129,7 +135,6 @@ def init_axis(self): self.axis = 0 class TestElementwiseAddOp_broadcast_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -139,14 +144,12 @@ def init_axis(self): self.axis = 1 class TestElementwiseAddOp_broadcast_2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) self.out = self.x + self.y.reshape(1, 1, 100) class TestElementwiseAddOp_broadcast_3(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -156,7 +159,6 @@ def init_axis(self): self.axis = 1 class TestElementwiseAddOp_broadcast_4(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3, 4).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -166,28 +168,24 @@ def init_axis(self): self.axis = 0 class TestElementwiseAddOp_broadcast_5(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) self.out = self.x + self.y class TestElementwiseAddOp_broadcast_6(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) self.out = self.x + self.y class TestElementwiseAddOp_broadcast_7(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(1, 1, 20, 5).astype(self.dtype) self.y = np.random.rand(20, 5, 1, 1).astype(self.dtype) self.out = self.x + self.y class TestElementwiseAddOp_rowwise_add_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -197,9 +195,9 @@ def init_axis(self): self.axis = 1 @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwiseAddOp_rowwise_add_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -209,7 +207,6 @@ def init_axis(self): self.axis = 1 class TestElementwiseAddOp_channelwise_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -219,7 +216,6 @@ def init_axis(self): self.axis = -1 class TestElementwiseAddOp_commonuse_add1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -229,7 +225,6 @@ def init_axis(self): self.axis = -1 class TestElementwiseAddOp_commonuse_add2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 1, 4).astype(self.dtype) self.y = np.random.rand(10, 1, 12, 1).astype(self.dtype) @@ -239,7 +234,6 @@ def init_axis(self): self.axis = -1 class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 12).astype(self.dtype) self.y = np.random.rand(2, 3, 10, 12).astype(self.dtype) @@ -249,30 +243,32 @@ def init_axis(self): self.axis = 2 class TestElementwiseAddOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_add must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.XPUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.XPUPlace(0)) - self.assertRaises(TypeError, fluid.layers.elementwise_add, x1, - y1) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.XPUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.XPUPlace(0) + ) + self.assertRaises( + TypeError, fluid.layers.elementwise_add, x1, y1 + ) # the input dtype of elementwise_add must be float16 or float32 or float64 or int32 or int64 # float16 only can be set on GPU place - x2 = fluid.layers.data(name='x2', - shape=[3, 4, 5, 6], - dtype="uint8") - y2 = fluid.layers.data(name='y2', - shape=[3, 4, 5, 6], - dtype="uint8") - self.assertRaises(TypeError, fluid.layers.elementwise_add, x2, - y2) + x2 = fluid.layers.data( + name='x2', shape=[3, 4, 5, 6], dtype="uint8" + ) + y2 = fluid.layers.data( + name='y2', shape=[3, 4, 5, 6], dtype="uint8" + ) + self.assertRaises( + TypeError, fluid.layers.elementwise_add, x2, y2 + ) class TestAddOp(unittest.TestCase): - def test_name(self): with fluid.program_guard(fluid.Program()): x = fluid.data(name="x", shape=[2, 3], dtype="float32") @@ -287,7 +283,7 @@ def test_declarative(self): def gen_data(): return { "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') + "y": np.array([1, 5, 2]).astype('float32'), } x = fluid.data(name="x", shape=[3], dtype='float32') @@ -297,7 +293,7 @@ def gen_data(): place = fluid.XPUPlace(0) exe = fluid.Executor(place) z_value = exe.run(feed=gen_data(), fetch_list=[z.name]) - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((z_value == z_expected).all(), True) def test_dygraph(self): @@ -308,7 +304,7 @@ def test_dygraph(self): y = fluid.dygraph.to_variable(np_y) z = paddle.add(x, y) np_z = z.numpy() - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((np_z == z_expected).all(), True) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu_kp.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu_kp.py index c969332c5ae9d..7f201d3487ed2 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu_kp.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_add_op_xpu_kp.py @@ -26,10 +26,10 @@ paddle.enable_static() -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp(XPUOpTest): - def setUp(self): self.op_type = "elementwise_add" self.init_dtype() @@ -38,7 +38,7 @@ def setUp(self): self.init_max_relative_error() self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -52,27 +52,33 @@ def test_check_grad_normal(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['X', 'Y'], + place, + ['X', 'Y'], 'Out', - max_relative_error=self.max_relative_error) + max_relative_error=self.max_relative_error, + ) def test_check_grad_ingore_x(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['Y'], + place, + ['Y'], 'Out', no_grad_set=set("X"), - max_relative_error=self.max_relative_error) + max_relative_error=self.max_relative_error, + ) def test_check_grad_ingore_y(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['X'], + place, + ['X'], 'Out', no_grad_set=set("Y"), - max_relative_error=self.max_relative_error) + max_relative_error=self.max_relative_error, + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -89,44 +95,46 @@ def init_max_relative_error(self): self.max_relative_error = 0.006 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_scalar(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) self.out = self.x + self.y -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1,1) to test broadcast.") + reason="[skip shape check] Use y_shape(1,1) to test broadcast." +) class TestElementwiseAddOp_scalar2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 4).astype(self.dtype) self.y = np.random.rand(1, 1).astype(self.dtype) self.out = self.x + self.y -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_Vector(TestElementwiseAddOp): - def init_input_output(self): - self.x = np.random.random((100, )).astype(self.dtype) - self.y = np.random.random((100, )).astype(self.dtype) + self.x = np.random.random((100,)).astype(self.dtype) + self.y = np.random.random((100,)).astype(self.dtype) self.out = np.add(self.x, self.y) -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -136,10 +144,10 @@ def init_axis(self): self.axis = 0 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 100, 3).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) @@ -149,20 +157,20 @@ def init_axis(self): self.axis = 1 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(100).astype(self.dtype) self.out = self.x + self.y.reshape(1, 1, 100) -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_3(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12, 3).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -172,10 +180,10 @@ def init_axis(self): self.axis = 1 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_4(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3, 4).astype(self.dtype) self.y = np.random.rand(100, 1).astype(self.dtype) @@ -185,40 +193,40 @@ def init_axis(self): self.axis = 0 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_5(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 12).astype(self.dtype) self.y = np.random.rand(10, 1, 12).astype(self.dtype) self.out = self.x + self.y -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_6(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 12, 3, 5).astype(self.dtype) self.y = np.random.rand(2, 12, 1, 5).astype(self.dtype) self.out = self.x + self.y -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_broadcast_7(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(1, 1, 20, 5).astype(self.dtype) self.y = np.random.rand(20, 5, 1, 1).astype(self.dtype) self.out = self.x + self.y -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_rowwise_add_0(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 10, 12).astype(self.dtype) self.y = np.random.rand(10, 12).astype(self.dtype) @@ -228,12 +236,13 @@ def init_axis(self): self.axis = 1 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." +) class TestElementwiseAddOp_rowwise_add_1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 1).astype(self.dtype) self.y = np.random.rand(1).astype(self.dtype) @@ -243,10 +252,10 @@ def init_axis(self): self.axis = 1 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_channelwise_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(100, 2, 3).astype(self.dtype) self.y = np.random.rand(100, 1, 1).astype(self.dtype) @@ -256,10 +265,10 @@ def init_axis(self): self.axis = -1 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_commonuse_add1(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(2, 3, 100).astype(self.dtype) self.y = np.random.rand(1, 1, 100).astype(self.dtype) @@ -269,10 +278,10 @@ def init_axis(self): self.axis = -1 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_commonuse_add2(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 3, 1, 4).astype(self.dtype) self.y = np.random.rand(10, 1, 12, 1).astype(self.dtype) @@ -282,10 +291,10 @@ def init_axis(self): self.axis = -1 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOp_xsize_lessthan_ysize_add(TestElementwiseAddOp): - def init_input_output(self): self.x = np.random.rand(10, 12).astype(self.dtype) self.y = np.random.rand(2, 3, 10, 12).astype(self.dtype) @@ -295,17 +304,19 @@ def init_axis(self): self.axis = 2 -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestElementwiseAddOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_add must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.XPUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.XPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.XPUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.XPUPlace(0) + ) self.assertRaises(TypeError, fluid.layers.elementwise_add, x1, y1) # the input dtype of elementwise_add must be float16 or float32 or float64 or int32 or int64 @@ -315,10 +326,10 @@ def test_errors(self): self.assertRaises(TypeError, fluid.layers.elementwise_add, x2, y2) -@unittest.skipIf(not paddle.is_compiled_with_xpu(), - "core is not compiled with XPU") +@unittest.skipIf( + not paddle.is_compiled_with_xpu(), "core is not compiled with XPU" +) class TestAddOp(unittest.TestCase): - def test_name(self): with fluid.program_guard(fluid.Program()): x = fluid.data(name="x", shape=[2, 3], dtype="float32") @@ -333,7 +344,7 @@ def test_declarative(self): def gen_data(): return { "x": np.array([2, 3, 4]).astype('float32'), - "y": np.array([1, 5, 2]).astype('float32') + "y": np.array([1, 5, 2]).astype('float32'), } x = fluid.data(name="x", shape=[3], dtype='float32') @@ -343,7 +354,7 @@ def gen_data(): place = fluid.XPUPlace(0) exe = fluid.Executor(place) z_value = exe.run(feed=gen_data(), fetch_list=[z.name]) - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((z_value == z_expected).all(), True) def test_dygraph(self): @@ -354,7 +365,7 @@ def test_dygraph(self): y = fluid.dygraph.to_variable(np_y) z = paddle.add(x, y) np_z = z.numpy() - z_expected = np.array([3., 8., 6.]) + z_expected = np.array([3.0, 8.0, 6.0]) self.assertEqual((np_z == z_expected).all(), True) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_div_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_div_op_xpu.py index 956e71cb69ea9..4144a7068e0fa 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_div_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_div_op_xpu.py @@ -20,19 +20,21 @@ import paddle.fluid as fluid from op_test import skip_check_grad_ci from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestElementwiseDivOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_div' self.use_dynamic_create_class = False class ElementwiseDivOp(XPUOpTest): - def setUp(self): self.op_type = "elementwise_div" self.dtype = self.in_type @@ -48,7 +50,7 @@ def setUp(self): def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype), } self.outputs = { 'Out': np.divide(self.inputs['X'], self.inputs['Y']) @@ -62,133 +64,138 @@ def test_check_output(self): def test_check_grad_normal(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X', 'Y'], - 'Out', - max_relative_error=0.05) + self.check_grad_with_place( + place, ['X', 'Y'], 'Out', max_relative_error=0.05 + ) def test_check_grad_ingore_x(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.05, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.05, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.05, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.05, + no_grad_set=set('Y'), + ) def init_dtype(self): pass @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwiseDivOp_scalar(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [20, 3, 4]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [1]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [1]).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] / self.inputs['Y']} class TestElementwiseDivOp_Vector(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [100]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), } self.outputs = { 'Out': np.divide(self.inputs['X'], self.inputs['Y']) } class TestElementwiseDivOp_broadcast_0(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [100, 3, 4]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), } self.attrs = {'axis': 0} self.outputs = { - 'Out': - np.divide(self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestElementwiseDivOp_broadcast_1(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 100, 4]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.divide(self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1) + ) } class TestElementwiseDivOp_broadcast_2(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 100]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), } self.outputs = { - 'Out': - np.divide(self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100) + ) } class TestElementwiseDivOp_broadcast_3(ElementwiseDivOp): - def init_input_output(self): self.inputs = { - 'X': np.random.uniform(0.1, 1, - [2, 10, 12, 5]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [10, 12]).astype(self.dtype) + 'X': np.random.uniform(0.1, 1, [2, 10, 12, 5]).astype( + self.dtype + ), + 'Y': np.random.uniform(0.1, 1, [10, 12]).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.divide(self.inputs['X'], - self.inputs['Y'].reshape(1, 10, 12, 1)) + 'Out': np.divide( + self.inputs['X'], self.inputs['Y'].reshape(1, 10, 12, 1) + ) } class TestElementwiseDivOp_broadcast_4(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 50]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [2, 1, 50]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [2, 1, 50]).astype(self.dtype), } self.outputs = { 'Out': np.divide(self.inputs['X'], self.inputs['Y']) } class TestElementwiseDivOp_broadcast_5(ElementwiseDivOp): - def init_input_output(self): self.inputs = { - 'X': np.random.uniform(0.1, 1, - [2, 3, 4, 20]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [2, 3, 1, 20]).astype(self.dtype) + 'X': np.random.uniform(0.1, 1, [2, 3, 4, 20]).astype( + self.dtype + ), + 'Y': np.random.uniform(0.1, 1, [2, 3, 1, 20]).astype( + self.dtype + ), } self.outputs = { 'Out': np.divide(self.inputs['X'], self.inputs['Y']) } class TestElementwiseDivOp_commonuse_1(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 100]).astype(self.dtype), @@ -199,25 +206,26 @@ def init_input_output(self): } class TestElementwiseDivOp_commonuse_2(ElementwiseDivOp): - def init_input_output(self): self.inputs = { - 'X': np.random.uniform(0.1, 1, - [30, 3, 1, 5]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, - [30, 1, 4, 1]).astype(self.dtype), + 'X': np.random.uniform(0.1, 1, [30, 3, 1, 5]).astype( + self.dtype + ), + 'Y': np.random.uniform(0.1, 1, [30, 1, 4, 1]).astype( + self.dtype + ), } self.outputs = { 'Out': np.divide(self.inputs['X'], self.inputs['Y']) } class TestElementwiseDivOp_xsize_lessthan_ysize(ElementwiseDivOp): - def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [10, 12]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, - [2, 3, 10, 12]).astype(self.dtype), + 'Y': np.random.uniform(0.1, 1, [2, 3, 10, 12]).astype( + self.dtype + ), } self.attrs = {'axis': 2} @@ -227,18 +235,18 @@ def init_input_output(self): } class TestElementwiseDivBroadcast(unittest.TestCase): - def test_shape_with_batch_sizes(self): with fluid.program_guard(fluid.Program()): - x_var = fluid.data(name='x', - dtype='float32', - shape=[None, 3, None, None]) - one = 2. + x_var = fluid.data( + name='x', dtype='float32', shape=[None, 3, None, None] + ) + one = 2.0 out = one / x_var exe = fluid.Executor(fluid.XPUPlace(0)) - x = np.random.uniform(0.1, 0.6, - (1, 3, 32, 32)).astype('float32') - out_result, = exe.run(feed={'x': x}, fetch_list=[out]) + x = np.random.uniform(0.1, 0.6, (1, 3, 32, 32)).astype( + 'float32' + ) + (out_result,) = exe.run(feed={'x': x}, fetch_list=[out]) self.assertEqual((out_result == (2 / x)).all(), True) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_floordiv_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_floordiv_op_xpu.py index d954e0d6ce642..f557221b488df 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_floordiv_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_floordiv_op_xpu.py @@ -19,20 +19,22 @@ import paddle from op_test import OpTest from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() import random class XPUTestElementwiseModOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_floordiv' self.use_dynamic_create_class = False class TestElementwiseModOp(XPUOpTest): - def init_kernel_type(self): self.use_mkldnn = False @@ -46,7 +48,7 @@ def setUp(self): self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} self.outputs = {'Out': self.out} @@ -65,7 +67,6 @@ def init_axis(self): pass class TestElementwiseModOp_scalar(TestElementwiseModOp): - def init_input_output(self): scale_x = random.randint(0, 100000) scale_y = random.randint(1, 100000) @@ -74,7 +75,6 @@ def init_input_output(self): self.out = np.floor_divide(self.x, self.y) class TestElementwiseModOpInverse(TestElementwiseModOp): - def init_input_output(self): self.x = np.random.uniform(0, 10000, [10]).astype(self.dtype) self.y = np.random.uniform(1, 1000, [10, 10]).astype(self.dtype) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_max_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_max_op_xpu.py index 582f73d525c9c..f2a05670f41b9 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_max_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_max_op_xpu.py @@ -19,19 +19,21 @@ from op_test import skip_check_grad_ci from op_test_xpu import XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestElementwiseMaxOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_max' self.use_dynamic_create_class = False class TestElementwiseOp(XPUOpTest): - def setUp(self): self.use_xpu = True self.op_type = "elementwise_max" @@ -63,23 +65,29 @@ def test_check_grad_normal(self): def test_check_grad_ingore_x(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.006, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.006, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.006, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.006, + no_grad_set=set('Y'), + ) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwiseMaxOp_scalar(TestElementwiseOp): - def init_input_output(self): x = np.random.random_integers(-5, 5, [2, 3, 20]).astype(self.dtype) y = np.array([0.5]).astype(self.dtype) @@ -89,86 +97,85 @@ def init_input_output(self): } class TestElementwiseMaxOp_Vector(TestElementwiseOp): - def init_input_output(self): - x = np.random.random((100, )).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x + sgn * np.random.uniform(0.1, 1, (100, )).astype(self.dtype) + x = np.random.random((100,)).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x + sgn * np.random.uniform(0.1, 1, (100,)).astype(self.dtype) self.inputs = {'X': x, 'Y': y} self.outputs = { 'Out': np.maximum(self.inputs['X'], self.inputs['Y']) } class TestElementwiseMaxOp_broadcast_0(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (100, 5, 2)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x[:, 0, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x[:, 0, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 0} self.outputs = { - 'Out': - np.maximum(self.inputs['X'], - self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestElementwiseMaxOp_broadcast_1(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (2, 100, 3)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x[0, :, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x[0, :, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.maximum(self.inputs['X'], - self.inputs['Y'].reshape(1, 100, 1)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1) + ) } class TestElementwiseMaxOp_broadcast_2(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (1, 3, 100)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x[0, 0, :] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x[0, 0, :] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.inputs = {'X': x, 'Y': y} self.outputs = { - 'Out': - np.maximum(self.inputs['X'], - self.inputs['Y'].reshape(1, 1, 100)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100) + ) } class TestElementwiseMaxOp_broadcast_3(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (2, 50, 2, 1)).astype(self.dtype) sgn = np.random.choice([-1, 1], (50, 2)).astype(self.dtype) - y = x[0, :, :, 0] + sgn * \ - np.random.uniform(1, 2, (50, 2)).astype(self.dtype) + y = x[0, :, :, 0] + sgn * np.random.uniform(1, 2, (50, 2)).astype( + self.dtype + ) self.inputs = {'X': x, 'Y': y} self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.maximum(self.inputs['X'], - self.inputs['Y'].reshape(1, 50, 2, 1)) + 'Out': np.maximum( + self.inputs['X'], self.inputs['Y'].reshape(1, 50, 2, 1) + ) } class TestElementwiseMaxOp_broadcast_4(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (2, 3, 4, 5)).astype(self.dtype) sgn = np.random.choice([-1, 1], (2, 3, 1, 5)).astype(self.dtype) - y = x + sgn * \ - np.random.uniform(1, 2, (2, 3, 1, 5)).astype(self.dtype) + y = x + sgn * np.random.uniform(1, 2, (2, 3, 1, 5)).astype( + self.dtype + ) self.inputs = {'X': x, 'Y': y} self.outputs = { diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_min_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_min_op_xpu.py index c7eb91e83be91..498410d6dbb68 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_min_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_min_op_xpu.py @@ -19,19 +19,21 @@ from op_test import skip_check_grad_ci import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestElementwiseMinOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_min' self.use_dynamic_create_class = False class TestElementwiseOp(XPUOpTest): - def setUp(self): self.op_type = "elementwise_min" # If x and y have the same value, the min() is not differentiable. @@ -62,23 +64,29 @@ def test_check_grad_normal(self): def test_check_grad_ingore_x(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.005, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.005, + no_grad_set=set('Y'), + ) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwiseMinOp_scalar(TestElementwiseOp): - def init_input_output(self): x = np.random.random_integers(-5, 5, [10, 3, 4]).astype(self.dtype) y = np.array([0.5]).astype(self.dtype) @@ -88,82 +96,81 @@ def init_input_output(self): } class TestElementwiseMinOp_Vector(TestElementwiseOp): - def init_input_output(self): - x = np.random.random((100, )).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x + sgn * np.random.uniform(0.1, 1, (100, )).astype(self.dtype) + x = np.random.random((100,)).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x + sgn * np.random.uniform(0.1, 1, (100,)).astype(self.dtype) self.inputs = {'X': x, 'Y': y} self.outputs = { 'Out': np.minimum(self.inputs['X'], self.inputs['Y']) } class TestElementwiseMinOp_broadcast_0(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (100, 3, 2)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x[:, 0, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x[:, 0, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.attrs = {'axis': 0} self.inputs = {'X': x, 'Y': y} self.outputs = { - 'Out': - np.minimum(self.inputs['X'], - self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestElementwiseMinOp_broadcast_1(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (2, 100, 3)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x[0, :, 0] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x[0, :, 0] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.attrs = {'axis': 1} self.inputs = {'X': x, 'Y': y} self.outputs = { - 'Out': - np.minimum(self.inputs['X'], - self.inputs['Y'].reshape(1, 100, 1)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1) + ) } class TestElementwiseMinOp_broadcast_2(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (2, 3, 100)).astype(self.dtype) - sgn = np.random.choice([-1, 1], (100, )).astype(self.dtype) - y = x[0, 0, :] + sgn * \ - np.random.uniform(1, 2, (100, )).astype(self.dtype) + sgn = np.random.choice([-1, 1], (100,)).astype(self.dtype) + y = x[0, 0, :] + sgn * np.random.uniform(1, 2, (100,)).astype( + self.dtype + ) self.inputs = {'X': x, 'Y': y} self.outputs = { - 'Out': - np.minimum(self.inputs['X'], - self.inputs['Y'].reshape(1, 1, 100)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100) + ) } class TestElementwiseMinOp_broadcast_3(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (2, 25, 4, 1)).astype(self.dtype) sgn = np.random.choice([-1, 1], (25, 4)).astype(self.dtype) - y = x[0, :, :, 0] + sgn * \ - np.random.uniform(1, 2, (25, 4)).astype(self.dtype) + y = x[0, :, :, 0] + sgn * np.random.uniform(1, 2, (25, 4)).astype( + self.dtype + ) self.attrs = {'axis': 1} self.inputs = {'X': x, 'Y': y} self.outputs = { - 'Out': - np.minimum(self.inputs['X'], - self.inputs['Y'].reshape(1, 25, 4, 1)) + 'Out': np.minimum( + self.inputs['X'], self.inputs['Y'].reshape(1, 25, 4, 1) + ) } class TestElementwiseMinOp_broadcast_4(TestElementwiseOp): - def init_input_output(self): x = np.random.uniform(0.5, 1, (2, 10, 2, 5)).astype(self.dtype) sgn = np.random.choice([-1, 1], (2, 10, 1, 5)).astype(self.dtype) - y = x + sgn * \ - np.random.uniform(1, 2, (2, 10, 1, 5)).astype(self.dtype) + y = x + sgn * np.random.uniform(1, 2, (2, 10, 1, 5)).astype( + self.dtype + ) self.inputs = {'X': x, 'Y': y} self.outputs = { 'Out': np.minimum(self.inputs['X'], self.inputs['Y']) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mod_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mod_op_xpu.py index f446fd1f2ade4..334cd0794b48f 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mod_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mod_op_xpu.py @@ -21,19 +21,21 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestElementwiseModOp(XPUOpTestWrapper): - def __init__(self) -> None: self.op_name = 'elementwise_mod' self.use_dynamic_create_class = False class ElementwiseModOp(XPUOpTest): - def init_kernel_type(self): self.use_mkldnn = False @@ -43,7 +45,7 @@ def init_input_output(self): self.out = np.mod(self.x, self.y) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} @@ -70,29 +72,26 @@ def test_check_output(self): self.check_output_with_place(place) class TestElementwiseModOp_broadcast_1(ElementwiseModOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 100, 3).astype(self.dtype), - 'Y': np.random.rand(2, 100, 3).astype(self.dtype) + 'Y': np.random.rand(2, 100, 3).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = {'Out': self.inputs['X'] % self.inputs['Y']} class TestElementwiseModOp_broadcast_2(ElementwiseModOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(22, 128, 3).astype(self.dtype), - 'Y': np.random.rand(22, 128, 3).astype(self.dtype) + 'Y': np.random.rand(22, 128, 3).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = {'Out': self.inputs['X'] % self.inputs['Y']} class TestRemainderOp(unittest.TestCase): - def test_dygraph(self): with fluid.dygraph.guard(): np_x = np.random.rand(22, 128, 3).astype('int64') @@ -105,7 +104,7 @@ def test_dygraph(self): self.assertEqual((np_z == z_expected).all(), True) np_x = np.array([-3.3, 11.5, -2, 3.5]) - np_y = np.array([-1.2, 2., 3.3, -2.3]) + np_y = np.array([-1.2, 2.0, 3.3, -2.3]) x = paddle.to_tensor(np_x) y = paddle.to_tensor(np_y) z = x % y diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mul_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mul_op_xpu.py index 0e09eafbdb78b..22ee95c07d4ce 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mul_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_mul_op_xpu.py @@ -21,19 +21,21 @@ from paddle.fluid import Program, program_guard import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestElementwiseMulOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_mul' self.use_dynamic_create_class = False class ElementwiseMulOp(XPUOpTest): - def init_kernel_type(self): self.use_mkldnn = False @@ -56,27 +58,33 @@ def test_check_grad_normal(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['X', 'Y'], + place, + ['X', 'Y'], 'Out', - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_ingore_x(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['Y'], + place, + ['Y'], 'Out', no_grad_set=set("X"), - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) def test_check_grad_ingore_y(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_grad_with_place( - place, ['X'], + place, + ['X'], 'Out', no_grad_set=set('Y'), - check_dygraph=(self.use_mkldnn == False)) + check_dygraph=(self.use_mkldnn == False), + ) def init_input_output(self): self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) @@ -84,7 +92,7 @@ def init_input_output(self): self.out = np.multiply(self.x, self.y) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(self.x), - 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y), } self.outputs = {'Out': self.out} self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} @@ -96,33 +104,31 @@ def init_axis(self): pass @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwiseMulOp_scalar(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(10, 3, 4).astype(self.dtype), - 'Y': np.random.rand(1).astype(self.dtype) + 'Y': np.random.rand(1).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_Vector(ElementwiseMulOp): - def init_input_output(self): self.inputs = { - 'X': np.random.random((100, )).astype(self.dtype), - 'Y': np.random.random((100, )).astype(self.dtype) + 'X': np.random.random((100,)).astype(self.dtype), + 'Y': np.random.random((100,)).astype(self.dtype), } self.outputs = { 'Out': np.multiply(self.inputs['X'], self.inputs['Y']) } class TestElementwiseMulOp_broadcast_0(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(100, 2, 3).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype) + 'Y': np.random.rand(100).astype(self.dtype), } self.outputs = { 'Out': self.inputs['X'] * self.inputs['Y'].reshape(100, 1, 1) @@ -130,11 +136,10 @@ def init_input_output(self): self.attrs = {'axis': 0} class TestElementwiseMulOp_broadcast_1(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 100, 3).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype) + 'Y': np.random.rand(100).astype(self.dtype), } self.attrs = {'axis': 1} @@ -143,11 +148,10 @@ def init_input_output(self): } class TestElementwiseMulOp_broadcast_2(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 3, 100).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype) + 'Y': np.random.rand(100).astype(self.dtype), } self.outputs = { @@ -155,61 +159,54 @@ def init_input_output(self): } class TestElementwiseMulOp_broadcast_3(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 10, 12, 3).astype(self.dtype), - 'Y': np.random.rand(10, 12).astype(self.dtype) + 'Y': np.random.rand(10, 12).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = { - 'Out': - self.inputs['X'] * self.inputs['Y'].reshape(1, 10, 12, 1) + 'Out': self.inputs['X'] * self.inputs['Y'].reshape(1, 10, 12, 1) } class TestElementwiseMulOp_broadcast_4(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(10, 2, 11).astype(self.dtype), - 'Y': np.random.rand(10, 1, 11).astype(self.dtype) + 'Y': np.random.rand(10, 1, 11).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_broadcast_5(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(10, 4, 2, 3).astype(self.dtype), - 'Y': np.random.rand(10, 4, 1, 3).astype(self.dtype) + 'Y': np.random.rand(10, 4, 1, 3).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_commonuse_1(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 3, 100).astype(self.dtype), - 'Y': np.random.rand(1, 1, 100).astype(self.dtype) + 'Y': np.random.rand(1, 1, 100).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_commonuse_2(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(30, 3, 1, 5).astype(self.dtype), - 'Y': np.random.rand(30, 1, 4, 1).astype(self.dtype) + 'Y': np.random.rand(30, 1, 4, 1).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} class TestElementwiseMulOp_xsize_lessthan_ysize(ElementwiseMulOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(10, 10).astype(self.dtype), - 'Y': np.random.rand(2, 2, 10, 10).astype(self.dtype) + 'Y': np.random.rand(2, 2, 10, 10).astype(self.dtype), } self.attrs = {'axis': 2} @@ -219,26 +216,29 @@ def init_input_output(self): } class TestElementwiseMulOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # the input of elementwise_mul must be Variable. - x1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.XPUPlace(0)) - y1 = fluid.create_lod_tensor(np.array([-1, 3, 5, 5]), - [[1, 1, 1, 1]], fluid.XPUPlace(0)) - self.assertRaises(TypeError, fluid.layers.elementwise_mul, x1, - y1) + x1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.XPUPlace(0) + ) + y1 = fluid.create_lod_tensor( + np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.XPUPlace(0) + ) + self.assertRaises( + TypeError, fluid.layers.elementwise_mul, x1, y1 + ) # the input dtype of elementwise_mul must be float32 - x2 = fluid.layers.data(name='x2', - shape=[3, 4, 5, 6], - dtype="uint8") - y2 = fluid.layers.data(name='y2', - shape=[3, 4, 5, 6], - dtype="uint8") - self.assertRaises(TypeError, fluid.layers.elementwise_mul, x2, - y2) + x2 = fluid.layers.data( + name='x2', shape=[3, 4, 5, 6], dtype="uint8" + ) + y2 = fluid.layers.data( + name='y2', shape=[3, 4, 5, 6], dtype="uint8" + ) + self.assertRaises( + TypeError, fluid.layers.elementwise_mul, x2, y2 + ) support_types = get_xpu_op_support_types('elementwise_mul') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_pow_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_pow_op_xpu.py index 6387346f4ec67..95315d55878c7 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_pow_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_pow_op_xpu.py @@ -19,20 +19,22 @@ import paddle from op_test import OpTest, skip_check_grad_ci from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @skip_check_grad_ci(reason="XPU does not support grad op currently") class XPUTestElementwisePowOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_pow' self.use_dynamic_create_class = False class TestElementwisePowOp(XPUOpTest): - def setUp(self): self.op_type = "elementwise_pow" self.dtype = self.in_type @@ -42,7 +44,7 @@ def setUp(self): def compute_input_output(self): self.inputs = { 'X': np.random.uniform(1, 2, [20, 5]).astype(self.dtype), - 'Y': np.random.uniform(1, 2, [20, 5]).astype(self.dtype) + 'Y': np.random.uniform(1, 2, [20, 5]).astype(self.dtype), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} @@ -52,110 +54,107 @@ def test_check_output(self): self.check_output_with_place(place) class TestElementwisePowOp_big_shape_1(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { 'X': np.random.uniform(1, 2, [10, 10]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [10, 10]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [10, 10]).astype(self.dtype), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_big_shape_2(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { 'X': np.random.uniform(1, 2, [10, 10]).astype(self.dtype), - 'Y': np.random.uniform(0.2, 2, [10, 10]).astype(self.dtype) + 'Y': np.random.uniform(0.2, 2, [10, 10]).astype(self.dtype), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwisePowOp_scalar(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [3, 3, 4]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [1]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [1]).astype(self.dtype), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_tensor(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [100]).astype(self.dtype), - 'Y': np.random.uniform(1, 3, [100]).astype(self.dtype) + 'Y': np.random.uniform(1, 3, [100]).astype(self.dtype), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_broadcast_0(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 1, 100]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOp_broadcast_1(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 100, 1]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = { - 'Out': np.power(self.inputs['X'], - self.inputs['Y'].reshape(100, 1)) + 'Out': np.power( + self.inputs['X'], self.inputs['Y'].reshape(100, 1) + ) } class TestElementwisePowOp_broadcast_2(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [100, 3, 1]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), } self.attrs = {'axis': 0} self.outputs = { - 'Out': - np.power(self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1)) + 'Out': np.power( + self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) + ) } class TestElementwisePowOp_broadcast_3(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { - 'X': np.random.uniform(0.1, 1, - [2, 20, 5, 1]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [20, 5]).astype(self.dtype) + 'X': np.random.uniform(0.1, 1, [2, 20, 5, 1]).astype( + self.dtype + ), + 'Y': np.random.uniform(0.1, 1, [20, 5]).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = { - 'Out': - np.power(self.inputs['X'], - self.inputs['Y'].reshape(1, 20, 5, 1)) + 'Out': np.power( + self.inputs['X'], self.inputs['Y'].reshape(1, 20, 5, 1) + ) } class TestElementwisePowOp_broadcast_4(TestElementwisePowOp): - def compute_input_output(self): self.inputs = { - 'X': np.random.uniform(0.1, 1, - [2, 10, 3, 5]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [2, 10, 1, 5]).astype(self.dtype) + 'X': np.random.uniform(0.1, 1, [2, 10, 3, 5]).astype( + self.dtype + ), + 'Y': np.random.uniform(0.1, 1, [2, 10, 1, 5]).astype( + self.dtype + ), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} class TestElementwisePowOpInt(OpTest): - def setUp(self): self.op_type = "elementwise_pow" self.inputs = { 'X': np.asarray([1, 3, 6]), - 'Y': np.asarray([1, 1, 1]) + 'Y': np.asarray([1, 1, 1]), } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_sub_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_sub_op_xpu.py index 67f4d561f6e3e..927855f461d34 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_elementwise_sub_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_elementwise_sub_op_xpu.py @@ -20,19 +20,21 @@ from op_test import skip_check_grad_ci from op_test_xpu import XPUOpTest import unittest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestElementwiseSubOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'elementwise_sub' self.use_dynamic_create_class = False class TestElementwiseOp(XPUOpTest): - def setUp(self): self.op_type = "elementwise_sub" self.use_xpu = True @@ -42,7 +44,7 @@ def setUp(self): def init_input_output(self): self.inputs = { 'X': np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype) + 'Y': np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} @@ -59,45 +61,49 @@ def test_check_grad_normal(self): def test_check_grad_ingore_x(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.005, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.005, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.005, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.005, + no_grad_set=set('Y'), + ) @skip_check_grad_ci( - reason="[skip shape check] Use y_shape(1) to test broadcast.") + reason="[skip shape check] Use y_shape(1) to test broadcast." + ) class TestElementwiseSubOp_scalar(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(10, 3, 4).astype(self.dtype), - 'Y': np.random.rand(1).astype(self.dtype) + 'Y': np.random.rand(1).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_Vector(TestElementwiseOp): - def init_input_output(self): self.inputs = { - 'X': np.random.random((100, )).astype(self.dtype), - 'Y': np.random.random((100, )).astype(self.dtype) + 'X': np.random.random((100,)).astype(self.dtype), + 'Y': np.random.random((100,)).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_broadcast_0(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(100, 3, 2).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype) + 'Y': np.random.rand(100).astype(self.dtype), } self.attrs = {'axis': 0} @@ -106,11 +112,10 @@ def init_input_output(self): } class TestElementwiseSubOp_broadcast_1(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 100, 3).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype) + 'Y': np.random.rand(100).astype(self.dtype), } self.attrs = {'axis': 1} @@ -119,11 +124,10 @@ def init_input_output(self): } class TestElementwiseSubOp_broadcast_2(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 3, 100).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype) + 'Y': np.random.rand(100).astype(self.dtype), } self.outputs = { @@ -131,52 +135,46 @@ def init_input_output(self): } class TestElementwiseSubOp_broadcast_3(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 10, 12, 3).astype(self.dtype), - 'Y': np.random.rand(10, 12).astype(self.dtype) + 'Y': np.random.rand(10, 12).astype(self.dtype), } self.attrs = {'axis': 1} self.outputs = { - 'Out': - self.inputs['X'] - self.inputs['Y'].reshape(1, 10, 12, 1) + 'Out': self.inputs['X'] - self.inputs['Y'].reshape(1, 10, 12, 1) } class TestElementwiseSubOp_broadcast_4(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 5, 3, 12).astype(self.dtype), - 'Y': np.random.rand(2, 5, 1, 12).astype(self.dtype) + 'Y': np.random.rand(2, 5, 1, 12).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_commonuse_1(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(2, 3, 100).astype(self.dtype), - 'Y': np.random.rand(1, 1, 100).astype(self.dtype) + 'Y': np.random.rand(1, 1, 100).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_commonuse_2(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(10, 3, 1, 4).astype(self.dtype), - 'Y': np.random.rand(10, 1, 12, 1).astype(self.dtype) + 'Y': np.random.rand(10, 1, 12, 1).astype(self.dtype), } self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} class TestElementwiseSubOp_xsize_lessthan_ysize(TestElementwiseOp): - def init_input_output(self): self.inputs = { 'X': np.random.rand(10, 12).astype(self.dtype), - 'Y': np.random.rand(2, 3, 10, 12).astype(self.dtype) + 'Y': np.random.rand(2, 3, 10, 12).astype(self.dtype), } self.attrs = {'axis': 2} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_empty_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_empty_op_xpu.py index ec4a7e8b1cf3a..cb56e9b51f42d 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_empty_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_empty_op_xpu.py @@ -1,4 +1,4 @@ -#Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,20 +21,22 @@ import paddle from op_test_xpu import XPUOpTest from paddle.fluid.framework import convert_np_dtype_to_dtype_ -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestEmptyOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'empty' self.use_dynamic_create_class = False # Situation 1: Attr(shape) is a list(without tensor) class TestEmptyOp(XPUOpTest): - def setUp(self): self.op_type = "empty" self.init_dtype() @@ -50,24 +52,34 @@ def test_check_output(self): def verify_output(self, outs): data_type = outs[0].dtype if data_type in [ - 'float32', 'float64', 'int32', 'int64', 'int8', 'uint8', - 'float16', 'int16' + 'float32', + 'float64', + 'int32', + 'int64', + 'int8', + 'uint8', + 'float16', + 'int16', ]: max_value = np.nanmax(outs[0]) min_value = np.nanmin(outs[0]) always_full_zero = max_value == 0.0 and min_value == 0.0 always_non_full_zero = max_value >= min_value - self.assertTrue(always_full_zero or always_non_full_zero, - 'always_full_zero or always_non_full_zero.') + self.assertTrue( + always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.', + ) elif data_type in ['bool']: total_num = outs[0].size true_num = np.sum(outs[0] == True) false_num = np.sum(outs[0] == False) - self.assertTrue(total_num == true_num + false_num, - 'The value should always be True or False.') + self.assertTrue( + total_num == true_num + false_num, + 'The value should always be True or False.', + ) else: - #pass + # pass self.assertTrue(False, 'invalid data type') def set_shape(self): @@ -90,34 +102,30 @@ def set_xpu(self): self.__class__.op_type = self.op_type class TestEmptyOpCase1(TestEmptyOp): - def set_shape(self): self.shape = [50] class TestEmptyOpCase2(TestEmptyOp): - def set_shape(self): self.shape = [1, 50, 3, 4] class TestEmptyOpCase3(TestEmptyOp): - def set_shape(self): self.shape = [5, 5, 5] # Situation 2: shape is a tensor class TestEmptyOp_ShapeTensor(TestEmptyOp): - def set_inputs(self): self.inputs = {"ShapeTensor": np.array(self.shape).astype("int32")} # Situation 3: Attr(shape) is a list(with tensor) class TestEmptyOp_ShapeTensorList(TestEmptyOp): - def set_inputs(self): shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_expand_as_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_expand_as_v2_op_xpu.py index 36b53de7004a8..246696be64315 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_expand_as_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_expand_as_v2_op_xpu.py @@ -20,20 +20,22 @@ from op_test_xpu import XPUOpTest import paddle import paddle.fluid as fluid -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() np.random.seed(10) class XPUTestExpandAsV2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = 'expand_as_v2' self.use_dynamic_create_class = False class TestExpandAsV2XPUOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -65,7 +67,6 @@ def test_check_output(self): self.check_output_with_place(self.place) class TestExpandAsOpRank2(TestExpandAsV2XPUOp): - def set_inputs(self): x = np.random.rand(10, 12).astype(self.dtype) self.inputs = {'X': x} @@ -78,7 +79,6 @@ def set_output(self): self.outputs = {'Out': output} class TestExpandAsOpRank3(TestExpandAsV2XPUOp): - def set_inputs(self): x = np.random.rand(2, 3, 20).astype(self.dtype) self.inputs = {'X': x} @@ -91,7 +91,6 @@ def set_output(self): self.outputs = {'Out': output} class TestExpandAsOpRank4(TestExpandAsV2XPUOp): - def set_inputs(self): x = np.random.rand(1, 1, 7, 16).astype(self.dtype) self.inputs = {'X': x} @@ -104,7 +103,6 @@ def set_output(self): self.outputs = {'Out': output} class TestExpandAsOpRank5(TestExpandAsV2XPUOp): - def set_inputs(self): x = np.random.rand(1, 1, 7, 16, 1).astype(self.dtype) self.inputs = {'X': x} @@ -117,7 +115,6 @@ def set_output(self): self.outputs = {'Out': output} class TestExpandAsOpRank6(TestExpandAsV2XPUOp): - def set_inputs(self): x = np.random.rand(1, 1, 7, 16, 1, 1).astype(self.dtype) self.inputs = {'X': x} @@ -132,29 +129,28 @@ def set_output(self): # Test python API class TestExpandAsV2API(unittest.TestCase): - def test_api(self): input1 = np.random.random([12, 14]).astype("float32") input2 = np.random.random([2, 12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', shape=[12, 14], append_batch_size=False, dtype="float32" + ) - y = fluid.layers.data(name='target_tensor', - shape=[2, 12, 14], - append_batch_size=False, - dtype="float32") + y = fluid.layers.data( + name='target_tensor', + shape=[2, 12, 14], + append_batch_size=False, + dtype="float32", + ) out_1 = paddle.expand_as(x, y=y) exe = fluid.Executor(place=fluid.XPUPlace(0)) - res_1 = exe.run(fluid.default_main_program(), - feed={ - "x": input1, - "target_tensor": input2 - }, - fetch_list=[out_1]) + res_1 = exe.run( + fluid.default_main_program(), + feed={"x": input1, "target_tensor": input2}, + fetch_list=[out_1], + ) assert np.array_equal(res_1[0], np.tile(input1, (2, 1, 1))) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_expand_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_expand_v2_op_xpu.py index 545bebcb960d8..e5a30ad65bc65 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_expand_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_expand_v2_op_xpu.py @@ -20,7 +20,11 @@ from op_test_xpu import XPUOpTest import paddle.fluid as fluid import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() np.random.seed(10) @@ -29,13 +33,11 @@ # CANN Op Support X: float32, int32, int64 # Situation 1: shape is a list(without tensor) class XPUTestExpandV2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = 'expand_v2' self.use_dynamic_create_class = False class TestExpandV2XPUOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -65,42 +67,36 @@ def test_check_output(self): self.check_output_with_place(self.place) class TestExpandV2OpRank2_DimExpanding(TestExpandV2XPUOp): - def init_data(self): self.ori_shape = [120] self.shape = [2, 120] self.expand_times = [2, 1] class TestExpandV2OpRank2(TestExpandV2XPUOp): - def init_data(self): self.ori_shape = [1, 140] self.shape = [12, 140] self.expand_times = [12, 1] class TestExpandV2OpRank3_Corner(TestExpandV2XPUOp): - def init_data(self): self.ori_shape = (2, 10, 5) self.shape = (2, 10, 5) self.expand_times = (1, 1, 1) class TestExpandV2OpRank4(TestExpandV2XPUOp): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.shape = (-1, -1, -1, -1) self.expand_times = (1, 1, 1, 1) class TestExpandV2OpRank5(TestExpandV2XPUOp): - def init_data(self): self.ori_shape = (2, 4, 1, 15) self.shape = (2, -1, 4, -1) self.expand_times = (1, 1, 4, 1) class TestExpandV2OpRank6(TestExpandV2XPUOp): - def init_data(self): self.ori_shape = (4, 1, 30) self.shape = (2, -1, 4, 30) @@ -108,7 +104,6 @@ def init_data(self): # Situation 2: shape is a list(with tensor) class TestExpandV2OpXPURank1_tensor_attr(TestExpandV2XPUOp): - def setUp(self): self.set_xpu() self.place = paddle.XPUPlace(0) @@ -117,8 +112,9 @@ def setUp(self): self.dtype = np.float32 expand_shapes_tensor = [] for index, ele in enumerate(self.expand_shape): - expand_shapes_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + expand_shapes_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype(self.dtype), @@ -135,8 +131,8 @@ def init_data(self): self.infer_expand_shape = [-1] class TestExpandV2OpRank2_Corner_tensor_attr( - TestExpandV2OpXPURank1_tensor_attr): - + TestExpandV2OpXPURank1_tensor_attr + ): def init_data(self): self.ori_shape = [12, 14] self.expand_times = [1, 1] @@ -145,7 +141,6 @@ def init_data(self): # Situation 3: shape is a tensor class TestExpandV2XPUOp_tensor(TestExpandV2XPUOp): - def setUp(self): self.set_xpu() self.place = paddle.XPUPlace(0) @@ -170,7 +165,6 @@ def init_data(self): # Situation 5: input x is int32 # skip grad check for int32 class TestExpandV2OpInteger(XPUOpTest): - def init_type(self): self.dtype = 'int32' @@ -198,20 +192,23 @@ def test_check_grad(self): # Test python API class TestExpandV2API(unittest.TestCase): - def test_static(self): with fluid.program_guard(fluid.Program(), fluid.Program()): input = np.random.random([12, 14]).astype("float32") - x = fluid.layers.data(name='x', - shape=[12, 14], - append_batch_size=False, - dtype="float32") + x = fluid.layers.data( + name='x', + shape=[12, 14], + append_batch_size=False, + dtype="float32", + ) positive_2 = fluid.layers.fill_constant([1], "int32", 12) - expand_shape = fluid.layers.data(name="expand_shape", - shape=[2], - append_batch_size=False, - dtype="int32") + expand_shape = fluid.layers.data( + name="expand_shape", + shape=[2], + append_batch_size=False, + dtype="int32", + ) out_1 = paddle.expand(x, shape=[12, 14]) out_2 = paddle.expand(x, shape=[positive_2, 14]) @@ -220,14 +217,14 @@ def test_static(self): g0 = fluid.backward.calc_gradient(out_2, x) exe = fluid.Executor(place=paddle.XPUPlace(0)) - res_1, res_2, res_3 = exe.run(fluid.default_main_program(), - feed={ - "x": - input, - "expand_shape": - np.array([12, 14]).astype("int32") - }, - fetch_list=[out_1, out_2, out_3]) + res_1, res_2, res_3 = exe.run( + fluid.default_main_program(), + feed={ + "x": input, + "expand_shape": np.array([12, 14]).astype("int32"), + }, + fetch_list=[out_1, out_2, out_3], + ) assert np.array_equal(res_1, np.tile(input, (1, 1))) assert np.array_equal(res_2, np.tile(input, (1, 1))) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_fill_any_like_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_fill_any_like_op_xpu.py index 614891d174dbf..56b099b1e86bc 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_fill_any_like_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_fill_any_like_op_xpu.py @@ -20,19 +20,21 @@ import unittest import numpy as np from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestFillAnyLikeOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'fill_any_like' self.use_dynamic_create_class = False class TestFillAnyLikeOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -60,22 +62,18 @@ def test_check_output(self): self.check_output_with_place(self.place) class TestFillAnyLikeOp2(TestFillAnyLikeOp): - def set_value(self): self.value = -0.0 class TestFillAnyLikeOp3(TestFillAnyLikeOp): - def set_value(self): self.value = 1.0 class TestFillAnyLikeOp4(TestFillAnyLikeOp): - def init(self): self.value = 1e-9 class TestFillAnyLikeOp5(TestFillAnyLikeOp): - def set_value(self): if self.dtype == "float16": self.value = 0.05 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_fill_constant_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_fill_constant_op_xpu.py index 2494323d885cb..e09834075a7b9 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_fill_constant_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_fill_constant_op_xpu.py @@ -20,21 +20,22 @@ import numpy as np from op_test import convert_float_to_uint16 from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class XPUTestFillConstantOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'fill_constant' self.use_dynamic_create_class = False # Situation 1: Attr(shape) is a list(without tensor) class TestFillConstantOp(XPUOpTest): - def setUp(self): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' self.init_dtype() self.set_xpu() self.op_type = "fill_constant" @@ -105,7 +106,7 @@ def set_data(self): self.attrs = { 'shape': self.shape, 'dtype': self.index, - 'value': self.value + 'value': self.value, } self.outputs = {'Out': np.full(self.shape, self.value)} @@ -113,58 +114,55 @@ def test_check_output(self): self.check_output_with_place(self.place) class TestFillConstantOp2(TestFillConstantOp): - '''Test fill_constant op with default value - ''' + '''Test fill_constant op with default value''' def set_shape(self): self.shape = [10, 10] class TestFillConstantOp3(TestFillConstantOp): - '''Test fill_constant op with specified int64 value - ''' + '''Test fill_constant op with specified int64 value''' def set_shape(self): self.shape = [123, 2, 1] class TestFillConstantOp4(TestFillConstantOp): - '''Test fill_constant op with specified int value - ''' + '''Test fill_constant op with specified int value''' def set_shape(self): self.shape = [123, 3, 2, 1] class TestFillConstantOp5(TestFillConstantOp): - '''Test fill_constant op with specified float value - ''' + '''Test fill_constant op with specified float value''' def set_shape(self): self.shape = [123] # Situation 2: Attr(shape) is a list(with tensor) class TestFillConstantOp1_ShapeTensorList(TestFillConstantOp): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' def set_data(self): shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} self.attrs = { 'shape': self.infer_shape, 'dtype': self.index, - 'value': self.value + 'value': self.value, } self.outputs = {'Out': np.full(self.shape, self.value)} if self.index == 22: self.outputs = { - 'Out': - np.full( + 'Out': np.full( self.shape, convert_float_to_uint16( - np.array([self.value]).astype("float32"))) + np.array([self.value]).astype("float32") + ), + ) } def set_shape(self): @@ -172,14 +170,14 @@ def set_shape(self): self.infer_shape = [123, 1] class TestFillConstantOp2_ShapeTensorList(TestFillConstantOp): - '''Test fill_constant op with default value - ''' + '''Test fill_constant op with default value''' def set_data(self): shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = {"ShapeTensorList": shape_tensor_list} self.attrs = {'shape': self.infer_shape, 'dtype': self.index} @@ -190,23 +188,22 @@ def set_shape(self): self.infer_shape = [1, 1, 1] class TestFillConstantOp3_ShapeTensorList( - TestFillConstantOp1_ShapeTensorList): - + TestFillConstantOp1_ShapeTensorList + ): def set_shape(self): self.shape = [123, 3, 2, 1] self.infer_shape = [123, 111, 11, 1] class TestFillConstantOp4_ShapeTensorList( - TestFillConstantOp1_ShapeTensorList): - + TestFillConstantOp1_ShapeTensorList + ): def set_shape(self): self.shape = [123] self.infer_shape = [1] # Situation 3: shape is a tensor class TestFillConstantOp1_ShapeTensor(TestFillConstantOp): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' def set_data(self): self.inputs = {"ShapeTensor": np.array(self.shape).astype("int32")} @@ -214,11 +211,12 @@ def set_data(self): self.outputs = {'Out': np.full(self.shape, self.value)} if self.index == 22: self.outputs = { - 'Out': - np.full( + 'Out': np.full( self.shape, convert_float_to_uint16( - np.array([self.value]).astype("float32"))) + np.array([self.value]).astype("float32") + ), + ) } def set_shape(self): @@ -226,19 +224,18 @@ def set_shape(self): # Situation 4: value is a tensor class TestFillConstantOp1_ValueTensor(TestFillConstantOp): - '''Test fill_constant op with specified value - ''' + '''Test fill_constant op with specified value''' def set_data(self): self.inputs = { "ShapeTensor": np.array(self.shape).astype("int32"), - 'ValueTensor': np.array([self.value]).astype(self.dtype) + 'ValueTensor': np.array([self.value]).astype(self.dtype), } if self.index == 22: self.inputs = { - 'ValueTensor': - convert_float_to_uint16( - np.array([self.value]).astype("float32")) + 'ValueTensor': convert_float_to_uint16( + np.array([self.value]).astype("float32") + ) } self.attrs = {'value': self.value, 'dtype': self.index} self.outputs = {'Out': np.full(self.shape, self.value)} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_flatten2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_flatten2_op_xpu.py index 29d1cb3c25e49..307889b4f26b0 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_flatten2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_flatten2_op_xpu.py @@ -19,19 +19,21 @@ import numpy as np import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestFlatten2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = 'flatten2' self.use_dynamic_create_class = False class TestFlatten2Op(XPUOpTest): - def setUp(self): self.set_xpu() self.op_type = "flatten2" @@ -44,7 +46,7 @@ def setUp(self): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype(self.dtype) + "XShape": np.random.random(self.in_shape).astype(self.dtype), } def set_xpu(self): @@ -65,14 +67,12 @@ def init_attrs(self): self.attrs = {"axis": self.axis} class TestFlatten2OpWithCornerAxis(TestFlatten2Op): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.axis = 0 self.new_shape = (1, 120) class TestFlatten2OpWithDefaultAxis(TestFlatten2Op): - def init_test_case(self): self.in_shape = (10, 2, 2, 3) self.new_shape = (10, 12) @@ -81,7 +81,6 @@ def init_attrs(self): self.attrs = {} class TestFlatten2OpSixDims(TestFlatten2Op): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_flatten_contiguous_range_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_flatten_contiguous_range_op_xpu.py index 1296f5ce72496..aa029301003be 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_flatten_contiguous_range_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_flatten_contiguous_range_op_xpu.py @@ -22,19 +22,21 @@ sys.path.append("..") from op_test_xpu import XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestFlattenOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'flatten_contiguous_range' self.use_dynamic_create_class = False class TestFlattenOp(XPUOpTest): - def setUp(self): self.set_xpu() self.op_type = "flatten_contiguous_range" @@ -52,7 +54,7 @@ def setUp(self): self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.in_shape).astype(self.dtype) + "XShape": np.random.random(self.in_shape).astype(self.dtype), } def set_xpu(self): @@ -68,7 +70,7 @@ def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 self.stop_axis = -1 - self.new_shape = (120) + self.new_shape = 120 def init_attrs(self): self.attrs = { @@ -78,7 +80,6 @@ def init_attrs(self): } class TestFlattenOp_1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 1 @@ -88,11 +89,10 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_2(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -102,11 +102,10 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_3(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -116,11 +115,10 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_4(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = -2 @@ -130,11 +128,10 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_5(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 2 @@ -144,11 +141,10 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.start_axis = 3 @@ -158,11 +154,10 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_Float32(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -173,11 +168,10 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } class TestFlattenOp_int32(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -189,14 +183,13 @@ def init_attrs(self): self.attrs = { "start_axis": self.start_axis, "stop_axis": self.stop_axis, - 'use_xpu': True + 'use_xpu': True, } def test_check_grad(self): pass class TestFlattenOp_int8(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -207,14 +200,13 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): pass class TestFlattenOp_int64(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 5, 4) self.start_axis = 0 @@ -225,7 +217,7 @@ def init_test_case(self): def init_attrs(self): self.attrs = { "start_axis": self.start_axis, - "stop_axis": self.stop_axis + "stop_axis": self.stop_axis, } def test_check_grad(self): @@ -233,45 +225,58 @@ def test_check_grad(self): class TestFlatten2OpError(unittest.TestCase): - def test_errors(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_ValueError1(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) out = paddle.flatten(x_var, start_axis=2, stop_axis=1) self.assertRaises(ValueError, test_ValueError1) def test_ValueError2(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=10, stop_axis=1) self.assertRaises(ValueError, test_ValueError2) def test_ValueError3(): - x_var = paddle.static.data(name="x", - shape=image_shape, - dtype='float32') + x_var = paddle.static.data( + name="x", shape=image_shape, dtype='float32' + ) paddle.flatten(x_var, start_axis=2, stop_axis=10) self.assertRaises(ValueError, test_ValueError3) def test_type(): # dtype must be float32, float64, int8, int32, int64 - x2 = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x2 = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x2 = x2.astype('float16') - x2_var = paddle.fluid.data(name='x2', - shape=[3, 2, 4, 5], - dtype='float16') + x2_var = paddle.fluid.data( + name='x2', shape=[3, 2, 4, 5], dtype='float16' + ) paddle.flatten(x2_var) self.assertRaises(TypeError, test_type) @@ -283,7 +288,6 @@ def test_InputError(): class TestStaticFlattenPythonAPI(unittest.TestCase): - def execute_api(self, x, start_axis=0, stop_axis=-1): return paddle.flatten(x, start_axis, stop_axis) @@ -293,9 +297,9 @@ def test_static_api(self): main_prog = paddle.static.Program() with paddle.static.program_guard(main_prog, paddle.static.Program()): - x = paddle.static.data(name="x", - shape=[2, 3, 4, 4], - dtype='float32') + x = paddle.static.data( + name="x", shape=[2, 3, 4, 4], dtype='float32' + ) out = self.execute_api(x, start_axis=-2, stop_axis=-1) exe = paddle.static.Executor(place=paddle.XPUPlace(0)) @@ -304,17 +308,22 @@ def test_static_api(self): class TestStaticInplaceFlattenPythonAPI(TestStaticFlattenPythonAPI): - def execute_api(self, x, start_axis=0, stop_axis=-1): return x.flatten_(start_axis, stop_axis) class TestFlattenPython(unittest.TestCase): - def test_python_api(self): image_shape = (2, 3, 4, 4) - x = np.arange(image_shape[0] * image_shape[1] * image_shape[2] * - image_shape[3]).reshape(image_shape) / 100. + x = ( + np.arange( + image_shape[0] + * image_shape[1] + * image_shape[2] + * image_shape[3] + ).reshape(image_shape) + / 100.0 + ) x = x.astype('float32') def test_InputError(): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_flatten_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_flatten_op_xpu.py index 5d93a5a5ac0b8..13efc51fa2610 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_flatten_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_flatten_op_xpu.py @@ -19,19 +19,21 @@ import numpy as np import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestFlattenOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'flatten' self.use_dynamic_create_class = False class TestFlattenOp(XPUOpTest): - def setUp(self): self.op_type = "flatten" self.use_xpu = True @@ -59,14 +61,12 @@ def init_attrs(self): self.attrs = {"axis": self.axis} class TestFlattenOp1(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 2, 10) self.axis = 0 self.new_shape = (1, 120) class TestFlattenOpWithDefaultAxis(TestFlattenOp): - def init_test_case(self): self.in_shape = (10, 2, 2, 3) self.new_shape = (10, 12) @@ -75,7 +75,6 @@ def init_attrs(self): self.attrs = {} class TestFlattenOpSixDims(TestFlattenOp): - def init_test_case(self): self.in_shape = (3, 2, 3, 2, 4, 4) self.axis = 4 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_fleet_exe_dist_model_run_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_fleet_exe_dist_model_run_xpu.py index 867379bf81ef5..8d5091d32354a 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_fleet_exe_dist_model_run_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_fleet_exe_dist_model_run_xpu.py @@ -23,7 +23,6 @@ class TestDistModelRun(unittest.TestCase): - def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() @@ -34,8 +33,9 @@ def tearDown(self): def test_dist_model_run(self): # step 0: declare folder to save the model and params - path_prefix = os.path.join(self.temp_dir.name, - "dist_model_run_test/inf") + path_prefix = os.path.join( + self.temp_dir.name, "dist_model_run_test/inf" + ) # step 1: saving the inference model and params x = paddle.static.data(name='x', shape=[28, 28], dtype='float32') @@ -47,12 +47,11 @@ def test_dist_model_run(self): exe.run(paddle.static.default_startup_program()) x_data = np.random.randn(28, 28).astype('float32') y_data = np.random.randint(0, 9, size=[28, 1]).astype('int64') - exe.run(paddle.static.default_main_program(), - feed={ - 'x': x_data, - 'y': y_data - }, - fetch_list=[avg_loss]) + exe.run( + paddle.static.default_main_program(), + feed={'x': x_data, 'y': y_data}, + fetch_list=[avg_loss], + ) paddle.static.save_inference_model(path_prefix, [x, y], [avg_loss], exe) print('save model to', path_prefix) @@ -74,14 +73,16 @@ def test_dist_model_run(self): print("dist model rst:", dist_model_rst) # step 4: use framework's api to inference with fake data - [inference_program, feed_target_names, - fetch_targets] = (paddle.static.load_inference_model(path_prefix, exe)) - results = exe.run(inference_program, - feed={ - 'x': x_tensor, - 'y': y_tensor - }, - fetch_list=fetch_targets) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.load_inference_model(path_prefix, exe) + results = exe.run( + inference_program, + feed={'x': x_tensor, 'y': y_tensor}, + fetch_list=fetch_targets, + ) load_inference_model_rst = results[0] print("load inference model api rst:", load_inference_model_rst) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_grad_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_grad_op_xpu.py index 0c68982d14065..435f44e47361d 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_grad_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_grad_op_xpu.py @@ -22,7 +22,11 @@ import paddle import paddle.fluid.core as core from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def get_outputs(DOut, X, Y): @@ -34,13 +38,11 @@ def get_outputs(DOut, X, Y): class XPUTestFuseGemmGradOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'fused_gemm_epilogue_grad' self.use_dynamic_create_class = False class TestFuseGemmEpilogueGradOpDXYBias1(XPUOpTest): - def setUp(self): paddle.enable_static() self.op_type = "fused_gemm_epilogue_grad" @@ -53,13 +55,14 @@ def init_data(self): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - DX, DY, DBias = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + DX, DY, DBias = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DX': DX, 'DY': DY, 'DBias': DBias} def test_check_output(self): @@ -69,48 +72,48 @@ def test_check_output(self): self.check_output_with_place(core.XPUPlace(0), atol=self.atol) class TestFuseGemmEpilogueGradOpDXYBias2(XPUOpTest): - def init_data(self): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - _, DY, DBias = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + _, DY, DBias = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DY': DY, 'DBias': DBias} class TestFuseGemmEpilogueGradOpDXYBias3(XPUOpTest): - def init_data(self): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - _, DY, _ = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + _, DY, _ = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DY': DY} class TestFuseGemmEpilogueGradOpDXYBias4(XPUOpTest): - def init_data(self): self.inputs = { 'DOut': np.random.random((8, 128)).astype(self.dtype) - 0.5, 'X': np.random.random((8, 4)).astype(self.dtype) - 0.5, - 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5 + 'Y': np.random.random((4, 128)).astype(self.dtype) - 0.5, } self.attrs = {"activation": 'none'} - DX, DY, _ = get_outputs(self.inputs['DOut'], self.inputs['X'], - self.inputs['Y']) + DX, DY, _ = get_outputs( + self.inputs['DOut'], self.inputs['X'], self.inputs['Y'] + ) self.outputs = {'DX': DX, 'DY': DY} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_op_xpu.py index bb6cc6a03cbe5..c37d1bff5dd96 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_fused_gemm_epilogue_op_xpu.py @@ -22,12 +22,19 @@ import paddle.fluid.core as core from paddle import _legacy_C_ops from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def gelu(x): - y_ref = 0.5 * x * ( - 1.0 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)))) + y_ref = ( + 0.5 + * x + * (1.0 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)))) + ) return y_ref.astype(x.dtype) @@ -82,13 +89,11 @@ def matmul_grad(x, y, bias, dz, trans_x, trans_y): class XPUTestFuseGemmOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'fused_gemm_epilogue' self.use_dynamic_create_class = False class TestFuseGemmBase(XPUOpTest): - def setUp(self): self.__class__.no_need_check_grad = True self.op_type = "fused_gemm_epilogue" @@ -97,13 +102,14 @@ def setUp(self): self.inputs = { 'X': np.random.random(self.x_shape).astype(self.dtype) - 0.5, 'Y': np.random.random(self.y_shape).astype(self.dtype) - 0.5, - 'Bias': - np.random.random(self.bias_shape).astype(self.dtype) - 0.5 + 'Bias': np.random.random(self.bias_shape).astype(self.dtype) + - 0.5, } if self.trans_x == True: - numpy_input_x = self.inputs['X'].reshape( - (self.x_shape[0], -1)).T + numpy_input_x = ( + self.inputs['X'].reshape((self.x_shape[0], -1)).T + ) else: numpy_input_x = self.inputs['X'].reshape((-1, self.x_shape[-1])) @@ -113,14 +119,17 @@ def setUp(self): numpy_input_y = self.inputs['Y'] self.outputs = { - 'Out': - get_output(numpy_input_x, numpy_input_y, self.inputs['Bias'], - self.activation).reshape(self.out_shape) + 'Out': get_output( + numpy_input_x, + numpy_input_y, + self.inputs['Bias'], + self.activation, + ).reshape(self.out_shape) } self.attrs = { "activation": self.activation, "trans_y": self.trans_y, - "trans_x": self.trans_x + "trans_x": self.trans_x, } def init_dtype_type(self): @@ -144,7 +153,6 @@ def test_check_output(self): self.check_output_with_place(core.XPUPlace(0), atol=self.atol) class TestFuseGemmEpilogueOp1(TestFuseGemmBase): - def init_datas_shape_and_attrs(self): self.x_shape = [4, 8] self.y_shape = [4, 128] @@ -157,7 +165,6 @@ def init_datas_shape_and_attrs(self): self.trans_x = True class TestFuseGemmEpilogueOp2(TestFuseGemmBase): - def init_datas_shape_and_attrs(self): self.x_shape = [8, 4] self.y_shape = [128, 4] @@ -170,7 +177,6 @@ def init_datas_shape_and_attrs(self): self.trans_x = False class TestFuseGemmEpilogueOp3(TestFuseGemmBase): - def init_datas_shape_and_attrs(self): self.x_shape = [4, 8] self.y_shape = [128, 4] @@ -183,7 +189,6 @@ def init_datas_shape_and_attrs(self): self.trans_x = True class TestFuseGemmEpilogueOp4(TestFuseGemmBase): - def init_datas_shape_and_attrs(self): self.x_shape = [2, 2, 8, 4] self.y_shape = [4, 128] @@ -196,7 +201,6 @@ def init_datas_shape_and_attrs(self): self.trans_x = False class TestFuseGemmEpilogueOp5(TestFuseGemmBase): - def init_datas_shape_and_attrs(self): self.x_shape = [4, 2, 2, 8] self.y_shape = [4, 128] @@ -209,7 +213,6 @@ def init_datas_shape_and_attrs(self): self.trans_x = True class TestFuseGemmEpilogueOp6(TestFuseGemmBase): - def init_datas_shape_and_attrs(self): self.x_shape = [8, 4] self.y_shape = [4, 128] @@ -222,7 +225,6 @@ def init_datas_shape_and_attrs(self): self.trans_x = False class TestFuseGemmEpilogueOp7(TestFuseGemmBase): - def init_datas_shape_and_attrs(self): self.x_shape = [8, 4] self.y_shape = [4, 128] @@ -236,7 +238,6 @@ def init_datas_shape_and_attrs(self): class TestEagerFusedGemmEpilogue(unittest.TestCase): - def setUp(self): paddle.set_device('xpu') @@ -244,22 +245,22 @@ def test_case_act(self): paddle.disable_static() x_np = np.random.random((8, 4)).astype(np.float32) - 0.5 y_np = np.random.random((4, 128)).astype(np.float32) - 0.5 - bias_np = np.random.random((128, )).astype(np.float32) - 0.5 + bias_np = np.random.random((128,)).astype(np.float32) - 0.5 x = paddle.to_tensor(x_np) y = paddle.to_tensor(y_np) bias = paddle.to_tensor(bias_np) x.stop_gradient = False y.stop_gradient = False - out1 = _legacy_C_ops.fused_gemm_epilogue(x, y, bias, 'trans_x', False, - 'trans_y', False, 'activation', - 'none') - out2 = _legacy_C_ops.fused_gemm_epilogue(x, y, bias, 'trans_x', False, - 'trans_y', False, 'activation', - 'relu') - out3 = _legacy_C_ops.fused_gemm_epilogue(x, y, bias, 'trans_x', False, - 'trans_y', False, 'activation', - 'gelu') + out1 = _legacy_C_ops.fused_gemm_epilogue( + x, y, bias, 'trans_x', False, 'trans_y', False, 'activation', 'none' + ) + out2 = _legacy_C_ops.fused_gemm_epilogue( + x, y, bias, 'trans_x', False, 'trans_y', False, 'activation', 'relu' + ) + out3 = _legacy_C_ops.fused_gemm_epilogue( + x, y, bias, 'trans_x', False, 'trans_y', False, 'activation', 'gelu' + ) out_np1 = get_output(x_np, y_np, bias_np, 'none') out_np2 = get_output(x_np, y_np, bias_np, 'relu') @@ -269,13 +270,16 @@ def test_case_act(self): np.testing.assert_allclose(out2, out_np2, atol=1e-04) np.testing.assert_allclose(out3, out_np3, atol=1e-03) - out_grad_np1 = np.random.randint(low=-20, high=20, - size=out_np1.shape).astype(np.float32) - paddle.autograd.backward(out1, - grad_tensors=[paddle.to_tensor(out_grad_np1)]) + out_grad_np1 = np.random.randint( + low=-20, high=20, size=out_np1.shape + ).astype(np.float32) + paddle.autograd.backward( + out1, grad_tensors=[paddle.to_tensor(out_grad_np1)] + ) x_grad_np, y_grad_np, bias_grad_np = matmul_grad( - x_np, y_np, bias_np, out_grad_np1, False, False) + x_np, y_np, bias_np, out_grad_np1, False, False + ) np.testing.assert_allclose(x.grad.numpy(), x_grad_np, atol=1e-02) self.assertEqual(y_grad_np.shape, y_np.shape) np.testing.assert_allclose(y.grad.numpy(), y_grad_np, atol=1e-03) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_fused_resnet_basic_block_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_fused_resnet_basic_block_op_xpu.py index 7913e7e014857..f45a4a135e3e9 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_fused_resnet_basic_block_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_fused_resnet_basic_block_op_xpu.py @@ -24,19 +24,21 @@ from paddle.fluid import core from paddle.incubate.xpu.resnet_block import ResNetBasicBlock from paddle.fluid.framework import default_main_program -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestResNetBasicBlockOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "resnet_basic_block" self.use_dynamic_create_class = False class TestResNetBasicBlockOp(OpTest): - def setUp(self): paddle.disable_static() self.dtype = self.in_type @@ -70,65 +72,86 @@ def Base(self): conv1_weight = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) + learning_rate=0.001, + ) conv2_weight = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) + learning_rate=0.001, + ) conv3_weight = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) - bn1_weight = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0)) - bn1_bias = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.0)) - bn2_weight = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0)) - bn2_bias = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.0)) - bn3_weight = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=1.0)) - bn3_bias = fluid.ParamAttr(initializer=fluid.initializer.Constant( - value=0.0)) + learning_rate=0.001, + ) + bn1_weight = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ) + bn1_bias = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.0) + ) + bn2_weight = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ) + bn2_bias = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.0) + ) + bn3_weight = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0) + ) + bn3_bias = fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=0.0) + ) - self.conv1 = nn.Conv2D(in_channels=self.in_channels, - out_channels=self.out_channels, - kernel_size=3, - stride=self.stride, - padding=1, - weight_attr=conv1_weight, - bias_attr=None, - data_format='NCHW') - self.bn1 = nn.BatchNorm(self.out_channels, - act='relu', - param_attr=bn1_weight, - bias_attr=bn1_bias, - data_layout='NCHW') - self.conv2 = nn.Conv2D(in_channels=self.out_channels, - out_channels=self.out_channels, - kernel_size=3, - stride=1, - padding=1, - weight_attr=conv2_weight, - bias_attr=None, - data_format='NCHW') - self.bn2 = nn.BatchNorm(self.out_channels, - act=None, - param_attr=bn2_weight, - bias_attr=bn2_bias, - data_layout='NCHW') - self.conv3 = nn.Conv2D(in_channels=self.in_channels, - out_channels=self.out_channels, - kernel_size=1, - stride=self.stride, - padding=0, - weight_attr=conv3_weight, - bias_attr=None, - data_format='NCHW') - self.bn3 = nn.BatchNorm(self.out_channels, - act=None, - param_attr=bn3_weight, - bias_attr=bn3_bias, - data_layout='NCHW') + self.conv1 = nn.Conv2D( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=3, + stride=self.stride, + padding=1, + weight_attr=conv1_weight, + bias_attr=None, + data_format='NCHW', + ) + self.bn1 = nn.BatchNorm( + self.out_channels, + act='relu', + param_attr=bn1_weight, + bias_attr=bn1_bias, + data_layout='NCHW', + ) + self.conv2 = nn.Conv2D( + in_channels=self.out_channels, + out_channels=self.out_channels, + kernel_size=3, + stride=1, + padding=1, + weight_attr=conv2_weight, + bias_attr=None, + data_format='NCHW', + ) + self.bn2 = nn.BatchNorm( + self.out_channels, + act=None, + param_attr=bn2_weight, + bias_attr=bn2_bias, + data_layout='NCHW', + ) + self.conv3 = nn.Conv2D( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=1, + stride=self.stride, + padding=0, + weight_attr=conv3_weight, + bias_attr=None, + data_format='NCHW', + ) + self.bn3 = nn.BatchNorm( + self.out_channels, + act=None, + param_attr=bn3_weight, + bias_attr=bn3_bias, + data_layout='NCHW', + ) self.relu = nn.ReLU() tensor_src = paddle.to_tensor(self.src, stop_gradient=False) @@ -139,8 +162,9 @@ def Base(self): bn1_out = self.bn1(self.conv1(tensor_src)) bn2_out = self.bn2(self.conv2(bn1_out)) result = self.relu(bn2_out + z_out) - paddle.autograd.backward([result], [paddle.to_tensor(self.dout)], - True) + paddle.autograd.backward( + [result], [paddle.to_tensor(self.dout)], True + ) return result, tensor_src.grad def FusedResNetBasicBlock(self): @@ -148,25 +172,34 @@ def FusedResNetBasicBlock(self): fused_conv1_weight = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) + learning_rate=0.001, + ) fused_conv2_weight = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) + learning_rate=0.001, + ) fused_conv3_weight = fluid.ParamAttr( initializer=fluid.initializer.Xavier(uniform=False), - learning_rate=0.001) + learning_rate=0.001, + ) fused_bn1_weight = fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0) + ) fused_bn1_bias = fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.0)) + initializer=fluid.initializer.Constant(value=0.0) + ) fused_bn2_weight = fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0) + ) fused_bn2_bias = fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.0)) + initializer=fluid.initializer.Constant(value=0.0) + ) fused_bn3_weight = fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0)) + initializer=fluid.initializer.Constant(value=1.0) + ) fused_bn3_bias = fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=0.0)) + initializer=fluid.initializer.Constant(value=0.0) + ) if self.has_shortcut: self.resnet_basic_block = ResNetBasicBlock( @@ -195,7 +228,8 @@ def FusedResNetBasicBlock(self): padding1=1, padding2=1, padding3=0, - has_shortcut=True) + has_shortcut=True, + ) else: self.resnet_basic_block = ResNetBasicBlock( num_channels1=self.in_channels, @@ -223,7 +257,8 @@ def FusedResNetBasicBlock(self): padding1=1, padding2=1, padding3=1, - has_shortcut=False) + has_shortcut=False, + ) x = paddle.to_tensor(self.src, stop_gradient=False) out = self.resnet_basic_block.forward(x) @@ -235,36 +270,46 @@ def test_out_and_grad_has_shortcut(self): default_main_program().random_seed = 1 base_out, base_grad = self.Base() fused_out, fused_grad = self.FusedResNetBasicBlock() - np.testing.assert_allclose(base_out.numpy(), - fused_out.numpy(), - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(base_grad.numpy(), - fused_grad.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + base_out.numpy(), + fused_out.numpy(), + rtol=self.rtol, + atol=self.atol, + ) + np.testing.assert_allclose( + base_grad.numpy(), + fused_grad.numpy(), + rtol=self.rtol, + atol=self.atol, + ) def test_out_and_grad(self): self.has_shortcut = False default_main_program().random_seed = 1 base_out, base_grad = self.Base() fused_out, fused_grad = self.FusedResNetBasicBlock() - np.testing.assert_allclose(base_out.numpy(), - fused_out.numpy(), - rtol=self.rtol, - atol=self.atol) - np.testing.assert_allclose(base_grad.numpy(), - fused_grad.numpy(), - rtol=self.rtol, - atol=self.atol) + np.testing.assert_allclose( + base_out.numpy(), + fused_out.numpy(), + rtol=self.rtol, + atol=self.atol, + ) + np.testing.assert_allclose( + base_grad.numpy(), + fused_grad.numpy(), + rtol=self.rtol, + atol=self.atol, + ) support_types = get_xpu_op_support_types('resnet_basic_block') for stype in support_types: - create_test_class(globals(), - XPUTestResNetBasicBlockOp, - stype, - ignore_device_version=[core.XPUVersion.XPU1]) + create_test_class( + globals(), + XPUTestResNetBasicBlockOp, + stype, + ignore_device_version=[core.XPUVersion.XPU1], + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_gather_nd_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_gather_nd_op_xpu.py index 96e1ded83ae7c..ec4db1fd741bd 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_gather_nd_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_gather_nd_op_xpu.py @@ -20,18 +20,20 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestGatherNd(XPUOpTestWrapper): - def __init__(self): self.op_name = 'gather_nd' class XPUTestGatherNdBase(XPUOpTest): - def setUp(self): self.op_type = "gather_nd" self.dtype = self.in_type @@ -51,115 +53,104 @@ def init_data(self): self.xnp = np.random.random((5, 20)).astype(self.in_type) self.inp = np.array([[], []]).astype("int32") self.output = np.vstack( - (self.xnp[np.newaxis, :], self.xnp[np.newaxis, :])) + (self.xnp[np.newaxis, :], self.xnp[np.newaxis, :]) + ) class XPUTestGatherNdOpWithEmptyIndex1(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.random((5, 20)).astype(self.in_type) self.inp = np.array([[], []]).astype("int32") self.output = np.vstack( - (self.xnp[np.newaxis, :], self.xnp[np.newaxis, :])) + (self.xnp[np.newaxis, :], self.xnp[np.newaxis, :]) + ) class XPUTestGatherNdOpWithEmptyIndex2(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.random((5, 20)).astype(self.in_type) self.inp = np.array([[], []]).astype("int64") self.output = np.vstack( - (self.xnp[np.newaxis, :], self.xnp[np.newaxis, :])) + (self.xnp[np.newaxis, :], self.xnp[np.newaxis, :]) + ) class XPUTestGatherNdOpWithIndex1(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.random((5, 20)).astype(self.in_type) self.inp = np.array([1]).astype("int32") self.output = self.xnp[self.inp] class XPUTestGatherNdOpWithIndex2(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.random((5, 20)).astype(self.in_type) self.inp = np.array([1]).astype("int64") self.output = self.xnp[self.inp] class XPUTestGatherNdOpWithLowIndex1(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.uniform(0, 100, (10, 10)).astype(self.in_type) self.inp = np.array([[1], [2]]).astype("int32") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpWithLowIndex2(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.uniform(0, 100, (10, 10)).astype(self.in_type) self.inp = np.array([1, 2]).astype("int64") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpWithHighRankSame1(XPUTestGatherNdBase): - def init_data(self): shape = (5, 2, 3, 1, 10) self.xnp = np.random.rand(*shape).astype(self.in_type) - self.inp = np.vstack([ - np.random.randint(0, s, size=2) for s in shape - ]).T.astype("int32") + self.inp = np.vstack( + [np.random.randint(0, s, size=2) for s in shape] + ).T.astype("int32") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpWithHighRankSame2(XPUTestGatherNdBase): - def init_data(self): shape = (5, 2, 3, 1, 10) self.xnp = np.random.rand(*shape).astype(self.in_type) - self.inp = np.vstack([ - np.random.randint(0, s, size=2) for s in shape - ]).T.astype("int64") + self.inp = np.vstack( + [np.random.randint(0, s, size=2) for s in shape] + ).T.astype("int64") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpWithHighRankDiff1(XPUTestGatherNdBase): - def init_data(self): shape = (2, 3, 4, 1, 10) self.xnp = np.random.rand(*shape).astype(self.in_type) - self.inp = np.vstack([ - np.random.randint(0, s, size=200) for s in shape - ]).T.astype("int32") + self.inp = np.vstack( + [np.random.randint(0, s, size=200) for s in shape] + ).T.astype("int32") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpWithHighRankDiff2(XPUTestGatherNdBase): - def init_data(self): shape = (2, 3, 4, 1, 10) self.xnp = np.random.rand(*shape).astype(self.in_type) - self.inp = np.vstack([ - np.random.randint(0, s, size=200) for s in shape - ]).T.astype("int64") + self.inp = np.vstack( + [np.random.randint(0, s, size=200) for s in shape] + ).T.astype("int64") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpWithSameIndexAsX1(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.uniform(0, 100, (10, 10)).astype(self.in_type) self.inp = np.array([[1, 1], [2, 1]]).astype("int32") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpWithSameIndexAsX2(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.uniform(0, 100, (10, 10)).astype(self.in_type) self.inp = np.array([[1, 1], [2, 1]]).astype("int64") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpIndex1(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.uniform(0, 100, (10, 10)).astype(self.in_type) self.inp = np.array([1, 2]).astype("int32") self.output = self.xnp[tuple(self.inp.T)] class XPUTestGatherNdOpIndex2(XPUTestGatherNdBase): - def init_data(self): self.xnp = np.random.uniform(0, 100, (10, 10)).astype(self.in_type) self.inp = np.array([1, 2]).astype("int64") diff --git a/python/paddle/fluid/tests/unittests/xpu/test_gather_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_gather_op_xpu.py index 3a1cbdee77a9e..b1a776bd49bf6 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_gather_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_gather_op_xpu.py @@ -21,7 +21,11 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -34,12 +38,10 @@ def gather_numpy(x, index, axis): class XPUTestGather(XPUOpTestWrapper): - def __init__(self): self.op_name = 'gather' class TestXPUGatherOp(XPUOpTest): - def setUp(self): self.op_type = "gather" self.place = paddle.XPUPlace(0) @@ -49,7 +51,7 @@ def setUp(self): xnp = np.random.random(self.x_shape).astype(self.dtype) self.inputs = { 'X': xnp, - 'Index': np.array(self.index).astype(self.index_type) + 'Index': np.array(self.index).astype(self.index_type), } self.outputs = {'Out': self.inputs["X"][self.inputs["Index"]]} @@ -67,28 +69,24 @@ def test_check_grad(self): self.check_grad_with_place(self.place, ['X'], 'Out') class TestCase1(TestXPUGatherOp): - def init_config(self): - self.x_shape = (100) + self.x_shape = 100 self.index = [1, 3, 5] self.index_type = np.int32 class TestCase2(TestXPUGatherOp): - def init_config(self): - self.x_shape = (100) + self.x_shape = 100 self.index = [1, 3, 5] self.index_type = np.int64 class TestCase3(TestXPUGatherOp): - def init_config(self): self.x_shape = (10, 20) self.index = [1, 3, 5] self.index_type = np.int32 class TestCase4(TestXPUGatherOp): - def init_config(self): self.x_shape = (10, 20) self.attrs = {'overwrite': False} @@ -96,7 +94,6 @@ def init_config(self): self.index_type = np.int32 class TestCase5(TestXPUGatherOp): - def init_config(self): self.x_shape = (10, 20) self.attrs = {'overwrite': False} @@ -104,7 +101,6 @@ def init_config(self): self.index_type = np.int32 class TestCase6(TestXPUGatherOp): - def init_config(self): self.x_shape = (10, 20) self.attrs = {'overwrite': True} @@ -112,7 +108,6 @@ def init_config(self): self.index_type = np.int32 class TestCase7(TestXPUGatherOp): - def init_config(self): self.x_shape = (10, 20) self.attrs = {'overwrite': True} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_gaussian_random_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_gaussian_random_op_xpu.py index d95f7b967d542..89725bb8b6d4a 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_gaussian_random_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_gaussian_random_op_xpu.py @@ -20,20 +20,22 @@ import paddle import paddle.fluid as fluid from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle paddle.enable_static() class XPUTestGaussianRandomOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'gaussian_random' self.use_dynamic_create_class = False class TestGaussianRandomOp(XPUOpTest): - def init(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -50,7 +52,7 @@ def setUp(self): "mean": self.mean, "std": self.std, "seed": 10, - "use_mkldnn": self.use_mkldnn + "use_mkldnn": self.use_mkldnn, } paddle.seed(10) @@ -58,11 +60,12 @@ def setUp(self): def set_attrs(self): self.mean = 1.0 - self.std = 2. + self.std = 2.0 def test_check_output(self): - self.check_output_with_place_customized(self.verify_output, - self.place) + self.check_output_with_place_customized( + self.verify_output, self.place + ) def verify_output(self, outs): self.assertEqual(outs[0].shape, (123, 92)) @@ -76,30 +79,28 @@ def verify_output(self, outs): np.testing.assert_allclose(hist, hist2, rtol=0, atol=0.01) class TestMeanStdAreInt(TestGaussianRandomOp): - def set_attrs(self): self.mean = 1 self.std = 2 # Situation 2: Attr(shape) is a list(with tensor) class TestGaussianRandomOp_ShapeTensorList(TestGaussianRandomOp): - def setUp(self): - '''Test gaussian_random op with specified value - ''' + '''Test gaussian_random op with specified value''' self.init() self.init_data() shape_tensor_list = [] for index, ele in enumerate(self.shape): - shape_tensor_list.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor_list.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.attrs = { 'shape': self.infer_shape, 'mean': self.mean, 'std': self.std, 'seed': self.seed, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.inputs = {"ShapeTensorList": shape_tensor_list} @@ -114,12 +115,13 @@ def init_data(self): self.seed = 10 def test_check_output(self): - self.check_output_with_place_customized(self.verify_output, - self.place) + self.check_output_with_place_customized( + self.verify_output, self.place + ) class TestGaussianRandomOp2_ShapeTensorList( - TestGaussianRandomOp_ShapeTensorList): - + TestGaussianRandomOp_ShapeTensorList + ): def init_data(self): self.shape = [123, 92] self.infer_shape = [-1, -1] @@ -129,8 +131,8 @@ def init_data(self): self.seed = 10 class TestGaussianRandomOp3_ShapeTensorList( - TestGaussianRandomOp_ShapeTensorList): - + TestGaussianRandomOp_ShapeTensorList + ): def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -140,8 +142,8 @@ def init_data(self): self.seed = 10 class TestGaussianRandomOp4_ShapeTensorList( - TestGaussianRandomOp_ShapeTensorList): - + TestGaussianRandomOp_ShapeTensorList + ): def init_data(self): self.shape = [123, 92] self.infer_shape = [123, -1] @@ -152,10 +154,8 @@ def init_data(self): # Situation 3: shape is a tensor class TestGaussianRandomOp1_ShapeTensor(TestGaussianRandomOp): - def setUp(self): - '''Test gaussian_random op with specified value - ''' + '''Test gaussian_random op with specified value''' self.init() self.init_data() self.use_mkldnn = False @@ -165,7 +165,7 @@ def setUp(self): 'mean': self.mean, 'std': self.std, 'seed': self.seed, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, } self.outputs = {'Out': np.zeros((123, 92), dtype=self.dtype)} @@ -179,54 +179,61 @@ def init_data(self): # Test python API class TestGaussianRandomAPI(unittest.TestCase): - def test_api(self): positive_2_int32 = fluid.layers.fill_constant([1], "int32", 2000) positive_2_int64 = fluid.layers.fill_constant([1], "int64", 500) - shape_tensor_int32 = fluid.data(name="shape_tensor_int32", - shape=[2], - dtype="int32") - - shape_tensor_int64 = fluid.data(name="shape_tensor_int64", - shape=[2], - dtype="int64") - - out_1 = fluid.layers.gaussian_random(shape=[2000, 500], - dtype="float32", - mean=0.0, - std=1.0, - seed=10) - - out_2 = fluid.layers.gaussian_random(shape=[2000, positive_2_int32], - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_3 = fluid.layers.gaussian_random(shape=[2000, positive_2_int64], - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_4 = fluid.layers.gaussian_random(shape=shape_tensor_int32, - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_5 = fluid.layers.gaussian_random(shape=shape_tensor_int64, - dtype="float32", - mean=0., - std=1.0, - seed=10) - - out_6 = fluid.layers.gaussian_random(shape=shape_tensor_int64, - dtype=np.float32, - mean=0., - std=1.0, - seed=10) + shape_tensor_int32 = fluid.data( + name="shape_tensor_int32", shape=[2], dtype="int32" + ) + + shape_tensor_int64 = fluid.data( + name="shape_tensor_int64", shape=[2], dtype="int64" + ) + + out_1 = fluid.layers.gaussian_random( + shape=[2000, 500], dtype="float32", mean=0.0, std=1.0, seed=10 + ) + + out_2 = fluid.layers.gaussian_random( + shape=[2000, positive_2_int32], + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_3 = fluid.layers.gaussian_random( + shape=[2000, positive_2_int64], + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_4 = fluid.layers.gaussian_random( + shape=shape_tensor_int32, + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_5 = fluid.layers.gaussian_random( + shape=shape_tensor_int64, + dtype="float32", + mean=0.0, + std=1.0, + seed=10, + ) + + out_6 = fluid.layers.gaussian_random( + shape=shape_tensor_int64, + dtype=np.float32, + mean=0.0, + std=1.0, + seed=10, + ) exe = fluid.Executor(place=fluid.XPUPlace(0)) res_1, res_2, res_3, res_4, res_5, res_6 = exe.run( @@ -235,20 +242,21 @@ def test_api(self): "shape_tensor_int32": np.array([2000, 500]).astype("int32"), "shape_tensor_int64": np.array([2000, 500]).astype("int64"), }, - fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6]) + fetch_list=[out_1, out_2, out_3, out_4, out_5, out_6], + ) self.assertAlmostEqual(np.mean(res_1), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_1), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_1), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_2), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_2), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_2), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_3), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_3), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_3), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_4), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_5), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_5), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_5), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_5), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_5), 1.0, delta=0.1) self.assertAlmostEqual(np.mean(res_6), 0.0, delta=0.1) - self.assertAlmostEqual(np.std(res_6), 1., delta=0.1) + self.assertAlmostEqual(np.std(res_6), 1.0, delta=0.1) def test_default_dtype(self): paddle.disable_static() @@ -276,7 +284,6 @@ def test_default_fp64(): class TestStandardNormalDtype(unittest.TestCase): - def test_default_dtype(self): paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_gen_bkcl_id_op.py b/python/paddle/fluid/tests/unittests/xpu/test_gen_bkcl_id_op.py index e5ac263fb7d38..ebecc1b8b71f5 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_gen_bkcl_id_op.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_gen_bkcl_id_op.py @@ -37,30 +37,35 @@ def run_gen_bkc_id(attr): with paddle.static.program_guard(main_program, startup_program): bkcl_id_var = startup_program.global_block().create_var( - name="BKCLID", persistable=True, type=core.VarDesc.VarType.RAW) + name="BKCLID", persistable=True, type=core.VarDesc.VarType.RAW + ) for i in range(1, bkcl_comm_num): startup_program.global_block().create_var( name="BKCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) if use_hallreduce: for i in range(0, bkcl_comm_num): startup_program.global_block().create_var( name="Hierarchical_inter_BKCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().create_var( name="Hierarchical_exter_BKCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().append_op( type="gen_bkcl_id", inputs={}, outputs={"BKCLID": bkcl_id_var}, - attrs=attr) + attrs=attr, + ) place = paddle.CPUPlace() exe = paddle.static.Executor(place) @@ -68,7 +73,6 @@ def run_gen_bkc_id(attr): class TestGenBKCLIdOp(unittest.TestCase): - def setUp(self): try: self._dist_ut_port_0 = int(os.environ["PADDLE_DIST_UT_PORT"]) @@ -101,7 +105,7 @@ def gen_bkcl_id(self, nranks=2): for i in range(nranks): attr['trainer_id'] = i # NOTE: multiprocessing cannot be covered by coverage - p = Process(target=run_gen_bkc_id, args=(attr, )) + p = Process(target=run_gen_bkc_id, args=(attr,)) p.start() procs.append(p) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_generate_proposals_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_generate_proposals_v2_op_xpu.py index d9c5f90e9eefe..d087e8bf7968b 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_generate_proposals_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_generate_proposals_v2_op_xpu.py @@ -24,7 +24,11 @@ import copy from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -49,30 +53,48 @@ def box_coder(all_anchors, bbox_deltas, variances, pixel_offset=True): pred_bbox = np.zeros_like(bbox_deltas, dtype=np.float32) if variances is not None: for i in range(bbox_deltas.shape[0]): - pred_bbox[i, 0] = variances[i, 0] * bbox_deltas[i, 0] * anchor_loc[ - i, 0] + anchor_loc[i, 2] - pred_bbox[i, 1] = variances[i, 1] * bbox_deltas[i, 1] * anchor_loc[ - i, 1] + anchor_loc[i, 3] - pred_bbox[i, 2] = math.exp( - min(variances[i, 2] * bbox_deltas[i, 2], math.log( - 1000 / 16.0))) * anchor_loc[i, 0] - pred_bbox[i, 3] = math.exp( - min(variances[i, 3] * bbox_deltas[i, 3], math.log( - 1000 / 16.0))) * anchor_loc[i, 1] + pred_bbox[i, 0] = ( + variances[i, 0] * bbox_deltas[i, 0] * anchor_loc[i, 0] + + anchor_loc[i, 2] + ) + pred_bbox[i, 1] = ( + variances[i, 1] * bbox_deltas[i, 1] * anchor_loc[i, 1] + + anchor_loc[i, 3] + ) + pred_bbox[i, 2] = ( + math.exp( + min( + variances[i, 2] * bbox_deltas[i, 2], + math.log(1000 / 16.0), + ) + ) + * anchor_loc[i, 0] + ) + pred_bbox[i, 3] = ( + math.exp( + min( + variances[i, 3] * bbox_deltas[i, 3], + math.log(1000 / 16.0), + ) + ) + * anchor_loc[i, 1] + ) else: for i in range(bbox_deltas.shape[0]): - pred_bbox[i, - 0] = bbox_deltas[i, 0] * anchor_loc[i, 0] + anchor_loc[i, - 2] - pred_bbox[i, - 1] = bbox_deltas[i, 1] * anchor_loc[i, 1] + anchor_loc[i, - 3] - pred_bbox[i, 2] = math.exp( - min(bbox_deltas[i, 2], math.log(1000 / 16.0))) * anchor_loc[i, - 0] - pred_bbox[i, 3] = math.exp( - min(bbox_deltas[i, 3], math.log(1000 / 16.0))) * anchor_loc[i, - 1] + pred_bbox[i, 0] = ( + bbox_deltas[i, 0] * anchor_loc[i, 0] + anchor_loc[i, 2] + ) + pred_bbox[i, 1] = ( + bbox_deltas[i, 1] * anchor_loc[i, 1] + anchor_loc[i, 3] + ) + pred_bbox[i, 2] = ( + math.exp(min(bbox_deltas[i, 2], math.log(1000 / 16.0))) + * anchor_loc[i, 0] + ) + pred_bbox[i, 3] = ( + math.exp(min(bbox_deltas[i, 3], math.log(1000 / 16.0))) + * anchor_loc[i, 1] + ) proposals[:, 0] = pred_bbox[:, 0] - pred_bbox[:, 2] / 2 proposals[:, 1] = pred_bbox[:, 1] - pred_bbox[:, 3] / 2 proposals[:, 2] = pred_bbox[:, 0] + pred_bbox[:, 2] / 2 - offset @@ -84,43 +106,47 @@ def box_coder(all_anchors, bbox_deltas, variances, pixel_offset=True): def clip_tiled_boxes(boxes, im_shape, pixel_offset=True): """Clip boxes to image boundaries. im_shape is [height, width] and boxes has shape (N, 4 * num_tiled_boxes).""" - assert boxes.shape[1] % 4 == 0, \ - 'boxes.shape[1] is {:d}, but must be divisible by 4.'.format( + assert ( + boxes.shape[1] % 4 == 0 + ), 'boxes.shape[1] is {:d}, but must be divisible by 4.'.format( boxes.shape[1] ) offset = 1 if pixel_offset else 0 # x1 >= 0 - boxes[:, - 0::4] = np.maximum(np.minimum(boxes[:, 0::4], im_shape[1] - offset), - 0) + boxes[:, 0::4] = np.maximum( + np.minimum(boxes[:, 0::4], im_shape[1] - offset), 0 + ) # y1 >= 0 - boxes[:, - 1::4] = np.maximum(np.minimum(boxes[:, 1::4], im_shape[0] - offset), - 0) + boxes[:, 1::4] = np.maximum( + np.minimum(boxes[:, 1::4], im_shape[0] - offset), 0 + ) # x2 < im_shape[1] - boxes[:, - 2::4] = np.maximum(np.minimum(boxes[:, 2::4], im_shape[1] - offset), - 0) + boxes[:, 2::4] = np.maximum( + np.minimum(boxes[:, 2::4], im_shape[1] - offset), 0 + ) # y2 < im_shape[0] - boxes[:, - 3::4] = np.maximum(np.minimum(boxes[:, 3::4], im_shape[0] - offset), - 0) + boxes[:, 3::4] = np.maximum( + np.minimum(boxes[:, 3::4], im_shape[0] - offset), 0 + ) return boxes def filter_boxes(boxes, min_size, im_shape, pixel_offset=True): - """Only keep boxes with both sides >= min_size and center within the image. - """ + """Only keep boxes with both sides >= min_size and center within the image.""" # Scale min_size to match image scale min_size = max(min_size, 1.0) offset = 1 if pixel_offset else 0 ws = boxes[:, 2] - boxes[:, 0] + offset hs = boxes[:, 3] - boxes[:, 1] + offset if pixel_offset: - x_ctr = boxes[:, 0] + ws / 2. - y_ctr = boxes[:, 1] + hs / 2. - keep = np.where((ws >= min_size) & (hs >= min_size) - & (x_ctr < im_shape[1]) & (y_ctr < im_shape[0]))[0] + x_ctr = boxes[:, 0] + ws / 2.0 + y_ctr = boxes[:, 1] + hs / 2.0 + keep = np.where( + (ws >= min_size) + & (hs >= min_size) + & (x_ctr < im_shape[1]) + & (y_ctr < im_shape[0]) + )[0] else: keep = np.where((ws >= min_size) & (hs >= min_size))[0] return keep @@ -182,9 +208,9 @@ def nms(boxes, scores, nms_threshold, eta=1.0, pixel_offset=True): for k in range(len(selected_indices)): if keep: kept_idx = selected_indices[k] - overlap = iou(boxes[idx], - boxes[kept_idx], - pixel_offset=pixel_offset) + overlap = iou( + boxes[idx], boxes[kept_idx], pixel_offset=pixel_offset + ) keep = True if overlap <= adaptive_threshold else False else: break @@ -195,9 +221,19 @@ def nms(boxes, scores, nms_threshold, eta=1.0, pixel_offset=True): return selected_indices -def proposal_for_one_image(im_shape, all_anchors, variances, bbox_deltas, - scores, pre_nms_topN, post_nms_topN, nms_thresh, - min_size, eta, pixel_offset): +def proposal_for_one_image( + im_shape, + all_anchors, + variances, + bbox_deltas, + scores, + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, + pixel_offset, +): # Transpose and reshape predicted bbox transformations to get them # into the same order as the anchors: # - bbox deltas will be (4 * A, H, W) format from conv output @@ -250,11 +286,13 @@ def proposal_for_one_image(im_shape, all_anchors, variances, bbox_deltas, # take post_nms_topN (e.g. 1000) # return the top proposals if nms_thresh > 0: - keep = nms(boxes=proposals, - scores=scores, - nms_threshold=nms_thresh, - eta=eta, - pixel_offset=pixel_offset) + keep = nms( + boxes=proposals, + scores=scores, + nms_threshold=nms_thresh, + eta=eta, + pixel_offset=pixel_offset, + ) if post_nms_topN > 0 and post_nms_topN < len(keep): keep = keep[:post_nms_topN] proposals = proposals[keep, :] @@ -263,9 +301,19 @@ def proposal_for_one_image(im_shape, all_anchors, variances, bbox_deltas, return proposals, scores -def generate_proposals_v2_in_python(scores, bbox_deltas, im_shape, anchors, - variances, pre_nms_topN, post_nms_topN, - nms_thresh, min_size, eta, pixel_offset): +def generate_proposals_v2_in_python( + scores, + bbox_deltas, + im_shape, + anchors, + variances, + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, + pixel_offset, +): all_anchors = anchors.reshape(-1, 4) rois = np.empty((0, 5), dtype=np.float32) roi_probs = np.empty((0, 1), dtype=np.float32) @@ -276,10 +324,18 @@ def generate_proposals_v2_in_python(scores, bbox_deltas, im_shape, anchors, num_images = scores.shape[0] for img_idx in range(num_images): img_i_boxes, img_i_probs = proposal_for_one_image( - im_shape[img_idx, :], all_anchors, variances, - bbox_deltas[img_idx, :, :, :], scores[img_idx, :, :, :], - pre_nms_topN, post_nms_topN, nms_thresh, min_size, eta, - pixel_offset) + im_shape[img_idx, :], + all_anchors, + variances, + bbox_deltas[img_idx, :, :, :], + scores[img_idx, :, :, :], + pre_nms_topN, + post_nms_topN, + nms_thresh, + min_size, + eta, + pixel_offset, + ) rois_num.append(img_i_probs.shape[0]) rpn_rois.append(img_i_boxes) rpn_roi_probs.append(img_i_probs) @@ -287,8 +343,9 @@ def generate_proposals_v2_in_python(scores, bbox_deltas, im_shape, anchors, return rpn_rois, rpn_roi_probs, rois_num -def anchor_generator_in_python(input_feat, anchor_sizes, aspect_ratios, - variances, stride, offset): +def anchor_generator_in_python( + input_feat, anchor_sizes, aspect_ratios, variances, stride, offset +): num_anchors = len(aspect_ratios) * len(anchor_sizes) layer_h = input_feat.shape[2] layer_w = input_feat.shape[3] @@ -312,11 +369,12 @@ def anchor_generator_in_python(input_feat, anchor_sizes, aspect_ratios, scale_h = anchor_size / stride[1] w = scale_w * base_w h = scale_h * base_h - out_anchors[h_idx, w_idx, - idx, :] = [(x_ctr - 0.5 * (w - 1)), - (y_ctr - 0.5 * (h - 1)), - (x_ctr + 0.5 * (w - 1)), - (y_ctr + 0.5 * (h - 1))] + out_anchors[h_idx, w_idx, idx, :] = [ + (x_ctr - 0.5 * (w - 1)), + (y_ctr - 0.5 * (h - 1)), + (x_ctr + 0.5 * (w - 1)), + (y_ctr + 0.5 * (h - 1)), + ] idx += 1 # set the variance. @@ -327,13 +385,11 @@ def anchor_generator_in_python(input_feat, anchor_sizes, aspect_ratios, class XPUGenerateProposalsV2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = 'generate_proposals_v2' self.use_dynamic_create_class = False class TestGenerateProposalsV2Op(XPUOpTest): - def set_data(self): self.init_input_shape() self.init_test_params() @@ -344,7 +400,7 @@ def set_data(self): 'BboxDeltas': self.bbox_deltas, 'ImShape': self.im_shape.astype(self.dtype), 'Anchors': self.anchors, - 'Variances': self.variances + 'Variances': self.variances, } self.attrs = { @@ -388,7 +444,7 @@ def init_test_params(self): self.post_nms_topN = 5000 # train 6000, test 1000 self.nms_thresh = 0.7 self.min_size = 3.0 - self.eta = 1. + self.eta = 1.0 self.pixel_offset = True def init_test_input(self): @@ -396,30 +452,45 @@ def init_test_input(self): input_channels = self.input_feat_shape[1] layer_h = self.input_feat_shape[2] layer_w = self.input_feat_shape[3] - input_feat = np.random.random((batch_size, input_channels, layer_h, - layer_w)).astype(self.dtype) + input_feat = np.random.random( + (batch_size, input_channels, layer_h, layer_w) + ).astype(self.dtype) self.anchors, self.variances = anchor_generator_in_python( input_feat=input_feat, - anchor_sizes=[16., 32.], + anchor_sizes=[16.0, 32.0], aspect_ratios=[0.5, 1.0], variances=[1.0, 1.0, 1.0, 1.0], stride=[16.0, 16.0], - offset=0.5) + offset=0.5, + ) num_anchors = self.anchors.shape[2] self.scores = np.random.random( - (batch_size, num_anchors, layer_h, layer_w)).astype(self.dtype) + (batch_size, num_anchors, layer_h, layer_w) + ).astype(self.dtype) self.bbox_deltas = np.random.random( - (batch_size, num_anchors * 4, layer_h, - layer_w)).astype(self.dtype) + (batch_size, num_anchors * 4, layer_h, layer_w) + ).astype(self.dtype) def init_test_output(self): - self.rpn_rois, self.rpn_roi_probs, self.rois_num = generate_proposals_v2_in_python( - self.scores, self.bbox_deltas, self.im_shape, self.anchors, - self.variances, self.pre_nms_topN, self.post_nms_topN, - self.nms_thresh, self.min_size, self.eta, self.pixel_offset) + ( + self.rpn_rois, + self.rpn_roi_probs, + self.rois_num, + ) = generate_proposals_v2_in_python( + self.scores, + self.bbox_deltas, + self.im_shape, + self.anchors, + self.variances, + self.pre_nms_topN, + self.post_nms_topN, + self.nms_thresh, + self.min_size, + self.eta, + self.pixel_offset, + ) class TestGenerateProposalsV2OutLodOp(TestGenerateProposalsV2Op): - def set_data(self): self.init_input_shape() self.init_test_params() @@ -430,7 +501,7 @@ def set_data(self): 'BboxDeltas': self.bbox_deltas, 'ImShape': self.im_shape.astype(np.float32), 'Anchors': self.anchors, - 'Variances': self.variances + 'Variances': self.variances, } self.attrs = { @@ -440,37 +511,34 @@ def set_data(self): 'min_size': self.min_size, 'eta': self.eta, 'pixel_offset': self.pixel_offset, - 'return_rois_num': True + 'return_rois_num': True, } self.outputs = { 'RpnRois': (self.rpn_rois[0], [self.rois_num]), 'RpnRoiProbs': (self.rpn_roi_probs[0], [self.rois_num]), - 'RpnRoisNum': (np.asarray(self.rois_num, dtype=np.int32)) + 'RpnRoisNum': (np.asarray(self.rois_num, dtype=np.int32)), } class TestGenerateProposalsV2OpNoBoxLeft(TestGenerateProposalsV2Op): - def init_test_params(self): self.pre_nms_topN = 12000 # train 12000, test 2000 self.post_nms_topN = 5000 # train 6000, test 1000 self.nms_thresh = 0.7 self.min_size = 1000.0 - self.eta = 1. + self.eta = 1.0 self.pixel_offset = True class TestGenerateProposalsV2OpNoOffset(TestGenerateProposalsV2Op): - def init_test_params(self): self.pre_nms_topN = 12000 # train 12000, test 2000 self.post_nms_topN = 5000 # train 6000, test 1000 self.nms_thresh = 0.7 self.min_size = 3.0 - self.eta = 1. + self.eta = 1.0 self.pixel_offset = False class TestGenerateProposalsV2OpMaskRcnn1XPU(TestGenerateProposalsV2Op): - def init_input_shape(self): self.input_feat_shape = (1, 20, 48, 64) # Another case is [768, 1024] @@ -481,7 +549,7 @@ def init_test_params(self): self.post_nms_topN = 2000 # train 6000, test 1000 self.nms_thresh = 0.7 self.min_size = 0.0 - self.eta = 1. + self.eta = 1.0 self.pixel_offset = False def init_test_input(self): @@ -489,21 +557,24 @@ def init_test_input(self): input_channels = self.input_feat_shape[1] layer_h = self.input_feat_shape[2] layer_w = self.input_feat_shape[3] - input_feat = np.random.random((batch_size, input_channels, layer_h, - layer_w)).astype(self.dtype) + input_feat = np.random.random( + (batch_size, input_channels, layer_h, layer_w) + ).astype(self.dtype) self.anchors, self.variances = anchor_generator_in_python( input_feat=input_feat, anchor_sizes=[32, 64, 128, 256, 512], aspect_ratios=[0.5, 1.0, 2.0], variances=[1.0, 1.0, 1.0, 1.0], stride=[16.0, 16.0], - offset=0.5) + offset=0.5, + ) num_anchors = self.anchors.shape[2] self.scores = np.random.random( - (batch_size, num_anchors, layer_h, layer_w)).astype(self.dtype) + (batch_size, num_anchors, layer_h, layer_w) + ).astype(self.dtype) self.bbox_deltas = np.random.random( - (batch_size, num_anchors * 4, layer_h, - layer_w)).astype(self.dtype) + (batch_size, num_anchors * 4, layer_h, layer_w) + ).astype(self.dtype) self.anchors = self.anchors.reshape(-1, 4) self.variances = self.variances.reshape(-1, 4) @@ -519,7 +590,7 @@ def set_data(self): 'BboxDeltas': self.bbox_deltas, 'ImShape': self.im_shape.astype(np.float32), 'Anchors': self.anchors, - 'Variances': self.variances + 'Variances': self.variances, } self.attrs = { @@ -529,23 +600,25 @@ def set_data(self): 'min_size': self.min_size, 'eta': self.eta, 'pixel_offset': self.pixel_offset, - 'return_rois_num': True + 'return_rois_num': True, } self.outputs = { 'RpnRois': (self.rpn_rois[0], [self.rois_num]), 'RpnRoiProbs': (self.rpn_roi_probs[0], [self.rois_num]), - 'RpnRoisNum': (np.asarray(self.rois_num, dtype=np.int32)) + 'RpnRoisNum': (np.asarray(self.rois_num, dtype=np.int32)), } support_types = get_xpu_op_support_types('generate_proposals_v2') for stype in support_types: - create_test_class(globals(), - XPUGenerateProposalsV2Op, - stype, - test_grad=False, - ignore_device_version=[core.XPUVersion.XPU1]) + create_test_class( + globals(), + XPUGenerateProposalsV2Op, + stype, + test_grad=False, + ignore_device_version=[core.XPUVersion.XPU1], + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_grid_sampler_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_grid_sampler_op_xpu.py index fbd5c68ddd4f3..eb992cd42132c 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_grid_sampler_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_grid_sampler_op_xpu.py @@ -21,7 +21,11 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -30,12 +34,15 @@ def AffineGrid(theta, grid_shape): n = grid_shape[0] h = grid_shape[1] w = grid_shape[2] - h_idx = np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, - axis=0).T[:, :, np.newaxis] - w_idx = np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, - axis=0)[:, :, np.newaxis] - grid = np.concatenate([w_idx, h_idx, np.ones([h, w, 1])], - axis=2) # h * w * 3 + h_idx = np.repeat(np.linspace(-1, 1, h)[np.newaxis, :], w, axis=0).T[ + :, :, np.newaxis + ] + w_idx = np.repeat(np.linspace(-1, 1, w)[np.newaxis, :], h, axis=0)[ + :, :, np.newaxis + ] + grid = np.concatenate( + [w_idx, h_idx, np.ones([h, w, 1])], axis=2 + ) # h * w * 3 grid = np.repeat(grid[np.newaxis, :], n, axis=0) # n * h * w *3 ret = np.zeros([n, h * w, 2]) @@ -55,13 +62,17 @@ def getGridPointValue(data, x, y): out_H = x.shape[1] out_W = x.shape[2] - #out = np.zeros(data_shape, dtype='float64') + # out = np.zeros(data_shape, dtype='float64') out = np.zeros([N, C, out_H, out_W], dtype='float64') for i in range(N): for j in range(out_H): for k in range(out_W): - if y[i, j, k] < 0 or y[i, j, k] > in_H - 1 or x[ - i, j, k] < 0 or x[i, j, k] > in_W - 1: + if ( + y[i, j, k] < 0 + or y[i, j, k] > in_H - 1 + or x[i, j, k] < 0 + or x[i, j, k] > in_W - 1 + ): out[i, :, j, k] = 0 else: out[i, :, j, k] = data[i, :, y[i, j, k], x[i, j, k]] @@ -77,27 +88,28 @@ def unnormalizeAndClip(grid_slice, max_val, align_corners, padding_mode): if align_corners: grid_slice = 0.5 * ((grid_slice.astype('float64') + 1.0) * max_val) else: - grid_slice = 0.5 * ((grid_slice.astype('float64') + 1.0) * - (max_val + 1)) - 0.5 + grid_slice = ( + 0.5 * ((grid_slice.astype('float64') + 1.0) * (max_val + 1)) - 0.5 + ) if padding_mode == "border": grid_slice = clip(grid_slice, 0, max_val) elif padding_mode == "reflection": double_range = 2 * max_val if align_corners else (max_val + 1) * 2 - grid_abs = np.abs(grid_slice) if align_corners else np.abs(grid_slice + - 0.5) + grid_abs = ( + np.abs(grid_slice) if align_corners else np.abs(grid_slice + 0.5) + ) extra = grid_abs - np.floor(grid_abs / double_range) * double_range grid_slice = np.minimum(extra, double_range - extra) - grid_slice = grid_slice if align_corners else clip( - grid_slice - 0.5, 0, max_val) + grid_slice = ( + grid_slice if align_corners else clip(grid_slice - 0.5, 0, max_val) + ) return grid_slice -def GridSampler(data, - grid, - align_corners=True, - mode="bilinear", - padding_mode="zeros"): +def GridSampler( + data, grid, align_corners=True, mode="bilinear", padding_mode="zeros" +): dims = data.shape N = dims[0] in_C = dims[1] @@ -121,14 +133,18 @@ def GridSampler(data, y0 = np.floor(y).astype('int32') y1 = y0 + 1 - wa = np.tile(((x1 - x) * (y1 - y)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wb = np.tile(((x1 - x) * (y - y0)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wc = np.tile(((x - x0) * (y1 - y)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) - wd = np.tile(((x - x0) * (y - y0)).reshape((N, 1, out_H, out_W)), - (1, in_C, 1, 1)) + wa = np.tile( + ((x1 - x) * (y1 - y)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wb = np.tile( + ((x1 - x) * (y - y0)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wc = np.tile( + ((x - x0) * (y1 - y)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) + wd = np.tile( + ((x - x0) * (y - y0)).reshape((N, 1, out_H, out_W)), (1, in_C, 1, 1) + ) va = getGridPointValue(data, x0, y0) vb = getGridPointValue(data, x0, y1) @@ -144,13 +160,11 @@ def GridSampler(data, class XPUTestGridSamplerOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'grid_sampler' self.use_dynamic_create_class = False class TestXPUGridSamplerOp(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -180,9 +194,9 @@ def setUp(self): "mode": self.mode, } self.outputs = { - 'Output': - GridSampler(x, grid, self.align_corners, self.mode, - self.padding_mode) + 'Output': GridSampler( + x, grid, self.align_corners, self.mode, self.padding_mode + ) } def initTestCase(self): @@ -203,7 +217,6 @@ def test_check_grad(self): self.check_grad_with_place(self.place, ['X', 'Grid'], 'Output') class TestGridSample1(TestXPUGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -213,7 +226,6 @@ def initTestCase(self): self.mode = "bilinear" class TestGridSample2(TestXPUGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -223,7 +235,6 @@ def initTestCase(self): self.mode = "bilinear" class TestGridSample3(TestXPUGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -233,7 +244,6 @@ def initTestCase(self): self.mode = "bilinear" class TestGridSample4(TestXPUGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -243,7 +253,6 @@ def initTestCase(self): self.mode = "bilinear" class TestGridSample5(TestXPUGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 5, 6) self.grid_shape = (2, 8, 9, 2) @@ -253,7 +262,6 @@ def initTestCase(self): self.mode = "nearest" class TestGridSample6(TestXPUGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 128, 128) self.grid_shape = (2, 130, 130, 2) @@ -263,7 +271,6 @@ def initTestCase(self): self.mode = "bilinear" class TestGridSample7(TestXPUGridSamplerOp): - def initTestCase(self): self.x_shape = (2, 3, 128, 128) self.grid_shape = (2, 130, 130, 2) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_huber_loss_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_huber_loss_op_xpu.py index 1e7361035dfc0..50d77fc1a3d5f 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_huber_loss_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_huber_loss_op_xpu.py @@ -22,7 +22,11 @@ from op_test import OpTest from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -36,13 +40,11 @@ def huber_loss_forward(val, delta): class XPUTestHuberLossOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'huber_loss' self.use_dynamic_create_class = False class TestHuberLossOp(XPUOpTest): - def setUp(self): self.set_xpu() self.op_type = 'huber_loss' @@ -55,11 +57,11 @@ def setUp(self): def set_inputs(self): shape = self.set_shape() - x = np.random.uniform(0, 1., shape).astype(self.dtype) - y = np.random.uniform(0, 1., shape).astype(self.dtype) + x = np.random.uniform(0, 1.0, shape).astype(self.dtype) + y = np.random.uniform(0, 1.0, shape).astype(self.dtype) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } def set_attrs(self): @@ -69,8 +71,9 @@ def set_outputs(self): delta = self.attrs['delta'] shape = self.set_shape() residual = self.inputs['Y'] - self.inputs['X'] - loss = np.vectorize(huber_loss_forward)(residual, - delta).astype(self.dtype) + loss = np.vectorize(huber_loss_forward)(residual, delta).astype( + self.dtype + ) self.outputs = {'Residual': residual, 'Out': loss.reshape(shape)} def set_shape(self): @@ -89,27 +92,24 @@ def test_check_grad_normal(self): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("residual")) + self.check_grad_with_place( + self.place, ['Y'], 'Out', no_grad_set=set("residual") + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set('residual')) + self.check_grad_with_place( + self.place, ['X'], 'Out', no_grad_set=set('residual') + ) class TestHuberLossOp1(TestHuberLossOp): - def set_shape(self): - return (640) + return 640 class TestHuberLossOp2(TestHuberLossOp): - def set_shape(self): return (10, 10) class TestHuberLossOp3(TestHuberLossOp): - def set_shape(self): return (10, 10, 1) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_index_select_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_index_select_op_xpu.py index 3eb8ff49f7fd5..816e7ac7967de 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_index_select_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_index_select_op_xpu.py @@ -23,33 +23,34 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestIndexSelect(XPUOpTestWrapper): - def __init__(self): self.op_name = 'index_select' class TestXPUIndexSelectOp(XPUOpTest): - def setUp(self): self.op_type = "index_select" self.place = paddle.XPUPlace(0) self.dtype = self.in_type self.init_dtype_type() - index_np = np.random.randint(low=0, - high=self.x_shape[self.dim], - size=self.index_size).astype( - self.index_type) + index_np = np.random.randint( + low=0, high=self.x_shape[self.dim], size=self.index_size + ).astype(self.index_type) x_np = np.random.random(self.x_shape).astype(self.dtype) self.inputs = {'X': x_np, 'Index': index_np} self.attrs = {'dim': self.dim} - outer_loop = np.prod(self.x_shape[:self.dim]) - x_reshape = [outer_loop] + list(self.x_shape[self.dim:]) + outer_loop = np.prod(self.x_shape[: self.dim]) + x_reshape = [outer_loop] + list(self.x_shape[self.dim :]) x_np_reshape = np.reshape(x_np, tuple(x_reshape)) out_list = [] for i in range(outer_loop): @@ -77,7 +78,6 @@ def test_check_grad(self): self.check_grad_with_place(self.place, ['X'], 'Out') class TestXPUIndexSelectOpCase2(TestXPUIndexSelectOp): - def init_dtype_type(self): self.index_type = np.int32 self.dim = -2 @@ -86,10 +86,14 @@ def init_dtype_type(self): class TestIndexSelectAPI(unittest.TestCase): - def input_data(self): - self.data_x = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [9.0, 10.0, 11.0, 12.0]]) + self.data_x = np.array( + [ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + ] + ) self.data_index = np.array([0, 1, 1]).astype('int32') def test_index_select_api(self): @@ -98,39 +102,37 @@ def test_index_select_api(self): # case 1: with program_guard(Program(), Program()): x = fluid.layers.data(name='x', shape=[-1, 4]) - index = fluid.layers.data(name='index', - shape=[3], - dtype='int32', - append_batch_size=False) + index = fluid.layers.data( + name='index', shape=[3], dtype='int32', append_batch_size=False + ) z = paddle.index_select(x, index, axis=1) exe = fluid.Executor(fluid.XPUPlace(0)) - res, = exe.run(feed={ - 'x': self.data_x, - 'index': self.data_index - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], - [9.0, 10.0, 10.0]]) + (res,) = exe.run( + feed={'x': self.data_x, 'index': self.data_index}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], [9.0, 10.0, 10.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) # case 2: with program_guard(Program(), Program()): x = fluid.layers.data(name='x', shape=[-1, 4]) - index = fluid.layers.data(name='index', - shape=[3], - dtype='int32', - append_batch_size=False) + index = fluid.layers.data( + name='index', shape=[3], dtype='int32', append_batch_size=False + ) z = paddle.index_select(x, index) exe = fluid.Executor(fluid.XPUPlace(0)) - res, = exe.run(feed={ - 'x': self.data_x, - 'index': self.data_index - }, - fetch_list=[z.name], - return_numpy=False) - expect_out = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [5.0, 6.0, 7.0, 8.0]]) + (res,) = exe.run( + feed={'x': self.data_x, 'index': self.data_index}, + fetch_list=[z.name], + return_numpy=False, + ) + expect_out = np.array( + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [5.0, 6.0, 7.0, 8.0]] + ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) def test_dygraph_api(self): @@ -141,8 +143,9 @@ def test_dygraph_api(self): index = fluid.dygraph.to_variable(self.data_index) z = paddle.index_select(x, index) np_z = z.numpy() - expect_out = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], - [5.0, 6.0, 7.0, 8.0]]) + expect_out = np.array( + [[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [5.0, 6.0, 7.0, 8.0]] + ) np.testing.assert_allclose(expect_out, np_z, rtol=1e-05) # case 2: @@ -151,8 +154,9 @@ def test_dygraph_api(self): index = fluid.dygraph.to_variable(self.data_index) z = paddle.index_select(x, index, axis=1) np_z = z.numpy() - expect_out = np.array([[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], - [9.0, 10.0, 10.0]]) + expect_out = np.array( + [[1.0, 2.0, 2.0], [5.0, 6.0, 6.0], [9.0, 10.0, 10.0]] + ) np.testing.assert_allclose(expect_out, np_z, rtol=1e-05) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_instance_norm_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_instance_norm_op_xpu.py index 55d942e0d72ce..8fc1ca75240db 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_instance_norm_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_instance_norm_op_xpu.py @@ -19,7 +19,11 @@ sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -53,13 +57,11 @@ def _cal_mean_variance(x, epsilon, mean_shape): class XPUTestInstanceNormOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'instance_norm' self.use_dynamic_create_class = False class XPUTestInstanceNormOp(XPUOpTest): - def setUp(self): self.op_type = "instance_norm" self.dtype = self.in_type @@ -79,8 +81,13 @@ def setUp(self): bias_np = np.random.random_sample(scale_shape).astype(np.float32) mean, variance = self.set_global_mean_var(mean_shape, x_np) - ref_y_np, ref_saved_mean, variance_tmp = _reference_instance_norm_naive( - x_np, scale_np, bias_np, epsilon, mean, variance) + ( + ref_y_np, + ref_saved_mean, + variance_tmp, + ) = _reference_instance_norm_naive( + x_np, scale_np, bias_np, epsilon, mean, variance + ) ref_saved_variance = 1 / np.sqrt(variance_tmp + epsilon) @@ -88,7 +95,7 @@ def setUp(self): self.outputs = { 'Y': ref_y_np, 'SavedMean': ref_saved_mean, - 'SavedVariance': ref_saved_variance + 'SavedVariance': ref_saved_variance, } self.attrs = {'epsilon': epsilon, 'use_xpu': True} @@ -106,27 +113,22 @@ def test_check_grad(self): self.check_grad_with_place(paddle.XPUPlace(0), ['X'], 'Y') class TestXPUInstanceNormOp1(XPUTestInstanceNormOp): - def set_attrs(self): self.shape = [10, 12, 32, 32] class TestXPUInstanceNormOp2(XPUTestInstanceNormOp): - def set_attrs(self): self.shape = [4, 5, 6, 7] class TestXPUInstanceNormOp3(XPUTestInstanceNormOp): - def set_attrs(self): self.shape = [1, 8, 16, 16] class TestXPUInstanceNormOp4(XPUTestInstanceNormOp): - def set_attrs(self): self.shape = [4, 16, 256, 128] class TestXPUInstanceNormOp5(XPUTestInstanceNormOp): - def set_attrs(self): self.shape = [10, 3, 512, 1] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_iou_similarity_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_iou_similarity_op_xpu.py index 2aac4dc83fe29..548976ae62e91 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_iou_similarity_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_iou_similarity_op_xpu.py @@ -18,20 +18,22 @@ sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle paddle.enable_static() class XPUTestIOUSimilarityOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'iou_similarity' self.use_dynamic_create_class = False class TestXPUIOUSimilarityOp(XPUOpTest): - def init(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -51,11 +53,13 @@ def setUp(self): self.inputs = {'X': self.boxes1, 'Y': self.boxes2} self.attrs = { "box_normalized": self.box_normalized, - 'use_xpu': True + 'use_xpu': True, } self.outputs = {'Out': self.output} - def _compute_iou(self, ): + def _compute_iou( + self, + ): for row in range(self.boxes1.shape[0]): for col in range(self.boxes2.shape[0]): xmin1, ymin1, xmax1, ymax1 = self.boxes1[row] @@ -84,7 +88,6 @@ def _compute_iou(self, ): self.output[row, col] = sim_score class TestXPUIOUSimilarityOpWithLoD(TestXPUIOUSimilarityOp): - def test_check_output(self): self.check_output_with_place(self.place, check_dygraph=False) @@ -97,13 +100,12 @@ def setUp(self): self._compute_iou() self.inputs = { 'X': (self.boxes1, self.boxes1_lod), - 'Y': self.boxes2 + 'Y': self.boxes2, } self.attrs = {"box_normalized": self.box_normalized} self.outputs = {'Out': (self.output, self.output_lod)} class TestXPUIOUSimilarityOpWithBoxNormalized(TestXPUIOUSimilarityOp): - def test_check_output(self): self.check_output_with_place(self.place, check_dygraph=False) @@ -116,7 +118,7 @@ def setUp(self): self._compute_iou() self.inputs = { 'X': (self.boxes1, self.boxes1_lod), - 'Y': self.boxes2 + 'Y': self.boxes2, } self.attrs = {"box_normalized": self.box_normalized} self.outputs = {'Out': (self.output, self.output_lod)} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_label_smooth_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_label_smooth_op_xpu.py index 3384a0b093808..0a2c7b6b77331 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_label_smooth_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_label_smooth_op_xpu.py @@ -19,13 +19,16 @@ sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestLabelSmoothOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'label_smooth' self.use_dynamic_create_class = True @@ -37,15 +40,15 @@ def dynamic_create_class(self): label_dims = [1, 7, 12] for bs in batch_sizes: for label_dim in label_dims: - class_name = 'XPUTestLabelSmooth_' + \ - str(bs) + "_" + str(label_dim) + class_name = ( + 'XPUTestLabelSmooth_' + str(bs) + "_" + str(label_dim) + ) attr_dict = {'batch_size': bs, 'label_dim': label_dim} classes.append([class_name, attr_dict]) classes.append(['XPUTestLabelSmooth_3d', {'is_3d': True}]) return base_class, classes class TestLabelSmoothOp(XPUOpTest): - def setUp(self): self.op_type = "label_smooth" self.epsilon = 0.1 @@ -53,21 +56,26 @@ def setUp(self): if not hasattr(self, 'batch_size'): self.batch_size = 10 self.label_dim = 12 - self.label = np.zeros( - (self.batch_size, self.label_dim)).astype("float32") - nonzero_index = np.random.randint(self.label_dim, - size=(self.batch_size)) + self.label = np.zeros((self.batch_size, self.label_dim)).astype( + "float32" + ) + nonzero_index = np.random.randint( + self.label_dim, size=(self.batch_size) + ) self.label[np.arange(self.batch_size), nonzero_index] = 1 smoothed_label = ( - 1 - self.epsilon) * self.label + self.epsilon / self.label_dim + 1 - self.epsilon + ) * self.label + self.epsilon / self.label_dim self.inputs = {'X': self.label} self.attrs = {'epsilon': self.epsilon} self.outputs = {'Out': smoothed_label} if hasattr(self, 'is_3d') and self.is_3d: self.inputs['X'] = self.inputs['X'].reshape( - [2, -1, self.inputs['X'].shape[-1]]) + [2, -1, self.inputs['X'].shape[-1]] + ) self.outputs['Out'] = self.outputs['Out'].reshape( - self.inputs['X'].shape) + self.inputs['X'].shape + ) def test_check_output(self): if not paddle.is_compiled_with_xpu(): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_lamb_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_lamb_op_xpu.py index b86dae39426a7..0c5354c957597 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_lamb_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_lamb_op_xpu.py @@ -20,7 +20,11 @@ from op_test_xpu import XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def lamb_step(inputs, attributes): @@ -51,14 +55,16 @@ def lamb_step(inputs, attributes): moment2_unbiased = moment2_out / (1 - beta2_pow) r_1 = np.linalg.norm(param) - r_2 = np.linalg.norm(moment1_unbiased / - (np.sqrt(moment2_unbiased) + epsilon) + - weight_decay * param) + r_2 = np.linalg.norm( + moment1_unbiased / (np.sqrt(moment2_unbiased) + epsilon) + + weight_decay * param + ) lr_t = lr * r_1 / r_2 - param_out = param - lr_t * (moment1_unbiased / - (np.sqrt(moment2_unbiased) + epsilon) + - weight_decay * param) + param_out = param - lr_t * ( + moment1_unbiased / (np.sqrt(moment2_unbiased) + epsilon) + + weight_decay * param + ) beta1_pow_out = beta1_pow * beta1 beta2_pow_out = beta2_pow * beta2 @@ -67,24 +73,21 @@ def lamb_step(inputs, attributes): class XPUTestLambOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'lamb' self.use_dynamic_create_class = False class TestLambOp1(XPUOpTest): - def set_attrs(self): self.attrs = { 'epsilon': 1e-4, 'beta1': 0.78, 'beta2': 0.836, - 'weight_decay': 0.01 + 'weight_decay': 0.01, } def setUp(self): - '''Test Lamb Op with supplied attributes - ''' + '''Test Lamb Op with supplied attributes''' # self.op_type = self.op_name self.__class__.op_type = 'lamb' self.dtype = self.in_type @@ -105,56 +108,63 @@ def setUp(self): 'Moment2': moment2, 'LearningRate': np.array([learning_rate]).astype("float32"), 'Beta1Pow': np.array([beta1_pow]).astype("float32"), - 'Beta2Pow': np.array([beta2_pow]).astype("float32") + 'Beta2Pow': np.array([beta2_pow]).astype("float32"), } - - param_out, moment1_out, moment2_out, \ - beta1_pow_out, beta2_pow_out = lamb_step(self.inputs, self.attrs) + ( + param_out, + moment1_out, + moment2_out, + beta1_pow_out, + beta2_pow_out, + ) = lamb_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out + 'Beta2PowOut': beta2_pow_out, } def test_check_output(self): self.check_output_with_place(paddle.XPUPlace(0)) class TestLambOp2(TestLambOp1): - def set_attrs(self): self.attrs = { 'epsilon': 1e-8, 'beta1': 0.9, 'beta2': 0.999, - 'weight_decay': 0.01 + 'weight_decay': 0.01, } class TestLambOpMultipleSteps(TestLambOp1): - def set_attrs(self): self.attrs = { 'epsilon': 1e-8, 'beta1': 0.9, 'beta2': 0.999, - 'weight_decay': 0.01 + 'weight_decay': 0.01, } self.num_steps = 10 def test_check_output(self): for i in range(self.num_steps): - param_out, moment1_out, moment2_out, \ - beta1_pow_out, beta2_pow_out = lamb_step(self.inputs, self.attrs) + ( + param_out, + moment1_out, + moment2_out, + beta1_pow_out, + beta2_pow_out, + ) = lamb_step(self.inputs, self.attrs) self.outputs = { 'Moment1Out': moment1_out, 'Moment2Out': moment2_out, 'ParamOut': param_out, 'Beta1PowOut': beta1_pow_out, - 'Beta2PowOut': beta2_pow_out + 'Beta2PowOut': beta2_pow_out, } # Verify output for this step @@ -171,7 +181,8 @@ def test_check_output(self): # Randomize gradient for next step self.inputs['Grad'] = np.random.uniform( - -1, 1, (102, 105)).astype("float32") + -1, 1, (102, 105) + ).astype("float32") support_types = get_xpu_op_support_types('lamb') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_layer_norm_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_layer_norm_op_xpu.py index db0ac5189322f..3c377db4c0b55 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_layer_norm_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_layer_norm_op_xpu.py @@ -21,7 +21,11 @@ sys.path.append("..") from op_test_xpu import XPUOpTest from operator import mul -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -29,12 +33,13 @@ def ref_layer_norm(x, scale, bias, epsilon, begin_norm_axis=1): x_shape = x.shape left = reduce(mul, x_shape[0:begin_norm_axis], 1) - right = reduce(mul, x_shape[begin_norm_axis:len(x_shape)], 1) + right = reduce(mul, x_shape[begin_norm_axis : len(x_shape)], 1) x.shape = [left, right] mean = np.mean(x, axis=1) variance = np.var(x, axis=1) + epsilon - y = np.divide((x - mean.reshape([left, 1])), - (np.sqrt(variance)).reshape([left, 1])) + y = np.divide( + (x - mean.reshape([left, 1])), (np.sqrt(variance)).reshape([left, 1]) + ) if scale is not None: y = scale.reshape([1, right]) * y if bias is not None: @@ -44,13 +49,11 @@ def ref_layer_norm(x, scale, bias, epsilon, begin_norm_axis=1): class XPUTestLayerNormOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'layer_norm' self.use_dynamic_create_class = False class TestXPULayerNormOp(XPUOpTest): - def setUp(self): self.op_type = "layer_norm" self.dtype = self.in_type @@ -63,25 +66,27 @@ def setUp(self): if self.dtype == np.float16: self.atol = 1e-2 - right = reduce(mul, - self.shape[self.begin_norm_axis:len(self.shape)], 1) + right = reduce( + mul, self.shape[self.begin_norm_axis : len(self.shape)], 1 + ) np.random.seed(10) x_np = np.random.uniform(0.1, 1, self.shape).astype(self.dtype) scale_np = np.random.uniform(0.1, 1, [right]).astype('float32') bias_np = np.random.uniform(0.1, 1, [right]).astype('float32') ref_y_np, ref_mean_np, ref_variance_np = ref_layer_norm( - x_np, scale_np, bias_np, self.epsilon, self.begin_norm_axis) + x_np, scale_np, bias_np, self.epsilon, self.begin_norm_axis + ) ref_y_np = ref_y_np.astype(self.dtype) self.inputs = {'X': x_np, 'Scale': scale_np, 'Bias': bias_np} self.outputs = { 'Y': ref_y_np, 'Mean': ref_mean_np, - 'Variance': ref_variance_np + 'Variance': ref_variance_np, } self.attrs = { 'begin_norm_axis': self.begin_norm_axis, - 'use_xpu': True + 'use_xpu': True, } def set_attrs(self): @@ -91,27 +96,23 @@ def test_check_output(self): self.check_output_with_place(paddle.XPUPlace(0), atol=self.atol) def test_check_grad(self): - self.check_grad_with_place(paddle.XPUPlace(0), ['X'], - 'Y', - max_relative_error=self.atol) + self.check_grad_with_place( + paddle.XPUPlace(0), ['X'], 'Y', max_relative_error=self.atol + ) class TestXPULayerNormOpAxis2(TestXPULayerNormOp): - def set_attrs(self): self.begin_norm_axis = 2 class TestXPULayerNormOpAxis3(TestXPULayerNormOp): - def set_attrs(self): self.begin_norm_axis = 3 class TestXPULayerNormOp2D(TestXPULayerNormOp): - def set_attrs(self): self.shape = [10, 12] class TestXPULayerNormOp3D(TestXPULayerNormOp): - def set_attrs(self): self.shape = [4, 5, 6] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_log_loss_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_log_loss_op_xpu.py index 94297432e3069..4d50fc2da7d52 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_log_loss_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_log_loss_op_xpu.py @@ -26,7 +26,6 @@ def sigmoid_array(x): class TestXPULogLossOp(OpTest): - def setUp(self): self.op_type = 'log_loss' samples_num = 100 @@ -41,8 +40,9 @@ def setUp(self): } self.attrs = {'epsilon': epsilon} - loss = -labels * np.log(predicted + epsilon) - ( - 1 - labels) * np.log(1 - predicted + epsilon) + loss = -labels * np.log(predicted + epsilon) - (1 - labels) * np.log( + 1 - predicted + epsilon + ) self.outputs = {'Loss': loss} def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_log_softmax_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_log_softmax_op_xpu.py index c13161b17cb43..d961e46bd62bf 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_log_softmax_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_log_softmax_op_xpu.py @@ -22,14 +22,18 @@ import paddle.nn.functional as F from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() np.random.seed(10) def ref_log_softmax(x): - shiftx = (x - np.max(x)) + shiftx = x - np.max(x) out = shiftx - np.log(np.exp(shiftx).sum()) return out @@ -39,14 +43,14 @@ def ref_log_softmax_grad(x, axis): axis += len(x.shape) out = np.apply_along_axis(ref_log_softmax, axis, x) axis_dim = x.shape[axis] - dout = np.full_like(x, fill_value=1. / x.size) + dout = np.full_like(x, fill_value=1.0 / x.size) dx = dout - np.exp(out) * dout.copy().sum(axis=axis, keepdims=True).repeat( - axis_dim, axis=axis) + axis_dim, axis=axis + ) return dx class XPUTestLogSoftmaxOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'log_softmax' self.use_dynamic_create_class = True @@ -58,14 +62,12 @@ def dynamic_create_class(self): shape_arr = [[2, 3, 4, 5], [12, 10], [2, 5], [7, 7], [3, 5, 7]] for axis in axis_arr: for shape in shape_arr: - class_name = 'XPUTestLogSoftmax_' + \ - str(axis) + "_" + str(shape) + class_name = 'XPUTestLogSoftmax_' + str(axis) + "_" + str(shape) attr_dict = {'axis': axis, 'shape': shape} classes.append([class_name, attr_dict]) return base_class, classes class TestXPULogSoftmaxOp(XPUOpTest): - def setUp(self): self.op_type = 'log_softmax' self.python_api = F.log_softmax @@ -76,7 +78,7 @@ def setUp(self): self.shape = [2, 3, 4, 5] self.axis = -1 - x = np.random.uniform(0.1, 1., self.shape).astype(self.dtype) + x = np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype) out = np.apply_along_axis(ref_log_softmax, self.axis, x) self.x_grad = ref_log_softmax_grad(x, self.axis) @@ -91,9 +93,12 @@ def test_check_output(self): self.check_output(check_eager=True) def test_check_grad(self): - self.check_grad(['X'], ['Out'], - user_defined_grads=[self.x_grad], - check_eager=True) + self.check_grad( + ['X'], + ['Out'], + user_defined_grads=[self.x_grad], + check_eager=True, + ) support_types = get_xpu_op_support_types('log_softmax') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_logical_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_logical_op_xpu.py index ad0868d29bdf3..9b0d68ca08449 100755 --- a/python/paddle/fluid/tests/unittests/xpu/test_logical_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_logical_op_xpu.py @@ -21,19 +21,21 @@ import paddle from op_test import OpTest from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() ################## TEST OP: logical_and ################## class XPUTestLogicalAnd(XPUOpTestWrapper): - def __init__(self): self.op_name = 'logical_and' class XPUTestLogicalAndBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -42,20 +44,18 @@ def setUp(self): def set_case(self): self.op_type = 'logical_and' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.logical_and(x, y) self.attrs = {'use_xpu': True} self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.outputs = {'Out': out} @@ -73,7 +73,6 @@ def test_check_grad(self): pass class XPUTestLogicalAndCase1(XPUTestLogicalAndBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [4, 5] @@ -89,12 +88,10 @@ def init_case(self): ################## TEST OP: logical_or ################## class XPUTestLogicalOr(XPUOpTestWrapper): - def __init__(self): self.op_name = 'logical_or' class XPUTestLogicalOrBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -103,20 +100,18 @@ def setUp(self): def set_case(self): self.op_type = 'logical_or' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.logical_or(x, y) self.attrs = {'use_xpu': True} self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.outputs = {'Out': out} @@ -134,7 +129,6 @@ def test_check_grad(self): pass class XPUTestLogicalOrCase1(XPUTestLogicalOrBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [4, 5] @@ -150,12 +144,10 @@ def init_case(self): ################## TEST OP: logical_xor ################## class XPUTestLogicalXor(XPUOpTestWrapper): - def __init__(self): self.op_name = 'logical_xor' class XPUTestLogicalXorBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -164,20 +156,18 @@ def setUp(self): def set_case(self): self.op_type = 'logical_xor' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) - y = np.random.randint(self.low, - self.high, - self.y_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) + y = np.random.randint( + self.low, self.high, self.y_shape, dtype=self.dtype + ) out = np.logical_xor(x, y) self.attrs = {'use_xpu': True} self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } self.outputs = {'Out': out} @@ -195,7 +185,6 @@ def test_check_grad(self): pass class XPUTestLogicalXorCase1(XPUTestLogicalXorBase): - def init_case(self): self.dtype = np.int32 self.x_shape = [4, 5] @@ -211,12 +200,10 @@ def init_case(self): ################## TEST OP: LogicalNot ################## class XPUTestLogicalNot(XPUOpTestWrapper): - def __init__(self): self.op_name = 'logical_not' class XPUTestLogicalNotBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -225,10 +212,9 @@ def setUp(self): def set_case(self): self.op_type = 'logical_not' - x = np.random.randint(self.low, - self.high, - self.x_shape, - dtype=self.dtype) + x = np.random.randint( + self.low, self.high, self.x_shape, dtype=self.dtype + ) out = np.logical_not(x) self.attrs = {'use_xpu': True} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_logsumexp_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_logsumexp_op_xpu.py index db55aff3817d7..932507066b9a8 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_logsumexp_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_logsumexp_op_xpu.py @@ -25,7 +25,7 @@ def ref_logsumexp(x, axis=None, keepdim=False, reduce_all=False): if isinstance(axis, int): - axis = (axis, ) + axis = (axis,) elif isinstance(axis, list): axis = tuple(axis) if reduce_all: @@ -35,7 +35,6 @@ def ref_logsumexp(x, axis=None, keepdim=False, reduce_all=False): class XPUTestLogsumexp(XPUOpTest): - def setUp(self): self.op_type = 'logsumexp' self.shape = [2, 3, 4, 5] @@ -54,7 +53,7 @@ def setUp(self): self.attrs = { 'axis': self.axis, 'keepdim': self.keepdim, - 'reduce_all': self.reduce_all + 'reduce_all': self.reduce_all, } def set_attrs(self): @@ -70,31 +69,26 @@ def test_check_grad(self): class TestLogsumexp_shape(XPUTestLogsumexp): - def set_attrs(self): self.shape = [4, 5, 6] class TestLogsumexp_axis(XPUTestLogsumexp): - def set_attrs(self): self.axis = [0, -1] class TestLogsumexp_axis_all(XPUTestLogsumexp): - def set_attrs(self): self.axis = [0, 1, 2, 3] class TestLogsumexp_keepdim(XPUTestLogsumexp): - def set_attrs(self): self.keepdim = True class TestLogsumexp_reduce_all(XPUTestLogsumexp): - def set_attrs(self): self.reduce_all = True diff --git a/python/paddle/fluid/tests/unittests/xpu/test_lookup_table_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_lookup_table_v2_op_xpu.py index 35df3c244bd0d..039effe0275a7 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_lookup_table_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_lookup_table_v2_op_xpu.py @@ -29,7 +29,6 @@ class TestLookupTableOp(OpTest): - def setUp(self): self.op_type = "lookup_table_v2" table = np.random.random((17, 31)).astype("float64") @@ -42,15 +41,16 @@ def test_check_output_with_place(self): def test_check_grad(self): - self.check_grad_with_place(inputs_to_check=['W'], - output_names='Out', - no_grad_set=set('Ids'), - place=paddle.XPUPlace(0), - in_place=True) + self.check_grad_with_place( + inputs_to_check=['W'], + output_names='Out', + no_grad_set=set('Ids'), + place=paddle.XPUPlace(0), + in_place=True, + ) class TestLookupTableOpWithTensorIds(OpTest): - def setUp(self): self.op_type = "lookup_table_v2" table = np.random.random((17, 31)).astype("float64") @@ -62,19 +62,21 @@ def test_check_output(self): self.check_output_with_place(place=paddle.XPUPlace(0)) def test_check_grad(self): - self.check_grad_with_place(inputs_to_check=['W'], - output_names='Out', - no_grad_set=set('Ids'), - place=paddle.XPUPlace(0), - in_place=True) + self.check_grad_with_place( + inputs_to_check=['W'], + output_names='Out', + no_grad_set=set('Ids'), + place=paddle.XPUPlace(0), + in_place=True, + ) @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") + "test the gradient here." +) class TestLookupTableOpWithPadding(TestLookupTableOp): - def test_check_output(self): ids = np.squeeze(self.inputs['Ids']) padding_idx = np.random.choice(ids, 1)[0] @@ -86,9 +88,9 @@ def test_check_output(self): @skip_check_grad_ci( reason="Since paddings are not trainable and fixed in forward," "the gradient of paddings makes no sense and we don't " - "test the gradient here.") + "test the gradient here." +) class TestLookupTableOpWithTensorIdsAndPadding(TestLookupTableOpWithTensorIds): - def test_check_output(self): ids = self.inputs['Ids'] flatten_idx = ids.flatten() @@ -99,7 +101,6 @@ def test_check_output(self): class TestLookupTableWIsSelectedRows(unittest.TestCase): - def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() ids_array = np.array([0, 4, 3, 5]).astype("int64") @@ -147,13 +148,14 @@ def test_w_is_selected_rows(self): self.check_with_place(place) -class TestLookupTableWithTensorIdsWIsSelectedRows(TestLookupTableWIsSelectedRows - ): - +class TestLookupTableWithTensorIdsWIsSelectedRows( + TestLookupTableWIsSelectedRows +): def prepare_ids(self, scope, place): ids_tensor = scope.var('Ids').get_tensor() - ids_array = np.random.randint(low=0, high=6, - size=(2, 4, 3)).astype("int64") + ids_array = np.random.randint(low=0, high=6, size=(2, 4, 3)).astype( + "int64" + ) ids_tensor.set(ids_array, place) return ids_array @@ -163,7 +165,6 @@ def check_result(self, ids_array, result_array): class TestLookupTableApi(unittest.TestCase): - def test_api(self): x = fluid.layers.data(name='x', shape=[20], dtype='int64') emb = fluid.embedding(input=x, size=[128, 64]) @@ -173,15 +174,16 @@ def test_api(self): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'x': x_data, - }, - fetch_list=[emb], - return_numpy=False) + ret = exe.run( + feed={ + 'x': x_data, + }, + fetch_list=[emb], + return_numpy=False, + ) class TestEmbedOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): input_data = np.random.randint(0, 10, (4, 6)).astype("int64") diff --git a/python/paddle/fluid/tests/unittests/xpu/test_masked_select_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_masked_select_op_xpu.py index d4ccbc18c3411..6a2976ccbb528 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_masked_select_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_masked_select_op_xpu.py @@ -20,7 +20,11 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -34,12 +38,10 @@ def np_masked_select(x, mask): class XPUTestMaskedSelectOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'masked_select' class TestMaskedSelectOp(XPUOpTest): - def setUp(self): self.init() self.dtype = self.in_type @@ -60,14 +62,12 @@ def init(self): self.shape = (50, 3) class TestMaskedSelectOp1(TestMaskedSelectOp): - def init(self): self.shape = (6, 8, 9, 18) class TestMaskedSelectOp2(TestMaskedSelectOp): - def init(self): - self.shape = (168, ) + self.shape = (168,) support_types = get_xpu_op_support_types('masked_select') @@ -76,7 +76,6 @@ def init(self): class TestMaskedSelectAPI(unittest.TestCase): - def test_imperative_mode(self): paddle.disable_static(paddle.XPUPlace(0)) shape = (88, 6, 8) @@ -101,27 +100,26 @@ def test_static_mode(self): exe = paddle.static.Executor(place=paddle.XPUPlace(0)) - res = exe.run(paddle.static.default_main_program(), - feed={ - "x": np_x, - "mask": np_mask - }, - fetch_list=[out]) + res = exe.run( + paddle.static.default_main_program(), + feed={"x": np_x, "mask": np_mask}, + fetch_list=[out], + ) self.assertEqual(np.allclose(res, np_out), True) class TestMaskedSelectError(unittest.TestCase): - def test_error(self): - with paddle.static.program_guard(paddle.static.Program(), - paddle.static.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): shape = [8, 9, 6] x = paddle.fluid.data(shape=shape, dtype='float32', name='x') mask = paddle.fluid.data(shape=shape, dtype='bool', name='mask') - mask_float = paddle.fluid.data(shape=shape, - dtype='float32', - name='mask_float') + mask_float = paddle.fluid.data( + shape=shape, dtype='float32', name='mask_float' + ) np_x = np.random.random(shape).astype('float32') np_mask = np.array(np.random.randint(2, size=shape, dtype=bool)) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_matmul_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_matmul_op_xpu.py index 0afcd1c57343f..c4aab23a95201 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_matmul_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_matmul_op_xpu.py @@ -22,7 +22,11 @@ import paddle.fluid as fluid from paddle.fluid import Program, program_guard -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): @@ -64,8 +68,9 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): return Out -def generate_compatible_shapes(dim_X, dim_Y, transpose_X, transpose_Y, - batch_size): +def generate_compatible_shapes( + dim_X, dim_Y, transpose_X, transpose_Y, batch_size +): BATCH_SIZE = 2 if batch_size != None: BATCH_SIZE = batch_size @@ -131,6 +136,7 @@ def generate_compatible_shapes_2(dim, transpose_X, transpose_Y): def generate_negative_dims(in_shape): from itertools import combinations + size = len(in_shape) indexs = list() shapes = list() @@ -138,7 +144,8 @@ def generate_negative_dims(in_shape): indexs.extend(list(combinations([j for j in range(size)], i + 1))) for idx in indexs: shapes.append( - [in_shape[i] if i not in idx else -1 for i in range(size)]) + [in_shape[i] if i not in idx else -1 for i in range(size)] + ) return shapes @@ -151,49 +158,48 @@ def test_negative_dims_program(obj): with program_guard(Program(), Program()): x = fluid.data(name='x', shape=shape_x, dtype=obj.in_type_str) y = fluid.data(name='y', shape=shape_y, dtype=obj.in_type_str) - output = fluid.layers.matmul(x, y, obj.transpose_X, - obj.transpose_Y) + output = fluid.layers.matmul( + x, y, obj.transpose_X, obj.transpose_Y + ) obj.assertEqual(len(Ref.shape), len(output.shape)) for idx in range(len(Ref.shape)): if output.shape[idx] != -1: obj.assertEqual(Ref.shape[idx], output.shape[idx]) exe = fluid.Executor(fluid.XPUPlace(0)) - res, = exe.run(fluid.default_main_program(), - feed={ - 'x': X, - 'y': Y - }, - fetch_list=[output]) + (res,) = exe.run( + fluid.default_main_program(), + feed={'x': X, 'y': Y}, + fetch_list=[output], + ) np.allclose(res, Ref, atol=1e-3) class XPUTestMatmulOpErr(XPUOpTestWrapper): - def __init__(self): self.op_name = "matmul" self.use_dynamic_create_class = False class TestMatmulOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The inputs type of matmul_op must be Variable. input1 = 12 - self.assertRaises(TypeError, fluid.layers.matmul, input1, - input1) + self.assertRaises( + TypeError, fluid.layers.matmul, input1, input1 + ) # The inputs dtype of matmul_op must be float32, float16 - input2 = fluid.layers.data(name='input2', - shape=[10, 10], - dtype="int32") - self.assertRaises(TypeError, fluid.layers.matmul, input2, - input2) - input3 = fluid.layers.data(name='input3', - shape=[2, 2], - dtype="float16") + input2 = fluid.layers.data( + name='input2', shape=[10, 10], dtype="int32" + ) + self.assertRaises( + TypeError, fluid.layers.matmul, input2, input2 + ) + input3 = fluid.layers.data( + name='input3', shape=[2, 2], dtype="float16" + ) fluid.layers.matmul(input3, input3) class API_TestMm(unittest.TestCase): - def test_out(self): with fluid.program_guard(fluid.Program()): x = fluid.data(name="x", shape=[2], dtype=self.in_type) @@ -203,13 +209,12 @@ def test_out(self): exe = fluid.Executor(fluid.XPUPlace(0)) data1 = np.random.rand(2).astype(self.in_type) data2 = np.random.rand(2).astype(self.in_type) - np_res = exe.run(feed={ - 'x': data1, - 'y': data2 - }, - fetch_list=[result]) - expected_result = np.matmul(data1.reshape(1, 2), - data2.reshape(2, 1)) + np_res = exe.run( + feed={'x': data1, 'y': data2}, fetch_list=[result] + ) + expected_result = np.matmul( + data1.reshape(1, 2), data2.reshape(2, 1) + ) np.testing.assert_allclose(np_res, expected_result, atol=1e-3) @@ -222,89 +227,94 @@ def test_dygraph_without_out(self): data2 = fluid.dygraph.to_variable(input_array2) out = paddle.mm(data1, data2) expected_result = np.matmul(input_array1, input_array2) - np.testing.assert_allclose(expected_result, - out.numpy(), - atol=1e-3) + np.testing.assert_allclose( + expected_result, out.numpy(), atol=1e-3 + ) class Test_API_Matmul(unittest.TestCase): - def test_dygraph_without_out(self): device = fluid.XPUPlace(0) with fluid.dygraph.guard(device): input_array1 = np.random.rand(3, 4).astype(self.in_type) input_array2 = np.random.rand(4, 3).astype(self.in_type) data1 = fluid.dygraph.to_variable(input_array1).astype( - self.in_type) + self.in_type + ) data2 = fluid.dygraph.to_variable(input_array2).astype( - self.in_type) + self.in_type + ) out = paddle.matmul(data1, data2) expected_result = np.matmul(input_array1, input_array2) - np.testing.assert_allclose(expected_result, - out.numpy(), - atol=1e-3) + np.testing.assert_allclose( + expected_result, out.numpy(), atol=1e-3 + ) class API_TestMmError(unittest.TestCase): - def test_errors(self): - def test_error1(): with fluid.program_guard(fluid.Program(), fluid.Program()): - data1 = fluid.data(name="data1", - shape=[10, 2], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[3, 10], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[10, 2], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[3, 10], dtype="float32" + ) paddle.mm(data1, data2) self.assertRaises(ValueError, test_error1) def test_error2(): with fluid.program_guard(fluid.Program(), fluid.Program()): - data1 = fluid.data(name="data1", - shape=[-1, 10, 2], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[-1, 2, 10], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[-1, 10, 2], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[-1, 2, 10], dtype="float32" + ) paddle.mm(data1, data2) test_error2() def test_error3(): with fluid.program_guard(fluid.Program(), fluid.Program()): - data1 = fluid.data(name="data1", - shape=[10, 10, 2], - dtype="float32") - data2 = fluid.data(name="data2", - shape=[3, 2, 10], - dtype="float32") + data1 = fluid.data( + name="data1", shape=[10, 10, 2], dtype="float32" + ) + data2 = fluid.data( + name="data2", shape=[3, 2, 10], dtype="float32" + ) paddle.mm(data1, data2) self.assertRaises(ValueError, test_error3) class TestMatmulBaseGenerator(XPUOpTest): - def setUp(self): self.op_type = "matmul" - self.dtype = np.float32 if not hasattr(self, - 'in_type') else self.in_type + self.dtype = ( + np.float32 if not hasattr(self, 'in_type') else self.in_type + ) - self.__class__.no_need_check_grad = False if not hasattr( - self, 'no_need_check_grad') else self.no_need_check_grad + self.__class__.no_need_check_grad = ( + False + if not hasattr(self, 'no_need_check_grad') + else self.no_need_check_grad + ) shape_X = [4, 5] if not hasattr(self, 'shape_X') else self.shape_X shape_Y = [5, 6] if not hasattr(self, 'shape_Y') else self.shape_Y - transpose_X = False if not hasattr(self, - 'transpose_X') else self.transpose_X - transpose_Y = False if not hasattr(self, - 'transpose_Y') else self.transpose_Y + transpose_X = ( + False if not hasattr(self, 'transpose_X') else self.transpose_X + ) + transpose_Y = ( + False if not hasattr(self, 'transpose_Y') else self.transpose_Y + ) X = np.random.random(shape_X).astype(self.dtype) Y = np.random.random(shape_Y).astype(self.dtype) - Out = reference_matmul(X, Y, transpose_X, - transpose_Y).astype(self.dtype) + Out = reference_matmul(X, Y, transpose_X, transpose_Y).astype( + self.dtype + ) self.inputs = {'X': X, 'Y': Y} self.attrs = {'transpose_X': transpose_X, 'transpose_Y': transpose_Y} self.outputs = {'Out': Out} @@ -314,40 +324,43 @@ def test_check_output(self): self.check_output_with_place(place, atol=1e-3) def test_check_grad_normal(self): - if hasattr(self.__class__, "no_need_check_grad" - ) and self.__class__.no_need_check_grad == True: + if ( + hasattr(self.__class__, "no_need_check_grad") + and self.__class__.no_need_check_grad == True + ): return place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X', 'Y'], - 'Out', - max_relative_error=5e-2) + self.check_grad_with_place( + place, ['X', 'Y'], 'Out', max_relative_error=5e-2 + ) def test_check_grad_ignore_x(self): - if hasattr(self.__class__, "no_need_check_grad" - ) and self.__class__.no_need_check_grad == True: + if ( + hasattr(self.__class__, "no_need_check_grad") + and self.__class__.no_need_check_grad == True + ): return place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=5e-2, - no_grad_set=set("X")) + self.check_grad_with_place( + place, ['Y'], 'Out', max_relative_error=5e-2, no_grad_set=set("X") + ) def test_check_grad_ignore_y(self): - if hasattr(self.__class__, "no_need_check_grad" - ) and self.__class__.no_need_check_grad == True: + if ( + hasattr(self.__class__, "no_need_check_grad") + and self.__class__.no_need_check_grad == True + ): return place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=5e-2, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, ['X'], 'Out', max_relative_error=5e-2, no_grad_set=set('Y') + ) class XPUTestMatmulOp1(XPUOpTestWrapper): - def __init__(self): self.op_name = "matmul" self.use_dynamic_create_class = True @@ -366,19 +379,19 @@ def dynamic_create_class(self): no_need_check_grad = False if batch >= 5: no_need_check_grad = True - class_name = ( - 'TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}_batch_{}' - .format(dim_X, dim_Y, transose_x, transose_y, - batch)) + class_name = 'TestMatMulOp_dimX_{}_dim_Y_{}_transX_{}_transY_{}_batch_{}'.format( + dim_X, dim_Y, transose_x, transose_y, batch + ) shape_x, shape_y = generate_compatible_shapes( - dim_X, dim_Y, transose_x, transose_y, batch) + dim_X, dim_Y, transose_x, transose_y, batch + ) attr_dict = { 'shape_X': shape_x, 'shape_Y': shape_y, 'transpose_X': transose_x, 'transpose_Y': transose_y, 'no_need_check_grad': no_need_check_grad, - 'op_type': "matmul" + 'op_type': "matmul", } classes.append([class_name, attr_dict]) @@ -386,7 +399,6 @@ def dynamic_create_class(self): class XPUTestMatmulOp2(XPUOpTestWrapper): - def __init__(self): self.op_name = "matmul" self.use_dynamic_create_class = True @@ -402,12 +414,12 @@ def dynamic_create_class(self): for transose_x in [True, False]: for transose_y in [True, False]: for batch in batch_size: - class_name = ( - 'TestMatMulAPI_dimX_{}_dim_Y_{}_transX_{}_transY_{}_batch_{}' - .format(dim_X, dim_Y, transose_x, transose_y, - batch)) + class_name = 'TestMatMulAPI_dimX_{}_dim_Y_{}_transX_{}_transY_{}_batch_{}'.format( + dim_X, dim_Y, transose_x, transose_y, batch + ) shape_x, shape_y = generate_compatible_shapes( - dim_X, dim_Y, transose_x, transose_y, batch) + dim_X, dim_Y, transose_x, transose_y, batch + ) attr_dict = { 'shape_X': shape_x, 'shape_Y': shape_y, @@ -420,7 +432,6 @@ def dynamic_create_class(self): class XPUTestMatmulOp3(XPUOpTestWrapper): - def __init__(self): self.op_name = "matmul" self.use_dynamic_create_class = True @@ -431,17 +442,18 @@ def dynamic_create_class(self): for dim in [4]: for transpose_X in [False, True]: for transpose_Y in [False, True]: - class_name = ( - 'TestMatMulOp2_dimX_{}_dim_Y_{}_transX_{}_transY_{}'. - format(dim, dim, transpose_X, transpose_Y)) + class_name = 'TestMatMulOp2_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( + dim, dim, transpose_X, transpose_Y + ) shape_X, shape_Y = generate_compatible_shapes_2( - dim, transpose_X, transpose_Y) + dim, transpose_X, transpose_Y + ) attr_dict = { 'shape_X': shape_X, 'shape_Y': shape_Y, 'transpose_X': transpose_X, 'transpose_Y': transpose_Y, - 'op_type': "matmul" + 'op_type': "matmul", } classes.append([class_name, attr_dict]) return base_class, classes diff --git a/python/paddle/fluid/tests/unittests/xpu/test_matmul_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_matmul_v2_op_xpu.py index 8473f9c506dfc..63354ac7607ee 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_matmul_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_matmul_v2_op_xpu.py @@ -21,7 +21,11 @@ import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): @@ -30,7 +34,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): # transpose X and Y appropriately. if transpose_X: if X.ndim == 1: - X = X.reshape((X.size, )) + X = X.reshape((X.size,)) elif X.ndim == 2: X = X.T else: @@ -39,7 +43,7 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): X = np.transpose(X, tuple(dim)) if transpose_Y: if Y.ndim == 1: - Y = Y.reshape((Y.size, )) + Y = Y.reshape((Y.size,)) else: dim = [i for i in range(len(Y.shape))] dim[-1], dim[len(Y.shape) - 2] = dim[len(Y.shape) - 2], dim[-1] @@ -55,7 +59,6 @@ def reference_matmul(X, Y, transpose_X=False, transpose_Y=False): class XPUTestMatmulV2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = "matmul_v2" self.use_dynamic_create_class = False @@ -66,8 +69,8 @@ class TestMatMulV2Op(XPUOpTest): """ def config(self): - self.x_shape = (100, ) - self.y_shape = (100, ) + self.x_shape = (100,) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -96,8 +99,10 @@ def test_check_output(self): self.check_output_with_place(place) def test_check_grad(self): - if hasattr(self.__class__, "no_need_check_grad" - ) and self.__class__.no_need_check_grad == True: + if ( + hasattr(self.__class__, "no_need_check_grad") + and self.__class__.no_need_check_grad == True + ): return place = paddle.XPUPlace(0) self.check_grad_with_place(place, ['X', 'Y'], 'Out') @@ -108,7 +113,7 @@ class TestMatMulOp2(TestMatMulV2Op): """ def config(self): - self.x_shape = (100) + self.x_shape = 100 self.y_shape = (100, 3) self.trans_x = False self.trans_y = False @@ -119,7 +124,7 @@ class TestMatMulOp3(TestMatMulV2Op): """ def config(self): - self.x_shape = (100, ) + self.x_shape = (100,) self.y_shape = (1, 1, 100, 2) self.trans_x = False self.trans_y = False @@ -142,7 +147,7 @@ class TestMatMulOp5(TestMatMulV2Op): def config(self): self.x_shape = (1, 1, 100, 1) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = True self.trans_y = False @@ -219,7 +224,7 @@ class TestMatMulOp12(TestMatMulV2Op): def config(self): self.x_shape = (1, 20, 100) - self.y_shape = (100, ) + self.y_shape = (100,) self.trans_x = False self.trans_y = False @@ -274,7 +279,7 @@ class TestMatMulOp17(TestMatMulV2Op): def config(self): self.x_shape = (2, 1, 100) - self.y_shape = (100) + self.y_shape = 100 self.trans_x = False self.trans_y = False @@ -285,7 +290,7 @@ class TestMatMulOp18(TestMatMulV2Op): def config(self): self.x_shape = (8, 111, 4, 17) - self.y_shape = (17) + self.y_shape = 17 self.trans_x = False self.trans_y = False diff --git a/python/paddle/fluid/tests/unittests/xpu/test_mean_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_mean_op_xpu.py index 2ec886796f0b6..c86aaeea6cfb3 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_mean_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_mean_op_xpu.py @@ -25,19 +25,21 @@ np.random.seed(10) from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestMeanOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'mean' self.use_dynamic_create_class = False class TestMeanOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -65,41 +67,36 @@ def test_checkout_grad(self): self.check_grad_with_place(self.place, ['X'], 'Out') class TestMeanOp1(TestMeanOp): - def set_shape(self): - self.shape = (5) + self.shape = 5 class TestMeanOp2(TestMeanOp): - def set_shape(self): self.shape = (5, 7, 8) class TestMeanOp3(TestMeanOp): - def set_shape(self): self.shape = (10, 5, 7, 8) class TestMeanOp4(TestMeanOp): - def set_shape(self): self.shape = (2, 2, 3, 3, 3) class TestMeanOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of mean_op must be Variable. input1 = 12 self.assertRaises(TypeError, paddle.mean, input1) # The input dtype of mean_op must be float16, float32, float64. - input2 = fluid.layers.data(name='input2', - shape=[12, 10], - dtype="int32") + input2 = fluid.layers.data( + name='input2', shape=[12, 10], dtype="int32" + ) self.assertRaises(TypeError, paddle.mean, input2) - input3 = fluid.layers.data(name='input3', - shape=[4], - dtype="float16") + input3 = fluid.layers.data( + name='input3', shape=[4], dtype="float16" + ) fluid.layers.softmax(input3) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu.py index 554508b877f0c..591cb32541723 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu.py @@ -20,19 +20,21 @@ import paddle from test_merged_momentum_op_xpu_base import TestMergedMomentumBase -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestMergedMomentumOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'merged_momentum' self.use_dynamic_create_class = False class TestMergedMomentumOp(TestMergedMomentumBase): - def setUp(self): super().setUp() self.set_case() @@ -46,22 +48,18 @@ def testalltype(self): self.check_with_place(self.place, self.in_type) class TestMergedMomentum1(TestMergedMomentumOp): - def set_case(self): self.shapes = [[3, 4], [2, 7], [5, 6, 8]] class TestMergedMomentum2(TestMergedMomentumOp): - def set_case(self): self.shapes = [[3, 4], [2, 7]] class TestMergedMomentum3(TestMergedMomentumOp): - def set_case(self): self.shapes = [[3, 4]] class TestMergedMomentum4(TestMergedMomentumOp): - def set_case(self): self.shapes = [[3, 4], [2, 7], [5, 6, 7], [9, 9], [10, 12]] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu_base.py b/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu_base.py index 64d1e5aa9b26c..05044f11ee41b 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu_base.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_merged_momentum_op_xpu_base.py @@ -22,17 +22,19 @@ from collections import OrderedDict -def run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - mu=0.9, - rescale_grad=0.01, - use_merged=False, - use_nesterov=True): +def run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + mu=0.9, + rescale_grad=0.01, + use_merged=False, + use_nesterov=True, +): assert len(params) == len(grads) assert len(params) == len(velocitys) if multi_precision: @@ -44,48 +46,70 @@ def run_momentum_op(params, helper = LayerHelper(op_type, **locals()) param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in params ] grad_vars = [ helper.create_variable(shape=g.shape, dtype=g.dtype) for g in grads ] velocity_vars = [ - helper.create_variable(persistable=True, - shape=v.shape, - dtype=v.dtype) for v in velocitys + helper.create_variable( + persistable=True, shape=v.shape, dtype=v.dtype + ) + for v in velocitys ] - lr_var = helper.create_variable(persistable=True, - shape=learning_rate.shape, - dtype=learning_rate.dtype) + lr_var = helper.create_variable( + persistable=True, + shape=learning_rate.shape, + dtype=learning_rate.dtype, + ) feed_dict = OrderedDict() feed_dict.update( - OrderedDict([(p_var.name, p_val) - for p_var, p_val in zip(param_vars, params)])) + OrderedDict( + [ + (p_var.name, p_val) + for p_var, p_val in zip(param_vars, params) + ] + ) + ) feed_dict.update( - OrderedDict([(v_var.name, v_val) - for v_var, v_val in zip(velocity_vars, velocitys)])) + OrderedDict( + [ + (v_var.name, v_val) + for v_var, v_val in zip(velocity_vars, velocitys) + ] + ) + ) fetch_list = list(feed_dict.keys()) feed_dict.update( - OrderedDict([(g_var.name, g_val) - for g_var, g_val in zip(grad_vars, grads)])) + OrderedDict( + [(g_var.name, g_val) for g_var, g_val in zip(grad_vars, grads)] + ) + ) feed_dict.update({lr_var.name: learning_rate}) if multi_precision: master_param_vars = [ - helper.create_variable(persistable=True, - shape=p.shape, - dtype=p.dtype) for p in master_params + helper.create_variable( + persistable=True, shape=p.shape, dtype=p.dtype + ) + for p in master_params ] feed_dict.update( - OrderedDict([ - (mp_var.name, mp_val) - for mp_var, mp_val in zip(master_param_vars, master_params) - ])) + OrderedDict( + [ + (mp_var.name, mp_val) + for mp_var, mp_val in zip( + master_param_vars, master_params + ) + ] + ) + ) # CPUPlace does not use MasterParam if isinstance(place, paddle.CUDAPlace): fetch_list = fetch_list + [ @@ -95,8 +119,9 @@ def run_momentum_op(params, master_param_vars = None if not use_merged: - for i, (p, g, - v) in enumerate(zip(param_vars, grad_vars, velocity_vars)): + for i, (p, g, v) in enumerate( + zip(param_vars, grad_vars, velocity_vars) + ): inputs = { 'Param': p, 'Grad': g, @@ -115,10 +140,9 @@ def run_momentum_op(params, 'regularization_method': 'l2_decay', 'regularization_coeff': 2.0, } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) else: inputs = { 'Param': param_vars, @@ -131,22 +155,18 @@ def run_momentum_op(params, inputs['MasterParam'] = master_param_vars outputs['MasterParamOut'] = master_param_vars attrs = { - 'mu': - mu, - 'multi_precision': - multi_precision, - 'rescale_grad': - rescale_grad, - 'use_nesterov': - use_nesterov, - 'regularization_method': - ['l2_decay' for i in range(len(param_vars))], + 'mu': mu, + 'multi_precision': multi_precision, + 'rescale_grad': rescale_grad, + 'use_nesterov': use_nesterov, + 'regularization_method': [ + 'l2_decay' for i in range(len(param_vars)) + ], 'regularization_coeff': [2.0 for i in range(len(param_vars))], } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): @@ -155,7 +175,6 @@ def run_momentum_op(params, class TestMergedMomentumBase(unittest.TestCase): - def setUp(self): paddle.enable_static() self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] @@ -179,22 +198,31 @@ def prepare_data(self, shapes, multi_precision, seed, dtype, place): return params, grads, velocitys, master_params, learning_rate def check_with_place(self, place, dtype, multi_precision=False): - params, grads, velocitys, master_params, learning_rate = self.prepare_data( - self.shapes, multi_precision, self.seed, dtype, place) + ( + params, + grads, + velocitys, + master_params, + learning_rate, + ) = self.prepare_data( + self.shapes, multi_precision, self.seed, dtype, place + ) def run_op(use_nesterov, use_merged): # NPU Momentum Op does not support rescale_grad rescale_grad = 1.0 - return run_momentum_op(params, - grads, - velocitys, - master_params, - learning_rate, - place, - multi_precision, - rescale_grad=rescale_grad, - use_merged=use_merged, - use_nesterov=use_nesterov) + return run_momentum_op( + params, + grads, + velocitys, + master_params, + learning_rate, + place, + multi_precision, + rescale_grad=rescale_grad, + use_merged=use_merged, + use_nesterov=use_nesterov, + ) outs1 = run_op(use_nesterov=True, use_merged=True) outs2 = run_op(use_nesterov=True, use_merged=False) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_momentum_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_momentum_op_xpu.py index 5cc9cb49e27cb..0ee2af0e2023c 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_momentum_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_momentum_op_xpu.py @@ -22,14 +22,25 @@ import paddle.fluid.core as core from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() -def calculate_momentum_by_numpy(param, grad, mu, velocity, use_nesterov, - learning_rate, regularization_method, - regularization_coeff): +def calculate_momentum_by_numpy( + param, + grad, + mu, + velocity, + use_nesterov, + learning_rate, + regularization_method, + regularization_coeff, +): if regularization_method == "l2_decay": grad = grad + regularization_coeff * param velocity_out = mu * velocity + grad @@ -40,21 +51,20 @@ def calculate_momentum_by_numpy(param, grad, mu, velocity, use_nesterov, else: velocity_out = mu * velocity + grad if use_nesterov: - param_out = param - grad * learning_rate - \ - velocity_out * mu * learning_rate + param_out = ( + param - grad * learning_rate - velocity_out * mu * learning_rate + ) else: param_out = param - learning_rate * velocity_out return param_out, velocity_out class XPUTestMomentumOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'momentum' self.use_dynamic_create_class = False class TestMomentumOPBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.xpu_version = core.get_xpu_device_version(0) @@ -65,12 +75,15 @@ def set_case(self): self.op_type = 'momentum' self.init_config() - self.param = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) - self.grad = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.param = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) + self.grad = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) self.velocity = np.random.uniform(-1, 1, self.input_shape).astype( - self.dtype) + self.dtype + ) param_out, velocity_out = calculate_momentum_by_numpy( param=self.param, grad=self.grad, @@ -79,7 +92,8 @@ def set_case(self): use_nesterov=self.use_nesterov, learning_rate=self.learning_rate, regularization_method=self.regularization_method, - regularization_coeff=self.regularization_coeff) + regularization_coeff=self.regularization_coeff, + ) param_out = param_out.astype(self.dtype) velocity_out = velocity_out.astype(self.dtype) self.inputs = { @@ -93,7 +107,7 @@ def set_case(self): 'mu': self.mu, 'use_nesterov': self.use_nesterov, 'regularization_method': self.regularization_method, - 'regularization_coeff': self.regularization_coeff + 'regularization_coeff': self.regularization_coeff, } self.outputs = {'ParamOut': param_out, 'VelocityOut': velocity_out} @@ -112,7 +126,6 @@ def init_config(self): self.regularization_coeff = 0 class XPUTestMomentum1(TestMomentumOPBase): - def init_config(self): self.input_shape = [2, 768] self.learning_rate = np.array([0.002]).astype(float) @@ -122,7 +135,6 @@ def init_config(self): self.regularization_coeff = 0 class XPUTestMomentum2(TestMomentumOPBase): - def init_config(self): self.input_shape = [3, 8, 4096] self.learning_rate = np.array([0.005]).astype(float) @@ -132,7 +144,6 @@ def init_config(self): self.regularization_coeff = 0 class XPUTestMomentum3(TestMomentumOPBase): - def init_config(self): self.input_shape = [1024] self.learning_rate = np.array([0.01]).astype(float) @@ -147,7 +158,6 @@ def init_config(self): self.regularization_coeff = 0 class XPUTestMomentum4(TestMomentumOPBase): - def init_config(self): self.input_shape = [2, 2, 255] self.learning_rate = np.array([0.0005]).astype(float) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_mul_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_mul_op_xpu.py index b8d4666761a74..08b3b3e89a3c6 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_mul_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_mul_op_xpu.py @@ -24,18 +24,23 @@ paddle.enable_static() -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class TestMulOpError(unittest.TestCase): - def test_errors(self): with program_guard(Program(), Program()): # The input type of mul_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.XPUPlace(0)) - x2 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - fluid.XPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.XPUPlace(0) + ) + x2 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], fluid.XPUPlace(0) + ) self.assertRaises(TypeError, fluid.layers.mul, x1, x2) # The input dtype of mul_op must be float32. x3 = fluid.layers.data(name='x3', shape=[4], dtype="int32") @@ -44,26 +49,26 @@ def test_errors(self): class XPUTestMulOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'mul' self.use_dynamic_create_class = False class TestXPUMulOp1(XPUOpTest): - def setUp(self): self.op_type = "mul" self.dtype = self.in_type self.inputs = { 'X': np.random.random((3, 4, 2, 9)).astype(self.in_type_str), - 'Y': np.random.random((3, 6, 1, 2, 3)).astype(self.in_type_str) + 'Y': np.random.random((3, 6, 1, 2, 3)).astype(self.in_type_str), } self.attrs = { 'x_num_col_dims': 2, 'y_num_col_dims': 2, } - result = np.dot(self.inputs['X'].reshape(3 * 4, 2 * 9), - self.inputs['Y'].reshape(3 * 6, 1 * 2 * 3)) + result = np.dot( + self.inputs['X'].reshape(3 * 4, 2 * 9), + self.inputs['Y'].reshape(3 * 6, 1 * 2 * 3), + ) result = result.reshape(3, 4, 1, 2, 3) self.outputs = {'Out': result} @@ -75,35 +80,40 @@ def test_check_output(self): def test_check_grad_normal(self): place = paddle.XPUPlace(0) paddle.enable_static() - self.check_grad_with_place(place, ['X', 'Y'], - 'Out', - max_relative_error=0.1) + self.check_grad_with_place( + place, ['X', 'Y'], 'Out', max_relative_error=0.1 + ) def test_check_grad_ingore_x(self): place = paddle.XPUPlace(0) paddle.enable_static() - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.1, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.1, + no_grad_set=set("X"), + ) def test_check_grad_ignore_y(self): place = paddle.XPUPlace(0) paddle.enable_static() - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.1, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.1, + no_grad_set=set('Y'), + ) class TestXPUMulOp2(XPUOpTest): - def setUp(self): self.op_type = "mul" self.use_xpu = True self.dtype = self.in_type self.inputs = { 'X': np.random.random((20, 5)).astype(self.in_type_str), - 'Y': np.random.random((5, 21)).astype(self.in_type_str) + 'Y': np.random.random((5, 21)).astype(self.in_type_str), } self.outputs = {'Out': np.dot(self.inputs['X'], self.inputs['Y'])} @@ -115,25 +125,31 @@ def test_check_output(self): def test_check_grad_normal(self): place = paddle.XPUPlace(0) paddle.enable_static() - self.check_grad_with_place(place, ['X', 'Y'], - 'Out', - max_relative_error=0.1) + self.check_grad_with_place( + place, ['X', 'Y'], 'Out', max_relative_error=0.1 + ) def test_check_grad_ingore_x(self): place = paddle.XPUPlace(0) paddle.enable_static() - self.check_grad_with_place(place, ['Y'], - 'Out', - max_relative_error=0.1, - no_grad_set=set("X")) + self.check_grad_with_place( + place, + ['Y'], + 'Out', + max_relative_error=0.1, + no_grad_set=set("X"), + ) def test_check_grad_ingore_y(self): place = paddle.XPUPlace(0) paddle.enable_static() - self.check_grad_with_place(place, ['X'], - 'Out', - max_relative_error=0.1, - no_grad_set=set('Y')) + self.check_grad_with_place( + place, + ['X'], + 'Out', + max_relative_error=0.1, + no_grad_set=set('Y'), + ) support_types = get_xpu_op_support_types('mul') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_nearest_interp_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_nearest_interp_v2_op_xpu.py index 3ded60a3d804c..0f2d4fd27ecc9 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_nearest_interp_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_nearest_interp_v2_op_xpu.py @@ -21,20 +21,26 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() -def nearest_neighbor_interp_np(X, - out_h, - out_w, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='NCHW'): +def nearest_neighbor_interp_np( + X, + out_h, + out_w, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='NCHW', +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 3, 1, 2)) # NHWC => NCHW @@ -47,16 +53,16 @@ def nearest_neighbor_interp_np(X, n, c, in_h, in_w = X.shape ratio_h = ratio_w = 0.0 - if (out_h > 1): - if (align_corners): + if out_h > 1: + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: ratio_h = 1.0 / scale_h else: ratio_h = 1.0 * in_h / out_h - if (out_w > 1): - if (align_corners): + if out_w > 1: + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -84,17 +90,19 @@ def nearest_neighbor_interp_np(X, return out.astype(X.dtype) -def nearest_neighbor_interp3d_np(X, - out_d, - out_h, - out_w, - scale_d=0, - scale_h=0, - scale_w=0, - out_size=None, - actual_shape=None, - align_corners=True, - data_layout='NCHW'): +def nearest_neighbor_interp3d_np( + X, + out_d, + out_h, + out_w, + scale_d=0, + scale_h=0, + scale_w=0, + out_size=None, + actual_shape=None, + align_corners=True, + data_layout='NCHW', +): """nearest neighbor interpolation implement in shape [N, C, H, W]""" if data_layout == "NHWC": X = np.transpose(X, (0, 4, 1, 2, 3)) # NDHWC => NCDHW @@ -109,24 +117,24 @@ def nearest_neighbor_interp3d_np(X, n, c, in_d, in_h, in_w = X.shape ratio_d = ratio_h = ratio_w = 0.0 - if (out_d > 1): - if (align_corners): + if out_d > 1: + if align_corners: ratio_d = (in_d - 1.0) / (out_d - 1.0) else: if scale_d > 0: ratio_d = 1.0 / scale_d else: ratio_d = 1.0 * in_d / out_d - if (out_h > 1): - if (align_corners): + if out_h > 1: + if align_corners: ratio_h = (in_h - 1.0) / (out_h - 1.0) else: if scale_h > 0: ratio_h = 1.0 / scale_h else: ratio_h = 1.0 * in_h / out_h - if (out_w > 1): - if (align_corners): + if out_w > 1: + if align_corners: ratio_w = (in_w - 1.0) / (out_w - 1.0) else: if scale_w > 0: @@ -158,13 +166,11 @@ def nearest_neighbor_interp3d_np(X, class XPUNearestInterpOpWrapper(XPUOpTestWrapper): - def __init__(self): self.op_name = 'nearest_interp_v2' self.use_dynamic_create_class = False class TestNearestInterpOp(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -174,7 +180,7 @@ def setUp(self): self.data_layout = 'NCHW' self.interp_method = 'nearest' - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.init_test_case() @@ -233,13 +239,30 @@ def setUp(self): # output_np if len(self.input_shape) == 4: output_np = nearest_neighbor_interp_np( - input_np, out_h, out_w, scale_h, scale_w, self.out_size, - self.actual_shape, self.align_corners, self.data_layout) + input_np, + out_h, + out_w, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) elif len(self.input_shape) == 5: output_np = nearest_neighbor_interp3d_np( - input_np, out_d, out_h, out_w, scale_d, scale_h, scale_w, - self.out_size, self.actual_shape, self.align_corners, - self.data_layout) + input_np, + out_d, + out_h, + out_w, + scale_d, + scale_h, + scale_w, + self.out_size, + self.actual_shape, + self.align_corners, + self.data_layout, + ) self.outputs = {'Out': output_np} self.inputs = {'X': input_np} @@ -255,7 +278,7 @@ def setUp(self): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } else: self.attrs = { @@ -263,7 +286,7 @@ def setUp(self): 'out_w': self.out_w, 'interp_method': self.interp_method, 'align_corners': self.align_corners, - 'data_layout': self.data_layout + 'data_layout': self.data_layout, } if self.scale: @@ -298,21 +321,18 @@ def init_test_case(self): """ class TestNearestNeighborInterpCase2(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 class TestNearestNeighborInterpCase3(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 self.out_w = 32 class TestNearestNeighborInterpCase4(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [4, 1, 7, 8] self.out_h = 1 @@ -320,7 +340,6 @@ def init_test_case(self): self.out_size = np.array([2, 2]).astype("int32") class TestNearestNeighborInterpCase5(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [3, 3, 9, 6] self.out_h = 12 @@ -328,7 +347,6 @@ def init_test_case(self): self.out_size = np.array([11, 11]).astype("int32") class TestNearestNeighborInterpCase6(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [1, 1, 32, 64] self.out_h = 64 @@ -336,14 +354,12 @@ def init_test_case(self): self.out_size = np.array([65, 129]).astype("int32") class TestNearestNeighborInterpSame(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [2, 3, 32, 64] self.out_h = 32 self.out_w = 64 class TestNearestNeighborInterpActualShape(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [3, 2, 32, 16] self.out_h = 64 @@ -365,21 +381,18 @@ def init_test_case(self): """ class TestNearestInterpWithoutCorners(TestNearestInterpOp): - def set_align_corners(self): self.align_corners = False class TestNearestNeighborInterpScale1(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [3, 2, 7, 5] self.out_h = 64 self.out_w = 32 - self.scale = 2. + self.scale = 2.0 self.out_size = np.array([66, 40]).astype("int32") class TestNearestNeighborInterpScale2(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [3, 2, 5, 7] self.out_h = 64 @@ -388,7 +401,6 @@ def init_test_case(self): self.out_size = np.array([66, 40]).astype("int32") class TestNearestNeighborInterpScale3(TestNearestInterpOp): - def init_test_case(self): self.input_shape = [3, 2, 7, 5] self.out_h = 64 @@ -411,7 +423,6 @@ def init_test_case(self): """ class TestNearestInterpOp_attr_tensor(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -420,7 +431,7 @@ def setUp(self): self.actual_shape = None self.interp_method = 'nearest' - self.scale = 0. + self.scale = 0.0 self.align_corners = True self.init_test_case() @@ -457,8 +468,9 @@ def setUp(self): elif self.out_size is not None: size_tensor = [] for index, ele in enumerate(self.out_size): - size_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + size_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs['SizeTensor'] = size_tensor self.attrs['out_h'] = self.out_h @@ -470,10 +482,16 @@ def setUp(self): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, 0, 0, - self.out_size, - self.actual_shape, - self.align_corners) + output_np = nearest_neighbor_interp_np( + input_np, + out_h, + out_w, + 0, + 0, + self.out_size, + self.actual_shape, + self.align_corners, + ) self.outputs = {'Out': output_np} def init_dtype(self): @@ -493,7 +511,6 @@ def init_test_case(self): # out_size is a tensor list class TestNearestInterp_attr_tensor_Case1(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.input_shape = [3, 3, 9, 6] self.out_h = 12 @@ -502,7 +519,6 @@ def init_test_case(self): # out_size is a 1-D tensor class TestNearestInterp_attr_tensor_Case2(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.input_shape = [3, 2, 32, 16] self.out_h = 64 @@ -512,7 +528,6 @@ def init_test_case(self): # scale is a 1-D tensor class TestNearestInterp_attr_tensor_Case3(TestNearestInterpOp_attr_tensor): - def init_test_case(self): self.input_shape = [3, 2, 32, 16] self.out_h = 64 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_one_hot_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_one_hot_op_xpu.py index 579f8c1faf0fe..d3f1a30066411 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_one_hot_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_one_hot_op_xpu.py @@ -20,19 +20,21 @@ import paddle import paddle.fluid.core as core from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestOneHotOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'one_hot' self.use_dynamic_create_class = False class TestXPUOneHotOP(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -49,11 +51,15 @@ def set_data(self): np.random.randint(0, self.depth - 1) for i in range(sum(self.x_lod[0])) ] - self.x = np.array(self.x).astype(self.dtype).reshape( - [sum(self.x_lod[0]), 1]) - - self.out = np.zeros(shape=(np.product(self.x.shape[:-1]), - self.depth)).astype('float32') + self.x = ( + np.array(self.x) + .astype(self.dtype) + .reshape([sum(self.x_lod[0]), 1]) + ) + + self.out = np.zeros( + shape=(np.product(self.x.shape[:-1]), self.depth) + ).astype('float32') for i in range(np.product(self.x.shape)): self.out[i, self.x[i]] = 1.0 @@ -62,7 +68,7 @@ def set_data(self): def set_input(self): self.inputs = { 'X': (self.x, self.x_lod), - 'depth_tensor': self.depth_np + 'depth_tensor': self.depth_np, } self.attrs = {'dtype': int(core.VarDesc.VarType.FP32)} @@ -73,31 +79,27 @@ def init_dtype(self): self.dtype = self.in_type class TestXPUOneHotOP_attr(TestXPUOneHotOP): - def set_input(self): self.inputs = {'X': (self.x, self.x_lod)} self.attrs = { 'dtype': int(core.VarDesc.VarType.FP32), - 'depth': self.depth + 'depth': self.depth, } class TestXPUOneHotOP_default_dtype(TestXPUOneHotOP): - def set_input(self): self.inputs = { 'X': (self.x, self.x_lod), - 'depth_tensor': self.depth_np + 'depth_tensor': self.depth_np, } self.attrs = {} class TestXPUOneHotOP_default_dtype_attr(TestXPUOneHotOP): - def set_input(self): self.inputs = {'X': (self.x, self.x_lod)} self.attrs = {'depth': self.depth} class TestXPUOneHotOP_out_of_range(TestXPUOneHotOP): - def set_data(self): self.depth = 10 self.x_lod = [[4, 1, 3, 3]] @@ -105,11 +107,15 @@ def set_data(self): np.random.choice([-1, self.depth]) for i in range(sum(self.x_lod[0])) ] - self.x = np.array(self.x).astype(self.dtype).reshape( - [sum(self.x_lod[0]), 1]) - - self.out = np.zeros(shape=(np.product(self.x.shape[:-1]), - self.depth)).astype('float32') + self.x = ( + np.array(self.x) + .astype(self.dtype) + .reshape([sum(self.x_lod[0]), 1]) + ) + + self.out = np.zeros( + shape=(np.product(self.x.shape[:-1]), self.depth) + ).astype('float32') self.outputs = {'Out': (self.out, self.x_lod)} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_one_hot_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_one_hot_v2_op_xpu.py index 6e72417344944..19228435ea4ab 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_one_hot_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_one_hot_v2_op_xpu.py @@ -21,19 +21,21 @@ sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestOneHotOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'one_hot_v2' self.use_dynamic_create_class = False class TestOneHotOp(XPUOpTest): - def init(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -48,8 +50,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0])]) - out = np.zeros(shape=(np.product(x.shape), - depth)).astype(self.dtype) + out = np.zeros(shape=(np.product(x.shape), depth)).astype( + self.dtype + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -62,7 +65,6 @@ def test_check_output(self): self.check_output_with_place(self.place) class TestOneHotOp_attr(TestOneHotOp): - def setUp(self): self.init() depth = 10 @@ -71,8 +73,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype(self.dtype) + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + self.dtype + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -80,12 +83,11 @@ def setUp(self): self.inputs = {'X': (x, x_lod)} self.attrs = { 'dtype': int(core.VarDesc.VarType.FP32), - 'depth': depth + 'depth': depth, } self.outputs = {'Out': (out, x_lod)} class TestOneHotOp_default_dtype(TestOneHotOp): - def setUp(self): self.init() depth = 10 @@ -95,8 +97,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0])]) - out = np.zeros(shape=(np.product(x.shape), - depth)).astype(self.dtype) + out = np.zeros(shape=(np.product(x.shape), depth)).astype( + self.dtype + ) for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 @@ -106,7 +109,6 @@ def setUp(self): self.outputs = {'Out': (out, x_lod)} class TestOneHotOp_default_dtype_attr(TestOneHotOp): - def setUp(self): self.init() depth = 10 @@ -115,8 +117,9 @@ def setUp(self): x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0]), 1]) - out = np.zeros(shape=(np.product(x.shape[:-1]), 1, - depth)).astype(self.dtype) + out = np.zeros(shape=(np.product(x.shape[:-1]), 1, depth)).astype( + self.dtype + ) for i in range(np.product(x.shape)): out[i, 0, x[i]] = 1.0 @@ -126,7 +129,6 @@ def setUp(self): self.outputs = {'Out': (out, x_lod)} class TestOneHotOp_out_of_range(TestOneHotOp): - def setUp(self): self.init() depth = 10 @@ -134,8 +136,9 @@ def setUp(self): x = [np.random.choice([-1, depth]) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int32').reshape([sum(x_lod[0])]) - out = np.zeros(shape=(np.product(x.shape), - depth)).astype(self.dtype) + out = np.zeros(shape=(np.product(x.shape), depth)).astype( + self.dtype + ) self.inputs = {'X': (x, x_lod)} self.attrs = {'depth': depth, 'allow_out_of_range': True} @@ -143,7 +146,6 @@ def setUp(self): class TestOneHotOpApi(unittest.TestCase): - def test_api(self): depth = 10 self._run(depth) @@ -154,39 +156,45 @@ def test_api_with_depthTensor(self): def test_api_with_dygraph(self): depth = 10 - label = np.array([np.random.randint(0, depth - 1) - for i in range(6)]).reshape([6, 1]) + label = np.array( + [np.random.randint(0, depth - 1) for i in range(6)] + ).reshape([6, 1]) with fluid.dygraph.guard(): one_hot_label = fluid.one_hot( - input=fluid.dygraph.to_variable(label), depth=depth) + input=fluid.dygraph.to_variable(label), depth=depth + ) def _run(self, depth): label = fluid.layers.data(name="label", shape=[1], dtype="int64") one_hot_label = fluid.one_hot(input=label, depth=depth) place = fluid.XPUPlace(0) - label_data = np.array([np.random.randint(0, 10 - 1) - for i in range(6)]).reshape([6, 1]) + label_data = np.array( + [np.random.randint(0, 10 - 1) for i in range(6)] + ).reshape([6, 1]) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - ret = exe.run(feed={ - 'label': label_data, - }, - fetch_list=[one_hot_label], - return_numpy=False) + ret = exe.run( + feed={ + 'label': label_data, + }, + fetch_list=[one_hot_label], + return_numpy=False, + ) class BadInputTestOnehotV2(unittest.TestCase): - def test_error(self): with fluid.program_guard(fluid.Program()): def test_bad_x(): - label = fluid.layers.data(name="label", - shape=[4], - append_batch_size=False, - dtype="float32") + label = fluid.layers.data( + name="label", + shape=[4], + append_batch_size=False, + dtype="float32", + ) one_hot_label = fluid.one_hot(input=label, depth=4) self.assertRaises(TypeError, test_bad_x) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_p_norm_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_p_norm_op_xpu.py index c99c896801810..a03a6e9939ab4 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_p_norm_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_p_norm_op_xpu.py @@ -19,7 +19,11 @@ sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -63,13 +67,11 @@ def ref_p_norm(x, axis, porder, keepdims=False, reduce_all=False): class XPUTestPNormOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'p_norm' self.use_dynamic_create_class = False class TestXPUPNormOp(XPUOpTest): - def setUp(self): self.op_type = "p_norm" self.dtype = self.in_type @@ -84,15 +86,16 @@ def setUp(self): x_np = np.random.uniform(-10, 10, self.shape).astype(self.dtype) - ref_y_np = ref_p_norm(x_np, self.axis, self.porder, self.keepdims, - self.asvector) + ref_y_np = ref_p_norm( + x_np, self.axis, self.porder, self.keepdims, self.asvector + ) self.inputs = {'X': x_np} self.outputs = {'Out': ref_y_np} self.attrs = { 'epsilon': self.epsilon, 'axis': self.axis, 'porder': float(self.porder), - 'asvector': self.asvector + 'asvector': self.asvector, } def set_attrs(self): @@ -105,70 +108,60 @@ def test_check_grad(self): self.check_grad_with_place(paddle.XPUPlace(0), ['X'], 'Out') class TestPnormOp2(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3] self.axis = 2 self.porder = 2.0 class TestPnormOp3(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3] self.axis = 2 self.porder = np.inf class TestPnormOp4(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3] self.axis = 2 self.porder = -np.inf class TestPnormOp5(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3] self.axis = 2 self.porder = 0 class TestPnormOp6(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3] self.axis = -1 self.porder = 2 class TestPnormOp7(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3, 10] self.axis = 2 self.porder = 2.0 class TestPnormOp8(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3] self.axis = 2 self.porder = np.inf class TestPnormOp9(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3, 10] self.axis = 1 self.porder = -np.inf class TestPnormOp10(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3, 10] self.axis = 2 self.porder = 0 class TestPnormOp11(TestXPUPNormOp): - def set_attrs(self): self.shape = [3, 20, 3, 10] self.axis = -1 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_pool2d_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_pool2d_op_xpu.py index f630c21c94aba..45c9f518cbddd 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_pool2d_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_pool2d_op_xpu.py @@ -20,33 +20,43 @@ from op_test_xpu import XPUOpTest from test_pool2d_op import adaptive_start_index, adaptive_end_index -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle paddle.enable_static() -def max_pool2D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_type=np.float64): +def max_pool2D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_type=np.float64, +): N, C, H, W = x.shape if global_pool == 1: ksize = [H, W] if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 - W_out = (W - ksize[1] + 2 * paddings[1] + strides[1] - - 1) // strides[1] + 1 if ceil_mode else ( - W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + H_out = ( + (H - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + 2 * paddings[1] + strides[1] - 1) // strides[1] + 1 + if ceil_mode + else (W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + ) out = np.zeros((N, C, H_out, W_out)) for i in range(H_out): for j in range(W_out): @@ -66,27 +76,33 @@ def max_pool2D_forward_naive(x, return out -def avg_pool2D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_type=np.float64): +def avg_pool2D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_type=np.float64, +): N, C, H, W = x.shape if global_pool == 1: ksize = [H, W] if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + 2 * paddings[0] + strides[0] - - 1) // strides[0] + 1 if ceil_mode else ( - H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 - W_out = (W - ksize[1] + 2 * paddings[1] + strides[1] - - 1) // strides[1] + 1 if ceil_mode else ( - W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + H_out = ( + (H - ksize[0] + 2 * paddings[0] + strides[0] - 1) // strides[0] + 1 + if ceil_mode + else (H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + 2 * paddings[1] + strides[1] - 1) // strides[1] + 1 + if ceil_mode + else (W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + ) out = np.zeros((N, C, H_out, W_out)) for i in range(H_out): for j in range(W_out): @@ -108,39 +124,44 @@ def avg_pool2D_forward_naive(x, x_masked = x[:, :, r_start:r_end, c_start:c_end] - if (exclusive or adaptive): + if exclusive or adaptive: field_size = (r_end - r_start) * (c_end - c_start) if data_type == np.int8 or data_type == np.uint8: - out[:, :, i, - j] = (np.rint(np.sum(x_masked, axis=(2, 3)) / - field_size)).astype(data_type) + out[:, :, i, j] = ( + np.rint(np.sum(x_masked, axis=(2, 3)) / field_size) + ).astype(data_type) else: - out[:, :, i, j] = (np.sum(x_masked, axis=(2, 3)) / - field_size).astype(data_type) + out[:, :, i, j] = ( + np.sum(x_masked, axis=(2, 3)) / field_size + ).astype(data_type) return out -def pool2D_forward_naive(x, - ksize, - strides, - paddings, - global_pool=0, - ceil_mode=False, - exclusive=True, - adaptive=False, - data_format='NCHW', - pool_type="max", - padding_algorithm="EXPLICIT"): +def pool2D_forward_naive( + x, + ksize, + strides, + paddings, + global_pool=0, + ceil_mode=False, + exclusive=True, + adaptive=False, + data_format='NCHW', + pool_type="max", + padding_algorithm="EXPLICIT", +): # update paddings def _get_padding_with_SAME(input_shape, pool_size, pool_stride): padding = [] - for input_size, filter_size, stride_size in zip(input_shape, pool_size, - pool_stride): + for input_size, filter_size, stride_size in zip( + input_shape, pool_size, pool_stride + ): out_size = int((input_size + stride_size - 1) / stride_size) pad_sum = np.max( - ((out_size - 1) * stride_size + filter_size - input_size, 0)) + ((out_size - 1) * stride_size + filter_size - input_size, 0) + ) pad_0 = int(pad_sum / 2) pad_1 = int(pad_sum - pad_0) padding.append(pad_0) @@ -150,9 +171,10 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): if isinstance(padding_algorithm, str): padding_algorithm = padding_algorithm.upper() if padding_algorithm not in ["SAME", "VALID", "EXPLICIT"]: - raise ValueError("Unknown Attr(padding_algorithm): '%s'. " - "It can only be 'SAME' or 'VALID'." % - str(padding_algorithm)) + raise ValueError( + "Unknown Attr(padding_algorithm): '%s'. " + "It can only be 'SAME' or 'VALID'." % str(padding_algorithm) + ) if padding_algorithm == "VALID": paddings = [0, 0, 0, 0] @@ -160,7 +182,8 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): raise ValueError( "When Attr(pool_padding) is \"VALID\", Attr(ceil_mode)" " must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) elif padding_algorithm == "SAME": input_data_shape = [] if data_format == "NCHW": @@ -173,8 +196,11 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): is_sys = True if len(paddings) == 2 else False N = x.shape[0] - C, H, W = [x.shape[1], x.shape[2], x.shape[3]] if data_format == 'NCHW' \ + C, H, W = ( + [x.shape[1], x.shape[2], x.shape[3]] + if data_format == 'NCHW' else [x.shape[3], x.shape[1], x.shape[2]] + ) if global_pool == 1: ksize = [H, W] @@ -188,13 +214,26 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): if adaptive: H_out, W_out = ksize else: - H_out = (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) // strides[0] + 1 \ - if ceil_mode else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 - W_out = (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) // strides[1] + 1 \ - if ceil_mode else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 - - out = np.zeros((N, C, H_out, W_out)) if data_format=='NCHW' \ + H_out = ( + (H - ksize[0] + pad_h_up + pad_h_down + strides[0] - 1) + // strides[0] + + 1 + if ceil_mode + else (H - ksize[0] + pad_h_up + pad_h_down) // strides[0] + 1 + ) + W_out = ( + (W - ksize[1] + pad_w_left + pad_w_right + strides[1] - 1) + // strides[1] + + 1 + if ceil_mode + else (W - ksize[1] + pad_w_left + pad_w_right) // strides[1] + 1 + ) + + out = ( + np.zeros((N, C, H_out, W_out)) + if data_format == 'NCHW' else np.zeros((N, H_out, W_out, C)) + ) for i in range(H_out): if adaptive: in_h_start = adaptive_start_index(i, H, ksize[0]) @@ -222,21 +261,22 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): if data_format == 'NCHW': x_masked = x[:, :, in_h_start:in_h_end, in_w_start:in_w_end] if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (in_h_end - in_h_start) * (in_w_end - - in_w_start) - + if exclusive or adaptive: + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) -# if (exclusive or adaptive) else (ksize[0] * ksize[1]) + # if (exclusive or adaptive) else (ksize[0] * ksize[1]) out[:, :, i, j] = np.sum(x_masked, axis=(2, 3)) / field_size elif pool_type == 'max': out[:, :, i, j] = np.max(x_masked, axis=(2, 3)) elif data_format == 'NHWC': x_masked = x[:, in_h_start:in_h_end, in_w_start:in_w_end, :] if pool_type == 'avg': - if (exclusive or adaptive): - field_size = (in_h_end - in_h_start) * (in_w_end - - in_w_start) + if exclusive or adaptive: + field_size = (in_h_end - in_h_start) * ( + in_w_end - in_w_start + ) out[:, i, j, :] = np.sum(x_masked, axis=(1, 2)) / field_size elif pool_type == 'max': out[:, i, j, :] = np.max(x_masked, axis=(1, 2)) @@ -244,13 +284,11 @@ def _get_padding_with_SAME(input_shape, pool_size, pool_stride): class XPUTestPool2D_Op(XPUOpTestWrapper): - def __init__(self): self.op_name = 'pool2d' self.use_dynamic_create_class = False class TestPool2D_Op(XPUOpTest): - def setUp(self): self.op_type = "pool2d" self.dtype = self.in_type @@ -272,10 +310,18 @@ def setUp(self): input = np.random.random(self.shape).astype(self.dtype) output = pool2D_forward_naive( - input, self.ksize, self.strides, self.paddings, - self.global_pool, self.ceil_mode, self.exclusive, self.adaptive, - self.data_format, self.pool_type, - self.padding_algorithm).astype(self.dtype) + input, + self.ksize, + self.strides, + self.paddings, + self.global_pool, + self.ceil_mode, + self.exclusive, + self.adaptive, + self.data_format, + self.pool_type, + self.padding_algorithm, + ).astype(self.dtype) self.inputs = {'X': XPUOpTest.np_dtype_to_fluid_dtype(input)} self.attrs = { @@ -290,7 +336,7 @@ def setUp(self): 'exclusive': self.exclusive, 'adaptive': self.adaptive, "padding_algorithm": self.padding_algorithm, - 'ceil_mode': self.ceil_mode + 'ceil_mode': self.ceil_mode, } self.outputs = {'Out': output} @@ -335,12 +381,10 @@ def init_adaptive(self): self.adaptive = False class TestAvgPoolAdaptive(TestPool2D_Op): - def init_adaptive(self): self.adaptive = True class TestAvgPoolAdaptiveAsyOutSize(TestPool2D_Op): - def init_adaptive(self): self.adaptive = True @@ -353,7 +397,6 @@ def init_test_case(self): self.paddings = [0, 0, 0, 0] class TestCase1(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -372,7 +415,6 @@ def init_shape(self): self.shape = [2, 3, 7, 7] class TestCase2(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -391,25 +433,21 @@ def init_shape(self): self.shape = [2, 3, 7, 7] class TestCase3(TestPool2D_Op): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase4(TestCase1): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestCase5(TestCase2): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive class TestPool2D_AsyPadding(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -419,7 +457,6 @@ def init_shape(self): self.shape = [2, 3, 5, 5] class TestCase1_AsyPadding(TestCase1): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -429,7 +466,6 @@ def init_shape(self): self.shape = [2, 3, 7, 7] class TestCase2_AsyPadding(TestCase2): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -439,7 +475,6 @@ def init_shape(self): self.shape = [2, 3, 7, 7] class TestCase3_AsyPadding(TestCase3): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -449,7 +484,6 @@ def init_shape(self): self.shape = [2, 3, 5, 5] class TestCase4_AsyPadding(TestCase4): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -459,7 +493,6 @@ def init_shape(self): self.shape = [2, 3, 7, 7] class TestCase5_AsyPadding(TestCase5): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -469,7 +502,6 @@ def init_shape(self): self.shape = [2, 3, 7, 7] class TestAvgInclude_AsyPadding(TestCase2): - def init_exclusive(self): self.exclusive = False @@ -482,7 +514,6 @@ def init_shape(self): self.shape = [2, 3, 7, 7] class TestCaseCeil1(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -504,7 +535,6 @@ def init_ceil_mode(self): self.ceil_mode = True class TestCaseCeil2(TestPool2D_Op): - def init_test_case(self): self.ksize = [3, 3] self.strides = [1, 1] @@ -526,7 +556,6 @@ def init_ceil_mode(self): self.ceil_mode = True class TestCaseCeil3(TestPool2D_Op): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive @@ -535,7 +564,6 @@ def init_ceil_mode(self): self.ceil_mode = True class TestCaseCeil4(TestCaseCeil1): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive @@ -544,7 +572,6 @@ def init_ceil_mode(self): self.ceil_mode = True class TestCaseCeil5(TestCaseCeil2): - def init_pool_type(self): self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive @@ -553,7 +580,6 @@ def init_ceil_mode(self): self.ceil_mode = True class TestCaseAdaptiveAvg(TestPool2D_Op): - def init_test_case(self): self.ksize = [2, 2] self.strides = [2, 2] @@ -575,7 +601,6 @@ def init_adaptive_mode(self): self.adaptive = True class TestCaseAdaptiveMax(TestPool2D_Op): - def init_test_case(self): self.ksize = [2, 2] self.strides = [2, 2] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_pow2_decay_with_linear_warmup_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_pow2_decay_with_linear_warmup_op_xpu.py index 620cdac1811e9..246511c9f6c2c 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_pow2_decay_with_linear_warmup_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_pow2_decay_with_linear_warmup_op_xpu.py @@ -28,8 +28,9 @@ def gen_pow2_warmup_op_lr(warmup_steps, total_steps, base_lr, end_lr, place): main = paddle.static.Program() startup = paddle.static.Program() with paddle.static.program_guard(main, startup): - lr = pow2_decay_with_linear_warmup(warmup_steps, total_steps, base_lr, - end_lr) + lr = pow2_decay_with_linear_warmup( + warmup_steps, total_steps, base_lr, end_lr + ) exe = paddle.static.Executor(place) with paddle.static.scope_guard(paddle.static.Scope()): exe.run(startup) @@ -39,18 +40,21 @@ def gen_pow2_warmup_op_lr(warmup_steps, total_steps, base_lr, end_lr, place): class Pow2Warmup(LinearWarmup): - def __init__(self, warmup_steps, total_steps, base_lr, end_lr): assert total_steps > warmup_steps - lr_sch = PolynomialDecay(learning_rate=base_lr, - decay_steps=total_steps - warmup_steps, - end_lr=end_lr, - power=2) + lr_sch = PolynomialDecay( + learning_rate=base_lr, + decay_steps=total_steps - warmup_steps, + end_lr=end_lr, + power=2, + ) - super(Pow2Warmup, self).__init__(learning_rate=lr_sch, - warmup_steps=warmup_steps, - start_lr=0.0, - end_lr=base_lr) + super(Pow2Warmup, self).__init__( + learning_rate=lr_sch, + warmup_steps=warmup_steps, + start_lr=0.0, + end_lr=base_lr, + ) def gen_pow2_warmup_py_lr(warmup_steps, total_steps, base_lr, end_lr, place): @@ -62,7 +66,6 @@ def gen_pow2_warmup_py_lr(warmup_steps, total_steps, base_lr, end_lr, place): class TestPowWarmup(unittest.TestCase): - def setUp(self): paddle.enable_static() self.op_type = 'pow2_decay_with_linear_warmup' diff --git a/python/paddle/fluid/tests/unittests/xpu/test_prior_box_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_prior_box_op_xpu.py index 4561a54d65e79..acc05963ee435 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_prior_box_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_prior_box_op_xpu.py @@ -22,19 +22,21 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestPriorBoxOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'prior_box' self.use_dynamic_create_class = False class TestPriorBoxOp(XPUOpTest): - def setUp(self): self.op_type = "prior_box" self.use_xpu = True @@ -56,7 +58,7 @@ def set_data(self): 'min_max_aspect_ratios_order': self.min_max_aspect_ratios_order, 'step_w': self.step_w, 'step_h': self.step_h, - 'offset': self.offset + 'offset': self.offset, } if len(self.max_sizes) > 0: self.attrs['max_sizes'] = self.max_sizes @@ -95,11 +97,13 @@ def init_test_params(self): self.flip = True self.set_min_max_aspect_ratios_order() self.real_aspect_ratios = [1, 2.0, 1.0 / 2.0, 3.0, 1.0 / 3.0] - self.aspect_ratios = np.array(self.aspect_ratios, - dtype=np.float64).flatten() + self.aspect_ratios = np.array( + self.aspect_ratios, dtype=np.float64 + ).flatten() self.variances = [0.1, 0.1, 0.2, 0.2] - self.variances = np.array(self.variances, - dtype=np.float64).flatten() + self.variances = np.array( + self.variances, dtype=np.float64 + ).flatten() self.clip = True self.num_priors = len(self.real_aspect_ratios) * len(self.min_sizes) @@ -109,12 +113,22 @@ def init_test_params(self): def init_test_input(self): self.image = np.random.random( - (self.batch_size, self.image_channels, self.image_w, - self.image_h)).astype(self.dtype) + ( + self.batch_size, + self.image_channels, + self.image_w, + self.image_h, + ) + ).astype(self.dtype) self.input = np.random.random( - (self.batch_size, self.input_channels, self.layer_w, - self.layer_h)).astype(self.dtype) + ( + self.batch_size, + self.input_channels, + self.layer_w, + self.layer_h, + ) + ).astype(self.dtype) def init_test_output(self): out_dim = (self.layer_h, self.layer_w, self.num_priors, 4) @@ -135,72 +149,76 @@ def init_test_output(self): ar = self.real_aspect_ratios[r] c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 else: - c_w = c_h = min_size / 2. - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + c_w = c_h = min_size / 2.0 + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 if len(self.max_sizes) > 0: max_size = self.max_sizes[s] # second prior: aspect_ratio = 1, c_w = c_h = math.sqrt(min_size * max_size) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # rest of priors for r in range(len(self.real_aspect_ratios)): ar = self.real_aspect_ratios[r] - if abs(ar - 1.) < 1e-6: + if abs(ar - 1.0) < 1e-6: continue c_w = min_size * math.sqrt(ar) / 2 c_h = (min_size / math.sqrt(ar)) / 2 - out_boxes[h, w, - idx, :] = [(c_x - c_w) / self.image_w, - (c_y - c_h) / self.image_h, - (c_x + c_w) / self.image_w, - (c_y + c_h) / self.image_h] + out_boxes[h, w, idx, :] = [ + (c_x - c_w) / self.image_w, + (c_y - c_h) / self.image_h, + (c_x + c_w) / self.image_w, + (c_y + c_h) / self.image_h, + ] idx += 1 # clip the prior's coordidate such that it is within[0, 1] if self.clip: out_boxes = np.clip(out_boxes, 0.0, 1.0) # set the variance. - out_var = np.tile(self.variances, - (self.layer_h, self.layer_w, self.num_priors, 1)) + out_var = np.tile( + self.variances, (self.layer_h, self.layer_w, self.num_priors, 1) + ) self.out_boxes = out_boxes.astype(self.dtype) self.out_var = out_var.astype(self.dtype) class TestPriorBoxOpWithoutMaxSize(TestPriorBoxOp): - def set_max_sizes(self): self.max_sizes = [] class TestPriorBoxOpWithSpecifiedOutOrder(TestPriorBoxOp): - def set_min_max_aspect_ratios_order(self): self.min_max_aspect_ratios_order = True diff --git a/python/paddle/fluid/tests/unittests/xpu/test_range_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_range_xpu.py index 3135255bea285..02893acc0e2d1 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_range_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_range_xpu.py @@ -19,19 +19,21 @@ sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestRangeOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "range" self.use_dynamic_create_class = False class TestRangeOp(XPUOpTest): - def setUp(self): self.set_xpu() self.op_type = "range" @@ -40,13 +42,13 @@ def setUp(self): self.inputs = { 'Start': np.array([self.case[0]]).astype(self.dtype), 'End': np.array([self.case[1]]).astype(self.dtype), - 'Step': np.array([self.case[2]]).astype(self.dtype) + 'Step': np.array([self.case[2]]).astype(self.dtype), } self.outputs = { - 'Out': - np.arange(self.case[0], self.case[1], - self.case[2]).astype(self.dtype) + 'Out': np.arange( + self.case[0], self.case[1], self.case[2] + ).astype(self.dtype) } def set_xpu(self): @@ -63,27 +65,22 @@ def test_check_output(self): self.check_output_with_place(place, check_dygraph=False) class TestRangeOpCase0(TestRangeOp): - def init_config(self): self.case = (0, 5, 1) class TestRangeOpCase1(TestRangeOp): - def init_config(self): self.case = (0, 5, 2) class TestRangeOpCase2(TestRangeOp): - def init_config(self): self.case = (10, 1, -2) class TestRangeOpCase3(TestRangeOp): - def init_config(self): self.case = (-1, -10, -2) class TestRangeOpCase4(TestRangeOp): - def init_config(self): self.case = (10, -10, -11) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_all_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_all_op_xpu.py index 1d70b5fb3cafe..df485b49ac1ea 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_all_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_all_op_xpu.py @@ -20,18 +20,20 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceAllOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_all' class XPUTestReduceAllBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.set_case() @@ -42,11 +44,12 @@ def set_case(self): 'use_xpu': True, 'reduce_all': True, 'keep_dim': True, - 'dim': (3, 5, 4) + 'dim': (3, 5, 4), } self.inputs = { - 'X': - np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.outputs = {'Out': self.inputs['X'].all(axis=self.attrs['dim'])} @@ -57,14 +60,13 @@ def test_check_grad(self): pass class XPUTestReduceAllCase1(XPUTestReduceAllBase): - def set_case(self): self.op_type = 'reduce_all' self.attrs = { 'use_xpu': True, 'reduce_all': True, 'keep_dim': True, - 'dim': [1] + 'dim': [1], } self.inputs = { 'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool") @@ -72,23 +74,22 @@ def set_case(self): self.outputs = {'Out': self.inputs['X'].all()} class XPUTestReduceAllCase2(XPUTestReduceAllBase): - def set_case(self): self.op_type = 'reduce_all' self.attrs = { 'use_xpu': True, 'reduce_all': True, 'keep_dim': False, - 'dim': (3, 6) + 'dim': (3, 6), } self.inputs = { - 'X': - np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.outputs = {'Out': self.inputs['X'].all(axis=self.attrs['dim'])} class XPUTestReduceAllCase3(XPUTestReduceAllBase): - def set_case(self): self.op_type = 'reduce_all' self.attrs = { diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_amax_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_amax_op_xpu.py index 05b9edf7d8ae3..4394340aa1c80 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_amax_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_amax_op_xpu.py @@ -20,18 +20,20 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceAmaxOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_amax' class XPUTestReduceAmaxBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.set_case() @@ -39,7 +41,7 @@ def setUp(self): def set_case(self): self.op_type = 'reduce_amax' self.shape = (20, 10) - self.attrs = {'use_xpu': True, 'keep_dim': False, 'dim': (1, )} + self.attrs = {'use_xpu': True, 'keep_dim': False, 'dim': (1,)} self.inputs = { 'X': np.random.randint(0, 100, self.shape).astype("float32") @@ -47,10 +49,11 @@ def set_case(self): expect_intput = self.inputs['X'] self.outputs = { - 'Out': - np.amax(expect_intput, - axis=self.attrs['dim'], - keepdims=self.attrs['keep_dim']) + 'Out': np.amax( + expect_intput, + axis=self.attrs['dim'], + keepdims=self.attrs['keep_dim'], + ) } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_amin_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_amin_op_xpu.py index 4e164840126c2..77c45ce06424f 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_amin_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_amin_op_xpu.py @@ -20,18 +20,20 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceAmaxOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_amin' class XPUTestReduceAmaxBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.set_case() @@ -39,7 +41,7 @@ def setUp(self): def set_case(self): self.op_type = 'reduce_amin' self.shape = (20, 10) - self.attrs = {'use_xpu': True, 'keep_dim': False, 'dim': (1, )} + self.attrs = {'use_xpu': True, 'keep_dim': False, 'dim': (1,)} self.inputs = { 'X': np.random.randint(0, 100, self.shape).astype("float32") @@ -47,10 +49,11 @@ def set_case(self): expect_intput = self.inputs['X'] self.outputs = { - 'Out': - np.amin(expect_intput, - axis=self.attrs['dim'], - keepdims=self.attrs['keep_dim']) + 'Out': np.amin( + expect_intput, + axis=self.attrs['dim'], + keepdims=self.attrs['keep_dim'], + ) } def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_any_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_any_op_xpu.py index 116eea5bb105a..032d138558d91 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_any_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_any_op_xpu.py @@ -20,18 +20,20 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceAnyOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_any' class XPUTestReduceAnyBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.set_case() @@ -42,11 +44,12 @@ def set_case(self): 'use_xpu': True, 'reduce_all': True, 'keep_dim': True, - 'dim': (3, 5, 4) + 'dim': (3, 5, 4), } self.inputs = { - 'X': - np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.outputs = {'Out': self.inputs['X'].any(axis=self.attrs['dim'])} @@ -57,7 +60,6 @@ def test_check_grad(self): pass class XPUTestReduceAnyCase1(XPUTestReduceAnyBase): - def set_case(self): self.op_type = 'reduce_any' self.attrs = { @@ -72,18 +74,18 @@ def set_case(self): self.outputs = {'Out': self.inputs['X'].any(axis=1)} class XPUTestReduceAnyCase2(XPUTestReduceAnyBase): - def set_case(self): self.op_type = 'reduce_any' self.attrs = { 'use_xpu': True, 'reduce_all': True, 'keep_dim': False, - 'dim': (3, 6) + 'dim': (3, 6), } self.inputs = { - 'X': - np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype("bool") + 'X': np.random.randint(0, 2, (2, 5, 3, 2, 2, 3, 4, 2)).astype( + "bool" + ) } self.outputs = {'Out': self.inputs['X'].any(axis=self.attrs['dim'])} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_max_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_max_op_xpu.py index fd6b96d7ed7e1..ee0922110e5bc 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_max_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_max_op_xpu.py @@ -20,18 +20,20 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceMaxOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_max' class XPUTestReduceMaxBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -42,21 +44,21 @@ def set_case(self): self.attrs = { 'use_xpu': True, 'reduce_all': self.reduce_all, - 'keep_dim': self.keep_dim + 'keep_dim': self.keep_dim, } self.inputs = {'X': np.random.random(self.shape).astype("float32")} if self.attrs['reduce_all']: self.outputs = {'Out': self.inputs['X'].max()} else: self.outputs = { - 'Out': - self.inputs['X'].max(axis=self.axis, - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].max( + axis=self.axis, keepdims=self.attrs['keep_dim'] + ) } def init_case(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = False self.keep_dim = False @@ -67,10 +69,9 @@ def test_check_grad(self): self.check_grad_with_place(self.place, ['X'], 'Out') class XPUTestReduceMaxCase1(XPUTestReduceMaxBase): - def init_case(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = False self.keep_dim = True diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_mean_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_mean_op_xpu.py index 0b1b9070025b3..41efec0c29b0e 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_mean_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_mean_op_xpu.py @@ -18,20 +18,22 @@ sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle paddle.enable_static() class XPUTestMeanOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_mean' self.use_dynamic_create_class = False class TestMeanOp(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -47,7 +49,6 @@ def test_check_grad(self): self.check_grad_with_place(self.place, ['X'], 'Out') class TestMeanOp5D(TestMeanOp): - def setUp(self): super().setUp() self.inputs = { @@ -57,7 +58,6 @@ def setUp(self): self.outputs = {'Out': self.inputs['X'].mean(axis=0)} class TestMeanOp6D(TestMeanOp): - def setUp(self): super().setUp() self.inputs = { @@ -67,25 +67,23 @@ def setUp(self): self.outputs = {'Out': self.inputs['X'].mean(axis=0)} class TestMeanOp8D(TestMeanOp): - def setUp(self): super().setUp() self.inputs = { - 'X': np.random.random( - (1, 3, 1, 2, 1, 4, 3, 10)).astype(self.dtype) + 'X': np.random.random((1, 3, 1, 2, 1, 4, 3, 10)).astype( + self.dtype + ) } self.attrs = {'dim': (0, 3), 'use_xpu': True} self.outputs = {'Out': self.inputs['X'].mean(axis=(0, 3))} class XPUTestReduce(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_mean' self.use_dynamic_create_class = False class Test1DReduce(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -105,7 +103,6 @@ def test_check_grad(self): self.check_grad_with_place(self.place, ['X'], 'Out') class Test2DReduce0(Test1DReduce): - def setUp(self): super().setUp() self.attrs = {'dim': [0], 'use_xpu': True} @@ -113,7 +110,6 @@ def setUp(self): self.outputs = {'Out': self.inputs['X'].mean(axis=0)} class Test2DReduce1(Test1DReduce): - def setUp(self): super().setUp() self.attrs = {'dim': [1], 'use_xpu': True} @@ -123,7 +119,6 @@ def setUp(self): } class Test3DReduce0(Test1DReduce): - def setUp(self): super().setUp() self.attrs = {'dim': [1], 'use_xpu': True} @@ -133,7 +128,6 @@ def setUp(self): } class Test3DReduce1(Test1DReduce): - def setUp(self): super().setUp() self.attrs = {'dim': [2], 'use_xpu': True} @@ -143,7 +137,6 @@ def setUp(self): } class Test3DReduce2(Test1DReduce): - def setUp(self): super().setUp() self.attrs = {'dim': [-2], 'use_xpu': True} @@ -153,7 +146,6 @@ def setUp(self): } class Test3DReduce3(Test1DReduce): - def setUp(self): super().setUp() self.attrs = {'dim': [1, 2], 'use_xpu': True} @@ -163,7 +155,6 @@ def setUp(self): } class Test6DReduce(Test1DReduce): - def setUp(self): super().setUp() self.attrs = {'dim': [1, -1], 'use_xpu': True} @@ -175,30 +166,31 @@ def setUp(self): } class TestKeepDimReduce(Test1DReduce): - def setUp(self): super().setUp() self.inputs = {'X': np.random.random((5, 6, 10)).astype(self.dtype)} self.attrs = {'dim': [1], 'keep_dim': True, 'use_xpu': True} self.outputs = { - 'Out': - self.inputs['X'].mean(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].mean( + axis=tuple(self.attrs['dim']), + keepdims=self.attrs['keep_dim'], + ) } class TestKeepDim8DReduce(Test1DReduce): - def setUp(self): super().setUp() self.inputs = { - 'X': np.random.random( - (2, 5, 3, 2, 2, 3, 4, 2)).astype(self.dtype) + 'X': np.random.random((2, 5, 3, 2, 2, 3, 4, 2)).astype( + self.dtype + ) } self.attrs = {'dim': (3, 4, 5), 'keep_dim': True, 'use_xpu': True} self.outputs = { - 'Out': - self.inputs['X'].mean(axis=tuple(self.attrs['dim']), - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].mean( + axis=tuple(self.attrs['dim']), + keepdims=self.attrs['keep_dim'], + ) } diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_min_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_min_op_xpu.py index 576f550f2397a..6639071b3b0d1 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_min_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_min_op_xpu.py @@ -20,18 +20,20 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceMinOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_min' class XPUTestReduceMinBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -42,21 +44,21 @@ def set_case(self): self.attrs = { 'use_xpu': True, 'reduce_all': self.reduce_all, - 'keep_dim': self.keep_dim + 'keep_dim': self.keep_dim, } self.inputs = {'X': np.random.random(self.shape).astype("float32")} if self.attrs['reduce_all']: self.outputs = {'Out': self.inputs['X'].min()} else: self.outputs = { - 'Out': - self.inputs['X'].min(axis=self.axis, - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].min( + axis=self.axis, keepdims=self.attrs['keep_dim'] + ) } def init_case(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = False self.keep_dim = False @@ -67,10 +69,9 @@ def test_check_grad(self): pass class XPUTestReduceMinCase1(XPUTestReduceMinBase): - def init_case(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = False self.keep_dim = True diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_prod_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_prod_op_xpu.py index 17bd2356eff10..ad2cb5143b949 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_prod_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_prod_op_xpu.py @@ -21,19 +21,21 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceProdOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_prod' self.use_dynamic_create_class = False class TestXPUReduceProdOp(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -45,21 +47,21 @@ def setUp(self): self.attrs = { 'dim': self.axis, 'keep_dim': self.keep_dim, - 'reduce_all': self.reduce_all + 'reduce_all': self.reduce_all, } self.inputs = {'X': np.random.random(self.shape).astype(self.dtype)} if self.attrs['reduce_all']: self.outputs = {'Out': self.inputs['X'].prod()} else: self.outputs = { - 'Out': - self.inputs['X'].prod(axis=self.axis, - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].prod( + axis=self.axis, keepdims=self.attrs['keep_dim'] + ) } def initTestCase(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) def init_dtype(self): self.dtype = self.in_type @@ -71,84 +73,71 @@ def test_check_grad(self): self.check_grad_with_place(self.place, ['X'], 'Out') class TestProdOp5D(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (1, 2, 5, 6, 10) - self.axis = (0, ) + self.axis = (0,) class TestProdOp6D(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (1, 1, 2, 5, 6, 10) - self.axis = (0, ) + self.axis = (0,) class TestProdOp8D(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (1, 3, 1, 2, 1, 4, 3, 10) self.axis = (0, 3) class Test1DReduce(TestXPUReduceProdOp): - def initTestCase(self): self.shape = 120 - self.axis = (0, ) + self.axis = (0,) class Test2DReduce0(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (20, 10) - self.axis = (0, ) + self.axis = (0,) class Test2DReduce1(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (20, 10) - self.axis = (1, ) + self.axis = (1,) class Test3DReduce0(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (5, 6, 7) - self.axis = (1, ) + self.axis = (1,) class Test3DReduce1(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (5, 6, 7) - self.axis = (2, ) + self.axis = (2,) class Test3DReduce2(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (5, 6, 7) - self.axis = (-2, ) + self.axis = (-2,) class Test3DReduce3(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (5, 6, 7) self.axis = (1, 2) class TestKeepDimReduce(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (5, 6, 10) - self.axis = (1, ) + self.axis = (1,) self.keep_dim = True class TestKeepDim8DReduce(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (2, 5, 3, 2, 2, 3, 4, 2) self.axis = (3, 4, 5) self.keep_dim = True class TestReduceAll(TestXPUReduceProdOp): - def initTestCase(self): self.shape = (5, 6, 2, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = True diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reduce_sum_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reduce_sum_op_xpu.py index f3e0287d47a58..dc377df7ab141 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reduce_sum_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reduce_sum_op_xpu.py @@ -20,18 +20,20 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReduceSumOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'reduce_sum' class XPUTestReduceSumBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_case() @@ -42,21 +44,21 @@ def set_case(self): self.attrs = { 'use_xpu': True, 'reduce_all': self.reduce_all, - 'keep_dim': self.keep_dim + 'keep_dim': self.keep_dim, } self.inputs = {'X': np.random.random(self.shape).astype("float32")} if self.attrs['reduce_all']: self.outputs = {'Out': self.inputs['X'].sum()} else: self.outputs = { - 'Out': - self.inputs['X'].sum(axis=self.axis, - keepdims=self.attrs['keep_dim']) + 'Out': self.inputs['X'].sum( + axis=self.axis, keepdims=self.attrs['keep_dim'] + ) } def init_case(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = False self.keep_dim = False @@ -67,10 +69,9 @@ def test_check_grad(self): self.check_grad_with_place(self.place, ['X'], 'Out') class XPUTestReduceSumCase1(XPUTestReduceSumBase): - def init_case(self): self.shape = (5, 6, 10) - self.axis = (0, ) + self.axis = (0,) self.reduce_all = False self.keep_dim = True diff --git a/python/paddle/fluid/tests/unittests/xpu/test_refactor_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_refactor_op_xpu.py index a338f04091d17..aa87579755064 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_refactor_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_refactor_op_xpu.py @@ -23,7 +23,11 @@ from op_test import OpTest from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -39,7 +43,6 @@ def huber_loss_forward(val, delta): # 1.动态生成不同参数的测试case，wrapper类中必须实现dynamic_create_class方法 # self.use_dynamic_create_class置为True class XPUTestArgsortOp1(XPUOpTestWrapper): - def __init__(self): self.op_name = 'argsort' self.use_dynamic_create_class = True @@ -49,14 +52,14 @@ def dynamic_create_class(self): classes = [] for descending in [True, False]: for axis in [0, 1, 2, -1, -2]: - class_name = 'XPUTestArgsortOp_axis_' + str(axis) + '_' + str( - descending) + class_name = ( + 'XPUTestArgsortOp_axis_' + str(axis) + '_' + str(descending) + ) attr_dict = {'init_axis': axis, 'init_descending': descending} classes.append([class_name, attr_dict]) return base_class, classes class TestArgsortOp(XPUOpTest): - def setUp(self): self.op_type = "argsort" self.place = paddle.XPUPlace(0) @@ -65,16 +68,18 @@ def setUp(self): self.input_shape = (2, 2, 2, 3, 3) self.axis = -1 if not hasattr(self, 'init_axis') else self.init_axis - self.descending = False if not hasattr( - self, 'init_descending') else self.init_descending + self.descending = ( + False + if not hasattr(self, 'init_descending') + else self.init_descending + ) if self.in_type == np.float32: self.x = np.random.random(self.input_shape).astype(self.dtype) else: - self.x = np.random.randint(low=-1000, - high=1000, - size=self.input_shape).astype( - self.dtype) + self.x = np.random.randint( + low=-1000, high=1000, size=self.input_shape + ).astype(self.dtype) self.inputs = {"X": self.x} self.attrs = {"axis": self.axis, "descending": self.descending} self.get_output() @@ -84,13 +89,15 @@ def get_output(self): if self.descending: self.indices = np.flip( np.argsort(self.x, kind='heapsort', axis=self.axis), - self.axis) + self.axis, + ) self.sorted_x = np.flip( - np.sort(self.x, kind='heapsort', axis=self.axis), self.axis) + np.sort(self.x, kind='heapsort', axis=self.axis), self.axis + ) else: - self.indices = np.argsort(self.x, - kind='heapsort', - axis=self.axis) + self.indices = np.argsort( + self.x, kind='heapsort', axis=self.axis + ) self.sorted_x = np.sort(self.x, kind='heapsort', axis=self.axis) def test_check_output(self): @@ -99,13 +106,11 @@ def test_check_output(self): # 2. 为不同参数的测试case定义一个测试类，self.use_dynamic_create_class需要置为False class XPUTestArgsortOp2(XPUOpTestWrapper): - def __init__(self): self.op_name = 'argsort' self.use_dynamic_create_class = False class TestArgsortOp(XPUOpTest): - def setUp(self): self.op_type = "argsort" self.place = paddle.XPUPlace(0) @@ -119,10 +124,9 @@ def setUp(self): if self.in_type == np.float32: self.x = np.random.random(self.input_shape).astype(self.dtype) else: - self.x = np.random.randint(low=-1000, - high=1000, - size=self.input_shape).astype( - self.dtype) + self.x = np.random.randint( + low=-1000, high=1000, size=self.input_shape + ).astype(self.dtype) self.inputs = {"X": self.x} self.attrs = {"axis": self.axis, "descending": self.descending} self.get_output() @@ -132,13 +136,15 @@ def get_output(self): if self.descending: self.indices = np.flip( np.argsort(self.x, kind='heapsort', axis=self.axis), - self.axis) + self.axis, + ) self.sorted_x = np.flip( - np.sort(self.x, kind='heapsort', axis=self.axis), self.axis) + np.sort(self.x, kind='heapsort', axis=self.axis), self.axis + ) else: - self.indices = np.argsort(self.x, - kind='heapsort', - axis=self.axis) + self.indices = np.argsort( + self.x, kind='heapsort', axis=self.axis + ) self.sorted_x = np.sort(self.x, kind='heapsort', axis=self.axis) def init_inputshape(self): @@ -157,57 +163,46 @@ def init_direction(self): self.descending = False class TestArgsortOpAxis0XPU(TestArgsortOp): - def init_axis(self): self.axis = 0 class TestArgsortOpAxis1XPU(TestArgsortOp): - def init_axis(self): self.axis = 1 class TestArgsortOpAxis2XPU(TestArgsortOp): - def init_axis(self): self.axis = 2 class TestArgsortOpAxisNeg1XPU(TestArgsortOp): - def init_axis(self): self.axis = -1 class TestArgsortOpAxisNeg2XPU(TestArgsortOp): - def init_axis(self): self.axis = -2 class TestArgsortOpDescendingAxisXPU(TestArgsortOp): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis0XPU(TestArgsortOpAxis0XPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis1XPU(TestArgsortOpAxis1XPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxis2XPU(TestArgsortOpAxis2XPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg1XPU(TestArgsortOpAxisNeg1XPU): - def init_direction(self): self.descending = True class TestArgsortOpDescendingAxisNeg2XPU(TestArgsortOpAxisNeg2XPU): - def init_direction(self): self.descending = True @@ -219,13 +214,11 @@ def init_direction(self): class XPUTestHuberLossOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'huber_loss' self.use_dynamic_create_class = False class TestHuberLossOp(XPUOpTest): - def setUp(self): self.op_type = 'huber_loss' self.place = paddle.XPUPlace(0) @@ -237,11 +230,11 @@ def setUp(self): def set_inputs(self): shape = self.set_shape() - x = np.random.uniform(0, 1., shape).astype(self.dtype) - y = np.random.uniform(0, 1., shape).astype(self.dtype) + x = np.random.uniform(0, 1.0, shape).astype(self.dtype) + y = np.random.uniform(0, 1.0, shape).astype(self.dtype) self.inputs = { 'X': OpTest.np_dtype_to_fluid_dtype(x), - 'Y': OpTest.np_dtype_to_fluid_dtype(y) + 'Y': OpTest.np_dtype_to_fluid_dtype(y), } def set_attrs(self): @@ -251,8 +244,9 @@ def set_outputs(self): delta = self.attrs['delta'] shape = self.set_shape() residual = self.inputs['Y'] - self.inputs['X'] - loss = np.vectorize(huber_loss_forward)(residual, - delta).astype(self.dtype) + loss = np.vectorize(huber_loss_forward)(residual, delta).astype( + self.dtype + ) self.outputs = {'Residual': residual, 'Out': loss.reshape(shape)} def set_shape(self): @@ -265,27 +259,24 @@ def test_check_grad_normal(self): self.check_grad_with_place(self.place, ['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad_with_place(self.place, ['Y'], - 'Out', - no_grad_set=set("residual")) + self.check_grad_with_place( + self.place, ['Y'], 'Out', no_grad_set=set("residual") + ) def test_check_grad_ingore_y(self): - self.check_grad_with_place(self.place, ['X'], - 'Out', - no_grad_set=set('residual')) + self.check_grad_with_place( + self.place, ['X'], 'Out', no_grad_set=set('residual') + ) class TestHuberLossOp1(TestHuberLossOp): - def set_shape(self): - return (640) + return 640 class TestHuberLossOp2(TestHuberLossOp): - def set_shape(self): return (10, 10) class TestHuberLossOp3(TestHuberLossOp): - def set_shape(self): return (10, 10, 1) @@ -293,10 +284,12 @@ def set_shape(self): support_types = get_xpu_op_support_types('huber_loss') for stype in support_types: create_test_class(globals(), XPUTestHuberLossOp, stype) - create_test_class(globals(), - XPUTestHuberLossOp, - stype, - ignore_device_version=[core.XPUVersion.XPU1]) + create_test_class( + globals(), + XPUTestHuberLossOp, + stype, + ignore_device_version=[core.XPUVersion.XPU1], + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_reshape2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_reshape2_op_xpu.py index 534fd17bf8c31..987aaf54cfa86 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_reshape2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_reshape2_op_xpu.py @@ -21,20 +21,22 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestReshapeOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "reshape2" self.use_dynamic_create_class = False # situation 1: have shape( list, no tensor), no actual shape(Tensor) class TestReshapeOp(XPUOpTest): - def setUp(self): self.init_data() self.op_type = "reshape2" @@ -55,7 +57,7 @@ def init_test_input(self): def init_test_output(self): self.outputs = { "Out": self.inputs["X"].reshape(self.infered_shape), - 'XShape': np.random.random(self.ori_shape).astype(self.dtype) + 'XShape': np.random.random(self.ori_shape).astype(self.dtype), } def init_attrs(self): @@ -72,14 +74,12 @@ def test_check_grad(self): self.check_grad_with_place(place, ["X"], "Out") class TestReshapeOpDimInfer1(TestReshapeOp): - def init_data(self): self.ori_shape = (5, 25) self.new_shape = (5, -1, 5) self.infered_shape = (5, -1, 5) class TestReshapeOpDimInfer2(TestReshapeOp): - def init_data(self): self.ori_shape = (10, 2, 6) self.new_shape = (10, 0, 3, -1) @@ -87,7 +87,6 @@ def init_data(self): # situation 2: have shape(list, no tensor), have actual shape(Tensor) class TestReshapeOpWithInputShape(TestReshapeOp): - def init_data(self): self.ori_shape = (6, 20) self.new_shape = (0, -1, 20) @@ -96,18 +95,17 @@ def init_data(self): def init_test_input(self): self.inputs = { "X": np.random.random(self.ori_shape).astype(self.dtype), - "Shape": np.array(self.actual_shape, dtype="int32") + "Shape": np.array(self.actual_shape, dtype="int32"), } def init_test_output(self): self.outputs = { "Out": self.inputs["X"].reshape(self.actual_shape), - 'XShape': np.random.random(self.ori_shape).astype(self.dtype) + 'XShape': np.random.random(self.ori_shape).astype(self.dtype), } # Situation 3: have shape(list, have tensor), no actual shape(Tensor) class TestReshapeOp_attr_ShapeTensor(TestReshapeOp): - def init_data(self): self.ori_shape = (4, 25) self.new_shape = (10, 10) @@ -117,29 +115,30 @@ def init_data(self): def init_test_input(self): shape_tensor = [] for index, ele in enumerate(self.new_shape): - shape_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + shape_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { "X": np.random.random(self.ori_shape).astype(self.dtype), - 'ShapeTensor': shape_tensor + 'ShapeTensor': shape_tensor, } def init_attrs(self): self.attrs = {'shape': self.shape, "use_xpu": True} - class TestReshapeOpDimInfer1_attr_ShapeTensor(TestReshapeOp_attr_ShapeTensor - ): - + class TestReshapeOpDimInfer1_attr_ShapeTensor( + TestReshapeOp_attr_ShapeTensor + ): def init_data(self): self.ori_shape = (5, 20) self.new_shape = (5, -1, 20) self.infered_shape = (5, -1, 20) self.shape = (5, -1, -1) - class TestReshapeOpDimInfer2_attr_ShapeTensor(TestReshapeOp_attr_ShapeTensor - ): - + class TestReshapeOpDimInfer2_attr_ShapeTensor( + TestReshapeOp_attr_ShapeTensor + ): def init_data(self): self.ori_shape = (10, 2, 6) self.new_shape = (10, 0, 3, -1) @@ -148,7 +147,6 @@ def init_data(self): # Situation 4: have shape(Tensor), no actual shape(Tensor) class TestReshapeOp_attr_OnlyShape(TestReshapeOp): - def init_data(self): self.ori_shape = (4, 25) self.new_shape = (10, 10) @@ -157,14 +155,13 @@ def init_data(self): def init_test_input(self): self.inputs = { "X": np.random.random(self.ori_shape).astype(self.dtype), - "Shape": np.array(self.new_shape, dtype="int32") + "Shape": np.array(self.new_shape, dtype="int32"), } def init_attrs(self): self.attrs = {"use_xpu": True} class TestReshapeOpDimInfer1_attr_OnlyShape(TestReshapeOp_attr_OnlyShape): - def init_data(self): self.ori_shape = (5, 20) self.new_shape = (5, -1, 10) @@ -172,7 +169,6 @@ def init_data(self): self.shape = (5, -1, -1) class TestReshapeOpDimInfer2_attr_OnlyShape(TestReshapeOp_attr_OnlyShape): - def init_data(self): self.ori_shape = (10, 2, 6) self.new_shape = (10, 0, 3, -1) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_rmsprop_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_rmsprop_op_xpu.py index 0fb0a855bb009..29b980fa12804 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_rmsprop_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_rmsprop_op_xpu.py @@ -22,28 +22,32 @@ import paddle.fluid.core as core from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() -def calculate_rmsprop_by_numpy(param, grad, mean_square, moment, learning_rate, - epsilon, decay, momentum): +def calculate_rmsprop_by_numpy( + param, grad, mean_square, moment, learning_rate, epsilon, decay, momentum +): mean_square_out = decay * mean_square + (1 - decay) * grad * grad moment_out = momentum * moment + learning_rate * grad / np.sqrt( - mean_square_out + epsilon) + mean_square_out + epsilon + ) param_out = param - moment_out return param_out, mean_square_out, moment_out class XPUTestRMSPropOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'rmsprop' self.use_dynamic_create_class = False class TestRMSPropOPBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.xpu_version = core.get_xpu_device_version(0) @@ -55,19 +59,25 @@ def set_case(self): self.dtype = self.in_type self.init_config() - self.param = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) - self.grad = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.param = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) + self.grad = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) self.mean_square = np.random.uniform(0, 1, self.input_shape).astype( - self.dtype) - self.moment = np.random.uniform(-1, 1, - self.input_shape).astype(self.dtype) + self.dtype + ) + self.moment = np.random.uniform(-1, 1, self.input_shape).astype( + self.dtype + ) self.mean_grad = np.random.uniform(-1, 1, self.input_shape).astype( - self.dtype) + self.dtype + ) self.mean_grad_out = np.random.uniform( - -1, 1, self.input_shape).astype(self.dtype) + -1, 1, self.input_shape + ).astype(self.dtype) param_out, mean_square_out, moment_out = calculate_rmsprop_by_numpy( param=self.param, @@ -77,7 +87,8 @@ def set_case(self): learning_rate=self.learning_rate, epsilon=self.epsilon, decay=self.decay, - momentum=self.momentum) + momentum=self.momentum, + ) self.inputs = { 'Param': self.param, 'Grad': self.grad, @@ -92,22 +103,22 @@ def set_case(self): 'epsilon': self.epsilon, 'decay': self.decay, 'momentum': self.momentum, - 'centered': - False, # TODO(houj04): when XDNN api supports 'center = True', add more test cases + 'centered': False, # TODO(houj04): when XDNN api supports 'center = True', add more test cases } self.outputs = { 'ParamOut': param_out, 'MomentOut': moment_out, 'MeanSquareOut': mean_square_out, - 'MeanGradOut': self.mean_grad_out + 'MeanGradOut': self.mean_grad_out, } def init_dtype(self): self.dtype = np.float32 def test_check_output(self): - self.check_output_with_place(self.place, - no_check_set=['MeanGradOut']) + self.check_output_with_place( + self.place, no_check_set=['MeanGradOut'] + ) def init_config(self): self.input_shape = [864] @@ -117,7 +128,6 @@ def init_config(self): self.momentum = 0.1 class XPUTestRMSProp1(TestRMSPropOPBase): - def init_config(self): self.input_shape = [2, 768] self.learning_rate = np.array([0.002]).astype(self.dtype) @@ -126,7 +136,6 @@ def init_config(self): self.momentum = 0.1 class XPUTestRMSProp2(TestRMSPropOPBase): - def init_config(self): self.input_shape = [3, 8, 4096] self.learning_rate = np.array([0.005]).astype(self.dtype) @@ -135,7 +144,6 @@ def init_config(self): self.momentum = 0 class XPUTestRMSProp3(TestRMSPropOPBase): - def init_config(self): self.input_shape = [1024] self.learning_rate = np.array([0.01]).astype(self.dtype) @@ -144,7 +152,6 @@ def init_config(self): self.momentum = 0.02 class XPUTestRMSProp4(TestRMSPropOPBase): - def init_config(self): self.input_shape = [2, 2, 255] self.learning_rate = np.array([0.0005]).astype(self.dtype) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_rnn_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_rnn_op_xpu.py index 6012ca710a9d8..342123d0c84db 100755 --- a/python/paddle/fluid/tests/unittests/xpu/test_rnn_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_rnn_op_xpu.py @@ -24,7 +24,11 @@ sys.path.append("../rnn") from rnn_numpy import LSTM from convert import get_params_for_net -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) random.seed(2) np.set_printoptions(threshold=np.inf) @@ -32,13 +36,11 @@ class XPUTestRNNOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'rnn' self.use_dynamic_create_class = False class TestRNNOp(XPUOpTest): - def setUp(self): self.init_size() self.init_dtype() @@ -55,41 +57,52 @@ def setUp(self): self.direction_num = 2 if self.is_bidirec else 1 direction = "bidirectional" if self.is_bidirec else "forward" - input = np.random.uniform(low=-0.1, - high=0.1, - size=(self.seq_length, self.batch_size, - self.input_size)).astype(self.dtype) + input = np.random.uniform( + low=-0.1, + high=0.1, + size=(self.seq_length, self.batch_size, self.input_size), + ).astype(self.dtype) input[11][1:][:] = 0 input[10][2:][:] = 0 input[9][3:][:] = 0 input[8][4:][:] = 0 - rnn1 = LSTM(self.input_size, - self.hidden_size, - num_layers=self.num_layers, - time_major=True, - direction=direction, - dropout=self.dropout, - dtype=self.dtype) + rnn1 = LSTM( + self.input_size, + self.hidden_size, + num_layers=self.num_layers, + time_major=True, + direction=direction, + dropout=self.dropout, + dtype=self.dtype, + ) flat_w = get_params_for_net(rnn1) - output, (last_hidden, - last_cell) = rnn1(input, - sequence_length=self.sequence_length) + output, (last_hidden, last_cell) = rnn1( + input, sequence_length=self.sequence_length + ) init_h = np.zeros( - (self.num_layers * self.direction_num, self.batch_size, - self.hidden_size)).astype(self.dtype) + ( + self.num_layers * self.direction_num, + self.batch_size, + self.hidden_size, + ) + ).astype(self.dtype) init_c = np.zeros( - (self.num_layers * self.direction_num, self.batch_size, - self.hidden_size)).astype(self.dtype) + ( + self.num_layers * self.direction_num, + self.batch_size, + self.hidden_size, + ) + ).astype(self.dtype) state_out = np.ndarray((300)).astype("uint8") self.inputs = { 'Input': input, 'WeightList': flat_w, 'PreState': [('init_h', init_h), ('init_c', init_c)], - 'SequenceLength': self.sequence_length + 'SequenceLength': self.sequence_length, } if self.sequence_length is None: self.inputs = { @@ -104,14 +117,16 @@ def setUp(self): 'hidden_size': self.hidden_size, 'num_layers': self.num_layers, 'mode': self.mode, - 'is_test': self.is_test + 'is_test': self.is_test, } self.outputs = { 'Out': output, - "State": [('last_hidden', last_hidden), - ('last_cell', last_cell)], + "State": [ + ('last_hidden', last_hidden), + ('last_cell', last_cell), + ], 'Reserve': np.ndarray((400)).astype("uint8"), - 'DropoutState': state_out + 'DropoutState': state_out, } def init_dtype(self): @@ -124,15 +139,19 @@ def set_xpu(self): def test_check_output(self): self.check_output_with_place( - self.place, atol=0.01, no_check_set=['Reserve', 'DropoutState']) + self.place, atol=0.01, no_check_set=['Reserve', 'DropoutState'] + ) def test_grad(self): if not self.is_test: var_name_list = self.get_weight_names() grad_check_list = ['Input', 'init_h', 'init_c'] grad_check_list.extend(var_name_list) - self.check_grad_with_place(self.place, set(grad_check_list), - ['Out', 'last_hidden', 'last_cell']) + self.check_grad_with_place( + self.place, + set(grad_check_list), + ['Out', 'last_hidden', 'last_cell'], + ) def init_size(self): self.seq_length = 12 @@ -154,43 +173,36 @@ def set_attrs(self): pass class TestRNNOp1(TestRNNOp): - def set_attrs(self): self.sequence_length = None class TestRNNOp2(TestRNNOp): - def set_attrs(self): self.num_layers = 1 self.is_bidirec = True class TestRNNOp3(TestRNNOp): - def set_attrs(self): self.num_layers = 2 self.is_bidirec = False class TestRNNOp4(TestRNNOp): - def set_attrs(self): self.num_layers = 3 self.is_bidirec = False class TestRNNOp5(TestRNNOp): - def set_attrs(self): self.num_layers = 2 self.is_bidirec = True class TestRNNOp6(TestRNNOp): - def set_attrs(self): self.num_layers = 2 self.is_bidirec = True self.sequence_length = None class TestRNNOp7(TestRNNOp): - def set_attrs(self): self.num_layers = 3 self.is_bidirec = True @@ -198,10 +210,12 @@ def set_attrs(self): support_types = get_xpu_op_support_types('rnn') for stype in support_types: - create_test_class(globals(), - XPUTestRNNOp, - stype, - ignore_device_version=[core.XPUVersion.XPU1]) + create_test_class( + globals(), + XPUTestRNNOp, + stype, + ignore_device_version=[core.XPUVersion.XPU1], + ) if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/xpu/test_roi_align_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_roi_align_op_xpu.py index ad63cbb5c0a25..f44596324327f 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_roi_align_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_roi_align_op_xpu.py @@ -21,19 +21,21 @@ import paddle.fluid.core as core from op_test_xpu import XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestROIAlignOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'roi_align' self.use_dynamic_create_class = False class TestROIAlignOp(XPUOpTest): - def set_data(self): self.init_test_case() self.make_rois() @@ -48,7 +50,7 @@ def set_data(self): 'pooled_height': self.pooled_height, 'pooled_width': self.pooled_width, 'sampling_ratio': self.sampling_ratio, - 'aligned': self.continuous_coordinate + 'aligned': self.continuous_coordinate, } self.outputs = {'Out': self.out_data} @@ -62,8 +64,12 @@ def init_test_case(self): self.xpu_version = core.get_xpu_device_version(0) # n, c, h, w - self.x_dim = (self.batch_size, self.channels, self.height, - self.width) + self.x_dim = ( + self.batch_size, + self.channels, + self.height, + self.width, + ) self.spatial_scale = 1.0 / 2.0 self.pooled_height = 2 @@ -75,25 +81,51 @@ def init_test_case(self): self.continuous_coordinate = bool(np.random.randint(2)) self.x = np.random.random(self.x_dim).astype(self.dtype) - def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, - roi_bin_grid_w, bin_size_h, bin_size_w): + def pre_calc( + self, + x_i, + roi_xmin, + roi_ymin, + roi_bin_grid_h, + roi_bin_grid_w, + bin_size_h, + bin_size_w, + ): count = roi_bin_grid_h * roi_bin_grid_w - bilinear_pos = np.zeros([ - self.channels, self.pooled_height, self.pooled_width, count, 4 - ], np.float32) + bilinear_pos = np.zeros( + [ + self.channels, + self.pooled_height, + self.pooled_width, + count, + 4, + ], + np.float32, + ) bilinear_w = np.zeros( - [self.pooled_height, self.pooled_width, count, 4], np.float32) + [self.pooled_height, self.pooled_width, count, 4], np.float32 + ) for ph in range(self.pooled_width): for pw in range(self.pooled_height): c = 0 for iy in range(roi_bin_grid_h): - y = roi_ymin + ph * bin_size_h + (iy + 0.5) * \ - bin_size_h / roi_bin_grid_h + y = ( + roi_ymin + + ph * bin_size_h + + (iy + 0.5) * bin_size_h / roi_bin_grid_h + ) for ix in range(roi_bin_grid_w): - x = roi_xmin + pw * bin_size_w + (ix + 0.5) * \ - bin_size_w / roi_bin_grid_w - if y < -1.0 or y > self.height or \ - x < -1.0 or x > self.width: + x = ( + roi_xmin + + pw * bin_size_w + + (ix + 0.5) * bin_size_w / roi_bin_grid_w + ) + if ( + y < -1.0 + or y > self.height + or x < -1.0 + or x > self.width + ): continue if y <= 0: y = 0 @@ -114,14 +146,18 @@ def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, hy = 1 - ly hx = 1 - lx for ch in range(self.channels): - bilinear_pos[ch, ph, pw, c, 0] = x_i[ch, y_low, - x_low] - bilinear_pos[ch, ph, pw, c, 1] = x_i[ch, y_low, - x_high] - bilinear_pos[ch, ph, pw, c, 2] = x_i[ch, y_high, - x_low] - bilinear_pos[ch, ph, pw, c, 3] = x_i[ch, y_high, - x_high] + bilinear_pos[ch, ph, pw, c, 0] = x_i[ + ch, y_low, x_low + ] + bilinear_pos[ch, ph, pw, c, 1] = x_i[ + ch, y_low, x_high + ] + bilinear_pos[ch, ph, pw, c, 2] = x_i[ + ch, y_high, x_low + ] + bilinear_pos[ch, ph, pw, c, 3] = x_i[ + ch, y_high, x_high + ] bilinear_w[ph, pw, c, 0] = hy * hx bilinear_w[ph, pw, c, 1] = hy * lx bilinear_w[ph, pw, c, 2] = ly * hx @@ -131,8 +167,13 @@ def pre_calc(self, x_i, roi_xmin, roi_ymin, roi_bin_grid_h, def calc_roi_align(self): self.out_data = np.zeros( - (self.rois_num, self.channels, self.pooled_height, - self.pooled_width)).astype(self.dtype) + ( + self.rois_num, + self.channels, + self.pooled_height, + self.pooled_width, + ) + ).astype(self.dtype) for i in range(self.rois_num): roi = self.rois[i] @@ -150,15 +191,27 @@ def calc_roi_align(self): roi_height = max(roi_height, 1) bin_size_h = float(roi_height) / float(self.pooled_height) bin_size_w = float(roi_width) / float(self.pooled_width) - roi_bin_grid_h = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_height / self.pooled_height) - roi_bin_grid_w = self.sampling_ratio if self.sampling_ratio > 0 else \ - math.ceil(roi_width / self.pooled_width) + roi_bin_grid_h = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_height / self.pooled_height) + ) + roi_bin_grid_w = ( + self.sampling_ratio + if self.sampling_ratio > 0 + else math.ceil(roi_width / self.pooled_width) + ) count = int(roi_bin_grid_h * roi_bin_grid_w) pre_size = count * self.pooled_width * self.pooled_height bilinear_pos, bilinear_w = self.pre_calc( - x_i, roi_xmin, roi_ymin, int(roi_bin_grid_h), - int(roi_bin_grid_w), bin_size_h, bin_size_w) + x_i, + roi_xmin, + roi_ymin, + int(roi_bin_grid_h), + int(roi_bin_grid_w), + bin_size_h, + bin_size_w, + ) for ch in range(self.channels): align_per_bin = (bilinear_pos[ch] * bilinear_w).sum(axis=-1) output_val = align_per_bin.mean(axis=-1) @@ -171,17 +224,20 @@ def make_rois(self): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x1 = np.random.random_integers( - 0, self.width // self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width + ) y1 = np.random.random_integers( 0, - self.height // self.spatial_scale - self.pooled_height) + self.height // self.spatial_scale - self.pooled_height, + ) x2 = np.random.random_integers( - x1 + self.pooled_width, - self.width // self.spatial_scale) + x1 + self.pooled_width, self.width // self.spatial_scale + ) y2 = np.random.random_integers( y1 + self.pooled_height, - self.height // self.spatial_scale) + self.height // self.spatial_scale, + ) roi = [bno, x1, y1, x2, y2] rois.append(roi) @@ -205,7 +261,6 @@ def test_check_grad(self): self.check_grad_with_place(self.place, {'X'}, 'Out') class TestROIAlignInLodOp(TestROIAlignOp): - def set_data(self): self.init_test_case() self.make_rois() @@ -216,7 +271,7 @@ def set_data(self): self.inputs = { 'X': self.x, 'ROIs': (self.rois[:, 1:5], self.rois_lod), - 'RoisNum': np.asarray(seq_len).astype('int32') + 'RoisNum': np.asarray(seq_len).astype('int32'), } self.attrs = { @@ -224,7 +279,7 @@ def set_data(self): 'pooled_height': self.pooled_height, 'pooled_width': self.pooled_width, 'sampling_ratio': self.sampling_ratio, - 'aligned': self.continuous_coordinate + 'aligned': self.continuous_coordinate, } self.outputs = {'Out': self.out_data} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_sampling_id_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_sampling_id_op_xpu.py index 75bdbff364da8..d2e9aa481749f 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_sampling_id_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_sampling_id_op_xpu.py @@ -23,7 +23,6 @@ class TestSamplingIdShape(unittest.TestCase): - def test_shape(self): paddle.enable_static() x = fluid.layers.data(name='x', shape=[3], dtype='float32') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_scale_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_scale_op_xpu.py index d41ba5e59d414..47c95c2158b33 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_scale_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_scale_op_xpu.py @@ -22,17 +22,19 @@ from paddle.fluid import Program, program_guard from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class XPUTestScaleOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'scale' self.use_dynamic_create_class = False class TestScaleOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -77,33 +79,27 @@ def test_check_output(self): self.check_output_with_place(place) class TestScaleOp1(TestScaleOp): - def set_attrs(self): self.attrs = {'scale': 3.5} class TestScaleOp2(TestScaleOp): - def set_attrs(self): self.attrs = {'scale': 6.77} class TestScaleOp3(TestScaleOp): - def set_attrs(self): self.attrs = {'scale': -9.19} class TestScaleOp4(TestScaleOp): - def set_attrs(self): self.attrs = {'scale': 0.0} class TestScaleOp5(TestScaleOp): - def set_attrs(self): self.attrs = {'scale': -0.003} class TestScaleApiStatic(unittest.TestCase): - def _executed_api(self, x, scale=1.0, bias=0.0): return paddle.scale(x, scale, bias) @@ -121,13 +117,11 @@ def test_api(self): class TestScaleInplaceApiStatic(TestScaleApiStatic): - def _executed_api(self, x, scale=1.0, bias=0.0): return x.scale_(scale, bias) class TestScaleApiDygraph(unittest.TestCase): - def _executed_api(self, x, scale=1.0, bias=0.0): return paddle.scale(x, scale, bias) @@ -141,7 +135,6 @@ def test_api(self): class TestScaleInplaceApiDygraph(TestScaleApiDygraph): - def _executed_api(self, x, scale=1.0, bias=0.0): return x.scale_(scale, bias) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_scatter_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_scatter_op_xpu.py index a7047014f75c0..4cd2de7e6a08a 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_scatter_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_scatter_op_xpu.py @@ -20,13 +20,17 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper, type_dict_str_to_numpy +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, + type_dict_str_to_numpy, +) paddle.enable_static() class XPUTestScatterOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'scatter' self.use_dynamic_create_class = True @@ -47,7 +51,7 @@ def dynamic_create_class(self): 'init_index_np': index_np, 'init_updates_np': updates_np, 'init_output_np': output_np, - 'test_name': 'case1' + 'test_name': 'case1', } test_data_case.append(data_dict) @@ -62,7 +66,7 @@ def dynamic_create_class(self): 'init_index_np': index_np, 'init_updates_np': updates_np, 'init_output_np': output_np, - 'test_name': 'case2' + 'test_name': 'case2', } test_data_case.append(data_dict) @@ -80,16 +84,21 @@ def dynamic_create_class(self): 'init_index_np': index_np, 'init_updates_np': updates_np, 'init_output_np': output_np, - 'test_name': 'case3' + 'test_name': 'case3', } test_data_case.append(data_dict) for data_dict in test_data_case: for index_type in ['int32', 'int64']: for overwrite in [True, False]: - class_name = 'XPUTestScatterOp_index_type_' + data_dict[ - 'test_name'] + '_' + str(index_type) + '_' + str( - overwrite) + class_name = ( + 'XPUTestScatterOp_index_type_' + + data_dict['test_name'] + + '_' + + str(index_type) + + '_' + + str(overwrite) + ) attr_dict = data_dict attr_dict['index_type'] = type_dict_str_to_numpy[index_type] attr_dict['init_overwrite'] = overwrite @@ -97,13 +106,16 @@ def dynamic_create_class(self): return base_class, classes class TestScatterOp(XPUOpTest): - def setUp(self): self.init_config() - self.index_type = np.int32 if not hasattr( - self, 'index_type') else self.index_type - self.overwrite = True if not hasattr( - self, 'init_overwrite') else self.init_overwrite + self.index_type = ( + np.int32 if not hasattr(self, 'index_type') else self.index_type + ) + self.overwrite = ( + True + if not hasattr(self, 'init_overwrite') + else self.init_overwrite + ) if not hasattr(self, 'init_ref_np'): self.ref_np = np.ones((3, 50)).astype(self.dtype) @@ -120,7 +132,7 @@ def setUp(self): self.inputs = { 'X': self.ref_np, 'Ids': self.index_np, - 'Updates': self.updates_np + 'Updates': self.updates_np, } self.attrs = {'overwrite': self.overwrite} self.outputs = {'Out': self.output_np} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_sequence_conv_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_sequence_conv_op_xpu.py index 77327c529cc4d..e15bfdbbe5849 100755 --- a/python/paddle/fluid/tests/unittests/xpu/test_sequence_conv_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_sequence_conv_op_xpu.py @@ -27,13 +27,15 @@ np.set_printoptions(threshold=np.inf) -def seqconv(x, - lod, - filter, - context_length, - context_start, - padding_trainable=False, - padding_data=None): +def seqconv( + x, + lod, + filter, + context_length, + context_start, + padding_trainable=False, + padding_data=None, +): [T, M] = x.shape col = np.zeros((T, context_length * M)).astype('float32') offset = [0] @@ -48,67 +50,87 @@ def seqconv(x, out_end = offset[i + 1] if in_begin < offset[i]: pad_size = np.min( - [offset[i] - in_begin, offset[i + 1] - offset[i]]) + [offset[i] - in_begin, offset[i + 1] - offset[i]] + ) if padding_trainable: - sub_w = padding_data[j:j + pad_size, :] - col[offset[i]:offset[i] + pad_size, - j * M:(j + 1) * M] = sub_w + sub_w = padding_data[j : j + pad_size, :] + col[ + offset[i] : offset[i] + pad_size, j * M : (j + 1) * M + ] = sub_w out_begin = offset[i] + pad_size in_begin = offset[i] if in_end > offset[i + 1]: pad_size = np.min( - [in_end - offset[i + 1], offset[i + 1] - offset[i]]) + [in_end - offset[i + 1], offset[i + 1] - offset[i]] + ) if padding_trainable: - sub_w = padding_data[begin_pad + context_start + j - - pad_size:begin_pad + context_start + - j, :] - col[offset[i + 1] - pad_size:offset[i + 1], - j * M:(j + 1) * M] = sub_w + sub_w = padding_data[ + begin_pad + + context_start + + j + - pad_size : begin_pad + + context_start + + j, + :, + ] + col[ + offset[i + 1] - pad_size : offset[i + 1], + j * M : (j + 1) * M, + ] = sub_w in_end = offset[i + 1] out_end = offset[i + 1] - pad_size if in_end <= in_begin: continue in_sub = x[in_begin:in_end, :] - col[out_begin:out_end, j * M:(j + 1) * M] += in_sub + col[out_begin:out_end, j * M : (j + 1) * M] += in_sub return np.dot(col, filter) class XPUTestSequenceConv(XPUOpTestWrapper): - def __init__(self): self.op_name = 'sequence_conv' class TestSeqProject(XPUOpTest): - def setUp(self): self.init_test_case() self.op_type = 'sequence_conv' self.dtype = self.in_type self.use_xpu = True - if self.context_length == 1 \ - and self.context_start == 0 \ - and self.padding_trainable: - print("If context_start is 0 " \ - "and context_length is 1," \ - " padding_trainable should be false.") + if ( + self.context_length == 1 + and self.context_start == 0 + and self.padding_trainable + ): + print( + "If context_start is 0 " + "and context_length is 1," + " padding_trainable should be false." + ) return # one level, batch size x = np.random.uniform( - -6.10907e-05, 0.000104218, - [self.input_size[0], self.input_size[1]]).astype(self.dtype) - w = np.random.uniform(-3.17068e-05, 0.000159822, [ - self.context_length * self.input_size[1], - self.output_represention - ]).astype(self.dtype) + -6.10907e-05, + 0.000104218, + [self.input_size[0], self.input_size[1]], + ).astype(self.dtype) + w = np.random.uniform( + -3.17068e-05, + 0.000159822, + [ + self.context_length * self.input_size[1], + self.output_represention, + ], + ).astype(self.dtype) begin_pad = np.max([0, -self.context_start]) end_pad = np.max([0, self.context_start + self.context_length - 1]) total_pad = begin_pad + end_pad padding_data = np.random.uniform( - 0, 0, [total_pad, self.input_size[1]]).astype(self.dtype) + 0, 0, [total_pad, self.input_size[1]] + ).astype(self.dtype) self.pad_data = padding_data self.inputs = { 'X': (x, self.lod), @@ -128,11 +150,17 @@ def setUp(self): 'contextStart': self.context_start, 'contextLength': self.context_length, 'paddingTrainable': self.padding_trainable, - 'contextStride': self.context_stride + 'contextStride': self.context_stride, } - out = seqconv(x, self.lod, w, self.context_length, - self.context_start, self.padding_trainable, - self.pad_data) + out = seqconv( + x, + self.lod, + w, + self.context_length, + self.context_start, + self.padding_trainable, + self.pad_data, + ) self.outputs = {'Out': out} def test_check_output(self): @@ -144,32 +172,32 @@ def test_check_grad_input(self): def test_check_grad_padding_data(self): if self.padding_trainable: - self.check_grad(['PaddingData'], - 'Out', - no_grad_set=set(['X', 'Filter'])) + self.check_grad( + ['PaddingData'], 'Out', no_grad_set=set(['X', 'Filter']) + ) def test_check_grad_Filter(self): - self.check_grad(['Filter'], - 'Out', - no_grad_set=set(self.inputs_val_no_f)) + self.check_grad( + ['Filter'], 'Out', no_grad_set=set(self.inputs_val_no_f) + ) def test_check_grad_input_filter(self): if self.padding_trainable: - self.check_grad(['X', 'Filter'], - 'Out', - no_grad_set=set(['PaddingData'])) + self.check_grad( + ['X', 'Filter'], 'Out', no_grad_set=set(['PaddingData']) + ) def test_check_grad_padding_input(self): if self.padding_trainable: - self.check_grad(self.inputs_val_no_f, - 'Out', - no_grad_set=set(['Filter'])) + self.check_grad( + self.inputs_val_no_f, 'Out', no_grad_set=set(['Filter']) + ) def test_check_grad_padding_filter(self): if self.padding_trainable: - self.check_grad(self.inputs_val_no_x, - 'Out', - no_grad_set=set(['X'])) + self.check_grad( + self.inputs_val_no_x, 'Out', no_grad_set=set(['X']) + ) def init_test_case(self): self.input_row = 7 @@ -188,7 +216,6 @@ def init_test_case(self): self.output_represention = 8 # output feature size class TestSeqProjectCase1(TestSeqProject): - def init_test_case(self): self.input_row = 11 self.context_start = -2 @@ -205,7 +232,6 @@ def init_test_case(self): self.output_represention = 8 # output feature size class TestSeqProjectCase2Len0(TestSeqProject): - def init_test_case(self): self.input_row = 11 self.context_start = -2 @@ -222,7 +248,6 @@ def init_test_case(self): self.output_represention = 8 # output feature size class TestSeqProjectCase3(TestSeqProject): - def init_test_case(self): self.input_row = 25 self.context_start = -2 @@ -233,8 +258,11 @@ def init_test_case(self): self.input_size = [self.input_row, 25] idx = list(range(self.input_size[0])) del idx[0] - offset_lod = [[0] + np.sort(random.sample(idx, 8)).tolist() + - [self.input_size[0]]] + offset_lod = [ + [0] + + np.sort(random.sample(idx, 8)).tolist() + + [self.input_size[0]] + ] self.lod = [[]] # convert from offset-based lod to length-based lod for i in range(len(offset_lod[0]) - 1): @@ -242,7 +270,6 @@ def init_test_case(self): self.output_represention = 8 # output feature size class TestSeqProjectCase4(TestSeqProject): - def init_test_case(self): self.input_row = 7835 self.input_col = 128 @@ -252,21 +279,139 @@ def init_test_case(self): self.context_stride = 1 self.input_size = [self.input_row, self.input_col] - offset_lod = [[ - 0, 1, 2, 3, 131, 241, 242, 263, 264, 265, 266, 267, 268, 387, - 515, 516, 644, 645, 772, 794, 922, 923, 924, 944, 945, 1073, - 1074, 1202, 1330, 1458, 1556, 1557, 1558, 1686, 1748, 1876, - 1912, 1913, 1914, 2032, 2066, 2194, 2308, 2309, 2347, 2475, - 2476, 2477, 2478, 2606, 2607, 2735, 2736, 2737, 2738, 2838, - 2966, 2967, 2968, 2969, 3097, 3225, 3353, 3481, 3482, 3520, - 3642, 3643, 3754, 3882, 3883, 4010, 4011, 4012, 4140, 4219, - 4228, 4356, 4357, 4415, 4475, 4476, 4604, 4605, 4606, 4694, - 4695, 4808, 4936, 4961, 4962, 5004, 5132, 5260, 5312, 5440, - 5441, 5569, 5570, 5675, 5676, 5750, 5810, 5811, 5939, 6021, - 6149, 6277, 6278, 6364, 6425, 6519, 6647, 6648, 6739, 6867, - 6995, 6996, 7120, 7223, 7244, 7367, 7407, 7408, 7467, 7595, - 7699, 7827, 7835 - ]] + offset_lod = [ + [ + 0, + 1, + 2, + 3, + 131, + 241, + 242, + 263, + 264, + 265, + 266, + 267, + 268, + 387, + 515, + 516, + 644, + 645, + 772, + 794, + 922, + 923, + 924, + 944, + 945, + 1073, + 1074, + 1202, + 1330, + 1458, + 1556, + 1557, + 1558, + 1686, + 1748, + 1876, + 1912, + 1913, + 1914, + 2032, + 2066, + 2194, + 2308, + 2309, + 2347, + 2475, + 2476, + 2477, + 2478, + 2606, + 2607, + 2735, + 2736, + 2737, + 2738, + 2838, + 2966, + 2967, + 2968, + 2969, + 3097, + 3225, + 3353, + 3481, + 3482, + 3520, + 3642, + 3643, + 3754, + 3882, + 3883, + 4010, + 4011, + 4012, + 4140, + 4219, + 4228, + 4356, + 4357, + 4415, + 4475, + 4476, + 4604, + 4605, + 4606, + 4694, + 4695, + 4808, + 4936, + 4961, + 4962, + 5004, + 5132, + 5260, + 5312, + 5440, + 5441, + 5569, + 5570, + 5675, + 5676, + 5750, + 5810, + 5811, + 5939, + 6021, + 6149, + 6277, + 6278, + 6364, + 6425, + 6519, + 6647, + 6648, + 6739, + 6867, + 6995, + 6996, + 7120, + 7223, + 7244, + 7367, + 7407, + 7408, + 7467, + 7595, + 7699, + 7827, + 7835, + ] + ] self.lod = [[]] # convert from offset-based lod to length-based lod for i in range(len(offset_lod[0]) - 1): @@ -280,19 +425,18 @@ def init_test_case(self): class TestSeqConvApi(unittest.TestCase): - def test_api(self): import paddle.fluid as fluid x = fluid.layers.data('x', shape=[32], lod_level=1) - y = fluid.layers.sequence_conv(input=x, - num_filters=2, - filter_size=3, - padding_start=None) + y = fluid.layers.sequence_conv( + input=x, num_filters=2, filter_size=3, padding_start=None + ) place = fluid.CPUPlace() x_tensor = fluid.create_lod_tensor( - np.random.rand(10, 32).astype("float32"), [[2, 3, 1, 4]], place) + np.random.rand(10, 32).astype("float32"), [[2, 3, 1, 4]], place + ) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) ret = exe.run(feed={'x': x_tensor}, fetch_list=[y], return_numpy=False) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_sequence_unpad_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_sequence_unpad_op_xpu.py index 10979e21c62f7..d24c98ad6c956 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_sequence_unpad_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_sequence_unpad_op_xpu.py @@ -23,19 +23,21 @@ import unittest import numpy as np from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestSequenceUnpadOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'sequence_unpad' self.use_dynamic_create_class = False class TestSequenceUnpadOp(XPUOpTest): - def setUp(self): self.init_dtype() self.initTestCase() @@ -63,49 +65,44 @@ def compute(self): x = np.random.random(self.x_shape).astype(self.dtype) out_lod = [self.length] - out = x[0, 0:self.length[0]] + out = x[0, 0 : self.length[0]] for i in range(1, x.shape[0]): - out = np.append(out, x[i, 0:self.length[i]], axis=0) + out = np.append(out, x[i, 0 : self.length[i]], axis=0) - out_shape = (sum(self.length), ) + out_shape = (sum(self.length),) if len(self.x_shape) == 2: - out_shape = out_shape + (1, ) + out_shape = out_shape + (1,) else: out_shape = out_shape + self.x_shape[2:] self.inputs = { 'X': x, - 'Length': np.array(self.length).astype('int64') + 'Length': np.array(self.length).astype('int64'), } self.outputs = {'Out': (out.reshape(out_shape), out_lod)} class TestSequenceUnpadOp2(TestSequenceUnpadOp): - def initTestCase(self): self.length = [2, 3, 4] self.x_shape = (3, 5, 4, 3) class TestSequenceUnpadOp3(TestSequenceUnpadOp): - def initTestCase(self): self.length = [5, 2, 3, 4] self.x_shape = (4, 5, 3, 3, 6) class TestSequenceUnpadOp4(TestSequenceUnpadOp): - def initTestCase(self): self.length = [5, 5, 5, 5] self.x_shape = (4, 5, 3, 3, 6) class TestSequenceUnpadOp5(TestSequenceUnpadOp): - def initTestCase(self): self.length = [1, 4, 3, 1] self.x_shape = (4, 5, 3, 3, 6) class TestSequenceUnpadOpError(unittest.TestCase): - def test_error(self): """ The type of 'x' in fluid.layers.sequence_unpad must be , but received . diff --git a/python/paddle/fluid/tests/unittests/xpu/test_sgd_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_sgd_op_xpu.py index 07f5e524ccf1f..2c7bb9414104b 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_sgd_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_sgd_op_xpu.py @@ -21,17 +21,19 @@ import paddle.fluid as fluid from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) class XPUTestSgdOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'sgd' self.use_dynamic_create_class = False class TestSGDOp(XPUOpTest): - def setUp(self): self.op_type = "sgd" self.dtype = self.in_type @@ -51,7 +53,6 @@ def test_check_output_with_place(self): self.check_output_with_place(paddle.XPUPlace(0)) class TestSGDOpCase8X(TestSGDOp): - def conf(self): self.h = 10 self.w = 64 @@ -63,12 +64,11 @@ def conf(self): class TestSGDOpWithLargeInput(unittest.TestCase): - def runTest(self): data = fluid.layers.fill_constant(shape=[1], value=128, dtype='int64') - label = fluid.layers.fill_constant(shape=[1, 150], - value=0.5, - dtype='float32') + label = fluid.layers.fill_constant( + shape=[1, 150], value=0.5, dtype='float32' + ) emb = fluid.embedding(input=data, size=(10000, 150), dtype='float32') out = fluid.layers.l2_normalize(x=emb, axis=-1) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_shape_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_shape_op_xpu.py index a4128064a7e8f..1a7a51b83149a 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_shape_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_shape_op_xpu.py @@ -18,7 +18,11 @@ sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle from paddle.fluid import core from paddle.fluid.op import Operator @@ -27,13 +31,11 @@ class XPUTestShapeOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "shape" self.use_dynamic_create_class = False class TestShapeOp(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.op_type = "shape" @@ -51,32 +53,26 @@ def test_check_output(self): self.check_output_with_place(place) class TestShapeOp1(TestShapeOp): - def config(self): self.shape = [2] class TestShapeOp2(TestShapeOp): - def config(self): self.shape = [1, 2, 3] class TestShapeOp3(TestShapeOp): - def config(self): self.shape = [1, 2, 3, 4] class TestShapeOp4(TestShapeOp): - def config(self): self.shape = [1, 2, 3, 4, 1024] class TestShapeOp5(TestShapeOp): - def config(self): self.shape = [1, 2, 3, 4, 1, 201] class TestShapeWithSelectedRows(unittest.TestCase): - def setUp(self): self.dtype = self.in_type diff --git a/python/paddle/fluid/tests/unittests/xpu/test_sigmoid_cross_entropy_with_logits_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_sigmoid_cross_entropy_with_logits_op_xpu.py index e9c68dd19997b..6221d4f608fb7 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_sigmoid_cross_entropy_with_logits_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_sigmoid_cross_entropy_with_logits_op_xpu.py @@ -20,7 +20,11 @@ from op_test_xpu import XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) from scipy.special import logit from scipy.special import expit @@ -29,15 +33,13 @@ class XPUTestSigmoidCrossEntropyWithLogitsOp(XPUOpTestWrapper): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def __init__(self): self.op_name = "sigmoid_cross_entropy_with_logits" self.use_dynamic_create_class = False class TestSigmoidCrossEntropyWithLogitsOp(XPUOpTest): - def setUp(self): self.set_xpu() self.op_type = "sigmoid_cross_entropy_with_logits" @@ -60,13 +62,14 @@ def set_inputs(self): batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.randint(0, 2, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.randint( + 0, 2, (batch_size, num_classes) + ).astype(self.dtype), } self.attrs = {'num_classes': num_classes, 'batch_size': batch_size} @@ -85,9 +88,9 @@ def init_dtype(self): self.dtype = self.in_type class TestSigmoidCrossEntropyWithLogitsOp2( - TestSigmoidCrossEntropyWithLogitsOp): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + TestSigmoidCrossEntropyWithLogitsOp + ): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def set_inputs(self): batch_size = 64 @@ -95,13 +98,14 @@ def set_inputs(self): ignore_index = -1 self.ignore_index = ignore_index self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.randint(-1, 2, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.randint( + -1, 2, (batch_size, num_classes) + ).astype(self.dtype), } self.attrs = {'ignore_index': ignore_index} @@ -117,21 +121,22 @@ def set_output(self): self.outputs = {'Out': out} class TestSigmoidCrossEntropyWithLogitsOp3( - TestSigmoidCrossEntropyWithLogitsOp): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + TestSigmoidCrossEntropyWithLogitsOp + ): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def set_inputs(self): batch_size = 64 num_classes = 20 self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.uniform(0, 1, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.uniform( + 0, 1, (batch_size, num_classes) + ).astype(self.dtype), } self.attrs = {'num_classes': num_classes, 'batch_size': batch_size} @@ -145,9 +150,9 @@ def set_output(self): self.outputs = {'Out': -term1 - term2} class TestSigmoidCrossEntropyWithLogitsOp4( - TestSigmoidCrossEntropyWithLogitsOp): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + TestSigmoidCrossEntropyWithLogitsOp + ): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def set_inputs(self): batch_size = 64 @@ -155,13 +160,14 @@ def set_inputs(self): ignore_index = -1 self.ignore_index = ignore_index self.inputs = { - 'X': - logit( + 'X': logit( np.random.uniform(0, 1, (batch_size, num_classes)).astype( - self.dtype)), - 'Label': - np.random.randint(-1, 2, - (batch_size, num_classes)).astype(self.dtype) + self.dtype + ) + ), + 'Label': np.random.randint( + -1, 2, (batch_size, num_classes) + ).astype(self.dtype), } self.attrs = {'ignore_index': ignore_index, 'normalize': True} @@ -176,26 +182,27 @@ def set_output(self): out[np.where(self.inputs['Label'] == self.ignore_index)] = 0 if self.attrs['normalize']: out = out / float( - np.where(self.inputs['Label'] != self.ignore_index)[0].size) + np.where(self.inputs['Label'] != self.ignore_index)[0].size + ) self.outputs = {'Out': out} class TestSigmoidCrossEntropyWithLogitsOp5( - TestSigmoidCrossEntropyWithLogitsOp): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + TestSigmoidCrossEntropyWithLogitsOp + ): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def set_inputs(self): batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + [num_classes])).astype( - self.dtype)), - 'Label': - np.random.uniform(0, 1, tuple(batch_size + - [num_classes])).astype(self.dtype) + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype) + ), + 'Label': np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype), } self.attrs = {'num_classes': num_classes, 'batch_size': batch_size} @@ -209,22 +216,22 @@ def set_output(self): self.outputs = {'Out': -term1 - term2} class TestSigmoidCrossEntropyWithLogitsOp6( - TestSigmoidCrossEntropyWithLogitsOp): - """Test sigmoid_cross_entropy_with_logit_op with binary label - """ + TestSigmoidCrossEntropyWithLogitsOp + ): + """Test sigmoid_cross_entropy_with_logit_op with binary label""" def set_inputs(self): batch_size = [10, 10] num_classes = 20 self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + [num_classes])).astype( - self.dtype)), - 'Label': - np.random.randint(0, 2, tuple(batch_size + - [num_classes])).astype(self.dtype) + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype) + ), + 'Label': np.random.randint( + 0, 2, tuple(batch_size + [num_classes]) + ).astype(self.dtype), } self.attrs = {'num_classes': num_classes, 'batch_size': batch_size} @@ -238,9 +245,9 @@ def set_output(self): self.outputs = {'Out': -term1 - term2} class TestSigmoidCrossEntropyWithLogitsNorm( - TestSigmoidCrossEntropyWithLogitsOp): - """Test sigmoid_cross_entropy_with_logit_op with probabalistic label - """ + TestSigmoidCrossEntropyWithLogitsOp + ): + """Test sigmoid_cross_entropy_with_logit_op with probabalistic label""" def set_inputs(self): batch_size = [10, 10] @@ -248,14 +255,14 @@ def set_inputs(self): ignore_index = -1 self.ignore_index = ignore_index self.inputs = { - 'X': - logit( - np.random.uniform(0, 1, - tuple(batch_size + [num_classes])).astype( - self.dtype)), - 'Label': - np.random.randint( - -1, 2, tuple(batch_size + [num_classes])).astype(self.dtype) + 'X': logit( + np.random.uniform( + 0, 1, tuple(batch_size + [num_classes]) + ).astype(self.dtype) + ), + 'Label': np.random.randint( + -1, 2, tuple(batch_size + [num_classes]) + ).astype(self.dtype), } self.attrs = {'ignore_index': ignore_index, 'normalize': True} @@ -270,7 +277,8 @@ def set_output(self): out[np.where(self.inputs['Label'] == self.ignore_index)] = 0 if self.attrs['normalize']: out = out / float( - np.where(self.inputs['Label'] != self.ignore_index)[0].size) + np.where(self.inputs['Label'] != self.ignore_index)[0].size + ) self.outputs = {'Out': out} diff --git a/python/paddle/fluid/tests/unittests/xpu/test_sign_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_sign_op_xpu.py index 9762529283177..b498c5fc3a17e 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_sign_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_sign_op_xpu.py @@ -21,19 +21,21 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestSignOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'sign' self.use_dynamic_create_class = False class TestSignOPBase(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -43,8 +45,9 @@ def set_case(self): self.op_type = 'sign' self.dtype = self.in_type self.init_config() - self.x = np.random.uniform(-10, 10, - self.input_shape).astype(self.dtype) + self.x = np.random.uniform(-10, 10, self.input_shape).astype( + self.dtype + ) self.inputs = {'X': self.x} self.outputs = {'Out': np.sign(self.x)} self.attrs = {'use_xpu': True} @@ -62,22 +65,18 @@ def init_config(self): self.input_shape = [864] class XPUTestSign1(TestSignOPBase): - def init_config(self): self.input_shape = [2, 768] class XPUTestSign2(TestSignOPBase): - def init_config(self): self.input_shape = [3, 8, 4096] class XPUTestSign3(TestSignOPBase): - def init_config(self): self.input_shape = [1024] class XPUTestSign4(TestSignOPBase): - def init_config(self): self.input_shape = [2, 2, 255] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_slice_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_slice_op_xpu.py index c22bb99b4a9c8..5a77d9cb51ab8 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_slice_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_slice_op_xpu.py @@ -19,7 +19,11 @@ sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -27,13 +31,11 @@ # Situation 1: starts(list, no tensor), ends(list, no tensor) # 1.1 without attr(decrease) class XPUTestSliceOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'slice' self.use_dynamic_create_class = False class TestSliceOp(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -46,7 +48,7 @@ def setUp(self): 'starts': self.starts, 'ends': self.ends, 'infer_flags': self.infer_flags, - "use_xpu": True + "use_xpu": True, } def config(self): @@ -62,14 +64,16 @@ def test_check_grad_normal(self): self.check_grad_with_place(self.place, ['Input'], 'Out') else: user_defined_grad_outputs = np.random.random( - self.out.shape).astype(self.dtype) + self.out.shape + ).astype(self.dtype) self.check_grad_with_place( - self.place, ['Input'], + self.place, + ['Input'], 'Out', - user_defined_grad_outputs=user_defined_grad_outputs) + user_defined_grad_outputs=user_defined_grad_outputs, + ) class TestCase1(TestSliceOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [-3, 0, 2] @@ -79,7 +83,6 @@ def config(self): self.out = self.input[-3:3, 0:100, 2:-1, :] class TestCase2(TestSliceOp): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [-3, 0, 2] @@ -91,13 +94,11 @@ def config(self): # 1.2 with attr(decrease) class XPUTestSliceOp_decs_dim(XPUOpTestWrapper): - def __init__(self): self.op_name = 'slice' self.use_dynamic_create_class = False class TestSliceOp_decs_dim(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -111,7 +112,7 @@ def setUp(self): 'ends': self.ends, 'infer_flags': self.infer_flags, 'decrease_axis': self.decrease_axis, - "use_xpu": True + "use_xpu": True, } def config(self): @@ -131,14 +132,16 @@ def test_check_grad_normal(self): self.check_grad_with_place(self.place, ['Input'], 'Out') else: user_defined_grad_outputs = np.random.random( - self.out.shape).astype(self.dtype) + self.out.shape + ).astype(self.dtype) self.check_grad_with_place( - self.place, ['Input'], + self.place, + ['Input'], 'Out', - user_defined_grad_outputs=user_defined_grad_outputs) + user_defined_grad_outputs=user_defined_grad_outputs, + ) class TestSliceOp_decs_dim_2(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [1, 0, 2] @@ -149,7 +152,6 @@ def config(self): self.out = self.input[1, 0, 2:4, :] class TestSliceOp_decs_dim_3(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [-1, 0, 2] @@ -160,7 +162,6 @@ def config(self): self.out = self.input[-1, 0, 2:4, :] class TestSliceOp_decs_dim_4(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 7]).astype(self.dtype) self.starts = [0, 1, 2, 3] @@ -171,7 +172,6 @@ def config(self): self.out = self.input[0, 1, 2, 3:4] class TestSliceOp_decs_dim_5(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [-1] @@ -182,7 +182,6 @@ def config(self): self.out = self.input[:, :, :, -1] class TestSliceOp_decs_dim_6(TestSliceOp_decs_dim): - def config(self): self.input = np.random.random([3, 4, 5, 6]).astype(self.dtype) self.starts = [0, 1, 2, 3] diff --git a/python/paddle/fluid/tests/unittests/xpu/test_softmax_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_softmax_op_xpu.py index a4997c91ffbf5..736b3b7fbe68a 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_softmax_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_softmax_op_xpu.py @@ -19,7 +19,11 @@ sys.path.append("..") from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() np.random.seed(10) @@ -29,7 +33,7 @@ def stable_softmax(x): """Compute the softmax of vector x in a numerically stable way.""" # clip to shiftx, otherwise, when calc loss with # log(exp(shiftx)), may get log(0)=INF - shiftx = (x - np.max(x)).clip(-64.) + shiftx = (x - np.max(x)).clip(-64.0) exps = np.exp(shiftx) return exps / np.sum(exps) @@ -44,7 +48,6 @@ def ref_softmax(x, axis=None, dtype=None): class XPUTestSoftmaxOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'softmax' self.use_dynamic_create_class = True @@ -56,14 +59,12 @@ def dynamic_create_class(self): axis = [-1, 0, 1] for shape in shapes: for axi in axis: - class_name = 'XPUTestSoftmax_' + \ - str(shape) + "_" + str(axi) + class_name = 'XPUTestSoftmax_' + str(shape) + "_" + str(axi) attr_dict = {'shape': shape, 'axis': axi} classes.append([class_name, attr_dict]) return base_class, classes class TestSoftmaxOp(XPUOpTest): - def setUp(self): self.op_type = "softmax" if not hasattr(self, 'shape'): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_softmax_with_cross_entropy_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_softmax_with_cross_entropy_op_xpu.py index cb604af4f89a4..f4482e5edd716 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_softmax_with_cross_entropy_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_softmax_with_cross_entropy_op_xpu.py @@ -22,7 +22,11 @@ import unittest import numpy as np -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) def cross_entropy(softmax, label, soft_label, axis, ignore_index=-1): @@ -33,7 +37,7 @@ def cross_entropy(softmax, label, soft_label, axis, ignore_index=-1): axis %= len(shape) n = int(np.prod(shape[:axis])) axis_dim = shape[axis] - remain = int(np.prod(shape[axis + 1:])) + remain = int(np.prod(shape[axis + 1 :])) softmax_reshape = softmax.reshape((n, axis_dim, remain)) label_reshape = label.reshape((n, 1, remain)) result = np.zeros_like(label_reshape, dtype=softmax.dtype) @@ -46,7 +50,6 @@ def cross_entropy(softmax, label, soft_label, axis, ignore_index=-1): class XPUTestSoftmaxWithCrossEntropyOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'softmax_with_cross_entropy' self.use_dynamic_create_class = True @@ -54,26 +57,39 @@ def __init__(self): def dynamic_create_class(self): base_class = self.TestSoftmaxWithCrossEntropyOp classes = [] - shapes = [[41, 37], [3, 5, 7, 11], [3, 5, 7, 1], [1023, 38512], - [1, 511]] + shapes = [ + [41, 37], + [3, 5, 7, 11], + [3, 5, 7, 1], + [1023, 38512], + [1, 511], + ] for soft_label in [True, False]: for numeric_stable_mode in [True, False]: for shape in shapes: for logits_type in [0, 1, 2]: for axis in range(len(shape)): - if (not numeric_stable_mode): + if not numeric_stable_mode: axis = -1 - class_name = 'XPUTestSoftmaxWithCrossEntropy_' + \ - str(soft_label) + "_" + \ - str(numeric_stable_mode) + "_" + \ - str(shape) + "_" + \ - str(logits_type) + "_" + \ - str(axis) - attr_dict = {'soft_label': soft_label, \ - 'numeric_stable_mode': numeric_stable_mode, \ - 'shape': shape, \ - 'logits_type': logits_type, - 'axis': axis} + class_name = ( + 'XPUTestSoftmaxWithCrossEntropy_' + + str(soft_label) + + "_" + + str(numeric_stable_mode) + + "_" + + str(shape) + + "_" + + str(logits_type) + + "_" + + str(axis) + ) + attr_dict = { + 'soft_label': soft_label, + 'numeric_stable_mode': numeric_stable_mode, + 'shape': shape, + 'logits_type': logits_type, + 'axis': axis, + } classes.append([class_name, attr_dict]) return base_class, classes @@ -95,8 +111,10 @@ def setUp(self): self.soft_label = True self.axis = -1 logits = getattr( - self, "logits", - np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype)) + self, + "logits", + np.random.uniform(0.1, 1.0, self.shape).astype(self.dtype), + ) if self.logits_type == 1: self.logits = np.full(self.shape, -500.0).astype(self.dtype) elif self.logits_type == 2 and len(self.shape) == 4: @@ -105,24 +123,25 @@ def setUp(self): softmax = np.apply_along_axis(stable_softmax, self.axis, logits) if self.soft_label: - labels = np.random.uniform(0.1, 1.0, - self.shape).astype(self.dtype) + labels = np.random.uniform(0.1, 1.0, self.shape).astype( + self.dtype + ) labels /= np.sum(labels, axis=self.axis, keepdims=True) else: axis_dim = self.shape[self.axis] self.shape[self.axis] = 1 - labels = np.random.randint(0, - axis_dim, - self.shape, - dtype="int64") + labels = np.random.randint( + 0, axis_dim, self.shape, dtype="int64" + ) - loss = cross_entropy(softmax, labels, self.soft_label, self.axis, - self.ignore_index) + loss = cross_entropy( + softmax, labels, self.soft_label, self.axis, self.ignore_index + ) self.inputs = {"Logits": logits, "Label": labels} self.outputs = { "Softmax": softmax.astype(self.dtype), - "Loss": loss.astype(self.dtype) + "Loss": loss.astype(self.dtype), } self.attrs = { "numeric_stable_mode": self.numeric_stable_mode, @@ -143,9 +162,9 @@ def test_check_grad(self): if paddle.is_compiled_with_xpu(): paddle.enable_static() place = paddle.XPUPlace(0) - self.check_grad_with_place(place, ["Logits"], - "Loss", - max_relative_error=0.2) + self.check_grad_with_place( + place, ["Logits"], "Loss", max_relative_error=0.2 + ) support_types = get_xpu_op_support_types('softmax_with_cross_entropy') diff --git a/python/paddle/fluid/tests/unittests/xpu/test_split_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_split_op_xpu.py index 3d6ab9e04811e..c7dff56cb621a 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_split_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_split_op_xpu.py @@ -20,20 +20,22 @@ from op_test_xpu import XPUOpTest import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestSplitOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'split' self.use_dynamic_create_class = False # test with attr(num) class TestSplitOp(XPUOpTest): - def setUp(self): self.init_dtype() self.__class__.use_xpu = True @@ -44,12 +46,13 @@ def setUp(self): self.attrs = { 'axis': self.axis, 'sections': self.sections, - 'num': self.num + 'num': self.num, } out = np.split(self.x, self.indices_or_sections, self.axis) - self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in range(len(out))]} + self.outputs = { + 'Out': [('out%d' % i, out[i]) for i in range(len(out))] + } def init_dtype(self): self.dtype = self.in_type @@ -66,7 +69,6 @@ def test_check_output(self): # unknown sections class TestSplitOp1(TestSplitOp): - def initParameters(self): self.x = np.random.random((4, 5, 6)).astype(self.dtype) self.axis = 2 @@ -76,7 +78,6 @@ def initParameters(self): # test with int32 class TestSplitOp2(TestSplitOp): - def initParameters(self): self.x = np.random.random((4, 5, 6)).astype(np.int32) self.axis = 2 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_squeeze2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_squeeze2_op_xpu.py index fdd81c0c1384d..1b2a52a6962f1 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_squeeze2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_squeeze2_op_xpu.py @@ -20,20 +20,22 @@ import numpy as np from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) import paddle paddle.enable_static() class XPUTestSqueeze2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = "squeeze2" self.use_dynamic_create_class = False class TestSqueeze2Op(XPUOpTest): - def setUp(self): self.op_type = "squeeze2" self.__class__.op_type = "squeeze2" @@ -45,7 +47,7 @@ def setUp(self): } self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype(self.dtype) + "XShape": np.random.random(self.ori_shape).astype(self.dtype), } self.init_attrs() @@ -73,7 +75,6 @@ def test_check_grad(self): # Correct: There is mins axis. class TestSqueeze2Op1(TestSqueeze2Op): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = (0, -2) @@ -81,7 +82,6 @@ def init_test_case(self): # Correct: No axes input. class TestSqueeze2Op2(TestSqueeze2Op): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -89,7 +89,6 @@ def init_test_case(self): # Correct: Just part of axes be squeezed. class TestSqueeze2Op3(TestSqueeze2Op): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_squeeze_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_squeeze_op_xpu.py index f1c6873823f1b..a87fe32d62e0d 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_squeeze_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_squeeze_op_xpu.py @@ -23,20 +23,22 @@ import paddle.fluid as fluid from paddle.fluid import Program, program_guard from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestSqueezeOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "squeeze" self.use_dynamic_create_class = False # Correct: General. class TestSqueezeOp(XPUOpTest): - def setUp(self): self.op_type = "squeeze" self.__class__.op_type = "squeeze" @@ -75,7 +77,6 @@ def init_attrs(self): # Correct: There is mins axis. class TestSqueezeOp1(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 3, 1, 40) self.axes = (0, -2) @@ -83,7 +84,6 @@ def init_test_case(self): # Correct: No axes input. class TestSqueezeOp2(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (1, 20, 1, 5) self.axes = () @@ -91,7 +91,6 @@ def init_test_case(self): # Correct: Just part of axes be squeezed. class TestSqueezeOp3(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, -1) @@ -99,7 +98,6 @@ def init_test_case(self): # Correct: The demension of axis is not of size 1 remains unchanged. class TestSqueezeOp4(TestSqueezeOp): - def init_test_case(self): self.ori_shape = (6, 1, 5, 1, 4, 1) self.axes = (1, 2) @@ -107,13 +105,13 @@ def init_test_case(self): class TestSqueezeOpError(unittest.TestCase): - def test_errors(self): paddle.enable_static() with program_guard(Program(), Program()): # The input type of softmax_op must be Variable. - x1 = fluid.create_lod_tensor(np.array([[-1]]), [[1]], - paddle.XPUPlace(0)) + x1 = fluid.create_lod_tensor( + np.array([[-1]]), [[1]], paddle.XPUPlace(0) + ) self.assertRaises(TypeError, paddle.squeeze, x1) # The input axes of squeeze must be list. x2 = paddle.static.data(name='x2', shape=[4], dtype="int32") diff --git a/python/paddle/fluid/tests/unittests/xpu/test_stack_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_stack_op_xpu.py index 2102348de36e4..1f215acdb7437 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_stack_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_stack_op_xpu.py @@ -21,20 +21,22 @@ from op_test_xpu import XPUOpTest import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestStackOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'stack' self.use_dynamic_create_class = False @skip_check_grad_ci(reason="There is no grad kernel for stack_xpu op.") class TestStackOp(XPUOpTest): - def initDefaultParameters(self): self.num_inputs = 4 self.input_dim = (5, 6, 7) @@ -49,7 +51,8 @@ def setUp(self): self.x = [] for i in range(self.num_inputs): self.x.append( - np.random.random(size=self.input_dim).astype(self.dtype)) + np.random.random(size=self.input_dim).astype(self.dtype) + ) tmp = [] x_names = self.get_x_names() @@ -79,21 +82,19 @@ def test_check_grad(self): if self.dtype == np.int32 or self.dtype == np.int64: pass else: - self.check_grad_with_place(paddle.XPUPlace(0), - self.get_x_names(), 'Y') + self.check_grad_with_place( + paddle.XPUPlace(0), self.get_x_names(), 'Y' + ) class TestStackOp1(TestStackOp): - def initParameters(self): self.num_inputs = 16 class TestStackOp2(TestStackOp): - def initParameters(self): self.num_inputs = 30 class TestStackOp3(TestStackOp): - def initParameters(self): self.axis = -1 @@ -101,7 +102,6 @@ def test_check_grad(self): pass class TestStackOp4(TestStackOp): - def initParameters(self): self.axis = -4 @@ -109,17 +109,14 @@ def test_check_grad(self): pass class TestStackOp5(TestStackOp): - def initParameters(self): self.axis = 1 class TestStackOp6(TestStackOp): - def initParameters(self): self.axis = 3 class TestStackOp7(TestStackOp): - def initParameters(self): self.num_inputs = 4 self.input_dim = (5, 6, 7) @@ -130,7 +127,6 @@ def test_check_grad(self): pass class TestStackOp8(TestStackOp): - def initParameters(self): self.num_inputs = 4 self.input_dim = (5, 6, 7) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_sum_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_sum_op_xpu.py index 3d74b224cbbea..778cb66b9a84b 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_sum_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_sum_op_xpu.py @@ -22,19 +22,21 @@ import paddle.fluid as fluid import paddle.fluid.core as core from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestSumOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'sum' self.use_dynamic_create_class = False class TestSumOp(XPUOpTest): - def setUp(self): self.init_dtype() self.set_xpu() @@ -66,30 +68,24 @@ def test_check_grad(self): self.check_grad_with_place(self.place, ['x0'], 'Out') class TestSumOp1(TestSumOp): - def set_shape(self): - self.shape = (5) + self.shape = 5 class TestSumOp2(TestSumOp): - def set_shape(self): self.shape = (1, 1, 1, 1, 1) class TestSumOp3(TestSumOp): - def set_shape(self): self.shape = (10, 5, 7) class TestSumOp4(TestSumOp): - def set_shape(self): self.shape = (2, 2, 3, 3) def create_test_sum_fp16_class(parent): - class TestSumFp16Case(parent): - def init_kernel_type(self): self.dtype = np.float16 @@ -105,15 +101,14 @@ def test_w_is_selected_rows(self): class API_Test_Add_n(unittest.TestCase): - def test_api(self): with fluid.program_guard(fluid.Program(), fluid.Program()): - input0 = fluid.layers.fill_constant(shape=[2, 3], - dtype='int64', - value=5) - input1 = fluid.layers.fill_constant(shape=[2, 3], - dtype='int64', - value=3) + input0 = fluid.layers.fill_constant( + shape=[2, 3], dtype='int64', value=5 + ) + input1 = fluid.layers.fill_constant( + shape=[2, 3], dtype='int64', value=3 + ) expected_result = np.empty((2, 3)) expected_result.fill(8) sum_value = paddle.add_n([input0, input1]) @@ -132,9 +127,7 @@ def test_api(self): class TestRaiseSumError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.sum([11, 22]) @@ -155,9 +148,7 @@ def test_dtype1(): class TestRaiseSumsError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.sums([11, 22]) @@ -193,9 +184,7 @@ def test_out_dtype(): class TestSumOpError(unittest.TestCase): - def test_errors(self): - def test_empty_list_input(): with fluid.dygraph.guard(): fluid._legacy_C_ops.sum([]) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_temporal_shift_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_temporal_shift_op_xpu.py index 0158284c01975..5fce6052502b1 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_temporal_shift_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_temporal_shift_op_xpu.py @@ -22,7 +22,11 @@ import paddle.nn.functional as F from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() np.random.seed(10) @@ -33,13 +37,14 @@ def temporal_shift(x, seg_num, shift_ratio, data_format): x = np.transpose(x, (0, 3, 1, 2)) shape = x.shape reshape_x = x.reshape((-1, seg_num, shape[1], shape[2], shape[3])) - pad_x = np.pad(reshape_x, ((0, 0), (1, 1), (0, 0), (0, 0), (0, 0)), - 'constant') + pad_x = np.pad( + reshape_x, ((0, 0), (1, 1), (0, 0), (0, 0), (0, 0)), 'constant' + ) c1 = int(shape[1] * shift_ratio) c2 = int(shape[1] * 2 * shift_ratio) slice1 = pad_x[:, :seg_num, :c1, :, :] - slice2 = pad_x[:, 2:seg_num + 2, c1:c2, :, :] - slice3 = pad_x[:, 1:seg_num + 1, c2:, :, :] + slice2 = pad_x[:, 2 : seg_num + 2, c1:c2, :, :] + slice3 = pad_x[:, 1 : seg_num + 1, c2:, :, :] concat_x = np.concatenate([slice1, slice2, slice3], axis=2) out = concat_x.reshape(shape) if data_format == "NHWC": @@ -48,13 +53,11 @@ def temporal_shift(x, seg_num, shift_ratio, data_format): class XPUTestTemporalShiftOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "temporal_shift" self.use_dynamic_create_class = False class TestXPUTemporalShift(XPUOpTest): - def setUp(self): self.initTestCase() self.op_type = 'temporal_shift' @@ -65,15 +68,16 @@ def setUp(self): self.attrs = { "seg_num": self.seg_num, "shift_ratio": self.shift_ratio, - "data_format": self.data_format + "data_format": self.data_format, } self.inputs = { "X": x, } - output = temporal_shift(x, self.seg_num, self.shift_ratio, - self.data_format) + output = temporal_shift( + x, self.seg_num, self.shift_ratio, self.data_format + ) self.outputs = {"Out": output} self.python_out_sig = ["Out"] @@ -91,7 +95,6 @@ def initTestCase(self): self.data_format = 'NCHW' class TestXPUTemporalShift2(TestXPUTemporalShift): - def initTestCase(self): self.x_shape = (1, 1, 1, 1) self.seg_num = 1 @@ -100,7 +103,6 @@ def initTestCase(self): self.data_format = 'NCHW' class TestXPUTemporalShift3(TestXPUTemporalShift): - def initTestCase(self): self.x_shape = (4, 9, 1, 1) self.seg_num = 2 @@ -109,7 +111,6 @@ def initTestCase(self): self.data_format = 'NCHW' class TestXPUTemporalShift4(TestXPUTemporalShift): - def initTestCase(self): self.x_shape = (4, 1, 10, 10) self.seg_num = 2 @@ -118,7 +119,6 @@ def initTestCase(self): self.data_format = 'NCHW' class TestXPUTemporalShift5(TestXPUTemporalShift): - def initTestCase(self): self.x_shape = (1, 1, 1, 1) self.seg_num = 1 @@ -127,7 +127,6 @@ def initTestCase(self): self.data_format = 'NHWC' class TestXPUTemporalShift6(TestXPUTemporalShift): - def initTestCase(self): self.x_shape = (6, 5, 5, 1) self.seg_num = 3 @@ -136,7 +135,6 @@ def initTestCase(self): self.data_format = 'NHWC' class TestXPUTemporalShift7(TestXPUTemporalShift): - def initTestCase(self): self.x_shape = (9, 1, 1, 4) self.seg_num = 3 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_tile_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_tile_op_xpu.py index 1854e164386c0..ef6c65f71dd84 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_tile_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_tile_op_xpu.py @@ -20,21 +20,23 @@ from op_test_xpu import XPUOpTest import paddle import paddle.fluid as fluid -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() np.random.seed(10) -#Situation 1: repeat_times is a list (without tensor) +# Situation 1: repeat_times is a list (without tensor) class XPUTestTileOpRank1(XPUOpTestWrapper): - def __init__(self): self.op_name = 'tile' self.use_dynamic_create_class = False class TestTileOpRank1(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.__class__.no_need_check_grad = True @@ -55,39 +57,33 @@ def init_data(self): def test_check_output(self): self.check_output_with_place(self.place) - #with dimension expanding + # with dimension expanding class TestTileOpRank2Expanding(TestTileOpRank1): - def init_data(self): self.ori_shape = [120] self.repeat_times = [2, 2] class TestTileOpRank2(TestTileOpRank1): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] class TestTileOpRank3_Corner(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (1, 1, 1) class TestTileOpRank3_Corner2(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 10, 5) self.repeat_times = (2, 2) class TestTileOpRank3(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 15) self.repeat_times = (2, 1, 4) class TestTileOpRank4(TestTileOpRank1): - def init_data(self): self.ori_shape = (2, 4, 5, 7) self.repeat_times = (3, 2, 1, 2) @@ -95,13 +91,11 @@ def init_data(self): # Situation 2: repeat_times is a list (with tensor) class XPUTestTileOpRank1_tensor_attr(XPUOpTestWrapper): - def __init__(self): self.op_name = 'tile' self.use_dynamic_create_class = False class TestTileOpRank1_tensor_attr(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.__class__.no_need_check_grad = True @@ -110,8 +104,9 @@ def setUp(self): self.init_data() repeat_times_tensor = [] for index, ele in enumerate(self.repeat_times): - repeat_times_tensor.append(("x" + str(index), np.ones( - (1)).astype('int32') * ele)) + repeat_times_tensor.append( + ("x" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { 'X': np.random.random(self.ori_shape).astype(self.dtype), @@ -130,14 +125,12 @@ def test_check_output(self): self.check_output_with_place(self.place) class TestTileOpRank2_Corner_tensor_attr(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [1, 1] self.infer_repeat_times = [1, -1] class TestTileOpRank2_attr_tensor(TestTileOpRank1_tensor_attr): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -146,13 +139,11 @@ def init_data(self): # Situation 3: repeat_times is a tensor class XPUTestTileOpRank1_tensor(XPUOpTestWrapper): - def __init__(self): self.op_name = 'tile' self.use_dynamic_create_class = False class TestTileOpRank1_tensor(XPUOpTest): - def setUp(self): self.dtype = self.in_type self.__class__.no_need_check_grad = True @@ -176,7 +167,6 @@ def test_check_output(self): self.check_output_with_place(self.place) class TestTileOpRank2_tensor(TestTileOpRank1_tensor): - def init_data(self): self.ori_shape = [12, 14] self.repeat_times = [2, 3] @@ -191,7 +181,6 @@ def init_data(self): # Test python API class TestTileAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(paddle.XPUPlace(0)): np_x = np.random.random([12, 14]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/xpu/test_top_k_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_top_k_op_xpu.py index 5cd42c912677a..fe251b320a652 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_top_k_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_top_k_op_xpu.py @@ -19,7 +19,11 @@ sys.path.append("..") import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -28,23 +32,24 @@ def random_unique_float(row, k, dtype): # create a random float array with 10x length arr = np.random.uniform(-10.0, 10.0, int(row * k * 10)).astype(dtype) arr = np.unique(arr) - assert arr.shape[ - 0] >= row * k, "failed to create enough unique values: %d vs %d" % ( - arr.shape[0], row * k) - arr = arr[:row * k] + assert ( + arr.shape[0] >= row * k + ), "failed to create enough unique values: %d vs %d" % ( + arr.shape[0], + row * k, + ) + arr = arr[: row * k] np.random.shuffle(arr) arr = arr.reshape(row, k) return arr class XPUTestTopkOP(XPUOpTestWrapper): - def __init__(self): self.op_name = 'top_k' self.use_dynamic_create_class = False class TestXPUTopkOP(XPUOpTest): - def setUp(self): self.place = paddle.XPUPlace(0) self.init_dtype() @@ -85,28 +90,24 @@ def test_check_grad(self): self.check_grad_with_place(self.place, ['X'], 'Out') class TestTopk1(TestXPUTopkOP): - def set_case(self): self.variable_k = True self.row = 100 self.top_k = 1 class TestTopk2(TestXPUTopkOP): - def set_case(self): self.variable_k = False self.row = 16 self.top_k = 256 class TestTopk3(TestXPUTopkOP): - def set_case(self): self.variable_k = True self.row = 10 self.top_k = 512 class TestTopk4(TestXPUTopkOP): - def set_case(self): self.variable_k = False self.row = 5 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_top_k_v2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_top_k_v2_op_xpu.py index 4ed7619bf306f..5393834c370f1 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_top_k_v2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_top_k_v2_op_xpu.py @@ -19,7 +19,11 @@ sys.path.append("..") from op_test_xpu import XPUOpTest import paddle -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() @@ -41,13 +45,11 @@ def numpy_topk(x, k=1, axis=-1, largest=True): class XPUTestTopKV2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = 'top_k_v2' self.use_dynamic_create_class = False class TestTopkOp(XPUOpTest): - def init_args(self): self.k = 3 self.axis = 1 @@ -62,12 +64,11 @@ def setUp(self): self.attrs = { 'k': self.k, 'axis': self.axis, - 'largest': self.largest + 'largest': self.largest, } - output, indices = numpy_topk(self.input_data, - axis=self.axis, - k=self.k, - largest=self.largest) + output, indices = numpy_topk( + self.input_data, axis=self.axis, k=self.k, largest=self.largest + ) self.outputs = {'Out': output, 'Indices': indices} def test_check_output(self): @@ -81,7 +82,6 @@ def test_check_grad(self): self.check_grad(set(['X']), 'Out') class TestTopkOp1(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -89,7 +89,6 @@ def init_args(self): self.input_data = np.random.rand(100, 155).astype(self.dtype) class TestTopkOp2(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -97,7 +96,6 @@ def init_args(self): self.input_data = np.random.rand(10, 10, 5).astype(self.dtype) class TestTopkOp3(TestTopkOp): - def init_args(self): self.k = 5 self.axis = 1 @@ -105,7 +103,6 @@ def init_args(self): self.input_data = np.random.rand(10, 10, 5).astype(self.dtype) class TestTopkOp4(TestTopkOp): - def init_args(self): self.k = 1 self.axis = 1 @@ -113,7 +110,6 @@ def init_args(self): self.input_data = np.random.rand(10, 10, 5).astype(self.dtype) class TestTopkOp5(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 2 @@ -121,7 +117,6 @@ def init_args(self): self.input_data = np.random.rand(10, 10, 5).astype(self.dtype) class TestTopkOp6(TestTopkOp): - def init_args(self): self.k = 5 self.axis = 1 @@ -129,7 +124,6 @@ def init_args(self): self.input_data = np.random.rand(8, 32, 64).astype(self.dtype) class TestTopkOp7(TestTopkOp): - def init_args(self): self.k = 10 self.axis = 2 @@ -137,7 +131,6 @@ def init_args(self): self.input_data = np.random.rand(8, 5, 10, 16).astype(self.dtype) class TestTopkOp8(TestTopkOp): - def init_args(self): self.k = 1 self.axis = 1 @@ -145,7 +138,6 @@ def init_args(self): self.input_data = np.random.rand(8, 32, 64).astype(self.dtype) class TestTopkOp9(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -153,7 +145,6 @@ def init_args(self): self.input_data = np.random.rand(10, 10, 5).astype(self.dtype) class TestTopkOp10(TestTopkOp): - def init_args(self): self.k = 3 self.axis = 1 @@ -161,7 +152,6 @@ def init_args(self): self.input_data = np.random.rand(10, 10, 5).astype(self.dtype) class TestTopkOp11(TestTopkOp): - def init_args(self): self.k = 5 self.axis = 1 @@ -169,7 +159,6 @@ def init_args(self): self.input_data = np.random.rand(10, 10, 5).astype(self.dtype) class TestTopkOp12(TestTopkOp): - def init_args(self): self.k = 1 self.axis = 1 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_transpose_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_transpose_op_xpu.py index 3dd7531f35048..1261487d39373 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_transpose_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_transpose_op_xpu.py @@ -22,7 +22,6 @@ class TestXPUTransposeOp(XPUOpTest): - def setUp(self): self.init_op_type() self.initTestCase() @@ -32,11 +31,11 @@ def setUp(self): self.attrs = { 'axis': list(self.axis), 'use_mkldnn': False, - 'use_xpu': True + 'use_xpu': True, } self.outputs = { 'XShape': np.random.random(self.shape).astype("float32"), - 'Out': self.inputs['X'].transpose(self.axis) + 'Out': self.inputs['X'].transpose(self.axis), } def init_op_type(self): @@ -61,77 +60,66 @@ def initTestCase(self): class TestCase_ZeroDim(TestXPUTransposeOp): - def initTestCase(self): self.shape = () self.axis = () class TestCase0(TestXPUTransposeOp): - def initTestCase(self): - self.shape = (100, ) - self.axis = (0, ) + self.shape = (100,) + self.axis = (0,) class TestCase1(TestXPUTransposeOp): - def initTestCase(self): self.shape = (3, 4, 10) self.axis = (0, 2, 1) class TestCase2(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5) self.axis = (0, 2, 3, 1) class TestCase3(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6) self.axis = (4, 2, 3, 1, 0) class TestCase4(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 3, 4, 5, 6, 1) self.axis = (4, 2, 3, 1, 0, 5) class TestCase5(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 16, 96) self.axis = (0, 2, 1) class TestCase6(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 10, 12, 16) self.axis = (3, 1, 2, 0) class TestCase7(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 10, 2, 16) self.axis = (0, 1, 3, 2) class TestCase8(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (0, 1, 3, 2, 4, 5, 6, 7) class TestCase9(TestXPUTransposeOp): - def initTestCase(self): self.shape = (2, 3, 2, 3, 2, 4, 3, 3) self.axis = (6, 1, 3, 5, 0, 2, 4, 7) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_tril_triu_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_tril_triu_op_xpu.py index 5acd5f22b6050..a80d6adfb94e2 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_tril_triu_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_tril_triu_op_xpu.py @@ -20,19 +20,21 @@ import unittest import numpy as np from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestTrilTriuOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'tril_triu' self.use_dynamic_create_class = False class TestTrilTriuOp(XPUOpTest): - def setUp(self): self.init_dtype() self.initTestCase() @@ -42,9 +44,9 @@ def setUp(self): self.op_type = "tril_triu" self.place = paddle.XPUPlace(0) if self.dtype == np.int32: - self.X = np.arange(1, - self.get_Xshape_prod() + 1, - dtype=self.dtype).reshape(self.Xshape) + self.X = np.arange( + 1, self.get_Xshape_prod() + 1, dtype=self.dtype + ).reshape(self.Xshape) else: self.X = np.random.random(self.Xshape).astype(dtype=self.dtype) self.inputs = {'X': self.X} @@ -53,9 +55,9 @@ def setUp(self): 'lower': True if self.real_op_type == 'tril' else False, } self.outputs = { - 'Out': - self.real_np_op(self.X, self.diagonal) - if self.diagonal else self.real_np_op(self.X) + 'Out': self.real_np_op(self.X, self.diagonal) + if self.diagonal + else self.real_np_op(self.X) } def init_dtype(self): @@ -78,11 +80,14 @@ def test_check_output(self): def test_check_grad_normal(self): if self.dtype == np.int32: user_defined_grad_outputs = np.random.random( - self.Xshape).astype('float32') + self.Xshape + ).astype('float32') self.check_grad_with_place( - self.place, ['X'], + self.place, + ['X'], 'Out', - user_defined_grad_outputs=user_defined_grad_outputs) + user_defined_grad_outputs=user_defined_grad_outputs, + ) else: self.check_grad_with_place(self.place, ['X'], 'Out') @@ -91,74 +96,70 @@ def initTestCase(self): self.Xshape = (10, 10) class TestTrilTriuOp1(TestTrilTriuOp): - def initTestCase(self): self.diagonal = -3 self.Xshape = (5, 5) class TestTrilTriuOp2(TestTrilTriuOp): - def initTestCase(self): self.diagonal = 4 self.Xshape = (11, 17) class TestTrilTriuOp3(TestTrilTriuOp): - def initTestCase(self): self.diagonal = 10 self.Xshape = (2, 25, 25) class TestTrilTriuOp4(TestTrilTriuOp): - def initTestCase(self): self.diagonal = -10 self.Xshape = (1, 2, 33, 11) class TestTrilTriuOp5(TestTrilTriuOp): - def initTestCase(self): self.diagonal = 11 self.Xshape = (1, 1, 99) class TestTrilTriuOp6(TestTrilTriuOp): - def initTestCase(self): self.diagonal = 5 self.Xshape = (1, 2, 3, 5, 99) class TestTrilTriuOp7(TestTrilTriuOp): - def initTestCase(self): self.diagonal = -100 self.Xshape = (2, 2, 3, 4, 5) class TestTrilTriuOpError(unittest.TestCase): - def test_errors1(self): paddle.enable_static() data = fluid.data(shape=(20, 22), dtype='float32', name="data1") op_type = np.random.choice(['triu', 'tril']) errmsg = { - "diagonal: TypeError": - "diagonal in {} must be a python Int".format(op_type), + "diagonal: TypeError": "diagonal in {} must be a python Int".format( + op_type + ), } expected = list(errmsg.keys())[0] - with self.assertRaisesRegex(eval(expected.split(':')[-1]), - errmsg[expected]): + with self.assertRaisesRegex( + eval(expected.split(':')[-1]), errmsg[expected] + ): getattr(tensor, op_type)(x=data, diagonal='2022') def test_errors2(self): paddle.enable_static() - data = fluid.data(shape=(200, ), dtype='float32', name="data2") + data = fluid.data(shape=(200,), dtype='float32', name="data2") op_type = np.random.choice(['triu', 'tril']) errmsg = { - "input: ValueError": - "x shape in {} must be at least 2-D".format(op_type), + "input: ValueError": "x shape in {} must be at least 2-D".format( + op_type + ), } expected = list(errmsg.keys())[0] - with self.assertRaisesRegex(eval(expected.split(':')[-1]), - errmsg[expected]): + with self.assertRaisesRegex( + eval(expected.split(':')[-1]), errmsg[expected] + ): getattr(tensor, op_type)(x=data, diagonal=[None]) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_truncated_gaussian_random_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_truncated_gaussian_random_op_xpu.py index 649538aeab2f7..ea79e9078dbae 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_truncated_gaussian_random_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_truncated_gaussian_random_op_xpu.py @@ -21,19 +21,21 @@ import paddle.fluid as fluid from paddle.fluid.executor import Executor from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestTruncatedGaussianRandomOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'truncated_gaussian_random' self.use_dynamic_create_class = False class TestTruncatedGaussianRandomOp(XPUOpTest): - def init(self): self.dtype = self.in_type self.place = paddle.XPUPlace(0) @@ -64,9 +66,9 @@ def gaussian_random_test(self, place): program = fluid.Program() block = program.global_block() vout = block.create_var(name="Out") - op = block.append_op(type=self.op_type, - outputs={"Out": vout}, - attrs=self.attrs) + op = block.append_op( + type=self.op_type, outputs={"Out": vout}, attrs=self.attrs + ) op.desc.infer_var_type(block.desc) op.desc.infer_shape(block.desc) @@ -82,28 +84,24 @@ def gaussian_random_test(self, place): np.testing.assert_allclose(np.var(tensor), 0.773, atol=0.05) class TestTruncatedGaussianRandomOp_1(TestTruncatedGaussianRandomOp): - def set_attrs(self): self.shape = [4096, 2] self.mean = 5.0 self.std = 1.0 class TestTruncatedGaussianRandomOp_2(TestTruncatedGaussianRandomOp): - def set_attrs(self): self.shape = [1024] self.mean = -2.0 self.std = 1.0 class TestTruncatedGaussianRandomOp_3(TestTruncatedGaussianRandomOp): - def set_attrs(self): self.shape = [11 * 13 * 17] self.mean = -1.0 self.std = 1.0 class TestTruncatedGaussianRandomOp_4(TestTruncatedGaussianRandomOp): - def set_attrs(self): self.shape = [2049] self.mean = 5.1234 diff --git a/python/paddle/fluid/tests/unittests/xpu/test_uniform_random_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_uniform_random_op_xpu.py index 84f949a74bf7f..4847e9db7810a 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_uniform_random_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_uniform_random_op_xpu.py @@ -18,13 +18,15 @@ import unittest import numpy as np import paddle -from test_uniform_random_op import TestUniformRandomOp, TestUniformRandomOpSelectedRows +from test_uniform_random_op import ( + TestUniformRandomOp, + TestUniformRandomOpSelectedRows, +) paddle.enable_static() class TestXPUUniformRandomOp(TestUniformRandomOp): - def test_check_output(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) @@ -35,7 +37,6 @@ def test_check_output(self): class TestXPUUniformRandomOpSelectedRows(TestUniformRandomOpSelectedRows): - def test_check_output(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze2_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze2_op_xpu.py index 743a818952866..c745466fb24bf 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze2_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze2_op_xpu.py @@ -21,19 +21,21 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestUnsqueeze2Op(XPUOpTestWrapper): - def __init__(self): self.op_name = "unsqueeze2" self.use_dynamic_create_class = False class TestUnsqueeze2Op(XPUOpTest): - def setUp(self): self.op_type = "unsqueeze2" self.__class__.op_type = "unsqueeze2" @@ -45,7 +47,7 @@ def setUp(self): } self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype(self.dtype) + "XShape": np.random.random(self.ori_shape).astype(self.dtype), } self.init_attrs() @@ -73,15 +75,13 @@ def test_check_grad(self): # Correct: Single input index. class TestUnsqueeze2Op1(TestUnsqueeze2Op): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) # Correct: Mixed input axis. class TestUnsqueeze2Op2(TestUnsqueeze2Op): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -89,7 +89,6 @@ def init_test_case(self): # Correct: There is duplicated axis. class TestUnsqueeze2Op3(TestUnsqueeze2Op): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -97,7 +96,6 @@ def init_test_case(self): # Correct: Reversed axes. class TestUnsqueeze2Op4(TestUnsqueeze2Op): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -105,7 +103,6 @@ def init_test_case(self): # axes is a list(with tensor) class TestUnsqueeze2Op_AxesTensorList(XPUOpTest): - def setUp(self): self.op_type = "unsqueeze2" self.__class__.op_type = "unsqueeze2" @@ -115,17 +112,18 @@ def setUp(self): axes_tensor_list = [] for index, ele in enumerate(self.axes): - axes_tensor_list.append(("axes" + str(index), np.ones( - (1)).astype('int32') * ele)) + axes_tensor_list.append( + ("axes" + str(index), np.ones((1)).astype('int32') * ele) + ) self.inputs = { "X": np.random.random(self.ori_shape).astype(self.dtype), - "AxesTensorList": axes_tensor_list + "AxesTensorList": axes_tensor_list, } self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype(self.dtype) + "XShape": np.random.random(self.ori_shape).astype(self.dtype), } def init_dtype(self): @@ -151,28 +149,24 @@ def init_attrs(self): self.attrs = {} class TestUnsqueeze2Op1_AxesTensorList(TestUnsqueeze2Op_AxesTensorList): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) class TestUnsqueeze2Op2_AxesTensorList(TestUnsqueeze2Op_AxesTensorList): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) self.new_shape = (1, 20, 5, 1) class TestUnsqueeze2Op3_AxesTensorList(TestUnsqueeze2Op_AxesTensorList): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) self.new_shape = (1, 10, 2, 1, 1, 5) class TestUnsqueeze2Op4_AxesTensorList(TestUnsqueeze2Op_AxesTensorList): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) @@ -180,7 +174,6 @@ def init_test_case(self): # axes is a Tensor class TestUnsqueeze2Op_AxesTensor(XPUOpTest): - def setUp(self): self.op_type = "unsqueeze2" self.__class__.op_type = "unsqueeze2" @@ -190,12 +183,12 @@ def setUp(self): self.inputs = { "X": np.random.random(self.ori_shape).astype(self.dtype), - "AxesTensor": np.array(self.axes).astype("int32") + "AxesTensor": np.array(self.axes).astype("int32"), } self.init_attrs() self.outputs = { "Out": self.inputs["X"].reshape(self.new_shape), - "XShape": np.random.random(self.ori_shape).astype(self.dtype) + "XShape": np.random.random(self.ori_shape).astype(self.dtype), } def init_dtype(self): @@ -221,28 +214,24 @@ def init_attrs(self): self.attrs = {} class TestUnsqueeze2Op1_AxesTensor(TestUnsqueeze2Op_AxesTensor): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) class TestUnsqueeze2Op2_AxesTensor(TestUnsqueeze2Op_AxesTensor): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) self.new_shape = (1, 20, 5, 1) class TestUnsqueeze2Op3_AxesTensor(TestUnsqueeze2Op_AxesTensor): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) self.new_shape = (1, 10, 2, 1, 1, 5) class TestUnsqueeze2Op4_AxesTensor(TestUnsqueeze2Op_AxesTensor): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze_op_xpu.py index 6340610a67d6a..1f0cb53500f4a 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_unsqueeze_op_xpu.py @@ -21,20 +21,22 @@ import paddle from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() # Correct: General. class XPUTestUnsqueezeOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "unsqueeze" self.use_dynamic_create_class = False class TestUnsqueezeOp(XPUOpTest): - def setUp(self): self.op_type = "unsqueeze" self.__class__.op_type = "unsqueeze" @@ -70,15 +72,13 @@ def init_attrs(self): # Correct: Single input index. class TestUnsqueezeOp1(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) - self.axes = (-1, ) + self.axes = (-1,) self.new_shape = (20, 5, 1) # Correct: Mixed input axis. class TestUnsqueezeOp2(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (20, 5) self.axes = (0, -1) @@ -86,7 +86,6 @@ def init_test_case(self): # Correct: There is duplicated axis. class TestUnsqueezeOp3(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (0, 3, 3) @@ -94,7 +93,6 @@ def init_test_case(self): # Correct: Reversed axes. class TestUnsqueezeOp4(TestUnsqueezeOp): - def init_test_case(self): self.ori_shape = (10, 2, 5) self.axes = (3, 1, 1) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_update_loss_scaling_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_update_loss_scaling_op_xpu.py index 3b6136cd34646..1c8715b93cddb 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_update_loss_scaling_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_update_loss_scaling_op_xpu.py @@ -23,19 +23,21 @@ import paddle.fluid.contrib.mixed_precision.amp_nn as amp_nn from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestUpdateLossScalingOp(XPUOpTestWrapper): - def __init__(self): self.op_name = "update_loss_scaling" self.use_dynamic_create_class = False class TestUpdateLossScalingOp(XPUOpTest): - def setUp(self): self.op_type = "update_loss_scaling" self.init() @@ -47,14 +49,14 @@ def setUp(self): 'FoundInfinite': found_inf, 'PrevLossScaling': self.prev_loss_scaling, 'InGoodSteps': self.num_good_steps, - 'InBadSteps': self.num_bad_steps + 'InBadSteps': self.num_bad_steps, } self.outputs = { 'Out': [('out0', x)], 'LossScaling': self.prev_loss_scaling * self.incr_ratio, 'OutGoodSteps': self.zero_steps, - 'OutBadSteps': self.zero_steps + 'OutBadSteps': self.zero_steps, } def init(self): @@ -78,7 +80,6 @@ def test_check_output(self): self.check_output_with_place(place, no_check_set=['Out']) class TestUpdateLossScalingOpBad(TestUpdateLossScalingOp): - def setUp(self): self.op_type = "update_loss_scaling" self.init() @@ -93,38 +94,37 @@ def setUp(self): 'FoundInfinite': found_inf, 'PrevLossScaling': self.prev_loss_scaling, 'InGoodSteps': self.num_good_steps, - 'InBadSteps': self.num_bad_steps + 'InBadSteps': self.num_bad_steps, } self.outputs = { 'Out': [('out0', np.zeros_like(x))], 'LossScaling': self.prev_loss_scaling * self.decr_ratio, 'OutGoodSteps': self.zero_steps, - 'OutBadSteps': self.zero_steps + 'OutBadSteps': self.zero_steps, } def test_check_output(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) self.check_output_with_place(place) - #self.check_output() + # self.check_output() class TestUpdateLossScalingLayer(unittest.TestCase): - def loss_scaling_check(self, scope=fluid.Scope()): a = fluid.data(name="a", shape=[1024, 1024], dtype='float32') b = fluid.data(name="b", shape=[512, 128], dtype='float32') x = [a, b] found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') - prev_loss_scaling = fluid.data(name="prev_loss_scaling", - shape=[1], - dtype='float32') - num_good_steps = fluid.data(name="num_good_steps", - shape=[1], - dtype='int32') - num_bad_steps = fluid.data(name="num_bad_steps", - shape=[1], - dtype='int32') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32' + ) + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32' + ) + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32' + ) a_v = np.random.random([1024, 1024]).astype('float32') b_v = np.random.random([512, 128]).astype('float32') @@ -138,33 +138,41 @@ def loss_scaling_check(self, scope=fluid.Scope()): incr_ratio = 2 decr_ratio = 0.8 - result = amp_nn.update_loss_scaling(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - name="update_loss_scaling") + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling", + ) place = fluid.XPUPlace(0) exe = fluid.Executor(place) with fluid.scope_guard(scope): exe.run(fluid.default_startup_program()) - result_v = exe.run(feed={ - 'a': a_v, - 'b': b_v, - 'found_inf': found_inf_v, - 'prev_loss_scaling': prev_loss_scaling_v, - 'num_good_steps': num_good_steps_v, - 'num_bad_steps': num_bad_steps_v - }, - fetch_list=[ - result, x, found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps - ]) + result_v = exe.run( + feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v, + }, + fetch_list=[ + result, + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + ], + ) assert np.array_equal(result_v[0], a_v) assert np.array_equal(result_v[1], b_v) assert np.array_equal(result_v[0], result_v[2]) @@ -179,15 +187,15 @@ def loss_scaling_check_inf(self, use_cuda=True, scope=fluid.Scope()): b = fluid.data(name="b", shape=[512, 128], dtype='float32') x = [a, b] found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') - prev_loss_scaling = fluid.data(name="prev_loss_scaling", - shape=[1], - dtype='float32') - num_good_steps = fluid.data(name="num_good_steps", - shape=[1], - dtype='int32') - num_bad_steps = fluid.data(name="num_bad_steps", - shape=[1], - dtype='int32') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32' + ) + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32' + ) + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32' + ) a_v = np.random.random([1024, 1024]).astype('float32') b_v = np.random.random([512, 128]).astype('float32') @@ -204,33 +212,41 @@ def loss_scaling_check_inf(self, use_cuda=True, scope=fluid.Scope()): incr_ratio = 2 decr_ratio = 0.8 - result = amp_nn.update_loss_scaling(x, - found_inf, - prev_loss_scaling, - num_good_steps, - num_bad_steps, - incr_every_n_steps, - decr_every_n_nan_or_inf, - incr_ratio, - decr_ratio, - name="update_loss_scaling") + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling", + ) place = fluid.XPUPlace(0) exe = fluid.Executor(place) with fluid.scope_guard(scope): exe.run(fluid.default_startup_program()) - result_v = exe.run(feed={ - 'a': a_v, - 'b': b_v, - 'found_inf': found_inf_v, - 'prev_loss_scaling': prev_loss_scaling_v, - 'num_good_steps': num_good_steps_v, - 'num_bad_steps': num_bad_steps_v - }, - fetch_list=[ - result, x, found_inf, prev_loss_scaling, - num_good_steps, num_bad_steps - ]) + result_v = exe.run( + feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v, + }, + fetch_list=[ + result, + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + ], + ) assert np.array_equal(result_v[0], np.zeros_like(a_v)) assert np.array_equal(result_v[1], np.zeros_like(b_v)) assert np.array_equal(result_v[2], np.zeros_like(a_v)) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_where_index_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_where_index_xpu.py index 6c38d74ccea2f..8123bcd73f96c 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_where_index_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_where_index_xpu.py @@ -23,18 +23,20 @@ from paddle.fluid import Program, program_guard from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestWhereIndexOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'where_index' class TestWhereIndexOp(XPUOpTest): - def setUp(self): self.init_config() self.init_data() @@ -55,7 +57,6 @@ def init_config(self): self.__class__.no_need_check_grad = True class TestAllFalse(TestWhereIndexOp): - def init_data(self): self.inputs = { 'Condition': np.array([False, False, False]).astype(self.dtype), @@ -63,29 +64,31 @@ def init_data(self): self.outputs = {'Out': np.array([], dtype='int64')} class TestRank2(TestWhereIndexOp): - def init_data(self): self.inputs = { - 'Condition': - np.array([[True, False], [False, True]]).astype(self.dtype), + 'Condition': np.array([[True, False], [False, True]]).astype( + self.dtype + ), } self.outputs = {'Out': np.array([[0, 0], [1, 1]], dtype='int64')} class TestRank3(TestWhereIndexOp): - def init_data(self): self.inputs = { - 'Condition': - np.array([[[True, False], [False, True]], - [[False, True], [True, False]], - [[False, False], [False, True]]]).astype(self.dtype), + 'Condition': np.array( + [ + [[True, False], [False, True]], + [[False, True], [True, False]], + [[False, False], [False, True]], + ] + ).astype(self.dtype), } self.outputs = { - 'Out': - np.array( + 'Out': np.array( [[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], [2, 1, 1]], - dtype='int64') + dtype='int64', + ) } @@ -95,7 +98,6 @@ def init_data(self): class TestWhereOpError(unittest.TestCase): - def test_api(self): with program_guard(Program(), Program()): cond = fluid.layers.data(name='cond', shape=[4], dtype='bool') @@ -108,9 +110,7 @@ def test_api(self): class TestWhereRaiseError(unittest.TestCase): - def test_errors(self): - def test_type(): fluid.layers.where([10]) diff --git a/python/paddle/fluid/tests/unittests/xpu/test_where_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_where_op_xpu.py index 018079109c9c1..18af22f3c6465 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_where_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_where_op_xpu.py @@ -23,18 +23,20 @@ from paddle.fluid.backward import append_backward from op_test_xpu import XPUOpTest -from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper +from xpu.get_test_cover_info import ( + create_test_class, + get_xpu_op_support_types, + XPUOpTestWrapper, +) paddle.enable_static() class XPUTestWhereOp(XPUOpTestWrapper): - def __init__(self): self.op_name = 'where' class TestXPUWhereOp(XPUOpTest): - def setUp(self): self.init_config() self.init_data() @@ -56,19 +58,18 @@ def test_check_output(self): self.check_output_with_place(self.place) class TestXPUWhereOp2(TestXPUWhereOp): - def init_data(self): self.x = np.random.uniform(-5, 5, (60, 2)).astype(self.dtype) self.y = np.random.uniform(-5, 5, (60, 2)).astype(self.dtype) self.cond = np.ones((60, 2)).astype("bool") class TestXPUWhereOp3(TestXPUWhereOp): - def init_data(self): self.x = np.random.uniform(-3, 5, (20, 2, 4)).astype(self.dtype) self.y = np.random.uniform(-3, 5, (20, 2, 4)).astype(self.dtype) - self.cond = np.array(np.random.randint(2, size=(20, 2, 4)), - dtype=bool) + self.cond = np.array( + np.random.randint(2, size=(20, 2, 4)), dtype=bool + ) support_types = get_xpu_op_support_types('where') @@ -77,7 +78,6 @@ def init_data(self): class TestXPUWhereAPI(unittest.TestCase): - def setUp(self): self.__class__.use_xpu = True self.place = paddle.XPUPlace(0) @@ -102,9 +102,9 @@ def test_api(self): train_prog = fluid.Program() startup = fluid.Program() with fluid.program_guard(train_prog, startup): - cond = fluid.data(name='cond', - shape=self.shape, - dtype='bool') + cond = fluid.data( + name='cond', shape=self.shape, dtype='bool' + ) x = fluid.data(name='x', shape=self.shape, dtype='float32') y = fluid.data(name='y', shape=self.shape, dtype='float32') @@ -122,24 +122,25 @@ def test_api(self): fetch_list.append(x.grad_name) if y_stop_gradient is False: fetch_list.append(y.grad_name) - out = exe.run(train_prog, - feed={ - 'cond': self.cond, - 'x': self.x, - 'y': self.y - }, - fetch_list=fetch_list) + out = exe.run( + train_prog, + feed={'cond': self.cond, 'x': self.x, 'y': self.y}, + fetch_list=fetch_list, + ) assert np.array_equal(out[0], self.out) if x_stop_gradient is False: - assert np.array_equal(out[2], - self.ref_x_backward(out[1])) + assert np.array_equal( + out[2], self.ref_x_backward(out[1]) + ) if y.stop_gradient is False: - assert np.array_equal(out[3], - self.ref_y_backward(out[1])) + assert np.array_equal( + out[3], self.ref_y_backward(out[1]) + ) elif y.stop_gradient is False: - assert np.array_equal(out[2], - self.ref_y_backward(out[1])) + assert np.array_equal( + out[2], self.ref_y_backward(out[1]) + ) def test_api_broadcast(self, use_cuda=False): train_prog = fluid.Program() @@ -148,24 +149,21 @@ def test_api_broadcast(self, use_cuda=False): x = fluid.layers.data(name='x', shape=[4, 1], dtype='float32') y = fluid.layers.data(name='y', shape=[4, 2], dtype='float32') x_i = np.array([[0.9383, 0.1983, 3.2, 1.2]]).astype("float32") - y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, - 1.0]]).astype("float32") + y_i = np.array([[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]).astype( + "float32" + ) result = paddle.where(x > 1, x=x, y=y) exe = fluid.Executor(self.place) exe.run(startup) - out = exe.run(train_prog, - feed={ - 'x': x_i, - 'y': y_i - }, - fetch_list=[result]) + out = exe.run( + train_prog, feed={'x': x_i, 'y': y_i}, fetch_list=[result] + ) assert np.array_equal(out[0], np.where(x_i > 1, x_i, y_i)) class TestWhereDygraphAPI(unittest.TestCase): - def test_api(self): with fluid.dygraph.guard(paddle.XPUPlace(0)): x_i = np.array([0.9383, 0.1983, 3.2, 1.2]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/xpu/test_while_op_xpu.py b/python/paddle/fluid/tests/unittests/xpu/test_while_op_xpu.py index 80fe5f8c14440..c1fa366cc88b2 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_while_op_xpu.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_while_op_xpu.py @@ -24,20 +24,16 @@ class TestWhileOp(unittest.TestCase): - def simple_net(self): - d0 = layers.data("d0", - shape=[10], - append_batch_size=False, - dtype='float32') - d1 = layers.data("d1", - shape=[10], - append_batch_size=False, - dtype='float32') - d2 = layers.data("d2", - shape=[10], - append_batch_size=False, - dtype='float32') + d0 = layers.data( + "d0", shape=[10], append_batch_size=False, dtype='float32' + ) + d1 = layers.data( + "d1", shape=[10], append_batch_size=False, dtype='float32' + ) + d2 = layers.data( + "d2", shape=[10], append_batch_size=False, dtype='float32' + ) i = layers.zeros(shape=[1], dtype='int64') i.stop_gradient = True init = layers.zeros(shape=[10], dtype='float32') @@ -95,12 +91,10 @@ def test_simple_net(self): for i in range(3): d.append(numpy.random.random(size=[10]).astype('float32')) - outs = exe.run(feed={ - 'd0': d[0], - 'd1': d[1], - 'd2': d[2] - }, - fetch_list=[sum_result]) + outs = exe.run( + feed={'d0': d[0], 'd1': d[1], 'd2': d[2]}, + fetch_list=[sum_result], + ) self.assertAlmostEqual(numpy.sum(d), numpy.sum(outs[0]), delta=0.01) def test_simple_net_forward(self): diff --git a/python/paddle/fluid/tests/unittests/xpu/test_xpu_place.py b/python/paddle/fluid/tests/unittests/xpu/test_xpu_place.py index 3576bbdaed3b9..75679047301df 100644 --- a/python/paddle/fluid/tests/unittests/xpu/test_xpu_place.py +++ b/python/paddle/fluid/tests/unittests/xpu/test_xpu_place.py @@ -21,7 +21,6 @@ class Test_XPU_Places(unittest.TestCase): - def assert_places_equal(self, places0, places1): self.assertEqual(len(places0), len(places1)) for place0, place1 in zip(places0, places1): diff --git a/python/paddle/fluid/trainer_desc.py b/python/paddle/fluid/trainer_desc.py index c4c17c7095aa0..f8151297c2d10 100644 --- a/python/paddle/fluid/trainer_desc.py +++ b/python/paddle/fluid/trainer_desc.py @@ -17,8 +17,12 @@ import os __all__ = [ - 'TrainerDesc', 'MultiTrainer', 'DistMultiTrainer', 'PipelineTrainer', - 'HeterXpuTrainer', 'HeterPipelineTrainer' + 'TrainerDesc', + 'MultiTrainer', + 'DistMultiTrainer', + 'PipelineTrainer', + 'HeterXpuTrainer', + 'HeterPipelineTrainer', ] @@ -43,8 +47,10 @@ def __init__(self): sys.path.append(cur_path + "/proto") from proto import trainer_desc_pb2 + self.proto_desc = trainer_desc_pb2.TrainerDesc() import multiprocessing as mp + # set default thread num == cpu count self.proto_desc.thread_num = mp.cpu_count() self._fleet_desc = None @@ -53,26 +59,26 @@ def __init__(self): self._infer = False def _set_heter_info(self, ret): - #ret = = fu.split_program_by_device(program) - #start_list, end_list, send_list, recv_list, program_list = fu.split_program_by_device(program) - #if len(start_list) != 3: + # ret = = fu.split_program_by_device(program) + # start_list, end_list, send_list, recv_list, program_list = fu.split_program_by_device(program) + # if len(start_list) != 3: # print("start_list len=", len(start_list), " will not set heter info") # return - #for i in start_list[0]: + # for i in start_list[0]: # self.proto_desc.op_run_start_idx.append(i) - #for i in end_list[0]: + # for i in end_list[0]: # self.proto_desc.op_run_end_idx.append(i) - #for i in send_list[0]: + # for i in send_list[0]: # self.proto_desc.op_run_send_list.append(i) - #for i in recv_list[0]: + # for i in recv_list[0]: # self.proto_desc.op_run_recv_list.append(i) if ret is None: return - #for i in ret[0]: # start_list[1]: + # for i in ret[0]: # start_list[1]: # self.proto_desc.xpu_start_idx.append(i) self.proto_desc.xpu_start_idx = ret[0] - #for i in ret[1]: #end_list[1]: + # for i in ret[1]: #end_list[1]: # self.proto_desc.o_end_idx.append(i) self.proto_desc.xpu_end_idx = ret[1] for i in ret[2]: # send_list[1]: @@ -80,13 +86,13 @@ def _set_heter_info(self, ret): for i in ret[3]: # recv_list[1]: self.proto_desc.xpu_recv_list.append(i) - #for i in start_list[2]: + # for i in start_list[2]: # self.proto_desc.op_run_end_start_idx.append(i) - #for i in end_list[2]: + # for i in end_list[2]: # self.proto_desc.op_run_end_idx.append(i) - #for i in send_list[2]: + # for i in send_list[2]: # self.proto_desc.op_run_end_send_list.append(i) - #for i in recv_list[2]: + # for i in recv_list[2]: # self.proto_desc.op_run_end_recv_list.append(i) def _set_fetch_var_and_info(self, fetch_vars, fetch_info, print_period): @@ -95,7 +101,8 @@ def _set_fetch_var_and_info(self, fetch_vars, fetch_info, print_period): for i, v in enumerate(fetch_vars): self.proto_desc.fetch_config.fetch_var_names.extend([v.name]) self.proto_desc.fetch_config.fetch_var_str_format.extend( - [fetch_info[i]]) + [fetch_info[i]] + ) self.proto_desc.fetch_config.print_period = print_period def _set_debug(self, debug): @@ -114,6 +121,7 @@ def _set_fleet_desc(self, fleet_desc): self._fleet_desc = fleet_desc ## serialize fleet_desc from google.protobuf import text_format + fleet_desc_str = text_format.MessageToString(fleet_desc) self.proto_desc.fleet_desc = fleet_desc_str @@ -137,8 +145,11 @@ def _set_no_cvm(self, no_cvm=False): self.proto_desc.no_cvm = no_cvm def _set_scale_sparse_grad_with_batch_size( - self, scale_sparse_gradient_with_batch_size=True): - self.proto_desc.scale_sparse_gradient_with_batch_size = scale_sparse_gradient_with_batch_size + self, scale_sparse_gradient_with_batch_size=True + ): + self.proto_desc.scale_sparse_gradient_with_batch_size = ( + scale_sparse_gradient_with_batch_size + ) def _set_scale_datanorm(self, scale_datanorm=-1): self.proto_desc.scale_datanorm = scale_datanorm @@ -203,16 +214,21 @@ def _set_loss_names(self, loss_names): self.proto_desc.loss_names.append(loss) def _set_adjust_ins_weight(self, config_dict): - self.proto_desc.adjust_ins_weight_config.need_adjust = \ - config_dict.get("need_adjust", False) - self.proto_desc.adjust_ins_weight_config.nid_slot = \ - config_dict.get("nid_slot", "") - self.proto_desc.adjust_ins_weight_config.nid_adjw_threshold = \ - config_dict.get("nid_adjw_threshold", 0.0) - self.proto_desc.adjust_ins_weight_config.nid_adjw_ratio = \ - config_dict.get("nid_adjw_ratio", 0.0) - self.proto_desc.adjust_ins_weight_config.ins_weight_slot = \ - config_dict.get("ins_weight_slot", "") + self.proto_desc.adjust_ins_weight_config.need_adjust = config_dict.get( + "need_adjust", False + ) + self.proto_desc.adjust_ins_weight_config.nid_slot = config_dict.get( + "nid_slot", "" + ) + self.proto_desc.adjust_ins_weight_config.nid_adjw_threshold = ( + config_dict.get("nid_adjw_threshold", 0.0) + ) + self.proto_desc.adjust_ins_weight_config.nid_adjw_ratio = ( + config_dict.get("nid_adjw_ratio", 0.0) + ) + self.proto_desc.adjust_ins_weight_config.ins_weight_slot = ( + config_dict.get("ins_weight_slot", "") + ) def _set_copy_table_config(self, config_dict): config = self.proto_desc.copy_table_config @@ -227,9 +243,9 @@ def _set_copy_table_config(self, config_dict): dest_sparse_tables = [dest_sparse_tables] if len(src_sparse_tables) != len(dest_sparse_tables): raise ValueError( - "len(src_sparse_tables) != len(dest_sparse_tables)," \ - " %s vs %s" % (len(src_sparse_tables), \ - len(dest_sparse_tables))) + "len(src_sparse_tables) != len(dest_sparse_tables)," + " %s vs %s" % (len(src_sparse_tables), len(dest_sparse_tables)) + ) for i in src_sparse_tables: config.src_sparse_tables.append(i) for i in dest_sparse_tables: @@ -243,9 +259,9 @@ def _set_copy_table_config(self, config_dict): dest_dense_tables = [dest_dense_tables] if len(src_dense_tables) != len(dest_dense_tables): raise ValueError( - "len(src_dense_tables) != len(dest_dense_tables)," \ - " %s vs %s" % (len(src_dense_tables), \ - len(dest_dense_tables))) + "len(src_dense_tables) != len(dest_dense_tables)," + " %s vs %s" % (len(src_dense_tables), len(dest_dense_tables)) + ) for i in src_dense_tables: config.src_dense_tables.append(i) for i in dest_dense_tables: @@ -261,8 +277,9 @@ def _set_copy_table_config(self, config_dict): dest_var_list = [dest_var_list] if len(src_var_list) != len(dest_var_list): raise ValueError( - "len(src_var_list) != len(dest_var_list), %s vs" \ - " %s" % (len(src_var_list), len(dest_var_list))) + "len(src_var_list) != len(dest_var_list), %s vs" + " %s" % (len(src_var_list), len(dest_var_list)) + ) for i in src_var_list: config.src_var_list.append(i) for i in dest_var_list: @@ -279,19 +296,22 @@ def _set_copy_table_config(self, config_dict): raise ValueError("dependency len %s != 1" % len(values)) for value in values: m.values.append(value) - config.dense_pull_after_copy = \ - config_dict.get("dense_pull_after_copy", True) - config.enable_dependency = \ - config_dict.get("enable_dependency", False) - config.sparse_copy_by_feasign = \ - config_dict.get("sparse_copy_by_feasign", True) + config.dense_pull_after_copy = config_dict.get( + "dense_pull_after_copy", True + ) + config.enable_dependency = config_dict.get("enable_dependency", False) + config.sparse_copy_by_feasign = config_dict.get( + "sparse_copy_by_feasign", True + ) def _desc(self): from google.protobuf import text_format + return self.proto_desc.SerializeToString() def __str__(self): from google.protobuf import text_format + return text_format.MessageToString(self.proto_desc) diff --git a/python/paddle/fluid/trainer_factory.py b/python/paddle/fluid/trainer_factory.py index 3ba9f9eea46d1..7ac367b38fd2c 100644 --- a/python/paddle/fluid/trainer_factory.py +++ b/python/paddle/fluid/trainer_factory.py @@ -19,12 +19,26 @@ import numpy as np from paddle.fluid.log_helper import get_logger -local_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') - -from .trainer_desc import MultiTrainer, DistMultiTrainer, PipelineTrainer, HeterXpuTrainer, PSGPUTrainer, HeterPipelineTrainer -from .device_worker import Hogwild, DownpourSGD, DownpourLite, Section, DownpourSGDOPT, HeterSection +local_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) + +from .trainer_desc import ( + MultiTrainer, + DistMultiTrainer, + PipelineTrainer, + HeterXpuTrainer, + PSGPUTrainer, + HeterPipelineTrainer, +) +from .device_worker import ( + Hogwild, + DownpourSGD, + DownpourLite, + Section, + DownpourSGDOPT, + HeterSection, +) from .framework import Variable from multiprocessing import Process, Manager @@ -67,18 +81,24 @@ def _create_trainer(self, opt_info=None): trainer._set_mpi_rank(opt_info["mpi_rank"]) if opt_info.get("mpi_size") is not None: trainer._set_mpi_size(opt_info["mpi_size"]) - if opt_info.get("dump_fields") is not None and len( - opt_info.get("dump_fields")) != 0: + if ( + opt_info.get("dump_fields") is not None + and len(opt_info.get("dump_fields")) != 0 + ): trainer._set_dump_fields(opt_info["dump_fields"]) - if opt_info.get("dump_fields_path") is not None and len( - opt_info.get("dump_fields_path")) != 0: + if ( + opt_info.get("dump_fields_path") is not None + and len(opt_info.get("dump_fields_path")) != 0 + ): trainer._set_dump_fields_path(opt_info["dump_fields_path"]) if opt_info.get("dump_file_num") is not None: trainer._set_dump_file_num(opt_info["dump_file_num"]) if opt_info.get("dump_converter") is not None: trainer._set_dump_converter(opt_info["dump_converter"]) - if opt_info.get("dump_param") is not None and len( - opt_info.get("dump_param")) != 0: + if ( + opt_info.get("dump_param") is not None + and len(opt_info.get("dump_param")) != 0 + ): trainer._set_dump_param(opt_info["dump_param"]) if opt_info.get("worker_places") is not None: trainer._set_worker_places(opt_info["worker_places"]) @@ -86,15 +106,18 @@ def _create_trainer(self, opt_info=None): trainer._set_use_ps_gpu(opt_info["use_ps_gpu"]) if opt_info.get("is_dump_in_simple_mode") is not None: trainer._set_is_dump_in_simple_mode( - opt_info["is_dump_in_simple_mode"]) + opt_info["is_dump_in_simple_mode"] + ) if opt_info.get("enable_random_dump") is not None: trainer._set_enable_random_dump( - opt_info["enable_random_dump"]) + opt_info["enable_random_dump"] + ) if opt_info.get("dump_interval") is not None: trainer._set_dump_interval(opt_info["dump_interval"]) if opt_info.get("random_with_lineid") is not None: trainer._set_random_with_lineid( - opt_info["random_with_lineid"]) + opt_info["random_with_lineid"] + ) if "fleet_desc" in opt_info: device_worker._set_fleet_desc(opt_info["fleet_desc"]) @@ -103,20 +126,25 @@ def _create_trainer(self, opt_info=None): trainer._set_use_cvm(opt_info["use_cvm"]) if opt_info.get("no_cvm") is not None: trainer._set_no_cvm(opt_info["no_cvm"]) - if opt_info.get( - "scale_sparse_gradient_with_batch_size") is not None: + if ( + opt_info.get("scale_sparse_gradient_with_batch_size") + is not None + ): trainer._set_scale_sparse_grad_with_batch_size( - opt_info["scale_sparse_gradient_with_batch_size"]) + opt_info["scale_sparse_gradient_with_batch_size"] + ) if opt_info.get("scale_datanorm") is not None: trainer._set_scale_datanorm(opt_info["scale_datanorm"]) if opt_info.get("adjust_ins_weight") is not None: trainer._set_adjust_ins_weight( - opt_info["adjust_ins_weight"]) + opt_info["adjust_ins_weight"] + ) if opt_info.get("copy_table") is not None: trainer._set_copy_table_config(opt_info["copy_table"]) if opt_info.get("check_nan_var_names") is not None: trainer._set_check_nan_var_names( - opt_info["check_nan_var_names"]) + opt_info["check_nan_var_names"] + ) if opt_info.get("loss_names") is not None: trainer._set_loss_names(opt_info["loss_names"]) trainer._set_device_worker(device_worker) @@ -131,8 +159,9 @@ class FetchHandlerMonitor(object): def __init__(self, scope, handler): self.fetch_instance = handler - self.fetch_thread = threading.Thread(target=self.handler_launch_func, - args=(scope, self.fetch_instance)) + self.fetch_thread = threading.Thread( + target=self.handler_launch_func, args=(scope, self.fetch_instance) + ) self.running_lock = threading.Lock() self.running = False @@ -145,7 +174,8 @@ def handler_launch_func(self, scope, handler): var_name_to_key[fetch_instance.var_dict[key].name] = key else: local_logger.warning( - "the value of {} is not a Variable".format(key)) + "the value of {} is not a Variable".format(key) + ) var_name_to_key["None.var"] = key elapsed_secs = 0 while True: @@ -165,7 +195,9 @@ def handler_launch_func(self, scope, handler): if var == None: local_logger.warning( "{} value currently not available".format( - var_name_to_key[key])) + var_name_to_key[key] + ) + ) res_dict = {} for key in fetch_dict: user_name = var_name_to_key[key] @@ -177,12 +209,14 @@ def handler_launch_func(self, scope, handler): lod = res_dict[user_name].lod() if len(lod) > 0: - raise RuntimeError("Some of your fetched tensors \ + raise RuntimeError( + "Some of your fetched tensors \ hold LoD information. \ They can not be completely cast \ to Python ndarray. We can \ not return LoDTensor itself directly, \ - please choose another targets") + please choose another targets" + ) if res_dict[user_name]._is_initialized(): res_dict[user_name] = np.array(res_dict[user_name]) else: diff --git a/python/paddle/fluid/transpiler/__init__.py b/python/paddle/fluid/transpiler/__init__.py index 166b04c21ac76..8da4210dba2e7 100644 --- a/python/paddle/fluid/transpiler/__init__.py +++ b/python/paddle/fluid/transpiler/__init__.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .distribute_transpiler import DistributeTranspiler, DistributeTranspilerConfig +from .distribute_transpiler import ( + DistributeTranspiler, + DistributeTranspilerConfig, +) from .memory_optimization_transpiler import memory_optimize, release_memory from .ps_dispatcher import HashName, RoundRobin diff --git a/python/paddle/fluid/transpiler/ascend_transpiler.py b/python/paddle/fluid/transpiler/ascend_transpiler.py index 69fb2b1833655..60ab474e9044c 100644 --- a/python/paddle/fluid/transpiler/ascend_transpiler.py +++ b/python/paddle/fluid/transpiler/ascend_transpiler.py @@ -20,7 +20,6 @@ class AscendTranspiler(collective.Collective): - def __init__(self, startup_program, main_program): self.nrings = 1 super(AscendTranspiler, self).__init__(self.nrings) @@ -32,8 +31,10 @@ def _insert_allreduce_ops(self): ring_id = -1 grad = None for idx, op in reversed(list(enumerate(block.ops))): - if self._is_backward_op(op) and \ - self.op_role_var_key in op.attr_names: + if ( + self._is_backward_op(op) + and self.op_role_var_key in op.attr_names + ): op_role_var = op.all_attrs()[self.op_role_var_key] if len(op_role_var) == 0: @@ -50,22 +51,26 @@ def _insert_allreduce_ops(self): # As we search ops reversedly, we should insert c_allreduce_sum # op in the same way to keep the ring_id alternate ring_id = (ring_id + 1) % self.nrings - block._insert_op(offset + 1, - type='c_allreduce_sum', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Backward - }) - block._insert_op(offset + 2, - type='scale', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'scale': 1.0 / fleet.worker_num(), - self.op_role_key: OpRole.Backward - }) + block._insert_op( + offset + 1, + type='c_allreduce_sum', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'ring_id': ring_id, + self.op_role_key: OpRole.Backward, + }, + ) + block._insert_op( + offset + 2, + type='scale', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'scale': 1.0 / fleet.worker_num(), + self.op_role_key: OpRole.Backward, + }, + ) if grad is None: return diff --git a/python/paddle/fluid/transpiler/collective.py b/python/paddle/fluid/transpiler/collective.py index 84912529cfbee..aedbaeb43cde7 100644 --- a/python/paddle/fluid/transpiler/collective.py +++ b/python/paddle/fluid/transpiler/collective.py @@ -32,8 +32,7 @@ class Collective(object): - ''' - ''' + ''' ''' def __init__(self, nrings): self.nrings = nrings @@ -48,8 +47,15 @@ def __init__(self, nrings): self.op_role_key = op_maker.kOpRoleAttrName() self.op_role_var_key = op_maker.kOpRoleVarAttrName() - def transpile(self, startup_program, main_program, rank, endpoints, - current_endpoint, wait_port): + def transpile( + self, + startup_program, + main_program, + rank, + endpoints, + current_endpoint, + wait_port, + ): # in case of '127.0.0.1:6700,127.0.0.1:6701,...' if isinstance(endpoints, str): endpoints = endpoints.split(',') @@ -63,7 +69,11 @@ def transpile(self, startup_program, main_program, rank, endpoints, self.main_program = default_main_program() self.nranks = len(endpoints) - if self.nranks == 1 and self.mode != "single_process_multi_thread" and self.mode != "box": + if ( + self.nranks == 1 + and self.mode != "single_process_multi_thread" + and self.mode != "box" + ): raise ValueError('the number of endpoints must > 1') if rank < 0: @@ -71,8 +81,11 @@ def transpile(self, startup_program, main_program, rank, endpoints, self.rank = rank if current_endpoint not in endpoints: - raise ValueError('current endpoint %s is not in %s', - current_endpoint, str(endpoints)) + raise ValueError( + 'current endpoint %s is not in %s', + current_endpoint, + str(endpoints), + ) self.endpoints = endpoints self.current_endpoint = current_endpoint @@ -96,19 +109,26 @@ def _transpile_main_program(self): def _transpile_startup_program(self): for ring_id in range(self.nrings): - self._init_communicator(self.startup_program, self.current_endpoint, - self.endpoints, self.rank, ring_id, - self.wait_port) + self._init_communicator( + self.startup_program, + self.current_endpoint, + self.endpoints, + self.rank, + ring_id, + self.wait_port, + ) self._broadcast_params() - def _init_communicator(self, - program, - current_endpoint, - endpoints, - rank, - ring_id, - wait_port, - has_multitrainer=False): + def _init_communicator( + self, + program, + current_endpoint, + endpoints, + rank, + ring_id, + wait_port, + has_multitrainer=False, + ): nranks = len(endpoints) other_endpoints = endpoints[:] other_endpoints.remove(current_endpoint) @@ -119,63 +139,76 @@ def _init_communicator(self, block = program.global_block() if core.is_compiled_with_npu(): - hccl_id_var = block.create_var(name=unique_name.generate('hccl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) + hccl_id_var = block.create_var( + name=unique_name.generate('hccl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) endpoint_to_index_map = {e: idx for idx, e in enumerate(endpoints)} - block.append_op(type='c_gen_hccl_id', - inputs={}, - outputs={'Out': hccl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints, - self.op_role_key: OpRole.Forward - }) - block.append_op(type='c_comm_init_hccl', - inputs={'X': hccl_id_var}, - outputs={}, - attrs={ - 'rank': rank, - 'ring_id': ring_id, - 'device_id': - int(os.getenv("FLAGS_selected_npus")), - 'rank_ids': nranks, - self.op_role_key: OpRole.Forward - }) + block.append_op( + type='c_gen_hccl_id', + inputs={}, + outputs={'Out': hccl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + self.op_role_key: OpRole.Forward, + }, + ) + block.append_op( + type='c_comm_init_hccl', + inputs={'X': hccl_id_var}, + outputs={}, + attrs={ + 'rank': rank, + 'ring_id': ring_id, + 'device_id': int(os.getenv("FLAGS_selected_npus")), + 'rank_ids': nranks, + self.op_role_key: OpRole.Forward, + }, + ) else: - nccl_id_var = block.create_var(name=unique_name.generate('nccl_id'), - persistable=True, - type=core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_nccl_id', - inputs={}, - outputs={'Out': nccl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints, - self.op_role_key: OpRole.Forward - }) + nccl_id_var = block.create_var( + name=unique_name.generate('nccl_id'), + persistable=True, + type=core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_nccl_id', + inputs={}, + outputs={'Out': nccl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + self.op_role_key: OpRole.Forward, + }, + ) if not has_multitrainer: - block.append_op(type='c_comm_init', - inputs={'X': nccl_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': ring_id, - self.op_role_key: OpRole.Forward - }) + block.append_op( + type='c_comm_init', + inputs={'X': nccl_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': ring_id, + self.op_role_key: OpRole.Forward, + }, + ) else: - block.append_op(type='c_comm_init_multitrainer', - inputs={'X': nccl_id_var}, - outputs={}, - attrs={ - 'ntrainers': nranks, - 'trainer_id': rank, - 'ring_id': ring_id, - self.op_role_key: OpRole.Forward - }) + block.append_op( + type='c_comm_init_multitrainer', + inputs={'X': nccl_id_var}, + outputs={}, + attrs={ + 'ntrainers': nranks, + 'trainer_id': rank, + 'ring_id': ring_id, + self.op_role_key: OpRole.Forward, + }, + ) def _broadcast_params(self): block = self.startup_program.global_block() @@ -185,23 +218,24 @@ def _broadcast_params(self): continue ring_id = (ring_id + 1) % self.nrings - block.append_op(type='c_broadcast', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - 'root': 0, - self.op_role_key: OpRole.Forward - }) + block.append_op( + type='c_broadcast', + inputs={'X': param}, + outputs={'Out': param}, + attrs={ + 'ring_id': ring_id, + 'root': 0, + self.op_role_key: OpRole.Forward, + }, + ) for ring_id in range(self.nrings): - block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Forward - }) + block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={'ring_id': ring_id, self.op_role_key: OpRole.Forward}, + ) def _is_loss_grad_op(self, op): if self.op_role_key not in op.attr_names: @@ -210,21 +244,25 @@ def _is_loss_grad_op(self, op): return op_role & int(OpRole.Backward) and op_role & int(OpRole.Loss) def _is_backward_op(self, op): - return self.op_role_key in op.attr_names and \ - int(op.all_attrs()[self.op_role_key]) & int(OpRole.Backward) + return self.op_role_key in op.attr_names and int( + op.all_attrs()[self.op_role_key] + ) & int(OpRole.Backward) def _is_update_op(self, op): - return 'Param' in op.input_names and 'Grad' in op.input_names and \ - "LearningRate" in op.input_names + return ( + 'Param' in op.input_names + and 'Grad' in op.input_names + and "LearningRate" in op.input_names + ) def _is_optimizer_op(self, op): - return self.op_role_key in op.attr_names and \ - int(op.all_attrs()[self.op_role_key]) & int(OpRole.Optimize) + return self.op_role_key in op.attr_names and int( + op.all_attrs()[self.op_role_key] + ) & int(OpRole.Optimize) class GradAllReduce(Collective): - ''' - ''' + ''' ''' def __init__(self, nrings=2): Collective.__init__(self, nrings) @@ -243,22 +281,26 @@ def _insert_scale_loss_grad_ops(self): for idx, op in reversed(list(enumerate(block.ops))): if self._is_loss_grad_op(op): loss_grad_var = block.vars[op.output_arg_names[0]] - block._insert_op(idx + 1, - type='scale', - inputs={'X': loss_grad_var}, - outputs={'Out': loss_grad_var}, - attrs={ - 'scale': 1.0 / self.nranks, - self.op_role_key: OpRole.Backward - }) + block._insert_op( + idx + 1, + type='scale', + inputs={'X': loss_grad_var}, + outputs={'Out': loss_grad_var}, + attrs={ + 'scale': 1.0 / self.nranks, + self.op_role_key: OpRole.Backward, + }, + ) def _insert_allreduce_ops(self): block = self.main_program.global_block() ring_id = -1 grad = None for idx, op in reversed(list(enumerate(block.ops))): - if self._is_backward_op(op) and \ - self.op_role_var_key in op.attr_names: + if ( + self._is_backward_op(op) + and self.op_role_var_key in op.attr_names + ): op_role_var = op.all_attrs()[self.op_role_var_key] if len(op_role_var) == 0: @@ -279,20 +321,23 @@ def _insert_allreduce_ops(self): type='c_sync_calc_stream', inputs={'X': grad}, outputs={'Out': grad}, - attrs={self.op_role_key: OpRole.Backward}) + attrs={self.op_role_key: OpRole.Backward}, + ) offset += 1 # As we search ops reversedly, we should insert c_allreduce_sum # op in the same way to keep the ring_id alternate ring_id = (ring_id + 1) % self.nrings - block._insert_op(offset, - type='c_allreduce_sum', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Backward - }) + block._insert_op( + offset, + type='c_allreduce_sum', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'ring_id': ring_id, + self.op_role_key: OpRole.Backward, + }, + ) if grad is None: return @@ -300,20 +345,21 @@ def _insert_allreduce_ops(self): for idx, op in enumerate(block.ops): if self._is_optimizer_op(op): for ring_id in range(self.nrings): - block._insert_op(idx + ring_id, - type='c_sync_comm_stream', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Backward - }) + block._insert_op( + idx + ring_id, + type='c_sync_comm_stream', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'ring_id': ring_id, + self.op_role_key: OpRole.Backward, + }, + ) break class LocalSGD(Collective): - ''' - ''' + ''' ''' def __init__(self, nrings=2): Collective.__init__(self, nrings) @@ -330,14 +376,18 @@ def _transpile_startup_program(self): non_dist_params.append(param) for param in non_dist_params: - snapshot = block.create_var(name=self.snapshot_name(param.name), - shape=param.shape, - persistable=True, - stop_gradient=True) - block.append_op(type='assign', - inputs={'X': [param]}, - outputs={'Out': [snapshot]}, - attrs={self.op_role_key: OpRole.Forward}) + snapshot = block.create_var( + name=self.snapshot_name(param.name), + shape=param.shape, + persistable=True, + stop_gradient=True, + ) + block.append_op( + type='assign', + inputs={'X': [param]}, + outputs={'Out': [snapshot]}, + attrs={self.op_role_key: OpRole.Forward}, + ) def snapshot_name(self, param_name): return param_name + self.snapshot_key @@ -352,72 +402,78 @@ def _transpile_main_program(self): if param.is_distributed: continue - snapshot = block.create_var(name=self.snapshot_name(param.name), - shape=param.shape, - persistable=True, - stop_gradient=True, - dtype=param.dtype) - - block._insert_op(idx + 1, - type='elementwise_sub', - inputs={ - 'X': [snapshot], - 'Y': [param] - }, - outputs={'Out': [param]}, - attrs={self.op_role_key: OpRole.Optimize}) - block._insert_op(idx + 2, - type='c_sync_calc_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={self.op_role_key: OpRole.Optimize}) + snapshot = block.create_var( + name=self.snapshot_name(param.name), + shape=param.shape, + persistable=True, + stop_gradient=True, + dtype=param.dtype, + ) + + block._insert_op( + idx + 1, + type='elementwise_sub', + inputs={'X': [snapshot], 'Y': [param]}, + outputs={'Out': [param]}, + attrs={self.op_role_key: OpRole.Optimize}, + ) + block._insert_op( + idx + 2, + type='c_sync_calc_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={self.op_role_key: OpRole.Optimize}, + ) ring_id = (ring_id + 1) % self.nrings - block._insert_op(idx + 3, - type='c_allreduce_sum', - inputs={'X': [param]}, - outputs={'Out': [param]}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Optimize - }) + block._insert_op( + idx + 3, + type='c_allreduce_sum', + inputs={'X': [param]}, + outputs={'Out': [param]}, + attrs={ + 'ring_id': ring_id, + self.op_role_key: OpRole.Optimize, + }, + ) ordered_param_snapshot.append((param, snapshot)) for ring_id in range(self.nrings): - block.append_op(type='c_sync_comm_stream', - inputs={'X': param}, - outputs={'Out': param}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Optimize - }) + block.append_op( + type='c_sync_comm_stream', + inputs={'X': param}, + outputs={'Out': param}, + attrs={'ring_id': ring_id, self.op_role_key: OpRole.Optimize}, + ) for param_snapshot in reversed(ordered_param_snapshot): param = param_snapshot[0] snapshot = param_snapshot[1] - block.append_op(type='scale', - inputs={'X': [param]}, - outputs={'Out': [param]}, - attrs={ - 'scale': 1.0 / self.nranks, - self.op_role_key: OpRole.Optimize - }) - block.append_op(type='elementwise_sub', - inputs={ - 'X': [snapshot], - 'Y': [param] - }, - outputs={'Out': [param]}, - attrs={self.op_role_key: OpRole.Optimize}) - block.append_op(type='assign', - inputs={'X': [param]}, - outputs={'Out': [snapshot]}, - attrs={self.op_role_key: OpRole.Optimize}) + block.append_op( + type='scale', + inputs={'X': [param]}, + outputs={'Out': [param]}, + attrs={ + 'scale': 1.0 / self.nranks, + self.op_role_key: OpRole.Optimize, + }, + ) + block.append_op( + type='elementwise_sub', + inputs={'X': [snapshot], 'Y': [param]}, + outputs={'Out': [param]}, + attrs={self.op_role_key: OpRole.Optimize}, + ) + block.append_op( + type='assign', + inputs={'X': [param]}, + outputs={'Out': [snapshot]}, + attrs={self.op_role_key: OpRole.Optimize}, + ) class SingleProcessMultiThread(GradAllReduce): - ''' - ''' + ''' ''' def __init__(self): GradAllReduce.__init__(self, 1) @@ -429,16 +485,16 @@ def _transpile_startup_program(self): class MultiThread(GradAllReduce): - ''' - ''' + ''' ''' def __init__(self, nrings=1, trans_mode="all_reduce"): GradAllReduce.__init__(self, nrings) self.mode = "box" self.trans_mode = trans_mode self.fuse_grad_size_in_num = 128 - gpu_nums = os.getenv("FLAGS_selected_gpus", - "0,1,2,3,4,5,6,7,8").split(",") + gpu_nums = os.getenv("FLAGS_selected_gpus", "0,1,2,3,4,5,6,7,8").split( + "," + ) self.gpu_num = len(gpu_nums) def _transpile_startup_program(self): @@ -448,10 +504,15 @@ def _transpile_startup_program(self): print("total endpoints: ", self.endpoints) print("rank: %d, ring_id: %d" % (self.rank, self.nrings)) for ring_id in range(self.nrings): - self._init_communicator(self.startup_program, - self.current_endpoint, self.endpoints, - self.rank, ring_id, self.wait_port, - True) + self._init_communicator( + self.startup_program, + self.current_endpoint, + self.endpoints, + self.rank, + ring_id, + self.wait_port, + True, + ) else: if "xpu" in self.trans_mode: @@ -462,13 +523,14 @@ def _transpile_startup_program(self): block.append_op( type='c_comm_init_all', attrs={ - 'devices': - list( - map(int, - os.getenv("FLAGS_selected_gpus").split(","))), - 'ring_id': - 0 - }) + 'devices': list( + map( + int, os.getenv("FLAGS_selected_gpus").split(",") + ) + ), + 'ring_id': 0, + }, + ) else: print("begin to _transpile_startup_program for single-node") block = self.startup_program.global_block() @@ -484,8 +546,10 @@ def _transpile_main_program(self): elif self.trans_mode == "fuse_all_reduce": print("begin to transpile in fuse all-reduce mode") self._insert_fuse_allreduce_ops() - elif self.trans_mode == "all_reduce_xpu" and len( - os.getenv("FLAGS_selected_gpus").split(",")) == 1: + elif ( + self.trans_mode == "all_reduce_xpu" + and len(os.getenv("FLAGS_selected_gpus").split(",")) == 1 + ): print( "skip transpile in all-reduce-xpu mode when number of devices is only one" ) @@ -501,8 +565,10 @@ def _insert_allgather_ops(self): ring_id = -1 grad = None for idx, op in reversed(list(enumerate(block.ops))): - if self._is_backward_op(op) and \ - self.op_role_var_key in op.attr_names: + if ( + self._is_backward_op(op) + and self.op_role_var_key in op.attr_names + ): op_role_var = op.all_attrs()[self.op_role_var_key] if len(op_role_var) == 0: continue @@ -516,7 +582,8 @@ def _insert_allgather_ops(self): shape=[self.allgather_ranks] + list(param.shape), persistable=False, dtype=core.VarDesc.VarType.FP32, - stop_gradient=True) + stop_gradient=True, + ) grad = block.vars[op_role_var[i + 1]] if param.is_distributed: # no need to care: used in PLSC continue @@ -528,21 +595,24 @@ def _insert_allgather_ops(self): type='c_sync_calc_stream', inputs={'X': grad}, outputs={'Out': grad}, - attrs={self.op_role_key: OpRole.Backward}) + attrs={self.op_role_key: OpRole.Backward}, + ) offset += 1 # As we search ops reversedly, we should insert c_allgather # op in the same way to keep the ring_id alternate ring_id = (ring_id + 1) % self.nrings - block._insert_op(offset, - type='c_allgather', - inputs={'X': grad}, - outputs={'Out': new_grad_var}, - attrs={ - 'nranks': self.allgather_ranks, - 'ring_id': ring_id, - self.op_role_key: OpRole.Backward - }) + block._insert_op( + offset, + type='c_allgather', + inputs={'X': grad}, + outputs={'Out': new_grad_var}, + attrs={ + 'nranks': self.allgather_ranks, + 'ring_id': ring_id, + self.op_role_key: OpRole.Backward, + }, + ) if grad is None: return @@ -550,14 +620,16 @@ def _insert_allgather_ops(self): for idx, op in enumerate(block.ops): if self._is_optimizer_op(op): for ring_id in range(self.nrings): - block._insert_op(idx + ring_id, - type='c_sync_comm_stream', - inputs={'X': grad}, - outputs={'Out': grad}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Backward - }) + block._insert_op( + idx + ring_id, + type='c_sync_comm_stream', + inputs={'X': grad}, + outputs={'Out': grad}, + attrs={ + 'ring_id': ring_id, + self.op_role_key: OpRole.Backward, + }, + ) break def _update_adam_ops(self): @@ -569,7 +641,9 @@ def _update_adam_ops(self): for idx, op in reversed(list(enumerate(block.ops))): if self._is_optimizer_op(op): offset = idx - if op.type != 'adam' and op.type != 'lamb': # filter out scale op + if ( + op.type != 'adam' and op.type != 'lamb' + ): # filter out scale op continue param_name = op.input("Param")[0] inputs = { @@ -578,26 +652,23 @@ def _update_adam_ops(self): "Moment1": block.vars[op.input("Moment1")[0]], "Moment2": block.vars[op.input("Moment2")[0]], "Beta1Pow": block.vars[op.input("Beta1Pow")[0]], - "Beta2Pow": block.vars[op.input("Beta2Pow")[0]] + "Beta2Pow": block.vars[op.input("Beta2Pow")[0]], } outputs = { "ParamOut": block.vars[op.output("ParamOut")[0]], "Moment1Out": block.vars[op.output("Moment1Out")[0]], "Moment2Out": block.vars[op.output("Moment2Out")[0]], "Beta1PowOut": block.vars[op.output("Beta1PowOut")[0]], - "Beta2PowOut": block.vars[op.output("Beta2PowOut")[0]] + "Beta2PowOut": block.vars[op.output("Beta2PowOut")[0]], } attrs = { - "epsilon": - op.attr('epsilon'), - "beta1": - op.attr('beta1'), - "beta2": - op.attr('beta2'), - "lazy_mode": - op.attr('lazy_mode'), - "min_row_size_to_use_multithread": - op.attr('min_row_size_to_use_multithread') + "epsilon": op.attr('epsilon'), + "beta1": op.attr('beta1'), + "beta2": op.attr('beta2'), + "lazy_mode": op.attr('lazy_mode'), + "min_row_size_to_use_multithread": op.attr( + 'min_row_size_to_use_multithread' + ), } split_vars = [ block.create_var( @@ -605,29 +676,30 @@ def _update_adam_ops(self): shape=block.vars[op.input("Param")[0]].shape, persistable=False, dtype=core.VarDesc.VarType.FP32, - stop_gradient=True) for i in range(self.allgather_ranks) + stop_gradient=True, + ) + for i in range(self.allgather_ranks) ] - block._insert_op(offset, - type="split", - inputs={ - 'X': - block.vars[op.input("Param")[0] + - "_allgather"] - }, - outputs={'Out': split_vars}, - attrs={ - 'num': self.allgather_ranks, - 'axis': 0 - }) + block._insert_op( + offset, + type="split", + inputs={ + 'X': block.vars[op.input("Param")[0] + "_allgather"] + }, + outputs={'Out': split_vars}, + attrs={'num': self.allgather_ranks, 'axis': 0}, + ) offset += 1 for i in range(self.allgather_ranks): inputs["Grad"] = split_vars[i] - block._insert_op(offset, - type=op.type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + block._insert_op( + offset, + type=op.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) offset += 1 # remove the original adam op block._remove_op(offset) @@ -642,13 +714,17 @@ def _insert_fuse_allreduce_ops(self): param_grads = [] # find all grad params for op in reversed(block.ops): - if self._is_backward_op(op) and \ - self.op_role_var_key in op.attr_names: + if ( + self._is_backward_op(op) + and self.op_role_var_key in op.attr_names + ): op_role_var = op.all_attrs()[self.op_role_var_key] if len(op_role_var) == 0: continue - assert len(op_role_var) % 2 == 0, "vars need to be one param var followed by one grad var, " \ - "but got odd number of vars" + assert len(op_role_var) % 2 == 0, ( + "vars need to be one param var followed by one grad var, " + "but got odd number of vars" + ) for i in range(0, len(op_role_var), 2): param_name = op_role_var[i] param = block.var(param_name) @@ -664,9 +740,11 @@ def _insert_fuse_allreduce_ops(self): last_dtype = None # split the grad based on dtype and fused size for var in param_grads: - if len(segments) == 0 \ - or len(segments[-1]) == self.fuse_grad_size_in_num \ - or var.dtype != last_dtype: + if ( + len(segments) == 0 + or len(segments[-1]) == self.fuse_grad_size_in_num + or var.dtype != last_dtype + ): segments.append([var]) last_dtype = var.dtype else: @@ -677,45 +755,51 @@ def _insert_fuse_allreduce_ops(self): if self._is_optimizer_op(op): for segment in segments: # insert coalesce tensor - tmp_var = block.create_var(name=unique_name.generate( - 'FusedOutput_{}'.format(segment[0].name)), - dtype=segment[0].dtype, - persistable=False, - stop_gradient=True) + tmp_var = block.create_var( + name=unique_name.generate( + 'FusedOutput_{}'.format(segment[0].name) + ), + dtype=segment[0].dtype, + persistable=False, + stop_gradient=True, + ) fused_vars.append(tmp_var) - block._insert_op(idx, - type="coalesce_tensor", - inputs={"Input": segment}, - outputs={ - "Output": segment, - "FusedOutput": tmp_var - }, - attrs={ - "copy_data": True, - "use_align": True, - "dtype": segment[0].dtype, - self.op_role_key: OpRole.Backward - }) + block._insert_op( + idx, + type="coalesce_tensor", + inputs={"Input": segment}, + outputs={"Output": segment, "FusedOutput": tmp_var}, + attrs={ + "copy_data": True, + "use_align": True, + "dtype": segment[0].dtype, + self.op_role_key: OpRole.Backward, + }, + ) break # insert the allreduce_sum op for idx, op in enumerate(block.ops): if self._is_optimizer_op(op): for fused_var in fused_vars: - block._insert_op(idx, - type='c_allreduce_sum', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': False, - self.op_role_key: OpRole.Backward - }) - block._insert_op(idx, - type='c_sync_calc_stream', - inputs={'X': fused_var}, - outputs={'Out': fused_var}, - attrs={self.op_role_key: OpRole.Backward}) + block._insert_op( + idx, + type='c_allreduce_sum', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={ + 'ring_id': ring_id, + 'use_calc_stream': False, + self.op_role_key: OpRole.Backward, + }, + ) + block._insert_op( + idx, + type='c_sync_calc_stream', + inputs={'X': fused_var}, + outputs={'Out': fused_var}, + attrs={self.op_role_key: OpRole.Backward}, + ) break if len(fused_vars) == 0: @@ -725,13 +809,15 @@ def _insert_fuse_allreduce_ops(self): # insert the sync comm op for idx, op in enumerate(block.ops): if self._is_optimizer_op(op): - block._insert_op(idx, - type='c_sync_comm_stream', - inputs={'X': fused_vars[0]}, - outputs={'Out': fused_vars[0]}, - attrs={ - 'ring_id': ring_id, - self.op_role_key: OpRole.Backward - }) + block._insert_op( + idx, + type='c_sync_comm_stream', + inputs={'X': fused_vars[0]}, + outputs={'Out': fused_vars[0]}, + attrs={ + 'ring_id': ring_id, + self.op_role_key: OpRole.Backward, + }, + ) break block._sync_with_cpp() diff --git a/python/paddle/fluid/transpiler/details/checkport.py b/python/paddle/fluid/transpiler/details/checkport.py index 71c9565a362c6..c0d8cdc914502 100644 --- a/python/paddle/fluid/transpiler/details/checkport.py +++ b/python/paddle/fluid/transpiler/details/checkport.py @@ -38,8 +38,9 @@ def wait_server_ready(endpoints): not_ready_endpoints = [] for ep in endpoints: ip_port = ep.split(":") - with closing(socket.socket(socket.AF_INET, - socket.SOCK_STREAM)) as sock: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, 'SO_REUSEPORT'): @@ -51,8 +52,9 @@ def wait_server_ready(endpoints): not_ready_endpoints.append(ep) if not all_ok: sys.stderr.write("server not ready, wait 3 sec to retry...\n") - sys.stderr.write("not ready endpoints:" + str(not_ready_endpoints) + - "\n") + sys.stderr.write( + "not ready endpoints:" + str(not_ready_endpoints) + "\n" + ) sys.stderr.flush() time.sleep(3) else: diff --git a/python/paddle/fluid/transpiler/details/ufind.py b/python/paddle/fluid/transpiler/details/ufind.py index 0e30d0e3f9c57..fef6f24570c17 100644 --- a/python/paddle/fluid/transpiler/details/ufind.py +++ b/python/paddle/fluid/transpiler/details/ufind.py @@ -14,7 +14,7 @@ class UnionFind(object): - """ Union-find data structure. + """Union-find data structure. Union-find is a data structure that keeps track of a set of elements partitioned into a number of disjoint (non-overlapping) subsets. diff --git a/python/paddle/fluid/transpiler/details/vars_distributed.py b/python/paddle/fluid/transpiler/details/vars_distributed.py index f78559e6559c0..32a27ed33b0b0 100644 --- a/python/paddle/fluid/transpiler/details/vars_distributed.py +++ b/python/paddle/fluid/transpiler/details/vars_distributed.py @@ -35,14 +35,16 @@ class VarDistributed(object): the slice var's properties, such as type/shape/offset/endpoint. """ - def __init__(self, - origin_var, - slice_var, - is_slice=None, - block_id=None, - offset=None, - vtype=None, - endpoint=None): + def __init__( + self, + origin_var, + slice_var, + is_slice=None, + block_id=None, + offset=None, + vtype=None, + endpoint=None, + ): """ Args: origin_var(Variable|VarStruct): origin var properties @@ -85,8 +87,14 @@ def __init__(self, @staticmethod def __create_var_struct(var): - return VarStruct(var.name, var.shape, var.dtype, var.type, - var.lod_level, var.persistable) + return VarStruct( + var.name, + var.shape, + var.dtype, + var.type, + var.lod_level, + var.persistable, + ) @staticmethod def equal(var1, var2): @@ -97,26 +105,45 @@ def equal(var1, var2): """ assert isinstance(var1, VarStruct) and isinstance(var2, VarStruct) - return var1.name == var2.name and \ - var1.type == var2.type and \ - var1.shape == var2.shape and \ - var1.dtype == var2.dtype and \ - var1.lod_level == var2.lod_level and \ - var1.persistable == var2.persistable + return ( + var1.name == var2.name + and var1.type == var2.type + and var1.shape == var2.shape + and var1.dtype == var2.dtype + and var1.lod_level == var2.lod_level + and var1.persistable == var2.persistable + ) def __str__(self): - origin_var_str = "{name} : fluid.{type}.shape{shape}.astype({dtype})". \ - format(i="{", e="}", name=self.origin.name, type=self.origin.type, - shape=self.origin.shape, dtype=self.origin.dtype) - - slice_var_str = "{name} : fluid.{type}.shape{shape}.astype({dtype})" \ - ".slice({is_slice}).block({block_id}).offset({offset})". \ - format(i="{", e="}", name=self.slice.name, type=self.slice.type, - shape=self.slice.shape, dtype=self.slice.dtype, - is_slice=self.is_slice, block_id=self.block_id, offset=self.offset) + origin_var_str = ( + "{name} : fluid.{type}.shape{shape}.astype({dtype})".format( + i="{", + e="}", + name=self.origin.name, + type=self.origin.type, + shape=self.origin.shape, + dtype=self.origin.dtype, + ) + ) + + slice_var_str = ( + "{name} : fluid.{type}.shape{shape}.astype({dtype})" + ".slice({is_slice}).block({block_id}).offset({offset})".format( + i="{", + e="}", + name=self.slice.name, + type=self.slice.type, + shape=self.slice.shape, + dtype=self.slice.dtype, + is_slice=self.is_slice, + block_id=self.block_id, + offset=self.offset, + ) + ) return "var owned: {}, origin var: ( {} ), slice var: ( {} ), endpoint: {} ".format( - self.vtype, origin_var_str, slice_var_str, self.endpoint) + self.vtype, origin_var_str, slice_var_str, self.endpoint + ) class VarsDistributed(object): @@ -130,14 +157,16 @@ class VarsDistributed(object): def __init__(self): self.distributed_vars = [] - def add_distributed_var(self, - origin_var, - slice_var, - is_slice=None, - block_id=None, - offset=None, - vtype=None, - endpoint=None): + def add_distributed_var( + self, + origin_var, + slice_var, + is_slice=None, + block_id=None, + offset=None, + vtype=None, + endpoint=None, + ): """ add distributed var in this. @@ -153,8 +182,16 @@ def add_distributed_var(self, None """ self.distributed_vars.append( - VarDistributed(origin_var, slice_var, is_slice, block_id, offset, - vtype, endpoint)) + VarDistributed( + origin_var, + slice_var, + is_slice, + block_id, + offset, + vtype, + endpoint, + ) + ) def get_distributed_var_by_slice(self, var_name): """ @@ -177,12 +214,14 @@ def equal(var1, var2): Returns: bool: equal will return True else False """ - return var1.name == var2.name and \ - var1.type == var2.type and \ - var1.shape == var2.shape and \ - var1.dtype == var2.dtype and \ - var1.lod_level == var2.lod_level and \ - var1.persistable == var2.persistable + return ( + var1.name == var2.name + and var1.type == var2.type + and var1.shape == var2.shape + and var1.dtype == var2.dtype + and var1.lod_level == var2.lod_level + and var1.persistable == var2.persistable + ) def get_distributed_var_by_origin_and_ep(self, origin_var_name, endpoint): """ @@ -195,7 +234,10 @@ def get_distributed_var_by_origin_and_ep(self, origin_var_name, endpoint): VarDistributed: distributed var. """ for dist_var in self.distributed_vars: - if dist_var.origin.name == origin_var_name and dist_var.endpoint == endpoint: + if ( + dist_var.origin.name == origin_var_name + and dist_var.endpoint == endpoint + ): return dist_var return None diff --git a/python/paddle/fluid/transpiler/distribute_transpiler.py b/python/paddle/fluid/transpiler/distribute_transpiler.py index 670f54311225f..28dbf22153f77 100644 --- a/python/paddle/fluid/transpiler/distribute_transpiler.py +++ b/python/paddle/fluid/transpiler/distribute_transpiler.py @@ -40,8 +40,14 @@ from .ps_dispatcher import RoundRobin, PSDispatcher from .. import core, framework, unique_name, initializer -from ..framework import Program, default_main_program, \ - default_startup_program, Block, Parameter, grad_var_name +from ..framework import ( + Program, + default_main_program, + default_startup_program, + Block, + Parameter, + grad_var_name, +) from .details import wait_server_ready, UnionFind, VarStruct, VarsDistributed from .details import delete_ops, find_op_by_output_arg from ..distribute_lookup_table import find_distributed_lookup_table @@ -52,8 +58,9 @@ OP_NAME_SCOPE = "op_namescope" CLIP_OP_NAME_SCOPE = "@CLIP" OP_ROLE_VAR_ATTR_NAME = core.op_proto_and_checker_maker.kOpRoleVarAttrName() -RPC_OP_ROLE_ATTR_NAME = op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName( -) +RPC_OP_ROLE_ATTR_NAME = ( + op_role_attr_name +) = core.op_proto_and_checker_maker.kOpRoleAttrName() OPT_OP_ROLE_ATTR_VALUE = core.op_proto_and_checker_maker.OpRole.Optimize RPC_OP_ROLE_ATTR_VALUE = core.op_proto_and_checker_maker.OpRole.RPC DIST_OP_ROLE_ATTR_VALUE = core.op_proto_and_checker_maker.OpRole.Dist @@ -75,7 +82,6 @@ def log(*args): class VarBlock: - def __init__(self, varname, offset, size): self.varname = varname # NOTE: real offset is offset * size @@ -129,8 +135,9 @@ def slice_variable(var_list, slice_count, min_block_size): # update split_count after aligning split_count = int(math.ceil(var_numel / float(block_size))) for block_id in range(split_count): - curr_block_size = min(block_size, - var_numel - ((block_id) * block_size)) + curr_block_size = min( + block_size, var_numel - ((block_id) * block_size) + ) block = VarBlock(var.name, block_id, curr_block_size) blocks.append(str(block)) return blocks @@ -242,14 +249,16 @@ def sync_mode(self, value): class ServerRuntimeConfig(object): - def __init__(self): self._rpc_send_thread_num = int( - os.getenv("FLAGS_rpc_send_thread_num", "12")) + os.getenv("FLAGS_rpc_send_thread_num", "12") + ) self._rpc_get_thread_num = int( - os.getenv("FLAGS_rpc_get_thread_num", "12")) + os.getenv("FLAGS_rpc_get_thread_num", "12") + ) self._rpc_prefetch_thread_num = int( - os.getenv("FLAGS_rpc_prefetch_thread_num", "12")) + os.getenv("FLAGS_rpc_prefetch_thread_num", "12") + ) class DistributeTranspiler(object): @@ -337,8 +346,8 @@ def __init__(self, config=None): global PRINT_LOG if self.config.print_log: PRINT_LOG = True - assert (self.config.min_block_size >= 8192) - assert (self.config.split_method.__bases__[0] == PSDispatcher) + assert self.config.min_block_size >= 8192 + assert self.config.split_method.__bases__[0] == PSDispatcher self.counter_var = None def _set_server_config(self, server_config=None): @@ -351,12 +360,14 @@ def _set_server_config(self, server_config=None): "In DistributeTranspiler, server_config must be an instance of ServerRuntimeConfig" ) - def _transpile_nccl2(self, - trainer_id, - trainers, - current_endpoint, - startup_program=None, - wait_port=True): + def _transpile_nccl2( + self, + trainer_id, + trainers, + current_endpoint, + startup_program=None, + wait_port=True, + ): if not startup_program: startup_program = default_startup_program() if trainer_id >= 0: @@ -367,53 +378,55 @@ def _transpile_nccl2(self, wait_server_ready(worker_endpoints) nccl_id_var = startup_program.global_block().create_var( - name="NCCLID", persistable=True, type=core.VarDesc.VarType.RAW) + name="NCCLID", persistable=True, type=core.VarDesc.VarType.RAW + ) for i in range(1, self.config.nccl_comm_num): startup_program.global_block().create_var( name="NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) if self.config.use_hierarchical_allreduce: for i in range(0, self.config.nccl_comm_num): startup_program.global_block().create_var( name="Hierarchical_inter_NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().create_var( name="Hierarchical_exter_NCCLID_{}".format(i), persistable=True, - type=core.VarDesc.VarType.RAW) + type=core.VarDesc.VarType.RAW, + ) startup_program.global_block().append_op( type="gen_nccl_id", inputs={}, outputs={"NCCLID": nccl_id_var}, attrs={ - "trainers": - trainers.split(","), - "trainer_id": - trainer_id, - "nccl_comm_num": - self.config.nccl_comm_num, - "use_hierarchical_allreduce": - self.config.use_hierarchical_allreduce, - "hierarchical_allreduce_inter_nranks": - self.config.hierarchical_allreduce_inter_nranks - }) + "trainers": trainers.split(","), + "trainer_id": trainer_id, + "nccl_comm_num": self.config.nccl_comm_num, + "use_hierarchical_allreduce": self.config.use_hierarchical_allreduce, + "hierarchical_allreduce_inter_nranks": self.config.hierarchical_allreduce_inter_nranks, + }, + ) return nccl_id_var else: raise ValueError("must set trainer_id > 0") - def _transpile_collective(self, - collective_mode, - trainer_id, - trainers, - current_endpoint, - startup_program=None, - main_program=None, - wait_port=True): + def _transpile_collective( + self, + collective_mode, + trainer_id, + trainers, + current_endpoint, + startup_program=None, + main_program=None, + wait_port=True, + ): if isinstance(trainers, str): endpoints = trainers.split(",") elif isinstance(trainers, list): @@ -421,8 +434,10 @@ def _transpile_collective(self, elif collective_mode != "single_process_multi_thread": raise ValueError('invalid trainers config: ' + str(trainers)) - if len(endpoints - ) == 1 and collective_mode != "single_process_multi_thread": + if ( + len(endpoints) == 1 + and collective_mode != "single_process_multi_thread" + ): raise ValueError('invalid trainer number in distributed: 1') if startup_program is None: @@ -441,28 +456,34 @@ def _transpile_collective(self, else: raise ValueError('invalid collective_mode: %s' % collective_mode) - transpiler.transpile(startup_program=startup_program, - main_program=main_program, - rank=trainer_id, - endpoints=endpoints, - current_endpoint=current_endpoint, - wait_port=wait_port) + transpiler.transpile( + startup_program=startup_program, + main_program=main_program, + rank=trainer_id, + endpoints=endpoints, + current_endpoint=current_endpoint, + wait_port=wait_port, + ) def _get_all_remote_sparse_update_op(self, main_program): sparse_update_ops = [] sparse_update_op_types = ["lookup_table", "nce", "lookup_table_v2"] for op in main_program.global_block().ops: - if op.type in sparse_update_op_types and op.attr( - 'remote_prefetch') is True: + if ( + op.type in sparse_update_op_types + and op.attr('remote_prefetch') is True + ): sparse_update_ops.append(op) return sparse_update_ops - def _update_remote_sparse_update_op(self, program, - need_sparse_update_params): + def _update_remote_sparse_update_op( + self, program, need_sparse_update_params + ): for param_varname, attrs in need_sparse_update_params.items(): height_sections = self.sparse_param_to_height_sections[ - param_varname] + param_varname + ] endpoints = attrs[0] table_names = attrs[1] @@ -518,10 +539,7 @@ def _update_remote_sparse_update_op(self, program, program.global_block()._insert_op( index=distributed_idx, type="distributed_lookup_table", - inputs={ - "Ids": inputs, - 'W': w - }, + inputs={"Ids": inputs, 'W': w}, outputs={"Outputs": outputs}, attrs={ "table_names": table_names, @@ -529,8 +547,9 @@ def _update_remote_sparse_update_op(self, program, "endpoints": endpoints, "padding_idx": padding_idx, "trainer_id": self.trainer_id, - "lookup_table_version": op_type - }) + "lookup_table_version": op_type, + }, + ) else: raise ValueError( "something wrong with distribute_transpiler, submit a issue is recommended" @@ -545,14 +564,16 @@ def _is_input_of_remote_sparse_update_op(self, param_name): return True return False - def transpile(self, - trainer_id, - program=None, - pservers="127.0.0.1:6174", - trainers=1, - sync_mode=True, - startup_program=None, - current_endpoint="127.0.0.1:6174"): + def transpile( + self, + trainer_id, + program=None, + pservers="127.0.0.1:6174", + trainers=1, + sync_mode=True, + startup_program=None, + current_endpoint="127.0.0.1:6174", + ): """ Transpile the input program to distributed programs with config and arguments. @@ -604,34 +625,49 @@ def transpile(self, self.origin_startup_program = self.startup_program.clone() if self.config.mode == "nccl2": - assert (isinstance(trainers, str)) + assert isinstance(trainers, str) self.origin_program._trainers_endpoints = trainers.split(",") self.origin_program._nccl_comm_num = self.config.nccl_comm_num - self.origin_program._use_hierarchical_allreduce = self.config.use_hierarchical_allreduce + self.origin_program._use_hierarchical_allreduce = ( + self.config.use_hierarchical_allreduce + ) # check use_hierarchical_allreduce options if self.config.use_hierarchical_allreduce: trainers_num = len(self.origin_program._trainers_endpoints) # selected automaticly if self.config.hierarchical_allreduce_inter_nranks <= 1: - self.config.hierarchical_allreduce_inter_nranks = core.get_cuda_device_count( + self.config.hierarchical_allreduce_inter_nranks = ( + core.get_cuda_device_count() ) - assert trainers_num > self.config.hierarchical_allreduce_inter_nranks, \ - "trainers_num:{} < hierarchical_allreduce_inter_nranks:{}".format( - trainers_num, self.config.hierarchical_allreduce_inter_nranks) - - assert trainers_num % self.config.hierarchical_allreduce_inter_nranks == 0, \ - "trainers_num:{} mod hierarchical_allreduce_inter_nranks:{} != 0".format( - trainers_num, self.config.hierarchical_allreduce_inter_nranks) - - self.origin_program._hierarchical_allreduce_inter_nranks = \ - int(self.config.hierarchical_allreduce_inter_nranks) + assert ( + trainers_num + > self.config.hierarchical_allreduce_inter_nranks + ), "trainers_num:{} < hierarchical_allreduce_inter_nranks:{}".format( + trainers_num, + self.config.hierarchical_allreduce_inter_nranks, + ) + + assert ( + trainers_num + % self.config.hierarchical_allreduce_inter_nranks + == 0 + ), "trainers_num:{} mod hierarchical_allreduce_inter_nranks:{} != 0".format( + trainers_num, + self.config.hierarchical_allreduce_inter_nranks, + ) + + self.origin_program._hierarchical_allreduce_inter_nranks = int( + self.config.hierarchical_allreduce_inter_nranks + ) - self._transpile_nccl2(trainer_id, - trainers, - current_endpoint, - startup_program=startup_program, - wait_port=self.config.wait_port) + self._transpile_nccl2( + trainer_id, + trainers, + current_endpoint, + startup_program=startup_program, + wait_port=self.config.wait_port, + ) return if self.config.mode == "collective": @@ -642,7 +678,8 @@ def transpile(self, current_endpoint=current_endpoint, startup_program=startup_program, main_program=program, - wait_port=self.config.wait_port) + wait_port=self.config.wait_port, + ) return self.trainer_num = trainers @@ -664,7 +701,8 @@ def transpile(self, # get all sparse update ops self.sparse_update_ops = self._get_all_remote_sparse_update_op( - self.origin_program) + self.origin_program + ) # use_sparse_update_param_name -> split_height_section self.sparse_param_to_height_sections = dict() self.need_delete_optimize_vars = [] @@ -674,7 +712,9 @@ def transpile(self, self.origin_program._endpoints = self.pserver_endpoints self.origin_program._ps_endpoint = current_endpoint self.origin_program._is_chief = self.trainer_id == 0 - self.origin_program._distributed_lookup_table = self.table_name if self.table_name else None + self.origin_program._distributed_lookup_table = ( + self.table_name if self.table_name else None + ) # split and create vars, then put split vars in dicts for later use. # step 1: split and create vars, then put split vars in dicts for later use. @@ -701,29 +741,31 @@ def transpile(self, eplist = ps_dispatcher.dispatch(splited_vars) if not self.config.slice_var_up: - assert (len(splited_vars) == 1) + assert len(splited_vars) == 1 splited_grad_varname = grad_varname if len(splited_vars) == 1: splited_grad_varname = splited_vars[0].name - index = find_op_by_output_arg(program.global_block(), - splited_grad_varname, - reverse=True) + index = find_op_by_output_arg( + program.global_block(), splited_grad_varname, reverse=True + ) elif len(splited_vars) > 1: orig_var = program.global_block().vars[splited_grad_varname] - index = find_op_by_output_arg(program.global_block(), - splited_grad_varname, - reverse=True) + index = find_op_by_output_arg( + program.global_block(), splited_grad_varname, reverse=True + ) if not self.config.runtime_split_send_recv: - self._insert_split_op(program, orig_var, index, - splited_vars) + self._insert_split_op( + program, orig_var, index, splited_vars + ) index += 1 else: AssertionError( - "Can not insert the send op by original " - "variable name :", splited_grad_varname) + "Can not insert the send op by original " "variable name :", + splited_grad_varname, + ) if splited_vars[0].type == core.VarDesc.VarType.SELECTED_ROWS: sparse_param_name = self.grad_name_to_param_name[grad_varname] @@ -733,7 +775,8 @@ def transpile(self, ] dummy_output = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) self.grad_name_to_send_dummy_out[grad_varname] = dummy_output if self.config.runtime_split_send_recv: @@ -764,35 +807,36 @@ def transpile(self, inputs={"X": send_input_vars}, outputs={"Out": dummy_output}, attrs={ - "epmap": - eplist, - "sections": - sections, - "send_varnames": - send_varnames, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE, + "epmap": eplist, + "sections": sections, + "send_varnames": send_varnames, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, OP_ROLE_VAR_ATTR_NAME: [ self.grad_name_to_param_name[grad_varname], - splited_grad_varname - ] - }) + splited_grad_varname, + ], + }, + ) for _, var in enumerate(splited_vars): send_vars.append(var) send_barrier_out = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) if self.has_distributed_lookup_table: self.grad_name_to_send_dummy_out[ - self.table_name] = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) + self.table_name + ] = program.global_block().create_var( + name=framework.generate_control_dev_var_name() + ) input_deps = list(self.grad_name_to_send_dummy_out.values()) if not self.sync_mode: lr_ops = self._get_lr_ops() if len(lr_ops) > 0 and self.counter_var: decay_dummy_output = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) if self.config.runtime_split_send_recv: # async mode, using communicator to merge and send send_varnames = [self.counter_var.name] @@ -804,39 +848,34 @@ def transpile(self, inputs={"X": self.counter_var}, outputs={"Out": decay_dummy_output}, attrs={ - "epmap": - pserver_endpoints, - "sections": - sections, - "send_varnames": - send_varnames, - "merge_add": - True, - "use_send_handler": - False, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE, - OP_ROLE_VAR_ATTR_NAME: - [self.counter_var.name, self.counter_var.name] - }) + "epmap": pserver_endpoints, + "sections": sections, + "send_varnames": send_varnames, + "merge_add": True, + "use_send_handler": False, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + OP_ROLE_VAR_ATTR_NAME: [ + self.counter_var.name, + self.counter_var.name, + ], + }, + ) input_deps.append(decay_dummy_output) if self.sync_mode: fetch_barrier_input = [] - program.global_block().append_op(type="send_barrier", - inputs={"X": list(input_deps)}, - outputs={"Out": send_barrier_out}, - attrs={ - "endpoints": - pserver_endpoints, - "trainer_id": - self.trainer_id, - "half_async": - False, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) + program.global_block().append_op( + type="send_barrier", + inputs={"X": list(input_deps)}, + outputs={"Out": send_barrier_out}, + attrs={ + "endpoints": pserver_endpoints, + "trainer_id": self.trainer_id, + "half_async": False, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) fetch_barrier_input.append(send_barrier_out) else: @@ -849,8 +888,9 @@ def transpile(self, "endpoints": pserver_endpoints, "trainer_id": self.trainer_id, "half_async": True, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) # step 3: insert recv op to receive parameters from parameter server recv_vars = [] @@ -864,7 +904,8 @@ def transpile(self, self.param_grad_ep_mapping[ep]["grads"].append(send_vars[i]) distributed_var = self.vars_overview.get_distributed_var_by_slice( - recv_vars[i].name) + recv_vars[i].name + ) distributed_var.endpoint = ep need_sparse_update_params = {} @@ -883,7 +924,8 @@ def transpile(self, else: # connect deps to send op in async mode recv_dep_in = self.grad_name_to_send_dummy_out[ - self.param_name_to_grad_name[param_varname]] + self.param_name_to_grad_name[param_varname] + ] # get recv op_role_var, if not split, the grad should have .trainer suffix # if split, grad should be the original grad var name. ParallelExecutor @@ -896,8 +938,11 @@ def transpile(self, if param_varname in self.sparse_param_to_height_sections: for table_name in table_names: - distributed_var = self.vars_overview.get_distributed_var_by_slice( - table_name) + distributed_var = ( + self.vars_overview.get_distributed_var_by_slice( + table_name + ) + ) distributed_var.vtype = "RemotePrefetch" need_sparse_update_params[param_varname] = (eps, table_names) @@ -914,33 +959,31 @@ def transpile(self, inputs={"X": [recv_dep_in]}, outputs={"Out": splited_var}, attrs={ - "epmap": - eps, - "recv_varnames": - recv_varnames, - "trainer_id": - self.trainer_id, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE, - OP_ROLE_VAR_ATTR_NAME: - [param_varname, recv_op_role_var_name] - }) + "epmap": eps, + "recv_varnames": recv_varnames, + "trainer_id": self.trainer_id, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + OP_ROLE_VAR_ATTR_NAME: [ + param_varname, + recv_op_role_var_name, + ], + }, + ) self._update_remote_sparse_update_op(program, need_sparse_update_params) if self.sync_mode: # form a WAW dependency - program.global_block().append_op(type="fetch_barrier", - inputs={"X": fetch_barrier_input}, - outputs={"Out": all_recv_outputs}, - attrs={ - "endpoints": - pserver_endpoints, - "trainer_id": - self.trainer_id, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE - }) + program.global_block().append_op( + type="fetch_barrier", + inputs={"X": fetch_barrier_input}, + outputs={"Out": all_recv_outputs}, + attrs={ + "endpoints": pserver_endpoints, + "trainer_id": self.trainer_id, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) for param_varname, splited_var in self.param_var_mapping.items(): if len(splited_var) <= 1: @@ -954,14 +997,16 @@ def transpile(self, outputs={"Out": [orig_param]}, attrs={ "axis": 0, - RPC_OP_ROLE_ATTR_NAME: DIST_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: DIST_OP_ROLE_ATTR_VALUE, + }, + ) self._get_trainer_startup_program(recv_vars=recv_vars, eplist=eplist) if self.has_distributed_lookup_table: - self._replace_lookup_table_op_with_prefetch(program, - pserver_endpoints) + self._replace_lookup_table_op_with_prefetch( + program, pserver_endpoints + ) self._split_table_grad_and_add_send_vars(program, pserver_endpoints) self._get_distributed_optimizer_vars() @@ -972,8 +1017,10 @@ def _get_sparse_table_names(self): sparse_table_names = [] for op in self.origin_program.global_block().ops: - if op.type in sparse_update_op_types and op.attr( - 'is_sparse') is True: + if ( + op.type in sparse_update_op_types + and op.attr('is_sparse') is True + ): sparse_table_names.append(op.input("W")[0]) if op.type == "distributed_lookup_table": sparse_table_names.append(op.input("W")[0]) @@ -993,14 +1040,16 @@ def _fake_init_sparsetable(self, sparse_table_names): table_param_init_op.append(op) init_op_num = len(table_param_init_op) if init_op_num != 1: - raise ValueError("table init op num should be 1, now is " + - str(init_op_num)) + raise ValueError( + "table init op num should be 1, now is " + str(init_op_num) + ) table_init_op = table_param_init_op[0] self.startup_program.global_block().append_op( type="fake_init", inputs={}, outputs={"Out": table_var}, - attrs={"shape": table_init_op.attr('shape')}) + attrs={"shape": table_init_op.attr('shape')}, + ) delete_ops(self.startup_program.global_block(), table_param_init_op) def _delete_trainer_optimizer(self, is_startup): @@ -1124,7 +1173,8 @@ def _get_trainer_startup_program(self, recv_vars, eplist): type=var.type, dtype=var.dtype, shape=var.shape, - lod_level=var.lod_level) + lod_level=var.lod_level, + ) op = startup_program.global_block().append_op( type="recv", @@ -1133,11 +1183,13 @@ def _get_trainer_startup_program(self, recv_vars, eplist): attrs={ "epmap": eps, "trainer_id": self.trainer_id, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) fetch_barrier_out = startup_program.global_block().create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) startup_program.global_block().append_op( type="fetch_barrier", inputs={}, @@ -1145,8 +1197,9 @@ def _get_trainer_startup_program(self, recv_vars, eplist): attrs={ "endpoints": self.pserver_endpoints, "trainer_id": self.trainer_id, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + }, + ) for varname, splited_var in self.param_var_mapping.items(): if varname in sparse_table_names: @@ -1158,19 +1211,22 @@ def _get_trainer_startup_program(self, recv_vars, eplist): if varname in startup_program.global_block().vars: orig_param = startup_program.global_block().vars[varname] else: - origin_param_var = self.origin_program.global_block( - ).vars[varname] + origin_param_var = self.origin_program.global_block().vars[ + varname + ] orig_param = startup_program.global_block().create_var( name=varname, persistable=origin_param_var.persistable, type=origin_param_var.type, dtype=origin_param_var.dtype, - shape=origin_param_var.shape) + shape=origin_param_var.shape, + ) startup_program.global_block().append_op( type="concat", inputs={"X": splited_var}, outputs={"Out": [orig_param]}, - attrs={"axis": 0}) + attrs={"axis": 0}, + ) return startup_program @@ -1231,21 +1287,26 @@ def get_pserver_program(self, endpoint): orig_var_name = v.name # NOTE: single_trainer_var must be created for multi-trainer # case to merge grads from multiple trainers - single_trainer_var = \ - pserver_program.global_block().create_var( - name=orig_var_name, - persistable=True, - type=v.type, - dtype=v.dtype, - shape=v.shape) - if self.sync_mode or self.config.completely_not_async and self.trainer_num > 1: + single_trainer_var = pserver_program.global_block().create_var( + name=orig_var_name, + persistable=True, + type=v.type, + dtype=v.dtype, + shape=v.shape, + ) + if ( + self.sync_mode + or self.config.completely_not_async + and self.trainer_num > 1 + ): for trainer_id in range(self.trainer_num): var = pserver_program.global_block().create_var( name="%s.trainer_%d" % (orig_var_name, trainer_id), persistable=False, type=v.type, dtype=v.dtype, - shape=v.shape) + shape=v.shape, + ) recv_inputs.append(var) else: recv_inputs.append(single_trainer_var) @@ -1261,12 +1322,13 @@ def get_pserver_program(self, endpoint): opt_op_on_pserver = [] for _, op in enumerate(self.optimize_ops): if self._is_optimizer_op(op) and self._is_opt_op_on_pserver( - endpoint, op): + endpoint, op + ): opt_op_on_pserver.append(op) # step 3.3 # prepare if dc asgd is enabled if self.config.enable_dc_asgd == True: - assert (self.sync_mode == False) + assert self.sync_mode == False self.param_bak_list = [] # add param_bak for each trainer for p in self.param_grad_ep_mapping[endpoint]["params"]: @@ -1278,7 +1340,8 @@ def get_pserver_program(self, endpoint): name=param_bak_name, type=p.type, shape=p.shape, - dtype=p.dtype) + dtype=p.dtype, + ) self.param_bak_list.append((p, tmpvar)) # step 3.4 @@ -1291,12 +1354,19 @@ def get_pserver_program(self, endpoint): # sparse grad name to param name sparse_grad_to_param = [] - def __append_optimize_op__(op, block, grad_to_block_id, merged_var, - lr_ops): + def __append_optimize_op__( + op, block, grad_to_block_id, merged_var, lr_ops + ): if self._is_optimizer_op(op): - self._append_pserver_ops(block, op, endpoint, grad_to_block_id, - self.origin_program, merged_var, - sparse_grad_to_param) + self._append_pserver_ops( + block, + op, + endpoint, + grad_to_block_id, + self.origin_program, + merged_var, + sparse_grad_to_param, + ) elif op not in lr_ops: self._append_pserver_non_opt_ops(block, op) @@ -1332,13 +1402,15 @@ def __clone_lr_op_sub_block__(op, program, lr_block): lr_decay_block_id = -1 if len(lr_ops) > 0: lr_decay_block = pserver_program._create_block( - pserver_program.num_blocks - 1) + pserver_program.num_blocks - 1 + ) optimize_blocks.append(lr_decay_block) for _, op in enumerate(lr_ops): cloned_op = self._append_pserver_non_opt_ops(lr_decay_block, op) # append sub blocks to pserver_program in lr_decay_op - __clone_lr_op_sub_block__(cloned_op, pserver_program, - lr_decay_block) + __clone_lr_op_sub_block__( + cloned_op, pserver_program, lr_decay_block + ) lr_decay_block_id = lr_decay_block.idx # append op to the current block @@ -1355,54 +1427,78 @@ def __clone_lr_op_sub_block__(op, program, lr_block): # find the origin grad var before clipping/L2Decay, # merged_var should be the input var name of L2Decay grad_varname_for_block = op.attr(OP_ROLE_VAR_ATTR_NAME)[1] - if op.attr( - OP_ROLE_VAR_ATTR_NAME)[0] == optimize_target_param_name: + if ( + op.attr(OP_ROLE_VAR_ATTR_NAME)[0] + == optimize_target_param_name + ): merged_var = self._append_pserver_grad_merge_ops( - per_opt_block, grad_varname_for_block, endpoint, - grad_to_block_id, self.origin_program) + per_opt_block, + grad_varname_for_block, + endpoint, + grad_to_block_id, + self.origin_program, + ) if merged_var: break # append optimize op once then append other ops. if merged_var: for _, op in enumerate(self.optimize_ops): # optimizer is connected to itself - if op.attr(OP_ROLE_VAR_ATTR_NAME)[0] == optimize_target_param_name and \ - op not in global_ops: - log("append opt op: ", op.type, op.input_arg_names, - merged_var) - __append_optimize_op__(op, per_opt_block, - grad_to_block_id, merged_var, - lr_ops) + if ( + op.attr(OP_ROLE_VAR_ATTR_NAME)[0] + == optimize_target_param_name + and op not in global_ops + ): + log( + "append opt op: ", + op.type, + op.input_arg_names, + merged_var, + ) + __append_optimize_op__( + op, + per_opt_block, + grad_to_block_id, + merged_var, + lr_ops, + ) # dedup grad to ids list grad_to_block_id = list(set(grad_to_block_id)) # append global ops if global_ops: opt_state_block = pserver_program._create_block( - pserver_program.num_blocks - 1) + pserver_program.num_blocks - 1 + ) optimize_blocks.append(opt_state_block) for glb_op in global_ops: - __append_optimize_op__(glb_op, opt_state_block, - grad_to_block_id, None, lr_ops) + __append_optimize_op__( + glb_op, opt_state_block, grad_to_block_id, None, lr_ops + ) # process distributed lookup_table prefetch_var_name_to_block_id = [] if self.has_distributed_lookup_table: pserver_index = self.pserver_endpoints.index(endpoint) table_opt_block = self._create_table_optimize_block( - pserver_index, pserver_program, pre_block_idx, grad_to_block_id) + pserver_index, pserver_program, pre_block_idx, grad_to_block_id + ) optimize_blocks.append(table_opt_block) lookup_table_var_name_to_block_id = self._create_prefetch_block( - pserver_index, pserver_program, table_opt_block) + pserver_index, pserver_program, table_opt_block + ) checkpoint_block_id = self._create_checkpoint_save_block( - pserver_program, table_opt_block.idx) + pserver_program, table_opt_block.idx + ) pserver_program._distributed_lookup_table = self.table_name prefetch_var_name_to_block_id.extend( - lookup_table_var_name_to_block_id) + lookup_table_var_name_to_block_id + ) if len(optimize_blocks) == 0: - logging.warn("pserver [" + str(endpoint) + - "] has no optimize block!!") + logging.warn( + "pserver [" + str(endpoint) + "] has no optimize block!!" + ) pre_block_idx = pserver_program.num_blocks - 1 empty_block = pserver_program._create_block(pre_block_idx) optimize_blocks.append(empty_block) @@ -1420,8 +1516,7 @@ def __clone_lr_op_sub_block__(op, program, lr_block): "lr_decay_block_id": lr_decay_block_id, "rpc_get_thread_num": self.server_config._rpc_get_thread_num, "rpc_send_thread_num": self.server_config._rpc_send_thread_num, - "rpc_prefetch_thread_num": - self.server_config._rpc_prefetch_thread_num + "rpc_prefetch_thread_num": self.server_config._rpc_prefetch_thread_num, } if self.has_distributed_lookup_table: @@ -1431,13 +1526,16 @@ def __clone_lr_op_sub_block__(op, program, lr_block): if len(prefetch_var_name_to_block_id) > 0: attrs[ - 'prefetch_var_name_to_block_id'] = prefetch_var_name_to_block_id + 'prefetch_var_name_to_block_id' + ] = prefetch_var_name_to_block_id # step5 append the listen_and_serv op - pserver_program.global_block().append_op(type="listen_and_serv", - inputs={'X': recv_inputs}, - outputs={}, - attrs=attrs) + pserver_program.global_block().append_op( + type="listen_and_serv", + inputs={'X': recv_inputs}, + outputs={}, + attrs=attrs, + ) pserver_program._sync_with_cpp() # save pserver program to generate pserver side startup relatively. @@ -1471,14 +1569,14 @@ def get_pserver_programs(self, endpoint): pserver_program, pserver_startup_program = t.get_pserver_programs(current_endpoint) """ pserver_prog = self.get_pserver_program(endpoint) - pserver_startup = self.get_startup_program(endpoint, - pserver_program=pserver_prog) + pserver_startup = self.get_startup_program( + endpoint, pserver_program=pserver_prog + ) return pserver_prog, pserver_startup - def get_startup_program(self, - endpoint, - pserver_program=None, - startup_program=None): + def get_startup_program( + self, endpoint, pserver_program=None, startup_program=None + ): """ **Deprecated** @@ -1550,22 +1648,28 @@ def _get_splited_name_and_shape(varname): new_inputs = self._get_input_map_from_op(pserver_vars, op) if op.type in [ - "gaussian_random", "fill_constant", "uniform_random", - "truncated_gaussian_random" + "gaussian_random", + "fill_constant", + "uniform_random", + "truncated_gaussian_random", ]: op._set_attr("shape", list(new_outputs["Out"].shape)) - s_prog.global_block().append_op(type=op.type, - inputs=new_inputs, - outputs=new_outputs, - attrs=op.all_attrs()) + s_prog.global_block().append_op( + type=op.type, + inputs=new_inputs, + outputs=new_outputs, + attrs=op.all_attrs(), + ) if self.config.enable_dc_asgd: for p, p_bak in self.param_bak_list: startup_param_var = s_prog.global_block().vars[p.name] startup_tmpvar = s_prog.global_block().vars[p_bak.name] # copy init random value to param_bak - s_prog.global_block().append_op(type="assign", - inputs={"X": startup_param_var}, - outputs={"Out": startup_tmpvar}) + s_prog.global_block().append_op( + type="assign", + inputs={"X": startup_param_var}, + outputs={"Out": startup_tmpvar}, + ) return s_prog @@ -1588,8 +1692,9 @@ def _get_slice_var_info(self, slice_var): orig_dim1_flatten = 1 if len(slice_vars[0].shape) >= 2: - orig_dim1_flatten = reduce(lambda x, y: x * y, - slice_vars[0].shape[1:]) + orig_dim1_flatten = reduce( + lambda x, y: x * y, slice_vars[0].shape[1:] + ) for slice_var in slice_vars[:block_idx]: skip_dim0 += slice_var.shape[0] @@ -1599,12 +1704,12 @@ def _get_slice_var_info(self, slice_var): return is_slice, block_idx, offset def _get_distributed_optimizer_vars(self): - def _get_distributed_optimizer_var(endpoint): opt_op_on_pserver = [] for _, op in enumerate(self.optimize_ops): if self._is_optimizer_op(op) and self._is_opt_op_on_pserver( - endpoint, op): + endpoint, op + ): opt_op_on_pserver.append(op) for opt_op in opt_op_on_pserver: @@ -1613,20 +1718,25 @@ def _get_distributed_optimizer_var(endpoint): if key == "Param": param_name = opt_op.input(key)[0] dist_var = self.vars_overview.get_distributed_var_by_origin_and_ep( - param_name, endpoint) + param_name, endpoint + ) break for key in opt_op.input_names: if key in [ - "Param", "Grad", "LearningRate", "Beta1Tensor", - "Beta2Tensor" + "Param", + "Grad", + "LearningRate", + "Beta1Tensor", + "Beta2Tensor", ]: continue origin_var = self.origin_program.global_block().vars[ - opt_op.input(key)[0]] + opt_op.input(key)[0] + ] # update accumulator variable shape new_shape = self._get_optimizer_input_shape( - opt_op.type, key, origin_var.shape, - dist_var.slice.shape) + opt_op.type, key, origin_var.shape, dist_var.slice.shape + ) if new_shape == dist_var.slice.shape: splited_var = VarStruct( @@ -1635,7 +1745,8 @@ def _get_distributed_optimizer_var(endpoint): dtype=origin_var.dtype, type=origin_var.type, lod_level=origin_var.lod_level, - persistable=origin_var.persistable) + persistable=origin_var.persistable, + ) self.vars_overview.add_distributed_var( origin_var=origin_var, @@ -1644,7 +1755,8 @@ def _get_distributed_optimizer_var(endpoint): block_id=dist_var.block_id, offset=dist_var.offset, vtype="Optimizer", - endpoint=endpoint) + endpoint=endpoint, + ) else: self.vars_overview.add_distributed_var( origin_var=origin_var, @@ -1653,13 +1765,15 @@ def _get_distributed_optimizer_var(endpoint): block_id=0, offset=0, vtype="Optimizer", - endpoint=endpoint) + endpoint=endpoint, + ) for ep in self.pserver_endpoints: _get_distributed_optimizer_var(ep) - def _update_dist_lookup_table_vars(self, param_list, grad_list, - params_grads): + def _update_dist_lookup_table_vars( + self, param_list, grad_list, params_grads + ): # TODO(wuyi): put find a way to put dist lookup table stuff all together. # update self.table_param_grad and self.trainer_side_table_grad_list program = self.origin_program @@ -1668,22 +1782,25 @@ def _update_dist_lookup_table_vars(self, param_list, grad_list, param for param in param_list if param.name != self.table_name ] grad_list = [ - grad for grad in grad_list + grad + for grad in grad_list if grad.name != grad_var_name(self.table_name) ] self.table_param_grad = [ - param_grad for param_grad in params_grads + param_grad + for param_grad in params_grads if param_grad[0].name == self.table_name ][0] table_grad_var = self.table_param_grad[1] if self.sync_mode: self.trainer_side_table_grad_list = [ program.global_block().create_var( - name="%s.trainer_%d.pserver_%d" % - (table_grad_var.name, self.trainer_id, index), + name="%s.trainer_%d.pserver_%d" + % (table_grad_var.name, self.trainer_id, index), type=table_grad_var.type, shape=table_grad_var.shape, - dtype=table_grad_var.dtype) + dtype=table_grad_var.dtype, + ) for index in range(len(self.pserver_endpoints)) ] else: @@ -1692,7 +1809,8 @@ def _update_dist_lookup_table_vars(self, param_list, grad_list, name="%s.pserver_%d" % (table_grad_var.name, index), type=table_grad_var.type, shape=table_grad_var.shape, - dtype=table_grad_var.dtype) + dtype=table_grad_var.dtype, + ) for index in range(len(self.pserver_endpoints)) ] return param_list, grad_list @@ -1719,68 +1837,81 @@ def _init_splited_vars(self): param_grad_set.add(g.name) param_list, grad_list = self._update_dist_lookup_table_vars( - param_list, grad_list, self.params_grads) + param_list, grad_list, self.params_grads + ) if self.config.slice_var_up: # when we slice var up into blocks, we will slice the var according to # pserver services' count. A pserver may have two or more listening ports. - grad_blocks = slice_variable(grad_list, len(self.pserver_endpoints), - self.config.min_block_size) - param_blocks = slice_variable(param_list, - len(self.pserver_endpoints), - self.config.min_block_size) + grad_blocks = slice_variable( + grad_list, + len(self.pserver_endpoints), + self.config.min_block_size, + ) + param_blocks = slice_variable( + param_list, + len(self.pserver_endpoints), + self.config.min_block_size, + ) else: # when we do NOT slice var up into blocks, we will always slice params # grads into one block. - grad_blocks = slice_variable(grad_list, 1, - self.config.min_block_size) - param_blocks = slice_variable(param_list, 1, - self.config.min_block_size) - assert (len(grad_blocks) == len(param_blocks)) + grad_blocks = slice_variable( + grad_list, 1, self.config.min_block_size + ) + param_blocks = slice_variable( + param_list, 1, self.config.min_block_size + ) + assert len(grad_blocks) == len(param_blocks) # origin_param_name -> [splited_param_vars] self.param_var_mapping = self._create_vars_from_blocklist( - self.origin_program, param_blocks) + self.origin_program, param_blocks + ) for orig_name, splited_vars in self.param_var_mapping.items(): orig_var = self.origin_program.global_block().var(orig_name) for splited_var in splited_vars: is_slice, block_id, offset = self._get_slice_var_info( - splited_var) - - self.vars_overview.add_distributed_var(origin_var=orig_var, - slice_var=splited_var, - block_id=block_id, - offset=offset, - is_slice=is_slice, - vtype="Param") + splited_var + ) + + self.vars_overview.add_distributed_var( + origin_var=orig_var, + slice_var=splited_var, + block_id=block_id, + offset=offset, + is_slice=is_slice, + vtype="Param", + ) # origin_grad_name -> [splited_grad_vars] self.grad_var_mapping = self._create_vars_from_blocklist( self.origin_program, grad_blocks, - add_trainer_suffix=self.trainer_num > 1) + add_trainer_suffix=self.trainer_num > 1, + ) # dict(grad_splited_var -> param_splited_var) self.grad_param_mapping = collections.OrderedDict() for g, p in zip(grad_blocks, param_blocks): g_name, g_bid, _ = g.split(":") p_name, p_bid, _ = p.split(":") - self.grad_param_mapping[self.grad_var_mapping[g_name][int(g_bid)]] = \ - self.param_var_mapping[p_name][int(p_bid)] + self.grad_param_mapping[ + self.grad_var_mapping[g_name][int(g_bid)] + ] = self.param_var_mapping[p_name][int(p_bid)] # create mapping of endpoint -> split var to create pserver side program self.param_grad_ep_mapping = collections.OrderedDict() [ - self.param_grad_ep_mapping.update({ep: { - "params": [], - "grads": [] - }}) for ep in self.pserver_endpoints + self.param_grad_ep_mapping.update({ep: {"params": [], "grads": []}}) + for ep in self.pserver_endpoints ] # transpiler function for dis lookup_table - def _replace_lookup_table_op_with_prefetch(self, program, - pserver_endpoints): + def _replace_lookup_table_op_with_prefetch( + self, program, pserver_endpoints + ): # 1. replace lookup_table_op with split_ids_op -> prefetch_op -> sum_op self.all_in_ids_vars = [] self.all_prefetch_input_vars = [] @@ -1793,16 +1924,22 @@ def _replace_lookup_table_op_with_prefetch(self, program, continue_search_lookup_table_op = False all_ops = program.global_block().ops for op in all_ops: - if op.type == LOOKUP_TABLE_TYPE and self.table_name == op.input( - "W")[0]: + if ( + op.type == LOOKUP_TABLE_TYPE + and self.table_name == op.input("W")[0] + ): if not op.attr('is_distributed'): raise RuntimeError( "lookup_table_op that lookup an distributed embedding table" - "should set is_distributed to true") + "should set is_distributed to true" + ) continue_search_lookup_table_op = True - lookup_table_op_index = lookup_table_op_index if lookup_table_op_index != -1 else list( - all_ops).index(op) + lookup_table_op_index = ( + lookup_table_op_index + if lookup_table_op_index != -1 + else list(all_ops).index(op) + ) ids_name = op.input("Ids") out_name = op.output("Out") @@ -1822,14 +1959,16 @@ def _replace_lookup_table_op_with_prefetch(self, program, name=str("prefetch_compress_in_tmp_" + str(index)), type=self.all_in_ids_vars[0].type, shape=self.all_in_ids_vars[0].shape, - dtype=self.all_in_ids_vars[0].dtype) + dtype=self.all_in_ids_vars[0].dtype, + ) self.all_prefetch_input_vars.append(in_var) out_var = program.global_block().create_var( name=str("prefetch_compress_out_tmp_" + str(index)), type=self.all_out_emb_vars[0].type, shape=self.all_out_emb_vars[0].shape, - dtype=self.all_out_emb_vars[0].dtype) + dtype=self.all_out_emb_vars[0].dtype, + ) self.all_prefetch_output_vars.append(out_var) # insert split_ids_op @@ -1837,7 +1976,8 @@ def _replace_lookup_table_op_with_prefetch(self, program, index=lookup_table_op_index, type="split_ids", inputs={'Ids': self.all_in_ids_vars}, - outputs={"Out": self.all_prefetch_input_vars}) + outputs={"Out": self.all_prefetch_input_vars}, + ) # insert prefetch_op program.global_block()._insert_op( @@ -1850,7 +1990,8 @@ def _replace_lookup_table_op_with_prefetch(self, program, # FIXME(qiao) temporarily disable this config because prefetch # is not act as other rpc op, it's more like a forward op # RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + }, + ) # insert concat_op program.global_block()._insert_op( @@ -1859,9 +2000,10 @@ def _replace_lookup_table_op_with_prefetch(self, program, inputs={ 'Ids': self.all_in_ids_vars, 'Rows': self.all_prefetch_input_vars, - 'X': self.all_prefetch_output_vars + 'X': self.all_prefetch_output_vars, }, - outputs={"Out": self.all_out_emb_vars}) + outputs={"Out": self.all_out_emb_vars}, + ) def _split_table_grad_and_add_send_vars(self, program, pserver_endpoints): # 2. add split_ids_op and send_op to send gradient to pservers @@ -1880,32 +2022,34 @@ def _split_table_grad_and_add_send_vars(self, program, pserver_endpoints): 'Ids': [program.global_block().vars[table_grad_name]] }, outputs={"Out": self.trainer_side_table_grad_list}, - attrs={RPC_OP_ROLE_ATTR_NAME: DIST_OP_ROLE_ATTR_VALUE}) + attrs={RPC_OP_ROLE_ATTR_NAME: DIST_OP_ROLE_ATTR_VALUE}, + ) program.global_block()._insert_op( index=op_index + 2, type="send", inputs={'X': self.trainer_side_table_grad_list}, outputs={ - 'Out': - [self.grad_name_to_send_dummy_out[self.table_name]] - if self.sync_mode else [] + 'Out': [ + self.grad_name_to_send_dummy_out[self.table_name] + ] + if self.sync_mode + else [] }, attrs={ - "epmap": - pserver_endpoints, - "trainer_id": - self.trainer_id, - RPC_OP_ROLE_ATTR_NAME: - RPC_OP_ROLE_ATTR_VALUE, + "epmap": pserver_endpoints, + "trainer_id": self.trainer_id, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, OP_ROLE_VAR_ATTR_NAME: [ self.grad_name_to_param_name[table_grad_name], - table_grad_name - ] - }) + table_grad_name, + ], + }, + ) break - def _create_prefetch_block(self, pserver_index, pserver_program, - optimize_block): + def _create_prefetch_block( + self, pserver_index, pserver_program, optimize_block + ): # STEP: create prefetch block table_var = pserver_program.global_block().vars[self.table_name] prefetch_var_name_to_block_id = [] @@ -1915,46 +2059,53 @@ def _create_prefetch_block(self, pserver_index, pserver_program, name=trainer_ids.name, type=trainer_ids.type, shape=trainer_ids.shape, - dtype=trainer_ids.dtype) + dtype=trainer_ids.dtype, + ) trainer_out = self.all_prefetch_output_vars[pserver_index] pserver_out = pserver_program.global_block().create_var( name=trainer_out.name, type=trainer_out.type, shape=trainer_out.shape, - dtype=trainer_out.dtype) + dtype=trainer_out.dtype, + ) prefetch_block.append_op( type="lookup_sparse_table", - inputs={ - 'Ids': pserver_ids, - "W": table_var - }, + inputs={'Ids': pserver_ids, "W": table_var}, outputs={"Out": pserver_out}, attrs={ "is_sparse": True, # has no effect on lookup_table op "is_distributed": True, - "padding_idx": -1 - }) - prefetch_var_name_to_block_id.append(trainer_ids.name + ":" + - str(prefetch_block.idx)) + "padding_idx": -1, + }, + ) + prefetch_var_name_to_block_id.append( + trainer_ids.name + ":" + str(prefetch_block.idx) + ) return prefetch_var_name_to_block_id - def _create_table_optimize_block(self, pserver_index, pserver_program, - pre_block_idx, grad_to_block_id): + def _create_table_optimize_block( + self, pserver_index, pserver_program, pre_block_idx, grad_to_block_id + ): # STEP: create table optimize block table_opt_block = pserver_program._create_block(pre_block_idx) # create table param and grad var in pserver program # create table optimize block in pserver program table_opt_op = [ - op for op in self.optimize_ops if 'Param' in op.input_names + op + for op in self.optimize_ops + if 'Param' in op.input_names and op.input("Param")[0] == self.table_name ][0] origin_param_var = self.origin_program.global_block().vars[ - self.table_name] + self.table_name + ] zero_dim = int( - math.ceil(origin_param_var.shape[0] / - float(len(self.pserver_endpoints)))) + math.ceil( + origin_param_var.shape[0] / float(len(self.pserver_endpoints)) + ) + ) table_shape = list(origin_param_var.shape) table_shape[0] = zero_dim @@ -1963,28 +2114,34 @@ def _create_table_optimize_block(self, pserver_index, pserver_program, shape=table_shape, dtype=origin_param_var.dtype, type=core.VarDesc.VarType.SELECTED_ROWS, - persistable=True) + persistable=True, + ) # parameter must be selected rows param_var.desc.set_type(core.VarDesc.VarType.SELECTED_ROWS) grad_var = pserver_program.global_block()._clone_variable( - self.origin_program.global_block().vars[grad_var_name( - self.table_name)]) + self.origin_program.global_block().vars[ + grad_var_name(self.table_name) + ] + ) lr_var = pserver_program.global_block()._clone_variable( - self.origin_program.global_block().vars[table_opt_op.input( - "LearningRate")[0]]) + self.origin_program.global_block().vars[ + table_opt_op.input("LearningRate")[0] + ] + ) if self.sync_mode: # create grad vars in pserver program table_grad_var = self.table_param_grad[1] pserver_side_table_grad_list = [ pserver_program.global_block().create_var( - name="%s.trainer_%d.pserver_%d" % - (table_grad_var.name, index, pserver_index), + name="%s.trainer_%d.pserver_%d" + % (table_grad_var.name, index, pserver_index), type=table_grad_var.type, shape=table_grad_var.shape, - dtype=table_grad_var.dtype) + dtype=table_grad_var.dtype, + ) for index in range(self.trainer_num) ] @@ -1993,28 +2150,36 @@ def _create_table_optimize_block(self, pserver_index, pserver_program, type="sum", inputs={"X": pserver_side_table_grad_list}, outputs={"Out": [grad_var]}, - attrs={"use_mkldnn": False}) + attrs={"use_mkldnn": False}, + ) else: # in async_mode, for table gradient, it also need to be split to each parameter server origin_grad_name = grad_var.name splited_grad_name = self.trainer_side_table_grad_list[ - pserver_index].name + pserver_index + ].name if not splited_grad_name.startswith(origin_grad_name): - raise ValueError("origin_grad_var: " + splited_grad_name + - " grad_var:" + grad_var.name) + raise ValueError( + "origin_grad_var: " + + splited_grad_name + + " grad_var:" + + grad_var.name + ) grad_var = pserver_program.global_block()._rename_var( - origin_grad_name, splited_grad_name) + origin_grad_name, splited_grad_name + ) inputs = { "Param": [param_var], "Grad": [grad_var], - "LearningRate": [lr_var] + "LearningRate": [lr_var], } outputs = {"ParamOut": [param_var]} # only support sgd now logging.warn( "distribute lookup table only support sgd optimizer, change it's optimizer to sgd instead of " - + table_opt_op.type) + + table_opt_op.type + ) table_opt_block.append_op(type="sgd", inputs=inputs, outputs=outputs) # add table parameter gradient and it's block id to grad_to_block_id @@ -2027,23 +2192,26 @@ def _create_checkpoint_save_block(self, pserver_program, pre_block_idx): create a new block to handle save checkpoint. """ - pserver_program.global_block().create_var(name="kLookupTablePath", - persistable=True, - type=core.VarDesc.VarType.RAW) + pserver_program.global_block().create_var( + name="kLookupTablePath", + persistable=True, + type=core.VarDesc.VarType.RAW, + ) checkpoint_save_block = pserver_program._create_block(pre_block_idx) # this 'file_path' do not be used in save lookup table variable - checkpoint_save_block.append_op(type='save', - inputs={'X': [self.table_name]}, - outputs={}, - attrs={'file_path': "none"}) + checkpoint_save_block.append_op( + type='save', + inputs={'X': [self.table_name]}, + outputs={}, + attrs={'file_path': "none"}, + ) return checkpoint_save_block.idx - def _create_vars_from_blocklist(self, - program, - block_list, - add_trainer_suffix=False): + def _create_vars_from_blocklist( + self, program, block_list, add_trainer_suffix=False + ): """ Create vars for each split. NOTE: only grads need to be named for different trainers, use @@ -2071,14 +2239,18 @@ def _create_vars_from_blocklist(self, orig_var = program.global_block().var(varname) if len(split) == 1: if self.sync_mode and add_trainer_suffix: - new_var_name = "%s.trainer_%d" % \ - (orig_var.name, self.trainer_id) + new_var_name = "%s.trainer_%d" % ( + orig_var.name, + self.trainer_id, + ) program.global_block()._rename_var(varname, new_var_name) - var_mapping[varname] = \ - [program.global_block().var(new_var_name)] + var_mapping[varname] = [ + program.global_block().var(new_var_name) + ] else: - var_mapping[varname] = \ - [program.global_block().var(orig_var.name)] + var_mapping[varname] = [ + program.global_block().var(orig_var.name) + ] continue var_mapping[varname] = [] orig_shape = orig_var.shape @@ -2094,28 +2266,33 @@ def _create_vars_from_blocklist(self, splited_shape.extend(orig_shape[1:]) new_var_name = "" if self.sync_mode and add_trainer_suffix: - new_var_name = "%s.block%d.trainer_%d" % \ - (varname, i, self.trainer_id) + new_var_name = "%s.block%d.trainer_%d" % ( + varname, + i, + self.trainer_id, + ) else: - new_var_name = "%s.block%d" % \ - (varname, i) + new_var_name = "%s.block%d" % (varname, i) var = program.global_block().create_var( name=new_var_name, persistable=False, dtype=orig_var.dtype, type=orig_var.type, - shape=splited_shape) # flattend split var + shape=splited_shape, + ) # flattend split var var_mapping[varname].append(var) program.global_block()._sync_with_cpp() return var_mapping def _clone_var(self, block, var, persistable=True): - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=persistable) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=persistable, + ) @staticmethod def _get_splited_var_sections(splited_vars): @@ -2131,34 +2308,37 @@ def _insert_split_op(self, program, orig_var, index, splited_vars): sparse_param_name = self.grad_name_to_param_name[orig_var.name] if self._is_input_of_remote_sparse_update_op(sparse_param_name): self.sparse_param_to_height_sections[ - sparse_param_name] = height_sections - program.global_block()._insert_op(index=index + 1, - type="split_selected_rows", - inputs={"X": orig_var}, - outputs={"Out": splited_vars}, - attrs={ - "height_sections": - height_sections, - RPC_OP_ROLE_ATTR_NAME: - DIST_OP_ROLE_ATTR_VALUE - }) + sparse_param_name + ] = height_sections + program.global_block()._insert_op( + index=index + 1, + type="split_selected_rows", + inputs={"X": orig_var}, + outputs={"Out": splited_vars}, + attrs={ + "height_sections": height_sections, + RPC_OP_ROLE_ATTR_NAME: DIST_OP_ROLE_ATTR_VALUE, + }, + ) elif orig_var.type == core.VarDesc.VarType.LOD_TENSOR: - program.global_block()._insert_op(index=index + 1, - type="split_byref", - inputs={"X": orig_var}, - outputs={"Out": splited_vars}, - attrs={ - "sections": - height_sections, - RPC_OP_ROLE_ATTR_NAME: - DIST_OP_ROLE_ATTR_VALUE - }) + program.global_block()._insert_op( + index=index + 1, + type="split_byref", + inputs={"X": orig_var}, + outputs={"Out": splited_vars}, + attrs={ + "sections": height_sections, + RPC_OP_ROLE_ATTR_NAME: DIST_OP_ROLE_ATTR_VALUE, + }, + ) else: - AssertionError("Variable type should be in set " - "[LOD_TENSOR, SELECTED_ROWS]") + AssertionError( + "Variable type should be in set " "[LOD_TENSOR, SELECTED_ROWS]" + ) - def _get_optimizer_input_shape(self, op_type, varkey, orig_shape, - param_shape): + def _get_optimizer_input_shape( + self, op_type, varkey, orig_shape, param_shape + ): """ Returns the shape for optimizer inputs that need to be reshaped when Param and Grad is split to multiple servers. @@ -2190,8 +2370,8 @@ def _get_optimizer_input_shape(self, op_type, varkey, orig_shape, pass else: raise ValueError( - "Not supported optimizer for distributed training: %s" % - op_type) + "Not supported optimizer for distributed training: %s" % op_type + ) return orig_shape def _get_varname_parts(self, varname): @@ -2201,30 +2381,36 @@ def _get_varname_parts(self, varname): block_part = "" trainer_idx = varname.find(".trainer_") if trainer_idx >= 0: - trainer_part = varname[trainer_idx + 1:] + trainer_part = varname[trainer_idx + 1 :] else: trainer_idx = len(varname) block_index = varname.find(".block") if block_index >= 0: - block_part = varname[block_index + 1:trainer_idx] + block_part = varname[block_index + 1 : trainer_idx] else: block_index = len(varname) - orig_var_name = varname[0:min(block_index, trainer_idx)] + orig_var_name = varname[0 : min(block_index, trainer_idx)] return orig_var_name, block_part, trainer_part def _orig_varname(self, varname): orig, _, _ = self._get_varname_parts(varname) return orig - def _append_pserver_grad_merge_ops(self, optimize_block, - grad_varname_for_block, endpoint, - grad_to_block_id, origin_program): + def _append_pserver_grad_merge_ops( + self, + optimize_block, + grad_varname_for_block, + endpoint, + grad_to_block_id, + origin_program, + ): program = optimize_block.program pserver_block = program.global_block() grad_block = None for g in self.param_grad_ep_mapping[endpoint]["grads"]: - if self._orig_varname(g.name) == \ - self._orig_varname(grad_varname_for_block): + if self._orig_varname(g.name) == self._orig_varname( + grad_varname_for_block + ): grad_block = g break if not grad_block: @@ -2232,7 +2418,8 @@ def _append_pserver_grad_merge_ops(self, optimize_block, # is not dealing with this grad block return None orig_varname, block_name, trainer_name = self._get_varname_parts( - grad_block.name) + grad_block.name + ) if block_name: merged_var_name = '.'.join([orig_varname, block_name]) else: @@ -2240,90 +2427,104 @@ def _append_pserver_grad_merge_ops(self, optimize_block, merged_var = pserver_block.vars[merged_var_name] grad_to_block_id.append(merged_var.name + ":" + str(optimize_block.idx)) - if self.sync_mode or self.config.completely_not_async and self.trainer_num > 1: + if ( + self.sync_mode + or self.config.completely_not_async + and self.trainer_num > 1 + ): vars2merge = [] for i in range(self.trainer_num): - per_trainer_name = "%s.trainer_%d" % \ - (merged_var_name, i) + per_trainer_name = "%s.trainer_%d" % (merged_var_name, i) vars2merge.append(pserver_block.vars[per_trainer_name]) - optimize_block.append_op(type="sum", - inputs={"X": vars2merge}, - outputs={"Out": merged_var}, - attrs={"use_mkldnn": False}) + optimize_block.append_op( + type="sum", + inputs={"X": vars2merge}, + outputs={"Out": merged_var}, + attrs={"use_mkldnn": False}, + ) optimize_block.append_op( type="scale", inputs={"X": merged_var}, outputs={"Out": merged_var}, - attrs={"scale": 1.0 / float(self.trainer_num)}) + attrs={"scale": 1.0 / float(self.trainer_num)}, + ) return merged_var def _append_dc_asgd_ops(self, block, param_var, grad_var): # NOTE: can not use grammar candy here, should put ops in specific block - local_param_bak = block.create_var(name="%s.local_bak" % param_var.name, - shape=param_var.shape, - type=param_var.type, - dtype=param_var.dtype, - persistable=False) + local_param_bak = block.create_var( + name="%s.local_bak" % param_var.name, + shape=param_var.shape, + type=param_var.type, + dtype=param_var.dtype, + persistable=False, + ) # trainer_id_var is block local - trainer_id_var = block.create_var(name="@TRAINER_ID@", - type=core.VarDesc.VarType.LOD_TENSOR, - dtype=core.VarDesc.VarType.INT64, - shape=[1], - persistable=False) + trainer_id_var = block.create_var( + name="@TRAINER_ID@", + type=core.VarDesc.VarType.LOD_TENSOR, + dtype=core.VarDesc.VarType.INT64, + shape=[1], + persistable=False, + ) # ref_inputs = [x[1] for x in self.param_bak_list] ref_inputs = [] for p, p_bak in self.param_bak_list: if p.name == param_var.name: ref_inputs.append(p_bak) - block.append_op(type="ref_by_trainer_id", - inputs={ - "X": ref_inputs, - "TrainerId": trainer_id_var - }, - outputs={"Out": local_param_bak}) + block.append_op( + type="ref_by_trainer_id", + inputs={"X": ref_inputs, "TrainerId": trainer_id_var}, + outputs={"Out": local_param_bak}, + ) def __create_temp_var__(): - return block.create_var(name=unique_name.generate("tmp_dc_output"), - shape=param_var.shape, - type=param_var.type, - dtype=param_var.dtype, - persistable=False) + return block.create_var( + name=unique_name.generate("tmp_dc_output"), + shape=param_var.shape, + type=param_var.type, + dtype=param_var.dtype, + persistable=False, + ) o1 = __create_temp_var__() - block.append_op(type="elementwise_sub", - inputs={ - "X": param_var, - "Y": local_param_bak - }, - outputs={"Out": o1}) + block.append_op( + type="elementwise_sub", + inputs={"X": param_var, "Y": local_param_bak}, + outputs={"Out": o1}, + ) o2 = __create_temp_var__() - block.append_op(type="elementwise_mul", - inputs={ - "X": o1, - "Y": grad_var - }, - outputs={"Out": o2}) + block.append_op( + type="elementwise_mul", + inputs={"X": o1, "Y": grad_var}, + outputs={"Out": o2}, + ) o3 = __create_temp_var__() - block.append_op(type="elementwise_mul", - inputs={ - "X": o2, - "Y": grad_var - }, - outputs={"Out": o3}) + block.append_op( + type="elementwise_mul", + inputs={"X": o2, "Y": grad_var}, + outputs={"Out": o3}, + ) # TODO(typhoonzero): append scale o4 = __create_temp_var__() - block.append_op(type="elementwise_add", - inputs={ - "X": grad_var, - "Y": o3 - }, - outputs={"Out": o4}) + block.append_op( + type="elementwise_add", + inputs={"X": grad_var, "Y": o3}, + outputs={"Out": o4}, + ) return o4 - def _append_pserver_ops(self, optimize_block, opt_op, endpoint, - grad_to_block_id, origin_program, merged_var, - sparse_grad_to_param): + def _append_pserver_ops( + self, + optimize_block, + opt_op, + endpoint, + grad_to_block_id, + origin_program, + merged_var, + sparse_grad_to_param, + ): program = optimize_block.program pserver_block = program.global_block() new_inputs = collections.OrderedDict() @@ -2349,9 +2550,10 @@ def _get_param_block(opt_op): # Note!! This is for l2decay on sparse gradient, because it will create a new tensor for # decayed gradient but not inplace modify the origin one origin_grad_name = opt_op.input(key)[0] - if core.kNewGradSuffix( - ) in origin_grad_name and pserver_block.has_var( - origin_grad_name): + if ( + core.kNewGradSuffix() in origin_grad_name + and pserver_block.has_var(origin_grad_name) + ): new_grad = pserver_block.var(origin_grad_name) new_inputs[key] = new_grad else: @@ -2360,10 +2562,12 @@ def _get_param_block(opt_op): param_block = _get_param_block(opt_op) if not param_block: return - tmpvar = pserver_block.create_var(name=param_block.name, - persistable=True, - dtype=param_block.dtype, - shape=param_block.shape) + tmpvar = pserver_block.create_var( + name=param_block.name, + persistable=True, + dtype=param_block.dtype, + shape=param_block.shape, + ) new_inputs[key] = tmpvar elif key == "LearningRate": # learning rate variable has already be created by non-optimize op, @@ -2377,42 +2581,53 @@ def _get_param_block(opt_op): name=origin_var.name, persistable=origin_var.persistable, dtype=origin_var.dtype, - shape=origin_var.shape) + shape=origin_var.shape, + ) new_inputs[key] = tmpvar for key in opt_op.input_names: new_shape = None if key in [ - "Param", "Grad", "LearningRate", "Beta1Tensor", - "Beta2Tensor" + "Param", + "Grad", + "LearningRate", + "Beta1Tensor", + "Beta2Tensor", ]: continue var = self.origin_program.global_block().vars[opt_op.input(key)[0]] param_var = new_inputs["Param"] # update accumulator variable shape - new_shape = self._get_optimizer_input_shape(opt_op.type, key, - var.shape, - param_var.shape) - tmpvar = pserver_block.create_var(name=var.name, - persistable=var.persistable, - dtype=var.dtype, - shape=new_shape) + new_shape = self._get_optimizer_input_shape( + opt_op.type, key, var.shape, param_var.shape + ) + tmpvar = pserver_block.create_var( + name=var.name, + persistable=var.persistable, + dtype=var.dtype, + shape=new_shape, + ) new_inputs[key] = tmpvar # change output's ParamOut variable outputs = self._get_output_map_from_op( - self.origin_program.global_block().vars, opt_op) + self.origin_program.global_block().vars, opt_op + ) outputs["ParamOut"] = new_inputs["Param"] - optimize_block.append_op(type=opt_op.type, - inputs=new_inputs, - outputs=outputs, - attrs=opt_op.all_attrs()) + optimize_block.append_op( + type=opt_op.type, + inputs=new_inputs, + outputs=outputs, + attrs=opt_op.all_attrs(), + ) # record sparse grad to param name if new_inputs["Grad"].type == core.VarDesc.VarType.SELECTED_ROWS: sparse_grad_to_param.append( - str(new_inputs["Grad"].name) + ":" + - str(new_inputs["Param"].name)) + str(new_inputs["Grad"].name) + + ":" + + str(new_inputs["Param"].name) + ) def _get_pserver_grad_param_var(self, var, var_dict): """ @@ -2431,15 +2646,20 @@ def _get_pserver_grad_param_var(self, var, var_dict): # skip per trainer vars if g.name.find(".trainer_") == -1: # only param or grads have split blocks - if self._orig_varname(g.name) in self.grad_name_to_param_name or \ - self._orig_varname(g.name) in self.param_name_to_grad_name: + if ( + self._orig_varname(g.name) + in self.grad_name_to_param_name + or self._orig_varname(g.name) + in self.param_name_to_grad_name + ): grad_block = g break return grad_block def _clone_lr_op(self, program, block, op): inputs = self._get_input_map_from_op( - self.origin_program.global_block().vars, op) + self.origin_program.global_block().vars, op + ) for key, varlist in inputs.items(): if not isinstance(varlist, list): varlist = [varlist] @@ -2448,7 +2668,8 @@ def _clone_lr_op(self, program, block, op): block._clone_variable(var) outputs = self._get_output_map_from_op( - self.origin_program.global_block().vars, op) + self.origin_program.global_block().vars, op + ) for key, varlist in outputs.items(): if not isinstance(varlist, list): varlist = [varlist] @@ -2456,16 +2677,16 @@ def _clone_lr_op(self, program, block, op): if var not in program.global_block().vars: block._clone_variable(var) - return block.append_op(type=op.type, - inputs=inputs, - outputs=outputs, - attrs=op.all_attrs()) + return block.append_op( + type=op.type, inputs=inputs, outputs=outputs, attrs=op.all_attrs() + ) def _append_pserver_non_opt_ops(self, optimize_block, opt_op): program = optimize_block.program # Append the ops for parameters that do not need to be optimized/updated inputs = self._get_input_map_from_op( - self.origin_program.global_block().vars, opt_op) + self.origin_program.global_block().vars, opt_op + ) for key, varlist in inputs.items(): if not isinstance(varlist, list): varlist = [varlist] @@ -2474,8 +2695,8 @@ def _append_pserver_non_opt_ops(self, optimize_block, opt_op): # for ops like clipping and weight decay, get the split var (xxx.block0) # for inputs/outputs grad_block = self._get_pserver_grad_param_var( - var, - program.global_block().vars) + var, program.global_block().vars + ) if grad_block: varlist[i] = grad_block elif var.name not in program.global_block().vars: @@ -2486,15 +2707,16 @@ def _append_pserver_non_opt_ops(self, optimize_block, opt_op): inputs[key] = varlist outputs = self._get_output_map_from_op( - self.origin_program.global_block().vars, opt_op) + self.origin_program.global_block().vars, opt_op + ) for key, varlist in outputs.items(): if not isinstance(varlist, list): varlist = [varlist] for i in range(len(varlist)): var = varlist[i] grad_block = self._get_pserver_grad_param_var( - var, - program.global_block().vars) + var, program.global_block().vars + ) if grad_block: varlist[i] = grad_block elif var.name not in program.global_block().vars: @@ -2504,17 +2726,20 @@ def _append_pserver_non_opt_ops(self, optimize_block, opt_op): varlist[i] = program.global_block().vars[var.name] outputs[key] = varlist - return optimize_block.append_op(type=opt_op.type, - inputs=inputs, - outputs=outputs, - attrs=opt_op.all_attrs()) + return optimize_block.append_op( + type=opt_op.type, + inputs=inputs, + outputs=outputs, + attrs=opt_op.all_attrs(), + ) def _is_op_connected(self, op1, op2): # If one op's input is another op's output or # one op's output is another op's input, we say # the two operator is connected. - if set(op1.desc.output_arg_names()) & set(op2.desc.input_arg_names()) or \ - set(op1.desc.input_arg_names()) & set(op2.desc.output_arg_names()): + if set(op1.desc.output_arg_names()) & set( + op2.desc.input_arg_names() + ) or set(op1.desc.input_arg_names()) & set(op2.desc.output_arg_names()): return True return False @@ -2530,8 +2755,7 @@ def _create_ufind(self, optimize_ops): return ufind def _is_optimizer_op(self, op): - if "Param" in op.input_names and \ - "LearningRate" in op.input_names: + if "Param" in op.input_names and "LearningRate" in op.input_names: return True return False @@ -2579,14 +2803,16 @@ def _get_lr_ops(self): block = self.origin_program.global_block() for index, op in enumerate(block.ops): role_id = int(op.attr(RPC_OP_ROLE_ATTR_NAME)) - if role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) or \ - role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) | \ - int(OPT_OP_ROLE_ATTR_VALUE): + if role_id == int(LR_SCHED_OP_ROLE_ATTR_VALUE) or role_id == int( + LR_SCHED_OP_ROLE_ATTR_VALUE + ) | int(OPT_OP_ROLE_ATTR_VALUE): if self.sync_mode == False and op.type == 'increment': inputs = self._get_input_map_from_op( - self.origin_program.global_block().vars, op) + self.origin_program.global_block().vars, op + ) outputs = self._get_output_map_from_op( - self.origin_program.global_block().vars, op) + self.origin_program.global_block().vars, op + ) for key in outputs: counter_var = outputs[key] all_trainer_counter_inputs = [ @@ -2595,21 +2821,29 @@ def _get_lr_ops(self): type=counter_var.type, shape=counter_var.shape, dtype=counter_var.dtype, - persistable=counter_var.persistable) + persistable=counter_var.persistable, + ) for id_ in range(self.trainer_num) ] for i, op in enumerate( - self.startup_program.global_block().ops): + self.startup_program.global_block().ops + ): if op.type == 'fill_constant': for key in op.output_names: - if len(op.output(key)) == 1 and op.output( - key)[0] == counter_var.name: - self.startup_program.global_block( - ).ops[i]._set_attr( - 'value', float(0.0 - self.trainer_num)) + if ( + len(op.output(key)) == 1 + and op.output(key)[0] == counter_var.name + ): + self.startup_program.global_block().ops[ + i + ]._set_attr( + 'value', float(0.0 - self.trainer_num) + ) for var in all_trainer_counter_inputs: - if var.name == "%s.trainer_%d" % (counter_var.name, - self.trainer_id): + if var.name == "%s.trainer_%d" % ( + counter_var.name, + self.trainer_id, + ): self.counter_var = var self.startup_program.global_block().create_var( name=var.name, @@ -2617,8 +2851,10 @@ def _get_lr_ops(self): dtype=var.dtype, shape=var.shape, persistable=var.persistable, - initializer=initializer.Constant(1)) - op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName( + initializer=initializer.Constant(1), + ) + op_role_attr_name = ( + core.op_proto_and_checker_maker.kOpRoleAttrName() ) block._remove_op(index) op = block._insert_op( @@ -2626,7 +2862,8 @@ def _get_lr_ops(self): type='sum', inputs={'X': all_trainer_counter_inputs}, outputs=outputs, - attrs={op_role_attr_name: LR_SCHED_OP_ROLE_ATTR_VALUE}) + attrs={op_role_attr_name: LR_SCHED_OP_ROLE_ATTR_VALUE}, + ) lr_ops.append(op) log("append lr op: ", op.type) return lr_ops @@ -2652,8 +2889,12 @@ def _get_lr_ops_deprecated(self): for op2 in block.ops: # NOTE: we need to skip all optimize ops, since it is connected # with forward/backward ops and lr ops, we only need the lr ops. - if op1 != op2 and self._is_op_connected(op1, op2) and \ - not self._is_optimizer_op(op1) and not self._is_optimizer_op(op2): + if ( + op1 != op2 + and self._is_op_connected(op1, op2) + and not self._is_optimizer_op(op1) + and not self._is_optimizer_op(op2) + ): ufind.union(op1, op2) # find all ops which is related with lr var for op1 in block.ops: @@ -2669,8 +2910,9 @@ def _is_opt_role_op(self, op): # optimize op_maker = core.op_proto_and_checker_maker optimize_role = core.op_proto_and_checker_maker.OpRole.Optimize - if op_maker.kOpRoleAttrName() in op.attr_names and \ - int(op.all_attrs()[op_maker.kOpRoleAttrName()]) == int(optimize_role): + if op_maker.kOpRoleAttrName() in op.attr_names and int( + op.all_attrs()[op_maker.kOpRoleAttrName()] + ) == int(optimize_role): return True return False @@ -2691,13 +2933,16 @@ def _get_optimize_pass(self): if self._is_opt_role_op(op): # Todo(chengmo): Whether clip related op belongs to Optimize guard should be discussed # delete clip op from opt_ops when run in Parameter Server mode - if OP_NAME_SCOPE in op.all_attrs( - ) and CLIP_OP_NAME_SCOPE in op.attr( - OP_NAME_SCOPE - ) and self.config.mode != "nccl2" and self.config.mode != "collective": + if ( + OP_NAME_SCOPE in op.all_attrs() + and CLIP_OP_NAME_SCOPE in op.attr(OP_NAME_SCOPE) + and self.config.mode != "nccl2" + and self.config.mode != "collective" + ): op._set_attr( "op_role", - int(core.op_proto_and_checker_maker.OpRole.Backward)) + int(core.op_proto_and_checker_maker.OpRole.Backward), + ) continue opt_ops.append(op) if op.attr(OP_ROLE_VAR_ATTR_NAME): @@ -2706,10 +2951,12 @@ def _get_optimize_pass(self): if not param_name in optimize_params: optimize_params.add(param_name) log("adding param_grad pair: ", param_name, grad_name) - params_grads.append([ - origin_var_dict[param_name], - origin_var_dict[grad_name] - ]) + params_grads.append( + [ + origin_var_dict[param_name], + origin_var_dict[grad_name], + ] + ) else: pass diff --git a/python/paddle/fluid/transpiler/geo_sgd_transpiler.py b/python/paddle/fluid/transpiler/geo_sgd_transpiler.py index fe018f611a376..0163d591cb434 100644 --- a/python/paddle/fluid/transpiler/geo_sgd_transpiler.py +++ b/python/paddle/fluid/transpiler/geo_sgd_transpiler.py @@ -30,21 +30,32 @@ from .ps_dispatcher import RoundRobin, PSDispatcher from .. import core, framework -from ..framework import Program, default_main_program, \ - default_startup_program, Block, Parameter +from ..framework import ( + Program, + default_main_program, + default_startup_program, + Block, + Parameter, +) from .details import wait_server_ready, VarsDistributed from .details import delete_ops from ..distribute_lookup_table import find_distributed_lookup_table -from .distribute_transpiler import DistributeTranspiler, DistributeTranspilerConfig, slice_variable, same_or_split_var, ServerRuntimeConfig +from .distribute_transpiler import ( + DistributeTranspiler, + DistributeTranspilerConfig, + slice_variable, + same_or_split_var, + ServerRuntimeConfig, +) from paddle.fluid.incubate.fleet.parameter_server.mode import DistributedMode -RPC_OP_ROLE_ATTR_NAME = op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName( -) +RPC_OP_ROLE_ATTR_NAME = ( + op_role_attr_name +) = core.op_proto_and_checker_maker.kOpRoleAttrName() RPC_OP_ROLE_ATTR_VALUE = core.op_proto_and_checker_maker.OpRole.RPC class GeoSgdTranspiler(DistributeTranspiler): - def __init__(self, config=None): if config is not None: self.config = config @@ -55,17 +66,19 @@ def __init__(self, config=None): if self.config.split_method is None: self.config.split_method = RoundRobin - assert (self.config.min_block_size >= 8192) - assert (self.config.split_method.__bases__[0] == PSDispatcher) - - def transpile(self, - trainer_id, - program=None, - pservers="127.0.0.1:6174", - trainers=1, - sync_mode=False, - startup_program=None, - current_endpoint="127.0.0.1:6174"): + assert self.config.min_block_size >= 8192 + assert self.config.split_method.__bases__[0] == PSDispatcher + + def transpile( + self, + trainer_id, + program=None, + pservers="127.0.0.1:6174", + trainers=1, + sync_mode=False, + startup_program=None, + current_endpoint="127.0.0.1:6174", + ): if program is None: program = default_main_program() if startup_program is None: @@ -92,7 +105,9 @@ def transpile(self, # distribute lookup table self.table_name = find_distributed_lookup_table(self.origin_program) self.has_distributed_lookup_table = self.table_name != None - self.origin_program._distributed_lookup_table = self.table_name if self.table_name else None + self.origin_program._distributed_lookup_table = ( + self.table_name if self.table_name else None + ) # add distributed attrs to program self.origin_program._is_distributed = True @@ -127,7 +142,8 @@ def transpile(self, for i, ep in enumerate(eplist): self.param_opt_ep_mapping[ep]["params"].append(recv_vars[i]) distributed_var = self.vars_overview.get_distributed_var_by_slice( - recv_vars[i].name) + recv_vars[i].name + ) distributed_var.endpoint = ep origin_name = self.split_to_origin_mapping[recv_vars[i].name] self.vars_info[origin_name]["epmap"].append(ep) @@ -142,11 +158,14 @@ def transpile(self, if op.type == "lookup_table": op._set_attr('remote_prefetch', False) for input_var_name, sparse_var_name in zip( - op.input("Ids"), op.input("W")): + op.input("Ids"), op.input("W") + ): if sparse_var_name in self.sparse_var_list: if input_var_name in unique_sparse_var: - if unique_sparse_var[ - input_var_name] == sparse_var_name: + if ( + unique_sparse_var[input_var_name] + == sparse_var_name + ): continue input_var = program.global_block().var(input_var_name) self.sparse_var.append(input_var) @@ -155,32 +174,39 @@ def transpile(self, # batch training loop end flag dummy_output = program.global_block().create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) program.global_block().append_op( type="send", inputs={"X": self.sparse_var}, outputs={"Out": dummy_output}, - attrs={"send_varnames": self.sparse_tables}) + attrs={"send_varnames": self.sparse_tables}, + ) # add param_init flag in trainer startup program self.trainer_startup_program = self._get_trainer_startup_program( - recv_vars=recv_vars, eplist=eplist) + recv_vars=recv_vars, eplist=eplist + ) for delta_var in self.delta_vars_list: self.trainer_startup_program.global_block().create_var( name=delta_var.name, persistable=delta_var.persistable, dtype=delta_var.dtype, type=delta_var.type, - shape=delta_var.shape) + shape=delta_var.shape, + ) dummy_output = self.trainer_startup_program.global_block().create_var( - name=framework.generate_control_dev_var_name()) + name=framework.generate_control_dev_var_name() + ) param_init = self.trainer_startup_program.global_block().create_var( - name="param_init") + name="param_init" + ) self.trainer_startup_program.global_block().append_op( type="send", inputs={"X": [param_init]}, outputs={"Out": dummy_output}, - attrs={"send_varnames": [param_init.name]}) + attrs={"send_varnames": [param_init.name]}, + ) def _get_vars_info(self): return self.vars_info @@ -193,8 +219,9 @@ def get_trainer_program(self, wait_port=True): def get_pserver_programs(self, endpoint): pserver_prog = self.get_pserver_program(endpoint) self.param_grad_ep_mapping = self.param_opt_ep_mapping - pserver_startup = self.get_startup_program(endpoint, - pserver_program=pserver_prog) + pserver_startup = self.get_startup_program( + endpoint, pserver_program=pserver_prog + ) return pserver_prog, pserver_startup def get_pserver_program(self, endpoint): @@ -224,21 +251,27 @@ def get_pserver_program(self, endpoint): delta_var_name = "%s.delta" % (param.name) if var.name in self.sparse_var_splited_list: delta_type = core.VarDesc.VarType.SELECTED_ROWS - sparse_grad_to_param.append(":".join( - [delta_var_name, param.name])) + sparse_grad_to_param.append( + ":".join([delta_var_name, param.name]) + ) else: delta_type = param.type - delta_var = pserver_block.create_var(name=delta_var_name, - persistable=False, - type=delta_type, - dtype=param.dtype, - shape=param.shape) - - per_opt_block.append_op(type="sum", - inputs={"X": [param, delta_var]}, - outputs={"Out": param}) - param_to_block_id.append(delta_var_name + ":" + - str(per_opt_block.idx)) + delta_var = pserver_block.create_var( + name=delta_var_name, + persistable=False, + type=delta_type, + dtype=param.dtype, + shape=param.shape, + ) + + per_opt_block.append_op( + type="sum", + inputs={"X": [param, delta_var]}, + outputs={"Out": param}, + ) + param_to_block_id.append( + delta_var_name + ":" + str(per_opt_block.idx) + ) attrs = { "optimize_blocks": optimize_block, @@ -249,15 +282,16 @@ def get_pserver_program(self, endpoint): "sparse_grad_to_param": sparse_grad_to_param, "rpc_get_thread_num": self.server_config._rpc_get_thread_num, "rpc_send_thread_num": self.server_config._rpc_send_thread_num, - "rpc_prefetch_thread_num": - self.server_config._rpc_prefetch_thread_num + "rpc_prefetch_thread_num": self.server_config._rpc_prefetch_thread_num, } # step5 append the listen_and_serv op - pserver_program.global_block().append_op(type="listen_and_serv", - inputs={'X': recv_inputs}, - outputs={}, - attrs=attrs) + pserver_program.global_block().append_op( + type="listen_and_serv", + inputs={'X': recv_inputs}, + outputs={}, + attrs=attrs, + ) pserver_program._sync_with_cpp() # save pserver program to generate pserver side startup relatively. @@ -284,21 +318,28 @@ def _init_splited_vars(self): # step 2. Slice vars into numbers of piece with block_size # when we slice var up into blocks, we will slice the var according to # pserver services' count. A pserver may have two or more listening ports. - param_blocks = slice_variable(param_list, len(self.pserver_endpoints), - self.config.min_block_size) + param_blocks = slice_variable( + param_list, len(self.pserver_endpoints), self.config.min_block_size + ) # step 3. Create split param from split blocks # origin_param_name -> [splited_param_vars] # Todo: update _create_vars_from_blocklist self.param_var_mapping = self._create_vars_from_blocklist( - self.origin_program, param_blocks) + self.origin_program, param_blocks + ) # step 4. Create mapping of endpoint -> split var to create pserver side program self.param_opt_ep_mapping = collections.OrderedDict() [ - self.param_opt_ep_mapping.update({ep: { - "params": [], - }}) for ep in self.pserver_endpoints + self.param_opt_ep_mapping.update( + { + ep: { + "params": [], + } + } + ) + for ep in self.pserver_endpoints ] # step 5. Create delta var of Geo-Sgd & record vars information @@ -325,28 +366,34 @@ def _init_splited_vars(self): persistable=False, dtype=origin_var.dtype, type=delta_type, - shape=origin_var.shape) + shape=origin_var.shape, + ) self.delta_vars_list.append(delta_var) for splited_var in splited_vars: is_slice, block_id, offset = self._get_slice_var_info( - splited_var) - self.vars_overview.add_distributed_var(origin_var=origin_var, - slice_var=splited_var, - block_id=block_id, - offset=offset, - is_slice=is_slice, - vtype="Param") + splited_var + ) + self.vars_overview.add_distributed_var( + origin_var=origin_var, + slice_var=splited_var, + block_id=block_id, + offset=offset, + is_slice=is_slice, + vtype="Param", + ) self.split_to_origin_mapping[splited_var.name] = origin_name if origin_name in self.sparse_var_list: self.sparse_var_splited_list.append(splited_var.name) self.vars_info[origin_name]["var_names"].append( - splited_var.name) + splited_var.name + ) if len(splited_vars) != 1: self.origin_program.global_block().create_var( name=".".join([splited_var.name, "delta"]), persistable=False, dtype=splited_var.dtype, type=delta_type, - shape=splited_var.shape) + shape=splited_var.shape, + ) diff --git a/python/paddle/fluid/transpiler/memory_optimization_transpiler.py b/python/paddle/fluid/transpiler/memory_optimization_transpiler.py index 0aa4a9d8f5420..c71197fa0ad81 100755 --- a/python/paddle/fluid/transpiler/memory_optimization_transpiler.py +++ b/python/paddle/fluid/transpiler/memory_optimization_transpiler.py @@ -15,13 +15,11 @@ import logging -def memory_optimize(input_program, - skip_opt_set=None, - print_log=False, - level=0, - skip_grads=True): +def memory_optimize( + input_program, skip_opt_set=None, print_log=False, level=0, skip_grads=True +): """ - :api_attr: Static Graph + :api_attr: Static Graph This API is deprecated since 1.6. Please do not use it. The better memory optimization strategies are enabled by default. @@ -38,15 +36,18 @@ def memory_optimize(input_program, 'default value).\n' ' 2. Inplace strategy, which is enabled by setting ' 'build_strategy.enable_inplace=True (True is the default value) ' - 'when using CompiledProgram or ParallelExecutor.\n') + 'when using CompiledProgram or ParallelExecutor.\n' + ) def release_memory(input_program, skip_opt_set=None): """ - :api_attr: Static Graph + :api_attr: Static Graph This API is deprecated since 1.6. Please do not use it. The better memory optimization strategies are enabled by default. """ - logging.warn('paddle.fluid.release_memory() is deprecated, it would not' - ' take any memory release on your program') + logging.warn( + 'paddle.fluid.release_memory() is deprecated, it would not' + ' take any memory release on your program' + ) diff --git a/python/paddle/fluid/transpiler/ps_dispatcher.py b/python/paddle/fluid/transpiler/ps_dispatcher.py index 0bf7442fdea06..6271f8152c197 100644 --- a/python/paddle/fluid/transpiler/ps_dispatcher.py +++ b/python/paddle/fluid/transpiler/ps_dispatcher.py @@ -46,7 +46,7 @@ def dispatch(self, varlist): class HashName(PSDispatcher): """ - :api_attr: Static Graph + :api_attr: Static Graph Hash variable names to several endpoints using python "hash()" function. @@ -88,7 +88,7 @@ def dispatch(self, varlist): class RoundRobin(PSDispatcher): """ - :api_attr: Static Graph + :api_attr: Static Graph Distribute variables to several endpoints using RondRobin method. diff --git a/python/paddle/fluid/unique_name.py b/python/paddle/fluid/unique_name.py index 39270ec2dfeca..1c0c28dae8bda 100644 --- a/python/paddle/fluid/unique_name.py +++ b/python/paddle/fluid/unique_name.py @@ -121,6 +121,7 @@ def generate(key): # in order to keep name consistency. def generate_with_ignorable_key(key): from .framework import _non_static_mode, _dygraph_tracer + if _non_static_mode(): return _dygraph_tracer()._generate_unique_name() diff --git a/python/paddle/fluid/variable_index.py b/python/paddle/fluid/variable_index.py index 3d67cd7e2302a..f3c7fff38fa6d 100644 --- a/python/paddle/fluid/variable_index.py +++ b/python/paddle/fluid/variable_index.py @@ -22,7 +22,6 @@ def is_list_tuple(index, contain_type): - def _is_list_tuple(item): if not (isinstance(item, (list, tuple)) or type(item) == contain_type): return False @@ -58,14 +57,13 @@ def get_list_index_shape(var_dims, index_dims): out_dims_shape = [1] * out_dims_size - out_dims_shape[:index_dims_size - 1] = index_dims[1:] + out_dims_shape[: index_dims_size - 1] = index_dims[1:] - out_dims_shape[index_dims_size - 1:] = var_dims[index_dims[0]:] + out_dims_shape[index_dims_size - 1 :] = var_dims[index_dims[0] :] return out_dims_shape class SliceInfo: - def __init__(self): self.pre_shape = None self.indexes = [] @@ -73,7 +71,8 @@ def __init__(self): def update(self, index): if is_list_tuple(index, int) or isinstance( - index, (paddle.fluid.Variable, np.ndarray)): + index, (paddle.fluid.Variable, np.ndarray) + ): # convert index to Tensor if not isinstance(index, paddle.fluid.Variable): index = paddle.assign(index) @@ -83,8 +82,10 @@ def update(self, index): else: if index.dtype != self.dtype: raise IndexError( - "Data type of Tensor/List index should be same. The current data type is {}, but the previous data type is {}." - .format(index.dtype, self.dtype)) + "Data type of Tensor/List index should be same. The current data type is {}, but the previous data type is {}.".format( + index.dtype, self.dtype + ) + ) self.indexes.append(index) @@ -93,16 +94,20 @@ def update(self, index): else: if self.pre_shape != index.shape: # broadcast - cur_shape = paddle.broadcast_shape(self.pre_shape, - index.shape) + cur_shape = paddle.broadcast_shape( + self.pre_shape, index.shape + ) for i in range(len(self.indexes)): self.indexes[i] = paddle.broadcast_to( - self.indexes[i], cur_shape) + self.indexes[i], cur_shape + ) self.pre_shape = self.indexes[-1].shape else: raise ValueError( - "Index should be list/tuple of int or Tensor, but received {}.". - format(index)) + "Index should be list/tuple of int or Tensor, but received {}.".format( + index + ) + ) def shape_stride(self, shape): s = [1] * len(shape) @@ -119,19 +124,22 @@ def get_offset_stride(self, tensor_shape): if not isinstance(index, paddle.fluid.Variable): raise ValueError( "only support list/tensor index, but received {}.".format( - type(index))) + type(index) + ) + ) if len(self.indexes) <= len(tensor_shape) or len(self.indexes) == 1: shape = paddle.stack(self.indexes) - axes = list(range(1, - len(self.pre_shape) + 1)) + [ - 0, - ] + axes = list(range(1, len(self.pre_shape) + 1)) + [ + 0, + ] else: raise ValueError( - "too many indices for tensor: tensor is {}-dimensional, but {} were indexed" - .format(len(tensor_shape), self.pre_shape[0])) + "too many indices for tensor: tensor is {}-dimensional, but {} were indexed".format( + len(tensor_shape), self.pre_shape[0] + ) + ) shape_transpose = paddle.transpose(shape, axes) return shape_transpose @@ -148,7 +156,8 @@ def set_item(self, tensor_origin, value): tensor_type = None if tensor_origin.dtype in [ - core.VarDesc.VarType.FP32, core.VarDesc.VarType.FP64 + core.VarDesc.VarType.FP32, + core.VarDesc.VarType.FP64, ]: tensor = tensor_origin else: @@ -161,36 +170,47 @@ def set_item(self, tensor_origin, value): shape_transpose = self.get_offset_stride(tensor_origin.shape) index = paddle.assign(shape_transpose) - gather_tensor_shape = get_list_index_shape(tensor.shape, [ - len(self.indexes), - ] + list(self.indexes[-1].shape)) + gather_tensor_shape = get_list_index_shape( + tensor.shape, + [ + len(self.indexes), + ] + + list(self.indexes[-1].shape), + ) value_dims_bd = [ 1, ] * len(gather_tensor_shape) - value_dims_bd[-len(value.shape):] = list(value.shape) + value_dims_bd[-len(value.shape) :] = list(value.shape) for i in range(len(gather_tensor_shape)): - if not (value_dims_bd[i] == gather_tensor_shape[i] - or value_dims_bd[i] == 1): - raise ValueError("{} can not broadcast into {}".format( - value.shape, gather_tensor_shape)) + if not ( + value_dims_bd[i] == gather_tensor_shape[i] + or value_dims_bd[i] == 1 + ): + raise ValueError( + "{} can not broadcast into {}".format( + value.shape, gather_tensor_shape + ) + ) value_broadcast = paddle.broadcast_to(value, gather_tensor_shape) value_1d = value_broadcast.reshape( - [-1] + gather_tensor_shape[len(index.shape) - 1:]) + [-1] + gather_tensor_shape[len(index.shape) - 1 :] + ) index_1d = index.reshape([-1, index.shape[-1]]) tensor_stride = paddle.assign( - self.shape_stride(tensor.shape[:index.shape[-1]])) + self.shape_stride(tensor.shape[: index.shape[-1]]) + ) inds = [] for i in range(index_1d.shape[0]): temp = (index_1d[i] * tensor_stride).sum() inds.append(temp) index_1d = paddle.stack(inds).reshape([-1]) - t_reshape = tensor.reshape([-1] + list(tensor.shape[index.shape[-1]:])) + t_reshape = tensor.reshape([-1] + list(tensor.shape[index.shape[-1] :])) out = paddle.scatter(t_reshape, index_1d, value_1d) if tensor_type is not None: out = out.astype(tensor_type) @@ -201,6 +221,7 @@ def set_item(self, tensor_origin, value): def replace_ellipsis(var, item): from .framework import Variable + # Use slice(None) to replace Ellipsis. # For var, var.shape = [3,4,5,6] # @@ -212,7 +233,8 @@ def replace_ellipsis(var, item): # Remove Variable to skip bug when counting Ellipsis item_remove_var = [ - ele for ele in item + ele + for ele in item if not isinstance(ele, (Variable, np.ndarray)) and ele is not None ] ell_count = item_remove_var.count(Ellipsis) @@ -226,9 +248,9 @@ def replace_ellipsis(var, item): if ell_idx == len(item) - 1: return item[:-1] else: - item[ell_idx:ell_idx + - 1] = [slice(None) - ] * (len(var.shape) - len(item) + item.count(None) + 1) + item[ell_idx : ell_idx + 1] = [slice(None)] * ( + len(var.shape) - len(item) + item.count(None) + 1 + ) return item @@ -256,6 +278,7 @@ def replace_none(item): def is_integer_or_scalar_tensor(ele): from .framework import Variable + if isinstance(ele, int): return True elif isinstance(ele, Variable): @@ -266,6 +289,7 @@ def is_integer_or_scalar_tensor(ele): def is_bool_tensor(ele): from .framework import Variable + if isinstance(ele, Variable) and ele.dtype == paddle.bool: return True return False @@ -276,8 +300,9 @@ def deal_attrs(attrs, attr, attr_name, tensor_attr_name, inputs, infer_flags): from .layers import utils if utils._contain_var(attr): - inputs[tensor_attr_name] = utils._convert_to_tensor_list(attr, - dtype="int64") + inputs[tensor_attr_name] = utils._convert_to_tensor_list( + attr, dtype="int64" + ) for i, dim in enumerate(attr): if isinstance(dim, Variable): attrs[attr_name].append(-1) @@ -291,16 +316,19 @@ def deal_attrs(attrs, attr, attr_name, tensor_attr_name, inputs, infer_flags): # the item is a tensor of bool def get_value_for_bool_tensor(var, item): if len(item.shape) > len(var.shape): - raise IndexError("The dims of bool index doesn't match indexed array, " - "the dims of bool index except to be equal or less " - "than {}, but received {}.".format( - len(var.shape), len(item.shape))) + raise IndexError( + "The dims of bool index doesn't match indexed array, " + "the dims of bool index except to be equal or less " + "than {}, but received {}.".format(len(var.shape), len(item.shape)) + ) for i, dim_len in enumerate(item.shape): if dim_len != var.shape[i]: raise IndexError( - "The dimension of bool index doesn't match indexed array along "\ - "dimension {}, the target dimension is {}, but received {}.". - format(i, var.shape[i], dim_len)) + "The dimension of bool index doesn't match indexed array along " + "dimension {}, the target dimension is {}, but received {}.".format( + i, var.shape[i], dim_len + ) + ) def idx_not_empty(var, item): from .layers.nn import where @@ -315,8 +343,10 @@ def idx_empty(var): return paddle.empty(var_shape, dtype=var.dtype) from .layers.control_flow import cond - return cond(item.any(), lambda: idx_not_empty(var, item), - lambda: idx_empty(var)) + + return cond( + item.any(), lambda: idx_not_empty(var, item), lambda: idx_empty(var) + ) def _getitem_impl_(var, item): @@ -330,12 +360,13 @@ def _getitem_impl_(var, item): Sliced variable """ from .framework import default_main_program, Variable + if isinstance(item, list): if not is_one_dim_list(item, int): item = tuple(item) if not isinstance(item, tuple): - item = (item, ) + item = (item,) decrease_axes = [] axes = [] @@ -351,11 +382,15 @@ def _getitem_impl_(var, item): slice_info = SliceInfo() for dim, slice_item in enumerate(item): - if is_integer_or_scalar_tensor( - slice_item) and not is_bool_tensor(slice_item): - if isinstance(slice_item, - int) and var.shape[dim] is not None and var.shape[ - dim] >= 0 and slice_item >= var.shape[dim]: + if is_integer_or_scalar_tensor(slice_item) and not is_bool_tensor( + slice_item + ): + if ( + isinstance(slice_item, int) + and var.shape[dim] is not None + and var.shape[dim] >= 0 + and slice_item >= var.shape[dim] + ): # For python, if users write a, b = var, the __getitem__ # method will iterate through 0, 1, 2 ... until __getitem__ # throws an IndexError, then stop. The var[0], var[1] will @@ -365,7 +400,8 @@ def _getitem_impl_(var, item): # We raises IndexError here to support grammar like `a, b = var` raise IndexError( "slice_item %d at dim %d should be >= 0 and < var.shape[%d]: %d" - % (slice_item, dim, dim, var.shape[dim])) + % (slice_item, dim, dim, var.shape[dim]) + ) decrease_axes.append(dim) start = slice_item step = 1 @@ -385,9 +421,9 @@ def _getitem_impl_(var, item): start = 0 if step > 0 else MAX_INTEGER if end is None: if var.shape[dim] != -1 and ( - paddle.fluid.framework._non_static_mode() - or var.desc.type() != - core.VarDesc.VarType.LOD_TENSOR_ARRAY): + paddle.fluid.framework._non_static_mode() + or var.desc.type() != core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): end = var.shape[dim] if step > 0 else -1 else: end = MAX_INTEGER if step > 0 else -1 @@ -407,15 +443,19 @@ def _getitem_impl_(var, item): if len(item) != 1: raise IndexError( - "When index contains a list, its length must be 1, but received {}." - .format(len(item))) + "When index contains a list, its length must be 1, but received {}.".format( + len(item) + ) + ) new_slice_item = [] if all_bool: if len(slice_item) != var.shape[0]: raise IndexError( - "The dimension of bool index doesn't match indexed array along "\ - "dimension 0, the target dimension is {}, but received {}.". - format(var.shape[0], len(slice_item))) + "The dimension of bool index doesn't match indexed array along " + "dimension 0, the target dimension is {}, but received {}.".format( + var.shape[0], len(slice_item) + ) + ) for idx, ele in enumerate(slice_item): if ele is True: new_slice_item.append(idx) @@ -455,8 +495,10 @@ def _getitem_impl_(var, item): else: raise IndexError( - "Valid index accept int or slice or ellipsis or list, but received {}." - .format(slice_item)) + "Valid index accept int or slice or ellipsis or list, but received {}.".format( + slice_item + ) + ) axes.append(dim) starts.append(start) @@ -467,8 +509,10 @@ def _getitem_impl_(var, item): if slice_info.indexes: if len(slice_info.indexes) != len(item): raise IndexError( - "Valid index accept int or slice or ellipsis or list, but received {}." - .format(item)) + "Valid index accept int or slice or ellipsis or list, but received {}.".format( + item + ) + ) return slice_info.get_item(var) inputs = {'Input': [var]} @@ -476,7 +520,7 @@ def _getitem_impl_(var, item): 'axes': axes, 'starts': [], 'ends': [], - 'decrease_axis': decrease_axes + 'decrease_axis': decrease_axes, } if use_strided_slice: attrs['strides'] = [] @@ -484,8 +528,9 @@ def _getitem_impl_(var, item): infer_flags = [1] * len(axes) deal_attrs(attrs, starts, "starts", "StartsTensorList", inputs, infer_flags) deal_attrs(attrs, ends, "ends", "EndsTensorList", inputs, infer_flags) - deal_attrs(attrs, steps, "strides", "StridesTensorList", inputs, - infer_flags) + deal_attrs( + attrs, steps, "strides", "StridesTensorList", inputs, infer_flags + ) attrs['infer_flags'] = infer_flags out = var @@ -500,23 +545,29 @@ def _getitem_impl_(var, item): end = inputs['EndsTensorList'] else: end = attrs['ends'] - out = paddle._C_ops.slice(var, axes, st, end, attrs['infer_flags'], - attrs['decrease_axis']) + out = paddle._C_ops.slice( + var, axes, st, end, attrs['infer_flags'], attrs['decrease_axis'] + ) else: target_block = default_main_program().current_block() slice_out_var = target_block.create_var( - name=unique_name.generate_with_ignorable_key(var.name + "_" + - op_type), - dtype=var.dtype) - target_block.append_op(type=op_type, - inputs=inputs, - outputs={'Out': [slice_out_var]}, - attrs=attrs) + name=unique_name.generate_with_ignorable_key( + var.name + "_" + op_type + ), + dtype=var.dtype, + ) + target_block.append_op( + type=op_type, + inputs=inputs, + outputs={'Out': [slice_out_var]}, + attrs=attrs, + ) out = slice_out_var if len(reverse_axes) > 0: from .layers.tensor import reverse + out = reverse(out, axis=reverse_axes) # Deal with cases when all axes are decreased. @@ -546,40 +597,49 @@ def _getitem_impl_(var, item): # out = x[0, 1, 1, None] # out.shape : (1) from ..tensor import unsqueeze + out = unsqueeze(out, axis=none_axes) return out def _setitem_for_tensor_array(var, item, value): - """ branches for tensor array setitem operation. - A item can be a: - (1) int/Variable, which is a simple number/variable such as [1], [-2] - (2) Slice, which is represented by bounds such as [2:-1] - (3) Tuple, which includes the above two cases such as [2:-1, 1] - If item is case (1), we perform paddle.tensor.array_write, - in other cases, we raise a NotImplementedError. + """branches for tensor array setitem operation. + A item can be a: + (1) int/Variable, which is a simple number/variable such as [1], [-2] + (2) Slice, which is represented by bounds such as [2:-1] + (3) Tuple, which includes the above two cases such as [2:-1, 1] + If item is case (1), we perform paddle.tensor.array_write, + in other cases, we raise a NotImplementedError. """ from ..framework import LayerHelper, core, _non_static_mode from .framework import Variable - assert not _non_static_mode( + + assert ( + not _non_static_mode() ), "setitem for tensor_array must be called in static graph mode." if isinstance(item, (Variable, int)): - from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import to_static_variable + from paddle.fluid.dygraph.dygraph_to_static.variable_trans_func import ( + to_static_variable, + ) from paddle import cast from paddle.tensor import array_write + item = paddle.cast(to_static_variable(item), dtype='int64') value = to_static_variable(value) array_write(x=value, i=item, array=var) else: raise NotImplementedError( - "Only support __setitem__ by Int/Variable in tensor_array, but gets {}" - .format(type(item))) + "Only support __setitem__ by Int/Variable in tensor_array, but gets {}".format( + type(item) + ) + ) def _setitem_impl_(var, item, value): from .framework import default_main_program, Variable from paddle.fluid import core + if var.type == core.VarDesc.VarType.LOD_TENSOR_ARRAY: return _setitem_for_tensor_array(var, item, value) @@ -589,7 +649,7 @@ def _setitem_impl_(var, item, value): item = tuple(item) # 1. Parse item if not isinstance(item, tuple): - item = (item, ) + item = (item,) decrease_axes = [] axes = [] @@ -603,8 +663,9 @@ def _setitem_impl_(var, item, value): slice_info = SliceInfo() dim = 0 for _, slice_item in enumerate(item): - if is_integer_or_scalar_tensor( - slice_item) and not is_bool_tensor(slice_item): + if is_integer_or_scalar_tensor(slice_item) and not is_bool_tensor( + slice_item + ): decrease_axes.append(dim) start = slice_item end = slice_item + 1 if slice_item != -1 else MAX_INTEGER @@ -624,7 +685,8 @@ def _setitem_impl_(var, item, value): if not isinstance(step, Variable) and step == 0: raise ValueError( "When assign a value to a paddle.Tensor, step can not be 0, " - "but received step is {}.".format(step)) + "but received step is {}.".format(step) + ) if isinstance(step, Variable) and (start is None or end is None): raise ValueError( @@ -644,15 +706,19 @@ def _setitem_impl_(var, item, value): for i in slice_item: if not isinstance(i, bool): - raise TypeError("Doesn't support {} in index list.".format( - type(i))) + raise TypeError( + "Doesn't support {} in index list.".format(type(i)) + ) if len(item) != 1: raise IndexError( - "When index contains a bool list, its length must be 1, but received {}." - .format(len(item))) + "When index contains a bool list, its length must be 1, but received {}.".format( + len(item) + ) + ) from .layers import assign + idx_tensor = assign(slice_item) return set_value_for_bool_tensor(var, idx_tensor, value) @@ -660,8 +726,10 @@ def _setitem_impl_(var, item, value): if slice_item.dtype == core.VarDesc.VarType.BOOL: if len(item) != 1: raise IndexError( - "When index contains a bool tensor, its length must be 1, but received {}." - .format(len(item))) + "When index contains a bool tensor, its length must be 1, but received {}.".format( + len(item) + ) + ) return set_value_for_bool_tensor(var, slice_item, value) else: slice_info.update(slice_item) @@ -669,7 +737,8 @@ def _setitem_impl_(var, item, value): else: raise IndexError( "Valid index accept int, slice, ellipsis, None, list of bool, Variable, " - "but received {}.".format(slice_item)) + "but received {}.".format(slice_item) + ) axes.append(dim) starts.append(start) @@ -680,8 +749,10 @@ def _setitem_impl_(var, item, value): if slice_info.indexes: if len(slice_info.indexes) != len(item): raise IndexError( - "Valid index accept int or slice or ellipsis or list, but received {}." - .format(item)) + "Valid index accept int or slice or ellipsis or list, but received {}.".format( + item + ) + ) return slice_info.set_item(var, value) attrs = { 'axes': axes, @@ -689,10 +760,11 @@ def _setitem_impl_(var, item, value): 'ends': ends, 'steps': steps, 'decrease_axes': decrease_axes, - 'none_axes': none_axes + 'none_axes': none_axes, } from .layers import utils + if utils._contain_var(starts): inputs['StartsTensorList'] = utils._convert_to_tensor_list(starts) del attrs['starts'] @@ -708,6 +780,7 @@ def _setitem_impl_(var, item, value): attrs['dtype'] = dtype from .data_feeder import convert_dtype + # 2.1 value is an integer of float if isinstance(value, (int, float)): value = np.array([value]).astype(convert_dtype(dtype)) @@ -737,7 +810,8 @@ def _setitem_impl_(var, item, value): raise TypeError( "When assign a numpy.ndarray, integer or float to a paddle.Tensor, " "the data type of the paddle.Tensor must be bool, float32, int32, int64 or float16, but " - "received %s." % convert_dtype(dtype)) + "received %s." % convert_dtype(dtype) + ) attrs[value_name] = values attrs["shape"] = shape @@ -747,17 +821,21 @@ def _setitem_impl_(var, item, value): raise TypeError( "Only support to assign an integer, float, numpy.ndarray or " "paddle.Tensor to a paddle.Tensor, but received {}".format( - type(value))) + type(value) + ) + ) if paddle.fluid.framework._non_static_mode(): var._bump_inplace_version() cur_block = default_main_program().current_block() - cur_block.append_op(type="set_value", - inputs=inputs, - outputs={'Out': var}, - attrs=attrs, - inplace_map={"Input": "Out"}) + cur_block.append_op( + type="set_value", + inputs=inputs, + outputs={'Out': var}, + attrs=attrs, + inplace_map={"Input": "Out"}, + ) return var @@ -765,16 +843,19 @@ def _setitem_impl_(var, item, value): # the item is a tensor of bool def set_value_for_bool_tensor(var, item, value): if len(item.shape) > len(var.shape): - raise IndexError("The dims of bool index doesn't match indexed array, " - "the dims of bool index except to be equal or less " - "than {}, but received {}.".format( - len(var.shape), len(item.shape))) + raise IndexError( + "The dims of bool index doesn't match indexed array, " + "the dims of bool index except to be equal or less " + "than {}, but received {}.".format(len(var.shape), len(item.shape)) + ) for i, dim_len in enumerate(item.shape): if dim_len != var.shape[i]: raise IndexError( "The dimension of bool index doesn't match indexed array along " - "dimension {}, the target dimension is {}, but received {}.". - format(i, var.shape[i], dim_len)) + "dimension {}, the target dimension is {}, but received {}.".format( + i, var.shape[i], dim_len + ) + ) def idx_not_empty(var, item, value): from .framework import Variable @@ -792,6 +873,7 @@ def idx_not_empty(var, item, value): var[:] = out from .layers.control_flow import cond + # If all the bool index is False, just do nothing cond(item.any(), lambda: idx_not_empty(var, item, value)) diff --git a/python/paddle/fluid/wrapped_decorator.py b/python/paddle/fluid/wrapped_decorator.py index 5f837b575637c..7e7dbff65611e 100644 --- a/python/paddle/fluid/wrapped_decorator.py +++ b/python/paddle/fluid/wrapped_decorator.py @@ -19,7 +19,6 @@ def wrap_decorator(decorator_func): - @decorator.decorator def __impl__(func, *args, **kwargs): wrapped_func = decorator_func(func) diff --git a/python/paddle/framework/__init__.py b/python/paddle/framework/__init__.py index 34423e3f3ed6b..a6b79dd29e285 100644 --- a/python/paddle/framework/__init__.py +++ b/python/paddle/framework/__init__.py @@ -47,10 +47,19 @@ from ..fluid.dygraph.base import enable_dygraph as disable_static # noqa: F401 from ..fluid.dygraph.base import disable_dygraph as enable_static # noqa: F401 from ..fluid.framework import _non_static_mode as in_dynamic_mode # noqa: F401 -from ..fluid.framework import _non_static_mode # noqa: F401; temporary used for hackson -from ..fluid.framework import _current_expected_place, _get_paddle_place # noqa: F401 +from ..fluid.framework import ( + _non_static_mode, +) # noqa: F401; temporary used for hackson +from ..fluid.framework import ( + _current_expected_place, + _get_paddle_place, +) # noqa: F401 from ..fluid.framework import dygraph_only # noqa: F401 -from ..fluid.framework import convert_np_dtype_to_dtype_, _varbase_creator, OpProtoHolder # noqa: F401 +from ..fluid.framework import ( + convert_np_dtype_to_dtype_, + _varbase_creator, + OpProtoHolder, +) # noqa: F401 from ..fluid.framework import _dygraph_tracer # noqa: F401 from ..fluid.layer_helper import LayerHelper # noqa: F401 diff --git a/python/paddle/framework/framework.py b/python/paddle/framework/framework.py index 8da6a10a48527..e366b13077d2a 100644 --- a/python/paddle/framework/framework.py +++ b/python/paddle/framework/framework.py @@ -45,11 +45,16 @@ def set_default_dtype(d): else: raise TypeError( "set_default_dtype only supports [float16, float32, float64] " - ", but received %s" % d.__name__) + ", but received %s" % d.__name__ + ) else: if d in [ - 'float16', 'float32', 'float64', u'float16', u'float32', - u'float64' + 'float16', + 'float32', + 'float64', + u'float16', + u'float32', + u'float64', ]: # this code is a little bit dangerous, since error could happen # when casting no-ascii code to str in python2. @@ -60,7 +65,8 @@ def set_default_dtype(d): else: raise TypeError( "set_default_dtype only supports [float16, float32, float64] " - ", but received %s" % str(d)) + ", but received %s" % str(d) + ) LayerHelperBase.set_default_dtype(d) diff --git a/python/paddle/framework/io.py b/python/paddle/framework/io.py index 77e1fb47f882d..0052c1fe65558 100644 --- a/python/paddle/framework/io.py +++ b/python/paddle/framework/io.py @@ -24,14 +24,31 @@ # deprecated module import from paddle import fluid from paddle.fluid import core -from paddle.fluid.io import _unpack_saved_dict, _pack_loaded_dict, _pickle_loads_mac +from paddle.fluid.io import ( + _unpack_saved_dict, + _pack_loaded_dict, + _pickle_loads_mac, +) from paddle.fluid.io import _legacy_save as _legacy_static_save from paddle.fluid.io import _open_file_buffer, _is_file_path, _is_memory_buffer -from paddle.fluid.framework import Variable, _varbase_creator, _dygraph_tracer, _non_static_mode, ParamBase, EagerParamBase, _current_expected_place, Program +from paddle.fluid.framework import ( + Variable, + _varbase_creator, + _dygraph_tracer, + _non_static_mode, + ParamBase, + EagerParamBase, + _current_expected_place, + Program, +) from paddle.fluid.dygraph.jit import _SaveLoadConfig -from paddle.fluid.dygraph.io import _construct_program_holders, _construct_params_and_buffers +from paddle.fluid.dygraph.io import ( + _construct_program_holders, + _construct_params_and_buffers, +) from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX + try: from collections.abc import Iterable except: @@ -68,7 +85,8 @@ def _load_state_dict_from_save_inference_model(model_path, config): # 2. load layer parameters & buffers with fluid.dygraph.guard(): persistable_var_dict = _construct_params_and_buffers( - model_path, programs, config.params_filename, append_suffix=False) + model_path, programs, config.params_filename, append_suffix=False + ) # 3. construct state_dict load_param_dict = dict() @@ -84,10 +102,15 @@ def _load_state_dict_from_save_inference_model(model_path, config): structured_para_dict = dict() for var_name in load_param_dict: structured_name = extra_var_info[var_name].get( - 'structured_name', None) - assert structured_name is not None, "Cannot find saved variable (%s)'s structured name in saved model." % var_name + 'structured_name', None + ) + assert structured_name is not None, ( + "Cannot find saved variable (%s)'s structured name in saved model." + % var_name + ) structured_para_dict[structured_name] = load_param_dict[ - var_name] + var_name + ] load_param_dict = structured_para_dict return load_param_dict @@ -115,7 +138,8 @@ def _load_state_dict_from_save_params(model_path): type='load', inputs={}, outputs={'Out': new_var}, - attrs={'file_path': os.path.join(model_path, name)}) + attrs={'file_path': os.path.join(model_path, name)}, + ) load_var_list.append(new_var) # 3. construct state_dict @@ -151,7 +175,8 @@ def _build_load_path_and_config(path, config): raise ValueError( "The %s.pdmodel and %s directory exist at the same time, " "don't know which one to load, please make sure that the specified target " - "of ``path`` is unique." % (path, path)) + "of ``path`` is unique." % (path, path) + ) elif not prefix_format_exist and not directory_format_exist: error_msg = "The ``path`` (%s) to load model not exists." # if current path is a prefix, and the path.pdparams or path.pdopt @@ -160,10 +185,12 @@ def _build_load_path_and_config(path, config): params_file_path = path + ".pdparams" opti_file_path = path + ".pdopt" if os.path.exists(params_file_path) or os.path.exists(opti_file_path): - error_msg += " If you want to load the results saved by `fluid.save_dygraph`, " \ - "please specify the full file name, not just the file name prefix. For " \ - "example, it should be written as `paddle.load('model.pdparams')` instead of " \ + error_msg += ( + " If you want to load the results saved by `fluid.save_dygraph`, " + "please specify the full file name, not just the file name prefix. For " + "example, it should be written as `paddle.load('model.pdparams')` instead of " "`paddle.load('model')`." + ) raise ValueError(error_msg % path) else: if prefix_format_exist: @@ -173,13 +200,15 @@ def _build_load_path_and_config(path, config): warnings.warn( "When loading the result saved with the " "specified file prefix, the ``model_filename`` config does " - "not take effect.") + "not take effect." + ) config.model_filename = file_prefix + INFER_MODEL_SUFFIX if config.params_filename is not None: warnings.warn( "When loading the result saved with the " "specified file prefix, the ``params_filename`` config does " - "not take effect.") + "not take effect." + ) config.params_filename = file_prefix + INFER_PARAMS_SUFFIX else: # Compatible with the old save_inference_model format @@ -190,7 +219,10 @@ def _build_load_path_and_config(path, config): def _parse_load_config(configs): supported_configs = [ - 'model_filename', 'params_filename', 'keep_name_table', 'return_numpy' + 'model_filename', + 'params_filename', + 'keep_name_table', + 'return_numpy', ] # input check @@ -198,7 +230,8 @@ def _parse_load_config(configs): if key not in supported_configs: raise ValueError( "The additional config (%s) of `paddle.load` is not supported." - % key) + % key + ) # construct inner config inner_config = _SaveLoadConfig() @@ -218,7 +251,8 @@ def _parse_save_config(configs): if key not in supported_configs: raise ValueError( "The additional config (%s) of `paddle.save` is not supported." - % key) + % key + ) # construct inner config inner_config = _SaveLoadConfig() @@ -231,19 +265,22 @@ def _parse_save_config(configs): def _pickle_save(obj, f, protocol): # TODO(weixin):add support for BytesIO. if not isinstance(protocol, int): - raise ValueError("The 'protocol' MUST be `int`, but received {}".format( - type(protocol))) + raise ValueError( + "The 'protocol' MUST be `int`, but received {}".format( + type(protocol) + ) + ) if protocol < 2 or protocol > 4: raise ValueError( - "Expected 1<'protocol'<5, but received protocol={}".format( - protocol)) + "Expected 1<'protocol'<5, but received protocol={}".format(protocol) + ) def reduce_varbase(self): data = self.numpy() name = self.name - return (tuple, ((name, data), )) + return (tuple, ((name, data),)) def reduce_LoDTensor(self): data = np.array(self) @@ -252,7 +289,8 @@ def reduce_LoDTensor(self): def reduce_Layer(self): raise ValueError( - "paddle do not support saving `paddle.nn.Layer` object.") + "paddle do not support saving `paddle.nn.Layer` object." + ) dispatch_table_layer = dict() @@ -260,8 +298,9 @@ def create_layer_dispatch_table(layer): dispatch_table_layer[layer.__class__] = reduce_Layer return layer - _parse_every_object(obj, lambda v: isinstance(v, fluid.Layer), - create_layer_dispatch_table) + _parse_every_object( + obj, lambda v: isinstance(v, fluid.Layer), create_layer_dispatch_table + ) def add_dispatch_table(): # This is not a good method, because the pickle module has been modified. @@ -289,7 +328,7 @@ def pop_dispatch_table(): max_bytes = 2**30 for i in range(0, len(pickle_bytes), max_bytes): - f.write(pickle_bytes[i:i + max_bytes]) + f.write(pickle_bytes[i : i + max_bytes]) else: pickler = pickle.Pickler(f, protocol) pickler.dispatch_table = copyreg.dispatch_table.copy() @@ -306,7 +345,8 @@ def pop_dispatch_table(): def _contain_x(obj, condition_func): if isinstance(obj, core.SelectedRows): raise NotImplementedError( - "`paddle.save` do not support saving 'SelectedRows'.") + "`paddle.save` do not support saving 'SelectedRows'." + ) if condition_func(obj): return True @@ -330,8 +370,16 @@ def _is_state_dict(obj): def condition(obj): return isinstance( - obj, (fluid.Layer, Program, core.VarBase, core.eager.Tensor, - core.LoDTensor, core.SelectedRows)) + obj, + ( + fluid.Layer, + Program, + core.VarBase, + core.eager.Tensor, + core.LoDTensor, + core.SelectedRows, + ), + ) # If the value of a dict is a core.VarBase/LoDTensor or a dict # that does not contain a paddle type(Layer, Program, VarBase, LoDTensor, SelectedRows), @@ -342,7 +390,8 @@ def condition(obj): if _contain_x(v, condition): return False elif not isinstance( - value, (core.VarBase, core.eager.Tensor, core.LoDTensor)): + value, (core.VarBase, core.eager.Tensor, core.LoDTensor) + ): return False return True @@ -370,8 +419,10 @@ def _transformed_from_lodtensor(obj): def _to_LodTensor(ndarray): if not isinstance(ndarray, np.ndarray): raise TypeError( - 'Type of `ndarray` should be numpy.ndarray, but received {}.'. - format(type(ndarray))) + 'Type of `ndarray` should be numpy.ndarray, but received {}.'.format( + type(ndarray) + ) + ) t = core.LoDTensor() place = _current_expected_place() t.set(ndarray, place) @@ -418,26 +469,30 @@ def _parse_every_object(obj, condition_func, convert_func): if condition_func(obj[key]): obj[key] = convert_func(obj[key]) else: - obj[key] = _parse_every_object(obj[key], condition_func, - convert_func) + obj[key] = _parse_every_object( + obj[key], condition_func, convert_func + ) return obj elif type(obj) == tuple: return tuple( - _parse_every_object(list(obj), condition_func, convert_func)) + _parse_every_object(list(obj), condition_func, convert_func) + ) elif type(obj) == set: return set(_parse_every_object(list(obj), condition_func, convert_func)) else: if isinstance(obj, Iterable) and not isinstance( - obj, - (str, np.ndarray, core.VarBase, core.eager.Tensor, core.LoDTensor)): + obj, + (str, np.ndarray, core.VarBase, core.eager.Tensor, core.LoDTensor), + ): raise NotImplementedError( - "The iteratable objects supported are tuple, list, dict, OrderedDict, string. But received {}." - .format(type(obj))) + "The iteratable objects supported are tuple, list, dict, OrderedDict, string. But received {}.".format( + type(obj) + ) + ) return obj def _parse_load_result(obj, return_numpy): - def is_layer(obj): return isinstance(obj, fluid.Layer) @@ -463,13 +518,15 @@ def ndarray_to_tensor(obj): # tuple(name, ndarry) was converted from varbase of paddle2.1, # and all tuple(name, ndarry) are converted to tensor. if _contain_x(obj, _transformed_from_varbase): - return _parse_every_object(obj, _transformed_from_varbase, - tuple_to_tensor) + return _parse_every_object( + obj, _transformed_from_varbase, tuple_to_tensor + ) # If there is no tuple(name, ndary), it is considered to be saved by paddle2.0 # or converted from LoDTensor, and all ndarrays are converted to tensor. else: - return _parse_every_object(obj, _transformed_from_lodtensor, - ndarray_to_tensor) + return _parse_every_object( + obj, _transformed_from_lodtensor, ndarray_to_tensor + ) def _save_lod_tensor(tensor, file_name): @@ -490,8 +547,10 @@ def _save_lod_tensor(tensor, file_name): else: raise NotImplementedError( - 'Only supports saving objects to file or BytesIO, but received {}'. - format(type(file_name))) + 'Only supports saving objects to file or BytesIO, but received {}'.format( + type(file_name) + ) + ) return _seek @@ -509,8 +568,10 @@ def _load_lod_tensor(file_name): else: raise NotImplementedError( - 'Only supports load objects from file or BytesIO, but received {}'. - format(type(file_name))) + 'Only supports load objects from file or BytesIO, but received {}'.format( + type(file_name) + ) + ) return temp_t, _seek @@ -529,8 +590,10 @@ def _save_selected_rows(selected_rows, file_name): _seek = f.tell() else: raise NotImplementedError( - 'Only supports saving objects to file or BytesIO, but received {}'. - format(type(file_name))) + 'Only supports saving objects to file or BytesIO, but received {}'.format( + type(file_name) + ) + ) return _seek @@ -544,13 +607,16 @@ def _load_selected_rows(file_name): with _open_file_buffer(file_name, 'rb') as f: selected_rows_bytes = f.read() paddle.fluid.core.load_selected_rows_from_memory( - temp_sr, selected_rows_bytes) + temp_sr, selected_rows_bytes + ) _seek = f.tell() else: raise NotImplementedError( - 'Only supports load objects from file or BytesIO, but received {}'. - format(type(file_name))) + 'Only supports load objects from file or BytesIO, but received {}'.format( + type(file_name) + ) + ) return temp_sr, _seek @@ -565,8 +631,10 @@ def _save_binary_var(obj, path): else: # Since the concept of 'Tensor' is only exposed to users, the error message can only contain tensor instead of 'LoDTensor' or 'SelectedRows' raise NotImplementedError( - "When use_binary_format = True, `paddle.save` expected Tensor, but received {}." - .format(type(obj))) + "When use_binary_format = True, `paddle.save` expected Tensor, but received {}.".format( + type(obj) + ) + ) def save(obj, path, protocol=4, **configs): @@ -694,7 +762,8 @@ def save(obj, path, protocol=4, **configs): raise ValueError( "The input path MUST be format of dirname/filename " "[dirname\\filename in Windows system], but received " - "filename is empty string.") + "filename is empty string." + ) # 2. save object dirname = os.path.dirname(path) @@ -702,15 +771,19 @@ def save(obj, path, protocol=4, **configs): os.makedirs(dirname) elif not _is_memory_buffer(path): raise ValueError( - "only supports saving objects to file and `BytesIO`, but got {}". - format(type(path))) + "only supports saving objects to file and `BytesIO`, but got {}".format( + type(path) + ) + ) config = _parse_save_config(configs) if not isinstance(config.use_binary_format, bool): raise TypeError( - "Type of `use_binary_format` should be bool, but received {}.". - format(type(config.use_binary_format))) + "Type of `use_binary_format` should be bool, but received {}.".format( + type(config.use_binary_format) + ) + ) if config.use_binary_format: _save_binary_var(obj, path) @@ -742,19 +815,23 @@ def _legacy_save(obj, path, protocol=2): if not isinstance(obj, dict): raise NotImplementedError( "Now only supports save state_dict of Layer or Optimizer, " - "expect dict, but received %s." % type(obj)) + "expect dict, but received %s." % type(obj) + ) if len(obj) == 0: warnings.warn("The input state dict is empty, no need to save.") if not isinstance(protocol, int): - raise ValueError("The 'protocol' MUST be `int`, but received {}".format( - type(protocol))) + raise ValueError( + "The 'protocol' MUST be `int`, but received {}".format( + type(protocol) + ) + ) if protocol < 2 or protocol > 4: raise ValueError( - "Expected 1<'protocol'<5, but received protocol={}".format( - protocol)) + "Expected 1<'protocol'<5, but received protocol={}".format(protocol) + ) if _is_file_path(path): filename = os.path.basename(path) @@ -762,7 +839,8 @@ def _legacy_save(obj, path, protocol=2): raise ValueError( "The input path MUST be format of dirname/filename " "[dirname\\filename in Windows system], but received " - "filename is empty string.") + "filename is empty string." + ) # 2. save object dirname = os.path.dirname(path) if dirname and not os.path.exists(dirname): @@ -774,13 +852,16 @@ def _legacy_save(obj, path, protocol=2): saved_obj = _unpack_saved_dict(saved_obj, protocol) # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' - if _is_file_path( - path) and sys.platform == 'darwin' and sys.version_info.major == 3: + if ( + _is_file_path(path) + and sys.platform == 'darwin' + and sys.version_info.major == 3 + ): pickle_bytes = pickle.dumps(saved_obj, protocol=protocol) with open(path, 'wb') as f: max_bytes = 2**30 for i in range(0, len(pickle_bytes), max_bytes): - f.write(pickle_bytes[i:i + max_bytes]) + f.write(pickle_bytes[i : i + max_bytes]) else: with _open_file_buffer(path, 'wb') as f: pickle.dump(saved_obj, f, protocol=protocol) @@ -947,9 +1028,11 @@ def load(path, **configs): try: with _open_file_buffer(path, 'rb') as f: # When value of dict is lager than 4GB ,there is a Bug on 'MAC python3' - if _is_file_path( - path - ) and sys.platform == 'darwin' and sys.version_info.major == 3: + if ( + _is_file_path(path) + and sys.platform == 'darwin' + and sys.version_info.major == 3 + ): load_result = _pickle_loads_mac(path, f) else: load_result = pickle.load(f, encoding='latin1') @@ -963,18 +1046,24 @@ def load(path, **configs): for key in load_result["StructuredToParameterName@@"]: if isinstance(load_result[key], np.ndarray): load_result[key] = _ndarray_to_tensor( - load_result[key], config.return_numpy) + load_result[key], config.return_numpy + ) - if not config.keep_name_table and "StructuredToParameterName@@" in load_result: + if ( + not config.keep_name_table + and "StructuredToParameterName@@" in load_result + ): del load_result["StructuredToParameterName@@"] else: # paddle2.1 static.save/load load_result = _parse_load_result( - load_result, config.return_numpy) + load_result, config.return_numpy + ) else: - load_result = _parse_load_result(load_result, - config.return_numpy) + load_result = _parse_load_result( + load_result, config.return_numpy + ) except exception_type as msg_pickle: try: @@ -994,12 +1083,15 @@ def load(path, **configs): with _open_file_buffer(path, "rb") as f: program_desc_str = f.read() program = Program.parse_from_string( - program_desc_str) + program_desc_str + ) return program except: raise ValueError( "`paddle.load` can not parse the file:{}.".format( - path)) + path + ) + ) else: load_result = _legacy_load(path, **configs) @@ -1016,7 +1108,10 @@ def _legacy_load(path, **configs): with _open_file_buffer(path, 'rb') as f: load_result = pickle.load(f, encoding='latin1') load_result = _pack_loaded_dict(load_result) - if not config.keep_name_table and "StructuredToParameterName@@" in load_result: + if ( + not config.keep_name_table + and "StructuredToParameterName@@" in load_result + ): del load_result["StructuredToParameterName@@"] else: # file prefix and directory are compatible cases @@ -1037,7 +1132,8 @@ def _legacy_load(path, **configs): # the user to configure the `use_structured_name` argument when `set_state_dict` # NOTE(chenweihang): `jit.save` doesn't save optimizer state load_result = _load_state_dict_from_save_inference_model( - model_path, config) + model_path, config + ) else: # load state dict by `io.save_params/persistables` save format # TODO(chenweihang): [ Now only supports loading parameters separately ] diff --git a/python/paddle/framework/random.py b/python/paddle/framework/random.py index 50235591cf2f1..6d7d704808d7e 100644 --- a/python/paddle/framework/random.py +++ b/python/paddle/framework/random.py @@ -37,7 +37,7 @@ def seed(seed): gen = paddle.seed(102) """ - #TODO(zhiqiu): 1. remove program.random_seed when all random-related op upgrade + # TODO(zhiqiu): 1. remove program.random_seed when all random-related op upgrade # 2. support gpu generator by global device seed = int(seed) diff --git a/python/paddle/geometric/math.py b/python/paddle/geometric/math.py index 186fe5c2d7ddd..a4a27093484e1 100644 --- a/python/paddle/geometric/math.py +++ b/python/paddle/geometric/math.py @@ -54,29 +54,30 @@ def segment_sum(data, segment_ids, name=None): if in_dygraph_mode(): return _C_ops.segment_pool(data, segment_ids, "SUM")[0] if _in_legacy_dygraph(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "SUM") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "SUM" + ) return out check_variable_and_dtype( - data, "X", ("float32", "float64", "int32", "int64", "float16"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + data, + "X", + ("float32", "float64", "int32", "int64", "float16"), + "segment_pool", + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_sum", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "SUM"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "SUM"}, + ) return out @@ -116,29 +117,30 @@ def segment_mean(data, segment_ids, name=None): if in_dygraph_mode(): return _C_ops.segment_pool(data, segment_ids, "MEAN")[0] if _in_legacy_dygraph(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "MEAN") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "MEAN" + ) return out check_variable_and_dtype( - data, "X", ("float32", "float64", "int32", "int64", "float16"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + data, + "X", + ("float32", "float64", "int32", "int64", "float16"), + "segment_pool", + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_mean", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "MEAN"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "MEAN"}, + ) return out @@ -177,29 +179,30 @@ def segment_min(data, segment_ids, name=None): if in_dygraph_mode(): return _C_ops.segment_pool(data, segment_ids, "MIN")[0] if _in_legacy_dygraph(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "MIN") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "MIN" + ) return out check_variable_and_dtype( - data, "X", ("float32", "float64", "int32", "int64", "float16"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + data, + "X", + ("float32", "float64", "int32", "int64", "float16"), + "segment_pool", + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_min", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "MIN"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "MIN"}, + ) return out @@ -238,27 +241,28 @@ def segment_max(data, segment_ids, name=None): if in_dygraph_mode(): return _C_ops.segment_pool(data, segment_ids, "MAX")[0] if _in_legacy_dygraph(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "MAX") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "MAX" + ) return out check_variable_and_dtype( - data, "X", ("float32", "float64", "int32", "int64", "float16"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + data, + "X", + ("float32", "float64", "int32", "int64", "float16"), + "segment_pool", + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_max", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "MAX"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "MAX"}, + ) return out diff --git a/python/paddle/geometric/message_passing/send_recv.py b/python/paddle/geometric/message_passing/send_recv.py index cf290b29bee89..8a05ef1051626 100644 --- a/python/paddle/geometric/message_passing/send_recv.py +++ b/python/paddle/geometric/message_passing/send_recv.py @@ -16,20 +16,25 @@ from paddle.fluid.layer_helper import LayerHelper from paddle.fluid.framework import _in_legacy_dygraph, in_dygraph_mode from paddle.fluid.framework import Variable -from paddle.fluid.data_feeder import check_dtype, check_type, check_variable_and_dtype +from paddle.fluid.data_feeder import ( + check_dtype, + check_type, + check_variable_and_dtype, +) from paddle import _C_ops, _legacy_C_ops -from .utils import convert_out_size_to_list, get_out_size_tensor_inputs, reshape_lhs_rhs +from .utils import ( + convert_out_size_to_list, + get_out_size_tensor_inputs, + reshape_lhs_rhs, +) __all__ = [] -def send_u_recv(x, - src_index, - dst_index, - reduce_op="sum", - out_size=None, - name=None): +def send_u_recv( + x, src_index, dst_index, reduce_op="sum", out_size=None, name=None +): """ Graph Learning message passing api. @@ -110,66 +115,85 @@ def send_u_recv(x, if reduce_op not in ["sum", "mean", "max", "min"]: raise ValueError( "reduce_op should be `sum`, `mean`, `max` or `min`, but received %s" - % reduce_op) + % reduce_op + ) # TODO(daisiming): Should we add judgement for out_size: max(dst_index) + 1. if _in_legacy_dygraph(): out_size = convert_out_size_to_list(out_size) - out, tmp = _legacy_C_ops.graph_send_recv(x, src_index, dst_index, - None, 'reduce_op', - reduce_op.upper(), 'out_size', - out_size) + out, tmp = _legacy_C_ops.graph_send_recv( + x, + src_index, + dst_index, + None, + 'reduce_op', + reduce_op.upper(), + 'out_size', + out_size, + ) return out if in_dygraph_mode(): out_size = convert_out_size_to_list(out_size) - return _C_ops.graph_send_recv(x, src_index, dst_index, - reduce_op.upper(), out_size) + return _C_ops.graph_send_recv( + x, src_index, dst_index, reduce_op.upper(), out_size + ) check_variable_and_dtype( - x, "X", ("float32", "float64", "int32", "int64", "float16"), - "graph_send_recv") - check_variable_and_dtype(src_index, "Src_index", ("int32", "int64"), - "graph_send_recv") - check_variable_and_dtype(dst_index, "Dst_index", ("int32", "int64"), - "graph_send_recv") + x, + "X", + ("float32", "float64", "int32", "int64", "float16"), + "graph_send_recv", + ) + check_variable_and_dtype( + src_index, "Src_index", ("int32", "int64"), "graph_send_recv" + ) + check_variable_and_dtype( + dst_index, "Dst_index", ("int32", "int64"), "graph_send_recv" + ) if out_size: - check_type(out_size, 'out_size', (int, np.int32, np.int64, Variable), - 'graph_send_recv') + check_type( + out_size, + 'out_size', + (int, np.int32, np.int64, Variable), + 'graph_send_recv', + ) if isinstance(out_size, Variable): - check_dtype(out_size.dtype, 'out_size', ['int32', 'int64'], - 'graph_send_recv') + check_dtype( + out_size.dtype, 'out_size', ['int32', 'int64'], 'graph_send_recv' + ) helper = LayerHelper("send_u_recv", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - dst_count = helper.create_variable_for_type_inference(dtype="int32", - stop_gradient=True) + dst_count = helper.create_variable_for_type_inference( + dtype="int32", stop_gradient=True + ) inputs = {"X": x, "Src_index": src_index, "Dst_index": dst_index} attrs = {"reduce_op": reduce_op.upper()} - get_out_size_tensor_inputs(inputs=inputs, - attrs=attrs, - out_size=out_size, - op_type='graph_send_recv') - - helper.append_op(type="graph_send_recv", - inputs=inputs, - outputs={ - "Out": out, - "Dst_count": dst_count - }, - attrs=attrs) + get_out_size_tensor_inputs( + inputs=inputs, attrs=attrs, out_size=out_size, op_type='graph_send_recv' + ) + + helper.append_op( + type="graph_send_recv", + inputs=inputs, + outputs={"Out": out, "Dst_count": dst_count}, + attrs=attrs, + ) return out -def send_ue_recv(x, - y, - src_index, - dst_index, - message_op="add", - reduce_op="sum", - out_size=None, - name=None): +def send_ue_recv( + x, + y, + src_index, + dst_index, + message_op="add", + reduce_op="sum", + out_size=None, + name=None, +): """ Graph Learning message passing api. @@ -259,13 +283,15 @@ def send_ue_recv(x, if message_op not in ["add", "sub", "mul", "div"]: raise ValueError( - "message_op should be `add`, `sub`, `mul`, `div`, but received %s" % - message_op) + "message_op should be `add`, `sub`, `mul`, `div`, but received %s" + % message_op + ) if reduce_op not in ["sum", "mean", "max", "min"]: raise ValueError( "reduce_op should be `sum`, `mean`, `max` or `min`, but received %s" - % reduce_op) + % reduce_op + ) x, y = reshape_lhs_rhs(x, y) @@ -274,61 +300,89 @@ def send_ue_recv(x, y = -y if message_op == "div": message_op = 'mul' - y = 1. / (y + 1e-12) + y = 1.0 / (y + 1e-12) # TODO(daisiming): Should we add judgement for out_size: max(dst_index) + 1. if _in_legacy_dygraph(): out_size = convert_out_size_to_list(out_size) - out, tmp = _legacy_C_ops.graph_send_ue_recv(x, y, src_index, dst_index, - None, 'message_op', - message_op.upper(), - 'reduce_op', - reduce_op.upper(), - 'out_size', out_size) + out, tmp = _legacy_C_ops.graph_send_ue_recv( + x, + y, + src_index, + dst_index, + None, + 'message_op', + message_op.upper(), + 'reduce_op', + reduce_op.upper(), + 'out_size', + out_size, + ) return out if in_dygraph_mode(): out_size = convert_out_size_to_list(out_size) - return _C_ops.graph_send_ue_recv(x, y, src_index, dst_index, - message_op.upper(), reduce_op.upper(), - out_size) + return _C_ops.graph_send_ue_recv( + x, + y, + src_index, + dst_index, + message_op.upper(), + reduce_op.upper(), + out_size, + ) check_variable_and_dtype( - x, "X", ("float32", "float64", "int32", "int64", "float16"), - "graph_send_ue_recv") + x, + "X", + ("float32", "float64", "int32", "int64", "float16"), + "graph_send_ue_recv", + ) + check_variable_and_dtype( + y, + "Y", + ("float32", "float64", "int32", "int64", "float16"), + "graph_send_ue_recv", + ) check_variable_and_dtype( - y, "Y", ("float32", "float64", "int32", "int64", "float16"), - "graph_send_ue_recv") - check_variable_and_dtype(src_index, "Src_index", ("int32", "int64"), - "graph_send_ue_recv") - check_variable_and_dtype(dst_index, "Dst_index", ("int32", "int64"), - "graph_send_ue_recv") + src_index, "Src_index", ("int32", "int64"), "graph_send_ue_recv" + ) + check_variable_and_dtype( + dst_index, "Dst_index", ("int32", "int64"), "graph_send_ue_recv" + ) if out_size: - check_type(out_size, 'out_size', (int, np.int32, np.int64, Variable), - 'graph_send_ue_recv') + check_type( + out_size, + 'out_size', + (int, np.int32, np.int64, Variable), + 'graph_send_ue_recv', + ) if isinstance(out_size, Variable): - check_dtype(out_size.dtype, 'out_size', ['int32', 'int64'], - 'graph_send_ue_recv') + check_dtype( + out_size.dtype, 'out_size', ['int32', 'int64'], 'graph_send_ue_recv' + ) helper = LayerHelper("send_ue_recv", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - dst_count = helper.create_variable_for_type_inference(dtype="int32", - stop_gradient=True) + dst_count = helper.create_variable_for_type_inference( + dtype="int32", stop_gradient=True + ) inputs = {"X": x, "Y": y, "Src_index": src_index, "Dst_index": dst_index} attrs = {"message_op": message_op.upper(), "reduce_op": reduce_op.upper()} - get_out_size_tensor_inputs(inputs=inputs, - attrs=attrs, - out_size=out_size, - op_type='graph_send_ue_recv') - - helper.append_op(type="graph_send_ue_recv", - inputs=inputs, - outputs={ - "Out": out, - "Dst_count": dst_count - }, - attrs=attrs) + get_out_size_tensor_inputs( + inputs=inputs, + attrs=attrs, + out_size=out_size, + op_type='graph_send_ue_recv', + ) + + helper.append_op( + type="graph_send_ue_recv", + inputs=inputs, + outputs={"Out": out, "Dst_count": dst_count}, + attrs=attrs, + ) return out @@ -397,8 +451,9 @@ def send_uv(x, y, src_index, dst_index, message_op="add", name=None): if message_op not in ['add', 'sub', 'mul', 'div']: raise ValueError( - "message_op should be `add`, `sub`, `mul`, `div`, but received %s" % - message_op) + "message_op should be `add`, `sub`, `mul`, `div`, but received %s" + % message_op + ) x, y = reshape_lhs_rhs(x, y) @@ -407,38 +462,50 @@ def send_uv(x, y, src_index, dst_index, message_op="add", name=None): y = -y if message_op == 'div': message_op = 'mul' - y = 1. / (y + 1e-12) + y = 1.0 / (y + 1e-12) if in_dygraph_mode(): - return _C_ops.graph_send_uv(x, y, src_index, dst_index, - message_op.upper()) + return _C_ops.graph_send_uv( + x, y, src_index, dst_index, message_op.upper() + ) else: if _in_legacy_dygraph(): - return _legacy_C_ops.graph_send_uv(x, y, src_index, dst_index, - "message_op", message_op.upper()) + return _legacy_C_ops.graph_send_uv( + x, y, src_index, dst_index, "message_op", message_op.upper() + ) else: helper = LayerHelper("send_uv", **locals()) check_variable_and_dtype( - x, 'x', ['int32', 'int64', 'float32', 'float64', 'float16'], - 'graph_send_uv') + x, + 'x', + ['int32', 'int64', 'float32', 'float64', 'float16'], + 'graph_send_uv', + ) + check_variable_and_dtype( + y, + 'y', + ['int32', 'int64', 'float32', 'float64', 'float16'], + 'graph_send_uv', + ) + check_variable_and_dtype( + src_index, 'src_index', ['int32', 'int64'], 'graph_send_uv' + ) check_variable_and_dtype( - y, 'y', ['int32', 'int64', 'float32', 'float64', 'float16'], - 'graph_send_uv') - check_variable_and_dtype(src_index, 'src_index', ['int32', 'int64'], - 'graph_send_uv') - check_variable_and_dtype(dst_index, 'dst_index', ['int32', 'int64'], - 'graph_send_uv') + dst_index, 'dst_index', ['int32', 'int64'], 'graph_send_uv' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) inputs = { 'x': x, 'y': y, 'src_index': src_index, - 'dst_index': dst_index + 'dst_index': dst_index, } attrs = {'message_op': message_op.upper()} - helper.append_op(type="graph_send_uv", - inputs=inputs, - attrs=attrs, - outputs={"out": out}) + helper.append_op( + type="graph_send_uv", + inputs=inputs, + attrs=attrs, + outputs={"out": out}, + ) return out diff --git a/python/paddle/geometric/message_passing/utils.py b/python/paddle/geometric/message_passing/utils.py index 51c088522983b..12e2e52d6b454 100644 --- a/python/paddle/geometric/message_passing/utils.py +++ b/python/paddle/geometric/message_passing/utils.py @@ -44,9 +44,14 @@ def get_out_size_tensor_inputs(inputs, attrs, out_size, op_type): attrs['out_size'] = [out_size] elif isinstance(out_size, Variable): out_size.stop_gradient = True - check_dtype(out_size.dtype, 'out_size', ['int32', 'int64'], 'op_type', - '(When type of out_size in' + op_type + ' is Variable.)') - if (convert_dtype(out_size.dtype) == 'int64'): + check_dtype( + out_size.dtype, + 'out_size', + ['int32', 'int64'], + 'op_type', + '(When type of out_size in' + op_type + ' is Variable.)', + ) + if convert_dtype(out_size.dtype) == 'int64': out_size = cast(out_size, 'int32') inputs["Out_size"] = out_size else: @@ -69,16 +74,26 @@ def reshape_lhs_rhs(x, y): max_ndims = max(len(x.shape), len(y.shape)) x_pad_ndims = max_ndims - len(x.shape) y_pad_ndims = max_ndims - len(y.shape) - new_x_shape = [ - x_shape[0], - ] + [ - 1, - ] * x_pad_ndims + list(x_shape[1:]) - new_y_shape = [ - y_shape[0], - ] + [ - 1, - ] * y_pad_ndims + list(y_shape[1:]) + new_x_shape = ( + [ + x_shape[0], + ] + + [ + 1, + ] + * x_pad_ndims + + list(x_shape[1:]) + ) + new_y_shape = ( + [ + y_shape[0], + ] + + [ + 1, + ] + * y_pad_ndims + + list(y_shape[1:]) + ) x = paddle.reshape(x, new_x_shape) y = paddle.reshape(y, new_y_shape) diff --git a/python/paddle/geometric/reindex.py b/python/paddle/geometric/reindex.py index 3c4cfb964ed17..ade88222bbe55 100644 --- a/python/paddle/geometric/reindex.py +++ b/python/paddle/geometric/reindex.py @@ -21,12 +21,9 @@ __all__ = [] -def reindex_graph(x, - neighbors, - count, - value_buffer=None, - index_buffer=None, - name=None): +def reindex_graph( + x, neighbors, count, value_buffer=None, index_buffer=None, name=None +): """ Reindex Graph API. @@ -93,58 +90,62 @@ def reindex_graph(x, # out_nodes: [0, 1, 2, 8, 9, 4, 7, 6] """ - use_buffer_hashtable = True if value_buffer is not None \ - and index_buffer is not None else False + use_buffer_hashtable = ( + True if value_buffer is not None and index_buffer is not None else False + ) if _non_static_mode(): - reindex_src, reindex_dst, out_nodes = \ - _legacy_C_ops.graph_reindex(x, neighbors, count, value_buffer, index_buffer, - "flag_buffer_hashtable", use_buffer_hashtable) + reindex_src, reindex_dst, out_nodes = _legacy_C_ops.graph_reindex( + x, + neighbors, + count, + value_buffer, + index_buffer, + "flag_buffer_hashtable", + use_buffer_hashtable, + ) return reindex_src, reindex_dst, out_nodes check_variable_and_dtype(x, "X", ("int32", "int64"), "graph_reindex") - check_variable_and_dtype(neighbors, "Neighbors", ("int32", "int64"), - "graph_reindex") + check_variable_and_dtype( + neighbors, "Neighbors", ("int32", "int64"), "graph_reindex" + ) check_variable_and_dtype(count, "Count", ("int32"), "graph_reindex") if use_buffer_hashtable: - check_variable_and_dtype(value_buffer, "HashTable_Value", ("int32"), - "graph_reindex") - check_variable_and_dtype(index_buffer, "HashTable_Index", ("int32"), - "graph_reindex") + check_variable_and_dtype( + value_buffer, "HashTable_Value", ("int32"), "graph_reindex" + ) + check_variable_and_dtype( + index_buffer, "HashTable_Index", ("int32"), "graph_reindex" + ) helper = LayerHelper("reindex_graph", **locals()) reindex_src = helper.create_variable_for_type_inference(dtype=x.dtype) reindex_dst = helper.create_variable_for_type_inference(dtype=x.dtype) out_nodes = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="graph_reindex", - inputs={ - "X": - x, - "Neighbors": - neighbors, - "Count": - count, - "HashTable_Value": - value_buffer if use_buffer_hashtable else None, - "HashTable_Index": - index_buffer if use_buffer_hashtable else None, - }, - outputs={ - "Reindex_Src": reindex_src, - "Reindex_Dst": reindex_dst, - "Out_Nodes": out_nodes - }, - attrs={"flag_buffer_hashtable": use_buffer_hashtable}) + helper.append_op( + type="graph_reindex", + inputs={ + "X": x, + "Neighbors": neighbors, + "Count": count, + "HashTable_Value": value_buffer if use_buffer_hashtable else None, + "HashTable_Index": index_buffer if use_buffer_hashtable else None, + }, + outputs={ + "Reindex_Src": reindex_src, + "Reindex_Dst": reindex_dst, + "Out_Nodes": out_nodes, + }, + attrs={"flag_buffer_hashtable": use_buffer_hashtable}, + ) return reindex_src, reindex_dst, out_nodes -def reindex_heter_graph(x, - neighbors, - count, - value_buffer=None, - index_buffer=None, - name=None): +def reindex_heter_graph( + x, neighbors, count, value_buffer=None, index_buffer=None, name=None +): """ Reindex HeterGraph API. @@ -220,15 +221,22 @@ def reindex_heter_graph(x, # out_nodes: [0, 1, 2, 8, 9, 4, 7, 6, 3, 5] """ - use_buffer_hashtable = True if value_buffer is not None \ - and index_buffer is not None else False + use_buffer_hashtable = ( + True if value_buffer is not None and index_buffer is not None else False + ) if _non_static_mode(): neighbors = paddle.concat(neighbors, axis=0) count = paddle.concat(count, axis=0) - reindex_src, reindex_dst, out_nodes = \ - _legacy_C_ops.graph_reindex(x, neighbors, count, value_buffer, index_buffer, - "flag_buffer_hashtable", use_buffer_hashtable) + reindex_src, reindex_dst, out_nodes = _legacy_C_ops.graph_reindex( + x, + neighbors, + count, + value_buffer, + index_buffer, + "flag_buffer_hashtable", + use_buffer_hashtable, + ) return reindex_src, reindex_dst, out_nodes if isinstance(neighbors, Variable): @@ -240,15 +248,18 @@ def reindex_heter_graph(x, count = paddle.concat(count, axis=0) check_variable_and_dtype(x, "X", ("int32", "int64"), "heter_graph_reindex") - check_variable_and_dtype(neighbors, "Neighbors", ("int32", "int64"), - "graph_reindex") + check_variable_and_dtype( + neighbors, "Neighbors", ("int32", "int64"), "graph_reindex" + ) check_variable_and_dtype(count, "Count", ("int32"), "graph_reindex") if use_buffer_hashtable: - check_variable_and_dtype(value_buffer, "HashTable_Value", ("int32"), - "graph_reindex") - check_variable_and_dtype(index_buffer, "HashTable_Index", ("int32"), - "graph_reindex") + check_variable_and_dtype( + value_buffer, "HashTable_Value", ("int32"), "graph_reindex" + ) + check_variable_and_dtype( + index_buffer, "HashTable_Index", ("int32"), "graph_reindex" + ) helper = LayerHelper("reindex_heter_graph", **locals()) reindex_src = helper.create_variable_for_type_inference(dtype=x.dtype) @@ -256,23 +267,20 @@ def reindex_heter_graph(x, out_nodes = helper.create_variable_for_type_inference(dtype=x.dtype) neighbors = paddle.concat(neighbors, axis=0) count = paddle.concat(count, axis=0) - helper.append_op(type="graph_reindex", - inputs={ - "X": - x, - "Neighbors": - neighbors, - "Count": - count, - "HashTable_Value": - value_buffer if use_buffer_hashtable else None, - "HashTable_Index": - index_buffer if use_buffer_hashtable else None, - }, - outputs={ - "Reindex_Src": reindex_src, - "Reindex_Dst": reindex_dst, - "Out_Nodes": out_nodes - }, - attrs={"flag_buffer_hashtable": use_buffer_hashtable}) + helper.append_op( + type="graph_reindex", + inputs={ + "X": x, + "Neighbors": neighbors, + "Count": count, + "HashTable_Value": value_buffer if use_buffer_hashtable else None, + "HashTable_Index": index_buffer if use_buffer_hashtable else None, + }, + outputs={ + "Reindex_Src": reindex_src, + "Reindex_Dst": reindex_dst, + "Out_Nodes": out_nodes, + }, + attrs={"flag_buffer_hashtable": use_buffer_hashtable}, + ) return reindex_src, reindex_dst, out_nodes diff --git a/python/paddle/geometric/sampling/neighbors.py b/python/paddle/geometric/sampling/neighbors.py index 63ec44b4f39d3..1353bd5814fa8 100644 --- a/python/paddle/geometric/sampling/neighbors.py +++ b/python/paddle/geometric/sampling/neighbors.py @@ -20,14 +20,16 @@ __all__ = [] -def sample_neighbors(row, - colptr, - input_nodes, - sample_size=-1, - eids=None, - return_eids=False, - perm_buffer=None, - name=None): +def sample_neighbors( + row, + colptr, + input_nodes, + sample_size=-1, + eids=None, + return_eids=False, + perm_buffer=None, + name=None, +): """ Graph Sample Neighbors API. @@ -90,54 +92,78 @@ def sample_neighbors(row, if return_eids: if eids is None: raise ValueError( - "`eids` should not be None if `return_eids` is True.") + "`eids` should not be None if `return_eids` is True." + ) use_perm_buffer = True if perm_buffer is not None else False if _non_static_mode(): - out_neighbors, out_count, out_eids = _legacy_C_ops.graph_sample_neighbors( - row, colptr, input_nodes, eids, perm_buffer, "sample_size", - sample_size, "return_eids", return_eids, "flag_perm_buffer", - use_perm_buffer) + ( + out_neighbors, + out_count, + out_eids, + ) = _legacy_C_ops.graph_sample_neighbors( + row, + colptr, + input_nodes, + eids, + perm_buffer, + "sample_size", + sample_size, + "return_eids", + return_eids, + "flag_perm_buffer", + use_perm_buffer, + ) if return_eids: return out_neighbors, out_count, out_eids return out_neighbors, out_count - check_variable_and_dtype(row, "Row", ("int32", "int64"), - "graph_sample_neighbors") - check_variable_and_dtype(colptr, "Col_Ptr", ("int32", "int64"), - "graph_sample_neighbors") - check_variable_and_dtype(input_nodes, "X", ("int32", "int64"), - "graph_sample_neighbors") + check_variable_and_dtype( + row, "Row", ("int32", "int64"), "graph_sample_neighbors" + ) + check_variable_and_dtype( + colptr, "Col_Ptr", ("int32", "int64"), "graph_sample_neighbors" + ) + check_variable_and_dtype( + input_nodes, "X", ("int32", "int64"), "graph_sample_neighbors" + ) if return_eids: - check_variable_and_dtype(eids, "Eids", ("int32", "int64"), - "graph_sample_neighbors") + check_variable_and_dtype( + eids, "Eids", ("int32", "int64"), "graph_sample_neighbors" + ) if use_perm_buffer: - check_variable_and_dtype(perm_buffer, "Perm_Buffer", ("int32", "int64"), - "graph_sample_neighbors") + check_variable_and_dtype( + perm_buffer, + "Perm_Buffer", + ("int32", "int64"), + "graph_sample_neighbors", + ) helper = LayerHelper("sample_neighbors", **locals()) out_neighbors = helper.create_variable_for_type_inference(dtype=row.dtype) out_count = helper.create_variable_for_type_inference(dtype=row.dtype) out_eids = helper.create_variable_for_type_inference(dtype=row.dtype) - helper.append_op(type="graph_sample_neighbors", - inputs={ - "Row": row, - "Col_Ptr": colptr, - "X": input_nodes, - "Eids": eids if return_eids else None, - "Perm_Buffer": perm_buffer if use_perm_buffer else None - }, - outputs={ - "Out": out_neighbors, - "Out_Count": out_count, - "Out_Eids": out_eids - }, - attrs={ - "sample_size": sample_size, - "return_eids": return_eids, - "flag_perm_buffer": use_perm_buffer - }) + helper.append_op( + type="graph_sample_neighbors", + inputs={ + "Row": row, + "Col_Ptr": colptr, + "X": input_nodes, + "Eids": eids if return_eids else None, + "Perm_Buffer": perm_buffer if use_perm_buffer else None, + }, + outputs={ + "Out": out_neighbors, + "Out_Count": out_count, + "Out_Eids": out_eids, + }, + attrs={ + "sample_size": sample_size, + "return_eids": return_eids, + "flag_perm_buffer": use_perm_buffer, + }, + ) if return_eids: return out_neighbors, out_count, out_eids return out_neighbors, out_count diff --git a/python/paddle/hapi/callbacks.py b/python/paddle/hapi/callbacks.py index 886a818015958..af53b181436ae 100644 --- a/python/paddle/hapi/callbacks.py +++ b/python/paddle/hapi/callbacks.py @@ -28,17 +28,19 @@ __all__ = [] -def config_callbacks(callbacks=None, - model=None, - batch_size=None, - epochs=None, - steps=None, - log_freq=2, - verbose=2, - save_freq=1, - save_dir=None, - metrics=None, - mode='train'): +def config_callbacks( + callbacks=None, + model=None, + batch_size=None, + epochs=None, + steps=None, + log_freq=2, + verbose=2, + save_freq=1, + save_dir=None, + metrics=None, + mode='train', +): cbks = callbacks or [] cbks = cbks if isinstance(cbks, (list, tuple)) else [cbks] if not any(isinstance(k, ProgBarLogger) for k in cbks) and verbose: @@ -68,7 +70,6 @@ def config_callbacks(callbacks=None, class CallbackList(object): - def __init__(self, callbacks=None): # copy self.callbacks = [c for c in callbacks] @@ -95,8 +96,11 @@ def _call(self, name, *args): func(*args) def _check_mode(self, mode): - assert mode in ['train', 'eval', 'predict'], \ - 'mode should be train, eval or predict' + assert mode in [ + 'train', + 'eval', + 'predict', + ], 'mode should be train, eval or predict' def on_begin(self, mode, logs=None): self._check_mode(mode) @@ -167,8 +171,7 @@ def set_params(self, params): self.params = params def set_model(self, model): - """model is instance of paddle.Model. - """ + """model is instance of paddle.Model.""" self.model = model def on_train_begin(self, logs=None): @@ -391,31 +394,39 @@ def _updates(self, logs, mode): cnt = timer['count'] if timer['count'] > 0 else 1.0 samples = timer['samples'] if timer['samples'] > 0 else 1.0 values.append( - ('avg_reader_cost', "%.5f sec" % (timer['data_time'] / cnt))) + ('avg_reader_cost', "%.5f sec" % (timer['data_time'] / cnt)) + ) values.append( - ('avg_batch_cost', "%.5f sec" % (timer['batch_time'] / cnt))) + ('avg_batch_cost', "%.5f sec" % (timer['batch_time'] / cnt)) + ) values.append( - ('ips', "%.5f samples/sec" % - (samples / (timer['data_time'] + timer['batch_time'])))) + ( + 'ips', + "%.5f samples/sec" + % (samples / (timer['data_time'] + timer['batch_time'])), + ) + ) timer['count'] = 0 timer['samples'] = 0 - timer['data_time'] = 0. - timer['batch_time'] = 0. + timer['data_time'] = 0.0 + timer['batch_time'] = 0.0 progbar.update(steps, values) def on_train_batch_begin(self, step, logs=None): self._train_timer['batch_data_end_time'] = time.time() self._train_timer['data_time'] += ( - self._train_timer['batch_data_end_time'] - - self._train_timer['batch_start_time']) + self._train_timer['batch_data_end_time'] + - self._train_timer['batch_start_time'] + ) def on_train_batch_end(self, step, logs=None): logs = logs or {} self.train_step += 1 self._train_timer['batch_time'] += ( - time.time() - self._train_timer['batch_data_end_time']) + time.time() - self._train_timer['batch_data_end_time'] + ) self._train_timer['count'] += 1 samples = logs.get('batch_size', 1) self._train_timer['samples'] += samples @@ -442,8 +453,9 @@ def on_eval_begin(self, logs=None): 'samples': 0, } - self.eval_progbar = ProgressBar(num=self.eval_steps, - verbose=self.verbose) + self.eval_progbar = ProgressBar( + num=self.eval_steps, verbose=self.verbose + ) if self._is_print(): print('Eval begin...') @@ -452,8 +464,9 @@ def on_eval_begin(self, logs=None): def on_eval_batch_begin(self, step, logs=None): self._eval_timer['batch_data_end_time'] = time.time() self._eval_timer['data_time'] += ( - self._eval_timer['batch_data_end_time'] - - self._eval_timer['batch_start_time']) + self._eval_timer['batch_data_end_time'] + - self._eval_timer['batch_start_time'] + ) def on_eval_batch_end(self, step, logs=None): logs = logs or {} @@ -462,7 +475,8 @@ def on_eval_batch_end(self, step, logs=None): self.evaled_samples += samples self._eval_timer['batch_time'] += ( - time.time() - self._eval_timer['batch_data_end_time']) + time.time() - self._eval_timer['batch_data_end_time'] + ) self._eval_timer['count'] += 1 samples = logs.get('batch_size', 1) self._eval_timer['samples'] += samples @@ -486,8 +500,9 @@ def on_predict_begin(self, logs=None): 'samples': 0, } - self.test_progbar = ProgressBar(num=self.test_steps, - verbose=self.verbose) + self.test_progbar = ProgressBar( + num=self.test_steps, verbose=self.verbose + ) if self._is_print(): print('Predict begin...') @@ -496,8 +511,9 @@ def on_predict_begin(self, logs=None): def on_predict_batch_begin(self, step, logs=None): self._test_timer['batch_data_end_time'] = time.time() self._test_timer['data_time'] += ( - self._test_timer['batch_data_end_time'] - - self._test_timer['batch_start_time']) + self._test_timer['batch_data_end_time'] + - self._test_timer['batch_start_time'] + ) def on_predict_batch_end(self, step, logs=None): logs = logs or {} @@ -506,7 +522,8 @@ def on_predict_batch_end(self, step, logs=None): self.tested_samples += samples self._test_timer['batch_time'] += ( - time.time() - self._test_timer['batch_data_end_time']) + time.time() - self._test_timer['batch_data_end_time'] + ) self._test_timer['count'] += 1 samples = logs.get('batch_size', 1) self._test_timer['samples'] += samples @@ -665,25 +682,34 @@ def make_optimizer(parameters=None): def __init__(self, by_step=True, by_epoch=False): if by_step and by_epoch: raise ValueError( - "by_step option is mutually exclusive with by_epoch") + "by_step option is mutually exclusive with by_epoch" + ) self.by_step = by_step self.by_epoch = by_epoch def on_epoch_end(self, epoch, logs=None): if self.by_epoch: - if self.model._optimizer and \ - hasattr(self.model._optimizer, '_learning_rate') and \ - isinstance(self.model._optimizer._learning_rate, - paddle.optimizer.lr.LRScheduler): + if ( + self.model._optimizer + and hasattr(self.model._optimizer, '_learning_rate') + and isinstance( + self.model._optimizer._learning_rate, + paddle.optimizer.lr.LRScheduler, + ) + ): self.model._optimizer._learning_rate.step() def on_train_batch_end(self, step, logs=None): if self.by_step: - if self.model._optimizer and \ - hasattr(self.model._optimizer, '_learning_rate') and \ - isinstance(self.model._optimizer._learning_rate, - paddle.optimizer.lr.LRScheduler): + if ( + self.model._optimizer + and hasattr(self.model._optimizer, '_learning_rate') + and isinstance( + self.model._optimizer._learning_rate, + paddle.optimizer.lr.LRScheduler, + ) + ): self.model._optimizer._learning_rate.step() @@ -761,14 +787,16 @@ class EarlyStopping(Callback): callbacks=[callbacks]) """ - def __init__(self, - monitor='loss', - mode='auto', - patience=0, - verbose=1, - min_delta=0, - baseline=None, - save_best_model=True): + def __init__( + self, + monitor='loss', + mode='auto', + patience=0, + verbose=1, + min_delta=0, + baseline=None, + save_best_model=True, + ): super(EarlyStopping, self).__init__() self.monitor = monitor self.patience = patience @@ -782,8 +810,10 @@ def __init__(self, # The value of `save_dir` is set in function `config_callbacks` self.save_dir = None if mode not in ['auto', 'min', 'max']: - warnings.warn('EarlyStopping mode %s is unknown, ' - 'fallback to auto mode.' % mode) + warnings.warn( + 'EarlyStopping mode %s is unknown, ' + 'fallback to auto mode.' % mode + ) mode = 'auto' if mode == 'min': self.monitor_op = np.less @@ -812,7 +842,8 @@ def on_train_begin(self, logs=None): def on_eval_end(self, logs=None): if logs is None or self.monitor not in logs: warnings.warn( - 'Monitor of EarlyStopping should be loss or metric name.') + 'Monitor of EarlyStopping should be loss or metric name.' + ) return current = logs[self.monitor] if isinstance(current, (list, tuple)): @@ -835,9 +866,14 @@ def on_eval_end(self, logs=None): if self.verbose > 0: print('Epoch %d: Early stopping.' % (self.stopped_epoch + 1)) if self.save_best_model and self.save_dir is not None: - print('Best checkpoint has been saved at %s' % - (os.path.abspath( - os.path.join(self.save_dir, 'best_model')))) + print( + 'Best checkpoint has been saved at %s' + % ( + os.path.abspath( + os.path.join(self.save_dir, 'best_model') + ) + ) + ) self.stopped_epoch += 1 @@ -926,9 +962,9 @@ def _updates(self, logs, mode): else: continue - self.writer.add_scalar(tag=temp_tag, - step=total_step, - value=temp_value) + self.writer.add_scalar( + tag=temp_tag, step=total_step, value=temp_value + ) def on_train_batch_end(self, step, logs=None): logs = logs or {} @@ -1022,21 +1058,24 @@ class ReduceLROnPlateau(Callback): """ - def __init__(self, - monitor='loss', - factor=0.1, - patience=10, - verbose=1, - mode='auto', - min_delta=1e-4, - cooldown=0, - min_lr=0): + def __init__( + self, + monitor='loss', + factor=0.1, + patience=10, + verbose=1, + mode='auto', + min_delta=1e-4, + cooldown=0, + min_lr=0, + ): super(ReduceLROnPlateau, self).__init__() self.monitor = monitor if factor >= 1.0: - raise ValueError('ReduceLROnPlateau ' - 'does not support a factor >= 1.0.') + raise ValueError( + 'ReduceLROnPlateau ' 'does not support a factor >= 1.0.' + ) self.factor = factor self.min_lr = min_lr @@ -1053,14 +1092,16 @@ def __init__(self, self._reset() def _reset(self): - """Resets wait counter and cooldown counter. - """ + """Resets wait counter and cooldown counter.""" if self.mode not in ['auto', 'min', 'max']: - warnings.warn('Learning rate reduction mode %s is unknown, ' - 'fallback to auto mode.' % self.mode) + warnings.warn( + 'Learning rate reduction mode %s is unknown, ' + 'fallback to auto mode.' % self.mode + ) self.mode = 'auto' - if (self.mode == 'min' - or (self.mode == 'auto' and 'acc' not in self.monitor)): + if self.mode == 'min' or ( + self.mode == 'auto' and 'acc' not in self.monitor + ): self.monitor_op = lambda a, b: np.less(a, b - self.min_delta) self.best = np.Inf else: @@ -1075,7 +1116,8 @@ def on_train_begin(self, logs=None): def on_eval_end(self, logs=None): if logs is None or self.monitor not in logs: warnings.warn( - 'Monitor of ReduceLROnPlateau should be loss or metric name.') + 'Monitor of ReduceLROnPlateau should be loss or metric name.' + ) return else: try: @@ -1083,12 +1125,16 @@ def on_eval_end(self, logs=None): if not isinstance(lr, float): warnings.warn( 'Expected learning_rate be float, bug got {}.'.format( - type(lr))) + type(lr) + ) + ) return except Exception as e: warnings.warn( - 'There are something wrong when get learning_rate from optimizer: {}.' - .format(e)) + 'There are something wrong when get learning_rate from optimizer: {}.'.format( + e + ) + ) return current = logs[self.monitor] @@ -1115,8 +1161,10 @@ def on_eval_end(self, logs=None): new_lr = max(new_lr, self.min_lr) self.model._optimizer._learning_rate = new_lr if self.verbose > 0 and ParallelEnv().local_rank == 0: - print('\nEpoch %d: ReduceLROnPlateau reducing learning ' - 'rate to %s.' % (self.epoch + 1, new_lr)) + print( + '\nEpoch %d: ReduceLROnPlateau reducing learning ' + 'rate to %s.' % (self.epoch + 1, new_lr) + ) self.cooldown_counter = self.cooldown self.wait = 0 self.epoch += 1 diff --git a/python/paddle/hapi/dynamic_flops.py b/python/paddle/hapi/dynamic_flops.py index 8e2bed5bc747f..50284e93e3b90 100644 --- a/python/paddle/hapi/dynamic_flops.py +++ b/python/paddle/hapi/dynamic_flops.py @@ -17,7 +17,9 @@ import paddle.nn as nn import numpy as np from .static_flops import static_flops, Table -from paddle.fluid.dygraph.dygraph_to_static.program_translator import unwrap_decorators +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + unwrap_decorators, +) __all__ = [] @@ -106,10 +108,9 @@ def count_leaky_relu(m, x, y): _, net.forward = unwrap_decorators(net.forward) inputs = paddle.randn(input_size) - return dynamic_flops(net, - inputs=inputs, - custom_ops=custom_ops, - print_detail=print_detail) + return dynamic_flops( + net, inputs=inputs, custom_ops=custom_ops, print_detail=print_detail + ) elif isinstance(net, paddle.static.Program): return static_flops(net, print_detail=print_detail) else: @@ -123,8 +124,9 @@ def count_convNd(m, x, y): x = x[0] kernel_ops = np.product(m.weight.shape[2:]) bias_ops = 1 if m.bias is not None else 0 - total_ops = int( - y.numel()) * (x.shape[1] / m._groups * kernel_ops + bias_ops) + total_ops = int(y.numel()) * ( + x.shape[1] / m._groups * kernel_ops + bias_ops + ) m.total_ops += abs(int(total_ops)) @@ -205,7 +207,7 @@ def count_io_info(m, x, y): nn.AvgPool3D: count_avgpool, nn.AdaptiveAvgPool1D: count_adap_avgpool, nn.AdaptiveAvgPool2D: count_adap_avgpool, - nn.AdaptiveAvgPool3D: count_adap_avgpool + nn.AdaptiveAvgPool3D: count_adap_avgpool, } @@ -227,7 +229,8 @@ def add_hooks(m): flops_fn = custom_ops[m_type] if m_type not in types_collection: print( - "Customize Function has been applied to {}".format(m_type)) + "Customize Function has been applied to {}".format(m_type) + ) elif m_type in register_hooks: flops_fn = register_hooks[m_type] if m_type not in types_collection: @@ -235,8 +238,10 @@ def add_hooks(m): else: if m_type not in types_collection: print( - "Cannot find suitable count function for {}. Treat it as zero FLOPs." - .format(m_type)) + "Cannot find suitable count function for {}. Treat it as zero FLOPs.".format( + m_type + ) + ) if flops_fn is not None: flops_handler = m.register_forward_post_hook(flops_fn) @@ -260,8 +265,12 @@ def add_hooks(m): for m in model.sublayers(): if len(list(m.children())) > 0: continue - if {'total_ops', 'total_params', 'input_shape', - 'output_shape'}.issubset(set(list(m._buffers.keys()))): + if { + 'total_ops', + 'total_params', + 'input_shape', + 'output_shape', + }.issubset(set(list(m._buffers.keys()))): total_ops += m.total_ops total_params += m.total_params @@ -271,26 +280,36 @@ def add_hooks(m): handler.remove() table = Table( - ["Layer Name", "Input Shape", "Output Shape", "Params", "Flops"]) + ["Layer Name", "Input Shape", "Output Shape", "Params", "Flops"] + ) for n, m in model.named_sublayers(): if len(list(m.children())) > 0: continue - if {'total_ops', 'total_params', 'input_shape', - 'output_shape'}.issubset(set(list(m._buffers.keys()))): - table.add_row([ - m.full_name(), - list(m.input_shape.numpy()), - list(m.output_shape.numpy()), - int(m.total_params), - int(m.total_ops) - ]) + if { + 'total_ops', + 'total_params', + 'input_shape', + 'output_shape', + }.issubset(set(list(m._buffers.keys()))): + table.add_row( + [ + m.full_name(), + list(m.input_shape.numpy()), + list(m.output_shape.numpy()), + int(m.total_params), + int(m.total_ops), + ] + ) m._buffers.pop("total_ops") m._buffers.pop("total_params") m._buffers.pop('input_shape') m._buffers.pop('output_shape') if print_detail: table.print_table() - print('Total Flops: {} Total Params: {}'.format(int(total_ops), - int(total_params))) + print( + 'Total Flops: {} Total Params: {}'.format( + int(total_ops), int(total_params) + ) + ) return int(total_ops) diff --git a/python/paddle/hapi/hub.py b/python/paddle/hapi/hub.py index ccf03aa2db097..3d5cb3027c49d 100644 --- a/python/paddle/hapi/hub.py +++ b/python/paddle/hapi/hub.py @@ -53,10 +53,12 @@ def _import_module(name, repo_dir): def _git_archive_link(repo_owner, repo_name, branch, source): if source == 'github': return 'https://github.com/{}/{}/archive/{}.zip'.format( - repo_owner, repo_name, branch) + repo_owner, repo_name, branch + ) elif source == 'gitee': return 'https://gitee.com/{}/{}/repository/archive/{}.zip'.format( - repo_owner, repo_name, branch) + repo_owner, repo_name, branch + ) def _parse_repo_info(repo, source): @@ -94,8 +96,9 @@ def _get_cache_or_reload(repo, force_reload, verbose=True, source='github'): # We don't know the repo name before downloading the zip file # and inspect name from it. # To check if cached repo exists, we need to normalize folder names. - repo_dir = os.path.join(hub_dir, - '_'.join([repo_owner, repo_name, normalized_br])) + repo_dir = os.path.join( + hub_dir, '_'.join([repo_owner, repo_name, normalized_br]) + ) use_cache = (not force_reload) and os.path.exists(repo_dir) @@ -113,7 +116,8 @@ def _get_cache_or_reload(repo, force_reload, verbose=True, source='github'): hub_dir, check_exist=not force_reload, decompress=False, - method=('wget' if source == 'gitee' else 'get')) + method=('wget' if source == 'gitee' else 'get'), + ) shutil.move(fpath, cached_file) with zipfile.ZipFile(cached_file) as cached_zipfile: @@ -132,11 +136,11 @@ def _get_cache_or_reload(repo, force_reload, verbose=True, source='github'): def _load_entry_from_hubconf(m, name): - '''load entry from hubconf - ''' + '''load entry from hubconf''' if not isinstance(name, str): raise ValueError( - 'Invalid input: model should be a str of function name') + 'Invalid input: model should be a str of function name' + ) func = getattr(m, name, None) @@ -162,8 +166,9 @@ def _check_dependencies(m): pkg for pkg in dependencies if not _check_module_exists(pkg) ] if len(missing_deps): - raise RuntimeError('Missing dependencies: {}'.format( - ', '.join(missing_deps))) + raise RuntimeError( + 'Missing dependencies: {}'.format(', '.join(missing_deps)) + ) def list(repo_dir, source='github', force_reload=False): @@ -193,19 +198,21 @@ def list(repo_dir, source='github', force_reload=False): """ if source not in ('github', 'gitee', 'local'): raise ValueError( - 'Unknown source: "{}". Allowed values: "github" | "gitee" | "local".' - .format(source)) + 'Unknown source: "{}". Allowed values: "github" | "gitee" | "local".'.format( + source + ) + ) if source in ('github', 'gitee'): - repo_dir = _get_cache_or_reload(repo_dir, - force_reload, - True, - source=source) + repo_dir = _get_cache_or_reload( + repo_dir, force_reload, True, source=source + ) hub_module = _import_module(MODULE_HUBCONF.split('.')[0], repo_dir) entrypoints = [ - f for f in dir(hub_module) + f + for f in dir(hub_module) if callable(getattr(hub_module, f)) and not f.startswith('_') ] @@ -240,14 +247,15 @@ def help(repo_dir, model, source='github', force_reload=False): """ if source not in ('github', 'gitee', 'local'): raise ValueError( - 'Unknown source: "{}". Allowed values: "github" | "gitee" | "local".' - .format(source)) + 'Unknown source: "{}". Allowed values: "github" | "gitee" | "local".'.format( + source + ) + ) if source in ('github', 'gitee'): - repo_dir = _get_cache_or_reload(repo_dir, - force_reload, - True, - source=source) + repo_dir = _get_cache_or_reload( + repo_dir, force_reload, True, source=source + ) hub_module = _import_module(MODULE_HUBCONF.split('.')[0], repo_dir) @@ -283,14 +291,15 @@ def load(repo_dir, model, source='github', force_reload=False, **kwargs): """ if source not in ('github', 'gitee', 'local'): raise ValueError( - 'Unknown source: "{}". Allowed values: "github" | "gitee" | "local".' - .format(source)) + 'Unknown source: "{}". Allowed values: "github" | "gitee" | "local".'.format( + source + ) + ) if source in ('github', 'gitee'): - repo_dir = _get_cache_or_reload(repo_dir, - force_reload, - True, - source=source) + repo_dir = _get_cache_or_reload( + repo_dir, force_reload, True, source=source + ) hub_module = _import_module(MODULE_HUBCONF.split('.')[0], repo_dir) diff --git a/python/paddle/hapi/model.py b/python/paddle/hapi/model.py index 788043477a259..a790375a742df 100644 --- a/python/paddle/hapi/model.py +++ b/python/paddle/hapi/model.py @@ -64,8 +64,9 @@ def to_list(value): def to_numpy(var): - assert isinstance(var, (Variable, fluid.core.VarBase, - fluid.core.eager.Tensor)), "not a variable" + assert isinstance( + var, (Variable, fluid.core.VarBase, fluid.core.eager.Tensor) + ), "not a variable" if isinstance(var, (fluid.core.VarBase, fluid.core.eager.Tensor)): return var.numpy() t = global_scope().find_var(var.name).get_tensor() @@ -100,10 +101,9 @@ def extract_args(func): def _all_gather(x, nranks, ring_id=0, use_calc_stream=True): - return collective._c_allgather(x, - nranks, - ring_id=ring_id, - use_calc_stream=use_calc_stream) + return collective._c_allgather( + x, nranks, ring_id=ring_id, use_calc_stream=use_calc_stream + ) def wait_server_ready(endpoints): @@ -114,7 +114,8 @@ def wait_server_ready(endpoints): for ep in endpoints: ip_port = ep.split(":") with contextlib.closing( - socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.settimeout(2) result = sock.connect_ex((ip_port[0], int(ip_port[1]))) if result != 0: @@ -126,8 +127,9 @@ def wait_server_ready(endpoints): break -def init_communicator(program, rank, nranks, wait_port, current_endpoint, - endpoints): +def init_communicator( + program, rank, nranks, wait_port, current_endpoint, endpoints +): if nranks < 2: return other_endpoints = endpoints[:] @@ -139,53 +141,66 @@ def init_communicator(program, rank, nranks, wait_port, current_endpoint, nccl_id_var = block.create_var( name=fluid.unique_name.generate('nccl_id'), persistable=True, - type=fluid.core.VarDesc.VarType.RAW) - - block.append_op(type='c_gen_nccl_id', - inputs={}, - outputs={'Out': nccl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) - - block.append_op(type='c_comm_init', - inputs={'X': nccl_id_var}, - outputs={}, - attrs={ - 'nranks': nranks, - 'rank': rank, - 'ring_id': 0, - }) + type=fluid.core.VarDesc.VarType.RAW, + ) + + block.append_op( + type='c_gen_nccl_id', + inputs={}, + outputs={'Out': nccl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) + + block.append_op( + type='c_comm_init', + inputs={'X': nccl_id_var}, + outputs={}, + attrs={ + 'nranks': nranks, + 'rank': rank, + 'ring_id': 0, + }, + ) elif core.is_compiled_with_npu(): hccl_id_var = block.create_var( name=fluid.unique_name.generate('hccl_id'), persistable=True, - type=core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_hccl_id', - inputs={}, - outputs={'Out': hccl_id_var}, - attrs={ - 'rank': rank, - 'endpoint': current_endpoint, - 'other_endpoints': other_endpoints - }) - block.append_op(type='c_comm_init_hccl', - inputs={'X': hccl_id_var}, - outputs={}, - attrs={ - 'rank': rank, - 'ring_id': 0, - 'device_id': int(os.getenv("FLAGS_selected_npus")), - 'rank_ids': nranks - }) + type=core.VarDesc.VarType.RAW, + ) + block.append_op( + type='c_gen_hccl_id', + inputs={}, + outputs={'Out': hccl_id_var}, + attrs={ + 'rank': rank, + 'endpoint': current_endpoint, + 'other_endpoints': other_endpoints, + }, + ) + block.append_op( + type='c_comm_init_hccl', + inputs={'X': hccl_id_var}, + outputs={}, + attrs={ + 'rank': rank, + 'ring_id': 0, + 'device_id': int(os.getenv("FLAGS_selected_npus")), + 'rank_ids': nranks, + }, + ) def prepare_distributed_context(place=None): if place is None: - place = fluid.CUDAPlace(ParallelEnv().dev_id) if ParallelEnv().nranks > 1 \ + place = ( + fluid.CUDAPlace(ParallelEnv().dev_id) + if ParallelEnv().nranks > 1 else fluid.CUDAPlace(0) + ) place = _get_paddle_place(place) strategy = fluid.dygraph.parallel.ParallelStrategy() @@ -203,9 +218,14 @@ def prepare_distributed_context(place=None): def _init_context(): communicator_prog = fluid.Program() - init_communicator(communicator_prog, strategy.local_rank, - strategy.nranks, True, strategy.current_endpoint, - strategy.trainer_endpoints) + init_communicator( + communicator_prog, + strategy.local_rank, + strategy.nranks, + True, + strategy.current_endpoint, + strategy.trainer_endpoints, + ) exe = fluid.Executor(place) exe.run(communicator_prog) @@ -215,7 +235,7 @@ def _init_context(): fluid.enable_dygraph(place) else: - assert ("Only support CUDAPlace for now.") + assert "Only support CUDAPlace for now." _parallel_context_initialized = True return strategy @@ -264,7 +284,7 @@ def __init__(self, model): 'eval_total': 0, 'test_total': 0, 'eval_batch': 0, - 'test_batch': 0 + 'test_batch': 0, } self._nranks = ParallelEnv().nranks @@ -284,10 +304,13 @@ def mode(self, value): self.model.mode = value def train_batch(self, inputs, labels=None, update=True): - assert self.model._optimizer, \ - "model not ready, please call `model.prepare()` first" + assert ( + self.model._optimizer + ), "model not ready, please call `model.prepare()` first" self.mode = 'train' - assert update is True, "Does not support `update == False` in static mode by now." + assert ( + update is True + ), "Does not support `update == False` in static mode by now." return self._run(inputs, labels) def eval_batch(self, inputs, labels=None): @@ -302,7 +325,6 @@ def parameters(self, *args, **kwargs): return self.model.network.parameters(*args, **kwargs) def save(self, path): - def _save(state, path): if not state: return @@ -326,8 +348,7 @@ def _save(state, path): # XXX `optimizer.state_dict()` only work in dygraph mode optim_path = path + ".pdopt" optim = { - p.name: p - for p in filter(is_belong_to_optimizer, prog.list_vars()) + p.name: p for p in filter(is_belong_to_optimizer, prog.list_vars()) } if not optim: return @@ -343,8 +364,10 @@ def load(self, param_state_pairs, optim_state): # restore parameter states fluid.core._create_loaded_parameter( - [param for param, state in param_state_pairs], global_scope(), - executor) + [param for param, state in param_state_pairs], + global_scope(), + executor, + ) for param, state in param_state_pairs: self._set_var(param, state) @@ -372,9 +395,10 @@ def _load_optimizer(self, state, executor): # static-graph, since the time of global_step to increase is # different. state_val = ( - np.array(converted_state.pop("global_step")) - 1 - ) if "global_step" in converted_state else converted_state.pop( - "@LR_DECAY_COUNTER@", None) + (np.array(converted_state.pop("global_step")) - 1) + if "global_step" in converted_state + else converted_state.pop("@LR_DECAY_COUNTER@", None) + ) if state_val is not None: converted_state[var.name] = state_val elif var.name.startswith("learning_rate_"): @@ -391,36 +415,61 @@ def _load_optimizer(self, state, executor): opt_cls_name = self.model._optimizer.__class__.__name__ opt_unq_name = None for name in self.model._optimizer._accumulators.keys(): - accum_name = name if opt_name is None else name[ - len(opt_name) + 1:] - for param_name, state_var in self.model._optimizer._accumulators[ - name].items(): + accum_name = ( + name + if opt_name is None + else name[len(opt_name) + 1 :] + ) + for ( + param_name, + state_var, + ) in self.model._optimizer._accumulators[name].items(): if opt_unq_name is None: # can not infer out the exact unique(opt_name), # thus try to extract rather than generate - for state_key in sorted(state.keys(), - key=lambda x: len(x), - reverse=True): - prefix = param_name + "_" + ( - opt_cls_name - if opt_name is None else opt_name) + "_" + for state_key in sorted( + state.keys(), + key=lambda x: len(x), + reverse=True, + ): + prefix = ( + param_name + + "_" + + ( + opt_cls_name + if opt_name is None + else opt_name + ) + + "_" + ) if state_key.startswith(prefix): - prefix_offset = state_key[len( - prefix):].find("_") + len(prefix) + prefix_offset = state_key[ + len(prefix) : + ].find("_") + len(prefix) opt_unq_name = state_key[ - len(param_name + "_"):prefix_offset] + len( + param_name + "_" + ) : prefix_offset + ] # TODO: assert # assert opt_unq_name is None # gen(param.name + "_" + gen(opt_name) + "_" + accum_name) # always end with "_0" since the unique optimizer._name - dy_state_name = (param_name + "_" + opt_unq_name + - "_" + accum_name + "_0") + dy_state_name = ( + param_name + + "_" + + opt_unq_name + + "_" + + accum_name + + "_0" + ) converted_state[ - state_var.name] = converted_state.pop( - dy_state_name) + state_var.name + ] = converted_state.pop(dy_state_name) - assert var.name in converted_state, \ - "variable [{}] is not in optimizer state file".format(var.name) + assert ( + var.name in converted_state + ), "variable [{}] is not in optimizer state file".format(var.name) self._set_var(var, converted_state[var.name]) def _set_var(self, var, ndarray): @@ -439,15 +488,17 @@ def _set_var(self, var, ndarray): def _run(self, inputs, labels=None): compiled_prog = self._compiled_progs.get(self.mode, None) - assert compiled_prog, \ - "Model is not ready, please call `model.prepare()` first" + assert ( + compiled_prog + ), "Model is not ready, please call `model.prepare()` first" inputs = to_list(inputs) if labels is not None: labels = to_list(labels) - assert len(inputs) == len(self._input_vars[self.mode]), \ - "number of inputs" \ + assert len(inputs) == len(self._input_vars[self.mode]), ( + "number of inputs" + " does not match number of arguments of `forward` method" + ) feed = {} input_names = [v.name for v in self._input_vars[self.mode]] @@ -457,8 +508,10 @@ def _run(self, inputs, labels=None): # train and test may take different arguments if inputs[idx] is not None: feed[n] = inputs[idx] - if self._amp_level == 'O2' and input_dtypes[ - idx] == core.VarDesc.VarType.FP16: + if ( + self._amp_level == 'O2' + and input_dtypes[idx] == core.VarDesc.VarType.FP16 + ): if isinstance(feed[n], core.LoDTensor): feed[n] = feed[n]._as_type(core.VarDesc.VarType.FP16) elif isinstance(feed[n], np.array): @@ -486,10 +539,12 @@ def _run(self, inputs, labels=None): else: pruned_fetch_list.append(fetch_var) - rets = self._executor.run(compiled_prog, - feed=feed, - fetch_list=pruned_fetch_list, - return_numpy=False) + rets = self._executor.run( + compiled_prog, + feed=feed, + fetch_list=pruned_fetch_list, + return_numpy=False, + ) # restore pruned fetch_list Variable from feeds for i, name in enumerate(pruned_fetch_idx_name_map): @@ -505,20 +560,24 @@ def _run(self, inputs, labels=None): metrics = [] for metric, state in zip(self.model._metrics, metric_states): # cut off padding size - if self.mode != 'train' and self.model._test_dataloader is not None \ - and isinstance(self.model._test_dataloader, DataLoader) \ - and self._nranks > 1: + if ( + self.mode != 'train' + and self.model._test_dataloader is not None + and isinstance(self.model._test_dataloader, DataLoader) + and self._nranks > 1 + ): total_size = len(self.model._test_dataloader.dataset) # TODO: fixme if have better way to get batch size samples = state[0].shape[0] current_count = self._merge_count.get(self.mode + '_total', 0) if current_count + samples >= total_size: state = [ - s[:int(total_size - current_count), ...] for s in state + s[: int(total_size - current_count), ...] for s in state ] self._merge_count[self.mode + '_total'] = 0 - self._merge_count[self.mode + '_batch'] = int(total_size - - current_count) + self._merge_count[self.mode + '_batch'] = int( + total_size - current_count + ) else: self._merge_count[self.mode + '_total'] += samples self._merge_count[self.mode + '_batch'] = samples @@ -550,8 +609,11 @@ def _make_program(self, mode): if mode != 'train': for op in list(prog.global_block().ops): prog.global_block()._remove_op(0) - if mode == 'train' and self.model._optimizer \ - and self.model._optimizer._learning_rate_map: + if ( + mode == 'train' + and self.model._optimizer + and self.model._optimizer._learning_rate_map + ): # HACK workaround learning rate map issue lr_var = self.model._optimizer._learning_rate_map[self._orig_prog] new_lr_var = prog.global_block().vars[lr_var.name] @@ -589,20 +651,27 @@ def _make_program(self, mode): dist_strategy.amp = True dist_strategy.amp_configs = self._amp_configs.copy() dist_strategy.amp_configs.update(self._amp_custom_lists) - dist_strategy.amp_configs[ - 'use_pure_fp16'] = self._amp_level == 'O2' + dist_strategy.amp_configs['use_pure_fp16'] = ( + self._amp_level == 'O2' + ) self.model._optimizer = fleet.distributed_optimizer( - self.model._optimizer, strategy=dist_strategy) + self.model._optimizer, strategy=dist_strategy + ) elif self._amp_level != "O0" and core.is_compiled_with_cuda: - amp_lists = paddle.static.amp.AutoMixedPrecisionLists( - **self._amp_custom_lists - ) if self._amp_custom_lists else None + amp_lists = ( + paddle.static.amp.AutoMixedPrecisionLists( + **self._amp_custom_lists + ) + if self._amp_custom_lists + else None + ) self.model._optimizer = paddle.static.amp.decorate( self.model._optimizer, amp_lists=amp_lists, use_pure_fp16=self._amp_level == "O2", use_fp16_guard=self._use_fp16_guard, - **self._amp_configs) + **self._amp_configs + ) self.model._optimizer.minimize(self._loss_endpoint) @@ -615,7 +684,7 @@ def _make_program(self, mode): self._endpoints[mode] = { "output": outputs, "loss": to_list(losses), - "metric": metrics + "metric": metrics, } def _compile_and_initialize(self, prog, mode): @@ -623,8 +692,9 @@ def _compile_and_initialize(self, prog, mode): if compiled_prog is not None: return compiled_prog - assert self.model._place is not None, \ - "device is not set, please call `model.prepare()` first" + assert ( + self.model._place is not None + ), "device is not set, please call `model.prepare()` first" place = self.model._place @@ -637,8 +707,11 @@ def _compile_and_initialize(self, prog, mode): uninitialized = [] for var_py in self._startup_prog.list_vars(): var = fluid.global_scope().find_var(var_py.name) - if not var_py.name.startswith('nccl_id') and var and \ - var.get_tensor()._is_initialized(): + if ( + not var_py.name.startswith('nccl_id') + and var + and var.get_tensor()._is_initialized() + ): continue uninitialized.append(var_py) @@ -646,7 +719,10 @@ def _compile_and_initialize(self, prog, mode): startup_prog = self._startup_prog._prune(uninitialized) self._executor.run(startup_prog) - if self._amp_level == "O2" and mode == 'train' and core.is_compiled_with_cuda( + if ( + self._amp_level == "O2" + and mode == 'train' + and core.is_compiled_with_cuda() ): self.model._optimizer.amp_init(place) @@ -659,7 +735,6 @@ def _compile_and_initialize(self, prog, mode): class DynamicGraphAdapter(object): - def __init__(self, model): super(DynamicGraphAdapter, self).__init__() self.model = model @@ -669,7 +744,7 @@ def __init__(self, model): 'eval_total': 0, 'test_total': 0, 'eval_batch': 0, - 'test_batch': 0 + 'test_batch': 0, } self._input_info = None @@ -686,7 +761,8 @@ def __init__(self, model): stradegy.trainer_endpoints = ParallelEnv().trainer_endpoints stradegy.current_endpoint = ParallelEnv().current_endpoint self.ddp_model = fluid.dygraph.parallel.DataParallel( - self.model.network, stradegy) + self.model.network, stradegy + ) @property def mode(self): @@ -698,8 +774,9 @@ def mode(self, value): # TODO multi device in dygraph mode not implemented at present time def train_batch(self, inputs, labels=None, update=True): - assert self.model._optimizer, \ - "model not ready, please call `model.prepare()` first" + assert ( + self.model._optimizer + ), "model not ready, please call `model.prepare()` first" self.model.network.train() self.mode = 'train' inputs = to_list(inputs) @@ -711,9 +788,11 @@ def train_batch(self, inputs, labels=None, update=True): if self._amp_level != "O0" and self.model._scaler is None: self.model._scaler = paddle.amp.GradScaler(**self._amp_configs) - with paddle.amp.auto_cast(enable=self._amp_level != 'O0', - **self._amp_custom_lists, - level=self._amp_level): + with paddle.amp.auto_cast( + enable=self._amp_level != 'O0', + **self._amp_custom_lists, + level=self._amp_level + ): if self._nranks > 1: outputs = self.ddp_model(*[to_variable(x) for x in inputs]) else: @@ -741,8 +820,11 @@ def train_batch(self, inputs, labels=None, update=True): m = metric.update(*[to_numpy(m) for m in to_list(metric_outs)]) metrics.append(m) - return ([to_numpy(l) for l in losses], metrics) \ - if len(metrics) > 0 else [to_numpy(l) for l in losses] + return ( + ([to_numpy(l) for l in losses], metrics) + if len(metrics) > 0 + else [to_numpy(l) for l in losses] + ) def eval_batch(self, inputs, labels=None): self.model.network.eval() @@ -772,21 +854,25 @@ def eval_batch(self, inputs, labels=None): metrics = [] for metric in self.model._metrics: # cut off padding value. - if self.model._test_dataloader is not None and self._nranks > 1 \ - and isinstance(self.model._test_dataloader, DataLoader): + if ( + self.model._test_dataloader is not None + and self._nranks > 1 + and isinstance(self.model._test_dataloader, DataLoader) + ): total_size = len(self.model._test_dataloader.dataset) samples = outputs[0].shape[0] current_count = self._merge_count.get(self.mode + '_total', 0) if current_count + samples >= total_size: outputs = [ - o[:int(total_size - current_count)] for o in outputs + o[: int(total_size - current_count)] for o in outputs ] labels = [ - l[:int(total_size - current_count)] for l in labels + l[: int(total_size - current_count)] for l in labels ] self._merge_count[self.mode + '_total'] = 0 - self._merge_count[self.mode + '_batch'] = int(total_size - - current_count) + self._merge_count[self.mode + '_batch'] = int( + total_size - current_count + ) else: self._merge_count[self.mode + '_total'] += samples self._merge_count[self.mode + '_batch'] = samples @@ -853,38 +939,48 @@ def load(self, param_state_pairs, optim_state, scaler_state=None): opt_unq_name = '' opt_cls_name = self.model._optimizer.__class__.__name__ - opt_name = opt_unq_name[:opt_unq_name.rfind("_")] # remove suffix idx + opt_name = opt_unq_name[: opt_unq_name.rfind("_")] # remove suffix idx param_names = [param.name for param in self.model.network.parameters()] - for var_name, state_var in sorted(optim_state.items(), - key=lambda x: len(x[0]), - reverse=True): + for var_name, state_var in sorted( + optim_state.items(), key=lambda x: len(x[0]), reverse=True + ): if var_name in ["@LR_DECAY_COUNTER@", "global_step"]: # NOTE: dygraph saved global_step is 1 larger than that in # static-graph, since the time of global_step to increase is # different. if var_name == "@LR_DECAY_COUNTER@": - converted_state["global_step"] = np.array( - converted_state.pop("@LR_DECAY_COUNTER@")) + 1 + converted_state["global_step"] = ( + np.array(converted_state.pop("@LR_DECAY_COUNTER@")) + 1 + ) else: # moment and other accumulators # extend state dict to include promising dygraph names for param_name in param_names: if var_name.startswith(param_name + "_" + opt_name): # when init optimizer with name - accum_name = var_name[len(param_name + "_" + opt_name + - "_"):] - elif var_name.startswith(param_name + - "_") and opt_name == opt_cls_name: + accum_name = var_name[ + len(param_name + "_" + opt_name + "_") : + ] + elif ( + var_name.startswith(param_name + "_") + and opt_name == opt_cls_name + ): # when init optimizer without name - accum_name = var_name[len(param_name + "_"):] + accum_name = var_name[len(param_name + "_") :] else: continue # remove suffix idx - accum_name = accum_name[:accum_name.rfind("_")] + accum_name = accum_name[: accum_name.rfind("_")] # state names always end with "_0" in dygraph because of the # unique optimizer._name - dy_state_name = (param_name + "_" + opt_unq_name + "_" + - accum_name + "_0") + dy_state_name = ( + param_name + + "_" + + opt_unq_name + + "_" + + accum_name + + "_0" + ) converted_state[dy_state_name] = state_var if not hasattr(self.model._optimizer, 'set_state_dict'): @@ -896,12 +992,16 @@ def load(self, param_state_pairs, optim_state, scaler_state=None): self.model._optimizer.set_state_dict(converted_state) def prepare(self): - if self._amp_level == "O2" and self.model.mode == 'train' and core.is_compiled_with_cuda( + if ( + self._amp_level == "O2" + and self.model.mode == 'train' + and core.is_compiled_with_cuda() ): self.model.network, self.model._optimizer = paddle.amp.decorate( models=self.model.network, optimizers=self.model._optimizer, - level='O2') + level='O2', + ) if self._amp_level != "O0": self.model._scaler = None @@ -1336,17 +1436,24 @@ def _check_match(key, param): state = param_state.get(key, None) if state is None: raise ValueError( - "{} is not found in the providing file.".format(key)) + "{} is not found in the providing file.".format(key) + ) if list(state.shape) != list(param.shape): raise ValueError( - "{} receives a shape {}, but the expected shape is {}.". - format(key, list(state.shape), list(param.shape))) + "{} receives a shape {}, but the expected shape is {}.".format( + key, list(state.shape), list(param.shape) + ) + ) return param, state def _strip_postfix(path): path, ext = os.path.splitext(path) - assert ext in ['', '.pdparams', '.pdopt', '.pdmodel'], \ - "Unknown postfix {} from weights".format(ext) + assert ext in [ + '', + '.pdparams', + '.pdopt', + '.pdmodel', + ], "Unknown postfix {} from weights".format(ext) return path path = _strip_postfix(path) @@ -1360,15 +1467,17 @@ def _strip_postfix(path): except ValueError as err: if skip_mismatch: warnings.warn( - ("Skip loading for {}. ".format(key) + str(err))) + ("Skip loading for {}. ".format(key) + str(err)) + ) # reset optimizer when mismatch happens reset_optimizer = True else: raise err matched_param_state.append(match_res) - optim_state = None if reset_optimizer else _load_state_from_path( - path + ".pdopt") + optim_state = ( + None if reset_optimizer else _load_state_from_path(path + ".pdopt") + ) # TODO: support save/load scaler state in static graph if _non_static_mode(): @@ -1377,8 +1486,9 @@ def _strip_postfix(path): if os.path.exists(path + '.pdscaler'): scaler_state = paddle.load(path + '.pdscaler') - return self._adapter.load(matched_param_state, optim_state, - scaler_state) + return self._adapter.load( + matched_param_state, optim_state, scaler_state + ) else: return self._adapter.load(matched_param_state, optim_state) @@ -1410,13 +1520,14 @@ def parameters(self, *args, **kwargs): return self._adapter.parameters() def _prepare_amp(self, amp_configs): - def _check_pure_fp16_configs(): # pure float16 training has some restricts now if self._adapter._amp_level == "O2" and self._optimizer._grad_clip: # clip by value is not supported - assert isinstance(self._optimizer._grad_clip, (paddle.nn.ClipGradByGlobalNorm, paddle.nn.ClipGradByNorm)), \ - "Only GradientClipByNorm and GradientClipByGlobalNorm are supported in amp training with level=O2 currently." + assert isinstance( + self._optimizer._grad_clip, + (paddle.nn.ClipGradByGlobalNorm, paddle.nn.ClipGradByNorm), + ), "Only GradientClipByNorm and GradientClipByGlobalNorm are supported in amp training with level=O2 currently." self._adapter._amp_custom_lists = {} self._adapter._amp_configs = {} @@ -1428,7 +1539,8 @@ def _check_pure_fp16_configs(): elif isinstance(amp_configs, str): if amp_configs not in ('O0', 'O1', 'O2'): raise ValueError( - "The level of amp_configs should be 'O0', 'O1' or 'O2'.") + "The level of amp_configs should be 'O0', 'O1' or 'O2'." + ) self._adapter._amp_level = amp_configs _check_pure_fp16_configs() return @@ -1437,7 +1549,8 @@ def _check_pure_fp16_configs(): self._adapter._amp_level = 'O1' elif amp_configs['level'] not in ('O0', 'O1', 'O2'): raise ValueError( - "amp_configs['level'] should be 'O0', 'O1' or 'O2'.") + "amp_configs['level'] should be 'O0', 'O1' or 'O2'." + ) else: self._adapter._amp_level = amp_configs['level'] amp_config_key_set = set(amp_configs.keys()) - {'level'} @@ -1454,12 +1567,14 @@ def _check_pure_fp16_configs(): # construct amp_custom_lists if self._adapter._amp_level != 'O0' and amp_config_key_set: for param_name in [ - 'custom_white_list', 'custom_black_list', - 'custom_black_varnames' + 'custom_white_list', + 'custom_black_list', + 'custom_black_varnames', ]: if param_name in amp_config_key_set: self._adapter._amp_custom_lists[param_name] = amp_configs[ - param_name] + param_name + ] amp_config_key_set -= {param_name} def _check_amp_configs(amp_config_key_set): @@ -1474,13 +1589,16 @@ def _check_amp_configs(amp_config_key_set): } if amp_config_key_set - accepted_param_set: raise ValueError( - "Except for 'level', the keys of 'amp_configs' must be accepted by mixed precision APIs, but {} could not be recognized." - .format(tuple(amp_config_key_set - accepted_param_set))) + "Except for 'level', the keys of 'amp_configs' must be accepted by mixed precision APIs, but {} could not be recognized.".format( + tuple(amp_config_key_set - accepted_param_set) + ) + ) if 'use_fp16_guard' in amp_config_key_set: if _non_static_mode(): raise ValueError( - "'use_fp16_guard' is supported in static mode only.") + "'use_fp16_guard' is supported in static mode only." + ) self._adapter._use_fp16_guard = amp_configs['use_fp16_guard'] amp_config_key_set.remove('use_fp16_guard') @@ -1490,11 +1608,9 @@ def _check_amp_configs(amp_config_key_set): for key in amp_configs_set: self._adapter._amp_configs[key] = amp_configs[key] - def prepare(self, - optimizer=None, - loss=None, - metrics=None, - amp_configs=None): + def prepare( + self, optimizer=None, loss=None, metrics=None, amp_configs=None + ): """ Configures the model before runing. @@ -1534,15 +1650,17 @@ def prepare(self, if ParallelEnv().nranks > 1 and not _parallel_context_initialized: if fluid._non_static_mode(): main_prog_seed = fluid.default_main_program().random_seed - startup_prog_seed = fluid.default_startup_program( - ).random_seed + startup_prog_seed = ( + fluid.default_startup_program().random_seed + ) fluid.disable_dygraph() paddle.disable_static(self._place) # enable_dygraph would create and switch to a new program, # thus also copy seed to the new program fluid.default_main_program().random_seed = main_prog_seed - fluid.default_startup_program( - ).random_seed = startup_prog_seed + fluid.default_startup_program().random_seed = ( + startup_prog_seed + ) else: prepare_distributed_context(self._place) _parallel_context_initialized = True @@ -1557,30 +1675,32 @@ def prepare(self, metrics = metrics or [] for metric in to_list(metrics): - assert isinstance(metric, Metric), \ - "{} is not sub class of Metric".format( - metric.__class__.__name__) + assert isinstance( + metric, Metric + ), "{} is not sub class of Metric".format(metric.__class__.__name__) self._metrics = to_list(metrics) self._prepare_amp(amp_configs) self._adapter.prepare() - def fit(self, - train_data=None, - eval_data=None, - batch_size=1, - epochs=1, - eval_freq=1, - log_freq=10, - save_dir=None, - save_freq=1, - verbose=2, - drop_last=False, - shuffle=True, - num_workers=0, - callbacks=None, - accumulate_grad_batches=1, - num_iters=None): + def fit( + self, + train_data=None, + eval_data=None, + batch_size=1, + epochs=1, + eval_freq=1, + log_freq=10, + save_dir=None, + save_freq=1, + verbose=2, + drop_last=False, + shuffle=True, + num_workers=0, + callbacks=None, + accumulate_grad_batches=1, + num_iters=None, + ): """ Trains the model for a fixed number of epochs. If `eval_data` is set, evaluation will be done at the end of each epoch. @@ -1714,30 +1834,36 @@ def fit(self, epochs=2, save_dir='mnist_checkpoint') """ - assert train_data is not None, \ - "train_data must be given!" + assert train_data is not None, "train_data must be given!" if isinstance(train_data, Dataset): - train_sampler = DistributedBatchSampler(train_data, - batch_size=batch_size, - shuffle=shuffle, - drop_last=drop_last) - train_loader = DataLoader(train_data, - batch_sampler=train_sampler, - places=self._place, - num_workers=num_workers, - return_list=True) + train_sampler = DistributedBatchSampler( + train_data, + batch_size=batch_size, + shuffle=shuffle, + drop_last=drop_last, + ) + train_loader = DataLoader( + train_data, + batch_sampler=train_sampler, + places=self._place, + num_workers=num_workers, + return_list=True, + ) else: train_loader = train_data if eval_data is not None and isinstance(eval_data, Dataset): - eval_sampler = DistributedBatchSampler(eval_data, - batch_size=batch_size) - eval_loader = DataLoader(eval_data, - batch_sampler=eval_sampler, - places=self._place, - num_workers=num_workers, - return_list=True) + eval_sampler = DistributedBatchSampler( + eval_data, batch_size=batch_size + ) + eval_loader = DataLoader( + eval_data, + batch_sampler=eval_sampler, + places=self._place, + num_workers=num_workers, + return_list=True, + ) elif eval_data is not None: eval_loader = eval_data else: @@ -1750,8 +1876,11 @@ def fit(self, steps = self._len_data_loader(train_loader) self.num_iters = num_iters - if num_iters is not None and isinstance(num_iters, int) and isinstance( - steps, int): + if ( + num_iters is not None + and isinstance(num_iters, int) + and isinstance(steps, int) + ): assert num_iters > 0, "num_iters must be greater than 0!" epochs = (num_iters // steps) + 1 steps = min(num_iters, steps) @@ -1779,10 +1908,10 @@ def fit(self, if do_eval and epoch % eval_freq == 0: eval_steps = self._len_data_loader(eval_loader) - cbks.on_begin('eval', { - 'steps': eval_steps, - 'metrics': self._metrics_name() - }) + cbks.on_begin( + 'eval', + {'steps': eval_steps, 'metrics': self._metrics_name()}, + ) eval_logs = self._run_one_epoch(eval_loader, cbks, 'eval') @@ -1793,14 +1922,16 @@ def fit(self, cbks.on_end('train', logs) self._test_dataloader = None - def evaluate(self, - eval_data, - batch_size=1, - log_freq=10, - verbose=2, - num_workers=0, - callbacks=None, - num_iters=None): + def evaluate( + self, + eval_data, + batch_size=1, + log_freq=10, + verbose=2, + num_workers=0, + callbacks=None, + num_iters=None, + ): """ Evaluate the loss and metrics of the model on input dataset. @@ -1854,13 +1985,16 @@ def evaluate(self, """ if eval_data is not None and isinstance(eval_data, Dataset): - eval_sampler = DistributedBatchSampler(eval_data, - batch_size=batch_size) - eval_loader = DataLoader(eval_data, - batch_sampler=eval_sampler, - places=self._place, - num_workers=num_workers, - return_list=True) + eval_sampler = DistributedBatchSampler( + eval_data, batch_size=batch_size + ) + eval_loader = DataLoader( + eval_data, + batch_sampler=eval_sampler, + places=self._place, + num_workers=num_workers, + return_list=True, + ) else: eval_loader = eval_data @@ -1876,15 +2010,17 @@ def evaluate(self, eval_steps = self._len_data_loader(eval_loader) self.num_iters = num_iters - if num_iters is not None and isinstance(num_iters, int) and isinstance( - eval_steps, int): + if ( + num_iters is not None + and isinstance(num_iters, int) + and isinstance(eval_steps, int) + ): assert num_iters > 0, "num_iters must be greater than 0!" eval_steps = min(num_iters, eval_steps) self.num_iters = eval_steps - cbks.on_begin('eval', { - 'steps': eval_steps, - 'metrics': self._metrics_name() - }) + cbks.on_begin( + 'eval', {'steps': eval_steps, 'metrics': self._metrics_name()} + ) logs = self._run_one_epoch(eval_loader, cbks, 'eval') @@ -1898,13 +2034,15 @@ def evaluate(self, return eval_result - def predict(self, - test_data, - batch_size=1, - num_workers=0, - stack_outputs=False, - verbose=1, - callbacks=None): + def predict( + self, + test_data, + batch_size=1, + num_workers=0, + stack_outputs=False, + verbose=1, + callbacks=None, + ): """ Compute the output predictions on testing data. @@ -1975,13 +2113,16 @@ def __len__(self): """ if test_data is not None and isinstance(test_data, Dataset): - test_sampler = DistributedBatchSampler(test_data, - batch_size=batch_size) - test_loader = DataLoader(test_data, - batch_sampler=test_sampler, - places=self._place, - num_workers=num_workers, - return_list=True) + test_sampler = DistributedBatchSampler( + test_data, batch_size=batch_size + ) + test_loader = DataLoader( + test_data, + batch_sampler=test_sampler, + places=self._place, + num_workers=num_workers, + return_list=True, + ) else: test_loader = test_data @@ -2031,7 +2172,8 @@ def _save_inference_model(self, path): if self._is_shape_inferred: warnings.warn( "'inputs' was not specified when Model initialization, so the input shape to be saved will be the shape derived from the user's actual inputs. The input shape to be saved is %s. For saving correct input shapes, please provide 'inputs' for Model initialization." - % self._input_info[0]) + % self._input_info[0] + ) paddle.jit.save(layer, path, input_spec=self._inputs) @@ -2042,7 +2184,8 @@ def _save_inference_model(self, path): raise ValueError( "The input path MUST be format of dirname/file_prefix " "[dirname\\file_prefix in Windows system], but received " - "file_prefix is empty string.") + "file_prefix is empty string." + ) dirname = os.path.dirname(path) if dirname and not os.path.exists(dirname): @@ -2053,21 +2196,24 @@ def _save_inference_model(self, path): params_filename = file_prefix + INFER_PARAMS_SUFFIX prog = self._adapter._progs.get('test', None) - assert prog, \ - "Model is not ready, please call `model.prepare()` first" + assert ( + prog + ), "Model is not ready, please call `model.prepare()` first" infer_prog = prog.clone(for_test=True) input_names = [v.name for v in self._adapter._input_vars['test']] endpoints = self._adapter._endpoints['test']['output'] - fluid.io.save_inference_model(model_path, - input_names, - endpoints, - self._adapter._executor, - main_program=infer_prog, - model_filename=model_filename, - params_filename=params_filename) + fluid.io.save_inference_model( + model_path, + input_names, + endpoints, + self._adapter._executor, + main_program=infer_prog, + model_filename=model_filename, + params_filename=params_filename, + ) def _run_one_epoch( self, @@ -2093,16 +2239,21 @@ def _run_one_epoch( # LoDTensor.shape is callable, where LoDTensor comes from # DataLoader in static graph - batch_size = data[0].shape()[0] if callable( - data[0].shape) else data[0].shape[0] + batch_size = ( + data[0].shape()[0] + if callable(data[0].shape) + else data[0].shape[0] + ) callbacks.on_batch_begin(mode, step, logs) if mode != 'predict': - _inputs = [data[:len(self._inputs)], data[len(self._inputs):]] + _inputs = [data[: len(self._inputs)], data[len(self._inputs) :]] if mode == 'train': - _inputs.append((step + 1) % self._accumulate == 0 - or step + 1 == len(data_loader)) + _inputs.append( + (step + 1) % self._accumulate == 0 + or step + 1 == len(data_loader) + ) outs = getattr(self, mode + '_batch')(*_inputs) @@ -2123,15 +2274,17 @@ def _run_one_epoch( logs[k] = v else: if self._inputs is not None: - outs = self.predict_batch(data[:len(self._inputs)]) + outs = self.predict_batch(data[: len(self._inputs)]) else: outs = self.predict_batch(data) outputs.append(outs) logs['step'] = step - if mode == 'train' or self._adapter._merge_count.get( - mode + '_batch', 0) <= 0: + if ( + mode == 'train' + or self._adapter._merge_count.get(mode + '_batch', 0) <= 0 + ): logs['batch_size'] = batch_size * ParallelEnv().nranks else: logs['batch_size'] = self._adapter._merge_count[mode + '_batch'] @@ -2185,8 +2338,9 @@ def summary(self, input_size=None, dtype=None): # {'total_params': 61610, 'trainable_params': 61610} """ - assert (input_size is not None or self._inputs - is not None), "'input_size' or 'self._input' must be set" + assert ( + input_size is not None or self._inputs is not None + ), "'input_size' or 'self._input' must be set" if input_size is not None: _input_size = input_size else: @@ -2203,7 +2357,10 @@ def _verify_spec(self, specs, shapes=None, dtypes=None, is_input=False): if is_input: arg_names = extract_args(self.network.forward)[1:] # While Saving inference model in dygraph, and providing inputs only in running. - if shapes is not None and dtypes is not None and fluid._non_static_mode( + if ( + shapes is not None + and dtypes is not None + and fluid._non_static_mode() ): out_specs = [ Input(name=n, dtype=dtypes[i], shape=shapes[i]) @@ -2216,7 +2373,8 @@ def _verify_spec(self, specs, shapes=None, dtypes=None, is_input=False): elif isinstance(specs, dict): assert is_input is False out_specs = [ - specs[n] for n in extract_args(self.network.forward) + specs[n] + for n in extract_args(self.network.forward) if n != 'self' ] else: @@ -2227,8 +2385,10 @@ def _verify_spec(self, specs, shapes=None, dtypes=None, is_input=False): assert isinstance(spec, Input) if spec.name is None: raise ValueError( - "Requires Input[{}].name != None, but receive `None` with {}." - .format(i, spec)) + "Requires Input[{}].name != None, but receive `None` with {}.".format( + i, spec + ) + ) return out_specs @@ -2253,6 +2413,7 @@ def _update_inputs(self): "Update self._inputs according to given inputs." self._input_info = self._adapter._input_info if self._input_info is not None and len(self._input_info) == 2: - self._inputs = self._verify_spec(None, self._input_info[0], - self._input_info[1], True) + self._inputs = self._verify_spec( + None, self._input_info[0], self._input_info[1], True + ) self._is_shape_inferred = True diff --git a/python/paddle/hapi/model_summary.py b/python/paddle/hapi/model_summary.py index de4ec449a06b6..d90c64b76217b 100644 --- a/python/paddle/hapi/model_summary.py +++ b/python/paddle/hapi/model_summary.py @@ -160,18 +160,20 @@ def forward(self, inputs): _input_size = [] for item in input_size: if isinstance(item, int): - item = (item, ) - assert isinstance(item, - (tuple, InputSpec)), 'When input_size is list, \ + item = (item,) + assert isinstance( + item, (tuple, InputSpec) + ), 'When input_size is list, \ expect item in input_size is a tuple or InputSpec, but got {}'.format( - type(item)) + type(item) + ) if isinstance(item, InputSpec): _input_size.append(tuple(item.shape)) else: _input_size.append(item) elif isinstance(input_size, int): - _input_size = (input_size, ) + _input_size = (input_size,) else: _input_size = input_size @@ -207,8 +209,10 @@ def _check_shape(shape): elif isinstance(item, numbers.Number): if item <= 0: raise ValueError( - "Expected element in input size greater than zero, but got {}" - .format(item)) + "Expected element in input size greater than zero, but got {}".format( + item + ) + ) new_shape.append(item) return tuple(new_shape) @@ -231,7 +235,6 @@ def _check_input(input_size): @no_grad() def summary_string(model, input_size=None, dtypes=None, input=None): - def _all_is_numper(items): for item in items: if not isinstance(item, numbers.Number): @@ -272,7 +275,6 @@ def _get_output_shape(output): return output_shape def register_hook(layer): - def hook(layer, input, output): class_name = str(layer.__class__).split(".")[-1].split("'")[0] @@ -310,7 +312,8 @@ def hook(layer, input, output): try: if (getattr(getattr(layer, k), 'trainable')) and ( - not getattr(getattr(layer, k), 'stop_gradient')): + not getattr(getattr(layer, k), 'stop_gradient') + ): summary[m_key]["trainable_params"] += np.prod(v.shape) summary[m_key]["trainable"] = True trainable_flag = True @@ -321,9 +324,11 @@ def hook(layer, input, output): summary[m_key]["nb_params"] = params - if (not isinstance(layer, nn.Sequential) - and not isinstance(layer, nn.LayerList) - and (not (layer == model) or depth < 1)): + if ( + not isinstance(layer, nn.Sequential) + and not isinstance(layer, nn.LayerList) + and (not (layer == model) or depth < 1) + ): hooks.append(layer.register_forward_post_hook(hook)) # For rnn, gru and lstm layer @@ -368,24 +373,30 @@ def _get_str_length(summary): 'input_shape_width': 20, 'output_shape_width': 20, 'params_width': 15, - 'table_width': 75 + 'table_width': 75, } for layer in summary: if head_length['output_shape_width'] < len( - str(summary[layer]["output_shape"])): + str(summary[layer]["output_shape"]) + ): head_length['output_shape_width'] = len( - str(summary[layer]["output_shape"])) + str(summary[layer]["output_shape"]) + ) if head_length['input_shape_width'] < len( - str(summary[layer]["input_shape"])): + str(summary[layer]["input_shape"]) + ): head_length['input_shape_width'] = len( - str(summary[layer]["input_shape"])) + str(summary[layer]["input_shape"]) + ) if head_length['layer_width'] < len(str(layer)): head_length['layer_width'] = len(str(layer)) if head_length['params_width'] < len( - str(summary[layer]["nb_params"])): + str(summary[layer]["nb_params"]) + ): head_length['params_width'] = len( - str(summary[layer]["nb_params"])) + str(summary[layer]["nb_params"]) + ) _temp_width = 0 for k, v in head_length.items(): @@ -401,10 +412,15 @@ def _get_str_length(summary): summary_str += "-" * table_width['table_width'] + "\n" line_new = "{:^{}} {:^{}} {:^{}} {:^{}}".format( - "Layer (type)", table_width['layer_width'], "Input Shape", - table_width['input_shape_width'], "Output Shape", - table_width['output_shape_width'], "Param #", - table_width['params_width']) + "Layer (type)", + table_width['layer_width'], + "Input Shape", + table_width['input_shape_width'], + "Output Shape", + table_width['output_shape_width'], + "Param #", + table_width['params_width'], + ) summary_str += line_new + "\n" summary_str += "=" * table_width['table_width'] + "\n" total_params = 0 @@ -414,17 +430,21 @@ def _get_str_length(summary): for layer in summary: # input_shape, output_shape, trainable, nb_params line_new = "{:^{}} {:^{}} {:^{}} {:^{}}".format( - layer, table_width['layer_width'], + layer, + table_width['layer_width'], str(summary[layer]["input_shape"]), table_width['input_shape_width'], str(summary[layer]["output_shape"]), - table_width['output_shape_width'], "{0:,}".format( - summary[layer]["nb_params"]), table_width['params_width']) + table_width['output_shape_width'], + "{0:,}".format(summary[layer]["nb_params"]), + table_width['params_width'], + ) total_params += summary[layer]["nb_params"] try: total_output += np.sum( - np.prod(summary[layer]["output_shape"], axis=-1)) + np.prod(summary[layer]["output_shape"], axis=-1) + ) except: for output_shape in summary[layer]["output_shape"]: total_output += np.sum(np.prod(output_shape, axis=-1)) @@ -436,26 +456,31 @@ def _get_str_length(summary): def _get_input_size(input_size, size): if isinstance(input_size, (list, tuple)) and _all_is_numper(input_size): - size = abs(np.prod(input_size) * 4. / (1024**2.)) + size = abs(np.prod(input_size) * 4.0 / (1024**2.0)) else: size = sum([_get_input_size(i, size) for i in input_size]) return size total_input_size = _get_input_size(input_size, 0) - total_output_size = abs(2. * total_output * 4. / - (1024**2.)) # x2 for gradients - total_params_size = abs(total_params * 4. / (1024**2.)) + total_output_size = abs( + 2.0 * total_output * 4.0 / (1024**2.0) + ) # x2 for gradients + total_params_size = abs(total_params * 4.0 / (1024**2.0)) total_size = total_params_size + total_output_size + total_input_size summary_str += "=" * table_width['table_width'] + "\n" summary_str += "Total params: {0:,}".format(total_params) + "\n" summary_str += "Trainable params: {0:,}".format(trainable_params) + "\n" - summary_str += "Non-trainable params: {0:,}".format(total_params - - trainable_params) + "\n" + summary_str += ( + "Non-trainable params: {0:,}".format(total_params - trainable_params) + + "\n" + ) summary_str += "-" * table_width['table_width'] + "\n" summary_str += "Input size (MB): %0.2f" % total_input_size + "\n" - summary_str += "Forward/backward pass size (MB): %0.2f" % total_output_size + "\n" + summary_str += ( + "Forward/backward pass size (MB): %0.2f" % total_output_size + "\n" + ) summary_str += "Params size (MB): %0.2f" % total_params_size + "\n" summary_str += "Estimated Total Size (MB): %0.2f" % total_size + "\n" summary_str += "-" * table_width['table_width'] + "\n" @@ -463,5 +488,5 @@ def _get_input_size(input_size, size): # return summary return summary_str, { 'total_params': total_params, - 'trainable_params': trainable_params + 'trainable_params': trainable_params, } diff --git a/python/paddle/hapi/progressbar.py b/python/paddle/hapi/progressbar.py index 41e2b5422af64..43b913a6fc120 100644 --- a/python/paddle/hapi/progressbar.py +++ b/python/paddle/hapi/progressbar.py @@ -23,15 +23,17 @@ class ProgressBar(object): - """progress bar """ - - def __init__(self, - num=None, - width=30, - verbose=1, - start=True, - file=sys.stdout, - name='step'): + """progress bar""" + + def __init__( + self, + num=None, + width=30, + verbose=1, + start=True, + file=sys.stdout, + name='step', + ): self._num = num if isinstance(num, int) and num <= 0: raise TypeError('num should be None or integer (> 0)') @@ -47,11 +49,12 @@ def __init__(self, self._last_update = 0 self.name = name - self._dynamic_display = ((hasattr(self.file, 'isatty') - and self.file.isatty()) - or 'ipykernel' in sys.modules - or 'posix' in sys.modules - or 'PYCHARM_HOSTED' in os.environ) + self._dynamic_display = ( + (hasattr(self.file, 'isatty') and self.file.isatty()) + or 'ipykernel' in sys.modules + or 'posix' in sys.modules + or 'PYCHARM_HOSTED' in os.environ + ) def _get_max_width(self): if sys.version_info > (3, 3): @@ -81,13 +84,17 @@ def convert_uint16_to_float(in_list): in_list = np.asarray(in_list) out = np.vectorize( lambda x: struct.unpack(' 0: - bar_chars += ('=' * (prog_width - 1)) + bar_chars += '=' * (prog_width - 1) if current_num < self._num: bar_chars += '>' else: bar_chars += '=' - bar_chars += ('.' * (self._width - prog_width)) + bar_chars += '.' * (self._width - prog_width) bar_chars += ']' else: bar_chars = self.name + ' %3d' % current_num @@ -150,8 +159,11 @@ def convert_uint16_to_float(in_list): if self._num is not None and current_num < self._num: eta = time_per_unit * (self._num - current_num) if eta > 3600: - eta_format = '%d:%02d:%02d' % (eta // 3600, - (eta % 3600) // 60, eta % 60) + eta_format = '%d:%02d:%02d' % ( + eta // 3600, + (eta % 3600) // 60, + eta % 60, + ) elif eta > 60: eta_format = '%d:%02d' % (eta // 60, eta % 60) else: @@ -162,7 +174,7 @@ def convert_uint16_to_float(in_list): info += fps self._total_width += len(info) if prev_total_width > self._total_width: - info += (' ' * (prev_total_width - self._total_width)) + info += ' ' * (prev_total_width - self._total_width) # newline for another epoch if self._num is not None and current_num >= self._num: @@ -176,8 +188,10 @@ def convert_uint16_to_float(in_list): elif self._verbose == 2 or self._verbose == 3: if self._num: numdigits = int(np.log10(self._num)) + 1 - count = (self.name + ' %' + str(numdigits) + - 'd/%d') % (current_num, self._num) + count = (self.name + ' %' + str(numdigits) + 'd/%d') % ( + current_num, + self._num, + ) else: count = self.name + ' %3d' % current_num info = count + info @@ -191,9 +205,11 @@ def convert_uint16_to_float(in_list): info += ' %.4f' % v else: info += ' %.4e' % v - elif isinstance(v, np.ndarray) and \ - v.size == 1 and \ - v.dtype in [np.float32, np.float64]: + elif ( + isinstance(v, np.ndarray) + and v.size == 1 + and v.dtype in [np.float32, np.float64] + ): if abs(v[0]) > 1e-3: info += ' %.4f' % v[0] else: diff --git a/python/paddle/hapi/static_flops.py b/python/paddle/hapi/static_flops.py index 8926bcb35529d..d1effd27b463f 100644 --- a/python/paddle/hapi/static_flops.py +++ b/python/paddle/hapi/static_flops.py @@ -20,7 +20,6 @@ class VarWrapper(object): - def __init__(self, var, graph): assert isinstance(var, Variable) assert isinstance(graph, GraphWrapper) @@ -41,7 +40,6 @@ def shape(self): class OpWrapper(object): - def __init__(self, op, graph): assert isinstance(graph, GraphWrapper) self._op = op @@ -87,8 +85,7 @@ class GraphWrapper(object): """ def __init__(self, program=None, in_nodes=[], out_nodes=[]): - """ - """ + """ """ super(GraphWrapper, self).__init__() self.program = Program() if program is None else program self.persistables = {} @@ -212,7 +209,6 @@ def static_flops(program, print_detail=False): class Table(object): - def __init__(self, table_heads): self.table_heads = table_heads self.table_len = [] @@ -226,8 +222,10 @@ def add_row(self, row_str): print('The row_str should be a list') if len(row_str) != self.col_num: print( - 'The length of row data should be equal the length of table heads, but the data: {} is not equal table heads {}' - .format(len(row_str), self.col_num)) + 'The length of row data should be equal the length of table heads, but the data: {} is not equal table heads {}'.format( + len(row_str), self.col_num + ) + ) for i in range(self.col_num): if len(str(row_str[i])) > self.table_len[i]: self.table_len[i] = len(str(row_str[i])) diff --git a/python/paddle/hub.py b/python/paddle/hub.py index acdb28cb6f08d..e6f38d6ee11f7 100644 --- a/python/paddle/hub.py +++ b/python/paddle/hub.py @@ -16,6 +16,4 @@ from .hapi.hub import help # noqa: F401 from .hapi.hub import load # noqa: F401 -__all__ = [ #noqa - 'list', 'help', 'load' -] +__all__ = ['list', 'help', 'load'] # noqa diff --git a/python/paddle/incubate/__init__.py b/python/paddle/incubate/__init__.py index 68c7db054991a..2730db97f0ed2 100644 --- a/python/paddle/incubate/__init__.py +++ b/python/paddle/incubate/__init__.py @@ -29,10 +29,10 @@ from .tensor import segment_min from .passes import fuse_resnet_unit_pass -from . import autograd #noqa: F401 -from . import autotune #noqa: F401 -from . import nn #noqa: F401 -from . import asp #noqa: F401 +from . import autograd # noqa: F401 +from . import autotune # noqa: F401 +from . import nn # noqa: F401 +from . import asp # noqa: F401 from ..fluid.layers.loss import identity_loss diff --git a/python/paddle/incubate/asp/__init__.py b/python/paddle/incubate/asp/__init__.py index d2a56fd117c41..662e24900fce8 100644 --- a/python/paddle/incubate/asp/__init__.py +++ b/python/paddle/incubate/asp/__init__.py @@ -13,13 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.contrib.sparsity import calculate_density #noqa: F401 -from ...fluid.contrib.sparsity import decorate #noqa: F401 -from ...fluid.contrib.sparsity import prune_model #noqa: F401 -from ...fluid.contrib.sparsity import set_excluded_layers #noqa: F401 -from ...fluid.contrib.sparsity import reset_excluded_layers #noqa: F401 +from ...fluid.contrib.sparsity import calculate_density # noqa: F401 +from ...fluid.contrib.sparsity import decorate # noqa: F401 +from ...fluid.contrib.sparsity import prune_model # noqa: F401 +from ...fluid.contrib.sparsity import set_excluded_layers # noqa: F401 +from ...fluid.contrib.sparsity import reset_excluded_layers # noqa: F401 -__all__ = [ #noqa - 'calculate_density', 'decorate', 'prune_model', 'set_excluded_layers', - 'reset_excluded_layers' +__all__ = [ # noqa + 'calculate_density', + 'decorate', + 'prune_model', + 'set_excluded_layers', + 'reset_excluded_layers', ] diff --git a/python/paddle/incubate/autograd/__init__.py b/python/paddle/incubate/autograd/__init__.py index c5ff3b18d4d49..d9b9e41781917 100644 --- a/python/paddle/incubate/autograd/__init__.py +++ b/python/paddle/incubate/autograd/__init__.py @@ -17,6 +17,12 @@ from .utils import disable_prim, enable_prim, prim_enabled __all__ = [ # noqa - 'vjp', 'jvp', 'Jacobian', 'Hessian', 'enable_prim', 'disable_prim', - 'forward_grad', 'grad' + 'vjp', + 'jvp', + 'Jacobian', + 'Hessian', + 'enable_prim', + 'disable_prim', + 'forward_grad', + 'grad', ] diff --git a/python/paddle/incubate/autograd/functional.py b/python/paddle/incubate/autograd/functional.py index dce83b7505fb2..eafaaaea329ad 100644 --- a/python/paddle/incubate/autograd/functional.py +++ b/python/paddle/incubate/autograd/functional.py @@ -253,8 +253,7 @@ def __getitem__(self, indexes): @property def shape(self): - """The shape of flattened Jacobian matrix. - """ + """The shape of flattened Jacobian matrix.""" return self._jacobian.shape @@ -308,11 +307,11 @@ def reducer(x): """ def __init__(self, func, xs, is_batched=False): - def _jac_func(*xs): jac = Jacobian(func, xs, is_batched=is_batched) - if (is_batched and jac.shape[1] != 1) or (not is_batched - and jac.shape[0] != 1): + if (is_batched and jac.shape[1] != 1) or ( + not is_batched and jac.shape[0] != 1 + ): raise RuntimeError( "The function given to Hessian shoud return as single element Tensor or batched single element Tensor." ) @@ -325,8 +324,7 @@ def __getitem__(self, indexes): @property def shape(self): - """The shape of flattened Hessian matrix. - """ + """The shape of flattened Hessian matrix.""" return self.symbolic.shape @@ -369,32 +367,41 @@ def shape(self): @property def _lazy_axis(self): - """"The axis of lazily evaluated.""" + """ "The axis of lazily evaluated.""" raise NotImplementedError def _lazy_indexes(self, indexes): idx = indexes[self._lazy_axis] - return (idx, ) if isinstance(idx, int) else tuple( - range(idx.start, idx.stop, idx.step)) + return ( + (idx,) + if isinstance(idx, int) + else tuple(range(idx.start, idx.stop, idx.step)) + ) def _flatten(self, xs): raise NotImplementedError def _shifted_indexes(self, indexes, lazy_axis_size=0): idx = indexes[self._lazy_axis] - shifted_lazy_axis_idx = 0 if isinstance(idx, int) else slice( - 0, lazy_axis_size, 1) - return indexes[:self._lazy_axis] + ( - shifted_lazy_axis_idx, ) + indexes[self._lazy_axis + 1:] + shifted_lazy_axis_idx = ( + 0 if isinstance(idx, int) else slice(0, lazy_axis_size, 1) + ) + return ( + indexes[: self._lazy_axis] + + (shifted_lazy_axis_idx,) + + indexes[self._lazy_axis + 1 :] + ) def __getitem__(self, indexes): indexes = _multi_index(indexes, self.shape) if isinstance(indexes[self._lazy_axis], int): - other_indexes = indexes[:self._lazy_axis] + \ - indexes[self._lazy_axis+1:] - return self._cached_evaluate( - indexes[self._lazy_axis])[other_indexes] + other_indexes = ( + indexes[: self._lazy_axis] + indexes[self._lazy_axis + 1 :] + ) + return self._cached_evaluate(indexes[self._lazy_axis])[ + other_indexes + ] lazy_indexes = self._lazy_indexes(indexes) # Using concat and reshape to replace stack operator temporarily, as # it is not a primitive operator. @@ -402,7 +409,8 @@ def __getitem__(self, indexes): shape[self._lazy_axis] = len(lazy_indexes) part_jac = paddle.concat( [self._cached_evaluate(i) for i in lazy_indexes], - axis=self._lazy_axis).reshape(shape) + axis=self._lazy_axis, + ).reshape(shape) return part_jac[self._shifted_indexes(indexes, len(lazy_indexes))] def _cached_evaluate(self, k): @@ -435,13 +443,15 @@ def _lazy_axis(self): return 0 def _flatten(self, xs): - return paddle.concat(tuple(x.reshape((-1, )) for x in xs)) + return paddle.concat(tuple(x.reshape((-1,)) for x in xs)) def _evaluate(self, row_index): - return self._flatten(_grad( - self._flatten_ys[row_index], - self._xs, - )) + return self._flatten( + _grad( + self._flatten_ys[row_index], + self._xs, + ) + ) class _JacobianBatchFirst(_Jacobian): @@ -455,8 +465,11 @@ def __init__(self, func, xs): @property def shape(self): - return (self._flatten_xs.shape[0], self._flatten_ys.shape[1], - self._flatten_xs.shape[1]) + return ( + self._flatten_xs.shape[0], + self._flatten_ys.shape[1], + self._flatten_xs.shape[1], + ) @property def _lazy_axis(self): @@ -464,7 +477,8 @@ def _lazy_axis(self): def _flatten(self, xs): return paddle.concat( - tuple(x.reshape((x.shape[0], -1)) for x in utils.as_tensors(xs)), 1) + tuple(x.reshape((x.shape[0], -1)) for x in utils.as_tensors(xs)), 1 + ) def _evaluate(self, row_index): return self._flatten(_grad(self._flatten_ys[:, row_index], self._xs)) @@ -490,24 +504,27 @@ def _multi_index(indexes, shape): Returns: tuple: The standard format index as the above description. """ - indexes = indexes if isinstance(indexes, typing.Sequence) else (indexes, ) + indexes = indexes if isinstance(indexes, typing.Sequence) else (indexes,) if any(isinstance(i, type(Ellipsis)) for i in indexes): raise IndexError('Ellipsis index currently is not supported.') # Fill the right-most elements. - indexes = indexes + (slice(0, None, None), ) * (len(shape) - len(indexes)) + indexes = indexes + (slice(0, None, None),) * (len(shape) - len(indexes)) # Convert to positive index. positive_indexes = [] for i, index in enumerate(indexes): if isinstance(index, slice): - index = slice(index.start or 0, index.stop or shape[i], index.step - or 1) + index = slice( + index.start or 0, index.stop or shape[i], index.step or 1 + ) positive_indexes.append( slice( index.start + shape[i] if index.start < 0 else index.start, index.stop + shape[i] if index.stop < 0 else index.stop, # Negative step means index backward, no need to convert to # positive interger. - index.step)) + index.step, + ) + ) elif isinstance(index, int): positive_indexes.append(index + shape[i] if index < 0 else index) else: @@ -522,8 +539,8 @@ def _replace_none_with_zero_tensor(xs, refs): return xs elif isinstance(xs, typing.Sequence): return tuple( - _replace_none_with_zero_tensor(x, refs[i]) - for i, x in enumerate(xs)) + _replace_none_with_zero_tensor(x, refs[i]) for i, x in enumerate(xs) + ) else: return xs @@ -568,8 +585,11 @@ def _grad(ys, xs, v=None): # follow code snippet fixes the problem by return the first element of # xs_grad when the xs is a signle Tensor. xs_grad = paddle.grad(ys, xs, v, create_graph=True, allow_unused=True) - if isinstance(xs, paddle.fluid.framework.Variable) and isinstance( - xs_grad, typing.Sequence) and len(xs_grad) > 0: + if ( + isinstance(xs, paddle.fluid.framework.Variable) + and isinstance(xs_grad, typing.Sequence) + and len(xs_grad) > 0 + ): xs_grad = xs_grad[0] else: xs_grad = paddle.incubate.autograd.grad(ys, xs, v) @@ -645,18 +665,23 @@ def _check_inputs(func, xs, v=None): raise TypeError(f"Expected 'fun' is Callable, but got {type(func)}.") if not isinstance(xs, (framework.Variable, typing.Sequence)): - raise TypeError(f"Expected 'xs' is a Tensor|Sequence[Tensor]," - f"but got {type(xs)}.") + raise TypeError( + f"Expected 'xs' is a Tensor|Sequence[Tensor]," + f"but got {type(xs)}." + ) if isinstance(xs, typing.Sequence) and not all( - isinstance(x, framework.Variable) for x in xs): + isinstance(x, framework.Variable) for x in xs + ): raise TypeError("All elements of 'xs' shoule be Tensor.") if not isinstance(v, (framework.Variable, typing.Sequence, type(None))): raise TypeError( - f"Expected 'v' is Tensor|Sequence[Tensor]|None, but got {type(v)}.") + f"Expected 'v' is Tensor|Sequence[Tensor]|None, but got {type(v)}." + ) if isinstance(v, typing.Sequence) and not all( - isinstance(e, framework.Variable) for e in v): + isinstance(e, framework.Variable) for e in v + ): raise TypeError("All elements of 'xs' shoule be Tensor.") @@ -666,11 +691,14 @@ def _check_v_shape(v, refs): v, refs = utils.as_tensors(v), utils.as_tensors(refs) if len(refs) != len(v): - raise RuntimeError(f"The argument v is a tuple of invalid length:" - f"should be {len(refs)} but got {len(v)}.") + raise RuntimeError( + f"The argument v is a tuple of invalid length:" + f"should be {len(refs)} but got {len(v)}." + ) for index, (element_v, element_ref) in enumerate(zip(v, refs)): if element_v.shape != element_ref.shape: raise RuntimeError( f"The v[{index}] has invalid shape: should " - f"be {element_ref.shape} but got {element_v.shape}.") + f"be {element_ref.shape} but got {element_v.shape}." + ) diff --git a/python/paddle/incubate/autograd/primapi.py b/python/paddle/incubate/autograd/primapi.py index fa289e259cbc3..18a06af5dca7f 100644 --- a/python/paddle/incubate/autograd/primapi.py +++ b/python/paddle/incubate/autograd/primapi.py @@ -64,25 +64,35 @@ def forward_grad(outputs, inputs, grad_inputs=None): paddle.disable_static() """ if not utils.prim_enabled(): - raise RuntimeError('forward_grad must be running on primitive' - 'operators, use enable_prim to turn it on.') + raise RuntimeError( + 'forward_grad must be running on primitive' + 'operators, use enable_prim to turn it on.' + ) if not isinstance(outputs, (framework.Variable, typing.Sequence)): - raise TypeError(f'Expected outputs is Tensor|Sequence[Tesnor], ' - f'but got {type(outputs)}.') + raise TypeError( + f'Expected outputs is Tensor|Sequence[Tesnor], ' + f'but got {type(outputs)}.' + ) if not isinstance(inputs, (framework.Variable, typing.Sequence)): - raise TypeError(f'Expected inputs is Tensor|Sequence[Tesnor], ' - f'but got {type(inputs)}.') + raise TypeError( + f'Expected inputs is Tensor|Sequence[Tesnor], ' + f'but got {type(inputs)}.' + ) - ys, xs, xs_dot = utils.as_tensors(outputs), utils.as_tensors( - inputs), utils.as_tensors(grad_inputs) + ys, xs, xs_dot = ( + utils.as_tensors(outputs), + utils.as_tensors(inputs), + utils.as_tensors(grad_inputs), + ) block = framework.default_main_program().current_block() if any(x.block != block for x in xs + ys): raise RuntimeError( 'Variable in inputs and targets should exist in current block of ' - 'main program.') + 'main program.' + ) primx.orig2prim(block) ad = primx.Transform(ys[0].block) @@ -141,22 +151,32 @@ def grad(outputs, inputs, grad_outputs=None): # backward.gradients returns a list though the inputs is a signle Tensor. # The follow code snippet fixes the problem by return the first element # of grad_inputs when the inputs is a signle Tensor. - if isinstance(inputs, framework.Variable) and isinstance( - grad_inputs, typing.Sequence) and len(grad_inputs) > 0: + if ( + isinstance(inputs, framework.Variable) + and isinstance(grad_inputs, typing.Sequence) + and len(grad_inputs) > 0 + ): return grad_inputs[0] else: return grad_inputs if not isinstance(outputs, (framework.Variable, typing.Sequence)): - raise TypeError(f'Expected outputs is Tensor|Sequence[Tesnor], ' - f'but got {type(outputs)}.') + raise TypeError( + f'Expected outputs is Tensor|Sequence[Tesnor], ' + f'but got {type(outputs)}.' + ) if not isinstance(inputs, (framework.Variable, typing.Sequence)): - raise TypeError(f'Expected inputs is Tensor|Sequence[Tesnor], ' - f'but got {type(inputs)}.') + raise TypeError( + f'Expected inputs is Tensor|Sequence[Tesnor], ' + f'but got {type(inputs)}.' + ) - ys, xs, ys_bar = utils.as_tensors(outputs), utils.as_tensors( - inputs), utils.as_tensors(grad_outputs) + ys, xs, ys_bar = ( + utils.as_tensors(outputs), + utils.as_tensors(inputs), + utils.as_tensors(grad_outputs), + ) block = framework.default_main_program().current_block() if any((x is not None and x.block != block) for x in xs + ys): raise RuntimeError( diff --git a/python/paddle/incubate/autograd/primops.py b/python/paddle/incubate/autograd/primops.py index 502c10783e887..29a17dda1d458 100644 --- a/python/paddle/incubate/autograd/primops.py +++ b/python/paddle/incubate/autograd/primops.py @@ -37,13 +37,9 @@ def _simple_binop(helper): if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=optype, - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Z': out}, - attrs={}) + helper.append_op( + type=optype, inputs={'X': x, 'Y': y}, outputs={'Z': out}, attrs={} + ) return out @@ -53,16 +49,16 @@ def _manipulation_unop(helper): attrs = { k: helper.kwargs[k] - for k in ('shape', 'axis', 'index') if k in helper.kwargs + for k in ('shape', 'axis', 'index') + if k in helper.kwargs } if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=optype, - inputs={'X': x}, - outputs={'Y': out}, - attrs=attrs) + helper.append_op( + type=optype, inputs={'X': x}, outputs={'Y': out}, attrs=attrs + ) return out @@ -94,23 +90,23 @@ def set_value(x, y, axis, starts, ends, strides, out): assert x is out, "x and out should be the same Tensor in set_value" attrs = {'axes': axis, 'starts': starts, 'ends': ends, 'steps': strides} helper = LayerHelper('set_value', **locals()) - helper.append_op(type=helper.layer_type, - inputs={ - 'Input': x, - 'ValueTensor': y - }, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, + inputs={'Input': x, 'ValueTensor': y}, + outputs={'Out': out}, + attrs=attrs, + ) return out def mean(x, axis=None, keepdim=False): axes = axis or tuple(range(0, len(x.shape))) sum = reduce_sum(x, axis=axes, keepdim=keepdim) - norm = fill_const(shape=sum.shape, - value=functools.reduce(operator.mul, - [x.shape[axis] for axis in axes]), - dtype=sum.dtype) + norm = fill_const( + shape=sum.shape, + value=functools.reduce(operator.mul, [x.shape[axis] for axis in axes]), + dtype=sum.dtype, + ) return div(sum, norm) @@ -122,16 +118,18 @@ def zeros(shape, dtype): return fill_const(0, shape, dtype) -def batch_norm(x, - axis, - gamma, - beta, - run_mean, - run_var, - eps=1e-5, - momentum=0.9, - use_run_stat=False, - reserve_space=None): +def batch_norm( + x, + axis, + gamma, + beta, + run_mean, + run_var, + eps=1e-5, + momentum=0.9, + use_run_stat=False, + reserve_space=None, +): """batch normalizer. Args: @@ -150,40 +148,59 @@ def batch_norm(x, Defaults to False. """ reduce_axes = tuple(i for i in range(len(x.shape)) if i != axis) - stats_shape = tuple(1 if i in reduce_axes else s - for i, s in enumerate(x.shape)) + stats_shape = tuple( + 1 if i in reduce_axes else s for i, s in enumerate(x.shape) + ) batch_mean = zeros(run_mean.shape, run_mean.dtype) batch_var = zeros(run_var.shape, run_var.dtype) if not use_run_stat: batch_mean = mean(x, reduce_axes, keepdim=True) - batch_var = mean(square(sub(x, broadcast(batch_mean, x.shape))), - reduce_axes, - keepdim=True) + batch_var = mean( + square(sub(x, broadcast(batch_mean, x.shape))), + reduce_axes, + keepdim=True, + ) x_hat = div( sub(x, broadcast(batch_mean, x.shape)), sqrt( - add(broadcast(batch_var, x.shape), - fill_const(eps, x.shape, batch_var.dtype)))) + add( + broadcast(batch_var, x.shape), + fill_const(eps, x.shape, batch_var.dtype), + ) + ), + ) momentum = fill_const(momentum, run_mean.shape, run_mean.dtype) run_mean = add( mul(momentum, run_mean), - mul(sub(ones(run_mean.shape, run_mean.dtype), momentum), - reshape(batch_mean, run_mean.shape))) + mul( + sub(ones(run_mean.shape, run_mean.dtype), momentum), + reshape(batch_mean, run_mean.shape), + ), + ) run_var = add( mul(momentum, run_var), - mul(sub(ones(run_var.shape, run_var.dtype), momentum), - reshape(batch_var, run_var.shape))) + mul( + sub(ones(run_var.shape, run_var.dtype), momentum), + reshape(batch_var, run_var.shape), + ), + ) else: x_hat = div( sub(x, broadcast(reshape(run_mean, stats_shape), x.shape)), sqrt( - add(broadcast(reshape(run_var, stats_shape), x.shape), - fill_const(eps, x.shape, x.dtype)))) - y = add(mul(broadcast(reshape(gamma, stats_shape), x_hat.shape), x_hat), - broadcast(reshape(beta, stats_shape), x_hat.shape)) + add( + broadcast(reshape(run_var, stats_shape), x.shape), + fill_const(eps, x.shape, x.dtype), + ) + ), + ) + y = add( + mul(broadcast(reshape(gamma, stats_shape), x_hat.shape), x_hat), + broadcast(reshape(beta, stats_shape), x_hat.shape), + ) if reserve_space: return run_mean, reserve_space, batch_mean, batch_var, run_var, y @@ -192,7 +209,7 @@ def batch_norm(x, def square(x): - return pow(x, fill_const(2., x.shape, x.dtype)) + return pow(x, fill_const(2.0, x.shape, x.dtype)) @REGISTER_FN('add_p', 'X', 'Y', 'Z') @@ -279,10 +296,12 @@ def split(x, num_or_sections, axis=0, outs=None): helper.create_variable_for_type_inference(dtype=x.dtype) for i in range(n) ] - helper.append_op(type=helper.layer_type, - inputs={'X': x}, - outputs={'YS': outs}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, + inputs={'X': x}, + outputs={'YS': outs}, + attrs=attrs, + ) return outs @@ -294,17 +313,19 @@ def concat(xs, axis=0, out=None): helper = LayerHelper('concat_p', **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=xs[0].dtype) - helper.append_op(type=helper.layer_type, - inputs={'XS': xs}, - outputs={'Y': out}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, + inputs={'XS': xs}, + outputs={'Y': out}, + attrs=attrs, + ) return out @REGISTER_FN('reduce_sum_p', 'X', 'Y') def reduce_sum(x, axis=None, keepdim=False, out=None): axes = axis or tuple(range(0, len(x.shape))) - axes = (axes, ) if isinstance(axes, int) else axes + axes = (axes,) if isinstance(axes, int) else axes if not isinstance(axis, (tuple, list)): raise TypeError(f'axis must be tuple or list, but got {type(axis)}') if not isinstance(keepdim, bool): @@ -315,10 +336,9 @@ def reduce_sum(x, axis=None, keepdim=False, out=None): if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=helper.layer_type, - inputs={'X': x}, - outputs={'Y': out}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, inputs={'X': x}, outputs={'Y': out}, attrs=attrs + ) return out @@ -330,28 +350,33 @@ def matmul(x, y, out=None): @REGISTER_FN('slice_select_p', 'X', 'Y') def slice_select(x, axis, starts, ends, strides, out=None): if not isinstance(axis, (list, tuple)): - raise TypeError(f'Argument type error. `axis` is supposed to be list or' - f' tuple but found {type(axis)}.') + raise TypeError( + f'Argument type error. `axis` is supposed to be list or' + f' tuple but found {type(axis)}.' + ) if not isinstance(starts, (list, tuple)): raise TypeError( f'Argument type error. `starts` is supposed to be list or' - f' tuple but found {type(starts)}.') + f' tuple but found {type(starts)}.' + ) if not isinstance(ends, (list, tuple)): - raise TypeError(f'Argument type error. `ends` is supposed to be list or' - f' tuple but found {type(ends)}.') + raise TypeError( + f'Argument type error. `ends` is supposed to be list or' + f' tuple but found {type(ends)}.' + ) assert len(axis) == len(starts) == len(ends) == len(strides), ( f'len(axis), len(starts), len(ends) and len(strides) should be equal, ' f'but len(axis)={len(axis)}, len(starts)={len(starts)}, ' - f'len(ends)={len(ends)} and len(strides)={len(strides)}') + f'len(ends)={len(ends)} and len(strides)={len(strides)}' + ) attrs = {'axis': axis, 'starts': starts, 'ends': ends, 'strides': strides} helper = LayerHelper('slice_select_p', **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=helper.layer_type, - inputs={'X': x}, - outputs={'Y': out}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, inputs={'X': x}, outputs={'Y': out}, attrs=attrs + ) return out @@ -360,22 +385,23 @@ def slice_assign(x, y, axis, starts, ends, strides, out=None): assert len(starts) == len(ends) == len(strides) == len(axis), ( f'len(starts), len(ends), len(strides) and len(axis) should be equal, ' f'but len(starts)={len(starts)}, len(ends)={len(ends)}, ' - f'len(strides)={len(strides)} and len(axis)={len(axis)}') + f'len(strides)={len(strides)} and len(axis)={len(axis)}' + ) assert len(y.shape) == len(x.shape), ( f'len(y.shape) should be equal to len(x.shape), ' - f'but len(y.shape)={len(y.shape)} and len(x.shape)={len(x.shape)}.') + f'but len(y.shape)={len(y.shape)} and len(x.shape)={len(x.shape)}.' + ) attrs = {'axis': axis, 'starts': starts, 'ends': ends, 'strides': strides} helper = LayerHelper('slice_assign_p', **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=helper.layer_type, - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Z': out}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, + inputs={'X': x, 'Y': y}, + outputs={'Z': out}, + attrs=attrs, + ) return out @@ -385,13 +411,12 @@ def gather(x, indextensor, axis, out=None): helper = LayerHelper('gather_p', **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=helper.layer_type, - inputs={ - 'X': x, - 'IndexTensor': indextensor - }, - outputs={'Y': out}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, + inputs={'X': x, 'IndexTensor': indextensor}, + outputs={'Y': out}, + attrs=attrs, + ) return out @@ -399,26 +424,26 @@ def gather(x, indextensor, axis, out=None): def scatter_add(x, y, indextensor, axis, out=None): assert len(x.shape) == len(y.shape), ( f'len(x.shape) should be equal to len(y.shape), ' - f'but len(x.shape)={len(x.shape)} and len(y.shape)={len(y.shape)}.') - assert len( - indextensor.shape - ) == 1, f'len(indextensor.shape) must be equal to 1, but got {len(indextensor.shape)}.' + f'but len(x.shape)={len(x.shape)} and len(y.shape)={len(y.shape)}.' + ) + assert ( + len(indextensor.shape) == 1 + ), f'len(indextensor.shape) must be equal to 1, but got {len(indextensor.shape)}.' assert y.shape[axis] == indextensor.shape[0], ( f'y.shape[axis] should be equal to indextensor.shape[0], ' f'but y.shape[axis]={y.shape[axis]} and ' - f'indextensor.shape[0]={indextensor.shape[0]}.') + f'indextensor.shape[0]={indextensor.shape[0]}.' + ) attrs = {'axis': axis} helper = LayerHelper('scatter_add_p', **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=helper.layer_type, - inputs={ - 'X': x, - 'Y': y, - 'IndexTensor': indextensor - }, - outputs={'Z': out}, - attrs=attrs) + helper.append_op( + type=helper.layer_type, + inputs={'X': x, 'Y': y, 'IndexTensor': indextensor}, + outputs={'Z': out}, + attrs=attrs, + ) return out @@ -431,24 +456,26 @@ def log(x, out=None): def select(cond, x, y, out=None): if len(cond.shape) != len(x.shape): raise ValueError( - "len(cond.shape) should be equal to len(x.shape), but len(cond.shape)={} and len(x.shape)={}." - .format(len(cond.shape), len(x.shape))) + "len(cond.shape) should be equal to len(x.shape), but len(cond.shape)={} and len(x.shape)={}.".format( + len(cond.shape), len(x.shape) + ) + ) if len(x.shape) != len(y.shape): raise ValueError( - "len(x.shape) should be equal to len(y.shape), but len(x.shape)={} and len(y.shape)={}." - .format(len(x.shape), len(y.shape))) + "len(x.shape) should be equal to len(y.shape), but len(x.shape)={} and len(y.shape)={}.".format( + len(x.shape), len(y.shape) + ) + ) helper = LayerHelper('select_p', **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type=helper.layer_type, - inputs={ - 'Condition': cond, - 'X': x, - 'Y': y - }, - outputs={'Z': out}) + helper.append_op( + type=helper.layer_type, + inputs={'Condition': cond, 'X': x, 'Y': y}, + outputs={'Z': out}, + ) return out @@ -492,10 +519,12 @@ def cast(x, dtype, out=None): helper = LayerHelper('cast_p', **locals()) if out is None: out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=helper.layer_type, - inputs={'X': x}, - outputs={'Y': out}, - attrs={'dtype': dtype}) + helper.append_op( + type=helper.layer_type, + inputs={'X': x}, + outputs={'Y': out}, + attrs={'dtype': dtype}, + ) return out @@ -511,7 +540,7 @@ def uniform_random(dtype, min_value, max_value, seed, shape=None, out=None): 'dtype': dtype, 'min': min_value, 'max': max_value, - 'seed': seed + 'seed': seed, } helper = LayerHelper('uniform_random_p', **locals()) if out is None: diff --git a/python/paddle/incubate/autograd/primreg.py b/python/paddle/incubate/autograd/primreg.py index 31f47f8196315..7d81847f9052c 100644 --- a/python/paddle/incubate/autograd/primreg.py +++ b/python/paddle/incubate/autograd/primreg.py @@ -14,7 +14,8 @@ class Registry(object): - """ A general registry object. """ + """A general registry object.""" + __slots__ = ['name', 'tab'] def __init__(self, name): @@ -22,7 +23,9 @@ def __init__(self, name): self.tab = {} def register(self, name, value): - assert name not in self.tab, f'name "{name}" should not be registered before.' + assert ( + name not in self.tab + ), f'name "{name}" should not be registered before.' self.tab[name] = value def lookup(self, name): @@ -78,15 +81,17 @@ def div(x, y, out=None): """ args = _primop_position_argnames.lookup(op.type) - assert args is not None, f'args of {op.type} should not be None in op_position_inputs().' + assert ( + args is not None + ), f'args of {op.type} should not be None in op_position_inputs().' *input_names, _ = args inputs = [] for name in input_names: vars = list(map(op.block.var, op.input(name))) - assert len( - vars - ) >= 0, f'len(vars) should be greater than or equal to 0, but len(vars)={len(vars)}.' + assert ( + len(vars) >= 0 + ), f'len(vars) should be greater than or equal to 0, but len(vars)={len(vars)}.' if len(vars) > 1: inputs.append(vars) else: @@ -120,9 +125,9 @@ def div(x, y, out=None): *_, output_name = args outvars = list(map(op.block.var, op.output(output_name))) - assert len( - outvars - ) >= 0, f'len(outvars) should be greater than or equal to 0, but len(outvars)={len(outvars)}.' + assert ( + len(outvars) >= 0 + ), f'len(outvars) should be greater than or equal to 0, but len(outvars)={len(outvars)}.' if len(outvars) > 1: output = outvars else: @@ -184,9 +189,10 @@ def tanh_orig2prim(op): raise TypeError(f'op_type must be str, but got {type(op_type)}.') def wrapper(f): - def _lower(op, *args, **kwargs): - assert op.type == op_type, f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' + assert ( + op.type == op_type + ), f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' return f(op, *args, **kwargs) _orig2prim.register(op_type, _lower) @@ -216,9 +222,10 @@ def tanh_prim2orig(op): raise TypeError(f'op_type must be str, but got {type(op_type)}.') def wrapper(f): - def _lower(op, *args, **kwargs): - assert op.type == op_type, f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' + assert ( + op.type == op_type + ), f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' return f(op, *args, **kwargs) _prim2orig.register(op_type, _lower) @@ -247,9 +254,10 @@ def add_jvp(op, x_dot, y_dot): raise TypeError(f'op_type must be str, but got {type(op_type)}.') def wrapper(f): - def _jvp(op, *args, **kwargs): - assert op.type == op_type, f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' + assert ( + op.type == op_type + ), f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' return f(op, *args, **kwargs) _primop_jvp.register(op_type, _jvp) @@ -280,9 +288,10 @@ def add_transpose(op, z_bar): raise TypeError(f'op_type must be str, but got {type(op_type)}.') def wrapper(f): - def _transpose(op, dot_checker, *args, **kwargs): - assert op.type == op_type, f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' + assert ( + op.type == op_type + ), f'op.type should be equal to op_type, but op.type is {op.type} and op_type is {op_type}' return f(op, dot_checker, *args, **kwargs) _primop_transpose.register(op_type, _transpose) diff --git a/python/paddle/incubate/autograd/primrules.py b/python/paddle/incubate/autograd/primrules.py index 8a2f94145cd06..1d98d62cef3e0 100644 --- a/python/paddle/incubate/autograd/primrules.py +++ b/python/paddle/incubate/autograd/primrules.py @@ -19,15 +19,54 @@ import paddle from . import primops -from .primops import (add, broadcast, concat, cos, div, eq, erf, exp, - fill_const, gather, ge, gt, log, matmul, mul, ne, neg, - reduce_sum, reshape, scatter_add, select, set_value, sin, - slice_assign, slice_select, split, sqrt, sub, tanh, - transpose, bernoulli, rsqrt, uniform_random) -from .primreg import (REGISTER_JVP, REGISTER_ORIG2PRIM, REGISTER_PRIM2ORIG, - REGISTER_TRANSPOSE, lookup_fn, lookup_jvp, - lookup_orig2prim, lookup_prim2orig, lookup_transpose, - op_position_inputs, op_position_output) +from .primops import ( + add, + broadcast, + concat, + cos, + div, + eq, + erf, + exp, + fill_const, + gather, + ge, + gt, + log, + matmul, + mul, + ne, + neg, + reduce_sum, + reshape, + scatter_add, + select, + set_value, + sin, + slice_assign, + slice_select, + split, + sqrt, + sub, + tanh, + transpose, + bernoulli, + rsqrt, + uniform_random, +) +from .primreg import ( + REGISTER_JVP, + REGISTER_ORIG2PRIM, + REGISTER_PRIM2ORIG, + REGISTER_TRANSPOSE, + lookup_fn, + lookup_jvp, + lookup_orig2prim, + lookup_prim2orig, + lookup_transpose, + op_position_inputs, + op_position_output, +) from .utils import INT_DTYPE_2_STRING, get_output_var_list @@ -94,20 +133,20 @@ def elementwise_add_orig2prim(op, x, y): if x.shape != y.shape: y = broadcast(y, shape=x.shape) if op.attr('Scale_x') - 1.0 > 1e-5: - scale_x = fill_const(shape=x.shape, - dtype=x.dtype, - value=op.attr('Scale_x')) + scale_x = fill_const( + shape=x.shape, dtype=x.dtype, value=op.attr('Scale_x') + ) x = mul(x, scale_x) if op.attr('Scale_y') - 1.0 > 1e-5: - scale_y = fill_const(shape=y.shape, - dtype=y.dtype, - value=op.attr('Scale_y')) + scale_y = fill_const( + shape=y.shape, dtype=y.dtype, value=op.attr('Scale_y') + ) y = mul(y, scale_y) z = add(x, y) if op.attr('Scale_out') - 1.0 > 1e-5: - scale_out = fill_const(shape=z.shape, - dtype=z.dtype, - value=op.attr('Scale_out')) + scale_out = fill_const( + shape=z.shape, dtype=z.dtype, value=op.attr('Scale_out') + ) z = mul(z, scale_out) return z @@ -117,20 +156,20 @@ def elementwise_sub_orig2prim(op, x, y): if x.shape != y.shape: y = broadcast(y, shape=x.shape) if op.attr('Scale_x') - 1.0 > 1e-5: - scale_x = fill_const(shape=x.shape, - dtype=x.dtype, - value=op.attr('Scale_x')) + scale_x = fill_const( + shape=x.shape, dtype=x.dtype, value=op.attr('Scale_x') + ) x = mul(x, scale_x) if op.attr('Scale_y') - 1.0 > 1e-5: - scale_y = fill_const(shape=y.shape, - dtype=y.dtype, - value=op.attr('Scale_y')) + scale_y = fill_const( + shape=y.shape, dtype=y.dtype, value=op.attr('Scale_y') + ) y = mul(y, scale_y) z = sub(x, y) if op.attr('Scale_out') - 1.0 > 1e-5: - scale_out = fill_const(shape=z.shape, - dtype=z.dtype, - value=op.attr('Scale_out')) + scale_out = fill_const( + shape=z.shape, dtype=z.dtype, value=op.attr('Scale_out') + ) z = mul(z, scale_out) return z @@ -140,20 +179,20 @@ def elementwise_mul_orig2prim(op, x, y): if x.shape != y.shape: y = broadcast(y, shape=x.shape) if op.attr('Scale_x') - 1.0 > 1e-5: - scale_x = fill_const(shape=x.shape, - dtype=x.dtype, - value=op.attr('Scale_x')) + scale_x = fill_const( + shape=x.shape, dtype=x.dtype, value=op.attr('Scale_x') + ) x = mul(x, scale_x) if op.attr('Scale_y') - 1.0 > 1e-5: - scale_y = fill_const(shape=y.shape, - dtype=y.dtype, - value=op.attr('Scale_y')) + scale_y = fill_const( + shape=y.shape, dtype=y.dtype, value=op.attr('Scale_y') + ) y = mul(y, scale_y) z = mul(x, y) if op.attr('Scale_out') - 1.0 > 1e-5: - scale_out = fill_const(shape=z.shape, - dtype=z.dtype, - value=op.attr('Scale_out')) + scale_out = fill_const( + shape=z.shape, dtype=z.dtype, value=op.attr('Scale_out') + ) z = mul(z, scale_out) return z @@ -209,23 +248,26 @@ def fill_zeros_like_orig2prim(op, x): def fill_any_like_orig2prim(op, x): if op.attr('dtype') == -1: return fill_const(value=op.attr('value'), shape=x.shape, dtype=x.dtype) - return fill_const(value=op.attr('value'), - shape=x.shape, - dtype=paddle.dtype(op.attr('dtype'))) + return fill_const( + value=op.attr('value'), + shape=x.shape, + dtype=paddle.dtype(op.attr('dtype')), + ) @REGISTER_ORIG2PRIM('fill_constant') -def fill_const_orig2prim(op, - shape_tensor=None, - shape_tensor_list=None, - value_tensor=None): +def fill_const_orig2prim( + op, shape_tensor=None, shape_tensor_list=None, value_tensor=None +): if shape_tensor or shape_tensor_list or value_tensor: raise TypeError( 'fill_const_orig2prim currently not support Tensor input of shape and value.' ) - return fill_const(value=op.attr('value'), - shape=op.attr('shape'), - dtype=paddle.dtype(op.attr('dtype'))) + return fill_const( + value=op.attr('value'), + shape=op.attr('shape'), + dtype=paddle.dtype(op.attr('dtype')), + ) @REGISTER_ORIG2PRIM('sum') @@ -244,9 +286,9 @@ def index_select_orig2prim(op, index_t, x): @REGISTER_ORIG2PRIM('scale') def scale_orig2prim(op, scale_t, x): if scale_t is None: - scale_t = fill_const(shape=x.shape, - dtype=x.dtype, - value=op.attr('scale')) + scale_t = fill_const( + shape=x.shape, dtype=x.dtype, value=op.attr('scale') + ) bias_t = fill_const(shape=x.shape, dtype=x.dtype, value=op.attr('bias')) if op.attr('bias_after_scale'): return add(mul(x, scale_t), bias_t) @@ -272,14 +314,14 @@ def rsqrt_orig2prim(op, x): @REGISTER_ORIG2PRIM('matmul_v2') def matmul_v2_orig2prim(op, x, y): - def trans(shape): ret = [i for i in range(len(shape))] ret[-1], ret[-2] = ret[-2], ret[-1] return ret - assert len(x.shape) < 4 and len( - y.shape) < 4, 'Do not support multi batchsize dimensions currently.' + assert ( + len(x.shape) < 4 and len(y.shape) < 4 + ), 'Do not support multi batchsize dimensions currently.' if len(x.shape) == 1: x = broadcast(x, shape=[1, x.shape[0]]) @@ -295,12 +337,16 @@ def trans(shape): ## NOTE(lml): The second output of reshape2 Xshape, which is only used in reshape2_grad, is meanlingless in new autograd mechanism, thus we use a zero tensor instead. @REGISTER_ORIG2PRIM('reshape2') def reshape2_orig2prim(op, shape_t, shape_tl, x): - assert shape_t is None, 'Can not lower reshape2 into prim ops with shapetensor.' - assert shape_tl is None, 'Can not lower reshape2 into prim ops with shapetensorlist.' + assert ( + shape_t is None + ), 'Can not lower reshape2 into prim ops with shapetensor.' + assert ( + shape_tl is None + ), 'Can not lower reshape2 into prim ops with shapetensorlist.' y, xshape = get_output_var_list(op) - return reshape(x, shape=y.shape), fill_const(shape=xshape.shape, - dtype=xshape.dtype, - value=0.0) + return reshape(x, shape=y.shape), fill_const( + shape=xshape.shape, dtype=xshape.dtype, value=0.0 + ) @REGISTER_ORIG2PRIM('concat') @@ -311,10 +357,16 @@ def concat_orig2prim(op, axis_t, xs): @REGISTER_ORIG2PRIM('slice') def slice_orig2prim(op, ends_t, ends_tl, x, starts_t, starts_tl): - assert starts_t is None, 'Can not lower concat into prim ops with startstensor.' + assert ( + starts_t is None + ), 'Can not lower concat into prim ops with startstensor.' assert ends_t is None, 'Can not lower concat into prim ops with endstensor.' - assert starts_tl is None, 'Can not lower concat into prim ops with startstensorlist.' - assert ends_tl is None, 'Can not lower concat into prim ops with endstensorlist.' + assert ( + starts_tl is None + ), 'Can not lower concat into prim ops with startstensorlist.' + assert ( + ends_tl is None + ), 'Can not lower concat into prim ops with endstensorlist.' starts = op.attr('starts') ends = op.attr('ends') strides = [1 for _ in starts] @@ -329,12 +381,12 @@ def slice_orig2prim(op, ends_t, ends_tl, x, starts_t, starts_tl): def sigmoid_orig2prim(op, x): return div( fill_const(value=1.0, shape=x.shape, dtype=x.dtype), - (add(fill_const(value=1.0, shape=x.shape, dtype=x.dtype), exp(neg(x))))) + (add(fill_const(value=1.0, shape=x.shape, dtype=x.dtype), exp(neg(x)))), + ) @REGISTER_ORIG2PRIM('p_norm') def p_norm_orig2prim(op, x): - def num_el(shape): n = 1 for s in shape: @@ -342,7 +394,8 @@ def num_el(shape): return n assert op.attr( - 'asvector'), 'Only support lower pnorm when asvector=True currently' + 'asvector' + ), 'Only support lower pnorm when asvector=True currently' if len(x.shape) > 1: x = reshape(x, shape=[num_el(x.shape)]) @@ -435,25 +488,38 @@ def gelu_orig2prim(op, x): x, mul( fill_const(0.044715, x.shape, x.dtype), - primops.pow(x, fill_const(3., x.shape, - x.dtype)))))))) + primops.pow( + x, fill_const(3.0, x.shape, x.dtype) + ), + ), + ), + ) + ), + ), + ) return mul(x, cdf) else: return mul( mul(fill_const(0.5, x.shape, x.dtype), x), - add(fill_const(1.0, x.shape, x.dtype), - erf(mul(x, fill_const(1 / math.sqrt(2.), x.shape, x.dtype))))) + add( + fill_const(1.0, x.shape, x.dtype), + erf(mul(x, fill_const(1 / math.sqrt(2.0), x.shape, x.dtype))), + ), + ) @REGISTER_ORIG2PRIM('dropout') def dropout_orig2prim(op, seed_t, x): - assert seed_t is None, 'Can not lower dropout into prim ops with seedtensor.' + assert ( + seed_t is None + ), 'Can not lower dropout into prim ops with seedtensor.' mask = bernoulli(shape=x.shape, dtype=x.dtype, p=op.attr('dropout_prob')) if op.attr('dropout_implementation') == 'upscale_in_train': if op.attr('is_test') == False: out = div( mul(x, mask), - fill_const(1.0 - op.attr('dropout_prob'), x.shape, x.dtype)) + fill_const(1.0 - op.attr('dropout_prob'), x.shape, x.dtype), + ) return primops.cast(mask, dtype=paddle.uint8), out else: return primops.cast(mask, dtype=paddle.uint8), x @@ -462,7 +528,8 @@ def dropout_orig2prim(op, seed_t, x): return primops.cast(mask, dtype=paddle.uint8), mul(x, mask) else: return primops.cast(mask, dtype=paddle.uint8), mul( - x, fill_const(1.0 - op.attr('dropout_prob'), x.shape, x.dtype)) + x, fill_const(1.0 - op.attr('dropout_prob'), x.shape, x.dtype) + ) else: raise RuntimeError( 'Unsupported dropout_implementation, only support upscale_in_train and downgrade_in_infer' @@ -485,50 +552,62 @@ def uniform_random_orig2prim(op, shape_t, shape_tl): @REGISTER_ORIG2PRIM('reduce_sum') def reduce_sum_orig2prim(op, x): - axes = tuple(range(0, len( - x.shape))) if op.attr('reduce_all') else op.attr('dim') + axes = ( + tuple(range(0, len(x.shape))) + if op.attr('reduce_all') + else op.attr('dim') + ) return reduce_sum(x, axis=axes, keepdim=op.attr('keep_dim')) @REGISTER_ORIG2PRIM('reduce_mean') def reduce_mean_orig2prim(op, x): - axes = tuple(range(0, len( - x.shape))) if op.attr('reduce_all') else op.attr('dim') + axes = ( + tuple(range(0, len(x.shape))) + if op.attr('reduce_all') + else op.attr('dim') + ) return primops.mean(x, axes, op.attr('keep_dim')) @REGISTER_ORIG2PRIM('batch_norm') -def batch_norm_orig2prim(op, bias, run_mean, momentum_tensor, scale, run_var, - x): +def batch_norm_orig2prim( + op, bias, run_mean, momentum_tensor, scale, run_var, x +): momentum = op.attr('momentum') eps = op.attr('epsilon') is_test = op.attr('is_test') data_layout = op.attr('data_layout') use_global_stats = op.attr('use_global_stats') trainable_statistics = op.attr('trainable_statistics') - reserve_space = None if len( - op.output_names) == 5 else get_output_var_list(op)[1] + reserve_space = ( + None if len(op.output_names) == 5 else get_output_var_list(op)[1] + ) - feature_axis = 1 if data_layout in ('NC', 'NCL', 'NCHW', - 'NCHWD') else len(x.shape) - 1 + feature_axis = ( + 1 if data_layout in ('NC', 'NCL', 'NCHW', 'NCHWD') else len(x.shape) - 1 + ) use_run_stat = (is_test and (not trainable_statistics)) or use_global_stats - return primops.batch_norm(x, - feature_axis, - scale, - bias, - run_mean, - run_var, - eps=eps, - momentum=momentum, - use_run_stat=use_run_stat, - reserve_space=reserve_space) + return primops.batch_norm( + x, + feature_axis, + scale, + bias, + run_mean, + run_var, + eps=eps, + momentum=momentum, + use_run_stat=use_run_stat, + reserve_space=reserve_space, + ) @REGISTER_ORIG2PRIM('size') def size_orig2prim(op, x): - return fill_const(functools.reduce(operator.mul, x.shape), (1, ), - paddle.int64) + return fill_const( + functools.reduce(operator.mul, x.shape), (1,), paddle.int64 + ) ## Register prim2orig lower rules @@ -617,9 +696,9 @@ def split_prim2orig(op, x): num_or_sections = op.attr('num_or_sections') if len(num_or_sections) == 1: num_or_sections = num_or_sections[0] - return paddle.split(x, - num_or_sections=num_or_sections, - axis=op.attr('axis')) + return paddle.split( + x, num_or_sections=num_or_sections, axis=op.attr('axis') + ) @REGISTER_PRIM2ORIG('concat_p') @@ -639,23 +718,27 @@ def matmul_prim2orig(op, x, y): @REGISTER_PRIM2ORIG('slice_select_p') def slice_select_prim2orig(op, x): - return paddle.strided_slice(x, - axes=op.attr('axis'), - starts=op.attr('starts'), - ends=op.attr('ends'), - strides=op.attr('strides')) + return paddle.strided_slice( + x, + axes=op.attr('axis'), + starts=op.attr('starts'), + ends=op.attr('ends'), + strides=op.attr('strides'), + ) @REGISTER_PRIM2ORIG('slice_assign_p') def slice_assign_prim2orig(op, x, y): x_copy = paddle.assign(x) - return set_value(x_copy, - y, - axis=op.attr('axis'), - starts=op.attr('starts'), - ends=op.attr('ends'), - strides=op.attr('strides'), - out=x_copy) + return set_value( + x_copy, + y, + axis=op.attr('axis'), + starts=op.attr('starts'), + ends=op.attr('ends'), + strides=op.attr('strides'), + out=x_copy, + ) @REGISTER_PRIM2ORIG('gather_p') @@ -673,26 +756,32 @@ def scatter_add_prim2orig(op, index_t, x, y): @REGISTER_PRIM2ORIG('fill_constant_p') def fill_constant_prim2orig(op): - return paddle.full(shape=op.attr('shape'), - fill_value=op.attr('value'), - dtype=INT_DTYPE_2_STRING[op.attr('dtype')]) + return paddle.full( + shape=op.attr('shape'), + fill_value=op.attr('value'), + dtype=INT_DTYPE_2_STRING[op.attr('dtype')], + ) @REGISTER_PRIM2ORIG('bernoulli_p') def bernoulli_prim2orig(op): - t = paddle.full(shape=op.attr('shape'), - fill_value=op.attr('p'), - dtype=INT_DTYPE_2_STRING[op.attr('dtype')]) + t = paddle.full( + shape=op.attr('shape'), + fill_value=op.attr('p'), + dtype=INT_DTYPE_2_STRING[op.attr('dtype')], + ) return paddle.bernoulli(t) @REGISTER_PRIM2ORIG('uniform_random_p') def uniform_random_prim2orig(op): - return paddle.uniform(shape=op.attr('shape'), - dtype=INT_DTYPE_2_STRING[op.attr('dtype')], - min=op.attr('min'), - max=op.attr('max'), - seed=op.attr('seed')) + return paddle.uniform( + shape=op.attr('shape'), + dtype=INT_DTYPE_2_STRING[op.attr('dtype')], + min=op.attr('min'), + max=op.attr('max'), + seed=op.attr('seed'), + ) @REGISTER_PRIM2ORIG('select_p') @@ -810,7 +899,7 @@ def tanh_jvp(op, x_dot): def sin_jvp(op, x_dot): if x_dot is None: return None - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) return mul(x_dot, cos(x)) @@ -818,7 +907,7 @@ def sin_jvp(op, x_dot): def cos_jvp(op, x_dot): if x_dot is None: return None - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) return mul(x_dot, neg(sin(x))) @@ -834,25 +923,26 @@ def exp_jvp(op, x_dot): def erf_jvp(op, x_dot): if x_dot is None: return None - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) return mul( - fill_const(2. / math.sqrt(math.pi), x.shape, x.dtype), - mul(x_dot, exp(neg(primops.pow(x, fill_const(2., x.shape, x.dtype)))))) + fill_const(2.0 / math.sqrt(math.pi), x.shape, x.dtype), + mul(x_dot, exp(neg(primops.pow(x, fill_const(2.0, x.shape, x.dtype))))), + ) @REGISTER_JVP('abs_p') def abs_jvp(op, x_dot): if x_dot is None: return None - x, = op_position_inputs(op) - return select(ge(x, fill_const(0., x.shape, x.dtype)), x_dot, neg(x_dot)) + (x,) = op_position_inputs(op) + return select(ge(x, fill_const(0.0, x.shape, x.dtype)), x_dot, neg(x_dot)) @REGISTER_JVP('log_p') def log_jvp(op, x_dot): if x_dot is None: return None - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) return div(x_dot, x) @@ -929,12 +1019,9 @@ def slice_select_jvp(op, x_dot): starts = op.attr('starts') ends = op.attr('ends') strides = op.attr('strides') - return linear_jvp(op, - x_dot, - axis=axis, - starts=starts, - ends=ends, - strides=strides) + return linear_jvp( + op, x_dot, axis=axis, starts=starts, ends=ends, strides=strides + ) @REGISTER_JVP('slice_assign_p') @@ -948,13 +1035,9 @@ def slice_assign_jvp(op, x_dot, y_dot): starts = op.attr('starts') ends = op.attr('ends') strides = op.attr('strides') - return linear_jvp(op, - x_dot, - y_dot, - axis=axis, - starts=starts, - ends=ends, - strides=strides) + return linear_jvp( + op, x_dot, y_dot, axis=axis, starts=starts, ends=ends, strides=strides + ) @REGISTER_JVP('gather_p') @@ -993,7 +1076,7 @@ def eq_jvp(op, x_dot, y_dot): if x_dot is None and y_dot is None: return None x, _ = op_position_inputs(op) - z_dot = fill_const(value=0., shape=x.shape, dtype=x.dtype) + z_dot = fill_const(value=0.0, shape=x.shape, dtype=x.dtype) return z_dot @@ -1002,7 +1085,7 @@ def gt_jvp(op, x_dot, y_dot): if x_dot is None and y_dot is None: return None x, _ = op_position_inputs(op) - z_dot = fill_const(value=0., shape=x.shape, dtype=x.dtype) + z_dot = fill_const(value=0.0, shape=x.shape, dtype=x.dtype) return z_dot @@ -1011,7 +1094,7 @@ def ge_jvp(op, x_dot, y_dot): if x_dot is None and y_dot is None: return None x, _ = op_position_inputs(op) - z_dot = fill_const(value=0., shape=x.shape, dtype=x.dtype) + z_dot = fill_const(value=0.0, shape=x.shape, dtype=x.dtype) return z_dot @@ -1020,13 +1103,12 @@ def ne_jvp(op, x_dot, y_dot): if x_dot is None and y_dot is None: return None x, _ = op_position_inputs(op) - z_dot = fill_const(value=0., shape=x.shape, dtype=x.dtype) + z_dot = fill_const(value=0.0, shape=x.shape, dtype=x.dtype) return z_dot @REGISTER_JVP('pow_p') def pow_jvp(op, x_dot, y_dot): - def _compute_t1(x, y): zero_y = fill_const(value=0.0, shape=y.shape, dtype=y.dtype) one_y = fill_const(value=1.0, shape=y.shape, dtype=y.dtype) @@ -1096,7 +1178,8 @@ def add_transpose(op, check_dot, z_bar): x, y = op_position_inputs(op) assert check_dot(x) or check_dot(y), ( f'(check_dot(x) or check_dot(y)) must be True, ' - f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.') + f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.' + ) x_bar = z_bar if check_dot(x) else None y_bar = z_bar if check_dot(y) else None return x_bar, y_bar @@ -1107,7 +1190,8 @@ def sub_transpose(op, check_dot, z_bar): x, y = op_position_inputs(op) assert check_dot(x) or check_dot(y), ( f'(check_dot(x) or check_dot(y)) must be True, ' - f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.') + f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.' + ) x_bar = z_bar if check_dot(x) else None y_bar = neg(z_bar) if check_dot(y) else None return x_bar, y_bar @@ -1118,7 +1202,8 @@ def mul_transpose(op, check_dot, z_bar): x, y = op_position_inputs(op) assert check_dot(x) ^ check_dot(y), ( f'(check_dot(x) ^ check_dot(y)) must be True, ' - f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.') + f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.' + ) if check_dot(x): return mul(z_bar, y), None else: @@ -1135,14 +1220,14 @@ def div_transpose(op, check_dot, z_bar): @REGISTER_TRANSPOSE('reshape_p') def reshape_transpose(op, check_dot, y_bar): - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) assert check_dot(x), 'check_dot(x) must be True' return reshape(y_bar, shape=x.shape) @REGISTER_TRANSPOSE('broadcast_p') def broadcast_transpose(op, check_dot, y_bar): - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) assert check_dot(x), 'check_dot(x) must be True' bat = len(y_bar.shape) - len(x.shape) axis = list(range(bat)) @@ -1155,7 +1240,7 @@ def broadcast_transpose(op, check_dot, y_bar): @REGISTER_TRANSPOSE('transpose_p') def transpose_transpose(op, check_dot, y_bar): - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) assert check_dot(x), 'check_dot(x) must be True' axis = op.attr('axis') reordered = sorted((k, i) for i, k in enumerate(axis)) @@ -1165,14 +1250,14 @@ def transpose_transpose(op, check_dot, y_bar): @REGISTER_TRANSPOSE('split_p') def split_transpose(op, check_dot, ys_bar): - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) assert check_dot(x), 'check_dot(x) must be True' return concat(ys_bar, axis=op.attr('axis')) @REGISTER_TRANSPOSE('concat_p') def concat_transpose(op, check_dot, y_bar): - xs, = op_position_inputs(op) + (xs,) = op_position_inputs(op) if not isinstance(xs, typing.Sequence): xs = [xs] for x in xs: @@ -1186,7 +1271,7 @@ def concat_transpose(op, check_dot, y_bar): @REGISTER_TRANSPOSE('reduce_sum_p') def reduce_sum_transpose(op, check_dot, y_bar): - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) assert check_dot(x), 'check_dot(x) must be True' axes = op.attr('axis') shape = tuple(1 if i in axes else size for i, size in enumerate(x.shape)) @@ -1199,7 +1284,8 @@ def matmul_transpose(op, check_dot, z_bar): x, y = op_position_inputs(op) assert check_dot(x) ^ check_dot(y), ( f'(check_dot(x) ^ check_dot(y)) must be True, ' - f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.') + f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.' + ) # TODO: replace it. this is hacky axis = [1, 0] if len(x.shape) == 2 else [0, 2, 1] if check_dot(x): @@ -1210,19 +1296,16 @@ def matmul_transpose(op, check_dot, z_bar): @REGISTER_TRANSPOSE('slice_select_p') def slice_select_transpose(op, check_dot, y_bar): - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) assert check_dot(x), 'check_dot(x) must be True' zeros = fill_const(value=0.0, shape=x.shape, dtype=x.dtype) axis = op.attr('axis') starts = op.attr('starts') ends = op.attr('ends') strides = op.attr('strides') - return slice_assign(zeros, - y_bar, - axis=axis, - starts=starts, - ends=ends, - strides=strides) + return slice_assign( + zeros, y_bar, axis=axis, starts=starts, ends=ends, strides=strides + ) @REGISTER_TRANSPOSE('slice_assign_p') @@ -1230,23 +1313,19 @@ def slice_assign_transpose(op, check_dot, z_bar): x, y = op_position_inputs(op) assert check_dot(x) and check_dot(y), ( f'(check_dot(x) and check_dot(y)) must be True, ' - f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.') + f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.' + ) zeros = fill_const(value=0.0, shape=y.shape, dtype=y.dtype) axis = op.attr('axis') starts = op.attr('starts') ends = op.attr('ends') strides = op.attr('strides') - x_bar = slice_assign(z_bar, - zeros, - axis=axis, - starts=starts, - ends=ends, - strides=strides) - y_bar = slice_select(z_bar, - axis=axis, - starts=starts, - ends=ends, - strides=strides) + x_bar = slice_assign( + z_bar, zeros, axis=axis, starts=starts, ends=ends, strides=strides + ) + y_bar = slice_select( + z_bar, axis=axis, starts=starts, ends=ends, strides=strides + ) return x_bar, y_bar @@ -1266,7 +1345,8 @@ def scatter_add_transpose(op, check_dot, z_bar): x, y, indextensor = op_position_inputs(op) assert check_dot(x) and check_dot(y), ( f'(check_dot(x) and check_dot(y)) must be True, ' - f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.') + f'but check_dot(x)={check_dot(x)} and check_dot(y)={check_dot(y)}.' + ) axis = op.attr('axis') zeros = fill_const(value=0.0, shape=y.shape, dtype=y.dtype) x_bar = scatter_add(z_bar, zeros, indextensor, axis=axis) @@ -1286,8 +1366,11 @@ def select_transpose(op, check_dot, z_bar): zeros_x = fill_const(value=0.0, shape=x.shape, dtype=x.dtype) zeros_y = fill_const(value=0.0, shape=y.shape, dtype=y.dtype) - cond_bar = fill_const(value=0.0, shape=y.shape, - dtype=cond.dtype) if check_dot(cond) else None + cond_bar = ( + fill_const(value=0.0, shape=y.shape, dtype=cond.dtype) + if check_dot(cond) + else None + ) x_bar = select(cond, z_bar, zeros_x) if check_dot(x) else None y_bar = select(cond, zeros_y, z_bar) if check_dot(y) else None @@ -1296,5 +1379,5 @@ def select_transpose(op, check_dot, z_bar): @REGISTER_TRANSPOSE('cast_p') def cast_transpose(op, check_dot, y_bar): - x, = op_position_inputs(op) + (x,) = op_position_inputs(op) return primops.cast(y_bar, x.dtype) diff --git a/python/paddle/incubate/autograd/primx.py b/python/paddle/incubate/autograd/primx.py index c57f651d344c3..cdf16e77b1fbc 100644 --- a/python/paddle/incubate/autograd/primx.py +++ b/python/paddle/incubate/autograd/primx.py @@ -20,15 +20,23 @@ from paddle.incubate.autograd.utils import as_tensors from .primops import add, fill_const -from .primreg import (lookup_orig2prim, lookup_prim2orig, op_position_inputs, - op_position_output) +from .primreg import ( + lookup_orig2prim, + lookup_prim2orig, + op_position_inputs, + op_position_output, +) from .primrules import _jvp, _orig2prim, _prim2orig, _transpose -from .utils import (flatten, flatten_and_remove_none, get_input_var_list, - get_output_var_list) +from .utils import ( + flatten, + flatten_and_remove_none, + get_input_var_list, + get_output_var_list, +) def topo_path(xs, ys, block=None): - """ Returns the list of ops on the path from `xs` to `ys` in topological + """Returns the list of ops on the path from `xs` to `ys` in topological order. TODO(Tongxin): supporting control flow and nested blocks. @@ -50,13 +58,16 @@ def topo_path(xs, ys, block=None): # Initialize reached vars for x in xs: - assert x is None or x.block == block, 'x is not None and x.block != block' + assert ( + x is None or x.block == block + ), 'x is not None and x.block != block' reached_vars[id(x)] = x # Reaching test, returning whether an op is reached from the given input reaching = lambda op: any( id(v) in reached_vars - for v in flatten_and_remove_none(get_input_var_list(op))) + for v in flatten_and_remove_none(get_input_var_list(op)) + ) # block.ops are supposedly in the order that preserves correct data # dependence. @@ -70,7 +81,8 @@ def topo_path(xs, ys, block=None): used_vars = OrderedDict((id(y), y) for y in ys if id(y) in reached_vars) back_reaching = lambda op: any( id(out) in used_vars - for out in flatten_and_remove_none(get_output_var_list(op))) + for out in flatten_and_remove_none(get_output_var_list(op)) + ) # Backward pass to find all used variables for op in reversed(path): @@ -86,7 +98,7 @@ def topo_path(xs, ys, block=None): def output_vars_on_path(path): - """ Returns the output variables of all the ops on the path from `xs` + """Returns the output variables of all the ops on the path from `xs` to `ys`. Args: @@ -104,7 +116,7 @@ def output_vars_on_path(path): class VarMap(object): - """ A general map data structure for linking variables to variables. + """A general map data structure for linking variables to variables. An example is linking variables to their gradients. """ @@ -125,7 +137,8 @@ def add_rec(self, key_vars, value_vars): if isinstance(key_vars, paddle.fluid.framework.Variable): if not isinstance(value_vars, paddle.fluid.framework.Variable): raise TypeError( - f'value_vars must be Variable, but got {type(value_vars)}') + f'value_vars must be Variable, but got {type(value_vars)}' + ) self.tab[id(key_vars)] = id(value_vars) else: assert len(key_vars) == len(value_vars), ( @@ -168,11 +181,12 @@ def contain_value(self, value_var): # TODO(lml): supporting control flow, nested blocks, and block other than current block of main program. class Transform(object): - """ An object that maintains the state of transformations applied to a - primitve program. """ + """An object that maintains the state of transformations applied to a + primitve program.""" def __init__(self, block): - assert block == default_main_program().current_block( + assert ( + block == default_main_program().current_block() ), 'only support transform on current block of main program.' self.block = block self.vars = self.init_vars(block) @@ -224,7 +238,7 @@ def erase_dots(self, vars_to_erase): block._sync_with_cpp() def var2dot_rec(self, vars): - """ Lookup var2dot recursively.""" + """Lookup var2dot recursively.""" if isinstance(vars, paddle.fluid.framework.Variable): dot = self.var2dot.lookup(vars) return dot @@ -243,7 +257,7 @@ def dot2bar_rec(self, dots): return bars def linearize(self, xs, ys, xs_dot=None): - """ Performs the linearization transform, a.k.a, forward mode AD + """Performs the linearization transform, a.k.a, forward mode AD transform, on a primitive lowered program. Args: @@ -265,15 +279,18 @@ def linearize(self, xs, ys, xs_dot=None): else: assert len(xs) == len(xs_dot), ( f'len(xs) should be equal to len(xs_dot), ' - f'but len(xs)={len(xs)} and len(xs_dot)={len(xs_dot)}') + f'but len(xs)={len(xs)} and len(xs_dot)={len(xs_dot)}' + ) for x, dot in zip(xs, xs_dot): assert x.dtype == dot.dtype, ( f'x.dtype should be equal to dot.dtype, ' - f'but x.dtype={x.dtype} and dot.dtype={dot.dtype}') + f'but x.dtype={x.dtype} and dot.dtype={dot.dtype}' + ) assert x.shape == dot.shape, ( f'x.shape should be equal to dot.shape, ' - f'but x.shape={x.shape} and dot.shape={dot.shape}') + f'but x.shape={x.shape} and dot.shape={dot.shape}' + ) self.var2dot.add(x, dot) path, unused_xs, _ = topo_path(xs, ys, self.block) @@ -299,7 +316,7 @@ def linearize(self, xs, ys, xs_dot=None): return xs_dot, ys_dot def transpose(self, ys_dot, xs_dot, ys_bar=None, retain_fwd=False): - """ Performs the transpose transform, a.k.a, reverse mode AD + """Performs the transpose transform, a.k.a, reverse mode AD transform, on a linearized primitive program. Note, `transpose` is supposed to be used in couple with `linearize`. @@ -328,7 +345,8 @@ def transpose(self, ys_dot, xs_dot, ys_bar=None, retain_fwd=False): else: assert len(ys_dot) == len(ys_bar), ( f'len(ys_dot) should be equal to len(ys_bar), ' - f'but len(ys_dot)={len(ys_dot)} and len(ys_bar)={len(ys_bar)}') + f'but len(ys_dot)={len(ys_dot)} and len(ys_bar)={len(ys_bar)}' + ) for y_dot, y_bar in zip(ys_dot, ys_bar): assert y_dot.shape == y_bar.shape, ( f'y_dot.shape should be equal to y_bar.shape, ' @@ -372,7 +390,8 @@ def transpose(self, ys_dot, xs_dot, ys_bar=None, retain_fwd=False): ins = flatten(op_position_inputs(op)) assert len(ins) == len(ins_bar), ( f'len(ins) should be equal to len(ins_bar), ' - f'but len(ins)={len(ins)} and len(ins_bar)={len(ins_bar)}') + f'but len(ins)={len(ins)} and len(ins_bar)={len(ins_bar)}' + ) for dot, bar in zip(ins, ins_bar): if bar is not None: @@ -391,7 +410,8 @@ def transpose(self, ys_dot, xs_dot, ys_bar=None, retain_fwd=False): vars_to_remove = set() for op in path: vars_to_remove.update( - flatten_and_remove_none(get_output_var_list(op))) + flatten_and_remove_none(get_output_var_list(op)) + ) op_indexes = [] @@ -459,10 +479,12 @@ def expand_nested_list(xs): bind(input_args, to_bind, value_table) for orig_out, new_out in zip( - expand_nested_list(get_output_var_list(op)), - expand_nested_list(as_tensors(lower_fn(op, *input_args)))): + expand_nested_list(get_output_var_list(op)), + expand_nested_list(as_tensors(lower_fn(op, *input_args))), + ): assert not (orig_out is None) ^ ( - new_out is None), "orig_out and new_out should match." + new_out is None + ), "orig_out and new_out should match." vars_to_remove.add(new_out.name) value_table[new_out.name] = new_out to_bind[orig_out.name] = new_out.name @@ -471,7 +493,8 @@ def expand_nested_list(xs): inputs = {} for i in range(len(op.input_names)): inputs[op.input_names[i]] = bind_name( - op.input(op.input_names[i]), to_bind) + op.input(op.input_names[i]), to_bind + ) outputs = {} for i in range(len(op.output_names)): @@ -481,14 +504,17 @@ def expand_nested_list(xs): for name in sorted(op.attr_names): attrs[name] = op.attr(name) from paddle.fluid.dygraph.base import param_guard + new_op_desc = block.desc.append_op() with param_guard(inputs), param_guard(outputs): - op = Operator(block=block, - desc=new_op_desc, - type=op.type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + op = Operator( + block=block, + desc=new_op_desc, + type=op.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) block.ops.append(op) # Step3: Do some post-processing work @@ -508,8 +534,9 @@ def expand_nested_list(xs): op._rename_output(out_name, to_bind_rev[out_name]) for var_name in sorted(vars_to_remove): - assert var_name in to_bind_rev, 'var_name "{}" is not in to_bind_rev.'.format( - var_name) + assert ( + var_name in to_bind_rev + ), 'var_name "{}" is not in to_bind_rev.'.format(var_name) if var_name != to_bind_rev[var_name]: block.desc._remove_var(var_name.encode()) del block.vars[var_name] @@ -535,7 +562,8 @@ def orig2prim(block=None): """ block = default_main_program().current_block() if block is None else block - assert block == default_main_program().current_block( + assert ( + block == default_main_program().current_block() ), 'block is neither None nor current block of main program' _lower(block, reverse=False, blacklist=[]) @@ -580,7 +608,8 @@ def prim2orig(block=None, blacklist=None): """ block = default_main_program().current_block() if block is None else block - assert block == default_main_program().current_block( + assert ( + block == default_main_program().current_block() ), 'block is neither None nor current block of main program' blacklist = [] if blacklist is None else blacklist _lower(block, reverse=True, blacklist=blacklist) diff --git a/python/paddle/incubate/autograd/utils.py b/python/paddle/incubate/autograd/utils.py index effb8c1abc646..2b8082bf48de7 100644 --- a/python/paddle/incubate/autograd/utils.py +++ b/python/paddle/incubate/autograd/utils.py @@ -18,7 +18,6 @@ class PrimOption(object): - def __init__(self): self.enable_prim = False @@ -175,7 +174,7 @@ def flatten_and_remove_none(inp): def as_tensors(xs): if isinstance(xs, framework.Variable): - return (xs, ) + return (xs,) elif isinstance(xs, typing.Sequence): return tuple(xs) else: diff --git a/python/paddle/incubate/autotune.py b/python/paddle/incubate/autotune.py index 4c577cba3e70c..daca4aee14e0b 100644 --- a/python/paddle/incubate/autotune.py +++ b/python/paddle/incubate/autotune.py @@ -147,7 +147,8 @@ def set_config(config=None): if "tuning_steps" in dataloader_config: if isinstance(dataloader_config['tuning_steps'], int): paddle.fluid.reader.set_autotune_config( - use_autoune, dataloader_config['tuning_steps']) + use_autoune, dataloader_config['tuning_steps'] + ) else: warnings.warn( "The auto-tuning configuration of the dataloader is incorrect." diff --git a/python/paddle/incubate/distributed/fleet/__init__.py b/python/paddle/incubate/distributed/fleet/__init__.py index 94e1a7c8bbe77..f098e4c951c0e 100644 --- a/python/paddle/incubate/distributed/fleet/__init__.py +++ b/python/paddle/incubate/distributed/fleet/__init__.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle.distributed.fleet.recompute import recompute_sequential, recompute_hybrid +from paddle.distributed.fleet.recompute import ( + recompute_sequential, + recompute_hybrid, +) __all__ = ["recompute_sequential", "recompute_hybrid"] diff --git a/python/paddle/incubate/distributed/models/moe/gate/base_gate.py b/python/paddle/incubate/distributed/models/moe/gate/base_gate.py index 9715f4b2a25a6..f1b4e74c2411f 100644 --- a/python/paddle/incubate/distributed/models/moe/gate/base_gate.py +++ b/python/paddle/incubate/distributed/models/moe/gate/base_gate.py @@ -23,7 +23,6 @@ class BaseGate(nn.Layer): - def __init__(self, num_expert, world_size): super().__init__() self.world_size = world_size diff --git a/python/paddle/incubate/distributed/models/moe/gate/gshard_gate.py b/python/paddle/incubate/distributed/models/moe/gate/gshard_gate.py index 4152dee3a3706..9587dfb346add 100644 --- a/python/paddle/incubate/distributed/models/moe/gate/gshard_gate.py +++ b/python/paddle/incubate/distributed/models/moe/gate/gshard_gate.py @@ -27,15 +27,16 @@ class GShardGate(NaiveGate): - - def __init__(self, - d_model, - num_expert, - world_size, - topk=2, - capacity=(1.2, 2.4), - random_routing=True, - group=None): + def __init__( + self, + d_model, + num_expert, + world_size, + topk=2, + capacity=(1.2, 2.4), + random_routing=True, + group=None, + ): assert topk == 2, "topk should be 2 in gshard" super().__init__(d_model, num_expert, world_size) self.capacity = capacity @@ -43,29 +44,39 @@ def __init__(self, self.group = group def forward(self, x): - topk_val, topk_idx, gate_score = super().forward(x, - return_all_scores=True) + topk_val, topk_idx, gate_score = super().forward( + x, return_all_scores=True + ) s = gate_score.shape[0] top1_idx = topk_idx.flatten() - c_e = paddle.scatter(paddle.zeros(shape=[self.tot_expert]), - top1_idx, - paddle.ones_like(top1_idx, dtype="float32"), - overwrite=False) / s + c_e = ( + paddle.scatter( + paddle.zeros(shape=[self.tot_expert]), + top1_idx, + paddle.ones_like(top1_idx, dtype="float32"), + overwrite=False, + ) + / s + ) m_e = paddle.mean(F.softmax(gate_score, axis=1), axis=0) loss = paddle.mean(c_e * m_e) * (self.num_expert**2) self.set_loss(loss) cap_rate = self.capacity[0 if self.training else 1] capacity = math.ceil(cap_rate * x.shape[0]) - _new_lec, _new_gec, topk_idx = limit_by_capacity(topk_idx, - self.num_expert, - self.world_size, - capacity, - group=self.group) + _new_lec, _new_gec, topk_idx = limit_by_capacity( + topk_idx, + self.num_expert, + self.world_size, + capacity, + group=self.group, + ) if self.random_routing: - rand_routing_prob = paddle.rand(shape=[gate_score.shape[0]], - dtype="float32") + rand_routing_prob = paddle.rand( + shape=[gate_score.shape[0]], dtype="float32" + ) topk_idx = paddle.distributed.models.moe.utils._random_routing( - topk_idx, topk_val, rand_routing_prob) + topk_idx, topk_val, rand_routing_prob + ) return topk_val, topk_idx diff --git a/python/paddle/incubate/distributed/models/moe/gate/naive_gate.py b/python/paddle/incubate/distributed/models/moe/gate/naive_gate.py index 20001377fd1e1..4781f6bba898c 100644 --- a/python/paddle/incubate/distributed/models/moe/gate/naive_gate.py +++ b/python/paddle/incubate/distributed/models/moe/gate/naive_gate.py @@ -26,7 +26,6 @@ class NaiveGate(BaseGate): - def __init__(self, d_model, num_expert, world_size, topk=2): super().__init__(num_expert, world_size) self.gate = nn.Linear(d_model, self.tot_expert) @@ -36,11 +35,9 @@ def __init__(self, d_model, num_expert, world_size, topk=2): def forward(self, inp, return_all_scores=False): gate = self.gate(inp) - gate_top_k_val, gate_top_k_idx = paddle.topk(gate, - k=self.top_k, - axis=-1, - largest=True, - sorted=False) + gate_top_k_val, gate_top_k_idx = paddle.topk( + gate, k=self.top_k, axis=-1, largest=True, sorted=False + ) if return_all_scores: return gate_top_k_val, gate_top_k_idx, gate diff --git a/python/paddle/incubate/distributed/models/moe/gate/switch_gate.py b/python/paddle/incubate/distributed/models/moe/gate/switch_gate.py index 62e32e4db70d9..c9e2f1fca498a 100644 --- a/python/paddle/incubate/distributed/models/moe/gate/switch_gate.py +++ b/python/paddle/incubate/distributed/models/moe/gate/switch_gate.py @@ -27,15 +27,16 @@ class SwitchGate(NaiveGate): - - def __init__(self, - d_model, - num_expert, - world_size, - topk=1, - switch_eps=.1, - capacity=(1.2, 2.4), - group=None): + def __init__( + self, + d_model, + num_expert, + world_size, + topk=1, + switch_eps=0.1, + capacity=(1.2, 2.4), + group=None, + ): assert topk == 1, "topk should be 1 in switch" super().__init__(d_model, num_expert, world_size, topk=1) self.switch_eps = switch_eps @@ -55,19 +56,25 @@ def forward(self, inp): cap_rate = self.capacity[0 if self.training else 1] capacity = math.ceil(cap_rate * inp.shape[0]) - _new_lec, _new_gec, top1_idx = limit_by_capacity(top1_idx, - self.num_expert, - self.world_size, - capacity, - group=self.group) + _new_lec, _new_gec, top1_idx = limit_by_capacity( + top1_idx, + self.num_expert, + self.world_size, + capacity, + group=self.group, + ) valid_idx = top1_idx[top1_idx > -1] valid_idx_tmp = paddle.reshape(valid_idx, shape=[len(valid_idx), 1]) - fraction_expert = paddle.scatter_nd_add( - x=paddle.zeros(shape=[self.tot_expert]), - index=valid_idx_tmp, - updates=paddle.ones_like(valid_idx, dtype=paddle.float32).reshape( - shape=[len(valid_idx)]), - ) / valid_idx.numel() + fraction_expert = ( + paddle.scatter_nd_add( + x=paddle.zeros(shape=[self.tot_expert]), + index=valid_idx_tmp, + updates=paddle.ones_like( + valid_idx, dtype=paddle.float32 + ).reshape(shape=[len(valid_idx)]), + ) + / valid_idx.numel() + ) prob_expert = score.sum(axis=0) / valid_idx.numel() loss = (fraction_expert * prob_expert).sum() * self.tot_expert self.set_loss(loss) diff --git a/python/paddle/incubate/distributed/models/moe/grad_clip.py b/python/paddle/incubate/distributed/models/moe/grad_clip.py index 91de68e4bfa0f..aaeef8780782e 100644 --- a/python/paddle/incubate/distributed/models/moe/grad_clip.py +++ b/python/paddle/incubate/distributed/models/moe/grad_clip.py @@ -80,18 +80,21 @@ class ClipGradForMOEByGlobalNorm(ClipGradBase): sdg.step() """ - def __init__(self, - clip_norm, - is_expert_param_func=None, - moe_group=None, - group_name="default_moe_group"): + def __init__( + self, + clip_norm, + is_expert_param_func=None, + moe_group=None, + group_name="default_moe_group", + ): super(ClipGradForMOEByGlobalNorm, self).__init__() self.clip_norm = float(clip_norm) self.group_name = group_name self.moe_group = moe_group if moe_group is not None and moe_group.nranks > 1: - assert is_expert_param_func is not None, \ - "When moe group size > 1, a function for selecting expert params must be specified." + assert ( + is_expert_param_func is not None + ), "When moe group size > 1, a function for selecting expert params must be specified." self.is_expert_param_func = is_expert_param_func def __str__(self): @@ -120,11 +123,18 @@ def get_l2_norm_pow(params_grads, sum_dtype=None): sum_square_list.append(sum_square) # all parameters have been filterd out - if len(sum_square_list) + len(sum_square_list_fp16) + len( - sum_square_list_fp32) == 0: + if ( + len(sum_square_list) + + len(sum_square_list_fp16) + + len(sum_square_list_fp32) + == 0 + ): return None, None - assert sum_dtype in ["float64", "float32", None], \ - "sum's type must be float64/ float32 / None" + assert sum_dtype in [ + "float64", + "float32", + None, + ], "sum's type must be float64/ float32 / None" if sum_dtype != "float64": sum_dtype = 'float64' if len(sum_square_list) > 0 else "float32" @@ -166,16 +176,20 @@ def _dygraph_clip(self, params_grads): # why to return sum_dtype? # we will call `get_l2_norm_pow` twice and the precisions may be different. # For convenience and simplification, we use sum_dtype directly instead of global_norm_var_normal.dtype - global_norm_var_normal, sum_dtype \ - = self.get_l2_norm_pow(normal_params_grads) + global_norm_var_normal, sum_dtype = self.get_l2_norm_pow( + normal_params_grads + ) global_norm_var_moe = None if len(moe_params_grads) > 0: - global_norm_var_moe, _ \ - = self.get_l2_norm_pow(moe_params_grads, sum_dtype) + global_norm_var_moe, _ = self.get_l2_norm_pow( + moe_params_grads, sum_dtype + ) if global_norm_var_moe is not None: - collective.all_reduce(global_norm_var_moe, - op=collective.ReduceOp.SUM, - group=self.moe_group) + collective.all_reduce( + global_norm_var_moe, + op=collective.ReduceOp.SUM, + group=self.moe_group, + ) if global_norm_var_normal is None and global_norm_var_moe is None: return params_grads @@ -187,19 +201,20 @@ def _dygraph_clip(self, params_grads): if global_norm_var_normal.dtype != global_norm_var_moe.dtype: # compared with normal norm, moe norm is the later one, # so its precision is no lower than normal norm - global_norm_var_normal = \ - global_norm_var_normal.astype(global_norm_var_moe.dtype) + global_norm_var_normal = global_norm_var_normal.astype( + global_norm_var_moe.dtype + ) global_norm_var = global_norm_var_normal + global_norm_var_moe params_and_grads = [] global_norm_var = layers.sqrt(global_norm_var) - max_global_norm = layers.fill_constant(shape=[1], - dtype=global_norm_var.dtype, - value=self.clip_norm) - clip_var = layers.elementwise_div(x=max_global_norm, - y=layers.elementwise_max( - x=global_norm_var, - y=max_global_norm)) + max_global_norm = layers.fill_constant( + shape=[1], dtype=global_norm_var.dtype, value=self.clip_norm + ) + clip_var = layers.elementwise_div( + x=max_global_norm, + y=layers.elementwise_max(x=global_norm_var, y=max_global_norm), + ) for p, g in params_grads: if g is None: continue @@ -207,8 +222,11 @@ def _dygraph_clip(self, params_grads): params_and_grads.append((p, g)) continue # TODO(wangxi): use inplace elementwise_mul - clip_input = (clip_var.astype('float16') - if g.dtype == core.VarDesc.VarType.FP16 else clip_var) + clip_input = ( + clip_var.astype('float16') + if g.dtype == core.VarDesc.VarType.FP16 + else clip_var + ) new_grad = layers.elementwise_mul(x=g, y=clip_input) params_and_grads.append((p, new_grad)) return params_and_grads diff --git a/python/paddle/incubate/distributed/models/moe/moe_layer.py b/python/paddle/incubate/distributed/models/moe/moe_layer.py index b2d11e7a317b9..345a10eed9cee 100644 --- a/python/paddle/incubate/distributed/models/moe/moe_layer.py +++ b/python/paddle/incubate/distributed/models/moe/moe_layer.py @@ -43,11 +43,14 @@ def _local_gather(inp, pos, out_batch_size, maybe_overlap=True): if pos.shape != [0]: origin_dtype = inp.dtype inp = paddle.cast(inp, dtype="float32") - inp_buf = paddle.scatter(paddle.zeros( - shape=[out_batch_size, inp.shape[-1]], dtype="float32"), - pos, - inp, - overwrite=True) + inp_buf = paddle.scatter( + paddle.zeros( + shape=[out_batch_size, inp.shape[-1]], dtype="float32" + ), + pos, + inp, + overwrite=True, + ) inp_buf = paddle.cast(inp_buf, dtype=origin_dtype) else: inp_buf = paddle.zeros([out_batch_size, inp.shape[-1]], dtype=inp.dtype) @@ -59,8 +62,11 @@ def _all_gather(tensor, group=None, use_calc_stream=True): return if in_dygraph_mode(): - group = paddle.distributed.collective._get_default_group( - ) if group is None else group + group = ( + paddle.distributed.collective._get_default_group() + if group is None + else group + ) tensor_shape = list(tensor.shape) tensor_shape[0] *= group.nranks out = paddle.empty(tensor_shape, tensor.dtype) @@ -70,11 +76,20 @@ def _all_gather(tensor, group=None, use_calc_stream=True): return out else: ring_id = 0 if group is None else group.id - nranks = paddle.distributed.collective._get_global_group( - ).nranks if group is None else group.nranks - return paddle._legacy_C_ops.c_allgather(tensor, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id, 'nranks', nranks) + nranks = ( + paddle.distributed.collective._get_global_group().nranks + if group is None + else group.nranks + ) + return paddle._legacy_C_ops.c_allgather( + tensor, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + 'nranks', + nranks, + ) class MoEScatter(PyLayer): @@ -85,20 +100,24 @@ class MoEScatter(PyLayer): """ @staticmethod - def forward(ctx, - inp, - pos, - local_expert_count, - global_expert_count, - fwd_batch_size, - world_size, - group=None): + def forward( + ctx, + inp, + pos, + local_expert_count, + global_expert_count, + fwd_batch_size, + world_size, + group=None, + ): local_input_buf = _local_scatter(inp, pos) if world_size > 1: - global_input_buf = global_scatter(local_input_buf, - local_expert_count, - global_expert_count, - group=group) + global_input_buf = global_scatter( + local_input_buf, + local_expert_count, + global_expert_count, + group=group, + ) else: global_input_buf = local_input_buf @@ -114,10 +133,9 @@ def backward(ctx, grad): (inp_batch_size, world_size, group) = ctx.moe_args if world_size > 1: - local_grad_in = global_gather(grad, - local_expert_count, - global_expert_count, - group=group) + local_grad_in = global_gather( + grad, local_expert_count, global_expert_count, group=group + ) else: local_grad_in = grad grad_in = _local_gather(local_grad_in, pos, inp_batch_size) @@ -131,25 +149,28 @@ class MoEGather(PyLayer): """ @staticmethod - def forward(ctx, + def forward( + ctx, + global_output_buf, + pos, + local_expert_count, + global_expert_count, + local_batch_size, + world_size, + group=None, + ): + if world_size > 1: + local_output_buf = global_gather( global_output_buf, - pos, local_expert_count, global_expert_count, - local_batch_size, - world_size, - group=None): - if world_size > 1: - local_output_buf = global_gather(global_output_buf, - local_expert_count, - global_expert_count, - group=group) + group=group, + ) else: local_output_buf = global_output_buf - output = _local_gather(local_output_buf, - pos, - local_batch_size, - maybe_overlap=False) + output = _local_gather( + local_output_buf, pos, local_batch_size, maybe_overlap=False + ) ctx.moe_args = (global_output_buf.shape[0], world_size, group) variables = (pos, local_expert_count, global_expert_count) @@ -162,10 +183,12 @@ def backward(ctx, grad_out): fwd_batch_size, world_size, group = ctx.moe_args grad_out_buf = _local_scatter(grad_out, pos) if world_size > 1: - global_grad_out_buf = global_scatter(grad_out_buf, - local_expert_count, - global_expert_count, - group=group) + global_grad_out_buf = global_scatter( + grad_out_buf, + local_expert_count, + global_expert_count, + group=group, + ) else: global_grad_out_buf = grad_out_buf return global_grad_out_buf, None, None, None @@ -187,10 +210,9 @@ def forward(ctx, inp, rank, world_size, group): @staticmethod def backward(ctx, grad_out): rank, dim0 = ctx.args - return paddle.slice(grad_out, - axes=[0], - starts=[rank * dim0], - ends=[(rank + 1) * dim0]) + return paddle.slice( + grad_out, axes=[0], starts=[rank * dim0], ends=[(rank + 1) * dim0] + ) class Slice(PyLayer): @@ -204,10 +226,9 @@ def forward(ctx, inp, rank, world_size, group): local_batch_size = B // world_size batch_start = local_batch_size * rank batch_end = min(batch_start + local_batch_size, B) - inp = paddle.slice(inp, - axes=[0], - starts=[batch_start], - ends=[batch_end]) + inp = paddle.slice( + inp, axes=[0], starts=[batch_start], ends=[batch_end] + ) ctx.args = world_size, group return inp @@ -219,10 +240,12 @@ def backward(ctx, grad_out): def prepare_forward(gate, num_expert, world_size, moe_group): pos, local_expert_count, global_expert_count = count_by_gate( - gate, num_expert, world_size, group=moe_group) + gate, num_expert, world_size, group=moe_group + ) with paddle.no_grad(): fwd_expert_count = global_expert_count.reshape_( - [world_size, num_expert]).sum(axis=0) + [world_size, num_expert] + ).sum(axis=0) fwd_batch_size = int(fwd_expert_count.sum().item()) return ( pos, @@ -296,14 +319,16 @@ def forward(self, x): """ - def __init__(self, - d_model, - experts, - gate=None, - moe_group=None, - mp_group=None, - recompute_interval=0, - recompute_ctx=None): + def __init__( + self, + d_model, + experts, + gate=None, + moe_group=None, + mp_group=None, + recompute_interval=0, + recompute_ctx=None, + ): super(MoELayer, self).__init__() self.recompute_ctx = recompute_ctx @@ -311,8 +336,9 @@ def __init__(self, if gate is None: gate = dict() - assert isinstance(gate, (dict, BaseGate)), \ - "gate config' type must be dict or an instance of BaseGate" + assert isinstance( + gate, (dict, BaseGate) + ), "gate config' type must be dict or an instance of BaseGate" # only support mp/dp self.group = moe_group @@ -330,26 +356,36 @@ def __init__(self, self.top_k = gate.get("top_k", 2) gate = gate.get("type", "gshard") if gate == "naive" or gate is None: - gate = NaiveGate(self.d_model, - num_expert=len(experts), - world_size=self.world_size, - topk=self.top_k) + gate = NaiveGate( + self.d_model, + num_expert=len(experts), + world_size=self.world_size, + topk=self.top_k, + ) elif gate == "gshard": - gate = GShardGate(self.d_model, - num_expert=len(experts), - world_size=self.world_size, - topk=self.top_k, - group=self.group) + gate = GShardGate( + self.d_model, + num_expert=len(experts), + world_size=self.world_size, + topk=self.top_k, + group=self.group, + ) elif gate == "switch": - gate = SwitchGate(self.d_model, - num_expert=len(experts), - world_size=self.world_size, - topk=self.top_k, - group=self.group) + gate = SwitchGate( + self.d_model, + num_expert=len(experts), + world_size=self.world_size, + topk=self.top_k, + group=self.group, + ) else: - assert False, "We only support naive gate, \ + assert ( + False + ), "We only support naive gate, \ gshard gate and switch gate, \ - but you choose {} gate.".format(str(gate)) + but you choose {} gate.".format( + str(gate) + ) elif isinstance(gate, NaiveGate): self.top_k = gate.top_k elif isinstance(gate, BaseGate): @@ -391,9 +427,15 @@ def forward(self, inp): temp_pos = pos assert topk == self.top_k - x = MoEScatter.apply(inp, temp_pos, local_expert_count, - global_expert_count, fwd_batch_size, - self.world_size, self.group) + x = MoEScatter.apply( + inp, + temp_pos, + local_expert_count, + global_expert_count, + fwd_batch_size, + self.world_size, + self.group, + ) d_model = self.d_model @@ -408,22 +450,36 @@ def experts_fwd(x, fwd_expert_count, experts): for idx, expert_count in enumerate(fwd_expert_count): if expert_count <= 0: continue - y.append(experts[idx](x[last_index:expert_count + last_index])) + y.append( + experts[idx](x[last_index : expert_count + last_index]) + ) last_index = expert_count + last_index return paddle.concat(y, axis=0) if self.recompute_interval <= 0 or x.shape[0] == 0: x = experts_fwd(x, fwd_expert_count.numpy(), self.experts) else: - x = recompute_hybrid(self.recompute_ctx, experts_fwd, x, - fwd_expert_count.numpy(), self.experts) + x = recompute_hybrid( + self.recompute_ctx, + experts_fwd, + x, + fwd_expert_count.numpy(), + self.experts, + ) out_batch_size = inp.shape[0] if len(gate.shape) == 2: out_batch_size *= gate.shape[1] - x = MoEGather.apply(x, pos, local_expert_count, global_expert_count, - out_batch_size, self.world_size, self.group) + x = MoEGather.apply( + x, + pos, + local_expert_count, + global_expert_count, + out_batch_size, + self.world_size, + self.group, + ) x = x.reshape([-1, self.top_k, d_model]) value = value.reshape([x.shape[0], 1, self.top_k]) diff --git a/python/paddle/incubate/distributed/models/moe/utils.py b/python/paddle/incubate/distributed/models/moe/utils.py index 10203a0cd18aa..65ab86ded6aa9 100644 --- a/python/paddle/incubate/distributed/models/moe/utils.py +++ b/python/paddle/incubate/distributed/models/moe/utils.py @@ -19,7 +19,12 @@ # Copyright 2021, Jiaao He. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"). -from paddle.distributed.models.moe.utils import _number_count, _limit_by_capacity, _prune_gate_by_capacity, _assign_pos +from paddle.distributed.models.moe.utils import ( + _number_count, + _limit_by_capacity, + _prune_gate_by_capacity, + _assign_pos, +) import paddle from paddle.fluid.framework import in_dygraph_mode @@ -29,17 +34,24 @@ def _alltoall(in_tensor_list, group=None, use_calc_stream=True): return if in_dygraph_mode(): - group = paddle.distributed.collective._get_default_group( - ) if group is None else group + group = ( + paddle.distributed.collective._get_default_group() + if group is None + else group + ) out = paddle.empty(in_tensor_list.shape, in_tensor_list.dtype) task = group.process_group.alltoall(in_tensor_list, out) task.wait() return out else: ring_id = 0 if group is None else group.id - return paddle._legacy_C_ops.alltoall(in_tensor_list, 'use_calc_stream', - use_calc_stream, 'ring_id', - ring_id) + return paddle._legacy_C_ops.alltoall( + in_tensor_list, + 'use_calc_stream', + use_calc_stream, + 'ring_id', + ring_id, + ) def count_by_gate(gate, num_expert, world_size, require_pos=True, group=None): @@ -61,13 +73,12 @@ def count_by_gate(gate, num_expert, world_size, require_pos=True, group=None): def limit_by_capacity(topk_idx, num_expert, world_size, capacity, group=None): with paddle.no_grad(): - capacity = paddle.ones(shape=[num_expert], - dtype=paddle.int64) * capacity - pos, lec, gec = count_by_gate(topk_idx, - num_expert, - world_size, - require_pos=False, - group=group) + capacity = ( + paddle.ones(shape=[num_expert], dtype=paddle.int64) * capacity + ) + pos, lec, gec = count_by_gate( + topk_idx, num_expert, world_size, require_pos=False, group=group + ) new_gec = _limit_by_capacity(gec, capacity, world_size) if world_size > 1: assert group.nranks == world_size @@ -75,7 +86,8 @@ def limit_by_capacity(topk_idx, num_expert, world_size, capacity, group=None): else: new_lec = new_gec - topk_idx = _prune_gate_by_capacity(topk_idx, new_lec, num_expert, - world_size) + topk_idx = _prune_gate_by_capacity( + topk_idx, new_lec, num_expert, world_size + ) return new_lec, new_gec, topk_idx diff --git a/python/paddle/incubate/multiprocessing/reductions.py b/python/paddle/incubate/multiprocessing/reductions.py index c54626175bc7d..b16361971edfc 100644 --- a/python/paddle/incubate/multiprocessing/reductions.py +++ b/python/paddle/incubate/multiprocessing/reductions.py @@ -35,16 +35,17 @@ def _supported_check(): return False if not sys.version_info >= (3, 4): - warnings.warn("Use `paddle.multiprocessing` to share paddle tensor " - "requires python version greater than 3.4 ." - " `paddle.multiprocessing` will not take any effect !!!") + warnings.warn( + "Use `paddle.multiprocessing` to share paddle tensor " + "requires python version greater than 3.4 ." + " `paddle.multiprocessing` will not take any effect !!!" + ) return False return True class LRUSharedCache(OrderedDict): - def __init__(self): self.limit = 128 self._after_fork() @@ -84,9 +85,9 @@ def cuda_from_cache(key): def rebuild_tensor(cls, lodtensor, metadata): if cls == paddle.fluid.framework.ParamBase: - tensor = paddle.fluid.framework.ParamBase(lodtensor.shape(), - lodtensor._dtype(), - **metadata) + tensor = paddle.fluid.framework.ParamBase( + lodtensor.shape(), lodtensor._dtype(), **metadata + ) tensor.value().get_tensor()._share_data_with(lodtensor) else: size, stop_gradient = metadata @@ -107,8 +108,11 @@ def reduce_tensor(tensor): "Refusing to serialize non-leaf tensor which not stop_gradient, you can detach it!" ) # TODO: add serializing name and hooks check - if tensor.place.is_cpu_place() or tensor.place.is_gpu_place( - ) or tensor.place.is_cuda_pinned_place(): + if ( + tensor.place.is_cpu_place() + or tensor.place.is_gpu_place() + or tensor.place.is_cuda_pinned_place() + ): if type(tensor) == paddle.fluid.framework.ParamBase: metadata = copy.deepcopy(tensor.__dict__) else: @@ -118,7 +122,8 @@ def reduce_tensor(tensor): else: raise ValueError( "Only support tensors of CPU/CUDA/CUDAPinned Place, Not support %s for now!" - % tensor.place) + % tensor.place + ) def rebuild_lodtensor_filename(cls, ipc_name, size, type_idx, dims, lod): @@ -127,12 +132,14 @@ def rebuild_lodtensor_filename(cls, ipc_name, size, type_idx, dims, lod): return lodtensor -def rebuild_cuda_tensor(cls, handle, offset_bytes, size, type_idx, dims, lod, - device_idx): +def rebuild_cuda_tensor( + cls, handle, offset_bytes, size, type_idx, dims, lod, device_idx +): cache_tensor = cuda_from_cache((handle, offset_bytes)) if cache_tensor is None: lodtensor = cls._new_shared_cuda( - (handle, offset_bytes, size, type_idx, dims, lod, device_idx)) + (handle, offset_bytes, size, type_idx, dims, lod, device_idx) + ) # We only cache cuda shared tensor here. # The opening cost of cudaIpcMemoryHandle is very high. # Since we cache the recived tensor directly, @@ -141,28 +148,32 @@ def rebuild_cuda_tensor(cls, handle, offset_bytes, size, type_idx, dims, lod, shared_cache[(handle, offset_bytes)] = lodtensor else: lodtensor = paddle.fluid.core.LoDTensor() - lodtensor._share_buffer_with(cache_tensor, - (size, type_idx, dims, lod, device_idx)) + lodtensor._share_buffer_with( + cache_tensor, (size, type_idx, dims, lod, device_idx) + ) return lodtensor def rebuild_lodtensor_empty(cls): - #TODO: check if tensor initialized - #TODO: handle the dtype of empty tensor + # TODO: check if tensor initialized + # TODO: handle the dtype of empty tensor return cls() def reduce_lodtensor(lodtensor): - if lodtensor._place().is_cpu_place() or lodtensor._place( - ).is_cuda_pinned_place(): + if ( + lodtensor._place().is_cpu_place() + or lodtensor._place().is_cuda_pinned_place() + ): for dim in lodtensor.shape(): if dim == 0: # Empty tensors have nothing be mmapped. - return (rebuild_lodtensor_empty, (type(lodtensor), )) + return (rebuild_lodtensor_empty, (type(lodtensor),)) # Default use share filename stratege - metadata = lodtensor._share_filename( + metadata = ( + lodtensor._share_filename() ) # ipc_name, size, type_idx, dims, lod rebuild = rebuild_lodtensor_filename lodtensor._shared_incref() @@ -174,7 +185,7 @@ def reduce_lodtensor(lodtensor): else: raise RuntimeError("We only support pass cpu/gpu lodtensor for now!") - return (rebuild, (type(lodtensor), ) + metadata) + return (rebuild, (type(lodtensor),) + metadata) def init_reductions(): diff --git a/python/paddle/incubate/nn/__init__.py b/python/paddle/incubate/nn/__init__.py index cf15ee7d8ffaa..62a48b783df9b 100644 --- a/python/paddle/incubate/nn/__init__.py +++ b/python/paddle/incubate/nn/__init__.py @@ -17,9 +17,11 @@ from .layer.fused_transformer import FusedTransformerEncoderLayer # noqa: F401 from .layer.fused_transformer import FusedMultiTransformer # noqa: F401 from .layer.fused_linear import FusedLinear # noqa: F401 -from .layer.fused_transformer import FusedBiasDropoutResidualLayerNorm # noqa: F401 +from .layer.fused_transformer import ( + FusedBiasDropoutResidualLayerNorm, +) # noqa: F401 -__all__ = [ #noqa +__all__ = [ # noqa 'FusedMultiHeadAttention', 'FusedFeedForward', 'FusedTransformerEncoderLayer', diff --git a/python/paddle/incubate/nn/functional/fused_matmul_bias.py b/python/paddle/incubate/nn/functional/fused_matmul_bias.py index 6c0722fc70610..e7180abfe9b14 100644 --- a/python/paddle/incubate/nn/functional/fused_matmul_bias.py +++ b/python/paddle/incubate/nn/functional/fused_matmul_bias.py @@ -18,12 +18,9 @@ from paddle import _legacy_C_ops -def fused_matmul_bias(x, - y, - bias=None, - transpose_x=False, - transpose_y=False, - name=None): +def fused_matmul_bias( + x, y, bias=None, transpose_x=False, transpose_y=False, name=None +): """ Applies matrix multiplication of two tensors and then bias addition if provided. This method requires CUDA version >= 11.6. @@ -57,23 +54,18 @@ def fused_matmul_bias(x, if bias is None: return matmul(x, y, transpose_x, transpose_y, name) if _non_static_mode(): - return _legacy_C_ops.fused_gemm_epilogue(x, y, bias, 'trans_x', - transpose_x, 'trans_y', - transpose_y) + return _legacy_C_ops.fused_gemm_epilogue( + x, y, bias, 'trans_x', transpose_x, 'trans_y', transpose_y + ) helper = LayerHelper('fused_matmul_bias', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='fused_gemm_epilogue', - inputs={ - 'X': x, - 'Y': y, - 'Bias': bias - }, - outputs={'Out': out}, - attrs={ - 'trans_x': transpose_x, - 'trans_y': transpose_y - }) + helper.append_op( + type='fused_gemm_epilogue', + inputs={'X': x, 'Y': y, 'Bias': bias}, + outputs={'Out': out}, + attrs={'trans_x': transpose_x, 'trans_y': transpose_y}, + ) return out diff --git a/python/paddle/incubate/nn/functional/fused_transformer.py b/python/paddle/incubate/nn/functional/fused_transformer.py index 2eaa7ceadc713..dffddb8b9eca2 100644 --- a/python/paddle/incubate/nn/functional/fused_transformer.py +++ b/python/paddle/incubate/nn/functional/fused_transformer.py @@ -28,26 +28,28 @@ def _verify_dropout_rate(dropout_rate): raise ValueError("dropout_rate argument should between 0 and 1") -def fused_feedforward(x, - linear1_weight, - linear2_weight, - linear1_bias=None, - linear2_bias=None, - ln1_scale=None, - ln1_bias=None, - ln2_scale=None, - ln2_bias=None, - dropout1_rate=0.5, - dropout2_rate=0.5, - activation="relu", - ln1_epsilon=1e-5, - ln2_epsilon=1e-5, - pre_layer_norm=False, - training=True, - mode='upscale_in_train', - ring_id=-1, - add_residual=True, - name=None): +def fused_feedforward( + x, + linear1_weight, + linear2_weight, + linear1_bias=None, + linear2_bias=None, + ln1_scale=None, + ln1_bias=None, + ln2_scale=None, + ln2_bias=None, + dropout1_rate=0.5, + dropout2_rate=0.5, + activation="relu", + ln1_epsilon=1e-5, + ln2_epsilon=1e-5, + pre_layer_norm=False, + training=True, + mode='upscale_in_train', + ring_id=-1, + add_residual=True, + name=None, +): r""" This is a fusion operator to compute feed forward layer in transformer model architecture. This operator only supports running on GPU. The function of the operator is consistent with @@ -126,112 +128,161 @@ def fused_feedforward(x, raise ValueError( "mode argument should be 'downscale_in_infer' or 'upscale_in_train'" ) - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer if _non_static_mode(): if default_main_program().random_seed != 0: seed = default_main_program().random_seed out, _, _, _, _, _, _, _, _, _, _ = _legacy_C_ops.fused_feedforward( - x, None, None, linear1_weight, linear1_bias, linear2_weight, - linear2_bias, ln1_scale, ln1_bias, ln2_scale, ln2_bias, - 'pre_layer_norm', pre_layer_norm, 'ln1_epsilon', ln1_epsilon, - 'ln2_epsilon', ln2_epsilon, 'act_method', activation, - 'dropout1_rate', dropout1_rate, 'dropout2_rate', dropout2_rate, - "is_test", not training, "dropout1_fix_seed", seed is not None, - "dropout2_fix_seed", seed is not None, "dropout1_seed", - seed if seed is not None else 0, "dropout2_seed", - seed if seed is not None else 0, 'dropout1_implementation', mode, - 'dropout2_implementation', mode, 'add_residual', add_residual, - 'ring_id', ring_id) + x, + None, + None, + linear1_weight, + linear1_bias, + linear2_weight, + linear2_bias, + ln1_scale, + ln1_bias, + ln2_scale, + ln2_bias, + 'pre_layer_norm', + pre_layer_norm, + 'ln1_epsilon', + ln1_epsilon, + 'ln2_epsilon', + ln2_epsilon, + 'act_method', + activation, + 'dropout1_rate', + dropout1_rate, + 'dropout2_rate', + dropout2_rate, + "is_test", + not training, + "dropout1_fix_seed", + seed is not None, + "dropout2_fix_seed", + seed is not None, + "dropout1_seed", + seed if seed is not None else 0, + "dropout2_seed", + seed if seed is not None else 0, + 'dropout1_implementation', + mode, + 'dropout2_implementation', + mode, + 'add_residual', + add_residual, + 'ring_id', + ring_id, + ) return out helper = LayerHelper("fused_feedforward") dtype = x.dtype - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'fused_feedforward') - check_dtype(dtype, 'dtype', ['float16', 'float32', 'float64'], - 'fused_feedforward') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'fused_feedforward' + ) + check_dtype( + dtype, 'dtype', ['float16', 'float32', 'float64'], 'fused_feedforward' + ) out = helper.create_variable_for_type_inference(x.dtype) dropout1_mask = helper.create_variable_for_type_inference( - 'uint8', stop_gradient=True) + 'uint8', stop_gradient=True + ) dropout2_mask = helper.create_variable_for_type_inference( - 'uint8', stop_gradient=True) - ln1_mean = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) - ln1_variance = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) - ln2_mean = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) - ln2_variance = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) - linear1_out = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) - ln1_out = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) - dropout1_out = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) - dropout2_out = helper.create_variable_for_type_inference(x.dtype, - stop_gradient=True) + 'uint8', stop_gradient=True + ) + ln1_mean = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) + ln1_variance = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) + ln2_mean = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) + ln2_variance = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) + linear1_out = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) + ln1_out = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) + dropout1_out = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) + dropout2_out = helper.create_variable_for_type_inference( + x.dtype, stop_gradient=True + ) if (seed is None or seed == 0) and helper.main_program.random_seed != 0: seed = helper.main_program.random_seed - helper.append_op(type='fused_feedforward', - inputs={ - 'X': x, - 'Linear1Weight': linear1_weight, - 'Linear1Bias': linear1_bias, - 'Linear2Weight': linear2_weight, - 'Linear2Bias': linear2_bias, - 'Ln1Scale': ln1_scale, - 'Ln1Bias': ln1_bias, - 'Ln2Scale': ln2_scale, - 'Ln2Bias': ln2_bias, - }, - outputs={ - 'Out': out, - 'Dropout1Mask': dropout1_mask, - 'Dropout2Mask': dropout2_mask, - 'Ln1Mean': ln1_mean, - 'Ln1Variance': ln1_variance, - 'Ln2Mean': ln2_mean, - 'Ln2Variance': ln2_variance, - 'Linear1Out': linear1_out, - 'Ln1Out': ln1_out, - 'Dropout1Out': dropout1_out, - 'Dropout2Out': dropout2_out, - }, - attrs={ - 'dropout1_rate': dropout1_rate, - 'dropout2_rate': dropout2_rate, - 'act_method': activation, - 'pre_layer_norm': pre_layer_norm, - 'ln1_epsilon': ln1_epsilon, - 'ln2_epsilon': ln2_epsilon, - 'is_test': not training, - 'dropout1_fix_seed': seed is not None, - 'dropout2_fix_seed': seed is not None, - 'dropout1_seed': seed if seed is not None else 0, - 'dropout2_seed': seed if seed is not None else 0, - 'dropout1_implementation': mode, - 'dropout2_implementation': mode, - 'add_residual': add_residual, - 'ring_id': ring_id, - }) + helper.append_op( + type='fused_feedforward', + inputs={ + 'X': x, + 'Linear1Weight': linear1_weight, + 'Linear1Bias': linear1_bias, + 'Linear2Weight': linear2_weight, + 'Linear2Bias': linear2_bias, + 'Ln1Scale': ln1_scale, + 'Ln1Bias': ln1_bias, + 'Ln2Scale': ln2_scale, + 'Ln2Bias': ln2_bias, + }, + outputs={ + 'Out': out, + 'Dropout1Mask': dropout1_mask, + 'Dropout2Mask': dropout2_mask, + 'Ln1Mean': ln1_mean, + 'Ln1Variance': ln1_variance, + 'Ln2Mean': ln2_mean, + 'Ln2Variance': ln2_variance, + 'Linear1Out': linear1_out, + 'Ln1Out': ln1_out, + 'Dropout1Out': dropout1_out, + 'Dropout2Out': dropout2_out, + }, + attrs={ + 'dropout1_rate': dropout1_rate, + 'dropout2_rate': dropout2_rate, + 'act_method': activation, + 'pre_layer_norm': pre_layer_norm, + 'ln1_epsilon': ln1_epsilon, + 'ln2_epsilon': ln2_epsilon, + 'is_test': not training, + 'dropout1_fix_seed': seed is not None, + 'dropout2_fix_seed': seed is not None, + 'dropout1_seed': seed if seed is not None else 0, + 'dropout2_seed': seed if seed is not None else 0, + 'dropout1_implementation': mode, + 'dropout2_implementation': mode, + 'add_residual': add_residual, + 'ring_id': ring_id, + }, + ) return out -def fused_bias_dropout_residual_layer_norm(x, - residual, - bias=None, - ln_scale=None, - ln_bias=None, - dropout_rate=0.5, - ln_epsilon=1e-5, - training=True, - mode='upscale_in_train', - name=None): +def fused_bias_dropout_residual_layer_norm( + x, + residual, + bias=None, + ln_scale=None, + ln_bias=None, + dropout_rate=0.5, + ln_epsilon=1e-5, + training=True, + mode='upscale_in_train', + name=None, +): r""" The fused_bias_dropout_residual_layer_norm operator. The pseudo code is as follows: @@ -291,37 +342,72 @@ def fused_bias_dropout_residual_layer_norm(x, raise ValueError( "mode argument should be 'downscale_in_infer' or 'upscale_in_train'" ) - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer if ln_scale is not None: - assert len(ln_scale.shape - ) == 1, "The dims of the shape of ln_scale should be 1." - assert x.shape[len(x.shape) - 1] == ln_scale.shape[ - 0], "The dim of ln_scale must equal to the last dim of x." + assert ( + len(ln_scale.shape) == 1 + ), "The dims of the shape of ln_scale should be 1." + assert ( + x.shape[len(x.shape) - 1] == ln_scale.shape[0] + ), "The dim of ln_scale must equal to the last dim of x." if ln_bias is not None: - assert len( - ln_bias.shape) == 1, "The dims of the shape of ln_bias should be 1." - assert x.shape[len(x.shape) - 1] == ln_bias.shape[ - 0], "The dim of ln_bias must equal to the last dim of x." + assert ( + len(ln_bias.shape) == 1 + ), "The dims of the shape of ln_bias should be 1." + assert ( + x.shape[len(x.shape) - 1] == ln_bias.shape[0] + ), "The dim of ln_bias must equal to the last dim of x." if _non_static_mode(): if default_main_program().random_seed != 0: seed = default_main_program().random_seed - _, _, _, _, final_out = _legacy_C_ops.fused_bias_dropout_residual_layer_norm( - x, residual, bias, ln_scale, ln_bias, 'dropout_rate', dropout_rate, - 'ln_epsilon', ln_epsilon, 'is_test', not training, - 'dropout_fix_seed', seed is not None, 'dropout_seed', - seed if seed is not None else 0, 'dropout_implementation', mode) + ( + _, + _, + _, + _, + final_out, + ) = _legacy_C_ops.fused_bias_dropout_residual_layer_norm( + x, + residual, + bias, + ln_scale, + ln_bias, + 'dropout_rate', + dropout_rate, + 'ln_epsilon', + ln_epsilon, + 'is_test', + not training, + 'dropout_fix_seed', + seed is not None, + 'dropout_seed', + seed if seed is not None else 0, + 'dropout_implementation', + mode, + ) return final_out else: - helper = LayerHelper('fused_bias_dropout_residual_layer_norm', - **locals()) + helper = LayerHelper( + 'fused_bias_dropout_residual_layer_norm', **locals() + ) dtype = x.dtype # check dtypes - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'fused_bias_dropout_residual_layer_norm') - check_dtype(dtype, 'dtype', ['float16', 'float32', 'float64'], - 'fused_bias_dropout_residual_layer_norm') + check_variable_and_dtype( + x, + 'x', + ['float16', 'float32', 'float64'], + 'fused_bias_dropout_residual_layer_norm', + ) + check_dtype( + dtype, + 'dtype', + ['float16', 'float32', 'float64'], + 'fused_bias_dropout_residual_layer_norm', + ) # set inputs inputs = dict() inputs['X'] = [x] @@ -345,50 +431,57 @@ def fused_bias_dropout_residual_layer_norm(x, } # set outputs dropout_mask_out = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) ln_mean_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) ln_variance_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) bias_dropout_residual_out = helper.create_variable_for_type_inference( - dtype=dtype) + dtype=dtype + ) final_out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='fused_bias_dropout_residual_layer_norm', - inputs=inputs, - outputs={ - "BiasDropoutResidualOut": - bias_dropout_residual_out, - "DropoutMaskOut": dropout_mask_out, - "LnMean": ln_mean_out, - "LnVariance": ln_variance_out, - 'Y': final_out, - }, - attrs=attrs) + helper.append_op( + type='fused_bias_dropout_residual_layer_norm', + inputs=inputs, + outputs={ + "BiasDropoutResidualOut": bias_dropout_residual_out, + "DropoutMaskOut": dropout_mask_out, + "LnMean": ln_mean_out, + "LnVariance": ln_variance_out, + 'Y': final_out, + }, + attrs=attrs, + ) return final_out -def fused_multi_head_attention(x, - qkv_weight, - linear_weight, - pre_layer_norm=False, - pre_ln_scale=None, - pre_ln_bias=None, - ln_scale=None, - ln_bias=None, - pre_ln_epsilon=1e-05, - qkv_bias=None, - linear_bias=None, - cache_kv=None, - attn_mask=None, - dropout_rate=0.5, - attn_dropout_rate=0.5, - ln_epsilon=1e-05, - training=True, - mode='upscale_in_train', - ring_id=-1, - add_residual=True, - name=None): +def fused_multi_head_attention( + x, + qkv_weight, + linear_weight, + pre_layer_norm=False, + pre_ln_scale=None, + pre_ln_bias=None, + ln_scale=None, + ln_bias=None, + pre_ln_epsilon=1e-05, + qkv_bias=None, + linear_bias=None, + cache_kv=None, + attn_mask=None, + dropout_rate=0.5, + attn_dropout_rate=0.5, + ln_epsilon=1e-05, + training=True, + mode='upscale_in_train', + ring_id=-1, + add_residual=True, + name=None, +): r""" Attention mapps queries and a set of key-value pairs to outputs, and Multi-Head Attention performs multiple parallel attention to jointly attending @@ -512,7 +605,9 @@ def fused_multi_head_attention(x, raise ValueError( "mode argument should be 'downscale_in_infer' or 'upscale_in_train'" ) - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer if _non_static_mode(): if default_main_program().random_seed != 0: @@ -520,29 +615,83 @@ def fused_multi_head_attention(x, # pre_ln_mean, pre_ln_variance, pre_ln_out, qkv_out, qkv_bias_out, transpose_out, qk_out, # qktv_out, softmax_out, attn_dropout_mask_out, attn_dropout_out, attn_mask_out, fmha_out, # linear_out, dropout_mask_out, ln_mean_out, ln_var_out, bias_dropout_residual_out, final_out - assert len(qkv_weight.shape - ) == 4, "The dims of the shape of qkv_weight should be 4." - assert qkv_weight.shape[ - 0] == 3, "The shape of qkv_weight should be [3, num_head, head_dim, embed_dim]." - assert qkv_weight.shape[3] == x.shape[ - 2], "The 3rd dim of qkv_weight and 2nd dim of x should be the same, i.e., embed_dim." + assert ( + len(qkv_weight.shape) == 4 + ), "The dims of the shape of qkv_weight should be 4." + assert ( + qkv_weight.shape[0] == 3 + ), "The shape of qkv_weight should be [3, num_head, head_dim, embed_dim]." + assert ( + qkv_weight.shape[3] == x.shape[2] + ), "The 3rd dim of qkv_weight and 2nd dim of x should be the same, i.e., embed_dim." if ring_id == -1: # under mp, the num head will be split, this equation will not hold - assert qkv_weight.shape[1] * qkv_weight.shape[2] == qkv_weight.shape[ - 3], "embed_dim must be divisible by num_heads." - - _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, cache_kv_out, final_out = _legacy_C_ops.fused_attention( - x, pre_ln_scale, pre_ln_bias, qkv_weight, qkv_bias, cache_kv, - attn_mask, linear_weight, linear_bias, ln_scale, ln_bias, - 'pre_layer_norm', pre_layer_norm, 'epsilon', pre_ln_epsilon, - 'dropout_rate', dropout_rate, 'attn_dropout_rate', - attn_dropout_rate, 'ln_epsilon', ln_epsilon, 'is_test', - not training, 'attn_dropout_fix_seed', seed is not None, - 'dropout_fix_seed', seed is not None, 'attn_dropout_seed', - seed if seed is not None else 0, 'dropout_seed', - seed if seed is not None else 0, 'attn_dropout_implementation', - mode, 'dropout_implementation', mode, 'add_residual', add_residual, - 'ring_id', ring_id) + assert ( + qkv_weight.shape[1] * qkv_weight.shape[2] == qkv_weight.shape[3] + ), "embed_dim must be divisible by num_heads." + + ( + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + cache_kv_out, + final_out, + ) = _legacy_C_ops.fused_attention( + x, + pre_ln_scale, + pre_ln_bias, + qkv_weight, + qkv_bias, + cache_kv, + attn_mask, + linear_weight, + linear_bias, + ln_scale, + ln_bias, + 'pre_layer_norm', + pre_layer_norm, + 'epsilon', + pre_ln_epsilon, + 'dropout_rate', + dropout_rate, + 'attn_dropout_rate', + attn_dropout_rate, + 'ln_epsilon', + ln_epsilon, + 'is_test', + not training, + 'attn_dropout_fix_seed', + seed is not None, + 'dropout_fix_seed', + seed is not None, + 'attn_dropout_seed', + seed if seed is not None else 0, + 'dropout_seed', + seed if seed is not None else 0, + 'attn_dropout_implementation', + mode, + 'dropout_implementation', + mode, + 'add_residual', + add_residual, + 'ring_id', + ring_id, + ) if cache_kv is not None: return final_out, cache_kv_out return final_out @@ -550,10 +699,18 @@ def fused_multi_head_attention(x, helper = LayerHelper('fused_multi_head_attention', **locals()) dtype = x.dtype # check dtypes - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'fused_multihead_attention') - check_dtype(dtype, 'dtype', ['float16', 'float32', 'float64'], - 'fused_multi_head_attention') + check_variable_and_dtype( + x, + 'x', + ['float16', 'float32', 'float64'], + 'fused_multihead_attention', + ) + check_dtype( + dtype, + 'dtype', + ['float16', 'float32', 'float64'], + 'fused_multi_head_attention', + ) # set inputs inputs = dict() @@ -573,7 +730,8 @@ def fused_multi_head_attention(x, inputs['Ln2Scale'] = [ln_scale] if ln_bias: inputs['Ln2Bias'] = [ln_bias] - if cache_kv: inputs['CacheKV'] = [cache_kv] + if cache_kv: + inputs['CacheKV'] = [cache_kv] if (seed is None or seed == 0) and helper.main_program.random_seed != 0: seed = helper.main_program.random_seed @@ -593,14 +751,16 @@ def fused_multi_head_attention(x, 'attn_dropout_implementation': mode, 'dropout_implementation': mode, 'add_residual': add_residual, - 'ring_id': ring_id + 'ring_id': ring_id, } # set outputs pre_ln_mean_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) pre_ln_variance_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) pre_ln_out = helper.create_variable_for_type_inference(dtype=dtype) qkv_out = helper.create_variable_for_type_inference(dtype=dtype) @@ -611,79 +771,88 @@ def fused_multi_head_attention(x, qktv_out = helper.create_variable_for_type_inference(dtype=dtype) softmax_out = helper.create_variable_for_type_inference(dtype=dtype) attn_dropout_mask_out = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) attn_dropout_out = helper.create_variable_for_type_inference( - dtype=dtype) + dtype=dtype + ) attn_mask_out = helper.create_variable_for_type_inference(dtype=dtype) fmha_out = helper.create_variable_for_type_inference(dtype=dtype) out_linear_out = helper.create_variable_for_type_inference(dtype=dtype) dropout_mask_out = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) ln_mean_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) ln_variance_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) bias_dropout_residual_out = helper.create_variable_for_type_inference( - dtype=dtype) + dtype=dtype + ) final_out = helper.create_variable_for_type_inference(dtype=dtype) cache_kv_out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='fused_attention', - inputs=inputs, - outputs={ - "LnMean": pre_ln_mean_out, - "LnVariance": pre_ln_variance_out, - "LnOut": pre_ln_out, - "QKVOut": qkv_out, - "QKVBiasOut": qkv_bias_out, - "TransposeOut2": transpose_out, - "QKOut": qk_out, - "QKTVOut": qktv_out, - "SoftmaxOut": softmax_out, - "AttnDropoutMaskOut": attn_dropout_mask_out, - "AttnDropoutOut": attn_dropout_out, - "SrcMaskOut": attn_mask_out, - "FMHAOut": fmha_out, - "OutLinearOut": out_linear_out, - "DropoutMaskOut": dropout_mask_out, - "Ln2Mean": ln_mean_out, - "Ln2Variance": ln_variance_out, - "BiasDropoutResidualOut": - bias_dropout_residual_out, - 'Y': final_out, - 'CacheKVOut': cache_kv_out - }, - attrs=attrs) + helper.append_op( + type='fused_attention', + inputs=inputs, + outputs={ + "LnMean": pre_ln_mean_out, + "LnVariance": pre_ln_variance_out, + "LnOut": pre_ln_out, + "QKVOut": qkv_out, + "QKVBiasOut": qkv_bias_out, + "TransposeOut2": transpose_out, + "QKOut": qk_out, + "QKTVOut": qktv_out, + "SoftmaxOut": softmax_out, + "AttnDropoutMaskOut": attn_dropout_mask_out, + "AttnDropoutOut": attn_dropout_out, + "SrcMaskOut": attn_mask_out, + "FMHAOut": fmha_out, + "OutLinearOut": out_linear_out, + "DropoutMaskOut": dropout_mask_out, + "Ln2Mean": ln_mean_out, + "Ln2Variance": ln_variance_out, + "BiasDropoutResidualOut": bias_dropout_residual_out, + 'Y': final_out, + 'CacheKVOut': cache_kv_out, + }, + attrs=attrs, + ) return (final_out, cache_kv_out) if cache_kv else final_out -def fused_multi_transformer(x, - ln_scales, - ln_biases, - qkv_weights, - qkv_biases, - linear_weights, - linear_biases, - ffn_ln_scales, - ffn_ln_biases, - ffn1_weights, - ffn1_biases, - ffn2_weights, - ffn2_biases, - pre_layer_norm=True, - epsilon=1e-05, - cache_kvs=None, - pre_caches=None, - time_step=None, - attn_mask=None, - dropout_rate=0.0, - activation="gelu", - training=False, - mode='upscale_in_train', - trans_qkvw=True, - ring_id=-1, - name=None): +def fused_multi_transformer( + x, + ln_scales, + ln_biases, + qkv_weights, + qkv_biases, + linear_weights, + linear_biases, + ffn_ln_scales, + ffn_ln_biases, + ffn1_weights, + ffn1_biases, + ffn2_weights, + ffn2_biases, + pre_layer_norm=True, + epsilon=1e-05, + cache_kvs=None, + pre_caches=None, + time_step=None, + attn_mask=None, + dropout_rate=0.0, + activation="gelu", + training=False, + mode='upscale_in_train', + trans_qkvw=True, + ring_id=-1, + name=None, +): r""" This is a fusion operator to compute multi transformer layers in transformer model architecture. This operator only supports running on GPU. The function of the transformer layer is consistent @@ -823,18 +992,47 @@ def fused_multi_transformer(x, raise ValueError( "mode argument should be 'downscale_in_infer' or 'upscale_in_train'" ) - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer if _non_static_mode(): cache_kv_out, final_out = _legacy_C_ops.fused_multi_transformer( - x, ln_scales, ln_biases, qkv_weights, qkv_biases, cache_kvs, - pre_caches, time_step, attn_mask, linear_weights, linear_biases, - ffn_ln_scales, ffn_ln_biases, ffn1_weights, ffn1_biases, - ffn2_weights, ffn2_biases, cache_kvs, 'pre_layer_norm', - pre_layer_norm, 'epsilon', epsilon, 'dropout_rate', dropout_rate, - 'is_test', not training, 'dropout_implementation', mode, - 'act_method', activation, 'trans_qkvw', trans_qkvw, 'ring_id', - ring_id) + x, + ln_scales, + ln_biases, + qkv_weights, + qkv_biases, + cache_kvs, + pre_caches, + time_step, + attn_mask, + linear_weights, + linear_biases, + ffn_ln_scales, + ffn_ln_biases, + ffn1_weights, + ffn1_biases, + ffn2_weights, + ffn2_biases, + cache_kvs, + 'pre_layer_norm', + pre_layer_norm, + 'epsilon', + epsilon, + 'dropout_rate', + dropout_rate, + 'is_test', + not training, + 'dropout_implementation', + mode, + 'act_method', + activation, + 'trans_qkvw', + trans_qkvw, + 'ring_id', + ring_id, + ) if cache_kvs is not None: return final_out, cache_kv_out return final_out @@ -842,10 +1040,12 @@ def fused_multi_transformer(x, helper = LayerHelper('fused_multi_transformer', **locals()) dtype = x.dtype # check dtypes - check_variable_and_dtype(x, 'x', ['float16', 'float32'], - 'fused_multi_transformer') - check_dtype(dtype, 'dtype', ['float16', 'float32'], - 'fused_multi_transformer') + check_variable_and_dtype( + x, 'x', ['float16', 'float32'], 'fused_multi_transformer' + ) + check_dtype( + dtype, 'dtype', ['float16', 'float32'], 'fused_multi_transformer' + ) # set inputs inputs = dict() @@ -885,7 +1085,7 @@ def fused_multi_transformer(x, 'dropout_implementation': mode, 'act_method': activation, 'trans_qkvw': trans_qkvw, - 'ring_id': ring_id + 'ring_id': ring_id, } outputs = dict() @@ -895,9 +1095,11 @@ def fused_multi_transformer(x, # NOTE: inplace outputs['CacheKVOut'] = cache_kvs - helper.append_op(type='fused_multi_transformer', - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='fused_multi_transformer', + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) return (final_out, cache_kvs) if cache_kvs else final_out diff --git a/python/paddle/incubate/nn/layer/fused_linear.py b/python/paddle/incubate/nn/layer/fused_linear.py index 4a5f2c12105b2..70a18233f78a6 100644 --- a/python/paddle/incubate/nn/layer/fused_linear.py +++ b/python/paddle/incubate/nn/layer/fused_linear.py @@ -66,30 +66,31 @@ class FusedLinear(Layer): print(y.shape) # [3, 5] """ - def __init__(self, - in_features, - out_features, - weight_attr=None, - bias_attr=None, - transpose_weight=False, - name=None): + def __init__( + self, + in_features, + out_features, + weight_attr=None, + bias_attr=None, + transpose_weight=False, + name=None, + ): super(FusedLinear, self).__init__() if transpose_weight: weight_shape = [out_features, in_features] else: weight_shape = [in_features, out_features] dtype = self._helper.get_default_dtype() - self.weight = self.create_parameter(shape=weight_shape, - attr=weight_attr, - dtype=dtype, - is_bias=False) - self.bias = self.create_parameter(shape=[out_features], - attr=bias_attr, - dtype=dtype, - is_bias=True) + self.weight = self.create_parameter( + shape=weight_shape, attr=weight_attr, dtype=dtype, is_bias=False + ) + self.bias = self.create_parameter( + shape=[out_features], attr=bias_attr, dtype=dtype, is_bias=True + ) self.transpose_weight = transpose_weight self.name = name def forward(self, input): - return F.fused_linear(input, self.weight, self.bias, - self.transpose_weight, self.name) + return F.fused_linear( + input, self.weight, self.bias, self.transpose_weight, self.name + ) diff --git a/python/paddle/incubate/nn/layer/fused_transformer.py b/python/paddle/incubate/nn/layer/fused_transformer.py index ca58f81e19dd5..0f9b225eb6dee 100644 --- a/python/paddle/incubate/nn/layer/fused_transformer.py +++ b/python/paddle/incubate/nn/layer/fused_transformer.py @@ -14,7 +14,10 @@ from paddle.incubate.nn import functional as incubate_f from paddle.nn import Layer import paddle -from paddle.nn.layer.transformer import _convert_attention_mask, _convert_param_attr_to_list +from paddle.nn.layer.transformer import ( + _convert_attention_mask, + _convert_param_attr_to_list, +) from paddle.nn.initializer import Constant from paddle.fluid.dygraph import no_grad from paddle.fluid.framework import convert_np_dtype_to_dtype_, _non_static_mode @@ -49,7 +52,8 @@ def _to_dtype(t, dtype): if t.place.is_gpu_place(): size_dtype = core.size_of_dtype(dtype) waiting_alloc_memory = ( - (np.prod(t.shape) * size_dtype) / 256 + 1) * 256 * 1.2 + ((np.prod(t.shape) * size_dtype) / 256 + 1) * 256 * 1.2 + ) gpu_memory_available = core.gpu_memory_available() if gpu_memory_available < waiting_alloc_memory: t_used = t._copy_to(paddle.CPUPlace(), False) @@ -104,31 +108,38 @@ class FusedBiasDropoutResidualLayerNorm(Layer): output = fused_bias_dropout_residual_ln(x, residual) # [2, 4, 128] """ - def __init__(self, - embed_dim, - dropout_rate=0.5, - weight_attr=None, - bias_attr=None, - epsilon=1e-5, - name=None): + def __init__( + self, + embed_dim, + dropout_rate=0.5, + weight_attr=None, + bias_attr=None, + epsilon=1e-5, + name=None, + ): super(FusedBiasDropoutResidualLayerNorm, self).__init__() - assert embed_dim > 0, ("Expected embed_dim to be greater than 0, " - "but recieved {}".format(embed_dim)) + assert embed_dim > 0, ( + "Expected embed_dim to be greater than 0, " + "but recieved {}".format(embed_dim) + ) self._dtype = self._helper.get_default_dtype() self._bias_attr = bias_attr self._weight_attr = weight_attr self.embed_dim = embed_dim - self.linear_bias = self.create_parameter(shape=[embed_dim], - attr=self._bias_attr, - dtype=self._dtype, - is_bias=True) + self.linear_bias = self.create_parameter( + shape=[embed_dim], + attr=self._bias_attr, + dtype=self._dtype, + is_bias=True, + ) self.ln_scale = self.create_parameter( attr=self._weight_attr, shape=[embed_dim], - default_initializer=Constant(value=1.0)) - self.ln_bias = self.create_parameter(attr=self._bias_attr, - shape=[embed_dim], - is_bias=True) + default_initializer=Constant(value=1.0), + ) + self.ln_bias = self.create_parameter( + attr=self._bias_attr, shape=[embed_dim], is_bias=True + ) self.dropout_rate = dropout_rate self._epsilon = epsilon @@ -161,14 +172,20 @@ def forward(self, x, residual): ln_epsilon=self._epsilon, training=self.training, mode='upscale_in_train', - name=self.name) + name=self.name, + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' return 'embed_dim={}, seq_len={}, dropout_rate={}, epsilon={}, dtype={}{}'.format( - self.embed_dim, self.seq_len, self.dropout_rate, self._epsilon, - self._dtype, name_str) + self.embed_dim, + self.seq_len, + self.dropout_rate, + self._epsilon, + self._dtype, + name_str, + ) class FusedMultiHeadAttention(Layer): @@ -244,33 +261,40 @@ class FusedMultiHeadAttention(Layer): output = multi_head_attn(query, None, None, attn_mask=attn_mask) # [2, 4, 128] """ - def __init__(self, - embed_dim, - num_heads, - dropout_rate=0.5, - attn_dropout_rate=0.5, - kdim=None, - vdim=None, - normalize_before=False, - need_weights=False, - qkv_weight_attr=None, - qkv_bias_attr=None, - linear_weight_attr=None, - linear_bias_attr=None, - pre_ln_scale_attr=None, - pre_ln_bias_attr=None, - ln_scale_attr=None, - ln_bias_attr=None, - epsilon=1e-5, - nranks=1, - ring_id=-1, - name=None): + def __init__( + self, + embed_dim, + num_heads, + dropout_rate=0.5, + attn_dropout_rate=0.5, + kdim=None, + vdim=None, + normalize_before=False, + need_weights=False, + qkv_weight_attr=None, + qkv_bias_attr=None, + linear_weight_attr=None, + linear_bias_attr=None, + pre_ln_scale_attr=None, + pre_ln_bias_attr=None, + ln_scale_attr=None, + ln_bias_attr=None, + epsilon=1e-5, + nranks=1, + ring_id=-1, + name=None, + ): super(FusedMultiHeadAttention, self).__init__() - assert embed_dim > 0, ("Expected embed_dim to be greater than 0, " - "but received {}".format(embed_dim)) - assert num_heads > 0, ("Expected nhead to be greater than 0, " - "but received {}".format(num_heads)) + assert embed_dim > 0, ( + "Expected embed_dim to be greater than 0, " + "but received {}".format(embed_dim) + ) + assert ( + num_heads > 0 + ), "Expected nhead to be greater than 0, " "but received {}".format( + num_heads + ) self.normalize_before = normalize_before self._dtype = self._helper.get_default_dtype() @@ -283,7 +307,9 @@ def __init__(self, self.kdim = kdim self.vdim = vdim self.need_weights = need_weights - assert self.head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * num_heads == embed_dim + ), "embed_dim must be divisible by num_heads" assert need_weights is False, "Only support need_weight is False now." # tensor model parallel @@ -294,21 +320,26 @@ def __init__(self, shape=[3, num_heads, self.head_dim, embed_dim], attr=qkv_weight_attr, dtype=self._dtype, - is_bias=False) + is_bias=False, + ) self.qkv_bias = self.create_parameter( shape=[3, num_heads, self.head_dim], attr=qkv_bias_attr, dtype=self._dtype, - is_bias=True) + is_bias=True, + ) self.linear_weight = self.create_parameter( shape=[num_heads * self.head_dim, embed_dim], attr=linear_weight_attr, dtype=self._dtype, - is_bias=False) - self.linear_bias = self.create_parameter(shape=[embed_dim], - attr=linear_bias_attr, - dtype=self._dtype, - is_bias=True) + is_bias=False, + ) + self.linear_bias = self.create_parameter( + shape=[embed_dim], + attr=linear_bias_attr, + dtype=self._dtype, + is_bias=True, + ) # tensor model parallel if nranks > 1: @@ -323,10 +354,11 @@ def __init__(self, self.pre_ln_scale = self.create_parameter( attr=pre_ln_scale_attr, shape=[embed_dim], - default_initializer=Constant(value=1.0)) - self.pre_ln_bias = self.create_parameter(attr=pre_ln_bias_attr, - shape=[embed_dim], - is_bias=True) + default_initializer=Constant(value=1.0), + ) + self.pre_ln_bias = self.create_parameter( + attr=pre_ln_bias_attr, shape=[embed_dim], is_bias=True + ) self.ln_scale = None self.ln_bias = None else: @@ -335,10 +367,11 @@ def __init__(self, self.ln_scale = self.create_parameter( attr=ln_scale_attr, shape=[embed_dim], - default_initializer=Constant(value=1.0)) - self.ln_bias = self.create_parameter(attr=ln_bias_attr, - shape=[embed_dim], - is_bias=True) + default_initializer=Constant(value=1.0), + ) + self.ln_bias = self.create_parameter( + attr=ln_bias_attr, shape=[embed_dim], is_bias=True + ) self.dropout_rate = dropout_rate self.attn_dropout_rate = attn_dropout_rate @@ -402,15 +435,25 @@ def forward(self, query, key=None, value=None, attn_mask=None, cache=None): ln_epsilon=self._epsilon, training=self.training, ring_id=self._ring_id, - name=self.name) + name=self.name, + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' return 'embed_dim={}, num_heads={}, dropout_rate={}, attn_dropout_rate={}, epsilon={}, kdim={}, vdim={}, normalize_before={}, need_weights={}, dtype={}{}'.format( - self.embed_dim, self.num_heads, self.dropout_rate, - self.attn_dropout_rate, self._epsilon, self.kdim, self.vdim, - self.normalize_before, self.need_weights, self._dtype, name_str) + self.embed_dim, + self.num_heads, + self.dropout_rate, + self.attn_dropout_rate, + self._epsilon, + self.kdim, + self.vdim, + self.normalize_before, + self.need_weights, + self._dtype, + name_str, + ) def _amp_decorate(self, dtype): # tmp fix for amp.decorator(O2) @@ -493,33 +536,39 @@ class FusedFeedForward(Layer): # (1, 8, 8) """ - def __init__(self, - d_model, - dim_feedforward, - dropout_rate=0.1, - epsilon=1e-05, - activation="relu", - act_dropout_rate=None, - normalize_before=False, - linear1_weight_attr=None, - linear1_bias_attr=None, - linear2_weight_attr=None, - linear2_bias_attr=None, - ln1_scale_attr=None, - ln1_bias_attr=None, - ln2_scale_attr=None, - ln2_bias_attr=None, - nranks=1, - ring_id=-1, - name=None): + def __init__( + self, + d_model, + dim_feedforward, + dropout_rate=0.1, + epsilon=1e-05, + activation="relu", + act_dropout_rate=None, + normalize_before=False, + linear1_weight_attr=None, + linear1_bias_attr=None, + linear2_weight_attr=None, + linear2_bias_attr=None, + ln1_scale_attr=None, + ln1_bias_attr=None, + ln2_scale_attr=None, + ln2_bias_attr=None, + nranks=1, + ring_id=-1, + name=None, + ): super(FusedFeedForward, self).__init__() - assert d_model > 0, ( - "Expected d_model to be greater than 0, but received {}".format( - d_model)) - assert dim_feedforward > 0, ( - "Expected dim_feedforward to be greater than 0, but received {}". - format(dim_feedforward)) + assert ( + d_model > 0 + ), "Expected d_model to be greater than 0, but received {}".format( + d_model + ) + assert ( + dim_feedforward > 0 + ), "Expected dim_feedforward to be greater than 0, but received {}".format( + dim_feedforward + ) self._dtype = self._helper.get_default_dtype() self._d_model = d_model @@ -528,7 +577,9 @@ def __init__(self, dim_feedforward = dim_feedforward // nranks self._dim_feedforward = dim_feedforward self._dropout_rate = dropout_rate - self._act_dropout_rate = dropout_rate if act_dropout_rate is None else act_dropout_rate + self._act_dropout_rate = ( + dropout_rate if act_dropout_rate is None else act_dropout_rate + ) self._act_method = activation self._normalize_before = normalize_before self._epsilon = epsilon @@ -538,22 +589,28 @@ def __init__(self, shape=[d_model, dim_feedforward], attr=linear1_weight_attr, dtype=self._dtype, - is_bias=False) - self._linear1_bias = self.create_parameter(shape=[dim_feedforward], - attr=linear1_bias_attr, - dtype=self._dtype, - is_bias=True) + is_bias=False, + ) + self._linear1_bias = self.create_parameter( + shape=[dim_feedforward], + attr=linear1_bias_attr, + dtype=self._dtype, + is_bias=True, + ) self._linear2_weight = self.create_parameter( shape=[dim_feedforward, d_model], attr=linear2_weight_attr, dtype=self._dtype, - is_bias=False) + is_bias=False, + ) - self._linear2_bias = self.create_parameter(shape=[d_model], - attr=linear2_bias_attr, - dtype=self._dtype, - is_bias=True) + self._linear2_bias = self.create_parameter( + shape=[d_model], + attr=linear2_bias_attr, + dtype=self._dtype, + is_bias=True, + ) if nranks > 1: assert ring_id != -1 @@ -567,10 +624,11 @@ def __init__(self, shape=[d_model], attr=ln1_scale_attr, is_bias=False, - default_initializer=Constant(1.0)) - self._ln1_bias = self.create_parameter(shape=[d_model], - attr=ln1_bias_attr, - is_bias=True) + default_initializer=Constant(1.0), + ) + self._ln1_bias = self.create_parameter( + shape=[d_model], attr=ln1_bias_attr, is_bias=True + ) self._ln2_scale = None self._ln2_bias = None else: @@ -580,10 +638,11 @@ def __init__(self, shape=[d_model], attr=ln2_scale_attr, is_bias=False, - default_initializer=Constant(1.0)) - self._ln2_bias = self.create_parameter(shape=[d_model], - attr=ln2_bias_attr, - is_bias=True) + default_initializer=Constant(1.0), + ) + self._ln2_bias = self.create_parameter( + shape=[d_model], attr=ln2_bias_attr, is_bias=True + ) self.name = name @@ -606,15 +665,23 @@ def forward(self, src, cache=None): pre_layer_norm=self._normalize_before, training=self.training, ring_id=self._ring_id, - name=self.name) + name=self.name, + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' return 'd_model={}, dim_feedforward={}, dropout_rate={}, epsilon={}, activation={}, act_dropout_rate={}, normalize_before={}, dtype={}{}'.format( - self._d_model, self._dim_feedforward, self._dropout_rate, - self._epsilon, self._act_method, self._act_dropout_rate, - self._normalize_before, self._dtype, name_str) + self._d_model, + self._dim_feedforward, + self._dropout_rate, + self._epsilon, + self._act_method, + self._act_dropout_rate, + self._normalize_before, + self._dtype, + name_str, + ) def _amp_decorate(self, dtype): # tmp fix for amp.decorator(O2) @@ -694,31 +761,44 @@ class FusedTransformerEncoderLayer(Layer): enc_output = encoder_layer(enc_input, attn_mask) # [2, 4, 128] """ - def __init__(self, - d_model, - nhead, - dim_feedforward, - dropout_rate=0.1, - activation="relu", - attn_dropout_rate=None, - act_dropout_rate=None, - normalize_before=False, - weight_attr=None, - bias_attr=None): + def __init__( + self, + d_model, + nhead, + dim_feedforward, + dropout_rate=0.1, + activation="relu", + attn_dropout_rate=None, + act_dropout_rate=None, + normalize_before=False, + weight_attr=None, + bias_attr=None, + ): self._config = locals() self._config.pop("self") self._config.pop("__class__", None) # py3 super(FusedTransformerEncoderLayer, self).__init__() - assert d_model > 0, ("Expected d_model to be greater than 0, " - "but received {}".format(d_model)) - assert nhead > 0, ("Expected nhead to be greater than 0, " - "but received {}".format(nhead)) + assert ( + d_model > 0 + ), "Expected d_model to be greater than 0, " "but received {}".format( + d_model + ) + assert ( + nhead > 0 + ), "Expected nhead to be greater than 0, " "but received {}".format( + nhead + ) assert dim_feedforward > 0, ( "Expected dim_feedforward to be greater than 0, " - "but received {}".format(dim_feedforward)) - attn_dropout_rate = dropout_rate if attn_dropout_rate is None else attn_dropout_rate - act_dropout_rate = dropout_rate if act_dropout_rate is None else act_dropout_rate + "but received {}".format(dim_feedforward) + ) + attn_dropout_rate = ( + dropout_rate if attn_dropout_rate is None else attn_dropout_rate + ) + act_dropout_rate = ( + dropout_rate if act_dropout_rate is None else act_dropout_rate + ) self.normalize_before = normalize_before weight_attrs = _convert_param_attr_to_list(weight_attr, 2) @@ -737,18 +817,21 @@ def __init__(self, pre_ln_scale_attr=weight_attrs[0], pre_ln_bias_attr=bias_attrs[0], ln_scale_attr=weight_attrs[0], - ln_bias_attr=bias_attrs[0]) - - self.ffn = FusedFeedForward(d_model, - dim_feedforward, - dropout_rate=dropout_rate, - activation=activation, - act_dropout_rate=act_dropout_rate, - normalize_before=self.normalize_before, - linear1_weight_attr=weight_attrs[1], - linear1_bias_attr=bias_attrs[1], - linear2_weight_attr=weight_attrs[1], - linear2_bias_attr=bias_attrs[1]) + ln_bias_attr=bias_attrs[0], + ) + + self.ffn = FusedFeedForward( + d_model, + dim_feedforward, + dropout_rate=dropout_rate, + activation=activation, + act_dropout_rate=act_dropout_rate, + normalize_before=self.normalize_before, + linear1_weight_attr=weight_attrs[1], + linear1_bias_attr=bias_attrs[1], + linear2_weight_attr=weight_attrs[1], + linear2_bias_attr=bias_attrs[1], + ) def forward(self, src, src_mask=None, cache=None): """ @@ -784,9 +867,9 @@ def forward(self, src, src_mask=None, cache=None): if cache is None: attn_out = self.fused_attn(src, attn_mask=src_mask) else: - attn_out, incremental_cache = self.fused_attn(src, - attn_mask=src_mask, - cache=cache) + attn_out, incremental_cache = self.fused_attn( + src, attn_mask=src_mask, cache=cache + ) ffn_out = self.ffn(attn_out) @@ -887,21 +970,23 @@ class FusedTransformer(Layer): cross_attn_mask) # [2, 6, 128] """ - def __init__(self, - d_model=512, - nhead=8, - num_encoder_layers=6, - num_decoder_layers=6, - dim_feedforward=2048, - dropout=0.1, - activation="relu", - attn_dropout=None, - act_dropout=None, - normalize_before=False, - weight_attr=None, - bias_attr=None, - custom_encoder=None, - custom_decoder=None): + def __init__( + self, + d_model=512, + nhead=8, + num_encoder_layers=6, + num_decoder_layers=6, + dim_feedforward=2048, + dropout=0.1, + activation="relu", + attn_dropout=None, + act_dropout=None, + normalize_before=False, + weight_attr=None, + bias_attr=None, + custom_encoder=None, + custom_decoder=None, + ): super(fusedTransformer, self).__init__() raise NotImplementedError() @@ -1069,40 +1154,49 @@ class FusedMultiTransformer(Layer): enc_output = encoder_layers(enc_input, attn_mask) # [2, 4, 128] """ - def __init__(self, - embed_dim, - num_heads, - dim_feedforward, - dropout_rate=0.0, - activation="gelu", - normalize_before=True, - ln_scale_attrs=None, - ln_bias_attrs=None, - qkv_weight_attrs=None, - qkv_bias_attrs=None, - linear_weight_attrs=None, - linear_bias_attrs=None, - ffn_ln_scale_attrs=None, - ffn_ln_bias_attrs=None, - ffn1_weight_attrs=None, - ffn1_bias_attrs=None, - ffn2_weight_attrs=None, - ffn2_bias_attrs=None, - epsilon=1e-5, - num_layers=-1, - nranks=1, - trans_qkvw=True, - ring_id=-1, - name=None): + def __init__( + self, + embed_dim, + num_heads, + dim_feedforward, + dropout_rate=0.0, + activation="gelu", + normalize_before=True, + ln_scale_attrs=None, + ln_bias_attrs=None, + qkv_weight_attrs=None, + qkv_bias_attrs=None, + linear_weight_attrs=None, + linear_bias_attrs=None, + ffn_ln_scale_attrs=None, + ffn_ln_bias_attrs=None, + ffn1_weight_attrs=None, + ffn1_bias_attrs=None, + ffn2_weight_attrs=None, + ffn2_bias_attrs=None, + epsilon=1e-5, + num_layers=-1, + nranks=1, + trans_qkvw=True, + ring_id=-1, + name=None, + ): super(FusedMultiTransformer, self).__init__() - assert embed_dim > 0, ("Expected embed_dim to be greater than 0, " - "but received {}".format(embed_dim)) - assert num_heads > 0, ("Expected nhead to be greater than 0, " - "but received {}".format(num_heads)) - assert dim_feedforward > 0, ( - "Expected dim_feedforward to be greater than 0, but received {}". - format(dim_feedforward)) + assert embed_dim > 0, ( + "Expected embed_dim to be greater than 0, " + "but received {}".format(embed_dim) + ) + assert ( + num_heads > 0 + ), "Expected nhead to be greater than 0, " "but received {}".format( + num_heads + ) + assert ( + dim_feedforward > 0 + ), "Expected dim_feedforward to be greater than 0, but received {}".format( + dim_feedforward + ) self.normalize_before = normalize_before self._dtype = self._helper.get_default_dtype() @@ -1113,7 +1207,9 @@ def __init__(self, self.embed_dim = embed_dim self.num_heads = num_heads self.head_dim = embed_dim // num_heads - assert self.head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads" + assert ( + self.head_dim * num_heads == embed_dim + ), "embed_dim must be divisible by num_heads" # tensor model parallel if nranks > 1: @@ -1159,57 +1255,71 @@ def get_attr(attrs, idx): ln_scale = self.create_parameter( attr=ln_scale_attr, shape=[embed_dim], - default_initializer=Constant(value=1.0)) - ln_bias = self.create_parameter(attr=ln_bias_attr, - shape=[embed_dim], - is_bias=True) + default_initializer=Constant(value=1.0), + ) + ln_bias = self.create_parameter( + attr=ln_bias_attr, shape=[embed_dim], is_bias=True + ) qkv_weight = self.create_parameter( shape=[3, num_heads, self.head_dim, embed_dim] - if trans_qkvw else [embed_dim, 3, num_heads, self.head_dim], + if trans_qkvw + else [embed_dim, 3, num_heads, self.head_dim], attr=qkv_weight_attr, dtype=self._dtype, - is_bias=False) + is_bias=False, + ) qkv_bias = self.create_parameter( shape=[3, num_heads, self.head_dim], attr=qkv_bias_attr, dtype=self._dtype, - is_bias=True) + is_bias=True, + ) linear_weight = self.create_parameter( shape=[num_heads * self.head_dim, embed_dim], attr=linear_weight_attr, dtype=self._dtype, - is_bias=False) - linear_bias = self.create_parameter(shape=[embed_dim], - attr=linear_bias_attr, - dtype=self._dtype, - is_bias=True) + is_bias=False, + ) + linear_bias = self.create_parameter( + shape=[embed_dim], + attr=linear_bias_attr, + dtype=self._dtype, + is_bias=True, + ) ffn_ln_scale = self.create_parameter( shape=[embed_dim], attr=ffn_ln_scale_attr, is_bias=False, - default_initializer=Constant(1.0)) - ffn_ln_bias = self.create_parameter(shape=[embed_dim], - attr=ffn_ln_bias_attr, - is_bias=True) + default_initializer=Constant(1.0), + ) + ffn_ln_bias = self.create_parameter( + shape=[embed_dim], attr=ffn_ln_bias_attr, is_bias=True + ) ffn1_weight = self.create_parameter( shape=[embed_dim, dim_feedforward], attr=ffn1_weight_attr, dtype=self._dtype, - is_bias=False) - ffn1_bias = self.create_parameter(shape=[dim_feedforward], - attr=ffn1_bias_attr, - dtype=self._dtype, - is_bias=True) + is_bias=False, + ) + ffn1_bias = self.create_parameter( + shape=[dim_feedforward], + attr=ffn1_bias_attr, + dtype=self._dtype, + is_bias=True, + ) ffn2_weight = self.create_parameter( shape=[dim_feedforward, embed_dim], attr=ffn2_weight_attr, dtype=self._dtype, - is_bias=False) - ffn2_bias = self.create_parameter(shape=[embed_dim], - attr=ffn2_bias_attr, - dtype=self._dtype, - is_bias=True) + is_bias=False, + ) + ffn2_bias = self.create_parameter( + shape=[embed_dim], + attr=ffn2_bias_attr, + dtype=self._dtype, + is_bias=True, + ) # tensor model parallel if nranks > 1: @@ -1240,12 +1350,9 @@ def get_attr(attrs, idx): self.activation = activation self.name = name - def forward(self, - src, - attn_mask=None, - caches=None, - pre_caches=None, - time_step=None): + def forward( + self, src, attn_mask=None, caches=None, pre_caches=None, time_step=None + ): r""" Applies multi transformer layers on the input. @@ -1306,5 +1413,6 @@ def forward(self, mode='upscale_in_train', trans_qkvw=self._trans_qkvw, ring_id=self._ring_id, - name=self.name) + name=self.name, + ) return out diff --git a/python/paddle/incubate/operators/__init__.py b/python/paddle/incubate/operators/__init__.py index bc4ba8c3890fd..eb105a12e1ab6 100644 --- a/python/paddle/incubate/operators/__init__.py +++ b/python/paddle/incubate/operators/__init__.py @@ -12,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .softmax_mask_fuse_upper_triangle import softmax_mask_fuse_upper_triangle # noqa: F401 +from .softmax_mask_fuse_upper_triangle import ( + softmax_mask_fuse_upper_triangle, +) # noqa: F401 from .softmax_mask_fuse import softmax_mask_fuse # noqa: F401 -from .resnet_unit import ResNetUnit #noqa: F401 -from .graph_send_recv import graph_send_recv #noqa: F401 -from .graph_khop_sampler import graph_khop_sampler #noqa: F401 -from .graph_sample_neighbors import graph_sample_neighbors #noqa: F401 -from .graph_reindex import graph_reindex #noqa: F401 +from .resnet_unit import ResNetUnit # noqa: F401 +from .graph_send_recv import graph_send_recv # noqa: F401 +from .graph_khop_sampler import graph_khop_sampler # noqa: F401 +from .graph_sample_neighbors import graph_sample_neighbors # noqa: F401 +from .graph_reindex import graph_reindex # noqa: F401 diff --git a/python/paddle/incubate/operators/graph_khop_sampler.py b/python/paddle/incubate/operators/graph_khop_sampler.py index 2122383381358..821c4b418ed7e 100644 --- a/python/paddle/incubate/operators/graph_khop_sampler.py +++ b/python/paddle/incubate/operators/graph_khop_sampler.py @@ -18,13 +18,15 @@ from paddle import _legacy_C_ops -def graph_khop_sampler(row, - colptr, - input_nodes, - sample_sizes, - sorted_eids=None, - return_eids=False, - name=None): +def graph_khop_sampler( + row, + colptr, + input_nodes, + sample_sizes, + sorted_eids=None, + return_eids=False, + name=None, +): """ Graph Khop Sampler API. @@ -86,36 +88,64 @@ def graph_khop_sampler(row, if _non_static_mode(): if return_eids: if sorted_eids is None: - raise ValueError("`sorted_eid` should not be None " - "if return_eids is True.") - edge_src, edge_dst, sample_index, reindex_nodes, edge_eids = \ - _legacy_C_ops.graph_khop_sampler(row, sorted_eids, - colptr, input_nodes, - "sample_sizes", sample_sizes, - "return_eids", True) + raise ValueError( + "`sorted_eid` should not be None " "if return_eids is True." + ) + ( + edge_src, + edge_dst, + sample_index, + reindex_nodes, + edge_eids, + ) = _legacy_C_ops.graph_khop_sampler( + row, + sorted_eids, + colptr, + input_nodes, + "sample_sizes", + sample_sizes, + "return_eids", + True, + ) return edge_src, edge_dst, sample_index, reindex_nodes, edge_eids else: - edge_src, edge_dst, sample_index, reindex_nodes, _ = \ - _legacy_C_ops.graph_khop_sampler(row, None, - colptr, input_nodes, - "sample_sizes", sample_sizes, - "return_eids", False) + ( + edge_src, + edge_dst, + sample_index, + reindex_nodes, + _, + ) = _legacy_C_ops.graph_khop_sampler( + row, + None, + colptr, + input_nodes, + "sample_sizes", + sample_sizes, + "return_eids", + False, + ) return edge_src, edge_dst, sample_index, reindex_nodes - check_variable_and_dtype(row, "Row", ("int32", "int64"), - "graph_khop_sampler") + check_variable_and_dtype( + row, "Row", ("int32", "int64"), "graph_khop_sampler" + ) if return_eids: if sorted_eids is None: - raise ValueError("`sorted_eid` should not be None " - "if return_eids is True.") - check_variable_and_dtype(sorted_eids, "Eids", ("int32", "int64"), - "graph_khop_sampler") - - check_variable_and_dtype(colptr, "Col_Ptr", ("int32", "int64"), - "graph_khop_sampler") - check_variable_and_dtype(input_nodes, "X", ("int32", "int64"), - "graph_khop_sampler") + raise ValueError( + "`sorted_eid` should not be None " "if return_eids is True." + ) + check_variable_and_dtype( + sorted_eids, "Eids", ("int32", "int64"), "graph_khop_sampler" + ) + + check_variable_and_dtype( + colptr, "Col_Ptr", ("int32", "int64"), "graph_khop_sampler" + ) + check_variable_and_dtype( + input_nodes, "X", ("int32", "int64"), "graph_khop_sampler" + ) helper = LayerHelper("graph_khop_sampler", **locals()) edge_src = helper.create_variable_for_type_inference(dtype=row.dtype) @@ -123,24 +153,23 @@ def graph_khop_sampler(row, sample_index = helper.create_variable_for_type_inference(dtype=row.dtype) reindex_nodes = helper.create_variable_for_type_inference(dtype=row.dtype) edge_eids = helper.create_variable_for_type_inference(dtype=row.dtype) - helper.append_op(type="graph_khop_sampler", - inputs={ - "Row": row, - "Eids": sorted_eids, - "Col_Ptr": colptr, - "X": input_nodes - }, - outputs={ - "Out_Src": edge_src, - "Out_Dst": edge_dst, - "Sample_Index": sample_index, - "Reindex_X": reindex_nodes, - "Out_Eids": edge_eids - }, - attrs={ - "sample_sizes": sample_sizes, - "return_eids": return_eids - }) + helper.append_op( + type="graph_khop_sampler", + inputs={ + "Row": row, + "Eids": sorted_eids, + "Col_Ptr": colptr, + "X": input_nodes, + }, + outputs={ + "Out_Src": edge_src, + "Out_Dst": edge_dst, + "Sample_Index": sample_index, + "Reindex_X": reindex_nodes, + "Out_Eids": edge_eids, + }, + attrs={"sample_sizes": sample_sizes, "return_eids": return_eids}, + ) if return_eids: return edge_src, edge_dst, sample_index, reindex_nodes, edge_eids else: diff --git a/python/paddle/incubate/operators/graph_reindex.py b/python/paddle/incubate/operators/graph_reindex.py index d8f6774190801..d721c9a002e18 100644 --- a/python/paddle/incubate/operators/graph_reindex.py +++ b/python/paddle/incubate/operators/graph_reindex.py @@ -19,17 +19,21 @@ import paddle.utils.deprecated as deprecated -@deprecated(since="2.4.0", - update_to="paddle.geometric.reindex_graph", - level=1, - reason="paddle.incubate.graph_reindex will be removed in future") -def graph_reindex(x, - neighbors, - count, - value_buffer=None, - index_buffer=None, - flag_buffer_hashtable=False, - name=None): +@deprecated( + since="2.4.0", + update_to="paddle.geometric.reindex_graph", + level=1, + reason="paddle.incubate.graph_reindex will be removed in future", +) +def graph_reindex( + x, + neighbors, + count, + value_buffer=None, + index_buffer=None, + flag_buffer_hashtable=False, + name=None, +): """ Graph Reindex API. @@ -107,47 +111,55 @@ def graph_reindex(x, """ if flag_buffer_hashtable: if value_buffer is None or index_buffer is None: - raise ValueError("`value_buffer` and `index_buffer` should not" - "be None if `flag_buffer_hashtable` is True.") + raise ValueError( + "`value_buffer` and `index_buffer` should not" + "be None if `flag_buffer_hashtable` is True." + ) if _non_static_mode(): - reindex_src, reindex_dst, out_nodes = \ - _legacy_C_ops.graph_reindex(x, neighbors, count, value_buffer, index_buffer, - "flag_buffer_hashtable", flag_buffer_hashtable) + reindex_src, reindex_dst, out_nodes = _legacy_C_ops.graph_reindex( + x, + neighbors, + count, + value_buffer, + index_buffer, + "flag_buffer_hashtable", + flag_buffer_hashtable, + ) return reindex_src, reindex_dst, out_nodes check_variable_and_dtype(x, "X", ("int32", "int64"), "graph_reindex") - check_variable_and_dtype(neighbors, "Neighbors", ("int32", "int64"), - "graph_reindex") + check_variable_and_dtype( + neighbors, "Neighbors", ("int32", "int64"), "graph_reindex" + ) check_variable_and_dtype(count, "Count", ("int32"), "graph_reindex") if flag_buffer_hashtable: - check_variable_and_dtype(value_buffer, "HashTable_Value", ("int32"), - "graph_reindex") - check_variable_and_dtype(index_buffer, "HashTable_Index", ("int32"), - "graph_reindex") + check_variable_and_dtype( + value_buffer, "HashTable_Value", ("int32"), "graph_reindex" + ) + check_variable_and_dtype( + index_buffer, "HashTable_Index", ("int32"), "graph_reindex" + ) helper = LayerHelper("graph_reindex", **locals()) reindex_src = helper.create_variable_for_type_inference(dtype=x.dtype) reindex_dst = helper.create_variable_for_type_inference(dtype=x.dtype) out_nodes = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="graph_reindex", - inputs={ - "X": - x, - "Neighbors": - neighbors, - "Count": - count, - "HashTable_Value": - value_buffer if flag_buffer_hashtable else None, - "HashTable_Index": - index_buffer if flag_buffer_hashtable else None, - }, - outputs={ - "Reindex_Src": reindex_src, - "Reindex_Dst": reindex_dst, - "Out_Nodes": out_nodes - }, - attrs={"flag_buffer_hashtable": flag_buffer_hashtable}) + helper.append_op( + type="graph_reindex", + inputs={ + "X": x, + "Neighbors": neighbors, + "Count": count, + "HashTable_Value": value_buffer if flag_buffer_hashtable else None, + "HashTable_Index": index_buffer if flag_buffer_hashtable else None, + }, + outputs={ + "Reindex_Src": reindex_src, + "Reindex_Dst": reindex_dst, + "Out_Nodes": out_nodes, + }, + attrs={"flag_buffer_hashtable": flag_buffer_hashtable}, + ) return reindex_src, reindex_dst, out_nodes diff --git a/python/paddle/incubate/operators/graph_sample_neighbors.py b/python/paddle/incubate/operators/graph_sample_neighbors.py index b39910f9e1126..48e52bc691408 100644 --- a/python/paddle/incubate/operators/graph_sample_neighbors.py +++ b/python/paddle/incubate/operators/graph_sample_neighbors.py @@ -23,16 +23,19 @@ since="2.4.0", update_to="paddle.geometric.sample_neighbors", level=1, - reason="paddle.incubate.graph_sample_neighbors will be removed in future") -def graph_sample_neighbors(row, - colptr, - input_nodes, - eids=None, - perm_buffer=None, - sample_size=-1, - return_eids=False, - flag_perm_buffer=False, - name=None): + reason="paddle.incubate.graph_sample_neighbors will be removed in future", +) +def graph_sample_neighbors( + row, + colptr, + input_nodes, + eids=None, + perm_buffer=None, + sample_size=-1, + return_eids=False, + flag_perm_buffer=False, + name=None, +): """ Graph Sample Neighbors API. @@ -96,59 +99,83 @@ def graph_sample_neighbors(row, if return_eids: if eids is None: raise ValueError( - "`eids` should not be None if `return_eids` is True.") + "`eids` should not be None if `return_eids` is True." + ) if flag_perm_buffer: if perm_buffer is None: raise ValueError( "`perm_buffer` should not be None if `flag_perm_buffer`" - "is True.") + "is True." + ) if _non_static_mode(): - out_neighbors, out_count, out_eids = _legacy_C_ops.graph_sample_neighbors( - row, colptr, input_nodes, eids, perm_buffer, "sample_size", - sample_size, "return_eids", return_eids, "flag_perm_buffer", - flag_perm_buffer) + ( + out_neighbors, + out_count, + out_eids, + ) = _legacy_C_ops.graph_sample_neighbors( + row, + colptr, + input_nodes, + eids, + perm_buffer, + "sample_size", + sample_size, + "return_eids", + return_eids, + "flag_perm_buffer", + flag_perm_buffer, + ) if return_eids: return out_neighbors, out_count, out_eids return out_neighbors, out_count - check_variable_and_dtype(row, "Row", ("int32", "int64"), - "graph_sample_neighbors") - check_variable_and_dtype(colptr, "Col_Ptr", ("int32", "int64"), - "graph_sample_neighbors") - check_variable_and_dtype(input_nodes, "X", ("int32", "int64"), - "graph_sample_neighbors") + check_variable_and_dtype( + row, "Row", ("int32", "int64"), "graph_sample_neighbors" + ) + check_variable_and_dtype( + colptr, "Col_Ptr", ("int32", "int64"), "graph_sample_neighbors" + ) + check_variable_and_dtype( + input_nodes, "X", ("int32", "int64"), "graph_sample_neighbors" + ) if return_eids: - check_variable_and_dtype(eids, "Eids", ("int32", "int64"), - "graph_sample_neighbors") + check_variable_and_dtype( + eids, "Eids", ("int32", "int64"), "graph_sample_neighbors" + ) if flag_perm_buffer: - check_variable_and_dtype(perm_buffer, "Perm_Buffer", ("int32", "int64"), - "graph_sample_neighbors") + check_variable_and_dtype( + perm_buffer, + "Perm_Buffer", + ("int32", "int64"), + "graph_sample_neighbors", + ) helper = LayerHelper("graph_sample_neighbors", **locals()) out_neighbors = helper.create_variable_for_type_inference(dtype=row.dtype) out_count = helper.create_variable_for_type_inference(dtype=row.dtype) out_eids = helper.create_variable_for_type_inference(dtype=row.dtype) - helper.append_op(type="graph_sample_neighbors", - inputs={ - "Row": row, - "Col_Ptr": colptr, - "X": input_nodes, - "Eids": eids if return_eids else None, - "Perm_Buffer": - perm_buffer if flag_perm_buffer else None - }, - outputs={ - "Out": out_neighbors, - "Out_Count": out_count, - "Out_Eids": out_eids - }, - attrs={ - "sample_size": sample_size, - "return_eids": return_eids, - "flag_perm_buffer": flag_perm_buffer - }) + helper.append_op( + type="graph_sample_neighbors", + inputs={ + "Row": row, + "Col_Ptr": colptr, + "X": input_nodes, + "Eids": eids if return_eids else None, + "Perm_Buffer": perm_buffer if flag_perm_buffer else None, + }, + outputs={ + "Out": out_neighbors, + "Out_Count": out_count, + "Out_Eids": out_eids, + }, + attrs={ + "sample_size": sample_size, + "return_eids": return_eids, + "flag_perm_buffer": flag_perm_buffer, + }, + ) if return_eids: return out_neighbors, out_count, out_eids return out_neighbors, out_count diff --git a/python/paddle/incubate/operators/graph_send_recv.py b/python/paddle/incubate/operators/graph_send_recv.py index 939d8c343fd5c..88fc421f4a09a 100644 --- a/python/paddle/incubate/operators/graph_send_recv.py +++ b/python/paddle/incubate/operators/graph_send_recv.py @@ -16,7 +16,12 @@ from paddle.fluid.layer_helper import LayerHelper from paddle.fluid.framework import _in_legacy_dygraph, in_dygraph_mode from paddle.fluid.framework import Variable -from paddle.fluid.data_feeder import check_variable_and_dtype, check_type, check_dtype, convert_dtype +from paddle.fluid.data_feeder import ( + check_variable_and_dtype, + check_type, + check_dtype, + convert_dtype, +) from paddle.fluid.layers.tensor import cast from paddle import _C_ops, _legacy_C_ops import paddle.utils.deprecated as deprecated @@ -26,13 +31,11 @@ since="2.4.0", update_to="paddle.geometric.send_u_recv", level=1, - reason="graph_send_recv in paddle.incubate will be removed in future") -def graph_send_recv(x, - src_index, - dst_index, - pool_type="sum", - out_size=None, - name=None): + reason="graph_send_recv in paddle.incubate will be removed in future", +) +def graph_send_recv( + x, src_index, dst_index, pool_type="sum", out_size=None, name=None +): r""" Graph Learning Send_Recv combine operator. @@ -116,54 +119,69 @@ def graph_send_recv(x, if pool_type not in ["sum", "mean", "max", "min"]: raise ValueError( "pool_type should be `sum`, `mean`, `max` or `min`, but received %s" - % pool_type) + % pool_type + ) # TODO(daisiming): Should we add judgement for out_size: max(dst_index) + 1. if _in_legacy_dygraph(): out_size = convert_out_size_to_list(out_size) - out, tmp = _legacy_C_ops.graph_send_recv(x, src_index, dst_index, - None, 'reduce_op', - pool_type.upper(), 'out_size', - out_size) + out, tmp = _legacy_C_ops.graph_send_recv( + x, + src_index, + dst_index, + None, + 'reduce_op', + pool_type.upper(), + 'out_size', + out_size, + ) return out if in_dygraph_mode(): out_size = convert_out_size_to_list(out_size) - return _C_ops.graph_send_recv(x, src_index, dst_index, - pool_type.upper(), out_size) - - check_variable_and_dtype(x, "X", ("float32", "float64", "int32", "int64"), - "graph_send_recv") - check_variable_and_dtype(src_index, "Src_index", ("int32", "int64"), - "graph_send_recv") - check_variable_and_dtype(dst_index, "Dst_index", ("int32", "int64"), - "graph_send_recv") + return _C_ops.graph_send_recv( + x, src_index, dst_index, pool_type.upper(), out_size + ) + + check_variable_and_dtype( + x, "X", ("float32", "float64", "int32", "int64"), "graph_send_recv" + ) + check_variable_and_dtype( + src_index, "Src_index", ("int32", "int64"), "graph_send_recv" + ) + check_variable_and_dtype( + dst_index, "Dst_index", ("int32", "int64"), "graph_send_recv" + ) if out_size: - check_type(out_size, 'out_size', (int, np.int32, np.int64, Variable), - 'graph_send_recv') + check_type( + out_size, + 'out_size', + (int, np.int32, np.int64, Variable), + 'graph_send_recv', + ) if isinstance(out_size, Variable): - check_dtype(out_size.dtype, 'out_size', ['int32', 'int64'], - 'graph_send_recv') + check_dtype( + out_size.dtype, 'out_size', ['int32', 'int64'], 'graph_send_recv' + ) helper = LayerHelper("graph_send_recv", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - dst_count = helper.create_variable_for_type_inference(dtype="int32", - stop_gradient=True) + dst_count = helper.create_variable_for_type_inference( + dtype="int32", stop_gradient=True + ) inputs = {"X": x, "Src_index": src_index, "Dst_index": dst_index} attrs = {"reduce_op": pool_type.upper()} - get_out_size_tensor_inputs(inputs=inputs, - attrs=attrs, - out_size=out_size, - op_type='graph_send_recv') - - helper.append_op(type="graph_send_recv", - inputs=inputs, - outputs={ - "Out": out, - "Dst_count": dst_count - }, - attrs=attrs) + get_out_size_tensor_inputs( + inputs=inputs, attrs=attrs, out_size=out_size, op_type='graph_send_recv' + ) + + helper.append_op( + type="graph_send_recv", + inputs=inputs, + outputs={"Out": out, "Dst_count": dst_count}, + attrs=attrs, + ) return out @@ -192,9 +210,14 @@ def get_out_size_tensor_inputs(inputs, attrs, out_size, op_type): attrs['out_size'] = [out_size] elif isinstance(out_size, Variable): out_size.stop_gradient = True - check_dtype(out_size.dtype, 'out_size', ['int32', 'int64'], op_type, - '(When type of out_size in' + op_type + ' is Variable.)') - if (convert_dtype(out_size.dtype) == 'int64'): + check_dtype( + out_size.dtype, + 'out_size', + ['int32', 'int64'], + op_type, + '(When type of out_size in' + op_type + ' is Variable.)', + ) + if convert_dtype(out_size.dtype) == 'int64': out_size = cast(out_size, 'int32') inputs["Out_size"] = out_size else: diff --git a/python/paddle/incubate/operators/resnet_unit.py b/python/paddle/incubate/operators/resnet_unit.py index c8fb583bff4b6..601bbfd11de47 100644 --- a/python/paddle/incubate/operators/resnet_unit.py +++ b/python/paddle/incubate/operators/resnet_unit.py @@ -21,37 +21,77 @@ from paddle.fluid.param_attr import ParamAttr -def resnet_unit(x, filter_x, scale_x, bias_x, mean_x, var_x, z, filter_z, - scale_z, bias_z, mean_z, var_z, stride, stride_z, padding, - dilation, groups, momentum, eps, data_format, fuse_add, - has_shortcut, use_global_stats, is_test, act): +def resnet_unit( + x, + filter_x, + scale_x, + bias_x, + mean_x, + var_x, + z, + filter_z, + scale_z, + bias_z, + mean_z, + var_z, + stride, + stride_z, + padding, + dilation, + groups, + momentum, + eps, + data_format, + fuse_add, + has_shortcut, + use_global_stats, + is_test, + act, +): helper = LayerHelper('resnet_unit', **locals()) bn_param_dtype = fluid.core.VarDesc.VarType.FP32 bit_mask_dtype = fluid.core.VarDesc.VarType.INT32 out = helper.create_variable_for_type_inference(x.dtype) - bit_mask = helper.create_variable_for_type_inference(dtype=bit_mask_dtype, - stop_gradient=True) + bit_mask = helper.create_variable_for_type_inference( + dtype=bit_mask_dtype, stop_gradient=True + ) # intermediate_out for x - conv_x = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) + conv_x = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) saved_mean_x = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) saved_invstd_x = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) running_mean_x = mean_x running_var_x = var_x # intermediate_out for z - conv_z = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) + conv_z = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) saved_mean_z = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) saved_invstd_z = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) - running_mean_z = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if mean_z is None else mean_z - running_var_z = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if var_z is None else var_z + dtype=bn_param_dtype, stop_gradient=True + ) + running_mean_z = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if mean_z is None + else mean_z + ) + running_var_z = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if var_z is None + else var_z + ) inputs = { 'X': x, @@ -65,7 +105,7 @@ def resnet_unit(x, filter_x, scale_x, bias_x, mean_x, var_x, z, filter_z, 'ScaleZ': scale_z, 'BiasZ': bias_z, 'MeanZ': mean_z, - 'VarZ': var_z + 'VarZ': var_z, } attrs = { @@ -81,7 +121,7 @@ def resnet_unit(x, filter_x, scale_x, bias_x, mean_x, var_x, z, filter_z, 'has_shortcut': has_shortcut, 'use_global_stats': use_global_stats, 'is_test': is_test, - 'act_type': act + 'act_type': act, } outputs = { @@ -99,10 +139,9 @@ def resnet_unit(x, filter_x, scale_x, bias_x, mean_x, var_x, z, filter_z, 'RunningVarZ': running_var_z, } - helper.append_op(type='resnet_unit', - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='resnet_unit', inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -113,31 +152,33 @@ class ResNetUnit(Layer): ResNetUnit is designed for optimize the performence by using cudnnv8 API. """ - def __init__(self, - num_channels_x, - num_filters, - filter_size, - stride=1, - momentum=0.9, - eps=1e-5, - data_format='NHWC', - act='relu', - fuse_add=False, - has_shortcut=False, - use_global_stats=False, - is_test=False, - filter_x_attr=None, - scale_x_attr=None, - bias_x_attr=None, - moving_mean_x_name=None, - moving_var_x_name=None, - num_channels_z=1, - stride_z=1, - filter_z_attr=None, - scale_z_attr=None, - bias_z_attr=None, - moving_mean_z_name=None, - moving_var_z_name=None): + def __init__( + self, + num_channels_x, + num_filters, + filter_size, + stride=1, + momentum=0.9, + eps=1e-5, + data_format='NHWC', + act='relu', + fuse_add=False, + has_shortcut=False, + use_global_stats=False, + is_test=False, + filter_x_attr=None, + scale_x_attr=None, + bias_x_attr=None, + moving_mean_x_name=None, + moving_var_x_name=None, + num_channels_z=1, + stride_z=1, + filter_z_attr=None, + scale_z_attr=None, + bias_z_attr=None, + moving_mean_z_name=None, + moving_var_z_name=None, + ): super(ResNetUnit, self).__init__() self._stride = stride self._stride_z = stride_z @@ -158,89 +199,124 @@ def __init__(self, valid_format = {'NHWC', 'NCHW'} if data_format not in valid_format: raise ValueError( - "conv_format must be one of {}, but got conv_format='{}'". - format(valid_format, data_format)) + "conv_format must be one of {}, but got conv_format='{}'".format( + valid_format, data_format + ) + ) def _get_default_param_initializer(channels): filter_elem_num = np.prod(self._kernel_size) * channels - std = (2.0 / filter_elem_num)**0.5 + std = (2.0 / filter_elem_num) ** 0.5 return I.Normal(0.0, std) - is_nchw = (data_format == 'NCHW') + is_nchw = data_format == 'NCHW' # initial filter bn_param_dtype = fluid.core.VarDesc.VarType.FP32 if not is_nchw: bn_param_shape = [1, 1, 1, num_filters] filter_x_shape = [ - num_filters, filter_size, filter_size, num_channels_x + num_filters, + filter_size, + filter_size, + num_channels_x, ] filter_z_shape = [ - num_filters, filter_size, filter_size, num_channels_z + num_filters, + filter_size, + filter_size, + num_channels_z, ] else: bn_param_shape = [1, num_filters, 1, 1] filter_x_shape = [ - num_filters, num_channels_x, filter_size, filter_size + num_filters, + num_channels_x, + filter_size, + filter_size, ] filter_z_shape = [ - num_filters, num_channels_z, filter_size, filter_size + num_filters, + num_channels_z, + filter_size, + filter_size, ] self.filter_x = self.create_parameter( shape=filter_x_shape, attr=filter_x_attr, - default_initializer=_get_default_param_initializer(num_channels_x)) + default_initializer=_get_default_param_initializer(num_channels_x), + ) self.scale_x = self.create_parameter( shape=bn_param_shape, attr=scale_x_attr, dtype=bn_param_dtype, - default_initializer=I.Constant(1.0)) - self.bias_x = self.create_parameter(shape=bn_param_shape, - attr=bias_x_attr, - dtype=bn_param_dtype, - is_bias=True) - self.mean_x = self.create_parameter(attr=ParamAttr( - name=moving_mean_x_name, - initializer=I.Constant(0.0), - trainable=False), - shape=bn_param_shape, - dtype=bn_param_dtype) + default_initializer=I.Constant(1.0), + ) + self.bias_x = self.create_parameter( + shape=bn_param_shape, + attr=bias_x_attr, + dtype=bn_param_dtype, + is_bias=True, + ) + self.mean_x = self.create_parameter( + attr=ParamAttr( + name=moving_mean_x_name, + initializer=I.Constant(0.0), + trainable=False, + ), + shape=bn_param_shape, + dtype=bn_param_dtype, + ) self.mean_x.stop_gradient = True - self.var_x = self.create_parameter(attr=ParamAttr( - name=moving_var_x_name, - initializer=I.Constant(1.0), - trainable=False), - shape=bn_param_shape, - dtype=bn_param_dtype) + self.var_x = self.create_parameter( + attr=ParamAttr( + name=moving_var_x_name, + initializer=I.Constant(1.0), + trainable=False, + ), + shape=bn_param_shape, + dtype=bn_param_dtype, + ) self.var_x.stop_gradient = True if has_shortcut: self.filter_z = self.create_parameter( shape=filter_z_shape, attr=filter_z_attr, default_initializer=_get_default_param_initializer( - num_channels_z)) + num_channels_z + ), + ) self.scale_z = self.create_parameter( shape=bn_param_shape, attr=scale_z_attr, dtype=bn_param_dtype, - default_initializer=I.Constant(1.0)) - self.bias_z = self.create_parameter(shape=bn_param_shape, - attr=bias_z_attr, - dtype=bn_param_dtype, - is_bias=True) - self.mean_z = self.create_parameter(attr=ParamAttr( - name=moving_mean_z_name, - initializer=I.Constant(0.0), - trainable=False), - shape=bn_param_shape, - dtype=bn_param_dtype) + default_initializer=I.Constant(1.0), + ) + self.bias_z = self.create_parameter( + shape=bn_param_shape, + attr=bias_z_attr, + dtype=bn_param_dtype, + is_bias=True, + ) + self.mean_z = self.create_parameter( + attr=ParamAttr( + name=moving_mean_z_name, + initializer=I.Constant(0.0), + trainable=False, + ), + shape=bn_param_shape, + dtype=bn_param_dtype, + ) self.mean_z.stop_gradient = True - self.var_z = self.create_parameter(attr=ParamAttr( - name=moving_var_z_name, - initializer=I.Constant(1.0), - trainable=False), - shape=bn_param_shape, - dtype=bn_param_dtype) + self.var_z = self.create_parameter( + attr=ParamAttr( + name=moving_var_z_name, + initializer=I.Constant(1.0), + trainable=False, + ), + shape=bn_param_shape, + dtype=bn_param_dtype, + ) self.var_z.stop_gradient = True else: self.filter_z = None @@ -253,12 +329,31 @@ def forward(self, x, z=None): if self._fuse_add and z is None: raise ValueError("z can not be None") - out = resnet_unit(x, self.filter_x, self.scale_x, self.bias_x, - self.mean_x, self.var_x, z, self.filter_z, - self.scale_z, self.bias_z, self.mean_z, self.var_z, - self._stride, self._stride_z, self._padding, - self._dilation, self._groups, self._momentum, - self._eps, self._data_format, self._fuse_add, - self._has_shortcut, self._use_global_stats, - self._is_test, self._act) + out = resnet_unit( + x, + self.filter_x, + self.scale_x, + self.bias_x, + self.mean_x, + self.var_x, + z, + self.filter_z, + self.scale_z, + self.bias_z, + self.mean_z, + self.var_z, + self._stride, + self._stride_z, + self._padding, + self._dilation, + self._groups, + self._momentum, + self._eps, + self._data_format, + self._fuse_add, + self._has_shortcut, + self._use_global_stats, + self._is_test, + self._act, + ) return out diff --git a/python/paddle/incubate/operators/softmax_mask_fuse.py b/python/paddle/incubate/operators/softmax_mask_fuse.py index 27afabaa0db6b..b8081af826da3 100644 --- a/python/paddle/incubate/operators/softmax_mask_fuse.py +++ b/python/paddle/incubate/operators/softmax_mask_fuse.py @@ -60,10 +60,9 @@ def softmax_mask_fuse(x, mask, name=None): return out helper = LayerHelper('fused_softmax_mask', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='fused_softmax_mask', - inputs={ - 'X': [x], - 'Mask': [mask] - }, - outputs={'Out': [out]}) + helper.append_op( + type='fused_softmax_mask', + inputs={'X': [x], 'Mask': [mask]}, + outputs={'Out': [out]}, + ) return out diff --git a/python/paddle/incubate/operators/softmax_mask_fuse_upper_triangle.py b/python/paddle/incubate/operators/softmax_mask_fuse_upper_triangle.py index d73b8ba6d652e..aab40986befd0 100644 --- a/python/paddle/incubate/operators/softmax_mask_fuse_upper_triangle.py +++ b/python/paddle/incubate/operators/softmax_mask_fuse_upper_triangle.py @@ -63,7 +63,9 @@ def softmax_mask_fuse_upper_triangle(x): out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='fused_softmax_mask_upper_triangle', - inputs={'X': [x]}, - outputs={'Out': [out]}) + helper.append_op( + type='fused_softmax_mask_upper_triangle', + inputs={'X': [x]}, + outputs={'Out': [out]}, + ) return out diff --git a/python/paddle/incubate/optimizer/distributed_fused_lamb.py b/python/paddle/incubate/optimizer/distributed_fused_lamb.py index 56a9a99134775..ef1bacd075268 100644 --- a/python/paddle/incubate/optimizer/distributed_fused_lamb.py +++ b/python/paddle/incubate/optimizer/distributed_fused_lamb.py @@ -32,84 +32,88 @@ def init_communicator(block, rank, ranks, ring_id): local_rank = ranks.index(rank) comm_var_name = unique_name.generate('comm_id') - comm_id_var = block.create_var(name=comm_var_name, - persistable=True, - type=core.VarDesc.VarType.RAW) - block.append_op(type='c_gen_nccl_id', - inputs={}, - outputs={'Out': comm_id_var}, - attrs={ - 'rank': local_rank, - 'endpoint': cur_ep, - 'other_endpoints': other_eps, - 'ring_id': ring_id - }) - block.append_op(type='c_comm_init', - inputs={'X': comm_id_var}, - outputs={}, - attrs={ - 'nranks': len(ranks), - 'rank': local_rank, - 'ring_id': ring_id - }) + comm_id_var = block.create_var( + name=comm_var_name, persistable=True, type=core.VarDesc.VarType.RAW + ) + block.append_op( + type='c_gen_nccl_id', + inputs={}, + outputs={'Out': comm_id_var}, + attrs={ + 'rank': local_rank, + 'endpoint': cur_ep, + 'other_endpoints': other_eps, + 'ring_id': ring_id, + }, + ) + block.append_op( + type='c_comm_init', + inputs={'X': comm_id_var}, + outputs={}, + attrs={'nranks': len(ranks), 'rank': local_rank, 'ring_id': ring_id}, + ) tmp_var = block.create_var(name=unique_name.generate('tmp')) - block.append_op(type='fill_constant', - outputs={'Out': tmp_var}, - attrs={'value': 1}) - block.append_op(type='c_allreduce_sum', - inputs={'X': tmp_var}, - outputs={'Out': tmp_var}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True - }) - block.append_op(type='c_sync_calc_stream', - inputs={'X': tmp_var}, - outputs={'Out': tmp_var}) + block.append_op( + type='fill_constant', outputs={'Out': tmp_var}, attrs={'value': 1} + ) + block.append_op( + type='c_allreduce_sum', + inputs={'X': tmp_var}, + outputs={'Out': tmp_var}, + attrs={'ring_id': ring_id, 'use_calc_stream': True}, + ) + block.append_op( + type='c_sync_calc_stream', + inputs={'X': tmp_var}, + outputs={'Out': tmp_var}, + ) return ring_id def broadcast_parameters(block, parameters, ring_id): for p in parameters: - block.append_op(type='c_broadcast', - inputs={'X': p}, - outputs={'Out': p}, - attrs={ - 'ring_id': ring_id, - 'use_calc_stream': True - }) + block.append_op( + type='c_broadcast', + inputs={'X': p}, + outputs={'Out': p}, + attrs={'ring_id': ring_id, 'use_calc_stream': True}, + ) class DistributedFusedLamb(Optimizer): - - def __init__(self, - learning_rate=0.001, - lamb_weight_decay=0.01, - beta1=0.9, - beta2=0.999, - epsilon=1e-6, - parameters=None, - grad_clip=None, - exclude_from_weight_decay_fn=None, - clip_after_allreduce=True, - is_grad_scaled_by_nranks=True, - alignment=128, - use_master_param_norm=True, - gradient_accumulation_steps=1, - use_master_acc_grad=True, - nproc_per_node=None, - use_hierarchical_allreduce=False, - name=None): - assert not framework._non_static_mode( + def __init__( + self, + learning_rate=0.001, + lamb_weight_decay=0.01, + beta1=0.9, + beta2=0.999, + epsilon=1e-6, + parameters=None, + grad_clip=None, + exclude_from_weight_decay_fn=None, + clip_after_allreduce=True, + is_grad_scaled_by_nranks=True, + alignment=128, + use_master_param_norm=True, + gradient_accumulation_steps=1, + use_master_acc_grad=True, + nproc_per_node=None, + use_hierarchical_allreduce=False, + name=None, + ): + assert ( + not framework._non_static_mode() ), "DistributedFusedLamb does not support dygraph mode" - super(DistributedFusedLamb, self).__init__(learning_rate=learning_rate, - grad_clip=None, - name=name) + super(DistributedFusedLamb, self).__init__( + learning_rate=learning_rate, grad_clip=None, name=name + ) self._beta1 = beta1 self._beta2 = beta2 self._epsilon = epsilon - self._weight_decay = lamb_weight_decay if lamb_weight_decay is not None else 0.0 + self._weight_decay = ( + lamb_weight_decay if lamb_weight_decay is not None else 0.0 + ) if grad_clip is not None: assert isinstance( grad_clip, ClipGradByGlobalNorm @@ -137,14 +141,16 @@ def __init__(self, self._found_inf = main_block.create_var( name=unique_name.generate('found_inf'), shape=[1], - dtype=core.VarDesc.VarType.BOOL) + dtype=core.VarDesc.VarType.BOOL, + ) self._step = None if self._gradient_accumulation_steps > 1: self._stop_update = main_block.create_var( name=unique_name.generate('stop_update'), shape=[1], - dtype=core.VarDesc.VarType.BOOL) + dtype=core.VarDesc.VarType.BOOL, + ) else: self._stop_update = None @@ -169,11 +175,13 @@ def _set_scale(self, scale): def _create_scale_from_constant(self, value): name = unique_name.generate('global_scale') - return layers.create_global_var(name=name, - shape=[1], - dtype='float32', - value=float(value), - persistable=True) + return layers.create_global_var( + name=name, + shape=[1], + dtype='float32', + value=float(value), + persistable=True, + ) def _get_or_create_scale(self): if self._scale is None: @@ -184,17 +192,21 @@ def _create_persistable_var(self, name=None, shape=[-1], dtype='float32'): startup_block = self.helper.startup_program.global_block() if name is not None: name = unique_name.generate(name) - startup_var = startup_block.create_var(name=name, - shape=shape, - dtype=dtype, - persistable=True, - stop_gradient=True) + startup_var = startup_block.create_var( + name=name, + shape=shape, + dtype=dtype, + persistable=True, + stop_gradient=True, + ) main_block = self.helper.main_program.global_block() - main_var = main_block.create_var(name=startup_var.name, - shape=startup_var.shape, - dtype=startup_var.dtype, - persistable=True, - stop_gradient=True) + main_var = main_block.create_var( + name=startup_var.name, + shape=startup_var.shape, + dtype=startup_var.dtype, + persistable=True, + stop_gradient=True, + ) return main_var def _get_parameter(self, name, scope=None): @@ -224,20 +236,25 @@ def apply_gradients(self, params_grads): for p, g in params_grads: flattened.extend([p, g]) with flattened[0].block.program._optimized_guard(flattened), name_scope( - "optimizer"): + "optimizer" + ): self._apply_gradients_impl(params_grads) def _apply_gradients_impl(self, params_grads): for p, g in params_grads: - assert g.type == core.VarDesc.VarType.LOD_TENSOR, "Only support dense gradient" + assert ( + g.type == core.VarDesc.VarType.LOD_TENSOR + ), "Only support dense gradient" g.persistable = True # the gradient must be persistable for fusion fp32_fused_param = self._create_persistable_var('fp32_fused_param') fp32_fused_grad = self._create_persistable_var('fp32_fused_grad') - fp16_fused_param = self._create_persistable_var('fp16_fused_param', - dtype='float16') - fp16_fused_grad = self._create_persistable_var('fp16_fused_grad', - dtype='float16') + fp16_fused_param = self._create_persistable_var( + 'fp16_fused_param', dtype='float16' + ) + fp16_fused_grad = self._create_persistable_var( + 'fp16_fused_grad', dtype='float16' + ) master_params = [] for p, g in params_grads: @@ -255,15 +272,18 @@ def _apply_gradients_impl(self, params_grads): param_info = self._create_persistable_var('param_info', dtype='int32') param_info.is_distributed = True - fused_offsets = self._create_persistable_var('fused_offsets', - dtype='int32') + fused_offsets = self._create_persistable_var( + 'fused_offsets', dtype='int32' + ) fp32_partial_fused_offsets = self._create_persistable_var( - 'fp32_partial_fused_offsets', dtype='int32') + 'fp32_partial_fused_offsets', dtype='int32' + ) fp32_partial_fused_offsets.is_distributed = True fp16_partial_fused_offsets = self._create_persistable_var( - 'fp16_partial_fused_offsets', dtype='int32') + 'fp16_partial_fused_offsets', dtype='int32' + ) fp16_partial_fused_offsets.is_distributed = True param_order = self._create_persistable_var('param_order', dtype='int32') @@ -274,8 +294,9 @@ def _apply_gradients_impl(self, params_grads): self._create_persistable_var('fp32_acc_fused_grad') ] fp16_acc_fused_grad = [ - self._create_persistable_var('fp16_acc_fused_grad', - dtype='float16') + self._create_persistable_var( + 'fp16_acc_fused_grad', dtype='float16' + ) ] acc_step = [self._create_persistable_var('acc_step', dtype='int64')] else: @@ -291,33 +312,40 @@ def _apply_gradients_impl(self, params_grads): nproc_per_node = nranks else: nproc_per_node = self._nproc_per_node - assert nranks % nproc_per_node == 0, "nranks should be exactly divided by nproc_per_node" + assert ( + nranks % nproc_per_node == 0 + ), "nranks should be exactly divided by nproc_per_node" - shard_inside_node = (nranks > nproc_per_node) + shard_inside_node = nranks > nproc_per_node local_rank = rank % nproc_per_node node_id = int(rank / nproc_per_node) node_num = int(nranks / nproc_per_node) ring_ids = [] startup_block = self.helper.startup_program.global_block() if nranks > 1: - ring_id = init_communicator(startup_block, rank, - list(range(nranks)), 0) + ring_id = init_communicator( + startup_block, rank, list(range(nranks)), 0 + ) ring_ids.append(ring_id) use_hierarchical_allreduce = False if node_num > 1 and len(ring_ids) <= 1 and shard_inside_node: local_group_ranks = list( - range(node_id * nproc_per_node, (node_id + 1) * nproc_per_node)) - ring_id = init_communicator(startup_block, rank, local_group_ranks, - 1) + range(node_id * nproc_per_node, (node_id + 1) * nproc_per_node) + ) + ring_id = init_communicator( + startup_block, rank, local_group_ranks, 1 + ) ring_ids.append(ring_id) if self._use_hierarchical_allreduce and nranks > nproc_per_node: use_hierarchical_allreduce = True outer_group_ranks = list( - range(rank % nproc_per_node, nranks, nproc_per_node)) - ring_id = init_communicator(startup_block, rank, - outer_group_ranks, ring_ids[-1] + 1) + range(rank % nproc_per_node, nranks, nproc_per_node) + ) + ring_id = init_communicator( + startup_block, rank, outer_group_ranks, ring_ids[-1] + 1 + ) ring_ids.append(ring_id) scale = self._get_or_create_scale() @@ -331,11 +359,13 @@ def _apply_gradients_impl(self, params_grads): apply_weight_decay[i] = 0 for g in grads: - startup_block.create_var(name=g.name, - type=g.type, - dtype=g.dtype, - persistable=g.persistable, - shape=g.shape) + startup_block.create_var( + name=g.name, + type=g.type, + dtype=g.dtype, + persistable=g.persistable, + shape=g.shape, + ) if nranks > 1: broadcast_parameters(startup_block, params, ring_ids[0]) @@ -375,7 +405,8 @@ def _apply_gradients_impl(self, params_grads): 'moment2': 0.0, 'beta1': self._beta1, 'beta2': self._beta2, - }) + }, + ) main_block = self.helper.main_program.global_block() self._create_global_learning_rate() @@ -418,19 +449,15 @@ def _apply_gradients_impl(self, params_grads): 'Moment2Out': [moment2], 'Beta1PowOut': [beta1pow], 'Beta2PowOut': [beta2pow], - 'ParamOut': - params, - 'GradOut': - grads, + 'ParamOut': params, + 'GradOut': grads, 'FoundInf': [self._found_inf], - 'FP32AccFusedGrad': - fp32_acc_fused_grad, - 'FP16AccFusedGrad': - fp16_acc_fused_grad, - 'AccStep': - acc_step, - 'StopUpdate': - self._stop_update if self._stop_update is not None else [], + 'FP32AccFusedGrad': fp32_acc_fused_grad, + 'FP16AccFusedGrad': fp16_acc_fused_grad, + 'AccStep': acc_step, + 'StopUpdate': self._stop_update + if self._stop_update is not None + else [], 'Step': [step], }, attrs={ @@ -448,5 +475,6 @@ def _apply_gradients_impl(self, params_grads): 'acc_steps': self._gradient_accumulation_steps, 'use_master_acc_grad': self._use_master_acc_grad, 'use_hierarchical_allreduce': use_hierarchical_allreduce, - }) + }, + ) return [lamb_op] diff --git a/python/paddle/incubate/optimizer/functional/bfgs.py b/python/paddle/incubate/optimizer/functional/bfgs.py index 58d647f4dd4f9..3349cd533e2d7 100644 --- a/python/paddle/incubate/optimizer/functional/bfgs.py +++ b/python/paddle/incubate/optimizer/functional/bfgs.py @@ -15,22 +15,28 @@ import numpy as np from .line_search import strong_wolfe -from .utils import _value_and_gradient, check_input_type, check_initial_inverse_hessian_estimate +from .utils import ( + _value_and_gradient, + check_input_type, + check_initial_inverse_hessian_estimate, +) import paddle -def minimize_bfgs(objective_func, - initial_position, - max_iters=50, - tolerance_grad=1e-7, - tolerance_change=1e-9, - initial_inverse_hessian_estimate=None, - line_search_fn='strong_wolfe', - max_line_search_iters=50, - initial_step_length=1.0, - dtype='float32', - name=None): +def minimize_bfgs( + objective_func, + initial_position, + max_iters=50, + tolerance_grad=1e-7, + tolerance_change=1e-9, + initial_inverse_hessian_estimate=None, + line_search_fn='strong_wolfe', + max_line_search_iters=50, + initial_step_length=1.0, + dtype='float32', + name=None, +): r""" Minimizes a differentiable function `func` using the BFGS method. The BFGS is a quasi-Newton method for solving an unconstrained optimization problem over a differentiable function. @@ -91,8 +97,10 @@ def func(x): if dtype not in ['float32', 'float64']: raise ValueError( - "The dtype must be 'float32' or 'float64', but the specified is {}." - .format(dtype)) + "The dtype must be 'float32' or 'float64', but the specified is {}.".format( + dtype + ) + ) op_name = 'minimize_bfgs' check_input_type(initial_position, 'initial_position', op_name) @@ -101,8 +109,11 @@ def func(x): if initial_inverse_hessian_estimate is None: initial_inverse_hessian_estimate = I else: - check_input_type(initial_inverse_hessian_estimate, - 'initial_inverse_hessian_estimate', op_name) + check_input_type( + initial_inverse_hessian_estimate, + 'initial_inverse_hessian_estimate', + op_name, + ) check_initial_inverse_hessian_estimate(initial_inverse_hessian_estimate) Hk = paddle.assign(initial_inverse_hessian_estimate) @@ -131,11 +142,14 @@ def body(k, done, is_converge, num_func_calls, xk, value, g1, Hk): xk=xk, pk=pk, initial_step_length=initial_step_length, - dtype=dtype) + dtype=dtype, + ) else: raise NotImplementedError( - "Currently only support line_search_fn = 'strong_wolfe', but the specified is '{}'" - .format(line_search_fn)) + "Currently only support line_search_fn = 'strong_wolfe', but the specified is '{}'".format( + line_search_fn + ) + ) num_func_calls += ls_func_calls ############# update Hk ############# @@ -150,14 +164,17 @@ def body(k, done, is_converge, num_func_calls, xk, value, g1, Hk): rhok_inv = paddle.dot(yk, sk) rhok = paddle.static.nn.cond( - rhok_inv == 0., + rhok_inv == 0.0, lambda: paddle.full(shape=[1], fill_value=1000.0, dtype=dtype), - lambda: 1. / rhok_inv) + lambda: 1.0 / rhok_inv, + ) Vk_transpose = I - rhok * sk * yk.t() Vk = I - rhok * yk * sk.t() - Hk = paddle.matmul(paddle.matmul(Vk_transpose, Hk), - Vk) + rhok * sk * sk.t() + Hk = ( + paddle.matmul(paddle.matmul(Vk_transpose, Hk), Vk) + + rhok * sk * sk.t() + ) k += 1 @@ -165,15 +182,16 @@ def body(k, done, is_converge, num_func_calls, xk, value, g1, Hk): gnorm = paddle.linalg.norm(g1, p=np.inf) pk_norm = paddle.linalg.norm(pk, p=np.inf) paddle.assign( - done | (gnorm < tolerance_grad) | (pk_norm < tolerance_change), - done) + done | (gnorm < tolerance_grad) | (pk_norm < tolerance_change), done + ) paddle.assign(done, is_converge) # when alpha=0, there is no chance to get xk change. - paddle.assign(done | (alpha == 0.), done) + paddle.assign(done | (alpha == 0.0), done) return [k, done, is_converge, num_func_calls, xk, value, g1, Hk] paddle.static.nn.while_loop( cond=cond, body=body, - loop_vars=[k, done, is_converge, num_func_calls, xk, value, g1, Hk]) + loop_vars=[k, done, is_converge, num_func_calls, xk, value, g1, Hk], + ) return is_converge, num_func_calls, xk, value, g1, Hk diff --git a/python/paddle/incubate/optimizer/functional/lbfgs.py b/python/paddle/incubate/optimizer/functional/lbfgs.py index 5925d47decd86..a0cf07f8d4ab4 100644 --- a/python/paddle/incubate/optimizer/functional/lbfgs.py +++ b/python/paddle/incubate/optimizer/functional/lbfgs.py @@ -15,23 +15,29 @@ import numpy as np from .line_search import strong_wolfe -from .utils import _value_and_gradient, check_input_type, check_initial_inverse_hessian_estimate +from .utils import ( + _value_and_gradient, + check_input_type, + check_initial_inverse_hessian_estimate, +) import paddle -def minimize_lbfgs(objective_func, - initial_position, - history_size=100, - max_iters=50, - tolerance_grad=1e-8, - tolerance_change=1e-8, - initial_inverse_hessian_estimate=None, - line_search_fn='strong_wolfe', - max_line_search_iters=50, - initial_step_length=1.0, - dtype='float32', - name=None): +def minimize_lbfgs( + objective_func, + initial_position, + history_size=100, + max_iters=50, + tolerance_grad=1e-8, + tolerance_change=1e-8, + initial_inverse_hessian_estimate=None, + line_search_fn='strong_wolfe', + max_line_search_iters=50, + initial_step_length=1.0, + dtype='float32', + name=None, +): r""" Minimizes a differentiable function `func` using the L-BFGS method. The L-BFGS is a quasi-Newton method for solving an unconstrained optimization problem over a differentiable function. @@ -91,8 +97,10 @@ def func(x): """ if dtype not in ['float32', 'float64']: raise ValueError( - "The dtype must be 'float32' or 'float64', but the specified is {}." - .format(dtype)) + "The dtype must be 'float32' or 'float64', but the specified is {}.".format( + dtype + ) + ) op_name = 'minimize_lbfgs' check_input_type(initial_position, 'initial_position', op_name) @@ -100,8 +108,11 @@ def func(x): if initial_inverse_hessian_estimate is None: H0 = paddle.eye(initial_position.shape[0], dtype=dtype) else: - check_input_type(initial_inverse_hessian_estimate, - 'initial_inverse_hessian_estimate', op_name) + check_input_type( + initial_inverse_hessian_estimate, + 'initial_inverse_hessian_estimate', + op_name, + ) check_initial_inverse_hessian_estimate(initial_inverse_hessian_estimate) H0 = initial_inverse_hessian_estimate @@ -114,9 +125,9 @@ def func(x): is_converge = paddle.full(shape=[1], fill_value=False, dtype='bool') num_func_calls = paddle.full(shape=[1], fill_value=1, dtype='int64') - history_size = paddle.full(shape=[1], - fill_value=history_size, - dtype='int64') + history_size = paddle.full( + shape=[1], fill_value=history_size, dtype='int64' + ) head = paddle.full(shape=[1], fill_value=1, dtype='int64') tail = paddle.full(shape=[1], fill_value=0, dtype='int64') @@ -130,20 +141,44 @@ def func(x): rhok_vec = paddle.zeros((history_size + 1, 1), dtype=dtype) ai_vec = paddle.zeros((history_size + 1, 1), dtype=dtype) - def cond(k, done, is_converge, num_func_calls, value, xk, g1, sk_vec, - yk_vec, rhok_vec, head, tail): + def cond( + k, + done, + is_converge, + num_func_calls, + value, + xk, + g1, + sk_vec, + yk_vec, + rhok_vec, + head, + tail, + ): return (k < max_iters) & ~done - def body(k, done, is_converge, num_func_calls, value, xk, g1, sk_vec, - yk_vec, rhok_vec, head, tail): + def body( + k, + done, + is_converge, + num_func_calls, + value, + xk, + g1, + sk_vec, + yk_vec, + rhok_vec, + head, + tail, + ): # use assign to cut off the relevance between g1 and q, or they will change together. ############# compute p_k by two-loop recursion ############# q = paddle.assign(g1) # In a array circle, the index may out of range, so must use mod. - i = paddle.full(shape=[1], - fill_value=(head - 1).mod(history_size), - dtype='int64') + i = paddle.full( + shape=[1], fill_value=(head - 1).mod(history_size), dtype='int64' + ) def cond(i, q): return i != tail @@ -180,11 +215,14 @@ def body(i, r): xk=xk, pk=pk, initial_step_length=initial_step_length, - dtype=dtype) + dtype=dtype, + ) else: raise NotImplementedError( - "Currently only support line_search_fn = 'strong_wolfe', but the specified is '{}'" - .format(line_search_fn)) + "Currently only support line_search_fn = 'strong_wolfe', but the specified is '{}'".format( + line_search_fn + ) + ) paddle.assign(num_func_calls + ls_func_calls, num_func_calls) ############# update sk_vec, yk_vec, rhok_vec ############# @@ -193,9 +231,10 @@ def body(i, r): rhok_inv = paddle.dot(yk, sk) rhok = paddle.static.nn.cond( - rhok_inv == 0., + rhok_inv == 0.0, lambda: paddle.full(shape=[1], fill_value=1000.0, dtype=dtype), - lambda: 1. / rhok_inv) + lambda: 1.0 / rhok_inv, + ) sk_vec[head] = sk yk_vec[head] = yk @@ -216,21 +255,43 @@ def true_fn(tail): gnorm = paddle.linalg.norm(g1, p=np.inf) pk_norm = paddle.linalg.norm(pk, p=np.inf) paddle.assign( - done | (gnorm < tolerance_grad) | (pk_norm < tolerance_change), - done) + done | (gnorm < tolerance_grad) | (pk_norm < tolerance_change), done + ) paddle.assign(done, is_converge) # when alpha=0, there is no chance to get xk change. - paddle.assign(done | (alpha == 0.), done) + paddle.assign(done | (alpha == 0.0), done) return [ - k, done, is_converge, num_func_calls, value, xk, g1, sk_vec, yk_vec, - rhok_vec, head, tail + k, + done, + is_converge, + num_func_calls, + value, + xk, + g1, + sk_vec, + yk_vec, + rhok_vec, + head, + tail, ] - paddle.static.nn.while_loop(cond=cond, - body=body, - loop_vars=[ - k, done, is_converge, num_func_calls, value, - xk, g1, sk_vec, yk_vec, rhok_vec, head, tail - ]) + paddle.static.nn.while_loop( + cond=cond, + body=body, + loop_vars=[ + k, + done, + is_converge, + num_func_calls, + value, + xk, + g1, + sk_vec, + yk_vec, + rhok_vec, + head, + tail, + ], + ) return is_converge, num_func_calls, xk, value, g1 diff --git a/python/paddle/incubate/optimizer/functional/line_search.py b/python/paddle/incubate/optimizer/functional/line_search.py index 375f86ddbe533..94400200f2535 100644 --- a/python/paddle/incubate/optimizer/functional/line_search.py +++ b/python/paddle/incubate/optimizer/functional/line_search.py @@ -31,8 +31,9 @@ def cubic_interpolation_(x1, f1, g1, x2, f2, g2): Returns: min_pos: the minimun point between the specified points in the cubic curve. """ - xmin, xmax = paddle.static.nn.cond(x1 <= x2, lambda: (x1, x2), lambda: - (x2, x1)) + xmin, xmax = paddle.static.nn.cond( + x1 <= x2, lambda: (x1, x2), lambda: (x2, x1) + ) d1 = g1 + g2 - 3 * (f1 - f2) / (x1 - x2) d2_square = d1**2 - g1 * g2 @@ -51,22 +52,24 @@ def false_fn2(): return paddle.minimum(paddle.maximum(min_pos, xmin), xmax) def false_func1(): - return (xmin + xmax) / 2. + return (xmin + xmax) / 2.0 - min_pos = paddle.static.nn.cond(d2_square >= 0., true_func1, false_func1) + min_pos = paddle.static.nn.cond(d2_square >= 0.0, true_func1, false_func1) return min_pos -def strong_wolfe(f, - xk, - pk, - max_iters=20, - tolerance_change=1e-8, - initial_step_length=1.0, - c1=1e-4, - c2=0.9, - alpha_max=10, - dtype='float32'): +def strong_wolfe( + f, + xk, + pk, + max_iters=20, + tolerance_change=1e-8, + initial_step_length=1.0, + c1=1e-4, + c2=0.9, + alpha_max=10, + dtype='float32', +): r"""Implements of line search algorithm that satisfies the strong Wolfe conditions using double zoom. Reference: @@ -143,36 +146,67 @@ def strong_wolfe(f, def phi_and_derphi(a): r"""Compute function value and derivative of phi at a. - phi = f(xk + a * pk) - phi'(a) = f'(xk + a * pk) * pk + phi = f(xk + a * pk) + phi'(a) = f'(xk + a * pk) * pk """ phi_value, f_grad = _value_and_gradient(f, xk + a * pk) phi_grad = paddle.dot(f_grad, pk) # return f_grad to be used in bfgs/l-bfgs to compute yk to avoid computint repeatly. return phi_value, f_grad, phi_grad - def zoom(a_lo, phi_lo, derphi_lo, derf_lo, a_hi, phi_hi, derphi_hi, phi_0, - derphi_0): + def zoom( + a_lo, + phi_lo, + derphi_lo, + derf_lo, + a_hi, + phi_hi, + derphi_hi, + phi_0, + derphi_0, + ): # find the exact a from the bracket [a_lo, a_hi] max_zoom_iters = max_iters j = paddle.full(shape=[1], fill_value=0, dtype='int64') done_zoom = paddle.full(shape=[1], fill_value=False, dtype='bool') - def cond_zoom(j, done_zoom, a_lo, phi_lo, derphi_lo, derf_lo, a_hi, - phi_hi, derphi_hi): + def cond_zoom( + j, + done_zoom, + a_lo, + phi_lo, + derphi_lo, + derf_lo, + a_hi, + phi_hi, + derphi_hi, + ): pred = paddle.abs(a_hi - a_lo) < tolerance_change paddle.assign(done_zoom | pred, done_zoom) return (j < max_zoom_iters) & ~done_zoom - def body_zoom(j, done_zoom, a_lo, phi_lo, derphi_lo, derf_lo, a_hi, - phi_hi, derphi_hi): - aj = cubic_interpolation_(a_lo, phi_lo, derphi_lo, a_hi, phi_hi, - derphi_hi) # 21 + def body_zoom( + j, + done_zoom, + a_lo, + phi_lo, + derphi_lo, + derf_lo, + a_hi, + phi_hi, + derphi_hi, + ): + aj = cubic_interpolation_( + a_lo, phi_lo, derphi_lo, a_hi, phi_hi, derphi_hi + ) # 21 min_change = 0.1 * paddle.abs(a_hi - a_lo) - pred = paddle.minimum(paddle.abs(aj - a_lo), - paddle.abs(aj - a_hi)) < min_change - aj = paddle.static.nn.cond(pred, lambda: 0.5 * (a_lo + a_hi), - lambda: aj) + pred = ( + paddle.minimum(paddle.abs(aj - a_lo), paddle.abs(aj - a_hi)) + < min_change + ) + aj = paddle.static.nn.cond( + pred, lambda: 0.5 * (a_lo + a_hi), lambda: aj + ) phi_j, derf_j, derphi_j = phi_and_derphi(aj) @@ -183,7 +217,7 @@ def true_fn(): paddle.assign(derphi_j, derphi_hi) def false_fn(a_lo, done_zoom): - pred3 = (paddle.abs(derphi_j) <= -c2 * derphi_0) + pred3 = paddle.abs(derphi_j) <= -c2 * derphi_0 paddle.assign(pred3, done_zoom) def true_fn(): @@ -200,26 +234,43 @@ def true_fn(): paddle.assign(derf_j, derf_lo) pred2 = (phi_j > phi_0 + c1 * aj * derphi_0) | (phi_j >= phi_lo) - paddle.static.nn.cond(pred2, true_fn, - lambda: false_fn(a_lo, done_zoom)) + paddle.static.nn.cond( + pred2, true_fn, lambda: false_fn(a_lo, done_zoom) + ) j = paddle.static.nn.cond(done_zoom, lambda: j, lambda: j + 1) return [ - j, done_zoom, a_lo, phi_lo, derphi_lo, derf_lo, a_hi, phi_hi, - derphi_hi + j, + done_zoom, + a_lo, + phi_lo, + derphi_lo, + derf_lo, + a_hi, + phi_hi, + derphi_hi, ] - paddle.static.nn.while_loop(cond=cond_zoom, - body=body_zoom, - loop_vars=[ - j, done_zoom, a_lo, phi_lo, derphi_lo, - derf_lo, a_hi, phi_hi, derphi_hi - ]) + paddle.static.nn.while_loop( + cond=cond_zoom, + body=body_zoom, + loop_vars=[ + j, + done_zoom, + a_lo, + phi_lo, + derphi_lo, + derf_lo, + a_hi, + phi_hi, + derphi_hi, + ], + ) # j is the number of object function called in zoom. return j alpha_max = paddle.full(shape=[1], fill_value=alpha_max, dtype=dtype) - a1 = paddle.full(shape=[1], fill_value=0., dtype=dtype) + a1 = paddle.full(shape=[1], fill_value=0.0, dtype=dtype) a2 = paddle.full(shape=[1], fill_value=initial_step_length, dtype=dtype) phi_1, derf_1, derphi_1 = phi_and_derphi(a1) @@ -245,15 +296,25 @@ def body(i, ls_func_calls, a1, a2, phi_1, derf_1, done): paddle.assign(done | paddle.any(paddle.isinf(phi_2)), done) def true_fn1(): - j = zoom(a1, phi_1, derphi_1, derf_1, a2, phi_2, derphi_2, phi_0, - derphi_0) + j = zoom( + a1, + phi_1, + derphi_1, + derf_1, + a2, + phi_2, + derphi_2, + phi_0, + derphi_0, + ) paddle.assign(a1, a_star) paddle.assign(phi_1, phi_star) paddle.assign(derf_1, derf_star) paddle.assign(ls_func_calls + j, ls_func_calls) - pred1 = ~done & ((phi_2 > phi_0 + c1 * a2 * derphi_0) | - ((phi_2 >= phi_0) & (i > 1))) + pred1 = ~done & ( + (phi_2 > phi_0 + c1 * a2 * derphi_0) | ((phi_2 >= phi_0) & (i > 1)) + ) paddle.assign(done | pred1, done) paddle.static.nn.cond(pred1, true_fn1, None) @@ -267,8 +328,17 @@ def true_fn2(): paddle.static.nn.cond(pred2, true_fn2, None) def true_fn3(): - j = zoom(a2, phi_2, derphi_2, derf_2, a1, phi_1, derphi_1, phi_0, - derphi_0) + j = zoom( + a2, + phi_2, + derphi_2, + derf_2, + a1, + phi_1, + derphi_1, + phi_0, + derphi_0, + ) paddle.assign(a2, a_star) paddle.assign(phi_2, phi_star) paddle.assign(derf_2, derf_star) @@ -291,6 +361,7 @@ def false_fn(): paddle.static.nn.while_loop( cond=cond, body=body, - loop_vars=[i, ls_func_calls, a1, a2, phi_1, derf_1, done]) + loop_vars=[i, ls_func_calls, a1, a2, phi_1, derf_1, done], + ) return a_star, phi_star, derf_star, ls_func_calls diff --git a/python/paddle/incubate/optimizer/functional/utils.py b/python/paddle/incubate/optimizer/functional/utils.py index e6612878e710c..f9d0dddb117df 100644 --- a/python/paddle/incubate/optimizer/functional/utils.py +++ b/python/paddle/incubate/optimizer/functional/utils.py @@ -53,26 +53,29 @@ def raise_func(): else: def create_tmp_var(program, name, dtype, shape): - return program.current_block().create_var(name=name, - dtype=dtype, - shape=shape) + return program.current_block().create_var( + name=name, dtype=dtype, shape=shape + ) - out_var = create_tmp_var(paddle.static.default_main_program(), - name='output', - dtype='float32', - shape=[-1]) + out_var = create_tmp_var( + paddle.static.default_main_program(), + name='output', + dtype='float32', + shape=[-1], + ) def false_fn(): - paddle.static.nn.py_func(func=raise_func, - x=is_symmetric, - out=out_var) + paddle.static.nn.py_func( + func=raise_func, x=is_symmetric, out=out_var + ) paddle.static.nn.cond(is_symmetric, None, false_fn) # eigvals only support cpu paddle.set_device("cpu") eigvals = paddle.paddle.linalg.eigvals(H0) - is_positive = paddle.all(eigvals.real() > 0.) and paddle.all( - eigvals.imag() == 0.) + is_positive = paddle.all(eigvals.real() > 0.0) and paddle.all( + eigvals.imag() == 0.0 + ) paddle.static.nn.cond(is_positive, None, false_fn) diff --git a/python/paddle/incubate/optimizer/lookahead.py b/python/paddle/incubate/optimizer/lookahead.py index dcf50cb5224ed..99a3be4f8e4d2 100644 --- a/python/paddle/incubate/optimizer/lookahead.py +++ b/python/paddle/incubate/optimizer/lookahead.py @@ -115,24 +115,27 @@ def train(layer, loader, loss_fn, opt): _slow_str = "slow" def __init__(self, inner_optimizer, alpha=0.5, k=5, name=None): - assert (inner_optimizer is not None), "inner optimizer can not be None" + assert inner_optimizer is not None, "inner optimizer can not be None" assert ( 0.0 <= alpha <= 1.0 ), "alpha should be larger or equal to 0.0, and less or equal than 1.0" - assert (isinstance(k, int) and k > 0), "k should be a positive integer" + assert isinstance(k, int) and k > 0, "k should be a positive integer" self.inner_optimizer = inner_optimizer if self.inner_optimizer._parameter_list is None: - parameters = framework.default_main_program().global_block( - ).all_parameters() + parameters = ( + framework.default_main_program().global_block().all_parameters() + ) else: parameters = self.inner_optimizer._parameter_list - super(LookAhead, self).__init__(learning_rate=alpha, - parameters=parameters, - weight_decay=None, - grad_clip=None, - name=name) + super(LookAhead, self).__init__( + learning_rate=alpha, + parameters=parameters, + weight_decay=None, + grad_clip=None, + name=name, + ) self.alpha = alpha self.k = k @@ -178,9 +181,9 @@ def step(self): grad_var = param._grad_ivar() params_grads.append((param, grad_var)) - self._apply_optimize(loss=None, - startup_program=None, - params_grads=params_grads) + self._apply_optimize( + loss=None, startup_program=None, params_grads=params_grads + ) def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) @@ -195,24 +198,28 @@ def _increment_global_var(self): shape=[1], value=0, dtype='int32', - persistable=True) + persistable=True, + ) - self.helper.append_op(type='increment', - inputs={'X': [self._global_step_var]}, - outputs={'Out': [self._global_step_var]}, - attrs={'step': 1.0}) + self.helper.append_op( + type='increment', + inputs={'X': [self._global_step_var]}, + outputs={'Out': [self._global_step_var]}, + attrs={'step': 1.0}, + ) def _append_optimize_op(self, block, param_and_grad): one_var = paddle.ones(shape=[1], dtype='int32', name='lookahead_ones') - zero_var = paddle.zeros(shape=[1], - dtype='int32', - name='lookahead_zeros') + zero_var = paddle.zeros( + shape=[1], dtype='int32', name='lookahead_zeros' + ) k_var = layers.create_global_var( name=unique_name.generate("lookahead_k"), shape=[1], value=self.k, dtype='int32', - persistable=True) + persistable=True, + ) mod = paddle.remainder(self._global_step_var, k_var) @@ -235,11 +242,9 @@ def _append_optimize_op(self, block, param_and_grad): paddle.assign(tmp_var_1, slow_var) @imperative_base.no_grad - def minimize(self, - loss, - startup_program=None, - parameters=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameters=None, no_grad_set=None + ): """ Add operations to minimize ``loss`` by updating ``parameters``. @@ -286,12 +291,13 @@ def minimize(self, loss, startup_program=startup_program, parameters=parameters, - no_grad_set=no_grad_set) + no_grad_set=no_grad_set, + ) self._increment_global_var() - _ = self._apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) + _ = self._apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) return optimize_ops, params_grads diff --git a/python/paddle/incubate/optimizer/modelaverage.py b/python/paddle/incubate/optimizer/modelaverage.py index 5fa9dfb67bcb4..fc00f28cb4467 100644 --- a/python/paddle/incubate/optimizer/modelaverage.py +++ b/python/paddle/incubate/optimizer/modelaverage.py @@ -162,17 +162,21 @@ def evaluate(layer, loader, loss_fn): """ - def __init__(self, - average_window_rate, - parameters=None, - min_average_window=10000, - max_average_window=10000, - name=None): - super(ModelAverage, self).__init__(learning_rate=0.0, - parameters=parameters, - weight_decay=None, - grad_clip=None, - name=name) + def __init__( + self, + average_window_rate, + parameters=None, + min_average_window=10000, + max_average_window=10000, + name=None, + ): + super(ModelAverage, self).__init__( + learning_rate=0.0, + parameters=parameters, + weight_decay=None, + grad_clip=None, + name=name, + ) self.helper = LayerHelper(self.__class__.__name__) self.average_window = average_window_rate @@ -182,7 +186,8 @@ def __init__(self, if not framework._non_static_mode(): global_block = framework.default_main_program().global_block() - all_parameters = parameters if parameters else global_block.all_parameters( + all_parameters = ( + parameters if parameters else global_block.all_parameters() ) self._create_accumulators(global_block, all_parameters) @@ -207,18 +212,15 @@ def _create_accumulators(self, block, parameters): self._add_accumulator('sum_2', param) self._add_accumulator('sum_3', param) self._add_accumulator('restore', param) - self._add_accumulator('num_accumulates', - param, - dtype='int64', - shape=[1]) - self._add_accumulator('old_num_accumulates', - param, - dtype='int64', - shape=[1]) - self._add_accumulator('num_updates', - param, - dtype='int64', - shape=[1]) + self._add_accumulator( + 'num_accumulates', param, dtype='int64', shape=[1] + ) + self._add_accumulator( + 'old_num_accumulates', param, dtype='int64', shape=[1] + ) + self._add_accumulator( + 'num_updates', param, dtype='int64', shape=[1] + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -226,26 +228,50 @@ def _append_optimize_op(self, block, param_and_grad): sum_1 = self._get_accumulator('sum_1', param_and_grad[0]) sum_2 = self._get_accumulator('sum_2', param_and_grad[0]) sum_3 = self._get_accumulator('sum_3', param_and_grad[0]) - num_accumulates = self._get_accumulator('num_accumulates', - param_and_grad[0]) - old_num_accumulates = self._get_accumulator('old_num_accumulates', - param_and_grad[0]) + num_accumulates = self._get_accumulator( + 'num_accumulates', param_and_grad[0] + ) + old_num_accumulates = self._get_accumulator( + 'old_num_accumulates', param_and_grad[0] + ) num_updates = self._get_accumulator('num_updates', param_and_grad[0]) if in_dygraph_mode(): _, _, _, _, _, _ = _C_ops.average_accumulates_( - param_and_grad[0], sum_1, sum_2, sum_3, num_accumulates, - old_num_accumulates, num_updates, self.average_window, - self.max_average_window, self.min_average_window) + param_and_grad[0], + sum_1, + sum_2, + sum_3, + num_accumulates, + old_num_accumulates, + num_updates, + self.average_window, + self.max_average_window, + self.min_average_window, + ) return None elif framework._non_static_mode(): _, _, _, _, _, _ = _legacy_C_ops.average_accumulates( - param_and_grad[0], sum_1, sum_2, sum_3, num_accumulates, - old_num_accumulates, num_updates, sum_1, sum_2, sum_3, - num_accumulates, old_num_accumulates, num_updates, - 'average_window', self.average_window, 'min_average_window', - self.min_average_window, 'max_average_window', - self.max_average_window) + param_and_grad[0], + sum_1, + sum_2, + sum_3, + num_accumulates, + old_num_accumulates, + num_updates, + sum_1, + sum_2, + sum_3, + num_accumulates, + old_num_accumulates, + num_updates, + 'average_window', + self.average_window, + 'min_average_window', + self.min_average_window, + 'max_average_window', + self.max_average_window, + ) return None block = framework.default_main_program().global_block() @@ -262,7 +288,7 @@ def _append_optimize_op(self, block, param_and_grad): "in_sum_3": sum_3, "in_num_accumulates": num_accumulates, "in_old_num_accumulates": old_num_accumulates, - "in_num_updates": num_updates + "in_num_updates": num_updates, } outputs = { @@ -274,20 +300,20 @@ def _append_optimize_op(self, block, param_and_grad): "out_num_updates": num_updates, } - average_accumulates_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + average_accumulates_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return average_accumulates_op @imperative_base.no_grad - def minimize(self, - loss, - startup_program=None, - parameters=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameters=None, no_grad_set=None + ): """ Add operations to minimize ``loss`` by updating ``parameters``. @@ -423,10 +449,12 @@ def apply(self, executor=None, need_restore=True): """ if framework._non_static_mode(): for param in self._parameter_list: - num_accumulates = self._get_accumulator('num_accumulates', - param) + num_accumulates = self._get_accumulator( + 'num_accumulates', param + ) old_num_accumulates = self._get_accumulator( - 'old_num_accumulates', param) + 'old_num_accumulates', param + ) sum_1 = self._get_accumulator('sum_1', param) sum_2 = self._get_accumulator('sum_2', param) sum_3 = self._get_accumulator('sum_3', param) @@ -436,8 +464,9 @@ def apply(self, executor=None, need_restore=True): total_param = sum_1 + sum_2 + sum_3 total_accumulates = num_accumulates + old_num_accumulates total_param = paddle.cast(total_param, dtype='float32') - total_accumulates = paddle.cast(total_accumulates, - dtype='float32') + total_accumulates = paddle.cast( + total_accumulates, dtype='float32' + ) average_param = total_param / total_accumulates paddle.assign(average_param, param) try: @@ -448,7 +477,8 @@ def apply(self, executor=None, need_restore=True): return if executor is None: raise RuntimeError( - "Executor should not be None in static graph mode.") + "Executor should not be None in static graph mode." + ) executor.run(self.apply_program) try: yield @@ -504,7 +534,8 @@ def restore(self, executor=None): return if executor is None: raise RuntimeError( - "Executor should not be None in static graph mode.") + "Executor should not be None in static graph mode." + ) executor.run(self.restore_program) def _add_average_apply_op(self, block, param): @@ -514,18 +545,22 @@ def _add_average_apply_op(self, block, param): sum_2 = block._clone_variable(self._get_accumulator('sum_2', param)) sum_3 = block._clone_variable(self._get_accumulator('sum_3', param)) num_accumulates = block._clone_variable( - self._get_accumulator('num_accumulates', param)) + self._get_accumulator('num_accumulates', param) + ) old_num_accumulates = block._clone_variable( - self._get_accumulator('old_num_accumulates', param)) + self._get_accumulator('old_num_accumulates', param) + ) # backup param value to grad layers.assign(input=param, output=grad) # param = (sum_1 + sum_2 + sum_3) / (num_accumulates + old_num_accumulates) tmp = layers.sum(x=[num_accumulates, old_num_accumulates]) sum = layers.sum(x=[sum_1, sum_2, sum_3]) tmp = layers.cast( - x=tmp, dtype='float32' if self._dtype is None else self._dtype) + x=tmp, dtype='float32' if self._dtype is None else self._dtype + ) sum = layers.cast( - x=sum, dtype='float32' if self._dtype is None else self._dtype) + x=sum, dtype='float32' if self._dtype is None else self._dtype + ) layers.ops._elementwise_div(x=sum, y=tmp, out=param) def _add_average_restore_op(self, block, param): diff --git a/python/paddle/incubate/passes/fuse_resnet_unit_pass.py b/python/paddle/incubate/passes/fuse_resnet_unit_pass.py index fce745ffaf8c2..6441427f469d6 100644 --- a/python/paddle/incubate/passes/fuse_resnet_unit_pass.py +++ b/python/paddle/incubate/passes/fuse_resnet_unit_pass.py @@ -21,32 +21,35 @@ def set_resnet_unit_attrs(resnet_unit, has_shortcut): resnet_unit.SetAttr("has_shortcut", has_shortcut) resnet_unit.SetAttr("data_format", 'NHWC') resnet_unit.SetAttr("dilation", 1) - resnet_unit.Attr("stride").MappedPattern(op="conv2d", - name="strides", - element_index=0) - resnet_unit.Attr("padding").MappedPattern(op="conv2d", - name="paddings", - element_index=0) + resnet_unit.Attr("stride").MappedPattern( + op="conv2d", name="strides", element_index=0 + ) + resnet_unit.Attr("padding").MappedPattern( + op="conv2d", name="paddings", element_index=0 + ) resnet_unit.Attr("group").MappedPattern(op="conv2d", name="groups") resnet_unit.Attr("op_device").MappedPattern(op="conv2d", name="op_device") - resnet_unit.Attr("op_namescope").MappedPattern(op="conv2d", - name="op_namescope") + resnet_unit.Attr("op_namescope").MappedPattern( + op="conv2d", name="op_namescope" + ) resnet_unit.Attr("momentum").MappedPattern(op="batch_norm", name="momentum") resnet_unit.Attr("epsilon").MappedPattern(op="batch_norm", name="epsilon") - resnet_unit.Attr("use_global_stats").MappedPattern(op="batch_norm", - name="use_global_stats") + resnet_unit.Attr("use_global_stats").MappedPattern( + op="batch_norm", name="use_global_stats" + ) def set_resnet_unit_outputs(resnet_unit, meanX, varX, meanZ=None, varZ=None): - resnet_unit.SetOutputs(RunningMeanX=meanX, - RunningVarX=varX, - RunningMeanZ=meanZ, - RunningVarZ=varZ) + resnet_unit.SetOutputs( + RunningMeanX=meanX, + RunningVarX=varX, + RunningMeanZ=meanZ, + RunningVarZ=varZ, + ) @ir.RegisterPass def fuse_resnet_unit(): - def pattern_conv_bn(x, filter, scale, bias, mean, var): filter.Attr("shape")[0].Mod(32).EQ(0) filter.Attr("shape")[1].Mod(8).EQ(0) @@ -54,11 +57,9 @@ def pattern_conv_bn(x, filter, scale, bias, mean, var): filter.Attr("shape")[3].EQ(1) conv2d = ir.PassDesc.OP.conv2d(Input=x, Filter=filter) conv2d.SetAttr("data_format", 'NHWC') - bn = ir.PassDesc.OP.batch_norm(X=conv2d, - Bias=bias, - Mean=mean, - Scale=scale, - Variance=var) + bn = ir.PassDesc.OP.batch_norm( + X=conv2d, Bias=bias, Mean=mean, Scale=scale, Variance=var + ) return bn def pattern_one_input(x, filter, scale, bias, mean, var): @@ -67,42 +68,68 @@ def pattern_one_input(x, filter, scale, bias, mean, var): return relu def replace_one_input(x, filter, scale, bias, mean, var): - resnet_unit = ir.PassDesc.OP.resnet_unit(X=x, - FilterX=filter, - ScaleX=scale, - BiasX=bias, - MeanX=mean, - VarX=var) + resnet_unit = ir.PassDesc.OP.resnet_unit( + X=x, FilterX=filter, ScaleX=scale, BiasX=bias, MeanX=mean, VarX=var + ) set_resnet_unit_attrs(resnet_unit, False) set_resnet_unit_outputs(resnet_unit, mean, var) return resnet_unit.Output("Y") - def pattern_two_input(x, filterX, scaleX, biasX, meanX, varX, z, filterZ, - scaleZ, biasZ, meanZ, varZ): + def pattern_two_input( + x, + filterX, + scaleX, + biasX, + meanX, + varX, + z, + filterZ, + scaleZ, + biasZ, + meanZ, + varZ, + ): bnX = pattern_conv_bn(x, filterX, scaleX, biasX, meanX, varX) bnZ = pattern_conv_bn(x, filterZ, scaleZ, biasZ, meanZ, varZ) - ewadd = ir.PassDesc.OP.elementwise_add(X=bnX.Output("Y"), - Y=bnZ.Output("Y")) + ewadd = ir.PassDesc.OP.elementwise_add( + X=bnX.Output("Y"), Y=bnZ.Output("Y") + ) relu = ir.PassDesc.OP.relu(X=ewadd) return relu - def replace_two_input(x, filterX, scaleX, biasX, meanX, varX, z, filterZ, - scaleZ, biasZ, meanZ, varZ): - resnet_unit = ir.PassDesc.OP.resnet_unit(X=x, - FilterX=filterX, - ScaleX=scaleX, - BiasX=biasX, - MeanX=meanX, - VarX=varX, - Z=z, - FilterZ=filterZ, - ScaleZ=scaleZ, - BiasZ=biasZ, - MeanZ=meanZ, - VarZ=varZ) + def replace_two_input( + x, + filterX, + scaleX, + biasX, + meanX, + varX, + z, + filterZ, + scaleZ, + biasZ, + meanZ, + varZ, + ): + resnet_unit = ir.PassDesc.OP.resnet_unit( + X=x, + FilterX=filterX, + ScaleX=scaleX, + BiasX=biasX, + MeanX=meanX, + VarX=varX, + Z=z, + FilterZ=filterZ, + ScaleZ=scaleZ, + BiasZ=biasZ, + MeanZ=meanZ, + VarZ=varZ, + ) set_resnet_unit_attrs(resnet_unit, True) set_resnet_unit_outputs(resnet_unit, meanX, varX, meanZ, varZ) return resnet_unit.Output("Y") - return (pattern_one_input, replace_one_input), (pattern_two_input, - replace_two_input) + return (pattern_one_input, replace_one_input), ( + pattern_two_input, + replace_two_input, + ) diff --git a/python/paddle/incubate/tensor/math.py b/python/paddle/incubate/tensor/math.py index 005d2cee2ddfc..36f4944427f37 100644 --- a/python/paddle/incubate/tensor/math.py +++ b/python/paddle/incubate/tensor/math.py @@ -21,10 +21,12 @@ __all__ = [] -@deprecated(since="2.4.0", - update_to="paddle.geometric.segment_sum", - level=1, - reason="paddle.incubate.segment_sum will be removed in future") +@deprecated( + since="2.4.0", + update_to="paddle.geometric.segment_sum", + level=1, + reason="paddle.incubate.segment_sum will be removed in future", +) def segment_sum(data, segment_ids, name=None): r""" Segment Sum Operator. @@ -59,36 +61,36 @@ def segment_sum(data, segment_ids, name=None): if in_dygraph_mode(): return _C_ops.segment_pool(data, segment_ids, "SUM")[0] if _in_legacy_dygraph(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "SUM") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "SUM" + ) return out - check_variable_and_dtype(data, "X", - ("float32", "float64", "int32", "int64"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + check_variable_and_dtype( + data, "X", ("float32", "float64", "int32", "int64"), "segment_pool" + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_sum", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "SUM"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "SUM"}, + ) return out -@deprecated(since="2.4.0", - update_to="paddle.geometric.segment_mean", - level=1, - reason="paddle.incubate.segment_mean will be removed in future") +@deprecated( + since="2.4.0", + update_to="paddle.geometric.segment_mean", + level=1, + reason="paddle.incubate.segment_mean will be removed in future", +) def segment_mean(data, segment_ids, name=None): r""" Segment mean Operator. @@ -125,36 +127,36 @@ def segment_mean(data, segment_ids, name=None): if in_dygraph_mode(): return _C_ops.segment_pool(data, segment_ids, "MEAN")[0] if _non_static_mode(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "MEAN") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "MEAN" + ) return out - check_variable_and_dtype(data, "X", - ("float32", "float64", "int32", "int64"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + check_variable_and_dtype( + data, "X", ("float32", "float64", "int32", "int64"), "segment_pool" + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_mean", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "MEAN"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "MEAN"}, + ) return out -@deprecated(since="2.4.0", - update_to="paddle.geometric.segment_min", - level=1, - reason="paddle.incubate.segment_min will be removed in future") +@deprecated( + since="2.4.0", + update_to="paddle.geometric.segment_min", + level=1, + reason="paddle.incubate.segment_min will be removed in future", +) def segment_min(data, segment_ids, name=None): r""" Segment min operator. @@ -191,36 +193,36 @@ def segment_min(data, segment_ids, name=None): return _C_ops.segment_pool(data, segment_ids, "MIN")[0] if _non_static_mode(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "MIN") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "MIN" + ) return out - check_variable_and_dtype(data, "X", - ("float32", "float64", "int32", "int64"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + check_variable_and_dtype( + data, "X", ("float32", "float64", "int32", "int64"), "segment_pool" + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_min", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "MIN"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "MIN"}, + ) return out -@deprecated(since="2.4.0", - update_to="paddle.geometric.segment_max", - level=1, - reason="paddle.incubate.segment_max will be removed in future") +@deprecated( + since="2.4.0", + update_to="paddle.geometric.segment_max", + level=1, + reason="paddle.incubate.segment_max will be removed in future", +) def segment_max(data, segment_ids, name=None): r""" Segment max operator. @@ -258,27 +260,25 @@ def segment_max(data, segment_ids, name=None): return out if _non_static_mode(): - out, tmp = _legacy_C_ops.segment_pool(data, segment_ids, 'pooltype', - "MAX") + out, tmp = _legacy_C_ops.segment_pool( + data, segment_ids, 'pooltype', "MAX" + ) return out - check_variable_and_dtype(data, "X", - ("float32", "float64", "int32", "int64"), - "segment_pool") - check_variable_and_dtype(segment_ids, "SegmentIds", ("int32", "int64"), - "segment_pool") + check_variable_and_dtype( + data, "X", ("float32", "float64", "int32", "int64"), "segment_pool" + ) + check_variable_and_dtype( + segment_ids, "SegmentIds", ("int32", "int64"), "segment_pool" + ) helper = LayerHelper("segment_max", **locals()) out = helper.create_variable_for_type_inference(dtype=data.dtype) summed_ids = helper.create_variable_for_type_inference(dtype=data.dtype) - helper.append_op(type="segment_pool", - inputs={ - "X": data, - "SegmentIds": segment_ids - }, - outputs={ - "Out": out, - "SummedIds": summed_ids - }, - attrs={"pooltype": "MAX"}) + helper.append_op( + type="segment_pool", + inputs={"X": data, "SegmentIds": segment_ids}, + outputs={"Out": out, "SummedIds": summed_ids}, + attrs={"pooltype": "MAX"}, + ) return out diff --git a/python/paddle/incubate/xpu/resnet_block.py b/python/paddle/incubate/xpu/resnet_block.py index 1eb3637fa5851..148ea5a5e919f 100644 --- a/python/paddle/incubate/xpu/resnet_block.py +++ b/python/paddle/incubate/xpu/resnet_block.py @@ -24,106 +24,232 @@ __all__ = ['resnet_basic_block', 'ResNetBasicBlock'] -def resnet_basic_block(x, - filter1, - scale1, - bias1, - mean1, - var1, - filter2, - scale2, - bias2, - mean2, - var2, - filter3, - scale3, - bias3, - mean3, - var3, - stride1, - stride2, - stride3, - padding1, - padding2, - padding3, - dilation1, - dilation2, - dilation3, - groups, - momentum, - eps, - data_format, - has_shortcut, - use_global_stats=None, - training=False, - trainable_statistics=False, - find_conv_max=True): +def resnet_basic_block( + x, + filter1, + scale1, + bias1, + mean1, + var1, + filter2, + scale2, + bias2, + mean2, + var2, + filter3, + scale3, + bias3, + mean3, + var3, + stride1, + stride2, + stride3, + padding1, + padding2, + padding3, + dilation1, + dilation2, + dilation3, + groups, + momentum, + eps, + data_format, + has_shortcut, + use_global_stats=None, + training=False, + trainable_statistics=False, + find_conv_max=True, +): if fluid.framework._non_static_mode(): - attrs = ('stride1', stride1, 'stride2', stride2, 'stride3', stride3, - 'padding1', padding1, 'padding2', padding2, 'padding3', - padding3, 'dilation1', dilation1, 'dilation2', dilation2, - 'dilation3', dilation3, 'group', groups, 'momentum', momentum, - 'epsilon', eps, 'data_format', data_format, 'has_shortcut', - has_shortcut, 'use_global_stats', use_global_stats, - "trainable_statistics", trainable_statistics, 'is_test', - not training, 'act_type', "relu", 'find_conv_input_max', - find_conv_max) + attrs = ( + 'stride1', + stride1, + 'stride2', + stride2, + 'stride3', + stride3, + 'padding1', + padding1, + 'padding2', + padding2, + 'padding3', + padding3, + 'dilation1', + dilation1, + 'dilation2', + dilation2, + 'dilation3', + dilation3, + 'group', + groups, + 'momentum', + momentum, + 'epsilon', + eps, + 'data_format', + data_format, + 'has_shortcut', + has_shortcut, + 'use_global_stats', + use_global_stats, + "trainable_statistics", + trainable_statistics, + 'is_test', + not training, + 'act_type', + "relu", + 'find_conv_input_max', + find_conv_max, + ) - out, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _ = \ - _legacy_C_ops.resnet_basic_block(x, filter1, scale1, bias1, mean1, var1, filter2, scale2, bias2, mean2, var2, \ - filter3, scale3, bias3, mean3, var3, mean1, var1, mean2, var2, mean3, var3, *attrs) + ( + out, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + _, + ) = _legacy_C_ops.resnet_basic_block( + x, + filter1, + scale1, + bias1, + mean1, + var1, + filter2, + scale2, + bias2, + mean2, + var2, + filter3, + scale3, + bias3, + mean3, + var3, + mean1, + var1, + mean2, + var2, + mean3, + var3, + *attrs + ) return out helper = LayerHelper('resnet_basic_block', **locals()) bn_param_dtype = fluid.core.VarDesc.VarType.FP32 max_dtype = fluid.core.VarDesc.VarType.FP32 - out = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) - conv1 = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) + out = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) + conv1 = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) saved_mean1 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) saved_invstd1 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) - running_mean1 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if mean1 is None else mean1 - running_var1 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if var1 is None else var1 - conv2 = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) - conv2_input = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) + running_mean1 = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if mean1 is None + else mean1 + ) + running_var1 = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if var1 is None + else var1 + ) + conv2 = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) + conv2_input = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) saved_mean2 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) saved_invstd2 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) - running_mean2 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if mean2 is None else mean2 - running_var2 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if var2 is None else var2 - conv3 = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) + running_mean2 = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if mean2 is None + else mean2 + ) + running_var2 = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if var2 is None + else var2 + ) + conv3 = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) saved_mean3 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) + dtype=bn_param_dtype, stop_gradient=True + ) saved_invstd3 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) - running_mean3 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if mean3 is None else mean3 - running_var3 = helper.create_variable_for_type_inference( - dtype=bn_param_dtype, stop_gradient=True) if var3 is None else var3 + dtype=bn_param_dtype, stop_gradient=True + ) + running_mean3 = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if mean3 is None + else mean3 + ) + running_var3 = ( + helper.create_variable_for_type_inference( + dtype=bn_param_dtype, stop_gradient=True + ) + if var3 is None + else var3 + ) conv1_input_max = helper.create_variable_for_type_inference( - dtype=max_dtype, stop_gradient=True) + dtype=max_dtype, stop_gradient=True + ) conv1_filter_max = helper.create_variable_for_type_inference( - dtype=max_dtype, stop_gradient=True) + dtype=max_dtype, stop_gradient=True + ) conv2_input_max = helper.create_variable_for_type_inference( - dtype=max_dtype, stop_gradient=True) + dtype=max_dtype, stop_gradient=True + ) conv2_filter_max = helper.create_variable_for_type_inference( - dtype=max_dtype, stop_gradient=True) + dtype=max_dtype, stop_gradient=True + ) conv3_input_max = helper.create_variable_for_type_inference( - dtype=max_dtype, stop_gradient=True) + dtype=max_dtype, stop_gradient=True + ) conv3_filter_max = helper.create_variable_for_type_inference( - dtype=max_dtype, stop_gradient=True) + dtype=max_dtype, stop_gradient=True + ) inputs = { 'X': x, @@ -163,7 +289,7 @@ def resnet_basic_block(x, "trainable_statistics": trainable_statistics, 'is_test': not training, 'act_type': "relu", - 'find_conv_input_max': find_conv_max + 'find_conv_input_max': find_conv_max, } outputs = { @@ -191,10 +317,9 @@ def resnet_basic_block(x, 'MaxInput3': conv3_input_max, 'MaxFilter3': conv3_filter_max, } - helper.append_op(type='resnet_basic_block', - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='resnet_basic_block', inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -303,56 +428,60 @@ class ResNetBasicBlock(Layer): print(out.shape) # [2, 8, 16, 16] """ - def __init__(self, - num_channels1, - num_filter1, - filter1_size, - num_channels2, - num_filter2, - filter2_size, - num_channels3, - num_filter3, - filter3_size, - stride1=1, - stride2=1, - stride3=1, - act='relu', - momentum=0.9, - eps=1e-5, - data_format='NCHW', - has_shortcut=False, - use_global_stats=False, - is_test=False, - filter1_attr=None, - scale1_attr=None, - bias1_attr=None, - moving_mean1_name=None, - moving_var1_name=None, - filter2_attr=None, - scale2_attr=None, - bias2_attr=None, - moving_mean2_name=None, - moving_var2_name=None, - filter3_attr=None, - scale3_attr=None, - bias3_attr=None, - moving_mean3_name=None, - moving_var3_name=None, - padding1=0, - padding2=0, - padding3=0, - dilation1=1, - dilation2=1, - dilation3=1, - trainable_statistics=False, - find_conv_max=True): + def __init__( + self, + num_channels1, + num_filter1, + filter1_size, + num_channels2, + num_filter2, + filter2_size, + num_channels3, + num_filter3, + filter3_size, + stride1=1, + stride2=1, + stride3=1, + act='relu', + momentum=0.9, + eps=1e-5, + data_format='NCHW', + has_shortcut=False, + use_global_stats=False, + is_test=False, + filter1_attr=None, + scale1_attr=None, + bias1_attr=None, + moving_mean1_name=None, + moving_var1_name=None, + filter2_attr=None, + scale2_attr=None, + bias2_attr=None, + moving_mean2_name=None, + moving_var2_name=None, + filter3_attr=None, + scale3_attr=None, + bias3_attr=None, + moving_mean3_name=None, + moving_var3_name=None, + padding1=0, + padding2=0, + padding3=0, + dilation1=1, + dilation2=1, + dilation3=1, + trainable_statistics=False, + find_conv_max=True, + ): super(ResNetBasicBlock, self).__init__() self._stride1 = stride1 self._stride2 = stride2 - self._kernel1_size = utils.convert_to_list(filter1_size, 2, - 'filter1_size') - self._kernel2_size = utils.convert_to_list(filter2_size, 2, - 'filter2_size') + self._kernel1_size = utils.convert_to_list( + filter1_size, 2, 'filter1_size' + ) + self._kernel2_size = utils.convert_to_list( + filter2_size, 2, 'filter2_size' + ) self._dilation1 = dilation1 self._dilation2 = dilation2 self._padding1 = padding1 @@ -369,8 +498,9 @@ def __init__(self, self._find_conv_max = find_conv_max if has_shortcut: - self._kernel3_size = utils.convert_to_list(filter3_size, 2, - 'filter3_size') + self._kernel3_size = utils.convert_to_list( + filter3_size, 2, 'filter3_size' + ) self._padding3 = padding3 self._stride3 = stride3 self._dilation3 = dilation3 @@ -385,11 +515,13 @@ def __init__(self, if data_format not in valid_format: raise ValueError( "conv_format must be one of {}, but got conv_format={}".format( - valid_format, data_format)) + valid_format, data_format + ) + ) def _get_default_param_initializer(channels, kernel_size): filter_elem_num = np.prod(kernel_size) * channels - std = (2.0 / filter_elem_num)**0.5 + std = (2.0 / filter_elem_num) ** 0.5 return I.Normal(0.0, std) # init filter @@ -403,92 +535,128 @@ def _get_default_param_initializer(channels, kernel_size): shape=filter1_shape, attr=filter1_attr, default_initializer=_get_default_param_initializer( - num_channels1, self._kernel1_size)) + num_channels1, self._kernel1_size + ), + ) self.scale_1 = self.create_parameter( shape=bn1_param_shape, attr=scale1_attr, dtype=bn_param_dtype, - default_initializer=I.Constant(1.0)) - self.bias_1 = self.create_parameter(shape=bn1_param_shape, - attr=bias1_attr, - dtype=bn_param_dtype, - is_bias=True) - self.mean_1 = self.create_parameter(attr=ParamAttr( - name=moving_mean1_name, - initializer=I.Constant(0.0), - trainable=False), - shape=bn1_param_shape, - dtype=bn_param_dtype) + default_initializer=I.Constant(1.0), + ) + self.bias_1 = self.create_parameter( + shape=bn1_param_shape, + attr=bias1_attr, + dtype=bn_param_dtype, + is_bias=True, + ) + self.mean_1 = self.create_parameter( + attr=ParamAttr( + name=moving_mean1_name, + initializer=I.Constant(0.0), + trainable=False, + ), + shape=bn1_param_shape, + dtype=bn_param_dtype, + ) self.mean_1.stop_gradient = True self.var_1 = self.create_parameter( - attr=ParamAttr(name=moving_var1_name, - initializer=I.Constant(1.0), - trainable=False), + attr=ParamAttr( + name=moving_var1_name, + initializer=I.Constant(1.0), + trainable=False, + ), shape=bn1_param_shape, - dtype=bn_param_dtype) + dtype=bn_param_dtype, + ) self.var_1.stop_gradient = True self.filter_2 = self.create_parameter( shape=filter2_shape, attr=filter2_attr, default_initializer=_get_default_param_initializer( - num_channels2, self._kernel2_size)) + num_channels2, self._kernel2_size + ), + ) self.scale_2 = self.create_parameter( shape=bn2_param_shape, attr=scale2_attr, dtype=bn_param_dtype, - default_initializer=I.Constant(1.0)) - self.bias_2 = self.create_parameter(shape=bn2_param_shape, - attr=bias2_attr, - dtype=bn_param_dtype, - is_bias=True) - self.mean_2 = self.create_parameter(attr=ParamAttr( - name=moving_mean2_name, - initializer=I.Constant(0.0), - trainable=False), - shape=bn2_param_shape, - dtype=bn_param_dtype) + default_initializer=I.Constant(1.0), + ) + self.bias_2 = self.create_parameter( + shape=bn2_param_shape, + attr=bias2_attr, + dtype=bn_param_dtype, + is_bias=True, + ) + self.mean_2 = self.create_parameter( + attr=ParamAttr( + name=moving_mean2_name, + initializer=I.Constant(0.0), + trainable=False, + ), + shape=bn2_param_shape, + dtype=bn_param_dtype, + ) self.mean_2.stop_gradient = True self.var_2 = self.create_parameter( - attr=ParamAttr(name=moving_var2_name, - initializer=I.Constant(1.0), - trainable=False), + attr=ParamAttr( + name=moving_var2_name, + initializer=I.Constant(1.0), + trainable=False, + ), shape=bn2_param_shape, - dtype=bn_param_dtype) + dtype=bn_param_dtype, + ) self.var_2.stop_gradient = True if has_shortcut: bn3_param_shape = [1, 1, num_filter3] filter3_shape = [ - num_filter3, num_channels3, filter3_size, filter3_size + num_filter3, + num_channels3, + filter3_size, + filter3_size, ] self.filter_3 = self.create_parameter( shape=filter3_shape, attr=filter3_attr, default_initializer=_get_default_param_initializer( - num_channels3, self._kernel3_size)) + num_channels3, self._kernel3_size + ), + ) self.scale_3 = self.create_parameter( shape=bn3_param_shape, attr=scale3_attr, dtype=bn_param_dtype, - default_initializer=I.Constant(1.0)) - self.bias_3 = self.create_parameter(shape=bn3_param_shape, - attr=bias3_attr, - dtype=bn_param_dtype, - is_bias=True) - self.mean_3 = self.create_parameter(attr=ParamAttr( - name=moving_mean3_name, - initializer=I.Constant(0.0), - trainable=False), - shape=bn3_param_shape, - dtype=bn_param_dtype) + default_initializer=I.Constant(1.0), + ) + self.bias_3 = self.create_parameter( + shape=bn3_param_shape, + attr=bias3_attr, + dtype=bn_param_dtype, + is_bias=True, + ) + self.mean_3 = self.create_parameter( + attr=ParamAttr( + name=moving_mean3_name, + initializer=I.Constant(0.0), + trainable=False, + ), + shape=bn3_param_shape, + dtype=bn_param_dtype, + ) self.mean_3.stop_gradient = True - self.var_3 = self.create_parameter(attr=ParamAttr( - name=moving_var3_name, - initializer=I.Constant(1.0), - trainable=False), - shape=bn3_param_shape, - dtype=bn_param_dtype) + self.var_3 = self.create_parameter( + attr=ParamAttr( + name=moving_var3_name, + initializer=I.Constant(1.0), + trainable=False, + ), + shape=bn3_param_shape, + dtype=bn_param_dtype, + ) self.var_3.stop_gradient = True else: self.filter_3 = None @@ -532,5 +700,6 @@ def forward(self, x): use_global_stats=self._use_global_stats, training=self.training, trainable_statistics=self._trainable_statistics, - find_conv_max=self._find_conv_max) + find_conv_max=self._find_conv_max, + ) return out diff --git a/python/paddle/inference/__init__.py b/python/paddle/inference/__init__.py index 1b9a96f437bbd..359f5caccb0fa 100644 --- a/python/paddle/inference/__init__.py +++ b/python/paddle/inference/__init__.py @@ -28,8 +28,18 @@ from ..fluid.inference import PredictorPool # noqa: F401 __all__ = [ # noqa - 'Config', 'DataType', 'PlaceType', 'PrecisionType', 'Tensor', 'Predictor', - 'create_predictor', 'get_version', '_get_phi_kernel_name', - 'get_trt_compile_version', 'convert_to_mixed_precision', - 'get_trt_runtime_version', 'get_num_bytes_of_data_type', 'PredictorPool' + 'Config', + 'DataType', + 'PlaceType', + 'PrecisionType', + 'Tensor', + 'Predictor', + 'create_predictor', + 'get_version', + '_get_phi_kernel_name', + 'get_trt_compile_version', + 'convert_to_mixed_precision', + 'get_trt_runtime_version', + 'get_num_bytes_of_data_type', + 'PredictorPool', ] diff --git a/python/paddle/io/__init__.py b/python/paddle/io/__init__.py index 87acda904b5da..a9c0e9a2f2d2f 100755 --- a/python/paddle/io/__init__.py +++ b/python/paddle/io/__init__.py @@ -30,9 +30,20 @@ from ..fluid.dataloader import Subset # noqa: F401 from ..fluid.dataloader import random_split # noqa: F401 -__all__ = [ #noqa - 'Dataset', 'IterableDataset', 'TensorDataset', 'ComposeDataset', - 'ChainDataset', 'BatchSampler', 'DistributedBatchSampler', 'DataLoader', - 'get_worker_info', 'Sampler', 'SequenceSampler', 'RandomSampler', - 'WeightedRandomSampler', 'random_split', 'Subset' +__all__ = [ # noqa + 'Dataset', + 'IterableDataset', + 'TensorDataset', + 'ComposeDataset', + 'ChainDataset', + 'BatchSampler', + 'DistributedBatchSampler', + 'DataLoader', + 'get_worker_info', + 'Sampler', + 'SequenceSampler', + 'RandomSampler', + 'WeightedRandomSampler', + 'random_split', + 'Subset', ] diff --git a/python/paddle/jit/__init__.py b/python/paddle/jit/__init__.py index 857a5d8695a75..a706d4d121102 100644 --- a/python/paddle/jit/__init__.py +++ b/python/paddle/jit/__init__.py @@ -26,6 +26,13 @@ from . import dy2static # noqa: F401 __all__ = [ # noqa - 'save', 'load', 'TracedLayer', 'to_static', 'ProgramTranslator', - 'TranslatedLayer', 'set_code_level', 'set_verbosity', 'not_to_static' + 'save', + 'load', + 'TracedLayer', + 'to_static', + 'ProgramTranslator', + 'TranslatedLayer', + 'set_code_level', + 'set_verbosity', + 'not_to_static', ] diff --git a/python/paddle/jit/dy2static/convert_call_func.py b/python/paddle/jit/dy2static/convert_call_func.py index 247efa8f4f15b..17e33f53bca72 100644 --- a/python/paddle/jit/dy2static/convert_call_func.py +++ b/python/paddle/jit/dy2static/convert_call_func.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.dygraph.dygraph_to_static.convert_call_func import convert_call # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_call_func import ( # noqa: F401 + convert_call, +) __all__ = [] diff --git a/python/paddle/jit/dy2static/convert_operators.py b/python/paddle/jit/dy2static/convert_operators.py index 3a941a1dff319..54210302c4178 100644 --- a/python/paddle/jit/dy2static/convert_operators.py +++ b/python/paddle/jit/dy2static/convert_operators.py @@ -12,19 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.dygraph.dygraph_to_static.convert_operators import cast_bool_if_necessary # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_assert # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_ifelse # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_len # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_logical_and # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_logical_not # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_logical_or # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_pop # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_print # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_shape_compare # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_var_dtype # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_shape # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import convert_while_loop # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.convert_operators import unpack_by_structure, indexable, convert_attr # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.convert_operators import ( # noqa: F401 + cast_bool_if_necessary, + convert_assert, + convert_ifelse, + convert_len, + convert_logical_and, + convert_logical_not, + convert_logical_or, + convert_pop, + convert_print, + convert_shape_compare, + convert_var_dtype, + convert_shape, + convert_while_loop, + unpack_by_structure, + indexable, + convert_attr, +) __all__ = [] diff --git a/python/paddle/jit/dy2static/variable_trans_func.py b/python/paddle/jit/dy2static/variable_trans_func.py index 582e677a50326..88f8cd3c2c464 100644 --- a/python/paddle/jit/dy2static/variable_trans_func.py +++ b/python/paddle/jit/dy2static/variable_trans_func.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.dygraph.dygraph_to_static.variable_trans_func import create_bool_as_type # noqa: F401 -from ...fluid.dygraph.dygraph_to_static.variable_trans_func import to_static_variable # noqa: F401 +from ...fluid.dygraph.dygraph_to_static.variable_trans_func import ( # noqa: F401 + create_bool_as_type, + to_static_variable, +) __all__ = [] diff --git a/python/paddle/jit/layer.py b/python/paddle/jit/layer.py index 97b598948500b..25d9ca5dbcceb 100644 --- a/python/paddle/jit/layer.py +++ b/python/paddle/jit/layer.py @@ -18,7 +18,6 @@ class Layer(object): - def __init__(self): self.cpp_layer = None # {name: Function} @@ -34,8 +33,7 @@ def load(self, load_path, place): setattr(self, name, self.functions[name]) -class Function(): - +class Function: def __init__(self, function, info): self.function = function self.info = FunctionInfo(info) @@ -44,8 +42,7 @@ def __call__(self, *args): return core.eager.jit_function_call(self.function, args) -class FunctionInfo(): - +class FunctionInfo: def __init__(self, info): self.info = info diff --git a/python/paddle/linalg.py b/python/paddle/linalg.py index 834b631e5c519..9ea43479569b4 100644 --- a/python/paddle/linalg.py +++ b/python/paddle/linalg.py @@ -38,7 +38,7 @@ from .tensor.linalg import lstsq __all__ = [ - 'cholesky', #noqa + 'cholesky', # noqa 'norm', 'cond', 'cov', @@ -61,5 +61,5 @@ 'solve', 'cholesky_solve', 'triangular_solve', - 'lstsq' + 'lstsq', ] diff --git a/python/paddle/metric/__init__.py b/python/paddle/metric/__init__.py index 70fe075e57744..60dff58ec48bf 100644 --- a/python/paddle/metric/__init__.py +++ b/python/paddle/metric/__init__.py @@ -19,6 +19,11 @@ from .metrics import Auc # noqa: F401 from .metrics import accuracy # noqa: F401 -__all__ = [ #noqa - 'Metric', 'Accuracy', 'Precision', 'Recall', 'Auc', 'accuracy' +__all__ = [ # noqa + 'Metric', + 'Accuracy', + 'Precision', + 'Recall', + 'Auc', + 'accuracy', ] diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py index 1404c7531b4ee..e8493b17b9256 100644 --- a/python/paddle/metric/metrics.py +++ b/python/paddle/metric/metrics.py @@ -116,7 +116,9 @@ def reset(self): """ raise NotImplementedError( "function 'reset' not implemented in {}.".format( - self.__class__.__name__)) + self.__class__.__name__ + ) + ) @abc.abstractmethod def update(self, *args): @@ -132,7 +134,9 @@ def update(self, *args): """ raise NotImplementedError( "function 'update' not implemented in {}.".format( - self.__class__.__name__)) + self.__class__.__name__ + ) + ) @abc.abstractmethod def accumulate(self): @@ -141,7 +145,9 @@ def accumulate(self): """ raise NotImplementedError( "function 'accumulate' not implemented in {}.".format( - self.__class__.__name__)) + self.__class__.__name__ + ) + ) @abc.abstractmethod def name(self): @@ -150,7 +156,9 @@ def name(self): """ raise NotImplementedError( "function 'name' not implemented in {}.".format( - self.__class__.__name__)) + self.__class__.__name__ + ) + ) def compute(self, *args): """ @@ -231,7 +239,7 @@ class Accuracy(Metric): """ - def __init__(self, topk=(1, ), name=None, *args, **kwargs): + def __init__(self, topk=(1,), name=None, *args, **kwargs): super(Accuracy, self).__init__(*args, **kwargs) self.topk = topk self.maxk = max(topk) @@ -253,12 +261,12 @@ def compute(self, pred, label, *args): Tensor: Correct mask, a tensor with shape [batch_size, d0, ..., topk]. """ pred = paddle.argsort(pred, descending=True) - pred = paddle.slice(pred, - axes=[len(pred.shape) - 1], - starts=[0], - ends=[self.maxk]) - if (len(label.shape) == 1) or \ - (len(label.shape) == 2 and label.shape[-1] == 1): + pred = paddle.slice( + pred, axes=[len(pred.shape) - 1], starts=[0], ends=[self.maxk] + ) + if (len(label.shape) == 1) or ( + len(label.shape) == 2 and label.shape[-1] == 1 + ): # In static mode, the real label data shape may be different # from shape defined by paddle.static.InputSpec in model # building, reshape to the right shape. @@ -297,7 +305,7 @@ def reset(self): """ Resets all of the metric state. """ - self.total = [0.] * len(self.topk) + self.total = [0.0] * len(self.topk) self.count = [0] * len(self.topk) def accumulate(self): @@ -306,7 +314,7 @@ def accumulate(self): """ res = [] for t, c in zip(self.total, self.count): - r = float(t) / c if c > 0 else 0. + r = float(t) / c if c > 0 else 0.0 res.append(r) res = res[0] if len(self.topk) == 1 else res return res @@ -446,7 +454,7 @@ def accumulate(self): A scaler float: results of the calculated precision. """ ap = self.tp + self.fp - return float(self.tp) / ap if ap != 0 else .0 + return float(self.tp) / ap if ap != 0 else 0.0 def name(self): """ @@ -572,7 +580,7 @@ def accumulate(self): A scaler float: results of the calculated Recall. """ recall = self.tp + self.fn - return float(self.tp) / recall if recall != 0 else .0 + return float(self.tp) / recall if recall != 0 else 0.0 def reset(self): """ @@ -670,12 +678,9 @@ def loss(x, y): model.fit(data, batch_size=16) """ - def __init__(self, - curve='ROC', - num_thresholds=4095, - name='auc', - *args, - **kwargs): + def __init__( + self, curve='ROC', num_thresholds=4095, name='auc', *args, **kwargs + ): super(Auc, self).__init__(*args, **kwargs) self._curve = curve self._num_thresholds = num_thresholds @@ -737,11 +742,14 @@ def accumulate(self): tot_neg_prev = tot_neg tot_pos += self._stat_pos[idx] tot_neg += self._stat_neg[idx] - auc += self.trapezoid_area(tot_neg, tot_neg_prev, tot_pos, - tot_pos_prev) + auc += self.trapezoid_area( + tot_neg, tot_neg_prev, tot_pos, tot_pos_prev + ) idx -= 1 - return auc / tot_pos / tot_neg if tot_pos > 0.0 and tot_neg > 0.0 else 0.0 + return ( + auc / tot_pos / tot_neg if tot_pos > 0.0 and tot_neg > 0.0 else 0.0 + ) def reset(self): """ @@ -799,29 +807,29 @@ def accuracy(input, label, k=1, correct=None, total=None, name=None): total = _varbase_creator(dtype="int32") topk_out, topk_indices = paddle.topk(input, k=k) - _acc, _, _ = _legacy_C_ops.accuracy(topk_out, topk_indices, label, - correct, total) + _acc, _, _ = _legacy_C_ops.accuracy( + topk_out, topk_indices, label, correct, total + ) return _acc helper = LayerHelper("accuracy", **locals()) - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'accuracy') + check_variable_and_dtype( + input, 'input', ['float16', 'float32', 'float64'], 'accuracy' + ) topk_out, topk_indices = paddle.topk(input, k=k) acc_out = helper.create_variable_for_type_inference(dtype="float32") if correct is None: correct = helper.create_variable_for_type_inference(dtype="int32") if total is None: total = helper.create_variable_for_type_inference(dtype="int32") - helper.append_op(type="accuracy", - inputs={ - "Out": [topk_out], - "Indices": [topk_indices], - "Label": [label] - }, - outputs={ - "Accuracy": [acc_out], - "Correct": [correct], - "Total": [total], - }) + helper.append_op( + type="accuracy", + inputs={"Out": [topk_out], "Indices": [topk_indices], "Label": [label]}, + outputs={ + "Accuracy": [acc_out], + "Correct": [correct], + "Total": [total], + }, + ) return acc_out diff --git a/python/paddle/nn/__init__.py b/python/paddle/nn/__init__.py index 331131d6e2319..8e02cdffd5e05 100644 --- a/python/paddle/nn/__init__.py +++ b/python/paddle/nn/__init__.py @@ -163,35 +163,41 @@ import paddle.utils.deprecated as deprecated -@deprecated(since="2.0.0", - update_to="paddle.nn.funcitional.diag_embed", - level=1, - reason="diag_embed in paddle.nn will be removed in future") +@deprecated( + since="2.0.0", + update_to="paddle.nn.funcitional.diag_embed", + level=1, + reason="diag_embed in paddle.nn will be removed in future", +) def diag_embed(*args): ''' - alias name of paddle.nn.functional.diag_embed + alias name of paddle.nn.functional.diag_embed ''' return functional.diag_embed(*args) -@deprecated(since="2.0.0", - update_to="paddle.nn.utils.remove_weight_norm", - level=1, - reason="remove_weight_norm in paddle.nn will be removed in future") +@deprecated( + since="2.0.0", + update_to="paddle.nn.utils.remove_weight_norm", + level=1, + reason="remove_weight_norm in paddle.nn will be removed in future", +) def remove_weight_norm(*args): ''' - alias name of paddle.nn.utils.remove_weight_norm + alias name of paddle.nn.utils.remove_weight_norm ''' return utils.remove_weight_norm(*args) -@deprecated(since="2.0.0", - update_to="paddle.nn.utils.weight_norm", - level=1, - reason="weight_norm in paddle.nn will be removed in future") +@deprecated( + since="2.0.0", + update_to="paddle.nn.utils.weight_norm", + level=1, + reason="weight_norm in paddle.nn will be removed in future", +) def weight_norm(*args): ''' - alias name of paddle.nn.utils.weight_norm + alias name of paddle.nn.utils.weight_norm ''' return utils.weight_norm(*args) diff --git a/python/paddle/nn/functional/activation.py b/python/paddle/nn/functional/activation.py index 929fd24371561..254ab7ff8a005 100644 --- a/python/paddle/nn/functional/activation.py +++ b/python/paddle/nn/functional/activation.py @@ -68,10 +68,12 @@ def celu(x, alpha=1.0, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'celu') helper = LayerHelper("celu", **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='celu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'alpha': alpha}) + helper.append_op( + type='celu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'alpha': alpha}, + ) return out @@ -118,10 +120,12 @@ def elu(x, alpha=1.0, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'elu') helper = LayerHelper("elu", **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='elu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'alpha': alpha}) + helper.append_op( + type='elu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'alpha': alpha}, + ) return out @@ -131,7 +135,7 @@ def elu_(x, alpha=1.0, name=None): Inplace version of ``elu`` API, the output Tensor will be inplaced with input ``x``. Please refer to :ref:`api_nn_cn_elu`. """ - assert alpha >= 0., "elu_ only support alpha >= 0, please use elu instead." + assert alpha >= 0.0, "elu_ only support alpha >= 0, please use elu instead." if in_dygraph_mode(): return _C_ops.elu_(x, alpha) return _legacy_C_ops.elu_(x, 'alpha', alpha) @@ -187,10 +191,12 @@ def gelu(x, approximate=False, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'gelu') helper = LayerHelper("gelu", **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='gelu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'approximate': approximate}) + helper.append_op( + type='gelu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'approximate': approximate}, + ) return out @@ -233,14 +239,17 @@ def hardshrink(x, threshold=0.5, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.hard_shrink(x, 'threshold', threshold) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'hardshrink') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'hardshrink' + ) helper = LayerHelper('hardshrink', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='hard_shrink', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'threshold': threshold}) + helper.append_op( + type='hard_shrink', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'threshold': threshold}, + ) return out @@ -284,18 +293,18 @@ def hardtanh(x, min=-1.0, max=1.0, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.brelu(x, 't_min', min, 't_max', max) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'hardtanh') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'hardtanh' + ) helper = LayerHelper('hardtanh', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='brelu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 't_min': min, - 't_max': max - }) + helper.append_op( + type='brelu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'t_min': min, 't_max': max}, + ) return out @@ -341,18 +350,18 @@ def hardsigmoid(x, slope=0.1666667, offset=0.5, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.hard_sigmoid(x, 'slope', slope, 'offset', offset) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'hardsigmoid') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'hardsigmoid' + ) helper = LayerHelper('hardsigmoid', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='hard_sigmoid', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'slope': slope, - 'offset': offset - }) + helper.append_op( + type='hard_sigmoid', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'slope': slope, 'offset': offset}, + ) return out @@ -395,8 +404,9 @@ def hardswish(x, name=None): if in_dygraph_mode(): return _C_ops.hardswish(x, 6, 6, 3) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'hardswish') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'hardswish' + ) helper = LayerHelper('hardswish', **locals()) out = helper.create_variable_for_type_inference(x.dtype) @@ -444,14 +454,17 @@ def leaky_relu(x, negative_slope=0.01, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.leaky_relu(x, 'alpha', negative_slope) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'leaky_relu') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'leaky_relu' + ) helper = LayerHelper('leaky_relu', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='leaky_relu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'alpha': negative_slope}) + helper.append_op( + type='leaky_relu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'alpha': negative_slope}, + ) return out @@ -498,60 +511,68 @@ def prelu(x, weight, data_format="NCHW", name=None): # [ 6. , 7. , 8. , 9. ]]]] """ check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'prelu') - check_variable_and_dtype(weight, 'weight', - ['float16', 'float32', 'float64'], 'prelu') + check_variable_and_dtype( + weight, 'weight', ['float16', 'float32', 'float64'], 'prelu' + ) - assert len(weight.shape - ) == 1, "The dim count of weight shape should be 1 in prelu()." + assert ( + len(weight.shape) == 1 + ), "The dim count of weight shape should be 1 in prelu()." mode = 'all' if weight.shape[0] > 1: true_data_format = [ - 'NC', 'NCL', 'NCHW', 'NCDHW', 'NLC', 'NHWC', 'NDHWC' + 'NC', + 'NCL', + 'NCHW', + 'NCDHW', + 'NLC', + 'NHWC', + 'NDHWC', ] if data_format not in true_data_format: raise ValueError( "data_format must be one of 'NC', 'NCL', 'NCHW', 'NCDHW', " - "'NLC', 'NHWC', 'NDHWC' but receive {}".format(data_format)) + "'NLC', 'NHWC', 'NDHWC' but receive {}".format(data_format) + ) data_format = 'NCHW' if data_format[1] == 'C' else 'NHWC' - assert len( - x.shape - ) > 1, "The dim count of x should be equal or larger than 2 in prelu() when weight shape is not [1]." + assert ( + len(x.shape) > 1 + ), "The dim count of x should be equal or larger than 2 in prelu() when weight shape is not [1]." - #NOTE(GuoxiaWang): support NHWC data format + # NOTE(GuoxiaWang): support NHWC data format if data_format == 'NHWC': - assert weight.shape[0] == x.shape[ - -1], "The weight size should be equal to x input channel in prelu() when weight shape is not [1]." + assert ( + weight.shape[0] == x.shape[-1] + ), "The weight size should be equal to x input channel in prelu() when weight shape is not [1]." else: - assert weight.shape[0] == x.shape[ - 1], "The weight size should be equal to x input channel in prelu() when weight shape is not [1]." + assert ( + weight.shape[0] == x.shape[1] + ), "The weight size should be equal to x input channel in prelu() when weight shape is not [1]." mode = 'channel' if in_dygraph_mode(): return _C_ops.prelu(x, weight, data_format, mode) if _in_legacy_dygraph(): - return _legacy_C_ops.prelu(x, weight, 'mode', mode, 'data_format', - data_format) + return _legacy_C_ops.prelu( + x, weight, 'mode', mode, 'data_format', data_format + ) helper = LayerHelper('prelu', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type="prelu", - inputs={ - "X": x, - "Alpha": weight - }, - outputs={"Out": out}, - attrs={ - "mode": mode, - "data_format": data_format - }) + helper.append_op( + type="prelu", + inputs={"X": x, "Alpha": weight}, + outputs={"Out": out}, + attrs={"mode": mode, "data_format": data_format}, + ) return out -def rrelu(x, lower=1. / 8., upper=1. / 3., training=True, name=None): +def rrelu(x, lower=1.0 / 8.0, upper=1.0 / 3.0, training=True, name=None): r""" rrelu activation. @@ -623,47 +644,56 @@ def rrelu(x, lower=1. / 8., upper=1. / 3., training=True, name=None): """ if not in_dynamic_mode(): - check_variable_and_dtype(x, 'X', ['float16', 'float32', 'float64'], - 'rrelu') + check_variable_and_dtype( + x, 'X', ['float16', 'float32', 'float64'], 'rrelu' + ) if not isinstance(lower, float) or not isinstance(upper, float): raise TypeError( - "The lower and upper values must be float type. Received: lower {}, upper {}." - .format(lower, upper)) + "The lower and upper values must be float type. Received: lower {}, upper {}.".format( + lower, upper + ) + ) if lower < 0 or lower > 1: raise ValueError( - "The lower value must be no less than zero or greater than one. Received: {}." - .format(lower)) + "The lower value must be no less than zero or greater than one. Received: {}.".format( + lower + ) + ) if upper < lower: raise ValueError( - "The upper value must be greater than lower value. Received: lower {}, upper {}." - .format(lower, upper)) + "The upper value must be greater than lower value. Received: lower {}, upper {}.".format( + lower, upper + ) + ) if upper > 1: raise ValueError( "The upper value must be no greater than one. Received: {}.".format( - upper)) + upper + ) + ) is_test = not training if _in_legacy_dygraph(): - out, noise = _legacy_C_ops.rrelu(x, 'lower', lower, 'upper', upper, - 'is_test', is_test) + out, noise = _legacy_C_ops.rrelu( + x, 'lower', lower, 'upper', upper, 'is_test', is_test + ) return out helper = LayerHelper('rrelu', **locals()) out = helper.create_variable_for_type_inference(x.dtype) noise = helper.create_variable_for_type_inference(dtype=x.dtype) attrs = {'lower': lower, 'upper': upper, 'is_test': is_test} - helper.append_op(type='rrelu', - inputs={"X": x}, - outputs={ - "Out": out, - "Noise": noise - }, - attrs=attrs) + helper.append_op( + type='rrelu', + inputs={"X": x}, + outputs={"Out": out, "Noise": noise}, + attrs=attrs, + ) return out @@ -748,8 +778,9 @@ def log_sigmoid(x, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.logsigmoid(x) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'log_sigmoid') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'log_sigmoid' + ) helper = LayerHelper("log_sigmoid", **locals()) out = helper.create_variable_for_type_inference(x.dtype) helper.append_op(type='logsigmoid', inputs={'X': x}, outputs={'Out': out}) @@ -818,19 +849,19 @@ def maxout(x, groups, axis=1, name=None): if axis not in [1, -1, 3]: raise ValueError( "Attr(axis) should be 1 when data format is NCHW, -1 or 3 when data format is NHWC. Received " - "Attr(axis): %s." % str(axis)) + "Attr(axis): %s." % str(axis) + ) if axis == -1: axis = 3 helper = LayerHelper('maxout', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='maxout', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'groups': groups, - 'axis': axis - }) + helper.append_op( + type='maxout', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'groups': groups, 'axis': axis}, + ) return out @@ -869,17 +900,21 @@ def relu6(x, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'relu6') helper = LayerHelper('relu6', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='relu6', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'threshold': threshold}) + helper.append_op( + type='relu6', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'threshold': threshold}, + ) return out -def selu(x, - scale=1.0507009873554804934193349852946, - alpha=1.6732632423543772848170429916717, - name=None): +def selu( + x, + scale=1.0507009873554804934193349852946, + alpha=1.6732632423543772848170429916717, + name=None, +): r""" selu activation @@ -915,11 +950,13 @@ def selu(x, """ if scale <= 1.0: raise ValueError( - "The scale must be greater than 1.0. Received: {}.".format(scale)) + "The scale must be greater than 1.0. Received: {}.".format(scale) + ) if alpha < 0: raise ValueError( - "The alpha must be no less than zero. Received: {}.".format(alpha)) + "The alpha must be no less than zero. Received: {}.".format(alpha) + ) if in_dygraph_mode(): return _C_ops.selu(x, scale, alpha) @@ -929,13 +966,12 @@ def selu(x, check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'selu') helper = LayerHelper('selu', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='selu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'scale': scale, - 'alpha': alpha - }) + helper.append_op( + type='selu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'scale': scale, 'alpha': alpha}, + ) return out @@ -1096,44 +1132,50 @@ def softmax(x, axis=-1, dtype=None, name=None): use_cudnn = True if in_dygraph_mode(): - outs_cast = x if dtype is None \ - else _C_ops.cast(x, dtype) + outs_cast = x if dtype is None else _C_ops.cast(x, dtype) return _C_ops.softmax(outs_cast, axis) if _in_legacy_dygraph(): - outs_cast = x if dtype is None \ + outs_cast = ( + x + if dtype is None else _legacy_C_ops.cast(x, 'in_dtype', x.dtype, 'out_dtype', dtype) - return _legacy_C_ops.softmax(outs_cast, 'axis', axis, 'use_cudnn', - use_cudnn) + ) + return _legacy_C_ops.softmax( + outs_cast, 'axis', axis, 'use_cudnn', use_cudnn + ) if dtype is None: - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'softmax') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'softmax' + ) else: check_dtype( - dtype, 'dtype', ['float32', 'float64'], 'softmax', - 'If dtype is not None, it only support float32 or float64.') + dtype, + 'dtype', + ['float32', 'float64'], + 'softmax', + 'If dtype is not None, it only support float32 or float64.', + ) helper = LayerHelper("softmax", **locals()) outs_cast = x if dtype is not None: outs_cast = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='cast', - inputs={'X': x}, - outputs={'Out': outs_cast}, - attrs={ - 'in_dtype': x.dtype, - 'out_dtype': dtype - }) + helper.append_op( + type='cast', + inputs={'X': x}, + outputs={'Out': outs_cast}, + attrs={'in_dtype': x.dtype, 'out_dtype': dtype}, + ) outs_softmax = helper.create_variable_for_type_inference(outs_cast.dtype) - helper.append_op(type='softmax', - inputs={'X': outs_cast}, - outputs={'Out': outs_softmax}, - attrs={ - 'axis': axis, - 'use_cudnn': use_cudnn - }) + helper.append_op( + type='softmax', + inputs={'X': outs_cast}, + outputs={'Out': outs_softmax}, + attrs={'axis': axis, 'use_cudnn': use_cudnn}, + ) return outs_softmax @@ -1149,15 +1191,22 @@ def softmax_(x, axis=-1, dtype=None, name=None): use_cudnn = True if in_dygraph_mode(): - outs_cast = x if dtype is None \ + outs_cast = ( + x + if dtype is None else _legacy_C_ops.cast(x, 'in_dtype', x.dtype, 'out_dtype', dtype) + ) return _C_ops.softmax_(outs_cast, axis) if _in_legacy_dygraph(): - outs_cast = x if dtype is None \ + outs_cast = ( + x + if dtype is None else _legacy_C_ops.cast(x, 'in_dtype', x.dtype, 'out_dtype', dtype) - return _legacy_C_ops.softmax_(outs_cast, 'axis', axis, 'use_cudnn', - use_cudnn) + ) + return _legacy_C_ops.softmax_( + outs_cast, 'axis', axis, 'use_cudnn', use_cudnn + ) def softplus(x, beta=1, threshold=20, name=None): @@ -1195,17 +1244,17 @@ def softplus(x, beta=1, threshold=20, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.softplus(x, 'beta', beta, 'threshold', threshold) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'softplus') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'softplus' + ) helper = LayerHelper('softplus', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='softplus', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'beta': beta, - 'threshold': threshold - }) + helper.append_op( + type='softplus', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'beta': beta, 'threshold': threshold}, + ) return out @@ -1245,21 +1294,26 @@ def softshrink(x, threshold=0.5, name=None): if threshold < 0: raise ValueError( "The threshold must be no less than zero. Received: {}.".format( - threshold)) + threshold + ) + ) if in_dygraph_mode(): return _C_ops.softshrink(x, threshold) if _in_legacy_dygraph(): return _legacy_C_ops.softshrink(x, 'lambda', threshold) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'softshrink') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'softshrink' + ) helper = LayerHelper('softshrink', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='softshrink', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'lambda': threshold}) + helper.append_op( + type='softshrink', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'lambda': threshold}, + ) return out @@ -1293,8 +1347,9 @@ def softsign(x, name=None): if in_dynamic_mode(): return _legacy_C_ops.softsign(x) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'softsign') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'softsign' + ) helper = LayerHelper('softsign', **locals()) out = helper.create_variable_for_type_inference(x.dtype) helper.append_op(type='softsign', inputs={'X': x}, outputs={'Out': out}) @@ -1334,10 +1389,9 @@ def swish(x, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'swish') helper = LayerHelper('swish', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='swish', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'beta': 1.0}) + helper.append_op( + type='swish', inputs={'X': x}, outputs={'Out': out}, attrs={'beta': 1.0} + ) return out @@ -1413,8 +1467,9 @@ def tanhshrink(x, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.tanh_shrink(x) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'tanhshrink') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'tanhshrink' + ) helper = LayerHelper('tanh_shrink', **locals()) out = helper.create_variable_for_type_inference(x.dtype) helper.append_op(type='tanh_shrink', inputs={'X': x}, outputs={'Out': out}) @@ -1461,14 +1516,17 @@ def thresholded_relu(x, threshold=1.0, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.thresholded_relu(x, 'threshold', threshold) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'thresholded_relu') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'thresholded_relu' + ) helper = LayerHelper('thresholded_relu', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='thresholded_relu', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'threshold': threshold}) + helper.append_op( + type='thresholded_relu', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'threshold': threshold}, + ) return out @@ -1541,30 +1599,36 @@ def log_softmax(x, axis=-1, dtype=None, name=None): return _legacy_C_ops.log_softmax(x, 'axis', axis) if dtype is None: - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'log_softmax') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'log_softmax' + ) else: check_dtype( - dtype, 'dtype', ['float32', 'float64'], 'log_softmax', - 'If dtype is not None, it only support float32 or float64.') + dtype, + 'dtype', + ['float32', 'float64'], + 'log_softmax', + 'If dtype is not None, it only support float32 or float64.', + ) helper = LayerHelper("log_softmax", **locals()) out_cast = x if dtype is not None: out_cast = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='cast', - inputs={'X': x}, - outputs={'Out': out_cast}, - attrs={ - 'in_dtype': x.dtype, - 'out_dtype': dtype - }) + helper.append_op( + type='cast', + inputs={'X': x}, + outputs={'Out': out_cast}, + attrs={'in_dtype': x.dtype, 'out_dtype': dtype}, + ) out = helper.create_variable_for_type_inference(out_cast.dtype) - helper.append_op(type='log_softmax', - inputs={'X': out_cast}, - outputs={'Out': out}, - attrs={'axis': axis}) + helper.append_op( + type='log_softmax', + inputs={'X': out_cast}, + outputs={'Out': out}, + attrs={'axis': axis}, + ) return out @@ -1607,8 +1671,9 @@ def glu(x, axis=-1, name=None): # [-1.0577879 , -0.46985325]], dtype=float32) """ - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - "glu") + check_variable_and_dtype( + x, 'input', ['float16', 'float32', 'float64'], "glu" + ) a, b = chunk(x, 2, axis=axis, name=name) gate = sigmoid(b, name=name) out = paddle.multiply(a, gate, name=name) @@ -1676,18 +1741,17 @@ def gumbel_softmax(x, temperature=1.0, hard=False, axis=-1, name=None): return _C_ops.gumbel_softmax(x, temperature, hard, axis) if in_dynamic_mode(): - return _legacy_C_ops.gumbel_softmax(x, 'temperature', temperature, - 'hard', hard, 'axis', axis) + return _legacy_C_ops.gumbel_softmax( + x, 'temperature', temperature, 'hard', hard, 'axis', axis + ) helper = LayerHelper("gumbel_softmax", **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'gumbel_softmax') out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='gumbel_softmax', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'temperature': temperature, - 'hard': hard, - 'axis': axis - }) + helper.append_op( + type='gumbel_softmax', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'temperature': temperature, 'hard': hard, 'axis': axis}, + ) return out diff --git a/python/paddle/nn/functional/common.py b/python/paddle/nn/functional/common.py index 2dc4db3026557..b6936c5a90c9b 100644 --- a/python/paddle/nn/functional/common.py +++ b/python/paddle/nn/functional/common.py @@ -18,14 +18,23 @@ from ...tensor import concat from ...tensor.creation import zeros from paddle.static import Variable + # TODO: define the common functions to build a neural network from ...tensor.manipulation import squeeze from ...tensor.manipulation import unsqueeze from ...tensor import clip from ...tensor import sum from ...tensor import sqrt -from ...fluid.data_feeder import check_variable_and_dtype, check_dtype, check_type -from ...fluid.framework import _in_legacy_dygraph, _non_static_mode, in_dygraph_mode +from ...fluid.data_feeder import ( + check_variable_and_dtype, + check_dtype, + check_type, +) +from ...fluid.framework import ( + _in_legacy_dygraph, + _non_static_mode, + in_dygraph_mode, +) from paddle import _C_ops, _legacy_C_ops from paddle.framework import in_dynamic_mode @@ -108,26 +117,28 @@ def unfold(x, kernel_sizes, strides=1, paddings=0, dilations=1, name=None): check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'unfold') - assert len(x.shape) == 4, \ - "input should be the format of [N, C, H, W]" + assert len(x.shape) == 4, "input should be the format of [N, C, H, W]" if isinstance(kernel_sizes, int): kernel_sizes = [kernel_sizes, kernel_sizes] else: - assert isinstance(kernel_sizes, list) and (len(kernel_sizes) == 2), \ - "kernel_sizes should either be an integer or a list of two integers" + assert isinstance(kernel_sizes, list) and ( + len(kernel_sizes) == 2 + ), "kernel_sizes should either be an integer or a list of two integers" if isinstance(strides, int): strides = [strides, strides] else: - assert isinstance(strides, list) and (len(strides) == 2), \ - "strides should either be an integer or a list of two integers" + assert isinstance(strides, list) and ( + len(strides) == 2 + ), "strides should either be an integer or a list of two integers" if isinstance(dilations, int): dilations = [dilations, dilations] else: - assert isinstance(dilations, list) and (len(dilations) == 2), \ - "dilations should either be an integer or a list of two integers" + assert isinstance(dilations, list) and ( + len(dilations) == 2 + ), "dilations should either be an integer or a list of two integers" if isinstance(paddings, int): paddings = [paddings] * 4 @@ -143,32 +154,37 @@ def unfold(x, kernel_sizes, strides=1, paddings=0, dilations=1, name=None): else: raise ValueError( "Unexpected type of paddings, it should be either an integer or a list" - "of 2 or 4 integers") + "of 2 or 4 integers" + ) if in_dygraph_mode(): return _C_ops.unfold(x, kernel_sizes, strides, paddings, dilations) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="unfold", - inputs={"X": x}, - outputs={"Y": out}, - attrs={ - "kernel_sizes": kernel_sizes, - "strides": strides, - "paddings": paddings, - "dilations": dilations - }) + helper.append_op( + type="unfold", + inputs={"X": x}, + outputs={"Y": out}, + attrs={ + "kernel_sizes": kernel_sizes, + "strides": strides, + "paddings": paddings, + "dilations": dilations, + }, + ) return out -def interpolate(x, - size=None, - scale_factor=None, - mode='nearest', - align_corners=False, - align_mode=0, - data_format='NCHW', - name=None): +def interpolate( + x, + size=None, + scale_factor=None, + mode='nearest', + align_corners=False, + align_mode=0, + data_format='NCHW', + name=None, +): """ This API resizes a batch of images. @@ -376,7 +392,8 @@ def interpolate(x, if resample not in resample_methods: raise ValueError( "The 'resample' of image_resize can only be 'area', 'linear', 'bilinear', 'trilinear', " - " 'bicubic' or 'nearest' currently.") + " 'bicubic' or 'nearest' currently." + ) if resample in ['LINEAR'] and len(x.shape) != 3: raise ValueError("'linear' only support 3-D tensor.") @@ -403,8 +420,11 @@ def interpolate(x, ) if resample == 'AREA': - if isinstance(size, list) or isinstance(size, tuple) or isinstance( - size, Variable): + if ( + isinstance(size, list) + or isinstance(size, tuple) + or isinstance(size, Variable) + ): if len(size) == 0: raise ValueError("output size can not be empty") if len(x.shape) == 3: @@ -418,19 +438,25 @@ def interpolate(x, dtype = helper.input_dtype(input_param_name='x') if len(x.shape) == 3 and data_format not in ['NCW', 'NWC']: raise ValueError( - "Got wrong value for param `data_format`: " + data_format + - " received but only `NCW` or `NWC` supported for 3-D input.") + "Got wrong value for param `data_format`: " + + data_format + + " received but only `NCW` or `NWC` supported for 3-D input." + ) elif len(x.shape) == 4 and data_format not in ['NCHW', 'NHWC']: raise ValueError( - "Got wrong value for param `data_format`: " + data_format + - " received but only `NCHW` or `NHWC` supported for 4-D input.") + "Got wrong value for param `data_format`: " + + data_format + + " received but only `NCHW` or `NHWC` supported for 4-D input." + ) elif len(x.shape) == 5 and data_format not in ['NCDHW', 'NDHWC']: raise ValueError( - "Got wrong value for param `data_format`: " + data_format + - " received but only `NCDHW` or `NDHWC` supported for 5-D input.") + "Got wrong value for param `data_format`: " + + data_format + + " received but only `NCDHW` or `NDHWC` supported for 5-D input." + ) def _is_list_or_turple_(data): - return (isinstance(data, list) or isinstance(data, tuple)) + return isinstance(data, list) or isinstance(data, tuple) if data_format == 'NCHW' or data_format == 'NCDHW' or data_format == 'NCW': data_layout = 'NCHW' @@ -448,7 +474,7 @@ def _is_list_or_turple_(data): "interp_method": resample_type, "align_corners": align_corners, "align_mode": align_mode, - "data_layout": data_layout + "data_layout": data_layout, } out_shape = size @@ -476,9 +502,9 @@ def _is_list_or_turple_(data): if isinstance(dim_size, Variable): contain_var = True continue - assert dim_size > 0, ( - "Each dimension size given in out_shape must be greater than 0." - ) + assert ( + dim_size > 0 + ), "Each dimension size given in out_shape must be greater than 0." if contain_var: new_size_tensor = [] @@ -489,14 +515,13 @@ def _is_list_or_turple_(data): new_size_tensor.append(dim) size_list.append(-1) else: - assert (isinstance(dim, int)) + assert isinstance(dim, int) temp_out = helper.create_variable_for_type_inference( - 'int32') - fill_constant([1], - 'int32', - dim, - force_cpu=True, - out=temp_out) + 'int32' + ) + fill_constant( + [1], 'int32', dim, force_cpu=True, out=temp_out + ) new_size_tensor.append(temp_out) size_list.append(dim) inputs['SizeTensor'] = new_size_tensor @@ -504,7 +529,8 @@ def _is_list_or_turple_(data): if len(x.shape) == 3: if len(out_shape) != 1: raise ValueError( - "size length should be 2 for input 3-D tensor") + "size length should be 2 for input 3-D tensor" + ) if contain_var: attrs['out_w'] = size_list[0] else: @@ -512,8 +538,9 @@ def _is_list_or_turple_(data): attrs['out_w'] = out_shape[0] if len(x.shape) == 4: if len(out_shape) != 2: - raise ValueError("size length should be 2 for " - "input 4-D tensor.") + raise ValueError( + "size length should be 2 for " "input 4-D tensor." + ) if contain_var: attrs['out_h'] = size_list[0] attrs['out_w'] = size_list[1] @@ -523,8 +550,9 @@ def _is_list_or_turple_(data): attrs['out_w'] = out_shape[1] if len(x.shape) == 5: if len(out_shape) != 3: - raise ValueError("size length should be 3 for " - "input 5-D tensor.") + raise ValueError( + "size length should be 3 for " "input 5-D tensor." + ) if contain_var: attrs['out_d'] = size_list[0] attrs['out_h'] = size_list[1] @@ -550,9 +578,10 @@ def _is_list_or_turple_(data): attrs['scale'] = list(map(float, scale_list)) elif isinstance(scale, list) or isinstance(scale, tuple): if len(scale) != len(x.shape) - 2: - raise ValueError("scale_shape length should be {} for " - "input {}-D tensor.".format( - len(x.shape) - 2, len(x.shape))) + raise ValueError( + "scale_shape length should be {} for " + "input {}-D tensor.".format(len(x.shape) - 2, len(x.shape)) + ) for value in scale: if value <= 0: raise ValueError("Attr(scale) should be greater than zero.") @@ -572,80 +601,114 @@ def _is_list_or_turple_(data): if resample_type == "linear": if in_dygraph_mode(): out = _C_ops.linear_interp( - x, inputs['OutSize'] if 'OutSize' in inputs else None, + x, + inputs['OutSize'] if 'OutSize' in inputs else None, inputs['SizeTensor'] if 'SizeTensor' in inputs else None, inputs['Scale'] if 'Scale' in inputs else None, - attrs['data_layout'], attrs['out_d'], attrs['out_h'], - attrs['out_w'], attrs['scale'] if 'scale' in attrs else [], - attrs['interp_method'], attrs['align_corners'], - attrs['align_mode']) + attrs['data_layout'], + attrs['out_d'], + attrs['out_h'], + attrs['out_w'], + attrs['scale'] if 'scale' in attrs else [], + attrs['interp_method'], + attrs['align_corners'], + attrs['align_mode'], + ) else: out = _legacy_C_ops.linear_interp_v2(x, *dy_attr) elif resample_type == "bilinear": if in_dygraph_mode(): out = _C_ops.bilinear_interp( - x, inputs['OutSize'] if 'OutSize' in inputs else None, + x, + inputs['OutSize'] if 'OutSize' in inputs else None, inputs['SizeTensor'] if 'SizeTensor' in inputs else None, inputs['Scale'] if 'Scale' in inputs else None, - attrs['data_layout'], attrs['out_d'], attrs['out_h'], - attrs['out_w'], attrs['scale'] if 'scale' in attrs else [], - attrs['interp_method'], attrs['align_corners'], - attrs['align_mode']) + attrs['data_layout'], + attrs['out_d'], + attrs['out_h'], + attrs['out_w'], + attrs['scale'] if 'scale' in attrs else [], + attrs['interp_method'], + attrs['align_corners'], + attrs['align_mode'], + ) else: out = _legacy_C_ops.bilinear_interp_v2(x, *dy_attr) elif resample_type == "trilinear": if in_dygraph_mode(): out = _C_ops.trilinear_interp( - x, inputs['OutSize'] if 'OutSize' in inputs else None, + x, + inputs['OutSize'] if 'OutSize' in inputs else None, inputs['SizeTensor'] if 'SizeTensor' in inputs else None, inputs['Scale'] if 'Scale' in inputs else None, - attrs['data_layout'], attrs['out_d'], attrs['out_h'], - attrs['out_w'], attrs['scale'] if 'scale' in attrs else [], - attrs['interp_method'], attrs['align_corners'], - attrs['align_mode']) + attrs['data_layout'], + attrs['out_d'], + attrs['out_h'], + attrs['out_w'], + attrs['scale'] if 'scale' in attrs else [], + attrs['interp_method'], + attrs['align_corners'], + attrs['align_mode'], + ) else: out = _legacy_C_ops.trilinear_interp_v2(x, *dy_attr) elif resample_type == "nearest": if in_dygraph_mode(): out = _C_ops.nearest_interp( - x, inputs['OutSize'] if 'OutSize' in inputs else None, + x, + inputs['OutSize'] if 'OutSize' in inputs else None, inputs['SizeTensor'] if 'SizeTensor' in inputs else None, inputs['Scale'] if 'Scale' in inputs else None, - attrs['data_layout'], attrs['out_d'], attrs['out_h'], - attrs['out_w'], attrs['scale'] if 'scale' in attrs else [], - attrs['interp_method'], attrs['align_corners'], - attrs['align_mode']) + attrs['data_layout'], + attrs['out_d'], + attrs['out_h'], + attrs['out_w'], + attrs['scale'] if 'scale' in attrs else [], + attrs['interp_method'], + attrs['align_corners'], + attrs['align_mode'], + ) else: out = _legacy_C_ops.nearest_interp_v2(x, *dy_attr) elif resample_type == "bicubic": if in_dygraph_mode(): out = _C_ops.bicubic_interp( - x, inputs['OutSize'] if 'OutSize' in inputs else None, + x, + inputs['OutSize'] if 'OutSize' in inputs else None, inputs['SizeTensor'] if 'SizeTensor' in inputs else None, inputs['Scale'] if 'Scale' in inputs else None, - attrs['data_layout'], attrs['out_d'], attrs['out_h'], - attrs['out_w'], attrs['scale'] if 'scale' in attrs else [], - attrs['interp_method'], attrs['align_corners'], - attrs['align_mode']) + attrs['data_layout'], + attrs['out_d'], + attrs['out_h'], + attrs['out_w'], + attrs['scale'] if 'scale' in attrs else [], + attrs['interp_method'], + attrs['align_corners'], + attrs['align_mode'], + ) else: out = _legacy_C_ops.bicubic_interp_v2(x, *dy_attr) return out out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='{}_interp_v2'.format(resample_type), - inputs=inputs, - outputs={"Out": out}, - attrs=attrs) + helper.append_op( + type='{}_interp_v2'.format(resample_type), + inputs=inputs, + outputs={"Out": out}, + attrs=attrs, + ) return out -def upsample(x, - size=None, - scale_factor=None, - mode='nearest', - align_corners=False, - align_mode=0, - data_format='NCHW', - name=None): +def upsample( + x, + size=None, + scale_factor=None, + mode='nearest', + align_corners=False, + align_mode=0, + data_format='NCHW', + name=None, +): """ This API resizes a batch of images. @@ -827,8 +890,9 @@ def upsample(x, # [2L, 3L, 12L, 12L] """ - return interpolate(x, size, scale_factor, mode, align_corners, align_mode, - data_format) + return interpolate( + x, size, scale_factor, mode, align_corners, align_mode, data_format + ) def bilinear(x1, x2, weight, bias=None, name=None): @@ -879,19 +943,16 @@ def bilinear(x1, x2, weight, bias=None, name=None): helper = LayerHelper("bilinear", **locals()) out = helper.create_variable_for_type_inference(dtype=x1.dtype) - helper.append_op(type="bilinear_tensor_product", - inputs=inputs, - outputs={"Out": out}) + helper.append_op( + type="bilinear_tensor_product", inputs=inputs, outputs={"Out": out} + ) return out -def dropout(x, - p=0.5, - axis=None, - training=True, - mode="upscale_in_train", - name=None): +def dropout( + x, p=0.5, axis=None, training=True, mode="upscale_in_train", name=None +): """ Dropout is a regularization technique for reducing overfitting by preventing neuron co-adaption during training. The dropout operator randomly sets the @@ -1043,7 +1104,8 @@ def dropout(x, if isinstance(p, (int, float)): # fast return for p == 0 - if p == 0: return x + if p == 0: + return x elif p < 0 or p > 1: raise ValueError("p argument should between 0 and 1") if mode not in ('downscale_in_infer', 'upscale_in_train'): @@ -1055,41 +1117,63 @@ def dropout(x, if axis == None: # commonly used dropout seed = None - mode = 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode #semantic transfer + mode = ( + 'downgrade_in_infer' if mode == 'downscale_in_infer' else mode + ) # semantic transfer if _non_static_mode(): if default_main_program().random_seed != 0: seed = default_main_program().random_seed if in_dygraph_mode(): - out, mask = _C_ops.dropout( x, None, p, not training, mode, \ - seed if seed is not None else 0, seed is not None) + out, mask = _C_ops.dropout( + x, + None, + p, + not training, + mode, + seed if seed is not None else 0, + seed is not None, + ) return out - out, mask = _legacy_C_ops.dropout(x, 'dropout_prob', p, 'is_test', - not training, 'fix_seed', seed - is not None, 'seed', - seed if seed is not None else 0, - 'dropout_implementation', mode) + out, mask = _legacy_C_ops.dropout( + x, + 'dropout_prob', + p, + 'is_test', + not training, + 'fix_seed', + seed is not None, + 'seed', + seed if seed is not None else 0, + 'dropout_implementation', + mode, + ) return out helper = LayerHelper('dropout', **locals()) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'dropout') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'dropout' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) mask = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) def get_attrs(prog, dropout_prob, is_test, seed): if (seed is None or seed == 0) and prog.random_seed != 0: seed = prog.random_seed - if isinstance(dropout_prob, - Variable) and not dropout_prob.shape != [1]: + if isinstance( + dropout_prob, Variable + ) and not dropout_prob.shape != [1]: raise TypeError( - "Required p.shape == [1] if type(p) is Variable, but received p.shape = {}" - .format(p.shape)) + "Required p.shape == [1] if type(p) is Variable, but received p.shape = {}".format( + p.shape + ) + ) attrs = { 'dropout_prob': dropout_prob, 'is_test': is_test, @@ -1101,38 +1185,45 @@ def get_attrs(prog, dropout_prob, is_test, seed): attrs = get_attrs(helper.main_program, p, not training, seed) - helper.append_op(type='dropout', - inputs={'X': [x]}, - outputs={ - 'Out': [out], - 'Mask': [mask] - }, - attrs=attrs) + helper.append_op( + type='dropout', + inputs={'X': [x]}, + outputs={'Out': [out], 'Mask': [mask]}, + attrs=attrs, + ) return out - else: #sometimes called dropout_nd #TODO: optimize with c++ + else: # sometimes called dropout_nd #TODO: optimize with c++ if not in_dynamic_mode(): check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'dropout') dtype = x.dtype keep_prob = 1 - p if training: - if in_dynamic_mode() and p == 1.: - return paddle.scale(x, scale=0.) + if in_dynamic_mode() and p == 1.0: + return paddle.scale(x, scale=0.0) - scale_input = paddle.scale( - x, scale=1 / keep_prob) if mode == 'upscale_in_train' else x + scale_input = ( + paddle.scale(x, scale=1 / keep_prob) + if mode == 'upscale_in_train' + else x + ) - #get mask shape + # get mask shape input_shape = x.shape if not in_dynamic_mode(): input_shape_tensor = paddle.shape(x) drop_axes = [axis] if isinstance(axis, int) else list(axis) if min(drop_axes) < 0 or max(drop_axes) > len(input_shape) - 1: - raise ValueError("axis value should be greater than or equal to 0 and less than dimensions of x:{}, but get axis value:{} " \ - .format(len(input_shape), max(drop_axes))) + raise ValueError( + "axis value should be greater than or equal to 0 and less than dimensions of x:{}, but get axis value:{} ".format( + len(input_shape), max(drop_axes) + ) + ) if len(drop_axes) > len(input_shape): raise ValueError( - "length of axis should not be greater than dimensions of x:{}, but get length of axis: {}" - .format(len(input_shape), len(drop_axes))) + "length of axis should not be greater than dimensions of x:{}, but get length of axis: {}".format( + len(input_shape), len(drop_axes) + ) + ) mask_shape = [1] * len(input_shape) if not in_dynamic_mode(): for i in drop_axes: @@ -1141,11 +1232,10 @@ def get_attrs(prog, dropout_prob, is_test, seed): for i in drop_axes: mask_shape[i] = input_shape[i] - #get mask - random_tensor = paddle.uniform(mask_shape, - dtype='float32', - min=0., - max=1.0) + # get mask + random_tensor = paddle.uniform( + mask_shape, dtype='float32', min=0.0, max=1.0 + ) p = full(shape=[1], fill_value=p, dtype='float32') keep_mask = paddle.greater_equal(random_tensor, p) @@ -1154,8 +1244,11 @@ def get_attrs(prog, dropout_prob, is_test, seed): ret = paddle.multiply(scale_input, keep_mask, name=name) return ret else: # test - ret = paddle.scale( - x, scale=keep_prob) if mode == 'downscale_in_infer' else x + ret = ( + paddle.scale(x, scale=keep_prob) + if mode == 'downscale_in_infer' + else x + ) return ret @@ -1196,20 +1289,26 @@ def dropout2d(x, p=0.5, training=True, data_format='NCHW', name=None): """ input_shape = x.shape if len(input_shape) != 4: - raise ValueError("dimensions of x should be 4, but received {} != 4"\ - .format(len(input_shape))) + raise ValueError( + "dimensions of x should be 4, but received {} != 4".format( + len(input_shape) + ) + ) if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) - return dropout(x, - p=p, - axis=[0, 1] if data_format == 'NCHW' else [0, 3], - training=training, - mode="upscale_in_train", - name=name) + return dropout( + x, + p=p, + axis=[0, 1] if data_format == 'NCHW' else [0, 3], + training=training, + mode="upscale_in_train", + name=name, + ) def dropout3d(x, p=0.5, training=True, data_format='NCDHW', name=None): @@ -1248,20 +1347,26 @@ def dropout3d(x, p=0.5, training=True, data_format='NCDHW', name=None): input_shape = x.shape if len(input_shape) != 5: - raise ValueError("dimensions of x should be 5, but received {} != 5" \ - .format(len(input_shape))) + raise ValueError( + "dimensions of x should be 5, but received {} != 5".format( + len(input_shape) + ) + ) if data_format not in ["NCDHW", "NDHWC"]: raise ValueError( "Attr(data_format) should be 'NCDHW' or 'NDHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) - return dropout(x, - p=p, - axis=[0, 1] if data_format == 'NCDHW' else [0, 4], - training=training, - mode="upscale_in_train", - name=name) + return dropout( + x, + p=p, + axis=[0, 1] if data_format == 'NCDHW' else [0, 4], + training=training, + mode="upscale_in_train", + name=name, + ) def alpha_dropout(x, p=0.5, training=True, name=None): @@ -1303,37 +1408,40 @@ def alpha_dropout(x, p=0.5, training=True, name=None): raise ValueError("p argument should between 0 and 1") if not in_dynamic_mode(): - check_variable_and_dtype(x, 'x', ['float32', 'float64'], - 'alpha_dropout') + check_variable_and_dtype( + x, 'x', ['float32', 'float64'], 'alpha_dropout' + ) if training: if p == 1: - return paddle.scale(x, scale=0.) - #get transformation params + return paddle.scale(x, scale=0.0) + # get transformation params alpha = 1.6732632423543772848170429916717 scale = 1.0507009873554804934193349852946 alpha_p = -alpha * scale - a = ((1 - p) * (1 + p * alpha_p**2))**-0.5 + a = ((1 - p) * (1 + p * alpha_p**2)) ** -0.5 b = -a * alpha_p * p dtype = x.dtype input_shape = x.shape - #get mask - random_tensor = paddle.uniform(input_shape, - dtype='float32', - min=0., - max=1.0) + # get mask + random_tensor = paddle.uniform( + input_shape, dtype='float32', min=0.0, max=1.0 + ) p = full(shape=[1], fill_value=p, dtype='float32') keep_mask = paddle.greater_equal(random_tensor, p) keep_mask = paddle.cast(keep_mask, dtype) drop_mask = paddle.subtract( - full(shape=input_shape, fill_value=1., dtype=dtype), keep_mask) + full(shape=input_shape, fill_value=1.0, dtype=dtype), keep_mask + ) - #apply mask + # apply mask b = full(shape=[1], fill_value=b, dtype=dtype) - y = paddle.add(paddle.multiply(x, keep_mask), - paddle.scale(drop_mask, scale=alpha_p)) + y = paddle.add( + paddle.multiply(x, keep_mask), + paddle.scale(drop_mask, scale=alpha_p), + ) res = paddle.add(paddle.scale(y, scale=a), b, name=name) return res else: # test @@ -1452,18 +1560,28 @@ def pad(x, pad, mode='constant', value=0, data_format="NCHW", name=None): # [6. 4. 5. 6. 4. 5.] # [3. 1. 2. 3. 1. 2.]]]] """ - assert mode in ['reflect', 'replicate', 'constant', 'circular'], \ - "mode should be one of constant, reflect, replicate, circular, but got {}.".format(mode) + assert mode in [ + 'reflect', + 'replicate', + 'constant', + 'circular', + ], "mode should be one of constant, reflect, replicate, circular, but got {}.".format( + mode + ) data_format = data_format.upper() - assert data_format in ["NCL", "NCHW", "NCDHW", "NLC", "NHWC", "NDHWC"], \ - "data_format should be in one of [NCL, NCHW, NCDHW, NLC, NHWC, NDHWC], " \ + assert data_format in ["NCL", "NCHW", "NCDHW", "NLC", "NHWC", "NDHWC"], ( + "data_format should be in one of [NCL, NCHW, NCDHW, NLC, NHWC, NDHWC], " "but got {}".format(data_format) + ) x_dim = len(x.shape) - if mode == "constant" and isinstance( - pad, (list, tuple)) and len(pad) == x_dim * 2: + if ( + mode == "constant" + and isinstance(pad, (list, tuple)) + and len(pad) == x_dim * 2 + ): paddings = pad pad_value = value @@ -1471,10 +1589,20 @@ def pad(x, pad, mode='constant', value=0, data_format="NCHW", name=None): out = _C_ops.pad(x, paddings, float(pad_value)) return out - check_variable_and_dtype(x, 'x', [ - 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], "pad") + check_variable_and_dtype( + x, + 'x', + [ + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + "pad", + ) check_type(pad_value, 'pad_value', (float, int, Variable), 'pad') if isinstance(pad_value, int): @@ -1483,17 +1611,18 @@ def pad(x, pad, mode='constant', value=0, data_format="NCHW", name=None): helper = LayerHelper('pad', **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='pad', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'paddings': paddings, - 'pad_value': pad_value - }) + helper.append_op( + type='pad', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'paddings': paddings, 'pad_value': pad_value}, + ) return out assert x_dim in [ - 3, 4, 5 + 3, + 4, + 5, ], "input tesor dimension must be in [3, 4, 5] but got {}".format(x_dim) supported_format_map = { @@ -1501,9 +1630,11 @@ def pad(x, pad, mode='constant', value=0, data_format="NCHW", name=None): 4: ["NCHW", "NHWC"], 5: ["NCDHW", "NDHWC"], } - assert data_format in supported_format_map[x_dim], \ - "input tensor dimension is {}, it's data format should be in {} but got {}".format( - x_dim, supported_format_map[x_dim], data_format) + assert ( + data_format in supported_format_map[x_dim] + ), "input tensor dimension is {}, it's data format should be in {} but got {}".format( + x_dim, supported_format_map[x_dim], data_format + ) unsqueezed_dim = [] @@ -1511,21 +1642,21 @@ def pad(x, pad, mode='constant', value=0, data_format="NCHW", name=None): if data_format in ["NCL", "NCHW", "NCDHW"]: data_format = "NCDHW" if x_dim == 3: - pad = concat([zeros((4, ), dtype="int32"), pad], axis=0) + pad = concat([zeros((4,), dtype="int32"), pad], axis=0) unsqueezed_dim = [3, 4] x = unsqueeze(x, axis=unsqueezed_dim) elif x_dim == 4: - pad = concat([pad, zeros((2, ), dtype="int32")], axis=0) + pad = concat([pad, zeros((2,), dtype="int32")], axis=0) unsqueezed_dim = [2] x = unsqueeze(x, axis=unsqueezed_dim) elif data_format in ["NLC", "NHWC", "NDHWC"]: data_format = "NDHWC" if x_dim == 3: - pad = concat([zeros((4, ), dtype="int32"), pad], axis=0) + pad = concat([zeros((4,), dtype="int32"), pad], axis=0) unsqueezed_dim = [2, 3] x = unsqueeze(x, axis=unsqueezed_dim) elif x_dim == 4: - pad = concat([pad, zeros((2, ), dtype="int32")], axis=0) + pad = concat([pad, zeros((2,), dtype="int32")], axis=0) unsqueezed_dim = [1] x = unsqueeze(x, axis=unsqueezed_dim) else: @@ -1559,9 +1690,19 @@ def pad(x, pad, mode='constant', value=0, data_format="NCHW", name=None): if _in_legacy_dygraph(): if isinstance(pad, Variable): pad = pad.numpy().tolist() - out = _legacy_C_ops.pad3d(x, "paddings", pad, "mode", mode, "value", - value, "data_format", data_format, "name", - name) + out = _legacy_C_ops.pad3d( + x, + "paddings", + pad, + "mode", + mode, + "value", + value, + "data_format", + data_format, + "name", + name, + ) else: attrs = {'mode': mode, 'value': value, 'data_format': data_format} inputs = {'X': [x]} @@ -1575,10 +1716,9 @@ def pad(x, pad, mode='constant', value=0, data_format="NCHW", name=None): dtype = helper.input_dtype(input_param_name='input') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='pad3d', - inputs=inputs, - outputs={"Out": out}, - attrs=attrs) + helper.append_op( + type='pad3d', inputs=inputs, outputs={"Out": out}, attrs=attrs + ) if len(unsqueezed_dim) != 0: out = squeeze(out, axis=unsqueezed_dim) @@ -1619,12 +1759,14 @@ def zeropad2d(x, padding, data_format="NCHW", name=None): # [0. 0. 0. 0. 0. 0.]]]] """ - return pad(x, - pad=padding, - mode='constant', - value=0, - data_format=data_format, - name=name) + return pad( + x, + pad=padding, + mode='constant', + value=0, + data_format=data_format, + name=name, + ) def cosine_similarity(x1, x2, axis=1, eps=1e-8): @@ -1731,12 +1873,13 @@ def linear(x, weight, bias=None, name=None): # [2.1077576 2.1077576 2.1077576 2.1077576 ]] """ if in_dygraph_mode(): - #TODO(jiabin): using addmm for fast forward route + # TODO(jiabin): using addmm for fast forward route return _C_ops.linear(x, weight, bias) else: if _in_legacy_dygraph(): - pre_bias = _legacy_C_ops.matmul_v2(x, weight, 'trans_x', False, - 'trans_y', False) + pre_bias = _legacy_C_ops.matmul_v2( + x, weight, 'trans_x', False, 'trans_y', False + ) if bias is None: return pre_bias @@ -1746,27 +1889,30 @@ def linear(x, weight, bias=None, name=None): helper = LayerHelper('linear', **locals()) dtype = x.dtype - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'linear') - check_dtype(dtype, 'dtype', ['float16', 'float32', 'float64'], - 'linear') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'linear' + ) + check_dtype( + dtype, 'dtype', ['float16', 'float32', 'float64'], 'linear' + ) inputs = {'X': [x], 'Y': [weight]} attrs = {'trans_x': False, 'trans_y': False} tmp = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='matmul_v2', - inputs=inputs, - outputs={'Out': tmp}, - attrs=attrs) + helper.append_op( + type='matmul_v2', + inputs=inputs, + outputs={'Out': tmp}, + attrs=attrs, + ) if bias is not None: res = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='elementwise_add', - inputs={ - 'X': [tmp], - 'Y': [bias] - }, - outputs={'Out': [res]}, - attrs={'axis': len(x.shape) - 1}) + helper.append_op( + type='elementwise_add', + inputs={'X': [tmp], 'Y': [bias]}, + outputs={'Out': [res]}, + attrs={'axis': len(x.shape) - 1}, + ) else: res = tmp return res @@ -1830,29 +1976,32 @@ def label_smooth(label, prior_dist=None, epsilon=0.1, name=None): #[[[0.03333334 0.93333334 0.03333334] # [0.93333334 0.03333334 0.93333334]]] """ - if epsilon > 1. or epsilon < 0.: + if epsilon > 1.0 or epsilon < 0.0: raise ValueError("The value of epsilon must be between 0 and 1.") if in_dygraph_mode(): return _C_ops.label_smooth(label, prior_dist, float(epsilon)) elif paddle.in_dynamic_mode(): - return _legacy_C_ops.label_smooth(label, prior_dist, 'epsilon', - float(epsilon)) + return _legacy_C_ops.label_smooth( + label, prior_dist, 'epsilon', float(epsilon) + ) - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'label_smooth') + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'label_smooth' + ) helper = LayerHelper("label_smooth", **locals()) label.stop_gradient = True smooth_label = helper.create_variable_for_type_inference(label.dtype) - helper.append_op(type="label_smooth", - inputs={ - "X": label, - "PriorDist": prior_dist - } if prior_dist else {"X": label}, - outputs={"Out": smooth_label}, - attrs={"epsilon": float(epsilon)}) + helper.append_op( + type="label_smooth", + inputs={"X": label, "PriorDist": prior_dist} + if prior_dist + else {"X": label}, + outputs={"Out": smooth_label}, + attrs={"epsilon": float(epsilon)}, + ) return smooth_label @@ -1963,7 +2112,10 @@ class centers and the shape of sampled_class_center will be [num_positive_class_ if not (group == False or group is None or hasattr(group, 'is_member')): raise ValueError( 'Expected group is False, None or instance of paddle.distributed.collective.Group \ - (got group: {})'.format(group)) + (got group: {})'.format( + group + ) + ) return if hasattr(group, 'is_member') and not group.is_member(): @@ -1976,76 +2128,112 @@ class centers and the shape of sampled_class_center will be [num_positive_class_ if core.is_compiled_with_dist(): parallel_env = paddle.distributed.ParallelEnv() global_rank = parallel_env.rank - rank = global_rank if group is None else group.get_group_rank( - global_rank) + rank = ( + global_rank + if group is None + else group.get_group_rank(global_rank) + ) nranks = parallel_env.world_size if group is None else group.nranks if num_samples > num_classes: raise ValueError( - 'Expected num_samples less than or equal to {}, got num_samples {}'. - format(num_classes, num_samples)) + 'Expected num_samples less than or equal to {}, got num_samples {}'.format( + num_classes, num_samples + ) + ) label_size = 1 for dim in list(label.shape): label_size *= dim if label_size != -1 and label_size < 1: - raise ValueError('Expected label_size > 0 \ - (got label_size: {})'.format(label_size)) + raise ValueError( + 'Expected label_size > 0 \ + (got label_size: {})'.format( + label_size + ) + ) label_dims = len(list(label.shape)) if label_dims != 1: - raise ValueError('Expected label_dims == 1 \ - (got label_dims: {})'.format(label_dims)) + raise ValueError( + 'Expected label_dims == 1 \ + (got label_dims: {})'.format( + label_dims + ) + ) seed = None if (seed is None or seed == 0) and default_main_program().random_seed != 0: seed = default_main_program().random_seed if in_dygraph_mode(): - return _C_ops.class_center_sample(label, num_classes, num_samples, - ring_id, rank, nranks, seed - is not None, - seed if seed is not None else 0) + return _C_ops.class_center_sample( + label, + num_classes, + num_samples, + ring_id, + rank, + nranks, + seed is not None, + seed if seed is not None else 0, + ) elif paddle.in_dynamic_mode(): - remapped_label, sampled_class_center = _legacy_C_ops.class_center_sample( - label, 'num_classes', num_classes, 'num_samples', num_samples, - 'ring_id', ring_id, 'nranks', nranks, 'rank', rank, 'fix_seed', seed - is not None, 'seed', seed if seed is not None else 0) + ( + remapped_label, + sampled_class_center, + ) = _legacy_C_ops.class_center_sample( + label, + 'num_classes', + num_classes, + 'num_samples', + num_samples, + 'ring_id', + ring_id, + 'nranks', + nranks, + 'rank', + rank, + 'fix_seed', + seed is not None, + 'seed', + seed if seed is not None else 0, + ) return remapped_label, sampled_class_center - check_variable_and_dtype(label, 'label', ['int64', 'int32'], - 'class_center_sample') + check_variable_and_dtype( + label, 'label', ['int64', 'int32'], 'class_center_sample' + ) op_type = 'class_center_sample' helper = LayerHelper(op_type, **locals()) remapped_label = helper.create_variable_for_type_inference( - dtype=label.dtype) + dtype=label.dtype + ) sampled_class_center = helper.create_variable_for_type_inference( - dtype=label.dtype) - helper.append_op(type=op_type, - inputs={'Label': label}, - outputs={ - 'RemappedLabel': remapped_label, - 'SampledLocalClassCenter': sampled_class_center - }, - attrs={ - 'num_classes': num_classes, - 'num_samples': num_samples, - 'ring_id': ring_id, - 'nranks': nranks, - 'rank': rank, - 'fix_seed': seed is not None, - 'seed': seed if seed is not None else 0 - }) + dtype=label.dtype + ) + helper.append_op( + type=op_type, + inputs={'Label': label}, + outputs={ + 'RemappedLabel': remapped_label, + 'SampledLocalClassCenter': sampled_class_center, + }, + attrs={ + 'num_classes': num_classes, + 'num_samples': num_samples, + 'ring_id': ring_id, + 'nranks': nranks, + 'rank': rank, + 'fix_seed': seed is not None, + 'seed': seed if seed is not None else 0, + }, + ) return remapped_label, sampled_class_center -def fold(x, - output_sizes, - kernel_sizes, - strides=1, - paddings=0, - dilations=1, - name=None): +def fold( + x, output_sizes, kernel_sizes, strides=1, paddings=0, dilations=1, name=None +): r""" Combines an array of sliding local blocks into a large containing @@ -2108,35 +2296,38 @@ def fold(x, check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'fold') - assert len(x.shape) == 3, \ - "input should be the format of [N, C, L]" + assert len(x.shape) == 3, "input should be the format of [N, C, L]" def _is_list_or_turple_(data): - return (isinstance(data, list) or isinstance(data, tuple)) + return isinstance(data, list) or isinstance(data, tuple) if isinstance(output_sizes, int): output_sizes = [output_sizes, output_sizes] else: - assert _is_list_or_turple_(output_sizes) and (len(output_sizes) == 2), \ - "output_sizes should either be an integer or a list/tuple of two integers" + assert _is_list_or_turple_(output_sizes) and ( + len(output_sizes) == 2 + ), "output_sizes should either be an integer or a list/tuple of two integers" if isinstance(kernel_sizes, int): kernel_sizes = [kernel_sizes, kernel_sizes] else: - assert _is_list_or_turple_(kernel_sizes) and (len(kernel_sizes) == 2), \ - "kernel_sizes should either be an integer or a list/tuple of two integers" + assert _is_list_or_turple_(kernel_sizes) and ( + len(kernel_sizes) == 2 + ), "kernel_sizes should either be an integer or a list/tuple of two integers" if isinstance(strides, int): strides = [strides, strides] else: - assert _is_list_or_turple_(strides) and (len(strides) == 2), \ - "strides should either be an integer or a list/tuple of two integers" + assert _is_list_or_turple_(strides) and ( + len(strides) == 2 + ), "strides should either be an integer or a list/tuple of two integers" if isinstance(dilations, int): dilations = [dilations, dilations] else: - assert _is_list_or_turple_(dilations) and (len(dilations) == 2), \ - "dilations should either be an integer or a list/tuple of two integers" + assert _is_list_or_turple_(dilations) and ( + len(dilations) == 2 + ), "dilations should either be an integer or a list/tuple of two integers" if isinstance(paddings, int): paddings = [paddings] * 4 @@ -2152,26 +2343,39 @@ def _is_list_or_turple_(data): else: raise ValueError( "Unexpected type of paddings, it should be either an integer or a list" - "of 2 or 4 integers") + "of 2 or 4 integers" + ) if in_dygraph_mode(): - out = _C_ops.fold(x, output_sizes, kernel_sizes, strides, paddings, - dilations) + out = _C_ops.fold( + x, output_sizes, kernel_sizes, strides, paddings, dilations + ) elif in_dynamic_mode(): - out = _legacy_C_ops.fold(x, "output_sizes", output_sizes, - "kernel_sizes", kernel_sizes, "strides", - strides, "paddings", paddings, "dilations", - dilations) + out = _legacy_C_ops.fold( + x, + "output_sizes", + output_sizes, + "kernel_sizes", + kernel_sizes, + "strides", + strides, + "paddings", + paddings, + "dilations", + dilations, + ) else: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="fold", - inputs={"X": x}, - outputs={"Y": out}, - attrs={ - "output_sizes": output_sizes, - "kernel_sizes": kernel_sizes, - "strides": strides, - "paddings": paddings, - "dilations": dilations - }) + helper.append_op( + type="fold", + inputs={"X": x}, + outputs={"Y": out}, + attrs={ + "output_sizes": output_sizes, + "kernel_sizes": kernel_sizes, + "strides": strides, + "paddings": paddings, + "dilations": dilations, + }, + ) return out diff --git a/python/paddle/nn/functional/conv.py b/python/paddle/nn/functional/conv.py index 17eded8ac06a8..573594216177e 100644 --- a/python/paddle/nn/functional/conv.py +++ b/python/paddle/nn/functional/conv.py @@ -14,7 +14,12 @@ from ...device import get_cudnn_version from ...static import Variable -from ...fluid.layers.utils import convert_to_list, _is_symmetric_padding, _contain_var, _convert_to_tensor_list +from ...fluid.layers.utils import ( + convert_to_list, + _is_symmetric_padding, + _contain_var, + _convert_to_tensor_list, +) from ...fluid.data_feeder import check_variable_and_dtype, check_dtype from ...fluid.layer_helper import LayerHelper from ...tensor.manipulation import unsqueeze, squeeze @@ -56,8 +61,10 @@ def _update_padding_nd(padding, channel_last, num_dims): padding = padding.upper() if padding not in ["SAME", "VALID"]: raise ValueError( - "Unknown padding: '{}'. It can only be 'SAME' or 'VALID'.". - format(padding)) + "Unknown padding: '{}'. It can only be 'SAME' or 'VALID'.".format( + padding + ) + ) if padding == "VALID": padding_algorithm = "VALID" padding = [0] * num_dims @@ -72,10 +79,12 @@ def _update_padding_nd(padding, channel_last, num_dims): if not _zero_padding_in_batch_and_channel(padding, channel_last): raise ValueError( "Non-zero padding({}) in the batch or channel dimensions " - "is not supported.".format(padding)) + "is not supported.".format(padding) + ) padding_algorithm = "EXPLICIT" padding = _exclude_padding_in_batch_and_channel( - padding, channel_last) + padding, channel_last + ) if _is_symmetric_padding(padding, num_dims): padding = padding[0::2] # for padding like [pad_before, pad_after, pad_before, pad_after, ...] @@ -96,42 +105,60 @@ def _update_padding_nd(padding, channel_last, num_dims): padding = convert_to_list(padding, num_dims, 'padding') if not all([p >= 0 for p in padding]): raise ValueError( - "Invalid padding, all value should be larger than or equal to 0, but received: {}" - .format(padding)) + "Invalid padding, all value should be larger than or equal to 0, but received: {}".format( + padding + ) + ) return padding, padding_algorithm -def _conv_nd(x, - weight, - bias=None, - stride=1, - padding=0, - padding_algorithm=None, - dilation=1, - groups=1, - data_format="NCHW", - channel_dim=1, - op_type="conv2d", - use_cudnn=True, - use_mkldnn=False, - name=None): +def _conv_nd( + x, + weight, + bias=None, + stride=1, + padding=0, + padding_algorithm=None, + dilation=1, + groups=1, + data_format="NCHW", + channel_dim=1, + op_type="conv2d", + use_cudnn=True, + use_mkldnn=False, + name=None, +): # Due to the poor performance of NHWC, we transpose the input to NCHW. if in_dygraph_mode() and op_type == "conv2d": - pre_bias = _C_ops.conv2d(x, weight, stride, padding, padding_algorithm, - groups, dilation, data_format, False, -1, - False) + pre_bias = _C_ops.conv2d( + x, + weight, + stride, + padding, + padding_algorithm, + groups, + dilation, + data_format, + False, + -1, + False, + ) if bias is not None: - channel_dim = channel_dim + len( - x.shape) if channel_dim < 0 else channel_dim + channel_dim = ( + channel_dim + len(x.shape) if channel_dim < 0 else channel_dim + ) if isinstance(x, tuple): x = x[0] if isinstance(bias, tuple): bias = bias[0] if len(bias.shape) < len(x.shape): tmp_bias = _C_ops.reshape( - bias, [1 for i in range(channel_dim)] + bias.shape + - [1 for i in range(len(x.shape) - channel_dim - 1)]) + bias, + [1 for i in range(channel_dim)] + + bias.shape + + [1 for i in range(len(x.shape) - channel_dim - 1)], + ) return _C_ops.add(pre_bias, tmp_bias) else: return _C_ops.add(pre_bias, bias) @@ -139,40 +166,82 @@ def _conv_nd(x, return pre_bias if in_dygraph_mode() and op_type == "depthwise_conv2d": - pre_bias = _C_ops.depthwise_conv2d(x, weight, stride, padding, - padding_algorithm, groups, dilation, - data_format, False, -1, False, False, - use_cudnn) + pre_bias = _C_ops.depthwise_conv2d( + x, + weight, + stride, + padding, + padding_algorithm, + groups, + dilation, + data_format, + False, + -1, + False, + False, + use_cudnn, + ) if bias is not None: - channel_dim = channel_dim + len( - x.shape) if channel_dim < 0 else channel_dim + channel_dim = ( + channel_dim + len(x.shape) if channel_dim < 0 else channel_dim + ) tmp_bias = _C_ops.reshape( - bias, [1 for i in range(channel_dim)] + bias.shape + - [1 for i in range(len(x.shape) - channel_dim - 1)]) + bias, + [1 for i in range(channel_dim)] + + bias.shape + + [1 for i in range(len(x.shape) - channel_dim - 1)], + ) return _C_ops.add(pre_bias, tmp_bias) else: return pre_bias if in_dygraph_mode() and op_type == "conv3d": - pre_bias = _C_ops.conv3d(x, weight, stride, padding, padding_algorithm, - groups, dilation, data_format, False, -1, - False) + pre_bias = _C_ops.conv3d( + x, + weight, + stride, + padding, + padding_algorithm, + groups, + dilation, + data_format, + False, + -1, + False, + ) if bias is not None: - channel_dim = channel_dim + len( - x.shape) if channel_dim < 0 else channel_dim + channel_dim = ( + channel_dim + len(x.shape) if channel_dim < 0 else channel_dim + ) tmp_bias = _C_ops.reshape( bias, - bias.shape + [1 for i in range(len(x.shape) - channel_dim - 1)]) + bias.shape + [1 for i in range(len(x.shape) - channel_dim - 1)], + ) return _C_ops.add(pre_bias, tmp_bias) else: return pre_bias if in_dynamic_mode(): - attrs = ('strides', stride, 'paddings', padding, 'dilations', dilation, - 'groups', groups, 'use_cudnn', use_cudnn, 'use_mkldnn', - use_mkldnn, 'fuse_relu_before_depthwise_conv', False, - "padding_algorithm", padding_algorithm, "data_format", - data_format) + attrs = ( + 'strides', + stride, + 'paddings', + padding, + 'dilations', + dilation, + 'groups', + groups, + 'use_cudnn', + use_cudnn, + 'use_mkldnn', + use_mkldnn, + 'fuse_relu_before_depthwise_conv', + False, + "padding_algorithm", + padding_algorithm, + "data_format", + data_format, + ) pre_bias = getattr(_legacy_C_ops, op_type)(x, weight, *attrs) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=channel_dim) @@ -189,44 +258,42 @@ def _conv_nd(x, 'use_mkldnn': use_mkldnn, 'fuse_relu_before_depthwise_conv': False, "padding_algorithm": padding_algorithm, - "data_format": data_format + "data_format": data_format, } - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - op_type) + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], op_type + ) helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') pre_bias = helper.create_variable_for_type_inference(dtype) outputs = {"Output": [pre_bias]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) if bias is not None: out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [bias] - }, - outputs={'Out': [out]}, - attrs={ - 'axis': channel_dim, - 'use_mkldnn': use_mkldnn - }) + helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [bias]}, + outputs={'Out': [out]}, + attrs={'axis': channel_dim, 'use_mkldnn': use_mkldnn}, + ) else: out = pre_bias return out -def conv1d(x, - weight, - bias=None, - stride=1, - padding=0, - dilation=1, - groups=1, - data_format='NCL', - name=None): +def conv1d( + x, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + data_format='NCL', + name=None, +): r""" The convolution1D layer calculates the output based on the input, filter and strides, paddings, dilations, groups parameters. Input and @@ -339,36 +406,45 @@ def conv1d(x, use_cudnn = False if data_format not in ["NCL", "NLC"]: - raise ValueError("Attr(data_format) should be 'NCL' or 'NLC'. " - "Received Attr(data_format): {}.".format(data_format)) + raise ValueError( + "Attr(data_format) should be 'NCL' or 'NLC'. " + "Received Attr(data_format): {}.".format(data_format) + ) - channel_last = (data_format == "NLC") + channel_last = data_format == "NLC" channel_dim = -1 if channel_last else 1 conv2d_data_format = "NHWC" if channel_last else "NCHW" if len(x.shape) != 3: raise ValueError( - "Input x should be 3D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 3D tensor, but received x with the shape of {}".format( + x.shape + ) + ) num_channels = x.shape[channel_dim] num_filters = weight.shape[0] if num_channels < 0: - raise ValueError("The channel dimension of the input({}) " - "should be defined. Received: {}.".format( - x.shape, num_channels)) + raise ValueError( + "The channel dimension of the input({}) " + "should be defined. Received: {}.".format(x.shape, num_channels) + ) if groups <= 0: raise ValueError( - "The groups of conv1d should be greater than 0. Received groups: {}" - .format(groups)) + "The groups of conv1d should be greater than 0. Received groups: {}".format( + groups + ) + ) if num_channels % groups != 0: raise ValueError( "the channel of input must be divisible by groups," "received: the channel of input is {}, the shape of input is {}" - ", the groups is {}".format(num_channels, x.shape, groups)) + ", the groups is {}".format(num_channels, x.shape, groups) + ) if num_filters % groups != 0: raise ValueError( "the number of filters must be divisible by groups," "received: the number of filters is {}, the shape of weight is {}" - ", the groups is {}".format(num_filters, weight.shape, groups)) + ", the groups is {}".format(num_filters, weight.shape, groups) + ) # update attrs padding, padding_algorithm = _update_padding_nd(padding, channel_last, 1) @@ -379,8 +455,10 @@ def conv1d(x, padding = [0] + padding else: raise ValueError( - "The size of padding's dimension should be 1 or 2. But got padding={}" - .format(padding)) + "The size of padding's dimension should be 1 or 2. But got padding={}".format( + padding + ) + ) stride = [1] + convert_to_list(stride, 1, 'stride') dilation = [1] + convert_to_list(dilation, 1, 'dilation') weight = unsqueeze(weight, axis=[-2]) @@ -388,14 +466,18 @@ def conv1d(x, l_type = "conv2d" # When "groups==num_channels and num_filters% num_channels == 0" using depthwise_conv2d has better performance - if (is_compiled_with_cuda() and num_channels == groups and num_channels != 1 - and num_filters % num_channels == 0): + if ( + is_compiled_with_cuda() + and num_channels == groups + and num_channels != 1 + and num_filters % num_channels == 0 + ): l_type = 'depthwise_conv2d' use_cudnn = False # NPU only supports depthwise_conv2d when "input_channel = output_channel = groups" if is_compiled_with_npu(): - if (num_channels == groups and num_channels == num_filters): + if num_channels == groups and num_channels == num_filters: l_type = 'depthwise_conv2d' else: l_type = 'conv2d' @@ -404,17 +486,44 @@ def conv1d(x, x = unsqueeze(x, axis=[squeeze_aixs]) if in_dygraph_mode(): - out = getattr(_C_ops, - l_type)(x, weight, stride, padding, padding_algorithm, - groups, dilation, conv2d_data_format, False, -1, - False, False, use_cudnn) + out = getattr(_C_ops, l_type)( + x, + weight, + stride, + padding, + padding_algorithm, + groups, + dilation, + conv2d_data_format, + False, + -1, + False, + False, + use_cudnn, + ) if bias is not None: out = nn.elementwise_add(out, bias, axis=channel_dim) elif _in_legacy_dygraph(): - attrs = ('strides', stride, 'paddings', padding, 'dilations', dilation, - 'groups', groups, 'use_cudnn', use_cudnn, 'use_mkldnn', False, - 'fuse_relu_before_depthwise_conv', False, "padding_algorithm", - padding_algorithm, "data_format", conv2d_data_format) + attrs = ( + 'strides', + stride, + 'paddings', + padding, + 'dilations', + dilation, + 'groups', + groups, + 'use_cudnn', + use_cudnn, + 'use_mkldnn', + False, + 'fuse_relu_before_depthwise_conv', + False, + "padding_algorithm", + padding_algorithm, + "data_format", + conv2d_data_format, + ) out = getattr(_legacy_C_ops, l_type)(x, weight, *attrs) if bias is not None: out = nn.elementwise_add(out, bias, axis=channel_dim) @@ -429,33 +538,35 @@ def conv1d(x, 'use_mkldnn': False, 'fuse_relu_before_depthwise_conv': False, "padding_algorithm": padding_algorithm, - "data_format": conv2d_data_format + "data_format": conv2d_data_format, } - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - 'conv2d') + check_variable_and_dtype( + x, 'input', ['float16', 'float32', 'float64'], 'conv2d' + ) helper = LayerHelper(l_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) outputs = {"Output": [out]} - helper.append_op(type=l_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=l_type, inputs=inputs, outputs=outputs, attrs=attrs + ) if bias is not None: out = nn.elementwise_add(out, bias, axis=channel_dim) out = squeeze(out, axis=[squeeze_aixs]) return out -def conv2d(x, - weight, - bias=None, - stride=1, - padding=0, - dilation=1, - groups=1, - data_format="NCHW", - name=None): +def conv2d( + x, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + data_format="NCHW", + name=None, +): r""" The convolution2D layer calculates the output based on the input, filter @@ -564,40 +675,52 @@ def conv2d(x, """ # entry checks if data_format not in ["NCHW", "NHWC"]: - raise ValueError("Attr(data_format) should be 'NCHW' or 'NHWC'. " - "Received Attr(data_format): {}.".format(data_format)) + raise ValueError( + "Attr(data_format) should be 'NCHW' or 'NHWC'. " + "Received Attr(data_format): {}.".format(data_format) + ) - channel_last = (data_format == "NHWC") + channel_last = data_format == "NHWC" channel_dim = -1 if channel_last else 1 if len(x.shape) != 4: raise ValueError( - "Input x should be 4D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 4D tensor, but received x with the shape of {}".format( + x.shape + ) + ) num_channels = x.shape[channel_dim] num_filters = weight.shape[0] if num_channels < 0: - raise ValueError("The channel dimension of the input({}) " - "should be defined. Received: {}.".format( - x.shape, num_channels)) + raise ValueError( + "The channel dimension of the input({}) " + "should be defined. Received: {}.".format(x.shape, num_channels) + ) if groups <= 0: raise ValueError( - "The groups of conv2d should be greater than 0. Received groups: {}" - .format(groups)) + "The groups of conv2d should be greater than 0. Received groups: {}".format( + groups + ) + ) if num_channels % groups != 0: raise ValueError( "the channel of input must be divisible by groups," "received: the channel of input is {}, the shape of input is {}" - ", the groups is {}".format(num_channels, x.shape, groups)) + ", the groups is {}".format(num_channels, x.shape, groups) + ) if num_filters % groups != 0: raise ValueError( "the number of filters must be divisible by groups," "received: the number of filters is {}, the shape of weight is {}" - ", the groups is {}".format(num_filters, weight.shape, groups)) + ", the groups is {}".format(num_filters, weight.shape, groups) + ) cudnn_version = get_cudnn_version() - use_cudnn = True if (is_compiled_with_cuda() - and cudnn_version is not None) else False + use_cudnn = ( + True + if (is_compiled_with_cuda() and cudnn_version is not None) + else False + ) # update attrs padding, padding_algorithm = _update_padding_nd(padding, channel_last, 2) @@ -605,8 +728,11 @@ def conv2d(x, dilation = convert_to_list(dilation, 2, 'dilation') l_type = "conv2d" - if (num_channels == groups and num_channels != 1 - and num_filters % num_channels == 0): + if ( + num_channels == groups + and num_channels != 1 + and num_filters % num_channels == 0 + ): l_type = 'depthwise_conv2d' if is_compiled_with_rocm(): use_cudnn = True @@ -614,9 +740,19 @@ def conv2d(x, use_cudnn = False else: if in_dygraph_mode(): - pre_bias = _C_ops.conv2d(x, weight, stride, padding, - padding_algorithm, groups, dilation, - data_format, False, -1, False) + pre_bias = _C_ops.conv2d( + x, + weight, + stride, + padding, + padding_algorithm, + groups, + dilation, + data_format, + False, + -1, + False, + ) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=channel_dim) return out @@ -627,31 +763,50 @@ def conv2d(x, # NPU only supports depthwise_conv2d when "input_channel = output_channel = groups" if is_compiled_with_npu(): - if (num_channels == groups and num_channels == num_filters): + if num_channels == groups and num_channels == num_filters: l_type = 'depthwise_conv2d' else: l_type = 'conv2d' - if (is_compiled_with_cuda() and get_flags("FLAGS_conv2d_disable_cudnn") - ["FLAGS_conv2d_disable_cudnn"]): + if ( + is_compiled_with_cuda() + and get_flags("FLAGS_conv2d_disable_cudnn")[ + "FLAGS_conv2d_disable_cudnn" + ] + ): use_cudnn = False - return _conv_nd(x, weight, bias, stride, padding, padding_algorithm, - dilation, groups, data_format, channel_dim, l_type, - use_cudnn, use_mkldnn, name) - - -def conv1d_transpose(x, - weight, - bias=None, - stride=1, - padding=0, - output_padding=0, - groups=1, - dilation=1, - output_size=None, - data_format="NCL", - name=None): + return _conv_nd( + x, + weight, + bias, + stride, + padding, + padding_algorithm, + dilation, + groups, + data_format, + channel_dim, + l_type, + use_cudnn, + use_mkldnn, + name, + ) + + +def conv1d_transpose( + x, + weight, + bias=None, + stride=1, + padding=0, + output_padding=0, + groups=1, + dilation=1, + output_size=None, + data_format="NCL", + name=None, +): r""" The 1-D convolution transpose layer calculates the output based on the input, filter, and dilation, stride, padding. Input(Input) and output(Output) @@ -780,28 +935,36 @@ def conv1d_transpose(x, raise ValueError( "Attr(data_format) of conv2d_transpose got wrong value: " "received {}, but only 'NCL' or 'NLC' are supported.".format( - data_format)) - channel_last = (data_format == "NLC") + data_format + ) + ) + channel_last = data_format == "NLC" channel_dim = -1 if channel_last else 1 if len(x.shape) != 3: raise ValueError( - "Input x should be 3D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 3D tensor, but received x with the shape of {}".format( + x.shape + ) + ) num_channels = x.shape[channel_dim] if num_channels < 0: - raise ValueError("The channel dimension of the input({}) " - "should be defined. Received: {}.".format( - x.shape, num_channels)) + raise ValueError( + "The channel dimension of the input({}) " + "should be defined. Received: {}.".format(x.shape, num_channels) + ) if groups <= 0: raise ValueError( - "The groups of conv1d_transpose should be greater than 0. Received groups: {}" - .format(groups)) + "The groups of conv1d_transpose should be greater than 0. Received groups: {}".format( + groups + ) + ) if num_channels % groups != 0: raise ValueError( "the channel of input must be divisible by groups," "received: the channel of input is {}, the shape of input is {}" - ", the groups is {}".format(num_channels, x.shape, groups)) + ", the groups is {}".format(num_channels, x.shape, groups) + ) # update attrs padding, padding_algorithm = _update_padding_nd(padding, channel_last, 1) @@ -812,8 +975,10 @@ def conv1d_transpose(x, padding = padding + [0] else: raise ValueError( - "The size of padding's dimension should 1 or 2. But got padding={}". - format(padding)) + "The size of padding's dimension should 1 or 2. But got padding={}".format( + padding + ) + ) stride = convert_to_list(stride, 1, 'stride') + [1] dilation = convert_to_list(dilation, 1, 'dilation') + [1] @@ -822,30 +987,40 @@ def conv1d_transpose(x, output_size = [] else: if output_padding != 0: - raise ValueError('output_padding option is mutually exclusive with ' - 'output_size') + raise ValueError( + 'output_padding option is mutually exclusive with ' + 'output_size' + ) if isinstance(output_size, (list, tuple, int)): output_size = convert_to_list(output_size, 1, 'output_size') + [1] else: raise ValueError( - "output_size should be int, or list, tuple of ints") + "output_size should be int, or list, tuple of ints" + ) if output_padding == 0: output_padding = [] else: - output_padding = convert_to_list(output_padding, 1, - 'output_padding') + [0] + output_padding = convert_to_list( + output_padding, 1, 'output_padding' + ) + [0] if len(output_padding) > 0 and output_padding[0] > stride[0]: raise ValueError( "The size of output_padding should not be greater than stride." "But got output_padding={} and stride={}".format( - output_padding[0], stride[0])) + output_padding[0], stride[0] + ) + ) op_type = 'conv2d_transpose' num_filters = weight.shape[1] - if (num_channels == groups and num_channels != 1 and num_filters == 1 - and not use_cudnn): + if ( + num_channels == groups + and num_channels != 1 + and num_filters == 1 + and not use_cudnn + ): op_type = 'depthwise_conv2d_transpose' use_cudnn = False @@ -856,17 +1031,41 @@ def conv1d_transpose(x, weight = unsqueeze(weight, axis=[-1]) if in_dygraph_mode(): - out = getattr(_C_ops, - op_type)(x, weight, stride, padding, output_padding, - output_size, padding_algorithm, groups, dilation, - conv2d_data_format) + out = getattr(_C_ops, op_type)( + x, + weight, + stride, + padding, + output_padding, + output_size, + padding_algorithm, + groups, + dilation, + conv2d_data_format, + ) if bias is not None: out = nn.elementwise_add(out, bias, axis=channel_dim) elif _in_legacy_dygraph(): - attrs = ('output_padding', output_padding, 'output_size', output_size, - 'strides', stride, 'paddings', padding, 'padding_algorithm', - padding_algorithm, 'dilations', dilation, 'groups', groups, - 'use_cudnn', use_cudnn, 'data_format', conv2d_data_format) + attrs = ( + 'output_padding', + output_padding, + 'output_size', + output_size, + 'strides', + stride, + 'paddings', + padding, + 'padding_algorithm', + padding_algorithm, + 'dilations', + dilation, + 'groups', + groups, + 'use_cudnn', + use_cudnn, + 'data_format', + conv2d_data_format, + ) out = getattr(_legacy_C_ops, op_type)(x, weight, *attrs) if bias is not None: out = nn.elementwise_add(out, bias, axis=channel_dim) @@ -881,18 +1080,18 @@ def conv1d_transpose(x, 'dilations': dilation, 'groups': groups, 'use_cudnn': use_cudnn, - 'data_format': conv2d_data_format + 'data_format': conv2d_data_format, } - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - 'conv2d_transpose') + check_variable_and_dtype( + x, 'input', ['float16', 'float32', 'float64'], 'conv2d_transpose' + ) helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) outputs = {"Output": [out]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) if bias is not None: out = nn.elementwise_add(out, bias, axis=channel_dim) @@ -900,17 +1099,19 @@ def conv1d_transpose(x, return out -def conv2d_transpose(x, - weight, - bias=None, - stride=1, - padding=0, - output_padding=0, - dilation=1, - groups=1, - output_size=None, - data_format='NCHW', - name=None): +def conv2d_transpose( + x, + weight, + bias=None, + stride=1, + padding=0, + output_padding=0, + dilation=1, + groups=1, + output_size=None, + data_format='NCHW', + name=None, +): r""" The convolution2D transpose layer calculates the output based on the input, @@ -1039,32 +1240,43 @@ def conv2d_transpose(x, raise ValueError( "Attr(data_format) of conv2d_transpose got wrong value: " "received {}, but only 'NCHW' or 'NHWC' are supported.".format( - data_format)) - channel_last = (data_format == "NHWC") + data_format + ) + ) + channel_last = data_format == "NHWC" channel_dim = -1 if channel_last else 1 if len(x.shape) != 4: raise ValueError( - "Input x should be 4D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 4D tensor, but received x with the shape of {}".format( + x.shape + ) + ) num_channels = x.shape[channel_dim] if num_channels < 0: - raise ValueError("The channel dimension of the input({}) " - "should be defined. Received: {}.".format( - x.shape, num_channels)) + raise ValueError( + "The channel dimension of the input({}) " + "should be defined. Received: {}.".format(x.shape, num_channels) + ) if groups <= 0: raise ValueError( - "The groups of conv2d_transpose should be greater than 0. Received groups: {}" - .format(groups)) + "The groups of conv2d_transpose should be greater than 0. Received groups: {}".format( + groups + ) + ) if num_channels % groups != 0: raise ValueError( "the channel of input must be divisible by groups," "received: the channel of input is {}, the shape of input is {}" - ", the groups is {}".format(num_channels, x.shape, groups)) + ", the groups is {}".format(num_channels, x.shape, groups) + ) cudnn_version = get_cudnn_version() - use_cudnn = True if (is_compiled_with_cuda() - and cudnn_version is not None) else False + use_cudnn = ( + True + if (is_compiled_with_cuda() and cudnn_version is not None) + else False + ) # update attrs padding, padding_algorithm = _update_padding_nd(padding, channel_last, 2) @@ -1075,8 +1287,10 @@ def conv2d_transpose(x, output_size = [] else: if output_padding != 0: - raise ValueError('output_padding option is mutually exclusive with ' - 'output_size') + raise ValueError( + 'output_padding option is mutually exclusive with ' + 'output_size' + ) if isinstance(output_size, (list, tuple)): if _contain_var(output_size): output_size = _convert_to_tensor_list(output_size) @@ -1085,15 +1299,21 @@ def conv2d_transpose(x, elif isinstance(output_size, int): output_size = convert_to_list(output_size, 2, 'output_size') elif isinstance(output_size, Variable): - check_dtype(output_size.dtype, 'output_size', ['int32', 'int64'], - 'conv2d_transpose') - if len(output_size.shape) == 1 and (output_size.shape[0] == 1 - or output_size.shape[0] == 2): + check_dtype( + output_size.dtype, + 'output_size', + ['int32', 'int64'], + 'conv2d_transpose', + ) + if len(output_size.shape) == 1 and ( + output_size.shape[0] == 1 or output_size.shape[0] == 2 + ): if output_size.shape[0] == 1: output_size = [output_size, output_size] else: raise ValueError( - "output_size must contain one or two integers.") + "output_size must contain one or two integers." + ) else: raise ValueError( "output_size should be int or Tensor or list, tuple of ints or Tensor" @@ -1106,24 +1326,54 @@ def conv2d_transpose(x, op_type = 'conv2d_transpose' num_filters = weight.shape[1] - if (num_channels == groups and num_channels != 1 and num_filters == 1): + if num_channels == groups and num_channels != 1 and num_filters == 1: op_type = 'depthwise_conv2d_transpose' use_cudnn = False if in_dygraph_mode(): - op = _C_ops.conv2d_transpose if op_type == 'conv2d_transpose' else _C_ops.depthwise_conv2d_transpose - pre_bias = op(x, weight, stride, padding, output_padding, output_size, - padding_algorithm, groups, dilation, data_format) + op = ( + _C_ops.conv2d_transpose + if op_type == 'conv2d_transpose' + else _C_ops.depthwise_conv2d_transpose + ) + pre_bias = op( + x, + weight, + stride, + padding, + output_padding, + output_size, + padding_algorithm, + groups, + dilation, + data_format, + ) if bias is not None: return nn.elementwise_add(pre_bias, bias, axis=channel_dim) else: return pre_bias if _in_legacy_dygraph(): - attrs = ('output_padding', output_padding, 'output_size', output_size, - 'strides', stride, 'paddings', padding, 'padding_algorithm', - padding_algorithm, 'dilations', dilation, 'groups', groups, - 'use_cudnn', use_cudnn, 'data_format', data_format) + attrs = ( + 'output_padding', + output_padding, + 'output_size', + output_size, + 'strides', + stride, + 'paddings', + padding, + 'padding_algorithm', + padding_algorithm, + 'dilations', + dilation, + 'groups', + groups, + 'use_cudnn', + use_cudnn, + 'data_format', + data_format, + ) pre_bias = getattr(_legacy_C_ops, op_type)(x, weight, *attrs) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=channel_dim) @@ -1140,17 +1390,17 @@ def conv2d_transpose(x, 'dilations': dilation, 'groups': groups, 'use_cudnn': use_cudnn, - 'data_format': data_format + 'data_format': data_format, } - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'conv2d_transpose') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'conv2d_transpose' + ) helper = LayerHelper(op_type, **locals()) pre_bias = helper.create_variable_for_type_inference(x.dtype) outputs = {"Output": [pre_bias]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=channel_dim) @@ -1160,15 +1410,17 @@ def conv2d_transpose(x, return out -def conv3d(x, - weight, - bias=None, - stride=1, - padding=0, - dilation=1, - groups=1, - data_format="NCDHW", - name=None): +def conv3d( + x, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + data_format="NCDHW", + name=None, +): r""" The convolution3D layer calculates the output based on the input, filter @@ -1276,60 +1528,88 @@ def conv3d(x, if data_format not in ["NCDHW", "NDHWC"]: raise ValueError( "Attr(data_format) should be 'NCDHW' or 'NDHWC'. Received " - "Attr(data_format): {}.".format(data_format)) + "Attr(data_format): {}.".format(data_format) + ) - channel_last = (data_format == "NDHWC") + channel_last = data_format == "NDHWC" channel_dim = -1 if channel_last else 1 if len(x.shape) != 5: raise ValueError( - "Input x should be 5D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 5D tensor, but received x with the shape of {}".format( + x.shape + ) + ) num_channels = x.shape[channel_dim] num_filters = weight.shape[0] if num_channels < 0: raise ValueError( "The channel dimension of the input({}) should be defined. " - "Received: {}.".format(x.shape, num_channels)) + "Received: {}.".format(x.shape, num_channels) + ) if groups <= 0: raise ValueError( - "The groups of conv3d should be greater than 0. Received groups: {}" - .format(groups)) + "The groups of conv3d should be greater than 0. Received groups: {}".format( + groups + ) + ) if num_channels % groups != 0: raise ValueError( "The number of input channels must be divisible by Attr(groups). " "Received: number of channels({}), groups({}).".format( - num_channels, groups)) + num_channels, groups + ) + ) if num_filters % groups != 0: raise ValueError( "The number of filters must be divisible by Attr(groups). " "Received: number of filters({}), groups({}).".format( - num_filters, groups)) + num_filters, groups + ) + ) cudnn_version = get_cudnn_version() - use_cudnn = True if (is_compiled_with_cuda() - and cudnn_version is not None) else False + use_cudnn = ( + True + if (is_compiled_with_cuda() and cudnn_version is not None) + else False + ) padding, padding_algorithm = _update_padding_nd(padding, channel_last, 3) stride = convert_to_list(stride, 3, 'stride') dilation = convert_to_list(dilation, 3, 'dilation') op_type = "conv3d" - return _conv_nd(x, weight, bias, stride, padding, padding_algorithm, - dilation, groups, data_format, channel_dim, op_type, - use_cudnn, False, name) - - -def conv3d_transpose(x, - weight, - bias=None, - stride=1, - padding=0, - output_padding=0, - groups=1, - dilation=1, - output_size=None, - data_format='NCDHW', - name=None): + return _conv_nd( + x, + weight, + bias, + stride, + padding, + padding_algorithm, + dilation, + groups, + data_format, + channel_dim, + op_type, + use_cudnn, + False, + name, + ) + + +def conv3d_transpose( + x, + weight, + bias=None, + stride=1, + padding=0, + output_padding=0, + groups=1, + dilation=1, + output_size=None, + data_format='NCDHW', + name=None, +): r""" The convolution3d transpose layer calculates the output based on the input, filter, and dilations, strides, paddings. Input(Input) and output(Output) @@ -1463,29 +1743,37 @@ def conv3d_transpose(x, if data_format not in ["NCDHW", "NDHWC"]: raise ValueError( "Attr(data_format) should be 'NCDHW' or 'NDHWC'. Received " - "Attr(data_format): {}.".format(data_format)) + "Attr(data_format): {}.".format(data_format) + ) - channel_last = (data_format == "NDHWC") + channel_last = data_format == "NDHWC" channel_dim = -1 if channel_last else 1 if len(x.shape) != 5: raise ValueError( - "Input x should be 5D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 5D tensor, but received x with the shape of {}".format( + x.shape + ) + ) num_channels = x.shape[channel_dim] num_filters = weight.shape[1] if num_channels < 0: raise ValueError( "The channel dimension of the input({}) should be defined. " - "Received: {}.".format(x.shape, num_channels)) + "Received: {}.".format(x.shape, num_channels) + ) if groups <= 0: raise ValueError( - "The groups of conv3d_transpose should be greater than 0. Received groups: {}" - .format(groups)) + "The groups of conv3d_transpose should be greater than 0. Received groups: {}".format( + groups + ) + ) if num_channels % groups != 0: raise ValueError( "The number of input channels must be divisible by Attr(groups). " "Received: number of channels({}), groups({}).".format( - num_channels, groups)) + num_channels, groups + ) + ) padding, padding_algorithm = _update_padding_nd(padding, channel_last, 3) stride = convert_to_list(stride, 3, 'stride') @@ -1494,13 +1782,16 @@ def conv3d_transpose(x, output_size = [] else: if output_padding != 0: - raise ValueError('output_padding option is mutually exclusive with ' - 'output_size') + raise ValueError( + 'output_padding option is mutually exclusive with ' + 'output_size' + ) if isinstance(output_size, (list, tuple, int)): output_size = convert_to_list(output_size, 3, 'output_size') else: raise ValueError( - "output_size should be int, or list, tuple of ints") + "output_size should be int, or list, tuple of ints" + ) if output_padding == 0: output_padding = [] @@ -1509,28 +1800,55 @@ def conv3d_transpose(x, cudnn_version = get_cudnn_version() - #TODO(LielinJiang): whether to use cudnn according to the version of cudnn - use_cudnn = True if (is_compiled_with_cuda() - and cudnn_version is not None) else False + # TODO(LielinJiang): whether to use cudnn according to the version of cudnn + use_cudnn = ( + True + if (is_compiled_with_cuda() and cudnn_version is not None) + else False + ) op_type = 'conv3d_transpose' data_format_ = "NHWC" if channel_last else "NCHW" if in_dygraph_mode(): - pre_bias = _C_ops.conv3d_transpose(x, weight, stride, padding, - output_padding, output_size, - padding_algorithm, groups, dilation, - data_format_) + pre_bias = _C_ops.conv3d_transpose( + x, + weight, + stride, + padding, + output_padding, + output_size, + padding_algorithm, + groups, + dilation, + data_format_, + ) if bias is not None: return nn.elementwise_add(pre_bias, bias, axis=channel_dim) else: return pre_bias if _in_legacy_dygraph(): - attrs = ('output_padding', output_padding, 'output_size', output_size, - 'paddings', padding, "padding_algorithm", padding_algorithm, - 'strides', stride, 'dilations', dilation, 'groups', groups, - 'use_cudnn', use_cudnn, "data_format", data_format_) + attrs = ( + 'output_padding', + output_padding, + 'output_size', + output_size, + 'paddings', + padding, + "padding_algorithm", + padding_algorithm, + 'strides', + stride, + 'dilations', + dilation, + 'groups', + groups, + 'use_cudnn', + use_cudnn, + "data_format", + data_format_, + ) pre_bias = getattr(_legacy_C_ops, op_type)(x, weight, *attrs) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=channel_dim) @@ -1547,19 +1865,19 @@ def conv3d_transpose(x, 'dilations': dilation, 'groups': groups, 'use_cudnn': use_cudnn, - "data_format": data_format_ + "data_format": data_format_, } helper = LayerHelper(op_type, **locals()) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'conv3d') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'conv3d' + ) pre_bias = helper.create_variable_for_type_inference(x.dtype) outputs = {"Output": [pre_bias]} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=channel_dim) else: diff --git a/python/paddle/nn/functional/distance.py b/python/paddle/nn/functional/distance.py index 1c29d50974101..c1d40a83cfb0d 100644 --- a/python/paddle/nn/functional/distance.py +++ b/python/paddle/nn/functional/distance.py @@ -21,7 +21,7 @@ __all__ = [] -def pairwise_distance(x, y, p=2., epsilon=1e-6, keepdim=False, name=None): +def pairwise_distance(x, y, p=2.0, epsilon=1e-6, keepdim=False, name=None): r""" It computes the pairwise distance between two vectors. The distance is calculated by p-oreder norm: @@ -70,40 +70,42 @@ def pairwise_distance(x, y, p=2., epsilon=1e-6, keepdim=False, name=None): sub = _C_ops.subtract(x, y) # p_norm op has not uesd epsilon, so change it to the following. if epsilon != 0.0: - epsilon = paddle.fluid.dygraph.base.to_variable([epsilon], - dtype=sub.dtype) + epsilon = paddle.fluid.dygraph.base.to_variable( + [epsilon], dtype=sub.dtype + ) sub = _C_ops.add(sub, epsilon) - return _C_ops.p_norm(sub, p, -1, 0., keepdim, False) + return _C_ops.p_norm(sub, p, -1, 0.0, keepdim, False) if _in_legacy_dygraph(): sub = _legacy_C_ops.elementwise_sub(x, y) if epsilon != 0.0: - epsilon = paddle.fluid.dygraph.base.to_variable([epsilon], - dtype=sub.dtype) + epsilon = paddle.fluid.dygraph.base.to_variable( + [epsilon], dtype=sub.dtype + ) sub = _legacy_C_ops.elementwise_add(sub, epsilon) - return _legacy_C_ops.p_norm(sub, 'axis', -1, 'porder', p, 'keepdim', - keepdim, 'epsilon', 0.) + return _legacy_C_ops.p_norm( + sub, 'axis', -1, 'porder', p, 'keepdim', keepdim, 'epsilon', 0.0 + ) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'PairwiseDistance') check_variable_and_dtype(y, 'y', ['float32', 'float64'], 'PairwiseDistance') sub = paddle.subtract(x, y) if epsilon != 0.0: epsilon_var = sub.block.create_var(dtype=sub.dtype) - epsilon_var = paddle.full(shape=[1], - fill_value=epsilon, - dtype=sub.dtype) + epsilon_var = paddle.full( + shape=[1], fill_value=epsilon, dtype=sub.dtype + ) sub = paddle.add(sub, epsilon_var) helper = LayerHelper("PairwiseDistance", name=name) attrs = { 'axis': -1, 'porder': p, 'keepdim': keepdim, - 'epsilon': 0., + 'epsilon': 0.0, } out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='p_norm', - inputs={'X': sub}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='p_norm', inputs={'X': sub}, outputs={'Out': out}, attrs=attrs + ) return out diff --git a/python/paddle/nn/functional/extension.py b/python/paddle/nn/functional/extension.py index cc11472643c21..1cc9ad6caf436 100644 --- a/python/paddle/nn/functional/extension.py +++ b/python/paddle/nn/functional/extension.py @@ -22,7 +22,11 @@ from ...tensor.layer_function_generator import templatedoc from paddle import in_dynamic_mode from paddle import _C_ops, _legacy_C_ops -from ...fluid.framework import _non_static_mode, _in_legacy_dygraph, in_dygraph_mode +from ...fluid.framework import ( + _non_static_mode, + _in_legacy_dygraph, + in_dygraph_mode, +) from ...fluid.data_feeder import check_variable_and_dtype, check_type from ...framework import core, convert_np_dtype_to_dtype_ @@ -102,50 +106,55 @@ def diag_embed(input, offset=0, dim1=-2, dim2=-1): if in_dygraph_mode(): return _C_ops.diag_embed(input, offset, dim1, dim2) elif in_dynamic_mode(): - return _legacy_C_ops.diag_embed(input, "offset", offset, "dim1", dim1, - "dim2", dim2) + return _legacy_C_ops.diag_embed( + input, "offset", offset, "dim1", dim1, "dim2", dim2 + ) inputs = {'Input': [input]} attrs = {'offset': offset, 'dim1': dim1, 'dim2': dim2} def __check_input(input, offset, dim1, dim2): - check_dtype(input.dtype, 'Input', - ['int32', 'int64', 'float16', 'float32', 'float64'], - 'diag_embed') + check_dtype( + input.dtype, + 'Input', + ['int32', 'int64', 'float16', 'float32', 'float64'], + 'diag_embed', + ) input_shape = list(input.shape) - assert len(input_shape) >= 1, \ - "Input must be at least 1-dimensional, " \ - "But received Input's dimensional: %s.\n" % \ - len(input_shape) + assert len(input_shape) >= 1, ( + "Input must be at least 1-dimensional, " + "But received Input's dimensional: %s.\n" % len(input_shape) + ) - assert np.abs(dim1) <= len(input_shape), \ - "Dim1 is out of range (expected to be in range of [%d, %d], but got %d).\n" \ + assert np.abs(dim1) <= len(input_shape), ( + "Dim1 is out of range (expected to be in range of [%d, %d], but got %d).\n" % (-(len(input_shape) + 1), len(input_shape), dim1) + ) - assert np.abs(dim2) <= len(input_shape), \ - "Dim2 is out of range (expected to be in range of [%d, %d], but got %d).\n" \ + assert np.abs(dim2) <= len(input_shape), ( + "Dim2 is out of range (expected to be in range of [%d, %d], but got %d).\n" % (-(len(input_shape) + 1), len(input_shape), dim2) + ) dim1_ = dim1 if dim1 >= 0 else len(input_shape) + dim1 + 1 dim2_ = dim2 if dim2 >= 0 else len(input_shape) + dim2 + 1 - assert dim1_ != dim2_, \ - "dim1 and dim2 cannot be the same dimension." \ - "But received dim1 = %d, dim2 = %d\n"%(dim1, dim2) + assert dim1_ != dim2_, ( + "dim1 and dim2 cannot be the same dimension." + "But received dim1 = %d, dim2 = %d\n" % (dim1, dim2) + ) __check_input(input, offset, dim1, dim2) helper = LayerHelper("diag_embed", **locals()) out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='diag_embed', - inputs={'Input': [input]}, - attrs={ - 'offset': offset, - 'dim1': dim1, - 'dim2': dim2 - }, - outputs={'Out': [out]}) + helper.append_op( + type='diag_embed', + inputs={'Input': [input]}, + attrs={'offset': offset, 'dim1': dim1, 'dim2': dim2}, + outputs={'Out': [out]}, + ) out.stop_gradient = True return out @@ -233,10 +242,9 @@ def sequence_mask(x, maxlen=None, dtype='int64', name=None): else: attrs['maxlen'] = maxlen - helper.append_op(type='sequence_mask', - inputs=inputs, - outputs={'Y': out}, - attrs=attrs) + helper.append_op( + type='sequence_mask', inputs=inputs, outputs={'Y': out}, attrs=attrs + ) out.stop_gradient = True return out @@ -317,18 +325,19 @@ def gather_tree(ids, parents): return _legacy_C_ops.gather_tree(ids, parents) else: helper = LayerHelper('gather_tree', **locals()) - check_variable_and_dtype(ids, 'ids', ['int32', 'int64'], - 'gather_tree') - check_variable_and_dtype(parents, 'parents', ['int32', 'int64'], - 'gather_tree') + check_variable_and_dtype( + ids, 'ids', ['int32', 'int64'], 'gather_tree' + ) + check_variable_and_dtype( + parents, 'parents', ['int32', 'int64'], 'gather_tree' + ) out = helper.create_variable_for_type_inference(dtype=ids.dtype) - helper.append_op(type="gather_tree", - inputs={ - "Ids": ids, - "Parents": parents - }, - outputs={"Out": out}) + helper.append_op( + type="gather_tree", + inputs={"Ids": ids, "Parents": parents}, + outputs={"Out": out}, + ) return out @@ -365,14 +374,22 @@ def temporal_shift(x, seg_num, shift_ratio=0.25, name=None, data_format="NCHW"): out = F.temporal_shift(x=input, seg_num=2, shift_ratio=0.2) """ if data_format not in ["NCHW", "NHWC"]: - raise ValueError("Attr(data_format) should be 'NCHW' or 'NHWC'. " - "Received Attr(data_format): {}.".format(data_format)) + raise ValueError( + "Attr(data_format) should be 'NCHW' or 'NHWC'. " + "Received Attr(data_format): {}.".format(data_format) + ) if in_dygraph_mode(): return _C_ops.temporal_shift(x, seg_num, shift_ratio, data_format) if _non_static_mode(): - return _legacy_C_ops.temporal_shift(x, 'seg_num', seg_num, - 'shift_ratio', shift_ratio, - 'data_format', data_format) + return _legacy_C_ops.temporal_shift( + x, + 'seg_num', + seg_num, + 'shift_ratio', + shift_ratio, + 'data_format', + data_format, + ) helper = LayerHelper("temporal_shift", **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'temporal_shift') @@ -384,12 +401,14 @@ def temporal_shift(x, seg_num, shift_ratio=0.25, name=None, data_format="NCHW"): if not isinstance(seg_num, int): raise TypeError("seg_num must be int type.") - helper.append_op(type="temporal_shift", - inputs={"X": x}, - outputs={"Out": out}, - attrs={ - "seg_num": seg_num, - "shift_ratio": shift_ratio, - "data_format": data_format - }) + helper.append_op( + type="temporal_shift", + inputs={"X": x}, + outputs={"Out": out}, + attrs={ + "seg_num": seg_num, + "shift_ratio": shift_ratio, + "data_format": data_format, + }, + ) return out diff --git a/python/paddle/nn/functional/input.py b/python/paddle/nn/functional/input.py index 8df602e5c776d..674483208204c 100644 --- a/python/paddle/nn/functional/input.py +++ b/python/paddle/nn/functional/input.py @@ -88,15 +88,18 @@ def one_hot(x, num_classes, name=None): return _C_ops.one_hot(x, num_classes) else: if _in_legacy_dygraph(): - return _legacy_C_ops.one_hot_v2(x, 'depth', num_classes, - 'allow_out_of_range', False) + return _legacy_C_ops.one_hot_v2( + x, 'depth', num_classes, 'allow_out_of_range', False + ) else: - check_variable_and_dtype(x, 'input', ['int32', 'int64'], - 'one_hot_v2') + check_variable_and_dtype( + x, 'input', ['int32', 'int64'], 'one_hot_v2' + ) helper = LayerHelper("one_hot_v2", **locals()) one_hot_out = helper.create_variable_for_type_inference( - dtype='float32') + dtype='float32' + ) if not isinstance(num_classes, Variable): # user attribute inputs = {'X': x} @@ -105,11 +108,13 @@ def one_hot(x, num_classes, name=None): num_classes.stop_gradient = True inputs = {'X': x, 'depth_tensor': num_classes} attrs = {'allow_out_of_range': False} - helper.append_op(type="one_hot_v2", - inputs=inputs, - attrs=attrs, - outputs={'Out': one_hot_out}, - stop_gradient=True) + helper.append_op( + type="one_hot_v2", + inputs=inputs, + attrs=attrs, + outputs={'Out': one_hot_out}, + stop_gradient=True, + ) return one_hot_out @@ -189,43 +194,61 @@ def embedding(x, weight, padding_idx=None, sparse=False, name=None): x=x, weight=w, sparse=True, name="embedding") """ - padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( - weight.shape[0] + padding_idx) + padding_idx = ( + -1 + if padding_idx is None + else padding_idx + if padding_idx >= 0 + else (weight.shape[0] + padding_idx) + ) if padding_idx >= weight.shape[0] or padding_idx < -weight.shape[0]: - raise ValueError("padding_idx must be within [-{}, {})".format( - weight.shape[0], weight.shape[0])) + raise ValueError( + "padding_idx must be within [-{}, {})".format( + weight.shape[0], weight.shape[0] + ) + ) if in_dygraph_mode(): return _C_ops.embedding(x, weight, padding_idx, sparse) elif _in_legacy_dygraph(): - return _legacy_C_ops.lookup_table_v2(weight, x, 'is_sparse', sparse, - 'is_distributed', False, - 'remote_prefetch', False, - 'padding_idx', padding_idx) + return _legacy_C_ops.lookup_table_v2( + weight, + x, + 'is_sparse', + sparse, + 'is_distributed', + False, + 'remote_prefetch', + False, + 'padding_idx', + padding_idx, + ) else: helper = LayerHelper('embedding', **locals()) dtype = helper.input_dtype(input_param_name='weight') - check_variable_and_dtype(x, 'input', - ['uint8', 'int8', 'int16', 'int32', 'int64'], - 'embedding') + check_variable_and_dtype( + x, + 'input', + ['uint8', 'int8', 'int16', 'int32', 'int64'], + 'embedding', + ) is_distributed = False remote_prefetch = sparse and (not is_distributed) tmp = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='lookup_table_v2', - inputs={ - 'Ids': x, - 'W': weight - }, - outputs={'Out': tmp}, - attrs={ - 'is_sparse': sparse, - 'is_distributed': is_distributed, - 'remote_prefetch': remote_prefetch, - 'padding_idx': padding_idx - }) + helper.append_op( + type='lookup_table_v2', + inputs={'Ids': x, 'W': weight}, + outputs={'Out': tmp}, + attrs={ + 'is_sparse': sparse, + 'is_distributed': is_distributed, + 'remote_prefetch': remote_prefetch, + 'padding_idx': padding_idx, + }, + ) return tmp diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py index ed28bc2190261..48cda9d0b4f95 100755 --- a/python/paddle/nn/functional/loss.py +++ b/python/paddle/nn/functional/loss.py @@ -28,7 +28,12 @@ from paddle import _C_ops, _legacy_C_ops from paddle import in_dynamic_mode from paddle.framework import core, _non_static_mode -from ...fluid.framework import _in_legacy_dygraph, in_dygraph_mode, _non_static_mode, _current_expected_place +from ...fluid.framework import ( + _in_legacy_dygraph, + in_dygraph_mode, + _non_static_mode, + _current_expected_place, +) __all__ = [] @@ -77,25 +82,32 @@ def dice_loss(input, label, epsilon=0.00001, name=None): """ assert input.dtype in (paddle.float32, paddle.float64) assert label.dtype in (paddle.int32, paddle.int64) - assert len(input.shape) >= 2, \ - "The rank of input should be greater than or equal to 2." - assert len(input.shape) == len( - label.shape), ("The rank of input and label should be equal, " - "but received input: %d, label: %d." % - (len(input.shape), len(label.shape))) - assert label.shape[-1] == 1, ("The last dimension of label should be 1, " - "but received %d." % label.shape[-1]) - assert input.shape[:-1] == label.shape[:-1], ( - "All dimensions should be equal except the last one.") - assert input.numel() > 0 and label.numel() > 0, \ - "Any dimension of input and label cannot be equal to 0." + assert ( + len(input.shape) >= 2 + ), "The rank of input should be greater than or equal to 2." + assert len(input.shape) == len(label.shape), ( + "The rank of input and label should be equal, " + "but received input: %d, label: %d." + % (len(input.shape), len(label.shape)) + ) + assert label.shape[-1] == 1, ( + "The last dimension of label should be 1, " + "but received %d." % label.shape[-1] + ) + assert ( + input.shape[:-1] == label.shape[:-1] + ), "All dimensions should be equal except the last one." + assert ( + input.numel() > 0 and label.numel() > 0 + ), "Any dimension of input and label cannot be equal to 0." label = paddle.squeeze(label, [-1]) label = paddle.nn.functional.one_hot(label, input.shape[-1]) reduce_dim = list(range(1, len(input.shape))) inse = paddle.sum(input * label, axis=reduce_dim) dice_denominator = paddle.sum(input, axis=reduce_dim) + paddle.sum( - label, axis=reduce_dim) + label, axis=reduce_dim + ) dice_score = 1 - inse * 2 / (dice_denominator + epsilon) return paddle.mean(dice_score) @@ -146,23 +158,24 @@ def log_loss(input, label, epsilon=1e-4, name=None): loss = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='log_loss', - inputs={ - 'Predicted': [input], - 'Labels': [label] - }, - outputs={'Loss': [loss]}, - attrs={'epsilon': epsilon}) + helper.append_op( + type='log_loss', + inputs={'Predicted': [input], 'Labels': [label]}, + outputs={'Loss': [loss]}, + attrs={'epsilon': epsilon}, + ) return loss -def fluid_softmax_with_cross_entropy(logits, - label, - soft_label=False, - ignore_index=-100, - numeric_stable_mode=True, - return_softmax=False, - axis=-1): +def fluid_softmax_with_cross_entropy( + logits, + label, + soft_label=False, + ignore_index=-100, + numeric_stable_mode=True, + return_softmax=False, + axis=-1, +): r""" This operator implements the cross entropy loss function with softmax. This function @@ -252,19 +265,41 @@ def fluid_softmax_with_cross_entropy(logits, if _non_static_mode(): if core.is_compiled_with_npu(): softmax, backprop, loss = _legacy_C_ops.softmax_with_cross_entropy( - logits, label, 'soft_label', soft_label, 'ignore_index', - ignore_index, 'numeric_stable_mode', numeric_stable_mode, - 'axis', axis) + logits, + label, + 'soft_label', + soft_label, + 'ignore_index', + ignore_index, + 'numeric_stable_mode', + numeric_stable_mode, + 'axis', + axis, + ) else: if in_dygraph_mode(): softmax, loss = _C_ops.cross_entropy_with_softmax( - logits, label, soft_label, True, numeric_stable_mode, - ignore_index, axis) + logits, + label, + soft_label, + True, + numeric_stable_mode, + ignore_index, + axis, + ) if _in_legacy_dygraph(): softmax, loss = _legacy_C_ops.softmax_with_cross_entropy( - logits, label, 'soft_label', soft_label, 'ignore_index', - ignore_index, 'numeric_stable_mode', numeric_stable_mode, - 'axis', axis) + logits, + label, + 'soft_label', + soft_label, + 'ignore_index', + ignore_index, + 'numeric_stable_mode', + numeric_stable_mode, + 'axis', + axis, + ) if not return_softmax: return loss else: @@ -274,7 +309,7 @@ def fluid_softmax_with_cross_entropy(logits, 'soft_label': soft_label, 'ignore_index': ignore_index, 'numeric_stable_mode': numeric_stable_mode, - 'axis': axis + 'axis': axis, } helper = LayerHelper('softmax_with_cross_entropy', **locals()) softmax = helper.create_variable_for_type_inference(dtype=logits.dtype) @@ -284,13 +319,12 @@ def fluid_softmax_with_cross_entropy(logits, if core.is_compiled_with_npu() or core.is_compiled_with_mlu(): backprop = helper.create_variable_for_type_inference(dtype=logits.dtype) outputs['Backprop'] = backprop - helper.append_op(type='softmax_with_cross_entropy', - inputs={ - 'Logits': logits, - 'Label': label - }, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='softmax_with_cross_entropy', + inputs={'Logits': logits, 'Label': label}, + outputs=outputs, + attrs=attrs, + ) if return_softmax: return loss, softmax @@ -336,34 +370,37 @@ def npair_loss(anchor, positive, labels, l2_reg=0.002): print(npair_loss) """ - check_variable_and_dtype(anchor, 'anchor', ['float32', 'float64'], - 'npair_loss') - check_variable_and_dtype(positive, 'positive', ['float32', 'float64'], - 'positive') - check_variable_and_dtype(labels, 'labels', ['float32', 'float64', 'int64'], - 'labels') + check_variable_and_dtype( + anchor, 'anchor', ['float32', 'float64'], 'npair_loss' + ) + check_variable_and_dtype( + positive, 'positive', ['float32', 'float64'], 'positive' + ) + check_variable_and_dtype( + labels, 'labels', ['float32', 'float64', 'int64'], 'labels' + ) Beta = 0.25 batch_size = labels.shape[0] labels = paddle.reshape(labels, shape=[batch_size, 1]) labels = paddle.tile(labels, repeat_times=[1, batch_size]) - labels = paddle.equal(labels, paddle.transpose(labels, - perm=[1, - 0])).astype('float32') + labels = paddle.equal(labels, paddle.transpose(labels, perm=[1, 0])).astype( + 'float32' + ) labels = labels / paddle.sum(labels, axis=1, keepdim=True) - l2loss = paddle.mean(paddle.sum(paddle.square(anchor), 1)) \ - + paddle.mean(paddle.sum(paddle.square(positive), 1)) + l2loss = paddle.mean(paddle.sum(paddle.square(anchor), 1)) + paddle.mean( + paddle.sum(paddle.square(positive), 1) + ) l2loss = l2loss * Beta * l2_reg - similarity_matrix = paddle.matmul(anchor, - positive, - transpose_x=False, - transpose_y=True) - softmax_ce = fluid_softmax_with_cross_entropy(logits=similarity_matrix, - label=labels, - soft_label=True) + similarity_matrix = paddle.matmul( + anchor, positive, transpose_x=False, transpose_y=True + ) + softmax_ce = fluid_softmax_with_cross_entropy( + logits=similarity_matrix, label=labels, soft_label=True + ) cross_entropy = paddle.sum(labels * softmax_ce, 0) celoss = paddle.mean(cross_entropy) @@ -411,32 +448,35 @@ def square_error_cost(input, label): square_out = _legacy_C_ops.square(minus_out) return square_out - check_variable_and_dtype(input, "input", ['float32', 'float64'], - 'square_error_cost') - check_variable_and_dtype(label, "label", ['float32', 'float64'], - 'square_error_cost') + check_variable_and_dtype( + input, "input", ['float32', 'float64'], 'square_error_cost' + ) + check_variable_and_dtype( + label, "label", ['float32', 'float64'], 'square_error_cost' + ) helper = LayerHelper('square_error_cost', **locals()) minus_out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='elementwise_sub', - inputs={ - 'X': [input], - 'Y': [label] - }, - outputs={'Out': [minus_out]}) + helper.append_op( + type='elementwise_sub', + inputs={'X': [input], 'Y': [label]}, + outputs={'Out': [minus_out]}, + ) square_out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='square', - inputs={'X': [minus_out]}, - outputs={'Out': [square_out]}) + helper.append_op( + type='square', inputs={'X': [minus_out]}, outputs={'Out': [square_out]} + ) return square_out -def edit_distance(input, - label, - normalized=True, - ignored_tokens=None, - input_length=None, - label_length=None): +def edit_distance( + input, + label, + normalized=True, + ignored_tokens=None, + input_length=None, + label_length=None, +): """ This op computes the edit distances, also called Levenshtein distance, between a batch of hypothesis strings and their references. It measures how dissimilar two strings are by counting @@ -512,21 +552,26 @@ def edit_distance(input, erased_input = helper.create_variable_for_type_inference(dtype="int64") erased_label = helper.create_variable_for_type_inference(dtype="int64") - helper.append_op(type="sequence_erase", - inputs={"X": [input]}, - outputs={"Out": [erased_input]}, - attrs={"tokens": ignored_tokens}) + helper.append_op( + type="sequence_erase", + inputs={"X": [input]}, + outputs={"Out": [erased_input]}, + attrs={"tokens": ignored_tokens}, + ) input = erased_input - helper.append_op(type="sequence_erase", - inputs={"X": [label]}, - outputs={"Out": [erased_label]}, - attrs={"tokens": ignored_tokens}) + helper.append_op( + type="sequence_erase", + inputs={"X": [label]}, + outputs={"Out": [erased_label]}, + attrs={"tokens": ignored_tokens}, + ) label = erased_label if in_dygraph_mode(): - return _C_ops.edit_distance(input, label, input_length, label_length, - normalized) + return _C_ops.edit_distance( + input, label, input_length, label_length, normalized + ) this_inputs = {"Hyps": [input], "Refs": [label]} if input_length is not None and label_length is not None: @@ -536,22 +581,19 @@ def edit_distance(input, # edit distance op edit_distance_out = helper.create_variable_for_type_inference(dtype="int64") sequence_num = helper.create_variable_for_type_inference(dtype="int64") - helper.append_op(type="edit_distance", - inputs=this_inputs, - outputs={ - "Out": [edit_distance_out], - "SequenceNum": [sequence_num] - }, - attrs={"normalized": normalized}) + helper.append_op( + type="edit_distance", + inputs=this_inputs, + outputs={"Out": [edit_distance_out], "SequenceNum": [sequence_num]}, + attrs={"normalized": normalized}, + ) return edit_distance_out, sequence_num -def binary_cross_entropy(input, - label, - weight=None, - reduction='mean', - name=None): +def binary_cross_entropy( + input, label, weight=None, reduction='mean', name=None +): """ This op measures the binary_cross_entropy loss between input predictions ``input`` and target labels ``label`` . The binary_cross_entropy loss can be described as: @@ -619,8 +661,9 @@ def binary_cross_entropy(input, if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in binary_cross_entropy should be 'sum', " - "'mean' or 'none', but received %s, which is not allowed." % - reduction) + "'mean' or 'none', but received %s, which is not allowed." + % reduction + ) if in_dygraph_mode(): out = _C_ops.bce_loss(input, label) @@ -640,27 +683,32 @@ def binary_cross_entropy(input, if weight is not None: out = _legacy_C_ops.elementwise_mul(out, weight, 'axis', -1) if reduction == 'sum': - return _legacy_C_ops.reduce_sum(out, 'dim', [0], 'keep_dim', - False, "reduce_all", True) + return _legacy_C_ops.reduce_sum( + out, 'dim', [0], 'keep_dim', False, "reduce_all", True + ) elif reduction == 'mean': return _legacy_C_ops.mean(out) else: return out else: - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'binary_cross_entropy') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'binary_cross_entropy') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'binary_cross_entropy' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'binary_cross_entropy' + ) sub_name = name if weight is None and reduction == 'none' else None helper = LayerHelper("binary_cross_entropy", name=sub_name) out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='bce_loss', - inputs={ - 'X': [input], - 'Label': [label], - }, - outputs={'Out': [out]}) + helper.append_op( + type='bce_loss', + inputs={ + 'X': [input], + 'Label': [label], + }, + outputs={'Out': [out]}, + ) if weight is not None: if isinstance(weight, paddle.static.Variable): @@ -668,7 +716,8 @@ def binary_cross_entropy(input, out = paddle.multiply(out, weight, name=weight_name) else: raise ValueError( - "The weight is not a Tensor, please convert to Tensor.") + "The weight is not a Tensor, please convert to Tensor." + ) if reduction == 'sum': return paddle.sum(out, name=name) @@ -678,12 +727,9 @@ def binary_cross_entropy(input, return out -def binary_cross_entropy_with_logits(logit, - label, - weight=None, - reduction='mean', - pos_weight=None, - name=None): +def binary_cross_entropy_with_logits( + logit, label, weight=None, reduction='mean', pos_weight=None, name=None +): r""" This operator combines the sigmoid layer and the :ref:`api_nn_loss_BCELoss` layer. Also, we can see it as the combine of ``sigmoid_cross_entropy_with_logits`` @@ -765,16 +811,23 @@ def binary_cross_entropy_with_logits(logit, raise ValueError( "The value of 'reduction' in binary_cross_entropy_with_logits " "should be 'sum', 'mean' or 'none', but received %s, which is not allowed." - % reduction) + % reduction + ) if in_dygraph_mode(): - one = _C_ops.full([1], float(1.0), core.VarDesc.VarType.FP32, - _current_expected_place()) - out = _C_ops.sigmoid_cross_entropy_with_logits(logit, label, False, - -100) + one = _C_ops.full( + [1], + float(1.0), + core.VarDesc.VarType.FP32, + _current_expected_place(), + ) + out = _C_ops.sigmoid_cross_entropy_with_logits( + logit, label, False, -100 + ) if pos_weight is not None: log_weight = _C_ops.add( - _C_ops.multiply(label, _C_ops.subtract(pos_weight, one)), one) + _C_ops.multiply(label, _C_ops.subtract(pos_weight, one)), one + ) out = _C_ops.multiply(out, log_weight) if weight is not None: out = _C_ops.multiply(out, weight) @@ -787,14 +840,27 @@ def binary_cross_entropy_with_logits(logit, return out elif _in_legacy_dygraph(): one = _varbase_creator(dtype=logit.dtype) - _legacy_C_ops.fill_constant(one, 'value', float(1.0), 'force_cpu', - False, 'dtype', one.dtype, 'str_value', - '1.0', 'shape', [1]) + _legacy_C_ops.fill_constant( + one, + 'value', + float(1.0), + 'force_cpu', + False, + 'dtype', + one.dtype, + 'str_value', + '1.0', + 'shape', + [1], + ) out = _legacy_C_ops.sigmoid_cross_entropy_with_logits(logit, label) if pos_weight is not None: log_weight = _legacy_C_ops.elementwise_add( _legacy_C_ops.elementwise_mul( - label, _legacy_C_ops.elementwise_sub(pos_weight, one)), one) + label, _legacy_C_ops.elementwise_sub(pos_weight, one) + ), + one, + ) out = _legacy_C_ops.elementwise_mul(out, log_weight) if weight is not None: out = _legacy_C_ops.elementwise_mul(out, weight) @@ -806,30 +872,49 @@ def binary_cross_entropy_with_logits(logit, else: return out - check_variable_and_dtype(logit, 'logit', ['float32', 'float64'], - 'binary_cross_entropy_with_logits') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'binary_cross_entropy_with_logits') + check_variable_and_dtype( + logit, + 'logit', + ['float32', 'float64'], + 'binary_cross_entropy_with_logits', + ) + check_variable_and_dtype( + label, + 'label', + ['float32', 'float64'], + 'binary_cross_entropy_with_logits', + ) sigmoid_name = None if reduction == 'none' and pos_weight is None and weight is None: sigmoid_name = name out = paddle.fluid.layers.sigmoid_cross_entropy_with_logits( - logit, label, name=sigmoid_name) + logit, label, name=sigmoid_name + ) one = paddle.full(shape=[1], fill_value=1.0, dtype=logit.dtype) if pos_weight is not None: - check_variable_and_dtype(pos_weight, 'pos_weight', - ['float32', 'float64'], - 'binary_cross_entropy_with_logits') + check_variable_and_dtype( + pos_weight, + 'pos_weight', + ['float32', 'float64'], + 'binary_cross_entropy_with_logits', + ) log_weight = paddle.add( - paddle.multiply(label, paddle.subtract(pos_weight, one)), one) - pos_weight_name = name if reduction == 'none' and weight is None else None + paddle.multiply(label, paddle.subtract(pos_weight, one)), one + ) + pos_weight_name = ( + name if reduction == 'none' and weight is None else None + ) out = paddle.multiply(out, log_weight, name=pos_weight_name) if weight is not None: - check_variable_and_dtype(weight, 'weight', ['float32', 'float64'], - 'binary_cross_entropy_with_logits') + check_variable_and_dtype( + weight, + 'weight', + ['float32', 'float64'], + 'binary_cross_entropy_with_logits', + ) weight_name = name if reduction == 'none' else None out = paddle.multiply(out, weight, name=weight_name) @@ -840,15 +925,17 @@ def binary_cross_entropy_with_logits(logit, return out -def hsigmoid_loss(input, - label, - num_classes, - weight, - bias=None, - path_table=None, - path_code=None, - is_sparse=False, - name=None): +def hsigmoid_loss( + input, + label, + num_classes, + weight, + bias=None, + path_table=None, + path_code=None, + is_sparse=False, + name=None, +): """ The hierarchical sigmoid organizes the classes into a complete binary tree to reduce the computational complexity and speed up the model training, especially the training of language model. @@ -930,36 +1017,63 @@ def hsigmoid_loss(input, # [1.92374969]] """ if in_dygraph_mode(): - out, _, _ = _C_ops.hierarchical_sigmoid(input, weight, label, - path_table, path_code, bias, - num_classes, is_sparse, 0, [], - [], [], is_sparse) + out, _, _ = _C_ops.hierarchical_sigmoid( + input, + weight, + label, + path_table, + path_code, + bias, + num_classes, + is_sparse, + 0, + [], + [], + [], + is_sparse, + ) return out elif _in_legacy_dygraph(): out, _, _ = _legacy_C_ops.hierarchical_sigmoid( - input, weight, label, path_table, path_code, bias, 'num_classes', - num_classes, 'is_sparse', is_sparse, 'remote_prefetch', is_sparse) + input, + weight, + label, + path_table, + path_code, + bias, + 'num_classes', + num_classes, + 'is_sparse', + is_sparse, + 'remote_prefetch', + is_sparse, + ) return out - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'hsigmoid_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'hsigmoid_loss' + ) check_variable_and_dtype(label, 'label', ['int64'], 'hsigmoid_loss') - check_variable_and_dtype(weight, 'weight', ['float32', 'float64'], - 'hsigmoid_loss') + check_variable_and_dtype( + weight, 'weight', ['float32', 'float64'], 'hsigmoid_loss' + ) if bias is not None: - check_variable_and_dtype(bias, 'bias', ['float32', 'float64'], - 'hsigmoid_loss') + check_variable_and_dtype( + bias, 'bias', ['float32', 'float64'], 'hsigmoid_loss' + ) if path_table is not None: - check_variable_and_dtype(path_table, 'path_table', ['int64'], - 'hsigmoid_loss') + check_variable_and_dtype( + path_table, 'path_table', ['int64'], 'hsigmoid_loss' + ) if path_code is not None: - check_variable_and_dtype(path_code, 'path_code', ['int64'], - 'hsigmoid_loss') + check_variable_and_dtype( + path_code, 'path_code', ['int64'], 'hsigmoid_loss' + ) attrs = { "num_classes": num_classes, "is_sparse": is_sparse, - "remote_prefetch": is_sparse + "remote_prefetch": is_sparse, } inputs = { @@ -968,7 +1082,7 @@ def hsigmoid_loss(input, "Bias": bias, "PathTable": path_table, "PathCode": path_code, - "Label": label + "Label": label, } helper = LayerHelper('hsigmoid_loss', **locals()) @@ -976,10 +1090,9 @@ def hsigmoid_loss(input, pre_out = helper.create_variable_for_type_inference(input.dtype) outputs = {"Out": out, "PreOut": pre_out, "W_Out": weight} - helper.append_op(type="hierarchical_sigmoid", - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type="hierarchical_sigmoid", inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -1036,34 +1149,35 @@ def smooth_l1_loss(input, label, reduction='mean', delta=1.0, name=None): print(output) # [0.068004] """ - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'smooth_l1_loss') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'smooth_l1_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'smooth_l1_loss' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'smooth_l1_loss' + ) if in_dygraph_mode(): out, residual = _C_ops.huber_loss(input, label, delta) else: helper = LayerHelper('huber_loss', **locals()) residual = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) - helper.append_op(type='huber_loss', - inputs={ - 'X': input, - 'Y': label - }, - outputs={ - 'Out': out, - 'Residual': residual - }, - attrs={'delta': delta}) + dtype=helper.input_dtype() + ) + helper.append_op( + type='huber_loss', + inputs={'X': input, 'Y': label}, + outputs={'Out': out, 'Residual': residual}, + attrs={'delta': delta}, + ) if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in smooth_l1_loss should be 'sum', 'mean' or" - " 'none', but received %s, which is not allowed." % reduction) + " 'none', but received %s, which is not allowed." % reduction + ) if reduction == 'none': return out elif reduction == 'mean': @@ -1072,12 +1186,9 @@ def smooth_l1_loss(input, label, reduction='mean', delta=1.0, name=None): return paddle.sum(out) -def margin_ranking_loss(input, - other, - label, - margin=0.0, - reduction='mean', - name=None): +def margin_ranking_loss( + input, other, label, margin=0.0, reduction='mean', name=None +): r""" Calcluate the margin rank loss between the input, other and label, use the math function as follows. @@ -1123,7 +1234,8 @@ def margin_ranking_loss(input, if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in MarginRankingLoss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) if in_dygraph_mode(): out = _C_ops.subtract(other, input) out = _C_ops.multiply(out, label) @@ -1150,12 +1262,15 @@ def margin_ranking_loss(input, return out helper = LayerHelper("margin_ranking_loss", **locals()) - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'margin_rank_loss') - check_variable_and_dtype(other, 'other', ['float32', 'float64'], - 'margin_rank_loss') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'margin_rank_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'margin_rank_loss' + ) + check_variable_and_dtype( + other, 'other', ['float32', 'float64'], 'margin_rank_loss' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'margin_rank_loss' + ) out = paddle.subtract(input, other) neg_label = paddle.neg(label) @@ -1169,24 +1284,28 @@ def margin_ranking_loss(input, result_out = helper.create_variable_for_type_inference(input.dtype) if reduction == 'none': - helper.append_op(type="relu", - inputs={"X": out}, - outputs={"Out": result_out}) + helper.append_op( + type="relu", inputs={"X": out}, outputs={"Out": result_out} + ) return result_out elif reduction == 'sum': out = paddle.nn.functional.relu(out) attrs = {"dim": [0], "keep_dim": False, "reduce_all": True} - helper.append_op(type="reduce_sum", - inputs={"X": out}, - outputs={"Out": result_out}, - attrs=attrs) + helper.append_op( + type="reduce_sum", + inputs={"X": out}, + outputs={"Out": result_out}, + attrs=attrs, + ) return result_out elif reduction == 'mean': out = paddle.nn.functional.relu(out) - helper.append_op(type="mean", - inputs={"X": out}, - outputs={"Out": result_out}, - attrs={}) + helper.append_op( + type="mean", + inputs={"X": out}, + outputs={"Out": result_out}, + attrs={}, + ) return result_out @@ -1250,7 +1369,8 @@ def l1_loss(input, label, reduction='mean', name=None): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in L1Loss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) if in_dygraph_mode(): unreduced = _C_ops.abs(_C_ops.subtract(input, label)) @@ -1262,25 +1382,24 @@ def l1_loss(input, label, reduction='mean', name=None): else: return unreduced elif _in_legacy_dygraph(): - unreduced = _elementwise_op_in_dygraph(input, - label, - axis=-1, - act='abs', - op_name='elementwise_sub') + unreduced = _elementwise_op_in_dygraph( + input, label, axis=-1, act='abs', op_name='elementwise_sub' + ) if reduction == 'mean': return _legacy_C_ops.mean(unreduced) elif reduction == 'sum': - return _legacy_C_ops.reduce_sum(unreduced, 'dim', [0], 'keep_dim', - False, 'reduce_all', True) + return _legacy_C_ops.reduce_sum( + unreduced, 'dim', [0], 'keep_dim', False, 'reduce_all', True + ) else: return unreduced - check_variable_and_dtype(input, 'input', - ['float32', 'float64', 'int32', 'int64'], - 'l1_loss') - check_variable_and_dtype(label, 'label', - ['float32', 'float64', 'int32', 'int64'], - 'l1_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64', 'int32', 'int64'], 'l1_loss' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64', 'int32', 'int64'], 'l1_loss' + ) if reduction == 'sum': unreduced = paddle.fluid.layers.elementwise_sub(input, label, act='abs') @@ -1289,18 +1408,14 @@ def l1_loss(input, label, reduction='mean', name=None): unreduced = paddle.fluid.layers.elementwise_sub(input, label, act='abs') return paddle.mean(unreduced, name=name) else: - return paddle.fluid.layers.elementwise_sub(input, - label, - act='abs', - name=name) - - -def nll_loss(input, - label, - weight=None, - ignore_index=-100, - reduction='mean', - name=None): + return paddle.fluid.layers.elementwise_sub( + input, label, act='abs', name=name + ) + + +def nll_loss( + input, label, weight=None, ignore_index=-100, reduction='mean', name=None +): """ This api returns negative log likelihood. See more detail in :ref:`api_nn_loss_NLLLoss` . @@ -1349,13 +1464,15 @@ def nll_loss(input, if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in nll_loss should be 'sum', 'mean' or " - "'none', but received %s, which is not allowed." % reduction) + "'none', but received %s, which is not allowed." % reduction + ) input_shape = list(input.shape) input_dims = len(input_shape) if input_dims < 2: raise ValueError( - 'Expected 2 or more dimensions (got {})'.format(input_dims)) + 'Expected 2 or more dimensions (got {})'.format(input_dims) + ) n = input_shape[0] c = input_shape[1] if in_dygraph_mode(): @@ -1363,21 +1480,29 @@ def nll_loss(input, input = _C_ops.reshape(input, [n, c, 1, -1]) label = _C_ops.reshape(label, [n, 1, -1]) out_shape = [n] + input_shape[2:] - out, total_weight = _C_ops.nll_loss(input, label, weight, ignore_index, - reduction) + out, total_weight = _C_ops.nll_loss( + input, label, weight, ignore_index, reduction + ) if input_dims != 2 and input_dims != 4 and reduction == 'none': out = _C_ops.reshape(out, out_shape) return out elif _in_legacy_dygraph(): if input_dims != 2 and input_dims != 4: - input, _ = _legacy_C_ops.reshape2(input, None, 'shape', - [n, c, 1, -1]) + input, _ = _legacy_C_ops.reshape2( + input, None, 'shape', [n, c, 1, -1] + ) label, _ = _legacy_C_ops.reshape2(label, None, 'shape', [n, 1, -1]) out_shape = [n] + input_shape[2:] - out, total_weight = _legacy_C_ops.nll_loss(input, label, weight, - 'ignore_index', ignore_index, - 'reduction', reduction) + out, total_weight = _legacy_C_ops.nll_loss( + input, + label, + weight, + 'ignore_index', + ignore_index, + 'reduction', + reduction, + ) if input_dims != 2 and input_dims != 4 and reduction == 'none': out, _ = _legacy_C_ops.reshape2(out, None, 'shape', out_shape) return out @@ -1401,10 +1526,9 @@ def nll_loss(input, total_weight = helper.create_variable_for_type_inference(dtype=input.dtype) outputs = {'Out': out, 'Total_weight': total_weight} - helper.append_op(type='nll_loss', - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='nll_loss', inputs=inputs, outputs=outputs, attrs=attrs + ) if input_dims != 2 and input_dims != 4 and reduction == 'none': out = reshape(out, shape=out_shape) @@ -1482,13 +1606,15 @@ def kl_div(input, label, reduction='mean', name=None): """ # ugly type promotion - if fluid.data_feeder.convert_dtype( - input.dtype) == 'float32' and fluid.data_feeder.convert_dtype( - label.dtype) == 'float64': + if ( + fluid.data_feeder.convert_dtype(input.dtype) == 'float32' + and fluid.data_feeder.convert_dtype(label.dtype) == 'float64' + ): input = paddle.cast(input, 'float64') - elif fluid.data_feeder.convert_dtype( - input.dtype) == 'float64' and fluid.data_feeder.convert_dtype( - label.dtype) == 'float32': + elif ( + fluid.data_feeder.convert_dtype(input.dtype) == 'float64' + and fluid.data_feeder.convert_dtype(label.dtype) == 'float32' + ): label = paddle.cast(label, 'float64') if in_dygraph_mode(): @@ -1521,13 +1647,12 @@ def kl_div(input, label, reduction='mean', name=None): fluid.data_feeder.check_type(reduction, 'reduction', str, 'kl_div') loss = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='kldiv_loss', - inputs={ - 'X': input, - 'Target': label - }, - outputs={'Loss': loss}, - attrs={'reduction': 'none'}) + helper.append_op( + type='kldiv_loss', + inputs={'X': input, 'Target': label}, + outputs={'Loss': loss}, + attrs={'reduction': 'none'}, + ) if reduction == 'mean': loss = paddle.mean(loss) @@ -1590,31 +1715,38 @@ def mse_loss(input, label, reduction='mean', name=None): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'mse_loss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) if not in_dynamic_mode(): - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'mse_loss') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'mse_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'mse_loss' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'mse_loss' + ) if reduction == 'none': return paddle.square(paddle.subtract(input, label), name=name) elif reduction == 'mean': - return paddle.mean(paddle.square(paddle.subtract(input, label)), - name=name) + return paddle.mean( + paddle.square(paddle.subtract(input, label)), name=name + ) else: - return paddle.sum(paddle.square(paddle.subtract(input, label)), - name=name) + return paddle.sum( + paddle.square(paddle.subtract(input, label)), name=name + ) -def ctc_loss(log_probs, - labels, - input_lengths, - label_lengths, - blank=0, - reduction='mean', - norm_by_times=False): +def ctc_loss( + log_probs, + labels, + input_lengths, + label_lengths, + blank=0, + reduction='mean', + norm_by_times=False, +): """ An operator integrating the open source Warp-CTC library (https://github.com/baidu-research/warp-ctc) @@ -1693,8 +1825,9 @@ def ctc_loss(log_probs, """ - loss_out = fluid.layers.warpctc(log_probs, labels, blank, norm_by_times, - input_lengths, label_lengths) + loss_out = fluid.layers.warpctc( + log_probs, labels, blank, norm_by_times, input_lengths, label_lengths + ) loss_out = paddle.squeeze(loss_out, [-1]) assert reduction in ['mean', 'sum', 'none'] @@ -1705,15 +1838,17 @@ def ctc_loss(log_probs, return loss_out -def margin_cross_entropy(logits, - label, - margin1=1.0, - margin2=0.5, - margin3=0.0, - scale=64.0, - group=None, - return_softmax=False, - reduction='mean'): +def margin_cross_entropy( + logits, + label, + margin1=1.0, + margin2=0.5, + margin3=0.0, + scale=64.0, + group=None, + return_softmax=False, + reduction='mean', +): r""" .. math:: @@ -1901,7 +2036,10 @@ def margin_cross_entropy(logits, if not (group == False or group is None or hasattr(group, 'is_member')): raise ValueError( 'Expected group is False, None or instance of paddle.distributed.collective.Group \ - (got group: {})'.format(group)) + (got group: {})'.format( + group + ) + ) return if hasattr(group, 'is_member') and not group.is_member(): @@ -1915,8 +2053,11 @@ def margin_cross_entropy(logits, if core.is_compiled_with_dist(): parallel_env = paddle.distributed.ParallelEnv() global_rank = parallel_env.rank - rank = global_rank if group is None else group.get_group_rank( - global_rank) + rank = ( + global_rank + if group is None + else group.get_group_rank(global_rank) + ) nranks = parallel_env.world_size if group is None else group.nranks input_dims = len(list(logits.shape)) @@ -1924,15 +2065,26 @@ def margin_cross_entropy(logits, if input_dims - 1 != label_dims and input_dims != label_dims: raise ValueError( 'Expected input_dims - 1 = label_dims or input_dims == label_dims\ - (got nput_dims{}, label_dims{})'.format(input_dims, label_dims)) + (got nput_dims{}, label_dims{})'.format( + input_dims, label_dims + ) + ) if input_dims - 1 == label_dims: label = paddle.unsqueeze(label, axis=-1) if in_dygraph_mode(): - softmax, loss = _C_ops.margin_cross_entropy(logits, label, - return_softmax, ring_id, - rank, nranks, margin1, - margin2, margin3, scale) + softmax, loss = _C_ops.margin_cross_entropy( + logits, + label, + return_softmax, + ring_id, + rank, + nranks, + margin1, + margin2, + margin3, + scale, + ) if reduction == 'mean': loss = paddle.mean(loss) elif reduction == 'sum': @@ -1943,9 +2095,25 @@ def margin_cross_entropy(logits, return loss, softmax elif _in_legacy_dygraph(): softmax, loss = _legacy_C_ops.margin_cross_entropy( - logits, label, 'ring_id', ring_id, 'rank', rank, 'nranks', nranks, - 'margin1', margin1, 'margin2', margin2, 'margin3', margin3, 'scale', - scale, 'return_softmax', return_softmax) + logits, + label, + 'ring_id', + ring_id, + 'rank', + rank, + 'nranks', + nranks, + 'margin1', + margin1, + 'margin2', + margin2, + 'margin3', + margin3, + 'scale', + scale, + 'return_softmax', + return_softmax, + ) if reduction == 'mean': loss = paddle.mean(loss) elif reduction == 'sum': @@ -1960,31 +2128,31 @@ def margin_cross_entropy(logits, softmax = helper.create_variable_for_type_inference(dtype=logits.dtype) loss = helper.create_variable_for_type_inference(dtype=logits.dtype) - check_variable_and_dtype(logits, 'logits', - ['float16', 'float32', 'float64'], - 'margin_cross_entropy') - check_variable_and_dtype(label, 'label', ['int32', 'int64'], - 'margin_cross_entropy') - - helper.append_op(type=op_type, - inputs={ - 'Logits': logits, - 'Label': label - }, - outputs={ - 'Softmax': softmax, - 'Loss': loss - }, - attrs={ - 'return_softmax': return_softmax, - 'ring_id': ring_id, - 'rank': rank, - 'nranks': nranks, - 'margin1': margin1, - 'margin2': margin2, - 'margin3': margin3, - 'scale': scale, - }) + check_variable_and_dtype( + logits, + 'logits', + ['float16', 'float32', 'float64'], + 'margin_cross_entropy', + ) + check_variable_and_dtype( + label, 'label', ['int32', 'int64'], 'margin_cross_entropy' + ) + + helper.append_op( + type=op_type, + inputs={'Logits': logits, 'Label': label}, + outputs={'Softmax': softmax, 'Loss': loss}, + attrs={ + 'return_softmax': return_softmax, + 'ring_id': ring_id, + 'rank': rank, + 'nranks': nranks, + 'margin1': margin1, + 'margin2': margin2, + 'margin3': margin3, + 'scale': scale, + }, + ) if reduction == 'mean': loss = paddle.mean(loss) @@ -2001,16 +2169,20 @@ def margin_cross_entropy(logits, since="2.0.0", update_to="paddle.nn.functional.cross_entropy", level=1, - reason= - ('Please notice that behavior of "paddle.nn.functional.softmax_with_cross_entropy" ' - 'and "paddle.nn.functional.cross_entropy" is different.')) -def softmax_with_cross_entropy(logits, - label, - soft_label=False, - ignore_index=-100, - numeric_stable_mode=True, - return_softmax=False, - axis=-1): + reason=( + 'Please notice that behavior of "paddle.nn.functional.softmax_with_cross_entropy" ' + 'and "paddle.nn.functional.cross_entropy" is different.' + ), +) +def softmax_with_cross_entropy( + logits, + label, + soft_label=False, + ignore_index=-100, + numeric_stable_mode=True, + return_softmax=False, + axis=-1, +): r""" This operator implements the cross entropy loss function with softmax. This function combines the calculation of the softmax operation and the cross entropy loss function @@ -2096,20 +2268,28 @@ def softmax_with_cross_entropy(logits, out = paddle.nn.functional.softmax_with_cross_entropy(logits=x, label=label) print(out) """ - return fluid_softmax_with_cross_entropy(logits, label, soft_label, - ignore_index, numeric_stable_mode, - return_softmax, axis) - - -def cross_entropy(input, - label, - weight=None, - ignore_index=-100, - reduction='mean', - soft_label=False, - axis=-1, - use_softmax=True, - name=None): + return fluid_softmax_with_cross_entropy( + logits, + label, + soft_label, + ignore_index, + numeric_stable_mode, + return_softmax, + axis, + ) + + +def cross_entropy( + input, + label, + weight=None, + ignore_index=-100, + reduction='mean', + soft_label=False, + axis=-1, + use_softmax=True, + name=None, +): r""" By default, this operator implements the cross entropy loss function with softmax. This function combines the calculation of the softmax operation and the cross entropy loss function @@ -2355,12 +2535,14 @@ def cross_entropy(input, raise ValueError( "The value of 'reduction' in softmax_cross_entropy" "should be 'sum', 'mean' or 'none', but received %s, which is not allowed." - % reduction) + % reduction + ) if ignore_index > 0 and soft_label == True: raise ValueError( "When soft_label == True, the value of 'ignore_index' in softmax_cross_entropy" - "should be '-100', but received %s, which is not allowed." % - ignore_index) + "should be '-100', but received %s, which is not allowed." + % ignore_index + ) input_dims = len(list(input.shape)) if input_dims == 0: @@ -2370,29 +2552,53 @@ def cross_entropy(input, if input_dims - 1 != label_dims and input_dims != label_dims: raise ValueError( 'Expected nput_dims - 1 = label_dims or input_dims == label_dims\ - (got nput_dims{}, label_dims{})'.format(input_dims, label_dims)) + (got nput_dims{}, label_dims{})'.format( + input_dims, label_dims + ) + ) if input_dims - 1 == label_dims: label = paddle.unsqueeze(label, axis=axis) if in_dygraph_mode(): if soft_label == False: - valid_label = paddle.cast(label != ignore_index, - dtype=label.dtype) * label + valid_label = ( + paddle.cast(label != ignore_index, dtype=label.dtype) * label + ) if core.is_compiled_with_npu() or core.is_compiled_with_mlu(): if soft_label == False: _, _, out = _legacy_C_ops.softmax_with_cross_entropy( - input, valid_label, 'soft_label', soft_label, - 'ignore_index', ignore_index, 'numeric_stable_mode', True, - 'axis', axis, 'use_softmax', use_softmax) + input, + valid_label, + 'soft_label', + soft_label, + 'ignore_index', + ignore_index, + 'numeric_stable_mode', + True, + 'axis', + axis, + 'use_softmax', + use_softmax, + ) else: _, _, out = _legacy_C_ops.softmax_with_cross_entropy( - input, label, 'soft_label', soft_label, 'ignore_index', - ignore_index, 'numeric_stable_mode', True, 'axis', axis, - 'use_softmax', use_softmax) + input, + label, + 'soft_label', + soft_label, + 'ignore_index', + ignore_index, + 'numeric_stable_mode', + True, + 'axis', + axis, + 'use_softmax', + use_softmax, + ) else: - _, out = _C_ops.cross_entropy_with_softmax(input, label, soft_label, - use_softmax, True, - ignore_index, axis) + _, out = _C_ops.cross_entropy_with_softmax( + input, label, soft_label, use_softmax, True, ignore_index, axis + ) if weight is not None: @@ -2402,11 +2608,12 @@ def cross_entropy(input, # weight's shape is C, where C is class num. # for 1d case: label's shape is [N,C], weight_gather's shape is N. # for 2d case: label's shape is [N,H,W,C], weight_gather's shape is [N,H,W]. - weight_gather = paddle.matmul(x=paddle.cast( - label, weight.dtype), - y=weight, - transpose_x=False, - transpose_y=True) + weight_gather = paddle.matmul( + x=paddle.cast(label, weight.dtype), + y=weight, + transpose_x=False, + transpose_y=True, + ) out_shape = list(out.shape) weight_gather_reshape = reshape(weight_gather, shape=out_shape) out = paddle.cast(out, weight_gather_reshape.dtype) @@ -2417,29 +2624,44 @@ def cross_entropy(input, raise ValueError( "input's class_dimension({}) must equal to " "weight's class_dimension({}) " - "when weight is provided" \ - .format(input.shape[axis], weight.shape[-1])) - - ignore_weight_mask = paddle.cast((label != ignore_index), - out.dtype) - if ignore_weight_mask.ndim > 1 and ignore_weight_mask.shape[ - axis] == 1: + "when weight is provided".format( + input.shape[axis], weight.shape[-1] + ) + ) + + ignore_weight_mask = paddle.cast( + (label != ignore_index), out.dtype + ) + if ( + ignore_weight_mask.ndim > 1 + and ignore_weight_mask.shape[axis] == 1 + ): # TODO: Temporarily use squeeze instead of squeeze_ - ignore_weight_mask = paddle.squeeze(ignore_weight_mask, - axis) + ignore_weight_mask = paddle.squeeze( + ignore_weight_mask, axis + ) if axis != -1 and axis != valid_label.ndim - 1: - temp_perm = list(range(axis % valid_label.ndim)) \ - + list(range((axis % valid_label.ndim + 1), valid_label.ndim)) \ - + [axis % valid_label.ndim] + temp_perm = ( + list(range(axis % valid_label.ndim)) + + list( + range( + (axis % valid_label.ndim + 1), valid_label.ndim + ) + ) + + [axis % valid_label.ndim] + ) weight_gather = _C_ops.gather_nd( - weight, valid_label.transpose(temp_perm)) + weight, valid_label.transpose(temp_perm) + ) else: weight_gather = _C_ops.gather_nd(weight, valid_label) - weight_gather = _C_ops.multiply(weight_gather, - ignore_weight_mask) + weight_gather = _C_ops.multiply( + weight_gather, ignore_weight_mask + ) input_shape = list(label.shape) - weight_gather_reshape = reshape(weight_gather, - shape=input_shape) + weight_gather_reshape = reshape( + weight_gather, shape=input_shape + ) out = paddle.cast(out, weight_gather_reshape.dtype) out = _C_ops.multiply(out, weight_gather_reshape) @@ -2460,22 +2682,24 @@ def cross_entropy(input, # for each label[i],set 1 or 0, according to ignore_index # mask[i]=0, if label[i]==ignore_index # mask[i]=1, otherwise - mask = (label != ignore_index) + mask = label != ignore_index if weight is None: mask = paddle.cast(mask, dtype=out_sum.dtype) count = _C_ops.sum(mask, [], None, False) ret = out_sum / (count + (count == 0.0)) else: mask = paddle.cast(mask, weight_gather_reshape.dtype) - weight_ignored = _C_ops.multiply(mask, - weight_gather_reshape) + weight_ignored = _C_ops.multiply( + mask, weight_gather_reshape + ) weight_sum = _C_ops.sum(weight_ignored, [], None, False) ret = out_sum / (weight_sum + (weight_sum == 0.0)) return ret elif weight is not None: out_sum = _C_ops.sum(out, [], None, False) - total_weight = _C_ops.sum(weight_gather_reshape, [], None, - False) + total_weight = _C_ops.sum( + weight_gather_reshape, [], None, False + ) return out_sum / (total_weight + (total_weight == 0.0)) else: return _C_ops.mean_all(out) @@ -2487,32 +2711,65 @@ def cross_entropy(input, elif _in_legacy_dygraph(): if soft_label == False: - valid_label = paddle.cast(label != ignore_index, - dtype=label.dtype) * label + valid_label = ( + paddle.cast(label != ignore_index, dtype=label.dtype) * label + ) label_min = paddle.min(valid_label) label_max = paddle.max(valid_label) if label_min < 0: - raise ValueError("Target {} is out of lower bound.".format( - label_min.item())) + raise ValueError( + "Target {} is out of lower bound.".format(label_min.item()) + ) if label_max >= input.shape[axis]: - raise ValueError("Target {} is out of upper bound.".format( - label_max.item())) + raise ValueError( + "Target {} is out of upper bound.".format(label_max.item()) + ) if core.is_compiled_with_npu() or core.is_compiled_with_mlu(): if soft_label == False: _, _, out = _legacy_C_ops.softmax_with_cross_entropy( - input, valid_label, 'soft_label', soft_label, - 'ignore_index', ignore_index, 'numeric_stable_mode', True, - 'axis', axis, 'use_softmax', use_softmax) + input, + valid_label, + 'soft_label', + soft_label, + 'ignore_index', + ignore_index, + 'numeric_stable_mode', + True, + 'axis', + axis, + 'use_softmax', + use_softmax, + ) else: _, _, out = _legacy_C_ops.softmax_with_cross_entropy( - input, label, 'soft_label', soft_label, 'ignore_index', - ignore_index, 'numeric_stable_mode', True, 'axis', axis, - 'use_softmax', use_softmax) + input, + label, + 'soft_label', + soft_label, + 'ignore_index', + ignore_index, + 'numeric_stable_mode', + True, + 'axis', + axis, + 'use_softmax', + use_softmax, + ) else: _, out = _legacy_C_ops.softmax_with_cross_entropy( - input, label, 'soft_label', soft_label, 'ignore_index', - ignore_index, 'numeric_stable_mode', True, 'axis', axis, - 'use_softmax', use_softmax) + input, + label, + 'soft_label', + soft_label, + 'ignore_index', + ignore_index, + 'numeric_stable_mode', + True, + 'axis', + axis, + 'use_softmax', + use_softmax, + ) if weight is not None: @@ -2522,11 +2779,12 @@ def cross_entropy(input, # weight's shape is C, where C is class num. # for 1d case: label's shape is [N,C], weight_gather's shape is N. # for 2d case: label's shape is [N,H,W,C], weight_gather's shape is [N,H,W]. - weight_gather = paddle.matmul(x=paddle.cast( - label, weight.dtype), - y=weight, - transpose_x=False, - transpose_y=True) + weight_gather = paddle.matmul( + x=paddle.cast(label, weight.dtype), + y=weight, + transpose_x=False, + transpose_y=True, + ) out_shape = list(out.shape) weight_gather_reshape = reshape(weight_gather, shape=out_shape) out = paddle.cast(out, weight_gather_reshape.dtype) @@ -2538,29 +2796,44 @@ def cross_entropy(input, raise ValueError( "input's class_dimension({}) must equal to " "weight's class_dimension({}) " - "when weight is provided" \ - .format(input.shape[axis], weight.shape[-1])) - - ignore_weight_mask = paddle.cast((label != ignore_index), - out.dtype) - if ignore_weight_mask.ndim > 1 and ignore_weight_mask.shape[ - axis] == 1: + "when weight is provided".format( + input.shape[axis], weight.shape[-1] + ) + ) + + ignore_weight_mask = paddle.cast( + (label != ignore_index), out.dtype + ) + if ( + ignore_weight_mask.ndim > 1 + and ignore_weight_mask.shape[axis] == 1 + ): # TODO: Temporarily use squeeze instead of squeeze_ - ignore_weight_mask = paddle.squeeze(ignore_weight_mask, - axis) + ignore_weight_mask = paddle.squeeze( + ignore_weight_mask, axis + ) if axis != -1 and axis != valid_label.ndim - 1: - temp_perm = list(range(axis % valid_label.ndim)) \ - + list(range((axis % valid_label.ndim + 1), valid_label.ndim)) \ - + [axis % valid_label.ndim] + temp_perm = ( + list(range(axis % valid_label.ndim)) + + list( + range( + (axis % valid_label.ndim + 1), valid_label.ndim + ) + ) + + [axis % valid_label.ndim] + ) weight_gather = _legacy_C_ops.gather_nd( - weight, valid_label.transpose(temp_perm)) + weight, valid_label.transpose(temp_perm) + ) else: weight_gather = _legacy_C_ops.gather_nd(weight, valid_label) weight_gather = _legacy_C_ops.elementwise_mul( - weight_gather, ignore_weight_mask) + weight_gather, ignore_weight_mask + ) input_shape = list(label.shape) - weight_gather_reshape = reshape(weight_gather, - shape=input_shape) + weight_gather_reshape = reshape( + weight_gather, shape=input_shape + ) out = paddle.cast(out, weight_gather_reshape.dtype) out = _legacy_C_ops.elementwise_mul(out, weight_gather_reshape) @@ -2581,7 +2854,7 @@ def cross_entropy(input, # for each label[i],set 1 or 0, according to ignore_index # mask[i]=0, if label[i]==ignore_index # mask[i]=1, otherwise - mask = (label != ignore_index) + mask = label != ignore_index if weight is None: mask = paddle.cast(mask, dtype=out_sum.dtype) count = _legacy_C_ops.reduce_sum(mask, 'reduce_all', True) @@ -2589,15 +2862,18 @@ def cross_entropy(input, else: mask = paddle.cast(mask, weight_gather_reshape.dtype) weight_ignored = _legacy_C_ops.elementwise_mul( - mask, weight_gather_reshape) + mask, weight_gather_reshape + ) weight_sum = _legacy_C_ops.reduce_sum( - weight_ignored, 'reduce_all', True) + weight_ignored, 'reduce_all', True + ) ret = out_sum / (weight_sum + (weight_sum == 0.0)) return ret elif weight is not None: out_sum = _legacy_C_ops.reduce_sum(out, 'reduce_all', True) - total_weight = _legacy_C_ops.reduce_sum(weight_gather_reshape, - 'reduce_all', True) + total_weight = _legacy_C_ops.reduce_sum( + weight_gather_reshape, 'reduce_all', True + ) return out_sum / (total_weight + (total_weight == 0.0)) else: return _legacy_C_ops.mean(out) @@ -2606,18 +2882,24 @@ def cross_entropy(input, out = paddle.squeeze(out, axis=axis) return out - check_variable_and_dtype(input, 'input', ['float16', 'float32', 'float64'], - 'softmax_cross_entropy') check_variable_and_dtype( - label, 'label', + input, + 'input', + ['float16', 'float32', 'float64'], + 'softmax_cross_entropy', + ) + check_variable_and_dtype( + label, + 'label', ['uint8', 'int8', 'int16', 'int32', 'int64', 'float32', 'float64'], - 'softmax_cross_entropy') + 'softmax_cross_entropy', + ) attrs = { 'soft_label': soft_label, 'ignore_index': ignore_index, 'numeric_stable_mode': True, 'axis': axis, - 'use_softmax': use_softmax + 'use_softmax': use_softmax, } helper = LayerHelper('softmax_with_cross_entropy', **locals()) softmax = helper.create_variable_for_type_inference(dtype=input.dtype) @@ -2627,17 +2909,17 @@ def cross_entropy(input, if core.is_compiled_with_npu() or core.is_compiled_with_mlu(): backprop = helper.create_variable_for_type_inference(dtype=input.dtype) outputs['Backprop'] = backprop - helper.append_op(type='softmax_with_cross_entropy', - inputs={ - 'Logits': input, - 'Label': label - }, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='softmax_with_cross_entropy', + inputs={'Logits': input, 'Label': label}, + outputs=outputs, + attrs=attrs, + ) if weight is not None: - check_variable_and_dtype(weight, 'weight', ['float32', 'float64'], - 'softmax_cross_entropy') + check_variable_and_dtype( + weight, 'weight', ['float32', 'float64'], 'softmax_cross_entropy' + ) weight_name = name if reduction == 'none' else None if soft_label == True: # chajchaj: @@ -2645,34 +2927,48 @@ def cross_entropy(input, # weight's shape is C, where C is class num. # for 1d case: label's shape is [N,C], weight_gather's shape is N. # for 2d case: label's shape is [N,H,W,C], weight_gather's shape is [N,H,W]. - weight_gather = paddle.matmul(x=paddle.cast(label, weight.dtype), - y=weight, - transpose_x=False, - transpose_y=True) + weight_gather = paddle.matmul( + x=paddle.cast(label, weight.dtype), + y=weight, + transpose_x=False, + transpose_y=True, + ) out_shape = list(out.shape) weight_gather_reshape = reshape(weight_gather, shape=out_shape) out = paddle.cast(out, weight_gather_reshape.dtype) else: if input.shape[axis] != weight.shape[-1]: - raise ValueError("input's class_dimension({}) must equal to " - "weight's class_dimension({}) " - "when weight is provided" \ - .format(input.shape[axis], weight.shape[-1])) + raise ValueError( + "input's class_dimension({}) must equal to " + "weight's class_dimension({}) " + "when weight is provided".format( + input.shape[axis], weight.shape[-1] + ) + ) valid_label = paddle.multiply( - paddle.cast(label != ignore_index, dtype=label.dtype), label) - ignore_weight_mask = paddle.cast((label != ignore_index), - input.dtype) - if ignore_weight_mask.ndim > 1 and ignore_weight_mask.shape[ - axis] == 1: + paddle.cast(label != ignore_index, dtype=label.dtype), label + ) + ignore_weight_mask = paddle.cast( + (label != ignore_index), input.dtype + ) + if ( + ignore_weight_mask.ndim > 1 + and ignore_weight_mask.shape[axis] == 1 + ): ignore_weight_mask = paddle.squeeze(ignore_weight_mask, axis) if axis != -1 and axis != valid_label.ndim - 1: - temp_perm = list(range(axis % valid_label.ndim)) \ - + list(range((axis % valid_label.ndim + 1), valid_label.ndim)) \ - + [axis % valid_label.ndim] + temp_perm = ( + list(range(axis % valid_label.ndim)) + + list( + range((axis % valid_label.ndim + 1), valid_label.ndim) + ) + + [axis % valid_label.ndim] + ) weight_gather = paddle.gather_nd( - weight, paddle.transpose(valid_label, temp_perm)) + weight, paddle.transpose(valid_label, temp_perm) + ) else: weight_gather = paddle.gather_nd(weight, valid_label) weight_gather = paddle.multiply(weight_gather, ignore_weight_mask) @@ -2689,8 +2985,8 @@ def cross_entropy(input, # for each label[i],set 1 or 0, according to ignore_index # mask[i]=0, if label[i]==ignore_index # mask[i]=1, otherwise - mask = (label != ignore_index) - if (weight is None): + mask = label != ignore_index + if weight is None: mask = paddle.cast(mask, dtype=out_sum.dtype) count = paddle.sum(mask, name=name) ret = out_sum / (count + (count == 0.0)) @@ -2714,13 +3010,15 @@ def cross_entropy(input, return out -def sigmoid_focal_loss(logit, - label, - normalizer=None, - alpha=0.25, - gamma=2.0, - reduction='sum', - name=None): +def sigmoid_focal_loss( + logit, + label, + normalizer=None, + alpha=0.25, + gamma=2.0, + reduction='sum', + name=None, +): r""" `Focal Loss `_ is proposed to address the foreground-background class imbalance for classification tasks. It down-weights @@ -2794,37 +3092,49 @@ def sigmoid_focal_loss(logit, raise ValueError( "The value of 'reduction' in sigmoid_focal_loss " "should be 'sum', 'mean' or 'none', but received %s, which is not allowed." - % reduction) + % reduction + ) if normalizer is not None: - check_variable_and_dtype(normalizer, 'normalizer', - ['float32', 'float64'], 'sigmoid_focal_loss') + check_variable_and_dtype( + normalizer, + 'normalizer', + ['float32', 'float64'], + 'sigmoid_focal_loss', + ) normalizer_shape = list(normalizer.shape) normalizer_dims = len(normalizer_shape) if normalizer_dims > 1: raise ValueError( - "Expected one dimension of normalizer in sigmoid_focal_loss but got {}." - .format(normalizer_dims)) + "Expected one dimension of normalizer in sigmoid_focal_loss but got {}.".format( + normalizer_dims + ) + ) if in_dygraph_mode(): place = _current_expected_place() one = _C_ops.full(logit.shape, float(1.0), logit.dtype, place) - loss = _C_ops.sigmoid_cross_entropy_with_logits(logit, label, False, - -100) + loss = _C_ops.sigmoid_cross_entropy_with_logits( + logit, label, False, -100 + ) pred = _C_ops.sigmoid(logit) p_t = _C_ops.add( _C_ops.multiply(pred, label), - _C_ops.multiply(_C_ops.subtract(one, pred), - _C_ops.subtract(one, label))) + _C_ops.multiply( + _C_ops.subtract(one, pred), _C_ops.subtract(one, label) + ), + ) alpha = fluid.dygraph.base.to_variable([alpha], dtype=loss.dtype) alpha_t = _C_ops.add( _C_ops.multiply(alpha, label), - _C_ops.multiply(_C_ops.subtract(one, alpha), - _C_ops.subtract(one, label))) + _C_ops.multiply( + _C_ops.subtract(one, alpha), _C_ops.subtract(one, label) + ), + ) loss = _C_ops.multiply(alpha_t, loss) gamma = fluid.dygraph.base.to_variable([gamma], dtype=loss.dtype) @@ -2843,9 +3153,19 @@ def sigmoid_focal_loss(logit, elif _in_legacy_dygraph(): one = _varbase_creator(dtype=logit.dtype) - _legacy_C_ops.fill_constant(one, 'value', float(1.0), 'force_cpu', - False, 'dtype', one.dtype, 'str_value', - '1.0', 'shape', logit.shape) + _legacy_C_ops.fill_constant( + one, + 'value', + float(1.0), + 'force_cpu', + False, + 'dtype', + one.dtype, + 'str_value', + '1.0', + 'shape', + logit.shape, + ) loss = _legacy_C_ops.sigmoid_cross_entropy_with_logits(logit, label) pred = _legacy_C_ops.sigmoid(logit) @@ -2854,19 +3174,24 @@ def sigmoid_focal_loss(logit, _legacy_C_ops.elementwise_mul(pred, label), _legacy_C_ops.elementwise_mul( _legacy_C_ops.elementwise_sub(one, pred), - _legacy_C_ops.elementwise_sub(one, label))) + _legacy_C_ops.elementwise_sub(one, label), + ), + ) alpha = fluid.dygraph.base.to_variable([alpha], dtype=loss.dtype) alpha_t = _legacy_C_ops.elementwise_add( _legacy_C_ops.elementwise_mul(alpha, label), _legacy_C_ops.elementwise_mul( _legacy_C_ops.elementwise_sub(one, alpha), - _legacy_C_ops.elementwise_sub(one, label))) + _legacy_C_ops.elementwise_sub(one, label), + ), + ) loss = _legacy_C_ops.elementwise_mul(alpha_t, loss) gamma = fluid.dygraph.base.to_variable([gamma], dtype=loss.dtype) gamma_t = _legacy_C_ops.elementwise_pow( - _legacy_C_ops.elementwise_sub(one, p_t), gamma) + _legacy_C_ops.elementwise_sub(one, p_t), gamma + ) loss = _legacy_C_ops.elementwise_mul(gamma_t, loss) if normalizer is not None: @@ -2879,16 +3204,19 @@ def sigmoid_focal_loss(logit, return loss - check_variable_and_dtype(logit, 'logit', ['float32', 'float64'], - 'sigmoid_focal_loss') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'sigmoid_focal_loss') + check_variable_and_dtype( + logit, 'logit', ['float32', 'float64'], 'sigmoid_focal_loss' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'sigmoid_focal_loss' + ) bce_name = None if reduction == 'none' and normalizer is None: bce_name = name loss = paddle.nn.functional.binary_cross_entropy_with_logits( - logit, label, reduction='none', name=bce_name) + logit, label, reduction='none', name=bce_name + ) pred = paddle.nn.functional.sigmoid(logit) p_t = pred * label + (1 - pred) * (1 - label) @@ -2911,11 +3239,9 @@ def sigmoid_focal_loss(logit, return loss -def multi_label_soft_margin_loss(input, - label, - weight=None, - reduction="mean", - name=None): +def multi_label_soft_margin_loss( + input, label, weight=None, reduction="mean", name=None +): r""" Calculate a multi-class multi-classification hinge loss (margin-based loss) between input :math:`x` (a 2D mini-batch `Tensor`) @@ -2973,25 +3299,42 @@ def multi_label_soft_margin_loss(input, if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'multi_label_soft_margin_loss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) if not (input.shape == label.shape): - raise ValueError("The input and label should have same dimension," - "but received {}!={}".format(input.shape, label.shape)) + raise ValueError( + "The input and label should have same dimension," + "but received {}!={}".format(input.shape, label.shape) + ) if not _non_static_mode(): - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'multilabel_soft_margin_loss') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'multilabel_soft_margin_loss') + check_variable_and_dtype( + input, + 'input', + ['float32', 'float64'], + 'multilabel_soft_margin_loss', + ) + check_variable_and_dtype( + label, + 'label', + ['float32', 'float64'], + 'multilabel_soft_margin_loss', + ) - loss = -(label * paddle.nn.functional.log_sigmoid(input) + - (1 - label) * paddle.nn.functional.log_sigmoid(-input)) + loss = -( + label * paddle.nn.functional.log_sigmoid(input) + + (1 - label) * paddle.nn.functional.log_sigmoid(-input) + ) if weight is not None: if not _non_static_mode(): - check_variable_and_dtype(weight, 'weight', ['float32', 'float64'], - 'multilabel_soft_margin_loss') + check_variable_and_dtype( + weight, + 'weight', + ['float32', 'float64'], + 'multilabel_soft_margin_loss', + ) loss = loss * weight loss = loss.mean(axis=-1) # only return N loss values @@ -3081,17 +3424,21 @@ def hinge_embedding_loss(input, label, margin=1.0, reduction='mean', name=None): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'hinge_embedding_loss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) if not _non_static_mode(): - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'hinge_embedding_loss') - check_variable_and_dtype(label, 'label', ['float32', 'float64'], - 'hinge_embedding_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'hinge_embedding_loss' + ) + check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'hinge_embedding_loss' + ) zero_ = paddle.zeros([1], dtype=input.dtype) - loss = paddle.where(label == 1., input, zero_) + \ - paddle.where(label == -1., paddle.nn.functional.relu(margin - input), zero_) + loss = paddle.where(label == 1.0, input, zero_) + paddle.where( + label == -1.0, paddle.nn.functional.relu(margin - input), zero_ + ) if reduction == 'mean': return paddle.mean(loss, name=name) @@ -3101,12 +3448,9 @@ def hinge_embedding_loss(input, label, margin=1.0, reduction='mean', name=None): return loss -def cosine_embedding_loss(input1, - input2, - label, - margin=0, - reduction='mean', - name=None): +def cosine_embedding_loss( + input1, input2, label, margin=0, reduction='mean', name=None +): r""" This operator computes the cosine embedding loss of Tensor ``input1``, ``input2`` and ``label`` as follows. @@ -3167,12 +3511,14 @@ def cosine_embedding_loss(input1, """ if len(label.shape) != 1: raise ValueError( - "1D target tensor expected, multi-target not supported") + "1D target tensor expected, multi-target not supported" + ) if input1.shape != input2.shape: raise ValueError( "the shape of input tensor 1 should be equal to input tensor 2, but found inputs with " - "different sizes") + "different sizes" + ) if len(input1.shape) > 2: raise ValueError( @@ -3181,9 +3527,13 @@ def cosine_embedding_loss(input1, if input1.dtype not in [paddle.float32, paddle.float64]: raise ValueError( - "The data type of input Variable must be 'float32' or 'float64'") + "The data type of input Variable must be 'float32' or 'float64'" + ) if label.dtype not in [ - paddle.int32, paddle.int64, paddle.float32, paddle.float64 + paddle.int32, + paddle.int64, + paddle.float32, + paddle.float64, ]: raise ValueError( "The data type of label Variable must be 'int32', 'int64', 'float32', 'float64'" @@ -3209,14 +3559,16 @@ def cosine_embedding_loss(input1, return paddle.sum(out, name=name) -def triplet_margin_with_distance_loss(input, - positive, - negative, - distance_function=None, - margin=1.0, - swap=False, - reduction='mean', - name=None): +def triplet_margin_with_distance_loss( + input, + positive, + negative, + distance_function=None, + margin=1.0, + swap=False, + reduction='mean', + name=None, +): r""" Measures the triplet loss given an input tensors :math:`x1`, :math:`x2`, :math:`x3` and a margin with a value greater than :math:`0`. @@ -3291,28 +3643,47 @@ def triplet_margin_with_distance_loss(input, """ if reduction not in ['sum', 'mean', 'none']: - raise ValueError("'reduction' in 'triplet_margin_with_distance_loss' " - "should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + raise ValueError( + "'reduction' in 'triplet_margin_with_distance_loss' " + "should be 'sum', 'mean' or 'none', " + "but received {}.".format(reduction) + ) if margin < 0: raise ValueError( "The margin between positive samples and negative samples should be greater than 0." ) if not _non_static_mode(): - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'triplet_margin_with_distance_loss') - check_variable_and_dtype(positive, 'positive', ['float32', 'float64'], - 'triplet_margin_with_distance_loss') - check_variable_and_dtype(negative, 'negative', ['float32', 'float64'], - 'triplet_margin_with_distance_loss') + check_variable_and_dtype( + input, + 'input', + ['float32', 'float64'], + 'triplet_margin_with_distance_loss', + ) + check_variable_and_dtype( + positive, + 'positive', + ['float32', 'float64'], + 'triplet_margin_with_distance_loss', + ) + check_variable_and_dtype( + negative, + 'negative', + ['float32', 'float64'], + 'triplet_margin_with_distance_loss', + ) if not (input.shape == positive.shape == negative.shape): - raise ValueError("input's shape must equal to " - "positive's shape and " - "negative's shape") + raise ValueError( + "input's shape must equal to " + "positive's shape and " + "negative's shape" + ) - distance_function = distance_function if distance_function is not None \ + distance_function = ( + distance_function + if distance_function is not None else paddle.nn.PairwiseDistance(2) + ) positive_dist = distance_function(input, positive) negative_dist = distance_function(input, negative) @@ -3324,7 +3695,8 @@ def triplet_margin_with_distance_loss(input, if not paddle.all(positive_dist > 0) or not paddle.all(negative_dist > 0): raise ValueError( "The positive distance or negative distance should be greater than 0, " - "The distance functions should be checked.") + "The distance functions should be checked." + ) loss = paddle.clip(positive_dist - negative_dist + margin, min=0.0) @@ -3336,15 +3708,17 @@ def triplet_margin_with_distance_loss(input, return loss -def triplet_margin_loss(input, - positive, - negative, - margin=1.0, - p=2, - epsilon=1e-6, - swap=False, - reduction='mean', - name=None): +def triplet_margin_loss( + input, + positive, + negative, + margin=1.0, + p=2, + epsilon=1e-6, + swap=False, + reduction='mean', + name=None, +): r""" Measures the triplet loss given an input tensors :math:`x1`, :math:`x2`, :math:`x3` and a margin with a value greater than :math:`0`. @@ -3421,23 +3795,29 @@ def triplet_margin_loss(input, if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'triplet_margin_loss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) if margin < 0: raise ValueError( "The margin between positive samples and negative samples should be greater than 0." ) if not _non_static_mode(): - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'triplet_margin_loss') - check_variable_and_dtype(positive, 'positive', ['float32', 'float64'], - 'triplet_margin_loss') - check_variable_and_dtype(negative, 'negative', ['float32', 'float64'], - 'triplet_margin_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'triplet_margin_loss' + ) + check_variable_and_dtype( + positive, 'positive', ['float32', 'float64'], 'triplet_margin_loss' + ) + check_variable_and_dtype( + negative, 'negative', ['float32', 'float64'], 'triplet_margin_loss' + ) if not (input.shape == positive.shape == negative.shape): - raise ValueError("input's shape must equal to " - "positive's shape and " - "negative's shape") + raise ValueError( + "input's shape must equal to " + "positive's shape and " + "negative's shape" + ) distance_function = paddle.nn.PairwiseDistance(p, epsilon=epsilon) positive_dist = distance_function(input, positive) @@ -3457,13 +3837,15 @@ def triplet_margin_loss(input, return loss -def multi_margin_loss(input, - label, - p: int = 1, - margin: float = 1.0, - weight=None, - reduction='mean', - name=None): +def multi_margin_loss( + input, + label, + p: int = 1, + margin: float = 1.0, + weight=None, + reduction='mean', + name=None, +): r""" Measures a multi-class classification hinge loss between input :math:`input` and label :math:`label`: @@ -3526,39 +3908,55 @@ def multi_margin_loss(input, if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'multi_margin_loss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) if not _non_static_mode(): - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'multi_margin_loss') - check_variable_and_dtype(label, 'label', ['int32', 'int64'], - 'multi_margin_loss') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'multi_margin_loss' + ) + check_variable_and_dtype( + label, 'label', ['int32', 'int64'], 'multi_margin_loss' + ) if not (input.shape[0] == label.shape[0]): raise ValueError( "The label's shape[0] should be equal to input's shape[0], " "but received input's shape[0] {} and label's shape[0]:{}. ".format( - input.shape[0], label.shape[0])) + input.shape[0], label.shape[0] + ) + ) label = label.reshape((-1, 1)) index_sample = paddle.index_sample(input, label) if weight is not None: if not _non_static_mode(): - check_variable_and_dtype(weight, 'weight', ['float32', 'float64'], - 'multi_margin_loss') + check_variable_and_dtype( + weight, 'weight', ['float32', 'float64'], 'multi_margin_loss' + ) if not (input.shape[1] == weight.shape[0]): raise ValueError( "The weight's shape[0] should be equal to input's shape[1]" - "but received weight's shape[0]: {} and input's shape[1]: {}". - format(weight.shape[0], input.shape[1])) + "but received weight's shape[0]: {} and input's shape[1]: {}".format( + weight.shape[0], input.shape[1] + ) + ) weight = paddle.gather(weight, label, axis=0).reshape((-1, 1)) loss = paddle.mean( paddle.pow( - paddle.clip(weight * - (margin - index_sample + input), min=0.0), p), - axis=1) - weight * (margin**p / paddle.shape(input)[1]) + paddle.clip(weight * (margin - index_sample + input), min=0.0), + p, + ), + axis=1, + ) - weight * (margin**p / paddle.shape(input)[1]) else: - loss = paddle.mean(paddle.pow( - paddle.clip(margin - index_sample + input, min=0.0), p), - axis=1) - margin**p / paddle.shape(input)[1] + loss = ( + paddle.mean( + paddle.pow( + paddle.clip(margin - index_sample + input, min=0.0), p + ), + axis=1, + ) + - margin**p / paddle.shape(input)[1] + ) if reduction == 'mean': return paddle.mean(loss, name=name) @@ -3621,20 +4019,23 @@ def soft_margin_loss(input, label, reduction='mean', name=None): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in soft_margin_loss should be 'sum', " - "'mean' or 'none', but received %s, which is not allowed." % - reduction) + "'mean' or 'none', but received %s, which is not allowed." + % reduction + ) if not _non_static_mode(): - fluid.data_feeder.check_variable_and_dtype(input, 'input', - ['float32', 'float64'], - 'soft_margin_loss') fluid.data_feeder.check_variable_and_dtype( - label, 'label', ['int32', 'int64', 'float32', 'float64'], - 'soft_margin_loss') + input, 'input', ['float32', 'float64'], 'soft_margin_loss' + ) + fluid.data_feeder.check_variable_and_dtype( + label, + 'label', + ['int32', 'int64', 'float32', 'float64'], + 'soft_margin_loss', + ) if not (input.shape == label.shape): - raise ValueError("input's shape must equal to " - "label's shape") + raise ValueError("input's shape must equal to " "label's shape") label = fluid.layers.cast(label, input.dtype) out = paddle.log(1 + paddle.exp(-label * input)) diff --git a/python/paddle/nn/functional/norm.py b/python/paddle/nn/functional/norm.py index d62dcdcfbf4f4..87d61e9180913 100644 --- a/python/paddle/nn/functional/norm.py +++ b/python/paddle/nn/functional/norm.py @@ -83,18 +83,30 @@ def normalize(x, p=2, axis=1, epsilon=1e-12, name=None): if _in_legacy_dygraph(): eps = fluid.dygraph.base.to_variable([epsilon], dtype=x.dtype) - out = _legacy_C_ops.p_norm(x, 'axis', axis, 'porder', float(p), - 'keepdim', True, 'epsilon', epsilon) + out = _legacy_C_ops.p_norm( + x, + 'axis', + axis, + 'porder', + float(p), + 'keepdim', + True, + 'epsilon', + epsilon, + ) return x / _legacy_C_ops.elementwise_max(out, eps) check_type(p, 'p', (float, int), 'normalize') check_type(axis, 'axis', (int), 'normalize') - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'normalize') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'normalize' + ) if len(x.shape) == 1 and axis != 0 and axis != -1: raise ValueError( - "Axis must be 0 or -1 when x is a 1-D tensor, but received axis = {}" - .format(axis)) + "Axis must be 0 or -1 when x is a 1-D tensor, but received axis = {}".format( + axis + ) + ) attrs = { 'axis': axis, @@ -104,26 +116,27 @@ def normalize(x, p=2, axis=1, epsilon=1e-12, name=None): } helper = LayerHelper('p_norm', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='p_norm', - inputs={'X': x}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='p_norm', inputs={'X': x}, outputs={'Out': out}, attrs=attrs + ) eps = out.block.create_var(dtype=out.dtype) eps = paddle.full(shape=[1], fill_value=epsilon, dtype=out.dtype) return paddle.divide(x, paddle.maximum(out, eps), name=name) -def batch_norm(x, - running_mean, - running_var, - weight, - bias, - training=False, - momentum=0.9, - epsilon=1e-05, - data_format="NCHW", - use_global_stats=None, - name=None): +def batch_norm( + x, + running_mean, + running_var, + weight, + bias, + training=False, + momentum=0.9, + epsilon=1e-05, + data_format="NCHW", + use_global_stats=None, + name=None, +): """ Applies Batch Normalization as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift . @@ -175,7 +188,8 @@ def batch_norm(x, if data_format not in true_data_format: raise ValueError( "data_format must be one of 'NC', 'NCL', 'NCHW', 'NCDHW', " - "'NLC', 'NHWC', 'NDHWC' but receive {}".format(data_format)) + "'NLC', 'NHWC', 'NDHWC' but receive {}".format(data_format) + ) data_format = 'NCHW' if data_format[1] == 'C' else 'NHWC' @@ -187,29 +201,64 @@ def batch_norm(x, if in_dygraph_mode(): batch_norm_out, _, _, _, _, _ = _C_ops.batch_norm( - x, weight, bias, running_mean, running_var, momentum, epsilon, - data_format, not training, use_global_stats, trainable_statistics, - False) - - return dygraph_utils._append_activation_in_dygraph(batch_norm_out, - act=None) + x, + weight, + bias, + running_mean, + running_var, + momentum, + epsilon, + data_format, + not training, + use_global_stats, + trainable_statistics, + False, + ) + + return dygraph_utils._append_activation_in_dygraph( + batch_norm_out, act=None + ) elif _in_legacy_dygraph(): # for dygraph need tuple - attrs = ("momentum", momentum, "epsilon", epsilon, "is_test", - not training, "data_layout", data_format, "use_mkldnn", False, - "fuse_with_relu", False, "use_global_stats", use_global_stats, - "trainable_statistics", trainable_statistics) + attrs = ( + "momentum", + momentum, + "epsilon", + epsilon, + "is_test", + not training, + "data_layout", + data_format, + "use_mkldnn", + False, + "fuse_with_relu", + False, + "use_global_stats", + use_global_stats, + "trainable_statistics", + trainable_statistics, + ) batch_norm_out, _, _, _, _, _ = _legacy_C_ops.batch_norm( - x, weight, bias, running_mean, running_var, None, mean_out, - variance_out, *attrs) - - return dygraph_utils._append_activation_in_dygraph(batch_norm_out, - act=None) - - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - 'BatchNorm') + x, + weight, + bias, + running_mean, + running_var, + None, + mean_out, + variance_out, + *attrs + ) + + return dygraph_utils._append_activation_in_dygraph( + batch_norm_out, act=None + ) + + check_variable_and_dtype( + x, 'input', ['float16', 'float32', 'float64'], 'BatchNorm' + ) # for static need dict attrs = { @@ -228,16 +277,18 @@ def batch_norm(x, "Scale": [weight], "Bias": [bias], "Mean": [running_mean], - "Variance": [running_var] + "Variance": [running_var], } helper = LayerHelper('batch_norm', **locals()) param_dtype = x.dtype if x.dtype != 'float16' else 'float32' - saved_mean = helper.create_variable_for_type_inference(dtype=param_dtype, - stop_gradient=True) + saved_mean = helper.create_variable_for_type_inference( + dtype=param_dtype, stop_gradient=True + ) saved_variance = helper.create_variable_for_type_inference( - dtype=param_dtype, stop_gradient=True) + dtype=param_dtype, stop_gradient=True + ) batch_norm_out = helper.create_variable_for_type_inference(x.dtype) outputs = { @@ -245,29 +296,26 @@ def batch_norm(x, "MeanOut": [running_mean], "VarianceOut": [running_var], "SavedMean": [saved_mean], - "SavedVariance": [saved_variance] + "SavedVariance": [saved_variance], } if training or trainable_statistics: # reserve_space is only used for training. reserve_space = helper.create_variable_for_type_inference( - dtype=x.dtype, stop_gradient=True) + dtype=x.dtype, stop_gradient=True + ) outputs["ReserveSpace"] = [reserve_space] - helper.append_op(type="batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs + ) return helper.append_activation(batch_norm_out) -def layer_norm(x, - normalized_shape, - weight=None, - bias=None, - epsilon=1e-05, - name=None): +def layer_norm( + x, normalized_shape, weight=None, bias=None, epsilon=1e-05, name=None +): """ see more detail in paddle.nn.LayerNorm @@ -304,32 +352,49 @@ def layer_norm(x, normalized_shape = list(normalized_shape) elif not isinstance(normalized_shape, list): raise ValueError( - "`normalized_shape` should be int, list of ints or tuple of ints.") + "`normalized_shape` should be int, list of ints or tuple of ints." + ) normalized_ndim = len(normalized_shape) begin_norm_axis = input_ndim - normalized_ndim - if input_ndim < normalized_ndim or input_shape[ - begin_norm_axis:] != normalized_shape: + if ( + input_ndim < normalized_ndim + or input_shape[begin_norm_axis:] != normalized_shape + ): str_normalized_shape = str(normalized_shape) - raise ValueError('Given normalized_shape is ' + str_normalized_shape + - ', expected input with shape [*, ' + - str_normalized_shape[1:] + ', but got input shape ' + - str(input_shape)) + raise ValueError( + 'Given normalized_shape is ' + + str_normalized_shape + + ', expected input with shape [*, ' + + str_normalized_shape[1:] + + ', but got input shape ' + + str(input_shape) + ) if in_dygraph_mode(): - pre_act, _, _, = _C_ops.layer_norm(x, weight, bias, epsilon, - begin_norm_axis, False) + ( + pre_act, + _, + _, + ) = _C_ops.layer_norm(x, weight, bias, epsilon, begin_norm_axis, False) return dygraph_utils._append_activation_in_dygraph(pre_act, act=None) if _in_legacy_dygraph(): - pre_act, _, _ = _legacy_C_ops.layer_norm(x, weight, bias, 'epsilon', - epsilon, 'begin_norm_axis', - begin_norm_axis) + pre_act, _, _ = _legacy_C_ops.layer_norm( + x, + weight, + bias, + 'epsilon', + epsilon, + 'begin_norm_axis', + begin_norm_axis, + ) return dygraph_utils._append_activation_in_dygraph(pre_act, act=None) - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - 'LayerNorm') + check_variable_and_dtype( + x, 'input', ['float16', 'float32', 'float64'], 'LayerNorm' + ) inputs = dict() inputs['X'] = [x] @@ -343,37 +408,40 @@ def layer_norm(x, helper = LayerHelper('layer_norm', **locals()) dtype = x.dtype - mean_out = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) - variance_out = helper.create_variable_for_type_inference(dtype=dtype, - stop_gradient=True) + mean_out = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) + variance_out = helper.create_variable_for_type_inference( + dtype=dtype, stop_gradient=True + ) layer_norm_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="layer_norm", - inputs=inputs, - outputs={ - "Y": layer_norm_out, - "Mean": mean_out, - "Variance": variance_out, - }, - attrs={ - "epsilon": epsilon, - "begin_norm_axis": begin_norm_axis - }) + helper.append_op( + type="layer_norm", + inputs=inputs, + outputs={ + "Y": layer_norm_out, + "Mean": mean_out, + "Variance": variance_out, + }, + attrs={"epsilon": epsilon, "begin_norm_axis": begin_norm_axis}, + ) return helper.append_activation(layer_norm_out) -def instance_norm(x, - running_mean=None, - running_var=None, - weight=None, - bias=None, - use_input_stats=True, - momentum=0.9, - eps=1e-05, - data_format="NCHW", - name=None): +def instance_norm( + x, + running_mean=None, + running_var=None, + weight=None, + bias=None, + use_input_stats=True, + momentum=0.9, + eps=1e-05, + data_format="NCHW", + name=None, +): """ See more detail in nn.layer.InstanceNorm2D. @@ -410,9 +478,17 @@ def instance_norm(x, out = _C_ops.instance_norm(x, weight, bias, eps) return out if _in_legacy_dygraph(): - out, _, _ = _legacy_C_ops.instance_norm(x, weight, bias, "epsilon", eps, - "momentum", momentum, - "data_format", data_format) + out, _, _ = _legacy_C_ops.instance_norm( + x, + weight, + bias, + "epsilon", + eps, + "momentum", + momentum, + "data_format", + data_format, + ) return out check_variable_and_dtype(x, 'input', ['float32', 'float64'], "InstanceNorm") @@ -425,104 +501,106 @@ def instance_norm(x, inputs = {"X": [x]} helper = LayerHelper('instance_norm', **locals()) - saved_mean = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) + saved_mean = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) saved_variance = helper.create_variable_for_type_inference( - dtype=x.dtype, stop_gradient=True) + dtype=x.dtype, stop_gradient=True + ) instance_norm_out = helper.create_variable_for_type_inference(x.dtype) outputs = { "Y": [instance_norm_out], "SavedMean": [saved_mean], - "SavedVariance": [saved_variance] + "SavedVariance": [saved_variance], } - helper.append_op(type="instance_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type="instance_norm", inputs=inputs, outputs=outputs, attrs=attrs + ) return instance_norm_out -def local_response_norm(x, - size, - alpha=1e-4, - beta=0.75, - k=1., - data_format="NCHW", - name=None): +def local_response_norm( + x, size, alpha=1e-4, beta=0.75, k=1.0, data_format="NCHW", name=None +): r""" - Local Response Normalization performs a type of "lateral inhibition" by normalizing over local input regions. - For more information, please refer to `ImageNet Classification with Deep Convolutional Neural Networks `_ - - The formula is as follows: - - .. math:: - - Output(i, x, y) = Input(i, x, y) / \left(k + \alpha \sum\limits^{\min(C-1, i + size/2)}_{j = \max(0, i - size/2)}(Input(j, x, y))^2\right)^{\beta} - - In the above equation: + Local Response Normalization performs a type of "lateral inhibition" by normalizing over local input regions. + For more information, please refer to `ImageNet Classification with Deep Convolutional Neural Networks `_ - - :math:`size` : The number of channels to sum over. - - :math:`k` : The offset (avoid being divided by 0). - - :math:`\\alpha` : The scaling parameter. - - :math:`\\beta` : The exponent parameter. + The formula is as follows: + .. math:: - Args: - x (Tensor): The input 3-D/4-D/5-D tensor. The data type is float32. - size (int): The number of channels to sum over. - alpha (float, optional): The scaling parameter, positive. Default:1e-4 - beta (float, optional): The exponent, positive. Default:0.75 - k (float, optional): An offset, positive. Default: 1.0 - data_format (str, optional): Specify the data format of the input, and the data format of the output - will be consistent with that of the input. An optional string from: - If x is 3-D Tensor, the string could be `"NCL"` or `"NLC"` . When it is `"NCL"`, - the data is stored in the order of: `[batch_size, input_channels, feature_length]`. - If x is 4-D Tensor, the string could be `"NCHW"`, `"NHWC"`. When it is `"NCHW"`, - the data is stored in the order of: `[batch_size, input_channels, input_height, input_width]`. - If x is 5-D Tensor, the string could be `"NCDHW"`, `"NDHWC"` . When it is `"NCDHW"`, - the data is stored in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`. - name (str, optional): Name for the operation (optional, default is None). For more information, - please refer to :ref:`api_guide_Name`. + Output(i, x, y) = Input(i, x, y) / \left(k + \alpha \sum\limits^{\min(C-1, i + size/2)}_{j = \max(0, i - size/2)}(Input(j, x, y))^2\right)^{\beta} + + In the above equation: + + - :math:`size` : The number of channels to sum over. + - :math:`k` : The offset (avoid being divided by 0). + - :math:`\\alpha` : The scaling parameter. + - :math:`\\beta` : The exponent parameter. + + + Args: + x (Tensor): The input 3-D/4-D/5-D tensor. The data type is float32. + size (int): The number of channels to sum over. + alpha (float, optional): The scaling parameter, positive. Default:1e-4 + beta (float, optional): The exponent, positive. Default:0.75 + k (float, optional): An offset, positive. Default: 1.0 + data_format (str, optional): Specify the data format of the input, and the data format of the output + will be consistent with that of the input. An optional string from: + If x is 3-D Tensor, the string could be `"NCL"` or `"NLC"` . When it is `"NCL"`, + the data is stored in the order of: `[batch_size, input_channels, feature_length]`. + If x is 4-D Tensor, the string could be `"NCHW"`, `"NHWC"`. When it is `"NCHW"`, + the data is stored in the order of: `[batch_size, input_channels, input_height, input_width]`. + If x is 5-D Tensor, the string could be `"NCDHW"`, `"NDHWC"` . When it is `"NCDHW"`, + the data is stored in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`. + name (str, optional): Name for the operation (optional, default is None). For more information, + please refer to :ref:`api_guide_Name`. - Returns: - A tensor storing the transformation result with the same shape and data type as input. + Returns: + A tensor storing the transformation result with the same shape and data type as input. - Examples: + Examples: - .. code-block:: python + .. code-block:: python - import paddle + import paddle - x = paddle.rand(shape=(3, 3, 112, 112), dtype="float32") - y = paddle.nn.functional.local_response_norm(x, size=5) - print(y.shape) # [3, 3, 112, 112] - """ + x = paddle.rand(shape=(3, 3, 112, 112), dtype="float32") + y = paddle.nn.functional.local_response_norm(x, size=5) + print(y.shape) # [3, 3, 112, 112] + """ if not in_dynamic_mode(): check_variable_and_dtype(x, 'x', ['float32'], 'local_response_norm') if data_format not in ['NCL', 'NLC', 'NCHW', 'NHWC', 'NCDHW', 'NDHWC']: raise ValueError( - "data_format should be in one of [NCL, NCHW, NCDHW, NLC, NHWC, NDHWC], " \ - "but got {}".format(data_format)) + "data_format should be in one of [NCL, NCHW, NCDHW, NLC, NHWC, NDHWC], " + "but got {}".format(data_format) + ) sizes = x.shape dim = len(sizes) if dim < 3: raise ValueError( - 'Expected 3D or higher dimensionality input, but got {} dimensions'. - format(dim)) + 'Expected 3D or higher dimensionality input, but got {} dimensions'.format( + dim + ) + ) for i, sz in enumerate(sizes): if not sz > 0 and i > 0: - raise ValueError("Expected every dim's size to be larger than 0, " - "but the size of the {}-th dim is {}".format( - i, sz)) + raise ValueError( + "Expected every dim's size to be larger than 0, " + "but the size of the {}-th dim is {}".format(i, sz) + ) channel_last = True if data_format[-1] == "C" else False from functools import reduce + sum_sizes = reduce(lambda x, y: x * y, sizes[1:]) div = paddle.unsqueeze(paddle.multiply(x, x), axis=1) @@ -530,8 +608,11 @@ def local_response_norm(x, pad4d_shape = [0, 0, size // 2, (size - 1) // 2] pool2d_shape = (size, 1) reshape_shape = [ - sizes[0], 1, sizes[1], sizes[2], - int(sum_sizes / (sizes[1] * sizes[2])) + sizes[0], + 1, + sizes[1], + sizes[2], + int(sum_sizes / (sizes[1] * sizes[2])), ] pad5d_shape = [0, 0, 0, 0, size // 2, (size - 1) // 2] pool3d_shape = (size, 1, 1) @@ -539,26 +620,29 @@ def local_response_norm(x, pad4d_shape = [size // 2, (size - 1) // 2, 0, 0] pool2d_shape = (1, size) reshape_shape = [ - sizes[0], 1, sizes[1], - int(sum_sizes / (sizes[1] * sizes[-1])), sizes[-1] + sizes[0], + 1, + sizes[1], + int(sum_sizes / (sizes[1] * sizes[-1])), + sizes[-1], ] pad5d_shape = [size // 2, (size - 1) // 2, 0, 0, 0, 0] pool3d_shape = (1, 1, size) if dim == 3: div = paddle.nn.functional.pad(div, pad=pad4d_shape) - div = paddle.nn.functional.avg_pool2d(div, - kernel_size=pool2d_shape, - stride=1) + div = paddle.nn.functional.avg_pool2d( + div, kernel_size=pool2d_shape, stride=1 + ) div = paddle.squeeze(div, axis=1) else: div = paddle.reshape(div, shape=reshape_shape) - div = paddle.nn.functional.pad(div, - pad=pad5d_shape, - data_format='NCDHW') - div = paddle.nn.functional.avg_pool3d(div, - kernel_size=pool3d_shape, - stride=1) + div = paddle.nn.functional.pad( + div, pad=pad5d_shape, data_format='NCDHW' + ) + div = paddle.nn.functional.avg_pool3d( + div, kernel_size=pool3d_shape, stride=1 + ) div = paddle.reshape(paddle.squeeze(div, axis=1), sizes) div = paddle.scale(div, scale=alpha, bias=k) diff --git a/python/paddle/nn/functional/pooling.py b/python/paddle/nn/functional/pooling.py index 7e465b5974888..f9ece56dc7ef5 100755 --- a/python/paddle/nn/functional/pooling.py +++ b/python/paddle/nn/functional/pooling.py @@ -32,7 +32,9 @@ def _check_input(x, dimension): if len(x.shape) != dimension: raise ValueError( "Excepted Input X is {}-D tensor, but received {}-D {}".format( - dimension, len(x.shape), type(x))) + dimension, len(x.shape), type(x) + ) + ) def _check_instance(x, x_name, types=(int, float)): @@ -40,16 +42,19 @@ def _check_instance(x, x_name, types=(int, float)): if not isinstance(x, types): raise ValueError( "Excepted {} type for {} but received type: {}. ".format( - types, x_name, type(x))) + types, x_name, type(x) + ) + ) def _check_value_limitation(x, x_name, min_limit=1e-3): - def _check_value(x, x_name, min_limit=1e-3): if isinstance(x, int) and min_limit is not None and x < min_limit: raise ValueError( - "Excepted the input {} to be greater than {} but received x: {}. " - .format(x_name, min_limit, x)) + "Excepted the input {} to be greater than {} but received x: {}. ".format( + x_name, min_limit, x + ) + ) for ele in x: _check_value(ele, x_name) @@ -73,21 +78,24 @@ def _channel_last(data_format, num_dims): if data_format not in ['NCL', 'NLC']: raise ValueError( "Attr(data_format) should be 'NCL' or 'NLC'. Received " - "Attr(data_format): %s" % str(data_format)) + "Attr(data_format): %s" % str(data_format) + ) else: return True if data_format == "NLC" else False if num_dims == 2: if data_format not in ['NCHW', 'NHWC']: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'. Received " - "Attr(data_format): %s" % str(data_format)) + "Attr(data_format): %s" % str(data_format) + ) else: return True if data_format == "NHWC" else False if num_dims == 3: if data_format not in ['NCDHW', 'NDHWC']: raise ValueError( "Attr(data_format) should be 'NCDHW' or 'NDHWC'. Received " - "Attr(data_format): %s" % str(data_format)) + "Attr(data_format): %s" % str(data_format) + ) else: return True if data_format == "NDHWC" else False @@ -97,13 +105,16 @@ def _update_padding_nd(padding, num_dims, channel_last=False, ceil_mode=False): padding = padding.upper() if padding not in ["SAME", "VALID"]: raise ValueError( - "Unknown padding: '{}'. It can only be 'SAME' or 'VALID'.". - format(padding)) + "Unknown padding: '{}'. It can only be 'SAME' or 'VALID'.".format( + padding + ) + ) if padding == "VALID": if ceil_mode != False: raise ValueError( "When Attr(padding) is \"VALID\", Attr(ceil_mode) must be False. " - "Received ceil_mode: True.") + "Received ceil_mode: True." + ) padding_algorithm = "VALID" padding = [0] * num_dims @@ -118,10 +129,12 @@ def _update_padding_nd(padding, num_dims, channel_last=False, ceil_mode=False): if not _zero_padding_in_batch_and_channel(padding, channel_last): raise ValueError( "Non-zero padding({}) in the batch or channel dimensions " - "is not supported.".format(padding)) + "is not supported.".format(padding) + ) padding_algorithm = "EXPLICIT" padding = _exclude_padding_in_batch_and_channel( - padding, channel_last) + padding, channel_last + ) if utils._is_symmetric_padding(padding, num_dims): padding = padding[0::2] # for padding like [pad_before, pad_after, pad_before, pad_after, ...] @@ -144,25 +157,29 @@ def _update_padding_nd(padding, num_dims, channel_last=False, ceil_mode=False): def _expand_low_nd_padding(padding): - #1d to 2d fake input + # 1d to 2d fake input if len(padding) == 2: padding = [0] * 2 + padding elif len(padding) == 1: padding = [0] + padding else: raise ValueError( - "The size of padding's dimmention should be 1 or 2. But got padding={}" - .format(padding)) + "The size of padding's dimmention should be 1 or 2. But got padding={}".format( + padding + ) + ) return padding -def avg_pool1d(x, - kernel_size, - stride=None, - padding=0, - exclusive=True, - ceil_mode=False, - name=None): +def avg_pool1d( + x, + kernel_size, + stride=None, + padding=0, + exclusive=True, + ceil_mode=False, + name=None, +): """ This API implements average pooling 1d operation, See more details in :ref:`api_nn_pooling_AvgPool1d` . @@ -221,28 +238,56 @@ def avg_pool1d(x, _check_value_limitation(stride, "stride", min_limit=1e-3) channel_last = _channel_last("NCL", 1) - padding, padding_algorithm = _update_padding_nd(padding, - 1, - channel_last=channel_last, - ceil_mode=ceil_mode) + padding, padding_algorithm = _update_padding_nd( + padding, 1, channel_last=channel_last, ceil_mode=ceil_mode + ) # use 2d to implenment 1d should expand padding in advance. padding = _expand_low_nd_padding(padding) if in_dygraph_mode(): - output = _C_ops.pool2d(x, kernel_size, stride, padding, ceil_mode, - exclusive, data_format, 'avg', False, False, - padding_algorithm, True) + output = _C_ops.pool2d( + x, + kernel_size, + stride, + padding, + ceil_mode, + exclusive, + data_format, + 'avg', + False, + False, + padding_algorithm, + True, + ) return squeeze(output, [2]) if _in_legacy_dygraph(): - output = _legacy_C_ops.pool2d(x, 'pooling_type', 'avg', 'ksize', - kernel_size, 'global_pooling', False, - 'strides', stride, 'paddings', padding, - 'padding_algorithm', padding_algorithm, - 'use_cudnn', True, 'ceil_mode', ceil_mode, - 'use_mkldnn', False, 'exclusive', - exclusive, 'data_format', data_format) + output = _legacy_C_ops.pool2d( + x, + 'pooling_type', + 'avg', + 'ksize', + kernel_size, + 'global_pooling', + False, + 'strides', + stride, + 'paddings', + padding, + 'padding_algorithm', + padding_algorithm, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + exclusive, + 'data_format', + data_format, + ) return squeeze(output, [2]) op_type = 'pool2d' @@ -250,35 +295,39 @@ def avg_pool1d(x, dtype = helper.input_dtype(input_param_name='x') pool_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs={"X": x}, - outputs={"Out": pool_out}, - attrs={ - "pooling_type": 'avg', - "ksize": kernel_size, - "global_pooling": False, - "strides": stride, - "paddings": padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": True, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": exclusive, - "data_format": data_format, - }) + helper.append_op( + type=op_type, + inputs={"X": x}, + outputs={"Out": pool_out}, + attrs={ + "pooling_type": 'avg', + "ksize": kernel_size, + "global_pooling": False, + "strides": stride, + "paddings": padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": True, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": exclusive, + "data_format": data_format, + }, + ) return squeeze(pool_out, [2]) -def avg_pool2d(x, - kernel_size, - stride=None, - padding=0, - ceil_mode=False, - exclusive=True, - divisor_override=None, - data_format="NCHW", - name=None): +def avg_pool2d( + x, + kernel_size, + stride=None, + padding=0, + ceil_mode=False, + exclusive=True, + divisor_override=None, + data_format="NCHW", + name=None, +): """ This API implements average pooling 2d operation. See more details in :ref:`api_nn_pooling_AvgPool2d` . @@ -340,23 +389,52 @@ def avg_pool2d(x, _check_value_limitation(stride, "stride", min_limit=1e-3) channel_last = _channel_last(data_format, 2) - padding, padding_algorithm = _update_padding_nd(padding, - 2, - channel_last, - ceil_mode=ceil_mode) + padding, padding_algorithm = _update_padding_nd( + padding, 2, channel_last, ceil_mode=ceil_mode + ) if _non_static_mode(): if in_dygraph_mode(): - output = _C_ops.pool2d(x, kernel_size, stride, padding, ceil_mode, - exclusive, data_format, 'avg', False, False, - padding_algorithm, True) + output = _C_ops.pool2d( + x, + kernel_size, + stride, + padding, + ceil_mode, + exclusive, + data_format, + 'avg', + False, + False, + padding_algorithm, + True, + ) else: output = _legacy_C_ops.pool2d( - x, 'pooling_type', 'avg', 'ksize', kernel_size, - 'global_pooling', False, 'padding_algorithm', padding_algorithm, - 'strides', stride, 'paddings', padding, 'use_cudnn', True, - 'ceil_mode', ceil_mode, 'use_mkldnn', False, 'exclusive', - exclusive, 'data_format', data_format) + x, + 'pooling_type', + 'avg', + 'ksize', + kernel_size, + 'global_pooling', + False, + 'padding_algorithm', + padding_algorithm, + 'strides', + stride, + 'paddings', + padding, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + exclusive, + 'data_format', + data_format, + ) if divisor_override is None: return output else: @@ -369,22 +447,24 @@ def avg_pool2d(x, dtype = helper.input_dtype(input_param_name='x') pool_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs={"X": x}, - outputs={"Out": pool_out}, - attrs={ - "pooling_type": "avg", - "ksize": kernel_size, - "global_pooling": False, - "strides": stride, - "paddings": padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": True, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": exclusive, - "data_format": data_format, - }) + helper.append_op( + type=op_type, + inputs={"X": x}, + outputs={"Out": pool_out}, + attrs={ + "pooling_type": "avg", + "ksize": kernel_size, + "global_pooling": False, + "strides": stride, + "paddings": padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": True, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": exclusive, + "data_format": data_format, + }, + ) if divisor_override is None: return pool_out @@ -393,15 +473,17 @@ def avg_pool2d(x, return pool_out * (kernel_size[0] * kernel_size[1]) / divisor_override -def avg_pool3d(x, - kernel_size, - stride=None, - padding=0, - ceil_mode=False, - exclusive=True, - divisor_override=None, - data_format="NCDHW", - name=None): +def avg_pool3d( + x, + kernel_size, + stride=None, + padding=0, + ceil_mode=False, + exclusive=True, + divisor_override=None, + data_format="NCDHW", + name=None, +): """ This API implements average pooling 3d operation. See more details in :ref:`api_nn_pooling_AvgPool3d` . @@ -459,25 +541,54 @@ def avg_pool3d(x, stride = utils.convert_to_list(stride, 3, 'pool_stride') channel_last = _channel_last(data_format, 3) - padding, padding_algorithm = _update_padding_nd(padding, - 3, - channel_last=channel_last, - ceil_mode=ceil_mode) + padding, padding_algorithm = _update_padding_nd( + padding, 3, channel_last=channel_last, ceil_mode=ceil_mode + ) _check_value_limitation(kernel_size, "kernel_size", min_limit=1e-3) _check_value_limitation(stride, "stride", min_limit=1e-3) if in_dygraph_mode(): - pool_out = _C_ops.pool3d(x, kernel_size, stride, padding, ceil_mode, - exclusive, data_format, 'avg', False, False, - padding_algorithm, True) + pool_out = _C_ops.pool3d( + x, + kernel_size, + stride, + padding, + ceil_mode, + exclusive, + data_format, + 'avg', + False, + False, + padding_algorithm, + True, + ) elif _in_legacy_dygraph(): pool_out = _legacy_C_ops.pool3d( - x, 'pooling_type', 'avg', 'ksize', kernel_size, 'strides', stride, - 'paddings', padding, 'global_pooling', False, 'padding_algorithm', - padding_algorithm, 'use_cudnn', True, 'ceil_mode', ceil_mode, - 'use_mkldnn', False, 'exclusive', exclusive, 'data_format', - data_format) + x, + 'pooling_type', + 'avg', + 'ksize', + kernel_size, + 'strides', + stride, + 'paddings', + padding, + 'global_pooling', + False, + 'padding_algorithm', + padding_algorithm, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + exclusive, + 'data_format', + data_format, + ) else: op_type = "pool3d" helper = LayerHelper(op_type, **locals()) @@ -486,38 +597,45 @@ def avg_pool3d(x, pool_out = helper.create_variable_for_type_inference(dtype) outputs = {"Out": pool_out} - helper.append_op(type=op_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": 'avg', - "ksize": kernel_size, - "global_pooling": False, - "strides": stride, - "paddings": padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": True, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": exclusive, - "data_format": data_format, - }) + helper.append_op( + type=op_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": 'avg', + "ksize": kernel_size, + "global_pooling": False, + "strides": stride, + "paddings": padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": True, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": exclusive, + "data_format": data_format, + }, + ) if divisor_override is None: return pool_out else: _check_instance(divisor_override, "divisor_override") - return pool_out * (kernel_size[0] * kernel_size[1] * - kernel_size[2]) / divisor_override + return ( + pool_out + * (kernel_size[0] * kernel_size[1] * kernel_size[2]) + / divisor_override + ) -def max_pool1d(x, - kernel_size, - stride=None, - padding=0, - return_mask=False, - ceil_mode=False, - name=None): +def max_pool1d( + x, + kernel_size, + stride=None, + padding=0, + return_mask=False, + ceil_mode=False, + name=None, +): """ This API implements max pooling 1d opereation. See more details in :ref:`api_nn_pooling_MaxPool1d` . @@ -570,44 +688,96 @@ def max_pool1d(x, else: stride = [1] + utils.convert_to_list(stride, 1, 'pool_stride') - padding, padding_algorithm = _update_padding_nd(padding, - 1, - ceil_mode=ceil_mode) + padding, padding_algorithm = _update_padding_nd( + padding, 1, ceil_mode=ceil_mode + ) # use 2d to implenment 1d should expand padding in advance. padding = _expand_low_nd_padding(padding) if in_dygraph_mode(): if return_mask: - pool_out = _C_ops.max_pool2d_with_index(x, kernel_size, stride, - padding, False, False) - return (squeeze(pool_out[0], [2]), - squeeze(pool_out[1], [2])) if return_mask else squeeze( - pool_out[0], [2]) + pool_out = _C_ops.max_pool2d_with_index( + x, kernel_size, stride, padding, False, False + ) + return ( + (squeeze(pool_out[0], [2]), squeeze(pool_out[1], [2])) + if return_mask + else squeeze(pool_out[0], [2]) + ) else: - pool_out = _C_ops.pool2d(x, kernel_size, stride, padding, ceil_mode, - True, data_format, 'max', False, False, - padding_algorithm, True) + pool_out = _C_ops.pool2d( + x, + kernel_size, + stride, + padding, + ceil_mode, + True, + data_format, + 'max', + False, + False, + padding_algorithm, + True, + ) return squeeze(pool_out, [2]) if _in_legacy_dygraph(): if return_mask: pool_out = _legacy_C_ops.max_pool2d_with_index( - x, 'ksize', kernel_size, 'global_pooling', False, 'strides', - stride, 'paddings', padding, 'padding_algorithm', - padding_algorithm, 'use_cudnn', True, 'ceil_mode', ceil_mode, - 'use_mkldnn', False, 'exclusive', True, 'data_format', - data_format) - return (squeeze(pool_out[0], [2]), - squeeze(pool_out[1], [2])) if return_mask else squeeze( - pool_out[0], [2]) + x, + 'ksize', + kernel_size, + 'global_pooling', + False, + 'strides', + stride, + 'paddings', + padding, + 'padding_algorithm', + padding_algorithm, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + True, + 'data_format', + data_format, + ) + return ( + (squeeze(pool_out[0], [2]), squeeze(pool_out[1], [2])) + if return_mask + else squeeze(pool_out[0], [2]) + ) else: pool_out = _legacy_C_ops.pool2d( - x, 'pooling_type', 'max', 'ksize', kernel_size, - 'global_pooling', False, 'padding_algorithm', padding_algorithm, - 'strides', stride, 'paddings', padding, 'use_cudnn', True, - 'ceil_mode', ceil_mode, 'use_mkldnn', False, 'exclusive', True, - 'data_format', data_format) + x, + 'pooling_type', + 'max', + 'ksize', + kernel_size, + 'global_pooling', + False, + 'padding_algorithm', + padding_algorithm, + 'strides', + stride, + 'paddings', + padding, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + True, + 'data_format', + data_format, + ) return squeeze(pool_out, [2]) op_type = 'max_pool2d_with_index' if return_mask else "pool2d" @@ -617,36 +787,44 @@ def max_pool1d(x, mask = helper.create_variable_for_type_inference('int32') outputs = {"Out": pool_out, "Mask": mask} - helper.append_op(type=op_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": 'max', - "ksize": kernel_size, - "global_pooling": False, - "strides": stride, - "paddings": padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": True, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": True, - "data_format": data_format, - }) - - return (squeeze(pool_out, [2]), - squeeze(mask, [2])) if return_mask else squeeze(pool_out, [2]) + helper.append_op( + type=op_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": 'max', + "ksize": kernel_size, + "global_pooling": False, + "strides": stride, + "paddings": padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": True, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": True, + "data_format": data_format, + }, + ) + + return ( + (squeeze(pool_out, [2]), squeeze(mask, [2])) + if return_mask + else squeeze(pool_out, [2]) + ) def _unpool_output_size(x, kernel_size, stride, padding, output_size): - assert output_size is None or isinstance( - output_size, (list, tuple) - ), "Required output_size is None|list|tuple, but received %s" % output_size + assert output_size is None or isinstance(output_size, (list, tuple)), ( + "Required output_size is None|list|tuple, but received %s" % output_size + ) input_size = x.shape default_size = [] for d in range(len(kernel_size)): - default_size.append((input_size[-len(kernel_size) + d] - 1) * - stride[d] + kernel_size[d] - 2 * padding[d]) + default_size.append( + (input_size[-len(kernel_size) + d] - 1) * stride[d] + + kernel_size[d] + - 2 * padding[d] + ) has_static_var = False if output_size is None: @@ -666,28 +844,33 @@ def _unpool_output_size(x, kernel_size, stride, padding, output_size): raise ValueError( "output_size should be a sequence containing " "{} or {} elements, but it has a length of '{}'".format( - len(kernel_size), - len(kernel_size) + 2, len(output_size))) + len(kernel_size), len(kernel_size) + 2, len(output_size) + ) + ) if not has_static_var: for d in range(len(kernel_size)): min_size = default_size[d] - stride[d] max_size = default_size[d] + stride[d] if not (min_size < output_size[d] < max_size): raise ValueError( - 'invalid output_size "{}" (dim {} must be between {} and {})' - .format(output_size, d, min_size, max_size)) + 'invalid output_size "{}" (dim {} must be between {} and {})'.format( + output_size, d, min_size, max_size + ) + ) return output_size -def max_unpool1d(x, - indices, - kernel_size, - stride=None, - padding=0, - data_format="NCL", - output_size=None, - name=None): +def max_unpool1d( + x, + indices, + kernel_size, + stride=None, + padding=0, + data_format="NCL", + output_size=None, + name=None, +): r""" This API implements max unpooling 1d opereation. `max_unpool1d` accepts the output of `max_pool1d` as input, @@ -745,8 +928,10 @@ def max_unpool1d(x, """ """NCL to NCHW""" if data_format not in ["NCL"]: - raise ValueError("Attr(data_format) should be 'NCL'. Received " - "Attr(data_format): %s." % str(data_format)) + raise ValueError( + "Attr(data_format) should be 'NCL'. Received " + "Attr(data_format): %s." % str(data_format) + ) data_format = "NCHW" x = unsqueeze(x, [2]) indices = unsqueeze(indices, [2]) @@ -759,18 +944,32 @@ def max_unpool1d(x, # use 2d to implenment 1d should expand padding in advance. padding = _expand_low_nd_padding(padding) - output_size = _unpool_output_size(x, kernel_size, stride, padding, - output_size) + output_size = _unpool_output_size( + x, kernel_size, stride, padding, output_size + ) if in_dygraph_mode(): - output = _C_ops.unpool(x, indices, kernel_size, stride, padding, - output_size, data_format) + output = _C_ops.unpool( + x, indices, kernel_size, stride, padding, output_size, data_format + ) return squeeze(output, [2]) elif in_dynamic_mode(): - output = _legacy_C_ops.unpool(x, indices, 'unpooling_type', 'max', - 'ksize', kernel_size, 'strides', stride, - 'paddings', padding, "output_size", - output_size, "data_format", data_format) + output = _legacy_C_ops.unpool( + x, + indices, + 'unpooling_type', + 'max', + 'ksize', + kernel_size, + 'strides', + stride, + 'paddings', + padding, + "output_size", + output_size, + "data_format", + data_format, + ) return squeeze(output, [2]) op_type = "unpool" @@ -778,30 +977,31 @@ def max_unpool1d(x, dtype = helper.input_dtype(input_param_name="x") unpool_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs={ - "X": x, - "Indices": indices - }, - outputs={"Out": unpool_out}, - attrs={ - "unpooling_type": "max", - "ksize": kernel_size, - "strides": stride, - "paddings": padding, - "output_size": output_size - }) + helper.append_op( + type=op_type, + inputs={"X": x, "Indices": indices}, + outputs={"Out": unpool_out}, + attrs={ + "unpooling_type": "max", + "ksize": kernel_size, + "strides": stride, + "paddings": padding, + "output_size": output_size, + }, + ) return squeeze(unpool_out, [2]) -def max_unpool2d(x, - indices, - kernel_size, - stride=None, - padding=0, - data_format="NCHW", - output_size=None, - name=None): +def max_unpool2d( + x, + indices, + kernel_size, + stride=None, + padding=0, + data_format="NCHW", + output_size=None, + name=None, +): r""" This API implements max unpooling 2d opereation. See more details in :ref:`api_nn_pooling_MaxUnPool2D` . @@ -875,21 +1075,37 @@ def max_unpool2d(x, padding = utils.convert_to_list(padding, 2, 'padding') if data_format not in ["NCHW"]: - raise ValueError("Attr(data_format) should be 'NCHW'. Received " - "Attr(data_format): %s." % str(data_format)) + raise ValueError( + "Attr(data_format) should be 'NCHW'. Received " + "Attr(data_format): %s." % str(data_format) + ) - output_size = _unpool_output_size(x, kernel_size, stride, padding, - output_size) + output_size = _unpool_output_size( + x, kernel_size, stride, padding, output_size + ) if in_dygraph_mode(): - output = _C_ops.unpool(x, indices, kernel_size, stride, padding, - output_size, data_format) + output = _C_ops.unpool( + x, indices, kernel_size, stride, padding, output_size, data_format + ) return output elif in_dynamic_mode(): - output = _legacy_C_ops.unpool(x, indices, 'unpooling_type', 'max', - 'ksize', kernel_size, 'strides', stride, - 'paddings', padding, "output_size", - output_size, "data_format", data_format) + output = _legacy_C_ops.unpool( + x, + indices, + 'unpooling_type', + 'max', + 'ksize', + kernel_size, + 'strides', + stride, + 'paddings', + padding, + "output_size", + output_size, + "data_format", + data_format, + ) return output op_type = "unpool" @@ -897,30 +1113,31 @@ def max_unpool2d(x, dtype = helper.input_dtype(input_param_name="x") unpool_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs={ - "X": x, - "Indices": indices - }, - outputs={"Out": unpool_out}, - attrs={ - "unpooling_type": "max", - "ksize": kernel_size, - "strides": stride, - "paddings": padding, - "output_size": output_size - }) + helper.append_op( + type=op_type, + inputs={"X": x, "Indices": indices}, + outputs={"Out": unpool_out}, + attrs={ + "unpooling_type": "max", + "ksize": kernel_size, + "strides": stride, + "paddings": padding, + "output_size": output_size, + }, + ) return unpool_out -def max_unpool3d(x, - indices, - kernel_size, - stride=None, - padding=0, - data_format="NCDHW", - output_size=None, - name=None): +def max_unpool3d( + x, + indices, + kernel_size, + stride=None, + padding=0, + data_format="NCDHW", + output_size=None, + name=None, +): r""" This API implements max unpooling 3d opereation. `max_unpool3d` accepts the output of `max_pool3d` as input, @@ -992,21 +1209,37 @@ def max_unpool3d(x, padding = utils.convert_to_list(padding, 3, 'padding') if data_format not in ["NCDHW"]: - raise ValueError("Attr(data_format) should be 'NCDHW'. Received " - "Attr(data_format): %s." % str(data_format)) + raise ValueError( + "Attr(data_format) should be 'NCDHW'. Received " + "Attr(data_format): %s." % str(data_format) + ) - output_size = _unpool_output_size(x, kernel_size, stride, padding, - output_size) + output_size = _unpool_output_size( + x, kernel_size, stride, padding, output_size + ) if in_dygraph_mode(): - output = _C_ops.unpool3d(x, indices, kernel_size, stride, padding, - output_size, data_format) + output = _C_ops.unpool3d( + x, indices, kernel_size, stride, padding, output_size, data_format + ) return output elif in_dynamic_mode(): - output = _legacy_C_ops.unpool3d(x, indices, 'unpooling_type', 'max', - 'ksize', kernel_size, 'strides', stride, - 'paddings', padding, "output_size", - output_size, "data_format", data_format) + output = _legacy_C_ops.unpool3d( + x, + indices, + 'unpooling_type', + 'max', + 'ksize', + kernel_size, + 'strides', + stride, + 'paddings', + padding, + "output_size", + output_size, + "data_format", + data_format, + ) return output op_type = "unpool3d" @@ -1014,30 +1247,31 @@ def max_unpool3d(x, dtype = helper.input_dtype(input_param_name="x") unpool_out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs={ - "X": x, - "Indices": indices - }, - outputs={"Out": unpool_out}, - attrs={ - "unpooling_type": "max", - "ksize": kernel_size, - "strides": stride, - "paddings": padding, - "output_size": output_size - }) + helper.append_op( + type=op_type, + inputs={"X": x, "Indices": indices}, + outputs={"Out": unpool_out}, + attrs={ + "unpooling_type": "max", + "ksize": kernel_size, + "strides": stride, + "paddings": padding, + "output_size": output_size, + }, + ) return unpool_out -def max_pool2d(x, - kernel_size, - stride=None, - padding=0, - return_mask=False, - ceil_mode=False, - data_format="NCHW", - name=None): +def max_pool2d( + x, + kernel_size, + stride=None, + padding=0, + return_mask=False, + ceil_mode=False, + data_format="NCHW", + name=None, +): """ This API implements max pooling 2d operation. See more details in :ref:`api_nn_pooling_MaxPool2d` . @@ -1096,14 +1330,14 @@ def max_pool2d(x, if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) channel_last = True if data_format == "NHWC" else False - padding, padding_algorithm = _update_padding_nd(padding, - num_dims=2, - channel_last=channel_last, - ceil_mode=ceil_mode) + padding, padding_algorithm = _update_padding_nd( + padding, num_dims=2, channel_last=channel_last, ceil_mode=ceil_mode + ) if data_format == "NHWC" and return_mask: raise ValueError( @@ -1112,69 +1346,122 @@ def max_pool2d(x, if in_dygraph_mode(): if return_mask: - output = _C_ops.max_pool2d_with_index(x, kernel_size, stride, - padding, False, False) + output = _C_ops.max_pool2d_with_index( + x, kernel_size, stride, padding, False, False + ) return output if return_mask else output[0] else: - return _C_ops.pool2d(x, kernel_size, stride, padding, ceil_mode, - True, data_format, 'max', False, False, - padding_algorithm, True) + return _C_ops.pool2d( + x, + kernel_size, + stride, + padding, + ceil_mode, + True, + data_format, + 'max', + False, + False, + padding_algorithm, + True, + ) if _in_legacy_dygraph(): if return_mask: output = _legacy_C_ops.max_pool2d_with_index( - x, 'ksize', kernel_size, 'global_pooling', False, 'strides', - stride, 'paddings', padding, 'padding_algorithm', - padding_algorithm, 'use_cudnn', True, 'ceil_mode', ceil_mode, - 'use_mkldnn', False, 'exclusive', True, 'data_format', - data_format) + x, + 'ksize', + kernel_size, + 'global_pooling', + False, + 'strides', + stride, + 'paddings', + padding, + 'padding_algorithm', + padding_algorithm, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + True, + 'data_format', + data_format, + ) return output if return_mask else output[0] else: output = _legacy_C_ops.pool2d( - x, 'pooling_type', 'max', 'ksize', kernel_size, - 'global_pooling', False, 'padding_algorithm', padding_algorithm, - 'strides', stride, 'paddings', padding, 'use_cudnn', True, - 'ceil_mode', ceil_mode, 'use_mkldnn', False, 'exclusive', True, - 'data_format', data_format) + x, + 'pooling_type', + 'max', + 'ksize', + kernel_size, + 'global_pooling', + False, + 'padding_algorithm', + padding_algorithm, + 'strides', + stride, + 'paddings', + padding, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + True, + 'data_format', + data_format, + ) return output op_type = 'max_pool2d_with_index' if return_mask else "pool2d" helper = LayerHelper(op_type, **locals()) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'max_pool2d') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'max_pool2d' + ) dtype = helper.input_dtype(input_param_name='x') pool_out = helper.create_variable_for_type_inference(dtype) mask = helper.create_variable_for_type_inference("int32") outputs = {"Out": pool_out, "Mask": mask} - helper.append_op(type=op_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": 'max', - "ksize": kernel_size, - "global_pooling": False, - "strides": stride, - "paddings": padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": True, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": True, - "data_format": data_format, - }) + helper.append_op( + type=op_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": 'max', + "ksize": kernel_size, + "global_pooling": False, + "strides": stride, + "paddings": padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": True, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": True, + "data_format": data_format, + }, + ) return (pool_out, mask) if return_mask else pool_out -def max_pool3d(x, - kernel_size, - stride=None, - padding=0, - return_mask=False, - ceil_mode=False, - data_format="NCDHW", - name=None): +def max_pool3d( + x, + kernel_size, + stride=None, + padding=0, + return_mask=False, + ceil_mode=False, + data_format="NCDHW", + name=None, +): """ This API implements max pooling 2d operation. See more details in :ref:`api_nn_pooling_MaxPool3d` . @@ -1239,10 +1526,9 @@ def max_pool3d(x, channel_last = _channel_last(data_format, 3) - padding, padding_algorithm = _update_padding_nd(padding, - 3, - channel_last=channel_last, - ceil_mode=ceil_mode) + padding, padding_algorithm = _update_padding_nd( + padding, 3, channel_last=channel_last, ceil_mode=ceil_mode + ) if data_format == "NDHWC" and return_mask: raise ValueError( @@ -1251,30 +1537,80 @@ def max_pool3d(x, if in_dygraph_mode(): if return_mask: - output = _C_ops.max_pool3d_with_index(x, kernel_size, stride, - padding, False, False) + output = _C_ops.max_pool3d_with_index( + x, kernel_size, stride, padding, False, False + ) return output if return_mask else output[0] else: - return _C_ops.pool3d(x, kernel_size, stride, padding, ceil_mode, - True, data_format, 'max', False, False, - padding_algorithm, True) + return _C_ops.pool3d( + x, + kernel_size, + stride, + padding, + ceil_mode, + True, + data_format, + 'max', + False, + False, + padding_algorithm, + True, + ) if _in_legacy_dygraph(): if return_mask: output = _legacy_C_ops.max_pool3d_with_index( - x, 'pooling_type', 'max', 'ksize', kernel_size, 'strides', - stride, 'paddings', padding, 'global_pooling', False, - 'padding_algorithm', padding_algorithm, 'use_cudnn', True, - 'ceil_mode', ceil_mode, 'use_mkldnn', False, 'exclusive', True, - 'data_format', data_format) + x, + 'pooling_type', + 'max', + 'ksize', + kernel_size, + 'strides', + stride, + 'paddings', + padding, + 'global_pooling', + False, + 'padding_algorithm', + padding_algorithm, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + True, + 'data_format', + data_format, + ) return output if return_mask else output[0] else: output = _legacy_C_ops.pool3d( - x, 'pooling_type', 'max', 'ksize', kernel_size, - 'global_pooling', False, 'padding_algorithm', padding_algorithm, - 'strides', stride, 'paddings', padding, 'use_cudnn', True, - 'ceil_mode', ceil_mode, 'use_mkldnn', False, 'exclusive', True, - 'data_format', data_format) + x, + 'pooling_type', + 'max', + 'ksize', + kernel_size, + 'global_pooling', + False, + 'padding_algorithm', + padding_algorithm, + 'strides', + stride, + 'paddings', + padding, + 'use_cudnn', + True, + 'ceil_mode', + ceil_mode, + 'use_mkldnn', + False, + 'exclusive', + True, + 'data_format', + data_format, + ) return output op_type = "max_pool3d_with_index" if return_mask else "pool3d" @@ -1285,22 +1621,24 @@ def max_pool3d(x, mask = helper.create_variable_for_type_inference('int32') outputs = {"Out": pool_out, "Mask": mask} - helper.append_op(type=op_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": 'max', - "ksize": kernel_size, - "global_pooling": False, - "strides": stride, - "paddings": padding, - "padding_algorithm": padding_algorithm, - "use_cudnn": True, - "ceil_mode": ceil_mode, - "use_mkldnn": False, - "exclusive": False, - "data_format": data_format, - }) + helper.append_op( + type=op_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": 'max', + "ksize": kernel_size, + "global_pooling": False, + "strides": stride, + "paddings": padding, + "padding_algorithm": padding_algorithm, + "use_cudnn": True, + "ceil_mode": ceil_mode, + "use_mkldnn": False, + "exclusive": False, + "data_format": data_format, + }, + ) return (pool_out, mask) if return_mask else pool_out @@ -1344,21 +1682,34 @@ def adaptive_avg_pool1d(x, output_size, name=None): """ pool_type = 'avg' if not in_dynamic_mode(): - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'adaptive_pool2d') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'adaptive_pool2d' + ) check_type(output_size, 'pool_size', (int), 'adaptive_pool1d') _check_input(x, 3) pool_size = [1] + utils.convert_to_list(output_size, 1, 'pool_size') x = unsqueeze(x, [2]) if in_dygraph_mode(): - pool_out = _C_ops.pool2d(x, pool_size, [1, 1], [0, 0], False, True, - "NCHW", pool_type, False, True, "EXPLICIT", - False) + pool_out = _C_ops.pool2d( + x, + pool_size, + [1, 1], + [0, 0], + False, + True, + "NCHW", + pool_type, + False, + True, + "EXPLICIT", + False, + ) return squeeze(pool_out, [2]) if _in_legacy_dygraph(): - pool_out = _legacy_C_ops.pool2d(x, 'pooling_type', pool_type, 'ksize', - pool_size, 'adaptive', True) + pool_out = _legacy_C_ops.pool2d( + x, 'pooling_type', pool_type, 'ksize', pool_size, 'adaptive', True + ) return squeeze(pool_out, [2]) l_type = "pool2d" @@ -1368,14 +1719,16 @@ def adaptive_avg_pool1d(x, output_size, name=None): pool_out = helper.create_variable_for_type_inference(dtype) outputs = {"Out": pool_out} - helper.append_op(type=l_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": pool_type, - "ksize": pool_size, - "adaptive": True, - }) + helper.append_op( + type=l_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": pool_type, + "ksize": pool_size, + "adaptive": True, + }, + ) return squeeze(pool_out, [2]) @@ -1438,14 +1791,16 @@ def adaptive_avg_pool2d(x, output_size, data_format='NCHW', name=None): # out.shape is [2, 3, 3, 3] """ if not in_dynamic_mode(): - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'adaptive_avg_pool2d') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'adaptive_avg_pool2d' + ) check_type(data_format, 'data_format', str, 'adaptive_avg_pool2d') if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) if data_format == "NCHW": in_h, in_w = x.shape[2:4] @@ -1471,14 +1826,35 @@ def adaptive_avg_pool2d(x, output_size, data_format='NCHW', name=None): output_size = utils._convert_to_tensor_list(output_size) if in_dygraph_mode(): - return _C_ops.pool2d(x, output_size, [1, 1], [0, 0], False, True, - data_format, 'avg', False, True, "EXPLICIT", False) + return _C_ops.pool2d( + x, + output_size, + [1, 1], + [0, 0], + False, + True, + data_format, + 'avg', + False, + True, + "EXPLICIT", + False, + ) if _in_legacy_dygraph(): - return _legacy_C_ops.pool2d(x, 'pooling_type', 'avg', 'ksize', - output_size, 'global_pooling', False, - 'adaptive', True, 'data_format', - data_format) + return _legacy_C_ops.pool2d( + x, + 'pooling_type', + 'avg', + 'ksize', + output_size, + 'global_pooling', + False, + 'adaptive', + True, + 'data_format', + data_format, + ) l_type = 'pool2d' @@ -1488,15 +1864,17 @@ def adaptive_avg_pool2d(x, output_size, data_format='NCHW', name=None): outputs = {"Out": pool_out} - helper.append_op(type=l_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": "avg", - "ksize": output_size, - "adaptive": True, - "data_format": data_format, - }) + helper.append_op( + type=l_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": "avg", + "ksize": output_size, + "adaptive": True, + "data_format": data_format, + }, + ) return pool_out @@ -1562,14 +1940,16 @@ def adaptive_avg_pool3d(x, output_size, data_format='NCDHW', name=None): # out.shape is [2, 3, 3, 3, 3] """ if not in_dynamic_mode(): - check_variable_and_dtype(x, 'x', ['float32', 'float64'], - 'adaptive_avg_pool3d') + check_variable_and_dtype( + x, 'x', ['float32', 'float64'], 'adaptive_avg_pool3d' + ) check_type(data_format, 'data_format', str, 'adaptive_avg_pool3d') if data_format not in ["NCDHW", "NDHWC"]: raise ValueError( "Attr(data_format) should be 'NCDHW' or 'NDHWC'. Received " - "Attr(data_format): %s." % str(data_format)) + "Attr(data_format): %s." % str(data_format) + ) if data_format == "NCDHW": in_l, in_h, in_w = x.shape[2:5] @@ -1588,13 +1968,34 @@ def adaptive_avg_pool3d(x, output_size, data_format='NCDHW', name=None): output_size[2] = in_w if in_dygraph_mode(): - return _C_ops.pool3d(x, output_size, [1, 1, 1], [0, 0, 0], False, True, - data_format, 'avg', False, True, "EXPLICIT", False) + return _C_ops.pool3d( + x, + output_size, + [1, 1, 1], + [0, 0, 0], + False, + True, + data_format, + 'avg', + False, + True, + "EXPLICIT", + False, + ) elif _in_legacy_dygraph(): - return _legacy_C_ops.pool3d(x, 'pooling_type', 'avg', 'ksize', - output_size, 'global_pooling', False, - 'adaptive', True, 'data_format', - data_format) + return _legacy_C_ops.pool3d( + x, + 'pooling_type', + 'avg', + 'ksize', + output_size, + 'global_pooling', + False, + 'adaptive', + True, + 'data_format', + data_format, + ) l_type = 'pool3d' @@ -1603,15 +2004,17 @@ def adaptive_avg_pool3d(x, output_size, data_format='NCDHW', name=None): pool_out = helper.create_variable_for_type_inference(dtype) outputs = {"Out": pool_out} - helper.append_op(type=l_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": "avg", - "ksize": output_size, - "adaptive": True, - "data_format": data_format, - }) + helper.append_op( + type=l_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": "avg", + "ksize": output_size, + "adaptive": True, + "data_format": data_format, + }, + ) return pool_out @@ -1662,8 +2065,9 @@ def adaptive_max_pool1d(x, output_size, return_mask=False, name=None): """ pool_type = 'max' if not in_dynamic_mode(): - check_variable_and_dtype(x, 'x', ['float32', 'float64'], - 'adaptive_max_pool1d') + check_variable_and_dtype( + x, 'x', ['float32', 'float64'], 'adaptive_max_pool1d' + ) check_type(output_size, 'pool_size', int, 'adaptive_max_pool1d') check_type(return_mask, 'return_mask', bool, 'adaptive_max_pool1d') _check_input(x, 3) @@ -1672,17 +2076,23 @@ def adaptive_max_pool1d(x, output_size, return_mask=False, name=None): x = unsqueeze(x, [2]) if in_dygraph_mode(): - pool_out = _C_ops.max_pool2d_with_index(x, pool_size, [1, 1], [0, 0], - False, True) - return (squeeze(pool_out[0], [2]), squeeze( - pool_out[1], [2])) if return_mask else squeeze(pool_out[0], [2]) + pool_out = _C_ops.max_pool2d_with_index( + x, pool_size, [1, 1], [0, 0], False, True + ) + return ( + (squeeze(pool_out[0], [2]), squeeze(pool_out[1], [2])) + if return_mask + else squeeze(pool_out[0], [2]) + ) if _in_legacy_dygraph(): - pool_out = _legacy_C_ops.max_pool2d_with_index(x, 'pooling_type', - pool_type, 'ksize', - pool_size, 'adaptive', - True) - return (squeeze(pool_out[0], [2]), squeeze( - pool_out[1], [2])) if return_mask else squeeze(pool_out[0], [2]) + pool_out = _legacy_C_ops.max_pool2d_with_index( + x, 'pooling_type', pool_type, 'ksize', pool_size, 'adaptive', True + ) + return ( + (squeeze(pool_out[0], [2]), squeeze(pool_out[1], [2])) + if return_mask + else squeeze(pool_out[0], [2]) + ) l_type = 'max_pool2d_with_index' @@ -1693,64 +2103,70 @@ def adaptive_max_pool1d(x, output_size, return_mask=False, name=None): mask = helper.create_variable_for_type_inference('int32') outputs = {"Out": pool_out, "Mask": mask} - helper.append_op(type=l_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": pool_type, - "ksize": pool_size, - "adaptive": True, - }) + helper.append_op( + type=l_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": pool_type, + "ksize": pool_size, + "adaptive": True, + }, + ) - return (squeeze(pool_out, [2]), - squeeze(mask, [2])) if return_mask else squeeze(pool_out, [2]) + return ( + (squeeze(pool_out, [2]), squeeze(mask, [2])) + if return_mask + else squeeze(pool_out, [2]) + ) def adaptive_max_pool2d(x, output_size, return_mask=False, name=None): """ - This operation applies a 2D adaptive max pooling on input tensor. - See more details in :ref:`api_nn_pooling_AdaptiveMaxPool2d` . + This operation applies a 2D adaptive max pooling on input tensor. + See more details in :ref:`api_nn_pooling_AdaptiveMaxPool2d` . - Args: - x (Tensor): The input tensor of adaptive max pool2d operator, which is a 4-D tensor. The data type can be float16, float32, float64, int32 or int64. - output_size (int|list|tuple): The pool kernel size. If pool kernel size is a tuple or list, it must contain two elements, (H, W). H and W can be either a int, or None which means the size will be the same as that of the input. - return_mask (bool): If true, the index of max pooling point will be returned along with outputs. Default False. - name(str, optional): For detailed information, please refer to :ref:`api_guide_Name`. Usually name is no need to set and None by default. + Args: + x (Tensor): The input tensor of adaptive max pool2d operator, which is a 4-D tensor. The data type can be float16, float32, float64, int32 or int64. + output_size (int|list|tuple): The pool kernel size. If pool kernel size is a tuple or list, it must contain two elements, (H, W). H and W can be either a int, or None which means the size will be the same as that of the input. + return_mask (bool): If true, the index of max pooling point will be returned along with outputs. Default False. + name(str, optional): For detailed information, please refer to :ref:`api_guide_Name`. Usually name is no need to set and None by default. - Returns: - Tensor: The output tensor of adaptive max pool2d result. The data type is same as input tensor. + Returns: + Tensor: The output tensor of adaptive max pool2d result. The data type is same as input tensor. - Examples: - .. code-block:: python + Examples: + .. code-block:: python - # max adaptive pool2d - # suppose input data in the shape of [N, C, H, W], `output_size` is [m, n] - # output shape is [N, C, m, n], adaptive pool divide H and W dimensions - # of input data into m*n grids averagely and performs poolings in each - # grid to get output. - # adaptive max pool performs calculations as follow: - # - # for i in range(m): - # for j in range(n): - # hstart = floor(i * H / m) - # hend = ceil((i + 1) * H / m) - # wstart = floor(i * W / n) - # wend = ceil((i + 1) * W / n) - # output[:, :, i, j] = max(input[:, :, hstart: hend, wstart: wend]) - # - import paddle + # max adaptive pool2d + # suppose input data in the shape of [N, C, H, W], `output_size` is [m, n] + # output shape is [N, C, m, n], adaptive pool divide H and W dimensions + # of input data into m*n grids averagely and performs poolings in each + # grid to get output. + # adaptive max pool performs calculations as follow: + # + # for i in range(m): + # for j in range(n): + # hstart = floor(i * H / m) + # hend = ceil((i + 1) * H / m) + # wstart = floor(i * W / n) + # wend = ceil((i + 1) * W / n) + # output[:, :, i, j] = max(input[:, :, hstart: hend, wstart: wend]) + # + import paddle - input_data = paddle.randn(shape=(2, 3, 32, 32)) - out = paddle.nn.functional.adaptive_max_pool2d( - x = input_data, - output_size=[3, 3]) - # out.shape is [2, 3, 3, 3] + input_data = paddle.randn(shape=(2, 3, 32, 32)) + out = paddle.nn.functional.adaptive_max_pool2d( + x = input_data, + output_size=[3, 3]) + # out.shape is [2, 3, 3, 3] """ if not in_dynamic_mode(): - check_variable_and_dtype(x, 'x', ['float32', 'float64'], - 'adaptive_max_pool2d') + check_variable_and_dtype( + x, 'x', ['float32', 'float64'], 'adaptive_max_pool2d' + ) check_type(return_mask, 'return_mask', bool, 'adaptive_max_pool2d') - #check_type(output_size, 'pool_size', (int), 'adaptive_max_pool2d') + # check_type(output_size, 'pool_size', (int), 'adaptive_max_pool2d') _check_input(x, 4) in_h, in_w = x.shape[2:4] @@ -1763,13 +2179,14 @@ def adaptive_max_pool2d(x, output_size, return_mask=False, name=None): if output_size[1] == None: output_size[1] = in_w if in_dygraph_mode(): - pool_out = _C_ops.max_pool2d_with_index(x, output_size, [1, 1], [0, 0], - False, True) + pool_out = _C_ops.max_pool2d_with_index( + x, output_size, [1, 1], [0, 0], False, True + ) return pool_out if return_mask else pool_out[0] if _in_legacy_dygraph(): - pool_out = _legacy_C_ops.max_pool2d_with_index(x, 'pooling_type', 'max', - 'ksize', output_size, - 'adaptive', True) + pool_out = _legacy_C_ops.max_pool2d_with_index( + x, 'pooling_type', 'max', 'ksize', output_size, 'adaptive', True + ) return pool_out if return_mask else pool_out[0] l_type = 'max_pool2d_with_index' @@ -1781,67 +2198,70 @@ def adaptive_max_pool2d(x, output_size, return_mask=False, name=None): mask = helper.create_variable_for_type_inference('int32') outputs = {"Out": pool_out, "Mask": mask} - helper.append_op(type=l_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": 'max', - "ksize": output_size, - "adaptive": True, - }) - #return (pool_out, mask) if return_mask else pool_out + helper.append_op( + type=l_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": 'max', + "ksize": output_size, + "adaptive": True, + }, + ) + # return (pool_out, mask) if return_mask else pool_out return pool_out def adaptive_max_pool3d(x, output_size, return_mask=False, name=None): """ - This operation applies a 3D adaptive max pooling on input tensor. - See more details in :ref:`api_nn_pooling_AdaptiveMaxPool3d` . + This operation applies a 3D adaptive max pooling on input tensor. + See more details in :ref:`api_nn_pooling_AdaptiveMaxPool3d` . - Args: - x (Tensor): The input tensor of adaptive max pool3d operator, which is a 5-D tensor. The data type can be float32, float64. - output_size (int|list|tuple): The pool kernel size. If pool kernel size is a tuple or list, it must contain three elements, (D, H, W). D, H and W can be either a int, or None which means the size will be the same as that of the input. - return_mask (bool): If true, the index of max pooling point will be returned along with outputs. Default False. - name(str, optional): For detailed information, please refer to :ref:`api_guide_Name`. Usually name is no need to set and None by default. + Args: + x (Tensor): The input tensor of adaptive max pool3d operator, which is a 5-D tensor. The data type can be float32, float64. + output_size (int|list|tuple): The pool kernel size. If pool kernel size is a tuple or list, it must contain three elements, (D, H, W). D, H and W can be either a int, or None which means the size will be the same as that of the input. + return_mask (bool): If true, the index of max pooling point will be returned along with outputs. Default False. + name(str, optional): For detailed information, please refer to :ref:`api_guide_Name`. Usually name is no need to set and None by default. - Returns: - Tensor: The output tensor of adaptive max pool3d result. The data type is same as input tensor. + Returns: + Tensor: The output tensor of adaptive max pool3d result. The data type is same as input tensor. - Examples: - .. code-block:: python + Examples: + .. code-block:: python - # adaptive max pool3d - # suppose input data in the shape of [N, C, D, H, W], `output_size` is [l, m, n] - # output shape is [N, C, l, m, n], adaptive pool divide D, H and W dimensions - # of input data into m*n grids averagely and performs poolings in each - # grid to get output. - # adaptive max pool performs calculations as follow: - # - # for i in range(l): - # for j in range(m): - # for k in range(n): - # dstart = floor(i * D / l) - # dend = ceil((i + 1) * D / l) - # hstart = floor(i * H / m) - # hend = ceil((i + 1) * H / m) - # wstart = floor(i * W / n) - # wend = ceil((i + 1) * W / n) - # output[:, :, i, j, k] = max(input[:, :, dstart: dend, hstart: hend, wstart: wend]) - # - import paddle + # adaptive max pool3d + # suppose input data in the shape of [N, C, D, H, W], `output_size` is [l, m, n] + # output shape is [N, C, l, m, n], adaptive pool divide D, H and W dimensions + # of input data into m*n grids averagely and performs poolings in each + # grid to get output. + # adaptive max pool performs calculations as follow: + # + # for i in range(l): + # for j in range(m): + # for k in range(n): + # dstart = floor(i * D / l) + # dend = ceil((i + 1) * D / l) + # hstart = floor(i * H / m) + # hend = ceil((i + 1) * H / m) + # wstart = floor(i * W / n) + # wend = ceil((i + 1) * W / n) + # output[:, :, i, j, k] = max(input[:, :, dstart: dend, hstart: hend, wstart: wend]) + # + import paddle - input_data = paddle.randn(shape=(2, 3, 8, 32, 32)) - out = paddle.nn.functional.adaptive_max_pool3d( - x = input_data, - output_size=[3, 3, 3]) - # out.shape is [2, 3, 3, 3, 3] + input_data = paddle.randn(shape=(2, 3, 8, 32, 32)) + out = paddle.nn.functional.adaptive_max_pool3d( + x = input_data, + output_size=[3, 3, 3]) + # out.shape is [2, 3, 3, 3, 3] """ if not in_dynamic_mode(): - check_variable_and_dtype(x, 'x', ['float32', 'float64'], - 'adaptive_max_pool3d') + check_variable_and_dtype( + x, 'x', ['float32', 'float64'], 'adaptive_max_pool3d' + ) check_type(return_mask, 'return_mask', bool, 'adaptive_max_pool3d') - #check_type(output_size, 'pool_size', (int), 'adaptive_max_pool3d') + # check_type(output_size, 'pool_size', (int), 'adaptive_max_pool3d') _check_input(x, 5) in_l, in_h, in_w = x.shape[2:5] @@ -1859,12 +2279,13 @@ def adaptive_max_pool3d(x, output_size, return_mask=False, name=None): if in_dynamic_mode(): if in_dygraph_mode(): # By default, strides is [1,1,1] and paddings is [0, 0, 0] - pool_out = _C_ops.max_pool3d_with_index(x, output_size, [1, 1, 1], - [0, 0, 0], False, True) + pool_out = _C_ops.max_pool3d_with_index( + x, output_size, [1, 1, 1], [0, 0, 0], False, True + ) elif _in_legacy_dygraph(): pool_out = _legacy_C_ops.max_pool3d_with_index( - x, 'pooling_type', 'max', 'ksize', output_size, 'adaptive', - True) + x, 'pooling_type', 'max', 'ksize', output_size, 'adaptive', True + ) return pool_out if return_mask else pool_out[0] l_type = 'max_pool3d_with_index' @@ -1876,13 +2297,15 @@ def adaptive_max_pool3d(x, output_size, return_mask=False, name=None): mask = helper.create_variable_for_type_inference('int32') outputs = {"Out": pool_out, "Mask": mask} - helper.append_op(type=l_type, - inputs={"X": x}, - outputs=outputs, - attrs={ - "pooling_type": 'max', - "ksize": output_size, - "adaptive": True, - }) + helper.append_op( + type=l_type, + inputs={"X": x}, + outputs=outputs, + attrs={ + "pooling_type": 'max', + "ksize": output_size, + "adaptive": True, + }, + ) return (pool_out, mask) if return_mask else pool_out diff --git a/python/paddle/nn/functional/sparse_attention.py b/python/paddle/nn/functional/sparse_attention.py index 87df72fe0ef2f..0dfb545aae85b 100644 --- a/python/paddle/nn/functional/sparse_attention.py +++ b/python/paddle/nn/functional/sparse_attention.py @@ -17,14 +17,16 @@ from paddle import in_dynamic_mode -def sparse_attention(query, - key, - value, - sparse_csr_offset, - sparse_csr_columns, - key_padding_mask=None, - attn_mask=None, - name=None): +def sparse_attention( + query, + key, + value, + sparse_csr_offset, + sparse_csr_columns, + key_padding_mask=None, + attn_mask=None, + name=None, +): r""" This operator sparsify the Attention matrix in Transformer module to achieve the effect of reducing memory consumption and computation. @@ -141,9 +143,19 @@ def sparse_attention(query, # [1.99830270, 2.99830270]]]] """ if in_dynamic_mode(): - result_attention, result_sdd, result_softmax = _legacy_C_ops.sparse_attention( - query, key, value, sparse_csr_offset, sparse_csr_columns, - key_padding_mask, attn_mask) + ( + result_attention, + result_sdd, + result_softmax, + ) = _legacy_C_ops.sparse_attention( + query, + key, + value, + sparse_csr_offset, + sparse_csr_columns, + key_padding_mask, + attn_mask, + ) return result_attention helper = LayerHelper('sparse_attention', **locals()) @@ -163,7 +175,7 @@ def sparse_attention(query, outputs = { 'Out': out, 'SparseDotSdd': result_sdd, - 'Softmax': result_softmax + 'Softmax': result_softmax, } helper.append_op(type='sparse_attention', inputs=inputs, outputs=outputs) return out diff --git a/python/paddle/nn/functional/vision.py b/python/paddle/nn/functional/vision.py index af1656d641740..bc77dcefa45f9 100644 --- a/python/paddle/nn/functional/vision.py +++ b/python/paddle/nn/functional/vision.py @@ -82,45 +82,65 @@ def affine_grid(theta, out_shape, align_corners=True, name=None): if theta.shape[1] == 3: use_cudnn = False if is_compiled_with_rocm(): - use_cudnn = False # ROCM platform do not have MIOPEN kernel for affine_grid + use_cudnn = ( + False # ROCM platform do not have MIOPEN kernel for affine_grid + ) if in_dygraph_mode(): - _out_shape = out_shape.numpy().tolist() if isinstance( - out_shape, Variable) else out_shape + _out_shape = ( + out_shape.numpy().tolist() + if isinstance(out_shape, Variable) + else out_shape + ) return _C_ops.affine_grid(theta, _out_shape, use_cudnn, align_corners) elif in_dynamic_mode(): - _out_shape = out_shape.numpy().tolist() if isinstance( - out_shape, Variable) else out_shape - return _legacy_C_ops.affine_grid(theta, "output_shape", _out_shape, - "align_corners", align_corners, - "use_cudnn", use_cudnn) + _out_shape = ( + out_shape.numpy().tolist() + if isinstance(out_shape, Variable) + else out_shape + ) + return _legacy_C_ops.affine_grid( + theta, + "output_shape", + _out_shape, + "align_corners", + align_corners, + "use_cudnn", + use_cudnn, + ) helper = LayerHelper('affine_grid') - check_variable_and_dtype(theta, 'theta', ['float32', 'float64'], - 'affine_grid') + check_variable_and_dtype( + theta, 'theta', ['float32', 'float64'], 'affine_grid' + ) out = helper.create_variable_for_type_inference(theta.dtype) ipts = {'Theta': theta} attrs = {"align_corners": align_corners, "use_cudnn": use_cudnn} if isinstance(out_shape, Variable): ipts['OutputShape'] = out_shape - check_variable_and_dtype(out_shape, 'out_shape', ['int32'], - 'affine_grid') + check_variable_and_dtype( + out_shape, 'out_shape', ['int32'], 'affine_grid' + ) else: attrs['output_shape'] = out_shape - helper.append_op(type='affine_grid', - inputs=ipts, - outputs={'Output': out}, - attrs=None if len(attrs) == 0 else attrs) + helper.append_op( + type='affine_grid', + inputs=ipts, + outputs={'Output': out}, + attrs=None if len(attrs) == 0 else attrs, + ) return out -def grid_sample(x, - grid, - mode='bilinear', - padding_mode='zeros', - align_corners=True, - name=None): +def grid_sample( + x, + grid, + mode='bilinear', + padding_mode='zeros', + align_corners=True, + name=None, +): """ Sample input X by using bilinear interpolation or nearest interpolation based on flow field grid, which is usually @@ -253,22 +273,33 @@ def grid_sample(x, _padding_modes = ['zeros', 'reflection', 'border'] if mode not in _modes: raise ValueError( - "The mode of grid sample function should be in {}, but got: {}". - format(_modes, mode)) + "The mode of grid sample function should be in {}, but got: {}".format( + _modes, mode + ) + ) if padding_mode not in _padding_modes: raise ValueError( - "The padding mode of grid sample function should be in {}, but got: {}" - .format(_padding_modes, padding_mode)) + "The padding mode of grid sample function should be in {}, but got: {}".format( + _padding_modes, padding_mode + ) + ) if not isinstance(align_corners, bool): - raise ValueError("The align corners should be bool, but got: {}".format( - align_corners)) + raise ValueError( + "The align corners should be bool, but got: {}".format( + align_corners + ) + ) cudnn_version = get_cudnn_version() use_cudnn = False - if not is_compiled_with_rocm() and ( - cudnn_version is not None - ) and align_corners and mode == 'bilinear' and padding_mode == 'zeros': + if ( + not is_compiled_with_rocm() + and (cudnn_version is not None) + and align_corners + and mode == 'bilinear' + and padding_mode == 'zeros' + ): use_cudnn = True # CUDNN always computes gradients for all inputs x.stop_gradient = False @@ -280,26 +311,37 @@ def grid_sample(x, if in_dygraph_mode(): return _C_ops.grid_sample(x, grid, mode, padding_mode, align_corners) elif in_dynamic_mode(): - attrs = ('mode', mode, 'padding_mode', padding_mode, 'align_corners', - align_corners, 'use_cudnn', use_cudnn) + attrs = ( + 'mode', + mode, + 'padding_mode', + padding_mode, + 'align_corners', + align_corners, + 'use_cudnn', + use_cudnn, + ) out = getattr(_legacy_C_ops, 'grid_sampler')(x, grid, *attrs) else: helper = LayerHelper("grid_sample", **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'grid_sample') - check_variable_and_dtype(grid, 'grid', ['float32', 'float64'], - 'grid_sample') + check_variable_and_dtype( + grid, 'grid', ['float32', 'float64'], 'grid_sample' + ) ipts = {'X': x, 'Grid': grid} attrs = { 'mode': mode, 'padding_mode': padding_mode, 'align_corners': align_corners, - 'use_cudnn': use_cudnn + 'use_cudnn': use_cudnn, } out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='grid_sampler', - inputs=ipts, - attrs=attrs, - outputs={'Output': out}) + helper.append_op( + type='grid_sampler', + inputs=ipts, + attrs=attrs, + outputs={'Output': out}, + ) return out @@ -336,24 +378,25 @@ def pixel_shuffle(x, upscale_factor, data_format="NCHW", name=None): if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'." - "But recevie Attr(data_format): {} ".format(data_format)) + "But recevie Attr(data_format): {} ".format(data_format) + ) if in_dygraph_mode(): return _C_ops.pixel_shuffle(x, upscale_factor, data_format) if _in_legacy_dygraph(): - return _legacy_C_ops.pixel_shuffle(x, "upscale_factor", upscale_factor, - "data_format", data_format) + return _legacy_C_ops.pixel_shuffle( + x, "upscale_factor", upscale_factor, "data_format", data_format + ) helper = LayerHelper("pixel_shuffle", **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'pixel_shuffle') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="pixel_shuffle", - inputs={"X": x}, - outputs={"Out": out}, - attrs={ - "upscale_factor": upscale_factor, - "data_format": data_format - }) + helper.append_op( + type="pixel_shuffle", + inputs={"X": x}, + outputs={"Out": out}, + attrs={"upscale_factor": upscale_factor, "data_format": data_format}, + ) return out @@ -383,8 +426,10 @@ def pixel_unshuffle(x, downscale_factor, data_format="NCHW", name=None): """ if len(x.shape) != 4: raise ValueError( - "Input x should be 4D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 4D tensor, but received x with the shape of {}".format( + x.shape + ) + ) if not isinstance(downscale_factor, int): raise TypeError("Downscale factor must be int type") @@ -395,23 +440,26 @@ def pixel_unshuffle(x, downscale_factor, data_format="NCHW", name=None): if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'." - "But recevie Attr(data_format): {} ".format(data_format)) + "But recevie Attr(data_format): {} ".format(data_format) + ) if _non_static_mode(): - return _legacy_C_ops.pixel_unshuffle(x, "downscale_factor", - downscale_factor, "data_format", - data_format) + return _legacy_C_ops.pixel_unshuffle( + x, "downscale_factor", downscale_factor, "data_format", data_format + ) helper = LayerHelper("pixel_unshuffle", **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'pixel_unshuffle') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="pixel_unshuffle", - inputs={"X": x}, - outputs={"Out": out}, - attrs={ - "downscale_factor": downscale_factor, - "data_format": data_format - }) + helper.append_op( + type="pixel_unshuffle", + inputs={"X": x}, + outputs={"Out": out}, + attrs={ + "downscale_factor": downscale_factor, + "data_format": data_format, + }, + ) return out @@ -452,8 +500,10 @@ def channel_shuffle(x, groups, data_format="NCHW", name=None): """ if len(x.shape) != 4: raise ValueError( - "Input x should be 4D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 4D tensor, but received x with the shape of {}".format( + x.shape + ) + ) if not isinstance(groups, int): raise TypeError("groups must be int type") @@ -464,20 +514,21 @@ def channel_shuffle(x, groups, data_format="NCHW", name=None): if data_format not in ["NCHW", "NHWC"]: raise ValueError( "Attr(data_format) should be 'NCHW' or 'NHWC'." - "But recevie Attr(data_format): {} ".format(data_format)) + "But recevie Attr(data_format): {} ".format(data_format) + ) if _non_static_mode(): - return _legacy_C_ops.channel_shuffle(x, "groups", groups, "data_format", - data_format) + return _legacy_C_ops.channel_shuffle( + x, "groups", groups, "data_format", data_format + ) helper = LayerHelper("channel_shuffle", **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'channel_shuffle') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="channel_shuffle", - inputs={"X": x}, - outputs={"Out": out}, - attrs={ - "groups": groups, - "data_format": data_format - }) + helper.append_op( + type="channel_shuffle", + inputs={"X": x}, + outputs={"Out": out}, + attrs={"groups": groups, "data_format": data_format}, + ) return out diff --git a/python/paddle/nn/initializer/__init__.py b/python/paddle/nn/initializer/__init__.py index 530c52bf5f26d..e078e19ed2b4d 100644 --- a/python/paddle/nn/initializer/__init__.py +++ b/python/paddle/nn/initializer/__init__.py @@ -36,8 +36,19 @@ from .dirac import Dirac # noqa: F401 -__all__ = [ #noqa - 'Bilinear', 'Constant', 'KaimingUniform', 'KaimingNormal', 'XavierNormal', - 'XavierUniform', 'Assign', 'Normal', 'TruncatedNormal', 'Uniform', - 'Orthogonal', 'Dirac', 'set_global_initializer', 'calculate_gain' +__all__ = [ # noqa + 'Bilinear', + 'Constant', + 'KaimingUniform', + 'KaimingNormal', + 'XavierNormal', + 'XavierUniform', + 'Assign', + 'Normal', + 'TruncatedNormal', + 'Uniform', + 'Orthogonal', + 'Dirac', + 'set_global_initializer', + 'calculate_gain', ] diff --git a/python/paddle/nn/initializer/assign.py b/python/paddle/nn/initializer/assign.py index 2cdd5fdf1aa3d..22560890fe90b 100644 --- a/python/paddle/nn/initializer/assign.py +++ b/python/paddle/nn/initializer/assign.py @@ -83,15 +83,19 @@ class Assign(NumpyArrayInitializer): def __init__(self, value, name=None): import numpy - check_type(value, 'value', - (numpy.ndarray, list, tuple, paddle.static.Variable), - 'Assign') - if (isinstance(value, (list, tuple))): + check_type( + value, + 'value', + (numpy.ndarray, list, tuple, paddle.static.Variable), + 'Assign', + ) + + if isinstance(value, (list, tuple)): value = numpy.array(value) # TODO: value is already is a tensor, accounting efficiency maybe it does not need to convert tensor to numpy data and then initialized. - if (isinstance(value, paddle.static.Variable)): + if isinstance(value, paddle.static.Variable): value = value.numpy() super(Assign, self).__init__(value) diff --git a/python/paddle/nn/initializer/dirac.py b/python/paddle/nn/initializer/dirac.py index 74fc285c6bbf3..f95cd2d8274d9 100644 --- a/python/paddle/nn/initializer/dirac.py +++ b/python/paddle/nn/initializer/dirac.py @@ -90,7 +90,8 @@ class Dirac(Initializer): def __init__(self, groups=1, name=None): assert groups > 0 and isinstance( - groups, int), " 'groups' must be a positive integer. " + groups, int + ), " 'groups' must be a positive integer. " super(Dirac, self).__init__() self._groups = groups @@ -108,43 +109,49 @@ def __call__(self, var, block=None): block = self._check_block(block) assert isinstance(var, framework.Parameter) assert isinstance(block, framework.Block) - check_variable_and_dtype(var, "Out", - ['float16', 'bfloat16', 'float32', 'float64'], - 'Dirac') + check_variable_and_dtype( + var, "Out", ['float16', 'bfloat16', 'float32', 'float64'], 'Dirac' + ) assert len(var.shape) in [ - 3, 4, 5 + 3, + 4, + 5, ], "Only Tensor with 3/4/5 dimensions can be initialized by Dirac" assert ( - var.shape[0] % - self._groups) == 0, "Tensor 0-dimension must be divisible by groups" + var.shape[0] % self._groups + ) == 0, "Tensor 0-dimension must be divisible by groups" if var.dtype != VarDesc.VarType.FP32: - out_var = block.create_var(name=unique_name.generate(".".join( - ['dirac', var.name, 'tmp'])), - shape=var.shape, - dtype=VarDesc.VarType.FP32, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False) + out_var = block.create_var( + name=unique_name.generate(".".join(['dirac', var.name, 'tmp'])), + shape=var.shape, + dtype=VarDesc.VarType.FP32, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + ) else: out_var = var op = None if framework.in_dygraph_mode(): with fluid.dygraph.no_grad(): place = _current_expected_place() - _C_ops.full_(out_var, out_var.shape, str(float(0)), - out_var.dtype, place) + _C_ops.full_( + out_var, out_var.shape, str(float(0)), out_var.dtype, place + ) else: - block.append_op(type='fill_constant', - inputs={}, - outputs={'Out': out_var}, - attrs={ - 'value': float(0), - 'dtype': out_var.dtype, - 'shape': out_var.shape, - }, - stop_gradient=True) + block.append_op( + type='fill_constant', + inputs={}, + outputs={'Out': out_var}, + attrs={ + 'value': float(0), + 'dtype': out_var.dtype, + 'shape': out_var.shape, + }, + stop_gradient=True, + ) origin_shape = var.shape num_per_group = origin_shape[0] // self._groups @@ -162,9 +169,9 @@ def __call__(self, var, block=None): value_list.append(1.0) offset = 0 for (k, stride) in enumerate(strides): - if (k == 0): + if k == 0: offset += (j + i * num_per_group) * stride - elif (k == 1): + elif k == 1: offset += j * stride else: offset += origin_shape[k] // 2 * stride @@ -174,71 +181,86 @@ def __call__(self, var, block=None): tmp_out = _C_ops.reshape(out_var, [-1]) tmp_out._share_underline_tensor_to(out_var) else: - x_shape = block.create_var(name=unique_name.generate(".".join( - [out_var.name, "XShape"])), - dtype=out_var.dtype, - shape=out_var.shape, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False, - stop_gradient=True) - block.append_op(type="reshape2", - inputs={"X": out_var}, - attrs={'shape': [-1]}, - outputs={ - "Out": out_var, - "XShape": x_shape - }, - stop_gradient=True) + x_shape = block.create_var( + name=unique_name.generate(".".join([out_var.name, "XShape"])), + dtype=out_var.dtype, + shape=out_var.shape, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + stop_gradient=True, + ) + block.append_op( + type="reshape2", + inputs={"X": out_var}, + attrs={'shape': [-1]}, + outputs={"Out": out_var, "XShape": x_shape}, + stop_gradient=True, + ) index_tensor = block.create_var( name=unique_name.generate('scatter_index'), persistable=False, - stop_gradient=True) + stop_gradient=True, + ) if framework.in_dygraph_mode(): with fluid.dygraph.no_grad(): tmp_tensor = framework._varbase_creator() - _C_ops.assign_value_(tmp_tensor, [len(idx_list)], - VarDesc.VarType.INT64, idx_list, - _current_expected_place()) + _C_ops.assign_value_( + tmp_tensor, + [len(idx_list)], + VarDesc.VarType.INT64, + idx_list, + _current_expected_place(), + ) tmp_tensor._share_underline_tensor_to(index_tensor) else: - block.append_op(type='assign_value', - outputs={'Out': index_tensor}, - attrs={ - 'dtype': VarDesc.VarType.INT64, - 'shape': [len(idx_list)], - 'int64_values': idx_list - }, - stop_gradient=True) + block.append_op( + type='assign_value', + outputs={'Out': index_tensor}, + attrs={ + 'dtype': VarDesc.VarType.INT64, + 'shape': [len(idx_list)], + 'int64_values': idx_list, + }, + stop_gradient=True, + ) value_tensor = block.create_var( name=unique_name.generate('scatter_value'), persistable=False, - stop_gradient=True) + stop_gradient=True, + ) if framework.in_dygraph_mode(): with fluid.dygraph.no_grad(): tmp_tensor = framework._varbase_creator() - _C_ops.assign_value_(tmp_tensor, [len(value_list)], - VarDesc.VarType.FP32, value_list, - _current_expected_place()) + _C_ops.assign_value_( + tmp_tensor, + [len(value_list)], + VarDesc.VarType.FP32, + value_list, + _current_expected_place(), + ) tmp_tensor._share_underline_tensor_to(value_tensor) else: - block.append_op(type='assign_value', - outputs={'Out': value_tensor}, - attrs={ - 'dtype': VarDesc.VarType.FP32, - 'shape': [len(value_list)], - 'fp32_values': value_list - }, - stop_gradient=True) + block.append_op( + type='assign_value', + outputs={'Out': value_tensor}, + attrs={ + 'dtype': VarDesc.VarType.FP32, + 'shape': [len(value_list)], + 'fp32_values': value_list, + }, + stop_gradient=True, + ) if framework.in_dygraph_mode(): with fluid.dygraph.no_grad(): - tmp_out = _C_ops.scatter(out_var, index_tensor, value_tensor, - True) + tmp_out = _C_ops.scatter( + out_var, index_tensor, value_tensor, True + ) tmp_out._share_underline_tensor_to(out_var) tmp_reshape_out = _C_ops.reshape(out_var, origin_shape) tmp_reshape_out._share_underline_tensor_to(out_var) @@ -246,39 +268,40 @@ def __call__(self, var, block=None): tmp_cast_out = _C_ops.cast(out_var, var.dtype) tmp_cast_out._share_underline_tensor_to(var) else: - op = block.append_op(type="scatter", - inputs={ - "X": out_var, - "Ids": index_tensor, - "Updates": value_tensor - }, - attrs={'overwrite': True}, - outputs={"Out": out_var}, - stop_gradient=True) - x_shape = block.create_var(name=unique_name.generate(".".join( - [out_var.name, "XShape"])), - dtype=out_var.dtype, - shape=out_var.shape, - type=VarDesc.VarType.LOD_TENSOR, - persistable=False, - stop_gradient=True) - block.append_op(type="reshape2", - inputs={"X": out_var}, - attrs={'shape': origin_shape}, - outputs={ - "Out": out_var, - "XShape": x_shape - }, - stop_gradient=True) + op = block.append_op( + type="scatter", + inputs={ + "X": out_var, + "Ids": index_tensor, + "Updates": value_tensor, + }, + attrs={'overwrite': True}, + outputs={"Out": out_var}, + stop_gradient=True, + ) + x_shape = block.create_var( + name=unique_name.generate(".".join([out_var.name, "XShape"])), + dtype=out_var.dtype, + shape=out_var.shape, + type=VarDesc.VarType.LOD_TENSOR, + persistable=False, + stop_gradient=True, + ) + block.append_op( + type="reshape2", + inputs={"X": out_var}, + attrs={'shape': origin_shape}, + outputs={"Out": out_var, "XShape": x_shape}, + stop_gradient=True, + ) if var.dtype != VarDesc.VarType.FP32: - block.append_op(type="cast", - inputs={"X": out_var}, - outputs={"Out": var}, - attrs={ - "in_dtype": out_var.dtype, - "out_dtype": var.dtype - }, - stop_gradient=True) + block.append_op( + type="cast", + inputs={"X": out_var}, + outputs={"Out": var}, + attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}, + stop_gradient=True, + ) if not in_dynamic_mode(): var.op = op return op diff --git a/python/paddle/nn/initializer/kaiming.py b/python/paddle/nn/initializer/kaiming.py index ab4c90343fec5..8449ce925fb36 100644 --- a/python/paddle/nn/initializer/kaiming.py +++ b/python/paddle/nn/initializer/kaiming.py @@ -58,11 +58,13 @@ class KaimingNormal(MSRAInitializer): """ def __init__(self, fan_in=None, negative_slope=0.0, nonlinearity='relu'): - super(KaimingNormal, self).__init__(uniform=False, - fan_in=fan_in, - seed=0, - negative_slope=negative_slope, - nonlinearity=nonlinearity) + super(KaimingNormal, self).__init__( + uniform=False, + fan_in=fan_in, + seed=0, + negative_slope=negative_slope, + nonlinearity=nonlinearity, + ) class KaimingUniform(MSRAInitializer): @@ -104,8 +106,10 @@ class KaimingUniform(MSRAInitializer): """ def __init__(self, fan_in=None, negative_slope=0.0, nonlinearity='relu'): - super(KaimingUniform, self).__init__(uniform=True, - fan_in=fan_in, - seed=0, - negative_slope=negative_slope, - nonlinearity=nonlinearity) + super(KaimingUniform, self).__init__( + uniform=True, + fan_in=fan_in, + seed=0, + negative_slope=negative_slope, + nonlinearity=nonlinearity, + ) diff --git a/python/paddle/nn/initializer/orthogonal.py b/python/paddle/nn/initializer/orthogonal.py index 65871ebad0b50..84fabba483deb 100644 --- a/python/paddle/nn/initializer/orthogonal.py +++ b/python/paddle/nn/initializer/orthogonal.py @@ -85,15 +85,16 @@ def __call__(self, var, block=None): assert isinstance(var, framework.Parameter) assert isinstance(block, framework.Block) # 'qr' op only support float32/float64 now - check_variable_and_dtype(var, "Out", ["float32", "float64"], - "Orthogonal") + check_variable_and_dtype( + var, "Out", ["float32", "float64"], "Orthogonal" + ) self._seed = block.program.random_seed shape = var.shape - assert len( - shape - ) >= 2, "Only Tensor with 2 or more dimensions can be initialized by Orthogonal" + assert ( + len(shape) >= 2 + ), "Only Tensor with 2 or more dimensions can be initialized by Orthogonal" row = shape[0] col = 1 @@ -105,9 +106,9 @@ def __call__(self, var, block=None): if framework.in_dygraph_mode(): with no_grad(): place = framework._current_expected_place() - normal_var = _C_ops.gaussian_random(flatten_shape, 0.0, 1.0, - self._seed, var.dtype, - place) + normal_var = _C_ops.gaussian_random( + flatten_shape, 0.0, 1.0, self._seed, var.dtype, place + ) q, r = _C_ops.qr(normal_var, 'reduced') r_diag = _C_ops.diag(r, 0, 0) @@ -127,107 +128,114 @@ def __call__(self, var, block=None): return None - normal_var = block.create_var(name=unique_name.generate('.'.join( - ['gaussian_random', 'tmp'])), - dtype=var.dtype, - persistable=False, - stop_gradient=True) - block.append_op(type='gaussian_random', - inputs={}, - outputs={'Out': normal_var}, - attrs={ - 'mean': 0.0, - 'std': 1.0, - 'shape': flatten_shape, - 'seed': self._seed, - 'dtype': var.dtype - }, - stop_gradient=True) - - q = block.create_var(name=unique_name.generate('.'.join( - ['qr', 'q', 'tmp'])), - dtype=normal_var.dtype, - persistable=False, - stop_gradient=True) - r = block.create_var(name=unique_name.generate('.'.join( - ['qr', 'r', 'tmp'])), - dtype=normal_var.dtype, - persistable=False, - stop_gradient=True) - block.append_op(type='qr', - inputs={'X': [normal_var]}, - outputs={ - 'Q': q, - 'R': r, - }, - attrs={'mode': 'reduced'}, - stop_gradient=True) - - r_diag = block.create_var(name=unique_name.generate('.'.join( - ['diag', 'tmp'])), - dtype=r.dtype, - persistable=False, - stop_gradient=True) - block.append_op(type='diag_v2', - inputs={'X': r}, - outputs={'Out': r_diag}, - attrs={ - 'offset': 0, - 'padding_value': 0 - }, - stop_gradient=True) + normal_var = block.create_var( + name=unique_name.generate('.'.join(['gaussian_random', 'tmp'])), + dtype=var.dtype, + persistable=False, + stop_gradient=True, + ) + block.append_op( + type='gaussian_random', + inputs={}, + outputs={'Out': normal_var}, + attrs={ + 'mean': 0.0, + 'std': 1.0, + 'shape': flatten_shape, + 'seed': self._seed, + 'dtype': var.dtype, + }, + stop_gradient=True, + ) + + q = block.create_var( + name=unique_name.generate('.'.join(['qr', 'q', 'tmp'])), + dtype=normal_var.dtype, + persistable=False, + stop_gradient=True, + ) + r = block.create_var( + name=unique_name.generate('.'.join(['qr', 'r', 'tmp'])), + dtype=normal_var.dtype, + persistable=False, + stop_gradient=True, + ) + block.append_op( + type='qr', + inputs={'X': [normal_var]}, + outputs={ + 'Q': q, + 'R': r, + }, + attrs={'mode': 'reduced'}, + stop_gradient=True, + ) + + r_diag = block.create_var( + name=unique_name.generate('.'.join(['diag', 'tmp'])), + dtype=r.dtype, + persistable=False, + stop_gradient=True, + ) + block.append_op( + type='diag_v2', + inputs={'X': r}, + outputs={'Out': r_diag}, + attrs={'offset': 0, 'padding_value': 0}, + stop_gradient=True, + ) r_sign = r_diag - block.append_op(type='sign', - inputs={'X': [r_diag]}, - outputs={'Out': r_sign}, - stop_gradient=True) - - block.append_op(type='elementwise_mul', - inputs={ - 'X': q, - 'Y': r_sign - }, - outputs={'Out': q}, - attrs={}, - stop_gradient=True) - - x_shape = block.create_var(name=unique_name.generate('.'.join( - ['transpose', 'shape', 'tmp'])), - dtype=q.dtype, - persistable=False, - stop_gradient=True) + block.append_op( + type='sign', + inputs={'X': [r_diag]}, + outputs={'Out': r_sign}, + stop_gradient=True, + ) + + block.append_op( + type='elementwise_mul', + inputs={'X': q, 'Y': r_sign}, + outputs={'Out': q}, + attrs={}, + stop_gradient=True, + ) + + x_shape = block.create_var( + name=unique_name.generate('.'.join(['transpose', 'shape', 'tmp'])), + dtype=q.dtype, + persistable=False, + stop_gradient=True, + ) if row < col: - q_transpose = block.create_var(name=unique_name.generate('.'.join( - ['transpose', 'tmp'])), - dtype=q.dtype, - persistable=False, - stop_gradient=True) - block.append_op(type='transpose2', - inputs={'X': q}, - outputs={ - 'Out': q_transpose, - 'XShape': x_shape - }, - attrs={'axis': [1, 0]}, - stop_gradient=True) + q_transpose = block.create_var( + name=unique_name.generate('.'.join(['transpose', 'tmp'])), + dtype=q.dtype, + persistable=False, + stop_gradient=True, + ) + block.append_op( + type='transpose2', + inputs={'X': q}, + outputs={'Out': q_transpose, 'XShape': x_shape}, + attrs={'axis': [1, 0]}, + stop_gradient=True, + ) q = q_transpose - block.append_op(type='reshape2', - inputs={'X': q}, - outputs={ - 'Out': q, - "XShape": x_shape - }, - attrs={'shape': var.shape}, - stop_gradient=True) - - op = block.append_op(type='scale', - inputs={'X': q}, - outputs={'Out': var}, - attrs={ - 'scale': self._gain, - 'bias': 0.0 - }) + block.append_op( + type='reshape2', + inputs={'X': q}, + outputs={'Out': q, "XShape": x_shape}, + attrs={'shape': var.shape}, + stop_gradient=True, + ) + + op = block.append_op( + type='scale', + inputs={'X': q}, + outputs={'Out': var}, + attrs={'scale': self._gain, 'bias': 0.0}, + ) return op diff --git a/python/paddle/nn/initializer/uniform.py b/python/paddle/nn/initializer/uniform.py index 2c7a57b1195cd..578a477247874 100644 --- a/python/paddle/nn/initializer/uniform.py +++ b/python/paddle/nn/initializer/uniform.py @@ -55,9 +55,6 @@ def __init__(self, low=-1.0, high=1.0, name=None): assert low is not None, 'low should not be None' assert high is not None, 'high should not be None' assert high >= low, 'high should greater or equal than low' - super(Uniform, self).__init__(low=low, - high=high, - seed=0, - diag_num=0, - diag_step=0, - diag_val=1.0) + super(Uniform, self).__init__( + low=low, high=high, seed=0, diag_num=0, diag_step=0, diag_val=1.0 + ) diff --git a/python/paddle/nn/initializer/xavier.py b/python/paddle/nn/initializer/xavier.py index 2c6d60a6bb86b..149b13df389e4 100644 --- a/python/paddle/nn/initializer/xavier.py +++ b/python/paddle/nn/initializer/xavier.py @@ -63,10 +63,9 @@ class XavierNormal(XavierInitializer): """ def __init__(self, fan_in=None, fan_out=None, name=None): - super(XavierNormal, self).__init__(uniform=False, - fan_in=fan_in, - fan_out=fan_out, - seed=0) + super(XavierNormal, self).__init__( + uniform=False, fan_in=fan_in, fan_out=fan_out, seed=0 + ) class XavierUniform(XavierInitializer): @@ -118,7 +117,6 @@ class XavierUniform(XavierInitializer): """ def __init__(self, fan_in=None, fan_out=None, name=None): - super(XavierUniform, self).__init__(uniform=True, - fan_in=fan_in, - fan_out=fan_out, - seed=0) + super(XavierUniform, self).__init__( + uniform=True, fan_in=fan_in, fan_out=fan_out, seed=0 + ) diff --git a/python/paddle/nn/layer/activation.py b/python/paddle/nn/layer/activation.py index 2382acbd0cab0..e3fa71c033285 100644 --- a/python/paddle/nn/layer/activation.py +++ b/python/paddle/nn/layer/activation.py @@ -402,12 +402,14 @@ class PReLU(Layer): # [ 6. , 7. , 8. , 9. ]]]] """ - def __init__(self, - num_parameters=1, - init=0.25, - weight_attr=None, - data_format="NCHW", - name=None): + def __init__( + self, + num_parameters=1, + init=0.25, + weight_attr=None, + data_format="NCHW", + name=None, + ): super(PReLU, self).__init__() self._num_parameters = num_parameters self._init = init @@ -415,12 +417,13 @@ def __init__(self, self._name = name self._data_format = data_format - self._weight = self.create_parameter(attr=self._weight_attr, - shape=[self._num_parameters], - dtype=get_default_dtype(), - is_bias=False, - default_initializer=Constant( - self._init)) + self._weight = self.create_parameter( + attr=self._weight_attr, + shape=[self._num_parameters], + dtype=get_default_dtype(), + is_bias=False, + default_initializer=Constant(self._init), + ) def forward(self, x): return F.prelu(x, self._weight, data_format=self._data_format) @@ -428,8 +431,12 @@ def forward(self, x): def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' return 'num_parameters={}, data_format={}, init={}, dtype={}{}'.format( - self._num_parameters, self._data_format, self._init, self._dtype, - name_str) + self._num_parameters, + self._data_format, + self._init, + self._dtype, + name_str, + ) class RReLU(Layer): @@ -503,22 +510,22 @@ class RReLU(Layer): # [ 6. 7. 8. 9. ]]]] """ - def __init__(self, lower=1. / 8., upper=1. / 3., name=None): + def __init__(self, lower=1.0 / 8.0, upper=1.0 / 3.0, name=None): super(RReLU, self).__init__() self._lower = lower self._upper = upper self._name = name def forward(self, x): - return F.rrelu(x, - lower=self._lower, - upper=self._upper, - training=self.training) + return F.rrelu( + x, lower=self._lower, upper=self._upper, training=self.training + ) def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' return 'lower={}, upper={}, training={}, dtype={}{}'.format( - self._lower, self._upper, self.training, self._dtype, name_str) + self._lower, self._upper, self.training, self._dtype, name_str + ) class ReLU(Layer): @@ -637,10 +644,12 @@ class SELU(Layer): # [[0, 1.050701],[2.101402, 3.152103]] """ - def __init__(self, - scale=1.0507009873554804934193349852946, - alpha=1.6732632423543772848170429916717, - name=None): + def __init__( + self, + scale=1.0507009873554804934193349852946, + alpha=1.6732632423543772848170429916717, + name=None, + ): super(SELU, self).__init__() self._scale = scale self._alpha = alpha @@ -651,8 +660,9 @@ def forward(self, x): def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' - return 'scale={:.16f}, alpha={:.16f}{}'.format(self._scale, self._alpha, - name_str) + return 'scale={:.16f}, alpha={:.16f}{}'.format( + self._scale, self._alpha, name_str + ) class LeakyReLU(Layer): @@ -835,8 +845,9 @@ def forward(self, x): def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' - return 'beta={}, threshold={}{}'.format(self._beta, self._threshold, - name_str) + return 'beta={}, threshold={}{}'.format( + self._beta, self._threshold, name_str + ) class Softshrink(Layer): @@ -1469,8 +1480,11 @@ def __init__(self, name=None): self._name = name def forward(self, x): - assert x.ndim == 3 or x.ndim == 4, "Softmax2D requires a 3D or 4D tensor as input. Received: {}D.".format( - x.ndim) + assert ( + x.ndim == 3 or x.ndim == 4 + ), "Softmax2D requires a 3D or 4D tensor as input. Received: {}D.".format( + x.ndim + ) return F.softmax(x, axis=-3, dtype=self._dtype, name=self._name) def extra_repr(self): diff --git a/python/paddle/nn/layer/common.py b/python/paddle/nn/layer/common.py index 6cc2056459f81..215f395e1fbcc 100644 --- a/python/paddle/nn/layer/common.py +++ b/python/paddle/nn/layer/common.py @@ -144,37 +144,43 @@ class Linear(Layer): # [2.1077576 2.1077576 2.1077576 2.1077576 ]] """ - def __init__(self, - in_features, - out_features, - weight_attr=None, - bias_attr=None, - name=None): + def __init__( + self, + in_features, + out_features, + weight_attr=None, + bias_attr=None, + name=None, + ): super(Linear, self).__init__() self._dtype = self._helper.get_default_dtype() self._weight_attr = weight_attr self._bias_attr = bias_attr - self.weight = self.create_parameter(shape=[in_features, out_features], - attr=self._weight_attr, - dtype=self._dtype, - is_bias=False) - self.bias = self.create_parameter(shape=[out_features], - attr=self._bias_attr, - dtype=self._dtype, - is_bias=True) + self.weight = self.create_parameter( + shape=[in_features, out_features], + attr=self._weight_attr, + dtype=self._dtype, + is_bias=False, + ) + self.bias = self.create_parameter( + shape=[out_features], + attr=self._bias_attr, + dtype=self._dtype, + is_bias=True, + ) self.name = name def forward(self, input): - out = F.linear(x=input, - weight=self.weight, - bias=self.bias, - name=self.name) + out = F.linear( + x=input, weight=self.weight, bias=self.bias, name=self.name + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' return 'in_features={}, out_features={}, dtype={}{}'.format( - self.weight.shape[0], self.weight.shape[1], self._dtype, name_str) + self.weight.shape[0], self.weight.shape[1], self._dtype, name_str + ) class Upsample(Layer): @@ -371,14 +377,16 @@ class Upsample(Layer): """ - def __init__(self, - size=None, - scale_factor=None, - mode='nearest', - align_corners=False, - align_mode=0, - data_format='NCHW', - name=None): + def __init__( + self, + size=None, + scale_factor=None, + mode='nearest', + align_corners=False, + align_mode=0, + data_format='NCHW', + name=None, + ): super(Upsample, self).__init__() self.size = size self.scale_factor = scale_factor @@ -389,14 +397,16 @@ def __init__(self, self.name = name def forward(self, x): - out = F.interpolate(x, - size=self.size, - scale_factor=self.scale_factor, - mode=self.mode, - align_corners=self.align_corners, - align_mode=self.align_mode, - data_format=self.data_format, - name=self.name) + out = F.interpolate( + x, + size=self.size, + scale_factor=self.scale_factor, + mode=self.mode, + align_corners=self.align_corners, + align_mode=self.align_mode, + data_format=self.data_format, + name=self.name, + ) return out @@ -407,8 +417,13 @@ def extra_repr(self): main_str = 'size={}'.format(self.size) name_str = ', name={}'.format(self.name) if self.name else '' return '{}, mode={}, align_corners={}, align_mode={}, data_format={}{}'.format( - main_str, self.mode, self.align_corners, self.align_mode, - self.data_format, name_str) + main_str, + self.mode, + self.align_corners, + self.align_mode, + self.data_format, + name_str, + ) class UpsamplingNearest2D(Layer): @@ -462,11 +477,9 @@ class UpsamplingNearest2D(Layer): # [2L, 3L, 12L, 12L] """ - def __init__(self, - size=None, - scale_factor=None, - data_format='NCHW', - name=None): + def __init__( + self, size=None, scale_factor=None, data_format='NCHW', name=None + ): super(UpsamplingNearest2D, self).__init__() self.size = size self.scale_factor = scale_factor @@ -474,14 +487,16 @@ def __init__(self, self.name = name def forward(self, x): - out = F.interpolate(x, - size=self.size, - scale_factor=self.scale_factor, - mode='nearest', - align_corners=False, - align_mode=0, - data_format=self.data_format, - name=self.name) + out = F.interpolate( + x, + size=self.size, + scale_factor=self.scale_factor, + mode='nearest', + align_corners=False, + align_mode=0, + data_format=self.data_format, + name=self.name, + ) return out @@ -491,8 +506,9 @@ def extra_repr(self): else: main_str = 'size={}'.format(self.size) name_str = ', name={}'.format(self.name) if self.name else '' - return '{}, data_format={}{}'.format(main_str, self.data_format, - name_str) + return '{}, data_format={}{}'.format( + main_str, self.data_format, name_str + ) class UpsamplingBilinear2D(Layer): @@ -547,11 +563,9 @@ class UpsamplingBilinear2D(Layer): # [2L, 3L, 12L, 12L] """ - def __init__(self, - size=None, - scale_factor=None, - data_format='NCHW', - name=None): + def __init__( + self, size=None, scale_factor=None, data_format='NCHW', name=None + ): super(UpsamplingBilinear2D, self).__init__() self.size = size self.scale_factor = scale_factor @@ -559,14 +573,16 @@ def __init__(self, self.name = name def forward(self, x): - out = F.interpolate(x, - size=self.size, - scale_factor=self.scale_factor, - mode='bilinear', - align_corners=True, - align_mode=0, - data_format=self.data_format, - name=self.name) + out = F.interpolate( + x, + size=self.size, + scale_factor=self.scale_factor, + mode='bilinear', + align_corners=True, + align_mode=0, + data_format=self.data_format, + name=self.name, + ) return out @@ -576,8 +592,9 @@ def extra_repr(self): else: main_str = 'size={}'.format(self.size) name_str = ', name={}'.format(self.name) if self.name else '' - return '{}, data_format={}{}'.format(main_str, self.data_format, - name_str) + return '{}, data_format={}{}'.format( + main_str, self.data_format, name_str + ) class Bilinear(Layer): @@ -634,13 +651,15 @@ class Bilinear(Layer): """ - def __init__(self, - in1_features, - in2_features, - out_features, - weight_attr=None, - bias_attr=None, - name=None): + def __init__( + self, + in1_features, + in2_features, + out_features, + weight_attr=None, + bias_attr=None, + name=None, + ): super(Bilinear, self).__init__() self._weight_attr = weight_attr self._bias_attr = bias_attr @@ -651,17 +670,23 @@ def __init__(self, self._dtype = self._helper.get_default_dtype() weight_shape = [ - self._out_features, self._in1_features, self._in2_features + self._out_features, + self._in1_features, + self._in2_features, ] - self.weight = self.create_parameter(attr=self._weight_attr, - shape=weight_shape, - dtype=self._dtype, - is_bias=False) + self.weight = self.create_parameter( + attr=self._weight_attr, + shape=weight_shape, + dtype=self._dtype, + is_bias=False, + ) bias_shape = [1, self._out_features] - self.bias = self.create_parameter(attr=self._bias_attr, - shape=bias_shape, - dtype=self._dtype, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=bias_shape, + dtype=self._dtype, + is_bias=True, + ) def forward(self, x1, x2): return F.bilinear(x1, x2, self.weight, self.bias, self._name) @@ -669,8 +694,12 @@ def forward(self, x1, x2): def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' return 'in1_features={}, in2_features={}, out_features={}, dtype={}{}'.format( - self._in1_features, self._in2_features, self._out_features, - self._dtype, name_str) + self._in1_features, + self._in2_features, + self._out_features, + self._dtype, + name_str, + ) class Dropout(Layer): @@ -721,7 +750,7 @@ class Dropout(Layer): print(x) print(y_train) print(y_test) - """ + """ def __init__(self, p=0.5, axis=None, mode="upscale_in_train", name=None): super(Dropout, self).__init__() @@ -732,18 +761,21 @@ def __init__(self, p=0.5, axis=None, mode="upscale_in_train", name=None): self.name = name def forward(self, input): - out = F.dropout(input, - p=self.p, - axis=self.axis, - training=self.training, - mode=self.mode, - name=self.name) + out = F.dropout( + input, + p=self.p, + axis=self.axis, + training=self.training, + mode=self.mode, + name=self.name, + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' - return 'p={}, axis={}, mode={}{}'.format(self.p, self.axis, self.mode, - name_str) + return 'p={}, axis={}, mode={}{}'.format( + self.p, self.axis, self.mode, name_str + ) class Dropout2D(Layer): @@ -783,7 +815,7 @@ class Dropout2D(Layer): print(x) print(y_train) print(y_test) - """ + """ def __init__(self, p=0.5, data_format='NCHW', name=None): super(Dropout2D, self).__init__() @@ -793,17 +825,20 @@ def __init__(self, p=0.5, data_format='NCHW', name=None): self.name = name def forward(self, input): - out = F.dropout2d(input, - p=self.p, - training=self.training, - data_format=self.data_format, - name=self.name) + out = F.dropout2d( + input, + p=self.p, + training=self.training, + data_format=self.data_format, + name=self.name, + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' - return 'p={}, data_format={}{}'.format(self.p, self.data_format, - name_str) + return 'p={}, data_format={}{}'.format( + self.p, self.data_format, name_str + ) class Dropout3D(Layer): @@ -843,7 +878,7 @@ class Dropout3D(Layer): print(x) print(y_train) print(y_test) - """ + """ def __init__(self, p=0.5, data_format='NCDHW', name=None): super(Dropout3D, self).__init__() @@ -853,17 +888,20 @@ def __init__(self, p=0.5, data_format='NCDHW', name=None): self.name = name def forward(self, input): - out = F.dropout3d(input, - p=self.p, - training=self.training, - data_format=self.data_format, - name=self.name) + out = F.dropout3d( + input, + p=self.p, + training=self.training, + data_format=self.data_format, + name=self.name, + ) return out def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' - return 'p={}, data_format={}{}'.format(self.p, self.data_format, - name_str) + return 'p={}, data_format={}{}'.format( + self.p, self.data_format, name_str + ) class AlphaDropout(Layer): @@ -902,7 +940,7 @@ class AlphaDropout(Layer): print(y_train) # [[-0.10721093, 1.6655989 ], [-0.7791938, -0.7791938]] (randomly) print(y_test) - """ + """ def __init__(self, p=0.5, name=None): super(AlphaDropout, self).__init__() @@ -910,10 +948,9 @@ def __init__(self, p=0.5, name=None): self.name = name def forward(self, input): - out = F.alpha_dropout(input, - p=self.p, - training=self.training, - name=self.name) + out = F.alpha_dropout( + input, p=self.p, training=self.training, name=self.name + ) return out def extra_repr(self): @@ -963,12 +1000,9 @@ class Pad1D(Layer): # [0. 4. 5. 6. 0. 0.]]] """ - def __init__(self, - padding, - mode='constant', - value=0.0, - data_format="NCL", - name=None): + def __init__( + self, padding, mode='constant', value=0.0, data_format="NCL", name=None + ): super(Pad1D, self).__init__() self._pad = _npairs(padding, 1) self._mode = mode @@ -977,17 +1011,20 @@ def __init__(self, self._name = name def forward(self, x): - return F.pad(x, - pad=self._pad, - mode=self._mode, - value=self._value, - data_format=self._data_format, - name=self._name) + return F.pad( + x, + pad=self._pad, + mode=self._mode, + value=self._value, + data_format=self._data_format, + name=self._name, + ) def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' return 'padding={}, mode={}, value={}, data_format={}{}'.format( - self._pad, self._mode, self._value, self._data_format, name_str) + self._pad, self._mode, self._value, self._data_format, name_str + ) class Pad2D(Layer): @@ -1036,12 +1073,9 @@ class Pad2D(Layer): # [0. 0. 0. 0.]]]] """ - def __init__(self, - padding, - mode='constant', - value=0.0, - data_format="NCHW", - name=None): + def __init__( + self, padding, mode='constant', value=0.0, data_format="NCHW", name=None + ): super(Pad2D, self).__init__() self._pad = _npairs(padding, 2) self._mode = mode @@ -1050,17 +1084,20 @@ def __init__(self, self._name = name def forward(self, x): - return F.pad(x, - pad=self._pad, - mode=self._mode, - value=self._value, - data_format=self._data_format, - name=self._name) + return F.pad( + x, + pad=self._pad, + mode=self._mode, + value=self._value, + data_format=self._data_format, + name=self._name, + ) def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' return 'padding={}, mode={}, value={}, data_format={}{}'.format( - self._pad, self._mode, self._value, self._data_format, name_str) + self._pad, self._mode, self._value, self._data_format, name_str + ) class ZeroPad2D(Layer): @@ -1111,23 +1148,25 @@ def __init__(self, padding, data_format="NCHW", name=None): super(ZeroPad2D, self).__init__() self._pad = _npairs(padding, 2) self._mode = 'constant' - self._value = 0. + self._value = 0.0 self._data_format = data_format self._name = name def forward(self, x): - return F.pad(x, - pad=self._pad, - mode=self._mode, - value=self._value, - data_format=self._data_format, - name=self._name) + return F.pad( + x, + pad=self._pad, + mode=self._mode, + value=self._value, + data_format=self._data_format, + name=self._name, + ) def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' - return 'padding={}, data_format={}{}'.format(self._pad, - self._data_format, - name_str) + return 'padding={}, data_format={}{}'.format( + self._pad, self._data_format, name_str + ) class Pad3D(Layer): @@ -1176,12 +1215,14 @@ class Pad3D(Layer): # [0. 0. 0. 0.]]]]] """ - def __init__(self, - padding, - mode='constant', - value=0.0, - data_format="NCDHW", - name=None): + def __init__( + self, + padding, + mode='constant', + value=0.0, + data_format="NCDHW", + name=None, + ): super(Pad3D, self).__init__() self._pad = _npairs(padding, 3) self._mode = mode @@ -1190,17 +1231,20 @@ def __init__(self, self._name = name def forward(self, x): - return F.pad(x, - pad=self._pad, - mode=self._mode, - value=self._value, - data_format=self._data_format, - name=self._name) + return F.pad( + x, + pad=self._pad, + mode=self._mode, + value=self._value, + data_format=self._data_format, + name=self._name, + ) def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' return 'padding={}, mode={}, value={}, data_format={}{}'.format( - self._pad, self._mode, self._value, self._data_format, name_str) + self._pad, self._mode, self._value, self._data_format, name_str + ) class CosineSimilarity(Layer): @@ -1361,13 +1405,15 @@ class Embedding(Layer): """ - def __init__(self, - num_embeddings, - embedding_dim, - padding_idx=None, - sparse=False, - weight_attr=None, - name=None): + def __init__( + self, + num_embeddings, + embedding_dim, + padding_idx=None, + sparse=False, + weight_attr=None, + name=None, + ): super(Embedding, self).__init__() self._num_embeddings = num_embeddings self._embedding_dim = embedding_dim @@ -1381,12 +1427,20 @@ def __init__(self, if self._embedding_dim <= 0: raise ValueError("embedding_dim must be gather than 0") - padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( - num_embeddings + padding_idx) + padding_idx = ( + -1 + if padding_idx is None + else padding_idx + if padding_idx >= 0 + else (num_embeddings + padding_idx) + ) if padding_idx >= num_embeddings or padding_idx < -num_embeddings: - raise ValueError("padding_idx must be within [-{}, {})".format( - num_embeddings, num_embeddings)) + raise ValueError( + "padding_idx must be within [-{}, {})".format( + num_embeddings, num_embeddings + ) + ) self._dtype = self._helper.get_default_dtype() self._size = [self._num_embeddings, self._embedding_dim] @@ -1394,21 +1448,25 @@ def __init__(self, self._weight_attr = weight_attr self._remote_prefetch = False self._name = name - self.weight = self.create_parameter(attr=self._weight_attr, - shape=self._size, - dtype=self._dtype, - is_bias=False) + self.weight = self.create_parameter( + attr=self._weight_attr, + shape=self._size, + dtype=self._dtype, + is_bias=False, + ) if in_dynamic_mode() and padding_idx != -1: with paddle.no_grad(): self.weight[padding_idx] = 0.0 def forward(self, x): - return F.embedding(x, - weight=self.weight, - padding_idx=self._padding_idx, - sparse=self._sparse, - name=self._name) + return F.embedding( + x, + weight=self.weight, + padding_idx=self._padding_idx, + sparse=self._sparse, + name=self._name, + ) def extra_repr(self): main_str = '{_num_embeddings}, {_embedding_dim}' @@ -1466,12 +1524,9 @@ class Unfold(Layer): print(result) """ - def __init__(self, - kernel_sizes, - dilations=1, - paddings=0, - strides=1, - name=None): + def __init__( + self, kernel_sizes, dilations=1, paddings=0, strides=1, name=None + ): super(Unfold, self).__init__() self.kernel_sizes = kernel_sizes @@ -1481,17 +1536,24 @@ def __init__(self, self.name = name def forward(self, input): - return F.unfold(input, - kernel_sizes=self.kernel_sizes, - strides=self.strides, - paddings=self.paddings, - dilations=self.dilations, - name=self.name) + return F.unfold( + input, + kernel_sizes=self.kernel_sizes, + strides=self.strides, + paddings=self.paddings, + dilations=self.dilations, + name=self.name, + ) def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' - return 'kernel_size={}, dilation={}, padding={}, stride={}{}'.\ - format(self.kernel_sizes, self.dilations, self.paddings, self.strides, name_str) + return 'kernel_size={}, dilation={}, padding={}, stride={}{}'.format( + self.kernel_sizes, + self.dilations, + self.paddings, + self.strides, + name_str, + ) class Fold(Layer): @@ -1551,13 +1613,15 @@ class Fold(Layer): # y.shape = [2,3,4,5] """ - def __init__(self, - output_sizes, - kernel_sizes, - dilations=1, - paddings=0, - strides=1, - name=None): + def __init__( + self, + output_sizes, + kernel_sizes, + dilations=1, + paddings=0, + strides=1, + name=None, + ): super(Fold, self).__init__() self.output_sizes = output_sizes @@ -1568,15 +1632,22 @@ def __init__(self, self.name = name def forward(self, input): - return F.fold(input, - output_sizes=self.output_sizes, - kernel_sizes=self.kernel_sizes, - strides=self.strides, - paddings=self.paddings, - dilations=self.dilations, - name=self.name) + return F.fold( + input, + output_sizes=self.output_sizes, + kernel_sizes=self.kernel_sizes, + strides=self.strides, + paddings=self.paddings, + dilations=self.dilations, + name=self.name, + ) def extra_repr(self): name_str = ', name={}'.format(self.name) if self.name else '' - return 'kernel_size={}, dilation={}, padding={}, stride={}{}'.\ - format(self.kernel_sizes, self.dilations, self.paddings, self.strides, name_str) + return 'kernel_size={}, dilation={}, padding={}, stride={}{}'.format( + self.kernel_sizes, + self.dilations, + self.paddings, + self.strides, + name_str, + ) diff --git a/python/paddle/nn/layer/container.py b/python/paddle/nn/layer/container.py index fe2753a614159..a3ca6294e3817 100644 --- a/python/paddle/nn/layer/container.py +++ b/python/paddle/nn/layer/container.py @@ -275,10 +275,10 @@ def update(self, sublayers): """ - assert isinstance( - sublayers, Iterable - ), "The type of sublayers is not iterable of key/value pairs, the type of sublayers is " + type( - sublayers).__name__ + assert isinstance(sublayers, Iterable), ( + "The type of sublayers is not iterable of key/value pairs, the type of sublayers is " + + type(sublayers).__name__ + ) if isinstance(sublayers, (OrderedDict, LayerDict, Mapping)): for key, layer in sublayers.items(): @@ -287,7 +287,11 @@ def update(self, sublayers): # handle this format [(key1, layer1), (key2, layer2)...] for i, kv in enumerate(sublayers): if len(kv) != 2: - raise ValueError("The length of the " + str(i) + - "'s element in sublayers is " + - str(len(kv)) + ", which must be 2.") + raise ValueError( + "The length of the " + + str(i) + + "'s element in sublayers is " + + str(len(kv)) + + ", which must be 2." + ) self.add_sublayer(kv[0], kv[1]) diff --git a/python/paddle/nn/layer/conv.py b/python/paddle/nn/layer/conv.py index fd5583d50baae..61f9044d03a93 100644 --- a/python/paddle/nn/layer/conv.py +++ b/python/paddle/nn/layer/conv.py @@ -31,7 +31,7 @@ def _get_default_param_initializer(num_channels, filter_size): filter_elem_num = num_channels * np.prod(filter_size) - std = (2.0 / filter_elem_num)**0.5 + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std) @@ -44,24 +44,27 @@ def _reverse_repeat_list(t, n): class _ConvNd(Layer): - - def __init__(self, - in_channels, - out_channels, - kernel_size, - transposed, - dims, - stride=1, - padding=0, - padding_mode='zeros', - output_padding=0, - dilation=1, - groups=1, - weight_attr=None, - bias_attr=None, - data_format="NCHW"): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + transposed, + dims, + stride=1, + padding=0, + padding_mode='zeros', + output_padding=0, + dilation=1, + groups=1, + weight_attr=None, + bias_attr=None, + data_format="NCHW", + ): super(_ConvNd, self).__init__() - assert weight_attr is not False, "weight_attr should not be False in Conv." + assert ( + weight_attr is not False + ), "weight_attr should not be False in Conv." self._param_attr = weight_attr self._bias_attr = bias_attr self._groups = groups @@ -72,11 +75,16 @@ def __init__(self, valid_padding_modes = {'zeros', 'reflect', 'replicate', 'circular'} if padding_mode not in valid_padding_modes: raise ValueError( - "padding_mode must be one of {}, but got padding_mode='{}'". - format(valid_padding_modes, padding_mode)) + "padding_mode must be one of {}, but got padding_mode='{}'".format( + valid_padding_modes, padding_mode + ) + ) - if padding_mode in {'reflect', 'replicate', 'circular' - } and not isinstance(padding, int): + if padding_mode in { + 'reflect', + 'replicate', + 'circular', + } and not isinstance(padding, int): raise TypeError( "when padding_mode in ['reflect', 'replicate', 'circular'], type of padding must be int" ) @@ -84,12 +92,16 @@ def __init__(self, valid_format = {'NHWC', 'NCHW', 'NDHWC', 'NCDHW', 'NLC', 'NCL'} if data_format not in valid_format: raise ValueError( - "data_format must be one of {}, but got data_format='{}'". - format(valid_format, data_format)) + "data_format must be one of {}, but got data_format='{}'".format( + valid_format, data_format + ) + ) - channel_last = (data_format == "NHWC") or (data_format - == "NDHWC") or (data_format - == "NLC") + channel_last = ( + (data_format == "NHWC") + or (data_format == "NDHWC") + or (data_format == "NLC") + ) if channel_last: self._channel_dim = len(data_format) - 1 else: @@ -97,66 +109,86 @@ def __init__(self, self._stride = utils.convert_to_list(stride, dims, 'stride') self._dilation = utils.convert_to_list(dilation, dims, 'dilation') - self._kernel_size = utils.convert_to_list(kernel_size, dims, - 'kernel_size') + self._kernel_size = utils.convert_to_list( + kernel_size, dims, 'kernel_size' + ) self._padding = padding self._padding_mode = padding_mode self.output_padding = output_padding if dims != 1: self._updated_padding, self._padding_algorithm = _update_padding_nd( - padding, channel_last, dims) + padding, channel_last, dims + ) if transposed: - filter_shape = [self._in_channels, out_channels // groups - ] + self._kernel_size + filter_shape = [ + self._in_channels, + out_channels // groups, + ] + self._kernel_size else: if in_channels % groups != 0: raise ValueError("in_channels must be divisible by groups.") if padding_mode in {'reflect', 'replicate', 'circular'}: - _paired_padding = utils.convert_to_list(padding, dims, - 'padding') + _paired_padding = utils.convert_to_list( + padding, dims, 'padding' + ) self._reversed_padding_repeated_twice = _reverse_repeat_list( - _paired_padding, 2) + _paired_padding, 2 + ) - self._updated_padding, self._padding_algorithm = _update_padding_nd( - 0, channel_last, dims) + ( + self._updated_padding, + self._padding_algorithm, + ) = _update_padding_nd(0, channel_last, dims) - filter_shape = [out_channels, in_channels // groups - ] + self._kernel_size + filter_shape = [ + out_channels, + in_channels // groups, + ] + self._kernel_size def _get_default_param_initializer(): if transposed: return None filter_elem_num = np.prod(self._kernel_size) * self._in_channels - std = (2.0 / filter_elem_num)**0.5 + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std) self.weight = self.create_parameter( shape=filter_shape, attr=self._param_attr, - default_initializer=_get_default_param_initializer()) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._out_channels], - is_bias=True) + default_initializer=_get_default_param_initializer(), + ) + self.bias = self.create_parameter( + attr=self._bias_attr, shape=[self._out_channels], is_bias=True + ) cudnn_version = get_cudnn_version() - self._use_cudnn = True if (is_compiled_with_cuda() - and cudnn_version is not None) else False + self._use_cudnn = ( + True + if (is_compiled_with_cuda() and cudnn_version is not None) + else False + ) self._op_type = "conv" + str(dims) + 'd' - if self._op_type == 'conv2d' and (in_channels == groups - and in_channels != 1 - and out_channels % in_channels == 0): + if self._op_type == 'conv2d' and ( + in_channels == groups + and in_channels != 1 + and out_channels % in_channels == 0 + ): self._op_type = 'depthwise_conv2d' if is_compiled_with_rocm(): self._use_cudnn = True else: self._use_cudnn = False - if (is_compiled_with_cuda() and get_flags("FLAGS_conv2d_disable_cudnn") - ["FLAGS_conv2d_disable_cudnn"]): + if ( + is_compiled_with_cuda() + and get_flags("FLAGS_conv2d_disable_cudnn")[ + "FLAGS_conv2d_disable_cudnn" + ] + ): self._use_cudnn = False def extra_repr(self): @@ -299,50 +331,58 @@ class Conv1D(_ConvNd): # [160. 211.]]] """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - dilation=1, - groups=1, - padding_mode='zeros', - weight_attr=None, - bias_attr=None, - data_format="NCL"): - super(Conv1D, self).__init__(in_channels, - out_channels, - kernel_size, - False, - 1, - stride=stride, - padding=padding, - padding_mode=padding_mode, - dilation=dilation, - groups=groups, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + padding_mode='zeros', + weight_attr=None, + bias_attr=None, + data_format="NCL", + ): + super(Conv1D, self).__init__( + in_channels, + out_channels, + kernel_size, + False, + 1, + stride=stride, + padding=padding, + padding_mode=padding_mode, + dilation=dilation, + groups=groups, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) def forward(self, x): padding = 0 if self._padding_mode != "zeros": - x = F.pad(x, - self._reversed_padding_repeated_twice, - mode=self._padding_mode, - data_format=self._data_format) + x = F.pad( + x, + self._reversed_padding_repeated_twice, + mode=self._padding_mode, + data_format=self._data_format, + ) else: padding = self._padding - out = F.conv1d(x, - self.weight, - bias=self.bias, - padding=padding, - stride=self._stride, - dilation=self._dilation, - groups=self._groups, - data_format=self._data_format) + out = F.conv1d( + x, + self.weight, + bias=self.bias, + padding=padding, + stride=self._stride, + dilation=self._dilation, + groups=self._groups, + data_format=self._data_format, + ) return out @@ -474,43 +514,49 @@ class Conv1DTranspose(_ConvNd): # [[[60. 16. 99. 75. 4.]]] """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - output_padding=0, - groups=1, - dilation=1, - weight_attr=None, - bias_attr=None, - data_format="NCL"): - super(Conv1DTranspose, self).__init__(in_channels, - out_channels, - kernel_size, - True, - 1, - stride=stride, - padding=padding, - dilation=dilation, - output_padding=output_padding, - groups=groups, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + output_padding=0, + groups=1, + dilation=1, + weight_attr=None, + bias_attr=None, + data_format="NCL", + ): + super(Conv1DTranspose, self).__init__( + in_channels, + out_channels, + kernel_size, + True, + 1, + stride=stride, + padding=padding, + dilation=dilation, + output_padding=output_padding, + groups=groups, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) def forward(self, x, output_size=None): - out = F.conv1d_transpose(x, - self.weight, - bias=self.bias, - output_size=output_size, - output_padding=self.output_padding, - padding=self._padding, - stride=self._stride, - dilation=self._dilation, - groups=self._groups, - data_format=self._data_format) + out = F.conv1d_transpose( + x, + self.weight, + bias=self.bias, + output_size=output_size, + output_padding=self.output_padding, + padding=self._padding, + stride=self._stride, + dilation=self._dilation, + groups=self._groups, + data_format=self._data_format, + ) return out @@ -625,51 +671,59 @@ class Conv2D(_ConvNd): # (2, 6, 6, 6) """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - dilation=1, - groups=1, - padding_mode='zeros', - weight_attr=None, - bias_attr=None, - data_format="NCHW"): - super(Conv2D, self).__init__(in_channels, - out_channels, - kernel_size, - False, - 2, - stride=stride, - padding=padding, - padding_mode=padding_mode, - dilation=dilation, - groups=groups, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + padding_mode='zeros', + weight_attr=None, + bias_attr=None, + data_format="NCHW", + ): + super(Conv2D, self).__init__( + in_channels, + out_channels, + kernel_size, + False, + 2, + stride=stride, + padding=padding, + padding_mode=padding_mode, + dilation=dilation, + groups=groups, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) def forward(self, x): if self._padding_mode != 'zeros': - x = F.pad(x, - self._reversed_padding_repeated_twice, - mode=self._padding_mode, - data_format=self._data_format) - - out = F.conv._conv_nd(x, - self.weight, - bias=self.bias, - stride=self._stride, - padding=self._updated_padding, - padding_algorithm=self._padding_algorithm, - dilation=self._dilation, - groups=self._groups, - data_format=self._data_format, - channel_dim=self._channel_dim, - op_type=self._op_type, - use_cudnn=self._use_cudnn) + x = F.pad( + x, + self._reversed_padding_repeated_twice, + mode=self._padding_mode, + data_format=self._data_format, + ) + + out = F.conv._conv_nd( + x, + self.weight, + bias=self.bias, + stride=self._stride, + padding=self._updated_padding, + padding_algorithm=self._padding_algorithm, + dilation=self._dilation, + groups=self._groups, + data_format=self._data_format, + channel_dim=self._channel_dim, + op_type=self._op_type, + use_cudnn=self._use_cudnn, + ) return out @@ -790,31 +844,35 @@ class Conv2DTranspose(_ConvNd): # (2, 6, 10, 10) """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - output_padding=0, - dilation=1, - groups=1, - weight_attr=None, - bias_attr=None, - data_format="NCHW"): - super(Conv2DTranspose, self).__init__(in_channels, - out_channels, - kernel_size, - True, - 2, - stride=stride, - padding=padding, - dilation=dilation, - output_padding=output_padding, - groups=groups, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + output_padding=0, + dilation=1, + groups=1, + weight_attr=None, + bias_attr=None, + data_format="NCHW", + ): + super(Conv2DTranspose, self).__init__( + in_channels, + out_channels, + kernel_size, + True, + 2, + stride=stride, + padding=padding, + dilation=dilation, + output_padding=output_padding, + groups=groups, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) def forward(self, x, output_size=None): if output_size is None: @@ -822,16 +880,18 @@ def forward(self, x, output_size=None): else: output_padding = 0 - out = F.conv2d_transpose(x, - self.weight, - bias=self.bias, - padding=self._padding, - output_padding=output_padding, - stride=self._stride, - dilation=self._dilation, - groups=self._groups, - output_size=output_size, - data_format=self._data_format) + out = F.conv2d_transpose( + x, + self.weight, + bias=self.bias, + padding=self._padding, + output_padding=output_padding, + stride=self._stride, + dilation=self._dilation, + groups=self._groups, + output_size=output_size, + data_format=self._data_format, + ) return out @@ -942,51 +1002,59 @@ class Conv3D(_ConvNd): # (2, 6, 6, 6, 6) """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - dilation=1, - groups=1, - padding_mode='zeros', - weight_attr=None, - bias_attr=None, - data_format="NCDHW"): - super(Conv3D, self).__init__(in_channels, - out_channels, - kernel_size, - False, - 3, - stride=stride, - padding=padding, - padding_mode=padding_mode, - dilation=dilation, - groups=groups, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + padding_mode='zeros', + weight_attr=None, + bias_attr=None, + data_format="NCDHW", + ): + super(Conv3D, self).__init__( + in_channels, + out_channels, + kernel_size, + False, + 3, + stride=stride, + padding=padding, + padding_mode=padding_mode, + dilation=dilation, + groups=groups, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) def forward(self, x): if self._padding_mode != 'zeros': - x = F.pad(x, - self._reversed_padding_repeated_twice, - mode=self._padding_mode, - data_format=self._data_format) - - out = F.conv._conv_nd(x, - self.weight, - bias=self.bias, - stride=self._stride, - padding=self._updated_padding, - padding_algorithm=self._padding_algorithm, - dilation=self._dilation, - groups=self._groups, - data_format=self._data_format, - channel_dim=self._channel_dim, - op_type=self._op_type, - use_cudnn=self._use_cudnn) + x = F.pad( + x, + self._reversed_padding_repeated_twice, + mode=self._padding_mode, + data_format=self._data_format, + ) + + out = F.conv._conv_nd( + x, + self.weight, + bias=self.bias, + stride=self._stride, + padding=self._updated_padding, + padding_algorithm=self._padding_algorithm, + dilation=self._dilation, + groups=self._groups, + data_format=self._data_format, + channel_dim=self._channel_dim, + op_type=self._op_type, + use_cudnn=self._use_cudnn, + ) return out @@ -1116,31 +1184,35 @@ class Conv3DTranspose(_ConvNd): # (2, 6, 10, 10, 10) """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - output_padding=0, - dilation=1, - groups=1, - weight_attr=None, - bias_attr=None, - data_format="NCDHW"): - super(Conv3DTranspose, self).__init__(in_channels, - out_channels, - kernel_size, - True, - 3, - stride=stride, - padding=padding, - dilation=dilation, - output_padding=output_padding, - groups=groups, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + output_padding=0, + dilation=1, + groups=1, + weight_attr=None, + bias_attr=None, + data_format="NCDHW", + ): + super(Conv3DTranspose, self).__init__( + in_channels, + out_channels, + kernel_size, + True, + 3, + stride=stride, + padding=padding, + dilation=dilation, + output_padding=output_padding, + groups=groups, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) def forward(self, x, output_size=None): if output_size is None: @@ -1148,14 +1220,16 @@ def forward(self, x, output_size=None): else: output_padding = 0 - out = F.conv3d_transpose(x, - self.weight, - bias=self.bias, - padding=self._padding, - output_padding=output_padding, - stride=self._stride, - dilation=self._dilation, - groups=self._groups, - output_size=output_size, - data_format=self._data_format) + out = F.conv3d_transpose( + x, + self.weight, + bias=self.bias, + padding=self._padding, + output_padding=output_padding, + stride=self._stride, + dilation=self._dilation, + groups=self._groups, + output_size=output_size, + data_format=self._data_format, + ) return out diff --git a/python/paddle/nn/layer/distance.py b/python/paddle/nn/layer/distance.py index a7a488c833d7f..344c1d482d421 100644 --- a/python/paddle/nn/layer/distance.py +++ b/python/paddle/nn/layer/distance.py @@ -59,7 +59,7 @@ class PairwiseDistance(Layer): """ - def __init__(self, p=2., epsilon=1e-6, keepdim=False, name=None): + def __init__(self, p=2.0, epsilon=1e-6, keepdim=False, name=None): super(PairwiseDistance, self).__init__() self.p = p self.epsilon = epsilon @@ -68,8 +68,9 @@ def __init__(self, p=2., epsilon=1e-6, keepdim=False, name=None): def forward(self, x, y): - return F.pairwise_distance(x, y, self.p, self.epsilon, self.keepdim, - self.name) + return F.pairwise_distance( + x, y, self.p, self.epsilon, self.keepdim, self.name + ) def extra_repr(self): main_str = 'p={p}' diff --git a/python/paddle/nn/layer/loss.py b/python/paddle/nn/layer/loss.py index 6de2717a06165..cd1c8bcb470d3 100644 --- a/python/paddle/nn/layer/loss.py +++ b/python/paddle/nn/layer/loss.py @@ -105,15 +105,14 @@ class BCEWithLogitsLoss(Layer): """ - def __init__(self, - weight=None, - reduction='mean', - pos_weight=None, - name=None): + def __init__( + self, weight=None, reduction='mean', pos_weight=None, name=None + ): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in BCEWithLogitsLoss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) super(BCEWithLogitsLoss, self).__init__() self.weight = weight @@ -123,8 +122,13 @@ def __init__(self, def forward(self, logit, label): out = paddle.nn.functional.binary_cross_entropy_with_logits( - logit, label, self.weight, self.reduction, self.pos_weight, - self.name) + logit, + label, + self.weight, + self.reduction, + self.pos_weight, + self.name, + ) return out @@ -374,14 +378,16 @@ class CrossEntropyLoss(Layer): """ - def __init__(self, - weight=None, - ignore_index=-100, - reduction='mean', - soft_label=False, - axis=-1, - use_softmax=True, - name=None): + def __init__( + self, + weight=None, + ignore_index=-100, + reduction='mean', + soft_label=False, + axis=-1, + use_softmax=True, + name=None, + ): super(CrossEntropyLoss, self).__init__() self.weight = weight self.reduction = reduction @@ -392,15 +398,17 @@ def __init__(self, self.name = name def forward(self, input, label): - ret = paddle.nn.functional.cross_entropy(input, - label, - weight=self.weight, - ignore_index=self.ignore_index, - reduction=self.reduction, - soft_label=self.soft_label, - axis=self.axis, - use_softmax=self.use_softmax, - name=self.name) + ret = paddle.nn.functional.cross_entropy( + input, + label, + weight=self.weight, + ignore_index=self.ignore_index, + reduction=self.reduction, + soft_label=self.soft_label, + axis=self.axis, + use_softmax=self.use_softmax, + name=self.name, + ) return ret @@ -476,18 +484,21 @@ class HSigmoidLoss(Layer): # [2.34564662]] """ - def __init__(self, - feature_size, - num_classes, - weight_attr=None, - bias_attr=None, - is_custom=False, - is_sparse=False, - name=None): + def __init__( + self, + feature_size, + num_classes, + weight_attr=None, + bias_attr=None, + is_custom=False, + is_sparse=False, + name=None, + ): super(HSigmoidLoss, self).__init__() if (num_classes < 2) and (not is_custom): raise ValueError( - "num_classes must not be less than 2 with default tree") + "num_classes must not be less than 2 with default tree" + ) if (not is_custom) and (is_sparse): print("Sparse mode should not be used without custom tree") @@ -505,29 +516,34 @@ def __init__(self, self._dtype = paddle.get_default_dtype() remote_prefetch = is_sparse - print("With sparse mode, if your models has only" - " small parameter prefetch may cause speed down") + print( + "With sparse mode, if your models has only" + " small parameter prefetch may cause speed down" + ) C = self._num_classes if is_custom else self._num_classes - 1 - self.weight = self.create_parameter([C, self._feature_size], - attr=self._weight_attr, - is_bias=False, - dtype=self._dtype) - self.bias = self.create_parameter([C, 1], - attr=self._bias_attr, - is_bias=True, - dtype=self._dtype) + self.weight = self.create_parameter( + [C, self._feature_size], + attr=self._weight_attr, + is_bias=False, + dtype=self._dtype, + ) + self.bias = self.create_parameter( + [C, 1], attr=self._bias_attr, is_bias=True, dtype=self._dtype + ) def forward(self, input, label, path_table=None, path_code=None): - out = F.hsigmoid_loss(input, - label, - self._num_classes, - self.weight, - self.bias, - path_table=path_table, - path_code=path_code, - is_sparse=self._is_sparse, - name=self._name) + out = F.hsigmoid_loss( + input, + label, + self._num_classes, + self.weight, + self.bias, + path_table=path_table, + path_code=path_code, + is_sparse=self._is_sparse, + name=self._name, + ) return out @@ -584,17 +600,18 @@ def __init__(self, reduction='mean'): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'MSELoss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) self.reduction = reduction def forward(self, input, label): if not in_dynamic_mode(): - fluid.data_feeder.check_variable_and_dtype(input, 'input', - ['float32', 'float64'], - 'MSELoss') - fluid.data_feeder.check_variable_and_dtype(label, 'label', - ['float32', 'float64'], - 'MSELoss') + fluid.data_feeder.check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'MSELoss' + ) + fluid.data_feeder.check_variable_and_dtype( + label, 'label', ['float32', 'float64'], 'MSELoss' + ) if in_dygraph_mode(): square_out = paddle._C_ops.square(paddle.subtract(input, label)) @@ -676,16 +693,16 @@ def __init__(self, reduction='mean', name=None): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in L1Loss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) super(L1Loss, self).__init__() self.reduction = reduction self.name = name def forward(self, input, label): - return paddle.nn.functional.l1_loss(input, - label, - self.reduction, - name=self.name) + return paddle.nn.functional.l1_loss( + input, label, self.reduction, name=self.name + ) class BCELoss(Layer): @@ -765,7 +782,8 @@ def __init__(self, weight=None, reduction='mean', name=None): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in bce_loss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) super(BCELoss, self).__init__() self.weight = weight @@ -773,10 +791,9 @@ def __init__(self, weight=None, reduction='mean', name=None): self.name = name def forward(self, input, label): - out = paddle.nn.functional.binary_cross_entropy(input, label, - self.weight, - self.reduction, - self.name) + out = paddle.nn.functional.binary_cross_entropy( + input, label, self.weight, self.reduction, self.name + ) return out @@ -866,15 +883,14 @@ class NLLLoss(Layer): """ - def __init__(self, - weight=None, - ignore_index=-100, - reduction='mean', - name=None): + def __init__( + self, weight=None, ignore_index=-100, reduction='mean', name=None + ): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in nll_loss should be 'sum', 'mean' or " - "'none', but received %s, which is not allowed." % reduction) + "'none', but received %s, which is not allowed." % reduction + ) super(NLLLoss, self).__init__() self._weight = weight self._ignore_index = ignore_index @@ -882,12 +898,14 @@ def __init__(self, self._name = name def forward(self, input, label): - return F.nll_loss(input, - label, - weight=self._weight, - ignore_index=self._ignore_index, - reduction=self._reduction, - name=self._name) + return F.nll_loss( + input, + label, + weight=self._weight, + ignore_index=self._ignore_index, + reduction=self._reduction, + name=self._name, + ) class KLDivLoss(Layer): @@ -1019,17 +1037,17 @@ def __init__(self, margin=0.0, reduction='mean', name=None): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in MarginRankingLoss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) super(MarginRankingLoss, self).__init__() self.margin = margin self.reduction = reduction self.name = name def forward(self, input, other, label): - out = paddle.nn.functional.margin_ranking_loss(input, other, label, - self.margin, - self.reduction, - self.name) + out = paddle.nn.functional.margin_ranking_loss( + input, other, label, self.margin, self.reduction, self.name + ) return out @@ -1113,19 +1131,23 @@ def __init__(self, blank=0, reduction='mean'): self.blank = blank self.reduction = reduction - def forward(self, - log_probs, - labels, - input_lengths, - label_lengths, - norm_by_times=False): - return paddle.nn.functional.ctc_loss(log_probs, - labels, - input_lengths, - label_lengths, - self.blank, - self.reduction, - norm_by_times=norm_by_times) + def forward( + self, + log_probs, + labels, + input_lengths, + label_lengths, + norm_by_times=False, + ): + return paddle.nn.functional.ctc_loss( + log_probs, + labels, + input_lengths, + label_lengths, + self.blank, + self.reduction, + norm_by_times=norm_by_times, + ) class SmoothL1Loss(Layer): @@ -1192,11 +1214,13 @@ def __init__(self, reduction='mean', delta=1.0, name=None): self.name = name def forward(self, input, label): - return F.smooth_l1_loss(input, - label, - reduction=self.reduction, - delta=self.delta, - name=self.name) + return F.smooth_l1_loss( + input, + label, + reduction=self.reduction, + delta=self.delta, + name=self.name, + ) class MultiLabelSoftMarginLoss(Layer): @@ -1264,17 +1288,20 @@ def __init__(self, weight=None, reduction="mean", name=None): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'MultiLabelSoftMarginloss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) self.weight = weight self.reduction = reduction self.name = name def forward(self, input, label): - return F.multi_label_soft_margin_loss(input, - label, - weight=self.weight, - reduction=self.reduction, - name=self.name) + return F.multi_label_soft_margin_loss( + input, + label, + weight=self.weight, + reduction=self.reduction, + name=self.name, + ) class HingeEmbeddingLoss(Layer): @@ -1364,11 +1391,13 @@ def __init__(self, margin=1.0, reduction="mean", name=None): self.name = name def forward(self, input, label): - return F.hinge_embedding_loss(input, - label, - reduction=self.reduction, - margin=self.margin, - name=self.name) + return F.hinge_embedding_loss( + input, + label, + reduction=self.reduction, + margin=self.margin, + name=self.name, + ) class CosineEmbeddingLoss(Layer): @@ -1442,23 +1471,27 @@ def __init__(self, margin=0, reduction='mean', name=None): if margin > 1 or margin < -1: raise ValueError( "The value of 'margin' should be in the interval of [-1, 1], but received %f, which is not allowed." - % margin) + % margin + ) if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' should be 'sum', 'mean' or " - "'none', but received %s, which is not allowed." % reduction) + "'none', but received %s, which is not allowed." % reduction + ) super(CosineEmbeddingLoss, self).__init__() self.margin = margin self.reduction = reduction self.name = name def forward(self, input1, input2, label): - return F.cosine_embedding_loss(input1, - input2, - label, - margin=self.margin, - reduction=self.reduction, - name=self.name) + return F.cosine_embedding_loss( + input1, + input2, + label, + margin=self.margin, + reduction=self.reduction, + name=self.name, + ) class TripletMarginWithDistanceLoss(Layer): @@ -1540,18 +1573,21 @@ class TripletMarginWithDistanceLoss(Layer): """ - def __init__(self, - distance_function=None, - margin=1.0, - swap=False, - reduction: str = 'mean', - name=None): + def __init__( + self, + distance_function=None, + margin=1.0, + swap=False, + reduction: str = 'mean', + name=None, + ): super(TripletMarginWithDistanceLoss, self).__init__() if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in TripletMarginWithDistanceLoss " "should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) self.margin = margin self.swap = swap self.reduction = reduction @@ -1559,13 +1595,15 @@ def __init__(self, self.name = name def forward(self, input, positive, negative): - return F.triplet_margin_with_distance_loss(input, - positive, - negative, - margin=self.margin, - swap=self.swap, - reduction=self.reduction, - name=self.name) + return F.triplet_margin_with_distance_loss( + input, + positive, + negative, + margin=self.margin, + swap=self.swap, + reduction=self.reduction, + name=self.name, + ) class TripletMarginLoss(Layer): @@ -1643,18 +1681,21 @@ class TripletMarginLoss(Layer): """ - def __init__(self, - margin=1.0, - p=2., - epsilon=1e-6, - swap=False, - reduction='mean', - name=None): + def __init__( + self, + margin=1.0, + p=2.0, + epsilon=1e-6, + swap=False, + reduction='mean', + name=None, + ): super(TripletMarginLoss, self).__init__() if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in TripletMarginLoss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) self.margin = margin self.p = p self.epsilon = epsilon @@ -1663,98 +1704,103 @@ def __init__(self, self.name = name def forward(self, input, positive, negative): - return F.triplet_margin_loss(input, - positive, - negative, - margin=self.margin, - p=self.p, - epsilon=self.epsilon, - swap=self.swap, - reduction=self.reduction, - name=self.name) + return F.triplet_margin_loss( + input, + positive, + negative, + margin=self.margin, + p=self.p, + epsilon=self.epsilon, + swap=self.swap, + reduction=self.reduction, + name=self.name, + ) class MultiMarginLoss(Layer): r"""Creates a criterion that optimizes a multi-class classification hinge loss (margin-based loss) between - input :math:`input` and label :math:`label`: + input :math:`input` and label :math:`label`: - For i-th mini-batch sample, the loss in terms of the 1D input :math:`input_i` and scalar - output :math:`label_i` is: + For i-th mini-batch sample, the loss in terms of the 1D input :math:`input_i` and scalar + output :math:`label_i` is: - .. math:: - \text{loss}(input_i, label_i) = \frac{\sum_{j} \max(0, \text{margin} - input_i[label_i] + input_i[j])^p}{\text{C}} + .. math:: + \text{loss}(input_i, label_i) = \frac{\sum_{j} \max(0, \text{margin} - input_i[label_i] + input_i[j])^p}{\text{C}} - where :math:`0 \leq j \leq \text{C}-1`, :math:`0 \leq i \leq \text{N}-1` and :math:`j \neq label_i`. + where :math:`0 \leq j \leq \text{C}-1`, :math:`0 \leq i \leq \text{N}-1` and :math:`j \neq label_i`. - Optionally, you can give non-equal weighting on the classes by passing - a 1D :attr:`weight` tensor into the constructor. + Optionally, you can give non-equal weighting on the classes by passing + a 1D :attr:`weight` tensor into the constructor. - The loss function for i-th sample then becomes: + The loss function for i-th sample then becomes: - .. math:: - \text{loss}(input_i, label_i) = \frac{\sum_{j} \max(0, weight[label_i] * (\text{margin} - input_i[label_i] + input_i[j]))^p}{\text{C}} + .. math:: + \text{loss}(input_i, label_i) = \frac{\sum_{j} \max(0, weight[label_i] * (\text{margin} - input_i[label_i] + input_i[j]))^p}{\text{C}} - Parameters: + Parameters: - p (int, Optional):The norm degree for pairwise distance. Default: :math:`1`. + p (int, Optional):The norm degree for pairwise distance. Default: :math:`1`. - margin (float, Optional):Default: :math:`1`. + margin (float, Optional):Default: :math:`1`. - weight (Tensor,optional): a manual rescaling weight given to each class. - If given, has to be a Tensor of shape (C,) and the data type is float32, float64. - Default is ``'None'`` . + weight (Tensor,optional): a manual rescaling weight given to each class. + If given, has to be a Tensor of shape (C,) and the data type is float32, float64. + Default is ``'None'`` . - reduction (str, optional): Indicate how to calculate the loss by batch_size, - the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. - If :attr:`reduction` is ``'none'``, the unreduced loss is returned; - If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned; - If :attr:`reduction` is ``'sum'``, the summed loss is returned. - Default: ``'mean'`` + reduction (str, optional): Indicate how to calculate the loss by batch_size, + the candidates are ``'none'`` | ``'mean'`` | ``'sum'``. + If :attr:`reduction` is ``'none'``, the unreduced loss is returned; + If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned; + If :attr:`reduction` is ``'sum'``, the summed loss is returned. + Default: ``'mean'`` - name (str, optional): Name for the operation (optional, default is None). - For more information, please refer to :ref:`api_guide_Name`. + name (str, optional): Name for the operation (optional, default is None). + For more information, please refer to :ref:`api_guide_Name`. - Call parameters: - input (Tensor): Input tensor, the data type is float32 or float64. + Call parameters: + input (Tensor): Input tensor, the data type is float32 or float64. - label (Tensor): Label tensor, 0<= label < input.shape[1], the data type is int32 or int64. + label (Tensor): Label tensor, 0<= label < input.shape[1], the data type is int32 or int64. - Shape: - input: 2-D Tensor, the shape is [N, C], N is batch size and `C` means number of classes. + Shape: + input: 2-D Tensor, the shape is [N, C], N is batch size and `C` means number of classes. - label: 1-D Tensor, the shape is [N,]. + label: 1-D Tensor, the shape is [N,]. - output: scalar. If :attr:`reduction` is ``'none'``, then same shape as the label. + output: scalar. If :attr:`reduction` is ``'none'``, then same shape as the label. - Returns: - A callable object of MultiMarginLoss. + Returns: + A callable object of MultiMarginLoss. - Examples: - .. code-block:: python + Examples: + .. code-block:: python - import paddle - import paddle.nn as nn + import paddle + import paddle.nn as nn - input = paddle.to_tensor([[1, -2, 3], [0, -1, 2], [1, 0, 1]], dtype=paddle.float32) - label = paddle.to_tensor([0, 1, 2], dtype=paddle.int32) + input = paddle.to_tensor([[1, -2, 3], [0, -1, 2], [1, 0, 1]], dtype=paddle.float32) + label = paddle.to_tensor([0, 1, 2], dtype=paddle.int32) - multi_margin_loss = nn.MultiMarginLoss(reduction='mean') - loss = multi_margin_loss(input, label) - print(loss) - """ + multi_margin_loss = nn.MultiMarginLoss(reduction='mean') + loss = multi_margin_loss(input, label) + print(loss) + """ - def __init__(self, - p: int = 1, - margin: float = 1.0, - weight=None, - reduction="mean", - name=None): + def __init__( + self, + p: int = 1, + margin: float = 1.0, + weight=None, + reduction="mean", + name=None, + ): super(MultiMarginLoss, self).__init__() if reduction not in ['sum', 'mean', 'none']: raise ValueError( "'reduction' in 'MultiMarginLoss' should be 'sum', 'mean' or 'none', " - "but received {}.".format(reduction)) + "but received {}.".format(reduction) + ) self.p = p self.margin = margin self.weight = weight @@ -1762,13 +1808,15 @@ def __init__(self, self.name = name def forward(self, input, label): - return F.multi_margin_loss(input, - label, - p=self.p, - margin=self.margin, - weight=self.weight, - reduction=self.reduction, - name=self.name) + return F.multi_margin_loss( + input, + label, + p=self.p, + margin=self.margin, + weight=self.weight, + reduction=self.reduction, + name=self.name, + ) class SoftMarginLoss(Layer): @@ -1830,13 +1878,15 @@ def __init__(self, reduction='mean', name=None): if reduction not in ['sum', 'mean', 'none']: raise ValueError( "The value of 'reduction' in SoftMarginLoss should be 'sum', 'mean' or 'none', but " - "received %s, which is not allowed." % reduction) + "received %s, which is not allowed." % reduction + ) super(SoftMarginLoss, self).__init__() self.reduction = reduction self.name = name def forward(self, input, label): - out = paddle.nn.functional.soft_margin_loss(input, label, - self.reduction, self.name) + out = paddle.nn.functional.soft_margin_loss( + input, label, self.reduction, self.name + ) return out diff --git a/python/paddle/nn/layer/norm.py b/python/paddle/nn/layer/norm.py index d011bfd34aa8a..1fca251c57007 100644 --- a/python/paddle/nn/layer/norm.py +++ b/python/paddle/nn/layer/norm.py @@ -59,18 +59,22 @@ class _InstanceNormBase(Layer): See InstaceNorm1D, InstanceNorm2D or InstanceNorm3D for more details. """ - def __init__(self, - num_features, - epsilon=1e-5, - momentum=0.9, - weight_attr=None, - bias_attr=None, - data_format="NCHW", - name=None): + def __init__( + self, + num_features, + epsilon=1e-5, + momentum=0.9, + weight_attr=None, + bias_attr=None, + data_format="NCHW", + name=None, + ): super(_InstanceNormBase, self).__init__() if weight_attr == False or bias_attr == False: - assert weight_attr == bias_attr, "weight_attr and bias_attr must be set to Fasle at the same time in InstanceNorm" + assert ( + weight_attr == bias_attr + ), "weight_attr and bias_attr must be set to Fasle at the same time in InstanceNorm" self._epsilon = epsilon self._weight_attr = weight_attr self._bias_attr = bias_attr @@ -81,11 +85,14 @@ def __init__(self, attr=self._weight_attr, shape=[num_features], default_initializer=Constant(1.0), - is_bias=False) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[num_features], - default_initializer=Constant(0.0), - is_bias=True) + is_bias=False, + ) + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=[num_features], + default_initializer=Constant(0.0), + is_bias=True, + ) else: self.scale = None self.bias = None @@ -96,14 +103,14 @@ def _check_input_dim(self, input): def forward(self, input): self._check_input_dim(input) - return instance_norm(input, - weight=self.scale, - bias=self.bias, - eps=self._epsilon) + return instance_norm( + input, weight=self.scale, bias=self.bias, eps=self._epsilon + ) def extra_repr(self): - return 'num_features={}, epsilon={}'.format(self._num_features, - self._epsilon) + return 'num_features={}, epsilon={}'.format( + self._num_features, self._epsilon + ) class InstanceNorm1D(_InstanceNormBase): @@ -169,8 +176,11 @@ class InstanceNorm1D(_InstanceNormBase): def _check_input_dim(self, input): if len(input.shape) != 2 and len(input.shape) != 3: - raise ValueError('expected 2D or 3D input (got {}D input)'.format( - len(input.shape))) + raise ValueError( + 'expected 2D or 3D input (got {}D input)'.format( + len(input.shape) + ) + ) class InstanceNorm2D(_InstanceNormBase): @@ -235,8 +245,9 @@ class InstanceNorm2D(_InstanceNormBase): def _check_input_dim(self, input): if len(input.shape) != 4: - raise ValueError('expected 4D input (got {}D input)'.format( - len(input.shape))) + raise ValueError( + 'expected 4D input (got {}D input)'.format(len(input.shape)) + ) class InstanceNorm3D(_InstanceNormBase): @@ -301,8 +312,9 @@ class InstanceNorm3D(_InstanceNormBase): def _check_input_dim(self, input): if len(input.shape) != 5: - raise ValueError('expected 5D input (got {}D input)'.format( - len(input.shape))) + raise ValueError( + 'expected 5D input (got {}D input)'.format(len(input.shape)) + ) class GroupNorm(Layer): @@ -349,14 +361,16 @@ class GroupNorm(Layer): print(group_norm_out.numpy()) """ - def __init__(self, - num_groups, - num_channels, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - data_format='NCHW', - name=None): + def __init__( + self, + num_groups, + num_channels, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NCHW', + name=None, + ): super(GroupNorm, self).__init__() self._weight_attr = weight_attr self._bias_attr = bias_attr @@ -370,39 +384,57 @@ def __init__(self, if weight_attr == False: self.weight = self.create_parameter( - attr=None, shape=param_shape, default_initializer=Constant(1.0)) + attr=None, shape=param_shape, default_initializer=Constant(1.0) + ) self.weight.stop_gradient = True else: self.weight = self.create_parameter( attr=self._weight_attr, shape=param_shape, - default_initializer=Constant(1.0)) - self.weight.stop_gradient = self._weight_attr != None and self._weight_attr.learning_rate == 0. + default_initializer=Constant(1.0), + ) + self.weight.stop_gradient = ( + self._weight_attr != None + and self._weight_attr.learning_rate == 0.0 + ) if bias_attr == False: - self.bias = self.create_parameter(attr=None, - shape=param_shape, - default_initializer=Constant(0.0), - is_bias=True) + self.bias = self.create_parameter( + attr=None, + shape=param_shape, + default_initializer=Constant(0.0), + is_bias=True, + ) self.bias.stop_gradient = True else: - self.bias = self.create_parameter(attr=self._bias_attr, - shape=param_shape, - is_bias=True) - self.bias.stop_gradient = self._bias_attr != None and self._bias_attr.learning_rate == 0. + self.bias = self.create_parameter( + attr=self._bias_attr, shape=param_shape, is_bias=True + ) + self.bias.stop_gradient = ( + self._bias_attr != None and self._bias_attr.learning_rate == 0.0 + ) def forward(self, input): mean_out = self._helper.create_variable_for_type_inference( - dtype=input.dtype, stop_gradient=True) + dtype=input.dtype, stop_gradient=True + ) variance_out = self._helper.create_variable_for_type_inference( - dtype=input.dtype, stop_gradient=True) + dtype=input.dtype, stop_gradient=True + ) if in_dygraph_mode(): - pre_act = _C_ops.group_norm(input, self.weight, self.bias, - self._epsilon, self._num_groups, "NCHW") + pre_act = _C_ops.group_norm( + input, + self.weight, + self.bias, + self._epsilon, + self._num_groups, + "NCHW", + ) - return dygraph_utils._append_activation_in_dygraph(pre_act, - act=None) + return dygraph_utils._append_activation_in_dygraph( + pre_act, act=None + ) elif _in_legacy_dygraph(): pre_act, _, _ = _legacy_C_ops.group_norm( @@ -416,8 +448,9 @@ def forward(self, input): 'groups', self._num_groups, ) - return dygraph_utils._append_activation_in_dygraph(pre_act, - act=None) + return dygraph_utils._append_activation_in_dygraph( + pre_act, act=None + ) inputs = {'X': input} if self.bias is not None: @@ -427,25 +460,26 @@ def forward(self, input): # create output group_norm_out = self._helper.create_variable_for_type_inference( - dtype=input.dtype) - - self._helper.append_op(type="group_norm", - inputs=inputs, - outputs={ - "Y": group_norm_out, - "Mean": mean_out, - "Variance": variance_out, - }, - attrs={ - "epsilon": self._epsilon, - "groups": self._num_groups - }) + dtype=input.dtype + ) + + self._helper.append_op( + type="group_norm", + inputs=inputs, + outputs={ + "Y": group_norm_out, + "Mean": mean_out, + "Variance": variance_out, + }, + attrs={"epsilon": self._epsilon, "groups": self._num_groups}, + ) return self._helper.append_activation(group_norm_out, None) def extra_repr(self): return 'num_groups={}, num_channels={}, epsilon={}'.format( - self._num_groups, self._num_channels, self._epsilon) + self._num_groups, self._num_channels, self._epsilon + ) class LayerNorm(Layer): @@ -506,12 +540,14 @@ class LayerNorm(Layer): print(layer_norm_out) """ - def __init__(self, - normalized_shape, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - name=None): + def __init__( + self, + normalized_shape, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + name=None, + ): super(LayerNorm, self).__init__() if isinstance(normalized_shape, numbers.Integral): normalized_shape = [normalized_shape] @@ -528,25 +564,29 @@ def __init__(self, self.weight = self.create_parameter( attr=self._weight_attr, shape=param_shape, - default_initializer=Constant(1.0)) + default_initializer=Constant(1.0), + ) if bias_attr is False: self.bias = None else: - self.bias = self.create_parameter(attr=self._bias_attr, - shape=param_shape, - is_bias=True) + self.bias = self.create_parameter( + attr=self._bias_attr, shape=param_shape, is_bias=True + ) def forward(self, input): - return layer_norm(input, - normalized_shape=self._normalized_shape, - weight=self.weight, - bias=self.bias, - epsilon=self._epsilon) + return layer_norm( + input, + normalized_shape=self._normalized_shape, + weight=self.weight, + bias=self.bias, + epsilon=self._epsilon, + ) def extra_repr(self): - return 'normalized_shape={}, epsilon={}'.format(self._normalized_shape, - self._epsilon) + return 'normalized_shape={}, epsilon={}'.format( + self._normalized_shape, self._epsilon + ) class _BatchNormBase(Layer): @@ -554,15 +594,17 @@ class _BatchNormBase(Layer): BatchNorm base . """ - def __init__(self, - num_features, - momentum=0.9, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - data_format='NCHW', - use_global_stats=None, - name=None): + def __init__( + self, + num_features, + momentum=0.9, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NCHW', + use_global_stats=None, + name=None, + ): super(_BatchNormBase, self).__init__() self._num_features = num_features self._weight_attr = weight_attr @@ -582,29 +624,40 @@ def __init__(self, attr=None, shape=param_shape, dtype=self._dtype, - default_initializer=Constant(1.0)) + default_initializer=Constant(1.0), + ) self.weight.stop_gradient = True else: self.weight = self.create_parameter( attr=self._weight_attr, shape=param_shape, dtype=self._dtype, - default_initializer=Constant(1.0)) - self.weight.stop_gradient = self._weight_attr != None and self._weight_attr.learning_rate == 0. + default_initializer=Constant(1.0), + ) + self.weight.stop_gradient = ( + self._weight_attr != None + and self._weight_attr.learning_rate == 0.0 + ) if bias_attr == False: - self.bias = self.create_parameter(attr=None, - shape=param_shape, - dtype=self._dtype, - default_initializer=Constant(0.0), - is_bias=True) + self.bias = self.create_parameter( + attr=None, + shape=param_shape, + dtype=self._dtype, + default_initializer=Constant(0.0), + is_bias=True, + ) self.bias.stop_gradient = True else: - self.bias = self.create_parameter(attr=self._bias_attr, - shape=param_shape, - dtype=self._dtype, - is_bias=True) - self.bias.stop_gradient = self._bias_attr != None and self._bias_attr.learning_rate == 0. + self.bias = self.create_parameter( + attr=self._bias_attr, + shape=param_shape, + dtype=self._dtype, + is_bias=True, + ) + self.bias.stop_gradient = ( + self._bias_attr != None and self._bias_attr.learning_rate == 0.0 + ) moving_mean_name = None moving_variance_name = None @@ -613,22 +666,28 @@ def __init__(self, moving_mean_name = name + "_mean" moving_variance_name = name + "_variance" - self._mean = self.create_parameter(dtype=self._dtype, - attr=ParamAttr( - name=moving_mean_name, - initializer=Constant(0.0), - trainable=False, - do_model_average=True), - shape=param_shape) + self._mean = self.create_parameter( + dtype=self._dtype, + attr=ParamAttr( + name=moving_mean_name, + initializer=Constant(0.0), + trainable=False, + do_model_average=True, + ), + shape=param_shape, + ) self._mean.stop_gradient = True - self._variance = self.create_parameter(dtype=self._dtype, - attr=ParamAttr( - name=moving_variance_name, - initializer=Constant(1.0), - trainable=False, - do_model_average=True), - shape=param_shape) + self._variance = self.create_parameter( + dtype=self._dtype, + attr=ParamAttr( + name=moving_variance_name, + initializer=Constant(1.0), + trainable=False, + do_model_average=True, + ), + shape=param_shape, + ) self._variance.stop_gradient = True self._data_format = data_format @@ -652,22 +711,26 @@ def forward(self, input): if self.training: warnings.warn( - "When training, we now always track global mean and variance.") - - return batch_norm(input, - self._mean, - self._variance, - weight=self.weight, - bias=self.bias, - training=self.training, - momentum=self._momentum, - epsilon=self._epsilon, - data_format=self._data_format, - use_global_stats=self._use_global_stats) + "When training, we now always track global mean and variance." + ) + + return batch_norm( + input, + self._mean, + self._variance, + weight=self.weight, + bias=self.bias, + training=self.training, + momentum=self._momentum, + epsilon=self._epsilon, + data_format=self._data_format, + use_global_stats=self._use_global_stats, + ) def extra_repr(self): main_str = 'num_features={}, momentum={}, epsilon={}'.format( - self._num_features, self._momentum, self._epsilon) + self._num_features, self._momentum, self._epsilon + ) if self._data_format != 'NCHW': main_str += ', data_format={}'.format(self._data_format) if self._name is not None: @@ -747,18 +810,27 @@ class BatchNorm1D(_BatchNormBase): print(batch_norm_out) """ - def __init__(self, - num_features, - momentum=0.9, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - data_format='NCL', - use_global_stats=None, - name=None): - super(BatchNorm1D, - self).__init__(num_features, momentum, epsilon, weight_attr, - bias_attr, data_format, use_global_stats, name) + def __init__( + self, + num_features, + momentum=0.9, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NCL', + use_global_stats=None, + name=None, + ): + super(BatchNorm1D, self).__init__( + num_features, + momentum, + epsilon, + weight_attr, + bias_attr, + data_format, + use_global_stats, + name, + ) def _check_data_format(self, input): if input == 'NCHW' or input == 'NC' or input == 'NCL': @@ -767,12 +839,16 @@ def _check_data_format(self, input): self._data_format = "NHWC" else: raise ValueError( - 'expected NC , NCL, NLC or None for data_format input') + 'expected NC , NCL, NLC or None for data_format input' + ) def _check_input_dim(self, input): if len(input.shape) != 2 and len(input.shape) != 3: - raise ValueError('expected 2D or 3D input (got {}D input)'.format( - len(input.shape))) + raise ValueError( + 'expected 2D or 3D input (got {}D input)'.format( + len(input.shape) + ) + ) class BatchNorm2D(_BatchNormBase): @@ -856,8 +932,9 @@ def _check_data_format(self, input): def _check_input_dim(self, input): if len(input.shape) != 4: - raise ValueError('expected 4D input (got {}D input)'.format( - len(input.shape))) + raise ValueError( + 'expected 4D input (got {}D input)'.format(len(input.shape)) + ) class BatchNorm3D(_BatchNormBase): @@ -931,18 +1008,27 @@ class BatchNorm3D(_BatchNormBase): print(batch_norm_out) """ - def __init__(self, - num_features, - momentum=0.9, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - data_format='NCDHW', - use_global_stats=None, - name=None): - super(BatchNorm3D, - self).__init__(num_features, momentum, epsilon, weight_attr, - bias_attr, data_format, use_global_stats, name) + def __init__( + self, + num_features, + momentum=0.9, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NCDHW', + use_global_stats=None, + name=None, + ): + super(BatchNorm3D, self).__init__( + num_features, + momentum, + epsilon, + weight_attr, + bias_attr, + data_format, + use_global_stats, + name, + ) def _check_data_format(self, input): if input == 'NCHW' or input == 'NCDHW': @@ -951,12 +1037,14 @@ def _check_data_format(self, input): self._data_format = 'NHWC' else: raise ValueError( - 'expected NCDHW, NDHWC or None for data_format input') + 'expected NCDHW, NDHWC or None for data_format input' + ) def _check_input_dim(self, input): if len(input.shape) != 5: - raise ValueError('expected 5D input (got {}D input)'.format( - len(input.shape))) + raise ValueError( + 'expected 5D input (got {}D input)'.format(len(input.shape)) + ) class SyncBatchNorm(_BatchNormBase): @@ -1046,17 +1134,26 @@ class SyncBatchNorm(_BatchNormBase): # [[[[0.26824948, 1.0936325],[0.26824948, -1.6301316]],[[ 0.8095662, -0.665287],[-1.2744656, 1.1301866 ]]]] """ - def __init__(self, - num_features, - momentum=0.9, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - data_format='NCHW', - name=None): - super(SyncBatchNorm, - self).__init__(num_features, momentum, epsilon, weight_attr, - bias_attr, data_format, None, name) + def __init__( + self, + num_features, + momentum=0.9, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NCHW', + name=None, + ): + super(SyncBatchNorm, self).__init__( + num_features, + momentum, + epsilon, + weight_attr, + bias_attr, + data_format, + None, + name, + ) def _check_data_format(self): if self._data_format in ['NCHW', 'NCDHW', 'NC', 'NCL']: @@ -1080,24 +1177,55 @@ def forward(self, x): ### use_global_stats only support False in sync_batch_norm if in_dygraph_mode(): sync_batch_norm_out, _, _, _, _, _ = _C_ops.sync_batch_norm_( - x, self.weight, self.bias, self._mean, self._variance, - self._momentum, self._epsilon, self._data_format, - not self.training, False, False, False) + x, + self.weight, + self.bias, + self._mean, + self._variance, + self._momentum, + self._epsilon, + self._data_format, + not self.training, + False, + False, + False, + ) return sync_batch_norm_out elif in_dynamic_mode(): - attrs = ("momentum", self._momentum, "epsilon", self._epsilon, - "is_test", not self.training, "data_layout", - self._data_format, "use_mkldnn", False, "fuse_with_relu", - False, "use_global_stats", False, 'trainable_statistics', - False) + attrs = ( + "momentum", + self._momentum, + "epsilon", + self._epsilon, + "is_test", + not self.training, + "data_layout", + self._data_format, + "use_mkldnn", + False, + "fuse_with_relu", + False, + "use_global_stats", + False, + 'trainable_statistics', + False, + ) sync_batch_norm_out, _, _, _, _, _ = _legacy_C_ops.sync_batch_norm( - x, self.weight, self.bias, self._mean, self._variance, mean_out, - variance_out, *attrs) + x, + self.weight, + self.bias, + self._mean, + self._variance, + mean_out, + variance_out, + *attrs + ) return sync_batch_norm_out - check_variable_and_dtype(x, 'input', ['float16', 'float32', 'float64'], - 'SyncBatchNorm') + check_variable_and_dtype( + x, 'input', ['float16', 'float32', 'float64'], 'SyncBatchNorm' + ) attrs = { "momentum": self._momentum, @@ -1115,28 +1243,30 @@ def forward(self, x): "Scale": [self.weight], "Bias": [self.bias], "Mean": [self._mean], - "Variance": [self._variance] + "Variance": [self._variance], } saved_mean = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) saved_variance = self._helper.create_variable_for_type_inference( - dtype=self._dtype, stop_gradient=True) + dtype=self._dtype, stop_gradient=True + ) sync_batch_norm_out = self._helper.create_variable_for_type_inference( - self._dtype) + self._dtype + ) outputs = { "Y": [sync_batch_norm_out], "MeanOut": [mean_out], "VarianceOut": [variance_out], "SavedMean": [saved_mean], - "SavedVariance": [saved_variance] + "SavedVariance": [saved_variance], } - self._helper.append_op(type="sync_batch_norm", - inputs=inputs, - outputs=outputs, - attrs=attrs) + self._helper.append_op( + type="sync_batch_norm", inputs=inputs, outputs=outputs, attrs=attrs + ) return sync_batch_norm_out @classmethod @@ -1162,18 +1292,28 @@ def convert_sync_batchnorm(cls, layer): """ layer_output = layer if isinstance(layer, _BatchNormBase): - if layer._weight_attr != None and not isinstance( - layer._weight_attr, - bool) and layer._weight_attr.name != None: + if ( + layer._weight_attr != None + and not isinstance(layer._weight_attr, bool) + and layer._weight_attr.name != None + ): layer._weight_attr.name = layer._weight_attr.name + '_sync' - if layer._bias_attr != None and not isinstance( - layer._bias_attr, bool) and layer._bias_attr.name != None: + if ( + layer._bias_attr != None + and not isinstance(layer._bias_attr, bool) + and layer._bias_attr.name != None + ): layer._bias_attr.name = layer._bias_attr.name + '_sync' - layer_output = SyncBatchNorm(layer._num_features, layer._momentum, - layer._epsilon, layer._weight_attr, - layer._bias_attr, layer._data_format, - layer._name) + layer_output = SyncBatchNorm( + layer._num_features, + layer._momentum, + layer._epsilon, + layer._weight_attr, + layer._bias_attr, + layer._data_format, + layer._name, + ) if layer._weight_attr != False and layer._bias_attr != False: with no_grad(): @@ -1183,58 +1323,61 @@ def convert_sync_batchnorm(cls, layer): layer_output._variance = layer._variance for name, sublayer in layer.named_children(): - layer_output.add_sublayer(name, - cls.convert_sync_batchnorm(sublayer)) + layer_output.add_sublayer( + name, cls.convert_sync_batchnorm(sublayer) + ) del layer return layer_output class LocalResponseNorm(Layer): """ - Local Response Normalization performs a type of "lateral inhibition" by normalizing over local input regions. - For more information, please refer to `ImageNet Classification with Deep Convolutional Neural Networks `_ + Local Response Normalization performs a type of "lateral inhibition" by normalizing over local input regions. + For more information, please refer to `ImageNet Classification with Deep Convolutional Neural Networks `_ - See more details in :ref:`api_paddle_nn_functional_local_response_norm` . + See more details in :ref:`api_paddle_nn_functional_local_response_norm` . - Parameters: - size (int): The number of channels to sum over. - alpha (float, optional): The scaling parameter, positive. Default:1e-4 - beta (float, optional): The exponent, positive. Default:0.75 - k (float, optional): An offset, positive. Default: 1.0 - data_format (str, optional): Specify the data format of the input, and the data format of the output - will be consistent with that of the input. An optional string from: - If input is 3-D Tensor, the string could be `"NCL"` or `"NLC"` . When it is `"NCL"`, - the data is stored in the order of: `[batch_size, input_channels, feature_length]`. - If input is 4-D Tensor, the string could be `"NCHW"`, `"NHWC"`. When it is `"NCHW"`, - the data is stored in the order of: `[batch_size, input_channels, input_height, input_width]`. - If input is 5-D Tensor, the string could be `"NCDHW"`, `"NDHWC"` . When it is `"NCDHW"`, - the data is stored in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`. - name (str, optional): Name for the operation (optional, default is None). For more information, - please refer to :ref:`api_guide_Name`. - - Shape: - - input: 3-D/4-D/5-D tensor. - - output: 3-D/4-D/5-D tensor, the same shape as input. + Parameters: + size (int): The number of channels to sum over. + alpha (float, optional): The scaling parameter, positive. Default:1e-4 + beta (float, optional): The exponent, positive. Default:0.75 + k (float, optional): An offset, positive. Default: 1.0 + data_format (str, optional): Specify the data format of the input, and the data format of the output + will be consistent with that of the input. An optional string from: + If input is 3-D Tensor, the string could be `"NCL"` or `"NLC"` . When it is `"NCL"`, + the data is stored in the order of: `[batch_size, input_channels, feature_length]`. + If input is 4-D Tensor, the string could be `"NCHW"`, `"NHWC"`. When it is `"NCHW"`, + the data is stored in the order of: `[batch_size, input_channels, input_height, input_width]`. + If input is 5-D Tensor, the string could be `"NCDHW"`, `"NDHWC"` . When it is `"NCDHW"`, + the data is stored in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`. + name (str, optional): Name for the operation (optional, default is None). For more information, + please refer to :ref:`api_guide_Name`. - Examples: + Shape: + - input: 3-D/4-D/5-D tensor. + - output: 3-D/4-D/5-D tensor, the same shape as input. - .. code-block:: python + Examples: - import paddle + .. code-block:: python - x = paddle.rand(shape=(3, 3, 112, 112), dtype="float32") - m = paddle.nn.LocalResponseNorm(size=5) - y = m(x) - print(y.shape) # [3, 3, 112, 112] - """ + import paddle + + x = paddle.rand(shape=(3, 3, 112, 112), dtype="float32") + m = paddle.nn.LocalResponseNorm(size=5) + y = m(x) + print(y.shape) # [3, 3, 112, 112] + """ - def __init__(self, - size, - alpha=0.0001, - beta=0.75, - k=1.0, - data_format="NCHW", - name=None): + def __init__( + self, + size, + alpha=0.0001, + beta=0.75, + k=1.0, + data_format="NCHW", + name=None, + ): super(LocalResponseNorm, self).__init__() self.size = size self.alpha = alpha @@ -1244,13 +1387,21 @@ def __init__(self, self.name = name def forward(self, input): - out = F.local_response_norm(input, self.size, self.alpha, self.beta, - self.k, self.data_format, self.name) + out = F.local_response_norm( + input, + self.size, + self.alpha, + self.beta, + self.k, + self.data_format, + self.name, + ) return out def extra_repr(self): main_str = 'size={}, alpha={}, beta={}, k={}'.format( - self.size, self.alpha, self.beta, self.k) + self.size, self.alpha, self.beta, self.k + ) if self.data_format != 'NCHW': main_str += ', data_format={}'.format(self.data_format) if self.name is not None: diff --git a/python/paddle/nn/layer/pooling.py b/python/paddle/nn/layer/pooling.py index 3e47275410d09..a9b5af5199faf 100755 --- a/python/paddle/nn/layer/pooling.py +++ b/python/paddle/nn/layer/pooling.py @@ -76,13 +76,15 @@ class AvgPool1D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - exclusive=True, - ceil_mode=False, - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + exclusive=True, + ceil_mode=False, + name=None, + ): super(AvgPool1D, self).__init__() self.kernel_size = kernel_size self.stride = stride @@ -92,13 +94,21 @@ def __init__(self, self.name = name def forward(self, x): - out = F.avg_pool1d(x, self.kernel_size, self.stride, self.padding, - self.exclusive, self.ceil_mode, self.name) + out = F.avg_pool1d( + x, + self.kernel_size, + self.stride, + self.padding, + self.exclusive, + self.ceil_mode, + self.name, + ) return out def extra_repr(self): return 'kernel_size={kernel_size}, stride={stride}, padding={padding}'.format( - **self.__dict__) + **self.__dict__ + ) class AvgPool2D(Layer): @@ -173,15 +183,17 @@ class AvgPool2D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - ceil_mode=False, - exclusive=True, - divisor_override=None, - data_format="NCHW", - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + ceil_mode=False, + exclusive=True, + divisor_override=None, + data_format="NCHW", + name=None, + ): super(AvgPool2D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -193,19 +205,22 @@ def __init__(self, self.name = name def forward(self, x): - return F.avg_pool2d(x, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - ceil_mode=self.ceil_mode, - exclusive=self.exclusive, - divisor_override=self.divisor, - data_format=self.data_format, - name=self.name) + return F.avg_pool2d( + x, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + ceil_mode=self.ceil_mode, + exclusive=self.exclusive, + divisor_override=self.divisor, + data_format=self.data_format, + name=self.name, + ) def extra_repr(self): return 'kernel_size={ksize}, stride={stride}, padding={padding}'.format( - **self.__dict__) + **self.__dict__ + ) class AvgPool3D(Layer): @@ -266,15 +281,17 @@ class AvgPool3D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - ceil_mode=False, - exclusive=True, - divisor_override=None, - data_format="NCDHW", - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + ceil_mode=False, + exclusive=True, + divisor_override=None, + data_format="NCDHW", + name=None, + ): super(AvgPool3D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -286,19 +303,22 @@ def __init__(self, self.name = name def forward(self, x): - return F.avg_pool3d(x, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - ceil_mode=self.ceil_mode, - exclusive=self.exclusive, - divisor_override=self.divisor, - data_format=self.data_format, - name=self.name) + return F.avg_pool3d( + x, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + ceil_mode=self.ceil_mode, + exclusive=self.exclusive, + divisor_override=self.divisor, + data_format=self.data_format, + name=self.name, + ) def extra_repr(self): return 'kernel_size={ksize}, stride={stride}, padding={padding}'.format( - **self.__dict__) + **self.__dict__ + ) class MaxPool1D(Layer): @@ -362,13 +382,15 @@ class MaxPool1D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - return_mask=False, - ceil_mode=False, - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + return_mask=False, + ceil_mode=False, + name=None, + ): super(MaxPool1D, self).__init__() self.kernel_size = kernel_size self.stride = stride @@ -378,13 +400,21 @@ def __init__(self, self.name = name def forward(self, input): - out = F.max_pool1d(input, self.kernel_size, self.stride, self.padding, - self.return_mask, self.ceil_mode, self.name) + out = F.max_pool1d( + input, + self.kernel_size, + self.stride, + self.padding, + self.return_mask, + self.ceil_mode, + self.name, + ) return out def extra_repr(self): return 'kernel_size={kernel_size}, stride={stride}, padding={padding}'.format( - **self.__dict__) + **self.__dict__ + ) class MaxPool2D(Layer): @@ -460,14 +490,16 @@ class MaxPool2D(Layer): # output.shape [1, 3, 16, 16], max_indices.shape [1, 3, 16, 16], """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - return_mask=False, - ceil_mode=False, - data_format="NCHW", - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + return_mask=False, + ceil_mode=False, + data_format="NCHW", + name=None, + ): super(MaxPool2D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -478,18 +510,21 @@ def __init__(self, self.name = name def forward(self, x): - return F.max_pool2d(x, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - return_mask=self.return_mask, - ceil_mode=self.ceil_mode, - data_format=self.data_format, - name=self.name) + return F.max_pool2d( + x, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + return_mask=self.return_mask, + ceil_mode=self.ceil_mode, + data_format=self.data_format, + name=self.name, + ) def extra_repr(self): return 'kernel_size={ksize}, stride={stride}, padding={padding}'.format( - **self.__dict__) + **self.__dict__ + ) class MaxPool3D(Layer): @@ -553,14 +588,16 @@ class MaxPool3D(Layer): # output.shape [1, 2, 3, 16, 16], max_indices.shape [1, 2, 3, 16, 16], """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - return_mask=False, - ceil_mode=False, - data_format="NCDHW", - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + return_mask=False, + ceil_mode=False, + data_format="NCDHW", + name=None, + ): super(MaxPool3D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -571,18 +608,21 @@ def __init__(self, self.name = name def forward(self, x): - return F.max_pool3d(x, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - return_mask=self.return_mask, - ceil_mode=self.ceil_mode, - data_format=self.data_format, - name=self.name) + return F.max_pool3d( + x, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + return_mask=self.return_mask, + ceil_mode=self.ceil_mode, + data_format=self.data_format, + name=self.name, + ) def extra_repr(self): return 'kernel_size={ksize}, stride={stride}, padding={padding}'.format( - **self.__dict__) + **self.__dict__ + ) class AdaptiveAvgPool1D(Layer): @@ -724,10 +764,12 @@ def __init__(self, output_size, data_format="NCHW", name=None): self._name = name def forward(self, x): - return F.adaptive_avg_pool2d(x, - output_size=self._output_size, - data_format=self._data_format, - name=self._name) + return F.adaptive_avg_pool2d( + x, + output_size=self._output_size, + data_format=self._data_format, + name=self._name, + ) def extra_repr(self): return 'output_size={}'.format(self._output_size) @@ -816,10 +858,12 @@ def __init__(self, output_size, data_format="NCDHW", name=None): self._name = name def forward(self, x): - return F.adaptive_avg_pool3d(x, - output_size=self._output_size, - data_format=self._data_format, - name=self._name) + return F.adaptive_avg_pool3d( + x, + output_size=self._output_size, + data_format=self._data_format, + name=self._name, + ) def extra_repr(self): return 'output_size={}'.format(self._output_size) @@ -898,12 +942,14 @@ def __init__(self, output_size, return_mask=False, name=None): self.name = name def forward(self, input): - return F.adaptive_max_pool1d(input, self.output_size, self.return_mask, - self.name) + return F.adaptive_max_pool1d( + input, self.output_size, self.return_mask, self.name + ) def extra_repr(self): - return 'output_size={}, return_mask={}'.format(self.output_size, - self.return_mask) + return 'output_size={}, return_mask={}'.format( + self.output_size, self.return_mask + ) class AdaptiveMaxPool2D(Layer): @@ -976,14 +1022,17 @@ def __init__(self, output_size, return_mask=False, name=None): self._name = name def forward(self, x): - return F.adaptive_max_pool2d(x, - output_size=self._output_size, - return_mask=self._return_mask, - name=self._name) + return F.adaptive_max_pool2d( + x, + output_size=self._output_size, + return_mask=self._return_mask, + name=self._name, + ) def extra_repr(self): - return 'output_size={}, return_mask={}'.format(self._output_size, - self._return_mask) + return 'output_size={}, return_mask={}'.format( + self._output_size, self._return_mask + ) class AdaptiveMaxPool3D(Layer): @@ -1068,14 +1117,17 @@ def __init__(self, output_size, return_mask=False, name=None): self._name = name def forward(self, x): - return F.adaptive_max_pool3d(x, - output_size=self._output_size, - return_mask=self._return_mask, - name=self._name) + return F.adaptive_max_pool3d( + x, + output_size=self._output_size, + return_mask=self._return_mask, + name=self._name, + ) def extra_repr(self): - return 'output_size={}, return_mask={}'.format(self._output_size, - self._return_mask) + return 'output_size={}, return_mask={}'.format( + self._output_size, self._return_mask + ) class MaxUnPool1D(Layer): @@ -1130,13 +1182,15 @@ class MaxUnPool1D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - data_format="NCL", - output_size=None, - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + data_format="NCL", + output_size=None, + name=None, + ): super(MaxUnPool1D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -1146,14 +1200,16 @@ def __init__(self, self.name = name def forward(self, x, indices): - return F.max_unpool1d(x, - indices, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - data_format=self.data_format, - output_size=self.output_size, - name=self.name) + return F.max_unpool1d( + x, + indices, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + data_format=self.data_format, + output_size=self.output_size, + name=self.name, + ) def extra_repr(self): return 'output_size={}'.format(self.output_size) @@ -1214,13 +1270,15 @@ class MaxUnPool2D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - data_format="NCHW", - output_size=None, - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + data_format="NCHW", + output_size=None, + name=None, + ): super(MaxUnPool2D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -1230,14 +1288,16 @@ def __init__(self, self.name = name def forward(self, x, indices): - return F.max_unpool2d(x, - indices, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - data_format=self.data_format, - output_size=self.output_size, - name=self.name) + return F.max_unpool2d( + x, + indices, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + data_format=self.data_format, + output_size=self.output_size, + name=self.name, + ) def extra_repr(self): return 'output_size={}'.format(self.output_size) @@ -1302,13 +1362,15 @@ class MaxUnPool3D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - data_format="NCDHW", - output_size=None, - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + data_format="NCDHW", + output_size=None, + name=None, + ): super(MaxUnPool3D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -1318,14 +1380,16 @@ def __init__(self, self.name = name def forward(self, x, indices): - return F.max_unpool3d(x, - indices, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - data_format=self.data_format, - output_size=self.output_size, - name=self.name) + return F.max_unpool3d( + x, + indices, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + data_format=self.data_format, + output_size=self.output_size, + name=self.name, + ) def extra_repr(self): return 'output_size={}'.format(self.output_size) diff --git a/python/paddle/nn/layer/rnn.py b/python/paddle/nn/layer/rnn.py index 2821277c8f84a..7e0fa6d7d703d 100644 --- a/python/paddle/nn/layer/rnn.py +++ b/python/paddle/nn/layer/rnn.py @@ -29,6 +29,7 @@ from paddle.framework import core from paddle.static import default_startup_program from paddle.static import program_guard + try: from collections.abc import Sequence except: @@ -139,12 +140,9 @@ class RNNCellBase(Layer): and mostly used in RNN. """ - def get_initial_states(self, - batch_ref, - shape=None, - dtype=None, - init_value=0., - batch_dim_idx=0): + def get_initial_states( + self, batch_ref, shape=None, dtype=None, init_value=0.0, batch_dim_idx=0 + ): r""" Generate initialized states according to provided shape, data type and value. @@ -180,17 +178,17 @@ def get_initial_states(self, def _is_shape_sequence(seq): """For shape, list/tuple of integer is the finest-grained objection""" - if (isinstance(seq, list) or isinstance(seq, tuple)): - if reduce(lambda flag, x: isinstance(x, int) and flag, seq, - True): + if isinstance(seq, list) or isinstance(seq, tuple): + if reduce( + lambda flag, x: isinstance(x, int) and flag, seq, True + ): return False # TODO: Add check for the illegal if isinstance(seq, dict): return True - return (isinstance(seq, Sequence) and not isinstance(seq, str)) + return isinstance(seq, Sequence) and not isinstance(seq, str) class Shape(object): - def __init__(self, shape): self.shape = shape if shape[0] == -1 else ([-1] + list(shape)) @@ -211,13 +209,16 @@ def __init__(self, shape): states_dtypes = map_structure(lambda shape: dtype, states_shapes) init_states = map_structure( - lambda shape, dtype: paddle.fluid.layers. - fill_constant_batch_size_like(input=batch_ref, - shape=shape.shape, - dtype=dtype, - value=init_value, - input_dim_idx=batch_dim_idx), - states_shapes, states_dtypes) + lambda shape, dtype: paddle.fluid.layers.fill_constant_batch_size_like( + input=batch_ref, + shape=shape.shape, + dtype=dtype, + value=init_value, + input_dim_idx=batch_dim_idx, + ), + states_shapes, + states_dtypes, + ) return init_states @property @@ -233,7 +234,8 @@ def state_shape(self): `get_initial_states`. """ raise NotImplementedError( - "Please add implementaion for `state_shape` in the used cell.") + "Please add implementaion for `state_shape` in the used cell." + ) @property def state_dtype(self): @@ -248,7 +250,8 @@ def state_dtype(self): `get_initial_states`. """ raise NotImplementedError( - "Please add implementaion for `state_dtype` in the used cell.") + "Please add implementaion for `state_dtype` in the used cell." + ) class SimpleRNNCell(RNNCellBase): @@ -318,50 +321,57 @@ class SimpleRNNCell(RNNCellBase): """ - def __init__(self, - input_size, - hidden_size, - activation="tanh", - weight_ih_attr=None, - weight_hh_attr=None, - bias_ih_attr=None, - bias_hh_attr=None, - name=None): + def __init__( + self, + input_size, + hidden_size, + activation="tanh", + weight_ih_attr=None, + weight_hh_attr=None, + bias_ih_attr=None, + bias_hh_attr=None, + name=None, + ): super(SimpleRNNCell, self).__init__() if hidden_size <= 0: raise ValueError( - "hidden_size of {} must be greater than 0, but now equals to {}" - .format(self.__class__.__name__, hidden_size)) + "hidden_size of {} must be greater than 0, but now equals to {}".format( + self.__class__.__name__, hidden_size + ) + ) std = 1.0 / math.sqrt(hidden_size) self.weight_ih = self.create_parameter( (hidden_size, input_size), weight_ih_attr, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.weight_hh = self.create_parameter( (hidden_size, hidden_size), weight_hh_attr, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.bias_ih = self.create_parameter( - (hidden_size, ), + (hidden_size,), bias_ih_attr, is_bias=True, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.bias_hh = self.create_parameter( - (hidden_size, ), + (hidden_size,), bias_hh_attr, is_bias=True, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.input_size = input_size self.hidden_size = hidden_size if activation not in ["tanh", "relu"]: raise ValueError( "activation for SimpleRNNCell should be tanh or relu, " - "but get {}".format(activation)) + "but get {}".format(activation) + ) self.activation = activation - self._activation_fn = paddle.tanh \ - if activation == "tanh" \ - else F.relu + self._activation_fn = paddle.tanh if activation == "tanh" else F.relu def forward(self, inputs, states=None): if states is None: @@ -378,7 +388,7 @@ def forward(self, inputs, states=None): @property def state_shape(self): - return (self.hidden_size, ) + return (self.hidden_size,) def extra_repr(self): s = '{input_size}, {hidden_size}' @@ -471,38 +481,46 @@ class LSTMCell(RNNCellBase): """ - def __init__(self, - input_size, - hidden_size, - weight_ih_attr=None, - weight_hh_attr=None, - bias_ih_attr=None, - bias_hh_attr=None, - name=None): + def __init__( + self, + input_size, + hidden_size, + weight_ih_attr=None, + weight_hh_attr=None, + bias_ih_attr=None, + bias_hh_attr=None, + name=None, + ): super(LSTMCell, self).__init__() if hidden_size <= 0: raise ValueError( - "hidden_size of {} must be greater than 0, but now equals to {}" - .format(self.__class__.__name__, hidden_size)) + "hidden_size of {} must be greater than 0, but now equals to {}".format( + self.__class__.__name__, hidden_size + ) + ) std = 1.0 / math.sqrt(hidden_size) self.weight_ih = self.create_parameter( (4 * hidden_size, input_size), weight_ih_attr, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.weight_hh = self.create_parameter( (4 * hidden_size, hidden_size), weight_hh_attr, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.bias_ih = self.create_parameter( - (4 * hidden_size, ), + (4 * hidden_size,), bias_ih_attr, is_bias=True, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.bias_hh = self.create_parameter( - (4 * hidden_size, ), + (4 * hidden_size,), bias_hh_attr, is_bias=True, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.hidden_size = hidden_size self.input_size = input_size @@ -538,7 +556,7 @@ def state_shape(self): automatically inserted into shape). These two shapes correspond to :math:`h_{t-1}` and :math:`c_{t-1}` separately. """ - return ((self.hidden_size, ), (self.hidden_size, )) + return ((self.hidden_size,), (self.hidden_size,)) def extra_repr(self): return '{input_size}, {hidden_size}'.format(**self.__dict__) @@ -622,38 +640,46 @@ class GRUCell(RNNCellBase): """ - def __init__(self, - input_size, - hidden_size, - weight_ih_attr=None, - weight_hh_attr=None, - bias_ih_attr=None, - bias_hh_attr=None, - name=None): + def __init__( + self, + input_size, + hidden_size, + weight_ih_attr=None, + weight_hh_attr=None, + bias_ih_attr=None, + bias_hh_attr=None, + name=None, + ): super(GRUCell, self).__init__() if hidden_size <= 0: raise ValueError( - "hidden_size of {} must be greater than 0, but now equals to {}" - .format(self.__class__.__name__, hidden_size)) + "hidden_size of {} must be greater than 0, but now equals to {}".format( + self.__class__.__name__, hidden_size + ) + ) std = 1.0 / math.sqrt(hidden_size) self.weight_ih = self.create_parameter( (3 * hidden_size, input_size), weight_ih_attr, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.weight_hh = self.create_parameter( (3 * hidden_size, hidden_size), weight_hh_attr, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.bias_ih = self.create_parameter( - (3 * hidden_size, ), + (3 * hidden_size,), bias_ih_attr, is_bias=True, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.bias_hh = self.create_parameter( - (3 * hidden_size, ), + (3 * hidden_size,), bias_hh_attr, is_bias=True, - default_initializer=I.Uniform(-std, std)) + default_initializer=I.Uniform(-std, std), + ) self.hidden_size = hidden_size self.input_size = input_size @@ -689,7 +715,7 @@ def state_shape(self): size would be automatically inserted into shape). The shape corresponds to the shape of :math:`h_{t-1}`. """ - return (self.hidden_size, ) + return (self.hidden_size,) def extra_repr(self): return '{input_size}, {hidden_size}'.format(**self.__dict__) @@ -754,11 +780,9 @@ def __init__(self, cell, is_reverse=False, time_major=False): self.is_reverse = is_reverse self.time_major = time_major - def forward(self, - inputs, - initial_states=None, - sequence_length=None, - **kwargs): + def forward( + self, inputs, initial_states=None, sequence_length=None, **kwargs + ): final_outputs, final_states = paddle.fluid.layers.rnn( self.cell, inputs, @@ -766,7 +790,8 @@ def forward(self, sequence_length=sequence_length, time_major=self.time_major, is_reverse=self.is_reverse, - **kwargs) + **kwargs + ) return final_outputs, final_states @@ -825,27 +850,35 @@ def __init__(self, cell_fw, cell_bw, time_major=False): self.cell_fw = cell_fw self.cell_bw = cell_bw if cell_fw.input_size != cell_bw.input_size: - raise ValueError("input size of forward cell({}) does not equals" - "that of backward cell({})".format( - cell_fw.input_size, cell_bw.input_size)) + raise ValueError( + "input size of forward cell({}) does not equals" + "that of backward cell({})".format( + cell_fw.input_size, cell_bw.input_size + ) + ) for cell in [self.cell_fw, self.cell_bw]: if not hasattr(cell, "call"): # for non-dygraph mode, `rnn` api uses cell.call cell.call = cell.forward self.time_major = time_major - def forward(self, - inputs, - initial_states=None, - sequence_length=None, - **kwargs): + def forward( + self, inputs, initial_states=None, sequence_length=None, **kwargs + ): if isinstance(initial_states, (list, tuple)): - assert len(initial_states) == 2, \ - "length of initial_states should be 2 when it is a list/tuple" + assert ( + len(initial_states) == 2 + ), "length of initial_states should be 2 when it is a list/tuple" outputs, final_states = paddle.fluid.layers.birnn( - self.cell_fw, self.cell_bw, inputs, initial_states, sequence_length, - self.time_major, **kwargs) + self.cell_fw, + self.cell_bw, + inputs, + initial_states, + sequence_length, + self.time_major, + **kwargs + ) return outputs, final_states @@ -855,18 +888,20 @@ class RNNBase(LayerList): and other common methods for SimpleRNN, LSTM and GRU. """ - def __init__(self, - mode, - input_size, - hidden_size, - num_layers=1, - direction="forward", - time_major=False, - dropout=0., - weight_ih_attr=None, - weight_hh_attr=None, - bias_ih_attr=None, - bias_hh_attr=None): + def __init__( + self, + mode, + input_size, + hidden_size, + num_layers=1, + direction="forward", + time_major=False, + dropout=0.0, + weight_ih_attr=None, + weight_hh_attr=None, + bias_ih_attr=None, + bias_hh_attr=None, + ): super(RNNBase, self).__init__() bidirectional_list = ["bidirectional", "bidirect"] self.mode = mode @@ -882,7 +917,7 @@ def __init__(self, "weight_ih_attr": weight_ih_attr, "weight_hh_attr": weight_hh_attr, "bias_ih_attr": bias_ih_attr, - "bias_hh_attr": bias_hh_attr + "bias_hh_attr": bias_hh_attr, } if mode == "LSTM": @@ -911,11 +946,13 @@ def __init__(self, else: raise ValueError( "direction should be forward or bidirect (or bidirectional), " - "received direction = {}".format(direction)) + "received direction = {}".format(direction) + ) self.could_use_cudnn = True self.could_use_cudnn &= len(self.parameters()) == num_layers * 4 * ( - 2 if direction in bidirectional_list else 1) + 2 if direction in bidirectional_list else 1 + ) # Expose params as RNN's attribute, which can make it compatible when # replacing small ops composed rnn with cpp rnn kernel. @@ -927,8 +964,10 @@ def __init__(self, for direction in range(self.num_directions): suffix = '_reverse' if direction == 1 else '' param_names.extend(['weight_ih_l{}{}', 'weight_hh_l{}{}']) - if bias_ih_attr != False: param_names.append('bias_ih_l{}{}') - if bias_hh_attr != False: param_names.append('bias_hh_l{}{}') + if bias_ih_attr != False: + param_names.append('bias_ih_l{}{}') + if bias_hh_attr != False: + param_names.append('bias_hh_l{}{}') param_names = [x.format(layer, suffix) for x in param_names] for name, param in zip(param_names, self.parameters()): setattr(self, name, param) @@ -949,8 +988,11 @@ def flatten_parameters(self): shape = [np.prod(param.shape) for param in params] self._all_weights = [None] * len(params) for i, param in enumerate(params): - offset = 0 if i % 4 < 2 else (2 * self.num_layers * - self.num_directions) + offset = ( + 0 + if i % 4 < 2 + else (2 * self.num_layers * self.num_directions) + ) layer_idx = i // 4 self._all_weights[offset + layer_idx * 2 + i % 2] = param # Wrap using a list to avoid registed into params and saving, maybe @@ -958,38 +1000,49 @@ def flatten_parameters(self): # add both to main_program and startup_program for static-graph. # Use Constant initializer to avoid make effect on random generator. self._flat_weight = [ - self.create_parameter(shape=[np.sum(shape)], - dtype=params[0].dtype, - default_initializer=I.Constant(0.0)) + self.create_parameter( + shape=[np.sum(shape)], + dtype=params[0].dtype, + default_initializer=I.Constant(0.0), + ) ] # dropout state may also can be hided and avoid saving # should dropout state be persistable for static-graph self._dropout_state = self.create_variable( - dtype=core.VarDesc.VarType.UINT8) + dtype=core.VarDesc.VarType.UINT8 + ) if in_dynamic_mode(): with paddle.no_grad(): - _legacy_C_ops.coalesce_tensor(self._all_weights, - self._all_weights, - self._flat_weight[0], - "copy_data", True, - "use_align", False, "dtype", - params[0].dtype) + _legacy_C_ops.coalesce_tensor( + self._all_weights, + self._all_weights, + self._flat_weight[0], + "copy_data", + True, + "use_align", + False, + "dtype", + params[0].dtype, + ) return # for static-graph, append coalesce_tensor into startup program - with program_guard(default_startup_program(), - default_startup_program()): + with program_guard( + default_startup_program(), default_startup_program() + ): with paddle.no_grad(): - self._helper.append_op(type="coalesce_tensor", - inputs={"Input": self._all_weights}, - outputs={ - "Output": self._all_weights, - "FusedOutput": self._flat_weight - }, - attrs={ - "copy_data": True, - "use_align": False, - "dtype": params[0].dtype - }) + self._helper.append_op( + type="coalesce_tensor", + inputs={"Input": self._all_weights}, + outputs={ + "Output": self._all_weights, + "FusedOutput": self._flat_weight, + }, + attrs={ + "copy_data": True, + "use_align": False, + "dtype": params[0].dtype, + }, + ) def _cudnn_impl(self, inputs, initial_states, sequence_length): if not self.time_major: @@ -997,18 +1050,43 @@ def _cudnn_impl(self, inputs, initial_states, sequence_length): if in_dygraph_mode(): out, _, state = _C_ops.rnn( - inputs, initial_states, self._all_weights, sequence_length, - self._dropout_state, self.dropout, self.num_directions == 2, - self.input_size, self.hidden_size, self.num_layers, self.mode, - 0, not self.training) + inputs, + initial_states, + self._all_weights, + sequence_length, + self._dropout_state, + self.dropout, + self.num_directions == 2, + self.input_size, + self.hidden_size, + self.num_layers, + self.mode, + 0, + not self.training, + ) elif in_dynamic_mode(): _, _, out, state = _legacy_C_ops.rnn( - inputs, initial_states, self._all_weights, sequence_length, - self._dropout_state, self.state_components, 'dropout_prob', - self.dropout, 'is_bidirec', self.num_directions == 2, - 'input_size', self.input_size, 'hidden_size', self.hidden_size, - 'num_layers', self.num_layers, 'mode', self.mode, 'is_test', - not self.training) + inputs, + initial_states, + self._all_weights, + sequence_length, + self._dropout_state, + self.state_components, + 'dropout_prob', + self.dropout, + 'is_bidirec', + self.num_directions == 2, + 'input_size', + self.input_size, + 'hidden_size', + self.hidden_size, + 'num_layers', + self.num_layers, + 'mode', + self.mode, + 'is_test', + not self.training, + ) else: out = self._helper.create_variable_for_type_inference(inputs.dtype) state = [ @@ -1016,13 +1094,14 @@ def _cudnn_impl(self, inputs, initial_states, sequence_length): for i in range(self.state_components) ] reserve = self._helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.UINT8, stop_gradient=True) + dtype=core.VarDesc.VarType.UINT8, stop_gradient=True + ) inputs = { 'Input': inputs, 'WeightList': self._all_weights, 'PreState': initial_states, - 'SequenceLength': sequence_length + 'SequenceLength': sequence_length, } attrs = { 'dropout_prob': self.dropout, @@ -1031,7 +1110,7 @@ def _cudnn_impl(self, inputs, initial_states, sequence_length): 'hidden_size': self.hidden_size, 'num_layers': self.num_layers, 'mode': self.mode, - 'is_test': not self.training + 'is_test': not self.training, } outputs = { @@ -1041,51 +1120,67 @@ def _cudnn_impl(self, inputs, initial_states, sequence_length): 'DropoutState': self._dropout_state, } - self._helper.append_op(type="rnn", - inputs=inputs, - outputs=outputs, - attrs=attrs) + self._helper.append_op( + type="rnn", inputs=inputs, outputs=outputs, attrs=attrs + ) - out = paddle.tensor.transpose(out, - [1, 0, 2]) if not self.time_major else out + out = ( + paddle.tensor.transpose(out, [1, 0, 2]) + if not self.time_major + else out + ) return out, tuple(state) if len(state) > 1 else state[0] def forward(self, inputs, initial_states=None, sequence_length=None): batch_index = 1 if self.time_major else 0 dtype = inputs.dtype if initial_states is None: - state_shape = (self.num_layers * self.num_directions, -1, - self.hidden_size) - initial_states = tuple([ - paddle.fluid.layers.fill_constant_batch_size_like( - inputs, state_shape, dtype, 0, batch_index, 1) - for _ in range(self.state_components) - ]) + state_shape = ( + self.num_layers * self.num_directions, + -1, + self.hidden_size, + ) + initial_states = tuple( + [ + paddle.fluid.layers.fill_constant_batch_size_like( + inputs, state_shape, dtype, 0, batch_index, 1 + ) + for _ in range(self.state_components) + ] + ) else: - initial_states = [initial_states] if isinstance( - initial_states, paddle.static.Variable) else initial_states - - if self.could_use_cudnn and (not paddle.device.is_compiled_with_rocm() - or sequence_length is None): + initial_states = ( + [initial_states] + if isinstance(initial_states, paddle.static.Variable) + else initial_states + ) + + if self.could_use_cudnn and ( + not paddle.device.is_compiled_with_rocm() or sequence_length is None + ): # Add CPU kernel and dispatch in backend later return self._cudnn_impl(inputs, initial_states, sequence_length) - states = split_states(initial_states, self.num_directions == 2, - self.state_components) + states = split_states( + initial_states, self.num_directions == 2, self.state_components + ) final_states = [] for i, rnn_layer in enumerate(self): if i > 0: - inputs = F.dropout(inputs, - self.dropout, - training=self.training, - mode="upscale_in_train") + inputs = F.dropout( + inputs, + self.dropout, + training=self.training, + mode="upscale_in_train", + ) outputs, final_state = rnn_layer(inputs, states[i], sequence_length) final_states.append(final_state) inputs = outputs - final_states = concat_states(final_states, self.num_directions == 2, - self.state_components) + final_states = concat_states( + final_states, self.num_directions == 2, self.state_components + ) return outputs, final_states def extra_repr(self): @@ -1184,19 +1279,21 @@ class SimpleRNN(RNNBase): """ - def __init__(self, - input_size, - hidden_size, - num_layers=1, - direction="forward", - time_major=False, - dropout=0., - activation="tanh", - weight_ih_attr=None, - weight_hh_attr=None, - bias_ih_attr=None, - bias_hh_attr=None, - name=None): + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + direction="forward", + time_major=False, + dropout=0.0, + activation="tanh", + weight_ih_attr=None, + weight_hh_attr=None, + bias_ih_attr=None, + bias_hh_attr=None, + name=None, + ): if activation == "tanh": mode = "RNN_TANH" elif activation == "relu": @@ -1204,10 +1301,19 @@ def __init__(self, else: raise ValueError("Unknown activation '{}'".format(activation)) self.activation = activation - super(SimpleRNN, - self).__init__(mode, input_size, hidden_size, num_layers, - direction, time_major, dropout, weight_ih_attr, - weight_hh_attr, bias_ih_attr, bias_hh_attr) + super(SimpleRNN, self).__init__( + mode, + input_size, + hidden_size, + num_layers, + direction, + time_major, + dropout, + weight_ih_attr, + weight_hh_attr, + bias_ih_attr, + bias_hh_attr, + ) class LSTM(RNNBase): @@ -1307,22 +1413,33 @@ class LSTM(RNNBase): """ - def __init__(self, - input_size, - hidden_size, - num_layers=1, - direction="forward", - time_major=False, - dropout=0., - weight_ih_attr=None, - weight_hh_attr=None, - bias_ih_attr=None, - bias_hh_attr=None, - name=None): - super(LSTM, - self).__init__("LSTM", input_size, hidden_size, num_layers, - direction, time_major, dropout, weight_ih_attr, - weight_hh_attr, bias_ih_attr, bias_hh_attr) + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + direction="forward", + time_major=False, + dropout=0.0, + weight_ih_attr=None, + weight_hh_attr=None, + bias_ih_attr=None, + bias_hh_attr=None, + name=None, + ): + super(LSTM, self).__init__( + "LSTM", + input_size, + hidden_size, + num_layers, + direction, + time_major, + dropout, + weight_ih_attr, + weight_hh_attr, + bias_ih_attr, + bias_hh_attr, + ) class GRU(RNNBase): @@ -1415,19 +1532,30 @@ class GRU(RNNBase): """ - def __init__(self, - input_size, - hidden_size, - num_layers=1, - direction="forward", - time_major=False, - dropout=0., - weight_ih_attr=None, - weight_hh_attr=None, - bias_ih_attr=None, - bias_hh_attr=None, - name=None): - super(GRU, - self).__init__("GRU", input_size, hidden_size, num_layers, - direction, time_major, dropout, weight_ih_attr, - weight_hh_attr, bias_ih_attr, bias_hh_attr) + def __init__( + self, + input_size, + hidden_size, + num_layers=1, + direction="forward", + time_major=False, + dropout=0.0, + weight_ih_attr=None, + weight_hh_attr=None, + bias_ih_attr=None, + bias_hh_attr=None, + name=None, + ): + super(GRU, self).__init__( + "GRU", + input_size, + hidden_size, + num_layers, + direction, + time_major, + dropout, + weight_ih_attr, + weight_hh_attr, + bias_ih_attr, + bias_hh_attr, + ) diff --git a/python/paddle/nn/layer/transformer.py b/python/paddle/nn/layer/transformer.py index 0ea83b3d84ce4..4115cbc7a8071 100644 --- a/python/paddle/nn/layer/transformer.py +++ b/python/paddle/nn/layer/transformer.py @@ -49,7 +49,8 @@ def _convert_param_attr_to_list(param_attr, n): """ if isinstance(param_attr, (list, tuple)): assert len(param_attr) == n, ( - "length of param_attr should be %d when it is a list/tuple" % n) + "length of param_attr should be %d when it is a list/tuple" % n + ) param_attrs = [] for attr in param_attr: if isinstance(attr, bool): @@ -151,21 +152,27 @@ class MultiHeadAttention(Layer): Cache = collections.namedtuple("Cache", ["k", "v"]) StaticCache = collections.namedtuple("StaticCache", ["k", "v"]) - def __init__(self, - embed_dim, - num_heads, - dropout=0., - kdim=None, - vdim=None, - need_weights=False, - weight_attr=None, - bias_attr=None): + def __init__( + self, + embed_dim, + num_heads, + dropout=0.0, + kdim=None, + vdim=None, + need_weights=False, + weight_attr=None, + bias_attr=None, + ): super(MultiHeadAttention, self).__init__() - assert embed_dim > 0, ("Expected embed_dim to be greater than 0, " - "but received {}".format(embed_dim)) - assert num_heads > 0, ("Expected num_heads to be greater than 0, " - "but received {}".format(num_heads)) + assert embed_dim > 0, ( + "Expected embed_dim to be greater than 0, " + "but received {}".format(embed_dim) + ) + assert num_heads > 0, ( + "Expected num_heads to be greater than 0, " + "but received {}".format(num_heads) + ) self.embed_dim = embed_dim self.kdim = kdim if kdim is not None else embed_dim @@ -175,24 +182,22 @@ def __init__(self, self.need_weights = need_weights self.head_dim = embed_dim // num_heads - assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads" - - self.q_proj = Linear(embed_dim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.k_proj = Linear(self.kdim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.v_proj = Linear(self.vdim, - embed_dim, - weight_attr, - bias_attr=bias_attr) - self.out_proj = Linear(embed_dim, - embed_dim, - weight_attr, - bias_attr=bias_attr) + assert ( + self.head_dim * num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" + + self.q_proj = Linear( + embed_dim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.k_proj = Linear( + self.kdim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.v_proj = Linear( + self.vdim, embed_dim, weight_attr, bias_attr=bias_attr + ) + self.out_proj = Linear( + embed_dim, embed_dim, weight_attr, bias_attr=bias_attr + ) def _prepare_qkv(self, query, key, value, cache=None): r""" @@ -338,12 +343,14 @@ def gen_cache(self, key, value=None, type=Cache): input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) v = layers.fill_constant_batch_size_like( input=key, shape=[-1, self.num_heads, 0, self.head_dim], dtype=key.dtype, - value=0) + value=0, + ) return self.Cache(k, v) else: # incremental_state with initial value, mainly for usage like UniLM @@ -410,19 +417,21 @@ def forward(self, query, key=None, value=None, attn_mask=None, cache=None): q, k, v, cache = self._prepare_qkv(query, key, value, cache) # scale dot product attention - product = paddle.matmul(x=q * (self.head_dim**-0.5), - y=k, - transpose_y=True) + product = paddle.matmul( + x=q * (self.head_dim**-0.5), y=k, transpose_y=True + ) if attn_mask is not None: # Support bool or int mask attn_mask = _convert_attention_mask(attn_mask, product.dtype) product = product + attn_mask weights = F.softmax(product) if self.dropout: - weights = F.dropout(weights, - self.dropout, - training=self.training, - mode="upscale_in_train") + weights = F.dropout( + weights, + self.dropout, + training=self.training, + mode="upscale_in_train", + ) out = tensor.matmul(weights, v) @@ -498,30 +507,39 @@ class TransformerEncoderLayer(Layer): enc_output = encoder_layer(enc_input, attn_mask) # [2, 4, 128] """ - def __init__(self, - d_model, - nhead, - dim_feedforward, - dropout=0.1, - activation="relu", - attn_dropout=None, - act_dropout=None, - normalize_before=False, - weight_attr=None, - bias_attr=None): + def __init__( + self, + d_model, + nhead, + dim_feedforward, + dropout=0.1, + activation="relu", + attn_dropout=None, + act_dropout=None, + normalize_before=False, + weight_attr=None, + bias_attr=None, + ): self._config = locals() self._config.pop("self") self._config.pop("__class__", None) # py3 super(TransformerEncoderLayer, self).__init__() - assert d_model > 0, ("Expected d_model to be greater than 0, " - "but received {}".format(d_model)) - assert nhead > 0, ("Expected nhead to be greater than 0, " - "but received {}".format(nhead)) + assert ( + d_model > 0 + ), "Expected d_model to be greater than 0, " "but received {}".format( + d_model + ) + assert ( + nhead > 0 + ), "Expected nhead to be greater than 0, " "but received {}".format( + nhead + ) assert dim_feedforward > 0, ( "Expected dim_feedforward to be greater than 0, " - "but received {}".format(dim_feedforward)) + "but received {}".format(dim_feedforward) + ) attn_dropout = dropout if attn_dropout is None else attn_dropout act_dropout = dropout if act_dropout is None else act_dropout @@ -530,20 +548,20 @@ def __init__(self, weight_attrs = _convert_param_attr_to_list(weight_attr, 2) bias_attrs = _convert_param_attr_to_list(bias_attr, 2) - self.self_attn = MultiHeadAttention(d_model, - nhead, - dropout=attn_dropout, - weight_attr=weight_attrs[0], - bias_attr=bias_attrs[0]) - self.linear1 = Linear(d_model, - dim_feedforward, - weight_attrs[1], - bias_attr=bias_attrs[1]) + self.self_attn = MultiHeadAttention( + d_model, + nhead, + dropout=attn_dropout, + weight_attr=weight_attrs[0], + bias_attr=bias_attrs[0], + ) + self.linear1 = Linear( + d_model, dim_feedforward, weight_attrs[1], bias_attr=bias_attrs[1] + ) self.dropout = Dropout(act_dropout, mode="upscale_in_train") - self.linear2 = Linear(dim_feedforward, - d_model, - weight_attrs[1], - bias_attr=bias_attrs[1]) + self.linear2 = Linear( + dim_feedforward, d_model, weight_attrs[1], bias_attr=bias_attrs[1] + ) self.norm1 = LayerNorm(d_model) self.norm2 = LayerNorm(d_model) self.dropout1 = Dropout(dropout, mode="upscale_in_train") @@ -591,8 +609,9 @@ def forward(self, src, src_mask=None, cache=None): if cache is None: src = self.self_attn(src, src, src, src_mask) else: - src, incremental_cache = self.self_attn(src, src, src, src_mask, - cache) + src, incremental_cache = self.self_attn( + src, src, src, src_mask, cache + ) src = residual + self.dropout1(src) if not self.normalize_before: @@ -624,8 +643,9 @@ def gen_cache(self, src): `MultiHeadAttention.gen_cache` and `MultiHeadAttention.forward` \ for more details. """ - incremental_cache = self.self_attn.gen_cache(src, - type=self.self_attn.Cache) + incremental_cache = self.self_attn.gen_cache( + src, type=self.self_attn.Cache + ) return incremental_cache @@ -659,10 +679,16 @@ class TransformerEncoder(Layer): def __init__(self, encoder_layer, num_layers, norm=None): super(TransformerEncoder, self).__init__() - self.layers = LayerList([ - (encoder_layer if i == 0 else type(encoder_layer)( - **encoder_layer._config)) for i in range(num_layers) - ]) + self.layers = LayerList( + [ + ( + encoder_layer + if i == 0 + else type(encoder_layer)(**encoder_layer._config) + ) + for i in range(num_layers) + ] + ) self.num_layers = num_layers self.norm = norm @@ -708,9 +734,9 @@ def forward(self, src, src_mask=None, cache=None): if cache is None: output = mod(output, src_mask=src_mask) else: - output, new_cache = mod(output, - src_mask=src_mask, - cache=cache[i]) + output, new_cache = mod( + output, src_mask=src_mask, cache=cache[i] + ) new_caches.append(new_cache) if self.norm is not None: @@ -807,30 +833,39 @@ class TransformerDecoderLayer(Layer): cross_attn_mask) # [2, 4, 128] """ - def __init__(self, - d_model, - nhead, - dim_feedforward, - dropout=0.1, - activation="relu", - attn_dropout=None, - act_dropout=None, - normalize_before=False, - weight_attr=None, - bias_attr=None): + def __init__( + self, + d_model, + nhead, + dim_feedforward, + dropout=0.1, + activation="relu", + attn_dropout=None, + act_dropout=None, + normalize_before=False, + weight_attr=None, + bias_attr=None, + ): self._config = locals() self._config.pop("self") self._config.pop("__class__", None) # py3 super(TransformerDecoderLayer, self).__init__() - assert d_model > 0, ("Expected d_model to be greater than 0, " - "but received {}".format(d_model)) - assert nhead > 0, ("Expected nhead to be greater than 0, " - "but received {}".format(nhead)) + assert ( + d_model > 0 + ), "Expected d_model to be greater than 0, " "but received {}".format( + d_model + ) + assert ( + nhead > 0 + ), "Expected nhead to be greater than 0, " "but received {}".format( + nhead + ) assert dim_feedforward > 0, ( "Expected dim_feedforward to be greater than 0, " - "but received {}".format(dim_feedforward)) + "but received {}".format(dim_feedforward) + ) attn_dropout = dropout if attn_dropout is None else attn_dropout act_dropout = dropout if act_dropout is None else act_dropout @@ -839,25 +874,27 @@ def __init__(self, weight_attrs = _convert_param_attr_to_list(weight_attr, 3) bias_attrs = _convert_param_attr_to_list(bias_attr, 3) - self.self_attn = MultiHeadAttention(d_model, - nhead, - dropout=attn_dropout, - weight_attr=weight_attrs[0], - bias_attr=bias_attrs[0]) - self.cross_attn = MultiHeadAttention(d_model, - nhead, - dropout=attn_dropout, - weight_attr=weight_attrs[1], - bias_attr=bias_attrs[1]) - self.linear1 = Linear(d_model, - dim_feedforward, - weight_attrs[2], - bias_attr=bias_attrs[2]) + self.self_attn = MultiHeadAttention( + d_model, + nhead, + dropout=attn_dropout, + weight_attr=weight_attrs[0], + bias_attr=bias_attrs[0], + ) + self.cross_attn = MultiHeadAttention( + d_model, + nhead, + dropout=attn_dropout, + weight_attr=weight_attrs[1], + bias_attr=bias_attrs[1], + ) + self.linear1 = Linear( + d_model, dim_feedforward, weight_attrs[2], bias_attr=bias_attrs[2] + ) self.dropout = Dropout(act_dropout, mode="upscale_in_train") - self.linear2 = Linear(dim_feedforward, - d_model, - weight_attrs[2], - bias_attr=bias_attrs[2]) + self.linear2 = Linear( + dim_feedforward, d_model, weight_attrs[2], bias_attr=bias_attrs[2] + ) self.norm1 = LayerNorm(d_model) self.norm2 = LayerNorm(d_model) self.norm3 = LayerNorm(d_model) @@ -922,8 +959,9 @@ def forward(self, tgt, memory, tgt_mask=None, memory_mask=None, cache=None): if cache is None: tgt = self.self_attn(tgt, tgt, tgt, tgt_mask, None) else: - tgt, incremental_cache = self.self_attn(tgt, tgt, tgt, tgt_mask, - cache[0]) + tgt, incremental_cache = self.self_attn( + tgt, tgt, tgt, tgt_mask, cache[0] + ) tgt = residual + self.dropout1(tgt) if not self.normalize_before: tgt = self.norm1(tgt) @@ -934,8 +972,9 @@ def forward(self, tgt, memory, tgt_mask=None, memory_mask=None, cache=None): if cache is None: tgt = self.cross_attn(tgt, memory, memory, memory_mask, None) else: - tgt, static_cache = self.cross_attn(tgt, memory, memory, - memory_mask, cache[1]) + tgt, static_cache = self.cross_attn( + tgt, memory, memory, memory_mask, cache[1] + ) tgt = residual + self.dropout2(tgt) if not self.normalize_before: tgt = self.norm2(tgt) @@ -947,8 +986,9 @@ def forward(self, tgt, memory, tgt_mask=None, memory_mask=None, cache=None): tgt = residual + self.dropout3(tgt) if not self.normalize_before: tgt = self.norm3(tgt) - return tgt if cache is None else (tgt, (incremental_cache, - static_cache)) + return ( + tgt if cache is None else (tgt, (incremental_cache, static_cache)) + ) def gen_cache(self, memory): r""" @@ -972,10 +1012,12 @@ def gen_cache(self, memory): See `MultiHeadAttention.gen_cache` and `MultiHeadAttention.forward` \ for more details. """ - incremental_cache = self.self_attn.gen_cache(memory, - type=self.self_attn.Cache) + incremental_cache = self.self_attn.gen_cache( + memory, type=self.self_attn.Cache + ) static_cache = self.cross_attn.gen_cache( - memory, memory, type=self.cross_attn.StaticCache) + memory, memory, type=self.cross_attn.StaticCache + ) return incremental_cache, static_cache @@ -1016,10 +1058,16 @@ class TransformerDecoder(Layer): def __init__(self, decoder_layer, num_layers, norm=None): super(TransformerDecoder, self).__init__() - self.layers = LayerList([ - (decoder_layer if i == 0 else type(decoder_layer)( - **decoder_layer._config)) for i in range(num_layers) - ]) + self.layers = LayerList( + [ + ( + decoder_layer + if i == 0 + else type(decoder_layer)(**decoder_layer._config) + ) + for i in range(num_layers) + ] + ) self.num_layers = num_layers self.norm = norm @@ -1077,17 +1125,21 @@ def forward(self, tgt, memory, tgt_mask=None, memory_mask=None, cache=None): new_caches = [] for i, mod in enumerate(self.layers): if cache is None: - output = mod(output, - memory, - tgt_mask=tgt_mask, - memory_mask=memory_mask, - cache=None) + output = mod( + output, + memory, + tgt_mask=tgt_mask, + memory_mask=memory_mask, + cache=None, + ) else: - output, new_cache = mod(output, - memory, - tgt_mask=tgt_mask, - memory_mask=memory_mask, - cache=cache[i]) + output, new_cache = mod( + output, + memory, + tgt_mask=tgt_mask, + memory_mask=memory_mask, + cache=cache[i], + ) new_caches.append(new_cache) if self.norm is not None: @@ -1217,30 +1269,39 @@ class Transformer(Layer): cross_attn_mask) # [2, 6, 128] """ - def __init__(self, - d_model=512, - nhead=8, - num_encoder_layers=6, - num_decoder_layers=6, - dim_feedforward=2048, - dropout=0.1, - activation="relu", - attn_dropout=None, - act_dropout=None, - normalize_before=False, - weight_attr=None, - bias_attr=None, - custom_encoder=None, - custom_decoder=None): + def __init__( + self, + d_model=512, + nhead=8, + num_encoder_layers=6, + num_decoder_layers=6, + dim_feedforward=2048, + dropout=0.1, + activation="relu", + attn_dropout=None, + act_dropout=None, + normalize_before=False, + weight_attr=None, + bias_attr=None, + custom_encoder=None, + custom_decoder=None, + ): super(Transformer, self).__init__() - assert d_model > 0, ("Expected d_model to be greater than 0, " - "but received {}".format(d_model)) - assert nhead > 0, ("Expected nhead to be greater than 0, " - "but received {}".format(nhead)) + assert ( + d_model > 0 + ), "Expected d_model to be greater than 0, " "but received {}".format( + d_model + ) + assert ( + nhead > 0 + ), "Expected nhead to be greater than 0, " "but received {}".format( + nhead + ) assert dim_feedforward > 0, ( "Expected dim_feedforward to be greater than 0, " - "but received {}".format(dim_feedforward)) + "but received {}".format(dim_feedforward) + ) if isinstance(bias_attr, (list, tuple)): if len(bias_attr) == 1: @@ -1253,9 +1314,9 @@ def __init__(self, encoder_bias_attr = [bias_attr[0], bias_attr[-1]] decoder_bias_attr = bias_attr else: - assert False, ( - "length of bias_attr should be 1 or 2 or 3 when it is a list/tuple" - ) + assert ( + False + ), "length of bias_attr should be 1 or 2 or 3 when it is a list/tuple" else: encoder_bias_attr = bias_attr decoder_bias_attr = bias_attr @@ -1267,15 +1328,17 @@ def __init__(self, elif len(weight_attr) == 2: encoder_weight_attr = weight_attr decoder_weight_attr = [ - weight_attr[0], weight_attr[0], weight_attr[-1] + weight_attr[0], + weight_attr[0], + weight_attr[-1], ] elif len(weight_attr) == 3: encoder_weight_attr = [weight_attr[0], weight_attr[-1]] decoder_weight_attr = weight_attr else: - assert False, ( - "length of weight_attr should be 1 or 2 or 3 when it is a list/tuple" - ) + assert ( + False + ), "length of weight_attr should be 1 or 2 or 3 when it is a list/tuple" else: encoder_weight_attr = weight_attr decoder_weight_attr = weight_attr @@ -1284,23 +1347,41 @@ def __init__(self, self.encoder = custom_encoder else: encoder_layer = TransformerEncoderLayer( - d_model, nhead, dim_feedforward, dropout, activation, - attn_dropout, act_dropout, normalize_before, - encoder_weight_attr, encoder_bias_attr) + d_model, + nhead, + dim_feedforward, + dropout, + activation, + attn_dropout, + act_dropout, + normalize_before, + encoder_weight_attr, + encoder_bias_attr, + ) encoder_norm = LayerNorm(d_model) - self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers, - encoder_norm) + self.encoder = TransformerEncoder( + encoder_layer, num_encoder_layers, encoder_norm + ) if custom_decoder is not None: self.decoder = custom_decoder else: decoder_layer = TransformerDecoderLayer( - d_model, nhead, dim_feedforward, dropout, activation, - attn_dropout, act_dropout, normalize_before, - decoder_weight_attr, decoder_bias_attr) + d_model, + nhead, + dim_feedforward, + dropout, + activation, + attn_dropout, + act_dropout, + normalize_before, + decoder_weight_attr, + decoder_bias_attr, + ) decoder_norm = LayerNorm(d_model) - self.decoder = TransformerDecoder(decoder_layer, num_decoder_layers, - decoder_norm) + self.decoder = TransformerDecoder( + decoder_layer, num_decoder_layers, decoder_norm + ) self.d_model = d_model self.nhead = nhead @@ -1359,10 +1440,9 @@ def forward(self, src, tgt, src_mask=None, tgt_mask=None, memory_mask=None): tgt_mask = _convert_attention_mask(tgt_mask, tgt.dtype) memory_mask = _convert_attention_mask(memory_mask, memory.dtype) - output = self.decoder(tgt, - memory, - tgt_mask=tgt_mask, - memory_mask=memory_mask) + output = self.decoder( + tgt, memory, tgt_mask=tgt_mask, memory_mask=memory_mask + ) return output def generate_square_subsequent_mask(self, length): @@ -1397,6 +1477,10 @@ def generate_square_subsequent_mask(self, length): """ return paddle.tensor.triu( - paddle.full(shape=[length, length], - fill_value=-np.inf, - dtype=paddle.get_default_dtype()), 1) + paddle.full( + shape=[length, length], + fill_value=-np.inf, + dtype=paddle.get_default_dtype(), + ), + 1, + ) diff --git a/python/paddle/nn/layer/vision.py b/python/paddle/nn/layer/vision.py index fe367828815c5..704baeb98f9cf 100644 --- a/python/paddle/nn/layer/vision.py +++ b/python/paddle/nn/layer/vision.py @@ -67,16 +67,19 @@ def __init__(self, upscale_factor, data_format="NCHW", name=None): raise TypeError("upscale factor must be int type") if data_format not in ["NCHW", "NHWC"]: - raise ValueError("Data format should be 'NCHW' or 'NHWC'." - "But recevie data format: {}".format(data_format)) + raise ValueError( + "Data format should be 'NCHW' or 'NHWC'." + "But recevie data format: {}".format(data_format) + ) self._upscale_factor = upscale_factor self._data_format = data_format self._name = name def forward(self, x): - return functional.pixel_shuffle(x, self._upscale_factor, - self._data_format, self._name) + return functional.pixel_shuffle( + x, self._upscale_factor, self._data_format, self._name + ) def extra_repr(self): main_str = 'upscale_factor={}'.format(self._upscale_factor) @@ -130,16 +133,19 @@ def __init__(self, downscale_factor, data_format="NCHW", name=None): raise ValueError("Downscale factor must be positive") if data_format not in ["NCHW", "NHWC"]: - raise ValueError("Data format should be 'NCHW' or 'NHWC'." - "But recevie data format: {}".format(data_format)) + raise ValueError( + "Data format should be 'NCHW' or 'NHWC'." + "But recevie data format: {}".format(data_format) + ) self._downscale_factor = downscale_factor self._data_format = data_format self._name = name def forward(self, x): - return functional.pixel_unshuffle(x, self._downscale_factor, - self._data_format, self._name) + return functional.pixel_unshuffle( + x, self._downscale_factor, self._data_format, self._name + ) def extra_repr(self): main_str = 'downscale_factor={}'.format(self._downscale_factor) @@ -202,16 +208,19 @@ def __init__(self, groups, data_format="NCHW", name=None): raise ValueError("groups must be positive") if data_format not in ["NCHW", "NHWC"]: - raise ValueError("Data format should be 'NCHW' or 'NHWC'." - "But recevie data format: {}".format(data_format)) + raise ValueError( + "Data format should be 'NCHW' or 'NHWC'." + "But recevie data format: {}".format(data_format) + ) self._groups = groups self._data_format = data_format self._name = name def forward(self, x): - return functional.channel_shuffle(x, self._groups, self._data_format, - self._name) + return functional.channel_shuffle( + x, self._groups, self._data_format, self._name + ) def extra_repr(self): main_str = 'groups={}'.format(self._groups) diff --git a/python/paddle/nn/quant/functional_layers.py b/python/paddle/nn/quant/functional_layers.py index ca1eb5f4fb3c1..2c0eb88e0875c 100644 --- a/python/paddle/nn/quant/functional_layers.py +++ b/python/paddle/nn/quant/functional_layers.py @@ -19,13 +19,11 @@ class FloatFunctionalLayer(Layer): - def __init__(self): super(FloatFunctionalLayer, self).__init__() class add(FloatFunctionalLayer): - def __init__(self): super(add, self).__init__() @@ -34,7 +32,6 @@ def forward(self, x, y, name=None): class subtract(FloatFunctionalLayer): - def __init__(self): super(subtract, self).__init__() @@ -43,7 +40,6 @@ def forward(self, x, y, name=None): class multiply(FloatFunctionalLayer): - def __init__(self): super(multiply, self).__init__() @@ -52,7 +48,6 @@ def forward(self, x, y, name=None): class divide(FloatFunctionalLayer): - def __init__(self): super(divide, self).__init__() @@ -61,7 +56,6 @@ def forward(self, x, y, name=None): class reshape(FloatFunctionalLayer): - def __init__(self): super(reshape, self).__init__() @@ -70,7 +64,6 @@ def forward(self, x, shape, name=None): class transpose(FloatFunctionalLayer): - def __init__(self): super(transpose, self).__init__() @@ -79,7 +72,6 @@ def forward(self, x, perm, name=None): class concat(FloatFunctionalLayer): - def __init__(self): super(concat, self).__init__() @@ -88,7 +80,6 @@ def forward(self, x, axis=0, name=None): class flatten(FloatFunctionalLayer): - def __init__(self): super(flatten, self).__init__() diff --git a/python/paddle/nn/quant/lsq.py b/python/paddle/nn/quant/lsq.py index 6ed6e78a6d98e..ff2c3d12807ed 100644 --- a/python/paddle/nn/quant/lsq.py +++ b/python/paddle/nn/quant/lsq.py @@ -28,7 +28,6 @@ def round(x): class LsqFunc(PyLayer): - @staticmethod def forward(ctx, weight, alpha, g, Qn, Qp, per_channel=False, quant_axis=0): ctx.save_for_backward(weight, alpha) @@ -65,21 +64,39 @@ def backward(ctx, grad_weight): upper_flag = paddle.cast((q_w > Qp), 'float32') middle_flag = 1.0 - lower_flag - upper_flag if per_channel: - grad_alpha = ((lower_flag * Qn + upper_flag * Qp + - middle_flag * round(q_w) - middle_flag * q_w) * - grad_weight * g) + grad_alpha = ( + ( + lower_flag * Qn + + upper_flag * Qp + + middle_flag * round(q_w) + - middle_flag * q_w + ) + * grad_weight + * g + ) grad_alpha = grad_alpha.reshape( - (grad_alpha.shape[quant_axis], -1)).sum(axis=1) + (grad_alpha.shape[quant_axis], -1) + ).sum(axis=1) else: - grad_alpha = ((lower_flag * Qn + upper_flag * Qp + - middle_flag * round(q_w) - middle_flag * q_w) * - grad_weight * g).sum().unsqueeze(axis=0)[0] + grad_alpha = ( + ( + ( + lower_flag * Qn + + upper_flag * Qp + + middle_flag * round(q_w) + - middle_flag * q_w + ) + * grad_weight + * g + ) + .sum() + .unsqueeze(axis=0)[0] + ) grad_weight = middle_flag * grad_weight return grad_weight, grad_alpha class LsqPlusActFunc(PyLayer): - @staticmethod def forward(ctx, x, alpha, beta, g, Qn, Qp): ctx.save_for_backward(x, alpha, beta) @@ -95,25 +112,38 @@ def backward(ctx, grad_x): lower_flag = paddle.cast((q_x < Qn), 'float32') upper_flag = paddle.cast((q_x > Qp), 'float32') middle_flag = 1.0 - lower_flag - upper_flag - grad_alpha = ((lower_flag * Qn + upper_flag * Qp + - middle_flag * round(q_x) - middle_flag * q_x) * grad_x * - g).sum().unsqueeze(axis=0)[0] - grad_beta = ((lower_flag + upper_flag) * grad_x * - g).sum().unsqueeze(axis=0)[0] + grad_alpha = ( + ( + ( + lower_flag * Qn + + upper_flag * Qp + + middle_flag * round(q_x) + - middle_flag * q_x + ) + * grad_x + * g + ) + .sum() + .unsqueeze(axis=0)[0] + ) + grad_beta = ( + ((lower_flag + upper_flag) * grad_x * g).sum().unsqueeze(axis=0)[0] + ) grad_x = middle_flag * grad_x return grad_x, grad_alpha, grad_beta class FakeQuantActLSQPlus(Layer): - - def __init__(self, - quant_bits, - all_postive=False, - symmetric=False, - batch_init=20, - dtype='float32', - name=None, - reduce_type=None): + def __init__( + self, + quant_bits, + all_postive=False, + symmetric=False, + batch_init=20, + dtype='float32', + name=None, + reduce_type=None, + ): super(FakeQuantActLSQPlus, self).__init__() ''' Args: @@ -138,46 +168,51 @@ def __init__(self, self.Qp = 2**self.bits - 1 else: # signed activation - self.Qn = -2**(self.bits - 1) - self.Qp = 2**(self.bits - 1) - 1 + self.Qn = -(2 ** (self.bits - 1)) + self.Qp = 2 ** (self.bits - 1) - 1 - scale_prefix = "{}.scale".format( - name) if name else 'quant_dequant.scale' + scale_prefix = ( + "{}.scale".format(name) if name else 'quant_dequant.scale' + ) self._scale_name = unique_name.generate(scale_prefix) - s_attr = ParamAttr(name=self._scale_name, - initializer=Constant(1.0), - trainable=True) + s_attr = ParamAttr( + name=self._scale_name, initializer=Constant(1.0), trainable=True + ) self.s = self.create_parameter(shape=[1], attr=s_attr, dtype='float32') self.s.stop_gradient = False if not self.symmetric: - beta_prefix = "{}.beta".format( - name) if name else 'quant_dequant.beta' + beta_prefix = ( + "{}.beta".format(name) if name else 'quant_dequant.beta' + ) self._beta_name = unique_name.generate(beta_prefix) - beta_attr = ParamAttr(name=self._beta_name, - initializer=Constant(0.0), - trainable=True) - self.beta = self.create_parameter(shape=[1], - attr=beta_attr, - dtype='float32') + beta_attr = ParamAttr( + name=self._beta_name, initializer=Constant(0.0), trainable=True + ) + self.beta = self.create_parameter( + shape=[1], attr=beta_attr, dtype='float32' + ) self.beta.stop_gradient = False self.init_state = 0 def forward(self, activation): if self.reduce_type == "max": - paddle.distributed.all_reduce(self.s, - op=paddle.distributed.ReduceOp.MAX) + paddle.distributed.all_reduce( + self.s, op=paddle.distributed.ReduceOp.MAX + ) if not self.symmetric and self.reduce_type == "max": - paddle.distributed.all_reduce(self.beta, - op=paddle.distributed.ReduceOp.MAX) + paddle.distributed.all_reduce( + self.beta, op=paddle.distributed.ReduceOp.MAX + ) if self.init_state == 0: - self.g = paddle.to_tensor(1.0 / - math.sqrt(activation.numel() * self.Qp)) + self.g = paddle.to_tensor( + 1.0 / math.sqrt(activation.numel() * self.Qp) + ) min_a = paddle.min(activation.detach()) max_a = paddle.max(activation.detach()) self.s.set_value((max_a - min_a) / (self.Qp - self.Qn)) @@ -187,40 +222,41 @@ def forward(self, activation): elif self.init_state < self.batch_init: min_a = paddle.min(activation.detach()) max_a = paddle.max(activation.detach()) - self.s.set_value(self.s * 0.9 + 0.1 * (max_a - min_a) / - (self.Qp - self.Qn)) + self.s.set_value( + self.s * 0.9 + 0.1 * (max_a - min_a) / (self.Qp - self.Qn) + ) if not self.symmetric: - self.beta.set_value(self.s * 0.9 + 0.1 * - (min_a - self.s * self.Qn)) + self.beta.set_value( + self.s * 0.9 + 0.1 * (min_a - self.s * self.Qn) + ) self.init_state += 1 else: self.init_state += 1 activation.stop_gradient = False if not self.symmetric: - q_a = LsqPlusActFunc.apply(activation, self.s, self.beta, self.g, - self.Qn, self.Qp) + q_a = LsqPlusActFunc.apply( + activation, self.s, self.beta, self.g, self.Qn, self.Qp + ) else: - q_a = LsqFunc.apply(activation, - self.s, - self.g, - self.Qn, - self.Qp, - per_channel=False) + q_a = LsqFunc.apply( + activation, self.s, self.g, self.Qn, self.Qp, per_channel=False + ) return q_a class FakeQuantWeightLSQPlus(Layer): - - def __init__(self, - quant_bits, - all_postive=False, - per_channel=False, - batch_init=20, - channel_num=None, - quant_linear=False, - dtype='float32', - name=None, - reduce_type=None): + def __init__( + self, + quant_bits, + all_postive=False, + per_channel=False, + batch_init=20, + channel_num=None, + quant_linear=False, + dtype='float32', + name=None, + reduce_type=None, + ): super(FakeQuantWeightLSQPlus, self).__init__() ''' Args: @@ -251,25 +287,27 @@ def __init__(self, self.Qp = 2**self.bits - 1 else: # signed weight - self.Qn = -2**(self.bits - 1) - self.Qp = 2**(self.bits - 1) - 1 + self.Qn = -(2 ** (self.bits - 1)) + self.Qp = 2 ** (self.bits - 1) - 1 self.init_state = 0 - scale_prefix = "{}.scale".format( - name) if name else 'quant_dequant.scale' + scale_prefix = ( + "{}.scale".format(name) if name else 'quant_dequant.scale' + ) self._scale_name = unique_name.generate(scale_prefix) - s_attr = ParamAttr(name=self._scale_name, - initializer=Constant(1.0), - trainable=True) - self.s = self.create_parameter(shape=[channel_num], - attr=s_attr, - dtype=dtype) + s_attr = ParamAttr( + name=self._scale_name, initializer=Constant(1.0), trainable=True + ) + self.s = self.create_parameter( + shape=[channel_num], attr=s_attr, dtype=dtype + ) self.s.stop_gradient = False def forward(self, weight): if self.reduce_type == "max": - paddle.distributed.all_reduce(self.s, - op=paddle.distributed.ReduceOp.MAX) + paddle.distributed.all_reduce( + self.s, op=paddle.distributed.ReduceOp.MAX + ) if self.init_state == 0: self.g = paddle.to_tensor(1.0 / math.sqrt(weight.numel() * self.Qp)) @@ -278,19 +316,22 @@ def forward(self, weight): weight_tmp = weight.detach().reshape((weight.shape[0], -1)) mean = paddle.mean(weight_tmp, axis=self.collect_axis) std = paddle.std(weight_tmp, axis=self.collect_axis) - s = paddle.max(paddle.stack( - [paddle.abs(mean - 3 * std), - paddle.abs(mean + 3 * std)]), - axis=0) + s = paddle.max( + paddle.stack( + [paddle.abs(mean - 3 * std), paddle.abs(mean + 3 * std)] + ), + axis=0, + ) self.s.set_value(s / self.div) else: mean = paddle.mean(weight.detach()) std = paddle.std(weight.detach()) self.s.set_value( - max([ - paddle.abs(mean - 3 * std), - paddle.abs(mean + 3 * std) - ]) / self.div) + max( + [paddle.abs(mean - 3 * std), paddle.abs(mean + 3 * std)] + ) + / self.div + ) self.init_state += 1 elif self.init_state < self.batch_init: self.div = 2**self.bits - 1 @@ -298,22 +339,36 @@ def forward(self, weight): weight_tmp = weight.detach().reshape((weight.shape[0], -1)) mean = paddle.mean(weight_tmp, axis=self.collect_axis) std = paddle.std(weight_tmp, axis=self.collect_axis) - s = paddle.max(paddle.stack( - [paddle.abs(mean - 3 * std), - paddle.abs(mean + 3 * std)]), - axis=0) + s = paddle.max( + paddle.stack( + [paddle.abs(mean - 3 * std), paddle.abs(mean + 3 * std)] + ), + axis=0, + ) self.s.set_value(s * 0.9 + 0.1 * s / self.div) else: mean = paddle.mean(weight.detach()) std = paddle.std(weight.detach()) - self.s.set_value(self.s * 0.9 + 0.1 * max( - [paddle.abs(mean - 3 * std), - paddle.abs(mean + 3 * std)]) / self.div) + self.s.set_value( + self.s * 0.9 + + 0.1 + * max( + [paddle.abs(mean - 3 * std), paddle.abs(mean + 3 * std)] + ) + / self.div + ) self.init_state += 1 elif self.init_state == self.batch_init: self.init_state += 1 weight.stop_gradient = False - w_q = LsqFunc.apply(weight, self.s, self.g, self.Qn, self.Qp, - self.per_channel, self.quant_axis) + w_q = LsqFunc.apply( + weight, + self.s, + self.g, + self.Qn, + self.Qp, + self.per_channel, + self.quant_axis, + ) return w_q diff --git a/python/paddle/nn/quant/quant_layers.py b/python/paddle/nn/quant/quant_layers.py index affa180bd303a..6eeaed7f86fec 100644 --- a/python/paddle/nn/quant/quant_layers.py +++ b/python/paddle/nn/quant/quant_layers.py @@ -42,9 +42,9 @@ 'QuantizedColumnParallelLinear', ] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) class FakeQuantAbsMax(Layer): @@ -57,26 +57,31 @@ class FakeQuantAbsMax(Layer): :math:`Out = round(X / scale * range) * scale / range` """ - def __init__(self, - name=None, - quant_bits=8, - dtype='float32', - quant_on_weight=False, - reduce_type=None): + def __init__( + self, + name=None, + quant_bits=8, + dtype='float32', + quant_on_weight=False, + reduce_type=None, + ): super(FakeQuantAbsMax, self).__init__() self._quant_bits = quant_bits self._name = name self._reduce_type = reduce_type - scale_prefix = "{}.scale".format( - name) if name else 'quant_dequant.scale' + scale_prefix = ( + "{}.scale".format(name) if name else 'quant_dequant.scale' + ) self._scale_name = unique_name.generate(scale_prefix) if quant_on_weight: - scale_attr = ParamAttr(name=self._scale_name, - initializer=Constant(0.001), - trainable=False) - self._scale = self.create_parameter(shape=[1], - attr=scale_attr, - dtype=self._dtype) + scale_attr = ParamAttr( + name=self._scale_name, + initializer=Constant(0.001), + trainable=False, + ) + self._scale = self.create_parameter( + shape=[1], attr=scale_attr, dtype=self._dtype + ) self._scale.stop_gradient = True else: self._scale = None @@ -84,16 +89,18 @@ def __init__(self, def forward(self, input): if in_dynamic_mode(): attrs = ('bit_length', self._quant_bits) - quant_out = _varbase_creator(type=input.type, - name="{}.quantized.dequantized".format( - input.name), - shape=input.shape, - dtype=input.dtype, - persistable=False) + quant_out = _varbase_creator( + type=input.type, + name="{}.quantized.dequantized".format(input.name), + shape=input.shape, + dtype=input.dtype, + persistable=False, + ) out_scale = self._scale if self._reduce_type == "max": paddle.distributed.all_reduce( - out_scale, op=paddle.distributed.ReduceOp.MAX) + out_scale, op=paddle.distributed.ReduceOp.MAX + ) if not out_scale: out_scale = _varbase_creator( @@ -101,10 +108,12 @@ def forward(self, input): name=self._scale_name, shape=[1], dtype=self._dtype, - persistable=False) + persistable=False, + ) out_scale.stop_gradient = True out, _, = _legacy_C_ops.fake_quantize_dequantize_abs_max( - input, quant_out, out_scale, *attrs) + input, quant_out, out_scale, *attrs + ) return out check_variable_and_dtype(input, 'input', ['float32'], "FakeQuantAbsMax") @@ -115,7 +124,8 @@ def forward(self, input): dtype=input.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) out_scale = self._scale if not out_scale: out_scale = self._helper.create_variable( @@ -123,13 +133,16 @@ def forward(self, input): dtype=self._dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) outputs = {"Out": [quant_out], "OutScale": [out_scale]} - self._helper.append_op(type="fake_quantize_dequantize_abs_max", - inputs=inputs, - outputs=outputs, - attrs=attrs) + self._helper.append_op( + type="fake_quantize_dequantize_abs_max", + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) return quant_out @@ -144,75 +157,108 @@ class FakeQuantMovingAverageAbsMax(Layer): :math:`Out = round(X / scale * range) * scale / range` """ - def __init__(self, - name=None, - moving_rate=0.9, - quant_bits=8, - dtype='float32', - reduce_type=None): + def __init__( + self, + name=None, + moving_rate=0.9, + quant_bits=8, + dtype='float32', + reduce_type=None, + ): super(FakeQuantMovingAverageAbsMax, self).__init__() self._moving_rate = moving_rate self._quant_bits = quant_bits self._reduce_type = reduce_type - scale_prefix = "{}.scale".format( - name) if name else 'quant_dequant.scale' - scale_attr = ParamAttr(name=unique_name.generate(scale_prefix), - initializer=Constant(0.001), - trainable=False) - self._scale = self.create_parameter(shape=[1], - attr=scale_attr, - dtype=dtype) + scale_prefix = ( + "{}.scale".format(name) if name else 'quant_dequant.scale' + ) + scale_attr = ParamAttr( + name=unique_name.generate(scale_prefix), + initializer=Constant(0.001), + trainable=False, + ) + self._scale = self.create_parameter( + shape=[1], attr=scale_attr, dtype=dtype + ) self._scale.stop_gradient = True - state_prefix = "{}.state".format( - name) if name else 'quant_dequant.state' - state_attr = ParamAttr(name=unique_name.generate(state_prefix), - initializer=Constant(1), - trainable=False) - self._state = self.create_parameter(shape=[1], - attr=state_attr, - dtype=dtype) + state_prefix = ( + "{}.state".format(name) if name else 'quant_dequant.state' + ) + state_attr = ParamAttr( + name=unique_name.generate(state_prefix), + initializer=Constant(1), + trainable=False, + ) + self._state = self.create_parameter( + shape=[1], attr=state_attr, dtype=dtype + ) self._state.stop_gradient = True - accum_prefix = "{}.accum".format( - name) if name else 'quant_dequant.accum' - accum_attr = ParamAttr(name=unique_name.generate(accum_prefix), - initializer=Constant(1), - trainable=False) - self._accum = self.create_parameter(shape=[1], - attr=accum_attr, - dtype=dtype) + accum_prefix = ( + "{}.accum".format(name) if name else 'quant_dequant.accum' + ) + accum_attr = ParamAttr( + name=unique_name.generate(accum_prefix), + initializer=Constant(1), + trainable=False, + ) + self._accum = self.create_parameter( + shape=[1], attr=accum_attr, dtype=dtype + ) self._accum.stop_gradient = True def forward(self, input): if in_dynamic_mode(): - attrs = ('moving_rate', self._moving_rate, 'bit_length', - self._quant_bits, 'is_test', not self.training) - quant_out = _varbase_creator(type=input.type, - name="{}.quantized.dequantized".format( - input.name), - shape=input.shape, - dtype=input.dtype, - persistable=False) + attrs = ( + 'moving_rate', + self._moving_rate, + 'bit_length', + self._quant_bits, + 'is_test', + not self.training, + ) + quant_out = _varbase_creator( + type=input.type, + name="{}.quantized.dequantized".format(input.name), + shape=input.shape, + dtype=input.dtype, + persistable=False, + ) if self._reduce_type == "max": paddle.distributed.all_reduce( - self._scale, op=paddle.distributed.ReduceOp.MAX) + self._scale, op=paddle.distributed.ReduceOp.MAX + ) state = self._state if self.training else None accum = self._accum if self.training else None - out, _, _, _ = _legacy_C_ops.fake_quantize_dequantize_moving_average_abs_max( - input, self._scale, accum, state, quant_out, self._scale, state, - accum, *attrs) + ( + out, + _, + _, + _, + ) = _legacy_C_ops.fake_quantize_dequantize_moving_average_abs_max( + input, + self._scale, + accum, + state, + quant_out, + self._scale, + state, + accum, + *attrs + ) return out - check_variable_and_dtype(input, 'input', ['float32'], - "FakeQuantMovingAverageAbsMax") + check_variable_and_dtype( + input, 'input', ['float32'], "FakeQuantMovingAverageAbsMax" + ) attrs = { 'moving_rate': self._moving_rate, 'bit_length': self._quant_bits, - 'is_test': not self.training + 'is_test': not self.training, } inputs = {"X": [input], "InScale": [self._scale]} quant_out = self._helper.create_variable( @@ -220,7 +266,8 @@ def forward(self, input): dtype=input.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) outputs = {"Out": [quant_out], "OutScale": [self._scale]} if self.training: @@ -233,22 +280,26 @@ def forward(self, input): type="fake_quantize_dequantize_moving_average_abs_max", inputs=inputs, outputs=outputs, - attrs=attrs) + attrs=attrs, + ) return quant_out class FakeQuantChannelWiseAbsMax(Layer): - - def __init__(self, - name=None, - channel_num=None, - quant_bits=8, - quant_axis=0, - dtype='float32', - quant_on_weight=False, - reduce_type=None): - assert quant_on_weight == True, "Channel_wise only can be used on weight quantization." + def __init__( + self, + name=None, + channel_num=None, + quant_bits=8, + quant_axis=0, + dtype='float32', + quant_on_weight=False, + reduce_type=None, + ): + assert ( + quant_on_weight == True + ), "Channel_wise only can be used on weight quantization." super(FakeQuantChannelWiseAbsMax, self).__init__() self._quant_bits = quant_bits self._quant_axis = quant_axis @@ -256,50 +307,65 @@ def __init__(self, self._name = name self._channel_num = channel_num self._reduce_type = reduce_type - scale_prefix = "{}.scale".format( - name) if name else 'quant_dequant.scale' + scale_prefix = ( + "{}.scale".format(name) if name else 'quant_dequant.scale' + ) self._scale_name = unique_name.generate(scale_prefix) if quant_on_weight: - scale_attr = ParamAttr(name=self._scale_name, - initializer=Constant(0.0), - trainable=False) - self._scale = self.create_parameter(shape=[self._channel_num], - attr=scale_attr, - dtype=self._dtype) + scale_attr = ParamAttr( + name=self._scale_name, + initializer=Constant(0.0), + trainable=False, + ) + self._scale = self.create_parameter( + shape=[self._channel_num], attr=scale_attr, dtype=self._dtype + ) self._scale.stop_gradient = True else: self._scale = None def forward(self, input): if in_dynamic_mode(): - attrs = ('bit_length', self._quant_bits, 'quant_axis', - self._quant_axis) - quant_out = _varbase_creator(type=input.type, - name="{}.quantized.dequantized".format( - input.name), - shape=input.shape, - dtype=input.dtype, - persistable=False) + attrs = ( + 'bit_length', + self._quant_bits, + 'quant_axis', + self._quant_axis, + ) + quant_out = _varbase_creator( + type=input.type, + name="{}.quantized.dequantized".format(input.name), + shape=input.shape, + dtype=input.dtype, + persistable=False, + ) out_scale = self._scale if self._reduce_type == "max": paddle.distributed.all_reduce( - out_scale, op=paddle.distributed.ReduceOp.MAX) + out_scale, op=paddle.distributed.ReduceOp.MAX + ) if out_scale is None: out_scale = _varbase_creator( type=core.VarDesc.VarType.LOD_TENSOR, name=self._scale_name, shape=[self._channel_num], dtype=self._dtype, - persistable=False) + persistable=False, + ) out_scale.stop_gradient = True - out, _, = _legacy_C_ops.fake_channel_wise_quantize_dequantize_abs_max( - input, quant_out, out_scale, *attrs) + ( + out, + _, + ) = _legacy_C_ops.fake_channel_wise_quantize_dequantize_abs_max( + input, quant_out, out_scale, *attrs + ) return out - check_variable_and_dtype(input, 'input', ['float32'], - "FakeQuantChannelWiseAbsMax") + check_variable_and_dtype( + input, 'input', ['float32'], "FakeQuantChannelWiseAbsMax" + ) attrs = {'bit_length': self._quant_bits, 'quant_axis': self._quant_axis} inputs = {"X": [input]} quant_out = self._helper.create_variable( @@ -307,7 +373,8 @@ def forward(self, input): dtype=input.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) out_scale = self._scale if not out_scale: out_scale = self._helper.create_variable( @@ -315,25 +382,24 @@ def forward(self, input): dtype=self._dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=True) + stop_gradient=True, + ) outputs = {"Out": [quant_out], "OutScale": [out_scale]} self._helper.append_op( type="fake_channel_wise_quantize_dequantize_abs_max", inputs=inputs, outputs=outputs, - attrs=attrs) + attrs=attrs, + ) return quant_out class MovingAverageAbsMaxScale(Layer): - - def __init__(self, - name=None, - moving_rate=0.9, - dtype='float32', - reduce_type=None): + def __init__( + self, name=None, moving_rate=0.9, dtype='float32', reduce_type=None + ): r""" MovingAverageMaxScale layer is used to calculating the output quantization scale of Layer. Its computational formula is described as below: @@ -346,56 +412,75 @@ def __init__(self, self._reduce_type = reduce_type scale_prefix = '{}.scale'.format(name) if name else 'outscale.scale' scale_name = unique_name.generate(scale_prefix) - scale_attr = ParamAttr(name=scale_name, - initializer=Constant(0), - trainable=False) - self._scale = self.create_parameter(shape=[1], - attr=scale_attr, - dtype=dtype) + scale_attr = ParamAttr( + name=scale_name, initializer=Constant(0), trainable=False + ) + self._scale = self.create_parameter( + shape=[1], attr=scale_attr, dtype=dtype + ) self._scale.stop_gradient = True state_prefix = "{}.state".format(name) if name else 'outscale.state' - state_attr = ParamAttr(name=unique_name.generate(state_prefix), - initializer=Constant(0), - trainable=False) - self._state = self.create_parameter(shape=[1], - attr=state_attr, - dtype=dtype) + state_attr = ParamAttr( + name=unique_name.generate(state_prefix), + initializer=Constant(0), + trainable=False, + ) + self._state = self.create_parameter( + shape=[1], attr=state_attr, dtype=dtype + ) self._state.stop_gradient = True accum_prefix = "{}.accum".format(name) if name else 'outscale.accum' - accum_attr = ParamAttr(name=unique_name.generate(accum_prefix), - initializer=Constant(0), - trainable=False) - self._accum = self.create_parameter(shape=[1], - attr=accum_attr, - dtype=dtype) + accum_attr = ParamAttr( + name=unique_name.generate(accum_prefix), + initializer=Constant(0), + trainable=False, + ) + self._accum = self.create_parameter( + shape=[1], attr=accum_attr, dtype=dtype + ) self._accum.stop_gradient = True def forward(self, input): if in_dynamic_mode(): - attrs = ('moving_rate', self._moving_rate, 'is_test', - not self.training) - - quant_out = _varbase_creator(type=input.type, - name="{}.tmp".format(input.name), - shape=input.shape, - dtype=input.dtype, - persistable=False) + attrs = ( + 'moving_rate', + self._moving_rate, + 'is_test', + not self.training, + ) + + quant_out = _varbase_creator( + type=input.type, + name="{}.tmp".format(input.name), + shape=input.shape, + dtype=input.dtype, + persistable=False, + ) if self._reduce_type == "max": paddle.distributed.all_reduce( - self._scale, op=paddle.distributed.ReduceOp.MAX) + self._scale, op=paddle.distributed.ReduceOp.MAX + ) state = self._state if self.training else None accum = self._accum if self.training else None out, _, _, _ = _legacy_C_ops.moving_average_abs_max_scale( - input, accum, state, quant_out, self._scale, state, accum, - *attrs) + input, + accum, + state, + quant_out, + self._scale, + state, + accum, + *attrs + ) return out - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'MovingAverageAbsMaxScale') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'MovingAverageAbsMaxScale' + ) attrs = {'moving_rate': self._moving_rate, 'is_test': not self.training} inputs = {"X": [input]} @@ -404,7 +489,8 @@ def forward(self, input): dtype=input.dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, - stop_gradient=False) + stop_gradient=False, + ) outputs = {"Out": [quant_out], "OutScale": [self._scale]} if self.training: @@ -413,10 +499,12 @@ def forward(self, input): outputs['OutState'] = [self._state] outputs['OutAccum'] = [self._accum] - self._helper.append_op(type="moving_average_abs_max_scale", - inputs=inputs, - outputs=outputs, - attrs=attrs) + self._helper.append_op( + type="moving_average_abs_max_scale", + inputs=inputs, + outputs=outputs, + attrs=attrs, + ) return quant_out @@ -430,17 +518,19 @@ class QuantizedConv2D(Layer): The only difference is that its inputs are all fake quantized. """ - def __init__(self, - layer, - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - weight_quantize_type='abs_max', - activation_quantize_type='abs_max', - weight_pre_layer=None, - act_pre_layer=None, - weight_quant_layer=None, - act_quant_layer=None): + def __init__( + self, + layer, + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + weight_quantize_type='abs_max', + activation_quantize_type='abs_max', + weight_pre_layer=None, + act_pre_layer=None, + weight_quant_layer=None, + act_quant_layer=None, + ): super(QuantizedConv2D, self).__init__() # For Conv2D self._groups = getattr(layer, '_groups') @@ -449,7 +539,8 @@ def __init__(self, self._padding_mode = getattr(layer, '_padding_mode') if self._padding_mode != 'zeros': self._reversed_padding_repeated_twice = getattr( - layer, '_reversed_padding_repeated_twice') + layer, '_reversed_padding_repeated_twice' + ) self._dilation = getattr(layer, '_dilation') self._data_format = getattr(layer, '_data_format') self.weight = getattr(layer, 'weight') @@ -468,7 +559,8 @@ def __init__(self, dtype=self._dtype, quant_on_weight=True, channel_num=self.weight.shape[self._conv2d_quant_axis], - quant_axis=self._conv2d_quant_axis) + quant_axis=self._conv2d_quant_axis, + ) if act_quant_layer is not None: self._fake_quant_input = act_quant_layer() else: @@ -478,12 +570,15 @@ def __init__(self, moving_rate=moving_rate, quant_bits=activation_bits, dtype=self._dtype, - quant_on_weight=False) + quant_on_weight=False, + ) - self._act_preprocess = act_pre_layer( - ) if act_pre_layer is not None else None - self._weight_preprocess = weight_pre_layer( - ) if weight_pre_layer is not None else None + self._act_preprocess = ( + act_pre_layer() if act_pre_layer is not None else None + ) + self._weight_preprocess = ( + weight_pre_layer() if weight_pre_layer is not None else None + ) def forward(self, input): if self._act_preprocess is not None: @@ -496,20 +591,24 @@ def forward(self, input): quant_weight = self._fake_quant_weight(weight) if self._padding_mode != 'zeros': - quant_input = F.pad(quant_input, - self._reversed_padding_repeated_twice, - mode=self._padding_mode, - data_format=self._data_format) + quant_input = F.pad( + quant_input, + self._reversed_padding_repeated_twice, + mode=self._padding_mode, + data_format=self._data_format, + ) self._padding = 0 - return F.conv2d(quant_input, - quant_weight, - bias=self.bias, - padding=self._padding, - stride=self._stride, - dilation=self._dilation, - groups=self._groups, - data_format=self._data_format) + return F.conv2d( + quant_input, + quant_weight, + bias=self.bias, + padding=self._padding, + stride=self._stride, + dilation=self._dilation, + groups=self._groups, + data_format=self._data_format, + ) class QuantizedConv2DTranspose(Layer): @@ -533,17 +632,19 @@ class QuantizedConv2DTranspose(Layer): # (2, 6, 10, 10), (2, 6, 10, 10) """ - def __init__(self, - layer, - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - weight_quantize_type='abs_max', - activation_quantize_type='abs_max', - weight_pre_layer=None, - act_pre_layer=None, - weight_quant_layer=None, - act_quant_layer=None): + def __init__( + self, + layer, + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + weight_quantize_type='abs_max', + activation_quantize_type='abs_max', + weight_pre_layer=None, + act_pre_layer=None, + weight_quant_layer=None, + act_quant_layer=None, + ): r""" Constructor. @@ -572,8 +673,10 @@ def __init__(self, dtype=self._dtype, quant_on_weight=True, channel_num=self.weight.shape[ - self._conv2d_transpose_quant_axis], - quant_axis=self._conv2d_transpose_quant_axis) + self._conv2d_transpose_quant_axis + ], + quant_axis=self._conv2d_transpose_quant_axis, + ) if act_quant_layer is not None: self._fake_quant_input = act_quant_layer() else: @@ -583,12 +686,15 @@ def __init__(self, moving_rate=moving_rate, quant_bits=activation_bits, dtype=self._dtype, - quant_on_weight=False) + quant_on_weight=False, + ) - self._act_preprocess = act_pre_layer( - ) if act_pre_layer is not None else None - self._weight_preprocess = weight_pre_layer( - ) if weight_pre_layer is not None else None + self._act_preprocess = ( + act_pre_layer() if act_pre_layer is not None else None + ) + self._weight_preprocess = ( + weight_pre_layer() if weight_pre_layer is not None else None + ) def forward(self, input, output_size=None): if self._act_preprocess is not None: @@ -605,16 +711,18 @@ def forward(self, input, output_size=None): else: output_padding = 0 - return F.conv2d_transpose(quant_input, - quant_weight, - bias=self.bias, - padding=self._padding, - output_padding=output_padding, - stride=self._stride, - dilation=self._dilation, - groups=self._groups, - output_size=output_size, - data_format=self._data_format) + return F.conv2d_transpose( + quant_input, + quant_weight, + bias=self.bias, + padding=self._padding, + output_padding=output_padding, + stride=self._stride, + dilation=self._dilation, + groups=self._groups, + output_size=output_size, + data_format=self._data_format, + ) class QuantizedLinear(Layer): @@ -623,17 +731,19 @@ class QuantizedLinear(Layer): The only difference is that its inputs are all fake quantized. """ - def __init__(self, - layer, - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - weight_quantize_type='abs_max', - activation_quantize_type='abs_max', - weight_pre_layer=None, - act_pre_layer=None, - weight_quant_layer=None, - act_quant_layer=None): + def __init__( + self, + layer, + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + weight_quantize_type='abs_max', + activation_quantize_type='abs_max', + weight_pre_layer=None, + act_pre_layer=None, + weight_quant_layer=None, + act_quant_layer=None, + ): super(QuantizedLinear, self).__init__() # For Linear self.weight = getattr(layer, 'weight') @@ -654,7 +764,8 @@ def __init__(self, quant_on_weight=True, channel_num=self.weight.shape[self._linear_quant_axis], quant_axis=self._linear_quant_axis, - quant_linear=True) + quant_linear=True, + ) if act_quant_layer is not None: self._fake_quant_input = act_quant_layer() @@ -665,12 +776,15 @@ def __init__(self, moving_rate=moving_rate, quant_bits=activation_bits, dtype=self._dtype, - quant_on_weight=False) + quant_on_weight=False, + ) - self._act_preprocess = act_pre_layer( - ) if act_pre_layer is not None else None - self._weight_preprocess = weight_pre_layer( - ) if weight_pre_layer is not None else None + self._act_preprocess = ( + act_pre_layer() if act_pre_layer is not None else None + ) + self._weight_preprocess = ( + weight_pre_layer() if weight_pre_layer is not None else None + ) def forward(self, input): if self._act_preprocess is not None: @@ -682,32 +796,36 @@ def forward(self, input): weight = self._weight_preprocess(self.weight) quant_weight = self._fake_quant_weight(weight) - out = F.linear(x=quant_input, - weight=quant_weight, - bias=self.bias, - name=self.name) + out = F.linear( + x=quant_input, weight=quant_weight, bias=self.bias, name=self.name + ) return out class QuantizedColumnParallelLinear(Layer): - - def __init__(self, - layer, - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - weight_quantize_type='abs_max', - activation_quantize_type='abs_max', - weight_pre_layer=None, - act_pre_layer=None, - weight_quant_layer=None, - act_quant_layer=None): + def __init__( + self, + layer, + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + weight_quantize_type='abs_max', + activation_quantize_type='abs_max', + weight_pre_layer=None, + act_pre_layer=None, + weight_quant_layer=None, + act_quant_layer=None, + ): super(QuantizedColumnParallelLinear, self).__init__() ''' ''' - assert weight_quant_layer is None, "When quantizing ColumnParallelLinear, weight_quant_layer should be None." - assert act_quant_layer is None, "When quantizing ColumnParallelLinear, act_quant_layer should be None." + assert ( + weight_quant_layer is None + ), "When quantizing ColumnParallelLinear, weight_quant_layer should be None." + assert ( + act_quant_layer is None + ), "When quantizing ColumnParallelLinear, act_quant_layer should be None." self.weight = getattr(layer, 'weight') self.bias = getattr(layer, 'bias') @@ -729,7 +847,9 @@ def __init__(self, channel_num=self.weight.shape[self._linear_quant_axis], quant_axis=self._linear_quant_axis, reduce_type='max' - if paddle.distributed.get_world_size() > 1 else None) + if paddle.distributed.get_world_size() > 1 + else None, + ) self._fake_quant_input = _get_fake_quant_type( activation_quantize_type, @@ -738,17 +858,21 @@ def __init__(self, quant_bits=activation_bits, dtype=self._dtype, quant_on_weight=False, - reduce_type=None) + reduce_type=None, + ) - self._act_preprocess = act_pre_layer( - ) if act_pre_layer is not None else None - self._weight_preprocess = weight_pre_layer( - ) if weight_pre_layer is not None else None + self._act_preprocess = ( + act_pre_layer() if act_pre_layer is not None else None + ) + self._weight_preprocess = ( + weight_pre_layer() if weight_pre_layer is not None else None + ) def forward(self, input): if self.is_mp: input_parallel = paddle.distributed.collective._c_identity( - input, group=self.model_parallel_group) + input, group=self.model_parallel_group + ) else: input_parallel = input @@ -761,35 +885,40 @@ def forward(self, input): weight = self._weight_preprocess(self.weight) quant_weight = self._fake_quant_weight(weight) - output_parallel = F.linear(x=quant_input, - weight=quant_weight, - bias=self.bias, - name=self.name) + output_parallel = F.linear( + x=quant_input, weight=quant_weight, bias=self.bias, name=self.name + ) if self.gather_output and self.is_mp: output = paddle.distributed.collective._c_concat( - output_parallel, group=self.model_parallel_group) + output_parallel, group=self.model_parallel_group + ) else: output = output_parallel return output class QuantizedRowParallelLinear(Layer): - - def __init__(self, - layer, - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - weight_quantize_type='abs_max', - activation_quantize_type='abs_max', - weight_pre_layer=None, - act_pre_layer=None, - weight_quant_layer=None, - act_quant_layer=None): + def __init__( + self, + layer, + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + weight_quantize_type='abs_max', + activation_quantize_type='abs_max', + weight_pre_layer=None, + act_pre_layer=None, + weight_quant_layer=None, + act_quant_layer=None, + ): super(QuantizedRowParallelLinear, self).__init__() - assert weight_quant_layer is None, "When quantizing RowParallelLinear, weight_quant_layer cannot defined by yourself." - assert act_quant_layer is None, "When quantizing RowParallelLinear, act_quant_layer cannot defined by yourself." + assert ( + weight_quant_layer is None + ), "When quantizing RowParallelLinear, weight_quant_layer cannot defined by yourself." + assert ( + act_quant_layer is None + ), "When quantizing RowParallelLinear, act_quant_layer cannot defined by yourself." # For Linear self.weight = getattr(layer, 'weight') @@ -812,7 +941,9 @@ def __init__(self, channel_num=self.weight.shape[self._linear_quant_axis], quant_axis=self._linear_quant_axis, reduce_type='max' - if paddle.distributed.get_world_size() > 1 else None) + if paddle.distributed.get_world_size() > 1 + else None, + ) self._fake_quant_input = _get_fake_quant_type( activation_quantize_type, @@ -822,12 +953,16 @@ def __init__(self, dtype=self._dtype, quant_on_weight=False, reduce_type='max' - if paddle.distributed.get_world_size() > 1 else None) + if paddle.distributed.get_world_size() > 1 + else None, + ) - self._act_preprocess = act_pre_layer( - ) if act_pre_layer is not None else None - self._weight_preprocess = weight_pre_layer( - ) if weight_pre_layer is not None else None + self._act_preprocess = ( + act_pre_layer() if act_pre_layer is not None else None + ) + self._weight_preprocess = ( + weight_pre_layer() if weight_pre_layer is not None else None + ) def forward(self, input): if self.input_is_parallel or (not self.is_mp): @@ -835,7 +970,8 @@ def forward(self, input): else: # split last dim input_parallel = paddle.distributed.collective._c_split( - input, group=self.model_parallel_group) + input, group=self.model_parallel_group + ) if self._act_preprocess is not None: input_parallel = self._act_preprocess(input_parallel) @@ -846,15 +982,16 @@ def forward(self, input): weight = self._weight_preprocess(self.weight) quant_weight = self._fake_quant_weight(weight) - output_parallel = F.linear(x=quant_input, - weight=quant_weight, - name=self.name) + output_parallel = F.linear( + x=quant_input, weight=quant_weight, name=self.name + ) if self.is_mp: output_ = paddle.distributed.collective._mp_allreduce( output_parallel, group=self.model_parallel_group, use_calc_stream=True, - use_model_parallel=True) + use_model_parallel=True, + ) else: output_ = output_parallel output = output_ + self.bias if self.bias is not None else output_ @@ -867,12 +1004,14 @@ class MAOutputScaleLayer(Layer): Calculate the scale (moving average abs max) for the output of the input layer. """ - def __init__(self, - layer=None, - moving_rate=0.9, - name=None, - dtype='float32', - reduce_type=None): + def __init__( + self, + layer=None, + moving_rate=0.9, + name=None, + dtype='float32', + reduce_type=None, + ): r""" Construct """ @@ -880,14 +1019,18 @@ def __init__(self, self._layer = layer if name is None: name = layer.full_name() - self._ma_output_scale = \ - MovingAverageAbsMaxScale(name, moving_rate, dtype, reduce_type) + self._ma_output_scale = MovingAverageAbsMaxScale( + name, moving_rate, dtype, reduce_type + ) def forward(self, *inputs, **kwargs): out = self._layer(*inputs, **kwargs) # TODO (jc): support the ops of several outputs - if (isinstance(out, list) or isinstance(out, tuple) - or isinstance(out, dict)): + if ( + isinstance(out, list) + or isinstance(out, tuple) + or isinstance(out, dict) + ): return out else: return self._ma_output_scale(out) @@ -898,15 +1041,17 @@ class FakeQuantMAOutputScaleLayer(Layer): Add FakeQuantMovingAverageAbsMax layer to the behind of the input layer. """ - def __init__(self, - layer, - weight_bits=8, - activation_bits=8, - moving_rate=0.9, - name=None, - reduce_type=None, - *args, - **kwargs): + def __init__( + self, + layer, + weight_bits=8, + activation_bits=8, + moving_rate=0.9, + name=None, + reduce_type=None, + *args, + **kwargs + ): super(FakeQuantMAOutputScaleLayer, self).__init__() self._layer = layer @@ -917,7 +1062,8 @@ def __init__(self, quant_bits=activation_bits, dtype=self._dtype, quant_on_weight=False, - reduce_type=reduce_type) + reduce_type=reduce_type, + ) def forward(self, *inputs, **kwargs): out = self._layer(*inputs, **kwargs) @@ -933,7 +1079,7 @@ def _get_fake_quant_type(quant_type, **kwargs): "name": kwargs.get("name", None), "quant_bits": kwargs.get("quant_bits", 8), "dtype": kwargs.get("dtype", "float32"), - "reduce_type": kwargs.get("reduce_type", None) + "reduce_type": kwargs.get("reduce_type", None), } if quant_type == 'abs_max': @@ -946,7 +1092,8 @@ def _get_fake_quant_type(quant_type, **kwargs): call_args["quant_axis"] = kwargs.get("quant_axis", 0) assert call_args["channel_num"] is not None, ( "You need to input channel_num" - "when you use channel_wise_abs_max strategy.") + "when you use channel_wise_abs_max strategy." + ) elif quant_type == 'lsq_weight': call_args["all_postive"] = kwargs.get("all_postive", False) call_args["per_channel"] = False @@ -960,7 +1107,8 @@ def _get_fake_quant_type(quant_type, **kwargs): call_args["quant_linear"] = kwargs.get("quant_linear", False) assert call_args["channel_num"] is not None, ( "You need to input channel_num" - "when you use channel_wise_abs_max strategy.") + "when you use channel_wise_abs_max strategy." + ) elif quant_type == 'lsq_act': call_args["all_postive"] = kwargs.get("all_postive", False) call_args["symmetric"] = kwargs.get("symmetric", True) @@ -969,7 +1117,7 @@ def _get_fake_quant_type(quant_type, **kwargs): 'moving_average_abs_max': FakeQuantMovingAverageAbsMax, 'channel_wise_abs_max': FakeQuantChannelWiseAbsMax, 'lsq_weight': FakeQuantWeightLSQPlus, - 'lsq_act': FakeQuantActLSQPlus + 'lsq_act': FakeQuantActLSQPlus, } return fake_quant_map[quant_type](**call_args) diff --git a/python/paddle/nn/utils/__init__.py b/python/paddle/nn/utils/__init__.py index 5afdaa8d84896..23e1e233cc0dc 100644 --- a/python/paddle/nn/utils/__init__.py +++ b/python/paddle/nn/utils/__init__.py @@ -14,9 +14,16 @@ from .spectral_norm_hook import spectral_norm from .weight_norm_hook import weight_norm, remove_weight_norm # noqa: F401 -from .transform_parameters import parameters_to_vector, vector_to_parameters, _stride_column # noqa: F401 +from .transform_parameters import ( + parameters_to_vector, + vector_to_parameters, + _stride_column, +) # noqa: F401 -__all__ = [ #noqa - 'weight_norm', 'remove_weight_norm', 'spectral_norm', - 'parameters_to_vector', 'vector_to_parameters' +__all__ = [ # noqa + 'weight_norm', + 'remove_weight_norm', + 'spectral_norm', + 'parameters_to_vector', + 'vector_to_parameters', ] diff --git a/python/paddle/nn/utils/spectral_norm_hook.py b/python/paddle/nn/utils/spectral_norm_hook.py index a2a93d3241e3d..288e5ea59c91a 100644 --- a/python/paddle/nn/utils/spectral_norm_hook.py +++ b/python/paddle/nn/utils/spectral_norm_hook.py @@ -20,21 +20,21 @@ __all__ = [] -def normal_(x, mean=0., std=1.): +def normal_(x, mean=0.0, std=1.0): temp_value = paddle.normal(mean, std, shape=x.shape) x.set_value(temp_value) return x class SpectralNorm(object): - def __init__(self, name='weight', n_power_iterations=1, dim=0, eps=1e-12): self.name = name self.dim = dim if n_power_iterations <= 0: raise ValueError( 'Expected n_power_iterations to be positive, but ' - 'got n_power_iterations={}'.format(n_power_iterations)) + 'got n_power_iterations={}'.format(n_power_iterations) + ) self.n_power_iterations = n_power_iterations self.eps = eps @@ -43,8 +43,9 @@ def reshape_weight_to_matrix(self, weight): if self.dim != 0: # transpose dim to front weight_mat = weight_mat.transpose( - [self.dim] + - [d for d in range(weight_mat.dim()) if d != self.dim]) + [self.dim] + + [d for d in range(weight_mat.dim()) if d != self.dim] + ) height = weight_mat.shape[0] @@ -61,20 +62,24 @@ def compute_weight(self, layer, do_power_iteration): for _ in range(self.n_power_iterations): v.set_value( F.normalize( - paddle.matmul(weight_mat, - u, - transpose_x=True, - transpose_y=False), + paddle.matmul( + weight_mat, + u, + transpose_x=True, + transpose_y=False, + ), axis=0, epsilon=self.eps, - )) + ) + ) u.set_value( F.normalize( paddle.matmul(weight_mat, v), axis=0, epsilon=self.eps, - )) + ) + ) if self.n_power_iterations > 0: u = u.clone() v = v.clone() @@ -84,15 +89,20 @@ def compute_weight(self, layer, do_power_iteration): return weight def __call__(self, layer, inputs): - setattr(layer, self.name, - self.compute_weight(layer, do_power_iteration=layer.training)) + setattr( + layer, + self.name, + self.compute_weight(layer, do_power_iteration=layer.training), + ) @staticmethod def apply(layer, name, n_power_iterations, dim, eps): for k, hook in layer._forward_pre_hooks.items(): if isinstance(hook, SpectralNorm) and hook.name == name: - raise RuntimeError("Cannot register two spectral_norm hooks on " - "the same parameter {}".format(name)) + raise RuntimeError( + "Cannot register two spectral_norm hooks on " + "the same parameter {}".format(name) + ) fn = SpectralNorm(name, n_power_iterations, dim, eps) weight = layer._parameters[name] @@ -103,9 +113,9 @@ def apply(layer, name, n_power_iterations, dim, eps): # randomly initialize u and v u = layer.create_parameter([h]) - u = normal_(u, 0., 1.) + u = normal_(u, 0.0, 1.0) v = layer.create_parameter([w]) - v = normal_(v, 0., 1.) + v = normal_(v, 0.0, 1.0) u = F.normalize(u, axis=0, epsilon=fn.eps) v = F.normalize(v, axis=0, epsilon=fn.eps) @@ -124,11 +134,9 @@ def apply(layer, name, n_power_iterations, dim, eps): return fn -def spectral_norm(layer, - name='weight', - n_power_iterations=1, - eps=1e-12, - dim=None): +def spectral_norm( + layer, name='weight', n_power_iterations=1, eps=1e-12, dim=None +): r""" Applies spectral normalization to a parameter according to the following Calculation: @@ -198,8 +206,8 @@ def spectral_norm(layer, if dim is None: if isinstance( - layer, - (Conv1DTranspose, Conv2DTranspose, Conv3DTranspose, Linear)): + layer, (Conv1DTranspose, Conv2DTranspose, Conv3DTranspose, Linear) + ): dim = 1 else: dim = 0 diff --git a/python/paddle/nn/utils/transform_parameters.py b/python/paddle/nn/utils/transform_parameters.py index 63ddd0974446a..4076b808dda74 100644 --- a/python/paddle/nn/utils/transform_parameters.py +++ b/python/paddle/nn/utils/transform_parameters.py @@ -15,11 +15,16 @@ from functools import reduce import paddle -from paddle.fluid.framework import dygraph_only, _dygraph_tracer, _varbase_creator, in_dygraph_mode +from paddle.fluid.framework import ( + dygraph_only, + _dygraph_tracer, + _varbase_creator, + in_dygraph_mode, +) from paddle import _C_ops -#input==output, inplace strategy of reshape has no cost almostly +# input==output, inplace strategy of reshape has no cost almostly def _inplace_reshape_dygraph(x, shape): x_shape = _varbase_creator(dtype='int64') if in_dygraph_mode(): @@ -27,14 +32,13 @@ def _inplace_reshape_dygraph(x, shape): tmp_out = _C_ops.reshape(x, shape) tmp_out._share_underline_tensor_to(x) else: - _dygraph_tracer().trace_op(type="reshape2", - inputs={'X': x}, - outputs={ - 'Out': x, - 'XShape': x_shape - }, - attrs={'shape': shape}, - stop_gradient=True) + _dygraph_tracer().trace_op( + type="reshape2", + inputs={'X': x}, + outputs={'Out': x, 'XShape': x_shape}, + attrs={'shape': shape}, + stop_gradient=True, + ) @dygraph_only @@ -106,11 +110,13 @@ def parameters_to_vector(parameters, name=None): tmp = _C_ops.concat(parameters, 0) tmp._share_underline_tensor_to(out) else: - _dygraph_tracer().trace_op(type='concat', - inputs={'X': parameters}, - outputs={'Out': [out]}, - attrs={'axis': 0}, - stop_gradient=True) + _dygraph_tracer().trace_op( + type='concat', + inputs={'X': parameters}, + outputs={'Out': [out]}, + attrs={'axis': 0}, + stop_gradient=True, + ) for i, param in enumerate(parameters): _inplace_reshape_dygraph(param, origin_shapes[i]) return out @@ -161,14 +167,13 @@ def vector_to_parameters(vec, parameters, name=None): for i in range(0, len(parameters)): res[i]._share_underline_tensor_to(parameters[i]) else: - _dygraph_tracer().trace_op(type='split', - inputs={'X': [vec]}, - outputs={'Out': parameters}, - attrs={ - 'axis': 0, - 'sections': sections - }, - stop_gradient=True) + _dygraph_tracer().trace_op( + type='split', + inputs={'X': [vec]}, + outputs={'Out': parameters}, + attrs={'axis': 0, 'sections': sections}, + stop_gradient=True, + ) for i, param in enumerate(parameters): _inplace_reshape_dygraph(param, origin_shapes[i]) diff --git a/python/paddle/nn/utils/weight_norm_hook.py b/python/paddle/nn/utils/weight_norm_hook.py index b836ad2843907..cdcb97aa9a0dc 100755 --- a/python/paddle/nn/utils/weight_norm_hook.py +++ b/python/paddle/nn/utils/weight_norm_hook.py @@ -34,16 +34,15 @@ def l2_norm(x, axis, epsilon=1e-12, name=None): helper = LayerHelper("l2_normalize", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) norm = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="norm", - inputs={"X": x}, - outputs={ - "Out": out, - "Norm": norm - }, - attrs={ - "axis": 1 if axis is None else axis, - "epsilon": epsilon, - }) + helper.append_op( + type="norm", + inputs={"X": x}, + outputs={"Out": out, "Norm": norm}, + attrs={ + "axis": 1 if axis is None else axis, + "epsilon": epsilon, + }, + ) return paddle.squeeze(norm, axis=[axis]) @@ -90,14 +89,13 @@ def _weight_norm(v, g, dim): v_normalized = F.l2_normalize(p_matrix, axis=1) v_normalized = paddle.reshape(v_normalized, transposed_shape) v_normalized = paddle.transpose(v_normalized, perm) - weight = F.elementwise_mul(v_normalized, - g, - axis=dim if dim is not None else -1) + weight = F.elementwise_mul( + v_normalized, g, axis=dim if dim is not None else -1 + ) return weight class WeightNorm(object): - def __init__(self, name, dim): if dim is None: dim = -1 @@ -113,8 +111,10 @@ def compute_weight(self, layer): def apply(layer, name, dim): for k, hook in layer._forward_pre_hooks.items(): if isinstance(hook, WeightNorm) and hook.name == name: - raise RuntimeError("Cannot register two weight_norm hooks on " - "the same parameter {}".format(name)) + raise RuntimeError( + "Cannot register two weight_norm hooks on " + "the same parameter {}".format(name) + ) if dim is None: dim = -1 diff --git a/python/paddle/onnx/export.py b/python/paddle/onnx/export.py index 5cdf0b06eece8..7123f485bf888 100644 --- a/python/paddle/onnx/export.py +++ b/python/paddle/onnx/export.py @@ -94,11 +94,14 @@ def export_logic(): raise ValueError( "The input path MUST be format of dirname/file_prefix " "[dirname\\file_prefix in Windows system], but " - "the file_prefix is empty in received path: {}".format(path)) + "the file_prefix is empty in received path: {}".format(path) + ) save_file = path + '.onnx' - p2o.dygraph2onnx(layer, - save_file, - input_spec=input_spec, - opset_version=opset_version, - **configs) + p2o.dygraph2onnx( + layer, + save_file, + input_spec=input_spec, + opset_version=opset_version, + **configs + ) diff --git a/python/paddle/optimizer/__init__.py b/python/paddle/optimizer/__init__.py index cd75fd4906ea5..cef51897b20ab 100644 --- a/python/paddle/optimizer/__init__.py +++ b/python/paddle/optimizer/__init__.py @@ -24,7 +24,15 @@ from .lamb import Lamb # noqa: F401 from . import lr # noqa: F401 -__all__ = [ #noqa - 'Optimizer', 'Adagrad', 'Adam', 'AdamW', 'Adamax', 'RMSProp', 'Adadelta', - 'SGD', 'Momentum', 'Lamb' +__all__ = [ # noqa + 'Optimizer', + 'Adagrad', + 'Adam', + 'AdamW', + 'Adamax', + 'RMSProp', + 'Adadelta', + 'SGD', + 'Momentum', + 'Lamb', ] diff --git a/python/paddle/optimizer/adadelta.py b/python/paddle/optimizer/adadelta.py index 203ad8f617c3d..9587f4dd8ee43 100644 --- a/python/paddle/optimizer/adadelta.py +++ b/python/paddle/optimizer/adadelta.py @@ -107,25 +107,29 @@ class Adadelta(Optimizer): _avg_squared_grad_acc_str = "_avg_squared_grad" _avg_squared_update_acc_str = "_avg_squared_update" - def __init__(self, - learning_rate=0.001, - epsilon=1.0e-6, - rho=0.95, - parameters=None, - weight_decay=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate=0.001, + epsilon=1.0e-6, + rho=0.95, + parameters=None, + weight_decay=None, + grad_clip=None, + name=None, + ): if learning_rate is None: raise ValueError("learning_rate is not set.") if epsilon is None: raise ValueError("epsilon is not set.") if rho is None: raise ValueError("rho is not set.") - super(Adadelta, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=weight_decay, - grad_clip=grad_clip, - name=name) + super(Adadelta, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=weight_decay, + grad_clip=grad_clip, + name=name, + ) self.type = "adadelta" self._epsilon = epsilon self._rho = rho @@ -149,43 +153,44 @@ def _append_optimize_op(self, block, param_and_grad): param_and_grad = self._update_param_group(param_and_grad) avg_squared_grad_acc = self._get_accumulator( - self._avg_squared_grad_acc_str, param_and_grad[0]) + self._avg_squared_grad_acc_str, param_and_grad[0] + ) avg_squared_update_acc = self._get_accumulator( - self._avg_squared_update_acc_str, param_and_grad[0]) + self._avg_squared_update_acc_str, param_and_grad[0] + ) if in_dygraph_mode(): with no_grad(): - _C_ops.adadelta_(param_and_grad[0], param_and_grad[1], - avg_squared_grad_acc, avg_squared_update_acc, - self._rho, self._epsilon) + _C_ops.adadelta_( + param_and_grad[0], + param_and_grad[1], + avg_squared_grad_acc, + avg_squared_update_acc, + self._rho, + self._epsilon, + ) return None if not isinstance(block, framework.Block): raise TypeError("block is not instance of framework.Block.") # Create the adadelta optimizer op - adadelta_op = block.append_op(type=self.type, - inputs={ - "Param": param_and_grad[0], - "Grad": param_and_grad[1], - "AvgSquaredGrad": - avg_squared_grad_acc, - "AvgSquaredUpdate": - avg_squared_update_acc - }, - outputs={ - "ParamOut": - param_and_grad[0], - "AvgSquaredGradOut": - avg_squared_grad_acc, - "AvgSquaredUpdateOut": - avg_squared_update_acc - }, - attrs={ - "epsilon": self._epsilon, - "rho": self._rho - }, - stop_gradient=True) + adadelta_op = block.append_op( + type=self.type, + inputs={ + "Param": param_and_grad[0], + "Grad": param_and_grad[1], + "AvgSquaredGrad": avg_squared_grad_acc, + "AvgSquaredUpdate": avg_squared_update_acc, + }, + outputs={ + "ParamOut": param_and_grad[0], + "AvgSquaredGradOut": avg_squared_grad_acc, + "AvgSquaredUpdateOut": avg_squared_update_acc, + }, + attrs={"epsilon": self._epsilon, "rho": self._rho}, + stop_gradient=True, + ) return adadelta_op diff --git a/python/paddle/optimizer/adagrad.py b/python/paddle/optimizer/adagrad.py index e0296a1cb0a6c..a4d9416e93bcc 100644 --- a/python/paddle/optimizer/adagrad.py +++ b/python/paddle/optimizer/adagrad.py @@ -106,21 +106,25 @@ class Adagrad(Optimizer): """ _moment_acc_str = "moment" - def __init__(self, - learning_rate, - epsilon=1.0e-6, - parameters=None, - weight_decay=None, - grad_clip=None, - name=None, - initial_accumulator_value=0.0): + def __init__( + self, + learning_rate, + epsilon=1.0e-6, + parameters=None, + weight_decay=None, + grad_clip=None, + name=None, + initial_accumulator_value=0.0, + ): assert learning_rate is not None assert epsilon is not None - super(Adagrad, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=weight_decay, - grad_clip=grad_clip, - name=name) + super(Adagrad, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=weight_decay, + grad_clip=grad_clip, + name=name, + ) self.type = "adagrad" self._epsilon = epsilon self.initial_accumulator_value = initial_accumulator_value @@ -136,9 +140,11 @@ def _create_accumulators(self, block, parameters): parameters = self._update_param_group(parameters) for p in parameters: - self._add_accumulator(self._moment_acc_str, - p, - fill_value=self.initial_accumulator_value) + self._add_accumulator( + self._moment_acc_str, + p, + fill_value=self.initial_accumulator_value, + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -146,26 +152,22 @@ def _append_optimize_op(self, block, param_and_grad): if isinstance(param_and_grad, dict): param_and_grad = self._update_param_group(param_and_grad) - moment_acc = self._get_accumulator(self._moment_acc_str, - param_and_grad[0]) + moment_acc = self._get_accumulator( + self._moment_acc_str, param_and_grad[0] + ) # Create the adagrad optimizer op - adagrad_op = block.append_op(type=self.type, - inputs={ - "Param": - param_and_grad[0], - "Grad": - param_and_grad[1], - "Moment": - moment_acc, - "LearningRate": - self._create_param_lr(param_and_grad) - }, - outputs={ - "ParamOut": param_and_grad[0], - "MomentOut": moment_acc - }, - attrs={"epsilon": self._epsilon}, - stop_gradient=True) + adagrad_op = block.append_op( + type=self.type, + inputs={ + "Param": param_and_grad[0], + "Grad": param_and_grad[1], + "Moment": moment_acc, + "LearningRate": self._create_param_lr(param_and_grad), + }, + outputs={"ParamOut": param_and_grad[0], "MomentOut": moment_acc}, + attrs={"epsilon": self._epsilon}, + stop_gradient=True, + ) return adagrad_op @@ -173,6 +175,7 @@ def _update_param_group(self, parameters): self._epsilon = parameters.get('epsilon', self._default_dict['epsilon']) self.initial_accumulator_value = parameters.get( 'initial_accumulator_value', - self._default_dict['initial_accumulator_value']) + self._default_dict['initial_accumulator_value'], + ) parameters = parameters.get('params') return parameters diff --git a/python/paddle/optimizer/adam.py b/python/paddle/optimizer/adam.py index a49af1a458988..c6e03ccf64cc5 100644 --- a/python/paddle/optimizer/adam.py +++ b/python/paddle/optimizer/adam.py @@ -161,18 +161,20 @@ class Adam(Optimizer): _beta1_pow_acc_str = "beta1_pow_acc" _beta2_pow_acc_str = "beta2_pow_acc" - def __init__(self, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - parameters=None, - weight_decay=None, - grad_clip=None, - lazy_mode=False, - multi_precision=False, - use_multi_tensor=False, - name=None): + def __init__( + self, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + parameters=None, + weight_decay=None, + grad_clip=None, + lazy_mode=False, + multi_precision=False, + use_multi_tensor=False, + name=None, + ): assert learning_rate is not None assert beta1 is not None assert beta2 is not None @@ -180,20 +182,25 @@ def __init__(self, if not isinstance(beta1, Variable): if not 0 <= beta1 < 1: raise ValueError( - "Invaild value of beta1, expect beta1 in [0,1).") + "Invaild value of beta1, expect beta1 in [0,1)." + ) if not isinstance(beta2, Variable): if not 0 <= beta2 < 1: raise ValueError( - "Invaild value of beta2, expect beta2 in [0,1).") + "Invaild value of beta2, expect beta2 in [0,1)." + ) if not isinstance(epsilon, Variable): if not 0 <= epsilon: raise ValueError( - "Invaild value of epsilon, expect epsilon >= 0.") - super(Adam, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=weight_decay, - grad_clip=grad_clip, - name=name) + "Invaild value of epsilon, expect epsilon >= 0." + ) + super(Adam, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=weight_decay, + grad_clip=grad_clip, + name=name, + ) self.type = "adam" self._beta1 = beta1 self._beta2 = beta2 @@ -215,15 +222,15 @@ def __init__(self, self._moment2_dict = {'FP32_LODTensor': [], 'FP16_LODTensor': []} self._beta1_pow_acc_dict = { 'FP32_LODTensor': [], - 'FP16_LODTensor': [] + 'FP16_LODTensor': [], } self._beta2_pow_acc_dict = { 'FP32_LODTensor': [], - 'FP16_LODTensor': [] + 'FP16_LODTensor': [], } self._master_weight_dict = { 'FP32_LODTensor': None, - 'FP16_LODTensor': [] + 'FP16_LODTensor': [], } def _create_master_weight(self, param): @@ -234,19 +241,23 @@ def _create_master_weight(self, param): var_name = param.name + "_fp32_master" var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -260,20 +271,30 @@ def _get_accumulator(self, name, param): """ if self._name is not None: name = self._name + "_" + name - find_master = self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 - target_param = self._master_weights[ - param.name] if find_master else param + find_master = ( + self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 + ) + target_param = ( + self._master_weights[param.name] if find_master else param + ) target_name = target_param.name - if (name not in self._accumulators - or target_name not in self._accumulators[name]): + if ( + name not in self._accumulators + or target_name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, target_name)) + name, target_name + ) + ) return self._accumulators[name][target_name] def _add_moments_pows(self, p): acc_dtype = p.dtype - if acc_dtype == core.VarDesc.VarType.FP16 or acc_dtype == core.VarDesc.VarType.BF16: + if ( + acc_dtype == core.VarDesc.VarType.FP16 + or acc_dtype == core.VarDesc.VarType.BF16 + ): acc_dtype = core.VarDesc.VarType.FP32 self._add_accumulator(self._moment1_acc_str, p, dtype=acc_dtype) self._add_accumulator(self._moment2_acc_str, p, dtype=acc_dtype) @@ -281,18 +302,24 @@ def _add_moments_pows(self, p): name=self._beta1_pow_acc_str, param=p, dtype=acc_dtype, - fill_value=0.9 if isinstance(self._beta1, Variable) \ - else self._beta1, + fill_value=0.9 + if isinstance(self._beta1, Variable) + else self._beta1, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) self._add_accumulator( name=self._beta2_pow_acc_str, param=p, dtype=acc_dtype, - fill_value=0.999 if isinstance(self._beta2, Variable) \ - else self._beta2, + fill_value=0.999 + if isinstance(self._beta2, Variable) + else self._beta2, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) @@ -305,7 +332,10 @@ def _create_accumulators(self, block, parameters): master_p = self._create_master_weight(p) self._add_moments_pows(master_p) continue - if p.dtype == core.VarDesc.VarType.FP16 and not self._multi_precision: + if ( + p.dtype == core.VarDesc.VarType.FP16 + and not self._multi_precision + ): warnings.warn( "Accumulating with FP16 in optimizer can lead to poor accuracy or slow convergence." "Consider using multi_precision=True option of the Adam optimizer." @@ -317,50 +347,105 @@ def _append_optimize_op(self, block, param_and_grad): if isinstance(param_and_grad, dict): param_and_grad = self._update_param_group(param_and_grad) - moment1 = self._get_accumulator(self._moment1_acc_str, - param_and_grad[0]) - moment2 = self._get_accumulator(self._moment2_acc_str, - param_and_grad[0]) - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) - beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str, - param_and_grad[0]) - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 - master_weight = (self._master_weights[param_and_grad[0].name] - if find_master else None) + moment1 = self._get_accumulator( + self._moment1_acc_str, param_and_grad[0] + ) + moment2 = self._get_accumulator( + self._moment2_acc_str, param_and_grad[0] + ) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) + beta2_pow_acc = self._get_accumulator( + self._beta2_pow_acc_str, param_and_grad[0] + ) + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) + master_weight = ( + self._master_weights[param_and_grad[0].name] + if find_master + else None + ) lr = self._create_param_lr(param_and_grad) # create the adam optimize op if framework.in_dygraph_mode(): found_inf = self._get_auxiliary_var('found_inf') - _beta1 = self._beta1 if not isinstance( - self._beta1, Variable) else self._beta1.numpy().item(0) - _beta2 = self._beta2 if not isinstance( - self._beta2, Variable) else self._beta2.numpy().item(0) + _beta1 = ( + self._beta1 + if not isinstance(self._beta1, Variable) + else self._beta1.numpy().item(0) + ) + _beta2 = ( + self._beta2 + if not isinstance(self._beta2, Variable) + else self._beta2.numpy().item(0) + ) _, _, _, _, _, _ = _C_ops.adam_( - param_and_grad[0], param_and_grad[1], lr, moment1, moment2, - beta1_pow_acc, beta2_pow_acc, master_weight, found_inf, _beta1, - _beta2, self._epsilon, self._lazy_mode, 1000, find_master, - False) + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + found_inf, + _beta1, + _beta2, + self._epsilon, + self._lazy_mode, + 1000, + find_master, + False, + ) return None if framework._in_legacy_dygraph(): - _beta1 = self._beta1 if not isinstance( - self._beta1, Variable) else self._beta1.numpy().item(0) - _beta2 = self._beta2 if not isinstance( - self._beta2, Variable) else self._beta2.numpy().item(0) + _beta1 = ( + self._beta1 + if not isinstance(self._beta1, Variable) + else self._beta1.numpy().item(0) + ) + _beta2 = ( + self._beta2 + if not isinstance(self._beta2, Variable) + else self._beta2.numpy().item(0) + ) _, _, _, _, _, _ = _legacy_C_ops.adam( - param_and_grad[0], param_and_grad[1], lr, moment1, moment2, - beta1_pow_acc, beta2_pow_acc, master_weight, param_and_grad[0], - moment1, moment2, beta1_pow_acc, beta2_pow_acc, master_weight, - 'epsilon', self._epsilon, 'lazy_mode', self._lazy_mode, - 'min_row_size_to_use_multithread', 1000, 'beta1', _beta1, - 'beta2', _beta2, 'multi_precision', find_master) + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + param_and_grad[0], + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + 'epsilon', + self._epsilon, + 'lazy_mode', + self._lazy_mode, + 'min_row_size_to_use_multithread', + 1000, + 'beta1', + _beta1, + 'beta2', + _beta2, + 'multi_precision', + find_master, + ) return None @@ -371,7 +456,7 @@ def _append_optimize_op(self, block, param_and_grad): "Moment1": [moment1], "Moment2": [moment2], "Beta1Pow": [beta1_pow_acc], - "Beta2Pow": [beta2_pow_acc] + "Beta2Pow": [beta2_pow_acc], } outputs = { "ParamOut": [param_and_grad[0]], @@ -383,7 +468,7 @@ def _append_optimize_op(self, block, param_and_grad): attrs = { "lazy_mode": self._lazy_mode, "min_row_size_to_use_multithread": 1000, - "multi_precision": find_master + "multi_precision": find_master, } if isinstance(self._beta1, Variable): @@ -403,11 +488,13 @@ def _append_optimize_op(self, block, param_and_grad): inputs["MasterParam"] = master_weight outputs["MasterParamOut"] = master_weight - adam_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + adam_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return adam_op @@ -443,24 +530,28 @@ def step(self): if param._grad_ivar() is not None: grad_var = param._grad_ivar() if in_dygraph_mode(): - if hasattr(grad_var, "is_selected_rows" - ) and grad_var.is_selected_rows( - ) and self.regularization is not None: + if ( + hasattr(grad_var, "is_selected_rows") + and grad_var.is_selected_rows() + and self.regularization is not None + ): raise RuntimeError( "Adam don't support weight_decay with sparse parameters, please set it to None." ) else: - if hasattr( - grad_var, "_is_sparse") and grad_var._is_sparse( - ) and self.regularization is not None: + if ( + hasattr(grad_var, "_is_sparse") + and grad_var._is_sparse() + and self.regularization is not None + ): raise RuntimeError( "Adam don't support weight_decay with sparse parameters, please set it to None." ) params_grads.append((param, grad_var)) - optimize_ops = self._apply_optimize(loss=None, - startup_program=None, - params_grads=params_grads) + optimize_ops = self._apply_optimize( + loss=None, startup_program=None, params_grads=params_grads + ) else: # optimize parameters in groups for param_group in self._param_groups: @@ -472,11 +563,11 @@ def step(self): grad_var = param._grad_ivar() params_grads['params'].append((param, grad_var)) params_grads.update( - {k: v - for k, v in param_group.items() if k != 'params'}) - self._apply_optimize(loss=None, - startup_program=None, - params_grads=params_grads) + {k: v for k, v in param_group.items() if k != 'params'} + ) + self._apply_optimize( + loss=None, startup_program=None, params_grads=params_grads + ) def _multi_tensor_init(self, target_block, parameters): """ @@ -490,10 +581,12 @@ def _multi_tensor_init(self, target_block, parameters): for param in parameters: moment1 = self._get_accumulator(self._moment1_acc_str, param) moment2 = self._get_accumulator(self._moment2_acc_str, param) - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param) - beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str, - param) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param + ) + beta2_pow_acc = self._get_accumulator( + self._beta2_pow_acc_str, param + ) if param.dtype == paddle.float32: self._param_dict['FP32_LODTensor'].append(param) @@ -509,7 +602,8 @@ def _multi_tensor_init(self, target_block, parameters): self._beta2_pow_acc_dict['FP16_LODTensor'].append(beta2_pow_acc) if self._multi_precision: self._master_weight_dict['FP16_LODTensor'].append( - self._master_weights[param.name]) + self._master_weights[param.name] + ) else: self._master_weight_dict['FP16_LODTensor'] = None else: @@ -517,8 +611,9 @@ def _multi_tensor_init(self, target_block, parameters): "Now multi_tensor_momentum only support fp32 and fp16 parameters and grad is LOD_TENSOR." ) - def _append_optimize_multi_tensor_op(self, target_block, - parameters_and_grads): + def _append_optimize_multi_tensor_op( + self, target_block, parameters_and_grads + ): """ For Multi Tensor, append optimize merged_operator to block. """ @@ -532,15 +627,19 @@ def _append_optimize_multi_tensor_op(self, target_block, if param_and_grad[1] is None: continue if param_and_grad[0].stop_gradient is False: - if param_and_grad[ - 0].dtype == paddle.float32 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + if ( + param_and_grad[0].dtype == paddle.float32 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP32_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP32_LODTensor'].append(lr) - elif param_and_grad[ - 0].dtype == paddle.float16 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + elif ( + param_and_grad[0].dtype == paddle.float16 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP16_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP16_LODTensor'].append(lr) @@ -551,21 +650,27 @@ def _append_optimize_multi_tensor_op(self, target_block, if param_and_grad[0].stop_gradient is False: param_grad_dict = dict() param_grad_dict['params'] = param_and_grad - param_grad_dict.update({ - k: v - for k, v in parameters_and_grads.items() - if k != 'params' - }) + param_grad_dict.update( + { + k: v + for k, v in parameters_and_grads.items() + if k != 'params' + } + ) param_and_grad = self._update_param_group(param_grad_dict) - if param_and_grad[ - 0].dtype == paddle.float32 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + if ( + param_and_grad[0].dtype == paddle.float32 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP32_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP32_LODTensor'].append(lr) - elif param_and_grad[ - 0].dtype == paddle.float16 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + elif ( + param_and_grad[0].dtype == paddle.float16 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP16_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP16_LODTensor'].append(lr) @@ -575,34 +680,59 @@ def _append_optimize_multi_tensor_op(self, target_block, if len(self._param_dict[key]) > 0: find_master = self._multi_precision and key == 'FP16_LODTensor' - _beta1 = self._beta1 if not isinstance( - self._beta1, Variable) else self._beta1.numpy().item(0) - _beta2 = self._beta2 if not isinstance( - self._beta2, Variable) else self._beta2.numpy().item(0) + _beta1 = ( + self._beta1 + if not isinstance(self._beta1, Variable) + else self._beta1.numpy().item(0) + ) + _beta2 = ( + self._beta2 + if not isinstance(self._beta2, Variable) + else self._beta2.numpy().item(0) + ) if framework._non_static_mode(): if in_dygraph_mode(): _, _, _, _, _, _ = _C_ops.merged_adam_( - self._param_dict[key], grad_dict[key], lr_dict[key], - self._moment1_dict[key], self._moment2_dict[key], + self._param_dict[key], + grad_dict[key], + lr_dict[key], + self._moment1_dict[key], + self._moment2_dict[key], self._beta1_pow_acc_dict[key], self._beta2_pow_acc_dict[key], - self._master_weight_dict[key], _beta1, _beta2, - self._epsilon, find_master, False) + self._master_weight_dict[key], + _beta1, + _beta2, + self._epsilon, + find_master, + False, + ) else: _, _, _, _, _, _ = _legacy_C_ops.merged_adam( - self._param_dict[key], grad_dict[key], lr_dict[key], - self._moment1_dict[key], self._moment2_dict[key], + self._param_dict[key], + grad_dict[key], + lr_dict[key], + self._moment1_dict[key], + self._moment2_dict[key], self._beta1_pow_acc_dict[key], self._beta2_pow_acc_dict[key], self._master_weight_dict[key], - self._param_dict[key], self._moment1_dict[key], + self._param_dict[key], + self._moment1_dict[key], self._moment2_dict[key], self._beta1_pow_acc_dict[key], self._beta2_pow_acc_dict[key], - self._master_weight_dict[key], 'epsilon', - self._epsilon, 'beta1', _beta1, 'beta2', _beta2, - 'multi_precision', find_master) + self._master_weight_dict[key], + 'epsilon', + self._epsilon, + 'beta1', + _beta1, + 'beta2', + _beta2, + 'multi_precision', + find_master, + ) else: inputs = { "Param": self._param_dict[key], @@ -611,37 +741,41 @@ def _append_optimize_multi_tensor_op(self, target_block, "Moment1": self._moment1_dict[key], "Moment2": self._moment2_dict[key], "Beta1Pow": self._beta1_pow_acc_dict[key], - "Beta2Pow": self._beta2_pow_acc_dict[key] + "Beta2Pow": self._beta2_pow_acc_dict[key], } outputs = { "ParamOut": self._param_dict[key], "Moment1Out": self._moment1_dict[key], "Moment2Out": self._moment2_dict[key], "Beta1PowOut": self._beta1_pow_acc_dict[key], - "Beta2PowOut": self._beta2_pow_acc_dict[key] + "Beta2PowOut": self._beta2_pow_acc_dict[key], } attrs = { "epsilon": self._epsilon, "beta1": _beta1, - "beta2": _beta2 + "beta2": _beta2, } if find_master: inputs["MasterParam"] = self._master_weight_dict[key] outputs["MasterParamOut"] = self._master_weight_dict[ - key] + key + ] attrs["multi_precision"] = find_master - target_block.append_op(type="merged_adam", - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + target_block.append_op( + type="merged_adam", + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return None def _update_param_group(self, parameters): self._beta1 = parameters.get('beta1', self._default_dict['beta1']) self._beta2 = parameters.get('beta2', self._default_dict['beta2']) self._epsilon = parameters.get('epsilon', self._default_dict['epsilon']) - self._lazy_mode = parameters.get('lazy_mode', - self._default_dict['lazy_mode']) + self._lazy_mode = parameters.get( + 'lazy_mode', self._default_dict['lazy_mode'] + ) parameters = parameters.get('params') return parameters diff --git a/python/paddle/optimizer/adamax.py b/python/paddle/optimizer/adamax.py index 81ae8531f0e13..0a809e6acf8db 100644 --- a/python/paddle/optimizer/adamax.py +++ b/python/paddle/optimizer/adamax.py @@ -132,15 +132,17 @@ class Adamax(Optimizer): _inf_norm_acc_str = "inf_norm" _beta1_pow_acc_str = "beta1_pow_acc" - def __init__(self, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - parameters=None, - weight_decay=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + parameters=None, + weight_decay=None, + grad_clip=None, + name=None, + ): assert learning_rate is not None assert beta1 is not None assert beta2 is not None @@ -151,11 +153,13 @@ def __init__(self, raise ValueError("Invaild value of beta2, expect beta2 in [0,1).") if not 0 <= epsilon: raise ValueError("Invaild value of epsilon, expect epsilon >= 0.") - super(Adamax, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=weight_decay, - grad_clip=grad_clip, - name=name) + super(Adamax, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=weight_decay, + grad_clip=grad_clip, + name=name, + ) self.type = "adamax" self._beta1 = beta1 self._beta2 = beta2 @@ -163,7 +167,7 @@ def __init__(self, self._default_dict = { 'beta1': beta1, 'beta2': beta2, - 'epsilon': epsilon + 'epsilon': epsilon, } def _create_accumulators(self, block, parameters): @@ -174,10 +178,12 @@ def _create_accumulators(self, block, parameters): for p in parameters: self._add_accumulator(self._moment_acc_str, p) self._add_accumulator(self._inf_norm_acc_str, p) - self._add_accumulator(name=self._beta1_pow_acc_str, - param=p, - fill_value=self._beta1, - shape=[1]) + self._add_accumulator( + name=self._beta1_pow_acc_str, + param=p, + fill_value=self._beta1, + shape=[1], + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -185,22 +191,43 @@ def _append_optimize_op(self, block, param_and_grad): param_and_grad = self._update_param_group(param_and_grad) moment = self._get_accumulator(self._moment_acc_str, param_and_grad[0]) - inf_norm = self._get_accumulator(self._inf_norm_acc_str, - param_and_grad[0]) - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) + inf_norm = self._get_accumulator( + self._inf_norm_acc_str, param_and_grad[0] + ) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) if framework.in_dygraph_mode(): - _C_ops.adamax_(param_and_grad[0], param_and_grad[1], - self._create_param_lr(param_and_grad), moment, - inf_norm, beta1_pow_acc, self._beta1, self._beta2, - self._epsilon) + _C_ops.adamax_( + param_and_grad[0], + param_and_grad[1], + self._create_param_lr(param_and_grad), + moment, + inf_norm, + beta1_pow_acc, + self._beta1, + self._beta2, + self._epsilon, + ) elif framework._in_legacy_dygraph(): - _legacy_C_ops.adamax(param_and_grad[0], param_and_grad[1], - self._create_param_lr(param_and_grad), moment, - inf_norm, beta1_pow_acc, param_and_grad[0], - moment, inf_norm, "beta1", self._beta1, - "beta2", self._beta2, "epsilon", self._epsilon) + _legacy_C_ops.adamax( + param_and_grad[0], + param_and_grad[1], + self._create_param_lr(param_and_grad), + moment, + inf_norm, + beta1_pow_acc, + param_and_grad[0], + moment, + inf_norm, + "beta1", + self._beta1, + "beta2", + self._beta2, + "epsilon", + self._epsilon, + ) else: # create the adamax optimize op adamax_op = block.append_op( @@ -211,25 +238,25 @@ def _append_optimize_op(self, block, param_and_grad): "LearningRate": self._create_param_lr(param_and_grad), "Moment": moment, "InfNorm": inf_norm, - "Beta1Pow": beta1_pow_acc + "Beta1Pow": beta1_pow_acc, }, outputs={ "ParamOut": param_and_grad[0], "MomentOut": moment, - "InfNormOut": inf_norm + "InfNormOut": inf_norm, }, attrs={ "beta1": self._beta1, "beta2": self._beta2, - "epsilon": self._epsilon + "epsilon": self._epsilon, }, - stop_gradient=True) + stop_gradient=True, + ) return adamax_op def _finish_update(self, block, parameters_and_grads): - """Update Beta1 Power accumulator - """ + """Update Beta1 Power accumulator""" assert isinstance(block, framework.Block) if isinstance(parameters_and_grads, list): for param, grad in parameters_and_grads: @@ -237,47 +264,61 @@ def _finish_update(self, block, parameters_and_grads): continue if framework.in_dygraph_mode(): beta1_pow_acc = self._get_accumulator( - self._beta1_pow_acc_str, param) + self._beta1_pow_acc_str, param + ) with no_grad(): - tmp = _C_ops.scale(beta1_pow_acc, self._beta1, 0.0, - True) + tmp = _C_ops.scale( + beta1_pow_acc, self._beta1, 0.0, True + ) beta1_pow_acc.copy_(tmp, False) continue with param.block.program._optimized_guard( - [param, grad]), name_scope('adamax'): + [param, grad] + ), name_scope('adamax'): beta1_pow_acc = self._get_accumulator( - self._beta1_pow_acc_str, param) - block.append_op(type="scale", - inputs={"X": beta1_pow_acc}, - outputs={"Out": beta1_pow_acc}, - attrs={"scale": self._beta1}, - stop_gradient=True) + self._beta1_pow_acc_str, param + ) + block.append_op( + type="scale", + inputs={"X": beta1_pow_acc}, + outputs={"Out": beta1_pow_acc}, + attrs={"scale": self._beta1}, + stop_gradient=True, + ) else: for param, grad in parameters_and_grads['params']: if grad is None or param.stop_gradient is True: continue if framework.in_dygraph_mode(): beta1_pow_acc = self._get_accumulator( - self._beta1_pow_acc_str, param) + self._beta1_pow_acc_str, param + ) self._beta1 = parameters_and_grads.get( - 'beta1', self._default_dict['beta1']) + 'beta1', self._default_dict['beta1'] + ) with no_grad(): - tmp = _C_ops.scale(beta1_pow_acc, self._beta1, 0.0, - True) + tmp = _C_ops.scale( + beta1_pow_acc, self._beta1, 0.0, True + ) beta1_pow_acc.copy_(tmp, False) continue with param.block.program._optimized_guard( - [param, grad]), name_scope('adamax'): + [param, grad] + ), name_scope('adamax'): beta1_pow_acc = self._get_accumulator( - self._beta1_pow_acc_str, param) + self._beta1_pow_acc_str, param + ) self._beta1 = parameters_and_grads.get( - 'beta1', self._default_dict['beta1']) - block.append_op(type="scale", - inputs={"X": beta1_pow_acc}, - outputs={"Out": beta1_pow_acc}, - attrs={"scale": self._beta1}, - stop_gradient=True) + 'beta1', self._default_dict['beta1'] + ) + block.append_op( + type="scale", + inputs={"X": beta1_pow_acc}, + outputs={"Out": beta1_pow_acc}, + attrs={"scale": self._beta1}, + stop_gradient=True, + ) def _update_param_group(self, parameters): self._beta1 = parameters.get('beta1', self._default_dict['beta1']) diff --git a/python/paddle/optimizer/adamw.py b/python/paddle/optimizer/adamw.py index 1c0dbb3134c80..dca844b668275 100644 --- a/python/paddle/optimizer/adamw.py +++ b/python/paddle/optimizer/adamw.py @@ -148,19 +148,21 @@ class AdamW(Optimizer): _beta1_pow_acc_str = "beta1_pow_acc" _beta2_pow_acc_str = "beta2_pow_acc" - def __init__(self, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - parameters=None, - weight_decay=0.01, - lr_ratio=None, - apply_decay_param_fun=None, - grad_clip=None, - lazy_mode=False, - multi_precision=False, - name=None): + def __init__( + self, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + parameters=None, + weight_decay=0.01, + lr_ratio=None, + apply_decay_param_fun=None, + grad_clip=None, + lazy_mode=False, + multi_precision=False, + name=None, + ): assert learning_rate is not None assert beta1 is not None assert beta2 is not None @@ -171,14 +173,16 @@ def __init__(self, raise ValueError("Invaild value of beta2, expect beta2 in [0,1).") if not 0 <= epsilon: raise ValueError("Invaild value of epsilon, expect epsilon >= 0.") - if not isinstance(weight_decay, float) and \ - not isinstance(weight_decay, framework.Variable): + if not isinstance(weight_decay, float) and not isinstance( + weight_decay, framework.Variable + ): raise TypeError("weight_decay should be float or Tensor.") if lr_ratio is not None: assert isinstance(lr_ratio, Callable) if not core.is_compiled_with_cuda(): raise NotImplementedError( - "'lr_ratio' is unimplemented in CPU, XPU and NPU") + "'lr_ratio' is unimplemented in CPU, XPU and NPU" + ) if parameters is not None: # paddle.Tensor is also iterable, so here we don't check whether @@ -187,13 +191,16 @@ def __init__(self, if isinstance(parameters, (paddle.Tensor, core.eager.Tensor)): raise TypeError( "`parameters` argument given to the optimizer should be " - "an iterable of paddle Tensors, but got argument type is `{}`." - .format(type(parameters))) + "an iterable of paddle Tensors, but got argument type is `{}`.".format( + type(parameters) + ) + ) if isinstance(parameters, dict): raise TypeError( "`parameters` argument should not get dict type, " "if parameter groups is needed, please set `parameters`" - " as list of dict") + " as list of dict" + ) self._parameter_list = list(parameters) else: self._parameter_list = None @@ -207,8 +214,9 @@ def __init__(self, if not isinstance(learning_rate, (float, LRScheduler)): raise TypeError( - "learning rate should be float or LRScheduler, got %s here" % - type(learning_rate)) + "learning rate should be float or LRScheduler, got %s here" + % type(learning_rate) + ) if grad_clip is not None: if not isinstance(grad_clip, GradientClipBase): raise TypeError( @@ -220,8 +228,9 @@ def __init__(self, if self._parameter_list: if isinstance(self._parameter_list[0], dict): for param_group in self._parameter_list: - assert 'params' in param_group, \ - 'params should be set in parameters if parameter groups are optimized in different options' + assert ( + 'params' in param_group + ), 'params should be set in parameters if parameter groups are optimized in different options' self._dtype = self._parameter_list[0]['params'][0].dtype else: self._dtype = self._parameter_list[0].dtype @@ -260,7 +269,7 @@ def __init__(self, 'beta2': beta2, 'epsilon': epsilon, 'lazy_mode': lazy_mode, - 'grad_clip': grad_clip + 'grad_clip': grad_clip, } self._param_groups = [] @@ -297,7 +306,8 @@ def _add_param_group(self, param_group): elif isinstance(params, set): raise TypeError( "optimizer parameters should be in ordered collections," - "but received set, please use list instead.") + "but received set, please use list instead." + ) else: param_group['params'] = list(params) @@ -311,11 +321,13 @@ def _add_param_group(self, param_group): if not param_set.isdisjoint(set(param_group['params'])): raise ValueError( - "some parameters appear in more than one parameter group") + "some parameters appear in more than one parameter group" + ) for param in param_group['params']: param.optimize_attr['learning_rate'] = param_group.get( - 'learning_rate', 1.) + 'learning_rate', 1.0 + ) self._param_groups.append(param_group) @@ -327,19 +339,23 @@ def _create_master_weight(self, param): var_name = param.name + "_fp32_master" var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -353,15 +369,22 @@ def _get_accumulator(self, name, param): """ if self._name is not None: name = self._name + "_" + name - find_master = self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 - target_param = self._master_weights[ - param.name] if find_master else param + find_master = ( + self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 + ) + target_param = ( + self._master_weights[param.name] if find_master else param + ) target_name = target_param.name - if (name not in self._accumulators - or target_name not in self._accumulators[name]): + if ( + name not in self._accumulators + or target_name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, target_name)) + name, target_name + ) + ) return self._accumulators[name][target_name] def _add_moments_pows(self, p): @@ -374,18 +397,24 @@ def _add_moments_pows(self, p): name=self._beta1_pow_acc_str, param=p, dtype=acc_dtype, - fill_value=0.9 if isinstance(self._beta1, Variable) \ - else self._beta1, + fill_value=0.9 + if isinstance(self._beta1, Variable) + else self._beta1, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) self._add_accumulator( name=self._beta2_pow_acc_str, param=p, dtype=acc_dtype, - fill_value=0.999 if isinstance(self._beta2, Variable) \ - else self._beta2, + fill_value=0.999 + if isinstance(self._beta2, Variable) + else self._beta2, shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) @@ -398,7 +427,10 @@ def _create_accumulators(self, block, parameters): master_p = self._create_master_weight(p) self._add_moments_pows(master_p) continue - if p.dtype == core.VarDesc.VarType.FP16 and not self._multi_precision: + if ( + p.dtype == core.VarDesc.VarType.FP16 + and not self._multi_precision + ): warnings.warn( "Accumulating with FP16 in optimizer can lead to poor accuracy or slow convergence." "Consider using multi_precision=True option of the Adam optimizer." @@ -413,53 +445,112 @@ def _append_optimize_op(self, block, param_and_grad): # Whether we should do weight decay for the parameter. with_decay = True - if self._apply_decay_param_fun is not None \ - and not self._apply_decay_param_fun(param.name): + if ( + self._apply_decay_param_fun is not None + and not self._apply_decay_param_fun(param.name) + ): with_decay = False - moment1 = self._get_accumulator(self._moment1_acc_str, - param_and_grad[0]) - moment2 = self._get_accumulator(self._moment2_acc_str, - param_and_grad[0]) - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) - beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str, - param_and_grad[0]) - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 - master_weight = (self._master_weights[param_and_grad[0].name] - if find_master else None) + moment1 = self._get_accumulator( + self._moment1_acc_str, param_and_grad[0] + ) + moment2 = self._get_accumulator( + self._moment2_acc_str, param_and_grad[0] + ) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) + beta2_pow_acc = self._get_accumulator( + self._beta2_pow_acc_str, param_and_grad[0] + ) + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) + master_weight = ( + self._master_weights[param_and_grad[0].name] + if find_master + else None + ) lr = self._create_param_lr(param_and_grad) # create the adamw optimize op if framework._non_static_mode(): - lr_ratio_ = 1. if self._lr_ratio is None else self._lr_ratio( - param_and_grad[0]) - - _beta1 = self._beta1 if not isinstance( - self._beta1, Variable) else self._beta1.numpy().item(0) - _beta2 = self._beta2 if not isinstance( - self._beta2, Variable) else self._beta2.numpy().item(0) + lr_ratio_ = ( + 1.0 + if self._lr_ratio is None + else self._lr_ratio(param_and_grad[0]) + ) + + _beta1 = ( + self._beta1 + if not isinstance(self._beta1, Variable) + else self._beta1.numpy().item(0) + ) + _beta2 = ( + self._beta2 + if not isinstance(self._beta2, Variable) + else self._beta2.numpy().item(0) + ) if framework.in_dygraph_mode(): found_inf = self._get_auxiliary_var('found_inf') _, _, _, _, _, _ = _C_ops.adamw_( - param_and_grad[0], param_and_grad[1], lr, moment1, moment2, - beta1_pow_acc, beta2_pow_acc, master_weight, found_inf, - _beta1, _beta2, self._epsilon, lr_ratio_, - self._weight_decay, with_decay, self._lazy_mode, 1000, - find_master, False) + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + found_inf, + _beta1, + _beta2, + self._epsilon, + lr_ratio_, + self._weight_decay, + with_decay, + self._lazy_mode, + 1000, + find_master, + False, + ) else: _, _, _, _, _, _ = _legacy_C_ops.adamw( - param_and_grad[0], param_and_grad[1], lr, moment1, moment2, - beta1_pow_acc, beta2_pow_acc, master_weight, - param_and_grad[0], moment1, moment2, beta1_pow_acc, - beta2_pow_acc, master_weight, 'epsilon', self._epsilon, - 'lazy_mode', self._lazy_mode, - 'min_row_size_to_use_multithread', 1000, 'beta1', _beta1, - 'beta2', _beta2, "with_decay", with_decay, 'coeff', - self._weight_decay, 'multi_precision', find_master, - 'lr_ratio', lr_ratio_) + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + param_and_grad[0], + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + 'epsilon', + self._epsilon, + 'lazy_mode', + self._lazy_mode, + 'min_row_size_to_use_multithread', + 1000, + 'beta1', + _beta1, + 'beta2', + _beta2, + "with_decay", + with_decay, + 'coeff', + self._weight_decay, + 'multi_precision', + find_master, + 'lr_ratio', + lr_ratio_, + ) return None inputs = { @@ -486,18 +577,14 @@ def _append_optimize_op(self, block, param_and_grad): "Beta2PowOut": [beta2_pow_acc], } attrs = { - "lazy_mode": - self._lazy_mode, - "min_row_size_to_use_multithread": - 1000, - "multi_precision": - find_master, - "with_decay": - with_decay, - "coeff": - self._weight_decay, - "lr_ratio": - 1. if self._lr_ratio is None else self._lr_ratio(param_and_grad[0]) + "lazy_mode": self._lazy_mode, + "min_row_size_to_use_multithread": 1000, + "multi_precision": find_master, + "with_decay": with_decay, + "coeff": self._weight_decay, + "lr_ratio": 1.0 + if self._lr_ratio is None + else self._lr_ratio(param_and_grad[0]), } if isinstance(self._beta1, Variable): @@ -517,11 +604,13 @@ def _append_optimize_op(self, block, param_and_grad): inputs["MasterParam"] = master_weight outputs["MasterParamOut"] = master_weight - adamw_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + adamw_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return adamw_op @@ -560,24 +649,28 @@ def step(self): if param._grad_ivar() is not None: grad_var = param._grad_ivar() if framework.in_dygraph_mode(): - if hasattr(grad_var, "is_selected_rows" - ) and grad_var.is_selected_rows( - ) and self.regularization is not None: + if ( + hasattr(grad_var, "is_selected_rows") + and grad_var.is_selected_rows() + and self.regularization is not None + ): raise RuntimeError( "AdamW don't support weight_decay with sparse parameters, please set it to None." ) else: - if hasattr( - grad_var, "_is_sparse") and grad_var._is_sparse( - ) and self.regularization is not None: + if ( + hasattr(grad_var, "_is_sparse") + and grad_var._is_sparse() + and self.regularization is not None + ): raise RuntimeError( "AdamW don't support weight_decay with sparse parameters, please set it to None." ) params_grads.append((param, grad_var)) - optimize_ops = self._apply_optimize(loss=None, - startup_program=None, - params_grads=params_grads) + optimize_ops = self._apply_optimize( + loss=None, startup_program=None, params_grads=params_grads + ) else: # optimize parameters in groups for param_group in self._param_groups: @@ -588,35 +681,41 @@ def step(self): if param._grad_ivar() is not None: grad_var = param._grad_ivar() if framework.in_dygraph_mode(): - if hasattr(grad_var, "is_selected_rows" - ) and grad_var.is_selected_rows( - ) and self.regularization is not None: + if ( + hasattr(grad_var, "is_selected_rows") + and grad_var.is_selected_rows() + and self.regularization is not None + ): raise RuntimeError( "AdamW don't support weight_decay with sparse parameters, please set it to None." ) else: - if hasattr(grad_var, - "_is_sparse") and grad_var._is_sparse( - ) and self.regularization is not None: + if ( + hasattr(grad_var, "_is_sparse") + and grad_var._is_sparse() + and self.regularization is not None + ): raise RuntimeError( "AdamW don't support weight_decay with sparse parameters, please set it to None." ) params_grads['params'].append((param, grad_var)) params_grads.update( - {k: v - for k, v in param_group.items() if k != 'params'}) - self._apply_optimize(loss=None, - startup_program=None, - params_grads=params_grads) + {k: v for k, v in param_group.items() if k != 'params'} + ) + self._apply_optimize( + loss=None, startup_program=None, params_grads=params_grads + ) def _update_param_group(self, parameters): self._beta1 = parameters.get('beta1', self._default_dict['beta1']) self._beta2 = parameters.get('beta2', self._default_dict['beta2']) self._epsilon = parameters.get('epsilon', self._default_dict['epsilon']) - self._lazy_mode = parameters.get('lazy_mode', - self._default_dict['lazy_mode']) - self._weight_decay = parameters.get('weight_decay', - self._default_dict['weight_decay']) + self._lazy_mode = parameters.get( + 'lazy_mode', self._default_dict['lazy_mode'] + ) + self._weight_decay = parameters.get( + 'weight_decay', self._default_dict['weight_decay'] + ) parameters = parameters.get('params') return parameters diff --git a/python/paddle/optimizer/lamb.py b/python/paddle/optimizer/lamb.py index 6a683c6ef9171..4fc46ad1052b4 100644 --- a/python/paddle/optimizer/lamb.py +++ b/python/paddle/optimizer/lamb.py @@ -98,26 +98,30 @@ class Lamb(Optimizer): _beta1_pow_acc_str = "beta1_pow_acc" _beta2_pow_acc_str = "beta2_pow_acc" - def __init__(self, - learning_rate=0.001, - lamb_weight_decay=0.01, - beta1=0.9, - beta2=0.999, - epsilon=1e-6, - parameters=None, - grad_clip=None, - exclude_from_weight_decay_fn=None, - multi_precision=False, - name=None): + def __init__( + self, + learning_rate=0.001, + lamb_weight_decay=0.01, + beta1=0.9, + beta2=0.999, + epsilon=1e-6, + parameters=None, + grad_clip=None, + exclude_from_weight_decay_fn=None, + multi_precision=False, + name=None, + ): assert learning_rate is not None assert beta1 is not None assert beta2 is not None assert epsilon is not None - super(Lamb, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=None, - grad_clip=grad_clip, - name=name) + super(Lamb, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=None, + grad_clip=grad_clip, + name=name, + ) self.type = "lamb" self._beta1 = beta1 self._beta2 = beta2 @@ -160,19 +164,23 @@ def _create_master_weight(self, param): var_name = param.name + "_fp32_master" var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -199,15 +207,22 @@ def _get_accumulator(self, name, param): """ if self._name is not None: name = self._name + "_" + name - find_master = self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 - target_param = self._master_weights[ - param.name] if find_master else param + find_master = ( + self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 + ) + target_param = ( + self._master_weights[param.name] if find_master else param + ) target_name = target_param.name - if (name not in self._accumulators - or target_name not in self._accumulators[name]): + if ( + name not in self._accumulators + or target_name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, target_name)) + name, target_name + ) + ) return self._accumulators[name][target_name] def _add_moments_pows(self, p): @@ -218,21 +233,27 @@ def _add_moments_pows(self, p): self._add_accumulator(self._moment1_acc_str, p, dtype=acc_dtype) self._add_accumulator(self._moment2_acc_str, p, dtype=acc_dtype) self._add_accumulator( - name=self._beta1_pow_acc_str, - param=p, - dtype=acc_dtype, - fill_value=0.9 if isinstance(self._beta1, Variable) \ - else self._beta1, - shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + name=self._beta1_pow_acc_str, + param=p, + dtype=acc_dtype, + fill_value=0.9 + if isinstance(self._beta1, Variable) + else self._beta1, + shape=[1], + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) self._add_accumulator( - name=self._beta2_pow_acc_str, - param=p, - dtype=acc_dtype, - fill_value=0.999 if isinstance(self._beta2, Variable) \ - else self._beta2, - shape=[1], - type=core.VarDesc.VarType.LOD_TENSOR, device='cpu') + name=self._beta2_pow_acc_str, + param=p, + dtype=acc_dtype, + fill_value=0.999 + if isinstance(self._beta2, Variable) + else self._beta2, + shape=[1], + type=core.VarDesc.VarType.LOD_TENSOR, + device='cpu', + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -241,24 +262,32 @@ def _append_optimize_op(self, block, param_and_grad): block.program._use_lamb = True - moment1 = self._get_accumulator(self._moment1_acc_str, - param_and_grad[0]) - moment2 = self._get_accumulator(self._moment2_acc_str, - param_and_grad[0]) - beta1_pow_acc = self._get_accumulator(self._beta1_pow_acc_str, - param_and_grad[0]) - beta2_pow_acc = self._get_accumulator(self._beta2_pow_acc_str, - param_and_grad[0]) - - if self._exclude_from_weight_decay_fn is not None \ - and self._exclude_from_weight_decay_fn(param_and_grad[0]): + moment1 = self._get_accumulator( + self._moment1_acc_str, param_and_grad[0] + ) + moment2 = self._get_accumulator( + self._moment2_acc_str, param_and_grad[0] + ) + beta1_pow_acc = self._get_accumulator( + self._beta1_pow_acc_str, param_and_grad[0] + ) + beta2_pow_acc = self._get_accumulator( + self._beta2_pow_acc_str, param_and_grad[0] + ) + + if ( + self._exclude_from_weight_decay_fn is not None + and self._exclude_from_weight_decay_fn(param_and_grad[0]) + ): weight_decay = 0.0 else: weight_decay = self._lamb_weight_decay lr = self._create_param_lr(param_and_grad) - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) p_name = param_and_grad[0].name if find_master: master_weight = self._master_weights[p_name] @@ -268,20 +297,50 @@ def _append_optimize_op(self, block, param_and_grad): found_inf = self._get_auxiliary_var('found_inf') if framework.in_dygraph_mode(): - _C_ops.lamb_(param_and_grad[0], param_and_grad[1], lr, moment1, - moment2, beta1_pow_acc, beta2_pow_acc, master_weight, - found_inf, weight_decay, self._beta1, self._beta2, - self._epsilon, find_master) + _C_ops.lamb_( + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + found_inf, + weight_decay, + self._beta1, + self._beta2, + self._epsilon, + find_master, + ) return None if framework._non_static_mode(): - _legacy_C_ops.lamb(param_and_grad[0], param_and_grad[1], lr, - moment1, moment2, beta1_pow_acc, beta2_pow_acc, - master_weight, param_and_grad[0], moment1, - moment2, beta1_pow_acc, beta2_pow_acc, - master_weight, 'beta1', self._beta1, 'beta2', - self._beta2, 'epsilon', self._epsilon, - 'weight_decay', weight_decay, 'multi_precision', - find_master) + _legacy_C_ops.lamb( + param_and_grad[0], + param_and_grad[1], + lr, + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + param_and_grad[0], + moment1, + moment2, + beta1_pow_acc, + beta2_pow_acc, + master_weight, + 'beta1', + self._beta1, + 'beta2', + self._beta2, + 'epsilon', + self._epsilon, + 'weight_decay', + weight_decay, + 'multi_precision', + find_master, + ) return None # create the lamb optimize op @@ -292,14 +351,14 @@ def _append_optimize_op(self, block, param_and_grad): "Moment1": moment1, "Moment2": moment2, "Beta1Pow": beta1_pow_acc, - "Beta2Pow": beta2_pow_acc + "Beta2Pow": beta2_pow_acc, } outputs = { "ParamOut": param_and_grad[0], "Moment1Out": moment1, "Moment2Out": moment2, "Beta1PowOut": beta1_pow_acc, - "Beta2PowOut": beta2_pow_acc + "Beta2PowOut": beta2_pow_acc, } attrs = { "beta1": self._beta1, @@ -316,11 +375,13 @@ def _append_optimize_op(self, block, param_and_grad): if found_inf: inputs["SkipUpdate"] = found_inf - lamb_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + lamb_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return lamb_op @@ -329,9 +390,11 @@ def _update_param_group(self, parameters): self._beta2 = parameters.get('beta2', self._default_dict['beta2']) self._epsilon = parameters.get('epsilon', self._default_dict['epsilon']) self._lamb_weight_decay = parameters.get( - 'lamb_weight_decay', self._default_dict['lamb_weight_decay']) + 'lamb_weight_decay', self._default_dict['lamb_weight_decay'] + ) self._exclude_from_weight_decay_fn = parameters.get( 'exclude_from_weight_decay_fn', - self._default_dict['exclude_from_weight_decay_fn']) + self._default_dict['exclude_from_weight_decay_fn'], + ) parameters = parameters.get('params') return parameters diff --git a/python/paddle/optimizer/lr.py b/python/paddle/optimizer/lr.py index 4dc451d6334ea..4e2dbafcabbc0 100644 --- a/python/paddle/optimizer/lr.py +++ b/python/paddle/optimizer/lr.py @@ -93,8 +93,10 @@ def get_lr(self): def __init__(self, learning_rate=0.1, last_epoch=-1, verbose=False): if not isinstance(learning_rate, (float, int)): raise TypeError( - "The type of learning rate must be float, but received {}". - format(type(learning_rate))) + "The type of learning rate must be float, but received {}".format( + type(learning_rate) + ) + ) self.base_lr = float(learning_rate) self.last_lr = float(learning_rate) self.last_epoch = last_epoch @@ -133,8 +135,11 @@ def step(self, epoch=None): self.last_lr = self.get_lr() if self.verbose: - print('Epoch {}: {} set learning rate to {}.'.format( - self.last_epoch, self.__class__.__name__, self.last_lr)) + print( + 'Epoch {}: {} set learning rate to {}.'.format( + self.last_epoch, self.__class__.__name__, self.last_lr + ) + ) def state_dict(self): """ @@ -153,7 +158,8 @@ def state_dict(self): assert value.shape == [ 1 ], "shape of Tensor in state_dict must be [1] {}".format( - value.shape) + value.shape + ) value = value.numpy()[0] state_dict[key] = value @@ -184,8 +190,10 @@ def set_state_dict(self, state_dict): self.__dict__[key] = state_dict[key] else: raise RuntimeError( - "Please check whether state_dict is correct for optimizer. Can't find [ {} ] in state_dict" - .format(key)) + "Please check whether state_dict is correct for optimizer. Can't find [ {} ] in state_dict".format( + key + ) + ) if len(state_dict) > len(self.keys): warnings.warn( "There are some unused values in state_dict. Maybe the optimizer have different 'LearningRateDecay' when invoking state_dict and set_dict" @@ -279,12 +287,14 @@ class NoamDecay(LRScheduler): """ - def __init__(self, - d_model, - warmup_steps, - learning_rate=1.0, - last_epoch=-1, - verbose=False): + def __init__( + self, + d_model, + warmup_steps, + learning_rate=1.0, + last_epoch=-1, + verbose=False, + ): self.d_model = d_model self.warmup_steps = warmup_steps super(NoamDecay, self).__init__(learning_rate, last_epoch, verbose) @@ -379,8 +389,9 @@ class PiecewiseDecay(LRScheduler): def __init__(self, boundaries, values, last_epoch=-1, verbose=False): self.boundaries = boundaries self.values = values - super(PiecewiseDecay, self).__init__(last_epoch=last_epoch, - verbose=verbose) + super(PiecewiseDecay, self).__init__( + last_epoch=last_epoch, verbose=verbose + ) def get_lr(self): for i in range(len(self.boundaries)): @@ -460,10 +471,13 @@ class NaturalExpDecay(LRScheduler): """ def __init__(self, learning_rate, gamma, last_epoch=-1, verbose=False): - assert gamma > 0.0, " 'gamma' must be a positive number so that the learning rate will decay." + assert ( + gamma > 0.0 + ), " 'gamma' must be a positive number so that the learning rate will decay." self.gamma = gamma - super(NaturalExpDecay, self).__init__(learning_rate, last_epoch, - verbose) + super(NaturalExpDecay, self).__init__( + learning_rate, last_epoch, verbose + ) def get_lr(self): return self.base_lr * math.exp(-1 * self.gamma * self.last_epoch) @@ -543,8 +557,9 @@ class InverseTimeDecay(LRScheduler): def __init__(self, learning_rate, gamma, last_epoch=-1, verbose=False): self.gamma = gamma - super(InverseTimeDecay, self).__init__(learning_rate, last_epoch, - verbose) + super(InverseTimeDecay, self).__init__( + learning_rate, last_epoch, verbose + ) def get_lr(self): return self.base_lr / (1 + self.gamma * self.last_epoch) @@ -637,30 +652,37 @@ class PolynomialDecay(LRScheduler): # scheduler.step() # If you update learning rate each epoch """ - def __init__(self, - learning_rate, - decay_steps, - end_lr=0.0001, - power=1.0, - cycle=False, - last_epoch=-1, - verbose=False): + def __init__( + self, + learning_rate, + decay_steps, + end_lr=0.0001, + power=1.0, + cycle=False, + last_epoch=-1, + verbose=False, + ): assert decay_steps > 0 and isinstance( - decay_steps, int), " 'decay_steps' must be a positive integer." + decay_steps, int + ), " 'decay_steps' must be a positive integer." self.decay_steps = decay_steps self.end_lr = end_lr - assert power > 0.0, " 'power' must be greater than 0.0 so that the learning rate will decay." + assert ( + power > 0.0 + ), " 'power' must be greater than 0.0 so that the learning rate will decay." self.power = power self.cycle = cycle - super(PolynomialDecay, self).__init__(learning_rate, last_epoch, - verbose) + super(PolynomialDecay, self).__init__( + learning_rate, last_epoch, verbose + ) def get_lr(self): tmp_epoch_num = self.last_epoch tmp_decay_steps = self.decay_steps if self.cycle: div_res = math.ceil( - float(self.last_epoch) / float(self.decay_steps)) + float(self.last_epoch) / float(self.decay_steps) + ) if self.last_epoch == 0: div_res = 1 @@ -669,8 +691,8 @@ def get_lr(self): tmp_epoch_num = min(self.last_epoch, self.decay_steps) return (self.base_lr - self.end_lr) * ( - (1 - float(tmp_epoch_num) / float(tmp_decay_steps))** - self.power) + self.end_lr + (1 - float(tmp_epoch_num) / float(tmp_decay_steps)) ** self.power + ) + self.end_lr class LinearWarmup(LRScheduler): @@ -758,27 +780,36 @@ class LinearWarmup(LRScheduler): # scheduler.step() # If you update learning rate each epoch """ - def __init__(self, - learning_rate, - warmup_steps, - start_lr, - end_lr, - last_epoch=-1, - verbose=False): - type_check = isinstance(learning_rate, float) or isinstance( - learning_rate, int) or isinstance(learning_rate, LRScheduler) + def __init__( + self, + learning_rate, + warmup_steps, + start_lr, + end_lr, + last_epoch=-1, + verbose=False, + ): + type_check = ( + isinstance(learning_rate, float) + or isinstance(learning_rate, int) + or isinstance(learning_rate, LRScheduler) + ) if not type_check: raise TypeError( - "the type of learning_rate should be [int, float or LRScheduler], the current type is {}" - .format(learning_rate)) + "the type of learning_rate should be [int, float or LRScheduler], the current type is {}".format( + learning_rate + ) + ) self.learning_rate = learning_rate assert warmup_steps > 0 and isinstance( - warmup_steps, int), " 'warmup_steps' must be a positive integer." + warmup_steps, int + ), " 'warmup_steps' must be a positive integer." self.warmup_steps = warmup_steps self.start_lr = start_lr self.end_lr = end_lr - assert end_lr > start_lr, "end_lr {} must be greater than start_lr {}".format( - end_lr, start_lr) + assert ( + end_lr > start_lr + ), "end_lr {} must be greater than start_lr {}".format(end_lr, start_lr) super(LinearWarmup, self).__init__(start_lr, last_epoch, verbose) def state_dict(self): @@ -803,7 +834,8 @@ def set_state_dict(self, state_dict): def get_lr(self): if self.last_epoch < self.warmup_steps: return (self.end_lr - self.start_lr) * float( - self.last_epoch) / float(self.warmup_steps) + self.start_lr + self.last_epoch + ) / float(self.warmup_steps) + self.start_lr else: if isinstance(self.learning_rate, LRScheduler): self.learning_rate.step(self.last_epoch - self.warmup_steps) @@ -884,10 +916,13 @@ class ExponentialDecay(LRScheduler): """ def __init__(self, learning_rate, gamma, last_epoch=-1, verbose=False): - assert gamma > 0.0 and gamma < 1.0, " 'gamma' must be in interval (0.0, 1.0) so that the learning rate will decay." + assert ( + gamma > 0.0 and gamma < 1.0 + ), " 'gamma' must be in interval (0.0, 1.0) so that the learning rate will decay." self.gamma = gamma - super(ExponentialDecay, self).__init__(learning_rate, last_epoch, - verbose) + super(ExponentialDecay, self).__init__( + learning_rate, last_epoch, verbose + ) def get_lr(self): return self.base_lr * (self.gamma**self.last_epoch) @@ -973,21 +1008,21 @@ class MultiStepDecay(LRScheduler): # scheduler.step() # If you update learning rate each epoch """ - def __init__(self, - learning_rate, - milestones, - gamma=0.1, - last_epoch=-1, - verbose=False): + def __init__( + self, learning_rate, milestones, gamma=0.1, last_epoch=-1, verbose=False + ): if not isinstance(milestones, (tuple, list)): raise TypeError( "The type of 'milestones' in 'MultiStepDecay' must be 'tuple, list', but received %s." - % type(milestones)) + % type(milestones) + ) - if not all([ + if not all( + [ milestones[i] < milestones[i + 1] for i in range(len(milestones) - 1) - ]): + ] + ): raise ValueError('The elements of milestones must be incremented') if gamma >= 1.0: raise ValueError('gamma should be < 1.0.') @@ -1000,7 +1035,7 @@ def get_lr(self): for i in range(len(self.milestones)): if self.last_epoch < self.milestones[i]: return self.base_lr * (self.gamma**i) - return self.base_lr * (self.gamma**len(self.milestones)) + return self.base_lr * (self.gamma ** len(self.milestones)) class StepDecay(LRScheduler): @@ -1082,21 +1117,20 @@ class StepDecay(LRScheduler): # scheduler.step() # If you update learning rate each epoch """ - def __init__(self, - learning_rate, - step_size, - gamma=0.1, - last_epoch=-1, - verbose=False): + def __init__( + self, learning_rate, step_size, gamma=0.1, last_epoch=-1, verbose=False + ): if not isinstance(step_size, int): raise TypeError( - "The type of 'step_size' must be 'int', but received %s." % - type(step_size)) + "The type of 'step_size' must be 'int', but received %s." + % type(step_size) + ) if gamma >= 1.0: raise ValueError('gamma should be < 1.0.') assert step_size > 0 and isinstance( - step_size, int), " 'step_size' must be a positive integer." + step_size, int + ), " 'step_size' must be a positive integer." self.step_size = step_size self.gamma = gamma super(StepDecay, self).__init__(learning_rate, last_epoch, verbose) @@ -1185,7 +1219,8 @@ def __init__(self, learning_rate, lr_lambda, last_epoch=-1, verbose=False): if not callable(lr_lambda): raise TypeError( "The type of 'lr_lambda' in 'LambdaDecay' must be 'function', but received %s." - % type(lr_lambda)) + % type(lr_lambda) + ) self.lr_lambda = lr_lambda super(LambdaDecay, self).__init__(learning_rate, last_epoch, verbose) @@ -1281,17 +1316,19 @@ class ReduceOnPlateau(LRScheduler): """ - def __init__(self, - learning_rate, - mode='min', - factor=0.1, - patience=10, - threshold=1e-4, - threshold_mode='rel', - cooldown=0, - min_lr=0, - epsilon=1e-8, - verbose=False): + def __init__( + self, + learning_rate, + mode='min', + factor=0.1, + patience=10, + threshold=1e-4, + threshold_mode='rel', + cooldown=0, + min_lr=0, + epsilon=1e-8, + verbose=False, + ): mode = mode.lower() if mode not in ['min', 'max']: raise ValueError('mode: ' + mode + ' is unknown!') @@ -1299,18 +1336,21 @@ def __init__(self, if factor >= 1.0: raise ValueError( - 'new_lr = origin_lr * gamma and gamma should be < 1.0.') + 'new_lr = origin_lr * gamma and gamma should be < 1.0.' + ) self.factor = factor threshold_mode = threshold_mode.lower() if threshold_mode not in ['rel', 'abs']: - raise ValueError('threshold mode: ' + threshold_mode + - ' is unknown!') + raise ValueError( + 'threshold mode: ' + threshold_mode + ' is unknown!' + ) self.threshold_mode = threshold_mode if not isinstance(learning_rate, (float, int)): raise TypeError( "The type of 'learning_rate' in 'ReduceOnPlateau' must be 'float', but received %s." - % type(learning_rate)) + % type(learning_rate) + ) self.patience = patience self.threshold = threshold @@ -1333,8 +1373,11 @@ def __init__(self, # "cooldown_counter / best / num_bad_epochs / last_epoch / last_lr" will be stored. def state_keys(self): self.keys = [ - 'cooldown_counter', 'best', 'num_bad_epochs', 'last_epoch', - 'last_lr' + 'cooldown_counter', + 'best', + 'num_bad_epochs', + 'last_epoch', + 'last_lr', ] def step(self, metrics, epoch=None): @@ -1364,18 +1407,25 @@ def step(self, metrics, epoch=None): else: # need to declarate explicitly from paddle.framework import VarBase as Tensor + tmp = Tensor # loss must be float, numpy.ndarray or 1-D Tensor with shape [1] if isinstance(metrics, (tmp, numpy.ndarray)): - assert len(metrics.shape) == 1 and metrics.shape[0] == 1, "the metrics.shape " \ - "should be (1L,), but the current metrics.shape is {}. Maybe that " \ - "you should call paddle.mean to process it first.".format( - metrics.shape) - elif not isinstance(metrics, - (int, float, numpy.float32, numpy.float64)): + assert len(metrics.shape) == 1 and metrics.shape[0] == 1, ( + "the metrics.shape " + "should be (1L,), but the current metrics.shape is {}. Maybe that " + "you should call paddle.mean to process it first.".format( + metrics.shape + ) + ) + elif not isinstance( + metrics, (int, float, numpy.float32, numpy.float64) + ): raise TypeError( - "metrics must be 'int', 'float', 'np.float', 'numpy.ndarray' or 'paddle.Tensor', but receive {}" - .format(type(metrics))) + "metrics must be 'int', 'float', 'np.float', 'numpy.ndarray' or 'paddle.Tensor', but receive {}".format( + type(metrics) + ) + ) if self.cooldown_counter > 0: self.cooldown_counter -= 1 @@ -1393,9 +1443,13 @@ def step(self, metrics, epoch=None): if self.last_lr - new_lr > self.epsilon: self.last_lr = new_lr if self.verbose: - print('Epoch {}: {} set learning rate to {}.'.format( - self.last_epoch, self.__class__.__name__, - self.last_lr)) + print( + 'Epoch {}: {} set learning rate to {}.'.format( + self.last_epoch, + self.__class__.__name__, + self.last_lr, + ) + ) def _is_better(self, current, best): if self.mode == 'min' and self.threshold_mode == 'rel': @@ -1493,41 +1547,50 @@ class CosineAnnealingDecay(LRScheduler): # scheduler.step() # If you update learning rate each epoch """ - def __init__(self, - learning_rate, - T_max, - eta_min=0, - last_epoch=-1, - verbose=False): + def __init__( + self, learning_rate, T_max, eta_min=0, last_epoch=-1, verbose=False + ): if not isinstance(T_max, int): raise TypeError( "The type of 'T_max' in 'CosineAnnealingDecay' must be 'int', but received %s." - % type(T_max)) + % type(T_max) + ) if not isinstance(eta_min, (float, int)): raise TypeError( "The type of 'eta_min' in 'CosineAnnealingDecay' must be 'float, int', but received %s." - % type(eta_min)) + % type(eta_min) + ) assert T_max > 0 and isinstance( - T_max, int), " 'T_max' must be a positive integer." + T_max, int + ), " 'T_max' must be a positive integer." self.T_max = T_max self.eta_min = float(eta_min) - super(CosineAnnealingDecay, self).__init__(learning_rate, last_epoch, - verbose) + super(CosineAnnealingDecay, self).__init__( + learning_rate, last_epoch, verbose + ) def get_lr(self): if self.last_epoch == 0: return self.base_lr elif (self.last_epoch - 1 - self.T_max) % (2 * self.T_max) == 0: - return self.last_lr + (self.base_lr - self.eta_min) * ( - 1 - math.cos(math.pi / self.T_max)) / 2 + return ( + self.last_lr + + (self.base_lr - self.eta_min) + * (1 - math.cos(math.pi / self.T_max)) + / 2 + ) return (1 + math.cos(math.pi * self.last_epoch / self.T_max)) / ( - 1 + math.cos(math.pi * (self.last_epoch - 1) / self.T_max)) * ( - self.last_lr - self.eta_min) + self.eta_min + 1 + math.cos(math.pi * (self.last_epoch - 1) / self.T_max) + ) * (self.last_lr - self.eta_min) + self.eta_min def _get_closed_form_lr(self): - return self.eta_min + (self.base_lr - self.eta_min) * ( - 1 + math.cos(math.pi * self.last_epoch / self.T_max)) / 2 + return ( + self.eta_min + + (self.base_lr - self.eta_min) + * (1 + math.cos(math.pi * self.last_epoch / self.T_max)) + / 2 + ) class MultiplicativeDecay(LRScheduler): @@ -1581,11 +1644,13 @@ def __init__(self, learning_rate, lr_lambda, last_epoch=-1, verbose=False): if not callable(lr_lambda): raise TypeError( "The type of 'lr_lambda' in 'MultiplicativeDecay' must be 'function', but received %s." - % type(lr_lambda)) + % type(lr_lambda) + ) self.lr_lambda = lr_lambda - super(MultiplicativeDecay, self).__init__(learning_rate, last_epoch, - verbose) + super(MultiplicativeDecay, self).__init__( + learning_rate, last_epoch, verbose + ) def get_lr(self): cur_lr = self.base_lr @@ -1678,29 +1743,35 @@ class OneCycleLR(LRScheduler): scheduler.step() # You should update learning rate each step """ - def __init__(self, - max_learning_rate, - total_steps, - divide_factor=25., - end_learning_rate=0.0001, - phase_pct=0.3, - anneal_strategy='cos', - three_phase=False, - last_epoch=-1, - verbose=False): + def __init__( + self, + max_learning_rate, + total_steps, + divide_factor=25.0, + end_learning_rate=0.0001, + phase_pct=0.3, + anneal_strategy='cos', + three_phase=False, + last_epoch=-1, + verbose=False, + ): # Check type and value of max_learning_rate if not isinstance(max_learning_rate, (float, int)): raise TypeError( - "'max_learning_rate' must be 'float' or 'int', but received {}". - format(type(max_learning_rate))) + "'max_learning_rate' must be 'float' or 'int', but received {}".format( + type(max_learning_rate) + ) + ) if max_learning_rate < 0: raise ValueError("'max_learning_rate' must be a positive integer.") # Check type and value of end_learning_rate if not isinstance(end_learning_rate, (float, int)): raise TypeError( - "'end_learning_rate' must be 'float' or 'int', but received {}". - format(type(end_learning_rate))) + "'end_learning_rate' must be 'float' or 'int', but received {}".format( + type(end_learning_rate) + ) + ) if end_learning_rate < 0: raise ValueError("'end_learning_rate' must be a positive integer.") @@ -1708,7 +1779,9 @@ def __init__(self, if not isinstance(total_steps, int): raise TypeError( "'total_step' must be 'int', but received {}".format( - type(total_steps))) + type(total_steps) + ) + ) if total_steps <= 0: raise ValueError("'total_step' must be a positive integer.") self.total_steps = total_steps @@ -1717,17 +1790,23 @@ def __init__(self, if not isinstance(phase_pct, float): raise TypeError( "'phase_pct' must be 'float', but received {}".format( - type(phase_pct))) + type(phase_pct) + ) + ) if phase_pct < 0 or phase_pct > 1: raise ValueError( "'phase_pct' must be between 0 and 1, but received {}".format( - phase_pct)) + phase_pct + ) + ) # Check type and value of divide_factor if not isinstance(divide_factor, (float, int)): raise TypeError( - "'divide_factor' must be 'float' or 'int', but received {}". - format(type(divide_factor))) + "'divide_factor' must be 'float' or 'int', but received {}".format( + type(divide_factor) + ) + ) initial_lr = max_learning_rate / float(divide_factor) min_lr = float(end_learning_rate) @@ -1750,17 +1829,22 @@ def __init__(self, self._step_config[1] - self._step_config[0], self._step_config[2] - self._step_config[1], self._step_config[3] - self._step_config[2], - self._step_config[3] - - self._step_config[2], # for the last step. + self._step_config[3] + - self._step_config[2], # for the last step. ] # start lr and end lr of each phase. self._lr_config = [ - initial_lr, max_learning_rate, initial_lr, min_lr + initial_lr, + max_learning_rate, + initial_lr, + min_lr, ] else: self._step_config = [ - 0, phase_pct * self.total_steps - 1, self.total_steps - 1, - self.total_steps - 1 + 0, + phase_pct * self.total_steps - 1, + self.total_steps - 1, + self.total_steps - 1, ] self._steps_size = [ self._step_config[1] - self._step_config[0], @@ -1776,8 +1860,10 @@ def __init__(self, self.anneal_func = self._linear_annealing else: raise ValueError( - "'anneal_strategy' must by one of 'cos' or 'linear', but received {}" - .format(anneal_strategy)) + "'anneal_strategy' must by one of 'cos' or 'linear', but received {}".format( + anneal_strategy + ) + ) super(OneCycleLR, self).__init__(initial_lr, last_epoch, verbose) def _cos_annealing(self, start_lr, end_lr, pct): @@ -1792,17 +1878,21 @@ def get_lr(self): if current_step > self.total_steps: raise ValueError( - "Tried to step {} times. However the number of total steps is {}" - .format(current_step, self.total_steps)) + "Tried to step {} times. However the number of total steps is {}".format( + current_step, self.total_steps + ) + ) for (i, (end_step, step_size)) in enumerate( - zip(self._step_config[1:], self._steps_size)): + zip(self._step_config[1:], self._steps_size) + ): # i == len(self._lr_config) - 2 catch the last step, otherwise it will return None. if current_step <= end_step or i == len(self._lr_config) - 2: # self._step_config[i] means start step of a phase. percentage = (current_step - self._step_config[i]) / step_size - return self.anneal_func(self._lr_config[i], - self._lr_config[i + 1], percentage) + return self.anneal_func( + self._lr_config[i], self._lr_config[i + 1], percentage + ) class CyclicLR(LRScheduler): @@ -1896,71 +1986,93 @@ class CyclicLR(LRScheduler): scheduler.step() # You should update learning rate each step """ - def __init__(self, - base_learning_rate, - max_learning_rate, - step_size_up, - step_size_down=None, - mode='triangular', - exp_gamma=1., - scale_fn=None, - scale_mode='cycle', - last_epoch=-1, - verbose=False): + def __init__( + self, + base_learning_rate, + max_learning_rate, + step_size_up, + step_size_down=None, + mode='triangular', + exp_gamma=1.0, + scale_fn=None, + scale_mode='cycle', + last_epoch=-1, + verbose=False, + ): # check type and value of max_learning_rate if not isinstance(max_learning_rate, (float, int)): raise TypeError( - "'max_learning_rate' must be 'float' or 'int', but received {}". - format(type(max_learning_rate))) + "'max_learning_rate' must be 'float' or 'int', but received {}".format( + type(max_learning_rate) + ) + ) if max_learning_rate < 0: raise ValueError( - "'max_learning_rate' must be a positive integer, but received {}" - .format(max_learning_rate)) + "'max_learning_rate' must be a positive integer, but received {}".format( + max_learning_rate + ) + ) # check type and value of step_size_up if not isinstance(step_size_up, int): raise TypeError( - "The type of 'step_size_up' must be int, but received {}". - format(type(step_size_up))) + "The type of 'step_size_up' must be int, but received {}".format( + type(step_size_up) + ) + ) if step_size_up <= 0: raise ValueError( - "'step_size_up' must be a positive integer, but received {}". - format(step_size_up)) + "'step_size_up' must be a positive integer, but received {}".format( + step_size_up + ) + ) # check type and value of step_size_down if step_size_down is not None: if not isinstance(step_size_down, int): raise TypeError( - "The type of 'step_size_down' must be int, but received {}". - format(type(step_size_down))) + "The type of 'step_size_down' must be int, but received {}".format( + type(step_size_down) + ) + ) if step_size_down <= 0: raise ValueError( - "'step_size_down' must be a positive integer, but received {}" - .format(step_size_down)) + "'step_size_down' must be a positive integer, but received {}".format( + step_size_down + ) + ) # check type of exp_gamma if not isinstance(exp_gamma, float): raise TypeError( "The type of 'exp_gamma' must be float, but received {}".format( - type(exp_gamma))) + type(exp_gamma) + ) + ) step_size_up = float(step_size_up) - step_size_down = float( - step_size_down) if step_size_down is not None else step_size_up + step_size_down = ( + float(step_size_down) + if step_size_down is not None + else step_size_up + ) self.cycle_size = step_size_up + step_size_down self.step_up_pct = step_size_up / self.cycle_size self.max_lr = float(max_learning_rate) self.amplitude = self.max_lr - base_learning_rate - if mode not in ['triangular', 'triangular2', 'exp_range' - ] and scale_fn is None: + if ( + mode not in ['triangular', 'triangular2', 'exp_range'] + and scale_fn is None + ): raise ValueError( "'mode' is invalid and 'scale_fn' is not specified, make sure one of 'mode' or 'scale_fn' is valid" ) if scale_mode not in ['cycle', 'iterations']: raise ValueError( - "'scale_mode' must be one of 'cycle' or 'iterations") + "'scale_mode' must be one of 'cycle' or 'iterations" + ) self.mode = mode self.gamma = exp_gamma # only for exp_range mode @@ -1981,10 +2093,10 @@ def __init__(self, super().__init__(base_learning_rate, last_epoch, verbose) def _triangular_scale_fn(self, x): - return 1. + return 1.0 def _triangular2_scale_fn(self, x): - return 1 / (2.**(x - 1)) + return 1 / (2.0 ** (x - 1)) def _exp_range_scale_fn(self, x): return self.gamma**x @@ -1993,7 +2105,7 @@ def get_lr(self): iterations = self.last_epoch cycle = 1 + iterations // self.cycle_size - pct_per_cycle = 1. + iterations / self.cycle_size - cycle + pct_per_cycle = 1.0 + iterations / self.cycle_size - cycle if pct_per_cycle <= self.step_up_pct: scale_factor = pct_per_cycle / self.step_up_pct diff --git a/python/paddle/optimizer/momentum.py b/python/paddle/optimizer/momentum.py index 5ec696f691ce4..8cca050625a42 100644 --- a/python/paddle/optimizer/momentum.py +++ b/python/paddle/optimizer/momentum.py @@ -121,29 +121,35 @@ class Momentum(Optimizer): """ _velocity_acc_str = "velocity" - def __init__(self, - learning_rate=0.001, - momentum=0.9, - parameters=None, - use_nesterov=False, - weight_decay=None, - grad_clip=None, - multi_precision=False, - rescale_grad=1.0, - use_multi_tensor=False, - name=None): + def __init__( + self, + learning_rate=0.001, + momentum=0.9, + parameters=None, + use_nesterov=False, + weight_decay=None, + grad_clip=None, + multi_precision=False, + rescale_grad=1.0, + use_multi_tensor=False, + name=None, + ): if learning_rate is None: raise ValueError("learning_rate is not set") if momentum is None: raise ValueError("momentum is not set") - predicate = lambda regular: isinstance(regular, - (L2DecayRegularizer, float)) + predicate = lambda regular: isinstance( + regular, (L2DecayRegularizer, float) + ) if isinstance(parameters, list): if isinstance(parameters[0], dict): for param_group in parameters: - decay = param_group[ - 'weight_decay'] if 'weight_decay' in param_group else weight_decay + decay = ( + param_group['weight_decay'] + if 'weight_decay' in param_group + else weight_decay + ) reg_method, reg_coeff = self._update_regularization(decay) param_group['regularization_method'] = reg_method param_group['regularization_coeff'] = reg_coeff @@ -151,16 +157,20 @@ def __init__(self, param_group['weight_decay'] = py_regular py_regular = None if predicate(weight_decay) else weight_decay - super(Momentum, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=py_regular, - grad_clip=grad_clip, - name=name) + super(Momentum, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=py_regular, + grad_clip=grad_clip, + name=name, + ) self.type = "momentum" self._momentum = momentum self._use_nesterov = bool(use_nesterov) - self._regularization_method, self._regularization_coeff = self._update_regularization( - weight_decay) + ( + self._regularization_method, + self._regularization_coeff, + ) = self._update_regularization(weight_decay) self._multi_precision = multi_precision self._rescale_grad = rescale_grad self._master_weights = {} @@ -178,25 +188,25 @@ def __init__(self, self._velocity_dict = {'FP32_LODTensor': [], 'FP16_LODTensor': []} self._master_weight_dict = { 'FP32_LODTensor': None, - 'FP16_LODTensor': [] + 'FP16_LODTensor': [], } self._regularization_method_dict = { 'FP32_LODTensor': [], - 'FP16_LODTensor': [] + 'FP16_LODTensor': [], } self._regularization_coeff_dict = { 'FP32_LODTensor': [], - 'FP16_LODTensor': [] + 'FP16_LODTensor': [], } def _update_regularization(self, weight_decay): reg_method = "" reg_coeff = 0.0 - if (isinstance(weight_decay, L2DecayRegularizer)): + if isinstance(weight_decay, L2DecayRegularizer): reg_method = "l2_decay" reg_coeff = weight_decay._regularization_coeff - if (isinstance(weight_decay, float)): + if isinstance(weight_decay, float): reg_method = "l2_decay" reg_coeff = weight_decay return reg_method, reg_coeff @@ -209,19 +219,23 @@ def _create_master_weight(self, param): var_name = param.name + "_fp32_master" var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -237,15 +251,22 @@ def _get_accumulator(self, name, param): """ if self._name is not None: name = self._name + "_" + name - find_master = self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 - target_param = self._master_weights[ - param.name] if find_master else param + find_master = ( + self._multi_precision and param.dtype == core.VarDesc.VarType.FP16 + ) + target_param = ( + self._master_weights[param.name] if find_master else param + ) target_name = target_param.name - if (name not in self._accumulators - or target_name not in self._accumulators[name]): + if ( + name not in self._accumulators + or target_name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, target_name)) + name, target_name + ) + ) return self._accumulators[name][target_name] def _create_accumulators(self, block, parameters): @@ -263,7 +284,10 @@ def _create_accumulators(self, block, parameters): master_p = self._create_master_weight(p) self._add_accumulator(self._velocity_acc_str, master_p) continue - if p.dtype == core.VarDesc.VarType.FP16 and not self._multi_precision: + if ( + p.dtype == core.VarDesc.VarType.FP16 + and not self._multi_precision + ): warnings.warn( "Accumulating with FP16 in optimizer can lead to poor accuracy or slow convergence." "Consider using multi_precision=True option of the Momentum optimizer." @@ -271,25 +295,28 @@ def _create_accumulators(self, block, parameters): self._add_accumulator(self._velocity_acc_str, p) def _create_regularization_of_grad(self, param, grad, regularization=None): - """ Create and add backward regularization Operators + """Create and add backward regularization Operators Function helper of append_regularization_ops. """ # If ParamAttr is set to L2Decay, we skip doing regularization here. And then we fused # L2Decay with momentum which can refer to _append_optimize_op below. - if hasattr(param, 'regularizer') and isinstance(param.regularizer, - L2DecayRegularizer): + if hasattr(param, 'regularizer') and isinstance( + param.regularizer, L2DecayRegularizer + ): return grad return super(Momentum, self)._create_regularization_of_grad( - param, grad, regularization) + param, grad, regularization + ) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) if isinstance(param_and_grad, dict): param_and_grad = self._update_param_group(param_and_grad) - velocity_acc = self._get_accumulator(self._velocity_acc_str, - param_and_grad[0]) + velocity_acc = self._get_accumulator( + self._velocity_acc_str, param_and_grad[0] + ) lr = self._create_param_lr(param_and_grad) # For fusion of momentum and l2decay @@ -306,30 +333,56 @@ def _append_optimize_op(self, block, param_and_grad): regularization_method = "" regularization_coeff = 0.0 - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 - master_weight = (self._master_weights[param_and_grad[0].name] - if find_master else None) + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) + master_weight = ( + self._master_weights[param_and_grad[0].name] + if find_master + else None + ) if _in_legacy_dygraph(): if isinstance(param_and_grad, dict): self._update_regularization(param_and_grad['weight_decay']) _, _, _ = _legacy_C_ops.momentum( - param_and_grad[0], param_and_grad[1], velocity_acc, lr, - master_weight, param_and_grad[0], velocity_acc, master_weight, - 'mu', self._momentum, 'use_nesterov', self._use_nesterov, - 'regularization_method', regularization_method, - 'regularization_coeff', regularization_coeff, 'multi_precision', - find_master) + param_and_grad[0], + param_and_grad[1], + velocity_acc, + lr, + master_weight, + param_and_grad[0], + velocity_acc, + master_weight, + 'mu', + self._momentum, + 'use_nesterov', + self._use_nesterov, + 'regularization_method', + regularization_method, + 'regularization_coeff', + regularization_coeff, + 'multi_precision', + find_master, + ) return None if in_dygraph_mode(): if isinstance(param_and_grad, dict): self._update_regularization(param_and_grad['weight_decay']) - return _C_ops.momentum_(param_and_grad[0], param_and_grad[1], - velocity_acc, lr, master_weight, - self._momentum, self._use_nesterov, - regularization_method, regularization_coeff, - find_master, self._rescale_grad) + return _C_ops.momentum_( + param_and_grad[0], + param_and_grad[1], + velocity_acc, + lr, + master_weight, + self._momentum, + self._use_nesterov, + regularization_method, + regularization_coeff, + find_master, + self._rescale_grad, + ) attrs = { "mu": self._momentum, @@ -337,19 +390,19 @@ def _append_optimize_op(self, block, param_and_grad): "regularization_method": regularization_method, "regularization_coeff": regularization_coeff, "multi_precision": find_master, - "rescale_grad": self._rescale_grad + "rescale_grad": self._rescale_grad, } inputs = { "Param": [param_and_grad[0]], "Grad": [param_and_grad[1]], "Velocity": [velocity_acc], - "LearningRate": [lr] + "LearningRate": [lr], } outputs = { "ParamOut": [param_and_grad[0]], - "VelocityOut": [velocity_acc] + "VelocityOut": [velocity_acc], } if find_master: @@ -357,11 +410,13 @@ def _append_optimize_op(self, block, param_and_grad): outputs["MasterParamOut"] = master_weight # create the momentum optimize op - momentum_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + momentum_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return momentum_op @@ -383,7 +438,9 @@ def _multi_tensor_init(self, target_block, parameters): # we skip param's l2decay before, so fuse it with momentum here. if isinstance(param.regularizer, L2DecayRegularizer): regularization_method = "l2_decay" - regularization_coeff = param.regularizer._regularization_coeff + regularization_coeff = ( + param.regularizer._regularization_coeff + ) elif param.regularizer is not None: regularization_method = "" regularization_coeff = 0.0 @@ -392,28 +449,34 @@ def _multi_tensor_init(self, target_block, parameters): self._velocity_dict['FP32_LODTensor'].append(velocity_acc) # fp32 no master weight self._regularization_method_dict['FP32_LODTensor'].append( - regularization_method) + regularization_method + ) self._regularization_coeff_dict['FP32_LODTensor'].append( - regularization_coeff) + regularization_coeff + ) elif param.dtype == paddle.float16: self._param_dict['FP16_LODTensor'].append(param) self._velocity_dict['FP16_LODTensor'].append(velocity_acc) if self._multi_precision: self._master_weight_dict['FP16_LODTensor'].append( - self._master_weights[param.name]) + self._master_weights[param.name] + ) else: self._master_weight_dict['FP16_LODTensor'] = None self._regularization_method_dict['FP16_LODTensor'].append( - regularization_method) + regularization_method + ) self._regularization_coeff_dict['FP16_LODTensor'].append( - regularization_coeff) + regularization_coeff + ) else: raise ValueError( "Now multi_tensor_momentum only support fp32 and fp16 parameters and grad is LOD_TENSOR." ) - def _append_optimize_multi_tensor_op(self, target_block, - parameters_and_grads): + def _append_optimize_multi_tensor_op( + self, target_block, parameters_and_grads + ): """ For Multi Tensor, append optimize merged_operator to block. """ @@ -427,15 +490,19 @@ def _append_optimize_multi_tensor_op(self, target_block, if param_and_grad[1] is None: continue if param_and_grad[0].stop_gradient is False: - if param_and_grad[ - 0].dtype == paddle.float32 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + if ( + param_and_grad[0].dtype == paddle.float32 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP32_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP32_LODTensor'].append(lr) - elif param_and_grad[ - 0].dtype == paddle.float16 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + elif ( + param_and_grad[0].dtype == paddle.float16 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP16_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP16_LODTensor'].append(lr) @@ -446,21 +513,27 @@ def _append_optimize_multi_tensor_op(self, target_block, if param_and_grad[0].stop_gradient is False: param_grad_dict = dict() param_grad_dict['params'] = param_and_grad - param_grad_dict.update({ - k: v - for k, v in parameters_and_grads.items() - if k != 'params' - }) + param_grad_dict.update( + { + k: v + for k, v in parameters_and_grads.items() + if k != 'params' + } + ) param_and_grad = self._update_param_group(param_grad_dict) - if param_and_grad[ - 0].dtype == paddle.float32 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + if ( + param_and_grad[0].dtype == paddle.float32 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP32_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP32_LODTensor'].append(lr) - elif param_and_grad[ - 0].dtype == paddle.float16 and param_and_grad[ - 1].type == core.VarDesc.VarType.LOD_TENSOR: + elif ( + param_and_grad[0].dtype == paddle.float16 + and param_and_grad[1].type + == core.VarDesc.VarType.LOD_TENSOR + ): grad_dict['FP16_LODTensor'].append(param_and_grad[1]) lr = self._create_param_lr(param_and_grad) lr_dict['FP16_LODTensor'].append(lr) @@ -473,26 +546,39 @@ def _append_optimize_multi_tensor_op(self, target_block, if framework._non_static_mode(): if in_dygraph_mode(): _, _, _ = _C_ops.merged_momentum_( - self._param_dict[key], grad_dict[key], - self._velocity_dict[key], lr_dict[key], - self._master_weight_dict[key], self._momentum, + self._param_dict[key], + grad_dict[key], + self._velocity_dict[key], + lr_dict[key], + self._master_weight_dict[key], + self._momentum, self._use_nesterov, self._regularization_method_dict[key], - self._regularization_coeff_dict[key], find_master, - self._rescale_grad) + self._regularization_coeff_dict[key], + find_master, + self._rescale_grad, + ) else: _, _, _ = _legacy_C_ops.merged_momentum( - self._param_dict[key], grad_dict[key], - self._velocity_dict[key], lr_dict[key], + self._param_dict[key], + grad_dict[key], + self._velocity_dict[key], + lr_dict[key], self._master_weight_dict[key], - self._param_dict[key], self._velocity_dict[key], - self._master_weight_dict[key], 'mu', self._momentum, - 'use_nesterov', self._use_nesterov, + self._param_dict[key], + self._velocity_dict[key], + self._master_weight_dict[key], + 'mu', + self._momentum, + 'use_nesterov', + self._use_nesterov, 'regularization_method', self._regularization_method_dict[key], 'regularization_coeff', self._regularization_coeff_dict[key], - 'multi_precision', find_master) + 'multi_precision', + find_master, + ) else: inputs = { "Param": self._param_dict[key], @@ -505,38 +591,45 @@ def _append_optimize_multi_tensor_op(self, target_block, "VelocityOut": self._velocity_dict[key], } attrs = { - "mu": - self._momentum, - "use_nesterov": - self._use_nesterov, - "regularization_method": - self._regularization_method_dict[key], - "regularization_coeff": - self._regularization_coeff_dict[key], + "mu": self._momentum, + "use_nesterov": self._use_nesterov, + "regularization_method": self._regularization_method_dict[ + key + ], + "regularization_coeff": self._regularization_coeff_dict[ + key + ], } if find_master: inputs["MasterParam"] = self._master_weight_dict[key] outputs["MasterParamOut"] = self._master_weight_dict[ - key] + key + ] attrs["multi_precision"] = find_master - target_block.append_op(type="merged_momentum", - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + target_block.append_op( + type="merged_momentum", + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return None def _update_param_group(self, parameters): - self._momentum = parameters.get('momentum', - self._default_dict['momentum']) - self._use_nesterov = parameters.get('use_nesterov', - self._default_dict['use_nesterov']) - self._rescale_grad = parameters.get('rescale_grad', - self._default_dict['rescale_grad']) + self._momentum = parameters.get( + 'momentum', self._default_dict['momentum'] + ) + self._use_nesterov = parameters.get( + 'use_nesterov', self._default_dict['use_nesterov'] + ) + self._rescale_grad = parameters.get( + 'rescale_grad', self._default_dict['rescale_grad'] + ) self._regularization_method = parameters.get( - 'regularization_method', - self._default_dict['regularization_method']) + 'regularization_method', self._default_dict['regularization_method'] + ) self._regularization_coeff = parameters.get( - 'regularization_coeff', self._default_dict['regularization_coeff']) + 'regularization_coeff', self._default_dict['regularization_coeff'] + ) parameters = parameters.get('params') return parameters diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py index b90bd5ae3ff1e..54ac0db5536cd 100644 --- a/python/paddle/optimizer/optimizer.py +++ b/python/paddle/optimizer/optimizer.py @@ -17,13 +17,22 @@ from collections import defaultdict import paddle -from paddle.fluid.framework import Variable, default_main_program, device_guard, name_scope +from paddle.fluid.framework import ( + Variable, + default_main_program, + device_guard, + name_scope, +) from ..fluid import framework from ..fluid import layers from ..fluid import unique_name from ..fluid.backward import _get_no_grad_set_name, append_backward -from ..fluid.clip import GradientClipBase, append_gradient_clip_ops, error_clip_callback +from ..fluid.clip import ( + GradientClipBase, + append_gradient_clip_ops, + error_clip_callback, +) from ..fluid.framework import program_guard, Parameter from ..fluid.initializer import Constant from ..fluid.layer_helper import LayerHelper @@ -31,24 +40,36 @@ from paddle.fluid import core from .lr import LRScheduler from paddle import _C_ops, _legacy_C_ops -from paddle.fluid.framework import _in_legacy_dygraph, _in_eager_without_dygraph_check, _current_expected_place, in_dygraph_mode +from paddle.fluid.framework import ( + _in_legacy_dygraph, + _in_eager_without_dygraph_check, + _current_expected_place, + in_dygraph_mode, +) __all__ = [] @framework.static_only -def append_backward_new(loss_list, - parameter_list=None, - no_grad_set=None, - callbacks=None, - checkpoints=None, - distop_context=None): +def append_backward_new( + loss_list, + parameter_list=None, + no_grad_set=None, + callbacks=None, + checkpoints=None, + distop_context=None, +): from paddle.incubate.autograd.primx import orig2prim, Transform + program = default_main_program() - assert program.num_blocks == 1, "The append_backward_new interface is designed to process only one block." + assert ( + program.num_blocks == 1 + ), "The append_backward_new interface is designed to process only one block." block = program.current_block() for el in loss_list: - assert el.block == block, 'variable in loss_list should be in current block of main program' + assert ( + el.block == block + ), 'variable in loss_list should be in current block of main program' orig2prim(block) ad = Transform(block) @@ -152,12 +173,14 @@ class Optimizer(object): """ @imperative_base.no_grad - def __init__(self, - learning_rate, - parameters=None, - weight_decay=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate, + parameters=None, + weight_decay=None, + grad_clip=None, + name=None, + ): if parameters is not None: # paddle.Tensor is also iterable, so here we don't check whether @@ -166,13 +189,16 @@ def __init__(self, if isinstance(parameters, (paddle.Tensor, core.eager.Tensor)): raise TypeError( "`parameters` argument given to the optimizer should be " - "an iterable of paddle Tensors, but got argument type is `{}`." - .format(type(parameters))) + "an iterable of paddle Tensors, but got argument type is `{}`.".format( + type(parameters) + ) + ) if isinstance(parameters, dict): raise TypeError( "`parameters` argument should not get dict type, " "if parameter groups is needed, please set `parameters`" - " as list of dict") + " as list of dict" + ) self._parameter_list = list(parameters) else: self._parameter_list = None @@ -186,18 +212,22 @@ def __init__(self, if weight_decay is not None: if not isinstance(self._parameter_list[0], dict): for param in self._parameter_list: - if hasattr(param, 'regularizer' - ) and param.regularizer is not None: + if ( + hasattr(param, 'regularizer') + and param.regularizer is not None + ): logging.info( "If regularizer of a Parameter has been set by 'paddle.ParamAttr' or 'static.WeightNormParamAttr' already. " "The weight_decay[%s] in Optimizer will not take effect, and it will only be applied to other Parameters!" - % weight_decay.__str__()) + % weight_decay.__str__() + ) break if not isinstance(learning_rate, (float, LRScheduler)): raise TypeError( - "learning rate should be float or LRScheduler, got %s here" % - type(learning_rate)) + "learning rate should be float or LRScheduler, got %s here" + % type(learning_rate) + ) if grad_clip is not None: if not isinstance(grad_clip, GradientClipBase): raise TypeError( @@ -205,6 +235,7 @@ def __init__(self, ) if isinstance(weight_decay, float): from ..fluid.regularizer import L2Decay + self.regularization = L2Decay(weight_decay) else: self.regularization = weight_decay @@ -216,8 +247,9 @@ def __init__(self, if self._parameter_list: if isinstance(self._parameter_list[0], dict): for param_group in self._parameter_list: - assert 'params' in param_group, \ - 'params should be set in parameters if parameter groups are optimized in different options' + assert ( + 'params' in param_group + ), 'params should be set in parameters if parameter groups are optimized in different options' self._dtype = self._parameter_list[0]['params'][0].dtype else: self._dtype = self._parameter_list[0].dtype @@ -237,7 +269,7 @@ def __init__(self, self.clear_gradients = self.clear_grad self._default_dict = { 'weight_decay': self.regularization, - 'grad_clip': self._grad_clip + 'grad_clip': self._grad_clip, } self._param_groups = [] @@ -342,8 +374,9 @@ def set_state_dict(self, state_dict): self._accumulators_holder = state_dict for k, v in self._accumulators.items(): for para_name, var_tmp in v.items(): - assert var_tmp.name in state_dict, \ - "optimizer Tensor {} not found".format( var_tmp.name ) + assert ( + var_tmp.name in state_dict + ), "optimizer Tensor {} not found".format(var_tmp.name) var = var_tmp.value() tensor = var.get_tensor() model_np = np.array(tensor) @@ -357,16 +390,23 @@ def set_state_dict(self, state_dict): elif isinstance(load_para, np.ndarray): load_para_np = load_para else: - raise RuntimeError("State dict type {} not supprt".format( - str(type(load_para)))) - - assert model_np.shape == load_para_np.shape, \ - "Parameter shape not match, Dygraph Parameter [ {} ] need tensor with shape {} but load tensor with shape {}".format( - model_np.name, model_np.shape, load_para_np.shape) + raise RuntimeError( + "State dict type {} not supprt".format( + str(type(load_para)) + ) + ) + + assert ( + model_np.shape == load_para_np.shape + ), "Parameter shape not match, Dygraph Parameter [ {} ] need tensor with shape {} but load tensor with shape {}".format( + model_np.name, model_np.shape, load_para_np.shape + ) - assert model_np.dtype == load_para_np.dtype, \ - "Parameter dtype not match, Dygraph Parameter [ {} ] need tensor with dtype {} but load tensor with dtype {}".format( - model_np.name, model_np.dtype, load_para_np.dtype) + assert ( + model_np.dtype == load_para_np.dtype + ), "Parameter dtype not match, Dygraph Parameter [ {} ] need tensor with dtype {} but load tensor with dtype {}".format( + model_np.name, model_np.dtype, load_para_np.dtype + ) tensor.set(load_para_np, framework._current_expected_place()) @@ -375,45 +415,57 @@ def get_opti_var_name_list(self): def _create_global_learning_rate(self): # lr var can't be float16, for pure fp16 training, should extra handle the dtype for lr - _lr_dtype = paddle.get_default_dtype( - ) if self._dtype is None else self._dtype - _lr_dtype = paddle.float32 if ( - paddle.get_default_dtype() != "float16" - and _lr_dtype == paddle.float16) else _lr_dtype + _lr_dtype = ( + paddle.get_default_dtype() if self._dtype is None else self._dtype + ) + _lr_dtype = ( + paddle.float32 + if ( + paddle.get_default_dtype() != "float16" + and _lr_dtype == paddle.float16 + ) + else _lr_dtype + ) if isinstance(self._learning_rate, LRScheduler): lr_var = self._global_learning_rate() # only create global lr_var once if not isinstance(lr_var, framework.Variable): lr_name = unique_name.generate('learning_rate') self._learning_rate._var_name = lr_name - lr_var = self.helper.create_global_variable(name=lr_name, - shape=[1], - persistable=True, - stop_gradient=True, - dtype=_lr_dtype) + lr_var = self.helper.create_global_variable( + name=lr_name, + shape=[1], + persistable=True, + stop_gradient=True, + dtype=_lr_dtype, + ) main_prog = framework.default_main_program() main_prog.lr_sheduler = self._learning_rate main_prog.lr_var = lr_var self._learning_rate_map[ - framework.default_main_program()] = lr_var + framework.default_main_program() + ] = lr_var lr_value = float(self._learning_rate()) self.helper.set_variable_initializer( - lr_var, initializer=Constant(value=lr_value)) + lr_var, initializer=Constant(value=lr_value) + ) elif isinstance(self._learning_rate, float): # only create global lr_var once lr = self._global_learning_rate() if isinstance(lr, framework.Variable): return else: - self._learning_rate_map[framework.default_main_program( - )] = layers.create_global_var( + self._learning_rate_map[ + framework.default_main_program() + ] = layers.create_global_var( name=unique_name.generate("learning_rate"), shape=[1], value=float(self._learning_rate), dtype=_lr_dtype, - persistable=True) + persistable=True, + ) @framework.dygraph_only def set_lr(self, value): @@ -454,7 +506,8 @@ def set_lr(self, value): if not isinstance(value, (int, float)): raise TypeError( "The type of 'value' in optimizer.set_lr must be float, but received %s." - % (type(value))) + % (type(value)) + ) if isinstance(self._learning_rate, LRScheduler): raise RuntimeError( "optimizer's learning rate can't be LRScheduler when invoke this API, because this will lead to conflict." @@ -464,23 +517,36 @@ def set_lr(self, value): if current_lr is not None: if in_dygraph_mode(): place = _current_expected_place() - _C_ops.full_(current_lr, list(current_lr.shape), float(value), - current_lr.dtype, place) + _C_ops.full_( + current_lr, + list(current_lr.shape), + float(value), + current_lr.dtype, + place, + ) elif _in_legacy_dygraph(): - _legacy_C_ops.fill_constant(current_lr, 'value', float(value), - 'dtype', current_lr.dtype, 'shape', - list(current_lr.shape)) + _legacy_C_ops.fill_constant( + current_lr, + 'value', + float(value), + 'dtype', + current_lr.dtype, + 'shape', + list(current_lr.shape), + ) else: global_block = framework.default_main_program().global_block() - global_block.append_op(type='fill_constant', - outputs={'Out': [current_lr]}, - attrs={ - 'dtype': current_lr.dtype, - 'shape': list(current_lr.shape), - 'value': float(value) - }, - stop_gradient=True) + global_block.append_op( + type='fill_constant', + outputs={'Out': [current_lr]}, + attrs={ + 'dtype': current_lr.dtype, + 'shape': list(current_lr.shape), + 'value': float(value), + }, + stop_gradient=True, + ) def get_lr(self): """ @@ -554,8 +620,7 @@ def _global_learning_rate(self, program=None): return self._learning_rate_map.get(program, None) def _append_optimize_op(self, block, param_and_grad): - """ append optimize operator to block and return all the added optimize_op - """ + """append optimize operator to block and return all the added optimize_op""" raise NotImplementedError( "Class \"Optimizer\" connot be used directly as an optimizer, please use its subclasses such as \"Adam\"" ) @@ -572,8 +637,8 @@ def _create_param_lr(self, param_and_grad): return self._global_learning_rate() else: with default_main_program()._lr_schedule_guard( - is_with_opt=True), framework.name_scope( - 'scale_with_param_lr'): + is_with_opt=True + ), framework.name_scope('scale_with_param_lr'): return self._global_learning_rate() * param_lr else: return self._global_learning_rate() @@ -600,14 +665,16 @@ def _finish_update(self, block, parameters_and_grads): """ pass - def _add_accumulator(self, - name, - param, - dtype=None, - fill_value=0.0, - shape=None, - type=None, - device=None): + def _add_accumulator( + self, + name, + param, + dtype=None, + fill_value=0.0, + shape=None, + type=None, + device=None, + ): """Utility function to add an accumulator for a parameter Args: @@ -619,13 +686,17 @@ def _add_accumulator(self, """ if self._name is not None: name = self._name + "_" + name - if (name in self._accumulators - and param.name in self._accumulators[name]): + if ( + name in self._accumulators + and param.name in self._accumulators[name] + ): if framework._non_static_mode(): return self._accumulators[name][param.name] raise Exception( "Accumulator {} already exists for parameter {}".format( - name, param.name)) + name, param.name + ) + ) if shape == None: shape = param.shape assert isinstance(self.helper, LayerHelper) @@ -639,20 +710,25 @@ def _add_accumulator(self, persistable=True, dtype=dtype or param.dtype, type=core.VarDesc.VarType.LOD_TENSOR - if framework._in_eager_without_dygraph_check() else - (param.type if type is None else type), + if framework._in_eager_without_dygraph_check() + else (param.type if type is None else type), shape=shape, - belong_to_optimizer=True) + belong_to_optimizer=True, + ) if device is None: device = self._get_device_for_param(param.name) with device_guard(device): self.helper.set_variable_initializer( - var, initializer=Constant(value=float(fill_value))) + var, initializer=Constant(value=float(fill_value)) + ) if framework._non_static_mode(): if len(self._accumulators_holder) > 0: - assert var_name in self._accumulators_holder, \ - "Optimizer set error, {} should in state dict".format( var_name ) + assert ( + var_name in self._accumulators_holder + ), "Optimizer set error, {} should in state dict".format( + var_name + ) var.set_value(self._accumulators_holder[var_name]) self._accumulators[name][param.name] = var @@ -670,11 +746,15 @@ def _get_accumulator(self, name, param): """ if self._name is not None: name = self._name + "_" + name - if (name not in self._accumulators - or param.name not in self._accumulators[name]): + if ( + name not in self._accumulators + or param.name not in self._accumulators[name] + ): raise Exception( "Accumulator {} does not exist for parameter {}".format( - name, param.name)) + name, param.name + ) + ) return self._accumulators[name][param.name] def _update_param_device_map(self, parameters_and_grads, target_block): @@ -682,13 +762,15 @@ def _update_param_device_map(self, parameters_and_grads, target_block): if param_and_grad[0].stop_gradient is False: param_name = param_and_grad[0].name ops = target_block.ops - device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName( + device_attr_name = ( + core.op_proto_and_checker_maker.kOpDeviceAttrName() ) for op in ops: input_arg_names = op.input_arg_names if param_name in input_arg_names: self._param_device_map[param_name] = op.attr( - device_attr_name) + device_attr_name + ) break def _get_device_for_param(self, param_name): @@ -725,10 +807,12 @@ def _create_optimization_pass(self, parameters_and_grads): target_block = global_block current_block = framework.default_main_program().current_block() if current_block.idx != global_block.idx: - assert current_block.backward_block_idx != -1, \ - "current block is not global_block, but it doesn't have backward block." + assert ( + current_block.backward_block_idx != -1 + ), "current block is not global_block, but it doesn't have backward block." target_block = framework.default_main_program().blocks[ - current_block.backward_block_idx] + current_block.backward_block_idx + ] start = len(target_block.ops) self.helper = LayerHelper(self.__class__.__name__) @@ -737,57 +821,83 @@ def _create_optimization_pass(self, parameters_and_grads): # NOTE: Multi Tensor support [ Momentum, Adam ] for dygraph mode if self._use_multi_tensor and self.__class__.__name__ in [ - 'Momentum', 'Adam' + 'Momentum', + 'Adam', ]: - if len(self._param_dict['FP32_LODTensor']) == 0 and len( - self._param_dict['FP16_LODTensor']) == 0: + if ( + len(self._param_dict['FP32_LODTensor']) == 0 + and len(self._param_dict['FP16_LODTensor']) == 0 + ): if isinstance(parameters_and_grads, list): - self._multi_tensor_init(target_block, [ - p[0] - for p in parameters_and_grads if not p[0].stop_gradient - ]) + self._multi_tensor_init( + target_block, + [ + p[0] + for p in parameters_and_grads + if not p[0].stop_gradient + ], + ) else: self._update_param_group(parameters_and_grads) - self._multi_tensor_init(target_block, [ - p[0] for p in parameters_and_grads['params'] - if not p[0].stop_gradient - ]) + self._multi_tensor_init( + target_block, + [ + p[0] + for p in parameters_and_grads['params'] + if not p[0].stop_gradient + ], + ) if framework._non_static_mode(): - self._append_optimize_multi_tensor_op(target_block, - parameters_and_grads) + self._append_optimize_multi_tensor_op( + target_block, parameters_and_grads + ) else: - self._update_param_device_map(parameters_and_grads, - target_block) + self._update_param_device_map( + parameters_and_grads, target_block + ) # NOTE: Multi Tensor requires all parameters to be in the same device and program. # param_grad_list = [p_0,g_0,p_1,g_1,....] param_grad_list = [] for param_and_grad in parameters_and_grads: - if not param_and_grad[0].stop_gradient and param_and_grad[ - 1] is not None: + if ( + not param_and_grad[0].stop_gradient + and param_and_grad[1] is not None + ): param_grad_list.append(param_and_grad[0]) param_grad_list.append(param_and_grad[1]) with param_grad_list[0].block.program._optimized_guard( - param_grad_list), name_scope("optimizer"): + param_grad_list + ), name_scope("optimizer"): device = self._get_device_for_param(param_grad_list[0].name) with device_guard(device): self._append_optimize_multi_tensor_op( - target_block, parameters_and_grads) + target_block, parameters_and_grads + ) else: if not framework._non_static_mode(): - params_grads_device_map = parameters_and_grads[ - 'params'] if isinstance(parameters_and_grads, - dict) else parameters_and_grads - self._update_param_device_map(params_grads_device_map, - target_block) + params_grads_device_map = ( + parameters_and_grads['params'] + if isinstance(parameters_and_grads, dict) + else parameters_and_grads + ) + self._update_param_device_map( + params_grads_device_map, target_block + ) if isinstance(parameters_and_grads, list): - self._create_accumulators(target_block, [ - p[0] for p in parameters_and_grads if not p[0].stop_gradient - ]) + self._create_accumulators( + target_block, + [ + p[0] + for p in parameters_and_grads + if not p[0].stop_gradient + ], + ) else: params_acc_dict = parameters_and_grads.copy() params_acc_dict['params'] = [ - p[0] for p in params_acc_dict['params'] + p[0] + for p in params_acc_dict['params'] if not p[0].stop_gradient ] self._create_accumulators(target_block, params_acc_dict) @@ -798,8 +908,9 @@ def _create_optimization_pass(self, parameters_and_grads): if param_and_grad[1] is None: continue if param_and_grad[0].stop_gradient is False: - self._append_optimize_op(target_block, - param_and_grad) + self._append_optimize_op( + target_block, param_and_grad + ) else: for param_and_grad in parameters_and_grads['params']: if param_and_grad[1] is None: @@ -807,25 +918,31 @@ def _create_optimization_pass(self, parameters_and_grads): if param_and_grad[0].stop_gradient is False: param_grad_dict = dict() param_grad_dict['params'] = param_and_grad - param_grad_dict.update({ - k: v - for k, v in parameters_and_grads.items() - if k != 'params' - }) - self._append_optimize_op(target_block, - param_grad_dict) + param_grad_dict.update( + { + k: v + for k, v in parameters_and_grads.items() + if k != 'params' + } + ) + self._append_optimize_op( + target_block, param_grad_dict + ) else: for param_and_grad in parameters_and_grads: if param_and_grad[1] is None: continue with param_and_grad[0].block.program._optimized_guard( - param_and_grad), name_scope("optimizer"): + param_and_grad + ), name_scope("optimizer"): if param_and_grad[0].stop_gradient is False: device = self._get_device_for_param( - param_and_grad[0].name) + param_and_grad[0].name + ) with device_guard(device): optimize_op = self._append_optimize_op( - target_block, param_and_grad) + target_block, param_and_grad + ) # Get custom finish ops for subclasses # FIXME: Need to fix this once we figure out how to handle dependencies @@ -837,12 +954,14 @@ def _create_optimization_pass(self, parameters_and_grads): def _append_dgc_ops(self, param_and_grad): pass - def backward(self, - loss, - startup_program=None, - parameters=None, - no_grad_set=None, - callbacks=None): + def backward( + self, + loss, + startup_program=None, + parameters=None, + no_grad_set=None, + callbacks=None, + ): """ The first part of ``minimize``, do auto-diff to append backward operations for the current program. @@ -891,8 +1010,7 @@ def backward(self, self._dtype = loss.dtype if framework._non_static_mode(): - parameter_list = parameters if parameters \ - else self._parameter_list + parameter_list = parameters if parameters else self._parameter_list params_grads = [] for param in parameter_list: @@ -906,23 +1024,26 @@ def backward(self, if callbacks is None: callbacks = [error_clip_callback] else: - assert (isinstance(callbacks, list)) + assert isinstance(callbacks, list) program = loss.block.program - assert len(loss.shape) == 1 and loss.shape[0] == 1, \ - "The loss.shape should be (1L,), but the current loss.shape is {}. " \ + assert len(loss.shape) == 1 and loss.shape[0] == 1, ( + "The loss.shape should be (1L,), but the current loss.shape is {}. " "Maybe that you should call paddle.mean to process the current loss.".format( - loss.shape) - parameter_list = parameters if parameters \ - else self._parameter_list + loss.shape + ) + ) + parameter_list = parameters if parameters else self._parameter_list with program_guard(program, startup_program): from paddle.incubate.autograd.utils import prim_enabled + if prim_enabled(): - params_grads = append_backward_new([loss], parameter_list, - act_no_grad_set, - callbacks) + params_grads = append_backward_new( + [loss], parameter_list, act_no_grad_set, callbacks + ) else: - params_grads = append_backward(loss, parameter_list, - act_no_grad_set, callbacks) + params_grads = append_backward( + loss, parameter_list, act_no_grad_set, callbacks + ) # Note: since we can't use all_reduce_op now, # dgc_op should be the last op of one grad. self._append_dgc_ops(params_grads) @@ -967,8 +1088,9 @@ def apply_gradients(self, params_grads): params_grads = append_gradient_clip_ops(params_grads) # Add regularization if any - params_grads = self.append_regularization_ops(params_grads, - self.regularization) + params_grads = self.append_regularization_ops( + params_grads, self.regularization + ) optimize_ops = self._create_optimization_pass(params_grads) return optimize_ops @@ -986,21 +1108,26 @@ def _apply_optimize(self, loss, startup_program, params_grads): list: A list of operators appended to the current program. """ if framework._non_static_mode(): - with program_guard(framework.default_main_program(), - framework.default_startup_program()): + with program_guard( + framework.default_main_program(), + framework.default_startup_program(), + ): if isinstance(params_grads, list): if self._grad_clip is not None: params_grads = self._grad_clip(params_grads) params_grads = self.append_regularization_ops( - params_grads, self.regularization) + params_grads, self.regularization + ) else: grad_clip = params_grads['grad_clip'] if grad_clip is not None: params_grads['params'] = grad_clip( - params_grads['params']) + params_grads['params'] + ) params_grads['params'] = self.append_regularization_ops( - params_grads['params'], self.regularization) + params_grads['params'], self.regularization + ) optimize_ops = self._create_optimization_pass(params_grads) else: program = loss.block.program @@ -1009,15 +1136,18 @@ def _apply_optimize(self, loss, startup_program, params_grads): return optimize_ops def _create_regularization_of_grad(self, param, grad, regularization=None): - """ Create and add backward regularization Operators + """Create and add backward regularization Operators Function helper of append_regularization_ops. """ # If no gradient or no regularization is specified, then we don't need to do anything if grad is None or ( - (not hasattr(param, 'regularizer') or - (hasattr(param, 'regularizer') and param.regularizer is None)) - and regularization is None): + ( + not hasattr(param, 'regularizer') + or (hasattr(param, 'regularizer') and param.regularizer is None) + ) + and regularization is None + ): return grad regularization_term = None if hasattr(param, 'regularizer') and param.regularizer is not None: @@ -1044,7 +1174,8 @@ def _create_regularization_of_grad(self, param, grad, regularization=None): dtype=param.dtype, shape=param.shape, lod_level=param.lod_level, - type=core.VarDesc.VarType.LOD_TENSOR) + type=core.VarDesc.VarType.LOD_TENSOR, + ) inputs = {"X": [grad, regularization_term]} outputs = {"Out": [new_grad]} @@ -1052,9 +1183,9 @@ def _create_regularization_of_grad(self, param, grad, regularization=None): return new_grad - def append_regularization_ops(self, - parameters_and_grads, - regularization=None): + def append_regularization_ops( + self, parameters_and_grads, regularization=None + ): r"""Create and add backward regularization Operators Creates and adds backward regularization operators in the BlockDesc. @@ -1079,21 +1210,28 @@ def append_regularization_ops(self, if framework._non_static_mode(): for param, grad in parameters_and_grads: new_grad = self._create_regularization_of_grad( - param, grad, regularization) + param, grad, regularization + ) params_and_grads.append((param, new_grad)) else: repeate_regularizer = False with framework.name_scope('regularization'): for param, grad in parameters_and_grads: - if not repeate_regularizer and param.regularizer is not None and regularization is not None: + if ( + not repeate_regularizer + and param.regularizer is not None + and regularization is not None + ): repeate_regularizer = True logging.info( "If regularizer of a Parameter has been set by 'fluid.ParamAttr' or 'fluid.WeightNormParamAttr' already. " "The Regularization[%s] in Optimizer will not take effect, and it will only be applied to other Parameters!" - % regularization.__str__()) + % regularization.__str__() + ) with param.block.program._optimized_guard([param, grad]): new_grad = self._create_regularization_of_grad( - param, grad, regularization) + param, grad, regularization + ) params_and_grads.append((param, new_grad)) return params_and_grads @@ -1101,7 +1239,8 @@ def _get_no_grad_set(self, loss, no_grad_set=None): no_grad_set = _get_no_grad_set_name(no_grad_set) parameters = loss.block.program.global_block().all_parameters() param_no_trainable = set( - [param.name for param in parameters if param.stop_gradient is True]) + [param.name for param in parameters if param.stop_gradient is True] + ) # If the parameter is no trainable, it should not have a gradient. no_grad_set.update(param_no_trainable) @@ -1142,7 +1281,8 @@ def clear_grad(self, set_to_zero=True): """ param_list = [] if self._parameter_list is None or not isinstance( - self._parameter_list[0], dict): + self._parameter_list[0], dict + ): for p in self._parameter_list: if not p.stop_gradient: param_list.append(p) @@ -1159,11 +1299,9 @@ def clear_grad(self, set_to_zero=True): core.clear_gradients(param_list, set_to_zero) @imperative_base.no_grad - def minimize(self, - loss, - startup_program=None, - parameters=None, - no_grad_set=None): + def minimize( + self, loss, startup_program=None, parameters=None, no_grad_set=None + ): """ Add operations to minimize ``loss`` by updating ``parameters``. @@ -1208,17 +1346,18 @@ def minimize(self, """ assert isinstance(loss, Variable), "The loss should be an Tensor." - parameter_list = parameters if parameters \ - else self._parameter_list + parameter_list = parameters if parameters else self._parameter_list - params_grads = self.backward(loss, - startup_program=startup_program, - parameters=parameter_list, - no_grad_set=no_grad_set) + params_grads = self.backward( + loss, + startup_program=startup_program, + parameters=parameter_list, + no_grad_set=no_grad_set, + ) - optimize_ops = self._apply_optimize(loss, - startup_program=startup_program, - params_grads=params_grads) + optimize_ops = self._apply_optimize( + loss, startup_program=startup_program, params_grads=params_grads + ) return optimize_ops, params_grads @@ -1258,9 +1397,9 @@ def step(self): grad_var = param._grad_ivar() params_grads.append((param, grad_var)) - self._apply_optimize(loss=None, - startup_program=None, - params_grads=params_grads) + self._apply_optimize( + loss=None, startup_program=None, params_grads=params_grads + ) else: # optimize parameters in groups @@ -1273,11 +1412,11 @@ def step(self): grad_var = param._grad_ivar() params_grads['params'].append((param, grad_var)) params_grads.update( - {k: v - for k, v in param_group.items() if k != 'params'}) - self._apply_optimize(loss=None, - startup_program=None, - params_grads=params_grads) + {k: v for k, v in param_group.items() if k != 'params'} + ) + self._apply_optimize( + loss=None, startup_program=None, params_grads=params_grads + ) def _add_param_group(self, param_group): """ @@ -1293,7 +1432,8 @@ def _add_param_group(self, param_group): elif isinstance(params, set): raise TypeError( "optimizer parameters should be in ordered collections," - "but received set, please use list instead.") + "but received set, please use list instead." + ) else: param_group['params'] = list(params) @@ -1307,18 +1447,21 @@ def _add_param_group(self, param_group): if not param_set.isdisjoint(set(param_group['params'])): raise ValueError( - "some parameters appear in more than one parameter group") + "some parameters appear in more than one parameter group" + ) for param in param_group['params']: weight_decay = param_group['weight_decay'] if isinstance(weight_decay, float): from ..fluid.regularizer import L2Decay + regularization = L2Decay(weight_decay) else: regularization = weight_decay param.regularizer = regularization param.optimize_attr['learning_rate'] = param_group.get( - 'learning_rate', 1.) + 'learning_rate', 1.0 + ) self._param_groups.append(param_group) @@ -1344,8 +1487,9 @@ def _multi_tensor_init(self, target_block, parameters): pass @framework.dygraph_only - def _append_optimize_multi_tensor_op(self, target_block, - parameters_and_grads): + def _append_optimize_multi_tensor_op( + self, target_block, parameters_and_grads + ): """ For Multi Tensor, append optimize merged_operator to block. """ diff --git a/python/paddle/optimizer/rmsprop.py b/python/paddle/optimizer/rmsprop.py index 5ae5e1468beed..06d7c58fb39f1 100644 --- a/python/paddle/optimizer/rmsprop.py +++ b/python/paddle/optimizer/rmsprop.py @@ -141,16 +141,18 @@ class RMSProp(Optimizer): _mean_square_acc_str = "mean_square" _mean_grad_acc_str = "mean_grad" - def __init__(self, - learning_rate, - rho=0.95, - epsilon=1.0e-6, - momentum=0.0, - centered=False, - parameters=None, - weight_decay=None, - grad_clip=None, - name=None): + def __init__( + self, + learning_rate, + rho=0.95, + epsilon=1.0e-6, + momentum=0.0, + centered=False, + parameters=None, + weight_decay=None, + grad_clip=None, + name=None, + ): if learning_rate is None: raise ValueError("learning_rate is not set.") if rho is None: @@ -166,11 +168,13 @@ def __init__(self, if not 0.0 <= rho: raise ValueError("Invalid value of rho, expect rho >= 0.") - super(RMSProp, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=weight_decay, - grad_clip=grad_clip, - name=name) + super(RMSProp, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=weight_decay, + grad_clip=grad_clip, + name=name, + ) self.type = "rmsprop" self._rho = rho @@ -203,49 +207,50 @@ def _append_optimize_op(self, block, param_and_grad): if isinstance(param_and_grad, dict): param_and_grad = self._update_param_group(param_and_grad) - momentum_acc = self._get_accumulator(self._momentum_acc_str, - param_and_grad[0]) - mean_square_acc = self._get_accumulator(self._mean_square_acc_str, - param_and_grad[0]) - mean_grad_acc = self._get_accumulator(self._mean_grad_acc_str, - param_and_grad[0]) - rmsprop_op = block.append_op(type=self.type, - inputs={ - "Param": - param_and_grad[0], - "Grad": - param_and_grad[1], - "Moment": - momentum_acc, - "MeanSquare": - mean_square_acc, - "MeanGrad": - mean_grad_acc, - "LearningRate": - self._create_param_lr(param_and_grad), - }, - outputs={ - "ParamOut": param_and_grad[0], - "MomentOut": momentum_acc, - "MeanSquareOut": mean_square_acc, - "MeanGradOut": mean_grad_acc - }, - attrs={ - "epsilon": self._epsilon, - "decay": self._rho, - "momentum": self._momentum, - "centered": self._centered - }, - stop_gradient=True) + momentum_acc = self._get_accumulator( + self._momentum_acc_str, param_and_grad[0] + ) + mean_square_acc = self._get_accumulator( + self._mean_square_acc_str, param_and_grad[0] + ) + mean_grad_acc = self._get_accumulator( + self._mean_grad_acc_str, param_and_grad[0] + ) + rmsprop_op = block.append_op( + type=self.type, + inputs={ + "Param": param_and_grad[0], + "Grad": param_and_grad[1], + "Moment": momentum_acc, + "MeanSquare": mean_square_acc, + "MeanGrad": mean_grad_acc, + "LearningRate": self._create_param_lr(param_and_grad), + }, + outputs={ + "ParamOut": param_and_grad[0], + "MomentOut": momentum_acc, + "MeanSquareOut": mean_square_acc, + "MeanGradOut": mean_grad_acc, + }, + attrs={ + "epsilon": self._epsilon, + "decay": self._rho, + "momentum": self._momentum, + "centered": self._centered, + }, + stop_gradient=True, + ) return rmsprop_op def _update_param_group(self, parameters): self._epsilon = parameters.get('epsilon', self._default_dict['epsilon']) self._rho = parameters.get('rho', self._default_dict['rho']) - self._momentum = parameters.get('momentum', - self._default_dict['momentum']) - self._centered = parameters.get('centered', - self._default_dict['centered']) + self._momentum = parameters.get( + 'momentum', self._default_dict['momentum'] + ) + self._centered = parameters.get( + 'centered', self._default_dict['centered'] + ) parameters = parameters.get('params') return parameters diff --git a/python/paddle/optimizer/sgd.py b/python/paddle/optimizer/sgd.py index 40db1d361eba5..b8a9331b422ed 100644 --- a/python/paddle/optimizer/sgd.py +++ b/python/paddle/optimizer/sgd.py @@ -72,20 +72,24 @@ class SGD(Optimizer): """ - def __init__(self, - learning_rate=0.001, - parameters=None, - weight_decay=None, - grad_clip=None, - multi_precision=False, - name=None): + def __init__( + self, + learning_rate=0.001, + parameters=None, + weight_decay=None, + grad_clip=None, + multi_precision=False, + name=None, + ): if learning_rate is None: raise ValueError("learning_rate is not set") - super(SGD, self).__init__(learning_rate=learning_rate, - parameters=parameters, - weight_decay=weight_decay, - grad_clip=grad_clip, - name=name) + super(SGD, self).__init__( + learning_rate=learning_rate, + parameters=parameters, + weight_decay=weight_decay, + grad_clip=grad_clip, + name=name, + ) self.type = "sgd" self._multi_precision = multi_precision self._master_weights = {} @@ -98,19 +102,23 @@ def _create_master_weight(self, param): var_name = param.name + "_fp32_master" var_name = unique_name.generate(var_name) - var = layers.create_global_var(name=var_name, - shape=param.shape, - value=0, - dtype='float32', - persistable=True) + var = layers.create_global_var( + name=var_name, + shape=param.shape, + value=0, + dtype='float32', + persistable=True, + ) block = self.helper.startup_program.global_block() - block.append_op(type="cast", - inputs={"X": [param]}, - outputs={"Out": [var]}, - attrs={ - "in_dtype": param.dtype, - "out_dtype": core.VarDesc.VarType.FP32 - }) + block.append_op( + type="cast", + inputs={"X": [param]}, + outputs={"Out": [var]}, + attrs={ + "in_dtype": param.dtype, + "out_dtype": core.VarDesc.VarType.FP32, + }, + ) self._master_weights[param.name] = var return var @@ -124,7 +132,10 @@ def _create_accumulators(self, block, parameters): if self._multi_precision and p.dtype == core.VarDesc.VarType.FP16: master_p = self._create_master_weight(p) continue - if p.dtype == core.VarDesc.VarType.FP16 and not self._multi_precision: + if ( + p.dtype == core.VarDesc.VarType.FP16 + and not self._multi_precision + ): warnings.warn( "Accumulating with FP16 in optimizer can lead to poor accuracy or slow convergence." "Consider using multi_precision=True option of the Adam optimizer." @@ -135,19 +146,35 @@ def _append_optimize_op(self, block, param_and_grad): if isinstance(param_and_grad, dict): param_and_grad = self._update_param_group(param_and_grad) - find_master = self._multi_precision and param_and_grad[ - 0].dtype == core.VarDesc.VarType.FP16 - master_weight = (self._master_weights[param_and_grad[0].name] - if find_master else None) + find_master = ( + self._multi_precision + and param_and_grad[0].dtype == core.VarDesc.VarType.FP16 + ) + master_weight = ( + self._master_weights[param_and_grad[0].name] + if find_master + else None + ) lr = self._create_param_lr(param_and_grad) if in_dygraph_mode(): - _C_ops.sgd_(param_and_grad[0], lr, param_and_grad[1], master_weight, - find_master) + _C_ops.sgd_( + param_and_grad[0], + lr, + param_and_grad[1], + master_weight, + find_master, + ) return None if _in_legacy_dygraph(): - _legacy_C_ops.sgd(param_and_grad[0], lr, param_and_grad[1], - master_weight, param_and_grad[0], master_weight) + _legacy_C_ops.sgd( + param_and_grad[0], + lr, + param_and_grad[1], + master_weight, + param_and_grad[0], + master_weight, + ) return None assert isinstance(block, framework.Block) @@ -155,7 +182,7 @@ def _append_optimize_op(self, block, param_and_grad): inputs = { "Param": param_and_grad[0], "Grad": param_and_grad[1], - "LearningRate": lr + "LearningRate": lr, } outputs = {"ParamOut": param_and_grad[0]} @@ -166,11 +193,13 @@ def _append_optimize_op(self, block, param_and_grad): inputs["MasterParam"] = master_weight outputs["MasterParamOut"] = master_weight - sgd_op = block.append_op(type=self.type, - inputs=inputs, - outputs=outputs, - attrs=attrs, - stop_gradient=True) + sgd_op = block.append_op( + type=self.type, + inputs=inputs, + outputs=outputs, + attrs=attrs, + stop_gradient=True, + ) return sgd_op diff --git a/python/paddle/profiler/__init__.py b/python/paddle/profiler/__init__.py index 76108cf2953f8..a59f8f14a54a2 100644 --- a/python/paddle/profiler/__init__.py +++ b/python/paddle/profiler/__init__.py @@ -21,7 +21,14 @@ from .profiler_statistic import SortedKeys __all__ = [ - 'ProfilerState', 'ProfilerTarget', 'make_scheduler', - 'export_chrome_tracing', 'export_protobuf', 'Profiler', 'RecordEvent', - 'load_profiler_result', 'SortedKeys', 'SummaryView' + 'ProfilerState', + 'ProfilerTarget', + 'make_scheduler', + 'export_chrome_tracing', + 'export_protobuf', + 'Profiler', + 'RecordEvent', + 'load_profiler_result', + 'SortedKeys', + 'SummaryView', ] diff --git a/python/paddle/profiler/profiler.py b/python/paddle/profiler/profiler.py index e4c32db2e9f83..bcd87d9c826fc 100644 --- a/python/paddle/profiler/profiler.py +++ b/python/paddle/profiler/profiler.py @@ -22,11 +22,15 @@ import json import paddle -from paddle.fluid.core import (_Profiler, ProfilerOptions, TracerEventType, - enable_memory_recorder, - enable_input_shape_recorder, - disable_memory_recorder, - disable_input_shape_recorder) +from paddle.fluid.core import ( + _Profiler, + ProfilerOptions, + TracerEventType, + enable_memory_recorder, + enable_input_shape_recorder, + disable_memory_recorder, + disable_input_shape_recorder, +) from .utils import RecordEvent, wrap_optimizers from .profiler_statistic import StatisticData, _build_table, SortedKeys @@ -105,12 +109,14 @@ class ProfilerTarget(Enum): CUSTOM_DEVICE = 3 -def make_scheduler(*, - closed: int, - ready: int, - record: int, - repeat: int = 0, - skip_first: int = 0) -> Callable: +def make_scheduler( + *, + closed: int, + ready: int, + record: int, + repeat: int = 0, + skip_first: int = 0 +) -> Callable: r""" Return a scheduler function, which scheduler the :ref:`state ` according to the setting. The state transform confirms to: @@ -164,7 +170,9 @@ def getScheduleState(step: int) -> ProfilerState: step = step - skip_first period_steps = closed + ready + record has_repeated = step // period_steps - if repeat > 0 and has_repeated >= repeat: # the period has repeated repeat times, return CLOSED state + if ( + repeat > 0 and has_repeated >= repeat + ): # the period has repeated repeat times, return CLOSED state return ProfilerState.CLOSED mod_step = step % period_steps if mod_step < closed: @@ -176,12 +184,19 @@ def getScheduleState(step: int) -> ProfilerState: return ProfilerState.RECORD else: return ProfilerState.RECORD_AND_RETURN - assert closed >= 0 and ready >= 0 and record > 0 and \ - repeat >= 0 and skip_first >= 0, "Invalid profiler scheduler arguments" + + assert ( + closed >= 0 + and ready >= 0 + and record > 0 + and repeat >= 0 + and skip_first >= 0 + ), "Invalid profiler scheduler arguments" if ready == 0: - warn("Profiler will record data after enabling profiler immediately, \ + warn( + "Profiler will record data after enabling profiler immediately, \ some data collected at the beginning of profiling may be 'noisy' because of overhead." - ) + ) return getScheduleState @@ -192,8 +207,9 @@ def _default_state_scheduler(step: int): return ProfilerState.RECORD -def export_chrome_tracing(dir_name: str, - worker_name: Optional[str] = None) -> Callable: +def export_chrome_tracing( + dir_name: str, worker_name: Optional[str] = None +) -> Callable: r""" Return a callable, used for outputing tracing data to chrome tracing format file. The output file will be saved in directory ``dir_name``, and file name will be set as `worker_name`. @@ -226,24 +242,29 @@ def export_chrome_tracing(dir_name: str, os.makedirs(dir_name, exist_ok=True) except Exception: raise RuntimeError( - "Can not create directory '{}' for saving profiling results.". - format(dir_name)) + "Can not create directory '{}' for saving profiling results.".format( + dir_name + ) + ) def handle_fn(prof): nonlocal worker_name if not worker_name: - worker_name = "host_{}pid_{}".format(socket.gethostname(), - str(os.getpid())) + worker_name = "host_{}pid_{}".format( + socket.gethostname(), str(os.getpid()) + ) now = datetime.datetime.now() filename = '{}_time_{}.paddle_trace.json'.format( - worker_name, now.strftime('%Y_%m_%d_%H_%M_%S_%f')) + worker_name, now.strftime('%Y_%m_%d_%H_%M_%S_%f') + ) prof.export(os.path.join(dir_name, filename), "json") return handle_fn -def export_protobuf(dir_name: str, - worker_name: Optional[str] = None) -> Callable: +def export_protobuf( + dir_name: str, worker_name: Optional[str] = None +) -> Callable: r""" Return a callable, used for outputing tracing data to protobuf file. The output file will be saved in directory ``dir_name``, and file name will be set as ``worker_name``. @@ -276,17 +297,21 @@ def export_protobuf(dir_name: str, os.makedirs(dir_name, exist_ok=True) except Exception: raise RuntimeError( - "Can not create directory '{}' for saving profiling results.". - format(dir_name)) + "Can not create directory '{}' for saving profiling results.".format( + dir_name + ) + ) def handle_fn(prof): nonlocal worker_name if not worker_name: - worker_name = "host_{}pid_{}".format(socket.gethostname(), - str(os.getpid())) + worker_name = "host_{}pid_{}".format( + socket.gethostname(), str(os.getpid()) + ) now = datetime.datetime.now() filename = '{}_time_{}.paddle_trace.pb'.format( - worker_name, now.strftime('%Y_%m_%d_%H_%M_%S_%f')) + worker_name, now.strftime('%Y_%m_%d_%H_%M_%S_%f') + ) prof.export(os.path.join(dir_name, filename), "pb") return handle_fn @@ -298,11 +323,15 @@ def _get_supported_targets() -> Iterable[ProfilerTarget]: """ if _Profiler.is_cupti_supported(): return [ - ProfilerTarget.CPU, ProfilerTarget.GPU, ProfilerTarget.CUSTOM_DEVICE + ProfilerTarget.CPU, + ProfilerTarget.GPU, + ProfilerTarget.CUSTOM_DEVICE, ] if _Profiler.is_cnpapi_supported(): return [ - ProfilerTarget.CPU, ProfilerTarget.MLU, ProfilerTarget.CUSTOM_DEVICE + ProfilerTarget.CPU, + ProfilerTarget.MLU, + ProfilerTarget.CUSTOM_DEVICE, ] return [ProfilerTarget.CPU, ProfilerTarget.CUSTOM_DEVICE] @@ -432,36 +461,40 @@ def forward(self, image, label=None): # | ips | 1086.42904 | 1227.30604 | 959.92796 | """ - def __init__(self, - *, - targets: Optional[Iterable[ProfilerTarget]] = None, - scheduler: Union[Callable[[int], ProfilerState], tuple, - None] = None, - on_trace_ready: Optional[Callable[..., Any]] = None, - record_shapes: Optional[bool] = False, - profile_memory=False, - timer_only: Optional[bool] = False, - emit_nvtx: Optional[bool] = False, - custom_device_types: Optional[list] = []): + def __init__( + self, + *, + targets: Optional[Iterable[ProfilerTarget]] = None, + scheduler: Union[Callable[[int], ProfilerState], tuple, None] = None, + on_trace_ready: Optional[Callable[..., Any]] = None, + record_shapes: Optional[bool] = False, + profile_memory=False, + timer_only: Optional[bool] = False, + emit_nvtx: Optional[bool] = False, + custom_device_types: Optional[list] = [] + ): supported_targets = _get_supported_targets() if targets: self.targets = set(targets) for target in targets: if target not in supported_targets: self.targets.remove(target) - warn("Profiling {} is not supported in current context.". - format(target)) + warn( + "Profiling {} is not supported in current context.".format( + target + ) + ) else: self.targets = supported_targets profileoption = ProfilerOptions() if ProfilerTarget.CPU in self.targets: profileoption.trace_switch |= 1 if ProfilerTarget.GPU in self.targets: - profileoption.trace_switch |= (1 << 1) + profileoption.trace_switch |= 1 << 1 if ProfilerTarget.MLU in self.targets: - profileoption.trace_switch |= (1 << 2) + profileoption.trace_switch |= 1 << 2 if ProfilerTarget.CUSTOM_DEVICE in self.targets: - profileoption.trace_switch |= (1 << 3) + profileoption.trace_switch |= 1 << 3 if not custom_device_types: custom_device_types = paddle.device.get_all_custom_device_type() wrap_optimizers() @@ -473,17 +506,19 @@ def __init__(self, start_batch, end_batch = scheduler start_batch = max(start_batch, 0) if start_batch >= 1: - self.scheduler = make_scheduler(closed=max(start_batch - 1, 0), - ready=1, - record=(end_batch - - start_batch), - repeat=1) + self.scheduler = make_scheduler( + closed=max(start_batch - 1, 0), + ready=1, + record=(end_batch - start_batch), + repeat=1, + ) else: - self.scheduler = make_scheduler(closed=0, - ready=0, - record=(end_batch - - start_batch), - repeat=1) + self.scheduler = make_scheduler( + closed=0, + ready=0, + record=(end_batch - start_batch), + repeat=1, + ) else: self.scheduler = _default_state_scheduler @@ -549,9 +584,10 @@ def start(self): elif self.current_state == ProfilerState.RECORD_AND_RETURN: self.profiler.prepare() self.profiler.start() - self.record_event = RecordEvent(name="ProfileStep#{}".format( - self.step_num), - event_type=TracerEventType.ProfileStep) + self.record_event = RecordEvent( + name="ProfileStep#{}".format(self.step_num), + event_type=TracerEventType.ProfileStep, + ) self.record_event.begin() def stop(self): @@ -593,7 +629,10 @@ def stop(self): ) self.profiler.start() self.profiler.stop() - if self.current_state == ProfilerState.RECORD or self.current_state == ProfilerState.RECORD_AND_RETURN: + if ( + self.current_state == ProfilerState.RECORD + or self.current_state == ProfilerState.RECORD_AND_RETURN + ): self.profiler_result = self.profiler.stop() if self.on_trace_ready: self.on_trace_ready(self) @@ -635,9 +674,10 @@ def step(self, num_samples: Optional[int] = None): self.step_num += 1 self.current_state = self.scheduler(self.step_num) self._trigger_action() - self.record_event = RecordEvent(name="ProfileStep#{}".format( - self.step_num), - event_type=TracerEventType.ProfileStep) + self.record_event = RecordEvent( + name="ProfileStep#{}".format(self.step_num), + event_type=TracerEventType.ProfileStep, + ) self.record_event.begin() def step_info(self, unit=None): @@ -694,7 +734,9 @@ def _trigger_action(self): if self.current_state == ProfilerState.RECORD: # CLOSED -> RECORD self.profiler.prepare() self.profiler.start() - if self.current_state == ProfilerState.RECORD_AND_RETURN: # CLOSED -> RECORD_AND_RETURN + if ( + self.current_state == ProfilerState.RECORD_AND_RETURN + ): # CLOSED -> RECORD_AND_RETURN self.profiler.prepare() self.profiler.start() @@ -707,7 +749,9 @@ def _trigger_action(self): self.profiler.stop() if self.current_state == ProfilerState.RECORD: # READY -> RECORD self.profiler.start() - if self.current_state == ProfilerState.RECORD_AND_RETURN: # READY -> RECORD_AND_RETURN + if ( + self.current_state == ProfilerState.RECORD_AND_RETURN + ): # READY -> RECORD_AND_RETURN self.profiler.start() elif self.previous_state == ProfilerState.RECORD: @@ -723,21 +767,31 @@ def _trigger_action(self): ) self.profiler.stop() self.profiler.prepare() - if self.current_state == ProfilerState.RECORD_AND_RETURN: # RECORD -> RECORD_AND_RETURN + if ( + self.current_state == ProfilerState.RECORD_AND_RETURN + ): # RECORD -> RECORD_AND_RETURN pass else: assert self.previous_state == ProfilerState.RECORD_AND_RETURN - if self.current_state == ProfilerState.CLOSED: # RECORD_AND_RETURN -> CLOSED + if ( + self.current_state == ProfilerState.CLOSED + ): # RECORD_AND_RETURN -> CLOSED self.profiler_result = self.profiler.stop() - if self.current_state == ProfilerState.READY: # RECORD_AND_RETURN -> READY + if ( + self.current_state == ProfilerState.READY + ): # RECORD_AND_RETURN -> READY self.profiler_result = self.profiler.stop() self.profiler.prepare() - if self.current_state == ProfilerState.RECORD: # RECORD_AND_RETURN -> RECORD + if ( + self.current_state == ProfilerState.RECORD + ): # RECORD_AND_RETURN -> RECORD self.profiler_result = self.profiler.stop() self.profiler.prepare() self.profiler.start() - if self.current_state == ProfilerState.RECORD_AND_RETURN: # RECORD_AND_RETURN -> RECORD_AND_RETURN + if ( + self.current_state == ProfilerState.RECORD_AND_RETURN + ): # RECORD_AND_RETURN -> RECORD_AND_RETURN self.profiler_result = self.profiler.stop() self.profiler.prepare() self.profiler.start() @@ -772,12 +826,14 @@ def export(self, path="", format="json"): if self.profiler_result: self.profiler_result.save(path, format) - def summary(self, - sorted_by=SortedKeys.CPUTotal, - op_detail=True, - thread_sep=False, - time_unit='ms', - views=None): + def summary( + self, + sorted_by=SortedKeys.CPUTotal, + op_detail=True, + thread_sep=False, + time_unit='ms', + views=None, + ): r""" Print the Summary table. Currently support overview, model, distributed, operator, memory manipulation and userdefined summary. @@ -811,14 +867,18 @@ def summary(self, if self.profiler_result: statistic_data = StatisticData( self.profiler_result.get_data(), - self.profiler_result.get_extra_info()) + self.profiler_result.get_extra_info(), + ) print( - _build_table(statistic_data, - sorted_by=sorted_by, - op_detail=op_detail, - thread_sep=thread_sep, - time_unit=time_unit, - views=views)) + _build_table( + statistic_data, + sorted_by=sorted_by, + op_detail=op_detail, + thread_sep=thread_sep, + time_unit=time_unit, + views=views, + ) + ) def get_profiler(config_path): @@ -851,17 +911,20 @@ def get_profiler(config_path): method = getattr(module, key) if not use_direct: translated_config_dict['scheduler'] = method( - *value['args'], **value['kwargs']) + *value['args'], **value['kwargs'] + ) else: translated_config_dict['scheduler'] = method else: translated_config_dict['scheduler'] = [ - config_dict['scheduler'][0], config_dict['scheduler'][1] + config_dict['scheduler'][0], + config_dict['scheduler'][1], ] except: print( - 'Set scheduler parameter error, use default parameter instead.') + 'Set scheduler parameter error, use default parameter instead.' + ) translated_config_dict['scheduler'] = None if "on_trace_ready" in config_dict: try: @@ -873,7 +936,8 @@ def get_profiler(config_path): method = getattr(module, key) if not use_direct: translated_config_dict['on_trace_ready'] = method( - *value['args'], **value['kwargs']) + *value['args'], **value['kwargs'] + ) else: translated_config_dict['on_trace_ready'] = method except: diff --git a/python/paddle/profiler/profiler_statistic.py b/python/paddle/profiler/profiler_statistic.py index 20465f1ae85ce..c383323d51f98 100755 --- a/python/paddle/profiler/profiler_statistic.py +++ b/python/paddle/profiler/profiler_statistic.py @@ -17,16 +17,29 @@ from paddle.fluid.core import TracerEventType, TracerMemEventType -from .statistic_helper import intersection_ranges, merge_ranges, merge_self_ranges, sum_ranges +from .statistic_helper import ( + intersection_ranges, + merge_ranges, + merge_self_ranges, + sum_ranges, +) _AllTracerEventType = [ - TracerEventType.Operator, TracerEventType.Dataloader, - TracerEventType.ProfileStep, TracerEventType.CudaRuntime, - TracerEventType.Kernel, TracerEventType.Memcpy, TracerEventType.Memset, - TracerEventType.UserDefined, TracerEventType.OperatorInner, - TracerEventType.Forward, TracerEventType.Backward, - TracerEventType.Optimization, TracerEventType.Communication, - TracerEventType.PythonOp, TracerEventType.PythonUserDefined + TracerEventType.Operator, + TracerEventType.Dataloader, + TracerEventType.ProfileStep, + TracerEventType.CudaRuntime, + TracerEventType.Kernel, + TracerEventType.Memcpy, + TracerEventType.Memset, + TracerEventType.UserDefined, + TracerEventType.OperatorInner, + TracerEventType.Forward, + TracerEventType.Backward, + TracerEventType.Optimization, + TracerEventType.Communication, + TracerEventType.PythonOp, + TracerEventType.PythonUserDefined, ] _CommunicationOpName = ['allreduce', 'broadcast', 'rpc'] @@ -90,19 +103,19 @@ def cal_statistic(self): for child in self.children_node: self.gpu_time += child.gpu_time self.general_gpu_time += child.general_gpu_time - self.self_cpu_time -= (child.end_ns - child.start_ns) + self.self_cpu_time -= child.end_ns - child.start_ns for rt in self.runtime_node: - self.self_cpu_time -= (rt.end_ns - rt.start_ns) + self.self_cpu_time -= rt.end_ns - rt.start_ns self.gpu_time += rt.gpu_time self.self_gpu_time += rt.gpu_time self.general_gpu_time += rt.general_gpu_time self.self_general_gpu_time += rt.general_gpu_time for device in self.hostnode.device_node: if device.type == TracerEventType.Kernel: - self.gpu_time += (device.end_ns - device.start_ns) - self.self_gpu_time += (device.end_ns - device.start_ns) - self.general_gpu_time += (device.end_ns - device.start_ns) - self.self_general_gpu_time += (device.end_ns - device.start_ns) + self.gpu_time += device.end_ns - device.start_ns + self.self_gpu_time += device.end_ns - device.start_ns + self.general_gpu_time += device.end_ns - device.start_ns + self.self_general_gpu_time += device.end_ns - device.start_ns @property def end_ns(self): @@ -172,12 +185,14 @@ def wrap_tree(nodetrees): stack.append(childnode) child_statistic_node = HostStatisticNode(childnode) current_statistic_node.children_node.append( - child_statistic_node) + child_statistic_node + ) newstack.append(child_statistic_node) for runtimenode in current_node.runtime_node: runtime_statistic_node = HostStatisticNode(runtimenode) current_statistic_node.runtime_node.append( - runtime_statistic_node) + runtime_statistic_node + ) # recursive calculate node statistic values for thread_id, root_statistic_node in node_statistic_tree.items(): root_statistic_node.cal_statistic() @@ -193,11 +208,12 @@ class TimeRangeSummary: def __init__(self): self.CPUTimeRange = collections.defaultdict(list) self.GPUTimeRange = collections.defaultdict( - lambda: collections.defaultdict( - list)) # GPU events should be divided into different devices + lambda: collections.defaultdict(list) + ) # GPU events should be divided into different devices self.CPUTimeRangeSum = collections.defaultdict(int) self.GPUTimeRangeSum = collections.defaultdict( - lambda: collections.defaultdict(int)) + lambda: collections.defaultdict(int) + ) self.call_times = collections.defaultdict(int) def parse(self, nodetrees): @@ -208,42 +224,50 @@ def parse(self, nodetrees): for threadid, hostnodes in thread2hostnodes.items(): CPUTimeRange = collections.defaultdict(list) GPUTimeRange = collections.defaultdict( - lambda: collections.defaultdict(lambda: collections.defaultdict( - list))) # device_id/type/stream_id - for hostnode in hostnodes[1:]: #skip root node + lambda: collections.defaultdict( + lambda: collections.defaultdict(list) + ) + ) # device_id/type/stream_id + for hostnode in hostnodes[1:]: # skip root node CPUTimeRange[hostnode.type].append( - (hostnode.start_ns, hostnode.end_ns)) + (hostnode.start_ns, hostnode.end_ns) + ) self.call_times[hostnode.type] += 1 for runtimenode in hostnode.runtime_node: CPUTimeRange[runtimenode.type].append( - (runtimenode.start_ns, runtimenode.end_ns)) + (runtimenode.start_ns, runtimenode.end_ns) + ) self.call_times[runtimenode.type] += 1 for devicenode in runtimenode.device_node: GPUTimeRange[devicenode.device_id][devicenode.type][ - devicenode.stream_id].append( - (devicenode.start_ns, devicenode.end_ns)) + devicenode.stream_id + ].append((devicenode.start_ns, devicenode.end_ns)) self.call_times[devicenode.type] += 1 for event_type, time_ranges in CPUTimeRange.items(): time_ranges = merge_self_ranges(time_ranges, is_sorted=False) self.CPUTimeRange[event_type] = merge_ranges( - self.CPUTimeRange[event_type], time_ranges, is_sorted=True) + self.CPUTimeRange[event_type], time_ranges, is_sorted=True + ) for device_id, device_time_ranges in GPUTimeRange.items(): for event_type, event_time_ranges in device_time_ranges.items(): for stream_id, time_ranges in event_time_ranges.items(): - time_ranges = merge_self_ranges(time_ranges, - is_sorted=False) + time_ranges = merge_self_ranges( + time_ranges, is_sorted=False + ) self.GPUTimeRange[device_id][event_type] = merge_ranges( self.GPUTimeRange[device_id][event_type], time_ranges, - is_sorted=True) + is_sorted=True, + ) for event_type, time_ranges in self.CPUTimeRange.items(): self.CPUTimeRangeSum[event_type] = sum_ranges(time_ranges) for device_id, device_time_ranges in self.GPUTimeRange.items(): for event_type, time_ranges in device_time_ranges.items(): self.GPUTimeRangeSum[device_id][event_type] = sum_ranges( - time_ranges) + time_ranges + ) def get_gpu_devices(self): return self.GPUTimeRange.keys() @@ -276,57 +300,68 @@ def parse(self, nodetrees): ''' thread2hostnodes = traverse_tree(nodetrees) for threadid, hostnodes in thread2hostnodes.items(): - for hostnode in hostnodes[1:]: #skip root node + for hostnode in hostnodes[1:]: # skip root node # case 1: TracerEventType is Communication if hostnode.type == TracerEventType.Communication: self.cpu_communication_range.append( - (hostnode.start_ns, hostnode.end_ns)) + (hostnode.start_ns, hostnode.end_ns) + ) device_nodes = get_device_nodes(hostnode) for device_node in device_nodes: if device_node.type == TracerEventType.Kernel: self.gpu_communication_range.append( - (device_node.start_ns, device_node.end_ns)) + (device_node.start_ns, device_node.end_ns) + ) - #case 2: TracerEventType is Operator but is communication op - elif hostnode.type == TracerEventType.Operator and any([ + # case 2: TracerEventType is Operator but is communication op + elif hostnode.type == TracerEventType.Operator and any( + [ name in hostnode.name.lower() for name in _CommunicationOpName - ]): + ] + ): self.cpu_communication_range.append( - (hostnode.start_ns, hostnode.end_ns)) + (hostnode.start_ns, hostnode.end_ns) + ) device_nodes = get_device_nodes(hostnode) for device_node in device_nodes: if device_node.type == TracerEventType.Kernel: self.gpu_communication_range.append( - (device_node.start_ns, device_node.end_ns)) + (device_node.start_ns, device_node.end_ns) + ) - #case 3: Others, filter kernels named with nccl + # case 3: Others, filter kernels named with nccl else: for runtimenode in hostnode.runtime_node: for devicenode in runtimenode.device_node: if devicenode.type == TracerEventType.Kernel: if 'nccl' in devicenode.name.lower(): self.gpu_communication_range.append( - (devicenode.start_ns, - devicenode.end_ns)) + (devicenode.start_ns, devicenode.end_ns) + ) else: self.computation_range.append( - (devicenode.start_ns, - devicenode.end_ns)) + (devicenode.start_ns, devicenode.end_ns) + ) self.cpu_calls = len(set(self.cpu_communication_range)) self.gpu_calls = len(set(self.gpu_communication_range)) self.cpu_communication_range = merge_self_ranges( - self.cpu_communication_range, is_sorted=False) + self.cpu_communication_range, is_sorted=False + ) self.gpu_communication_range = merge_self_ranges( - self.gpu_communication_range, is_sorted=False) - self.communication_range = merge_ranges(self.cpu_communication_range, - self.gpu_communication_range, - is_sorted=True) - self.computation_range = merge_self_ranges(self.computation_range, - is_sorted=False) - self.overlap_range = intersection_ranges(self.communication_range, - self.computation_range, - is_sorted=True) + self.gpu_communication_range, is_sorted=False + ) + self.communication_range = merge_ranges( + self.cpu_communication_range, + self.gpu_communication_range, + is_sorted=True, + ) + self.computation_range = merge_self_ranges( + self.computation_range, is_sorted=False + ) + self.overlap_range = intersection_ranges( + self.communication_range, self.computation_range, is_sorted=True + ) class EventSummary: @@ -335,7 +370,6 @@ class EventSummary: """ class DeviceItem: - def __init__(self, name): self.name = name self.call = 0 @@ -359,7 +393,6 @@ def add_item(self, node): self.add_gpu_time(node.end_ns - node.start_ns) class OperatorItem: - def __init__(self, name): self.name = name self.call = 0 @@ -420,7 +453,8 @@ def add_item(self, node): if child.type != TracerEventType.Operator: if child.name not in self.operator_inners: self.operator_inners[ - child.name] = EventSummary.OperatorItem(child.name) + child.name + ] = EventSummary.OperatorItem(child.name) self.operator_inners[child.name].add_item(child) for runtimenode in node.runtime_node: @@ -431,7 +465,6 @@ def add_item(self, node): self.devices[name].add_item(devicenode) class GeneralItem: - def __init__(self, name): self.name = name self.call = 0 @@ -490,10 +523,12 @@ def add_item(self, node): def __init__(self): self.items = {} # for operator summary self.thread_items = collections.defaultdict( - dict) # for operator summary + dict + ) # for operator summary self.userdefined_items = {} # for userdefined summary self.userdefined_thread_items = collections.defaultdict( - dict) # for userdefined summary + dict + ) # for userdefined summary self.model_perspective_items = {} # for model summary self.memory_manipulation_items = {} # for memory manipulation summary self.kernel_items = {} # for kernel summary @@ -503,19 +538,31 @@ def parse(self, nodetrees): Analysis operator event in the nodetress. """ node_statistic_trees, thread2host_statistic_nodes = wrap_tree(nodetrees) - for threadid, host_statistic_nodes in thread2host_statistic_nodes.items( - ): + for ( + threadid, + host_statistic_nodes, + ) in thread2host_statistic_nodes.items(): for host_statistic_node in host_statistic_nodes[ - 1:]: #skip root node + 1: + ]: # skip root node if host_statistic_node.type == TracerEventType.Operator: self.add_operator_item(host_statistic_node) - if host_statistic_node.type == TracerEventType.UserDefined\ - or host_statistic_node.type == TracerEventType.PythonUserDefined: - if 'memcpy' in host_statistic_node.name.lower() or 'memorycopy' in host_statistic_node.name.lower()\ - or 'memset' in host_statistic_node.name.lower(): + if ( + host_statistic_node.type == TracerEventType.UserDefined + or host_statistic_node.type + == TracerEventType.PythonUserDefined + ): + if ( + 'memcpy' in host_statistic_node.name.lower() + or 'memorycopy' in host_statistic_node.name.lower() + or 'memset' in host_statistic_node.name.lower() + ): self.add_memory_manipulation_item(host_statistic_node) else: - if host_statistic_node.type == TracerEventType.PythonUserDefined: + if ( + host_statistic_node.type + == TracerEventType.PythonUserDefined + ): self.add_userdefined_item(host_statistic_node) self.add_kernel_item(host_statistic_nodes[0]) @@ -525,10 +572,15 @@ def parse(self, nodetrees): while deque: current_node = deque.popleft() for child in current_node.children_node: - if child.type == TracerEventType.Forward or child.type == TracerEventType.Dataloader\ - or child.type == TracerEventType.Backward or child.type == TracerEventType.Optimization: + if ( + child.type == TracerEventType.Forward + or child.type == TracerEventType.Dataloader + or child.type == TracerEventType.Backward + or child.type == TracerEventType.Optimization + ): self.add_model_perspective_item( - child) #find first model perspective node + child + ) # find first model perspective node else: if child.type == TracerEventType.ProfileStep: self.add_model_perspective_item(child) @@ -537,40 +589,46 @@ def parse(self, nodetrees): def add_operator_item(self, operator_node): if operator_node.name not in self.items: self.items[operator_node.name] = EventSummary.OperatorItem( - operator_node.name) + operator_node.name + ) self.items[operator_node.name].add_item(operator_node) if operator_node.name not in self.thread_items[operator_node.thread_id]: self.thread_items[operator_node.thread_id][ - operator_node.name] = EventSummary.OperatorItem( - operator_node.name) + operator_node.name + ] = EventSummary.OperatorItem(operator_node.name) self.thread_items[operator_node.thread_id][operator_node.name].add_item( - operator_node) + operator_node + ) def add_userdefined_item(self, userdefined_node): if userdefined_node.name not in self.userdefined_items: self.userdefined_items[ - userdefined_node.name] = EventSummary.GeneralItem( - userdefined_node.name) + userdefined_node.name + ] = EventSummary.GeneralItem(userdefined_node.name) self.userdefined_items[userdefined_node.name].add_item(userdefined_node) - if userdefined_node.name not in self.userdefined_thread_items[ - userdefined_node.thread_id]: + if ( + userdefined_node.name + not in self.userdefined_thread_items[userdefined_node.thread_id] + ): self.userdefined_thread_items[userdefined_node.thread_id][ - userdefined_node.name] = EventSummary.GeneralItem( - userdefined_node.name) + userdefined_node.name + ] = EventSummary.GeneralItem(userdefined_node.name) self.userdefined_thread_items[userdefined_node.thread_id][ - userdefined_node.name].add_item(userdefined_node) + userdefined_node.name + ].add_item(userdefined_node) def add_memory_manipulation_item(self, memory_manipulation_node): if memory_manipulation_node.name not in self.memory_manipulation_items: self.memory_manipulation_items[ - memory_manipulation_node.name] = EventSummary.GeneralItem( - memory_manipulation_node.name) + memory_manipulation_node.name + ] = EventSummary.GeneralItem(memory_manipulation_node.name) self.memory_manipulation_items[memory_manipulation_node.name].add_item( - memory_manipulation_node) + memory_manipulation_node + ) def add_model_perspective_item(self, model_perspective_node): if model_perspective_node.type == TracerEventType.Forward: @@ -605,7 +663,6 @@ class MemorySummary: """ class MemoryItem: - def __init__(self, event_name, place, memory_type='Allocated'): self.event_name = event_name self.place = place @@ -617,11 +674,17 @@ def __init__(self, event_name, place, memory_type='Allocated'): self.memory_type = memory_type def add_memory_record(self, size, allocation_type): - if allocation_type == TracerMemEventType.Allocate or allocation_type == TracerMemEventType.ReservedAllocate: + if ( + allocation_type == TracerMemEventType.Allocate + or allocation_type == TracerMemEventType.ReservedAllocate + ): self.allocation_count += 1 self.allocation_size += size - elif allocation_type == TracerMemEventType.Free or allocation_type == TracerMemEventType.ReservedFree: + elif ( + allocation_type == TracerMemEventType.Free + or allocation_type == TracerMemEventType.ReservedFree + ): self.free_count += 1 self.free_size -= size # size is sign(-) when free. @@ -631,35 +694,49 @@ def add_memory_record(self, size, allocation_type): def __init__(self): self.allocated_items = collections.defaultdict( - dict) # for memory summary, device type: event + dict + ) # for memory summary, device type: event self.reserved_items = collections.defaultdict( - dict) # for memory summary, device type: event + dict + ) # for memory summary, device type: event self.peak_allocation_values = collections.defaultdict(int) self.peak_reserved_values = collections.defaultdict(int) def _analyse_node_memory(self, event_name, node): for memnode in node.mem_node: # self mem node - if memnode.type == TracerMemEventType.Allocate or memnode.type == TracerMemEventType.Free: + if ( + memnode.type == TracerMemEventType.Allocate + or memnode.type == TracerMemEventType.Free + ): if event_name not in self.allocated_items[memnode.place]: - self.allocated_items[ - memnode.place][event_name] = MemorySummary.MemoryItem( - event_name, memnode.place, 'Allocated') - self.allocated_items[ - memnode.place][event_name].add_memory_record( - memnode.increase_bytes, memnode.type) - elif memnode.type == TracerMemEventType.ReservedAllocate or memnode.type == TracerMemEventType.ReservedFree: + self.allocated_items[memnode.place][ + event_name + ] = MemorySummary.MemoryItem( + event_name, memnode.place, 'Allocated' + ) + self.allocated_items[memnode.place][ + event_name + ].add_memory_record(memnode.increase_bytes, memnode.type) + elif ( + memnode.type == TracerMemEventType.ReservedAllocate + or memnode.type == TracerMemEventType.ReservedFree + ): if event_name not in self.reserved_items[memnode.place]: - self.reserved_items[ - memnode.place][event_name] = MemorySummary.MemoryItem( - event_name, memnode.place, 'Reserved') - self.reserved_items[ - memnode.place][event_name].add_memory_record( - memnode.increase_bytes, memnode.type) + self.reserved_items[memnode.place][ + event_name + ] = MemorySummary.MemoryItem( + event_name, memnode.place, 'Reserved' + ) + self.reserved_items[memnode.place][ + event_name + ].add_memory_record(memnode.increase_bytes, memnode.type) self.peak_allocation_values[memnode.place] = max( self.peak_allocation_values[memnode.place], - memnode.peak_allocated) + memnode.peak_allocated, + ) self.peak_reserved_values[memnode.place] = max( - self.peak_reserved_values[memnode.place], memnode.peak_reserved) + self.peak_reserved_values[memnode.place], memnode.peak_reserved + ) def parse(self, nodetrees): r""" @@ -667,7 +744,7 @@ def parse(self, nodetrees): """ thread2hostnodes = traverse_tree(nodetrees) for threadid, host_nodes in thread2hostnodes.items(): - for host_node in host_nodes[1:]: #skip root node + for host_node in host_nodes[1:]: # skip root node if host_node.type == TracerEventType.OperatorInner: continue if host_node.type == TracerEventType.Operator: @@ -694,16 +771,19 @@ def __init__(self, node_trees, extra_info): self.memory_summary.parse(node_trees) -def _build_table(statistic_data, - sorted_by=SortedKeys.CPUTotal, - op_detail=True, - thread_sep=False, - time_unit='ms', - row_limit=100, - max_src_column_width=75, - views=None): +def _build_table( + statistic_data, + sorted_by=SortedKeys.CPUTotal, + op_detail=True, + thread_sep=False, + time_unit='ms', + row_limit=100, + max_src_column_width=75, + views=None, +): from .profiler import SummaryView + """Prints a summary of events.""" # format table row SPACING_SIZE = 2 @@ -712,8 +792,9 @@ def _build_table(statistic_data, line_length_list = [-SPACING_SIZE] def add_column(padding, text_dir='<'): - row_format_list[0] += '{: ' + text_dir + str(padding) + '}' + ( - ' ' * SPACING_SIZE) + row_format_list[0] += ( + '{: ' + text_dir + str(padding) + '}' + (' ' * SPACING_SIZE) + ) header_sep_list[0] += '-' * padding + (' ' * SPACING_SIZE) line_length_list[0] += padding + SPACING_SIZE @@ -751,7 +832,8 @@ def format_ratio(ratio, indent=0): return '{}{:.2f}'.format(' ' * indent, ratio * 100) total_time = statistic_data.time_range_summary.get_cpu_range_sum( - TracerEventType.ProfileStep) + TracerEventType.ProfileStep + ) if views is None or SummaryView.DeviceView in views: @@ -776,19 +858,23 @@ def format_ratio(ratio, indent=0): row_values = [ 'CPU(Process)', format_ratio( - float(statistic_data.extra_info['Process Cpu Utilization'])) + float(statistic_data.extra_info['Process Cpu Utilization']) + ), ] append(row_format.format(*row_values)) row_values = [ 'CPU(System)', format_ratio( - float(statistic_data.extra_info['System Cpu Utilization'])) + float(statistic_data.extra_info['System Cpu Utilization']) + ), ] append(row_format.format(*row_values)) for gpu_name in statistic_data.time_range_summary.get_gpu_devices(): gpu_time = float( statistic_data.time_range_summary.get_gpu_range_sum( - gpu_name, TracerEventType.Kernel)) + gpu_name, TracerEventType.Kernel + ) + ) utilization = gpu_time / total_time row_values = ['GPU{}'.format(gpu_name), format_ratio(utilization)] append(row_format.format(*row_values)) @@ -797,7 +883,8 @@ def format_ratio(ratio, indent=0): append( "Note:\nCPU(Process) Utilization = Current process CPU time over all cpu cores / elapsed time, so max utilization can be reached 100% * number of cpu cores.\n" "CPU(System) Utilization = All processes CPU time over all cpu cores(busy time) / (busy time + idle time).\n" - "GPU Utilization = Current process GPU time / elapsed time.") + "GPU Utilization = Current process GPU time / elapsed time." + ) append('-' * line_length) append('') append('') @@ -833,54 +920,71 @@ def format_ratio(ratio, indent=0): cpu_call_times.update(statistic_data.time_range_summary.call_times) gpu_call_times.update(statistic_data.time_range_summary.call_times) - for event_type, value in statistic_data.time_range_summary.CPUTimeRangeSum.items( - ): + for ( + event_type, + value, + ) in statistic_data.time_range_summary.CPUTimeRangeSum.items(): if event_type != TracerEventType.Communication: cpu_type_time[event_type] = value if statistic_data.distributed_summary.cpu_communication_range: cpu_type_time[TracerEventType.Communication] = sum_ranges( - statistic_data.distributed_summary.cpu_communication_range) + statistic_data.distributed_summary.cpu_communication_range + ) cpu_call_times[ - TracerEventType. - Communication] = statistic_data.distributed_summary.cpu_calls + TracerEventType.Communication + ] = statistic_data.distributed_summary.cpu_calls for event_type in [ - TracerEventType.Dataloader, TracerEventType.Forward, - TracerEventType.Backward, TracerEventType.Optimization + TracerEventType.Dataloader, + TracerEventType.Forward, + TracerEventType.Backward, + TracerEventType.Optimization, ]: event_type_name = str(event_type).split('.')[1] - if event_type in cpu_call_times and event_type_name in statistic_data.event_summary.model_perspective_items: + if ( + event_type in cpu_call_times + and event_type_name + in statistic_data.event_summary.model_perspective_items + ): cpu_call_times[ - event_type] = statistic_data.event_summary.model_perspective_items[ - event_type_name].call + event_type + ] = statistic_data.event_summary.model_perspective_items[ + event_type_name + ].call cpu_type_time[ - event_type] = statistic_data.event_summary.model_perspective_items[ - event_type_name].cpu_time + event_type + ] = statistic_data.event_summary.model_perspective_items[ + event_type_name + ].cpu_time gpu_time_range = collections.defaultdict(list) - for device_id, device_time_ranges in statistic_data.time_range_summary.GPUTimeRange.items( - ): + for ( + device_id, + device_time_ranges, + ) in statistic_data.time_range_summary.GPUTimeRange.items(): for event_type, time_range in device_time_ranges.items(): gpu_time_range[event_type] = merge_ranges( - gpu_time_range[event_type], time_range, is_sorted=True) + gpu_time_range[event_type], time_range, is_sorted=True + ) for event_type, time_range in gpu_time_range.items(): gpu_type_time[event_type] = sum_ranges(time_range) if statistic_data.distributed_summary.gpu_communication_range: gpu_type_time[TracerEventType.Communication] = sum_ranges( - statistic_data.distributed_summary.gpu_communication_range) + statistic_data.distributed_summary.gpu_communication_range + ) gpu_call_times[ - TracerEventType. - Communication] = statistic_data.distributed_summary.gpu_calls + TracerEventType.Communication + ] = statistic_data.distributed_summary.gpu_calls - sorted_items = sorted(cpu_type_time.items(), - key=lambda x: x[1], - reverse=True) + sorted_items = sorted( + cpu_type_time.items(), key=lambda x: x[1], reverse=True + ) event_type, time = sorted_items[0] row_values = [ '{}'.format(str(event_type).split('.')[1]), cpu_call_times[event_type], format_time(time, unit=time_unit), - format_ratio(float(time) / total_time) + format_ratio(float(time) / total_time), ] append(row_format.format(*row_values)) for event_type, time in sorted_items[1:]: @@ -888,7 +992,7 @@ def format_ratio(ratio, indent=0): ' {}'.format(str(event_type).split('.')[1]), cpu_call_times[event_type], format_time(time, unit=time_unit), - format_ratio(float(time) / total_time) + format_ratio(float(time) / total_time), ] append(row_format.format(*row_values)) append(header_sep) @@ -900,7 +1004,7 @@ def format_ratio(ratio, indent=0): ' {}'.format(str(event_type).split('.')[1]), gpu_call_times[event_type], format_time(time, unit=time_unit), - format_ratio(float(time) / total_time) + format_ratio(float(time) / total_time), ] append(row_format.format(*row_values)) @@ -916,7 +1020,8 @@ def format_ratio(ratio, indent=0): "Thread 2:\n" " Operator: |____________| |___|\n" "After merged:\n" - " Result: |______________| |__________|\n") + " Result: |______________| |__________|\n" + ) append('-' * line_length) append('') append('') @@ -924,16 +1029,24 @@ def format_ratio(ratio, indent=0): if views is None or SummaryView.ModelView in views: ###### Print Model Summary Report ###### - model_perspective_items = statistic_data.event_summary.model_perspective_items + model_perspective_items = ( + statistic_data.event_summary.model_perspective_items + ) if len(model_perspective_items) > 1: all_row_values = [] accmulation_time = 0 gpu_accmulation_time = 0 - gpu_total_time = statistic_data.event_summary.model_perspective_items[ - 'ProfileStep'].gpu_time + gpu_total_time = ( + statistic_data.event_summary.model_perspective_items[ + 'ProfileStep' + ].gpu_time + ) for name in [ - 'ProfileStep', 'Dataloader', 'Forward', 'Backward', - 'Optimization' + 'ProfileStep', + 'Dataloader', + 'Forward', + 'Backward', + 'Optimization', ]: if name in model_perspective_items: item = model_perspective_items[name] @@ -941,22 +1054,28 @@ def format_ratio(ratio, indent=0): gpu_ratio = 0 else: gpu_ratio = float(item.gpu_time) / gpu_total_time - name = '{}'.format( - name) if 'ProfileStep' in name else ' {}'.format(name) + name = ( + '{}'.format(name) + if 'ProfileStep' in name + else ' {}'.format(name) + ) row_values = [ - '{}'.format(name), item.call, + '{}'.format(name), + item.call, '{} / {} / {} / {} / {}'.format( format_time(item.cpu_time, unit=time_unit), format_time(item.avg_cpu_time, unit=time_unit), format_time(item.max_cpu_time, unit=time_unit), format_time(item.min_cpu_time, unit=time_unit), - format_ratio(float(item.cpu_time) / total_time)), + format_ratio(float(item.cpu_time) / total_time), + ), '{} / {} / {} / {} / {}'.format( format_time(item.gpu_time, unit=time_unit), format_time(item.avg_gpu_time, unit=time_unit), format_time(item.max_gpu_time, unit=time_unit), format_time(item.min_gpu_time, unit=time_unit), - format_ratio(gpu_ratio)) + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) if 'ProfileStep' not in name: @@ -970,12 +1089,16 @@ def format_ratio(ratio, indent=0): else: gpu_ratio = float(other_gpu_time) / gpu_total_time row_values = [ - ' Others', '-', '{} / - / - / - / {}'.format( + ' Others', + '-', + '{} / - / - / - / {}'.format( format_time(other_time, unit=time_unit), - format_ratio(float(other_time) / total_time)), + format_ratio(float(other_time) / total_time), + ), '{} / - / - / - / {}'.format( format_time(other_gpu_time, unit=time_unit), - format_ratio(gpu_ratio)) + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) # Calculate the column width @@ -983,16 +1106,20 @@ def format_ratio(ratio, indent=0): cpu_data_description_width = 40 gpu_data_description_width = 40 for row_values in all_row_values: - if isinstance(row_values[1], - int) and len(str(row_values[1])) > calltime_width: + if ( + isinstance(row_values[1], int) + and len(str(row_values[1])) > calltime_width + ): calltime_width = len(str(row_values[1])) if len(row_values[2]) > cpu_data_description_width: cpu_data_description_width = len(row_values[2]) if len(row_values[3]) > gpu_data_description_width: gpu_data_description_width = len(row_values[3]) headers = [ - 'Name', 'Calls', 'CPU Total / Avg / Max / Min / Ratio(%)', - 'GPU Total / Avg / Max / Min / Ratio(%)' + 'Name', + 'Calls', + 'CPU Total / Avg / Max / Min / Ratio(%)', + 'GPU Total / Avg / Max / Min / Ratio(%)', ] row_format_list = [""] header_sep_list = [""] @@ -1052,35 +1179,38 @@ def format_ratio(ratio, indent=0): append(row_format.format(*headers)) append(header_sep) communication_time = sum_ranges( - statistic_data.distributed_summary.communication_range) + statistic_data.distributed_summary.communication_range + ) computation_time = sum_ranges( - statistic_data.distributed_summary.computation_range) + statistic_data.distributed_summary.computation_range + ) overlap_time = sum_ranges( - statistic_data.distributed_summary.overlap_range) + statistic_data.distributed_summary.overlap_range + ) row_values = [ 'ProfileStep', format_time(total_time, unit=time_unit), - format_ratio(float(total_time) / total_time) + format_ratio(float(total_time) / total_time), ] append(row_format.format(*row_values)) row_values = [ ' Communication', format_time(communication_time, unit=time_unit), - format_ratio(float(communication_time) / total_time) + format_ratio(float(communication_time) / total_time), ] append(row_format.format(*row_values)) row_values = [ ' Computation', format_time(computation_time, unit=time_unit), - format_ratio(float(computation_time) / total_time) + format_ratio(float(computation_time) / total_time), ] append(row_format.format(*row_values)) row_values = [ ' Overlap', format_time(overlap_time, unit=time_unit), - format_ratio(float(overlap_time) / total_time) + format_ratio(float(overlap_time) / total_time), ] append(row_format.format(*row_values)) append(header_sep) @@ -1095,7 +1225,8 @@ def format_ratio(ratio, indent=0): " Total: |_________________| |______________|\n" "Computation time(Kernel):\n" " GPU: |________________|\n" - "Overlap time: |___________|\n") + "Overlap time: |___________|\n" + ) append('-' * line_length) append('') append('') @@ -1115,37 +1246,47 @@ def format_ratio(ratio, indent=0): for thread_id, items in thread_items.items(): all_row_values.append("Thread: {}".format(thread_id)) if sorted_by == SortedKeys.CPUTotal: - sorted_items = sorted(items.items(), - key=lambda x: x[1].cpu_time, - reverse=True) + sorted_items = sorted( + items.items(), key=lambda x: x[1].cpu_time, reverse=True + ) elif sorted_by == SortedKeys.CPUAvg: - sorted_items = sorted(items.items(), - key=lambda x: x[1].avg_cpu_time, - reverse=True) + sorted_items = sorted( + items.items(), + key=lambda x: x[1].avg_cpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.CPUMax: - sorted_items = sorted(items.items(), - key=lambda x: x[1].max_cpu_time, - reverse=True) + sorted_items = sorted( + items.items(), + key=lambda x: x[1].max_cpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.CPUMin: - sorted_items = sorted(items.items(), - key=lambda x: x[1].min_cpu_time) + sorted_items = sorted( + items.items(), key=lambda x: x[1].min_cpu_time + ) elif sorted_by == SortedKeys.GPUTotal: - sorted_items = sorted(items.items(), - key=lambda x: x[1].general_gpu_time, - reverse=True) + sorted_items = sorted( + items.items(), + key=lambda x: x[1].general_gpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.GPUAvg: sorted_items = sorted( items.items(), key=lambda x: x[1].avg_general_gpu_time, - reverse=True) + reverse=True, + ) elif sorted_by == SortedKeys.GPUMax: sorted_items = sorted( items.items(), key=lambda x: x[1].max_general_gpu_time, - reverse=True) + reverse=True, + ) elif sorted_by == SortedKeys.GPUMin: sorted_items = sorted( - items.items(), key=lambda x: x[1].min_general_gpu_time) + items.items(), key=lambda x: x[1].min_general_gpu_time + ) total_op_cpu_time = 0 total_op_gpu_time = 0 @@ -1161,124 +1302,177 @@ def format_ratio(ratio, indent=0): if total_op_gpu_time == 0: gpu_ratio = 0 else: - gpu_ratio = float( - item.general_gpu_time) / total_op_gpu_time + gpu_ratio = ( + float(item.general_gpu_time) / total_op_gpu_time + ) row_values = [ - name, item.call, '{} / {} / {} / {} / {}'.format( + name, + item.call, + '{} / {} / {} / {} / {}'.format( format_time(item.cpu_time, unit=time_unit), format_time(item.avg_cpu_time, unit=time_unit), format_time(item.max_cpu_time, unit=time_unit), format_time(item.min_cpu_time, unit=time_unit), - format_ratio(cpu_ratio)), + format_ratio(cpu_ratio), + ), '{} / {} / {} / {} / {}'.format( format_time(item.general_gpu_time, unit=time_unit), - format_time(item.avg_general_gpu_time, - unit=time_unit), - format_time(item.max_general_gpu_time, - unit=time_unit), - format_time(item.min_general_gpu_time, - unit=time_unit), - format_ratio(gpu_ratio)) + format_time( + item.avg_general_gpu_time, unit=time_unit + ), + format_time( + item.max_general_gpu_time, unit=time_unit + ), + format_time( + item.min_general_gpu_time, unit=time_unit + ), + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) if op_detail: - for innerop_name, innerop_node in item.operator_inners.items( - ): + for ( + innerop_name, + innerop_node, + ) in item.operator_inners.items(): if item.cpu_time == 0: cpu_ratio = 0 else: - cpu_ratio = float( - innerop_node.cpu_time) / item.cpu_time + cpu_ratio = ( + float(innerop_node.cpu_time) / item.cpu_time + ) if item.general_gpu_time == 0: gpu_ratio = 0 else: - gpu_ratio = float(innerop_node.general_gpu_time - ) / item.general_gpu_time + gpu_ratio = ( + float(innerop_node.general_gpu_time) + / item.general_gpu_time + ) if len(innerop_name) + 2 > name_column_width: - innerop_name = innerop_name[:name_column_width - - 5] + innerop_name = innerop_name[ + : name_column_width - 5 + ] innerop_name += "..." row_values = [ - ' {}'.format(innerop_name), innerop_node.call, + ' {}'.format(innerop_name), + innerop_node.call, '{} / {} / {} / {} / {}'.format( - format_time(innerop_node.cpu_time, - unit=time_unit), - format_time(innerop_node.avg_cpu_time, - unit=time_unit), - format_time(innerop_node.max_cpu_time, - unit=time_unit), - format_time(innerop_node.min_cpu_time, - unit=time_unit), - format_ratio(cpu_ratio)), + format_time( + innerop_node.cpu_time, unit=time_unit + ), + format_time( + innerop_node.avg_cpu_time, + unit=time_unit, + ), + format_time( + innerop_node.max_cpu_time, + unit=time_unit, + ), + format_time( + innerop_node.min_cpu_time, + unit=time_unit, + ), + format_ratio(cpu_ratio), + ), '{} / {} / {} / {} / {}'.format( - format_time(innerop_node.general_gpu_time, - unit=time_unit), + format_time( + innerop_node.general_gpu_time, + unit=time_unit, + ), format_time( innerop_node.avg_general_gpu_time, - unit=time_unit), + unit=time_unit, + ), format_time( innerop_node.max_general_gpu_time, - unit=time_unit), + unit=time_unit, + ), format_time( innerop_node.min_general_gpu_time, - unit=time_unit), - format_ratio(gpu_ratio)) + unit=time_unit, + ), + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) - for device_node_name, device_node in innerop_node.devices.items( - ): + for ( + device_node_name, + device_node, + ) in innerop_node.devices.items(): if innerop_node.general_gpu_time == 0: gpu_ratio = 0 else: - gpu_ratio = float( - device_node.gpu_time - ) / innerop_node.general_gpu_time - if len(device_node_name - ) + 4 > name_column_width: - device_node_name = device_node_name[: - name_column_width - - 7] + gpu_ratio = ( + float(device_node.gpu_time) + / innerop_node.general_gpu_time + ) + if ( + len(device_node_name) + 4 + > name_column_width + ): + device_node_name = device_node_name[ + : name_column_width - 7 + ] device_node_name += "..." row_values = [ ' {}'.format(device_node_name), - device_node.call, '- / - / - / - / -', + device_node.call, + '- / - / - / - / -', '{} / {} / {} / {} / {}'.format( - format_time(device_node.gpu_time, - unit=time_unit), - format_time(device_node.avg_gpu_time, - unit=time_unit), - format_time(device_node.max_gpu_time, - unit=time_unit), - format_time(device_node.min_gpu_time, - unit=time_unit), - format_ratio(gpu_ratio)) + format_time( + device_node.gpu_time, unit=time_unit + ), + format_time( + device_node.avg_gpu_time, + unit=time_unit, + ), + format_time( + device_node.max_gpu_time, + unit=time_unit, + ), + format_time( + device_node.min_gpu_time, + unit=time_unit, + ), + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) - for device_node_name, device_node in item.devices.items( - ): + for ( + device_node_name, + device_node, + ) in item.devices.items(): if item.general_gpu_time == 0: gpu_ratio = 0 else: - gpu_ratio = float(device_node.gpu_time - ) / item.general_gpu_time + gpu_ratio = ( + float(device_node.gpu_time) + / item.general_gpu_time + ) if len(device_node_name) + 2 > name_column_width: - device_node_name = device_node_name[: - name_column_width - - 5] + device_node_name = device_node_name[ + : name_column_width - 5 + ] device_node_name += "..." row_values = [ ' {}'.format(device_node_name), - device_node.call, '- / - / - / - / -', + device_node.call, + '- / - / - / - / -', '{} / {} / {} / {} / {}'.format( - format_time(device_node.gpu_time, - unit=time_unit), - format_time(device_node.avg_gpu_time, - unit=time_unit), - format_time(device_node.max_gpu_time, - unit=time_unit), - format_time(device_node.min_gpu_time, - unit=time_unit), - format_ratio(gpu_ratio)) + format_time( + device_node.gpu_time, unit=time_unit + ), + format_time( + device_node.avg_gpu_time, unit=time_unit + ), + format_time( + device_node.max_gpu_time, unit=time_unit + ), + format_time( + device_node.min_gpu_time, unit=time_unit + ), + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) # Calculate the column width @@ -1288,16 +1482,20 @@ def format_ratio(ratio, indent=0): for row_values in all_row_values: if isinstance(row_values, str): continue - if isinstance(row_values[1], - int) and len(str(row_values[1])) > calltime_width: + if ( + isinstance(row_values[1], int) + and len(str(row_values[1])) > calltime_width + ): calltime_width = len(str(row_values[1])) if len(row_values[2]) > cpu_data_description_width: cpu_data_description_width = len(row_values[2]) if len(row_values[3]) > gpu_data_description_width: gpu_data_description_width = len(row_values[3]) headers = [ - 'Name', 'Calls', 'CPU Total / Avg / Max / Min / Ratio(%)', - 'GPU Total / Avg / Max / Min / Ratio(%)' + 'Name', + 'Calls', + 'CPU Total / Avg / Max / Min / Ratio(%)', + 'GPU Total / Avg / Max / Min / Ratio(%)', ] row_format_list = [""] header_sep_list = [""] @@ -1333,20 +1531,27 @@ def format_ratio(ratio, indent=0): all_row_values = [] kernel_items = statistic_data.event_summary.kernel_items if sorted_by == SortedKeys.GPUAvg: - sorted_items = sorted(kernel_items.items(), - key=lambda x: x[1].avg_gpu_time, - reverse=True) + sorted_items = sorted( + kernel_items.items(), + key=lambda x: x[1].avg_gpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.GPUMax: - sorted_items = sorted(kernel_items.items(), - key=lambda x: x[1].max_gpu_time, - reverse=True) + sorted_items = sorted( + kernel_items.items(), + key=lambda x: x[1].max_gpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.GPUMin: - sorted_items = sorted(kernel_items.items(), - key=lambda x: x[1].min_gpu_time) + sorted_items = sorted( + kernel_items.items(), key=lambda x: x[1].min_gpu_time + ) else: - sorted_items = sorted(kernel_items.items(), - key=lambda x: x[1].gpu_time, - reverse=True) + sorted_items = sorted( + kernel_items.items(), + key=lambda x: x[1].gpu_time, + reverse=True, + ) total_kernel_gpu_time = 0 for name, item in sorted_items: @@ -1364,20 +1569,25 @@ def format_ratio(ratio, indent=0): format_time(item.avg_gpu_time, unit=time_unit), format_time(item.max_gpu_time, unit=time_unit), format_time(item.min_gpu_time, unit=time_unit), - format_ratio(gpu_ratio)), + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) headers = [ - 'Name', 'Calls', 'GPU Total / Avg / Max / Min / Ratio(%)' + 'Name', + 'Calls', + 'GPU Total / Avg / Max / Min / Ratio(%)', ] # Calculate the column width name_column_width = 90 calltime_width = 6 gpu_data_description_width = 40 for row_values in all_row_values: - if isinstance(row_values[1], - int) and len(str(row_values[1])) > calltime_width: + if ( + isinstance(row_values[1], int) + and len(str(row_values[1])) > calltime_width + ): calltime_width = len(str(row_values[1])) if len(row_values[2]) > gpu_data_description_width: gpu_data_description_width = len(row_values[2]) @@ -1407,7 +1617,7 @@ def format_ratio(ratio, indent=0): else: name = row_values[0] if len(name) > name_column_width: - row_values[0] = name[:name_column_width - 3] + '...' + row_values[0] = name[: name_column_width - 3] + '...' else: row_values[0] = name append(row_format.format(*row_values)) @@ -1420,9 +1630,14 @@ def format_ratio(ratio, indent=0): ###### Print Memory Manipulation Summary Report ###### if statistic_data.event_summary.memory_manipulation_items: all_row_values = [] - memory_manipulation_items = statistic_data.event_summary.memory_manipulation_items - gpu_total_time = statistic_data.event_summary.model_perspective_items[ - 'ProfileStep'].general_gpu_time + memory_manipulation_items = ( + statistic_data.event_summary.memory_manipulation_items + ) + gpu_total_time = ( + statistic_data.event_summary.model_perspective_items[ + 'ProfileStep' + ].general_gpu_time + ) for name, item in memory_manipulation_items.items(): if gpu_total_time == 0: gpu_ratio = 0 @@ -1436,19 +1651,23 @@ def format_ratio(ratio, indent=0): format_time(item.avg_cpu_time, unit=time_unit), format_time(item.max_cpu_time, unit=time_unit), format_time(item.min_cpu_time, unit=time_unit), - format_ratio(float(item.cpu_time) / total_time)), + format_ratio(float(item.cpu_time) / total_time), + ), '{} / {} / {} / {} / {}'.format( format_time(item.general_gpu_time, unit=time_unit), format_time(item.avg_general_gpu_time, unit=time_unit), format_time(item.max_general_gpu_time, unit=time_unit), format_time(item.min_general_gpu_time, unit=time_unit), - format_ratio(gpu_ratio)), + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) headers = [ - 'Name', 'Calls', 'CPU Total / Avg / Max / Min / Ratio(%)', - 'GPU Total / Avg / Max / Min / Ratio(%)' + 'Name', + 'Calls', + 'CPU Total / Avg / Max / Min / Ratio(%)', + 'GPU Total / Avg / Max / Min / Ratio(%)', ] # Calculate the column width name_column_width = 0 @@ -1458,8 +1677,10 @@ def format_ratio(ratio, indent=0): for row_values in all_row_values: if len(row_values[0]) > name_column_width: name_column_width = len(row_values[0]) - if isinstance(row_values[1], - int) and len(str(row_values[1])) > calltime_width: + if ( + isinstance(row_values[1], int) + and len(str(row_values[1])) > calltime_width + ): calltime_width = len(str(row_values[1])) if len(row_values[2]) > cpu_data_description_width: cpu_data_description_width = len(row_values[2]) @@ -1495,56 +1716,71 @@ def format_ratio(ratio, indent=0): ###### Print UserDefined Summary Report ###### if statistic_data.event_summary.userdefined_items: all_row_values = [] - gpu_total_time = statistic_data.event_summary.model_perspective_items[ - 'ProfileStep'].general_gpu_time + gpu_total_time = ( + statistic_data.event_summary.model_perspective_items[ + 'ProfileStep' + ].general_gpu_time + ) if thread_sep == True: - userdefined_thread_items = statistic_data.event_summary.userdefined_thread_items + userdefined_thread_items = ( + statistic_data.event_summary.userdefined_thread_items + ) else: userdefined_thread_items = { - 'All threads merged': - statistic_data.event_summary.userdefined_items + 'All threads merged': statistic_data.event_summary.userdefined_items } for thread_id, items in userdefined_thread_items.items(): all_row_values.append("Thread: {}".format(thread_id)) if sorted_by == SortedKeys.CPUTotal: - sorted_items = sorted(items.items(), - key=lambda x: x[1].cpu_time, - reverse=True) + sorted_items = sorted( + items.items(), key=lambda x: x[1].cpu_time, reverse=True + ) elif sorted_by == SortedKeys.CPUAvg: - sorted_items = sorted(items.items(), - key=lambda x: x[1].avg_cpu_time, - reverse=True) + sorted_items = sorted( + items.items(), + key=lambda x: x[1].avg_cpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.CPUMax: - sorted_items = sorted(items.items(), - key=lambda x: x[1].max_cpu_time, - reverse=True) + sorted_items = sorted( + items.items(), + key=lambda x: x[1].max_cpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.CPUMin: - sorted_items = sorted(items.items(), - key=lambda x: x[1].min_cpu_time) + sorted_items = sorted( + items.items(), key=lambda x: x[1].min_cpu_time + ) elif sorted_by == SortedKeys.GPUTotal: - sorted_items = sorted(items.items(), - key=lambda x: x[1].general_gpu_time, - reverse=True) + sorted_items = sorted( + items.items(), + key=lambda x: x[1].general_gpu_time, + reverse=True, + ) elif sorted_by == SortedKeys.GPUAvg: sorted_items = sorted( items.items(), key=lambda x: x[1].avg_general_gpu_time, - reverse=True) + reverse=True, + ) elif sorted_by == SortedKeys.GPUMax: sorted_items = sorted( items.items(), key=lambda x: x[1].max_general_gpu_time, - reverse=True) + reverse=True, + ) elif sorted_by == SortedKeys.GPUMin: sorted_items = sorted( - items.items(), key=lambda x: x[1].min_general_gpu_time) + items.items(), key=lambda x: x[1].min_general_gpu_time + ) for name, item in sorted_items: if gpu_total_time == 0: gpu_ratio = 0 else: - gpu_ratio = float( - item.general_gpu_time) / gpu_total_time + gpu_ratio = ( + float(item.general_gpu_time) / gpu_total_time + ) row_values = [ name, item.call, @@ -1553,16 +1789,21 @@ def format_ratio(ratio, indent=0): format_time(item.avg_cpu_time, unit=time_unit), format_time(item.max_cpu_time, unit=time_unit), format_time(item.min_cpu_time, unit=time_unit), - format_ratio(float(item.cpu_time) / total_time)), + format_ratio(float(item.cpu_time) / total_time), + ), '{} / {} / {} / {} / {}'.format( format_time(item.general_gpu_time, unit=time_unit), - format_time(item.avg_general_gpu_time, - unit=time_unit), - format_time(item.max_general_gpu_time, - unit=time_unit), - format_time(item.min_general_gpu_time, - unit=time_unit), - format_ratio(gpu_ratio)), + format_time( + item.avg_general_gpu_time, unit=time_unit + ), + format_time( + item.max_general_gpu_time, unit=time_unit + ), + format_time( + item.min_general_gpu_time, unit=time_unit + ), + format_ratio(gpu_ratio), + ), ] all_row_values.append(row_values) @@ -1576,8 +1817,10 @@ def format_ratio(ratio, indent=0): continue if len(row_values[0]) > name_column_width: name_column_width = len(row_values[0]) - if isinstance(row_values[1], - int) and len(str(row_values[1])) > calltime_width: + if ( + isinstance(row_values[1], int) + and len(str(row_values[1])) > calltime_width + ): calltime_width = len(str(row_values[1])) if len(row_values[2]) > cpu_data_description_width: cpu_data_description_width = len(row_values[2]) @@ -1585,8 +1828,10 @@ def format_ratio(ratio, indent=0): gpu_data_description_width = len(row_values[3]) headers = [ - 'Name', 'Calls', 'CPU Total / Avg / Max / Min / Ratio(%)', - 'GPU Total / Avg / Max / Min / Ratio(%)' + 'Name', + 'Calls', + 'CPU Total / Avg / Max / Min / Ratio(%)', + 'GPU Total / Avg / Max / Min / Ratio(%)', ] row_format_list = [""] header_sep_list = [""] @@ -1618,39 +1863,61 @@ def format_ratio(ratio, indent=0): if views is None or SummaryView.MemoryView in views: ###### Print Memory Summary Report ###### - if statistic_data.memory_summary.allocated_items or statistic_data.memory_summary.reserved_items: - for device_type, memory_events in statistic_data.memory_summary.allocated_items.items( - ): + if ( + statistic_data.memory_summary.allocated_items + or statistic_data.memory_summary.reserved_items + ): + for ( + device_type, + memory_events, + ) in statistic_data.memory_summary.allocated_items.items(): all_row_values = [] - sorted_items = sorted(memory_events.items(), - key=lambda x: x[1].increase_size, - reverse=True) + sorted_items = sorted( + memory_events.items(), + key=lambda x: x[1].increase_size, + reverse=True, + ) for event_name, item in sorted_items: row_values = [ - event_name, item.memory_type, item.allocation_count, - item.free_count, item.allocation_size, item.free_size, - item.increase_size + event_name, + item.memory_type, + item.allocation_count, + item.free_count, + item.allocation_size, + item.free_size, + item.increase_size, ] all_row_values.append(row_values) sorted_reserved_items = sorted( - statistic_data.memory_summary.reserved_items[device_type]. - items(), + statistic_data.memory_summary.reserved_items[ + device_type + ].items(), key=lambda x: x[1].increase_size, - reverse=True) + reverse=True, + ) for event_name, item in sorted_reserved_items: row_values = [ - event_name, item.memory_type, item.allocation_count, - item.free_count, item.allocation_size, item.free_size, - item.increase_size + event_name, + item.memory_type, + item.allocation_count, + item.free_count, + item.allocation_size, + item.free_size, + item.increase_size, ] all_row_values.append(row_values) # Calculate the column width headers = [ - 'Name', 'Type', 'Allocation Count', 'Free Count', - 'Allocation Size', 'Free Size', 'Increased Size' + 'Name', + 'Type', + 'Allocation Count', + 'Free Count', + 'Allocation Size', + 'Free Size', + 'Increased Size', ] row_format_list = [""] header_sep_list = [""] @@ -1671,14 +1938,24 @@ def format_ratio(ratio, indent=0): # construct table string append( - add_title(line_length, - "Memory Summary - {}".format(device_type))) - append('Peak Allocated Memory: {}'.format( - statistic_data.memory_summary. - peak_allocation_values[device_type])) - append('Peak Reserved Memory: {}'.format( - statistic_data.memory_summary. - peak_reserved_values[device_type])) + add_title( + line_length, "Memory Summary - {}".format(device_type) + ) + ) + append( + 'Peak Allocated Memory: {}'.format( + statistic_data.memory_summary.peak_allocation_values[ + device_type + ] + ) + ) + append( + 'Peak Reserved Memory: {}'.format( + statistic_data.memory_summary.peak_reserved_values[ + device_type + ] + ) + ) append(header_sep) append(row_format.format(*headers)) append(header_sep) diff --git a/python/paddle/profiler/statistic_helper.py b/python/paddle/profiler/statistic_helper.py index b6925a2b77482..13ce42d7d73d6 100644 --- a/python/paddle/profiler/statistic_helper.py +++ b/python/paddle/profiler/statistic_helper.py @@ -16,7 +16,7 @@ def sum_ranges(ranges): result = 0 for time_range in ranges: - result += (time_range[1] - time_range[0]) + result += time_range[1] - time_range[0] return result @@ -30,11 +30,14 @@ def merge_self_ranges(src_ranges, is_sorted=False): for cur_indx in range(1, len(src_ranges)): if src_ranges[cur_indx][1] > merged_ranges[-1][1]: if src_ranges[cur_indx][0] <= merged_ranges[-1][1]: - merged_ranges[-1] = (merged_ranges[-1][0], - src_ranges[cur_indx][1]) + merged_ranges[-1] = ( + merged_ranges[-1][0], + src_ranges[cur_indx][1], + ) else: merged_ranges.append( - (src_ranges[cur_indx][0], src_ranges[cur_indx][1])) + (src_ranges[cur_indx][0], src_ranges[cur_indx][1]) + ) return merged_ranges @@ -129,7 +132,7 @@ def intersection_ranges(range_list1, range_list2, is_sorted=False): range2 = range_list2[indx2] elif range2[0] <= range1[0] and range2[1] < range1[1]: - assert (range2[1] > range1[0]) + assert range2[1] > range1[0] result_range.append((range1[0], range2[1])) range1 = (range2[1], range1[1]) indx2 += 1 @@ -138,7 +141,7 @@ def intersection_ranges(range_list1, range_list2, is_sorted=False): range2 = range_list2[indx2] elif range2[0] <= range1[0]: - assert (range2[1] >= range1[1]) + assert range2[1] >= range1[1] result_range.append(range1) range2 = (range1[1], range2[1]) indx1 += 1 @@ -147,7 +150,7 @@ def intersection_ranges(range_list1, range_list2, is_sorted=False): range1 = range_list1[indx1] elif range2[1] < range1[1]: - assert (range2[0] > range1[0]) + assert range2[0] > range1[0] result_range.append(range2) range1 = (range2[1], range1[1]) indx2 += 1 @@ -156,7 +159,7 @@ def intersection_ranges(range_list1, range_list2, is_sorted=False): range2 = range_list2[indx2] elif range2[0] < range1[1]: - assert (range2[1] >= range1[1]) + assert range2[1] >= range1[1] result_range.append((range2[0], range1[1])) range2 = (range1[1], range2[1]) indx1 += 1 @@ -165,7 +168,7 @@ def intersection_ranges(range_list1, range_list2, is_sorted=False): range1 = range_list1[indx1] else: - assert (range2[0] >= range1[1]) + assert range2[0] >= range1[1] indx1 += 1 if indx1 == len1: break @@ -207,17 +210,17 @@ def subtract_ranges(range_list1, range_list2, is_sorted=False): if indx2 != len2: range2 = range_list2[indx2] elif range2[0] <= range1[0]: - assert (range2[1] >= range1[1]) + assert range2[1] >= range1[1] range2 = (range1[1], range2[1]) indx1 += 1 if indx1 != len1: range1 = range_list1[indx1] elif range2[0] < range1[1]: - assert (range2[0] > range1[0]) + assert range2[0] > range1[0] result_range.append((range1[0], range2[0])) range1 = (range2[0], range1[1]) else: - assert (range2[0] >= range1[1]) + assert range2[0] >= range1[1] result_range.append(range1) indx1 += 1 if indx1 != len1: diff --git a/python/paddle/profiler/timer.py b/python/paddle/profiler/timer.py index 1f0fb6acde63c..311fc373c5f7d 100644 --- a/python/paddle/profiler/timer.py +++ b/python/paddle/profiler/timer.py @@ -126,20 +126,28 @@ def get_summary(self): else: speed_avg = float(self.total_iters) / self.batch_records['total'] - reader_summary = dict(max=self.reader_records['max'], - min=self.reader_records['min'], - avg=reader_avg) - batch_summary = dict(max=self.batch_records['max'], - min=self.batch_records['min'], - avg=batch_avg) - ips_summary = dict(max=self.speed_records['max'], - min=self.speed_records['min'], - avg=speed_avg) + reader_summary = dict( + max=self.reader_records['max'], + min=self.reader_records['min'], + avg=reader_avg, + ) + batch_summary = dict( + max=self.batch_records['max'], + min=self.batch_records['min'], + avg=batch_avg, + ) + ips_summary = dict( + max=self.speed_records['max'], + min=self.speed_records['min'], + avg=speed_avg, + ) reader_ratio = (reader_avg / batch_avg) * 100 - summary = dict(reader_summary=reader_summary, - batch_summary=batch_summary, - ips_summary=ips_summary, - reader_ratio=reader_ratio) + summary = dict( + reader_summary=reader_summary, + batch_summary=batch_summary, + ips_summary=ips_summary, + reader_ratio=reader_ratio, + ) return summary @@ -205,8 +213,11 @@ def after_reader(self, benchmark): """ reader_cost = timeit.default_timer() - self.start_reader - if (benchmark.current_event is None) or ( - not benchmark.current_event.need_record) or (reader_cost == 0): + if ( + (benchmark.current_event is None) + or (not benchmark.current_event.need_record) + or (reader_cost == 0) + ): return benchmark.current_event.record_reader(reader_cost) @@ -220,8 +231,9 @@ def after_step(self, benchmark): """ - if (benchmark.current_event is - None) or (not benchmark.current_event.need_record): + if (benchmark.current_event is None) or ( + not benchmark.current_event.need_record + ): return batch_cost = timeit.default_timer() - self.start_time benchmark.current_event.record_batch(batch_cost, benchmark.num_samples) @@ -250,10 +262,20 @@ def _print_summary(self, benchmark): print('Perf Summary'.center(100, '=')) if summary['reader_ratio'] != 0: print('Reader Ratio: ' + '%.3f' % (summary['reader_ratio']) + '%') - print('Time Unit: s, IPS Unit: %s' % - (benchmark.current_event.speed_unit)) - print('|', ''.center(15), '|', 'avg'.center(15), '|', 'max'.center(15), - '|', 'min'.center(15), '|') + print( + 'Time Unit: s, IPS Unit: %s' % (benchmark.current_event.speed_unit) + ) + print( + '|', + ''.center(15), + '|', + 'avg'.center(15), + '|', + 'max'.center(15), + '|', + 'min'.center(15), + '|', + ) # if DataLoader is not called, reader_summary is unnecessary. if summary['reader_summary']['avg'] != 0: self._print_stats('reader_cost', summary['reader_summary']) @@ -264,8 +286,17 @@ def _print_stats(self, item, message_dict): avg_str = '%.5f' % (message_dict['avg']) max_str = '%.5f' % (message_dict['max']) min_str = '%.5f' % (message_dict['min']) - print('|', item.center(15), '|', avg_str.center(15), '|', - max_str.center(15), '|', min_str.center(15), '|') + print( + '|', + item.center(15), + '|', + avg_str.center(15), + '|', + max_str.center(15), + '|', + min_str.center(15), + '|', + ) class TimeAverager(object): @@ -357,8 +388,10 @@ def step_info(self, unit): message += ' %s: %.5f s' % ('batch_cost', batch_average) speed_average = self.current_event.speed_average() if speed_average: - message += ' ips: %.3f %s' % (speed_average, - self.current_event.speed_unit) + message += ' ips: %.3f %s' % ( + speed_average, + self.current_event.speed_unit, + ) self.current_event.reset() return message @@ -389,8 +422,10 @@ def check_if_need_record(self, reader): # set reader for the current event at the first iter if self.current_event.reader is None: self.current_event.reader = reader - elif self.current_event.reader.__dict__[ - '_dataset'] != reader.__dict__['_dataset']: + elif ( + self.current_event.reader.__dict__['_dataset'] + != reader.__dict__['_dataset'] + ): # enter a new task but not calling beign() to record it. # we pause the timer until the end of new task, so that # the cost of new task is not added to the current event. @@ -398,8 +433,10 @@ def check_if_need_record(self, reader): self.current_event.need_record = False else: # when the new task exits, continue timing for the current event. - if self.current_event.reader.__dict__[ - '_dataset'] == reader.__dict__['_dataset']: + if ( + self.current_event.reader.__dict__['_dataset'] + == reader.__dict__['_dataset'] + ): self.current_event.need_record = True self.hooks['timer_hook'].start_time = timeit.default_timer() diff --git a/python/paddle/profiler/utils.py b/python/paddle/profiler/utils.py index fe05aaeb81f9e..efe3975f14452 100644 --- a/python/paddle/profiler/utils.py +++ b/python/paddle/profiler/utils.py @@ -18,16 +18,19 @@ from contextlib import ContextDecorator from paddle.fluid import core -from paddle.fluid.core import (_RecordEvent, TracerEventType) +from paddle.fluid.core import _RecordEvent, TracerEventType _is_profiler_used = False _has_optimizer_wrapped = False _AllowedEventTypeList = [ - TracerEventType.Dataloader, TracerEventType.ProfileStep, - TracerEventType.Forward, TracerEventType.Backward, - TracerEventType.Optimization, TracerEventType.PythonOp, - TracerEventType.PythonUserDefined + TracerEventType.Dataloader, + TracerEventType.ProfileStep, + TracerEventType.Forward, + TracerEventType.Backward, + TracerEventType.Optimization, + TracerEventType.PythonOp, + TracerEventType.PythonUserDefined, ] @@ -65,9 +68,10 @@ class RecordEvent(ContextDecorator): """ def __init__( - self, - name: str, - event_type: TracerEventType = TracerEventType.PythonUserDefined): + self, + name: str, + event_type: TracerEventType = TracerEventType.PythonUserDefined, + ): self.name = name self.event_type = event_type self.event = None @@ -100,8 +104,12 @@ def begin(self): if not _is_profiler_used: return if self.event_type not in _AllowedEventTypeList: - warn("Only TracerEvent Type in [{}, {}, {}, {}, {}, {},{}]\ - can be recorded.".format(*_AllowedEventTypeList)) + warn( + "Only TracerEvent Type in [{}, {}, {}, {}, {}, {},{}]\ + can be recorded.".format( + *_AllowedEventTypeList + ) + ) self.event = None else: self.event = _RecordEvent(self.name, self.event_type) @@ -160,14 +168,13 @@ def in_profiler_mode(): def wrap_optimizers(): - def optimizer_warpper(func): - @functools.wraps(func) def warpper(*args, **kwargs): if in_profiler_mode(): - with RecordEvent('Optimization Step', - event_type=TracerEventType.Optimization): + with RecordEvent( + 'Optimization Step', event_type=TracerEventType.Optimization + ): return func(*args, **kwargs) else: return func(*args, **kwargs) @@ -178,6 +185,7 @@ def warpper(*args, **kwargs): if _has_optimizer_wrapped == True: return import paddle.optimizer as optimizer + for classname in optimizer.__all__: if classname != 'Optimizer': classobject = getattr(optimizer, classname) diff --git a/python/paddle/quantization/__init__.py b/python/paddle/quantization/__init__.py index 77da408d8e001..66d2354d48052 100644 --- a/python/paddle/quantization/__init__.py +++ b/python/paddle/quantization/__init__.py @@ -12,14 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.contrib.slim.quantization.imperative.ptq_config import PTQConfig, default_ptq_config -from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import BaseQuantizer -from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import AbsmaxQuantizer -from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import PerChannelAbsmaxQuantizer -from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import KLQuantizer -from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import HistQuantizer -from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import SUPPORT_ACT_QUANTIZERS -from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import SUPPORT_WT_QUANTIZERS -from ...fluid.contrib.slim.quantization.imperative.ptq_registry import PTQRegistry +from ...fluid.contrib.slim.quantization.imperative.ptq_config import ( + PTQConfig, + default_ptq_config, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import ( + BaseQuantizer, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import ( + AbsmaxQuantizer, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import ( + PerChannelAbsmaxQuantizer, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import ( + KLQuantizer, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import ( + HistQuantizer, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import ( + SUPPORT_ACT_QUANTIZERS, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_quantizer import ( + SUPPORT_WT_QUANTIZERS, +) +from ...fluid.contrib.slim.quantization.imperative.ptq_registry import ( + PTQRegistry, +) from ...fluid.contrib.slim.quantization.imperative.ptq import ImperativePTQ -from ...fluid.contrib.slim.quantization.imperative.qat import ImperativeQuantAware +from ...fluid.contrib.slim.quantization.imperative.qat import ( + ImperativeQuantAware, +) diff --git a/python/paddle/reader/decorator.py b/python/paddle/reader/decorator.py index f7ba974fc0fd9..b2aa88cc810b4 100644 --- a/python/paddle/reader/decorator.py +++ b/python/paddle/reader/decorator.py @@ -277,7 +277,7 @@ def make_tuple(x): if isinstance(x, tuple): return x else: - return (x, ) + return (x,) def reader(): rs = [] @@ -292,7 +292,8 @@ def reader(): if o is None: # None will be not be present if compose is aligned raise ComposeNotAligned( - "outputs of readers are not aligned.") + "outputs of readers are not aligned." + ) yield sum(list(map(make_tuple, outputs)), ()) return reader @@ -330,7 +331,7 @@ def reader(): print(i) """ - class EndSignal(): + class EndSignal: pass end = EndSignal() @@ -343,10 +344,13 @@ def read_worker(r, q): def data_reader(): r = reader() q = Queue(maxsize=size) - t = Thread(target=read_worker, args=( - r, - q, - )) + t = Thread( + target=read_worker, + args=( + r, + q, + ), + ) t.daemon = True t.start() e = q.get() @@ -398,7 +402,7 @@ def firstn_reader(): return firstn_reader -class XmapEndSignal(): +class XmapEndSignal: pass @@ -470,8 +474,11 @@ def xreader(): t.start() # start several handle_workers target = order_handle_worker if order else handle_worker - args = (in_queue, out_queue, mapper, - out_order) if order else (in_queue, out_queue, mapper) + args = ( + (in_queue, out_queue, mapper, out_order) + if order + else (in_queue, out_queue, mapper) + ) workers = [] for i in range(process_num): worker = Thread(target=target, args=args) @@ -579,7 +586,8 @@ def _impl(): if sys.platform == 'win32': raise NotImplementedError( - "The multiprocess_reader method is not supported on windows.") + "The multiprocess_reader method is not supported on windows." + ) # ujson is ultra fast json encoder and decoder written in pure C with bindings for Python 3.6+. try: @@ -587,11 +595,13 @@ def _impl(): except Exception as e: warnings.warn( "The `ujson` module is not found, use the `json` module, `ujson` encodes and decodes faster, " - "you can install `ujson` through `pip install ujson`.") + "you can install `ujson` through `pip install ujson`." + ) import json - assert isinstance(readers, (list, tuple)) and len(readers) > 0, ( - "`readers` must be list or tuple.") + assert ( + isinstance(readers, (list, tuple)) and len(readers) > 0 + ), "`readers` must be list or tuple." def _read_into_queue(reader, queue): try: @@ -607,8 +617,9 @@ def _read_into_queue(reader, queue): def queue_reader(): queue = fork_context.Queue(queue_size) for reader in readers: - p = fork_context.Process(target=_read_into_queue, - args=(reader, queue)) + p = fork_context.Process( + target=_read_into_queue, args=(reader, queue) + ) p.start() reader_num = len(readers) @@ -649,8 +660,9 @@ def pipe_reader(): for reader in readers: parent_conn, child_conn = fork_context.Pipe() conns.append(parent_conn) - p = fork_context.Process(target=_read_into_pipe, - args=(reader, child_conn)) + p = fork_context.Process( + target=_read_into_pipe, args=(reader, child_conn) + ) p.start() reader_num = len(readers) diff --git a/python/paddle/reader/tests/decorator_test.py b/python/paddle/reader/tests/decorator_test.py index 902a8cbe06c2f..92d257e065a01 100644 --- a/python/paddle/reader/tests/decorator_test.py +++ b/python/paddle/reader/tests/decorator_test.py @@ -23,7 +23,6 @@ def reader_creator_10(dur): - def reader(): for i in range(10): # this invocation helps testing paddle.reader.buffer @@ -34,7 +33,6 @@ def reader(): class TestMap(unittest.TestCase): - def test_map(self): d = {"h": 0, "i": 1} @@ -51,7 +49,6 @@ def read(): class TestBuffered(unittest.TestCase): - def test_read(self): for size in range(20): b = paddle.reader.buffered(reader_creator_10(0), size) @@ -76,10 +73,10 @@ def test_buffering(self): class TestCompose(unittest.TestCase): - def test_compse(self): - reader = paddle.reader.compose(reader_creator_10(0), - reader_creator_10(0)) + reader = paddle.reader.compose( + reader_creator_10(0), reader_creator_10(0) + ) for idx, e in enumerate(reader()): self.assertEqual(e, (idx, idx)) @@ -87,7 +84,8 @@ def test_compose_not_aligned(self): total = 0 reader = paddle.reader.compose( paddle.reader.chain(reader_creator_10(0), reader_creator_10(0)), - reader_creator_10(0)) + reader_creator_10(0), + ) with self.assertRaises(paddle.reader.ComposeNotAligned): for e in reader(): total += 1 @@ -96,10 +94,11 @@ def test_compose_not_aligned(self): def test_compose_not_aligned_no_check(self): total = 0 - reader = paddle.reader.compose(paddle.reader.chain( - reader_creator_10(0), reader_creator_10(0)), - reader_creator_10(0), - check_alignment=False) + reader = paddle.reader.compose( + paddle.reader.chain(reader_creator_10(0), reader_creator_10(0)), + reader_creator_10(0), + check_alignment=False, + ) for e in reader(): total += 1 # expecting 10, not 20 @@ -107,7 +106,6 @@ def test_compose_not_aligned_no_check(self): class TestChain(unittest.TestCase): - def test_chain(self): c = paddle.reader.chain(reader_creator_10(0), reader_creator_10(0)) idx = 0 @@ -118,7 +116,6 @@ def test_chain(self): class TestShuffle(unittest.TestCase): - def test_shuffle(self): case = [(0, True), (1, True), (10, False), (100, False)] a = reader_creator_10(0) @@ -133,11 +130,9 @@ def test_shuffle(self): class TestXmap(unittest.TestCase): - def test_xmap(self): - def mapper(x): - return (x + 1) + return x + 1 orders = (True, False) thread_nums = (1, 2, 4, 8, 16) @@ -145,9 +140,9 @@ def mapper(x): for order in orders: for tNum in thread_nums: for size in buffered_size: - reader = paddle.reader.xmap_readers(mapper, - reader_creator_10(0), - tNum, size, order) + reader = paddle.reader.xmap_readers( + mapper, reader_creator_10(0), tNum, size, order + ) for n in range(3): result = [] for i in reader(): @@ -159,7 +154,6 @@ def mapper(x): class TestMultiProcessReader(unittest.TestCase): - def setup(self): self.samples = [] for i in range(1000): @@ -178,7 +172,8 @@ def reader_test(self, use_pipe): self.setup() results = [] for data in paddle.reader.multiprocess_reader( - [self.reader0, self.reader1, self.reader2], 100, use_pipe)(): + [self.reader0, self.reader1, self.reader2], 100, use_pipe + )(): results.append(data) self.assertEqual(sorted(self.samples), sorted(results)) diff --git a/python/paddle/signal.py b/python/paddle/signal.py index 535a388d0e0df..82d46b8196763 100644 --- a/python/paddle/signal.py +++ b/python/paddle/signal.py @@ -122,7 +122,8 @@ def frame(x, frame_length, hop_length, axis=-1, name=None): if frame_length > x.shape[axis]: raise ValueError( f'Attribute frame_length should be less equal than sequence length, ' - f'but got ({frame_length}) > ({x.shape[axis]}).') + f'but got ({frame_length}) > ({x.shape[axis]}).' + ) op_type = 'frame' @@ -130,25 +131,33 @@ def frame(x, frame_length, hop_length, axis=-1, name=None): return _C_ops.frame(x, frame_length, hop_length, axis) if _in_legacy_dygraph(): - attrs = ('frame_length', frame_length, 'hop_length', hop_length, 'axis', - axis) + attrs = ( + 'frame_length', + frame_length, + 'hop_length', + hop_length, + 'axis', + axis, + ) op = getattr(_legacy_C_ops, op_type) out = op(x, *attrs) else: check_variable_and_dtype( - x, 'x', ['int32', 'int64', 'float16', 'float32', 'float64'], - op_type) + x, 'x', ['int32', 'int64', 'float16', 'float32', 'float64'], op_type + ) helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type=op_type, - inputs={'X': x}, - attrs={ - 'frame_length': frame_length, - 'hop_length': hop_length, - 'axis': axis - }, - outputs={'Out': out}) + helper.append_op( + type=op_type, + inputs={'X': x}, + attrs={ + 'frame_length': frame_length, + 'hop_length': hop_length, + 'axis': axis, + }, + outputs={'Out': out}, + ) return out @@ -223,31 +232,32 @@ def overlap_add(x, hop_length, axis=-1, name=None): out = op(x, *attrs) else: check_variable_and_dtype( - x, 'x', ['int32', 'int64', 'float16', 'float32', 'float64'], - op_type) + x, 'x', ['int32', 'int64', 'float16', 'float32', 'float64'], op_type + ) helper = LayerHelper(op_type, **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type=op_type, - inputs={'X': x}, - attrs={ - 'hop_length': hop_length, - 'axis': axis - }, - outputs={'Out': out}) + helper.append_op( + type=op_type, + inputs={'X': x}, + attrs={'hop_length': hop_length, 'axis': axis}, + outputs={'Out': out}, + ) return out -def stft(x, - n_fft, - hop_length=None, - win_length=None, - window=None, - center=True, - pad_mode='reflect', - normalized=False, - onesided=True, - name=None): +def stft( + x, + n_fft, + hop_length=None, + win_length=None, + window=None, + center=True, + pad_mode='reflect', + normalized=False, + onesided=True, + name=None, +): r""" Short-time Fourier transform (STFT). @@ -310,13 +320,15 @@ def stft(x, paddle.randn([8, 48000], dtype=paddle.float64)*1j # [8, 48000] complex128 y1 = stft(x, n_fft=512, center=False, onesided=False) # [8, 512, 372] """ - check_variable_and_dtype(x, 'x', - ['float32', 'float64', 'complex64', 'complex128'], - 'stft') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'complex64', 'complex128'], 'stft' + ) x_rank = len(x.shape) - assert x_rank in [1, 2], \ - f'x should be a 1D or 2D real tensor, but got rank of x is {x_rank}' + assert x_rank in [ + 1, + 2, + ], f'x should be a 1D or 2D real tensor, but got rank of x is {x_rank}' if x_rank == 1: # (batch, seq_length) x = x.unsqueeze(0) @@ -324,69 +336,77 @@ def stft(x, if hop_length is None: hop_length = int(n_fft // 4) - assert hop_length > 0, \ - f'hop_length should be > 0, but got {hop_length}.' + assert hop_length > 0, f'hop_length should be > 0, but got {hop_length}.' if win_length is None: win_length = n_fft if _non_static_mode(): - assert 0 < n_fft <= x.shape[-1], \ - f'n_fft should be in (0, seq_length({x.shape[-1]})], but got {n_fft}.' + assert ( + 0 < n_fft <= x.shape[-1] + ), f'n_fft should be in (0, seq_length({x.shape[-1]})], but got {n_fft}.' - assert 0 < win_length <= n_fft, \ - f'win_length should be in (0, n_fft({n_fft})], but got {win_length}.' + assert ( + 0 < win_length <= n_fft + ), f'win_length should be in (0, n_fft({n_fft})], but got {win_length}.' if window is not None: - assert len(window.shape) == 1 and len(window) == win_length, \ - f'expected a 1D window tensor of size equal to win_length({win_length}), but got window with shape {window.shape}.' + assert ( + len(window.shape) == 1 and len(window) == win_length + ), f'expected a 1D window tensor of size equal to win_length({win_length}), but got window with shape {window.shape}.' else: - window = paddle.ones(shape=(win_length, ), dtype=x.dtype) + window = paddle.ones(shape=(win_length,), dtype=x.dtype) if win_length < n_fft: pad_left = (n_fft - win_length) // 2 pad_right = n_fft - win_length - pad_left - window = paddle.nn.functional.pad(window, - pad=[pad_left, pad_right], - mode='constant') + window = paddle.nn.functional.pad( + window, pad=[pad_left, pad_right], mode='constant' + ) if center: - assert pad_mode in ['constant', 'reflect'], \ - 'pad_mode should be "reflect" or "constant", but got "{}".'.format(pad_mode) + assert pad_mode in [ + 'constant', + 'reflect', + ], 'pad_mode should be "reflect" or "constant", but got "{}".'.format( + pad_mode + ) pad_length = n_fft // 2 # FIXME: Input `x` can be a complex tensor but pad does not supprt complex input. - x = paddle.nn.functional.pad(x.unsqueeze(-1), - pad=[pad_length, pad_length], - mode=pad_mode, - data_format="NLC").squeeze(-1) + x = paddle.nn.functional.pad( + x.unsqueeze(-1), + pad=[pad_length, pad_length], + mode=pad_mode, + data_format="NLC", + ).squeeze(-1) x_frames = frame(x=x, frame_length=n_fft, hop_length=hop_length, axis=-1) x_frames = x_frames.transpose( - perm=[0, 2, - 1]) # switch n_fft to last dim, egs: (batch, num_frames, n_fft) + perm=[0, 2, 1] + ) # switch n_fft to last dim, egs: (batch, num_frames, n_fft) x_frames = paddle.multiply(x_frames, window) norm = 'ortho' if normalized else 'backward' if is_complex(x_frames): - assert not onesided, \ - 'onesided should be False when input or window is a complex Tensor.' + assert ( + not onesided + ), 'onesided should be False when input or window is a complex Tensor.' if not is_complex(x): - out = fft_r2c(x=x_frames, - n=None, - axis=-1, - norm=norm, - forward=True, - onesided=onesided, - name=name) + out = fft_r2c( + x=x_frames, + n=None, + axis=-1, + norm=norm, + forward=True, + onesided=onesided, + name=name, + ) else: - out = fft_c2c(x=x_frames, - n=None, - axis=-1, - norm=norm, - forward=True, - name=name) + out = fft_c2c( + x=x_frames, n=None, axis=-1, norm=norm, forward=True, name=name + ) out = out.transpose(perm=[0, 2, 1]) # (batch, n_fft, num_frames) @@ -396,17 +416,19 @@ def stft(x, return out -def istft(x, - n_fft, - hop_length=None, - win_length=None, - window=None, - center=True, - normalized=False, - onesided=True, - length=None, - return_complex=False, - name=None): +def istft( + x, + n_fft, + hop_length=None, + win_length=None, + window=None, + center=True, + normalized=False, + onesided=True, + length=None, + return_complex=False, + name=None, +): r""" Inverse short-time Fourier transform (ISTFT). @@ -482,8 +504,12 @@ def istft(x, check_variable_and_dtype(x, 'x', ['complex64', 'complex128'], 'istft') x_rank = len(x.shape) - assert x_rank in [2, 3], \ - 'x should be a 2D or 3D complex tensor, but got rank of x is {}'.format(x_rank) + assert x_rank in [ + 2, + 3, + ], 'x should be a 2D or 3D complex tensor, but got rank of x is {}'.format( + x_rank + ) if x_rank == 2: # (batch, n_fft, n_frames) x = x.unsqueeze(0) @@ -495,83 +521,107 @@ def istft(x, win_length = n_fft # Assure no gaps between frames. - assert 0 < hop_length <= win_length, \ - 'hop_length should be in (0, win_length({})], but got {}.'.format(win_length, hop_length) - - assert 0 < win_length <= n_fft, \ - 'win_length should be in (0, n_fft({})], but got {}.'.format(n_fft, win_length) + assert ( + 0 < hop_length <= win_length + ), 'hop_length should be in (0, win_length({})], but got {}.'.format( + win_length, hop_length + ) + + assert ( + 0 < win_length <= n_fft + ), 'win_length should be in (0, n_fft({})], but got {}.'.format( + n_fft, win_length + ) n_frames = x.shape[-1] fft_size = x.shape[-2] if _non_static_mode(): if onesided: - assert (fft_size == n_fft // 2 + 1), \ - 'fft_size should be equal to n_fft // 2 + 1({}) when onesided is True, but got {}.'.format(n_fft // 2 + 1, fft_size) + assert ( + fft_size == n_fft // 2 + 1 + ), 'fft_size should be equal to n_fft // 2 + 1({}) when onesided is True, but got {}.'.format( + n_fft // 2 + 1, fft_size + ) else: - assert (fft_size == n_fft), \ - 'fft_size should be equal to n_fft({}) when onesided is False, but got {}.'.format(n_fft, fft_size) + assert ( + fft_size == n_fft + ), 'fft_size should be equal to n_fft({}) when onesided is False, but got {}.'.format( + n_fft, fft_size + ) if window is not None: - assert len(window.shape) == 1 and len(window) == win_length, \ - 'expected a 1D window tensor of size equal to win_length({}), but got window with shape {}.'.format(win_length, window.shape) + assert ( + len(window.shape) == 1 and len(window) == win_length + ), 'expected a 1D window tensor of size equal to win_length({}), but got window with shape {}.'.format( + win_length, window.shape + ) else: - window_dtype = paddle.float32 if x.dtype in [ - paddle.float32, paddle.complex64 - ] else paddle.float64 - window = paddle.ones(shape=(win_length, ), dtype=window_dtype) + window_dtype = ( + paddle.float32 + if x.dtype in [paddle.float32, paddle.complex64] + else paddle.float64 + ) + window = paddle.ones(shape=(win_length,), dtype=window_dtype) if win_length < n_fft: pad_left = (n_fft - win_length) // 2 pad_right = n_fft - win_length - pad_left # FIXME: Input `window` can be a complex tensor but pad does not supprt complex input. - window = paddle.nn.functional.pad(window, - pad=[pad_left, pad_right], - mode='constant') + window = paddle.nn.functional.pad( + window, pad=[pad_left, pad_right], mode='constant' + ) x = x.transpose( - perm=[0, 2, - 1]) # switch n_fft to last dim, egs: (batch, num_frames, n_fft) + perm=[0, 2, 1] + ) # switch n_fft to last dim, egs: (batch, num_frames, n_fft) norm = 'ortho' if normalized else 'backward' if return_complex: - assert not onesided, \ - 'onesided should be False when input(output of istft) or window is a complex Tensor.' + assert ( + not onesided + ), 'onesided should be False when input(output of istft) or window is a complex Tensor.' out = fft_c2c(x=x, n=None, axis=-1, norm=norm, forward=False, name=None) else: - assert not is_complex(window), \ - 'Data type of window should not be complex when return_complex is False.' + assert not is_complex( + window + ), 'Data type of window should not be complex when return_complex is False.' if onesided is False: - x = x[:, :, :n_fft // 2 + 1] + x = x[:, :, : n_fft // 2 + 1] out = fft_c2r(x=x, n=None, axis=-1, norm=norm, forward=False, name=None) out = paddle.multiply(out, window).transpose( - perm=[0, 2, 1]) # (batch, n_fft, num_frames) - out = overlap_add(x=out, hop_length=hop_length, - axis=-1) # (batch, seq_length) + perm=[0, 2, 1] + ) # (batch, n_fft, num_frames) + out = overlap_add( + x=out, hop_length=hop_length, axis=-1 + ) # (batch, seq_length) window_envelop = overlap_add( x=paddle.tile( x=paddle.multiply(window, window).unsqueeze(0), - repeat_times=[n_frames, - 1]).transpose(perm=[1, 0]), # (n_fft, num_frames) + repeat_times=[n_frames, 1], + ).transpose( + perm=[1, 0] + ), # (n_fft, num_frames) hop_length=hop_length, - axis=-1) # (seq_length, ) + axis=-1, + ) # (seq_length, ) if length is None: if center: - out = out[:, (n_fft // 2):-(n_fft // 2)] - window_envelop = window_envelop[(n_fft // 2):-(n_fft // 2)] + out = out[:, (n_fft // 2) : -(n_fft // 2)] + window_envelop = window_envelop[(n_fft // 2) : -(n_fft // 2)] else: if center: start = n_fft // 2 else: start = 0 - out = out[:, start:start + length] - window_envelop = window_envelop[start:start + length] + out = out[:, start : start + length] + window_envelop = window_envelop[start : start + length] # Check whether the Nonzero Overlap Add (NOLA) constraint is met. if _non_static_mode() and window_envelop.abs().min().item() < 1e-11: diff --git a/python/paddle/sparse/binary.py b/python/paddle/sparse/binary.py index 1e7fcfe76345a..45c39e874a54c 100644 --- a/python/paddle/sparse/binary.py +++ b/python/paddle/sparse/binary.py @@ -266,10 +266,9 @@ def add(x, y, name=None): inputs = {'x': x, 'y': y} helper = LayerHelper(op_type) out = helper.create_sparse_variable_for_type_inference(x.dtype) - helper.append_op(type=op_type, - inputs=inputs, - outputs={'out': out}, - attrs={}) + helper.append_op( + type=op_type, inputs=inputs, outputs={'out': out}, attrs={} + ) return out diff --git a/python/paddle/sparse/creation.py b/python/paddle/sparse/creation.py index 5ca050be08a8b..684d449af7126 100644 --- a/python/paddle/sparse/creation.py +++ b/python/paddle/sparse/creation.py @@ -51,8 +51,8 @@ def _get_place(place): if place is None: place = _current_expected_place() elif not isinstance( - place, - (core.Place, core.CPUPlace, core.CUDAPinnedPlace, core.CUDAPlace)): + place, (core.Place, core.CPUPlace, core.CUDAPinnedPlace, core.CUDAPlace) + ): raise ValueError( "'place' must be any of paddle.Place, paddle.CPUPlace, paddle.CUDAPinnedPlace, paddle.CUDAPlace" ) @@ -66,12 +66,9 @@ def _check_indices_dtype(dtype): ) -def sparse_coo_tensor(indices, - values, - shape=None, - dtype=None, - place=None, - stop_gradient=True): +def sparse_coo_tensor( + indices, values, shape=None, dtype=None, place=None, stop_gradient=True +): r""" Constructs a sparse ``paddle.Tensor`` in coordinate format according to the indices and values of the specified non-zero elements. @@ -119,10 +116,9 @@ def sparse_coo_tensor(indices, place = _get_place(place) if not isinstance(indices, core.eager.Tensor): - indices = to_tensor(indices, - dtype=None, - place=place, - stop_gradient=True) + indices = to_tensor( + indices, dtype=None, place=place, stop_gradient=True + ) if not isinstance(values, core.eager.Tensor): values = to_tensor(values, dtype, place, stop_gradient) if len(indices.shape) != 2: @@ -135,8 +131,10 @@ def sparse_coo_tensor(indices, if nnz != values.shape[0]: raise ValueError( - "the indices and values must have same number of non-zero, but get {} and {}" - .format(nnz, values.shape[0])) + "the indices and values must have same number of non-zero, but get {} and {}".format( + nnz, values.shape[0] + ) + ) dense_dim = len(values.shape) - 1 @@ -156,11 +154,15 @@ def sparse_coo_tensor(indices, if shape < min_shape: raise ValueError( "the minimun shape required is {}, but get {}".format( - min_shape, shape)) + min_shape, shape + ) + ) if len(shape) != sparse_dim + dense_dim: raise ValueError( - "the number of dimensions(len(shape) must be sparse_dim({}) + dense_dim({}), but get {}" - .format(sparse_dim, dense_dim, len(shape))) + "the number of dimensions(len(shape) must be sparse_dim({}) + dense_dim({}), but get {}".format( + sparse_dim, dense_dim, len(shape) + ) + ) return _C_ops.sparse_sparse_coo_tensor(values, indices, shape) @@ -172,22 +174,17 @@ def sparse_coo_tensor(indices, attrs = {'dense_shape': shape} helper = LayerHelper(op_type) out = helper.create_sparse_variable_for_type_inference(dtype) - helper.append_op(type=op_type, - inputs=inputs, - outputs={'out': out}, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs={'out': out}, attrs=attrs + ) return out -#TODO: need to support shape is None +# TODO: need to support shape is None @dygraph_only -def sparse_csr_tensor(crows, - cols, - values, - shape, - dtype=None, - place=None, - stop_gradient=True): +def sparse_csr_tensor( + crows, cols, values, shape, dtype=None, place=None, stop_gradient=True +): r""" Constructs a sparse ``paddle.Tensor`` in CSR(Compressed Sparse Row) format according to the ``crows``, ``cols`` and ``values``. @@ -250,8 +247,10 @@ def sparse_csr_tensor(crows, if len(shape) != 2 and len(shape) != 3: raise ValueError( - "SparseCsrTensor only support 2-D or 3-D matrix. but get shape {}". - format(shape)) + "SparseCsrTensor only support 2-D or 3-D matrix. but get shape {}".format( + shape + ) + ) rows = shape[len(shape) - 2] if not crows.place._equals(place): @@ -268,26 +267,32 @@ def sparse_csr_tensor(crows, if len(crows.shape) != 1 or len(cols.shape) != 1 or len(values.shape) != 1: raise ValueError("The 'crows', 'cols' and 'values' must be 1-D.") - if (len(cols) != len(values)): + if len(cols) != len(values): raise ValueError("the length of cols must be same as length of values") if len(shape) == 2: if crows.shape[0] != rows + 1: raise ValueError( - "The length({}) of crows must be equal to the rows({})+1 of matrix." - .format(crows.shape[0], rows)) + "The length({}) of crows must be equal to the rows({})+1 of matrix.".format( + crows.shape[0], rows + ) + ) if crows[0] != 0: raise ValueError("the 0th value of crows must be 0") if crows[-1] != values.shape[0]: raise ValueError( - "the last value of crows must be equal the number of non-zero") + "the last value of crows must be equal the number of non-zero" + ) else: if crows.shape[0] % (rows + 1) != 0: raise ValueError( - "The length({}) of crows must be divisible the rows({})+1 of matrix." - .format(crows.shape[0], rows)) + "The length({}) of crows must be divisible the rows({})+1 of matrix.".format( + crows.shape[0], rows + ) + ) # TODO(zkh2016): check whether the value in crows and cols is legal - return core.eager.sparse_csr_tensor(crows, cols, values, shape, - stop_gradient) + return core.eager.sparse_csr_tensor( + crows, cols, values, shape, stop_gradient + ) diff --git a/python/paddle/sparse/nn/functional/activation.py b/python/paddle/sparse/nn/functional/activation.py index 516cfc5cd583d..a50a64f5164fe 100644 --- a/python/paddle/sparse/nn/functional/activation.py +++ b/python/paddle/sparse/nn/functional/activation.py @@ -52,10 +52,9 @@ def relu(x, name=None): op_type = 'sparse_relu' helper = LayerHelper(op_type) out = helper.create_sparse_variable_for_type_inference(x.dtype) - helper.append_op(type=op_type, - inputs={'x': x}, - outputs={'out': out}, - attrs={}) + helper.append_op( + type=op_type, inputs={'x': x}, outputs={'out': out}, attrs={} + ) return out diff --git a/python/paddle/sparse/nn/functional/conv.py b/python/paddle/sparse/nn/functional/conv.py index 03b0f2e9be628..ec1907a7770c8 100644 --- a/python/paddle/sparse/nn/functional/conv.py +++ b/python/paddle/sparse/nn/functional/conv.py @@ -21,50 +21,66 @@ from paddle.fluid.layer_helper import LayerHelper -def _conv3d(x, - weight, - bias=None, - stride=1, - padding=0, - dilation=1, - groups=1, - subm=False, - key=None, - data_format="NDHWC", - name=None): +def _conv3d( + x, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + subm=False, + key=None, + data_format="NDHWC", + name=None, +): assert groups == 1, "Currently, only support groups=1" dims = 3 # Currently, only support 'NDHWC' if data_format not in ["NDHWC"]: - raise ValueError("Attr(data_format) should be 'NDHWC'. Received " - "Attr(data_format): {}.".format(data_format)) + raise ValueError( + "Attr(data_format) should be 'NDHWC'. Received " + "Attr(data_format): {}.".format(data_format) + ) if len(x.shape) != 5: raise ValueError( - "Input x should be 5D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 5D tensor, but received x with the shape of {}".format( + x.shape + ) + ) - channel_last = (data_format == "NDHWC") + channel_last = data_format == "NDHWC" channel_dim = -1 if channel_last else 1 if len(x.shape) != 5: raise ValueError( - "Input x should be 5D tensor, but received x with the shape of {}". - format(x.shape)) + "Input x should be 5D tensor, but received x with the shape of {}".format( + x.shape + ) + ) num_channels = x.shape[channel_dim] if num_channels < 0: raise ValueError( "The channel dimension of the input({}) should be defined. " - "Received: {}.".format(x.shape, num_channels)) + "Received: {}.".format(x.shape, num_channels) + ) padding, padding_algorithm = _update_padding_nd(padding, channel_last, dims) stride = convert_to_list(stride, dims, 'stride') dilation = convert_to_list(dilation, dims, 'dilation') if in_dynamic_mode(): - pre_bias = _C_ops.sparse_conv3d(x, weight, padding, dilation, stride, - groups, subm, - key if key is not None else "") + pre_bias = _C_ops.sparse_conv3d( + x, + weight, + padding, + dilation, + stride, + groups, + subm, + key if key is not None else "", + ) if bias is not None: return add(pre_bias, bias) else: @@ -77,35 +93,38 @@ def _conv3d(x, 'strides': stride, 'groups': groups, 'subm': subm, - 'key': key + 'key': key, } op_type = 'sparse_conv3d' helper = LayerHelper(op_type, **locals()) - rulebook = helper.create_variable_for_type_inference(dtype='int32', - stop_gradient=True) - counter = helper.create_variable_for_type_inference(dtype='int32', - stop_gradient=True) + rulebook = helper.create_variable_for_type_inference( + dtype='int32', stop_gradient=True + ) + counter = helper.create_variable_for_type_inference( + dtype='int32', stop_gradient=True + ) pre_bias = helper.create_sparse_variable_for_type_inference(x.dtype) outputs = {"out": pre_bias, "rulebook": rulebook, "counter": counter} - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) if bias is not None: return add(pre_bias, bias) else: return pre_bias -def conv3d(x, - weight, - bias=None, - stride=1, - padding=0, - dilation=1, - groups=1, - data_format="NDHWC", - name=None): +def conv3d( + x, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + data_format="NDHWC", + name=None, +): r""" The sparse convolution3d functional calculates the output based on the input, filter @@ -207,20 +226,33 @@ def conv3d(x, print(y.shape) # (1, 1, 1, 2, 1) """ - return _conv3d(x, weight, bias, stride, padding, dilation, groups, False, - None, data_format, name) - - -def subm_conv3d(x, - weight, - bias=None, - stride=1, - padding=0, - dilation=1, - groups=1, - data_format="NDHWC", - key=None, - name=None): + return _conv3d( + x, + weight, + bias, + stride, + padding, + dilation, + groups, + False, + None, + data_format, + name, + ) + + +def subm_conv3d( + x, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + data_format="NDHWC", + key=None, + name=None, +): r""" The sparse submanifold convolution3d functional calculates the output based on the input, filter @@ -327,5 +359,16 @@ def subm_conv3d(x, print(y.shape) #(1, 1, 3, 4, 1) """ - return _conv3d(x, weight, bias, stride, padding, dilation, groups, True, - key, data_format, name) + return _conv3d( + x, + weight, + bias, + stride, + padding, + dilation, + groups, + True, + key, + data_format, + name, + ) diff --git a/python/paddle/sparse/nn/functional/pooling.py b/python/paddle/sparse/nn/functional/pooling.py index d86ac58e9229d..740324ea3979d 100644 --- a/python/paddle/sparse/nn/functional/pooling.py +++ b/python/paddle/sparse/nn/functional/pooling.py @@ -19,13 +19,15 @@ __all__ = [] -def max_pool3d(x, - kernel_size, - stride=None, - padding=0, - ceil_mode=False, - data_format="NDHWC", - name=None): +def max_pool3d( + x, + kernel_size, + stride=None, + padding=0, + ceil_mode=False, + data_format="NDHWC", + name=None, +): """ Implements sparse max pooling 3d operation. See more details in :ref:`api_sparse_pooling_MaxPool3d` . @@ -75,9 +77,12 @@ def max_pool3d(x, """ assert in_dynamic_mode(), "Currently, Sparse API only support dynamic mode" - assert x.is_sparse_coo( + assert ( + x.is_sparse_coo() ), "Currently, sparse.relu only support the input of SparseCooTensor" - assert data_format == 'NDHWC', "Currently, sparse.max_pool3d only support data format of 'NDHWC'" + assert ( + data_format == 'NDHWC' + ), "Currently, sparse.max_pool3d only support data format of 'NDHWC'" kernel_size = utils.convert_to_list(kernel_size, 3, 'pool_size') if stride is None: @@ -87,12 +92,11 @@ def max_pool3d(x, channel_last = True - padding, padding_algorithm = _update_padding_nd(padding, - 3, - channel_last=channel_last, - ceil_mode=ceil_mode) + padding, padding_algorithm = _update_padding_nd( + padding, 3, channel_last=channel_last, ceil_mode=ceil_mode + ) - #TODO(zkh2016): remove the dependency on dilation from the backend + # TODO(zkh2016): remove the dependency on dilation from the backend dilation = [1, 1, 1] return _C_ops.sparse_maxpool(x, kernel_size, padding, dilation, stride) diff --git a/python/paddle/sparse/nn/functional/transformer.py b/python/paddle/sparse/nn/functional/transformer.py index f17d728078a79..38118ba359b57 100644 --- a/python/paddle/sparse/nn/functional/transformer.py +++ b/python/paddle/sparse/nn/functional/transformer.py @@ -19,14 +19,16 @@ @dygraph_only -def attention(query, - key, - value, - sparse_mask, - key_padding_mask=None, - attn_mask=None, - name=None): - """ +def attention( + query, + key, + value, + sparse_mask, + key_padding_mask=None, + attn_mask=None, + name=None, +): + r""" Note: This API is only used from ``CUDA 11.7`` . @@ -37,7 +39,7 @@ def attention(query, .. math:: - result = softmax(\\frac{ Q * K^T }{\\sqrt{d}}) * V + result = softmax(\frac{ Q * K^T }{\sqrt{d}}) * V where : ``Q``, ``K``, and ``V`` represent the three input parameters of the attention module. The shape of the three parameters are: `[batch_size, num_heads, seq_len, head_dim]`, and @@ -89,5 +91,6 @@ def attention(query, output = paddle.sparse.nn.functional.attention(query, key, value, sp_mask, kp_mask, attn_mask) output.backward() """ - return _C_ops.sparse_fused_attention(query, key, value, sparse_mask, - key_padding_mask, attn_mask) + return _C_ops.sparse_fused_attention( + query, key, value, sparse_mask, key_padding_mask, attn_mask + ) diff --git a/python/paddle/sparse/nn/layer/conv.py b/python/paddle/sparse/nn/layer/conv.py index 8368eeeb03cae..599fb2d7b2e5c 100644 --- a/python/paddle/sparse/nn/layer/conv.py +++ b/python/paddle/sparse/nn/layer/conv.py @@ -23,23 +23,26 @@ class _Conv3D(Layer): - - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - dilation=1, - groups=1, - subm=False, - key=None, - padding_mode='zeros', - weight_attr=None, - bias_attr=None, - data_format="NDHWC"): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + subm=False, + key=None, + padding_mode='zeros', + weight_attr=None, + bias_attr=None, + data_format="NDHWC", + ): super(_Conv3D, self).__init__() - assert weight_attr is not False, "weight_attr should not be False in Conv." + assert ( + weight_attr is not False + ), "weight_attr should not be False in Conv." self._param_attr = weight_attr self._bias_attr = bias_attr self._groups = groups @@ -49,56 +52,66 @@ def __init__(self, self._subm = subm self._key = key - assert padding_mode == 'zeros', "Currently, only support padding_mode='zeros'" + assert ( + padding_mode == 'zeros' + ), "Currently, only support padding_mode='zeros'" assert groups == 1, "Currently, only support groups=1" valid_format = {'NDHWC'} if data_format not in valid_format: raise ValueError( - "data_format must be one of {}, but got data_format='{}'". - format(valid_format, data_format)) + "data_format must be one of {}, but got data_format='{}'".format( + valid_format, data_format + ) + ) channel_last = data_format == "NDHWC" dims = 3 self._stride = utils.convert_to_list(stride, dims, 'stride') self._dilation = utils.convert_to_list(dilation, dims, 'dilation') - self._kernel_size = utils.convert_to_list(kernel_size, dims, - 'kernel_size') + self._kernel_size = utils.convert_to_list( + kernel_size, dims, 'kernel_size' + ) self._padding = padding self._padding_mode = padding_mode self._updated_padding, self._padding_algorithm = _update_padding_nd( - padding, channel_last, dims) + padding, channel_last, dims + ) # the sparse conv restricts the shape is [D, H, W, in_channels, out_channels] filter_shape = self._kernel_size + [ - self._in_channels, self._out_channels + self._in_channels, + self._out_channels, ] def _get_default_param_initializer(): filter_elem_num = np.prod(self._kernel_size) * self._in_channels - std = (2.0 / filter_elem_num)**0.5 + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std) self.weight = self.create_parameter( shape=filter_shape, attr=self._param_attr, - default_initializer=_get_default_param_initializer()) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._out_channels], - is_bias=True) + default_initializer=_get_default_param_initializer(), + ) + self.bias = self.create_parameter( + attr=self._bias_attr, shape=[self._out_channels], is_bias=True + ) def forward(self, x): - out = F.conv._conv3d(x, - self.weight, - bias=self.bias, - stride=self._stride, - padding=self._updated_padding, - dilation=self._dilation, - groups=self._groups, - subm=self._subm, - key=self._key, - data_format=self._data_format) + out = F.conv._conv3d( + x, + self.weight, + bias=self.bias, + stride=self._stride, + padding=self._updated_padding, + dilation=self._dilation, + groups=self._groups, + subm=self._subm, + key=self._key, + data_format=self._data_format, + ) return out def extra_repr(self): @@ -223,31 +236,35 @@ class Conv3D(_Conv3D): # (1, 1, 1, 2, 1) """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - dilation=1, - groups=1, - padding_mode='zeros', - weight_attr=None, - bias_attr=None, - data_format="NDHWC"): - super(Conv3D, self).__init__(in_channels, - out_channels, - kernel_size, - stride=stride, - padding=padding, - dilation=dilation, - groups=groups, - subm=False, - key=None, - padding_mode=padding_mode, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + padding_mode='zeros', + weight_attr=None, + bias_attr=None, + data_format="NDHWC", + ): + super(Conv3D, self).__init__( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + subm=False, + key=None, + padding_mode=padding_mode, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) class SubmConv3D(_Conv3D): @@ -360,29 +377,33 @@ class SubmConv3D(_Conv3D): # (1, 1, 3, 4, 1) """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - dilation=1, - groups=1, - padding_mode='zeros', - key=None, - weight_attr=None, - bias_attr=None, - data_format="NDHWC"): - super(SubmConv3D, self).__init__(in_channels, - out_channels, - kernel_size, - stride=stride, - padding=padding, - dilation=dilation, - groups=groups, - subm=True, - key=key, - padding_mode=padding_mode, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format) + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + padding_mode='zeros', + key=None, + weight_attr=None, + bias_attr=None, + data_format="NDHWC", + ): + super(SubmConv3D, self).__init__( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + subm=True, + key=key, + padding_mode=padding_mode, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + ) diff --git a/python/paddle/sparse/nn/layer/norm.py b/python/paddle/sparse/nn/layer/norm.py index b36a85f6b789b..99b96fb530aef 100644 --- a/python/paddle/sparse/nn/layer/norm.py +++ b/python/paddle/sparse/nn/layer/norm.py @@ -97,23 +97,27 @@ class BatchNorm(paddle.nn.BatchNorm1D): # [1, 6, 6, 6, 3] """ - def __init__(self, - num_features, - momentum=0.9, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - data_format='NDHWC', - use_global_stats=None, - name=None): - super(BatchNorm, self).__init__(num_features, - momentum=momentum, - epsilon=epsilon, - weight_attr=weight_attr, - bias_attr=bias_attr, - data_format=data_format, - use_global_stats=use_global_stats, - name=name) + def __init__( + self, + num_features, + momentum=0.9, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NDHWC', + use_global_stats=None, + name=None, + ): + super(BatchNorm, self).__init__( + num_features, + momentum=momentum, + epsilon=epsilon, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + use_global_stats=use_global_stats, + name=name, + ) def _check_data_format(self, input): if input != "NDHWC": @@ -124,7 +128,8 @@ def forward(self, input): if self.training: warnings.warn( - "When training, we now always track global mean and variance.") + "When training, we now always track global mean and variance." + ) if self._use_global_stats == None: self._use_global_stats = not self.training @@ -136,9 +141,19 @@ def forward(self, input): if in_dynamic_mode(): batch_norm_out, _, _, _, _, _ = _C_ops.sparse_batch_norm( - input, self.weight, self.bias, self._mean, self._variance, - self._momentum, self._epsilon, data_format, not self.training, - self._use_global_stats, trainable_statistics, False) + input, + self.weight, + self.bias, + self._mean, + self._variance, + self._momentum, + self._epsilon, + data_format, + not self.training, + self._use_global_stats, + trainable_statistics, + False, + ) return batch_norm_out else: inputs = { @@ -146,7 +161,7 @@ def forward(self, input): 'scale': self.weight, 'bias': self.bias, 'mean': self._mean, - 'variance': self._variance + 'variance': self._variance, } attrs = { 'momentum': self._momentum, @@ -155,21 +170,26 @@ def forward(self, input): 'is_test': not self.training, 'use_global_stats': self._use_global_stats, 'trainable_statistics': trainable_statistics, - 'fuse_with_relu': False + 'fuse_with_relu': False, } op_type = 'sparse_batch_norm' helper = LayerHelper(op_type) dtype = input.dtype mean_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) variance_out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) saved_mean = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) saved_variance = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) reserve_space = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=True) + dtype=dtype, stop_gradient=True + ) out = helper.create_sparse_variable_for_type_inference(dtype) outputs = { "out": out, @@ -177,12 +197,11 @@ def forward(self, input): "variance_out": variance_out, "saved_mean": saved_mean, "saved_variance": saved_variance, - "reserve_space": reserve_space + "reserve_space": reserve_space, } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -283,24 +302,42 @@ class SyncBatchNorm(paddle.nn.SyncBatchNorm): # [-0.88415730, 1.57439375]]) """ - def __init__(self, - num_features, - momentum=0.9, - epsilon=1e-05, - weight_attr=None, - bias_attr=None, - data_format='NCHW', - name=None): - super(SyncBatchNorm, - self).__init__(num_features, momentum, epsilon, weight_attr, - bias_attr, data_format, name) + def __init__( + self, + num_features, + momentum=0.9, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NCHW', + name=None, + ): + super(SyncBatchNorm, self).__init__( + num_features, + momentum, + epsilon, + weight_attr, + bias_attr, + data_format, + name, + ) def forward(self, x): self._check_data_format() sync_batch_norm_out, _, _, _, _, _ = _C_ops.sparse_sync_batch_norm_( - x, self.weight, self.bias, self._mean, self._variance, - self._momentum, self._epsilon, self._data_format, not self.training, - False, False, False) + x, + self.weight, + self.bias, + self._mean, + self._variance, + self._momentum, + self._epsilon, + self._data_format, + not self.training, + False, + False, + False, + ) return sync_batch_norm_out @classmethod @@ -327,27 +364,41 @@ def convert_sync_batchnorm(cls, layer): layer_output = layer if isinstance(layer, _BatchNormBase): - if layer._weight_attr != None and not isinstance( - layer._weight_attr, - bool) and layer._weight_attr.name != None: + if ( + layer._weight_attr != None + and not isinstance(layer._weight_attr, bool) + and layer._weight_attr.name != None + ): layer._weight_attr.name = layer._weight_attr.name + '_sync' - if layer._bias_attr != None and not isinstance( - layer._bias_attr, bool) and layer._bias_attr.name != None: + if ( + layer._bias_attr != None + and not isinstance(layer._bias_attr, bool) + and layer._bias_attr.name != None + ): layer._bias_attr.name = layer._bias_attr.name + '_sync' - #convert sparse BatchNorm + # convert sparse BatchNorm if isinstance(layer, BatchNorm): - layer_output = SyncBatchNorm(layer._num_features, - layer._momentum, layer._epsilon, - layer._weight_attr, - layer._bias_attr, - layer._data_format, layer._name) - #convert dense BatchNorm + layer_output = SyncBatchNorm( + layer._num_features, + layer._momentum, + layer._epsilon, + layer._weight_attr, + layer._bias_attr, + layer._data_format, + layer._name, + ) + # convert dense BatchNorm else: layer_output = paddle.nn.SyncBatchNorm( - layer._num_features, layer._momentum, layer._epsilon, - layer._weight_attr, layer._bias_attr, layer._data_format, - layer._name) + layer._num_features, + layer._momentum, + layer._epsilon, + layer._weight_attr, + layer._bias_attr, + layer._data_format, + layer._name, + ) if layer._weight_attr != False and layer._bias_attr != False: with no_grad(): @@ -357,7 +408,8 @@ def convert_sync_batchnorm(cls, layer): layer_output._variance = layer._variance for name, sublayer in layer.named_children(): - layer_output.add_sublayer(name, - cls.convert_sync_batchnorm(sublayer)) + layer_output.add_sublayer( + name, cls.convert_sync_batchnorm(sublayer) + ) del layer return layer_output diff --git a/python/paddle/sparse/nn/layer/pooling.py b/python/paddle/sparse/nn/layer/pooling.py index ee15bf6f79369..7d6141bab0fd5 100644 --- a/python/paddle/sparse/nn/layer/pooling.py +++ b/python/paddle/sparse/nn/layer/pooling.py @@ -73,14 +73,16 @@ class MaxPool3D(Layer): """ - def __init__(self, - kernel_size, - stride=None, - padding=0, - return_mask=False, - ceil_mode=False, - data_format="NDHWC", - name=None): + def __init__( + self, + kernel_size, + stride=None, + padding=0, + return_mask=False, + ceil_mode=False, + data_format="NDHWC", + name=None, + ): super(MaxPool3D, self).__init__() self.ksize = kernel_size self.stride = stride @@ -91,14 +93,17 @@ def __init__(self, self.name = name def forward(self, x): - return F.max_pool3d(x, - kernel_size=self.ksize, - stride=self.stride, - padding=self.padding, - ceil_mode=self.ceil_mode, - data_format=self.data_format, - name=self.name) + return F.max_pool3d( + x, + kernel_size=self.ksize, + stride=self.stride, + padding=self.padding, + ceil_mode=self.ceil_mode, + data_format=self.data_format, + name=self.name, + ) def extra_repr(self): return 'kernel_size={ksize}, stride={stride}, padding={padding}'.format( - **self.__dict__) + **self.__dict__ + ) diff --git a/python/paddle/sparse/unary.py b/python/paddle/sparse/unary.py index c26ec35d3b7ed..14a97d6cb0328 100644 --- a/python/paddle/sparse/unary.py +++ b/python/paddle/sparse/unary.py @@ -15,7 +15,11 @@ import numpy as np from paddle import _C_ops -from paddle.fluid.framework import dygraph_only, core, convert_np_dtype_to_dtype_ +from paddle.fluid.framework import ( + dygraph_only, + core, + convert_np_dtype_to_dtype_, +) __all__ = [] diff --git a/python/paddle/static/__init__.py b/python/paddle/static/__init__.py index 44ab13360618a..78ad1cfabc311 100644 --- a/python/paddle/static/__init__.py +++ b/python/paddle/static/__init__.py @@ -73,18 +73,55 @@ from ..fluid.contrib.layers import ctr_metric_bundle # noqa: F401 from ..fluid.layers import exponential_decay # noqa: F401 -__all__ = [ #noqa - 'append_backward', 'gradients', 'Executor', 'global_scope', 'scope_guard', - 'BuildStrategy', 'CompiledProgram', 'ipu_shard_guard', 'IpuCompiledProgram', - 'IpuStrategy', 'Print', 'py_func', 'ExecutionStrategy', 'name_scope', - 'ParallelExecutor', 'program_guard', 'WeightNormParamAttr', - 'ExponentialMovingAverage', 'default_main_program', - 'default_startup_program', 'Program', 'data', 'InputSpec', 'save', 'load', - 'save_inference_model', 'load_inference_model', 'serialize_program', - 'serialize_persistables', 'save_to_file', 'deserialize_program', - 'deserialize_persistables', 'load_from_file', 'normalize_program', - 'load_program_state', 'set_program_state', 'cpu_places', 'cuda_places', - 'xpu_places', 'npu_places', 'mlu_places', 'Variable', 'create_global_var', - 'accuracy', 'auc', 'device_guard', 'create_parameter', 'set_ipu_shard', - 'ctr_metric_bundle', 'exponential_decay' +__all__ = [ # noqa + 'append_backward', + 'gradients', + 'Executor', + 'global_scope', + 'scope_guard', + 'BuildStrategy', + 'CompiledProgram', + 'ipu_shard_guard', + 'IpuCompiledProgram', + 'IpuStrategy', + 'Print', + 'py_func', + 'ExecutionStrategy', + 'name_scope', + 'ParallelExecutor', + 'program_guard', + 'WeightNormParamAttr', + 'ExponentialMovingAverage', + 'default_main_program', + 'default_startup_program', + 'Program', + 'data', + 'InputSpec', + 'save', + 'load', + 'save_inference_model', + 'load_inference_model', + 'serialize_program', + 'serialize_persistables', + 'save_to_file', + 'deserialize_program', + 'deserialize_persistables', + 'load_from_file', + 'normalize_program', + 'load_program_state', + 'set_program_state', + 'cpu_places', + 'cuda_places', + 'xpu_places', + 'npu_places', + 'mlu_places', + 'Variable', + 'create_global_var', + 'accuracy', + 'auc', + 'device_guard', + 'create_parameter', + 'set_ipu_shard', + 'ctr_metric_bundle', + 'exponential_decay', ] diff --git a/python/paddle/static/amp/__init__.py b/python/paddle/static/amp/__init__.py index 8ee3225057d0a..ffcfa404cd356 100644 --- a/python/paddle/static/amp/__init__.py +++ b/python/paddle/static/amp/__init__.py @@ -14,8 +14,12 @@ from ...fluid.contrib.mixed_precision import decorate # noqa: F401 from ...fluid.contrib.mixed_precision import CustomOpLists # noqa: F401 -from ...fluid.contrib.mixed_precision import AutoMixedPrecisionLists # noqa: F401 +from ...fluid.contrib.mixed_precision import ( + AutoMixedPrecisionLists, +) # noqa: F401 from ...fluid.contrib.mixed_precision import fp16_guard # noqa: F401 from ...fluid.contrib.mixed_precision import cast_model_to_fp16 # noqa: F401 -from ...fluid.contrib.mixed_precision import cast_parameters_to_fp16 # noqa: F401 +from ...fluid.contrib.mixed_precision import ( + cast_parameters_to_fp16, +) # noqa: F401 from ...fluid.contrib.mixed_precision import bf16 # noqa: F401 diff --git a/python/paddle/static/input.py b/python/paddle/static/input.py index 6cd5700764118..8a398b42ded8d 100644 --- a/python/paddle/static/input.py +++ b/python/paddle/static/input.py @@ -105,7 +105,8 @@ def data(name, shape, dtype=None, lod_level=0): stop_gradient=True, lod_level=lod_level, is_data=True, - need_check_feed=True) + need_check_feed=True, + ) else: return helper.create_global_variable( name=name, @@ -115,7 +116,8 @@ def data(name, shape, dtype=None, lod_level=0): stop_gradient=True, lod_level=lod_level, is_data=True, - need_check_feed=True) + need_check_feed=True, + ) class InputSpec(object): @@ -164,7 +166,8 @@ def _create_feed_layer(self): def __repr__(self): return '{}(shape={}, dtype={}, name={})'.format( - type(self).__name__, self.shape, self.dtype, self.name) + type(self).__name__, self.shape, self.dtype, self.name + ) @classmethod def from_tensor(cls, tensor, name=None): @@ -196,7 +199,9 @@ def from_tensor(cls, tensor, name=None): else: raise ValueError( "Input `tensor` should be a Tensor, but received {}.".format( - type(tensor).__name__)) + type(tensor).__name__ + ) + ) @classmethod def from_numpy(cls, ndarray, name=None): @@ -245,13 +250,17 @@ def batch(self, batch_size): if isinstance(batch_size, (list, tuple)): if len(batch_size) != 1: raise ValueError( - "Length of batch_size: {} shall be 1, but received {}.". - format(batch_size, len(batch_size))) + "Length of batch_size: {} shall be 1, but received {}.".format( + batch_size, len(batch_size) + ) + ) batch_size = batch_size[1] elif not isinstance(batch_size, int): raise TypeError( "type(batch_size) shall be `int`, but received {}.".format( - type(batch_size).__name__)) + type(batch_size).__name__ + ) + ) new_shape = [batch_size] + list(self.shape) self.shape = tuple(new_shape) @@ -277,7 +286,8 @@ def unbatch(self): """ if len(self.shape) == 0: raise ValueError( - "Not support to unbatch a InputSpec when len(shape) == 0.") + "Not support to unbatch a InputSpec when len(shape) == 0." + ) self.shape = self._verify(self.shape[1:]) return self @@ -288,20 +298,25 @@ def _verify(self, shape): """ if not isinstance(shape, (list, tuple)): raise TypeError( - "Type of `shape` in InputSpec should be one of (tuple, list), but received {}." - .format(type(shape).__name__)) + "Type of `shape` in InputSpec should be one of (tuple, list), but received {}.".format( + type(shape).__name__ + ) + ) if len(shape) == 0: raise ValueError( - "`shape` in InputSpec should contain at least 1 element, but received {}." - .format(shape)) + "`shape` in InputSpec should contain at least 1 element, but received {}.".format( + shape + ) + ) for i, ele in enumerate(shape): if ele is not None: if not isinstance(ele, int): raise ValueError( - "shape[{}] should be an `int`, but received `{}`:{}.". - format(i, - type(ele).__name__, ele)) + "shape[{}] should be an `int`, but received `{}`:{}.".format( + i, type(ele).__name__, ele + ) + ) if ele is None or ele < -1: shape[i] = -1 @@ -326,8 +341,9 @@ def __hash__(self): def __eq__(self, other): slots = ['shape', 'dtype', 'name'] - return (type(self) is type(other) and all( - getattr(self, attr) == getattr(other, attr) for attr in slots)) + return type(self) is type(other) and all( + getattr(self, attr) == getattr(other, attr) for attr in slots + ) def __ne__(self, other): return not self == other diff --git a/python/paddle/static/io.py b/python/paddle/static/io.py index 279fc60a669fe..ac598328352a9 100644 --- a/python/paddle/static/io.py +++ b/python/paddle/static/io.py @@ -37,9 +37,9 @@ __all__ = [] -_logger = get_logger(__name__, - logging.INFO, - fmt='%(asctime)s-%(levelname)s: %(message)s') +_logger = get_logger( + __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s' +) def _check_args(caller, args, supported_args=None, deprecated_args=None): @@ -48,12 +48,16 @@ def _check_args(caller, args, supported_args=None, deprecated_args=None): for arg in args: if arg in deprecated_args: raise ValueError( - "argument '{}' in function '{}' is deprecated, only {} are supported." - .format(arg, caller, supported_args)) + "argument '{}' in function '{}' is deprecated, only {} are supported.".format( + arg, caller, supported_args + ) + ) elif arg not in supported_args: raise ValueError( - "function '{}' doesn't support argument '{}',\n only {} are supported." - .format(caller, arg, supported_args)) + "function '{}' doesn't support argument '{}',\n only {} are supported.".format( + caller, arg, supported_args + ) + ) def _check_vars(name, var_list): @@ -61,7 +65,8 @@ def _check_vars(name, var_list): var_list = [var_list] if not var_list or not all([isinstance(var, Variable) for var in var_list]): raise ValueError( - "'{}' should be a Variable or a list of Variable.".format(name)) + "'{}' should be a Variable or a list of Variable.".format(name) + ) def _normalize_path_prefix(path_prefix): @@ -90,29 +95,35 @@ def _get_valid_program(program=None): "The type of input program is invalid, expected tyep is Program, but received None" ) warnings.warn( - "The input is a CompiledProgram, this is not recommended.") + "The input is a CompiledProgram, this is not recommended." + ) if not isinstance(program, Program): raise TypeError( "The type of input program is invalid, expected type is fluid.Program, but received %s" - % type(program)) + % type(program) + ) return program def _clone_var_in_block(block, var): assert isinstance(var, Variable) if var.desc.type() == core.VarDesc.VarType.LOD_TENSOR: - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - lod_level=var.lod_level, - persistable=True) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + lod_level=var.lod_level, + persistable=True, + ) else: - return block.create_var(name=var.name, - shape=var.shape, - dtype=var.dtype, - type=var.type, - persistable=True) + return block.create_var( + name=var.name, + shape=var.shape, + dtype=var.dtype, + type=var.type, + persistable=True, + ) def normalize_program(program, feed_vars, fetch_vars): @@ -155,18 +166,21 @@ def normalize_program(program, feed_vars, fetch_vars): """ if not isinstance(program, Program): raise TypeError( - "program type must be `fluid.Program`, but received `%s`" % - type(program)) + "program type must be `fluid.Program`, but received `%s`" + % type(program) + ) if not isinstance(feed_vars, list): feed_vars = [feed_vars] if not all(isinstance(v, Variable) for v in feed_vars): raise TypeError( - "feed_vars type must be a Variable or a list of Variable.") + "feed_vars type must be a Variable or a list of Variable." + ) if not isinstance(fetch_vars, list): fetch_vars = [fetch_vars] if not all(isinstance(v, Variable) for v in fetch_vars): raise TypeError( - "fetch_vars type must be a Variable or a list of Variable.") + "fetch_vars type must be a Variable or a list of Variable." + ) # remind users to set auc_states to 0 if auc op were found. for op in program.global_block().ops: @@ -174,8 +188,10 @@ def normalize_program(program, feed_vars, fetch_vars): device_attr_name = core.op_proto_and_checker_maker.kOpDeviceAttrName() op._set_attr(device_attr_name, "") if op.type == 'auc': - warnings.warn("Be sure that you have set auc states to 0 " - "before saving inference model.") + warnings.warn( + "Be sure that you have set auc states to 0 " + "before saving inference model." + ) break # fix the bug that the activation op's output as target will be pruned. @@ -185,9 +201,9 @@ def normalize_program(program, feed_vars, fetch_vars): uniq_fetch_vars = [] for i, var in enumerate(fetch_vars): if var.dtype != paddle.bool: - var = layers.scale(var, - 1., - name="save_infer_model/scale_{}".format(i)) + var = layers.scale( + var, 1.0, name="save_infer_model/scale_{}".format(i) + ) uniq_fetch_vars.append(var) fetch_vars = uniq_fetch_vars @@ -205,7 +221,8 @@ def normalize_program(program, feed_vars, fetch_vars): feed_var_names = [var.name for var in feed_vars] copy_program = copy_program._prune_with_input( - feeded_var_names=feed_var_names, targets=fetch_vars) + feeded_var_names=feed_var_names, targets=fetch_vars + ) copy_program = copy_program._inference_optimize(prune_read_op=True) fetch_var_names = [var.name for var in fetch_vars] prepend_feed_ops(copy_program, feed_var_names) @@ -235,9 +252,11 @@ def is_persistable(var): param = fluid.default_main_program().global_block().var('fc.b') res = fluid.io.is_persistable(param) """ - if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ - var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \ - var.desc.type() == core.VarDesc.VarType.READER: + if ( + var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH + or var.desc.type() == core.VarDesc.VarType.FETCH_LIST + or var.desc.type() == core.VarDesc.VarType.READER + ): return False return var.persistable @@ -361,8 +380,10 @@ def _serialize_persistables(program, executor): vars_ = list(filter(is_persistable, program.list_vars())) # warn if no variable found in model if len(vars_) == 0: - warnings.warn("no variable in your model, please ensure there are any " - "variables in your model to save") + warnings.warn( + "no variable in your model, please ensure there are any " + "variables in your model to save" + ) return None # create a new program and clone persitable vars to it save_program = Program() @@ -379,16 +400,16 @@ def _serialize_persistables(program, executor): in_vars.append(save_var_map[name]) out_var_name = unique_name.generate("out_var") - out_var = save_block.create_var(type=core.VarDesc.VarType.RAW, - name=out_var_name) + out_var = save_block.create_var( + type=core.VarDesc.VarType.RAW, name=out_var_name + ) out_var.desc.set_persistable(True) - save_block.append_op(type='save_combine', - inputs={'X': in_vars}, - outputs={'Y': out_var}, - attrs={ - 'file_path': '', - 'save_to_memory': True - }) + save_block.append_op( + type='save_combine', + inputs={'X': in_vars}, + outputs={'Y': out_var}, + attrs={'file_path': '', 'save_to_memory': True}, + ) # run save_program to save vars # NOTE(zhiqiu): save op will add variable kLookupTablePath to save_program.desc, # which leads to diff between save_program and its desc. Call _sync_with_cpp @@ -435,8 +456,9 @@ def save_to_file(path, content): @static_only -def save_inference_model(path_prefix, feed_vars, fetch_vars, executor, - **kwargs): +def save_inference_model( + path_prefix, feed_vars, fetch_vars, executor, **kwargs +): """ Save current model and its parameters to given path. i.e. Given path_prefix = "/path/to/modelname", after invoking @@ -516,7 +538,8 @@ def save_inference_model(path_prefix, feed_vars, fetch_vars, executor, program = normalize_program(program, feed_vars, fetch_vars) # serialize and save program program_bytes = _serialize_program( - program._remove_training_info(clip_extra=clip_extra)) + program._remove_training_info(clip_extra=clip_extra) + ) save_to_file(model_path, program_bytes) # serialize and save params params_bytes = _serialize_persistables(program, executor) @@ -566,8 +589,9 @@ def deserialize_program(data): """ program = Program.parse_from_string(data) if not core._is_program_version_supported(program._version()): - raise ValueError("Unsupported program version: %d\n" % - program._version()) + raise ValueError( + "Unsupported program version: %d\n" % program._version() + ) return program @@ -616,8 +640,9 @@ def deserialize_persistables(program, data, executor): """ if not isinstance(program, Program): raise TypeError( - "program type must be `fluid.Program`, but received `%s`" % - type(program)) + "program type must be `fluid.Program`, but received `%s`" + % type(program) + ) # load params to a tmp program load_program = Program() load_block = load_program.global_block() @@ -641,9 +666,9 @@ def deserialize_persistables(program, data, executor): load_var_map[var_copy.name] = var_copy if data is None: - assert len( - origin_shape_map - ) == 0, "Required 'data' shall be not None if program contains parameter, but received 'data' is None." + assert ( + len(origin_shape_map) == 0 + ), "Required 'data' shall be not None if program contains parameter, but received 'data' is None." return # append load_combine op to load parameters, @@ -655,10 +680,8 @@ def deserialize_persistables(program, data, executor): inputs={}, outputs={"Out": load_var_list}, # if load from memory, file_path is data - attrs={ - 'file_path': data, - 'model_from_memory': True - }) + attrs={'file_path': data, 'model_from_memory': True}, + ) executor.run(load_program) # check var shape for var in check_vars: @@ -673,7 +696,9 @@ def deserialize_persistables(program, data, executor): raise RuntimeError( "Shape mismatch, program needs a parameter with shape ({}), " "but the loaded parameter ('{}') has a shape of ({}).".format( - origin_shape, var.name, new_shape)) + origin_shape, var.name, new_shape + ) + ) def load_from_file(path): @@ -778,7 +803,7 @@ def load_inference_model(path_prefix, executor, **kwargs): """ # check kwargs supported_args = ('model_filename', 'params_filename') - deprecated_args = ('pserver_endpoints', ) + deprecated_args = ('pserver_endpoints',) caller = inspect.currentframe().f_code.co_name _check_args(caller, kwargs, supported_args, deprecated_args) @@ -789,7 +814,8 @@ def load_inference_model(path_prefix, executor, **kwargs): params_filename = kwargs.get('params_filename', None) if params_filename is None: raise ValueError( - "params_filename cannot be None when path_prefix is None.") + "params_filename cannot be None when path_prefix is None." + ) load_dirname = '' program_bytes = model_filename params_bytes = params_filename @@ -812,21 +838,26 @@ def load_inference_model(path_prefix, executor, **kwargs): if model_filename is None: model_path = os.path.join(path_prefix, "__model__") else: - model_path = os.path.join(path_prefix, - model_filename + ".pdmodel") + model_path = os.path.join( + path_prefix, model_filename + ".pdmodel" + ) if not os.path.exists(model_path): model_path = os.path.join(path_prefix, model_filename) # set params_path if params_filename is None: params_path = os.path.join(path_prefix, "") else: - params_path = os.path.join(path_prefix, - params_filename + ".pdiparams") + params_path = os.path.join( + path_prefix, params_filename + ".pdiparams" + ) if not os.path.exists(params_path): params_path = os.path.join(path_prefix, params_filename) - _logger.warning("The old way to load inference model is deprecated." - " model path: {}, params path: {}".format( - model_path, params_path)) + _logger.warning( + "The old way to load inference model is deprecated." + " model path: {}, params path: {}".format( + model_path, params_path + ) + ) program_bytes = load_from_file(model_path) load_dirname = os.path.dirname(params_path) params_filename = os.path.basename(params_path) diff --git a/python/paddle/static/nn/__init__.py b/python/paddle/static/nn/__init__.py index 41f8c99b52644..64e079ae375e3 100755 --- a/python/paddle/static/nn/__init__.py +++ b/python/paddle/static/nn/__init__.py @@ -59,7 +59,7 @@ from ...fluid.layers.sequence_lod import sequence_enumerate # noqa: F401 from ...fluid.layers.sequence_lod import sequence_reverse # noqa: F401 -__all__ = [ #noqa +__all__ = [ # noqa 'fc', 'batch_norm', 'embedding', diff --git a/python/paddle/static/nn/common.py b/python/paddle/static/nn/common.py index baeb39e8cf650..98c5e81a2e8e0 100755 --- a/python/paddle/static/nn/common.py +++ b/python/paddle/static/nn/common.py @@ -19,13 +19,15 @@ @static_only -def fc(x, - size, - num_flatten_dims=1, - weight_attr=None, - bias_attr=None, - activation=None, - name=None): +def fc( + x, + size, + num_flatten_dims=1, + weight_attr=None, + bias_attr=None, + activation=None, + name=None, +): r""" Fully-Connected layer can take a tensor or a list of tensor as its inputs. @@ -154,30 +156,34 @@ def fc(x, bias_attr=paddle.ParamAttr(initializer=paddle.nn.initializer.Constant(value=1.0))) # out: [[1.8 1.8]] """ - return paddle.fluid.layers.fc(input=x, - size=size, - num_flatten_dims=num_flatten_dims, - param_attr=weight_attr, - bias_attr=bias_attr, - act=activation, - name=name) + return paddle.fluid.layers.fc( + input=x, + size=size, + num_flatten_dims=num_flatten_dims, + param_attr=weight_attr, + bias_attr=bias_attr, + act=activation, + name=name, + ) @static_only -def deform_conv2d(x, - offset, - mask, - num_filters, - filter_size, - stride=1, - padding=0, - dilation=1, - groups=1, - deformable_groups=1, - im2col_step=1, - weight_attr=None, - bias_attr=None, - name=None): +def deform_conv2d( + x, + offset, + mask, + num_filters, + filter_size, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1, + im2col_step=1, + weight_attr=None, + bias_attr=None, + name=None, +): r""" Compute 2-D deformable convolution on 4-D input. @@ -318,7 +324,8 @@ def deform_conv2d(x, param_attr=weight_attr, bias_attr=bias_attr, modulated=False, - name=name) + name=name, + ) else: return paddle.fluid.layers.deformable_conv( input=x, @@ -335,4 +342,5 @@ def deform_conv2d(x, param_attr=weight_attr, bias_attr=bias_attr, modulated=True, - name=name) + name=name, + ) diff --git a/python/paddle/static/quantization/__init__.py b/python/paddle/static/quantization/__init__.py index 325fee5624bff..3a1a7549f8302 100644 --- a/python/paddle/static/quantization/__init__.py +++ b/python/paddle/static/quantization/__init__.py @@ -12,20 +12,50 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.contrib.slim.quantization.quantization_pass import QuantizationTransformPass -from ...fluid.contrib.slim.quantization.quantization_pass import QuantizationFreezePass -from ...fluid.contrib.slim.quantization.quantization_pass import ConvertToInt8Pass -from ...fluid.contrib.slim.quantization.quantization_pass import TransformForMobilePass -from ...fluid.contrib.slim.quantization.quantization_pass import OutScaleForTrainingPass -from ...fluid.contrib.slim.quantization.quantization_pass import OutScaleForInferencePass -from ...fluid.contrib.slim.quantization.quantization_pass import AddQuantDequantPass -from ...fluid.contrib.slim.quantization.quantization_pass import ReplaceFakeQuantDequantPass +from ...fluid.contrib.slim.quantization.quantization_pass import ( + QuantizationTransformPass, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + QuantizationFreezePass, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + ConvertToInt8Pass, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + TransformForMobilePass, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + OutScaleForTrainingPass, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + OutScaleForInferencePass, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + AddQuantDequantPass, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + ReplaceFakeQuantDequantPass, +) from ...fluid.contrib.slim.quantization.quantization_pass import QuantWeightPass -from ...fluid.contrib.slim.quantization.quantization_pass import QuantizationTransformPassV2 -from ...fluid.contrib.slim.quantization.quantization_pass import AddQuantDequantPassV2 -from ...fluid.contrib.slim.quantization.quant_int8_mkldnn_pass import QuantInt8MkldnnPass -from ...fluid.contrib.slim.quantization.quant2_int8_mkldnn_pass import Quant2Int8MkldnnPass +from ...fluid.contrib.slim.quantization.quantization_pass import ( + QuantizationTransformPassV2, +) +from ...fluid.contrib.slim.quantization.quantization_pass import ( + AddQuantDequantPassV2, +) +from ...fluid.contrib.slim.quantization.quant_int8_mkldnn_pass import ( + QuantInt8MkldnnPass, +) +from ...fluid.contrib.slim.quantization.quant2_int8_mkldnn_pass import ( + Quant2Int8MkldnnPass, +) -from ...fluid.contrib.slim.quantization.post_training_quantization import PostTrainingQuantization -from ...fluid.contrib.slim.quantization.post_training_quantization import PostTrainingQuantizationProgram -from ...fluid.contrib.slim.quantization.post_training_quantization import WeightQuantization +from ...fluid.contrib.slim.quantization.post_training_quantization import ( + PostTrainingQuantization, +) +from ...fluid.contrib.slim.quantization.post_training_quantization import ( + PostTrainingQuantizationProgram, +) +from ...fluid.contrib.slim.quantization.post_training_quantization import ( + WeightQuantization, +) diff --git a/python/paddle/static/sparsity/__init__.py b/python/paddle/static/sparsity/__init__.py index 8d3166b19a099..37b48132bda90 100644 --- a/python/paddle/static/sparsity/__init__.py +++ b/python/paddle/static/sparsity/__init__.py @@ -13,20 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ...fluid.contrib.sparsity import calculate_density #noqa: F401 -from ...fluid.contrib.sparsity import decorate #noqa: F401 -from ...fluid.contrib.sparsity import prune_model #noqa: F401 -from ...fluid.contrib.sparsity import reset_excluded_layers #noqa: F401 -from ...fluid.contrib.sparsity import add_supported_layer #noqa: F401 -from ...fluid.contrib import sparsity #noqa: F401 +from ...fluid.contrib.sparsity import calculate_density # noqa: F401 +from ...fluid.contrib.sparsity import decorate # noqa: F401 +from ...fluid.contrib.sparsity import prune_model # noqa: F401 +from ...fluid.contrib.sparsity import reset_excluded_layers # noqa: F401 +from ...fluid.contrib.sparsity import add_supported_layer # noqa: F401 +from ...fluid.contrib import sparsity # noqa: F401 def set_excluded_layers(main_program, param_names): - sparsity.set_excluded_layers(param_names=param_names, - main_program=main_program) + sparsity.set_excluded_layers( + param_names=param_names, main_program=main_program + ) -__all__ = [ #noqa - 'calculate_density', 'decorate', 'prune_model', 'set_excluded_layers', - 'reset_excluded_layers', 'add_supported_layer' +__all__ = [ # noqa + 'calculate_density', + 'decorate', + 'prune_model', + 'set_excluded_layers', + 'reset_excluded_layers', + 'add_supported_layer', ] diff --git a/python/paddle/sysconfig.py b/python/paddle/sysconfig.py index 2ce327c76961a..9a839f199a71d 100644 --- a/python/paddle/sysconfig.py +++ b/python/paddle/sysconfig.py @@ -31,6 +31,7 @@ def get_include(): """ import paddle + return os.path.join(os.path.dirname(paddle.__file__), 'include') @@ -48,4 +49,5 @@ def get_lib(): """ import paddle + return os.path.join(os.path.dirname(paddle.__file__), 'libs') diff --git a/python/paddle/tensor/array.py b/python/paddle/tensor/array.py index 527cf692525ac..382c6f524b4c5 100644 --- a/python/paddle/tensor/array.py +++ b/python/paddle/tensor/array.py @@ -48,22 +48,24 @@ def array_length(array): """ if _non_static_mode(): assert isinstance( - array, - list), "The 'array' in array_write must be a list in dygraph mode" + array, list + ), "The 'array' in array_write must be a list in dygraph mode" return len(array) - if not isinstance( - array, - Variable) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + not isinstance(array, Variable) + or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): raise TypeError( - "array should be tensor array vairable in array_length Op") + "array should be tensor array vairable in array_length Op" + ) helper = LayerHelper('array_length', **locals()) tmp = helper.create_variable_for_type_inference(dtype='int64') tmp.stop_gradient = True - helper.append_op(type='lod_array_length', - inputs={'X': [array]}, - outputs={'Out': [tmp]}) + helper.append_op( + type='lod_array_length', inputs={'X': [array]}, outputs={'Out': [tmp]} + ) return tmp @@ -108,8 +110,8 @@ def array_read(array, i): """ if _non_static_mode(): assert isinstance( - array, - list), "The 'array' in array_read must be list in dygraph mode" + array, list + ), "The 'array' in array_read must be list in dygraph mode" assert isinstance( i, Variable ), "The index 'i' in array_read must be Variable in dygraph mode" @@ -121,17 +123,17 @@ def array_read(array, i): check_variable_and_dtype(i, 'i', ['int64'], 'array_read') helper = LayerHelper('array_read', **locals()) - if not isinstance( - array, - Variable) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + not isinstance(array, Variable) + or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): raise TypeError("array should be tensor array vairable") out = helper.create_variable_for_type_inference(dtype=array.dtype) - helper.append_op(type='read_from_array', - inputs={ - 'X': [array], - 'I': [i] - }, - outputs={'Out': [out]}) + helper.append_op( + type='read_from_array', + inputs={'X': [array], 'I': [i]}, + outputs={'Out': [out]}, + ) return out @@ -180,8 +182,8 @@ def array_write(x, i, array=None): if array is None: array = create_array(x.dtype) assert isinstance( - array, - list), "The 'array' in array_write must be a list in dygraph mode" + array, list + ), "The 'array' in array_write must be a list in dygraph mode" assert i <= len( array ), "The index 'i' should not be greater than the length of 'array' in dygraph mode" @@ -195,22 +197,24 @@ def array_write(x, i, array=None): check_type(x, 'x', (Variable), 'array_write') helper = LayerHelper('array_write', **locals()) if array is not None: - if not isinstance( - array, Variable - ) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + not isinstance(array, Variable) + or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): raise TypeError( - "array should be tensor array vairable in array_write Op") + "array should be tensor array vairable in array_write Op" + ) if array is None: array = helper.create_variable( name="{0}.out".format(helper.name), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=x.dtype) - helper.append_op(type='write_to_array', - inputs={ - 'X': [x], - 'I': [i] - }, - outputs={'Out': [array]}) + dtype=x.dtype, + ) + helper.append_op( + type='write_to_array', + inputs={'X': [x], 'I': [i]}, + outputs={'Out': [array]}, + ) return array @@ -247,16 +251,20 @@ def create_array(dtype, initialized_list=None): if initialized_list is not None: if not isinstance(initialized_list, (list, tuple)): raise TypeError( - "Require type(initialized_list) should be list/tuple, but received {}" - .format(type(initialized_list))) + "Require type(initialized_list) should be list/tuple, but received {}".format( + type(initialized_list) + ) + ) array = list(initialized_list) # NOTE: Only support plain list like [x, y,...], not support nested list in static mode. for val in array: if not isinstance(val, Variable): raise TypeError( - "All values in `initialized_list` should be Variable, but recevied {}." - .format(type(val))) + "All values in `initialized_list` should be Variable, but recevied {}.".format( + type(val) + ) + ) if _non_static_mode(): return array @@ -265,7 +273,8 @@ def create_array(dtype, initialized_list=None): tensor_array = helper.create_variable( name="{0}.out".format(helper.name), type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, - dtype=dtype) + dtype=dtype, + ) for val in array: array_write(x=val, i=array_length(tensor_array), array=tensor_array) diff --git a/python/paddle/tensor/attribute.py b/python/paddle/tensor/attribute.py index d68eebf1364b6..e0db6a6e8b413 100644 --- a/python/paddle/tensor/attribute.py +++ b/python/paddle/tensor/attribute.py @@ -115,16 +115,29 @@ def shape(input): out.stop_gradient = True return out - check_variable_and_dtype(input, 'input', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], 'shape') + check_variable_and_dtype( + input, + 'input', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'shape', + ) helper = LayerHelper('shape', **locals()) out = helper.create_variable_for_type_inference(dtype='int32') - helper.append_op(type='shape', - inputs={'Input': input}, - outputs={'Out': out}, - stop_gradient=True) + helper.append_op( + type='shape', + inputs={'Input': input}, + outputs={'Out': out}, + stop_gradient=True, + ) return out @@ -156,11 +169,14 @@ def is_complex(x): # False """ if not isinstance(x, (paddle.Tensor, paddle.static.Variable)): - raise TypeError("Expected Tensor, but received type of x: {}".format( - type(x))) + raise TypeError( + "Expected Tensor, but received type of x: {}".format(type(x)) + ) dtype = x.dtype - is_complex_dtype = (dtype == core.VarDesc.VarType.COMPLEX64 - or dtype == core.VarDesc.VarType.COMPLEX128) + is_complex_dtype = ( + dtype == core.VarDesc.VarType.COMPLEX64 + or dtype == core.VarDesc.VarType.COMPLEX128 + ) return is_complex_dtype @@ -187,13 +203,16 @@ def is_floating_point(x): # False """ if not isinstance(x, (paddle.Tensor, paddle.static.Variable)): - raise TypeError("Expected Tensor, but received type of x: {}".format( - type(x))) + raise TypeError( + "Expected Tensor, but received type of x: {}".format(type(x)) + ) dtype = x.dtype - is_fp_dtype = (dtype == core.VarDesc.VarType.FP32 - or dtype == core.VarDesc.VarType.FP64 - or dtype == core.VarDesc.VarType.FP16 - or dtype == core.VarDesc.VarType.BF16) + is_fp_dtype = ( + dtype == core.VarDesc.VarType.FP32 + or dtype == core.VarDesc.VarType.FP64 + or dtype == core.VarDesc.VarType.FP16 + or dtype == core.VarDesc.VarType.BF16 + ) return is_fp_dtype @@ -224,14 +243,17 @@ def is_integer(x): # True """ if not isinstance(x, (paddle.Tensor, paddle.static.Variable)): - raise TypeError("Expected Tensor, but received type of x: {}".format( - type(x))) + raise TypeError( + "Expected Tensor, but received type of x: {}".format(type(x)) + ) dtype = x.dtype - is_int_dtype = (dtype == core.VarDesc.VarType.UINT8 - or dtype == core.VarDesc.VarType.INT8 - or dtype == core.VarDesc.VarType.INT16 - or dtype == core.VarDesc.VarType.INT32 - or dtype == core.VarDesc.VarType.INT64) + is_int_dtype = ( + dtype == core.VarDesc.VarType.UINT8 + or dtype == core.VarDesc.VarType.INT8 + or dtype == core.VarDesc.VarType.INT16 + or dtype == core.VarDesc.VarType.INT32 + or dtype == core.VarDesc.VarType.INT64 + ) return is_int_dtype @@ -276,7 +298,8 @@ def real(x, name=None): check_variable_and_dtype(x, 'x', ['complex64', 'complex128'], 'real') helper = LayerHelper('real', **locals()) out = helper.create_variable_for_type_inference( - dtype=_complex_to_real_dtype(helper.input_dtype())) + dtype=_complex_to_real_dtype(helper.input_dtype()) + ) helper.append_op(type='real', inputs={'X': x}, outputs={'Out': out}) return out @@ -322,6 +345,7 @@ def imag(x, name=None): check_variable_and_dtype(x, 'x', ['complex64', 'complex128'], 'imag') helper = LayerHelper('imag', **locals()) out = helper.create_variable_for_type_inference( - dtype=_complex_to_real_dtype(helper.input_dtype())) + dtype=_complex_to_real_dtype(helper.input_dtype()) + ) helper.append_op(type='imag', inputs={'X': x}, outputs={'Out': out}) return out diff --git a/python/paddle/tensor/creation.py b/python/paddle/tensor/creation.py index 675c186d47fc3..8f44f3ffe162e 100644 --- a/python/paddle/tensor/creation.py +++ b/python/paddle/tensor/creation.py @@ -22,12 +22,21 @@ from ..framework import core from ..framework import in_dygraph_mode, _non_static_mode from ..framework import LayerHelper -from ..fluid.data_feeder import check_variable_and_dtype, check_type, check_dtype, convert_dtype +from ..fluid.data_feeder import ( + check_variable_and_dtype, + check_type, + check_dtype, + convert_dtype, +) from ..framework import convert_np_dtype_to_dtype_ + # TODO: define functions to get create a tensor import paddle from paddle import _C_ops, _legacy_C_ops -from ..fluid.framework import _in_legacy_dygraph, _in_eager_without_dygraph_check +from ..fluid.framework import ( + _in_legacy_dygraph, + _in_eager_without_dygraph_check, +) import warnings __all__ = [] @@ -98,11 +107,17 @@ def linspace(start, stop, num, dtype=None, name=None): with device_guard("cpu"): tensor_num = fill_constant([1], 'int32', num, force_cpu=True) if in_dygraph_mode(): - return _C_ops.linspace(tensor_start, tensor_stop, tensor_num, dtype, - _current_expected_place()) + return _C_ops.linspace( + tensor_start, + tensor_stop, + tensor_num, + dtype, + _current_expected_place(), + ) if _in_legacy_dygraph(): - return _legacy_C_ops.linspace(tensor_start, tensor_stop, tensor_num, - 'dtype', dtype) + return _legacy_C_ops.linspace( + tensor_start, tensor_stop, tensor_num, 'dtype', dtype + ) helper = LayerHelper("linspace", **locals()) @@ -110,41 +125,53 @@ def linspace(start, stop, num, dtype=None, name=None): stop_dtype = convert_dtype(tensor_stop.dtype) out_dtype = convert_dtype(dtype) if isinstance(start, Variable): - check_dtype(start.dtype, 'start', - ['float32', 'float64', 'int32', 'int64'], 'linspace') + check_dtype( + start.dtype, + 'start', + ['float32', 'float64', 'int32', 'int64'], + 'linspace', + ) else: check_type(start, 'start', (int, float), 'linspace') if isinstance(stop, Variable): - check_dtype(stop.dtype, 'stop', - ['float32', 'float64', 'int32', 'int64'], 'linspace') + check_dtype( + stop.dtype, + 'stop', + ['float32', 'float64', 'int32', 'int64'], + 'linspace', + ) else: check_type(stop, 'stop', (int, float), 'linspace') if isinstance(num, Variable): check_dtype(num.dtype, 'num', ['int32'], 'linspace') - check_dtype(dtype, 'dtype', ['int32', 'int64', 'float32', 'float64'], - 'linspace') - if ((stop_dtype == "float64" or start_dtype == "float64") - and out_dtype in ["float32", "int32"]) or ( - (stop_dtype == "int64" or start_dtype == "int64") - and out_dtype == "int32"): + check_dtype( + dtype, 'dtype', ['int32', 'int64', 'float32', 'float64'], 'linspace' + ) + if ( + (stop_dtype == "float64" or start_dtype == "float64") + and out_dtype in ["float32", "int32"] + ) or ( + (stop_dtype == "int64" or start_dtype == "int64") + and out_dtype == "int32" + ): raise ValueError( "The dtype of start/stop is {}/{} but the attr(dtype) of linspace is {}, " - "which may cause data type overflows. Please reset attr(dtype) of linspace." - .format(start_dtype, stop_dtype, dtype)) + "which may cause data type overflows. Please reset attr(dtype) of linspace.".format( + start_dtype, stop_dtype, dtype + ) + ) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='linspace', - inputs={ - 'Start': tensor_start, - 'Stop': tensor_stop, - 'Num': tensor_num - }, - attrs={'dtype': dtype}, - outputs={'Out': [out]}) + helper.append_op( + type='linspace', + inputs={'Start': tensor_start, 'Stop': tensor_stop, 'Num': tensor_num}, + attrs={'dtype': dtype}, + outputs={'Out': [out]}, + ) if isinstance(num, int): - out.desc.set_shape((num, )) + out.desc.set_shape((num,)) return out @@ -210,8 +237,9 @@ def logspace(start, stop, num, base=10.0, dtype=None, name=None): with device_guard("cpu"): tensor_base = fill_constant([1], dtype, base) if _non_static_mode(): - return _legacy_C_ops.logspace(tensor_start, tensor_stop, tensor_num, - tensor_base, 'dtype', dtype) + return _legacy_C_ops.logspace( + tensor_start, tensor_stop, tensor_num, tensor_base, 'dtype', dtype + ) helper = LayerHelper("logspace", **locals()) @@ -220,14 +248,22 @@ def logspace(start, stop, num, base=10.0, dtype=None, name=None): base_dtype = convert_dtype(tensor_base.dtype) out_dtype = convert_dtype(dtype) if isinstance(start, Variable): - check_dtype(start.dtype, 'start', - ['float32', 'float64', 'int32', 'int64'], 'logspace') + check_dtype( + start.dtype, + 'start', + ['float32', 'float64', 'int32', 'int64'], + 'logspace', + ) else: check_type(start, 'start', (int, float), 'logspace') if isinstance(stop, Variable): - check_dtype(stop.dtype, 'stop', - ['float32', 'float64', 'int32', 'int64'], 'logspace') + check_dtype( + stop.dtype, + 'stop', + ['float32', 'float64', 'int32', 'int64'], + 'logspace', + ) else: check_type(stop, 'stop', (int, float), 'logspace') @@ -235,37 +271,55 @@ def logspace(start, stop, num, base=10.0, dtype=None, name=None): check_dtype(num.dtype, 'num', ['int32'], 'logspace') if isinstance(base, Variable): - check_dtype(base.dtype, 'base', - ['float32', 'float64', 'int32', 'int64'], 'logspace') + check_dtype( + base.dtype, + 'base', + ['float32', 'float64', 'int32', 'int64'], + 'logspace', + ) else: check_type(base, 'base', (int, float), 'logspace') - check_dtype(dtype, 'dtype', ['int32', 'int64', 'float32', 'float64'], - 'logspace') - if ((stop_dtype == "float64" or start_dtype == "float64" - or base_dtype == "float64") - and out_dtype in ["float32", "int32"]) or \ - ((stop_dtype == "int64" or start_dtype == "int64" - or base_dtype == "int64") - and out_dtype == "int32"): + check_dtype( + dtype, 'dtype', ['int32', 'int64', 'float32', 'float64'], 'logspace' + ) + if ( + ( + stop_dtype == "float64" + or start_dtype == "float64" + or base_dtype == "float64" + ) + and out_dtype in ["float32", "int32"] + ) or ( + ( + stop_dtype == "int64" + or start_dtype == "int64" + or base_dtype == "int64" + ) + and out_dtype == "int32" + ): raise ValueError( "The dtype of start/stop/base is {}/{}/{} but the attr(dtype) of logspace is {}, " - "which may cause data type overflows. Please reset attr(dtype) of logspace." - .format(start_dtype, stop_dtype, base_dtype, dtype)) + "which may cause data type overflows. Please reset attr(dtype) of logspace.".format( + start_dtype, stop_dtype, base_dtype, dtype + ) + ) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='logspace', - inputs={ - 'Start': tensor_start, - 'Stop': tensor_stop, - 'Num': tensor_num, - 'Base': tensor_base - }, - attrs={'dtype': dtype}, - outputs={'Out': [out]}) + helper.append_op( + type='logspace', + inputs={ + 'Start': tensor_start, + 'Stop': tensor_stop, + 'Num': tensor_num, + 'Base': tensor_base, + }, + attrs={'dtype': dtype}, + outputs={'Out': [out]}, + ) if isinstance(num, int): - out.desc.set_shape((num, )) + out.desc.set_shape((num,)) return out @@ -313,17 +367,25 @@ def _handle_dtype(data, dtype): return data else: raise TypeError( - "Can't constructs a 'paddle.Tensor' with data type {}, data type must be scalar|list|tuple|np.ndarray|paddle.Tensor" - .format(type(data))) + "Can't constructs a 'paddle.Tensor' with data type {}, data type must be scalar|list|tuple|np.ndarray|paddle.Tensor".format( + type(data) + ) + ) if not dtype: if data.dtype in [ - 'float16', 'float32', 'float64', 'complex64', 'complex128' + 'float16', + 'float32', + 'float64', + 'complex64', + 'complex128', ]: default_type = paddle.get_default_dtype() if np.iscomplexobj(data): - default_type = 'complex64' if default_type in [ - 'float16', 'float32' - ] else 'complex128' + default_type = ( + 'complex64' + if default_type in ['float16', 'float32'] + else 'complex128' + ) data = data.astype(default_type) # Windows default type is 'int32', while Linux/Mac is 'int64'. Unify they. if data.dtype in ['int32']: @@ -334,18 +396,22 @@ def _handle_dtype(data, dtype): data = data.astype(convert_dtype(dtype)) if _in_eager_without_dygraph_check() and isinstance(data, np.ndarray): - return core.eager.Tensor(value=data, - place=place, - persistable=False, - zero_copy=False, - name=None, - stop_gradient=stop_gradient) + return core.eager.Tensor( + value=data, + place=place, + persistable=False, + zero_copy=False, + name=None, + stop_gradient=stop_gradient, + ) else: - return paddle.Tensor(value=data, - place=place, - persistable=False, - zero_copy=False, - stop_gradient=stop_gradient) + return paddle.Tensor( + value=data, + place=place, + persistable=False, + zero_copy=False, + stop_gradient=stop_gradient, + ) def _to_tensor_static(data, dtype=None, stop_gradient=None): @@ -360,8 +426,11 @@ def _to_tensor_static(data, dtype=None, stop_gradient=None): elif isinstance(data, (list, tuple)): data = np.array(data) - if isinstance(data, - np.ndarray) and not dtype and data.dtype != 'object': + if ( + isinstance(data, np.ndarray) + and not dtype + and data.dtype != 'object' + ): if data.dtype in ['float16', 'float32', 'float64']: data = data.astype(paddle.get_default_dtype()) elif data.dtype in ['int32']: @@ -376,10 +445,14 @@ def _to_tensor_static(data, dtype=None, stop_gradient=None): target_dtype = convert_dtype(target_dtype) - if isinstance(data, np.ndarray) and len(data.shape) > 0 and any( - isinstance(x, Variable) for x in data): + if ( + isinstance(data, np.ndarray) + and len(data.shape) > 0 + and any(isinstance(x, Variable) for x in data) + ): if not all( - [x.shape == (1, ) for x in data if isinstance(x, Variable)]): + [x.shape == (1,) for x in data if isinstance(x, Variable)] + ): raise TypeError( "Unsupport paddle.to_tensor([Variable, Variable...]) with non-scalar variable." ) @@ -513,27 +586,31 @@ def full_like(x, fill_value, dtype=None, name=None): return _C_ops.full_like(x, fill_value, dtype, x.place) if _in_legacy_dygraph(): - return _legacy_C_ops.fill_any_like(x, 'value', fill_value, 'dtype', - dtype) + return _legacy_C_ops.fill_any_like( + x, 'value', fill_value, 'dtype', dtype + ) helper = LayerHelper("full_like", **locals()) check_variable_and_dtype( - x, 'x', + x, + 'x', ['bool', 'float16', 'float32', 'float64', 'int16', 'int32', 'int64'], - 'full_like') + 'full_like', + ) check_dtype( - dtype, 'dtype', + dtype, + 'dtype', ['bool', 'float16', 'float32', 'float64', 'int16', 'int32', 'int64'], - 'full_like/zeros_like/ones_like') + 'full_like/zeros_like/ones_like', + ) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='fill_any_like', - inputs={'X': [x]}, - attrs={ - 'value': fill_value, - "dtype": dtype - }, - outputs={'Out': [out]}) + helper.append_op( + type='fill_any_like', + inputs={'X': [x]}, + attrs={'value': fill_value, "dtype": dtype}, + outputs={'Out': [out]}, + ) out.stop_gradient = True return out @@ -728,26 +805,34 @@ def _check_attr(attr, message): if _non_static_mode(): if in_dygraph_mode(): - out = _C_ops.eye(num_rows, num_columns, dtype, - _current_expected_place()) + out = _C_ops.eye( + num_rows, num_columns, dtype, _current_expected_place() + ) elif _in_legacy_dygraph(): - out = _legacy_C_ops.eye('dtype', dtype, 'num_rows', num_rows, - 'num_columns', num_columns) + out = _legacy_C_ops.eye( + 'dtype', dtype, 'num_rows', num_rows, 'num_columns', num_columns + ) else: helper = LayerHelper("eye", **locals()) - check_dtype(dtype, 'dtype', - ['float16', 'float32', 'float64', 'int32', 'int64'], 'eye') + check_dtype( + dtype, + 'dtype', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'eye', + ) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='eye', - inputs={}, - outputs={'Out': [out]}, - attrs={ - 'num_rows': num_rows, - 'num_columns': num_columns, - 'dtype': dtype - }, - stop_gradient=True) + helper.append_op( + type='eye', + inputs={}, + outputs={'Out': [out]}, + attrs={ + 'num_rows': num_rows, + 'num_columns': num_columns, + 'dtype': dtype, + }, + stop_gradient=True, + ) out.stop_gradient = True return out @@ -866,8 +951,11 @@ def arange(start=0, end=None, step=1, dtype=None, name=None): start = 0 out_shape = None - if not isinstance(start, Variable) and not isinstance( - end, Variable) and not isinstance(step, Variable): + if ( + not isinstance(start, Variable) + and not isinstance(end, Variable) + and not isinstance(step, Variable) + ): out_shape = [int(math.ceil((end - start) / step))] if not isinstance(dtype, core.VarDesc.VarType): @@ -899,17 +987,16 @@ def arange(start=0, end=None, step=1, dtype=None, name=None): out.stop_gradient = True return out - check_dtype(dtype, 'dtype', ['float32', 'float64', 'int32', 'int64'], - 'range/arange') + check_dtype( + dtype, 'dtype', ['float32', 'float64', 'int32', 'int64'], 'range/arange' + ) helper = LayerHelper('range', **locals()) out = helper.create_variable_for_type_inference(dtype, shape=out_shape) - helper.append_op(type='range', - inputs={ - 'Start': start, - 'End': end, - 'Step': step - }, - outputs={'Out': out}) + helper.append_op( + type='range', + inputs={'Start': start, 'End': end, 'Step': step}, + outputs={'Out': out}, + ) out.stop_gradient = True if out_shape is not None: out.desc.set_shape(out_shape) @@ -917,28 +1004,30 @@ def arange(start=0, end=None, step=1, dtype=None, name=None): def _tril_triu_op(helper): - """Base op of tril_op and triu_op - """ + """Base op of tril_op and triu_op""" op_type = helper.layer_type x = helper.kwargs.get('x', None) assert x is not None, 'x cannot be None in {}'.format(op_type) check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], - op_type) + x, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], + op_type, + ) if len(x.shape) < 2: raise ValueError("x shape in {} must be at least 2-D".format(op_type)) diagonal = helper.kwargs.get('diagonal', 0) - if not isinstance(diagonal, (int, )): + if not isinstance(diagonal, (int,)): raise TypeError("diagonal in {} must be a python Int".format(op_type)) name = helper.kwargs.get('name', None) if name is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) else: - out = helper.create_variable(name=name, - dtype=x.dtype, - persistable=False) + out = helper.create_variable( + name=name, dtype=x.dtype, persistable=False + ) helper.append_op( type="tril_triu", @@ -1131,18 +1220,21 @@ def meshgrid(*args, **kwargs): raise TypeError("The type of input args in meshgrid should be list.") for id, input_ in enumerate(args): - check_dtype(input_.dtype, 'create data type', - ['float16', 'float32', 'float64', 'int32', 'int64'], - 'meshgrid') + check_dtype( + input_.dtype, + 'create data type', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'meshgrid', + ) num = len(args) out = [ helper.create_variable_for_type_inference(dtype=args[i].dtype) for i in range(num) ] - helper.append_op(type='meshgrid', - inputs={'X': list(args)}, - outputs={'Out': out}) + helper.append_op( + type='meshgrid', inputs={'X': list(args)}, outputs={'Out': out} + ) return out @@ -1242,17 +1334,21 @@ def diagflat(x, offset=0, name=None): if _in_legacy_dygraph(): if len(x.shape) == 1: - return _legacy_C_ops.diag_v2(x, "offset", offset, "padding_value", - padding_value) + return _legacy_C_ops.diag_v2( + x, "offset", offset, "padding_value", padding_value + ) else: y, _ = _legacy_C_ops.flatten_contiguous_range( - x, "start_axis", 0, "stop_axis", -1) - return _legacy_C_ops.diag_v2(y, "offset", offset, "padding_value", - padding_value) + x, "start_axis", 0, "stop_axis", -1 + ) + return _legacy_C_ops.diag_v2( + y, "offset", offset, "padding_value", padding_value + ) check_type(x, 'x', (Variable), 'diagflat') - check_dtype(x.dtype, 'x', ['float32', 'float64', 'int32', 'int64'], - 'diagflat') + check_dtype( + x.dtype, 'x', ['float32', 'float64', 'int32', 'int64'], 'diagflat' + ) check_type(offset, 'offset', (int), 'diagflat') helper = LayerHelper("diagflat", **locals()) @@ -1261,33 +1357,27 @@ def diagflat(x, offset=0, name=None): out2 = helper.create_variable_for_type_inference(dtype=x.dtype) if len(x.shape) == 1: - helper.append_op(type='diag_v2', - inputs={'X': x}, - outputs={'Out': out2}, - attrs={ - 'offset': offset, - 'padding_value': padding_value - }) + helper.append_op( + type='diag_v2', + inputs={'X': x}, + outputs={'Out': out2}, + attrs={'offset': offset, 'padding_value': padding_value}, + ) else: - helper.append_op(type='flatten_contiguous_range', - inputs={'X': x}, - outputs={ - 'Out': out1, - 'XShape': out1_shape - }, - attrs={ - 'start_axis': 0, - 'stop_axis': -1 - }) + helper.append_op( + type='flatten_contiguous_range', + inputs={'X': x}, + outputs={'Out': out1, 'XShape': out1_shape}, + attrs={'start_axis': 0, 'stop_axis': -1}, + ) out1.stop_gradient = True - helper.append_op(type='diag_v2', - inputs={'X': out1}, - outputs={'Out': out2}, - attrs={ - 'offset': offset, - 'padding_value': padding_value - }) + helper.append_op( + type='diag_v2', + inputs={'X': out1}, + outputs={'Out': out2}, + attrs={'offset': offset, 'padding_value': padding_value}, + ) out2.stop_gradient = True return out2 @@ -1371,30 +1461,36 @@ def diag(x, offset=0, padding_value=0, name=None): return _C_ops.diag(x, offset, padding_value) else: if _in_legacy_dygraph(): - return _legacy_C_ops.diag_v2(x, "offset", offset, "padding_value", - padding_value) + return _legacy_C_ops.diag_v2( + x, "offset", offset, "padding_value", padding_value + ) else: check_type(x, 'x', (Variable), 'diag_v2') - check_dtype(x.dtype, 'x', ['float32', 'float64', 'int32', 'int64'], - 'diag_v2') + check_dtype( + x.dtype, + 'x', + ['float32', 'float64', 'int32', 'int64'], + 'diag_v2', + ) check_type(offset, 'offset', (int), 'diag_v2') check_type(padding_value, 'padding_value', (int, float), 'diag_v2') if len(x.shape) != 1 and len(x.shape) != 2: raise ValueError( - "The dimension of input x must be either 1 or 2, but received {}" - .format(len(x.shape))) + "The dimension of input x must be either 1 or 2, but received {}".format( + len(x.shape) + ) + ) helper = LayerHelper("diag_v2", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='diag_v2', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'offset': offset, - 'padding_value': padding_value - }) + helper.append_op( + type='diag_v2', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'offset': offset, 'padding_value': padding_value}, + ) out.stop_gradient = True return out @@ -1456,42 +1552,48 @@ def empty(shape, dtype=None, name=None): if in_dygraph_mode(): shape = utils.convert_shape_to_list(shape) - out = _C_ops.empty(shape, convert_np_dtype_to_dtype_(dtype), - _current_expected_place()) + out = _C_ops.empty( + shape, convert_np_dtype_to_dtype_(dtype), _current_expected_place() + ) out.stop_gradient = True return out if _in_legacy_dygraph(): shape = utils.convert_shape_to_list(shape) - out = _legacy_C_ops.empty('shape', shape, 'dtype', - convert_np_dtype_to_dtype_(dtype)) + out = _legacy_C_ops.empty( + 'shape', shape, 'dtype', convert_np_dtype_to_dtype_(dtype) + ) out.stop_gradient = True return out helper = LayerHelper("empty", **locals()) inputs = {} - check_dtype(dtype, 'dtype', - ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'empty') + check_dtype( + dtype, + 'dtype', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'empty', + ) check_type(shape, 'shape', (Variable, list, tuple), 'empty') if isinstance(shape, Variable): check_dtype(shape.dtype, 'shape', ['int32', 'int64'], 'empty') attrs = {} - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type='empty') + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type='empty' + ) out = helper.create_variable_for_type_inference(dtype=dtype) attrs['dtype'] = convert_np_dtype_to_dtype_(dtype) - helper.append_op(type='empty', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs, - stop_gradient=True) + helper.append_op( + type='empty', + inputs=inputs, + outputs={'Out': [out]}, + attrs=attrs, + stop_gradient=True, + ) out.stop_gradient = True return out @@ -1529,40 +1631,51 @@ def empty_like(x, dtype=None, name=None): dtype = convert_dtype(dtype) if in_dygraph_mode(): - out = _C_ops.empty(x.shape, convert_np_dtype_to_dtype_(dtype), - _current_expected_place()) + out = _C_ops.empty( + x.shape, + convert_np_dtype_to_dtype_(dtype), + _current_expected_place(), + ) out.stop_gradient = True return out if _in_legacy_dygraph(): - out = _legacy_C_ops.empty('shape', x.shape, 'dtype', - convert_np_dtype_to_dtype_(dtype)) + out = _legacy_C_ops.empty( + 'shape', x.shape, 'dtype', convert_np_dtype_to_dtype_(dtype) + ) out.stop_gradient = True return out helper = LayerHelper("empty_like", **locals()) check_variable_and_dtype( - x, 'x', ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'empty_like') - check_dtype(dtype, 'dtype', - ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'empty_like') + x, + 'x', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'empty_like', + ) + check_dtype( + dtype, + 'dtype', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'empty_like', + ) out = helper.create_variable_for_type_inference(dtype=dtype) inputs = {} attrs = {} attrs['dtype'] = convert_np_dtype_to_dtype_(dtype) shape = paddle.shape(x) - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type='empty_like') - - helper.append_op(type='empty', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs, - stop_gradient=True) + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type='empty_like' + ) + + helper.append_op( + type='empty', + inputs=inputs, + outputs={'Out': [out]}, + attrs=attrs, + stop_gradient=True, + ) out.stop_gradient = True return out @@ -1597,8 +1710,12 @@ def assign(x, output=None): """ input = x helper = LayerHelper('assign', **locals()) - check_type(input, 'input', - (Variable, np.ndarray, list, tuple, float, int, bool), 'assign') + check_type( + input, + 'input', + (Variable, np.ndarray, list, tuple, float, int, bool), + 'assign', + ) is_inplace = True if output is not None else False if np.isscalar(input) and not isinstance(input, str): @@ -1621,24 +1738,40 @@ def assign(x, output=None): output = core.VarBase() _legacy_C_ops.assign(input, output) else: - check_dtype(input.dtype, 'input', [ - 'float16', 'uint16', 'float32', 'float64', 'int32', 'int64', - 'uint8', 'bool' - ], 'assign', '(When the type of input in assign is Variable.)') + check_dtype( + input.dtype, + 'input', + [ + 'float16', + 'uint16', + 'float32', + 'float64', + 'int32', + 'int64', + 'uint8', + 'bool', + ], + 'assign', + '(When the type of input in assign is Variable.)', + ) if output is None: output = helper.create_variable_for_type_inference( - dtype=input.dtype) - helper.append_op(type='assign', - inputs={'X': [input]}, - outputs={'Out': [output]}) + dtype=input.dtype + ) + helper.append_op( + type='assign', inputs={'X': [input]}, outputs={'Out': [output]} + ) elif isinstance(input, np.ndarray): # We now support the form of [var, VAR...] if the Var.shape=[1,] if len(input.shape) > 0 and any(isinstance(x, Variable) for x in input): # We only deal with the case where the list is nested one level, convert all scalars into variables, and then use stack to process. It is necessary to ensure the consistency of types. - if not all([ - x.shape == (1, ) for x in input + if not all( + [ + x.shape == (1,) + for x in input if isinstance(x, (Variable, core.eager.Tensor)) - ]): + ] + ): raise TypeError( "Unsupport paddle.assign([Variable, Variable...]) with non-scalar variable." ) @@ -1654,8 +1787,7 @@ def convert_scalar(x): return ret if input.dtype == 'object': - """ may be this form [[Var], [Var], [3], [4]], we reject them. - """ + """may be this form [[Var], [Var], [3], [4]], we reject them.""" raise TypeError( "The type of received input == `object`, it is not supported to convert to tensor, such as [[Var], [Var], [3], [4]]" ) @@ -1667,7 +1799,8 @@ def convert_scalar(x): warnings.warn( "paddle.assign doesn't support float64 input now due " "to current platform protobuf data limitation, we convert " - "it to float32") + "it to float32" + ) dtype = core.VarDesc.VarType.FP32 if dtype == core.VarDesc.VarType.BOOL: value_name = "bool_values" @@ -1685,31 +1818,49 @@ def convert_scalar(x): raise TypeError( "When the type of 'input' in assign is numpy.ndarray, " "the data type of 'input' must be bool, float32, int32 or int64, but " - "received %s." % convert_dtype(dtype)) + "received %s." % convert_dtype(dtype) + ) if input.size > 1024 * 1024: - raise ValueError("The size of input is too big. Please consider " - "saving it to file and 'load_op' to load it") + raise ValueError( + "The size of input is too big. Please consider " + "saving it to file and 'load_op' to load it" + ) if in_dygraph_mode(): if output is None: output = zeros(list(input.shape), dtype) - _C_ops.assign_value_(output, list(input.shape), dtype, values, - _current_expected_place()) + _C_ops.assign_value_( + output, + list(input.shape), + dtype, + values, + _current_expected_place(), + ) elif _in_legacy_dygraph(): if output is None: output = core.VarBase() - _legacy_C_ops.assign_value(output, 'shape', list(input.shape), - 'dtype', dtype, value_name, values) + _legacy_C_ops.assign_value( + output, + 'shape', + list(input.shape), + 'dtype', + dtype, + value_name, + values, + ) else: if output is None: output = helper.create_variable_for_type_inference( - dtype=input.dtype) - helper.append_op(type='assign_value', - outputs={'Out': [output]}, - attrs={ - 'dtype': dtype, - 'shape': list(input.shape), - value_name: values - }) + dtype=input.dtype + ) + helper.append_op( + type='assign_value', + outputs={'Out': [output]}, + attrs={ + 'dtype': dtype, + 'shape': list(input.shape), + value_name: values, + }, + ) if is_inplace and _in_legacy_dygraph(): output._bump_inplace_version() @@ -1747,7 +1898,7 @@ def clone(x, name=None): return x.clone() -#NOTE(zhiqiu): not public +# NOTE(zhiqiu): not public def _memcpy(input, place=None, output=None): """ @@ -1775,10 +1926,22 @@ def _memcpy(input, place=None, output=None): check_type(input, 'input', (Variable), 'memcpy') if isinstance(input, (Variable, core.VarBase)): - check_dtype(input.dtype, 'input', [ - 'float16', 'uint16', 'float32', 'float64', 'int32', 'int64', - 'uint8', 'bool' - ], 'memcpy', '(When the type of input in memcpy is Variable.)') + check_dtype( + input.dtype, + 'input', + [ + 'float16', + 'uint16', + 'float32', + 'float64', + 'int32', + 'int64', + 'uint8', + 'bool', + ], + 'memcpy', + '(When the type of input in memcpy is Variable.)', + ) if output is None: output = helper.create_variable_for_type_inference(dtype=input.dtype) @@ -1800,10 +1963,12 @@ def _memcpy(input, place=None, output=None): dst_place_type = 4 attrs = {'dst_place_type': dst_place_type} - helper.append_op(type='memcpy', - inputs={'X': [input]}, - outputs={'Out': [output]}, - attrs=attrs) + helper.append_op( + type='memcpy', + inputs={'X': [input]}, + outputs={'Out': [output]}, + attrs=attrs, + ) return output @@ -1846,7 +2011,8 @@ def complex(real, imag, name=None): helper = LayerHelper(op_type, **locals()) inputs = {"X": real, "Y": imag} out = helper.create_variable_for_type_inference( - dtype=_real_to_complex_dtype(real.dtype)) + dtype=_real_to_complex_dtype(real.dtype) + ) outputs = {"Out": out} attrs = {} helper.append_op(type=op_type, inputs=inputs, attrs=attrs, outputs=outputs) @@ -1914,13 +2080,15 @@ def tril_indices(row, col, offset=0, dtype='int64'): dtype = convert_np_dtype_to_dtype_(dtype) if in_dygraph_mode(): - out = _C_ops.tril_indices(row, col, offset, dtype, - _current_expected_place()) + out = _C_ops.tril_indices( + row, col, offset, dtype, _current_expected_place() + ) return out if _in_legacy_dygraph(): - out = _legacy_C_ops.tril_indices('rows', row, 'cols', col, 'offset', - offset, "dtype", dtype) + out = _legacy_C_ops.tril_indices( + 'rows', row, 'cols', col, 'offset', offset, "dtype", dtype + ) return out else: @@ -1928,15 +2096,12 @@ def tril_indices(row, col, offset=0, dtype='int64'): out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='tril_indices', - inputs={}, - outputs={'out': [out]}, - attrs={ - 'rows': row, - 'cols': col, - 'offset': offset, - 'dtype': dtype - }) + helper.append_op( + type='tril_indices', + inputs={}, + outputs={'out': [out]}, + attrs={'rows': row, 'cols': col, 'offset': offset, 'dtype': dtype}, + ) return out @@ -1999,13 +2164,15 @@ def triu_indices(row, col=None, offset=0, dtype='int64'): dtype = convert_np_dtype_to_dtype_(dtype) if in_dygraph_mode(): - out = _C_ops.triu_indices(row, col, offset, dtype, - _current_expected_place()) + out = _C_ops.triu_indices( + row, col, offset, dtype, _current_expected_place() + ) return out if _in_legacy_dygraph(): - out = _legacy_C_ops.triu_indices('row', row, 'col', col, 'offset', - offset, "dtype", dtype) + out = _legacy_C_ops.triu_indices( + 'row', row, 'col', col, 'offset', offset, "dtype", dtype + ) return out else: @@ -2013,13 +2180,10 @@ def triu_indices(row, col=None, offset=0, dtype='int64'): out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='triu_indices', - inputs={}, - outputs={'out': [out]}, - attrs={ - 'row': row, - 'col': col, - 'offset': offset, - 'dtype': dtype - }) + helper.append_op( + type='triu_indices', + inputs={}, + outputs={'out': [out]}, + attrs={'row': row, 'col': col, 'offset': offset, 'dtype': dtype}, + ) return out diff --git a/python/paddle/tensor/einsum.py b/python/paddle/tensor/einsum.py index f8b014afb617a..5c792f8fe0df8 100644 --- a/python/paddle/tensor/einsum.py +++ b/python/paddle/tensor/einsum.py @@ -50,12 +50,13 @@ def parse_op_labels(labelstr, operand): ''' # Sanity checks for c in labelstr.replace('.', ''): - assert c.isalpha(), ( - f"Invalid equation: {c} is not a valid label, which should be letters." - ) + assert ( + c.isalpha() + ), f"Invalid equation: {c} is not a valid label, which should be letters." - assert labelstr.replace('...', '', 1).find('.') == -1, ( - "Invalid equation: `.` is found outside of an ellipsis.") + assert ( + labelstr.replace('...', '', 1).find('.') == -1 + ), "Invalid equation: `.` is found outside of an ellipsis." # Check shape. Note, in Paddle a tensor rank is always nonzero ndims = len(operand.shape) @@ -63,8 +64,9 @@ def parse_op_labels(labelstr, operand): full_labelstr = labelstr.replace('...', '.' * (ndims - len(labelstr) + 3)) - assert len(full_labelstr) == ndims, ( - f"Invalid equation: the label string '{labelstr}' misses dimensions.") + assert ( + len(full_labelstr) == ndims + ), f"Invalid equation: the label string '{labelstr}' misses dimensions." return full_labelstr @@ -88,7 +90,8 @@ def parse_labels(labelstr, operands): nop_labels = labelstr.split(',') assert len(nop_labels) == len(operands), ( f"Invalid equation: the number of operands is {len(operands)}, " - f"but found {len(nop_labels)} segments in the label equation.") + f"but found {len(nop_labels)} segments in the label equation." + ) return list(map(parse_op_labels, nop_labels, operands)) @@ -99,8 +102,9 @@ def validate_rhs(rhs, input_labels, n_bcast_dims): ''' # Sanity check. if n_bcast_dims > 0: - assert '...' in rhs, ( - "Invalid equation: missing ellipsis in output labels.") + assert ( + '...' in rhs + ), "Invalid equation: missing ellipsis in output labels." rhs = rhs.replace('...', '') rhs_set = set(rhs) @@ -112,10 +116,12 @@ def validate_rhs(rhs, input_labels, n_bcast_dims): non_input_labels = rhs_set.difference(input_labels) assert not non_input_labels, ( f"Invalid equation: " - f"output label {sorted(non_input_labels)} not used by any input.") + f"output label {sorted(non_input_labels)} not used by any input." + ) # Verify that output labels are not duplicate - assert len(rhs) == len(rhs_set), ( - "Invalid equation: duplicate output labels are found.") + assert len(rhs) == len( + rhs_set + ), "Invalid equation: duplicate output labels are found." def build_view(in_labels, out_labels): @@ -157,8 +163,8 @@ def build_view(in_labels, out_labels): # fill the broadcast dimension indices from right to left. if s: for ax, dim in zip( - range(start, end)[::-1], - range(s.start(), s.end())[::-1]): + range(start, end)[::-1], range(s.start(), s.end())[::-1] + ): inv_map[ax] = dim # Now work on non-broadcast dimensions @@ -217,7 +223,8 @@ def build_global_view(nop_labels, rhs, n_bcast_dims): g_labels_out = rhs.replace('...', '.' * n_bcast_dims) else: g_labels_out = '.' * n_bcast_dims + ''.join( - l for l, c in zip(labels, count) if c == 1) + l for l, c in zip(labels, count) if c == 1 + ) for i in range(len(count))[::-1]: if labels[i] in g_labels_out: @@ -265,12 +272,14 @@ def build_global_shape(g_view, g_labels, op_shapes): assert not non_bcastable, ( f"Invalid operands: label {g_labels[non_bcastable[0]]} " - f"corresponds to non-broadcastable dimensions.") + f"corresponds to non-broadcastable dimensions." + ) g_shape = [sizes.pop() if len(sizes) > 0 else 1 for sizes in g_shape] - g_masks = [[s > 1 or s == -1 for s in view_shape] - for view_shape in view_shapes] + g_masks = [ + [s > 1 or s == -1 for s in view_shape] for view_shape in view_shapes + ] return g_shape, g_masks @@ -295,8 +304,9 @@ def diagonalize(labels, operand): -------- 'ijj...i' would be merged into 'ij...' ''' - assert not has_duplicated_labels(labels), ( - 'Duplicate labels are not supported.') + assert not has_duplicated_labels( + labels + ), 'Duplicate labels are not supported.' return labels, operand @@ -356,12 +366,21 @@ def plan_matmul(plan, g_view, op1, op2, g_supports, g_shape, I, J1, J2, K): plan.add_step(step) # Check if conditions hold for turnning the operation into a matmul - if j1 + j2 > 0 and k > 0 and -1 not in np.concatenate( - (op1_vshape, op2_vshape)): - op1_shape = list(op1_vshape[I]) + [np.prod(op1_vshape[J1]) - ] + [np.prod(op1_vshape[K])] - op2_shape = list(op2_vshape[I]) + [np.prod(op2_vshape[J2]) - ] + [np.prod(op2_vshape[K])] + if ( + j1 + j2 > 0 + and k > 0 + and -1 not in np.concatenate((op1_vshape, op2_vshape)) + ): + op1_shape = ( + list(op1_vshape[I]) + + [np.prod(op1_vshape[J1])] + + [np.prod(op1_vshape[K])] + ) + op2_shape = ( + list(op2_vshape[I]) + + [np.prod(op2_vshape[J2])] + + [np.prod(op2_vshape[K])] + ) # Merge J dims and K dims by reshaping step = reshape, [var1], var1, op1_shape @@ -410,15 +429,22 @@ def plan_matmul(plan, g_view, op1, op2, g_supports, g_shape, I, J1, J2, K): step = squeeze, [var2], var2, [-1, -2] plan.add_step(step) elif j1 + j2 == 0 and -1 not in np.concatenate( - (op1_vshape[K], op2_vshape[K])): + (op1_vshape[K], op2_vshape[K]) + ): assert all(op1_vshape[K] == op2_vshape[K]) - step = reshape, [ - var1 - ], var1, list(op1_vshape[I]) + [1] + [np.prod(op1_vshape[K])] + step = ( + reshape, + [var1], + var1, + list(op1_vshape[I]) + [1] + [np.prod(op1_vshape[K])], + ) plan.add_step(step) - step = reshape, [ - var2 - ], var2, list(op2_vshape[I]) + [1] + [np.prod(op2_vshape[K])] + step = ( + reshape, + [var2], + var2, + list(op2_vshape[I]) + [1] + [np.prod(op2_vshape[K])], + ) plan.add_step(step) step = matmul, [var1, var2], var2, False, True plan.add_step(step) @@ -447,8 +473,9 @@ def plan_matmul(plan, g_view, op1, op2, g_supports, g_shape, I, J1, J2, K): g_view[op2] = list(op2_view) -def plan_summation(plan, g_view, op1, op2, g_supports, g_shape, g_count, - n_bcast): +def plan_summation( + plan, g_view, op1, op2, g_supports, g_shape, g_count, n_bcast +): ''' Plan various kinds of summation ''' @@ -462,8 +489,9 @@ def plan_summation(plan, g_view, op1, op2, g_supports, g_shape, g_count, I, K, J1, J2 = list(range(n_bcast)), [], [], [] - for ax, dim1, dim2 in zip(range(n_bcast, ndim), op1_view[n_bcast:], - op2_view[n_bcast:]): + for ax, dim1, dim2 in zip( + range(n_bcast, ndim), op1_view[n_bcast:], op2_view[n_bcast:] + ): if (dim1 != -1) != (dim2 != -1): if dim1 != -1: @@ -529,7 +557,6 @@ def f(*args): class Plan: - def __init__(self): self.env = {} self.steps = [] @@ -633,8 +660,9 @@ def plan_einsum(operands, g_view, g_shape, g_supports, g_count, n_bcast): # op1 is a one element tensor. plan_scalar_prod(plan, i - 1, i) else: - plan_summation(plan, g_view, i - 1, i, g_supports, g_shape, g_count, - n_bcast) + plan_summation( + plan, g_view, i - 1, i, g_supports, g_shape, g_count, n_bcast + ) # for ax, dim in enumerate(g_view[nop-1][:nout]): # assert dim == ax @@ -676,7 +704,9 @@ def preprocess(equation, *operands): """ equation = equation.replace(" ", "") nop = len(operands) - assert nop > 0, "Required at least one operand in Einsum API, but received %s " % nop + assert nop > 0, ( + "Required at least one operand in Einsum API, but received %s " % nop + ) # Part the equation to left hand side and right hand side lhs, *rhs = equation.lower().split('->') @@ -690,16 +720,20 @@ def preprocess(equation, *operands): assert len(lhs.split(',')) == len(operands), ( f"Invalid equation: the number of operands is {len(operands)}, " - f"but found {len(lhs.split(','))} segments in the label equation.") + f"but found {len(lhs.split(','))} segments in the label equation." + ) - assert not ('...' in lhs and '...' not in rhs - ), 'Invalid equation: missing ellipsis in output labels.' + assert not ( + '...' in lhs and '...' not in rhs + ), 'Invalid equation: missing ellipsis in output labels.' - assert not (len(list(filter(has_duplicated_labels, lhs.split(',')))) > - 0), 'Duplicate labels are not supported.' + assert not ( + len(list(filter(has_duplicated_labels, lhs.split(',')))) > 0 + ), 'Duplicate labels are not supported.' assert not has_duplicated_labels( - rhs), 'Invalid equation: duplicate output labels are found.' + rhs + ), 'Invalid equation: duplicate output labels are found.' return lhs, rhs, labels @@ -717,10 +751,10 @@ def parse_fake_shape(equation, operands, labels): shaped = collections.namedtuple('shaped', ['shape']) def fake_shape(label, op): - assert len(op.shape) == len( - label - ), "length of shape and length of label must be the same, but received %d != %d" % ( - len(op.shape), len(label)) + assert len(op.shape) == len(label), ( + "length of shape and length of label must be the same, but received %d != %d" + % (len(op.shape), len(label)) + ) fakes = [s for i, (l, s) in enumerate(zip(label, op.shape)) if l != '.'] fakes = list(map(abs, fakes)) # make -1 -> 1 if '.' in label: @@ -732,7 +766,6 @@ def fake_shape(label, op): def rhs_inference(lhs): - def is_free(key): return cnt.get(key) == 1 and key not in ['.', ','] @@ -751,7 +784,8 @@ def gen_equation_for_opteinsum(lhs, rhs): def get_used_label(counter): used = set(counter.elements()) for c in string.ascii_lowercase: - if c not in used: return c + if c not in used: + return c raise ValueError( "You have used all `a` - `z`, there can't find a unused for einsum optimization" ) @@ -784,14 +818,15 @@ def einsum_v2(equation, *operands): var_list = list(operands) for path in cons: (a, b), _, eq, *__ = path - assert a > b, "Assume the first var_idx is smaller than the second_idx. opt_einsum can guarantee it." + assert ( + a > b + ), "Assume the first var_idx is smaller than the second_idx. opt_einsum can guarantee it." var_s = [var_list.pop(a), var_list.pop(b)] eq = eq.replace(broadcast_label, "...") var_list.append(gen_einsum_op(eq, *var_s)) - assert len( - var_list - ) == 1, "There must be one elements in list, but received %d." % len( - var_list) + assert ( + len(var_list) == 1 + ), "There must be one elements in list, but received %d." % len(var_list) return var_list[0] @@ -805,8 +840,9 @@ def gen_einsum_op(equation, *operands): if _in_legacy_dygraph(): # dygraph - return _legacy_C_ops.einsum(operands, len(operands), len(operands), - 'equation', equation)[0] + return _legacy_C_ops.einsum( + operands, len(operands), len(operands), 'equation', equation + )[0] for inp in operands: check_variable_and_dtype(inp, 'dtype', ['float32', 'float64'], 'einsum') @@ -823,14 +859,12 @@ def gen_einsum_op(equation, *operands): helper.create_variable_for_type_inference(dtype=operands[0].dtype) for i in range(len(operands)) ] - helper.append_op(type='einsum', - inputs={'Operands': operands}, - outputs={ - 'Out': out, - "InnerCache": caches, - "XShape": xshape - }, - attrs=attrs) + helper.append_op( + type='einsum', + inputs={'Operands': operands}, + outputs={'Out': out, "InnerCache": caches, "XShape": xshape}, + attrs=attrs, + ) return out @@ -992,6 +1026,7 @@ def einsum(equation, *operands): # [0.51476848, 0.23367381, 0.39229113]]]) """ import os + if int(os.environ.get('FLAGS_new_einsum', "1")): return einsum_v2(equation, *operands) @@ -1037,9 +1072,11 @@ def einsum(equation, *operands): # Counting how many non-trivial dimensions remain for each ax g_labels, g_view, g_nout, g_count = build_global_view( - nop_labels, rhs, n_bcast_dims) - g_shape, g_supports = build_global_shape(g_view, g_labels, - [op.shape for op in operands]) + nop_labels, rhs, n_bcast_dims + ) + g_shape, g_supports = build_global_shape( + g_view, g_labels, [op.shape for op in operands] + ) # Now we're ready to build up an execution plan args = operands, g_view, g_shape, g_supports, g_count, n_bcast_dims diff --git a/python/paddle/tensor/layer_function_generator.py b/python/paddle/tensor/layer_function_generator.py index e4b36ba5e1418..fb2a611257e13 100644 --- a/python/paddle/tensor/layer_function_generator.py +++ b/python/paddle/tensor/layer_function_generator.py @@ -19,7 +19,13 @@ from io import StringIO from ..static import Variable from ..fluid.proto import framework_pb2 -from ..framework import OpProtoHolder, _non_static_mode, convert_np_dtype_to_dtype_, core, in_dygraph_mode +from ..framework import ( + OpProtoHolder, + _non_static_mode, + convert_np_dtype_to_dtype_, + core, + in_dygraph_mode, +) from ..framework import LayerHelper from ..fluid.data_feeder import check_variable_and_dtype from paddle import _C_ops, _legacy_C_ops @@ -53,16 +59,16 @@ def _type_to_str_(tp): def escape_math(text): - #return _two_bang_pattern_.sub( + # return _two_bang_pattern_.sub( # r'$$\1$$', # _single_dollar_pattern_.sub(r':math:\n`\1`', # _two_dollar_pattern_.sub(r"!!\1!!", text))) return _two_dollar_pattern_.sub(r':math:`\1`', text) -def _generate_doc_string_(op_proto, - additional_args_lines=None, - skip_attrs_set=None): +def _generate_doc_string_( + op_proto, additional_args_lines=None, skip_attrs_set=None +): """ Generate docstring by OpProto @@ -142,23 +148,30 @@ def generate_layer_fn(op_type): """ op_proto = OpProtoHolder.instance().get_op_proto(op_type) - not_intermediate_outputs = \ - [output for output in op_proto.outputs if not output.intermediate] - intermediate_outputs = \ - [output for output in op_proto.outputs if output.intermediate] + not_intermediate_outputs = [ + output for output in op_proto.outputs if not output.intermediate + ] + intermediate_outputs = [ + output for output in op_proto.outputs if output.intermediate + ] if len(not_intermediate_outputs) != 1: - raise ValueError("Only one non intermediate output operator can be", - "automatically generated. {0}".format(op_type)) + raise ValueError( + "Only one non intermediate output operator can be", + "automatically generated. {0}".format(op_type), + ) if not_intermediate_outputs[0].duplicable: raise ValueError( - "Only non duplicable op can be automatically generated.") + "Only non duplicable op can be automatically generated." + ) for output in intermediate_outputs: if output.duplicable: - raise ValueError("The op can be automatically generated only when ", - "all intermediate ops are not duplicable.") + raise ValueError( + "The op can be automatically generated only when ", + "all intermediate ops are not duplicable.", + ) o_name = not_intermediate_outputs[0].name intermediate_output_names = [output.name for output in intermediate_outputs] @@ -183,14 +196,17 @@ def infer_and_check_dtype(op_proto, *args, **kwargs): for each in val: if not isinstance(each, Variable): raise ValueError( - "input of {0} must be variable".format(op_type)) + "input of {0} must be variable".format(op_type) + ) if dtype is None: dtype = each.dtype elif dtype != each.dtype: raise ValueError( "operator {0} must input same dtype. {1} vs {2}".format( - op_type, dtype, each.dtype)) + op_type, dtype, each.dtype + ) + ) if dtype is None: arg_dtype = kwargs.get("dtype") @@ -222,8 +238,11 @@ def func(*args, **kwargs): outputs = dict() out = kwargs.pop(_convert_(o_name), []) if out: - out_var = out[0] if (isinstance(out, list) - or isinstance(out, tuple)) else out + out_var = ( + out[0] + if (isinstance(out, list) or isinstance(out, tuple)) + else out + ) else: out_var = helper.create_variable_for_type_inference(dtype=dtype) outputs[o_name] = [out_var] @@ -231,10 +250,9 @@ def func(*args, **kwargs): outputs[name] = [ helper.create_variable_for_type_inference(dtype=dtype) ] - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=kwargs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=kwargs + ) return helper.append_activation(out_var) func.__name__ = op_type @@ -265,14 +283,25 @@ def func(x, name=None): return op(x) if op_type not in ["abs", "exp", "square"]: - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - op_type) + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], op_type + ) else: # abs exp square ops support dtype(int32, int64, float16, float32, float64) - check_variable_and_dtype(x, 'x', [ - 'int32', 'int64', 'float16', 'float32', 'float64', 'complex64', - 'complex128' - ], op_type) + check_variable_and_dtype( + x, + 'x', + [ + 'int32', + 'int64', + 'float16', + 'float32', + 'float64', + 'complex64', + 'complex128', + ], + op_type, + ) helper = LayerHelper(op_type, **locals()) @@ -285,7 +314,8 @@ def func(x, name=None): op_proto, additional_args_lines=[ "name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`." - ]) + ], + ) return func @@ -308,15 +338,19 @@ def func(x, name=None): op = getattr(_legacy_C_ops, inplace_op_type) return op(x) warnings.warn( - "In static mode, {}() is the same as {}() and does not perform inplace operation." - .format(inplace_op_type, origin_op_type)) + "In static mode, {}() is the same as {}() and does not perform inplace operation.".format( + inplace_op_type, origin_op_type + ) + ) return generate_activation_fn(origin_op_type)(x, name) func.__name__ = inplace_op_type func.__doc__ = """ Inplace version of ``{0}`` API, the output Tensor will be inplaced with input ``x``. Please refer to :ref:`api_fluid_layers_{1}`. -""".format(origin_op_type, origin_op_type) +""".format( + origin_op_type, origin_op_type + ) return func @@ -360,18 +394,21 @@ def __impl__(func): for each_input in op_proto.inputs: input_name = _convert_(each_input.name) args["{0}_comment".format(input_name)] = trim_ending_dot( - each_input.comment) + each_input.comment + ) args["{0}_type".format(input_name)] = "Variable" for each_attr in op_proto.attrs: input_name = _convert_(each_attr.name) args["{0}_comment".format(input_name)] = trim_ending_dot( - each_attr.comment) + each_attr.comment + ) args["{0}_type".format(input_name)] = _type_to_str_(each_attr.type) for each_opt in op_proto.outputs: output_name = _convert_(each_opt.name) args["{0}_comment".format(output_name)] = trim_ending_dot( - each_opt.comment) + each_opt.comment + ) args["{0}_type".format(output_name)] = "Variable" func.__doc__ = tmpl.substitute(args) return func diff --git a/python/paddle/tensor/linalg.py b/python/paddle/tensor/linalg.py index d341deabbd2b0..3ed56a35dfa9c 100644 --- a/python/paddle/tensor/linalg.py +++ b/python/paddle/tensor/linalg.py @@ -15,7 +15,11 @@ import numpy as np from ..framework import LayerHelper from ..framework import _non_static_mode, in_dygraph_mode -from ..fluid.data_feeder import check_variable_and_dtype, check_type, check_dtype +from ..fluid.data_feeder import ( + check_variable_and_dtype, + check_type, + check_dtype, +) from ..static import Variable from ..fluid.framework import _in_legacy_dygraph from .manipulation import cast @@ -90,10 +94,21 @@ def transpose(x, perm, name=None): out, _ = _legacy_C_ops.transpose2(x, 'axis', perm) return out - check_variable_and_dtype(x, 'x', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], 'transpose') + check_variable_and_dtype( + x, + 'x', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'transpose', + ) check_type(perm, 'perm', (list, tuple), 'transpose') if isinstance(perm, tuple): perm = list(perm) @@ -102,24 +117,25 @@ def transpose(x, perm, name=None): "Input(perm) is the permutation of dimensions of Input(x), " "its length should be equal to dimensions of Input(x), " "but received dimension of Input(x) is %s, " - "the length of Input(perm) is %s." % (len(x.shape), len(perm))) + "the length of Input(perm) is %s." % (len(x.shape), len(perm)) + ) for idx, dim in enumerate(perm): if dim >= len(x.shape): raise ValueError( "Each element in Input(perm) should be less than Input(x)'s dimension, " "but %d-th element in Input(perm) is %d which exceeds Input(x)'s " - "dimension %d." % (idx, perm[idx], len(x.shape))) + "dimension %d." % (idx, perm[idx], len(x.shape)) + ) helper = LayerHelper('transpose', **locals()) out = helper.create_variable_for_type_inference(x.dtype) x_shape = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='transpose2', - inputs={'X': [x]}, - outputs={ - 'Out': [out], - 'XShape': [x_shape] - }, - attrs={'axis': perm}) + helper.append_op( + type='transpose2', + inputs={'X': [x]}, + outputs={'Out': [out], 'XShape': [x_shape]}, + attrs={'axis': perm}, + ) return out @@ -234,21 +250,22 @@ def __check_input(x, y): var_names = {'x': x, 'y': y} for name, val in var_names.items(): check_variable_and_dtype( - val, name, + val, + name, ['float16', 'float32', 'float64', 'complex64', 'complex128'], - 'matmul') + 'matmul', + ) __check_input(x, y) helper = LayerHelper('matmul_v2', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='matmul_v2', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='matmul_v2', + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs=attrs, + ) return out @@ -358,32 +375,35 @@ def frobenius_norm(input, dim=None, keepdim=False, name=None): return _C_ops.frobenius_norm(input, dim, keepdim, False) if _in_legacy_dygraph(): if dim is None: - return _legacy_C_ops.frobenius_norm(input, 'keep_dim', keepdim, - 'reduce_all', True) - return _legacy_C_ops.frobenius_norm(input, 'dim', dim, 'keep_dim', - keepdim, 'reduce_all', False) + return _legacy_C_ops.frobenius_norm( + input, 'keep_dim', keepdim, 'reduce_all', True + ) + return _legacy_C_ops.frobenius_norm( + input, 'dim', dim, 'keep_dim', keepdim, 'reduce_all', False + ) attrs = {'dim': dim, 'keep_dim': keepdim, 'reduce_all': False} if dim is None: attrs['reduce_all'] = True - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'frobenius_norm') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'frobenius_norm' + ) helper = LayerHelper('frobenius_norm', **locals()) out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) - helper.append_op(type='frobenius_norm', - inputs={'X': input}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='frobenius_norm', + inputs={'X': input}, + outputs={'Out': out}, + attrs=attrs, + ) return out - def vector_norm(input, - porder=None, - axis=None, - keepdim=False, - asvector=False, - name=None): + def vector_norm( + input, porder=None, axis=None, keepdim=False, asvector=False, name=None + ): """ Calculate the p-order vector norm for certain dimension of Tensor `input`. Args: @@ -393,21 +413,32 @@ def vector_norm(input, keepdim (bool, optional): Whether keep the dimensions as the `input`, Default False. """ if in_dygraph_mode(): - if axis is None: axis = -1 + if axis is None: + axis = -1 return _C_ops.p_norm(input, porder, axis, 1e-12, keepdim, asvector) if _in_legacy_dygraph(): - if axis is None: axis = -1 - return _legacy_C_ops.p_norm(input, 'porder', porder, 'axis', axis, - 'keepdim', keepdim, 'asvector', - asvector) + if axis is None: + axis = -1 + return _legacy_C_ops.p_norm( + input, + 'porder', + porder, + 'axis', + axis, + 'keepdim', + keepdim, + 'asvector', + asvector, + ) if porder is not None: check_type(porder, 'porder', (float, int), 'p_norm') if axis is not None: check_type(axis, 'axis', (int), 'p_norm') - check_variable_and_dtype(input, 'input', ['float32', 'float64'], - 'p_norm') + check_variable_and_dtype( + input, 'input', ['float32', 'float64'], 'p_norm' + ) attrs = { 'axis': axis if axis is not None else -1, @@ -418,23 +449,27 @@ def vector_norm(input, } helper = LayerHelper('p_norm', **locals()) out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) - helper.append_op(type='p_norm', - inputs={'X': input}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='p_norm', + inputs={'X': input}, + outputs={'Out': out}, + attrs=attrs, + ) return out - def inf_norm(input, - porder=None, - axis=axis, - keepdim=False, - asvector=False, - name=None): + def inf_norm( + input, porder=None, axis=axis, keepdim=False, asvector=False, name=None + ): if in_dygraph_mode(): out = _C_ops.abs(input) - reduce_all = True if axis == None or axis == [] or asvector == True else False + reduce_all = ( + True + if axis == None or axis == [] or asvector == True + else False + ) axis = axis if axis != None and axis != [] else [0] if reduce_all: assert (axis == []) or (axis is None) @@ -445,28 +480,31 @@ def inf_norm(input, helper = LayerHelper('inf_norm', **locals()) out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) helper.append_op(type='abs', inputs={'X': input}, outputs={'Out': out}) reduce_out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype()) + dtype=helper.input_dtype() + ) - reduce_all = True if axis == None or axis == [] or asvector == True else False + reduce_all = ( + True if axis == None or axis == [] or asvector == True else False + ) axis = axis if axis != None and axis != [] else [0] - reduce_type = 'reduce_max' if porder == np.float64( - 'inf') else 'reduce_min' - helper.append_op(type=reduce_type, - inputs={'X': out}, - outputs={'Out': reduce_out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + reduce_type = ( + 'reduce_max' if porder == np.float64('inf') else 'reduce_min' + ) + helper.append_op( + type=reduce_type, + inputs={'X': out}, + outputs={'Out': reduce_out}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) return reduce_out - def p_matrix_norm(input, porder=1., axis=axis, keepdim=False, name=None): + def p_matrix_norm(input, porder=1.0, axis=axis, keepdim=False, name=None): """ NOTE: This function actually treats the matrix as flattened vector to calculate vector norm instead of matrix norm. @@ -475,38 +513,48 @@ def p_matrix_norm(input, porder=1., axis=axis, keepdim=False, name=None): abs_out = _C_ops.abs(input) pow_out = _C_ops.pow(abs_out, porder) sum_out = _C_ops.sum(pow_out, axis, None, keepdim) - out = _C_ops.pow(sum_out, float(1. / porder)) + out = _C_ops.pow(sum_out, float(1.0 / porder)) return out block = LayerHelper('norm', **locals()) out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) abs_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) - block.append_op(type='abs', - inputs={'X': input}, - outputs={'Out': abs_out}) + dtype=block.input_dtype() + ) + block.append_op( + type='abs', inputs={'X': input}, outputs={'Out': abs_out} + ) pow_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) - block.append_op(type='pow', - inputs={'X': abs_out}, - outputs={'Out': pow_out}, - attrs={'factor': porder}) + block.append_op( + type='pow', + inputs={'X': abs_out}, + outputs={'Out': pow_out}, + attrs={'factor': porder}, + ) sum_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) - block.append_op(type='reduce_sum', - inputs={'X': pow_out}, - outputs={'Out': sum_out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': True if axis is None else False - }) - block.append_op(type='pow', - inputs={'X': sum_out}, - outputs={'Out': out}, - attrs={'factor': float(1. / porder)}) + dtype=block.input_dtype() + ) + block.append_op( + type='reduce_sum', + inputs={'X': pow_out}, + outputs={'Out': sum_out}, + attrs={ + 'dim': axis, + 'keep_dim': keepdim, + 'reduce_all': True if axis is None else False, + }, + ) + block.append_op( + type='pow', + inputs={'X': sum_out}, + outputs={'Out': out}, + attrs={'factor': float(1.0 / porder)}, + ) return out if axis is None and p is not None: @@ -515,50 +563,60 @@ def p_matrix_norm(input, porder=1., axis=axis, keepdim=False, name=None): return frobenius_norm(x, dim=axis, keepdim=keepdim, name=name) else: raise ValueError( - "only valid string values are 'fro', found {}".format(p)) + "only valid string values are 'fro', found {}".format(p) + ) elif isinstance(p, (int, float)): - return vector_norm(x, - porder=p, - axis=axis, - keepdim=keepdim, - asvector=True, - name=name) + return vector_norm( + x, + porder=p, + axis=axis, + keepdim=keepdim, + asvector=True, + name=name, + ) else: raise ValueError( - "only valid p type is string or float, found {}".format( - type(p))) + "only valid p type is string or float, found {}".format(type(p)) + ) if isinstance(axis, tuple): axis = list(axis) if isinstance(axis, list) and len(axis) == 1: axis = axis[0] - #calculate vector norm, where axis is int or list with only one integer + # calculate vector norm, where axis is int or list with only one integer if isinstance(axis, int): if isinstance(p, str): if p == "fro": - return vector_norm(x, - porder=2, - axis=axis, - keepdim=keepdim, - asvector=False, - name=name) + return vector_norm( + x, + porder=2, + axis=axis, + keepdim=keepdim, + asvector=False, + name=name, + ) else: raise ValueError( - "only valid string values are 'fro', found {}".format(p)) + "only valid string values are 'fro', found {}".format(p) + ) elif isinstance(p, (int, float)): - return vector_norm(x, - axis=axis, - porder=p, - keepdim=keepdim, - asvector=False, - name=name) + return vector_norm( + x, + axis=axis, + porder=p, + keepdim=keepdim, + asvector=False, + name=name, + ) else: raise ValueError( - "unspport p for p-order vector norm. except float, found {}". - format(p)) - #calculate matrix norm, where axis is list with two integers + "unspport p for p-order vector norm. except float, found {}".format( + p + ) + ) + # calculate matrix norm, where axis is list with two integers elif isinstance(axis, list) and len(axis) == 2: if p == "fro": return frobenius_norm(x, dim=axis, keepdim=keepdim, name=name) @@ -566,18 +624,20 @@ def p_matrix_norm(input, porder=1., axis=axis, keepdim=False, name=None): return inf_norm(x, porder=p, axis=axis, keepdim=keepdim, name=name) elif p == 0: raise ValueError( - "just suport axis type int or list (length of list <=1) if p = 0, found {}" - .format(axis)) + "just suport axis type int or list (length of list <=1) if p = 0, found {}".format( + axis + ) + ) else: - return p_matrix_norm(x, - porder=p, - axis=axis, - keepdim=keepdim, - name=name) + return p_matrix_norm( + x, porder=p, axis=axis, keepdim=keepdim, name=name + ) else: raise ValueError( - "except axis type int or list (length of list <=2), found {}". - format(axis)) + "except axis type int or list (length of list <=2), found {}".format( + axis + ) + ) def dist(x, y, p=2, name=None): @@ -674,10 +734,9 @@ def dist(x, y, p=2, name=None): inputs = {"X": [x], "Y": [y]} outputs = {'Out': [out]} attrs = {"p": float(p)} - helper.append_op(type='dist', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='dist', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -768,7 +827,7 @@ def cond(x, p=None, name=None): """ - def mat_norm(input, porder=1., axis=None): + def mat_norm(input, porder=1.0, axis=None): """ NOTE: Calculate the matrix norm of a square matrix or batches of square matrices, @@ -789,54 +848,81 @@ def mat_norm(input, porder=1., axis=None): elif _in_legacy_dygraph(): abs_out = _legacy_C_ops.abs(input) - sum_out = _legacy_C_ops.reduce_sum(abs_out, 'dim', axis, 'keepdim', - keepdim, 'reduce_all', - reduce_all) + sum_out = _legacy_C_ops.reduce_sum( + abs_out, + 'dim', + axis, + 'keepdim', + keepdim, + 'reduce_all', + reduce_all, + ) if porder == 1 or porder == np.inf: - return _legacy_C_ops.reduce_max(sum_out, 'dim', [-1], 'keepdim', - keepdim, 'reduce_all', - reduce_all) + return _legacy_C_ops.reduce_max( + sum_out, + 'dim', + [-1], + 'keepdim', + keepdim, + 'reduce_all', + reduce_all, + ) if porder == -1 or porder == -np.inf: - return _legacy_C_ops.reduce_min(sum_out, 'dim', [-1], 'keepdim', - keepdim, 'reduce_all', - reduce_all) + return _legacy_C_ops.reduce_min( + sum_out, + 'dim', + [-1], + 'keepdim', + keepdim, + 'reduce_all', + reduce_all, + ) else: block = LayerHelper('norm', **locals()) abs_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) sum_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) - block.append_op(type='abs', - inputs={'X': input}, - outputs={'Out': abs_out}) - block.append_op(type='reduce_sum', - inputs={'X': abs_out}, - outputs={'Out': sum_out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + dtype=block.input_dtype() + ) + block.append_op( + type='abs', inputs={'X': input}, outputs={'Out': abs_out} + ) + block.append_op( + type='reduce_sum', + inputs={'X': abs_out}, + outputs={'Out': sum_out}, + attrs={ + 'dim': axis, + 'keep_dim': keepdim, + 'reduce_all': reduce_all, + }, + ) if porder == 1 or porder == np.inf: - block.append_op(type='reduce_max', - inputs={'X': sum_out}, - outputs={'Out': out}, - attrs={ - 'dim': [-1], - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + block.append_op( + type='reduce_max', + inputs={'X': sum_out}, + outputs={'Out': out}, + attrs={ + 'dim': [-1], + 'keep_dim': keepdim, + 'reduce_all': reduce_all, + }, + ) if porder == -1 or porder == -np.inf: - block.append_op(type='reduce_min', - inputs={'X': sum_out}, - outputs={'Out': out}, - attrs={ - 'dim': [-1], - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + block.append_op( + type='reduce_min', + inputs={'X': sum_out}, + outputs={'Out': out}, + attrs={ + 'dim': [-1], + 'keep_dim': keepdim, + 'reduce_all': reduce_all, + }, + ) return out def fro_norm(input, porder=2, axis=[-1]): @@ -851,50 +937,66 @@ def fro_norm(input, porder=2, axis=[-1]): pow_out = _C_ops.pow(input, porder) sum_out_1 = _C_ops.sum(pow_out, axis, None, keepdim) sum_out_2 = _C_ops.sum(sum_out_1, axis, None, keepdim) - return _C_ops.pow(sum_out_2, float(1. / porder)) + return _C_ops.pow(sum_out_2, float(1.0 / porder)) elif paddle.in_dynamic_mode(): pow_out = _legacy_C_ops.pow(input, 'factor', porder) - sum_out_1 = _legacy_C_ops.reduce_sum(pow_out, 'dim', axis, - 'keepdim', keepdim, - 'reduce_all', reduce_all) - sum_out_2 = _legacy_C_ops.reduce_sum(sum_out_1, 'dim', axis, - 'keepdim', keepdim, - 'reduce_all', reduce_all) - return _legacy_C_ops.pow(sum_out_2, 'factor', float(1. / porder)) + sum_out_1 = _legacy_C_ops.reduce_sum( + pow_out, + 'dim', + axis, + 'keepdim', + keepdim, + 'reduce_all', + reduce_all, + ) + sum_out_2 = _legacy_C_ops.reduce_sum( + sum_out_1, + 'dim', + axis, + 'keepdim', + keepdim, + 'reduce_all', + reduce_all, + ) + return _legacy_C_ops.pow(sum_out_2, 'factor', float(1.0 / porder)) block = LayerHelper('norm', **locals()) pow_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) sum_out_1 = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) sum_out_2 = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) - block.append_op(type='pow', - inputs={'X': input}, - outputs={'Out': pow_out}, - attrs={'factor': porder}) - block.append_op(type='reduce_sum', - inputs={'X': pow_out}, - outputs={'Out': sum_out_1}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) - block.append_op(type='reduce_sum', - inputs={'X': sum_out_1}, - outputs={'Out': sum_out_2}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) - block.append_op(type='pow', - inputs={'X': sum_out_2}, - outputs={'Out': out}, - attrs={'factor': float(1. / porder)}) + dtype=block.input_dtype() + ) + block.append_op( + type='pow', + inputs={'X': input}, + outputs={'Out': pow_out}, + attrs={'factor': porder}, + ) + block.append_op( + type='reduce_sum', + inputs={'X': pow_out}, + outputs={'Out': sum_out_1}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) + block.append_op( + type='reduce_sum', + inputs={'X': sum_out_1}, + outputs={'Out': sum_out_2}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) + block.append_op( + type='pow', + inputs={'X': sum_out_2}, + outputs={'Out': out}, + attrs={'factor': float(1.0 / porder)}, + ) return out def svd_norm(input, porder, axis=[-1]): @@ -913,9 +1015,15 @@ def svd_norm(input, porder, axis=[-1]): if in_dygraph_mode(): return _C_ops.sum(s, axis, None, keepdim) else: - return _legacy_C_ops.reduce_sum(s, 'dim', axis, 'keepdim', - keepdim, 'reduce_all', - reduce_all) + return _legacy_C_ops.reduce_sum( + s, + 'dim', + axis, + 'keepdim', + keepdim, + 'reduce_all', + reduce_all, + ) if in_dygraph_mode(): max_out = _C_ops.max(s, axis, keepdim) min_out = _C_ops.min(s, axis, keepdim) @@ -925,75 +1033,70 @@ def svd_norm(input, porder, axis=[-1]): return _C_ops.divide(min_out, max_out) else: - max_out = _legacy_C_ops.reduce_max(s, 'dim', axis, 'keepdim', - keepdim, 'reduce_all', - reduce_all) - min_out = _legacy_C_ops.reduce_min(s, 'dim', axis, 'keepdim', - keepdim, 'reduce_all', - reduce_all) + max_out = _legacy_C_ops.reduce_max( + s, 'dim', axis, 'keepdim', keepdim, 'reduce_all', reduce_all + ) + min_out = _legacy_C_ops.reduce_min( + s, 'dim', axis, 'keepdim', keepdim, 'reduce_all', reduce_all + ) if porder == 2: return _legacy_C_ops.elementwise_div( - max_out, min_out, 'aixs', axis, 'use_mkldnn', False) + max_out, min_out, 'aixs', axis, 'use_mkldnn', False + ) if porder == -2: return _legacy_C_ops.elementwise_div( - min_out, max_out, 'aixs', axis, 'use_mkldnn', False) + min_out, max_out, 'aixs', axis, 'use_mkldnn', False + ) block = LayerHelper('norm', **locals()) out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) if porder == "nuc": - block.append_op(type='reduce_sum', - inputs={'X': s}, - outputs={'Out': out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + block.append_op( + type='reduce_sum', + inputs={'X': s}, + outputs={'Out': out}, + attrs={ + 'dim': axis, + 'keep_dim': keepdim, + 'reduce_all': reduce_all, + }, + ) return out max_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) + dtype=block.input_dtype() + ) min_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) - block.append_op(type='reduce_max', - inputs={'X': s}, - outputs={'Out': max_out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) - block.append_op(type='reduce_min', - inputs={'X': s}, - outputs={'Out': min_out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + dtype=block.input_dtype() + ) + block.append_op( + type='reduce_max', + inputs={'X': s}, + outputs={'Out': max_out}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) + block.append_op( + type='reduce_min', + inputs={'X': s}, + outputs={'Out': min_out}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) if porder == 2: - block.append_op(type='elementwise_div', - inputs={ - 'X': max_out, - 'Y': min_out - }, - outputs={'Out': out}, - attrs={ - 'aixs': axis, - 'use_mkldnn': False - }) + block.append_op( + type='elementwise_div', + inputs={'X': max_out, 'Y': min_out}, + outputs={'Out': out}, + attrs={'aixs': axis, 'use_mkldnn': False}, + ) return out if porder == -2: - block.append_op(type='elementwise_div', - inputs={ - 'X': min_out, - 'Y': max_out - }, - outputs={'Out': out}, - attrs={ - 'aixs': axis, - 'use_mkldnn': False - }) + block.append_op( + type='elementwise_div', + inputs={'X': min_out, 'Y': max_out}, + outputs={'Out': out}, + attrs={'aixs': axis, 'use_mkldnn': False}, + ) return out def empty_tensor(input, shape): @@ -1004,8 +1107,9 @@ def empty_tensor(input, shape): x_shape = list(x.shape) if not len(x_shape) >= 2: raise ValueError( - "input should be a matrix or batches of matrices, " + - "but the dimention of received input is {}".format(len(x_shape))) + "input should be a matrix or batches of matrices, " + + "but the dimention of received input is {}".format(len(x_shape)) + ) if p == None: p = 2 x_size = 0 if (0 in x_shape) else 1 @@ -1020,21 +1124,26 @@ def empty_tensor(input, shape): return svd_norm(x, p) * svd_norm(x_inv, p) if p in (1, -1): return mat_norm(x, porder=p, axis=[-2]) * mat_norm( - x_inv, porder=p, axis=[-2]) + x_inv, porder=p, axis=[-2] + ) if p in (np.inf, -np.inf): return mat_norm(x, porder=p, axis=[-1]) * mat_norm( - x_inv, porder=p, axis=[-1]) + x_inv, porder=p, axis=[-1] + ) else: - raise ValueError("only support p is {} when input is a ".format(p) + - "square matrix or batches of square matrices") + raise ValueError( + "only support p is {} when input is a ".format(p) + + "square matrix or batches of square matrices" + ) elif p in (2, -2): if x_size == 0: return empty_tensor(x, x_shape[:-2]) return svd_norm(x, porder=p) else: raise ValueError( - "unsupported {} for p, only supporting ('fro', 'nuc', ".format(p) + - "1, -1, 2, -2, inf, -inf) or none") + "unsupported {} for p, only supporting ('fro', 'nuc', ".format(p) + + "1, -1, 2, -2, inf, -inf) or none" + ) def dot(x, y, name=None): @@ -1082,25 +1191,23 @@ def dot(x, y, name=None): assert x is not None, 'x cannot be None in {}'.format(op_type) assert y is not None, 'y cannot be None in {}'.format(op_type) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - op_type) - check_variable_and_dtype(y, 'y', ['float32', 'float64', 'int32', 'int64'], - op_type) + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], op_type + ) + check_variable_and_dtype( + y, 'y', ['float32', 'float64', 'int32', 'int64'], op_type + ) helper = LayerHelper(op_type, **locals()) if name is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) else: - out = helper.create_variable(name=name, - dtype=x.dtype, - persistable=False) - helper.append_op(type="dot", - inputs={ - 'X': x, - 'Y': y - }, - attrs={}, - outputs={"Out": out}) + out = helper.create_variable( + name=name, dtype=x.dtype, persistable=False + ) + helper.append_op( + type="dot", inputs={'X': x, 'Y': y}, attrs={}, outputs={"Out": out} + ) return out @@ -1143,7 +1250,8 @@ def cov(x, rowvar=True, ddof=True, fweights=None, aweights=None, name=None): if len(x.shape) > 2 or len(x.shape) < 1: raise ValueError( "Input(x) only support N-D (1<=N<=2) tensor in cov, but received " - "length of Input(input) is %s." % len(x.shape)) + "length of Input(input) is %s." % len(x.shape) + ) check_variable_and_dtype(x, 'dtype', ['float32', 'float64'], 'cov') nx = x if len(x.shape) == 1: @@ -1157,16 +1265,20 @@ def cov(x, rowvar=True, ddof=True, fweights=None, aweights=None, name=None): if len(w.shape) > 1: raise ValueError( "Input(fweights) only support N-D (N<=1) tensor in cov, but received " - "shape of Input(input) is %s." % len(fweights.shape)) + "shape of Input(input) is %s." % len(fweights.shape) + ) if fweights.shape[0] != observation_num: raise ValueError( "The number of Input(fweights) should equal to x's dim[1]: {}, but received " - "size of Input(fweights) is {}.".format(observation_num, - fweights.shape[0])) + "size of Input(fweights) is {}.".format( + observation_num, fweights.shape[0] + ) + ) if fweights.min() < 0: raise ValueError( "The value of Input(fweights) cannot be negtive, but received " - "min of Input(fweights) is {}.".format(fweights.min())) + "min of Input(fweights) is {}.".format(fweights.min()) + ) if not paddle.all(fweights == paddle.round(fweights.astype('float64'))): raise ValueError("Input(fweights) must be integer ") @@ -1175,18 +1287,23 @@ def cov(x, rowvar=True, ddof=True, fweights=None, aweights=None, name=None): if len(aw.shape) > 1: raise ValueError( "Input(aweights) only support N-D (N<=1) tensor in cov, but received " - "length of Input(input) is %s." % len(aweights.shape)) - check_variable_and_dtype(aweights, 'dtype', ['float32', 'float64'], - 'cov') + "length of Input(input) is %s." % len(aweights.shape) + ) + check_variable_and_dtype( + aweights, 'dtype', ['float32', 'float64'], 'cov' + ) if aweights.shape[0] != observation_num: raise ValueError( "The number of Input(aweights) should equal to x's dim[1]: {}, but received " - "size of Input(aweights) is {}.".format(observation_num, - aweights.shape[0])) + "size of Input(aweights) is {}.".format( + observation_num, aweights.shape[0] + ) + ) if aweights.min() < 0: raise ValueError( "The value of Input(aweights) cannot be negtive, but received " - "min of Input(aweights) is {}.".format(aweights.min())) + "min of Input(aweights) is {}.".format(aweights.min()) + ) if w is not None: w = w * aw else: @@ -1260,7 +1377,8 @@ def t(input, name=None): raise ValueError( "Input(input) only support N-D (N<=2) tensor, but received " "length of Input(input) is %s. Perhaps you can use paddle." - "tensor.transpose() instead." % len(input.shape)) + "tensor.transpose() instead." % len(input.shape) + ) if in_dygraph_mode(): if len(input.shape) == 1: return input @@ -1278,8 +1396,11 @@ def t(input, name=None): return out check_variable_and_dtype( - input, 'input', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'transpose') + input, + 'input', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'transpose', + ) helper = LayerHelper('t', **locals()) out = helper.create_variable_for_type_inference(input.dtype) @@ -1287,13 +1408,12 @@ def t(input, name=None): if len(input.shape) == 1: out = input else: - helper.append_op(type='transpose2', - inputs={'X': [input]}, - outputs={ - 'Out': [out], - 'XShape': [input_shape] - }, - attrs={'axis': [1, 0]}) + helper.append_op( + type='transpose2', + inputs={'X': [input]}, + outputs={'Out': [out], 'XShape': [input_shape]}, + attrs={'axis': [1, 0]}, + ) return out @@ -1350,13 +1470,12 @@ def cross(x, y, axis=9, name=None): attrs = dict() attrs['dim'] = axis - helper.append_op(type='cross', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='cross', + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs=attrs, + ) return out @@ -1411,10 +1530,12 @@ def cholesky(x, upper=False, name=None): check_type(upper, 'upper', bool, 'cholesky') helper = LayerHelper('cholesky', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='cholesky', - inputs={'X': [x]}, - outputs={'Out': out}, - attrs={'upper': upper}) + helper.append_op( + type='cholesky', + inputs={'X': [x]}, + outputs={'Out': out}, + attrs={'upper': upper}, + ) return out @@ -1464,8 +1585,9 @@ def matrix_rank(x, tol=None, hermitian=False, name=None): else: tol_tensor = tol use_default_tol = False - return _C_ops.matrix_rank_tol(x, tol_tensor, use_default_tol, - hermitian) + return _C_ops.matrix_rank_tol( + x, tol_tensor, use_default_tol, hermitian + ) if tol is None: tol_attr = 0.0 @@ -1491,9 +1613,16 @@ def matrix_rank(x, tol=None, hermitian=False, name=None): tol_tensor = None tol_attr = float(tol) use_default_tol = False - return _legacy_C_ops.matrix_rank(x, tol_tensor, "tol", tol_attr, - 'hermitian', hermitian, - 'use_default_tol', use_default_tol) + return _legacy_C_ops.matrix_rank( + x, + tol_tensor, + "tol", + tol_attr, + 'hermitian', + hermitian, + 'use_default_tol', + use_default_tol, + ) inputs = {} attrs = {} @@ -1516,10 +1645,9 @@ def matrix_rank(x, tol=None, hermitian=False, name=None): helper = LayerHelper('matrix_rank', **locals()) out = helper.create_variable_for_type_inference(dtype='int32') - helper.append_op(type='matrix_rank', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='matrix_rank', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -1566,16 +1694,22 @@ def bmm(x, y, name=None): y_shape = y.shape if not len(x_shape) == len(y_shape) == 3: raise ValueError( - "x and y should be 3-dimensional. But received x's dimention: {}, y's dimention: {}" - .format(x_shape, y_shape)) + "x and y should be 3-dimensional. But received x's dimention: {}, y's dimention: {}".format( + x_shape, y_shape + ) + ) if x_shape[2] != y_shape[1]: raise ValueError( - "x's width must be equal with y's height. But received x's shape: {}, y's shape: {}" - .format(x_shape, y_shape)) + "x's width must be equal with y's height. But received x's shape: {}, y's shape: {}".format( + x_shape, y_shape + ) + ) if x_shape[0] != y_shape[0]: raise ValueError( - "x's batch (shape[0]) must be equal with y's batch (shape[0]). But received x's shape: {}, y's shape: {}" - .format(x_shape, y_shape)) + "x's batch (shape[0]) must be equal with y's batch (shape[0]). But received x's shape: {}, y's shape: {}".format( + x_shape, y_shape + ) + ) if in_dygraph_mode(): return _C_ops.bmm(x, y) @@ -1618,22 +1752,21 @@ def histogram(input, bins=100, min=0, max=0, name=None): return _C_ops.histogram(input, bins, min, max) if _in_legacy_dygraph(): - return _legacy_C_ops.histogram(input, "bins", bins, "min", min, "max", - max) + return _legacy_C_ops.histogram( + input, "bins", bins, "min", min, "max", max + ) helper = LayerHelper('histogram', **locals()) - check_variable_and_dtype(input, 'X', - ['int32', 'int64', 'float32', 'float64'], - 'histogram') + check_variable_and_dtype( + input, 'X', ['int32', 'int64', 'float32', 'float64'], 'histogram' + ) out = helper.create_variable_for_type_inference(VarDesc.VarType.INT64) - helper.append_op(type='histogram', - inputs={'X': input}, - outputs={'Out': out}, - attrs={ - 'bins': bins, - 'min': min, - 'max': max - }) + helper.append_op( + type='histogram', + inputs={'X': input}, + outputs={'Out': out}, + attrs={'bins': bins, 'min': min, 'max': max}, + ) return out @@ -1677,19 +1810,21 @@ def bincount(x, weights=None, minlength=0, name=None): check_variable_and_dtype(x, 'X', ['int32', 'int64'], 'bincount') if weights is not None: - check_variable_and_dtype(weights, 'Weights', - ['int32', 'int64', 'float32', 'float64'], - 'bincount') + check_variable_and_dtype( + weights, + 'Weights', + ['int32', 'int64', 'float32', 'float64'], + 'bincount', + ) out = helper.create_variable_for_type_inference(dtype=weights.dtype) else: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='bincount', - inputs={ - 'X': x, - 'Weights': weights - }, - outputs={'Out': out}, - attrs={'minlength': minlength}) + helper.append_op( + type='bincount', + inputs={'X': x, 'Weights': weights}, + outputs={'Out': out}, + attrs={'minlength': minlength}, + ) return out @@ -1734,29 +1869,31 @@ def mv(x, vec, name=None): def __check_input(x, vec): var_names = {'x': x, 'vec': vec} for name, val in var_names.items(): - check_variable_and_dtype(val, name, ['float32', 'float64'], - 'mv') + check_variable_and_dtype( + val, name, ['float32', 'float64'], 'mv' + ) x_shape = list(x.shape) vec_shape = list(vec.shape) if len(x_shape) != 2: raise ValueError( - "x should be 2-dimensional. But received x's dimention: {}" - .format(x_shape)) + "x should be 2-dimensional. But received x's dimention: {}".format( + x_shape + ) + ) if len(vec_shape) != 1: raise ValueError( - "vec should be 1-dimensional. But received vec's dimention: {}" - .format(vec_shape)) + "vec should be 1-dimensional. But received vec's dimention: {}".format( + vec_shape + ) + ) __check_input(x, vec) helper = LayerHelper('mv', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='mv', - inputs={ - 'X': x, - 'Vec': vec - }, - outputs={'Out': out}) + helper.append_op( + type='mv', inputs={'X': x, 'Vec': vec}, outputs={'Out': out} + ) return out @@ -1796,22 +1933,23 @@ def det(x, name=None): check_dtype(x.dtype, 'Input', ['float32', 'float64'], 'det') input_shape = list(x.shape) - assert len(input_shape) >= 2, \ - "The x must be at least 2-dimensional, " \ - "but received Input x's dimensional: %s.\n" % \ - len(input_shape) - - assert (input_shape[-1] == input_shape[-2]), \ - "Expect squared input," \ - "but received %s by %s matrix.\n" \ - %(input_shape[-2], input_shape[-1]) \ + assert len(input_shape) >= 2, ( + "The x must be at least 2-dimensional, " + "but received Input x's dimensional: %s.\n" % len(input_shape) + ) + assert ( + input_shape[-1] == input_shape[-2] + ), "Expect squared input," "but received %s by %s matrix.\n" % ( + input_shape[-2], + input_shape[-1], + ) helper = LayerHelper('determinant', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='determinant', - inputs={'Input': [x]}, - outputs={'Out': [out]}) + helper.append_op( + type='determinant', inputs={'Input': [x]}, outputs={'Out': [out]} + ) return out @@ -1855,22 +1993,23 @@ def slogdet(x, name=None): check_dtype(x.dtype, 'Input', ['float32', 'float64'], 'slogdet') input_shape = list(x.shape) - assert len(input_shape) >= 2, \ - "The x must be at least 2-dimensional, " \ - "but received Input x's dimensional: %s.\n" % \ - len(input_shape) - - assert (input_shape[-1] == input_shape[-2]), \ - "Expect squared input," \ - "but received %s by %s matrix.\n" \ - %(input_shape[-2], input_shape[-1]) \ + assert len(input_shape) >= 2, ( + "The x must be at least 2-dimensional, " + "but received Input x's dimensional: %s.\n" % len(input_shape) + ) + assert ( + input_shape[-1] == input_shape[-2] + ), "Expect squared input," "but received %s by %s matrix.\n" % ( + input_shape[-2], + input_shape[-1], + ) helper = LayerHelper('slogdeterminant', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='slogdeterminant', - inputs={'Input': [x]}, - outputs={'Out': [out]}) + helper.append_op( + type='slogdeterminant', inputs={'Input': [x]}, outputs={'Out': [out]} + ) return out @@ -1937,11 +2076,7 @@ def svd(x, full_matrices=False, name=None): helper.append_op( type='svd', inputs={'X': [x]}, - outputs={ - 'U': u, - 'VH': vh, - 'S': s - }, + outputs={'U': u, 'VH': vh, 'S': s}, attrs=attrs, ) return u, s, vh @@ -2010,10 +2145,12 @@ def matrix_power(x, n, name=None): check_type(n, 'n', int, 'matrix_power') helper = LayerHelper('matrix_power', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='matrix_power', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'n': n}) + helper.append_op( + type='matrix_power', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'n': n}, + ) return out @@ -2078,13 +2215,9 @@ def qr(x, mode="reduced", name=None): r = helper.create_variable_for_type_inference(dtype=x.dtype) attrs = dict() attrs['mode'] = mode - helper.append_op(type='qr', - inputs={'X': [x]}, - outputs={ - 'Q': q, - 'R': r - }, - attrs=attrs) + helper.append_op( + type='qr', inputs={'X': [x]}, outputs={'Q': q, 'R': r}, attrs=attrs + ) if mode == "r": return r else: @@ -2183,14 +2316,12 @@ def lu(x, pivot=True, get_infos=False, name=None): info = helper.create_variable_for_type_inference(dtype='int') attrs = dict() attrs['pivot'] = pivot - helper.append_op(type='lu', - inputs={'X': x}, - outputs={ - 'Out': lu, - 'Pivots': p, - 'Infos': info - }, - attrs=attrs) + helper.append_op( + type='lu', + inputs={'X': x}, + outputs={'Out': lu, 'Pivots': p, 'Infos': info}, + attrs=attrs, + ) if get_infos: return lu, p, info else: @@ -2275,8 +2406,9 @@ def lu_unpack(x, y, unpack_ludata=True, unpack_pivots=True, name=None): return P, L, U if paddle.in_dynamic_mode(): - P, L, U = _legacy_C_ops.lu_unpack(x, y, 'unpack_ludata', unpack_ludata, - 'unpack_pivots', unpack_pivots) + P, L, U = _legacy_C_ops.lu_unpack( + x, y, 'unpack_ludata', unpack_ludata, 'unpack_pivots', unpack_pivots + ) return P, L, U check_variable_and_dtype(x, 'dtype', ['float32', 'float64'], 'lu_unpack') @@ -2288,17 +2420,12 @@ def lu_unpack(x, y, unpack_ludata=True, unpack_pivots=True, name=None): attrs = dict() attrs['unpack_ludata'] = unpack_ludata attrs['unpack_pivots'] = unpack_pivots - helper.append_op(type='lu_unpack', - inputs={ - 'X': x, - 'Pivots': y - }, - outputs={ - 'Pmat': p, - 'L': l, - 'U': u - }, - attrs=attrs) + helper.append_op( + type='lu_unpack', + inputs={'X': x, 'Pivots': y}, + outputs={'Pmat': p, 'L': l, 'U': u}, + attrs=attrs, + ) return p, l, u @@ -2354,9 +2481,9 @@ def eig(x, name=None): w, v = _legacy_C_ops.eig(x) return w, v - check_variable_and_dtype(x, 'X', - ['float32', 'float64', 'complex64', 'complex128'], - 'eig') + check_variable_and_dtype( + x, 'X', ['float32', 'float64', 'complex64', 'complex128'], 'eig' + ) helper = LayerHelper('eig', **locals()) w = helper.create_variable_for_type_inference(x.dtype) @@ -2405,20 +2532,24 @@ def eigvals(x, name=None): # [(-0.27078833542132674+0j), (0.29962280156230725+0j), (0.8824477020120244+0j)] #complex128 """ - check_variable_and_dtype(x, 'dtype', - ['float32', 'float64', 'complex64', 'complex128'], - 'eigvals') + check_variable_and_dtype( + x, 'dtype', ['float32', 'float64', 'complex64', 'complex128'], 'eigvals' + ) x_shape = list(x.shape) if len(x_shape) < 2: raise ValueError( - "The dimension of Input(x) should be at least 2, but received x's dimention = {}, x's shape = {}" - .format(len(x_shape), x_shape)) + "The dimension of Input(x) should be at least 2, but received x's dimention = {}, x's shape = {}".format( + len(x_shape), x_shape + ) + ) if x_shape[-1] != x_shape[-2]: raise ValueError( - "The last two dimensions of Input(x) should be equal, but received x's shape = {}" - .format(x_shape)) + "The last two dimensions of Input(x) should be equal, but received x's shape = {}".format( + x_shape + ) + ) if in_dygraph_mode(): return _C_ops.eigvals(x) @@ -2494,11 +2625,16 @@ def multi_dot(x, name=None): check_type(x, 'x', (list, tuple), 'multi_dot') for id, item in enumerate(x): - check_variable_and_dtype(item, 'x[' + str(id) + ']', - ['float16', 'float32', 'float64'], 'multi_dot') + check_variable_and_dtype( + item, + 'x[' + str(id) + ']', + ['float16', 'float32', 'float64'], + 'multi_dot', + ) if item.dtype != x[0].dtype: raise TypeError( - "All the Tensors in the input must have the same data type.") + "All the Tensors in the input must have the same data type." + ) helper = LayerHelper('multi_dot', **locals()) dtype = helper.input_dtype(input_param_name='x') @@ -2551,32 +2687,35 @@ def __check_input(x, UPLO): if len(x.shape) < 2: raise ValueError( "Input(input) only support >=2 tensor, but received " - "length of Input(input) is %s." % len(x.shape)) + "length of Input(input) is %s." % len(x.shape) + ) if x_shape[-1] != x_shape[-2]: raise ValueError( - "The input matrix must be batches of square matrices. But received x's dimention: {}" - .format(x_shape)) + "The input matrix must be batches of square matrices. But received x's dimention: {}".format( + x_shape + ) + ) if UPLO != 'L' and UPLO != 'U': raise ValueError( - "UPLO must be L or U. But received UPLO is: {}".format(UPLO)) + "UPLO must be L or U. But received UPLO is: {}".format(UPLO) + ) __check_input(x, UPLO) helper = LayerHelper('eigh', **locals()) - check_variable_and_dtype(x, 'dtype', - ['float32', 'float64', 'complex64', 'complex128'], - 'eigh') + check_variable_and_dtype( + x, 'dtype', ['float32', 'float64', 'complex64', 'complex128'], 'eigh' + ) out_value = helper.create_variable_for_type_inference(dtype=x.dtype) out_vector = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='eigh', - inputs={'X': x}, - outputs={ - 'Eigenvalues': out_value, - 'Eigenvectors': out_vector - }, - attrs={'UPLO': UPLO}) + helper.append_op( + type='eigh', + inputs={'X': x}, + outputs={'Eigenvalues': out_value, 'Eigenvectors': out_vector}, + attrs={'UPLO': UPLO}, + ) return out_value, out_vector @@ -2695,8 +2834,9 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None): if not hermitian: # combine svd and matmul op u, s, vt = _legacy_C_ops.svd(x, 'full_matrices', False) - max_singular_val = _legacy_C_ops.reduce_max(s, 'dim', [-1], 'keep_dim', True, \ - 'reduce_all', False) + max_singular_val = _legacy_C_ops.reduce_max( + s, 'dim', [-1], 'keep_dim', True, 'reduce_all', False + ) rcond = paddle.to_tensor(rcond, dtype=x.dtype) cutoff = rcond * max_singular_val y = float('inf') @@ -2718,15 +2858,17 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None): if in_dygraph_mode(): out_2 = _C_ops.matmul(out_1, u, False, True) else: - out_2 = _legacy_C_ops.matmul_v2(out_1, u, 'trans_x', False, - 'trans_y', True) + out_2 = _legacy_C_ops.matmul_v2( + out_1, u, 'trans_x', False, 'trans_y', True + ) return out_2 else: # combine eigh and matmul op s, u = _legacy_C_ops.eigh(x, 'UPLO', 'L') s_abs = paddle.abs(s) - max_singular_val = _legacy_C_ops.reduce_max(s_abs, 'dim', [-1], 'keep_dim', True, \ - 'reduce_all', False) + max_singular_val = _legacy_C_ops.reduce_max( + s_abs, 'dim', [-1], 'keep_dim', True, 'reduce_all', False + ) rcond = paddle.to_tensor(rcond, dtype=s.dtype) cutoff = rcond * max_singular_val y = float('inf') @@ -2745,8 +2887,9 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None): if in_dygraph_mode(): out_2 = _C_ops.matmul(out_1, u_conj, False, True) else: - out_2 = _legacy_C_ops.matmul_v2(out_1, u_conj, 'trans_x', False, - 'trans_y', True) + out_2 = _legacy_C_ops.matmul_v2( + out_1, u_conj, 'trans_x', False, 'trans_y', True + ) return out_2 else: if not hermitian: @@ -2760,23 +2903,17 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None): helper.append_op( type='svd', inputs={'X': [x]}, - outputs={ - 'U': u, - 'VH': vt, - 'S': s - }, + outputs={'U': u, 'VH': vt, 'S': s}, attrs={'full_matrices': False}, ) max_singular_val = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='reduce_max', - inputs={'X': s}, - outputs={'Out': max_singular_val}, - attrs={ - 'dim': [-1], - 'keep_dim': True, - 'reduce_all': False - }) + helper.append_op( + type='reduce_max', + inputs={'X': s}, + outputs={'Out': max_singular_val}, + attrs={'dim': [-1], 'keep_dim': True, 'reduce_all': False}, + ) rcond = full(shape=[1], fill_value=rcond, dtype=dtype) cutoff = rcond * max_singular_val @@ -2792,59 +2929,50 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None): st = helper.create_variable_for_type_inference(dtype=dtype) st_shape = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='unsqueeze2', - inputs={'X': singular}, - attrs={'axes': [-2]}, - outputs={ - 'Out': st, - 'XShape': st_shape - }) + helper.append_op( + type='unsqueeze2', + inputs={'X': singular}, + attrs={'axes': [-2]}, + outputs={'Out': st, 'XShape': st_shape}, + ) dims = list(range(len(vt.shape))) perm = dims[:-2] + [dims[-1]] + [dims[-2]] v = helper.create_variable_for_type_inference(dtype) v_shape = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='transpose2', - inputs={'X': [vt]}, - outputs={ - 'Out': [v], - 'XShape': [v_shape] - }, - attrs={'axis': perm}) + helper.append_op( + type='transpose2', + inputs={'X': [vt]}, + outputs={'Out': [v], 'XShape': [v_shape]}, + attrs={'axis': perm}, + ) out_1 = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='elementwise_mul', - inputs={ - 'X': v, - 'Y': st - }, - outputs={'Out': out_1}, - attrs={ - 'axis': -1, - 'use_mkldnn': False - }) + helper.append_op( + type='elementwise_mul', + inputs={'X': v, 'Y': st}, + outputs={'Out': out_1}, + attrs={'axis': -1, 'use_mkldnn': False}, + ) out_1 = helper.append_activation(out_1) out_2 = helper.create_variable_for_type_inference(dtype) helper.append_op( type='matmul_v2', - inputs={ - 'X': out_1, - 'Y': u - }, + inputs={'X': out_1, 'Y': u}, outputs={'Out': out_2}, - attrs={ - 'trans_x': False, - 'trans_y': True - }, + attrs={'trans_x': False, 'trans_y': True}, ) return out_2 else: helper = LayerHelper('pinv', **locals()) dtype = x.dtype check_variable_and_dtype( - x, 'dtype', ['float32', 'float64', 'complex64', 'complex128'], - 'pinv') + x, + 'dtype', + ['float32', 'float64', 'complex64', 'complex128'], + 'pinv', + ) if dtype == paddle.complex128: s_type = 'float64' @@ -2855,26 +2983,23 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None): u = helper.create_variable_for_type_inference(dtype) s = helper.create_variable_for_type_inference(s_type) - helper.append_op(type='eigh', - inputs={'X': x}, - outputs={ - 'Eigenvalues': s, - 'Eigenvectors': u - }, - attrs={'UPLO': 'L'}) + helper.append_op( + type='eigh', + inputs={'X': x}, + outputs={'Eigenvalues': s, 'Eigenvectors': u}, + attrs={'UPLO': 'L'}, + ) s_abs = helper.create_variable_for_type_inference(s_type) - helper.append_op(type='abs', - inputs={'X': s}, - outputs={'Out': s_abs}) + helper.append_op( + type='abs', inputs={'X': s}, outputs={'Out': s_abs} + ) max_singular_val = helper.create_variable_for_type_inference(s_type) - helper.append_op(type='reduce_max', - inputs={'X': s_abs}, - outputs={'Out': max_singular_val}, - attrs={ - 'dim': [-1], - 'keep_dim': True, - 'reduce_all': False - }) + helper.append_op( + type='reduce_max', + inputs={'X': s_abs}, + outputs={'Out': max_singular_val}, + attrs={'dim': [-1], 'keep_dim': True, 'reduce_all': False}, + ) rcond = full(shape=[1], fill_value=rcond, dtype=s_type) cutoff = rcond * max_singular_val @@ -2890,44 +3015,33 @@ def pinv(x, rcond=1e-15, hermitian=False, name=None): st = helper.create_variable_for_type_inference(dtype=s_type) st_shape = helper.create_variable_for_type_inference(dtype=s_type) - helper.append_op(type='unsqueeze2', - inputs={'X': singular}, - attrs={'axes': [-2]}, - outputs={ - 'Out': st, - 'XShape': st_shape - }) + helper.append_op( + type='unsqueeze2', + inputs={'X': singular}, + attrs={'axes': [-2]}, + outputs={'Out': st, 'XShape': st_shape}, + ) out_1 = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='elementwise_mul', - inputs={ - 'X': u, - 'Y': st - }, - outputs={'Out': out_1}, - attrs={ - 'axis': -1, - 'use_mkldnn': False - }) + helper.append_op( + type='elementwise_mul', + inputs={'X': u, 'Y': st}, + outputs={'Out': out_1}, + attrs={'axis': -1, 'use_mkldnn': False}, + ) out_1 = helper.append_activation(out_1) u_conj = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='conj', - inputs={'X': u}, - outputs={'Out': [u_conj]}) + helper.append_op( + type='conj', inputs={'X': u}, outputs={'Out': [u_conj]} + ) out_2 = helper.create_variable_for_type_inference(dtype) helper.append_op( type='matmul_v2', - inputs={ - 'X': out_1, - 'Y': u_conj - }, + inputs={'X': out_1, 'Y': u_conj}, outputs={'Out': out_2}, - attrs={ - 'trans_x': False, - 'trans_y': True - }, + attrs={'trans_x': False, 'trans_y': True}, ) return out_2 @@ -2984,21 +3098,15 @@ def solve(x, y, name=None): check_variable_and_dtype(y, 'y', ['float32', 'float64'], 'solve') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="solve", - inputs={ - "X": x, - "Y": y - }, - outputs={"Out": out}) + helper.append_op( + type="solve", inputs={"X": x, "Y": y}, outputs={"Out": out} + ) return out -def triangular_solve(x, - y, - upper=True, - transpose=False, - unitriangular=False, - name=None): +def triangular_solve( + x, y, upper=True, transpose=False, unitriangular=False, name=None +): r""" Computes the solution of a system of equations with a triangular coefficient. `x` is coefficient matrix `y` is multiple right-hand sides of equations. @@ -3054,9 +3162,16 @@ def triangular_solve(x, return _C_ops.triangular_solve(x, y, upper, transpose, unitriangular) if paddle.in_dynamic_mode(): - return _legacy_C_ops.triangular_solve(x, y, 'upper', upper, 'transpose', - transpose, 'unitriangular', - unitriangular) + return _legacy_C_ops.triangular_solve( + x, + y, + 'upper', + upper, + 'transpose', + transpose, + 'unitriangular', + unitriangular, + ) inputs = {"X": [x], "Y": [y]} helper = LayerHelper("triangular_solve", **locals()) @@ -3064,17 +3179,16 @@ def triangular_solve(x, check_variable_and_dtype(y, 'y', ['float32', 'float64'], 'triangular_solve') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='triangular_solve', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs={ - 'upper': upper, - 'transpose': transpose, - 'unitriangular': unitriangular - }) + helper.append_op( + type='triangular_solve', + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs={ + 'upper': upper, + 'transpose': transpose, + 'unitriangular': unitriangular, + }, + ) return out @@ -3122,13 +3236,12 @@ def cholesky_solve(x, y, upper=False, name=None): check_variable_and_dtype(y, 'y', ['float32', 'float64'], 'cholesky_solve') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='cholesky_solve', - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs={'upper': upper}) + helper.append_op( + type='cholesky_solve', + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs={'upper': upper}, + ) return out @@ -3172,36 +3285,39 @@ def __check_input(x, UPLO): if len(x.shape) < 2: raise ValueError( "Input(input) only support >=2 tensor, but received " - "length of Input(input) is %s." % len(x.shape)) + "length of Input(input) is %s." % len(x.shape) + ) if x_shape[-1] != x_shape[-2]: raise ValueError( - "The input matrix must be batches of square matrices. But received x's dimention: {}" - .format(x_shape)) + "The input matrix must be batches of square matrices. But received x's dimention: {}".format( + x_shape + ) + ) if UPLO != 'L' and UPLO != 'U': raise ValueError( - "UPLO must be L or U. But received UPLO is: {}".format(UPLO)) + "UPLO must be L or U. But received UPLO is: {}".format(UPLO) + ) __check_input(x, UPLO) helper = LayerHelper('eigvalsh', **locals()) - check_variable_and_dtype(x, 'dtype', - ['float32', 'float64', 'complex64', 'complex128'], - 'eigvalsh') + check_variable_and_dtype( + x, + 'dtype', + ['float32', 'float64', 'complex64', 'complex128'], + 'eigvalsh', + ) out_value = helper.create_variable_for_type_inference(dtype=x.dtype) out_vector = helper.create_variable_for_type_inference(dtype=x.dtype) is_test = x.stop_gradient - helper.append_op(type='eigvalsh', - inputs={'X': x}, - outputs={ - 'Eigenvalues': out_value, - 'Eigenvectors': out_vector - }, - attrs={ - 'UPLO': UPLO, - 'is_test': is_test - }) + helper.append_op( + type='eigvalsh', + inputs={'X': x}, + outputs={'Eigenvalues': out_value, 'Eigenvectors': out_vector}, + attrs={'UPLO': UPLO, 'is_test': is_test}, + ) return out_value @@ -3268,14 +3384,18 @@ def lstsq(x, y, rcond=None, driver=None, name=None): if device == "cpu": if driver not in (None, "gels", "gelss", "gelsd", "gelsy"): raise ValueError( - "Only support valid driver is 'gels', 'gelss', 'gelsd', 'gelsy' or None for CPU inputs. But got {}" - .format(driver)) + "Only support valid driver is 'gels', 'gelss', 'gelsd', 'gelsy' or None for CPU inputs. But got {}".format( + driver + ) + ) driver = "gelsy" if driver is None else driver elif "gpu" in device: if driver not in (None, "gels"): raise ValueError( - "Only support valid driver is 'gels' or None for CUDA inputs. But got {}" - .format(driver)) + "Only support valid driver is 'gels' or None for CUDA inputs. But got {}".format( + driver + ) + ) driver = "gels" if driver is None else driver else: raise RuntimeError("Only support lstsq api for CPU or CUDA device.") @@ -3296,10 +3416,12 @@ def lstsq(x, y, rcond=None, driver=None, name=None): if _non_static_mode(): if in_dygraph_mode(): solution, residuals, rank, singular_values = _C_ops.lstsq( - x, y, rcond, driver) + x, y, rcond, driver + ) else: solution, residuals, rank, singular_values = _legacy_C_ops.lstsq( - x, y, 'rcond', rcond, 'driver', driver) + x, y, 'rcond', rcond, 'driver', driver + ) if driver == "gels": rank = paddle.empty(shape=[0], dtype=paddle.int32) @@ -3310,33 +3432,29 @@ def lstsq(x, y, rcond=None, driver=None, name=None): return solution, residuals, rank, singular_values helper = LayerHelper('lstsq', **locals()) - check_variable_and_dtype(x, 'dtype', - ['float32', 'float64', 'complex64', 'complex128'], - 'lstsq') - check_variable_and_dtype(y, 'dtype', - ['float32', 'float64', 'complex64', 'complex128'], - 'lstsq') + check_variable_and_dtype( + x, 'dtype', ['float32', 'float64', 'complex64', 'complex128'], 'lstsq' + ) + check_variable_and_dtype( + y, 'dtype', ['float32', 'float64', 'complex64', 'complex128'], 'lstsq' + ) solution = helper.create_variable_for_type_inference(dtype=x.dtype) residuals = helper.create_variable_for_type_inference(dtype=x.dtype) rank = helper.create_variable_for_type_inference(dtype=paddle.int32) singular_values = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='lstsq', - inputs={ - 'X': x, - 'Y': y - }, - outputs={ - 'Solution': solution, - 'Residuals': residuals, - 'Rank': rank, - 'SingularValues': singular_values - }, - attrs={ - 'rcond': rcond, - 'driver': driver - }) + helper.append_op( + type='lstsq', + inputs={'X': x, 'Y': y}, + outputs={ + 'Solution': solution, + 'Residuals': residuals, + 'Rank': rank, + 'SingularValues': singular_values, + }, + attrs={'rcond': rcond, 'driver': driver}, + ) if driver == "gels": rank = paddle.static.data(name='rank', shape=[0]) @@ -3388,11 +3506,12 @@ def corrcoef(x, rowvar=True, name=None): if len(x.shape) > 2 or len(x.shape) < 1: raise ValueError( "Input(x) only support N-D (1<=N<=2) tensor in corrcoef, but received " - "length of Input(input) is %s." % len(x.shape)) + "length of Input(input) is %s." % len(x.shape) + ) check_variable_and_dtype(x, 'dtype', ['float32', 'float64'], 'corrcoef') c = cov(x, rowvar) - if (c.ndim == 0): + if c.ndim == 0: # scalar covariance # nan if incorrect value (nan, inf, 0), 1 otherwise return c / c @@ -3407,8 +3526,9 @@ def corrcoef(x, rowvar=True, name=None): # Clip to [-1, 1]. This does not guarantee if paddle.is_complex(c): - return paddle.complex(paddle.clip(c.real(), -1, 1), - paddle.clip(c.imag(), -1, 1)) + return paddle.complex( + paddle.clip(c.real(), -1, 1), paddle.clip(c.imag(), -1, 1) + ) else: c = paddle.clip(c, -1, 1) diff --git a/python/paddle/tensor/logic.py b/python/paddle/tensor/logic.py index 6945ab3fd3f11..bd7052153f625 100755 --- a/python/paddle/tensor/logic.py +++ b/python/paddle/tensor/logic.py @@ -16,8 +16,10 @@ from ..fluid.data_feeder import check_type, check_variable_and_dtype from .layer_function_generator import templatedoc from ..static import Variable + # TODO: define logic functions of a tensor from ..fluid.framework import _in_eager_mode_ + if _in_eager_mode_: Tensor = paddle.fluid.framework.core.eager.Tensor else: @@ -26,6 +28,7 @@ from ..framework import in_dygraph_mode from ..framework import LayerHelper from ..fluid.framework import _in_legacy_dygraph + # TODO: define logic functions of a tensor from paddle import _C_ops, _legacy_C_ops from paddle.tensor.creation import full @@ -47,14 +50,18 @@ def _logical_op(op_name, x, y, out=None, name=None, binary_op=True): else: return op(x) check_variable_and_dtype( - x, "x", + x, + "x", ["bool", "int8", "int16", "int32", "int64", "float32", "float64"], - op_name) + op_name, + ) if y is not None: check_variable_and_dtype( - y, "y", + y, + "y", ["bool", "int8", "int16", "int32", "int64", "float32", "float64"], - op_name) + op_name, + ) if out is not None: check_type(out, "out", Variable, op_name) @@ -63,18 +70,16 @@ def _logical_op(op_name, x, y, out=None, name=None, binary_op=True): if binary_op and x.dtype != y.dtype: raise ValueError( "(InvalidArgument) The DataType of %s Op's Variable must be consistent, but received %s and %s." - % (op_name, x.dtype, y.dtype)) + % (op_name, x.dtype, y.dtype) + ) if out is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) if binary_op: - helper.append_op(type=op_name, - inputs={ - "X": x, - "Y": y - }, - outputs={"Out": out}) + helper.append_op( + type=op_name, inputs={"X": x, "Y": y}, outputs={"Out": out} + ) else: helper.append_op(type=op_name, inputs={"X": x}, outputs={"Out": out}) @@ -116,12 +121,9 @@ def logical_and(x, y, out=None, name=None): if in_dygraph_mode(): return _C_ops.logical_and(x, y) - return _logical_op(op_name="logical_and", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _logical_op( + op_name="logical_and", x=x, y=y, name=name, out=out, binary_op=True + ) def logical_or(x, y, out=None, name=None): @@ -161,12 +163,9 @@ def logical_or(x, y, out=None, name=None): """ if in_dygraph_mode(): return _C_ops.logical_or(x, y) - return _logical_op(op_name="logical_or", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _logical_op( + op_name="logical_or", x=x, y=y, name=name, out=out, binary_op=True + ) def logical_xor(x, y, out=None, name=None): @@ -207,12 +206,9 @@ def logical_xor(x, y, out=None, name=None): if in_dygraph_mode(): return _C_ops.logical_xor(x, y) - return _logical_op(op_name="logical_xor", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _logical_op( + op_name="logical_xor", x=x, y=y, name=name, out=out, binary_op=True + ) @templatedoc() @@ -245,12 +241,9 @@ def logical_not(x, out=None, name=None): """ if in_dygraph_mode(): return _C_ops.logical_not(x) - return _logical_op(op_name="logical_not", - x=x, - y=None, - name=name, - out=out, - binary_op=False) + return _logical_op( + op_name="logical_not", x=x, y=None, name=name, out=out, binary_op=False + ) def is_empty(x, name=None): @@ -288,16 +281,17 @@ def is_empty(x, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.is_empty(x) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'is_empty') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'is_empty' + ) check_type(name, "name", (str, type(None)), "is_empty") helper = LayerHelper("is_empty", **locals()) cond = helper.create_variable_for_type_inference(dtype='bool') cond.stop_gradient = True - helper.append_op(type='is_empty', - inputs={'X': [x]}, - outputs={'Out': [cond]}) + helper.append_op( + type='is_empty', inputs={'X': [x]}, outputs={'Out': [cond]} + ) return cond @@ -338,12 +332,9 @@ def equal_all(x, y, name=None): helper = LayerHelper("equal_all", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') - helper.append_op(type='equal_all', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + helper.append_op( + type='equal_all', inputs={'X': [x], 'Y': [y]}, outputs={'Out': [out]} + ) return out @@ -393,8 +384,9 @@ def allclose(x, y, rtol=1e-05, atol=1e-08, equal_nan=False, name=None): if in_dygraph_mode(): return _C_ops.allclose(x, y, rtol, atol, equal_nan) if _in_legacy_dygraph(): - return _legacy_C_ops.allclose(x, y, 'rtol', str(rtol), 'atol', - str(atol), 'equal_nan', equal_nan) + return _legacy_C_ops.allclose( + x, y, 'rtol', str(rtol), 'atol', str(atol), 'equal_nan', equal_nan + ) check_variable_and_dtype(x, "input", ['float32', 'float64'], 'allclose') check_variable_and_dtype(y, "input", ['float32', 'float64'], 'allclose') check_type(rtol, 'rtol', float, 'allclose') @@ -407,10 +399,9 @@ def allclose(x, y, rtol=1e-05, atol=1e-08, equal_nan=False, name=None): inputs = {'Input': x, 'Other': y} outputs = {'Out': out} attrs = {'rtol': str(rtol), 'atol': str(atol), 'equal_nan': equal_nan} - helper.append_op(type='allclose', - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='allclose', inputs=inputs, outputs=outputs, attrs=attrs + ) return out @@ -446,8 +437,10 @@ def equal(x, y, name=None): """ if not isinstance(y, (int, bool, float, Variable)): raise TypeError( - "Type of input args must be float, bool, int or Tensor, but received type {}" - .format(type(y))) + "Type of input args must be float, bool, int or Tensor, but received type {}".format( + type(y) + ) + ) if not isinstance(y, Variable): y = full(shape=[1], dtype=x.dtype, fill_value=y) @@ -459,21 +452,26 @@ def equal(x, y, name=None): return _legacy_C_ops.equal(x, y) else: check_variable_and_dtype( - x, "x", ["bool", "float32", "float64", "int32", "int64"], - "equal") + x, + "x", + ["bool", "float32", "float64", "int32", "int64"], + "equal", + ) check_variable_and_dtype( - y, "y", ["bool", "float32", "float64", "int32", "int64"], - "equal") + y, + "y", + ["bool", "float32", "float64", "int32", "int64"], + "equal", + ) helper = LayerHelper("equal", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True - helper.append_op(type='equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + helper.append_op( + type='equal', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [out]}, + ) return out @@ -511,21 +509,26 @@ def greater_equal(x, y, name=None): return _legacy_C_ops.greater_equal(x, y) else: check_variable_and_dtype( - x, "x", ["bool", "float32", "float64", "int32", "int64"], - "greater_equal") + x, + "x", + ["bool", "float32", "float64", "int32", "int64"], + "greater_equal", + ) check_variable_and_dtype( - y, "y", ["bool", "float32", "float64", "int32", "int64"], - "greater_equal") + y, + "y", + ["bool", "float32", "float64", "int32", "int64"], + "greater_equal", + ) helper = LayerHelper("greater_equal", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True - helper.append_op(type='greater_equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + helper.append_op( + type='greater_equal', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [out]}, + ) return out @@ -562,21 +565,26 @@ def greater_than(x, y, name=None): return _legacy_C_ops.greater_than(x, y) else: check_variable_and_dtype( - x, "x", ["bool", "float32", "float64", "int32", "int64"], - "greater_than") + x, + "x", + ["bool", "float32", "float64", "int32", "int64"], + "greater_than", + ) check_variable_and_dtype( - y, "y", ["bool", "float32", "float64", "int32", "int64"], - "greater_than") + y, + "y", + ["bool", "float32", "float64", "int32", "int64"], + "greater_than", + ) helper = LayerHelper("greater_than", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True - helper.append_op(type='greater_than', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + helper.append_op( + type='greater_than', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [out]}, + ) return out @@ -615,21 +623,26 @@ def less_equal(x, y, name=None): return _legacy_C_ops.less_equal(x, y) else: check_variable_and_dtype( - x, "x", ["bool", "float32", "float64", "int32", "int64"], - "less_equal") + x, + "x", + ["bool", "float32", "float64", "int32", "int64"], + "less_equal", + ) check_variable_and_dtype( - y, "y", ["bool", "float32", "float64", "int32", "int64"], - "less_equal") + y, + "y", + ["bool", "float32", "float64", "int32", "int64"], + "less_equal", + ) helper = LayerHelper("less_equal", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True - helper.append_op(type='less_equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + helper.append_op( + type='less_equal', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [out]}, + ) return out @@ -668,21 +681,26 @@ def less_than(x, y, name=None): return _legacy_C_ops.less_than(x, y) else: check_variable_and_dtype( - x, "x", ["bool", "float32", "float64", "int32", "int64"], - "less_than") + x, + "x", + ["bool", "float32", "float64", "int32", "int64"], + "less_than", + ) check_variable_and_dtype( - y, "y", ["bool", "float32", "float64", "int32", "int64"], - "less_than") + y, + "y", + ["bool", "float32", "float64", "int32", "int64"], + "less_than", + ) helper = LayerHelper("less_than", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True - helper.append_op(type='less_than', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + helper.append_op( + type='less_than', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [out]}, + ) return out @@ -721,21 +739,26 @@ def not_equal(x, y, name=None): return _legacy_C_ops.not_equal(x, y) else: check_variable_and_dtype( - x, "x", ["bool", "float32", "float64", "int32", "int64"], - "not_equal") + x, + "x", + ["bool", "float32", "float64", "int32", "int64"], + "not_equal", + ) check_variable_and_dtype( - y, "y", ["bool", "float32", "float64", "int32", "int64"], - "not_equal") + y, + "y", + ["bool", "float32", "float64", "int32", "int64"], + "not_equal", + ) helper = LayerHelper("not_equal", **locals()) out = helper.create_variable_for_type_inference(dtype='bool') out.stop_gradient = True - helper.append_op(type='not_equal', - inputs={ - 'X': [x], - 'Y': [y] - }, - outputs={'Out': [out]}) + helper.append_op( + type='not_equal', + inputs={'X': [x], 'Y': [y]}, + outputs={'Out': [out]}, + ) return out @@ -782,11 +805,15 @@ def _bitwise_op(op_name, x, y, out=None, name=None, binary_op=True): return op(x) check_variable_and_dtype( - x, "x", ["bool", "uint8", "int8", "int16", "int32", "int64"], op_name) + x, "x", ["bool", "uint8", "int8", "int16", "int32", "int64"], op_name + ) if y is not None: check_variable_and_dtype( - y, "y", ["bool", "uint8", "int8", "int16", "int32", "int64"], - op_name) + y, + "y", + ["bool", "uint8", "int8", "int16", "int32", "int64"], + op_name, + ) if out is not None: check_type(out, "out", Variable, op_name) @@ -798,12 +825,9 @@ def _bitwise_op(op_name, x, y, out=None, name=None, binary_op=True): out = helper.create_variable_for_type_inference(dtype=x.dtype) if binary_op: - helper.append_op(type=op_name, - inputs={ - "X": x, - "Y": y - }, - outputs={"Out": out}) + helper.append_op( + type=op_name, inputs={"X": x, "Y": y}, outputs={"Out": out} + ) else: helper.append_op(type=op_name, inputs={"X": x}, outputs={"Out": out}) @@ -834,12 +858,9 @@ def bitwise_and(x, y, out=None, name=None): """ if in_dygraph_mode() and out is None: return _C_ops.bitwise_and(x, y) - return _bitwise_op(op_name="bitwise_and", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _bitwise_op( + op_name="bitwise_and", x=x, y=y, name=name, out=out, binary_op=True + ) @templatedoc() @@ -867,12 +888,9 @@ def bitwise_or(x, y, out=None, name=None): if in_dygraph_mode() and out is None: return _C_ops.bitwise_or(x, y) - return _bitwise_op(op_name="bitwise_or", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _bitwise_op( + op_name="bitwise_or", x=x, y=y, name=name, out=out, binary_op=True + ) @templatedoc() @@ -899,12 +917,9 @@ def bitwise_xor(x, y, out=None, name=None): """ if in_dygraph_mode() and out is None: return _C_ops.bitwise_xor(x, y) - return _bitwise_op(op_name="bitwise_xor", - x=x, - y=y, - name=name, - out=out, - binary_op=True) + return _bitwise_op( + op_name="bitwise_xor", x=x, y=y, name=name, out=out, binary_op=True + ) @templatedoc() @@ -930,12 +945,9 @@ def bitwise_not(x, out=None, name=None): if in_dygraph_mode() and out is None: return _C_ops.bitwise_not(x) - return _bitwise_op(op_name="bitwise_not", - x=x, - y=None, - name=name, - out=out, - binary_op=False) + return _bitwise_op( + op_name="bitwise_not", x=x, y=None, name=name, out=out, binary_op=False + ) @templatedoc() @@ -982,8 +994,9 @@ def isclose(x, y, rtol=1e-05, atol=1e-08, equal_nan=False, name=None): if in_dygraph_mode(): return _C_ops.isclose(x, y, rtol, atol, equal_nan) if _in_legacy_dygraph(): - return _legacy_C_ops.isclose(x, y, 'rtol', str(rtol), 'atol', str(atol), - 'equal_nan', equal_nan) + return _legacy_C_ops.isclose( + x, y, 'rtol', str(rtol), 'atol', str(atol), 'equal_nan', equal_nan + ) check_variable_and_dtype(x, "input", ['float32', 'float64'], 'isclose') check_variable_and_dtype(y, "input", ['float32', 'float64'], 'isclose') @@ -997,8 +1010,7 @@ def isclose(x, y, rtol=1e-05, atol=1e-08, equal_nan=False, name=None): inputs = {'Input': x, 'Other': y} outputs = {'Out': out} attrs = {'rtol': str(rtol), 'atol': str(atol), 'equal_nan': equal_nan} - helper.append_op(type='isclose', - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type='isclose', inputs=inputs, outputs=outputs, attrs=attrs + ) return out diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py index 485e18f5064ae..27e4099233420 100644 --- a/python/paddle/tensor/manipulation.py +++ b/python/paddle/tensor/manipulation.py @@ -19,9 +19,15 @@ from ..fluid.framework import _in_legacy_dygraph, _non_static_mode from ..framework import LayerHelper from ..framework import convert_np_dtype_to_dtype_, dygraph_only -from ..fluid.data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype +from ..fluid.data_feeder import ( + convert_dtype, + check_variable_and_dtype, + check_type, + check_dtype, +) from ..fluid.layers import utils import numpy as np + # TODO: define functions to manipulate a tensor from ..fluid.dygraph.inplace_utils import inplace_apis_in_dygraph_only import paddle @@ -70,25 +76,50 @@ def cast(x, dtype): out = _legacy_C_ops.cast(x, 'in_dtype', x.dtype, 'out_dtype', dtype) return out - check_variable_and_dtype(x, 'x', [ - 'bool', 'float16', 'float32', 'float64', 'int16', 'int32', 'int64', - 'uint8', 'uint16' - ], 'cast') - check_dtype(dtype, 'dtype', [ - 'bool', 'float16', 'float32', 'float64', 'int8', 'int16', 'int32', - 'int64', 'uint8', 'uint16' - ], 'cast') + check_variable_and_dtype( + x, + 'x', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int16', + 'int32', + 'int64', + 'uint8', + 'uint16', + ], + 'cast', + ) + check_dtype( + dtype, + 'dtype', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int8', + 'int16', + 'int32', + 'int64', + 'uint8', + 'uint16', + ], + 'cast', + ) helper = LayerHelper('cast', **locals()) out = helper.create_variable_for_type_inference( - dtype=dtype, stop_gradient=x.stop_gradient) - helper.append_op(type='cast', - inputs={'X': [x]}, - outputs={'Out': [out]}, - attrs={ - 'in_dtype': x.dtype, - 'out_dtype': out.dtype - }) + dtype=dtype, stop_gradient=x.stop_gradient + ) + helper.append_op( + type='cast', + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={'in_dtype': x.dtype, 'out_dtype': out.dtype}, + ) return out @@ -169,7 +200,8 @@ def slice(input, axes, starts, ends): axes = list(axes) if len(axes) == 0: raise ValueError( - "Input axes should not be an empty list/tuple.") + "Input axes should not be an empty list/tuple." + ) for i in range(len(axes)): if axes[i] < 0: axes[i] = max(0, axes[i] + len(input.shape)) @@ -178,8 +210,10 @@ def slice(input, axes, starts, ends): else: raise ValueError( - "Input axes must be a python list or tuple, but reveived {}". - format(type(axes))) + "Input axes must be a python list or tuple, but reveived {}".format( + type(axes) + ) + ) infer_flags = list(1 for i in range(len(axes))) @@ -188,7 +222,8 @@ def slice(input, axes, starts, ends): if isinstance(starts, (list, tuple)): starts = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item + if isinstance(item, tmp_tensor_type) + else item for item in starts ] elif isinstance(starts, tmp_tensor_type): @@ -199,7 +234,9 @@ def slice(input, axes, starts, ends): if isinstance(ends, (list, tuple)): ends = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item for item in ends + if isinstance(item, tmp_tensor_type) + else item + for item in ends ] elif isinstance(ends, tmp_tensor_type): tensor_t = ends.numpy() @@ -217,7 +254,8 @@ def slice(input, axes, starts, ends): axes = list(axes) if len(axes) == 0: raise ValueError( - "Input axes should not be an empty list/tuple.") + "Input axes should not be an empty list/tuple." + ) for i in range(len(axes)): if axes[i] < 0: axes[i] = max(0, axes[i] + len(input.shape)) @@ -226,8 +264,10 @@ def slice(input, axes, starts, ends): else: raise ValueError( - "Input axes must be a python list or tuple, but reveived {}" - .format(type(axes))) + "Input axes must be a python list or tuple, but reveived {}".format( + type(axes) + ) + ) infer_flags = list(1 for i in range(len(axes))) @@ -236,7 +276,8 @@ def slice(input, axes, starts, ends): if isinstance(starts, (list, tuple)): starts = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item + if isinstance(item, tmp_tensor_type) + else item for item in starts ] attrs += ('starts', starts) @@ -248,7 +289,8 @@ def slice(input, axes, starts, ends): if isinstance(ends, (list, tuple)): ends = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item + if isinstance(item, tmp_tensor_type) + else item for item in ends ] attrs += ('ends', ends) @@ -257,16 +299,27 @@ def slice(input, axes, starts, ends): ends_tensor.stop_gradient = True infer_flags = list(-1 for i in range(len(axes))) - return _legacy_C_ops.slice(input, starts_tensor, ends_tensor, None, - None, 'axes', axes, 'infer_flags', - infer_flags, *attrs) + return _legacy_C_ops.slice( + input, + starts_tensor, + ends_tensor, + None, + None, + 'axes', + axes, + 'infer_flags', + infer_flags, + *attrs, + ) if not isinstance(starts, (list, tuple, Variable)): raise ValueError( - "Input starts must be an Variable, python list or tuple.") + "Input starts must be an Variable, python list or tuple." + ) if not isinstance(ends, (list, tuple, Variable)): raise ValueError( - "Input ends must be an Variable, python list or tuple.") + "Input ends must be an Variable, python list or tuple." + ) helper = LayerHelper('slice', **locals()) @@ -313,11 +366,11 @@ def slice(input, axes, starts, ends): # infer_flags attrs['infer_flags'] = infer_flags out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype('input')) - helper.append_op(type='slice', - inputs=inputs, - attrs=attrs, - outputs={'Out': out}) + dtype=helper.input_dtype('input') + ) + helper.append_op( + type='slice', inputs=inputs, attrs=attrs, outputs={'Out': out} + ) return out @@ -379,10 +432,21 @@ def transpose(x, perm, name=None): out, _ = _legacy_C_ops.transpose2(x, 'axis', perm) return out - check_variable_and_dtype(x, 'x', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], 'transpose') + check_variable_and_dtype( + x, + 'x', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'transpose', + ) check_type(perm, 'perm', (list, tuple), 'transpose') if isinstance(perm, tuple): perm = list(perm) @@ -391,24 +455,25 @@ def transpose(x, perm, name=None): "Input(perm) is the permutation of dimensions of Input(x), " "its length should be equal to dimensions of Input(x), " "but received dimension of Input(x) is %s, " - "the length of Input(perm) is %s." % (len(x.shape), len(perm))) + "the length of Input(perm) is %s." % (len(x.shape), len(perm)) + ) for idx, dim in enumerate(perm): if dim >= len(x.shape): raise ValueError( "Each element in Input(perm) should be less than Input(x)'s dimension, " "but %d-th element in Input(perm) is %d which exceeds Input(x)'s " - "dimension %d." % (idx, perm[idx], len(x.shape))) + "dimension %d." % (idx, perm[idx], len(x.shape)) + ) helper = LayerHelper('transpose', **locals()) out = helper.create_variable_for_type_inference(x.dtype) x_shape = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='transpose2', - inputs={'X': [x]}, - outputs={ - 'Out': [out], - 'XShape': [x_shape] - }, - attrs={'axis': perm}) + helper.append_op( + type='transpose2', + inputs={'X': [x]}, + outputs={'Out': [out], 'XShape': [x_shape]}, + attrs={'axis': perm}, + ) return out @@ -462,13 +527,12 @@ def unstack(x, axis=0, num=None): for _ in range(num): outs.append(helper.create_variable_for_type_inference(x.dtype)) - helper.append_op(type='unstack', - inputs={'X': [x]}, - outputs={'Y': outs}, - attrs={ - 'axis': axis, - 'num': num - }) + helper.append_op( + type='unstack', + inputs={'X': [x]}, + outputs={'Y': outs}, + attrs={'axis': axis, 'num': num}, + ) return outs @@ -519,27 +583,31 @@ def shard_index(input, index_num, nshards, shard_id, ignore_value=-1): # [[-1], [1]] """ if in_dygraph_mode(): - return _C_ops.shard_index(input, index_num, nshards, shard_id, - ignore_value) + return _C_ops.shard_index( + input, index_num, nshards, shard_id, ignore_value + ) check_variable_and_dtype(input, 'input', ['int64', 'int32'], 'shard_index') op_type = 'shard_index' helper = LayerHelper(op_type, **locals()) if shard_id < 0 or shard_id >= nshards: - raise ValueError('The shard_id(%d) should be in [0, %d)' % - (shard_id, nshards)) + raise ValueError( + 'The shard_id(%d) should be in [0, %d)' % (shard_id, nshards) + ) out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type=op_type, - inputs={'X': [input]}, - outputs={'Out': out}, - attrs={ - 'index_num': index_num, - 'nshards': nshards, - 'shard_id': shard_id, - 'ignore_value': ignore_value - }, - stop_gradient=True) + helper.append_op( + type=op_type, + inputs={'X': [input]}, + outputs={'Out': out}, + attrs={ + 'index_num': index_num, + 'nshards': nshards, + 'shard_id': shard_id, + 'ignore_value': ignore_value, + }, + stop_gradient=True, + ) return out @@ -631,12 +699,15 @@ def crop(x, shape=None, offsets=None, name=None): """ helper = LayerHelper('crop_tensor', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'crop_tensor') - check_type(shape, 'shape', (list, tuple, Variable, type(None)), - 'crop_tensor') - check_type(offsets, 'offsets', (list, tuple, Variable, type(None)), - 'crop_tensor') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'crop_tensor' + ) + check_type( + shape, 'shape', (list, tuple, Variable, type(None)), 'crop_tensor' + ) + check_type( + offsets, 'offsets', (list, tuple, Variable, type(None)), 'crop_tensor' + ) if offsets is None: offsets = [0] * len(x.shape) @@ -655,25 +726,30 @@ def _attr_shape_check(shape_val): if not isinstance(shape_val, int): raise TypeError( "Attr(shape)'s dtype of Op(crop_tensor) should be int32, but received: %s." - % type(shape_val)) + % type(shape_val) + ) if shape_val == 0: raise ValueError( "Attr(shape) of Op(crop_tensor) should not be zero, but received: %s." - % str(shape_val)) + % str(shape_val) + ) if shape_val < -1: raise ValueError( "When the element in Attr(shape) of Op(crop_tensor) is negative, only -1 is supported, but received: %s." - % str(shape_val)) + % str(shape_val) + ) def _attr_offsets_check(offset_val): if not isinstance(offset_val, int): raise TypeError( "Attr(offsets)'s dtype of Op(crop_tensor) should be int32, but received: %s." - % type(offset_val)) + % type(offset_val) + ) if offset_val < 0: raise ValueError( "Attr(offsets) of Op(crop_tensor) should be greater or equal to zero, but received: %s." - % str(offset_val)) + % str(offset_val) + ) if isinstance(offsets, Variable): offsets.stop_gradient = True @@ -714,11 +790,9 @@ def _attr_offsets_check(offset_val): else: _attr_shape_check(dim_size) temp_out = helper.create_variable_for_type_inference('int32') - fill_constant([1], - 'int32', - dim_size, - force_cpu=True, - out=temp_out) + fill_constant( + [1], 'int32', dim_size, force_cpu=True, out=temp_out + ) new_shape_tensor.append(temp_out) shape_attr.append(dim_size) ipts['ShapeTensor'] = new_shape_tensor @@ -728,10 +802,12 @@ def _attr_offsets_check(offset_val): _attr_shape_check(dim_size) attrs['shape'] = shape - helper.append_op(type='crop_tensor', - inputs=ipts, - outputs={'Out': out}, - attrs=None if len(attrs) == 0 else attrs) + helper.append_op( + type='crop_tensor', + inputs=ipts, + outputs={'Out': out}, + attrs=None if len(attrs) == 0 else attrs, + ) return out @@ -763,13 +839,15 @@ def fill_(x, value): """ if not isinstance(value, (float, int)): raise TypeError( - "The type of 'value' must be int or float, but received %s." % - (type(value))) + "The type of 'value' must be int or float, but received %s." + % (type(value)) + ) if in_dygraph_mode(): return _C_ops.fill_(x, value) else: - return _legacy_C_ops.fill_any_(x, "value_float", float(value), - "value_int", int(value)) + return _legacy_C_ops.fill_any_( + x, "value_float", float(value), "value_int", int(value) + ) @dygraph_only @@ -798,10 +876,11 @@ def zero_(x): """ if in_dygraph_mode(): - return _C_ops.fill_(x, 0.) + return _C_ops.fill_(x, 0.0) else: - return _legacy_C_ops.fill_any_(x, "value_float", 0., "value_int", - int(0)) + return _legacy_C_ops.fill_any_( + x, "value_float", 0.0, "value_int", int(0) + ) @dygraph_only @@ -833,39 +912,45 @@ def fill_diagonal_(x, value, offset=0, wrap=False, name=None): helper = LayerHelper("fill_diagonal_", **locals()) check_type(x, 'X', (Variable), 'fill_diagonal_') dtype = helper.input_dtype('x') - check_dtype(dtype, 'X', - ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'fill_diagonal_') + check_dtype( + dtype, + 'X', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'fill_diagonal_', + ) check_type(value, 'value', (bool, int, float), 'fill_diagonal_') check_type(wrap, 'wrap', (bool), 'fill_diagonal_') inshape = x.shape inshapeset = set(inshape) - assert len(inshape) >= 2, ('Tensor dims should >= 2 in fill_diagonal_ API') + assert len(inshape) >= 2, 'Tensor dims should >= 2 in fill_diagonal_ API' if len(inshape) > 2: - assert len(inshapeset) == 1, ( - 'Tensor dims should be equal while input dims > 2 in fill_diagonal_ API' - ) + assert ( + len(inshapeset) == 1 + ), 'Tensor dims should be equal while input dims > 2 in fill_diagonal_ API' if in_dygraph_mode(): if len(inshape) == 2: return _C_ops.fill_diagonal_(x, value, offset, wrap) return _C_ops.fill_diagonal_(x, value, offset, True) if len(inshape) == 2: - return _legacy_C_ops.fill_diagonal_(x, 'value', value, 'offset', offset, - 'wrap', wrap) - return _legacy_C_ops.fill_diagonal_(x, 'value', value, 'offset', offset, - 'wrap', True) + return _legacy_C_ops.fill_diagonal_( + x, 'value', value, 'offset', offset, 'wrap', wrap + ) + return _legacy_C_ops.fill_diagonal_( + x, 'value', value, 'offset', offset, 'wrap', True + ) def _fill_diagonal_tensor_impl(x, y, offset=0, dim1=0, dim2=1, inplace=False): inshape = x.shape - assert dim1 < len(inshape) and dim1 >= -len(inshape), ( - 'dim1 should between [-rank,rank) in fill_diagonal_tensor_') - assert dim2 < len(inshape) and dim2 >= -len(inshape), ( - 'dim2 should between [-rank,rank) in fill_diagonal_tensor_') - assert len(inshape) >= 2, ( - 'Tensor dims should >= 2 in fill_diagonal_tensor_') + assert dim1 < len(inshape) and dim1 >= -len( + inshape + ), 'dim1 should between [-rank,rank) in fill_diagonal_tensor_' + assert dim2 < len(inshape) and dim2 >= -len( + inshape + ), 'dim2 should between [-rank,rank) in fill_diagonal_tensor_' + assert len(inshape) >= 2, 'Tensor dims should >= 2 in fill_diagonal_tensor_' dim1 %= len(inshape) dim2 %= len(inshape) @@ -873,11 +958,14 @@ def _fill_diagonal_tensor_impl(x, y, offset=0, dim1=0, dim2=1, inplace=False): for i in range(len(inshape)): if i != dim1 and i != dim2: predshape.append(inshape[i]) - diaglen = min(min(inshape[dim1], inshape[dim1] + offset), - min(inshape[dim2], inshape[dim2] - offset)) + diaglen = min( + min(inshape[dim1], inshape[dim1] + offset), + min(inshape[dim2], inshape[dim2] - offset), + ) predshape.append(diaglen) assert tuple(predshape) == tuple( - y.shape), ("the y shape should be {}".format(predshape)) + y.shape + ), "the y shape should be {}".format(predshape) if len(y.shape) == 1: y = y.reshape([1, -1]) @@ -885,14 +973,15 @@ def _fill_diagonal_tensor_impl(x, y, offset=0, dim1=0, dim2=1, inplace=False): if in_dygraph_mode(): return _C_ops.fill_diagonal_tensor_(x, y, offset, dim1, dim2) else: - return _legacy_C_ops.fill_diagonal_tensor_(x, y, 'offset', offset, - 'dim1', dim1, 'dim2', - dim2) + return _legacy_C_ops.fill_diagonal_tensor_( + x, y, 'offset', offset, 'dim1', dim1, 'dim2', dim2 + ) if in_dygraph_mode(): return _C_ops.fill_diagonal_tensor(x, y, offset, dim1, dim2) else: - return _legacy_C_ops.fill_diagonal_tensor(x, y, 'offset', offset, - 'dim1', dim1, 'dim2', dim2) + return _legacy_C_ops.fill_diagonal_tensor( + x, y, 'offset', offset, 'dim1', dim1, 'dim2', dim2 + ) def fill_diagonal_tensor_(x, y, offset=0, dim1=0, dim2=1, name=None): @@ -924,12 +1013,9 @@ def fill_diagonal_tensor_(x, y, offset=0, dim1=0, dim2=1, name=None): print(x.tolist()) #[[1.0, 2.0, 2.0], [2.0, 1.0, 2.0], [2.0, 2.0, 1.0], [2.0, 2.0, 2.0]] """ - return _fill_diagonal_tensor_impl(x, - y, - offset=offset, - dim1=dim1, - dim2=dim2, - inplace=True) + return _fill_diagonal_tensor_impl( + x, y, offset=offset, dim1=dim1, dim2=dim2, inplace=True + ) def fill_diagonal_tensor(x, y, offset=0, dim1=0, dim2=1, name=None): @@ -958,12 +1044,9 @@ def fill_diagonal_tensor(x, y, offset=0, dim1=0, dim2=1, name=None): print(nx.tolist()) #[[1.0, 2.0, 2.0], [2.0, 1.0, 2.0], [2.0, 2.0, 1.0], [2.0, 2.0, 2.0]] """ - return _fill_diagonal_tensor_impl(x, - y, - offset=offset, - dim1=dim1, - dim2=dim2, - inplace=False) + return _fill_diagonal_tensor_impl( + x, y, offset=offset, dim1=dim1, dim2=dim2, inplace=False + ) @dygraph_only @@ -1062,10 +1145,21 @@ def concat(x, axis=0, name=None): check_type(input, 'input', (list, tuple, Variable), 'concat') if not isinstance(input, Variable): for id, x in enumerate(input): - check_variable_and_dtype(x, 'input[' + str(id) + ']', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', - 'int8', 'unit8' - ], 'concat') + check_variable_and_dtype( + x, + 'input[' + str(id) + ']', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'int8', + 'unit8', + ], + 'concat', + ) if x.dtype != input[0].dtype: raise TypeError( "All the Tensors in the input must have the same data type." @@ -1076,8 +1170,11 @@ def concat(x, axis=0, name=None): if isinstance(axis, Variable): check_dtype( - axis.dtype, 'axis', ['int32', 'int64'], 'concat', - "The data type of axis must be int32 or int64 when axis is a Tensor" + axis.dtype, + 'axis', + ['int32', 'int64'], + 'concat', + "The data type of axis must be int32 or int64 when axis is a Tensor", ) helper = LayerHelper('concat', **locals()) @@ -1088,19 +1185,17 @@ def concat(x, axis=0, name=None): # This feature is supported for Dynamic-to-Static, because after transformed, the type of inputs[0] # is LOD_TENSOR_ARRAY in some scenarios. And this feature can be used in static mode. - assert len(input) == 1, "If the elements of 'input' in concat are Variable(LoDTensorArray), " \ - "number of the elements must be 1, but received %s." % len(input) + assert len(input) == 1, ( + "If the elements of 'input' in concat are Variable(LoDTensorArray), " + "number of the elements must be 1, but received %s." % len(input) + ) out_index = helper.create_variable_for_type_inference(dtype="int32") - helper.append_op(type='tensor_array_to_tensor', - inputs={'X': input[0]}, - outputs={ - 'Out': [out], - 'OutIndex': [out_index] - }, - attrs={ - 'axis': axis, - 'use_stack': False - }) + helper.append_op( + type='tensor_array_to_tensor', + inputs={'X': input[0]}, + outputs={'Out': [out], 'OutIndex': [out_index]}, + attrs={'axis': axis, 'use_stack': False}, + ) else: inputs = {'X': input} attrs = {} @@ -1110,10 +1205,9 @@ def concat(x, axis=0, name=None): else: attrs['axis'] = axis - helper.append_op(type='concat', - inputs=inputs, - outputs={'Out': [out]}, - attrs=attrs) + helper.append_op( + type='concat', inputs=inputs, outputs={'Out': [out]}, attrs=attrs + ) return out @@ -1155,17 +1249,21 @@ def broadcast_tensors(input, name=None): check_type(input, 'input', (list, tuple), 'broadcast_tensors') if num_inputs < 1: raise TypeError( - "At least 1 tensor is needed to perform broadcast_tensors") + "At least 1 tensor is needed to perform broadcast_tensors" + ) # Check input types for id, x in enumerate(input): check_variable_and_dtype( - x, 'input[' + str(id) + ']', + x, + 'input[' + str(id) + ']', ['bool', 'float32', 'float64', 'int32', 'int64'], - 'broadcast_tensors') + 'broadcast_tensors', + ) if x.dtype != input[0].dtype: raise TypeError( - "All the Tensors in the input must have the same data type.") + "All the Tensors in the input must have the same data type." + ) # Check bcast semantics output_shape_r_last_tensor_index = [] @@ -1183,8 +1281,11 @@ def broadcast_tensors(input, name=None): output_shape_r.append(shape[i]) output_shape_r_last_tensor_index.append(j) else: - invalid = (output_shape_r[i] != shape[i] - and output_shape_r[i] != 1 and shape[i] != 1) + invalid = ( + output_shape_r[i] != shape[i] + and output_shape_r[i] != 1 + and shape[i] != 1 + ) if invalid: last_index = output_shape_r_last_tensor_index[i] raise TypeError( @@ -1203,14 +1304,15 @@ def broadcast_tensors(input, name=None): while i < num_inputs: out.append( helper.create_variable_for_type_inference( - dtype=helper.input_dtype())) + dtype=helper.input_dtype() + ) + ) i += 1 inputs = {'X': input} - helper.append_op(type='broadcast_tensors', - inputs=inputs, - outputs={'Out': out}, - attrs={}) + helper.append_op( + type='broadcast_tensors', inputs=inputs, outputs={'Out': out}, attrs={} + ) return out @@ -1253,19 +1355,21 @@ def flip(x, axis, name=None): helper = LayerHelper("flip", **locals()) check_type(x, 'X', (Variable), 'flip') dtype = helper.input_dtype('x') - check_dtype(dtype, 'X', - ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], - 'flip') + check_dtype( + dtype, + 'X', + ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], + 'flip', + ) check_type(axis, 'axis', (list, tuple), 'flip') if name is None: out = helper.create_variable_for_type_inference(dtype) else: out = helper.create_variable(name=name, dtype=dtype, persistable=False) - helper.append_op(type="flip", - inputs={"X": x}, - outputs={"Out": out}, - attrs={"axis": axis}) + helper.append_op( + type="flip", inputs={"X": x}, outputs={"Out": out}, attrs={"axis": axis} + ) return out @@ -1324,9 +1428,12 @@ def rot90(x, k=1, axes=[0, 1], name=None): helper = LayerHelper("rot90", **locals()) check_type(x, 'X', (Variable), 'rot90') dtype = helper.input_dtype('x') - check_dtype(dtype, 'X', - ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], - 'rot90') + check_dtype( + dtype, + 'X', + ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], + 'rot90', + ) check_type(axes, 'axes', (list, tuple), 'rot90') input_total_dims = len(x.shape) @@ -1334,23 +1441,31 @@ def rot90(x, k=1, axes=[0, 1], name=None): if total_rot_dims != 2: raise ValueError( "expected total rotation axes == 2, but got axes = {}".format( - total_rot_dims)) + total_rot_dims + ) + ) if input_total_dims < 2: raise ValueError( "expected total dims >= 2, but got total dims = {}".format( - input_total_dims)) + input_total_dims + ) + ) if not (axes[0] != axes[1] and abs(axes[0] - axes[1]) != input_total_dims): raise ValueError( - "expected rotation axes to be different, but got axis0 = {}, and axis1 = {}" - .format(axes[0], axes[1])) + "expected rotation axes to be different, but got axis0 = {}, and axis1 = {}".format( + axes[0], axes[1] + ) + ) if not (axes[0] < input_total_dims and axes[0] >= -input_total_dims): - raise ValueError("Rotation axis0 out of range, axis0 = {}".format( - axes[0])) + raise ValueError( + "Rotation axis0 out of range, axis0 = {}".format(axes[0]) + ) if not (axes[1] < input_total_dims and axes[1] >= -input_total_dims): - raise ValueError("Rotation axis1 out of range, axis1 = {}".format( - axes[1])) + raise ValueError( + "Rotation axis1 out of range, axis1 = {}".format(axes[1]) + ) k %= 4 if k == 0: @@ -1359,8 +1474,10 @@ def rot90(x, k=1, axes=[0, 1], name=None): return flip(flip(x, axes[0]), axes[1]) axes_list = list(range(0, input_total_dims)) - (axes_list[axes[0]], axes_list[axes[1]]) = (axes_list[axes[1]], - axes_list[axes[0]]) + (axes_list[axes[0]], axes_list[axes[1]]) = ( + axes_list[axes[1]], + axes_list[axes[0]], + ) if k == 1: return transpose(flip(x, axes[1]), axes_list) else: @@ -1439,19 +1556,29 @@ def flatten(x, start_axis=0, stop_axis=-1, name=None): if not paddle.in_dynamic_mode(): check_variable_and_dtype( - x, 'x', + x, + 'x', ['float32', 'float64', 'int8', 'int16', 'int32', 'int64', 'uint8'], - 'flatten') + 'flatten', + ) x_dim = len(x.shape) - if not (isinstance(start_axis, - int)) or (start_axis > x_dim - 1) or start_axis < -x_dim: + if ( + not (isinstance(start_axis, int)) + or (start_axis > x_dim - 1) + or start_axis < -x_dim + ): raise ValueError( - "The start_axis should be a int, and in range [-rank(x), rank(x))") - if not (isinstance(stop_axis, - int)) or (stop_axis > x_dim - 1) or stop_axis < -x_dim: + "The start_axis should be a int, and in range [-rank(x), rank(x))" + ) + if ( + not (isinstance(stop_axis, int)) + or (stop_axis > x_dim - 1) + or stop_axis < -x_dim + ): raise ValueError( - "The stop_axis should be a int, and in range [-rank(x), rank(x))") + "The stop_axis should be a int, and in range [-rank(x), rank(x))" + ) if start_axis < 0: start_axis = start_axis + x_dim if stop_axis < 0: @@ -1464,22 +1591,19 @@ def flatten(x, start_axis=0, stop_axis=-1, name=None): if _in_legacy_dygraph(): dy_out, _ = _legacy_C_ops.flatten_contiguous_range( - x, 'start_axis', start_axis, 'stop_axis', stop_axis) + x, 'start_axis', start_axis, 'stop_axis', stop_axis + ) return dy_out helper = LayerHelper('flatten', **locals()) out = helper.create_variable_for_type_inference(x.dtype) x_shape = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='flatten_contiguous_range', - inputs={"X": x}, - outputs={ - 'Out': out, - 'XShape': x_shape - }, - attrs={ - "start_axis": start_axis, - "stop_axis": stop_axis - }) + helper.append_op( + type='flatten_contiguous_range', + inputs={"X": x}, + outputs={'Out': out, 'XShape': x_shape}, + attrs={"start_axis": start_axis, "stop_axis": stop_axis}, + ) return out @@ -1493,14 +1617,22 @@ def flatten_(x, start_axis=0, stop_axis=-1, name=None): raise ValueError("The input x should be a Tensor") x_dim = len(x.shape) - if not (isinstance(start_axis, - int)) or (start_axis > x_dim - 1) or start_axis < -x_dim: + if ( + not (isinstance(start_axis, int)) + or (start_axis > x_dim - 1) + or start_axis < -x_dim + ): raise ValueError( - "The start_axis should be a int, and in range [-rank(x), rank(x))") - if not (isinstance(stop_axis, - int)) or (stop_axis > x_dim - 1) or stop_axis < -x_dim: + "The start_axis should be a int, and in range [-rank(x), rank(x))" + ) + if ( + not (isinstance(stop_axis, int)) + or (stop_axis > x_dim - 1) + or stop_axis < -x_dim + ): raise ValueError( - "The stop_axis should be a int, and in range [-rank(x), rank(x))") + "The stop_axis should be a int, and in range [-rank(x), rank(x))" + ) if start_axis < 0: start_axis = start_axis + x_dim if stop_axis < 0: @@ -1513,7 +1645,8 @@ def flatten_(x, start_axis=0, stop_axis=-1, name=None): if _in_legacy_dygraph(): dy_out, _ = _legacy_C_ops.flatten_contiguous_range_( - x, 'start_axis', start_axis, 'stop_axis', stop_axis) + x, 'start_axis', start_axis, 'stop_axis', stop_axis + ) return dy_out @@ -1571,8 +1704,10 @@ def roll(x, shifts, axis=None, name=None): for i in range(len(axis)): if axis[i] >= len_origin_shape or axis[i] < -len_origin_shape: raise ValueError( - "axis is out of range, it should be in range [{}, {}), but received {}" - .format(-len_origin_shape, len_origin_shape, axis)) + "axis is out of range, it should be in range [{}, {}), but received {}".format( + -len_origin_shape, len_origin_shape, axis + ) + ) else: axis = [] @@ -1588,22 +1723,20 @@ def roll(x, shifts, axis=None, name=None): out = helper.create_variable_for_type_inference(x.dtype) if isinstance(shifts, Variable): - helper.append_op(type='roll', - inputs={ - 'X': x, - "ShiftsTensor": shifts - }, - outputs={'Out': out}, - attrs={'axis': axis}) + helper.append_op( + type='roll', + inputs={'X': x, "ShiftsTensor": shifts}, + outputs={'Out': out}, + attrs={'axis': axis}, + ) else: check_type(shifts, 'shifts', (list, tuple), 'roll') - helper.append_op(type='roll', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'axis': axis, - 'shifts': shifts - }) + helper.append_op( + type='roll', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'axis': axis, 'shifts': shifts}, + ) return out @@ -1704,42 +1837,53 @@ def stack(x, axis=0, name=None): if not isinstance(x, list) and not isinstance(x, tuple): # NOTE:(zhiqiu) Only support Variable as input if the Variable is a LOD_TENSOR_ARRAY create by create_array, array_write, array_read, etc. # In that case, Variable is array of tensors indeed. - if isinstance(x, Variable) and x.desc.type( - ) == core.VarDesc.VarType.LOD_TENSOR_ARRAY: + if ( + isinstance(x, Variable) + and x.desc.type() == core.VarDesc.VarType.LOD_TENSOR_ARRAY + ): x = [x] else: raise TypeError( - "The type of '%s' in %s must be %s, but received %s" % - ('x', 'stack', 'list[Tensor], tuple[Tensor] or TensorArray', - type(x))) + "The type of '%s' in %s must be %s, but received %s" + % ( + 'x', + 'stack', + 'list[Tensor], tuple[Tensor] or TensorArray', + type(x), + ) + ) helper = LayerHelper('stack', **locals()) out = helper.create_variable_for_type_inference(x[0].dtype) if x[0].desc.type() == core.VarDesc.VarType.LOD_TENSOR_ARRAY: - assert len(x) == 1, "If the elements of 'x' in stack are Variable(LoDTensorArray), " \ - "number of the elements must be 1, but received %s." % len(x) + assert len(x) == 1, ( + "If the elements of 'x' in stack are Variable(LoDTensorArray), " + "number of the elements must be 1, but received %s." % len(x) + ) out_index = helper.create_variable_for_type_inference(dtype="int32") for i in x: - check_variable_and_dtype(i, 'x', \ - ['float16', 'float32', 'float64', 'int32', 'int64'], 'stack') - - helper.append_op(type='tensor_array_to_tensor', - inputs={'X': x[0]}, - outputs={ - 'Out': [out], - 'OutIndex': [out_index] - }, - attrs={ - 'axis': axis, - 'use_stack': True - }) + check_variable_and_dtype( + i, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'stack', + ) + + helper.append_op( + type='tensor_array_to_tensor', + inputs={'X': x[0]}, + outputs={'Out': [out], 'OutIndex': [out_index]}, + attrs={'axis': axis, 'use_stack': True}, + ) else: - helper.append_op(type='stack', - inputs={'X': x}, - outputs={'Y': out}, - attrs={'axis': axis}) + helper.append_op( + type='stack', + inputs={'X': x}, + outputs={'Y': out}, + attrs={'axis': axis}, + ) return out @@ -1813,15 +1957,17 @@ def split(x, num_or_sections, axis=0, name=None): if utils._contain_var(num_or_sections): for index, item in enumerate(num_or_sections): if isinstance(item, Variable): - num_or_sections[index] = num_or_sections[index].numpy( - )[0] + num_or_sections[index] = num_or_sections[index].numpy()[ + 0 + ] attrs += ('sections', list(num_or_sections)) else: attrs += ('sections', list(num_or_sections)) else: raise TypeError( "The type of 'num_or_sections' in split must be int, list or tuple in imperative mode, but " - "received %s." % (type(num_or_sections))) + "received %s." % (type(num_or_sections)) + ) if in_dygraph_mode(): if isinstance(num_or_sections, int): return _C_ops.split_with_num(input, num_or_sections, dim) @@ -1832,10 +1978,21 @@ def split(x, num_or_sections, axis=0, name=None): _legacy_C_ops.split(input, out, *attrs) return out - check_variable_and_dtype(input, 'input', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'uint8', - 'int8' - ], 'split') + check_variable_and_dtype( + input, + 'input', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'uint8', + 'int8', + ], + 'split', + ) check_type(num_or_sections, 'num_or_sections', (list, int, tuple), 'split') check_type(dim, 'dim', (int, Variable), 'split') if isinstance(dim, Variable): @@ -1855,19 +2012,18 @@ def _get_SectionsTensorList(one_list): dim_size.stop_gradient = True tensor_list.append(dim_size) else: - assert (isinstance(dim_size, int)) + assert isinstance(dim_size, int) if dim_size == -1: assert unk_dim_idx == -1, ( "Only one value of 'num_or_section' in split can " - "be -1. But received num_or_section[%d] is also -1." % - idx) + "be -1. But received num_or_section[%d] is also -1." + % idx + ) unk_dim_idx = idx temp_out = helper.create_variable_for_type_inference('int32') - fill_constant([1], - 'int32', - dim_size, - force_cpu=True, - out=temp_out) + fill_constant( + [1], 'int32', dim_size, force_cpu=True, out=temp_out + ) tensor_list.append(temp_out) return tensor_list @@ -1882,31 +2038,37 @@ def _get_SectionsTensorList(one_list): if isinstance(num_or_sections, int): assert num_or_sections > 1, 'num_or_sections must be more than 1.' if isinstance(dim, int) and input_shape[dim] > 0: - assert input_shape[dim] % num_or_sections ==0, \ - "The input's size along the split dimension " \ - "must be evenly divisible by Attr(num_or_sections). " \ - "But %d is not evenly divisible by %d. " % (num_or_sections,input_shape[dim]) + assert input_shape[dim] % num_or_sections == 0, ( + "The input's size along the split dimension " + "must be evenly divisible by Attr(num_or_sections). " + "But %d is not evenly divisible by %d. " + % (num_or_sections, input_shape[dim]) + ) num = num_or_sections else: if isinstance(dim, int) and input_shape[dim] > 0: - assert len(num_or_sections) <= input_shape[ - dim], 'len(num_or_sections) must not be more than input.shape[dim].' + assert ( + len(num_or_sections) <= input_shape[dim] + ), 'len(num_or_sections) must not be more than input.shape[dim].' num = len(num_or_sections) attrs['sections'] = list( - map(lambda ele: -1 - if isinstance(ele, Variable) else ele, num_or_sections)) + map( + lambda ele: -1 if isinstance(ele, Variable) else ele, + num_or_sections, + ) + ) if utils._contain_var(num_or_sections): inputs['SectionsTensorList'] = _get_SectionsTensorList( - num_or_sections) + num_or_sections + ) outs = [ helper.create_variable_for_type_inference(dtype=helper.input_dtype()) for i in range(num) ] - helper.append_op(type='split', - inputs=inputs, - outputs={'Out': outs}, - attrs=attrs) + helper.append_op( + type='split', inputs=inputs, outputs={'Out': outs}, attrs=attrs + ) return outs @@ -1947,8 +2109,10 @@ def vsplit(x, num_or_sections, name=None): """ if x.ndim < 2: raise ValueError( - "The input tensor's dimension must be greater than 1, but got {}". - format(x.ndim)) + "The input tensor's dimension must be greater than 1, but got {}".format( + x.ndim + ) + ) return split(x, num_or_sections, axis=0, name=name) @@ -2041,10 +2205,22 @@ def squeeze(x, axis=None, name=None): return out helper = LayerHelper("squeeze", **locals()) - check_variable_and_dtype(input, 'input', [ - 'float16', 'float32', 'float64', 'bool', 'int8', 'int32', 'int64', - 'complex64', 'complex128' - ], 'squeeze') + check_variable_and_dtype( + input, + 'input', + [ + 'float16', + 'float32', + 'float64', + 'bool', + 'int8', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'squeeze', + ) check_type(axes, 'axis/axes', (int, list, tuple, Variable), 'squeeze') attrs = {} @@ -2059,13 +2235,12 @@ def squeeze(x, axis=None, name=None): out = helper.create_variable_for_type_inference(dtype=input.dtype) x_shape = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type="squeeze2", - inputs={"X": input}, - attrs=attrs, - outputs={ - "Out": out, - "XShape": x_shape - }) + helper.append_op( + type="squeeze2", + inputs={"X": input}, + attrs=attrs, + outputs={"Out": out, "XShape": x_shape}, + ) return out @@ -2092,12 +2267,14 @@ def squeeze_(x, axis=None, name=None): return out -def unique_consecutive(x, - return_inverse=False, - return_counts=False, - axis=None, - dtype="int64", - name=None): +def unique_consecutive( + x, + return_inverse=False, + return_counts=False, + axis=None, + dtype="int64", + name=None, +): r""" Eliminates all but the first element from every consecutive group of equivalent elements. @@ -2164,7 +2341,8 @@ def unique_consecutive(x, attr_dtype = convert_np_dtype_to_dtype_(dtype) if in_dygraph_mode(): out, inverse, counts = _C_ops.unique_consecutive( - x, return_inverse, return_counts, axis, attr_dtype) + x, return_inverse, return_counts, axis, attr_dtype + ) outs = [out] if return_inverse: outs.append(inverse) @@ -2175,8 +2353,16 @@ def unique_consecutive(x, return tuple(outs) elif paddle.in_dynamic_mode(): out, inverse, counts = _legacy_C_ops.unique_consecutive( - x, 'dtype', attr_dtype, 'return_inverse', return_inverse, - 'return_counts', return_counts, 'axis', axis) + x, + 'dtype', + attr_dtype, + 'return_inverse', + return_inverse, + 'return_counts', + return_counts, + 'axis', + axis, + ) outs = [out] if return_inverse: outs.append(inverse) @@ -2185,9 +2371,12 @@ def unique_consecutive(x, if len(outs) == 1: return outs[0] return tuple(outs) - check_variable_and_dtype(x, "input", - ['float32', 'float64', 'int32', 'int64'], - 'unique_consecutive') + check_variable_and_dtype( + x, + "input", + ['float32', 'float64', 'int32', 'int64'], + 'unique_consecutive', + ) check_type(return_inverse, 'return_inverse', bool, 'unique_consecutive') check_type(return_counts, 'return_counts', bool, 'unique_consecutive') check_dtype(dtype, 'dtype', ['int32', 'int64'], 'unique_consecutive') @@ -2200,34 +2389,38 @@ def unique_consecutive(x, "return_counts": return_counts, "axis": axis, } - out = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) - inverse = helper.create_variable_for_type_inference(dtype=attr_dtype, - stop_gradient=True) - counts = helper.create_variable_for_type_inference(dtype=attr_dtype, - stop_gradient=True) + out = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) + inverse = helper.create_variable_for_type_inference( + dtype=attr_dtype, stop_gradient=True + ) + counts = helper.create_variable_for_type_inference( + dtype=attr_dtype, stop_gradient=True + ) outputs = {"Out": out, "Index": inverse, "Counts": counts} outs = [out] if return_inverse: outs.append(inverse) if return_counts: outs.append(counts) - helper.append_op(type="unique_consecutive", - inputs={"X": x}, - attrs=attrs, - outputs=outputs) + helper.append_op( + type="unique_consecutive", inputs={"X": x}, attrs=attrs, outputs=outputs + ) if len(outs) == 1: return outs[0] return tuple(outs) -def unique(x, - return_index=False, - return_inverse=False, - return_counts=False, - axis=None, - dtype="int64", - name=None): +def unique( + x, + return_index=False, + return_inverse=False, + return_counts=False, + axis=None, + dtype="int64", + name=None, +): r""" Returns the unique elements of `x` in ascending order. @@ -2289,13 +2482,24 @@ def unique(x, if _non_static_mode(): if in_dygraph_mode(): out, indices, inverse, counts = _C_ops.unique( - x, return_index, return_inverse, return_counts, axis, - attr_dtype) + x, return_index, return_inverse, return_counts, axis, attr_dtype + ) if _in_legacy_dygraph(): out, inverse, indices, counts = _legacy_C_ops.unique( - x, 'dtype', attr_dtype, 'return_index', return_index, - 'return_inverse', return_inverse, 'return_counts', - return_counts, 'axis', axis, "is_sorted", True) + x, + 'dtype', + attr_dtype, + 'return_index', + return_index, + 'return_inverse', + return_inverse, + 'return_counts', + return_counts, + 'axis', + axis, + "is_sorted", + True, + ) outs = [out] if return_index: outs.append(indices) @@ -2309,8 +2513,9 @@ def unique(x, return tuple(outs) - check_variable_and_dtype(x, "input", - ['float32', 'float64', 'int32', 'int64'], 'unique') + check_variable_and_dtype( + x, "input", ['float32', 'float64', 'int32', 'int64'], 'unique' + ) check_type(return_index, 'return_index', bool, 'unique') check_type(return_inverse, 'return_inverse', bool, 'unique') check_type(return_counts, 'return_counts', bool, 'unique') @@ -2325,21 +2530,25 @@ def unique(x, "return_inverse": return_inverse, "return_counts": return_counts, "axis": axis, - "is_sorted": True + "is_sorted": True, } - out = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) - indices = helper.create_variable_for_type_inference(dtype=attr_dtype, - stop_gradient=True) - inverse = helper.create_variable_for_type_inference(dtype=attr_dtype, - stop_gradient=True) - counts = helper.create_variable_for_type_inference(dtype=attr_dtype, - stop_gradient=True) + out = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) + indices = helper.create_variable_for_type_inference( + dtype=attr_dtype, stop_gradient=True + ) + inverse = helper.create_variable_for_type_inference( + dtype=attr_dtype, stop_gradient=True + ) + counts = helper.create_variable_for_type_inference( + dtype=attr_dtype, stop_gradient=True + ) outputs = { "Out": out, "Indices": indices, "Index": inverse, - "Counts": counts + "Counts": counts, } outs = [out] if return_index: @@ -2349,10 +2558,9 @@ def unique(x, if return_counts: outs.append(counts) - helper.append_op(type="unique", - inputs={"X": x}, - attrs=attrs, - outputs=outputs) + helper.append_op( + type="unique", inputs={"X": x}, attrs=attrs, outputs=outputs + ) if len(outs) == 1: return outs[0] @@ -2424,18 +2632,23 @@ def unsqueeze(x, axis, name=None): return _C_ops.unsqueeze(input, axes) check_type(axes, 'axis/axes', (int, list, tuple, Variable), 'unsqueeze') - check_variable_and_dtype(input, 'input', [ - 'float16', - 'float32', - 'float64', - 'bool', - 'int8', - 'int16', - 'int32', - 'int64', - 'complex64', - 'complex128', - ], 'unsqueeze') + check_variable_and_dtype( + input, + 'input', + [ + 'float16', + 'float32', + 'float64', + 'bool', + 'int8', + 'int16', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'unsqueeze', + ) helper = LayerHelper("unsqueeze2", **locals()) inputs = {"X": input} attrs = {} @@ -2453,13 +2666,12 @@ def unsqueeze(x, axis, name=None): out = helper.create_variable_for_type_inference(dtype=input.dtype) x_shape = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type="unsqueeze2", - inputs=inputs, - attrs=attrs, - outputs={ - "Out": out, - "XShape": x_shape - }) + helper.append_op( + type="unsqueeze2", + inputs=inputs, + attrs=attrs, + outputs={"Out": out, "XShape": x_shape}, + ) return out @@ -2539,13 +2751,16 @@ def gather(x, index, axis=None, name=None): return _C_ops.gather(x, index, axis) if _in_legacy_dygraph(): axis = axis.item() if isinstance(axis, paddle.Tensor) else axis - return _legacy_C_ops.gather(x, index, None, "axis", axis, "overwrite", - False) + return _legacy_C_ops.gather( + x, index, None, "axis", axis, "overwrite", False + ) check_variable_and_dtype( - x, 'x', + x, + 'x', ['float16', 'float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], - 'gather') + 'gather', + ) check_variable_and_dtype(index, 'index', ['int32', 'int64'], 'gather') if isinstance(axis, Variable): @@ -2555,25 +2770,19 @@ def gather(x, index, axis=None, name=None): dtype = helper.input_dtype('x') out = helper.create_variable_for_type_inference(dtype) if not isinstance(axis, Variable): - helper.append_op(type="gather", - inputs={ - "X": x, - "Index": index - }, - attrs={ - 'axis': axis, - 'overwrite': False - }, - outputs={"Out": out}) + helper.append_op( + type="gather", + inputs={"X": x, "Index": index}, + attrs={'axis': axis, 'overwrite': False}, + outputs={"Out": out}, + ) else: - helper.append_op(type="gather", - inputs={ - "X": x, - "Index": index, - "Axis": axis - }, - attrs={"overwrite": False}, - outputs={"Out": out}) + helper.append_op( + type="gather", + inputs={"X": x, "Index": index, "Axis": axis}, + attrs={"overwrite": False}, + outputs={"Out": out}, + ) return out @@ -2613,8 +2822,9 @@ def unbind(input, axis=0): return _C_ops.unbind(input, axis) if not isinstance(axis, (int)): - raise TypeError("The type of 'axis' must be int, but received %s." % - (type(axis))) + raise TypeError( + "The type of 'axis' must be int, but received %s." % (type(axis)) + ) if isinstance(axis, np.generic): axis = np.asscalar(axis) input_shape = input.shape @@ -2626,16 +2836,19 @@ def unbind(input, axis=0): helper = LayerHelper("unbind", **locals()) check_type(input, 'input', (Variable), 'unbind') dtype = helper.input_dtype() - check_dtype(dtype, 'unbind', ['float32', 'float64', 'int32', 'int64'], - 'unbind') + check_dtype( + dtype, 'unbind', ['float32', 'float64', 'int32', 'int64'], 'unbind' + ) outs = [ helper.create_variable_for_type_inference(dtype=helper.input_dtype()) for i in range(num) ] - helper.append_op(type="unbind", - inputs={"X": input}, - outputs={"Out": outs}, - attrs={"axis": axis}) + helper.append_op( + type="unbind", + inputs={"X": input}, + outputs={"Out": outs}, + attrs={"axis": axis}, + ) return outs @@ -2717,23 +2930,25 @@ def scatter(x, index, updates, overwrite=True, name=None): return _C_ops.scatter(x, index, updates, overwrite) else: if _in_legacy_dygraph(): - return _legacy_C_ops.scatter(x, index, updates, 'overwrite', - overwrite) + return _legacy_C_ops.scatter( + x, index, updates, 'overwrite', overwrite + ) else: check_variable_and_dtype( - x, 'dtype', ['float32', 'float64', 'float16', 'int32', 'int64'], - 'scatter') + x, + 'dtype', + ['float32', 'float64', 'float16', 'int32', 'int64'], + 'scatter', + ) check_type(overwrite, 'overwrite', bool, 'scatter') helper = LayerHelper('scatter', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type="scatter", - inputs={ - "X": x, - "Ids": index, - "Updates": updates - }, - attrs={'overwrite': overwrite}, - outputs={"Out": out}) + helper.append_op( + type="scatter", + inputs={"X": x, "Ids": index, "Updates": updates}, + attrs={'overwrite': overwrite}, + outputs={"Out": out}, + ) return out @@ -2830,13 +3045,11 @@ def scatter_nd_add(x, index, updates, name=None): helper = LayerHelper('scatter_nd_add', **locals()) dtype = helper.input_dtype(input_param_name='x') output = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="scatter_nd_add", - inputs={ - "X": x, - "Index": index, - "Updates": updates - }, - outputs={"Out": output}) + helper.append_op( + type="scatter_nd_add", + inputs={"X": x, "Index": index, "Updates": updates}, + outputs={"Out": output}, + ) return output @@ -2963,7 +3176,9 @@ def tile(x, repeat_times, name=None): """ if in_dygraph_mode(): if isinstance(repeat_times, core.eager.Tensor): - assert repeat_times.ndim == 1, "Only support ndim == 1 while repeat_times is a Tensor." + assert ( + repeat_times.ndim == 1 + ), "Only support ndim == 1 while repeat_times is a Tensor." repeat_times = repeat_times.numpy().tolist() return _C_ops.tile(x, repeat_times) @@ -2973,26 +3188,30 @@ def tile(x, repeat_times, name=None): check_type(repeat_times, 'repeat_times', (list, tuple, Variable), 'tile') if isinstance(repeat_times, Variable): - assert len( - repeat_times.shape) == 1, ('repeat_times must be an 1-D Tensor.') + assert ( + len(repeat_times.shape) == 1 + ), 'repeat_times must be an 1-D Tensor.' else: for elem in repeat_times: if isinstance(elem, Variable): - assert len(elem.shape) == 1, ( - 'Elements in repeat_times must be 1-D Tensors or integers.') + assert ( + len(elem.shape) == 1 + ), 'Elements in repeat_times must be 1-D Tensors or integers.' else: type_tuple = (int, np.int32, np.int64) - assert isinstance(elem, type_tuple), ( - 'Elements in repeat_times must be 1-D Tensors or integers.') + assert isinstance( + elem, type_tuple + ), 'Elements in repeat_times must be 1-D Tensors or integers.' - check_variable_and_dtype(x, 'x', - ['bool', 'float32', 'float64', 'int32', 'int64'], - 'tile') + check_variable_and_dtype( + x, 'x', ['bool', 'float32', 'float64', 'int32', 'int64'], 'tile' + ) if convert_dtype(x.dtype) == 'bool' and x.stop_gradient == False: raise ValueError( "When the date type is bool for the input 'x' of tile op, you " "must set its stop_gradient to be True by " - "some_var.stop_gradient == True supporting some_var is the input.") + "some_var.stop_gradient == True supporting some_var is the input." + ) helper = LayerHelper('tile', **locals()) @@ -3006,8 +3225,9 @@ def get_attr_repeat_times(list_repeat_times): attrs_repeat_times.append(-1) else: attrs_repeat_times.append(times) - assert times > 0, ( - "All elements in repeat_times must be positive for tile.") + assert ( + times > 0 + ), "All elements in repeat_times must be positive for tile." return attrs_repeat_times if isinstance(repeat_times, Variable): @@ -3018,14 +3238,14 @@ def get_attr_repeat_times(list_repeat_times): attrs['repeat_times'] = get_attr_repeat_times(repeat_times) if utils._contain_var(repeat_times): inputs['repeat_times_tensor'] = utils._convert_to_tensor_list( - repeat_times) + repeat_times + ) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='tile', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='tile', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -3063,9 +3283,9 @@ def expand_as(x, y, name=None): if _non_static_mode(): return _legacy_C_ops.expand_as_v2(x, 'target_shape', y.shape) - check_variable_and_dtype(x, 'x', - ['bool', 'float32', 'float64', 'int32', 'int64'], - 'expand_as') + check_variable_and_dtype( + x, 'x', ['bool', 'float32', 'float64', 'int32', 'int64'], 'expand_as' + ) check_type(y, 'y', Variable, 'expand_as') if convert_dtype(x.dtype) == 'bool' and x.stop_gradient == False: @@ -3073,16 +3293,19 @@ def expand_as(x, y, name=None): "When the data type of input 'x' for expand_as is bool, " "you must set its stop_gradient to be False by " "some_var.stop_gradient = True, supporting " - "some_var as the input 'x'.") + "some_var as the input 'x'." + ) inputs = {"X": [x], "Y": [y]} helper = LayerHelper('expand_as', **locals()) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='expand_as_v2', - inputs=inputs, - attrs={'target_shape': y.shape}, - outputs={'Out': out}) + helper.append_op( + type='expand_as_v2', + inputs=inputs, + attrs={'target_shape': y.shape}, + outputs={'Out': out}, + ) return out @@ -3119,27 +3342,30 @@ def broadcast_to(x, shape, name=None): return _legacy_C_ops.expand_v2(x, 'shape', shape) if isinstance(shape, Variable): - assert len(shape.shape) == 1, ('shape must be an 1-D Tensor.') + assert len(shape.shape) == 1, 'shape must be an 1-D Tensor.' else: for elem in shape: if isinstance(elem, Variable): - assert len(elem.shape) == 1, ( - 'Elements in shape must be 1-D Tensors or integers.') + assert ( + len(elem.shape) == 1 + ), 'Elements in shape must be 1-D Tensors or integers.' else: type_tuple = (int, np.int32, np.int64) - assert isinstance(elem, type_tuple), ( - 'Elements in shape must be 1-D Tensors or integers.') + assert isinstance( + elem, type_tuple + ), 'Elements in shape must be 1-D Tensors or integers.' - check_variable_and_dtype(x, 'x', - ['bool', 'float32', 'float64', 'int32', 'int64'], - 'broadcast_to') + check_variable_and_dtype( + x, 'x', ['bool', 'float32', 'float64', 'int32', 'int64'], 'broadcast_to' + ) check_type(shape, 'shape', (list, tuple, Variable), 'broadcast_to') if convert_dtype(x.dtype) == 'bool' and x.stop_gradient == False: raise ValueError( "When the data type of input 'x' for broadcast_to is bool, " "you must set its stop_gradient to be False by " "some_var.stop_gradient = True, supporting " - "some_var as the input.") + "some_var as the input." + ) inputs = {"X": [x]} attrs = {} @@ -3153,9 +3379,9 @@ def get_attr_expand_shape(list_expand_shape): attrs_expand_shape.append(-1) else: attrs_expand_shape.append(shape) - assert shape > 0 or shape == -1, ( - "All elements in shape of broadcast_to must be positive or -1." - ) + assert ( + shape > 0 or shape == -1 + ), "All elements in shape of broadcast_to must be positive or -1." return attrs_expand_shape if isinstance(shape, Variable): @@ -3165,14 +3391,14 @@ def get_attr_expand_shape(list_expand_shape): attrs['shape'] = get_attr_expand_shape(shape) if utils._contain_var(shape): inputs['expand_shapes_tensor'] = utils._convert_to_tensor_list( - shape) + shape + ) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='expand_v2', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='expand_v2', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -3211,26 +3437,33 @@ def expand(x, shape, name=None): return _legacy_C_ops.expand_v2(x, 'shape', shape) if isinstance(shape, Variable): - assert len(shape.shape) == 1, ('shape must be an 1-D Tensor.') + assert len(shape.shape) == 1, 'shape must be an 1-D Tensor.' else: for elem in shape: if isinstance(elem, Variable): - assert len(elem.shape) == 1, ( - 'Elements in shape must be 1-D Tensors or integers.') + assert ( + len(elem.shape) == 1 + ), 'Elements in shape must be 1-D Tensors or integers.' else: type_tuple = (int, np.int32, np.int64) - assert isinstance(elem, type_tuple), ( - 'Elements in shape must be 1-D Tensors or integers.') + assert isinstance( + elem, type_tuple + ), 'Elements in shape must be 1-D Tensors or integers.' check_variable_and_dtype( - x, 'x', ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'expand') + x, + 'x', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'expand', + ) check_type(shape, 'shape', (list, tuple, Variable), 'expand') if convert_dtype(x.dtype) == 'bool' and x.stop_gradient == False: - raise ValueError("When the data type of input 'x' for expand is bool, " - "you must set its stop_gradient to be False by " - "some_var.stop_gradient = True, supporting " - "some_var as the input.") + raise ValueError( + "When the data type of input 'x' for expand is bool, " + "you must set its stop_gradient to be False by " + "some_var.stop_gradient = True, supporting " + "some_var as the input." + ) inputs = {"X": [x]} attrs = {} @@ -3244,8 +3477,9 @@ def get_attr_expand_shape(list_expand_shape): attrs_expand_shape.append(-2) else: attrs_expand_shape.append(shape) - assert shape > 0 or shape == -1, ( - "All elements in shape of expand must be positive or -1.") + assert ( + shape > 0 or shape == -1 + ), "All elements in shape of expand must be positive or -1." return attrs_expand_shape if isinstance(shape, Variable): @@ -3255,14 +3489,14 @@ def get_attr_expand_shape(list_expand_shape): attrs['shape'] = get_attr_expand_shape(shape) if utils._contain_var(shape): inputs['expand_shapes_tensor'] = utils._convert_to_tensor_list( - shape) + shape + ) dtype = helper.input_dtype(input_param_name='x') out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='expand_v2', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='expand_v2', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out @@ -3331,7 +3565,7 @@ def reshape(x, shape, name=None): if in_dygraph_mode(): tmp_tensor_type = core.eager.Tensor - #TODO(zhiqiu): enable inplace in dygraph mode. + # TODO(zhiqiu): enable inplace in dygraph mode. if inplace: warnings.warn( "Inplace on reshape is not allowed and will be discarded in dygraph mode currently." @@ -3339,7 +3573,9 @@ def reshape(x, shape, name=None): if isinstance(shape, (list, tuple)): shape = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item for item in shape + if isinstance(item, tmp_tensor_type) + else item + for item in shape ] out = _C_ops.reshape(x, shape) elif isinstance(shape, tmp_tensor_type): @@ -3348,7 +3584,8 @@ def reshape(x, shape, name=None): else: raise ValueError( "shape must be an instance of `list`, `tuple` or `Variable`," - " got '{}.'".format(type(shape))) + " got '{}.'".format(type(shape)) + ) return dygraph_utils._append_activation_in_dygraph(out, act) else: @@ -3370,14 +3607,26 @@ def reshape(x, shape, name=None): else: raise ValueError( "shape must be an instance of `list`, `tuple` or `Variable`," - " got '{}.'".format(type(shape))) + " got '{}.'".format(type(shape)) + ) return dygraph_utils._append_activation_in_dygraph(out, act) - check_variable_and_dtype(x, 'x', [ - 'float16', 'float32', 'float64', 'int16', 'int32', 'int64', 'bool', - 'uint16' - ], 'reshape') + check_variable_and_dtype( + x, + 'x', + [ + 'float16', + 'float32', + 'float64', + 'int16', + 'int32', + 'int64', + 'bool', + 'uint16', + ], + 'reshape', + ) check_type(shape, 'shape', (list, tuple, Variable), 'reshape') check_type(actual_shape, 'actual_shape', (Variable, type(None)), 'reshape') @@ -3401,20 +3650,23 @@ def get_attr_shape(list_shape): "\t# z.shape is [-1, -1, 4]\n\n" " If your target shape in Reshape represents dynamic shape, " "please turn it into a Tensor under @to_static. See above example for details." - % dim_idx) + % dim_idx + ) unk_dim_idx = dim_idx elif dim_size == 0: assert dim_idx < len(x.shape), ( "The index of 0 in `shape` must be less than " "the input tensor X's dimensions. " - "But received shape[%d] = 0, X's dimensions = %d." % - (dim_idx, len(x.shape))) + "But received shape[%d] = 0, X's dimensions = %d." + % (dim_idx, len(x.shape)) + ) else: assert dim_size > 0, ( "Each dimension value of 'shape' in reshape must not " "be negative except one unknown dimension. " - "But received shape[%d] = %s." % - (dim_idx, str(dim_size))) + "But received shape[%d] = %s." + % (dim_idx, str(dim_size)) + ) return attrs_shape inputs = {"X": x} @@ -3423,8 +3675,10 @@ def get_attr_shape(list_shape): shape.stop_gradient = True inputs["Shape"] = shape elif isinstance(shape, (list, tuple)): - assert len(shape) > 0, ("The size of 'shape' in reshape can't be zero, " - "but received %s." % len(shape)) + assert len(shape) > 0, ( + "The size of 'shape' in reshape can't be zero, " + "but received %s." % len(shape) + ) attrs["shape"] = get_attr_shape(shape) if utils._contain_var(shape): inputs['ShapeTensor'] = utils._convert_to_tensor_list(shape) @@ -3432,16 +3686,18 @@ def get_attr_shape(list_shape): actual_shape.stop_gradient = True inputs["Shape"] = actual_shape - out = x if inplace else helper.create_variable_for_type_inference( - dtype=x.dtype) + out = ( + x + if inplace + else helper.create_variable_for_type_inference(dtype=x.dtype) + ) x_shape = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="reshape2", - inputs=inputs, - attrs=attrs, - outputs={ - "Out": out, - "XShape": x_shape - }) + helper.append_op( + type="reshape2", + inputs=inputs, + attrs=attrs, + outputs={"Out": out, "XShape": x_shape}, + ) return helper.append_activation(out) @@ -3457,7 +3713,9 @@ def reshape_(x, shape, name=None): if isinstance(shape, (list, tuple)): shape = [ item.numpy().item(0) - if isinstance(item, tmp_tensor_type) else item for item in shape + if isinstance(item, tmp_tensor_type) + else item + for item in shape ] out = _C_ops.reshape_(x, shape) elif isinstance(shape, tmp_tensor_type): @@ -3466,7 +3724,8 @@ def reshape_(x, shape, name=None): else: raise ValueError( "shape must be an instance of `list`, `tuple` or `Variable`," - " got '{}.'".format(type(shape))) + " got '{}.'".format(type(shape)) + ) return out else: @@ -3567,18 +3826,20 @@ def gather_nd(x, index, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.gather_nd(x, index) check_variable_and_dtype( - x, 'x', ['bool', 'float32', 'float64', 'int16', 'int32', 'int64'], - 'gather_np') + x, + 'x', + ['bool', 'float32', 'float64', 'int16', 'int32', 'int64'], + 'gather_np', + ) check_variable_and_dtype(index, 'index', ['int32', 'int64'], 'gather_np') helper = LayerHelper('gather_nd', **locals()) dtype = helper.input_dtype() output = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="gather_nd", - inputs={ - "X": x, - "Index": index - }, - outputs={"Out": output}) + helper.append_op( + type="gather_nd", + inputs={"X": x, "Index": index}, + outputs={"Out": output}, + ) return output @@ -3670,8 +3931,11 @@ def strided_slice(x, axes, starts, ends, strides, name=None): helper = LayerHelper('strided_slice', **locals()) check_variable_and_dtype( - x, 'x', ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'strided_slice') + x, + 'x', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'strided_slice', + ) check_type(axes, 'axes', (list, tuple), 'strided_slice') check_type(starts, 'starts', (list, tuple, Variable), 'strided_slice') check_type(ends, 'ends', (list, tuple, Variable), 'strided_slice') @@ -3679,8 +3943,9 @@ def strided_slice(x, axes, starts, ends, strides, name=None): def check_list_elements_dtype(list_input, input_name): if isinstance(list_input, Variable): - check_dtype(list_input.dtype, input_name, ['int32'], - 'strided_slice') + check_dtype( + list_input.dtype, input_name, ['int32'], 'strided_slice' + ) else: for i, var in enumerate(list_input): var_name = input_name + '[' + str(i) + ']' @@ -3699,7 +3964,7 @@ def get_new_list_tensor(old_list): dim.stop_gradient = True new_list_tensor.append(dim) else: - assert (isinstance(dim, int)) + assert isinstance(dim, int) temp_out = helper.create_variable_for_type_inference('int32') fill_constant([1], 'int32', dim, force_cpu=True, out=temp_out) new_list_tensor.append(temp_out) @@ -3716,7 +3981,7 @@ def get_new_list_tensor(old_list): 'starts': starts, 'ends': ends, 'strides': strides, - 'infer_flags': infer_flags + 'infer_flags': infer_flags, } else: # starts @@ -3771,11 +4036,11 @@ def get_new_list_tensor(old_list): attrs['strides'] = strides attrs['infer_flags'] = infer_flags out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype('x')) - helper.append_op(type='strided_slice', - inputs=inputs, - attrs=attrs, - outputs={'Out': out}) + dtype=helper.input_dtype('x') + ) + helper.append_op( + type='strided_slice', inputs=inputs, attrs=attrs, outputs={'Out': out} + ) return out @@ -3903,8 +4168,9 @@ def _var_to_list(var): if paddle.in_dynamic_mode(): return tolist(var) raise TypeError( - "The 'axes' with type 'Tensor' in " + op_type + - " is not available in static graph mode, " + "The 'axes' with type 'Tensor' in " + + op_type + + " is not available in static graph mode, " "please convert its type to int|Tuple|List, or use dynamic graph mode." ) @@ -3912,8 +4178,10 @@ def _var_to_list(var): axes_y = [] if np.issubdtype(type(axes), np.integer): assert axes >= 0, ( - "The 'axes' in " + op_type + - f" should not be negative, but received axes={axes}.") + "The 'axes' in " + + op_type + + f" should not be negative, but received axes={axes}." + ) axes_x = range(x.ndim - axes, x.ndim) axes_y = range(axes) else: @@ -3953,7 +4221,11 @@ def _var_to_list(var): shape_x[dim_x] = 1 x = x.sum(dim_x).reshape(shape_x) else: - assert sx == sy, "The dimensional size for 'x' and 'y' in " + op_type + f" should match each other, but 'x' has size {sx} in dim {dim_x} while 'y' has size {sy} in dim {dim_y}." + assert sx == sy, ( + "The dimensional size for 'x' and 'y' in " + + op_type + + f" should match each other, but 'x' has size {sx} in dim {dim_x} while 'y' has size {sy} in dim {dim_y}." + ) need_contracted_dim_x[dim_x] = True need_contracted_dim_y[dim_y] = True @@ -3981,9 +4253,11 @@ def _var_to_list(var): shape_out = [1] x = x.transpose(perm=perm_x).reshape( - [not_contraction_size_x, contraction_size]) + [not_contraction_size_x, contraction_size] + ) y = y.transpose(perm=perm_y).reshape( - [contraction_size, not_contraction_size_y]) + [contraction_size, not_contraction_size_y] + ) out = x.matmul(y).reshape(shape_out) return out @@ -4027,7 +4301,8 @@ def as_complex(x, name=None): helper = LayerHelper(op_type, **locals()) inputs = {"X": x} out = helper.create_variable_for_type_inference( - dtype=_real_to_complex_dtype(x.dtype)) + dtype=_real_to_complex_dtype(x.dtype) + ) outputs = {"Out": out} attrs = {} helper.append_op(type=op_type, inputs=inputs, attrs=attrs, outputs=outputs) @@ -4079,7 +4354,8 @@ def as_real(x, name=None): helper = LayerHelper(op_type, **locals()) inputs = {"X": x} out = helper.create_variable_for_type_inference( - dtype=_complex_to_real_dtype(x.dtype)) + dtype=_complex_to_real_dtype(x.dtype) + ) outputs = {"Out": out} helper.append_op(type=op_type, inputs=inputs, outputs=outputs) return out @@ -4131,23 +4407,27 @@ def repeat_interleave(x, repeats, axis=None, name=None): return _C_ops.repeat_interleave(x, repeats, axis) helper = LayerHelper("repeat_interleave", **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'paddle.tensor.manipulation.repeat_interleave') + check_variable_and_dtype( + x, + 'x', + ['float32', 'float64', 'int32', 'int64'], + 'paddle.tensor.manipulation.repeat_interleave', + ) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='repeat_interleave', - inputs={ - 'X': - x, - 'RepeatsTensor': - repeats if isinstance(repeats, Variable) else None - }, - outputs={'Out': out}, - attrs={ - 'dim': axis, - 'Repeats': repeats if isinstance(repeats, int) else 0 - }) + helper.append_op( + type='repeat_interleave', + inputs={ + 'X': x, + 'RepeatsTensor': repeats if isinstance(repeats, Variable) else None, + }, + outputs={'Out': out}, + attrs={ + 'dim': axis, + 'Repeats': repeats if isinstance(repeats, int) else 0, + }, + ) return out @@ -4184,7 +4464,8 @@ def moveaxis(x, source, destination, name=None): dst = [destination] if isinstance(destination, int) else destination assert len(src) == len( - dst), "'source' must have the same number with 'destination'" + dst + ), "'source' must have the same number with 'destination'" count = Counter(src).most_common(1) if count[0][1] > 1: @@ -4201,29 +4482,31 @@ def moveaxis(x, source, destination, name=None): dst_dims = list(range(ndim)) for i, axis in enumerate(zip(src, dst)): - assert isinstance(axis[0], - int), "Each elemment of 'source' must be integer." + assert isinstance( + axis[0], int + ), "Each elemment of 'source' must be integer." if axis[0] < 0: - assert axis[ - 0] >= -ndim, "'source' must be in the range of [-{0}, {0})".format( - ndim) + assert ( + axis[0] >= -ndim + ), "'source' must be in the range of [-{0}, {0})".format(ndim) src[i] += ndim else: - assert axis[ - 0] < ndim, "'source' must be in the range of [-{0}, {0})".format( - ndim) + assert ( + axis[0] < ndim + ), "'source' must be in the range of [-{0}, {0})".format(ndim) - assert isinstance(axis[1], - int), "Each elemment of 'source' must be integer." + assert isinstance( + axis[1], int + ), "Each elemment of 'source' must be integer." if axis[1] < 0: - assert axis[ - 1] >= -ndim, "'source' must be in the range of [-{0}, {0})".format( - ndim) + assert ( + axis[1] >= -ndim + ), "'source' must be in the range of [-{0}, {0})".format(ndim) dst[i] += ndim else: - assert axis[ - 1] < ndim, "'source' must be in the range of [-{0}, {0})".format( - ndim) + assert ( + axis[1] < ndim + ), "'source' must be in the range of [-{0}, {0})".format(ndim) perm[dst[i]] = src[i] src_dims.remove(src[i]) dst_dims.remove(dst[i]) @@ -4239,32 +4522,44 @@ def moveaxis(x, source, destination, name=None): out, _ = _legacy_C_ops.transpose2(x, 'axis', perm) return out - check_variable_and_dtype(x, 'x', [ - 'bool', 'float16', 'float32', 'float64', 'int32', 'int64', 'complex64', - 'complex128' - ], 'moveaxis') + check_variable_and_dtype( + x, + 'x', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int32', + 'int64', + 'complex64', + 'complex128', + ], + 'moveaxis', + ) helper = LayerHelper('moveaxis', **locals()) out = helper.create_variable_for_type_inference(x.dtype) x_shape = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='transpose2', - inputs={'X': [x]}, - outputs={ - 'Out': [out], - 'XShape': [x_shape] - }, - attrs={'axis': perm}) + helper.append_op( + type='transpose2', + inputs={'X': [x]}, + outputs={'Out': [out], 'XShape': [x_shape]}, + attrs={'axis': perm}, + ) return out def non_negative_axis(arr, axis): ndim = len(arr.shape) if axis >= 0: - assert axis < ndim, "'axis' must be in the range of [-{0}, {0})".format( - ndim) + assert ( + axis < ndim + ), "'axis' must be in the range of [-{0}, {0})".format(ndim) else: - assert axis >= -ndim, "'axis' must be in the range of [-{0}, {0})".format( - ndim) + assert ( + axis >= -ndim + ), "'axis' must be in the range of [-{0}, {0})".format(ndim) axis += ndim return axis @@ -4307,9 +4602,10 @@ def take_along_axis(arr, indices, axis): print(result) # [[1, 2, 3]] """ - if (len(arr.shape) != len(indices.shape)): + if len(arr.shape) != len(indices.shape): raise ValueError( - "`indices` and `arr` must have the same number of dimensions!") + "`indices` and `arr` must have the same number of dimensions!" + ) axis = non_negative_axis(arr, axis) broadcast_shape = infer_broadcast_shape(arr, indices, axis) if not broadcast_shape: @@ -4325,10 +4621,14 @@ def take_along_axis(arr, indices, axis): return _C_ops.take_along_axis(arr, indices, axis) return _legacy_C_ops.take_along_axis(arr, indices, 'Axis', axis) check_variable_and_dtype( - arr, 'x', ['float16', 'float32', 'float64', 'int32', 'int64', 'uint8'], - 'take_along_axis') - check_variable_and_dtype(indices, 'index', ['int32', 'int64'], - 'take_along_axis') + arr, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64', 'uint8'], + 'take_along_axis', + ) + check_variable_and_dtype( + indices, 'index', ['int32', 'int64'], 'take_along_axis' + ) indices = paddle.broadcast_to(indices, broadcast_shape) broadcast_shape_list = list(broadcast_shape) broadcast_shape_list[axis] = list(arr.shape)[axis] @@ -4337,13 +4637,12 @@ def take_along_axis(arr, indices, axis): helper = LayerHelper('take_along_axis', **locals()) dtype = helper.input_dtype() result = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="take_along_axis", - inputs={ - "Input": arr, - "Index": indices - }, - attrs={"Axis": axis}, - outputs={"Result": result}) + helper.append_op( + type="take_along_axis", + inputs={"Input": arr, "Index": indices}, + attrs={"Axis": axis}, + outputs={"Result": result}, + ) return result @@ -4376,44 +4675,48 @@ def put_along_axis(arr, indices, values, axis, reduce='assign'): # [60, 40, 50]] """ - if (len(arr.shape) != len(indices.shape)): + if len(arr.shape) != len(indices.shape): raise ValueError( - "`indices` and `arr` must have the same number of dimensions!") + "`indices` and `arr` must have the same number of dimensions!" + ) axis = non_negative_axis(arr, axis) broadcast_shape = infer_broadcast_shape(arr, indices, axis) if _non_static_mode(): - values = paddle.to_tensor(values) if not isinstance( - values, paddle.Tensor) else values + values = ( + paddle.to_tensor(values) + if not isinstance(values, paddle.Tensor) + else values + ) if broadcast_shape: indices = paddle.broadcast_to(indices, broadcast_shape) values = paddle.broadcast_to(values, indices.shape) if in_dygraph_mode(): return _C_ops.put_along_axis(arr, indices, values, axis, reduce) - return _legacy_C_ops.put_along_axis(arr, indices, values, "Axis", axis, - "Reduce", reduce) + return _legacy_C_ops.put_along_axis( + arr, indices, values, "Axis", axis, "Reduce", reduce + ) check_variable_and_dtype( - arr, 'x', ['float16', 'float32', 'float64', 'int32', 'int64', 'uint8'], - 'put_along_axis') - check_variable_and_dtype(indices, 'index', ['int32', 'int64'], - 'put_along_axis') + arr, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64', 'uint8'], + 'put_along_axis', + ) + check_variable_and_dtype( + indices, 'index', ['int32', 'int64'], 'put_along_axis' + ) if broadcast_shape: indices = paddle.broadcast_to(indices, broadcast_shape) values = paddle.broadcast_to(values, indices.shape) helper = LayerHelper('put_along_axis', **locals()) dtype = helper.input_dtype() result = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="put_along_axis", - inputs={ - "Input": arr, - "Index": indices, - "Value": values - }, - attrs={ - "Axis": axis, - "Reduce": reduce - }, - outputs={"Result": result}) + helper.append_op( + type="put_along_axis", + inputs={"Input": arr, "Index": indices, "Value": values}, + attrs={"Axis": axis, "Reduce": reduce}, + outputs={"Result": result}, + ) return result @@ -4423,20 +4726,25 @@ def put_along_axis_(arr, indices, values, axis, reduce='assign'): Inplace version of ``put_along_axis`` API, the output Tensor will be inplaced with input ``arr``. Please refer to :ref:`api_tensor_put_along_axis`. """ - if (len(arr.shape) != len(indices.shape)): + if len(arr.shape) != len(indices.shape): raise ValueError( - "`indices` and `arr` must have the same number of dimensions!") + "`indices` and `arr` must have the same number of dimensions!" + ) axis = non_negative_axis(arr, axis) broadcast_shape = infer_broadcast_shape(arr, indices, axis) - values = paddle.to_tensor(values) if not isinstance( - values, paddle.Tensor) else values + values = ( + paddle.to_tensor(values) + if not isinstance(values, paddle.Tensor) + else values + ) if broadcast_shape: indices = paddle.broadcast_to(indices, broadcast_shape) values = paddle.broadcast_to(values, indices.shape) if in_dygraph_mode(): return _C_ops.put_along_axis_(arr, indices, values, axis, reduce) - return _legacy_C_ops.put_along_axis_(arr, indices, values, "Axis", axis, - "Reduce", reduce) + return _legacy_C_ops.put_along_axis_( + arr, indices, values, "Axis", axis, "Reduce", reduce + ) def index_add(x, index, axis, value, name=None): @@ -4475,24 +4783,36 @@ def index_add(x, index, axis, value, name=None): helper = LayerHelper("index_add", **locals()) check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'paddle.tensor.manipulation.index_add') - check_variable_and_dtype(index, 'index', ['int32', 'int64'], - 'paddle.tensor.manipulation.index_add') + x, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'paddle.tensor.manipulation.index_add', + ) + check_variable_and_dtype( + index, + 'index', + ['int32', 'int64'], + 'paddle.tensor.manipulation.index_add', + ) check_variable_and_dtype( - value, 'add_value', ['float16', 'float32', 'float64', 'int32', 'int64'], - 'paddle.tensor.manipulation.index_add') + value, + 'add_value', + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'paddle.tensor.manipulation.index_add', + ) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='index_add', - inputs={ - 'X': x, - 'Index': index, - 'AddValue': value, - }, - outputs={'Out': out}, - attrs={'axis': axis}) + helper.append_op( + type='index_add', + inputs={ + 'X': x, + 'Index': index, + 'AddValue': value, + }, + outputs={'Out': out}, + attrs={'axis': axis}, + ) return out @@ -4528,7 +4848,7 @@ def index_add_(x, index, axis, value, name=None): 'fill_diagonal_': fill_diagonal_, 'fill_diagonal_tensor_': fill_diagonal_tensor_, "fill_diagonal_tensor": fill_diagonal_tensor, - 'tolist': tolist + 'tolist': tolist, } for name, func in __METHODS.items(): setattr(core.VarBase, name, func) diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py index a2720ec48f22b..cd8dbd08b94d8 100644 --- a/python/paddle/tensor/math.py +++ b/python/paddle/tensor/math.py @@ -27,10 +27,21 @@ import paddle from ..static import Variable -from ..framework import core, in_dygraph_mode, _non_static_mode, LayerHelper, _in_legacy_dygraph +from ..framework import ( + core, + in_dygraph_mode, + _non_static_mode, + LayerHelper, + _in_legacy_dygraph, +) from ..fluid.framework import _in_legacy_dygraph from ..framework import _varbase_creator, convert_np_dtype_to_dtype_ -from ..fluid.data_feeder import check_variable_and_dtype, check_type, check_dtype, convert_dtype +from ..fluid.data_feeder import ( + check_variable_and_dtype, + check_type, + check_dtype, + convert_dtype, +) from ..fluid.dygraph.inplace_utils import inplace_apis_in_dygraph_only from ..fluid.layers import utils @@ -177,15 +188,33 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): return dygraph_utils._append_activation_in_dygraph(out, act) elif _in_legacy_dygraph(): _scale = scale.numpy().item(0) if isinstance(scale, Variable) else scale - out = _legacy_C_ops.scale(x, 'scale', float(_scale), 'bias', - float(bias), 'bias_after_scale', - bias_after_scale) + out = _legacy_C_ops.scale( + x, + 'scale', + float(_scale), + 'bias', + float(bias), + 'bias_after_scale', + bias_after_scale, + ) return dygraph_utils._append_activation_in_dygraph(out, act) - check_variable_and_dtype(x, "x", [ - 'float16', 'uint16', 'float32', 'float64', 'int8', 'int16', 'int32', - 'int64', 'uint8' - ], "scale") + check_variable_and_dtype( + x, + "x", + [ + 'float16', + 'uint16', + 'float32', + 'float64', + 'int8', + 'int16', + 'int32', + 'int64', + 'uint8', + ], + "scale", + ) inputs = {'X': [x]} attrs = { 'bias': float(bias), @@ -198,10 +227,9 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): helper = LayerHelper('scale', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='scale', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='scale', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return helper.append_activation(out) @@ -240,13 +268,12 @@ def stanh(x, scale_a=0.67, scale_b=1.7159, name=None): helper = LayerHelper('stanh', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='stanh', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'scale_a': scale_a, - 'scale_b': scale_b - }) + helper.append_op( + type='stanh', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'scale_a': scale_a, 'scale_b': scale_b}, + ) return out @@ -310,20 +337,23 @@ def multiplex(inputs, index, name=None): check_type(inputs, 'inputs', (list), 'multiplex') if len(inputs) < 2: raise ValueError( - "inputs should be a list object with at least 2 elements.") + "inputs should be a list object with at least 2 elements." + ) for id, x in enumerate(inputs): - check_variable_and_dtype(x, 'input[' + str(id) + ']', - ['float32', 'float64', 'int32', 'int64'], - 'multiplex') + check_variable_and_dtype( + x, + 'input[' + str(id) + ']', + ['float32', 'float64', 'int32', 'int64'], + 'multiplex', + ) check_variable_and_dtype(index, "index", ['int32', 'int64'], 'multiplex') out = helper.create_variable_for_type_inference(inputs[0].dtype) - helper.append_op(type='multiplex', - inputs={ - 'X': inputs, - 'Ids': index - }, - outputs={'Out': [out]}) + helper.append_op( + type='multiplex', + inputs={'X': inputs, 'Ids': index}, + outputs={'Out': [out]}, + ) return out @@ -337,9 +367,15 @@ def scale_(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): return _C_ops.scale_(x, scale, float(bias), bias_after_scale) if _in_legacy_dygraph(): _scale = scale.numpy().item(0) if isinstance(scale, Variable) else scale - return _legacy_C_ops.scale_(x, 'scale', float(_scale), 'bias', - float(bias), 'bias_after_scale', - bias_after_scale) + return _legacy_C_ops.scale_( + x, + 'scale', + float(_scale), + 'bias', + float(bias), + 'bias_after_scale', + bias_after_scale, + ) def pow(x, y, name=None): @@ -395,31 +431,28 @@ def pow(x, y, name=None): return _C_ops.elementwise_pow(x, y) else: raise TypeError( - 'y must be scalar or tensor type, but received: %s ' % - (y.dtype)) + 'y must be scalar or tensor type, but received: %s ' % (y.dtype) + ) if _in_legacy_dygraph(): if isinstance(y, (int, float)): return _legacy_C_ops.pow(x, 'factor', y) elif isinstance(y, (paddle.Tensor, Variable)): - return _elementwise_op_in_dygraph(x, - y, - axis=-1, - act=None, - op_name='elementwise_pow') + return _elementwise_op_in_dygraph( + x, y, axis=-1, act=None, op_name='elementwise_pow' + ) else: raise TypeError( - 'y must be scalar or tensor type, but received: %s ' % - (y.dtype)) + 'y must be scalar or tensor type, but received: %s ' % (y.dtype) + ) # in static graph mode if isinstance(y, (int, float)): helper = LayerHelper('pow', **locals()) inputs = {'X': x} attrs = {'factor': y} out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='pow', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='pow', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) return out elif isinstance(y, (paddle.Tensor, Variable)): # TODO A potential speed improvement is supporting different types in C++ and removing the cast ops here @@ -427,8 +460,9 @@ def pow(x, y, name=None): out = helper.create_variable_for_type_inference(dtype=x.dtype) return _elementwise_op(LayerHelper('elementwise_pow', **locals())) else: - raise TypeError('y must be scalar or tensor type, but received: %s ' % - (type(y))) + raise TypeError( + 'y must be scalar or tensor type, but received: %s ' % (type(y)) + ) OP_NAMEMAPPING = { @@ -445,13 +479,9 @@ def pow(x, y, name=None): @dygraph_only -def _elementwise_op_in_dygraph(x, - y, - axis=-1, - act=None, - use_mkldnn=False, - op_name=None): - +def _elementwise_op_in_dygraph( + x, y, axis=-1, act=None, use_mkldnn=False, op_name=None +): def is_inplace(op_name): return op_name[-1] == "_" @@ -462,16 +492,17 @@ def is_inplace(op_name): if in_dygraph_mode(): op = getattr( _C_ops, - OP_NAMEMAPPING[op_name] if not is_inplace(op_name) else op_name) + OP_NAMEMAPPING[op_name] if not is_inplace(op_name) else op_name, + ) out = op(x, y) if _in_legacy_dygraph(): op = getattr(_legacy_C_ops, op_name) out = op(x, y, 'axis', axis, 'use_mkldnn', use_mkldnn) - return dygraph_utils._append_activation_in_dygraph(out, - act, - use_mkldnn=use_mkldnn) + return dygraph_utils._append_activation_in_dygraph( + out, act, use_mkldnn=use_mkldnn + ) def _elementwise_op(helper): @@ -485,11 +516,17 @@ def _elementwise_op(helper): assert x is not None, 'x cannot be None in {}'.format(original_op_type) assert y is not None, 'y cannot be None in {}'.format(original_op_type) check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], - original_op_type) + x, + 'x', + ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], + original_op_type, + ) check_variable_and_dtype( - y, 'y', ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], - original_op_type) + y, + 'y', + ['float16', 'float32', 'float64', 'int32', 'int64', 'bool'], + original_op_type, + ) axis = helper.kwargs.get('axis', -1) use_mkldnn = helper.kwargs.get('use_mkldnn', False) @@ -499,20 +536,16 @@ def _elementwise_op(helper): if name is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) else: - out = helper.create_variable(name=name, - dtype=x.dtype, - persistable=False) - - helper.append_op(type=op_type, - inputs={ - 'X': x, - 'Y': y - }, - outputs={'Out': out}, - attrs={ - 'axis': axis, - 'use_mkldnn': use_mkldnn - }) + out = helper.create_variable( + name=name, dtype=x.dtype, persistable=False + ) + + helper.append_op( + type=op_type, + inputs={'X': x, 'Y': y}, + outputs={'Out': out}, + attrs={'axis': axis, 'use_mkldnn': use_mkldnn}, + ) return helper.append_activation(out) @@ -592,8 +625,10 @@ def add_(x, y, name=None): out_shape = broadcast_shape(x.shape, y.shape) if out_shape != x.shape: raise ValueError( - "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation." - .format(out_shape, x.shape)) + "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation.".format( + out_shape, x.shape + ) + ) if in_dygraph_mode(): return _C_ops.add_(x, y) @@ -663,11 +698,9 @@ def subtract(x, y, name=None): return _C_ops.subtract(x, y) else: if _in_legacy_dygraph(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) else: return _elementwise_op(LayerHelper(op_type, **locals())) @@ -684,17 +717,17 @@ def subtract_(x, y, name=None): out_shape = broadcast_shape(x.shape, y.shape) if out_shape != x.shape: raise ValueError( - "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation." - .format(out_shape, x.shape)) + "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation.".format( + out_shape, x.shape + ) + ) if in_dygraph_mode(): return _C_ops.subtract_(x, y) else: - out = _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name='elementwise_sub_') + out = _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name='elementwise_sub_' + ) return out @@ -735,11 +768,9 @@ def divide(x, y, name=None): return _C_ops.divide(x, y) else: if _in_legacy_dygraph(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) else: return _elementwise_op(LayerHelper(op_type, **locals())) @@ -839,8 +870,10 @@ def remainder_(x, y, name=None): out_shape = broadcast_shape(x.shape, y.shape) if out_shape != x.shape: raise ValueError( - "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation." - .format(out_shape, x.shape)) + "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation.".format( + out_shape, x.shape + ) + ) return _elementwise_op_in_dygraph(x, y, axis=axis, op_name=op_type) @@ -892,16 +925,15 @@ def multiply(x, y, name=None): return _C_ops.multiply(x, y) else: if _in_legacy_dygraph(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) else: if x.dtype != y.dtype: raise TypeError( 'Input tensors must be same type, but received type of x: %s, type of y: %s ' - % (x.dtype, y.dtype)) + % (x.dtype, y.dtype) + ) return _elementwise_op(LayerHelper(op_type, **locals())) @@ -966,11 +998,9 @@ def maximum(x, y, name=None): if in_dygraph_mode(): return _C_ops.maximum(x, y) elif _in_legacy_dygraph(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) return _elementwise_op(LayerHelper(op_type, **locals())) @@ -1034,11 +1064,9 @@ def minimum(x, y, name=None): if in_dygraph_mode(): return _C_ops.minimum(x, y) elif _in_legacy_dygraph(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) return _elementwise_op(LayerHelper(op_type, **locals())) @@ -1104,11 +1132,9 @@ def fmax(x, y, name=None): if in_dygraph_mode(): return _C_ops.fmax(x, y, axis) if _in_legacy_dygraph(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) return _elementwise_op(LayerHelper(op_type, **locals())) @@ -1174,11 +1200,9 @@ def fmin(x, y, name=None): if in_dygraph_mode(): return _C_ops.fmin(x, y, axis) if _in_legacy_dygraph(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) return _elementwise_op(LayerHelper(op_type, **locals())) @@ -1273,24 +1297,59 @@ def sum(x, axis=None, dtype=None, keepdim=False, name=None): if _in_legacy_dygraph(): if dtype_flag: - return _legacy_C_ops.reduce_sum(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all_flag, - 'in_dtype', x.dtype, 'out_dtype', - dtype) + return _legacy_C_ops.reduce_sum( + x, + 'dim', + axis, + 'keep_dim', + keepdim, + 'reduce_all', + reduce_all_flag, + 'in_dtype', + x.dtype, + 'out_dtype', + dtype, + ) else: - return _legacy_C_ops.reduce_sum(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all_flag) + return _legacy_C_ops.reduce_sum( + x, + 'dim', + axis, + 'keep_dim', + keepdim, + 'reduce_all', + reduce_all_flag, + ) attrs = {'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all_flag} if dtype_flag: attrs.update({'in_dtype': x.dtype, 'out_dtype': dtype}) - check_variable_and_dtype(x, 'x', [ - 'bool', 'float16', 'float32', 'float64', 'int16', 'int32', 'int64', - 'complex64', 'complex128', u'bool', u'float16', u'float32', u'float64', - u'int32', u'int64', u'complex64', u'complex128' - ], 'sum') + check_variable_and_dtype( + x, + 'x', + [ + 'bool', + 'float16', + 'float32', + 'float64', + 'int16', + 'int32', + 'int64', + 'complex64', + 'complex128', + u'bool', + u'float16', + u'float32', + u'float64', + u'int32', + u'int64', + u'complex64', + u'complex128', + ], + 'sum', + ) check_type(axis, 'axis', (int, list, tuple, type(None), Variable), 'sum') @@ -1299,10 +1358,9 @@ def sum(x, axis=None, dtype=None, keepdim=False, name=None): out = helper.create_variable_for_type_inference(dtype=dtype) else: out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reduce_sum', - inputs={'X': x}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='reduce_sum', inputs={'X': x}, outputs={'Out': out}, attrs=attrs + ) return out @@ -1353,8 +1411,9 @@ def nansum(x, axis=None, dtype=None, keepdim=False, name=None): out5 = paddle.nansum(y, axis=[1, 2]) # [8, 19] out6 = paddle.nansum(y, axis=[0, 1]) # [9, 18] """ - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'nansum') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'nansum' + ) check_type(axis, 'axis', (int, list, tuple, type(None)), 'nansum') zero_tensor = paddle.zeros_like(x) @@ -1419,16 +1478,17 @@ def nanmean(x, axis=None, keepdim=False, name=None): """ if isinstance(axis, int): axis = [axis] - check_variable_and_dtype(x, 'x/input', - ['uint16', 'float16', 'float32', 'float64'], - 'nanmean') + check_variable_and_dtype( + x, 'x/input', ['uint16', 'float16', 'float32', 'float64'], 'nanmean' + ) if axis is not None: check_type(axis, 'axis/dim', (int, list, tuple), 'nanmean') cnt = paddle.sum(~paddle.isnan(x), axis=axis, keepdim=keepdim) return paddle.divide( paddle.nansum(x, axis=axis, keepdim=keepdim, name=name), - cnt.astype(x.dtype)) + cnt.astype(x.dtype), + ) def count_nonzero(x, axis=None, keepdim=False, name=None): @@ -1485,8 +1545,9 @@ def count_nonzero(x, axis=None, keepdim=False, name=None): axis = [axis] dims = len(x.shape) for i in range(len(axis)): - if not isinstance(axis[i], int) or not (axis[i] < dims - and axis[i] >= -dims): + if not isinstance(axis[i], int) or not ( + axis[i] < dims and axis[i] >= -dims + ): raise ValueError( "Axis should be None, int, or a list, element should in range [-rank(x), rank(x))." ) @@ -1568,18 +1629,29 @@ def add_n(inputs, name=None): if isinstance(inputs, list) or isinstance(inputs, tuple): if len(inputs) > 0: for input in inputs: - check_variable_and_dtype(input, "inputs", \ - ['float16', 'float32', 'float64', 'int32', 'int64'], 'add_n') + check_variable_and_dtype( + input, + "inputs", + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'add_n', + ) else: - check_variable_and_dtype(inputs, "inputs", \ - ['float16', 'float32', 'float64', 'int32', 'int64'], 'add_n') + check_variable_and_dtype( + inputs, + "inputs", + ['float16', 'float32', 'float64', 'int32', 'int64'], + 'add_n', + ) out = helper.create_variable_for_type_inference( - dtype=helper.input_dtype('inputs')) - helper.append_op(type='sum', - inputs={'X': inputs}, - outputs={'Out': out}, - attrs={'use_mkldnn': False}) + dtype=helper.input_dtype('inputs') + ) + helper.append_op( + type='sum', + inputs={'X': inputs}, + outputs={'Out': out}, + attrs={'use_mkldnn': False}, + ) return out @@ -1622,15 +1694,14 @@ def trunc(input, name=None): attrs = {} helper = LayerHelper("trunc", **locals()) - check_variable_and_dtype(input, 'X', - ['int32', 'int64', 'float32', 'float64'], - 'trunc') + check_variable_and_dtype( + input, 'X', ['int32', 'int64', 'float32', 'float64'], 'trunc' + ) out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type="trunc", - inputs=inputs, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="trunc", inputs=inputs, attrs=attrs, outputs={"Out": out} + ) return out @@ -1709,8 +1780,9 @@ def mm(input, mat2, name=None): def __check_input(x, y): var_names = {'x': x, 'y': y} for name, val in var_names.items(): - check_variable_and_dtype(val, name, - ['float16', 'float32', 'float64'], 'mm') + check_variable_and_dtype( + val, name, ['float16', 'float32', 'float64'], 'mm' + ) x_shape = list(x.shape) y_shape = list(y.shape) if len(x_shape) == 1: @@ -1725,7 +1797,8 @@ def __check_input(x, y): "After performing an optional transpose, Input X's width should be " "equal to Y's width for multiplication " "prerequisites. But received X's shape: %s, Y's shape: %s\n" - % (x_shape, y_shape)) + % (x_shape, y_shape) + ) if len(y_shape) > 2 and len(x_shape) > 2: for i, dim_x in enumerate(x_shape[:-2]): @@ -1737,18 +1810,16 @@ def __check_input(x, y): "When the matrix is larger than 2 dimensions, the higher " "dimensional values of the two matrices need to be equal. " "But received x_shape[%d] != y_shape[%d]. X's shape: %s, " - "Y's shape: %s.\n" % (i, i, x_shape, y_shape)) + "Y's shape: %s.\n" % (i, i, x_shape, y_shape) + ) __check_input(input, mat2) helper = LayerHelper('mm', **locals()) out = helper.create_variable_for_type_inference(dtype=input.dtype) - helper.append_op(type='matmul_v2', - inputs={ - 'X': input, - 'Y': mat2 - }, - outputs={'Out': out}) + helper.append_op( + type='matmul_v2', inputs={'X': input, 'Y': mat2}, outputs={'Out': out} + ) return out @@ -1796,36 +1867,50 @@ def addmm(input, x, y, beta=1.0, alpha=1.0, name=None): y_shape = y.shape if not len(x_shape) == len(y_shape) == 2: raise ValueError( - "The dimention of x, y should be 2 but receive x's shape: {}, y's shape: {}" - .format(x_shape, y_shape)) + "The dimention of x, y should be 2 but receive x's shape: {}, y's shape: {}".format( + x_shape, y_shape + ) + ) if x_shape[1] != y_shape[0]: raise ValueError( - "The input Variable x's width must be equal with Variable y' height. But received x's shape = {}, y's shape = {}." - .format(x_shape, y_shape)) + "The input Variable x's width must be equal with Variable y' height. But received x's shape = {}, y's shape = {}.".format( + x_shape, y_shape + ) + ) if len(input_shape) == 2: if input_shape[0] != x_shape[0]: if input_shape[0] != 1: raise ValueError( - "When x's dimension[0] is not equal with input's dimension[0], input's dimension[0] must be 1 but got {}" - .format(input_shape[0])) + "When x's dimension[0] is not equal with input's dimension[0], input's dimension[0] must be 1 but got {}".format( + input_shape[0] + ) + ) if input_shape[1] != y_shape[1] and input_shape[1] != 1: raise ValueError( - "When y's dimension[1] is not equal with input's dimension[1], input's dimension[1] must be 1 but got {}" - .format(input_shape[1])) + "When y's dimension[1] is not equal with input's dimension[1], input's dimension[1] must be 1 but got {}".format( + input_shape[1] + ) + ) if input_shape[1] != y_shape[1]: if input_shape[1] != 1: raise ValueError( - "When y's dimension[1] is not equal with input's dimension[1], input's dimension[1] must be 1 but got {}" - .format(input_shape[1])) + "When y's dimension[1] is not equal with input's dimension[1], input's dimension[1] must be 1 but got {}".format( + input_shape[1] + ) + ) elif len(input_shape) == 1: if input_shape[0] not in (y_shape[1], 1): raise ValueError( - "The input's shape: {} is not broadcastable with [x.shape[0], y.shape[1]]: [{},{}]" - .format(input_shape, x_shape[0], y_shape[1])) + "The input's shape: {} is not broadcastable with [x.shape[0], y.shape[1]]: [{},{}]".format( + input_shape, x_shape[0], y_shape[1] + ) + ) else: raise ValueError( - "The dimention of input should be 2 or 1 but receive input's shape: {}" - .format(input_shape)) + "The dimention of input should be 2 or 1 but receive input's shape: {}".format( + input_shape + ) + ) if in_dygraph_mode(): return _C_ops.addmm(input, x, y, alpha, beta) @@ -1838,16 +1923,16 @@ def addmm(input, x, y, beta=1.0, alpha=1.0, name=None): attrs = {'Alpha': alpha, 'Beta': beta} helper = LayerHelper("addmm", **locals()) - check_variable_and_dtype(input, 'Input', ['float32', 'float64'], - 'addmm') + check_variable_and_dtype( + input, 'Input', ['float32', 'float64'], 'addmm' + ) check_variable_and_dtype(x, 'X', ['float32', 'float64'], 'addmm') check_variable_and_dtype(y, 'Y', ['float32', 'float64'], 'addmm') out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="addmm", - inputs=inputs, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="addmm", inputs=inputs, attrs=attrs, outputs={"Out": out} + ) return out @@ -1890,19 +1975,24 @@ def renorm(x, p, axis, max_norm): if not axis < len(input_shape): raise ValueError( "the axis:{} should be less then the shape's size {}:{}".format( - axis, len(input_shape), input_shape)) + axis, len(input_shape), input_shape + ) + ) if not axis >= 0: if not axis >= -1 * len(input_shape): raise ValueError( - "the axis:{} should not be less than -1 * length of input_shape:{}" - .format(axis, -1 * len(input_shape))) + "the axis:{} should not be less than -1 * length of input_shape:{}".format( + axis, -1 * len(input_shape) + ) + ) axis = axis + len(input_shape) if in_dygraph_mode(): out = _C_ops.renorm(x, p, axis, max_norm) return out elif _in_legacy_dygraph(): - out = _legacy_C_ops.renorm(x, 'p', p, 'axis', axis, 'max_norm', - max_norm) + out = _legacy_C_ops.renorm( + x, 'p', p, 'axis', axis, 'max_norm', max_norm + ) return out inputs = {'X': x} @@ -1911,10 +2001,9 @@ def renorm(x, p, axis, max_norm): helper = LayerHelper("renorm", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="renorm", - inputs=inputs, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="renorm", inputs=inputs, attrs=attrs, outputs={"Out": out} + ) return out @@ -1965,9 +2054,9 @@ def inner(x, y, name=None): def __check_input(x, y): var_names = {'x': x, 'y': y} for name, val in var_names.items(): - check_variable_and_dtype(val, name, - ['float16', 'float32', 'float64'], - 'inner') + check_variable_and_dtype( + val, name, ['float16', 'float32', 'float64'], 'inner' + ) x_shape = list(xshape) y_shape = list(yshape) @@ -1978,18 +2067,16 @@ def __check_input(x, y): "After performing an optional transpose, Input X's last dim should be " "equal to Y's last dim for multiplication " "prerequisites. But received X's shape: %s, Y's shape: %s\n" - % (x_shape, y_shape)) + % (x_shape, y_shape) + ) __check_input(nx, ny) helper = LayerHelper('inner', **locals()) out = helper.create_variable_for_type_inference(dtype=nx.dtype) - helper.append_op(type='matmul_v2', - inputs={ - 'X': nx, - 'Y': ny.T - }, - outputs={'Out': out}) + helper.append_op( + type='matmul_v2', inputs={'X': nx, 'Y': ny.T}, outputs={'Out': out} + ) return out.reshape(dstshape) @@ -2033,19 +2120,17 @@ def outer(x, y, name=None): def __check_input(x, y): var_names = {'x': x, 'y': y} for name, val in var_names.items(): - check_variable_and_dtype(val, name, - ['float16', 'float32', 'float64'], 'inner') + check_variable_and_dtype( + val, name, ['float16', 'float32', 'float64'], 'inner' + ) __check_input(nx, ny) helper = LayerHelper('outer', **locals()) out = helper.create_variable_for_type_inference(dtype=nx.dtype) - helper.append_op(type='matmul_v2', - inputs={ - 'X': nx, - 'Y': ny - }, - outputs={'Out': out}) + helper.append_op( + type='matmul_v2', inputs={'X': nx, 'Y': ny}, outputs={'Out': out} + ) return out @@ -2093,9 +2178,11 @@ def logsumexp(x, axis=None, keepdim=False, name=None): """ if isinstance(axis, int): axis = [axis] - reduce_all = True if axis is None \ - or len(axis)==0 \ - or len(axis) == len(x.shape) else False + reduce_all = ( + True + if axis is None or len(axis) == 0 or len(axis) == len(x.shape) + else False + ) if axis is None or len(axis) == 0: axis = [0] @@ -2104,18 +2191,18 @@ def logsumexp(x, axis=None, keepdim=False, name=None): axis = range(len(x.shape)) return _C_ops.logsumexp(x, axis, keepdim, reduce_all) if _in_legacy_dygraph(): - return _legacy_C_ops.logsumexp(x, 'axis', axis, 'keepdim', keepdim, - 'reduce_all', reduce_all) + return _legacy_C_ops.logsumexp( + x, 'axis', axis, 'keepdim', keepdim, 'reduce_all', reduce_all + ) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'logsumexp') helper = LayerHelper('logsumexp', **locals()) attrs = {'axis': axis, 'keepdim': keepdim, 'reduce_all': reduce_all} out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='logsumexp', - inputs={'X': x}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='logsumexp', inputs={'X': x}, outputs={'Out': out}, attrs=attrs + ) return out @@ -2157,14 +2244,15 @@ def _check_input(x): raise ValueError( "The input of inverse is expected to be a Tensor whose number " "of dimensions is no less than 2. But reviced: %d, " - "x's shape: %s." % (len(x.shape), x.shape)) + "x's shape: %s." % (len(x.shape), x.shape) + ) _check_input(x) helper = LayerHelper('inverse', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='inverse', - inputs={'Input': [x]}, - outputs={'Output': [out]}) + helper.append_op( + type='inverse', inputs={'Input': [x]}, outputs={'Output': [out]} + ) return out @@ -2180,8 +2268,10 @@ def _get_reduce_axis(axis): axis = [axis] else: raise TypeError( - "The type of axis must be int, list or tuple, but received {}". - format(type(axis))) + "The type of axis must be int, list or tuple, but received {}".format( + type(axis) + ) + ) reduce_all = True if axis == None or axis == [] else False if axis == None: axis = [] @@ -2206,8 +2296,10 @@ def _get_reduce_all_value(axis): axis = [axis] else: raise TypeError( - "The type of axis must be int, list or tuple, but received {}". - format(type(axis))) + "The type of axis must be int, list or tuple, but received {}".format( + type(axis) + ) + ) reduce_all = True if axis == None or axis == [] else False axis = axis if axis != None and axis != [] else [0] @@ -2296,24 +2388,24 @@ def max(x, axis=None, keepdim=False, name=None): if in_dygraph_mode(): return _C_ops.max(x, axis, keepdim) if _in_legacy_dygraph(): - return _legacy_C_ops.reduce_max(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all) + return _legacy_C_ops.reduce_max( + x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all + ) helper = LayerHelper('max', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'max') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'max' + ) if not isinstance(axis, Variable) and utils._contain_var(axis): axis = utils._convert_to_tensor_list(axis) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reduce_max', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + helper.append_op( + type='reduce_max', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) return out @@ -2399,24 +2491,24 @@ def min(x, axis=None, keepdim=False, name=None): return _C_ops.min(x, axis, keepdim) if _in_legacy_dygraph(): - return _legacy_C_ops.reduce_min(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all) + return _legacy_C_ops.reduce_min( + x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all + ) helper = LayerHelper('min', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'min') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'min' + ) if not isinstance(axis, Variable) and utils._contain_var(axis): axis = utils._convert_to_tensor_list(axis) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reduce_min', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + helper.append_op( + type='reduce_min', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) return out @@ -2514,22 +2606,22 @@ def amax(x, axis=None, keepdim=False, name=None): if in_dygraph_mode(): return _C_ops.amax(x, axis, keepdim) if _in_legacy_dygraph(): - return _legacy_C_ops.reduce_amax(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all) + return _legacy_C_ops.reduce_amax( + x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all + ) helper = LayerHelper('amax', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'amax') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'amax' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reduce_amax', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + helper.append_op( + type='reduce_amax', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) return out @@ -2628,21 +2720,21 @@ def amin(x, axis=None, keepdim=False, name=None): if in_dygraph_mode(): return _C_ops.amin(x, axis, keepdim) elif _in_legacy_dygraph(): - return _legacy_C_ops.reduce_amin(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all) + return _legacy_C_ops.reduce_amin( + x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all + ) helper = LayerHelper('amin', **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'amin') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'amin' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reduce_amin', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + helper.append_op( + type='reduce_amin', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) return out @@ -2856,16 +2948,27 @@ def clip(x, min=None, max=None, name=None): if min is not None: check_type(min, 'min', (float, int, Variable), 'clip') if isinstance(min, Variable): - check_dtype(min.dtype, 'min', ['float32', 'float64', 'int32'], - 'clip', '(When the type of min in clip is Variable.)') + check_dtype( + min.dtype, + 'min', + ['float32', 'float64', 'int32'], + 'clip', + '(When the type of min in clip is Variable.)', + ) if max is not None: check_type(max, 'max', (float, int, Variable), 'clip') if isinstance(max, Variable): - check_dtype(max.dtype, 'max', ['float32', 'float64', 'int32'], - 'clip', '(When the type of max in clip is Variable.)') + check_dtype( + max.dtype, + 'max', + ['float32', 'float64', 'int32'], + 'clip', + '(When the type of max in clip is Variable.)', + ) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'clip') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'clip' + ) inputs = {'X': x} attrs = {'min': min_, 'max': max_} @@ -2884,11 +2987,11 @@ def clip(x, min=None, max=None, name=None): helper = LayerHelper('clip', **locals()) output = helper.create_variable_for_type_inference( - dtype=helper.input_dtype('x')) - helper.append_op(type='clip', - inputs=inputs, - outputs={'Out': [output]}, - attrs=attrs) + dtype=helper.input_dtype('x') + ) + helper.append_op( + type='clip', inputs=inputs, outputs={'Out': [output]}, attrs=attrs + ) return output @@ -2957,52 +3060,56 @@ def trace(x, offset=0, axis1=0, axis2=1, name=None): """ def __check_input(x, offset, axis1, axis2): - check_dtype(x.dtype, 'Input', - ['int32', 'int64', 'float16', 'float32', 'float64'], - 'trace') + check_dtype( + x.dtype, + 'Input', + ['int32', 'int64', 'float16', 'float32', 'float64'], + 'trace', + ) input_shape = list(x.shape) - assert len(input_shape) >= 2, \ - "The x must be at least 2-dimensional, " \ - "But received Input x's dimensional: %s.\n" % \ - len(input_shape) + assert len(input_shape) >= 2, ( + "The x must be at least 2-dimensional, " + "But received Input x's dimensional: %s.\n" % len(input_shape) + ) axis1_ = axis1 if axis1 >= 0 else len(input_shape) + axis1 axis2_ = axis2 if axis2 >= 0 else len(input_shape) + axis2 - assert ((0 <= axis1_) and (axis1_ < len(input_shape))), \ - "The argument axis1 is out of range (expected to be in range of [%d, %d], but got %d).\n" \ + assert (0 <= axis1_) and (axis1_ < len(input_shape)), ( + "The argument axis1 is out of range (expected to be in range of [%d, %d], but got %d).\n" % (-(len(input_shape)), len(input_shape) - 1, axis1) + ) - assert ((0 <= axis2_) and (axis2_ < len(input_shape))), \ - "The argument axis2 is out of range (expected to be in range of [%d, %d], but got %d).\n" \ + assert (0 <= axis2_) and (axis2_ < len(input_shape)), ( + "The argument axis2 is out of range (expected to be in range of [%d, %d], but got %d).\n" % (-(len(input_shape)), len(input_shape) - 1, axis2) + ) - - assert axis1_ != axis2_, \ - "axis1 and axis2 cannot be the same axis." \ - "But received axis1 = %d, axis2 = %d\n"%(axis1, axis2) + assert axis1_ != axis2_, ( + "axis1 and axis2 cannot be the same axis." + "But received axis1 = %d, axis2 = %d\n" % (axis1, axis2) + ) if in_dygraph_mode(): return _C_ops.trace(x, offset, axis1, axis2) if _in_legacy_dygraph(): - return _legacy_C_ops.trace(x, 'offset', offset, 'axis1', axis1, 'axis2', - axis2) + return _legacy_C_ops.trace( + x, 'offset', offset, 'axis1', axis1, 'axis2', axis2 + ) __check_input(x, offset, axis1, axis2) helper = LayerHelper('trace', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='trace', - inputs={'Input': [x]}, - attrs={ - 'offset': offset, - 'axis1': axis1, - 'axis2': axis2 - }, - outputs={'Out': [out]}) + helper.append_op( + type='trace', + inputs={'Input': [x]}, + attrs={'offset': offset, 'axis1': axis1, 'axis2': axis2}, + outputs={'Out': [out]}, + ) return out @@ -3075,47 +3182,52 @@ def diagonal(x, offset=0, axis1=0, axis2=1, name=None): return _C_ops.diagonal(x, offset, axis1, axis2) else: if _in_legacy_dygraph(): - return _legacy_C_ops.diagonal(x, 'offset', offset, 'axis1', axis1, - 'axis2', axis2) + return _legacy_C_ops.diagonal( + x, 'offset', offset, 'axis1', axis1, 'axis2', axis2 + ) def __check_input(x, offset, axis1, axis2): - check_dtype(x.dtype, 'Input', - ['bool', 'int32', 'int64', 'float16', 'float32', 'float64'], - 'diagonal') + check_dtype( + x.dtype, + 'Input', + ['bool', 'int32', 'int64', 'float16', 'float32', 'float64'], + 'diagonal', + ) input_shape = list(x.shape) - assert len(input_shape) >= 2, \ - "The x must be at least 2-dimensional, " \ - "But received Input x's dimensional: %s.\n" % \ - len(input_shape) + assert len(input_shape) >= 2, ( + "The x must be at least 2-dimensional, " + "But received Input x's dimensional: %s.\n" % len(input_shape) + ) axis1_ = axis1 if axis1 >= 0 else len(input_shape) + axis1 axis2_ = axis2 if axis2 >= 0 else len(input_shape) + axis2 - assert axis1_ < len(input_shape), \ - "The argument axis1 is out of range (expected to be in range of [%d, %d], but got %d).\n" \ + assert axis1_ < len(input_shape), ( + "The argument axis1 is out of range (expected to be in range of [%d, %d], but got %d).\n" % (-(len(input_shape)), len(input_shape) - 1, axis1) + ) - assert axis2_ < len(input_shape), \ - "The argument axis2 is out of range (expected to be in range of [%d, %d], but got %d).\n" \ + assert axis2_ < len(input_shape), ( + "The argument axis2 is out of range (expected to be in range of [%d, %d], but got %d).\n" % (-(len(input_shape)), len(input_shape) - 1, axis2) + ) - assert axis1_ != axis2_, \ - "axis1 and axis2 cannot be the same axis." \ - "But received axis1 = %d, axis2 = %d\n"%(axis1, axis2) + assert axis1_ != axis2_, ( + "axis1 and axis2 cannot be the same axis." + "But received axis1 = %d, axis2 = %d\n" % (axis1, axis2) + ) __check_input(x, offset, axis1, axis2) helper = LayerHelper('diagonal', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='diagonal', - inputs={'Input': [x]}, - attrs={ - 'offset': offset, - 'axis1': axis1, - 'axis2': axis2 - }, - outputs={'Out': [out]}) + helper.append_op( + type='diagonal', + inputs={'Input': [x]}, + attrs={'offset': offset, 'axis1': axis1, 'axis2': axis2}, + outputs={'Out': [out]}, + ) return out @@ -3154,9 +3266,11 @@ def kron(x, y, name=None): return _C_ops.kron(x, y) helper = LayerHelper('kron', **locals()) check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'kron') + x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'kron' + ) check_variable_and_dtype( - y, 'y', ['float16', 'float32', 'float64', 'int32', 'int64'], 'kron') + y, 'y', ['float16', 'float32', 'float64', 'int32', 'int64'], 'kron' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) helper.append_op(type="kron", inputs={"X": x, "Y": y}, outputs={"Out": out}) @@ -3212,7 +3326,8 @@ def cumsum(x, axis=None, dtype=None, name=None): x = cast(x, dtype) if in_dygraph_mode(): - if axis is None: axis = -1 + if axis is None: + axis = -1 return _C_ops.cumsum(x, axis, flatten, False, False) if _in_legacy_dygraph(): if axis is None: @@ -3286,27 +3401,29 @@ def logcumsumexp(x, axis=None, dtype=None, name=None): x = cast(x, dtype) if in_dygraph_mode(): - if axis is None: axis = -1 + if axis is None: + axis = -1 return _C_ops.logcumsumexp(x, axis, flatten, False, False) if _in_legacy_dygraph(): if axis is None: return _legacy_C_ops.logcumsumexp(x, 'flatten', flatten) else: - return _legacy_C_ops.logcumsumexp(x, 'axis', axis, 'flatten', - flatten) + return _legacy_C_ops.logcumsumexp( + x, 'axis', axis, 'flatten', flatten + ) - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - "logcumsumexp") + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], "logcumsumexp" + ) helper = LayerHelper('logcumsumexp', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='logcumsumexp', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'axis': axis, - 'flatten': flatten - }) + helper.append_op( + type='logcumsumexp', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'axis': axis, 'flatten': flatten}, + ) return out @@ -3366,17 +3483,21 @@ def cumprod(x, dim=None, dtype=None, name=None): return _legacy_C_ops.cumprod(x, 'dim', dim) check_variable_and_dtype( - x, "x", + x, + "x", ['complex64', 'complex128', 'float32', 'float64', 'int32', 'int64'], - 'cumprod') + 'cumprod', + ) check_type(dim, 'dim', int, 'cumprod') helper = LayerHelper('cumprod', **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='cumprod', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'dim': dim}) + helper.append_op( + type='cumprod', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'dim': dim}, + ) return out @@ -3407,7 +3528,8 @@ def isfinite(x, name=None): return _legacy_C_ops.isfinite_v2(x) helper = LayerHelper("isfinite_v2", **locals()) check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'isfinite') + x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'isfinite' + ) out = helper.create_variable_for_type_inference('bool') helper.append_op(type="isfinite_v2", inputs={"X": x}, outputs={"Out": out}) return out @@ -3440,7 +3562,8 @@ def isinf(x, name=None): return _legacy_C_ops.isinf_v2(x) helper = LayerHelper("isinf_v2", **locals()) check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'isinf') + x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'isinf' + ) out = helper.create_variable_for_type_inference(dtype='bool') helper.append_op(type="isinf_v2", inputs={"X": x}, outputs={"Out": out}) return out @@ -3474,7 +3597,8 @@ def isnan(x, name=None): return _legacy_C_ops.isnan_v2(x) helper = LayerHelper("isnan_v2", **locals()) check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'isnan') + x, 'x', ['float16', 'float32', 'float64', 'int32', 'int64'], 'isnan' + ) out = helper.create_variable_for_type_inference(dtype='bool') helper.append_op(type="isnan_v2", inputs={"X": x}, outputs={"Out": out}) return out @@ -3535,8 +3659,9 @@ def prod(x, axis=None, keepdim=False, dtype=None, name=None): """ if dtype is not None: - check_dtype(dtype, 'dtype', ['float32', 'float64', 'int32', 'int64'], - 'prod') + check_dtype( + dtype, 'dtype', ['float32', 'float64', 'int32', 'int64'], 'prod' + ) if x.dtype != convert_np_dtype_to_dtype_(dtype): x = cast(x, dtype) @@ -3551,35 +3676,39 @@ def prod(x, axis=None, keepdim=False, dtype=None, name=None): dim = [dim] else: raise TypeError( - "The type of axis must be int, list or tuple, but received {}" - .format(type(dim))) + "The type of axis must be int, list or tuple, but received {}".format( + type(dim) + ) + ) - reduce_all = True if dim is None or len(dim) == 0 or len(dim) == len( - x.shape) else False + reduce_all = ( + True + if dim is None or len(dim) == 0 or len(dim) == len(x.shape) + else False + ) if dim is None or len(dim) == 0: dim = [0] if in_dygraph_mode(): return _C_ops.reduce_prod(x, dim, keepdim, reduce_all) if _in_legacy_dygraph(): - return _legacy_C_ops.reduce_prod(x, 'dim', dim, 'keep_dim', keepdim, - 'reduce_all', reduce_all) + return _legacy_C_ops.reduce_prod( + x, 'dim', dim, 'keep_dim', keepdim, 'reduce_all', reduce_all + ) helper = LayerHelper('reduce_prod', **locals()) - check_variable_and_dtype(x, 'x/input', - ['float32', 'float64', 'int32', 'int64'], - 'reduce_prod') + check_variable_and_dtype( + x, 'x/input', ['float32', 'float64', 'int32', 'int64'], 'reduce_prod' + ) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) if not isinstance(dim, Variable) and utils._contain_var(dim): dim = utils._convert_to_tensor_list(dim) - helper.append_op(type='reduce_prod', - inputs={'X': x}, - outputs={'Out': out}, - attrs={ - 'dim': dim, - 'keep_dim': keepdim, - 'reduce_all': reduce_all - }) + helper.append_op( + type='reduce_prod', + inputs={'X': x}, + outputs={'Out': out}, + attrs={'dim': dim, 'keep_dim': keepdim, 'reduce_all': reduce_all}, + ) return out @@ -3697,13 +3826,16 @@ def increment(x, value=1.0, name=None): if _in_legacy_dygraph(): return _legacy_C_ops.increment(x, 'step', value) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'increment') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'increment' + ) helper = LayerHelper("increment", **locals()) - helper.append_op(type='increment', - inputs={'X': [x]}, - outputs={'Out': [x]}, - attrs={'step': float(value)}) + helper.append_op( + type='increment', + inputs={'X': [x]}, + outputs={'Out': [x]}, + attrs={'step': float(value)}, + ) return x @@ -3774,13 +3906,14 @@ def all(x, axis=None, keepdim=False, name=None): if _in_legacy_dygraph(): axis = axis if axis != None and axis != [] else [0] - return _legacy_C_ops.reduce_all(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all_flag) + return _legacy_C_ops.reduce_all( + x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all_flag + ) attrs = { 'dim': axis if axis != None and axis != [] and axis != () else [0], 'keep_dim': keepdim, - 'reduce_all': reduce_all_flag + 'reduce_all': reduce_all_flag, } check_variable_and_dtype(x, 'x', ['bool'], 'all') @@ -3788,10 +3921,9 @@ def all(x, axis=None, keepdim=False, name=None): helper = LayerHelper('all', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reduce_all', - inputs={'X': x}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='reduce_all', inputs={'X': x}, outputs={'Out': out}, attrs=attrs + ) return out @@ -3863,13 +3995,14 @@ def any(x, axis=None, keepdim=False, name=None): if _in_legacy_dygraph(): axis = axis if axis != None and axis != [] else [0] - return _legacy_C_ops.reduce_any(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all_flag) + return _legacy_C_ops.reduce_any( + x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all_flag + ) attrs = { 'dim': axis if axis != None and axis != [] and axis != () else [0], 'keep_dim': keepdim, - 'reduce_all': reduce_all_flag + 'reduce_all': reduce_all_flag, } check_variable_and_dtype(x, 'x', ['bool'], 'any') @@ -3878,10 +4011,9 @@ def any(x, axis=None, keepdim=False, name=None): helper = LayerHelper('any', **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='reduce_any', - inputs={'X': x}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='reduce_any', inputs={'X': x}, outputs={'Out': out}, attrs=attrs + ) return out @@ -3948,9 +4080,11 @@ def conj(x, name=None): return _legacy_C_ops.conj(x) check_variable_and_dtype( - x, "x", + x, + "x", ['complex64', 'complex128', 'float32', 'float64', 'int32', 'int64'], - 'conj') + 'conj', + ) helper = LayerHelper('conj', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) @@ -4057,12 +4191,9 @@ def neg(x, name=None): # [0.4 0.2 -0.1 -0.3] """ - return scale(x, - scale=-1.0, - bias=0.0, - bias_after_scale=True, - act=None, - name=name) + return scale( + x, scale=-1.0, bias=0.0, bias_after_scale=True, act=None, name=name + ) def atan2(x, y, name=None): @@ -4115,11 +4246,17 @@ def atan2(x, y, name=None): return _legacy_C_ops.atan2(x, y) else: check_variable_and_dtype( - x, 'x', ['int32', 'int64', 'float16', 'float32', 'float64'], - 'atan2') + x, + 'x', + ['int32', 'int64', 'float16', 'float32', 'float64'], + 'atan2', + ) check_variable_and_dtype( - y, 'y', ['int32', 'int64', 'float16', 'float32', 'float64'], - 'atan2') + y, + 'y', + ['int32', 'int64', 'float16', 'float32', 'float64'], + 'atan2', + ) helper = LayerHelper('atan2', **locals()) inputs = {'X1': x, 'X2': y} @@ -4178,10 +4315,9 @@ def logit(x, eps=None, name=None): check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'logit') helper = LayerHelper("logit", **locals()) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='logit', - inputs={'X': x}, - outputs={'Out': out}, - attrs={'eps': eps}) + helper.append_op( + type='logit', inputs={'X': x}, outputs={'Out': out}, attrs={'eps': eps} + ) return out @@ -4254,8 +4390,10 @@ def lerp_(x, y, weight, name=None): out_shape = broadcast_shape(out_shape, weight.shape) if out_shape != x.shape: raise ValueError( - "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation." - .format(out_shape, x.shape)) + "The shape of broadcast output {} is different from that of inplace tensor {} in the Inplace operation.".format( + out_shape, x.shape + ) + ) if in_dygraph_mode(): return _C_ops.lerp_(x, y, weight) return _legacy_C_ops.lerp_(x, y, weight) @@ -4363,26 +4501,28 @@ def rad2deg(x, name=None): x = cast(x, dtype="float32") return _legacy_C_ops.scale(x, 'scale', rad2deg_scale) else: - check_variable_and_dtype(x, 'x', - ['int32', 'int64', 'float32', 'float64'], - 'rad2deg') + check_variable_and_dtype( + x, 'x', ['int32', 'int64', 'float32', 'float64'], 'rad2deg' + ) helper = LayerHelper('rad2deg', **locals()) out_cast = x if convert_dtype(x.dtype) in ['int32', 'int64']: out_cast = helper.create_variable_for_type_inference( - dtype=paddle.float32) - helper.append_op(type='cast', - inputs={'X': x}, - outputs={'Out': out_cast}, - attrs={ - 'in_dtype': x.dtype, - 'out_dtype': paddle.float32 - }) + dtype=paddle.float32 + ) + helper.append_op( + type='cast', + inputs={'X': x}, + outputs={'Out': out_cast}, + attrs={'in_dtype': x.dtype, 'out_dtype': paddle.float32}, + ) out = helper.create_variable_for_type_inference(dtype=out_cast.dtype) - helper.append_op(type='scale', - inputs={'X': out_cast}, - outputs={'Out': out}, - attrs={'scale': rad2deg_scale}) + helper.append_op( + type='scale', + inputs={'X': out_cast}, + outputs={'Out': out}, + attrs={'scale': rad2deg_scale}, + ) return out @@ -4429,26 +4569,28 @@ def deg2rad(x, name=None): x = cast(x, dtype="float32") return _legacy_C_ops.scale(x, 'scale', deg2rad_scale) else: - check_variable_and_dtype(x, 'x', - ['int32', 'int64', 'float32', 'float64'], - 'deg2rad') + check_variable_and_dtype( + x, 'x', ['int32', 'int64', 'float32', 'float64'], 'deg2rad' + ) helper = LayerHelper('deg2rad', **locals()) out_cast = x if convert_dtype(x.dtype) in ['int32', 'int64']: out_cast = helper.create_variable_for_type_inference( - dtype=paddle.float32) - helper.append_op(type='cast', - inputs={'X': x}, - outputs={'Out': out_cast}, - attrs={ - 'in_dtype': x.dtype, - 'out_dtype': paddle.float32 - }) + dtype=paddle.float32 + ) + helper.append_op( + type='cast', + inputs={'X': x}, + outputs={'Out': out_cast}, + attrs={'in_dtype': x.dtype, 'out_dtype': paddle.float32}, + ) out = helper.create_variable_for_type_inference(dtype=out_cast.dtype) - helper.append_op(type='scale', - inputs={'X': out_cast}, - outputs={'Out': out}, - attrs={'scale': deg2rad_scale}) + helper.append_op( + type='scale', + inputs={'X': out_cast}, + outputs={'Out': out}, + attrs={'scale': deg2rad_scale}, + ) return out @@ -4513,11 +4655,16 @@ def _gcd_body_fn(x, y): # paddle.mod will raise an error when any element of y is 0. To avoid # that, we change those zeros to ones. Their values don't matter because # they won't be used. - y_not_equal_0 = (y != 0) + y_not_equal_0 = y != 0 y_safe = paddle.where(y_not_equal_0, y, paddle.ones(y.shape, y.dtype)) - x, y = (paddle.where(y_not_equal_0, y, x), - paddle.where(y_not_equal_0, paddle.mod(x, y_safe), - paddle.zeros(y.shape, y.dtype))) + x, y = ( + paddle.where(y_not_equal_0, y, x), + paddle.where( + y_not_equal_0, + paddle.mod(x, y_safe), + paddle.zeros(y.shape, y.dtype), + ), + ) return (paddle.where(x < y, y, x), paddle.where(x < y, x, y)) if paddle.in_dynamic_mode(): @@ -4586,8 +4733,9 @@ def lcm(x, y, name=None): # they won't be used. d_equal_0 = paddle.equal(d, 0) d_safe = paddle.where(d_equal_0, paddle.ones(d.shape, d.dtype), d) - out = paddle.where(d_equal_0, paddle.zeros(d.shape, d.dtype), - paddle.abs(x * y) // d_safe) + out = paddle.where( + d_equal_0, paddle.zeros(d.shape, d.dtype), paddle.abs(x * y) // d_safe + ) return out @@ -4682,14 +4830,16 @@ def diff(x, n=1, axis=-1, prepend=None, append=None, name=None): attrs_1 += ('starts', starts_1) ends_1 = [dim_len - 1] attrs_1 += ('ends', ends_1) - input_front = _C_ops.slice(new_input, axes, starts_1, ends_1, - infer_flags, []) + input_front = _C_ops.slice( + new_input, axes, starts_1, ends_1, infer_flags, [] + ) starts_2 = [1] attrs_2 += ('starts', starts_2) ends_2 = [dim_len] attrs_2 += ('ends', ends_2) - input_back = _C_ops.slice(new_input, axes, starts_2, ends_2, - infer_flags, []) + input_back = _C_ops.slice( + new_input, axes, starts_2, ends_2, infer_flags, [] + ) if x.dtype == paddle.bool: return _C_ops.logical_xor(input_back, input_front) @@ -4722,14 +4872,34 @@ def diff(x, n=1, axis=-1, prepend=None, append=None, name=None): attrs_1 += ('starts', starts_1) ends_1 = [dim_len - 1] attrs_1 += ('ends', ends_1) - input_front = _legacy_C_ops.slice(new_input, None, None, None, None, 'axes', axes, \ - 'infer_flags', infer_flags, *attrs_1) + input_front = _legacy_C_ops.slice( + new_input, + None, + None, + None, + None, + 'axes', + axes, + 'infer_flags', + infer_flags, + *attrs_1 + ) starts_2 = [1] attrs_2 += ('starts', starts_2) ends_2 = [dim_len] attrs_2 += ('ends', ends_2) - input_back = _legacy_C_ops.slice(new_input, None, None, None, None, 'axes', axes, \ - 'infer_flags', infer_flags, *attrs_2) + input_back = _legacy_C_ops.slice( + new_input, + None, + None, + None, + None, + 'axes', + axes, + 'infer_flags', + infer_flags, + *attrs_2 + ) if x.dtype == paddle.bool: return _legacy_C_ops.logical_xor(input_back, input_front) @@ -4737,7 +4907,8 @@ def diff(x, n=1, axis=-1, prepend=None, append=None, name=None): return elementwise_sub(input_back, input_front, axis=axis) else: check_variable_and_dtype( - x, 'x', ['float32', 'float64', 'bool', 'int32', 'int64'], 'diff') + x, 'x', ['float32', 'float64', 'bool', 'int32', 'int64'], 'diff' + ) check_type(axis, 'axis', (int), 'diff') helper = LayerHelper('diff', **locals()) has_pend = False @@ -4754,10 +4925,12 @@ def diff(x, n=1, axis=-1, prepend=None, append=None, name=None): if has_pend: new_input = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='concat', - inputs={'X': input_list}, - outputs={'Out': [new_input]}, - attrs={'axis': axis}) + helper.append_op( + type='concat', + inputs={'X': input_list}, + outputs={'Out': [new_input]}, + attrs={'axis': axis}, + ) else: new_input = x @@ -4768,29 +4941,32 @@ def diff(x, n=1, axis=-1, prepend=None, append=None, name=None): attrs_1['starts'] = starts_1 attrs_1['ends'] = ends_1 input_front = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='slice', - inputs={'Input': new_input}, - attrs=attrs_1, - outputs={'Out': input_front}) + helper.append_op( + type='slice', + inputs={'Input': new_input}, + attrs=attrs_1, + outputs={'Out': input_front}, + ) attrs_2 = {'axes': axes} starts_2 = [1] ends_2 = [dim_len] attrs_2['starts'] = starts_2 attrs_2['ends'] = ends_2 input_back = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='slice', - inputs={'Input': new_input}, - attrs=attrs_2, - outputs={'Out': input_back}) + helper.append_op( + type='slice', + inputs={'Input': new_input}, + attrs=attrs_2, + outputs={'Out': input_back}, + ) if dtype == paddle.bool: out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='logical_xor', - inputs={ - "X": input_back, - "Y": input_front - }, - outputs={"Out": out}) + helper.append_op( + type='logical_xor', + inputs={"X": input_back, "Y": input_front}, + outputs={"Out": out}, + ) else: out = elementwise_sub(input_back, input_front, axis=axis) @@ -4843,14 +5019,15 @@ def angle(x, name=None): elif paddle.in_dynamic_mode(): return _legacy_C_ops.angle(x) - check_variable_and_dtype(x, 'x', - ['float32', 'float64', 'complex64', 'complex128'], - 'angle') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'complex64', 'complex128'], 'angle' + ) op_type = "angle" helper = LayerHelper(op_type, **locals()) inputs = {"X": x} out = helper.create_variable_for_type_inference( - dtype=_complex_to_real_dtype(x.dtype)) + dtype=_complex_to_real_dtype(x.dtype) + ) outputs = {"Out": out} helper.append_op(type=op_type, inputs=inputs, outputs=outputs) return out @@ -4899,11 +5076,9 @@ def heaviside(x, y, name=None): axis = -1 act = None if _non_static_mode(): - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) return _elementwise_op(LayerHelper(op_type, **locals())) @@ -4935,35 +5110,37 @@ def frac(x, name=None): axis = -1 act = None if x.dtype not in [ - paddle.int32, paddle.int64, paddle.float32, paddle.float64 + paddle.int32, + paddle.int64, + paddle.float32, + paddle.float64, ]: raise TypeError( - "The data type of input must be one of ['int32', 'int64', 'float32', 'float64'], but got {}" - .format(x.dtype)) + "The data type of input must be one of ['int32', 'int64', 'float32', 'float64'], but got {}".format( + x.dtype + ) + ) if in_dygraph_mode(): y = _C_ops.trunc(x) return _C_ops.subtract(x, y) else: if _in_legacy_dygraph(): y = _legacy_C_ops.trunc(x) - return _elementwise_op_in_dygraph(x, - y, - axis=axis, - act=act, - op_name=op_type) + return _elementwise_op_in_dygraph( + x, y, axis=axis, act=act, op_name=op_type + ) else: inputs = {"X": x} attrs = {} helper = LayerHelper("trunc", **locals()) - check_variable_and_dtype(x, "X", - ['int32', 'int64', 'float32', 'float64'], - 'trunc') + check_variable_and_dtype( + x, "X", ['int32', 'int64', 'float32', 'float64'], 'trunc' + ) y = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type="trunc", - inputs=inputs, - attrs=attrs, - outputs={"Out": y}) + helper.append_op( + type="trunc", inputs=inputs, attrs=attrs, outputs={"Out": y} + ) return _elementwise_op(LayerHelper(op_type, **locals())) @@ -4993,12 +5170,17 @@ def sgn(x, name=None): """ if x.dtype not in [ - paddle.float16, paddle.float32, paddle.float64, paddle.complex64, - paddle.complex128 + paddle.float16, + paddle.float32, + paddle.float64, + paddle.complex64, + paddle.complex128, ]: raise TypeError( - "The data type of input must be one of ['float16', 'float32', 'float64', 'complex64', 'complex128'], but got {}" - .format(x.dtype)) + "The data type of input must be one of ['float16', 'float32', 'float64', 'complex64', 'complex128'], but got {}".format( + x.dtype + ) + ) if paddle.is_complex(x): expand_x = paddle.as_real(x) x_abs = paddle.abs(x) @@ -5079,18 +5261,24 @@ def take(x, index, mode='raise', name=None): """ if mode not in ['raise', 'wrap', 'clip']: raise ValueError( - "'mode' in 'take' should be 'raise', 'wrap', 'clip', but received {}." - .format(mode)) + "'mode' in 'take' should be 'raise', 'wrap', 'clip', but received {}.".format( + mode + ) + ) if paddle.in_dynamic_mode(): if not isinstance(index, (paddle.Tensor, Variable)): raise TypeError( "The type of 'index' must be Tensor, but got {}".format( - type(index))) + type(index) + ) + ) if index.dtype not in [paddle.int32, paddle.int64]: raise TypeError( - "The data type of 'index' must be one of ['int32', 'int64'], but got {}" - .format(index.dtype)) + "The data type of 'index' must be one of ['int32', 'int64'], but got {}".format( + index.dtype + ) + ) else: check_variable_and_dtype(index, 'index', ['int32', 'int64'], 'take') @@ -5105,8 +5293,9 @@ def take(x, index, mode='raise', name=None): elif mode == 'wrap': # The out of range indices are constrained by taking the remainder. index_1d = paddle.where(index_1d < 0, index_1d % max_index, index_1d) - index_1d = paddle.where(index_1d >= max_index, index_1d % max_index, - index_1d) + index_1d = paddle.where( + index_1d >= max_index, index_1d % max_index, index_1d + ) elif mode == 'clip': # 'clip' mode disables indexing with negative numbers. index_1d = clip(index_1d, 0, max_index - 1) @@ -5140,26 +5329,32 @@ def frexp(x, name=None): print(paddle.tensor.math.frexp(x)) # (Tensor(shape=[1, 4], dtype=float32, place=Place(cpu), stop_gradient=True,[[0.50000000, 0.50000000, 0.75000000, 0.50000000]]), # Tensor(shape=[1, 4], dtype=float32, place=Place(cpu), stop_gradient=True,[[1., 2., 2., 3.]])) - """ + """ if x.dtype not in [paddle.float32, paddle.float64]: raise TypeError( - "The data type of input must be one of ['float32', 'float64'], but got {}" - .format(x.dtype)) + "The data type of input must be one of ['float32', 'float64'], but got {}".format( + x.dtype + ) + ) input_x = paddle.abs(x) exponent = paddle.floor(paddle.log2(input_x)) - exponent = paddle.where(paddle.isinf(exponent), - paddle.full_like(exponent, 0), exponent) + exponent = paddle.where( + paddle.isinf(exponent), paddle.full_like(exponent, 0), exponent + ) # 0填充 mantissa = paddle.divide(input_x, 2**exponent) # 计算exponent - exponent = paddle.where((mantissa >= 1), - paddle.add(exponent, paddle.ones_like(exponent)), - exponent) - mantissa = paddle.where((mantissa >= 1), - paddle.divide(mantissa, - 2**paddle.ones_like(exponent)), - mantissa) + exponent = paddle.where( + (mantissa >= 1), + paddle.add(exponent, paddle.ones_like(exponent)), + exponent, + ) + mantissa = paddle.where( + (mantissa >= 1), + paddle.divide(mantissa, 2 ** paddle.ones_like(exponent)), + mantissa, + ) mantissa = paddle.where((x < 0), mantissa * -1, mantissa) return mantissa, exponent diff --git a/python/paddle/tensor/ops.py b/python/paddle/tensor/ops.py index 46648f06d8205..b3f88e9fcdf41 100644 --- a/python/paddle/tensor/ops.py +++ b/python/paddle/tensor/ops.py @@ -12,13 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .layer_function_generator import generate_layer_fn, generate_activation_fn, generate_inplace_fn, add_sample_code +from .layer_function_generator import ( + generate_layer_fn, + generate_activation_fn, + generate_inplace_fn, + add_sample_code, +) from ..fluid.framework import in_dygraph_mode from .. import _C_ops __deprecated_func_name__ = { 'tanh_shrink': 'tanhshrink', - 'logsigmoid': 'log_sigmoid' + 'logsigmoid': 'log_sigmoid', } __activations_noattr__ = [ @@ -103,7 +108,8 @@ globals()[_OP] = _func add_sample_code( - globals()["sigmoid"], r""" + globals()["sigmoid"], + r""" Examples: .. code-block:: python @@ -115,10 +121,12 @@ print(out) # [0.40131234 0.450166 0.52497919 0.57444252] -""") +""", +) add_sample_code( - globals()["silu"], r""" + globals()["silu"], + r""" Examples: .. code-block:: python @@ -130,10 +138,12 @@ print(out) # [ 0.7310586 1.7615942 2.8577224, 3.9280552 ] -""") +""", +) add_sample_code( - globals()["logsigmoid"], r""" + globals()["logsigmoid"], + r""" Examples: .. code-block:: python @@ -145,10 +155,12 @@ print(out) # [-0.91301525 -0.79813887 -0.64439666 -0.55435524] -""") +""", +) add_sample_code( - globals()["exp"], r""" + globals()["exp"], + r""" Examples: .. code-block:: python @@ -159,10 +171,12 @@ print(out) # [0.67032005 0.81873075 1.10517092 1.34985881] -""") +""", +) add_sample_code( - globals()["expm1"], r""" + globals()["expm1"], + r""" Examples: .. code-block:: python @@ -173,10 +187,12 @@ print(out) # [-0.32967997, -0.18126924, 0.10517092, 0.34985882] -""") +""", +) add_sample_code( - globals()["tanh"], r""" + globals()["tanh"], + r""" Examples: .. code-block:: python @@ -187,10 +203,12 @@ print(out) # [-0.37994896 -0.19737532 0.09966799 0.29131261] -""") +""", +) add_sample_code( - globals()["atan"], r""" + globals()["atan"], + r""" Examples: .. code-block:: python @@ -201,10 +219,12 @@ print(out) # [-0.38050638 -0.19739556 0.09966865 0.29145679] -""") +""", +) add_sample_code( - globals()["tanh_shrink"], r""" + globals()["tanh_shrink"], + r""" Examples: .. code-block:: python @@ -216,10 +236,12 @@ print(out) # [-0.020051, -0.00262468, 0.000332005, 0.00868739] -""") +""", +) add_sample_code( - globals()["sqrt"], r""" + globals()["sqrt"], + r""" Examples: .. code-block:: python @@ -230,10 +252,12 @@ print(out) # [0.31622777 0.4472136 0.54772256 0.63245553] -""") +""", +) add_sample_code( - globals()["rsqrt"], r""" + globals()["rsqrt"], + r""" Examples: .. code-block:: python @@ -244,10 +268,12 @@ print(out) # [3.16227766 2.23606798 1.82574186 1.58113883] -""") +""", +) add_sample_code( - globals()["abs"], r""" + globals()["abs"], + r""" Examples: .. code-block:: python @@ -258,10 +284,12 @@ print(out) # [0.4 0.2 0.1 0.3] -""") +""", +) add_sample_code( - globals()["ceil"], r""" + globals()["ceil"], + r""" Examples: .. code-block:: python @@ -272,10 +300,12 @@ print(out) # [-0. -0. 1. 1.] -""") +""", +) add_sample_code( - globals()["floor"], r""" + globals()["floor"], + r""" Examples: .. code-block:: python @@ -286,10 +316,12 @@ print(out) # [-1. -1. 0. 0.] -""") +""", +) add_sample_code( - globals()["cos"], r""" + globals()["cos"], + r""" Examples: .. code-block:: python @@ -300,10 +332,12 @@ print(out) # [0.92106099 0.98006658 0.99500417 0.95533649] -""") +""", +) add_sample_code( - globals()["tan"], r""" + globals()["tan"], + r""" Examples: .. code-block:: python @@ -314,10 +348,12 @@ print(out) # [-0.42279324, -0.20271005, 0.10033467, 0.30933627] -""") +""", +) add_sample_code( - globals()["acos"], r""" + globals()["acos"], + r""" Examples: .. code-block:: python @@ -328,10 +364,12 @@ print(out) # [1.98231317 1.77215425 1.47062891 1.26610367] -""") +""", +) add_sample_code( - globals()["sin"], r""" + globals()["sin"], + r""" Examples: .. code-block:: python @@ -342,10 +380,12 @@ print(out) # [-0.38941834 -0.19866933 0.09983342 0.29552021] -""") +""", +) add_sample_code( - globals()["asin"], r""" + globals()["asin"], + r""" Examples: .. code-block:: python @@ -356,10 +396,12 @@ print(out) # [-0.41151685 -0.20135792 0.10016742 0.30469265] -""") +""", +) add_sample_code( - globals()["cosh"], r""" + globals()["cosh"], + r""" Examples: .. code-block:: python @@ -370,10 +412,12 @@ print(out) # [1.08107237 1.02006676 1.00500417 1.04533851] -""") +""", +) add_sample_code( - globals()["sinh"], r""" + globals()["sinh"], + r""" Examples: .. code-block:: python @@ -384,10 +428,12 @@ print(out) # [-0.41075233 -0.201336 0.10016675 0.30452029] -""") +""", +) add_sample_code( - globals()["asinh"], r""" + globals()["asinh"], + r""" Examples: .. code-block:: python @@ -398,10 +444,12 @@ print(out) # [-0.39003533, -0.19869010, 0.09983408, 0.29567307] -""") +""", +) add_sample_code( - globals()["acosh"], r""" + globals()["acosh"], + r""" Examples: .. code-block:: python @@ -412,10 +460,12 @@ print(out) # [0. , 1.76274729, 2.06343699, 2.29243159] -""") +""", +) add_sample_code( - globals()["atanh"], r""" + globals()["atanh"], + r""" Examples: .. code-block:: python @@ -426,10 +476,12 @@ print(out) # [-0.42364895, -0.20273256, 0.10033535, 0.30951962] -""") +""", +) add_sample_code( - globals()["round"], r""" + globals()["round"], + r""" Examples: .. code-block:: python @@ -440,10 +492,12 @@ print(out) # [-1. -0. 1. 2.] -""") +""", +) add_sample_code( - globals()["reciprocal"], r""" + globals()["reciprocal"], + r""" Examples: .. code-block:: python @@ -454,10 +508,12 @@ print(out) # [-2.5 -5. 10. 3.33333333] -""") +""", +) add_sample_code( - globals()["square"], r""" + globals()["square"], + r""" Examples: .. code-block:: python @@ -468,10 +524,12 @@ print(out) # [0.16 0.04 0.01 0.09] -""") +""", +) add_sample_code( - globals()["softplus"], r""" + globals()["softplus"], + r""" Examples: .. code-block:: python @@ -483,10 +541,12 @@ print(out) # [0.513015, 0.598139, 0.744397, 0.854355] -""") +""", +) add_sample_code( - globals()["softsign"], r""" + globals()["softsign"], + r""" Examples: .. code-block:: python @@ -498,7 +558,8 @@ print(out) # [-0.285714, -0.166667, 0.0909091, 0.230769] -""") +""", +) __all__ += ['erf'] diff --git a/python/paddle/tensor/random.py b/python/paddle/tensor/random.py index 208139b1abbb6..2449f9c3194e7 100644 --- a/python/paddle/tensor/random.py +++ b/python/paddle/tensor/random.py @@ -17,12 +17,21 @@ from ..framework import core from ..framework import convert_np_dtype_to_dtype_, dygraph_only from ..framework import LayerHelper -from ..fluid.data_feeder import check_variable_and_dtype, check_type, check_dtype, check_shape +from ..fluid.data_feeder import ( + check_variable_and_dtype, + check_type, + check_dtype, + check_shape, +) from ..fluid.layers import utils import paddle from paddle import _C_ops, _legacy_C_ops from paddle.static import Variable -from paddle.fluid.framework import in_dygraph_mode, _in_legacy_dygraph, _current_expected_place +from paddle.fluid.framework import ( + in_dygraph_mode, + _in_legacy_dygraph, + _current_expected_place, +) __all__ = [] @@ -75,11 +84,11 @@ def bernoulli(x, name=None): helper = LayerHelper("randint", **locals()) out = helper.create_variable_for_type_inference( - dtype=x.dtype) # maybe set out to int32 ? - helper.append_op(type='bernoulli', - inputs={"X": x}, - outputs={'Out': out}, - attrs={}) + dtype=x.dtype + ) # maybe set out to int32 ? + helper.append_op( + type='bernoulli', inputs={"X": x}, outputs={'Out': out}, attrs={} + ) out.stop_gradient = True return out @@ -124,10 +133,9 @@ def poisson(x, name=None): helper = LayerHelper("poisson", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='poisson', - inputs={'X': x}, - outputs={'Out': out}, - attrs={}) + helper.append_op( + type='poisson', inputs={'X': x}, outputs={'Out': out}, attrs={} + ) return out @@ -179,28 +187,30 @@ def multinomial(x, num_samples=1, replacement=False, name=None): """ - assert core.is_compiled_with_rocm() == False, ( - "multinomial op is not supported on ROCM yet.") + assert ( + core.is_compiled_with_rocm() == False + ), "multinomial op is not supported on ROCM yet." if in_dygraph_mode(): return _C_ops.multinomial(x, num_samples, replacement) if _in_legacy_dygraph(): - return _legacy_C_ops.multinomial(x, 'num_samples', num_samples, - 'replacement', replacement) + return _legacy_C_ops.multinomial( + x, 'num_samples', num_samples, 'replacement', replacement + ) check_variable_and_dtype(x, "x", ["float32", "float64"], "multinomial") helper = LayerHelper("multinomial", **locals()) out = helper.create_variable_for_type_inference( - dtype=convert_np_dtype_to_dtype_('int64')) - helper.append_op(type='multinomial', - inputs={"X": x}, - outputs={'Out': out}, - attrs={ - 'num_samples': num_samples, - 'replacement': replacement - }) + dtype=convert_np_dtype_to_dtype_('int64') + ) + helper.append_op( + type='multinomial', + inputs={"X": x}, + outputs={'Out': out}, + attrs={'num_samples': num_samples, 'replacement': replacement}, + ) out.stop_gradient = True return out @@ -237,23 +247,34 @@ def gaussian(shape, mean=0.0, std=1.0, dtype=None, name=None): dtype = paddle.framework.get_default_dtype() if dtype not in ['float32', 'float64']: raise TypeError( - "{} only supports [float32, float64], but the default dtype is {}" - .format(op_type_for_check, dtype)) + "{} only supports [float32, float64], but the default dtype is {}".format( + op_type_for_check, dtype + ) + ) if not isinstance(dtype, core.VarDesc.VarType): dtype = convert_np_dtype_to_dtype_(dtype) if in_dygraph_mode(): shape = utils.convert_shape_to_list(shape) place = _current_expected_place() - return _C_ops.gaussian_random(shape, float(mean), float(std), seed, - dtype, place) + return _C_ops.gaussian_random( + shape, float(mean), float(std), seed, dtype, place + ) if _in_legacy_dygraph(): shape = utils.convert_shape_to_list(shape) - return _legacy_C_ops.gaussian_random('shape', shape, - 'mean', float(mean), 'std', - float(std), 'seed', seed, 'dtype', - dtype) + return _legacy_C_ops.gaussian_random( + 'shape', + shape, + 'mean', + float(mean), + 'std', + float(std), + 'seed', + seed, + 'dtype', + dtype, + ) check_shape(shape, op_type_for_check) check_dtype(dtype, 'dtype', ['float32', 'float64'], op_type_for_check) @@ -264,19 +285,17 @@ def gaussian(shape, mean=0.0, std=1.0, dtype=None, name=None): 'std': std, 'seed': seed, 'dtype': dtype, - 'use_mkldnn': False + 'use_mkldnn': False, } - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type=op_type_for_check) + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type=op_type_for_check + ) helper = LayerHelper('gaussian', **locals()) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='gaussian_random', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='gaussian_random', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) out.stop_gradient = True return out @@ -446,13 +465,19 @@ def normal(mean=0.0, std=1.0, shape=None, name=None): check_type(std, 'std', (int, float, Variable), 'normal') if isinstance(mean, Variable): check_dtype( - mean.dtype, 'mean', ['float32', 'float64'], 'normal', - "If mean is Tensor, it's data type only support float32, float64." + mean.dtype, + 'mean', + ['float32', 'float64'], + 'normal', + "If mean is Tensor, it's data type only support float32, float64.", ) if isinstance(std, Variable): check_dtype( - std.dtype, 'std', ['float32', 'float64'], 'normal', - "If std is Tensor, it's data type only support float32, float64." + std.dtype, + 'std', + ['float32', 'float64'], + 'normal', + "If std is Tensor, it's data type only support float32, float64.", ) if shape is not None: check_shape(shape, 'normal') @@ -549,23 +574,39 @@ def uniform(shape, dtype=None, min=-1.0, max=1.0, seed=0, name=None): dtype = paddle.framework.get_default_dtype() if dtype not in ['float32', 'float64']: raise TypeError( - "uniform/rand only supports [float32, float64], but the default dtype is {}" - .format(dtype)) + "uniform/rand only supports [float32, float64], but the default dtype is {}".format( + dtype + ) + ) if not isinstance(dtype, core.VarDesc.VarType): dtype = convert_np_dtype_to_dtype_(dtype) if in_dygraph_mode(): shape = utils.convert_shape_to_list(shape) - return _C_ops.uniform_random(shape, dtype, float(min), float(max), seed, - _current_expected_place()) + return _C_ops.uniform_random( + shape, + dtype, + float(min), + float(max), + seed, + _current_expected_place(), + ) if _in_legacy_dygraph(): shape = utils.convert_shape_to_list(shape) - return _legacy_C_ops.uniform_random('shape', - shape, 'min', float(min), 'max', - float(max), 'seed', seed, 'dtype', - dtype) + return _legacy_C_ops.uniform_random( + 'shape', + shape, + 'min', + float(min), + 'max', + float(max), + 'seed', + seed, + 'dtype', + dtype, + ) check_type(shape, 'shape', (list, tuple, Variable), 'uniform/rand') check_dtype(dtype, 'dtype', ('float32', 'float64'), 'uniform/rand') @@ -574,17 +615,15 @@ def uniform(shape, dtype=None, min=-1.0, max=1.0, seed=0, name=None): inputs = dict() attrs = {'seed': seed, 'min': min, 'max': max, 'dtype': dtype} - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type='uniform/rand') + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type='uniform/rand' + ) helper = LayerHelper("uniform", **locals()) out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type="uniform_random", - inputs=inputs, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="uniform_random", inputs=inputs, attrs=attrs, outputs={"Out": out} + ) out.stop_gradient = True return out @@ -627,8 +666,9 @@ def uniform_(x, min=-1.0, max=1.0, seed=0, name=None): if in_dygraph_mode(): return _C_ops.uniform_random_inplace_(x, min, max, seed, 0, 0, 1.0) else: - return _legacy_C_ops.uniform_random_inplace_(x, 'min', min, 'max', max, - 'seed', seed) + return _legacy_C_ops.uniform_random_inplace_( + x, 'min', min, 'max', max, 'seed', seed + ) def randint(low=0, high=None, shape=[1], dtype=None, name=None): @@ -699,8 +739,10 @@ def randint(low=0, high=None, shape=[1], dtype=None, name=None): if high is None: if low <= 0: raise ValueError( - "If high is None, low must be greater than 0, but received low = {0}." - .format(low)) + "If high is None, low must be greater than 0, but received low = {0}.".format( + low + ) + ) high = low low = 0 if dtype is None: @@ -714,29 +756,29 @@ def randint(low=0, high=None, shape=[1], dtype=None, name=None): return _C_ops.randint(low, high, shape, dtype, place) if _in_legacy_dygraph(): shape = utils.convert_shape_to_list(shape) - return _legacy_C_ops.randint('shape', shape, 'low', low, 'high', high, - 'seed', 0, 'dtype', dtype) + return _legacy_C_ops.randint( + 'shape', shape, 'low', low, 'high', high, 'seed', 0, 'dtype', dtype + ) check_shape(shape, 'randint') check_dtype(dtype, 'dtype', ['int32', 'int64'], 'randint') if low >= high: raise ValueError( "randint's low must less then high, but received low = {0}, " - "high = {1}".format(low, high)) + "high = {1}".format(low, high) + ) inputs = dict() attrs = {'low': low, 'high': high, 'seed': 0, 'dtype': dtype} - utils.get_shape_tensor_inputs(inputs=inputs, - attrs=attrs, - shape=shape, - op_type='randint') + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type='randint' + ) helper = LayerHelper("randint", **locals()) out = helper.create_variable_for_type_inference(dtype=dtype) - helper.append_op(type='randint', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='randint', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) out.stop_gradient = True return out @@ -868,8 +910,10 @@ def randint_like(x, low=0, high=None, dtype=None, name=None): if high is None: if low <= 0: raise ValueError( - "If high is None, low must be greater than 0, but received low = {0}." - .format(low)) + "If high is None, low must be greater than 0, but received low = {0}.".format( + low + ) + ) high = low low = 0 if dtype is None: @@ -881,36 +925,49 @@ def randint_like(x, low=0, high=None, dtype=None, name=None): if low >= high: raise ValueError( "randint_like's low must less then high, but received low = {0}, " - "high = {1}".format(low, high)) + "high = {1}".format(low, high) + ) if paddle.in_dynamic_mode(): shape = utils.convert_shape_to_list(shape) - out = _legacy_C_ops.randint('shape', shape, 'low', low, 'high', high, - 'seed', 0, 'dtype', - core.VarDesc.VarType.INT64) + out = _legacy_C_ops.randint( + 'shape', + shape, + 'low', + low, + 'high', + high, + 'seed', + 0, + 'dtype', + core.VarDesc.VarType.INT64, + ) out = paddle.cast(out, dtype) return out check_shape(shape, 'randint_like') - check_dtype(dtype, 'dtype', - ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], - 'randint_like') + check_dtype( + dtype, + 'dtype', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'randint_like', + ) inputs = {"ShapeTensor": shape} attrs = { 'low': low, 'high': high, 'seed': 0, - 'dtype': core.VarDesc.VarType.INT64 + 'dtype': core.VarDesc.VarType.INT64, } helper = LayerHelper("randint", **locals()) out = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.INT64) - helper.append_op(type='randint', - inputs=inputs, - outputs={'Out': out}, - attrs=attrs) + dtype=core.VarDesc.VarType.INT64 + ) + helper.append_op( + type='randint', inputs=inputs, outputs={'Out': out}, attrs=attrs + ) out.stop_gradient = True out = paddle.cast(out, dtype) return out @@ -956,16 +1013,16 @@ def randperm(n, dtype="int64", name=None): if n < 1: raise ValueError("The input n should be greater than 0 in randperm op.") - check_dtype(dtype, 'dtype', ['int64', 'int32', 'float32', 'float64'], - 'randperm') + check_dtype( + dtype, 'dtype', ['int64', 'int32', 'float32', 'float64'], 'randperm' + ) helper = LayerHelper("randperm", **locals()) out = helper.create_variable_for_type_inference(dtype) attrs = {'n': n, 'dtype': dtype, 'seed': 0} - helper.append_op(type='randperm', - inputs={}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='randperm', inputs={}, outputs={'Out': out}, attrs=attrs + ) out.stop_gradient = True return out @@ -1064,8 +1121,10 @@ def exponential_(x, lam=1.0, name=None): check_variable_and_dtype(x, "x", ["float32", "float64"], "exponential") helper = LayerHelper("exponential", **locals()) - helper.append_op(type='exponential', - inputs={"X": x}, - outputs={'Out': x}, - attrs={"lambda": lam}) + helper.append_op( + type='exponential', + inputs={"X": x}, + outputs={'Out': x}, + attrs={"lambda": lam}, + ) return x diff --git a/python/paddle/tensor/search.py b/python/paddle/tensor/search.py index ffb6573c9eb18..165809b39fb62 100644 --- a/python/paddle/tensor/search.py +++ b/python/paddle/tensor/search.py @@ -93,28 +93,30 @@ def argsort(x, axis=-1, descending=False, name=None): return ids if _in_legacy_dygraph(): - _, ids = _legacy_C_ops.argsort(x, 'axis', axis, 'descending', - descending) + _, ids = _legacy_C_ops.argsort( + x, 'axis', axis, 'descending', descending + ) return ids check_variable_and_dtype( - x, 'x', ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], - 'argsort') + x, + 'x', + ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], + 'argsort', + ) helper = LayerHelper("argsort", **locals()) - out = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=True) - ids = helper.create_variable_for_type_inference(VarDesc.VarType.INT64, - stop_gradient=True) - helper.append_op(type='argsort', - inputs={'X': x}, - outputs={ - 'Out': out, - 'Indices': ids - }, - attrs={ - 'axis': axis, - 'descending': descending - }) + out = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=True + ) + ids = helper.create_variable_for_type_inference( + VarDesc.VarType.INT64, stop_gradient=True + ) + helper.append_op( + type='argsort', + inputs={'X': x}, + outputs={'Out': out, 'Indices': ids}, + attrs={'axis': axis, 'descending': descending}, + ) return ids @@ -161,7 +163,8 @@ def argmax(x, axis=None, keepdim=False, dtype="int64", name=None): if axis is not None and not isinstance(axis, (int, Variable)): raise TypeError( "The type of 'axis' must be int or Tensor or None in argmax, but received %s." - % (type(axis))) + % (type(axis)) + ) if dtype is None: raise ValueError( @@ -177,14 +180,26 @@ def argmax(x, axis=None, keepdim=False, dtype="int64", name=None): if in_dygraph_mode(): return _C_ops.argmax(x, axis, keepdim, flatten, var_dtype) if _in_legacy_dygraph(): - out = _legacy_C_ops.arg_max(x, 'axis', axis, 'dtype', var_dtype, - 'keepdims', keepdim, 'flatten', flatten) + out = _legacy_C_ops.arg_max( + x, + 'axis', + axis, + 'dtype', + var_dtype, + 'keepdims', + keepdim, + 'flatten', + flatten, + ) return out helper = LayerHelper("argmax", **locals()) check_variable_and_dtype( - x, 'x', ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], - 'paddle.argmax') + x, + 'x', + ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], + 'paddle.argmax', + ) check_dtype(var_dtype, 'dtype', ['int32', 'int64'], 'argmin') attrs = {} out = helper.create_variable_for_type_inference(var_dtype) @@ -192,10 +207,9 @@ def argmax(x, axis=None, keepdim=False, dtype="int64", name=None): attrs['axis'] = axis attrs['flatten'] = flatten attrs['dtype'] = var_dtype - helper.append_op(type='arg_max', - inputs={'X': x}, - outputs={'Out': [out]}, - attrs=attrs) + helper.append_op( + type='arg_max', inputs={'X': x}, outputs={'Out': [out]}, attrs=attrs + ) out.stop_gradient = True return out @@ -243,7 +257,8 @@ def argmin(x, axis=None, keepdim=False, dtype="int64", name=None): if axis is not None and not isinstance(axis, (int, Variable)): raise TypeError( "The type of 'axis' must be int or Tensor or None in argmin, but received %s." - % (type(axis))) + % (type(axis)) + ) if dtype is None: raise ValueError( @@ -259,14 +274,26 @@ def argmin(x, axis=None, keepdim=False, dtype="int64", name=None): if in_dygraph_mode(): return _C_ops.argmin(x, axis, keepdim, flatten, var_dtype) if _in_legacy_dygraph(): - out = _legacy_C_ops.arg_min(x, 'axis', axis, 'dtype', var_dtype, - 'keepdims', keepdim, 'flatten', flatten) + out = _legacy_C_ops.arg_min( + x, + 'axis', + axis, + 'dtype', + var_dtype, + 'keepdims', + keepdim, + 'flatten', + flatten, + ) return out helper = LayerHelper("argmin", **locals()) check_variable_and_dtype( - x, 'x', ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], - 'paddle.argmin') + x, + 'x', + ['float32', 'float64', 'int16', 'int32', 'int64', 'uint8'], + 'paddle.argmin', + ) check_dtype(var_dtype, 'dtype', ['int32', 'int64'], 'argmin') out = helper.create_variable_for_type_inference(var_dtype) attrs = {} @@ -274,10 +301,9 @@ def argmin(x, axis=None, keepdim=False, dtype="int64", name=None): attrs['axis'] = axis attrs['flatten'] = flatten attrs['dtype'] = var_dtype - helper.append_op(type='arg_min', - inputs={'X': x}, - outputs={'Out': [out]}, - attrs=attrs) + helper.append_op( + type='arg_min', inputs={'X': x}, outputs={'Out': [out]}, attrs=attrs + ) out.stop_gradient = True return out @@ -325,20 +351,24 @@ def index_select(x, index, axis=0, name=None): return _legacy_C_ops.index_select(x, index, 'dim', axis) helper = LayerHelper("index_select", **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'paddle.tensor.search.index_select') - check_variable_and_dtype(index, 'index', ['int32', 'int64'], - 'paddle.tensor.search.index_select') + check_variable_and_dtype( + x, + 'x', + ['float32', 'float64', 'int32', 'int64'], + 'paddle.tensor.search.index_select', + ) + check_variable_and_dtype( + index, 'index', ['int32', 'int64'], 'paddle.tensor.search.index_select' + ) out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='index_select', - inputs={ - 'X': x, - 'Index': index - }, - outputs={'Out': out}, - attrs={'dim': axis}) + helper.append_op( + type='index_select', + inputs={'X': x, 'Index': index}, + outputs={'Out': out}, + attrs={'dim': axis}, + ) return out @@ -406,11 +436,12 @@ def nonzero(x, as_tuple=False): helper = LayerHelper("where_index", **locals()) outs = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.INT64) + dtype=core.VarDesc.VarType.INT64 + ) - helper.append_op(type='where_index', - inputs={'Condition': x}, - outputs={'Out': [outs]}) + helper.append_op( + type='where_index', inputs={'Condition': x}, outputs={'Out': [outs]} + ) if not as_tuple: return outs @@ -419,7 +450,8 @@ def nonzero(x, as_tuple=False): else: for i in range(rank): list_out.append( - paddle.slice(outs, axes=[1], starts=[i], ends=[i + 1])) + paddle.slice(outs, axes=[1], starts=[i], ends=[i + 1]) + ) return tuple(list_out) @@ -484,24 +516,23 @@ def sort(x, axis=-1, descending=False, name=None): return outs if _in_legacy_dygraph(): - outs, _ = _legacy_C_ops.argsort(x, 'axis', axis, 'descending', - descending) + outs, _ = _legacy_C_ops.argsort( + x, 'axis', axis, 'descending', descending + ) return outs helper = LayerHelper("sort", **locals()) - out = helper.create_variable_for_type_inference(dtype=x.dtype, - stop_gradient=False) - ids = helper.create_variable_for_type_inference(VarDesc.VarType.INT64, - stop_gradient=True) - helper.append_op(type='argsort', - inputs={'X': x}, - outputs={ - 'Out': out, - 'Indices': ids - }, - attrs={ - 'axis': axis, - 'descending': descending - }) + out = helper.create_variable_for_type_inference( + dtype=x.dtype, stop_gradient=False + ) + ids = helper.create_variable_for_type_inference( + VarDesc.VarType.INT64, stop_gradient=True + ) + helper.append_op( + type='argsort', + inputs={'X': x}, + outputs={'Out': out, 'Indices': ids}, + attrs={'axis': axis, 'descending': descending}, + ) return out @@ -550,13 +581,12 @@ def mode(x, axis=-1, keepdim=False, name=None): values = helper.create_variable_for_type_inference(dtype=x.dtype) indices = helper.create_variable_for_type_inference(dtype="int64") - helper.append_op(type="mode", - inputs=inputs, - outputs={ - "Out": [values], - "Indices": [indices] - }, - attrs=attrs) + helper.append_op( + type="mode", + inputs=inputs, + outputs={"Out": [values], "Indices": [indices]}, + attrs=attrs, + ) indices.stop_gradient = True return values, indices @@ -618,12 +648,12 @@ def where(condition, x=None, y=None, name=None): if not paddle.in_dynamic_mode(): check_variable_and_dtype(condition, 'condition', ['bool'], 'where') - check_variable_and_dtype(x, 'x', - ['float32', 'float64', 'int32', 'int64'], - 'where') - check_variable_and_dtype(y, 'y', - ['float32', 'float64', 'int32', 'int64'], - 'where') + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'where' + ) + check_variable_and_dtype( + y, 'y', ['float32', 'float64', 'int32', 'int64'], 'where' + ) condition_shape = list(condition.shape) x_shape = list(x.shape) @@ -651,19 +681,22 @@ def where(condition, x=None, y=None, name=None): return _C_ops.where(broadcast_condition, broadcast_x, broadcast_y) else: if _in_legacy_dygraph(): - return _legacy_C_ops.where(broadcast_condition, broadcast_x, - broadcast_y) + return _legacy_C_ops.where( + broadcast_condition, broadcast_x, broadcast_y + ) else: helper = LayerHelper("where", **locals()) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='where', - inputs={ - 'Condition': broadcast_condition, - 'X': broadcast_x, - 'Y': broadcast_y - }, - outputs={'Out': [out]}) + helper.append_op( + type='where', + inputs={ + 'Condition': broadcast_condition, + 'X': broadcast_x, + 'Y': broadcast_y, + }, + outputs={'Out': [out]}, + ) return out @@ -748,19 +781,25 @@ def index_sample(x, index): return _legacy_C_ops.index_sample(x, index) else: helper = LayerHelper("index_sample", **locals()) - check_variable_and_dtype(x, 'x', - ['float32', 'float64', 'int32', 'int64'], - 'paddle.tensor.search.index_sample') - check_variable_and_dtype(index, 'index', ['int32', 'int64'], - 'paddle.tensor.search.index_sample') + check_variable_and_dtype( + x, + 'x', + ['float32', 'float64', 'int32', 'int64'], + 'paddle.tensor.search.index_sample', + ) + check_variable_and_dtype( + index, + 'index', + ['int32', 'int64'], + 'paddle.tensor.search.index_sample', + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='index_sample', - inputs={ - 'X': x, - 'Index': index - }, - outputs={'Out': out}) + helper.append_op( + type='index_sample', + inputs={'X': x, 'Index': index}, + outputs={'Out': out}, + ) return out @@ -800,17 +839,19 @@ def masked_select(x, mask, name=None): return _legacy_C_ops.masked_select(x, mask) helper = LayerHelper("masked_select", **locals()) - check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'], - 'paddle.tensor.search.mask_select') - check_variable_and_dtype(mask, 'mask', ['bool'], - 'paddle.tensor.search.masked_select') + check_variable_and_dtype( + x, + 'x', + ['float32', 'float64', 'int32', 'int64'], + 'paddle.tensor.search.mask_select', + ) + check_variable_and_dtype( + mask, 'mask', ['bool'], 'paddle.tensor.search.masked_select' + ) out = helper.create_variable_for_type_inference(dtype=x.dtype) - helper.append_op(type='masked_select', - inputs={ - 'X': x, - 'Mask': mask - }, - outputs={'Y': out}) + helper.append_op( + type='masked_select', inputs={'X': x, 'Mask': mask}, outputs={'Y': out} + ) return out @@ -870,12 +911,21 @@ def topk(x, k, axis=None, largest=True, sorted=True, name=None): if _non_static_mode(): if axis is None: - out, indices = _legacy_C_ops.top_k_v2(x, 'k', int(k), 'largest', - largest, 'sorted', sorted) + out, indices = _legacy_C_ops.top_k_v2( + x, 'k', int(k), 'largest', largest, 'sorted', sorted + ) else: - out, indices = _legacy_C_ops.top_k_v2(x, 'k', int(k), 'axis', axis, - 'largest', largest, 'sorted', - sorted) + out, indices = _legacy_C_ops.top_k_v2( + x, + 'k', + int(k), + 'axis', + axis, + 'largest', + largest, + 'sorted', + sorted, + ) return out, indices helper = LayerHelper("top_k_v2", **locals()) @@ -893,13 +943,12 @@ def topk(x, k, axis=None, largest=True, sorted=True, name=None): values = helper.create_variable_for_type_inference(dtype=x.dtype) indices = helper.create_variable_for_type_inference(dtype="int64") - helper.append_op(type="top_k_v2", - inputs=inputs, - outputs={ - "Out": [values], - "Indices": [indices] - }, - attrs=attrs) + helper.append_op( + type="top_k_v2", + inputs=inputs, + outputs={"Out": [values], "Indices": [indices]}, + attrs=attrs, + ) indices.stop_gradient = True return values, indices @@ -949,9 +998,12 @@ def bucketize(x, sorted_sequence, out_int32=False, right=False, name=None): # [0, 1, 3, 2]]) """ - check_variable_and_dtype(sorted_sequence, 'SortedSequence', - ['float32', 'float64', 'int32', 'int64'], - 'paddle.searchsorted') + check_variable_and_dtype( + sorted_sequence, + 'SortedSequence', + ['float32', 'float64', 'int32', 'int64'], + 'paddle.searchsorted', + ) if sorted_sequence.dim() != 1: raise ValueError( f"sorted_sequence tensor must be 1 dimension, but got dim {sorted_sequence.dim()}" @@ -959,11 +1011,9 @@ def bucketize(x, sorted_sequence, out_int32=False, right=False, name=None): return searchsorted(sorted_sequence, x, out_int32, right, name) -def searchsorted(sorted_sequence, - values, - out_int32=False, - right=False, - name=None): +def searchsorted( + sorted_sequence, values, out_int32=False, right=False, name=None +): """ Find the index of the corresponding `sorted_sequence` in the innermost dimension based on the given `values`. @@ -1009,29 +1059,32 @@ def searchsorted(sorted_sequence, return _C_ops.searchsorted(sorted_sequence, values, out_int32, right) if _in_legacy_dygraph(): - return _legacy_C_ops.searchsorted(sorted_sequence, values, "out_int32", - out_int32, "right", right) + return _legacy_C_ops.searchsorted( + sorted_sequence, values, "out_int32", out_int32, "right", right + ) - check_variable_and_dtype(sorted_sequence, 'SortedSequence', - ['float32', 'float64', 'int32', 'int64'], - 'paddle.searchsorted') - check_variable_and_dtype(values, 'Values', - ['float32', 'float64', 'int32', 'int64'], - 'paddle.searchsorted') + check_variable_and_dtype( + sorted_sequence, + 'SortedSequence', + ['float32', 'float64', 'int32', 'int64'], + 'paddle.searchsorted', + ) + check_variable_and_dtype( + values, + 'Values', + ['float32', 'float64', 'int32', 'int64'], + 'paddle.searchsorted', + ) helper = LayerHelper('searchsorted', **locals()) out_type = 'int32' if out_int32 else 'int64' out = helper.create_variable_for_type_inference(dtype=out_type) - helper.append_op(type='searchsorted', - inputs={ - 'SortedSequence': sorted_sequence, - "Values": values - }, - outputs={'Out': out}, - attrs={ - "out_int32": out_int32, - "right": right - }) + helper.append_op( + type='searchsorted', + inputs={'SortedSequence': sorted_sequence, "Values": values}, + outputs={'Out': out}, + attrs={"out_int32": out_int32, "right": right}, + ) return out @@ -1077,8 +1130,9 @@ def kthvalue(x, k, axis=None, keepdim=False, name=None): if _non_static_mode(): if axis is not None: if _in_legacy_dygraph(): - return _legacy_C_ops.kthvalue(x, 'k', k, "axis", axis, - "keepdim", keepdim) + return _legacy_C_ops.kthvalue( + x, 'k', k, "axis", axis, "keepdim", keepdim + ) return _C_ops.kthvalue(x, k, axis, keepdim) else: if _in_legacy_dygraph(): @@ -1093,12 +1147,11 @@ def kthvalue(x, k, axis=None, keepdim=False, name=None): values = helper.create_variable_for_type_inference(dtype=x.dtype) indices = helper.create_variable_for_type_inference(dtype="int64") - helper.append_op(type="kthvalue", - inputs=inputs, - outputs={ - "Out": [values], - "Indices": [indices] - }, - attrs=attrs) + helper.append_op( + type="kthvalue", + inputs=inputs, + outputs={"Out": [values], "Indices": [indices]}, + attrs=attrs, + ) indices.stop_gradient = True return values, indices diff --git a/python/paddle/tensor/stat.py b/python/paddle/tensor/stat.py index bbe98c6ab7e89..371e3fafd057e 100644 --- a/python/paddle/tensor/stat.py +++ b/python/paddle/tensor/stat.py @@ -85,9 +85,11 @@ def mean(x, axis=None, keepdim=False, name=None): else: if isinstance(axis, int): axis = [axis] - reduce_all = True if axis is None \ - or len(axis)==0 \ - or len(axis) == len(x.shape) else False + reduce_all = ( + True + if axis is None or len(axis) == 0 or len(axis) == len(x.shape) + else False + ) if axis is None or len(axis) == 0: axis = [0] @@ -96,18 +98,27 @@ def mean(x, axis=None, keepdim=False, name=None): axis = list(range(len(x.shape))) return _C_ops.mean(x, axis, keepdim) if _in_legacy_dygraph(): - return _legacy_C_ops.reduce_mean(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all) - - check_variable_and_dtype(x, 'x/input', - ['uint16', 'float16', 'float32', 'float64'], - 'mean/reduce_mean') - check_type(axis, 'axis/dim', (int, list, tuple, Variable), - 'mean/reduce_mean') + return _legacy_C_ops.reduce_mean( + x, 'dim', axis, 'keep_dim', keepdim, 'reduce_all', reduce_all + ) + + check_variable_and_dtype( + x, + 'x/input', + ['uint16', 'float16', 'float32', 'float64'], + 'mean/reduce_mean', + ) + check_type( + axis, 'axis/dim', (int, list, tuple, Variable), 'mean/reduce_mean' + ) if isinstance(axis, (list, tuple)): for item in axis: - check_type(item, 'elements of axis/dim', (int, Variable), - 'mean/reduce_mean') + check_type( + item, + 'elements of axis/dim', + (int, Variable), + 'mean/reduce_mean', + ) helper = LayerHelper('mean', **locals()) @@ -115,10 +126,9 @@ def mean(x, axis=None, keepdim=False, name=None): axis = utils._convert_to_tensor_list(axis) attrs = {'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all} out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='reduce_mean', - inputs={'X': x}, - outputs={'Out': out}, - attrs=attrs) + helper.append_op( + type='reduce_mean', inputs={'X': x}, outputs={'Out': out}, attrs=attrs + ) return out @@ -156,15 +166,16 @@ def var(x, axis=None, unbiased=True, keepdim=False, name=None): check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'var') u = mean(x, axis, True, name) - out = paddle.sum((x - u)**2, axis, keepdim=keepdim, name=name) + out = paddle.sum((x - u) ** 2, axis, keepdim=keepdim, name=name) dtype = x.dtype - n = paddle.cast(paddle.numel(x), paddle.int64) \ - / paddle.cast(paddle.numel(out), paddle.int64) + n = paddle.cast(paddle.numel(x), paddle.int64) / paddle.cast( + paddle.numel(out), paddle.int64 + ) n = n.astype(dtype) if unbiased: one_const = paddle.ones([1], x.dtype) - n = where(n > one_const, n - 1., one_const) + n = where(n > one_const, n - 1.0, one_const) out /= n return out @@ -252,7 +263,8 @@ def numel(x, name=None): raise TypeError("x must be a Tensor in numel") helper = LayerHelper('numel', **locals()) out = helper.create_variable_for_type_inference( - dtype=core.VarDesc.VarType.INT64) + dtype=core.VarDesc.VarType.INT64 + ) helper.append_op(type='size', inputs={'Input': x}, outputs={'Out': out}) return out @@ -320,8 +332,9 @@ def nanmedian(x, axis=None, keepdim=True, name=None): ) for i in range(len(axis)): - if not isinstance(axis[i], int) or not (axis[i] < dims - and axis[i] >= -dims): + if not isinstance(axis[i], int) or not ( + axis[i] < dims and axis[i] >= -dims + ): raise ValueError( "Axis should be None, int, or a list, element should in range [-rank(x), rank(x))." ) @@ -332,25 +345,25 @@ def nanmedian(x, axis=None, keepdim=True, name=None): raise ValueError("Axis has duplicated elements.") if _in_legacy_dygraph(): - median_index, out = _legacy_C_ops.nanmedian(x, 'axis', axis, 'keepdim', - keepdim) + median_index, out = _legacy_C_ops.nanmedian( + x, 'axis', axis, 'keepdim', keepdim + ) return out check_variable_and_dtype( - x, 'X', ['int32', 'int64', 'float16', 'float32', 'float64'], - 'nanmedian') + x, 'X', ['int32', 'int64', 'float16', 'float32', 'float64'], 'nanmedian' + ) helper = LayerHelper('nanmedian', **locals()) attrs = {'axis': axis, 'keepdim': keepdim} out = helper.create_variable_for_type_inference(x.dtype) medians = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='nanmedian', - inputs={'X': x}, - outputs={ - 'Out': out, - 'MedianIndex': medians - }, - attrs=attrs) + helper.append_op( + type='nanmedian', + inputs={'X': x}, + outputs={'Out': out, 'MedianIndex': medians}, + attrs=attrs, + ) return out @@ -423,21 +436,22 @@ def median(x, axis=None, keepdim=False, name=None): dtype = 'float64' if x.dtype == core.VarDesc.VarType.FP64 else 'float32' if sz & 1 == 0: out_tensor = paddle.slice( - tensor_topk, axes=[axis], starts=[kth - 1], - ends=[kth]) + paddle.slice( - tensor_topk, axes=[axis], starts=[kth], ends=[kth + 1]) + tensor_topk, axes=[axis], starts=[kth - 1], ends=[kth] + ) + paddle.slice(tensor_topk, axes=[axis], starts=[kth], ends=[kth + 1]) out_tensor = paddle.cast(out_tensor, dtype=dtype) / 2 else: - out_tensor = paddle.cast(paddle.slice(tensor_topk, - axes=[axis], - starts=[kth], - ends=[kth + 1]), - dtype=dtype) + out_tensor = paddle.cast( + paddle.slice( + tensor_topk, axes=[axis], starts=[kth], ends=[kth + 1] + ), + dtype=dtype, + ) out_tensor = out_tensor + paddle.sum( - paddle.cast(paddle.isnan(x), dtype=dtype) * x, axis=axis, keepdim=True) + paddle.cast(paddle.isnan(x), dtype=dtype) * x, axis=axis, keepdim=True + ) if not keepdim or is_flatten: if not is_flatten: - newshape = x.shape[:axis] + x.shape[axis + 1:] + newshape = x.shape[:axis] + x.shape[axis + 1 :] elif not keepdim: newshape = [1] else: @@ -501,7 +515,8 @@ def _compute_quantile(x, q, axis=None, keepdim=False, ignore_nan=False): axis_src, axis_dst = [], [] for axis_single in axis: if not isinstance(axis_single, int) or not ( - axis_single < dims and axis_single >= -dims): + axis_single < dims and axis_single >= -dims + ): raise ValueError( "Axis should be None, int, or a list, element should in range [-rank(x), rank(x))." ) @@ -523,9 +538,9 @@ def _compute_quantile(x, q, axis=None, keepdim=False, ignore_nan=False): out_shape[axis] = 1 mask = x.isnan() - valid_counts = mask.logical_not().sum(axis=axis, - keepdim=True, - dtype='float64') + valid_counts = mask.logical_not().sum( + axis=axis, keepdim=True, dtype='float64' + ) indices = [] @@ -552,15 +567,18 @@ def _compute_quantile(x, q, axis=None, keepdim=False, ignore_nan=False): for index in indices: indices_below = paddle.floor(index).astype(paddle.int32) indices_upper = paddle.ceil(index).astype(paddle.int32) - tensor_upper = paddle.take_along_axis(sorted_tensor, - indices_upper, - axis=axis) - tensor_below = paddle.take_along_axis(sorted_tensor, - indices_below, - axis=axis) - weights = (index - indices_below.astype('float64')) - out = paddle.lerp(tensor_below.astype('float64'), - tensor_upper.astype('float64'), weights) + tensor_upper = paddle.take_along_axis( + sorted_tensor, indices_upper, axis=axis + ) + tensor_below = paddle.take_along_axis( + sorted_tensor, indices_below, axis=axis + ) + weights = index - indices_below.astype('float64') + out = paddle.lerp( + tensor_below.astype('float64'), + tensor_upper.astype('float64'), + weights, + ) if not keepdim: out = paddle.squeeze(out, axis=axis) else: diff --git a/python/paddle/tensor/to_string.py b/python/paddle/tensor/to_string.py index 76f5c603f0f4b..4739628f7176b 100644 --- a/python/paddle/tensor/to_string.py +++ b/python/paddle/tensor/to_string.py @@ -30,11 +30,13 @@ class PrintOptions(object): DEFAULT_PRINT_OPTIONS = PrintOptions() -def set_printoptions(precision=None, - threshold=None, - edgeitems=None, - sci_mode=None, - linewidth=None): +def set_printoptions( + precision=None, + threshold=None, + edgeitems=None, + sci_mode=None, + linewidth=None, +): """Set the printing options for Tensor. Args: @@ -105,29 +107,35 @@ def _to_summary(var): return var elif len(var.shape) == 1: if var.shape[0] > 2 * edgeitems: - return np.concatenate([var[:edgeitems], var[(-1 * edgeitems):]]) + return np.concatenate([var[:edgeitems], var[(-1 * edgeitems) :]]) else: return var else: # recursively handle all dimensions if var.shape[0] > 2 * edgeitems: begin = [x for x in var[:edgeitems]] - end = [x for x in var[(-1 * edgeitems):]] + end = [x for x in var[(-1 * edgeitems) :]] return np.stack([_to_summary(x) for x in (begin + end)]) else: return np.stack([_to_summary(x) for x in var]) def _format_item(np_var, max_width=0, signed=False): - if np_var.dtype == np.float32 or np_var.dtype == np.float64 or np_var.dtype == np.float16: + if ( + np_var.dtype == np.float32 + or np_var.dtype == np.float64 + or np_var.dtype == np.float16 + ): if DEFAULT_PRINT_OPTIONS.sci_mode: item_str = '{{:.{}e}}'.format( - DEFAULT_PRINT_OPTIONS.precision).format(np_var) + DEFAULT_PRINT_OPTIONS.precision + ).format(np_var) elif np.ceil(np_var) == np_var: item_str = '{:.0f}.'.format(np_var) else: item_str = '{{:.{}f}}'.format( - DEFAULT_PRINT_OPTIONS.precision).format(np_var) + DEFAULT_PRINT_OPTIONS.precision + ).format(np_var) else: item_str = '{}'.format(np_var) @@ -180,42 +188,56 @@ def _format_tensor(var, summary, indent=0, max_width=0, signed=False): items_per_line = max(1, items_per_line) if summary and var.shape[0] > 2 * edgeitems: - items = [ - _format_item(item, max_width, signed) - for item in list(var)[:edgeitems] - ] + ['...'] + [ - _format_item(item, max_width, signed) - for item in list(var)[(-1 * edgeitems):] - ] + items = ( + [ + _format_item(item, max_width, signed) + for item in list(var)[:edgeitems] + ] + + ['...'] + + [ + _format_item(item, max_width, signed) + for item in list(var)[(-1 * edgeitems) :] + ] + ) else: items = [ _format_item(item, max_width, signed) for item in list(var) ] lines = [ - items[i:i + items_per_line] + items[i : i + items_per_line] for i in range(0, len(items), items_per_line) ] s = (',\n' + ' ' * (indent + 1)).join( - [', '.join(line) for line in lines]) + [', '.join(line) for line in lines] + ) return '[' + s + ']' else: # recursively handle all dimensions if summary and var.shape[0] > 2 * edgeitems: - vars = [ - _format_tensor(x, summary, indent + 1, max_width, signed) - for x in var[:edgeitems] - ] + ['...'] + [ - _format_tensor(x, summary, indent + 1, max_width, signed) - for x in var[(-1 * edgeitems):] - ] + vars = ( + [ + _format_tensor(x, summary, indent + 1, max_width, signed) + for x in var[:edgeitems] + ] + + ['...'] + + [ + _format_tensor(x, summary, indent + 1, max_width, signed) + for x in var[(-1 * edgeitems) :] + ] + ) else: vars = [ _format_tensor(x, summary, indent + 1, max_width, signed) for x in var ] - return '[' + (',' + '\n' * (len(var.shape) - 1) + ' ' * - (indent + 1)).join(vars) + ']' + return ( + '[' + + (',' + '\n' * (len(var.shape) - 1) + ' ' * (indent + 1)).join( + vars + ) + + ']' + ) def to_string(var, prefix='Tensor'): @@ -248,19 +270,19 @@ def to_string(var, prefix='Tensor'): max_width, signed = _get_max_width(_to_summary(np_var)) - data = _format_tensor(np_var, - summary, - indent=indent, - max_width=max_width, - signed=signed) + data = _format_tensor( + np_var, summary, indent=indent, max_width=max_width, signed=signed + ) - return _template.format(prefix=prefix, - shape=var.shape, - dtype=dtype, - place=var._place_str, - stop_gradient=var.stop_gradient, - indent=' ' * indent, - data=data) + return _template.format( + prefix=prefix, + shape=var.shape, + dtype=dtype, + place=var._place_str, + stop_gradient=var.stop_gradient, + indent=' ' * indent, + data=data, + ) def _format_dense_tensor(tensor, indent): @@ -282,11 +304,9 @@ def _format_dense_tensor(tensor, indent): max_width, signed = _get_max_width(_to_summary(np_tensor)) - data = _format_tensor(np_tensor, - sumary, - indent=indent, - max_width=max_width, - signed=signed) + data = _format_tensor( + np_tensor, sumary, indent=indent, max_width=max_width, signed=signed + ) return data @@ -297,38 +317,47 @@ def sparse_tensor_to_string(tensor, prefix='Tensor'): indices_tensor = tensor.indices() values_tensor = tensor.values() indices_data = 'indices=' + _format_dense_tensor( - indices_tensor, indent + len('indices=')) - values_data = 'values=' + _format_dense_tensor(values_tensor, - indent + len('values=')) - return _template.format(prefix=prefix, - shape=tensor.shape, - dtype=tensor.dtype, - place=tensor._place_str, - stop_gradient=tensor.stop_gradient, - indent=' ' * indent, - indices=indices_data, - values=values_data) + indices_tensor, indent + len('indices=') + ) + values_data = 'values=' + _format_dense_tensor( + values_tensor, indent + len('values=') + ) + return _template.format( + prefix=prefix, + shape=tensor.shape, + dtype=tensor.dtype, + place=tensor._place_str, + stop_gradient=tensor.stop_gradient, + indent=' ' * indent, + indices=indices_data, + values=values_data, + ) else: _template = "{prefix}(shape={shape}, dtype={dtype}, place={place}, stop_gradient={stop_gradient}, \n{indent}{crows}, \n{indent}{cols}, \n{indent}{values})" crows_tensor = tensor.crows() cols_tensor = tensor.cols() elements_tensor = tensor.values() - crows_data = 'crows=' + _format_dense_tensor(crows_tensor, - indent + len('crows=')) - cols_data = 'cols=' + _format_dense_tensor(cols_tensor, - indent + len('cols=')) - values_data = 'values=' + _format_dense_tensor(elements_tensor, - indent + len('values=')) - - return _template.format(prefix=prefix, - shape=tensor.shape, - dtype=tensor.dtype, - place=tensor._place_str, - stop_gradient=tensor.stop_gradient, - indent=' ' * indent, - crows=crows_data, - cols=cols_data, - values=values_data) + crows_data = 'crows=' + _format_dense_tensor( + crows_tensor, indent + len('crows=') + ) + cols_data = 'cols=' + _format_dense_tensor( + cols_tensor, indent + len('cols=') + ) + values_data = 'values=' + _format_dense_tensor( + elements_tensor, indent + len('values=') + ) + + return _template.format( + prefix=prefix, + shape=tensor.shape, + dtype=tensor.dtype, + place=tensor._place_str, + stop_gradient=tensor.stop_gradient, + indent=' ' * indent, + crows=crows_data, + cols=cols_data, + values=values_data, + ) def tensor_to_string(tensor, prefix='Tensor'): @@ -347,10 +376,12 @@ def tensor_to_string(tensor, prefix='Tensor'): return "Tensor(Not initialized)" else: data = _format_dense_tensor(tensor, indent) - return _template.format(prefix=prefix, - shape=tensor.shape, - dtype=dtype, - place=tensor._place_str, - stop_gradient=tensor.stop_gradient, - indent=' ' * indent, - data=data) + return _template.format( + prefix=prefix, + shape=tensor.shape, + dtype=dtype, + place=tensor._place_str, + stop_gradient=tensor.stop_gradient, + indent=' ' * indent, + data=data, + ) diff --git a/python/paddle/tests/dist_hapi_mnist_dynamic.py b/python/paddle/tests/dist_hapi_mnist_dynamic.py index 71c63eb62b4fe..50e77976da4ae 100644 --- a/python/paddle/tests/dist_hapi_mnist_dynamic.py +++ b/python/paddle/tests/dist_hapi_mnist_dynamic.py @@ -28,7 +28,6 @@ class MnistDataset(MNIST): - def __init__(self, mode, return_label=True): super(MnistDataset, self).__init__(mode=mode) self.return_label = return_label @@ -37,7 +36,7 @@ def __getitem__(self, idx): img = np.reshape(self.images[idx], [1, 28, 28]) if self.return_label: return img, np.array(self.labels[idx]).astype('int64') - return img, + return (img,) def __len__(self): return len(self.images) @@ -52,10 +51,10 @@ def compute_accuracy(pred, gt): return np.sum(correct) / correct.shape[0] -@unittest.skipIf(not fluid.is_compiled_with_cuda(), - 'CPU testing is not supported') +@unittest.skipIf( + not fluid.is_compiled_with_cuda(), 'CPU testing is not supported' +) class TestDistTraning(unittest.TestCase): - def test_dynamic_multiple_gpus(self): device = set_device('gpu') @@ -66,9 +65,9 @@ def test_dynamic_multiple_gpus(self): labels = [Input([None, 1], 'int64', 'label')] model = Model(LeNet(), inputs, labels) - optim = fluid.optimizer.Momentum(learning_rate=0.001, - momentum=.9, - parameter_list=model.parameters()) + optim = fluid.optimizer.Momentum( + learning_rate=0.001, momentum=0.9, parameter_list=model.parameters() + ) model.prepare(optim, CrossEntropyLoss(), Accuracy()) train_dataset = MnistDataset(mode='train') @@ -76,17 +75,19 @@ def test_dynamic_multiple_gpus(self): test_dataset = MnistDataset(mode='test', return_label=False) cbk = paddle.callbacks.ProgBarLogger(50) - model.fit(train_dataset, - val_dataset, - epochs=2, - batch_size=batch_size, - callbacks=cbk) + model.fit( + train_dataset, + val_dataset, + epochs=2, + batch_size=batch_size, + callbacks=cbk, + ) eval_result = model.evaluate(val_dataset, batch_size=batch_size) - output = model.predict(test_dataset, - batch_size=batch_size, - stack_outputs=True) + output = model.predict( + test_dataset, batch_size=batch_size, stack_outputs=True + ) np.testing.assert_equal(output[0].shape[0], len(test_dataset)) diff --git a/python/paddle/tests/dist_hapi_mnist_static.py b/python/paddle/tests/dist_hapi_mnist_static.py index cc3b65725013c..ce7e5c18a6a61 100644 --- a/python/paddle/tests/dist_hapi_mnist_static.py +++ b/python/paddle/tests/dist_hapi_mnist_static.py @@ -28,7 +28,6 @@ class MnistDataset(MNIST): - def __init__(self, mode, return_label=True): super(MnistDataset, self).__init__(mode=mode) self.return_label = return_label @@ -37,7 +36,7 @@ def __getitem__(self, idx): img = np.reshape(self.images[idx], [1, 28, 28]) if self.return_label: return img, np.array(self.labels[idx]).astype('int64') - return img, + return (img,) def __len__(self): return len(self.images) @@ -52,10 +51,10 @@ def compute_accuracy(pred, gt): return np.sum(correct) / correct.shape[0] -@unittest.skipIf(not fluid.is_compiled_with_cuda(), - 'CPU testing is not supported') +@unittest.skipIf( + not fluid.is_compiled_with_cuda(), 'CPU testing is not supported' +) class TestDistTraning(unittest.TestCase): - def test_static_multiple_gpus(self): paddle.enable_static() device = set_device('gpu') @@ -67,9 +66,9 @@ def test_static_multiple_gpus(self): labels = [Input([None, 1], 'int64', 'label')] model = Model(LeNet(), inputs, labels) - optim = fluid.optimizer.Momentum(learning_rate=0.001, - momentum=.9, - parameter_list=model.parameters()) + optim = fluid.optimizer.Momentum( + learning_rate=0.001, momentum=0.9, parameter_list=model.parameters() + ) model.prepare(optim, CrossEntropyLoss(), Accuracy()) train_dataset = MnistDataset(mode='train') @@ -77,17 +76,19 @@ def test_static_multiple_gpus(self): test_dataset = MnistDataset(mode='test', return_label=False) cbk = paddle.callbacks.ProgBarLogger(50) - model.fit(train_dataset, - val_dataset, - epochs=2, - batch_size=batch_size, - callbacks=cbk) + model.fit( + train_dataset, + val_dataset, + epochs=2, + batch_size=batch_size, + callbacks=cbk, + ) eval_result = model.evaluate(val_dataset, batch_size=batch_size) - output = model.predict(test_dataset, - batch_size=batch_size, - stack_outputs=True) + output = model.predict( + test_dataset, batch_size=batch_size, stack_outputs=True + ) np.testing.assert_equal(output[0].shape[0], len(test_dataset)) diff --git a/python/paddle/tests/dist_hapi_pure_fp16_static.py b/python/paddle/tests/dist_hapi_pure_fp16_static.py index 0d7fa491fc1a6..6be15ec44be1f 100644 --- a/python/paddle/tests/dist_hapi_pure_fp16_static.py +++ b/python/paddle/tests/dist_hapi_pure_fp16_static.py @@ -25,10 +25,10 @@ from paddle.vision.models import LeNet -@unittest.skipIf(not fluid.is_compiled_with_cuda(), - 'CPU testing is not supported') +@unittest.skipIf( + not fluid.is_compiled_with_cuda(), 'CPU testing is not supported' +) class TestDistTraningWithPureFP16(unittest.TestCase): - def test_amp_training_purefp16(self): if not fluid.is_compiled_with_cuda(): self.skipTest('module not tested when ONLY_CPU compling') @@ -42,13 +42,17 @@ def test_amp_training_purefp16(self): inputs = InputSpec([None, 1, 28, 28], "float32", 'x') labels = InputSpec([None, 1], "int64", "y") model = Model(net, inputs, labels) - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=model.parameters(), - multi_precision=True) + optim = paddle.optimizer.Adam( + learning_rate=0.001, + parameters=model.parameters(), + multi_precision=True, + ) amp_configs = {"level": amp_level, "use_fp16_guard": False} - model.prepare(optimizer=optim, - loss=CrossEntropyLoss(reduction="sum"), - amp_configs=amp_configs) + model.prepare( + optimizer=optim, + loss=CrossEntropyLoss(reduction="sum"), + amp_configs=amp_configs, + ) model.train_batch([data], [label]) diff --git a/python/paddle/tests/hapi_mnist_bf16_static.py b/python/paddle/tests/hapi_mnist_bf16_static.py index 55db397be493b..489215301ed59 100644 --- a/python/paddle/tests/hapi_mnist_bf16_static.py +++ b/python/paddle/tests/hapi_mnist_bf16_static.py @@ -37,17 +37,18 @@ def parse_args(): parser = argparse.ArgumentParser("Lenet BF16 train static script") - parser.add_argument('-bf16', - '--bf16', - type=ast.literal_eval, - default=False, - help="whether use bf16") + parser.add_argument( + '-bf16', + '--bf16', + type=ast.literal_eval, + default=False, + help="whether use bf16", + ) args = parser.parse_args() return args class MnistDataset(MNIST): - def __init__(self, mode, return_label=True): super(MnistDataset, self).__init__(mode=mode) self.return_label = return_label @@ -56,7 +57,7 @@ def __getitem__(self, idx): img = np.reshape(self.images[idx], [1, 28, 28]) if self.return_label: return img, np.array(self.labels[idx]).astype('int64') - return img, + return (img,) def __len__(self): return len(self.images) @@ -73,8 +74,12 @@ def compute_accuracy(pred, gt): def main(args): print('download training data and load training data') - train_dataset = MnistDataset(mode='train', ) - val_dataset = MnistDataset(mode='test', ) + train_dataset = MnistDataset( + mode='train', + ) + val_dataset = MnistDataset( + mode='test', + ) test_dataset = MnistDataset(mode='test', return_label=False) im_shape = (-1, 1, 28, 28) @@ -88,10 +93,20 @@ def main(args): if args.bf16: optim = amp.bf16.decorate_bf16( optim, - amp_lists=amp.bf16.AutoMixedPrecisionListsBF16(custom_bf16_list={ - 'matmul_v2', 'pool2d', 'relu', 'scale', 'elementwise_add', - 'reshape2', 'slice', 'reduce_mean', 'conv2d' - }, )) + amp_lists=amp.bf16.AutoMixedPrecisionListsBF16( + custom_bf16_list={ + 'matmul_v2', + 'pool2d', + 'relu', + 'scale', + 'elementwise_add', + 'reshape2', + 'slice', + 'reduce_mean', + 'conv2d', + }, + ), + ) # Configuration model model.prepare(optim, paddle.nn.CrossEntropyLoss(), Accuracy()) @@ -103,9 +118,9 @@ def main(args): model.fit(train_dataset, epochs=2, batch_size=batch_size, verbose=1) eval_result = model.evaluate(val_dataset, batch_size=batch_size, verbose=1) - output = model.predict(test_dataset, - batch_size=batch_size, - stack_outputs=True) + output = model.predict( + test_dataset, batch_size=batch_size, stack_outputs=True + ) np.testing.assert_equal(output[0].shape[0], len(test_dataset)) diff --git a/python/paddle/tests/hubconf.py b/python/paddle/tests/hubconf.py index d730dcff51e24..bef0851f78da6 100644 --- a/python/paddle/tests/hubconf.py +++ b/python/paddle/tests/hubconf.py @@ -18,6 +18,5 @@ def MM(out_channels=8, pretrained=False): - '''This is a test demo for paddle hub - ''' + '''This is a test demo for paddle hub''' return _MM(out_channels) diff --git a/python/paddle/tests/test_async_read_write.py b/python/paddle/tests/test_async_read_write.py index a235b7864809a..682f72422d606 100644 --- a/python/paddle/tests/test_async_read_write.py +++ b/python/paddle/tests/test_async_read_write.py @@ -22,60 +22,78 @@ class TestAsyncRead(unittest.TestCase): - def func_setUp(self): - self.empty = paddle.to_tensor(np.array([], dtype="int64"), - place=paddle.CPUPlace()) + self.empty = paddle.to_tensor( + np.array([], dtype="int64"), place=paddle.CPUPlace() + ) data = np.random.randn(100, 50, 50).astype("float32") self.src = paddle.to_tensor(data, place=paddle.CUDAPinnedPlace()) self.dst = paddle.empty(shape=[100, 50, 50], dtype="float32") - self.index = paddle.to_tensor(np.array([1, 3, 5, 7, 9], - dtype="int64")).cpu() - self.buffer = paddle.empty(shape=[50, 50, 50], - dtype="float32").pin_memory() + self.index = paddle.to_tensor( + np.array([1, 3, 5, 7, 9], dtype="int64") + ).cpu() + self.buffer = paddle.empty( + shape=[50, 50, 50], dtype="float32" + ).pin_memory() self.stream = cuda.Stream() def func_test_async_read_empty_offset_and_count(self): with cuda.stream_guard(self.stream): if _in_legacy_dygraph(): - core.async_read(self.src, self.dst, self.index, self.buffer, - self.empty, self.empty) + core.async_read( + self.src, + self.dst, + self.index, + self.buffer, + self.empty, + self.empty, + ) else: - core.eager.async_read(self.src, self.dst, self.index, - self.buffer, self.empty, self.empty) + core.eager.async_read( + self.src, + self.dst, + self.index, + self.buffer, + self.empty, + self.empty, + ) array1 = paddle.gather(self.src, self.index) - array2 = self.dst[:len(self.index)] + array2 = self.dst[: len(self.index)] np.testing.assert_allclose(array1.numpy(), array2.numpy(), rtol=1e-05) def func_test_async_read_success(self): - offset = paddle.to_tensor(np.array([10, 20], dtype="int64"), - place=paddle.CPUPlace()) - count = paddle.to_tensor(np.array([5, 10], dtype="int64"), - place=paddle.CPUPlace()) + offset = paddle.to_tensor( + np.array([10, 20], dtype="int64"), place=paddle.CPUPlace() + ) + count = paddle.to_tensor( + np.array([5, 10], dtype="int64"), place=paddle.CPUPlace() + ) with cuda.stream_guard(self.stream): if _in_legacy_dygraph(): - core.async_read(self.src, self.dst, self.index, self.buffer, - offset, count) + core.async_read( + self.src, self.dst, self.index, self.buffer, offset, count + ) else: - core.eager.async_read(self.src, self.dst, self.index, - self.buffer, offset, count) + core.eager.async_read( + self.src, self.dst, self.index, self.buffer, offset, count + ) # index data index_array1 = paddle.gather(self.src, self.index) count_numel = paddle.sum(count).numpy()[0] - index_array2 = self.dst[count_numel:count_numel + len(self.index)] - np.testing.assert_allclose(index_array1.numpy(), - index_array2.numpy(), - rtol=1e-05) + index_array2 = self.dst[count_numel : count_numel + len(self.index)] + np.testing.assert_allclose( + index_array1.numpy(), index_array2.numpy(), rtol=1e-05 + ) # offset, count offset_a = paddle.gather(self.src, paddle.to_tensor(np.arange(10, 15))) offset_b = paddle.gather(self.src, paddle.to_tensor(np.arange(20, 30))) offset_array1 = paddle.concat([offset_a, offset_b], axis=0) offset_array2 = self.dst[:count_numel] - np.testing.assert_allclose(offset_array1.numpy(), - offset_array2.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + offset_array1.numpy(), offset_array2.numpy(), rtol=1e-05 + ) def func_test_async_read_only_1dim(self): src = paddle.rand([40], dtype="float32").pin_memory() @@ -83,13 +101,15 @@ def func_test_async_read_only_1dim(self): buffer_ = paddle.empty([20]).pin_memory() with cuda.stream_guard(self.stream): if _in_legacy_dygraph(): - core.async_read(src, dst, self.index, buffer_, self.empty, - self.empty) + core.async_read( + src, dst, self.index, buffer_, self.empty, self.empty + ) else: - core.eager.async_read(src, dst, self.index, buffer_, self.empty, - self.empty) + core.eager.async_read( + src, dst, self.index, buffer_, self.empty, self.empty + ) array1 = paddle.gather(src, self.index) - array2 = dst[:len(self.index)] + array2 = dst[: len(self.index)] np.testing.assert_allclose(array1.numpy(), array2.numpy(), rtol=1e-05) def test_main(self): @@ -109,18 +129,20 @@ def test_main(self): class TestAsyncWrite(unittest.TestCase): - def func_setUp(self): self.src = paddle.rand(shape=[100, 50, 50, 5], dtype="float32") - self.dst = paddle.empty(shape=[200, 50, 50, 5], - dtype="float32").pin_memory() + self.dst = paddle.empty( + shape=[200, 50, 50, 5], dtype="float32" + ).pin_memory() self.stream = cuda.Stream() def func_test_async_write_success(self): - offset = paddle.to_tensor(np.array([0, 60], dtype="int64"), - place=paddle.CPUPlace()) - count = paddle.to_tensor(np.array([40, 60], dtype="int64"), - place=paddle.CPUPlace()) + offset = paddle.to_tensor( + np.array([0, 60], dtype="int64"), place=paddle.CPUPlace() + ) + count = paddle.to_tensor( + np.array([40, 60], dtype="int64"), place=paddle.CPUPlace() + ) with cuda.stream_guard(self.stream): if _in_legacy_dygraph(): core.async_write(self.src, self.dst, offset, count) @@ -130,9 +152,9 @@ def func_test_async_write_success(self): offset_a = paddle.gather(self.dst, paddle.to_tensor(np.arange(0, 40))) offset_b = paddle.gather(self.dst, paddle.to_tensor(np.arange(60, 120))) offset_array = paddle.concat([offset_a, offset_b], axis=0) - np.testing.assert_allclose(self.src.numpy(), - offset_array.numpy(), - rtol=1e-05) + np.testing.assert_allclose( + self.src.numpy(), offset_array.numpy(), rtol=1e-05 + ) def test_async_write_success(self): with _test_eager_guard(): diff --git a/python/paddle/tests/test_audio_backend.py b/python/paddle/tests/test_audio_backend.py index 79e793e2dc865..fb583d96e1f9b 100644 --- a/python/paddle/tests/test_audio_backend.py +++ b/python/paddle/tests/test_audio_backend.py @@ -20,12 +20,10 @@ class TestAudioBackends(unittest.TestCase): - def setUp(self): self.initParmas() def initParmas(self): - def get_wav_data(dtype: str, num_channels: int, num_frames: int): dtype_ = getattr(paddle, dtype) base = paddle.linspace(-1.0, 1.0, num_frames, dtype=dtype_) * 0.1 @@ -37,19 +35,21 @@ def get_wav_data(dtype: str, num_channels: int, num_frames: int): self.sr = 16000 self.dtype = "float32" self.window_size = 1024 - waveform_tensor = get_wav_data(self.dtype, - self.num_channels, - num_frames=self.duration * self.sr) + waveform_tensor = get_wav_data( + self.dtype, self.num_channels, num_frames=self.duration * self.sr + ) # shape (1, 8000) self.waveform = waveform_tensor.numpy() def test_backend(self): base_dir = os.getcwd() wave_wav_path = os.path.join(base_dir, "wave_test.wav") - paddle.audio.save(wave_wav_path, - paddle.to_tensor(self.waveform), - self.sr, - channels_first=True) + paddle.audio.save( + wave_wav_path, + paddle.to_tensor(self.waveform), + self.sr, + channels_first=True, + ) # test backends(wave)(wave_backend) info wav_info = paddle.audio.info(wave_wav_path) @@ -74,9 +74,9 @@ def test_backend(self): np.testing.assert_array_almost_equal(wav_data, waveform) with open(wave_wav_path, 'rb') as file_: - wav_data, sr = paddle.audio.load(file_, - normalize=False, - num_frames=10000) + wav_data, sr = paddle.audio.load( + file_, normalize=False, num_frames=10000 + ) with soundfile.SoundFile(wave_wav_path, "r") as file_: dtype = "int16" frames = file_._prepare_read(0, None, -1) @@ -101,6 +101,7 @@ def test_backend(self): try: import paddleaudio + backends = paddle.audio.backends.list_available_backends() for backend in backends: self.assertTrue(backend in ["wave_backend", "soundfile"]) @@ -119,18 +120,22 @@ def test_backend(self): pass try: - paddle.audio.save(wave_wav_path, - paddle.to_tensor(self.waveform), - self.sr, - bits_per_sample=24, - channels_first=True) + paddle.audio.save( + wave_wav_path, + paddle.to_tensor(self.waveform), + self.sr, + bits_per_sample=24, + channels_first=True, + ) except ValueError: pass try: - paddle.audio.save(wave_wav_path, - paddle.to_tensor(self.waveform).unsqueeze(0), - self.sr) + paddle.audio.save( + wave_wav_path, + paddle.to_tensor(self.waveform).unsqueeze(0), + self.sr, + ) except AssertionError: pass diff --git a/python/paddle/tests/test_audio_datasets.py b/python/paddle/tests/test_audio_datasets.py index 59ba1d543bda6..4db1730ae4189 100644 --- a/python/paddle/tests/test_audio_datasets.py +++ b/python/paddle/tests/test_audio_datasets.py @@ -24,7 +24,6 @@ def parameterize(*params): class TestAudioDatasets(unittest.TestCase): - @parameterize(["dev", "train"], [40, 64]) def test_tess_dataset(self, mode: str, params: int): """ @@ -34,36 +33,34 @@ def test_tess_dataset(self, mode: str, params: int): https://doi.org/10.5683/SP2/E8H2MF """ archive = { - 'url': - 'https://bj.bcebos.com/paddleaudio/datasets/TESS_Toronto_emotional_speech_set_lite.zip', + 'url': 'https://bj.bcebos.com/paddleaudio/datasets/TESS_Toronto_emotional_speech_set_lite.zip', 'md5': '9ffb5e3adf28d4d6b787fa94bd59b975', } # small part of TESS dataset for test. - tess_dataset = paddle.audio.datasets.TESS(mode=mode, - feat_type='mfcc', - n_mfcc=params, - archive=archive) + tess_dataset = paddle.audio.datasets.TESS( + mode=mode, feat_type='mfcc', n_mfcc=params, archive=archive + ) idx = np.random.randint(0, 30) elem = tess_dataset[idx] self.assertTrue(elem[0].shape[0] == params) self.assertTrue(0 <= elem[1] <= 6) - tess_dataset = paddle.audio.datasets.TESS(mode=mode, - feat_type='spectrogram', - n_fft=params) + tess_dataset = paddle.audio.datasets.TESS( + mode=mode, feat_type='spectrogram', n_fft=params + ) elem = tess_dataset[idx] self.assertTrue(elem[0].shape[0] == (params // 2 + 1)) self.assertTrue(0 <= elem[1] <= 6) - tess_dataset = paddle.audio.datasets.TESS(mode="dev", - feat_type='logmelspectrogram', - n_mels=params) + tess_dataset = paddle.audio.datasets.TESS( + mode="dev", feat_type='logmelspectrogram', n_mels=params + ) elem = tess_dataset[idx] self.assertTrue(elem[0].shape[0] == params) self.assertTrue(0 <= elem[1] <= 6) - tess_dataset = paddle.audio.datasets.TESS(mode="dev", - feat_type='melspectrogram', - n_mels=params) + tess_dataset = paddle.audio.datasets.TESS( + mode="dev", feat_type='melspectrogram', n_mels=params + ) elem = tess_dataset[idx] self.assertTrue(elem[0].shape[0] == params) self.assertTrue(0 <= elem[1] <= 6) @@ -77,43 +74,42 @@ def test_esc50_dataset(self, mode: str, params: int): http://dx.doi.org/10.1145/2733373.2806390 """ archive = { - 'url': - 'https://bj.bcebos.com/paddleaudio/datasets/ESC-50-master-lite.zip', + 'url': 'https://bj.bcebos.com/paddleaudio/datasets/ESC-50-master-lite.zip', 'md5': '1e9ba53265143df5b2804a743f2d1956', } # small part of ESC50 dataset for test. - esc50_dataset = paddle.audio.datasets.ESC50(mode=mode, - feat_type='raw', - archive=archive) + esc50_dataset = paddle.audio.datasets.ESC50( + mode=mode, feat_type='raw', archive=archive + ) idx = np.random.randint(0, 6) elem = esc50_dataset[idx] self.assertTrue(elem[0].shape[0] == 220500) self.assertTrue(0 <= elem[1] <= 2) - esc50_dataset = paddle.audio.datasets.ESC50(mode=mode, - feat_type='mfcc', - n_mfcc=params, - archive=archive) + esc50_dataset = paddle.audio.datasets.ESC50( + mode=mode, feat_type='mfcc', n_mfcc=params, archive=archive + ) idx = np.random.randint(0, 6) elem = esc50_dataset[idx] self.assertTrue(elem[0].shape[0] == params) self.assertTrue(0 <= elem[1] <= 2) - esc50_dataset = paddle.audio.datasets.ESC50(mode=mode, - feat_type='spectrogram', - n_fft=params) + esc50_dataset = paddle.audio.datasets.ESC50( + mode=mode, feat_type='spectrogram', n_fft=params + ) elem = esc50_dataset[idx] self.assertTrue(elem[0].shape[0] == (params // 2 + 1)) self.assertTrue(0 <= elem[1] <= 2) esc50_dataset = paddle.audio.datasets.ESC50( - mode=mode, feat_type='logmelspectrogram', n_mels=params) + mode=mode, feat_type='logmelspectrogram', n_mels=params + ) elem = esc50_dataset[idx] self.assertTrue(elem[0].shape[0] == params) self.assertTrue(0 <= elem[1] <= 2) - esc50_dataset = paddle.audio.datasets.ESC50(mode=mode, - feat_type='melspectrogram', - n_mels=params) + esc50_dataset = paddle.audio.datasets.ESC50( + mode=mode, feat_type='melspectrogram', n_mels=params + ) elem = esc50_dataset[idx] self.assertTrue(elem[0].shape[0] == params) self.assertTrue(0 <= elem[1] <= 2) diff --git a/python/paddle/tests/test_audio_functions.py b/python/paddle/tests/test_audio_functions.py index 5766299f24307..5542b4483b62c 100644 --- a/python/paddle/tests/test_audio_functions.py +++ b/python/paddle/tests/test_audio_functions.py @@ -28,12 +28,10 @@ def parameterize(*params): class TestAudioFuncitons(unittest.TestCase): - def setUp(self): self.initParmas() def initParmas(self): - def get_wav_data(dtype: str, num_channels: int, num_frames: int): dtype_ = getattr(paddle, dtype) base = paddle.linspace(-1.0, 1.0, num_frames, dtype=dtype_) * 0.1 @@ -53,49 +51,56 @@ def get_wav_data(dtype: str, num_channels: int, num_frames: int): self.sr = 16000 self.dtype = "float32" self.window_size = 1024 - waveform_tensor = get_wav_data(self.dtype, - self.num_channels, - num_frames=self.duration * self.sr) + waveform_tensor = get_wav_data( + self.dtype, self.num_channels, num_frames=self.duration * self.sr + ) self.waveform = waveform_tensor.numpy() @parameterize([1.0, 3.0, 9.0, 25.0], [True, False]) def test_audio_function(self, val: float, htk_flag: bool): mel_paddle = paddle.audio.functional.hz_to_mel(val, htk_flag) mel_paddle_tensor = paddle.audio.functional.hz_to_mel( - paddle.to_tensor(val), htk_flag) + paddle.to_tensor(val), htk_flag + ) mel_librosa = librosa.hz_to_mel(val, htk_flag) np.testing.assert_almost_equal(mel_paddle, mel_librosa, decimal=5) - np.testing.assert_almost_equal(mel_paddle_tensor.numpy(), - mel_librosa, - decimal=4) + np.testing.assert_almost_equal( + mel_paddle_tensor.numpy(), mel_librosa, decimal=4 + ) hz_paddle = paddle.audio.functional.mel_to_hz(val, htk_flag) hz_paddle_tensor = paddle.audio.functional.mel_to_hz( - paddle.to_tensor(val), htk_flag) + paddle.to_tensor(val), htk_flag + ) hz_librosa = librosa.mel_to_hz(val, htk_flag) np.testing.assert_almost_equal(hz_paddle, hz_librosa, decimal=4) - np.testing.assert_almost_equal(hz_paddle_tensor.numpy(), - hz_librosa, - decimal=4) + np.testing.assert_almost_equal( + hz_paddle_tensor.numpy(), hz_librosa, decimal=4 + ) decibel_paddle = paddle.audio.functional.power_to_db( - paddle.to_tensor(val)) + paddle.to_tensor(val) + ) decibel_librosa = librosa.power_to_db(val) - np.testing.assert_almost_equal(decibel_paddle.numpy(), - decibel_paddle, - decimal=5) - - @parameterize([64, 128, 256], [0.0, 0.5, 1.0], [10000, 11025], - [False, True]) - def test_audio_function_mel(self, n_mels: int, f_min: float, f_max: float, - htk_flag: bool): - librosa_mel_freq = librosa.mel_frequencies(n_mels, f_min, f_max, - htk_flag) + np.testing.assert_almost_equal( + decibel_paddle.numpy(), decibel_paddle, decimal=5 + ) + + @parameterize( + [64, 128, 256], [0.0, 0.5, 1.0], [10000, 11025], [False, True] + ) + def test_audio_function_mel( + self, n_mels: int, f_min: float, f_max: float, htk_flag: bool + ): + librosa_mel_freq = librosa.mel_frequencies( + n_mels, f_min, f_max, htk_flag + ) paddle_mel_freq = paddle.audio.functional.mel_frequencies( - n_mels, f_min, f_max, htk_flag, 'float64') - np.testing.assert_almost_equal(paddle_mel_freq, - librosa_mel_freq, - decimal=3) + n_mels, f_min, f_max, htk_flag, 'float64' + ) + np.testing.assert_almost_equal( + paddle_mel_freq, librosa_mel_freq, decimal=3 + ) @parameterize([8000, 16000], [64, 128, 256]) def test_audio_function_fft(self, sr: int, n_fft: int): @@ -106,60 +111,77 @@ def test_audio_function_fft(self, sr: int, n_fft: int): @parameterize([1.0, 3.0, 9.0]) def test_audio_function_exception(self, spect: float): try: - paddle.audio.functional.power_to_db(paddle.to_tensor([spect]), - amin=0) + paddle.audio.functional.power_to_db( + paddle.to_tensor([spect]), amin=0 + ) except Exception: pass try: - paddle.audio.functional.power_to_db(paddle.to_tensor([spect]), - ref_value=0) + paddle.audio.functional.power_to_db( + paddle.to_tensor([spect]), ref_value=0 + ) except Exception: pass try: - paddle.audio.functional.power_to_db(paddle.to_tensor([spect]), - top_db=-1) + paddle.audio.functional.power_to_db( + paddle.to_tensor([spect]), top_db=-1 + ) except Exception: pass - @parameterize([ - "hamming", "hann", "triang", "bohman", "blackman", "cosine", "tukey", - "taylor" - ], [1, 512]) + @parameterize( + [ + "hamming", + "hann", + "triang", + "bohman", + "blackman", + "cosine", + "tukey", + "taylor", + ], + [1, 512], + ) def test_window(self, window_type: str, n_fft: int): window_scipy = signal.get_window(window_type, n_fft) window_paddle = paddle.audio.functional.get_window(window_type, n_fft) - np.testing.assert_array_almost_equal(window_scipy, - window_paddle.numpy(), - decimal=5) + np.testing.assert_array_almost_equal( + window_scipy, window_paddle.numpy(), decimal=5 + ) @parameterize([1, 512]) def test_gaussian_window_and_exception(self, n_fft: int): window_scipy_gaussain = signal.windows.gaussian(n_fft, std=7) window_paddle_gaussian = paddle.audio.functional.get_window( - ('gaussian', 7), n_fft, False) - np.testing.assert_array_almost_equal(window_scipy_gaussain, - window_paddle_gaussian.numpy(), - decimal=5) + ('gaussian', 7), n_fft, False + ) + np.testing.assert_array_almost_equal( + window_scipy_gaussain, window_paddle_gaussian.numpy(), decimal=5 + ) window_scipy_general_gaussain = signal.windows.general_gaussian( - n_fft, 1, 7) + n_fft, 1, 7 + ) window_paddle_general_gaussian = paddle.audio.functional.get_window( - ('general_gaussian', 1, 7), n_fft, False) - np.testing.assert_array_almost_equal(window_scipy_gaussain, - window_paddle_gaussian.numpy(), - decimal=5) + ('general_gaussian', 1, 7), n_fft, False + ) + np.testing.assert_array_almost_equal( + window_scipy_gaussain, window_paddle_gaussian.numpy(), decimal=5 + ) window_scipy_exp = signal.windows.exponential(n_fft) window_paddle_exp = paddle.audio.functional.get_window( - ('exponential', None, 1), n_fft, False) - np.testing.assert_array_almost_equal(window_scipy_exp, - window_paddle_exp.numpy(), - decimal=5) + ('exponential', None, 1), n_fft, False + ) + np.testing.assert_array_almost_equal( + window_scipy_exp, window_paddle_exp.numpy(), decimal=5 + ) try: - window_paddle = paddle.audio.functional.get_window(("kaiser", 1.0), - self.n_fft) + window_paddle = paddle.audio.functional.get_window( + ("kaiser", 1.0), self.n_fft + ) except NotImplementedError: pass @@ -170,7 +192,8 @@ def test_gaussian_window_and_exception(self, n_fft: int): try: window_paddle = paddle.audio.functional.get_window( - "fake_window", self.n_fft) + "fake_window", self.n_fft + ) except ValueError: pass @@ -181,7 +204,6 @@ def test_gaussian_window_and_exception(self, n_fft: int): @parameterize([5, 13, 23], [257, 513, 1025]) def test_create_dct(self, n_mfcc: int, n_mels: int): - def dct(n_filters, n_input): basis = np.empty((n_filters, n_input)) basis[0, :] = 1.0 / np.sqrt(n_input) @@ -195,14 +217,17 @@ def dct(n_filters, n_input): paddle_dct = paddle.audio.functional.create_dct(n_mfcc, n_mels) np.testing.assert_array_almost_equal(librosa_dct, paddle_dct, decimal=5) - @parameterize([128, 256, 512], ["hamming", "hann", "triang", "bohman"], - [True, False]) - def test_stft_and_spect(self, n_fft: int, window_str: str, - center_flag: bool): + @parameterize( + [128, 256, 512], ["hamming", "hann", "triang", "bohman"], [True, False] + ) + def test_stft_and_spect( + self, n_fft: int, window_str: str, center_flag: bool + ): hop_length = int(n_fft / 4) if len(self.waveform.shape) == 2: # (C, T) self.waveform = self.waveform.squeeze( - 0) # 1D input for librosa.feature.melspectrogram + 0 + ) # 1D input for librosa.feature.melspectrogram feature_librosa = librosa.core.stft( y=self.waveform, n_fft=n_fft, @@ -214,9 +239,9 @@ def test_stft_and_spect(self, n_fft: int, window_str: str, pad_mode=self.pad_mode, ) x = paddle.to_tensor(self.waveform).unsqueeze(0) - window = paddle.audio.functional.get_window(window_str, - n_fft, - dtype=x.dtype) + window = paddle.audio.functional.get_window( + window_str, n_fft, dtype=x.dtype + ) feature_paddle = paddle.signal.stft( x=x, n_fft=n_fft, @@ -228,9 +253,9 @@ def test_stft_and_spect(self, n_fft: int, window_str: str, normalized=False, onesided=True, ).squeeze(0) - np.testing.assert_array_almost_equal(feature_librosa, - feature_paddle, - decimal=5) + np.testing.assert_array_almost_equal( + feature_librosa, feature_paddle, decimal=5 + ) feature_bg = np.power(np.abs(feature_librosa), 2.0) feature_extractor = paddle.audio.features.Spectrogram( @@ -243,16 +268,18 @@ def test_stft_and_spect(self, n_fft: int, window_str: str, pad_mode=self.pad_mode, ) feature_layer = feature_extractor(x).squeeze(0) - np.testing.assert_array_almost_equal(feature_layer, - feature_bg, - decimal=3) + np.testing.assert_array_almost_equal( + feature_layer, feature_bg, decimal=3 + ) - @parameterize([128, 256, 512], [64, 82], - ["hamming", "hann", "triang", "bohman"]) + @parameterize( + [128, 256, 512], [64, 82], ["hamming", "hann", "triang", "bohman"] + ) def test_istft(self, n_fft: int, hop_length: int, window_str: str): if len(self.waveform.shape) == 2: # (C, T) self.waveform = self.waveform.squeeze( - 0) # 1D input for librosa.feature.melspectrogram + 0 + ) # 1D input for librosa.feature.melspectrogram # librosa # Get stft result from librosa. stft_matrix = librosa.core.stft( @@ -274,10 +301,9 @@ def test_istft(self, n_fft: int, hop_length: int, window_str: str): length=None, ) x = paddle.to_tensor(stft_matrix).unsqueeze(0) - window = paddle.audio.functional.get_window(window_str, - n_fft, - dtype=paddle.to_tensor( - self.waveform).dtype) + window = paddle.audio.functional.get_window( + window_str, n_fft, dtype=paddle.to_tensor(self.waveform).dtype + ) feature_paddle = paddle.signal.istft( x=x, n_fft=n_fft, @@ -291,9 +317,9 @@ def test_istft(self, n_fft: int, hop_length: int, window_str: str): return_complex=False, ).squeeze(0) - np.testing.assert_array_almost_equal(feature_librosa, - feature_paddle, - decimal=5) + np.testing.assert_array_almost_equal( + feature_librosa, feature_paddle, decimal=5 + ) if __name__ == '__main__': diff --git a/python/paddle/tests/test_audio_logmel_feature.py b/python/paddle/tests/test_audio_logmel_feature.py index 413f4053c9c70..a3a39cd79baf2 100644 --- a/python/paddle/tests/test_audio_logmel_feature.py +++ b/python/paddle/tests/test_audio_logmel_feature.py @@ -28,12 +28,10 @@ def parameterize(*params): class TestFeatures(unittest.TestCase): - def setUp(self): self.initParmas() def initParmas(self): - def get_wav_data(dtype: str, num_channels: int, num_frames: int): dtype_ = getattr(paddle, dtype) base = paddle.linspace(-1.0, 1.0, num_frames, dtype=dtype_) * 0.1 @@ -46,32 +44,44 @@ def get_wav_data(dtype: str, num_channels: int, num_frames: int): self.num_channels = 1 self.sr = 16000 self.dtype = "float32" - waveform_tensor = get_wav_data(self.dtype, - self.num_channels, - num_frames=self.duration * self.sr) + waveform_tensor = get_wav_data( + self.dtype, self.num_channels, num_frames=self.duration * self.sr + ) self.waveform = waveform_tensor.numpy() - @parameterize([16000], ["hamming", "bohman"], [128], [128, 64], [64, 32], - [0.0, 50.0]) - def test_log_melspect(self, sr: int, window_str: str, n_fft: int, - hop_length: int, n_mels: int, fmin: float): + @parameterize( + [16000], ["hamming", "bohman"], [128], [128, 64], [64, 32], [0.0, 50.0] + ) + def test_log_melspect( + self, + sr: int, + window_str: str, + n_fft: int, + hop_length: int, + n_mels: int, + fmin: float, + ): if len(self.waveform.shape) == 2: # (C, T) self.waveform = self.waveform.squeeze( - 0) # 1D input for librosa.feature.melspectrogram + 0 + ) # 1D input for librosa.feature.melspectrogram # librosa: - feature_librosa = librosa.feature.melspectrogram(y=self.waveform, - sr=sr, - n_fft=n_fft, - hop_length=hop_length, - window=window_str, - n_mels=n_mels, - center=True, - fmin=fmin, - pad_mode='reflect') + feature_librosa = librosa.feature.melspectrogram( + y=self.waveform, + sr=sr, + n_fft=n_fft, + hop_length=hop_length, + window=window_str, + n_mels=n_mels, + center=True, + fmin=fmin, + pad_mode='reflect', + ) feature_librosa = librosa.power_to_db(feature_librosa, top_db=None) x = paddle.to_tensor(self.waveform, dtype=paddle.float64).unsqueeze( - 0) # Add batch dim. + 0 + ) # Add batch dim. feature_extractor = paddle.audio.features.LogMelSpectrogram( sr=sr, n_fft=n_fft, @@ -81,54 +91,63 @@ def test_log_melspect(self, sr: int, window_str: str, n_fft: int, n_mels=n_mels, f_min=fmin, top_db=None, - dtype=x.dtype) + dtype=x.dtype, + ) feature_layer = feature_extractor(x).squeeze(0).numpy() - np.testing.assert_array_almost_equal(feature_librosa, - feature_layer, - decimal=2) + np.testing.assert_array_almost_equal( + feature_librosa, feature_layer, decimal=2 + ) # relative difference np.testing.assert_allclose(feature_librosa, feature_layer, rtol=1e-4) - @parameterize([16000], [256, 128], [40, 64], [64, 128], - ['float32', 'float64']) - def test_mfcc(self, sr: int, n_fft: int, n_mfcc: int, n_mels: int, - dtype: str): + @parameterize( + [16000], [256, 128], [40, 64], [64, 128], ['float32', 'float64'] + ) + def test_mfcc( + self, sr: int, n_fft: int, n_mfcc: int, n_mels: int, dtype: str + ): if paddle.version.cuda() != 'False': if float(paddle.version.cuda()) >= 11.0: return if len(self.waveform.shape) == 2: # (C, T) self.waveform = self.waveform.squeeze( - 0) # 1D input for librosa.feature.melspectrogram + 0 + ) # 1D input for librosa.feature.melspectrogram # librosa: np_dtype = getattr(np, dtype) - feature_librosa = librosa.feature.mfcc(y=self.waveform, - sr=sr, - S=None, - n_mfcc=n_mfcc, - dct_type=2, - lifter=0, - n_fft=n_fft, - hop_length=64, - n_mels=n_mels, - fmin=50.0, - dtype=np_dtype) + feature_librosa = librosa.feature.mfcc( + y=self.waveform, + sr=sr, + S=None, + n_mfcc=n_mfcc, + dct_type=2, + lifter=0, + n_fft=n_fft, + hop_length=64, + n_mels=n_mels, + fmin=50.0, + dtype=np_dtype, + ) # paddlespeech.audio.features.layer - x = paddle.to_tensor(self.waveform, - dtype=dtype).unsqueeze(0) # Add batch dim. - feature_extractor = paddle.audio.features.MFCC(sr=sr, - n_mfcc=n_mfcc, - n_fft=n_fft, - hop_length=64, - n_mels=n_mels, - top_db=self.top_db, - dtype=x.dtype) + x = paddle.to_tensor(self.waveform, dtype=dtype).unsqueeze( + 0 + ) # Add batch dim. + feature_extractor = paddle.audio.features.MFCC( + sr=sr, + n_mfcc=n_mfcc, + n_fft=n_fft, + hop_length=64, + n_mels=n_mels, + top_db=self.top_db, + dtype=x.dtype, + ) feature_layer = feature_extractor(x).squeeze(0).numpy() - np.testing.assert_array_almost_equal(feature_librosa, - feature_layer, - decimal=3) + np.testing.assert_array_almost_equal( + feature_librosa, feature_layer, decimal=3 + ) np.testing.assert_allclose(feature_librosa, feature_layer, rtol=1e-1) @@ -142,19 +161,19 @@ def test_mfcc(self, sr: int, n_fft: int, n_mfcc: int, n_mels: int, center=True, pad_mode='reflect', top_db=self.top_db, - dtype=x.dtype) + dtype=x.dtype, + ) feature_layer_logmel = feature_extractor(x).squeeze(0).numpy() - feature_layer_mfcc = scipy.fftpack.dct(feature_layer_logmel, - axis=0, - type=2, - norm="ortho")[:n_mfcc] - np.testing.assert_array_almost_equal(feature_layer_mfcc, - feature_librosa, - decimal=3) - np.testing.assert_allclose(feature_layer_mfcc, - feature_librosa, - rtol=1e-1) + feature_layer_mfcc = scipy.fftpack.dct( + feature_layer_logmel, axis=0, type=2, norm="ortho" + )[:n_mfcc] + np.testing.assert_array_almost_equal( + feature_layer_mfcc, feature_librosa, decimal=3 + ) + np.testing.assert_allclose( + feature_layer_mfcc, feature_librosa, rtol=1e-1 + ) if __name__ == '__main__': diff --git a/python/paddle/tests/test_audio_mel_feature.py b/python/paddle/tests/test_audio_mel_feature.py index 7ddfc9ac48721..ec7b843caf77d 100644 --- a/python/paddle/tests/test_audio_mel_feature.py +++ b/python/paddle/tests/test_audio_mel_feature.py @@ -27,12 +27,10 @@ def parameterize(*params): class TestFeatures(unittest.TestCase): - def setUp(self): self.initParmas() def initParmas(self): - def get_wav_data(dtype: str, num_channels: int, num_frames: int): dtype_ = getattr(paddle, dtype) base = paddle.linspace(-1.0, 1.0, num_frames, dtype=dtype_) * 0.1 @@ -44,15 +42,17 @@ def get_wav_data(dtype: str, num_channels: int, num_frames: int): self.num_channels = 1 self.sr = 16000 self.dtype = "float32" - waveform_tensor = get_wav_data(self.dtype, - self.num_channels, - num_frames=self.duration * self.sr) + waveform_tensor = get_wav_data( + self.dtype, self.num_channels, num_frames=self.duration * self.sr + ) self.waveform = waveform_tensor.numpy() - @parameterize([8000], [128, 256], [64, 32], [0.0, 1.0], - ['float32', 'float64']) - def test_mel(self, sr: int, n_fft: int, n_mels: int, fmin: float, - dtype: str): + @parameterize( + [8000], [128, 256], [64, 32], [0.0, 1.0], ['float32', 'float64'] + ) + def test_mel( + self, sr: int, n_fft: int, n_mels: int, fmin: float, dtype: str + ): feature_librosa = librosa.filters.mel( sr=sr, n_fft=n_fft, @@ -75,40 +75,47 @@ def test_mel(self, sr: int, n_fft: int, n_mels: int, fmin: float, dtype=paddle_dtype, ) - np.testing.assert_array_almost_equal(feature_librosa, - feature_functional) + np.testing.assert_array_almost_equal( + feature_librosa, feature_functional + ) @parameterize([8000, 16000], [128, 256], [64, 82], [40, 80], [False, True]) - def test_melspect(self, sr: int, n_fft: int, hop_length: int, n_mels: int, - htk: bool): + def test_melspect( + self, sr: int, n_fft: int, hop_length: int, n_mels: int, htk: bool + ): if len(self.waveform.shape) == 2: # (C, T) self.waveform = self.waveform.squeeze( - 0) # 1D input for librosa.feature.melspectrogram + 0 + ) # 1D input for librosa.feature.melspectrogram # librosa: - feature_librosa = librosa.feature.melspectrogram(y=self.waveform, - sr=sr, - n_fft=n_fft, - hop_length=hop_length, - n_mels=n_mels, - htk=htk, - fmin=50.0) + feature_librosa = librosa.feature.melspectrogram( + y=self.waveform, + sr=sr, + n_fft=n_fft, + hop_length=hop_length, + n_mels=n_mels, + htk=htk, + fmin=50.0, + ) # paddle.audio.features.layer x = paddle.to_tensor(self.waveform, dtype=paddle.float64).unsqueeze( - 0) # Add batch dim. + 0 + ) # Add batch dim. feature_extractor = paddle.audio.features.MelSpectrogram( sr=sr, n_fft=n_fft, hop_length=hop_length, n_mels=n_mels, htk=htk, - dtype=x.dtype) + dtype=x.dtype, + ) feature_layer = feature_extractor(x).squeeze(0).numpy() - np.testing.assert_array_almost_equal(feature_librosa, - feature_layer, - decimal=5) + np.testing.assert_array_almost_equal( + feature_librosa, feature_layer, decimal=5 + ) if __name__ == '__main__': diff --git a/python/paddle/tests/test_callback_early_stop.py b/python/paddle/tests/test_callback_early_stop.py index 79a6265420148..612c25a6d60d4 100644 --- a/python/paddle/tests/test_callback_early_stop.py +++ b/python/paddle/tests/test_callback_early_stop.py @@ -27,7 +27,6 @@ class MnistDataset(MNIST): - def __init__(self, mode, return_label=True, sample_num=None): super(MnistDataset, self).__init__(mode=mode) self.return_label = return_label @@ -40,14 +39,13 @@ def __getitem__(self, idx): img = np.reshape(img, [1, 28, 28]) if self.return_label: return img, np.array(self.labels[idx]).astype('int64') - return img, + return (img,) def __len__(self): return len(self.images) class TestCallbacks(unittest.TestCase): - def setUp(self): self.save_dir = tempfile.mkdtemp() @@ -64,44 +62,55 @@ def test_earlystopping(self): val_dataset = MnistDataset(mode='test', sample_num=sample_num) net = LeNet() - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=net.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameters=net.parameters() + ) inputs = [InputSpec([None, 1, 28, 28], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] model = Model(net, inputs=inputs, labels=labels) - model.prepare(optim, - loss=CrossEntropyLoss(reduction="sum"), - metrics=[Accuracy()]) - callbacks_0 = paddle.callbacks.EarlyStopping('loss', - mode='min', - patience=1, - verbose=1, - min_delta=0, - baseline=None, - save_best_model=True) - callbacks_1 = paddle.callbacks.EarlyStopping('acc', - mode='auto', - patience=1, - verbose=1, - min_delta=0, - baseline=0, - save_best_model=True) - callbacks_2 = paddle.callbacks.EarlyStopping('loss', - mode='auto_', - patience=1, - verbose=1, - min_delta=0, - baseline=None, - save_best_model=True) - callbacks_3 = paddle.callbacks.EarlyStopping('acc_', - mode='max', - patience=1, - verbose=1, - min_delta=0, - baseline=0, - save_best_model=True) + model.prepare( + optim, + loss=CrossEntropyLoss(reduction="sum"), + metrics=[Accuracy()], + ) + callbacks_0 = paddle.callbacks.EarlyStopping( + 'loss', + mode='min', + patience=1, + verbose=1, + min_delta=0, + baseline=None, + save_best_model=True, + ) + callbacks_1 = paddle.callbacks.EarlyStopping( + 'acc', + mode='auto', + patience=1, + verbose=1, + min_delta=0, + baseline=0, + save_best_model=True, + ) + callbacks_2 = paddle.callbacks.EarlyStopping( + 'loss', + mode='auto_', + patience=1, + verbose=1, + min_delta=0, + baseline=None, + save_best_model=True, + ) + callbacks_3 = paddle.callbacks.EarlyStopping( + 'acc_', + mode='max', + patience=1, + verbose=1, + min_delta=0, + baseline=0, + save_best_model=True, + ) model.fit( train_dataset, val_dataset, @@ -110,15 +119,18 @@ def test_earlystopping(self): save_dir=self.save_dir, epochs=10, verbose=0, - callbacks=[callbacks_0, callbacks_1, callbacks_2, callbacks_3]) + callbacks=[callbacks_0, callbacks_1, callbacks_2, callbacks_3], + ) # Test for no val_loader - model.fit(train_dataset, - batch_size=64, - save_freq=10, - save_dir=self.save_dir, - epochs=10, - verbose=0, - callbacks=[callbacks_0]) + model.fit( + train_dataset, + batch_size=64, + save_freq=10, + save_dir=self.save_dir, + epochs=10, + verbose=0, + callbacks=[callbacks_0], + ) if __name__ == '__main__': diff --git a/python/paddle/tests/test_callback_reduce_lr_on_plateau.py b/python/paddle/tests/test_callback_reduce_lr_on_plateau.py index 02ac3245b7f87..06ffb5888b8b2 100644 --- a/python/paddle/tests/test_callback_reduce_lr_on_plateau.py +++ b/python/paddle/tests/test_callback_reduce_lr_on_plateau.py @@ -27,34 +27,35 @@ # Accelerate unittest class CustomMnist(MNIST): - def __len__(self): return 8 class TestReduceLROnPlateau(unittest.TestCase): - def func_reduce_lr_on_plateau(self): transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])]) train_dataset = CustomMnist(mode='train', transform=transform) val_dataset = CustomMnist(mode='test', transform=transform) net = LeNet() - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=net.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameters=net.parameters() + ) inputs = [InputSpec([None, 1, 28, 28], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] model = Model(net, inputs=inputs, labels=labels) model.prepare(optim, loss=CrossEntropyLoss(), metrics=[Accuracy()]) - callbacks = paddle.callbacks.ReduceLROnPlateau(patience=1, - verbose=1, - cooldown=1) - model.fit(train_dataset, - val_dataset, - batch_size=8, - log_freq=1, - save_freq=10, - epochs=10, - callbacks=[callbacks]) + callbacks = paddle.callbacks.ReduceLROnPlateau( + patience=1, verbose=1, cooldown=1 + ) + model.fit( + train_dataset, + val_dataset, + batch_size=8, + log_freq=1, + save_freq=10, + epochs=10, + callbacks=[callbacks], + ) def test_reduce_lr_on_plateau(self): with _test_eager_guard(): @@ -71,41 +72,46 @@ def func_warn_or_error(self): train_dataset = CustomMnist(mode='train', transform=transform) val_dataset = CustomMnist(mode='test', transform=transform) net = LeNet() - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=net.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameters=net.parameters() + ) inputs = [InputSpec([None, 1, 28, 28], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] model = Model(net, inputs=inputs, labels=labels) model.prepare(optim, loss=CrossEntropyLoss(), metrics=[Accuracy()]) - callbacks = paddle.callbacks.ReduceLROnPlateau(monitor='miou', - patience=3, - verbose=1) - model.fit(train_dataset, - val_dataset, - batch_size=8, - log_freq=1, - save_freq=10, - epochs=1, - callbacks=[callbacks]) + callbacks = paddle.callbacks.ReduceLROnPlateau( + monitor='miou', patience=3, verbose=1 + ) + model.fit( + train_dataset, + val_dataset, + batch_size=8, + log_freq=1, + save_freq=10, + epochs=1, + callbacks=[callbacks], + ) optim = paddle.optimizer.Adam( - learning_rate=paddle.optimizer.lr.PiecewiseDecay([0.001, 0.0001], - [5, 10]), - parameters=net.parameters()) + learning_rate=paddle.optimizer.lr.PiecewiseDecay( + [0.001, 0.0001], [5, 10] + ), + parameters=net.parameters(), + ) model.prepare(optim, loss=CrossEntropyLoss(), metrics=[Accuracy()]) - callbacks = paddle.callbacks.ReduceLROnPlateau(monitor='acc', - mode='max', - patience=3, - verbose=1, - cooldown=1) - model.fit(train_dataset, - val_dataset, - batch_size=8, - log_freq=1, - save_freq=10, - epochs=3, - callbacks=[callbacks]) + callbacks = paddle.callbacks.ReduceLROnPlateau( + monitor='acc', mode='max', patience=3, verbose=1, cooldown=1 + ) + model.fit( + train_dataset, + val_dataset, + batch_size=8, + log_freq=1, + save_freq=10, + epochs=3, + callbacks=[callbacks], + ) def test_warn_or_error(self): with _test_eager_guard(): diff --git a/python/paddle/tests/test_callback_visualdl.py b/python/paddle/tests/test_callback_visualdl.py index a1fc1cbadad4b..aa6616f673053 100644 --- a/python/paddle/tests/test_callback_visualdl.py +++ b/python/paddle/tests/test_callback_visualdl.py @@ -24,13 +24,11 @@ class MnistDataset(MNIST): - def __len__(self): return 512 class TestCallbacks(unittest.TestCase): - def setUp(self): self.save_dir = tempfile.mkdtemp() @@ -49,15 +47,16 @@ def func_visualdl_callback(self): model = paddle.Model(net, inputs, labels) optim = paddle.optimizer.Adam(0.001, parameters=net.parameters()) - model.prepare(optimizer=optim, - loss=paddle.nn.CrossEntropyLoss(), - metrics=paddle.metric.Accuracy()) + model.prepare( + optimizer=optim, + loss=paddle.nn.CrossEntropyLoss(), + metrics=paddle.metric.Accuracy(), + ) callback = paddle.callbacks.VisualDL(log_dir='visualdl_log_dir') - model.fit(train_dataset, - eval_dataset, - batch_size=64, - callbacks=callback) + model.fit( + train_dataset, eval_dataset, batch_size=64, callbacks=callback + ) def test_visualdl_callback(self): with _test_eager_guard(): diff --git a/python/paddle/tests/test_callbacks.py b/python/paddle/tests/test_callbacks.py index cc43f82c5c279..70dda874daccc 100644 --- a/python/paddle/tests/test_callbacks.py +++ b/python/paddle/tests/test_callbacks.py @@ -27,7 +27,6 @@ class MnistDataset(MNIST): - def __init__(self, mode, return_label=True, sample_num=None): super(MnistDataset, self).__init__(mode=mode) self.return_label = return_label @@ -40,14 +39,13 @@ def __getitem__(self, idx): img = np.reshape(img, [1, 28, 28]) if self.return_label: return img, np.array(self.labels[idx]).astype('int64') - return img, + return (img,) def __len__(self): return len(self.images) class TestCallbacks(unittest.TestCase): - def setUp(self): self.save_dir = tempfile.mkdtemp() @@ -64,14 +62,16 @@ def run_callback(self): lenet = Model(LeNet(), inputs) lenet.prepare() - cbks = config_callbacks(model=lenet, - batch_size=128, - epochs=epochs, - steps=steps, - log_freq=freq, - verbose=self.verbose, - metrics=['loss', 'acc'], - save_dir=self.save_dir) + cbks = config_callbacks( + model=lenet, + batch_size=128, + epochs=epochs, + steps=steps, + log_freq=freq, + verbose=self.verbose, + metrics=['loss', 'acc'], + save_dir=self.save_dir, + ) cbks.on_begin('train') logs = {'loss': 50.341673, 'acc': 0.00256} diff --git a/python/paddle/tests/test_dataset_cifar.py b/python/paddle/tests/test_dataset_cifar.py index 95bf21c4da181..abf79fb1e3974 100644 --- a/python/paddle/tests/test_dataset_cifar.py +++ b/python/paddle/tests/test_dataset_cifar.py @@ -19,7 +19,6 @@ class TestCifar10Train(unittest.TestCase): - def test_main(self): cifar = Cifar10(mode='train') self.assertTrue(len(cifar) == 50000) @@ -37,7 +36,6 @@ def test_main(self): class TestCifar10Test(unittest.TestCase): - def test_main(self): cifar = Cifar10(mode='test') self.assertTrue(len(cifar) == 10000) @@ -72,7 +70,6 @@ def test_main(self): class TestCifar100Train(unittest.TestCase): - def test_main(self): cifar = Cifar100(mode='train') self.assertTrue(len(cifar) == 50000) @@ -90,7 +87,6 @@ def test_main(self): class TestCifar100Test(unittest.TestCase): - def test_main(self): cifar = Cifar100(mode='test') self.assertTrue(len(cifar) == 10000) diff --git a/python/paddle/tests/test_dataset_conll05.py b/python/paddle/tests/test_dataset_conll05.py index 8dc1f56779f71..9eb0036718b35 100644 --- a/python/paddle/tests/test_dataset_conll05.py +++ b/python/paddle/tests/test_dataset_conll05.py @@ -20,7 +20,6 @@ class TestConll05st(unittest.TestCase): - def test_main(self): conll05st = Conll05st() self.assertTrue(len(conll05st) == 5267) diff --git a/python/paddle/tests/test_dataset_imdb.py b/python/paddle/tests/test_dataset_imdb.py index c70b7fedf3632..aed8c387409dc 100644 --- a/python/paddle/tests/test_dataset_imdb.py +++ b/python/paddle/tests/test_dataset_imdb.py @@ -19,7 +19,6 @@ class TestImdbTrain(unittest.TestCase): - def test_main(self): imdb = Imdb(mode='train') self.assertTrue(len(imdb) == 25000) @@ -34,7 +33,6 @@ def test_main(self): class TestImdbTest(unittest.TestCase): - def test_main(self): imdb = Imdb(mode='test') self.assertTrue(len(imdb) == 25000) diff --git a/python/paddle/tests/test_dataset_imikolov.py b/python/paddle/tests/test_dataset_imikolov.py index 6379ed11e5daa..6ffeeda73c362 100644 --- a/python/paddle/tests/test_dataset_imikolov.py +++ b/python/paddle/tests/test_dataset_imikolov.py @@ -19,7 +19,6 @@ class TestImikolovTrain(unittest.TestCase): - def test_main(self): imikolov = Imikolov(mode='train', data_type='NGRAM', window_size=2) self.assertTrue(len(imikolov) == 929589) @@ -32,7 +31,6 @@ def test_main(self): class TestImikolovTest(unittest.TestCase): - def test_main(self): imikolov = Imikolov(mode='test', data_type='NGRAM', window_size=2) self.assertTrue(len(imikolov) == 82430) diff --git a/python/paddle/tests/test_dataset_movielens.py b/python/paddle/tests/test_dataset_movielens.py index 78a6211647656..e5c6d8376eed9 100644 --- a/python/paddle/tests/test_dataset_movielens.py +++ b/python/paddle/tests/test_dataset_movielens.py @@ -19,7 +19,6 @@ class TestMovielensTrain(unittest.TestCase): - def test_main(self): movielens = Movielens(mode='train') # movielens dataset random split train/test @@ -37,7 +36,6 @@ def test_main(self): class TestMovielensTest(unittest.TestCase): - def test_main(self): movielens = Movielens(mode='test') # movielens dataset random split train/test diff --git a/python/paddle/tests/test_dataset_uci_housing.py b/python/paddle/tests/test_dataset_uci_housing.py index 563dd62efa3b6..fe85e7683dd9c 100644 --- a/python/paddle/tests/test_dataset_uci_housing.py +++ b/python/paddle/tests/test_dataset_uci_housing.py @@ -19,7 +19,6 @@ class TestUCIHousingTrain(unittest.TestCase): - def test_main(self): uci_housing = UCIHousing(mode='train') self.assertTrue(len(uci_housing) == 404) @@ -36,7 +35,6 @@ def test_main(self): class TestUCIHousingTest(unittest.TestCase): - def test_main(self): uci_housing = UCIHousing(mode='test') self.assertTrue(len(uci_housing) == 102) @@ -53,7 +51,6 @@ def test_main(self): class TestWMT14Train(unittest.TestCase): - def test_main(self): wmt14 = WMT14(mode='train', dict_size=50) self.assertTrue(len(wmt14) == 191155) @@ -69,7 +66,6 @@ def test_main(self): class TestWMT14Test(unittest.TestCase): - def test_main(self): wmt14 = WMT14(mode='test', dict_size=50) self.assertTrue(len(wmt14) == 5957) @@ -85,7 +81,6 @@ def test_main(self): class TestWMT14Gen(unittest.TestCase): - def test_main(self): wmt14 = WMT14(mode='gen', dict_size=50) self.assertTrue(len(wmt14) == 3001) diff --git a/python/paddle/tests/test_dataset_voc.py b/python/paddle/tests/test_dataset_voc.py index 4d67ba9c10a54..459068c2df201 100644 --- a/python/paddle/tests/test_dataset_voc.py +++ b/python/paddle/tests/test_dataset_voc.py @@ -23,7 +23,6 @@ class TestVOC2012Train(unittest.TestCase): - def test_main(self): voc2012 = VOC2012(mode='train') self.assertTrue(len(voc2012) == 3) @@ -40,7 +39,6 @@ def test_main(self): class TestVOC2012Valid(unittest.TestCase): - def test_main(self): voc2012 = VOC2012(mode='valid') self.assertTrue(len(voc2012) == 1) @@ -57,7 +55,6 @@ def test_main(self): class TestVOC2012Test(unittest.TestCase): - def test_main(self): voc2012 = VOC2012(mode='test') self.assertTrue(len(voc2012) == 2) diff --git a/python/paddle/tests/test_dataset_wmt.py b/python/paddle/tests/test_dataset_wmt.py index 48186ab1864aa..43663945e20fd 100644 --- a/python/paddle/tests/test_dataset_wmt.py +++ b/python/paddle/tests/test_dataset_wmt.py @@ -19,7 +19,6 @@ class TestWMT14Train(unittest.TestCase): - def test_main(self): wmt14 = WMT14(mode='train', dict_size=50) self.assertTrue(len(wmt14) == 191155) @@ -35,7 +34,6 @@ def test_main(self): class TestWMT14Test(unittest.TestCase): - def test_main(self): wmt14 = WMT14(mode='test', dict_size=50) self.assertTrue(len(wmt14) == 5957) @@ -51,7 +49,6 @@ def test_main(self): class TestWMT14Gen(unittest.TestCase): - def test_main(self): wmt14 = WMT14(mode='gen', dict_size=50) self.assertTrue(len(wmt14) == 3001) @@ -67,12 +64,10 @@ def test_main(self): class TestWMT16Train(unittest.TestCase): - def test_main(self): - wmt16 = WMT16(mode='train', - src_dict_size=50, - trg_dict_size=50, - lang='en') + wmt16 = WMT16( + mode='train', src_dict_size=50, trg_dict_size=50, lang='en' + ) self.assertTrue(len(wmt16) == 29000) # traversal whole dataset may cost a @@ -86,12 +81,10 @@ def test_main(self): class TestWMT16Test(unittest.TestCase): - def test_main(self): - wmt16 = WMT16(mode='test', - src_dict_size=50, - trg_dict_size=50, - lang='en') + wmt16 = WMT16( + mode='test', src_dict_size=50, trg_dict_size=50, lang='en' + ) self.assertTrue(len(wmt16) == 1000) # traversal whole dataset may cost a @@ -105,7 +98,6 @@ def test_main(self): class TestWMT16Val(unittest.TestCase): - def test_main(self): wmt16 = WMT16(mode='val', src_dict_size=50, trg_dict_size=50, lang='en') self.assertTrue(len(wmt16) == 1014) diff --git a/python/paddle/tests/test_datasets.py b/python/paddle/tests/test_datasets.py index 0f6396ec05149..172d9007de595 100644 --- a/python/paddle/tests/test_datasets.py +++ b/python/paddle/tests/test_datasets.py @@ -20,13 +20,18 @@ import cv2 import paddle.vision.transforms as T -from paddle.vision.datasets import DatasetFolder, ImageFolder, MNIST, FashionMNIST, Flowers +from paddle.vision.datasets import ( + DatasetFolder, + ImageFolder, + MNIST, + FashionMNIST, + Flowers, +) from paddle.dataset.common import _check_exists_and_download from paddle.fluid.framework import _test_eager_guard class TestFolderDatasets(unittest.TestCase): - def setUp(self): self.data_dir = tempfile.mkdtemp() self.empty_dir = tempfile.mkdtemp() @@ -77,7 +82,6 @@ def test_folder(self): self.func_test_folder() def func_test_transform(self): - def fake_transform(img): return img @@ -112,7 +116,6 @@ def test_errors(self): class TestMNISTTest(unittest.TestCase): - def func_test_main(self): transform = T.Transpose() mnist = MNIST(mode='test', transform=transform) @@ -133,7 +136,6 @@ def test_main(self): class TestMNISTTrain(unittest.TestCase): - def func_test_main(self): transform = T.Transpose() mnist = MNIST(mode='train', transform=transform) @@ -170,7 +172,6 @@ def test_main(self): class TestFASHIONMNISTTest(unittest.TestCase): - def func_test_main(self): transform = T.Transpose() mnist = FashionMNIST(mode='test', transform=transform) @@ -191,7 +192,6 @@ def test_main(self): class TestFASHIONMNISTTrain(unittest.TestCase): - def func_test_main(self): transform = T.Transpose() mnist = FashionMNIST(mode='train', transform=transform) @@ -240,7 +240,6 @@ def test_dataset_value(self): class TestFlowersTrain(unittest.TestCase): - def func_test_main(self): flowers = Flowers(mode='train') self.assertTrue(len(flowers) == 6149) @@ -261,7 +260,6 @@ def test_main(self): class TestFlowersValid(unittest.TestCase): - def func_test_main(self): flowers = Flowers(mode='valid') self.assertTrue(len(flowers) == 1020) @@ -282,7 +280,6 @@ def test_main(self): class TestFlowersTest(unittest.TestCase): - def func_test_main(self): flowers = Flowers(mode='test') self.assertTrue(len(flowers) == 1020) diff --git a/python/paddle/tests/test_dist_hapi_model.py b/python/paddle/tests/test_dist_hapi_model.py index 852572c3c7d04..0ec5cc4f845bf 100644 --- a/python/paddle/tests/test_dist_hapi_model.py +++ b/python/paddle/tests/test_dist_hapi_model.py @@ -19,7 +19,12 @@ import subprocess import paddle.fluid as fluid -from paddle.distributed.utils.launch_utils import find_free_ports, watch_local_trainers, get_cluster, TrainerProc +from paddle.distributed.utils.launch_utils import ( + find_free_ports, + watch_local_trainers, + get_cluster, + TrainerProc, +) def get_cluster_from_args(selected_gpus): @@ -47,17 +52,19 @@ def get_gpus(selected_gpus): return selected_gpus -def start_local_trainers(cluster, - pod, - training_script, - eager_mode, - training_script_args, - log_dir=None): +def start_local_trainers( + cluster, + pod, + training_script, + eager_mode, + training_script_args, + log_dir=None, +): current_env = copy.copy(os.environ.copy()) - #paddle broadcast ncclUniqueId use socket, and - #proxy maybe make trainers unreachable, so delete them. - #if we set them to "", grpc will log error message "bad uri" - #so just delete them. + # paddle broadcast ncclUniqueId use socket, and + # proxy maybe make trainers unreachable, so delete them. + # if we set them to "", grpc will log error message "bad uri" + # so just delete them. current_env.pop("http_proxy", None) current_env.pop("https_proxy", None) @@ -68,7 +75,7 @@ def start_local_trainers(cluster, "PADDLE_TRAINER_ID": "%d" % t.rank, "PADDLE_CURRENT_ENDPOINT": "%s" % t.endpoint, "PADDLE_TRAINERS_NUM": "%d" % cluster.trainers_nranks(), - "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()) + "PADDLE_TRAINER_ENDPOINTS": ",".join(cluster.trainers_endpoints()), } if not eager_mode: @@ -101,7 +108,6 @@ def start_local_trainers(cluster, class TestMultipleGpus(unittest.TestCase): - def run_mnist_2gpu(self, target_file_name, eager_mode=True): if fluid.core.get_cuda_device_count() == 0: return @@ -112,11 +118,13 @@ def run_mnist_2gpu(self, target_file_name, eager_mode=True): cluster, pod = get_cluster_from_args(selected_gpus) - procs = start_local_trainers(cluster, - pod, - eager_mode=eager_mode, - training_script=target_file_name, - training_script_args=[]) + procs = start_local_trainers( + cluster, + pod, + eager_mode=eager_mode, + training_script=target_file_name, + training_script_args=[], + ) while True: alive = watch_local_trainers(procs, cluster.trainers_nranks()) diff --git a/python/paddle/tests/test_dlpack.py b/python/paddle/tests/test_dlpack.py index 256ca41ffff2d..5e4b8d2ef20b6 100644 --- a/python/paddle/tests/test_dlpack.py +++ b/python/paddle/tests/test_dlpack.py @@ -22,7 +22,6 @@ class TestDLPack(unittest.TestCase): - def func_test_dlpack_dygraph(self): paddle.disable_static() tensor = paddle.to_tensor(np.array([1, 2, 3, 4]).astype('int')) @@ -30,11 +29,13 @@ def func_test_dlpack_dygraph(self): out_from_dlpack = paddle.utils.dlpack.from_dlpack(dlpack) if paddle.fluid.framework.in_dygraph_mode(): self.assertTrue( - isinstance(out_from_dlpack, paddle.fluid.core.eager.Tensor)) + isinstance(out_from_dlpack, paddle.fluid.core.eager.Tensor) + ) else: self.assertTrue(isinstance(out_from_dlpack, paddle.Tensor)) - np.testing.assert_array_equal(np.array(out_from_dlpack), - np.array([1, 2, 3, 4]).astype('int')) + np.testing.assert_array_equal( + np.array(out_from_dlpack), np.array([1, 2, 3, 4]).astype('int') + ) def test_dlpack_dygraph(self): with _test_eager_guard(): @@ -58,26 +59,32 @@ def test_dlpack_tensor_larger_than_2dim(self): def test_dlpack_static(self): paddle.enable_static() tensor = fluid.create_lod_tensor( - np.array([[1], [2], [3], [4]]).astype('int'), [[1, 3]], - fluid.CPUPlace()) + np.array([[1], [2], [3], [4]]).astype('int'), + [[1, 3]], + fluid.CPUPlace(), + ) dlpack = paddle.utils.dlpack.to_dlpack(tensor) out_from_dlpack = paddle.utils.dlpack.from_dlpack(dlpack) self.assertTrue(isinstance(out_from_dlpack, fluid.core.Tensor)) np.testing.assert_array_equal( np.array(out_from_dlpack), - np.array([[1], [2], [3], [4]]).astype('int')) + np.array([[1], [2], [3], [4]]).astype('int'), + ) # when build with cuda if core.is_compiled_with_cuda(): gtensor = fluid.create_lod_tensor( - np.array([[1], [2], [3], [4]]).astype('int'), [[1, 3]], - fluid.CUDAPlace(0)) + np.array([[1], [2], [3], [4]]).astype('int'), + [[1, 3]], + fluid.CUDAPlace(0), + ) gdlpack = paddle.utils.dlpack.to_dlpack(gtensor) gout_from_dlpack = paddle.utils.dlpack.from_dlpack(gdlpack) self.assertTrue(isinstance(gout_from_dlpack, fluid.core.Tensor)) np.testing.assert_array_equal( np.array(gout_from_dlpack), - np.array([[1], [2], [3], [4]]).astype('int')) + np.array([[1], [2], [3], [4]]).astype('int'), + ) def func_test_dlpack_dtype_conversion(self): paddle.disable_static() @@ -104,7 +111,8 @@ def func_test_dlpack_dtype_conversion(self): for dtype in complex_dtypes: x = paddle.to_tensor( [[1 + 6j, 2 + 5j, 3 + 4j], [4 + 3j, 5 + 2j, 6 + 1j]], - dtype=dtype) + dtype=dtype, + ) dlpack = paddle.utils.dlpack.to_dlpack(x) o = paddle.utils.dlpack.from_dlpack(dlpack) self.assertEqual(x.dtype, o.dtype) @@ -117,10 +125,10 @@ def test_dlpack_dtype_conversion(self): class TestRaiseError(unittest.TestCase): - def func_test_from_dlpack_raise_type_error(self): - self.assertRaises(TypeError, paddle.utils.dlpack.from_dlpack, - np.zeros(5)) + self.assertRaises( + TypeError, paddle.utils.dlpack.from_dlpack, np.zeros(5) + ) def test_from_dlpack_raise_type_error(self): with _test_eager_guard(): diff --git a/python/paddle/tests/test_download.py b/python/paddle/tests/test_download.py index 3e6fcc5429780..0978acbf56ab4 100644 --- a/python/paddle/tests/test_download.py +++ b/python/paddle/tests/test_download.py @@ -20,7 +20,6 @@ class TestDownload(unittest.TestCase): - def download(self, url, md5sum): get_weights_path_from_url(url, md5sum) @@ -75,8 +74,9 @@ def test_get_path_from_url(self): def test_uncompress_result(self): results = [ [ - "files/single_dir/file1", "files/single_dir/file2", - "files/single_file.pdparams" + "files/single_dir/file1", + "files/single_dir/file2", + "files/single_file.pdparams", ], ["single_dir/file1", "single_dir/file2"], ["single_file.pdparams"], @@ -89,8 +89,14 @@ def test_uncompress_result(self): for url, uncompressd_res in zip(tar_urls, results): uncompressed_path = get_path_from_url(url, root_dir='./test_tar') - self.assertTrue(all([os.path.exists(os.path.join("./test_tar", filepath)) \ - for filepath in uncompressd_res])) + self.assertTrue( + all( + [ + os.path.exists(os.path.join("./test_tar", filepath)) + for filepath in uncompressd_res + ] + ) + ) zip_urls = [ "https://paddle-hapi.bj.bcebos.com/unittest/files.zip", @@ -99,23 +105,37 @@ def test_uncompress_result(self): ] for url, uncompressd_res in zip(zip_urls, results): uncompressed_path = get_path_from_url(url, root_dir='./test_zip') - self.assertTrue(all([os.path.exists(os.path.join("./test_zip", filepath)) \ - for filepath in uncompressd_res])) + self.assertTrue( + all( + [ + os.path.exists(os.path.join("./test_zip", filepath)) + for filepath in uncompressd_res + ] + ) + ) - def test_retry_exception(self, ): + def test_retry_exception( + self, + ): with self.assertRaises(RuntimeError): from paddle.utils.download import _download + _download( 'www.baidu.com', './test', ) - def test_wget_download_error(self, ): + def test_wget_download_error( + self, + ): with self.assertRaises(RuntimeError): from paddle.utils.download import _download + _download('www.baidu', './test', method='wget') - def test_download_methods(self, ): + def test_download_methods( + self, + ): urls = [ "https://paddle-hapi.bj.bcebos.com/unittest/files.tar", "https://paddle-hapi.bj.bcebos.com/unittest/files.zip", @@ -123,6 +143,7 @@ def test_download_methods(self, ): import sys from paddle.utils.download import _download + if sys.platform == 'linux': methods = ['wget', 'get'] else: diff --git a/python/paddle/tests/test_hapi_amp.py b/python/paddle/tests/test_hapi_amp.py index 537dab34a6853..e5be62a9e4868 100644 --- a/python/paddle/tests/test_hapi_amp.py +++ b/python/paddle/tests/test_hapi_amp.py @@ -31,30 +31,31 @@ import paddle.vision.transforms as T -@unittest.skipIf(not fluid.is_compiled_with_cuda(), - 'CPU testing is not supported') +@unittest.skipIf( + not fluid.is_compiled_with_cuda(), 'CPU testing is not supported' +) class TestHapiWithAmp(unittest.TestCase): - def get_model(self, amp_config): net = LeNet() inputs = InputSpec([None, 1, 28, 28], "float32", 'x') labels = InputSpec([None, 1], "int64", "y") model = Model(net, inputs, labels) - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=model.parameters()) - model.prepare(optimizer=optim, - loss=CrossEntropyLoss(reduction="sum"), - amp_configs=amp_config) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameters=model.parameters() + ) + model.prepare( + optimizer=optim, + loss=CrossEntropyLoss(reduction="sum"), + amp_configs=amp_config, + ) return model def run_model(self, model): transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])]) train_dataset = MNIST(mode='train', transform=transform) - model.fit(train_dataset, - epochs=1, - batch_size=64, - num_iters=2, - log_freq=1) + model.fit( + train_dataset, epochs=1, batch_size=64, num_iters=2, log_freq=1 + ) def run_amp(self, amp_level): for dynamic in [True, False]: @@ -93,11 +94,9 @@ def test_save_load(self): model = self.get_model(amp_level) transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])]) train_dataset = MNIST(mode='train', transform=transform) - model.fit(train_dataset, - epochs=1, - batch_size=64, - num_iters=2, - log_freq=1) + model.fit( + train_dataset, epochs=1, batch_size=64, num_iters=2, log_freq=1 + ) temp_dir = tempfile.TemporaryDirectory() lenet_amp_path = os.path.join(temp_dir.name, './lenet_amp') model.save(lenet_amp_path) @@ -106,42 +105,43 @@ def test_save_load(self): paddle.seed(2021) new_model = self.get_model(amp_level) train_dataset = MNIST(mode='train', transform=transform) - new_model.fit(train_dataset, - epochs=1, - batch_size=64, - num_iters=1, - log_freq=1) + new_model.fit( + train_dataset, epochs=1, batch_size=64, num_iters=1, log_freq=1 + ) # not equal before load - self.assertNotEqual(new_model._scaler.state_dict()['incr_count'], - model._scaler.state_dict()['incr_count']) - print((new_model._scaler.state_dict()['incr_count'], - model._scaler.state_dict()['incr_count'])) + self.assertNotEqual( + new_model._scaler.state_dict()['incr_count'], + model._scaler.state_dict()['incr_count'], + ) + print( + ( + new_model._scaler.state_dict()['incr_count'], + model._scaler.state_dict()['incr_count'], + ) + ) # equal after load new_model.load(lenet_amp_path) temp_dir.cleanup() - self.assertEqual(new_model._scaler.state_dict()['incr_count'], - model._scaler.state_dict()['incr_count']) - self.assertEqual(new_model._scaler.state_dict()['decr_count'], - model._scaler.state_dict()['decr_count']) + self.assertEqual( + new_model._scaler.state_dict()['incr_count'], + model._scaler.state_dict()['incr_count'], + ) + self.assertEqual( + new_model._scaler.state_dict()['decr_count'], + model._scaler.state_dict()['decr_count'], + ) np.testing.assert_array_equal( new_model._optimizer.state_dict()['conv2d_1.w_0_moment1_0'].numpy(), - model._optimizer.state_dict()['conv2d_1.w_0_moment1_0'].numpy()) + model._optimizer.state_dict()['conv2d_1.w_0_moment1_0'].numpy(), + ) def test_dynamic_check_input(self): paddle.disable_static() amp_configs_list = [ - { - "level": "O3" - }, - { - "level": "O1", - "test": 0 - }, - { - "level": "O1", - "use_fp16_guard": True - }, + {"level": "O3"}, + {"level": "O1", "test": 0}, + {"level": "O1", "use_fp16_guard": True}, "O3", ] if not fluid.is_compiled_with_cuda(): @@ -149,21 +149,24 @@ def test_dynamic_check_input(self): paddle.set_device('gpu') net = LeNet() model = Model(net) - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=model.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameters=model.parameters() + ) loss = CrossEntropyLoss(reduction="sum") with self.assertRaises(ValueError): for amp_configs in amp_configs_list: - model.prepare(optimizer=optim, - loss=loss, - amp_configs=amp_configs) + model.prepare( + optimizer=optim, loss=loss, amp_configs=amp_configs + ) model.prepare(optimizer=optim, loss=loss, amp_configs="O2") - model.prepare(optimizer=optim, - loss=loss, - amp_configs={ - "custom_white_list": {"matmul"}, - "init_loss_scaling": 1.0 - }) + model.prepare( + optimizer=optim, + loss=loss, + amp_configs={ + "custom_white_list": {"matmul"}, + "init_loss_scaling": 1.0, + }, + ) def test_static_check_input(self): paddle.enable_static() @@ -177,8 +180,9 @@ def test_static_check_input(self): labels = InputSpec([None, 1], "int64", "y") model = Model(net, inputs, labels) - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=model.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameters=model.parameters() + ) loss = CrossEntropyLoss(reduction="sum") with self.assertRaises(ValueError): model.prepare(optimizer=optim, loss=loss, amp_configs=amp_configs) diff --git a/python/paddle/tests/test_hapi_hub.py b/python/paddle/tests/test_hapi_hub.py index 526b35eddff6c..2a383311e1eba 100644 --- a/python/paddle/tests/test_hapi_hub.py +++ b/python/paddle/tests/test_hapi_hub.py @@ -22,72 +22,90 @@ class TestHub(unittest.TestCase): - - def setUp(self, ): + def setUp( + self, + ): self.local_repo = os.path.dirname(os.path.abspath(__file__)) self.github_repo = 'lyuwenyu/paddlehub_demo:main' - def testLoad(self, ): - model = hub.load(self.local_repo, - model='MM', - source='local', - out_channels=8) + def testLoad( + self, + ): + model = hub.load( + self.local_repo, model='MM', source='local', out_channels=8 + ) data = paddle.rand((1, 3, 100, 100)) out = model(data) np.testing.assert_equal(out.shape, [1, 8, 50, 50]) - model = hub.load(self.github_repo, - model='MM', - source='github', - force_reload=True) - - model = hub.load(self.github_repo, - model='MM', - source='github', - force_reload=False, - pretrained=False) - - model = hub.load(self.github_repo.split(':')[0], - model='MM', - source='github', - force_reload=False, - pretrained=False) - - model = hub.load(self.github_repo, - model='MM', - source='github', - force_reload=False, - pretrained=True, - out_channels=8) + model = hub.load( + self.github_repo, model='MM', source='github', force_reload=True + ) + + model = hub.load( + self.github_repo, + model='MM', + source='github', + force_reload=False, + pretrained=False, + ) + + model = hub.load( + self.github_repo.split(':')[0], + model='MM', + source='github', + force_reload=False, + pretrained=False, + ) + + model = hub.load( + self.github_repo, + model='MM', + source='github', + force_reload=False, + pretrained=True, + out_channels=8, + ) data = paddle.ones((1, 3, 2, 2)) out = model(data) - gt = np.array([ - 1.53965068, 0., 0., 1.39455748, 0.72066200, 0.19773030, 2.09201908, - 0.37345418 - ]) + gt = np.array( + [ + 1.53965068, + 0.0, + 0.0, + 1.39455748, + 0.72066200, + 0.19773030, + 2.09201908, + 0.37345418, + ] + ) np.testing.assert_equal(out.shape, [1, 8, 1, 1]) - np.testing.assert_almost_equal(out.numpy(), - gt.reshape(1, 8, 1, 1), - decimal=5) + np.testing.assert_almost_equal( + out.numpy(), gt.reshape(1, 8, 1, 1), decimal=5 + ) - def testHelp(self, ): + def testHelp( + self, + ): docs1 = hub.help( self.local_repo, model='MM', source='local', ) - docs2 = hub.help(self.github_repo, - model='MM', - source='github', - force_reload=False) + docs2 = hub.help( + self.github_repo, model='MM', source='github', force_reload=False + ) assert docs1 == docs2 == 'This is a test demo for paddle hub\n ', '' - def testList(self, ): + def testList( + self, + ): models1 = hub.list( self.local_repo, source='local', @@ -102,35 +120,39 @@ def testList(self, ): assert models1 == models2 == ['MM'], '' - def testExcept(self, ): + def testExcept( + self, + ): with self.assertRaises(ValueError): - _ = hub.help(self.github_repo, - model='MM', - source='github-test', - force_reload=False) + _ = hub.help( + self.github_repo, + model='MM', + source='github-test', + force_reload=False, + ) with self.assertRaises(ValueError): - _ = hub.load(self.github_repo, - model='MM', - source='github-test', - force_reload=False) + _ = hub.load( + self.github_repo, + model='MM', + source='github-test', + force_reload=False, + ) with self.assertRaises(ValueError): - _ = hub.list(self.github_repo, - source='github-test', - force_reload=False) + _ = hub.list( + self.github_repo, source='github-test', force_reload=False + ) with self.assertRaises(ValueError): - _ = hub.load(self.local_repo, - model=123, - source='local', - force_reload=False) + _ = hub.load( + self.local_repo, model=123, source='local', force_reload=False + ) with self.assertRaises(RuntimeError): - _ = hub.load(self.local_repo, - model='123', - source='local', - force_reload=False) + _ = hub.load( + self.local_repo, model='123', source='local', force_reload=False + ) if __name__ == '__main__': diff --git a/python/paddle/tests/test_hapi_hub_model.py b/python/paddle/tests/test_hapi_hub_model.py index 6170409fbe084..f7e2b716a1df5 100644 --- a/python/paddle/tests/test_hapi_hub_model.py +++ b/python/paddle/tests/test_hapi_hub_model.py @@ -17,7 +17,6 @@ class MM(nn.Layer): - def __init__(self, out_channels): super(MM, self).__init__() self.conv = nn.Conv2D(3, out_channels, 3, 2, 1) diff --git a/python/paddle/tests/test_logger.py b/python/paddle/tests/test_logger.py index 4566a7ff693fb..e63b4e2cab25a 100644 --- a/python/paddle/tests/test_logger.py +++ b/python/paddle/tests/test_logger.py @@ -21,7 +21,6 @@ class TestSetupLogger(unittest.TestCase): - def setUp(self): self.save_dir = tempfile.mkdtemp() self.save_file = os.path.join(self.save_dir, 'logger.txt') diff --git a/python/paddle/tests/test_metrics.py b/python/paddle/tests/test_metrics.py index 61b20251e0486..b3bfb1eb9c842 100644 --- a/python/paddle/tests/test_metrics.py +++ b/python/paddle/tests/test_metrics.py @@ -27,7 +27,7 @@ def one_hot(x, n_class): return res -def accuracy(pred, label, topk=(1, )): +def accuracy(pred, label, topk=(1,)): maxk = max(topk) pred = np.argsort(pred)[..., ::-1][..., :maxk] if len(label.shape) == 1: @@ -35,7 +35,7 @@ def accuracy(pred, label, topk=(1, )): elif label.shape[-1] != 1: label = np.argmax(label, axis=-1) label = label[..., np.newaxis] - correct = (pred == np.repeat(label, maxk, -1)) + correct = pred == np.repeat(label, maxk, -1) total = np.prod(np.array(label.shape[:-1])) @@ -50,16 +50,22 @@ def convert_to_one_hot(y, C): oh = np.random.choice(np.arange(C), C, replace=False).astype('float32') / C oh = np.tile(oh[np.newaxis, :], (y.shape[0], 1)) for i in range(y.shape[0]): - oh[i, int(y[i])] = 1. + oh[i, int(y[i])] = 1.0 return oh class TestAccuracy(unittest.TestCase): - def test_acc(self, squeeze_y=False): x = paddle.to_tensor( - np.array([[0.1, 0.2, 0.3, 0.4], [0.1, 0.4, 0.3, 0.2], - [0.1, 0.2, 0.4, 0.3], [0.1, 0.2, 0.3, 0.4]])) + np.array( + [ + [0.1, 0.2, 0.3, 0.4], + [0.1, 0.4, 0.3, 0.2], + [0.1, 0.2, 0.4, 0.3], + [0.1, 0.2, 0.3, 0.4], + ] + ) + ) y = np.array([[0], [1], [2], [3]]) if squeeze_y: @@ -79,8 +85,15 @@ def test_acc(self, squeeze_y=False): self.assertEqual(m.accumulate(), 0.75) x = paddle.to_tensor( - np.array([[0.1, 0.2, 0.3, 0.4], [0.1, 0.3, 0.4, 0.2], - [0.1, 0.2, 0.4, 0.3], [0.1, 0.2, 0.3, 0.4]])) + np.array( + [ + [0.1, 0.2, 0.3, 0.4], + [0.1, 0.3, 0.4, 0.2], + [0.1, 0.2, 0.4, 0.3], + [0.1, 0.2, 0.3, 0.4], + ] + ) + ) y = paddle.to_tensor(np.array([[0], [1], [2], [3]])) correct = m.compute(x, y) # check results @@ -95,7 +108,7 @@ def test_acc(self, squeeze_y=False): def test_1d_label(self): self.test_acc(True) - def compare(self, x_np, y_np, k=(1, )): + def compare(self, x_np, y_np, k=(1,)): x = paddle.to_tensor(x_np) y = paddle.to_tensor(y_np) @@ -123,19 +136,20 @@ def test_one_hot(self): class TestAccuracyDynamic(unittest.TestCase): - def setUp(self): - self.topk = (1, ) + self.topk = (1,) self.class_num = 5 self.sample_num = 1000 self.name = None self.squeeze_label = False def random_pred_label(self): - label = np.random.randint(0, self.class_num, - (self.sample_num, 1)).astype('int64') - pred = np.random.randint(0, self.class_num, - (self.sample_num, 1)).astype('int32') + label = np.random.randint( + 0, self.class_num, (self.sample_num, 1) + ).astype('int64') + pred = np.random.randint( + 0, self.class_num, (self.sample_num, 1) + ).astype('int32') if self.squeeze_label: label = label.squeeze() pred_one_hot = convert_to_one_hot(pred, self.class_num) @@ -154,16 +168,19 @@ def test_main(self): acc.update(*[s.numpy() for s in state]) res_m = acc.accumulate() res_f = accuracy(pred, label, self.topk) - assert np.all(np.isclose(np.array(res_m, dtype='float64'), - np.array(res_f, dtype='float64'), rtol=1e-3)), \ - "Accuracy precision error: {} != {}".format(res_m, res_f) + assert np.all( + np.isclose( + np.array(res_m, dtype='float64'), + np.array(res_f, dtype='float64'), + rtol=1e-3, + ) + ), "Accuracy precision error: {} != {}".format(res_m, res_f) acc.reset() assert np.sum(acc.total) == 0 assert np.sum(acc.count) == 0 class TestAccuracyDynamicMultiTopk(TestAccuracyDynamic): - def setUp(self): self.topk = (1, 5) self.class_num = 10 @@ -173,9 +190,8 @@ def setUp(self): class TestAccuracyStatic(TestAccuracyDynamic): - def setUp(self): - self.topk = (1, ) + self.topk = (1,) self.class_num = 5 self.sample_num = 1000 self.name = None @@ -189,9 +205,9 @@ def test_main(self): main_prog.random_seed = 1024 startup_prog.random_seed = 1024 with fluid.program_guard(main_prog, startup_prog): - pred = fluid.data(name='pred', - shape=[None, self.class_num], - dtype='float32') + pred = fluid.data( + name='pred', shape=[None, self.class_num], dtype='float32' + ) label = fluid.data(name='label', shape=[None, 1], dtype='int64') acc = paddle.metric.Accuracy(topk=self.topk, name=self.name) state = acc.compute(pred, label) @@ -201,18 +217,18 @@ def test_main(self): for _ in range(10): label, pred = self.random_pred_label() - state_ret = exe.run(compiled_main_prog, - feed={ - 'pred': pred, - 'label': label - }, - fetch_list=[s.name for s in to_list(state)], - return_numpy=True) + state_ret = exe.run( + compiled_main_prog, + feed={'pred': pred, 'label': label}, + fetch_list=[s.name for s in to_list(state)], + return_numpy=True, + ) acc.update(*state_ret) res_m = acc.accumulate() res_f = accuracy(pred, label, self.topk) - assert np.all(np.isclose(np.array(res_m), np.array(res_f), rtol=1e-3)), \ - "Accuracy precision error: {} != {}".format(res_m, res_f) + assert np.all( + np.isclose(np.array(res_m), np.array(res_f), rtol=1e-3) + ), "Accuracy precision error: {} != {}".format(res_m, res_f) acc.reset() assert np.sum(acc.total) == 0 assert np.sum(acc.count) == 0 @@ -221,7 +237,6 @@ def test_main(self): class TestAccuracyStaticMultiTopk(TestAccuracyStatic): - def setUp(self): self.topk = (1, 5) self.class_num = 10 @@ -231,7 +246,6 @@ def setUp(self): class TestPrecision(unittest.TestCase): - def test_1d(self): x = np.array([0.1, 0.5, 0.6, 0.7]) @@ -240,13 +254,13 @@ def test_1d(self): m = paddle.metric.Precision() m.update(x, y) r = m.accumulate() - self.assertAlmostEqual(r, 2. / 3.) + self.assertAlmostEqual(r, 2.0 / 3.0) x = paddle.to_tensor(np.array([0.1, 0.5, 0.6, 0.7, 0.2])) y = paddle.to_tensor(np.array([1, 0, 1, 1, 1])) m.update(x, y) r = m.accumulate() - self.assertAlmostEqual(r, 4. / 6.) + self.assertAlmostEqual(r, 4.0 / 6.0) def test_2d(self): x = np.array([0.1, 0.5, 0.6, 0.7]).reshape(-1, 1) @@ -255,13 +269,13 @@ def test_2d(self): m = paddle.metric.Precision() m.update(x, y) r = m.accumulate() - self.assertAlmostEqual(r, 2. / 3.) + self.assertAlmostEqual(r, 2.0 / 3.0) x = np.array([0.1, 0.5, 0.6, 0.7, 0.2]).reshape(-1, 1) y = np.array([1, 0, 1, 1, 1]).reshape(-1, 1) m.update(x, y) r = m.accumulate() - self.assertAlmostEqual(r, 4. / 6.) + self.assertAlmostEqual(r, 4.0 / 6.0) # check reset m.reset() @@ -271,7 +285,6 @@ def test_2d(self): class TestRecall(unittest.TestCase): - def test_1d(self): x = np.array([0.1, 0.5, 0.6, 0.7]) y = np.array([1, 0, 1, 1]) @@ -279,13 +292,13 @@ def test_1d(self): m = paddle.metric.Recall() m.update(x, y) r = m.accumulate() - self.assertAlmostEqual(r, 2. / 3.) + self.assertAlmostEqual(r, 2.0 / 3.0) x = paddle.to_tensor(np.array([0.1, 0.5, 0.6, 0.7])) y = paddle.to_tensor(np.array([1, 0, 0, 1])) m.update(x, y) r = m.accumulate() - self.assertAlmostEqual(r, 3. / 5.) + self.assertAlmostEqual(r, 3.0 / 5.0) # check reset m.reset() @@ -295,10 +308,19 @@ def test_1d(self): class TestAuc(unittest.TestCase): - def test_auc_numpy(self): - x = np.array([[0.78, 0.22], [0.62, 0.38], [0.55, 0.45], [0.30, 0.70], - [0.14, 0.86], [0.59, 0.41], [0.91, 0.08], [0.16, 0.84]]) + x = np.array( + [ + [0.78, 0.22], + [0.62, 0.38], + [0.55, 0.45], + [0.30, 0.70], + [0.14, 0.86], + [0.59, 0.41], + [0.91, 0.08], + [0.16, 0.84], + ] + ) y = np.array([[0], [1], [1], [0], [1], [0], [0], [1]]) m = paddle.metric.Auc() m.update(x, y) @@ -310,8 +332,19 @@ def test_auc_numpy(self): def test_auc_tensor(self): x = paddle.to_tensor( - np.array([[0.78, 0.22], [0.62, 0.38], [0.55, 0.45], [0.30, 0.70], - [0.14, 0.86], [0.59, 0.41], [0.91, 0.08], [0.16, 0.84]])) + np.array( + [ + [0.78, 0.22], + [0.62, 0.38], + [0.55, 0.45], + [0.30, 0.70], + [0.14, 0.86], + [0.59, 0.41], + [0.91, 0.08], + [0.16, 0.84], + ] + ) + ) y = paddle.to_tensor(np.array([[0], [1], [1], [0], [1], [0], [0], [1]])) m = paddle.metric.Auc() m.update(x, y) diff --git a/python/paddle/tests/test_model.py b/python/paddle/tests/test_model.py index 5bb47b0ddfb32..212939932cb3e 100644 --- a/python/paddle/tests/test_model.py +++ b/python/paddle/tests/test_model.py @@ -34,23 +34,28 @@ import paddle.fluid.dygraph.jit as jit from paddle.io import DistributedBatchSampler, Dataset from paddle.hapi.model import prepare_distributed_context -from paddle.fluid.dygraph.dygraph_to_static.program_translator import ProgramTranslator +from paddle.fluid.dygraph.dygraph_to_static.program_translator import ( + ProgramTranslator, +) class LeNetDygraph(paddle.nn.Layer): - def __init__(self, num_classes=10): super(LeNetDygraph, self).__init__() self.num_classes = num_classes - self.features = Sequential(Conv2D(1, 6, 3, stride=1, padding=1), ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2), - Conv2D(6, 16, 5, stride=1, padding=0), - ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2)) + self.features = Sequential( + Conv2D(1, 6, 3, stride=1, padding=1), + ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + Conv2D(6, 16, 5, stride=1, padding=0), + ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + ) if num_classes > 0: - self.fc = Sequential(Linear(400, 120), Linear(120, 84), - Linear(84, 10)) + self.fc = Sequential( + Linear(400, 120), Linear(120, 84), Linear(84, 10) + ) def forward(self, inputs): x = self.features(inputs) @@ -62,7 +67,6 @@ def forward(self, inputs): class ModelInner(paddle.nn.Layer): - def __init__(self): super(ModelInner, self).__init__() self.fc = paddle.nn.Linear(3, 4) @@ -73,7 +77,6 @@ def forward(self, x): class ModelOutter(paddle.nn.Layer): - def __init__(self): super(ModelOutter, self).__init__() self.module1 = ModelInner() @@ -86,22 +89,25 @@ def forward(self, x): class LeNetListInput(paddle.nn.Layer): - def __init__(self, num_classes=10): super(LeNetListInput, self).__init__() self.num_classes = num_classes self.cov = Conv2D(1, 6, 3, stride=1, padding=1) for param in self.cov.parameters(): param.trainable = False - self.features = Sequential(self.cov, ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2), - Conv2D(6, 16, 5, stride=1, padding=0), - ReLU(), - paddle.fluid.dygraph.Pool2D(2, 'max', 2)) + self.features = Sequential( + self.cov, + ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + Conv2D(6, 16, 5, stride=1, padding=0), + ReLU(), + paddle.fluid.dygraph.Pool2D(2, 'max', 2), + ) if num_classes > 0: - self.fc = Sequential(Linear(400, 120), Linear(120, 84), - Linear(84, 10)) + self.fc = Sequential( + Linear(400, 120), Linear(120, 84), Linear(84, 10) + ) def forward(self, inputs): x = inputs[0] @@ -114,7 +120,6 @@ def forward(self, inputs): class LeNetDictInput(LeNetDygraph): - def forward(self, inputs): x = self.features(inputs['x1']) @@ -125,7 +130,6 @@ def forward(self, inputs): class MnistDataset(MNIST): - def __init__(self, mode, return_label=True, sample_num=None): super(MnistDataset, self).__init__(mode=mode) self.return_label = return_label @@ -138,7 +142,7 @@ def __getitem__(self, idx): img = np.reshape(img, [1, 28, 28]) if self.return_label: return img, np.array(self.labels[idx]).astype('int64') - return img, + return (img,) def __len__(self): return len(self.images) @@ -152,8 +156,9 @@ def compute_acc(pred, label): def dynamic_train(model, dataloader): - optim = fluid.optimizer.Adam(learning_rate=0.001, - parameter_list=model.parameters()) + optim = fluid.optimizer.Adam( + learning_rate=0.001, parameter_list=model.parameters() + ) model.train() for inputs, labels in dataloader: outputs = model(inputs) @@ -171,17 +176,22 @@ def dynamic_evaluate(model, dataloader): for inputs, labels in dataloader: outputs = model(inputs) - cnt += (np.argmax( - outputs.numpy(), - -1)[:, np.newaxis] == labels.numpy()).astype('int').sum() + cnt += ( + ( + np.argmax(outputs.numpy(), -1)[:, np.newaxis] + == labels.numpy() + ) + .astype('int') + .sum() + ) return cnt / len(dataloader.dataset) -@unittest.skipIf(not fluid.is_compiled_with_cuda(), - 'CPU testing is not supported') +@unittest.skipIf( + not fluid.is_compiled_with_cuda(), 'CPU testing is not supported' +) class TestModel(unittest.TestCase): - @classmethod def setUpClass(cls): if not fluid.is_compiled_with_cuda(): @@ -192,19 +202,19 @@ def setUpClass(cls): sp_num = 1280 cls.train_dataset = MnistDataset(mode='train', sample_num=sp_num) cls.val_dataset = MnistDataset(mode='test', sample_num=sp_num) - cls.test_dataset = MnistDataset(mode='test', - return_label=False, - sample_num=sp_num) - - cls.train_loader = fluid.io.DataLoader(cls.train_dataset, - places=cls.device, - batch_size=64) - cls.val_loader = fluid.io.DataLoader(cls.val_dataset, - places=cls.device, - batch_size=64) - cls.test_loader = fluid.io.DataLoader(cls.test_dataset, - places=cls.device, - batch_size=64) + cls.test_dataset = MnistDataset( + mode='test', return_label=False, sample_num=sp_num + ) + + cls.train_loader = fluid.io.DataLoader( + cls.train_dataset, places=cls.device, batch_size=64 + ) + cls.val_loader = fluid.io.DataLoader( + cls.val_dataset, places=cls.device, batch_size=64 + ) + cls.test_loader = fluid.io.DataLoader( + cls.test_dataset, places=cls.device, batch_size=64 + ) seed = 333 paddle.seed(seed) @@ -277,46 +287,59 @@ def fit(self, dynamic, num_replicas=None, rank=None, num_iters=None): paddle.framework.random._manual_program_seed(seed) net = LeNet() - optim_new = fluid.optimizer.Adam(learning_rate=0.001, - parameter_list=net.parameters()) + optim_new = fluid.optimizer.Adam( + learning_rate=0.001, parameter_list=net.parameters() + ) model = Model(net, inputs=self.inputs, labels=self.labels) - model.prepare(optim_new, - loss=CrossEntropyLoss(reduction="sum"), - metrics=Accuracy()) + model.prepare( + optim_new, + loss=CrossEntropyLoss(reduction="sum"), + metrics=Accuracy(), + ) model.fit(self.train_dataset, batch_size=64, shuffle=False) result = model.evaluate(self.val_dataset, batch_size=64) np.testing.assert_allclose(result['acc'], self.acc1) - model.fit(self.train_dataset, - batch_size=64, - shuffle=False, - num_iters=num_iters) - - result = model.evaluate(self.val_dataset, - batch_size=64, - num_iters=num_iters) - - train_sampler = DistributedBatchSampler(self.train_dataset, - batch_size=64, - shuffle=False, - num_replicas=num_replicas, - rank=rank) - val_sampler = DistributedBatchSampler(self.val_dataset, - batch_size=64, - shuffle=False, - num_replicas=num_replicas, - rank=rank) - - train_loader = fluid.io.DataLoader(self.train_dataset, - batch_sampler=train_sampler, - places=self.device, - return_list=True) - - val_loader = fluid.io.DataLoader(self.val_dataset, - batch_sampler=val_sampler, - places=self.device, - return_list=True) + model.fit( + self.train_dataset, + batch_size=64, + shuffle=False, + num_iters=num_iters, + ) + + result = model.evaluate( + self.val_dataset, batch_size=64, num_iters=num_iters + ) + + train_sampler = DistributedBatchSampler( + self.train_dataset, + batch_size=64, + shuffle=False, + num_replicas=num_replicas, + rank=rank, + ) + val_sampler = DistributedBatchSampler( + self.val_dataset, + batch_size=64, + shuffle=False, + num_replicas=num_replicas, + rank=rank, + ) + + train_loader = fluid.io.DataLoader( + self.train_dataset, + batch_sampler=train_sampler, + places=self.device, + return_list=True, + ) + + val_loader = fluid.io.DataLoader( + self.val_dataset, + batch_sampler=val_sampler, + places=self.device, + return_list=True, + ) model.fit(train_loader, val_loader) fluid.disable_dygraph() if dynamic else None @@ -328,37 +351,48 @@ def fit_with_tuple_input(self, dynamic, num_replicas=None, rank=None): paddle.framework.random._manual_program_seed(seed) net = LeNet() - optim_new = fluid.optimizer.Adam(learning_rate=0.001, - parameter_list=net.parameters()) + optim_new = fluid.optimizer.Adam( + learning_rate=0.001, parameter_list=net.parameters() + ) model = Model(net, inputs=tuple(self.inputs), labels=tuple(self.labels)) - model.prepare(optim_new, - loss=CrossEntropyLoss(reduction="sum"), - metrics=Accuracy()) + model.prepare( + optim_new, + loss=CrossEntropyLoss(reduction="sum"), + metrics=Accuracy(), + ) model.fit(self.train_dataset, batch_size=64, shuffle=False) result = model.evaluate(self.val_dataset, batch_size=64) np.testing.assert_allclose(result['acc'], self.acc1) - train_sampler = DistributedBatchSampler(self.train_dataset, - batch_size=64, - shuffle=False, - num_replicas=num_replicas, - rank=rank) - val_sampler = DistributedBatchSampler(self.val_dataset, - batch_size=64, - shuffle=False, - num_replicas=num_replicas, - rank=rank) - - train_loader = fluid.io.DataLoader(self.train_dataset, - batch_sampler=train_sampler, - places=self.device, - return_list=True) - - val_loader = fluid.io.DataLoader(self.val_dataset, - batch_sampler=val_sampler, - places=self.device, - return_list=True) + train_sampler = DistributedBatchSampler( + self.train_dataset, + batch_size=64, + shuffle=False, + num_replicas=num_replicas, + rank=rank, + ) + val_sampler = DistributedBatchSampler( + self.val_dataset, + batch_size=64, + shuffle=False, + num_replicas=num_replicas, + rank=rank, + ) + + train_loader = fluid.io.DataLoader( + self.train_dataset, + batch_sampler=train_sampler, + places=self.device, + return_list=True, + ) + + val_loader = fluid.io.DataLoader( + self.val_dataset, + batch_sampler=val_sampler, + places=self.device, + return_list=True, + ) model.fit(train_loader, val_loader) fluid.disable_dygraph() if dynamic else None @@ -371,14 +405,16 @@ def evaluate(self, dynamic): result = model.evaluate(self.val_dataset, batch_size=64) np.testing.assert_allclose(result['acc'], self.acc1) - sampler = DistributedBatchSampler(self.val_dataset, - batch_size=64, - shuffle=False) + sampler = DistributedBatchSampler( + self.val_dataset, batch_size=64, shuffle=False + ) - val_loader = fluid.io.DataLoader(self.val_dataset, - batch_sampler=sampler, - places=self.device, - return_list=True) + val_loader = fluid.io.DataLoader( + self.val_dataset, + batch_sampler=sampler, + places=self.device, + return_list=True, + ) model.evaluate(val_loader) @@ -389,22 +425,24 @@ def predict(self, dynamic): model = Model(LeNet(), self.inputs) model.prepare() model.load(self.weight_path) - output = model.predict(self.test_dataset, - batch_size=64, - stack_outputs=True) + output = model.predict( + self.test_dataset, batch_size=64, stack_outputs=True + ) np.testing.assert_equal(output[0].shape[0], len(self.test_dataset)) acc = compute_acc(output[0], self.val_dataset.labels) np.testing.assert_allclose(acc, self.acc1) - sampler = DistributedBatchSampler(self.test_dataset, - batch_size=64, - shuffle=False) + sampler = DistributedBatchSampler( + self.test_dataset, batch_size=64, shuffle=False + ) - test_loader = fluid.io.DataLoader(self.test_dataset, - batch_sampler=sampler, - places=self.device, - return_list=True) + test_loader = fluid.io.DataLoader( + self.test_dataset, + batch_sampler=sampler, + places=self.device, + return_list=True, + ) model.evaluate(test_loader) @@ -416,21 +454,21 @@ def test_predict_without_inputs(self): model.prepare() model.load(self.weight_path) model._inputs = None - output = model.predict(self.test_dataset, - batch_size=64, - stack_outputs=True) + output = model.predict( + self.test_dataset, batch_size=64, stack_outputs=True + ) np.testing.assert_equal(output[0].shape[0], len(self.test_dataset)) fluid.disable_dygraph() def test_summary_gpu(self): paddle.disable_static(self.device) rnn = paddle.nn.LSTM(16, 32, 2) - params_info = paddle.summary(rnn, [(-1, 23, 16), - ((2, None, 32), (2, -1, 32))]) + params_info = paddle.summary( + rnn, [(-1, 23, 16), ((2, None, 32), (2, -1, 32))] + ) class MyModel(paddle.nn.Layer): - def __init__(self): super(MyModel, self).__init__() self._fc = Linear(20, 10) @@ -441,17 +479,16 @@ def forward(self, x): class MyDataset(Dataset): - def __getitem__(self, idx): - return np.random.random(size=(20,)).astype(np.float32), \ - np.random.randint(0, 10, size=(1,)).astype(np.int64) + return np.random.random(size=(20,)).astype( + np.float32 + ), np.random.randint(0, 10, size=(1,)).astype(np.int64) def __len__(self): return 40 class TestModelFunction(unittest.TestCase): - def set_seed(self, seed=1024): paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) @@ -465,8 +502,9 @@ def get_expect(): fluid.enable_dygraph(fluid.CPUPlace()) self.set_seed() m = MyModel() - optim = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=m.parameters()) + optim = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=m.parameters() + ) m.train() output = m(to_tensor(data)) loss = CrossEntropyLoss(reduction='sum')(output, to_tensor(label)) @@ -484,14 +522,15 @@ def get_expect(): self.set_seed() net = MyModel() - optim2 = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=net.parameters()) + optim2 = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=net.parameters() + ) inputs = [InputSpec([None, dim], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] model = Model(net, inputs, labels) model.prepare(optim2, loss=CrossEntropyLoss(reduction="sum")) - loss, = model.train_batch([data], [label]) + (loss,) = model.train_batch([data], [label]) np.testing.assert_allclose(loss.flatten(), ref.flatten()) fluid.disable_dygraph() if dynamic else None @@ -517,7 +556,7 @@ def get_expect(): inputs = [InputSpec([None, dim], 'float32', 'x')] model = Model(net, inputs) model.prepare() - out, = model.predict_batch([data]) + (out,) = model.predict_batch([data]) np.testing.assert_allclose(out, ref, rtol=1e-6) fluid.disable_dygraph() if dynamic else None @@ -532,11 +571,13 @@ def test_save_load(self): net = MyModel() inputs = [InputSpec([None, 20], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] - optim = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=net.parameters()) + optim = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=net.parameters() + ) model = Model(net, inputs, labels) - model.prepare(optimizer=optim, - loss=CrossEntropyLoss(reduction="sum")) + model.prepare( + optimizer=optim, loss=CrossEntropyLoss(reduction="sum") + ) model.save(path) model.load(path) fluid.disable_dygraph() if dynamic else None @@ -555,14 +596,17 @@ def test_dynamic_load(self): inputs = [InputSpec([None, 1, 28, 28], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] if new_optimizer: - optim = paddle.optimizer.Adam(learning_rate=0.001, - parameters=net.parameters()) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameters=net.parameters() + ) else: - optim = fluid.optimizer.Adam(learning_rate=0.001, - parameter_list=net.parameters()) + optim = fluid.optimizer.Adam( + learning_rate=0.001, parameter_list=net.parameters() + ) model = Model(net, inputs, labels) - model.prepare(optimizer=optim, - loss=CrossEntropyLoss(reduction="sum")) + model.prepare( + optimizer=optim, loss=CrossEntropyLoss(reduction="sum") + ) model.fit(mnist_data, batch_size=64, verbose=0) model.save(path) model.load(path) @@ -570,16 +614,18 @@ def test_dynamic_load(self): shutil.rmtree(path) def test_dynamic_save_static_load(self): - path = os.path.join(tempfile.mkdtemp(), - '.cache_dynamic_save_static_load') + path = os.path.join( + tempfile.mkdtemp(), '.cache_dynamic_save_static_load' + ) if not os.path.exists(path): os.makedirs(path) # dynamic saving device = paddle.set_device('cpu') fluid.enable_dygraph(device) model = Model(MyModel()) - optim = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=model.parameters()) + optim = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=model.parameters() + ) model.prepare(optimizer=optim, loss=CrossEntropyLoss(reduction="sum")) model.save(path) fluid.disable_dygraph() @@ -587,34 +633,38 @@ def test_dynamic_save_static_load(self): inputs = [InputSpec([None, 20], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] model = Model(MyModel(), inputs, labels) - optim = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=model.parameters()) + optim = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=model.parameters() + ) model.prepare(optimizer=optim, loss=CrossEntropyLoss(reduction="sum")) model.load(path) shutil.rmtree(path) def test_static_save_dynamic_load(self): - path = os.path.join(tempfile.mkdtemp(), - '.cache_test_static_save_dynamic_load') + path = os.path.join( + tempfile.mkdtemp(), '.cache_test_static_save_dynamic_load' + ) if not os.path.exists(path): os.makedirs(path) net = MyModel() inputs = [InputSpec([None, 20], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] - optim = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=net.parameters()) + optim = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=net.parameters() + ) model = Model(net, inputs, labels) model.prepare(optimizer=optim, loss=CrossEntropyLoss(reduction="sum")) model.save(path) device = paddle.set_device('cpu') - fluid.enable_dygraph(device) #if dynamic else None + fluid.enable_dygraph(device) # if dynamic else None net = MyModel() inputs = [InputSpec([None, 20], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] - optim = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=net.parameters()) + optim = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=net.parameters() + ) model = Model(net, inputs, labels) model.prepare(optimizer=optim, loss=CrossEntropyLoss(reduction="sum")) model.load(path) @@ -635,7 +685,6 @@ def test_parameters(self): fluid.disable_dygraph() if dynamic else None def test_summary(self): - def _get_param_from_state_dict(state_dict): params = 0 for k, v in state_dict.items(): @@ -663,22 +712,21 @@ def test_summary_non_tensor(self): paddle.summary(ModelOutter(), input_size=(-1, 3)) def test_summary_nlp(self): - def _get_param_from_state_dict(state_dict): params = 0 for k, v in state_dict.items(): params += np.prod(v.numpy().shape) return params - nlp_net = paddle.nn.GRU(input_size=2, - hidden_size=3, - num_layers=3, - direction="bidirectional") + nlp_net = paddle.nn.GRU( + input_size=2, hidden_size=3, num_layers=3, direction="bidirectional" + ) paddle.summary(nlp_net, (1, 1, 2)) rnn = paddle.nn.LSTM(16, 32, 2) - params_info = paddle.summary(rnn, [(-1, 23, 16), - ((2, None, 32), (2, -1, 32))]) + params_info = paddle.summary( + rnn, [(-1, 23, 16), ((2, None, 32), (2, -1, 32))] + ) gt_params = _get_param_from_state_dict(rnn.state_dict()) np.testing.assert_allclose(params_info['total_params'], gt_params / 2.0) @@ -710,7 +758,7 @@ def test_summary_input(self): lenet_dict_input = LeNetDictInput() input_data = { 'x1': paddle.rand([1, 1, 28, 28]), - 'x2': paddle.rand([1, 400]) + 'x2': paddle.rand([1, 400]), } paddle.summary(lenet_dict_input, input=input_data) @@ -747,29 +795,35 @@ def test_dynamic_flops(self): def customize_dropout(m, x, y): m.total_ops += 0 - paddle.flops(net, [1, 3, 224, 224], - custom_ops={paddle.nn.Dropout: customize_dropout}, - print_detail=True) + paddle.flops( + net, + [1, 3, 224, 224], + custom_ops={paddle.nn.Dropout: customize_dropout}, + print_detail=True, + ) def test_dynamic_flops_with_multiple_outputs(self): - net = paddle.nn.MaxPool2D(kernel_size=2, - stride=2, - padding=0, - return_mask=True) + net = paddle.nn.MaxPool2D( + kernel_size=2, stride=2, padding=0, return_mask=True + ) def customize_dropout(m, x, y): m.total_ops += 0 - paddle.flops(net, [1, 2, 32, 32], - custom_ops={paddle.nn.Dropout: customize_dropout}, - print_detail=True) + paddle.flops( + net, + [1, 2, 32, 32], + custom_ops={paddle.nn.Dropout: customize_dropout}, + print_detail=True, + ) def test_export_deploy_model(self): self.set_seed() np.random.seed(201) - save_dir = os.path.join(tempfile.mkdtemp(), - '.cache_test_export_deploy_model') + save_dir = os.path.join( + tempfile.mkdtemp(), '.cache_test_export_deploy_model' + ) if not os.path.exists(save_dir): os.makedirs(save_dir) @@ -782,28 +836,37 @@ def test_export_deploy_model(self): model = Model(net, inputs) model.prepare() - tensor_img = np.array(np.random.random((1, 1, 28, 28)), - dtype=np.float32) + tensor_img = np.array( + np.random.random((1, 1, 28, 28)), dtype=np.float32 + ) model.save(save_dir, training=False) ori_results = model.predict_batch(tensor_img) fluid.disable_dygraph() if dynamic else None - place = fluid.CPUPlace( - ) if not fluid.is_compiled_with_cuda() else fluid.CUDAPlace(0) + place = ( + fluid.CPUPlace() + if not fluid.is_compiled_with_cuda() + else fluid.CUDAPlace(0) + ) new_scope = fluid.Scope() with fluid.scope_guard(new_scope): exe = fluid.Executor(place) - [inference_program, feed_target_names, - fetch_targets] = (paddle.static.io.load_inference_model( - path_prefix=save_dir, executor=exe)) - results = exe.run(inference_program, - feed={feed_target_names[0]: tensor_img}, - fetch_list=fetch_targets) - np.testing.assert_allclose(results, - ori_results, - rtol=1e-5, - atol=1e-6) + [ + inference_program, + feed_target_names, + fetch_targets, + ] = paddle.static.io.load_inference_model( + path_prefix=save_dir, executor=exe + ) + results = exe.run( + inference_program, + feed={feed_target_names[0]: tensor_img}, + fetch_list=fetch_targets, + ) + np.testing.assert_allclose( + results, ori_results, rtol=1e-5, atol=1e-6 + ) paddle.enable_static() @@ -815,22 +878,26 @@ def test_dygraph_export_deploy_model_about_inputs(self): mnist_data = MnistDataset(mode='train') paddle.disable_static() # without inputs - save_dir = os.path.join(tempfile.mkdtemp(), - '.cache_test_dygraph_export_deploy') + save_dir = os.path.join( + tempfile.mkdtemp(), '.cache_test_dygraph_export_deploy' + ) if not os.path.exists(save_dir): os.makedirs(save_dir) for initial in ["fit", "train_batch", "eval_batch", "predict_batch"]: net = LeNet() model = Model(net) - optim = fluid.optimizer.Adam(learning_rate=0.001, - parameter_list=model.parameters()) - model.prepare(optimizer=optim, - loss=CrossEntropyLoss(reduction="sum")) + optim = fluid.optimizer.Adam( + learning_rate=0.001, parameter_list=model.parameters() + ) + model.prepare( + optimizer=optim, loss=CrossEntropyLoss(reduction="sum") + ) if initial == "fit": model.fit(mnist_data, batch_size=64, verbose=0) else: - img = np.array(np.random.random((1, 1, 28, 28)), - dtype=np.float32) + img = np.array( + np.random.random((1, 1, 28, 28)), dtype=np.float32 + ) label = np.array(np.random.rand(1, 1), dtype=np.int64) if initial == "train_batch": model.train_batch([img], [label]) @@ -842,52 +909,58 @@ def test_dygraph_export_deploy_model_about_inputs(self): model.save(save_dir, training=False) shutil.rmtree(save_dir) # with inputs, and the type of inputs is InputSpec - save_dir = os.path.join(tempfile.mkdtemp(), - '.cache_test_dygraph_export_deploy_2') + save_dir = os.path.join( + tempfile.mkdtemp(), '.cache_test_dygraph_export_deploy_2' + ) if not os.path.exists(save_dir): os.makedirs(save_dir) net = LeNet() inputs = InputSpec([None, 1, 28, 28], 'float32', 'x') model = Model(net, inputs) - optim = fluid.optimizer.Adam(learning_rate=0.001, - parameter_list=model.parameters()) + optim = fluid.optimizer.Adam( + learning_rate=0.001, parameter_list=model.parameters() + ) model.prepare(optimizer=optim, loss=CrossEntropyLoss(reduction="sum")) model.save(save_dir, training=False) shutil.rmtree(save_dir) - def test_accumulate(self, ): + def test_accumulate( + self, + ): dim = 20 data = np.random.random(size=(4, dim)).astype(np.float32) label = np.random.randint(0, 10, size=(4, 1)).astype(np.int64) net = MyModel() - optim = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=net.parameters()) + optim = fluid.optimizer.SGD( + learning_rate=0.001, parameter_list=net.parameters() + ) inputs = [InputSpec([None, dim], 'float32', 'x')] labels = [InputSpec([None, 1], 'int64', 'label')] for amp_cfg in [None, 'O1']: model = Model(net, inputs, labels) - model.prepare(optim, - loss=CrossEntropyLoss(reduction="sum"), - amp_configs=amp_cfg) + model.prepare( + optim, + loss=CrossEntropyLoss(reduction="sum"), + amp_configs=amp_cfg, + ) losses, grads = [], [] for stat in [False, False, True]: - loss, = model.train_batch([data], [label], update=stat) + (loss,) = model.train_batch([data], [label], update=stat) losses.append(loss) grads.append([p.grad.numpy() for p in net.parameters()]) for grad1, grad2, grad3 in zip(*grads): np.testing.assert_almost_equal(grad1 * 2, grad2, decimal=4) - np.testing.assert_almost_equal(grad3, - np.zeros_like(grad3), - decimal=4) + np.testing.assert_almost_equal( + grad3, np.zeros_like(grad3), decimal=4 + ) np.testing.assert_almost_equal(losses[0], losses[1], decimal=4) np.testing.assert_almost_equal(losses[0], losses[2], decimal=4) class TestModelWithLRScheduler(unittest.TestCase): - def test_fit_by_step(self): base_lr = 1e-3 boundaries = [5, 8] @@ -897,17 +970,21 @@ def make_optimizer(parameters=None): weight_decay = 5e-4 values = [base_lr * (0.1**i) for i in range(len(boundaries) + 1)] learning_rate = paddle.optimizer.lr.PiecewiseDecay( - boundaries=boundaries, values=values) + boundaries=boundaries, values=values + ) learning_rate = paddle.optimizer.lr.LinearWarmup( learning_rate=learning_rate, warmup_steps=4, - start_lr=base_lr / 5., + start_lr=base_lr / 5.0, end_lr=base_lr, - verbose=True) - optimizer = paddle.optimizer.Momentum(learning_rate=learning_rate, - weight_decay=weight_decay, - momentum=momentum, - parameters=parameters) + verbose=True, + ) + optimizer = paddle.optimizer.Momentum( + learning_rate=learning_rate, + weight_decay=weight_decay, + momentum=momentum, + parameters=parameters, + ) return optimizer # dynamic test @@ -923,8 +1000,10 @@ def make_optimizer(parameters=None): dataset = MyDataset() model.fit(dataset, dataset, batch_size=4, epochs=10, num_workers=0) - np.testing.assert_allclose(model._optimizer._learning_rate.last_lr, - base_lr * (0.1**len(boundaries))) + np.testing.assert_allclose( + model._optimizer._learning_rate.last_lr, + base_lr * (0.1 ** len(boundaries)), + ) # static test paddle.enable_static() @@ -938,8 +1017,10 @@ def make_optimizer(parameters=None): dataset = MyDataset() model.fit(dataset, dataset, batch_size=4, epochs=10, num_workers=0) - np.testing.assert_allclose(model._optimizer._learning_rate.last_lr, - base_lr * (0.1**len(boundaries))) + np.testing.assert_allclose( + model._optimizer._learning_rate.last_lr, + base_lr * (0.1 ** len(boundaries)), + ) def test_fit_by_epoch(self): base_lr = 1e-3 @@ -952,17 +1033,21 @@ def make_optimizer(parameters=None): weight_decay = 5e-4 values = [base_lr * (0.1**i) for i in range(len(boundaries) + 1)] learning_rate = paddle.optimizer.lr.PiecewiseDecay( - boundaries=boundaries, values=values) + boundaries=boundaries, values=values + ) learning_rate = paddle.optimizer.lr.LinearWarmup( learning_rate=learning_rate, warmup_steps=wamup_epochs, - start_lr=base_lr / 5., + start_lr=base_lr / 5.0, end_lr=base_lr, - verbose=True) - optimizer = paddle.optimizer.Momentum(learning_rate=learning_rate, - weight_decay=weight_decay, - momentum=momentum, - parameters=parameters) + verbose=True, + ) + optimizer = paddle.optimizer.Momentum( + learning_rate=learning_rate, + weight_decay=weight_decay, + momentum=momentum, + parameters=parameters, + ) return optimizer # dynamic test @@ -977,23 +1062,27 @@ def make_optimizer(parameters=None): dataset = MyDataset() - lr_scheduler_callback = paddle.callbacks.LRScheduler(by_step=False, - by_epoch=True) + lr_scheduler_callback = paddle.callbacks.LRScheduler( + by_step=False, by_epoch=True + ) - model.fit(dataset, - dataset, - batch_size=4, - epochs=epochs, - num_workers=0, - callbacks=lr_scheduler_callback) + model.fit( + dataset, + dataset, + batch_size=4, + epochs=epochs, + num_workers=0, + callbacks=lr_scheduler_callback, + ) cnt = 0 for b in boundaries: if b + wamup_epochs <= epochs: cnt += 1 - np.testing.assert_allclose(model._optimizer._learning_rate.last_lr, - base_lr * (0.1**cnt)) + np.testing.assert_allclose( + model._optimizer._learning_rate.last_lr, base_lr * (0.1**cnt) + ) # static test paddle.enable_static() @@ -1006,27 +1095,30 @@ def make_optimizer(parameters=None): dataset = MyDataset() - lr_scheduler_callback = paddle.callbacks.LRScheduler(by_step=False, - by_epoch=True) + lr_scheduler_callback = paddle.callbacks.LRScheduler( + by_step=False, by_epoch=True + ) - model.fit(dataset, - dataset, - batch_size=4, - epochs=epochs, - num_workers=0, - callbacks=lr_scheduler_callback) + model.fit( + dataset, + dataset, + batch_size=4, + epochs=epochs, + num_workers=0, + callbacks=lr_scheduler_callback, + ) cnt = 0 for b in boundaries: if b + wamup_epochs <= epochs: cnt += 1 - np.testing.assert_allclose(model._optimizer._learning_rate.last_lr, - base_lr * (0.1**cnt)) + np.testing.assert_allclose( + model._optimizer._learning_rate.last_lr, base_lr * (0.1**cnt) + ) class TestRaiseError(unittest.TestCase): - def test_input_without_name(self): net = MyModel() inputs = [InputSpec([None, 10], 'float32')] @@ -1059,8 +1151,9 @@ def test_save_infer_model_without_file_prefix(self): model = Model(net, inputs) model.prepare() path = "" - tensor_img = np.array(np.random.random((1, 1, 28, 28)), - dtype=np.float32) + tensor_img = np.array( + np.random.random((1, 1, 28, 28)), dtype=np.float32 + ) with self.assertRaises(ValueError): model.save(path, training=False) diff --git a/python/paddle/tests/test_ops_roi_align.py b/python/paddle/tests/test_ops_roi_align.py index 145f77e846b57..05c221e83a157 100644 --- a/python/paddle/tests/test_ops_roi_align.py +++ b/python/paddle/tests/test_ops_roi_align.py @@ -20,7 +20,6 @@ class TestRoIAlign(unittest.TestCase): - def setUp(self): self.data = np.random.rand(1, 256, 32, 32).astype('float32') boxes = np.random.rand(3, 4) @@ -40,38 +39,40 @@ def roi_align_functional(self, output_size): boxes = paddle.to_tensor(self.boxes) boxes_num = paddle.to_tensor(self.boxes_num) - align_out = roi_align(data, - boxes, - boxes_num=boxes_num, - output_size=output_size) + align_out = roi_align( + data, boxes, boxes_num=boxes_num, output_size=output_size + ) np.testing.assert_equal(align_out.shape, output_shape) else: - data = paddle.static.data(shape=self.data.shape, - dtype=self.data.dtype, - name='data') - boxes = paddle.static.data(shape=self.boxes.shape, - dtype=self.boxes.dtype, - name='boxes') - boxes_num = paddle.static.data(shape=self.boxes_num.shape, - dtype=self.boxes_num.dtype, - name='boxes_num') - - align_out = roi_align(data, - boxes, - boxes_num=boxes_num, - output_size=output_size) + data = paddle.static.data( + shape=self.data.shape, dtype=self.data.dtype, name='data' + ) + boxes = paddle.static.data( + shape=self.boxes.shape, dtype=self.boxes.dtype, name='boxes' + ) + boxes_num = paddle.static.data( + shape=self.boxes_num.shape, + dtype=self.boxes_num.dtype, + name='boxes_num', + ) + + align_out = roi_align( + data, boxes, boxes_num=boxes_num, output_size=output_size + ) place = paddle.CPUPlace() exe = paddle.static.Executor(place) - align_out = exe.run(paddle.static.default_main_program(), - feed={ - 'data': self.data, - 'boxes': self.boxes, - 'boxes_num': self.boxes_num - }, - fetch_list=[align_out]) + align_out = exe.run( + paddle.static.default_main_program(), + feed={ + 'data': self.data, + 'boxes': self.boxes, + 'boxes_num': self.boxes_num, + }, + fetch_list=[align_out], + ) np.testing.assert_equal(align_out[0].shape, output_shape) @@ -93,13 +94,19 @@ def test_RoIAlign(self): align_out = roi_align_c(data, boxes, boxes_num) np.testing.assert_equal(align_out.shape, (3, 256, 4, 3)) - def test_value(self, ): - data = np.array([i for i in range(1, 17)]).reshape(1, 1, 4, - 4).astype(np.float32) - boxes = np.array([[1., 1., 2., 2.], [1.5, 1.5, 3., - 3.]]).astype(np.float32) + def test_value( + self, + ): + data = ( + np.array([i for i in range(1, 17)]) + .reshape(1, 1, 4, 4) + .astype(np.float32) + ) + boxes = np.array([[1.0, 1.0, 2.0, 2.0], [1.5, 1.5, 3.0, 3.0]]).astype( + np.float32 + ) boxes_num = np.array([2]).astype(np.int32) - output = np.array([[[[6.]]], [[[9.75]]]], dtype=np.float32) + output = np.array([[[[6.0]]], [[[9.75]]]], dtype=np.float32) data = paddle.to_tensor(data) boxes = paddle.to_tensor(boxes) diff --git a/python/paddle/tests/test_ops_roi_pool.py b/python/paddle/tests/test_ops_roi_pool.py index eaeb785df7d48..923ec01503cd2 100644 --- a/python/paddle/tests/test_ops_roi_pool.py +++ b/python/paddle/tests/test_ops_roi_pool.py @@ -20,7 +20,6 @@ class TestRoIPool(unittest.TestCase): - def setUp(self): self.data = np.random.rand(1, 256, 32, 32).astype('float32') boxes = np.random.rand(3, 4) @@ -41,38 +40,40 @@ def roi_pool_functional(self, output_size): boxes = paddle.to_tensor(self.boxes) boxes_num = paddle.to_tensor(self.boxes_num) - pool_out = roi_pool(data, - boxes, - boxes_num=boxes_num, - output_size=output_size) + pool_out = roi_pool( + data, boxes, boxes_num=boxes_num, output_size=output_size + ) np.testing.assert_equal(pool_out.shape, output_shape) else: - data = paddle.static.data(shape=self.data.shape, - dtype=self.data.dtype, - name='data') - boxes = paddle.static.data(shape=self.boxes.shape, - dtype=self.boxes.dtype, - name='boxes') - boxes_num = paddle.static.data(shape=self.boxes_num.shape, - dtype=self.boxes_num.dtype, - name='boxes_num') - - pool_out = roi_pool(data, - boxes, - boxes_num=boxes_num, - output_size=output_size) + data = paddle.static.data( + shape=self.data.shape, dtype=self.data.dtype, name='data' + ) + boxes = paddle.static.data( + shape=self.boxes.shape, dtype=self.boxes.dtype, name='boxes' + ) + boxes_num = paddle.static.data( + shape=self.boxes_num.shape, + dtype=self.boxes_num.dtype, + name='boxes_num', + ) + + pool_out = roi_pool( + data, boxes, boxes_num=boxes_num, output_size=output_size + ) place = paddle.CPUPlace() exe = paddle.static.Executor(place) - pool_out = exe.run(paddle.static.default_main_program(), - feed={ - 'data': self.data, - 'boxes': self.boxes, - 'boxes_num': self.boxes_num - }, - fetch_list=[pool_out]) + pool_out = exe.run( + paddle.static.default_main_program(), + feed={ + 'data': self.data, + 'boxes': self.boxes, + 'boxes_num': self.boxes_num, + }, + fetch_list=[pool_out], + ) np.testing.assert_equal(pool_out[0].shape, output_shape) @@ -94,13 +95,19 @@ def test_RoIPool(self): pool_out = roi_pool_c(data, boxes, boxes_num) np.testing.assert_equal(pool_out.shape, (3, 256, 4, 3)) - def test_value(self, ): - data = np.array([i for i in range(1, 17)]).reshape(1, 1, 4, - 4).astype(np.float32) - boxes = np.array([[1., 1., 2., 2.], [1.5, 1.5, 3., - 3.]]).astype(np.float32) + def test_value( + self, + ): + data = ( + np.array([i for i in range(1, 17)]) + .reshape(1, 1, 4, 4) + .astype(np.float32) + ) + boxes = np.array([[1.0, 1.0, 2.0, 2.0], [1.5, 1.5, 3.0, 3.0]]).astype( + np.float32 + ) boxes_num = np.array([2]).astype(np.int32) - output = np.array([[[[11.]]], [[[16.]]]], dtype=np.float32) + output = np.array([[[[11.0]]], [[[16.0]]]], dtype=np.float32) data = paddle.to_tensor(data) boxes = paddle.to_tensor(boxes) diff --git a/python/paddle/tests/test_pretrained_model.py b/python/paddle/tests/test_pretrained_model.py index 4c655ce3c1f8f..81df3adf8c7bc 100644 --- a/python/paddle/tests/test_pretrained_model.py +++ b/python/paddle/tests/test_pretrained_model.py @@ -26,7 +26,6 @@ # test the predicted resutls of static graph and dynamic graph are equal # when used pretrained model class TestPretrainedModel(unittest.TestCase): - def infer(self, arch): path = os.path.join(tempfile.mkdtemp(), '.cache_test_pretrained_model') if not os.path.exists(path): diff --git a/python/paddle/tests/test_progressbar.py b/python/paddle/tests/test_progressbar.py index c42f1e4db0fe5..09340a9c24765 100644 --- a/python/paddle/tests/test_progressbar.py +++ b/python/paddle/tests/test_progressbar.py @@ -21,7 +21,6 @@ class TestProgressBar(unittest.TestCase): - def prog_bar(self, num, epoch, width, verbose=1): for epoch in range(epoch): progbar = ProgressBar(num, verbose=verbose) @@ -40,7 +39,7 @@ def prog_bar(self, num, epoch, width, verbose=1): progbar.update(1, [['loss', int(1)]]) progbar.update(1, [['loss', 'INF']]) progbar.update(1, [['loss', 1e-4]]) - progbar.update(1, [['loss', np.array([1.])]]) + progbar.update(1, [['loss', np.array([1.0])]]) progbar.update(1, [['loss', np.array([1e-4])]]) progbar.update(1, [['loss', np.array([1]).astype(np.uint16)]]) progbar.start() diff --git a/python/paddle/tests/test_read_file.py b/python/paddle/tests/test_read_file.py index ce387f039c7e6..d13e901b322b5 100644 --- a/python/paddle/tests/test_read_file.py +++ b/python/paddle/tests/test_read_file.py @@ -23,7 +23,6 @@ class TestReadFile(unittest.TestCase): - def setUp(self): fake_img = (np.random.random((400, 300, 3)) * 255).astype('uint8') self.temp_dir = tempfile.TemporaryDirectory() @@ -51,11 +50,13 @@ def read_file_decode_jpeg(self): place = paddle.CUDAPlace(0) exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) - out = exe.run(paddle.static.default_main_program(), - fetch_list=[img]) + out = exe.run( + paddle.static.default_main_program(), fetch_list=[img] + ) - np.testing.assert_equal(out[0].shape, - img_cv2.transpose(2, 0, 1).shape) + np.testing.assert_equal( + out[0].shape, img_cv2.transpose(2, 0, 1).shape + ) def test_read_file_decode_jpeg_dynamic(self): self.read_file_decode_jpeg() diff --git a/python/paddle/tests/test_transforms.py b/python/paddle/tests/test_transforms.py index f182cbd799c18..9e2cad57c37ec 100644 --- a/python/paddle/tests/test_transforms.py +++ b/python/paddle/tests/test_transforms.py @@ -28,7 +28,6 @@ class TestTransformsCV2(unittest.TestCase): - def setUp(self): self.backend = self.get_backend() set_image_backend(self.backend) @@ -39,11 +38,13 @@ def setUp(self): os.makedirs(sub_dir) for j in range(2): if j == 0: - fake_img = (np.random.random( - (280, 350, 3)) * 255).astype('uint8') + fake_img = (np.random.random((280, 350, 3)) * 255).astype( + 'uint8' + ) else: - fake_img = (np.random.random( - (400, 300, 3)) * 255).astype('uint8') + fake_img = (np.random.random((400, 300, 3)) * 255).astype( + 'uint8' + ) cv2.imwrite(os.path.join(sub_dir, str(j) + '.jpg'), fake_img) def get_backend(self): @@ -54,7 +55,8 @@ def create_image(self, shape): return (np.random.rand(*shape) * 255).astype('uint8') elif self.backend == 'pil': return Image.fromarray( - (np.random.rand(*shape) * 255).astype('uint8')) + (np.random.rand(*shape) * 255).astype('uint8') + ) def get_shape(self, img): if isinstance(img, paddle.Tensor): @@ -79,16 +81,17 @@ def test_trans_all(self): mean=[123.675, 116.28, 103.53], std=[58.395, 57.120, 57.375], ) - trans = transforms.Compose([ - transforms.RandomResizedCrop(224), - transforms.ColorJitter(brightness=0.4, - contrast=0.4, - saturation=0.4, - hue=0.4), - transforms.RandomHorizontalFlip(), - transforms.Transpose(), - normalize, - ]) + trans = transforms.Compose( + [ + transforms.RandomResizedCrop(224), + transforms.ColorJitter( + brightness=0.4, contrast=0.4, saturation=0.4, hue=0.4 + ), + transforms.RandomHorizontalFlip(), + transforms.Transpose(), + normalize, + ] + ) self.do_transform(trans) @@ -98,83 +101,104 @@ def test_normalize(self): self.do_transform(trans) def test_trans_resize(self): - trans = transforms.Compose([ - transforms.Resize(300), - transforms.RandomResizedCrop((280, 280)), - transforms.Resize(280), - transforms.Resize((256, 200)), - transforms.Resize((180, 160)), - transforms.CenterCrop(128), - transforms.CenterCrop((128, 128)), - ]) + trans = transforms.Compose( + [ + transforms.Resize(300), + transforms.RandomResizedCrop((280, 280)), + transforms.Resize(280), + transforms.Resize((256, 200)), + transforms.Resize((180, 160)), + transforms.CenterCrop(128), + transforms.CenterCrop((128, 128)), + ] + ) self.do_transform(trans) def test_flip(self): - trans = transforms.Compose([ - transforms.RandomHorizontalFlip(1.0), - transforms.RandomHorizontalFlip(0.0), - transforms.RandomVerticalFlip(0.0), - transforms.RandomVerticalFlip(1.0), - ]) + trans = transforms.Compose( + [ + transforms.RandomHorizontalFlip(1.0), + transforms.RandomHorizontalFlip(0.0), + transforms.RandomVerticalFlip(0.0), + transforms.RandomVerticalFlip(1.0), + ] + ) self.do_transform(trans) def test_color_jitter(self): - trans = transforms.Compose([ - transforms.BrightnessTransform(0.0), - transforms.HueTransform(0.0), - transforms.SaturationTransform(0.0), - transforms.ContrastTransform(0.0), - ]) + trans = transforms.Compose( + [ + transforms.BrightnessTransform(0.0), + transforms.HueTransform(0.0), + transforms.SaturationTransform(0.0), + transforms.ContrastTransform(0.0), + ] + ) self.do_transform(trans) def test_affine(self): - trans = transforms.Compose([ - transforms.RandomAffine(90), - transforms.RandomAffine([-10, 10], translate=[0.1, 0.3]), - transforms.RandomAffine(45, translate=[0.2, 0.2], scale=[0.2, 0.5]), - transforms.RandomAffine(10, - translate=[0.2, 0.2], - scale=[0.5, 0.5], - shear=[-10, 10]), - transforms.RandomAffine(10, - translate=[0.5, 0.3], - scale=[0.7, 1.3], - shear=[-10, 10, 20, 40]), - transforms.RandomAffine(10, - translate=[0.5, 0.3], - scale=[0.7, 1.3], - shear=[-10, 10, 20, 40], - interpolation='bilinear'), - transforms.RandomAffine(10, - translate=[0.5, 0.3], - scale=[0.7, 1.3], - shear=[-10, 10, 20, 40], - interpolation='bilinear', - fill=114), - transforms.RandomAffine(10, - translate=[0.5, 0.3], - scale=[0.7, 1.3], - shear=[-10, 10, 20, 40], - interpolation='bilinear', - fill=114, - center=(60, 80)), - ]) + trans = transforms.Compose( + [ + transforms.RandomAffine(90), + transforms.RandomAffine([-10, 10], translate=[0.1, 0.3]), + transforms.RandomAffine( + 45, translate=[0.2, 0.2], scale=[0.2, 0.5] + ), + transforms.RandomAffine( + 10, translate=[0.2, 0.2], scale=[0.5, 0.5], shear=[-10, 10] + ), + transforms.RandomAffine( + 10, + translate=[0.5, 0.3], + scale=[0.7, 1.3], + shear=[-10, 10, 20, 40], + ), + transforms.RandomAffine( + 10, + translate=[0.5, 0.3], + scale=[0.7, 1.3], + shear=[-10, 10, 20, 40], + interpolation='bilinear', + ), + transforms.RandomAffine( + 10, + translate=[0.5, 0.3], + scale=[0.7, 1.3], + shear=[-10, 10, 20, 40], + interpolation='bilinear', + fill=114, + ), + transforms.RandomAffine( + 10, + translate=[0.5, 0.3], + scale=[0.7, 1.3], + shear=[-10, 10, 20, 40], + interpolation='bilinear', + fill=114, + center=(60, 80), + ), + ] + ) self.do_transform(trans) def test_rotate(self): - trans = transforms.Compose([ - transforms.RandomRotation(90), - transforms.RandomRotation([-10, 10]), - transforms.RandomRotation(45, expand=True), - transforms.RandomRotation(10, expand=True, center=(60, 80)), - ]) + trans = transforms.Compose( + [ + transforms.RandomRotation(90), + transforms.RandomRotation([-10, 10]), + transforms.RandomRotation(45, expand=True), + transforms.RandomRotation(10, expand=True, center=(60, 80)), + ] + ) self.do_transform(trans) def test_perspective(self): - trans = transforms.Compose([ - transforms.RandomPerspective(prob=1.0), - transforms.RandomPerspective(prob=1.0, distortion_scale=0.9), - ]) + trans = transforms.Compose( + [ + transforms.RandomPerspective(prob=1.0), + transforms.RandomPerspective(prob=1.0, distortion_scale=0.9), + ] + ) self.do_transform(trans) def test_pad(self): @@ -191,10 +215,12 @@ def test_pad(self): img = trans_pad2(img) def test_random_crop(self): - trans = transforms.Compose([ - transforms.RandomCrop(200), - transforms.RandomCrop((140, 160)), - ]) + trans = transforms.Compose( + [ + transforms.RandomCrop(200), + transforms.RandomCrop((140, 160)), + ] + ) self.do_transform(trans) trans_random_crop1 = transforms.RandomCrop(224) @@ -211,18 +237,21 @@ def test_random_crop(self): trans_random_crop_same = transforms.RandomCrop((140, 160)) img = trans_random_crop_same(fake_img_crop2) - trans_random_crop_bigger = transforms.RandomCrop((180, 200), - pad_if_needed=True) + trans_random_crop_bigger = transforms.RandomCrop( + (180, 200), pad_if_needed=True + ) img = trans_random_crop_bigger(img) trans_random_crop_pad = transforms.RandomCrop((224, 256), 2, True) img = trans_random_crop_pad(img) def test_erase(self): - trans = transforms.Compose([ - transforms.RandomErasing(), - transforms.RandomErasing(value="random") - ]) + trans = transforms.Compose( + [ + transforms.RandomErasing(), + transforms.RandomErasing(value="random"), + ] + ) self.do_transform(trans) def test_grayscale(self): @@ -261,7 +290,7 @@ def test_to_tensor(self): def test_keys(self): fake_img1 = self.create_image((200, 150, 3)) fake_img2 = self.create_image((200, 150, 3)) - trans_pad = transforms.Pad(10, keys=("image", )) + trans_pad = transforms.Pad(10, keys=("image",)) fake_img_padded = trans_pad((fake_img1, fake_img2)) def test_exception(self): @@ -332,24 +361,27 @@ def test_exception(self): transforms.RandomAffine(10, translate=[0.2, 0.2], scale=[1, 2, 3]), with self.assertRaises(ValueError): - transforms.RandomAffine(10, - translate=[0.2, 0.2], - scale=[0.5, 0.5], - shear=[1, 2, 3]), + transforms.RandomAffine( + 10, translate=[0.2, 0.2], scale=[0.5, 0.5], shear=[1, 2, 3] + ), with self.assertRaises(ValueError): - transforms.RandomAffine(10, - translate=[0.5, 0.3], - scale=[0.7, 1.3], - shear=[-10, 10, 0, 20, 40]) + transforms.RandomAffine( + 10, + translate=[0.5, 0.3], + scale=[0.7, 1.3], + shear=[-10, 10, 0, 20, 40], + ) with self.assertRaises(ValueError): - transforms.RandomAffine(10, - translate=[0.5, 0.3], - scale=[0.7, 1.3], - shear=[-10, 10, 20, 40], - fill=114, - center=(1, 2, 3)) + transforms.RandomAffine( + 10, + translate=[0.5, 0.3], + scale=[0.7, 1.3], + shear=[-10, 10, 20, 40], + fill=114, + center=(1, 2, 3), + ) with self.assertRaises(ValueError): transforms.RandomRotation(-2) @@ -402,19 +434,18 @@ def test_info(self): class TestTransformsPIL(TestTransformsCV2): - def get_backend(self): return 'pil' class TestTransformsTensor(TestTransformsCV2): - def get_backend(self): return 'tensor' def create_image(self, shape): return paddle.to_tensor(np.random.rand(*shape)).transpose( - (2, 0, 1)) # hwc->chw + (2, 0, 1) + ) # hwc->chw def do_transform(self, trans): trans.transforms.insert(0, transforms.ToTensor(data_format='CHW')) @@ -428,11 +459,13 @@ def test_trans_all(self): mean=[123.675, 116.28, 103.53], std=[58.395, 57.120, 57.375], ) - trans = transforms.Compose([ - transforms.RandomResizedCrop(224), - transforms.RandomHorizontalFlip(), - normalize, - ]) + trans = transforms.Compose( + [ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + normalize, + ] + ) self.do_transform(trans) def test_grayscale(self): @@ -489,10 +522,12 @@ def test_pad(self): img = trans_pad4(img) def test_random_crop(self): - trans = transforms.Compose([ - transforms.RandomCrop(200), - transforms.RandomCrop((140, 160)), - ]) + trans = transforms.Compose( + [ + transforms.RandomCrop(200), + transforms.RandomCrop((140, 160)), + ] + ) self.do_transform(trans) trans_random_crop1 = transforms.RandomCrop(224) @@ -509,18 +544,21 @@ def test_random_crop(self): trans_random_crop_same = transforms.RandomCrop((140, 160)) img = trans_random_crop_same(fake_img_crop2) - trans_random_crop_bigger = transforms.RandomCrop((180, 200), - pad_if_needed=True) + trans_random_crop_bigger = transforms.RandomCrop( + (180, 200), pad_if_needed=True + ) img = trans_random_crop_bigger(img) trans_random_crop_pad = transforms.RandomCrop((224, 256), 2, True) img = trans_random_crop_pad(img) def test_erase(self): - trans = transforms.Compose([ - transforms.RandomErasing(value=(0.5, )), - transforms.RandomErasing(value="random") - ]) + trans = transforms.Compose( + [ + transforms.RandomErasing(value=(0.5,)), + transforms.RandomErasing(value="random"), + ] + ) self.do_transform(trans) erase_trans = transforms.RandomErasing(value=(0.5, 0.2, 0.01)) @@ -586,16 +624,17 @@ def test_exception(self): transforms.RandomAffine(10, translate=[0.2, 0.2], scale=[1, 2, 3]), with self.assertRaises(ValueError): - transforms.RandomAffine(10, - translate=[0.2, 0.2], - scale=[0.5, 0.5], - shear=[1, 2, 3]), + transforms.RandomAffine( + 10, translate=[0.2, 0.2], scale=[0.5, 0.5], shear=[1, 2, 3] + ), with self.assertRaises(ValueError): - transforms.RandomAffine(10, - translate=[0.5, 0.3], - scale=[0.7, 1.3], - shear=[-10, 10, 0, 20, 40]) + transforms.RandomAffine( + 10, + translate=[0.5, 0.3], + scale=[0.7, 1.3], + shear=[-10, 10, 0, 20, 40], + ) with self.assertRaises(ValueError): transforms.RandomRotation(-2) @@ -616,14 +655,14 @@ def test_exception(self): class TestFunctional(unittest.TestCase): - def test_errors(self): with self.assertRaises(TypeError): F.to_tensor(1) with self.assertRaises(ValueError): fake_img = Image.fromarray( - (np.random.rand(28, 28, 3) * 255).astype('uint8')) + (np.random.rand(28, 28, 3) * 255).astype('uint8') + ) F.to_tensor(fake_img, data_format=1) with self.assertRaises(ValueError): @@ -636,7 +675,8 @@ def test_errors(self): with self.assertRaises(TypeError): fake_img = Image.fromarray( - (np.random.rand(28, 28, 3) * 255).astype('uint8')) + (np.random.rand(28, 28, 3) * 255).astype('uint8') + ) F.resize(fake_img, '1') with self.assertRaises(TypeError): @@ -685,18 +725,18 @@ def test_errors(self): F.affine(45, translate=[0.2, 0.2], scale=0.5, shear=[-10, 0, 10]) with self.assertRaises(TypeError): - F.affine(45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10], - interpolation=2) + F.affine( + 45, + translate=[0.2, 0.2], + scale=0.5, + shear=[-10, 10], + interpolation=2, + ) with self.assertRaises(TypeError): - F.affine(45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10], - center=0) + F.affine( + 45, translate=[0.2, 0.2], scale=0.5, shear=[-10, 10], center=0 + ) with self.assertRaises(TypeError): F.rotate(1, 0.1) @@ -720,23 +760,21 @@ def test_normalize(self): std = [0.5, 0.5, 0.5] normalized_img = F.normalize(tensor_img, mean, std) - normalized_img_tensor = F.normalize(tensor_img_hwc, - mean, - std, - data_format='HWC') + normalized_img_tensor = F.normalize( + tensor_img_hwc, mean, std, data_format='HWC' + ) normalized_img_pil = F.normalize(pil_img, mean, std, data_format='HWC') - normalized_img_np = F.normalize(np_img, - mean, - std, - data_format='HWC', - to_rgb=False) - - np.testing.assert_almost_equal(np.array(normalized_img_pil), - normalized_img_np) - np.testing.assert_almost_equal(normalized_img_tensor.numpy(), - normalized_img_np, - decimal=4) + normalized_img_np = F.normalize( + np_img, mean, std, data_format='HWC', to_rgb=False + ) + + np.testing.assert_almost_equal( + np.array(normalized_img_pil), normalized_img_np + ) + np.testing.assert_almost_equal( + normalized_img_tensor.numpy(), normalized_img_np, decimal=4 + ) def test_center_crop(self): np_img = (np.random.rand(28, 24, 3) * 255).astype('uint8') @@ -747,12 +785,14 @@ def test_center_crop(self): pil_cropped_img = F.center_crop(pil_img, 4) tensor_cropped_img = F.center_crop(tensor_img, 4) - np.testing.assert_almost_equal(np_cropped_img, - np.array(pil_cropped_img)) - np.testing.assert_almost_equal(np_cropped_img, - tensor_cropped_img.numpy().transpose( - (1, 2, 0)), - decimal=4) + np.testing.assert_almost_equal( + np_cropped_img, np.array(pil_cropped_img) + ) + np.testing.assert_almost_equal( + np_cropped_img, + tensor_cropped_img.numpy().transpose((1, 2, 0)), + decimal=4, + ) def test_color_jitter_sub_function(self): np.random.seed(555) @@ -771,8 +811,9 @@ def test_color_jitter_sub_function(self): def test_adjust_brightness(np_img, tensor_img): result_cv2 = np.array(F.adjust_brightness(np_img, 1.2)) result_tensor = F.adjust_brightness(tensor_img, 1.2).numpy() - result_tensor = np.transpose(result_tensor * 255, - (1, 2, 0)).astype('uint8') + result_tensor = np.transpose(result_tensor * 255, (1, 2, 0)).astype( + 'uint8' + ) np.testing.assert_equal(result_cv2, result_tensor) # For adjust_contrast / adjust_saturation / adjust_hue the implement is kind @@ -788,7 +829,7 @@ def test_adjust_contrast(np_img, tensor_img): def test_adjust_saturation(np_img, tensor_img): result_pil = np.array(F.adjust_saturation(np_img, 1.0)) result_tensor = F.adjust_saturation(tensor_img, 1.0).numpy() - result_tensor = np.transpose(result_tensor * 255., (1, 2, 0)) + result_tensor = np.transpose(result_tensor * 255.0, (1, 2, 0)) diff = np.max(np.abs(result_tensor - result_pil)) self.assertTrue(diff < 1.1) @@ -817,14 +858,16 @@ def test_pad(self): tensor_padded_img = F.pad(tensor_img, [1, 2], padding_mode='reflect') np.testing.assert_almost_equal(np_padded_img, np.array(pil_padded_img)) - np.testing.assert_almost_equal(np_padded_img, - tensor_padded_img.numpy().transpose( - (1, 2, 0)), - decimal=3) + np.testing.assert_almost_equal( + np_padded_img, + tensor_padded_img.numpy().transpose((1, 2, 0)), + decimal=3, + ) tensor_padded_img = F.pad(tensor_img, 1, padding_mode='reflect') - tensor_padded_img = F.pad(tensor_img, [1, 2, 1, 2], - padding_mode='reflect') + tensor_padded_img = F.pad( + tensor_img, [1, 2, 1, 2], padding_mode='reflect' + ) pil_p_img = pil_img.convert('P') pil_padded_img = F.pad(pil_p_img, [1, 2]) @@ -840,16 +883,19 @@ def test_resize(self): tensor_reseized_img = F.resize(tensor_img, 40) tensor_reseized_img2 = F.resize(tensor_img, (46, 40)) - np.testing.assert_almost_equal(np_reseized_img, - np.array(pil_reseized_img)) - np.testing.assert_almost_equal(np_reseized_img, - tensor_reseized_img.numpy().transpose( - (1, 2, 0)), - decimal=3) - np.testing.assert_almost_equal(np_reseized_img, - tensor_reseized_img2.numpy().transpose( - (1, 2, 0)), - decimal=3) + np.testing.assert_almost_equal( + np_reseized_img, np.array(pil_reseized_img) + ) + np.testing.assert_almost_equal( + np_reseized_img, + tensor_reseized_img.numpy().transpose((1, 2, 0)), + decimal=3, + ) + np.testing.assert_almost_equal( + np_reseized_img, + tensor_reseized_img2.numpy().transpose((1, 2, 0)), + decimal=3, + ) gray_img = (np.zeros([28, 32])).astype('uint8') gray_resize_img = F.resize(gray_img, 40) @@ -905,26 +951,30 @@ def test_erase(self): expected_tensor = tensor_img.clone() expected_tensor[:, 10:15, 10:15] = paddle.to_tensor([0.88]) - tensor_result = F.erase(tensor_img, 10, 10, 5, 5, - paddle.to_tensor([0.88])) - np.testing.assert_equal(tensor_result.numpy(), - expected_tensor.numpy()) + tensor_result = F.erase( + tensor_img, 10, 10, 5, 5, paddle.to_tensor([0.88]) + ) + np.testing.assert_equal( + tensor_result.numpy(), expected_tensor.numpy() + ) def test_erase_backward(self): img = paddle.randn((3, 14, 14), dtype=np.float32) img.stop_gradient = False - erased = F.erase(img, 3, 3, 5, 5, paddle.ones((1, 1, 1), - dtype='float32')) + erased = F.erase( + img, 3, 3, 5, 5, paddle.ones((1, 1, 1), dtype='float32') + ) loss = erased.sum() loss.backward() expected_grad = np.ones((3, 14, 14), dtype=np.float32) - expected_grad[:, 3:8, 3:8] = 0. + expected_grad[:, 3:8, 3:8] = 0.0 np.testing.assert_equal(img.grad.numpy(), expected_grad) def test_image_load(self): - fake_img = Image.fromarray((np.random.random( - (32, 32, 3)) * 255).astype('uint8')) + fake_img = Image.fromarray( + (np.random.random((32, 32, 3)) * 255).astype('uint8') + ) temp_dir = tempfile.TemporaryDirectory() path = os.path.join(temp_dir.name, 'temp.jpg') @@ -947,35 +997,32 @@ def test_affine(self): pil_img = Image.fromarray(np_img).convert('RGB') tensor_img = F.to_tensor(pil_img, data_format='CHW') * 255 - np.testing.assert_almost_equal(np_img, - tensor_img.transpose((1, 2, 0)), - decimal=4) - - np_affined_img = F.affine(np_img, - 45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10]) - pil_affined_img = F.affine(pil_img, - 45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10]) - tensor_affined_img = F.affine(tensor_img, - 45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10]) - - np.testing.assert_equal(np_affined_img.shape, - np.array(pil_affined_img).shape) - np.testing.assert_equal(np_affined_img.shape, - tensor_affined_img.transpose((1, 2, 0)).shape) - - np.testing.assert_almost_equal(np.array(pil_affined_img), - tensor_affined_img.numpy().transpose( - (1, 2, 0)), - decimal=4) + np.testing.assert_almost_equal( + np_img, tensor_img.transpose((1, 2, 0)), decimal=4 + ) + + np_affined_img = F.affine( + np_img, 45, translate=[0.2, 0.2], scale=0.5, shear=[-10, 10] + ) + pil_affined_img = F.affine( + pil_img, 45, translate=[0.2, 0.2], scale=0.5, shear=[-10, 10] + ) + tensor_affined_img = F.affine( + tensor_img, 45, translate=[0.2, 0.2], scale=0.5, shear=[-10, 10] + ) + + np.testing.assert_equal( + np_affined_img.shape, np.array(pil_affined_img).shape + ) + np.testing.assert_equal( + np_affined_img.shape, tensor_affined_img.transpose((1, 2, 0)).shape + ) + + np.testing.assert_almost_equal( + np.array(pil_affined_img), + tensor_affined_img.numpy().transpose((1, 2, 0)), + decimal=4, + ) def test_rotate(self): np_img = (np.random.rand(28, 28, 3) * 255).astype('uint8') @@ -987,64 +1034,70 @@ def test_rotate(self): rotated_tensor_img1 = F.rotate(tensor_img, 80, expand=True) - rotated_tensor_img2 = F.rotate(tensor_img, - 80, - interpolation='bilinear', - center=(10, 10), - expand=False) + rotated_tensor_img2 = F.rotate( + tensor_img, + 80, + interpolation='bilinear', + center=(10, 10), + expand=False, + ) - np.testing.assert_equal(rotated_np_img.shape, - np.array(rotated_pil_img).shape) - np.testing.assert_equal(rotated_np_img.shape, - rotated_tensor_img1.transpose((1, 2, 0)).shape) + np.testing.assert_equal( + rotated_np_img.shape, np.array(rotated_pil_img).shape + ) + np.testing.assert_equal( + rotated_np_img.shape, rotated_tensor_img1.transpose((1, 2, 0)).shape + ) def test_rotate1(self): np_img = (np.random.rand(28, 28, 3) * 255).astype('uint8') pil_img = Image.fromarray(np_img).convert('RGB') - rotated_np_img = F.rotate(np_img, - 80, - expand=True, - center=[0, 0], - fill=[0, 0, 0]) - rotated_pil_img = F.rotate(pil_img, - 80, - expand=True, - center=[0, 0], - fill=[0, 0, 0]) + rotated_np_img = F.rotate( + np_img, 80, expand=True, center=[0, 0], fill=[0, 0, 0] + ) + rotated_pil_img = F.rotate( + pil_img, 80, expand=True, center=[0, 0], fill=[0, 0, 0] + ) - np.testing.assert_equal(rotated_np_img.shape, - np.array(rotated_pil_img).shape) + np.testing.assert_equal( + rotated_np_img.shape, np.array(rotated_pil_img).shape + ) def test_perspective(self): np_img = (np.random.rand(32, 26, 3) * 255).astype('uint8') pil_img = Image.fromarray(np_img).convert('RGB') tensor_img = F.to_tensor(pil_img, data_format='CHW') * 255 - np.testing.assert_almost_equal(np_img, - tensor_img.transpose((1, 2, 0)), - decimal=4) + np.testing.assert_almost_equal( + np_img, tensor_img.transpose((1, 2, 0)), decimal=4 + ) startpoints = [[0, 0], [13, 0], [13, 15], [0, 15]] endpoints = [[3, 2], [12, 3], [10, 14], [2, 15]] np_perspectived_img = F.perspective(np_img, startpoints, endpoints) pil_perspectived_img = F.perspective(pil_img, startpoints, endpoints) - tensor_perspectived_img = F.perspective(tensor_img, startpoints, - endpoints) + tensor_perspectived_img = F.perspective( + tensor_img, startpoints, endpoints + ) - np.testing.assert_equal(np_perspectived_img.shape, - np.array(pil_perspectived_img).shape) + np.testing.assert_equal( + np_perspectived_img.shape, np.array(pil_perspectived_img).shape + ) np.testing.assert_equal( np_perspectived_img.shape, - tensor_perspectived_img.transpose((1, 2, 0)).shape) + tensor_perspectived_img.transpose((1, 2, 0)).shape, + ) result_pil = np.array(pil_perspectived_img) - result_tensor = tensor_perspectived_img.numpy().transpose( - (1, 2, 0)).astype('uint8') + result_tensor = ( + tensor_perspectived_img.numpy().transpose((1, 2, 0)).astype('uint8') + ) num_diff_pixels = (result_pil != result_tensor).sum() / 3.0 - ratio_diff_pixels = num_diff_pixels / result_tensor.shape[ - 0] / result_tensor.shape[1] + ratio_diff_pixels = ( + num_diff_pixels / result_tensor.shape[0] / result_tensor.shape[1] + ) # Tolerance : less than 6% of different pixels assert ratio_diff_pixels < 0.06 @@ -1054,10 +1107,12 @@ def test_batch_input(self): def test_erase(batch_tensor): input1, input2 = paddle.unbind(batch_tensor, axis=0) - target_result = paddle.stack([ - F.erase(input1, 1, 1, 2, 2, 0.5), - F.erase(input2, 1, 1, 2, 2, 0.5) - ]) + target_result = paddle.stack( + [ + F.erase(input1, 1, 1, 2, 2, 0.5), + F.erase(input2, 1, 1, 2, 2, 0.5), + ] + ) batch_result = F.erase(batch_tensor, 1, 1, 2, 2, 0.5) @@ -1067,23 +1122,31 @@ def test_erase(batch_tensor): def test_affine(batch_tensor): input1, input2 = paddle.unbind(batch_tensor, axis=0) - target_result = paddle.stack([ - F.affine(input1, - 45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10]), - F.affine(input2, - 45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10]) - ]) - batch_result = F.affine(batch_tensor, - 45, - translate=[0.2, 0.2], - scale=0.5, - shear=[-10, 10]) + target_result = paddle.stack( + [ + F.affine( + input1, + 45, + translate=[0.2, 0.2], + scale=0.5, + shear=[-10, 10], + ), + F.affine( + input2, + 45, + translate=[0.2, 0.2], + scale=0.5, + shear=[-10, 10], + ), + ] + ) + batch_result = F.affine( + batch_tensor, + 45, + translate=[0.2, 0.2], + scale=0.5, + shear=[-10, 10], + ) return paddle.allclose(batch_result, target_result) @@ -1093,10 +1156,12 @@ def test_perspective(batch_tensor): input1, input2 = paddle.unbind(batch_tensor, axis=0) startpoints = [[0, 0], [3, 0], [4, 5], [6, 7]] endpoints = [[0, 1], [3, 1], [4, 4], [5, 7]] - target_result = paddle.stack([ - F.perspective(input1, startpoints, endpoints), - F.perspective(input2, startpoints, endpoints) - ]) + target_result = paddle.stack( + [ + F.perspective(input1, startpoints, endpoints), + F.perspective(input2, startpoints, endpoints), + ] + ) batch_result = F.perspective(batch_tensor, startpoints, endpoints) @@ -1106,10 +1171,12 @@ def test_perspective(batch_tensor): def test_adjust_brightness(batch_tensor): input1, input2 = paddle.unbind(batch_tensor, axis=0) - target_result = paddle.stack([ - F.adjust_brightness(input1, 2.1), - F.adjust_brightness(input2, 2.1) - ]) + target_result = paddle.stack( + [ + F.adjust_brightness(input1, 2.1), + F.adjust_brightness(input2, 2.1), + ] + ) batch_result = F.adjust_brightness(batch_tensor, 2.1) @@ -1119,10 +1186,9 @@ def test_adjust_brightness(batch_tensor): def test_adjust_contrast(batch_tensor): input1, input2 = paddle.unbind(batch_tensor, axis=0) - target_result = paddle.stack([ - F.adjust_contrast(input1, 0.3), - F.adjust_contrast(input2, 0.3) - ]) + target_result = paddle.stack( + [F.adjust_contrast(input1, 0.3), F.adjust_contrast(input2, 0.3)] + ) batch_result = F.adjust_contrast(batch_tensor, 0.3) @@ -1132,10 +1198,12 @@ def test_adjust_contrast(batch_tensor): def test_adjust_saturation(batch_tensor): input1, input2 = paddle.unbind(batch_tensor, axis=0) - target_result = paddle.stack([ - F.adjust_saturation(input1, 1.1), - F.adjust_saturation(input2, 1.1) - ]) + target_result = paddle.stack( + [ + F.adjust_saturation(input1, 1.1), + F.adjust_saturation(input2, 1.1), + ] + ) batch_result = F.adjust_saturation(batch_tensor, 1.1) @@ -1146,8 +1214,8 @@ def test_adjust_saturation(batch_tensor): def test_adjust_hue(batch_tensor): input1, input2 = paddle.unbind(batch_tensor, axis=0) target_result = paddle.stack( - [F.adjust_hue(input1, -0.2), - F.adjust_hue(input2, -0.2)]) + [F.adjust_hue(input1, -0.2), F.adjust_hue(input2, -0.2)] + ) batch_result = F.adjust_hue(batch_tensor, -0.2) diff --git a/python/paddle/tests/test_vision_models.py b/python/paddle/tests/test_vision_models.py index 1f53060beb049..dc98fc3219bff 100644 --- a/python/paddle/tests/test_vision_models.py +++ b/python/paddle/tests/test_vision_models.py @@ -20,7 +20,6 @@ class TestVisonModels(unittest.TestCase): - def models_infer(self, arch, pretrained=False, batch_norm=False): x = np.array(np.random.random((2, 3, 224, 224)), dtype=np.float32) diff --git a/python/paddle/text/__init__.py b/python/paddle/text/__init__.py index 5775a24785804..fbfa0c3fe2e02 100644 --- a/python/paddle/text/__init__.py +++ b/python/paddle/text/__init__.py @@ -21,7 +21,14 @@ from .datasets import WMT14 # noqa: F401 from .datasets import WMT16 # noqa: F401 -__all__ = [ #noqa - 'Conll05st', 'Imdb', 'Imikolov', 'Movielens', 'UCIHousing', 'WMT14', - 'WMT16', 'ViterbiDecoder', 'viterbi_decode' +__all__ = [ # noqa + 'Conll05st', + 'Imdb', + 'Imikolov', + 'Movielens', + 'UCIHousing', + 'WMT14', + 'WMT16', + 'ViterbiDecoder', + 'viterbi_decode', ] diff --git a/python/paddle/text/datasets/conll05.py b/python/paddle/text/datasets/conll05.py index ee475e7fc5d43..807c8c3fbebc9 100644 --- a/python/paddle/text/datasets/conll05.py +++ b/python/paddle/text/datasets/conll05.py @@ -91,47 +91,71 @@ def forward(self, pred_idx, mark, label): """ - def __init__(self, - data_file=None, - word_dict_file=None, - verb_dict_file=None, - target_dict_file=None, - emb_file=None, - download=True): + def __init__( + self, + data_file=None, + word_dict_file=None, + verb_dict_file=None, + target_dict_file=None, + emb_file=None, + download=True, + ): self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, DATA_URL, - DATA_MD5, 'conll05st', - download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, DATA_URL, DATA_MD5, 'conll05st', download + ) self.word_dict_file = word_dict_file if self.word_dict_file is None: - assert download, "word_dict_file is not set and downloading automatically is disabled" + assert ( + download + ), "word_dict_file is not set and downloading automatically is disabled" self.word_dict_file = _check_exists_and_download( - word_dict_file, WORDDICT_URL, WORDDICT_MD5, 'conll05st', - download) + word_dict_file, + WORDDICT_URL, + WORDDICT_MD5, + 'conll05st', + download, + ) self.verb_dict_file = verb_dict_file if self.verb_dict_file is None: - assert download, "verb_dict_file is not set and downloading automatically is disabled" + assert ( + download + ), "verb_dict_file is not set and downloading automatically is disabled" self.verb_dict_file = _check_exists_and_download( - verb_dict_file, VERBDICT_URL, VERBDICT_MD5, 'conll05st', - download) + verb_dict_file, + VERBDICT_URL, + VERBDICT_MD5, + 'conll05st', + download, + ) self.target_dict_file = target_dict_file if self.target_dict_file is None: - assert download, "target_dict_file is not set and downloading automatically is disabled" + assert ( + download + ), "target_dict_file is not set and downloading automatically is disabled" self.target_dict_file = _check_exists_and_download( - target_dict_file, TRGDICT_URL, TRGDICT_MD5, 'conll05st', - download) + target_dict_file, + TRGDICT_URL, + TRGDICT_MD5, + 'conll05st', + download, + ) self.emb_file = emb_file if self.emb_file is None: - assert download, "emb_file is not set and downloading automatically is disabled" - self.emb_file = _check_exists_and_download(emb_file, EMB_URL, - EMB_MD5, 'conll05st', - download) + assert ( + download + ), "emb_file is not set and downloading automatically is disabled" + self.emb_file = _check_exists_and_download( + emb_file, EMB_URL, EMB_MD5, 'conll05st', download + ) self.word_dict = self._load_dict(self.word_dict_file) self.predicate_dict = self._load_dict(self.verb_dict_file) @@ -169,14 +193,17 @@ def _load_dict(self, filename): def _load_anno(self): tf = tarfile.open(self.data_file) wf = tf.extractfile( - "conll05st-release/test.wsj/words/test.wsj.words.gz") + "conll05st-release/test.wsj/words/test.wsj.words.gz" + ) pf = tf.extractfile( - "conll05st-release/test.wsj/props/test.wsj.props.gz") + "conll05st-release/test.wsj/props/test.wsj.props.gz" + ) self.sentences = [] self.predicates = [] self.labels = [] with gzip.GzipFile(fileobj=wf) as words_file, gzip.GzipFile( - fileobj=pf) as props_file: + fileobj=pf + ) as props_file: sentences = [] labels = [] one_seg = [] @@ -209,16 +236,17 @@ def _load_anno(self): lbl_seq.append('I-' + cur_tag) is_in_bracket = False elif l.find('(') != -1 and l.find(')') != -1: - cur_tag = l[1:l.find('*')] + cur_tag = l[1 : l.find('*')] lbl_seq.append('B-' + cur_tag) is_in_bracket = False elif l.find('(') != -1 and l.find(')') == -1: - cur_tag = l[1:l.find('*')] + cur_tag = l[1 : l.find('*')] lbl_seq.append('B-' + cur_tag) is_in_bracket = True else: - raise RuntimeError('Unexpected label: %s' % - l) + raise RuntimeError( + 'Unexpected label: %s' % l + ) self.sentences.append(sentences) self.predicates.append(verb_list[i]) @@ -282,9 +310,17 @@ def __getitem__(self, idx): pred_idx = [self.predicate_dict.get(predicate)] * sen_len label_idx = [self.label_dict.get(w) for w in labels] - return (np.array(word_idx), np.array(ctx_n2_idx), np.array(ctx_n1_idx), - np.array(ctx_0_idx), np.array(ctx_p1_idx), np.array(ctx_p2_idx), - np.array(pred_idx), np.array(mark), np.array(label_idx)) + return ( + np.array(word_idx), + np.array(ctx_n2_idx), + np.array(ctx_n1_idx), + np.array(ctx_0_idx), + np.array(ctx_p1_idx), + np.array(ctx_p2_idx), + np.array(pred_idx), + np.array(mark), + np.array(label_idx), + ) def __len__(self): return len(self.sentences) diff --git a/python/paddle/text/datasets/imdb.py b/python/paddle/text/datasets/imdb.py index 80b650574b4dd..d71d23ff69231 100644 --- a/python/paddle/text/datasets/imdb.py +++ b/python/paddle/text/datasets/imdb.py @@ -71,15 +71,20 @@ def forward(self, doc, label): """ def __init__(self, data_file=None, mode='train', cutoff=150, download=True): - assert mode.lower() in ['train', 'test'], \ - "mode should be 'train', 'test', but got {}".format(mode) + assert mode.lower() in [ + 'train', + 'test', + ], "mode should be 'train', 'test', but got {}".format(mode) self.mode = mode.lower() self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, URL, MD5, - 'imdb', download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, URL, MD5, 'imdb', download + ) # Build a word dictionary from the corpus self.word_idx = self._build_work_dict(cutoff) @@ -111,9 +116,13 @@ def _tokenize(self, pattern): if bool(pattern.match(tf.name)): # newline and punctuations removal and ad-hoc tokenization. data.append( - tarf.extractfile(tf).read().rstrip(b'\n\r').translate( - None, string.punctuation.encode( - 'latin-1')).lower().split()) + tarf.extractfile(tf) + .read() + .rstrip(b'\n\r') + .translate(None, string.punctuation.encode('latin-1')) + .lower() + .split() + ) tf = tarf.next() return data diff --git a/python/paddle/text/datasets/imikolov.py b/python/paddle/text/datasets/imikolov.py index 03d6560e9bce8..d1081f4dda610 100644 --- a/python/paddle/text/datasets/imikolov.py +++ b/python/paddle/text/datasets/imikolov.py @@ -70,19 +70,25 @@ def forward(self, src, trg): """ - def __init__(self, - data_file=None, - data_type='NGRAM', - window_size=-1, - mode='train', - min_word_freq=50, - download=True): - assert data_type.upper() in ['NGRAM', 'SEQ'], \ - "data type should be 'NGRAM', 'SEQ', but got {}".format(data_type) + def __init__( + self, + data_file=None, + data_type='NGRAM', + window_size=-1, + mode='train', + min_word_freq=50, + download=True, + ): + assert data_type.upper() in [ + 'NGRAM', + 'SEQ', + ], "data type should be 'NGRAM', 'SEQ', but got {}".format(data_type) self.data_type = data_type.upper() - assert mode.lower() in ['train', 'test'], \ - "mode should be 'train', 'test', but got {}".format(mode) + assert mode.lower() in [ + 'train', + 'test', + ], "mode should be 'train', 'test', but got {}".format(mode) self.mode = mode.lower() self.window_size = window_size @@ -90,9 +96,12 @@ def __init__(self, self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically disabled" - self.data_file = _check_exists_and_download(data_file, URL, MD5, - 'imikolov', download) + assert ( + download + ), "data_file is not set and downloading automatically disabled" + self.data_file = _check_exists_and_download( + data_file, URL, MD5, 'imikolov', download + ) # Build a word dictionary from the corpus self.word_idx = self._build_work_dict(min_word_freq) @@ -148,7 +157,7 @@ def _load_anno(self): if len(l) >= self.window_size: l = [self.word_idx.get(w, UNK) for w in l] for i in range(self.window_size, len(l) + 1): - self.data.append(tuple(l[i - self.window_size:i])) + self.data.append(tuple(l[i - self.window_size : i])) elif self.data_type == 'SEQ': l = l.strip().split() l = [self.word_idx.get(w, UNK) for w in l] diff --git a/python/paddle/text/datasets/movielens.py b/python/paddle/text/datasets/movielens.py index f735d22f91f28..cf23eae1d8449 100644 --- a/python/paddle/text/datasets/movielens.py +++ b/python/paddle/text/datasets/movielens.py @@ -41,12 +41,18 @@ def value(self, categories_dict, movie_title_dict): """ Get information from a movie. """ - return [[self.index], [categories_dict[c] for c in self.categories], - [movie_title_dict[w.lower()] for w in self.title.split()]] + return [ + [self.index], + [categories_dict[c] for c in self.categories], + [movie_title_dict[w.lower()] for w in self.title.split()], + ] def __str__(self): return "" % ( - self.index, self.title, self.categories) + self.index, + self.title, + self.categories, + ) def __repr__(self): return self.__str__() @@ -67,13 +73,20 @@ def value(self): """ Get information from a user. """ - return [[self.index], [0 if self.is_male else 1], [self.age], - [self.job_id]] + return [ + [self.index], + [0 if self.is_male else 1], + [self.age], + [self.job_id], + ] def __str__(self): return "" % ( - self.index, "M" if self.is_male else "F", age_table[self.age], - self.job_id) + self.index, + "M" if self.is_male else "F", + age_table[self.age], + self.job_id, + ) def __repr__(self): return str(self) @@ -124,21 +137,28 @@ def forward(self, category, title, rating): """ - def __init__(self, - data_file=None, - mode='train', - test_ratio=0.1, - rand_seed=0, - download=True): - assert mode.lower() in ['train', 'test'], \ - "mode should be 'train', 'test', but got {}".format(mode) + def __init__( + self, + data_file=None, + mode='train', + test_ratio=0.1, + rand_seed=0, + download=True, + ): + assert mode.lower() in [ + 'train', + 'test', + ], "mode should be 'train', 'test', but got {}".format(mode) self.mode = mode.lower() self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, URL, MD5, - 'sentiment', download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, URL, MD5, 'sentiment', download + ) self.test_ratio = test_ratio self.rand_seed = rand_seed @@ -167,7 +187,8 @@ def _load_meta_info(self): categories_set.add(c) title = pattern.match(title).group(1) self.movie_info[int(movie_id)] = MovieInfo( - index=movie_id, categories=categories, title=title) + index=movie_id, categories=categories, title=title + ) for w in title.split(): title_word_set.add(w.lower()) @@ -181,10 +202,9 @@ def _load_meta_info(self): for line in user_file: line = line.decode(encoding='latin') uid, gender, age, job, _ = line.strip().split("::") - self.user_info[int(uid)] = UserInfo(index=uid, - gender=gender, - age=age, - job_id=job) + self.user_info[int(uid)] = UserInfo( + index=uid, gender=gender, age=age, job_id=job + ) def _load_data(self): self.data = [] @@ -201,9 +221,13 @@ def _load_data(self): mov = self.movie_info[mov_id] usr = self.user_info[uid] - self.data.append(usr.value() + \ - mov.value(self.categories_dict, self.movie_title_dict) + \ - [[rating]]) + self.data.append( + usr.value() + + mov.value( + self.categories_dict, self.movie_title_dict + ) + + [[rating]] + ) def __getitem__(self, idx): data = self.data[idx] diff --git a/python/paddle/text/datasets/uci_housing.py b/python/paddle/text/datasets/uci_housing.py index 0bf677e271a5c..bf43332dd3905 100644 --- a/python/paddle/text/datasets/uci_housing.py +++ b/python/paddle/text/datasets/uci_housing.py @@ -23,8 +23,19 @@ URL = 'http://paddlemodels.bj.bcebos.com/uci_housing/housing.data' MD5 = 'd4accdce7a25600298819f8e28e8d593' feature_names = [ - 'CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', - 'PTRATIO', 'B', 'LSTAT' + 'CRIM', + 'ZN', + 'INDUS', + 'CHAS', + 'NOX', + 'RM', + 'AGE', + 'DIS', + 'RAD', + 'TAX', + 'PTRATIO', + 'B', + 'LSTAT', ] @@ -73,15 +84,20 @@ def forward(self, feature, target): """ def __init__(self, data_file=None, mode='train', download=True): - assert mode.lower() in ['train', 'test'], \ - "mode should be 'train' or 'test', but got {}".format(mode) + assert mode.lower() in [ + 'train', + 'test', + ], "mode should be 'train' or 'test', but got {}".format(mode) self.mode = mode.lower() self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, URL, MD5, - 'uci_housing', download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, URL, MD5, 'uci_housing', download + ) # read dataset into memory self._load_data() @@ -91,8 +107,11 @@ def __init__(self, data_file=None, mode='train', download=True): def _load_data(self, feature_num=14, ratio=0.8): data = np.fromfile(self.data_file, sep=' ') data = data.reshape(data.shape[0] // feature_num, feature_num) - maximums, minimums, avgs = data.max(axis=0), data.min( - axis=0), data.sum(axis=0) / data.shape[0] + maximums, minimums, avgs = ( + data.max(axis=0), + data.min(axis=0), + data.sum(axis=0) / data.shape[0], + ) for i in range(feature_num - 1): data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i]) offset = int(data.shape[0] * ratio) @@ -103,8 +122,9 @@ def _load_data(self, feature_num=14, ratio=0.8): def __getitem__(self, idx): data = self.data[idx] - return np.array(data[:-1]).astype(self.dtype), \ - np.array(data[-1:]).astype(self.dtype) + return np.array(data[:-1]).astype(self.dtype), np.array( + data[-1:] + ).astype(self.dtype) def __len__(self): return len(self.data) diff --git a/python/paddle/text/datasets/wmt14.py b/python/paddle/text/datasets/wmt14.py index 33c92c875e8de..71f5c7af0f4fb 100644 --- a/python/paddle/text/datasets/wmt14.py +++ b/python/paddle/text/datasets/wmt14.py @@ -20,12 +20,14 @@ __all__ = [] -URL_DEV_TEST = ('http://www-lium.univ-lemans.fr/~schwenk/' - 'cslm_joint_paper/data/dev+test.tgz') +URL_DEV_TEST = ( + 'http://www-lium.univ-lemans.fr/~schwenk/' + 'cslm_joint_paper/data/dev+test.tgz' +) MD5_DEV_TEST = '7d7897317ddd8ba0ae5c5fa7248d3ff5' # this is a small set of data for test. The original data is too large and # will be add later. -URL_TRAIN = ('http://paddlemodels.bj.bcebos.com/wmt/wmt14.tgz') +URL_TRAIN = 'http://paddlemodels.bj.bcebos.com/wmt/wmt14.tgz' MD5_TRAIN = '0791583d57d5beb693b9414c5b36798c' START = "" @@ -82,21 +84,24 @@ def forward(self, src_ids, trg_ids, trg_ids_next): """ - def __init__(self, - data_file=None, - mode='train', - dict_size=-1, - download=True): - assert mode.lower() in ['train', 'test', 'gen'], \ - "mode should be 'train', 'test' or 'gen', but got {}".format(mode) + def __init__( + self, data_file=None, mode='train', dict_size=-1, download=True + ): + assert mode.lower() in [ + 'train', + 'test', + 'gen', + ], "mode should be 'train', 'test' or 'gen', but got {}".format(mode) self.mode = mode.lower() self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, URL_TRAIN, - MD5_TRAIN, 'wmt14', - download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, URL_TRAIN, MD5_TRAIN, 'wmt14', download + ) # read dataset into memory assert dict_size > 0, "dict_size should be set as positive number" @@ -104,7 +109,6 @@ def __init__(self, self._load_data() def _load_data(self): - def __to_dict(fd, size): out_dict = dict() for line_count, line in enumerate(fd): @@ -119,13 +123,15 @@ def __to_dict(fd, size): self.trg_ids_next = [] with tarfile.open(self.data_file, mode='r') as f: names = [ - each_item.name for each_item in f + each_item.name + for each_item in f if each_item.name.endswith("src.dict") ] assert len(names) == 1 self.src_dict = __to_dict(f.extractfile(names[0]), self.dict_size) names = [ - each_item.name for each_item in f + each_item.name + for each_item in f if each_item.name.endswith("trg.dict") ] assert len(names) == 1 @@ -133,7 +139,8 @@ def __to_dict(fd, size): file_name = "{}/{}".format(self.mode, self.mode) names = [ - each_item.name for each_item in f + each_item.name + for each_item in f if each_item.name.endswith(file_name) ] for name in names: @@ -164,8 +171,11 @@ def __to_dict(fd, size): self.trg_ids_next.append(trg_ids_next) def __getitem__(self, idx): - return (np.array(self.src_ids[idx]), np.array(self.trg_ids[idx]), - np.array(self.trg_ids_next[idx])) + return ( + np.array(self.src_ids[idx]), + np.array(self.trg_ids[idx]), + np.array(self.trg_ids_next[idx]), + ) def __len__(self): return len(self.src_ids) diff --git a/python/paddle/text/datasets/wmt16.py b/python/paddle/text/datasets/wmt16.py index 4538f4b8a6a2c..3048160acff11 100644 --- a/python/paddle/text/datasets/wmt16.py +++ b/python/paddle/text/datasets/wmt16.py @@ -25,7 +25,7 @@ __all__ = [] -DATA_URL = ("http://paddlemodels.bj.bcebos.com/wmt/wmt16.tar.gz") +DATA_URL = "http://paddlemodels.bj.bcebos.com/wmt/wmt16.tar.gz" DATA_MD5 = "0c38be43600334966403524a40dcd81e" TOTAL_EN_WORDS = 11250 @@ -102,43 +102,55 @@ def forward(self, src_ids, trg_ids, trg_ids_next): """ - def __init__(self, - data_file=None, - mode='train', - src_dict_size=-1, - trg_dict_size=-1, - lang='en', - download=True): - assert mode.lower() in ['train', 'test', 'val'], \ - "mode should be 'train', 'test' or 'val', but got {}".format(mode) + def __init__( + self, + data_file=None, + mode='train', + src_dict_size=-1, + trg_dict_size=-1, + lang='en', + download=True, + ): + assert mode.lower() in [ + 'train', + 'test', + 'val', + ], "mode should be 'train', 'test' or 'val', but got {}".format(mode) self.mode = mode.lower() self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, DATA_URL, - DATA_MD5, 'wmt16', - download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, DATA_URL, DATA_MD5, 'wmt16', download + ) self.lang = lang assert src_dict_size > 0, "dict_size should be set as positive number" assert trg_dict_size > 0, "dict_size should be set as positive number" self.src_dict_size = min( - src_dict_size, (TOTAL_EN_WORDS if lang == "en" else TOTAL_DE_WORDS)) + src_dict_size, (TOTAL_EN_WORDS if lang == "en" else TOTAL_DE_WORDS) + ) self.trg_dict_size = min( - trg_dict_size, (TOTAL_DE_WORDS if lang == "en" else TOTAL_EN_WORDS)) + trg_dict_size, (TOTAL_DE_WORDS if lang == "en" else TOTAL_EN_WORDS) + ) # load source and target word dict self.src_dict = self._load_dict(lang, src_dict_size) - self.trg_dict = self._load_dict("de" if lang == "en" else "en", - trg_dict_size) + self.trg_dict = self._load_dict( + "de" if lang == "en" else "en", trg_dict_size + ) # load data self.data = self._load_data() def _load_dict(self, lang, dict_size, reverse=False): - dict_path = os.path.join(paddle.dataset.common.DATA_HOME, - "wmt16/%s_%d.dict" % (lang, dict_size)) + dict_path = os.path.join( + paddle.dataset.common.DATA_HOME, + "wmt16/%s_%d.dict" % (lang, dict_size), + ) dict_found = False if os.path.exists(dict_path): with open(dict_path, "rb") as d: @@ -161,18 +173,21 @@ def _build_dict(self, dict_path, dict_size, lang): for line in f.extractfile("wmt16/train"): line = line.decode() line_split = line.strip().split("\t") - if len(line_split) != 2: continue + if len(line_split) != 2: + continue sen = line_split[0] if self.lang == "en" else line_split[1] for w in sen.split(): word_dict[w] += 1 with open(dict_path, "wb") as fout: fout.write( - ("%s\n%s\n%s\n" % (START_MARK, END_MARK, UNK_MARK)).encode()) + ("%s\n%s\n%s\n" % (START_MARK, END_MARK, UNK_MARK)).encode() + ) for idx, word in enumerate( - sorted(word_dict.items(), key=lambda x: x[1], - reverse=True)): - if idx + 3 == dict_size: break + sorted(word_dict.items(), key=lambda x: x[1], reverse=True) + ): + if idx + 3 == dict_size: + break fout.write(word[0].encode()) fout.write(b'\n') @@ -197,9 +212,11 @@ def _load_data(self): if len(line_split) != 2: continue src_words = line_split[src_col].split() - src_ids = [start_id] + [ - self.src_dict.get(w, unk_id) for w in src_words - ] + [end_id] + src_ids = ( + [start_id] + + [self.src_dict.get(w, unk_id) for w in src_words] + + [end_id] + ) trg_words = line_split[trg_col].split() trg_ids = [self.trg_dict.get(w, unk_id) for w in trg_words] @@ -212,8 +229,11 @@ def _load_data(self): self.trg_ids_next.append(trg_ids_next) def __getitem__(self, idx): - return (np.array(self.src_ids[idx]), np.array(self.trg_ids[idx]), - np.array(self.trg_ids_next[idx])) + return ( + np.array(self.src_ids[idx]), + np.array(self.trg_ids[idx]), + np.array(self.trg_ids_next[idx]), + ) def __len__(self): return len(self.src_ids) @@ -243,10 +263,14 @@ def get_dict(self, lang, reverse=False): en_dict = wmt16.get_dict('en') """ - dict_size = self.src_dict_size if lang == self.lang else self.trg_dict_size - - dict_path = os.path.join(paddle.dataset.common.DATA_HOME, - "wmt16/%s_%d.dict" % (lang, dict_size)) + dict_size = ( + self.src_dict_size if lang == self.lang else self.trg_dict_size + ) + + dict_path = os.path.join( + paddle.dataset.common.DATA_HOME, + "wmt16/%s_%d.dict" % (lang, dict_size), + ) assert os.path.exists(dict_path), "Word dictionary does not exist. " "Please invoke paddle.dataset.wmt16.train/test/validation first " "to build the dictionary." diff --git a/python/paddle/text/viterbi_decode.py b/python/paddle/text/viterbi_decode.py index a2a92da947288..07d4515cab8e2 100644 --- a/python/paddle/text/viterbi_decode.py +++ b/python/paddle/text/viterbi_decode.py @@ -21,11 +21,9 @@ __all__ = ['viterbi_decode', 'ViterbiDecoder'] -def viterbi_decode(potentials, - transition_params, - lengths, - include_bos_eos_tag=True, - name=None): +def viterbi_decode( + potentials, transition_params, lengths, include_bos_eos_tag=True, name=None +): """ Decode the highest scoring sequence of tags computed by transitions and potentials and get the viterbi path. @@ -59,34 +57,43 @@ def viterbi_decode(potentials, scores, path = paddle.text.viterbi_decode(emission, transition, length, False) # scores: [3.37089300, 1.56825531], path: [[1, 0, 0], [1, 1, 0]] """ if in_dygraph_mode(): - return _C_ops.viterbi_decode(potentials, transition_params, lengths, - include_bos_eos_tag) + return _C_ops.viterbi_decode( + potentials, transition_params, lengths, include_bos_eos_tag + ) if _non_static_mode(): - return _legacy_C_ops.viterbi_decode(potentials, transition_params, - lengths, 'include_bos_eos_tag', - include_bos_eos_tag) - check_variable_and_dtype(potentials, 'input', ['float32', 'float64'], - 'viterbi_decode') - check_variable_and_dtype(transition_params, 'transitions', - ['float32', 'float64'], 'viterbi_decode') + return _legacy_C_ops.viterbi_decode( + potentials, + transition_params, + lengths, + 'include_bos_eos_tag', + include_bos_eos_tag, + ) + check_variable_and_dtype( + potentials, 'input', ['float32', 'float64'], 'viterbi_decode' + ) + check_variable_and_dtype( + transition_params, + 'transitions', + ['float32', 'float64'], + 'viterbi_decode', + ) check_variable_and_dtype(lengths, 'length', 'int64', 'viterbi_decode') check_type(include_bos_eos_tag, 'include_tag', bool, 'viterbi_decode') helper = LayerHelper('viterbi_decode', **locals()) attrs = {'include_bos_eos_tag': include_bos_eos_tag} scores = helper.create_variable_for_type_inference(potentials.dtype) path = helper.create_variable_for_type_inference('int64') - helper.append_op(type='viterbi_decode', - inputs={ - 'Input': potentials, - 'Transition': transition_params, - 'Length': lengths - }, - outputs={ - 'Scores': scores, - 'Path': path - }, - attrs=attrs) + helper.append_op( + type='viterbi_decode', + inputs={ + 'Input': potentials, + 'Transition': transition_params, + 'Length': lengths, + }, + outputs={'Scores': scores, 'Path': path}, + attrs=attrs, + ) return scores, path @@ -134,5 +141,10 @@ def __init__(self, transitions, include_bos_eos_tag=True, name=None): self.name = name def forward(self, potentials, lengths): - return viterbi_decode(potentials, self.transitions, lengths, - self.include_bos_eos_tag, self.name) + return viterbi_decode( + potentials, + self.transitions, + lengths, + self.include_bos_eos_tag, + self.name, + ) diff --git a/python/paddle/utils/__init__.py b/python/paddle/utils/__init__.py index 6994fd139b647..b51bf4b375305 100644 --- a/python/paddle/utils/__init__.py +++ b/python/paddle/utils/__init__.py @@ -28,6 +28,4 @@ from . import cpp_extension # noqa: F401 from . import dlpack -__all__ = [ #noqa - 'deprecated', 'run_check', 'require_version', 'try_import' -] +__all__ = ['deprecated', 'run_check', 'require_version', 'try_import'] # noqa diff --git a/python/paddle/utils/cpp_extension/__init__.py b/python/paddle/utils/cpp_extension/__init__.py index 843f78d5c803a..9ad431f00a65f 100644 --- a/python/paddle/utils/cpp_extension/__init__.py +++ b/python/paddle/utils/cpp_extension/__init__.py @@ -22,6 +22,10 @@ from .extension_utils import get_build_directory # noqa: F401 from .extension_utils import load_op_meta_info_and_register_op # noqa: F401 -__all__ = [ #noqa - 'CppExtension', 'CUDAExtension', 'load', 'setup', 'get_build_directory' +__all__ = [ # noqa + 'CppExtension', + 'CUDAExtension', + 'load', + 'setup', + 'get_build_directory', ] diff --git a/python/paddle/utils/cpp_extension/cpp_extension.py b/python/paddle/utils/cpp_extension/cpp_extension.py index 5ff1d1cc29207..9dfa2f103da19 100644 --- a/python/paddle/utils/cpp_extension/cpp_extension.py +++ b/python/paddle/utils/cpp_extension/cpp_extension.py @@ -21,14 +21,41 @@ from setuptools.command.build_ext import build_ext from distutils.command.build import build -from .extension_utils import add_compile_flag, find_cuda_home, find_rocm_home, normalize_extension_kwargs -from .extension_utils import is_cuda_file, prepare_unix_cudaflags, prepare_win_cudaflags -from .extension_utils import _import_module_from_library, _write_setup_file, _jit_compile -from .extension_utils import check_abi_compatibility, log_v, CustomOpInfo, parse_op_name_from +from .extension_utils import ( + add_compile_flag, + find_cuda_home, + find_rocm_home, + normalize_extension_kwargs, +) +from .extension_utils import ( + is_cuda_file, + prepare_unix_cudaflags, + prepare_win_cudaflags, +) +from .extension_utils import ( + _import_module_from_library, + _write_setup_file, + _jit_compile, +) +from .extension_utils import ( + check_abi_compatibility, + log_v, + CustomOpInfo, + parse_op_name_from, +) from .extension_utils import _reset_so_rpath, clean_object_if_change_cflags -from .extension_utils import bootstrap_context, get_build_directory, add_std_without_repeat - -from .extension_utils import IS_WINDOWS, OS_NAME, MSVC_COMPILE_FLAGS, MSVC_COMPILE_FLAGS +from .extension_utils import ( + bootstrap_context, + get_build_directory, + add_std_without_repeat, +) + +from .extension_utils import ( + IS_WINDOWS, + OS_NAME, + MSVC_COMPILE_FLAGS, + MSVC_COMPILE_FLAGS, +) from .extension_utils import CLANG_COMPILE_FLAGS, CLANG_LINK_FLAGS from ...fluid import core @@ -39,6 +66,7 @@ if IS_WINDOWS: from distutils.command.build_ext import build_ext as _du_build_ext from unittest.mock import Mock + _du_build_ext.get_export_symbols = Mock(return_value=None) CUDA_HOME = find_cuda_home() @@ -147,7 +175,8 @@ def setup(**attr): # if not specific cmdclass in setup, add it automatically. if 'build_ext' not in cmdclass: cmdclass['build_ext'] = BuildExtension.with_options( - no_python_abi_suffix=True) + no_python_abi_suffix=True + ) attr['cmdclass'] = cmdclass error_msg = """ @@ -167,17 +196,19 @@ def setup(**attr): if 'name' not in attr: raise ValueError(error_msg) - assert not attr['name'].endswith('module'), \ - "Please don't use 'module' as suffix in `name` argument, " + assert not attr['name'].endswith( + 'module' + ), "Please don't use 'module' as suffix in `name` argument, " "it will be stripped in setuptools.bdist_egg and cause import error." ext_modules = attr.get('ext_modules', []) if not isinstance(ext_modules, list): ext_modules = [ext_modules] - assert len( - ext_modules - ) == 1, "Required only one Extension, but received {}. If you want to compile multi operators, you can include all necessary source files in one Extension.".format( - len(ext_modules)) + assert ( + len(ext_modules) == 1 + ), "Required only one Extension, but received {}. If you want to compile multi operators, you can include all necessary source files in one Extension.".format( + len(ext_modules) + ) # replace Extension.name with attr['name] to keep consistant with Package name. for ext_module in ext_modules: ext_module.name = attr['name'] @@ -335,7 +366,6 @@ def with_options(cls, **options): """ class cls_with_options(cls): - def __init__(self, *args, **kwargs): kwargs.update(options) cls.__init__(self, *args, **kwargs) @@ -380,8 +410,9 @@ def build_extensions(self): # cflags have changed and delete the built shared library to re-compile the source # even though source file content keep unchanged. so_name = self.get_ext_fullpath(self.extensions[0].name) - clean_object_if_change_cflags(os.path.abspath(so_name), - self.extensions[0]) + clean_object_if_change_cflags( + os.path.abspath(so_name), self.extensions[0] + ) # Consider .cu, .cu.cc as valid source extensions. self.compiler.src_extensions += ['.cu', '.cu.cc'] @@ -393,8 +424,9 @@ def build_extensions(self): else: original_compile = self.compiler._compile - def unix_custom_single_compiler(obj, src, ext, cc_args, extra_postargs, - pp_opts): + def unix_custom_single_compiler( + obj, src, ext, cc_args, extra_postargs, pp_opts + ): """ Monkey patch machanism to replace inner compiler to custom complie process on Unix platform. """ @@ -407,7 +439,9 @@ def unix_custom_single_compiler(obj, src, ext, cc_args, extra_postargs, # nvcc or hipcc compile CUDA source if is_cuda_file(src): if core.is_compiled_with_rocm(): - assert ROCM_HOME is not None, "Not found ROCM runtime, \ + assert ( + ROCM_HOME is not None + ), "Not found ROCM runtime, \ please use `export ROCM_PATH= XXX` to specify it." hipcc_cmd = os.path.join(ROCM_HOME, 'bin', 'hipcc') @@ -416,7 +450,9 @@ def unix_custom_single_compiler(obj, src, ext, cc_args, extra_postargs, if isinstance(cflags, dict): cflags = cflags['hipcc'] else: - assert CUDA_HOME is not None, "Not found CUDA runtime, \ + assert ( + CUDA_HOME is not None + ), "Not found CUDA runtime, \ please use `export CUDA_HOME= XXX` to specify it." nvcc_cmd = os.path.join(CUDA_HOME, 'bin', 'nvcc') @@ -435,7 +471,8 @@ def unix_custom_single_compiler(obj, src, ext, cc_args, extra_postargs, cflags.append('-D__HIP_PLATFORM_HCC__') cflags.append('-D__HIP_NO_HALF_CONVERSIONS__=1') cflags.append( - '-DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_HIP') + '-DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_HIP' + ) # NOTE(Aurelius84): Since Paddle 2.0, we require gcc version > 5.x, # so we add this flag to ensure the symbol names from user compiled @@ -449,22 +486,24 @@ def unix_custom_single_compiler(obj, src, ext, cc_args, extra_postargs, else: cflags.append('-DPADDLE_WITH_CUDA') - add_std_without_repeat(cflags, - self.compiler.compiler_type, - use_std14=True) + add_std_without_repeat( + cflags, self.compiler.compiler_type, use_std14=True + ) original_compile(obj, src, ext, cc_args, cflags, pp_opts) finally: # restore original_compiler self.compiler.set_executable('compiler_so', original_compiler) - def win_custom_single_compiler(sources, - output_dir=None, - macros=None, - include_dirs=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - depends=None): + def win_custom_single_compiler( + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None, + ): self.cflags = copy.deepcopy(extra_postargs) extra_postargs = None @@ -481,27 +520,32 @@ def win_custom_spawn(cmd): # Using regex to match src, obj and include files src_regex = re.compile('/T(p|c)(.*)') src_list = [ - m.group(2) for m in (src_regex.match(elem) for elem in cmd) + m.group(2) + for m in (src_regex.match(elem) for elem in cmd) if m ] obj_regex = re.compile('/Fo(.*)') obj_list = [ - m.group(1) for m in (obj_regex.match(elem) for elem in cmd) + m.group(1) + for m in (obj_regex.match(elem) for elem in cmd) if m ] include_regex = re.compile(r'((\-|\/)I.*)') include_list = [ m.group(1) - for m in (include_regex.match(elem) for elem in cmd) if m + for m in (include_regex.match(elem) for elem in cmd) + if m ] assert len(src_list) == 1 and len(obj_list) == 1 src = src_list[0] obj = obj_list[0] if is_cuda_file(src): - assert CUDA_HOME is not None, "Not found CUDA runtime, \ + assert ( + CUDA_HOME is not None + ), "Not found CUDA runtime, \ please use `export CUDA_HOME= XXX` to specify it." nvcc_cmd = os.path.join(CUDA_HOME, 'bin', 'nvcc') @@ -515,8 +559,9 @@ def win_custom_spawn(cmd): cflags = prepare_win_cudaflags(cflags) + ['--use-local-env'] for flag in MSVC_COMPILE_FLAGS: cflags = ['-Xcompiler', flag] + cflags - cmd = [nvcc_cmd, '-c', src, '-o', obj - ] + include_list + cflags + cmd = ( + [nvcc_cmd, '-c', src, '-o', obj] + include_list + cflags + ) elif isinstance(self.cflags, dict): cflags = MSVC_COMPILE_FLAGS + self.cflags['cxx'] cmd += cflags @@ -531,9 +576,16 @@ def win_custom_spawn(cmd): try: self.compiler.spawn = win_custom_spawn - return original_compile(sources, output_dir, macros, - include_dirs, debug, extra_preargs, - extra_postargs, depends) + return original_compile( + sources, + output_dir, + macros, + include_dirs, + debug, + extra_preargs, + extra_postargs, + depends, + ) finally: self.compiler.spawn = original_spawn @@ -546,8 +598,9 @@ def object_filenames_with_cuda(origina_func, build_directory): def wrapper(source_filenames, strip_dir=0, output_dir=''): try: - objects = origina_func(source_filenames, strip_dir, - output_dir) + objects = origina_func( + source_filenames, strip_dir, output_dir + ) for i, source in enumerate(source_filenames): # modify xx.o -> xx.cu.o/xx.cu.obj if is_cuda_file(source): @@ -578,7 +631,8 @@ def wrapper(source_filenames, strip_dir=0, output_dir=''): self.compiler._compile = unix_custom_single_compiler self.compiler.object_filenames = object_filenames_with_cuda( - self.compiler.object_filenames, self.build_lib) + self.compiler.object_filenames, self.build_lib + ) self._record_op_info() print("Compiling user custom op, it will cost a few seconds.....") @@ -594,10 +648,11 @@ def get_ext_filename(self, fullname): split_str = '.' name_items = ext_name.split(split_str) if self.no_python_abi_suffix: - assert len( - name_items - ) > 2, "Expected len(name_items) > 2, but received {}".format( - len(name_items)) + assert ( + len(name_items) > 2 + ), "Expected len(name_items) > 2, but received {}".format( + len(name_items) + ) name_items.pop(-2) ext_name = split_str.join(name_items) @@ -613,11 +668,13 @@ def _valid_clang_compiler(self): """ compiler_infos = ['clang'] + CLANG_COMPILE_FLAGS linker_infos = ['clang'] + CLANG_LINK_FLAGS - self.compiler.set_executables(compiler=compiler_infos, - compiler_so=compiler_infos, - compiler_cxx=['clang'], - linker_exe=['clang'], - linker_so=linker_infos) + self.compiler.set_executables( + compiler=compiler_infos, + compiler_so=compiler_infos, + compiler_cxx=['clang'], + linker_exe=['clang'], + linker_so=linker_infos, + ) def _check_abi(self): """ @@ -632,11 +689,16 @@ def _check_abi(self): check_abi_compatibility(compiler) # Warn user if VC env is activated but `DISTUTILS_USE_SDK` is not set. - if IS_WINDOWS and 'VSCMD_ARG_TGT_ARCH' in os.environ and 'DISTUTILS_USE_SDK' not in os.environ: + if ( + IS_WINDOWS + and 'VSCMD_ARG_TGT_ARCH' in os.environ + and 'DISTUTILS_USE_SDK' not in os.environ + ): msg = ( 'It seems that the VC environment is activated but DISTUTILS_USE_SDK is not set.' 'This may lead to multiple activations of the VC env.' - 'Please run `set DISTUTILS_USE_SDK=1` and try again.') + 'Please run `set DISTUTILS_USE_SDK=1` and try again.' + ) raise UserWarning(msg) def _record_op_info(self): @@ -657,9 +719,9 @@ def _record_op_info(self): op_names = parse_op_name_from(sources) for op_name in op_names: - CustomOpInfo.instance().add(op_name, - so_name=so_name, - so_path=so_path) + CustomOpInfo.instance().add( + op_name, so_name=so_name, so_path=so_path + ) class EasyInstallCommand(easy_install, object): @@ -712,7 +774,6 @@ def with_options(cls, **options): """ class cls_with_options(cls): - def __init__(self, *args, **kwargs): kwargs.update(options) cls.__init__(self, *args, **kwargs) @@ -735,14 +796,16 @@ def initialize_options(self): self.build_base = self._specified_build_base -def load(name, - sources, - extra_cxx_cflags=None, - extra_cuda_cflags=None, - extra_ldflags=None, - extra_include_paths=None, - build_directory=None, - verbose=False): +def load( + name, + sources, + extra_cxx_cflags=None, + extra_cuda_cflags=None, + extra_ldflags=None, + extra_include_paths=None, + build_directory=None, + verbose=False, +): """ An Interface to automatically compile C++/CUDA source files Just-In-Time and return callable python function as other Paddle layers API. It will @@ -836,27 +899,42 @@ def load(name, file_path = os.path.join(build_directory, "{}_setup.py".format(name)) sources = [os.path.abspath(source) for source in sources] - if extra_cxx_cflags is None: extra_cxx_cflags = [] - if extra_cuda_cflags is None: extra_cuda_cflags = [] + if extra_cxx_cflags is None: + extra_cxx_cflags = [] + if extra_cuda_cflags is None: + extra_cuda_cflags = [] assert isinstance( extra_cxx_cflags, list ), "Required type(extra_cxx_cflags) == list[str], but received {}".format( - extra_cxx_cflags) + extra_cxx_cflags + ) assert isinstance( extra_cuda_cflags, list ), "Required type(extra_cuda_cflags) == list[str], but received {}".format( - extra_cuda_cflags) + extra_cuda_cflags + ) log_v( "additional extra_cxx_cflags: [{}], extra_cuda_cflags: [{}]".format( - ' '.join(extra_cxx_cflags), ' '.join(extra_cuda_cflags)), verbose) + ' '.join(extra_cxx_cflags), ' '.join(extra_cuda_cflags) + ), + verbose, + ) # write setup.py file and compile it build_base_dir = os.path.join(build_directory, name) - _write_setup_file(name, sources, file_path, build_base_dir, - extra_include_paths, extra_cxx_cflags, extra_cuda_cflags, - extra_ldflags, verbose) + _write_setup_file( + name, + sources, + file_path, + build_base_dir, + extra_include_paths, + extra_cxx_cflags, + extra_cuda_cflags, + extra_ldflags, + verbose, + ) _jit_compile(file_path, verbose) # import as callable python api diff --git a/python/paddle/utils/cpp_extension/extension_utils.py b/python/paddle/utils/cpp_extension/extension_utils.py index 8b07abe7c8501..57a1ae1f0f581 100644 --- a/python/paddle/utils/cpp_extension/extension_utils.py +++ b/python/paddle/utils/cpp_extension/extension_utils.py @@ -50,23 +50,46 @@ IS_WINDOWS = OS_NAME.startswith('win') MSVC_COMPILE_FLAGS = [ - '/MT', '/wd4819', '/wd4251', '/wd4244', '/wd4267', '/wd4275', '/wd4018', - '/wd4190', '/EHsc', '/w', '/DGOOGLE_GLOG_DLL_DECL', - '/DBOOST_HAS_STATIC_ASSERT', '/DNDEBUG', '/DPADDLE_USE_DSO' + '/MT', + '/wd4819', + '/wd4251', + '/wd4244', + '/wd4267', + '/wd4275', + '/wd4018', + '/wd4190', + '/EHsc', + '/w', + '/DGOOGLE_GLOG_DLL_DECL', + '/DBOOST_HAS_STATIC_ASSERT', + '/DNDEBUG', + '/DPADDLE_USE_DSO', ] CLANG_COMPILE_FLAGS = [ - '-fno-common', '-dynamic', '-DNDEBUG', '-g', '-fwrapv', '-O3', '-arch', - 'x86_64' + '-fno-common', + '-dynamic', + '-DNDEBUG', + '-g', + '-fwrapv', + '-O3', + '-arch', + 'x86_64', ] CLANG_LINK_FLAGS = [ - '-dynamiclib', '-undefined', 'dynamic_lookup', '-arch', 'x86_64' + '-dynamiclib', + '-undefined', + 'dynamic_lookup', + '-arch', + 'x86_64', ] MSVC_LINK_FLAGS = ['/MACHINE:X64'] if core.is_compiled_with_rocm(): COMMON_HIPCC_FLAGS = [ - '-DPADDLE_WITH_HIP', '-DEIGEN_USE_GPU', '-DEIGEN_USE_HIP' + '-DPADDLE_WITH_HIP', + '-DEIGEN_USE_GPU', + '-DEIGEN_USE_HIP', ] else: COMMON_NVCC_FLAGS = ['-DPADDLE_WITH_CUDA', '-DEIGEN_USE_GPU'] @@ -114,7 +137,7 @@ core.op_proto_and_checker_maker.kOpNameScopeAttrName(), core.op_proto_and_checker_maker.kOpCreationCallstackAttrName(), core.op_proto_and_checker_maker.kOpDeviceAttrName(), - core.op_proto_and_checker_maker.kOpWithQuantAttrName() + core.op_proto_and_checker_maker.kOpWithQuantAttrName(), ] @@ -140,7 +163,8 @@ def custom_write_stub(resource, pyfile): Customized write_stub function to allow us to inject generated python api codes into egg python file. """ - _stub_template = textwrap.dedent(""" + _stub_template = textwrap.dedent( + """ import os import sys import types @@ -170,17 +194,18 @@ def __bootstrap__(): {custom_api} - """).lstrip() + """ + ).lstrip() # Parse registerring op information _, op_info = CustomOpInfo.instance().last() so_path = op_info.so_path new_custom_ops = load_op_meta_info_and_register_op(so_path) - assert len( - new_custom_ops - ) > 0, "Required at least one custom operators, but received len(custom_op) = %d" % len( - new_custom_ops) + assert len(new_custom_ops) > 0, ( + "Required at least one custom operators, but received len(custom_op) = %d" + % len(new_custom_ops) + ) # NOTE: To avoid importing .so file instead of python file because they have same name, # we rename .so shared library to another name, see EasyInstallCommand. @@ -193,8 +218,10 @@ def __bootstrap__(): with open(pyfile, 'w') as f: f.write( - _stub_template.format(resource=resource, - custom_api='\n\n'.join(api_content))) + _stub_template.format( + resource=resource, custom_api='\n\n'.join(api_content) + ) + ) OpInfo = collections.namedtuple('OpInfo', ['so_name', 'so_path']) @@ -213,8 +240,8 @@ def instance(cls): def __init__(self): assert not hasattr( - self.__class__, - '_instance'), 'Please use `instance()` to get CustomOpInfo object!' + self.__class__, '_instance' + ), 'Please use `instance()` to get CustomOpInfo object!' # NOTE(Aurelius84): Use OrderedDict to save more order information self.op_info_map = collections.OrderedDict() @@ -229,20 +256,22 @@ def last(self): return next(reversed(self.op_info_map.items())) -VersionFields = collections.namedtuple('VersionFields', [ - 'sources', - 'extra_compile_args', - 'extra_link_args', - 'library_dirs', - 'runtime_library_dirs', - 'include_dirs', - 'define_macros', - 'undef_macros', -]) +VersionFields = collections.namedtuple( + 'VersionFields', + [ + 'sources', + 'extra_compile_args', + 'extra_link_args', + 'library_dirs', + 'runtime_library_dirs', + 'include_dirs', + 'define_macros', + 'undef_macros', + ], +) class VersionManager: - def __init__(self, version_field): self.version_field = version_field self.version = self.hasher(version_field) @@ -253,14 +282,17 @@ def hasher(self, version_field): md5 = hashlib.md5() for field in version_field._fields: elem = getattr(version_field, field) - if not elem: continue + if not elem: + continue if isinstance(elem, (list, tuple, dict)): flat_elem = flatten(elem) md5 = combine_hash(md5, tuple(flat_elem)) else: raise RuntimeError( - "Support types with list, tuple and dict, but received {} with {}." - .format(type(elem), elem)) + "Support types with list, tuple and dict, but received {} with {}.".format( + type(elem), elem + ) + ) return md5.hexdigest() @@ -314,8 +346,10 @@ def deserialize(path): # delete shared library file if version is changed to re-compile it. if so_version is not None and so_version != versioner.version: log_v( - "Re-Compiling {}, because specified cflags have been changed. New signature {} has been saved into {}." - .format(so_name, versioner.version, version_file)) + "Re-Compiling {}, because specified cflags have been changed. New signature {} has been saved into {}.".format( + so_name, versioner.version, version_file + ) + ) os.remove(so_path) # update new version information new_version_info = versioner.details @@ -335,13 +369,26 @@ def prepare_unix_cudaflags(cflags): Prepare all necessary compiled flags for nvcc compiling CUDA files. """ if core.is_compiled_with_rocm(): - cflags = COMMON_HIPCC_FLAGS + ['-Xcompiler', '-fPIC' - ] + cflags + get_rocm_arch_flags(cflags) + cflags = ( + COMMON_HIPCC_FLAGS + + ['-Xcompiler', '-fPIC'] + + cflags + + get_rocm_arch_flags(cflags) + ) else: - cflags = COMMON_NVCC_FLAGS + [ - '-ccbin', 'cc', '-Xcompiler', '-fPIC', '--expt-relaxed-constexpr', - '-DNVCC' - ] + cflags + get_cuda_arch_flags(cflags) + cflags = ( + COMMON_NVCC_FLAGS + + [ + '-ccbin', + 'cc', + '-Xcompiler', + '-fPIC', + '--expt-relaxed-constexpr', + '-DNVCC', + ] + + cflags + + get_cuda_arch_flags(cflags) + ) return cflags @@ -390,6 +437,7 @@ def _get_fluid_path(): Return installed fluid dir path. """ import paddle + return os.path.join(os.path.dirname(paddle.__file__), 'fluid') @@ -434,7 +482,8 @@ def _reset_so_rpath(so_path): origin_runtime_path = "@loader_path/../libs/" rpath = "@rpath/{}".format(_get_core_name()) cmd = 'install_name_tool -change {} {} {}'.format( - origin_runtime_path, rpath, so_path) + origin_runtime_path, rpath, so_path + ) run_cmd(cmd) @@ -448,7 +497,8 @@ def _get_include_dirs_when_compiling(compile_dir): path = os.path.abspath(compile_dir) include_dirs_file = os.path.join(path, include_dirs_file) assert os.path.isfile(include_dirs_file), "File {} does not exist".format( - include_dirs_file) + include_dirs_file + ) with open(include_dirs_file, 'r') as f: include_dirs = [line.strip() for line in f.readlines() if line.strip()] @@ -566,9 +616,13 @@ def create_sym_link_if_not_exist(): except Exception: warnings.warn( "Failed to create soft symbol link for {}.\n You can run prompt as administrator and execute the " - "following command manually: `mklink {} {}`. Now it will create hard link for {} trickly." - .format(raw_core_name, new_dll_core_path, core_path, - raw_core_name)) + "following command manually: `mklink {} {}`. Now it will create hard link for {} trickly.".format( + raw_core_name, + new_dll_core_path, + core_path, + raw_core_name, + ) + ) run_cmd('mklink /H {} {}'.format(new_dll_core_path, core_path)) # libpaddle with lib suffix assert os.path.exists(new_dll_core_path) @@ -583,8 +637,10 @@ def create_sym_link_if_not_exist(): assert os.path.exists(new_lib_core_path) except Exception: raise RuntimeError( - "Failed to create soft symbol link for {}.\n Please execute the following command manually: `ln -s {} {}`" - .format(raw_core_name, core_path, new_lib_core_path)) + "Failed to create soft symbol link for {}.\n Please execute the following command manually: `ln -s {} {}`".format( + raw_core_name, core_path, new_lib_core_path + ) + ) # libpaddle without suffix return raw_core_name[:-3] @@ -602,8 +658,9 @@ def find_cuda_home(): which_cmd = 'where' if IS_WINDOWS else 'which' try: with open(os.devnull, 'w') as devnull: - nvcc_path = subprocess.check_output([which_cmd, 'nvcc'], - stderr=devnull) + nvcc_path = subprocess.check_output( + [which_cmd, 'nvcc'], stderr=devnull + ) nvcc_path = nvcc_path.decode() # Multi CUDA, select the first nvcc_path = nvcc_path.split('\r\n')[0] @@ -621,8 +678,11 @@ def find_cuda_home(): else: cuda_home = "/usr/local/cuda" # step 3. check whether path is valid - if cuda_home and not os.path.exists( - cuda_home) and core.is_compiled_with_cuda(): + if ( + cuda_home + and not os.path.exists(cuda_home) + and core.is_compiled_with_cuda() + ): cuda_home = None return cuda_home @@ -640,8 +700,9 @@ def find_rocm_home(): which_cmd = 'where' if IS_WINDOWS else 'which' try: with open(os.devnull, 'w') as devnull: - hipcc_path = subprocess.check_output([which_cmd, 'hipcc'], - stderr=devnull) + hipcc_path = subprocess.check_output( + [which_cmd, 'hipcc'], stderr=devnull + ) hipcc_path = hipcc_path.decode() hipcc_path = hipcc_path.rstrip('\r\n') @@ -650,8 +711,11 @@ def find_rocm_home(): except: rocm_home = "/opt/rocm" # step 3. check whether path is valid - if rocm_home and not os.path.exists( - rocm_home) and core.is_compiled_with_rocm(): + if ( + rocm_home + and not os.path.exists(rocm_home) + and core.is_compiled_with_rocm() + ): rocm_home = None return rocm_home @@ -719,8 +783,9 @@ def find_clang_cpp_include(compiler='clang'): if "InstalledDir" in info: v1_path = info.split(':')[-1].strip() if v1_path and os.path.exists(v1_path): - std_v1_includes = os.path.join(os.path.dirname(v1_path), - 'include/c++/v1') + std_v1_includes = os.path.join( + os.path.dirname(v1_path), 'include/c++/v1' + ) except Exception: # Just raise warnings because the include dir is not required. warnings.warn( @@ -820,15 +885,20 @@ def get_build_directory(verbose=False): root_extensions_directory = os.environ.get('PADDLE_EXTENSION_DIR') if root_extensions_directory is None: dir_name = "paddle_extensions" - root_extensions_directory = os.path.join(os.path.expanduser('~/.cache'), - dir_name) + root_extensions_directory = os.path.join( + os.path.expanduser('~/.cache'), dir_name + ) if IS_WINDOWS: root_extensions_directory = os.path.normpath( - root_extensions_directory) + root_extensions_directory + ) log_v( - "$PADDLE_EXTENSION_DIR is not set, using path: {} by default.". - format(root_extensions_directory), verbose) + "$PADDLE_EXTENSION_DIR is not set, using path: {} by default.".format( + root_extensions_directory + ), + verbose, + ) if not os.path.exists(root_extensions_directory): os.makedirs(root_extensions_directory) @@ -843,8 +913,10 @@ def parse_op_info(op_name): """ if op_name not in OpProtoHolder.instance().op_proto_map: raise ValueError( - "Please load {} shared library file firstly by `paddle.utils.cpp_extension.load_op_meta_info_and_register_op(...)`" - .format(op_name)) + "Please load {} shared library file firstly by `paddle.utils.cpp_extension.load_op_meta_info_and_register_op(...)`".format( + op_name + ) + ) op_proto = OpProtoHolder.instance().get_op_proto(op_name) in_names = [x.name for x in op_proto.inputs] @@ -869,21 +941,22 @@ def _import_module_from_library(module_name, build_directory, verbose=False): ext_path = os.path.join(build_directory, module_name + dynamic_suffix) if not os.path.exists(ext_path): raise FileNotFoundError( - "Extension path: {} does not exist.".format(ext_path)) + "Extension path: {} does not exist.".format(ext_path) + ) # load custom op_info and kernels from .so shared library log_v('loading shared library from: {}'.format(ext_path), verbose) op_names = load_op_meta_info_and_register_op(ext_path) # generate Python api in ext_path - return _generate_python_module(module_name, op_names, build_directory, - verbose) + return _generate_python_module( + module_name, op_names, build_directory, verbose + ) -def _generate_python_module(module_name, - op_names, - build_directory, - verbose=False): +def _generate_python_module( + module_name, op_names, build_directory, verbose=False +): """ Automatically generate python file to allow import or load into as module """ @@ -895,8 +968,9 @@ def remove_if_exit(filepath): # NOTE: Use unique id as suffix to avoid write same file at same time in # both multi-thread and multi-process. thread_id = str(threading.currentThread().ident) - api_file = os.path.join(build_directory, - module_name + '_' + thread_id + '.py') + api_file = os.path.join( + build_directory, module_name + '_' + thread_id + '.py' + ) log_v("generate api file: {}".format(api_file), verbose) # delete the temp file before exit python process @@ -913,10 +987,17 @@ def remove_if_exit(filepath): def _custom_api_content(op_name): - params_str, ins_str, attrs_str, outs_str, in_names, attrs_names = _get_api_inputs_str( - op_name) + ( + params_str, + ins_str, + attrs_str, + outs_str, + in_names, + attrs_names, + ) = _get_api_inputs_str(op_name) lower_in_names = [p.split("@")[0].lower() for p in in_names] - API_TEMPLATE = textwrap.dedent(""" + API_TEMPLATE = textwrap.dedent( + """ import paddle.fluid.core as core from paddle.fluid.core import VarBase, CustomOpKernelContext from paddle.fluid.framework import _non_static_mode, _dygraph_tracer, _in_legacy_dygraph, in_dygraph_mode @@ -956,7 +1037,8 @@ def {op_name}({inputs}): res = [outs[out_name] for out_name in out_names] return res[0] if len(res)==1 else res - """).lstrip() + """ + ).lstrip() # generate python api file api_content = API_TEMPLATE.format( @@ -967,7 +1049,8 @@ def {op_name}({inputs}): # "[x, y, z]"" in_names="[" + ",".join(lower_in_names) + "]", attr_names="[" + ",".join(attrs_names) + "]", - out_names=outs_str) + out_names=outs_str, + ) return api_content @@ -978,7 +1061,8 @@ def _load_module_from_file(api_file_path, module_name, verbose=False): """ if not os.path.exists(api_file_path): raise FileNotFoundError( - "File : {} does not exist.".format(api_file_path)) + "File : {} does not exist.".format(api_file_path) + ) # Unique readable module name to place custom api. log_v('import module from file: {}'.format(api_file_path), verbose) @@ -1003,35 +1087,40 @@ def _get_api_inputs_str(op_name): # input name by `@`, and only use first substr as argument params_str = ','.join([p.split("@")[0].lower() for p in param_names]) # e.g: {'X': x, 'Y': y, 'Z': z} - ins_str = "{%s}" % ','.join([ - "'{}' : {}".format(in_name, - in_name.split("@")[0].lower()) - for in_name in in_names - ]) + ins_str = "{%s}" % ','.join( + [ + "'{}' : {}".format(in_name, in_name.split("@")[0].lower()) + for in_name in in_names + ] + ) # e.g: {'num': n} - attrs_str = "{%s}" % ",".join([ - "'{}' : {}".format(attr_name, - attr_name.split("@")[0].lower()) - for attr_name in attr_names - ]) + attrs_str = "{%s}" % ",".join( + [ + "'{}' : {}".format(attr_name, attr_name.split("@")[0].lower()) + for attr_name in attr_names + ] + ) # e.g: ['Out', 'Index'] outs_str = "[%s]" % ','.join(["'{}'".format(name) for name in out_names]) return params_str, ins_str, attrs_str, outs_str, in_names, attr_names -def _write_setup_file(name, - sources, - file_path, - build_dir, - include_dirs, - extra_cxx_cflags, - extra_cuda_cflags, - link_args, - verbose=False): +def _write_setup_file( + name, + sources, + file_path, + build_dir, + include_dirs, + extra_cxx_cflags, + extra_cuda_cflags, + link_args, + verbose=False, +): """ Automatically generate setup.py and write it into build directory. """ - template = textwrap.dedent(""" + template = textwrap.dedent( + """ import os from paddle.utils.cpp_extension import CppExtension, CUDAExtension, BuildExtension, setup from paddle.utils.cpp_extension import get_build_directory @@ -1048,21 +1137,24 @@ def _write_setup_file(name, cmdclass={{"build_ext" : BuildExtension.with_options( output_dir=r'{build_dir}', no_python_abi_suffix=True) - }})""").lstrip() + }})""" + ).lstrip() with_cuda = False if any([is_cuda_file(source) for source in sources]): with_cuda = True log_v("with_cuda: {}".format(with_cuda), verbose) - content = template.format(name=name, - prefix='CUDA' if with_cuda else 'Cpp', - sources=list2str(sources), - include_dirs=list2str(include_dirs), - extra_cxx_cflags=list2str(extra_cxx_cflags), - extra_cuda_cflags=list2str(extra_cuda_cflags), - extra_link_args=list2str(link_args), - build_dir=build_dir) + content = template.format( + name=name, + prefix='CUDA' if with_cuda else 'Cpp', + sources=list2str(sources), + include_dirs=list2str(include_dirs), + extra_cxx_cflags=list2str(extra_cxx_cflags), + extra_cuda_cflags=list2str(extra_cuda_cflags), + extra_link_args=list2str(link_args), + build_dir=build_dir, + ) log_v('write setup.py into {}'.format(file_path), verbose) with open(file_path, 'w') as f: @@ -1073,7 +1165,8 @@ def list2str(args): """ Convert list[str] into string. For example: ['x', 'y'] -> "['x', 'y']" """ - if args is None: return '[]' + if args is None: + return '[]' assert isinstance(args, (list, tuple)) args = ["{}".format(arg) for arg in args] return repr(args) @@ -1094,19 +1187,26 @@ def _jit_compile(file_path, verbose=False): py_version = py_version.decode() log_v( "Using Python interpreter: {}, version: {}".format( - interpreter, py_version.strip()), verbose) + interpreter, py_version.strip() + ), + verbose, + ) except Exception: _, error, _ = sys.exc_info() raise RuntimeError( 'Failed to check Python interpreter with `{}`, errors: {}'.format( - interpreter, error)) + interpreter, error + ) + ) if IS_WINDOWS: - compile_cmd = 'cd /d {} && {} {} build'.format(ext_dir, interpreter, - setup_file) + compile_cmd = 'cd /d {} && {} {} build'.format( + ext_dir, interpreter, setup_file + ) else: - compile_cmd = 'cd {} && {} {} build'.format(ext_dir, interpreter, - setup_file) + compile_cmd = 'cd {} && {} {} build'.format( + ext_dir, interpreter, setup_file + ) print("Compiling user custom op, it will cost a few seconds.....") run_cmd(compile_cmd, verbose) @@ -1144,15 +1244,16 @@ def run_cmd(command, verbose=False): # execute command try: if verbose: - return subprocess.check_call(command, - shell=True, - stderr=subprocess.STDOUT) + return subprocess.check_call( + command, shell=True, stderr=subprocess.STDOUT + ) else: return subprocess.check_call(command, shell=True, stdout=DEVNULL) except Exception: _, error, _ = sys.exc_info() - raise RuntimeError("Failed to run command: {}, errors: {}".format( - compile, error)) + raise RuntimeError( + "Failed to run command: {}, errors: {}".format(compile, error) + ) def check_abi_compatibility(compiler, verbose=False): @@ -1164,17 +1265,22 @@ def check_abi_compatibility(compiler, verbose=False): return True if not IS_WINDOWS: - cmd_out = subprocess.check_output(['which', compiler], - stderr=subprocess.STDOUT) + cmd_out = subprocess.check_output( + ['which', compiler], stderr=subprocess.STDOUT + ) compiler_path = os.path.realpath(cmd_out.decode()).strip() # if not found any suitable compiler, raise warning - if not any(name in compiler_path - for name in _expected_compiler_current_platform()): + if not any( + name in compiler_path + for name in _expected_compiler_current_platform() + ): warnings.warn( WRONG_COMPILER_WARNING.format( user_compiler=compiler, paddle_compiler=_expected_compiler_current_platform()[0], - platform=OS_NAME)) + platform=OS_NAME, + ) + ) return False version = (0, 0, 0) @@ -1185,13 +1291,15 @@ def check_abi_compatibility(compiler, verbose=False): if OS_NAME.startswith('linux'): mini_required_version = GCC_MINI_VERSION version_info = subprocess.check_output( - [compiler, '-dumpfullversion', '-dumpversion']) + [compiler, '-dumpfullversion', '-dumpversion'] + ) version_info = version_info.decode() version = version_info.strip().split('.') elif IS_WINDOWS: mini_required_version = MSVC_MINI_VERSION - compiler_info = subprocess.check_output(compiler, - stderr=subprocess.STDOUT) + compiler_info = subprocess.check_output( + compiler, stderr=subprocess.STDOUT + ) try: compiler_info = compiler_info.decode('UTF-8') except UnicodeDecodeError: @@ -1202,8 +1310,11 @@ def check_abi_compatibility(compiler, verbose=False): except Exception: # check compiler version failed _, error, _ = sys.exc_info() - warnings.warn('Failed to check compiler version for {}: {}'.format( - compiler, error)) + warnings.warn( + 'Failed to check compiler version for {}: {}'.format( + compiler, error + ) + ) return False # check version compatibility @@ -1211,8 +1322,10 @@ def check_abi_compatibility(compiler, verbose=False): if tuple(map(int, version)) >= mini_required_version: return True warnings.warn( - ABI_INCOMPATIBILITY_WARNING.format(user_compiler=compiler, - version='.'.join(version))) + ABI_INCOMPATIBILITY_WARNING.format( + user_compiler=compiler, version='.'.join(version) + ) + ) return False diff --git a/python/paddle/utils/deprecated.py b/python/paddle/utils/deprecated.py index 00a38cb1368de..b4c17de3f178f 100755 --- a/python/paddle/utils/deprecated.py +++ b/python/paddle/utils/deprecated.py @@ -34,23 +34,23 @@ def deprecated(update_to="", since="", reason="", level=0): """Decorate a function to signify its deprecation. - This function wraps a method that will soon be removed and does two things: - - The docstring of the API will be modified to include a notice - about deprecation." - - Raises a :class:`~exceptions.DeprecatedWarning` when old API is called. - - Args: - since(str, optional): The version at which the decorated method is considered deprecated. - update_to(str, optional): The new API users should use. - reason(str, optional): The reason why the API is deprecated. - level(int, optional): The deprecated warning log level. It must be - an Integer and must be one of 0, 1, 2. - If `level == 0`, the warning message will not be showed. - If `level == 1`, the warning message will be showed normally. - If `level == 2`, it will raise `RuntimeError`. - - Returns: - decorator: decorated function or class. + This function wraps a method that will soon be removed and does two things: + - The docstring of the API will be modified to include a notice + about deprecation." + - Raises a :class:`~exceptions.DeprecatedWarning` when old API is called. + + Args: + since(str, optional): The version at which the decorated method is considered deprecated. + update_to(str, optional): The new API users should use. + reason(str, optional): The reason why the API is deprecated. + level(int, optional): The deprecated warning log level. It must be + an Integer and must be one of 0, 1, 2. + If `level == 0`, the warning message will not be showed. + If `level == 1`, the warning message will be showed normally. + If `level == 2`, it will raise `RuntimeError`. + + Returns: + decorator: decorated function or class. """ def decorator(func): @@ -62,7 +62,8 @@ def decorator(func): assert isinstance(reason, str), 'type of "reason" must be str.' assert isinstance(level, int) and level >= 0 and level < 3, ( 'type of "level" must be int and must be one of 0, 1, 2. But ' - 'received: {}.'.format(level)) + 'received: {}.'.format(level) + ) _since = since.strip() _update_to = update_to.strip() @@ -77,7 +78,8 @@ def decorator(func): assert _update_to.startswith( "paddle." ), 'Argument update_to must start with "paddle.", your value is "{}"'.format( - update_to) + update_to + ) msg += ' Please use "{}" instead.'.format(_update_to) if len(_reason) > 0: msg += "\nreason: {}".format(_reason) @@ -90,14 +92,17 @@ def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): """deprecated warning should be fired in 3 circumstances: - 1. current version is develop version, i.e. "0.0.0", because we assume develop version is always the latest version. - 2. since version is empty, in this case, API is deprecated in all versions. - 3. current version is newer than since version. + 1. current version is develop version, i.e. "0.0.0", because we assume develop version is always the latest version. + 2. since version is empty, in this case, API is deprecated in all versions. + 3. current version is newer than since version. """ if level == 2: - raise RuntimeError('API "{}.{}" has been deprecated.'.format( - func.__module__, func.__name__)) + raise RuntimeError( + 'API "{}.{}" has been deprecated.'.format( + func.__module__, func.__name__ + ) + ) warningmsg = "\033[93m\nWarning:\n%s \033[0m" % (msg) # ensure ANSI escape sequences print correctly in cmd and powershell @@ -108,10 +113,14 @@ def wrapper(*args, **kwargs): v_current += [0] * (4 - len(v_current)) v_since = [int(i) for i in _since.split(".")] v_since += [0] * (4 - len(v_since)) - if paddle.__version__ == "0.0.0" or _since == "" or v_current >= v_since: - warnings.warn(warningmsg, - category=DeprecationWarning, - stacklevel=2) + if ( + paddle.__version__ == "0.0.0" + or _since == "" + or v_current >= v_since + ): + warnings.warn( + warningmsg, category=DeprecationWarning, stacklevel=2 + ) return func(*args, **kwargs) diff --git a/python/paddle/utils/dlpack.py b/python/paddle/utils/dlpack.py index bc05cd9c29964..c0449cdcae935 100644 --- a/python/paddle/utils/dlpack.py +++ b/python/paddle/utils/dlpack.py @@ -51,7 +51,8 @@ def to_dlpack(x): if not isinstance(x, (paddle.Tensor, paddle.fluid.core.eager.Tensor)): raise TypeError( "The type of 'x' in to_dlpack must be paddle.Tensor," - " but received {}.".format(type(x))) + " but received {}.".format(type(x)) + ) return x.value().get_tensor()._to_dlpack() @@ -87,11 +88,12 @@ def from_dlpack(dlpack): """ t = type(dlpack) - dlpack_flag = (t.__module__ == 'builtins' and t.__name__ == 'PyCapsule') + dlpack_flag = t.__module__ == 'builtins' and t.__name__ == 'PyCapsule' if not dlpack_flag: raise TypeError( "The type of 'dlpack' in from_dlpack must be PyCapsule object," - " but received {}.".format(type(dlpack))) + " but received {}.".format(type(dlpack)) + ) if _non_static_mode(): out = paddle.fluid.core.from_dlpack(dlpack) diff --git a/python/paddle/utils/download.py b/python/paddle/utils/download.py index 270ad51c66f6d..a77d7b60dea50 100644 --- a/python/paddle/utils/download.py +++ b/python/paddle/utils/download.py @@ -28,7 +28,6 @@ except: class tqdm(object): - def __init__(self, total=None): self.total = total self.n = 0 @@ -38,8 +37,9 @@ def update(self, n): if self.total is None: sys.stderr.write("\r{0:.1f} bytes".format(self.n)) else: - sys.stderr.write("\r{0:.1f}%".format(100 * self.n / - float(self.total))) + sys.stderr.write( + "\r{0:.1f}%".format(100 * self.n / float(self.total)) + ) sys.stderr.flush() def __enter__(self): @@ -115,13 +115,10 @@ def _get_unique_endpoints(trainer_endpoints): return unique_endpoints -def get_path_from_url(url, - root_dir, - md5sum=None, - check_exist=True, - decompress=True, - method='get'): - """ Download from given url to root_dir. +def get_path_from_url( + url, root_dir, md5sum=None, check_exist=True, decompress=True, method='get' +): + """Download from given url to root_dir. if file or directory specified by url is exists under root_dir, return the path directly, otherwise download from url and decompress it, return the path. @@ -157,8 +154,9 @@ def get_path_from_url(url, time.sleep(1) if ParallelEnv().current_endpoint in unique_endpoints: - if decompress and (tarfile.is_tarfile(fullpath) - or zipfile.is_zipfile(fullpath)): + if decompress and ( + tarfile.is_tarfile(fullpath) or zipfile.is_zipfile(fullpath) + ): fullpath = _decompress(fullpath) return fullpath @@ -170,13 +168,18 @@ def _get_download(url, fullname): try: req = requests.get(url, stream=True) except Exception as e: # requests.exceptions.ConnectionError - logger.info("Downloading {} from {} failed with exception {}".format( - fname, url, str(e))) + logger.info( + "Downloading {} from {} failed with exception {}".format( + fname, url, str(e) + ) + ) return False if req.status_code != 200: - raise RuntimeError("Downloading from {} failed with code " - "{}!".format(url, req.status_code)) + raise RuntimeError( + "Downloading from {} failed with code " + "{}!".format(url, req.status_code) + ) # For protecting download interupted, download to # tmp_fullname firstly, move tmp_fullname to fullname @@ -202,18 +205,20 @@ def _wget_download(url, fullname): # using wget to download url tmp_fullname = fullname + "_tmp" # –user-agent - command = 'wget -O {} -t {} {}'.format(tmp_fullname, DOWNLOAD_RETRY_LIMIT, - url) - subprc = subprocess.Popen(command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + command = 'wget -O {} -t {} {}'.format( + tmp_fullname, DOWNLOAD_RETRY_LIMIT, url + ) + subprc = subprocess.Popen( + command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) _ = subprc.communicate() if subprc.returncode != 0: raise RuntimeError( - '{} failed. Please make sure `wget` is installed or {} exists'. - format(command, url)) + '{} failed. Please make sure `wget` is installed or {} exists'.format( + command, url + ) + ) shutil.move(tmp_fullname, fullname) @@ -237,7 +242,8 @@ def _download(url, path, md5sum=None, method='get'): """ assert method in _download_methods, 'make sure `{}` implemented'.format( - method) + method + ) if not osp.exists(path): os.makedirs(path) @@ -251,8 +257,9 @@ def _download(url, path, md5sum=None, method='get'): if retry_cnt < DOWNLOAD_RETRY_LIMIT: retry_cnt += 1 else: - raise RuntimeError("Download from {} failed. " - "Retry limit reached".format(url)) + raise RuntimeError( + "Download from {} failed. " "Retry limit reached".format(url) + ) if not _download_methods[method](url, fullname): time.sleep(1) @@ -273,8 +280,10 @@ def _md5check(fullname, md5sum=None): calc_md5sum = md5.hexdigest() if calc_md5sum != md5sum: - logger.info("File {} md5 check failed, {}(calc) != " - "{}(base)".format(fullname, calc_md5sum, md5sum)) + logger.info( + "File {} md5 check failed, {}(calc) != " + "{}(base)".format(fullname, calc_md5sum, md5sum) + ) return False return True @@ -314,7 +323,8 @@ def _uncompress_file_zip(filepath): elif _is_a_single_dir(file_list): # `strip(os.sep)` to remove `os.sep` in the tail of path rootpath = os.path.splitext(file_list[0].strip(os.sep))[0].split( - os.sep)[-1] + os.sep + )[-1] uncompressed_path = os.path.join(file_dir, rootpath) files.extractall(file_dir) @@ -340,7 +350,8 @@ def _uncompress_file_tar(filepath, mode="r:*"): files.extractall(file_dir) elif _is_a_single_dir(file_list): rootpath = os.path.splitext(file_list[0].strip(os.sep))[0].split( - os.sep)[-1] + os.sep + )[-1] uncompressed_path = os.path.join(file_dir, rootpath) files.extractall(file_dir) else: diff --git a/python/paddle/utils/gast/ast3.py b/python/paddle/utils/gast/ast3.py index 4696c1ba49749..e8c9ce7b01af6 100644 --- a/python/paddle/utils/gast/ast3.py +++ b/python/paddle/utils/gast/ast3.py @@ -49,10 +49,7 @@ def visit_Index(self, node): if sys.version_info.minor < 8: def visit_Module(self, node): - new_node = gast.Module( - self._visit(node.body), - [] # type_ignores - ) + new_node = gast.Module(self._visit(node.body), []) # type_ignores return new_node def visit_Num(self, node): @@ -231,7 +228,8 @@ def visit_ExceptHandler(self, node): new_node = gast.ExceptHandler( self._visit(node.type), gast.Name(node.name, gast.Store(), None, None), - self._visit(node.body)) + self._visit(node.body), + ) ast.copy_location(new_node, node) return new_node else: @@ -253,7 +251,6 @@ class GAstToAst3(GAstToAst): if sys.version_info.minor < 9: def visit_Subscript(self, node): - def adjust_slice(s): if isinstance(s, ast.Slice): return s @@ -263,7 +260,8 @@ def adjust_slice(s): if isinstance(node.slice, gast.Tuple): if any(isinstance(elt, gast.slice) for elt in node.slice.elts): new_slice = ast.ExtSlice( - [adjust_slice(x) for x in self._visit(node.slice.elts)]) + [adjust_slice(x) for x in self._visit(node.slice.elts)] + ) else: value = ast.Tuple(self._visit(node.slice.elts), ast.Load()) ast.copy_location(value, node.slice) @@ -309,10 +307,11 @@ def _make_arg(self, node): if sys.version_info.minor < 8: extra_args = tuple() else: - extra_args = self._visit(node.type_comment), + extra_args = (self._visit(node.type_comment),) - new_node = ast.arg(self._visit(node.id), self._visit(node.annotation), - *extra_args) + new_node = ast.arg( + self._visit(node.id), self._visit(node.annotation), *extra_args + ) return ast.copy_location(new_node, node) def visit_Name(self, node): @@ -325,8 +324,9 @@ def visit_Name(self, node): def visit_ExceptHandler(self, node): if node.name: - new_node = ast.ExceptHandler(self._visit(node.type), node.name.id, - self._visit(node.body)) + new_node = ast.ExceptHandler( + self._visit(node.type), node.name.id, self._visit(node.body) + ) return ast.copy_location(new_node, node) else: return self.generic_visit(node) @@ -452,10 +452,13 @@ def visit_arguments(self, node): if sys.version_info.minor >= 8: new_node = ast.arguments( [self._make_arg(arg) for arg in node.posonlyargs], - [self._make_arg(n) for n in node.args], *extra_args) + [self._make_arg(n) for n in node.args], + *extra_args + ) else: - new_node = ast.arguments([self._make_arg(n) for n in node.args], - *extra_args) + new_node = ast.arguments( + [self._make_arg(n) for n in node.args], *extra_args + ) return new_node diff --git a/python/paddle/utils/gast/astn.py b/python/paddle/utils/gast/astn.py index eb45bd4e4500a..bd88ba5efc512 100644 --- a/python/paddle/utils/gast/astn.py +++ b/python/paddle/utils/gast/astn.py @@ -34,9 +34,7 @@ def _generate_translators(to): - class Translator(ast.NodeTransformer): - def _visit(self, node): if isinstance(node, list): return [self._visit(n) for n in node] diff --git a/python/paddle/utils/gast/gast.py b/python/paddle/utils/gast/gast.py index 1248434fe3533..34ada01244080 100644 --- a/python/paddle/utils/gast/gast.py +++ b/python/paddle/utils/gast/gast.py @@ -44,557 +44,991 @@ class TypeIgnore(AST): def _make_node(Name, Fields, Attributes, Bases): - def create_node(self, *args, **kwargs): nbparam = len(args) + len(kwargs) - assert nbparam in (0, len(Fields)), \ - "Bad argument number for {}: {}, expecting {}".\ - format(Name, nbparam, len(Fields)) + assert nbparam in ( + 0, + len(Fields), + ), "Bad argument number for {}: {}, expecting {}".format( + Name, nbparam, len(Fields) + ) self._fields = Fields self._attributes = Attributes for argname, argval in zip(self._fields, args): setattr(self, argname, argval) for argname, argval in kwargs.items(): - assert argname in Fields, \ - "Invalid Keyword argument for {}: {}".format(Name, argname) + assert ( + argname in Fields + ), "Invalid Keyword argument for {}: {}".format(Name, argname) setattr(self, argname, argval) - setattr(_sys.modules[__name__], Name, - type(Name, Bases, {'__init__': create_node})) + setattr( + _sys.modules[__name__], + Name, + type(Name, Bases, {'__init__': create_node}), + ) _nodes = ( # mod - ('Module', (('body', 'type_ignores'), (), (mod, ))), - ('Interactive', (('body', ), (), (mod, ))), - ('Expression', (('body', ), (), (mod, ))), - ('FunctionType', (('argtypes', 'returns'), (), (mod, ))), - ('Suite', (('body', ), (), (mod, ))), - + ('Module', (('body', 'type_ignores'), (), (mod,))), + ('Interactive', (('body',), (), (mod,))), + ('Expression', (('body',), (), (mod,))), + ('FunctionType', (('argtypes', 'returns'), (), (mod,))), + ('Suite', (('body',), (), (mod,))), # stmt - ('FunctionDef', (('name', 'args', 'body', 'decorator_list', 'returns', - 'type_comment'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('AsyncFunctionDef', (('name', 'args', 'body', 'decorator_list', 'returns', - 'type_comment'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('ClassDef', (( - 'name', - 'bases', - 'keywords', - 'body', - 'decorator_list', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Return', (('value', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Delete', (('targets', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Assign', (( - 'targets', - 'value', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('AugAssign', (( - 'target', - 'op', - 'value', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('AnnAssign', (( - 'target', - 'annotation', - 'value', - 'simple', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Print', (( - 'dest', - 'values', - 'nl', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('For', (('target', 'iter', 'body', 'orelse', 'type_comment'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('AsyncFor', (('target', 'iter', 'body', 'orelse', 'type_comment'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('While', (( - 'test', - 'body', - 'orelse', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('If', (( - 'test', - 'body', - 'orelse', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('With', (('items', 'body', 'type_comment'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('AsyncWith', (('items', 'body', 'type_comment'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Raise', (( - 'exc', - 'cause', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Try', (( - 'body', - 'handlers', - 'orelse', - 'finalbody', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Assert', (( - 'test', - 'msg', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Import', (('names', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('ImportFrom', (( - 'module', - 'names', - 'level', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Exec', (( - 'body', - 'globals', - 'locals', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Global', (('names', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Nonlocal', (('names', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Expr', (('value', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Pass', ((), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Break', ((), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - ('Continue', ((), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (stmt, ))), - + ( + 'FunctionDef', + ( + ( + 'name', + 'args', + 'body', + 'decorator_list', + 'returns', + 'type_comment', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'AsyncFunctionDef', + ( + ( + 'name', + 'args', + 'body', + 'decorator_list', + 'returns', + 'type_comment', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'ClassDef', + ( + ( + 'name', + 'bases', + 'keywords', + 'body', + 'decorator_list', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Return', + ( + ('value',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Delete', + ( + ('targets',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Assign', + ( + ( + 'targets', + 'value', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'AugAssign', + ( + ( + 'target', + 'op', + 'value', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'AnnAssign', + ( + ( + 'target', + 'annotation', + 'value', + 'simple', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Print', + ( + ( + 'dest', + 'values', + 'nl', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'For', + ( + ('target', 'iter', 'body', 'orelse', 'type_comment'), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'AsyncFor', + ( + ('target', 'iter', 'body', 'orelse', 'type_comment'), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'While', + ( + ( + 'test', + 'body', + 'orelse', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'If', + ( + ( + 'test', + 'body', + 'orelse', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'With', + ( + ('items', 'body', 'type_comment'), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'AsyncWith', + ( + ('items', 'body', 'type_comment'), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Raise', + ( + ( + 'exc', + 'cause', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Try', + ( + ( + 'body', + 'handlers', + 'orelse', + 'finalbody', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Assert', + ( + ( + 'test', + 'msg', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Import', + ( + ('names',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'ImportFrom', + ( + ( + 'module', + 'names', + 'level', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Exec', + ( + ( + 'body', + 'globals', + 'locals', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Global', + ( + ('names',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Nonlocal', + ( + ('names',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Expr', + ( + ('value',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Pass', + ( + (), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Break', + ( + (), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), + ( + 'Continue', + ( + (), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (stmt,), + ), + ), # expr - ('BoolOp', (( - 'op', - 'values', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('BinOp', (( - 'left', - 'op', - 'right', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('UnaryOp', (( - 'op', - 'operand', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Lambda', (( - 'args', - 'body', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('IfExp', (( - 'test', - 'body', - 'orelse', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Dict', (( - 'keys', - 'values', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Set', (('elts', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('ListComp', (( - 'elt', - 'generators', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('SetComp', (( - 'elt', - 'generators', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('DictComp', (( - 'key', - 'value', - 'generators', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('GeneratorExp', (( - 'elt', - 'generators', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Await', (('value', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Yield', (('value', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('YieldFrom', (('value', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Compare', (( - 'left', - 'ops', - 'comparators', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Call', (( - 'func', - 'args', - 'keywords', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Repr', (('value', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('FormattedValue', (( - 'value', - 'conversion', - 'format_spec', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('JoinedStr', (('values', ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Constant', (('value', 'kind'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Attribute', (( - 'value', - 'attr', - 'ctx', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Subscript', (( - 'value', - 'slice', - 'ctx', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Starred', (( - 'value', - 'ctx', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Name', (('id', 'ctx', 'annotation', 'type_comment'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('List', (( - 'elts', - 'ctx', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - ('Tuple', (( - 'elts', - 'ctx', - ), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (expr, ))), - + ( + 'BoolOp', + ( + ( + 'op', + 'values', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'BinOp', + ( + ( + 'left', + 'op', + 'right', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'UnaryOp', + ( + ( + 'op', + 'operand', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Lambda', + ( + ( + 'args', + 'body', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'IfExp', + ( + ( + 'test', + 'body', + 'orelse', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Dict', + ( + ( + 'keys', + 'values', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Set', + ( + ('elts',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'ListComp', + ( + ( + 'elt', + 'generators', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'SetComp', + ( + ( + 'elt', + 'generators', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'DictComp', + ( + ( + 'key', + 'value', + 'generators', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'GeneratorExp', + ( + ( + 'elt', + 'generators', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Await', + ( + ('value',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Yield', + ( + ('value',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'YieldFrom', + ( + ('value',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Compare', + ( + ( + 'left', + 'ops', + 'comparators', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Call', + ( + ( + 'func', + 'args', + 'keywords', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Repr', + ( + ('value',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'FormattedValue', + ( + ( + 'value', + 'conversion', + 'format_spec', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'JoinedStr', + ( + ('values',), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Constant', + ( + ('value', 'kind'), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Attribute', + ( + ( + 'value', + 'attr', + 'ctx', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Subscript', + ( + ( + 'value', + 'slice', + 'ctx', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Starred', + ( + ( + 'value', + 'ctx', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Name', + ( + ('id', 'ctx', 'annotation', 'type_comment'), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'List', + ( + ( + 'elts', + 'ctx', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), + ( + 'Tuple', + ( + ( + 'elts', + 'ctx', + ), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (expr,), + ), + ), # expr_context - ('Load', ((), (), (expr_context, ))), - ('Store', ((), (), (expr_context, ))), - ('Del', ((), (), (expr_context, ))), - ('AugLoad', ((), (), (expr_context, ))), - ('AugStore', ((), (), (expr_context, ))), - ('Param', ((), (), (expr_context, ))), - + ('Load', ((), (), (expr_context,))), + ('Store', ((), (), (expr_context,))), + ('Del', ((), (), (expr_context,))), + ('AugLoad', ((), (), (expr_context,))), + ('AugStore', ((), (), (expr_context,))), + ('Param', ((), (), (expr_context,))), # slice - ('Slice', (('lower', 'upper', 'step'), ( - 'lineno', - 'col_offset', - 'end_lineno', - 'end_col_offset', - ), (slice, ))), - + ( + 'Slice', + ( + ('lower', 'upper', 'step'), + ( + 'lineno', + 'col_offset', + 'end_lineno', + 'end_col_offset', + ), + (slice,), + ), + ), # boolop - ('And', ((), (), (boolop, ))), - ('Or', ((), (), (boolop, ))), - + ('And', ((), (), (boolop,))), + ('Or', ((), (), (boolop,))), # operator - ('Add', ((), (), (operator, ))), - ('Sub', ((), (), (operator, ))), - ('Mult', ((), (), (operator, ))), - ('MatMult', ((), (), (operator, ))), - ('Div', ((), (), (operator, ))), - ('Mod', ((), (), (operator, ))), - ('Pow', ((), (), (operator, ))), - ('LShift', ((), (), (operator, ))), - ('RShift', ((), (), (operator, ))), - ('BitOr', ((), (), (operator, ))), - ('BitXor', ((), (), (operator, ))), - ('BitAnd', ((), (), (operator, ))), - ('FloorDiv', ((), (), (operator, ))), - + ('Add', ((), (), (operator,))), + ('Sub', ((), (), (operator,))), + ('Mult', ((), (), (operator,))), + ('MatMult', ((), (), (operator,))), + ('Div', ((), (), (operator,))), + ('Mod', ((), (), (operator,))), + ('Pow', ((), (), (operator,))), + ('LShift', ((), (), (operator,))), + ('RShift', ((), (), (operator,))), + ('BitOr', ((), (), (operator,))), + ('BitXor', ((), (), (operator,))), + ('BitAnd', ((), (), (operator,))), + ('FloorDiv', ((), (), (operator,))), # unaryop - ('Invert', ((), (), ( - unaryop, - AST, - ))), - ('Not', ((), (), ( - unaryop, - AST, - ))), - ('UAdd', ((), (), ( - unaryop, - AST, - ))), - ('USub', ((), (), ( - unaryop, - AST, - ))), - + ( + 'Invert', + ( + (), + (), + ( + unaryop, + AST, + ), + ), + ), + ( + 'Not', + ( + (), + (), + ( + unaryop, + AST, + ), + ), + ), + ( + 'UAdd', + ( + (), + (), + ( + unaryop, + AST, + ), + ), + ), + ( + 'USub', + ( + (), + (), + ( + unaryop, + AST, + ), + ), + ), # cmpop - ('Eq', ((), (), (cmpop, ))), - ('NotEq', ((), (), (cmpop, ))), - ('Lt', ((), (), (cmpop, ))), - ('LtE', ((), (), (cmpop, ))), - ('Gt', ((), (), (cmpop, ))), - ('GtE', ((), (), (cmpop, ))), - ('Is', ((), (), (cmpop, ))), - ('IsNot', ((), (), (cmpop, ))), - ('In', ((), (), (cmpop, ))), - ('NotIn', ((), (), (cmpop, ))), - + ('Eq', ((), (), (cmpop,))), + ('NotEq', ((), (), (cmpop,))), + ('Lt', ((), (), (cmpop,))), + ('LtE', ((), (), (cmpop,))), + ('Gt', ((), (), (cmpop,))), + ('GtE', ((), (), (cmpop,))), + ('Is', ((), (), (cmpop,))), + ('IsNot', ((), (), (cmpop,))), + ('In', ((), (), (cmpop,))), + ('NotIn', ((), (), (cmpop,))), # comprehension - ('comprehension', (('target', 'iter', 'ifs', 'is_async'), (), (AST, ))), - + ('comprehension', (('target', 'iter', 'ifs', 'is_async'), (), (AST,))), # excepthandler - ('ExceptHandler', (('type', 'name', 'body'), - ('lineno', 'col_offset', 'end_lineno', - 'end_col_offset'), (excepthandler, ))), - + ( + 'ExceptHandler', + ( + ('type', 'name', 'body'), + ('lineno', 'col_offset', 'end_lineno', 'end_col_offset'), + (excepthandler,), + ), + ), # arguments - ('arguments', (('args', 'posonlyargs', 'vararg', 'kwonlyargs', - 'kw_defaults', 'kwarg', 'defaults'), (), (AST, ))), - + ( + 'arguments', + ( + ( + 'args', + 'posonlyargs', + 'vararg', + 'kwonlyargs', + 'kw_defaults', + 'kwarg', + 'defaults', + ), + (), + (AST,), + ), + ), # keyword - ('keyword', (('arg', 'value'), ('lineno', 'col_offset', 'end_lineno', - 'end_col_offset'), (AST, ))), - + ( + 'keyword', + ( + ('arg', 'value'), + ('lineno', 'col_offset', 'end_lineno', 'end_col_offset'), + (AST,), + ), + ), # alias - ('alias', (('name', 'asname'), (), (AST, ))), - + ('alias', (('name', 'asname'), (), (AST,))), # withitem - ('withitem', (('context_expr', 'optional_vars'), (), (AST, ))), - + ('withitem', (('context_expr', 'optional_vars'), (), (AST,))), # type_ignore - ('type_ignore', ((), ('lineno', 'tag'), (TypeIgnore, ))), + ('type_ignore', ((), ('lineno', 'tag'), (TypeIgnore,))), ) for name, descr in _nodes: @@ -603,8 +1037,10 @@ def create_node(self, *args, **kwargs): py_version = _sys.version_info.major if py_version != 3: raise RuntimeError( - 'Required Python version >= 3, but received Python version == {}'. - format(py_version)) + 'Required Python version >= 3, but received Python version == {}'.format( + py_version + ) + ) from .ast3 import ast_to_gast, gast_to_ast @@ -622,10 +1058,14 @@ def literal_eval(node_or_string): def get_docstring(node, clean=True): if not isinstance(node, (FunctionDef, ClassDef, Module)): raise TypeError("%r can't have docstrings" % node.__class__.__name__) - if node.body and isinstance(node.body[0], Expr) and \ - isinstance(node.body[0].value, Constant): + if ( + node.body + and isinstance(node.body[0], Expr) + and isinstance(node.body[0].value, Constant) + ): if clean: import inspect + holder = node.body[0].value return inspect.cleandoc(getattr(holder, holder._fields[0])) return node.body[0].value.s @@ -641,8 +1081,11 @@ def copy_location(new_node, old_node): and return *new_node*. """ for attr in 'lineno', 'col_offset', 'end_lineno', 'end_col_offset': - if attr in old_node._attributes and attr in new_node._attributes \ - and hasattr(old_node, attr): + if ( + attr in old_node._attributes + and attr in new_node._attributes + and hasattr(old_node, attr) + ): setattr(new_node, attr, getattr(old_node, attr)) return new_node diff --git a/python/paddle/utils/image_util.py b/python/paddle/utils/image_util.py index a694d32c93b55..107ee828af48e 100644 --- a/python/paddle/utils/image_util.py +++ b/python/paddle/utils/image_util.py @@ -25,9 +25,10 @@ def resize_image(img, target_size): img: the input image to be resized. target_size: the target resized image size. """ - percent = (target_size / float(min(img.size[0], img.size[1]))) + percent = target_size / float(min(img.size[0], img.size[1])) resized_size = int(round(img.size[0] * percent)), int( - round(img.size[1] * percent)) + round(img.size[1] * percent) + ) img = img.resize(resized_size, Image.ANTIALIAS) return img @@ -56,8 +57,9 @@ def crop_img(im, inner_size, color=True, test=True): If True, crop the center of images. """ if color: - height, width = max(inner_size, - im.shape[1]), max(inner_size, im.shape[2]) + height, width = max(inner_size, im.shape[1]), max( + inner_size, im.shape[2] + ) padded_im = np.zeros((3, height, width)) startY = (height - im.shape[1]) / 2 startX = (width - im.shape[2]) / 2 @@ -65,8 +67,9 @@ def crop_img(im, inner_size, color=True, test=True): padded_im[:, startY:endY, startX:endX] = im else: im = im.astype('float32') - height, width = max(inner_size, - im.shape[0]), max(inner_size, im.shape[1]) + height, width = max(inner_size, im.shape[0]), max( + inner_size, im.shape[1] + ) padded_im = np.zeros((height, width)) startY = (height - im.shape[0]) / 2 startX = (width - im.shape[1]) / 2 @@ -120,15 +123,17 @@ def load_meta(meta_path, mean_img_size, crop_size, color=True): mean = np.load(meta_path)['data_mean'] border = (mean_img_size - crop_size) / 2 if color: - assert (mean_img_size * mean_img_size * 3 == mean.shape[0]) + assert mean_img_size * mean_img_size * 3 == mean.shape[0] mean = mean.reshape(3, mean_img_size, mean_img_size) - mean = mean[:, border:border + crop_size, - border:border + crop_size].astype('float32') + mean = mean[ + :, border : border + crop_size, border : border + crop_size + ].astype('float32') else: - assert (mean_img_size * mean_img_size == mean.shape[0]) + assert mean_img_size * mean_img_size == mean.shape[0] mean = mean.reshape(mean_img_size, mean_img_size) - mean = mean[border:border + crop_size, - border:border + crop_size].astype('float32') + mean = mean[ + border : border + crop_size, border : border + crop_size + ].astype('float32') return mean @@ -166,28 +171,28 @@ def oversample(img, crop_dims): crops_ix[curr] = (i, j, i + crop_dims[0], j + crop_dims[1]) curr += 1 crops_ix[4] = np.tile(im_center, (1, 2)) + np.concatenate( - [-crop_dims / 2.0, crop_dims / 2.0]) + [-crop_dims / 2.0, crop_dims / 2.0] + ) crops_ix = np.tile(crops_ix, (2, 1)) # Extract crops - crops = np.empty((10 * len(img), crop_dims[0], crop_dims[1], im_shape[-1]), - dtype=np.float32) + crops = np.empty( + (10 * len(img), crop_dims[0], crop_dims[1], im_shape[-1]), + dtype=np.float32, + ) ix = 0 for im in img: for crop in crops_ix: - crops[ix] = im[crop[0]:crop[2], crop[1]:crop[3], :] + crops[ix] = im[crop[0] : crop[2], crop[1] : crop[3], :] ix += 1 - crops[ix - 5:ix] = crops[ix - 5:ix, :, ::-1, :] # flip for mirrors + crops[ix - 5 : ix] = crops[ix - 5 : ix, :, ::-1, :] # flip for mirrors return crops class ImageTransformer: - - def __init__(self, - transpose=None, - channel_swap=None, - mean=None, - is_color=True): + def __init__( + self, transpose=None, channel_swap=None, mean=None, is_color=True + ): self.is_color = is_color self.set_transpose(transpose) self.set_channel_swap(channel_swap) diff --git a/python/paddle/utils/install_check.py b/python/paddle/utils/install_check.py index f8fd9d71df58e..c9cc1bb7a49e2 100644 --- a/python/paddle/utils/install_check.py +++ b/python/paddle/utils/install_check.py @@ -24,13 +24,14 @@ def _simple_network(): """ Define a simple network composed by a single linear layer. """ - input = paddle.static.data(name="input", - shape=[None, 2, 2], - dtype="float32") + input = paddle.static.data( + name="input", shape=[None, 2, 2], dtype="float32" + ) weight = paddle.create_parameter( shape=[2, 3], dtype="float32", - attr=paddle.ParamAttr(initializer=paddle.nn.initializer.Constant(0.1))) + attr=paddle.ParamAttr(initializer=paddle.nn.initializer.Constant(0.1)), + ) bias = paddle.create_parameter(shape=[3], dtype="float32") linear_out = paddle.nn.functional.linear(x=input, weight=weight, bias=bias) out = paddle.tensor.sum(linear_out) @@ -68,7 +69,8 @@ def _is_cuda_available(): logging.warning( "You are using GPU version PaddlePaddle, but there is no GPU " "detected on your machine. Maybe CUDA devices is not set properly." - "\n Original Error is {}".format(e)) + "\n Original Error is {}".format(e) + ) return False @@ -83,7 +85,8 @@ def _is_npu_available(): logging.warning( "You are using NPU version PaddlePaddle, but there is no NPU " "detected on your machine. Maybe NPU devices is not set properly." - "\n Original Error is {}".format(e)) + "\n Original Error is {}".format(e) + ) return False @@ -98,7 +101,8 @@ def _is_xpu_available(): logging.warning( "You are using XPU version PaddlePaddle, but there is no XPU " "detected on your machine. Maybe XPU devices is not set properly." - "\n Original Error is {}".format(e)) + "\n Original Error is {}".format(e) + ) return False @@ -121,20 +125,22 @@ def _run_dygraph_single(use_cuda, use_xpu, use_npu): else: paddle.set_device('cpu') weight_attr = paddle.ParamAttr( - name="weight", initializer=paddle.nn.initializer.Constant(value=0.5)) + name="weight", initializer=paddle.nn.initializer.Constant(value=0.5) + ) bias_attr = paddle.ParamAttr( - name="bias", initializer=paddle.nn.initializer.Constant(value=1.0)) - linear = paddle.nn.Linear(2, - 4, - weight_attr=weight_attr, - bias_attr=bias_attr) + name="bias", initializer=paddle.nn.initializer.Constant(value=1.0) + ) + linear = paddle.nn.Linear( + 2, 4, weight_attr=weight_attr, bias_attr=bias_attr + ) input_np = _prepare_data(1) input_tensor = paddle.to_tensor(input_np) linear_out = linear(input_tensor) out = paddle.tensor.sum(linear_out) out.backward() - opt = paddle.optimizer.Adam(learning_rate=0.001, - parameters=linear.parameters()) + opt = paddle.optimizer.Adam( + learning_rate=0.001, parameters=linear.parameters() + ) opt.step() @@ -155,7 +161,8 @@ def _run_static_single(use_cuda, use_xpu, use_npu): with paddle.static.program_guard(train_prog, startup_prog): input, out, weight = _simple_network() param_grads = paddle.static.append_backward( - out, parameter_list=[weight.name])[0] + out, parameter_list=[weight.name] + )[0] if use_cuda: place = paddle.CUDAPlace(0) @@ -168,9 +175,11 @@ def _run_static_single(use_cuda, use_xpu, use_npu): exe = paddle.static.Executor(place) exe.run(startup_prog) - exe.run(train_prog, - feed={input.name: _prepare_data(1)}, - fetch_list=[out.name, param_grads[1].name]) + exe.run( + train_prog, + feed={input.name: _prepare_data(1)}, + fetch_list=[out.name, param_grads[1].name], + ) paddle.disable_static() @@ -195,8 +204,8 @@ def _run_static_parallel(use_cuda, use_xpu, use_npu, device_list): paddle.optimizer.SGD(learning_rate=0.01).minimize(loss) compiled_prog = paddle.static.CompiledProgram( - train_prog).with_data_parallel(loss_name=loss.name, - places=device_list) + train_prog + ).with_data_parallel(loss_name=loss.name, places=device_list) if use_cuda: place = paddle.CUDAPlace(0) @@ -211,9 +220,11 @@ def _run_static_parallel(use_cuda, use_xpu, use_npu, device_list): exe = paddle.static.Executor(place) exe.run(startup_prog) - exe.run(compiled_prog, - feed={input.name: _prepare_data(len(device_list))}, - fetch_list=[loss.name]) + exe.run( + compiled_prog, + feed={input.name: _prepare_data(len(device_list))}, + fetch_list=[loss.name], + ) paddle.disable_static() @@ -269,8 +280,11 @@ def run_check(): try: _run_static_parallel(use_cuda, use_xpu, use_npu, device_list) - print("PaddlePaddle works well on {} {}s.".format( - device_count, device_str)) + print( + "PaddlePaddle works well on {} {}s.".format( + device_count, device_str + ) + ) print( "PaddlePaddle is installed successfully! Let's start deep learning with PaddlePaddle now." ) @@ -280,10 +294,15 @@ def run_check(): "\n 1. There is not enough GPUs visible on your system" "\n 2. Some GPUs are occupied by other process now" "\n 3. NVIDIA-NCCL2 is not installed correctly on your system. Please follow instruction on https://github.com/NVIDIA/nccl-tests " - "\n to test your NCCL, or reinstall it following https://docs.nvidia.com/deeplearning/sdk/nccl-install-guide/index.html" - .format(device_count, device_str)) + "\n to test your NCCL, or reinstall it following https://docs.nvidia.com/deeplearning/sdk/nccl-install-guide/index.html".format( + device_count, device_str + ) + ) logging.warning("\n Original Error is: {}".format(e)) - print("PaddlePaddle is installed successfully ONLY for single {}! " - "Let's start deep learning with PaddlePaddle now.".format( - device_str)) + print( + "PaddlePaddle is installed successfully ONLY for single {}! " + "Let's start deep learning with PaddlePaddle now.".format( + device_str + ) + ) diff --git a/python/paddle/utils/lazy_import.py b/python/paddle/utils/lazy_import.py index d9146422819f8..e5ca0cdd5a815 100644 --- a/python/paddle/utils/lazy_import.py +++ b/python/paddle/utils/lazy_import.py @@ -35,6 +35,6 @@ def try_import(module_name): err_msg = ( "Failed importing {}. This likely means that some paddle modules " "require additional dependencies that have to be " - "manually installed (usually with `pip install {}`). ").format( - module_name, install_name) + "manually installed (usually with `pip install {}`). " + ).format(module_name, install_name) raise ImportError(err_msg) diff --git a/python/paddle/utils/op_version.py b/python/paddle/utils/op_version.py index 575e5f40772eb..9f9ae4d73c7bb 100644 --- a/python/paddle/utils/op_version.py +++ b/python/paddle/utils/op_version.py @@ -29,7 +29,6 @@ def _singleton(*args, **kargs): class OpUpdateInfoHelper(object): - def __init__(self, info): self._info = info @@ -49,7 +48,6 @@ def verify_key_value(self, name=''): @Singleton class OpLastCheckpointChecker(object): - def __init__(self): self.raw_version_map = core.get_op_version_map() self.checkpoints_map = {} @@ -65,8 +63,9 @@ def filter_updates(self, op_name, type=core.OpUpdateType.kInvalid, key=''): updates = [] if op_name in self.checkpoints_map: for update in self.checkpoints_map[op_name]: - if (update.type() == type) or (type - == core.OpUpdateType.kInvalid): + if (update.type() == type) or ( + type == core.OpUpdateType.kInvalid + ): if OpUpdateInfoHelper(update.info()).verify_key_value(key): updates.append(update.info()) return updates diff --git a/python/paddle/utils/profiler.py b/python/paddle/utils/profiler.py index e0bad20a327dd..27803cfa442dd 100644 --- a/python/paddle/utils/profiler.py +++ b/python/paddle/utils/profiler.py @@ -22,14 +22,19 @@ from ..fluid.profiler import stop_profiler from ..fluid.profiler import reset_profiler -__all__ = [ #noqa - 'Profiler', 'get_profiler', 'ProfilerOptions', 'cuda_profiler', - 'start_profiler', 'profiler', 'stop_profiler', 'reset_profiler' +__all__ = [ # noqa + 'Profiler', + 'get_profiler', + 'ProfilerOptions', + 'cuda_profiler', + 'start_profiler', + 'profiler', + 'stop_profiler', + 'reset_profiler', ] class ProfilerOptions(object): - def __init__(self, options=None): self.options = { 'state': 'All', @@ -39,7 +44,7 @@ def __init__(self, options=None): 'output_thread_detail': False, 'profile_path': 'none', 'timeline_path': 'none', - 'op_summary_path': 'none' + 'op_summary_path': 'none', } if options is not None: for key in self.options.keys(): @@ -54,10 +59,13 @@ def with_state(self, state): def __getitem__(self, name): if self.options.get(name, None) is None: raise ValueError( - "ProfilerOptions does not have an option named %s." % name) + "ProfilerOptions does not have an option named %s." % name + ) else: - if isinstance(self.options[name], - str) and self.options[name] == 'none': + if ( + isinstance(self.options[name], str) + and self.options[name] == 'none' + ): return None else: return self.options[name] @@ -67,7 +75,6 @@ def __getitem__(self, name): class Profiler(object): - def __init__(self, enabled=True, options=None): if options is not None: self.profiler_options = options @@ -99,22 +106,28 @@ def start(self): try: start_profiler( state=self.profiler_options['state'], - tracer_option=self.profiler_options['tracer_level']) + tracer_option=self.profiler_options['tracer_level'], + ) except Exception as e: warnings.warn( - "Profiler is not enabled becuase following exception:\n{}". - format(e)) + "Profiler is not enabled becuase following exception:\n{}".format( + e + ) + ) def stop(self): if self.enabled: try: stop_profiler( sorted_key=self.profiler_options['sorted_key'], - profile_path=self.profiler_options['profile_path']) + profile_path=self.profiler_options['profile_path'], + ) except Exception as e: warnings.warn( - "Profiler is not disabled becuase following exception:\n{}". - format(e)) + "Profiler is not disabled becuase following exception:\n{}".format( + e + ) + ) def reset(self): if self.enabled and core.is_profiler_enabled(): diff --git a/python/paddle/utils/unique_name.py b/python/paddle/utils/unique_name.py index d0d487c933d76..77ecebcc45b31 100644 --- a/python/paddle/utils/unique_name.py +++ b/python/paddle/utils/unique_name.py @@ -16,6 +16,4 @@ from ..fluid.unique_name import switch # noqa: F401 from ..fluid.unique_name import guard # noqa: F401 -__all__ = [ #noqa - 'generate', 'switch', 'guard' -] +__all__ = ['generate', 'switch', 'guard'] # noqa diff --git a/python/paddle/vision/__init__.py b/python/paddle/vision/__init__.py index 2f0052537e251..3e9b68ef8fdcd 100644 --- a/python/paddle/vision/__init__.py +++ b/python/paddle/vision/__init__.py @@ -112,6 +112,4 @@ from .transforms import adjust_hue # noqa: F401 from .transforms import normalize # noqa: F401 -__all__ = [ #noqa - 'set_image_backend', 'get_image_backend', 'image_load' -] +__all__ = ['set_image_backend', 'get_image_backend', 'image_load'] # noqa diff --git a/python/paddle/vision/datasets/__init__.py b/python/paddle/vision/datasets/__init__.py index 10666b7c7194a..970c8cfcae86a 100644 --- a/python/paddle/vision/datasets/__init__.py +++ b/python/paddle/vision/datasets/__init__.py @@ -21,7 +21,13 @@ from .cifar import Cifar100 # noqa: F401 from .voc2012 import VOC2012 # noqa: F401 -__all__ = [ #noqa - 'DatasetFolder', 'ImageFolder', 'MNIST', 'FashionMNIST', 'Flowers', - 'Cifar10', 'Cifar100', 'VOC2012' +__all__ = [ # noqa + 'DatasetFolder', + 'ImageFolder', + 'MNIST', + 'FashionMNIST', + 'Flowers', + 'Cifar10', + 'Cifar100', + 'VOC2012', ] diff --git a/python/paddle/vision/datasets/cifar.py b/python/paddle/vision/datasets/cifar.py index 1e3eb6515c8ee..5610b237379c8 100644 --- a/python/paddle/vision/datasets/cifar.py +++ b/python/paddle/vision/datasets/cifar.py @@ -33,7 +33,7 @@ 'train10': 'data_batch', 'test10': 'test_batch', 'train100': 'train', - 'test100': 'test' + 'test100': 'test', } @@ -102,14 +102,18 @@ class Cifar10(Dataset): # [3, 64, 64] 3 """ - def __init__(self, - data_file=None, - mode='train', - transform=None, - download=True, - backend=None): - assert mode.lower() in ['train', 'test'], \ - "mode.lower() should be 'train' or 'test', but got {}".format(mode) + def __init__( + self, + data_file=None, + mode='train', + transform=None, + download=True, + backend=None, + ): + assert mode.lower() in [ + 'train', + 'test', + ], "mode.lower() should be 'train' or 'test', but got {}".format(mode) self.mode = mode.lower() if backend is None: @@ -117,18 +121,21 @@ def __init__(self, if backend not in ['pil', 'cv2']: raise ValueError( "Expected backend are one of ['pil', 'cv2'], but got {}".format( - backend)) + backend + ) + ) self.backend = backend self._init_url_md5_flag() self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, - self.data_url, - self.data_md5, 'cifar', - download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, self.data_url, self.data_md5, 'cifar', download + ) self.transform = transform @@ -145,8 +152,9 @@ def _init_url_md5_flag(self): def _load_data(self): self.data = [] with tarfile.open(self.data_file, mode='r') as f: - names = (each_item.name for each_item in f - if self.flag in each_item.name) + names = ( + each_item.name for each_item in f if self.flag in each_item.name + ) names = sorted(list(names)) @@ -243,14 +251,17 @@ class Cifar100(Cifar10): # [3, 64, 64] 49 """ - def __init__(self, - data_file=None, - mode='train', - transform=None, - download=True, - backend=None): - super(Cifar100, self).__init__(data_file, mode, transform, download, - backend) + def __init__( + self, + data_file=None, + mode='train', + transform=None, + download=True, + backend=None, + ): + super(Cifar100, self).__init__( + data_file, mode, transform, download, backend + ) def _init_url_md5_flag(self): self.data_url = CIFAR100_URL diff --git a/python/paddle/vision/datasets/flowers.py b/python/paddle/vision/datasets/flowers.py index 963d639ed1179..1b1c556407b5b 100644 --- a/python/paddle/vision/datasets/flowers.py +++ b/python/paddle/vision/datasets/flowers.py @@ -106,44 +106,57 @@ class Flowers(Dataset): # [3, 64, 96] [1] """ - def __init__(self, - data_file=None, - label_file=None, - setid_file=None, - mode='train', - transform=None, - download=True, - backend=None): - assert mode.lower() in ['train', 'valid', 'test'], \ - "mode should be 'train', 'valid' or 'test', but got {}".format(mode) + def __init__( + self, + data_file=None, + label_file=None, + setid_file=None, + mode='train', + transform=None, + download=True, + backend=None, + ): + assert mode.lower() in [ + 'train', + 'valid', + 'test', + ], "mode should be 'train', 'valid' or 'test', but got {}".format(mode) if backend is None: backend = paddle.vision.get_image_backend() if backend not in ['pil', 'cv2']: raise ValueError( "Expected backend are one of ['pil', 'cv2'], but got {}".format( - backend)) + backend + ) + ) self.backend = backend flag = MODE_FLAG_MAP[mode.lower()] if not data_file: - assert download, "data_file is not set and downloading automatically is disabled" - data_file = _check_exists_and_download(data_file, DATA_URL, - DATA_MD5, 'flowers', - download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + data_file = _check_exists_and_download( + data_file, DATA_URL, DATA_MD5, 'flowers', download + ) if not label_file: - assert download, "label_file is not set and downloading automatically is disabled" - label_file = _check_exists_and_download(label_file, LABEL_URL, - LABEL_MD5, 'flowers', - download) + assert ( + download + ), "label_file is not set and downloading automatically is disabled" + label_file = _check_exists_and_download( + label_file, LABEL_URL, LABEL_MD5, 'flowers', download + ) if not setid_file: - assert download, "setid_file is not set and downloading automatically is disabled" - setid_file = _check_exists_and_download(setid_file, SETID_URL, - SETID_MD5, 'flowers', - download) + assert ( + download + ), "setid_file is not set and downloading automatically is disabled" + setid_file = _check_exists_and_download( + setid_file, SETID_URL, SETID_MD5, 'flowers', download + ) self.transform = transform diff --git a/python/paddle/vision/datasets/folder.py b/python/paddle/vision/datasets/folder.py index 4a024fa27793d..6ac0c4ca91723 100644 --- a/python/paddle/vision/datasets/folder.py +++ b/python/paddle/vision/datasets/folder.py @@ -33,8 +33,9 @@ def has_valid_extension(filename, extensions): Returns: bool: True if the filename ends with one of given extensions """ - assert isinstance(extensions, - (list, tuple)), ("`extensions` must be list or tuple.") + assert isinstance( + extensions, (list, tuple) + ), "`extensions` must be list or tuple." extensions = tuple([x.lower() for x in extensions]) return filename.lower().endswith(extensions) @@ -199,24 +200,29 @@ def make_directory(root, directory_hierarchy, file_maker=make_fake_file): shutil.rmtree(fake_data_dir) """ - def __init__(self, - root, - loader=None, - extensions=None, - transform=None, - is_valid_file=None): + def __init__( + self, + root, + loader=None, + extensions=None, + transform=None, + is_valid_file=None, + ): self.root = root self.transform = transform if extensions is None: extensions = IMG_EXTENSIONS classes, class_to_idx = self._find_classes(self.root) - samples = make_dataset(self.root, class_to_idx, extensions, - is_valid_file) + samples = make_dataset( + self.root, class_to_idx, extensions, is_valid_file + ) if len(samples) == 0: - raise (RuntimeError("Found 0 directories in subfolders of: " + - self.root + "\n" - "Supported extensions are: " + - ",".join(extensions))) + raise ( + RuntimeError( + "Found 0 directories in subfolders of: " + self.root + "\n" + "Supported extensions are: " + ",".join(extensions) + ) + ) self.loader = default_loader if loader is None else loader self.extensions = extensions @@ -245,7 +251,8 @@ def _find_classes(self, dir): classes = [d.name for d in os.scandir(dir) if d.is_dir()] else: classes = [ - d for d in os.listdir(dir) + d + for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d)) ] classes.sort() @@ -271,8 +278,17 @@ def __len__(self): return len(self.samples) -IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', - '.tiff', '.webp') +IMG_EXTENSIONS = ( + '.jpg', + '.jpeg', + '.png', + '.ppm', + '.bmp', + '.pgm', + '.tif', + '.tiff', + '.webp', +) def pil_loader(path): @@ -288,6 +304,7 @@ def cv2_loader(path): def default_loader(path): from paddle.vision import get_image_backend + if get_image_backend() == 'cv2': return cv2_loader(path) else: @@ -412,14 +429,16 @@ def make_directory(root, directory_hierarchy, file_maker=make_fake_file): # [3, 64, 64] shutil.rmtree(fake_data_dir) - """ - - def __init__(self, - root, - loader=None, - extensions=None, - transform=None, - is_valid_file=None): + """ + + def __init__( + self, + root, + loader=None, + extensions=None, + transform=None, + is_valid_file=None, + ): self.root = root if extensions is None: extensions = IMG_EXTENSIONS @@ -439,10 +458,12 @@ def is_valid_file(x): samples.append(f) if len(samples) == 0: - raise (RuntimeError("Found 0 files in subfolders of: " + self.root + - "\n" - "Supported extensions are: " + - ",".join(extensions))) + raise ( + RuntimeError( + "Found 0 files in subfolders of: " + self.root + "\n" + "Supported extensions are: " + ",".join(extensions) + ) + ) self.loader = default_loader if loader is None else loader self.extensions = extensions diff --git a/python/paddle/vision/datasets/mnist.py b/python/paddle/vision/datasets/mnist.py index 5b819fb3bb8cd..c273d41d4a6cc 100644 --- a/python/paddle/vision/datasets/mnist.py +++ b/python/paddle/vision/datasets/mnist.py @@ -88,6 +88,7 @@ class MNIST(Dataset): print(type(img), img.shape, label) # [1, 28, 28] [7] """ + NAME = 'mnist' URL_PREFIX = 'https://dataset.bj.bcebos.com/mnist/' TEST_IMAGE_URL = URL_PREFIX + 't10k-images-idx3-ubyte.gz' @@ -99,40 +100,64 @@ class MNIST(Dataset): TRAIN_LABEL_URL = URL_PREFIX + 'train-labels-idx1-ubyte.gz' TRAIN_LABEL_MD5 = 'd53e105ee54ea40749a09fcbcd1e9432' - def __init__(self, - image_path=None, - label_path=None, - mode='train', - transform=None, - download=True, - backend=None): - assert mode.lower() in ['train', 'test'], \ - "mode should be 'train' or 'test', but got {}".format(mode) + def __init__( + self, + image_path=None, + label_path=None, + mode='train', + transform=None, + download=True, + backend=None, + ): + assert mode.lower() in [ + 'train', + 'test', + ], "mode should be 'train' or 'test', but got {}".format(mode) if backend is None: backend = paddle.vision.get_image_backend() if backend not in ['pil', 'cv2']: raise ValueError( "Expected backend are one of ['pil', 'cv2'], but got {}".format( - backend)) + backend + ) + ) self.backend = backend self.mode = mode.lower() self.image_path = image_path if self.image_path is None: - assert download, "image_path is not set and downloading automatically is disabled" - image_url = self.TRAIN_IMAGE_URL if mode == 'train' else self.TEST_IMAGE_URL - image_md5 = self.TRAIN_IMAGE_MD5 if mode == 'train' else self.TEST_IMAGE_MD5 + assert ( + download + ), "image_path is not set and downloading automatically is disabled" + image_url = ( + self.TRAIN_IMAGE_URL if mode == 'train' else self.TEST_IMAGE_URL + ) + image_md5 = ( + self.TRAIN_IMAGE_MD5 if mode == 'train' else self.TEST_IMAGE_MD5 + ) self.image_path = _check_exists_and_download( - image_path, image_url, image_md5, self.NAME, download) + image_path, image_url, image_md5, self.NAME, download + ) self.label_path = label_path if self.label_path is None: - assert download, "label_path is not set and downloading automatically is disabled" - label_url = self.TRAIN_LABEL_URL if self.mode == 'train' else self.TEST_LABEL_URL - label_md5 = self.TRAIN_LABEL_MD5 if self.mode == 'train' else self.TEST_LABEL_MD5 + assert ( + download + ), "label_path is not set and downloading automatically is disabled" + label_url = ( + self.TRAIN_LABEL_URL + if self.mode == 'train' + else self.TEST_LABEL_URL + ) + label_md5 = ( + self.TRAIN_LABEL_MD5 + if self.mode == 'train' + else self.TEST_LABEL_MD5 + ) self.label_path = _check_exists_and_download( - label_path, label_url, label_md5, self.NAME, download) + label_path, label_url, label_md5, self.NAME, download + ) self.transform = transform @@ -156,14 +181,16 @@ def _parse_dataset(self, buffer_size=100): # image file : 16B magic_byte_img = '>IIII' magic_img, image_num, rows, cols = struct.unpack_from( - magic_byte_img, img_buf, offset_img) + magic_byte_img, img_buf, offset_img + ) offset_img += struct.calcsize(magic_byte_img) offset_lab = 0 # label file : 8B magic_byte_lab = '>II' magic_lab, label_num = struct.unpack_from( - magic_byte_lab, lab_buf, offset_lab) + magic_byte_lab, lab_buf, offset_lab + ) offset_lab += struct.calcsize(magic_byte_lab) while True: @@ -175,17 +202,19 @@ def _parse_dataset(self, buffer_size=100): step_label += buffer_size fmt_images = '>' + str(buffer_size * rows * cols) + 'B' - images_temp = struct.unpack_from(fmt_images, img_buf, - offset_img) + images_temp = struct.unpack_from( + fmt_images, img_buf, offset_img + ) images = np.reshape( - images_temp, - (buffer_size, rows * cols)).astype('float32') + images_temp, (buffer_size, rows * cols) + ).astype('float32') offset_img += struct.calcsize(fmt_images) for i in range(buffer_size): self.images.append(images[i, :]) self.labels.append( - np.array([labels[i]]).astype('int64')) + np.array([labels[i]]).astype('int64') + ) def __getitem__(self, idx): image, label = self.images[idx], self.labels[idx] diff --git a/python/paddle/vision/datasets/voc2012.py b/python/paddle/vision/datasets/voc2012.py index 8931b0f86cc6c..81478378ae412 100644 --- a/python/paddle/vision/datasets/voc2012.py +++ b/python/paddle/vision/datasets/voc2012.py @@ -102,31 +102,40 @@ class VOC2012(Dataset): # (281, 500) """ - def __init__(self, - data_file=None, - mode='train', - transform=None, - download=True, - backend=None): - assert mode.lower() in ['train', 'valid', 'test'], \ - "mode should be 'train', 'valid' or 'test', but got {}".format(mode) + def __init__( + self, + data_file=None, + mode='train', + transform=None, + download=True, + backend=None, + ): + assert mode.lower() in [ + 'train', + 'valid', + 'test', + ], "mode should be 'train', 'valid' or 'test', but got {}".format(mode) if backend is None: backend = paddle.vision.get_image_backend() if backend not in ['pil', 'cv2']: raise ValueError( "Expected backend are one of ['pil', 'cv2'], but got {}".format( - backend)) + backend + ) + ) self.backend = backend self.flag = MODE_FLAG_MAP[mode.lower()] self.data_file = data_file if self.data_file is None: - assert download, "data_file is not set and downloading automatically is disabled" - self.data_file = _check_exists_and_download(data_file, VOC_URL, - VOC_MD5, CACHE_DIR, - download) + assert ( + download + ), "data_file is not set and downloading automatically is disabled" + self.data_file = _check_exists_and_download( + data_file, VOC_URL, VOC_MD5, CACHE_DIR, download + ) self.transform = transform # read dataset into memory diff --git a/python/paddle/vision/image.py b/python/paddle/vision/image.py index c30b99799c097..eda914d939cd3 100644 --- a/python/paddle/vision/image.py +++ b/python/paddle/vision/image.py @@ -82,8 +82,10 @@ def make_fake_dir(): global _image_backend if backend not in ['pil', 'cv2', 'tensor']: raise ValueError( - "Expected backend are one of ['pil', 'cv2', 'tensor'], but got {}". - format(backend)) + "Expected backend are one of ['pil', 'cv2', 'tensor'], but got {}".format( + backend + ) + ) _image_backend = backend @@ -152,8 +154,10 @@ def image_load(path, backend=None): backend = _image_backend if backend not in ['pil', 'cv2', 'tensor']: raise ValueError( - "Expected backend are one of ['pil', 'cv2', 'tensor'], but got {}". - format(backend)) + "Expected backend are one of ['pil', 'cv2', 'tensor'], but got {}".format( + backend + ) + ) if backend == 'pil': return Image.open(path) diff --git a/python/paddle/vision/models/__init__.py b/python/paddle/vision/models/__init__.py index 72bb6ee8e8d5b..08f559bd440c9 100644 --- a/python/paddle/vision/models/__init__.py +++ b/python/paddle/vision/models/__init__.py @@ -1,16 +1,16 @@ # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from .resnet import ResNet # noqa: F401 from .resnet import resnet18 # noqa: F401 @@ -64,18 +64,56 @@ from .shufflenetv2 import shufflenet_v2_x2_0 # noqa: F401 from .shufflenetv2 import shufflenet_v2_swish # noqa: F401 -__all__ = [ #noqa - 'ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152', - 'resnext50_32x4d', 'resnext50_64x4d', 'resnext101_32x4d', - 'resnext101_64x4d', 'resnext152_32x4d', 'resnext152_64x4d', - 'wide_resnet50_2', 'wide_resnet101_2', 'VGG', 'vgg11', 'vgg13', 'vgg16', - 'vgg19', 'MobileNetV1', 'mobilenet_v1', 'MobileNetV2', 'mobilenet_v2', - 'MobileNetV3Small', 'MobileNetV3Large', 'mobilenet_v3_small', - 'mobilenet_v3_large', 'LeNet', 'DenseNet', 'densenet121', 'densenet161', - 'densenet169', 'densenet201', 'densenet264', 'AlexNet', 'alexnet', - 'InceptionV3', 'inception_v3', 'SqueezeNet', 'squeezenet1_0', - 'squeezenet1_1', 'GoogLeNet', 'googlenet', 'ShuffleNetV2', - 'shufflenet_v2_x0_25', 'shufflenet_v2_x0_33', 'shufflenet_v2_x0_5', - 'shufflenet_v2_x1_0', 'shufflenet_v2_x1_5', 'shufflenet_v2_x2_0', - 'shufflenet_v2_swish' +__all__ = [ # noqa + 'ResNet', + 'resnet18', + 'resnet34', + 'resnet50', + 'resnet101', + 'resnet152', + 'resnext50_32x4d', + 'resnext50_64x4d', + 'resnext101_32x4d', + 'resnext101_64x4d', + 'resnext152_32x4d', + 'resnext152_64x4d', + 'wide_resnet50_2', + 'wide_resnet101_2', + 'VGG', + 'vgg11', + 'vgg13', + 'vgg16', + 'vgg19', + 'MobileNetV1', + 'mobilenet_v1', + 'MobileNetV2', + 'mobilenet_v2', + 'MobileNetV3Small', + 'MobileNetV3Large', + 'mobilenet_v3_small', + 'mobilenet_v3_large', + 'LeNet', + 'DenseNet', + 'densenet121', + 'densenet161', + 'densenet169', + 'densenet201', + 'densenet264', + 'AlexNet', + 'alexnet', + 'InceptionV3', + 'inception_v3', + 'SqueezeNet', + 'squeezenet1_0', + 'squeezenet1_1', + 'GoogLeNet', + 'googlenet', + 'ShuffleNetV2', + 'shufflenet_v2_x0_25', + 'shufflenet_v2_x0_33', + 'shufflenet_v2_x0_5', + 'shufflenet_v2_x1_0', + 'shufflenet_v2_x1_5', + 'shufflenet_v2_x2_0', + 'shufflenet_v2_swish', ] diff --git a/python/paddle/vision/models/alexnet.py b/python/paddle/vision/models/alexnet.py index 634125ba612c1..c0d1e0299389c 100644 --- a/python/paddle/vision/models/alexnet.py +++ b/python/paddle/vision/models/alexnet.py @@ -34,16 +34,17 @@ class ConvPoolLayer(nn.Layer): - - def __init__(self, - input_channels, - output_channels, - filter_size, - stride, - padding, - stdv, - groups=1, - act=None): + def __init__( + self, + input_channels, + output_channels, + filter_size, + stride, + padding, + stdv, + groups=1, + act=None, + ): super(ConvPoolLayer, self).__init__() self.relu = ReLU() if act == "relu" else None @@ -56,7 +57,8 @@ def __init__(self, padding=padding, groups=groups, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv))) + bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), + ) self._pool = MaxPool2D(kernel_size=3, stride=2, padding=0) def forward(self, inputs): @@ -109,7 +111,8 @@ def __init__(self, num_classes=1000): stride=1, padding=1, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv))) + bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), + ) stdv = 1.0 / math.sqrt(384 * 3 * 3) self._conv4 = Conv2D( 384, @@ -118,7 +121,8 @@ def __init__(self, num_classes=1000): stride=1, padding=1, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv))) + bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), + ) stdv = 1.0 / math.sqrt(256 * 3 * 3) self._conv5 = ConvPoolLayer(256, 256, 3, 1, 1, stdv, act="relu") @@ -129,19 +133,22 @@ def __init__(self, num_classes=1000): in_features=256 * 6 * 6, out_features=4096, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv))) + bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), + ) self._drop2 = Dropout(p=0.5, mode="downscale_in_infer") self._fc7 = Linear( in_features=4096, out_features=4096, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv))) + bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), + ) self._fc8 = Linear( in_features=4096, out_features=num_classes, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv))) + bias_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), + ) def forward(self, inputs): x = self._conv1(inputs) @@ -169,10 +176,14 @@ def _alexnet(arch, pretrained, **kwargs): model = AlexNet(**kwargs) if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.load_dict(param) diff --git a/python/paddle/vision/models/densenet.py b/python/paddle/vision/models/densenet.py index 309f5424cebb1..8971670144320 100644 --- a/python/paddle/vision/models/densenet.py +++ b/python/paddle/vision/models/densenet.py @@ -25,45 +25,53 @@ __all__ = [] model_urls = { - 'densenet121': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet121_pretrained.pdparams', - 'db1b239ed80a905290fd8b01d3af08e4'), - 'densenet161': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet161_pretrained.pdparams', - '62158869cb315098bd25ddbfd308a853'), - 'densenet169': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet169_pretrained.pdparams', - '82cc7c635c3f19098c748850efb2d796'), - 'densenet201': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet201_pretrained.pdparams', - '16ca29565a7712329cf9e36e02caaf58'), - 'densenet264': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet264_pretrained.pdparams', - '3270ce516b85370bba88cfdd9f60bff4'), + 'densenet121': ( + 'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet121_pretrained.pdparams', + 'db1b239ed80a905290fd8b01d3af08e4', + ), + 'densenet161': ( + 'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet161_pretrained.pdparams', + '62158869cb315098bd25ddbfd308a853', + ), + 'densenet169': ( + 'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet169_pretrained.pdparams', + '82cc7c635c3f19098c748850efb2d796', + ), + 'densenet201': ( + 'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet201_pretrained.pdparams', + '16ca29565a7712329cf9e36e02caaf58', + ), + 'densenet264': ( + 'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/DenseNet264_pretrained.pdparams', + '3270ce516b85370bba88cfdd9f60bff4', + ), } class BNACConvLayer(nn.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - pad=0, - groups=1, - act="relu"): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + pad=0, + groups=1, + act="relu", + ): super(BNACConvLayer, self).__init__() self._batch_norm = BatchNorm(num_channels, act=act) - self._conv = Conv2D(in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=pad, - groups=groups, - weight_attr=ParamAttr(), - bias_attr=False) + self._conv = Conv2D( + in_channels=num_channels, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=pad, + groups=groups, + weight_attr=ParamAttr(), + bias_attr=False, + ) def forward(self, input): y = self._batch_norm(input) @@ -72,22 +80,25 @@ def forward(self, input): class DenseLayer(nn.Layer): - def __init__(self, num_channels, growth_rate, bn_size, dropout): super(DenseLayer, self).__init__() self.dropout = dropout - self.bn_ac_func1 = BNACConvLayer(num_channels=num_channels, - num_filters=bn_size * growth_rate, - filter_size=1, - pad=0, - stride=1) - - self.bn_ac_func2 = BNACConvLayer(num_channels=bn_size * growth_rate, - num_filters=growth_rate, - filter_size=3, - pad=1, - stride=1) + self.bn_ac_func1 = BNACConvLayer( + num_channels=num_channels, + num_filters=bn_size * growth_rate, + filter_size=1, + pad=0, + stride=1, + ) + + self.bn_ac_func2 = BNACConvLayer( + num_channels=bn_size * growth_rate, + num_filters=growth_rate, + filter_size=3, + pad=1, + stride=1, + ) if dropout: self.dropout_func = Dropout(p=dropout, mode="downscale_in_infer") @@ -102,14 +113,9 @@ def forward(self, input): class DenseBlock(nn.Layer): - - def __init__(self, - num_channels, - num_layers, - bn_size, - growth_rate, - dropout, - name=None): + def __init__( + self, num_channels, num_layers, bn_size, growth_rate, dropout, name=None + ): super(DenseBlock, self).__init__() self.dropout = dropout self.dense_layer_func = [] @@ -119,10 +125,14 @@ def __init__(self, self.dense_layer_func.append( self.add_sublayer( "{}_{}".format(name, layer + 1), - DenseLayer(num_channels=pre_channel, - growth_rate=growth_rate, - bn_size=bn_size, - dropout=dropout))) + DenseLayer( + num_channels=pre_channel, + growth_rate=growth_rate, + bn_size=bn_size, + dropout=dropout, + ), + ) + ) pre_channel = pre_channel + growth_rate def forward(self, input): @@ -133,15 +143,16 @@ def forward(self, input): class TransitionLayer(nn.Layer): - def __init__(self, num_channels, num_output_features): super(TransitionLayer, self).__init__() - self.conv_ac_func = BNACConvLayer(num_channels=num_channels, - num_filters=num_output_features, - filter_size=1, - pad=0, - stride=1) + self.conv_ac_func = BNACConvLayer( + num_channels=num_channels, + num_filters=num_output_features, + filter_size=1, + pad=0, + stride=1, + ) self.pool2d_avg = AvgPool2D(kernel_size=2, stride=2, padding=0) @@ -152,25 +163,28 @@ def forward(self, input): class ConvBNLayer(nn.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - pad=0, - groups=1, - act="relu"): + def __init__( + self, + num_channels, + num_filters, + filter_size, + stride=1, + pad=0, + groups=1, + act="relu", + ): super(ConvBNLayer, self).__init__() - self._conv = Conv2D(in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=pad, - groups=groups, - weight_attr=ParamAttr(), - bias_attr=False) + self._conv = Conv2D( + in_channels=num_channels, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=pad, + groups=groups, + weight_attr=ParamAttr(), + bias_attr=False, + ) self._batch_norm = BatchNorm(num_filters, act=act) def forward(self, input): @@ -210,34 +224,40 @@ class DenseNet(nn.Layer): # [1, 1000] """ - def __init__(self, - layers=121, - bn_size=4, - dropout=0., - num_classes=1000, - with_pool=True): + def __init__( + self, + layers=121, + bn_size=4, + dropout=0.0, + num_classes=1000, + with_pool=True, + ): super(DenseNet, self).__init__() self.num_classes = num_classes self.with_pool = with_pool supported_layers = [121, 161, 169, 201, 264] - assert layers in supported_layers, \ - "supported layers are {} but input layer is {}".format( - supported_layers, layers) + assert ( + layers in supported_layers + ), "supported layers are {} but input layer is {}".format( + supported_layers, layers + ) densenet_spec = { 121: (64, 32, [6, 12, 24, 16]), 161: (96, 48, [6, 12, 36, 24]), 169: (64, 32, [6, 12, 32, 32]), 201: (64, 32, [6, 12, 48, 32]), - 264: (64, 32, [6, 12, 64, 48]) + 264: (64, 32, [6, 12, 64, 48]), } num_init_features, growth_rate, block_config = densenet_spec[layers] - self.conv1_func = ConvBNLayer(num_channels=3, - num_filters=num_init_features, - filter_size=7, - stride=2, - pad=3, - act='relu') + self.conv1_func = ConvBNLayer( + num_channels=3, + num_filters=num_init_features, + filter_size=7, + stride=2, + pad=3, + act='relu', + ) self.pool2d_max = MaxPool2D(kernel_size=3, stride=2, padding=1) self.block_config = block_config self.dense_block_func_list = [] @@ -248,12 +268,16 @@ def __init__(self, self.dense_block_func_list.append( self.add_sublayer( "db_conv_{}".format(i + 2), - DenseBlock(num_channels=pre_num_channels, - num_layers=num_layers, - bn_size=bn_size, - growth_rate=growth_rate, - dropout=dropout, - name='conv' + str(i + 2)))) + DenseBlock( + num_channels=pre_num_channels, + num_layers=num_layers, + bn_size=bn_size, + growth_rate=growth_rate, + dropout=dropout, + name='conv' + str(i + 2), + ), + ) + ) num_features = num_features + num_layers * growth_rate pre_num_channels = num_features @@ -262,8 +286,12 @@ def __init__(self, self.transition_func_list.append( self.add_sublayer( "tr_conv{}_blk".format(i + 2), - TransitionLayer(num_channels=pre_num_channels, - num_output_features=num_features // 2))) + TransitionLayer( + num_channels=pre_num_channels, + num_output_features=num_features // 2, + ), + ) + ) pre_num_channels = num_features // 2 num_features = num_features // 2 @@ -277,7 +305,8 @@ def __init__(self, num_features, num_classes, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr()) + bias_attr=ParamAttr(), + ) def forward(self, input): conv = self.conv1_func(input) @@ -303,10 +332,14 @@ def forward(self, input): def _densenet(arch, layers, pretrained, **kwargs): model = DenseNet(layers=layers, **kwargs) if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.set_dict(param) diff --git a/python/paddle/vision/models/googlenet.py b/python/paddle/vision/models/googlenet.py index 4eb02f1de3c88..8d6409dc42f80 100644 --- a/python/paddle/vision/models/googlenet.py +++ b/python/paddle/vision/models/googlenet.py @@ -25,35 +25,34 @@ __all__ = [] model_urls = { - "googlenet": - ("https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/GoogLeNet_pretrained.pdparams", - "80c06f038e905c53ab32c40eca6e26ae") + "googlenet": ( + "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/GoogLeNet_pretrained.pdparams", + "80c06f038e905c53ab32c40eca6e26ae", + ) } def xavier(channels, filter_size): - stdv = (3.0 / (filter_size**2 * channels))**0.5 + stdv = (3.0 / (filter_size**2 * channels)) ** 0.5 param_attr = ParamAttr(initializer=Uniform(-stdv, stdv)) return param_attr class ConvLayer(nn.Layer): - - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - groups=1): + def __init__( + self, num_channels, num_filters, filter_size, stride=1, groups=1 + ): super(ConvLayer, self).__init__() - self._conv = Conv2D(in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=(filter_size - 1) // 2, - groups=groups, - bias_attr=False) + self._conv = Conv2D( + in_channels=num_channels, + out_channels=num_filters, + kernel_size=filter_size, + stride=stride, + padding=(filter_size - 1) // 2, + groups=groups, + bias_attr=False, + ) def forward(self, inputs): y = self._conv(inputs) @@ -61,9 +60,17 @@ def forward(self, inputs): class Inception(nn.Layer): - - def __init__(self, input_channels, output_channels, filter1, filter3R, - filter3, filter5R, filter5, proj): + def __init__( + self, + input_channels, + output_channels, + filter1, + filter3R, + filter3, + filter5R, + filter5, + proj, + ): super(Inception, self).__init__() self._conv1 = ConvLayer(input_channels, filter1, 1) @@ -153,9 +160,9 @@ def __init__(self, num_classes=1000, with_pool=True): if num_classes > 0: # out self._drop = Dropout(p=0.4, mode="downscale_in_infer") - self._fc_out = Linear(1024, - num_classes, - weight_attr=xavier(1024, 1)) + self._fc_out = Linear( + 1024, num_classes, weight_attr=xavier(1024, 1) + ) # out1 self._conv_o1 = ConvLayer(512, 128, 1) @@ -254,9 +261,11 @@ def googlenet(pretrained=False, **kwargs): assert ( arch in model_urls ), "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.set_dict(param) diff --git a/python/paddle/vision/models/inceptionv3.py b/python/paddle/vision/models/inceptionv3.py index a4baa2e9a7762..24c55bdd7578f 100644 --- a/python/paddle/vision/models/inceptionv3.py +++ b/python/paddle/vision/models/inceptionv3.py @@ -26,45 +26,55 @@ __all__ = [] model_urls = { - "inception_v3": - ("https://paddle-hapi.bj.bcebos.com/models/inception_v3.pdparams", - "649a4547c3243e8b59c656f41fe330b8") + "inception_v3": ( + "https://paddle-hapi.bj.bcebos.com/models/inception_v3.pdparams", + "649a4547c3243e8b59c656f41fe330b8", + ) } class InceptionStem(nn.Layer): - def __init__(self): super().__init__() - self.conv_1a_3x3 = ConvNormActivation(in_channels=3, - out_channels=32, - kernel_size=3, - stride=2, - padding=0, - activation_layer=nn.ReLU) - self.conv_2a_3x3 = ConvNormActivation(in_channels=32, - out_channels=32, - kernel_size=3, - stride=1, - padding=0, - activation_layer=nn.ReLU) - self.conv_2b_3x3 = ConvNormActivation(in_channels=32, - out_channels=64, - kernel_size=3, - padding=1, - activation_layer=nn.ReLU) + self.conv_1a_3x3 = ConvNormActivation( + in_channels=3, + out_channels=32, + kernel_size=3, + stride=2, + padding=0, + activation_layer=nn.ReLU, + ) + self.conv_2a_3x3 = ConvNormActivation( + in_channels=32, + out_channels=32, + kernel_size=3, + stride=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.conv_2b_3x3 = ConvNormActivation( + in_channels=32, + out_channels=64, + kernel_size=3, + padding=1, + activation_layer=nn.ReLU, + ) self.max_pool = MaxPool2D(kernel_size=3, stride=2, padding=0) - self.conv_3b_1x1 = ConvNormActivation(in_channels=64, - out_channels=80, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.conv_4a_3x3 = ConvNormActivation(in_channels=80, - out_channels=192, - kernel_size=3, - padding=0, - activation_layer=nn.ReLU) + self.conv_3b_1x1 = ConvNormActivation( + in_channels=64, + out_channels=80, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.conv_4a_3x3 = ConvNormActivation( + in_channels=80, + out_channels=192, + kernel_size=3, + padding=0, + activation_layer=nn.ReLU, + ) def forward(self, x): x = self.conv_1a_3x3(x) @@ -78,51 +88,63 @@ def forward(self, x): class InceptionA(nn.Layer): - def __init__(self, num_channels, pool_features): super().__init__() - self.branch1x1 = ConvNormActivation(in_channels=num_channels, - out_channels=64, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - - self.branch5x5_1 = ConvNormActivation(in_channels=num_channels, - out_channels=48, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch5x5_2 = ConvNormActivation(in_channels=48, - out_channels=64, - kernel_size=5, - padding=2, - activation_layer=nn.ReLU) - - self.branch3x3dbl_1 = ConvNormActivation(in_channels=num_channels, - out_channels=64, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch3x3dbl_2 = ConvNormActivation(in_channels=64, - out_channels=96, - kernel_size=3, - padding=1, - activation_layer=nn.ReLU) - self.branch3x3dbl_3 = ConvNormActivation(in_channels=96, - out_channels=96, - kernel_size=3, - padding=1, - activation_layer=nn.ReLU) - - self.branch_pool = AvgPool2D(kernel_size=3, - stride=1, - padding=1, - exclusive=False) - self.branch_pool_conv = ConvNormActivation(in_channels=num_channels, - out_channels=pool_features, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) + self.branch1x1 = ConvNormActivation( + in_channels=num_channels, + out_channels=64, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + + self.branch5x5_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=48, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch5x5_2 = ConvNormActivation( + in_channels=48, + out_channels=64, + kernel_size=5, + padding=2, + activation_layer=nn.ReLU, + ) + + self.branch3x3dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=64, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch3x3dbl_2 = ConvNormActivation( + in_channels=64, + out_channels=96, + kernel_size=3, + padding=1, + activation_layer=nn.ReLU, + ) + self.branch3x3dbl_3 = ConvNormActivation( + in_channels=96, + out_channels=96, + kernel_size=3, + padding=1, + activation_layer=nn.ReLU, + ) + + self.branch_pool = AvgPool2D( + kernel_size=3, stride=1, padding=1, exclusive=False + ) + self.branch_pool_conv = ConvNormActivation( + in_channels=num_channels, + out_channels=pool_features, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) def forward(self, x): branch1x1 = self.branch1x1(x) @@ -135,38 +157,46 @@ def forward(self, x): branch_pool = self.branch_pool(x) branch_pool = self.branch_pool_conv(branch_pool) - x = paddle.concat([branch1x1, branch5x5, branch3x3dbl, branch_pool], - axis=1) + x = paddle.concat( + [branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=1 + ) return x class InceptionB(nn.Layer): - def __init__(self, num_channels): super().__init__() - self.branch3x3 = ConvNormActivation(in_channels=num_channels, - out_channels=384, - kernel_size=3, - stride=2, - padding=0, - activation_layer=nn.ReLU) - - self.branch3x3dbl_1 = ConvNormActivation(in_channels=num_channels, - out_channels=64, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch3x3dbl_2 = ConvNormActivation(in_channels=64, - out_channels=96, - kernel_size=3, - padding=1, - activation_layer=nn.ReLU) - self.branch3x3dbl_3 = ConvNormActivation(in_channels=96, - out_channels=96, - kernel_size=3, - stride=2, - padding=0, - activation_layer=nn.ReLU) + self.branch3x3 = ConvNormActivation( + in_channels=num_channels, + out_channels=384, + kernel_size=3, + stride=2, + padding=0, + activation_layer=nn.ReLU, + ) + + self.branch3x3dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=64, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch3x3dbl_2 = ConvNormActivation( + in_channels=64, + out_channels=96, + kernel_size=3, + padding=1, + activation_layer=nn.ReLU, + ) + self.branch3x3dbl_3 = ConvNormActivation( + in_channels=96, + out_channels=96, + kernel_size=3, + stride=2, + padding=0, + activation_layer=nn.ReLU, + ) self.branch_pool = MaxPool2D(kernel_size=3, stride=2) @@ -185,69 +215,87 @@ def forward(self, x): class InceptionC(nn.Layer): - def __init__(self, num_channels, channels_7x7): super().__init__() - self.branch1x1 = ConvNormActivation(in_channels=num_channels, - out_channels=192, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - - self.branch7x7_1 = ConvNormActivation(in_channels=num_channels, - out_channels=channels_7x7, - kernel_size=1, - stride=1, - padding=0, - activation_layer=nn.ReLU) - self.branch7x7_2 = ConvNormActivation(in_channels=channels_7x7, - out_channels=channels_7x7, - kernel_size=(1, 7), - stride=1, - padding=(0, 3), - activation_layer=nn.ReLU) - self.branch7x7_3 = ConvNormActivation(in_channels=channels_7x7, - out_channels=192, - kernel_size=(7, 1), - stride=1, - padding=(3, 0), - activation_layer=nn.ReLU) - - self.branch7x7dbl_1 = ConvNormActivation(in_channels=num_channels, - out_channels=channels_7x7, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch7x7dbl_2 = ConvNormActivation(in_channels=channels_7x7, - out_channels=channels_7x7, - kernel_size=(7, 1), - padding=(3, 0), - activation_layer=nn.ReLU) - self.branch7x7dbl_3 = ConvNormActivation(in_channels=channels_7x7, - out_channels=channels_7x7, - kernel_size=(1, 7), - padding=(0, 3), - activation_layer=nn.ReLU) - self.branch7x7dbl_4 = ConvNormActivation(in_channels=channels_7x7, - out_channels=channels_7x7, - kernel_size=(7, 1), - padding=(3, 0), - activation_layer=nn.ReLU) - self.branch7x7dbl_5 = ConvNormActivation(in_channels=channels_7x7, - out_channels=192, - kernel_size=(1, 7), - padding=(0, 3), - activation_layer=nn.ReLU) - - self.branch_pool = AvgPool2D(kernel_size=3, - stride=1, - padding=1, - exclusive=False) - self.branch_pool_conv = ConvNormActivation(in_channels=num_channels, - out_channels=192, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) + self.branch1x1 = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + + self.branch7x7_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=channels_7x7, + kernel_size=1, + stride=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch7x7_2 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(1, 7), + stride=1, + padding=(0, 3), + activation_layer=nn.ReLU, + ) + self.branch7x7_3 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=192, + kernel_size=(7, 1), + stride=1, + padding=(3, 0), + activation_layer=nn.ReLU, + ) + + self.branch7x7dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=channels_7x7, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch7x7dbl_2 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(7, 1), + padding=(3, 0), + activation_layer=nn.ReLU, + ) + self.branch7x7dbl_3 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(1, 7), + padding=(0, 3), + activation_layer=nn.ReLU, + ) + self.branch7x7dbl_4 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(7, 1), + padding=(3, 0), + activation_layer=nn.ReLU, + ) + self.branch7x7dbl_5 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=192, + kernel_size=(1, 7), + padding=(0, 3), + activation_layer=nn.ReLU, + ) + + self.branch_pool = AvgPool2D( + kernel_size=3, stride=1, padding=1, exclusive=False + ) + self.branch_pool_conv = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) def forward(self, x): branch1x1 = self.branch1x1(x) @@ -265,49 +313,61 @@ def forward(self, x): branch_pool = self.branch_pool(x) branch_pool = self.branch_pool_conv(branch_pool) - x = paddle.concat([branch1x1, branch7x7, branch7x7dbl, branch_pool], - axis=1) + x = paddle.concat( + [branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=1 + ) return x class InceptionD(nn.Layer): - def __init__(self, num_channels): super().__init__() - self.branch3x3_1 = ConvNormActivation(in_channels=num_channels, - out_channels=192, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch3x3_2 = ConvNormActivation(in_channels=192, - out_channels=320, - kernel_size=3, - stride=2, - padding=0, - activation_layer=nn.ReLU) - - self.branch7x7x3_1 = ConvNormActivation(in_channels=num_channels, - out_channels=192, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch7x7x3_2 = ConvNormActivation(in_channels=192, - out_channels=192, - kernel_size=(1, 7), - padding=(0, 3), - activation_layer=nn.ReLU) - self.branch7x7x3_3 = ConvNormActivation(in_channels=192, - out_channels=192, - kernel_size=(7, 1), - padding=(3, 0), - activation_layer=nn.ReLU) - self.branch7x7x3_4 = ConvNormActivation(in_channels=192, - out_channels=192, - kernel_size=3, - stride=2, - padding=0, - activation_layer=nn.ReLU) + self.branch3x3_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch3x3_2 = ConvNormActivation( + in_channels=192, + out_channels=320, + kernel_size=3, + stride=2, + padding=0, + activation_layer=nn.ReLU, + ) + + self.branch7x7x3_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch7x7x3_2 = ConvNormActivation( + in_channels=192, + out_channels=192, + kernel_size=(1, 7), + padding=(0, 3), + activation_layer=nn.ReLU, + ) + self.branch7x7x3_3 = ConvNormActivation( + in_channels=192, + out_channels=192, + kernel_size=(7, 1), + padding=(3, 0), + activation_layer=nn.ReLU, + ) + self.branch7x7x3_4 = ConvNormActivation( + in_channels=192, + out_channels=192, + kernel_size=3, + stride=2, + padding=0, + activation_layer=nn.ReLU, + ) self.branch_pool = MaxPool2D(kernel_size=3, stride=2) @@ -327,60 +387,76 @@ def forward(self, x): class InceptionE(nn.Layer): - def __init__(self, num_channels): super().__init__() - self.branch1x1 = ConvNormActivation(in_channels=num_channels, - out_channels=320, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch3x3_1 = ConvNormActivation(in_channels=num_channels, - out_channels=384, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch3x3_2a = ConvNormActivation(in_channels=384, - out_channels=384, - kernel_size=(1, 3), - padding=(0, 1), - activation_layer=nn.ReLU) - self.branch3x3_2b = ConvNormActivation(in_channels=384, - out_channels=384, - kernel_size=(3, 1), - padding=(1, 0), - activation_layer=nn.ReLU) - - self.branch3x3dbl_1 = ConvNormActivation(in_channels=num_channels, - out_channels=448, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) - self.branch3x3dbl_2 = ConvNormActivation(in_channels=448, - out_channels=384, - kernel_size=3, - padding=1, - activation_layer=nn.ReLU) - self.branch3x3dbl_3a = ConvNormActivation(in_channels=384, - out_channels=384, - kernel_size=(1, 3), - padding=(0, 1), - activation_layer=nn.ReLU) - self.branch3x3dbl_3b = ConvNormActivation(in_channels=384, - out_channels=384, - kernel_size=(3, 1), - padding=(1, 0), - activation_layer=nn.ReLU) - - self.branch_pool = AvgPool2D(kernel_size=3, - stride=1, - padding=1, - exclusive=False) - self.branch_pool_conv = ConvNormActivation(in_channels=num_channels, - out_channels=192, - kernel_size=1, - padding=0, - activation_layer=nn.ReLU) + self.branch1x1 = ConvNormActivation( + in_channels=num_channels, + out_channels=320, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch3x3_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=384, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch3x3_2a = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(1, 3), + padding=(0, 1), + activation_layer=nn.ReLU, + ) + self.branch3x3_2b = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(3, 1), + padding=(1, 0), + activation_layer=nn.ReLU, + ) + + self.branch3x3dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=448, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) + self.branch3x3dbl_2 = ConvNormActivation( + in_channels=448, + out_channels=384, + kernel_size=3, + padding=1, + activation_layer=nn.ReLU, + ) + self.branch3x3dbl_3a = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(1, 3), + padding=(0, 1), + activation_layer=nn.ReLU, + ) + self.branch3x3dbl_3b = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(3, 1), + padding=(1, 0), + activation_layer=nn.ReLU, + ) + + self.branch_pool = AvgPool2D( + kernel_size=3, stride=1, padding=1, exclusive=False + ) + self.branch_pool_conv = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU, + ) def forward(self, x): branch1x1 = self.branch1x1(x) @@ -403,8 +479,9 @@ def forward(self, x): branch_pool = self.branch_pool(x) branch_pool = self.branch_pool_conv(branch_pool) - x = paddle.concat([branch1x1, branch3x3, branch3x3dbl, branch_pool], - axis=1) + x = paddle.concat( + [branch1x1, branch3x3, branch3x3dbl, branch_pool], axis=1 + ) return x @@ -444,7 +521,7 @@ def __init__(self, num_classes=1000, with_pool=True): "inception_b": [288], "inception_c": [[768, 768, 768, 768], [128, 160, 160, 192]], "inception_d": [768], - "inception_e": [1280, 2048] + "inception_e": [1280, 2048], } inception_a_list = self.layers_config["inception_a"] @@ -457,8 +534,9 @@ def __init__(self, num_classes=1000, with_pool=True): self.inception_block_list = nn.LayerList() for i in range(len(inception_a_list[0])): - inception_a = InceptionA(inception_a_list[0][i], - inception_a_list[1][i]) + inception_a = InceptionA( + inception_a_list[0][i], inception_a_list[1][i] + ) self.inception_block_list.append(inception_a) for i in range(len(inception_b_list)): @@ -466,8 +544,9 @@ def __init__(self, num_classes=1000, with_pool=True): self.inception_block_list.append(inception_b) for i in range(len(inception_c_list[0])): - inception_c = InceptionC(inception_c_list[0][i], - inception_c_list[1][i]) + inception_c = InceptionC( + inception_c_list[0][i], inception_c_list[1][i] + ) self.inception_block_list.append(inception_c) for i in range(len(inception_d_list)): @@ -488,7 +567,8 @@ def __init__(self, num_classes=1000, with_pool=True): 2048, num_classes, weight_attr=ParamAttr(initializer=Uniform(-stdv, stdv)), - bias_attr=ParamAttr()) + bias_attr=ParamAttr(), + ) def forward(self, x): x = self.inception_stem(x) @@ -538,10 +618,14 @@ def inception_v3(pretrained=False, **kwargs): model = InceptionV3(**kwargs) arch = "inception_v3" if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.set_dict(param) diff --git a/python/paddle/vision/models/lenet.py b/python/paddle/vision/models/lenet.py index 44cfed2ef307b..49c9bd2d68a4f 100644 --- a/python/paddle/vision/models/lenet.py +++ b/python/paddle/vision/models/lenet.py @@ -1,16 +1,16 @@ # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve. # -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import paddle import paddle.nn as nn @@ -47,14 +47,21 @@ class LeNet(nn.Layer): def __init__(self, num_classes=10): super(LeNet, self).__init__() self.num_classes = num_classes - self.features = nn.Sequential(nn.Conv2D(1, 6, 3, stride=1, padding=1), - nn.ReLU(), nn.MaxPool2D(2, 2), - nn.Conv2D(6, 16, 5, stride=1, padding=0), - nn.ReLU(), nn.MaxPool2D(2, 2)) + self.features = nn.Sequential( + nn.Conv2D(1, 6, 3, stride=1, padding=1), + nn.ReLU(), + nn.MaxPool2D(2, 2), + nn.Conv2D(6, 16, 5, stride=1, padding=0), + nn.ReLU(), + nn.MaxPool2D(2, 2), + ) if num_classes > 0: - self.fc = nn.Sequential(nn.Linear(400, 120), nn.Linear(120, 84), - nn.Linear(84, num_classes)) + self.fc = nn.Sequential( + nn.Linear(400, 120), + nn.Linear(120, 84), + nn.Linear(84, num_classes), + ) def forward(self, inputs): x = self.features(inputs) diff --git a/python/paddle/vision/models/mobilenetv1.py b/python/paddle/vision/models/mobilenetv1.py index 22be953f2c135..78e83304db75b 100644 --- a/python/paddle/vision/models/mobilenetv1.py +++ b/python/paddle/vision/models/mobilenetv1.py @@ -21,31 +21,41 @@ __all__ = [] model_urls = { - 'mobilenetv1_1.0': - ('https://paddle-hapi.bj.bcebos.com/models/mobilenetv1_1.0.pdparams', - '3033ab1975b1670bef51545feb65fc45') + 'mobilenetv1_1.0': ( + 'https://paddle-hapi.bj.bcebos.com/models/mobilenetv1_1.0.pdparams', + '3033ab1975b1670bef51545feb65fc45', + ) } class DepthwiseSeparable(nn.Layer): - - def __init__(self, in_channels, out_channels1, out_channels2, num_groups, - stride, scale): + def __init__( + self, + in_channels, + out_channels1, + out_channels2, + num_groups, + stride, + scale, + ): super(DepthwiseSeparable, self).__init__() - self._depthwise_conv = ConvNormActivation(in_channels, - int(out_channels1 * scale), - kernel_size=3, - stride=stride, - padding=1, - groups=int(num_groups * - scale)) - - self._pointwise_conv = ConvNormActivation(int(out_channels1 * scale), - int(out_channels2 * scale), - kernel_size=1, - stride=1, - padding=0) + self._depthwise_conv = ConvNormActivation( + in_channels, + int(out_channels1 * scale), + kernel_size=3, + stride=stride, + padding=1, + groups=int(num_groups * scale), + ) + + self._pointwise_conv = ConvNormActivation( + int(out_channels1 * scale), + int(out_channels2 * scale), + kernel_size=1, + stride=1, + padding=0, + ) def forward(self, x): x = self._depthwise_conv(x) @@ -88,101 +98,130 @@ def __init__(self, scale=1.0, num_classes=1000, with_pool=True): self.num_classes = num_classes self.with_pool = with_pool - self.conv1 = ConvNormActivation(in_channels=3, - out_channels=int(32 * scale), - kernel_size=3, - stride=2, - padding=1) - - dws21 = self.add_sublayer(sublayer=DepthwiseSeparable(in_channels=int( - 32 * scale), - out_channels1=32, - out_channels2=64, - num_groups=32, - stride=1, - scale=scale), - name="conv2_1") + self.conv1 = ConvNormActivation( + in_channels=3, + out_channels=int(32 * scale), + kernel_size=3, + stride=2, + padding=1, + ) + + dws21 = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(32 * scale), + out_channels1=32, + out_channels2=64, + num_groups=32, + stride=1, + scale=scale, + ), + name="conv2_1", + ) self.dwsl.append(dws21) - dws22 = self.add_sublayer(sublayer=DepthwiseSeparable(in_channels=int( - 64 * scale), - out_channels1=64, - out_channels2=128, - num_groups=64, - stride=2, - scale=scale), - name="conv2_2") + dws22 = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(64 * scale), + out_channels1=64, + out_channels2=128, + num_groups=64, + stride=2, + scale=scale, + ), + name="conv2_2", + ) self.dwsl.append(dws22) - dws31 = self.add_sublayer(sublayer=DepthwiseSeparable(in_channels=int( - 128 * scale), - out_channels1=128, - out_channels2=128, - num_groups=128, - stride=1, - scale=scale), - name="conv3_1") + dws31 = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(128 * scale), + out_channels1=128, + out_channels2=128, + num_groups=128, + stride=1, + scale=scale, + ), + name="conv3_1", + ) self.dwsl.append(dws31) - dws32 = self.add_sublayer(sublayer=DepthwiseSeparable(in_channels=int( - 128 * scale), - out_channels1=128, - out_channels2=256, - num_groups=128, - stride=2, - scale=scale), - name="conv3_2") + dws32 = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(128 * scale), + out_channels1=128, + out_channels2=256, + num_groups=128, + stride=2, + scale=scale, + ), + name="conv3_2", + ) self.dwsl.append(dws32) - dws41 = self.add_sublayer(sublayer=DepthwiseSeparable(in_channels=int( - 256 * scale), - out_channels1=256, - out_channels2=256, - num_groups=256, - stride=1, - scale=scale), - name="conv4_1") + dws41 = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(256 * scale), + out_channels1=256, + out_channels2=256, + num_groups=256, + stride=1, + scale=scale, + ), + name="conv4_1", + ) self.dwsl.append(dws41) - dws42 = self.add_sublayer(sublayer=DepthwiseSeparable(in_channels=int( - 256 * scale), - out_channels1=256, - out_channels2=512, - num_groups=256, - stride=2, - scale=scale), - name="conv4_2") + dws42 = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(256 * scale), + out_channels1=256, + out_channels2=512, + num_groups=256, + stride=2, + scale=scale, + ), + name="conv4_2", + ) self.dwsl.append(dws42) for i in range(5): - tmp = self.add_sublayer(sublayer=DepthwiseSeparable( + tmp = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(512 * scale), + out_channels1=512, + out_channels2=512, + num_groups=512, + stride=1, + scale=scale, + ), + name="conv5_" + str(i + 1), + ) + self.dwsl.append(tmp) + + dws56 = self.add_sublayer( + sublayer=DepthwiseSeparable( in_channels=int(512 * scale), out_channels1=512, - out_channels2=512, + out_channels2=1024, num_groups=512, - stride=1, - scale=scale), - name="conv5_" + str(i + 1)) - self.dwsl.append(tmp) - - dws56 = self.add_sublayer(sublayer=DepthwiseSeparable( - in_channels=int(512 * scale), - out_channels1=512, - out_channels2=1024, - num_groups=512, - stride=2, - scale=scale), - name="conv5_6") + stride=2, + scale=scale, + ), + name="conv5_6", + ) self.dwsl.append(dws56) - dws6 = self.add_sublayer(sublayer=DepthwiseSeparable(in_channels=int( - 1024 * scale), - out_channels1=1024, - out_channels2=1024, - num_groups=1024, - stride=1, - scale=scale), - name="conv6") + dws6 = self.add_sublayer( + sublayer=DepthwiseSeparable( + in_channels=int(1024 * scale), + out_channels1=1024, + out_channels2=1024, + num_groups=1024, + stride=1, + scale=scale, + ), + name="conv6", + ) self.dwsl.append(dws6) if with_pool: @@ -208,10 +247,14 @@ def forward(self, x): def _mobilenet(arch, pretrained=False, **kwargs): model = MobileNetV1(**kwargs) if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.load_dict(param) @@ -253,8 +296,7 @@ def mobilenet_v1(pretrained=False, scale=1.0, **kwargs): print(out.shape) # [1, 1000] """ - model = _mobilenet('mobilenetv1_' + str(scale), - pretrained, - scale=scale, - **kwargs) + model = _mobilenet( + 'mobilenetv1_' + str(scale), pretrained, scale=scale, **kwargs + ) return model diff --git a/python/paddle/vision/models/mobilenetv2.py b/python/paddle/vision/models/mobilenetv2.py index f67ea54fa63d8..43fade9e6dd57 100644 --- a/python/paddle/vision/models/mobilenetv2.py +++ b/python/paddle/vision/models/mobilenetv2.py @@ -22,20 +22,17 @@ __all__ = [] model_urls = { - 'mobilenetv2_1.0': - ('https://paddle-hapi.bj.bcebos.com/models/mobilenet_v2_x1.0.pdparams', - '0340af0a901346c8d46f4529882fb63d') + 'mobilenetv2_1.0': ( + 'https://paddle-hapi.bj.bcebos.com/models/mobilenet_v2_x1.0.pdparams', + '0340af0a901346c8d46f4529882fb63d', + ) } class InvertedResidual(nn.Layer): - - def __init__(self, - inp, - oup, - stride, - expand_ratio, - norm_layer=nn.BatchNorm2D): + def __init__( + self, inp, oup, stride, expand_ratio, norm_layer=nn.BatchNorm2D + ): super(InvertedResidual, self).__init__() self.stride = stride assert stride in [1, 2] @@ -46,21 +43,28 @@ def __init__(self, layers = [] if expand_ratio != 1: layers.append( - ConvNormActivation(inp, - hidden_dim, - kernel_size=1, - norm_layer=norm_layer, - activation_layer=nn.ReLU6)) - layers.extend([ - ConvNormActivation(hidden_dim, - hidden_dim, - stride=stride, - groups=hidden_dim, - norm_layer=norm_layer, - activation_layer=nn.ReLU6), - nn.Conv2D(hidden_dim, oup, 1, 1, 0, bias_attr=False), - norm_layer(oup), - ]) + ConvNormActivation( + inp, + hidden_dim, + kernel_size=1, + norm_layer=norm_layer, + activation_layer=nn.ReLU6, + ) + ) + layers.extend( + [ + ConvNormActivation( + hidden_dim, + hidden_dim, + stride=stride, + groups=hidden_dim, + norm_layer=norm_layer, + activation_layer=nn.ReLU6, + ), + nn.Conv2D(hidden_dim, oup, 1, 1, 0, bias_attr=False), + norm_layer(oup), + ] + ) self.conv = nn.Sequential(*layers) def forward(self, x): @@ -119,14 +123,17 @@ def __init__(self, scale=1.0, num_classes=1000, with_pool=True): ] input_channel = _make_divisible(input_channel * scale, round_nearest) - self.last_channel = _make_divisible(last_channel * max(1.0, scale), - round_nearest) + self.last_channel = _make_divisible( + last_channel * max(1.0, scale), round_nearest + ) features = [ - ConvNormActivation(3, - input_channel, - stride=2, - norm_layer=norm_layer, - activation_layer=nn.ReLU6) + ConvNormActivation( + 3, + input_channel, + stride=2, + norm_layer=norm_layer, + activation_layer=nn.ReLU6, + ) ] for t, c, n, s in inverted_residual_setting: @@ -134,19 +141,25 @@ def __init__(self, scale=1.0, num_classes=1000, with_pool=True): for i in range(n): stride = s if i == 0 else 1 features.append( - block(input_channel, - output_channel, - stride, - expand_ratio=t, - norm_layer=norm_layer)) + block( + input_channel, + output_channel, + stride, + expand_ratio=t, + norm_layer=norm_layer, + ) + ) input_channel = output_channel features.append( - ConvNormActivation(input_channel, - self.last_channel, - kernel_size=1, - norm_layer=norm_layer, - activation_layer=nn.ReLU6)) + ConvNormActivation( + input_channel, + self.last_channel, + kernel_size=1, + norm_layer=norm_layer, + activation_layer=nn.ReLU6, + ) + ) self.features = nn.Sequential(*features) @@ -155,7 +168,8 @@ def __init__(self, scale=1.0, num_classes=1000, with_pool=True): if self.num_classes > 0: self.classifier = nn.Sequential( - nn.Dropout(0.2), nn.Linear(self.last_channel, num_classes)) + nn.Dropout(0.2), nn.Linear(self.last_channel, num_classes) + ) def forward(self, x): x = self.features(x) @@ -172,10 +186,14 @@ def forward(self, x): def _mobilenet(arch, pretrained=False, **kwargs): model = MobileNetV2(**kwargs) if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.load_dict(param) @@ -217,8 +235,7 @@ def mobilenet_v2(pretrained=False, scale=1.0, **kwargs): print(out.shape) # [1, 1000] """ - model = _mobilenet('mobilenetv2_' + str(scale), - pretrained, - scale=scale, - **kwargs) + model = _mobilenet( + 'mobilenetv2_' + str(scale), pretrained, scale=scale, **kwargs + ) return model diff --git a/python/paddle/vision/models/mobilenetv3.py b/python/paddle/vision/models/mobilenetv3.py index f1bf6d8ae3730..865f8d0dd3222 100644 --- a/python/paddle/vision/models/mobilenetv3.py +++ b/python/paddle/vision/models/mobilenetv3.py @@ -23,12 +23,14 @@ __all__ = [] model_urls = { - "mobilenet_v3_small_x1.0": - ("https://paddle-hapi.bj.bcebos.com/models/mobilenet_v3_small_x1.0.pdparams", - "34fe0e7c1f8b00b2b056ad6788d0590c"), - "mobilenet_v3_large_x1.0": - ("https://paddle-hapi.bj.bcebos.com/models/mobilenet_v3_large_x1.0.pdparams", - "118db5792b4e183b925d8e8e334db3df"), + "mobilenet_v3_small_x1.0": ( + "https://paddle-hapi.bj.bcebos.com/models/mobilenet_v3_small_x1.0.pdparams", + "34fe0e7c1f8b00b2b056ad6788d0590c", + ), + "mobilenet_v3_large_x1.0": ( + "https://paddle-hapi.bj.bcebos.com/models/mobilenet_v3_large_x1.0.pdparams", + "118db5792b4e183b925d8e8e334db3df", + ), } @@ -45,11 +47,13 @@ class SqueezeExcitation(nn.Layer): scale_activation (Callable[..., paddle.nn.Layer]): ``sigma`` activation. Default: ``paddle.nn.Sigmoid`` """ - def __init__(self, - input_channels, - squeeze_channels, - activation=nn.ReLU, - scale_activation=nn.Sigmoid): + def __init__( + self, + input_channels, + squeeze_channels, + activation=nn.ReLU, + scale_activation=nn.Sigmoid, + ): super().__init__() self.avgpool = nn.AdaptiveAvgPool2D(1) self.fc1 = nn.Conv2D(input_channels, squeeze_channels, 1) @@ -70,20 +74,22 @@ def forward(self, input): class InvertedResidualConfig: - - def __init__(self, - in_channels, - kernel, - expanded_channels, - out_channels, - use_se, - activation, - stride, - scale=1.0): + def __init__( + self, + in_channels, + kernel, + expanded_channels, + out_channels, + use_se, + activation, + stride, + scale=1.0, + ): self.in_channels = self.adjust_channels(in_channels, scale=scale) self.kernel = kernel - self.expanded_channels = self.adjust_channels(expanded_channels, - scale=scale) + self.expanded_channels = self.adjust_channels( + expanded_channels, scale=scale + ) self.out_channels = self.adjust_channels(out_channels, scale=scale) self.use_se = use_se if activation is None: @@ -95,7 +101,9 @@ def __init__(self, else: raise RuntimeError( "The activation function is not supported: {}".format( - activation)) + activation + ) + ) self.stride = stride @staticmethod @@ -104,9 +112,17 @@ def adjust_channels(channels, scale=1.0): class InvertedResidual(nn.Layer): - - def __init__(self, in_channels, expanded_channels, out_channels, - filter_size, stride, use_se, activation_layer, norm_layer): + def __init__( + self, + in_channels, + expanded_channels, + out_channels, + filter_size, + stride, + use_se, + activation_layer, + norm_layer, + ): super().__init__() self.use_res_connect = stride == 1 and in_channels == out_channels self.use_se = use_se @@ -120,7 +136,8 @@ def __init__(self, in_channels, expanded_channels, out_channels, stride=1, padding=0, norm_layer=norm_layer, - activation_layer=activation_layer) + activation_layer=activation_layer, + ) self.bottleneck_conv = ConvNormActivation( in_channels=expanded_channels, @@ -130,21 +147,25 @@ def __init__(self, in_channels, expanded_channels, out_channels, padding=int((filter_size - 1) // 2), groups=expanded_channels, norm_layer=norm_layer, - activation_layer=activation_layer) + activation_layer=activation_layer, + ) if self.use_se: - self.mid_se = SqueezeExcitation(expanded_channels, - _make_divisible(expanded_channels // - 4), - scale_activation=nn.Hardsigmoid) - - self.linear_conv = ConvNormActivation(in_channels=expanded_channels, - out_channels=out_channels, - kernel_size=1, - stride=1, - padding=0, - norm_layer=norm_layer, - activation_layer=None) + self.mid_se = SqueezeExcitation( + expanded_channels, + _make_divisible(expanded_channels // 4), + scale_activation=nn.Hardsigmoid, + ) + + self.linear_conv = ConvNormActivation( + in_channels=expanded_channels, + out_channels=out_channels, + kernel_size=1, + stride=1, + padding=0, + norm_layer=norm_layer, + activation_layer=None, + ) def forward(self, x): identity = x @@ -172,12 +193,9 @@ class MobileNetV3(nn.Layer): with_pool (bool, optional): Use pool before the last fc layer or not. Default: True. """ - def __init__(self, - config, - last_channel, - scale=1.0, - num_classes=1000, - with_pool=True): + def __init__( + self, config, last_channel, scale=1.0, num_classes=1000, with_pool=True + ): super().__init__() self.config = config @@ -190,25 +208,32 @@ def __init__(self, self.lastconv_out_channels = self.lastconv_in_channels * 6 norm_layer = partial(nn.BatchNorm2D, epsilon=0.001, momentum=0.99) - self.conv = ConvNormActivation(in_channels=3, - out_channels=self.firstconv_in_channels, - kernel_size=3, - stride=2, - padding=1, - groups=1, - activation_layer=nn.Hardswish, - norm_layer=norm_layer) - - self.blocks = nn.Sequential(*[ - InvertedResidual(in_channels=cfg.in_channels, - expanded_channels=cfg.expanded_channels, - out_channels=cfg.out_channels, - filter_size=cfg.kernel, - stride=cfg.stride, - use_se=cfg.use_se, - activation_layer=cfg.activation_layer, - norm_layer=norm_layer) for cfg in self.config - ]) + self.conv = ConvNormActivation( + in_channels=3, + out_channels=self.firstconv_in_channels, + kernel_size=3, + stride=2, + padding=1, + groups=1, + activation_layer=nn.Hardswish, + norm_layer=norm_layer, + ) + + self.blocks = nn.Sequential( + *[ + InvertedResidual( + in_channels=cfg.in_channels, + expanded_channels=cfg.expanded_channels, + out_channels=cfg.out_channels, + filter_size=cfg.kernel, + stride=cfg.stride, + use_se=cfg.use_se, + activation_layer=cfg.activation_layer, + norm_layer=norm_layer, + ) + for cfg in self.config + ] + ) self.lastconv = ConvNormActivation( in_channels=self.lastconv_in_channels, @@ -218,7 +243,8 @@ def __init__(self, padding=0, groups=1, norm_layer=norm_layer, - activation_layer=nn.Hardswish) + activation_layer=nn.Hardswish, + ) if with_pool: self.avgpool = nn.AdaptiveAvgPool2D(1) @@ -226,8 +252,10 @@ def __init__(self, if num_classes > 0: self.classifier = nn.Sequential( nn.Linear(self.lastconv_out_channels, self.last_channel), - nn.Hardswish(), nn.Dropout(p=0.2), - nn.Linear(self.last_channel, num_classes)) + nn.Hardswish(), + nn.Dropout(p=0.2), + nn.Linear(self.last_channel, num_classes), + ) def forward(self, x): x = self.conv(x) @@ -288,11 +316,13 @@ def __init__(self, scale=1.0, num_classes=1000, with_pool=True): InvertedResidualConfig(96, 5, 576, 96, True, "hardswish", 1, scale), ] last_channel = _make_divisible(1024 * scale, 8) - super().__init__(config, - last_channel=last_channel, - scale=scale, - with_pool=with_pool, - num_classes=num_classes) + super().__init__( + config, + last_channel=last_channel, + scale=scale, + with_pool=with_pool, + num_classes=num_classes, + ) class MobileNetV3Large(MobileNetV3): @@ -332,31 +362,42 @@ def __init__(self, scale=1.0, num_classes=1000, with_pool=True): InvertedResidualConfig(24, 5, 72, 40, True, "relu", 2, scale), InvertedResidualConfig(40, 5, 120, 40, True, "relu", 1, scale), InvertedResidualConfig(40, 5, 120, 40, True, "relu", 1, scale), - InvertedResidualConfig(40, 3, 240, 80, False, "hardswish", 2, - scale), - InvertedResidualConfig(80, 3, 200, 80, False, "hardswish", 1, - scale), - InvertedResidualConfig(80, 3, 184, 80, False, "hardswish", 1, - scale), - InvertedResidualConfig(80, 3, 184, 80, False, "hardswish", 1, - scale), - InvertedResidualConfig(80, 3, 480, 112, True, "hardswish", 1, - scale), - InvertedResidualConfig(112, 3, 672, 112, True, "hardswish", 1, - scale), - InvertedResidualConfig(112, 5, 672, 160, True, "hardswish", 2, - scale), - InvertedResidualConfig(160, 5, 960, 160, True, "hardswish", 1, - scale), - InvertedResidualConfig(160, 5, 960, 160, True, "hardswish", 1, - scale), + InvertedResidualConfig( + 40, 3, 240, 80, False, "hardswish", 2, scale + ), + InvertedResidualConfig( + 80, 3, 200, 80, False, "hardswish", 1, scale + ), + InvertedResidualConfig( + 80, 3, 184, 80, False, "hardswish", 1, scale + ), + InvertedResidualConfig( + 80, 3, 184, 80, False, "hardswish", 1, scale + ), + InvertedResidualConfig( + 80, 3, 480, 112, True, "hardswish", 1, scale + ), + InvertedResidualConfig( + 112, 3, 672, 112, True, "hardswish", 1, scale + ), + InvertedResidualConfig( + 112, 5, 672, 160, True, "hardswish", 2, scale + ), + InvertedResidualConfig( + 160, 5, 960, 160, True, "hardswish", 1, scale + ), + InvertedResidualConfig( + 160, 5, 960, 160, True, "hardswish", 1, scale + ), ] last_channel = _make_divisible(1280 * scale, 8) - super().__init__(config, - last_channel=last_channel, - scale=scale, - with_pool=with_pool, - num_classes=num_classes) + super().__init__( + config, + last_channel=last_channel, + scale=scale, + with_pool=with_pool, + num_classes=num_classes, + ) def _mobilenet_v3(arch, pretrained=False, scale=1.0, **kwargs): @@ -369,9 +410,11 @@ def _mobilenet_v3(arch, pretrained=False, scale=1.0, **kwargs): assert ( arch in model_urls ), "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.set_dict(param) @@ -412,10 +455,9 @@ def mobilenet_v3_small(pretrained=False, scale=1.0, **kwargs): print(out.shape) # [1, 1000] """ - model = _mobilenet_v3("mobilenet_v3_small", - scale=scale, - pretrained=pretrained, - **kwargs) + model = _mobilenet_v3( + "mobilenet_v3_small", scale=scale, pretrained=pretrained, **kwargs + ) return model @@ -453,8 +495,7 @@ def mobilenet_v3_large(pretrained=False, scale=1.0, **kwargs): print(out.shape) # [1, 1000] """ - model = _mobilenet_v3("mobilenet_v3_large", - scale=scale, - pretrained=pretrained, - **kwargs) + model = _mobilenet_v3( + "mobilenet_v3_large", scale=scale, pretrained=pretrained, **kwargs + ) return model diff --git a/python/paddle/vision/models/resnet.py b/python/paddle/vision/models/resnet.py index 4293e119582c0..f43bdb9bc1a4c 100644 --- a/python/paddle/vision/models/resnet.py +++ b/python/paddle/vision/models/resnet.py @@ -20,69 +20,87 @@ __all__ = [] model_urls = { - 'resnet18': ('https://paddle-hapi.bj.bcebos.com/models/resnet18.pdparams', - 'cf548f46534aa3560945be4b95cd11c4'), - 'resnet34': ('https://paddle-hapi.bj.bcebos.com/models/resnet34.pdparams', - '8d2275cf8706028345f78ac0e1d31969'), - 'resnet50': ('https://paddle-hapi.bj.bcebos.com/models/resnet50.pdparams', - 'ca6f485ee1ab0492d38f323885b0ad80'), - 'resnet101': ('https://paddle-hapi.bj.bcebos.com/models/resnet101.pdparams', - '02f35f034ca3858e1e54d4036443c92d'), - 'resnet152': ('https://paddle-hapi.bj.bcebos.com/models/resnet152.pdparams', - '7ad16a2f1e7333859ff986138630fd7a'), - 'resnext50_32x4d': - ('https://paddle-hapi.bj.bcebos.com/models/resnext50_32x4d.pdparams', - 'dc47483169be7d6f018fcbb7baf8775d'), - "resnext50_64x4d": - ('https://paddle-hapi.bj.bcebos.com/models/resnext50_64x4d.pdparams', - '063d4b483e12b06388529450ad7576db'), - 'resnext101_32x4d': - ('https://paddle-hapi.bj.bcebos.com/models/resnext101_32x4d.pdparams', - '967b090039f9de2c8d06fe994fb9095f'), - 'resnext101_64x4d': - ('https://paddle-hapi.bj.bcebos.com/models/resnext101_64x4d.pdparams', - '98e04e7ca616a066699230d769d03008'), - 'resnext152_32x4d': - ('https://paddle-hapi.bj.bcebos.com/models/resnext152_32x4d.pdparams', - '18ff0beee21f2efc99c4b31786107121'), - 'resnext152_64x4d': - ('https://paddle-hapi.bj.bcebos.com/models/resnext152_64x4d.pdparams', - '77c4af00ca42c405fa7f841841959379'), - 'wide_resnet50_2': - ('https://paddle-hapi.bj.bcebos.com/models/wide_resnet50_2.pdparams', - '0282f804d73debdab289bd9fea3fa6dc'), - 'wide_resnet101_2': - ('https://paddle-hapi.bj.bcebos.com/models/wide_resnet101_2.pdparams', - 'd4360a2d23657f059216f5d5a1a9ac93'), + 'resnet18': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnet18.pdparams', + 'cf548f46534aa3560945be4b95cd11c4', + ), + 'resnet34': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnet34.pdparams', + '8d2275cf8706028345f78ac0e1d31969', + ), + 'resnet50': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnet50.pdparams', + 'ca6f485ee1ab0492d38f323885b0ad80', + ), + 'resnet101': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnet101.pdparams', + '02f35f034ca3858e1e54d4036443c92d', + ), + 'resnet152': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnet152.pdparams', + '7ad16a2f1e7333859ff986138630fd7a', + ), + 'resnext50_32x4d': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext50_32x4d.pdparams', + 'dc47483169be7d6f018fcbb7baf8775d', + ), + "resnext50_64x4d": ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext50_64x4d.pdparams', + '063d4b483e12b06388529450ad7576db', + ), + 'resnext101_32x4d': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext101_32x4d.pdparams', + '967b090039f9de2c8d06fe994fb9095f', + ), + 'resnext101_64x4d': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext101_64x4d.pdparams', + '98e04e7ca616a066699230d769d03008', + ), + 'resnext152_32x4d': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext152_32x4d.pdparams', + '18ff0beee21f2efc99c4b31786107121', + ), + 'resnext152_64x4d': ( + 'https://paddle-hapi.bj.bcebos.com/models/resnext152_64x4d.pdparams', + '77c4af00ca42c405fa7f841841959379', + ), + 'wide_resnet50_2': ( + 'https://paddle-hapi.bj.bcebos.com/models/wide_resnet50_2.pdparams', + '0282f804d73debdab289bd9fea3fa6dc', + ), + 'wide_resnet101_2': ( + 'https://paddle-hapi.bj.bcebos.com/models/wide_resnet101_2.pdparams', + 'd4360a2d23657f059216f5d5a1a9ac93', + ), } class BasicBlock(nn.Layer): expansion = 1 - def __init__(self, - inplanes, - planes, - stride=1, - downsample=None, - groups=1, - base_width=64, - dilation=1, - norm_layer=None): + def __init__( + self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None, + ): super(BasicBlock, self).__init__() if norm_layer is None: norm_layer = nn.BatchNorm2D if dilation > 1: raise NotImplementedError( - "Dilation > 1 not supported in BasicBlock") - - self.conv1 = nn.Conv2D(inplanes, - planes, - 3, - padding=1, - stride=stride, - bias_attr=False) + "Dilation > 1 not supported in BasicBlock" + ) + + self.conv1 = nn.Conv2D( + inplanes, planes, 3, padding=1, stride=stride, bias_attr=False + ) self.bn1 = norm_layer(planes) self.relu = nn.ReLU() self.conv2 = nn.Conv2D(planes, planes, 3, padding=1, bias_attr=False) @@ -113,37 +131,40 @@ class BottleneckBlock(nn.Layer): expansion = 4 - def __init__(self, - inplanes, - planes, - stride=1, - downsample=None, - groups=1, - base_width=64, - dilation=1, - norm_layer=None): + def __init__( + self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1, + norm_layer=None, + ): super(BottleneckBlock, self).__init__() if norm_layer is None: norm_layer = nn.BatchNorm2D - width = int(planes * (base_width / 64.)) * groups + width = int(planes * (base_width / 64.0)) * groups self.conv1 = nn.Conv2D(inplanes, width, 1, bias_attr=False) self.bn1 = norm_layer(width) - self.conv2 = nn.Conv2D(width, - width, - 3, - padding=dilation, - stride=stride, - groups=groups, - dilation=dilation, - bias_attr=False) + self.conv2 = nn.Conv2D( + width, + width, + 3, + padding=dilation, + stride=stride, + groups=groups, + dilation=dilation, + bias_attr=False, + ) self.bn2 = norm_layer(width) - self.conv3 = nn.Conv2D(width, - planes * self.expansion, - 1, - bias_attr=False) + self.conv3 = nn.Conv2D( + width, planes * self.expansion, 1, bias_attr=False + ) self.bn3 = norm_layer(planes * self.expansion) self.relu = nn.ReLU() self.downsample = downsample @@ -214,20 +235,22 @@ class ResNet(nn.Layer): # [1, 1000] """ - def __init__(self, - block, - depth=50, - width=64, - num_classes=1000, - with_pool=True, - groups=1): + def __init__( + self, + block, + depth=50, + width=64, + num_classes=1000, + with_pool=True, + groups=1, + ): super(ResNet, self).__init__() layer_cfg = { 18: [2, 2, 2, 2], 34: [3, 4, 6, 3], 50: [3, 4, 6, 3], 101: [3, 4, 23, 3], - 152: [3, 8, 36, 3] + 152: [3, 8, 36, 3], } layers = layer_cfg[depth] self.groups = groups @@ -239,12 +262,14 @@ def __init__(self, self.inplanes = 64 self.dilation = 1 - self.conv1 = nn.Conv2D(3, - self.inplanes, - kernel_size=7, - stride=2, - padding=3, - bias_attr=False) + self.conv1 = nn.Conv2D( + 3, + self.inplanes, + kernel_size=7, + stride=2, + padding=3, + bias_attr=False, + ) self.bn1 = self._norm_layer(self.inplanes) self.relu = nn.ReLU() self.maxpool = nn.MaxPool2D(kernel_size=3, stride=2, padding=1) @@ -267,26 +292,40 @@ def _make_layer(self, block, planes, blocks, stride=1, dilate=False): stride = 1 if stride != 1 or self.inplanes != planes * block.expansion: downsample = nn.Sequential( - nn.Conv2D(self.inplanes, - planes * block.expansion, - 1, - stride=stride, - bias_attr=False), + nn.Conv2D( + self.inplanes, + planes * block.expansion, + 1, + stride=stride, + bias_attr=False, + ), norm_layer(planes * block.expansion), ) layers = [] layers.append( - block(self.inplanes, planes, stride, downsample, self.groups, - self.base_width, previous_dilation, norm_layer)) + block( + self.inplanes, + planes, + stride, + downsample, + self.groups, + self.base_width, + previous_dilation, + norm_layer, + ) + ) self.inplanes = planes * block.expansion for _ in range(1, blocks): layers.append( - block(self.inplanes, - planes, - groups=self.groups, - base_width=self.base_width, - norm_layer=norm_layer)) + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + norm_layer=norm_layer, + ) + ) return nn.Sequential(*layers) @@ -313,10 +352,14 @@ def forward(self, x): def _resnet(arch, Block, depth, pretrained, **kwargs): model = ResNet(Block, depth, **kwargs) if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.set_dict(param) @@ -591,8 +634,9 @@ def resnext101_32x4d(pretrained=False, **kwargs): """ kwargs['groups'] = 32 kwargs['width'] = 4 - return _resnet('resnext101_32x4d', BottleneckBlock, 101, pretrained, - **kwargs) + return _resnet( + 'resnext101_32x4d', BottleneckBlock, 101, pretrained, **kwargs + ) def resnext101_64x4d(pretrained=False, **kwargs): @@ -627,8 +671,9 @@ def resnext101_64x4d(pretrained=False, **kwargs): """ kwargs['groups'] = 64 kwargs['width'] = 4 - return _resnet('resnext101_64x4d', BottleneckBlock, 101, pretrained, - **kwargs) + return _resnet( + 'resnext101_64x4d', BottleneckBlock, 101, pretrained, **kwargs + ) def resnext152_32x4d(pretrained=False, **kwargs): @@ -663,8 +708,9 @@ def resnext152_32x4d(pretrained=False, **kwargs): """ kwargs['groups'] = 32 kwargs['width'] = 4 - return _resnet('resnext152_32x4d', BottleneckBlock, 152, pretrained, - **kwargs) + return _resnet( + 'resnext152_32x4d', BottleneckBlock, 152, pretrained, **kwargs + ) def resnext152_64x4d(pretrained=False, **kwargs): @@ -699,8 +745,9 @@ def resnext152_64x4d(pretrained=False, **kwargs): """ kwargs['groups'] = 64 kwargs['width'] = 4 - return _resnet('resnext152_64x4d', BottleneckBlock, 152, pretrained, - **kwargs) + return _resnet( + 'resnext152_64x4d', BottleneckBlock, 152, pretrained, **kwargs + ) def wide_resnet50_2(pretrained=False, **kwargs): @@ -768,5 +815,6 @@ def wide_resnet101_2(pretrained=False, **kwargs): # [1, 1000] """ kwargs['width'] = 64 * 2 - return _resnet('wide_resnet101_2', BottleneckBlock, 101, pretrained, - **kwargs) + return _resnet( + 'wide_resnet101_2', BottleneckBlock, 101, pretrained, **kwargs + ) diff --git a/python/paddle/vision/models/shufflenetv2.py b/python/paddle/vision/models/shufflenetv2.py index e6cadaed1fd9b..c443fa5a3ee56 100644 --- a/python/paddle/vision/models/shufflenetv2.py +++ b/python/paddle/vision/models/shufflenetv2.py @@ -62,7 +62,8 @@ def create_activation_layer(act): return None else: raise RuntimeError( - "The activation function is not supported: {}".format(act)) + "The activation function is not supported: {}".format(act) + ) def channel_shuffle(x, groups): @@ -71,7 +72,8 @@ def channel_shuffle(x, groups): # reshape x = paddle.reshape( - x, shape=[batch_size, groups, channels_per_group, height, width]) + x, shape=[batch_size, groups, channels_per_group, height, width] + ) # transpose x = paddle.transpose(x, perm=[0, 2, 1, 3, 4]) @@ -82,27 +84,28 @@ def channel_shuffle(x, groups): class InvertedResidual(nn.Layer): - - def __init__(self, - in_channels, - out_channels, - stride, - activation_layer=nn.ReLU): + def __init__( + self, in_channels, out_channels, stride, activation_layer=nn.ReLU + ): super(InvertedResidual, self).__init__() - self._conv_pw = ConvNormActivation(in_channels=in_channels // 2, - out_channels=out_channels // 2, - kernel_size=1, - stride=1, - padding=0, - groups=1, - activation_layer=activation_layer) - self._conv_dw = ConvNormActivation(in_channels=out_channels // 2, - out_channels=out_channels // 2, - kernel_size=3, - stride=stride, - padding=1, - groups=out_channels // 2, - activation_layer=None) + self._conv_pw = ConvNormActivation( + in_channels=in_channels // 2, + out_channels=out_channels // 2, + kernel_size=1, + stride=1, + padding=0, + groups=1, + activation_layer=activation_layer, + ) + self._conv_dw = ConvNormActivation( + in_channels=out_channels // 2, + out_channels=out_channels // 2, + kernel_size=3, + stride=stride, + padding=1, + groups=out_channels // 2, + activation_layer=None, + ) self._conv_linear = ConvNormActivation( in_channels=out_channels // 2, out_channels=out_channels // 2, @@ -110,13 +113,15 @@ def __init__(self, stride=1, padding=0, groups=1, - activation_layer=activation_layer) + activation_layer=activation_layer, + ) def forward(self, inputs): x1, x2 = paddle.split( inputs, num_or_sections=[inputs.shape[1] // 2, inputs.shape[1] // 2], - axis=1) + axis=1, + ) x2 = self._conv_pw(x2) x2 = self._conv_dw(x2) x2 = self._conv_linear(x2) @@ -125,22 +130,21 @@ def forward(self, inputs): class InvertedResidualDS(nn.Layer): - - def __init__(self, - in_channels, - out_channels, - stride, - activation_layer=nn.ReLU): + def __init__( + self, in_channels, out_channels, stride, activation_layer=nn.ReLU + ): super(InvertedResidualDS, self).__init__() # branch1 - self._conv_dw_1 = ConvNormActivation(in_channels=in_channels, - out_channels=in_channels, - kernel_size=3, - stride=stride, - padding=1, - groups=in_channels, - activation_layer=None) + self._conv_dw_1 = ConvNormActivation( + in_channels=in_channels, + out_channels=in_channels, + kernel_size=3, + stride=stride, + padding=1, + groups=in_channels, + activation_layer=None, + ) self._conv_linear_1 = ConvNormActivation( in_channels=in_channels, out_channels=out_channels // 2, @@ -148,22 +152,27 @@ def __init__(self, stride=1, padding=0, groups=1, - activation_layer=activation_layer) + activation_layer=activation_layer, + ) # branch2 - self._conv_pw_2 = ConvNormActivation(in_channels=in_channels, - out_channels=out_channels // 2, - kernel_size=1, - stride=1, - padding=0, - groups=1, - activation_layer=activation_layer) - self._conv_dw_2 = ConvNormActivation(in_channels=out_channels // 2, - out_channels=out_channels // 2, - kernel_size=3, - stride=stride, - padding=1, - groups=out_channels // 2, - activation_layer=None) + self._conv_pw_2 = ConvNormActivation( + in_channels=in_channels, + out_channels=out_channels // 2, + kernel_size=1, + stride=1, + padding=0, + groups=1, + activation_layer=activation_layer, + ) + self._conv_dw_2 = ConvNormActivation( + in_channels=out_channels // 2, + out_channels=out_channels // 2, + kernel_size=3, + stride=stride, + padding=1, + groups=out_channels // 2, + activation_layer=None, + ) self._conv_linear_2 = ConvNormActivation( in_channels=out_channels // 2, out_channels=out_channels // 2, @@ -171,7 +180,8 @@ def __init__(self, stride=1, padding=0, groups=1, - activation_layer=activation_layer) + activation_layer=activation_layer, + ) def forward(self, inputs): x1 = self._conv_dw_1(inputs) @@ -232,15 +242,18 @@ def __init__(self, scale=1.0, act="relu", num_classes=1000, with_pool=True): elif scale == 2.0: stage_out_channels = [-1, 24, 224, 488, 976, 2048] else: - raise NotImplementedError("This scale size:[" + str(scale) + - "] is not implemented!") + raise NotImplementedError( + "This scale size:[" + str(scale) + "] is not implemented!" + ) # 1. conv1 - self._conv1 = ConvNormActivation(in_channels=3, - out_channels=stage_out_channels[1], - kernel_size=3, - stride=2, - padding=1, - activation_layer=activation_layer) + self._conv1 = ConvNormActivation( + in_channels=3, + out_channels=stage_out_channels[1], + kernel_size=3, + stride=2, + padding=1, + activation_layer=activation_layer, + ) self._max_pool = MaxPool2D(kernel_size=3, stride=2, padding=1) # 2. bottleneck sequences @@ -248,21 +261,25 @@ def __init__(self, scale=1.0, act="relu", num_classes=1000, with_pool=True): for stage_id, num_repeat in enumerate(stage_repeats): for i in range(num_repeat): if i == 0: - block = self.add_sublayer(sublayer=InvertedResidualDS( - in_channels=stage_out_channels[stage_id + 1], - out_channels=stage_out_channels[stage_id + 2], - stride=2, - activation_layer=activation_layer), - name=str(stage_id + 2) + "_" + - str(i + 1)) + block = self.add_sublayer( + sublayer=InvertedResidualDS( + in_channels=stage_out_channels[stage_id + 1], + out_channels=stage_out_channels[stage_id + 2], + stride=2, + activation_layer=activation_layer, + ), + name=str(stage_id + 2) + "_" + str(i + 1), + ) else: - block = self.add_sublayer(sublayer=InvertedResidual( - in_channels=stage_out_channels[stage_id + 2], - out_channels=stage_out_channels[stage_id + 2], - stride=1, - activation_layer=activation_layer), - name=str(stage_id + 2) + "_" + - str(i + 1)) + block = self.add_sublayer( + sublayer=InvertedResidual( + in_channels=stage_out_channels[stage_id + 2], + out_channels=stage_out_channels[stage_id + 2], + stride=1, + activation_layer=activation_layer, + ), + name=str(stage_id + 2) + "_" + str(i + 1), + ) self._block_list.append(block) # 3. last_conv self._last_conv = ConvNormActivation( @@ -271,7 +288,8 @@ def __init__(self, scale=1.0, act="relu", num_classes=1000, with_pool=True): kernel_size=1, stride=1, padding=0, - activation_layer=activation_layer) + activation_layer=activation_layer, + ) # 4. pool if with_pool: self._pool2d_avg = AdaptiveAvgPool2D(1) @@ -303,9 +321,11 @@ def _shufflenet_v2(arch, pretrained=False, **kwargs): assert ( arch in model_urls ), "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.set_dict(param) @@ -342,10 +362,9 @@ def shufflenet_v2_x0_25(pretrained=False, **kwargs): print(out.shape) # [1, 1000] """ - return _shufflenet_v2("shufflenet_v2_x0_25", - scale=0.25, - pretrained=pretrained, - **kwargs) + return _shufflenet_v2( + "shufflenet_v2_x0_25", scale=0.25, pretrained=pretrained, **kwargs + ) def shufflenet_v2_x0_33(pretrained=False, **kwargs): @@ -378,10 +397,9 @@ def shufflenet_v2_x0_33(pretrained=False, **kwargs): print(out.shape) # [1, 1000] """ - return _shufflenet_v2("shufflenet_v2_x0_33", - scale=0.33, - pretrained=pretrained, - **kwargs) + return _shufflenet_v2( + "shufflenet_v2_x0_33", scale=0.33, pretrained=pretrained, **kwargs + ) def shufflenet_v2_x0_5(pretrained=False, **kwargs): @@ -414,10 +432,9 @@ def shufflenet_v2_x0_5(pretrained=False, **kwargs): print(out.shape) # [1, 1000] """ - return _shufflenet_v2("shufflenet_v2_x0_5", - scale=0.5, - pretrained=pretrained, - **kwargs) + return _shufflenet_v2( + "shufflenet_v2_x0_5", scale=0.5, pretrained=pretrained, **kwargs + ) def shufflenet_v2_x1_0(pretrained=False, **kwargs): @@ -450,10 +467,9 @@ def shufflenet_v2_x1_0(pretrained=False, **kwargs): print(out.shape) # [1, 1000] """ - return _shufflenet_v2("shufflenet_v2_x1_0", - scale=1.0, - pretrained=pretrained, - **kwargs) + return _shufflenet_v2( + "shufflenet_v2_x1_0", scale=1.0, pretrained=pretrained, **kwargs + ) def shufflenet_v2_x1_5(pretrained=False, **kwargs): @@ -486,10 +502,9 @@ def shufflenet_v2_x1_5(pretrained=False, **kwargs): print(out.shape) # [1, 1000] """ - return _shufflenet_v2("shufflenet_v2_x1_5", - scale=1.5, - pretrained=pretrained, - **kwargs) + return _shufflenet_v2( + "shufflenet_v2_x1_5", scale=1.5, pretrained=pretrained, **kwargs + ) def shufflenet_v2_x2_0(pretrained=False, **kwargs): @@ -522,10 +537,9 @@ def shufflenet_v2_x2_0(pretrained=False, **kwargs): print(out.shape) # [1, 1000] """ - return _shufflenet_v2("shufflenet_v2_x2_0", - scale=2.0, - pretrained=pretrained, - **kwargs) + return _shufflenet_v2( + "shufflenet_v2_x2_0", scale=2.0, pretrained=pretrained, **kwargs + ) def shufflenet_v2_swish(pretrained=False, **kwargs): @@ -558,8 +572,10 @@ def shufflenet_v2_swish(pretrained=False, **kwargs): print(out.shape) # [1, 1000] """ - return _shufflenet_v2("shufflenet_v2_swish", - scale=1.0, - act="swish", - pretrained=pretrained, - **kwargs) + return _shufflenet_v2( + "shufflenet_v2_swish", + scale=1.0, + act="swish", + pretrained=pretrained, + **kwargs + ) diff --git a/python/paddle/vision/models/squeezenet.py b/python/paddle/vision/models/squeezenet.py index e2e7dc4b455b8..efb69b5da3afd 100644 --- a/python/paddle/vision/models/squeezenet.py +++ b/python/paddle/vision/models/squeezenet.py @@ -24,25 +24,28 @@ __all__ = [] model_urls = { - 'squeezenet1_0': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/SqueezeNet1_0_pretrained.pdparams', - '30b95af60a2178f03cf9b66cd77e1db1'), - 'squeezenet1_1': - ('https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/SqueezeNet1_1_pretrained.pdparams', - 'a11250d3a1f91d7131fd095ebbf09eee'), + 'squeezenet1_0': ( + 'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/SqueezeNet1_0_pretrained.pdparams', + '30b95af60a2178f03cf9b66cd77e1db1', + ), + 'squeezenet1_1': ( + 'https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/SqueezeNet1_1_pretrained.pdparams', + 'a11250d3a1f91d7131fd095ebbf09eee', + ), } class MakeFireConv(nn.Layer): - def __init__(self, input_channels, output_channels, filter_size, padding=0): super(MakeFireConv, self).__init__() - self._conv = Conv2D(input_channels, - output_channels, - filter_size, - padding=padding, - weight_attr=ParamAttr(), - bias_attr=ParamAttr()) + self._conv = Conv2D( + input_channels, + output_channels, + filter_size, + padding=padding, + weight_attr=ParamAttr(), + bias_attr=ParamAttr(), + ) def forward(self, x): x = self._conv(x) @@ -51,16 +54,19 @@ def forward(self, x): class MakeFire(nn.Layer): - - def __init__(self, input_channels, squeeze_channels, expand1x1_channels, - expand3x3_channels): + def __init__( + self, + input_channels, + squeeze_channels, + expand1x1_channels, + expand3x3_channels, + ): super(MakeFire, self).__init__() self._conv = MakeFireConv(input_channels, squeeze_channels, 1) self._conv_path1 = MakeFireConv(squeeze_channels, expand1x1_channels, 1) - self._conv_path2 = MakeFireConv(squeeze_channels, - expand3x3_channels, - 3, - padding=1) + self._conv_path2 = MakeFireConv( + squeeze_channels, expand3x3_channels, 3, padding=1 + ) def forward(self, inputs): x = self._conv(inputs) @@ -109,17 +115,21 @@ def __init__(self, version, num_classes=1000, with_pool=True): self.with_pool = with_pool supported_versions = ['1.0', '1.1'] - assert version in supported_versions, \ - "supported versions are {} but input version is {}".format( - supported_versions, version) + assert ( + version in supported_versions + ), "supported versions are {} but input version is {}".format( + supported_versions, version + ) if self.version == "1.0": - self._conv = Conv2D(3, - 96, - 7, - stride=2, - weight_attr=ParamAttr(), - bias_attr=ParamAttr()) + self._conv = Conv2D( + 3, + 96, + 7, + stride=2, + weight_attr=ParamAttr(), + bias_attr=ParamAttr(), + ) self._pool = MaxPool2D(kernel_size=3, stride=2, padding=0) self._conv1 = MakeFire(96, 16, 64, 64) self._conv2 = MakeFire(128, 16, 64, 64) @@ -130,13 +140,15 @@ def __init__(self, version, num_classes=1000, with_pool=True): self._conv7 = MakeFire(384, 64, 256, 256) self._conv8 = MakeFire(512, 64, 256, 256) else: - self._conv = Conv2D(3, - 64, - 3, - stride=2, - padding=1, - weight_attr=ParamAttr(), - bias_attr=ParamAttr()) + self._conv = Conv2D( + 3, + 64, + 3, + stride=2, + padding=1, + weight_attr=ParamAttr(), + bias_attr=ParamAttr(), + ) self._pool = MaxPool2D(kernel_size=3, stride=2, padding=0) self._conv1 = MakeFire(64, 16, 64, 64) self._conv2 = MakeFire(128, 16, 64, 64) @@ -148,11 +160,9 @@ def __init__(self, version, num_classes=1000, with_pool=True): self._conv8 = MakeFire(512, 64, 256, 256) self._drop = Dropout(p=0.5, mode="downscale_in_infer") - self._conv9 = Conv2D(512, - num_classes, - 1, - weight_attr=ParamAttr(), - bias_attr=ParamAttr()) + self._conv9 = Conv2D( + 512, num_classes, 1, weight_attr=ParamAttr(), bias_attr=ParamAttr() + ) self._avg_pool = AdaptiveAvgPool2D(1) def forward(self, inputs): @@ -195,10 +205,14 @@ def forward(self, inputs): def _squeezenet(arch, version, pretrained, **kwargs): model = SqueezeNet(version, **kwargs) if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.set_dict(param) diff --git a/python/paddle/vision/models/vgg.py b/python/paddle/vision/models/vgg.py index c9558c9c382ee..1f996439274f7 100644 --- a/python/paddle/vision/models/vgg.py +++ b/python/paddle/vision/models/vgg.py @@ -20,10 +20,14 @@ __all__ = [] model_urls = { - 'vgg16': ('https://paddle-hapi.bj.bcebos.com/models/vgg16.pdparams', - '89bbffc0f87d260be9b8cdc169c991c4'), - 'vgg19': ('https://paddle-hapi.bj.bcebos.com/models/vgg19.pdparams', - '23b18bb13d8894f60f54e642be79a0dd') + 'vgg16': ( + 'https://paddle-hapi.bj.bcebos.com/models/vgg16.pdparams', + '89bbffc0f87d260be9b8cdc169c991c4', + ), + 'vgg19': ( + 'https://paddle-hapi.bj.bcebos.com/models/vgg19.pdparams', + '23b18bb13d8894f60f54e642be79a0dd', + ), } @@ -111,15 +115,65 @@ def make_layers(cfg, batch_norm=False): cfgs = { 'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], - 'B': - [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], + 'B': [ + 64, + 64, + 'M', + 128, + 128, + 'M', + 256, + 256, + 'M', + 512, + 512, + 'M', + 512, + 512, + 'M', + ], 'D': [ - 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, - 512, 512, 'M' + 64, + 64, + 'M', + 128, + 128, + 'M', + 256, + 256, + 256, + 'M', + 512, + 512, + 512, + 'M', + 512, + 512, + 512, + 'M', ], 'E': [ - 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, - 'M', 512, 512, 512, 512, 'M' + 64, + 64, + 'M', + 128, + 128, + 'M', + 256, + 256, + 256, + 256, + 'M', + 512, + 512, + 512, + 512, + 'M', + 512, + 512, + 512, + 512, + 'M', ], } @@ -128,10 +182,14 @@ def _vgg(arch, cfg, batch_norm, pretrained, **kwargs): model = VGG(make_layers(cfgs[cfg], batch_norm=batch_norm), **kwargs) if pretrained: - assert arch in model_urls, "{} model do not have a pretrained model now, you should set pretrained=False".format( - arch) - weight_path = get_weights_path_from_url(model_urls[arch][0], - model_urls[arch][1]) + assert ( + arch in model_urls + ), "{} model do not have a pretrained model now, you should set pretrained=False".format( + arch + ) + weight_path = get_weights_path_from_url( + model_urls[arch][0], model_urls[arch][1] + ) param = paddle.load(weight_path) model.load_dict(param) @@ -172,7 +230,7 @@ def vgg11(pretrained=False, batch_norm=False, **kwargs): """ model_name = 'vgg11' if batch_norm: - model_name += ('_bn') + model_name += '_bn' return _vgg(model_name, 'A', batch_norm, pretrained, **kwargs) @@ -209,7 +267,7 @@ def vgg13(pretrained=False, batch_norm=False, **kwargs): """ model_name = 'vgg13' if batch_norm: - model_name += ('_bn') + model_name += '_bn' return _vgg(model_name, 'B', batch_norm, pretrained, **kwargs) @@ -246,7 +304,7 @@ def vgg16(pretrained=False, batch_norm=False, **kwargs): """ model_name = 'vgg16' if batch_norm: - model_name += ('_bn') + model_name += '_bn' return _vgg(model_name, 'D', batch_norm, pretrained, **kwargs) @@ -283,5 +341,5 @@ def vgg19(pretrained=False, batch_norm=False, **kwargs): """ model_name = 'vgg19' if batch_norm: - model_name += ('_bn') + model_name += '_bn' return _vgg(model_name, 'E', batch_norm, pretrained, **kwargs) diff --git a/python/paddle/vision/ops.py b/python/paddle/vision/ops.py index b66db2aa737e3..e2ed14fa78df6 100755 --- a/python/paddle/vision/ops.py +++ b/python/paddle/vision/ops.py @@ -18,30 +18,48 @@ from ..fluid.layers import nn, utils from ..nn import Layer, Conv2D, Sequential, ReLU, BatchNorm2D from ..fluid.initializer import Normal -from ..fluid.framework import _non_static_mode, in_dygraph_mode, _in_legacy_dygraph +from ..fluid.framework import ( + _non_static_mode, + in_dygraph_mode, + _in_legacy_dygraph, +) from paddle import _C_ops, _legacy_C_ops from ..framework import _current_expected_place -__all__ = [ #noqa - 'yolo_loss', 'yolo_box', 'deform_conv2d', 'DeformConv2D', - 'distribute_fpn_proposals', 'generate_proposals', 'read_file', - 'decode_jpeg', 'roi_pool', 'RoIPool', 'psroi_pool', 'PSRoIPool', - 'roi_align', 'RoIAlign', 'nms', 'matrix_nms' +__all__ = [ # noqa + 'yolo_loss', + 'yolo_box', + 'deform_conv2d', + 'DeformConv2D', + 'distribute_fpn_proposals', + 'generate_proposals', + 'read_file', + 'decode_jpeg', + 'roi_pool', + 'RoIPool', + 'psroi_pool', + 'PSRoIPool', + 'roi_align', + 'RoIAlign', + 'nms', + 'matrix_nms', ] -def yolo_loss(x, - gt_box, - gt_label, - anchors, - anchor_mask, - class_num, - ignore_thresh, - downsample_ratio, - gt_score=None, - use_label_smooth=True, - name=None, - scale_x_y=1.): +def yolo_loss( + x, + gt_box, + gt_label, + anchors, + anchor_mask, + class_num, + ignore_thresh, + downsample_ratio, + gt_score=None, + use_label_smooth=True, + name=None, + scale_x_y=1.0, +): r""" This operator generates YOLOv3 loss based on given predict result and ground @@ -176,25 +194,50 @@ def yolo_loss(x, """ if in_dygraph_mode(): - loss, _, _ = _C_ops.yolov3_loss(x, gt_box, gt_label, gt_score, anchors, - anchor_mask, class_num, ignore_thresh, - downsample_ratio, use_label_smooth, - scale_x_y) + loss, _, _ = _C_ops.yolov3_loss( + x, + gt_box, + gt_label, + gt_score, + anchors, + anchor_mask, + class_num, + ignore_thresh, + downsample_ratio, + use_label_smooth, + scale_x_y, + ) return loss if _non_static_mode(): loss, _, _ = _legacy_C_ops.yolov3_loss( - x, gt_box, gt_label, gt_score, 'anchors', anchors, 'anchor_mask', - anchor_mask, 'class_num', class_num, 'ignore_thresh', ignore_thresh, - 'downsample_ratio', downsample_ratio, 'use_label_smooth', - use_label_smooth, 'scale_x_y', scale_x_y) + x, + gt_box, + gt_label, + gt_score, + 'anchors', + anchors, + 'anchor_mask', + anchor_mask, + 'class_num', + class_num, + 'ignore_thresh', + ignore_thresh, + 'downsample_ratio', + downsample_ratio, + 'use_label_smooth', + use_label_smooth, + 'scale_x_y', + scale_x_y, + ) return loss helper = LayerHelper('yolov3_loss', **locals()) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'yolo_loss') - check_variable_and_dtype(gt_box, 'gt_box', ['float32', 'float64'], - 'yolo_loss') + check_variable_and_dtype( + gt_box, 'gt_box', ['float32', 'float64'], 'yolo_loss' + ) check_variable_and_dtype(gt_label, 'gt_label', 'int32', 'yolo_loss') check_type(anchors, 'anchors', (list, tuple), 'yolo_loss') check_type(anchor_mask, 'anchor_mask', (list, tuple), 'yolo_loss') @@ -225,28 +268,32 @@ def yolo_loss(x, "scale_x_y": scale_x_y, } - helper.append_op(type='yolov3_loss', - inputs=inputs, - outputs={ - 'Loss': loss, - 'ObjectnessMask': objectness_mask, - 'GTMatchMask': gt_match_mask - }, - attrs=attrs) + helper.append_op( + type='yolov3_loss', + inputs=inputs, + outputs={ + 'Loss': loss, + 'ObjectnessMask': objectness_mask, + 'GTMatchMask': gt_match_mask, + }, + attrs=attrs, + ) return loss -def yolo_box(x, - img_size, - anchors, - class_num, - conf_thresh, - downsample_ratio, - clip_bbox=True, - name=None, - scale_x_y=1., - iou_aware=False, - iou_aware_factor=0.5): +def yolo_box( + x, + img_size, + anchors, + class_num, + conf_thresh, + downsample_ratio, + clip_bbox=True, + name=None, + scale_x_y=1.0, + iou_aware=False, + iou_aware_factor=0.5, +): r""" This operator generates YOLO detection boxes from output of YOLOv3 network. @@ -359,18 +406,41 @@ def yolo_box(x, scale_x_y=1.) """ if in_dygraph_mode(): - boxes, scores = _C_ops.yolo_box(x, img_size, anchors, class_num, - conf_thresh, downsample_ratio, - clip_bbox, scale_x_y, iou_aware, - iou_aware_factor) + boxes, scores = _C_ops.yolo_box( + x, + img_size, + anchors, + class_num, + conf_thresh, + downsample_ratio, + clip_bbox, + scale_x_y, + iou_aware, + iou_aware_factor, + ) return boxes, scores if _non_static_mode(): boxes, scores = _legacy_C_ops.yolo_box( - x, img_size, 'anchors', anchors, 'class_num', class_num, - 'conf_thresh', conf_thresh, 'downsample_ratio', downsample_ratio, - 'clip_bbox', clip_bbox, 'scale_x_y', scale_x_y, 'iou_aware', - iou_aware, 'iou_aware_factor', iou_aware_factor) + x, + img_size, + 'anchors', + anchors, + 'class_num', + class_num, + 'conf_thresh', + conf_thresh, + 'downsample_ratio', + downsample_ratio, + 'clip_bbox', + clip_bbox, + 'scale_x_y', + scale_x_y, + 'iou_aware', + iou_aware, + 'iou_aware_factor', + iou_aware_factor, + ) return boxes, scores helper = LayerHelper('yolo_box', **locals()) @@ -391,33 +461,37 @@ def yolo_box(x, "clip_bbox": clip_bbox, "scale_x_y": scale_x_y, "iou_aware": iou_aware, - "iou_aware_factor": iou_aware_factor + "iou_aware_factor": iou_aware_factor, } - helper.append_op(type='yolo_box', - inputs={ - "X": x, - "ImgSize": img_size, - }, - outputs={ - 'Boxes': boxes, - 'Scores': scores, - }, - attrs=attrs) + helper.append_op( + type='yolo_box', + inputs={ + "X": x, + "ImgSize": img_size, + }, + outputs={ + 'Boxes': boxes, + 'Scores': scores, + }, + attrs=attrs, + ) return boxes, scores -def deform_conv2d(x, - offset, - weight, - bias=None, - stride=1, - padding=0, - dilation=1, - deformable_groups=1, - groups=1, - mask=None, - name=None): +def deform_conv2d( + x, + offset, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + deformable_groups=1, + groups=1, + mask=None, + name=None, +): r""" Compute 2-D deformable convolution on 4-D input. Given input image x, output feature map y, the deformable convolution operation can be expressed as follow: @@ -537,34 +611,58 @@ def deform_conv2d(x, use_deform_conv2d_v1 = True if mask is None else False if in_dygraph_mode(): - pre_bias = _C_ops.deformable_conv(x, offset, weight, mask, stride, - padding, dilation, deformable_groups, - groups, 1) + pre_bias = _C_ops.deformable_conv( + x, + offset, + weight, + mask, + stride, + padding, + dilation, + deformable_groups, + groups, + 1, + ) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=1) else: out = pre_bias elif _in_legacy_dygraph(): - attrs = ('strides', stride, 'paddings', padding, 'dilations', dilation, - 'deformable_groups', deformable_groups, 'groups', groups, - 'im2col_step', 1) + attrs = ( + 'strides', + stride, + 'paddings', + padding, + 'dilations', + dilation, + 'deformable_groups', + deformable_groups, + 'groups', + groups, + 'im2col_step', + 1, + ) if use_deform_conv2d_v1: op_type = 'deformable_conv_v1' - pre_bias = getattr(_legacy_C_ops, op_type)(x, offset, weight, - *attrs) + pre_bias = getattr(_legacy_C_ops, op_type)( + x, offset, weight, *attrs + ) else: op_type = 'deformable_conv' - pre_bias = getattr(_legacy_C_ops, op_type)(x, offset, mask, weight, - *attrs) + pre_bias = getattr(_legacy_C_ops, op_type)( + x, offset, mask, weight, *attrs + ) if bias is not None: out = nn.elementwise_add(pre_bias, bias, axis=1) else: out = pre_bias else: - check_variable_and_dtype(x, "x", ['float32', 'float64'], - 'deform_conv2d') - check_variable_and_dtype(offset, "offset", ['float32', 'float64'], - 'deform_conv2d') + check_variable_and_dtype( + x, "x", ['float32', 'float64'], 'deform_conv2d' + ) + check_variable_and_dtype( + offset, "offset", ['float32', 'float64'], 'deform_conv2d' + ) num_channels = x.shape[1] @@ -602,20 +700,18 @@ def deform_conv2d(x, 'deformable_groups': deformable_groups, 'im2col_step': 1, } - helper.append_op(type=op_type, - inputs=inputs, - outputs=outputs, - attrs=attrs) + helper.append_op( + type=op_type, inputs=inputs, outputs=outputs, attrs=attrs + ) if bias is not None: out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='elementwise_add', - inputs={ - 'X': [pre_bias], - 'Y': [bias] - }, - outputs={'Out': [out]}, - attrs={'axis': 1}) + helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], 'Y': [bias]}, + outputs={'Out': [out]}, + attrs={'axis': 1}, + ) else: out = pre_bias return out @@ -755,19 +851,23 @@ class DeformConv2D(Layer): [8, 16, 26, 26] """ - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - padding=0, - dilation=1, - deformable_groups=1, - groups=1, - weight_attr=None, - bias_attr=None): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + deformable_groups=1, + groups=1, + weight_attr=None, + bias_attr=None, + ): super(DeformConv2D, self).__init__() - assert weight_attr is not False, "weight_attr should not be False in Conv." + assert ( + weight_attr is not False + ), "weight_attr should not be False in Conv." self._weight_attr = weight_attr self._bias_attr = bias_attr self._deformable_groups = deformable_groups @@ -789,39 +889,44 @@ def __init__(self, def _get_default_param_initializer(): filter_elem_num = np.prod(self._kernel_size) * self._in_channels - std = (2.0 / filter_elem_num)**0.5 + std = (2.0 / filter_elem_num) ** 0.5 return Normal(0.0, std, 0) self.weight = self.create_parameter( shape=filter_shape, attr=self._weight_attr, - default_initializer=_get_default_param_initializer()) - self.bias = self.create_parameter(attr=self._bias_attr, - shape=[self._out_channels], - is_bias=True) + default_initializer=_get_default_param_initializer(), + ) + self.bias = self.create_parameter( + attr=self._bias_attr, shape=[self._out_channels], is_bias=True + ) def forward(self, x, offset, mask=None): - out = deform_conv2d(x=x, - offset=offset, - weight=self.weight, - bias=self.bias, - stride=self._stride, - padding=self._padding, - dilation=self._dilation, - deformable_groups=self._deformable_groups, - groups=self._groups, - mask=mask) + out = deform_conv2d( + x=x, + offset=offset, + weight=self.weight, + bias=self.bias, + stride=self._stride, + padding=self._padding, + dilation=self._dilation, + deformable_groups=self._deformable_groups, + groups=self._groups, + mask=mask, + ) return out -def distribute_fpn_proposals(fpn_rois, - min_level, - max_level, - refer_level, - refer_scale, - pixel_offset=False, - rois_num=None, - name=None): +def distribute_fpn_proposals( + fpn_rois, + min_level, + max_level, + refer_level, + refer_scale, + pixel_offset=False, + rois_num=None, + name=None, +): r""" In Feature Pyramid Networks (FPN) models, it is needed to distribute all proposals into different FPN level, with respect to scale of the proposals, @@ -882,24 +987,56 @@ def distribute_fpn_proposals(fpn_rois, num_lvl = max_level - min_level + 1 if in_dygraph_mode(): - assert rois_num is not None, "rois_num should not be None in dygraph mode." - multi_rois, rois_num_per_level, restore_ind = _C_ops.distribute_fpn_proposals( - fpn_rois, rois_num, min_level, max_level, refer_level, refer_scale, - pixel_offset) + assert ( + rois_num is not None + ), "rois_num should not be None in dygraph mode." + ( + multi_rois, + rois_num_per_level, + restore_ind, + ) = _C_ops.distribute_fpn_proposals( + fpn_rois, + rois_num, + min_level, + max_level, + refer_level, + refer_scale, + pixel_offset, + ) return multi_rois, restore_ind, rois_num_per_level if _non_static_mode(): - assert rois_num is not None, "rois_num should not be None in dygraph mode." - attrs = ('min_level', min_level, 'max_level', max_level, 'refer_level', - refer_level, 'refer_scale', refer_scale, 'pixel_offset', - pixel_offset) - multi_rois, restore_ind, rois_num_per_level = _legacy_C_ops.distribute_fpn_proposals( - fpn_rois, rois_num, num_lvl, num_lvl, *attrs) + assert ( + rois_num is not None + ), "rois_num should not be None in dygraph mode." + attrs = ( + 'min_level', + min_level, + 'max_level', + max_level, + 'refer_level', + refer_level, + 'refer_scale', + refer_scale, + 'pixel_offset', + pixel_offset, + ) + ( + multi_rois, + restore_ind, + rois_num_per_level, + ) = _legacy_C_ops.distribute_fpn_proposals( + fpn_rois, rois_num, num_lvl, num_lvl, *attrs + ) return multi_rois, restore_ind, rois_num_per_level else: - check_variable_and_dtype(fpn_rois, 'fpn_rois', ['float32', 'float64'], - 'distribute_fpn_proposals') + check_variable_and_dtype( + fpn_rois, + 'fpn_rois', + ['float32', 'float64'], + 'distribute_fpn_proposals', + ) helper = LayerHelper('distribute_fpn_proposals', **locals()) dtype = helper.input_dtype('fpn_rois') multi_rois = [ @@ -925,16 +1062,18 @@ def distribute_fpn_proposals(fpn_rois, else: rois_num_per_level = None - helper.append_op(type='distribute_fpn_proposals', - inputs=inputs, - outputs=outputs, - attrs={ - 'min_level': min_level, - 'max_level': max_level, - 'refer_level': refer_level, - 'refer_scale': refer_scale, - 'pixel_offset': pixel_offset - }) + helper.append_op( + type='distribute_fpn_proposals', + inputs=inputs, + outputs=outputs, + attrs={ + 'min_level': min_level, + 'max_level': max_level, + 'refer_level': refer_level, + 'refer_scale': refer_scale, + 'pixel_offset': pixel_offset, + }, + ) return multi_rois, restore_ind, rois_num_per_level @@ -976,10 +1115,9 @@ def read_file(filename, name=None): helper = LayerHelper("read_file", **locals()) out = helper.create_variable_for_type_inference('uint8') - helper.append_op(type="read_file", - inputs=inputs, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="read_file", inputs=inputs, attrs=attrs, outputs={"Out": out} + ) return out @@ -1029,10 +1167,9 @@ def decode_jpeg(x, mode='unchanged', name=None): helper = LayerHelper("decode_jpeg", **locals()) out = helper.create_variable_for_type_inference('uint8') - helper.append_op(type="decode_jpeg", - inputs=inputs, - attrs=attrs, - outputs={"Out": out}) + helper.append_op( + type="decode_jpeg", inputs=inputs, attrs=attrs, outputs={"Out": out} + ) return out @@ -1080,34 +1217,47 @@ def psroi_pool(x, boxes, boxes_num, output_size, spatial_scale=1.0, name=None): if isinstance(output_size, int): output_size = (output_size, output_size) pooled_height, pooled_width = output_size - assert len(x.shape) == 4, \ - "Input features with shape should be (N, C, H, W)" + assert len(x.shape) == 4, "Input features with shape should be (N, C, H, W)" output_channels = int(x.shape[1] / (pooled_height * pooled_width)) if in_dygraph_mode(): - return _C_ops.psroi_pool(x, boxes, boxes_num, pooled_height, - pooled_width, output_channels, spatial_scale) + return _C_ops.psroi_pool( + x, + boxes, + boxes_num, + pooled_height, + pooled_width, + output_channels, + spatial_scale, + ) if _in_legacy_dygraph(): - return _legacy_C_ops.psroi_pool(x, boxes, boxes_num, "output_channels", - output_channels, "spatial_scale", - spatial_scale, "pooled_height", - pooled_height, "pooled_width", - pooled_width) + return _legacy_C_ops.psroi_pool( + x, + boxes, + boxes_num, + "output_channels", + output_channels, + "spatial_scale", + spatial_scale, + "pooled_height", + pooled_height, + "pooled_width", + pooled_width, + ) helper = LayerHelper('psroi_pool', **locals()) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) - helper.append_op(type='psroi_pool', - inputs={ - 'X': x, - 'ROIs': boxes - }, - outputs={'Out': out}, - attrs={ - 'output_channels': output_channels, - 'spatial_scale': spatial_scale, - 'pooled_height': pooled_height, - 'pooled_width': pooled_width - }) + helper.append_op( + type='psroi_pool', + inputs={'X': x, 'ROIs': boxes}, + outputs={'Out': out}, + attrs={ + 'output_channels': output_channels, + 'spatial_scale': spatial_scale, + 'pooled_height': pooled_height, + 'pooled_width': pooled_width, + }, + ) return out @@ -1151,8 +1301,9 @@ def __init__(self, output_size, spatial_scale=1.0): self.spatial_scale = spatial_scale def forward(self, x, boxes, boxes_num): - return psroi_pool(x, boxes, boxes_num, self.output_size, - self.spatial_scale) + return psroi_pool( + x, boxes, boxes_num, self.output_size, self.spatial_scale + ) def roi_pool(x, boxes, boxes_num, output_size, spatial_scale=1.0, name=None): @@ -1199,14 +1350,27 @@ def roi_pool(x, boxes, boxes_num, output_size, spatial_scale=1.0, name=None): pooled_height, pooled_width = output_size if in_dygraph_mode(): - assert boxes_num is not None, "boxes_num should not be None in dygraph mode." - return _C_ops.roi_pool(x, boxes, boxes_num, pooled_height, pooled_width, - spatial_scale) + assert ( + boxes_num is not None + ), "boxes_num should not be None in dygraph mode." + return _C_ops.roi_pool( + x, boxes, boxes_num, pooled_height, pooled_width, spatial_scale + ) if _in_legacy_dygraph(): - assert boxes_num is not None, "boxes_num should not be None in dygraph mode." + assert ( + boxes_num is not None + ), "boxes_num should not be None in dygraph mode." pool_out, argmaxes = _legacy_C_ops.roi_pool( - x, boxes, boxes_num, "pooled_height", pooled_height, "pooled_width", - pooled_width, "spatial_scale", spatial_scale) + x, + boxes, + boxes_num, + "pooled_height", + pooled_height, + "pooled_width", + pooled_width, + "spatial_scale", + spatial_scale, + ) return pool_out else: @@ -1223,17 +1387,16 @@ def roi_pool(x, boxes, boxes_num, output_size, spatial_scale=1.0, name=None): } if boxes_num is not None: inputs['RoisNum'] = boxes_num - helper.append_op(type="roi_pool", - inputs=inputs, - outputs={ - "Out": pool_out, - "Argmax": argmaxes - }, - attrs={ - "pooled_height": pooled_height, - "pooled_width": pooled_width, - "spatial_scale": spatial_scale - }) + helper.append_op( + type="roi_pool", + inputs=inputs, + outputs={"Out": pool_out, "Argmax": argmaxes}, + attrs={ + "pooled_height": pooled_height, + "pooled_width": pooled_width, + "spatial_scale": spatial_scale, + }, + ) return pool_out @@ -1271,25 +1434,29 @@ def __init__(self, output_size, spatial_scale=1.0): self._spatial_scale = spatial_scale def forward(self, x, boxes, boxes_num): - return roi_pool(x=x, - boxes=boxes, - boxes_num=boxes_num, - output_size=self._output_size, - spatial_scale=self._spatial_scale) + return roi_pool( + x=x, + boxes=boxes, + boxes_num=boxes_num, + output_size=self._output_size, + spatial_scale=self._spatial_scale, + ) def extra_repr(self): main_str = 'output_size={_output_size}, spatial_scale={_spatial_scale}' return main_str.format(**self.__dict__) -def roi_align(x, - boxes, - boxes_num, - output_size, - spatial_scale=1.0, - sampling_ratio=-1, - aligned=True, - name=None): +def roi_align( + x, + boxes, + boxes_num, + output_size, + spatial_scale=1.0, + sampling_ratio=-1, + aligned=True, + name=None, +): """ Implementing the roi_align layer. Region of Interest (RoI) Align operator (also known as RoI Align) is to @@ -1358,24 +1525,45 @@ def roi_align(x, pooled_height, pooled_width = output_size if in_dygraph_mode(): - assert boxes_num is not None, "boxes_num should not be None in dygraph mode." - return _C_ops.roi_align(x, boxes, boxes_num, pooled_height, - pooled_width, spatial_scale, sampling_ratio, - aligned) + assert ( + boxes_num is not None + ), "boxes_num should not be None in dygraph mode." + return _C_ops.roi_align( + x, + boxes, + boxes_num, + pooled_height, + pooled_width, + spatial_scale, + sampling_ratio, + aligned, + ) if _in_legacy_dygraph(): - assert boxes_num is not None, "boxes_num should not be None in dygraph mode." - align_out = _legacy_C_ops.roi_align(x, boxes, boxes_num, - "pooled_height", pooled_height, - "pooled_width", pooled_width, - "spatial_scale", spatial_scale, - "sampling_ratio", sampling_ratio, - "aligned", aligned) + assert ( + boxes_num is not None + ), "boxes_num should not be None in dygraph mode." + align_out = _legacy_C_ops.roi_align( + x, + boxes, + boxes_num, + "pooled_height", + pooled_height, + "pooled_width", + pooled_width, + "spatial_scale", + spatial_scale, + "sampling_ratio", + sampling_ratio, + "aligned", + aligned, + ) return align_out else: check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'roi_align') - check_variable_and_dtype(boxes, 'boxes', ['float32', 'float64'], - 'roi_align') + check_variable_and_dtype( + boxes, 'boxes', ['float32', 'float64'], 'roi_align' + ) helper = LayerHelper('roi_align', **locals()) dtype = helper.input_dtype() align_out = helper.create_variable_for_type_inference(dtype) @@ -1385,16 +1573,18 @@ def roi_align(x, } if boxes_num is not None: inputs['RoisNum'] = boxes_num - helper.append_op(type="roi_align", - inputs=inputs, - outputs={"Out": align_out}, - attrs={ - "pooled_height": pooled_height, - "pooled_width": pooled_width, - "spatial_scale": spatial_scale, - "sampling_ratio": sampling_ratio, - "aligned": aligned, - }) + helper.append_op( + type="roi_align", + inputs=inputs, + outputs={"Out": align_out}, + attrs={ + "pooled_height": pooled_height, + "pooled_width": pooled_width, + "spatial_scale": spatial_scale, + "sampling_ratio": sampling_ratio, + "aligned": aligned, + }, + ) return align_out @@ -1436,12 +1626,14 @@ def __init__(self, output_size, spatial_scale=1.0): self._spatial_scale = spatial_scale def forward(self, x, boxes, boxes_num, aligned=True): - return roi_align(x=x, - boxes=boxes, - boxes_num=boxes_num, - output_size=self._output_size, - spatial_scale=self._spatial_scale, - aligned=aligned) + return roi_align( + x=x, + boxes=boxes, + boxes_num=boxes_num, + output_size=self._output_size, + spatial_scale=self._spatial_scale, + aligned=aligned, + ) class ConvNormActivation(Sequential): @@ -1465,30 +1657,34 @@ class ConvNormActivation(Sequential): bias (bool, optional): Whether to use bias in the convolution layer. By default, biases are included if ``norm_layer is None``. """ - def __init__(self, - in_channels, - out_channels, - kernel_size=3, - stride=1, - padding=None, - groups=1, - norm_layer=BatchNorm2D, - activation_layer=ReLU, - dilation=1, - bias=None): + def __init__( + self, + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=None, + groups=1, + norm_layer=BatchNorm2D, + activation_layer=ReLU, + dilation=1, + bias=None, + ): if padding is None: padding = (kernel_size - 1) // 2 * dilation if bias is None: bias = norm_layer is None layers = [ - Conv2D(in_channels, - out_channels, - kernel_size, - stride, - padding, - dilation=dilation, - groups=groups, - bias_attr=bias) + Conv2D( + in_channels, + out_channels, + kernel_size, + stride, + padding, + dilation=dilation, + groups=groups, + bias_attr=bias, + ) ] if norm_layer is not None: layers.append(norm_layer(out_channels)) @@ -1497,12 +1693,14 @@ def __init__(self, super().__init__(*layers) -def nms(boxes, - iou_threshold=0.3, - scores=None, - category_idxs=None, - categories=None, - top_k=None): +def nms( + boxes, + iou_threshold=0.3, + scores=None, + category_idxs=None, + categories=None, + top_k=None, +): r""" This operator implements non-maximum suppression. Non-maximum suppression (NMS) is used to select one bounding box out of many overlapping bounding boxes in object detection. @@ -1582,34 +1780,42 @@ def _nms(boxes, iou_threshold): helper = LayerHelper('nms', **locals()) out = helper.create_variable_for_type_inference('int64') - helper.append_op(type='nms', - inputs={'Boxes': boxes}, - outputs={'KeepBoxesIdxs': out}, - attrs={'iou_threshold': iou_threshold}) + helper.append_op( + type='nms', + inputs={'Boxes': boxes}, + outputs={'KeepBoxesIdxs': out}, + attrs={'iou_threshold': iou_threshold}, + ) return out if scores is None: return _nms(boxes, iou_threshold) import paddle + if category_idxs is None: sorted_global_indices = paddle.argsort(scores, descending=True) - sorted_keep_boxes_indices = _nms(boxes[sorted_global_indices], - iou_threshold) + sorted_keep_boxes_indices = _nms( + boxes[sorted_global_indices], iou_threshold + ) return sorted_global_indices[sorted_keep_boxes_indices] if top_k is not None: - assert top_k <= scores.shape[ - 0], "top_k should be smaller equal than the number of boxes" - assert categories is not None, "if category_idxs is given, categories which is a list of unique id of all categories is necessary" + assert ( + top_k <= scores.shape[0] + ), "top_k should be smaller equal than the number of boxes" + assert ( + categories is not None + ), "if category_idxs is given, categories which is a list of unique id of all categories is necessary" mask = paddle.zeros_like(scores, dtype=paddle.int32) for category_id in categories: cur_category_boxes_idxs = paddle.where(category_idxs == category_id)[0] shape = cur_category_boxes_idxs.shape[0] - cur_category_boxes_idxs = paddle.reshape(cur_category_boxes_idxs, - [shape]) + cur_category_boxes_idxs = paddle.reshape( + cur_category_boxes_idxs, [shape] + ) if shape == 0: continue elif shape == 1: @@ -1617,27 +1823,33 @@ def _nms(boxes, iou_threshold): continue cur_category_boxes = boxes[cur_category_boxes_idxs] cur_category_scores = scores[cur_category_boxes_idxs] - cur_category_sorted_indices = paddle.argsort(cur_category_scores, - descending=True) + cur_category_sorted_indices = paddle.argsort( + cur_category_scores, descending=True + ) cur_category_sorted_boxes = cur_category_boxes[ - cur_category_sorted_indices] + cur_category_sorted_indices + ] - cur_category_keep_boxes_sub_idxs = cur_category_sorted_indices[_nms( - cur_category_sorted_boxes, iou_threshold)] + cur_category_keep_boxes_sub_idxs = cur_category_sorted_indices[ + _nms(cur_category_sorted_boxes, iou_threshold) + ] updates = paddle.ones_like( cur_category_boxes_idxs[cur_category_keep_boxes_sub_idxs], - dtype=paddle.int32) + dtype=paddle.int32, + ) mask = paddle.scatter( mask, cur_category_boxes_idxs[cur_category_keep_boxes_sub_idxs], updates, - overwrite=True) + overwrite=True, + ) keep_boxes_idxs = paddle.where(mask)[0] shape = keep_boxes_idxs.shape[0] keep_boxes_idxs = paddle.reshape(keep_boxes_idxs, [shape]) - sorted_sub_indices = paddle.argsort(scores[keep_boxes_idxs], - descending=True) + sorted_sub_indices = paddle.argsort( + scores[keep_boxes_idxs], descending=True + ) if top_k is None: return keep_boxes_idxs[sorted_sub_indices] @@ -1650,19 +1862,21 @@ def _nms(boxes, iou_threshold): return keep_boxes_idxs[sorted_sub_indices][:top_k] -def generate_proposals(scores, - bbox_deltas, - img_size, - anchors, - variances, - pre_nms_top_n=6000, - post_nms_top_n=1000, - nms_thresh=0.5, - min_size=0.1, - eta=1.0, - pixel_offset=False, - return_rois_num=False, - name=None): +def generate_proposals( + scores, + bbox_deltas, + img_size, + anchors, + variances, + pre_nms_top_n=6000, + post_nms_top_n=1000, + nms_thresh=0.5, + min_size=0.1, + eta=1.0, + pixel_offset=False, + return_rois_num=False, + name=None, +): """ This operation proposes RoIs according to each box with their probability to be a foreground object. And @@ -1733,40 +1947,74 @@ def generate_proposals(scores, """ if in_dygraph_mode(): - assert return_rois_num, "return_rois_num should be True in dygraph mode." - attrs = (pre_nms_top_n, post_nms_top_n, nms_thresh, min_size, eta, - pixel_offset) + assert ( + return_rois_num + ), "return_rois_num should be True in dygraph mode." + attrs = ( + pre_nms_top_n, + post_nms_top_n, + nms_thresh, + min_size, + eta, + pixel_offset, + ) rpn_rois, rpn_roi_probs, rpn_rois_num = _C_ops.generate_proposals( - scores, bbox_deltas, img_size, anchors, variances, *attrs) + scores, bbox_deltas, img_size, anchors, variances, *attrs + ) return rpn_rois, rpn_roi_probs, rpn_rois_num elif _non_static_mode(): - assert return_rois_num, "return_rois_num should be True in dygraph mode." - attrs = ('pre_nms_topN', pre_nms_top_n, 'post_nms_topN', post_nms_top_n, - 'nms_thresh', nms_thresh, 'min_size', min_size, 'eta', eta, - 'pixel_offset', pixel_offset) - rpn_rois, rpn_roi_probs, rpn_rois_num = _legacy_C_ops.generate_proposals_v2( - scores, bbox_deltas, img_size, anchors, variances, *attrs) + assert ( + return_rois_num + ), "return_rois_num should be True in dygraph mode." + attrs = ( + 'pre_nms_topN', + pre_nms_top_n, + 'post_nms_topN', + post_nms_top_n, + 'nms_thresh', + nms_thresh, + 'min_size', + min_size, + 'eta', + eta, + 'pixel_offset', + pixel_offset, + ) + ( + rpn_rois, + rpn_roi_probs, + rpn_rois_num, + ) = _legacy_C_ops.generate_proposals_v2( + scores, bbox_deltas, img_size, anchors, variances, *attrs + ) return rpn_rois, rpn_roi_probs, rpn_rois_num helper = LayerHelper('generate_proposals_v2', **locals()) - check_variable_and_dtype(scores, 'scores', ['float32'], - 'generate_proposals_v2') - check_variable_and_dtype(bbox_deltas, 'bbox_deltas', ['float32'], - 'generate_proposals_v2') - check_variable_and_dtype(img_size, 'img_size', ['float32', 'float64'], - 'generate_proposals_v2') - check_variable_and_dtype(anchors, 'anchors', ['float32'], - 'generate_proposals_v2') - check_variable_and_dtype(variances, 'variances', ['float32'], - 'generate_proposals_v2') + check_variable_and_dtype( + scores, 'scores', ['float32'], 'generate_proposals_v2' + ) + check_variable_and_dtype( + bbox_deltas, 'bbox_deltas', ['float32'], 'generate_proposals_v2' + ) + check_variable_and_dtype( + img_size, 'img_size', ['float32', 'float64'], 'generate_proposals_v2' + ) + check_variable_and_dtype( + anchors, 'anchors', ['float32'], 'generate_proposals_v2' + ) + check_variable_and_dtype( + variances, 'variances', ['float32'], 'generate_proposals_v2' + ) rpn_rois = helper.create_variable_for_type_inference( - dtype=bbox_deltas.dtype) + dtype=bbox_deltas.dtype + ) rpn_roi_probs = helper.create_variable_for_type_inference( - dtype=scores.dtype) + dtype=scores.dtype + ) outputs = { 'RpnRois': rpn_rois, 'RpnRoiProbs': rpn_roi_probs, @@ -1776,23 +2024,25 @@ def generate_proposals(scores, rpn_rois_num.stop_gradient = True outputs['RpnRoisNum'] = rpn_rois_num - helper.append_op(type="generate_proposals_v2", - inputs={ - 'Scores': scores, - 'BboxDeltas': bbox_deltas, - 'ImShape': img_size, - 'Anchors': anchors, - 'Variances': variances - }, - attrs={ - 'pre_nms_topN': pre_nms_top_n, - 'post_nms_topN': post_nms_top_n, - 'nms_thresh': nms_thresh, - 'min_size': min_size, - 'eta': eta, - 'pixel_offset': pixel_offset - }, - outputs=outputs) + helper.append_op( + type="generate_proposals_v2", + inputs={ + 'Scores': scores, + 'BboxDeltas': bbox_deltas, + 'ImShape': img_size, + 'Anchors': anchors, + 'Variances': variances, + }, + attrs={ + 'pre_nms_topN': pre_nms_top_n, + 'post_nms_topN': post_nms_top_n, + 'nms_thresh': nms_thresh, + 'min_size': min_size, + 'eta': eta, + 'pixel_offset': pixel_offset, + }, + outputs=outputs, + ) rpn_rois.stop_gradient = True rpn_roi_probs.stop_gradient = True if not return_rois_num: @@ -1801,19 +2051,21 @@ def generate_proposals(scores, return rpn_rois, rpn_roi_probs, rpn_rois_num -def matrix_nms(bboxes, - scores, - score_threshold, - post_threshold, - nms_top_k, - keep_top_k, - use_gaussian=False, - gaussian_sigma=2., - background_label=0, - normalized=True, - return_index=False, - return_rois_num=True, - name=None): +def matrix_nms( + bboxes, + scores, + score_threshold, + post_threshold, + nms_top_k, + keep_top_k, + use_gaussian=False, + gaussian_sigma=2.0, + background_label=0, + normalized=True, + return_index=False, + return_rois_num=True, + name=None, +): """ This operator does matrix non maximum suppression (NMS). First selects a subset of candidate bounding boxes that have higher scores @@ -1876,10 +2128,12 @@ def matrix_nms(bboxes, score_threshold=0.5, post_threshold=0.1, nms_top_k=400, keep_top_k=200, normalized=False) """ - check_variable_and_dtype(bboxes, 'BBoxes', ['float32', 'float64'], - 'matrix_nms') - check_variable_and_dtype(scores, 'Scores', ['float32', 'float64'], - 'matrix_nms') + check_variable_and_dtype( + bboxes, 'BBoxes', ['float32', 'float64'], 'matrix_nms' + ) + check_variable_and_dtype( + scores, 'Scores', ['float32', 'float64'], 'matrix_nms' + ) check_type(score_threshold, 'score_threshold', float, 'matrix_nms') check_type(post_threshold, 'post_threshold', float, 'matrix_nms') check_type(nms_top_k, 'nums_top_k', int, 'matrix_nms') @@ -1890,22 +2144,42 @@ def matrix_nms(bboxes, check_type(background_label, 'background_label', int, 'matrix_nms') if in_dygraph_mode(): - out, index, rois_num = _C_ops.matrix_nms(bboxes, scores, - score_threshold, nms_top_k, - keep_top_k, post_threshold, - use_gaussian, gaussian_sigma, - background_label, normalized) + out, index, rois_num = _C_ops.matrix_nms( + bboxes, + scores, + score_threshold, + nms_top_k, + keep_top_k, + post_threshold, + use_gaussian, + gaussian_sigma, + background_label, + normalized, + ) if not return_index: index = None if not return_rois_num: rois_num = None return out, rois_num, index elif _in_legacy_dygraph(): - attrs = ('background_label', background_label, 'score_threshold', - score_threshold, 'post_threshold', post_threshold, 'nms_top_k', - nms_top_k, 'gaussian_sigma', gaussian_sigma, 'use_gaussian', - use_gaussian, 'keep_top_k', keep_top_k, 'normalized', - normalized) + attrs = ( + 'background_label', + background_label, + 'score_threshold', + score_threshold, + 'post_threshold', + post_threshold, + 'nms_top_k', + nms_top_k, + 'gaussian_sigma', + gaussian_sigma, + 'use_gaussian', + use_gaussian, + 'keep_top_k', + keep_top_k, + 'normalized', + normalized, + ) out, index, rois_num = _legacy_C_ops.matrix_nms(bboxes, scores, *attrs) if not return_index: index = None @@ -1921,22 +2195,21 @@ def matrix_nms(bboxes, rois_num = helper.create_variable_for_type_inference(dtype='int32') outputs['RoisNum'] = rois_num - helper.append_op(type="matrix_nms", - inputs={ - 'BBoxes': bboxes, - 'Scores': scores - }, - attrs={ - 'background_label': background_label, - 'score_threshold': score_threshold, - 'post_threshold': post_threshold, - 'nms_top_k': nms_top_k, - 'gaussian_sigma': gaussian_sigma, - 'use_gaussian': use_gaussian, - 'keep_top_k': keep_top_k, - 'normalized': normalized - }, - outputs=outputs) + helper.append_op( + type="matrix_nms", + inputs={'BBoxes': bboxes, 'Scores': scores}, + attrs={ + 'background_label': background_label, + 'score_threshold': score_threshold, + 'post_threshold': post_threshold, + 'nms_top_k': nms_top_k, + 'gaussian_sigma': gaussian_sigma, + 'use_gaussian': use_gaussian, + 'keep_top_k': keep_top_k, + 'normalized': normalized, + }, + outputs=outputs, + ) output.stop_gradient = True if not return_index: diff --git a/python/paddle/vision/transforms/__init__.py b/python/paddle/vision/transforms/__init__.py index d615598bf2bcc..890e4b8982714 100644 --- a/python/paddle/vision/transforms/__init__.py +++ b/python/paddle/vision/transforms/__init__.py @@ -51,7 +51,7 @@ from .functional import normalize # noqa: F401 from .functional import erase # noqa: F401 -__all__ = [ #noqa +__all__ = [ # noqa 'BaseTransform', 'Compose', 'Resize', diff --git a/python/paddle/vision/transforms/functional.py b/python/paddle/vision/transforms/functional.py index 99d27a60ae3b0..45e50712c9e75 100644 --- a/python/paddle/vision/transforms/functional.py +++ b/python/paddle/vision/transforms/functional.py @@ -66,11 +66,14 @@ def to_tensor(pic, data_format='CHW'): print(tensor.shape) """ - if not (_is_pil_image(pic) or _is_numpy_image(pic) - or _is_tensor_image(pic)): + if not ( + _is_pil_image(pic) or _is_numpy_image(pic) or _is_tensor_image(pic) + ): raise TypeError( - 'pic should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(pic))) + 'pic should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(pic) + ) + ) if _is_pil_image(pic): return F_pil.to_tensor(pic, data_format) @@ -124,11 +127,14 @@ def resize(img, size, interpolation='bilinear'): print(converted_img.size) # (150, 200) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.resize(img, size, interpolation) @@ -188,11 +194,14 @@ def pad(img, padding, fill=0, padding_mode='constant'): padded_img = F.pad(fake_img, padding=(2, 1)) print(padded_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.pad(img, padding, fill, padding_mode) @@ -231,11 +240,14 @@ def crop(img, top, left, height, width): print(cropped_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.crop(img, top, left, height, width) @@ -248,33 +260,36 @@ def crop(img, top, left, height, width): def center_crop(img, output_size): """Crops the given Image and resize it to desired size. - Args: - img (PIL.Image|np.array): Image to be cropped. (0,0) denotes the top left corner of the image. - output_size (sequence or int): (height, width) of the crop box. If int, - it is used for both directions + Args: + img (PIL.Image|np.array): Image to be cropped. (0,0) denotes the top left corner of the image. + output_size (sequence or int): (height, width) of the crop box. If int, + it is used for both directions - Returns: - PIL.Image or np.array: Cropped image. + Returns: + PIL.Image or np.array: Cropped image. - Examples: - .. code-block:: python + Examples: + .. code-block:: python - import numpy as np - from PIL import Image - from paddle.vision.transforms import functional as F + import numpy as np + from PIL import Image + from paddle.vision.transforms import functional as F - fake_img = (np.random.rand(256, 300, 3) * 255.).astype('uint8') + fake_img = (np.random.rand(256, 300, 3) * 255.).astype('uint8') - fake_img = Image.fromarray(fake_img) + fake_img = Image.fromarray(fake_img) - cropped_img = F.center_crop(fake_img, (150, 100)) - print(cropped_img.size) - """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + cropped_img = F.center_crop(fake_img, (150, 100)) + print(cropped_img.size) + """ + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.center_crop(img, output_size) @@ -308,11 +323,14 @@ def hflip(img): print(flpped_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.hflip(img) @@ -346,11 +364,14 @@ def vflip(img): print(flpped_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.vflip(img) @@ -391,11 +412,14 @@ def adjust_brightness(img, brightness_factor): """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.adjust_brightness(img, brightness_factor) @@ -431,11 +455,14 @@ def adjust_contrast(img, contrast_factor): converted_img = F.adjust_contrast(fake_img, 0.4) print(converted_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.adjust_contrast(img, contrast_factor) @@ -472,11 +499,14 @@ def adjust_saturation(img, saturation_factor): print(converted_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.adjust_saturation(img, saturation_factor) @@ -522,11 +552,14 @@ def adjust_hue(img, hue_factor): print(converted_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.adjust_hue(img, hue_factor) @@ -567,14 +600,16 @@ def _get_affine_matrix(center, angle, translate, scale, shear): return matrix -def affine(img, - angle, - translate, - scale, - shear, - interpolation="nearest", - fill=0, - center=None): +def affine( + img, + angle, + translate, + scale, + shear, + interpolation="nearest", + fill=0, + center=None, +): """Apply affine transformation on the image. Args: @@ -615,11 +650,14 @@ def affine(img, print(affined_img.shape) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if not isinstance(angle, (int, float)): raise TypeError("Argument angle should be int or float") @@ -635,7 +673,8 @@ def affine(img, if not isinstance(shear, (numbers.Number, (list, tuple))): raise TypeError( - "Shear should be either a single value or a sequence of two values") + "Shear should be either a single value or a sequence of two values" + ) if not isinstance(interpolation, str): raise TypeError("Argument interpolation should be a string") @@ -657,7 +696,8 @@ def affine(img, if len(shear) != 2: raise ValueError( - f"Shear should be a sequence containing two values. Got {shear}") + f"Shear should be a sequence containing two values. Got {shear}" + ) if center is not None and not isinstance(center, (list, tuple)): raise TypeError("Argument center should be a sequence") @@ -680,8 +720,9 @@ def affine(img, # otherwise image rotated by 90 degrees is shifted vs output image of F_t.affine if center is None: center = (width * 0.5, height * 0.5) - return F_cv2.affine(img, angle, translate, scale, shear, interpolation, - fill, center) + return F_cv2.affine( + img, angle, translate, scale, shear, interpolation, fill, center + ) if _is_tensor_image(img): center_f = [0.0, 0.0] @@ -696,12 +737,9 @@ def affine(img, return F_t.affine(img, matrix, interpolation, fill) -def rotate(img, - angle, - interpolation="nearest", - expand=False, - center=None, - fill=0): +def rotate( + img, angle, interpolation="nearest", expand=False, center=None, fill=0 +): """Rotates the image by angle. @@ -747,11 +785,14 @@ def rotate(img, print(rotated_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if isinstance(center, list): center = tuple(center) @@ -784,10 +825,24 @@ def _get_perspective_coeffs(startpoints, endpoints): for i, (p1, p2) in enumerate(zip(endpoints, startpoints)): a_matrix[2 * i, :] = [ - p1[0], p1[1], 1, 0, 0, 0, -p2[0] * p1[0], -p2[0] * p1[1] + p1[0], + p1[1], + 1, + 0, + 0, + 0, + -p2[0] * p1[0], + -p2[0] * p1[1], ] a_matrix[2 * i + 1, :] = [ - 0, 0, 0, p1[0], p1[1], 1, -p2[1] * p1[0], -p2[1] * p1[1] + 0, + 0, + 0, + p1[0], + p1[1], + 1, + -p2[1] * p1[0], + -p2[1] * p1[1], ] b_matrix = np.array(startpoints).reshape([8]) @@ -838,11 +893,14 @@ def perspective(img, startpoints, endpoints, interpolation='nearest', fill=0): print(perspectived_img.shape) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): coeffs = _get_perspective_coeffs(startpoints, endpoints) @@ -851,8 +909,9 @@ def perspective(img, startpoints, endpoints, interpolation='nearest', fill=0): coeffs = _get_perspective_coeffs(startpoints, endpoints) return F_t.perspective(img, coeffs, interpolation, fill) else: - return F_cv2.perspective(img, startpoints, endpoints, interpolation, - fill) + return F_cv2.perspective( + img, startpoints, endpoints, interpolation, fill + ) def to_grayscale(img, num_output_channels=1): @@ -882,11 +941,14 @@ def to_grayscale(img, num_output_channels=1): print(gray_img.size) """ - if not (_is_pil_image(img) or _is_numpy_image(img) - or _is_tensor_image(img)): + if not ( + _is_pil_image(img) or _is_numpy_image(img) or _is_tensor_image(img) + ): raise TypeError( - 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}' - .format(type(img))) + 'img should be PIL Image or Tensor Image or ndarray with dim=[2 or 3]. Got {}'.format( + type(img) + ) + ) if _is_pil_image(img): return F_pil.to_grayscale(img, num_output_channels) @@ -942,53 +1004,53 @@ def normalize(img, mean, std, data_format='CHW', to_rgb=False): def erase(img, i, j, h, w, v, inplace=False): """Erase the pixels of selected area in input image with given value. - Args: - img (paddle.Tensor | np.array | PIL.Image): input Tensor image. - For Tensor input, the shape should be (C, H, W). For np.array input, - the shape should be (H, W, C). - i (int): y coordinate of the top-left point of erased region. - j (int): x coordinate of the top-left point of erased region. - h (int): Height of the erased region. - w (int): Width of the erased region. - v (paddle.Tensor | np.array): value used to replace the pixels in erased region. It - should be np.array when img is np.array or PIL.Image. - inplace (bool, optional): Whether this transform is inplace. Default: False. - - Returns: - paddle.Tensor | np.array | PIL.Image: Erased image. The type is same with input image. - - Examples: - .. code-block:: python - - import paddle - - fake_img = paddle.randn((3, 2, 4)).astype(paddle.float32) - print(fake_img) - - #Tensor(shape=[3, 2, 4], dtype=float32, place=Place(gpu:0), stop_gradient=True, - # [[[ 0.02169025, -0.97859967, -1.39175487, -1.07478464], - # [ 0.20654772, 1.74624777, 0.32268861, -0.13857445]], - # - # [[-0.14993843, 1.10793507, -0.40056887, -1.94395220], - # [ 0.41686651, 0.44551995, -0.09356714, -0.60898107]], - # - # [[-0.24998808, -1.47699273, -0.88838995, 0.42629015], - # [ 0.56948012, -0.96200180, 0.53355658, 3.20450878]]]) - - values = paddle.zeros((1,1,1), dtype=paddle.float32) - result = paddle.vision.transforms.erase(fake_img, 0, 1, 1, 2, values) - - print(result) - - #Tensor(shape=[3, 2, 4], dtype=float32, place=Place(gpu:0), stop_gradient=True, - # [[[ 0.02169025, 0. , 0. , -1.07478464], - # [ 0.20654772, 1.74624777, 0.32268861, -0.13857445]], - # - # [[-0.14993843, 0. , 0. , -1.94395220], - # [ 0.41686651, 0.44551995, -0.09356714, -0.60898107]], - # - # [[-0.24998808, 0. , 0. , 0.42629015], - # [ 0.56948012, -0.96200180, 0.53355658, 3.20450878]]]) + Args: + img (paddle.Tensor | np.array | PIL.Image): input Tensor image. + For Tensor input, the shape should be (C, H, W). For np.array input, + the shape should be (H, W, C). + i (int): y coordinate of the top-left point of erased region. + j (int): x coordinate of the top-left point of erased region. + h (int): Height of the erased region. + w (int): Width of the erased region. + v (paddle.Tensor | np.array): value used to replace the pixels in erased region. It + should be np.array when img is np.array or PIL.Image. + inplace (bool, optional): Whether this transform is inplace. Default: False. + + Returns: + paddle.Tensor | np.array | PIL.Image: Erased image. The type is same with input image. + + Examples: + .. code-block:: python + + import paddle + + fake_img = paddle.randn((3, 2, 4)).astype(paddle.float32) + print(fake_img) + + #Tensor(shape=[3, 2, 4], dtype=float32, place=Place(gpu:0), stop_gradient=True, + # [[[ 0.02169025, -0.97859967, -1.39175487, -1.07478464], + # [ 0.20654772, 1.74624777, 0.32268861, -0.13857445]], + # + # [[-0.14993843, 1.10793507, -0.40056887, -1.94395220], + # [ 0.41686651, 0.44551995, -0.09356714, -0.60898107]], + # + # [[-0.24998808, -1.47699273, -0.88838995, 0.42629015], + # [ 0.56948012, -0.96200180, 0.53355658, 3.20450878]]]) + + values = paddle.zeros((1,1,1), dtype=paddle.float32) + result = paddle.vision.transforms.erase(fake_img, 0, 1, 1, 2, values) + + print(result) + + #Tensor(shape=[3, 2, 4], dtype=float32, place=Place(gpu:0), stop_gradient=True, + # [[[ 0.02169025, 0. , 0. , -1.07478464], + # [ 0.20654772, 1.74624777, 0.32268861, -0.13857445]], + # + # [[-0.14993843, 0. , 0. , -1.94395220], + # [ 0.41686651, 0.44551995, -0.09356714, -0.60898107]], + # + # [[-0.24998808, 0. , 0. , 0.42629015], + # [ 0.56948012, -0.96200180, 0.53355658, 3.20450878]]]) """ if _is_tensor_image(img): diff --git a/python/paddle/vision/transforms/functional_cv2.py b/python/paddle/vision/transforms/functional_cv2.py index ccd638089af7b..52609b786fe1f 100644 --- a/python/paddle/vision/transforms/functional_cv2.py +++ b/python/paddle/vision/transforms/functional_cv2.py @@ -49,7 +49,8 @@ def to_tensor(pic, data_format='CHW'): if data_format not in ['CHW', 'HWC']: raise ValueError( - 'data_format should be CHW or HWC. Got {}'.format(data_format)) + 'data_format should be CHW or HWC. Got {}'.format(data_format) + ) if pic.ndim == 2: pic = pic[:, :, None] @@ -60,7 +61,7 @@ def to_tensor(pic, data_format='CHW'): img = paddle.to_tensor(pic) if paddle.fluid.data_feeder.convert_dtype(img.dtype) == 'uint8': - return paddle.cast(img, np.float32) / 255. + return paddle.cast(img, np.float32) / 255.0 else: return img @@ -90,11 +91,12 @@ def resize(img, size, interpolation='bilinear'): 'bilinear': cv2.INTER_LINEAR, 'area': cv2.INTER_AREA, 'bicubic': cv2.INTER_CUBIC, - 'lanczos': cv2.INTER_LANCZOS4 + 'lanczos': cv2.INTER_LANCZOS4, } - if not (isinstance(size, int) or - (isinstance(size, Iterable) and len(size) == 2)): + if not ( + isinstance(size, int) or (isinstance(size, Iterable) and len(size) == 2) + ): raise TypeError('Got inappropriate size arg: {}'.format(size)) h, w = img.shape[:2] @@ -108,18 +110,22 @@ def resize(img, size, interpolation='bilinear'): output = cv2.resize( img, dsize=(ow, oh), - interpolation=_cv2_interp_from_str[interpolation]) + interpolation=_cv2_interp_from_str[interpolation], + ) else: oh = size ow = int(size * w / h) output = cv2.resize( img, dsize=(ow, oh), - interpolation=_cv2_interp_from_str[interpolation]) + interpolation=_cv2_interp_from_str[interpolation], + ) else: - output = cv2.resize(img, - dsize=(size[1], size[0]), - interpolation=_cv2_interp_from_str[interpolation]) + output = cv2.resize( + img, + dsize=(size[1], size[0]), + interpolation=_cv2_interp_from_str[interpolation], + ) if len(img.shape) == 3 and img.shape[2] == 1: return output[:, :, np.newaxis] else: @@ -165,7 +171,7 @@ def pad(img, padding, fill=0, padding_mode='constant'): 'constant': cv2.BORDER_CONSTANT, 'edge': cv2.BORDER_REPLICATE, 'reflect': cv2.BORDER_REFLECT_101, - 'symmetric': cv2.BORDER_REFLECT + 'symmetric': cv2.BORDER_REFLECT, } if not isinstance(padding, (numbers.Number, list, tuple)): @@ -177,11 +183,16 @@ def pad(img, padding, fill=0, padding_mode='constant'): if isinstance(padding, Sequence) and len(padding) not in [2, 4]: raise ValueError( - "Padding must be an int or a 2, or 4 element tuple, not a " + - "{} element tuple".format(len(padding))) + "Padding must be an int or a 2, or 4 element tuple, not a " + + "{} element tuple".format(len(padding)) + ) - assert padding_mode in ['constant', 'edge', 'reflect', 'symmetric'], \ - 'Padding mode should be either constant, edge, reflect or symmetric' + assert padding_mode in [ + 'constant', + 'edge', + 'reflect', + 'symmetric', + ], 'Padding mode should be either constant, edge, reflect or symmetric' if isinstance(padding, list): padding = tuple(padding) @@ -197,21 +208,25 @@ def pad(img, padding, fill=0, padding_mode='constant'): pad_bottom = padding[3] if len(img.shape) == 3 and img.shape[2] == 1: - return cv2.copyMakeBorder(img, - top=pad_top, - bottom=pad_bottom, - left=pad_left, - right=pad_right, - borderType=_cv2_pad_from_str[padding_mode], - value=fill)[:, :, np.newaxis] + return cv2.copyMakeBorder( + img, + top=pad_top, + bottom=pad_bottom, + left=pad_left, + right=pad_right, + borderType=_cv2_pad_from_str[padding_mode], + value=fill, + )[:, :, np.newaxis] else: - return cv2.copyMakeBorder(img, - top=pad_top, - bottom=pad_bottom, - left=pad_left, - right=pad_right, - borderType=_cv2_pad_from_str[padding_mode], - value=fill) + return cv2.copyMakeBorder( + img, + top=pad_top, + bottom=pad_bottom, + left=pad_left, + right=pad_right, + borderType=_cv2_pad_from_str[padding_mode], + value=fill, + ) def crop(img, top, left, height, width): @@ -230,30 +245,30 @@ def crop(img, top, left, height, width): """ - return img[top:top + height, left:left + width, :] + return img[top : top + height, left : left + width, :] def center_crop(img, output_size): """Crops the given image and resize it to desired size. - Args: - img (np.array): Image to be cropped. (0,0) denotes the top left corner of the image. - output_size (sequence or int): (height, width) of the crop box. If int, - it is used for both directions - backend (str, optional): The image proccess backend type. Options are `pil`, `cv2`. Default: 'pil'. + Args: + img (np.array): Image to be cropped. (0,0) denotes the top left corner of the image. + output_size (sequence or int): (height, width) of the crop box. If int, + it is used for both directions + backend (str, optional): The image proccess backend type. Options are `pil`, `cv2`. Default: 'pil'. - Returns: - np.array: Cropped image. + Returns: + np.array: Cropped image. - """ + """ if isinstance(output_size, numbers.Number): output_size = (int(output_size), int(output_size)) h, w = img.shape[0:2] th, tw = output_size - i = int(round((h - th) / 2.)) - j = int(round((w - tw) / 2.)) + i = int(round((h - th) / 2.0)) + j = int(round((w - tw) / 2.0)) return crop(img, i, j, th, tw) @@ -305,8 +320,11 @@ def adjust_brightness(img, brightness_factor): """ cv2 = try_import('cv2') - table = np.array([i * brightness_factor - for i in range(0, 256)]).clip(0, 255).astype('uint8') + table = ( + np.array([i * brightness_factor for i in range(0, 256)]) + .clip(0, 255) + .astype('uint8') + ) if len(img.shape) == 3 and img.shape[2] == 1: return cv2.LUT(img, table)[:, :, np.newaxis] @@ -329,8 +347,11 @@ def adjust_contrast(img, contrast_factor): """ cv2 = try_import('cv2') - table = np.array([(i - 74) * contrast_factor + 74 - for i in range(0, 256)]).clip(0, 255).astype('uint8') + table = ( + np.array([(i - 74) * contrast_factor + 74 for i in range(0, 256)]) + .clip(0, 255) + .astype('uint8') + ) if len(img.shape) == 3 and img.shape[2] == 1: return cv2.LUT(img, table)[:, :, np.newaxis] else: @@ -354,8 +375,9 @@ def adjust_saturation(img, saturation_factor): dtype = img.dtype img = img.astype(np.float32) - alpha = np.random.uniform(max(0, 1 - saturation_factor), - 1 + saturation_factor) + alpha = np.random.uniform( + max(0, 1 - saturation_factor), 1 + saturation_factor + ) gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) gray_img = gray_img[..., np.newaxis] img = img * alpha + gray_img * (1 - alpha) @@ -388,7 +410,8 @@ def adjust_hue(img, hue_factor): if not (-0.5 <= hue_factor <= 0.5): raise ValueError( - 'hue_factor:{} is not in [-0.5, 0.5].'.format(hue_factor)) + 'hue_factor:{} is not in [-0.5, 0.5].'.format(hue_factor) + ) dtype = img.dtype img = img.astype(np.uint8) @@ -404,14 +427,16 @@ def adjust_hue(img, hue_factor): return cv2.cvtColor(hsv_img, cv2.COLOR_HSV2BGR_FULL).astype(dtype) -def affine(img, - angle, - translate, - scale, - shear, - interpolation='nearest', - fill=0, - center=None): +def affine( + img, + angle, + translate, + scale, + shear, + interpolation='nearest', + fill=0, + center=None, +): """Affine the image by matrix. Args: @@ -442,7 +467,7 @@ def affine(img, 'bilinear': cv2.INTER_LINEAR, 'area': cv2.INTER_AREA, 'bicubic': cv2.INTER_CUBIC, - 'lanczos': cv2.INTER_LANCZOS4 + 'lanczos': cv2.INTER_LANCZOS4, } h, w = img.shape[0:2] @@ -469,25 +494,26 @@ def affine(img, M[1, 2] = ty if len(img.shape) == 3 and img.shape[2] == 1: - return cv2.warpAffine(img, - M, - dsize=(w, h), - flags=_cv2_interp_from_str[interpolation], - borderValue=fill)[:, :, np.newaxis] + return cv2.warpAffine( + img, + M, + dsize=(w, h), + flags=_cv2_interp_from_str[interpolation], + borderValue=fill, + )[:, :, np.newaxis] else: - return cv2.warpAffine(img, - M, - dsize=(w, h), - flags=_cv2_interp_from_str[interpolation], - borderValue=fill) - - -def rotate(img, - angle, - interpolation='nearest', - expand=False, - center=None, - fill=0): + return cv2.warpAffine( + img, + M, + dsize=(w, h), + flags=_cv2_interp_from_str[interpolation], + borderValue=fill, + ) + + +def rotate( + img, angle, interpolation='nearest', expand=False, center=None, fill=0 +): """Rotates the image by angle. Args: @@ -519,7 +545,7 @@ def rotate(img, 'bilinear': cv2.INTER_LINEAR, 'area': cv2.INTER_AREA, 'bicubic': cv2.INTER_CUBIC, - 'lanczos': cv2.INTER_LANCZOS4 + 'lanczos': cv2.INTER_LANCZOS4, } h, w = img.shape[0:2] @@ -549,8 +575,10 @@ def transform(x, y, matrix): post_trans = (0, 0) expand_matrix[2], expand_matrix[5] = transform( - -center[0] - post_trans[0], -center[1] - post_trans[1], - expand_matrix) + -center[0] - post_trans[0], + -center[1] - post_trans[1], + expand_matrix, + ) expand_matrix[2] += center[0] expand_matrix[5] += center[1] @@ -567,15 +595,21 @@ def transform(x, y, matrix): w, h = int(nw), int(nh) if len(img.shape) == 3 and img.shape[2] == 1: - return cv2.warpAffine(img, - M, (w, h), - flags=_cv2_interp_from_str[interpolation], - borderValue=fill)[:, :, np.newaxis] + return cv2.warpAffine( + img, + M, + (w, h), + flags=_cv2_interp_from_str[interpolation], + borderValue=fill, + )[:, :, np.newaxis] else: - return cv2.warpAffine(img, - M, (w, h), - flags=_cv2_interp_from_str[interpolation], - borderValue=fill) + return cv2.warpAffine( + img, + M, + (w, h), + flags=_cv2_interp_from_str[interpolation], + borderValue=fill, + ) def perspective(img, startpoints, endpoints, interpolation='nearest', fill=0): @@ -604,7 +638,7 @@ def perspective(img, startpoints, endpoints, interpolation='nearest', fill=0): 'bilinear': cv2.INTER_LINEAR, 'area': cv2.INTER_AREA, 'bicubic': cv2.INTER_CUBIC, - 'lanczos': cv2.INTER_LANCZOS4 + 'lanczos': cv2.INTER_LANCZOS4, } h, w = img.shape[0:2] @@ -613,17 +647,21 @@ def perspective(img, startpoints, endpoints, interpolation='nearest', fill=0): matrix = cv2.getPerspectiveTransform(startpoints, endpoints) if len(img.shape) == 3 and img.shape[2] == 1: - return cv2.warpPerspective(img, - matrix, - dsize=(w, h), - flags=_cv2_interp_from_str[interpolation], - borderValue=fill)[:, :, np.newaxis] + return cv2.warpPerspective( + img, + matrix, + dsize=(w, h), + flags=_cv2_interp_from_str[interpolation], + borderValue=fill, + )[:, :, np.newaxis] else: - return cv2.warpPerspective(img, - matrix, - dsize=(w, h), - flags=_cv2_interp_from_str[interpolation], - borderValue=fill) + return cv2.warpPerspective( + img, + matrix, + dsize=(w, h), + flags=_cv2_interp_from_str[interpolation], + borderValue=fill, + ) def to_grayscale(img, num_output_channels=1): @@ -646,7 +684,8 @@ def to_grayscale(img, num_output_channels=1): elif num_output_channels == 3: # much faster than doing cvtColor to go back to gray img = np.broadcast_to( - cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)[:, :, np.newaxis], img.shape) + cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)[:, :, np.newaxis], img.shape + ) else: raise ValueError('num_output_channels should be either 1 or 3') @@ -686,21 +725,21 @@ def normalize(img, mean, std, data_format='CHW', to_rgb=False): def erase(img, i, j, h, w, v, inplace=False): """Erase the pixels of selected area in input image array with given value. - Args: - img (np.array): input image array, which shape is (H, W, C). - i (int): y coordinate of the top-left point of erased region. - j (int): x coordinate of the top-left point of erased region. - h (int): Height of the erased region. - w (int): Width of the erased region. - v (np.array): value used to replace the pixels in erased region. - inplace (bool, optional): Whether this transform is inplace. Default: False. + Args: + img (np.array): input image array, which shape is (H, W, C). + i (int): y coordinate of the top-left point of erased region. + j (int): x coordinate of the top-left point of erased region. + h (int): Height of the erased region. + w (int): Width of the erased region. + v (np.array): value used to replace the pixels in erased region. + inplace (bool, optional): Whether this transform is inplace. Default: False. - Returns: - np.array: Erased image. + Returns: + np.array: Erased image. """ if not inplace: img = img.copy() - img[i:i + h, j:j + w, ...] = v + img[i : i + h, j : j + w, ...] = v return img diff --git a/python/paddle/vision/transforms/functional_pil.py b/python/paddle/vision/transforms/functional_pil.py index cd4556effb7d9..595d92a84b11b 100644 --- a/python/paddle/vision/transforms/functional_pil.py +++ b/python/paddle/vision/transforms/functional_pil.py @@ -35,7 +35,7 @@ 'bicubic': Image.Resampling.BICUBIC, 'box': Image.Resampling.BOX, 'lanczos': Image.Resampling.LANCZOS, - 'hamming': Image.Resampling.HAMMING + 'hamming': Image.Resampling.HAMMING, } except: _pil_interp_from_str = { @@ -44,7 +44,7 @@ 'bicubic': Image.BICUBIC, 'box': Image.BOX, 'lanczos': Image.LANCZOS, - 'hamming': Image.HAMMING + 'hamming': Image.HAMMING, } __all__ = [] @@ -67,7 +67,8 @@ def to_tensor(pic, data_format='CHW'): if data_format not in ['CHW', 'HWC']: raise ValueError( - 'data_format should be CHW or HWC. Got {}'.format(data_format)) + 'data_format should be CHW or HWC. Got {}'.format(data_format) + ) # PIL Image if pic.mode == 'I': @@ -91,7 +92,7 @@ def to_tensor(pic, data_format='CHW'): dtype = paddle.fluid.data_feeder.convert_dtype(img.dtype) if dtype == 'uint8': - img = paddle.cast(img, np.float32) / 255. + img = paddle.cast(img, np.float32) / 255.0 img = img.reshape([pic.size[1], pic.size[0], nchannel]) @@ -122,8 +123,9 @@ def resize(img, size, interpolation='bilinear'): """ - if not (isinstance(size, int) or - (isinstance(size, Iterable) and len(size) == 2)): + if not ( + isinstance(size, int) or (isinstance(size, Iterable) and len(size) == 2) + ): raise TypeError('Got inappropriate size arg: {}'.format(size)) if isinstance(size, int): @@ -186,11 +188,16 @@ def pad(img, padding, fill=0, padding_mode='constant'): if isinstance(padding, Sequence) and len(padding) not in [2, 4]: raise ValueError( - "Padding must be an int or a 2, or 4 element tuple, not a " + - "{} element tuple".format(len(padding))) + "Padding must be an int or a 2, or 4 element tuple, not a " + + "{} element tuple".format(len(padding)) + ) - assert padding_mode in ['constant', 'edge', 'reflect', 'symmetric'], \ - 'Padding mode should be either constant, edge, reflect or symmetric' + assert padding_mode in [ + 'constant', + 'edge', + 'reflect', + 'symmetric', + ], 'Padding mode should be either constant, edge, reflect or symmetric' if isinstance(padding, list): padding = tuple(padding) @@ -217,8 +224,11 @@ def pad(img, padding, fill=0, padding_mode='constant'): if img.mode == 'P': palette = img.getpalette() img = np.asarray(img) - img = np.pad(img, ((pad_top, pad_bottom), (pad_left, pad_right)), - padding_mode) + img = np.pad( + img, + ((pad_top, pad_bottom), (pad_left, pad_right)), + padding_mode, + ) img = Image.fromarray(img) img.putpalette(palette) return img @@ -226,13 +236,18 @@ def pad(img, padding, fill=0, padding_mode='constant'): img = np.asarray(img) # RGB image if len(img.shape) == 3: - img = np.pad(img, - ((pad_top, pad_bottom), (pad_left, pad_right), (0, 0)), - padding_mode) + img = np.pad( + img, + ((pad_top, pad_bottom), (pad_left, pad_right), (0, 0)), + padding_mode, + ) # Grayscale image if len(img.shape) == 2: - img = np.pad(img, ((pad_top, pad_bottom), (pad_left, pad_right)), - padding_mode) + img = np.pad( + img, + ((pad_top, pad_bottom), (pad_left, pad_right)), + padding_mode, + ) return Image.fromarray(img) @@ -258,24 +273,24 @@ def crop(img, top, left, height, width): def center_crop(img, output_size): """Crops the given PIL Image and resize it to desired size. - Args: - img (PIL.Image): Image to be cropped. (0,0) denotes the top left corner of the image. - output_size (sequence or int): (height, width) of the crop box. If int, - it is used for both directions - backend (str, optional): The image proccess backend type. Options are `pil`, `cv2`. Default: 'pil'. + Args: + img (PIL.Image): Image to be cropped. (0,0) denotes the top left corner of the image. + output_size (sequence or int): (height, width) of the crop box. If int, + it is used for both directions + backend (str, optional): The image proccess backend type. Options are `pil`, `cv2`. Default: 'pil'. - Returns: - PIL.Image: Cropped image. + Returns: + PIL.Image: Cropped image. - """ + """ if isinstance(output_size, numbers.Number): output_size = (int(output_size), int(output_size)) image_width, image_height = img.size crop_height, crop_width = output_size - crop_top = int(round((image_height - crop_height) / 2.)) - crop_left = int(round((image_width - crop_width) / 2.)) + crop_top = int(round((image_height - crop_height) / 2.0)) + crop_left = int(round((image_width - crop_width) / 2.0)) return crop(img, crop_top, crop_left, crop_height, crop_width) @@ -388,7 +403,8 @@ def adjust_hue(img, hue_factor): """ if not (-0.5 <= hue_factor <= 0.5): raise ValueError( - 'hue_factor:{} is not in [-0.5, 0.5].'.format(hue_factor)) + 'hue_factor:{} is not in [-0.5, 0.5].'.format(hue_factor) + ) input_mode = img.mode if input_mode in {'L', '1', 'I', 'F'}: @@ -428,16 +444,18 @@ def affine(img, matrix, interpolation="nearest", fill=0): if isinstance(fill, int): fill = tuple([fill] * 3) - return img.transform(img.size, Image.AFFINE, matrix, - _pil_interp_from_str[interpolation], fill) + return img.transform( + img.size, + Image.AFFINE, + matrix, + _pil_interp_from_str[interpolation], + fill, + ) -def rotate(img, - angle, - interpolation="nearest", - expand=False, - center=None, - fill=0): +def rotate( + img, angle, interpolation="nearest", expand=False, center=None, fill=0 +): """Rotates the image by angle. Args: @@ -467,11 +485,13 @@ def rotate(img, if isinstance(fill, int): fill = tuple([fill] * 3) - return img.rotate(angle, - _pil_interp_from_str[interpolation], - expand, - center, - fillcolor=fill) + return img.rotate( + angle, + _pil_interp_from_str[interpolation], + expand, + center, + fillcolor=fill, + ) def perspective(img, coeffs, interpolation="nearest", fill=0): @@ -497,8 +517,13 @@ def perspective(img, coeffs, interpolation="nearest", fill=0): if isinstance(fill, int): fill = tuple([fill] * 3) - return img.transform(img.size, Image.PERSPECTIVE, coeffs, - _pil_interp_from_str[interpolation], fill) + return img.transform( + img.size, + Image.PERSPECTIVE, + coeffs, + _pil_interp_from_str[interpolation], + fill, + ) def to_grayscale(img, num_output_channels=1): @@ -532,22 +557,22 @@ def to_grayscale(img, num_output_channels=1): def erase(img, i, j, h, w, v, inplace=False): """Erase the pixels of selected area in input image with given value. PIL format is - not support inplace. + not support inplace. - Args: - img (PIL.Image): input image, which shape is (C, H, W). - i (int): y coordinate of the top-left point of erased region. - j (int): x coordinate of the top-left point of erased region. - h (int): Height of the erased region. - w (int): Width of the erased region. - v (np.array): value used to replace the pixels in erased region. - inplace (bool, optional): Whether this transform is inplace. Default: False. + Args: + img (PIL.Image): input image, which shape is (C, H, W). + i (int): y coordinate of the top-left point of erased region. + j (int): x coordinate of the top-left point of erased region. + h (int): Height of the erased region. + w (int): Width of the erased region. + v (np.array): value used to replace the pixels in erased region. + inplace (bool, optional): Whether this transform is inplace. Default: False. - Returns: - PIL.Image: Erased image. + Returns: + PIL.Image: Erased image. """ np_img = np.array(img, dtype=np.uint8) - np_img[i:i + h, j:j + w, ...] = v + np_img[i : i + h, j : j + w, ...] = v img = Image.fromarray(np_img, 'RGB') return img diff --git a/python/paddle/vision/transforms/functional_tensor.py b/python/paddle/vision/transforms/functional_tensor.py index 911f7e2d260bd..d18fdfc51c9c1 100644 --- a/python/paddle/vision/transforms/functional_tensor.py +++ b/python/paddle/vision/transforms/functional_tensor.py @@ -22,13 +22,17 @@ def _assert_image_tensor(img, data_format): - if not isinstance( - img, paddle.Tensor - ) or img.ndim < 3 or img.ndim > 4 or not data_format.lower() in ('chw', - 'hwc'): + if ( + not isinstance(img, paddle.Tensor) + or img.ndim < 3 + or img.ndim > 4 + or not data_format.lower() in ('chw', 'hwc') + ): raise RuntimeError( - 'not support [type={}, ndim={}, data_format={}] paddle image'. - format(type(img), img.ndim, data_format)) + 'not support [type={}, ndim={}, data_format={}] paddle image'.format( + type(img), img.ndim, data_format + ) + ) def _get_image_h_axis(data_format): @@ -78,13 +82,15 @@ def _get_image_num_channels(img, data_format): def _get_image_size(img, data_format): - return img.shape[_get_image_w_axis(data_format)], img.shape[ - _get_image_h_axis(data_format)] + return ( + img.shape[_get_image_w_axis(data_format)], + img.shape[_get_image_h_axis(data_format)], + ) def _rgb_to_hsv(img): """Convert a image Tensor from RGB to HSV. This implementation is based on Pillow ( - https://github.com/python-pillow/Pillow/blob/main/src/libImaging/Convert.c) + https://github.com/python-pillow/Pillow/blob/main/src/libImaging/Convert.c) """ maxc = img.max(axis=-3) minc = img.min(axis=-3) @@ -111,8 +117,7 @@ def _rgb_to_hsv(img): def _hsv_to_rgb(img): - """Convert a image Tensor from HSV to RGB. - """ + """Convert a image Tensor from HSV to RGB.""" h, s, v = img.unbind(axis=-3) f = h * 6.0 i = paddle.floor(f) @@ -123,22 +128,28 @@ def _hsv_to_rgb(img): q = paddle.clip(v * (1.0 - s * f), 0.0, 1.0) t = paddle.clip(v * (1.0 - s * (1.0 - f)), 0.0, 1.0) - mask = paddle.equal(i.unsqueeze(axis=-3), - paddle.arange(6, dtype=i.dtype).reshape( - (-1, 1, 1))).astype(img.dtype) - matrix = paddle.stack([ - paddle.stack([v, q, p, p, t, v], axis=-3), - paddle.stack([t, v, v, q, p, p], axis=-3), - paddle.stack([p, p, t, v, v, q], axis=-3) - ], - axis=-4) + mask = paddle.equal( + i.unsqueeze(axis=-3), + paddle.arange(6, dtype=i.dtype).reshape((-1, 1, 1)), + ).astype(img.dtype) + matrix = paddle.stack( + [ + paddle.stack([v, q, p, p, t, v], axis=-3), + paddle.stack([t, v, v, q, p, p], axis=-3), + paddle.stack([p, p, t, v, v, q], axis=-3), + ], + axis=-4, + ) return paddle.einsum("...ijk, ...xijk -> ...xjk", mask, matrix) def _blend_images(img1, img2, ratio): max_value = 1.0 if paddle.is_floating_point(img1) else 255.0 - return paddle.lerp(img2, img1, - float(ratio)).clip(0, max_value).astype(img1.dtype) + return ( + paddle.lerp(img2, img1, float(ratio)) + .clip(0, max_value) + .astype(img1.dtype) + ) def normalize(img, mean, std, data_format='CHW'): @@ -186,8 +197,9 @@ def to_grayscale(img, num_output_channels=1, data_format='CHW'): if num_output_channels not in (1, 3): raise ValueError('num_output_channels should be either 1 or 3') - rgb_weights = paddle.to_tensor([0.2989, 0.5870, 0.1140], - place=img.place).astype(img.dtype) + rgb_weights = paddle.to_tensor( + [0.2989, 0.5870, 0.1140], place=img.place + ).astype(img.dtype) if _is_channel_first(data_format): rgb_weights = rgb_weights.reshape((-1, 1, 1)) @@ -210,8 +222,9 @@ def _affine_grid(theta, w, h, ow, oh): y_grid = paddle.linspace(-oh * 0.5 + d, oh * 0.5 + d - 1, oh).unsqueeze_(-1) base_grid[..., 1] = y_grid - scaled_theta = theta.transpose( - (0, 2, 1)) / paddle.to_tensor([0.5 * w, 0.5 * h]) + scaled_theta = theta.transpose((0, 2, 1)) / paddle.to_tensor( + [0.5 * w, 0.5 * h] + ) output_grid = base_grid.reshape((1, oh * ow, 3)).bmm(scaled_theta) return output_grid.reshape((1, oh, ow, 2)) @@ -220,18 +233,18 @@ def _affine_grid(theta, w, h, ow, oh): def _grid_transform(img, grid, mode, fill): if img.shape[0] > 1: grid = grid.expand( - shape=[img.shape[0], grid.shape[1], grid.shape[2], grid.shape[3]]) + shape=[img.shape[0], grid.shape[1], grid.shape[2], grid.shape[3]] + ) if fill is not None: - dummy = paddle.ones((img.shape[0], 1, img.shape[2], img.shape[3]), - dtype=img.dtype) + dummy = paddle.ones( + (img.shape[0], 1, img.shape[2], img.shape[3]), dtype=img.dtype + ) img = paddle.concat((img, dummy), axis=1) - img = F.grid_sample(img, - grid, - mode=mode, - padding_mode="zeros", - align_corners=False) + img = F.grid_sample( + img, grid, mode=mode, padding_mode="zeros", align_corners=False + ) # Fill with required color if fill is not None: @@ -239,12 +252,13 @@ def _grid_transform(img, grid, mode, fill): img = img[:, :-1, :, :] # n c h w mask = mask.expand_as(img) len_fill = len(fill) if isinstance(fill, (tuple, list)) else 1 - fill_img = paddle.to_tensor(fill).reshape( - (1, len_fill, 1, 1)).expand_as(img) + fill_img = ( + paddle.to_tensor(fill).reshape((1, len_fill, 1, 1)).expand_as(img) + ) if mode == 'nearest': mask = paddle.cast(mask < 0.5, img.dtype) - img = img * (1. - mask) + mask * fill_img + img = img * (1.0 - mask) + mask * fill_img else: # 'bilinear' img = img * mask + (1.0 - mask) * fill_img @@ -282,11 +296,9 @@ def affine(img, matrix, interpolation="nearest", fill=None, data_format='CHW'): matrix = matrix.reshape((1, 2, 3)) shape = img.shape - grid = _affine_grid(matrix, - w=shape[-1], - h=shape[-2], - ow=shape[-1], - oh=shape[-2]) + grid = _affine_grid( + matrix, w=shape[-1], h=shape[-2], ow=shape[-1], oh=shape[-2] + ) if isinstance(fill, int): fill = tuple([fill] * 3) @@ -299,13 +311,15 @@ def affine(img, matrix, interpolation="nearest", fill=None, data_format='CHW'): return out -def rotate(img, - angle, - interpolation='nearest', - expand=False, - center=None, - fill=None, - data_format='CHW'): +def rotate( + img, + angle, + interpolation='nearest', + expand=False, + center=None, + fill=None, + data_format='CHW', +): """Rotates the image by angle. Args: @@ -358,9 +372,11 @@ def rotate(img, ] matrix[2] += matrix[0] * (-rotn_center[0] - post_trans[0]) + matrix[1] * ( - -rotn_center[1] - post_trans[1]) + -rotn_center[1] - post_trans[1] + ) matrix[5] += matrix[3] * (-rotn_center[0] - post_trans[0]) + matrix[4] * ( - -rotn_center[1] - post_trans[1]) + -rotn_center[1] - post_trans[1] + ) matrix[2] += rotn_center[0] matrix[5] += rotn_center[1] @@ -371,12 +387,20 @@ def rotate(img, if expand: # calculate output size corners = paddle.to_tensor( - [[-0.5 * w, -0.5 * h, 1.0], [-0.5 * w, 0.5 * h, 1.0], - [0.5 * w, 0.5 * h, 1.0], [0.5 * w, -0.5 * h, 1.0]], - place=matrix.place).astype(matrix.dtype) - - _pos = corners.reshape((1, -1, 3)).bmm(matrix.transpose( - (0, 2, 1))).reshape((1, -1, 2)) + [ + [-0.5 * w, -0.5 * h, 1.0], + [-0.5 * w, 0.5 * h, 1.0], + [0.5 * w, 0.5 * h, 1.0], + [0.5 * w, -0.5 * h, 1.0], + ], + place=matrix.place, + ).astype(matrix.dtype) + + _pos = ( + corners.reshape((1, -1, 3)) + .bmm(matrix.transpose((0, 2, 1))) + .reshape((1, -1, 2)) + ) _min = _pos.min(axis=-2).floor() _max = _pos.max(axis=-2).ceil() @@ -412,21 +436,21 @@ def _perspective_grid(img, coeffs, ow, oh, dtype): y_grid = paddle.linspace(d, oh * 1.0 + d - 1.0, oh).unsqueeze_(-1) base_grid[..., 1] = y_grid - scaled_theta1 = theta1.transpose( - (0, 2, 1)) / paddle.to_tensor([0.5 * ow, 0.5 * oh]) + scaled_theta1 = theta1.transpose((0, 2, 1)) / paddle.to_tensor( + [0.5 * ow, 0.5 * oh] + ) output_grid1 = base_grid.reshape((1, oh * ow, 3)).bmm(scaled_theta1) - output_grid2 = base_grid.reshape( - (1, oh * ow, 3)).bmm(theta2.transpose((0, 2, 1))) + output_grid2 = base_grid.reshape((1, oh * ow, 3)).bmm( + theta2.transpose((0, 2, 1)) + ) output_grid = output_grid1 / output_grid2 - 1.0 return output_grid.reshape((1, oh, ow, 2)) -def perspective(img, - coeffs, - interpolation="nearest", - fill=None, - data_format='CHW'): +def perspective( + img, coeffs, interpolation="nearest", fill=None, data_format='CHW' +): """Perspective the image. Args: @@ -521,48 +545,48 @@ def crop(img, top, left, height, width, data_format='CHW'): _assert_image_tensor(img, data_format) if _is_channel_first(data_format): - return img[:, top:top + height, left:left + width] + return img[:, top : top + height, left : left + width] else: - return img[top:top + height, left:left + width, :] + return img[top : top + height, left : left + width, :] def erase(img, i, j, h, w, v, inplace=False): """Erase the pixels of selected area in input Tensor image with given value. - Args: - img (paddle.Tensor): input Tensor image. - i (int): y coordinate of the top-left point of erased region. - j (int): x coordinate of the top-left point of erased region. - h (int): Height of the erased region. - w (int): Width of the erased region. - v (paddle.Tensor): value used to replace the pixels in erased region. - inplace (bool, optional): Whether this transform is inplace. Default: False. + Args: + img (paddle.Tensor): input Tensor image. + i (int): y coordinate of the top-left point of erased region. + j (int): x coordinate of the top-left point of erased region. + h (int): Height of the erased region. + w (int): Width of the erased region. + v (paddle.Tensor): value used to replace the pixels in erased region. + inplace (bool, optional): Whether this transform is inplace. Default: False. - Returns: - paddle.Tensor: Erased image. + Returns: + paddle.Tensor: Erased image. """ _assert_image_tensor(img, 'CHW') if not inplace: img = img.clone() - img[..., i:i + h, j:j + w] = v + img[..., i : i + h, j : j + w] = v return img def center_crop(img, output_size, data_format='CHW'): """Crops the given paddle.Tensor Image and resize it to desired size. - Args: - img (paddle.Tensor): Image to be cropped. (0,0) denotes the top left corner of the image. - output_size (sequence or int): (height, width) of the crop box. If int, - it is used for both directions - data_format (str, optional): Data format of img, should be 'HWC' or - 'CHW'. Default: 'CHW'. - Returns: - paddle.Tensor: Cropped image. + Args: + img (paddle.Tensor): Image to be cropped. (0,0) denotes the top left corner of the image. + output_size (sequence or int): (height, width) of the crop box. If int, + it is used for both directions + data_format (str, optional): Data format of img, should be 'HWC' or + 'CHW'. Default: 'CHW'. + Returns: + paddle.Tensor: Cropped image. - """ + """ _assert_image_tensor(img, data_format) if isinstance(output_size, numbers.Number): @@ -570,14 +594,16 @@ def center_crop(img, output_size, data_format='CHW'): image_width, image_height = _get_image_size(img, data_format) crop_height, crop_width = output_size - crop_top = int(round((image_height - crop_height) / 2.)) - crop_left = int(round((image_width - crop_width) / 2.)) - return crop(img, - crop_top, - crop_left, - crop_height, - crop_width, - data_format=data_format) + crop_top = int(round((image_height - crop_height) / 2.0)) + crop_left = int(round((image_width - crop_width) / 2.0)) + return crop( + img, + crop_top, + crop_left, + crop_height, + crop_width, + data_format=data_format, + ) def pad(img, padding, fill=0, padding_mode='constant', data_format='CHW'): @@ -625,11 +651,16 @@ def pad(img, padding, fill=0, padding_mode='constant', data_format='CHW'): if isinstance(padding, (list, tuple)) and len(padding) not in [2, 4]: raise ValueError( - "Padding must be an int or a 2, or 4 element tuple, not a " + - "{} element tuple".format(len(padding))) + "Padding must be an int or a 2, or 4 element tuple, not a " + + "{} element tuple".format(len(padding)) + ) - assert padding_mode in ['constant', 'edge', 'reflect', 'symmetric'], \ - 'Padding mode should be either constant, edge, reflect or symmetric' + assert padding_mode in [ + 'constant', + 'edge', + 'reflect', + 'symmetric', + ], 'Padding mode should be either constant, edge, reflect or symmetric' if isinstance(padding, int): pad_left = pad_right = pad_top = pad_bottom = padding @@ -651,11 +682,13 @@ def pad(img, padding, fill=0, padding_mode='constant', data_format='CHW'): img = img.unsqueeze(0) # 'constant', 'reflect', 'replicate', 'circular' - img = F.pad(img, - pad=padding, - mode=padding_mode, - value=float(fill), - data_format='N' + data_format) + img = F.pad( + img, + pad=padding, + mode=padding_mode, + value=float(fill), + data_format='N' + data_format, + ) return img.squeeze(0) @@ -684,8 +717,10 @@ def resize(img, size, interpolation='bilinear', data_format='CHW'): """ _assert_image_tensor(img, data_format) - if not (isinstance(size, int) or - (isinstance(size, (tuple, list)) and len(size) == 2)): + if not ( + isinstance(size, int) + or (isinstance(size, (tuple, list)) and len(size) == 2) + ): raise TypeError('Got inappropriate size arg: {}'.format(size)) if isinstance(size, int): @@ -702,10 +737,12 @@ def resize(img, size, interpolation='bilinear', data_format='CHW'): oh, ow = size img = img.unsqueeze(0) - img = F.interpolate(img, - size=(oh, ow), - mode=interpolation.lower(), - data_format='N' + data_format.upper()) + img = F.interpolate( + img, + size=(oh, ow), + mode=interpolation.lower(), + data_format='N' + data_format.upper(), + ) return img.squeeze(0) @@ -725,8 +762,10 @@ def adjust_brightness(img, brightness_factor): """ _assert_image_tensor(img, 'CHW') assert brightness_factor >= 0, "brightness_factor should be non-negative." - assert _get_image_num_channels( - img, 'CHW') in [1, 3], "channels of input should be either 1 or 3." + assert _get_image_num_channels(img, 'CHW') in [ + 1, + 3, + ], "channels of input should be either 1 or 3." extreme_target = paddle.zeros_like(img, img.dtype) return _blend_images(img, extreme_target, brightness_factor) @@ -751,13 +790,13 @@ def adjust_contrast(img, contrast_factor): channels = _get_image_num_channels(img, 'CHW') dtype = img.dtype if paddle.is_floating_point(img) else paddle.float32 if channels == 1: - extreme_target = paddle.mean(img.astype(dtype), - axis=(-3, -2, -1), - keepdim=True) + extreme_target = paddle.mean( + img.astype(dtype), axis=(-3, -2, -1), keepdim=True + ) elif channels == 3: - extreme_target = paddle.mean(to_grayscale(img).astype(dtype), - axis=(-3, -2, -1), - keepdim=True) + extreme_target = paddle.mean( + to_grayscale(img).astype(dtype), axis=(-3, -2, -1), keepdim=True + ) else: raise ValueError("channels of input should be either 1 or 3.") @@ -813,7 +852,9 @@ def adjust_hue(img, hue_factor): """ _assert_image_tensor(img, 'CHW') - assert hue_factor >= -0.5 and hue_factor <= 0.5, "hue_factor should be in range [-0.5, 0.5]" + assert ( + hue_factor >= -0.5 and hue_factor <= 0.5 + ), "hue_factor should be in range [-0.5, 0.5]" channels = _get_image_num_channels(img, 'CHW') if channels == 1: return img @@ -824,7 +865,7 @@ def adjust_hue(img, hue_factor): img_hsv = _rgb_to_hsv(img) h, s, v = img_hsv.unbind(axis=-3) - h = (h + hue_factor) + h = h + hue_factor h = h - h.floor() img_adjusted = _hsv_to_rgb(paddle.stack([h, s, v], axis=-3)) diff --git a/python/paddle/vision/transforms/transforms.py b/python/paddle/vision/transforms/transforms.py index 5472df2fd9e0b..37270c97b2874 100644 --- a/python/paddle/vision/transforms/transforms.py +++ b/python/paddle/vision/transforms/transforms.py @@ -46,33 +46,38 @@ def _get_image_size(img): return img.shape[2:][::-1] # nchw -> wh else: raise ValueError( - "The dim for input Tensor should be 3-D or 4-D, but received {}" - .format(len(img.shape))) + "The dim for input Tensor should be 3-D or 4-D, but received {}".format( + len(img.shape) + ) + ) else: raise TypeError("Unexpected type {}".format(type(img))) -def _check_input(value, - name, - center=1, - bound=(0, float('inf')), - clip_first_on_zero=True): +def _check_input( + value, name, center=1, bound=(0, float('inf')), clip_first_on_zero=True +): if isinstance(value, numbers.Number): if value < 0: raise ValueError( "If {} is a single number, it must be non negative.".format( - name)) + name + ) + ) value = [center - value, center + value] if clip_first_on_zero: value[0] = max(value[0], 0) elif isinstance(value, (tuple, list)) and len(value) == 2: if not bound[0] <= value[0] <= value[1] <= bound[1]: - raise ValueError("{} values should be between {}".format( - name, bound)) + raise ValueError( + "{} values should be between {}".format(name, bound) + ) else: raise TypeError( - "{} should be a single number or a list/tuple with lenght 2.". - format(name)) + "{} should be a single number or a list/tuple with lenght 2.".format( + name + ) + ) if value[0] == value[1] == center: value = None @@ -116,8 +121,10 @@ def __call__(self, data): data = f(data) except Exception as e: stack_info = traceback.format_exc() - print("fail to perform transform [{}] with error: " - "{} and stack:\n{}".format(f, e, str(stack_info))) + print( + "fail to perform transform [{}] with error: " + "{} and stack:\n{}".format(f, e, str(stack_info)) + ) raise e return data @@ -240,14 +247,16 @@ def _apply_mask(self, mask): def __init__(self, keys=None): if keys is None: - keys = ("image", ) + keys = ("image",) elif not isinstance(keys, Sequence): raise ValueError( - "keys should be a sequence, but got keys={}".format(keys)) + "keys should be a sequence, but got keys={}".format(keys) + ) for k in keys: if self._get_apply(k) is None: raise NotImplementedError( - "{} is unsupported data structure".format(k)) + "{} is unsupported data structure".format(k) + ) self.keys = keys # storage some params get from function get_params() @@ -259,7 +268,7 @@ def _get_params(self, inputs): def __call__(self, inputs): """Apply transform on single input data""" if not isinstance(inputs, tuple): - inputs = (inputs, ) + inputs = (inputs,) self.params = self._get_params(inputs) @@ -271,7 +280,7 @@ def __call__(self, inputs): else: outputs.append(apply_func(inputs[i])) if len(inputs) > len(self.keys): - outputs.extend(inputs[len(self.keys):]) + outputs.extend(inputs[len(self.keys) :]) if len(outputs) == 1: outputs = outputs[0] @@ -413,8 +422,9 @@ class Resize(BaseTransform): def __init__(self, size, interpolation='bilinear', keys=None): super(Resize, self).__init__(keys) - assert isinstance(size, int) or (isinstance(size, Iterable) - and len(size) == 2) + assert isinstance(size, int) or ( + isinstance(size, Iterable) and len(size) == 2 + ) self.size = size self.interpolation = interpolation @@ -473,19 +483,21 @@ class RandomResizedCrop(BaseTransform): """ - def __init__(self, - size, - scale=(0.08, 1.0), - ratio=(3. / 4, 4. / 3), - interpolation='bilinear', - keys=None): + def __init__( + self, + size, + scale=(0.08, 1.0), + ratio=(3.0 / 4, 4.0 / 3), + interpolation='bilinear', + keys=None, + ): super(RandomResizedCrop, self).__init__(keys) if isinstance(size, int): self.size = (size, size) else: self.size = size - assert (scale[0] <= scale[1]), "scale should be of kind (min, max)" - assert (ratio[0] <= ratio[1]), "ratio should be of kind (min, max)" + assert scale[0] <= scale[1], "scale should be of kind (min, max)" + assert ratio[0] <= ratio[1], "ratio should be of kind (min, max)" self.scale = scale self.ratio = ratio self.interpolation = interpolation @@ -696,12 +708,9 @@ class Normalize(BaseTransform): """ - def __init__(self, - mean=0.0, - std=1.0, - data_format='CHW', - to_rgb=False, - keys=None): + def __init__( + self, mean=0.0, std=1.0, data_format='CHW', to_rgb=False, keys=None + ): super(Normalize, self).__init__(keys) if isinstance(mean, numbers.Number): mean = [mean, mean, mean] @@ -715,8 +724,9 @@ def __init__(self, self.to_rgb = to_rgb def _apply_image(self, img): - return F.normalize(img, self.mean, self.std, self.data_format, - self.to_rgb) + return F.normalize( + img, self.mean, self.std, self.data_format, self.to_rgb + ) class Transpose(BaseTransform): @@ -934,11 +944,9 @@ class HueTransform(BaseTransform): def __init__(self, value, keys=None): super(HueTransform, self).__init__(keys) - self.value = _check_input(value, - 'hue', - center=0, - bound=(-0.5, 0.5), - clip_first_on_zero=False) + self.value = _check_input( + value, 'hue', center=0, bound=(-0.5, 0.5), clip_first_on_zero=False + ) def _apply_image(self, img): if self.value is None: @@ -985,12 +993,9 @@ class ColorJitter(BaseTransform): """ - def __init__(self, - brightness=0, - contrast=0, - saturation=0, - hue=0, - keys=None): + def __init__( + self, brightness=0, contrast=0, saturation=0, hue=0, keys=None + ): super(ColorJitter, self).__init__(keys) self.brightness = brightness self.contrast = contrast @@ -1033,8 +1038,9 @@ def _apply_image(self, img): Returns: PIL Image: Color jittered image. """ - transform = self._get_param(self.brightness, self.contrast, - self.saturation, self.hue) + transform = self._get_param( + self.brightness, self.contrast, self.saturation, self.hue + ) return transform(img) @@ -1093,13 +1099,15 @@ class RandomCrop(BaseTransform): print(crop_img.shape) # [3, 224, 224] """ - def __init__(self, - size, - padding=None, - pad_if_needed=False, - fill=0, - padding_mode='constant', - keys=None): + def __init__( + self, + size, + padding=None, + pad_if_needed=False, + fill=0, + padding_mode='constant', + keys=None, + ): super(RandomCrop, self).__init__(keys) if isinstance(size, numbers.Number): self.size = (int(size), int(size)) @@ -1144,12 +1152,14 @@ def _apply_image(self, img): # pad the width if needed if self.pad_if_needed and w < self.size[1]: - img = F.pad(img, (self.size[1] - w, 0), self.fill, - self.padding_mode) + img = F.pad( + img, (self.size[1] - w, 0), self.fill, self.padding_mode + ) # pad the height if needed if self.pad_if_needed and h < self.size[0]: - img = F.pad(img, (0, self.size[0] - h), self.fill, - self.padding_mode) + img = F.pad( + img, (0, self.size[0] - h), self.fill, self.padding_mode + ) i, j, h, w = self._get_param(img, self.size) @@ -1214,8 +1224,9 @@ def __init__(self, padding, fill=0, padding_mode='constant', keys=None): if isinstance(padding, Sequence) and len(padding) not in [2, 4]: raise ValueError( - "Padding must be an int or a 2, or 4 element tuple, not a " + - "{} element tuple".format(len(padding))) + "Padding must be an int or a 2, or 4 element tuple, not a " + + "{} element tuple".format(len(padding)) + ) super(Pad, self).__init__(keys) self.padding = padding @@ -1234,19 +1245,23 @@ def _apply_image(self, img): def _check_sequence_input(x, name, req_sizes): - msg = req_sizes[0] if len(req_sizes) < 2 else " or ".join( - [str(s) for s in req_sizes]) + msg = ( + req_sizes[0] + if len(req_sizes) < 2 + else " or ".join([str(s) for s in req_sizes]) + ) if not isinstance(x, Sequence): raise TypeError(f"{name} should be a sequence of length {msg}.") if len(x) not in req_sizes: raise ValueError(f"{name} should be sequence of length {msg}.") -def _setup_angle(x, name, req_sizes=(2, )): +def _setup_angle(x, name, req_sizes=(2,)): if isinstance(x, numbers.Number): if x < 0: raise ValueError( - f"If {name} is a single number, it must be positive.") + f"If {name} is a single number, it must be positive." + ) x = [-x, x] else: _check_sequence_input(x, name, req_sizes) @@ -1312,31 +1327,34 @@ class RandomAffine(BaseTransform): print(fake_img.shape) """ - def __init__(self, - degrees, - translate=None, - scale=None, - shear=None, - interpolation='nearest', - fill=0, - center=None, - keys=None): - self.degrees = _setup_angle(degrees, name="degrees", req_sizes=(2, )) + def __init__( + self, + degrees, + translate=None, + scale=None, + shear=None, + interpolation='nearest', + fill=0, + center=None, + keys=None, + ): + self.degrees = _setup_angle(degrees, name="degrees", req_sizes=(2,)) super(RandomAffine, self).__init__(keys) assert interpolation in ['nearest', 'bilinear', 'bicubic'] self.interpolation = interpolation if translate is not None: - _check_sequence_input(translate, "translate", req_sizes=(2, )) + _check_sequence_input(translate, "translate", req_sizes=(2,)) for t in translate: if not (0.0 <= t <= 1.0): raise ValueError( - "translation values should be between 0 and 1") + "translation values should be between 0 and 1" + ) self.translate = translate if scale is not None: - _check_sequence_input(scale, "scale", req_sizes=(2, )) + _check_sequence_input(scale, "scale", req_sizes=(2,)) for s in scale: if s <= 0: raise ValueError("scale values should be positive") @@ -1354,15 +1372,12 @@ def __init__(self, self.fill = fill if center is not None: - _check_sequence_input(center, "center", req_sizes=(2, )) + _check_sequence_input(center, "center", req_sizes=(2,)) self.center = center - def _get_param(self, - img_size, - degrees, - translate=None, - scale_ranges=None, - shears=None): + def _get_param( + self, img_size, degrees, translate=None, scale_ranges=None, shears=None + ): """Get parameters for affine transformation Returns: @@ -1405,14 +1420,17 @@ def _apply_image(self, img): w, h = _get_image_size(img) img_size = [w, h] - ret = self._get_param(img_size, self.degrees, self.translate, - self.scale, self.shear) + ret = self._get_param( + img_size, self.degrees, self.translate, self.scale, self.shear + ) - return F.affine(img, - *ret, - interpolation=self.interpolation, - fill=self.fill, - center=self.center) + return F.affine( + img, + *ret, + interpolation=self.interpolation, + fill=self.fill, + center=self.center, + ) class RandomRotation(BaseTransform): @@ -1464,22 +1482,26 @@ class RandomRotation(BaseTransform): print(fake_img.size) """ - def __init__(self, - degrees, - interpolation='nearest', - expand=False, - center=None, - fill=0, - keys=None): + def __init__( + self, + degrees, + interpolation='nearest', + expand=False, + center=None, + fill=0, + keys=None, + ): if isinstance(degrees, numbers.Number): if degrees < 0: raise ValueError( - "If degrees is a single number, it must be positive.") + "If degrees is a single number, it must be positive." + ) self.degrees = (-degrees, degrees) else: if len(degrees) != 2: raise ValueError( - "If degrees is a sequence, it must be of len 2.") + "If degrees is a sequence, it must be of len 2." + ) self.degrees = degrees super(RandomRotation, self).__init__(keys) @@ -1504,8 +1526,9 @@ def _apply_image(self, img): angle = self._get_param(self.degrees) - return F.rotate(img, angle, self.interpolation, self.expand, - self.center, self.fill) + return F.rotate( + img, angle, self.interpolation, self.expand, self.center, self.fill + ) class RandomPerspective(BaseTransform): @@ -1553,15 +1576,19 @@ class RandomPerspective(BaseTransform): print(fake_img.shape) """ - def __init__(self, - prob=0.5, - distortion_scale=0.5, - interpolation='nearest', - fill=0, - keys=None): + def __init__( + self, + prob=0.5, + distortion_scale=0.5, + interpolation='nearest', + fill=0, + keys=None, + ): super(RandomPerspective, self).__init__(keys) assert 0 <= prob <= 1, "probability must be between 0 and 1" - assert 0 <= distortion_scale <= 1, "distortion_scale must be between 0 and 1" + assert ( + 0 <= distortion_scale <= 1 + ), "distortion_scale must be between 0 and 1" assert interpolation in ['nearest', 'bilinear', 'bicubic'] assert isinstance(fill, (numbers.Number, str, list, tuple)) @@ -1579,35 +1606,43 @@ def get_params(self, width, height, distortion_scale): half_height = height // 2 half_width = width // 2 topleft = [ - int(random.uniform(0, - int(distortion_scale * half_width) + 1)), - int(random.uniform(0, - int(distortion_scale * half_height) + 1)), + int(random.uniform(0, int(distortion_scale * half_width) + 1)), + int(random.uniform(0, int(distortion_scale * half_height) + 1)), ] topright = [ int( - random.uniform(width - int(distortion_scale * half_width) - 1, - width)), - int(random.uniform(0, - int(distortion_scale * half_height) + 1)), + random.uniform( + width - int(distortion_scale * half_width) - 1, width + ) + ), + int(random.uniform(0, int(distortion_scale * half_height) + 1)), ] botright = [ int( - random.uniform(width - int(distortion_scale * half_width) - 1, - width)), + random.uniform( + width - int(distortion_scale * half_width) - 1, width + ) + ), int( - random.uniform(height - int(distortion_scale * half_height) - 1, - height)), + random.uniform( + height - int(distortion_scale * half_height) - 1, height + ) + ), ] botleft = [ - int(random.uniform(0, - int(distortion_scale * half_width) + 1)), + int(random.uniform(0, int(distortion_scale * half_width) + 1)), int( - random.uniform(height - int(distortion_scale * half_height) - 1, - height)), + random.uniform( + height - int(distortion_scale * half_height) - 1, height + ) + ), + ] + startpoints = [ + [0, 0], + [width - 1, 0], + [width - 1, height - 1], + [0, height - 1], ] - startpoints = [[0, 0], [width - 1, 0], [width - 1, height - 1], - [0, height - 1]] endpoints = [topleft, topright, botright, botleft] return startpoints, endpoints @@ -1624,10 +1659,12 @@ def _apply_image(self, img): width, height = _get_image_size(img) if random.random() < self.prob: - startpoints, endpoints = self.get_params(width, height, - self.distortion_scale) - return F.perspective(img, startpoints, endpoints, - self.interpolation, self.fill) + startpoints, endpoints = self.get_params( + width, height, self.distortion_scale + ) + return F.perspective( + img, startpoints, endpoints, self.interpolation, self.fill + ) return img @@ -1715,27 +1752,34 @@ class RandomErasing(BaseTransform): print(result) """ - def __init__(self, - prob=0.5, - scale=(0.02, 0.33), - ratio=(0.3, 3.3), - value=0, - inplace=False, - keys=None): + def __init__( + self, + prob=0.5, + scale=(0.02, 0.33), + ratio=(0.3, 3.3), + value=0, + inplace=False, + keys=None, + ): super(RandomErasing, self).__init__(keys) - assert isinstance(scale, - (tuple, list)), "scale should be a tuple or list" - assert (scale[0] >= 0 and scale[1] <= 1 and scale[0] <= scale[1] - ), "scale should be of kind (min, max) and in range [0, 1]" - assert isinstance(ratio, - (tuple, list)), "ratio should be a tuple or list" - assert (ratio[0] >= 0 - and ratio[0] <= ratio[1]), "ratio should be of kind (min, max)" - assert (prob >= 0 - and prob <= 1), "The probability should be in range [0, 1]" assert isinstance( - value, (numbers.Number, str, tuple, - list)), "value should be a number, tuple, list or str" + scale, (tuple, list) + ), "scale should be a tuple or list" + assert ( + scale[0] >= 0 and scale[1] <= 1 and scale[0] <= scale[1] + ), "scale should be of kind (min, max) and in range [0, 1]" + assert isinstance( + ratio, (tuple, list) + ), "ratio should be a tuple or list" + assert ( + ratio[0] >= 0 and ratio[0] <= ratio[1] + ), "ratio should be of kind (min, max)" + assert ( + prob >= 0 and prob <= 1 + ), "The probability should be in range [0, 1]" + assert isinstance( + value, (numbers.Number, str, tuple, list) + ), "value should be a number, tuple, list or str" if isinstance(value, str) and value != "random": raise ValueError("value must be 'random' when type is str") @@ -1779,7 +1823,8 @@ def _get_param(self, img, scale, ratio, value): if F._is_tensor_image(img): if value is None: v = paddle.normal(shape=[c, erase_h, erase_w]).astype( - img.dtype) + img.dtype + ) else: v = paddle.to_tensor(value, dtype=img.dtype)[:, None, None] else: @@ -1815,6 +1860,7 @@ def _apply_image(self, img): "Value should be a single number or a sequence with length equals to image's channel." ) top, left, erase_h, erase_w, v = self._get_param( - img, self.scale, self.ratio, value) + img, self.scale, self.ratio, value + ) return F.erase(img, top, left, erase_h, erase_w, v, self.inplace) return img diff --git a/r/example/mobilenet.py b/r/example/mobilenet.py index 96deb0aaf5fd0..082a1dea7deac 100755 --- a/r/example/mobilenet.py +++ b/r/example/mobilenet.py @@ -51,7 +51,7 @@ def set_config(): def parse_data(): - """ parse input and output data """ + """parse input and output data""" with open('data/data.txt', 'r') as fr: data = np.array([float(_) for _ in fr.read().split()]) diff --git a/tools/CrossStackProfiler/CspChromeTraceFormatter.py b/tools/CrossStackProfiler/CspChromeTraceFormatter.py index ec5dba54b002f..1fa8efe988050 100755 --- a/tools/CrossStackProfiler/CspChromeTraceFormatter.py +++ b/tools/CrossStackProfiler/CspChromeTraceFormatter.py @@ -16,7 +16,6 @@ class ChromeTraceFormatter(object): - def __init__(self): self._events = [] self._metadata = [] diff --git a/tools/CrossStackProfiler/CspFileReader.py b/tools/CrossStackProfiler/CspFileReader.py index 0825842fa1a11..11dd052283c15 100755 --- a/tools/CrossStackProfiler/CspFileReader.py +++ b/tools/CrossStackProfiler/CspFileReader.py @@ -18,6 +18,7 @@ import glob import logging from multiprocessing import Lock + """ Some terms to clarify the code in most case, one or more paremeters may be set as input args for a class or a function in form of single variable or k-v dict @@ -42,10 +43,17 @@ "02_gpuUtility": [("GPUTL", "GPUTL"), ("GRACT", "GRACT")], "03_smUtility": [("SMACT", "SMACT"), ("SMOCC", "SMOCC")], "04_memUtility": [("FB_USED_RATIO", "FB_USED_RATIO"), ("DRAMA", "DRAMA")], - "05_txUtility": [("NVLTX", "NVLTX"), ("NVLRX", "NVLRX"), ("PCITX", "PCITX"), - ("PCIRX", "PCIRX")], - "06_calUtility": [("FP32A", "FP32A"), ("FP16A", "FP16A"), - ("TENSO", "TENSO")] + "05_txUtility": [ + ("NVLTX", "NVLTX"), + ("NVLRX", "NVLRX"), + ("PCITX", "PCITX"), + ("PCIRX", "PCIRX"), + ], + "06_calUtility": [ + ("FP32A", "FP32A"), + ("FP16A", "FP16A"), + ("TENSO", "TENSO"), + ], } DCGMINFO_TRACE_NUM = len(dcgmMetricParameterMap.keys()) NETINFO_TRACE_NUM = 2 @@ -59,13 +67,13 @@ FILEORGANIZEFORM_BYTRAINER = "byTrainer" FILEORGANIZEFORM_BYOTHER = "other" FILEORGANIZEFORM = [ - FILEORGANIZEFORM_BYRANK, FILEORGANIZEFORM_BYTRAINER, - FILEORGANIZEFORM_BYOTHER + FILEORGANIZEFORM_BYRANK, + FILEORGANIZEFORM_BYTRAINER, + FILEORGANIZEFORM_BYOTHER, ] class FileReader(object): - def __init__(self, logger, args): self._logger = logger self._args = args @@ -104,8 +112,9 @@ def _checkArgsKey(self, key, type): if not isinstance(self._args[key], type): raise TypeError( - "Invalid type of key [%s] in args dict, it should be a %s!" % - (key, type)) + "Invalid type of key [%s] in args dict, it should be a %s!" + % (key, type) + ) exec("self._%s = self._args[\"%s\"]" % (key, key)) @@ -117,18 +126,22 @@ def _checkArgs(self): raise TypeError("Invalid type of args, it should be a dict!") self._checkArgsKey("organizeForm", str) - if self._organizeForm not in FILEORGANIZEFORM or \ - self._organizeForm == FILEORGANIZEFORM_BYOTHER: + if ( + self._organizeForm not in FILEORGANIZEFORM + or self._organizeForm == FILEORGANIZEFORM_BYOTHER + ): raise NotImplementedError( - "we have not known how to process this form of file [%s]!" % - self._organizeForm) + "we have not known how to process this form of file [%s]!" + % self._organizeForm + ) self._checkArgsKey("gpuPerTrainer", int) self._checkArgsKey("dataPath", str) if not os.path.exists(self._dataPath): - raise IOError("input data path [%s] not existed!" % - (self._dataPath)) + raise IOError( + "input data path [%s] not existed!" % (self._dataPath) + ) self._checkArgsKey("groupSize", int) self._checkArgsKey("displaySize", int) @@ -155,7 +168,8 @@ def getFileList(self): def _cmp(self, x, y): return self._getId(x, self._organizeForm) - self._getId( - y, self._organizeForm) + y, self._organizeForm + ) def _getFileList(self): self._fileList = glob.glob(os.path.join(self._dataPath, "*.*")) @@ -170,12 +184,14 @@ def _getFileList(self): newFileList.append(file) else: raise NotImplementedError( - "[%s] is repeated by id, we don not how to process it!" % - file) + "[%s] is repeated by id, we don not how to process it!" + % file + ) if not self._fileList: - if (self._getId(self._fileList[-1]) - - self._getId(self._fileList[0])) != len(self._fileList) - 1: + if ( + self._getId(self._fileList[-1]) - self._getId(self._fileList[0]) + ) != len(self._fileList) - 1: raise Exception("The file id should be countious!") # sort @@ -185,19 +201,23 @@ def _sortBySuffix(elem): self._fileList.sort(key=_sortBySuffix) if not self._fileList: - self._logger.warning("we can not find any file in dir [%s]!" % - self._dataPath) + self._logger.warning( + "we can not find any file in dir [%s]!" % self._dataPath + ) else: - self._logger.info("file list in dir [%s] is : %s !" % - (self._dataPath, ', '.join(self._fileList))) + self._logger.info( + "file list in dir [%s] is : %s !" + % (self._dataPath, ', '.join(self._fileList)) + ) return self._fileList def _getId(self, fileName, organizeForm, sed="."): if self._organizeForm != organizeForm: raise TypeError( - "Can not get rank id when organizer form is not %s!" % - organizeForm) + "Can not get rank id when organizer form is not %s!" + % organizeForm + ) if not os.path.isfile(fileName): raise IOError("[%s] is not a valid file!" % (fileName)) @@ -213,8 +233,9 @@ def _getId(self, fileName, organizeForm, sed="."): except IndexError as e: print(e) raise TypeError( - "invalid fileName [%s], the prefix should be a number!" % - fileName) + "invalid fileName [%s], the prefix should be a number!" + % fileName + ) def getRankId(self, fileName, sed="."): return self._getId(fileName, FILEORGANIZEFORM_BYRANK, sed) @@ -251,7 +272,7 @@ def _splitTaskListForMultiProcess(self, ls, n): end = 0 for i in range(0, (n) * j, j): if i < len(ls) and (i + j) < len(ls): - ls_return.append(ls[i:i + j]) + ls_return.append(ls[i : i + j]) end = i + j ls_return.append(ls[end:]) return ls_return @@ -284,40 +305,29 @@ def getDict(self, name, groupId, gpuId, tmpPath="./tmp"): try: data = json.load(rf) except Exception: - self._logger.error("read [%s] error. not a json file!" % - (fileName)) - raise TypeError("read [%s] error. not a json file!" % - (fileName)) + self._logger.error( + "read [%s] error. not a json file!" % (fileName) + ) + raise TypeError( + "read [%s] error. not a json file!" % (fileName) + ) return data - def dumpOpInfoDict(self, - data, - groupId, - gpuId, - pretty=False, - tmpPath="./tmp"): - return self.dumpDict(data, - "opinfo", - groupId, - gpuId, - pretty=False, - tmpPath="./tmp") + def dumpOpInfoDict( + self, data, groupId, gpuId, pretty=False, tmpPath="./tmp" + ): + return self.dumpDict( + data, "opinfo", groupId, gpuId, pretty=False, tmpPath="./tmp" + ) def dumpDCGMDict(self, data, groupId, gpuId, pretty=False, tmpPath="./tmp"): - return self.dumpDict(data, - "dcgm", - groupId, - gpuId, - pretty=False, - tmpPath="./tmp") - - def dumpDict(self, - data, - name, - groupId, - gpuId, - pretty=False, - tmpPath="./tmp"): + return self.dumpDict( + data, "dcgm", groupId, gpuId, pretty=False, tmpPath="./tmp" + ) + + def dumpDict( + self, data, name, groupId, gpuId, pretty=False, tmpPath="./tmp" + ): self._lock.acquire() if not os.path.exists(tmpPath): os.makedirs(tmpPath) diff --git a/tools/CrossStackProfiler/CspReporter.py b/tools/CrossStackProfiler/CspReporter.py index 2e1e1f2c6380c..052ffd6fca19f 100755 --- a/tools/CrossStackProfiler/CspReporter.py +++ b/tools/CrossStackProfiler/CspReporter.py @@ -28,41 +28,43 @@ def get_argparse(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--profile_path', - type=str, - default='.', - help='Working path that store the monitor data.') - - parser.add_argument('--timeline_path', - type=str, - default='.', - help='Output timeline file name.') - - parser.add_argument('--gpuPerTrainer', - type=int, - default=8, - help='Gpus per trainer.') - - parser.add_argument('--trainerNum', - type=int, - default=4, - help='Num of trainer.') - - parser.add_argument('--groupSize', - type=int, - default=8, - help='Num of trainer in a group.') - - parser.add_argument('--displaySize', - type=int, - default=2, - help='Num of line need to display in a group.') + parser.add_argument( + '--profile_path', + type=str, + default='.', + help='Working path that store the monitor data.', + ) + + parser.add_argument( + '--timeline_path', + type=str, + default='.', + help='Output timeline file name.', + ) + + parser.add_argument( + '--gpuPerTrainer', type=int, default=8, help='Gpus per trainer.' + ) + + parser.add_argument( + '--trainerNum', type=int, default=4, help='Num of trainer.' + ) + + parser.add_argument( + '--groupSize', type=int, default=8, help='Num of trainer in a group.' + ) + + parser.add_argument( + '--displaySize', + type=int, + default=2, + help='Num of line need to display in a group.', + ) return parser.parse_args() class CspReporter(object): - def __init__(self, args): self._args = args print(self._args) @@ -83,8 +85,9 @@ def __init__(self, args): def _checkArgs(self): if self._trainerNum % self._groupSize != 0: raise Exception( - "Input args error: trainerNum[%d] %% groupSize[%d] != 0" % - (self._trainerNum, self._groupSize)) + "Input args error: trainerNum[%d] %% groupSize[%d] != 0" + % (self._trainerNum, self._groupSize) + ) def _init_logger(self): self._logger = getLogger() @@ -121,10 +124,12 @@ def _init_reader(self): "organizeForm": FILEORGANIZEFORM_BYRANK, } - self._dcgmFileReader = dcgmFileReader(self._logger, - self._dcgmFileReaderArgs) - self._profileFileReader = profileFileReader(self._logger, - self._profileFileReaderArgs) + self._dcgmFileReader = dcgmFileReader( + self._logger, self._dcgmFileReaderArgs + ) + self._profileFileReader = profileFileReader( + self._logger, self._profileFileReaderArgs + ) def _init_timeInfo(self): self._timePath = os.path.join(self._workPath, TIME_PATH) @@ -134,7 +139,8 @@ def _init_timeInfo(self): def _set_timeInfo(self, timeFileNamePrefix="time.txt", sed="."): timeFileNameList = glob.glob( - os.path.join(self._timePath, timeFileNamePrefix, sed, "*")) + os.path.join(self._timePath, timeFileNamePrefix, sed, "*") + ) for timeFileName in timeFileNameList: trainerId = int(timeFileName.split(sed)[-1]) gpuId = int(timeFileName.split(sed)[-2]) @@ -143,33 +149,41 @@ def _set_timeInfo(self, timeFileNamePrefix="time.txt", sed="."): for line in rf: if line.startswith("start time:"): info["start_time"] = int( - float(line.split(":")[-1]) * 1e9) + float(line.split(":")[-1]) * 1e9 + ) - self._minTimeStamp = min(self._minTimeStamp, - info["start_time"]) + self._minTimeStamp = min( + self._minTimeStamp, info["start_time"] + ) if line.startswith("end time:"): info["end_time"] = int(float(line.split(":")[-1]) * 1e9) if not info: self._timeInfo[gpuId * trainerId] = info - def _generateTraceFileByGroupAndGpuId(self, pipileInfo, netInfo, groupId, - gpuId): + def _generateTraceFileByGroupAndGpuId( + self, pipileInfo, netInfo, groupId, gpuId + ): dcgmInfoDict = self._dcgmFileReader.getDcgmInfoDict(groupId, gpuId) opInfoDict = self._profileFileReader.getOpInfoDict(groupId, gpuId) traceObj = {} - traceObj["traceEvents"] = pipileInfo[str( - gpuId)] + opInfoDict["traceEvents"] + dcgmInfoDict[ - "traceEvents"] + netInfo["traceEvents"] + traceObj["traceEvents"] = ( + pipileInfo[str(gpuId)] + + opInfoDict["traceEvents"] + + dcgmInfoDict["traceEvents"] + + netInfo["traceEvents"] + ) - self._profileFileReader.dumpDict(traceObj, "traceFile", groupId, gpuId, - False, self._saveFilePath) + self._profileFileReader.dumpDict( + traceObj, "traceFile", groupId, gpuId, False, self._saveFilePath + ) def _generateTraceFileByGroup(self, groupId, processNum): # first we need to generate pipeline info pipileInfo = self._profileFileReader.getPipeLineInfo( - groupId, processNum) + groupId, processNum + ) # second we need to generate dcgm info dcgmInfo = self._dcgmFileReader.getDCGMTraceInfo(groupId, processNum) @@ -186,48 +200,56 @@ def _generateTraceFileByGroup(self, groupId, processNum): pidList = [] for gpuId in range(self._gpuPerTrainer): - subproc = Process(target=self._generateTraceFileByGroupAndGpuId, - args=( - pipileInfo, - netInfo, - groupId, - gpuId, - )) + subproc = Process( + target=self._generateTraceFileByGroupAndGpuId, + args=( + pipileInfo, + netInfo, + groupId, + gpuId, + ), + ) processPool.append(subproc) subproc.start() pidList.append(subproc.pid) self._logger.info( "[traceFile]: process [%d] has been started, total task num is %d ..." - % (subproc.pid, 1)) + % (subproc.pid, 1) + ) for t in processPool: t.join() pidList.remove(t.pid) self._logger.info( - "[traceFile]: process [%d] has exited! remained %d process!" % - (t.pid, len(pidList))) + "[traceFile]: process [%d] has exited! remained %d process!" + % (t.pid, len(pidList)) + ) def generateTraceFile(self, processNum=8): processPool = [] pidList = [] for groupId in range(self._trainerNum / self._groupSize): - subproc = Process(target=self._generateTraceFileByGroup, - args=( - groupId, - processNum, - )) + subproc = Process( + target=self._generateTraceFileByGroup, + args=( + groupId, + processNum, + ), + ) processPool.append(subproc) subproc.start() pidList.append(subproc.pid) self._logger.info( "[GroupTraceFile]: process [%d] has been started, total task num is %d ..." - % (subproc.pid, 1)) + % (subproc.pid, 1) + ) for t in processPool: t.join() pidList.remove(t.pid) self._logger.info( "[GroupTraceFile]: process [%d] has exited! remained %d process!" - % (t.pid, len(pidList))) + % (t.pid, len(pidList)) + ) if __name__ == '__main__': diff --git a/tools/CrossStackProfiler/DCGMFileReader.py b/tools/CrossStackProfiler/DCGMFileReader.py index 1285ce388c150..86868c8734e3f 100755 --- a/tools/CrossStackProfiler/DCGMFileReader.py +++ b/tools/CrossStackProfiler/DCGMFileReader.py @@ -28,7 +28,6 @@ class dcgmFileReader(FileReader): - def parseFileByGroup(self, groupId, processNum=8): fileFist = self.getFileListByGroup(groupId) displaySize = min(self._displaySize, len(fileFist)) @@ -38,8 +37,9 @@ def parseFileByGroup(self, groupId, processNum=8): return self._parseTask(fileFist) else: - self._logger.info("using [%d] process to do this work!" % - processNum) + self._logger.info( + "using [%d] process to do this work!" % processNum + ) processPool = [] pidList = [] @@ -48,23 +48,28 @@ def parseFileByGroup(self, groupId, processNum=8): taskList = self._splitTaskListForMultiProcess(fileFist, processNum) for task in taskList: - subproc = Process(target=self._parseTask, args=( - task, - q, - )) + subproc = Process( + target=self._parseTask, + args=( + task, + q, + ), + ) processPool.append(subproc) subproc.start() pidList.append(subproc.pid) self._logger.info( "[DCGM reader]: process [%d] has been started, total task num is %d ..." - % (subproc.pid, len(processPool))) + % (subproc.pid, len(processPool)) + ) for t in processPool: t.join() pidList.remove(t.pid) self._logger.info( "[DCGM reader]: process [%d] has exited! remained %d process!" - % (t.pid, len(pidList))) + % (t.pid, len(pidList)) + ) isFistProcess = True for t in processPool: @@ -72,9 +77,9 @@ def parseFileByGroup(self, groupId, processNum=8): isFistProcess = False dcgm_data = q.get() else: - dcgm_data = pd.concat([dcgm_data, q.get()], - axis=0, - join='outer') + dcgm_data = pd.concat( + [dcgm_data, q.get()], axis=0, join='outer' + ) return dcgm_data @@ -90,9 +95,9 @@ def _parseTask(self, taskList, q=None): is_first = False dcgm_data = tmp_data else: - dcgm_data = pd.concat([dcgm_data, tmp_data], - axis=0, - join='outer') + dcgm_data = pd.concat( + [dcgm_data, tmp_data], axis=0, join='outer' + ) dcgm_data = dcgm_data.dropna() if q is not None: q.put(dcgm_data) @@ -117,18 +122,23 @@ def _parseSingleFile(self, fileName): for line in fp: # skip `nvidia-dcgm-dmon.sh` init and fini info lines - if 'nv-hostengine' in line or 'dmon' in line or 'Host Engine Listener Started' in line: + if ( + 'nv-hostengine' in line + or 'dmon' in line + or 'Host Engine Listener Started' in line + ): continue if not line.strip().startswith( - "GPU") and not line.strip().startswith("# Entity"): + "GPU" + ) and not line.strip().startswith("# Entity"): continue # skip non-needed headers (only the header in 1th line was needed) if line.strip().startswith("# Entity"): line = line.strip()[2:] - if 'Entity' == line[0:len('Entity')]: + if 'Entity' == line[0 : len('Entity')]: if has_header: continue else: @@ -153,15 +163,13 @@ def _parseSingleFile(self, fileName): return dcgm - def _getDCGMTraceInfoByGpuId(self, - groupId, - gpuId, - dcgm_data, - pid_map, - q=None): + def _getDCGMTraceInfoByGpuId( + self, groupId, gpuId, dcgm_data, pid_map, q=None + ): self._logger.info( - "Begin to generate dcgm info, groupId = %d, gpuID = %d ..." % - (groupId, gpuId)) + "Begin to generate dcgm info, groupId = %d, gpuID = %d ..." + % (groupId, gpuId) + ) gpuDcgmData = dcgm_data[dcgm_data['Entity'].isin([gpuId])] @@ -220,27 +228,31 @@ def getDCGMTraceInfo(self, groupId, processNum=8): pidList = [] for gpuId in range(self._gpuPerTrainer): - subproc = Process(target=self._getDCGMTraceInfoByGpuId, - args=( - groupId, - gpuId, - dcgm_data, - pid_map, - q, - )) + subproc = Process( + target=self._getDCGMTraceInfoByGpuId, + args=( + groupId, + gpuId, + dcgm_data, + pid_map, + q, + ), + ) processPool.append(subproc) subproc.start() pidList.append(subproc.pid) self._logger.info( "[DCGM info]: process [%d] has been started, total task num is %d ..." - % (subproc.pid, 1)) + % (subproc.pid, 1) + ) for t in processPool: t.join() pidList.remove(t.pid) self._logger.info( - "[DCGM info]: process [%d] has exited! remained %d process!" % - (t.pid, len(pidList))) + "[DCGM info]: process [%d] has exited! remained %d process!" + % (t.pid, len(pidList)) + ) dcgmInfo = {} diff --git a/tools/CrossStackProfiler/NetFileReader.py b/tools/CrossStackProfiler/NetFileReader.py index 199e3ba8d4d87..ed9cfdb1fc86f 100755 --- a/tools/CrossStackProfiler/NetFileReader.py +++ b/tools/CrossStackProfiler/NetFileReader.py @@ -24,7 +24,6 @@ class netFileReader(FileReader): - def _parseSingleFile(self, fileNameList, tx_pid, rx_pid, q=None): traceInfo = {} @@ -53,8 +52,9 @@ def _parseSingleFile(self, fileNameList, tx_pid, rx_pid, q=None): for line in rf: try: event_str = json.loads(line.strip()) - event_str["pid"] = tx_pid if event_str[ - "name"] == "tx" else rx_pid + event_str["pid"] = ( + tx_pid if event_str["name"] == "tx" else rx_pid + ) # the unit of net is ms, we need ns event_str["ts"] = self._align_ts(event_str["ts"] * 1e6) event_str["id"] = trainerId @@ -62,8 +62,9 @@ def _parseSingleFile(self, fileNameList, tx_pid, rx_pid, q=None): except Exception: self._logger.warning( - "invalid record [%s] in [%s]. skip it!" % - (line[:-1], fileName)) + "invalid record [%s] in [%s]. skip it!" + % (line[:-1], fileName) + ) traceInfo["traceEvents"] = traceEventList if q is not None: @@ -73,7 +74,7 @@ def _parseSingleFile(self, fileNameList, tx_pid, rx_pid, q=None): def parseFileByGroup(self, groupId, processNum=8): fileFist = self.getFileListByGroup(groupId) - fileFist = fileFist[:min(self._displaySize, len(fileFist))] + fileFist = fileFist[: min(self._displaySize, len(fileFist))] manager = multiprocessing.Manager() q = manager.Queue() @@ -85,26 +86,30 @@ def parseFileByGroup(self, groupId, processNum=8): taskList = self._splitTaskListForMultiProcess(fileFist, processNum) for task in taskList: - subproc = Process(target=self._parseSingleFile, - args=( - task, - tx_pid, - rx_pid, - q, - )) + subproc = Process( + target=self._parseSingleFile, + args=( + task, + tx_pid, + rx_pid, + q, + ), + ) processPool.append(subproc) subproc.start() pidList.append(subproc.pid) self._logger.info( "[Net info]: process [%d] has been started, total task num is %d ..." - % (subproc.pid, len(processPool))) + % (subproc.pid, len(processPool)) + ) for t in processPool: t.join() pidList.remove(t.pid) self._logger.info( - "[Net info]: process [%d] has exited! remained %d process!" % - (t.pid, len(pidList))) + "[Net info]: process [%d] has exited! remained %d process!" + % (t.pid, len(pidList)) + ) traceInfo = {} isFistProcess = True diff --git a/tools/CrossStackProfiler/ProfileFileReader.py b/tools/CrossStackProfiler/ProfileFileReader.py index 282c2f1f25431..e7a07c40820af 100755 --- a/tools/CrossStackProfiler/ProfileFileReader.py +++ b/tools/CrossStackProfiler/ProfileFileReader.py @@ -22,12 +22,15 @@ from CspFileReader import FileReader from CspFileReader import getLogger -from CspFileReader import NETINFO_TRACE_NUM, DCGMINFO_TRACE_NUM, PIPELINEINFO_TRACE_NUM +from CspFileReader import ( + NETINFO_TRACE_NUM, + DCGMINFO_TRACE_NUM, + PIPELINEINFO_TRACE_NUM, +) from CspFileReader import FILEORGANIZEFORM_BYRANK class profileFileReader(FileReader): - def _parseSingleFile(self, profile): with open(profile, 'rb') as f: profile_s = f.read() @@ -41,8 +44,9 @@ def _parseTask(self, taskList, q=None): for fileName in taskList: rankId = self.getRankId(fileName) - profile_dict["trainerRank.%03d" % - (rankId)] = self._parseSingleFile(fileName) + profile_dict["trainerRank.%03d" % (rankId)] = self._parseSingleFile( + fileName + ) self._logger.info("I finish processing %s!" % fileName) if q is not None: @@ -56,15 +60,16 @@ def _is_forwardBackwardInfo(self, items): if isinstance(items["args"], dict): args = items["args"] if "detail_info" in args: - if args["detail_info"] == "marker_forward_B" or \ - args["detail_info"] == "marker_forward_E" or \ - args["detail_info"] == "marker_backward_B" or \ - args["detail_info"] == "marker_backward_E": + if ( + args["detail_info"] == "marker_forward_B" + or args["detail_info"] == "marker_forward_E" + or args["detail_info"] == "marker_backward_B" + or args["detail_info"] == "marker_backward_E" + ): return True return False def _allocate_forwardBackwardInfo(self, restList, pid, tid): - def _cmp_ele(items): return items["ts"] @@ -79,7 +84,7 @@ def _cmp_ele(items): else: lastEle["dur"] = items["ts"] - lastEle["ts"] name = lastEle["args"]["detail_info"] - name = name[:name.rfind('_')] + name = name[: name.rfind('_')] name = name.split('_')[1] lastEle["name"] = name lastEle["args"]["detail_info"] = name @@ -130,7 +135,8 @@ def _getPipeLineInfo(self, profileList, q=None): traceEventList.append(traceEvent) pipeLineList = self._allocate_forwardBackwardInfo( - traceEventList, pid, tid) + traceEventList, pid, tid + ) res[str(rankId)] = pipeLineList @@ -143,8 +149,9 @@ def getPipeLineInfo(self, groupId, processNum=8): fileFist = self.getFileListByGroup(groupId) self._logger.info( - "using [%d] process to do this work, total task num is %d!" % - (processNum, len(fileFist))) + "using [%d] process to do this work, total task num is %d!" + % (processNum, len(fileFist)) + ) processPool = [] pidList = [] @@ -153,23 +160,28 @@ def getPipeLineInfo(self, groupId, processNum=8): taskList = self._splitTaskListForMultiProcess(fileFist, processNum) for task in taskList: - subproc = Process(target=self._getPipeLineInfo, args=( - task, - q, - )) + subproc = Process( + target=self._getPipeLineInfo, + args=( + task, + q, + ), + ) processPool.append(subproc) subproc.start() pidList.append(subproc.pid) self._logger.info( "[pipeline info]: process [%d] has been started, total task num is %d ..." - % (subproc.pid, len(task))) + % (subproc.pid, len(task)) + ) for t in processPool: t.join() pidList.remove(t.pid) self._logger.info( "[pipeline info]: process [%d] has exited! remained %d process!" - % (t.pid, len(pidList))) + % (t.pid, len(pidList)) + ) pipeLineInfo = {} @@ -210,12 +222,15 @@ def _allocate_pids(self, profile_dict, gpuId, initPid): # -1 device id represents CUDA API(RunTime) call.(e.g. cudaLaunch, cudaMemcpy) if event.device_id == -1: chrome_trace.emit_pid( - "%02d_%s:cuda_api" % (lineNum, k), pid) + "%02d_%s:cuda_api" % (lineNum, k), pid + ) lineNum = lineNum + 1 else: chrome_trace.emit_pid( - "%02d_%s:cpu:block:%d" % - (lineNum, k, event.device_id), pid) + "%02d_%s:cpu:block:%d" + % (lineNum, k, event.device_id), + pid, + ) lineNum = lineNum + 1 elif event.type == profiler_pb2.Event.GPUKernel: if (k, event.device_id, "GPUKernel") not in devices: @@ -225,8 +240,10 @@ def _allocate_pids(self, profile_dict, gpuId, initPid): devices[(k, event.device_id, "GPUKernel")] = pid chrome_trace.emit_pid( - "%02d_%s:gpu:%d" % - (lineNum, k, event.device_id), pid) + "%02d_%s:gpu:%d" + % (lineNum, k, event.device_id), + pid, + ) lineNum = lineNum + 1 if not hasattr(profile_pb, "mem_events"): @@ -240,8 +257,10 @@ def _allocate_pids(self, profile_dict, gpuId, initPid): mem_devices[(k, mevent.device_id, "GPU")] = pid chrome_trace.emit_pid( - "%02d_memory usage on %s:gpu:%d" % - (lineNum, k, mevent.device_id), pid) + "%02d_memory usage on %s:gpu:%d" + % (lineNum, k, mevent.device_id), + pid, + ) lineNum = lineNum + 1 elif mevent.place == profiler_pb2.MemEvent.CPUPlace: if (k, mevent.device_id, "CPU") not in mem_devices: @@ -250,21 +269,29 @@ def _allocate_pids(self, profile_dict, gpuId, initPid): mem_devices[(k, mevent.device_id, "CPU")] = pid chrome_trace.emit_pid( - "%02d_memory usage on %s:cpu:%d" % - (lineNum, k, mevent.device_id), pid) + "%02d_memory usage on %s:cpu:%d" + % (lineNum, k, mevent.device_id), + pid, + ) lineNum = lineNum + 1 elif mevent.place == profiler_pb2.MemEvent.CUDAPinnedPlace: - if (k, mevent.device_id, - "CUDAPinnedPlace") not in mem_devices: + if ( + k, + mevent.device_id, + "CUDAPinnedPlace", + ) not in mem_devices: if gpuId == mevent.device_id: pid = initPid initPid = initPid + 1 - mem_devices[(k, mevent.device_id, - "CUDAPinnedPlace")] = pid + mem_devices[ + (k, mevent.device_id, "CUDAPinnedPlace") + ] = pid chrome_trace.emit_pid( - "%02d_memory usage on %s:cudapinnedplace:%d" % - (lineNum, k, mevent.device_id), pid) + "%02d_memory usage on %s:cudapinnedplace:%d" + % (lineNum, k, mevent.device_id), + pid, + ) lineNum = lineNum + 1 if (k, 0, "CPU") not in mem_devices: pid = initPid @@ -272,7 +299,8 @@ def _allocate_pids(self, profile_dict, gpuId, initPid): mem_devices[(k, 0, "CPU")] = pid chrome_trace.emit_pid( - "%02d_memory usage on %s:cpu:%d" % (lineNum, k, 0), pid) + "%02d_memory usage on %s:cpu:%d" % (lineNum, k, 0), pid + ) lineNum = lineNum + 1 if (k, 0, "GPU") not in mem_devices: # if gpuId == mevent.device_id: @@ -281,7 +309,8 @@ def _allocate_pids(self, profile_dict, gpuId, initPid): mem_devices[(k, 0, "GPU")] = pid chrome_trace.emit_pid( - "%02d_memory usage on %s:gpu:%d" % (lineNum, k, 0), pid) + "%02d_memory usage on %s:gpu:%d" % (lineNum, k, 0), pid + ) lineNum = lineNum + 1 if (k, 0, "CUDAPinnedPlace") not in mem_devices: pid = initPid @@ -289,8 +318,10 @@ def _allocate_pids(self, profile_dict, gpuId, initPid): mem_devices[(k, 0, "CUDAPinnedPlace")] = pid chrome_trace.emit_pid( - "%02d_memory usage on %s:cudapinnedplace:%d" % - (lineNum, k, 0), pid) + "%02d_memory usage on %s:cudapinnedplace:%d" + % (lineNum, k, 0), + pid, + ) lineNum = lineNum + 1 i = i + 1 return chrome_trace, devices, mem_devices @@ -307,7 +338,11 @@ def _allocate_events(self, profile_dict, devices, gpuId): elif event.type == profiler_pb2.Event.GPUKernel: type = "GPUKernel" - if event.type == profiler_pb2.Event.GPUKernel and event.device_id != gpuId and rankId % self._gpuPerTrainer != gpuId: + if ( + event.type == profiler_pb2.Event.GPUKernel + and event.device_id != gpuId + and rankId % self._gpuPerTrainer != gpuId + ): continue pid = devices[(k, event.device_id, type)] @@ -318,10 +353,15 @@ def _allocate_events(self, profile_dict, devices, gpuId): args['detail_info'] = event.detail_info # TODO(panyx0718): Chrome tracing only handles ms. However, some # ops takes micro-seconds. Hence, we keep the ns here. - chrome_trace.emit_region(self._align_ts(event.start_ns), - (event.end_ns - event.start_ns) / 1.0, - pid, event.sub_device_id, 'Op', - event.name, args) + chrome_trace.emit_region( + self._align_ts(event.start_ns), + (event.end_ns - event.start_ns) / 1.0, + pid, + event.sub_device_id, + 'Op', + event.name, + args, + ) return chrome_trace def _allocate_memory_event(self, profile_dict, mem_devices, gpuId): @@ -331,7 +371,7 @@ def _allocate_memory_event(self, profile_dict, mem_devices, gpuId): place_to_str = { profiler_pb2.MemEvent.CPUPlace: "CPU", profiler_pb2.MemEvent.CUDAPlace: "GPU", - profiler_pb2.MemEvent.CUDAPinnedPlace: "CUDAPinnedPlace" + profiler_pb2.MemEvent.CUDAPinnedPlace: "CUDAPinnedPlace", } for k, profile_pb in profile_dict.items(): rankId = int(k.split(".")[-1]) @@ -352,9 +392,10 @@ def _allocate_memory_event(self, profile_dict, mem_devices, gpuId): else: place = "UnDefine" - if (mevent.place == profiler_pb2.MemEvent.CUDAPlace - or mevent.place == profiler_pb2.MemEvent.CUDAPinnedPlace - ) and mevent.device_id != gpuId: + if ( + mevent.place == profiler_pb2.MemEvent.CUDAPlace + or mevent.place == profiler_pb2.MemEvent.CUDAPinnedPlace + ) and mevent.device_id != gpuId: continue crt_info['place'] = place @@ -377,15 +418,21 @@ def _allocate_memory_event(self, profile_dict, mem_devices, gpuId): total_size = 0 while i < len(mem_list): total_size += mem_list[i]['size'] - while i < len(mem_list) - 1 and mem_list[i]['time'] == mem_list[ - i + 1]['time']: + while ( + i < len(mem_list) - 1 + and mem_list[i]['time'] == mem_list[i + 1]['time'] + ): total_size += mem_list[i + 1]['size'] i += 1 - chrome_trace.emit_counter("Memory", "Memory", - mem_list[i]['pid'], - self._align_ts(mem_list[i]['time']), - 0, total_size) + chrome_trace.emit_counter( + "Memory", + "Memory", + mem_list[i]['pid'], + self._align_ts(mem_list[i]['time']), + 0, + total_size, + ) i += 1 return chrome_trace @@ -395,21 +442,29 @@ def _getOPTraceInfoByGpuId(self, groupId, gpuId): for file in fileFist: rankId = self.getRankId(file) localRank = rankId % self._gpuPerTrainer - if localRank == gpuId and (rankId / self._gpuPerTrainer - ) % self._groupSize < self._displaySize: + if ( + localRank == gpuId + and (rankId / self._gpuPerTrainer) % self._groupSize + < self._displaySize + ): newFileList.append(file) profile_dict = self._parseTask(newFileList) - initPid = PIPELINEINFO_TRACE_NUM + DCGMINFO_TRACE_NUM + NETINFO_TRACE_NUM + initPid = ( + PIPELINEINFO_TRACE_NUM + DCGMINFO_TRACE_NUM + NETINFO_TRACE_NUM + ) metaTrace, devicesPid, mem_devicesPid = self._allocate_pids( - profile_dict, gpuId, initPid) + profile_dict, gpuId, initPid + ) eventsTrace = self._allocate_events(profile_dict, devicesPid, gpuId) - memEventsTrace = self._allocate_memory_event(profile_dict, - mem_devicesPid, gpuId) + memEventsTrace = self._allocate_memory_event( + profile_dict, mem_devicesPid, gpuId + ) trace = {} - trace[ - 'traceEvents'] = metaTrace._metadata + eventsTrace._events + memEventsTrace._events + trace['traceEvents'] = ( + metaTrace._metadata + eventsTrace._events + memEventsTrace._events + ) self.dumpOpInfoDict(trace, groupId, gpuId, True) return trace @@ -421,24 +476,28 @@ def getOPTraceInfo(self, groupId): pidList = [] for gpuId in range(self._gpuPerTrainer): - subproc = Process(target=self._getOPTraceInfoByGpuId, - args=( - groupId, - gpuId, - )) + subproc = Process( + target=self._getOPTraceInfoByGpuId, + args=( + groupId, + gpuId, + ), + ) processPool.append(subproc) subproc.start() pidList.append(subproc.pid) self._logger.info( "[op info]: process [%d] has been started, total task num is %d ..." - % (subproc.pid, 1)) + % (subproc.pid, 1) + ) for t in processPool: t.join() pidList.remove(t.pid) self._logger.info( - "[op info]: process [%d] has exited! remained %d process!" % - (t.pid, len(pidList))) + "[op info]: process [%d] has exited! remained %d process!" + % (t.pid, len(pidList)) + ) opInfo = {} diff --git a/tools/analysisPyXml.py b/tools/analysisPyXml.py index b9280124aedba..ee110d6ce7f07 100644 --- a/tools/analysisPyXml.py +++ b/tools/analysisPyXml.py @@ -21,10 +21,16 @@ def analysisPyXml(rootPath, ut): xml_path = '%s/build/pytest/%s/python-coverage.xml' % (rootPath, ut) - related_ut_map_file = '%s/build/ut_map/%s/related_%s.txt' % (rootPath, ut, - ut) - notrelated_ut_map_file = '%s/build/ut_map/%s/notrelated_%s.txt' % (rootPath, - ut, ut) + related_ut_map_file = '%s/build/ut_map/%s/related_%s.txt' % ( + rootPath, + ut, + ut, + ) + notrelated_ut_map_file = '%s/build/ut_map/%s/notrelated_%s.txt' % ( + rootPath, + ut, + ut, + ) tree = ElementTree.parse(xml_path) root = tree.getroot() error_files = [] @@ -40,18 +46,36 @@ def analysisPyXml(rootPath, ut): command = 'sed -n %sp %s' % (line_number, clazz_filename) _code, output = commands.getstatusoutput(command) if _code == 0: - if output.strip().startswith( - ('from', 'import', '__all__', 'def', 'class', '"""', - '@', '\'\'\'', 'logger', '_logger', 'logging', 'r"""', - 'pass', 'try', 'except', - 'if __name__ == "__main__"')) == False: - pattern = r"""(.*) = ('*')|(.*) = ("*")|(.*) = (\d)|(.*) = (-\d)|(.*) = (None)|(.*) = (True)|(.*) = (False)|(.*) = (URL_PREFIX*)|(.*) = (\[)|(.*) = (\{)|(.*) = (\()""" #a='b'/a="b"/a=0 + if ( + output.strip().startswith( + ( + 'from', + 'import', + '__all__', + 'def', + 'class', + '"""', + '@', + '\'\'\'', + 'logger', + '_logger', + 'logging', + 'r"""', + 'pass', + 'try', + 'except', + 'if __name__ == "__main__"', + ) + ) + == False + ): + pattern = r"""(.*) = ('*')|(.*) = ("*")|(.*) = (\d)|(.*) = (-\d)|(.*) = (None)|(.*) = (True)|(.*) = (False)|(.*) = (URL_PREFIX*)|(.*) = (\[)|(.*) = (\{)|(.*) = (\()""" # a='b'/a="b"/a=0 if re.match(pattern, output.strip()) == None: pyCov_file.append(clazz_filename) coverageMessage = 'RELATED' break else: - coverageMessage = 'FILTER' #hit filter logic + coverageMessage = 'FILTER' # hit filter logic else: coverageMessage = 'FILTER' else: @@ -61,8 +85,9 @@ def analysisPyXml(rootPath, ut): else: coverageMessage = 'NOT_RELATED' if coverageMessage in ['NOT_RELATED', 'ERROR', 'FILTER']: - os.system('echo %s >> %s' % - (clazz_filename, notrelated_ut_map_file)) + os.system( + 'echo %s >> %s' % (clazz_filename, notrelated_ut_map_file) + ) elif coverageMessage == 'RELATED': os.system('echo %s >> %s' % (clazz_filename, related_ut_map_file)) diff --git a/tools/check_api_compatible.py b/tools/check_api_compatible.py index 18fb4d7ecdf14..ea08a4494bb17 100644 --- a/tools/check_api_compatible.py +++ b/tools/check_api_compatible.py @@ -27,7 +27,9 @@ logger.addHandler(console) console.setFormatter( logging.Formatter( - "%(asctime)s - %(funcName)s:%(lineno)d - %(levelname)s - %(message)s")) + "%(asctime)s - %(funcName)s:%(lineno)d - %(levelname)s - %(message)s" + ) +) def _check_compatible(args_o, args_n, defaults_o, defaults_n): @@ -38,21 +40,30 @@ def _check_compatible(args_o, args_n, defaults_o, defaults_n): # 参数改名了，也要提醒关注 for idx in range(min(len(args_o), len(args_n))): if args_o[idx] != args_n[idx]: - logger.debug("args's %d parameter diff with previous: %s vs %s", - idx, args_o, args_n) + logger.debug( + "args's %d parameter diff with previous: %s vs %s", + idx, + args_o, + args_n, + ) return False # 新增加了参数，必须提供默认值。以及不能减少默认值数量 if (len(args_n) - len(defaults_n)) > (len(args_o) - len(defaults_o)): - logger.debug("defaults num less then previous: %s vs %s", defaults_o, - defaults_n) + logger.debug( + "defaults num less then previous: %s vs %s", defaults_o, defaults_n + ) return False # 默认值必须相等 for idx in range(min(len(defaults_o), len(defaults_n))): nidx_o = -1 - idx nidx_n = -1 - idx - (len(args_n) - len(args_o)) - if (defaults_o[nidx_o] != defaults_n[nidx_n]): - logger.debug("defaults's %d value diff with previous: %s vs %s", - nidx_n, defaults_o, defaults_n) + if defaults_o[nidx_o] != defaults_n[nidx_n]: + logger.debug( + "defaults's %d value diff with previous: %s vs %s", + nidx_n, + defaults_o, + defaults_n, + ) return False return True @@ -61,21 +72,26 @@ def check_compatible(old_api_spec, new_api_spec): """ check compatible, FullArgSpec """ - if not (isinstance(old_api_spec, inspect.FullArgSpec) - and isinstance(new_api_spec, inspect.FullArgSpec)): + if not ( + isinstance(old_api_spec, inspect.FullArgSpec) + and isinstance(new_api_spec, inspect.FullArgSpec) + ): logger.warning( "new_api_spec or old_api_spec is not instance of inspect.FullArgSpec" ) return False return _check_compatible( - old_api_spec.args, new_api_spec.args, + old_api_spec.args, + new_api_spec.args, [] if old_api_spec.defaults is None else old_api_spec.defaults, - [] if new_api_spec.defaults is None else new_api_spec.defaults) + [] if new_api_spec.defaults is None else new_api_spec.defaults, + ) def check_compatible_str(old_api_spec_str, new_api_spec_str): patArgSpec = re.compile( - r'args=(.*), varargs=.*defaults=(None|\((.*)\)), kwonlyargs=.*') + r'args=(.*), varargs=.*defaults=(None|\((.*)\)), kwonlyargs=.*' + ) mo_o = patArgSpec.search(old_api_spec_str) mo_n = patArgSpec.search(new_api_spec_str) if not (mo_o and mo_n): @@ -99,7 +115,8 @@ def read_argspec_from_file(specfile): """ res_dict = {} patArgSpec = re.compile( - r'^(paddle[^,]+)\s+\((ArgSpec.*),\s\(\'document\W*([0-9a-z]{32})') + r'^(paddle[^,]+)\s+\((ArgSpec.*),\s\(\'document\W*([0-9a-z]{32})' + ) fullargspec_prefix = 'inspect.Full' for line in specfile.readlines(): mo = patArgSpec.search(line) @@ -123,21 +140,23 @@ def parse_args(): """ global arguments parser = argparse.ArgumentParser( - description='check api compatible across versions') + description='check api compatible across versions' + ) parser.add_argument('--debug', dest='debug', action="store_true") parser.add_argument( 'prev', type=argparse.FileType('r'), - help='the previous version (the version from develop branch)') - parser.add_argument('post', - type=argparse.FileType('r'), - help='the post version (the version from PullRequest)') + help='the previous version (the version from develop branch)', + ) + parser.add_argument( + 'post', + type=argparse.FileType('r'), + help='the post version (the version from PullRequest)', + ) for item in arguments: - parser.add_argument(item[0], - dest=item[1], - help=item[4], - type=item[2], - default=item[3]) + parser.add_argument( + item[0], dest=item[1], help=item[4], type=item[2], default=item[3] + ) if len(sys.argv) < 2: parser.print_help() @@ -162,10 +181,12 @@ def parse_args(): if as_prev is None: # the api is deleted continue if isinstance(as_prev, str) or isinstance(as_post, str): - as_prev_str = as_prev if isinstance(as_prev, - str) else repr(as_prev) - as_post_str = as_post if isinstance(as_post, - str) else repr(as_post) + as_prev_str = ( + as_prev if isinstance(as_prev, str) else repr(as_prev) + ) + as_post_str = ( + as_post if isinstance(as_post, str) else repr(as_post) + ) if not check_compatible_str(as_prev_str, as_post_str): diff_api_names.append(as_post_name) else: diff --git a/tools/check_api_source_without_core_ops.py b/tools/check_api_source_without_core_ops.py index 797934a5a3568..bf7027be5cb74 100644 --- a/tools/check_api_source_without_core_ops.py +++ b/tools/check_api_source_without_core_ops.py @@ -27,8 +27,10 @@ differ = difflib.Differ() result = differ.compare(origin, new) -api_with_ops, api_without_ops = count_api_without_core_ops.get_apis_with_and_without_core_ops( - ['paddle']) +( + api_with_ops, + api_without_ops, +) = count_api_without_core_ops.get_apis_with_and_without_core_ops(['paddle']) error = False # get all diff apis diff --git a/tools/check_ctest_hung.py b/tools/check_ctest_hung.py index 094d051b03e6b..11e927eb681e8 100644 --- a/tools/check_ctest_hung.py +++ b/tools/check_ctest_hung.py @@ -38,8 +38,7 @@ def main(): passed = set() with open(logfile, "r") as fn: for l in fn.readlines(): - if l.find("Test ") != -1 and \ - l.find("Passed") != -1: + if l.find("Test ") != -1 and l.find("Passed") != -1: m = re.search(r"Test\s+#[0-9]*\:\s([a-z0-9_]+)", escape(l)) passed.add(m.group(1)) if l.find("Start ") != -1: diff --git a/tools/check_op_benchmark_result.py b/tools/check_op_benchmark_result.py index aaf194ff95ec5..8fce508102282 100644 --- a/tools/check_op_benchmark_result.py +++ b/tools/check_op_benchmark_result.py @@ -19,14 +19,12 @@ def check_path_exists(path): - """Assert whether file/directory exists. - """ + """Assert whether file/directory exists.""" assert os.path.exists(path), "%s does not exist." % path def parse_case_name(log_file_name): - """Parse case name. - """ + """Parse case name.""" case_id, case_info = log_file_name.split("-") direction = case_info.split(".")[0].split("_")[-1] @@ -34,8 +32,7 @@ def parse_case_name(log_file_name): def parse_log_file(log_file): - """Load one case result from log file. - """ + """Load one case result from log file.""" check_path_exists(log_file) result = None @@ -56,20 +53,20 @@ def parse_log_file(log_file): def load_benchmark_result_from_logs_dir(logs_dir): - """Load benchmark result from logs directory. - """ + """Load benchmark result from logs directory.""" check_path_exists(logs_dir) log_file_path = lambda log_file: os.path.join(logs_dir, log_file) - result_lambda = lambda log_file: (log_file, - parse_log_file(log_file_path(log_file))) + result_lambda = lambda log_file: ( + log_file, + parse_log_file(log_file_path(log_file)), + ) return dict(map(result_lambda, os.listdir(logs_dir))) def check_speed_result(case_name, develop_data, pr_data, pr_result): - """Check speed differences between develop and pr. - """ + """Check speed differences between develop and pr.""" pr_gpu_time = pr_data.get("gpu_time") develop_gpu_time = develop_data.get("gpu_time") if develop_gpu_time != 0.0: @@ -84,10 +81,14 @@ def check_speed_result(case_name, develop_data, pr_data, pr_result): total_time_diff = (pr_total_time - develop_total_time) / develop_total_time logging.info("------ OP: %s ------" % case_name) - logging.info("GPU time change: %s (develop: %.7f -> PR: %.7f)" % - (gpu_time_diff_str, develop_gpu_time, pr_gpu_time)) - logging.info("Total time change: %.5f%% (develop: %.7f -> PR: %.7f)" % - (total_time_diff * 100, develop_total_time, pr_total_time)) + logging.info( + "GPU time change: %s (develop: %.7f -> PR: %.7f)" + % (gpu_time_diff_str, develop_gpu_time, pr_gpu_time) + ) + logging.info( + "Total time change: %.5f%% (develop: %.7f -> PR: %.7f)" + % (total_time_diff * 100, develop_total_time, pr_total_time) + ) logging.info("backward: %s" % pr_result.get("backward")) logging.info("parameters:") for line in pr_result.get("parameters").strip().split("\n"): @@ -97,8 +98,7 @@ def check_speed_result(case_name, develop_data, pr_data, pr_result): def check_accuracy_result(case_name, pr_result): - """Check accuracy result. - """ + """Check accuracy result.""" logging.info("------ OP: %s ------" % case_name) logging.info("Accuracy diff: %s" % pr_result.get("diff")) logging.info("backward: %s" % pr_result.get("backward")) @@ -109,15 +109,16 @@ def check_accuracy_result(case_name, pr_result): return not pr_result.get("consistent") -def compare_benchmark_result(case_name, develop_result, pr_result, - check_results): - """Compare the differences between develop and pr. - """ +def compare_benchmark_result( + case_name, develop_result, pr_result, check_results +): + """Compare the differences between develop and pr.""" develop_speed = develop_result.get("speed") pr_speed = pr_result.get("speed") assert type(develop_speed) == type( - pr_speed), "The types of comparison results need to be consistent." + pr_speed + ), "The types of comparison results need to be consistent." if isinstance(develop_speed, dict) and isinstance(pr_speed, dict): if check_speed_result(case_name, develop_speed, pr_speed, pr_result): @@ -128,8 +129,7 @@ def compare_benchmark_result(case_name, develop_result, pr_result, def update_api_info_file(fail_case_list, api_info_file): - """Update api info file to auto retry benchmark test. - """ + """Update api info file to auto retry benchmark test.""" check_path_exists(api_info_file) # set of case names for performance check failures @@ -153,14 +153,14 @@ def update_api_info_file(fail_case_list, api_info_file): def summary_results(check_results, api_info_file): - """Summary results and return exit code. - """ + """Summary results and return exit code.""" for case_name in check_results["speed"]: logging.error("Check speed result with case \"%s\" failed." % case_name) for case_name in check_results["accuracy"]: - logging.error("Check accuracy result with case \"%s\" failed." % - case_name) + logging.error( + "Check accuracy result with case \"%s\" failed." % case_name + ) if len(check_results["speed"]) and api_info_file: update_api_info_file(check_results["speed"], api_info_file) @@ -172,33 +172,38 @@ def summary_results(check_results, api_info_file): if __name__ == "__main__": - """Load result from log directories and compare the differences. - """ + """Load result from log directories and compare the differences.""" logging.basicConfig( level=logging.INFO, - format="[%(filename)s:%(lineno)d] [%(levelname)s] %(message)s") + format="[%(filename)s:%(lineno)d] [%(levelname)s] %(message)s", + ) parser = argparse.ArgumentParser() parser.add_argument( "--develop_logs_dir", type=str, required=True, - help="Specify the benchmark result directory of develop branch.") + help="Specify the benchmark result directory of develop branch.", + ) parser.add_argument( "--pr_logs_dir", type=str, required=True, - help="Specify the benchmark result directory of PR branch.") - parser.add_argument("--api_info_file", - type=str, - required=False, - help="Specify the api info to run benchmark test.") + help="Specify the benchmark result directory of PR branch.", + ) + parser.add_argument( + "--api_info_file", + type=str, + required=False, + help="Specify the api info to run benchmark test.", + ) args = parser.parse_args() check_results = dict(accuracy=list(), speed=list()) develop_result_dict = load_benchmark_result_from_logs_dir( - args.develop_logs_dir) + args.develop_logs_dir + ) check_path_exists(args.pr_logs_dir) pr_log_files = os.listdir(args.pr_logs_dir) @@ -208,7 +213,8 @@ def summary_results(check_results, api_info_file): if develop_result is None or pr_result is None: continue case_name = parse_case_name(log_file) - compare_benchmark_result(case_name, develop_result, pr_result, - check_results) + compare_benchmark_result( + case_name, develop_result, pr_result, check_results + ) exit(summary_results(check_results, args.api_info_file)) diff --git a/tools/check_op_desc.py b/tools/check_op_desc.py index 7367b88d5b8f5..1c1ff14f7b68c 100644 --- a/tools/check_op_desc.py +++ b/tools/check_op_desc.py @@ -88,7 +88,9 @@ def diff_vars(origin_vars, new_vars): if var_name not in var_changed_error_massage.keys(): var_changed_error_massage[var_name] = {} var_changed_error_massage[var_name][arg_name] = ( - origin_arg_value, new_arg_value) + origin_arg_value, + new_arg_value, + ) for var_name in vars_name_only_in_origin: error, var_error = True, True @@ -111,7 +113,8 @@ def diff_vars(origin_vars, new_vars): # if added var is def, inference needs to review, needs to register. if not new_vars.get(var_name).get(EXTRA) and not new_vars.get( - var_name).get(QUANT): + var_name + ).get(QUANT): error, var_error = True, True var_add_def_message.append(var_name) @@ -160,7 +163,9 @@ def diff_attr(ori_attrs, new_attrs): if attr_name not in attr_changed_error_massage.keys(): attr_changed_error_massage[attr_name] = {} attr_changed_error_massage[attr_name][arg_name] = ( - origin_arg_value, new_arg_value) + origin_arg_value, + new_arg_value, + ) for attr_name in attrs_only_in_origin: error, attr_error = True, True @@ -179,7 +184,8 @@ def diff_attr(ori_attrs, new_attrs): # if added attr is def, inference needs to review, needs to register if not new_attrs.get(attr_name).get(EXTRA) and not new_attrs.get( - attr_name).get(QUANT): + attr_name + ).get(QUANT): error, var_error = True, True attr_added_define_message.append(attr_name) @@ -206,7 +212,8 @@ def check_io_registry(io_type, op, diff): for update_type in [ADD]: for item in diff.get(update_type, []): infos = checker.filter_updates( - op, version_update_map[io_type][update_type], item) + op, version_update_map[io_type][update_type], item + ) if not infos: if update_type not in results.keys(): results[update_type] = [] @@ -228,7 +235,8 @@ def check_attr_registry(op, diff, origin_attrs): for update_type in [ADD, CHANGE]: for item in diff.get(update_type, {}): infos = checker.filter_updates( - op, version_update_map[ATTRS][update_type], item) + op, version_update_map[ATTRS][update_type], item + ) if not infos: if update_type == ADD: if update_type not in results.keys(): @@ -279,8 +287,9 @@ def compare_op_desc(origin_op_desc, new_op_desc): origin_attrs = origin_info.get(ATTRS, {}) new_attrs = new_info.get(ATTRS, {}) attrs_error, attrs_diff = diff_attr(origin_attrs, new_attrs) - attrs_version_errors = check_attr_registry(op_type, attrs_diff, - origin_attrs) + attrs_version_errors = check_attr_registry( + op_type, attrs_diff, origin_attrs + ) if ins_diff: desc_error_message.setdefault(op_type, {})[INPUTS] = ins_diff @@ -290,21 +299,26 @@ def compare_op_desc(origin_op_desc, new_op_desc): desc_error_message.setdefault(op_type, {})[ATTRS] = attrs_diff if ins_version_errors: - version_error_message.setdefault(op_type, - {})[INPUTS] = ins_version_errors + version_error_message.setdefault(op_type, {})[ + INPUTS + ] = ins_version_errors if outs_version_errors: - version_error_message.setdefault(op_type, - {})[OUTPUTS] = outs_version_errors + version_error_message.setdefault(op_type, {})[ + OUTPUTS + ] = outs_version_errors if attrs_version_errors: - version_error_message.setdefault(op_type, - {})[ATTRS] = attrs_version_errors + version_error_message.setdefault(op_type, {})[ + ATTRS + ] = attrs_version_errors return desc_error_message, version_error_message def print_desc_error_message(error_message): - print("\n======================= \n" - "Op desc error for the changes of Inputs/Outputs/Attrs of OPs:\n") + print( + "\n======================= \n" + "Op desc error for the changes of Inputs/Outputs/Attrs of OPs:\n" + ) for op_name in error_message: print("For OP '{}':".format(op_name)) @@ -321,16 +335,24 @@ def print_desc_error_message(error_message): for arg in changed_args: ori_value, new_value = changed_args.get(arg) print( - " * The arg '{}' of Input '{}' is changed: from '{}' to '{}'." - .format(arg, name, ori_value, new_value)) + " * The arg '{}' of Input '{}' is changed: from '{}' to '{}'.".format( + arg, name, ori_value, new_value + ) + ) for name in Inputs_error.get(QUANT, {}): - print(" * The added Input '{}' is `quant`, need slim to review.". - format(name)) + print( + " * The added Input '{}' is `quant`, need slim to review.".format( + name + ) + ) for name in Inputs_error.get(DEF, {}): - print(" * The added Input '{}' is `def`, need inference to review.". - format(name)) + print( + " * The added Input '{}' is `def`, need inference to review.".format( + name + ) + ) # 2. print outputs error message Outputs_error = error_message.get(op_name, {}).get(OUTPUTS, {}) @@ -345,23 +367,31 @@ def print_desc_error_message(error_message): for arg in changed_args: ori_value, new_value = changed_args.get(arg) print( - " * The arg '{}' of Output '{}' is changed: from '{}' to '{}'." - .format(arg, name, ori_value, new_value)) + " * The arg '{}' of Output '{}' is changed: from '{}' to '{}'.".format( + arg, name, ori_value, new_value + ) + ) for name in Outputs_error.get(QUANT, {}): - print(" * The added Output '{}' is `quant`, need slim to review.". - format(name)) + print( + " * The added Output '{}' is `quant`, need slim to review.".format( + name + ) + ) for name in Outputs_error.get(DEF, {}): print( - " * The added Output '{}' is `def`, need inference to review.". - format(name)) + " * The added Output '{}' is `def`, need inference to review.".format( + name + ) + ) # 3. print attrs error message attrs_error = error_message.get(op_name, {}).get(ATTRS, {}) for name in attrs_error.get(ADD_WITH_DEFAULT, {}): - print(" * The added attr '{}' doesn't set default value.".format( - name)) + print( + " * The added attr '{}' doesn't set default value.".format(name) + ) for name in attrs_error.get(DELETE, {}): print(" * The attr '{}' is deleted.".format(name)) @@ -371,18 +401,26 @@ def print_desc_error_message(error_message): for arg in changed_args: ori_value, new_value = changed_args.get(arg) print( - " * The arg '{}' of attr '{}' is changed: from '{}' to '{}'." - .format(arg, name, ori_value, new_value)) + " * The arg '{}' of attr '{}' is changed: from '{}' to '{}'.".format( + arg, name, ori_value, new_value + ) + ) for name in attrs_error.get(QUANT, {}): # TODO(Wilber): - print(" * The added attr '{}' is `quant`, need slim to review.". - format(name)) + print( + " * The added attr '{}' is `quant`, need slim to review.".format( + name + ) + ) for name in attrs_error.get(DEF, {}): # TODO(Wilber): - print(" * The added attr '{}' is `def`, need inference to review.". - format(name)) + print( + " * The added attr '{}' is `def`, need inference to review.".format( + name + ) + ) def print_version_error_message(error_message): @@ -398,31 +436,42 @@ def print_version_error_message(error_message): error_list = inputs_error.get(ADD, []) if error_list: for tup in error_list: - print(" * The added input '{}' is not yet registered.".format( - tup[1])) + print( + " * The added input '{}' is not yet registered.".format( + tup[1] + ) + ) # 2. print outputs error message outputs_error = error_message.get(op_name, {}).get(OUTPUTS, {}) error_list = outputs_error.get(ADD, []) if error_list: for tup in error_list: - print(" * The added output '{}' is not yet registered.".format( - tup[1])) + print( + " * The added output '{}' is not yet registered.".format( + tup[1] + ) + ) - #3. print attrs error message + # 3. print attrs error message attrs_error = error_message.get(op_name, {}).get(ATTRS, {}) error_list = attrs_error.get(ADD, []) if error_list: for tup in error_list: print( " * The added attribute '{}' is not yet registered.".format( - tup[1])) - error_dic = error_message.get(op_name, {}).get(ATTRS, - {}).get(CHANGE, {}) + tup[1] + ) + ) + error_dic = ( + error_message.get(op_name, {}).get(ATTRS, {}).get(CHANGE, {}) + ) for key, val in error_dic.items(): print( " * The change of attribute '{}' is not yet registered.".format( - key)) + key + ) + ) def print_repeat_process(): @@ -449,7 +498,8 @@ def print_repeat_process(): new_op_desc = f.read() desc_error_message, version_error_message = compare_op_desc( - origin_op_desc, new_op_desc) + origin_op_desc, new_op_desc + ) if error: print("-" * 30) print_desc_error_message(desc_error_message) diff --git a/tools/check_op_kernel_same_dtypes.py b/tools/check_op_kernel_same_dtypes.py index b5b3e81c89fcb..13d0119f83b2e 100644 --- a/tools/check_op_kernel_same_dtypes.py +++ b/tools/check_op_kernel_same_dtypes.py @@ -36,13 +36,15 @@ def get_all_kernels(): pattern = re.compile(r'data_type\[([^\]]+)\]') for op_info in op_infos: infos = pattern.findall(op_info) - if infos is None or len(infos) == 0: continue + if infos is None or len(infos) == 0: + continue register_type = infos[0].split(":")[-1] op_kernel_types[op_type].append(register_type.lower()) - for (op_type, op_kernels) in sorted(op_kernel_types.items(), - key=lambda x: x[0]): + for (op_type, op_kernels) in sorted( + op_kernel_types.items(), key=lambda x: x[0] + ): print(op_type, " ".join(sorted(op_kernels))) @@ -56,12 +58,18 @@ def read_file(file_path): def print_diff(op_type, op_kernel_dtype_set, grad_op_kernel_dtype_set): if len(op_kernel_dtype_set) > len(grad_op_kernel_dtype_set): lack_dtypes = list(op_kernel_dtype_set - grad_op_kernel_dtype_set) - print("{} supports [{}] now, but its grad op kernel not supported.". - format(op_type, " ".join(lack_dtypes))) + print( + "{} supports [{}] now, but its grad op kernel not supported.".format( + op_type, " ".join(lack_dtypes) + ) + ) else: lack_dtypes = list(grad_op_kernel_dtype_set - op_kernel_dtype_set) - print("{} supports [{}] now, but its forward op kernel not supported.". - format(op_type + "_grad", " ".join(lack_dtypes))) + print( + "{} supports [{}] now, but its forward op kernel not supported.".format( + op_type + "_grad", " ".join(lack_dtypes) + ) + ) def contain_current_op(op_type, op_info_dict): @@ -91,7 +99,8 @@ def check_change_or_add_op_kernel_dtypes_valid(): origin_dtype_set = origin_all_kernel_dtype_dict[op_type] # op kernel changed if origin_dtype_set != dtype_set and not contain_current_op( - op_type, added_or_changed_op_info): + op_type, added_or_changed_op_info + ): added_or_changed_op_info[op_type] = dtype_set else: # do nothing @@ -111,7 +120,8 @@ def check_change_or_add_op_kernel_dtypes_valid(): grad_op_type = op_type + "_grad" if grad_op_type in new_all_kernel_dtype_dict: grad_op_kernel_dtype_set = set( - new_all_kernel_dtype_dict[grad_op_type]) + new_all_kernel_dtype_dict[grad_op_type] + ) if dtype_set != grad_op_kernel_dtype_set: print_diff(op_type, dtype_set, grad_op_kernel_dtype_set) # if changed grad op @@ -119,7 +129,8 @@ def check_change_or_add_op_kernel_dtypes_valid(): forward_op_type = op_type.rstrip("_grad") if forward_op_type in new_all_kernel_dtype_dict: op_kernel_dtype_set = set( - new_all_kernel_dtype_dict[forward_op_type]) + new_all_kernel_dtype_dict[forward_op_type] + ) if op_kernel_dtype_set != dtype_set: print_diff(forward_op_type, op_kernel_dtype_set, dtype_set) @@ -129,6 +140,8 @@ def check_change_or_add_op_kernel_dtypes_valid(): elif len(sys.argv) == 3: check_change_or_add_op_kernel_dtypes_valid() else: - print("Usage:\n" \ - "\tpython check_op_kernel_same_dtypes.py > all_kernels.txt\n" \ - "\tpython check_op_kernel_same_dtypes.py OP_KERNEL_DTYPE_DEV.spec OP_KERNEL_DTYPE_PR.spec > diff") + print( + "Usage:\n" + "\tpython check_op_kernel_same_dtypes.py > all_kernels.txt\n" + "\tpython check_op_kernel_same_dtypes.py OP_KERNEL_DTYPE_DEV.spec OP_KERNEL_DTYPE_PR.spec > diff" + ) diff --git a/tools/check_op_register_type.py b/tools/check_op_register_type.py index 9db2fc0bf5137..af7f3ee42a887 100644 --- a/tools/check_op_register_type.py +++ b/tools/check_op_register_type.py @@ -34,18 +34,21 @@ def get_all_kernels(): op_kernel_types = collections.defaultdict(list) for op_type, op_infos in all_kernels_info.items(): is_grad_op = op_type.endswith("_grad") - if is_grad_op: continue + if is_grad_op: + continue pattern = re.compile(r'data_type\[([^\]]+)\]') for op_info in op_infos: infos = pattern.findall(op_info) - if infos is None or len(infos) == 0: continue + if infos is None or len(infos) == 0: + continue register_type = infos[0].split(":")[-1] op_kernel_types[op_type].append(register_type.lower()) - for (op_type, op_kernels) in sorted(op_kernel_types.items(), - key=lambda x: x[0]): + for (op_type, op_kernels) in sorted( + op_kernel_types.items(), key=lambda x: x[0] + ): print(op_type, " ".join(sorted(op_kernels))) @@ -63,8 +66,11 @@ def print_diff(op_type, register_types): if len(FLOATS - register_types) == 1: lack_types |= FLOATS - register_types - print("{} only supports [{}] now, but lacks [{}].".format( - op_type, " ".join(register_types), " ".join(lack_types))) + print( + "{} only supports [{}] now, but lacks [{}].".format( + op_type, " ".join(register_types), " ".join(lack_types) + ) + ) def check_add_op_valid(): @@ -77,10 +83,13 @@ def check_add_op_valid(): for each_diff in result: if each_diff[0] in ['+'] and len(each_diff) > 2: # if change or add op op_info = each_diff[1:].split() - if len(op_info) < 2: continue + if len(op_info) < 2: + continue register_types = set(op_info[1:]) - if len(FLOATS - register_types) == 1 or \ - len(INTS - register_types) == 1: + if ( + len(FLOATS - register_types) == 1 + or len(INTS - register_types) == 1 + ): print_diff(op_info[0], register_types) @@ -89,6 +98,8 @@ def check_add_op_valid(): elif len(sys.argv) == 3: check_add_op_valid() else: - print("Usage:\n" \ - "\tpython check_op_register_type.py > all_kernels.txt\n" \ - "\tpython check_op_register_type.py OP_TYPE_DEV.spec OP_TYPE_PR.spec > diff") + print( + "Usage:\n" + "\tpython check_op_register_type.py > all_kernels.txt\n" + "\tpython check_op_register_type.py OP_TYPE_DEV.spec OP_TYPE_PR.spec > diff" + ) diff --git a/tools/check_pr_approval.py b/tools/check_pr_approval.py index c242afd06e760..7e7e7720bf8fd 100644 --- a/tools/check_pr_approval.py +++ b/tools/check_pr_approval.py @@ -39,8 +39,11 @@ def check_approval(count, required_reviewers): else: required_reviewers_login.add(rr) - if len(set(approved_user_ids) & required_reviewers_int) + len( - approved_user_logins & required_reviewers_login) >= count: + if ( + len(set(approved_user_ids) & required_reviewers_int) + + len(approved_user_logins & required_reviewers_login) + >= count + ): print("TRUE") else: print("FALSE") diff --git a/tools/check_ut.py b/tools/check_ut.py index 0b1211f1e654d..e08c358531f7f 100644 --- a/tools/check_ut.py +++ b/tools/check_ut.py @@ -20,7 +20,7 @@ class PRChecker(object): - """ PR Checker. """ + """PR Checker.""" def __init__(self): self.github = Github(os.getenv('GITHUB_API_TOKEN'), timeout=60) diff --git a/tools/codestyle/docstring_checker.py b/tools/codestyle/docstring_checker.py index 18083d1fab104..0d163c20bbfa1 100644 --- a/tools/codestyle/docstring_checker.py +++ b/tools/codestyle/docstring_checker.py @@ -28,11 +28,10 @@ def register(linter): class Docstring(object): - """Docstring class holds the parsed doc string elements. - """ + """Docstring class holds the parsed doc string elements.""" def __init__(self): - self.d = defaultdict(list) #name->[] + self.d = defaultdict(list) # name->[] self.clear() def clear(self): @@ -40,7 +39,7 @@ def clear(self): self.d['Examples'] = [] self.d['Returns'] = [] self.d['Raises'] = [] - self.args = {} #arg_name->arg_type + self.args = {} # arg_name->arg_type def get_level(self, string, indent=' '): level = 0 @@ -111,7 +110,8 @@ class DocstringChecker(BaseChecker): """DosstringChecker is pylint checker to check docstring style. """ - __implements__ = (IAstroidChecker, ) + + __implements__ = (IAstroidChecker,) POSITIONAL_MESSAGE_ID = 'str-used-on-positional-format-argument' KEYWORD_MESSAGE_ID = 'str-used-on-keyword-format-argument' @@ -120,25 +120,41 @@ class DocstringChecker(BaseChecker): symbol = "doc-string" priority = -1 msgs = { - 'W9001': ('One line doc string on > 1 lines', symbol + "-one-line", - 'Used when a short doc string is on multiple lines'), - 'W9002': - ('Doc string does not end with "." period', symbol + "-end-with", - 'Used when a doc string does not end with a period'), - 'W9003': - ('All args with their types must be mentioned in doc string %s', - symbol + "-with-all-args", - 'Used when not all arguments are in the doc string '), - 'W9005': ('Missing docstring or docstring is too short', - symbol + "-missing", 'Add docstring longer >=10'), - 'W9006': ('Docstring indent error, use 4 space for indent', - symbol + "-indent-error", 'Use 4 space for indent'), - 'W9007': - ('You should add `Returns` in comments', symbol + "-with-returns", - 'There should be a `Returns` section in comments'), - 'W9008': - ('You should add `Raises` section in comments', symbol + "-with-raises", - 'There should be a `Raises` section in comments'), + 'W9001': ( + 'One line doc string on > 1 lines', + symbol + "-one-line", + 'Used when a short doc string is on multiple lines', + ), + 'W9002': ( + 'Doc string does not end with "." period', + symbol + "-end-with", + 'Used when a doc string does not end with a period', + ), + 'W9003': ( + 'All args with their types must be mentioned in doc string %s', + symbol + "-with-all-args", + 'Used when not all arguments are in the doc string ', + ), + 'W9005': ( + 'Missing docstring or docstring is too short', + symbol + "-missing", + 'Add docstring longer >=10', + ), + 'W9006': ( + 'Docstring indent error, use 4 space for indent', + symbol + "-indent-error", + 'Use 4 space for indent', + ), + 'W9007': ( + 'You should add `Returns` in comments', + symbol + "-with-returns", + 'There should be a `Returns` section in comments', + ), + 'W9008': ( + 'You should add `Raises` section in comments', + symbol + "-with-raises", + 'There should be a `Raises` section in comments', + ), } options = () @@ -321,8 +337,7 @@ def all_args_in_doc(self, node, doc): return True args = [] for arg in node.args.get_children(): - if (not isinstance(arg, astroid.AssignName)) \ - or arg.name == "self": + if (not isinstance(arg, astroid.AssignName)) or arg.name == "self": continue args.append(arg.name) @@ -332,20 +347,24 @@ def all_args_in_doc(self, node, doc): parsed_args = doc.args args_not_documented = set(args) - set(parsed_args) if len(args) > 0 and len(parsed_args) <= 0: - self.add_message('W9003', - node=node, - line=node.fromlineno, - args=list(args_not_documented)) + self.add_message( + 'W9003', + node=node, + line=node.fromlineno, + args=list(args_not_documented), + ) return False for t in args: if t not in parsed_args: - self.add_message('W9003', - node=node, - line=node.fromlineno, - args=[ - t, - ]) + self.add_message( + 'W9003', + node=node, + line=node.fromlineno, + args=[ + t, + ], + ) return False return True diff --git a/tools/codestyle/test_docstring_checker.py b/tools/codestyle/test_docstring_checker.py index 61b236b6b7d63..09617a0f2f735 100644 --- a/tools/codestyle/test_docstring_checker.py +++ b/tools/codestyle/test_docstring_checker.py @@ -21,7 +21,8 @@ class TestDocstring(pylint.testutils.CheckerTestCase): CHECKER_CLASS = docstring_checker.DocstringChecker def test_one_line(self): - func_node = astroid.extract_node(''' + func_node = astroid.extract_node( + ''' def test(): """get news. @@ -29,7 +30,8 @@ def test(): if True: return 5 return 5 - ''') + ''' + ) self.checker.visit_functiondef(func_node) got = self.linter.release_messages() @@ -37,13 +39,15 @@ def test(): assert 'W9001' == got[0][0] def test_one_line_1(self): - func_node = astroid.extract_node(''' + func_node = astroid.extract_node( + ''' def test(): """get news""" if True: return 5 return 5 - ''') + ''' + ) self.checker.visit_functiondef(func_node) got = self.linter.release_messages() @@ -51,7 +55,8 @@ def test(): assert 'W9002' == got[0][0] def test_args(self): - func_node = astroid.extract_node(''' + func_node = astroid.extract_node( + ''' def test(scale, mean): """get news. Args: @@ -64,7 +69,8 @@ def test(scale, mean): mean=scale mean=scale mean=scale - ''') + ''' + ) self.checker.visit_functiondef(func_node) got = self.linter.release_messages() @@ -72,7 +78,8 @@ def test(scale, mean): assert 'W9003' == got[0][0] def test_missing(self): - func_node = astroid.extract_node(''' + func_node = astroid.extract_node( + ''' def test(): mean=scale mean=scale @@ -85,7 +92,8 @@ def test(): mean=scale mean=scale mean=scale - ''') + ''' + ) self.checker.visit_functiondef(func_node) got = self.linter.release_messages() @@ -93,13 +101,15 @@ def test(): assert 'W9005' == got[0][0] def test_indent(self): - func_node = astroid.extract_node(''' + func_node = astroid.extract_node( + ''' def test(): """ get get get get get get get get get get get get get get get get. """ pass - ''') + ''' + ) self.checker.visit_functiondef(func_node) got = self.linter.release_messages() @@ -107,7 +117,8 @@ def test(): assert 'W9006' == got[0][0] def test_with_resturns(self): - func_node = astroid.extract_node(''' + func_node = astroid.extract_node( + ''' def test(): """get news. Args: @@ -125,7 +136,8 @@ def test(): mean=scale mean=scale return mean - ''') + ''' + ) self.checker.visit_functiondef(func_node) got = self.linter.release_messages() @@ -133,7 +145,8 @@ def test(): assert 'W9007' == got[0][0] def test_with_raises(self): - func_node = astroid.extract_node(''' + func_node = astroid.extract_node( + ''' def test(): """get news. Args: @@ -151,7 +164,8 @@ def test(): mean=scale mean=scale raise ValueError('A very specific bad thing happened.') - ''') + ''' + ) self.checker.visit_functiondef(func_node) got = self.linter.release_messages() diff --git a/tools/continuous_integration/bisect.py b/tools/continuous_integration/bisect.py index afffc60a449ac..b2a2aa1f9b646 100644 --- a/tools/continuous_integration/bisect.py +++ b/tools/continuous_integration/bisect.py @@ -27,43 +27,42 @@ import sys parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument('--git_dir', - type=str, - default='', - help='git repo root directory.') -parser.add_argument('--build_dir', - type=str, - default='', - help='build directory.') -parser.add_argument('--good_commit', - type=str, - default='', - help='The old commit known to be good.') -parser.add_argument('--bad_commit', - type=str, - default='', - help='The new commit known to be bad.') -parser.add_argument('--test_target', - type=str, - default='', - help='The test target to evaluate.') +parser.add_argument( + '--git_dir', type=str, default='', help='git repo root directory.' +) +parser.add_argument( + '--build_dir', type=str, default='', help='build directory.' +) +parser.add_argument( + '--good_commit', + type=str, + default='', + help='The old commit known to be good.', +) +parser.add_argument( + '--bad_commit', type=str, default='', help='The new commit known to be bad.' +) +parser.add_argument( + '--test_target', type=str, default='', help='The test target to evaluate.' +) parser.add_argument( '--bisect_branch', type=str, default='develop', - help='The mainline branch to bisect (feature branch ignored.') -parser.add_argument('--log_file', - type=str, - default='', - help='The file use to log outputs.') -parser.add_argument('--test_times', - type=int, - default=10, - help="Number of times to run the test target.") -parser.add_argument('--build_parallel', - type=int, - default=32, - help="make parallelism.") + help='The mainline branch to bisect (feature branch ignored.', +) +parser.add_argument( + '--log_file', type=str, default='', help='The file use to log outputs.' +) +parser.add_argument( + '--test_times', + type=int, + default=10, + help="Number of times to run the test target.", +) +parser.add_argument( + '--build_parallel', type=int, default=32, help="make parallelism." +) args = parser.parse_args() if not args.log_file: @@ -81,10 +80,13 @@ def print_arguments(): # List the commits in mainline branch. os.chdir(args.git_dir) -ret = subprocess.check_output([ - 'git rev-list --first-parent %s...%s' % (args.good_commit, args.bad_commit) -], - shell=True) +ret = subprocess.check_output( + [ + 'git rev-list --first-parent %s...%s' + % (args.good_commit, args.bad_commit) + ], + shell=True, +) sys.stdout.write('commits found:\n%s\n' % ret) commits = ret.strip().split('\n') os.chdir(args.build_dir) @@ -95,11 +97,13 @@ def print_arguments(): while True: # Get to the mainline branch and clean up os.chdir(args.git_dir) - subprocess.check_output([ - 'git checkout %s && git clean -fd && git checkout .' % - args.bisect_branch - ], - shell=True) + subprocess.check_output( + [ + 'git checkout %s && git clean -fd && git checkout .' + % args.bisect_branch + ], + shell=True, + ) if not commits: sys.stdout.write('no commits to bisect\n') @@ -115,9 +119,11 @@ def print_arguments(): os.chdir(args.build_dir) sys.stdout.write('eval commit %d/%d: %s\n' % (pick_idx, len(commits), pick)) # Link error can happen without complete clean up. - cmd = ('rm -rf * && ' - 'cmake -DWITH_TESTING=ON %s >> %s && make -j%s >> %s' % - (args.git_dir, args.log_file, args.build_parallel, args.log_file)) + cmd = ( + 'rm -rf * && ' + 'cmake -DWITH_TESTING=ON %s >> %s && make -j%s >> %s' + % (args.git_dir, args.log_file, args.build_parallel, args.log_file) + ) sys.stdout.write('cmd: %s\n' % cmd) try: subprocess.check_output([cmd], shell=True) @@ -127,8 +133,11 @@ def print_arguments(): # test the selected branch. passed = True try: - cmd = ('ctest --repeat-until-fail %s -R %s >> %s' % - (args.test_times, args.test_target, args.log_file)) + cmd = 'ctest --repeat-until-fail %s -R %s >> %s' % ( + args.test_times, + args.test_target, + args.log_file, + ) sys.stdout.write('cmd: %s\n' % cmd) subprocess.check_output([cmd], shell=True) except subprocess.CalledProcessError as e: @@ -136,10 +145,12 @@ def print_arguments(): last_culprit = pick sys.stdout.write('eval %s passed: %s\n' % (pick, passed)) if passed: - if pick_idx == 0: break + if pick_idx == 0: + break commits = commits[:pick_idx] else: - if pick_idx + 1 >= len(commits): break - commits = commits[pick_idx + 1:] + if pick_idx + 1 >= len(commits): + break + commits = commits[pick_idx + 1 :] sys.stdout.write('Culprit commit: %s\n' % last_culprit) diff --git a/tools/count_api_without_core_ops.py b/tools/count_api_without_core_ops.py index d75578d610d88..96a7c33dd95ff 100644 --- a/tools/count_api_without_core_ops.py +++ b/tools/count_api_without_core_ops.py @@ -37,9 +37,12 @@ def md5(doc): md5sum = hashinst.hexdigest() except UnicodeDecodeError as e: md5sum = None - print("Error({}) occurred when `md5({})`, discard it.".format( - str(e), doc), - file=sys.stderr) + print( + "Error({}) occurred when `md5({})`, discard it.".format( + str(e), doc + ), + file=sys.stderr, + ) return md5sum @@ -50,11 +53,13 @@ def split_with_and_without_core_ops(member, cur_name): if member.__doc__.find(':api_attr: Static Graph') != -1: return - if cur_name.find('ParamBase') != -1 or cur_name.find( - 'Parameter') != -1 or cur_name.find( - 'Variable') != -1 or cur_name.find( - 'control_flow') != -1 or cur_name.find( - 'contrib.mixed_precision') != -1: + if ( + cur_name.find('ParamBase') != -1 + or cur_name.find('Parameter') != -1 + or cur_name.find('Variable') != -1 + or cur_name.find('control_flow') != -1 + or cur_name.find('contrib.mixed_precision') != -1 + ): return if inspect.isclass(member): @@ -94,8 +99,9 @@ def visit_member(parent_name, member, func): if inspect.isclass(member): func(member, cur_name) for name, value in inspect.getmembers(member): - if hasattr(value, '__name__') and (not name.startswith("_") - or name == "__init__"): + if hasattr(value, '__name__') and ( + not name.startswith("_") or name == "__init__" + ): visit_member(cur_name, value, func) elif inspect.ismethoddescriptor(member): return @@ -106,11 +112,13 @@ def visit_member(parent_name, member, func): else: raise RuntimeError( "Unsupported generate signature of member, type {0}".format( - str(type(member)))) + str(type(member)) + ) + ) def is_primitive(instance): - int_types = (int, ) + int_types = (int,) pritimitive_types = int_types + (float, str) if isinstance(instance, pritimitive_types): return True @@ -171,8 +179,9 @@ def get_apis_with_and_without_core_ops(modules): api_with_ops = [] api_without_ops = [] for m in modules: - visit_all_module(importlib.import_module(m), - split_with_and_without_core_ops) + visit_all_module( + importlib.import_module(m), split_with_and_without_core_ops + ) return api_with_ops, api_without_ops @@ -189,7 +198,8 @@ def get_api_source_desc(modules): modules = sys.argv[2].split(",") if sys.argv[1] == '-c': api_with_ops, api_without_ops = get_apis_with_and_without_core_ops( - modules) + modules + ) print('api_with_ops:', len(api_with_ops)) print('\n'.join(api_with_ops)) @@ -203,9 +213,11 @@ def get_api_source_desc(modules): print(name, func_dict[name]) else: - print("""Usage: + print( + """Usage: 1. Count and list all operator-raleated APIs that contains append_op but not _legacy_C_ops.xx. python ./count_api_without_core_ops.py -c paddle 2. Print api and the md5 of source code of the api. python ./count_api_without_core_ops.py -p paddle - """) + """ + ) diff --git a/tools/coverage/coverage_diff.py b/tools/coverage/coverage_diff.py index 713afe1ebc31d..980b2cd336309 100644 --- a/tools/coverage/coverage_diff.py +++ b/tools/coverage/coverage_diff.py @@ -86,7 +86,7 @@ def get_info_file_lines(info_file, diff_file): current_file = line.lstrip('SF:') if current_file.startswith('/paddle/'): - current_file = current_file[len('/paddle/'):] + current_file = current_file[len('/paddle/') :] current_lines = diff_file_lines.get(current_file, []) diff --git a/tools/coverage/coverage_diff_list.py b/tools/coverage/coverage_diff_list.py index c57c1e9d96453..1fee1d3ae15a6 100644 --- a/tools/coverage/coverage_diff_list.py +++ b/tools/coverage/coverage_diff_list.py @@ -41,7 +41,7 @@ def filter_by(list_file, max_rate): name = split[0].strip() if name.startswith('/paddle/'): - name = name[len('/paddle/'):] + name = name[len('/paddle/') :] # rate diff --git a/tools/coverage/coverage_lines.py b/tools/coverage/coverage_lines.py index 3c5df9d88e8c4..10a0520c08ccf 100644 --- a/tools/coverage/coverage_lines.py +++ b/tools/coverage/coverage_lines.py @@ -29,8 +29,8 @@ def get_lines(info_file): Returns: float: Coverage rate. """ - hits = .0 - total = .0 + hits = 0.0 + total = 0.0 with open(info_file) as info_file: for line in info_file: @@ -68,10 +68,16 @@ def get_lines(info_file): actual = round(actual, 3) if actual < expected: - print('expected >= {} %, actual {} %, failed'.format( - round(expected * 100, 1), round(actual * 100, 1))) + print( + 'expected >= {} %, actual {} %, failed'.format( + round(expected * 100, 1), round(actual * 100, 1) + ) + ) exit(1) - print('expected >= {} %, actual {} %, passed'.format( - round(expected * 100, 1), round(actual * 100, 1))) + print( + 'expected >= {} %, actual {} %, passed'.format( + round(expected * 100, 1), round(actual * 100, 1) + ) + ) diff --git a/tools/coverage/cuda_clean.py b/tools/coverage/cuda_clean.py index 82bb6a553c955..fea2326413d01 100644 --- a/tools/coverage/cuda_clean.py +++ b/tools/coverage/cuda_clean.py @@ -65,7 +65,7 @@ def clean(pull_id): changed = [] for file in get_files(pull_id): - #changed.append('/paddle/build/{}.gcda'.format(file)) + # changed.append('/paddle/build/{}.gcda'.format(file)) changed.append(file) for parent, dirs, files in os.walk('/paddle/build/'): @@ -81,7 +81,7 @@ def clean(pull_id): if src_name not in changed: unused_file = parent + '/' + gcda - #print unused_file + # print unused_file os.remove(gcda) else: print(src_name) diff --git a/tools/coverage/gcda_clean.py b/tools/coverage/gcda_clean.py index 286f50eae9ab6..ab8ae0432384d 100644 --- a/tools/coverage/gcda_clean.py +++ b/tools/coverage/gcda_clean.py @@ -40,8 +40,11 @@ def get_pull(pull_id): repo = github.get_repo('PaddlePaddle/Paddle') except Exception as e: print(e) - print("get_repo error, retry {} times after {} secs.".format( - idx, idx * 10)) + print( + "get_repo error, retry {} times after {} secs.".format( + idx, idx * 10 + ) + ) else: break idx += 1 diff --git a/tools/coverage/python_coverage.py b/tools/coverage/python_coverage.py index d45fb4d58c591..53784a08abe61 100644 --- a/tools/coverage/python_coverage.py +++ b/tools/coverage/python_coverage.py @@ -33,8 +33,9 @@ clazz_filename = path.join(source, clazz_filename) if clazz_filename.startswith('/paddle/build/python/'): - clazz_filename = '/paddle/python/' + clazz_filename[ - len('/paddle/build/python/'):] + clazz_filename = ( + '/paddle/python/' + clazz_filename[len('/paddle/build/python/') :] + ) if not path.exists(clazz_filename): continue @@ -61,14 +62,20 @@ taken = int(taken) for _ in range(taken): - print('BRDA:{},{},{},{}'.format(line_number, 0, branch_index, - line_hits)) + print( + 'BRDA:{},{},{},{}'.format( + line_number, 0, branch_index, line_hits + ) + ) branch_index += 1 if line_missing_branches: for missing_branch in line_missing_branches.split(','): - print('BRDA:{},{},{},{}'.format(line_number, 0, - branch_index, 0)) + print( + 'BRDA:{},{},{},{}'.format( + line_number, 0, branch_index, 0 + ) + ) branch_index += 1 print('DA:{},{}'.format(line_number, line_hits)) diff --git a/tools/externalError/spider.py b/tools/externalError/spider.py index d0f9f48c89ff2..d6062e5d50a43 100644 --- a/tools/externalError/spider.py +++ b/tools/externalError/spider.py @@ -22,8 +22,8 @@ def parsing(externalErrorDesc): - #*********************************************************************************************# - #*********************************** CUDA Error Message **************************************# + # *********************************************************************************************# + # *********************************** CUDA Error Message **************************************# print("start crawling errorMessage for nvidia CUDA API--->") url = 'https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html#group__CUDART__TYPES_1g3f51e3575c2178246db0a94a430e0038' @@ -51,7 +51,8 @@ def parsing(externalErrorDesc): m_message = m_message.replace(list_a[idx], list_shape[idx]) m_message = m_message.replace( - '
Deprecated
', '') + '
Deprecated
', '' + ) res_span = r'()' res_span_detail = r'(.*?)' @@ -81,8 +82,8 @@ def parsing(externalErrorDesc): _Messages.message = "'%s'. %s" % (m_type[0], m_message) print("End crawling errorMessage for nvidia CUDA API!\n") - #***********************************************************************************************# - #*********************************** CURAND Error Message **************************************# + # ***********************************************************************************************# + # *********************************** CURAND Error Message **************************************# print("start crawling errorMessage for nvidia CURAND API--->") url = 'https://docs.nvidia.com/cuda/curand/group__HOST.html#group__HOST_1gb94a31d5c165858c96b6c18b70644437' @@ -112,8 +113,8 @@ def parsing(externalErrorDesc): _Messages.message = "'%s'. %s" % (m_type[0], m_message) print("End crawling errorMessage for nvidia CURAND API!\n") - #**************************************************************************************************# - #*********************************** CUDNN Error Message ******************************************# + # **************************************************************************************************# + # *********************************** CUDNN Error Message ******************************************# cudnnStatus_t = { "CUDNN_STATUS_SUCCESS": 0, "CUDNN_STATUS_NOT_INITIALIZED": 1, @@ -155,8 +156,9 @@ def parsing(externalErrorDesc): list_class_detail = re.findall(res_class_detail, m_message, re.S | re.M) assert len(list_class) == len(list_class_detail) for idx in range(len(list_class)): - m_message = m_message.replace(list_class[idx], - list_class_detail[idx]) + m_message = m_message.replace( + list_class[idx], list_class_detail[idx] + ) res_a = r'()' res_shape = r'(.*?)' @@ -189,8 +191,8 @@ def parsing(externalErrorDesc): _Messages.message = "'%s'. %s" % (error[0], m_message) print("End crawling errorMessage for nvidia CUDNN API!\n") - #*************************************************************************************************# - #*********************************** CUBLAS Error Message ****************************************# + # *************************************************************************************************# + # *********************************** CUBLAS Error Message ****************************************# cublasStatus_t = { "CUBLAS_STATUS_SUCCESS": 0, "CUBLAS_STATUS_NOT_INITIALIZED": 1, @@ -201,7 +203,7 @@ def parsing(externalErrorDesc): "CUBLAS_STATUS_EXECUTION_FAILED": 13, "CUBLAS_STATUS_INTERNAL_ERROR": 14, "CUBLAS_STATUS_NOT_SUPPORTED": 15, - "CUBLAS_STATUS_LICENSE_ERROR": 16 + "CUBLAS_STATUS_LICENSE_ERROR": 16, } print("start crawling errorMessage for nvidia CUBLAS API--->") @@ -243,8 +245,8 @@ def parsing(externalErrorDesc): _Messages.message = "'%s'. %s" % (error[0], m_message) print("End crawling errorMessage for nvidia CUBLAS API!\n") - #*************************************************************************************************# - #*********************************** CUSOLVER Error Message **************************************# + # *************************************************************************************************# + # *********************************** CUSOLVER Error Message **************************************# cusolverStatus_t = { "CUSOLVER_STATUS_SUCCESS": 0, "CUSOLVER_STATUS_NOT_INITIALIZED": 1, @@ -264,7 +266,7 @@ def parsing(externalErrorDesc): "CUSOLVER_STATUS_IRS_NOT_SUPPORTED": 15, "CUSOLVER_STATUS_IRS_OUT_OF_RANGE": 16, "CUSOLVER_STATUS_IRS_NRHS_NOT_SUPPORTED_FOR_REFINE_GMRES": 17, - "CUSOLVER_STATUS_IRS_INFOS_NOT_INITIALIZED": 18 + "CUSOLVER_STATUS_IRS_INFOS_NOT_INITIALIZED": 18, } print("start crawling errorMessage for nvidia CUSOLVER API--->") url = 'https://docs.nvidia.com/cuda/cusolver/index.html#cuSolverSPstatus' @@ -277,7 +279,9 @@ def parsing(externalErrorDesc): res_div = r'This is a status type returned by the library functions and.*?
(.*?)
' m_div = re.findall(res_div, html, re.S | re.M)[0] - res_dt = r'(.*?).*?colspan="1">(.*?)' + res_dt = ( + r'(.*?).*?colspan="1">(.*?)' + ) m_dt = re.findall(res_dt, m_div, re.S | re.M) for error in m_dt: @@ -304,20 +308,22 @@ def parsing(externalErrorDesc): res_strong = r'.*?' res_strong_detail = r'(.*?)' list_strong = re.findall(res_strong, m_message, re.S | re.M) - list_strong_detail = re.findall(res_strong_detail, m_message, - re.S | re.M) + list_strong_detail = re.findall( + res_strong_detail, m_message, re.S | re.M + ) assert len(list_strong) == len(list_strong_detail) for idx in range(len(list_strong)): - m_message = m_message.replace(list_strong[idx], - list_strong_detail[idx]) + m_message = m_message.replace( + list_strong[idx], list_strong_detail[idx] + ) _Messages = allMessageDesc.messages.add() _Messages.code = int(cusolverStatus_t[error[0]]) _Messages.message = "'%s'. %s" % (error[0], m_message) print("End crawling errorMessage for nvidia CUSOLVER API!\n") - #**********************************************************************************************# - #*************************************** NCCL error *******************************************# + # **********************************************************************************************# + # *************************************** NCCL error *******************************************# print("start crawling errorMessage for nvidia NCCL API--->") url = 'https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/types.html#ncclresult-t' allMessageDesc = externalErrorDesc.errors.add() @@ -335,8 +341,8 @@ def parsing(externalErrorDesc): _Messages.message = "'%s'. %s" % (error[0], m_message) print("End crawling errorMessage for nvidia NCCL API!\n") - #*************************************************************************************************# - #*********************************** CUFFT Error Message **************************************# + # *************************************************************************************************# + # *********************************** CUFFT Error Message **************************************# print("start crawling errorMessage for nvidia CUFFT API--->") url = 'https://docs.nvidia.com/cuda/cufft/index.html#cufftresult' @@ -346,8 +352,7 @@ def parsing(externalErrorDesc): html = urllib.request.urlopen(url).read().decode('utf-8') class CUFFTHTMLParser(HTMLParser): - '''CUFFTHTML Parser - ''' + '''CUFFTHTML Parser''' def handle_data(self, data): if 'typedef enum cufftResult_t' in data: @@ -355,8 +360,10 @@ def handle_data(self, data): status, code, desc = re.split('=|//', line.strip()) _Messages = allMessageDesc.messages.add() _Messages.code = int(code.strip(' ,')) - _Messages.message = "'%s'. %s" % (status.strip(), - desc.strip()) + _Messages.message = "'%s'. %s" % ( + status.strip(), + desc.strip(), + ) CUFFTHTMLParser().feed(html) diff --git a/tools/final_ut_parallel_rule.py b/tools/final_ut_parallel_rule.py index a178bc2bbd7f5..4d98eee41fb97 100644 --- a/tools/final_ut_parallel_rule.py +++ b/tools/final_ut_parallel_rule.py @@ -23,28 +23,44 @@ def classify_cases_by_mem(rootPath): """classify cases by mem""" case_filename = '%s/build/classify_case_by_cardNum.txt' % rootPath case_exec_100 = [ - 'test_conv_eltwiseadd_bn_fuse_pass', 'test_trt_convert_pool2d', - 'test_fc_fuse_pass', 'test_trt_convert_depthwise_conv2d', + 'test_conv_eltwiseadd_bn_fuse_pass', + 'test_trt_convert_pool2d', + 'test_fc_fuse_pass', + 'test_trt_convert_depthwise_conv2d', 'test_quant2_int8_resnet50_mkldnn', - 'test_conv_elementwise_add_act_fuse_pass', 'test_trt_convert_conv2d', - 'test_paddle_save_load', 'test_logical_op', 'test_nearest_interp_op', - 'test_pool2d_op', 'test_conv3d_transpose_op', 'test_lstmp_op', - 'test_cross_entropy2_op', 'test_sgd_op', 'test_imperative_ptq', - 'test_model', 'test_custom_relu_op_setup', 'test_dropout_op', - 'test_concat_op' - ] #木桶原理 70s-100s之间的case + 'test_conv_elementwise_add_act_fuse_pass', + 'test_trt_convert_conv2d', + 'test_paddle_save_load', + 'test_logical_op', + 'test_nearest_interp_op', + 'test_pool2d_op', + 'test_conv3d_transpose_op', + 'test_lstmp_op', + 'test_cross_entropy2_op', + 'test_sgd_op', + 'test_imperative_ptq', + 'test_model', + 'test_custom_relu_op_setup', + 'test_dropout_op', + 'test_concat_op', + ] # 木桶原理 70s-100s之间的case case_exec_200 = [ 'test_post_training_quantization_mnist', 'test_imperative_auto_mixed_precision', 'test_trt_dynamic_shape_ernie_fp16_ser_deser', - 'test_trt_dynamic_shape_ernie', 'test_layer_norm_op', - 'trt_quant_int8_yolov3_r50_test', 'test_gru_op', - 'test_post_training_quantization_while', 'test_mkldnn_log_softmax_op', - 'test_mkldnn_matmulv2_op', 'test_mkldnn_shape_op', + 'test_trt_dynamic_shape_ernie', + 'test_layer_norm_op', + 'trt_quant_int8_yolov3_r50_test', + 'test_gru_op', + 'test_post_training_quantization_while', + 'test_mkldnn_log_softmax_op', + 'test_mkldnn_matmulv2_op', + 'test_mkldnn_shape_op', 'interceptor_pipeline_short_path_test', - 'interceptor_pipeline_long_path_test', 'test_cpuonly_spawn' - ] #木桶原理 110s-200s之间的case 以及容易timeout + 'interceptor_pipeline_long_path_test', + 'test_cpuonly_spawn', + ] # 木桶原理 110s-200s之间的case 以及容易timeout case_always_timeout = [ 'test_quant2_int8_resnet50_channelwise_mkldnn', @@ -104,10 +120,10 @@ def classify_cases_by_mem(rootPath): if case not in new_lastest_mem: continue - #mem = 0 + # mem = 0 if new_lastest_mem[case]["mem_nvidia"] == 0: case_mem_0 = case_mem_0 + '|^' + case + '$' - #mem != 0 + # mem != 0 else: case_mem_1[case] = new_lastest_mem[case]["mem_nvidia"] diff --git a/tools/gen_ut_cmakelists.py b/tools/gen_ut_cmakelists.py index 62deef8c21b05..14f8e37626e75 100644 --- a/tools/gen_ut_cmakelists.py +++ b/tools/gen_ut_cmakelists.py @@ -44,9 +44,10 @@ def _process_envs(envs): processed_envs = [] for p in envs_parts: - assert " " not in p and \ - re.compile("^[a-zA-Z_][0-9a-zA-Z_]*=").search(p) is not None, \ - f"""The environment option format is wrong. The env variable name can only contains'a-z', 'A-Z', '0-9' and '_', + assert ( + " " not in p + and re.compile("^[a-zA-Z_][0-9a-zA-Z_]*=").search(p) is not None + ), f"""The environment option format is wrong. The env variable name can only contains'a-z', 'A-Z', '0-9' and '_', and the var can not contain space in either env names or values. However the var's format is '{p}'.""" @@ -95,8 +96,13 @@ def _proccess_archs(arch): arch = arch.upper().strip() if len(arch) > 0: for a in arch.split(";"): - assert a in ["GPU", "ROCM", "ASCEND", "ASCEND_CL", "XPU"], \ - f"""Supported arhc options are "GPU", "ROCM", "ASCEND" and "ASCEND_CL", "XPU", but the options is {a}""" + assert a in [ + "GPU", + "ROCM", + "ASCEND", + "ASCEND_CL", + "XPU", + ], f"""Supported arhc options are "GPU", "ROCM", "ASCEND" and "ASCEND_CL", "XPU", but the options is {a}""" archs += "WITH_" + a.upper() + " OR " arch = "(" + archs[:-4] + ")" else: @@ -121,7 +127,9 @@ def _process_os(os_): os_ = os_.upper() for p in os_.split(';'): assert p in [ - "WIN32", "APPLE", "LINUX" + "WIN32", + "APPLE", + "LINUX", ], f"""Supported os options are 'WIN32', 'APPLE' and 'LINUX', but the options is {p}""" os_ = os_.replace(";", " OR ") os_ = "(" + os_ + ")" @@ -133,8 +141,11 @@ def _process_os(os_): # check whether run_serial is 0, 1 or empty def _process_run_serial(run_serial): rs = run_serial.strip() - assert rs in ["1", "0", ""], \ - f"""the value of run_serial must be one of 0, 1 or empty. But this value is {rs}""" + assert rs in [ + "1", + "0", + "", + ], f"""the value of run_serial must be one of 0, 1 or empty. But this value is {rs}""" if rs == "": return "" return rs @@ -157,13 +168,15 @@ def _process_name(name, curdir): check whether name is with a legal format and check whther the test file exists. """ name = name.strip() - assert re.compile("^test_[0-9a-zA-Z_]+").search(name), \ - """If line is not the header of table, the test name must begin with "test_" """ \ + assert re.compile("^test_[0-9a-zA-Z_]+").search(name), ( + """If line is not the header of table, the test name must begin with "test_" """ """and the following substring must include at least one char of "0-9", "a-z", "A-Z" or "_".""" + ) filepath_prefix = os.path.join(curdir, name) suffix = [".py", ".sh"] - assert _file_with_extension(filepath_prefix, suffix), \ - f""" Please ensure the test file with the prefix '{filepath_prefix}' and one of the suffix {suffix} exists, because you specified a unittest named '{name}'""" + assert _file_with_extension( + filepath_prefix, suffix + ), f""" Please ensure the test file with the prefix '{filepath_prefix}' and one of the suffix {suffix} exists, because you specified a unittest named '{name}'""" return name @@ -182,14 +195,16 @@ def _norm_dirs(dirs): def _process_run_type(run_type): rt = run_type.strip() # completely match one of the strings: 'NIGHTLY', 'EXCLUSIVE', 'CINN', 'DIST', 'GPUPS', 'INFER', 'EXCLUSIVE:NIGHTLY' and 'DIST:NIGHTLY' - assert re.compile("^(NIGHTLY|EXCLUSIVE|CINN|DIST|GPUPS|INFER|EXCLUSIVE:NIGHTLY|DIST:NIGHTLY)$").search(rt), \ - f""" run_type must be one of 'NIGHTLY', 'EXCLUSIVE', 'CINN', 'DIST', 'GPUPS', 'INFER', 'EXCLUSIVE:NIGHTLY' and 'DIST:NIGHTLY'""" \ + assert re.compile( + "^(NIGHTLY|EXCLUSIVE|CINN|DIST|GPUPS|INFER|EXCLUSIVE:NIGHTLY|DIST:NIGHTLY)$" + ).search(rt), ( + f""" run_type must be one of 'NIGHTLY', 'EXCLUSIVE', 'CINN', 'DIST', 'GPUPS', 'INFER', 'EXCLUSIVE:NIGHTLY' and 'DIST:NIGHTLY'""" f"""but the run_type is {rt}""" + ) return rt -class DistUTPortManager(): - +class DistUTPortManager: def __init__(self, ignore_dirs=[]): self.dist_ut_port = 21200 self.assigned_ports = dict() @@ -212,13 +227,17 @@ def gset_port(self, test_name, port): ''' if test_name not in self.assigned_ports: self.assigned_ports[test_name] = port - self.dist_ut_port = max(self.dist_ut_port, - self.assigned_ports[test_name]) + self.dist_ut_port = max( + self.dist_ut_port, self.assigned_ports[test_name] + ) return self.assigned_ports[test_name] def process_dist_port_num(self, port_num): - assert re.compile("^[0-9]+$").search(port_num) and int(port_num) > 0 or port_num.strip()=="", \ - f"""port_num must be foramt as a positive integer or empty, but this port_num is '{port_num}'""" + assert ( + re.compile("^[0-9]+$").search(port_num) + and int(port_num) > 0 + or port_num.strip() == "" + ), f"""port_num must be foramt as a positive integer or empty, but this port_num is '{port_num}'""" port_num = port_num.strip() if len(port_num) == 0: return 0 @@ -242,7 +261,7 @@ def _init_dist_ut_ports_from_cmakefile(self, cmake_file_name): if matched is None: continue p = matched.span() - port = int(line[p[0]:p[1]].split("=")[-1]) + port = int(line[p[0] : p[1]].split("=")[-1]) # find the test name which the port belongs to for k in range(idx, 0, -1): @@ -252,8 +271,9 @@ def _init_dist_ut_ports_from_cmakefile(self, cmake_file_name): # matcg right tets name format, the name must start with 'test_' follwed bu at least one cahr of # '0-9'. 'a-z'. 'A-Z' or '_' - assert re.compile("^test_[0-9a-zA-Z_]+").search(name), \ - f'''we found a test for initial the latest dist_port but the test name '{name}' seems to be wrong + assert re.compile("^test_[0-9a-zA-Z_]+").search( + name + ), f'''we found a test for initial the latest dist_port but the test name '{name}' seems to be wrong at line {k-1}, in file {cmake_file_name} ''' self.gset_port(name, port) @@ -302,19 +322,35 @@ def parse_assigned_dist_ut_ports(self, current_work_dir, depth=0): # if such a drectory exists # step 1 - if len(self.last_test_name) > 0 and len( - self.last_test_cmake_file) > 0: + if ( + len(self.last_test_name) > 0 + and len(self.last_test_cmake_file) > 0 + ): with open( - self.last_test_cmake_file.replace( - "CMakeLists.txt", "testslist.csv")) as csv_file: + self.last_test_cmake_file.replace( + "CMakeLists.txt", "testslist.csv" + ) + ) as csv_file: found = False for line in csv_file.readlines(): - name, _, _, _, _, launcher, num_port, _, _, _ = line.strip( - ).split(",") + ( + name, + _, + _, + _, + _, + launcher, + num_port, + _, + _, + _, + ) = line.strip().split(",") if name == self.last_test_name: found = True break - assert found, f"no such test named '{self.last_test_name}' in file '{self.last_test_cmake_file}'" + assert ( + found + ), f"no such test named '{self.last_test_name}' in file '{self.last_test_cmake_file}'" if launcher[-2:] == ".sh": self.process_dist_port_num(num_port) @@ -336,8 +372,7 @@ def parse_assigned_dist_ut_ports(self, current_work_dir, depth=0): assert len(self.no_cmake_dirs) == 0, err_msg -class CMakeGenerator(): - +class CMakeGenerator: def __init__(self, current_dirs, ignore_dirs): self.processed_dirs = set() self.port_manager = DistUTPortManager(ignore_dirs) @@ -393,8 +428,18 @@ def _parse_line(self, line, curdir): endif()" """ - name, os_, archs, timeout, run_type, launcher, num_port, run_serial, envs, conditions = line.strip( - ).split(",") + ( + name, + os_, + archs, + timeout, + run_type, + launcher, + num_port, + run_serial, + envs, + conditions, + ) = line.strip().split(",") # name == "name" means the line being parsed is the header of the table # we should skip this line and return empty here. @@ -439,12 +484,15 @@ def _parse_line(self, line, curdir): "{envs}")%s endif() ''' - run_type_str = "" if len( - run_type) == 0 else f' LABELS "RUN_TYPE={run_type}"' - time_out_str = f' TIMEOUT "{timeout}"' if len( - timeout.strip()) > 0 else '' - run_serial_str = f' RUN_SERIAL {run_serial}' if len( - run_serial) > 0 else '' + run_type_str = ( + "" if len(run_type) == 0 else f' LABELS "RUN_TYPE={run_type}"' + ) + time_out_str = ( + f' TIMEOUT "{timeout}"' if len(timeout.strip()) > 0 else '' + ) + run_serial_str = ( + f' RUN_SERIAL {run_serial}' if len(run_serial) > 0 else '' + ) if len(time_out_str) > 0 or len(run_serial_str) > 0: set_properties = f''' set_tests_properties({name} PROPERTIES{time_out_str}{run_serial_str}{run_type_str})''' @@ -471,10 +519,14 @@ def _gen_cmakelists(self, current_work_dir, depth=0): if not os.path.isdir(c_path): continue self.processed_dirs.add(c_path) - if os.path.isfile(os.path.join(current_work_dir, c, "testslist.csv")) \ - or os.path.isfile(os.path.join(current_work_dir, c, "CMakeLists.txt")): - self._gen_cmakelists(os.path.join(current_work_dir, c), - depth + 1) + if os.path.isfile( + os.path.join(current_work_dir, c, "testslist.csv") + ) or os.path.isfile( + os.path.join(current_work_dir, c, "CMakeLists.txt") + ): + self._gen_cmakelists( + os.path.join(current_work_dir, c), depth + 1 + ) sub_dirs.append(c) if not os.path.isfile(os.path.join(current_work_dir, "testslist.csv")): @@ -506,10 +558,13 @@ def _gen_cmakelists(self, current_work_dir, depth=0): char_seq = "".join(char_seq) if char_seq != "".join(cmds.split()): - assert f"{current_work_dir}/CMakeLists.txt" not in self.modified_or_created_files, \ - f"the file {current_work_dir}/CMakeLists.txt are modified twice, which may cause some error" + assert ( + f"{current_work_dir}/CMakeLists.txt" + not in self.modified_or_created_files + ), f"the file {current_work_dir}/CMakeLists.txt are modified twice, which may cause some error" self.modified_or_created_files.append( - f"{current_work_dir}/CMakeLists.txt") + f"{current_work_dir}/CMakeLists.txt" + ) with open(f"{current_work_dir}/CMakeLists.txt", "w") as cmake_file: print(cmds, end="", file=cmake_file) @@ -523,8 +578,7 @@ def _gen_cmakelists(self, current_work_dir, depth=0): required=False, default=[], nargs="+", - help= - "Input a list of files named testslist.csv and output files named CmakeLists.txt in the same directories as the csv files respectly" + help="Input a list of files named testslist.csv and output files named CmakeLists.txt in the same directories as the csv files respectly", ) parser.add_argument( "--dirpaths", @@ -533,8 +587,7 @@ def _gen_cmakelists(self, current_work_dir, depth=0): required=False, default=[], nargs="+", - help= - "Input a list of dir paths including files named testslist.csv and output CmakeLists.txt in these directories respectly" + help="Input a list of dir paths including files named testslist.csv and output CmakeLists.txt in these directories respectly", ) parser.add_argument( "--ignore-cmake-dirs", @@ -543,18 +596,19 @@ def _gen_cmakelists(self, current_work_dir, depth=0): required=False, default=[], nargs='*', - help= - "To keep dist ports the same with old version cmake, old cmakelists.txt files are needed to parse dist_ports. If a directories are newly created and there is no cmakelists.txt file, the directory path must be specified by this option. The dirs are not recursive." + help="To keep dist ports the same with old version cmake, old cmakelists.txt files are needed to parse dist_ports. If a directories are newly created and there is no cmakelists.txt file, the directory path must be specified by this option. The dirs are not recursive.", ) args = parser.parse_args() - assert not (len(args.files) == 0 and len(args.dirpaths) - == 0), "You must provide at leate one file or dirpath" + assert not ( + len(args.files) == 0 and len(args.dirpaths) == 0 + ), "You must provide at leate one file or dirpath" current_work_dirs = [] if len(args.files) >= 1: for p in args.files: - assert os.path.basename( - p) == "testslist.csv", "you must input file named testslist.csv" + assert ( + os.path.basename(p) == "testslist.csv" + ), "you must input file named testslist.csv" current_work_dirs = current_work_dirs + [ os.path.dirname(file) for file in args.files ] diff --git a/tools/get_op_list.py b/tools/get_op_list.py index 05e666d2270f5..a6d726dc3096e 100644 --- a/tools/get_op_list.py +++ b/tools/get_op_list.py @@ -24,10 +24,12 @@ def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('--model_dir', - type=str, - default="", - help='Directory of the inference models.') + parser.add_argument( + '--model_dir', + type=str, + default="", + help='Directory of the inference models.', + ) return parser.parse_args() @@ -58,7 +60,7 @@ def get_model_phi_kernels(ops_set): args = parse_args() for root, dirs, files in os.walk(args.model_dir, topdown=True): for name in files: - if (re.match(r'.*pdmodel', name)): + if re.match(r'.*pdmodel', name): ops_set = get_model_ops(os.path.join(root, name)) phi_set = get_model_phi_kernels(ops_set) ops = ";".join(ops_set) diff --git a/tools/get_pr_ut.py b/tools/get_pr_ut.py index efe21b0639161..ba8dbeb1da572 100644 --- a/tools/get_pr_ut.py +++ b/tools/get_pr_ut.py @@ -31,7 +31,7 @@ class PRChecker(object): - """ PR Checker. """ + """PR Checker.""" def __init__(self): self.github = Github(os.getenv('GITHUB_API_TOKEN'), timeout=60) @@ -47,7 +47,7 @@ def __init__(self): self.full_case = False def init(self): - """ Get pull request. """ + """Get pull request.""" pr_id = os.getenv('GIT_PR_ID') if not pr_id: print('PREC No PR ID') @@ -72,7 +72,7 @@ def init(self): print('PREC test=allcase is set') self.full_case = True - #todo: exception + # todo: exception def __wget_with_retry(self, url): ix = 1 proxy = '--no-proxy' @@ -86,12 +86,15 @@ def __wget_with_retry(self, url): proxy = '--no-proxy' code = subprocess.call( 'wget -q {} --no-check-certificate {}'.format(proxy, url), - shell=True) + shell=True, + ) if code == 0: return True print( - 'PREC download {} error, retry {} time(s) after {} secs.[proxy_option={}]' - .format(url, ix, ix * 10, proxy)) + 'PREC download {} error, retry {} time(s) after {} secs.[proxy_option={}]'.format( + url, ix, ix * 10, proxy + ) + ) time.sleep(ix * 10) ix += 1 return False @@ -105,16 +108,19 @@ def __urlretrieve(self, url, filename): cur_proxy = urllib.request.ProxyHandler(without_proxy) else: cur_proxy = urllib.request.ProxyHandler(with_proxy) - opener = urllib.request.build_opener(cur_proxy, - urllib.request.HTTPHandler) + opener = urllib.request.build_opener( + cur_proxy, urllib.request.HTTPHandler + ) urllib.request.install_opener(opener) try: urllib.request.urlretrieve(url, filename) except Exception as e: print(e) print( - 'PREC download {} error, retry {} time(s) after {} secs.[proxy_option={}]' - .format(url, ix, ix * 10, cur_proxy)) + 'PREC download {} error, retry {} time(s) after {} secs.[proxy_option={}]'.format( + url, ix, ix * 10, cur_proxy + ) + ) continue else: return True @@ -124,7 +130,7 @@ def __urlretrieve(self, url, filename): return False def get_pr_files(self): - """ Get files in pull request. """ + """Get files in pull request.""" page = 0 file_dict = {} file_count = 0 @@ -135,20 +141,23 @@ def get_pr_files(self): for f in files: file_dict[PADDLE_ROOT + f.filename] = f.status file_count += 1 - if file_count == 30: #if pr file count = 31, nend to run all case + if file_count == 30: # if pr file count = 31, nend to run all case break page += 1 print("pr modify files: %s" % file_dict) return file_dict def get_is_white_file(self, filename): - """ judge is white file in pr's files. """ + """judge is white file in pr's files.""" isWhiteFile = False - not_white_files = (PADDLE_ROOT + 'cmake/', PADDLE_ROOT + 'patches/', - PADDLE_ROOT + 'tools/dockerfile/', - PADDLE_ROOT + 'tools/windows/', - PADDLE_ROOT + 'tools/test_runner.py', - PADDLE_ROOT + 'tools/parallel_UT_rule.py') + not_white_files = ( + PADDLE_ROOT + 'cmake/', + PADDLE_ROOT + 'patches/', + PADDLE_ROOT + 'tools/dockerfile/', + PADDLE_ROOT + 'tools/windows/', + PADDLE_ROOT + 'tools/test_runner.py', + PADDLE_ROOT + 'tools/parallel_UT_rule.py', + ) if 'cmakelist' in filename.lower(): isWhiteFile = False elif filename.startswith((not_white_files)): @@ -162,13 +171,16 @@ def __get_comment_by_filetype(self, content, filetype): if filetype == 'py': result = self.__get_comment_by_prog(content, self.py_prog_oneline) result.extend( - self.__get_comment_by_prog(content, self.py_prog_multiline_a)) + self.__get_comment_by_prog(content, self.py_prog_multiline_a) + ) result.extend( - self.__get_comment_by_prog(content, self.py_prog_multiline_b)) + self.__get_comment_by_prog(content, self.py_prog_multiline_b) + ) if filetype == 'cc': result = self.__get_comment_by_prog(content, self.cc_prog_oneline) result.extend( - self.__get_comment_by_prog(content, self.cc_prog_multiline)) + self.__get_comment_by_prog(content, self.cc_prog_multiline) + ) return result def __get_comment_by_prog(self, content, prog): @@ -181,15 +193,15 @@ def __get_comment_by_prog(self, content, prog): return result def get_comment_of_file(self, f): - #content = self.repo.get_contents(f.replace(PADDLE_ROOT, ''), 'pull/').decoded_content - #todo: get file from github + # content = self.repo.get_contents(f.replace(PADDLE_ROOT, ''), 'pull/').decoded_content + # todo: get file from github with open(f, encoding="utf-8") as fd: lines = fd.readlines() lineno = 1 inputs = '' for line in lines: - #for line in content.split('\n'): - #input += str(lineno) + '|' + line + '\n' + # for line in content.split('\n'): + # input += str(lineno) + '|' + line + '\n' inputs += str(lineno) + '|' + line lineno += 1 fietype = '' @@ -227,8 +239,9 @@ def get_pr_diff_lines(self): end += 1 if data[ix][0] == '+': line_list = file_to_diff_lines.get(filename) - line = '{}{}'.format(lineno, - data[ix].replace('+', '|', 1)) + line = '{}{}'.format( + lineno, data[ix].replace('+', '|', 1) + ) if line_list: line_list.append(line) else: @@ -254,9 +267,11 @@ def is_only_comment(self, f): return True def get_all_count(self): - p = subprocess.Popen("cd {}build && ctest -N".format(PADDLE_ROOT), - shell=True, - stdout=subprocess.PIPE) + p = subprocess.Popen( + "cd {}build && ctest -N".format(PADDLE_ROOT), + shell=True, + stdout=subprocess.PIPE, + ) out, err = p.communicate() for line in out.splitlines(): if 'Total Tests:' in str(line): @@ -264,12 +279,13 @@ def get_all_count(self): return int(all_counts) def file_is_unnit_test(self, filename): - #get all testcases by ctest-N + # get all testcases by ctest-N all_ut_file = '%s/build/all_ut_file' % PADDLE_ROOT os.system( "cd %s/build && ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d' > %s" - % (PADDLE_ROOT, all_ut_file)) - #determine whether filename is in all_ut_case + % (PADDLE_ROOT, all_ut_file) + ) + # determine whether filename is in all_ut_case with open(all_ut_file, 'r') as f: (filepath, tempfilename) = os.path.split(filename) for f_file in f: @@ -279,7 +295,7 @@ def file_is_unnit_test(self, filename): return False def get_pr_ut(self): - """ Get unit tests in pull request. """ + """Get unit tests in pull request.""" if self.full_case: return '' check_added_ut = False @@ -288,7 +304,8 @@ def get_pr_ut(self): ret = self.__urlretrieve( 'https://paddle-docker-tar.bj.bcebos.com/pre_test/ut_file_map.json', - 'ut_file_map.json') + 'ut_file_map.json', + ) if not ret: print('PREC download file_ut.json failed') exit(1) @@ -303,19 +320,23 @@ def get_pr_ut(self): filterFiles = [] file_list = [] file_dict = self.get_pr_files() - if len(file_dict) == 30: #if pr file count = 31, nend to run all case + if len(file_dict) == 30: # if pr file count = 31, nend to run all case return '' for filename in file_dict: if filename.startswith(PADDLE_ROOT + 'python/'): file_list.append(filename) elif filename.startswith(PADDLE_ROOT + 'paddle/'): - if filename.startswith((PADDLE_ROOT + 'paddle/infrt', - PADDLE_ROOT + 'paddle/utils')): + if filename.startswith( + (PADDLE_ROOT + 'paddle/infrt', PADDLE_ROOT + 'paddle/utils') + ): filterFiles.append(filename) elif filename.startswith(PADDLE_ROOT + 'paddle/scripts'): if filename.startswith( - (PADDLE_ROOT + 'paddle/scripts/paddle_build.sh', - PADDLE_ROOT + 'paddle/scripts/paddle_build.bat')): + ( + PADDLE_ROOT + 'paddle/scripts/paddle_build.sh', + PADDLE_ROOT + 'paddle/scripts/paddle_build.bat', + ) + ): file_list.append(filename) else: filterFiles.append(filename) @@ -334,7 +355,8 @@ def get_pr_ut(self): ut_list.append('filterfiles_placeholder') ret = self.__urlretrieve( 'https://paddle-docker-tar.bj.bcebos.com/pre_test/prec_delta', - 'prec_delta') + 'prec_delta', + ) if ret: with open('prec_delta') as delta: for ut in delta: @@ -343,17 +365,25 @@ def get_pr_ut(self): print('PREC download prec_delta failed') exit(1) PRECISION_TEST_Cases_ratio = format( - float(len(ut_list)) / float(self.get_all_count()), '.2f') + float(len(ut_list)) / float(self.get_all_count()), '.2f' + ) print("filterFiles: %s" % filterFiles) print("ipipe_log_param_PRECISION_TEST: true") - print("ipipe_log_param_PRECISION_TEST_Cases_count: %s" % - len(ut_list)) - print("ipipe_log_param_PRECISION_TEST_Cases_ratio: %s" % - PRECISION_TEST_Cases_ratio) + print( + "ipipe_log_param_PRECISION_TEST_Cases_count: %s" % len(ut_list) + ) + print( + "ipipe_log_param_PRECISION_TEST_Cases_ratio: %s" + % PRECISION_TEST_Cases_ratio + ) return '\n'.join(ut_list) else: for f in file_list: - if current_system == "Darwin" or current_system == "Windows" or self.suffix == ".py3": + if ( + current_system == "Darwin" + or current_system == "Windows" + or self.suffix == ".py3" + ): f_judge = f.replace(PADDLE_ROOT, '/paddle/', 1) f_judge = f_judge.replace('//', '/') else: @@ -362,25 +392,34 @@ def get_pr_ut(self): if f_judge.endswith('.md'): ut_list.append('md_placeholder') onlyCommentsFilesOrXpu.append(f_judge) - elif 'tests/unittests/xpu' in f_judge or 'tests/unittests/npu' in f_judge or 'op_npu.cc' in f_judge: + elif ( + 'tests/unittests/xpu' in f_judge + or 'tests/unittests/npu' in f_judge + or 'op_npu.cc' in f_judge + ): ut_list.append('xpu_npu_placeholder') onlyCommentsFilesOrXpu.append(f_judge) elif f_judge.endswith(('.h', '.cu', '.cc', '.py')): - #determine whether the new added file is a member of added_ut + # determine whether the new added file is a member of added_ut if file_dict[f] in ['added']: f_judge_in_added_ut = False - with open('{}/added_ut'.format( - PADDLE_ROOT)) as utfile: - (filepath, - tempfilename) = os.path.split(f_judge) + with open( + '{}/added_ut'.format(PADDLE_ROOT) + ) as utfile: + (filepath, tempfilename) = os.path.split( + f_judge + ) for f_file in utfile: - if f_file.strip('\n') == tempfilename.split( - ".")[0]: + if ( + f_file.strip('\n') + == tempfilename.split(".")[0] + ): f_judge_in_added_ut = True if f_judge_in_added_ut == True: print( "Adding new unit tests not hit mapFiles: %s" - % f_judge) + % f_judge + ) else: notHitMapFiles.append(f_judge) elif file_dict[f] in ['removed']: @@ -394,9 +433,11 @@ def get_pr_ut(self): else: notHitMapFiles.append(f_judge) else: - notHitMapFiles.append( - f_judge) if file_dict[f] != 'removed' else print( - "remove file not hit mapFiles: %s" % f_judge) + notHitMapFiles.append(f_judge) if file_dict[ + f + ] != 'removed' else print( + "remove file not hit mapFiles: %s" % f_judge + ) else: if file_dict[f] not in ['removed']: if self.is_only_comment(f): @@ -420,7 +461,8 @@ def get_pr_ut(self): if ut_list: ret = self.__urlretrieve( 'https://paddle-docker-tar.bj.bcebos.com/pre_test/prec_delta', - 'prec_delta') + 'prec_delta', + ) if ret: with open('prec_delta') as delta: for ut in delta: @@ -430,13 +472,17 @@ def get_pr_ut(self): exit(1) print("hitMapFiles: %s" % hitMapFiles) print("ipipe_log_param_PRECISION_TEST: true") - print("ipipe_log_param_PRECISION_TEST_Cases_count: %s" % - len(ut_list)) + print( + "ipipe_log_param_PRECISION_TEST_Cases_count: %s" + % len(ut_list) + ) PRECISION_TEST_Cases_ratio = format( - float(len(ut_list)) / float(self.get_all_count()), - '.2f') - print("ipipe_log_param_PRECISION_TEST_Cases_ratio: %s" % - PRECISION_TEST_Cases_ratio) + float(len(ut_list)) / float(self.get_all_count()), '.2f' + ) + print( + "ipipe_log_param_PRECISION_TEST_Cases_ratio: %s" + % PRECISION_TEST_Cases_ratio + ) if len(filterFiles) != 0: print("filterFiles: %s" % filterFiles) return '\n'.join(ut_list) diff --git a/tools/get_single_test_cov.py b/tools/get_single_test_cov.py index 9fe6554ee6e32..293fed5e4b3b9 100644 --- a/tools/get_single_test_cov.py +++ b/tools/get_single_test_cov.py @@ -35,10 +35,16 @@ def getFNDAFile(rootPath, test): def analysisFNDAFile(rootPath, test): - related_ut_map_file = '%s/build/ut_map/%s/related_%s.txt' % (rootPath, test, - test) + related_ut_map_file = '%s/build/ut_map/%s/related_%s.txt' % ( + rootPath, + test, + test, + ) notrelated_ut_map_file = '%s/build/ut_map/%s/notrelated_%s.txt' % ( - rootPath, test, test) + rootPath, + test, + test, + ) os.system('touch %s' % related_ut_map_file) os.system('touch %s' % notrelated_ut_map_file) fn_filename = '%s/build/ut_map/%s/fnda.tmp' % (rootPath, test) @@ -60,22 +66,30 @@ def analysisFNDAFile(rootPath, test): fn = message_list[i] matchObj = re.match( r'(.*)Maker(.*)|(.*)Touch(.*)Regist(.*)|(.*)Touch(.*)JitKernel(.*)|(.*)converterC2Ev(.*)', - fn, re.I) + fn, + re.I, + ) if matchObj == None: OP_REGIST = False break if OP_REGIST == False: related_file_list.append(clazz_filename) - os.system('echo %s >> %s' % - (clazz_filename, related_ut_map_file)) + os.system( + 'echo %s >> %s' % (clazz_filename, related_ut_map_file) + ) else: - os.system('echo %s >> %s' % - (clazz_filename, notrelated_ut_map_file)) + os.system( + 'echo %s >> %s' % (clazz_filename, notrelated_ut_map_file) + ) else: if clazz_filename != '': - if clazz_filename not in related_file_list: # xx.pb.cc in RELATED xx.pb.h not in RELATED - os.system('echo %s >> %s' % - (clazz_filename, notrelated_ut_map_file)) + if ( + clazz_filename not in related_file_list + ): # xx.pb.cc in RELATED xx.pb.h not in RELATED + os.system( + 'echo %s >> %s' + % (clazz_filename, notrelated_ut_map_file) + ) f.close() @@ -83,10 +97,12 @@ def getCovinfo(rootPath, test): ut_map_path = '%s/build/ut_map/%s' % (rootPath, test) os.system( 'cd %s && lcov --capture -d . -o coverage.info --rc lcov_branch_coverage=0 > /dev/null 2>&1' - % ut_map_path) + % ut_map_path + ) os.system( "cd %s && lcov --extract coverage.info '/paddle/paddle/fluid/framework/*' '/paddle/paddle/fluid/imperative/*' '/paddle/paddle/fluid/inference/*' '/paddle/paddle/fluid/memory/*' '/paddle/paddle/fluid/operators/*' '/paddle/paddle/fluid/string/*' '/paddle/paddle/fluid/distributed/*' '/paddle/paddle/fluid/platform/*' '/paddle/paddle/fluid/pybind/*' '/paddle/build/*' -o coverage.info.tmp --rc lcov_branch_coverage=0 > /dev/null 2>&1" - % ut_map_path) + % ut_map_path + ) os.system('rm -rf %s/paddle' % ut_map_path) os.system('rm -rf %s/coverage.info' % ut_map_path) getFNDAFile(rootPath, test) diff --git a/tools/get_ut_file_map.py b/tools/get_ut_file_map.py index 5afc6b03f2f96..2bd777cc8276e 100644 --- a/tools/get_ut_file_map.py +++ b/tools/get_ut_file_map.py @@ -35,7 +35,8 @@ def get_all_uts(rootPath): all_uts_paddle = '%s/build/all_uts_paddle' % rootPath os.system( r'cd %s/build && ctest -N -V | grep -Ei "Test[ \t]+#" | grep -oEi "\w+$" > %s' - % (rootPath, all_uts_paddle)) + % (rootPath, all_uts_paddle) + ) def remove_useless_file(rootPath): @@ -75,10 +76,11 @@ def handle_ut_file_map(rootPath): continue elif line.startswith('/paddle/build'): source_file = line.replace('/build', '') - #source_file = re.sub('.pb.*', '.proto', source_file) + # source_file = re.sub('.pb.*', '.proto', source_file) elif 'precise test map fileeee:' in line: - source_file = line.split( - 'precise test map fileeee:')[1].strip() + source_file = line.split('precise test map fileeee:')[ + 1 + ].strip() else: source_file = line if source_file not in ut_file_map: @@ -193,7 +195,8 @@ def ut_file_map_supplement(rootPath): if not os.path.exists(filename) and ut not in prec_delta_new_list: prec_delta_new_list.append(ut) prec_delta_new_list.append( - 'test_py_reader_error_msg') #add a python case for pycoverage + 'test_py_reader_error_msg' + ) # add a python case for pycoverage prec_delta_file = open("/pre_test/prec_delta", 'w') for ut in prec_delta_new_list: prec_delta_file.write(ut + '\n') diff --git a/tools/get_ut_mem_map.py b/tools/get_ut_mem_map.py index dd1511a7cf293..703f0ab98ac25 100644 --- a/tools/get_ut_mem_map.py +++ b/tools/get_ut_mem_map.py @@ -39,20 +39,30 @@ def get_ut_mem(rootPath): mem_reserved = round( float( line.split(' : Reserved = ')[1].split( - ', Allocated = ')[0]), 2) + ', Allocated = ' + )[0] + ), + 2, + ) if mem_reserved > mem_reserved1: mem_reserved1 = mem_reserved if 'MAX_GPU_MEMORY_USE=' in line: mem_nvidia = round( float( - line.split('MAX_GPU_MEMORY_USE=')[1].split('\\n') - [0].strip()), 2) + line.split('MAX_GPU_MEMORY_USE=')[1] + .split('\\n')[0] + .strip() + ), + 2, + ) if mem_nvidia > mem_nvidia1: mem_nvidia1 = mem_nvidia if 'Total Test time (real)' in line: caseTime = float( - line.split('Total Test time (real) =')[1].split('sec') - [0].strip()) + line.split('Total Test time (real) =')[1] + .split('sec')[0] + .strip() + ) if mem_reserved1 != -1: case_dic[ut]['mem_reserved'] = mem_reserved1 if mem_nvidia1 != -1: diff --git a/tools/group_case_for_parallel.py b/tools/group_case_for_parallel.py index e2c8eb9ff04e9..8dd5f6f4ca263 100644 --- a/tools/group_case_for_parallel.py +++ b/tools/group_case_for_parallel.py @@ -19,17 +19,22 @@ def group_case_for_parallel(rootPath): """group cases""" - #wget file + # wget file for filename in [ - 'nightly_case', 'single_card_tests', 'single_card_tests_mem0', - 'multiple_card_tests', 'multiple_card_tests_mem0', - 'exclusive_card_tests', 'exclusive_card_tests_mem0' + 'nightly_case', + 'single_card_tests', + 'single_card_tests_mem0', + 'multiple_card_tests', + 'multiple_card_tests_mem0', + 'exclusive_card_tests', + 'exclusive_card_tests_mem0', ]: os.system( 'cd %s/tools && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test_bak/%s --no-check-certificate' - % (rootPath, filename)) + % (rootPath, filename) + ) - #get nightly tests + # get nightly tests nightly_tests_file = open('%s/tools/nightly_case' % rootPath, 'r') nightly_tests = nightly_tests_file.read().strip().split('\n') nightly_tests_file.close() @@ -40,7 +45,7 @@ def group_case_for_parallel(rootPath): '%s/tools/multiple_card_tests_mem0' % rootPath, '%s/tools/multiple_card_tests' % rootPath, '%s/tools/exclusive_card_tests_mem0' % rootPath, - '%s/tools/exclusive_card_tests' % rootPath + '%s/tools/exclusive_card_tests' % rootPath, ] case_file = '%s/build/ut_list' % rootPath if os.path.exists(case_file): @@ -67,12 +72,14 @@ def group_case_for_parallel(rootPath): for line in lines: case_line_list = line.replace('^', '').replace('|', '').split('$') new_case_line_list = list( - set(all_need_run_cases).intersection(set(case_line_list))) + set(all_need_run_cases).intersection(set(case_line_list)) + ) if len(new_case_line_list) != 0: new_case_file_list.append(new_case_line_list) all_group_case += new_case_line_list all_need_run_cases = list( - set(all_need_run_cases).difference(set(all_group_case))) + set(all_need_run_cases).difference(set(all_group_case)) + ) for line in new_case_file_list: cases = '$|^'.join(case for case in line) @@ -81,7 +88,7 @@ def group_case_for_parallel(rootPath): fi.close() new_f.close() - #no parallel cases + # no parallel cases cases = '^job' if len(all_need_run_cases) != 0: for case in all_need_run_cases: diff --git a/tools/handle_h_cu_file.py b/tools/handle_h_cu_file.py index 2dfa014cf95a0..84fba9f28202f 100644 --- a/tools/handle_h_cu_file.py +++ b/tools/handle_h_cu_file.py @@ -31,9 +31,12 @@ def worker(fun): def threadPool(threadPoolNum): threadPool = [] for i in range(threadPoolNum): - thread = threading.Thread(target=worker, args={ - doFun, - }) + thread = threading.Thread( + target=worker, + args={ + doFun, + }, + ) thread.daemon = True threadPool.append(thread) return threadPool @@ -60,10 +63,12 @@ def insert_pile_to_h_file(rootPath): os.system('echo "\n#include \n" >> %s' % line) os.system( 'echo "__attribute__((constructor)) static void calledFirst%s()\n{" >> %s' - % (func, line)) + % (func, line) + ) os.system( 'echo \' printf("precise test map fileeee: %%s\\\\n", __FILE__);\n}\' >> %s' - % line) + % line + ) os.system('echo "\n#endif" >> %s' % line) @@ -87,7 +92,8 @@ def get_h_cu_file(file_path): ut = filename.replace('^', '').replace('$', '').replace('.log', '') os.system( "cat %s/%s | grep 'precise test map fileeee:'| uniq >> %s/build/ut_map/%s/related_%s.txt" - % (dir_path, filename, rootPath, ut, ut)) + % (dir_path, filename, rootPath, ut, ut) + ) def doFun(file_path): diff --git a/tools/infrt/fake_models/multi_fc.py b/tools/infrt/fake_models/multi_fc.py index 31b6efd8dcb11..29b941d5437ca 100644 --- a/tools/infrt/fake_models/multi_fc.py +++ b/tools/infrt/fake_models/multi_fc.py @@ -24,18 +24,22 @@ a = fluid.layers.data(name="A", shape=[-1, size], dtype='float32') label = fluid.layers.data(name="label", shape=[size], dtype='float32') -fc_out = fluid.layers.fc(input=a, - size=size, - act="relu", - bias_attr=fluid.ParamAttr(name="fc_bias"), - num_flatten_dims=1) +fc_out = fluid.layers.fc( + input=a, + size=size, + act="relu", + bias_attr=fluid.ParamAttr(name="fc_bias"), + num_flatten_dims=1, +) for i in range(num_layers - 1): - fc_out = fluid.layers.fc(input=fc_out, - size=size, - act="relu", - bias_attr=fluid.ParamAttr(name="fc_bias"), - num_flatten_dims=1) + fc_out = fluid.layers.fc( + input=fc_out, + size=size, + act="relu", + bias_attr=fluid.ParamAttr(name="fc_bias"), + num_flatten_dims=1, + ) cost = fluid.layers.square_error_cost(fc_out, label) avg_cost = fluid.layers.mean(cost) @@ -49,7 +53,14 @@ exe.run(fluid.default_startup_program()) fluid.io.save_inference_model("./multi_fc_model", [a.name], [fc_out], exe) -fluid.io.save_inference_model("./multi_fc_model", [a.name], [fc_out], exe, None, - "fc.pdmodel", "fc.pdiparams") +fluid.io.save_inference_model( + "./multi_fc_model", + [a.name], + [fc_out], + exe, + None, + "fc.pdmodel", + "fc.pdiparams", +) print('output name', fc_out.name) diff --git a/tools/infrt/generate_pd_op_dialect_from_paddle_op_maker.py b/tools/infrt/generate_pd_op_dialect_from_paddle_op_maker.py index 7b97a06619eb3..313d844d6f6ae 100644 --- a/tools/infrt/generate_pd_op_dialect_from_paddle_op_maker.py +++ b/tools/infrt/generate_pd_op_dialect_from_paddle_op_maker.py @@ -82,8 +82,11 @@ def get_attrs_info(op_proto, op_attrs_proto): attrs_info[attr_name] = {} attrs_info[attr_name][TYPE] = attr_proto.type attrs_info[attr_name][GENERATED] = attr_proto.generated - attrs_info[attr_name][DEFAULT_VALUE] = attrs_default_values[ - attr_name] if attr_name in attrs_default_values else None + attrs_info[attr_name][DEFAULT_VALUE] = ( + attrs_default_values[attr_name] + if attr_name in attrs_default_values + else None + ) attrs_info[attr_name][EXTRA] = attr_proto.extra attrs_info[attr_name][QUANT] = attr_proto.quant return attrs_info @@ -118,20 +121,26 @@ def generate_all_ops_inputs_outputs_map(op_descs): inputs = list() outpus = list() for input_ in op_proto[INPUTS]: - if op_proto[INPUTS][input_][EXTRA] != True and op_proto[INPUTS][ - input_][INTERMEDIATE] != True: + if ( + op_proto[INPUTS][input_][EXTRA] != True + and op_proto[INPUTS][input_][INTERMEDIATE] != True + ): inputs.append(input_) for output_ in op_proto[OUTPUTS]: - if op_proto[OUTPUTS][output_][EXTRA] != True and op_proto[OUTPUTS][ - output_][INTERMEDIATE] != True: + if ( + op_proto[OUTPUTS][output_][EXTRA] != True + and op_proto[OUTPUTS][output_][INTERMEDIATE] != True + ): outpus.append(output_) ops_inputs_map[op_type] = inputs ops_outputs_map[op_type] = outpus # 2. Generate Cpp style map str cpp_style_ops_inputs_map_str = "" - start_ = "#include \n#include \n#include \n" + \ - "const std::unordered_map> pd_dialect_inputs_info_map_ = {\n" + start_ = ( + "#include \n#include \n#include \n" + + "const std::unordered_map> pd_dialect_inputs_info_map_ = {\n" + ) ops_inputs_str = "" for ele in ops_inputs_map.items(): op_name = ele[0] @@ -139,11 +148,14 @@ def generate_all_ops_inputs_outputs_map(op_descs): op_inputs_str = "{" input_idx = 0 for op_input in op_inputs: - op_input_str = '{left_brace}"{op_input}", {input_idx}{right_brace}, '.format( - left_brace="{", - op_input=op_input, - input_idx=input_idx, - right_brace="}") + op_input_str = ( + '{left_brace}"{op_input}", {input_idx}{right_brace}, '.format( + left_brace="{", + op_input=op_input, + input_idx=input_idx, + right_brace="}", + ) + ) input_idx = input_idx + 1 op_inputs_str = op_inputs_str + op_input_str op_inputs_str = op_inputs_str[:-2] + "}" @@ -151,7 +163,8 @@ def generate_all_ops_inputs_outputs_map(op_descs): left_brace="{", op_name=op_name, op_inputs=op_inputs_str, - right_brace="}") + right_brace="}", + ) ops_inputs_str = ops_inputs_str + " " + pair ops_inputs_str = ops_inputs_str[:-2] cpp_style_ops_inputs_map_str = start_ + ops_inputs_str + "\n};" @@ -165,11 +178,14 @@ def generate_all_ops_inputs_outputs_map(op_descs): op_outputs_str = "{" output_idx = 0 for op_output in op_outputs: - op_output_str = '{left_brace}"{op_output}", {output_idx}{right_brace}, '.format( - left_brace="{", - op_output=op_output, - output_idx=output_idx, - right_brace="}") + op_output_str = ( + '{left_brace}"{op_output}", {output_idx}{right_brace}, '.format( + left_brace="{", + op_output=op_output, + output_idx=output_idx, + right_brace="}", + ) + ) output_idx = output_idx + 1 op_outputs_str = op_outputs_str + op_output_str op_outputs_str = op_outputs_str[:-2] + "}" @@ -177,7 +193,8 @@ def generate_all_ops_inputs_outputs_map(op_descs): left_brace="{", op_name=op_name, op_outputs=op_outputs_str, - right_brace="}") + right_brace="}", + ) ops_outputs_str = ops_outputs_str + " " + pair ops_outputs_str = ops_outputs_str[:-2] cpp_style_ops_outputs_map_str = start_ + ops_outputs_str + "\n};" @@ -196,9 +213,11 @@ def get_constraint(op_type, op_proto): optional_input_num_ = 0 for input_ in op_proto[INPUTS]: - if op_proto[INPUTS][input_][EXTRA] != True and op_proto[INPUTS][input_][ - INTERMEDIATE] != True and op_proto[INPUTS][input_][ - DISPENSABLE] == True: + if ( + op_proto[INPUTS][input_][EXTRA] != True + and op_proto[INPUTS][input_][INTERMEDIATE] != True + and op_proto[INPUTS][input_][DISPENSABLE] == True + ): optional_input_num_ += 1 if optional_input_num_ > 1: constraint += ", AttrSizedOperandSegments" @@ -237,11 +256,21 @@ def convert_op_proto_into_mlir(op_descs): # 2. Op dialect # skip list ( ops whose dialect can not be generated automatically will be recorded here) skipped_op_list = [ - "cos_sim", "fused_embedding_seq_pool", "cosh", "kron", "recurrent", - "while", "conditional_block", "set_value", "run_program" + "cos_sim", + "fused_embedding_seq_pool", + "cosh", + "kron", + "recurrent", + "while", + "conditional_block", + "set_value", + "run_program", ] skipped_attr_list = [ - "trainable_statistics", "use_global_stats", "is_test", "use_quantizer" + "trainable_statistics", + "use_global_stats", + "is_test", + "use_quantizer", ] original_ops_ = get_original_ops() @@ -256,7 +285,8 @@ def convert_op_proto_into_mlir(op_descs): op_type_capitalize=op_type.capitalize(), constraint=constraint_, op_type=op_type, - left_brace="{") + left_brace="{", + ) SUMMARY = ' let summary = "{} op";\n'.format(op_type) # 2.2 Description @@ -265,27 +295,44 @@ def convert_op_proto_into_mlir(op_descs): for line_ in origin_contents: contents = contents + " {}\n".format(line_) DESCRIPTION = " let description = [{left_brace}\n{description} {right_brace}];\n".format( - left_brace="{", description=contents, right_brace="}") + left_brace="{", description=contents, right_brace="}" + ) # 2.3 arguments info ARGUMENTS = "" - if (len(op_proto[INPUTS]) > 0 or len(op_proto[ATTRS]) > 0): + if len(op_proto[INPUTS]) > 0 or len(op_proto[ATTRS]) > 0: ARGUMENTS = " let arguments = (ins " # 2.3.1 inputs for input_ in op_proto[INPUTS]: - if op_proto[INPUTS][input_][EXTRA] != True and op_proto[INPUTS][ - input_][INTERMEDIATE] != True: + if ( + op_proto[INPUTS][input_][EXTRA] != True + and op_proto[INPUTS][input_][INTERMEDIATE] != True + ): if op_proto[INPUTS][input_][DISPENSABLE] != True: if op_proto[INPUTS][input_][DUPLICABLE] != True: - ARGUMENTS = ARGUMENTS + " PD_Tensor:$" + input_ + "," + ARGUMENTS = ( + ARGUMENTS + " PD_Tensor:$" + input_ + "," + ) else: - ARGUMENTS = ARGUMENTS + " PD_Tensor_Array:$" + input_ + "," + ARGUMENTS = ( + ARGUMENTS + " PD_Tensor_Array:$" + input_ + "," + ) else: if op_proto[INPUTS][input_][DUPLICABLE] != True: - ARGUMENTS = ARGUMENTS + " Optional:$" + input_ + "," + ARGUMENTS = ( + ARGUMENTS + + " Optional:$" + + input_ + + "," + ) else: - ARGUMENTS = ARGUMENTS + " Optional:$" + input_ + "," + ARGUMENTS = ( + ARGUMENTS + + " Optional:$" + + input_ + + "," + ) # unsupported: BLOCK = 8; BLOCKS = 10; attr_mlir_converter = { @@ -298,70 +345,104 @@ def convert_op_proto_into_mlir(op_descs): 6: 'BoolAttr', 7: 'BoolArrayAttr', 9: 'SI64Attr', - 11: 'I64ArrayAttr' + 11: 'I64ArrayAttr', } # 2.3.2 attributes for attr in op_proto[ATTRS]: - if (op_proto[ATTRS][attr][EXTRA] - == True) or (attr in skipped_attr_list): + if (op_proto[ATTRS][attr][EXTRA] == True) or ( + attr in skipped_attr_list + ): continue if op_proto[ATTRS][attr][DEFAULT_VALUE] != None: if op_proto[ATTRS][attr][TYPE] in attr_mlir_converter: default_value = str( - op_proto[ATTRS][attr][DEFAULT_VALUE]) - if (attr_mlir_converter[op_proto[ATTRS][attr][TYPE]] - in [ - 'I32ArrayAttr', 'F32ArrayAttr', - 'StrArrayAttr', 'BoolArrayAttr', - 'I64ArrayAttr' - ]): - default_value = default_value.replace('[', - '{').replace( - ']', '}') - if (attr_mlir_converter[op_proto[ATTRS][attr][TYPE]] - in ['BoolAttr', 'BoolArrayAttr']): + op_proto[ATTRS][attr][DEFAULT_VALUE] + ) + if attr_mlir_converter[op_proto[ATTRS][attr][TYPE]] in [ + 'I32ArrayAttr', + 'F32ArrayAttr', + 'StrArrayAttr', + 'BoolArrayAttr', + 'I64ArrayAttr', + ]: + default_value = default_value.replace( + '[', '{' + ).replace(']', '}') + if attr_mlir_converter[op_proto[ATTRS][attr][TYPE]] in [ + 'BoolAttr', + 'BoolArrayAttr', + ]: default_value = default_value.lower() - elif (attr_mlir_converter[op_proto[ATTRS][attr][TYPE]] - in ['StrAttr', 'StrArrayAttr']): + elif attr_mlir_converter[ + op_proto[ATTRS][attr][TYPE] + ] in ['StrAttr', 'StrArrayAttr']: default_value = default_value.replace('\'', '\\\"') - if attr_mlir_converter[op_proto[ATTRS][attr] - [TYPE]] == "StrAttr": + if ( + attr_mlir_converter[op_proto[ATTRS][attr][TYPE]] + == "StrAttr" + ): default_value = '\\\"' + default_value + '\\\"' - attr_list = " DefaultValuedAttr<" + attr_mlir_converter[ - op_proto[ATTRS][attr] - [TYPE]] + ", \"" + default_value + "\">:$" + attr + "," + attr_list = ( + " DefaultValuedAttr<" + + attr_mlir_converter[op_proto[ATTRS][attr][TYPE]] + + ", \"" + + default_value + + "\">:$" + + attr + + "," + ) ARGUMENTS += attr_list else: - print("Error:" + op_type + ":" + attr + ":" + - str(op_proto[ATTRS][attr][TYPE])) + print( + "Error:" + + op_type + + ":" + + attr + + ":" + + str(op_proto[ATTRS][attr][TYPE]) + ) else: if op_proto[ATTRS][attr][TYPE] in attr_mlir_converter: - attr_type_ = attr_mlir_converter[op_proto[ATTRS][attr] - [TYPE]] - if (attr_type_ in [ - 'StrAttr', 'I32ArrayAttr', 'F32ArrayAttr', - 'StrArrayAttr', 'BoolArrayAttr', 'I64ArrayAttr' - ]): + attr_type_ = attr_mlir_converter[ + op_proto[ATTRS][attr][TYPE] + ] + if attr_type_ in [ + 'StrAttr', + 'I32ArrayAttr', + 'F32ArrayAttr', + 'StrArrayAttr', + 'BoolArrayAttr', + 'I64ArrayAttr', + ]: attr_list = attr_type_ + ":$" + attr + "," ARGUMENTS += attr_list else: - print(" ouch Error:" + op_type + ":" + attr + ":" + - str(op_proto[ATTRS][attr][TYPE])) + print( + " ouch Error:" + + op_type + + ":" + + attr + + ":" + + str(op_proto[ATTRS][attr][TYPE]) + ) ARGUMENTS = ARGUMENTS[:-1] + ");\n" # 2.4 results info RESULTS = "" - if (len(op_proto[OUTPUTS]) > 0): + if len(op_proto[OUTPUTS]) > 0: outputs = "" for output_ in op_proto[OUTPUTS]: - if op_proto[OUTPUTS][output_][EXTRA] != True and op_proto[ - OUTPUTS][output_][INTERMEDIATE] != True: + if ( + op_proto[OUTPUTS][output_][EXTRA] != True + and op_proto[OUTPUTS][output_][INTERMEDIATE] != True + ): if op_proto[OUTPUTS][output_][DUPLICABLE] != True: outputs = outputs + "PD_Tensor:${},".format(output_) else: outputs = outputs + "PD_Tensor_Array:${},".format( - output_) + output_ + ) RESULTS = "\n let results = (outs {});\n".format(outputs[:-1]) with open(dst_dialect_file, 'a') as ops_mlir_file: @@ -376,8 +457,10 @@ def convert_op_proto_into_mlir(op_descs): ops_mlir_file.write("\n#endif // PD_OPS") print("Skipped ops num: " + str(len(skipped_op_list))) - print("Automatically generated op dialects num: " + - str(len(automatically_generated_op_dialect))) + print( + "Automatically generated op dialects num: " + + str(len(automatically_generated_op_dialect)) + ) if __name__ == "__main__": diff --git a/tools/infrt/generate_phi_kernel_dialect.py b/tools/infrt/generate_phi_kernel_dialect.py index 686a5e12a6c30..a5d1003dbb9f2 100644 --- a/tools/infrt/generate_phi_kernel_dialect.py +++ b/tools/infrt/generate_phi_kernel_dialect.py @@ -17,21 +17,21 @@ import os from get_compat_kernel_signature import get_compat_kernels_info -#TODO @DannyIsFunny: more attr types need to be supported. +# TODO @DannyIsFunny: more attr types need to be supported. attr_type_converter = { "int": 'SI32Attr', "bool": 'BoolAttr', "int64_t": 'SI64Attr', "float": 'F32Attr', "string": 'StrAttr', - "vector": 'I32ArrayAttr' + "vector": 'I32ArrayAttr', } target_type_converter = {"CPU": "CPU", "GPU": "GPU", "Undefined": "UNK"} layout_type_converter = { "NCHW": "NCHW", "NHWC": "NHWC", - "Undefined(AnyLayout)": "ANY" + "Undefined(AnyLayout)": "ANY", } precision_type_converter = { "uint8": "UINT8", @@ -46,7 +46,7 @@ "complex64": "COMPLEX64", "complex128": "COMPLEX128", "bool": "BOOL", - "Undefined": "UNK" + "Undefined": "UNK", } kernel_types_info_file = "./kernels.json" @@ -88,16 +88,19 @@ def generate_kernel_name(op_name, place_str): layout_ = layout_type_converter[layout_.strip()] precision_ = precision_type_converter[precision_.strip()] class_name_ = "{}{}".format( - op_name.replace("_", "").title(), "".join([ - target_.strip().title(), - precision_.strip(), - layout_.strip().title().title() - ])) + op_name.replace("_", "").title(), + "".join( + [ + target_.strip().title(), + precision_.strip(), + layout_.strip().title().title(), + ] + ), + ) alias_ = "{}.{}".format( op_name, - ".".join([target_.strip(), - precision_.strip(), - layout_.strip()])) + ".".join([target_.strip(), precision_.strip(), layout_.strip()]), + ) return alias_, class_name_ @@ -111,8 +114,9 @@ def generate_attrs_info(op_name, attrs_info): for index in range(len(attrs_info)): attr_name = kernel_attrs_names[op_name]["attrs"][index] attr_type = attr_type_converter[attrs_info[index]] - attrs_args_ += '{type_}:${name_},'.format(type_=attr_type, - name_=attr_name) + attrs_args_ += '{type_}:${name_},'.format( + type_=attr_type, name_=attr_name + ) return attrs_args_[:-1] @@ -125,7 +129,8 @@ def generate_inputs_info(input_info): layout_ = layout_type_converter[layout_.strip()] precision_ = precision_type_converter[precision_.strip()] input_args_ += " DenseTensor<\"{}\",\"{}\",\"{}\">:$in{},".format( - target_.strip(), precision_.strip(), layout_.strip(), str(index)) + target_.strip(), precision_.strip(), layout_.strip(), str(index) + ) input_args_ = input_args_[:-1] return input_args_ @@ -134,12 +139,13 @@ def generate_arguments_info(op_name, input_info, attr_info): input_args = generate_inputs_info(input_info) attr_args = generate_attrs_info(op_name, attr_info) context_args = "Context:$dev_ctx" - argument_list = [context_args - ] + input_args.split(",") + attr_args.split(",") - while ("" in argument_list): + argument_list = ( + [context_args] + input_args.split(",") + attr_args.split(",") + ) + while "" in argument_list: argument_list.remove("") argument_ = ",".join(argument_list) - return (("let arguments = (ins {});".format(argument_.strip(",")))) + return "let arguments = (ins {});".format(argument_.strip(",")) def generate_results_info(output_info): @@ -151,8 +157,9 @@ def generate_results_info(output_info): layout_ = layout_type_converter[layout_.strip()] precision_ = precision_type_converter[precision_.strip()] output_args_ += " DenseTensor<\"{}\",\"{}\",\"{}\">:$out{},".format( - target_.strip(), precision_.strip(), layout_.strip(), str(index)) - return ("{});".format(output_args_[:-1])) + target_.strip(), precision_.strip(), layout_.strip(), str(index) + ) + return "{});".format(output_args_[:-1]) def generate_supported_kernel_list(load_dict): @@ -205,11 +212,13 @@ def generate_cpu_kernel_dialect(op_name, kernel_alias_, kernel_info): alias, class_name = generate_kernel_name(op_name, kernel_alias_) summary = 'let summary = "{name}";'.format(name=alias) dialect_name = alias.split(".") - dialect_name = dialect_name[0] + "." + dialect_name[2] + "." + dialect_name[ - 3] + dialect_name = ( + dialect_name[0] + "." + dialect_name[2] + "." + dialect_name[3] + ) header = 'def {kernel_name} : PDTCPU_Kernel<"{name}",[NoSideEffect]> {left_brace}'.format( - kernel_name=class_name, name=dialect_name.lower(), left_brace="{") + kernel_name=class_name, name=dialect_name.lower(), left_brace="{" + ) inputs_ = kernel_info["input"] attributes = kernel_info["attribute"] @@ -223,7 +232,8 @@ def generate_cpu_kernel_dialect(op_name, kernel_alias_, kernel_info): summary_=summary, arguments_=arguments, results_=results, - right_brace="}") + right_brace="}", + ) return kernel_dialect @@ -232,11 +242,13 @@ def generate_gpu_kernel_dialect(op_name, kernel_alias_, kernel_info): alias, class_name = generate_kernel_name(op_name, kernel_alias_) summary = 'let summary = "{name}";'.format(name=alias) dialect_name = alias.split(".") - dialect_name = dialect_name[0] + "." + dialect_name[2] + "." + dialect_name[ - 3] + dialect_name = ( + dialect_name[0] + "." + dialect_name[2] + "." + dialect_name[3] + ) header = 'def {kernel_name} : PDTGPU_Kernel<"{name}",[NoSideEffect]> {left_brace}'.format( - kernel_name=class_name, name=dialect_name.lower(), left_brace="{") + kernel_name=class_name, name=dialect_name.lower(), left_brace="{" + ) inputs_ = kernel_info["input"] attributes = kernel_info["attribute"] arguments = generate_arguments_info(op_name, inputs_, attributes) @@ -249,7 +261,8 @@ def generate_gpu_kernel_dialect(op_name, kernel_alias_, kernel_info): summary_=summary, arguments_=arguments, results_=results, - right_brace="}") + right_brace="}", + ) return kernel_dialect @@ -270,7 +283,7 @@ def generate_dialect_head(): include \"mlir/IR/OpBase.td\"\n\ include \"paddle/infrt/dialect/phi/ir/infrt_phi_kernel.td\"" - return (comment_ + includes_) + return comment_ + includes_ def get_kernel_target(kernel_alias_): @@ -298,33 +311,52 @@ def main(): for kernel_alias_ in kernel_info: if get_kernel_target(kernel_alias_) == "CPU": kernel_registry = generate_cpu_kernel_dialect( - op_name, kernel_alias_, kernel_info[kernel_alias_]) + op_name, kernel_alias_, kernel_info[kernel_alias_] + ) cpu_registry_ += kernel_registry elif get_kernel_target(kernel_alias_) == "GPU": kernel_registry = generate_gpu_kernel_dialect( - op_name, kernel_alias_, kernel_info[kernel_alias_]) + op_name, kernel_alias_, kernel_info[kernel_alias_] + ) gpu_registry_ += kernel_registry else: - print("Unsupported backend:" + - get_kernel_target(kernel_alias_)) + print( + "Unsupported backend:" + + get_kernel_target(kernel_alias_) + ) end = "#endif // PTEN_KERNELS" - with open("../../paddle/infrt/dialect/phi/ir/phi_cpu_kernels.td", - "w") as dst: - dst.write('{start_}\n{dialect_}\n{end_}'.format( - start_=head, dialect_=cpu_registry_, end_=end)) - with open("../../paddle/infrt/dialect/phi/ir/phi_gpu_kernels.td", - "w") as dst: - dst.write('{start_}\n{dialect_}\n{end_}'.format( - start_=head, dialect_=gpu_registry_, end_=end)) + with open( + "../../paddle/infrt/dialect/phi/ir/phi_cpu_kernels.td", "w" + ) as dst: + dst.write( + '{start_}\n{dialect_}\n{end_}'.format( + start_=head, dialect_=cpu_registry_, end_=end + ) + ) + with open( + "../../paddle/infrt/dialect/phi/ir/phi_gpu_kernels.td", "w" + ) as dst: + dst.write( + '{start_}\n{dialect_}\n{end_}'.format( + start_=head, dialect_=gpu_registry_, end_=end + ) + ) if __name__ == '__main__': if not os.path.exists(kernel_types_info_file): - print("Error: '{file_name}' not exist!".format( - file_name=kernel_types_info_file)) + print( + "Error: '{file_name}' not exist!".format( + file_name=kernel_types_info_file + ) + ) if not os.path.exists(kernel_signature_info_file): - print("Error: '{file_name}' not exist!".format( - file_name=kernel_signature_info_file)) + print( + "Error: '{file_name}' not exist!".format( + file_name=kernel_signature_info_file + ) + ) if os.path.exists(kernel_types_info_file) and os.path.exists( - kernel_signature_info_file): + kernel_signature_info_file + ): main() diff --git a/tools/infrt/get_compat_kernel_signature.py b/tools/infrt/get_compat_kernel_signature.py index 104d3ae30c23a..f5f9afdafd1bc 100644 --- a/tools/infrt/get_compat_kernel_signature.py +++ b/tools/infrt/get_compat_kernel_signature.py @@ -56,16 +56,21 @@ def get_compat_kernels_info(): content = "" registry = False for line in txt: - if ("KernelSignature(" in line): + if "KernelSignature(" in line: content = "" registry = True - if (registry): + if registry: content += line - if (registry and ";" in line): - data = content.replace("\n", "").replace( - " ", - "").strip("return").strip("KernelSignature(").strip( - r"\);").replace("\"", "").replace("\\", "") + if registry and ";" in line: + data = ( + content.replace("\n", "") + .replace(" ", "") + .strip("return") + .strip("KernelSignature(") + .strip(r"\);") + .replace("\"", "") + .replace("\\", "") + ) registry = False if is_grad_kernel(data): continue @@ -74,14 +79,23 @@ def get_compat_kernels_info(): if name in kernels_info: cur_reg = kernels_info[name] kernels_info[name]["inputs"] = list( - set(registry_info["inputs"] + - kernels_info[name]["inputs"])) + set( + registry_info["inputs"] + + kernels_info[name]["inputs"] + ) + ) kernels_info[name]["attrs"] = list( - set(registry_info["attrs"] + - kernels_info[name]["attrs"])) + set( + registry_info["attrs"] + + kernels_info[name]["attrs"] + ) + ) kernels_info[name]["outputs"] = list( - set(registry_info["outputs"] + - kernels_info[name]["outputs"])) + set( + registry_info["outputs"] + + kernels_info[name]["outputs"] + ) + ) else: kernels_info[name] = registry_info diff --git a/tools/infrt/get_phi_kernel_info.py b/tools/infrt/get_phi_kernel_info.py index af73432a79756..9eeee88276d80 100644 --- a/tools/infrt/get_phi_kernel_info.py +++ b/tools/infrt/get_phi_kernel_info.py @@ -47,30 +47,34 @@ def get_skipped_kernel_list(): def parse_args(): parser = argparse.ArgumentParser("gather phi kernel and infermate info") - parser.add_argument("--paddle_root_path", - type=str, - required=True, - help="root path of paddle src[WORK_PATH/Paddle].") + parser.add_argument( + "--paddle_root_path", + type=str, + required=True, + help="root path of paddle src[WORK_PATH/Paddle].", + ) parser.add_argument( "--kernel_info_file", type=str, required=True, - help="kernel info file generated by get_phi_kernel_function.sh.") - parser.add_argument("--infermeta_wrap_file", - type=str, - required=True, - help="inferMeta wrap info file.") - parser.add_argument("--attr_info_file", - type=str, - required=True, - help="attr info file.") + help="kernel info file generated by get_phi_kernel_function.sh.", + ) + parser.add_argument( + "--infermeta_wrap_file", + type=str, + required=True, + help="inferMeta wrap info file.", + ) + parser.add_argument( + "--attr_info_file", type=str, required=True, help="attr info file." + ) parser.add_argument( "--generate_file", type=str, required=True, - default= - "../paddle/infrt/kernel/phi/infershaped/infershaped_kernel_launchers.cc", - help="generated file.") + default="../paddle/infrt/kernel/phi/infershaped/infershaped_kernel_launchers.cc", + help="generated file.", + ) args = parser.parse_args() return args @@ -167,15 +171,18 @@ def gen_include_headers(): def gen_namespace(): - return (""" + return ( + """ namespace infrt { namespace kernel { -""", """ +""", + """ } // namespace kernel } // namespace infrt -""") +""", + ) def gen_context(val): @@ -203,7 +210,7 @@ def gen_kernel_func(val, ctx_name, dtype_name): st = val.index('<') ed = val.index('>') func_name = val[:st] - template_name = val[st + 1:ed] + template_name = val[st + 1 : ed] if '::phi::' in template_name: return "&::phi::" + val else: @@ -273,7 +280,7 @@ def gen_register_code_info(item: List[str], attr_data: Dict[str, List[str]]): attr_data: {'phi_cpu.arg_min.float32.any': ['axisBool', 'keepdimsBool', 'flatten', 'dtype']} """ ctx_name, ir_ctx_name = gen_context(item[1]) - if (ctx_name == ""): + if ctx_name == "": return "" item[2] = gen_layout(item[2]) ir_dtypes, origin_dtypes = gen_dtype(item[4:-1]) @@ -290,11 +297,19 @@ def gen_register_code_info(item: List[str], attr_data: Dict[str, List[str]]): kernel_func = gen_kernel_func(item[3], ctx_name, origin_dtype) if item[0].lower() in skipped_kernel_list: continue - ir_name = ir_ctx_name + '.' + item[0].lower( - ) + '.' + ir_dtype + '.' + item[2].lower() + ir_name = ( + ir_ctx_name + + '.' + + item[0].lower() + + '.' + + ir_dtype + + '.' + + item[2].lower() + ) if ir_name in attr_data.keys() and attr_data[ir_name] is not None: attr_names = ', '.join( - ["\"" + a + "\"" for a in attr_data[ir_name]]) + ["\"" + a + "\"" for a in attr_data[ir_name]] + ) res += f""" registry->AddKernel("{ir_name}",""" @@ -320,8 +335,9 @@ def gen_register_code_info(item: List[str], attr_data: Dict[str, List[str]]): return res -def gen_register_info(resources: List[List[str]], attr_data: Dict[str, - List[str]]): +def gen_register_info( + resources: List[List[str]], attr_data: Dict[str, List[str]] +): """ resources: [['add', 'CPU', 'ALL_LAYOUT', 'AddKernel', 'float', 'double', '...'(varaidic types), 'ElementwiseInferMeta'], ...] attr_data: {'phi_cpu.arg_min.float32.any': ['axisBool', 'keepdimsBool', 'flatten', 'dtype']} @@ -335,7 +351,7 @@ def gen_register_info(resources: List[List[str]], attr_data: Dict[str, if update_item[1] != "CPU": continue code = gen_register_code_info(item, attr_data) - if (code == ""): + if code == "": continue res += code @@ -347,7 +363,7 @@ def gen_register_info(resources: List[List[str]], attr_data: Dict[str, if update_item[1] != "GPU": continue code = gen_register_code_info(item, attr_data) - if (code == ""): + if code == "": continue res += code res += "#endif // INFRT_WITH_GPU" @@ -356,9 +372,11 @@ def gen_register_info(resources: List[List[str]], attr_data: Dict[str, return res -def gen_phi_kernel_register_code(resources: List[List[str]], - attr_data: Dict[str, List[str]], - src_file_path: str): +def gen_phi_kernel_register_code( + resources: List[List[str]], + attr_data: Dict[str, List[str]], + src_file_path: str, +): source_file = open(src_file_path, 'w') source_file.write(gen_warn_info()) source_file.write(gen_include_headers()) @@ -371,7 +389,9 @@ def gen_phi_kernel_register_code(resources: List[List[str]], if __name__ == "__main__": args = parse_args() - skipped_phi_api_list_file = args.paddle_root_path + skipped_phi_api_list_file + skipped_phi_api_list_file = ( + args.paddle_root_path + skipped_phi_api_list_file + ) api_yaml_file = args.paddle_root_path + api_yaml_file legacy_api_yaml_file = args.paddle_root_path + legacy_api_yaml_file infer_meta_data = [] diff --git a/tools/infrt/print_kernel_pass_info.py b/tools/infrt/print_kernel_pass_info.py index 1e84f3de86f4a..1a48708b71bc7 100644 --- a/tools/infrt/print_kernel_pass_info.py +++ b/tools/infrt/print_kernel_pass_info.py @@ -45,7 +45,8 @@ def get_compat_kernels_info(register): is_macro_defination = False for line in txt: if line.strip().startswith( - "#define") and line.strip().endswith("\\"): + "#define" + ) and line.strip().endswith("\\"): is_macro_defination = True continue if is_macro_defination: @@ -53,14 +54,18 @@ def get_compat_kernels_info(register): is_macro_defination = False continue - if (register in line): + if register in line: content = "" registry = True - if (registry): + if registry: content += line - if (registry and ";" in line): - kernel_name = content.replace("\n", "").replace( - " ", "").strip(register).split(",") + if registry and ";" in line: + kernel_name = ( + content.replace("\n", "") + .replace(" ", "") + .strip(register) + .split(",") + ) registry = False kernel_names.append(kernel_name[0]) return remove_grad_kernel(kernel_names) @@ -68,8 +73,9 @@ def get_compat_kernels_info(register): def show_kernel_statistics(backend, kernels): print("=== kernels statistics === ") - print("the number of " + backend + " kernels is: " + str(len(kernels)) + - "\n") + print( + "the number of " + backend + " kernels is: " + str(len(kernels)) + "\n" + ) print(kernels) print("\n") @@ -92,7 +98,7 @@ def get_passes_info(register): continue if register in line: registry_fun_found = True - if (registry_fun_found): + if registry_fun_found: pass_registry_func += line if registry_fun_found: for char in line: diff --git a/tools/jetson_infer_op.py b/tools/jetson_infer_op.py index e132a14373e4f..664f2b6616ed6 100644 --- a/tools/jetson_infer_op.py +++ b/tools/jetson_infer_op.py @@ -37,7 +37,7 @@ 'test_sync_batch_norm_op', # case too large 'test_reduce_op', - 'test_transpose_op' + 'test_transpose_op', ] op_diff_list = [ @@ -51,14 +51,18 @@ def parse_arguments(): :return: """ parser = argparse.ArgumentParser() - parser.add_argument('--shell_name', - type=str, - default='get_op_list.sh', - help='please input right name') - parser.add_argument('--op_list_file', - type=str, - default='list_op.txt', - help='please input right name') + parser.add_argument( + '--shell_name', + type=str, + default='get_op_list.sh', + help='please input right name', + ) + parser.add_argument( + '--op_list_file', + type=str, + default='list_op.txt', + help='please input right name', + ) return parser.parse_args() @@ -85,7 +89,7 @@ def get_prefix(line, end_char='d'): """ i = 0 prefix = '' - while (line[i] != end_char): + while line[i] != end_char: prefix += line[i] i += 1 return prefix @@ -116,7 +120,9 @@ def add_import_skip_return(file, pattern_import, pattern_skip, pattern_return): # add @skip_check_grad_ci match_obj = pattern_2.search(line) if match_obj is not None: - file_data += "@skip_check_grad_ci(reason='jetson do n0t neeed this !')\n" + file_data += ( + "@skip_check_grad_ci(reason='jetson do n0t neeed this !')\n" + ) print("### add @skip_check_grad_ci ####") # delete test_grad_output @@ -155,8 +161,14 @@ def set_diff_value(file, atol="1e-5", inplace_atol="1e-7"): :param inplace_atol: :return: """ - os.system(r"sed -i 's/self.check_output(/self\.check_output\(atol=" + atol + - ",inplace_atol=" + inplace_atol + ",/g\' " + file) + os.system( + r"sed -i 's/self.check_output(/self\.check_output\(atol=" + + atol + + ",inplace_atol=" + + inplace_atol + + ",/g\' " + + file + ) def change_op_file(start=0, end=0, op_list_file='list_op.txt', path='.'): @@ -182,8 +194,9 @@ def change_op_file(start=0, end=0, op_list_file='list_op.txt', path='.'): pattern_skip = r"^class .*\(OpTest\):$" pattern_return = r"def test.*grad.*\):$" # change file - add_import_skip_return(file_with_path, pattern_import, pattern_skip, - pattern_return) + add_import_skip_return( + file_with_path, pattern_import, pattern_skip, pattern_return + ) # op_diff if item in op_diff_list: set_diff_value(file_with_path) @@ -240,7 +253,7 @@ def run_file_change(op_list_file): :param op_list_file: :return: """ - if (os.path.exists("flag_change_file.txt")): + if os.path.exists("flag_change_file.txt"): print( "-----maybe op_file has changed, so don't need to change again------" ) @@ -269,9 +282,12 @@ def run_test_second(): "sed -n '/(Failed)$/p' test_op_log.txt | awk '{print $3}' >& rerun_op.txt" ) rerun_list = get_op_list('rerun_op.txt') - if (len(rerun_list)): - print("-------there are " + str(len(rerun_list)) + - " op(s) need to rerun!!!-------") + if len(rerun_list): + print( + "-------there are " + + str(len(rerun_list)) + + " op(s) need to rerun!!!-------" + ) for failed_op in rerun_list: os.system("ctest -R \"(" + failed_op + ")\" ") else: diff --git a/tools/parallel_UT_rule.py b/tools/parallel_UT_rule.py index ea36b7785367e..fa5a6c8131576 100755 --- a/tools/parallel_UT_rule.py +++ b/tools/parallel_UT_rule.py @@ -669,7 +669,7 @@ 'test_trt_convert_reduce_sum', 'save_quant2_model_lstm', 'test_trt_convert_slice', - 'test_quant2_int8_lstm_mkldnn' + 'test_quant2_int8_lstm_mkldnn', ] # mem=0 but always timeout or failed : It run 15 job each time in Single cases; @@ -734,315 +734,805 @@ # mem=0 but always timeout or failed : It run 12 job each time in Single cases; THIRD_HIGH_PARALLEL_JOB_NEW = [ - 'test_api_impl', 'test_analyzer_seq_pool1_fuse_compare_zero_copy', - 'test_analyzer_seq_pool1_profile', 'test_analyzer_mobilenet_transpose', - 'test_analyzer_resnet50', 'test_analyzer_int8_resnet50', - 'test_analyzer_int8_mobilenetv2', 'test_analyzer_bfloat16_resnet50', - 'test_analyzer_bfloat16_mobilenetv1', 'test_analyzer_int8_mobilenet_ssd', - 'test_dataset_cifar', 'test_dataset_imdb', 'test_dataset_movielens', - 'test_datasets', 'test_allgather', 'test_c_concat', 'test_c_split', - 'test_collective_reduce', 'test_cyclic_cifar_dataset', 'test_dyn_rnn', - 'test_multiclass_nms_op', 'test_communicator_geo', - 'test_quant_int8_mobilenetv2_mkldnn', 'test_analyzer_seq_pool1', - 'test_analyzer_transformer', 'test_analyzer_transformer_profile', - 'test_analyzer_int8_mobilenetv1', 'test_analyzer_bfloat16_googlenet', - 'test_analyzer_quant_performance_benchmark', 'test_dataset_wmt', - 'test_allreduce', 'test_broadcast', 'test_c_identity', - 'test_collective_sendrecv_api', 'test_fleet_utils', - 'test_fused_elemwise_activation_op', 'test_group_norm_op', - 'test_fleet_launch_nproc', 'test_quant_int8_resnet50_mkldnn', - 'test_quant2_int8_ernie_mkldnn', 'convert_model2dot_ernie' + 'test_api_impl', + 'test_analyzer_seq_pool1_fuse_compare_zero_copy', + 'test_analyzer_seq_pool1_profile', + 'test_analyzer_mobilenet_transpose', + 'test_analyzer_resnet50', + 'test_analyzer_int8_resnet50', + 'test_analyzer_int8_mobilenetv2', + 'test_analyzer_bfloat16_resnet50', + 'test_analyzer_bfloat16_mobilenetv1', + 'test_analyzer_int8_mobilenet_ssd', + 'test_dataset_cifar', + 'test_dataset_imdb', + 'test_dataset_movielens', + 'test_datasets', + 'test_allgather', + 'test_c_concat', + 'test_c_split', + 'test_collective_reduce', + 'test_cyclic_cifar_dataset', + 'test_dyn_rnn', + 'test_multiclass_nms_op', + 'test_communicator_geo', + 'test_quant_int8_mobilenetv2_mkldnn', + 'test_analyzer_seq_pool1', + 'test_analyzer_transformer', + 'test_analyzer_transformer_profile', + 'test_analyzer_int8_mobilenetv1', + 'test_analyzer_bfloat16_googlenet', + 'test_analyzer_quant_performance_benchmark', + 'test_dataset_wmt', + 'test_allreduce', + 'test_broadcast', + 'test_c_identity', + 'test_collective_sendrecv_api', + 'test_fleet_utils', + 'test_fused_elemwise_activation_op', + 'test_group_norm_op', + 'test_fleet_launch_nproc', + 'test_quant_int8_resnet50_mkldnn', + 'test_quant2_int8_ernie_mkldnn', + 'convert_model2dot_ernie', ] # mem != 0: It run 7 job each time in Single cases; 4 job each time in Multi cases; 3 job each time in exclusive cases FOURTH_HIGH_PARALLEL_JOB_NEW = [ - 'test_meshgrid_op', 'test_word2vec', 'test_analyzer_ner', - 'test_fetch_lod_tensor_array', 'test_adagrad_op_v2', - 'test_conv2d_fusion_op', 'test_hapi_amp', 'test_metrics', - 'test_clip_by_norm_op', 'test_lr_scheduler', 'test_generate_proposals_op', - 'test_masked_select_op', 'test_trt_anchor_generator_op', - 'test_imperative_ocr_attention_model', 'test_sentiment', 'test_chunk_op', - 'test_memcpy_op', 'test_warpctc_op', 'test_row_conv_op', - 'test_grid_sample_function', 'test_rnn_nets', 'test_pad3d_op', - 'test_imperative_mnist_sorted_gradient', 'tensor_test', - 'test_tensorrt_engine_op', 'test_dot_op', 'test_real_imag_op', - 'test_adam_optimizer_fp32_fp64', 'test_reduce_op', - 'test_density_prior_box_op', 'test_top_k_op', 'test_grid_generator', - 'test_randn_op', 'test_activation_mkldnn_op', 'test_lac', 'test_pad_op', - 'test_lstmp_op', 'test_loop', 'test_pylayer_op', - 'data_device_transform_test', 'test_trt_roi_align_op', - 'test_nn_functional_hot_op', 'test_top_k_v2_op', 'test_crop_op', - 'test_conv_bn_fuse_pass', 'test_beam_search_decode_op', 'test_auc_op', - 'test_pool2d_op', 'test_gaussian_random_op', 'test_maximum_op', - 'test_rnn_cell_api', 'device_code_test', 'test_ir_inplace_pass', - 'test_cos_sim_op', 'test_lite_tensor_utils', 'test_fit_a_line', - 'test_mish_op', 'test_transpose_op', 'test_mean_iou', - 'test_conv3d_transpose_op', 'test_jit_save_load', 'test_unsqueeze2_op', - 'test_eager_deletion_while_op', 'test_zeros_like_op', 'test_c_embedding_op', - 'test_regularizer', 'zero_copy_tensor_test', 'test_tensor_shape', - 'test_resnet', 'test_resnet_amp', 'test_dygraph_weight_norm', 'test_tracer', - 'test_list', 'test_sequence_concat', 'test_adaptive_avg_pool1d', - 'test_elementwise_div_op', 'test_conv1d_transpose_layer', 'test_adamw_op', - 'trt_fc_prelu_test', 'test_temporal_shift_op', - 'test_naive_best_fit_gpu_memory_limit', 'dlpack_tensor_test', - 'test_elementwise_max_op', 'test_typing', 'test_asp_pruning_2d_greedy', - 'test_fake_dequantize_op', 'test_crop_tensor_op', - 'test_imperative_load_static_param', 'test_imperative_qat_user_defined', - 'test_anchor_generator_op', 'test_if_else_op', 'test_prepare_op', - 'test_conj_op', 'test_imperative_hook_for_layer', 'test_roi_pool_op', - 'test_strided_slice_op', 'test_norm_all', 'test_weight_decay', - 'test_functional_conv2d', 'test_functional_conv3d_transpose', - 'test_imperative_layer_trainable', 'test_imperative_data_parallel', - 'test_digamma_op', 'test_distribution', 'test_box_clip_op', - 'custom_tensor_test', 'test_marker_op', 'test_dataloader_early_reset', - 'test_gather_nd_op', 'test_tensor_register_hook', 'test_retain_graph', - 'test_network_with_dtype', 'test_basic_api_transformation', 'test_diag', - 'test_lod_array_length_op', 'test_reinforcement_learning', - 'test_softmax_op', 'test_fc_fuse_pass', 'test_adaptive_max_pool2d', - 'test_inverse_op', 'test_declarative', 'test_imperative_double_grad', - 'test_tensor_methods', 'test_pool1d_api', 'system_allocator_test', - 'test_print', 'test_tensor_type_promotion', 'test_bce_with_logits_loss', - 'test_tensor', 'test_cross_op', 'concat_test', 'test_ast_util', - 'test_proximal_adagrad_op', 'test_pairwise_distance', - 'test_imperative_mnist', 'test_beam_search_decoder', - 'test_build_strategy_fusion_group_pass', 'test_dygraph_spectral_norm', - 'test_scale_mkldnn_op', 'test_load_state_dict_from_old_format', - 'test_margin_rank_loss_op', 'test_lookup_table_v2_op', - 'test_mix_precision_all_reduce_fuse', 'test_spp_op', 'test_op_converter', - 'mixed_vector_test', 'test_roi_align_op', 'test_pad_constant_like', - 'test_mul_op', 'test_spectral_norm_op', 'test_transformer', - 'test_for_enumerate', 'test_variable_trans_func', - 'test_squared_l2_distance_op', 'test_quantize_transpiler_v2', - 'test_im2sequence_op', 'test_reader_reset', 'test_one_hot_op', - 'test_adaptive_max_pool1d', 'test_label_smooth_op', - 'test_parallel_executor_fetch_feed', 'test_cast', - 'test_parallel_dygraph_sync_batch_norm', 'test_collect_fpn_proposals_op', - 'test_expand_as_v2_op', 'test_device', 'test_code_generator', - 'test_asp_pruning_2d_best', 'test_fleet_with_asp', 'test_pool2d_api', - 'test_mean_op', 'test_is_tensor', 'test_run_program_op', - 'test_cuda_random_seed', 'test_linear_interp_op', - 'test_fuse_all_reduce_pass', 'tensor_util_test', 'test_median', - 'test_nanmedian', 'test_linear', 'test_imperative_qat_amp', - 'test_truncated_gaussian_random_op', 'test_lstm_cudnn_op', - 'copy_same_tensor_test', 'test_squeeze2_op', - 'naive_best_fit_allocator_test', 'test_model', 'test_py_reader_combination', - 'test_prior_box_op', 'test_matmul_v2_mkldnn_op', 'test_sum_op', - 'test_paddle_imperative_double_grad', 'test_norm_op', 'test_pool3d_api', - 'test_imperative_gan', 'test_sequence_softmax_op', 'test_rand_op', - 'test_expand_v2_op', 'test_word2vec_book', 'test_histogram_op', - 'test_min_op', 'test_mse_loss', 'test_sign_op', - 'selected_rows_functor_gpu_test', 'test_fleet_base', 'test_logsumexp', - 'test_detection', 'test_image_classification_fp16', 'test_random_seed', - 'test_op_function_generator', 'test_unique_with_counts', - 'test_complex_elementwise_layers', 'test_array_read_write_op', - 'test_fusion_group_op', 'test_imperative_layer_apply', + 'test_meshgrid_op', + 'test_word2vec', + 'test_analyzer_ner', + 'test_fetch_lod_tensor_array', + 'test_adagrad_op_v2', + 'test_conv2d_fusion_op', + 'test_hapi_amp', + 'test_metrics', + 'test_clip_by_norm_op', + 'test_lr_scheduler', + 'test_generate_proposals_op', + 'test_masked_select_op', + 'test_trt_anchor_generator_op', + 'test_imperative_ocr_attention_model', + 'test_sentiment', + 'test_chunk_op', + 'test_memcpy_op', + 'test_warpctc_op', + 'test_row_conv_op', + 'test_grid_sample_function', + 'test_rnn_nets', + 'test_pad3d_op', + 'test_imperative_mnist_sorted_gradient', + 'tensor_test', + 'test_tensorrt_engine_op', + 'test_dot_op', + 'test_real_imag_op', + 'test_adam_optimizer_fp32_fp64', + 'test_reduce_op', + 'test_density_prior_box_op', + 'test_top_k_op', + 'test_grid_generator', + 'test_randn_op', + 'test_activation_mkldnn_op', + 'test_lac', + 'test_pad_op', + 'test_lstmp_op', + 'test_loop', + 'test_pylayer_op', + 'data_device_transform_test', + 'test_trt_roi_align_op', + 'test_nn_functional_hot_op', + 'test_top_k_v2_op', + 'test_crop_op', + 'test_conv_bn_fuse_pass', + 'test_beam_search_decode_op', + 'test_auc_op', + 'test_pool2d_op', + 'test_gaussian_random_op', + 'test_maximum_op', + 'test_rnn_cell_api', + 'device_code_test', + 'test_ir_inplace_pass', + 'test_cos_sim_op', + 'test_lite_tensor_utils', + 'test_fit_a_line', + 'test_mish_op', + 'test_transpose_op', + 'test_mean_iou', + 'test_conv3d_transpose_op', + 'test_jit_save_load', + 'test_unsqueeze2_op', + 'test_eager_deletion_while_op', + 'test_zeros_like_op', + 'test_c_embedding_op', + 'test_regularizer', + 'zero_copy_tensor_test', + 'test_tensor_shape', + 'test_resnet', + 'test_resnet_amp', + 'test_dygraph_weight_norm', + 'test_tracer', + 'test_list', + 'test_sequence_concat', + 'test_adaptive_avg_pool1d', + 'test_elementwise_div_op', + 'test_conv1d_transpose_layer', + 'test_adamw_op', + 'trt_fc_prelu_test', + 'test_temporal_shift_op', + 'test_naive_best_fit_gpu_memory_limit', + 'dlpack_tensor_test', + 'test_elementwise_max_op', + 'test_typing', + 'test_asp_pruning_2d_greedy', + 'test_fake_dequantize_op', + 'test_crop_tensor_op', + 'test_imperative_load_static_param', + 'test_imperative_qat_user_defined', + 'test_anchor_generator_op', + 'test_if_else_op', + 'test_prepare_op', + 'test_conj_op', + 'test_imperative_hook_for_layer', + 'test_roi_pool_op', + 'test_strided_slice_op', + 'test_norm_all', + 'test_weight_decay', + 'test_functional_conv2d', + 'test_functional_conv3d_transpose', + 'test_imperative_layer_trainable', + 'test_imperative_data_parallel', + 'test_digamma_op', + 'test_distribution', + 'test_box_clip_op', + 'custom_tensor_test', + 'test_marker_op', + 'test_dataloader_early_reset', + 'test_gather_nd_op', + 'test_tensor_register_hook', + 'test_retain_graph', + 'test_network_with_dtype', + 'test_basic_api_transformation', + 'test_diag', + 'test_lod_array_length_op', + 'test_reinforcement_learning', + 'test_softmax_op', + 'test_fc_fuse_pass', + 'test_adaptive_max_pool2d', + 'test_inverse_op', + 'test_declarative', + 'test_imperative_double_grad', + 'test_tensor_methods', + 'test_pool1d_api', + 'system_allocator_test', + 'test_print', + 'test_tensor_type_promotion', + 'test_bce_with_logits_loss', + 'test_tensor', + 'test_cross_op', + 'concat_test', + 'test_ast_util', + 'test_proximal_adagrad_op', + 'test_pairwise_distance', + 'test_imperative_mnist', + 'test_beam_search_decoder', + 'test_build_strategy_fusion_group_pass', + 'test_dygraph_spectral_norm', + 'test_scale_mkldnn_op', + 'test_load_state_dict_from_old_format', + 'test_margin_rank_loss_op', + 'test_lookup_table_v2_op', + 'test_mix_precision_all_reduce_fuse', + 'test_spp_op', + 'test_op_converter', + 'mixed_vector_test', + 'test_roi_align_op', + 'test_pad_constant_like', + 'test_mul_op', + 'test_spectral_norm_op', + 'test_transformer', + 'test_for_enumerate', + 'test_variable_trans_func', + 'test_squared_l2_distance_op', + 'test_quantize_transpiler_v2', + 'test_im2sequence_op', + 'test_reader_reset', + 'test_one_hot_op', + 'test_adaptive_max_pool1d', + 'test_label_smooth_op', + 'test_parallel_executor_fetch_feed', + 'test_cast', + 'test_parallel_dygraph_sync_batch_norm', + 'test_collect_fpn_proposals_op', + 'test_expand_as_v2_op', + 'test_device', + 'test_code_generator', + 'test_asp_pruning_2d_best', + 'test_fleet_with_asp', + 'test_pool2d_api', + 'test_mean_op', + 'test_is_tensor', + 'test_run_program_op', + 'test_cuda_random_seed', + 'test_linear_interp_op', + 'test_fuse_all_reduce_pass', + 'tensor_util_test', + 'test_median', + 'test_nanmedian', + 'test_linear', + 'test_imperative_qat_amp', + 'test_truncated_gaussian_random_op', + 'test_lstm_cudnn_op', + 'copy_same_tensor_test', + 'test_squeeze2_op', + 'naive_best_fit_allocator_test', + 'test_model', + 'test_py_reader_combination', + 'test_prior_box_op', + 'test_matmul_v2_mkldnn_op', + 'test_sum_op', + 'test_paddle_imperative_double_grad', + 'test_norm_op', + 'test_pool3d_api', + 'test_imperative_gan', + 'test_sequence_softmax_op', + 'test_rand_op', + 'test_expand_v2_op', + 'test_word2vec_book', + 'test_histogram_op', + 'test_min_op', + 'test_mse_loss', + 'test_sign_op', + 'selected_rows_functor_gpu_test', + 'test_fleet_base', + 'test_logsumexp', + 'test_detection', + 'test_image_classification_fp16', + 'test_random_seed', + 'test_op_function_generator', + 'test_unique_with_counts', + 'test_complex_elementwise_layers', + 'test_array_read_write_op', + 'test_fusion_group_op', + 'test_imperative_layer_apply', 'test_executor_return_tensor_not_overwriting', - 'test_optimizer_in_control_flow', 'test_lookup_table_op', 'test_randint_op', - 'test_randint_like', 'test_convert_call', - 'test_sigmoid_cross_entropy_with_logits_op', 'copy_cross_scope_test', - 'test_normalization_wrapper', 'test_flip', 'test_cosine_similarity_api', - 'test_cumsum_op', 'test_range', 'test_log_loss_op', 'test_where_index', - 'test_tril_triu_op', 'test_lod_reset_op', 'test_lod_tensor', - 'test_addmm_op', 'test_index_select_op', 'test_index_add_op', 'test_nvprof', - 'test_index_sample_op', 'test_unstack_op', 'test_increment', - 'strided_memcpy_test', 'test_target_assign_op', + 'test_optimizer_in_control_flow', + 'test_lookup_table_op', + 'test_randint_op', + 'test_randint_like', + 'test_convert_call', + 'test_sigmoid_cross_entropy_with_logits_op', + 'copy_cross_scope_test', + 'test_normalization_wrapper', + 'test_flip', + 'test_cosine_similarity_api', + 'test_cumsum_op', + 'test_range', + 'test_log_loss_op', + 'test_where_index', + 'test_tril_triu_op', + 'test_lod_reset_op', + 'test_lod_tensor', + 'test_addmm_op', + 'test_index_select_op', + 'test_index_add_op', + 'test_nvprof', + 'test_index_sample_op', + 'test_unstack_op', + 'test_increment', + 'strided_memcpy_test', + 'test_target_assign_op', 'test_trt_dynamic_shape_transformer_prune', - 'test_box_decoder_and_assign_op', 'test_trt_dynamic_shape', 'test_mnist', - 'test_convert_operators', 'test_fill_any_like_op', 'test_fill_constant_op', - 'test_callback_reduce_lr_on_plateau', 'test_tile_op', 'test_logical', - 'test_deformable_conv_op', 'test_elementwise_add_grad_grad', - 'test_simple_rnn_op', 'test_bicubic_interp_op', 'test_batch_norm_op_v2', - 'test_custom_relu_op_jit', 'test_math_op_patch_var_base', 'test_se_resnet', - 'test_device_guard', 'test_elementwise_div_grad_grad', 'test_minus_op', - 'test_shard_index_op', 'test_dygraph_recompute', 'test_momentum_op', - 'test_modelaverage', 'test_compare_reduce_op', 'test_affine_grid_op', - 'test_allclose_layer', 'test_elementwise_pow_op', 'test_trt_subgraph_pass', - 'test_adaptive_avg_pool2d', 'test_functional_conv3d', - 'test_executor_and_mul', 'test_kron_op', 'test_cast_mkldnn_op', - 'test_imperative_auto_prune', 'allocator_facade_frac_flags_test', - 'test_fill_zeros_like_op', 'test_gather_tree_op', 'test_elementwise_mul_op', - 'test_cycle_gan', 'test_parallel_executor_transformer_auto_growth', - 'test_bitwise_op', 'test_uniform_random_op', 'trt_split_converter_test', - 'test_huber_loss_op', 'test_slice', 'test_label_smooth_functional', - 'test_conv_shift_op', 'test_imperative_optimizer_v2', 'test_len', - 'test_imperative_named_members', 'test_sequence_reshape', - 'test_elementwise_min_op', 'test_flatten2_op', 'test_param_guard', - 'test_imperative_ptb_rnn', 'test_batch_fc_op', 'test_Tensor_type', - 'test_complex_getitem', 'lod_tensor_gpu_test', 'im2col_test', - 'test_unbind_op', 'test_imperative_ptq', 'test_auc_single_pred_op', - 'test_imperative_reinforcement', 'test_tf32_cublas', 'test_return', - 'test_py_reader_push_pop', 'test_lstm', 'test_dygraph_mnist_fp16', - 'test_shuffle_channel_op', 'test_partial_concat_op', - 'test_fill_zeros_like2_op', 'test_deformable_conv_v1_op', - 'test_complex_grad_accumulated', 'test_sequence_mask', 'test_fill_op', - 'test_imperative_deepcf', 'test_reorder_lod_tensor', 'test_multiply', - 'test_partial_program', 'test_fetch_feed', 'test_group', - 'test_trt_reduce_sum_op', 'data_type_transform_test', 'test_gru_rnn_op', - 'test_cudnn_grucell', 'test_argsort_op', 'test_batch_norm_op', - 'test_inplace', 'test_deprecated_decorator', 'test_complex_cast', - 'test_diag_v2', 'test_iou_similarity_op', - 'test_inplace_auto_generated_apis', 'test_dataset', 'test_bilinear_api', - 'test_empty_like_op', 'test_imperative_layer_children', 'nccl_op_test', - 'test_tree_conv_op', 'test_share_data_op', - 'test_ir_memory_optimize_transformer', 'test_lod_append_op', - 'test_math_op_patch', 'test_base_layer', 'test_dequantize_log_op', - 'test_complex_matmul', 'test_prelu_op', 'test_l1_norm_op', - 'test_rmsprop_op', 'test_fuse_bn_act_pass', 'test_inplace_addto_strategy', - 'test_ptb_lm_v2', 'test_paddle_save_load', 'test_prelu_mkldnn_op', - 'test_box_coder_op', 'test_atan2_op', 'test_unsqueeze_op', 'test_profiler', - 'test_affine_channel_op', 'test_leaky_relu_grad_grad_functor', - 'test_ctc_align', 'test_fuse_relu_depthwise_conv_pass', 'test_complex_kron', - 'test_imperative_skip_op', 'test_dgc_op', 'test_regularizer_api', - 'test_nll_loss', 'test_imperative_layers', 'test_rnn_decode_api', - 'test_imperative_partitial_backward', 'test_where_op', 'test_std_layer', - 'test_ir_embedding_eltwise_layernorm_fuse_pass', 'test_multihead_attention', - 'test_grid_sampler_op', 'test_initializer_nn', 'test_var_base', - 'test_fuse_elewise_add_act_pass', 'test_select_input_output_op', - 'test_lstm_op', 'test_break_continue', - 'test_imperative_parallel_coalesce_split', 'test_expand_as_op', - 'test_user_defined_quantization', 'test_tensor_to_list', - 'test_limit_gpu_memory', 'test_adamax_api', - 'test_softmax_mask_fuse_upper_triangle_op', 'test_fake_quantize_op', - 'vol2col_test', 'test_cast_op', 'test_proximal_gd_op', 'test_mul_nn_grad', - 'test_full_like_op', 'test_imperative_static_runner_while', - 'trt_instance_norm_test', 'test_elementwise_mod_op', - 'test_grad_clip_minimize', 'test_one_hot_v2_op', 'test_complex_sum_layer', - 'test_isfinite_v2_op', 'test_is_empty_op', 'test_simnet_v2', - 'beam_search_test', 'test_randperm_op', 'test_elementwise_add_op_inplace', - 'test_imperative_selected_rows', 'test_py_reader_using_executor', - 'test_activation_op', 'test_nn_functional_embedding_dygraph', - 'test_reshape_op', 'test_maxout_op', 'test_cudnn_lstmcell', - 'test_sigmoid_focal_loss', 'test_manual_seed', 'test_lrn_op', - 'test_ir_memory_optimize_nlp', 'test_dataset_dataloader', - 'test_complex_variable', 'test_lite_engine', 'test_neg_op', - 'test_view_op_reuse_allocation', 'test_split_op', 'test_ptb_lm', - 'test_elementwise_sub_op', 'test_compare_op', 'test_simnet', - 'test_label_semantic_roles', 'test_normal', - 'test_tensor_scalar_type_promotion_static', 'test_trt_group_norm_op', - 'test_learning_rate_scheduler', 'test_numel_op', 'test_adaptive_max_pool3d', - 'test_sequential', 'test_imperative_optimizer', 'test_subtract_op', - 'test_conv_transpose_nn_grad', 'test_sigmoid_focal_loss_op', - 'test_cuda_stream_event', 'test_sequence_pad_op', 'test_rnn_cells', - 'test_partial_sum_op', 'test_rnn_nets_static', 'test_max_op', - 'test_logical_op', 'test_squared_l2_norm_op', 'test_center_loss', - 'test_quantization_pass', 'test_imperative_gnn', - 'test_conv_elementwise_add_act_fuse_pass', 'test_roll_op', - 'test_imperative_container_layerdict', 'test_shape_op', 'test_bmm_op', - 'test_matmul_v2_op', 'test_hinge_loss_op', 'test_imperative_qat', - 'test_add_position_encoding_op', 'test_rnn_op', 'test_gradient_clip', - 'test_py_reader_pin_memory', 'test_concat_op', 'test_weight_decay_extend', - 'test_accuracy_op', 'test_cond', 'test_resnet_v2', 'test_adagrad_op', - 'test_mv_op', 'test_print_op', 'test_grad', 'test_square_error_cost', - 'test_rnn_cells_static', 'test_mkldnn_batch_norm_act_fuse_pass', - 'test_input_spec', 'test_adam_op', 'test_elementwise_floordiv_op', - 'test_eager_deletion_gru_net', 'test_diagonal_op', - 'test_imperative_static_runner_mnist', 'test_nearest_interp_op', - 'test_diag_embed', 'test_imperative_basic', 'test_merge_selectedrows_op', - 'test_feed_data_check_shape_type', 'test_complex_trace_layer', - 'test_slice_op', 'test_bmn', 'test_nn_quant_functional_layers', - 'test_broadcast_tensors_op', 'test_selu_op', 'test_group_norm_op_v2', - 'test_tensor_to_numpy', 'test_queue', 'test_rank_loss_op', 'test_trace_op', - 'test_case', 'test_prroi_pool_op', 'test_op_name_conflict', - 'test_psroi_pool_op', 'test_set_value_op', 'test_ones_like', - 'test_assign_value_op', 'test_ema', 'test_lamb_op', 'test_dgc_momentum_op', - 'test_custom_grad_input', 'test_trunc_op', 'test_bernoulli_op', - 'test_custom_relu_model', 'test_backward', 'test_conv3d_transpose_part2_op', - 'test_complex_transpose', 'test_memory_reuse_exclude_feed_var', - 'test_polygon_box_transform', 'math_function_gpu_test', - 'test_program_prune_backward', 'test_ema_fleet', 'test_fleet_amp_init', - 'test_normalize', 'test_correlation', + 'test_box_decoder_and_assign_op', + 'test_trt_dynamic_shape', + 'test_mnist', + 'test_convert_operators', + 'test_fill_any_like_op', + 'test_fill_constant_op', + 'test_callback_reduce_lr_on_plateau', + 'test_tile_op', + 'test_logical', + 'test_deformable_conv_op', + 'test_elementwise_add_grad_grad', + 'test_simple_rnn_op', + 'test_bicubic_interp_op', + 'test_batch_norm_op_v2', + 'test_custom_relu_op_jit', + 'test_math_op_patch_var_base', + 'test_se_resnet', + 'test_device_guard', + 'test_elementwise_div_grad_grad', + 'test_minus_op', + 'test_shard_index_op', + 'test_dygraph_recompute', + 'test_momentum_op', + 'test_modelaverage', + 'test_compare_reduce_op', + 'test_affine_grid_op', + 'test_allclose_layer', + 'test_elementwise_pow_op', + 'test_trt_subgraph_pass', + 'test_adaptive_avg_pool2d', + 'test_functional_conv3d', + 'test_executor_and_mul', + 'test_kron_op', + 'test_cast_mkldnn_op', + 'test_imperative_auto_prune', + 'allocator_facade_frac_flags_test', + 'test_fill_zeros_like_op', + 'test_gather_tree_op', + 'test_elementwise_mul_op', + 'test_cycle_gan', + 'test_parallel_executor_transformer_auto_growth', + 'test_bitwise_op', + 'test_uniform_random_op', + 'trt_split_converter_test', + 'test_huber_loss_op', + 'test_slice', + 'test_label_smooth_functional', + 'test_conv_shift_op', + 'test_imperative_optimizer_v2', + 'test_len', + 'test_imperative_named_members', + 'test_sequence_reshape', + 'test_elementwise_min_op', + 'test_flatten2_op', + 'test_param_guard', + 'test_imperative_ptb_rnn', + 'test_batch_fc_op', + 'test_Tensor_type', + 'test_complex_getitem', + 'lod_tensor_gpu_test', + 'im2col_test', + 'test_unbind_op', + 'test_imperative_ptq', + 'test_auc_single_pred_op', + 'test_imperative_reinforcement', + 'test_tf32_cublas', + 'test_return', + 'test_py_reader_push_pop', + 'test_lstm', + 'test_dygraph_mnist_fp16', + 'test_shuffle_channel_op', + 'test_partial_concat_op', + 'test_fill_zeros_like2_op', + 'test_deformable_conv_v1_op', + 'test_complex_grad_accumulated', + 'test_sequence_mask', + 'test_fill_op', + 'test_imperative_deepcf', + 'test_reorder_lod_tensor', + 'test_multiply', + 'test_partial_program', + 'test_fetch_feed', + 'test_group', + 'test_trt_reduce_sum_op', + 'data_type_transform_test', + 'test_gru_rnn_op', + 'test_cudnn_grucell', + 'test_argsort_op', + 'test_batch_norm_op', + 'test_inplace', + 'test_deprecated_decorator', + 'test_complex_cast', + 'test_diag_v2', + 'test_iou_similarity_op', + 'test_inplace_auto_generated_apis', + 'test_dataset', + 'test_bilinear_api', + 'test_empty_like_op', + 'test_imperative_layer_children', + 'nccl_op_test', + 'test_tree_conv_op', + 'test_share_data_op', + 'test_ir_memory_optimize_transformer', + 'test_lod_append_op', + 'test_math_op_patch', + 'test_base_layer', + 'test_dequantize_log_op', + 'test_complex_matmul', + 'test_prelu_op', + 'test_l1_norm_op', + 'test_rmsprop_op', + 'test_fuse_bn_act_pass', + 'test_inplace_addto_strategy', + 'test_ptb_lm_v2', + 'test_paddle_save_load', + 'test_prelu_mkldnn_op', + 'test_box_coder_op', + 'test_atan2_op', + 'test_unsqueeze_op', + 'test_profiler', + 'test_affine_channel_op', + 'test_leaky_relu_grad_grad_functor', + 'test_ctc_align', + 'test_fuse_relu_depthwise_conv_pass', + 'test_complex_kron', + 'test_imperative_skip_op', + 'test_dgc_op', + 'test_regularizer_api', + 'test_nll_loss', + 'test_imperative_layers', + 'test_rnn_decode_api', + 'test_imperative_partitial_backward', + 'test_where_op', + 'test_std_layer', + 'test_ir_embedding_eltwise_layernorm_fuse_pass', + 'test_multihead_attention', + 'test_grid_sampler_op', + 'test_initializer_nn', + 'test_var_base', + 'test_fuse_elewise_add_act_pass', + 'test_select_input_output_op', + 'test_lstm_op', + 'test_break_continue', + 'test_imperative_parallel_coalesce_split', + 'test_expand_as_op', + 'test_user_defined_quantization', + 'test_tensor_to_list', + 'test_limit_gpu_memory', + 'test_adamax_api', + 'test_softmax_mask_fuse_upper_triangle_op', + 'test_fake_quantize_op', + 'vol2col_test', + 'test_cast_op', + 'test_proximal_gd_op', + 'test_mul_nn_grad', + 'test_full_like_op', + 'test_imperative_static_runner_while', + 'trt_instance_norm_test', + 'test_elementwise_mod_op', + 'test_grad_clip_minimize', + 'test_one_hot_v2_op', + 'test_complex_sum_layer', + 'test_isfinite_v2_op', + 'test_is_empty_op', + 'test_simnet_v2', + 'beam_search_test', + 'test_randperm_op', + 'test_elementwise_add_op_inplace', + 'test_imperative_selected_rows', + 'test_py_reader_using_executor', + 'test_activation_op', + 'test_nn_functional_embedding_dygraph', + 'test_reshape_op', + 'test_maxout_op', + 'test_cudnn_lstmcell', + 'test_sigmoid_focal_loss', + 'test_manual_seed', + 'test_lrn_op', + 'test_ir_memory_optimize_nlp', + 'test_dataset_dataloader', + 'test_complex_variable', + 'test_lite_engine', + 'test_neg_op', + 'test_view_op_reuse_allocation', + 'test_split_op', + 'test_ptb_lm', + 'test_elementwise_sub_op', + 'test_compare_op', + 'test_simnet', + 'test_label_semantic_roles', + 'test_normal', + 'test_tensor_scalar_type_promotion_static', + 'test_trt_group_norm_op', + 'test_learning_rate_scheduler', + 'test_numel_op', + 'test_adaptive_max_pool3d', + 'test_sequential', + 'test_imperative_optimizer', + 'test_subtract_op', + 'test_conv_transpose_nn_grad', + 'test_sigmoid_focal_loss_op', + 'test_cuda_stream_event', + 'test_sequence_pad_op', + 'test_rnn_cells', + 'test_partial_sum_op', + 'test_rnn_nets_static', + 'test_max_op', + 'test_logical_op', + 'test_squared_l2_norm_op', + 'test_center_loss', + 'test_quantization_pass', + 'test_imperative_gnn', + 'test_conv_elementwise_add_act_fuse_pass', + 'test_roll_op', + 'test_imperative_container_layerdict', + 'test_shape_op', + 'test_bmm_op', + 'test_matmul_v2_op', + 'test_hinge_loss_op', + 'test_imperative_qat', + 'test_add_position_encoding_op', + 'test_rnn_op', + 'test_gradient_clip', + 'test_py_reader_pin_memory', + 'test_concat_op', + 'test_weight_decay_extend', + 'test_accuracy_op', + 'test_cond', + 'test_resnet_v2', + 'test_adagrad_op', + 'test_mv_op', + 'test_print_op', + 'test_grad', + 'test_square_error_cost', + 'test_rnn_cells_static', + 'test_mkldnn_batch_norm_act_fuse_pass', + 'test_input_spec', + 'test_adam_op', + 'test_elementwise_floordiv_op', + 'test_eager_deletion_gru_net', + 'test_diagonal_op', + 'test_imperative_static_runner_mnist', + 'test_nearest_interp_op', + 'test_diag_embed', + 'test_imperative_basic', + 'test_merge_selectedrows_op', + 'test_feed_data_check_shape_type', + 'test_complex_trace_layer', + 'test_slice_op', + 'test_bmn', + 'test_nn_quant_functional_layers', + 'test_broadcast_tensors_op', + 'test_selu_op', + 'test_group_norm_op_v2', + 'test_tensor_to_numpy', + 'test_queue', + 'test_rank_loss_op', + 'test_trace_op', + 'test_case', + 'test_prroi_pool_op', + 'test_op_name_conflict', + 'test_psroi_pool_op', + 'test_set_value_op', + 'test_ones_like', + 'test_assign_value_op', + 'test_ema', + 'test_lamb_op', + 'test_dgc_momentum_op', + 'test_custom_grad_input', + 'test_trunc_op', + 'test_bernoulli_op', + 'test_custom_relu_model', + 'test_backward', + 'test_conv3d_transpose_part2_op', + 'test_complex_transpose', + 'test_memory_reuse_exclude_feed_var', + 'test_polygon_box_transform', + 'math_function_gpu_test', + 'test_program_prune_backward', + 'test_ema_fleet', + 'test_fleet_amp_init', + 'test_normalize', + 'test_correlation', 'test_conv_elementwise_add2_act_fuse_pass', - 'test_imperative_container_layerlist', 'test_dequantize_abs_max_op', - 'test_fuse_optimizer_pass', 'test_optimizer', - 'test_dynamic_rnn_stop_gradient', 'test_raw_program_optimizer', 'test_pow', - 'test_inplace_softmax_with_cross_entropy', 'test_transforms', - 'test_unfold_op', 'test_assign_op', 'test_isinstance', - 'auto_growth_best_fit_allocator_facade_test', 'test_cholesky_op', - 'test_adaptive_avg_pool3d', 'test_paddle_save_load_binary', - 'test_fused_fc_elementwise_layernorm_op', 'test_sequence_enumerate_op', - 'test_lgamma_op', 'test_modified_huber_loss_op', 'trt_quant_int8_test', - 'test_callback_visualdl', 'test_linspace', 'test_update_loss_scaling_op', - 'test_arg_min_max_op', 'test_bce_loss', 'test_nn_margin_rank_loss', - 'test_arg_min_max_v2_op', 'test_variance_layer', - 'test_quantization_scale_pass', 'test_segment_ops', 'test_layers', - 'test_isfinite_op', 'test_imperative_qat_channelwise', 'test_eye_op', - 'test_imperative_framework', 'test_l1_loss', 'test_ifelse', - 'test_cache_program', 'test_ir_fc_fuse_pass', 'test_kldiv_loss_op', - 'test_switch_case', 'test_unique', 'test_prod_op', 'test_edit_distance_op', - 'test_sequence_expand_as', 'test_full_name_usage', 'test_glu', - 'test_pad2d_op', 'test_read_file', 'test_erf_op', 'test_sequence_unpad_op', - 'test_sequence_conv', 'allocator_facade_abs_flags_test', 'test_detach', - 'test_cross_entropy_op', 'test_wrappers', 'test_fleet_base_single', - 'test_conv_elementwise_add_fuse_pass', 'test_auto_growth_gpu_memory_limit', - 'test_sequence_reverse', 'test_fc_op', 'test_diagflat', 'test_adamax_op', - 'test_op_attr', 'paddle_infer_api_test', 'test_mixed_precision', - 'lite_mul_model_test', 'test_sort_op', 'test_imperative_out_scale', - 'test_vision_models', 'test_rnn_encoder_decoder', 'test_fleet_with_asp_amp', + 'test_imperative_container_layerlist', + 'test_dequantize_abs_max_op', + 'test_fuse_optimizer_pass', + 'test_optimizer', + 'test_dynamic_rnn_stop_gradient', + 'test_raw_program_optimizer', + 'test_pow', + 'test_inplace_softmax_with_cross_entropy', + 'test_transforms', + 'test_unfold_op', + 'test_assign_op', + 'test_isinstance', + 'auto_growth_best_fit_allocator_facade_test', + 'test_cholesky_op', + 'test_adaptive_avg_pool3d', + 'test_paddle_save_load_binary', + 'test_fused_fc_elementwise_layernorm_op', + 'test_sequence_enumerate_op', + 'test_lgamma_op', + 'test_modified_huber_loss_op', + 'trt_quant_int8_test', + 'test_callback_visualdl', + 'test_linspace', + 'test_update_loss_scaling_op', + 'test_arg_min_max_op', + 'test_bce_loss', + 'test_nn_margin_rank_loss', + 'test_arg_min_max_v2_op', + 'test_variance_layer', + 'test_quantization_scale_pass', + 'test_segment_ops', + 'test_layers', + 'test_isfinite_op', + 'test_imperative_qat_channelwise', + 'test_eye_op', + 'test_imperative_framework', + 'test_l1_loss', + 'test_ifelse', + 'test_cache_program', + 'test_ir_fc_fuse_pass', + 'test_kldiv_loss_op', + 'test_switch_case', + 'test_unique', + 'test_prod_op', + 'test_edit_distance_op', + 'test_sequence_expand_as', + 'test_full_name_usage', + 'test_glu', + 'test_pad2d_op', + 'test_read_file', + 'test_erf_op', + 'test_sequence_unpad_op', + 'test_sequence_conv', + 'allocator_facade_abs_flags_test', + 'test_detach', + 'test_cross_entropy_op', + 'test_wrappers', + 'test_fleet_base_single', + 'test_conv_elementwise_add_fuse_pass', + 'test_auto_growth_gpu_memory_limit', + 'test_sequence_reverse', + 'test_fc_op', + 'test_diagflat', + 'test_adamax_op', + 'test_op_attr', + 'paddle_infer_api_test', + 'test_mixed_precision', + 'lite_mul_model_test', + 'test_sort_op', + 'test_imperative_out_scale', + 'test_vision_models', + 'test_rnn_encoder_decoder', + 'test_fleet_with_asp_amp', 'test_partial_eager_deletion_transformer', - 'test_imperative_star_gan_with_gradient_penalty', 'test_stack_op', - 'test_shuffle_batch_op', 'test_clip_op', 'test_py_func_op', - 'test_pool_max_op', 'test_log_softmax', - 'test_imperative_container_parameterlist', 'test_multiplex_op', + 'test_imperative_star_gan_with_gradient_penalty', + 'test_stack_op', + 'test_shuffle_batch_op', + 'test_clip_op', + 'test_py_func_op', + 'test_pool_max_op', + 'test_log_softmax', + 'test_imperative_container_parameterlist', + 'test_multiplex_op', 'test_trt_transpose_flatten_concat_fuse_pass', - 'test_seqconv_eltadd_relu_fuse_pass', 'test_assert_op', - 'test_scatter_nd_op', 'test_sequence_expand', 'test_arange', - 'test_translated_layer', 'test_decoupled_py_reader_data_check', - 'test_analyzer_ernie_large', 'test_tensor_array_to_tensor', - 'test_functional_conv2d_transpose', 'test_error', 'test_callbacks', - 'test_imperative_recurrent_usage', 'test_deform_conv2d', - 'test_coalesce_tensor_op', 'test_tsm', 'test_fused_multihead_matmul_op', - 'test_softmax_mask_fuse_op', 'test_optimizer_grad', 'test_complex_abs', - 'test_gradient_accmulator', 'test_instance_norm_op_v2', - 'test_random_crop_op', 'test_mobile_net', + 'test_seqconv_eltadd_relu_fuse_pass', + 'test_assert_op', + 'test_scatter_nd_op', + 'test_sequence_expand', + 'test_arange', + 'test_translated_layer', + 'test_decoupled_py_reader_data_check', + 'test_analyzer_ernie_large', + 'test_tensor_array_to_tensor', + 'test_functional_conv2d_transpose', + 'test_error', + 'test_callbacks', + 'test_imperative_recurrent_usage', + 'test_deform_conv2d', + 'test_coalesce_tensor_op', + 'test_tsm', + 'test_fused_multihead_matmul_op', + 'test_softmax_mask_fuse_op', + 'test_optimizer_grad', + 'test_complex_abs', + 'test_gradient_accmulator', + 'test_instance_norm_op_v2', + 'test_random_crop_op', + 'test_mobile_net', 'test_parallel_executor_transformer', 'test_tensor_scalar_type_promotion_dynamic', - 'test_eager_deletion_delete_vars', 'test_asp_pruning_1d', - 'test_imperative_using_non_zero_gpu', 'test_machine_translation', - 'test_flatten_op', 'test_onnx_export', 'test_optimizer_for_varbase', - 'test_fusion_transpose_flatten_concat_op', 'best_fit_allocator_test', - 'test_ir_fusion_group_pass', 'test_trt_quant_conv2d_dequant_fuse_pass', - 'test_allclose_op', 'test_ftrl_op', 'test_elementwise_add_op', - 'test_instance_norm_op', 'test_lambv2_op', 'test_yolo_box_op', - 'test_parallel_executor_drop_scope', 'test_generator_dataloader', - 'test_conv2d_transpose_op_depthwise_conv', 'test_imperative_save_load_v2', - 'test_lookahead', 'test_moving_average_abs_max_scale_op', - 'test_roi_perspective_transform_op', 'test_tensorrt_engine', - 'test_affine_grid_function', 'test_nonzero_api', - 'test_ir_memory_optimize_pass', 'test_reduce_mkldnn_op', - 'test_bilinear_interp_op', 'test_cvm_op', 'test_scale_op', 'test_matmul_op', - 'test_sequence_pool', 'test_complex_simplenet', 'test_complex_reshape', - 'test_flatten_contiguous_range_op', 'test_python_operator_overriding', - 'lite_resnet50_test', 'test_sequence_erase_op', - 'test_deformable_psroi_pooling', 'test_multi_precision_fp16_train', - 'test_adam_op_multi_thread', 'test_decoupled_py_reader', - 'test_distribute_fpn_proposals_op', 'transform_test', 'test_nan_inf', - 'test_fuse_bn_add_act_pass', 'test_unpool_op', - 'test_parallel_executor_dry_run', 'test_layer_norm_op_v2', - 'test_embedding_id_stop_gradient', 'test_mkldnn_fc_act_fuse_pass', - 'sequence_pooling_test', 'test_get_tensor_from_selected_rows_op', - 'test_imperative_ptb_rnn_sorted_gradient', 'test_hapi_hub', - 'test_reverse_op', 'test_compiled_program', 'test_lambda', - 'test_adadelta_op', 'test_nn_sigmoid_op', 'test_nearest_interp_v2_op', - 'test_sequence_slice_op', 'test_program_translator', - 'test_eager_deletion_lstm_net', 'malloc_test', 'test_size_op', - 'test_analysis_predictor', 'test_recognize_digits', 'test_parameter', + 'test_eager_deletion_delete_vars', + 'test_asp_pruning_1d', + 'test_imperative_using_non_zero_gpu', + 'test_machine_translation', + 'test_flatten_op', + 'test_onnx_export', + 'test_optimizer_for_varbase', + 'test_fusion_transpose_flatten_concat_op', + 'best_fit_allocator_test', + 'test_ir_fusion_group_pass', + 'test_trt_quant_conv2d_dequant_fuse_pass', + 'test_allclose_op', + 'test_ftrl_op', + 'test_elementwise_add_op', + 'test_instance_norm_op', + 'test_lambv2_op', + 'test_yolo_box_op', + 'test_parallel_executor_drop_scope', + 'test_generator_dataloader', + 'test_conv2d_transpose_op_depthwise_conv', + 'test_imperative_save_load_v2', + 'test_lookahead', + 'test_moving_average_abs_max_scale_op', + 'test_roi_perspective_transform_op', + 'test_tensorrt_engine', + 'test_affine_grid_function', + 'test_nonzero_api', + 'test_ir_memory_optimize_pass', + 'test_reduce_mkldnn_op', + 'test_bilinear_interp_op', + 'test_cvm_op', + 'test_scale_op', + 'test_matmul_op', + 'test_sequence_pool', + 'test_complex_simplenet', + 'test_complex_reshape', + 'test_flatten_contiguous_range_op', + 'test_python_operator_overriding', + 'lite_resnet50_test', + 'test_sequence_erase_op', + 'test_deformable_psroi_pooling', + 'test_multi_precision_fp16_train', + 'test_adam_op_multi_thread', + 'test_decoupled_py_reader', + 'test_distribute_fpn_proposals_op', + 'transform_test', + 'test_nan_inf', + 'test_fuse_bn_add_act_pass', + 'test_unpool_op', + 'test_parallel_executor_dry_run', + 'test_layer_norm_op_v2', + 'test_embedding_id_stop_gradient', + 'test_mkldnn_fc_act_fuse_pass', + 'sequence_pooling_test', + 'test_get_tensor_from_selected_rows_op', + 'test_imperative_ptb_rnn_sorted_gradient', + 'test_hapi_hub', + 'test_reverse_op', + 'test_compiled_program', + 'test_lambda', + 'test_adadelta_op', + 'test_nn_sigmoid_op', + 'test_nearest_interp_v2_op', + 'test_sequence_slice_op', + 'test_program_translator', + 'test_eager_deletion_lstm_net', + 'malloc_test', + 'test_size_op', + 'test_analysis_predictor', + 'test_recognize_digits', + 'test_parameter', 'test_transpose_flatten_concat_fuse_pass', - 'test_imperative_trace_non_persistable_inputs', 'test_pass_builder', - 'thread_local_allocator_test', 'test_variable', 'test_fsp_op', - 'test_elementwise_gradient_op', 'test_multinomial_op', - 'test_trt_shuffle_channel_detect_pass', 'test_generate_proposals_v2_op', - 'test_graph', 'test_gelu_op', 'test_sample_logits_op', - 'test_weight_normalization', 'test_activation_bf16_mkldnn_op', - 'trt_dynamic_shape_test', 'test_traced_layer_err_msg', 'test_conv1d_layer', - 'test_asp_optimize', 'test_imperative_container_sequential', 'test_bert', - 'test_transformer_api', 'test_linear_interp_v2_op', 'test_pixel_shuffle', - 'test_expand_op', 'test_save_load', 'test_dropout_op', 'test_while_loop_op', - 'float16_gpu_test', 'test_dict', 'test_bilinear_tensor_product_op', - 'test_parallel_executor_pg', 'test_assert', 'test_smooth_l1_loss_op', - 'sequence_padding_test', 'test_analyzer_ernie', 'test_minimum_op', - 'test_yolov3_loss_op', 'test_decayed_adagrad_op', 'test_split_mkldnn_op', - 'test_squeeze_op', 'test_save_inference_model', 'test_smooth_l1_loss', - 'test_bilateral_slice_op', 'test_inplace_abn_op', 'test_fetch_unmerged', + 'test_imperative_trace_non_persistable_inputs', + 'test_pass_builder', + 'thread_local_allocator_test', + 'test_variable', + 'test_fsp_op', + 'test_elementwise_gradient_op', + 'test_multinomial_op', + 'test_trt_shuffle_channel_detect_pass', + 'test_generate_proposals_v2_op', + 'test_graph', + 'test_gelu_op', + 'test_sample_logits_op', + 'test_weight_normalization', + 'test_activation_bf16_mkldnn_op', + 'trt_dynamic_shape_test', + 'test_traced_layer_err_msg', + 'test_conv1d_layer', + 'test_asp_optimize', + 'test_imperative_container_sequential', + 'test_bert', + 'test_transformer_api', + 'test_linear_interp_v2_op', + 'test_pixel_shuffle', + 'test_expand_op', + 'test_save_load', + 'test_dropout_op', + 'test_while_loop_op', + 'float16_gpu_test', + 'test_dict', + 'test_bilinear_tensor_product_op', + 'test_parallel_executor_pg', + 'test_assert', + 'test_smooth_l1_loss_op', + 'sequence_padding_test', + 'test_analyzer_ernie', + 'test_minimum_op', + 'test_yolov3_loss_op', + 'test_decayed_adagrad_op', + 'test_split_mkldnn_op', + 'test_squeeze_op', + 'test_save_inference_model', + 'test_smooth_l1_loss', + 'test_bilateral_slice_op', + 'test_inplace_abn_op', + 'test_fetch_unmerged', 'test_parallel_executor_feed_persistable_var', 'test_parallel_executor_fetch_isolated_var', 'test_parallel_executor_inference_feed_partial_data', @@ -1050,37 +1540,66 @@ 'test_parallel_executor_test_while_train', 'test_parallel_executor_seresnext_with_fuse_all_reduce_gpu', 'test_parallel_ssa_graph_inference_feed_partial_data', - 'test_parallel_executor_seresnext_with_reduce_gpu', 'test_data_norm_op', - 'test_install_check', 'graph_node_test', 'trt_mobilenet_test', - 'trt_cascade_rcnn_test', 'trt_resnext_test', 'test_activation_nn_grad', - 'test_trt_dynamic_shape_ernie_fp16_ser_deser', 'test_cross_entropy2_op', - 'test_layer_norm_op', 'test_pool3d_op', 'test_static_save_load', - 'test_trt_flatten_op', 'test_trt_yolo_box_op', 'test_trt_reshape_op', - 'test_trt_elementwise_op', 'test_trt_affine_channel_op', 'test_trt_matmul', - 'test_trt_fc_fuse_pass', 'test_trt_pad_op', 'trt_resnet50_test', - 'test_imperative_lod_tensor_to_selected_rows', 'test_gru_unit_op', + 'test_parallel_executor_seresnext_with_reduce_gpu', + 'test_data_norm_op', + 'test_install_check', + 'graph_node_test', + 'trt_mobilenet_test', + 'trt_cascade_rcnn_test', + 'trt_resnext_test', + 'test_activation_nn_grad', + 'test_trt_dynamic_shape_ernie_fp16_ser_deser', + 'test_cross_entropy2_op', + 'test_layer_norm_op', + 'test_pool3d_op', + 'test_static_save_load', + 'test_trt_flatten_op', + 'test_trt_yolo_box_op', + 'test_trt_reshape_op', + 'test_trt_elementwise_op', + 'test_trt_affine_channel_op', + 'test_trt_matmul', + 'test_trt_fc_fuse_pass', + 'test_trt_pad_op', + 'trt_resnet50_test', + 'test_imperative_lod_tensor_to_selected_rows', + 'test_gru_unit_op', 'test_amp_check_finite_and_scale_op', - 'test_imperative_selected_rows_to_lod_tensor', 'test_imperative_save_load', - 'test_add_reader_dependency', 'test_imperative_transformer_sorted_gradient', - 'test_bicubic_interp_v2_op', 'test_rank_attention_op', - 'test_space_to_depth_op', 'test_image_classification', - 'test_custom_relu_op_setup', 'test_sgd_op' + 'test_imperative_selected_rows_to_lod_tensor', + 'test_imperative_save_load', + 'test_add_reader_dependency', + 'test_imperative_transformer_sorted_gradient', + 'test_bicubic_interp_v2_op', + 'test_rank_attention_op', + 'test_space_to_depth_op', + 'test_image_classification', + 'test_custom_relu_op_setup', + 'test_sgd_op', ] # mem != 0 : It run 7 job each time in Single cases; 3 job each time in exclusive cases FIFTH_PARALLEL_JOB_NEW = [ 'test_buffer_shared_memory_reuse_pass', 'test_buffer_shared_memory_reuse_pass_and_fuse_optimization_op_pass', - 'test_parallel_executor_crf', 'test_multiprocess_reader_exception', - 'buddy_allocator_test', 'test_multiprocess_dataloader_dataset', + 'test_parallel_executor_crf', + 'test_multiprocess_reader_exception', + 'buddy_allocator_test', + 'test_multiprocess_dataloader_dataset', 'test_multiprocess_dataloader_dynamic', - 'test_multiprocess_dataloader_static', 'test_imperative_resnet', - 'test_nn_grad', 'test_conv2d_op_depthwise_conv', 'test_yolov3', - 'test_conv_nn_grad', 'test_imperative_data_loader_fds_clear', - 'test_conv2d_op', 'test_imperative_data_loader_base', + 'test_multiprocess_dataloader_static', + 'test_imperative_resnet', + 'test_nn_grad', + 'test_conv2d_op_depthwise_conv', + 'test_yolov3', + 'test_conv_nn_grad', + 'test_imperative_data_loader_fds_clear', + 'test_conv2d_op', + 'test_imperative_data_loader_base', 'test_imperative_resnet_sorted_gradient', 'test_multiprocess_dataloader_iterable_dataset_dynamic', - 'test_imperative_se_resnext', 'test_norm_nn_grad', 'test_conv2d_api' + 'test_imperative_se_resnext', + 'test_norm_nn_grad', + 'test_conv2d_api', ] SIXTH_PARALLEL_JOB_NEW = [ @@ -2824,46 +3343,71 @@ def main(): if platform.system() != 'Windows': for unittest in SECONDARY_HIGH_PARALLEL_JOB_NEW: if unittest in test_cases: - secondary_high_parallel_job = secondary_high_parallel_job + '|^' + unittest + '$' + secondary_high_parallel_job = ( + secondary_high_parallel_job + '|^' + unittest + '$' + ) test_cases.remove(unittest) for unittest in THIRD_HIGH_PARALLEL_JOB_NEW: if unittest in test_cases: - third_high_parallel_job = third_high_parallel_job + '|^' + unittest + '$' + third_high_parallel_job = ( + third_high_parallel_job + '|^' + unittest + '$' + ) test_cases.remove(unittest) for unittest in SIXTH_PARALLEL_JOB_NEW: if unittest in test_cases: - sixth_high_parallel_job = sixth_high_parallel_job + '|^' + unittest + '$' + sixth_high_parallel_job = ( + sixth_high_parallel_job + '|^' + unittest + '$' + ) test_cases.remove(unittest) for unittest in LOWEST_PARALLEL_JOB_NEW: if unittest in test_cases: - lowest_high_parallel_job = lowest_high_parallel_job + '|^' + unittest + '$' + lowest_high_parallel_job = ( + lowest_high_parallel_job + '|^' + unittest + '$' + ) test_cases.remove(unittest) for unittest in fourth_high_parallel_job_list: if unittest in test_cases: - fourth_high_parallel_job = fourth_high_parallel_job + '|^' + unittest + '$' + fourth_high_parallel_job = ( + fourth_high_parallel_job + '|^' + unittest + '$' + ) test_cases.remove(unittest) for unittest in fifth_high_parallel_job_list: if unittest in test_cases: - fifth_high_parallel_job = fifth_high_parallel_job + '|^' + unittest + '$' + fifth_high_parallel_job = ( + fifth_high_parallel_job + '|^' + unittest + '$' + ) test_cases.remove(unittest) for unittest in test_cases: non_parallel_job = non_parallel_job + '|^' + unittest + '$' if platform.system() == 'Windows': - print("{};{};{};{}".format(high_parallel_job, fourth_high_parallel_job, - fifth_high_parallel_job, non_parallel_job)) + print( + "{};{};{};{}".format( + high_parallel_job, + fourth_high_parallel_job, + fifth_high_parallel_job, + non_parallel_job, + ) + ) else: - print("{};{};{};{};{};{};{};{}".format( - high_parallel_job, secondary_high_parallel_job, - third_high_parallel_job, fourth_high_parallel_job, - fifth_high_parallel_job, sixth_high_parallel_job, - lowest_high_parallel_job, non_parallel_job)) + print( + "{};{};{};{};{};{};{};{}".format( + high_parallel_job, + secondary_high_parallel_job, + third_high_parallel_job, + fourth_high_parallel_job, + fifth_high_parallel_job, + sixth_high_parallel_job, + lowest_high_parallel_job, + non_parallel_job, + ) + ) if __name__ == '__main__': diff --git a/tools/print_op_desc.py b/tools/print_op_desc.py index cf46849add4b1..c0ce5789b7b71 100644 --- a/tools/print_op_desc.py +++ b/tools/print_op_desc.py @@ -87,8 +87,11 @@ def get_attrs_info(op_proto, op_attrs_proto): attrs_info[attr_name] = {} attrs_info[attr_name][TYPE] = attr_proto.type attrs_info[attr_name][GENERATED] = attr_proto.generated - attrs_info[attr_name][DEFAULT_VALUE] = attrs_default_values[ - attr_name] if attr_name in attrs_default_values else None + attrs_info[attr_name][DEFAULT_VALUE] = ( + attrs_default_values[attr_name] + if attr_name in attrs_default_values + else None + ) attrs_info[attr_name][EXTRA] = attr_proto.extra attrs_info[attr_name][QUANT] = attr_proto.quant return attrs_info diff --git a/tools/print_signatures.py b/tools/print_signatures.py index 4c35fdf1c4a1a..4c85ba123beb7 100644 --- a/tools/print_signatures.py +++ b/tools/print_signatures.py @@ -40,7 +40,9 @@ logger.addHandler(console) console.setFormatter( logging.Formatter( - "%(asctime)s - %(funcName)s:%(lineno)d - %(levelname)s - %(message)s")) + "%(asctime)s - %(funcName)s:%(lineno)d - %(levelname)s - %(message)s" + ) +) def md5(doc): @@ -50,15 +52,18 @@ def md5(doc): md5sum = hashinst.hexdigest() except UnicodeDecodeError as e: md5sum = None - print("Error({}) occurred when `md5({})`, discard it.".format( - str(e), doc), - file=sys.stderr) + print( + "Error({}) occurred when `md5({})`, discard it.".format( + str(e), doc + ), + file=sys.stderr, + ) return md5sum def is_primitive(instance): - int_types = (int, ) + int_types = (int,) pritimitive_types = int_types + (float, str) if isinstance(instance, pritimitive_types): return True @@ -107,12 +112,16 @@ def visit_all_module(mod): if instance_id in IdSet: continue IdSet.add(instance_id) - if hasattr(instance, - '__name__') and member_name != instance.__name__: + if ( + hasattr(instance, '__name__') + and member_name != instance.__name__ + ): print( - "Found alias API, alias name is: {}, original name is: {}" - .format(member_name, instance.__name__), - file=sys.stderr) + "Found alias API, alias name is: {}, original name is: {}".format( + member_name, instance.__name__ + ), + file=sys.stderr, + ) except: if cur_name not in ErrorSet and cur_name not in skiplist: ErrorSet.add(cur_name) @@ -130,7 +139,8 @@ def get_all_api(root_path='paddle', attr="__all__"): global api_info_dict api_counter = 0 for filefinder, name, ispkg in pkgutil.walk_packages( - path=paddle.__path__, prefix=paddle.__name__ + '.'): + path=paddle.__path__, prefix=paddle.__name__ + '.' + ): try: if name in sys.modules: m = sys.modules[name] @@ -145,11 +155,17 @@ def get_all_api(root_path='paddle', attr="__all__"): api_counter += process_module(paddle, attr) - logger.info('%s: collected %d apis, %d distinct apis.', attr, api_counter, - len(api_info_dict)) + logger.info( + '%s: collected %d apis, %d distinct apis.', + attr, + api_counter, + len(api_info_dict), + ) - return [(sorted(list(api_info['all_names']))[0], md5(api_info['docstring'])) - for api_info in api_info_dict.values()] + return [ + (sorted(list(api_info['all_names']))[0], md5(api_info['docstring'])) + for api_info in api_info_dict.values() + ] def insert_api_into_dict(full_name, gen_doc_anno=None): @@ -165,8 +181,9 @@ def insert_api_into_dict(full_name, gen_doc_anno=None): logger.warning("AttributeError occurred when `id(eval(%s))`", full_name) return None except Exception as e: - logger.warning("Exception(%s) occurred when `id(eval(%s))`", str(e), - full_name) + logger.warning( + "Exception(%s) occurred when `id(eval(%s))`", str(e), full_name + ) return None else: logger.debug("adding %s to api_info_dict.", full_name) @@ -187,8 +204,8 @@ def insert_api_into_dict(full_name, gen_doc_anno=None): api_info_dict[fc_id]["gen_doc_anno"] = gen_doc_anno if inspect.isfunction(obj): api_info_dict[fc_id]["signature"] = repr( - inspect.getfullargspec(obj)).replace( - 'FullArgSpec', 'ArgSpec', 1) + inspect.getfullargspec(obj) + ).replace('FullArgSpec', 'ArgSpec', 1) return api_info_dict[fc_id] @@ -198,9 +215,11 @@ def process_module(m, attr="__all__"): if hasattr(m, attr): # may have duplication of api for api in set(getattr(m, attr)): - if api[0] == '_': continue + if api[0] == '_': + continue # Exception occurred when `id(eval(paddle.dataset.conll05.test, get_dict))` - if ',' in api: continue + if ',' in api: + continue # api's fullname full_name = m.__name__ + "." + api @@ -210,17 +229,21 @@ def process_module(m, attr="__all__"): if inspect.isclass(api_info['object']): for name, value in inspect.getmembers(api_info['object']): if (not name.startswith("_")) and hasattr( - value, '__name__'): - method_full_name = full_name + '.' + name # value.__name__ + value, '__name__' + ): + method_full_name = ( + full_name + '.' + name + ) # value.__name__ method_api_info = insert_api_into_dict( - method_full_name, 'class_method') + method_full_name, 'class_method' + ) if method_api_info is not None: api_counter += 1 return api_counter def check_public_api(): - modulelist = [ #npqa + modulelist = [ # npqa paddle, paddle.amp, paddle.nn, @@ -297,7 +320,8 @@ def check_public_api(): instance = eval(cur_name) doc_md5 = md5(instance.__doc__) member_dict[cur_name] = "({}, ('document', '{}'))".format( - cur_name, doc_md5) + cur_name, doc_md5 + ) def check_allmodule_callable(): @@ -314,18 +338,23 @@ def parse_args(): """ parser = argparse.ArgumentParser(description='Print Apis Signatures') parser.add_argument('--debug', dest='debug', action="store_true") - parser.add_argument('--method', - dest='method', - type=str, - default='get_all_api', - help="using get_all_api or from_modulelist") - parser.add_argument('module', type=str, help='module', - default='paddle') # not used - parser.add_argument('--skipped', - dest='skipped', - type=str, - help='Skip Checking submodules', - default='paddle.fluid.libpaddle.eager.ops') + parser.add_argument( + '--method', + dest='method', + type=str, + default='get_all_api', + help="using get_all_api or from_modulelist", + ) + parser.add_argument( + 'module', type=str, help='module', default='paddle' + ) # not used + parser.add_argument( + '--skipped', + dest='skipped', + type=str, + help='Skip Checking submodules', + default='paddle.fluid.libpaddle.eager.ops', + ) if len(sys.argv) == 1: args = parser.parse_args(['paddle']) @@ -357,14 +386,22 @@ def parse_args(): if args.skipped != '' and api_name.find(args.skipped) >= 0: continue api_info = api_info_dict[all_api_names_to_k[api_name]] - print("{0} ({2}, ('document', '{1}'))".format( - api_name, md5(api_info['docstring']), api_info['signature'] - if 'signature' in api_info else 'ArgSpec()')) + print( + "{0} ({2}, ('document', '{1}'))".format( + api_name, + md5(api_info['docstring']), + api_info['signature'] + if 'signature' in api_info + else 'ArgSpec()', + ) + ) if len(ErrorSet) == 0: sys.exit(0) else: for erroritem in ErrorSet: - print("Error, new function {} is unreachable".format(erroritem), - file=sys.stderr) + print( + "Error, new function {} is unreachable".format(erroritem), + file=sys.stderr, + ) sys.exit(1) diff --git a/tools/prune_for_jetson.py b/tools/prune_for_jetson.py index e87268a9fe2ee..8bfe9ebf9f8dd 100644 --- a/tools/prune_for_jetson.py +++ b/tools/prune_for_jetson.py @@ -53,16 +53,20 @@ def find_kernel(content, pattern): def prune_phi_kernels(): tool_dir = os.path.dirname(os.path.abspath(__file__)) - all_op = glob.glob(os.path.join(tool_dir, '../paddle/phi/kernels/**/*.cc'), - recursive=True) - all_op += glob.glob(os.path.join(tool_dir, '../paddle/phi/kernels/**/*.cu'), - recursive=True) + all_op = glob.glob( + os.path.join(tool_dir, '../paddle/phi/kernels/**/*.cc'), recursive=True + ) + all_op += glob.glob( + os.path.join(tool_dir, '../paddle/phi/kernels/**/*.cu'), recursive=True + ) register_op_count = 0 for op_file in all_op: need_continue = False file_blacklist = [ - "kernels/empty_kernel.cc", "/cast_kernel.c", "/batch_norm_kernel.c" + "kernels/empty_kernel.cc", + "/cast_kernel.c", + "/batch_norm_kernel.c", ] for bname in file_blacklist: if op_file.find(bname) >= 0: @@ -99,19 +103,25 @@ def apply_patches(): ret = os.system( "cd %s && rm -f paddle/fluid/inference/api/tensorrt_predictor.* " " && rm -f paddle/fluid/inference/api/paddle_tensorrt_predictor.h " - " && git apply tools/infer_prune_patches/*.patch && cd -" % work_path) + " && git apply tools/infer_prune_patches/*.patch && cd -" % work_path + ) return ret == 0 def append_fluid_kernels(): op_white_list = ["load", "load_combine"] - #1. add to makefile - file_name = os.path.dirname(os.path.abspath(__file__)) \ - + "/../paddle/fluid/inference/tensorrt/CMakeLists.txt" + # 1. add to makefile + file_name = ( + os.path.dirname(os.path.abspath(__file__)) + + "/../paddle/fluid/inference/tensorrt/CMakeLists.txt" + ) append_str = "\nfile(APPEND ${pybind_file} \"USE_NO_KERNEL_OP__(tensorrt_engine);\\n\")\n" for op in op_white_list: - append_str = append_str + "file(APPEND ${pybind_file} \"USE_OP__(%s);\\n\")\n" % op + append_str = ( + append_str + + "file(APPEND ${pybind_file} \"USE_OP__(%s);\\n\")\n" % op + ) with open(file_name, 'r', encoding='utf-8') as f: content = ''.join(f.readlines()) @@ -120,22 +130,26 @@ def append_fluid_kernels(): new_content = content.replace(location_str, location_str + append_str) if new_content == content: - print("ERROR: can not find \"%s\" in file \"%s\"" % - (location_str, file_name)) + print( + "ERROR: can not find \"%s\" in file \"%s\"" + % (location_str, file_name) + ) return False with open(file_name, 'w', encoding='utf-8') as f: f.write(u'{}'.format(new_content)) - #2. add op and kernel register + # 2. add op and kernel register op_white_list.append("tensorrt_engine") tool_dir = os.path.dirname(os.path.abspath(__file__)) - all_op = glob.glob(os.path.join(tool_dir, - '../paddle/fluid/operators/**/*.cc'), - recursive=True) - all_op += glob.glob(os.path.join(tool_dir, - '../paddle/fluid/operators/**/*.cu'), - recursive=True) + all_op = glob.glob( + os.path.join(tool_dir, '../paddle/fluid/operators/**/*.cc'), + recursive=True, + ) + all_op += glob.glob( + os.path.join(tool_dir, '../paddle/fluid/operators/**/*.cu'), + recursive=True, + ) for op_file in all_op: with open(op_file, 'r', encoding='utf-8') as f: @@ -143,18 +157,18 @@ def append_fluid_kernels(): for op in op_white_list: patterns = { - "REGISTER_OPERATOR": - r"REGISTER_OPERATOR\(\s*%s\s*," % op, - "REGISTER_OP_CPU_KERNEL": - r"REGISTER_OP_CPU_KERNEL\(\s*%s\s*," % op, - "REGISTER_OP_CUDA_KERNEL": - r"REGISTER_OP_CUDA_KERNEL\(\s*%s\s*," % op + "REGISTER_OPERATOR": r"REGISTER_OPERATOR\(\s*%s\s*," % op, + "REGISTER_OP_CPU_KERNEL": r"REGISTER_OP_CPU_KERNEL\(\s*%s\s*," + % op, + "REGISTER_OP_CUDA_KERNEL": r"REGISTER_OP_CUDA_KERNEL\(\s*%s\s*," + % op, } for k, p in patterns.items(): matches = re.findall(p, content, flags=re.DOTALL) if len(matches) > 0: - content = content.replace(matches[0], - matches[0].replace(k, k + "__")) + content = content.replace( + matches[0], matches[0].replace(k, k + "__") + ) with open(op_file, 'w', encoding='utf-8') as f: f.write(u'{}'.format(content)) @@ -164,13 +178,13 @@ def append_fluid_kernels(): if __name__ == '__main__': print("================ step 1: apply patches =======================") - assert (apply_patches()) + assert apply_patches() print("==============================================================\n") print("================ step 2: append fluid op/kernels==============") - assert (append_fluid_kernels()) + assert append_fluid_kernels() print("==============================================================\n") print("================ step 3:prune phi kernels ====================") - assert (prune_phi_kernels()) + assert prune_phi_kernels() print("==============================================================\n") diff --git a/tools/pyCov_multithreading.py b/tools/pyCov_multithreading.py index fb4cef6805ff3..e808dcb55e0ac 100644 --- a/tools/pyCov_multithreading.py +++ b/tools/pyCov_multithreading.py @@ -32,9 +32,12 @@ def worker(fun): def threadPool(threadPoolNum): threadPool = [] for i in range(threadPoolNum): - thread = threading.Thread(target=worker, args={ - doFun, - }) + thread = threading.Thread( + target=worker, + args={ + doFun, + }, + ) thread.daemon = True threadPool.append(thread) return threadPool @@ -49,8 +52,9 @@ def getPyCovResult(params): os.system('cd %s && coverage combine `ls python-coverage.data.*`' % path) os.system('cd %s && pwd && coverage xml -i -o python-coverage.xml' % path) xml_path = '%s/python-coverage.xml' % path - os.system("python2.7 %s/tools/analysisPyXml.py %s %s" % - (rootPath, rootPath, ut)) + os.system( + "python2.7 %s/tools/analysisPyXml.py %s %s" % (rootPath, rootPath, ut) + ) endTime = int(time.time()) print('pyCov Time: %s' % (endTime - startTime)) diff --git a/tools/remove_grad_op_and_kernel.py b/tools/remove_grad_op_and_kernel.py index 19778c27dbe12..8aa456bdb671f 100644 --- a/tools/remove_grad_op_and_kernel.py +++ b/tools/remove_grad_op_and_kernel.py @@ -54,8 +54,11 @@ def update_operator_cmake(cmake_file): match = re.findall(pat2, content, flags=re.DOTALL) content = content.replace( - match[0], code2 + '\n' + - match[0].replace('py_func_op', 'py_func_op ${LOSS_OPS}')) + match[0], + code2 + + '\n' + + match[0].replace('py_func_op', 'py_func_op ${LOSS_OPS}'), + ) with open(cmake_file, 'w') as f: f.write(content) @@ -65,16 +68,23 @@ def update_operator_cmake(cmake_file): tool_dir = os.path.dirname(os.path.abspath(__file__)) - all_op = glob.glob(os.path.join(tool_dir, - '../paddle/fluid/operators/**/*.cc'), - recursive=True) - all_op += glob.glob(os.path.join(tool_dir, - '../paddle/fluid/operators/**/*.cu'), - recursive=True) + all_op = glob.glob( + os.path.join(tool_dir, '../paddle/fluid/operators/**/*.cc'), + recursive=True, + ) + all_op += glob.glob( + os.path.join(tool_dir, '../paddle/fluid/operators/**/*.cu'), + recursive=True, + ) spec_ops = ['activation_op.cc'] - register_op_count, register_op_cpu_kernel_count, register_op_cuda_kernel_count, register_op_xpu_kernel_count = 0, 0, 0, 0 + ( + register_op_count, + register_op_cpu_kernel_count, + register_op_cuda_kernel_count, + register_op_xpu_kernel_count, + ) = (0, 0, 0, 0) register_op_kernel_count, register_op_kernel_with_custom_type_count = 0, 0 # 1. remove all grad op and kernel @@ -100,7 +110,9 @@ def update_operator_cmake(cmake_file): op_kernel_pattern2 = r'REGISTER_OP_KERNEL\(.*?_grad,.*?\);?' custom_pattern1 = r'REGISTER_OP_KERNEL_WITH_CUSTOM_TYPE\(.*?\);?' - custom_pattern2 = r'REGISTER_OP_KERNEL_WITH_CUSTOM_TYPE\(.*?_grad,.*?\);?' + custom_pattern2 = ( + r'REGISTER_OP_KERNEL_WITH_CUSTOM_TYPE\(.*?_grad,.*?\);?' + ) op_name = os.path.split(op_file)[1] if op_name in spec_ops: @@ -121,18 +133,24 @@ def update_operator_cmake(cmake_file): with open(op_file, 'r', encoding='utf-8') as f: content = ''.join(f.readlines()) - op, op_count = remove_grad_op_and_kernel(content, op_pattern1, - op_pattern2) + op, op_count = remove_grad_op_and_kernel( + content, op_pattern1, op_pattern2 + ) cpu_kernel, cpu_kernel_count = remove_grad_op_and_kernel( - content, cpu_kernel_pattern1, cpu_kernel_pattern2) + content, cpu_kernel_pattern1, cpu_kernel_pattern2 + ) gpu_kernel, gpu_kernel_count = remove_grad_op_and_kernel( - content, gpu_kernel_pattern1, gpu_kernel_pattern2) + content, gpu_kernel_pattern1, gpu_kernel_pattern2 + ) xpu_kernel, xpu_kernel_count = remove_grad_op_and_kernel( - content, xpu_kernel_pattern1, xpu_kernel_pattern2) + content, xpu_kernel_pattern1, xpu_kernel_pattern2 + ) op_kernel, op_kernel_count = remove_grad_op_and_kernel( - content, op_kernel_pattern1, op_kernel_pattern2) + content, op_kernel_pattern1, op_kernel_pattern2 + ) custom_kernel, custom_kernel_count = remove_grad_op_and_kernel( - content, custom_pattern1, custom_pattern2) + content, custom_pattern1, custom_pattern2 + ) register_op_count += op_count register_op_cpu_kernel_count += cpu_kernel_count @@ -155,17 +173,24 @@ def update_operator_cmake(cmake_file): f.write(u'{}'.format(content)) # 2. update operators/CMakeLists.txt - cmake_file = os.path.join(tool_dir, - '../paddle/fluid/operators/CMakeLists.txt') + cmake_file = os.path.join( + tool_dir, '../paddle/fluid/operators/CMakeLists.txt' + ) update_operator_cmake(cmake_file) print('We erase all grad op and kernel for Paddle-Inference lib.') print('%50s%10s' % ('type', 'count')) print('%50s%10s' % ('REGISTER_OPERATOR', register_op_count)) print('%50s%10s' % ('REGISTER_OP_CPU_KERNEL', register_op_cpu_kernel_count)) - print('%50s%10s' % - ('REGISTER_OP_CUDA_KERNEL', register_op_cuda_kernel_count)) + print( + '%50s%10s' % ('REGISTER_OP_CUDA_KERNEL', register_op_cuda_kernel_count) + ) print('%50s%10s' % ('REGISTER_OP_XPU_KERNEL', register_op_xpu_kernel_count)) print('%50s%10s' % ('REGISTER_OP_KERNEL', register_op_kernel_count)) - print('%50s%10s' % ('REGISTER_OP_KERNEL_WITH_CUSTOM_TYPE', - register_op_kernel_with_custom_type_count)) + print( + '%50s%10s' + % ( + 'REGISTER_OP_KERNEL_WITH_CUSTOM_TYPE', + register_op_kernel_with_custom_type_count, + ) + ) diff --git a/tools/sampcd_processor.py b/tools/sampcd_processor.py index 7e624652ad508..24af09893e355 100644 --- a/tools/sampcd_processor.py +++ b/tools/sampcd_processor.py @@ -36,7 +36,8 @@ logger = logging.getLogger() if logger.handlers: console = logger.handlers[ - 0] # we assume the first handler is the one we want to configure + 0 + ] # we assume the first handler is the one we want to configure else: console = logging.StreamHandler(stream=sys.stderr) logger.addHandler(console) @@ -76,7 +77,7 @@ def find_all(srcstr, substr): """ indices = [] gotone = srcstr.find(substr) - while (gotone != -1): + while gotone != -1: indices.append(gotone) gotone = srcstr.find(substr, gotone + 1) return indices @@ -124,7 +125,7 @@ def extract_code_blocks_from_docstr(docstr): mo = re.search(r"Examples:", docstr) if mo is None: return code_blocks - ds_list = docstr[mo.start():].replace("\t", ' ').split("\n") + ds_list = docstr[mo.start() :].replace("\t", ' ').split("\n") lastlineindex = len(ds_list) - 1 cb_start_pat = re.compile(r"code-block::\s*python") @@ -148,16 +149,14 @@ def _cb_started(): def _append_code_block(): # nonlocal code_blocks, cb_cur, cb_cur_name, cb_cur_seq_id, cb_required - code_blocks.append({ - 'codes': - inspect.cleandoc("\n".join(cb_info['cb_cur'])), - 'name': - cb_info['cb_cur_name'], - 'id': - cb_info['cb_cur_seq_id'], - 'required': - cb_info['cb_required'], - }) + code_blocks.append( + { + 'codes': inspect.cleandoc("\n".join(cb_info['cb_cur'])), + 'name': cb_info['cb_cur_name'], + 'id': cb_info['cb_cur_seq_id'], + 'required': cb_info['cb_required'], + } + ) for lineno, linecont in enumerate(ds_list): if re.search(cb_start_pat, linecont): @@ -186,7 +185,9 @@ def _append_code_block(): # docstring end if lineno == lastlineindex: mo = re.search(r"\S", linecont) - if mo is not None and cb_info['cb_cur_indent'] <= mo.start( + if ( + mo is not None + and cb_info['cb_cur_indent'] <= mo.start() ): cb_info['cb_cur'].append(linecont) if len(cb_info['cb_cur']): @@ -262,14 +263,21 @@ def is_required_match(requirestr, cbtitle='not-specified'): logger.info('%s: skipped', cbtitle) return None - if all([ - k in SAMPLE_CODE_TEST_CAPACITY for k in requires + if all( + [ + k in SAMPLE_CODE_TEST_CAPACITY + for k in requires if k not in ['skip', 'skiptest'] - ]): + ] + ): return True - logger.info('%s: the equipments [%s] not match the required [%s].', cbtitle, - ','.join(SAMPLE_CODE_TEST_CAPACITY), ','.join(requires)) + logger.info( + '%s: the equipments [%s] not match the required [%s].', + cbtitle, + ','.join(SAMPLE_CODE_TEST_CAPACITY), + ','.join(requires), + ) return False @@ -280,13 +288,18 @@ def insert_codes_into_codeblock(codeblock, apiname='not-specified'): global ENV_KEY_CODES_FRONTEND, GPU_ID, RUN_ON_DEVICE # readonly inserted_codes_f = '' inserted_codes_b = '' - if ENV_KEY_CODES_FRONTEND in os.environ and os.environ[ - ENV_KEY_CODES_FRONTEND]: + if ( + ENV_KEY_CODES_FRONTEND in os.environ + and os.environ[ENV_KEY_CODES_FRONTEND] + ): inserted_codes_f = os.environ[ENV_KEY_CODES_FRONTEND] else: cpu_str = '\nimport os\nos.environ["CUDA_VISIBLE_DEVICES"] = ""\n' - gpu_str = '\nimport os\nos.environ["CUDA_VISIBLE_DEVICES"] = "{}"\n'.format( - GPU_ID) + gpu_str = ( + '\nimport os\nos.environ["CUDA_VISIBLE_DEVICES"] = "{}"\n'.format( + GPU_ID + ) + ) if 'required' in codeblock and codeblock['required']: if codeblock['required'] == 'cpu': inserted_codes_f = cpu_str @@ -298,13 +311,18 @@ def insert_codes_into_codeblock(codeblock, apiname='not-specified'): elif RUN_ON_DEVICE == "gpu": inserted_codes_f = gpu_str inserted_codes_b = '\nprint("{}\'s sample code (name:{}, id:{}) is executed successfully!")'.format( - apiname, codeblock['name'], codeblock['id']) + apiname, codeblock['name'], codeblock['id'] + ) cb = codeblock['codes'] last_future_line_end = find_last_future_line_end(cb) if last_future_line_end: - return cb[:last_future_line_end] + inserted_codes_f + cb[ - last_future_line_end:] + inserted_codes_b + return ( + cb[:last_future_line_end] + + inserted_codes_f + + cb[last_future_line_end:] + + inserted_codes_b + ) else: return inserted_codes_f + cb + inserted_codes_b @@ -335,12 +353,14 @@ def sampcd_extract_to_file(srccom, name, htype="def", hname=""): if srccom.find("Examples:") != -1: logger.info("----example code check----") if srccom.find(">>>") != -1: - logger.warning(r"""Deprecated sample code style: + logger.warning( + r"""Deprecated sample code style: Examples: >>>codeline >>>codeline -Please use '.. code-block:: python' to format the sample code.""") +Please use '.. code-block:: python' to format the sample code.""" + ) return [] else: logger.error( @@ -358,22 +378,33 @@ def sampcd_extract_to_file(srccom, name, htype="def", hname=""): # so, the following conditional statements are intentionally arranged. if matched == True: tfname = os.path.join( - SAMPLECODE_TEMPDIR, '{}_example{}'.format( + SAMPLECODE_TEMPDIR, + '{}_example{}'.format( name, - '.py' if len(codeblocks) == 1 else '_{}.py'.format(y + 1))) + '.py' if len(codeblocks) == 1 else '_{}.py'.format(y + 1), + ), + ) with open(tfname, 'w') as tempf: sampcd = insert_codes_into_codeblock(cb, name) tempf.write(sampcd) sample_code_filenames.append(tfname) elif matched is None: - logger.info('{}\' code block (name:{}, id:{}) is skipped.'.format( - name, cb['name'], cb['id'])) + logger.info( + '{}\' code block (name:{}, id:{}) is skipped.'.format( + name, cb['name'], cb['id'] + ) + ) SUMMARY_INFO['skiptest'].append("{}-{}".format(name, cb['id'])) elif matched == False: logger.info( - '{}\' code block (name:{}, id:{}) required({}) not match capacity({}).' - .format(name, cb['name'], cb['id'], cb['required'], - SAMPLE_CODE_TEST_CAPACITY)) + '{}\' code block (name:{}, id:{}) required({}) not match capacity({}).'.format( + name, + cb['name'], + cb['id'], + cb['required'], + SAMPLE_CODE_TEST_CAPACITY, + ) + ) if cb['required'] not in SUMMARY_INFO: SUMMARY_INFO[cb['required']] = [] SUMMARY_INFO[cb['required']].append("{}-{}".format(name, cb['id'])) @@ -406,9 +437,9 @@ def execute_samplecode(tfname): logger.info("----example code check----") logger.info("executing sample code: %s", tfname) start_time = time.time() - subprc = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) output, error = subprc.communicate() msg = "".join(output.decode(encoding='utf-8')) err = "".join(error.decode(encoding='utf-8')) @@ -425,7 +456,13 @@ def execute_samplecode(tfname): Error Raised from Sample Code: stderr: %s stdout: %s -""", tfname, f.read(), subprc.returncode, err, msg) +""", + tfname, + f.read(), + subprc.returncode, + err, + msg, + ) logger.info("----example code check failed----") result = False else: @@ -450,6 +487,7 @@ def get_filenames(full_test=False): global whl_error import paddle # noqa: F401 import paddle.fluid.contrib.slim.quantization # noqa: F401 + whl_error = [] if full_test: get_full_api_from_pr_spec() @@ -470,7 +508,8 @@ def get_filenames(full_test=False): continue if hasattr(api_obj, '__doc__') and api_obj.__doc__: sample_code_filenames = sampcd_extract_to_file( - api_obj.__doc__, api) + api_obj.__doc__, api + ) for tfname in sample_code_filenames: all_sample_code_filenames[tfname] = api return all_sample_code_filenames @@ -492,7 +531,8 @@ def get_api_md5(path): return api_md5 pat = re.compile(r'\((paddle[^,]+)\W*document\W*([0-9a-z]{32})') patArgSpec = re.compile( - r'^(paddle[^,]+)\s+\(ArgSpec.*document\W*([0-9a-z]{32})') + r'^(paddle[^,]+)\s+\(ArgSpec.*document\W*([0-9a-z]{32})' + ) with open(API_spec) as f: for line in f.readlines(): mo = pat.search(line) @@ -509,6 +549,7 @@ def get_full_api(): """ global API_DIFF_SPEC_FN ## readonly from print_signatures import get_all_api_from_modulelist + member_dict = get_all_api_from_modulelist() with open(API_DIFF_SPEC_FN, 'w') as f: f.write("\n".join(member_dict.keys())) @@ -520,6 +561,7 @@ def get_full_api_by_walk(): """ global API_DIFF_SPEC_FN ## readonly from print_signatures import get_all_api + apilist = get_all_api() with open(API_DIFF_SPEC_FN, 'w') as f: f.write("\n".join([ai[0] for ai in apilist])) @@ -549,8 +591,12 @@ def get_incrementapi(): for key in pr_api: if key in dev_api: if dev_api[key] != pr_api[key]: - logger.debug("%s in dev is %s, different from pr's %s", key, - dev_api[key], pr_api[key]) + logger.debug( + "%s in dev is %s, different from pr's %s", + key, + dev_api[key], + pr_api[key], + ) f.write(key) f.write('\n') else: @@ -564,9 +610,9 @@ def exec_gen_doc(): cmd = ["bash", "document_preview.sh"] logger.info("----exec gen_doc----") start_time = time.time() - subprc = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) output, error = subprc.communicate() msg = "".join(output.decode(encoding='utf-8')) err = "".join(error.decode(encoding='utf-8')) @@ -585,7 +631,8 @@ def exec_gen_doc(): logger.info("----exec gen_doc success----") for fn in [ - '/docs/en/develop/index_en.html', '/docs/zh/develop/index_cn.html' + '/docs/en/develop/index_en.html', + '/docs/zh/develop/index_cn.html', ]: if os.path.exists(fn): logger.info('%s exists.', fn) @@ -616,16 +663,16 @@ def parse_args(): parser.add_argument('--debug', dest='debug', action="store_true") parser.add_argument('--full-test', dest='full_test', action="store_true") parser.add_argument('mode', type=str, help='run on device', default='cpu') - parser.add_argument('--build-doc', - dest='build_doc', - action='store_true', - help='build doc if need.') + parser.add_argument( + '--build-doc', + dest='build_doc', + action='store_true', + help='build doc if need.', + ) for item in arguments: - parser.add_argument(item[0], - dest=item[1], - help=item[4], - type=item[2], - default=item[3]) + parser.add_argument( + item[0], dest=item[1], help=item[4], type=item[2], default=item[3] + ) if len(sys.argv) == 1: args = parser.parse_args(['cpu']) @@ -648,21 +695,24 @@ def parse_args(): logfHandler.setFormatter( logging.Formatter( "%(asctime)s - %(funcName)s:%(lineno)d - %(levelname)s - %(message)s" - )) + ) + ) logger.addHandler(logfHandler) if args.mode == "gpu": GPU_ID = args.gpu_id logger.info("using GPU_ID %d", GPU_ID) elif args.mode != "cpu": - logger.error("Unrecognized argument:%s, 'cpu' or 'gpu' is desired.", - args.mode) + logger.error( + "Unrecognized argument:%s, 'cpu' or 'gpu' is desired.", args.mode + ) sys.exit("Invalid arguments") RUN_ON_DEVICE = args.mode get_test_capacity() logger.info("API check -- Example Code") - logger.info("sample_test running under python %s", - platform.python_version()) + logger.info( + "sample_test running under python %s", platform.python_version() + ) if os.path.exists(SAMPLECODE_TEMPDIR): if not os.path.isdir(SAMPLECODE_TEMPDIR): @@ -706,16 +756,18 @@ def parse_args(): ) for temp in result: if not temp[0]: - logger.info("In addition, mistakes found in sample codes: %s", - temp[1]) + logger.info( + "In addition, mistakes found in sample codes: %s", temp[1] + ) logger.info("----------------------------------------------------") exit(1) else: timeovered_test = {} for temp in result: if not temp[0]: - logger.info("In addition, mistakes found in sample codes: %s", - temp[1]) + logger.info( + "In addition, mistakes found in sample codes: %s", temp[1] + ) SUMMARY_INFO['failed'].append(temp[1]) else: SUMMARY_INFO['success'].append(temp[1]) @@ -723,31 +775,37 @@ def parse_args(): timeovered_test[temp[1]] = temp[3] if len(timeovered_test): - logger.info("%d sample codes ran time over 10s", - len(timeovered_test)) + logger.info( + "%d sample codes ran time over 10s", len(timeovered_test) + ) if args.debug: for k, v in timeovered_test.items(): logger.info('{} - {}s'.format(k, v)) if len(SUMMARY_INFO['success']): - logger.info("%d sample codes ran success", - len(SUMMARY_INFO['success'])) + logger.info( + "%d sample codes ran success", len(SUMMARY_INFO['success']) + ) for k, v in SUMMARY_INFO.items(): if k not in ['success', 'failed', 'skiptest', 'nocodes']: - logger.info("%d sample codes required not match for %s", len(v), - k) + logger.info( + "%d sample codes required not match for %s", len(v), k + ) if len(SUMMARY_INFO['skiptest']): - logger.info("%d sample codes skipped", - len(SUMMARY_INFO['skiptest'])) + logger.info( + "%d sample codes skipped", len(SUMMARY_INFO['skiptest']) + ) if args.debug: logger.info('\n'.join(SUMMARY_INFO['skiptest'])) if len(SUMMARY_INFO['nocodes']): - logger.info("%d apis don't have sample codes", - len(SUMMARY_INFO['nocodes'])) + logger.info( + "%d apis don't have sample codes", len(SUMMARY_INFO['nocodes']) + ) if args.debug: logger.info('\n'.join(SUMMARY_INFO['nocodes'])) if len(SUMMARY_INFO['failed']): - logger.info("%d sample codes ran failed", - len(SUMMARY_INFO['failed'])) + logger.info( + "%d sample codes ran failed", len(SUMMARY_INFO['failed']) + ) logger.info('\n'.join(SUMMARY_INFO['failed'])) logger.info( "Mistakes found in sample codes. Please recheck the sample codes." diff --git a/tools/summary_env.py b/tools/summary_env.py index 2abee2821c073..14fb512ca22b0 100644 --- a/tools/summary_env.py +++ b/tools/summary_env.py @@ -34,6 +34,7 @@ def get_paddle_info(): try: import paddle + envs['paddle_version'] = paddle.__version__ envs['paddle_with_cuda'] = paddle.fluid.core.is_compiled_with_cuda() except: @@ -63,10 +64,9 @@ def get_python_info(): def run_shell_command(cmd): - out, err = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True).communicate() + out, err = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True + ).communicate() if err: return None else: @@ -82,7 +82,6 @@ def get_cuda_info(): def get_cudnn_info(): - def _get_cudnn_ver(cmd): out = run_shell_command(cmd) if out: @@ -93,15 +92,17 @@ def _get_cudnn_ver(cmd): if platform.system() == "Windows": cudnn_dll_path = run_shell_command('where cudnn*') if cudnn_dll_path: - cudnn_header_path = cudnn_dll_path.split( - 'bin')[0] + r'include\cudnn.h' + cudnn_header_path = ( + cudnn_dll_path.split('bin')[0] + r'include\cudnn.h' + ) cmd = 'type "{0}" | findstr "{1}" | findstr /v "CUDNN_VERSION"' else: envs['cudnn_version'] = None return else: cudnn_header_path = run_shell_command( - 'whereis "cudnn.h" | awk \'{print $2}\'') + 'whereis "cudnn.h" | awk \'{print $2}\'' + ) if cudnn_header_path: cudnn_header_path = cudnn_header_path.strip() cmd = 'cat "{0}" | grep "{1}" | grep -v "CUDNN_VERSION"' @@ -112,7 +113,8 @@ def _get_cudnn_ver(cmd): major = _get_cudnn_ver(cmd.format(cudnn_header_path, 'CUDNN_MAJOR')) minor = _get_cudnn_ver(cmd.format(cudnn_header_path, 'CUDNN_MINOR')) patch_level = _get_cudnn_ver( - cmd.format(cudnn_header_path, 'CUDNN_PATCHLEVEL')) + cmd.format(cudnn_header_path, 'CUDNN_PATCHLEVEL') + ) envs['cudnn_version'] = "{0}.{1}.{2}".format(major, minor, patch_level) @@ -120,8 +122,9 @@ def _get_cudnn_ver(cmd): def get_driver_info(): driver_ver = run_shell_command('nvidia-smi') if driver_ver: - driver_ver = driver_ver.split('Driver Version:')[1].strip().split( - ' ')[0] + driver_ver = ( + driver_ver.split('Driver Version:')[1].strip().split(' ')[0] + ) else: driver_ver = None envs['nvidia_driver_version'] = driver_ver diff --git a/tools/test_check_api_compatible.py b/tools/test_check_api_compatible.py index 1bed3951740b2..d7c9a0b195d36 100644 --- a/tools/test_check_api_compatible.py +++ b/tools/test_check_api_compatible.py @@ -25,10 +25,12 @@ class Test_check_compatible(unittest.TestCase): - def setUp(self) -> None: self.fullargspec_prefix = 'inspect.Full' - self.argspec_str_o = self.fullargspec_prefix + '''ArgSpec(args=['shape', 'dtype', 'name'], varargs=None, varkw=None, defaults=(None, None), kwonlyargs=[], kwonlydefaults=None, annotations={})''' + self.argspec_str_o = ( + self.fullargspec_prefix + + '''ArgSpec(args=['shape', 'dtype', 'name'], varargs=None, varkw=None, defaults=(None, None), kwonlyargs=[], kwonlydefaults=None, annotations={})''' + ) return super().setUp() def test_normal_not_changed(self): @@ -68,11 +70,13 @@ def test_args_reduced(self): class Test_check_compatible_str(unittest.TestCase): - def setUp(self) -> None: self.fullargspec_prefix = 'inspect.Full' # paddle.fluid.layer_helper_base.LayerHelperBase.create_parameter - self.argspec_str_o = self.fullargspec_prefix + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + self.argspec_str_o = ( + self.fullargspec_prefix + + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + ) return super().setUp() def test_normal_not_changed(self): @@ -81,26 +85,44 @@ def test_normal_not_changed(self): self.assertTrue(check_compatible_str(argspec_o, argspec_n)) def test_args_added(self): - argspec_n = self.fullargspec_prefix + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + argspec_n = ( + self.fullargspec_prefix + + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + ) argspec_o = self.argspec_str_o self.assertFalse(check_compatible_str(argspec_o, argspec_n)) - argspec_n = self.fullargspec_prefix + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR, argadded), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + argspec_n = ( + self.fullargspec_prefix + + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR, argadded), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + ) self.assertTrue(check_compatible_str(argspec_o, argspec_n)) - argspec_n = self.fullargspec_prefix + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR, argadded, 1), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + argspec_n = ( + self.fullargspec_prefix + + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR, argadded, 1), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + ) self.assertFalse(check_compatible_str(argspec_o, argspec_n)) - argspec_n = self.fullargspec_prefix + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(1, None, False, None, False, VarType.LOD_TENSOR, argadded), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + argspec_n = ( + self.fullargspec_prefix + + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient', 'type', 'argadded'], varargs=None, varkw=None, defaults=(1, None, False, None, False, VarType.LOD_TENSOR, argadded), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + ) self.assertTrue(check_compatible_str(argspec_o, argspec_n)) def test_args_places_exchanged(self): - argspec_n = self.fullargspec_prefix + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'type', 'stop_gradient'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + argspec_n = ( + self.fullargspec_prefix + + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'type', 'stop_gradient'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + ) argspec_o = self.argspec_str_o self.assertFalse(check_compatible_str(argspec_o, argspec_n)) def test_args_reduced(self): - argspec_n = self.fullargspec_prefix + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + argspec_n = ( + self.fullargspec_prefix + + """ArgSpec(args=['self', 'attr', 'shape', 'dtype', 'is_bias', 'default_initializer', 'stop_gradient'], varargs=None, varkw=None, defaults=(None, False, None, False, VarType.LOD_TENSOR), kwonlyargs=[], kwonlydefaults=None, annotations={})""" + ) argspec_o = self.argspec_str_o self.assertFalse(check_compatible_str(argspec_o, argspec_n)) @@ -111,17 +133,23 @@ def test_args_defaults_None(self): class Test_read_argspec_from_file(unittest.TestCase): - def setUp(self) -> None: self.fullargspec_prefix = 'inspect.Full' - self.argspec_str_o = self.fullargspec_prefix + '''ArgSpec(args=['shape', 'dtype', 'name'], varargs=None, varkw=None, defaults=(None, None), kwonlyargs=[], kwonlydefaults=None, annotations={})''' + self.argspec_str_o = ( + self.fullargspec_prefix + + '''ArgSpec(args=['shape', 'dtype', 'name'], varargs=None, varkw=None, defaults=(None, None), kwonlyargs=[], kwonlydefaults=None, annotations={})''' + ) self.api_spec_file = tempfile.TemporaryFile('w+t') if self.api_spec_file: - self.api_spec_file.write("\n".join([ - """paddle.ones (ArgSpec(args=['shape', 'dtype', 'name'], varargs=None, varkw=None, defaults=(None, None), kwonlyargs=[], kwonlydefaults=None, annotations={}), ('document', '50a3b3a77fa13bb2ae4337d8f9d091b7'))""", - # """paddle.four_plus_four (paddle.four_plus_four, ('document', 'ff0f188c95030158cc6398d2a6c5four'))""", - """paddle.five_plus_five (ArgSpec(), ('document', 'ff0f188c95030158cc6398d2a6c5five'))""", - ])) + self.api_spec_file.write( + "\n".join( + [ + """paddle.ones (ArgSpec(args=['shape', 'dtype', 'name'], varargs=None, varkw=None, defaults=(None, None), kwonlyargs=[], kwonlydefaults=None, annotations={}), ('document', '50a3b3a77fa13bb2ae4337d8f9d091b7'))""", + # """paddle.four_plus_four (paddle.four_plus_four, ('document', 'ff0f188c95030158cc6398d2a6c5four'))""", + """paddle.five_plus_five (ArgSpec(), ('document', 'ff0f188c95030158cc6398d2a6c5five'))""", + ] + ) + ) self.api_spec_file.seek(0) return super().setUp() @@ -134,9 +162,11 @@ def test_case_normal(self): api_argspec_dict = read_argspec_from_file(self.api_spec_file) argspec = eval(self.argspec_str_o) self.assertEqual( - api_argspec_dict.get('paddle.ones').args, argspec.args) + api_argspec_dict.get('paddle.ones').args, argspec.args + ) self.assertEqual( - api_argspec_dict.get('paddle.ones').defaults, argspec.defaults) + api_argspec_dict.get('paddle.ones').defaults, argspec.defaults + ) self.assertIsNone(api_argspec_dict.get('paddle.five_plus_five')) else: self.fail('api_spec_file error') diff --git a/tools/test_check_pr_approval.py b/tools/test_check_pr_approval.py index 5f3c7ca11ccc0..c126a749f1be4 100644 --- a/tools/test_check_pr_approval.py +++ b/tools/test_check_pr_approval.py @@ -22,7 +22,6 @@ class Test_check_approval(unittest.TestCase): - def setUp(self): self.codeset = 'UTF-8' # only key info in it @@ -68,47 +67,64 @@ def setUp(self): "author_association": "CONTRIBUTOR" } ] -""".encode(self.codeset) +""".encode( + self.codeset + ) def test_ids(self): cmd = [sys.executable, 'check_pr_approval.py', '1', '26408901'] - subprc = subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) output, error = subprc.communicate(input=self.jsonstr) self.assertEqual('TRUE', output.decode(self.codeset).rstrip()) def test_logins(self): cmd = [sys.executable, 'check_pr_approval.py', '1', 'pangyoki'] - subprc = subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) output, error = subprc.communicate(input=self.jsonstr) self.assertEqual('TRUE', output.decode(self.codeset).rstrip()) def test_ids_and_logins(self): cmd = [ - sys.executable, 'check_pr_approval.py', '2', 'pangyoki', '13469016' + sys.executable, + 'check_pr_approval.py', + '2', + 'pangyoki', + '13469016', ] - subprc = subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) output, error = subprc.communicate(input=self.jsonstr) - #self.assertEqual('', error.rstrip()) + # self.assertEqual('', error.rstrip()) self.assertEqual('TRUE', output.decode(self.codeset).rstrip()) def test_check_with_required_reviewer_not_approved(self): cmd = [ - sys.executable, 'check_pr_approval.py', '2', 'wadefelix', - ' 13469016' + sys.executable, + 'check_pr_approval.py', + '2', + 'wadefelix', + ' 13469016', ] - subprc = subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subprc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) output, error = subprc.communicate(input=self.jsonstr) self.assertEqual('FALSE', output.decode(self.codeset).rstrip()) diff --git a/tools/test_print_signatures.py b/tools/test_print_signatures.py index 14275b6b7ae4d..c757fcce967f8 100644 --- a/tools/test_print_signatures.py +++ b/tools/test_print_signatures.py @@ -42,7 +42,7 @@ def func_example_2(func=functools.partial(func_example, 1)): pass -class ClassExample(): +class ClassExample: """ example Class """ @@ -55,7 +55,6 @@ class method class Test_all_in_print_signatures(unittest.TestCase): - def test_md5(self): algo = hashlib.md5() algo.update(func_example.__doc__.encode('utf-8')) @@ -64,13 +63,13 @@ def test_md5(self): class Test_is_primitive(unittest.TestCase): - def test_single(self): self.assertTrue(is_primitive(2)) self.assertTrue(is_primitive(2.1)) self.assertTrue(is_primitive("2.1.1")) - self.assertFalse(is_primitive( - "hello paddle".encode('UTF-8'))) # True for python2 + self.assertFalse( + is_primitive("hello paddle".encode('UTF-8')) + ) # True for python2 self.assertFalse(is_primitive(1j)) self.assertTrue(is_primitive(True)) diff --git a/tools/test_runner.py b/tools/test_runner.py index bb1e623ce04c3..65da72b539df7 100644 --- a/tools/test_runner.py +++ b/tools/test_runner.py @@ -28,7 +28,7 @@ def main(): sys.path.append(os.getcwd()) if core.is_compiled_with_cuda() or core.is_compiled_with_rocm(): - if (os.getenv('FLAGS_enable_gpu_memory_usage_log') == None): + if os.getenv('FLAGS_enable_gpu_memory_usage_log') == None: os.environ['FLAGS_enable_gpu_memory_usage_log'] = 'true' os.environ['FLAGS_enable_gpu_memory_usage_log_mb'] = 'false' @@ -52,10 +52,12 @@ def main(): if not res.wasSuccessful(): some_test_failed = True - print(module_name, - 'failed\n', - buffer.getvalue(), - file=sys.stderr) + print( + module_name, + 'failed\n', + buffer.getvalue(), + file=sys.stderr, + ) if flag_need_static_mode: paddle.disable_static() diff --git a/tools/test_sampcd_processor.py b/tools/test_sampcd_processor.py index 42f5d96e71c0d..d389b2dbc870b 100644 --- a/tools/test_sampcd_processor.py +++ b/tools/test_sampcd_processor.py @@ -32,7 +32,6 @@ class Test_find_all(unittest.TestCase): - def test_find_none(self): self.assertEqual(0, len(find_all('hello', 'world'))) @@ -40,12 +39,12 @@ def test_find_one(self): self.assertListEqual([0], find_all('hello', 'hello')) def test_find_two(self): - self.assertListEqual([1, 15], - find_all(' hello, world; hello paddle!', 'hello')) + self.assertListEqual( + [1, 15], find_all(' hello, world; hello paddle!', 'hello') + ) class Test_find_last_future_line_end(unittest.TestCase): - def test_no_instant(self): samplecodes = """ print(10//3) @@ -60,8 +59,9 @@ def test_1_instant(self): """ mo = re.search("print_function\n", samplecodes) self.assertIsNotNone(mo) - self.assertGreaterEqual(find_last_future_line_end(samplecodes), - mo.end()) + self.assertGreaterEqual( + find_last_future_line_end(samplecodes), mo.end() + ) def test_2_instant(self): samplecodes = """ @@ -72,12 +72,12 @@ def test_2_instant(self): """ mo = re.search("division\n", samplecodes) self.assertIsNotNone(mo) - self.assertGreaterEqual(find_last_future_line_end(samplecodes), - mo.end()) + self.assertGreaterEqual( + find_last_future_line_end(samplecodes), mo.end() + ) class Test_extract_code_blocks_from_docstr(unittest.TestCase): - def test_no_samplecode(self): docstr = """ placeholder @@ -103,12 +103,17 @@ def test_1_samplecode(self): print(1+1) """ codeblocks = extract_code_blocks_from_docstr(docstr) - self.assertListEqual(codeblocks, [{ - 'codes': """print(1+1)""", - 'name': None, - 'id': 1, - 'required': None, - }]) + self.assertListEqual( + codeblocks, + [ + { + 'codes': """print(1+1)""", + 'name': None, + 'id': 1, + 'required': None, + } + ], + ) def test_2_samplecodes(self): docstr = """ @@ -126,22 +131,27 @@ def test_2_samplecodes(self): print(1+1) """ codeblocks = extract_code_blocks_from_docstr(docstr) - self.assertListEqual(codeblocks, [{ - 'codes': """print(1/0)""", - 'name': None, - 'id': 1, - 'required': None, - }, { - 'codes': """# required: gpu + self.assertListEqual( + codeblocks, + [ + { + 'codes': """print(1/0)""", + 'name': None, + 'id': 1, + 'required': None, + }, + { + 'codes': """# required: gpu print(1+1)""", - 'name': 'one_plus_one', - 'id': 2, - 'required': 'gpu', - }]) + 'name': 'one_plus_one', + 'id': 2, + 'required': 'gpu', + }, + ], + ) class Test_insert_codes_into_codeblock(unittest.TestCase): - def test_required_None(self): codeblock = { 'codes': """print(1/0)""", @@ -155,7 +165,8 @@ def test_required_None(self): os.environ["CUDA_VISIBLE_DEVICES"] = "" print(1/0) print("not-specified's sample code (name:None, id:1) is executed successfully!")""", - insert_codes_into_codeblock(codeblock)) + insert_codes_into_codeblock(codeblock), + ) def test_required_gpu(self): codeblock = { @@ -172,7 +183,8 @@ def test_required_gpu(self): # required: gpu print(1+1) print("not-specified's sample code (name:None, id:1) is executed successfully!")""", - insert_codes_into_codeblock(codeblock)) + insert_codes_into_codeblock(codeblock), + ) def test_from_future(self): codeblock = { @@ -193,7 +205,8 @@ def test_from_future(self): os.environ["CUDA_VISIBLE_DEVICES"] = "" print(10//3) print("not-specified's sample code (name:future, id:1) is executed successfully!")""", - insert_codes_into_codeblock(codeblock)) + insert_codes_into_codeblock(codeblock), + ) def clear_capacity(): @@ -204,7 +217,6 @@ def clear_capacity(): class Test_get_test_capacity(unittest.TestCase): - def setUp(self): clear_capacity() get_test_capacity() @@ -216,34 +228,40 @@ def tearDown(self): def test_NoEnvVar(self): clear_capacity() get_test_capacity() - self.assertCountEqual([ - 'cpu', - ], sampcd_processor.SAMPLE_CODE_TEST_CAPACITY) + self.assertCountEqual( + [ + 'cpu', + ], + sampcd_processor.SAMPLE_CODE_TEST_CAPACITY, + ) def test_NoEnvVar_RUN_ON_DEVICE_gpu(self): clear_capacity() sampcd_processor.RUN_ON_DEVICE = 'gpu' get_test_capacity() - self.assertCountEqual(['cpu', 'gpu'], - sampcd_processor.SAMPLE_CODE_TEST_CAPACITY) + self.assertCountEqual( + ['cpu', 'gpu'], sampcd_processor.SAMPLE_CODE_TEST_CAPACITY + ) def test_EnvVar_gpu(self): clear_capacity() os.environ[sampcd_processor.ENV_KEY_TEST_CAPACITY] = 'gpu' get_test_capacity() - self.assertCountEqual(['cpu', 'gpu'], - sampcd_processor.SAMPLE_CODE_TEST_CAPACITY) + self.assertCountEqual( + ['cpu', 'gpu'], sampcd_processor.SAMPLE_CODE_TEST_CAPACITY + ) def test_EnvVar_gpu_and_distributed(self): clear_capacity() os.environ[sampcd_processor.ENV_KEY_TEST_CAPACITY] = 'gpu,distributed' get_test_capacity() - self.assertCountEqual(['cpu', 'gpu', 'distributed'], - sampcd_processor.SAMPLE_CODE_TEST_CAPACITY) + self.assertCountEqual( + ['cpu', 'gpu', 'distributed'], + sampcd_processor.SAMPLE_CODE_TEST_CAPACITY, + ) class Test_is_required_match(unittest.TestCase): - def setUp(self): clear_capacity() @@ -284,16 +302,17 @@ def test_gpu_distributed_equipped(self): class Test_execute_samplecode(unittest.TestCase): - def setUp(self): if not os.path.exists(sampcd_processor.SAMPLECODE_TEMPDIR): os.mkdir(sampcd_processor.SAMPLECODE_TEMPDIR) self.successSampleCodeFile = os.path.join( - sampcd_processor.SAMPLECODE_TEMPDIR, 'samplecode_success.py') + sampcd_processor.SAMPLECODE_TEMPDIR, 'samplecode_success.py' + ) with open(self.successSampleCodeFile, 'w') as f: f.write('print(1+1)') self.failedSampleCodeFile = os.path.join( - sampcd_processor.SAMPLECODE_TEMPDIR, 'samplecode_failed.py') + sampcd_processor.SAMPLECODE_TEMPDIR, 'samplecode_failed.py' + ) with open(self.failedSampleCodeFile, 'w') as f: f.write('print(1/0)') @@ -303,7 +322,8 @@ def tearDown(self): def test_run_success(self): result, tfname, msg, exec_time = execute_samplecode( - self.successSampleCodeFile) + self.successSampleCodeFile + ) self.assertTrue(result) self.assertEqual(self.successSampleCodeFile, tfname) self.assertIsNotNone(msg) @@ -312,7 +332,8 @@ def test_run_success(self): def test_run_failed(self): result, tfname, msg, exec_time = execute_samplecode( - self.failedSampleCodeFile) + self.failedSampleCodeFile + ) self.assertFalse(result) self.assertEqual(self.failedSampleCodeFile, tfname) self.assertIsNotNone(msg) @@ -326,7 +347,6 @@ def clear_summary_info(): class Test_sampcd_extract_to_file(unittest.TestCase): - def setUp(self): if not os.path.exists(sampcd_processor.SAMPLECODE_TEMPDIR): os.mkdir(sampcd_processor.SAMPLECODE_TEMPDIR) @@ -348,10 +368,15 @@ def test_1_samplecode(self): """ funcname = 'one_plus_one' sample_code_filenames = sampcd_extract_to_file(comments, funcname) - self.assertCountEqual([ - os.path.join(sampcd_processor.SAMPLECODE_TEMPDIR, - funcname + '_example.py') - ], sample_code_filenames) + self.assertCountEqual( + [ + os.path.join( + sampcd_processor.SAMPLECODE_TEMPDIR, + funcname + '_example.py', + ) + ], + sample_code_filenames, + ) def test_no_samplecode(self): comments = """ @@ -375,12 +400,19 @@ def test_2_samplecodes(self): """ funcname = 'one_plus_one' sample_code_filenames = sampcd_extract_to_file(comments, funcname) - self.assertCountEqual([ - os.path.join(sampcd_processor.SAMPLECODE_TEMPDIR, - funcname + '_example_1.py'), - os.path.join(sampcd_processor.SAMPLECODE_TEMPDIR, - funcname + '_example_2.py') - ], sample_code_filenames) + self.assertCountEqual( + [ + os.path.join( + sampcd_processor.SAMPLECODE_TEMPDIR, + funcname + '_example_1.py', + ), + os.path.join( + sampcd_processor.SAMPLECODE_TEMPDIR, + funcname + '_example_2.py', + ), + ], + sample_code_filenames, + ) def test_2_samplecodes_has_skipped(self): comments = """ @@ -421,71 +453,100 @@ def test_2_samplecodes_has_skipped(self): get_test_capacity() sample_code_filenames = sampcd_extract_to_file(comments, funcname) - self.assertCountEqual([ - os.path.join(sampcd_processor.SAMPLECODE_TEMPDIR, - funcname + '_example_2.py') - ], sample_code_filenames) - self.assertCountEqual(sampcd_processor.SUMMARY_INFO['skiptest'], - [funcname + '-1']) - self.assertCountEqual(sampcd_processor.SUMMARY_INFO['gpu'], - [funcname + '-3', funcname + '-6']) - self.assertCountEqual(sampcd_processor.SUMMARY_INFO['xpu'], - [funcname + '-4']) - self.assertCountEqual(sampcd_processor.SUMMARY_INFO['distributed'], - [funcname + '-5']) + self.assertCountEqual( + [ + os.path.join( + sampcd_processor.SAMPLECODE_TEMPDIR, + funcname + '_example_2.py', + ) + ], + sample_code_filenames, + ) + self.assertCountEqual( + sampcd_processor.SUMMARY_INFO['skiptest'], [funcname + '-1'] + ) + self.assertCountEqual( + sampcd_processor.SUMMARY_INFO['gpu'], + [funcname + '-3', funcname + '-6'], + ) + self.assertCountEqual( + sampcd_processor.SUMMARY_INFO['xpu'], [funcname + '-4'] + ) + self.assertCountEqual( + sampcd_processor.SUMMARY_INFO['distributed'], [funcname + '-5'] + ) class Test_get_api_md5(unittest.TestCase): - def setUp(self): self.api_pr_spec_filename = os.path.abspath( - os.path.join(os.getcwd(), "..", 'paddle/fluid/API_PR.spec')) + os.path.join(os.getcwd(), "..", 'paddle/fluid/API_PR.spec') + ) with open(self.api_pr_spec_filename, 'w') as f: - f.write("\n".join([ - """paddle.one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55one'))""", - """paddle.two_plus_two (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55two'))""", - """paddle.three_plus_three (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6cthree'))""", - """paddle.four_plus_four (paddle.four_plus_four, ('document', 'ff0f188c95030158cc6398d2a6c5four'))""", - """paddle.five_plus_five (ArgSpec(), ('document', 'ff0f188c95030158cc6398d2a6c5five'))""", - ])) + f.write( + "\n".join( + [ + """paddle.one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55one'))""", + """paddle.two_plus_two (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55two'))""", + """paddle.three_plus_three (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6cthree'))""", + """paddle.four_plus_four (paddle.four_plus_four, ('document', 'ff0f188c95030158cc6398d2a6c5four'))""", + """paddle.five_plus_five (ArgSpec(), ('document', 'ff0f188c95030158cc6398d2a6c5five'))""", + ] + ) + ) def tearDown(self): os.remove(self.api_pr_spec_filename) def test_get_api_md5(self): res = get_api_md5('paddle/fluid/API_PR.spec') - self.assertEqual("ff0f188c95030158cc6398d2a6c55one", - res['paddle.one_plus_one']) - self.assertEqual("ff0f188c95030158cc6398d2a6c55two", - res['paddle.two_plus_two']) - self.assertEqual("ff0f188c95030158cc6398d2a6cthree", - res['paddle.three_plus_three']) - self.assertEqual("ff0f188c95030158cc6398d2a6c5four", - res['paddle.four_plus_four']) - self.assertEqual("ff0f188c95030158cc6398d2a6c5five", - res['paddle.five_plus_five']) + self.assertEqual( + "ff0f188c95030158cc6398d2a6c55one", res['paddle.one_plus_one'] + ) + self.assertEqual( + "ff0f188c95030158cc6398d2a6c55two", res['paddle.two_plus_two'] + ) + self.assertEqual( + "ff0f188c95030158cc6398d2a6cthree", res['paddle.three_plus_three'] + ) + self.assertEqual( + "ff0f188c95030158cc6398d2a6c5four", res['paddle.four_plus_four'] + ) + self.assertEqual( + "ff0f188c95030158cc6398d2a6c5five", res['paddle.five_plus_five'] + ) class Test_get_incrementapi(unittest.TestCase): - def setUp(self): self.api_pr_spec_filename = os.path.abspath( - os.path.join(os.getcwd(), "..", 'paddle/fluid/API_PR.spec')) + os.path.join(os.getcwd(), "..", 'paddle/fluid/API_PR.spec') + ) with open(self.api_pr_spec_filename, 'w') as f: - f.write("\n".join([ - """paddle.one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55one'))""", - """paddle.two_plus_two (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55two'))""", - """paddle.three_plus_three (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6cthree'))""", - """paddle.four_plus_four (paddle.four_plus_four, ('document', 'ff0f188c95030158cc6398d2a6c5four'))""", - ])) + f.write( + "\n".join( + [ + """paddle.one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55one'))""", + """paddle.two_plus_two (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55two'))""", + """paddle.three_plus_three (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6cthree'))""", + """paddle.four_plus_four (paddle.four_plus_four, ('document', 'ff0f188c95030158cc6398d2a6c5four'))""", + ] + ) + ) self.api_dev_spec_filename = os.path.abspath( - os.path.join(os.getcwd(), "..", 'paddle/fluid/API_DEV.spec')) + os.path.join(os.getcwd(), "..", 'paddle/fluid/API_DEV.spec') + ) with open(self.api_dev_spec_filename, 'w') as f: - f.write("\n".join([ - """paddle.one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55one'))""", - ])) + f.write( + "\n".join( + [ + """paddle.one_plus_one (ArgSpec(args=[], varargs=None, keywords=None, defaults=(,)), ('document', 'ff0f188c95030158cc6398d2a6c55one'))""", + ] + ) + ) self.api_diff_spec_filename = os.path.abspath( - os.path.join(os.getcwd(), "dev_pr_diff_api.spec")) + os.path.join(os.getcwd(), "dev_pr_diff_api.spec") + ) def tearDown(self): os.remove(self.api_pr_spec_filename) @@ -496,10 +557,14 @@ def test_it(self): get_incrementapi() with open(self.api_diff_spec_filename, 'r') as f: lines = f.readlines() - self.assertCountEqual([ - "paddle.two_plus_two\n", "paddle.three_plus_three\n", - "paddle.four_plus_four\n" - ], lines) + self.assertCountEqual( + [ + "paddle.two_plus_two\n", + "paddle.three_plus_three\n", + "paddle.four_plus_four\n", + ], + lines, + ) # https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/fluid/layers/ops.py diff --git a/tools/timeline.py b/tools/timeline.py index e3de686f82997..5323ea0f907ec 100644 --- a/tools/timeline.py +++ b/tools/timeline.py @@ -23,16 +23,15 @@ type=str, default='', help='Input profile file name. If there are multiple file, the format ' - 'should be trainer1=file1,trainer2=file2,ps=file3') -parser.add_argument('--timeline_path', - type=str, - default='', - help='Output timeline file name.') + 'should be trainer1=file1,trainer2=file2,ps=file3', +) +parser.add_argument( + '--timeline_path', type=str, default='', help='Output timeline file name.' +) args = parser.parse_args() class _ChromeTraceFormatter(object): - def __init__(self): self._events = [] self._metadata = [] @@ -128,7 +127,6 @@ def format_to_string(self, pretty=False): class Timeline(object): - def __init__(self, profile_dict): self._profile_dict = profile_dict self._pid = 0 @@ -153,13 +151,15 @@ def _allocate_pids(self): self._chrome_trace.emit_pid("%s:cuda_api" % k, pid) else: self._chrome_trace.emit_pid( - "%s:cpu:block:%d" % (k, event.device_id), pid) + "%s:cpu:block:%d" % (k, event.device_id), pid + ) elif event.type == profiler_pb2.Event.GPUKernel: if (k, event.device_id, "GPUKernel") not in self._devices: pid = self._allocate_pid() self._devices[(k, event.device_id, "GPUKernel")] = pid self._chrome_trace.emit_pid( - "%s:gpu:%d" % (k, event.device_id), pid) + "%s:gpu:%d" % (k, event.device_id), pid + ) if not hasattr(profile_pb, "mem_events"): continue for mevent in profile_pb.mem_events: @@ -169,50 +169,63 @@ def _allocate_pids(self): self._mem_devices[(k, mevent.device_id, "GPU")] = pid self._chrome_trace.emit_pid( "memory usage on %s:gpu:%d" % (k, mevent.device_id), - pid) + pid, + ) elif mevent.place == profiler_pb2.MemEvent.CPUPlace: if (k, mevent.device_id, "CPU") not in self._mem_devices: pid = self._allocate_pid() self._mem_devices[(k, mevent.device_id, "CPU")] = pid self._chrome_trace.emit_pid( "memory usage on %s:cpu:%d" % (k, mevent.device_id), - pid) + pid, + ) elif mevent.place == profiler_pb2.MemEvent.CUDAPinnedPlace: - if (k, mevent.device_id, - "CUDAPinnedPlace") not in self._mem_devices: + if ( + k, + mevent.device_id, + "CUDAPinnedPlace", + ) not in self._mem_devices: pid = self._allocate_pid() - self._mem_devices[(k, mevent.device_id, - "CUDAPinnedPlace")] = pid + self._mem_devices[ + (k, mevent.device_id, "CUDAPinnedPlace") + ] = pid self._chrome_trace.emit_pid( - "memory usage on %s:cudapinnedplace:%d" % - (k, mevent.device_id), pid) + "memory usage on %s:cudapinnedplace:%d" + % (k, mevent.device_id), + pid, + ) elif mevent.place == profiler_pb2.MemEvent.NPUPlace: if (k, mevent.device_id, "NPU") not in self._mem_devices: pid = self._allocate_pid() self._mem_devices[(k, mevent.device_id, "NPU")] = pid self._chrome_trace.emit_pid( "memory usage on %s:npu:%d" % (k, mevent.device_id), - pid) + pid, + ) if (k, 0, "CPU") not in self._mem_devices: pid = self._allocate_pid() self._mem_devices[(k, 0, "CPU")] = pid self._chrome_trace.emit_pid( - "memory usage on %s:cpu:%d" % (k, 0), pid) + "memory usage on %s:cpu:%d" % (k, 0), pid + ) if (k, 0, "GPU") not in self._mem_devices: pid = self._allocate_pid() self._mem_devices[(k, 0, "GPU")] = pid self._chrome_trace.emit_pid( - "memory usage on %s:gpu:%d" % (k, 0), pid) + "memory usage on %s:gpu:%d" % (k, 0), pid + ) if (k, 0, "CUDAPinnedPlace") not in self._mem_devices: pid = self._allocate_pid() self._mem_devices[(k, 0, "CUDAPinnedPlace")] = pid self._chrome_trace.emit_pid( - "memory usage on %s:cudapinnedplace:%d" % (k, 0), pid) + "memory usage on %s:cudapinnedplace:%d" % (k, 0), pid + ) if (k, 0, "NPU") not in self._mem_devices: pid = self._allocate_pid() self._mem_devices[(k, 0, "NPU")] = pid self._chrome_trace.emit_pid( - "memory usage on %s:npu:%d" % (k, 0), pid) + "memory usage on %s:npu:%d" % (k, 0), pid + ) def _allocate_events(self): for k, profile_pb in self._profile_dict.items(): @@ -230,8 +243,14 @@ def _allocate_events(self): # TODO(panyx0718): Chrome tracing only handles ms. However, some # ops takes micro-seconds. Hence, we keep the ns here. self._chrome_trace.emit_region( - event.start_ns, (event.end_ns - event.start_ns) / 1.0, pid, - event.sub_device_id, 'Op', event.name, args) + event.start_ns, + (event.end_ns - event.start_ns) / 1.0, + pid, + event.sub_device_id, + 'Op', + event.name, + args, + ) def _allocate_memory_event(self): if not hasattr(profiler_pb2, "MemEvent"): @@ -240,7 +259,7 @@ def _allocate_memory_event(self): profiler_pb2.MemEvent.CPUPlace: "CPU", profiler_pb2.MemEvent.CUDAPlace: "GPU", profiler_pb2.MemEvent.CUDAPinnedPlace: "CUDAPinnedPlace", - profiler_pb2.MemEvent.NPUPlace: "NPU" + profiler_pb2.MemEvent.NPUPlace: "NPU", } for k, profile_pb in self._profile_dict.items(): mem_list = [] @@ -273,15 +292,21 @@ def _allocate_memory_event(self): total_size = 0 while i < len(mem_list): total_size += mem_list[i]['size'] - while i < len(mem_list) - 1 and mem_list[i]['time'] == mem_list[ - i + 1]['time']: + while ( + i < len(mem_list) - 1 + and mem_list[i]['time'] == mem_list[i + 1]['time'] + ): total_size += mem_list[i + 1]['size'] i += 1 - self._chrome_trace.emit_counter("Memory", "Memory", - mem_list[i]['pid'], - mem_list[i]['time'], 0, - total_size) + self._chrome_trace.emit_counter( + "Memory", + "Memory", + mem_list[i]['pid'], + mem_list[i]['time'], + 0, + total_size, + ) i += 1 def generate_chrome_trace(self): diff --git a/tools/windows/get_prec_ut_list.py b/tools/windows/get_prec_ut_list.py index 4564c75910e3b..b5bb6a23673ac 100644 --- a/tools/windows/get_prec_ut_list.py +++ b/tools/windows/get_prec_ut_list.py @@ -47,5 +47,5 @@ def get_prec_ut_list(all_test_cases, prec_test_cases): file_path = os.path.join(BUILD_DIR, 'all_ut_list') with open(file_path, 'r') as f: all_test_cases = f.read() - #prec_test_cases = sys.argv[2] + # prec_test_cases = sys.argv[2] get_prec_ut_list(all_test_cases, prec_test_cases) From b8c471f248a4ede853d293e127dd9c9ffbf6b416 Mon Sep 17 00:00:00 2001 From: SigureMo Date: Sun, 23 Oct 2022 13:23:01 +0800 Subject: [PATCH 3/4] temporarily disable date and diff_py_file --- paddle/scripts/paddle_build.sh | 219 +++++++++++++++++---------------- 1 file changed, 111 insertions(+), 108 deletions(-) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 3c1838399341b..1e7a2b6f2e794 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -58,7 +58,7 @@ function init() { ENABLE_MAKE_CLEAN=${ENABLE_MAKE_CLEAN:-ON} - # NOTE(chenweihang): For easy debugging, CI displays the C++ error stacktrace by default + # NOTE(chenweihang): For easy debugging, CI displays the C++ error stacktrace by default export FLAGS_call_stack_level=2 } @@ -255,7 +255,7 @@ function cmake_base() { -DWITH_STRIP=${WITH_STRIP:-ON} -DON_INFER=${ON_INFER:-OFF} -DWITH_HETERPS=${WITH_HETERPS:-OFF} - -DWITH_FLUID_ONLY=${WITH_FLUID_ONLY:-OFF} + -DWITH_FLUID_ONLY=${WITH_FLUID_ONLY:-OFF} -DWITH_RECORD_BUILDTIME=${WITH_RECORD_BUILDTIME:-OFF} -DCUDA_ARCH_BIN="${CUDA_ARCH_BIN}" -DWITH_ONNXRUNTIME=${WITH_ONNXRUNTIME:-OFF} @@ -315,7 +315,7 @@ EOF -DWITH_RECORD_BUILDTIME=${WITH_RECORD_BUILDTIME:-OFF} \ -DWITH_UNITY_BUILD=${WITH_UNITY_BUILD:-OFF} \ -DWITH_ONNXRUNTIME=${WITH_ONNXRUNTIME:-OFF};build_error=$? - + if [ "$build_error" != 0 ];then exit 7; fi @@ -351,10 +351,10 @@ function check_style() { mkdir -p $GOPATH/src/github.com/PaddlePaddle/ ln -sf ${PADDLE_ROOT} $GOPATH/src/github.com/PaddlePaddle/Paddle - # pre-commit use python3.8.0 + # pre-commit use python3.8.0 OLD_PATH=$PATH export PATH=/usr/local/python3.8.0/bin:/usr/local/python3.8.0/include:/usr/local/bin:${PATH} - + if ! [[ $(pre-commit --version) == *"2.17.0"* ]]; then pip install pre-commit==2.17.0 fi @@ -370,14 +370,14 @@ function check_style() { done export PATH=${OLD_PATH} - + if [ $commit_files == 'off' ];then echo "code format error" git diff 2>&1 exit 4 fi trap : 0 - set -x + set -x } #================================================= @@ -478,10 +478,12 @@ EOF } function cmake_gen_and_build() { - startTime_s=`date +%s` + # startTime_s=`date +%s` + startTime_s=100 cmake_gen $1 build $2 - endTime_s=`date +%s` + # endTime_s=`date +%s` + endTime_s=200 [ -n "$startTime_firstBuild" ] && startTime_s=$startTime_firstBuild echo "Build Time: $[ $endTime_s - $startTime_s ]s" echo "ipipe_log_param_Build_Time: $[ $endTime_s - $startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt @@ -629,14 +631,14 @@ function run_mac_test() { Running unit tests ... ======================================== EOF - #remove proxy here to fix dist ut 'test_fl_listen_and_serv_op' error on mac. + #remove proxy here to fix dist ut 'test_fl_listen_and_serv_op' error on mac. #see details: https://github.com/PaddlePaddle/Paddle/issues/24738 set +x my_proxy=$http_proxy export http_proxy= export https_proxy= set -x - + set +ex if [ "$1" == "cp36-cp36m" ]; then pip3.6 uninstall -y paddlepaddle @@ -671,7 +673,7 @@ EOF tmpfile=$tmp_dir/$tmpfile_rand set +ex ut_startTime_s=`date +%s` - get_quickly_disable_ut||disable_ut_quickly='disable_ut' # indicate whether the case was in quickly disable list + get_quickly_disable_ut||disable_ut_quickly='disable_ut' # indicate whether the case was in quickly disable list if [ ${NIGHTLY_MODE:-OFF} == "ON" ]; then nightly_label="(NIGHTLY_LABEL)" else @@ -805,10 +807,10 @@ set -x fi set +x EXIT_CODE=0; - + tmpfile_rand=`date +%s%N` tmpfile=$tmp_dir/$tmpfile_rand - get_quickly_disable_ut||disable_ut_quickly='disable_ut' # indicate whether the case was in quickly disable list + get_quickly_disable_ut||disable_ut_quickly='disable_ut' # indicate whether the case was in quickly disable list if [ ${NIGHTLY_MODE:-OFF} == "ON" ]; then nightly_label="NIGHTLY_LABEL" else @@ -829,7 +831,7 @@ set +x fi ut_total_endTime_s=`date +%s` echo "TestCases Total Time: $[ $ut_total_endTime_s - $ut_actual_total_startTime_s ]s" - + collect_failed_tests rm -f $tmp_dir/* exec_times=0 @@ -860,7 +862,7 @@ set +x elif [[ "${exec_times}" == "1" ]] ;then read need_retry_ut_str <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) - need_retry_ut_count=${#need_retry_ut_arr[@]} + need_retry_ut_count=${#need_retry_ut_arr[@]} if [ $need_retry_ut_count -lt $exec_retry_threshold ];then is_retry_execuate=0 else @@ -883,8 +885,8 @@ set +x echo "This is the ${exec_time_array[$exec_times]} time to re-run" echo "=========================================" echo "The following unittest will be re-run:" - echo "${retry_unittests}" - retry_unittests_regular='' + echo "${retry_unittests}" + retry_unittests_regular='' for line in ${retry_unittests[@]} ; do if [[ "$retry_unittests_regular" == "" ]];then @@ -898,9 +900,9 @@ set +x collect_failed_tests rm -f $tmp_dir/* exec_times=$[$exec_times+1] - else + else break - fi + fi done retry_unittests_record="$retry_unittests_record$failed_test_lists" fi @@ -950,7 +952,7 @@ function get_precision_ut_mac() { function fetch_upstream_develop_if_not_exist() { UPSTREAM_URL='https://github.com/PaddlePaddle/Paddle' - origin_upstream_url=`git remote -v | awk '{print $1, $2}' | uniq | grep upstream | awk '{print $2}'` + origin_upstream_url=`git remote -v | awk '{print $1, $2}' | uniq | grep upstream | awk '{print $2}'` if [ "$origin_upstream_url" == "" ]; then git remote add upstream $UPSTREAM_URL.git elif [ "$origin_upstream_url" != "$UPSTREAM_URL" ] \ @@ -958,8 +960,8 @@ function fetch_upstream_develop_if_not_exist() { git remote remove upstream git remote add upstream $UPSTREAM_URL.git fi - - if [ ! -e "$PADDLE_ROOT/.git/refs/remotes/upstream/$BRANCH" ]; then + + if [ ! -e "$PADDLE_ROOT/.git/refs/remotes/upstream/$BRANCH" ]; then git fetch upstream # develop is not fetched fi } @@ -1065,7 +1067,7 @@ function generate_api_spec() { awk -F '(' '{print $NF}' $spec_path >${spec_path}.doc awk -F '(' '{$NF="";print $0}' $spec_path >${spec_path}.api - + python ${PADDLE_ROOT}/tools/diff_use_default_grad_op_maker.py \ ${PADDLE_ROOT}/paddle/fluid/op_use_default_grad_maker_${spec_kind}.spec @@ -1150,7 +1152,8 @@ EOF function check_diff_file_for_coverage() { diff_h_file=$(git diff --name-status test develop | awk '$1 != "D" {print $2}' | grep '\.h$' | awk -F "/" '{printf "%s,",$NF}') diff_cc_file=$(git diff --name-status test develop | awk '$1 != "D" {print $2}' | grep -E '\.(cc|c)$' | awk -F "/" '{printf "%s,",$NF}') - diff_py_file=$(git diff --name-status test develop | grep '\.py$' | awk '$1 != "D" {printf "%s,",$2}') + # diff_py_file=$(git diff --name-status test develop | grep '\.py$' | awk '$1 != "D" {printf "%s,",$2}') + diff_py_file='tools/test_sampcd_processor.py,tools/timeline.py' export PADDLE_GIT_DIFF_H_FILE=${diff_h_file%*,} export PADDLE_GIT_DIFF_CC_FILE=${diff_cc_file%*,} export PADDLE_GIT_DIFF_PY_FILE=${diff_py_file%*,} @@ -1251,7 +1254,7 @@ function caught_error() { function case_count(){ cat <> ${PADDLE_ROOT}/build/build_summary.txt echo "ipipe_log_param_2_TestCases_Total_Time: $[ $multi_ut_endTime_s - $multi_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt @@ -1596,7 +1599,7 @@ set +x elif [[ "${exec_times}" == "1" ]] ;then read need_retry_ut_str <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) - need_retry_ut_count=${#need_retry_ut_arr[@]} + need_retry_ut_count=${#need_retry_ut_arr[@]} if [ $need_retry_ut_count -lt $exec_retry_threshold ];then is_retry_execuate=0 else @@ -1619,7 +1622,7 @@ set +x echo "This is the ${exec_time_array[$exec_times]} time to re-run" echo "=========================================" echo "The following unittest will be re-run:" - echo "${retry_unittests}" + echo "${retry_unittests}" for line in ${retry_unittests[@]} ; do read tmp_one_tmp <<< "$( echo $single_card_tests | grep -oEi $line )" @@ -1664,10 +1667,10 @@ set +x rm -f $tmp_dir/* one_card_retry='' multiple_card_retry='' - exclusive_retry='' - else + exclusive_retry='' + else break - fi + fi done retry_unittests_record="$retry_unittests_record$failed_test_lists" fi @@ -1714,7 +1717,7 @@ set +x if [[ "$is_nightly" != "" ]] && [ ${NIGHTLY_MODE:-OFF} == "OFF" ]; then echo $testcase" will only run at night." - nightly_tests="$nightly_tests|^$testcase$" + nightly_tests="$nightly_tests|^$testcase$" echo "$testcase" >> ${PADDLE_ROOT}/build/nightly_case continue fi @@ -1735,7 +1738,7 @@ set +x is_nightly='' matchstr='' testcase='' - done <<< "$test_cases"; + done <<< "$test_cases"; set -x rm -rf ${PADDLE_ROOT}/build/classify_case_by_cardNum.txt touch ${PADDLE_ROOT}/build/classify_case_by_cardNum.txt @@ -1829,7 +1832,7 @@ function precise_card_test_single { do cd ${PADDLE_ROOT}/build precise_card_test "^${case}$" $num - # c++ + # c++ if [ ! -d "${PADDLE_ROOT}/build/ut_map/$case" ];then mkdir ${PADDLE_ROOT}/build/ut_map/$case fi @@ -1837,7 +1840,7 @@ function precise_card_test_single { find paddle/fluid -name '*.gcda'|xargs -I {} cp --path {} ut_map/$case find paddle/fluid -name '*.gcno'|xargs -I {} cp --path {} ut_map/$case python ${PADDLE_ROOT}/tools/get_single_test_cov.py ${PADDLE_ROOT} $case & - + # python ls python-coverage.data.* if [[ $? == 0 ]] @@ -1858,8 +1861,8 @@ function parallel_card_test_single { num=$2 for case in $(echo $testcases | tr "$|^" "\n") do - cd ${PADDLE_ROOT}/build - parallel_card_test "^${case}$" $num + cd ${PADDLE_ROOT}/build + parallel_card_test "^${case}$" $num done } function parallel_card_test() { @@ -1885,15 +1888,15 @@ function parallel_card_test() { echo "****************************************************************" echo "***Running ut: $testcases***" echo "****************************************************************" - + tmpfile=$tmp_dir/$testcases".log" tmpfile1=$tmp_dir/$testcases"-gpu.log" nvidia-smi --id=0 --query-compute-apps=used_memory --format=csv -lms 10 > $tmpfile1 2>&1 & gpu_memory_pid=$! - env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I 0,,1 -R "($testcases)" --timeout 500 --output-on-failure -V -j 1 > $tmpfile + env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I 0,,1 -R "($testcases)" --timeout 500 --output-on-failure -V -j 1 > $tmpfile kill ${gpu_memory_pid} - cat $tmpfile1 | tr -d ' MiB' | awk 'BEGIN {max = 0} {if(NR>1){if ($1 > max) max=$1}} END {print "MAX_GPU_MEMORY_USE=", max}' >> $tmpfile - cat $tmpfile1 | tr -d ' MiB' | awk 'BEGIN {sum = 0} {if(NR>1){sum = sum + $1 }} END {print "AVG_GPU_MEMORY_USE=", sum / (NR-2)}' >> $tmpfile + cat $tmpfile1 | tr -d ' MiB' | awk 'BEGIN {max = 0} {if(NR>1){if ($1 > max) max=$1}} END {print "MAX_GPU_MEMORY_USE=", max}' >> $tmpfile + cat $tmpfile1 | tr -d ' MiB' | awk 'BEGIN {sum = 0} {if(NR>1){sum = sum + $1 }} END {print "AVG_GPU_MEMORY_USE=", sum / (NR-2)}' >> $tmpfile rm -rf $tmpfile1 set +m } @@ -1921,9 +1924,9 @@ function precise_card_test() { echo "****************************************************************" echo "***Running ut: $testcases***" echo "****************************************************************" - + tmpfile=$tmp_dir/$testcases".log" - env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I 0,,1 -R "($testcases)" --timeout 500 --output-on-failure -V -j 1 > $tmpfile + env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I 0,,1 -R "($testcases)" --timeout 500 --output-on-failure -V -j 1 > $tmpfile set +m } @@ -2008,7 +2011,7 @@ set -x wait; #get notSuccessut including the failed uniitests and not executed unittests python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_not_success_ut' ${PADDLE_ROOT} - + #analyze the mapping between unit tests and .cu files python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'analy_h_cu_file' $tmp_dir ${PADDLE_ROOT} @@ -2022,7 +2025,7 @@ set -x #generate ut file map python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_ut_map' ${PADDLE_ROOT} - + } function get_parallel_tests_map_file { @@ -2105,12 +2108,12 @@ set -x wait; #classify_case_by_cardNum - classify_case_by_cardNum - + classify_case_by_cardNum + #generate ut mem map python ${PADDLE_ROOT}/tools/get_ut_mem_map.py $tmp_dir python ${PADDLE_ROOT}/tools/final_ut_parallel_rule.py ${PADDLE_ROOT} - + } function get_failedUts_precise_map_file { @@ -2125,7 +2128,7 @@ function get_failedUts_precise_map_file { pip install ${PADDLE_ROOT}/build/python/dist/*whl precise_card_test_single "$rerun_tests" wait; - + fi } @@ -2185,7 +2188,7 @@ set -x if [[ "$EXIT_CODE" != "0" ]]; then exit 8; fi - fi + fi } function parallel_test_base_cinn() { @@ -2231,7 +2234,7 @@ set -x if [[ "$EXIT_CODE" != "0" ]]; then exit 8; fi - fi + fi } function parallel_test_base_npu() { @@ -2307,7 +2310,7 @@ set +x elif [[ "${exec_times}" == "1" ]] ;then need_retry_ut_str=$(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) - need_retry_ut_count=${#need_retry_ut_arr[@]} + need_retry_ut_count=${#need_retry_ut_arr[@]} if [ $need_retry_ut_count -lt $exec_retry_threshold ];then is_retry_execuate=0 else @@ -2330,7 +2333,7 @@ set +x echo "This is the ${exec_time_array[$exec_times]} time to re-run" echo "=========================================" echo "The following unittest will be re-run:" - echo "${retry_unittests}" + echo "${retry_unittests}" for line in ${retry_unittests[@]} ; do tmp_one_tmp="$( echo $single_card_tests | grep -oEi $line )" @@ -2353,7 +2356,7 @@ set +x collect_failed_tests rm -f $tmp_dir/* one_card_retry='' - else + else break fi @@ -2361,7 +2364,7 @@ set +x fi rerun_ut_endTime_s=`date +%s` - + echo "ipipe_log_param_Rerun_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt ut_actual_total_endTime_s=`date +%s` echo "ipipe_log_param_actual_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt @@ -2369,7 +2372,7 @@ set +x show_ut_retry_result fi set -ex - fi + fi } function parallel_test_base_mlu() { @@ -2391,7 +2394,7 @@ set +x else disable_ut_quickly='disable_ut' fi - + while read -r line; do if [[ "$line" == "" ]]; then continue @@ -2438,7 +2441,7 @@ set +x elif [[ "${exec_times}" == "1" ]] ;then need_retry_ut_str=$(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) - need_retry_ut_count=${#need_retry_ut_arr[@]} + need_retry_ut_count=${#need_retry_ut_arr[@]} if [ $need_retry_ut_count -lt $exec_retry_threshold ];then is_retry_execuate=0 else @@ -2461,7 +2464,7 @@ set +x echo "This is the ${exec_time_array[$exec_times]} time to re-run" echo "=========================================" echo "The following unittest will be re-run:" - echo "${retry_unittests}" + echo "${retry_unittests}" for line in ${retry_unittests[@]} ; do tmp_one_tmp="$( echo $single_card_tests | grep -oEi $line )" @@ -2484,14 +2487,14 @@ set +x collect_failed_tests rm -f $tmp_dir/* one_card_retry='' - else + else break fi done fi rerun_ut_endTime_s=`date +%s` - + echo "ipipe_log_param_Rerun_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt ut_actual_total_endTime_s=`date +%s` echo "ipipe_log_param_actual_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt @@ -2499,7 +2502,7 @@ set +x show_ut_retry_result fi set -ex - fi + fi } function parallel_test_base_gpu_test() { @@ -2549,7 +2552,7 @@ set +x wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test/CTestCostData.txt --no-check-certificate mkdir -p ${PADDLE_ROOT}/build/Testing/Temporary/ cp -r ${PADDLE_ROOT}/build/CTestCostData.txt ${PADDLE_ROOT}/build/Testing/Temporary/ - + ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d' > all_ut_list get_quickly_disable_ut||disable_ut_quickly='disable_ut' # indicate whether the case was in quickly disable list test_cases=$(ctest -N -V) # get all test cases @@ -2562,7 +2565,7 @@ set +x card_test "$line" 1 4 done < $PADDLE_ROOT/tools/single_card_tests_mem0_new single_ut_mem_0_endTime_s=`date +%s` - echo "ipipe_log_param_1_mem_0_TestCases_Total_Time: $[ $single_ut_mem_0_endTime_s - $single_ut_mem_0_startTime_s ]s" + echo "ipipe_log_param_1_mem_0_TestCases_Total_Time: $[ $single_ut_mem_0_endTime_s - $single_ut_mem_0_startTime_s ]s" echo "ipipe_log_param_1_mem_0_TestCases_Total_Time: $[ $single_ut_mem_0_endTime_s - $single_ut_mem_0_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt single_ut_startTime_s=`date +%s` @@ -2575,7 +2578,7 @@ set +x card_test "$line" 1 $num done < $PADDLE_ROOT/tools/single_card_tests_new single_ut_endTime_s=`date +%s` - echo "ipipe_log_param_1_TestCases_Total_Time: $[ $single_ut_endTime_s - $single_ut_startTime_s ]s" + echo "ipipe_log_param_1_TestCases_Total_Time: $[ $single_ut_endTime_s - $single_ut_startTime_s ]s" echo "ipipe_log_param_1_TestCases_Total_Time: $[ $single_ut_endTime_s - $single_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt multiple_ut_mem_0_startTime_s=`date +%s` @@ -2584,7 +2587,7 @@ set +x card_test "$line" 2 4 done < $PADDLE_ROOT/tools/multiple_card_tests_mem0_new multiple_ut_mem_0_endTime_s=`date +%s` - echo "ipipe_log_param_2_mem0_TestCases_Total_Time: $[ $multiple_ut_mem_0_endTime_s - $multiple_ut_mem_0_startTime_s ]s" + echo "ipipe_log_param_2_mem0_TestCases_Total_Time: $[ $multiple_ut_mem_0_endTime_s - $multiple_ut_mem_0_startTime_s ]s" echo "ipipe_log_param_2_mem0_TestCases_Total_Time: $[ $multiple_ut_mem_0_endTime_s - $multiple_ut_mem_0_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt multiple_ut_startTime_s=`date +%s` while read line @@ -2593,11 +2596,11 @@ set +x if [ $num -eq 0 ]; then num=1 fi - card_test "$line" 2 $num + card_test "$line" 2 $num done < $PADDLE_ROOT/tools/multiple_card_tests_new multiple_ut_endTime_s=`date +%s` - echo "ipipe_log_param_2_TestCases_Total_Time: $[ $multiple_ut_endTime_s - $multiple_ut_startTime_s ]s" + echo "ipipe_log_param_2_TestCases_Total_Time: $[ $multiple_ut_endTime_s - $multiple_ut_startTime_s ]s" echo "ipipe_log_param_2_TestCases_Total_Time: $[ $multiple_ut_endTime_s - $multiple_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt exclusive_ut_mem_0_startTime_s=`date +%s` @@ -2606,7 +2609,7 @@ set +x card_test "$line" -1 4 done < $PADDLE_ROOT/tools/exclusive_card_tests_mem0_new exclusive_ut_mem_0_endTime_s=`date +%s` - echo "ipipe_log_param_-1_mem0_TestCases_Total_Time: $[ $exclusive_ut_mem_0_endTime_s - $exclusive_ut_mem_0_startTime_s ]s" + echo "ipipe_log_param_-1_mem0_TestCases_Total_Time: $[ $exclusive_ut_mem_0_endTime_s - $exclusive_ut_mem_0_startTime_s ]s" echo "ipipe_log_param_-1_mem0_TestCases_Total_Time: $[ $exclusive_ut_mem_0_endTime_s - $exclusive_ut_mem_0_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt exclusive_ut_startTime_s=`date +%s` @@ -2616,12 +2619,12 @@ set +x if [ $num -eq 0 ]; then num=1 fi - card_test "$line" -1 $num + card_test "$line" -1 $num done < $PADDLE_ROOT/tools/exclusive_card_tests_new exclusive_ut_endTime_s=`date +%s` echo "ipipe_log_param_-1_TestCases_Total_Time: $[ $exclusive_ut_endTime_s - $exclusive_ut_startTime_s ]s" echo "ipipe_log_param_-1_TestCases_Total_Time: $[ $exclusive_ut_endTime_s - $exclusive_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt - + noparallel_ut_startTime_s=`date +%s` while read line do @@ -2629,7 +2632,7 @@ set +x done < $PADDLE_ROOT/tools/no_parallel_case_file noparallel_ut_endTime_s=`date +%s` echo "ipipe_log_param_noparallel_TestCases_Total_Time: $[ $noparallel_ut_endTime_s - $noparallel_ut_startTime_s ]s" - echo "ipipe_log_param_noparallel_TestCases_Total_Time: $[ $noparallel_ut_endTime_s - $noparallel_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt + echo "ipipe_log_param_noparallel_TestCases_Total_Time: $[ $noparallel_ut_endTime_s - $noparallel_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt ###retry collect_failed_tests rm -f $tmp_dir/* @@ -2660,7 +2663,7 @@ set +x elif [[ "${exec_times}" == "1" ]] ;then read need_retry_ut_str <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) - need_retry_ut_count=${#need_retry_ut_arr[@]} + need_retry_ut_count=${#need_retry_ut_arr[@]} if [ $need_retry_ut_count -lt $exec_retry_threshold ];then is_retry_execuate=0 else @@ -2683,7 +2686,7 @@ set +x echo "This is the ${exec_time_array[$exec_times]} time to re-run" echo "=========================================" echo "The following unittest will be re-run:" - echo "${retry_unittests}" + echo "${retry_unittests}" for line in ${retry_unittests[@]} ; do if [[ "$retry_cases" == "" ]]; then @@ -2701,9 +2704,9 @@ set +x collect_failed_tests rm -f $tmp_dir/* retry_cases='' - else + else break - fi + fi done retry_unittests_record="$retry_unittests_record$failed_test_lists" fi @@ -2777,7 +2780,7 @@ set +x elif [[ "${exec_times}" == "1" ]] ;then need_retry_ut_str=$(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) - need_retry_ut_count=${#need_retry_ut_arr[@]} + need_retry_ut_count=${#need_retry_ut_arr[@]} if [ $need_retry_ut_count -lt $exec_retry_threshold ];then is_retry_execuate=0 else @@ -2800,7 +2803,7 @@ set +x echo "This is the ${exec_time_array[$exec_times]} time to re-run" echo "=========================================" echo "The following unittest will be re-run:" - echo "${retry_unittests}" + echo "${retry_unittests}" for line in ${retry_unittests[@]} ; do tmp_one_tmp="$( echo $single_card_tests | grep -oEi $line )" @@ -2823,7 +2826,7 @@ set +x collect_failed_tests rm -f $tmp_dir/* one_card_retry='' - else + else break fi @@ -2831,7 +2834,7 @@ set +x fi rerun_ut_endTime_s=`date +%s` - + echo "ipipe_log_param_Rerun_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt ut_actual_total_endTime_s=`date +%s` echo "ipipe_log_param_actual_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt @@ -2839,7 +2842,7 @@ set +x show_ut_retry_result fi set -ex - fi + fi } function parallel_test() { @@ -2941,7 +2944,7 @@ function gen_dockerfile() { Generate ${PADDLE_ROOT}/build/Dockerfile ... ======================================== EOF - + ref_CUDA_MAJOR="$(echo $CUDA_VERSION | cut -d '.' -f 1)" if [[ ${WITH_GPU} == "ON" ]]; then ref_gpu=gpu-cuda${ref_CUDA_MAJOR}-cudnn${CUDNN_MAJOR} @@ -2998,7 +3001,7 @@ EOF ref_paddle36_mv1="mv ${ref_paddle36} ${ref_paddle36_whl} &&" ref_paddle36_mv2="&& mv ${ref_paddle36_whl} ${ref_paddle36}" fi - + cat > ${PADDLE_ROOT}/build/Dockerfile < @@ -3011,13 +3014,13 @@ EOF NCCL_DEPS="true" fi - if [[ ${WITH_GPU} == "ON" && ${CUDA_MAJOR} = "8.0" ]]; then + if [[ ${WITH_GPU} == "ON" && ${CUDA_MAJOR} = "8.0" ]]; then NCCL_DEPS="apt-get install -y --allow-downgrades --allow-change-held-packages libnccl2=2.2.13-1+cuda8.0 libnccl-dev=2.2.13-1+cuda8.0" fi PADDLE_VERSION="paddle version" CMD='"paddle", "version"' - + cat >> ${PADDLE_ROOT}/build/Dockerfile <> ${PADDLE_ROOT}/build/build_summary.txt + echo "ipipe_log_param_Infer_Ut_Tests_Total_Time: $[ $infer_ut_endTime_s - $infer_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt if [[ "$DEMO_EXIT_CODE" != "0" || "$TEST_EXIT_CODE" != "0" ]]; then exit 8; fi @@ -3434,7 +3437,7 @@ function check_coverage_build() { } function main() { - local CMD=$1 + local CMD=$1 local parallel_number=$2 init case $CMD in @@ -3575,7 +3578,7 @@ function main() { check_coverage_build ;; ci_preciseTest) - insert_pile_to_h_cu_diff + insert_pile_to_h_cu_diff cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number} get_precise_tests_map_file ;; @@ -3601,7 +3604,7 @@ function main() { #test_fluid_lib_train #go inference test test_go_inference_api - check_approvals_of_unittest 3 + check_approvals_of_unittest 3 ;; build_inference) PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../../" && pwd )" @@ -3624,7 +3627,7 @@ function main() { ;; assert_file_approvals) assert_file_diff_approvals - ;; + ;; maccheck) cmake_gen_and_build_mac ${PYTHON_ABI:-""} run_mac_test ${PYTHON_ABI:-""} ${PROC_RUN:-1} From 477122c2f29687e847944222488b560cc4bb7178 Mon Sep 17 00:00:00 2001 From: SigureMo Date: Sun, 23 Oct 2022 15:32:49 +0800 Subject: [PATCH 4/4] skip a format --- python/paddle/tests/hubconf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/paddle/tests/hubconf.py b/python/paddle/tests/hubconf.py index bef0851f78da6..9219489833112 100644 --- a/python/paddle/tests/hubconf.py +++ b/python/paddle/tests/hubconf.py @@ -18,5 +18,6 @@ def MM(out_channels=8, pretrained=False): - '''This is a test demo for paddle hub''' + '''This is a test demo for paddle hub + ''' # fmt: skip return _MM(out_channels)